diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000000..1e8be1006aa9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*.{py,pyi,c,cpp,h,rst,md,yml,yaml,json,test}] +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = space + +[*.{py,pyi,c,h,json,test}] +indent_size = 4 + +[*.{yml,yaml}] +indent_size = 2 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000000..8d89ec6d6043 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,12 @@ +# Adopt black and isort +97c5ee99bc98dc475512e549b252b23a6e7e0997 +# Use builtin generics and PEP 604 for type annotations wherever possible (#13427) +23ee1e7aff357e656e3102435ad0fe3b5074571e +# Use variable annotations (#10723) +f98f78216ba9d6ab68c8e69c19e9f3c7926c5efe +# run pyupgrade (#12711) +fc335cb16315964b923eb1927e3aad1516891c28 +# update black to 23.3.0 (#15059) +4276308be01ea498d946a79554b4a10b1cf13ccb +# Update black to 24.1.1 (#16847) +8107e53158d83d30bb04d290ac10d8d3ccd344f8 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000000..840ba454b8e3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# We vendor typeshed from https://github.com/python/typeshed +mypy/typeshed/** linguist-vendored diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 000000000000..b5cf5bb4dc80 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,53 @@ +--- +name: Bug Report +about: Submit a bug report +labels: "bug" +--- + + + +**Bug Report** + + + +(A clear and concise description of what the bug is.) + +**To Reproduce** + +```python +# Ideally, a small sample program that demonstrates the problem. +# Or even better, a reproducible playground link https://mypy-play.net/ (use the "Gist" button) +``` + +**Expected Behavior** + + + +**Actual Behavior** + + + +**Your Environment** + + + +- Mypy version used: +- Mypy command-line flags: +- Mypy configuration options from `mypy.ini` (and other config files): +- Python version used: + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..a88773308d5e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +contact_links: + - about: "Please check the linked documentation page before filing new issues." + name: "Common issues and solutions" + url: "https://mypy.readthedocs.io/en/stable/common_issues.html" + - about: "Please ask and answer any questions on the python/typing Gitter." + name: "Questions or Chat" + url: "https://gitter.im/python/typing" diff --git a/.github/ISSUE_TEMPLATE/crash.md b/.github/ISSUE_TEMPLATE/crash.md new file mode 100644 index 000000000000..fed16a8d28ac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crash.md @@ -0,0 +1,41 @@ +--- +name: Crash Report +about: Crash (traceback or "INTERNAL ERROR") +labels: "crash" +--- + + + +**Crash Report** + +(Tell us what happened.) + +**Traceback** + +``` +(Insert traceback and other messages from mypy here -- use `--show-traceback`.) +``` + +**To Reproduce** + +(Write what you did to reproduce the crash. Full source code is +appreciated. We also very much appreciate it if you try to narrow the +source down to a small stand-alone example.) + +**Your Environment** + + + +- Mypy version used: +- Mypy command-line flags: +- Mypy configuration options from `mypy.ini` (and other config files): +- Python version used: +- Operating system and version: + + diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 000000000000..c765cc3141f8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,9 @@ +--- +name: Documentation +about: Report a problem with the documentation +labels: "documentation" +--- + +**Documentation** + +(A clear and concise description of the issue.) diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md new file mode 100644 index 000000000000..984e552e51b1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.md @@ -0,0 +1,13 @@ +--- +name: Feature +about: Submit a proposal for a new mypy feature +labels: "feature" +--- + +**Feature** + + + +**Pitch** + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000000..696eb8aee125 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,12 @@ + + +(Explain how this PR changes mypy.) + + diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml new file mode 100644 index 000000000000..8055cfd24180 --- /dev/null +++ b/.github/workflows/build_wheels.yml @@ -0,0 +1,23 @@ +name: Trigger wheel build + +on: + push: + branches: [main, master, 'release*'] + tags: ['*'] + +permissions: + contents: write + +jobs: + build-wheels: + if: github.repository == 'python/mypy' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Trigger script + env: + WHEELS_PUSH_TOKEN: ${{ secrets.WHEELS_PUSH_TOKEN }} + run: ./misc/trigger_wheel_build.sh diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000000..112102954dd3 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,46 @@ +name: Check documentation build + +on: + workflow_dispatch: + push: + branches: [main, master, 'release*'] + tags: ['*'] + pull_request: + paths: + - 'docs/**' + # We now have a docs check that fails if any error codes don't have documentation, + # so it's important to do the docs build on all PRs touching mypy/errorcodes.py + # in case somebody's adding a new error code without any docs + - 'mypy/errorcodes.py' + - 'mypyc/doc/**' + - '**/*.rst' + - '**/*.md' + - CREDITS + - LICENSE + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + docs: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + TOXENV: docs + TOX_SKIP_MISSING_INTERPRETERS: False + VERIFY_MYPY_ERROR_CODES: 1 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install tox + run: pip install tox==4.21.2 + - name: Setup tox environment + run: tox run -e ${{ env.TOXENV }} --notest + - name: Test + run: tox run -e ${{ env.TOXENV }} --skip-pkg-install diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml new file mode 100644 index 000000000000..2b2327798a72 --- /dev/null +++ b/.github/workflows/mypy_primer.yml @@ -0,0 +1,104 @@ +name: Run mypy_primer + +on: + # Only run on PR, since we diff against master + pull_request: + paths-ignore: + - 'docs/**' + - '**/*.rst' + - '**/*.md' + - 'misc/**' + - 'mypyc/**' + - 'mypy/stubtest.py' + - 'mypy/stubgen.py' + - 'mypy/stubgenc.py' + - 'mypy/test/**' + - 'test-data/**' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + mypy_primer: + name: Run mypy_primer + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + matrix: + shard-index: [0, 1, 2, 3, 4] + fail-fast: false + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + path: mypy_to_test + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install -U pip + pip install git+https://github.com/hauntsaninja/mypy_primer.git + - name: Run mypy_primer + shell: bash + run: | + cd mypy_to_test + echo "new commit" + git rev-list --format=%s --max-count=1 $GITHUB_SHA + + MERGE_BASE=$(git merge-base $GITHUB_SHA origin/$GITHUB_BASE_REF) + git checkout -b base_commit $MERGE_BASE + echo "base commit" + git rev-list --format=%s --max-count=1 base_commit + + echo '' + cd .. + # fail action if exit code isn't zero or one + ( + mypy_primer \ + --repo mypy_to_test \ + --new $GITHUB_SHA --old base_commit \ + --num-shards 5 --shard-index ${{ matrix.shard-index }} \ + --debug \ + --additional-flags="--debug-serialize" \ + --output concise \ + | tee diff_${{ matrix.shard-index }}.txt + ) || [ $? -eq 1 ] + - if: ${{ matrix.shard-index == 0 }} + name: Save PR number + run: | + echo ${{ github.event.pull_request.number }} | tee pr_number.txt + - if: ${{ matrix.shard-index == 0 }} + name: Upload mypy_primer diff + PR number + uses: actions/upload-artifact@v4 + with: + name: mypy_primer_diffs-${{ matrix.shard-index }} + path: | + diff_${{ matrix.shard-index }}.txt + pr_number.txt + - name: Upload mypy_primer diff + uses: actions/upload-artifact@v4 + if: ${{ matrix.shard-index != 0 }} + with: + name: mypy_primer_diffs-${{ matrix.shard-index }} + path: diff_${{ matrix.shard-index }}.txt + + join_artifacts: + name: Join artifacts + runs-on: ubuntu-latest + needs: [mypy_primer] + permissions: + contents: read + steps: + - name: Merge artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: mypy_primer_diffs + pattern: mypy_primer_diffs-* + delete-merged: true diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml new file mode 100644 index 000000000000..6e62d8c51713 --- /dev/null +++ b/.github/workflows/mypy_primer_comment.yml @@ -0,0 +1,98 @@ +name: Comment with mypy_primer diff + +on: + workflow_run: + workflows: + - Run mypy_primer + types: + - completed + +permissions: + contents: read + pull-requests: write + +jobs: + comment: + name: Comment PR from mypy_primer + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Download diffs + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + const [matchArtifact] = artifacts.data.artifacts.filter((artifact) => + artifact.name == "mypy_primer_diffs"); + + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: "zip", + }); + fs.writeFileSync("diff.zip", Buffer.from(download.data)); + + - run: unzip diff.zip + - run: | + cat diff_*.txt | tee fulldiff.txt + + - name: Post comment + id: post-comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const MAX_CHARACTERS = 30000 + const MAX_CHARACTERS_PER_PROJECT = MAX_CHARACTERS / 3 + + const fs = require('fs') + let data = fs.readFileSync('fulldiff.txt', { encoding: 'utf8' }) + + function truncateIfNeeded(original, maxLength) { + if (original.length <= maxLength) { + return original + } + let truncated = original.substring(0, maxLength) + // further, remove last line that might be truncated + truncated = truncated.substring(0, truncated.lastIndexOf('\n')) + let lines_truncated = original.split('\n').length - truncated.split('\n').length + return `${truncated}\n\n... (truncated ${lines_truncated} lines) ...` + } + + const projects = data.split('\n\n') + // don't let one project dominate + data = projects.map(project => truncateIfNeeded(project, MAX_CHARACTERS_PER_PROJECT)).join('\n\n') + // posting comment fails if too long, so truncate + data = truncateIfNeeded(data, MAX_CHARACTERS) + + console.log("Diff from mypy_primer:") + console.log(data) + + let body + if (data.trim()) { + body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```' + } else { + body = "According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change doesn't affect type check results on a corpus of open source code. ✅" + } + const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) + await github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) + return prNumber + + - name: Hide old comments + # v0.4.0 + uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + leave_visible: 1 + issue_number: ${{ steps.post-comment.outputs.result }} diff --git a/.github/workflows/sync_typeshed.yml b/.github/workflows/sync_typeshed.yml new file mode 100644 index 000000000000..84d246441f3d --- /dev/null +++ b/.github/workflows/sync_typeshed.yml @@ -0,0 +1,34 @@ +name: Sync typeshed + +on: + workflow_dispatch: + schedule: + - cron: "0 0 1,15 * *" + +permissions: + contents: write + pull-requests: write + +jobs: + sync_typeshed: + name: Sync typeshed + if: github.repository == 'python/mypy' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # TODO: use whatever solution ends up working for + # https://github.com/python/typeshed/issues/8434 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: git config + run: | + git config --global user.name mypybot + git config --global user.email '<>' + - name: Sync typeshed + run: | + python -m pip install requests==2.28.1 + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python misc/sync-typeshed.py --make-pr diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000000..9e6c9cd1d9b3 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,252 @@ +name: Tests + +on: + workflow_dispatch: + push: + branches: [main, master, 'release*'] + tags: ['*'] + pull_request: + paths-ignore: + - 'docs/**' + - 'mypyc/doc/**' + - '**/*.rst' + - '**/*.md' + - .gitignore + - CREDITS + - LICENSE + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + main: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Make sure to run mypyc compiled unit tests for both + # the oldest and newest supported Python versions + - name: Test suite with py38-ubuntu, mypyc-compiled + python: '3.8' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 4" + test_mypyc: true + - name: Test suite with py38-windows-64 + python: '3.8' + arch: x64 + os: windows-latest + toxenv: py38 + tox_extra_args: "-n 4" + - name: Test suite with py39-ubuntu + python: '3.9' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 4" + - name: Test suite with py310-ubuntu + python: '3.10' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 4" + - name: Test suite with py311-ubuntu, mypyc-compiled + python: '3.11' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 4" + test_mypyc: true + - name: Test suite with py312-ubuntu, mypyc-compiled + python: '3.12' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 4" + test_mypyc: true + - name: Test suite with py313-ubuntu, mypyc-compiled + python: '3.13' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 4" + test_mypyc: true + + # - name: Test suite with py314-dev-ubuntu + # python: '3.14-dev' + # arch: x64 + # os: ubuntu-latest + # toxenv: py + # tox_extra_args: "-n 4" + # allow_failure: true + # test_mypyc: true + + - name: mypyc runtime tests with py39-macos + python: '3.9.18' + arch: x64 + # TODO: macos-13 is the last one to support Python 3.9, change it to macos-latest when updating the Python version + os: macos-13 + toxenv: py + tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" + # This is broken. See + # - https://github.com/python/mypy/issues/17819 + # - https://github.com/python/mypy/pull/17822 + # - name: mypyc runtime tests with py38-debug-build-ubuntu + # python: '3.8.17' + # arch: x64 + # os: ubuntu-latest + # toxenv: py + # tox_extra_args: "-n 4 mypyc/test/test_run.py mypyc/test/test_external.py" + # debug_build: true + + - name: Type check our own code (py38-ubuntu) + python: '3.8' + arch: x64 + os: ubuntu-latest + toxenv: type + - name: Type check our own code (py38-windows-64) + python: '3.8' + arch: x64 + os: windows-latest + toxenv: type + + # We also run these checks with pre-commit in CI, + # but it's useful to run them with tox too, + # to ensure the tox env works as expected + - name: Formatting and code style with Black + ruff + python: '3.10' + arch: x64 + os: ubuntu-latest + toxenv: lint + + name: ${{ matrix.name }} + timeout-minutes: 60 + env: + TOX_SKIP_MISSING_INTERPRETERS: False + # Rich (pip) -- Disable color for windows + pytest + FORCE_COLOR: ${{ !(startsWith(matrix.os, 'windows-') && startsWith(matrix.toxenv, 'py')) && 1 || 0 }} + # Tox + PY_COLORS: 1 + # Mypy (see https://github.com/python/mypy/issues/7771) + TERM: xterm-color + MYPY_FORCE_COLOR: 1 + MYPY_FORCE_TERMINAL_WIDTH: 200 + # Pytest + PYTEST_ADDOPTS: --color=yes + + steps: + - uses: actions/checkout@v4 + + - name: Debug build + if: ${{ matrix.debug_build }} + run: | + PYTHONVERSION=${{ matrix.python }} + PYTHONDIR=~/python-debug/python-$PYTHONVERSION + VENV=$PYTHONDIR/env + ./misc/build-debug-python.sh $PYTHONVERSION $PYTHONDIR $VENV + # TODO: does this do anything? env vars aren't passed to the next step right + source $VENV/bin/activate + - name: Latest dev build + if: ${{ endsWith(matrix.python, '-dev') }} + run: | + git clone --depth 1 https://github.com/python/cpython.git /tmp/cpython --branch $( echo ${{ matrix.python }} | sed 's/-dev//' ) + cd /tmp/cpython + echo git rev-parse HEAD; git rev-parse HEAD + git show --no-patch + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential gdb lcov libbz2-dev libffi-dev libgdbm-dev liblzma-dev libncurses5-dev \ + libreadline6-dev libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g-dev + ./configure --prefix=/opt/pythondev + make -j$(nproc) + sudo make install + sudo ln -s /opt/pythondev/bin/python3 /opt/pythondev/bin/python + sudo ln -s /opt/pythondev/bin/pip3 /opt/pythondev/bin/pip + echo "/opt/pythondev/bin" >> $GITHUB_PATH + - uses: actions/setup-python@v5 + if: ${{ !(matrix.debug_build || endsWith(matrix.python, '-dev')) }} + with: + python-version: ${{ matrix.python }} + architecture: ${{ matrix.arch }} + + - name: Install tox + run: | + echo PATH; echo $PATH + echo which python; which python + echo which pip; which pip + echo python version; python -c 'import sys; print(sys.version)' + echo debug build; python -c 'import sysconfig; print(bool(sysconfig.get_config_var("Py_DEBUG")))' + echo os.cpu_count; python -c 'import os; print(os.cpu_count())' + echo os.sched_getaffinity; python -c 'import os; print(len(getattr(os, "sched_getaffinity", lambda *args: [])(0)))' + pip install setuptools==75.1.0 tox==4.21.2 + + - name: Compiled with mypyc + if: ${{ matrix.test_mypyc }} + run: | + pip install -r test-requirements.txt + CC=clang MYPYC_OPT_LEVEL=0 MYPY_USE_MYPYC=1 pip install -e . + + - name: Setup tox environment + run: | + tox run -e ${{ matrix.toxenv }} --notest + - name: Test + run: tox run -e ${{ matrix.toxenv }} --skip-pkg-install -- ${{ matrix.tox_extra_args }} + continue-on-error: ${{ matrix.allow_failure == 'true' }} + + - name: Mark as success (check failures manually) + if: ${{ matrix.allow_failure == 'true' }} + run: exit 0 + + python_32bits: + runs-on: ubuntu-latest + name: Test mypyc suite with 32-bit Python + timeout-minutes: 60 + env: + TOX_SKIP_MISSING_INTERPRETERS: False + # Rich (pip) + FORCE_COLOR: 1 + # Tox + PY_COLORS: 1 + # Mypy (see https://github.com/python/mypy/issues/7771) + TERM: xterm-color + MYPY_FORCE_COLOR: 1 + MYPY_FORCE_TERMINAL_WIDTH: 200 + # Pytest + PYTEST_ADDOPTS: --color=yes + CXX: i686-linux-gnu-g++ + CC: i686-linux-gnu-gcc + steps: + - uses: actions/checkout@v4 + - name: Install 32-bit build dependencies + run: | + sudo dpkg --add-architecture i386 && \ + sudo apt-get update && sudo apt-get install -y \ + zlib1g-dev:i386 \ + libgcc-s1:i386 \ + g++-i686-linux-gnu \ + gcc-i686-linux-gnu \ + libffi-dev:i386 \ + libssl-dev:i386 \ + libbz2-dev:i386 \ + libncurses-dev:i386 \ + libreadline-dev:i386 \ + libsqlite3-dev:i386 \ + liblzma-dev:i386 \ + uuid-dev:i386 + - name: Compile, install, and activate 32-bit Python + uses: gabrielfalcao/pyenv-action@v13 + with: + default: 3.11.1 + command: python -c "import platform; print(f'{platform.architecture()=} {platform.machine()=}');" + - name: Install tox + run: pip install setuptools==75.1.0 tox==4.21.2 + - name: Setup tox environment + run: tox run -e py --notest + - name: Test + run: tox run -e py --skip-pkg-install -- -n 4 mypyc/test/ diff --git a/.github/workflows/test_stubgenc.yml b/.github/workflows/test_stubgenc.yml new file mode 100644 index 000000000000..0652702a0fc0 --- /dev/null +++ b/.github/workflows/test_stubgenc.yml @@ -0,0 +1,39 @@ +name: Test stubgenc on pybind11_fixtures + +on: + workflow_dispatch: + push: + branches: [main, master, 'release*'] + tags: ['*'] + pull_request: + paths: + - 'misc/test-stubgenc.sh' + - 'mypy/stubgenc.py' + - 'mypy/stubdoc.py' + - 'mypy/stubutil.py' + - 'test-data/stubgen/**' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + stubgenc: + # Check stub file generation for a small pybind11 project + # (full text match is required to pass) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + + - uses: actions/checkout@v4 + + - name: Setup 🐍 3.8 + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + - name: Test stubgenc + run: misc/test-stubgenc.sh diff --git a/.gitignore b/.gitignore index 619fe676938d..6c35e3d89342 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,61 @@ build/ __pycache__ -*.pyc +*.py[cod] *~ -@* /build -/env +/env*/ docs/build/ +docs/source/_build +mypyc/doc/_build +*.iml +/out/ +.venv* +venv/ +.mypy_cache/ +.incremental_checker_cache.json +.cache +test-data/packages/.pip_lock +dmypy.json +.dmypy.json + +# Packages +*.egg +*.egg-info +*.eggs + +# IDEs +.idea +.vscode + +# vim temporary files +.*.sw? +*.sw? + +# Operating Systems +.DS_Store + +# Coverage Files +htmlcov +.coverage* + +# pytest cache +.pytest_cache/ + +# virtualenv +.Python +bin/ +lib/ +include/ +.python-version +pyvenv.cfg + +.tox +pip-wheel-metadata + + +test_capi +*.o +*.a +test_capi +/mypyc/lib-rt/build/ +/mypyc/lib-rt/*.so diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000000..d8e66ecb4dfc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +exclude: '^(mypyc/external/)|(mypy/typeshed/)|misc/typeshed_patches' # Exclude all vendored code from lints +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.8.0 + hooks: + - id: black + exclude: '^(test-data/)' + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.9 + hooks: + - id: ruff + args: [--exit-non-zero-on-fix] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.29.4 + hooks: + - id: check-dependabot + - id: check-github-workflows + - repo: https://github.com/rhysd/actionlint + rev: v1.7.3 + hooks: + - id: actionlint + args: [ + -ignore=property "debug_build" is not defined, + -ignore=property "allow_failure" is not defined, + -ignore=SC2(046|086), + ] +ci: + autoupdate_schedule: quarterly diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000000..8ec33ee641ed --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/source/conf.py + +formats: [pdf, htmlzip, epub] + +python: + install: + - requirements: docs/requirements-docs.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 702a78c143bf..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: python -python: - - "3.2" - - "3.3" - - "3.4" - - "pypy3" - -install: python setup.py install - -script: bash travis.sh diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000000..01c3ed16ddbb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,2807 @@ +# Mypy Release Notes + +## Next release + +... + +## Mypy 1.14 + +We’ve just uploaded mypy 1.14 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features and bug fixes. +You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Change to Enum Membership Semantics + +As per the updated [typing specification for enums](https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members), +enum members must be left unannotated. + +```python +class Pet(Enum): + CAT = 1 # Member attribute + DOG = 2 # Member attribute + + # New error: Enum members must be left unannotated + WOLF: int = 3 + + species: str # Considered a non-member attribute +``` + +In particular, the specification change can result in issues in type stubs (`.pyi` files), since +historically it was common to leave the value absent: + +```python +# In a type stub (.pyi file) + +class Pet(Enum): + # Change in semantics: previously considered members, + # now non-member attributes + CAT: int + DOG: int + + # Mypy will now issue a warning if it detects this + # situation in type stubs: + # > Detected enum "Pet" in a type stub with zero + # > members. There is a chance this is due to a recent + # > change in the semantics of enum membership. If so, + # > use `member = value` to mark an enum member, + # > instead of `member: type` + +class Pet(Enum): + # As per the specification, you should now do one of + # the following: + DOG = 1 # Member attribute with value 1 and known type + WOLF = cast(int, ...) # Member attribute with unknown + # value but known type + LION = ... # Member attribute with unknown value and + # # unknown type +``` + +Contributed by Terence Honles (PR [17207](https://github.com/python/mypy/pull/17207)) and +Shantanu Jain (PR [18068](https://github.com/python/mypy/pull/18068)). + +### Support for @deprecated Decorator (PEP 702) + +Mypy can now issue errors or notes when code imports a deprecated feature +explicitly with a `from mod import depr` statement, or uses a deprecated feature +imported otherwise or defined locally. Features are considered deprecated when +decorated with `warnings.deprecated`, as specified in [PEP 702](https://peps.python.org/pep-0702). + +You can enable the error code via `--enable-error-code=deprecated` on the mypy +command line or `enable_error_code = deprecated` in the mypy config file. +Use the command line flag `--report-deprecated-as-note` or config file option +`report_deprecated_as_note=True` to turn all such errors into notes. + +Deprecation errors will be enabled by default in a future mypy version. + +This feature was contributed by Christoph Tyralla. + +List of changes: + + * Add basic support for PEP 702 (`@deprecated`) (Christoph Tyralla, PR [17476](https://github.com/python/mypy/pull/17476)) + * Support descriptors with `@deprecated` (Christoph Tyralla, PR [18090](https://github.com/python/mypy/pull/18090)) + * Make "deprecated" note an error, disabled by default (Valentin Stanciu, PR [18192](https://github.com/python/mypy/pull/18192)) + * Consider all possible type positions with `@deprecated` (Christoph Tyralla, PR [17926](https://github.com/python/mypy/pull/17926)) + * Improve the handling of explicit type annotations in assignment statements with `@deprecated` (Christoph Tyralla, PR [17899](https://github.com/python/mypy/pull/17899)) + +### Optionally Analyzing Untyped Modules + +Mypy normally doesn't analyze imports from third-party modules (installed using pip, for example) +if there are no stubs or a py.typed marker file. To force mypy to analyze these imports, you +can now use the `--follow-untyped-imports` flag or set the `follow_untyped_imports` +config file option to True. This can be set either in the global section of your mypy config +file, or individually on a per-module basis. + +This feature was contributed by Jannick Kremer. + +List of changes: + + * Implement flag to allow type checking of untyped modules (Jannick Kremer, PR [17712](https://github.com/python/mypy/pull/17712)) + * Warn about `--follow-untyped-imports` (Shantanu, PR [18249](https://github.com/python/mypy/pull/18249)) + +### Support New Style Type Variable Defaults (PEP 696) + +Mypy now supports type variable defaults using the new syntax described in PEP 696, which +was introduced in Python 3.13. Example: + +```python +@dataclass +class Box[T = int]: # Set default for "T" + value: T | None = None + +reveal_type(Box()) # type is Box[int], since it's the default +reveal_type(Box(value="Hello World!")) # type is Box[str] +``` + +This feature was contributed by Marc Mueller (PR [17985](https://github.com/python/mypy/pull/17985)). + +### Improved For Loop Index Variable Type Narrowing + +Mypy now preserves the literal type of for loop index variables, to support `TypedDict` +lookups. Example: + +```python +from typing import TypedDict + +class X(TypedDict): + hourly: int + daily: int + +def func(x: X) -> int: + s = 0 + for var in ("hourly", "daily"): + # "Union[Literal['hourly']?, Literal['daily']?]" + reveal_type(var) + + # x[var] no longer triggers a literal-required error + s += x[var] + return s +``` + +This was contributed by Marc Mueller (PR [18014](https://github.com/python/mypy/pull/18014)). + +### Mypyc Improvements + + * Document optimized bytes operations and additional str operations (Jukka Lehtosalo, PR [18242](https://github.com/python/mypy/pull/18242)) + * Add primitives and specialization for `ord()` (Jukka Lehtosalo, PR [18240](https://github.com/python/mypy/pull/18240)) + * Optimize `str.encode` with specializations for common used encodings (Valentin Stanciu, PR [18232](https://github.com/python/mypy/pull/18232)) + * Fix fall back to generic operation for staticmethod and classmethod (Advait Dixit, PR [18228](https://github.com/python/mypy/pull/18228)) + * Support unicode surrogates in string literals (Jukka Lehtosalo, PR [18209](https://github.com/python/mypy/pull/18209)) + * Fix index variable in for loop with `builtins.enumerate` (Advait Dixit, PR [18202](https://github.com/python/mypy/pull/18202)) + * Fix check for enum classes (Advait Dixit, PR [18178](https://github.com/python/mypy/pull/18178)) + * Fix loading type from imported modules (Advait Dixit, PR [18158](https://github.com/python/mypy/pull/18158)) + * Fix initializers of final attributes in class body (Jared Hance, PR [18031](https://github.com/python/mypy/pull/18031)) + * Fix name generation for modules with similar full names (aatle, PR [18001](https://github.com/python/mypy/pull/18001)) + * Fix relative imports in `__init__.py` (Shantanu, PR [17979](https://github.com/python/mypy/pull/17979)) + * Optimize dunder methods (jairov4, PR [17934](https://github.com/python/mypy/pull/17934)) + * Replace deprecated `_PyDict_GetItemStringWithError` (Marc Mueller, PR [17930](https://github.com/python/mypy/pull/17930)) + * Fix wheel build for cp313-win (Marc Mueller, PR [17941](https://github.com/python/mypy/pull/17941)) + * Use public PyGen_GetCode instead of vendored implementation (Marc Mueller, PR [17931](https://github.com/python/mypy/pull/17931)) + * Optimize calls to final classes (jairov4, PR [17886](https://github.com/python/mypy/pull/17886)) + * Support ellipsis (`...`) expressions in class bodies (Newbyte, PR [17923](https://github.com/python/mypy/pull/17923)) + * Sync `pythoncapi_compat.h` (Marc Mueller, PR [17929](https://github.com/python/mypy/pull/17929)) + * Add `runtests.py mypyc-fast` for running fast mypyc tests (Jukka Lehtosalo, PR [17906](https://github.com/python/mypy/pull/17906)) + +### Stubgen Improvements + + * Do not include mypy generated symbols (Ali Hamdan, PR [18137](https://github.com/python/mypy/pull/18137)) + * Fix `FunctionContext.fullname` for nested classes (Chad Dombrova, PR [17963](https://github.com/python/mypy/pull/17963)) + * Add flagfile support (Ruslan Sayfutdinov, PR [18061](https://github.com/python/mypy/pull/18061)) + * Add support for PEP 695 and PEP 696 syntax (Ali Hamdan, PR [18054](https://github.com/python/mypy/pull/18054)) + +### Stubtest Improvements + + * Allow the use of `--show-traceback` and `--pdb` with stubtest (Stephen Morton, PR [18037](https://github.com/python/mypy/pull/18037)) + * Verify `__all__` exists in stub (Sebastian Rittau, PR [18005](https://github.com/python/mypy/pull/18005)) + * Stop telling people to use double underscores (Jelle Zijlstra, PR [17897](https://github.com/python/mypy/pull/17897)) + +### Documentation Updates + + * Update config file documentation (sobolevn, PR [18103](https://github.com/python/mypy/pull/18103)) + * Improve contributor documentation for Windows (ag-tafe, PR [18097](https://github.com/python/mypy/pull/18097)) + * Correct note about `--disallow-any-generics` flag in documentation (Abel Sen, PR [18055](https://github.com/python/mypy/pull/18055)) + * Further caution against `--follow-imports=skip` (Shantanu, PR [18048](https://github.com/python/mypy/pull/18048)) + * Fix the edit page buttton link in documentation (Kanishk Pachauri, PR [17933](https://github.com/python/mypy/pull/17933)) + +### Other Notables Fixes and Improvements + + * Show `Protocol` `__call__` for arguments with incompatible types (MechanicalConstruct, PR [18214](https://github.com/python/mypy/pull/18214)) + * Make join and meet symmetric with `strict_optional` (MechanicalConstruct, PR [18227](https://github.com/python/mypy/pull/18227)) + * Preserve block unreachablility when checking function definitions with constrained TypeVars (Brian Schubert, PR [18217](https://github.com/python/mypy/pull/18217)) + * Do not include non-init fields in the synthesized `__replace__` method for dataclasses (Victorien, PR [18221](https://github.com/python/mypy/pull/18221)) + * Disallow `TypeVar` constraints parameterized by type variables (Brian Schubert, PR [18186](https://github.com/python/mypy/pull/18186)) + * Always complain about invalid varargs and varkwargs (Shantanu, PR [18207](https://github.com/python/mypy/pull/18207)) + * Set default strict_optional state to True (Shantanu, PR [18198](https://github.com/python/mypy/pull/18198)) + * Preserve type variable default None in type alias (Sukhorosov Aleksey, PR [18197](https://github.com/python/mypy/pull/18197)) + * Add checks for invalid usage of continue/break/return in `except*` block (coldwolverine, PR [18132](https://github.com/python/mypy/pull/18132)) + * Do not consider bare TypeVar not overlapping with None for reachability analysis (Stanislav Terliakov, PR [18138](https://github.com/python/mypy/pull/18138)) + * Special case `types.DynamicClassAttribute` as property-like (Stephen Morton, PR [18150](https://github.com/python/mypy/pull/18150)) + * Disallow bare `ParamSpec` in type aliases (Brian Schubert, PR [18174](https://github.com/python/mypy/pull/18174)) + * Move long_description metadata to pyproject.toml (Marc Mueller, PR [18172](https://github.com/python/mypy/pull/18172)) + * Support `==`-based narrowing of Optional (Christoph Tyralla, PR [18163](https://github.com/python/mypy/pull/18163)) + * Allow TypedDict assignment of Required item to NotRequired ReadOnly item (Brian Schubert, PR [18164](https://github.com/python/mypy/pull/18164)) + * Allow nesting of Annotated with TypedDict special forms inside TypedDicts (Brian Schubert, PR [18165](https://github.com/python/mypy/pull/18165)) + * Infer generic type arguments for slice expressions (Brian Schubert, PR [18160](https://github.com/python/mypy/pull/18160)) + * Fix checking of match sequence pattern against bounded type variables (Brian Schubert, PR [18091](https://github.com/python/mypy/pull/18091)) + * Fix incorrect truthyness for Enum types and literals (David Salvisberg, PR [17337](https://github.com/python/mypy/pull/17337)) + * Move static project metadata to pyproject.toml (Marc Mueller, PR [18146](https://github.com/python/mypy/pull/18146)) + * Fallback to stdlib json if integer exceeds 64-bit range (q0w, PR [18148](https://github.com/python/mypy/pull/18148)) + * Fix 'or' pattern structural matching exhaustiveness (yihong, PR [18119](https://github.com/python/mypy/pull/18119)) + * Fix type inference of positional parameter in class pattern involving builtin subtype (Brian Schubert, PR [18141](https://github.com/python/mypy/pull/18141)) + * Fix `[override]` error with no line number when argument node has no line number (Brian Schubert, PR [18122](https://github.com/python/mypy/pull/18122)) + * Fix some dmypy crashes (Ivan Levkivskyi, PR [18098](https://github.com/python/mypy/pull/18098)) + * Fix subtyping between instance type and overloaded (Shantanu, PR [18102](https://github.com/python/mypy/pull/18102)) + * Clean up new_semantic_analyzer config (Shantanu, PR [18071](https://github.com/python/mypy/pull/18071)) + * Issue warning for enum with no members in stub (Shantanu, PR [18068](https://github.com/python/mypy/pull/18068)) + * Fix enum attributes are not members (Terence Honles, PR [17207](https://github.com/python/mypy/pull/17207)) + * Fix crash when checking slice expression with step 0 in tuple index (Brian Schubert, PR [18063](https://github.com/python/mypy/pull/18063)) + * Allow union-with-callable attributes to be overridden by methods (Brian Schubert, PR [18018](https://github.com/python/mypy/pull/18018)) + * Emit `[mutable-override]` for covariant override of attribute with method (Brian Schubert, PR [18058](https://github.com/python/mypy/pull/18058)) + * Support ParamSpec mapping with `functools.partial` (Stanislav Terliakov, PR [17355](https://github.com/python/mypy/pull/17355)) + * Fix approved stub ignore, remove normpath (Shantanu, PR [18045](https://github.com/python/mypy/pull/18045)) + * Make `disallow-any-unimported` flag invertible (Séamus Ó Ceanainn, PR [18030](https://github.com/python/mypy/pull/18030)) + * Filter to possible package paths before trying to resolve a module (falsedrow, PR [18038](https://github.com/python/mypy/pull/18038)) + * Fix overlap check for ParamSpec types (Jukka Lehtosalo, PR [18040](https://github.com/python/mypy/pull/18040)) + * Do not prioritize ParamSpec signatures during overload resolution (Stanislav Terliakov, PR [18033](https://github.com/python/mypy/pull/18033)) + * Fix ternary union for literals (Ivan Levkivskyi, PR [18023](https://github.com/python/mypy/pull/18023)) + * Fix compatibility checks for conditional function definitions using decorators (Brian Schubert, PR [18020](https://github.com/python/mypy/pull/18020)) + * TypeGuard should be bool not Any when matching TypeVar (Evgeniy Slobodkin, PR [17145](https://github.com/python/mypy/pull/17145)) + * Fix convert-cache tool (Shantanu, PR [17974](https://github.com/python/mypy/pull/17974)) + * Fix generator comprehension with mypyc (Shantanu, PR [17969](https://github.com/python/mypy/pull/17969)) + * Fix crash issue when using shadowfile with pretty (Max Chang, PR [17894](https://github.com/python/mypy/pull/17894)) + * Fix multiple nested classes with new generics syntax (Max Chang, PR [17820](https://github.com/python/mypy/pull/17820)) + * Better error for `mypy -p package` without py.typed (Joe Gordon, PR [17908](https://github.com/python/mypy/pull/17908)) + * Emit error for `raise NotImplemented` (Brian Schubert, PR [17890](https://github.com/python/mypy/pull/17890)) + * Add `is_lvalue` attribute to AttributeContext (Brian Schubert, PR [17881](https://github.com/python/mypy/pull/17881)) + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +- aatle +- Abel Sen +- Advait Dixit +- ag-tafe +- Alex Waygood +- Ali Hamdan +- Brian Schubert +- Carlton Gibson +- Chad Dombrova +- Chelsea Durazo +- chiri +- Christoph Tyralla +- coldwolverine +- David Salvisberg +- Ekin Dursun +- Evgeniy Slobodkin +- falsedrow +- Gaurav Giri +- Ihor +- Ivan Levkivskyi +- jairov4 +- Jannick Kremer +- Jared Hance +- Jelle Zijlstra +- jianghuyiyuan +- Joe Gordon +- John Doknjas +- Jukka Lehtosalo +- Kanishk Pachauri +- Marc Mueller +- Max Chang +- MechanicalConstruct +- Newbyte +- q0w +- Ruslan Sayfutdinov +- Sebastian Rittau +- Shantanu +- sobolevn +- Stanislav Terliakov +- Stephen Morton +- Sukhorosov Aleksey +- Séamus Ó Ceanainn +- Terence Honles +- Valentin Stanciu +- vasiliy +- Victorien +- yihong + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + + +## Mypy 1.13 + +We’ve just uploaded mypy 1.13 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +Note that unlike typical releases, Mypy 1.13 does not have any changes to type checking semantics +from 1.12.1. + +### Improved Performance + +Mypy 1.13 contains several performance improvements. Users can expect mypy to be 5-20% faster. +In environments with long search paths (such as environments using many editable installs), mypy +can be significantly faster, e.g. 2.2x faster in the use case targeted by these improvements. + +Mypy 1.13 allows use of the `orjson` library for handling the cache instead of the stdlib `json`, +for improved performance. You can ensure the presence of `orjson` using the `faster-cache` extra: + + python3 -m pip install -U mypy[faster-cache] + +Mypy may depend on `orjson` by default in the future. + +These improvements were contributed by Shantanu. + +List of changes: +* Significantly speed up file handling error paths (Shantanu, PR [17920](https://github.com/python/mypy/pull/17920)) +* Use fast path in modulefinder more often (Shantanu, PR [17950](https://github.com/python/mypy/pull/17950)) +* Let mypyc optimise os.path.join (Shantanu, PR [17949](https://github.com/python/mypy/pull/17949)) +* Make is_sub_path faster (Shantanu, PR [17962](https://github.com/python/mypy/pull/17962)) +* Speed up stubs suggestions (Shantanu, PR [17965](https://github.com/python/mypy/pull/17965)) +* Use sha1 for hashing (Shantanu, PR [17953](https://github.com/python/mypy/pull/17953)) +* Use orjson instead of json, when available (Shantanu, PR [17955](https://github.com/python/mypy/pull/17955)) +* Add faster-cache extra, test in CI (Shantanu, PR [17978](https://github.com/python/mypy/pull/17978)) + +### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Shantanu Jain +- Jukka Lehtosalo + +## Mypy 1.12 + +We’ve just uploaded mypy 1.12 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type +checker for Python. This release includes new features, performance improvements and bug fixes. +You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Support Python 3.12 Syntax for Generics (PEP 695) + +Support for the new type parameter syntax introduced in Python 3.12 is now enabled by default, +documented, and no longer experimental. It was available through a feature flag in +mypy 1.11 as an experimental feature. + +This example demonstrates the new syntax: + +```python +# Generic function +def f[T](x: T) -> T: ... + +reveal_type(f(1)) # Revealed type is 'int' + +# Generic class +class C[T]: + def __init__(self, x: T) -> None: + self.x = x + +c = C('a') +reveal_type(c.x) # Revealed type is 'str' + +# Type alias +type A[T] = C[list[T]] +``` + +For more information, refer to the [documentation](https://mypy.readthedocs.io/en/latest/generics.html). + +These improvements are included: + + * Document Python 3.12 type parameter syntax (Jukka Lehtosalo, PR [17816](https://github.com/python/mypy/pull/17816)) + * Further documentation updates (Jukka Lehtosalo, PR [17826](https://github.com/python/mypy/pull/17826)) + * Allow Self return types with contravariance (Jukka Lehtosalo, PR [17786](https://github.com/python/mypy/pull/17786)) + * Enable new type parameter syntax by default (Jukka Lehtosalo, PR [17798](https://github.com/python/mypy/pull/17798)) + * Generate error if new-style type alias used as base class (Jukka Lehtosalo, PR [17789](https://github.com/python/mypy/pull/17789)) + * Inherit variance if base class has explicit variance (Jukka Lehtosalo, PR [17787](https://github.com/python/mypy/pull/17787)) + * Fix crash on invalid type var reference (Jukka Lehtosalo, PR [17788](https://github.com/python/mypy/pull/17788)) + * Fix covariance of frozen dataclasses (Jukka Lehtosalo, PR [17783](https://github.com/python/mypy/pull/17783)) + * Allow covariance with attribute that has "`_`" name prefix (Jukka Lehtosalo, PR [17782](https://github.com/python/mypy/pull/17782)) + * Support `Annotated[...]` in new-style type aliases (Jukka Lehtosalo, PR [17777](https://github.com/python/mypy/pull/17777)) + * Fix nested generic classes (Jukka Lehtosalo, PR [17776](https://github.com/python/mypy/pull/17776)) + * Add detection and error reporting for the use of incorrect expressions within the scope of a type parameter and a type alias (Kirill Podoprigora, PR [17560](https://github.com/python/mypy/pull/17560)) + +### Basic Support for Python 3.13 + +This release adds partial support for Python 3.13 features and compiled binaries for +Python 3.13. Mypyc now also supports Python 3.13. + +In particular, these features are supported: + * Various new stdlib features and changes (through typeshed stub improvements) + * `typing.ReadOnly` (see below for more) + * `typing.TypeIs` (added in mypy 1.10, [PEP 742](https://peps.python.org/pep-0742/)) + * Type parameter defaults when using the legacy syntax ([PEP 696](https://peps.python.org/pep-0696/)) + +These features are not supported yet: + * `warnings.deprecated` ([PEP 702](https://peps.python.org/pep-0702/)) + * Type parameter defaults when using Python 3.12 type parameter syntax + +### Mypyc Support for Python 3.13 + +Mypyc now supports Python 3.13. This was contributed by Marc Mueller, with additional +fixes by Jukka Lehtosalo. Free threaded Python 3.13 builds are not supported yet. + +List of changes: + + * Add additional includes for Python 3.13 (Marc Mueller, PR [17506](https://github.com/python/mypy/pull/17506)) + * Add another include for Python 3.13 (Marc Mueller, PR [17509](https://github.com/python/mypy/pull/17509)) + * Fix ManagedDict functions for Python 3.13 (Marc Mueller, PR [17507](https://github.com/python/mypy/pull/17507)) + * Update mypyc test output for Python 3.13 (Marc Mueller, PR [17508](https://github.com/python/mypy/pull/17508)) + * Fix `PyUnicode` functions for Python 3.13 (Marc Mueller, PR [17504](https://github.com/python/mypy/pull/17504)) + * Fix `_PyObject_LookupAttrId` for Python 3.13 (Marc Mueller, PR [17505](https://github.com/python/mypy/pull/17505)) + * Fix `_PyList_Extend` for Python 3.13 (Marc Mueller, PR [17503](https://github.com/python/mypy/pull/17503)) + * Fix `gen_is_coroutine` for Python 3.13 (Marc Mueller, PR [17501](https://github.com/python/mypy/pull/17501)) + * Fix `_PyObject_FastCall` for Python 3.13 (Marc Mueller, PR [17502](https://github.com/python/mypy/pull/17502)) + * Avoid uses of `_PyObject_CallMethodOneArg` on 3.13 (Jukka Lehtosalo, PR [17526](https://github.com/python/mypy/pull/17526)) + * Don't rely on `_PyType_CalculateMetaclass` on 3.13 (Jukka Lehtosalo, PR [17525](https://github.com/python/mypy/pull/17525)) + * Don't use `_PyUnicode_FastCopyCharacters` on 3.13 (Jukka Lehtosalo, PR [17524](https://github.com/python/mypy/pull/17524)) + * Don't use `_PyUnicode_EQ` on 3.13, as it's no longer exported (Jukka Lehtosalo, PR [17523](https://github.com/python/mypy/pull/17523)) + +### Inferring Unions for Conditional Expressions + +Mypy now always tries to infer a union type for a conditional expression if left and right +operand types are different. This results in more precise inferred types and lets mypy detect +more issues. Example: + +```python +s = "foo" if cond() else 1 +# Type of "s" is now "str | int" (it used to be "object") +``` + +Notably, if one of the operands has type `Any`, the type of a conditional expression is +now ` | Any`. Previously the inferred type was just `Any`. The new type essentially +indicates that the value can be of type ``, and potentially of some (unknown) type. +Most operations performed on the result must also be valid for ``. +Example where this is relevant: + +```python +from typing import Any + +def func(a: Any, b: bool) -> None: + x = a if b else None + # Type of x is "Any | None" + print(x.y) # Error: None has no attribute "y" +``` + +This feature was contributed by Ivan Levkivskyi (PR [17427](https://github.com/python/mypy/pull/17427)). + +### ReadOnly Support for TypedDict (PEP 705) + +You can now use `typing.ReadOnly` to specity TypedDict items as +read-only ([PEP 705](https://peps.python.org/pep-0705/)): + +```python +from typing import TypedDict + +# Or "from typing ..." on Python 3.13 +from typing_extensions import ReadOnly + +class TD(TypedDict): + a: int + b: ReadOnly[int] + +d: TD = {"a": 1, "b": 2} +d["a"] = 3 # OK +d["b"] = 5 # Error: "b" is ReadOnly +``` + +This feature was contributed by Nikita Sobolev (PR [17644](https://github.com/python/mypy/pull/17644)). + +### Python 3.8 End of Life Approaching + +We are planning to drop support for Python 3.8 in the next mypy feature release or the +one after that. Python 3.8 reaches end of life in October 2024. + +### Planned Changes to Defaults + +We are planning to enable `--local-partial-types` by default in mypy 2.0. This will +often require at least minor code changes. This option is implicitly enabled by mypy +daemon, so this makes the behavior of daemon and non-daemon modes consistent. + +We recommend that mypy users start using local partial types soon (or to explicitly disable +them) to prepare for the change. + +This can also be configured in a mypy configuration file: + +``` +local_partial_types = True +``` + +For more information, refer to the +[documentation](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-local-partial-types). + +### Documentation Updates + +Mypy documentation now uses modern syntax variants and imports in many examples. Some +examples no longer work on Python 3.8, which is the earliest Python version that mypy supports. + +Notably, `Iterable` and other protocols/ABCs are imported from `collections.abc` instead of +`typing`: +```python +from collections.abc import Iterable, Callable +``` + +Examples also avoid the upper-case aliases to built-in types: `list[str]` is used instead +of `List[str]`. The `X | Y` union type syntax introduced in Python 3.10 is also now prevalent. + +List of documentation updates: + + * Document `--output=json` CLI option (Edgar Ramírez Mondragón, PR [17611](https://github.com/python/mypy/pull/17611)) + * Update various references to deprecated type aliases in docs (Jukka Lehtosalo, PR [17829](https://github.com/python/mypy/pull/17829)) + * Make "X | Y" union syntax more prominent in documentation (Jukka Lehtosalo, PR [17835](https://github.com/python/mypy/pull/17835)) + * Discuss upper bounds before self types in documentation (Jukka Lehtosalo, PR [17827](https://github.com/python/mypy/pull/17827)) + * Make changelog visible in mypy documentation (quinn-sasha, PR [17742](https://github.com/python/mypy/pull/17742)) + * List all incomplete features in `--enable-incomplete-feature` docs (sobolevn, PR [17633](https://github.com/python/mypy/pull/17633)) + * Remove the explicit setting of a pygments theme (Pradyun Gedam, PR [17571](https://github.com/python/mypy/pull/17571)) + * Document ReadOnly with TypedDict (Jukka Lehtosalo, PR [17905](https://github.com/python/mypy/pull/17905)) + * Document TypeIs (Chelsea Durazo, PR [17821](https://github.com/python/mypy/pull/17821)) + +### Experimental Inline TypedDict Syntax + +Mypy now supports a non-standard, experimental syntax for defining anonymous TypedDicts. +Example: + +```python +def func(n: str, y: int) -> {"name": str, "year": int}: + return {"name": n, "year": y} +``` + +The feature is disabled by default. Use `--enable-incomplete-feature=InlineTypedDict` to +enable it. *We might remove this feature in a future release.* + +This feature was contributed by Ivan Levkivskyi (PR [17457](https://github.com/python/mypy/pull/17457)). + +### Stubgen Improvements + + * Fix crash on literal class-level keywords (sobolevn, PR [17663](https://github.com/python/mypy/pull/17663)) + * Stubgen add `--version` (sobolevn, PR [17662](https://github.com/python/mypy/pull/17662)) + * Fix `stubgen --no-analysis/--parse-only` docs (sobolevn, PR [17632](https://github.com/python/mypy/pull/17632)) + * Include keyword only args when generating signatures in stubgenc (Eric Mark Martin, PR [17448](https://github.com/python/mypy/pull/17448)) + * Add support for detecting `Literal` types when extracting types from docstrings (Michael Carlstrom, PR [17441](https://github.com/python/mypy/pull/17441)) + * Use `Generator` type var defaults (Sebastian Rittau, PR [17670](https://github.com/python/mypy/pull/17670)) + +### Stubtest Improvements + * Add support for `cached_property` (Ali Hamdan, PR [17626](https://github.com/python/mypy/pull/17626)) + * Add `enable_incomplete_feature` validation to `stubtest` (sobolevn, PR [17635](https://github.com/python/mypy/pull/17635)) + * Fix error code handling in `stubtest` with `--mypy-config-file` (sobolevn, PR [17629](https://github.com/python/mypy/pull/17629)) + +### Other Notables Fixes and Improvements + + * Report error if using unsupported type parameter defaults (Jukka Lehtosalo, PR [17876](https://github.com/python/mypy/pull/17876)) + * Fix re-processing cross-reference in mypy daemon when node kind changes (Ivan Levkivskyi, PR [17883](https://github.com/python/mypy/pull/17883)) + * Don't use equality to narrow when value is IntEnum/StrEnum (Jukka Lehtosalo, PR [17866](https://github.com/python/mypy/pull/17866)) + * Don't consider None vs IntEnum comparison ambiguous (Jukka Lehtosalo, PR [17877](https://github.com/python/mypy/pull/17877)) + * Fix narrowing of IntEnum and StrEnum types (Jukka Lehtosalo, PR [17874](https://github.com/python/mypy/pull/17874)) + * Filter overload items based on self type during type inference (Jukka Lehtosalo, PR [17873](https://github.com/python/mypy/pull/17873)) + * Enable negative narrowing of union TypeVar upper bounds (Brian Schubert, PR [17850](https://github.com/python/mypy/pull/17850)) + * Fix issue with member expression formatting (Brian Schubert, PR [17848](https://github.com/python/mypy/pull/17848)) + * Avoid type size explosion when expanding types (Jukka Lehtosalo, PR [17842](https://github.com/python/mypy/pull/17842)) + * Fix negative narrowing of tuples in match statement (Brian Schubert, PR [17817](https://github.com/python/mypy/pull/17817)) + * Narrow falsey str/bytes/int to literal type (Brian Schubert, PR [17818](https://github.com/python/mypy/pull/17818)) + * Test against latest Python 3.13, make testing 3.14 easy (Shantanu, PR [17812](https://github.com/python/mypy/pull/17812)) + * Reject ParamSpec-typed callables calls with insufficient arguments (Stanislav Terliakov, PR [17323](https://github.com/python/mypy/pull/17323)) + * Fix crash when passing too many type arguments to generic base class accepting single ParamSpec (Brian Schubert, PR [17770](https://github.com/python/mypy/pull/17770)) + * Fix TypeVar upper bounds sometimes not being displayed in pretty callables (Brian Schubert, PR [17802](https://github.com/python/mypy/pull/17802)) + * Added error code for overlapping function signatures (Katrina Connors, PR [17597](https://github.com/python/mypy/pull/17597)) + * Check for `truthy-bool` in `not ...` unary expressions (sobolevn, PR [17773](https://github.com/python/mypy/pull/17773)) + * Add missing lines-covered and lines-valid attributes (Soubhik Kumar Mitra, PR [17738](https://github.com/python/mypy/pull/17738)) + * Fix another crash scenario with recursive tuple types (Ivan Levkivskyi, PR [17708](https://github.com/python/mypy/pull/17708)) + * Resolve TypeVar upper bounds in `functools.partial` (Shantanu, PR [17660](https://github.com/python/mypy/pull/17660)) + * Always reset binder when checking deferred nodes (Ivan Levkivskyi, PR [17643](https://github.com/python/mypy/pull/17643)) + * Fix crash on a callable attribute with single unpack (Ivan Levkivskyi, PR [17641](https://github.com/python/mypy/pull/17641)) + * Fix mismatched signature between checker plugin API and implementation (bzoracler, PR [17343](https://github.com/python/mypy/pull/17343)) + * Indexing a type also produces a GenericAlias (Shantanu, PR [17546](https://github.com/python/mypy/pull/17546)) + * Fix crash on self-type in callable protocol (Ivan Levkivskyi, PR [17499](https://github.com/python/mypy/pull/17499)) + * Fix crash on NamedTuple with method and error in function (Ivan Levkivskyi, PR [17498](https://github.com/python/mypy/pull/17498)) + * Add `__replace__` for dataclasses in 3.13 (Max Muoto, PR [17469](https://github.com/python/mypy/pull/17469)) + * Fix help message for `--no-namespace-packages` (Raphael Krupinski, PR [17472](https://github.com/python/mypy/pull/17472)) + * Fix typechecking for async generators (Danny Yang, PR [17452](https://github.com/python/mypy/pull/17452)) + * Fix strict optional handling in attrs plugin (Ivan Levkivskyi, PR [17451](https://github.com/python/mypy/pull/17451)) + * Allow mixing ParamSpec and TypeVarTuple in Generic (Ivan Levkivskyi, PR [17450](https://github.com/python/mypy/pull/17450)) + * Improvements to `functools.partial` of types (Shantanu, PR [17898](https://github.com/python/mypy/pull/17898)) + * Make ReadOnly TypedDict items covariant (Jukka Lehtosalo, PR [17904](https://github.com/python/mypy/pull/17904)) + * Fix union callees with `functools.partial` (Jukka Lehtosalo, PR [17903](https://github.com/python/mypy/pull/17903)) + * Improve handling of generic functions with `functools.partial` (Ivan Levkivskyi, PR [17925](https://github.com/python/mypy/pull/17925)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=91a58b07cdd807b1d965e04ba85af2adab8bf924+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Mypy 1.12.1 + * Fix crash when showing partially analyzed type in error message (Ivan Levkivskyi, PR [17961](https://github.com/python/mypy/pull/17961)) + * Fix iteration over union (when self type is involved) (Shantanu, PR [17976](https://github.com/python/mypy/pull/17976)) + * Fix type object with type var default in union context (Jukka Lehtosalo, PR [17991](https://github.com/python/mypy/pull/17991)) + * Revert change to `os.path` stubs affecting use of `os.PathLike[Any]` (Shantanu, PR [17995](https://github.com/python/mypy/pull/17995)) + +### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Ali Hamdan +- Anders Kaseorg +- Bénédikt Tran +- Brian Schubert +- bzoracler +- Chelsea Durazo +- Danny Yang +- Edgar Ramírez Mondragón +- Eric Mark Martin +- InSync +- Ivan Levkivskyi +- Jordandev678 +- Katrina Connors +- Kirill Podoprigora +- Marc Mueller +- Max Muoto +- Max Murin +- Michael Carlstrom +- Michael I Chen +- Pradyun Gedam +- quinn-sasha +- Raphael Krupinski +- Sebastian Rittau +- Shantanu +- sobolevn +- Soubhik Kumar Mitra +- Stanislav Terliakov +- wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + + +## Mypy 1.11 + +We’ve just uploaded mypy 1.11 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Support Python 3.12 Syntax for Generics (PEP 695) + +Mypy now supports the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). +This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag, or with `enable_incomplete_feature = NewGenericSyntax` in the mypy configuration file. +We plan to enable this by default in the next mypy feature release. + +This example demonstrates the new syntax: + +```python +# Generic function +def f[T](x: T) -> T: ... + +reveal_type(f(1)) # Revealed type is 'int' + +# Generic class +class C[T]: + def __init__(self, x: T) -> None: + self.x = x + +c = C('a') +reveal_type(c.x) # Revealed type is 'str' + +# Type alias +type A[T] = C[list[T]] +``` + +This feature was contributed by Jukka Lehtosalo. + + +### Support for `functools.partial` + +Mypy now type checks uses of `functools.partial`. Previously mypy would accept arbitrary arguments. + +This example will now produce an error: + +```python +from functools import partial + +def f(a: int, b: str) -> None: ... + +g = partial(f, 1) + +# Argument has incompatible type "int"; expected "str" +g(11) +``` + +This feature was contributed by Shantanu (PR [16939](https://github.com/python/mypy/pull/16939)). + + +### Stricter Checks for Untyped Overrides + +Past mypy versions didn't check if untyped methods were compatible with overridden methods. This would result in false negatives. Now mypy performs these checks when using `--check-untyped-defs`. + +For example, this now generates an error if using `--check-untyped-defs`: + +```python +class Base: + def f(self, x: int = 0) -> None: ... + +class Derived(Base): + # Signature incompatible with "Base" + def f(self): ... +``` + +This feature was contributed by Steven Troxler (PR [17276](https://github.com/python/mypy/pull/17276)). + + +### Type Inference Improvements + +The new polymorphic inference algorithm introduced in mypy 1.5 is now used in more situations. This improves type inference involving generic higher-order functions, in particular. + +This feature was contributed by Ivan Levkivskyi (PR [17348](https://github.com/python/mypy/pull/17348)). + +Mypy now uses unions of tuple item types in certain contexts to enable more precise inferred types. Example: + +```python +for x in (1, 'x'): + # Previously inferred as 'object' + reveal_type(x) # Revealed type is 'int | str' +``` + +This was also contributed by Ivan Levkivskyi (PR [17408](https://github.com/python/mypy/pull/17408)). + + +### Improvements to Detection of Overlapping Overloads + +The details of how mypy checks if two `@overload` signatures are unsafely overlapping were overhauled. This both fixes some false positives, and allows mypy to detect additional unsafe signatures. + +This feature was contributed by Ivan Levkivskyi (PR [17392](https://github.com/python/mypy/pull/17392)). + + +### Better Support for Type Hints in Expressions + +Mypy now allows more expressions that evaluate to valid type annotations in all expression contexts. The inferred types of these expressions are also sometimes more precise. Previously they were often `object`. + +This example uses a union type that includes a callable type as an expression, and it no longer generates an error: + +```python +from typing import Callable + +print(Callable[[], int] | None) # No error +``` + +This feature was contributed by Jukka Lehtosalo (PR [17404](https://github.com/python/mypy/pull/17404)). + + +### Mypyc Improvements + +Mypyc now supports the new syntax for generics introduced in Python 3.12 (see above). Another notable improvement is signficantly faster basic operations on `int` values. + + * Support Python 3.12 syntax for generic functions and classes (Jukka Lehtosalo, PR [17357](https://github.com/python/mypy/pull/17357)) + * Support Python 3.12 type alias syntax (Jukka Lehtosalo, PR [17384](https://github.com/python/mypy/pull/17384)) + * Fix ParamSpec (Shantanu, PR [17309](https://github.com/python/mypy/pull/17309)) + * Inline fast paths of integer unboxing operations (Jukka Lehtosalo, PR [17266](https://github.com/python/mypy/pull/17266)) + * Inline tagged integer arithmetic and bitwise operations (Jukka Lehtosalo, PR [17265](https://github.com/python/mypy/pull/17265)) + * Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) + + +### Changes to Stubtest + * Ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) + * Improve support for Python 3.13 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) + + +### Changes to Stubgen + * Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) + * Fix for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) + * Preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) + + +### Miscellaneous New Features + * Add error format support and JSON output option via `--output json` (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) + * Support `enum.member` in Python 3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) + * Support `enum.nonmember` in Python 3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) + * Support `namedtuple.__replace__` in Python 3.13 (Shantanu, PR [17259](https://github.com/python/mypy/pull/17259)) + * Support `rename=True` in collections.namedtuple (Jelle Zijlstra, PR [17247](https://github.com/python/mypy/pull/17247)) + * Add support for `__spec__` (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) + + +### Changes to Error Reporting + * Mention `--enable-incomplete-feature=NewGenericSyntax` in messages (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) + * Do not report plugin-generated methods with `explicit-override` (sobolevn, PR [17433](https://github.com/python/mypy/pull/17433)) + * Use and display namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) + * Fix false positive for Final local scope variable in Protocol (GiorgosPapoutsakis, PR [17308](https://github.com/python/mypy/pull/17308)) + * Use Never in more messages, use ambiguous in join (Shantanu, PR [17304](https://github.com/python/mypy/pull/17304)) + * Log full path to config file in verbose output (dexterkennedy, PR [17180](https://github.com/python/mypy/pull/17180)) + * Added `[prop-decorator]` code for unsupported property decorators (#14461) (Christopher Barber, PR [16571](https://github.com/python/mypy/pull/16571)) + * Suppress second error message with `:=` and `[truthy-bool]` (Nikita Sobolev, PR [15941](https://github.com/python/mypy/pull/15941)) + * Generate error for assignment of functional Enum to variable of different name (Shantanu, PR [16805](https://github.com/python/mypy/pull/16805)) + * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) + + +### Fixes for Crashes + * Fix daemon crash on invalid type in TypedDict (Ivan Levkivskyi, PR [17495](https://github.com/python/mypy/pull/17495)) + * Fix crash and bugs related to `partial()` (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) + * Fix crash when overriding with unpacked TypedDict (Ivan Levkivskyi, PR [17359](https://github.com/python/mypy/pull/17359)) + * Fix crash on TypedDict unpacking for ParamSpec (Ivan Levkivskyi, PR [17358](https://github.com/python/mypy/pull/17358)) + * Fix crash involving recursive union of tuples (Ivan Levkivskyi, PR [17353](https://github.com/python/mypy/pull/17353)) + * Fix crash on invalid callable property override (Ivan Levkivskyi, PR [17352](https://github.com/python/mypy/pull/17352)) + * Fix crash on unpacking self in NamedTuple (Ivan Levkivskyi, PR [17351](https://github.com/python/mypy/pull/17351)) + * Fix crash on recursive alias with an optional type (Ivan Levkivskyi, PR [17350](https://github.com/python/mypy/pull/17350)) + * Fix crash on type comment inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) + + +### Changes to Documentation + * Use inline config in documentation for optional error codes (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) + * Use lower-case generics in documentation (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) + * Add documentation for show-error-code-links (GiorgosPapoutsakis, PR [17144](https://github.com/python/mypy/pull/17144)) + * Update CONTRIBUTING.md to include commands for Windows (GiorgosPapoutsakis, PR [17142](https://github.com/python/mypy/pull/17142)) + + +### Other Notable Improvements and Fixes + * Fix ParamSpec inference against TypeVarTuple (Ivan Levkivskyi, PR [17431](https://github.com/python/mypy/pull/17431)) + * Fix explicit type for `partial` (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) + * Always allow lambda calls (Ivan Levkivskyi, PR [17430](https://github.com/python/mypy/pull/17430)) + * Fix isinstance checks with PEP 604 unions containing None (Shantanu, PR [17415](https://github.com/python/mypy/pull/17415)) + * Fix self-referential upper bound in new-style type variables (Ivan Levkivskyi, PR [17407](https://github.com/python/mypy/pull/17407)) + * Consider overlap between instances and callables (Ivan Levkivskyi, PR [17389](https://github.com/python/mypy/pull/17389)) + * Allow new-style self-types in classmethods (Ivan Levkivskyi, PR [17381](https://github.com/python/mypy/pull/17381)) + * Fix isinstance with type aliases to PEP 604 unions (Shantanu, PR [17371](https://github.com/python/mypy/pull/17371)) + * Properly handle unpacks in overlap checks (Ivan Levkivskyi, PR [17356](https://github.com/python/mypy/pull/17356)) + * Fix type application for classes with generic constructors (Ivan Levkivskyi, PR [17354](https://github.com/python/mypy/pull/17354)) + * Update `typing_extensions` to >=4.6.0 to fix Python 3.12 error (Ben Brown, PR [17312](https://github.com/python/mypy/pull/17312)) + * Avoid "does not return" error in lambda (Shantanu, PR [17294](https://github.com/python/mypy/pull/17294)) + * Fix bug with descriptors in non-strict-optional mode (Max Murin, PR [17293](https://github.com/python/mypy/pull/17293)) + * Don’t leak unreachability from lambda body to surrounding scope (Anders Kaseorg, PR [17287](https://github.com/python/mypy/pull/17287)) + * Fix issues with non-ASCII characters on Windows (Alexander Leopold Shon, PR [17275](https://github.com/python/mypy/pull/17275)) + * Fix for type narrowing of negative integer literals (gilesgc, PR [17256](https://github.com/python/mypy/pull/17256)) + * Fix confusion between .py and .pyi files in mypy daemon (Valentin Stanciu, PR [17245](https://github.com/python/mypy/pull/17245)) + * Fix type of `tuple[X, Y]` expression (urnest, PR [17235](https://github.com/python/mypy/pull/17235)) + * Don't forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred (Christoph Tyralla, PR [17226](https://github.com/python/mypy/pull/17226)) + * Mark annotated argument as having an explicit, not inferred type (bzoracler, PR [17217](https://github.com/python/mypy/pull/17217)) + * Don't consider Enum private attributes as enum members (Ali Hamdan, PR [17182](https://github.com/python/mypy/pull/17182)) + * Fix Literal strings containing pipe characters (Jelle Zijlstra, PR [17148](https://github.com/python/mypy/pull/17148)) + + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Mypy 1.11.1 + * Fix `RawExpressionType.accept` crash with `--cache-fine-grained` (Anders Kaseorg, PR [17588](https://github.com/python/mypy/pull/17588)) + * Fix PEP 604 isinstance caching (Shantanu, PR [17563](https://github.com/python/mypy/pull/17563)) + * Fix `typing.TypeAliasType` being undefined on python < 3.12 (Nikita Sobolev, PR [17558](https://github.com/python/mypy/pull/17558)) + * Fix `types.GenericAlias` lookup crash (Shantanu, PR [17543](https://github.com/python/mypy/pull/17543)) + +### Mypy 1.11.2 + * Alternative fix for a union-like literal string (Ivan Levkivskyi, PR [17639](https://github.com/python/mypy/pull/17639)) + * Unwrap `TypedDict` item types before storing (Ivan Levkivskyi, PR [17640](https://github.com/python/mypy/pull/17640)) + +### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Alex Waygood +- Alexander Leopold Shon +- Ali Hamdan +- Anders Kaseorg +- Ben Brown +- Bénédikt Tran +- bzoracler +- Christoph Tyralla +- Christopher Barber +- dexterkennedy +- gilesgc +- GiorgosPapoutsakis +- Ivan Levkivskyi +- Jelle Zijlstra +- Jukka Lehtosalo +- Marc Mueller +- Matthieu Devlin +- Michael R. Crusoe +- Nikita Sobolev +- Seo Sanghyeon +- Shantanu +- sobolevn +- Steven Troxler +- Tadeu Manoel +- Tamir Duberstein +- Tushar Sadhwani +- urnest +- Valentin Stanciu + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + + +## Mypy 1.10 + +We’ve just uploaded mypy 1.10 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Support TypeIs (PEP 742) + +Mypy now supports `TypeIs` ([PEP 742](https://peps.python.org/pep-0742/)), which allows +functions to narrow the type of a value, similar to `isinstance()`. Unlike `TypeGuard`, +`TypeIs` can narrow in both the `if` and `else` branches of an if statement: + +```python +from typing_extensions import TypeIs + +def is_str(s: object) -> TypeIs[str]: + return isinstance(s, str) + +def f(o: str | int) -> None: + if is_str(o): + # Type of o is 'str' + ... + else: + # Type of o is 'int' + ... +``` + +`TypeIs` will be added to the `typing` module in Python 3.13, but it +can be used on earlier Python versions by importing it from +`typing_extensions`. + +This feature was contributed by Jelle Zijlstra (PR [16898](https://github.com/python/mypy/pull/16898)). + +### Support TypeVar Defaults (PEP 696) + +[PEP 696](https://peps.python.org/pep-0696/) adds support for type parameter defaults. +Example: + +```python +from typing import Generic +from typing_extensions import TypeVar + +T = TypeVar("T", default=int) + +class C(Generic[T]): + ... + +x: C = ... +y: C[str] = ... +reveal_type(x) # C[int], because of the default +reveal_type(y) # C[str] +``` + +TypeVar defaults will be added to the `typing` module in Python 3.13, but they +can be used with earlier Python releases by importing `TypeVar` from +`typing_extensions`. + +This feature was contributed by Marc Mueller (PR [16878](https://github.com/python/mypy/pull/16878) +and PR [16925](https://github.com/python/mypy/pull/16925)). + +### Support TypeAliasType (PEP 695) +As part of the initial steps towards implementing [PEP 695](https://peps.python.org/pep-0695/), mypy now supports `TypeAliasType`. +`TypeAliasType` provides a backport of the new `type` statement in Python 3.12. + +```python +type ListOrSet[T] = list[T] | set[T] +``` + +is equivalent to: + +```python +T = TypeVar("T") +ListOrSet = TypeAliasType("ListOrSet", list[T] | set[T], type_params=(T,)) +``` + +Example of use in mypy: + +```python +from typing_extensions import TypeAliasType, TypeVar + +NewUnionType = TypeAliasType("NewUnionType", int | str) +x: NewUnionType = 42 +y: NewUnionType = 'a' +z: NewUnionType = object() # error: Incompatible types in assignment (expression has type "object", variable has type "int | str") [assignment] + +T = TypeVar("T") +ListOrSet = TypeAliasType("ListOrSet", list[T] | set[T], type_params=(T,)) +a: ListOrSet[int] = [1, 2] +b: ListOrSet[str] = {'a', 'b'} +c: ListOrSet[str] = 'test' # error: Incompatible types in assignment (expression has type "str", variable has type "list[str] | set[str]") [assignment] +``` + +`TypeAliasType` was added to the `typing` module in Python 3.12, but it can be used with earlier Python releases by importing from `typing_extensions`. + +This feature was contributed by Ali Hamdan (PR [16926](https://github.com/python/mypy/pull/16926), PR [17038](https://github.com/python/mypy/pull/17038) and PR [17053](https://github.com/python/mypy/pull/17053)) + +### Detect Additional Unsafe Uses of super() + +Mypy will reject unsafe uses of `super()` more consistently, when the target has a +trivial (empty) body. Example: + +```python +class Proto(Protocol): + def method(self) -> int: ... + +class Sub(Proto): + def method(self) -> int: + return super().meth() # Error (unsafe) +``` + +This feature was contributed by Shantanu (PR [16756](https://github.com/python/mypy/pull/16756)). + +### Stubgen Improvements +- Preserve empty tuple annotation (Ali Hamdan, PR [16907](https://github.com/python/mypy/pull/16907)) +- Add support for PEP 570 positional-only parameters (Ali Hamdan, PR [16904](https://github.com/python/mypy/pull/16904)) +- Replace obsolete typing aliases with builtin containers (Ali Hamdan, PR [16780](https://github.com/python/mypy/pull/16780)) +- Fix generated dataclass `__init__` signature (Ali Hamdan, PR [16906](https://github.com/python/mypy/pull/16906)) + +### Mypyc Improvements + +- Provide an easier way to define IR-to-IR transforms (Jukka Lehtosalo, PR [16998](https://github.com/python/mypy/pull/16998)) +- Implement lowering pass and add primitives for int (in)equality (Jukka Lehtosalo, PR [17027](https://github.com/python/mypy/pull/17027)) +- Implement lowering for remaining tagged integer comparisons (Jukka Lehtosalo, PR [17040](https://github.com/python/mypy/pull/17040)) +- Optimize away some bool/bit registers (Jukka Lehtosalo, PR [17022](https://github.com/python/mypy/pull/17022)) +- Remangle redefined names produced by async with (Richard Si, PR [16408](https://github.com/python/mypy/pull/16408)) +- Optimize TYPE_CHECKING to False at Runtime (Srinivas Lade, PR [16263](https://github.com/python/mypy/pull/16263)) +- Fix compilation of unreachable comprehensions (Richard Si, PR [15721](https://github.com/python/mypy/pull/15721)) +- Don't crash on non-inlinable final local reads (Richard Si, PR [15719](https://github.com/python/mypy/pull/15719)) + +### Documentation Improvements +- Import `TypedDict` from `typing` instead of `typing_extensions` (Riccardo Di Maio, PR [16958](https://github.com/python/mypy/pull/16958)) +- Add missing `mutable-override` to section title (James Braza, PR [16886](https://github.com/python/mypy/pull/16886)) + +### Error Reporting Improvements + +- Use lower-case generics more consistently in error messages (Jukka Lehtosalo, PR [17035](https://github.com/python/mypy/pull/17035)) + +### Other Notable Changes and Fixes +- Fix incorrect inferred type when accessing descriptor on union type (Matthieu Devlin, PR [16604](https://github.com/python/mypy/pull/16604)) +- Fix crash when expanding invalid `Unpack` in a `Callable` alias (Ali Hamdan, PR [17028](https://github.com/python/mypy/pull/17028)) +- Fix false positive when string formatting with string enum (roberfi, PR [16555](https://github.com/python/mypy/pull/16555)) +- Narrow individual items when matching a tuple to a sequence pattern (Loïc Simon, PR [16905](https://github.com/python/mypy/pull/16905)) +- Fix false positive from type variable within TypeGuard or TypeIs (Evgeniy Slobodkin, PR [17071](https://github.com/python/mypy/pull/17071)) +- Improve `yield from` inference for unions of generators (Shantanu, PR [16717](https://github.com/python/mypy/pull/16717)) +- Fix emulating hash method logic in `attrs` classes (Hashem, PR [17016](https://github.com/python/mypy/pull/17016)) +- Add reverted typeshed commit that uses `ParamSpec` for `functools.wraps` (Tamir Duberstein, PR [16942](https://github.com/python/mypy/pull/16942)) +- Fix type narrowing for `types.EllipsisType` (Shantanu, PR [17003](https://github.com/python/mypy/pull/17003)) +- Fix single item enum match type exhaustion (Oskari Lehto, PR [16966](https://github.com/python/mypy/pull/16966)) +- Improve type inference with empty collections (Marc Mueller, PR [16994](https://github.com/python/mypy/pull/16994)) +- Fix override checking for decorated property (Shantanu, PR [16856](https://github.com/python/mypy/pull/16856)) +- Fix narrowing on match with function subject (Edward Paget, PR [16503](https://github.com/python/mypy/pull/16503)) +- Allow `+N` within `Literal[...]` (Spencer Brown, PR [16910](https://github.com/python/mypy/pull/16910)) +- Experimental: Support TypedDict within `type[...]` (Marc Mueller, PR [16963](https://github.com/python/mypy/pull/16963)) +- Experimtental: Fix issue with TypedDict with optional keys in `type[...]` (Marc Mueller, PR [17068](https://github.com/python/mypy/pull/17068)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Mypy 1.10.1 + +- Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) + +### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Alex Waygood +- Ali Hamdan +- Edward Paget +- Evgeniy Slobodkin +- Hashem +- hesam +- Hugo van Kemenade +- Ihor +- James Braza +- Jelle Zijlstra +- jhance +- Jukka Lehtosalo +- Loïc Simon +- Marc Mueller +- Matthieu Devlin +- Michael R. Crusoe +- Nikita Sobolev +- Oskari Lehto +- Riccardo Di Maio +- Richard Si +- roberfi +- Roman Solomatin +- Sam Xifaras +- Shantanu +- Spencer Brown +- Srinivas Lade +- Tamir Duberstein +- youkaichao + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + + +## Mypy 1.9 + +We’ve just uploaded mypy 1.9 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Breaking Changes + +Because the version of typeshed we use in mypy 1.9 doesn't support 3.7, neither does mypy 1.9. (Jared Hance, PR [16883](https://github.com/python/mypy/pull/16883)) + +We are planning to enable +[local partial types](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-local-partial-types) (enabled via the +`--local-partial-types` flag) later this year by default. This change +was announced years ago, but now it's finally happening. This is a +major backward-incompatible change, so we'll probably include it as +part of the upcoming mypy 2.0 release. This makes daemon and +non-daemon mypy runs have the same behavior by default. + +Local partial types can also be enabled in the mypy config file: +``` +local_partial_types = True +``` + +We are looking at providing a tool to make it easier to migrate +projects to use `--local-partial-types`, but it's not yet clear whether +this is practical. The migration usually involves adding some +explicit type annotations to module-level and class-level variables. + +### Basic Support for Type Parameter Defaults (PEP 696) + +This release contains new experimental support for type parameter +defaults ([PEP 696](https://peps.python.org/pep-0696)). Please try it +out! This feature was contributed by Marc Mueller. + +Since this feature will be officially introduced in the next Python +feature release (3.13), you will need to import `TypeVar`, `ParamSpec` +or `TypeVarTuple` from `typing_extensions` to use defaults for now. + +This example adapted from the PEP defines a default for `BotT`: +```python +from typing import Generic +from typing_extensions import TypeVar + +class Bot: ... + +BotT = TypeVar("BotT", bound=Bot, default=Bot) + +class Context(Generic[BotT]): + bot: BotT + +class MyBot(Bot): ... + +# type is Bot (the default) +reveal_type(Context().bot) +# type is MyBot +reveal_type(Context[MyBot]().bot) +``` + +### Type-checking Improvements + * Fix missing type store for overloads (Marc Mueller, PR [16803](https://github.com/python/mypy/pull/16803)) + * Fix `'WriteToConn' object has no attribute 'flush'` (Charlie Denton, PR [16801](https://github.com/python/mypy/pull/16801)) + * Improve TypeAlias error messages (Marc Mueller, PR [16831](https://github.com/python/mypy/pull/16831)) + * Support narrowing unions that include `type[None]` (Christoph Tyralla, PR [16315](https://github.com/python/mypy/pull/16315)) + * Support TypedDict functional syntax as class base type (anniel-stripe, PR [16703](https://github.com/python/mypy/pull/16703)) + * Accept multiline quoted annotations (Shantanu, PR [16765](https://github.com/python/mypy/pull/16765)) + * Allow unary + in `Literal` (Jelle Zijlstra, PR [16729](https://github.com/python/mypy/pull/16729)) + * Substitute type variables in return type of static methods (Kouroche Bouchiat, PR [16670](https://github.com/python/mypy/pull/16670)) + * Consider TypeVarTuple to be invariant (Marc Mueller, PR [16759](https://github.com/python/mypy/pull/16759)) + * Add `alias` support to `field()` in `attrs` plugin (Nikita Sobolev, PR [16610](https://github.com/python/mypy/pull/16610)) + * Improve attrs hashability detection (Tin Tvrtković, PR [16556](https://github.com/python/mypy/pull/16556)) + +### Performance Improvements + + * Speed up finding function type variables (Jukka Lehtosalo, PR [16562](https://github.com/python/mypy/pull/16562)) + +### Documentation Updates + + * Document supported values for `--enable-incomplete-feature` in "mypy --help" (Froger David, PR [16661](https://github.com/python/mypy/pull/16661)) + * Update new type system discussion links (thomaswhaley, PR [16841](https://github.com/python/mypy/pull/16841)) + * Add missing class instantiation to cheat sheet (Aleksi Tarvainen, PR [16817](https://github.com/python/mypy/pull/16817)) + * Document how evil `--no-strict-optional` is (Shantanu, PR [16731](https://github.com/python/mypy/pull/16731)) + * Improve mypy daemon documentation note about local partial types (Makonnen Makonnen, PR [16782](https://github.com/python/mypy/pull/16782)) + * Fix numbering error (Stefanie Molin, PR [16838](https://github.com/python/mypy/pull/16838)) + * Various documentation improvements (Shantanu, PR [16836](https://github.com/python/mypy/pull/16836)) + +### Stubtest Improvements + * Ignore private function/method parameters when they are missing from the stub (private parameter names start with a single underscore and have a default) (PR [16507](https://github.com/python/mypy/pull/16507)) + * Ignore a new protocol dunder (Alex Waygood, PR [16895](https://github.com/python/mypy/pull/16895)) + * Private parameters can be omitted (Sebastian Rittau, PR [16507](https://github.com/python/mypy/pull/16507)) + * Add support for setting enum members to "..." (Jelle Zijlstra, PR [16807](https://github.com/python/mypy/pull/16807)) + * Adjust symbol table logic (Shantanu, PR [16823](https://github.com/python/mypy/pull/16823)) + * Fix posisitional-only handling in overload resolution (Shantanu, PR [16750](https://github.com/python/mypy/pull/16750)) + +### Stubgen Improvements + * Fix crash on star unpack of TypeVarTuple (Ali Hamdan, PR [16869](https://github.com/python/mypy/pull/16869)) + * Use PEP 604 unions everywhere (Ali Hamdan, PR [16519](https://github.com/python/mypy/pull/16519)) + * Do not ignore property deleter (Ali Hamdan, PR [16781](https://github.com/python/mypy/pull/16781)) + * Support type stub generation for `staticmethod` (WeilerMarcel, PR [14934](https://github.com/python/mypy/pull/14934)) + +### Acknowledgements + +​Thanks to all mypy contributors who contributed to this release: + +- Aleksi Tarvainen +- Alex Waygood +- Ali Hamdan +- anniel-stripe +- Charlie Denton +- Christoph Tyralla +- Dheeraj +- Fabian Keller +- Fabian Lewis +- Froger David +- Ihor +- Jared Hance +- Jelle Zijlstra +- Jukka Lehtosalo +- Kouroche Bouchiat +- Lukas Geiger +- Maarten Huijsmans +- Makonnen Makonnen +- Marc Mueller +- Nikita Sobolev +- Sebastian Rittau +- Shantanu +- Stefanie Molin +- Stephen Morton +- thomaswhaley +- Tin Tvrtković +- WeilerMarcel +- Wesley Collin Wright +- zipperer + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + +## Mypy 1.8 + +We’ve just uploaded mypy 1.8 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Type-checking Improvements + * Do not intersect types in isinstance checks if at least one is final (Christoph Tyralla, PR [16330](https://github.com/python/mypy/pull/16330)) + * Detect that `@final` class without `__bool__` cannot have falsey instances (Ilya Priven, PR [16566](https://github.com/python/mypy/pull/16566)) + * Do not allow `TypedDict` classes with extra keywords (Nikita Sobolev, PR [16438](https://github.com/python/mypy/pull/16438)) + * Do not allow class-level keywords for `NamedTuple` (Nikita Sobolev, PR [16526](https://github.com/python/mypy/pull/16526)) + * Make imprecise constraints handling more robust (Ivan Levkivskyi, PR [16502](https://github.com/python/mypy/pull/16502)) + * Fix strict-optional in extending generic TypedDict (Ivan Levkivskyi, PR [16398](https://github.com/python/mypy/pull/16398)) + * Allow type ignores of PEP 695 constructs (Shantanu, PR [16608](https://github.com/python/mypy/pull/16608)) + * Enable `type_check_only` support for `TypedDict` and `NamedTuple` (Nikita Sobolev, PR [16469](https://github.com/python/mypy/pull/16469)) + +### Performance Improvements + * Add fast path to analyzing special form assignments (Jukka Lehtosalo, PR [16561](https://github.com/python/mypy/pull/16561)) + +### Improvements to Error Reporting + * Don't show documentation links for plugin error codes (Ivan Levkivskyi, PR [16383](https://github.com/python/mypy/pull/16383)) + * Improve error messages for `super` checks and add more tests (Nikita Sobolev, PR [16393](https://github.com/python/mypy/pull/16393)) + * Add error code for mutable covariant override (Ivan Levkivskyi, PR [16399](https://github.com/python/mypy/pull/16399)) + +### Stubgen Improvements + * Preserve simple defaults in function signatures (Ali Hamdan, PR [15355](https://github.com/python/mypy/pull/15355)) + * Include `__all__` in output (Jelle Zijlstra, PR [16356](https://github.com/python/mypy/pull/16356)) + * Fix stubgen regressions with pybind11 and mypy 1.7 (Chad Dombrova, PR [16504](https://github.com/python/mypy/pull/16504)) + +### Stubtest Improvements + * Improve handling of unrepresentable defaults (Jelle Zijlstra, PR [16433](https://github.com/python/mypy/pull/16433)) + * Print more helpful errors if a function is missing from stub (Alex Waygood, PR [16517](https://github.com/python/mypy/pull/16517)) + * Support `@type_check_only` decorator (Nikita Sobolev, PR [16422](https://github.com/python/mypy/pull/16422)) + * Warn about missing `__del__` (Shantanu, PR [16456](https://github.com/python/mypy/pull/16456)) + * Fix crashes with some uses of `final` and `deprecated` (Shantanu, PR [16457](https://github.com/python/mypy/pull/16457)) + +### Fixes to Crashes + * Fix crash with type alias to `Callable[[Unpack[Tuple[Any, ...]]], Any]` (Alex Waygood, PR [16541](https://github.com/python/mypy/pull/16541)) + * Fix crash on TypeGuard in `__call__` (Ivan Levkivskyi, PR [16516](https://github.com/python/mypy/pull/16516)) + * Fix crash on invalid enum in method (Ivan Levkivskyi, PR [16511](https://github.com/python/mypy/pull/16511)) + * Fix crash on unimported Any in TypedDict (Ivan Levkivskyi, PR [16510](https://github.com/python/mypy/pull/16510)) + +### Documentation Updates + * Update soft-error-limit default value to -1 (Sveinung Gundersen, PR [16542](https://github.com/python/mypy/pull/16542)) + * Support Sphinx 7.x (Michael R. Crusoe, PR [16460](https://github.com/python/mypy/pull/16460)) + +### Other Notable Changes and Fixes + * Allow mypy to output a junit file with per-file results (Matthew Wright, PR [16388](https://github.com/python/mypy/pull/16388)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=4a854366e03dee700109f8e758a08b2457ea2f51+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +​Thanks to all mypy contributors who contributed to this release: + +- Alex Waygood +- Ali Hamdan +- Chad Dombrova +- Christoph Tyralla +- Ilya Priven +- Ivan Levkivskyi +- Jelle Zijlstra +- Jukka Lehtosalo +- Marcel Telka +- Matthew Wright +- Michael R. Crusoe +- Nikita Sobolev +- Ole Peder Brandtzæg +- robjhornby +- Shantanu +- Sveinung Gundersen +- Valentin Stanciu + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + +Posted by Wesley Collin Wright + +## Mypy 1.7 + +We’ve just uploaded mypy 1.7 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Using TypedDict for `**kwargs` Typing + +Mypy now has support for using `Unpack[...]` with a TypedDict type to annotate `**kwargs` arguments enabled by default. Example: + +```python +# Or 'from typing_extensions import ...' +from typing import TypedDict, Unpack + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]) -> None: + ... + +foo(name="x", age=1) # Ok +foo(name=1) # Error +``` + +The definition of `foo` above is equivalent to the one below, with keyword-only arguments `name` and `age`: + +```python +def foo(*, name: str, age: int) -> None: + ... +``` + +Refer to [PEP 692](https://peps.python.org/pep-0692/) for more information. Note that unlike in the current version of the PEP, mypy always treats signatures with `Unpack[SomeTypedDict]` as equivalent to their expanded forms with explicit keyword arguments, and there aren't special type checking rules for TypedDict arguments. + +This was contributed by Ivan Levkivskyi back in 2022 (PR [13471](https://github.com/python/mypy/pull/13471)). + +### TypeVarTuple Support Enabled (Experimental) + +Mypy now has support for variadic generics (TypeVarTuple) enabled by default, as an experimental feature. Refer to [PEP 646](https://peps.python.org/pep-0646/) for the details. + +TypeVarTuple was implemented by Jared Hance and Ivan Levkivskyi over several mypy releases, with help from Jukka Lehtosalo. + +Changes included in this release: + + * Fix handling of tuple type context with unpacks (Ivan Levkivskyi, PR [16444](https://github.com/python/mypy/pull/16444)) + * Handle TypeVarTuples when checking overload constraints (robjhornby, PR [16428](https://github.com/python/mypy/pull/16428)) + * Enable Unpack/TypeVarTuple support (Ivan Levkivskyi, PR [16354](https://github.com/python/mypy/pull/16354)) + * Fix crash on unpack call special-casing (Ivan Levkivskyi, PR [16381](https://github.com/python/mypy/pull/16381)) + * Some final touches for variadic types support (Ivan Levkivskyi, PR [16334](https://github.com/python/mypy/pull/16334)) + * Support PEP-646 and PEP-692 in the same callable (Ivan Levkivskyi, PR [16294](https://github.com/python/mypy/pull/16294)) + * Support new `*` syntax for variadic types (Ivan Levkivskyi, PR [16242](https://github.com/python/mypy/pull/16242)) + * Correctly handle variadic instances with empty arguments (Ivan Levkivskyi, PR [16238](https://github.com/python/mypy/pull/16238)) + * Correctly handle runtime type applications of variadic types (Ivan Levkivskyi, PR [16240](https://github.com/python/mypy/pull/16240)) + * Support variadic tuple packing/unpacking (Ivan Levkivskyi, PR [16205](https://github.com/python/mypy/pull/16205)) + * Better support for variadic calls and indexing (Ivan Levkivskyi, PR [16131](https://github.com/python/mypy/pull/16131)) + * Subtyping and inference of user-defined variadic types (Ivan Levkivskyi, PR [16076](https://github.com/python/mypy/pull/16076)) + * Complete type analysis of variadic types (Ivan Levkivskyi, PR [15991](https://github.com/python/mypy/pull/15991)) + +### New Way of Installing Mypyc Dependencies + +If you want to install package dependencies needed by mypyc (not just mypy), you should now install `mypy[mypyc]` instead of just `mypy`: + +``` +python3 -m pip install -U 'mypy[mypyc]' +``` + +Mypy has many more users than mypyc, so always installing mypyc dependencies would often bring unnecessary dependencies. + +This change was contributed by Shantanu (PR [16229](https://github.com/python/mypy/pull/16229)). + +### New Rules for Re-exports + +Mypy no longer considers an import such as `import a.b as b` as an explicit re-export. The old behavior was arguably inconsistent and surprising. This may impact some stub packages, such as older versions of `types-six`. You can change the import to `from a import b as b`, if treating the import as a re-export was intentional. + +This change was contributed by Anders Kaseorg (PR [14086](https://github.com/python/mypy/pull/14086)). + +### Improved Type Inference + +The new type inference algorithm that was recently introduced to mypy (but was not enabled by default) is now enabled by default. It improves type inference of calls to generic callables where an argument is also a generic callable, in particular. You can use `--old-type-inference` to disable the new behavior. + +The new algorithm can (rarely) produce different error messages, different error codes, or errors reported on different lines. This is more likely in cases where generic types were used incorrectly. + +The new type inference algorithm was contributed by Ivan Levkivskyi. PR [16345](https://github.com/python/mypy/pull/16345) enabled it by default. + +### Narrowing Tuple Types Using len() + +Mypy now can narrow tuple types using `len()` checks. Example: + +```python +def f(t: tuple[int, int] | tuple[int, int, int]) -> None: + if len(t) == 2: + a, b = t # Ok + ... +``` + +This feature was contributed by Ivan Levkivskyi (PR [16237](https://github.com/python/mypy/pull/16237)). + +### More Precise Tuple Lengths (Experimental) + +Mypy supports experimental, more precise checking of tuple type lengths through `--enable-incomplete-feature=PreciseTupleTypes`. Refer to the [documentation](https://mypy.readthedocs.io/en/latest/command_line.html#enabling-incomplete-experimental-features) for more information. + +More generally, we are planning to use `--enable-incomplete-feature` to introduce experimental features that would benefit from community feedback. + +This feature was contributed by Ivan Levkivskyi (PR [16237](https://github.com/python/mypy/pull/16237)). + +### Mypy Changelog + +We now maintain a [changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) in the mypy Git repository. It mirrors the contents of [mypy release blog posts](https://mypy-lang.blogspot.com/). We will continue to also publish release blog posts. In the future, release blog posts will be created based on the changelog near a release date. + +This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull/16280)). + +### Mypy Daemon Improvements + + * Fix daemon crash caused by deleted submodule (Jukka Lehtosalo, PR [16370](https://github.com/python/mypy/pull/16370)) + * Fix file reloading in dmypy with --export-types (Ivan Levkivskyi, PR [16359](https://github.com/python/mypy/pull/16359)) + * Fix dmypy inspect on Windows (Ivan Levkivskyi, PR [16355](https://github.com/python/mypy/pull/16355)) + * Fix dmypy inspect for namespace packages (Ivan Levkivskyi, PR [16357](https://github.com/python/mypy/pull/16357)) + * Fix return type change to optional in generic function (Jukka Lehtosalo, PR [16342](https://github.com/python/mypy/pull/16342)) + * Fix daemon false positives related to module-level `__getattr__` (Jukka Lehtosalo, PR [16292](https://github.com/python/mypy/pull/16292)) + * Fix daemon crash related to ABCs (Jukka Lehtosalo, PR [16275](https://github.com/python/mypy/pull/16275)) + * Stream dmypy output instead of dumping everything at the end (Valentin Stanciu, PR [16252](https://github.com/python/mypy/pull/16252)) + * Make sure all dmypy errors are shown (Valentin Stanciu, PR [16250](https://github.com/python/mypy/pull/16250)) + +### Mypyc Improvements + + * Generate error on duplicate function definitions (Jukka Lehtosalo, PR [16309](https://github.com/python/mypy/pull/16309)) + * Don't crash on unreachable statements (Jukka Lehtosalo, PR [16311](https://github.com/python/mypy/pull/16311)) + * Avoid cyclic reference in nested functions (Jukka Lehtosalo, PR [16268](https://github.com/python/mypy/pull/16268)) + * Fix direct `__dict__` access on inner functions in new Python (Shantanu, PR [16084](https://github.com/python/mypy/pull/16084)) + * Make tuple packing and unpacking more efficient (Jukka Lehtosalo, PR [16022](https://github.com/python/mypy/pull/16022)) + +### Improvements to Error Reporting + + * Update starred expression error message to match CPython (Cibin Mathew, PR [16304](https://github.com/python/mypy/pull/16304)) + * Fix error code of "Maybe you forgot to use await" note (Jelle Zijlstra, PR [16203](https://github.com/python/mypy/pull/16203)) + * Use error code `[unsafe-overload]` for unsafe overloads, instead of `[misc]` (Randolf Scholz, PR [16061](https://github.com/python/mypy/pull/16061)) + * Reword the error message related to void functions (Albert Tugushev, PR [15876](https://github.com/python/mypy/pull/15876)) + * Represent bottom type as Never in messages (Shantanu, PR [15996](https://github.com/python/mypy/pull/15996)) + * Add hint for AsyncIterator incompatible return type (Ilya Priven, PR [15883](https://github.com/python/mypy/pull/15883)) + * Don't suggest stubs packages where the runtime package now ships with types (Alex Waygood, PR [16226](https://github.com/python/mypy/pull/16226)) + +### Performance Improvements + + * Speed up type argument checking (Jukka Lehtosalo, PR [16353](https://github.com/python/mypy/pull/16353)) + * Add fast path for checking self types (Jukka Lehtosalo, PR [16352](https://github.com/python/mypy/pull/16352)) + * Cache information about whether file is typeshed file (Jukka Lehtosalo, PR [16351](https://github.com/python/mypy/pull/16351)) + * Skip expensive `repr()` in logging call when not needed (Jukka Lehtosalo, PR [16350](https://github.com/python/mypy/pull/16350)) + +### Attrs and Dataclass Improvements + + * `dataclass.replace`: Allow transformed classes (Ilya Priven, PR [15915](https://github.com/python/mypy/pull/15915)) + * `dataclass.replace`: Fall through to typeshed signature (Ilya Priven, PR [15962](https://github.com/python/mypy/pull/15962)) + * Document `dataclass_transform` behavior (Ilya Priven, PR [16017](https://github.com/python/mypy/pull/16017)) + * `attrs`: Remove fields type check (Ilya Priven, PR [15983](https://github.com/python/mypy/pull/15983)) + * `attrs`, `dataclasses`: Don't enforce slots when base class doesn't (Ilya Priven, PR [15976](https://github.com/python/mypy/pull/15976)) + * Fix crash on dataclass field / property collision (Nikita Sobolev, PR [16147](https://github.com/python/mypy/pull/16147)) + +### Stubgen Improvements + + * Write stubs with utf-8 encoding (Jørgen Lind, PR [16329](https://github.com/python/mypy/pull/16329)) + * Fix missing property setter in semantic analysis mode (Ali Hamdan, PR [16303](https://github.com/python/mypy/pull/16303)) + * Unify C extension and pure python stub generators with object oriented design (Chad Dombrova, PR [15770](https://github.com/python/mypy/pull/15770)) + * Multiple fixes to the generated imports (Ali Hamdan, PR [15624](https://github.com/python/mypy/pull/15624)) + * Generate valid dataclass stubs (Ali Hamdan, PR [15625](https://github.com/python/mypy/pull/15625)) + +### Fixes to Crashes + + * Fix incremental mode crash on TypedDict in method (Ivan Levkivskyi, PR [16364](https://github.com/python/mypy/pull/16364)) + * Fix crash on star unpack in TypedDict (Ivan Levkivskyi, PR [16116](https://github.com/python/mypy/pull/16116)) + * Fix crash on malformed TypedDict in incremental mode (Ivan Levkivskyi, PR [16115](https://github.com/python/mypy/pull/16115)) + * Fix crash with report generation on namespace packages (Shantanu, PR [16019](https://github.com/python/mypy/pull/16019)) + * Fix crash when parsing error code config with typo (Shantanu, PR [16005](https://github.com/python/mypy/pull/16005)) + * Fix `__post_init__()` internal error (Ilya Priven, PR [16080](https://github.com/python/mypy/pull/16080)) + +### Documentation Updates + + * Make it easier to copy commands from README (Hamir Mahal, PR [16133](https://github.com/python/mypy/pull/16133)) + * Document and rename `[overload-overlap]` error code (Shantanu, PR [16074](https://github.com/python/mypy/pull/16074)) + * Document `--force-uppercase-builtins` and `--force-union-syntax` (Nikita Sobolev, PR [16049](https://github.com/python/mypy/pull/16049)) + * Document `force_union_syntax` and `force_uppercase_builtins` (Nikita Sobolev, PR [16048](https://github.com/python/mypy/pull/16048)) + * Document we're not tracking relationships between symbols (Ilya Priven, PR [16018](https://github.com/python/mypy/pull/16018)) + +### Other Notable Changes and Fixes + + * Propagate narrowed types to lambda expressions (Ivan Levkivskyi, PR [16407](https://github.com/python/mypy/pull/16407)) + * Avoid importing from `setuptools._distutils` (Shantanu, PR [16348](https://github.com/python/mypy/pull/16348)) + * Delete recursive aliases flags (Ivan Levkivskyi, PR [16346](https://github.com/python/mypy/pull/16346)) + * Properly use proper subtyping for callables (Ivan Levkivskyi, PR [16343](https://github.com/python/mypy/pull/16343)) + * Use upper bound as inference fallback more consistently (Ivan Levkivskyi, PR [16344](https://github.com/python/mypy/pull/16344)) + * Add `[unimported-reveal]` error code (Nikita Sobolev, PR [16271](https://github.com/python/mypy/pull/16271)) + * Add `|=` and `|` operators support for `TypedDict` (Nikita Sobolev, PR [16249](https://github.com/python/mypy/pull/16249)) + * Clarify variance convention for Parameters (Ivan Levkivskyi, PR [16302](https://github.com/python/mypy/pull/16302)) + * Correctly recognize `typing_extensions.NewType` (Ganden Schaffner, PR [16298](https://github.com/python/mypy/pull/16298)) + * Fix partially defined in the case of missing type maps (Shantanu, PR [15995](https://github.com/python/mypy/pull/15995)) + * Use SPDX license identifier (Nikita Sobolev, PR [16230](https://github.com/python/mypy/pull/16230)) + * Make `__qualname__` and `__module__` available in class bodies (Anthony Sottile, PR [16215](https://github.com/python/mypy/pull/16215)) + * stubtest: Hint when args in stub need to be keyword-only (Alex Waygood, PR [16210](https://github.com/python/mypy/pull/16210)) + * Tuple slice should not propagate fallback (Thomas Grainger, PR [16154](https://github.com/python/mypy/pull/16154)) + * Fix cases of type object handling for overloads (Shantanu, PR [16168](https://github.com/python/mypy/pull/16168)) + * Fix walrus interaction with empty collections (Ivan Levkivskyi, PR [16197](https://github.com/python/mypy/pull/16197)) + * Use type variable bound when it appears as actual during inference (Ivan Levkivskyi, PR [16178](https://github.com/python/mypy/pull/16178)) + * Use upper bounds as fallback solutions for inference (Ivan Levkivskyi, PR [16184](https://github.com/python/mypy/pull/16184)) + * Special-case type inference of empty collections (Ivan Levkivskyi, PR [16122](https://github.com/python/mypy/pull/16122)) + * Allow TypedDict unpacking in Callable types (Ivan Levkivskyi, PR [16083](https://github.com/python/mypy/pull/16083)) + * Fix inference for overloaded `__call__` with generic self (Shantanu, PR [16053](https://github.com/python/mypy/pull/16053)) + * Call dynamic class hook on generic classes (Petter Friberg, PR [16052](https://github.com/python/mypy/pull/16052)) + * Preserve implicitly exported types via attribute access (Shantanu, PR [16129](https://github.com/python/mypy/pull/16129)) + * Fix a stubtest bug (Alex Waygood) + * Fix `tuple[Any, ...]` subtyping (Shantanu, PR [16108](https://github.com/python/mypy/pull/16108)) + * Lenient handling of trivial Callable suffixes (Ivan Levkivskyi, PR [15913](https://github.com/python/mypy/pull/15913)) + * Add `add_overloaded_method_to_class` helper for plugins (Nikita Sobolev, PR [16038](https://github.com/python/mypy/pull/16038)) + * Bundle `misc/proper_plugin.py` as a part of `mypy` (Nikita Sobolev, PR [16036](https://github.com/python/mypy/pull/16036)) + * Fix `case Any()` in match statement (DS/Charlie, PR [14479](https://github.com/python/mypy/pull/14479)) + * Make iterable logic more consistent (Shantanu, PR [16006](https://github.com/python/mypy/pull/16006)) + * Fix inference for properties with `__call__` (Shantanu, PR [15926](https://github.com/python/mypy/pull/15926)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=4a854366e03dee700109f8e758a08b2457ea2f51+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Albert Tugushev +* Alex Waygood +* Ali Hamdan +* Anders Kaseorg +* Anthony Sottile +* Chad Dombrova +* Cibin Mathew +* dinaldoap +* DS/Charlie +* Eli Schwartz +* Ganden Schaffner +* Hamir Mahal +* Ihor +* Ikko Eltociear Ashimine +* Ilya Priven +* Ivan Levkivskyi +* Jelle Zijlstra +* Jukka Lehtosalo +* Jørgen Lind +* KotlinIsland +* Matt Bogosian +* Nikita Sobolev +* Petter Friberg +* Randolf Scholz +* Shantanu +* Thomas Grainger +* Valentin Stanciu + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + +Posted by Jukka Lehtosalo + +## Mypy 1.6 + +[Tuesday, 10 October 2023](https://mypy-lang.blogspot.com/2023/10/mypy-16-released.html) + +We’ve just uploaded mypy 1.6 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Introduce Error Subcodes for Import Errors + +Mypy now uses the error code import-untyped if an import targets an installed library that doesn’t support static type checking, and no stub files are available. Other invalid imports produce the import-not-found error code. They both are subcodes of the import error code, which was previously used for both kinds of import-related errors. + +Use \--disable-error-code=import-untyped to only ignore import errors about installed libraries without stubs. This way mypy will still report errors about typos in import statements, for example. + +If you use \--warn-unused-ignore or \--strict, mypy will complain if you use \# type: ignore\[import\] to ignore an import error. You are expected to use one of the more specific error codes instead. Otherwise, ignoring the import error code continues to silence both errors. + +This feature was contributed by Shantanu (PR [15840](https://github.com/python/mypy/pull/15840), PR [14740](https://github.com/python/mypy/pull/14740)). + +### Remove Support for Targeting Python 3.6 and Earlier + +Running mypy with \--python-version 3.6, for example, is no longer supported. Python 3.6 hasn’t been properly supported by mypy for some time now, and this makes it explicit. This was contributed by Nikita Sobolev (PR [15668](https://github.com/python/mypy/pull/15668)). + +### Selective Filtering of \--disallow-untyped-calls Targets + +Using \--disallow-untyped-calls could be annoying when using libraries with missing type information, as mypy would generate many errors about code that uses the library. Now you can use \--untyped-calls-exclude=acme, for example, to disable these errors about calls targeting functions defined in the acme package. Refer to the [documentation](https://mypy.readthedocs.io/en/latest/command_line.html#cmdoption-mypy-untyped-calls-exclude) for more information. + +This feature was contributed by Ivan Levkivskyi (PR [15845](https://github.com/python/mypy/pull/15845)). + +### Improved Type Inference between Callable Types + +Mypy now does a better job inferring type variables inside arguments of callable types. For example, this code fragment now type checks correctly: + +```python +def f(c: Callable[[T, S], None]) -> Callable[[str, T, S], None]: ... +def g(*x: int) -> None: ... + +reveal_type(f(g)) # Callable[[str, int, int], None] +``` + +This was contributed by Ivan Levkivskyi (PR [15910](https://github.com/python/mypy/pull/15910)). + +### Don’t Consider None and TypeVar to Overlap in Overloads + +Mypy now doesn’t consider an overload item with an argument type None to overlap with a type variable: + +```python +@overload +def f(x: None) -> None: .. +@overload +def f(x: T) -> Foo[T]: ... +... +``` + +Previously mypy would generate an error about the definition of f above. This is slightly unsafe if the upper bound of T is object, since the value of the type variable could be None. We relaxed the rules a little, since this solves a common issue. + +This feature was contributed by Ivan Levkivskyi (PR [15846](https://github.com/python/mypy/pull/15846)). + +### Improvements to \--new-type-inference + +The experimental new type inference algorithm (polymorphic inference) introduced as an opt-in feature in mypy 1.5 has several improvements: + +* Improve transitive closure computation during constraint solving (Ivan Levkivskyi, PR [15754](https://github.com/python/mypy/pull/15754)) +* Add support for upper bounds and values with \--new-type-inference (Ivan Levkivskyi, PR [15813](https://github.com/python/mypy/pull/15813)) +* Basic support for variadic types with \--new-type-inference (Ivan Levkivskyi, PR [15879](https://github.com/python/mypy/pull/15879)) +* Polymorphic inference: support for parameter specifications and lambdas (Ivan Levkivskyi, PR [15837](https://github.com/python/mypy/pull/15837)) +* Invalidate cache when adding \--new-type-inference (Marc Mueller, PR [16059](https://github.com/python/mypy/pull/16059)) + +**Note:** We are planning to enable \--new-type-inference by default in mypy 1.7. Please try this out and let us know if you encounter any issues. + +### ParamSpec Improvements + +* Support self-types containing ParamSpec (Ivan Levkivskyi, PR [15903](https://github.com/python/mypy/pull/15903)) +* Allow “…” in Concatenate, and clean up ParamSpec literals (Ivan Levkivskyi, PR [15905](https://github.com/python/mypy/pull/15905)) +* Fix ParamSpec inference for callback protocols (Ivan Levkivskyi, PR [15986](https://github.com/python/mypy/pull/15986)) +* Infer ParamSpec constraint from arguments (Ivan Levkivskyi, PR [15896](https://github.com/python/mypy/pull/15896)) +* Fix crash on invalid type variable with ParamSpec (Ivan Levkivskyi, PR [15953](https://github.com/python/mypy/pull/15953)) +* Fix subtyping between ParamSpecs (Ivan Levkivskyi, PR [15892](https://github.com/python/mypy/pull/15892)) + +### Stubgen Improvements + +* Add option to include docstrings with stubgen (chylek, PR [13284](https://github.com/python/mypy/pull/13284)) +* Add required ... initializer to NamedTuple fields with default values (Nikita Sobolev, PR [15680](https://github.com/python/mypy/pull/15680)) + +### Stubtest Improvements + +* Fix \_\_mypy-replace false positives (Alex Waygood, PR [15689](https://github.com/python/mypy/pull/15689)) +* Fix edge case for bytes enum subclasses (Alex Waygood, PR [15943](https://github.com/python/mypy/pull/15943)) +* Generate error if typeshed is missing modules from the stdlib (Alex Waygood, PR [15729](https://github.com/python/mypy/pull/15729)) +* Fixes to new check for missing stdlib modules (Alex Waygood, PR [15960](https://github.com/python/mypy/pull/15960)) +* Fix stubtest enum.Flag edge case (Alex Waygood, PR [15933](https://github.com/python/mypy/pull/15933)) + +### Documentation Improvements + +* Do not advertise to create your own assert\_never helper (Nikita Sobolev, PR [15947](https://github.com/python/mypy/pull/15947)) +* Fix all the missing references found within the docs (Albert Tugushev, PR [15875](https://github.com/python/mypy/pull/15875)) +* Document await-not-async error code (Shantanu, PR [15858](https://github.com/python/mypy/pull/15858)) +* Improve documentation of disabling error codes (Shantanu, PR [15841](https://github.com/python/mypy/pull/15841)) + +### Other Notable Changes and Fixes + +* Make unsupported PEP 695 features (introduced in Python 3.12) give a reasonable error message (Shantanu, PR [16013](https://github.com/python/mypy/pull/16013)) +* Remove the \--py2 command-line argument (Marc Mueller, PR [15670](https://github.com/python/mypy/pull/15670)) +* Change empty tuple from tuple\[\] to tuple\[()\] in error messages (Nikita Sobolev, PR [15783](https://github.com/python/mypy/pull/15783)) +* Fix assert\_type failures when some nodes are deferred (Nikita Sobolev, PR [15920](https://github.com/python/mypy/pull/15920)) +* Generate error on unbound TypeVar with values (Nikita Sobolev, PR [15732](https://github.com/python/mypy/pull/15732)) +* Fix over-eager types-google-cloud-ndb suggestion (Shantanu, PR [15347](https://github.com/python/mypy/pull/15347)) +* Fix type narrowing of \== None and in (None,) conditions (Marti Raudsepp, PR [15760](https://github.com/python/mypy/pull/15760)) +* Fix inference for attrs.fields (Shantanu, PR [15688](https://github.com/python/mypy/pull/15688)) +* Make “await in non-async function” a non-blocking error and give it an error code (Gregory Santosa, PR [15384](https://github.com/python/mypy/pull/15384)) +* Add basic support for decorated overloads (Ivan Levkivskyi, PR [15898](https://github.com/python/mypy/pull/15898)) +* Fix TypeVar regression with self types (Ivan Levkivskyi, PR [15945](https://github.com/python/mypy/pull/15945)) +* Add \_\_match\_args\_\_ to dataclasses with no fields (Ali Hamdan, PR [15749](https://github.com/python/mypy/pull/15749)) +* Include stdout and stderr in dmypy verbose output (Valentin Stanciu, PR [15881](https://github.com/python/mypy/pull/15881)) +* Improve match narrowing and reachability analysis (Shantanu, PR [15882](https://github.com/python/mypy/pull/15882)) +* Support \_\_bool\_\_ with Literal in \--warn-unreachable (Jannic Warken, PR [15645](https://github.com/python/mypy/pull/15645)) +* Fix inheriting from generic @frozen attrs class (Ilya Priven, PR [15700](https://github.com/python/mypy/pull/15700)) +* Correctly narrow types for tuple\[type\[X\], ...\] (Nikita Sobolev, PR [15691](https://github.com/python/mypy/pull/15691)) +* Don't flag intentionally empty generators unreachable (Ilya Priven, PR [15722](https://github.com/python/mypy/pull/15722)) +* Add tox.ini to mypy sdist (Marcel Telka, PR [15853](https://github.com/python/mypy/pull/15853)) +* Fix mypyc regression with pretty (Shantanu, PR [16124](https://github.com/python/mypy/pull/16124)) + +### Typeshed Updates + +Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=6a8d653a671925b0a3af61729ff8cf3f90c9c662+0&branch=main&path=stdlib) for full list of typeshed changes. + +### Acknowledgements + +Thanks to Max Murin, who did most of the release manager work for this release (I just did the final steps). + +Thanks to all mypy contributors who contributed to this release: + +* Albert Tugushev +* Alex Waygood +* Ali Hamdan +* chylek +* EXPLOSION +* Gregory Santosa +* Ilya Priven +* Ivan Levkivskyi +* Jannic Warken +* KotlinIsland +* Marc Mueller +* Marcel Johannesmann +* Marcel Telka +* Mark Byrne +* Marti Raudsepp +* Max Murin +* Nikita Sobolev +* Shantanu +* Valentin Stanciu + +Posted by Jukka Lehtosalo + + +## Mypy 1.5 + +[Thursday, 10 August 2023](https://mypy-lang.blogspot.com/2023/08/mypy-15-released.html) + +We’ve just uploaded mypy 1.5 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, deprecations and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Drop Support for Python 3.7 + +Mypy no longer supports running with Python 3.7, which has reached end-of-life. This was contributed by Shantanu (PR [15566](https://github.com/python/mypy/pull/15566)). + +### Optional Check to Require Explicit @override + +If you enable the explicit-override error code, mypy will generate an error if a method override doesn’t use the @typing.override decorator (as discussed in [PEP 698](https://peps.python.org/pep-0698/#strict-enforcement-per-project)). This way mypy will detect accidentally introduced overrides. Example: + +```python +# mypy: enable-error-code="explicit-override" + +from typing_extensions import override + +class C: + def foo(self) -> None: pass + def bar(self) -> None: pass + +class D(C): + # Error: Method "foo" is not using @override but is + # overriding a method + def foo(self) -> None: + ... + + @override + def bar(self) -> None: # OK + ... +``` + +You can enable the error code via \--enable-error-code=explicit-override on the mypy command line or enable\_error\_code = explicit-override in the mypy config file. + +The override decorator will be available in typing in Python 3.12, but you can also use the backport from a recent version of `typing_extensions` on all supported Python versions. + +This feature was contributed by Marc Mueller(PR [15512](https://github.com/python/mypy/pull/15512)). + +### More Flexible TypedDict Creation and Update + +Mypy was previously overly strict when type checking TypedDict creation and update operations. Though these checks were often technically correct, they sometimes triggered for apparently valid code. These checks have now been relaxed by default. You can enable stricter checking by using the new \--extra-checks flag. + +Construction using the `**` syntax is now more flexible: + +```python +from typing import TypedDict + +class A(TypedDict): + foo: int + bar: int + +class B(TypedDict): + foo: int + +a: A = {"foo": 1, "bar": 2} +b: B = {"foo": 3} +a2: A = { **a, **b} # OK (previously an error) +``` + +You can also call update() with a TypedDict argument that contains a subset of the keys in the updated TypedDict: +```python +a.update(b) # OK (previously an error) +``` + +This feature was contributed by Ivan Levkivskyi (PR [15425](https://github.com/python/mypy/pull/15425)). + +### Deprecated Flag: \--strict-concatenate + +The behavior of \--strict-concatenate is now included in the new \--extra-checks flag, and the old flag is deprecated. + +### Optionally Show Links to Error Code Documentation + +If you use \--show-error-code-links, mypy will add documentation links to (many) reported errors. The links are not shown for error messages that are sufficiently obvious, and they are shown once per error code only. + +Example output: +``` +a.py:1: error: Need type annotation for "foo" (hint: "x: List[] = ...") [var-annotated] +a.py:1: note: See https://mypy.rtfd.io/en/stable/_refs.html#code-var-annotated for more info +``` +This was contributed by Ivan Levkivskyi (PR [15449](https://github.com/python/mypy/pull/15449)). + +### Consistently Avoid Type Checking Unreachable Code + +If a module top level has unreachable code, mypy won’t type check the unreachable statements. This is consistent with how functions behave. The behavior of \--warn-unreachable is also more consistent now. + +This was contributed by Ilya Priven (PR [15386](https://github.com/python/mypy/pull/15386)). + +### Experimental Improved Type Inference for Generic Functions + +You can use \--new-type-inference to opt into an experimental new type inference algorithm. It fixes issues when calling a generic functions with an argument that is also a generic function, in particular. This current implementation is still incomplete, but we encourage trying it out and reporting bugs if you encounter regressions. We are planning to enable the new algorithm by default in a future mypy release. + +This feature was contributed by Ivan Levkivskyi (PR [15287](https://github.com/python/mypy/pull/15287)). + +### Partial Support for Python 3.12 + +Mypy and mypyc now support running on recent Python 3.12 development versions. Not all new Python 3.12 features are supported, and we don’t ship compiled wheels for Python 3.12 yet. + +* Fix ast warnings for Python 3.12 (Nikita Sobolev, PR [15558](https://github.com/python/mypy/pull/15558)) +* mypyc: Fix multiple inheritance with a protocol on Python 3.12 (Jukka Lehtosalo, PR [15572](https://github.com/python/mypy/pull/15572)) +* mypyc: Fix self-compilation on Python 3.12 (Jukka Lehtosalo, PR [15582](https://github.com/python/mypy/pull/15582)) +* mypyc: Fix 3.12 issue with pickling of instances with \_\_dict\_\_ (Jukka Lehtosalo, PR [15574](https://github.com/python/mypy/pull/15574)) +* mypyc: Fix i16 on Python 3.12 (Jukka Lehtosalo, PR [15510](https://github.com/python/mypy/pull/15510)) +* mypyc: Fix int operations on Python 3.12 (Jukka Lehtosalo, PR [15470](https://github.com/python/mypy/pull/15470)) +* mypyc: Fix generators on Python 3.12 (Jukka Lehtosalo, PR [15472](https://github.com/python/mypy/pull/15472)) +* mypyc: Fix classes with \_\_dict\_\_ on 3.12 (Jukka Lehtosalo, PR [15471](https://github.com/python/mypy/pull/15471)) +* mypyc: Fix coroutines on Python 3.12 (Jukka Lehtosalo, PR [15469](https://github.com/python/mypy/pull/15469)) +* mypyc: Don't use \_PyErr\_ChainExceptions on 3.12, since it's deprecated (Jukka Lehtosalo, PR [15468](https://github.com/python/mypy/pull/15468)) +* mypyc: Add Python 3.12 feature macro (Jukka Lehtosalo, PR [15465](https://github.com/python/mypy/pull/15465)) + +### Improvements to Dataclasses + +* Improve signature of dataclasses.replace (Ilya Priven, PR [14849](https://github.com/python/mypy/pull/14849)) +* Fix dataclass/protocol crash on joining types (Ilya Priven, PR [15629](https://github.com/python/mypy/pull/15629)) +* Fix strict optional handling in dataclasses (Ivan Levkivskyi, PR [15571](https://github.com/python/mypy/pull/15571)) +* Support optional types for custom dataclass descriptors (Marc Mueller, PR [15628](https://github.com/python/mypy/pull/15628)) +* Add `__slots__` attribute to dataclasses (Nikita Sobolev, PR [15649](https://github.com/python/mypy/pull/15649)) +* Support better \_\_post\_init\_\_ method signature for dataclasses (Nikita Sobolev, PR [15503](https://github.com/python/mypy/pull/15503)) + +### Mypyc Improvements + +* Support unsigned 8-bit native integer type: mypy\_extensions.u8 (Jukka Lehtosalo, PR [15564](https://github.com/python/mypy/pull/15564)) +* Support signed 16-bit native integer type: mypy\_extensions.i16 (Jukka Lehtosalo, PR [15464](https://github.com/python/mypy/pull/15464)) +* Define mypy\_extensions.i16 in stubs (Jukka Lehtosalo, PR [15562](https://github.com/python/mypy/pull/15562)) +* Document more unsupported features and update supported features (Richard Si, PR [15524](https://github.com/python/mypy/pull/15524)) +* Fix final NamedTuple classes (Richard Si, PR [15513](https://github.com/python/mypy/pull/15513)) +* Use C99 compound literals for undefined tuple values (Jukka Lehtosalo, PR [15453](https://github.com/python/mypy/pull/15453)) +* Don't explicitly assign NULL values in setup functions (Logan Hunt, PR [15379](https://github.com/python/mypy/pull/15379)) + +### Stubgen Improvements + +* Teach stubgen to work with complex and unary expressions (Nikita Sobolev, PR [15661](https://github.com/python/mypy/pull/15661)) +* Support ParamSpec and TypeVarTuple (Ali Hamdan, PR [15626](https://github.com/python/mypy/pull/15626)) +* Fix crash on non-str docstring (Ali Hamdan, PR [15623](https://github.com/python/mypy/pull/15623)) + +### Documentation Updates + +* Add documentation for additional error codes (Ivan Levkivskyi, PR [15539](https://github.com/python/mypy/pull/15539)) +* Improve documentation of type narrowing (Ilya Priven, PR [15652](https://github.com/python/mypy/pull/15652)) +* Small improvements to protocol documentation (Shantanu, PR [15460](https://github.com/python/mypy/pull/15460)) +* Remove confusing instance variable example in cheat sheet (Adel Atallah, PR [15441](https://github.com/python/mypy/pull/15441)) + +### Other Notable Fixes and Improvements + +* Constant fold additional unary and binary expressions (Richard Si, PR [15202](https://github.com/python/mypy/pull/15202)) +* Exclude the same special attributes from Protocol as CPython (Kyle Benesch, PR [15490](https://github.com/python/mypy/pull/15490)) +* Change the default value of the slots argument of attrs.define to True, to match runtime behavior (Ilya Priven, PR [15642](https://github.com/python/mypy/pull/15642)) +* Fix type of class attribute if attribute is defined in both class and metaclass (Alex Waygood, PR [14988](https://github.com/python/mypy/pull/14988)) +* Handle type the same as typing.Type in the first argument of classmethods (Erik Kemperman, PR [15297](https://github.com/python/mypy/pull/15297)) +* Fix \--find-occurrences flag (Shantanu, PR [15528](https://github.com/python/mypy/pull/15528)) +* Fix error location for class patterns (Nikita Sobolev, PR [15506](https://github.com/python/mypy/pull/15506)) +* Fix re-added file with errors in mypy daemon (Ivan Levkivskyi, PR [15440](https://github.com/python/mypy/pull/15440)) +* Fix dmypy run on Windows (Ivan Levkivskyi, PR [15429](https://github.com/python/mypy/pull/15429)) +* Fix abstract and non-abstract variant error for property deleter (Shantanu, PR [15395](https://github.com/python/mypy/pull/15395)) +* Remove special casing for "cannot" in error messages (Ilya Priven, PR [15428](https://github.com/python/mypy/pull/15428)) +* Add runtime `__slots__` attribute to attrs classes (Nikita Sobolev, PR [15651](https://github.com/python/mypy/pull/15651)) +* Add get\_expression\_type to CheckerPluginInterface (Ilya Priven, PR [15369](https://github.com/python/mypy/pull/15369)) +* Remove parameters that no longer exist from NamedTuple.\_make() (Alex Waygood, PR [15578](https://github.com/python/mypy/pull/15578)) +* Allow using typing.Self in `__all__` with an explicit @staticmethod decorator (Erik Kemperman, PR [15353](https://github.com/python/mypy/pull/15353)) +* Fix self types in subclass methods without Self annotation (Ivan Levkivskyi, PR [15541](https://github.com/python/mypy/pull/15541)) +* Check for abstract class objects in tuples (Nikita Sobolev, PR [15366](https://github.com/python/mypy/pull/15366)) + +### Typeshed Updates + +Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=fc7d4722eaa54803926cee5730e1f784979c0531+0&branch=main&path=stdlib) for full list of typeshed changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Adel Atallah +* Alex Waygood +* Ali Hamdan +* Erik Kemperman +* Federico Padua +* Ilya Priven +* Ivan Levkivskyi +* Jelle Zijlstra +* Jared Hance +* Jukka Lehtosalo +* Kyle Benesch +* Logan Hunt +* Marc Mueller +* Nikita Sobolev +* Richard Si +* Shantanu +* Stavros Ntentos +* Valentin Stanciu + +Posted by Valentin Stanciu + + +## Mypy 1.4 + +[Tuesday, 20 June 2023](https://mypy-lang.blogspot.com/2023/06/mypy-140-released.html) + +We’ve just uploaded mypy 1.4 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### The Override Decorator + +Mypy can now ensure that when renaming a method, overrides are also renamed. You can explicitly mark a method as overriding a base class method by using the @typing.override decorator ([PEP 698](https://peps.python.org/pep-0698/)). If the method is then renamed in the base class while the method override is not, mypy will generate an error. The decorator will be available in typing in Python 3.12, but you can also use the backport from a recent version of `typing_extensions` on all supported Python versions. + +This feature was contributed byThomas M Kehrenberg (PR [14609](https://github.com/python/mypy/pull/14609)). + +### Propagating Type Narrowing to Nested Functions + +Previously, type narrowing was not propagated to nested functions because it would not be sound if the narrowed variable changed between the definition of the nested function and the call site. Mypy will now propagate the narrowed type if the variable is not assigned to after the definition of the nested function: + +```python +def outer(x: str | None = None) -> None: + if x is None: + x = calculate_default() + reveal_type(x) # "str" (narrowed) + + def nested() -> None: + reveal_type(x) # Now "str" (used to be "str | None") + + nested() +``` + +This may generate some new errors because asserts that were previously necessary may become tautological or no-ops. + +This was contributed by Jukka Lehtosalo (PR [15133](https://github.com/python/mypy/pull/15133)). + +### Narrowing Enum Values Using “==” + +Mypy now allows narrowing enum types using the \== operator. Previously this was only supported when using the is operator. This makes exhaustiveness checking with enum types more usable, as the requirement to use the is operator was not very intuitive. In this example mypy can detect that the developer forgot to handle the value MyEnum.C in example + +```python +from enum import Enum + +class MyEnum(Enum): + A = 0 + B = 1 + C = 2 + +def example(e: MyEnum) -> str: # Error: Missing return statement + if e == MyEnum.A: + return 'x' + elif e == MyEnum.B: + return 'y' +``` + +Adding an extra elif case resolves the error: + +```python +... +def example(e: MyEnum) -> str: # No error -- all values covered + if e == MyEnum.A: + return 'x' + elif e == MyEnum.B: + return 'y' + elif e == MyEnum.C: + return 'z' +``` + +This change can cause false positives in test cases that have assert statements like assert o.x == SomeEnum.X when using \--strict-equality. Example: + +```python +# mypy: strict-equality + +from enum import Enum + +class MyEnum(Enum): + A = 0 + B = 1 + +class C: + x: MyEnum + ... + +def test_something() -> None: + c = C(...) + assert c.x == MyEnum.A + c.do_something_that_changes_x() + assert c.x == MyEnum.B # Error: Non-overlapping equality check +``` + +These errors can be ignored using \# type: ignore\[comparison-overlap\], or you can perform the assertion using a temporary variable as a workaround: + +```python +... +def test_something() -> None: + ... + x = c.x + assert x == MyEnum.A # Does not narrow c.x + c.do_something_that_changes_x() + x = c.x + assert x == MyEnum.B # OK +``` + +This feature was contributed by Shantanu (PR [11521](https://github.com/python/mypy/pull/11521)). + +### Performance Improvements + +* Speed up simplification of large union types and also fix a recursive tuple crash (Shantanu, PR [15128](https://github.com/python/mypy/pull/15128)) +* Speed up union subtyping (Shantanu, PR [15104](https://github.com/python/mypy/pull/15104)) +* Don't type check most function bodies when type checking third-party library code, or generally when ignoring errors (Jukka Lehtosalo, PR [14150](https://github.com/python/mypy/pull/14150)) + +### Improvements to Plugins + +* attrs.evolve: Support generics and unions (Ilya Konstantinov, PR [15050](https://github.com/python/mypy/pull/15050)) +* Fix ctypes plugin (Alex Waygood) + +### Fixes to Crashes + +* Fix a crash when function-scope recursive alias appears as upper bound (Ivan Levkivskyi, PR [15159](https://github.com/python/mypy/pull/15159)) +* Fix crash on follow\_imports\_for\_stubs (Ivan Levkivskyi, PR [15407](https://github.com/python/mypy/pull/15407)) +* Fix stubtest crash in explicit init subclass (Shantanu, PR [15399](https://github.com/python/mypy/pull/15399)) +* Fix crash when indexing TypedDict with empty key (Shantanu, PR [15392](https://github.com/python/mypy/pull/15392)) +* Fix crash on NamedTuple as attribute (Ivan Levkivskyi, PR [15404](https://github.com/python/mypy/pull/15404)) +* Correctly track loop depth for nested functions/classes (Ivan Levkivskyi, PR [15403](https://github.com/python/mypy/pull/15403)) +* Fix crash on joins with recursive tuples (Ivan Levkivskyi, PR [15402](https://github.com/python/mypy/pull/15402)) +* Fix crash with custom ErrorCode subclasses (Marc Mueller, PR [15327](https://github.com/python/mypy/pull/15327)) +* Fix crash in dataclass protocol with self attribute assignment (Ivan Levkivskyi, PR [15157](https://github.com/python/mypy/pull/15157)) +* Fix crash on lambda in generic context with generic method in body (Ivan Levkivskyi, PR [15155](https://github.com/python/mypy/pull/15155)) +* Fix recursive type alias crash in make\_simplified\_union (Ivan Levkivskyi, PR [15216](https://github.com/python/mypy/pull/15216)) + +### Improvements to Error Messages + +* Use lower-case built-in collection types such as list\[…\] instead of List\[…\] in errors when targeting Python 3.9+ (Max Murin, PR [15070](https://github.com/python/mypy/pull/15070)) +* Use X | Y union syntax in error messages when targeting Python 3.10+ (Omar Silva, PR [15102](https://github.com/python/mypy/pull/15102)) +* Use type instead of Type in errors when targeting Python 3.9+ (Rohit Sanjay, PR [15139](https://github.com/python/mypy/pull/15139)) +* Do not show unused-ignore errors in unreachable code, and make it a real error code (Ivan Levkivskyi, PR [15164](https://github.com/python/mypy/pull/15164)) +* Don’t limit the number of errors shown by default (Rohit Sanjay, PR [15138](https://github.com/python/mypy/pull/15138)) +* Improver message for truthy functions (madt2709, PR [15193](https://github.com/python/mypy/pull/15193)) +* Output distinct types when type names are ambiguous (teresa0605, PR [15184](https://github.com/python/mypy/pull/15184)) +* Update message about invalid exception type in try (AJ Rasmussen, PR [15131](https://github.com/python/mypy/pull/15131)) +* Add explanation if argument type is incompatible because of an unsupported numbers type (Jukka Lehtosalo, PR [15137](https://github.com/python/mypy/pull/15137)) +* Add more detail to 'signature incompatible with supertype' messages for non-callables (Ilya Priven, PR [15263](https://github.com/python/mypy/pull/15263)) + +### Documentation Updates + +* Add \--local-partial-types note to dmypy docs (Alan Du, PR [15259](https://github.com/python/mypy/pull/15259)) +* Update getting started docs for mypyc for Windows (Valentin Stanciu, PR [15233](https://github.com/python/mypy/pull/15233)) +* Clarify usage of callables regarding type object in docs (Viicos, PR [15079](https://github.com/python/mypy/pull/15079)) +* Clarify difference between disallow\_untyped\_defs and disallow\_incomplete\_defs (Ilya Priven, PR [15247](https://github.com/python/mypy/pull/15247)) +* Use attrs and @attrs.define in documentation and tests (Ilya Priven, PR [15152](https://github.com/python/mypy/pull/15152)) + +### Mypyc Improvements + +* Fix unexpected TypeError for certain variables with an inferred optional type (Richard Si, PR [15206](https://github.com/python/mypy/pull/15206)) +* Inline math literals (Logan Hunt, PR [15324](https://github.com/python/mypy/pull/15324)) +* Support unpacking mappings in dict display (Richard Si, PR [15203](https://github.com/python/mypy/pull/15203)) + +### Changes to Stubgen + +* Do not remove Generic from base classes (Ali Hamdan, PR [15316](https://github.com/python/mypy/pull/15316)) +* Support yield from statements (Ali Hamdan, PR [15271](https://github.com/python/mypy/pull/15271)) +* Fix missing total from TypedDict class (Ali Hamdan, PR [15208](https://github.com/python/mypy/pull/15208)) +* Fix call-based namedtuple omitted from class bases (Ali Hamdan, PR [14680](https://github.com/python/mypy/pull/14680)) +* Support TypedDict alternative syntax (Ali Hamdan, PR [14682](https://github.com/python/mypy/pull/14682)) +* Make stubgen respect MYPY\_CACHE\_DIR (Henrik Bäärnhielm, PR [14722](https://github.com/python/mypy/pull/14722)) +* Fixes and simplifications (Ali Hamdan, PR [15232](https://github.com/python/mypy/pull/15232)) + +### Other Notable Fixes and Improvements + +* Fix nested async functions when using TypeVar value restriction (Jukka Lehtosalo, PR [14705](https://github.com/python/mypy/pull/14705)) +* Always allow returning Any from lambda (Ivan Levkivskyi, PR [15413](https://github.com/python/mypy/pull/15413)) +* Add foundation for TypeVar defaults (PEP 696) (Marc Mueller, PR [14872](https://github.com/python/mypy/pull/14872)) +* Update semantic analyzer for TypeVar defaults (PEP 696) (Marc Mueller, PR [14873](https://github.com/python/mypy/pull/14873)) +* Make dict expression inference more consistent (Ivan Levkivskyi, PR [15174](https://github.com/python/mypy/pull/15174)) +* Do not block on duplicate base classes (Nikita Sobolev, PR [15367](https://github.com/python/mypy/pull/15367)) +* Generate an error when both staticmethod and classmethod decorators are used (Juhi Chandalia, PR [15118](https://github.com/python/mypy/pull/15118)) +* Fix assert\_type behaviour with literals (Carl Karsten, PR [15123](https://github.com/python/mypy/pull/15123)) +* Fix match subject ignoring redefinitions (Vincent Vanlaer, PR [15306](https://github.com/python/mypy/pull/15306)) +* Support `__all__`.remove (Shantanu, PR [15279](https://github.com/python/mypy/pull/15279)) + +### Typeshed Updates + +Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=877e06ad1cfd9fd9967c0b0340a86d0c23ea89ce+0&branch=main&path=stdlib) for full list of typeshed changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Adrian Garcia Badaracco +* AJ Rasmussen +* Alan Du +* Alex Waygood +* Ali Hamdan +* Carl Karsten +* dosisod +* Ethan Smith +* Gregory Santosa +* Heather White +* Henrik Bäärnhielm +* Ilya Konstantinov +* Ilya Priven +* Ivan Levkivskyi +* Juhi Chandalia +* Jukka Lehtosalo +* Logan Hunt +* madt2709 +* Marc Mueller +* Max Murin +* Nikita Sobolev +* Omar Silva +* Özgür +* Richard Si +* Rohit Sanjay +* Shantanu +* teresa0605 +* Thomas M Kehrenberg +* Tin Tvrtković +* Tushar Sadhwani +* Valentin Stanciu +* Viicos +* Vincent Vanlaer +* Wesley Collin Wright +* William Santosa +* yaegassy + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + +Posted by Jared Hance + + +## Mypy 1.3 + +[Wednesday, 10 May 2023](https://mypy-lang.blogspot.com/2023/05/mypy-13-released.html) + + We’ve just uploaded mypy 1.3 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Performance Improvements + +* Improve performance of union subtyping (Shantanu, PR [15104](https://github.com/python/mypy/pull/15104)) +* Add negative subtype caches (Ivan Levkivskyi, PR [14884](https://github.com/python/mypy/pull/14884)) + +### Stub Tooling Improvements + +* Stubtest: Check that the stub is abstract if the runtime is, even when the stub is an overloaded method (Alex Waygood, PR [14955](https://github.com/python/mypy/pull/14955)) +* Stubtest: Verify stub methods or properties are decorated with @final if they are decorated with @final at runtime (Alex Waygood, PR [14951](https://github.com/python/mypy/pull/14951)) +* Stubtest: Fix stubtest false positives with TypedDicts at runtime (Alex Waygood, PR [14984](https://github.com/python/mypy/pull/14984)) +* Stubgen: Support @functools.cached\_property (Nikita Sobolev, PR [14981](https://github.com/python/mypy/pull/14981)) +* Improvements to stubgenc (Chad Dombrova, PR [14564](https://github.com/python/mypy/pull/14564)) + +### Improvements to attrs + +* Add support for converters with TypeVars on generic attrs classes (Chad Dombrova, PR [14908](https://github.com/python/mypy/pull/14908)) +* Fix attrs.evolve on bound TypeVar (Ilya Konstantinov, PR [15022](https://github.com/python/mypy/pull/15022)) + +### Documentation Updates + +* Improve async documentation (Shantanu, PR [14973](https://github.com/python/mypy/pull/14973)) +* Improvements to cheat sheet (Shantanu, PR [14972](https://github.com/python/mypy/pull/14972)) +* Add documentation for bytes formatting error code (Shantanu, PR [14971](https://github.com/python/mypy/pull/14971)) +* Convert insecure links to use HTTPS (Marti Raudsepp, PR [14974](https://github.com/python/mypy/pull/14974)) +* Also mention overloads in async iterator documentation (Shantanu, PR [14998](https://github.com/python/mypy/pull/14998)) +* stubtest: Improve allowlist documentation (Shantanu, PR [15008](https://github.com/python/mypy/pull/15008)) +* Clarify "Using types... but not at runtime" (Jon Shea, PR [15029](https://github.com/python/mypy/pull/15029)) +* Fix alignment of cheat sheet example (Ondřej Cvacho, PR [15039](https://github.com/python/mypy/pull/15039)) +* Fix error for callback protocol matching against callable type object (Shantanu, PR [15042](https://github.com/python/mypy/pull/15042)) + +### Error Reporting Improvements + +* Improve bytes formatting error (Shantanu, PR [14959](https://github.com/python/mypy/pull/14959)) + +### Mypyc Improvements + +* Fix unions of bools and ints (Tomer Chachamu, PR [15066](https://github.com/python/mypy/pull/15066)) + +### Other Fixes and Improvements + +* Fix narrowing union types that include Self with isinstance (Christoph Tyralla, PR [14923](https://github.com/python/mypy/pull/14923)) +* Allow objects matching SupportsKeysAndGetItem to be unpacked (Bryan Forbes, PR [14990](https://github.com/python/mypy/pull/14990)) +* Check type guard validity for staticmethods (EXPLOSION, PR [14953](https://github.com/python/mypy/pull/14953)) +* Fix sys.platform when cross-compiling with emscripten (Ethan Smith, PR [14888](https://github.com/python/mypy/pull/14888)) + +### Typeshed Updates + +Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=b0ed50e9392a23e52445b630a808153e0e256976+0&branch=main&path=stdlib) for full list of typeshed changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Alex Waygood +* Amin Alaee +* Bryan Forbes +* Chad Dombrova +* Charlie Denton +* Christoph Tyralla +* dosisod +* Ethan Smith +* EXPLOSION +* Ilya Konstantinov +* Ivan Levkivskyi +* Jon Shea +* Jukka Lehtosalo +* KotlinIsland +* Marti Raudsepp +* Nikita Sobolev +* Ondřej Cvacho +* Shantanu +* sobolevn +* Tomer Chachamu +* Yaroslav Halchenko + +Posted by Wesley Collin Wright. + + +## Mypy 1.2 + +[Thursday, 6 April 2023](https://mypy-lang.blogspot.com/2023/04/mypy-12-released.html) + +We’ve just uploaded mypy 1.2 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Improvements to Dataclass Transforms + +* Support implicit default for "init" parameter in field specifiers (Wesley Collin Wright and Jukka Lehtosalo, PR [15010](https://github.com/python/mypy/pull/15010)) +* Support descriptors in dataclass transform (Jukka Lehtosalo, PR [15006](https://github.com/python/mypy/pull/15006)) +* Fix frozen\_default in incremental mode (Wesley Collin Wright) +* Fix frozen behavior for base classes with direct metaclasses (Wesley Collin Wright, PR [14878](https://github.com/python/mypy/pull/14878)) + +### Mypyc: Native Floats + +Mypyc now uses a native, unboxed representation for values of type float. Previously these were heap-allocated Python objects. Native floats are faster and use less memory. Code that uses floating-point operations heavily can be several times faster when using native floats. + +Various float operations and math functions also now have optimized implementations. Refer to the [documentation](https://mypyc.readthedocs.io/en/latest/float_operations.html) for a full list. + +This can change the behavior of existing code that uses subclasses of float. When assigning an instance of a subclass of float to a variable with the float type, it gets implicitly converted to a float instance when compiled: + +```python +from lib import MyFloat # MyFloat ia a subclass of "float" + +def example() -> None: + x = MyFloat(1.5) + y: float = x # Implicit conversion from MyFloat to float + print(type(y)) # float, not MyFloat +``` + +Previously, implicit conversions were applied to int subclasses but not float subclasses. + +Also, int values can no longer be assigned to a variable with type float in compiled code, since these types now have incompatible representations. An explicit conversion is required: + +```python +def example(n: int) -> None: + a: float = 1 # Error: cannot assign "int" to "float" + b: float = 1.0 # OK + c: float = n # Error + d: float = float(n) # OK +``` + +This restriction only applies to assignments, since they could otherwise narrow down the type of a variable from float to int. int values can still be implicitly converted to float when passed as arguments to functions that expect float values. + +Note that mypyc still doesn’t support arrays of unboxed float values. Using list\[float\] involves heap-allocated float objects, since list can only store boxed values. Support for efficient floating point arrays is one of the next major planned mypyc features. + +Related changes: + +* Use a native unboxed representation for floats (Jukka Lehtosalo, PR [14880](https://github.com/python/mypy/pull/14880)) +* Document native floats and integers (Jukka Lehtosalo, PR [14927](https://github.com/python/mypy/pull/14927)) +* Fixes to float to int conversion (Jukka Lehtosalo, PR [14936](https://github.com/python/mypy/pull/14936)) + +### Mypyc: Native Integers + +Mypyc now supports signed 32-bit and 64-bit integer types in addition to the arbitrary-precision int type. You can use the types mypy\_extensions.i32 and mypy\_extensions.i64 to speed up code that uses integer operations heavily. + +Simple example: +```python +from mypy_extensions import i64 + +def inc(x: i64) -> i64: + return x + 1 +``` + +Refer to the [documentation](https://mypyc.readthedocs.io/en/latest/using_type_annotations.html#native-integer-types) for more information. This feature was contributed by Jukka Lehtosalo. + +### Other Mypyc Fixes and Improvements + +* Support iterating over a TypedDict (Richard Si, PR [14747](https://github.com/python/mypy/pull/14747)) +* Faster coercions between different tuple types (Jukka Lehtosalo, PR [14899](https://github.com/python/mypy/pull/14899)) +* Faster calls via type aliases (Jukka Lehtosalo, PR [14784](https://github.com/python/mypy/pull/14784)) +* Faster classmethod calls via cls (Jukka Lehtosalo, PR [14789](https://github.com/python/mypy/pull/14789)) + +### Fixes to Crashes + +* Fix crash on class-level import in protocol definition (Ivan Levkivskyi, PR [14926](https://github.com/python/mypy/pull/14926)) +* Fix crash on single item union of alias (Ivan Levkivskyi, PR [14876](https://github.com/python/mypy/pull/14876)) +* Fix crash on ParamSpec in incremental mode (Ivan Levkivskyi, PR [14885](https://github.com/python/mypy/pull/14885)) + +### Documentation Updates + +* Update adopting \--strict documentation for 1.0 (Shantanu, PR [14865](https://github.com/python/mypy/pull/14865)) +* Some minor documentation tweaks (Jukka Lehtosalo, PR [14847](https://github.com/python/mypy/pull/14847)) +* Improve documentation of top level mypy: disable-error-code comment (Nikita Sobolev, PR [14810](https://github.com/python/mypy/pull/14810)) + +### Error Reporting Improvements + +* Add error code to `typing_extensions` suggestion (Shantanu, PR [14881](https://github.com/python/mypy/pull/14881)) +* Add a separate error code for top-level await (Nikita Sobolev, PR [14801](https://github.com/python/mypy/pull/14801)) +* Don’t suggest two obsolete stub packages (Jelle Zijlstra, PR [14842](https://github.com/python/mypy/pull/14842)) +* Add suggestions for pandas-stubs and lxml-stubs (Shantanu, PR [14737](https://github.com/python/mypy/pull/14737)) + +### Other Fixes and Improvements + +* Multiple inheritance considers callable objects as subtypes of functions (Christoph Tyralla, PR [14855](https://github.com/python/mypy/pull/14855)) +* stubtest: Respect @final runtime decorator and enforce it in stubs (Nikita Sobolev, PR [14922](https://github.com/python/mypy/pull/14922)) +* Fix false positives related to type\[\] (sterliakov, PR [14756](https://github.com/python/mypy/pull/14756)) +* Fix duplication of ParamSpec prefixes and properly substitute ParamSpecs (EXPLOSION, PR [14677](https://github.com/python/mypy/pull/14677)) +* Fix line number if `__iter__` is incorrectly reported as missing (Jukka Lehtosalo, PR [14893](https://github.com/python/mypy/pull/14893)) +* Fix incompatible overrides of overloaded generics with self types (Shantanu, PR [14882](https://github.com/python/mypy/pull/14882)) +* Allow SupportsIndex in slice expressions (Shantanu, PR [14738](https://github.com/python/mypy/pull/14738)) +* Support if statements in bodies of dataclasses and classes that use dataclass\_transform (Jacek Chałupka, PR [14854](https://github.com/python/mypy/pull/14854)) +* Allow iterable class objects to be unpacked (including enums) (Alex Waygood, PR [14827](https://github.com/python/mypy/pull/14827)) +* Fix narrowing for walrus expressions used in match statements (Shantanu, PR [14844](https://github.com/python/mypy/pull/14844)) +* Add signature for attr.evolve (Ilya Konstantinov, PR [14526](https://github.com/python/mypy/pull/14526)) +* Fix Any inference when unpacking iterators that don't directly inherit from typing.Iterator (Alex Waygood, PR [14821](https://github.com/python/mypy/pull/14821)) +* Fix unpack with overloaded `__iter__` method (Nikita Sobolev, PR [14817](https://github.com/python/mypy/pull/14817)) +* Reduce size of JSON data in mypy cache (dosisod, PR [14808](https://github.com/python/mypy/pull/14808)) +* Improve “used before definition” checks when a local definition has the same name as a global definition (Stas Ilinskiy, PR [14517](https://github.com/python/mypy/pull/14517)) +* Honor NoReturn as \_\_setitem\_\_ return type to mark unreachable code (sterliakov, PR [12572](https://github.com/python/mypy/pull/12572)) + +### Typeshed Updates + +Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=a544b75320e97424d2d927605316383c755cdac0+0&branch=main&path=stdlib) for full list of typeshed changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Alex Waygood +* Avasam +* Christoph Tyralla +* dosisod +* EXPLOSION +* Ilya Konstantinov +* Ivan Levkivskyi +* Jacek Chałupka +* Jelle Zijlstra +* Jukka Lehtosalo +* Marc Mueller +* Max Murin +* Nikita Sobolev +* Richard Si +* Shantanu +* Stas Ilinskiy +* sterliakov +* Wesley Collin Wright + +Posted by Jukka Lehtosalo + + +## Mypy 1.1.1 + +[Monday, 6 March 2023](https://mypy-lang.blogspot.com/2023/03/mypy-111-released.html) + + We’ve just uploaded mypy 1.1.1 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Support for `dataclass_transform`` + +This release adds full support for the dataclass\_transform decorator defined in [PEP 681](https://peps.python.org/pep-0681/#decorator-function-example). This allows decorators, base classes, and metaclasses that generate a \_\_init\_\_ method or other methods based on the properties of that class (similar to dataclasses) to have those methods recognized by mypy. + +This was contributed by Wesley Collin Wright. + +### Dedicated Error Code for Method Assignments + +Mypy can’t safely check all assignments to methods (a form of monkey patching), so mypy generates an error by default. To make it easier to ignore this error, mypy now uses the new error code method-assign for this. By disabling this error code in a file or globally, mypy will no longer complain about assignments to methods if the signatures are compatible. + +Mypy also supports the old error code assignment for these assignments to prevent a backward compatibility break. More generally, we can use this mechanism in the future if we wish to split or rename another existing error code without causing backward compatibility issues. + +This was contributed by Ivan Levkivskyi (PR [14570](https://github.com/python/mypy/pull/14570)). + +### Fixes to Crashes + +* Fix a crash on walrus in comprehension at class scope (Ivan Levkivskyi, PR [14556](https://github.com/python/mypy/pull/14556)) +* Fix crash related to value-constrained TypeVar (Shantanu, PR [14642](https://github.com/python/mypy/pull/14642)) + +### Fixes to Cache Corruption + +* Fix generic TypedDict/NamedTuple caching (Ivan Levkivskyi, PR [14675](https://github.com/python/mypy/pull/14675)) + +### Mypyc Fixes and Improvements + +* Raise "non-trait base must be first..." error less frequently (Richard Si, PR [14468](https://github.com/python/mypy/pull/14468)) +* Generate faster code for bool comparisons and arithmetic (Jukka Lehtosalo, PR [14489](https://github.com/python/mypy/pull/14489)) +* Optimize \_\_(a)enter\_\_/\_\_(a)exit\_\_ for native classes (Jared Hance, PR [14530](https://github.com/python/mypy/pull/14530)) +* Detect if attribute definition conflicts with base class/trait (Jukka Lehtosalo, PR [14535](https://github.com/python/mypy/pull/14535)) +* Support \_\_(r)divmod\_\_ dunders (Richard Si, PR [14613](https://github.com/python/mypy/pull/14613)) +* Support \_\_pow\_\_, \_\_rpow\_\_, and \_\_ipow\_\_ dunders (Richard Si, PR [14616](https://github.com/python/mypy/pull/14616)) +* Fix crash on star unpacking to underscore (Ivan Levkivskyi, PR [14624](https://github.com/python/mypy/pull/14624)) +* Fix iterating over a union of dicts (Richard Si, PR [14713](https://github.com/python/mypy/pull/14713)) + +### Fixes to Detecting Undefined Names (used-before-def) + +* Correctly handle walrus operator (Stas Ilinskiy, PR [14646](https://github.com/python/mypy/pull/14646)) +* Handle walrus declaration in match subject correctly (Stas Ilinskiy, PR [14665](https://github.com/python/mypy/pull/14665)) + +### Stubgen Improvements + +Stubgen is a tool for automatically generating draft stubs for libraries. + +* Allow aliases below the top level (Chad Dombrova, PR [14388](https://github.com/python/mypy/pull/14388)) +* Fix crash with PEP 604 union in type variable bound (Shantanu, PR [14557](https://github.com/python/mypy/pull/14557)) +* Preserve PEP 604 unions in generated .pyi files (hamdanal, PR [14601](https://github.com/python/mypy/pull/14601)) + +### Stubtest Improvements + +Stubtest is a tool for testing that stubs conform to the implementations. + +* Update message format so that it’s easier to go to error location (Avasam, PR [14437](https://github.com/python/mypy/pull/14437)) +* Handle name-mangling edge cases better (Alex Waygood, PR [14596](https://github.com/python/mypy/pull/14596)) + +### Changes to Error Reporting and Messages + +* Add new TypedDict error code typeddict-unknown-key (JoaquimEsteves, PR [14225](https://github.com/python/mypy/pull/14225)) +* Give arguments a more reasonable location in error messages (Max Murin, PR [14562](https://github.com/python/mypy/pull/14562)) +* In error messages, quote just the module's name (Ilya Konstantinov, PR [14567](https://github.com/python/mypy/pull/14567)) +* Improve misleading message about Enum() (Rodrigo Silva, PR [14590](https://github.com/python/mypy/pull/14590)) +* Suggest importing from `typing_extensions` if definition is not in typing (Shantanu, PR [14591](https://github.com/python/mypy/pull/14591)) +* Consistently use type-abstract error code (Ivan Levkivskyi, PR [14619](https://github.com/python/mypy/pull/14619)) +* Consistently use literal-required error code for TypedDicts (Ivan Levkivskyi, PR [14621](https://github.com/python/mypy/pull/14621)) +* Adjust inconsistent dataclasses plugin error messages (Wesley Collin Wright, PR [14637](https://github.com/python/mypy/pull/14637)) +* Consolidate literal bool argument error messages (Wesley Collin Wright, PR [14693](https://github.com/python/mypy/pull/14693)) + +### Other Fixes and Improvements + +* Check that type guards accept a positional argument (EXPLOSION, PR [14238](https://github.com/python/mypy/pull/14238)) +* Fix bug with in operator used with a union of Container and Iterable (Max Murin, PR [14384](https://github.com/python/mypy/pull/14384)) +* Support protocol inference for type\[T\] via metaclass (Ivan Levkivskyi, PR [14554](https://github.com/python/mypy/pull/14554)) +* Allow overlapping comparisons between bytes-like types (Shantanu, PR [14658](https://github.com/python/mypy/pull/14658)) +* Fix mypy daemon documentation link in README (Ivan Levkivskyi, PR [14644](https://github.com/python/mypy/pull/14644)) + +### Typeshed Updates + +Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=5ebf892d0710a6e87925b8d138dfa597e7bb11cc+0&branch=main&path=stdlib) for full list of typeshed changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Alex Waygood +* Avasam +* Chad Dombrova +* dosisod +* EXPLOSION +* hamdanal +* Ilya Konstantinov +* Ivan Levkivskyi +* Jared Hance +* JoaquimEsteves +* Jukka Lehtosalo +* Marc Mueller +* Max Murin +* Michael Lee +* Michael R. Crusoe +* Richard Si +* Rodrigo Silva +* Shantanu +* Stas Ilinskiy +* Wesley Collin Wright +* Yilei "Dolee" Yang +* Yurii Karabas + +We’d also like to thank our employer, Dropbox, for funding the mypy core team. + +Posted by Max Murin + + +## Mypy 1.0 + +[Monday, 6 February 2023](https://mypy-lang.blogspot.com/2023/02/mypy-10-released.html) + +We’ve just uploaded mypy 1.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### New Release Versioning Scheme + +Now that mypy reached 1.0, we’ll switch to a new versioning scheme. Mypy version numbers will be of form x.y.z. + +Rules: + +* The major release number (x) is incremented if a feature release includes a significant backward incompatible change that affects a significant fraction of users. +* The minor release number (y) is incremented on each feature release. Minor releases include updated stdlib stubs from typeshed. +* The point release number (z) is incremented when there are fixes only. + +Mypy doesn't use SemVer, since most minor releases have at least minor backward incompatible changes in typeshed, at the very least. Also, many type checking features find new legitimate issues in code. These are not considered backward incompatible changes, unless the number of new errors is very high. + +Any significant backward incompatible change must be announced in the blog post for the previous feature release, before making the change. The previous release must also provide a flag to explicitly enable or disable the new behavior (whenever practical), so that users will be able to prepare for the changes and report issues. We should keep the feature flag for at least a few releases after we've switched the default. + +See [”Release Process” in the mypy wiki](https://github.com/python/mypy/wiki/Release-Process) for more details and for the most up-to-date version of the versioning scheme. + +### Performance Improvements + +Mypy 1.0 is up to 40% faster than mypy 0.991 when type checking the Dropbox internal codebase. We also set up a daily job to measure the performance of the most recent development version of mypy to make it easier to track changes in performance. + +Many optimizations contributed to this improvement: + +* Improve performance for errors on class with many attributes (Shantanu, PR [14379](https://github.com/python/mypy/pull/14379)) +* Speed up make\_simplified\_union (Jukka Lehtosalo, PR [14370](https://github.com/python/mypy/pull/14370)) +* Micro-optimize get\_proper\_type(s) (Jukka Lehtosalo, PR [14369](https://github.com/python/mypy/pull/14369)) +* Micro-optimize flatten\_nested\_unions (Jukka Lehtosalo, PR [14368](https://github.com/python/mypy/pull/14368)) +* Some semantic analyzer micro-optimizations (Jukka Lehtosalo, PR [14367](https://github.com/python/mypy/pull/14367)) +* A few miscellaneous micro-optimizations (Jukka Lehtosalo, PR [14366](https://github.com/python/mypy/pull/14366)) +* Optimization: Avoid a few uses of contextmanagers in semantic analyzer (Jukka Lehtosalo, PR [14360](https://github.com/python/mypy/pull/14360)) +* Optimization: Enable always defined attributes in Type subclasses (Jukka Lehtosalo, PR [14356](https://github.com/python/mypy/pull/14356)) +* Optimization: Remove expensive context manager in type analyzer (Jukka Lehtosalo, PR [14357](https://github.com/python/mypy/pull/14357)) +* subtypes: fast path for Union/Union subtype check (Hugues, PR [14277](https://github.com/python/mypy/pull/14277)) +* Micro-optimization: avoid Bogus\[int\] types that cause needless boxing (Jukka Lehtosalo, PR [14354](https://github.com/python/mypy/pull/14354)) +* Avoid slow error message logic if errors not shown to user (Jukka Lehtosalo, PR [14336](https://github.com/python/mypy/pull/14336)) +* Speed up the implementation of hasattr() checks (Jukka Lehtosalo, PR [14333](https://github.com/python/mypy/pull/14333)) +* Avoid the use of a context manager in hot code path (Jukka Lehtosalo, PR [14331](https://github.com/python/mypy/pull/14331)) +* Change various type queries into faster bool type queries (Jukka Lehtosalo, PR [14330](https://github.com/python/mypy/pull/14330)) +* Speed up recursive type check (Jukka Lehtosalo, PR [14326](https://github.com/python/mypy/pull/14326)) +* Optimize subtype checking by avoiding a nested function (Jukka Lehtosalo, PR [14325](https://github.com/python/mypy/pull/14325)) +* Optimize type parameter checks in subtype checking (Jukka Lehtosalo, PR [14324](https://github.com/python/mypy/pull/14324)) +* Speed up freshening type variables (Jukka Lehtosalo, PR [14323](https://github.com/python/mypy/pull/14323)) +* Optimize implementation of TypedDict types for \*\*kwds (Jukka Lehtosalo, PR [14316](https://github.com/python/mypy/pull/14316)) + +### Warn About Variables Used Before Definition + +Mypy will now generate an error if you use a variable before it’s defined. This feature is enabled by default. By default mypy reports an error when it infers that a variable is always undefined. +```python +y = x # E: Name "x" is used before definition [used-before-def] +x = 0 +``` +This feature was contributed by Stas Ilinskiy. + +### Detect Possibly Undefined Variables (Experimental) + +A new experimental possibly-undefined error code is now available that will detect variables that may be undefined: +```python + if b: + x = 0 + print(x) # Error: Name "x" may be undefined [possibly-undefined] +``` +The error code is disabled be default, since it can generate false positives. + +This feature was contributed by Stas Ilinskiy. + +### Support the “Self” Type + +There is now a simpler syntax for declaring [generic self types](https://mypy.readthedocs.io/en/stable/generics.html#generic-methods-and-generic-self) introduced in [PEP 673](https://peps.python.org/pep-0673/): the Self type. You no longer have to define a type variable to use “self types”, and you can use them with attributes. Example from mypy documentation: +```python +from typing import Self + +class Friend: + other: Self | None = None + + @classmethod + def make_pair(cls) -> tuple[Self, Self]: + a, b = cls(), cls() + a.other = b + b.other = a + return a, b + +class SuperFriend(Friend): + pass + +# a and b have the inferred type "SuperFriend", not "Friend" +a, b = SuperFriend.make_pair() +``` +The feature was introduced in Python 3.11. In earlier Python versions a backport of Self is available in `typing_extensions`. + +This was contributed by Ivan Levkivskyi (PR [14041](https://github.com/python/mypy/pull/14041)). + +### Support ParamSpec in Type Aliases + +ParamSpec and Concatenate can now be used in type aliases. Example: +```python +from typing import ParamSpec, Callable + +P = ParamSpec("P") +A = Callable[P, None] + +def f(c: A[int, str]) -> None: + c(1, "x") +``` +This feature was contributed by Ivan Levkivskyi (PR [14159](https://github.com/python/mypy/pull/14159)). + +### ParamSpec and Generic Self Types No Longer Experimental + +Support for ParamSpec ([PEP 612](https://www.python.org/dev/peps/pep-0612/)) and generic self types are no longer considered experimental. + +### Miscellaneous New Features + +* Minimal, partial implementation of dataclass\_transform ([PEP 681](https://peps.python.org/pep-0681/)) (Wesley Collin Wright, PR [14523](https://github.com/python/mypy/pull/14523)) +* Add basic support for `typing_extensions`.TypeVar (Marc Mueller, PR [14313](https://github.com/python/mypy/pull/14313)) +* Add \--debug-serialize option (Marc Mueller, PR [14155](https://github.com/python/mypy/pull/14155)) +* Constant fold initializers of final variables (Jukka Lehtosalo, PR [14283](https://github.com/python/mypy/pull/14283)) +* Enable Final instance attributes for attrs (Tin Tvrtković, PR [14232](https://github.com/python/mypy/pull/14232)) +* Allow function arguments as base classes (Ivan Levkivskyi, PR [14135](https://github.com/python/mypy/pull/14135)) +* Allow super() with mixin protocols (Ivan Levkivskyi, PR [14082](https://github.com/python/mypy/pull/14082)) +* Add type inference for dict.keys membership (Matthew Hughes, PR [13372](https://github.com/python/mypy/pull/13372)) +* Generate error for class attribute access if attribute is defined with `__slots__` (Harrison McCarty, PR [14125](https://github.com/python/mypy/pull/14125)) +* Support additional attributes in callback protocols (Ivan Levkivskyi, PR [14084](https://github.com/python/mypy/pull/14084)) + +### Fixes to Crashes + +* Fix crash on prefixed ParamSpec with forward reference (Ivan Levkivskyi, PR [14569](https://github.com/python/mypy/pull/14569)) +* Fix internal crash when resolving the same partial type twice (Shantanu, PR [14552](https://github.com/python/mypy/pull/14552)) +* Fix crash in daemon mode on new import cycle (Ivan Levkivskyi, PR [14508](https://github.com/python/mypy/pull/14508)) +* Fix crash in mypy daemon (Ivan Levkivskyi, PR [14497](https://github.com/python/mypy/pull/14497)) +* Fix crash on Any metaclass in incremental mode (Ivan Levkivskyi, PR [14495](https://github.com/python/mypy/pull/14495)) +* Fix crash in await inside comprehension outside function (Ivan Levkivskyi, PR [14486](https://github.com/python/mypy/pull/14486)) +* Fix crash in Self type on forward reference in upper bound (Ivan Levkivskyi, PR [14206](https://github.com/python/mypy/pull/14206)) +* Fix a crash when incorrect super() is used outside a method (Ivan Levkivskyi, PR [14208](https://github.com/python/mypy/pull/14208)) +* Fix crash on overriding with frozen attrs (Ivan Levkivskyi, PR [14186](https://github.com/python/mypy/pull/14186)) +* Fix incremental mode crash on generic function appearing in nested position (Ivan Levkivskyi, PR [14148](https://github.com/python/mypy/pull/14148)) +* Fix daemon crash on malformed NamedTuple (Ivan Levkivskyi, PR [14119](https://github.com/python/mypy/pull/14119)) +* Fix crash during ParamSpec inference (Ivan Levkivskyi, PR [14118](https://github.com/python/mypy/pull/14118)) +* Fix crash on nested generic callable (Ivan Levkivskyi, PR [14093](https://github.com/python/mypy/pull/14093)) +* Fix crashes with unpacking SyntaxError (Shantanu, PR [11499](https://github.com/python/mypy/pull/11499)) +* Fix crash on partial type inference within a lambda (Ivan Levkivskyi, PR [14087](https://github.com/python/mypy/pull/14087)) +* Fix crash with enums (Michael Lee, PR [14021](https://github.com/python/mypy/pull/14021)) +* Fix crash with malformed TypedDicts and disllow-any-expr (Michael Lee, PR [13963](https://github.com/python/mypy/pull/13963)) + +### Error Reporting Improvements + +* More helpful error for missing self (Shantanu, PR [14386](https://github.com/python/mypy/pull/14386)) +* Add error-code truthy-iterable (Marc Mueller, PR [13762](https://github.com/python/mypy/pull/13762)) +* Fix pluralization in error messages (KotlinIsland, PR [14411](https://github.com/python/mypy/pull/14411)) + +### Mypyc: Support Match Statement + +Mypyc can now compile Python 3.10 match statements. + +This was contributed by dosisod (PR [13953](https://github.com/python/mypy/pull/13953)). + +### Other Mypyc Fixes and Improvements + +* Optimize int(x)/float(x)/complex(x) on instances of native classes (Richard Si, PR [14450](https://github.com/python/mypy/pull/14450)) +* Always emit warnings (Richard Si, PR [14451](https://github.com/python/mypy/pull/14451)) +* Faster bool and integer conversions (Jukka Lehtosalo, PR [14422](https://github.com/python/mypy/pull/14422)) +* Support attributes that override properties (Jukka Lehtosalo, PR [14377](https://github.com/python/mypy/pull/14377)) +* Precompute set literals for "in" operations and iteration (Richard Si, PR [14409](https://github.com/python/mypy/pull/14409)) +* Don't load targets with forward references while setting up non-extension class `__all__` (Richard Si, PR [14401](https://github.com/python/mypy/pull/14401)) +* Compile away NewType type calls (Richard Si, PR [14398](https://github.com/python/mypy/pull/14398)) +* Improve error message for multiple inheritance (Joshua Bronson, PR [14344](https://github.com/python/mypy/pull/14344)) +* Simplify union types (Jukka Lehtosalo, PR [14363](https://github.com/python/mypy/pull/14363)) +* Fixes to union simplification (Jukka Lehtosalo, PR [14364](https://github.com/python/mypy/pull/14364)) +* Fix for typeshed changes to Collection (Shantanu, PR [13994](https://github.com/python/mypy/pull/13994)) +* Allow use of enum.Enum (Shantanu, PR [13995](https://github.com/python/mypy/pull/13995)) +* Fix compiling on Arch Linux (dosisod, PR [13978](https://github.com/python/mypy/pull/13978)) + +### Documentation Improvements + +* Various documentation and error message tweaks (Jukka Lehtosalo, PR [14574](https://github.com/python/mypy/pull/14574)) +* Improve Generics documentation (Shantanu, PR [14587](https://github.com/python/mypy/pull/14587)) +* Improve protocols documentation (Shantanu, PR [14577](https://github.com/python/mypy/pull/14577)) +* Improve dynamic typing documentation (Shantanu, PR [14576](https://github.com/python/mypy/pull/14576)) +* Improve the Common Issues page (Shantanu, PR [14581](https://github.com/python/mypy/pull/14581)) +* Add a top-level TypedDict page (Shantanu, PR [14584](https://github.com/python/mypy/pull/14584)) +* More improvements to getting started documentation (Shantanu, PR [14572](https://github.com/python/mypy/pull/14572)) +* Move truthy-function documentation from “optional checks” to “enabled by default” (Anders Kaseorg, PR [14380](https://github.com/python/mypy/pull/14380)) +* Avoid use of implicit optional in decorator factory documentation (Tom Schraitle, PR [14156](https://github.com/python/mypy/pull/14156)) +* Clarify documentation surrounding install-types (Shantanu, PR [14003](https://github.com/python/mypy/pull/14003)) +* Improve searchability for module level type ignore errors (Shantanu, PR [14342](https://github.com/python/mypy/pull/14342)) +* Advertise mypy daemon in README (Ivan Levkivskyi, PR [14248](https://github.com/python/mypy/pull/14248)) +* Add link to error codes in README (Ivan Levkivskyi, PR [14249](https://github.com/python/mypy/pull/14249)) +* Document that report generation disables cache (Ilya Konstantinov, PR [14402](https://github.com/python/mypy/pull/14402)) +* Stop saying mypy is beta software (Ivan Levkivskyi, PR [14251](https://github.com/python/mypy/pull/14251)) +* Flycheck-mypy is deprecated, since its functionality was merged to Flycheck (Ivan Levkivskyi, PR [14247](https://github.com/python/mypy/pull/14247)) +* Update code example in "Declaring decorators" (ChristianWitzler, PR [14131](https://github.com/python/mypy/pull/14131)) + +### Stubtest Improvements + +Stubtest is a tool for testing that stubs conform to the implementations. + +* Improve error message for `__all__`\-related errors (Alex Waygood, PR [14362](https://github.com/python/mypy/pull/14362)) +* Improve heuristics for determining whether global-namespace names are imported (Alex Waygood, PR [14270](https://github.com/python/mypy/pull/14270)) +* Catch BaseException on module imports (Shantanu, PR [14284](https://github.com/python/mypy/pull/14284)) +* Associate exported symbol error with `__all__` object\_path (Nikita Sobolev, PR [14217](https://github.com/python/mypy/pull/14217)) +* Add \_\_warningregistry\_\_ to the list of ignored module dunders (Nikita Sobolev, PR [14218](https://github.com/python/mypy/pull/14218)) +* If a default is present in the stub, check that it is correct (Jelle Zijlstra, PR [14085](https://github.com/python/mypy/pull/14085)) + +### Stubgen Improvements + +Stubgen is a tool for automatically generating draft stubs for libraries. + +* Treat dlls as C modules (Shantanu, PR [14503](https://github.com/python/mypy/pull/14503)) + +### Other Notable Fixes and Improvements + +* Update stub suggestions based on recent typeshed changes (Alex Waygood, PR [14265](https://github.com/python/mypy/pull/14265)) +* Fix attrs protocol check with cache (Marc Mueller, PR [14558](https://github.com/python/mypy/pull/14558)) +* Fix strict equality check if operand item type has custom \_\_eq\_\_ (Jukka Lehtosalo, PR [14513](https://github.com/python/mypy/pull/14513)) +* Don't consider object always truthy (Jukka Lehtosalo, PR [14510](https://github.com/python/mypy/pull/14510)) +* Properly support union of TypedDicts as dict literal context (Ivan Levkivskyi, PR [14505](https://github.com/python/mypy/pull/14505)) +* Properly expand type in generic class with Self and TypeVar with values (Ivan Levkivskyi, PR [14491](https://github.com/python/mypy/pull/14491)) +* Fix recursive TypedDicts/NamedTuples defined with call syntax (Ivan Levkivskyi, PR [14488](https://github.com/python/mypy/pull/14488)) +* Fix type inference issue when a class inherits from Any (Shantanu, PR [14404](https://github.com/python/mypy/pull/14404)) +* Fix false positive on generic base class with six (Ivan Levkivskyi, PR [14478](https://github.com/python/mypy/pull/14478)) +* Don't read scripts without extensions as modules in namespace mode (Tim Geypens, PR [14335](https://github.com/python/mypy/pull/14335)) +* Fix inference for constrained type variables within unions (Christoph Tyralla, PR [14396](https://github.com/python/mypy/pull/14396)) +* Fix Unpack imported from typing (Marc Mueller, PR [14378](https://github.com/python/mypy/pull/14378)) +* Allow trailing commas in ini configuration of multiline values (Nikita Sobolev, PR [14240](https://github.com/python/mypy/pull/14240)) +* Fix false negatives involving Unions and generators or coroutines (Shantanu, PR [14224](https://github.com/python/mypy/pull/14224)) +* Fix ParamSpec constraint for types as callable (Vincent Vanlaer, PR [14153](https://github.com/python/mypy/pull/14153)) +* Fix type aliases with fixed-length tuples (Jukka Lehtosalo, PR [14184](https://github.com/python/mypy/pull/14184)) +* Fix issues with type aliases and new style unions (Jukka Lehtosalo, PR [14181](https://github.com/python/mypy/pull/14181)) +* Simplify unions less aggressively (Ivan Levkivskyi, PR [14178](https://github.com/python/mypy/pull/14178)) +* Simplify callable overlap logic (Ivan Levkivskyi, PR [14174](https://github.com/python/mypy/pull/14174)) +* Try empty context when assigning to union typed variables (Ivan Levkivskyi, PR [14151](https://github.com/python/mypy/pull/14151)) +* Improvements to recursive types (Ivan Levkivskyi, PR [14147](https://github.com/python/mypy/pull/14147)) +* Make non-numeric non-empty FORCE\_COLOR truthy (Shantanu, PR [14140](https://github.com/python/mypy/pull/14140)) +* Fix to recursive type aliases (Ivan Levkivskyi, PR [14136](https://github.com/python/mypy/pull/14136)) +* Correctly handle Enum name on Python 3.11 (Ivan Levkivskyi, PR [14133](https://github.com/python/mypy/pull/14133)) +* Fix class objects falling back to metaclass for callback protocol (Ivan Levkivskyi, PR [14121](https://github.com/python/mypy/pull/14121)) +* Correctly support self types in callable ClassVar (Ivan Levkivskyi, PR [14115](https://github.com/python/mypy/pull/14115)) +* Fix type variable clash in nested positions and in attributes (Ivan Levkivskyi, PR [14095](https://github.com/python/mypy/pull/14095)) +* Allow class variable as implementation for read only attribute (Ivan Levkivskyi, PR [14081](https://github.com/python/mypy/pull/14081)) +* Prevent warnings from causing dmypy to fail (Andrzej Bartosiński, PR [14102](https://github.com/python/mypy/pull/14102)) +* Correctly process nested definitions in mypy daemon (Ivan Levkivskyi, PR [14104](https://github.com/python/mypy/pull/14104)) +* Don't consider a branch unreachable if there is a possible promotion (Ivan Levkivskyi, PR [14077](https://github.com/python/mypy/pull/14077)) +* Fix incompatible overrides of overloaded methods in concrete subclasses (Shantanu, PR [14017](https://github.com/python/mypy/pull/14017)) +* Fix new style union syntax in type aliases (Jukka Lehtosalo, PR [14008](https://github.com/python/mypy/pull/14008)) +* Fix and optimise overload compatibility checking (Shantanu, PR [14018](https://github.com/python/mypy/pull/14018)) +* Improve handling of redefinitions through imports (Shantanu, PR [13969](https://github.com/python/mypy/pull/13969)) +* Preserve (some) implicitly exported types (Shantanu, PR [13967](https://github.com/python/mypy/pull/13967)) + +### Typeshed Updates + +Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=ea0ae2155e8a04c9837903c3aff8dd5ad5f36ebc+0&branch=main&path=stdlib) for full list of typeshed changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Alessio Izzo +* Alex Waygood +* Anders Kaseorg +* Andrzej Bartosiński +* Avasam +* ChristianWitzler +* Christoph Tyralla +* dosisod +* Harrison McCarty +* Hugo van Kemenade +* Hugues +* Ilya Konstantinov +* Ivan Levkivskyi +* Jelle Zijlstra +* jhance +* johnthagen +* Jonathan Daniel +* Joshua Bronson +* Jukka Lehtosalo +* KotlinIsland +* Lakshay Bisht +* Lefteris Karapetsas +* Marc Mueller +* Matthew Hughes +* Michael Lee +* Nick Drozd +* Nikita Sobolev +* Richard Si +* Shantanu +* Stas Ilinskiy +* Tim Geypens +* Tin Tvrtković +* Tom Schraitle +* Valentin Stanciu +* Vincent Vanlaer + +We’d also like to thank our employer, Dropbox, for funding the mypy core team. + +Posted by Stas Ilinskiy + +## Previous releases + +For information about previous releases, refer to the posts at https://mypy-lang.blogspot.com/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000000..24f7e516e9e2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,208 @@ +# Contributing to Mypy + +Welcome! Mypy is a community project that aims to work for a wide +range of Python users and Python codebases. If you're trying mypy on +your Python code, your experience and what you can contribute are +important to the project's success. + +## Code of Conduct + +Everyone participating in the Mypy community, and in particular in our +issue tracker, pull requests, and chat, is expected to treat +other people with respect and more generally to follow the guidelines +articulated in the [Python Community Code of Conduct](https://www.python.org/psf/codeofconduct/). + +## Getting started with development + +### Setup + +#### (1) Fork the mypy repository + +Within Github, navigate to and fork the repository. + +#### (2) Clone the mypy repository and enter into it + +```bash +git clone git@github.com:/mypy.git +cd mypy +``` + +#### (3) Create then activate a virtual environment + +```bash +python3 -m venv venv +source venv/bin/activate +``` + +```bash +# For Windows use +python -m venv venv +. venv/Scripts/activate + +# For more details, see https://docs.python.org/3/library/venv.html#creating-virtual-environments +``` + +#### (4) Install the test requirements and the project + +```bash +python -m pip install -r test-requirements.txt +python -m pip install -e . +hash -r # This resets shell PATH cache, not necessary on Windows +``` + +> **Note** +> You'll need Python 3.8 or higher to install all requirements listed in +> test-requirements.txt + +### Running tests + +Running the full test suite can take a while, and usually isn't necessary when +preparing a PR. Once you file a PR, the full test suite will run on GitHub. +You'll then be able to see any test failures, and make any necessary changes to +your PR. + +However, if you wish to do so, you can run the full test suite +like this: + +```bash +python runtests.py +``` + +Some useful commands for running specific tests include: + +```bash +# Use mypy to check mypy's own code +python runtests.py self +# or equivalently: +python -m mypy --config-file mypy_self_check.ini -p mypy + +# Run a single test from the test suite +pytest -n0 -k 'test_name' + +# Run all test cases in the "test-data/unit/check-dataclasses.test" file +pytest mypy/test/testcheck.py::TypeCheckSuite::check-dataclasses.test + +# Run the formatters and linters +python runtests.py lint +``` + +For an in-depth guide on running and writing tests, +see [the README in the test-data directory](test-data/unit/README.md). + +#### Using `tox` + +You can also use [`tox`](https://tox.wiki/en/latest/) to run tests and other commands. +`tox` handles setting up test environments for you. + +```bash +# Run tests +tox run -e py + +# Run tests using some specific Python version +tox run -e py311 + +# Run a specific command +tox run -e lint + +# Run a single test from the test suite +tox run -e py -- -n0 -k 'test_name' + +# Run all test cases in the "test-data/unit/check-dataclasses.test" file using +# Python 3.11 specifically +tox run -e py311 -- mypy/test/testcheck.py::TypeCheckSuite::check-dataclasses.test + +# Set up a development environment with all the project libraries and run a command +tox -e dev -- mypy --verbose test_case.py +tox -e dev --override testenv:dev.allowlist_externals+=env -- env # inspect the environment +``` + +If you don't already have `tox` installed, you can use a virtual environment as +described above to install `tox` via `pip` (e.g., ``python -m pip install tox``). + +## First time contributors + +If you're looking for things to help with, browse our [issue tracker](https://github.com/python/mypy/issues)! + +In particular, look for: + +- [good first issues](https://github.com/python/mypy/labels/good-first-issue) +- [good second issues](https://github.com/python/mypy/labels/good-second-issue) +- [documentation issues](https://github.com/python/mypy/labels/documentation) + +You do not need to ask for permission to work on any of these issues. +Just fix the issue yourself, [try to add a unit test](#running-tests) and +[open a pull request](#submitting-changes). + +To get help fixing a specific issue, it's often best to comment on the issue +itself. You're much more likely to get help if you provide details about what +you've tried and where you've looked (maintainers tend to help those who help +themselves). [gitter](https://gitter.im/python/typing) can also be a good place +to ask for help. + +Interactive debuggers like `pdb` and `ipdb` are really useful for getting +started with the mypy codebase. This is a +[useful tutorial](https://realpython.com/python-debugging-pdb/). + +It's also extremely easy to get started contributing to our sister project +[typeshed](https://github.com/python/typeshed/issues) that provides type stubs +for libraries. This is a great way to become familiar with type syntax. + +## Submitting changes + +Even more excellent than a good bug report is a fix for a bug, or the +implementation of a much-needed new feature. We'd love to have +your contributions. + +We use the usual GitHub pull-request flow, which may be familiar to +you if you've contributed to other projects on GitHub. For the mechanics, +see [our git and GitHub workflow help page](https://github.com/python/mypy/wiki/Using-Git-And-GitHub), +or [GitHub's own documentation](https://help.github.com/articles/using-pull-requests/). + +Anyone interested in Mypy may review your code. One of the Mypy core +developers will merge your pull request when they think it's ready. + +If your change will be a significant amount of work +to write, we highly recommend starting by opening an issue laying out +what you want to do. That lets a conversation happen early in case +other contributors disagree with what you'd like to do or have ideas +that will help you do it. + +The best pull requests are focused, clearly describe what they're for +and why they're correct, and contain tests for whatever changes they +make to the code's behavior. As a bonus these are easiest for someone +to review, which helps your pull request get merged quickly! Standard +advice about good pull requests for open-source projects applies; we +have [our own writeup](https://github.com/python/mypy/wiki/Good-Pull-Request) +of this advice. + +Also, do not squash your commits after you have submitted a pull request, as this +erases context during review. We will squash commits when the pull request is merged. + +You may also find other pages in the +[Mypy developer guide](https://github.com/python/mypy/wiki/Developer-Guides) +helpful in developing your change. + +## Core developer guidelines + +Core developers should follow these rules when processing pull requests: + +- Always wait for tests to pass before merging PRs. +- Use "[Squash and merge](https://github.com/blog/2141-squash-your-commits)" + to merge PRs. +- Delete branches for merged PRs (by core devs pushing to the main repo). +- Edit the final commit message before merging to conform to the following + style (we wish to have a clean `git log` output): + - When merging a multi-commit PR make sure that the commit message doesn't + contain the local history from the committer and the review history from + the PR. Edit the message to only describe the end state of the PR. + - Make sure there is a *single* newline at the end of the commit message. + This way there is a single empty line between commits in `git log` + output. + - Split lines as needed so that the maximum line length of the commit + message is under 80 characters, including the subject line. + - Capitalize the subject and each paragraph. + - Make sure that the subject of the commit message has no trailing dot. + - Use the imperative mood in the subject line (e.g. "Fix typo in README"). + - If the PR fixes an issue, make sure something like "Fixes #xxx." occurs + in the body of the message (not in the subject). + - Use Markdown for formatting. diff --git a/CREDITS b/CREDITS index ebfc62ae15f0..fb2fe155a9b8 100644 --- a/CREDITS +++ b/CREDITS @@ -1,34 +1,41 @@ Credits ------- -Lead developer: +For a full list of contributors you can mine the commit history: +https://github.com/python/mypy/commits/master + +For lists of contributors per mypy release (including typeshed) see +the release blog posts at https://mypy-lang.blogspot.com/. + +Dropbox core team: Jukka Lehtosalo + Ivan Levkivskyi + Jared Hance + +Non-Dropbox core team members: + + Ethan Smith + Guido van Rossum + Jelle Zijlstra + Michael J. Sullivan + Shantanu Jain + Xuanda Yang + Jingchen Ye <97littleleaf11@gmail.com> + Nikita Sobolev + +Past Dropbox core team members: -Contributors (in alphabetical order): - - Steven Allen (@Stebalien) - Reid Barton (@rwbarton) - Miguel Garcia (@rockneurotiko) - Ryan Gonzalez (@kirbyfan64) - Ashley Hewson (@ashleyh) - Bob Ippolito (@etrepum) - Sander Kersten (@spkersten) - Ian Kronquist (@iankronquist) - Florian Ludwig (@FlorianLudwig) - Jared Pochtar (@jaredp) - Eric Price (@ecprice) - Ron Murawski - Sebastian Riikonen - Schuyler Smith - Marcell Vazquez-Chanlatte (@mvcisback) - Igor Vuk (@ivuk) - Jeff Walden (@jswalden) - Yuanchao Zhu (@yczhu) - Gennadiy Zlobin (@gennad) + David Fisher + Svyatoslav Ilinskiy + Greg Price + Naomi Seyfer + Michael Lee + Reid Barton Additional thanks to: + Alex Allain Max Bolingbroke Peter Calvert Kannan Goundan @@ -37,6 +44,7 @@ Additional thanks to: Riitta Ikonen Terho Ikonen Stephen Kell + Łukasz Langa Laura Lehtosalo Peter Ludemann Seppo Mattila @@ -44,9 +52,10 @@ Additional thanks to: Alan Mycroft Dominic Orchard Pekka Rapinoja - Guido van Rossum + Matt Robben Satnam Singh Juha Sorva Clay Sweetser Jorma Tarhio Jussi Tuovila + Andrey Vlasovskikh diff --git a/LICENSE b/LICENSE index 208ecb46a4fa..55d01ee19ad8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,10 +1,11 @@ -Mypy is licensed under the terms of the MIT license, reproduced below. +Mypy (and mypyc) are licensed under the terms of the MIT license, reproduced below. = = = = = The MIT License -Copyright (c) 2012 Jukka Lehtosalo and contributors +Copyright (c) 2012-2023 Jukka Lehtosalo and contributors +Copyright (c) 2015-2023 Dropbox, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), @@ -26,8 +27,11 @@ DEALINGS IN THE SOFTWARE. = = = = = -Portions of mypy are licensed under different licenses. The files under -python-lib are licensed under the PSF 2 License, reproduced below. +Portions of mypy and mypyc are licensed under different licenses. +The files +mypyc/lib-rt/pythonsupport.h, mypyc/lib-rt/getargs.c and +mypyc/lib-rt/getargsfast.c are licensed under the PSF 2 License, reproduced +below. = = = = = @@ -223,5 +227,3 @@ FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -= = = = = \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000000..c2399d2b00b6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,49 @@ +# some of the prunes here are so that check-manifest doesn't complain about their exclusion +# as such, be judicious in your use of prune + +# stubs +prune mypy/typeshed +include mypy/typeshed/LICENSE +include mypy/typeshed/stdlib/VERSIONS +recursive-include mypy/typeshed *.pyi + +# mypy and mypyc +include mypy/py.typed +recursive-include mypy *.py +recursive-include mypyc *.py + +# random +include mypy_bootstrap.ini +graft mypy/xml +graft scripts + +# docs +graft docs +prune docs/build +prune docs/source/_build + +# assorted mypyc requirements +graft mypyc/external +graft mypyc/lib-rt +graft mypyc/test-data +graft mypyc/doc + +# files necessary for testing sdist +include mypy-requirements.txt +include build-requirements.txt +include test-requirements.in +include test-requirements.txt +include mypy_self_check.ini +prune misc +graft test-data +include conftest.py +include runtests.py +include pytest.ini +include tox.ini + +include LICENSE mypyc/README.md CHANGELOG.md +exclude .gitmodules CONTRIBUTING.md CREDITS ROADMAP.md action.yml .editorconfig +exclude .git-blame-ignore-revs .pre-commit-config.yaml + +global-exclude *.py[cod] +global-exclude .DS_Store diff --git a/README.md b/README.md index 074389168fc6..07c170d46cb3 100644 --- a/README.md +++ b/README.md @@ -1,167 +1,193 @@ -Mypy: Optional Static Typing for Python +mypy logo + +Mypy: Static Typing for Python ======================================= -[![Build Status](https://travis-ci.org/JukkaL/mypy.svg)](https://travis-ci.org/JukkaL/mypy) +[![Stable Version](https://img.shields.io/pypi/v/mypy?color=blue)](https://pypi.org/project/mypy/) +[![Downloads](https://img.shields.io/pypi/dm/mypy)](https://pypistats.org/packages/mypy) +[![Build Status](https://github.com/python/mypy/actions/workflows/test.yml/badge.svg)](https://github.com/python/mypy/actions) +[![Documentation Status](https://readthedocs.org/projects/mypy/badge/?version=latest)](https://mypy.readthedocs.io/en/latest/?badge=latest) +[![Chat at https://gitter.im/python/typing](https://badges.gitter.im/python/typing.svg)](https://gitter.im/python/typing?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Linting: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +Got a question? +--------------- -What is mypy? -------------- +We are always happy to answer questions! Here are some good places to ask them: -Mypy is an optional static type checker for Python. You can add type -annotations to your Python programs and use mypy to type check them -statically to find errors before running them. You can also -seamlessly mix dynamic and static typing in your programs, so you can -always fall back to dynamic typing. Mypy programs are valid Python -3.x and you use a normal Python interpreter to run them. There is -essentially no performance overhead when using mypy, since mypy does -not introduce additional runtime type checking. +- for anything you're curious about, try [gitter chat](https://gitter.im/python/typing) +- for general questions about Python typing, try [typing discussions](https://github.com/python/typing/discussions) -Here is a small example to whet your appetite: +If you're just getting started, +[the documentation](https://mypy.readthedocs.io/en/stable/index.html) +and [type hints cheat sheet](https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html) +can also help answer questions. -```python -from typing import Iterator +If you think you've found a bug: -def fib(n: int) -> Iterator[int]: - a, b = 0, 1 - while a < n: - yield a - a, b = b, a + b -``` +- check our [common issues page](https://mypy.readthedocs.io/en/stable/common_issues.html) +- search our [issue tracker](https://github.com/python/mypy/issues) to see if + it's already been reported +- consider asking on [gitter chat](https://gitter.im/python/typing) -Mypy is in development; some features are missing and there are bugs. -See 'Development status' below. +To report a bug or request an enhancement: +- report at [our issue tracker](https://github.com/python/mypy/issues) +- if the issue is with a specific library or function, consider reporting it at + [typeshed tracker](https://github.com/python/typeshed/issues) or the issue + tracker for that library -Requirements ------------- +To discuss a new type system feature: -You need Python 3.2 or later to run mypy. You can have multiple Python -versions (2.x and 3.x) installed on the same system without problems. +- discuss at [discuss.python.org](https://discuss.python.org/c/typing/32) +- there is also some historical discussion at the [typing-sig mailing list](https://mail.python.org/archives/list/typing-sig@python.org/) and the [python/typing repo](https://github.com/python/typing/issues) + +What is mypy? +------------- -In Ubuntu, Mint and Debian you can install Python 3 like this: +Mypy is a static type checker for Python. - $ sudo apt-get install python3 +Type checkers help ensure that you're using variables and functions in your code +correctly. With mypy, add type hints ([PEP 484](https://www.python.org/dev/peps/pep-0484/)) +to your Python programs, and mypy will warn you when you use those types +incorrectly. -For other Linux flavors, OS X and Windows, packages are available at +Python is a dynamic language, so usually you'll only see errors in your code +when you attempt to run it. Mypy is a *static* checker, so it finds bugs +in your programs without even running them! - http://www.python.org/getit/ +Here is a small example to whet your appetite: +```python +number = input("What is your favourite number?") +print("It is", number + 1) # error: Unsupported operand types for + ("str" and "int") +``` -Quick start ------------ +Adding type hints for mypy does not interfere with the way your program would +otherwise run. Think of type hints as similar to comments! You can always use +the Python interpreter to run your code, even if mypy reports errors. -Mypy can be installed from pip: +Mypy is designed with gradual typing in mind. This means you can add type +hints to your code base slowly and that you can always fall back to dynamic +typing when static typing is not convenient. - $ pip install mypy-lang +Mypy has a powerful and easy-to-use type system, supporting features such as +type inference, generics, callable types, tuple types, union types, +structural subtyping and more. Using mypy will make your programs easier to +understand, debug, and maintain. -If you want to contribute and have git, -first clone the mypy git repository: +See [the documentation](https://mypy.readthedocs.io/en/stable/index.html) for +more examples and information. - $ git clone https://github.com/JukkaL/mypy.git +In particular, see: -Alternatively, you can download the latest development version as a -zip archive from this URL: +- [type hints cheat sheet](https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html) +- [getting started](https://mypy.readthedocs.io/en/stable/getting_started.html) +- [list of error codes](https://mypy.readthedocs.io/en/stable/error_code_list.html) - https://github.com/JukkaL/mypy/archive/master.zip +Quick start +----------- -Run the supplied setup.py script to install mypy: +Mypy can be installed using pip: - $ python3 setup.py install +```bash +python3 -m pip install -U mypy +``` -Replace 'python3' with your Python 3 interpreter. You may have to do -the above as root. For example, in Ubuntu and Mac OS X: +If you want to run the latest version of the code, you can install from the +repo directly: - $ sudo python3 setup.py install +```bash +python3 -m pip install -U git+https://github.com/python/mypy.git +# or if you don't have 'git' installed +python3 -m pip install -U https://github.com/python/mypy/zipball/master +``` -This installs the 'mypy' script and dependencies, including the -'typing' module, to system-dependent locations. Sometimes the script -directory will not be in PATH, and you have to add the target -directory to PATH manually or create a symbolic link to the script. -In particular, on Mac OS X, the script may be installed under -/Library/Frameworks: +Now you can type-check the [statically typed parts] of a program like this: - /Library/Frameworks/Python.framework/Versions//bin +```bash +mypy PROGRAM +``` -Now, on a Unix-like system, you can type check a program like this: +You can always use the Python interpreter to run your statically typed +programs, even if mypy reports type errors: - $ mypy PROGRAM +```bash +python3 PROGRAM +``` -In Windows, the script is generally installed in -\PythonNN\Scripts. So, type check a program like this (replace -\Python33 with your Python installation path): +You can also try mypy in an [online playground](https://mypy-play.net/) (developed by +Yusuke Miyazaki). If you are working with large code bases, you can run mypy in +[daemon mode], that will give much faster (often sub-second) incremental updates: - C:\>\Python33\python \Python33\Scripts\mypy PROGRAM +```bash +dmypy run -- PROGRAM +``` -You can always use a Python interpreter to run your statically typed -programs, even if they have type errors: +[statically typed parts]: https://mypy.readthedocs.io/en/latest/getting_started.html#function-signatures-and-dynamic-vs-static-typing +[daemon mode]: https://mypy.readthedocs.io/en/stable/mypy_daemon.html - $ python3 PROGRAM +Integrations +------------ +Mypy can be integrated into popular IDEs: + +- Vim: + - Using [Syntastic](https://github.com/vim-syntastic/syntastic): in `~/.vimrc` add + `let g:syntastic_python_checkers=['mypy']` + - Using [ALE](https://github.com/dense-analysis/ale): should be enabled by default when `mypy` is installed, + or can be explicitly enabled by adding `let b:ale_linters = ['mypy']` in `~/vim/ftplugin/python.vim` +- Emacs: using [Flycheck](https://github.com/flycheck/) +- Sublime Text: [SublimeLinter-contrib-mypy](https://github.com/fredcallaway/SublimeLinter-contrib-mypy) +- Atom: [linter-mypy](https://atom.io/packages/linter-mypy) +- PyCharm: [mypy plugin](https://github.com/dropbox/mypy-PyCharm-plugin) (PyCharm integrates + [its own implementation](https://www.jetbrains.com/help/pycharm/type-hinting-in-product.html) of [PEP 484](https://peps.python.org/pep-0484/)) +- VS Code: provides [basic integration](https://code.visualstudio.com/docs/python/linting#_mypy) with mypy. +- pre-commit: use [pre-commit mirrors-mypy](https://github.com/pre-commit/mirrors-mypy). Web site and documentation -------------------------- -Documentation and additional information is available at the web site: - - http://www.mypy-lang.org/ - - -Running tests -------------- - -To run tests, run the script 'tests.py' in the mypy repository: - - $ python3 tests.py - - -Development status ------------------- - -Mypy is work in progress and is not yet production quality (though -mypy development is already done in mypy!). +Additional information is available at the web site: -Here are some of the more significant Python features not supported -right now (but all of these will improve): + - - Python 2.x support not really yet usable - - properties with setters not supported - - somewhat limited operator overloading - - only a subset of Python standard library modules are supported, and some - only partially - - limited metaclass support +Jump straight to the documentation: -Some mypy-specific features are also not supported or only partially -supported, including these: + - - function overloading does not work properly in all cases, including - some instances of method overriding, and keyword arguments - - no 'Dynamic' classes - - there is no way to use dynamic typing by default for top-level code +Follow along our changelog at: -The current development focus is to support static type checking with a good -subset of Python features (both 2.x and 3.x). + +Contributing +------------ -Issue tracker -------------- +Help in testing, development, documentation and other tasks is +highly appreciated and useful to the project. There are tasks for +contributors of all experience levels. -Please report any bugs and enhancement ideas using the mypy issue -tracker: +To get started with developing mypy, see [CONTRIBUTING.md](CONTRIBUTING.md). - https://github.com/JukkaL/mypy/issues +If you need help getting started, don't hesitate to ask on [gitter](https://gitter.im/python/typing). +Mypyc and compiled version of mypy +---------------------------------- -Help wanted ------------ +[Mypyc](https://github.com/mypyc/mypyc) uses Python type hints to compile Python +modules to faster C extensions. Mypy is itself compiled using mypyc: this makes +mypy approximately 4 times faster than if interpreted! -Any help in testing, development, documentation and other tasks is -highly appreciated and useful to the project. Contact the developers -to join the project, or just start coding and send pull requests! -There are tasks for contributors of all skill levels. +To install an interpreted mypy instead, use: +```bash +python3 -m pip install --no-binary mypy -U mypy +``` -License -------- +To use a compiled version of a development +version of mypy, directly install a binary from +. -Mypy is licensed under the terms of the MIT License (see the file -LICENSE). +To contribute to the mypyc project, check out diff --git a/action.yml b/action.yml new file mode 100644 index 000000000000..df8715327830 --- /dev/null +++ b/action.yml @@ -0,0 +1,83 @@ +name: "Mypy" +description: "Optional Static Typing for Python." +author: "Jukka Lehtosalo and contributors" +inputs: + options: + description: > + Options passed to mypy. Use `mypy --help` to see available options. + required: false + paths: + description: > + Explicit paths to run mypy on. Defaults to the current directory. + required: false + default: "." + version: + description: > + Mypy version to use (PEP440) - e.g. "0.910" + required: false + default: "" + install_types: + description: > + Whether to automatically install missing library stub packages. + ('yes'|'no', default: 'yes') + default: "yes" + install_project_dependencies: + description: > + Whether to attempt to install project dependencies into mypy + environment. ('yes'|'no', default: 'yes') + default: "yes" +branding: + color: "blue" + icon: "check-circle" +runs: + using: composite + steps: + - name: mypy setup + shell: bash + run: | + echo ::group::Installing mypy... + export PIP_DISABLE_PIP_VERSION_CHECK=1 + + if [ "$RUNNER_OS" == "Windows" ]; then + HOST_PYTHON=python + else + HOST_PYTHON=python3 + fi + + venv_script="import os.path; import venv; import sys; + path = os.path.join(r'${{ github.action_path }}', '.mypy-venv'); + venv.main([path]); + bin_subdir = 'Scripts' if sys.platform == 'win32' else 'bin'; + print(os.path.join(path, bin_subdir, 'python')); + " + + VENV_PYTHON=$(echo $venv_script | "$HOST_PYTHON") + mypy_spec="mypy" + + if [ -n "${{ inputs.version }}" ]; then + mypy_spec+="==${{ inputs.version }}" + fi + + if ! "$VENV_PYTHON" -m pip install "$mypy_spec"; then + echo "::error::Could not install mypy." + exit 1 + fi + echo ::endgroup:: + + if [ "${{ inputs.install_project_dependencies }}" == "yes" ]; then + VENV=$("$VENV_PYTHON" -c 'import sys;print(sys.prefix)') + echo ::group::Installing project dependencies... + "$VENV_PYTHON" -m pip download --dest="$VENV"/deps . + "$VENV_PYTHON" -m pip install -U --find-links="$VENV"/deps "$VENV"/deps/* + echo ::endgroup:: + fi + + echo ::group::Running mypy... + mypy_opts="" + if [ "${{ inputs.install_types }}" == "yes" ]; then + mypy_opts+="--install-types --non-interactive" + fi + + echo "mypy $mypy_opts ${{ inputs.options }} ${{ inputs.paths }}" + "$VENV_PYTHON" -m mypy $mypy_opts ${{ inputs.options }} ${{ inputs.paths }} + echo ::endgroup:: diff --git a/build-requirements.txt b/build-requirements.txt new file mode 100644 index 000000000000..aac1b95eddf7 --- /dev/null +++ b/build-requirements.txt @@ -0,0 +1,4 @@ +# NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml +-r mypy-requirements.txt +types-psutil +types-setuptools diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000000..4454b02e7f3a --- /dev/null +++ b/conftest.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import os.path + +pytest_plugins = ["mypy.test.data"] + + +def pytest_configure(config): + mypy_source_root = os.path.dirname(os.path.abspath(__file__)) + if os.getcwd() != mypy_source_root: + os.chdir(mypy_source_root) + + +# This function name is special to pytest. See +# https://doc.pytest.org/en/latest/how-to/writing_plugins.html#initialization-command-line-and-configuration-hooks +def pytest_addoption(parser) -> None: + parser.addoption( + "--bench", action="store_true", default=False, help="Enable the benchmark test runs" + ) diff --git a/docs/Makefile b/docs/Makefile index be69e9d88281..c87c4c1abcb2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,7 +9,7 @@ BUILDDIR = build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from https://www.sphinx-doc.org/) endif # Internal variables. diff --git a/docs/README.md b/docs/README.md index 67e7d752d647..e72164c78560 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,19 +5,45 @@ What's this? ------------ This directory contains the source code for Mypy documentation (under `source/`) -and build scripts. The documentation uses Sphinx and reStructuredText. +and build scripts. The documentation uses Sphinx and reStructuredText. We use +`furo` as the documentation theme. Building the documentation -------------------------- -We use `sphinx_rtd_theme` as the theme. You should install it first: +Install Sphinx and other dependencies (i.e. theme) needed for the documentation. +From the `docs` directory, use `pip`: ``` -$ pip install sphinx_rtd_theme +pip install -r requirements-docs.txt ``` Build the documentation like this: ``` -$ make html +make html ``` + +The built documentation will be placed in the `docs/build` directory. Open +`docs/build/index.html` to view the documentation. + +Helpful documentation build commands +------------------------------------ + +Clean the documentation build: + +``` +make clean +``` + +Test and check the links found in the documentation: + +``` +make linkcheck +``` + +Documentation on Read The Docs +------------------------------ + +The mypy documentation is hosted on Read The Docs, and the latest version +can be found at https://mypy.readthedocs.io/en/latest. diff --git a/docs/make.bat b/docs/make.bat index 1e3d84320174..3664bed34b7e 100755 --- a/docs/make.bat +++ b/docs/make.bat @@ -56,7 +56,7 @@ if errorlevel 9009 ( echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ + echo.https://www.sphinx-doc.org/ exit /b 1 ) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 000000000000..a94c1b7ba95c --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1,3 @@ +sphinx>=8.1.0 +furo>=2022.3.4 +myst-parser>=4.0.0 diff --git a/docs/source/additional_features.rst b/docs/source/additional_features.rst index b9dd07f9bc3c..e7c162a0b0df 100644 --- a/docs/source/additional_features.rst +++ b/docs/source/additional_features.rst @@ -1,9 +1,483 @@ Additional features ------------------- -Several mypy features are not currently covered by this tutorial, -including the following: +This section discusses various features that did not fit in naturally in one +of the previous sections. -- inheritance between generic classes -- compatibility and subtyping of generic types, including covariance of generic types -- ``super()`` +.. _dataclasses_support: + +Dataclasses +*********** + +The :py:mod:`dataclasses` module allows defining and customizing simple +boilerplate-free classes. They can be defined using the +:py:func:`@dataclasses.dataclass ` decorator: + +.. code-block:: python + + from dataclasses import dataclass, field + + @dataclass + class Application: + name: str + plugins: list[str] = field(default_factory=list) + + test = Application("Testing...") # OK + bad = Application("Testing...", "with plugin") # Error: list[str] expected + +Mypy will detect special methods (such as :py:meth:`__lt__ `) depending on the flags used to +define dataclasses. For example: + +.. code-block:: python + + from dataclasses import dataclass + + @dataclass(order=True) + class OrderedPoint: + x: int + y: int + + @dataclass(order=False) + class UnorderedPoint: + x: int + y: int + + OrderedPoint(1, 2) < OrderedPoint(3, 4) # OK + UnorderedPoint(1, 2) < UnorderedPoint(3, 4) # Error: Unsupported operand types + +Dataclasses can be generic and can be used in any other way a normal +class can be used (Python 3.12 syntax): + +.. code-block:: python + + from dataclasses import dataclass + + @dataclass + class BoxedData[T]: + data: T + label: str + + def unbox[T](bd: BoxedData[T]) -> T: + ... + + val = unbox(BoxedData(42, "")) # OK, inferred type is int + +For more information see :doc:`official docs ` +and :pep:`557`. + +Caveats/Known Issues +==================== + +Some functions in the :py:mod:`dataclasses` module, such as :py:func:`~dataclasses.asdict`, +have imprecise (too permissive) types. This will be fixed in future releases. + +Mypy does not yet recognize aliases of :py:func:`dataclasses.dataclass `, and will +probably never recognize dynamically computed decorators. The following example +does **not** work: + +.. code-block:: python + + from dataclasses import dataclass + + dataclass_alias = dataclass + def dataclass_wrapper(cls): + return dataclass(cls) + + @dataclass_alias + class AliasDecorated: + """ + Mypy doesn't recognize this as a dataclass because it is decorated by an + alias of `dataclass` rather than by `dataclass` itself. + """ + attribute: int + + AliasDecorated(attribute=1) # error: Unexpected keyword argument + + +To have Mypy recognize a wrapper of :py:func:`dataclasses.dataclass ` +as a dataclass decorator, consider using the :py:func:`~typing.dataclass_transform` +decorator (example uses Python 3.12 syntax): + +.. code-block:: python + + from dataclasses import dataclass, Field + from typing import dataclass_transform + + @dataclass_transform(field_specifiers=(Field,)) + def my_dataclass[T](cls: type[T]) -> type[T]: + ... + return dataclass(cls) + + +Data Class Transforms +********************* + +Mypy supports the :py:func:`~typing.dataclass_transform` decorator as described in +`PEP 681 `_. + +.. note:: + + Pragmatically, mypy will assume such classes have the internal attribute :code:`__dataclass_fields__` + (even though they might lack it in runtime) and will assume functions such as :py:func:`dataclasses.is_dataclass` + and :py:func:`dataclasses.fields` treat them as if they were dataclasses + (even though they may fail at runtime). + +.. _attrs_package: + +The attrs package +***************** + +:doc:`attrs ` is a package that lets you define +classes without writing boilerplate code. Mypy can detect uses of the +package and will generate the necessary method definitions for decorated +classes using the type annotations it finds. +Type annotations can be added as follows: + +.. code-block:: python + + import attr + + @attrs.define + class A: + one: int + two: int = 7 + three: int = attrs.field(8) + +If you're using ``auto_attribs=False`` you must use ``attrs.field``: + +.. code-block:: python + + import attrs + + @attrs.define + class A: + one: int = attrs.field() # Variable annotation (Python 3.6+) + two = attrs.field() # type: int # Type comment + three = attrs.field(type=int) # type= argument + +Typeshed has a couple of "white lie" annotations to make type checking +easier. :py:func:`attrs.field` and :py:class:`attrs.Factory` actually return objects, but the +annotation says these return the types that they expect to be assigned to. +That enables this to work: + +.. code-block:: python + + import attrs + + @attrs.define + class A: + one: int = attrs.field(8) + two: dict[str, str] = attrs.Factory(dict) + bad: str = attrs.field(16) # Error: can't assign int to str + +Caveats/Known Issues +==================== + +* The detection of attr classes and attributes works by function name only. + This means that if you have your own helper functions that, for example, + ``return attrs.field()`` mypy will not see them. + +* All boolean arguments that mypy cares about must be literal ``True`` or ``False``. + e.g the following will not work: + + .. code-block:: python + + import attrs + YES = True + @attrs.define(init=YES) + class A: + ... + +* Currently, ``converter`` only supports named functions. If mypy finds something else it + will complain about not understanding the argument and the type annotation in + :py:meth:`__init__ ` will be replaced by ``Any``. + +* :ref:`Validator decorators ` + and `default decorators `_ + are not type-checked against the attribute they are setting/validating. + +* Method definitions added by mypy currently overwrite any existing method + definitions. + +.. _remote-cache: + +Using a remote cache to speed up mypy runs +****************************************** + +Mypy performs type checking *incrementally*, reusing results from +previous runs to speed up successive runs. If you are type checking a +large codebase, mypy can still be sometimes slower than desirable. For +example, if you create a new branch based on a much more recent commit +than the target of the previous mypy run, mypy may have to +process almost every file, as a large fraction of source files may +have changed. This can also happen after you've rebased a local +branch. + +Mypy supports using a *remote cache* to improve performance in cases +such as the above. In a large codebase, remote caching can sometimes +speed up mypy runs by a factor of 10, or more. + +Mypy doesn't include all components needed to set +this up -- generally you will have to perform some simple integration +with your Continuous Integration (CI) or build system to configure +mypy to use a remote cache. This discussion assumes you have a CI +system set up for the mypy build you want to speed up, and that you +are using a central git repository. Generalizing to different +environments should not be difficult. + +Here are the main components needed: + +* A shared repository for storing mypy cache files for all landed commits. + +* CI build that uploads mypy incremental cache files to the shared repository for + each commit for which the CI build runs. + +* A wrapper script around mypy that developers use to run mypy with remote + caching enabled. + +Below we discuss each of these components in some detail. + +Shared repository for cache files +================================= + +You need a repository that allows you to upload mypy cache files from +your CI build and make the cache files available for download based on +a commit id. A simple approach would be to produce an archive of the +``.mypy_cache`` directory (which contains the mypy cache data) as a +downloadable *build artifact* from your CI build (depending on the +capabilities of your CI system). Alternatively, you could upload the +data to a web server or to S3, for example. + +Continuous Integration build +============================ + +The CI build would run a regular mypy build and create an archive containing +the ``.mypy_cache`` directory produced by the build. Finally, it will produce +the cache as a build artifact or upload it to a repository where it is +accessible by the mypy wrapper script. + +Your CI script might work like this: + +* Run mypy normally. This will generate cache data under the + ``.mypy_cache`` directory. + +* Create a tarball from the ``.mypy_cache`` directory. + +* Determine the current git master branch commit id (say, using + ``git rev-parse HEAD``). + +* Upload the tarball to the shared repository with a name derived from the + commit id. + +Mypy wrapper script +=================== + +The wrapper script is used by developers to run mypy locally during +development instead of invoking mypy directly. The wrapper first +populates the local ``.mypy_cache`` directory from the shared +repository and then runs a normal incremental build. + +The wrapper script needs some logic to determine the most recent +central repository commit (by convention, the ``origin/master`` branch +for git) the local development branch is based on. In a typical git +setup you can do it like this: + +.. code:: + + git merge-base HEAD origin/master + +The next step is to download the cache data (contents of the +``.mypy_cache`` directory) from the shared repository based on the +commit id of the merge base produced by the git command above. The +script will decompress the data so that mypy will start with a fresh +``.mypy_cache``. Finally, the script runs mypy normally. And that's all! + +Caching with mypy daemon +======================== + +You can also use remote caching with the :ref:`mypy daemon `. +The remote cache will significantly speed up the first ``dmypy check`` +run after starting or restarting the daemon. + +The mypy daemon requires extra fine-grained dependency data in +the cache files which aren't included by default. To use caching with +the mypy daemon, use the :option:`--cache-fine-grained ` option in your CI +build:: + + $ mypy --cache-fine-grained + +This flag adds extra information for the daemon to the cache. In +order to use this extra information, you will also need to use the +``--use-fine-grained-cache`` option with ``dmypy start`` or +``dmypy restart``. Example:: + + $ dmypy start -- --use-fine-grained-cache + +Now your first ``dmypy check`` run should be much faster, as it can use +cache information to avoid processing the whole program. + +Refinements +=========== + +There are several optional refinements that may improve things further, +at least if your codebase is hundreds of thousands of lines or more: + +* If the wrapper script determines that the merge base hasn't changed + from a previous run, there's no need to download the cache data and + it's better to instead reuse the existing local cache data. + +* If you use the mypy daemon, you may want to restart the daemon each time + after the merge base or local branch has changed to avoid processing a + potentially large number of changes in an incremental build, as this can + be much slower than downloading cache data and restarting the daemon. + +* If the current local branch is based on a very recent master commit, + the remote cache data may not yet be available for that commit, as + there will necessarily be some latency to build the cache files. It + may be a good idea to look for cache data for, say, the 5 latest + master commits and use the most recent data that is available. + +* If the remote cache is not accessible for some reason (say, from a public + network), the script can still fall back to a normal incremental build. + +* You can have multiple local cache directories for different local branches + using the :option:`--cache-dir ` option. If the user switches to an existing + branch where downloaded cache data is already available, you can continue + to use the existing cache data instead of redownloading the data. + +* You can set up your CI build to use a remote cache to speed up the + CI build. This would be particularly useful if each CI build starts + from a fresh state without access to cache files from previous + builds. It's still recommended to run a full, non-incremental + mypy build to create the cache data, as repeatedly updating cache + data incrementally could result in drift over a long time period (due + to a mypy caching issue, perhaps). + +.. _extended_callable: + +Extended Callable types +*********************** + +.. note:: + + This feature is deprecated. You can use + :ref:`callback protocols ` as a replacement. + +As an experimental mypy extension, you can specify :py:class:`~collections.abc.Callable` types +that support keyword arguments, optional arguments, and more. When +you specify the arguments of a :py:class:`~collections.abc.Callable`, you can choose to supply just +the type of a nameless positional argument, or an "argument specifier" +representing a more complicated form of argument. This allows one to +more closely emulate the full range of possibilities given by the +``def`` statement in Python. + +As an example, here's a complicated function definition and the +corresponding :py:class:`~collections.abc.Callable`: + +.. code-block:: python + + from collections.abc import Callable + from mypy_extensions import (Arg, DefaultArg, NamedArg, + DefaultNamedArg, VarArg, KwArg) + + def func(__a: int, # This convention is for nameless arguments + b: int, + c: int = 0, + *args: int, + d: int, + e: int = 0, + **kwargs: int) -> int: + ... + + F = Callable[[int, # Or Arg(int) + Arg(int, 'b'), + DefaultArg(int, 'c'), + VarArg(int), + NamedArg(int, 'd'), + DefaultNamedArg(int, 'e'), + KwArg(int)], + int] + + f: F = func + +Argument specifiers are special function calls that can specify the +following aspects of an argument: + +- its type (the only thing that the basic format supports) + +- its name (if it has one) + +- whether it may be omitted + +- whether it may or must be passed using a keyword + +- whether it is a ``*args`` argument (representing the remaining + positional arguments) + +- whether it is a ``**kwargs`` argument (representing the remaining + keyword arguments) + +The following functions are available in ``mypy_extensions`` for this +purpose: + +.. code-block:: python + + def Arg(type=Any, name=None): + # A normal, mandatory, positional argument. + # If the name is specified it may be passed as a keyword. + + def DefaultArg(type=Any, name=None): + # An optional positional argument (i.e. with a default value). + # If the name is specified it may be passed as a keyword. + + def NamedArg(type=Any, name=None): + # A mandatory keyword-only argument. + + def DefaultNamedArg(type=Any, name=None): + # An optional keyword-only argument (i.e. with a default value). + + def VarArg(type=Any): + # A *args-style variadic positional argument. + # A single VarArg() specifier represents all remaining + # positional arguments. + + def KwArg(type=Any): + # A **kwargs-style variadic keyword argument. + # A single KwArg() specifier represents all remaining + # keyword arguments. + +In all cases, the ``type`` argument defaults to ``Any``, and if the +``name`` argument is omitted the argument has no name (the name is +required for ``NamedArg`` and ``DefaultNamedArg``). A basic +:py:class:`~collections.abc.Callable` such as + +.. code-block:: python + + MyFunc = Callable[[int, str, int], float] + +is equivalent to the following: + +.. code-block:: python + + MyFunc = Callable[[Arg(int), Arg(str), Arg(int)], float] + +A :py:class:`~collections.abc.Callable` with unspecified argument types, such as + +.. code-block:: python + + MyOtherFunc = Callable[..., int] + +is (roughly) equivalent to + +.. code-block:: python + + MyOtherFunc = Callable[[VarArg(), KwArg()], int] + +.. note:: + + Each of the functions above currently just returns its ``type`` + argument at runtime, so the information contained in the argument + specifiers is not available at runtime. This limitation is + necessary for backwards compatibility with the existing + ``typing.py`` module as present in the Python 3.5+ standard library + and distributed via PyPI. diff --git a/docs/source/basics.rst b/docs/source/basics.rst deleted file mode 100644 index e460ac26bea9..000000000000 --- a/docs/source/basics.rst +++ /dev/null @@ -1,170 +0,0 @@ -Basics -====== - -This chapter introduces some core concepts of mypy, including function -annotations, the ``typing`` module and library stubs. Read it carefully, -as the rest of documentation may not make much sense otherwise. - -Function signatures -******************* - -A function without a type signature is dynamically typed. You can -declare the signature of a function using the Python 3 annotation -syntax. This makes the function statically typed (the type checker -reports type errors within the function). A function without a -type annotation is dynamically typed, and identical to ordinary -Python: - -.. code-block:: python - - def greeting(name): - return 'Hello, {}'.format(name) - -This version of the above function is statically typed (but it's still -valid Python): - -.. code-block:: python - - def greeting(name: str) -> str: - return 'Hello, {}'.format(name) - -A ``None`` return type indicates a function that does not explicitly -return a value. Using a ``None`` result in a statically typed context -results in a type check error: - -.. code-block:: python - - def p() -> None: - print('hello') - - a = p() # Type check error: p has None return value - -The typing module -***************** - -We cheated a bit in the above examples: a module is type checked only -if it imports the module ``typing``. Here is a complete statically typed -example from the previous section: - -.. code-block:: python - - import typing - - def greeting(name: str) -> str: - return 'Hello, {}'.format(name) - -The ``typing`` module contains many definitions that are useful in -statically typed code. You can also use ``from ... import`` to import -them (we'll explain ``Iterable`` later in this document): - -.. code-block:: python - - from typing import Iterable - - def greet_all(names: Iterable[str]) -> None: - for name in names: - print('Hello, {}'.format(name)) - -For brevity, we often omit the ``typing`` import in code examples, but -you should always include it in modules that contain statically typed -code. - -You can still have dynamically typed functions in modules that import ``typing``: - -.. code-block:: python - - import typing - - def f(): - 1 + 'x' # No static type error (dynamically typed) - - def g() -> None: - 1 + 'x' # Type check error (statically typed) - -Mixing dynamic and static typing within a single file is often -useful. For example, if you are migrating existing Python code to -static typing, it may be easiest to do this incrementally, such as by -migrating a few functions at a time. Also, when prototyping a new -feature, you may decide to first implement the relevant code using -dynamic typing and only add type signatures later, when the code is -more stable. - -.. note:: - - Currently the type checker checks the top levels and annotated - functions of all modules, even those that don't import - ``typing``. However, you should not rely on this, as this will change - in the future. - -Type checking and running programs -********************************** - -You can type check a program by using the ``mypy`` tool, which is -basically a linter — it checks you program for errors without actually -running it:: - - $ mypy program.py - -You can always run a mypy program as a Python program, without type -checking, even it it has type errors:: - - $ python3 program.py - -All errors reported by mypy are essentially warnings that you are free -to ignore, if you so wish. - -The `README `_ -explains how to download and install mypy. - -.. note:: - - Depending on how mypy is configured, you may have to explicitly use - the Python interpreter to run mypy. The mypy tool is an ordinary - mypy (and so also Python) program. - -Library stubs -************* - -In order to type check code that uses library modules such those -included in the Python standard library, you need to have library -*stubs*. A library stub defines a skeleton of the public interface -of the library, including classes, variables and functions, and -their types. - -For example, consider this code: - -.. code-block:: python - - x = chr(4) - -Without a library stub, the type checker has no way of inferring the -type of ``x`` and checking that the argument to ``chr`` has a valid -type. Mypy comes with a library stub for Python builtins that contains -a definition like this for ``chr``: - -.. code-block:: python - - def chr(code: int) -> str: pass - -Mypy complains if it can't find a stub for a library module that you -import. You can create a stub easily; here is an overview: - -* Write a stub file for the library and store it as a ``.py`` file in - a directory reserved for stubs (e.g., ``myproject/stubs``). -* Set the environment variable ``MYPYPATH`` to refer to the above directory. - For example:: - - $ export MYPYPATH=~/work/myproject/stubs - -Use the normal Python file name conventions for modules, e.g. ``csv.py`` -for module ``csv``, and use a subdirectory with ``__init__.py`` for packages. - -That's it! Now you can access the module in mypy programs and type check -code that uses the library. If you write a stub for a library module, -consider making it available for other programmers that use mypy or -contributing it to mypy. - -There is more information about creating stubs in the -`mypy wiki `_. -The following sections explain the kinds of type annotations you can use -in your programs and stub files. diff --git a/docs/source/builtin_types.rst b/docs/source/builtin_types.rst index 4426df73868c..37b56169d879 100644 --- a/docs/source/builtin_types.rst +++ b/docs/source/builtin_types.rst @@ -1,37 +1,96 @@ Built-in types ============== -These are examples of some of the most common built-in types: - -=================== =============================== -Type Description -=================== =============================== -``int`` integer of arbitrary size -``float`` floating point number -``bool`` boolean value -``str`` unicode string -``bytes`` 8-bit string -``object`` an arbitrary object (``object`` is the common base class) -``List[str]`` list of ``str`` objects -``Dict[str, int]`` dictionary from ``str`` keys to ``int`` values -``Iterable[int]`` iterable object containing ints -``Sequence[bool]`` sequence of booleans -``Any`` dynamically typed value with an arbitrary type -=================== =============================== - -The type ``Any`` and type constructors ``List``, ``Dict``, -``Iterable`` and ``Sequence`` are defined in the ``typing`` module. - -The type ``Dict`` is a *generic* class, signified by type arguments within -``[...]``. For example, ``Dict[int, str]`` is a dictionary from integers to -strings and and ``Dict[Any, Any]`` is a dictionary of dynamically typed -(arbitrary) values and keys. ``List`` is another generic class. ``Dict`` and -``List`` are aliases for the built-ins ``dict`` and ``list``, respectively. - -``Iterable`` and ``Sequence`` are generic abstract base classes that -correspond to Python protocols. For example, a ``str`` object or a -``List[str]`` object is valid -when ``Iterable[str]`` or ``Sequence[str]`` is expected. Note that even though -they are similar to abstract base classes defined in ``abc.collections`` -(formerly ``collections``), they are not identical, since the built-in -collection type objects do not support indexing. +This chapter introduces some commonly used built-in types. We will +cover many other kinds of types later. + +Simple types +............ + +Here are examples of some common built-in types: + +====================== =============================== +Type Description +====================== =============================== +``int`` integer +``float`` floating point number +``bool`` boolean value (subclass of ``int``) +``str`` text, sequence of unicode codepoints +``bytes`` 8-bit string, sequence of byte values +``object`` an arbitrary object (``object`` is the common base class) +====================== =============================== + +All built-in classes can be used as types. + +Any type +........ + +If you can't find a good type for some value, you can always fall back +to ``Any``: + +====================== =============================== +Type Description +====================== =============================== +``Any`` dynamically typed value with an arbitrary type +====================== =============================== + +The type ``Any`` is defined in the :py:mod:`typing` module. +See :ref:`dynamic-typing` for more details. + +Generic types +............. + +In Python 3.9 and later, built-in collection type objects support +indexing: + +====================== =============================== +Type Description +====================== =============================== +``list[str]`` list of ``str`` objects +``tuple[int, int]`` tuple of two ``int`` objects (``tuple[()]`` is the empty tuple) +``tuple[int, ...]`` tuple of an arbitrary number of ``int`` objects +``dict[str, int]`` dictionary from ``str`` keys to ``int`` values +``Iterable[int]`` iterable object containing ints +``Sequence[bool]`` sequence of booleans (read-only) +``Mapping[str, int]`` mapping from ``str`` keys to ``int`` values (read-only) +``type[C]`` type object of ``C`` (``C`` is a class/type variable/union of types) +====================== =============================== + +The type ``dict`` is a *generic* class, signified by type arguments within +``[...]``. For example, ``dict[int, str]`` is a dictionary from integers to +strings and ``dict[Any, Any]`` is a dictionary of dynamically typed +(arbitrary) values and keys. ``list`` is another generic class. + +``Iterable``, ``Sequence``, and ``Mapping`` are generic types that correspond to +Python protocols. For example, a ``str`` object or a ``list[str]`` object is +valid when ``Iterable[str]`` or ``Sequence[str]`` is expected. +You can import them from :py:mod:`collections.abc` instead of importing from +:py:mod:`typing` in Python 3.9. + +See :ref:`generic-builtins` for more details, including how you can +use these in annotations also in Python 3.7 and 3.8. + +These legacy types defined in :py:mod:`typing` are needed if you need to support +Python 3.8 and earlier: + +====================== =============================== +Type Description +====================== =============================== +``List[str]`` list of ``str`` objects +``Tuple[int, int]`` tuple of two ``int`` objects (``Tuple[()]`` is the empty tuple) +``Tuple[int, ...]`` tuple of an arbitrary number of ``int`` objects +``Dict[str, int]`` dictionary from ``str`` keys to ``int`` values +``Iterable[int]`` iterable object containing ints +``Sequence[bool]`` sequence of booleans (read-only) +``Mapping[str, int]`` mapping from ``str`` keys to ``int`` values (read-only) +``Type[C]`` type object of ``C`` (``C`` is a class/type variable/union of types) +====================== =============================== + +``List`` is an alias for the built-in type ``list`` that supports +indexing (and similarly for ``dict``/``Dict`` and +``tuple``/``Tuple``). + +Note that even though ``Iterable``, ``Sequence`` and ``Mapping`` look +similar to abstract base classes defined in :py:mod:`collections.abc` +(formerly ``collections``), they are not identical, since the latter +don't support indexing prior to Python 3.9. diff --git a/docs/source/casts.rst b/docs/source/casts.rst deleted file mode 100644 index 525829012265..000000000000 --- a/docs/source/casts.rst +++ /dev/null @@ -1,42 +0,0 @@ -.. _casts: - -Casts -===== - -Mypy supports type casts that are usually used to coerce a statically -typed value to a subtype. Unlike languages such as Java or C#, -however, mypy casts are only used as hints for the type checker when -using Python semantics, and they have no runtime effect. Use the -function ``cast`` to perform a cast: - -.. code-block:: python - - from typing import cast - - o = [1] # type: object - x = cast(List[int], o) # OK - y = cast(List[str], o) # OK (cast performs no actual runtime check) - -Supporting runtime checking of casts such as the above when using -Python semantics would require emulating reified generics and this -would be difficult to do and would likely degrade performance and make -code more difficult to read. You should not rely in your programs on -casts being checked at runtime. Use an assertion if you want to -perform an actual runtime check. Casts are used to silence spurious -type checker warnings. - -You don't need a cast for expressions with type ``Any``, of when -assigning to a variable with type ``Any``, as was explained earlier. - -Any() as a cast -*************** - -You can cast to a dynamically typed value by just calling ``Any``; -this is equivalent to ``cast(Any, ...)`` but shorter: - -.. code-block:: python - - from typing import Any - - def f(x: object) -> None: - Any(x).foo() # OK diff --git a/docs/source/changelog.md b/docs/source/changelog.md new file mode 100644 index 000000000000..a490ada727a6 --- /dev/null +++ b/docs/source/changelog.md @@ -0,0 +1,3 @@ + +```{include} ../../CHANGELOG.md +``` diff --git a/docs/source/cheat_sheet_py3.rst b/docs/source/cheat_sheet_py3.rst new file mode 100644 index 000000000000..7385a66863bf --- /dev/null +++ b/docs/source/cheat_sheet_py3.rst @@ -0,0 +1,399 @@ +.. _cheat-sheet-py3: + +Type hints cheat sheet +====================== + +This document is a quick cheat sheet showing how to use type +annotations for various common types in Python. + +Variables +********* + +Technically many of the type annotations shown below are redundant, +since mypy can usually infer the type of a variable from its value. +See :ref:`type-inference-and-annotations` for more details. + +.. code-block:: python + + # This is how you declare the type of a variable + age: int = 1 + + # You don't need to initialize a variable to annotate it + a: int # Ok (no value at runtime until assigned) + + # Doing so can be useful in conditional branches + child: bool + if age < 18: + child = True + else: + child = False + + +Useful built-in types +********************* + +.. code-block:: python + + # For most types, just use the name of the type in the annotation + # Note that mypy can usually infer the type of a variable from its value, + # so technically these annotations are redundant + x: int = 1 + x: float = 1.0 + x: bool = True + x: str = "test" + x: bytes = b"test" + + # For collections on Python 3.9+, the type of the collection item is in brackets + x: list[int] = [1] + x: set[int] = {6, 7} + + # For mappings, we need the types of both keys and values + x: dict[str, float] = {"field": 2.0} # Python 3.9+ + + # For tuples of fixed size, we specify the types of all the elements + x: tuple[int, str, float] = (3, "yes", 7.5) # Python 3.9+ + + # For tuples of variable size, we use one type and ellipsis + x: tuple[int, ...] = (1, 2, 3) # Python 3.9+ + + # On Python 3.8 and earlier, the name of the collection type is + # capitalized, and the type is imported from the 'typing' module + from typing import List, Set, Dict, Tuple + x: List[int] = [1] + x: Set[int] = {6, 7} + x: Dict[str, float] = {"field": 2.0} + x: Tuple[int, str, float] = (3, "yes", 7.5) + x: Tuple[int, ...] = (1, 2, 3) + + from typing import Union, Optional + + # On Python 3.10+, use the | operator when something could be one of a few types + x: list[int | str] = [3, 5, "test", "fun"] # Python 3.10+ + # On earlier versions, use Union + x: list[Union[int, str]] = [3, 5, "test", "fun"] + + # Use X | None for a value that could be None on Python 3.10+ + # Use Optional[X] on 3.9 and earlier; Optional[X] is the same as 'X | None' + x: str | None = "something" if some_condition() else None + if x is not None: + # Mypy understands x won't be None here because of the if-statement + print(x.upper()) + # If you know a value can never be None due to some logic that mypy doesn't + # understand, use an assert + assert x is not None + print(x.upper()) + +Functions +********* + +.. code-block:: python + + from collections.abc import Iterator, Callable + from typing import Union, Optional + + # This is how you annotate a function definition + def stringify(num: int) -> str: + return str(num) + + # And here's how you specify multiple arguments + def plus(num1: int, num2: int) -> int: + return num1 + num2 + + # If a function does not return a value, use None as the return type + # Default value for an argument goes after the type annotation + def show(value: str, excitement: int = 10) -> None: + print(value + "!" * excitement) + + # Note that arguments without a type are dynamically typed (treated as Any) + # and that functions without any annotations are not checked + def untyped(x): + x.anything() + 1 + "string" # no errors + + # This is how you annotate a callable (function) value + x: Callable[[int, float], float] = f + def register(callback: Callable[[str], int]) -> None: ... + + # A generator function that yields ints is secretly just a function that + # returns an iterator of ints, so that's how we annotate it + def gen(n: int) -> Iterator[int]: + i = 0 + while i < n: + yield i + i += 1 + + # You can of course split a function annotation over multiple lines + def send_email( + address: str | list[str], + sender: str, + cc: list[str] | None, + bcc: list[str] | None, + subject: str = '', + body: list[str] | None = None, + ) -> bool: + ... + + # Mypy understands positional-only and keyword-only arguments + # Positional-only arguments can also be marked by using a name starting with + # two underscores + def quux(x: int, /, *, y: int) -> None: + pass + + quux(3, y=5) # Ok + quux(3, 5) # error: Too many positional arguments for "quux" + quux(x=3, y=5) # error: Unexpected keyword argument "x" for "quux" + + # This says each positional arg and each keyword arg is a "str" + def call(self, *args: str, **kwargs: str) -> str: + reveal_type(args) # Revealed type is "tuple[str, ...]" + reveal_type(kwargs) # Revealed type is "dict[str, str]" + request = make_request(*args, **kwargs) + return self.do_api_query(request) + +Classes +******* + +.. code-block:: python + + from typing import ClassVar + + class BankAccount: + # The "__init__" method doesn't return anything, so it gets return + # type "None" just like any other method that doesn't return anything + def __init__(self, account_name: str, initial_balance: int = 0) -> None: + # mypy will infer the correct types for these instance variables + # based on the types of the parameters. + self.account_name = account_name + self.balance = initial_balance + + # For instance methods, omit type for "self" + def deposit(self, amount: int) -> None: + self.balance += amount + + def withdraw(self, amount: int) -> None: + self.balance -= amount + + # User-defined classes are valid as types in annotations + account: BankAccount = BankAccount("Alice", 400) + def transfer(src: BankAccount, dst: BankAccount, amount: int) -> None: + src.withdraw(amount) + dst.deposit(amount) + + # Functions that accept BankAccount also accept any subclass of BankAccount! + class AuditedBankAccount(BankAccount): + # You can optionally declare instance variables in the class body + audit_log: list[str] + + def __init__(self, account_name: str, initial_balance: int = 0) -> None: + super().__init__(account_name, initial_balance) + self.audit_log: list[str] = [] + + def deposit(self, amount: int) -> None: + self.audit_log.append(f"Deposited {amount}") + self.balance += amount + + def withdraw(self, amount: int) -> None: + self.audit_log.append(f"Withdrew {amount}") + self.balance -= amount + + audited = AuditedBankAccount("Bob", 300) + transfer(audited, account, 100) # type checks! + + # You can use the ClassVar annotation to declare a class variable + class Car: + seats: ClassVar[int] = 4 + passengers: ClassVar[list[str]] + + # If you want dynamic attributes on your class, have it + # override "__setattr__" or "__getattr__" + class A: + # This will allow assignment to any A.x, if x is the same type as "value" + # (use "value: Any" to allow arbitrary types) + def __setattr__(self, name: str, value: int) -> None: ... + + # This will allow access to any A.x, if x is compatible with the return type + def __getattr__(self, name: str) -> int: ... + + a = A() + a.foo = 42 # Works + a.bar = 'Ex-parrot' # Fails type checking + +When you're puzzled or when things are complicated +************************************************** + +.. code-block:: python + + from typing import Union, Any, Optional, TYPE_CHECKING, cast + + # To find out what type mypy infers for an expression anywhere in + # your program, wrap it in reveal_type(). Mypy will print an error + # message with the type; remove it again before running the code. + reveal_type(1) # Revealed type is "builtins.int" + + # If you initialize a variable with an empty container or "None" + # you may have to help mypy a bit by providing an explicit type annotation + x: list[str] = [] + x: str | None = None + + # Use Any if you don't know the type of something or it's too + # dynamic to write a type for + x: Any = mystery_function() + # Mypy will let you do anything with x! + x.whatever() * x["you"] + x("want") - any(x) and all(x) is super # no errors + + # Use a "type: ignore" comment to suppress errors on a given line, + # when your code confuses mypy or runs into an outright bug in mypy. + # Good practice is to add a comment explaining the issue. + x = confusing_function() # type: ignore # confusing_function won't return None here because ... + + # "cast" is a helper function that lets you override the inferred + # type of an expression. It's only for mypy -- there's no runtime check. + a = [4] + b = cast(list[int], a) # Passes fine + c = cast(list[str], a) # Passes fine despite being a lie (no runtime check) + reveal_type(c) # Revealed type is "builtins.list[builtins.str]" + print(c) # Still prints [4] ... the object is not changed or casted at runtime + + # Use "TYPE_CHECKING" if you want to have code that mypy can see but will not + # be executed at runtime (or to have code that mypy can't see) + if TYPE_CHECKING: + import json + else: + import orjson as json # mypy is unaware of this + +In some cases type annotations can cause issues at runtime, see +:ref:`runtime_troubles` for dealing with this. + +See :ref:`silencing-type-errors` for details on how to silence errors. + +Standard "duck types" +********************* + +In typical Python code, many functions that can take a list or a dict +as an argument only need their argument to be somehow "list-like" or +"dict-like". A specific meaning of "list-like" or "dict-like" (or +something-else-like) is called a "duck type", and several duck types +that are common in idiomatic Python are standardized. + +.. code-block:: python + + from collections.abc import Mapping, MutableMapping, Sequence, Iterable + # or 'from typing import ...' (required in Python 3.8) + + # Use Iterable for generic iterables (anything usable in "for"), + # and Sequence where a sequence (supporting "len" and "__getitem__") is + # required + def f(ints: Iterable[int]) -> list[str]: + return [str(x) for x in ints] + + f(range(1, 3)) + + # Mapping describes a dict-like object (with "__getitem__") that we won't + # mutate, and MutableMapping one (with "__setitem__") that we might + def f(my_mapping: Mapping[int, str]) -> list[int]: + my_mapping[5] = 'maybe' # mypy will complain about this line... + return list(my_mapping.keys()) + + f({3: 'yes', 4: 'no'}) + + def f(my_mapping: MutableMapping[int, str]) -> set[str]: + my_mapping[5] = 'maybe' # ...but mypy is OK with this. + return set(my_mapping.values()) + + f({3: 'yes', 4: 'no'}) + + import sys + from typing import IO + + # Use IO[str] or IO[bytes] for functions that should accept or return + # objects that come from an open() call (note that IO does not + # distinguish between reading, writing or other modes) + def get_sys_IO(mode: str = 'w') -> IO[str]: + if mode == 'w': + return sys.stdout + elif mode == 'r': + return sys.stdin + else: + return sys.stdout + + +You can even make your own duck types using :ref:`protocol-types`. + +Forward references +****************** + +.. code-block:: python + + # You may want to reference a class before it is defined. + # This is known as a "forward reference". + def f(foo: A) -> int: # This will fail at runtime with 'A' is not defined + ... + + # However, if you add the following special import: + from __future__ import annotations + # It will work at runtime and type checking will succeed as long as there + # is a class of that name later on in the file + def f(foo: A) -> int: # Ok + ... + + # Another option is to just put the type in quotes + def f(foo: 'A') -> int: # Also ok + ... + + class A: + # This can also come up if you need to reference a class in a type + # annotation inside the definition of that class + @classmethod + def create(cls) -> A: + ... + +See :ref:`forward-references` for more details. + +Decorators +********** + +Decorator functions can be expressed via generics. See +:ref:`declaring-decorators` for more details. Example using Python 3.12 +syntax: + +.. code-block:: python + + from collections.abc import Callable + from typing import Any + + def bare_decorator[F: Callable[..., Any]](func: F) -> F: + ... + + def decorator_args[F: Callable[..., Any]](url: str) -> Callable[[F], F]: + ... + +The same example using pre-3.12 syntax: + +.. code-block:: python + + from collections.abc import Callable + from typing import Any, TypeVar + + F = TypeVar('F', bound=Callable[..., Any]) + + def bare_decorator(func: F) -> F: + ... + + def decorator_args(url: str) -> Callable[[F], F]: + ... + +Coroutines and asyncio +********************** + +See :ref:`async-and-await` for the full detail on typing coroutines and asynchronous code. + +.. code-block:: python + + import asyncio + + # A coroutine is typed like a normal function + async def countdown(tag: str, count: int) -> str: + while count > 0: + print(f'T-minus {count} ({tag})') + await asyncio.sleep(0.1) + count -= 1 + return "Blastoff!" diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 8b23d89e10a1..241dbeae0f44 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -1,10 +1,16 @@ +.. _class-basics: + Class basics ============ +This section will help get you started annotating your +classes. Built-in classes such as ``int`` also follow these same +rules. + Instance and class attributes ***************************** -Mypy type checker detects if you are trying to access a missing +The mypy type checker detects if you are trying to access a missing attribute, which is a very common programming error. For this to work correctly, instance and class attributes must be defined or initialized within the class. Mypy infers the types of attributes: @@ -13,52 +19,30 @@ initialized within the class. Mypy infers the types of attributes: class A: def __init__(self, x: int) -> None: - self.x = x # Attribute x of type int + self.x = x # Aha, attribute 'x' of type 'int' a = A(1) - a.x = 2 # OK - a.y = 3 # Error: A has no attribute y + a.x = 2 # OK! + a.y = 3 # Error: "A" has no attribute "y" This is a bit like each class having an implicitly defined -``__slots__`` attribute. In Python semantics this is only enforced -during type checking: at runtime we use standard Python semantics. You -can selectively define a class as *dynamic*; dynamic classes have -Python-like compile-time semantics, and they allow you to assign to -arbitrary attributes anywhere in a program without the type checker -complaining: - -.. code-block:: python - - from typing import Dynamic - - class A(Dynamic): - pass - - a = A() - a.x = 2 # OK, no need to define x explicitly. +:py:data:`__slots__ ` attribute. This is only enforced during type +checking and not when your program is running. -Mypy also lets you read arbitrary attributes of dynamic class -instances. This limits type checking effectiveness, so you should only -use dynamic classes when you really need them. - -.. note:: - - Dynamic classes are not implemented in the current mypy version. - -You can declare variables in the class body explicitly using -``Undefined`` or a type comment: +You can declare types of variables in the class body explicitly using +a type annotation: .. code-block:: python class A: - x = Undefined(List[int]) # Declare attribute y of type List[int] - y = 0 # type: Any # Declare attribute x of type Any + x: list[int] # Declare attribute 'x' of type list[int] a = A() a.x = [1] # OK -As in Python, a variable defined in the class body can used as a class -or an instance variable. +As in Python generally, a variable defined in the class body can be used +as a class or an instance variable. (As discussed in the next section, you +can override this with a :py:data:`~typing.ClassVar` annotation.) Similarly, you can give explicit types to instance variables defined in a method: @@ -67,10 +51,10 @@ in a method: class A: def __init__(self) -> None: - self.x = Undefined(List[int]) # OK + self.x: list[int] = [] def f(self) -> None: - self.y = 0 # type: Any # OK + self.y: Any = 0 You can only define an instance variable within a method if you assign to it explicitly using ``self``: @@ -79,9 +63,112 @@ to it explicitly using ``self``: class A: def __init__(self) -> None: - self.y = 1 # Define y + self.y = 1 # Define 'y' a = self - a.x = 1 # Error: x not defined + a.x = 1 # Error: 'x' not defined + +Annotating __init__ methods +*************************** + +The :py:meth:`__init__ ` method is somewhat special -- it doesn't return a +value. This is best expressed as ``-> None``. However, since many feel +this is redundant, it is allowed to omit the return type declaration +on :py:meth:`__init__ ` methods **if at least one argument is annotated**. For +example, in the following classes :py:meth:`__init__ ` is considered fully +annotated: + +.. code-block:: python + + class C1: + def __init__(self) -> None: + self.var = 42 + + class C2: + def __init__(self, arg: int): + self.var = arg + +However, if :py:meth:`__init__ ` has no annotated arguments and no return type +annotation, it is considered an untyped method: + +.. code-block:: python + + class C3: + def __init__(self): + # This body is not type checked + self.var = 42 + 'abc' + +Class attribute annotations +*************************** + +You can use a :py:data:`ClassVar[t] ` annotation to explicitly declare that a +particular attribute should not be set on instances: + +.. code-block:: python + + from typing import ClassVar + + class A: + x: ClassVar[int] = 0 # Class variable only + + A.x += 1 # OK + + a = A() + a.x = 1 # Error: Cannot assign to class variable "x" via instance + print(a.x) # OK -- can be read through an instance + +It's not necessary to annotate all class variables using +:py:data:`~typing.ClassVar`. An attribute without the :py:data:`~typing.ClassVar` annotation can +still be used as a class variable. However, mypy won't prevent it from +being used as an instance variable, as discussed previously: + +.. code-block:: python + + class A: + x = 0 # Can be used as a class or instance variable + + A.x += 1 # OK + + a = A() + a.x = 1 # Also OK + +Note that :py:data:`~typing.ClassVar` is not a class, and you can't use it with +:py:func:`isinstance` or :py:func:`issubclass`. It does not change Python +runtime behavior -- it's only for type checkers such as mypy (and +also helpful for human readers). + +You can also omit the square brackets and the variable type in +a :py:data:`~typing.ClassVar` annotation, but this might not do what you'd expect: + +.. code-block:: python + + class A: + y: ClassVar = 0 # Type implicitly Any! + +In this case the type of the attribute will be implicitly ``Any``. +This behavior will change in the future, since it's surprising. + +An explicit :py:data:`~typing.ClassVar` may be particularly handy to distinguish +between class and instance variables with callable types. For example: + +.. code-block:: python + + from collections.abc import Callable + from typing import ClassVar + + class A: + foo: Callable[[int], None] + bar: ClassVar[Callable[[A, int], None]] + bad: Callable[[A], None] + + A().foo(42) # OK + A().bar(42) # OK + A().bad() # Error: Too few arguments + +.. note:: + A :py:data:`~typing.ClassVar` type parameter cannot include type variables: + ``ClassVar[T]`` and ``ClassVar[list[T]]`` + are both invalid if ``T`` is a type variable (see :ref:`generic-classes` + for more about type variables). Overriding statically typed methods *********************************** @@ -91,90 +178,224 @@ override has a compatible signature: .. code-block:: python - class A: + class Base: def f(self, x: int) -> None: ... - class B(A): - def f(self, x: str) -> None: # Error: type of x incompatible + class Derived1(Base): + def f(self, x: str) -> None: # Error: type of 'x' incompatible ... - class C(A): + class Derived2(Base): def f(self, x: int, y: int) -> None: # Error: too many arguments ... - class D(A): + class Derived3(Base): def f(self, x: int) -> None: # OK ... + class Derived4(Base): + def f(self, x: float) -> None: # OK: mypy treats int as a subtype of float + ... + + class Derived5(Base): + def f(self, x: int, y: int = 0) -> None: # OK: accepts more than the base + ... # class method + .. note:: You can also vary return types **covariantly** in overriding. For - example, you could override the return type ``object`` with a subtype - such as ``int``. + example, you could override the return type ``Iterable[int]`` with a + subtype such as ``list[int]``. Similarly, you can vary argument types + **contravariantly** -- subclasses can have more general argument types. + +In order to ensure that your code remains correct when renaming methods, +it can be helpful to explicitly mark a method as overriding a base +method. This can be done with the ``@override`` decorator. ``@override`` +can be imported from ``typing`` starting with Python 3.12 or from +``typing_extensions`` for use with older Python versions. If the base +method is then renamed while the overriding method is not, mypy will +show an error: + +.. code-block:: python + + from typing import override + + class Base: + def f(self, x: int) -> None: + ... + def g_renamed(self, y: str) -> None: + ... + + class Derived1(Base): + @override + def f(self, x: int) -> None: # OK + ... + + @override + def g(self, y: str) -> None: # Error: no corresponding base method found + ... + +.. note:: + + Use :ref:`--enable-error-code explicit-override ` to require + that method overrides use the ``@override`` decorator. Emit an error if it is missing. You can also override a statically typed method with a dynamically typed one. This allows dynamically typed code to override methods defined in library classes without worrying about their type -signatures, similar to Python. +signatures. -There is no runtime enforcement that the method override returns a -value that is compatible with the original return type, since types -are erased in the Python semantics: +As always, relying on dynamically typed code can be unsafe. There is no +runtime enforcement that the method override returns a value that is +compatible with the original return type, since annotations have no +effect at runtime: .. code-block:: python - class A: + class Base: def inc(self, x: int) -> int: return x + 1 - class B(A): - def inc(self, x): # Override, dynamically typed - return 'hello' - - b = B() - print(b.inc(1)) # hello - a = b # type: A - print(a.inc(1)) # hello + class Derived(Base): + def inc(self, x): # Override, dynamically typed + return 'hello' # Incompatible with 'Base', but no mypy error Abstract base classes and multiple inheritance ********************************************** -Mypy uses Python abstract base classes for protocol types. There are -several built-in abstract base classes types (for example, -``Sequence``, ``Iterable`` and ``Iterator``). You can define abstract -base classes using the ``abc.ABCMeta`` metaclass and the -``abc.abstractmethod`` function decorator. +Mypy supports Python :doc:`abstract base classes ` (ABCs). Abstract classes +have at least one abstract method or property that must be implemented +by any *concrete* (non-abstract) subclass. You can define abstract base +classes using the :py:class:`abc.ABCMeta` metaclass and the :py:func:`@abc.abstractmethod ` +function decorator. Example: .. code-block:: python from abc import ABCMeta, abstractmethod - import typing - class A(metaclass=ABCMeta): + class Animal(metaclass=ABCMeta): @abstractmethod - def foo(self, x: int) -> None: pass + def eat(self, food: str) -> None: pass + + @property + @abstractmethod + def can_walk(self) -> bool: pass + + class Cat(Animal): + def eat(self, food: str) -> None: + ... # Body omitted + + @property + def can_walk(self) -> bool: + return True + + x = Animal() # Error: 'Animal' is abstract due to 'eat' and 'can_walk' + y = Cat() # OK + +Note that mypy performs checking for unimplemented abstract methods +even if you omit the :py:class:`~abc.ABCMeta` metaclass. This can be useful if the +metaclass would cause runtime metaclass conflicts. + +Since you can't create instances of ABCs, they are most commonly used in +type annotations. For example, this method accepts arbitrary iterables +containing arbitrary animals (instances of concrete ``Animal`` +subclasses): + +.. code-block:: python + + def feed_all(animals: Iterable[Animal], food: str) -> None: + for animal in animals: + animal.eat(food) + +There is one important peculiarity about how ABCs work in Python -- +whether a particular class is abstract or not is somewhat implicit. +In the example below, ``Derived`` is treated as an abstract base class +since ``Derived`` inherits an abstract ``f`` method from ``Base`` and +doesn't explicitly implement it. The definition of ``Derived`` +generates no errors from mypy, since it's a valid ABC: +.. code-block:: python + + from abc import ABCMeta, abstractmethod + + class Base(metaclass=ABCMeta): @abstractmethod - def bar(self) -> str: pass + def f(self, x: int) -> None: pass + + class Derived(Base): # No error -- Derived is implicitly abstract + def g(self) -> None: + ... + +Attempting to create an instance of ``Derived`` will be rejected, +however: - class B(A): - def foo(self, x: int) -> None: ... - def bar(self -> str: - return 'x' +.. code-block:: python + + d = Derived() # Error: 'Derived' is abstract + +.. note:: - a = A() # Error: A is abstract - b = B() # OK + It's a common error to forget to implement an abstract method. + As shown above, the class definition will not generate an error + in this case, but any attempt to construct an instance will be + flagged as an error. + +Mypy allows you to omit the body for an abstract method, but if you do so, +it is unsafe to call such method via ``super()``. For example: + +.. code-block:: python -Unlike most Python code, abstract base classes are likely to play a -significant role in many complex mypy programs. + from abc import abstractmethod + class Base: + @abstractmethod + def foo(self) -> int: pass + @abstractmethod + def bar(self) -> int: + return 0 + class Sub(Base): + def foo(self) -> int: + return super().foo() + 1 # error: Call to abstract method "foo" of "Base" + # with trivial body via super() is unsafe + @abstractmethod + def bar(self) -> int: + return super().bar() + 1 # This is OK however. A class can inherit any number of classes, both abstract and concrete. As with normal overrides, a dynamically typed method can -implement a statically typed abstract method defined in an abstract -base class. +override or implement a statically typed method defined in any base +class, including an abstract method defined in an abstract base class. -.. note:: +You can implement an abstract property using either a normal +property or an instance variable. + +Slots +***** + +When a class has explicitly defined :std:term:`__slots__`, +mypy will check that all attributes assigned to are members of ``__slots__``: + +.. code-block:: python + + class Album: + __slots__ = ('name', 'year') + + def __init__(self, name: str, year: int) -> None: + self.name = name + self.year = year + # Error: Trying to assign name "released" that is not in "__slots__" of type "Album" + self.released = True + + my_album = Album('Songs about Python', 2021) + +Mypy will only check attribute assignments against ``__slots__`` when +the following conditions hold: + +1. All base classes (except builtin ones) must have explicit + ``__slots__`` defined (this mirrors Python semantics). + +2. ``__slots__`` does not include ``__dict__``. If ``__slots__`` + includes ``__dict__``, arbitrary attributes can be set, similar to + when ``__slots__`` is not defined (this mirrors Python semantics). - There are also plans to support more Python-style "duck typing" in - the type system. The details are still open. +3. All values in ``__slots__`` must be string literals. diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst new file mode 100644 index 000000000000..ea96e9f64790 --- /dev/null +++ b/docs/source/command_line.rst @@ -0,0 +1,1148 @@ +.. _command-line: + +.. program:: mypy + +The mypy command line +===================== + +This section documents mypy's command line interface. You can view +a quick summary of the available flags by running :option:`mypy --help`. + +.. note:: + + Command line flags are liable to change between releases. + + +Specifying what to type check +***************************** + +By default, you can specify what code you want mypy to type check +by passing in the paths to what you want to have type checked:: + + $ mypy foo.py bar.py some_directory + +Note that directories are checked recursively. + +Mypy also lets you specify what code to type check in several other +ways. A short summary of the relevant flags is included below: +for full details, see :ref:`running-mypy`. + +.. option:: -m MODULE, --module MODULE + + Asks mypy to type check the provided module. This flag may be + repeated multiple times. + + Mypy *will not* recursively type check any submodules of the provided + module. + +.. option:: -p PACKAGE, --package PACKAGE + + Asks mypy to type check the provided package. This flag may be + repeated multiple times. + + Mypy *will* recursively type check any submodules of the provided + package. This flag is identical to :option:`--module` apart from this + behavior. + +.. option:: -c PROGRAM_TEXT, --command PROGRAM_TEXT + + Asks mypy to type check the provided string as a program. + + +.. option:: --exclude + + A regular expression that matches file names, directory names and paths + which mypy should ignore while recursively discovering files to check. + Use forward slashes on all platforms. + + For instance, to avoid discovering any files named `setup.py` you could + pass ``--exclude '/setup\.py$'``. Similarly, you can ignore discovering + directories with a given name by e.g. ``--exclude /build/`` or + those matching a subpath with ``--exclude /project/vendor/``. To ignore + multiple files / directories / paths, you can provide the --exclude + flag more than once, e.g ``--exclude '/setup\.py$' --exclude '/build/'``. + + Note that this flag only affects recursive directory tree discovery, that + is, when mypy is discovering files within a directory tree or submodules of + a package to check. If you pass a file or module explicitly it will still be + checked. For instance, ``mypy --exclude '/setup.py$' + but_still_check/setup.py``. + + In particular, ``--exclude`` does not affect mypy's :ref:`import following + `. You can use a per-module :confval:`follow_imports` config + option to additionally avoid mypy from following imports and checking code + you do not wish to be checked. + + Note that mypy will never recursively discover files and directories named + "site-packages", "node_modules" or "__pycache__", or those whose name starts + with a period, exactly as ``--exclude + '/(site-packages|node_modules|__pycache__|\..*)/$'`` would. Mypy will also + never recursively discover files with extensions other than ``.py`` or + ``.pyi``. + + +Optional arguments +****************** + +.. option:: -h, --help + + Show help message and exit. + +.. option:: -v, --verbose + + More verbose messages. + +.. option:: -V, --version + + Show program's version number and exit. + +.. option:: -O FORMAT, --output FORMAT {json} + + Set a custom output format. + +.. _config-file-flag: + +Config file +*********** + +.. option:: --config-file CONFIG_FILE + + This flag makes mypy read configuration settings from the given file. + + By default settings are read from ``mypy.ini``, ``.mypy.ini``, ``pyproject.toml``, or ``setup.cfg`` + in the current directory. Settings override mypy's built-in defaults and + command line flags can override settings. + + Specifying :option:`--config-file= <--config-file>` (with no filename) will ignore *all* + config files. + + See :ref:`config-file` for the syntax of configuration files. + +.. option:: --warn-unused-configs + + This flag makes mypy warn about unused ``[mypy-]`` config + file sections. + (This requires turning off incremental mode using :option:`--no-incremental`.) + + +.. _import-discovery: + +Import discovery +**************** + +The following flags customize how exactly mypy discovers and follows +imports. + +.. option:: --explicit-package-bases + + This flag tells mypy that top-level packages will be based in either the + current directory, or a member of the ``MYPYPATH`` environment variable or + :confval:`mypy_path` config option. This option is only useful + in the absence of `__init__.py`. See :ref:`Mapping file + paths to modules ` for details. + +.. option:: --ignore-missing-imports + + This flag makes mypy ignore all missing imports. It is equivalent + to adding ``# type: ignore`` comments to all unresolved imports + within your codebase. + + Note that this flag does *not* suppress errors about missing names + in successfully resolved modules. For example, if one has the + following files:: + + package/__init__.py + package/mod.py + + Then mypy will generate the following errors with :option:`--ignore-missing-imports`: + + .. code-block:: python + + import package.unknown # No error, ignored + x = package.unknown.func() # OK. 'func' is assumed to be of type 'Any' + + from package import unknown # No error, ignored + from package.mod import NonExisting # Error: Module has no attribute 'NonExisting' + + For more details, see :ref:`ignore-missing-imports`. + +.. option:: --follow-untyped-imports + + This flag makes mypy analyze imports from installed packages even if + missing a :ref:`py.typed marker or stubs `. + + .. warning:: + + Note that analyzing all unannotated modules might result in issues + when analyzing code not designed to be type checked and may significantly + increase how long mypy takes to run. + +.. option:: --follow-imports {normal,silent,skip,error} + + This flag adjusts how mypy follows imported modules that were not + explicitly passed in via the command line. + + The default option is ``normal``: mypy will follow and type check + all modules. For more information on what the other options do, + see :ref:`Following imports `. + +.. option:: --python-executable EXECUTABLE + + This flag will have mypy collect type information from :pep:`561` + compliant packages installed for the Python executable ``EXECUTABLE``. + If not provided, mypy will use PEP 561 compliant packages installed for + the Python executable running mypy. + + See :ref:`installed-packages` for more on making PEP 561 compliant packages. + +.. option:: --no-site-packages + + This flag will disable searching for :pep:`561` compliant packages. This + will also disable searching for a usable Python executable. + + Use this flag if mypy cannot find a Python executable for the version of + Python being checked, and you don't need to use PEP 561 typed packages. + Otherwise, use :option:`--python-executable`. + +.. option:: --no-silence-site-packages + + By default, mypy will suppress any error messages generated within :pep:`561` + compliant packages. Adding this flag will disable this behavior. + +.. option:: --fast-module-lookup + + The default logic used to scan through search paths to resolve imports has a + quadratic worse-case behavior in some cases, which is for instance triggered + by a large number of folders sharing a top-level namespace as in:: + + foo/ + company/ + foo/ + a.py + bar/ + company/ + bar/ + b.py + baz/ + company/ + baz/ + c.py + ... + + If you are in this situation, you can enable an experimental fast path by + setting the :option:`--fast-module-lookup` option. + + +.. option:: --no-namespace-packages + + This flag disables import discovery of namespace packages (see :pep:`420`). + In particular, this prevents discovery of packages that don't have an + ``__init__.py`` (or ``__init__.pyi``) file. + + This flag affects how mypy finds modules and packages explicitly passed on + the command line. It also affects how mypy determines fully qualified module + names for files passed on the command line. See :ref:`Mapping file paths to + modules ` for details. + + +.. _platform-configuration: + +Platform configuration +********************** + +By default, mypy will assume that you intend to run your code using the same +operating system and Python version you are using to run mypy itself. The +following flags let you modify this behavior. + +For more information on how to use these flags, see :ref:`version_and_platform_checks`. + +.. option:: --python-version X.Y + + This flag will make mypy type check your code as if it were + run under Python version X.Y. Without this option, mypy will default to using + whatever version of Python is running mypy. + + This flag will attempt to find a Python executable of the corresponding + version to search for :pep:`561` compliant packages. If you'd like to + disable this, use the :option:`--no-site-packages` flag (see + :ref:`import-discovery` for more details). + +.. option:: --platform PLATFORM + + This flag will make mypy type check your code as if it were + run under the given operating system. Without this option, mypy will + default to using whatever operating system you are currently using. + + The ``PLATFORM`` parameter may be any string supported by + :py:data:`sys.platform`. + +.. _always-true: + +.. option:: --always-true NAME + + This flag will treat all variables named ``NAME`` as + compile-time constants that are always true. This flag may + be repeated. + +.. option:: --always-false NAME + + This flag will treat all variables named ``NAME`` as + compile-time constants that are always false. This flag may + be repeated. + + +.. _disallow-dynamic-typing: + +Disallow dynamic typing +*********************** + +The ``Any`` type is used to represent a value that has a :ref:`dynamic type `. +The ``--disallow-any`` family of flags will disallow various uses of the ``Any`` type in +a module -- this lets us strategically disallow the use of dynamic typing in a controlled way. + +The following options are available: + +.. option:: --disallow-any-unimported + + This flag disallows usage of types that come from unfollowed imports + (such types become aliases for ``Any``). Unfollowed imports occur either + when the imported module does not exist or when :option:`--follow-imports=skip <--follow-imports>` + is set. + +.. option:: --disallow-any-expr + + This flag disallows all expressions in the module that have type ``Any``. + If an expression of type ``Any`` appears anywhere in the module + mypy will output an error unless the expression is immediately + used as an argument to :py:func:`~typing.cast` or assigned to a variable with an + explicit type annotation. + + In addition, declaring a variable of type ``Any`` + or casting to type ``Any`` is not allowed. Note that calling functions + that take parameters of type ``Any`` is still allowed. + +.. option:: --disallow-any-decorated + + This flag disallows functions that have ``Any`` in their signature + after decorator transformation. + +.. option:: --disallow-any-explicit + + This flag disallows explicit ``Any`` in type positions such as type + annotations and generic type parameters. + +.. option:: --disallow-any-generics + + This flag disallows usage of generic types that do not specify explicit + type parameters. For example, you can't use a bare ``x: list``. Instead, you + must always write something like ``x: list[int]``. + +.. option:: --disallow-subclassing-any + + This flag reports an error whenever a class subclasses a value of + type ``Any``. This may occur when the base class is imported from + a module that doesn't exist (when using + :option:`--ignore-missing-imports`) or is + ignored due to :option:`--follow-imports=skip <--follow-imports>` or a + ``# type: ignore`` comment on the ``import`` statement. + + Since the module is silenced, the imported class is given a type of ``Any``. + By default mypy will assume that the subclass correctly inherited + the base class even though that may not actually be the case. This + flag makes mypy raise an error instead. + + +.. _untyped-definitions-and-calls: + +Untyped definitions and calls +***************************** + +The following flags configure how mypy handles untyped function +definitions or calls. + +.. option:: --disallow-untyped-calls + + This flag reports an error whenever a function with type annotations + calls a function defined without annotations. + +.. option:: --untyped-calls-exclude + + This flag allows to selectively disable :option:`--disallow-untyped-calls` + for functions and methods defined in specific packages, modules, or classes. + Note that each exclude entry acts as a prefix. For example (assuming there + are no type annotations for ``third_party_lib`` available): + + .. code-block:: python + + # mypy --disallow-untyped-calls + # --untyped-calls-exclude=third_party_lib.module_a + # --untyped-calls-exclude=foo.A + from third_party_lib.module_a import some_func + from third_party_lib.module_b import other_func + import foo + + some_func() # OK, function comes from module `third_party_lib.module_a` + other_func() # E: Call to untyped function "other_func" in typed context + + foo.A().meth() # OK, method was defined in class `foo.A` + foo.B().meth() # E: Call to untyped function "meth" in typed context + + # file foo.py + class A: + def meth(self): pass + class B: + def meth(self): pass + +.. option:: --disallow-untyped-defs + + This flag reports an error whenever it encounters a function definition + without type annotations or with incomplete type annotations. + (a superset of :option:`--disallow-incomplete-defs`). + + For example, it would report an error for :code:`def f(a, b)` and :code:`def f(a: int, b)`. + +.. option:: --disallow-incomplete-defs + + This flag reports an error whenever it encounters a partly annotated + function definition, while still allowing entirely unannotated definitions. + + For example, it would report an error for :code:`def f(a: int, b)` but not :code:`def f(a, b)`. + +.. option:: --check-untyped-defs + + This flag is less severe than the previous two options -- it type checks + the body of every function, regardless of whether it has type annotations. + (By default the bodies of functions without annotations are not type + checked.) + + It will assume all arguments have type ``Any`` and always infer ``Any`` + as the return type. + +.. option:: --disallow-untyped-decorators + + This flag reports an error whenever a function with type annotations + is decorated with a decorator without annotations. + + +.. _none-and-optional-handling: + +None and Optional handling +************************** + +The following flags adjust how mypy handles values of type ``None``. + +.. _implicit-optional: + +.. option:: --implicit-optional + + This flag causes mypy to treat parameters with a ``None`` + default value as having an implicit optional type (``T | None``). + + For example, if this flag is set, mypy would assume that the ``x`` + parameter is actually of type ``int | None`` in the code snippet below, + since the default parameter is ``None``: + + .. code-block:: python + + def foo(x: int = None) -> None: + print(x) + + **Note:** This was disabled by default starting in mypy 0.980. + +.. _no_strict_optional: + +.. option:: --no-strict-optional + + This flag effectively disables checking of optional + types and ``None`` values. With this option, mypy doesn't + generally check the use of ``None`` values -- it is treated + as compatible with every type. + + .. warning:: + + ``--no-strict-optional`` is evil. Avoid using it and definitely do + not use it without understanding what it does. + + +.. _configuring-warnings: + +Configuring warnings +******************** + +The following flags enable warnings for code that is sound but is +potentially problematic or redundant in some way. + +.. option:: --warn-redundant-casts + + This flag will make mypy report an error whenever your code uses + an unnecessary cast that can safely be removed. + +.. option:: --warn-unused-ignores + + This flag will make mypy report an error whenever your code uses + a ``# type: ignore`` comment on a line that is not actually + generating an error message. + + This flag, along with the :option:`--warn-redundant-casts` flag, are both + particularly useful when you are upgrading mypy. Previously, + you may have needed to add casts or ``# type: ignore`` annotations + to work around bugs in mypy or missing stubs for 3rd party libraries. + + These two flags let you discover cases where either workarounds are + no longer necessary. + +.. option:: --no-warn-no-return + + By default, mypy will generate errors when a function is missing + return statements in some execution paths. The only exceptions + are when: + + - The function has a ``None`` or ``Any`` return type + - The function has an empty body and is marked as an abstract method, + is in a protocol class, or is in a stub file + - The execution path can never return; for example, if an exception + is always raised + + Passing in :option:`--no-warn-no-return` will disable these error + messages in all cases. + +.. option:: --warn-return-any + + This flag causes mypy to generate a warning when returning a value + with type ``Any`` from a function declared with a non-``Any`` return type. + +.. option:: --warn-unreachable + + This flag will make mypy report an error whenever it encounters + code determined to be unreachable or redundant after performing type analysis. + This can be a helpful way of detecting certain kinds of bugs in your code. + + For example, enabling this flag will make mypy report that the ``x > 7`` + check is redundant and that the ``else`` block below is unreachable. + + .. code-block:: python + + def process(x: int) -> None: + # Error: Right operand of "or" is never evaluated + if isinstance(x, int) or x > 7: + # Error: Unsupported operand types for + ("int" and "str") + print(x + "bad") + else: + # Error: 'Statement is unreachable' error + print(x + "bad") + + To help prevent mypy from generating spurious warnings, the "Statement is + unreachable" warning will be silenced in exactly two cases: + + 1. When the unreachable statement is a ``raise`` statement, is an + ``assert False`` statement, or calls a function that has the :py:data:`~typing.NoReturn` + return type hint. In other words, when the unreachable statement + throws an error or terminates the program in some way. + 2. When the unreachable statement was *intentionally* marked as unreachable + using :ref:`version_and_platform_checks`. + + .. note:: + + Mypy currently cannot detect and report unreachable or redundant code + inside any functions using :ref:`type-variable-value-restriction`. + + This limitation will be removed in future releases of mypy. + +.. option:: --report-deprecated-as-note + + If error code ``deprecated`` is enabled, mypy emits errors if your code + imports or uses deprecated features. This flag converts such errors to + notes, causing mypy to eventually finish with a zero exit code. Features + are considered deprecated when decorated with ``warnings.deprecated``. + +.. _miscellaneous-strictness-flags: + +Miscellaneous strictness flags +****************************** + +This section documents any other flags that do not neatly fall under any +of the above sections. + +.. option:: --allow-untyped-globals + + This flag causes mypy to suppress errors caused by not being able to fully + infer the types of global and class variables. + +.. option:: --allow-redefinition + + By default, mypy won't allow a variable to be redefined with an + unrelated type. This flag enables redefinition of a variable with an + arbitrary type *in some contexts*: only redefinitions within the + same block and nesting depth as the original definition are allowed. + Example where this can be useful: + + .. code-block:: python + + def process(items: list[str]) -> None: + # 'items' has type list[str] + items = [item.split() for item in items] + # 'items' now has type list[list[str]] + + The variable must be used before it can be redefined: + + .. code-block:: python + + def process(items: list[str]) -> None: + items = "mypy" # invalid redefinition to str because the variable hasn't been used yet + print(items) + items = "100" # valid, items now has type str + items = int(items) # valid, items now has type int + +.. option:: --local-partial-types + + In mypy, the most common cases for partial types are variables initialized using ``None``, + but without explicit ``X | None`` annotations. By default, mypy won't check partial types + spanning module top level or class top level. This flag changes the behavior to only allow + partial types at local level, therefore it disallows inferring variable type for ``None`` + from two assignments in different scopes. For example: + + .. code-block:: python + + a = None # Need type annotation here if using --local-partial-types + b: int | None = None + + class Foo: + bar = None # Need type annotation here if using --local-partial-types + baz: int | None = None + + def __init__(self) -> None: + self.bar = 1 + + reveal_type(Foo().bar) # 'int | None' without --local-partial-types + + Note: this option is always implicitly enabled in mypy daemon and will become + enabled by default for mypy in a future release. + +.. option:: --no-implicit-reexport + + By default, imported values to a module are treated as exported and mypy allows + other modules to import them. This flag changes the behavior to not re-export unless + the item is imported using from-as or is included in ``__all__``. Note this is + always treated as enabled for stub files. For example: + + .. code-block:: python + + # This won't re-export the value + from foo import bar + + # Neither will this + from foo import bar as bang + + # This will re-export it as bar and allow other modules to import it + from foo import bar as bar + + # This will also re-export bar + from foo import bar + __all__ = ['bar'] + + +.. option:: --strict-equality + + By default, mypy allows always-false comparisons like ``42 == 'no'``. + Use this flag to prohibit such comparisons of non-overlapping types, and + similar identity and container checks: + + .. code-block:: python + + items: list[int] + if 'some string' in items: # Error: non-overlapping container check! + ... + + text: str + if text != b'other bytes': # Error: non-overlapping equality check! + ... + + assert text is not None # OK, check against None is allowed as a special case. + +.. option:: --extra-checks + + This flag enables additional checks that are technically correct but may be + impractical in real code. In particular, it prohibits partial overlap in + ``TypedDict`` updates, and makes arguments prepended via ``Concatenate`` + positional-only. For example: + + .. code-block:: python + + from typing import TypedDict + + class Foo(TypedDict): + a: int + + class Bar(TypedDict): + a: int + b: int + + def test(foo: Foo, bar: Bar) -> None: + # This is technically unsafe since foo can have a subtype of Foo at + # runtime, where type of key "b" is incompatible with int, see below + bar.update(foo) + + class Bad(Foo): + b: str + bad: Bad = {"a": 0, "b": "no"} + test(bad, bar) + +.. option:: --strict + + This flag mode enables all optional error checking flags. You can see the + list of flags enabled by strict mode in the full :option:`mypy --help` output. + + Note: the exact list of flags enabled by running :option:`--strict` may change + over time. + +.. option:: --disable-error-code + + This flag allows disabling one or multiple error codes globally. + See :ref:`error-codes` for more information. + + .. code-block:: python + + # no flag + x = 'a string' + x.trim() # error: "str" has no attribute "trim" [attr-defined] + + # When using --disable-error-code attr-defined + x = 'a string' + x.trim() + +.. option:: --enable-error-code + + This flag allows enabling one or multiple error codes globally. + See :ref:`error-codes` for more information. + + Note: This flag will override disabled error codes from the + :option:`--disable-error-code ` flag. + + .. code-block:: python + + # When using --disable-error-code attr-defined + x = 'a string' + x.trim() + + # --disable-error-code attr-defined --enable-error-code attr-defined + x = 'a string' + x.trim() # error: "str" has no attribute "trim" [attr-defined] + +.. _configuring-error-messages: + +Configuring error messages +************************** + +The following flags let you adjust how much detail mypy displays +in error messages. + +.. option:: --show-error-context + + This flag will precede all errors with "note" messages explaining the + context of the error. For example, consider the following program: + + .. code-block:: python + + class Test: + def foo(self, x: int) -> int: + return x + "bar" + + Mypy normally displays an error message that looks like this:: + + main.py:3: error: Unsupported operand types for + ("int" and "str") + + If we enable this flag, the error message now looks like this:: + + main.py: note: In member "foo" of class "Test": + main.py:3: error: Unsupported operand types for + ("int" and "str") + +.. option:: --show-column-numbers + + This flag will add column offsets to error messages. + For example, the following indicates an error in line 12, column 9 + (note that column offsets are 0-based):: + + main.py:12:9: error: Unsupported operand types for / ("int" and "str") + +.. option:: --show-error-code-links + + This flag will also display a link to error code documentation, anchored to the error code reported by mypy. + The corresponding error code will be highlighted within the documentation page. + If we enable this flag, the error message now looks like this:: + + main.py:3: error: Unsupported operand types for - ("int" and "str") [operator] + main.py:3: note: See 'https://mypy.rtfd.io/en/stable/_refs.html#code-operator' for more info + + + +.. option:: --show-error-end + + This flag will make mypy show not just that start position where + an error was detected, but also the end position of the relevant expression. + This way various tools can easily highlight the whole error span. The format is + ``file:line:column:end_line:end_column``. This option implies + ``--show-column-numbers``. + +.. option:: --hide-error-codes + + This flag will hide the error code ``[]`` from error messages. By default, the error + code is shown after each error message:: + + prog.py:1: error: "str" has no attribute "trim" [attr-defined] + + See :ref:`error-codes` for more information. + +.. option:: --pretty + + Use visually nicer output in error messages: use soft word wrap, + show source code snippets, and show error location markers. + +.. option:: --no-color-output + + This flag will disable color output in error messages, enabled by default. + +.. option:: --no-error-summary + + This flag will disable error summary. By default mypy shows a summary line + including total number of errors, number of files with errors, and number + of files checked. + +.. option:: --show-absolute-path + + Show absolute paths to files. + +.. option:: --soft-error-limit N + + This flag will adjust the limit after which mypy will (sometimes) + disable reporting most additional errors. The limit only applies + if it seems likely that most of the remaining errors will not be + useful or they may be overly noisy. If ``N`` is negative, there is + no limit. The default limit is -1. + +.. option:: --force-uppercase-builtins + + Always use ``List`` instead of ``list`` in error messages, + even on Python 3.9+. + +.. option:: --force-union-syntax + + Always use ``Union[]`` and ``Optional[]`` for union types + in error messages (instead of the ``|`` operator), + even on Python 3.10+. + + +.. _incremental: + +Incremental mode +**************** + +By default, mypy will store type information into a cache. Mypy +will use this information to avoid unnecessary recomputation when +it type checks your code again. This can help speed up the type +checking process, especially when most parts of your program have +not changed since the previous mypy run. + +If you want to speed up how long it takes to recheck your code +beyond what incremental mode can offer, try running mypy in +:ref:`daemon mode `. + +.. option:: --no-incremental + + This flag disables incremental mode: mypy will no longer reference + the cache when re-run. + + Note that mypy will still write out to the cache even when + incremental mode is disabled: see the :option:`--cache-dir` flag below + for more details. + +.. option:: --cache-dir DIR + + By default, mypy stores all cache data inside of a folder named + ``.mypy_cache`` in the current directory. This flag lets you + change this folder. This flag can also be useful for controlling + cache use when using :ref:`remote caching `. + + This setting will override the ``MYPY_CACHE_DIR`` environment + variable if it is set. + + Mypy will also always write to the cache even when incremental + mode is disabled so it can "warm up" the cache. To disable + writing to the cache, use ``--cache-dir=/dev/null`` (UNIX) + or ``--cache-dir=nul`` (Windows). + +.. option:: --sqlite-cache + + Use an `SQLite`_ database to store the cache. + +.. option:: --cache-fine-grained + + Include fine-grained dependency information in the cache for the mypy daemon. + +.. option:: --skip-version-check + + By default, mypy will ignore cache data generated by a different + version of mypy. This flag disables that behavior. + +.. option:: --skip-cache-mtime-checks + + Skip cache internal consistency checks based on mtime. + + +Advanced options +**************** + +The following flags are useful mostly for people who are interested +in developing or debugging mypy internals. + +.. option:: --pdb + + This flag will invoke the Python debugger when mypy encounters + a fatal error. + +.. option:: --show-traceback, --tb + + If set, this flag will display a full traceback when mypy + encounters a fatal error. + +.. option:: --raise-exceptions + + Raise exception on fatal error. + +.. option:: --custom-typing-module MODULE + + This flag lets you use a custom module as a substitute for the + :py:mod:`typing` module. + +.. option:: --custom-typeshed-dir DIR + + This flag specifies the directory where mypy looks for standard library typeshed + stubs, instead of the typeshed that ships with mypy. This is + primarily intended to make it easier to test typeshed changes before + submitting them upstream, but also allows you to use a forked version of + typeshed. + + Note that this doesn't affect third-party library stubs. To test third-party stubs, + for example try ``MYPYPATH=stubs/six mypy ...``. + +.. _warn-incomplete-stub: + +.. option:: --warn-incomplete-stub + + This flag modifies both the :option:`--disallow-untyped-defs` and + :option:`--disallow-incomplete-defs` flags so they also report errors + if stubs in typeshed are missing type annotations or has incomplete + annotations. If both flags are missing, :option:`--warn-incomplete-stub` + also does nothing. + + This flag is mainly intended to be used by people who want contribute + to typeshed and would like a convenient way to find gaps and omissions. + + If you want mypy to report an error when your codebase *uses* an untyped + function, whether that function is defined in typeshed or not, use the + :option:`--disallow-untyped-calls` flag. See :ref:`untyped-definitions-and-calls` + for more details. + +.. _shadow-file: + +.. option:: --shadow-file SOURCE_FILE SHADOW_FILE + + When mypy is asked to type check ``SOURCE_FILE``, this flag makes mypy + read from and type check the contents of ``SHADOW_FILE`` instead. However, + diagnostics will continue to refer to ``SOURCE_FILE``. + + Specifying this argument multiple times (``--shadow-file X1 Y1 --shadow-file X2 Y2``) + will allow mypy to perform multiple substitutions. + + This allows tooling to create temporary files with helpful modifications + without having to change the source file in place. For example, suppose we + have a pipeline that adds ``reveal_type`` for certain variables. + This pipeline is run on ``original.py`` to produce ``temp.py``. + Running ``mypy --shadow-file original.py temp.py original.py`` will then + cause mypy to type check the contents of ``temp.py`` instead of ``original.py``, + but error messages will still reference ``original.py``. + + +Report generation +***************** + +If these flags are set, mypy will generate a report in the specified +format into the specified directory. + +.. option:: --any-exprs-report DIR + + Causes mypy to generate a text file report documenting how many + expressions of type ``Any`` are present within your codebase. + +.. option:: --cobertura-xml-report DIR + + Causes mypy to generate a Cobertura XML type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + +.. option:: --html-report / --xslt-html-report DIR + + Causes mypy to generate an HTML type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + +.. option:: --linecount-report DIR + + Causes mypy to generate a text file report documenting the functions + and lines that are typed and untyped within your codebase. + +.. option:: --linecoverage-report DIR + + Causes mypy to generate a JSON file that maps each source file's + absolute filename to a list of line numbers that belong to typed + functions in that file. + +.. option:: --lineprecision-report DIR + + Causes mypy to generate a flat text file report with per-module + statistics of how many lines are typechecked etc. + +.. option:: --txt-report / --xslt-txt-report DIR + + Causes mypy to generate a text file type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + +.. option:: --xml-report DIR + + Causes mypy to generate an XML type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + + +Enabling incomplete/experimental features +***************************************** + +.. option:: --enable-incomplete-feature {PreciseTupleTypes, InlineTypedDict} + + Some features may require several mypy releases to implement, for example + due to their complexity, potential for backwards incompatibility, or + ambiguous semantics that would benefit from feedback from the community. + You can enable such features for early preview using this flag. Note that + it is not guaranteed that all features will be ultimately enabled by + default. In *rare cases* we may decide to not go ahead with certain + features. + +List of currently incomplete/experimental features: + +* ``PreciseTupleTypes``: this feature will infer more precise tuple types in + various scenarios. Before variadic types were added to the Python type system + by :pep:`646`, it was impossible to express a type like "a tuple with + at least two integers". The best type available was ``tuple[int, ...]``. + Therefore, mypy applied very lenient checking for variable-length tuples. + Now this type can be expressed as ``tuple[int, int, *tuple[int, ...]]``. + For such more precise types (when explicitly *defined* by a user) mypy, + for example, warns about unsafe index access, and generally handles them + in a type-safe manner. However, to avoid problems in existing code, mypy + does not *infer* these precise types when it technically can. Here are + notable examples where ``PreciseTupleTypes`` infers more precise types: + + .. code-block:: python + + numbers: tuple[int, ...] + + more_numbers = (1, *numbers, 1) + reveal_type(more_numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, *tuple[int, ...], int] + + other_numbers = (1, 1) + numbers + reveal_type(other_numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, int, *tuple[int, ...]] + + if len(numbers) > 2: + reveal_type(numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, int, int, *tuple[int, ...]] + else: + reveal_type(numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int] + +* ``InlineTypedDict``: this feature enables non-standard syntax for inline + :ref:`TypedDicts `, for example: + + .. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + + +Miscellaneous +************* + +.. option:: --install-types + + This flag causes mypy to install known missing stub packages for + third-party libraries using pip. It will display the pip command + that will be run, and expects a confirmation before installing + anything. For security reasons, these stubs are limited to only a + small subset of manually selected packages that have been + verified by the typeshed team. These packages include only stub + files and no executable code. + + If you use this option without providing any files or modules to + type check, mypy will install stub packages suggested during the + previous mypy run. If there are files or modules to type check, + mypy first type checks those, and proposes to install missing + stubs at the end of the run, but only if any missing modules were + detected. + + .. note:: + + This is new in mypy 0.900. Previous mypy versions included a + selection of third-party package stubs, instead of having + them installed separately. + +.. option:: --non-interactive + + When used together with :option:`--install-types `, this causes mypy to install all suggested stub + packages using pip without asking for confirmation, and then + continues to perform type checking using the installed stubs, if + some files or modules are provided to type check. + + This is implemented as up to two mypy runs internally. The first run + is used to find missing stub packages, and output is shown from + this run only if no missing stub packages were found. If missing + stub packages were found, they are installed and then another run + is performed. + +.. option:: --junit-xml JUNIT_XML + + Causes mypy to generate a JUnit XML test result document with + type checking results. This can make it easier to integrate mypy + with continuous integration (CI) tools. + +.. option:: --find-occurrences CLASS.MEMBER + + This flag will make mypy print out all usages of a class member + based on static type information. This feature is experimental. + +.. option:: --scripts-are-modules + + This flag will give command line arguments that appear to be + scripts (i.e. files whose name does not end in ``.py``) + a module name derived from the script name rather than the fixed + name :py:mod:`__main__`. + + This lets you check more than one script in a single mypy invocation. + (The default :py:mod:`__main__` is technically more correct, but if you + have many scripts that import a large package, the behavior enabled + by this flag is often more convenient.) + +.. _lxml: https://pypi.org/project/lxml/ +.. _SQLite: https://www.sqlite.org/ diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index 5a6ed4786ad5..39954b8e332a 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -1,83 +1,271 @@ .. _common_issues: -Dealing with common issues -========================== +Common issues and solutions +=========================== -Statically typed function bodies are often identical to normal Python -code, but sometimes you need to do things slightly differently. This -section introduces some of the most common cases which require -different conventions in statically typed code. +This section has examples of cases when you need to update your code +to use static typing, and ideas for working around issues if mypy +doesn't work as expected. Statically typed code is often identical to +normal Python code (except for type annotations), but sometimes you need +to do things slightly differently. -Types of empty collections --------------------------- +.. _annotations_needed: -You need to specify the type when you assign an empty list or -dict to a new variable, as mentioned earlier: +No errors reported for obviously wrong code +------------------------------------------- + +There are several common reasons why obviously wrong code is not +flagged as an error. + +**The function containing the error is not annotated.** + +Functions that +do not have any annotations (neither for any argument nor for the +return type) are not type-checked, and even the most blatant type +errors (e.g. ``2 + 'a'``) pass silently. The solution is to add +annotations. Where that isn't possible, functions without annotations +can be checked using :option:`--check-untyped-defs `. + +Example: .. code-block:: python - a = List[int]() # Explicit type required in statically typed code + def foo(a): + return '(' + a.split() + ')' # No error! -Alternatively, you can use a type annotation in a comment: +This gives no error even though ``a.split()`` is "obviously" a list +(the author probably meant ``a.strip()``). The error is reported +once you add annotations: .. code-block:: python - a = [] # type: List[int] + def foo(a: str) -> str: + return '(' + a.split() + ')' + # error: Unsupported operand types for + ("str" and "list[str]") -Without the annotation the type checker has no way of figuring out the -precise type of ``a``. +If you don't know what types to add, you can use ``Any``, but beware: -You can use a simple empty list literal in a dynamically typed function (as the -type of ``a`` would be implicitly ``Any`` and need not be inferred), or if type -of the variable has been declared or inferred before: +**One of the values involved has type 'Any'.** + +Extending the above +example, if we were to leave out the annotation for ``a``, we'd get +no error: .. code-block:: python - a = [] # Okay if type of a known + def foo(a) -> str: + return '(' + a.split() + ')' # No error! + +The reason is that if the type of ``a`` is unknown, the type of +``a.split()`` is also unknown, so it is inferred as having type +``Any``, and it is no error to add a string to an ``Any``. + +If you're having trouble debugging such situations, +:ref:`reveal_type() ` might come in handy. -Sometimes you can avoid the explicit list item type by using a list -comprehension. Here a type annotation is needed: +Note that sometimes library stubs with imprecise type information +can be a source of ``Any`` values. + +:py:meth:`__init__ ` **method has no annotated +arguments and no return type annotation.** + +This is basically a combination of the two cases above, in that ``__init__`` +without annotations can cause ``Any`` types leak into instance variables: .. code-block:: python - l = List[int]() - for i in range(n): - l.append(i * i) + class Bad: + def __init__(self): + self.value = "asdf" + 1 + "asdf" # No error! + + bad = Bad() + bad.value + 1 # No error! + reveal_type(bad) # Revealed type is "__main__.Bad" + reveal_type(bad.value) # Revealed type is "Any" + + class Good: + def __init__(self) -> None: # Explicitly return None + self.value = value + + +**Some imports may be silently ignored**. + +A common source of unexpected ``Any`` values is the +:option:`--ignore-missing-imports ` flag. + +When you use :option:`--ignore-missing-imports `, +any imported module that cannot be found is silently replaced with ``Any``. + +To help debug this, simply leave out +:option:`--ignore-missing-imports `. +As mentioned in :ref:`fix-missing-imports`, setting ``ignore_missing_imports=True`` +on a per-module basis will make bad surprises less likely and is highly encouraged. + +Use of the :option:`--follow-imports=skip ` flags can also +cause problems. Use of these flags is strongly discouraged and only required in +relatively niche situations. See :ref:`follow-imports` for more information. + +**mypy considers some of your code unreachable**. + +See :ref:`unreachable` for more information. + +**A function annotated as returning a non-optional type returns 'None' +and mypy doesn't complain**. + +.. code-block:: python + + def foo() -> str: + return None # No error! + +You may have disabled strict optional checking (see +:ref:`--no-strict-optional ` for more). + +.. _silencing_checker: + +Spurious errors and locally silencing the checker +------------------------------------------------- + +You can use a ``# type: ignore`` comment to silence the type checker +on a particular line. For example, let's say our code is using +the C extension module ``frobnicate``, and there's no stub available. +Mypy will complain about this, as it has no information about the +module: + +.. code-block:: python + + import frobnicate # Error: No module "frobnicate" + frobnicate.start() + +You can add a ``# type: ignore`` comment to tell mypy to ignore this +error: + +.. code-block:: python + + import frobnicate # type: ignore + frobnicate.start() # Okay! + +The second line is now fine, since the ignore comment causes the name +``frobnicate`` to get an implicit ``Any`` type. + +.. note:: + + You can use the form ``# type: ignore[]`` to only ignore + specific errors on the line. This way you are less likely to + silence unexpected errors that are not safe to ignore, and this + will also document what the purpose of the comment is. See + :ref:`error-codes` for more information. .. note:: - A future mypy version may be able to deal with cases such as the - above without type annotations. + The ``# type: ignore`` comment will only assign the implicit ``Any`` + type if mypy cannot find information about that particular module. So, + if we did have a stub available for ``frobnicate`` then mypy would + ignore the ``# type: ignore`` comment and typecheck the stub as usual. + +Another option is to explicitly annotate values with type ``Any`` -- +mypy will let you perform arbitrary operations on ``Any`` +values. Sometimes there is no more precise type you can use for a +particular value, especially if you use dynamic Python features +such as :py:meth:`__getattr__ `: + +.. code-block:: python + + class Wrapper: + ... + def __getattr__(self, a: str) -> Any: + return getattr(self._wrapped, a) + +Finally, you can create a stub file (``.pyi``) for a file that +generates spurious errors. Mypy will only look at the stub file +and ignore the implementation, since stub files take precedence +over ``.py`` files. + +Ignoring a whole file +--------------------- + +* To only ignore errors, use a top-level ``# mypy: ignore-errors`` comment instead. +* To only ignore errors with a specific error code, use a top-level + ``# mypy: disable-error-code="..."`` comment. Example: ``# mypy: disable-error-code="truthy-bool, ignore-without-code"`` +* To replace the contents of a module with ``Any``, use a per-module ``follow_imports = skip``. + See :ref:`Following imports ` for details. + +Note that a ``# type: ignore`` comment at the top of a module (before any statements, +including imports or docstrings) has the effect of ignoring the entire contents of the module. +This behaviour can be surprising and result in +"Module ... has no attribute ... [attr-defined]" errors. + +Issues with code at runtime +--------------------------- + +Idiomatic use of type annotations can sometimes run up against what a given +version of Python considers legal code. These can result in some of the +following errors when trying to run your code: + +* ``ImportError`` from circular imports +* ``NameError: name "X" is not defined`` from forward references +* ``TypeError: 'type' object is not subscriptable`` from types that are not generic at runtime +* ``ImportError`` or ``ModuleNotFoundError`` from use of stub definitions not available at runtime +* ``TypeError: unsupported operand type(s) for |: 'type' and 'type'`` from use of new syntax + +For dealing with these, see :ref:`runtime_troubles`. + +Mypy runs are slow +------------------ + +If your mypy runs feel slow, you should probably use the :ref:`mypy +daemon `, which can speed up incremental mypy runtimes by +a factor of 10 or more. :ref:`Remote caching ` can +make cold mypy runs several times faster. + +Types of empty collections +-------------------------- + +You often need to specify the type when you assign an empty list or +dict to a new variable, as mentioned earlier: + +.. code-block:: python + + a: list[int] = [] + +Without the annotation mypy can't always figure out the +precise type of ``a``. -No type annotation needed if using a list comprehension: +You can use a simple empty list literal in a dynamically typed function (as the +type of ``a`` would be implicitly ``Any`` and need not be inferred), if type +of the variable has been declared or inferred before, or if you perform a simple +modification operation in the same scope (such as ``append`` for a list): .. code-block:: python - l = [i * i for i in range(n)] + a = [] # Okay because followed by append, inferred type list[int] + for i in range(n): + a.append(i * i) -However, in more complex cases the explicit type annotation can -improve the clarity of your code, whereas a complex list comprehension -can make your code difficult to understand. +However, in more complex cases an explicit type annotation can be +required (mypy will tell you this). Often the annotation can +make your code easier to understand, so it doesn't only help mypy but +everybody who is reading the code! Redefinitions with incompatible types ------------------------------------- Each name within a function only has a single 'declared' type. You can reuse for loop indices etc., but if you want to use a variable with -multiple types within a single function, you may need to declare it -with the ``Any`` type. +multiple types within a single function, you may need to instead use +multiple variables (or maybe declare the variable with an ``Any`` type). .. code-block:: python def f() -> None: n = 1 ... - n = 'x' # Type error: n has type int + n = 'x' # error: Incompatible types in assignment (expression has type "str", variable has type "int") .. note:: - This is another limitation that could be lifted in a future mypy - version. + Using the :option:`--allow-redefinition ` + flag can suppress this error in several cases. Note that you can redefine a variable with a more *precise* or a more concrete type. For example, you can redefine a sequence (which does @@ -88,80 +276,546 @@ not support ``sort()``) as a list and sort it in-place: def f(x: Sequence[int]) -> None: # Type of x is Sequence[int] here; we don't know the concrete type. x = list(x) - # Type of x is List[int] here. + # Type of x is list[int] here. x.sort() # Okay! +See :ref:`type-narrowing` for more information. + +.. _variance: + +Invariance vs covariance +------------------------ + +Most mutable generic collections are invariant, and mypy considers all +user-defined generic classes invariant by default +(see :ref:`variance-of-generics` for motivation). This could lead to some +unexpected errors when combined with type inference. For example: + +.. code-block:: python + + class A: ... + class B(A): ... + + lst = [A(), A()] # Inferred type is list[A] + new_lst = [B(), B()] # inferred type is list[B] + lst = new_lst # mypy will complain about this, because List is invariant + +Possible strategies in such situations are: + +* Use an explicit type annotation: + + .. code-block:: python + + new_lst: list[A] = [B(), B()] + lst = new_lst # OK + +* Make a copy of the right hand side: + + .. code-block:: python + + lst = list(new_lst) # Also OK + +* Use immutable collections as annotations whenever possible: + + .. code-block:: python + + def f_bad(x: list[A]) -> A: + return x[0] + f_bad(new_lst) # Fails + + def f_good(x: Sequence[A]) -> A: + return x[0] + f_good(new_lst) # OK + Declaring a supertype as variable type -------------------------------------- Sometimes the inferred type is a subtype (subclass) of the desired type. The type inference uses the first assignment to infer the type -of a name (assume here that ``Shape`` is the base class of both -``Circle`` and ``Triangle``): +of a name: .. code-block:: python - shape = Circle() # Infer shape to be Circle - ... - shape = Triangle() # Type error: Triangle is not a Circle + class Shape: ... + class Circle(Shape): ... + class Triangle(Shape): ... + + shape = Circle() # mypy infers the type of shape to be Circle + shape = Triangle() # error: Incompatible types in assignment (expression has type "Triangle", variable has type "Circle") You can just give an explicit type for the variable in cases such the above example: .. code-block:: python - shape = Circle() # type: Shape # The variable s can be any Shape, - # not just Circle - ... - shape = Triangle() # OK + shape: Shape = Circle() # The variable s can be any Shape, not just Circle + shape = Triangle() # OK -Complex isinstance tests ------------------------- +Complex type tests +------------------ -If you use ``isinstance()`` tests or other kinds of runtime type -tests, you may have to add casts (this is similar to ``instanceof`` tests -in Java): +Mypy can usually infer the types correctly when using :py:func:`isinstance `, +:py:func:`issubclass `, +or ``type(obj) is some_class`` type tests, +and even :ref:`user-defined type guards `, +but for other kinds of checks you may need to add an +explicit type cast: .. code-block:: python - def f(o: object, x: int) -> None: - if isinstance(o, int) and x > 1: - n = cast(int, o) - g(n + 1) # o + 1 would be an error - ... + from collections.abc import Sequence + from typing import cast + + def find_first_str(a: Sequence[object]) -> str: + index = next((i for i, s in enumerate(a) if isinstance(s, str)), -1) + if index < 0: + raise ValueError('No str found') + + found = a[index] # Has type "object", despite the fact that we know it is "str" + return cast(str, found) # We need an explicit cast to make mypy happy + +Alternatively, you can use an ``assert`` statement together with some +of the supported type inference techniques: + +.. code-block:: python + + def find_first_str(a: Sequence[object]) -> str: + index = next((i for i, s in enumerate(a) if isinstance(s, str)), -1) + if index < 0: + raise ValueError('No str found') + + found = a[index] # Has type "object", despite the fact that we know it is "str" + assert isinstance(found, str) # Now, "found" will be narrowed to "str" + return found # No need for the explicit "cast()" anymore .. note:: - Note that the ``object`` type used in the above example is similar + Note that the :py:class:`object` type used in the above example is similar to ``Object`` in Java: it only supports operations defined for *all* - objects, such as equality and ``isinstance()``. The type ``Any``, + objects, such as equality and :py:func:`isinstance`. The type ``Any``, in contrast, supports all operations, even if they may fail at runtime. The cast above would have been unnecessary if the type of ``o`` was ``Any``. -Mypy can't infer the type of ``o`` after the ``isinstance()`` check -because of the ``and`` operator (this limitation will likely be lifted -in the future). We can write the above code without a cast by using a -nested if statemet: - -.. code-block:: python - - def f(o: object, x: int) -> None: - if isinstance(o, int): # Mypy understands a lone isinstance check - if x > 1: - g(o + 1) # Okay; type of o is inferred as int here - ... +.. note:: -Some consider casual use of ``isinstance()`` tests a sign of bad -programming style. Often a method override or a ``hasattr`` check -is a cleaner way of implementing functionality that depends on the -runtime types of values. However, use whatever techniques that work -for you. Sometimes ``isinstance`` tests *are* the cleanest way of -implementing a piece of functionality. + You can read more about type narrowing techniques :ref:`here `. -Type inference in mypy is designed to work well in common cases, to be +Type inference in Mypy is designed to work well in common cases, to be predictable and to let the type checker give useful error messages. More powerful type inference strategies often have complex -and difficult-to-prefict failure modes and could result in very +and difficult-to-predict failure modes and could result in very confusing error messages. The tradeoff is that you as a programmer sometimes have to give the type checker a little help. + +.. _version_and_platform_checks: + +Python version and system platform checks +----------------------------------------- + +Mypy supports the ability to perform Python version checks and platform +checks (e.g. Windows vs Posix), ignoring code paths that won't be run on +the targeted Python version or platform. This allows you to more effectively +typecheck code that supports multiple versions of Python or multiple operating +systems. + +More specifically, mypy will understand the use of :py:data:`sys.version_info` and +:py:data:`sys.platform` checks within ``if/elif/else`` statements. For example: + +.. code-block:: python + + import sys + + # Distinguishing between different versions of Python: + if sys.version_info >= (3, 8): + # Python 3.8+ specific definitions and imports + else: + # Other definitions and imports + + # Distinguishing between different operating systems: + if sys.platform.startswith("linux"): + # Linux-specific code + elif sys.platform == "darwin": + # Mac-specific code + elif sys.platform == "win32": + # Windows-specific code + else: + # Other systems + +As a special case, you can also use one of these checks in a top-level +(unindented) ``assert``; this makes mypy skip the rest of the file. +Example: + +.. code-block:: python + + import sys + + assert sys.platform != 'win32' + + # The rest of this file doesn't apply to Windows. + +Some other expressions exhibit similar behavior; in particular, +:py:data:`~typing.TYPE_CHECKING`, variables named ``MYPY``, and any variable +whose name is passed to :option:`--always-true ` or :option:`--always-false `. +(However, ``True`` and ``False`` are not treated specially!) + +.. note:: + + Mypy currently does not support more complex checks, and does not assign + any special meaning when assigning a :py:data:`sys.version_info` or :py:data:`sys.platform` + check to a variable. This may change in future versions of mypy. + +By default, mypy will use your current version of Python and your current +operating system as default values for :py:data:`sys.version_info` and +:py:data:`sys.platform`. + +To target a different Python version, use the :option:`--python-version X.Y ` flag. +For example, to verify your code typechecks if were run using Python 3.8, pass +in :option:`--python-version 3.8 ` from the command line. Note that you do not need +to have Python 3.8 installed to perform this check. + +To target a different operating system, use the :option:`--platform PLATFORM ` flag. +For example, to verify your code typechecks if it were run in Windows, pass +in :option:`--platform win32 `. See the documentation for :py:data:`sys.platform` +for examples of valid platform parameters. + +.. _reveal-type: + +Displaying the type of an expression +------------------------------------ + +You can use ``reveal_type(expr)`` to ask mypy to display the inferred +static type of an expression. This can be useful when you don't quite +understand how mypy handles a particular piece of code. Example: + +.. code-block:: python + + reveal_type((1, 'hello')) # Revealed type is "tuple[builtins.int, builtins.str]" + +You can also use ``reveal_locals()`` at any line in a file +to see the types of all local variables at once. Example: + +.. code-block:: python + + a = 1 + b = 'one' + reveal_locals() + # Revealed local types are: + # a: builtins.int + # b: builtins.str +.. note:: + + ``reveal_type`` and ``reveal_locals`` are only understood by mypy and + don't exist in Python. If you try to run your program, you'll have to + remove any ``reveal_type`` and ``reveal_locals`` calls before you can + run your code. Both are always available and you don't need to import + them. + +.. _silencing-linters: + +Silencing linters +----------------- + +In some cases, linters will complain about unused imports or code. In +these cases, you can silence them with a comment after type comments, or on +the same line as the import: + +.. code-block:: python + + # to silence complaints about unused imports + from typing import List # noqa + a = None # type: List[int] + + +To silence the linter on the same line as a type comment +put the linter comment *after* the type comment: + +.. code-block:: python + + a = some_complex_thing() # type: ignore # noqa + +Covariant subtyping of mutable protocol members is rejected +----------------------------------------------------------- + +Mypy rejects this because this is potentially unsafe. +Consider this example: + +.. code-block:: python + + from typing import Protocol + + class P(Protocol): + x: float + + def fun(arg: P) -> None: + arg.x = 3.14 + + class C: + x = 42 + c = C() + fun(c) # This is not safe + c.x << 5 # Since this will fail! + +To work around this problem consider whether "mutating" is actually part +of a protocol. If not, then one can use a :py:class:`@property ` in +the protocol definition: + +.. code-block:: python + + from typing import Protocol + + class P(Protocol): + @property + def x(self) -> float: + pass + + def fun(arg: P) -> None: + ... + + class C: + x = 42 + fun(C()) # OK + +Dealing with conflicting names +------------------------------ + +Suppose you have a class with a method whose name is the same as an +imported (or built-in) type, and you want to use the type in another +method signature. E.g.: + +.. code-block:: python + + class Message: + def bytes(self): + ... + def register(self, path: bytes): # error: Invalid type "mod.Message.bytes" + ... + +The third line elicits an error because mypy sees the argument type +``bytes`` as a reference to the method by that name. Other than +renaming the method, a workaround is to use an alias: + +.. code-block:: python + + bytes_ = bytes + class Message: + def bytes(self): + ... + def register(self, path: bytes_): + ... + +Using a development mypy build +------------------------------ + +You can install the latest development version of mypy from source. Clone the +`mypy repository on GitHub `_, and then run +``pip install`` locally: + +.. code-block:: text + + git clone https://github.com/python/mypy.git + cd mypy + python3 -m pip install --upgrade . + +To install a development version of mypy that is mypyc-compiled, see the +instructions at the `mypyc wheels repo `_. + +Variables vs type aliases +------------------------- + +Mypy has both *type aliases* and variables with types like ``type[...]``. These are +subtly different, and it's important to understand how they differ to avoid pitfalls. + +1. A variable with type ``type[...]`` is defined using an assignment with an + explicit type annotation: + + .. code-block:: python + + class A: ... + tp: type[A] = A + +2. You can define a type alias using an assignment without an explicit type annotation + at the top level of a module: + + .. code-block:: python + + class A: ... + Alias = A + + You can also use ``TypeAlias`` (:pep:`613`) to define an *explicit type alias*: + + .. code-block:: python + + from typing import TypeAlias # "from typing_extensions" in Python 3.9 and earlier + + class A: ... + Alias: TypeAlias = A + + You should always use ``TypeAlias`` to define a type alias in a class body or + inside a function. + +The main difference is that the target of an alias is precisely known statically, and this +means that they can be used in type annotations and other *type contexts*. Type aliases +can't be defined conditionally (unless using +:ref:`supported Python version and platform checks `): + + .. code-block:: python + + class A: ... + class B: ... + + if random() > 0.5: + Alias = A + else: + # error: Cannot assign multiple types to name "Alias" without an + # explicit "Type[...]" annotation + Alias = B + + tp: type[object] # "tp" is a variable with a type object value + if random() > 0.5: + tp = A + else: + tp = B # This is OK + + def fun1(x: Alias) -> None: ... # OK + def fun2(x: tp) -> None: ... # Error: "tp" is not valid as a type + +Incompatible overrides +---------------------- + +It's unsafe to override a method with a more specific argument type, +as it violates the `Liskov substitution principle +`_. +For return types, it's unsafe to override a method with a more general +return type. + +Other incompatible signature changes in method overrides, such as +adding an extra required parameter, or removing an optional parameter, +will also generate errors. The signature of a method in a subclass +should accept all valid calls to the base class method. Mypy +treats a subclass as a subtype of the base class. An instance of a +subclass is valid everywhere where an instance of the base class is +valid. + +This example demonstrates both safe and unsafe overrides: + +.. code-block:: python + + from collections.abc import Sequence, Iterable + + class A: + def test(self, t: Sequence[int]) -> Sequence[str]: + ... + + class GeneralizedArgument(A): + # A more general argument type is okay + def test(self, t: Iterable[int]) -> Sequence[str]: # OK + ... + + class NarrowerArgument(A): + # A more specific argument type isn't accepted + def test(self, t: list[int]) -> Sequence[str]: # Error + ... + + class NarrowerReturn(A): + # A more specific return type is fine + def test(self, t: Sequence[int]) -> List[str]: # OK + ... + + class GeneralizedReturn(A): + # A more general return type is an error + def test(self, t: Sequence[int]) -> Iterable[str]: # Error + ... + +You can use ``# type: ignore[override]`` to silence the error. Add it +to the line that generates the error, if you decide that type safety is +not necessary: + +.. code-block:: python + + class NarrowerArgument(A): + def test(self, t: List[int]) -> Sequence[str]: # type: ignore[override] + ... + +.. _unreachable: + +Unreachable code +---------------- + +Mypy may consider some code as *unreachable*, even if it might not be +immediately obvious why. It's important to note that mypy will *not* +type check such code. Consider this example: + +.. code-block:: python + + class Foo: + bar: str = '' + + def bar() -> None: + foo: Foo = Foo() + return + x: int = 'abc' # Unreachable -- no error + +It's easy to see that any statement after ``return`` is unreachable, +and hence mypy will not complain about the mis-typed code below +it. For a more subtle example, consider this code: + +.. code-block:: python + + class Foo: + bar: str = '' + + def bar() -> None: + foo: Foo = Foo() + assert foo.bar is None + x: int = 'abc' # Unreachable -- no error + +Again, mypy will not report any errors. The type of ``foo.bar`` is +``str``, and mypy reasons that it can never be ``None``. Hence the +``assert`` statement will always fail and the statement below will +never be executed. (Note that in Python, ``None`` is not an empty +reference but an object of type ``None``.) + +In this example mypy will go on to check the last line and report an +error, since mypy thinks that the condition could be either True or +False: + +.. code-block:: python + + class Foo: + bar: str = '' + + def bar() -> None: + foo: Foo = Foo() + if not foo.bar: + return + x: int = 'abc' # Reachable -- error + +If you use the :option:`--warn-unreachable ` flag, mypy will generate +an error about each unreachable code block. + +Narrowing and inner functions +----------------------------- + +Because closures in Python are late-binding (https://docs.python-guide.org/writing/gotchas/#late-binding-closures), +mypy will not narrow the type of a captured variable in an inner function. +This is best understood via an example: + +.. code-block:: python + + def foo(x: int | None) -> Callable[[], int]: + if x is None: + x = 5 + print(x + 1) # mypy correctly deduces x must be an int here + def inner() -> int: + return x + 1 # but (correctly) complains about this line + + x = None # because x could later be assigned None + return inner + + inner = foo(5) + inner() # this will raise an error when called + +To get this code to type check, you could assign ``y = x`` after ``x`` has been +narrowed, and use ``y`` in the inner function, or add an assert in the inner +function. diff --git a/docs/source/conf.py b/docs/source/conf.py index 49cd56403c9f..ddc9923c6c93 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,58 +12,65 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys +from __future__ import annotations + import os +import sys + +from sphinx.application import Sphinx +from sphinx.util.docfields import Field # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath("../..")) + +from mypy.version import __version__ as mypy_version # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ["sphinx.ext.intersphinx", "docs.source.html_builder", "myst_parser"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'Mypy' -copyright = u'2014, Jukka Lehtosalo' +project = "mypy" +copyright = "2012-%Y Jukka Lehtosalo and mypy contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.0.1.dev1' +version = mypy_version.split("-")[0] # The full version, including alpha/beta/rc tags. -release = '0.0.1.dev1' +release = mypy_version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -71,173 +78,165 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +# pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -try: - import sphinx_rtd_theme -except: - html_theme = 'default' -else: - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_theme = "furo" + +html_theme_options = { + "source_repository": "https://github.com/python/mypy", + "source_branch": "master", + "source_directory": "docs/source", +} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +html_logo = "mypy_light.svg" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Mypydoc' +htmlhelp_basename = "mypydoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'Mypy.tex', u'Mypy Documentation', - u'Jukka', 'manual'), -] +latex_documents = [("index", "Mypy.tex", "Mypy Documentation", "Jukka", "manual")] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'mypy', u'Mypy Documentation', - [u'Jukka Lehtosalo'], 1) -] +man_pages = [("index", "mypy", "Mypy Documentation", ["Jukka Lehtosalo"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -246,21 +245,48 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Mypy', u'Mypy Documentation', - u'Jukka', 'Mypy', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "Mypy", + "Mypy Documentation", + "Jukka", + "Mypy", + "One line description of project.", + "Miscellaneous", + ) ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False + +rst_prolog = ".. |...| unicode:: U+2026 .. ellipsis\n" + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "attrs": ("https://www.attrs.org/en/stable/", None), + "cython": ("https://docs.cython.org/en/latest", None), + "monkeytype": ("https://monkeytype.readthedocs.io/en/latest", None), + "setuptools": ("https://setuptools.readthedocs.io/en/latest", None), +} + -rst_prolog = '.. |...| unicode:: U+2026 .. ellipsis\n' +def setup(app: Sphinx) -> None: + app.add_object_type( + "confval", + "confval", + objname="configuration value", + indextemplate="pair: %s; configuration value", + doc_field_types=[ + Field("type", label="Type", has_arg=False, names=("type",)), + Field("default", label="Default", has_arg=False, names=("default",)), + ], + ) diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst new file mode 100644 index 000000000000..d7ae1b7a00df --- /dev/null +++ b/docs/source/config_file.rst @@ -0,0 +1,1200 @@ +.. _config-file: + +The mypy configuration file +=========================== + +Mypy is very configurable. This is most useful when introducing typing to +an existing codebase. See :ref:`existing-code` for concrete advice for +that situation. + +Mypy supports reading configuration settings from a file with the following precedence order: + + 1. ``./mypy.ini`` + 2. ``./.mypy.ini`` + 3. ``./pyproject.toml`` + 4. ``./setup.cfg`` + 5. ``$XDG_CONFIG_HOME/mypy/config`` + 6. ``~/.config/mypy/config`` + 7. ``~/.mypy.ini`` + +It is important to understand that there is no merging of configuration +files, as it would lead to ambiguity. The :option:`--config-file ` +command-line flag has the highest precedence and +must be correct; otherwise mypy will report an error and exit. Without the +command line option, mypy will look for configuration files in the +precedence order above. + +Most flags correspond closely to :ref:`command-line flags +` but there are some differences in flag names and some +flags may take a different value based on the module being processed. + +Some flags support user home directory and environment variable expansion. +To refer to the user home directory, use ``~`` at the beginning of the path. +To expand environment variables use ``$VARNAME`` or ``${VARNAME}``. + + +Config file format +****************** + +The configuration file format is the usual +:doc:`ini file ` format. It should contain +section names in square brackets and flag settings of the form +`NAME = VALUE`. Comments start with ``#`` characters. + +- A section named ``[mypy]`` must be present. This specifies + the global flags. + +- Additional sections named ``[mypy-PATTERN1,PATTERN2,...]`` may be + present, where ``PATTERN1``, ``PATTERN2``, etc., are comma-separated + patterns of fully-qualified module names, with some components optionally + replaced by the '*' character (e.g. ``foo.bar``, ``foo.bar.*``, ``foo.*.baz``). + These sections specify additional flags that only apply to *modules* + whose name matches at least one of the patterns. + + A pattern of the form ``qualified_module_name`` matches only the named module, + while ``dotted_module_name.*`` matches ``dotted_module_name`` and any + submodules (so ``foo.bar.*`` would match all of ``foo.bar``, + ``foo.bar.baz``, and ``foo.bar.baz.quux``). + + Patterns may also be "unstructured" wildcards, in which stars may + appear in the middle of a name (e.g + ``site.*.migrations.*``). Stars match zero or more module + components (so ``site.*.migrations.*`` can match ``site.migrations``). + + .. _config-precedence: + + When options conflict, the precedence order for configuration is: + + 1. :ref:`Inline configuration ` in the source file + 2. Sections with concrete module names (``foo.bar``) + 3. Sections with "unstructured" wildcard patterns (``foo.*.baz``), + with sections later in the configuration file overriding + sections earlier. + 4. Sections with "well-structured" wildcard patterns + (``foo.bar.*``), with more specific overriding more general. + 5. Command line options. + 6. Top-level configuration file options. + +The difference in precedence order between "structured" patterns (by +specificity) and "unstructured" patterns (by order in the file) is +unfortunate, and is subject to change in future versions. + +.. note:: + + The :confval:`warn_unused_configs` flag may be useful to debug misspelled + section names. + +.. note:: + + Configuration flags are liable to change between releases. + + +Per-module and global options +***************************** + +Some of the config options may be set either globally (in the ``[mypy]`` section) +or on a per-module basis (in sections like ``[mypy-foo.bar]``). + +If you set an option both globally and for a specific module, the module configuration +options take precedence. This lets you set global defaults and override them on a +module-by-module basis. If multiple pattern sections match a module, :ref:`the options from the +most specific section are used where they disagree `. + +Some other options, as specified in their description, +may only be set in the global section (``[mypy]``). + + +Inverting option values +*********************** + +Options that take a boolean value may be inverted by adding ``no_`` to +their name or by (when applicable) swapping their prefix from +``disallow`` to ``allow`` (and vice versa). + + +Example ``mypy.ini`` +******************** + +Here is an example of a ``mypy.ini`` file. To use this config file, place it at the root +of your repo and run mypy. + +.. code-block:: ini + + # Global options: + + [mypy] + warn_return_any = True + warn_unused_configs = True + + # Per-module options: + + [mypy-mycode.foo.*] + disallow_untyped_defs = True + + [mypy-mycode.bar] + warn_return_any = False + + [mypy-somelibrary] + ignore_missing_imports = True + +This config file specifies two global options in the ``[mypy]`` section. These two +options will: + +1. Report an error whenever a function returns a value that is inferred + to have type ``Any``. + +2. Report any config options that are unused by mypy. (This will help us catch typos + when making changes to our config file). + +Next, this module specifies three per-module options. The first two options change how mypy +type checks code in ``mycode.foo.*`` and ``mycode.bar``, which we assume here are two modules +that you wrote. The final config option changes how mypy type checks ``somelibrary``, which we +assume here is some 3rd party library you've installed and are importing. These options will: + +1. Selectively disallow untyped function definitions only within the ``mycode.foo`` + package -- that is, only for function definitions defined in the + ``mycode/foo`` directory. + +2. Selectively *disable* the "function is returning any" warnings within + ``mycode.bar`` only. This overrides the global default we set earlier. + +3. Suppress any error messages generated when your codebase tries importing the + module ``somelibrary``. This is useful if ``somelibrary`` is some 3rd party library + missing type hints. + + +.. _config-file-import-discovery: + +Import discovery +**************** + +For more information, see the :ref:`Import discovery ` +section of the command line docs. + +.. confval:: mypy_path + + :type: string + + Specifies the paths to use, after trying the paths from ``MYPYPATH`` environment + variable. Useful if you'd like to keep stubs in your repo, along with the config file. + Multiple paths are always separated with a ``:`` or ``,`` regardless of the platform. + User home directory and environment variables will be expanded. + + Relative paths are treated relative to the working directory of the mypy command, + not the config file. + Use the ``MYPY_CONFIG_FILE_DIR`` environment variable to refer to paths relative to + the config file (e.g. ``mypy_path = $MYPY_CONFIG_FILE_DIR/src``). + + This option may only be set in the global section (``[mypy]``). + + **Note:** On Windows, use UNC paths to avoid using ``:`` (e.g. ``\\127.0.0.1\X$\MyDir`` where ``X`` is the drive letter). + +.. confval:: files + + :type: comma-separated list of strings + + A comma-separated list of paths which should be checked by mypy if none are given on the command + line. Supports recursive file globbing using :py:mod:`glob`, where ``*`` (e.g. ``*.py``) matches + files in the current directory and ``**/`` (e.g. ``**/*.py``) matches files in any directories below + the current one. User home directory and environment variables will be expanded. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: modules + + :type: comma-separated list of strings + + A comma-separated list of packages which should be checked by mypy if none are given on the command + line. Mypy *will not* recursively type check any submodules of the provided + module. + + This option may only be set in the global section (``[mypy]``). + + +.. confval:: packages + + :type: comma-separated list of strings + + A comma-separated list of packages which should be checked by mypy if none are given on the command + line. Mypy *will* recursively type check any submodules of the provided + package. This flag is identical to :confval:`modules` apart from this + behavior. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: exclude + + :type: regular expression + + A regular expression that matches file names, directory names and paths + which mypy should ignore while recursively discovering files to check. + Use forward slashes (``/``) as directory separators on all platforms. + + .. code-block:: ini + + [mypy] + exclude = (?x)( + ^one\.py$ # files named "one.py" + | two\.pyi$ # or files ending with "two.pyi" + | ^three\. # or files starting with "three." + ) + + Crafting a single regular expression that excludes multiple files while remaining + human-readable can be a challenge. The above example demonstrates one approach. + ``(?x)`` enables the ``VERBOSE`` flag for the subsequent regular expression, which + :py:data:`ignores most whitespace and supports comments `. + The above is equivalent to: ``(^one\.py$|two\.pyi$|^three\.)``. + + For more details, see :option:`--exclude `. + + This option may only be set in the global section (``[mypy]``). + + .. note:: + + Note that the TOML equivalent differs slightly. It can be either a single string + (including a multi-line string) -- which is treated as a single regular + expression -- or an array of such strings. The following TOML examples are + equivalent to the above INI example. + + Array of strings: + + .. code-block:: toml + + [tool.mypy] + exclude = [ + "^one\\.py$", # TOML's double-quoted strings require escaping backslashes + 'two\.pyi$', # but TOML's single-quoted strings do not + '^three\.', + ] + + A single, multi-line string: + + .. code-block:: toml + + [tool.mypy] + exclude = '''(?x)( + ^one\.py$ # files named "one.py" + | two\.pyi$ # or files ending with "two.pyi" + | ^three\. # or files starting with "three." + )''' # TOML's single-quoted strings do not require escaping backslashes + + See :ref:`using-a-pyproject-toml`. + +.. confval:: namespace_packages + + :type: boolean + :default: True + + Enables :pep:`420` style namespace packages. See the + corresponding flag :option:`--no-namespace-packages ` + for more information. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: explicit_package_bases + + :type: boolean + :default: False + + This flag tells mypy that top-level packages will be based in either the + current directory, or a member of the ``MYPYPATH`` environment variable or + :confval:`mypy_path` config option. This option is only useful in + the absence of `__init__.py`. See :ref:`Mapping file + paths to modules ` for details. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: ignore_missing_imports + + :type: boolean + :default: False + + Suppresses error messages about imports that cannot be resolved. + + If this option is used in a per-module section, the module name should + match the name of the *imported* module, not the module containing the + import statement. + +.. confval:: follow_untyped_imports + + :type: boolean + :default: False + + Makes mypy analyze imports from installed packages even if missing a + :ref:`py.typed marker or stubs `. + + If this option is used in a per-module section, the module name should + match the name of the *imported* module, not the module containing the + import statement. + + .. warning:: + + Note that analyzing all unannotated modules might result in issues + when analyzing code not designed to be type checked and may significantly + increase how long mypy takes to run. + +.. confval:: follow_imports + + :type: string + :default: ``normal`` + + Directs what to do with imports when the imported module is found + as a ``.py`` file and not part of the files, modules and packages + provided on the command line. + + The four possible values are ``normal``, ``silent``, ``skip`` and + ``error``. For explanations see the discussion for the + :option:`--follow-imports ` command line flag. + + Using this option in a per-module section (potentially with a wildcard, + as described at the top of this page) is a good way to prevent mypy from + checking portions of your code. + + If this option is used in a per-module section, the module name should + match the name of the *imported* module, not the module containing the + import statement. + +.. confval:: follow_imports_for_stubs + + :type: boolean + :default: False + + Determines whether to respect the :confval:`follow_imports` setting even for + stub (``.pyi``) files. + + Used in conjunction with :confval:`follow_imports=skip `, this can be used + to suppress the import of a module from ``typeshed``, replacing it + with ``Any``. + + Used in conjunction with :confval:`follow_imports=error `, this can be used + to make any use of a particular ``typeshed`` module an error. + + .. note:: + + This is not supported by the mypy daemon. + +.. confval:: python_executable + + :type: string + + Specifies the path to the Python executable to inspect to collect + a list of available :ref:`PEP 561 packages `. User + home directory and environment variables will be expanded. Defaults to + the executable used to run mypy. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: no_site_packages + + :type: boolean + :default: False + + Disables using type information in installed packages (see :pep:`561`). + This will also disable searching for a usable Python executable. This acts + the same as :option:`--no-site-packages ` command + line flag. + +.. confval:: no_silence_site_packages + + :type: boolean + :default: False + + Enables reporting error messages generated within installed packages (see + :pep:`561` for more details on distributing type information). Those error + messages are suppressed by default, since you are usually not able to + control errors in 3rd party code. + + This option may only be set in the global section (``[mypy]``). + + +Platform configuration +********************** + +.. confval:: python_version + + :type: string + + Specifies the Python version used to parse and check the target + program. The string should be in the format ``MAJOR.MINOR`` -- + for example ``2.7``. The default is the version of the Python + interpreter used to run mypy. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: platform + + :type: string + + Specifies the OS platform for the target program, for example + ``darwin`` or ``win32`` (meaning OS X or Windows, respectively). + The default is the current platform as revealed by Python's + :py:data:`sys.platform` variable. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: always_true + + :type: comma-separated list of strings + + Specifies a list of variables that mypy will treat as + compile-time constants that are always true. + +.. confval:: always_false + + :type: comma-separated list of strings + + Specifies a list of variables that mypy will treat as + compile-time constants that are always false. + + +Disallow dynamic typing +*********************** + +For more information, see the :ref:`Disallow dynamic typing ` +section of the command line docs. + +.. confval:: disallow_any_unimported + + :type: boolean + :default: False + + Disallows usage of types that come from unfollowed imports (anything imported from + an unfollowed import is automatically given a type of ``Any``). + +.. confval:: disallow_any_expr + + :type: boolean + :default: False + + Disallows all expressions in the module that have type ``Any``. + +.. confval:: disallow_any_decorated + + :type: boolean + :default: False + + Disallows functions that have ``Any`` in their signature after decorator transformation. + +.. confval:: disallow_any_explicit + + :type: boolean + :default: False + + Disallows explicit ``Any`` in type positions such as type annotations and generic + type parameters. + +.. confval:: disallow_any_generics + + :type: boolean + :default: False + + Disallows usage of generic types that do not specify explicit type parameters. + +.. confval:: disallow_subclassing_any + + :type: boolean + :default: False + + Disallows subclassing a value of type ``Any``. + + +Untyped definitions and calls +***************************** + +For more information, see the :ref:`Untyped definitions and calls ` +section of the command line docs. + +.. confval:: disallow_untyped_calls + + :type: boolean + :default: False + + Disallows calling functions without type annotations from functions with type + annotations. Note that when used in per-module options, it enables/disables + this check **inside** the module(s) specified, not for functions that come + from that module(s), for example config like this: + + .. code-block:: ini + + [mypy] + disallow_untyped_calls = True + + [mypy-some.library.*] + disallow_untyped_calls = False + + will disable this check inside ``some.library``, not for your code that + imports ``some.library``. If you want to selectively disable this check for + all your code that imports ``some.library`` you should instead use + :confval:`untyped_calls_exclude`, for example: + + .. code-block:: ini + + [mypy] + disallow_untyped_calls = True + untyped_calls_exclude = some.library + +.. confval:: untyped_calls_exclude + + :type: comma-separated list of strings + + Selectively excludes functions and methods defined in specific packages, + modules, and classes from action of :confval:`disallow_untyped_calls`. + This also applies to all submodules of packages (i.e. everything inside + a given prefix). Note, this option does not support per-file configuration, + the exclusions list is defined globally for all your code. + +.. confval:: disallow_untyped_defs + + :type: boolean + :default: False + + Disallows defining functions without type annotations or with incomplete type + annotations (a superset of :confval:`disallow_incomplete_defs`). + + For example, it would report an error for :code:`def f(a, b)` and :code:`def f(a: int, b)`. + +.. confval:: disallow_incomplete_defs + + :type: boolean + :default: False + + Disallows defining functions with incomplete type annotations, while still + allowing entirely unannotated definitions. + + For example, it would report an error for :code:`def f(a: int, b)` but not :code:`def f(a, b)`. + +.. confval:: check_untyped_defs + + :type: boolean + :default: False + + Type-checks the interior of functions without type annotations. + +.. confval:: disallow_untyped_decorators + + :type: boolean + :default: False + + Reports an error whenever a function with type annotations is decorated with a + decorator without annotations. + + +.. _config-file-none-and-optional-handling: + +None and Optional handling +************************** + +For more information, see the :ref:`None and Optional handling ` +section of the command line docs. + +.. confval:: implicit_optional + + :type: boolean + :default: False + + Causes mypy to treat parameters with a ``None`` + default value as having an implicit optional type (``T | None``). + + **Note:** This was True by default in mypy versions 0.980 and earlier. + +.. confval:: strict_optional + + :type: boolean + :default: True + + Effectively disables checking of optional + types and ``None`` values. With this option, mypy doesn't + generally check the use of ``None`` values -- it is treated + as compatible with every type. + + .. warning:: + + ``strict_optional = false`` is evil. Avoid using it and definitely do + not use it without understanding what it does. + + +Configuring warnings +******************** + +For more information, see the :ref:`Configuring warnings ` +section of the command line docs. + +.. confval:: warn_redundant_casts + + :type: boolean + :default: False + + Warns about casting an expression to its inferred type. + + This option may only be set in the global section (``[mypy]``). + +.. confval:: warn_unused_ignores + + :type: boolean + :default: False + + Warns about unneeded ``# type: ignore`` comments. + +.. confval:: warn_no_return + + :type: boolean + :default: True + + Shows errors for missing return statements on some execution paths. + +.. confval:: warn_return_any + + :type: boolean + :default: False + + Shows a warning when returning a value with type ``Any`` from a function + declared with a non- ``Any`` return type. + +.. confval:: warn_unreachable + + :type: boolean + :default: False + + Shows a warning when encountering any code inferred to be unreachable or + redundant after performing type analysis. + + +Suppressing errors +****************** + +Note: these configuration options are available in the config file only. There is +no analog available via the command line options. + +.. confval:: ignore_errors + + :type: boolean + :default: False + + Ignores all non-fatal errors. + + +Miscellaneous strictness flags +****************************** + +For more information, see the :ref:`Miscellaneous strictness flags ` +section of the command line docs. + +.. confval:: allow_untyped_globals + + :type: boolean + :default: False + + Causes mypy to suppress errors caused by not being able to fully + infer the types of global and class variables. + +.. confval:: allow_redefinition + + :type: boolean + :default: False + + Allows variables to be redefined with an arbitrary type, as long as the redefinition + is in the same block and nesting level as the original definition. + Example where this can be useful: + + .. code-block:: python + + def process(items: list[str]) -> None: + # 'items' has type list[str] + items = [item.split() for item in items] + # 'items' now has type list[list[str]] + + The variable must be used before it can be redefined: + + .. code-block:: python + + def process(items: list[str]) -> None: + items = "mypy" # invalid redefinition to str because the variable hasn't been used yet + print(items) + items = "100" # valid, items now has type str + items = int(items) # valid, items now has type int + +.. confval:: local_partial_types + + :type: boolean + :default: False + + Disallows inferring variable type for ``None`` from two assignments in different scopes. + This is always implicitly enabled when using the :ref:`mypy daemon `. + +.. confval:: disable_error_code + + :type: comma-separated list of strings + + Allows disabling one or multiple error codes globally. + +.. confval:: enable_error_code + + :type: comma-separated list of strings + + Allows enabling one or multiple error codes globally. + + Note: This option will override disabled error codes from the disable_error_code option. + +.. confval:: extra_checks + + :type: boolean + :default: False + + This flag enables additional checks that are technically correct but may be impractical in real code. + See :option:`mypy --extra-checks` for more info. + +.. confval:: implicit_reexport + + :type: boolean + :default: True + + By default, imported values to a module are treated as exported and mypy allows + other modules to import them. When false, mypy will not re-export unless + the item is imported using from-as or is included in ``__all__``. Note that mypy + treats stub files as if this is always disabled. For example: + + .. code-block:: python + + # This won't re-export the value + from foo import bar + # This will re-export it as bar and allow other modules to import it + from foo import bar as bar + # This will also re-export bar + from foo import bar + __all__ = ['bar'] + +.. confval:: strict_concatenate + + :type: boolean + :default: False + + Make arguments prepended via ``Concatenate`` be truly positional-only. + +.. confval:: strict_equality + + :type: boolean + :default: False + + Prohibit equality checks, identity checks, and container checks between + non-overlapping types. + +.. confval:: strict + + :type: boolean + :default: False + + Enable all optional error checking flags. You can see the list of + flags enabled by strict mode in the full :option:`mypy --help` + output. + + Note: the exact list of flags enabled by :confval:`strict` may + change over time. + + +Configuring error messages +************************** + +For more information, see the :ref:`Configuring error messages ` +section of the command line docs. + +These options may only be set in the global section (``[mypy]``). + +.. confval:: show_error_context + + :type: boolean + :default: False + + Prefixes each error with the relevant context. + +.. confval:: show_column_numbers + + :type: boolean + :default: False + + Shows column numbers in error messages. + +.. confval:: show_error_code_links + + :type: boolean + :default: False + + Shows documentation link to corresponding error code. + +.. confval:: hide_error_codes + + :type: boolean + :default: False + + Hides error codes in error messages. See :ref:`error-codes` for more information. + +.. confval:: pretty + + :type: boolean + :default: False + + Use visually nicer output in error messages: use soft word wrap, + show source code snippets, and show error location markers. + +.. confval:: color_output + + :type: boolean + :default: True + + Shows error messages with color enabled. + +.. confval:: error_summary + + :type: boolean + :default: True + + Shows a short summary line after error messages. + +.. confval:: show_absolute_path + + :type: boolean + :default: False + + Show absolute paths to files. + +.. confval:: force_uppercase_builtins + + :type: boolean + :default: False + + Always use ``List`` instead of ``list`` in error messages, + even on Python 3.9+. + +.. confval:: force_union_syntax + + :type: boolean + :default: False + + Always use ``Union[]`` and ``Optional[]`` for union types + in error messages (instead of the ``|`` operator), + even on Python 3.10+. + +Incremental mode +**************** + +These options may only be set in the global section (``[mypy]``). + +.. confval:: incremental + + :type: boolean + :default: True + + Enables :ref:`incremental mode `. + +.. confval:: cache_dir + + :type: string + :default: ``.mypy_cache`` + + Specifies the location where mypy stores incremental cache info. + User home directory and environment variables will be expanded. + This setting will be overridden by the ``MYPY_CACHE_DIR`` environment + variable. + + Note that the cache is only read when incremental mode is enabled + but is always written to, unless the value is set to ``/dev/null`` + (UNIX) or ``nul`` (Windows). + +.. confval:: sqlite_cache + + :type: boolean + :default: False + + Use an `SQLite`_ database to store the cache. + +.. confval:: cache_fine_grained + + :type: boolean + :default: False + + Include fine-grained dependency information in the cache for the mypy daemon. + +.. confval:: skip_version_check + + :type: boolean + :default: False + + Makes mypy use incremental cache data even if it was generated by a + different version of mypy. (By default, mypy will perform a version + check and regenerate the cache if it was written by older versions of mypy.) + +.. confval:: skip_cache_mtime_checks + + :type: boolean + :default: False + + Skip cache internal consistency checks based on mtime. + + +Advanced options +**************** + +These options may only be set in the global section (``[mypy]``). + +.. confval:: plugins + + :type: comma-separated list of strings + + A comma-separated list of mypy plugins. See :ref:`extending-mypy-using-plugins`. + +.. confval:: pdb + + :type: boolean + :default: False + + Invokes :mod:`pdb` on fatal error. + +.. confval:: show_traceback + + :type: boolean + :default: False + + Shows traceback on fatal error. + +.. confval:: raise_exceptions + + :type: boolean + :default: False + + Raise exception on fatal error. + +.. confval:: custom_typing_module + + :type: string + + Specifies a custom module to use as a substitute for the :py:mod:`typing` module. + +.. confval:: custom_typeshed_dir + + :type: string + + This specifies the directory where mypy looks for standard library typeshed + stubs, instead of the typeshed that ships with mypy. This is + primarily intended to make it easier to test typeshed changes before + submitting them upstream, but also allows you to use a forked version of + typeshed. + + User home directory and environment variables will be expanded. + + Note that this doesn't affect third-party library stubs. To test third-party stubs, + for example try ``MYPYPATH=stubs/six mypy ...``. + +.. confval:: warn_incomplete_stub + + :type: boolean + :default: False + + Warns about missing type annotations in typeshed. This is only relevant + in combination with :confval:`disallow_untyped_defs` or :confval:`disallow_incomplete_defs`. + + +Report generation +***************** + +If these options are set, mypy will generate a report in the specified +format into the specified directory. + +.. warning:: + + Generating reports disables incremental mode and can significantly slow down + your workflow. It is recommended to enable reporting only for specific runs + (e.g. in CI). + +.. confval:: any_exprs_report + + :type: string + + Causes mypy to generate a text file report documenting how many + expressions of type ``Any`` are present within your codebase. + +.. confval:: cobertura_xml_report + + :type: string + + Causes mypy to generate a Cobertura XML type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + +.. confval:: html_report / xslt_html_report + + :type: string + + Causes mypy to generate an HTML type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + +.. confval:: linecount_report + + :type: string + + Causes mypy to generate a text file report documenting the functions + and lines that are typed and untyped within your codebase. + +.. confval:: linecoverage_report + + :type: string + + Causes mypy to generate a JSON file that maps each source file's + absolute filename to a list of line numbers that belong to typed + functions in that file. + +.. confval:: lineprecision_report + + :type: string + + Causes mypy to generate a flat text file report with per-module + statistics of how many lines are typechecked etc. + +.. confval:: txt_report / xslt_txt_report + + :type: string + + Causes mypy to generate a text file type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + +.. confval:: xml_report + + :type: string + + Causes mypy to generate an XML type checking coverage report. + + To generate this report, you must either manually install the `lxml`_ + library or specify mypy installation with the setuptools extra + ``mypy[reports]``. + + +Miscellaneous +************* + +These options may only be set in the global section (``[mypy]``). + +.. confval:: junit_xml + + :type: string + + Causes mypy to generate a JUnit XML test result document with + type checking results. This can make it easier to integrate mypy + with continuous integration (CI) tools. + +.. confval:: scripts_are_modules + + :type: boolean + :default: False + + Makes script ``x`` become module ``x`` instead of ``__main__``. This is + useful when checking multiple scripts in a single run. + +.. confval:: warn_unused_configs + + :type: boolean + :default: False + + Warns about per-module sections in the config file that do not + match any files processed when invoking mypy. + (This requires turning off incremental mode using :confval:`incremental = False `.) + +.. confval:: verbosity + + :type: integer + :default: 0 + + Controls how much debug output will be generated. Higher numbers are more verbose. + + +.. _using-a-pyproject-toml: + +Using a pyproject.toml file +*************************** + +Instead of using a ``mypy.ini`` file, a ``pyproject.toml`` file (as specified by +`PEP 518`_) may be used instead. A few notes on doing so: + +* The ``[mypy]`` section should have ``tool.`` prepended to its name: + + * I.e., ``[mypy]`` would become ``[tool.mypy]`` + +* The module specific sections should be moved into ``[[tool.mypy.overrides]]`` sections: + + * For example, ``[mypy-packagename]`` would become: + +.. code-block:: toml + + [[tool.mypy.overrides]] + module = 'packagename' + ... + +* Multi-module specific sections can be moved into a single ``[[tool.mypy.overrides]]`` section with a + module property set to an array of modules: + + * For example, ``[mypy-packagename,packagename2]`` would become: + +.. code-block:: toml + + [[tool.mypy.overrides]] + module = [ + 'packagename', + 'packagename2' + ] + ... + +* The following care should be given to values in the ``pyproject.toml`` files as compared to ``ini`` files: + + * Strings must be wrapped in double quotes, or single quotes if the string contains special characters + + * Boolean values should be all lower case + +Please see the `TOML Documentation`_ for more details and information on +what is allowed in a ``toml`` file. See `PEP 518`_ for more information on the layout +and structure of the ``pyproject.toml`` file. + +Example ``pyproject.toml`` +************************** + +Here is an example of a ``pyproject.toml`` file. To use this config file, place it at the root +of your repo (or append it to the end of an existing ``pyproject.toml`` file) and run mypy. + +.. code-block:: toml + + # mypy global options: + + [tool.mypy] + python_version = "2.7" + warn_return_any = true + warn_unused_configs = true + exclude = [ + '^file1\.py$', # TOML literal string (single-quotes, no escaping necessary) + "^file2\\.py$", # TOML basic string (double-quotes, backslash and other characters need escaping) + ] + + # mypy per-module options: + + [[tool.mypy.overrides]] + module = "mycode.foo.*" + disallow_untyped_defs = true + + [[tool.mypy.overrides]] + module = "mycode.bar" + warn_return_any = false + + [[tool.mypy.overrides]] + module = [ + "somelibrary", + "some_other_library" + ] + ignore_missing_imports = true + +.. _lxml: https://pypi.org/project/lxml/ +.. _SQLite: https://www.sqlite.org/ +.. _PEP 518: https://www.python.org/dev/peps/pep-0518/ +.. _TOML Documentation: https://toml.io/ diff --git a/docs/source/duck_type_compatibility.rst b/docs/source/duck_type_compatibility.rst index b49aa3c536c1..e801f9251db5 100644 --- a/docs/source/duck_type_compatibility.rst +++ b/docs/source/duck_type_compatibility.rst @@ -3,54 +3,29 @@ Duck type compatibility In Python, certain types are compatible even though they aren't subclasses of each other. For example, ``int`` objects are valid whenever ``float`` objects -are expected. Mypy supports this idiom via *duck type compatibility*. You can -specify a type to be a valid substitute for another type using the ``ducktype`` -class decorator: +are expected. Mypy supports this idiom via *duck type compatibility*. This is +supported for a small set of built-in types: -.. code-block:: python - - from typing import ducktype - - @ducktype(str) - class MyString: - def __init__(self, ...): ... - ... - -Now mypy considers a ``MyString`` instance to be valid whenever a -``str`` object is expected, independent of whether ``MyString`` -actually is a perfect substitute for strings. You can think of this as -a class-level cast as opposed to a value-level cast. This is a powerful -feature but you can easily abuse it and make it easy to write programs -that pass type checking but will crash and burn when run! - -The most common case where ``ducktype`` is useful is for certain -well-known standard library classes: - -* ``int`` is duck type compatible with ``float`` +* ``int`` is duck type compatible with ``float`` and ``complex``. * ``float`` is duck type compatible with ``complex``. +* ``bytearray`` and ``memoryview`` are duck type compatible with ``bytes``. -Thus code like this is nice and clean and also behaves as expected: +For example, mypy considers an ``int`` object to be valid whenever a +``float`` object is expected. Thus code like this is nice and clean +and also behaves as expected: .. code-block:: python - def degrees_to_radians(x: float) -> float: + import math + + def degrees_to_radians(degrees: float) -> float: return math.pi * degrees / 180 n = 90 # Inferred type 'int' - print(degrees_to_radians(n)) # Okay! - -Also, in Python 2 ``str`` would be duck type compatible with ``unicode``. - -.. note:: - - Note that in Python 2 a ``str`` object with non-ASCII characters is - often *not valid* when a unicode string is expected. The mypy type - system does not consider a string with non-ASCII values as a - separate type so some programs with this kind of error will - silently pass type checking. In Python 3 ``str`` and ``bytes`` are - separate, unrelated types and this kind of error is easy to - detect. This a good reason for preferring Python 3 over Python 2! - -.. note:: + print(degrees_to_radians(n)) # Okay! - Mypy support for Python 2 is still work in progress. +You can also often use :ref:`protocol-types` to achieve a similar effect in +a more principled and extensible fashion. Protocols don't apply to +cases like ``int`` being compatible with ``float``, since ``float`` is not +a protocol class but a regular, concrete class, and many standard library +functions expect concrete instances of ``float`` (or ``int``). diff --git a/docs/source/dynamic_typing.rst b/docs/source/dynamic_typing.rst index 538d0ab37eda..304e25c085a8 100644 --- a/docs/source/dynamic_typing.rst +++ b/docs/source/dynamic_typing.rst @@ -1,50 +1,42 @@ -Dynamically typed code -====================== - -As mentioned earlier, bodies of functions that don't have have an -explicit return type are dynamically typed (operations are checked at -runtime). Code outside functions is statically typed by default, and -types of variables are inferred. This does usually the right thing, -but you can also make any variable dynamically typed by defining it -explicitly with the type ``Any``: +.. _dynamic-typing: -.. code-block:: python - from typing import Any +Dynamically typed code +====================== - s = 1 # Statically typed (type int) - d = 1 # type: Any # Dynamically typed (type Any) - s = 'x' # Type check error - d = 'x' # OK +In :ref:`getting-started-dynamic-vs-static`, we discussed how bodies of functions +that don't have any explicit type annotations in their function are "dynamically typed" +and that mypy will not check them. In this section, we'll talk a little bit more +about what that means and how you can enable dynamic typing on a more fine grained basis. -Alternatively, you can use the ``Undefined`` construct to define -dynamically typed variables, as ``Any`` can be used anywhere any other -type is valid: +In cases where your code is too magical for mypy to understand, you can make a +variable or parameter dynamically typed by explicitly giving it the type +``Any``. Mypy will let you do basically anything with a value of type ``Any``, +including assigning a value of type ``Any`` to a variable of any type (or vice +versa). .. code-block:: python - from typing import Undefined, Any + from typing import Any - d = Undefined(Any) - d = 1 # OK - d = 'x' # OK + num = 1 # Statically typed (inferred to be int) + num = 'x' # error: Incompatible types in assignment (expression has type "str", variable has type "int") -Additionally, if you don't import the ``typing`` module in a file, all -code outside functions will be dynamically typed by default, and the -file is not type checked at all. This mode makes it easy to include -existing Python code that is not trivially compatible with static -typing. + dyn: Any = 1 # Dynamically typed (type Any) + dyn = 'x' # OK -.. note:: + num = dyn # No error, mypy will let you assign a value of type Any to any variable + num += 1 # Oops, mypy still thinks num is an int - The current mypy version type checks all modules, even those that - don't import ``typing``. This will change in a future version. +You can think of ``Any`` as a way to locally disable type checking. +See :ref:`silencing-type-errors` for other ways you can shut up +the type checker. Operations on Any values ------------------------ -You can do anything using a value with type ``Any``, and type checker -does not complain: +You can do anything using a value with type ``Any``, and the type checker +will not complain: .. code-block:: python @@ -57,23 +49,58 @@ does not complain: open(x).read() return x -Values derived from an ``Any`` value also have the value ``Any`` -implicitly. For example, if you get the attribute of an ``Any`` -value or call a ``Any`` value the result is ``Any``: +Values derived from an ``Any`` value also usually have the type ``Any`` +implicitly, as mypy can't infer a more precise result type. For +example, if you get the attribute of an ``Any`` value or call a +``Any`` value the result is ``Any``: .. code-block:: python def f(x: Any) -> None: - y = x.foo() # y has type Any - y.bar() # Okay as well! + y = x.foo() + reveal_type(y) # Revealed type is "Any" + z = y.bar("mypy will let you do anything to y") + reveal_type(z) # Revealed type is "Any" + +``Any`` types may propagate through your program, making type checking +less effective, unless you are careful. + +Function parameters without annotations are also implicitly ``Any``: + +.. code-block:: python + + def f(x) -> None: + reveal_type(x) # Revealed type is "Any" + x.can.do["anything", x]("wants", 2) + +You can make mypy warn you about untyped function parameters using the +:option:`--disallow-untyped-defs ` flag. + +Generic types missing type parameters will have those parameters implicitly +treated as ``Any``: + +.. code-block:: python + + def f(x: list) -> None: + reveal_type(x) # Revealed type is "builtins.list[Any]" + reveal_type(x[0]) # Revealed type is "Any" + x[0].anything_goes() # OK + +You can make mypy warn you about missing generic parameters using the +:option:`--disallow-any-generics ` flag. + +Finally, another major source of ``Any`` types leaking into your program is from +third party libraries that mypy does not know about. This is particularly the case +when using the :option:`--ignore-missing-imports ` +flag. See :ref:`fix-missing-imports` for more information about this. Any vs. object -------------- -The type ``object`` is another type that can have an instance of arbitrary -type as a value. Unlike ``Any``, ``object`` is an ordinary static type (it +The type :py:class:`object` is another type that can have an instance of arbitrary +type as a value. Unlike ``Any``, :py:class:`object` is an ordinary static type (it is similar to ``Object`` in Java), and only operations valid for *all* -types are accepted for ``object`` values. These are all valid: +types are accepted for :py:class:`object` values. These are all valid: .. code-block:: python @@ -93,10 +120,15 @@ operations: o.foo() # Error! o + 2 # Error! open(o) # Error! - n = Undefined(int) + n: int = 1 n = o # Error! -You can use ``cast()`` (see chapter :ref:`casts`) to go from a general -type such as ``object`` to a more specific type (subtype) such as -``int``. ``cast()`` is not needed with dynamically typed values -(values with type ``Any``). + +If you're not sure whether you need to use :py:class:`object` or ``Any``, use +:py:class:`object` -- only switch to using ``Any`` if you get a type checker +complaint. + +You can use different :ref:`type narrowing ` +techniques to narrow :py:class:`object` to a more specific +type (subtype) such as ``int``. Type narrowing is not needed with +dynamically typed values (values with type ``Any``). diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst new file mode 100644 index 000000000000..73171131bc8d --- /dev/null +++ b/docs/source/error_code_list.rst @@ -0,0 +1,1261 @@ +.. _error-code-list: + +Error codes enabled by default +============================== + +This section documents various errors codes that mypy can generate +with default options. See :ref:`error-codes` for general documentation +about error codes. :ref:`error-codes-optional` documents additional +error codes that you can enable. + +.. _code-attr-defined: + +Check that attribute exists [attr-defined] +------------------------------------------ + +Mypy checks that an attribute is defined in the target class or module +when using the dot operator. This applies to both getting and setting +an attribute. New attributes are defined by assignments in the class +body, or assignments to ``self.x`` in methods. These assignments don't +generate ``attr-defined`` errors. + +Example: + +.. code-block:: python + + class Resource: + def __init__(self, name: str) -> None: + self.name = name + + r = Resource('x') + print(r.name) # OK + print(r.id) # Error: "Resource" has no attribute "id" [attr-defined] + r.id = 5 # Error: "Resource" has no attribute "id" [attr-defined] + +This error code is also generated if an imported name is not defined +in the module in a ``from ... import`` statement (as long as the +target module can be found): + +.. code-block:: python + + # Error: Module "os" has no attribute "non_existent" [attr-defined] + from os import non_existent + +A reference to a missing attribute is given the ``Any`` type. In the +above example, the type of ``non_existent`` will be ``Any``, which can +be important if you silence the error. + +.. _code-union-attr: + +Check that attribute exists in each union item [union-attr] +----------------------------------------------------------- + +If you access the attribute of a value with a union type, mypy checks +that the attribute is defined for *every* type in that +union. Otherwise the operation can fail at runtime. This also applies +to optional types. + +Example: + +.. code-block:: python + + class Cat: + def sleep(self) -> None: ... + def miaow(self) -> None: ... + + class Dog: + def sleep(self) -> None: ... + def follow_me(self) -> None: ... + + def func(animal: Cat | Dog) -> None: + # OK: 'sleep' is defined for both Cat and Dog + animal.sleep() + # Error: Item "Cat" of "Cat | Dog" has no attribute "follow_me" [union-attr] + animal.follow_me() + +You can often work around these errors by using ``assert isinstance(obj, ClassName)`` +or ``assert obj is not None`` to tell mypy that you know that the type is more specific +than what mypy thinks. + +.. _code-name-defined: + +Check that name is defined [name-defined] +----------------------------------------- + +Mypy expects that all references to names have a corresponding +definition in an active scope, such as an assignment, function +definition or an import. This can catch missing definitions, missing +imports, and typos. + +This example accidentally calls ``sort()`` instead of :py:func:`sorted`: + +.. code-block:: python + + x = sort([3, 2, 4]) # Error: Name "sort" is not defined [name-defined] + +.. _code-used-before-def: + +Check that a variable is not used before it's defined [used-before-def] +----------------------------------------------------------------------- + +Mypy will generate an error if a name is used before it's defined. +While the name-defined check will catch issues with names that are undefined, +it will not flag if a variable is used and then defined later in the scope. +used-before-def check will catch such cases. + +Example: + +.. code-block:: python + + print(x) # Error: Name "x" is used before definition [used-before-def] + x = 123 + +.. _code-call-arg: + +Check arguments in calls [call-arg] +----------------------------------- + +Mypy expects that the number and names of arguments match the called function. +Note that argument type checks have a separate error code ``arg-type``. + +Example: + +.. code-block:: python + + def greet(name: str) -> None: + print('hello', name) + + greet('jack') # OK + greet('jill', 'jack') # Error: Too many arguments for "greet" [call-arg] + +.. _code-arg-type: + +Check argument types [arg-type] +------------------------------- + +Mypy checks that argument types in a call match the declared argument +types in the signature of the called function (if one exists). + +Example: + +.. code-block:: python + + def first(x: list[int]) -> int: + return x[0] if x else 0 + + t = (5, 4) + # Error: Argument 1 to "first" has incompatible type "tuple[int, int]"; + # expected "list[int]" [arg-type] + print(first(t)) + +.. _code-call-overload: + +Check calls to overloaded functions [call-overload] +--------------------------------------------------- + +When you call an overloaded function, mypy checks that at least one of +the signatures of the overload items match the argument types in the +call. + +Example: + +.. code-block:: python + + from typing import overload + + @overload + def inc_maybe(x: None) -> None: ... + + @overload + def inc_maybe(x: int) -> int: ... + + def inc_maybe(x: int | None) -> int | None: + if x is None: + return None + else: + return x + 1 + + inc_maybe(None) # OK + inc_maybe(5) # OK + + # Error: No overload variant of "inc_maybe" matches argument type "float" [call-overload] + inc_maybe(1.2) + +.. _code-valid-type: + +Check validity of types [valid-type] +------------------------------------ + +Mypy checks that each type annotation and any expression that +represents a type is a valid type. Examples of valid types include +classes, union types, callable types, type aliases, and literal types. +Examples of invalid types include bare integer literals, functions, +variables, and modules. + +This example incorrectly uses the function ``log`` as a type: + +.. code-block:: python + + def log(x: object) -> None: + print('log:', repr(x)) + + # Error: Function "t.log" is not valid as a type [valid-type] + def log_all(objs: list[object], f: log) -> None: + for x in objs: + f(x) + +You can use :py:class:`~collections.abc.Callable` as the type for callable objects: + +.. code-block:: python + + from collections.abc import Callable + + # OK + def log_all(objs: list[object], f: Callable[[object], None]) -> None: + for x in objs: + f(x) + +.. _code-var-annotated: + +Require annotation if variable type is unclear [var-annotated] +-------------------------------------------------------------- + +In some cases mypy can't infer the type of a variable without an +explicit annotation. Mypy treats this as an error. This typically +happens when you initialize a variable with an empty collection or +``None``. If mypy can't infer the collection item type, mypy replaces +any parts of the type it couldn't infer with ``Any`` and generates an +error. + +Example with an error: + +.. code-block:: python + + class Bundle: + def __init__(self) -> None: + # Error: Need type annotation for "items" + # (hint: "items: list[] = ...") [var-annotated] + self.items = [] + + reveal_type(Bundle().items) # list[Any] + +To address this, we add an explicit annotation: + +.. code-block:: python + + class Bundle: + def __init__(self) -> None: + self.items: list[str] = [] # OK + + reveal_type(Bundle().items) # list[str] + +.. _code-override: + +Check validity of overrides [override] +-------------------------------------- + +Mypy checks that an overridden method or attribute is compatible with +the base class. A method in a subclass must accept all arguments +that the base class method accepts, and the return type must conform +to the return type in the base class (Liskov substitution principle). + +Argument types can be more general is a subclass (i.e., they can vary +contravariantly). The return type can be narrowed in a subclass +(i.e., it can vary covariantly). It's okay to define additional +arguments in a subclass method, as long all extra arguments have default +values or can be left out (``*args``, for example). + +Example: + +.. code-block:: python + + class Base: + def method(self, + arg: int) -> int | None: + ... + + class Derived(Base): + def method(self, + arg: int | str) -> int: # OK + ... + + class DerivedBad(Base): + # Error: Argument 1 of "method" is incompatible with "Base" [override] + def method(self, + arg: bool) -> int: + ... + +.. _code-return: + +Check that function returns a value [return] +-------------------------------------------- + +If a function has a non-``None`` return type, mypy expects that the +function always explicitly returns a value (or raises an exception). +The function should not fall off the end of the function, since this +is often a bug. + +Example: + +.. code-block:: python + + # Error: Missing return statement [return] + def show(x: int) -> int: + print(x) + + # Error: Missing return statement [return] + def pred1(x: int) -> int: + if x > 0: + return x - 1 + + # OK + def pred2(x: int) -> int: + if x > 0: + return x - 1 + else: + raise ValueError('not defined for zero') + +.. _code-empty-body: + +Check that functions don't have empty bodies outside stubs [empty-body] +----------------------------------------------------------------------- + +This error code is similar to the ``[return]`` code but is emitted specifically +for functions and methods with empty bodies (if they are annotated with +non-trivial return type). Such a distinction exists because in some contexts +an empty body can be valid, for example for an abstract method or in a stub +file. Also old versions of mypy used to unconditionally allow functions with +empty bodies, so having a dedicated error code simplifies cross-version +compatibility. + +Note that empty bodies are allowed for methods in *protocols*, and such methods +are considered implicitly abstract: + +.. code-block:: python + + from abc import abstractmethod + from typing import Protocol + + class RegularABC: + @abstractmethod + def foo(self) -> int: + pass # OK + def bar(self) -> int: + pass # Error: Missing return statement [empty-body] + + class Proto(Protocol): + def bar(self) -> int: + pass # OK + +.. _code-return-value: + +Check that return value is compatible [return-value] +---------------------------------------------------- + +Mypy checks that the returned value is compatible with the type +signature of the function. + +Example: + +.. code-block:: python + + def func(x: int) -> str: + # Error: Incompatible return value type (got "int", expected "str") [return-value] + return x + 1 + +.. _code-assignment: + +Check types in assignment statement [assignment] +------------------------------------------------ + +Mypy checks that the assigned expression is compatible with the +assignment target (or targets). + +Example: + +.. code-block:: python + + class Resource: + def __init__(self, name: str) -> None: + self.name = name + + r = Resource('A') + + r.name = 'B' # OK + + # Error: Incompatible types in assignment (expression has type "int", + # variable has type "str") [assignment] + r.name = 5 + +.. _code-method-assign: + +Check that assignment target is not a method [method-assign] +------------------------------------------------------------ + +In general, assigning to a method on class object or instance (a.k.a. +monkey-patching) is ambiguous in terms of types, since Python's static type +system cannot express the difference between bound and unbound callable types. +Consider this example: + +.. code-block:: python + + class A: + def f(self) -> None: pass + def g(self) -> None: pass + + def h(self: A) -> None: pass + + A.f = h # Type of h is Callable[[A], None] + A().f() # This works + A.f = A().g # Type of A().g is Callable[[], None] + A().f() # ...but this also works at runtime + +To prevent the ambiguity, mypy will flag both assignments by default. If this +error code is disabled, mypy will treat the assigned value in all method assignments as unbound, +so only the second assignment will still generate an error. + +.. note:: + + This error code is a subcode of the more general ``[assignment]`` code. + +.. _code-type-var: + +Check type variable values [type-var] +------------------------------------- + +Mypy checks that value of a type variable is compatible with a value +restriction or the upper bound type. + +Example (Python 3.12 syntax): + +.. code-block:: python + + def add[T1: (int, float)](x: T1, y: T1) -> T1: + return x + y + + add(4, 5.5) # OK + + # Error: Value of type variable "T1" of "add" cannot be "str" [type-var] + add('x', 'y') + +.. _code-operator: + +Check uses of various operators [operator] +------------------------------------------ + +Mypy checks that operands support a binary or unary operation, such as +``+`` or ``~``. Indexing operations are so common that they have their +own error code ``index`` (see below). + +Example: + +.. code-block:: python + + # Error: Unsupported operand types for + ("int" and "str") [operator] + 1 + 'x' + +.. _code-index: + +Check indexing operations [index] +--------------------------------- + +Mypy checks that the indexed value in indexing operation such as +``x[y]`` supports indexing, and that the index expression has a valid +type. + +Example: + +.. code-block:: python + + a = {'x': 1, 'y': 2} + + a['x'] # OK + + # Error: Invalid index type "int" for "dict[str, int]"; expected type "str" [index] + print(a[1]) + + # Error: Invalid index type "bytes" for "dict[str, int]"; expected type "str" [index] + a[b'x'] = 4 + +.. _code-list-item: + +Check list items [list-item] +---------------------------- + +When constructing a list using ``[item, ...]``, mypy checks that each item +is compatible with the list type that is inferred from the surrounding +context. + +Example: + +.. code-block:: python + + # Error: List item 0 has incompatible type "int"; expected "str" [list-item] + a: list[str] = [0] + +.. _code-dict-item: + +Check dict items [dict-item] +---------------------------- + +When constructing a dictionary using ``{key: value, ...}`` or ``dict(key=value, ...)``, +mypy checks that each key and value is compatible with the dictionary type that is +inferred from the surrounding context. + +Example: + +.. code-block:: python + + # Error: Dict entry 0 has incompatible type "str": "str"; expected "str": "int" [dict-item] + d: dict[str, int] = {'key': 'value'} + +.. _code-typeddict-item: + +Check TypedDict items [typeddict-item] +-------------------------------------- + +When constructing a TypedDict object, mypy checks that each key and value is compatible +with the TypedDict type that is inferred from the surrounding context. + +When getting a TypedDict item, mypy checks that the key +exists. When assigning to a TypedDict, mypy checks that both the +key and the value are valid. + +Example: + +.. code-block:: python + + from typing import TypedDict + + class Point(TypedDict): + x: int + y: int + + # Error: Incompatible types (expression has type "float", + # TypedDict item "x" has type "int") [typeddict-item] + p: Point = {'x': 1.2, 'y': 4} + +.. _code-typeddict-unknown-key: + +Check TypedDict Keys [typeddict-unknown-key] +-------------------------------------------- + +When constructing a TypedDict object, mypy checks whether the +definition contains unknown keys, to catch invalid keys and +misspellings. On the other hand, mypy will not generate an error when +a previously constructed TypedDict value with extra keys is passed +to a function as an argument, since TypedDict values support +structural subtyping ("static duck typing") and the keys are assumed +to have been validated at the point of construction. Example: + +.. code-block:: python + + from typing import TypedDict + + class Point(TypedDict): + x: int + y: int + + class Point3D(Point): + z: int + + def add_x_coordinates(a: Point, b: Point) -> int: + return a["x"] + b["x"] + + a: Point = {"x": 1, "y": 4} + b: Point3D = {"x": 2, "y": 5, "z": 6} + + add_x_coordinates(a, b) # OK + + # Error: Extra key "z" for TypedDict "Point" [typeddict-unknown-key] + add_x_coordinates(a, {"x": 1, "y": 4, "z": 5}) + +Setting a TypedDict item using an unknown key will also generate this +error, since it could be a misspelling: + +.. code-block:: python + + a: Point = {"x": 1, "y": 2} + # Error: Extra key "z" for TypedDict "Point" [typeddict-unknown-key] + a["z"] = 3 + +Reading an unknown key will generate the more general (and serious) +``typeddict-item`` error, which is likely to result in an exception at +runtime: + +.. code-block:: python + + a: Point = {"x": 1, "y": 2} + # Error: TypedDict "Point" has no key "z" [typeddict-item] + _ = a["z"] + +.. note:: + + This error code is a subcode of the wider ``[typeddict-item]`` code. + +.. _code-has-type: + +Check that type of target is known [has-type] +--------------------------------------------- + +Mypy sometimes generates an error when it hasn't inferred any type for +a variable being referenced. This can happen for references to +variables that are initialized later in the source file, and for +references across modules that form an import cycle. When this +happens, the reference gets an implicit ``Any`` type. + +In this example the definitions of ``x`` and ``y`` are circular: + +.. code-block:: python + + class Problem: + def set_x(self) -> None: + # Error: Cannot determine type of "y" [has-type] + self.x = self.y + + def set_y(self) -> None: + self.y = self.x + +To work around this error, you can add an explicit type annotation to +the target variable or attribute. Sometimes you can also reorganize +the code so that the definition of the variable is placed earlier than +the reference to the variable in a source file. Untangling cyclic +imports may also help. + +We add an explicit annotation to the ``y`` attribute to work around +the issue: + +.. code-block:: python + + class Problem: + def set_x(self) -> None: + self.x = self.y # OK + + def set_y(self) -> None: + self.y: int = self.x # Added annotation here + +.. _code-import: + +Check for an issue with imports [import] +---------------------------------------- + +Mypy generates an error if it can't resolve an `import` statement. +This is a parent error code of `import-not-found` and `import-untyped` + +See :ref:`ignore-missing-imports` for how to work around these errors. + +.. _code-import-not-found: + +Check that import target can be found [import-not-found] +-------------------------------------------------------- + +Mypy generates an error if it can't find the source code or a stub file +for an imported module. + +Example: + +.. code-block:: python + + # Error: Cannot find implementation or library stub for module named "m0dule_with_typo" [import-not-found] + import m0dule_with_typo + +See :ref:`ignore-missing-imports` for how to work around these errors. + +.. _code-import-untyped: + +Check that import target can be found [import-untyped] +-------------------------------------------------------- + +Mypy generates an error if it can find the source code for an imported module, +but that module does not provide type annotations (via :ref:`PEP 561 `). + +Example: + +.. code-block:: python + + # Error: Library stubs not installed for "bs4" [import-untyped] + import bs4 + # Error: Skipping analyzing "no_py_typed": module is installed, but missing library stubs or py.typed marker [import-untyped] + import no_py_typed + +In some cases, these errors can be fixed by installing an appropriate +stub package. See :ref:`ignore-missing-imports` for more details. + +.. _code-no-redef: + +Check that each name is defined once [no-redef] +----------------------------------------------- + +Mypy may generate an error if you have multiple definitions for a name +in the same namespace. The reason is that this is often an error, as +the second definition may overwrite the first one. Also, mypy often +can't be able to determine whether references point to the first or +the second definition, which would compromise type checking. + +If you silence this error, all references to the defined name refer to +the *first* definition. + +Example: + +.. code-block:: python + + class A: + def __init__(self, x: int) -> None: ... + + class A: # Error: Name "A" already defined on line 1 [no-redef] + def __init__(self, x: str) -> None: ... + + # Error: Argument 1 to "A" has incompatible type "str"; expected "int" + # (the first definition wins!) + A('x') + +.. _code-func-returns-value: + +Check that called function returns a value [func-returns-value] +--------------------------------------------------------------- + +Mypy reports an error if you call a function with a ``None`` +return type and don't ignore the return value, as this is +usually (but not always) a programming error. + +In this example, the ``if f()`` check is always false since ``f`` +returns ``None``: + +.. code-block:: python + + def f() -> None: + ... + + # OK: we don't do anything with the return value + f() + + # Error: "f" does not return a value (it only ever returns None) [func-returns-value] + if f(): + print("not false") + +.. _code-abstract: + +Check instantiation of abstract classes [abstract] +-------------------------------------------------- + +Mypy generates an error if you try to instantiate an abstract base +class (ABC). An abstract base class is a class with at least one +abstract method or attribute. (See also :py:mod:`abc` module documentation) + +Sometimes a class is made accidentally abstract, often due to an +unimplemented abstract method. In a case like this you need to provide +an implementation for the method to make the class concrete +(non-abstract). + +Example: + +.. code-block:: python + + from abc import ABCMeta, abstractmethod + + class Persistent(metaclass=ABCMeta): + @abstractmethod + def save(self) -> None: ... + + class Thing(Persistent): + def __init__(self) -> None: + ... + + ... # No "save" method + + # Error: Cannot instantiate abstract class "Thing" with abstract attribute "save" [abstract] + t = Thing() + +.. _code-type-abstract: + +Safe handling of abstract type object types [type-abstract] +----------------------------------------------------------- + +Mypy always allows instantiating (calling) type objects typed as ``type[t]``, +even if it is not known that ``t`` is non-abstract, since it is a common +pattern to create functions that act as object factories (custom constructors). +Therefore, to prevent issues described in the above section, when an abstract +type object is passed where ``type[t]`` is expected, mypy will give an error. +Example (Python 3.12 syntax): + +.. code-block:: python + + from abc import ABCMeta, abstractmethod + + class Config(metaclass=ABCMeta): + @abstractmethod + def get_value(self, attr: str) -> str: ... + + def make_many[T](typ: type[T], n: int) -> list[T]: + return [typ() for _ in range(n)] # This will raise if typ is abstract + + # Error: Only concrete class can be given where "type[Config]" is expected [type-abstract] + make_many(Config, 5) + +.. _code-safe-super: + +Check that call to an abstract method via super is valid [safe-super] +--------------------------------------------------------------------- + +Abstract methods often don't have any default implementation, i.e. their +bodies are just empty. Calling such methods in subclasses via ``super()`` +will cause runtime errors, so mypy prevents you from doing so: + +.. code-block:: python + + from abc import abstractmethod + class Base: + @abstractmethod + def foo(self) -> int: ... + class Sub(Base): + def foo(self) -> int: + return super().foo() + 1 # error: Call to abstract method "foo" of "Base" with + # trivial body via super() is unsafe [safe-super] + Sub().foo() # This will crash at runtime. + +Mypy considers the following as trivial bodies: a ``pass`` statement, a literal +ellipsis ``...``, a docstring, and a ``raise NotImplementedError`` statement. + +.. _code-valid-newtype: + +Check the target of NewType [valid-newtype] +------------------------------------------- + +The target of a :py:class:`~typing.NewType` definition must be a class type. It can't +be a union type, ``Any``, or various other special types. + +You can also get this error if the target has been imported from a +module whose source mypy cannot find, since any such definitions are +treated by mypy as values with ``Any`` types. Example: + +.. code-block:: python + + from typing import NewType + + # The source for "acme" is not available for mypy + from acme import Entity # type: ignore + + # Error: Argument 2 to NewType(...) must be subclassable (got "Any") [valid-newtype] + UserEntity = NewType('UserEntity', Entity) + +To work around the issue, you can either give mypy access to the sources +for ``acme`` or create a stub file for the module. See :ref:`ignore-missing-imports` +for more information. + +.. _code-exit-return: + +Check the return type of __exit__ [exit-return] +----------------------------------------------- + +If mypy can determine that :py:meth:`__exit__ ` always returns ``False``, mypy +checks that the return type is *not* ``bool``. The boolean value of +the return type affects which lines mypy thinks are reachable after a +``with`` statement, since any :py:meth:`__exit__ ` method that can return +``True`` may swallow exceptions. An imprecise return type can result +in mysterious errors reported near ``with`` statements. + +To fix this, use either ``typing.Literal[False]`` or +``None`` as the return type. Returning ``None`` is equivalent to +returning ``False`` in this context, since both are treated as false +values. + +Example: + +.. code-block:: python + + class MyContext: + ... + def __exit__(self, exc, value, tb) -> bool: # Error + print('exit') + return False + +This produces the following output from mypy: + +.. code-block:: text + + example.py:3: error: "bool" is invalid as return type for "__exit__" that always returns False + example.py:3: note: Use "typing_extensions.Literal[False]" as the return type or change it to + "None" + example.py:3: note: If return type of "__exit__" implies that it may return True, the context + manager may swallow exceptions + +You can use ``Literal[False]`` to fix the error: + +.. code-block:: python + + from typing import Literal + + class MyContext: + ... + def __exit__(self, exc, value, tb) -> Literal[False]: # OK + print('exit') + return False + +You can also use ``None``: + +.. code-block:: python + + class MyContext: + ... + def __exit__(self, exc, value, tb) -> None: # Also OK + print('exit') + +.. _code-name-match: + +Check that naming is consistent [name-match] +-------------------------------------------- + +The definition of a named tuple or a TypedDict must be named +consistently when using the call-based syntax. Example: + +.. code-block:: python + + from typing import NamedTuple + + # Error: First argument to namedtuple() should be "Point2D", not "Point" + Point2D = NamedTuple("Point", [("x", int), ("y", int)]) + +.. _code-literal-required: + +Check that literal is used where expected [literal-required] +------------------------------------------------------------ + +There are some places where only a (string) literal value is expected for +the purposes of static type checking, for example a ``TypedDict`` key, or +a ``__match_args__`` item. Providing a ``str``-valued variable in such contexts +will result in an error. Note that in many cases you can also use ``Final`` +or ``Literal`` variables. Example: + +.. code-block:: python + + from typing import Final, Literal, TypedDict + + class Point(TypedDict): + x: int + y: int + + def test(p: Point) -> None: + X: Final = "x" + p[X] # OK + + Y: Literal["y"] = "y" + p[Y] # OK + + key = "x" # Inferred type of key is `str` + # Error: TypedDict key must be a string literal; + # expected one of ("x", "y") [literal-required] + p[key] + +.. _code-no-overload-impl: + +Check that overloaded functions have an implementation [no-overload-impl] +------------------------------------------------------------------------- + +Overloaded functions outside of stub files must be followed by a non overloaded +implementation. + +.. code-block:: python + + from typing import overload + + @overload + def func(value: int) -> int: + ... + + @overload + def func(value: str) -> str: + ... + + # presence of required function below is checked + def func(value): + pass # actual implementation + +.. _code-unused-coroutine: + +Check that coroutine return value is used [unused-coroutine] +------------------------------------------------------------ + +Mypy ensures that return values of async def functions are not +ignored, as this is usually a programming error, as the coroutine +won't be executed at the call site. + +.. code-block:: python + + async def f() -> None: + ... + + async def g() -> None: + f() # Error: missing await + await f() # OK + +You can work around this error by assigning the result to a temporary, +otherwise unused variable: + +.. code-block:: python + + _ = f() # No error + +.. _code-top-level-await: + +Warn about top level await expressions [top-level-await] +-------------------------------------------------------- + +This error code is separate from the general ``[syntax]`` errors, because in +some environments (e.g. IPython) a top level ``await`` is allowed. In such +environments a user may want to use ``--disable-error-code=top-level-await``, +that allows to still have errors for other improper uses of ``await``, for +example: + +.. code-block:: python + + async def f() -> None: + ... + + top = await f() # Error: "await" outside function [top-level-await] + +.. _code-await-not-async: + +Warn about await expressions used outside of coroutines [await-not-async] +------------------------------------------------------------------------- + +``await`` must be used inside a coroutine. + +.. code-block:: python + + async def f() -> None: + ... + + def g() -> None: + await f() # Error: "await" outside coroutine ("async def") [await-not-async] + +.. _code-assert-type: + +Check types in assert_type [assert-type] +---------------------------------------- + +The inferred type for an expression passed to ``assert_type`` must match +the provided type. + +.. code-block:: python + + from typing_extensions import assert_type + + assert_type([1], list[int]) # OK + + assert_type([1], list[str]) # Error + +.. _code-truthy-function: + +Check that function isn't used in boolean context [truthy-function] +------------------------------------------------------------------- + +Functions will always evaluate to true in boolean contexts. + +.. code-block:: python + + def f(): + ... + + if f: # Error: Function "Callable[[], Any]" could always be true in boolean context [truthy-function] + pass + +.. _code-str-format: + +Check that string formatting/interpolation is type-safe [str-format] +-------------------------------------------------------------------- + +Mypy will check that f-strings, ``str.format()`` calls, and ``%`` interpolations +are valid (when corresponding template is a literal string). This includes +checking number and types of replacements, for example: + +.. code-block:: python + + # Error: Cannot find replacement for positional format specifier 1 [str-format] + "{} and {}".format("spam") + "{} and {}".format("spam", "eggs") # OK + # Error: Not all arguments converted during string formatting [str-format] + "{} and {}".format("spam", "eggs", "cheese") + + # Error: Incompatible types in string interpolation + # (expression has type "float", placeholder has type "int") [str-format] + "{:d}".format(3.14) + +.. _code-str-bytes-safe: + +Check for implicit bytes coercions [str-bytes-safe] +------------------------------------------------------------------- + +Warn about cases where a bytes object may be converted to a string in an unexpected manner. + +.. code-block:: python + + b = b"abc" + + # Error: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". + # If this is desired behavior, use f"{x!r}" or "{!r}".format(x). + # Otherwise, decode the bytes [str-bytes-safe] + print(f"The alphabet starts with {b}") + + # Okay + print(f"The alphabet starts with {b!r}") # The alphabet starts with b'abc' + print(f"The alphabet starts with {b.decode('utf-8')}") # The alphabet starts with abc + +.. _code-overload-overlap: + +Check that overloaded functions don't overlap [overload-overlap] +---------------------------------------------------------------- + +Warn if multiple ``@overload`` variants overlap in potentially unsafe ways. +This guards against the following situation: + +.. code-block:: python + + from typing import overload + + class A: ... + class B(A): ... + + @overload + def foo(x: B) -> int: ... # Error: Overloaded function signatures 1 and 2 overlap with incompatible return types [overload-overlap] + @overload + def foo(x: A) -> str: ... + def foo(x): ... + + def takes_a(a: A) -> str: + return foo(a) + + a: A = B() + value = takes_a(a) + # mypy will think that value is a str, but it could actually be an int + reveal_type(value) # Revealed type is "builtins.str" + + +Note that in cases where you ignore this error, mypy will usually still infer the +types you expect. + +See :ref:`overloading ` for more explanation. + + +.. _code-overload-cannot-match: + +Check for overload signatures that cannot match [overload-cannot-match] +-------------------------------------------------------------------------- + +Warn if an ``@overload`` variant can never be matched, because an earlier +overload has a wider signature. For example, this can happen if the two +overloads accept the same parameters and each parameter on the first overload +has the same type or a wider type than the corresponding parameter on the second +overload. + +Example: + +.. code-block:: python + + from typing import overload, Union + + @overload + def process(response1: object, response2: object) -> object: + ... + @overload + def process(response1: int, response2: int) -> int: # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [overload-cannot-match] + ... + + def process(response1: object, response2: object) -> object: + return response1 + response2 + +.. _code-annotation-unchecked: + +Notify about an annotation in an unchecked function [annotation-unchecked] +-------------------------------------------------------------------------- + +Sometimes a user may accidentally omit an annotation for a function, and mypy +will not check the body of this function (unless one uses +:option:`--check-untyped-defs ` or +:option:`--disallow-untyped-defs `). To avoid +such situations go unnoticed, mypy will show a note, if there are any type +annotations in an unchecked function: + +.. code-block:: python + + def test_assignment(): # "-> None" return annotation is missing + # Note: By default the bodies of untyped functions are not checked, + # consider using --check-untyped-defs [annotation-unchecked] + x: int = "no way" + +Note that mypy will still exit with return code ``0``, since such behaviour is +specified by :pep:`484`. + +.. _code-prop-decorator: + +Decorator preceding property not supported [prop-decorator] +----------------------------------------------------------- + +Mypy does not yet support analysis of decorators that precede the property +decorator. If the decorator does not preserve the declared type of the property, +mypy will not infer the correct type for the declaration. If the decorator cannot +be moved after the ``@property`` decorator, then you must use a type ignore +comment: + +.. code-block:: python + + class MyClass: + @special # type: ignore[prop-decorator] + @property + def magic(self) -> str: + return "xyzzy" + +.. note:: + + For backward compatibility, this error code is a subcode of the generic ``[misc]`` code. + +.. _code-syntax: + +Report syntax errors [syntax] +----------------------------- + +If the code being checked is not syntactically valid, mypy issues a +syntax error. Most, but not all, syntax errors are *blocking errors*: +they can't be ignored with a ``# type: ignore`` comment. + +.. _code-typeddict-readonly-mutated: + +ReadOnly key of a TypedDict is mutated [typeddict-readonly-mutated] +------------------------------------------------------------------- + +Consider this example: + +.. code-block:: python + + from datetime import datetime + from typing import TypedDict + from typing_extensions import ReadOnly + + class User(TypedDict): + username: ReadOnly[str] + last_active: datetime + + user: User = {'username': 'foobar', 'last_active': datetime.now()} + user['last_active'] = datetime.now() # ok + user['username'] = 'other' # error: ReadOnly TypedDict key "key" TypedDict is mutated [typeddict-readonly-mutated] + +`PEP 705 `_ specifies +how ``ReadOnly`` special form works for ``TypedDict`` objects. + +.. _code-misc: + +Miscellaneous checks [misc] +--------------------------- + +Mypy performs numerous other, less commonly failing checks that don't +have specific error codes. These use the ``misc`` error code. Other +than being used for multiple unrelated errors, the ``misc`` error code +is not special. For example, you can ignore all errors in this +category by using ``# type: ignore[misc]`` comment. Since these errors +are not expected to be common, it's unlikely that you'll see two +*different* errors with the ``misc`` code on a single line -- though +this can certainly happen once in a while. + +.. note:: + + Future mypy versions will likely add new error codes for some errors + that currently use the ``misc`` error code. diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst new file mode 100644 index 000000000000..df8b696745fc --- /dev/null +++ b/docs/source/error_code_list2.rst @@ -0,0 +1,611 @@ +.. _error-codes-optional: + +Error codes for optional checks +=============================== + +This section documents various errors codes that mypy generates only +if you enable certain options. See :ref:`error-codes` for general +documentation about error codes and their configuration. +:ref:`error-code-list` documents error codes that are enabled by default. + +.. note:: + + The examples in this section use :ref:`inline configuration + ` to specify mypy options. You can also set the same + options by using a :ref:`configuration file ` or + :ref:`command-line options `. + +.. _code-type-arg: + +Check that type arguments exist [type-arg] +------------------------------------------ + +If you use :option:`--disallow-any-generics `, mypy requires that each generic +type has values for each type argument. For example, the types ``list`` or +``dict`` would be rejected. You should instead use types like ``list[int]`` or +``dict[str, int]``. Any omitted generic type arguments get implicit ``Any`` +values. The type ``list`` is equivalent to ``list[Any]``, and so on. + +Example: + +.. code-block:: python + + # mypy: disallow-any-generics + + # Error: Missing type parameters for generic type "list" [type-arg] + def remove_dups(items: list) -> list: + ... + +.. _code-no-untyped-def: + +Check that every function has an annotation [no-untyped-def] +------------------------------------------------------------ + +If you use :option:`--disallow-untyped-defs `, mypy requires that all functions +have annotations (either a Python 3 annotation or a type comment). + +Example: + +.. code-block:: python + + # mypy: disallow-untyped-defs + + def inc(x): # Error: Function is missing a type annotation [no-untyped-def] + return x + 1 + + def inc_ok(x: int) -> int: # OK + return x + 1 + + class Counter: + # Error: Function is missing a type annotation [no-untyped-def] + def __init__(self): + self.value = 0 + + class CounterOk: + # OK: An explicit "-> None" is needed if "__init__" takes no arguments + def __init__(self) -> None: + self.value = 0 + +.. _code-redundant-cast: + +Check that cast is not redundant [redundant-cast] +------------------------------------------------- + +If you use :option:`--warn-redundant-casts `, mypy will generate an error if the source +type of a cast is the same as the target type. + +Example: + +.. code-block:: python + + # mypy: warn-redundant-casts + + from typing import cast + + Count = int + + def example(x: Count) -> int: + # Error: Redundant cast to "int" [redundant-cast] + return cast(int, x) + +.. _code-redundant-self: + +Check that methods do not have redundant Self annotations [redundant-self] +-------------------------------------------------------------------------- + +If a method uses the ``Self`` type in the return type or the type of a +non-self argument, there is no need to annotate the ``self`` argument +explicitly. Such annotations are allowed by :pep:`673` but are +redundant. If you enable this error code, mypy will generate an error if +there is a redundant ``Self`` type. + +Example: + +.. code-block:: python + + # mypy: enable-error-code="redundant-self" + + from typing import Self + + class C: + # Error: Redundant "Self" annotation for the first method argument + def copy(self: Self) -> Self: + return type(self)() + +.. _code-comparison-overlap: + +Check that comparisons are overlapping [comparison-overlap] +----------------------------------------------------------- + +If you use :option:`--strict-equality `, mypy will generate an error if it +thinks that a comparison operation is always true or false. These are +often bugs. Sometimes mypy is too picky and the comparison can +actually be useful. Instead of disabling strict equality checking +everywhere, you can use ``# type: ignore[comparison-overlap]`` to +ignore the issue on a particular line only. + +Example: + +.. code-block:: python + + # mypy: strict-equality + + def is_magic(x: bytes) -> bool: + # Error: Non-overlapping equality check (left operand type: "bytes", + # right operand type: "str") [comparison-overlap] + return x == 'magic' + +We can fix the error by changing the string literal to a bytes +literal: + +.. code-block:: python + + # mypy: strict-equality + + def is_magic(x: bytes) -> bool: + return x == b'magic' # OK + +.. _code-no-untyped-call: + +Check that no untyped functions are called [no-untyped-call] +------------------------------------------------------------ + +If you use :option:`--disallow-untyped-calls `, mypy generates an error when you +call an unannotated function in an annotated function. + +Example: + +.. code-block:: python + + # mypy: disallow-untyped-calls + + def do_it() -> None: + # Error: Call to untyped function "bad" in typed context [no-untyped-call] + bad() + + def bad(): + ... + +.. _code-no-any-return: + +Check that function does not return Any value [no-any-return] +------------------------------------------------------------- + +If you use :option:`--warn-return-any `, mypy generates an error if you return a +value with an ``Any`` type in a function that is annotated to return a +non-``Any`` value. + +Example: + +.. code-block:: python + + # mypy: warn-return-any + + def fields(s): + return s.split(',') + + def first_field(x: str) -> str: + # Error: Returning Any from function declared to return "str" [no-any-return] + return fields(x)[0] + +.. _code-no-any-unimported: + +Check that types have no Any components due to missing imports [no-any-unimported] +---------------------------------------------------------------------------------- + +If you use :option:`--disallow-any-unimported `, mypy generates an error if a component of +a type becomes ``Any`` because mypy couldn't resolve an import. These "stealth" +``Any`` types can be surprising and accidentally cause imprecise type checking. + +In this example, we assume that mypy can't find the module ``animals``, which means +that ``Cat`` falls back to ``Any`` in a type annotation: + +.. code-block:: python + + # mypy: disallow-any-unimported + + from animals import Cat # type: ignore + + # Error: Argument 1 to "feed" becomes "Any" due to an unfollowed import [no-any-unimported] + def feed(cat: Cat) -> None: + ... + +.. _code-unreachable: + +Check that statement or expression is unreachable [unreachable] +--------------------------------------------------------------- + +If you use :option:`--warn-unreachable `, mypy generates an error if it +thinks that a statement or expression will never be executed. In most cases, this is due to +incorrect control flow or conditional checks that are accidentally always true or false. + +.. code-block:: python + + # mypy: warn-unreachable + + def example(x: int) -> None: + # Error: Right operand of "or" is never evaluated [unreachable] + assert isinstance(x, int) or x == 'unused' + + return + # Error: Statement is unreachable [unreachable] + print('unreachable') + +.. _code-deprecated: + +Check that imported or used feature is deprecated [deprecated] +-------------------------------------------------------------- + +If you use :option:`--enable-error-code deprecated `, +mypy generates an error if your code imports a deprecated feature explicitly with a +``from mod import depr`` statement or uses a deprecated feature imported otherwise or defined +locally. Features are considered deprecated when decorated with ``warnings.deprecated``, as +specified in `PEP 702 `_. +Use the :option:`--report-deprecated-as-note ` option to +turn all such errors into notes. + +.. note:: + + The ``warnings`` module provides the ``@deprecated`` decorator since Python 3.13. + To use it with older Python versions, import it from ``typing_extensions`` instead. + +Examples: + +.. code-block:: python + + # mypy: report-deprecated-as-error + + # Error: abc.abstractproperty is deprecated: Deprecated, use 'property' with 'abstractmethod' instead + from abc import abstractproperty + + from typing_extensions import deprecated + + @deprecated("use new_function") + def old_function() -> None: + print("I am old") + + # Error: __main__.old_function is deprecated: use new_function + old_function() + old_function() # type: ignore[deprecated] + + +.. _code-redundant-expr: + +Check that expression is redundant [redundant-expr] +--------------------------------------------------- + +If you use :option:`--enable-error-code redundant-expr `, +mypy generates an error if it thinks that an expression is redundant. + +.. code-block:: python + + # mypy: enable-error-code="redundant-expr" + + def example(x: int) -> None: + # Error: Left operand of "and" is always true [redundant-expr] + if isinstance(x, int) and x > 0: + pass + + # Error: If condition is always true [redundant-expr] + 1 if isinstance(x, int) else 0 + + # Error: If condition in comprehension is always true [redundant-expr] + [i for i in range(x) if isinstance(i, int)] + + +.. _code-possibly-undefined: + +Warn about variables that are defined only in some execution paths [possibly-undefined] +--------------------------------------------------------------------------------------- + +If you use :option:`--enable-error-code possibly-undefined `, +mypy generates an error if it cannot verify that a variable will be defined in +all execution paths. This includes situations when a variable definition +appears in a loop, in a conditional branch, in an except handler, etc. For +example: + +.. code-block:: python + + # mypy: enable-error-code="possibly-undefined" + + from collections.abc import Iterable + + def test(values: Iterable[int], flag: bool) -> None: + if flag: + a = 1 + z = a + 1 # Error: Name "a" may be undefined [possibly-undefined] + + for v in values: + b = v + z = b + 1 # Error: Name "b" may be undefined [possibly-undefined] + +.. _code-truthy-bool: + +Check that expression is not implicitly true in boolean context [truthy-bool] +----------------------------------------------------------------------------- + +Warn when the type of an expression in a boolean context does not +implement ``__bool__`` or ``__len__``. Unless one of these is +implemented by a subtype, the expression will always be considered +true, and there may be a bug in the condition. + +As an exception, the ``object`` type is allowed in a boolean context. +Using an iterable value in a boolean context has a separate error code +(see below). + +.. code-block:: python + + # mypy: enable-error-code="truthy-bool" + + class Foo: + pass + foo = Foo() + # Error: "foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context + if foo: + ... + +.. _code-truthy-iterable: + +Check that iterable is not implicitly true in boolean context [truthy-iterable] +------------------------------------------------------------------------------- + +Generate an error if a value of type ``Iterable`` is used as a boolean +condition, since ``Iterable`` does not implement ``__len__`` or ``__bool__``. + +Example: + +.. code-block:: python + + from collections.abc import Iterable + + def transform(items: Iterable[int]) -> list[int]: + # Error: "items" has type "Iterable[int]" which can always be true in boolean context. Consider using "Collection[int]" instead. [truthy-iterable] + if not items: + return [42] + return [x + 1 for x in items] + +If ``transform`` is called with a ``Generator`` argument, such as +``int(x) for x in []``, this function would not return ``[42]`` unlike +what might be intended. Of course, it's possible that ``transform`` is +only called with ``list`` or other container objects, and the ``if not +items`` check is actually valid. If that is the case, it is +recommended to annotate ``items`` as ``Collection[int]`` instead of +``Iterable[int]``. + +.. _code-ignore-without-code: + +Check that ``# type: ignore`` include an error code [ignore-without-code] +------------------------------------------------------------------------- + +Warn when a ``# type: ignore`` comment does not specify any error codes. +This clarifies the intent of the ignore and ensures that only the +expected errors are silenced. + +Example: + +.. code-block:: python + + # mypy: enable-error-code="ignore-without-code" + + class Foo: + def __init__(self, name: str) -> None: + self.name = name + + f = Foo('foo') + + # This line has a typo that mypy can't help with as both: + # - the expected error 'assignment', and + # - the unexpected error 'attr-defined' + # are silenced. + # Error: "type: ignore" comment without error code (consider "type: ignore[attr-defined]" instead) + f.nme = 42 # type: ignore + + # This line warns correctly about the typo in the attribute name + # Error: "Foo" has no attribute "nme"; maybe "name"? + f.nme = 42 # type: ignore[assignment] + +.. _code-unused-awaitable: + +Check that awaitable return value is used [unused-awaitable] +------------------------------------------------------------ + +If you use :option:`--enable-error-code unused-awaitable `, +mypy generates an error if you don't use a returned value that defines ``__await__``. + +Example: + +.. code-block:: python + + # mypy: enable-error-code="unused-awaitable" + + import asyncio + + async def f() -> int: ... + + async def g() -> None: + # Error: Value of type "Task[int]" must be used + # Are you missing an await? + asyncio.create_task(f()) + +You can assign the value to a temporary, otherwise unused variable to +silence the error: + +.. code-block:: python + + async def g() -> None: + _ = asyncio.create_task(f()) # No error + +.. _code-unused-ignore: + +Check that ``# type: ignore`` comment is used [unused-ignore] +------------------------------------------------------------- + +If you use :option:`--enable-error-code unused-ignore `, +or :option:`--warn-unused-ignores ` +mypy generates an error if you don't use a ``# type: ignore`` comment, i.e. if +there is a comment, but there would be no error generated by mypy on this line +anyway. + +Example: + +.. code-block:: python + + # Use "mypy --warn-unused-ignores ..." + + def add(a: int, b: int) -> int: + # Error: unused "type: ignore" comment + return a + b # type: ignore + +Note that due to a specific nature of this comment, the only way to selectively +silence it, is to include the error code explicitly. Also note that this error is +not shown if the ``# type: ignore`` is not used due to code being statically +unreachable (e.g. due to platform or version checks). + +Example: + +.. code-block:: python + + # Use "mypy --warn-unused-ignores ..." + + import sys + + try: + # The "[unused-ignore]" is needed to get a clean mypy run + # on both Python 3.8, and 3.9 where this module was added + import graphlib # type: ignore[import,unused-ignore] + except ImportError: + pass + + if sys.version_info >= (3, 9): + # The following will not generate an error on either + # Python 3.8, or Python 3.9 + 42 + "testing..." # type: ignore + +.. _code-explicit-override: + +Check that ``@override`` is used when overriding a base class method [explicit-override] +---------------------------------------------------------------------------------------- + +If you use :option:`--enable-error-code explicit-override ` +mypy generates an error if you override a base class method without using the +``@override`` decorator. An error will not be emitted for overrides of ``__init__`` +or ``__new__``. See `PEP 698 `_. + +.. note:: + + Starting with Python 3.12, the ``@override`` decorator can be imported from ``typing``. + To use it with older Python versions, import it from ``typing_extensions`` instead. + +Example: + +.. code-block:: python + + # mypy: enable-error-code="explicit-override" + + from typing import override + + class Parent: + def f(self, x: int) -> None: + pass + + def g(self, y: int) -> None: + pass + + + class Child(Parent): + def f(self, x: int) -> None: # Error: Missing @override decorator + pass + + @override + def g(self, y: int) -> None: + pass + +.. _code-mutable-override: + +Check that overrides of mutable attributes are safe [mutable-override] +---------------------------------------------------------------------- + +`mutable-override` will enable the check for unsafe overrides of mutable attributes. +For historical reasons, and because this is a relatively common pattern in Python, +this check is not enabled by default. The example below is unsafe, and will be +flagged when this error code is enabled: + +.. code-block:: python + + from typing import Any + + class C: + x: float + y: float + z: float + + class D(C): + x: int # Error: Covariant override of a mutable attribute + # (base class "C" defined the type as "float", + # expression has type "int") [mutable-override] + y: float # OK + z: Any # OK + + def f(c: C) -> None: + c.x = 1.1 + d = D() + f(d) + d.x >> 1 # This will crash at runtime, because d.x is now float, not an int + +.. _code-unimported-reveal: + +Check that ``reveal_type`` is imported from typing or typing_extensions [unimported-reveal] +------------------------------------------------------------------------------------------- + +Mypy used to have ``reveal_type`` as a special builtin +that only existed during type-checking. +In runtime it fails with expected ``NameError``, +which can cause real problem in production, hidden from mypy. + +But, in Python3.11 :py:func:`typing.reveal_type` was added. +``typing_extensions`` ported this helper to all supported Python versions. + +Now users can actually import ``reveal_type`` to make the runtime code safe. + +.. note:: + + Starting with Python 3.11, the ``reveal_type`` function can be imported from ``typing``. + To use it with older Python versions, import it from ``typing_extensions`` instead. + +.. code-block:: python + + # mypy: enable-error-code="unimported-reveal" + + x = 1 + reveal_type(x) # Note: Revealed type is "builtins.int" \ + # Error: Name "reveal_type" is not defined + +Correct usage: + +.. code-block:: python + + # mypy: enable-error-code="unimported-reveal" + from typing import reveal_type # or `typing_extensions` + + x = 1 + # This won't raise an error: + reveal_type(x) # Note: Revealed type is "builtins.int" + +When this code is enabled, using ``reveal_locals`` is always an error, +because there's no way one can import it. + +.. _code-narrowed-type-not-subtype: + +Check that ``TypeIs`` narrows types [narrowed-type-not-subtype] +--------------------------------------------------------------- + +:pep:`742` requires that when ``TypeIs`` is used, the narrowed +type must be a subtype of the original type:: + + from typing_extensions import TypeIs + + def f(x: int) -> TypeIs[str]: # Error, str is not a subtype of int + ... + + def g(x: object) -> TypeIs[str]: # OK + ... diff --git a/docs/source/error_codes.rst b/docs/source/error_codes.rst new file mode 100644 index 000000000000..485d70cb59bc --- /dev/null +++ b/docs/source/error_codes.rst @@ -0,0 +1,118 @@ +.. _error-codes: + +Error codes +=========== + +Mypy can optionally display an error code such as ``[attr-defined]`` +after each error message. Error codes serve two purposes: + +1. It's possible to silence specific error codes on a line using ``# + type: ignore[code]``. This way you won't accidentally ignore other, + potentially more serious errors. + +2. The error code can be used to find documentation about the error. + The next two topics (:ref:`error-code-list` and + :ref:`error-codes-optional`) document the various error codes + mypy can report. + +Most error codes are shared between multiple related error messages. +Error codes may change in future mypy releases. + + +.. _silence-error-codes: + +Silencing errors based on error codes +------------------------------------- + +You can use a special comment ``# type: ignore[code, ...]`` to only +ignore errors with a specific error code (or codes) on a particular +line. This can be used even if you have not configured mypy to show +error codes. + +This example shows how to ignore an error about an imported name mypy +thinks is undefined: + +.. code-block:: python + + # 'foo' is defined in 'foolib', even though mypy can't see the + # definition. + from foolib import foo # type: ignore[attr-defined] + +Enabling/disabling specific error codes globally +------------------------------------------------ + +There are command-line flags and config file settings for enabling +certain optional error codes, such as :option:`--disallow-untyped-defs `, +which enables the ``no-untyped-def`` error code. + +You can use :option:`--enable-error-code ` +and :option:`--disable-error-code ` +to enable or disable specific error codes that don't have a dedicated +command-line flag or config file setting. + +Per-module enabling/disabling error codes +----------------------------------------- + +You can use :ref:`configuration file ` sections to enable or +disable specific error codes only in some modules. For example, this ``mypy.ini`` +config will enable non-annotated empty containers in tests, while keeping +other parts of code checked in strict mode: + +.. code-block:: ini + + [mypy] + strict = True + + [mypy-tests.*] + allow_untyped_defs = True + allow_untyped_calls = True + disable_error_code = var-annotated, has-type + +Note that per-module enabling/disabling acts as override over the global +options. So that you don't need to repeat the error code lists for each +module if you have them in global config section. For example: + +.. code-block:: ini + + [mypy] + enable_error_code = truthy-bool, ignore-without-code, unused-awaitable + + [mypy-extensions.*] + disable_error_code = unused-awaitable + +The above config will allow unused awaitables in extension modules, but will +still keep the other two error codes enabled. The overall logic is following: + +* Command line and/or config main section set global error codes + +* Individual config sections *adjust* them per glob/module + +* Inline ``# mypy: disable-error-code="..."`` and ``# mypy: enable-error-code="..."`` + comments can further *adjust* them for a specific file. + For example: + +.. code-block:: python + + # mypy: enable-error-code="truthy-bool, ignore-without-code" + +So one can e.g. enable some code globally, disable it for all tests in +the corresponding config section, and then re-enable it with an inline +comment in some specific test. + +Subcodes of error codes +----------------------- + +In some cases, mostly for backwards compatibility reasons, an error +code may be covered also by another, wider error code. For example, an error with +code ``[method-assign]`` can be ignored by ``# type: ignore[assignment]``. +Similar logic works for disabling error codes globally. If a given error code +is a subcode of another one, it will be mentioned in the documentation for the narrower +code. This hierarchy is not nested: there cannot be subcodes of other +subcodes. + + +Requiring error codes +--------------------- + +It's possible to require error codes be specified in ``type: ignore`` comments. +See :ref:`ignore-without-code` for more information. diff --git a/docs/source/existing_code.rst b/docs/source/existing_code.rst new file mode 100644 index 000000000000..0a5ac2bfa8f6 --- /dev/null +++ b/docs/source/existing_code.rst @@ -0,0 +1,251 @@ +.. _existing-code: + +Using mypy with an existing codebase +==================================== + +This section explains how to get started using mypy with an existing, +significant codebase that has little or no type annotations. If you are +a beginner, you can skip this section. + +Start small +----------- + +If your codebase is large, pick a subset of your codebase (say, 5,000 to 50,000 +lines) and get mypy to run successfully only on this subset at first, *before +adding annotations*. This should be doable in a day or two. The sooner you get +some form of mypy passing on your codebase, the sooner you benefit. + +You'll likely need to fix some mypy errors, either by inserting +annotations requested by mypy or by adding ``# type: ignore`` +comments to silence errors you don't want to fix now. + +We'll mention some tips for getting mypy passing on your codebase in various +sections below. + +Run mypy consistently and prevent regressions +--------------------------------------------- + +Make sure all developers on your codebase run mypy the same way. +One way to ensure this is adding a small script with your mypy +invocation to your codebase, or adding your mypy invocation to +existing tools you use to run tests, like ``tox``. + +* Make sure everyone runs mypy with the same options. Checking a mypy + :ref:`configuration file ` into your codebase is the + easiest way to do this. + +* Make sure everyone type checks the same set of files. See + :ref:`specifying-code-to-be-checked` for details. + +* Make sure everyone runs mypy with the same version of mypy, for instance + by pinning mypy with the rest of your dev requirements. + +In particular, you'll want to make sure to run mypy as part of your +Continuous Integration (CI) system as soon as possible. This will +prevent new type errors from being introduced into your codebase. + +A simple CI script could look something like this: + +.. code-block:: text + + python3 -m pip install mypy==1.8 + # Run your standardised mypy invocation, e.g. + mypy my_project + # This could also look like `scripts/run_mypy.sh`, `tox run -e mypy`, `make mypy`, etc + +Ignoring errors from certain modules +------------------------------------ + +By default mypy will follow imports in your code and try to check everything. +This means even if you only pass in a few files to mypy, it may still process a +large number of imported files. This could potentially result in lots of errors +you don't want to deal with at the moment. + +One way to deal with this is to ignore errors in modules you aren't yet ready to +type check. The :confval:`ignore_errors` option is useful for this, for instance, +if you aren't yet ready to deal with errors from ``package_to_fix_later``: + +.. code-block:: text + + [mypy-package_to_fix_later.*] + ignore_errors = True + +You could even invert this, by setting ``ignore_errors = True`` in your global +config section and only enabling error reporting with ``ignore_errors = False`` +for the set of modules you are ready to type check. + +The per-module configuration that mypy's configuration file allows can be +extremely useful. Many configuration options can be enabled or disabled +only for specific modules. In particular, you can also enable or disable +various error codes on a per-module basis, see :ref:`error-codes`. + +Fixing errors related to imports +-------------------------------- + +A common class of error you will encounter is errors from mypy about modules +that it can't find, that don't have types, or don't have stub files: + +.. code-block:: text + + core/config.py:7: error: Cannot find implementation or library stub for module named 'frobnicate' + core/model.py:9: error: Cannot find implementation or library stub for module named 'acme' + ... + +Sometimes these can be fixed by installing the relevant packages or +stub libraries in the environment you're running ``mypy`` in. + +See :ref:`fix-missing-imports` for a complete reference on these errors +and the ways in which you can fix them. + +You'll likely find that you want to suppress all errors from importing +a given module that doesn't have types. If you only import that module +in one or two places, you can use ``# type: ignore`` comments. For example, +here we ignore an error about a third-party module ``frobnicate`` that +doesn't have stubs using ``# type: ignore``: + +.. code-block:: python + + import frobnicate # type: ignore + ... + frobnicate.initialize() # OK (but not checked) + +But if you import the module in many places, this becomes unwieldy. In this +case, we recommend using a :ref:`configuration file `. For example, +to disable errors about importing ``frobnicate`` and ``acme`` everywhere in your +codebase, use a config like this: + +.. code-block:: text + + [mypy-frobnicate.*] + ignore_missing_imports = True + + [mypy-acme.*] + ignore_missing_imports = True + +If you get a large number of errors, you may want to ignore all errors +about missing imports, for instance by setting +:option:`--disable-error-code=import-untyped `. +or setting :confval:`ignore_missing_imports` to true globally. +This can hide errors later on, so we recommend avoiding this +if possible. + +Finally, mypy allows fine-grained control over specific import following +behaviour. It's very easy to silently shoot yourself in the foot when playing +around with these, so this should be a last resort. For more +details, look :ref:`here `. + +Prioritise annotating widely imported modules +--------------------------------------------- + +Most projects have some widely imported modules, such as utilities or +model classes. It's a good idea to annotate these pretty early on, +since this allows code using these modules to be type checked more +effectively. + +Mypy is designed to support gradual typing, i.e. letting you add annotations at +your own pace, so it's okay to leave some of these modules unannotated. The more +you annotate, the more useful mypy will be, but even a little annotation +coverage is useful. + +Write annotations as you go +--------------------------- + +Consider adding something like these in your code style +conventions: + +1. Developers should add annotations for any new code. +2. It's also encouraged to write annotations when you modify existing code. + +This way you'll gradually increase annotation coverage in your +codebase without much effort. + +Automate annotation of legacy code +---------------------------------- + +There are tools for automatically adding draft annotations based on simple +static analysis or on type profiles collected at runtime. Tools include +:doc:`monkeytype:index`, `autotyping`_ and `PyAnnotate`_. + +A simple approach is to collect types from test runs. This may work +well if your test coverage is good (and if your tests aren't very +slow). + +Another approach is to enable type collection for a small, random +fraction of production network requests. This clearly requires more +care, as type collection could impact the reliability or the +performance of your service. + +.. _getting-to-strict: + +Introduce stricter options +-------------------------- + +Mypy is very configurable. Once you get started with static typing, you may want +to explore the various strictness options mypy provides to catch more bugs. For +example, you can ask mypy to require annotations for all functions in certain +modules to avoid accidentally introducing code that won't be type checked using +:confval:`disallow_untyped_defs`. Refer to :ref:`config-file` for the details. + +An excellent goal to aim for is to have your codebase pass when run against ``mypy --strict``. +This basically ensures that you will never have a type related error without an explicit +circumvention somewhere (such as a ``# type: ignore`` comment). + +The following config is equivalent to ``--strict`` (as of mypy 1.0): + +.. code-block:: text + + # Start off with these + warn_unused_configs = True + warn_redundant_casts = True + warn_unused_ignores = True + + # Getting these passing should be easy + strict_equality = True + strict_concatenate = True + + # Strongly recommend enabling this one as soon as you can + check_untyped_defs = True + + # These shouldn't be too much additional work, but may be tricky to + # get passing if you use a lot of untyped libraries + disallow_subclassing_any = True + disallow_untyped_decorators = True + disallow_any_generics = True + + # These next few are various gradations of forcing use of type annotations + disallow_untyped_calls = True + disallow_incomplete_defs = True + disallow_untyped_defs = True + + # This one isn't too hard to get passing, but return on investment is lower + no_implicit_reexport = True + + # This one can be tricky to get passing if you use a lot of untyped libraries + warn_return_any = True + +Note that you can also start with ``--strict`` and subtract, for instance: + +.. code-block:: text + + strict = True + warn_return_any = False + +Remember that many of these options can be enabled on a per-module basis. For instance, +you may want to enable ``disallow_untyped_defs`` for modules which you've completed +annotations for, in order to prevent new code from being added without annotations. + +And if you want, it doesn't stop at ``--strict``. Mypy has additional checks +that are not part of ``--strict`` that can be useful. See the complete +:ref:`command-line` reference and :ref:`error-codes-optional`. + +Speed up mypy runs +------------------ + +You can use :ref:`mypy daemon ` to get much faster +incremental mypy runs. The larger your project is, the more useful +this will be. If your project has at least 100,000 lines of code or +so, you may also want to set up :ref:`remote caching ` +for further speedups. + +.. _PyAnnotate: https://github.com/dropbox/pyannotate +.. _autotyping: https://github.com/JelleZijlstra/autotyping diff --git a/docs/source/extending_mypy.rst b/docs/source/extending_mypy.rst new file mode 100644 index 000000000000..0df45ea22d33 --- /dev/null +++ b/docs/source/extending_mypy.rst @@ -0,0 +1,248 @@ +.. _extending-mypy: + +Extending and integrating mypy +============================== + +.. _integrating-mypy: + +Integrating mypy into another Python application +************************************************ + +It is possible to integrate mypy into another Python 3 application by +importing ``mypy.api`` and calling the ``run`` function with a parameter of type ``list[str]``, containing +what normally would have been the command line arguments to mypy. + +Function ``run`` returns a ``tuple[str, str, int]``, namely +``(, , )``, in which ```` +is what mypy normally writes to :py:data:`sys.stdout`, ```` is what mypy +normally writes to :py:data:`sys.stderr` and ``exit_status`` is the exit status mypy normally +returns to the operating system. + +A trivial example of using the api is the following + +.. code-block:: python + + import sys + from mypy import api + + result = api.run(sys.argv[1:]) + + if result[0]: + print('\nType checking report:\n') + print(result[0]) # stdout + + if result[1]: + print('\nError report:\n') + print(result[1]) # stderr + + print('\nExit status:', result[2]) + + +.. _extending-mypy-using-plugins: + +Extending mypy using plugins +**************************** + +Python is a highly dynamic language and has extensive metaprogramming +capabilities. Many popular libraries use these to create APIs that may +be more flexible and/or natural for humans, but are hard to express using +static types. Extending the :pep:`484` type system to accommodate all existing +dynamic patterns is impractical and often just impossible. + +Mypy supports a plugin system that lets you customize the way mypy type checks +code. This can be useful if you want to extend mypy so it can type check code +that uses a library that is difficult to express using just :pep:`484` types. + +The plugin system is focused on improving mypy's understanding +of *semantics* of third party frameworks. There is currently no way to define +new first class kinds of types. + +.. note:: + + The plugin system is experimental and prone to change. If you want to write + a mypy plugin, we recommend you start by contacting the mypy core developers + on `gitter `_. In particular, there are + no guarantees about backwards compatibility. + + Backwards incompatible changes may be made without a deprecation period, + but we will announce them in + `the plugin API changes announcement issue `_. + +Configuring mypy to use plugins +******************************* + +Plugins are Python files that can be specified in a mypy +:ref:`config file ` using the :confval:`plugins` option and one of the two formats: relative or +absolute path to the plugin file, or a module name (if the plugin +is installed using ``pip install`` in the same virtual environment where mypy +is running). The two formats can be mixed, for example: + +.. code-block:: ini + + [mypy] + plugins = /one/plugin.py, other.plugin + +Mypy will try to import the plugins and will look for an entry point function +named ``plugin``. If the plugin entry point function has a different name, it +can be specified after colon: + +.. code-block:: ini + + [mypy] + plugins = custom_plugin:custom_entry_point + +In the following sections we describe the basics of the plugin system with +some examples. For more technical details, please read the docstrings in +`mypy/plugin.py `_ +in mypy source code. Also you can find good examples in the bundled plugins +located in `mypy/plugins `_. + +High-level overview +******************* + +Every entry point function should accept a single string argument +that is a full mypy version and return a subclass of ``mypy.plugin.Plugin``: + +.. code-block:: python + + from mypy.plugin import Plugin + + class CustomPlugin(Plugin): + def get_type_analyze_hook(self, fullname: str): + # see explanation below + ... + + def plugin(version: str): + # ignore version argument if the plugin works with all mypy versions. + return CustomPlugin + +During different phases of analyzing the code (first in semantic analysis, +and then in type checking) mypy calls plugin methods such as +``get_type_analyze_hook()`` on user plugins. This particular method, for example, +can return a callback that mypy will use to analyze unbound types with the given +full name. See the full plugin hook method list :ref:`below `. + +Mypy maintains a list of plugins it gets from the config file plus the default +(built-in) plugin that is always enabled. Mypy calls a method once for each +plugin in the list until one of the methods returns a non-``None`` value. +This callback will be then used to customize the corresponding aspect of +analyzing/checking the current abstract syntax tree node. + +The callback returned by the ``get_xxx`` method will be given a detailed +current context and an API to create new nodes, new types, emit error messages, +etc., and the result will be used for further processing. + +Plugin developers should ensure that their plugins work well in incremental and +daemon modes. In particular, plugins should not hold global state due to caching +of plugin hook results. + +.. _plugin_hooks: + +Current list of plugin hooks +**************************** + +**get_type_analyze_hook()** customizes behaviour of the type analyzer. +For example, :pep:`484` doesn't support defining variadic generic types: + +.. code-block:: python + + from lib import Vector + + a: Vector[int, int] + b: Vector[int, int, int] + +When analyzing this code, mypy will call ``get_type_analyze_hook("lib.Vector")``, +so the plugin can return some valid type for each variable. + +**get_function_hook()** is used to adjust the return type of a function call. +This hook will be also called for instantiation of classes. +This is a good choice if the return type is too complex +to be expressed by regular python typing. + +**get_function_signature_hook()** is used to adjust the signature of a function. + +**get_method_hook()** is the same as ``get_function_hook()`` but for methods +instead of module level functions. + +**get_method_signature_hook()** is used to adjust the signature of a method. +This includes special Python methods except :py:meth:`~object.__init__` and :py:meth:`~object.__new__`. +For example in this code: + +.. code-block:: python + + from ctypes import Array, c_int + + x: Array[c_int] + x[0] = 42 + +mypy will call ``get_method_signature_hook("ctypes.Array.__setitem__")`` +so that the plugin can mimic the :py:mod:`ctypes` auto-convert behavior. + +**get_attribute_hook()** overrides instance member field lookups and property +access (not method calls). This hook is only called for +fields which already exist on the class. *Exception:* if :py:meth:`__getattr__ ` or +:py:meth:`__getattribute__ ` is a method on the class, the hook is called for all +fields which do not refer to methods. + +**get_class_attribute_hook()** is similar to above, but for attributes on classes rather than instances. +Unlike above, this does not have special casing for :py:meth:`__getattr__ ` or +:py:meth:`__getattribute__ `. + +**get_class_decorator_hook()** can be used to update class definition for +given class decorators. For example, you can add some attributes to the class +to match runtime behaviour: + +.. code-block:: python + + from dataclasses import dataclass + + @dataclass # built-in plugin adds `__init__` method here + class User: + name: str + + user = User(name='example') # mypy can understand this using a plugin + +**get_metaclass_hook()** is similar to above, but for metaclasses. + +**get_base_class_hook()** is similar to above, but for base classes. + +**get_dynamic_class_hook()** can be used to allow dynamic class definitions +in mypy. This plugin hook is called for every assignment to a simple name +where right hand side is a function call: + +.. code-block:: python + + from lib import dynamic_class + + X = dynamic_class('X', []) + +For such definition, mypy will call ``get_dynamic_class_hook("lib.dynamic_class")``. +The plugin should create the corresponding ``mypy.nodes.TypeInfo`` object, and +place it into a relevant symbol table. (Instances of this class represent +classes in mypy and hold essential information such as qualified name, +method resolution order, etc.) + +**get_customize_class_mro_hook()** can be used to modify class MRO (for example +insert some entries there) before the class body is analyzed. + +**get_additional_deps()** can be used to add new dependencies for a +module. It is called before semantic analysis. For example, this can +be used if a library has dependencies that are dynamically loaded +based on configuration information. + +**report_config_data()** can be used if the plugin has some sort of +per-module configuration that can affect typechecking. In that case, +when the configuration for a module changes, we want to invalidate +mypy's cache for that module so that it can be rechecked. This hook +should be used to report to mypy any relevant configuration data, +so that mypy knows to recheck the module if the configuration changes. +The hooks should return data encodable as JSON. + +Useful tools +************ + +Mypy ships ``mypy.plugins.proper_plugin`` plugin which can be useful +for plugin authors, since it finds missing ``get_proper_type()`` calls, +which is a pretty common mistake. + +It is recommended to enable it as a part of your plugin's CI. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index f73ecedfd1ed..b7f5e3759a7e 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -6,7 +6,8 @@ Why have both dynamic and static typing? Dynamic typing can be flexible, powerful, convenient and easy. But it's not always the best approach; there are good reasons why many -developers choose to use staticaly typed languages. +developers choose to use statically typed languages or static typing +for Python. Here are some potential benefits of mypy-style static typing: @@ -35,7 +36,7 @@ Here are some potential benefits of mypy-style static typing: grows, you can adapt tricky application logic to static typing to help maintenance. -See also the `front page `_ of the mypy web +See also the `front page `_ of the mypy web site. Would my project benefit from static typing? @@ -54,25 +55,26 @@ may be useful: - Multiple developers are working on the same code. -- Running tests takes a lot of time or work (type checking may help - you find errors early in development, reducing the number of testing - iterations). +- Running tests takes a lot of time or work (type checking helps + you find errors quickly early in development, reducing the number of + testing iterations). - Some project members (devs or management) don't like dynamic typing, but others prefer dynamic typing and Python syntax. Mypy could be a solution that everybody finds easy to accept. - You want to future-proof your project even if currently none of the - above really apply. + above really apply. The earlier you start, the easier it will be to + adopt static typing. Can I use mypy to type check my existing Python code? ***************************************************** -It depends. Compatibility is pretty good, but several Python features -are not yet implemented. The ultimate goal is to make using mypy -practical for most Python code. Code that uses complex introspection -or metaprogramming may be impractical to type check, but it should -still be possible to use static typing in other parts of a program. +Mypy supports most Python features and idioms, and many large Python +projects are using mypy successfully. Code that uses complex +introspection or metaprogramming may be impractical to type check, but +it should still be possible to use static typing in other parts of a +codebase that are less dynamic. Will static typing make my programs run faster? *********************************************** @@ -80,20 +82,8 @@ Will static typing make my programs run faster? Mypy only does static type checking and it does not improve performance. It has a minimal performance impact. In the future, there could be other tools that can compile statically typed mypy code to C -modules or to efficient JVM bytecode, for example, but this outside -the scope of the mypy project. It may also be possible to modify -existing Python VMs to take advantage of static type information, but -whether this is feasible is still unknown. This is nontrivial since -the runtime types do not necessarily correspond to the static types. - -All of my code is still in Python 2. What are my options? -********************************************************* - -Mypy currently supports Python 3 syntax. Python 2 support is still in -early stages of development. However, Python 2 support will be -improving. Mypy includes a custom codec that lets you use Python 3 -function annotations in Python 2 code. The codec just removes the -annotations before bytecode compilation. +modules or to efficient JVM bytecode, for example, but this is outside +the scope of the mypy project. Is mypy free? ************* @@ -101,44 +91,48 @@ Is mypy free? Yes. Mypy is free software, and it can also be used for commercial and proprietary projects. Mypy is available under the MIT license. -Why not use structural subtyping? -********************************* +Can I use duck typing with mypy? +******************************** -Mypy primarily uses `nominal subtyping -`_ instead of +Mypy provides support for both `nominal subtyping +`_ and `structural subtyping -`_. Some argue -that structural subtyping is better suited for languages with duck -typing such as Python. - -Here are some reasons why mypy uses nominal subtyping: +`_. +Structural subtyping can be thought of as "static duck typing". +Some argue that structural subtyping is better suited for languages with duck +typing such as Python. Mypy however primarily uses nominal subtyping, +leaving structural subtyping mostly opt-in (except for built-in protocols +such as :py:class:`~collections.abc.Iterable` that always support structural +subtyping). Here are some reasons why: 1. It is easy to generate short and informative error messages when using a nominal type system. This is especially important when using type inference. -2. Python supports basically nominal isinstance tests and they are - widely used in programs. It is not clear how to support isinstance - in a purely structural type system while remaining compatible with - Python idioms. +2. Python provides built-in support for nominal :py:func:`isinstance` tests and + they are widely used in programs. Only limited support for structural + :py:func:`isinstance` is available, and it's less type safe than nominal type tests. -3. Many programmers are already familiar with nominal subtyping and it +3. Many programmers are already familiar with static, nominal subtyping and it has been successfully used in languages such as Java, C++ and - C#. Only few languages use structural subtyping. + C#. Fewer languages use structural subtyping. -However, structural subtyping can also be useful. Structural subtyping -is a likely feature to be added to mypy in the future, even though we -expect that most mypy programs will still primarily use nominal -subtyping. +However, structural subtyping can also be useful. For example, a "public API" +may be more flexible if it is typed with protocols. Also, using protocol types +removes the necessity to explicitly declare implementations of ABCs. +As a rule of thumb, we recommend using nominal classes where possible, and +protocols where necessary. For more details about protocol types and structural +subtyping see :ref:`protocol-types` and :pep:`544`. I like Python and I have no need for static typing ************************************************** -That wasn't really a question, was it? Mypy is not aimed at replacing -Python. The goal is to give more options for Python programmers, to +The aim of mypy is not to convince everybody to write statically typed +Python -- static typing is entirely optional, now and in the +future. The goal is to give more options for Python programmers, to make Python a more competitive alternative to other statically typed -languages in large projects, to improve programmer productivity and to -improve software quality. +languages in large projects, to improve programmer productivity, and +to improve software quality. How are mypy programs different from normal Python? *************************************************** @@ -146,37 +140,23 @@ How are mypy programs different from normal Python? Since you use a vanilla Python implementation to run mypy programs, mypy programs are also Python programs. The type checker may give warnings for some valid Python code, but the code is still always -runnable. Also, some Python features and syntax are still not +runnable. Also, a few Python features are still not supported by mypy, but this is gradually improving. The obvious difference is the availability of static type checking. The section :ref:`common_issues` mentions some modifications to Python code that may be required to make code type -check without errors. Also, your code must make attributes explicit and -use a explicit protocol representation. For example, you may want to -subclass an Abstract Base Class such as ``typing.Iterable``. +check without errors. Also, your code must make defined +attributes explicit. -Mypy will support modular, efficient type checking, and this seems to +Mypy supports modular, efficient type checking, and this seems to rule out type checking some language features, such as arbitrary -runtime addition of methods. However, it is likely that many of these -features will be supported in a restricted form (for example, runtime -modification is only supported for classes or methods registered as -dynamic or 'patchable'). - -How is mypy different from PyPy? -******************************** - -*This answer relates to PyPy as a Python implementation. See also the answer related to RPython below.* - -Mypy and PyPy are orthogonal. Mypy does static type checking, i.e. it -is basically a linter, but static typing has no runtime effect, -whereas the PyPy is an Python implementation. You can use PyPy to run -mypy programs. +monkey patching of methods. How is mypy different from Cython? ********************************** -`Cython `_ is a variant of Python that supports +:doc:`Cython ` is a variant of Python that supports compilation to CPython C modules. It can give major speedups to certain classes of programs compared to CPython, and it provides static typing (though this is different from mypy). Mypy differs in @@ -211,58 +191,19 @@ the following aspects, among others: Python semantics, and mypy does not deal with accessing C library functionality. -How is mypy different from Nuitka? -********************************** - -`Nuitka `_ is a static compiler that can translate -Python programs to C++. Nuitka integrates with the CPython -runtime. Nuitka has additional future goals, such as using type -inference and whole-program analysis to further speed up code. Here -are some differences: - -- Nuitka is primarily focused on speeding up Python code. Mypy focuses - on static type checking and facilitating better tools. - -- Whole-program analysis tends to be slow and scale poorly to large or - complex programs. It is still unclear if Nuitka can solve these - issues. Mypy does not use whole-program analysis and will support - modular type checking (though this has not been implemented yet). - -How is mypy different from RPython or Shed Skin? -************************************************ - -`RPython `_ and `Shed -Skin `_ are basically statically -typed subsets of Python. Mypy does the following important things -differently: - -- RPython is primarily designed for implementing virtual machines; - mypy is a general-purpose language. - -- Mypy supports both static and dynamic typing. Dynamically typed and - statically typed code can be freely mixed and can interact - seamlessly. - -- Mypy aims to support (in the future) fast and modular type - checking. Both RPython and Shed Skin use whole-program type - inference which is very slow, does not scale well to large programs - and often produces confusing error messages. Mypy can support - modularity since it only uses local type inference; static type - checking depends on having type annotatations for functions - signatures. - -- Mypy will support introspection, dynamic loading of code and many - other dynamic language features (though using these may make static - typing less effective). RPython and Shed Skin only support a - restricted Python subset without several of these features. +Does it run on PyPy? +********************* -- Mypy supports user-defined generic types. +Somewhat. With PyPy 3.8, mypy is at least able to type check itself. +With older versions of PyPy, mypy relies on `typed-ast +`_, which uses several APIs that +PyPy does not support (including some internal CPython APIs). Mypy is a cool project. Can I help? *********************************** Any help is much appreciated! `Contact -`_ the developers if you would +`_ the developers if you would like to contribute. Any help related to development, design, publicity, documentation, testing, web site maintenance, financing, etc. can be helpful. You can learn a lot by contributing, and anybody diff --git a/docs/source/final_attrs.rst b/docs/source/final_attrs.rst new file mode 100644 index 000000000000..81bfba650430 --- /dev/null +++ b/docs/source/final_attrs.rst @@ -0,0 +1,237 @@ +.. _final_attrs: + +Final names, methods and classes +================================ + +This section introduces these related features: + +1. *Final names* are variables or attributes that should not be reassigned after + initialization. They are useful for declaring constants. +2. *Final methods* should not be overridden in a subclass. +3. *Final classes* should not be subclassed. + +All of these are only enforced by mypy, and only in annotated code. +There is no runtime enforcement by the Python runtime. + +.. note:: + + The examples in this page import ``Final`` and ``final`` from the + ``typing`` module. These types were added to ``typing`` in Python 3.8, + but are also available for use in Python 3.4 - 3.7 via the + ``typing_extensions`` package. + +Final names +----------- + +You can use the ``typing.Final`` qualifier to indicate that +a name or attribute should not be reassigned, redefined, or +overridden. This is often useful for module and class-level +constants to prevent unintended modification. Mypy will prevent +further assignments to final names in type-checked code: + +.. code-block:: python + + from typing import Final + + RATE: Final = 3_000 + + class Base: + DEFAULT_ID: Final = 0 + + RATE = 300 # Error: can't assign to final attribute + Base.DEFAULT_ID = 1 # Error: can't override a final attribute + +Another use case for final attributes is to protect certain attributes +from being overridden in a subclass: + +.. code-block:: python + + from typing import Final + + class Window: + BORDER_WIDTH: Final = 2.5 + ... + + class ListView(Window): + BORDER_WIDTH = 3 # Error: can't override a final attribute + +You can use :py:class:`@property ` to make an attribute read-only, but unlike ``Final``, +it doesn't work with module attributes, and it doesn't prevent overriding in +subclasses. + +Syntax variants +*************** + +You can use ``Final`` in one of these forms: + +* You can provide an explicit type using the syntax ``Final[]``. Example: + + .. code-block:: python + + ID: Final[int] = 1 + + Here, mypy will infer type ``int`` for ``ID``. + +* You can omit the type: + + .. code-block:: python + + ID: Final = 1 + + Here, mypy will infer type ``Literal[1]`` for ``ID``. Note that unlike for + generic classes, this is *not* the same as ``Final[Any]``. + +* In class bodies and stub files, you can omit the right-hand side and just write + ``ID: Final[int]``. + +* Finally, you can write ``self.id: Final = 1`` (also optionally with + a type in square brackets). This is allowed *only* in + :py:meth:`__init__ ` methods so the final instance attribute is + assigned only once when an instance is created. + +Details of using ``Final`` +************************** + +These are the two main rules for defining a final name: + +* There can be *at most one* final declaration per module or class for + a given attribute. There can't be separate class-level and instance-level + constants with the same name. + +* There must be *exactly one* assignment to a final name. + +A final attribute declared in a class body without an initializer must +be initialized in the :py:meth:`__init__ ` method (you can skip the +initializer in stub files): + +.. code-block:: python + + class ImmutablePoint: + x: Final[int] + y: Final[int] # Error: final attribute without an initializer + + def __init__(self) -> None: + self.x = 1 # Good + +``Final`` can only be used as the outermost type in assignments or variable +annotations. Using it in any other position is an error. In particular, +``Final`` can't be used in annotations for function arguments: + +.. code-block:: python + + x: list[Final[int]] = [] # Error! + + def fun(x: Final[list[int]]) -> None: # Error! + ... + +``Final`` and :py:data:`~typing.ClassVar` should not be used together. Mypy will infer +the scope of a final declaration automatically depending on whether it was +initialized in the class body or in :py:meth:`__init__ `. + +A final attribute can't be overridden by a subclass (even with another +explicit final declaration). Note, however, that a final attribute can +override a read-only property: + +.. code-block:: python + + class Base: + @property + def ID(self) -> int: ... + + class Derived(Base): + ID: Final = 1 # OK + +Declaring a name as final only guarantees that the name will not be re-bound +to another value. It doesn't make the value immutable. You can use immutable ABCs +and containers to prevent mutating such values: + +.. code-block:: python + + x: Final = ['a', 'b'] + x.append('c') # OK + + y: Final[Sequence[str]] = ['a', 'b'] + y.append('x') # Error: Sequence is immutable + z: Final = ('a', 'b') # Also an option + +Final methods +------------- + +Like with attributes, sometimes it is useful to protect a method from +overriding. You can use the ``typing.final`` decorator for this purpose: + +.. code-block:: python + + from typing import final + + class Base: + @final + def common_name(self) -> None: + ... + + class Derived(Base): + def common_name(self) -> None: # Error: cannot override a final method + ... + +This ``@final`` decorator can be used with instance methods, class methods, +static methods, and properties. + +For overloaded methods, you should add ``@final`` on the implementation +to make it final (or on the first overload in stubs): + +.. code-block:: python + + from typing import final, overload + + class Base: + @overload + def method(self) -> None: ... + @overload + def method(self, arg: int) -> int: ... + @final + def method(self, x=None): + ... + +Final classes +------------- + +You can apply the ``typing.final`` decorator to a class to indicate +to mypy that it should not be subclassed: + +.. code-block:: python + + from typing import final + + @final + class Leaf: + ... + + class MyLeaf(Leaf): # Error: Leaf can't be subclassed + ... + +The decorator acts as a declaration for mypy (and as documentation for +humans), but it doesn't actually prevent subclassing at runtime. + +Here are some situations where using a final class may be useful: + +* A class wasn't designed to be subclassed. Perhaps subclassing would not + work as expected, or subclassing would be error-prone. +* Subclassing would make code harder to understand or maintain. + For example, you may want to prevent unnecessarily tight coupling between + base classes and subclasses. +* You want to retain the freedom to arbitrarily change the class implementation + in the future, and these changes might break subclasses. + +An abstract class that defines at least one abstract method or +property and has ``@final`` decorator will generate an error from +mypy since those attributes could never be implemented. + +.. code-block:: python + + from abc import ABCMeta, abstractmethod + from typing import final + + @final + class A(metaclass=ABCMeta): # error: Final class A has abstract attributes "f" + @abstractmethod + def f(self, x: int) -> None: pass diff --git a/docs/source/function_overloading.rst b/docs/source/function_overloading.rst deleted file mode 100644 index e47ed7e8d4ba..000000000000 --- a/docs/source/function_overloading.rst +++ /dev/null @@ -1,42 +0,0 @@ -Function overloading -==================== - -You can define multiple instances of a function with the same name but -different signatures. The first matching signature is selected at -runtime when evaluating each individual call. This enables also a form -of multiple dispatch. - -.. code-block:: python - - from typing import overload - - @overload - def abs(n: int) -> int: - return n if n >= 0 else -n - - @overload - def abs(n: float) -> float: - return n if n >= 0.0 else -n - - abs(-2) # 2 (int) - abs(-1.5) # 1.5 (float) - -Overloaded function variants still define a single runtime object; the -following code is valid: - -.. code-block:: python - - my_abs = abs - my_abs(-2) # 2 (int) - my_abs(-1.5) # 1.5 (float) - -The overload variants must be adjacent in the code. This makes code -clearer, and otherwise there would be awkward corner cases such as -partially defined overloaded functions that could surprise the unwary -programmer. - -.. note:: - - As generic type variables are erased at runtime, an overloaded - function cannot dispatch based on a generic type argument, - e.g. ``List[int]`` versus ``List[str]``. diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 3b3543615fde..4ba6d322417d 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -1,26 +1,64 @@ Generics ======== +This section explains how you can define your own generic classes that take +one or more type arguments, similar to built-in types such as ``list[T]``. +User-defined generics are a moderately advanced feature and you can get far +without ever using them -- feel free to skip this section and come back later. + +.. _generic-classes: + Defining generic classes ************************ The built-in collection classes are generic classes. Generic types -have one or more type parameters, which can be arbitrary types. For -example, ``Dict[int, str]`` has the type parameters ``int`` and -``str``, and ``List[int]`` has a type parameter ``int``. +accept one or more type arguments within ``[...]``, which can be +arbitrary types. For example, the type ``dict[int, str]`` has the +type arguments ``int`` and ``str``, and ``list[int]`` has the type +argument ``int``. Programs can also define new generic classes. Here is a very simple -generic class that represents a stack: +generic class that represents a stack (using the syntax introduced in +Python 3.12): + +.. code-block:: python + + class Stack[T]: + def __init__(self) -> None: + # Create an empty list with items of type T + self.items: list[T] = [] + + def push(self, item: T) -> None: + self.items.append(item) + + def pop(self) -> T: + return self.items.pop() + + def empty(self) -> bool: + return not self.items + +There are two syntax variants for defining generic classes in Python. +Python 3.12 introduced a +`new dedicated syntax `_ +for defining generic classes (and also functions and type aliases, which +we will discuss later). The above example used the new syntax. Most examples are +given using both the new and the old (or legacy) syntax variants. +Unless mentioned otherwise, they work the same -- but the new syntax +is more readable and more convenient. + +Here is the same example using the old syntax (required for Python 3.11 +and earlier, but also supported on newer Python versions): .. code-block:: python - from typing import typevar, Generic + from typing import TypeVar, Generic - T = typevar('T') + T = TypeVar('T') # Define type variable "T" class Stack(Generic[T]): def __init__(self) -> None: - self.items = List[T]() # Create an empty list with items of type T + # Create an empty list with items of type T + self.items: list[T] = [] def push(self, item: T) -> None: self.items.append(item) @@ -31,102 +69,228 @@ generic class that represents a stack: def empty(self) -> bool: return not self.items +.. note:: + + There are currently no plans to deprecate the legacy syntax. + You can freely mix code using the new and old syntax variants, + even within a single file (but *not* within a single class). + The ``Stack`` class can be used to represent a stack of any type: -``Stack[int]``, ``Stack[Tuple[int, str]]``, etc. +``Stack[int]``, ``Stack[tuple[int, str]]``, etc. You can think of +``Stack[int]`` as referring to the definition of ``Stack`` above, +but with all instances of ``T`` replaced with ``int``. Using ``Stack`` is similar to built-in container types: .. code-block:: python - stack = Stack[int]() # Construct an empty Stack[int] instance + # Construct an empty Stack[int] instance + stack = Stack[int]() stack.push(2) stack.pop() - stack.push('x') # Type error -Type inference works for user-defined generic types as well: + # error: Argument 1 to "push" of "Stack" has incompatible type "str"; expected "int" + stack.push('x') + + stack2: Stack[str] = Stack() + stack2.append('x') + +Construction of instances of generic types is type checked (Python 3.12 syntax): .. code-block:: python - def process(stack: Stack[int]) -> None: ... + class Box[T]: + def __init__(self, content: T) -> None: + self.content = content - process(Stack()) # Argument has inferred type Stack[int] + Box(1) # OK, inferred type is Box[int] + Box[int](1) # Also OK -Generic class internals -*********************** + # error: Argument 1 to "Box" has incompatible type "str"; expected "int" + Box[int]('some string') -You may wonder what happens at runtime when you index -``Stack``. Actually, indexing ``Stack`` just returns ``Stack``: +Here is the definition of ``Box`` using the legacy syntax (Python 3.11 and earlier): ->>> print(Stack) - ->>> print(Stack[int]) - +.. code-block:: python -Note that built-in types ``list``, ``dict`` and so on do not support -indexing in Python. This is why we have the aliases ``List``, ``Dict`` -and so on in the ``typing`` module. Indexing these aliases just gives -you the target class in Python, similar to ``Stack``: + from typing import TypeVar, Generic ->>> from typing import List ->>> List[int] - + T = TypeVar('T') -The above examples illustrate that type variables are erased at -runtime. Generic ``Stack`` or ``list`` instances are just ordinary -Python objects, and they have no extra runtime overhead or magic due -to being generic, other than a metaclass that overloads the indexing -operator. If you worry about the overhead introduced by the type -indexing operation when constructing instances, you can usually -rewrite such code using a ``# type:`` annotation, which has no runtime -impact: + class Box(Generic[T]): + def __init__(self, content: T) -> None: + self.content = content + +.. note:: + + Before moving on, let's clarify some terminology. + The name ``T`` in ``class Stack[T]`` or ``class Stack(Generic[T])`` + declares a *type parameter* ``T`` (of class ``Stack``). + ``T`` is also called a *type variable*, especially in a type annotation, + such as in the signature of ``push`` above. + When the type ``Stack[...]`` is used in a type annotation, the type + within square brackets is called a *type argument*. + This is similar to the distinction between function parameters and arguments. + +.. _generic-subclasses: + +Defining subclasses of generic classes +************************************** + +User-defined generic classes and generic classes defined in :py:mod:`typing` +can be used as a base class for another class (generic or non-generic). For +example (Python 3.12 syntax): + +.. code-block:: python + + from typing import Mapping, Iterator + + # This is a generic subclass of Mapping + class MyMap[KT, VT](Mapping[KT, VT]): + def __getitem__(self, k: KT) -> VT: ... + def __iter__(self) -> Iterator[KT]: ... + def __len__(self) -> int: ... + + items: MyMap[str, int] # OK + + # This is a non-generic subclass of dict + class StrDict(dict[str, str]): + def __str__(self) -> str: + return f'StrDict({super().__str__()})' + + data: StrDict[int, int] # Error! StrDict is not generic + data2: StrDict # OK + + # This is a user-defined generic class + class Receiver[T]: + def accept(self, value: T) -> None: ... + + # This is a generic subclass of Receiver + class AdvancedReceiver[T](Receiver[T]): ... + +Here is the above example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from typing import Generic, TypeVar, Mapping, Iterator + + KT = TypeVar('KT') + VT = TypeVar('VT') + + # This is a generic subclass of Mapping + class MyMap(Mapping[KT, VT]): + def __getitem__(self, k: KT) -> VT: ... + def __iter__(self) -> Iterator[KT]: ... + def __len__(self) -> int: ... + + items: MyMap[str, int] # OK + + # This is a non-generic subclass of dict + class StrDict(dict[str, str]): + def __str__(self) -> str: + return f'StrDict({super().__str__()})' + + data: StrDict[int, int] # Error! StrDict is not generic + data2: StrDict # OK + + # This is a user-defined generic class + class Receiver(Generic[T]): + def accept(self, value: T) -> None: ... + + # This is a generic subclass of Receiver + class AdvancedReceiver(Receiver[T]): ... + +.. note:: + + You have to add an explicit :py:class:`~collections.abc.Mapping` base class + if you want mypy to consider a user-defined class as a mapping (and + :py:class:`~collections.abc.Sequence` for sequences, etc.). This is because + mypy doesn't use *structural subtyping* for these ABCs, unlike simpler protocols + like :py:class:`~collections.abc.Iterable`, which use + :ref:`structural subtyping `. + +When using the legacy syntax, :py:class:`Generic ` can be omitted +from bases if there are +other base classes that include type variables, such as ``Mapping[KT, VT]`` +in the above example. If you include ``Generic[...]`` in bases, then +it should list all type variables present in other bases (or more, +if needed). The order of type parameters is defined by the following +rules: + +* If ``Generic[...]`` is present, then the order of parameters is + always determined by their order in ``Generic[...]``. +* If there are no ``Generic[...]`` in bases, then all type parameters + are collected in the lexicographic order (i.e. by first appearance). + +Example: .. code-block:: python - x = List[int]() - x = [] # type: List[int] # Like the above but faster. + from typing import Generic, TypeVar, Any + + T = TypeVar('T') + S = TypeVar('S') + U = TypeVar('U') -The savings are rarely significant, but it could make a difference in -a performance-critical loop or function. Function annotations, on the -other hand, are only evaluated during the defintion of the function, -not during every call. Constructing type objects in function -signatures rarely has any noticeable performance impact. + class One(Generic[T]): ... + class Another(Generic[T]): ... + + class First(One[T], Another[S]): ... + class Second(One[T], Another[S], Generic[S, U, T]): ... + + x: First[int, str] # Here T is bound to int, S is bound to str + y: Second[int, str, Any] # Here T is Any, S is int, and U is str + +When using the Python 3.12 syntax, all type parameters must always be +explicitly defined immediately after the class name within ``[...]``, and the +``Generic[...]`` base class is never used. .. _generic-functions: Generic functions ***************** -Generic type variables can also be used to define generic functions: +Functions can also be generic, i.e. they can have type parameters (Python 3.12 syntax): .. code-block:: python - from typing import typevar, Sequence - - T = typevar('T') # Declare type variable + from collections.abc import Sequence - def first(seq: Sequence[T]) -> T: # Generic function + # A generic function! + def first[T](seq: Sequence[T]) -> T: return seq[0] -As with generic classes, the type variable can be replaced with any -type. That means first can we used with any sequence type, and the -return type is derived from the sequence item type. For example: +Here is the same example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python - # Assume first defined as above. + from typing import TypeVar, Sequence + + T = TypeVar('T') + + # A generic function! + def first(seq: Sequence[T]) -> T: + return seq[0] + +As with generic classes, the type parameter ``T`` can be replaced with any +type. That means ``first`` can be passed an argument with any sequence type, +and the return type is derived from the sequence item type. Example: + +.. code-block:: python - s = first('foo') # s has type str. - n = first([1, 2, 3]) # n has type int. + reveal_type(first([1, 2, 3])) # Revealed type is "builtins.int" + reveal_type(first(('a', 'b'))) # Revealed type is "builtins.str" -Note also that a single definition of a type variable (such as ``T`` -above) can be used in multiple generic functions or classes. In this -example we use the same type variable in two generic functions: +When using the legacy syntax, a single definition of a type variable +(such as ``T`` above) can be used in multiple generic functions or +classes. In this example we use the same type variable in two generic +functions to declarare type parameters: .. code-block:: python - from typing typevar, Sequence + from typing import TypeVar, Sequence - T = typevar('T') # Declare type variable + T = TypeVar('T') # Define type variable def first(seq: Sequence[T]) -> T: return seq[0] @@ -134,70 +298,451 @@ example we use the same type variable in two generic functions: def last(seq: Sequence[T]) -> T: return seq[-1] -You can also define generic methods — just use a type variable in the -method signature that is different from class type variables. +Since the Python 3.12 syntax is more concise, it doesn't need (or have) +an equivalent way of sharing type parameter definitions. -Type variables with value restriction -************************************* +A variable cannot have a type variable in its type unless the type +variable is bound in a containing generic class or function. -By default, a type variable can be replaced with any type. However, sometimes -it's useful to have a type variable that can only have some specific types -as its value. A typical example is a type variable that can only have values -``str`` and ``bytes``: +When calling a generic function, you can't explicitly pass the values of +type parameters as type arguments. The values of type parameters are always +inferred by mypy. This is not valid: .. code-block:: python - from typing import typevar + first[int]([1, 2]) # Error: can't use [...] with generic function + +If you really need this, you can define a generic class with a ``__call__`` +method. - AnyStr = typevar('AnyStr', values=(str, bytes)) +.. _type-variable-upper-bound: -This is actually such a common type variable that ``AnyStr`` is -defined in ``typing`` and we don't need to define it ourselves. +Type variables with upper bounds +******************************** -We can use ``AnyStr`` to define a function that can concatenate -two strings or bytes objects, but it can't be called with other -argument types: +A type variable can also be restricted to having values that are +subtypes of a specific type. This type is called the upper bound of +the type variable, and it is specified using ``T: `` when using the +Python 3.12 syntax. In the definition of a generic function or a generic +class that uses such a type variable ``T``, the type represented by ``T`` +is assumed to be a subtype of its upper bound, so you can use methods +of the upper bound on values of type ``T`` (Python 3.12 syntax): .. code-block:: python - from typing import AnyStr + from typing import SupportsAbs - def concat(x: AnyStr, y: AnyStr) -> AnyStr: + def max_by_abs[T: SupportsAbs[float]](*xs: T) -> T: + # We can use abs(), because T is a subtype of SupportsAbs[float]. + return max(xs, key=abs) + +An upper bound can also be specified with the ``bound=...`` keyword +argument to :py:class:`~typing.TypeVar`. +Here is the example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar, SupportsAbs + + T = TypeVar('T', bound=SupportsAbs[float]) + + def max_by_abs(*xs: T) -> T: + return max(xs, key=abs) + +In a call to such a function, the type ``T`` must be replaced by a +type that is a subtype of its upper bound. Continuing the example +above: + +.. code-block:: python + + max_by_abs(-3.5, 2) # Okay, has type 'float' + max_by_abs(5+6j, 7) # Okay, has type 'complex' + max_by_abs('a', 'b') # Error: 'str' is not a subtype of SupportsAbs[float] + +Type parameters of generic classes may also have upper bounds, which +restrict the valid values for the type parameter in the same way. + +.. _generic-methods-and-generic-self: + +Generic methods and generic self +******************************** + +You can also define generic methods. In +particular, the ``self`` parameter may also be generic, allowing a +method to return the most precise type known at the point of access. +In this way, for example, you can type check a chain of setter +methods (Python 3.12 syntax): + +.. code-block:: python + + class Shape: + def set_scale[T: Shape](self: T, scale: float) -> T: + self.scale = scale + return self + + class Circle(Shape): + def set_radius(self, r: float) -> 'Circle': + self.radius = r + return self + + class Square(Shape): + def set_width(self, w: float) -> 'Square': + self.width = w + return self + + circle: Circle = Circle().set_scale(0.5).set_radius(2.7) + square: Square = Square().set_scale(0.5).set_width(3.2) + +Without using generic ``self``, the last two lines could not be type +checked properly, since the return type of ``set_scale`` would be +``Shape``, which doesn't define ``set_radius`` or ``set_width``. + +When using the legacy syntax, just use a type variable in the +method signature that is different from class type parameters (if any +are defined). Here is the above example using the legacy +syntax (3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar + + T = TypeVar('T', bound='Shape') + + class Shape: + def set_scale(self: T, scale: float) -> T: + self.scale = scale + return self + + class Circle(Shape): + def set_radius(self, r: float) -> 'Circle': + self.radius = r + return self + + class Square(Shape): + def set_width(self, w: float) -> 'Square': + self.width = w + return self + + circle: Circle = Circle().set_scale(0.5).set_radius(2.7) + square: Square = Square().set_scale(0.5).set_width(3.2) + +Other uses include factory methods, such as copy and deserialization methods. +For class methods, you can also define generic ``cls``, using ``type[T]`` +or :py:class:`Type[T] ` (Python 3.12 syntax): + +.. code-block:: python + + class Friend: + other: "Friend | None" = None + + @classmethod + def make_pair[T: Friend](cls: type[T]) -> tuple[T, T]: + a, b = cls(), cls() + a.other = b + b.other = a + return a, b + + class SuperFriend(Friend): + pass + + a, b = SuperFriend.make_pair() + +Here is the same example using the legacy syntax (3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar + + T = TypeVar('T', bound='Friend') + + class Friend: + other: "Friend | None" = None + + @classmethod + def make_pair(cls: type[T]) -> tuple[T, T]: + a, b = cls(), cls() + a.other = b + b.other = a + return a, b + + class SuperFriend(Friend): + pass + + a, b = SuperFriend.make_pair() + +Note that when overriding a method with generic ``self``, you must either +return a generic ``self`` too, or return an instance of the current class. +In the latter case, you must implement this method in all future subclasses. + +Note also that mypy cannot always verify that the implementation of a copy +or a deserialization method returns the actual type of self. Therefore +you may need to silence mypy inside these methods (but not at the call site), +possibly by making use of the ``Any`` type or a ``# type: ignore`` comment. + +Mypy lets you use generic self types in certain unsafe ways +in order to support common idioms. For example, using a generic +self type in an argument type is accepted even though it's unsafe (Python 3.12 +syntax): + +.. code-block:: python + + class Base: + def compare[T: Base](self: T, other: T) -> bool: + return False + + class Sub(Base): + def __init__(self, x: int) -> None: + self.x = x + + # This is unsafe (see below) but allowed because it's + # a common pattern and rarely causes issues in practice. + def compare(self, other: 'Sub') -> bool: + return self.x > other.x + + b: Base = Sub(42) + b.compare(Base()) # Runtime error here: 'Base' object has no attribute 'x' + +For some advanced uses of self types, see :ref:`additional examples `. + +Automatic self types using typing.Self +************************************** + +Since the patterns described above are quite common, mypy supports a +simpler syntax, introduced in :pep:`673`, to make them easier to use. +Instead of introducing a type parameter and using an explicit annotation +for ``self``, you can import the special type ``typing.Self`` that is +automatically transformed into a method-level type parameter with the +current class as the upper bound, and you don't need an annotation for +``self`` (or ``cls`` in class methods). The example from the previous +section can be made simpler by using ``Self``: + +.. code-block:: python + + from typing import Self + + class Friend: + other: Self | None = None + + @classmethod + def make_pair(cls) -> tuple[Self, Self]: + a, b = cls(), cls() + a.other = b + b.other = a + return a, b + + class SuperFriend(Friend): + pass + + a, b = SuperFriend.make_pair() + +This is more compact than using explicit type parameters. Also, you can +use ``Self`` in attribute annotations in addition to methods. + +.. note:: + + To use this feature on Python versions earlier than 3.11, you will need to + import ``Self`` from ``typing_extensions`` (version 4.0 or newer). + +.. _variance-of-generics: + +Variance of generic types +************************* + +There are three main kinds of generic types with respect to subtype +relations between them: invariant, covariant, and contravariant. +Assuming that we have a pair of types ``A`` and ``B``, and ``B`` is +a subtype of ``A``, these are defined as follows: + +* A generic class ``MyCovGen[T]`` is called covariant in type variable + ``T`` if ``MyCovGen[B]`` is always a subtype of ``MyCovGen[A]``. +* A generic class ``MyContraGen[T]`` is called contravariant in type + variable ``T`` if ``MyContraGen[A]`` is always a subtype of + ``MyContraGen[B]``. +* A generic class ``MyInvGen[T]`` is called invariant in ``T`` if neither + of the above is true. + +Let us illustrate this by few simple examples: + +.. code-block:: python + + # We'll use these classes in the examples below + class Shape: ... + class Triangle(Shape): ... + class Square(Shape): ... + +* Most immutable container types, such as :py:class:`~collections.abc.Sequence` + and :py:class:`~frozenset` are covariant. Union types are + also covariant in all union items: ``Triangle | int`` is + a subtype of ``Shape | int``. + + .. code-block:: python + + def count_lines(shapes: Sequence[Shape]) -> int: + return sum(shape.num_sides for shape in shapes) + + triangles: Sequence[Triangle] + count_lines(triangles) # OK + + def foo(triangle: Triangle, num: int) -> None: + shape_or_number: Union[Shape, int] + # a Triangle is a Shape, and a Shape is a valid Union[Shape, int] + shape_or_number = triangle + + Covariance should feel relatively intuitive, but contravariance and invariance + can be harder to reason about. + +* :py:class:`~collections.abc.Callable` is an example of type that behaves contravariant + in types of arguments. That is, ``Callable[[Shape], int]`` is a subtype of + ``Callable[[Triangle], int]``, despite ``Shape`` being a supertype of + ``Triangle``. To understand this, consider: + + .. code-block:: python + + def cost_of_paint_required( + triangle: Triangle, + area_calculator: Callable[[Triangle], float] + ) -> float: + return area_calculator(triangle) * DOLLAR_PER_SQ_FT + + # This straightforwardly works + def area_of_triangle(triangle: Triangle) -> float: ... + cost_of_paint_required(triangle, area_of_triangle) # OK + + # But this works as well! + def area_of_any_shape(shape: Shape) -> float: ... + cost_of_paint_required(triangle, area_of_any_shape) # OK + + ``cost_of_paint_required`` needs a callable that can calculate the area of a + triangle. If we give it a callable that can calculate the area of an + arbitrary shape (not just triangles), everything still works. + +* ``list`` is an invariant generic type. Naively, one would think + that it is covariant, like :py:class:`~collections.abc.Sequence` above, but consider this code: + + .. code-block:: python + + class Circle(Shape): + # The rotate method is only defined on Circle, not on Shape + def rotate(self): ... + + def add_one(things: list[Shape]) -> None: + things.append(Shape()) + + my_circles: list[Circle] = [] + add_one(my_circles) # This may appear safe, but... + my_circles[-1].rotate() # ...this will fail, since my_circles[0] is now a Shape, not a Circle + + Another example of invariant type is ``dict``. Most mutable containers + are invariant. + +When using the Python 3.12 syntax for generics, mypy will automatically +infer the most flexible variance for each class type variable. Here +``Box`` will be inferred as covariant: + +.. code-block:: python + + class Box[T]: # this type is implicitly covariant + def __init__(self, content: T) -> None: + self._content = content + + def get_content(self) -> T: + return self._content + + def look_into(box: Box[Shape]): ... + + my_box = Box(Square()) + look_into(my_box) # OK, but mypy would complain here for an invariant type + +Here the underscore prefix for ``_content`` is significant. Without an +underscore prefix, the class would be invariant, as the attribute would +be understood as a public, mutable attribute (a single underscore prefix +has no special significance for mypy in most other contexts). By declaring +the attribute as ``Final``, the class could still be made covariant: + +.. code-block:: python + + from typing import Final + + class Box[T]: # this type is implicitly covariant + def __init__(self, content: T) -> None: + self.content: Final = content + + def get_content(self) -> T: + return self.content + +When using the legacy syntax, mypy assumes that all user-defined generics +are invariant by default. To declare a given generic class as covariant or +contravariant, use type variables defined with special keyword arguments +``covariant`` or ``contravariant``. For example (Python 3.11 or earlier): + +.. code-block:: python + + from typing import Generic, TypeVar + + T_co = TypeVar('T_co', covariant=True) + + class Box(Generic[T_co]): # this type is declared covariant + def __init__(self, content: T_co) -> None: + self._content = content + + def get_content(self) -> T_co: + return self._content + + def look_into(box: Box[Shape]): ... + + my_box = Box(Square()) + look_into(my_box) # OK, but mypy would complain here for an invariant type + +.. _type-variable-value-restriction: + +Type variables with value restriction +************************************* + +By default, a type variable can be replaced with any type -- or any type that +is a subtype of the upper bound, which defaults to ``object``. However, sometimes +it's useful to have a type variable that can only have some specific types +as its value. A typical example is a type variable that can only have values +``str`` and ``bytes``. This lets us define a function that can concatenate +two strings or bytes objects, but it can't be called with other argument +types (Python 3.12 syntax): + +.. code-block:: python + + def concat[S: (str, bytes)](x: S, y: S) -> S: return x + y concat('a', 'b') # Okay concat(b'a', b'b') # Okay concat(1, 2) # Error! -Note that this is different from a union type, since combinations -of ``str`` and ``bytes`` are not accepted: + +The same thing is also possibly using the legacy syntax (Python 3.11 or earlier): .. code-block:: python - concat('string', b'bytes') # Error! + from typing import TypeVar -In this case, this is exactly what we want, since it's not possible -to concatenate a string and a bytes object! The type checker -will reject this function: + AnyStr = TypeVar('AnyStr', str, bytes) + + def concat(x: AnyStr, y: AnyStr) -> AnyStr: + return x + y + +No matter which syntax you use, such a type variable is called a type variable +with a value restriction. Importantly, this is different from a union type, +since combinations of ``str`` and ``bytes`` are not accepted: .. code-block:: python - def union_concat(x: Union[str, bytes], y: Union[str, bytes]) -> Union[str, bytes]: - return x + y # Error: can't concatenate str and bytes + concat('string', b'bytes') # Error! -The original, valid definition of ``concat`` is more or less -equivalent to this overloaded function, but it's much shorter, -cleaner and more efficient: +In this case, this is exactly what we want, since it's not possible +to concatenate a string and a bytes object! If we tried to use +a union type, the type checker would complain about this possibility: .. code-block:: python - @overload - def overload_concat(x: str, y: str) -> str: - return x + y - - @overload - def overload_concat(x: bytes, y: bytes) -> bytes: - return x + y + def union_concat(x: str | bytes, y: str | bytes) -> str | bytes: + return x + y # Error: can't concatenate str and bytes Another interesting special case is calling ``concat()`` with a subtype of ``str``: @@ -206,22 +751,687 @@ subtype of ``str``: class S(str): pass - ss = concat(S('foo'), S('bar'))) + ss = concat(S('foo'), S('bar')) + reveal_type(ss) # Revealed type is "builtins.str" You may expect that the type of ``ss`` is ``S``, but the type is actually ``str``: a subtype gets promoted to one of the valid values -for the type variable, which in this case is ``str``. This is thus -subtly different from *bounded quantification* in languages such as -Java, where the return type would be ``S``. The way mypy implements -this is correct for ``concat``, since ``concat`` actually returns a -``str`` instance in the above example: +for the type variable, which in this case is ``str``. + +This is thus subtly different from using ``str | bytes`` as an upper bound, +where the return type would be ``S`` (see :ref:`type-variable-upper-bound`). +Using a value restriction is correct for ``concat``, since ``concat`` +actually returns a ``str`` instance in the above example: .. code-block:: python - >>> prin(type(ss)) + >>> print(type(ss)) -You can also use a ``typevar`` with ``values`` when defining a generic -class. For example, mypy uses the type ``typing.Pattern[AnyStr]`` for the -return value of ``re.compile``, since regular expressions can be based -on a string or a bytes pattern. +You can also use type variables with a restricted set of possible +values when defining a generic class. For example, the type +:py:class:`Pattern[S] ` is used for the return +value of :py:func:`re.compile`, where ``S`` can be either ``str`` +or ``bytes``. Regular expressions can be based on a string or a +bytes pattern. + +A type variable may not have both a value restriction and an upper bound. + +Note that you may come across :py:data:`~typing.AnyStr` imported from +:py:mod:`typing`. This feature is now deprecated, but it means the same +as our definition of ``AnyStr`` above. + +.. _declaring-decorators: + +Declaring decorators +******************** + +Decorators are typically functions that take a function as an argument and +return another function. Describing this behaviour in terms of types can +be a little tricky; we'll show how you can use type variables and a special +kind of type variable called a *parameter specification* to do so. + +Suppose we have the following decorator, not type annotated yet, +that preserves the original function's signature and merely prints the decorated +function's name: + +.. code-block:: python + + def printing_decorator(func): + def wrapper(*args, **kwds): + print("Calling", func) + return func(*args, **kwds) + return wrapper + +We can use it to decorate function ``add_forty_two``: + +.. code-block:: python + + # A decorated function. + @printing_decorator + def add_forty_two(value: int) -> int: + return value + 42 + + a = add_forty_two(3) + +Since ``printing_decorator`` is not type-annotated, the following won't get type checked: + +.. code-block:: python + + reveal_type(a) # Revealed type is "Any" + add_forty_two('foo') # No type checker error :( + +This is a sorry state of affairs! If you run with ``--strict``, mypy will +even alert you to this fact: +``Untyped decorator makes function "add_forty_two" untyped`` + +Note that class decorators are handled differently than function decorators in +mypy: decorating a class does not erase its type, even if the decorator has +incomplete type annotations. + +Here's how one could annotate the decorator (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Callable + from typing import Any, cast + + # A decorator that preserves the signature. + def printing_decorator[F: Callable[..., Any]](func: F) -> F: + def wrapper(*args, **kwds): + print("Calling", func) + return func(*args, **kwds) + return cast(F, wrapper) + + @printing_decorator + def add_forty_two(value: int) -> int: + return value + 42 + + a = add_forty_two(3) + reveal_type(a) # Revealed type is "builtins.int" + add_forty_two('x') # Argument 1 to "add_forty_two" has incompatible type "str"; expected "int" + +Here is the example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Callable + from typing import Any, TypeVar, cast + + F = TypeVar('F', bound=Callable[..., Any]) + + # A decorator that preserves the signature. + def printing_decorator(func: F) -> F: + def wrapper(*args, **kwds): + print("Calling", func) + return func(*args, **kwds) + return cast(F, wrapper) + + @printing_decorator + def add_forty_two(value: int) -> int: + return value + 42 + + a = add_forty_two(3) + reveal_type(a) # Revealed type is "builtins.int" + add_forty_two('x') # Argument 1 to "add_forty_two" has incompatible type "str"; expected "int" + +This still has some shortcomings. First, we need to use the unsafe +:py:func:`~typing.cast` to convince mypy that ``wrapper()`` has the same +signature as ``func`` (see :ref:`casts `). + +Second, the ``wrapper()`` function is not tightly type checked, although +wrapper functions are typically small enough that this is not a big +problem. This is also the reason for the :py:func:`~typing.cast` call in the +``return`` statement in ``printing_decorator()``. + +However, we can use a parameter specification, introduced using ``**P``, +for a more faithful type annotation (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Callable + + def printing_decorator[**P, T](func: Callable[P, T]) -> Callable[P, T]: + def wrapper(*args: P.args, **kwds: P.kwargs) -> T: + print("Calling", func) + return func(*args, **kwds) + return wrapper + +The same is possible using the legacy syntax with :py:class:`~typing.ParamSpec` +(Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Callable + from typing import TypeVar + from typing_extensions import ParamSpec + + P = ParamSpec('P') + T = TypeVar('T') + + def printing_decorator(func: Callable[P, T]) -> Callable[P, T]: + def wrapper(*args: P.args, **kwds: P.kwargs) -> T: + print("Calling", func) + return func(*args, **kwds) + return wrapper + +Parameter specifications also allow you to describe decorators that +alter the signature of the input function (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Callable + + # We reuse 'P' in the return type, but replace 'T' with 'str' + def stringify[**P, T](func: Callable[P, T]) -> Callable[P, str]: + def wrapper(*args: P.args, **kwds: P.kwargs) -> str: + return str(func(*args, **kwds)) + return wrapper + + @stringify + def add_forty_two(value: int) -> int: + return value + 42 + + a = add_forty_two(3) + reveal_type(a) # Revealed type is "builtins.str" + add_forty_two('x') # error: Argument 1 to "add_forty_two" has incompatible type "str"; expected "int" + +Here is the above example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Callable + from typing import TypeVar + from typing_extensions import ParamSpec + + P = ParamSpec('P') + T = TypeVar('T') + + # We reuse 'P' in the return type, but replace 'T' with 'str' + def stringify(func: Callable[P, T]) -> Callable[P, str]: + def wrapper(*args: P.args, **kwds: P.kwargs) -> str: + return str(func(*args, **kwds)) + return wrapper + +You can also insert an argument in a decorator (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Callable + from typing import Concatenate + + def printing_decorator[**P, T](func: Callable[P, T]) -> Callable[Concatenate[str, P], T]: + def wrapper(msg: str, /, *args: P.args, **kwds: P.kwargs) -> T: + print("Calling", func, "with", msg) + return func(*args, **kwds) + return wrapper + + @printing_decorator + def add_forty_two(value: int) -> int: + return value + 42 + + a = add_forty_two('three', 3) + +Here is the same function using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Callable + from typing import TypeVar + from typing_extensions import Concatenate, ParamSpec + + P = ParamSpec('P') + T = TypeVar('T') + + def printing_decorator(func: Callable[P, T]) -> Callable[Concatenate[str, P], T]: + def wrapper(msg: str, /, *args: P.args, **kwds: P.kwargs) -> T: + print("Calling", func, "with", msg) + return func(*args, **kwds) + return wrapper + +.. _decorator-factories: + +Decorator factories +------------------- + +Functions that take arguments and return a decorator (also called second-order decorators), are +similarly supported via generics (Python 3.12 syntax): + +.. code-block:: python + + from colletions.abc import Callable + from typing import Any + + def route[F: Callable[..., Any]](url: str) -> Callable[[F], F]: + ... + + @route(url='/') + def index(request: Any) -> str: + return 'Hello world' + +Note that mypy infers that ``F`` is used to make the ``Callable`` return value +of ``route`` generic, instead of making ``route`` itself generic, since ``F`` is +only used in the return type. Python has no explicit syntax to mark that ``F`` +is only bound in the return value. + +Here is the example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Callable + from typing import Any, TypeVar + + F = TypeVar('F', bound=Callable[..., Any]) + + def route(url: str) -> Callable[[F], F]: + ... + + @route(url='/') + def index(request: Any) -> str: + return 'Hello world' + +Sometimes the same decorator supports both bare calls and calls with arguments. This can be +achieved by combining with :py:func:`@overload ` (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Callable + from typing import Any, overload + + # Bare decorator usage + @overload + def atomic[F: Callable[..., Any]](func: F, /) -> F: ... + # Decorator with arguments + @overload + def atomic[F: Callable[..., Any]](*, savepoint: bool = True) -> Callable[[F], F]: ... + + # Implementation + def atomic(func: Callable[..., Any] | None = None, /, *, savepoint: bool = True): + def decorator(func: Callable[..., Any]): + ... # Code goes here + if __func is not None: + return decorator(__func) + else: + return decorator + + # Usage + @atomic + def func1() -> None: ... + + @atomic(savepoint=False) + def func2() -> None: ... + +Here is the decorator from the example using the legacy syntax +(Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Callable + from typing import Any, Optional, TypeVar, overload + + F = TypeVar('F', bound=Callable[..., Any]) + + # Bare decorator usage + @overload + def atomic(func: F, /) -> F: ... + # Decorator with arguments + @overload + def atomic(*, savepoint: bool = True) -> Callable[[F], F]: ... + + # Implementation + def atomic(func: Optional[Callable[..., Any]] = None, /, *, savepoint: bool = True): + ... # Same as above + +Generic protocols +***************** + +Mypy supports generic protocols (see also :ref:`protocol-types`). Several +:ref:`predefined protocols ` are generic, such as +:py:class:`Iterable[T] `, and you can define additional +generic protocols. Generic protocols mostly follow the normal rules for +generic classes. Example (Python 3.12 syntax): + +.. code-block:: python + + from typing import Protocol + + class Box[T](Protocol): + content: T + + def do_stuff(one: Box[str], other: Box[bytes]) -> None: + ... + + class StringWrapper: + def __init__(self, content: str) -> None: + self.content = content + + class BytesWrapper: + def __init__(self, content: bytes) -> None: + self.content = content + + do_stuff(StringWrapper('one'), BytesWrapper(b'other')) # OK + + x: Box[float] = ... + y: Box[int] = ... + x = y # Error -- Box is invariant + +Here is the definition of ``Box`` from the above example using the legacy +syntax (Python 3.11 and earlier): + +.. code-block:: python + + from typing import Protocol, TypeVar + + T = TypeVar('T') + + class Box(Protocol[T]): + content: T + +Note that ``class ClassName(Protocol[T])`` is allowed as a shorthand for +``class ClassName(Protocol, Generic[T])`` when using the legacy syntax, +as per :pep:`PEP 544: Generic protocols <544#generic-protocols>`. +This form is only valid when using the legacy syntax. + +When using the legacy syntax, there is an important difference between +generic protocols and ordinary generic classes: mypy checks that the +declared variances of generic type variables in a protocol match how +they are used in the protocol definition. The protocol in this example +is rejected, since the type variable ``T`` is used covariantly as +a return type, but the type variable is invariant: + +.. code-block:: python + + from typing import Protocol, TypeVar + + T = TypeVar('T') + + class ReadOnlyBox(Protocol[T]): # error: Invariant type variable "T" used in protocol where covariant one is expected + def content(self) -> T: ... + +This example correctly uses a covariant type variable: + +.. code-block:: python + + from typing import Protocol, TypeVar + + T_co = TypeVar('T_co', covariant=True) + + class ReadOnlyBox(Protocol[T_co]): # OK + def content(self) -> T_co: ... + + ax: ReadOnlyBox[float] = ... + ay: ReadOnlyBox[int] = ... + ax = ay # OK -- ReadOnlyBox is covariant + +See :ref:`variance-of-generics` for more about variance. + +Generic protocols can also be recursive. Example (Python 3.12 synta): + +.. code-block:: python + + class Linked[T](Protocol): + val: T + def next(self) -> 'Linked[T]': ... + + class L: + val: int + def next(self) -> 'L': ... + + def last(seq: Linked[T]) -> T: ... + + result = last(L()) + reveal_type(result) # Revealed type is "builtins.int" + +Here is the definition of ``Linked`` using the legacy syntax +(Python 3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar + + T = TypeVar('T') + + class Linked(Protocol[T]): + val: T + def next(self) -> 'Linked[T]': ... + +.. _generic-type-aliases: + +Generic type aliases +******************** + +Type aliases can be generic. In this case they can be used in two ways. +First, subscripted aliases are equivalent to original types with substituted type +variables. Second, unsubscripted aliases are treated as original types with type +parameters replaced with ``Any``. + +The ``type`` statement introduced in Python 3.12 is used to define generic +type aliases (it also supports non-generic type aliases): + +.. code-block:: python + + from collections.abc import Callable, Iterable + + type TInt[S] = tuple[int, S] + type UInt[S] = S | int + type CBack[S] = Callable[..., S] + + def response(query: str) -> UInt[str]: # Same as str | int + ... + def activate[S](cb: CBack[S]) -> S: # Same as Callable[..., S] + ... + table_entry: TInt # Same as tuple[int, Any] + + type Vec[T: (int, float, complex)] = Iterable[tuple[T, T]] + + def inproduct[T: (int, float, complex)](v: Vec[T]) -> T: + return sum(x*y for x, y in v) + + def dilate[T: (int, float, complex)](v: Vec[T], scale: T) -> Vec[T]: + return ((x * scale, y * scale) for x, y in v) + + v1: Vec[int] = [] # Same as Iterable[tuple[int, int]] + v2: Vec = [] # Same as Iterable[tuple[Any, Any]] + v3: Vec[int, int] = [] # Error: Invalid alias, too many type arguments! + +There is also a legacy syntax that relies on ``TypeVar``. +Here the number of type arguments must match the number of free type variables +in the generic type alias definition. A type variables is free if it's not +a type parameter of a surrounding class or function. Example (following +:pep:`PEP 484: Type aliases <484#type-aliases>`, Python 3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar, Iterable, Union, Callable + + S = TypeVar('S') + + TInt = tuple[int, S] # 1 type parameter, since only S is free + UInt = Union[S, int] + CBack = Callable[..., S] + + def response(query: str) -> UInt[str]: # Same as Union[str, int] + ... + def activate(cb: CBack[S]) -> S: # Same as Callable[..., S] + ... + table_entry: TInt # Same as tuple[int, Any] + + T = TypeVar('T', int, float, complex) + + Vec = Iterable[tuple[T, T]] + + def inproduct(v: Vec[T]) -> T: + return sum(x*y for x, y in v) + + def dilate(v: Vec[T], scale: T) -> Vec[T]: + return ((x * scale, y * scale) for x, y in v) + + v1: Vec[int] = [] # Same as Iterable[tuple[int, int]] + v2: Vec = [] # Same as Iterable[tuple[Any, Any]] + v3: Vec[int, int] = [] # Error: Invalid alias, too many type arguments! + +Type aliases can be imported from modules just like other names. An +alias can also target another alias, although building complex chains +of aliases is not recommended -- this impedes code readability, thus +defeating the purpose of using aliases. Example (Python 3.12 syntax): + +.. code-block:: python + + from example1 import AliasType + from example2 import Vec + + # AliasType and Vec are type aliases (Vec as defined above) + + def fun() -> AliasType: + ... + + type OIntVec = Vec[int] | None + +Type aliases defined using the ``type`` statement are not valid as +base classes, and they can't be used to construct instances: + +.. code-block:: python + + from example1 import AliasType + from example2 import Vec + + # AliasType and Vec are type aliases (Vec as defined above) + + class NewVec[T](Vec[T]): # Error: not valid as base class + ... + + x = AliasType() # Error: can't be used to create instances + +Here are examples using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar, Generic, Optional + from example1 import AliasType + from example2 import Vec + + # AliasType and Vec are type aliases (Vec as defined above) + + def fun() -> AliasType: + ... + + OIntVec = Optional[Vec[int]] + + T = TypeVar('T') + + # Old-style type aliases can be used as base classes and you can + # construct instances using them + + class NewVec(Vec[T]): + ... + + x = AliasType() + + for i, j in NewVec[int](): + ... + +Using type variable bounds or value restriction in generic aliases has +the same effect as in generic classes and functions. + + +Differences between the new and old syntax +****************************************** + +There are a few notable differences between the new (Python 3.12 and later) +and the old syntax for generic classes, functions and type aliases, beyond +the obvious syntactic differences: + + * Type variables defined using the old syntax create definitions at runtime + in the surrounding namespace, whereas the type variables defined using the + new syntax are only defined within the class, function or type variable + that uses them. + * Type variable definitions can be shared when using the old syntax, but + the new syntax doesn't support this. + * When using the new syntax, the variance of class type variables is always + inferred. + * Type aliases defined using the new syntax can contain forward references + and recursive references without using string literal escaping. The + same is true for the bounds and constraints of type variables. + * The new syntax lets you define a generic alias where the definition doesn't + contain a reference to a type parameter. This is occasionally useful, at + least when conditionally defining type aliases. + * Type aliases defined using the new syntax can't be used as base classes + and can't be used to construct instances, unlike aliases defined using the + old syntax. + + +Generic class internals +*********************** + +You may wonder what happens at runtime when you index a generic class. +Indexing returns a *generic alias* to the original class that returns instances +of the original class on instantiation (Python 3.12 syntax): + +.. code-block:: python + + >>> class Stack[T]: ... + >>> Stack + __main__.Stack + >>> Stack[int] + __main__.Stack[int] + >>> instance = Stack[int]() + >>> instance.__class__ + __main__.Stack + +Here is the same example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + >>> from typing import TypeVar, Generic + >>> T = TypeVar('T') + >>> class Stack(Generic[T]): ... + >>> Stack + __main__.Stack + >>> Stack[int] + __main__.Stack[int] + >>> instance = Stack[int]() + >>> instance.__class__ + __main__.Stack + +Generic aliases can be instantiated or subclassed, similar to real +classes, but the above examples illustrate that type variables are +erased at runtime. Generic ``Stack`` instances are just ordinary +Python objects, and they have no extra runtime overhead or magic due +to being generic, other than the ``Generic`` base class that overloads +the indexing operator using ``__class_getitem__``. ``typing.Generic`` +is included as an implicit base class even when using the new syntax: + +.. code-block:: python + + >>> class Stack[T]: ... + >>> Stack.mro() + [, , ] + +Note that in Python 3.8 and earlier, the built-in types +:py:class:`list`, :py:class:`dict` and others do not support indexing. +This is why we have the aliases :py:class:`~typing.List`, +:py:class:`~typing.Dict` and so on in the :py:mod:`typing` +module. Indexing these aliases gives you a generic alias that +resembles generic aliases constructed by directly indexing the target +class in more recent versions of Python: + +.. code-block:: python + + >>> # Only relevant for Python 3.8 and below + >>> # If using Python 3.9 or newer, prefer the 'list[int]' syntax + >>> from typing import List + >>> List[int] + typing.List[int] + +Note that the generic aliases in ``typing`` don't support constructing +instances, unlike the corresponding built-in classes: + +.. code-block:: python + + >>> list[int]() + [] + >>> from typing import List + >>> List[int]() + Traceback (most recent call last): + ... + TypeError: Type List cannot be instantiated; use list() instead diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst new file mode 100644 index 000000000000..28a4481e502e --- /dev/null +++ b/docs/source/getting_started.rst @@ -0,0 +1,345 @@ +.. _getting-started: + +Getting started +=============== + +This chapter introduces some core concepts of mypy, including function +annotations, the :py:mod:`typing` module, stub files, and more. + +If you're looking for a quick intro, see the +:ref:`mypy cheatsheet `. + +If you're unfamiliar with the concepts of static and dynamic type checking, +be sure to read this chapter carefully, as the rest of the documentation +may not make much sense otherwise. + +Installing and running mypy +*************************** + +Mypy requires Python 3.8 or later to run. You can install mypy using pip: + +.. code-block:: shell + + $ python3 -m pip install mypy + +Once mypy is installed, run it by using the ``mypy`` tool: + +.. code-block:: shell + + $ mypy program.py + +This command makes mypy *type check* your ``program.py`` file and print +out any errors it finds. Mypy will type check your code *statically*: this +means that it will check for errors without ever running your code, just +like a linter. + +This also means that you are always free to ignore the errors mypy reports, +if you so wish. You can always use the Python interpreter to run your code, +even if mypy reports errors. + +However, if you try directly running mypy on your existing Python code, it +will most likely report little to no errors. This is a feature! It makes it +easy to adopt mypy incrementally. + +In order to get useful diagnostics from mypy, you must add *type annotations* +to your code. See the section below for details. + +.. _getting-started-dynamic-vs-static: + +Dynamic vs static typing +************************ + +A function without type annotations is considered to be *dynamically typed* by mypy: + +.. code-block:: python + + def greeting(name): + return 'Hello ' + name + +By default, mypy will **not** type check dynamically typed functions. This means +that with a few exceptions, mypy will not report any errors with regular unannotated Python. + +This is the case even if you misuse the function! + +.. code-block:: python + + def greeting(name): + return 'Hello ' + name + + # These calls will fail when the program runs, but mypy does not report an error + # because "greeting" does not have type annotations. + greeting(123) + greeting(b"Alice") + +We can get mypy to detect these kinds of bugs by adding *type annotations* (also +known as *type hints*). For example, you can tell mypy that ``greeting`` both accepts +and returns a string like so: + +.. code-block:: python + + # The "name: str" annotation says that the "name" argument should be a string + # The "-> str" annotation says that "greeting" will return a string + def greeting(name: str) -> str: + return 'Hello ' + name + +This function is now *statically typed*: mypy will use the provided type hints +to detect incorrect use of the ``greeting`` function and incorrect use of +variables within the ``greeting`` function. For example: + +.. code-block:: python + + def greeting(name: str) -> str: + return 'Hello ' + name + + greeting(3) # Argument 1 to "greeting" has incompatible type "int"; expected "str" + greeting(b'Alice') # Argument 1 to "greeting" has incompatible type "bytes"; expected "str" + greeting("World!") # No error + + def bad_greeting(name: str) -> str: + return 'Hello ' * name # Unsupported operand types for * ("str" and "str") + +Being able to pick whether you want a function to be dynamically or statically +typed can be very helpful. For example, if you are migrating an existing +Python codebase to use static types, it's usually easier to migrate by incrementally +adding type hints to your code rather than adding them all at once. Similarly, +when you are prototyping a new feature, it may be convenient to initially implement +the code using dynamic typing and only add type hints later once the code is more stable. + +Once you are finished migrating or prototyping your code, you can make mypy warn you +if you add a dynamic function by mistake by using the :option:`--disallow-untyped-defs ` +flag. You can also get mypy to provide some limited checking of dynamically typed +functions by using the :option:`--check-untyped-defs ` flag. +See :ref:`command-line` for more information on configuring mypy. + +Strict mode and configuration +***************************** + +Mypy has a *strict mode* that enables a number of additional checks, +like :option:`--disallow-untyped-defs `. + +If you run mypy with the :option:`--strict ` flag, you +will basically never get a type related error at runtime without a corresponding +mypy error, unless you explicitly circumvent mypy somehow. + +However, this flag will probably be too aggressive if you are trying +to add static types to a large, existing codebase. See :ref:`existing-code` +for suggestions on how to handle that case. + +Mypy is very configurable, so you can start with using ``--strict`` +and toggle off individual checks. For instance, if you use many third +party libraries that do not have types, +:option:`--ignore-missing-imports ` +may be useful. See :ref:`getting-to-strict` for how to build up to ``--strict``. + +See :ref:`command-line` and :ref:`config-file` for a complete reference on +configuration options. + +More complex types +****************** + +So far, we've added type hints that use only basic concrete types like +``str`` and ``float``. What if we want to express more complex types, +such as "a list of strings" or "an iterable of ints"? + +For example, to indicate that some function can accept a list of +strings, use the ``list[str]`` type (Python 3.9 and later): + +.. code-block:: python + + def greet_all(names: list[str]) -> None: + for name in names: + print('Hello ' + name) + + names = ["Alice", "Bob", "Charlie"] + ages = [10, 20, 30] + + greet_all(names) # Ok! + greet_all(ages) # Error due to incompatible types + +The :py:class:`list` type is an example of something called a *generic type*: it can +accept one or more *type parameters*. In this case, we *parameterized* :py:class:`list` +by writing ``list[str]``. This lets mypy know that ``greet_all`` accepts specifically +lists containing strings, and not lists containing ints or any other type. + +In the above examples, the type signature is perhaps a little too rigid. +After all, there's no reason why this function must accept *specifically* a list -- +it would run just fine if you were to pass in a tuple, a set, or any other custom iterable. + +You can express this idea using :py:class:`collections.abc.Iterable`: + +.. code-block:: python + + from collections.abc import Iterable # or "from typing import Iterable" + + def greet_all(names: Iterable[str]) -> None: + for name in names: + print('Hello ' + name) + +This behavior is actually a fundamental aspect of the PEP 484 type system: when +we annotate some variable with a type ``T``, we are actually telling mypy that +variable can be assigned an instance of ``T``, or an instance of a *subtype* of ``T``. +That is, ``list[str]`` is a subtype of ``Iterable[str]``. + +This also applies to inheritance, so if you have a class ``Child`` that inherits from +``Parent``, then a value of type ``Child`` can be assigned to a variable of type ``Parent``. +For example, a ``RuntimeError`` instance can be passed to a function that is annotated +as taking an ``Exception``. + +As another example, suppose you want to write a function that can accept *either* +ints or strings, but no other types. You can express this using a +union type. For example, ``int`` is a subtype of ``int | str``: + +.. code-block:: python + + def normalize_id(user_id: int | str) -> str: + if isinstance(user_id, int): + return f'user-{100_000 + user_id}' + else: + return user_id + +.. note:: + + If using Python 3.9 or earlier, use ``typing.Union[int, str]`` instead of + ``int | str``, or use ``from __future__ import annotations`` at the top of + the file (see :ref:`runtime_troubles`). + +The :py:mod:`typing` module contains many other useful types. + +For a quick overview, look through the :ref:`mypy cheatsheet `. + +For a detailed overview (including information on how to make your own +generic types or your own type aliases), look through the +:ref:`type system reference `. + +.. note:: + + When adding types, the convention is to import types + using the form ``from typing import `` (as opposed to doing + just ``import typing`` or ``import typing as t`` or ``from typing import *``). + + For brevity, we often omit imports from :py:mod:`typing` or :py:mod:`collections.abc` + in code examples, but mypy will give an error if you use types such as + :py:class:`~collections.abc.Iterable` without first importing them. + +.. note:: + + In some examples we use capitalized variants of types, such as + ``List``, and sometimes we use plain ``list``. They are equivalent, + but the prior variant is needed if you are using Python 3.8 or earlier. + +Local type inference +******************** + +Once you have added type hints to a function (i.e. made it statically typed), +mypy will automatically type check that function's body. While doing so, +mypy will try and *infer* as many details as possible. + +We saw an example of this in the ``normalize_id`` function above -- mypy understands +basic :py:func:`isinstance ` checks and so can infer that the ``user_id`` variable was of +type ``int`` in the if-branch and of type ``str`` in the else-branch. + +As another example, consider the following function. Mypy can type check this function +without a problem: it will use the available context and deduce that ``output`` must be +of type ``list[float]`` and that ``num`` must be of type ``float``: + +.. code-block:: python + + def nums_below(numbers: Iterable[float], limit: float) -> list[float]: + output = [] + for num in numbers: + if num < limit: + output.append(num) + return output + +For more details, see :ref:`type-inference-and-annotations`. + +Types from libraries +******************** + +Mypy can also understand how to work with types from libraries that you use. + +For instance, mypy comes out of the box with an intimate knowledge of the +Python standard library. For example, here is a function which uses the +``Path`` object from the :doc:`pathlib standard library module `: + +.. code-block:: python + + from pathlib import Path + + def load_template(template_path: Path, name: str) -> str: + # Mypy knows that `template_path` has a `read_text` method that returns a str + template = template_path.read_text() + # ...so it understands this line type checks + return template.replace('USERNAME', name) + +If a third party library you use :ref:`declares support for type checking `, +mypy will type check your use of that library based on the type hints +it contains. + +However, if the third party library does not have type hints, mypy will +complain about missing type information. + +.. code-block:: text + + prog.py:1: error: Library stubs not installed for "yaml" + prog.py:1: note: Hint: "python3 -m pip install types-PyYAML" + prog.py:2: error: Library stubs not installed for "requests" + prog.py:2: note: Hint: "python3 -m pip install types-requests" + ... + +In this case, you can provide mypy a different source of type information, +by installing a *stub* package. A stub package is a package that contains +type hints for another library, but no actual code. + +.. code-block:: shell + + $ python3 -m pip install types-PyYAML types-requests + +Stubs packages for a distribution are often named ``types-``. +Note that a distribution name may be different from the name of the package that +you import. For example, ``types-PyYAML`` contains stubs for the ``yaml`` +package. + +For more discussion on strategies for handling errors about libraries without +type information, refer to :ref:`fix-missing-imports`. + +For more information about stubs, see :ref:`stub-files`. + +Next steps +********** + +If you are in a hurry and don't want to read lots of documentation +before getting started, here are some pointers to quick learning +resources: + +* Read the :ref:`mypy cheatsheet `. + +* Read :ref:`existing-code` if you have a significant existing + codebase without many type annotations. + +* Read the `blog post `_ + about the Zulip project's experiences with adopting mypy. + +* If you prefer watching talks instead of reading, here are + some ideas: + + * Carl Meyer: + `Type Checked Python in the Real World `_ + (PyCon 2018) + + * Greg Price: + `Clearer Code at Scale: Static Types at Zulip and Dropbox `_ + (PyCon 2018) + +* Look at :ref:`solutions to common issues ` with mypy if + you encounter problems. + +* You can ask questions about mypy in the + `mypy issue tracker `_ and + typing `Gitter chat `_. + +* For general questions about Python typing, try posting at + `typing discussions `_. + +You can also continue reading this document and skip sections that +aren't relevant for you. You don't need to read sections in order. diff --git a/docs/source/html_builder.py b/docs/source/html_builder.py new file mode 100644 index 000000000000..ea3594e0617b --- /dev/null +++ b/docs/source/html_builder.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +import os +import textwrap +from pathlib import Path +from typing import Any + +from sphinx.addnodes import document +from sphinx.application import Sphinx +from sphinx.builders.html import StandaloneHTMLBuilder +from sphinx.environment import BuildEnvironment + + +class MypyHTMLBuilder(StandaloneHTMLBuilder): + def __init__(self, app: Sphinx, env: BuildEnvironment) -> None: + super().__init__(app, env) + self._ref_to_doc = {} + + def write_doc(self, docname: str, doctree: document) -> None: + super().write_doc(docname, doctree) + self._ref_to_doc.update({_id: docname for _id in doctree.ids}) + + def _verify_error_codes(self) -> None: + from mypy.errorcodes import error_codes + + missing_error_codes = {c for c in error_codes if f"code-{c}" not in self._ref_to_doc} + if missing_error_codes: + raise ValueError( + f"Some error codes are not documented: {', '.join(sorted(missing_error_codes))}" + ) + + def _write_ref_redirector(self) -> None: + if os.getenv("VERIFY_MYPY_ERROR_CODES"): + self._verify_error_codes() + p = Path(self.outdir) / "_refs.html" + data = f""" + + + + + + """ + p.write_text(textwrap.dedent(data)) + + def finish(self) -> None: + super().finish() + self._write_ref_redirector() + + +def setup(app: Sphinx) -> dict[str, Any]: + app.add_builder(MypyHTMLBuilder, override=True) + + return {"version": "0.1", "parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/source/index.rst b/docs/source/index.rst index 342dc7daa970..de3286d58ace 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,29 +3,114 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to Mypy documentation! +Welcome to mypy documentation! ============================== +Mypy is a static type checker for Python. + +Type checkers help ensure that you're using variables and functions in your code +correctly. With mypy, add type hints (:pep:`484`) +to your Python programs, and mypy will warn you when you use those types +incorrectly. + +Python is a dynamic language, so usually you'll only see errors in your code +when you attempt to run it. Mypy is a *static* checker, so it finds bugs +in your programs without even running them! + +Here is a small example to whet your appetite: + +.. code-block:: python + + number = input("What is your favourite number?") + print("It is", number + 1) # error: Unsupported operand types for + ("str" and "int") + +Adding type hints for mypy does not interfere with the way your program would +otherwise run. Think of type hints as similar to comments! You can always use +the Python interpreter to run your code, even if mypy reports errors. + +Mypy is designed with gradual typing in mind. This means you can add type +hints to your code base slowly and that you can always fall back to dynamic +typing when static typing is not convenient. + +Mypy has a powerful and easy-to-use type system, supporting features such as +type inference, generics, callable types, tuple types, union types, +structural subtyping and more. Using mypy will make your programs easier to +understand, debug, and maintain. + +.. note:: + + Although mypy is production ready, there may be occasional changes + that break backward compatibility. The mypy development team tries to + minimize the impact of changes to user code. In case of a major breaking + change, mypy's major version will be bumped. + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + :caption: First steps + + getting_started + cheat_sheet_py3 + existing_code + +.. _overview-type-system-reference: + .. toctree:: :maxdepth: 2 + :caption: Type system reference - introduction - basics builtin_types type_inference_and_annotations kinds_of_types class_basics + runtime_troubles + protocols dynamic_typing - function_overloading - casts + type_narrowing duck_type_compatibility - common_issues + stubs generics + more_types + literal_types + typed_dict + final_attrs + metaclasses + +.. toctree:: + :maxdepth: 2 + :caption: Configuring and running mypy + + running_mypy + command_line + config_file + inline_config + mypy_daemon + installed_packages + extending_mypy + stubgen + stubtest + +.. toctree:: + :maxdepth: 2 + :caption: Miscellaneous + + common_issues supported_python_features + error_codes + error_code_list + error_code_list2 additional_features - planned_features faq - revision_history + changelog + +.. toctree:: + :hidden: + :caption: Project Links + + GitHub + Website Indices and tables ================== diff --git a/docs/source/inline_config.rst b/docs/source/inline_config.rst new file mode 100644 index 000000000000..a587983cc74a --- /dev/null +++ b/docs/source/inline_config.rst @@ -0,0 +1,43 @@ +.. _inline-config: + +Inline configuration +==================== + +Mypy supports setting per-file configuration options inside files themselves +using ``# mypy:`` comments. For example: + +.. code-block:: python + + # mypy: disallow-any-generics + +Inline configuration comments take precedence over all other +configuration mechanisms. + +Configuration comment format +**************************** + +Flags correspond to :ref:`config file flags ` but allow +hyphens to be substituted for underscores. + +Values are specified using ``=``, but ``= True`` may be omitted: + +.. code-block:: python + + # mypy: disallow-any-generics + # mypy: always-true=FOO + +Multiple flags can be separated by commas or placed on separate +lines. To include a comma as part of an option's value, place the +value inside quotes: + +.. code-block:: python + + # mypy: disallow-untyped-defs, always-false="FOO,BAR" + +Like in the configuration file, options that take a boolean value may be +inverted by adding ``no-`` to their name or by (when applicable) +swapping their prefix from ``disallow`` to ``allow`` (and vice versa): + +.. code-block:: python + + # mypy: allow-untyped-defs, no-strict-optional diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst new file mode 100644 index 000000000000..fa4fae1a0b3e --- /dev/null +++ b/docs/source/installed_packages.rst @@ -0,0 +1,196 @@ +.. _installed-packages: + +Using installed packages +======================== + +Packages installed with pip can declare that they support type +checking. For example, the `aiohttp +`_ package has built-in support +for type checking. + +Packages can also provide stubs for a library. For example, +``types-requests`` is a stub-only package that provides stubs for the +`requests `_ package. +Stub packages are usually published from `typeshed +`_, a shared repository for Python +library stubs, and have a name of form ``types-``. Note that +many stub packages are not maintained by the original maintainers of +the package. + +The sections below explain how mypy can use these packages, and how +you can create such packages. + +.. note:: + + :pep:`561` specifies how a package can declare that it supports + type checking. + +.. note:: + + New versions of stub packages often use type system features not + supported by older, and even fairly recent mypy versions. If you + pin to an older version of mypy (using ``requirements.txt``, for + example), it is recommended that you also pin the versions of all + your stub package dependencies. + +.. note:: + + Starting in mypy 0.900, most third-party package stubs must be + installed explicitly. This decouples mypy and stub versioning, + allowing stubs to updated without updating mypy. This also allows + stubs not originally included with mypy to be installed. Earlier + mypy versions included a fixed set of stubs for third-party + packages. + +Using installed packages with mypy (PEP 561) +******************************************** + +Typically mypy will automatically find and use installed packages that +support type checking or provide stubs. This requires that you install +the packages in the Python environment that you use to run mypy. As +many packages don't support type checking yet, you may also have to +install a separate stub package, usually named +``types-``. (See :ref:`fix-missing-imports` for how to deal +with libraries that don't support type checking and are also missing +stubs.) + +If you have installed typed packages in another Python installation or +environment, mypy won't automatically find them. One option is to +install another copy of those packages in the environment in which you +installed mypy. Alternatively, you can use the +:option:`--python-executable ` flag to point +to the Python executable for another environment, and mypy will find +packages installed for that Python executable. + +Note that mypy does not support some more advanced import features, +such as zip imports and custom import hooks. + +If you don't want to use installed packages that provide type +information at all, use the :option:`--no-site-packages ` flag to disable searching for installed packages. + +Note that stub-only packages cannot be used with ``MYPYPATH``. If you +want mypy to find the package, it must be installed. For a package +``foo``, the name of the stub-only package (``foo-stubs``) is not a +legal package name, so mypy will not find it, unless it is installed +(see :pep:`PEP 561: Stub-only Packages <561#stub-only-packages>` for +more information). + +Creating PEP 561 compatible packages +************************************ + +.. note:: + + You can generally ignore this section unless you maintain a package on + PyPI, or want to publish type information for an existing PyPI + package. + +:pep:`561` describes three main ways to distribute type +information: + +1. A package has inline type annotations in the Python implementation. + +2. A package ships :ref:`stub files ` with type + information alongside the Python implementation. + +3. A package ships type information for another package separately as + stub files (also known as a "stub-only package"). + +If you want to create a stub-only package for an existing library, the +simplest way is to contribute stubs to the `typeshed +`_ repository, and a stub package +will automatically be uploaded to PyPI. + +If you would like to publish a library package to a package repository +yourself (e.g. on PyPI) for either internal or external use in type +checking, packages that supply type information via type comments or +annotations in the code should put a ``py.typed`` file in their +package directory. For example, here is a typical directory structure: + +.. code-block:: text + + setup.py + package_a/ + __init__.py + lib.py + py.typed + +The ``setup.py`` file could look like this: + +.. code-block:: python + + from setuptools import setup + + setup( + name="SuperPackageA", + author="Me", + version="0.1", + package_data={"package_a": ["py.typed"]}, + packages=["package_a"] + ) + +Some packages have a mix of stub files and runtime files. These packages also +require a ``py.typed`` file. An example can be seen below: + +.. code-block:: text + + setup.py + package_b/ + __init__.py + lib.py + lib.pyi + py.typed + +The ``setup.py`` file might look like this: + +.. code-block:: python + + from setuptools import setup + + setup( + name="SuperPackageB", + author="Me", + version="0.1", + package_data={"package_b": ["py.typed", "lib.pyi"]}, + packages=["package_b"] + ) + +In this example, both ``lib.py`` and the ``lib.pyi`` stub file exist. At +runtime, the Python interpreter will use ``lib.py``, but mypy will use +``lib.pyi`` instead. + +If the package is stub-only (not imported at runtime), the package should have +a prefix of the runtime package name and a suffix of ``-stubs``. +A ``py.typed`` file is not needed for stub-only packages. For example, if we +had stubs for ``package_c``, we might do the following: + +.. code-block:: text + + setup.py + package_c-stubs/ + __init__.pyi + lib.pyi + +The ``setup.py`` might look like this: + +.. code-block:: python + + from setuptools import setup + + setup( + name="SuperPackageC", + author="Me", + version="0.1", + package_data={"package_c-stubs": ["__init__.pyi", "lib.pyi"]}, + packages=["package_c-stubs"] + ) + +The instructions above are enough to ensure that the built wheels +contain the appropriate files. However, to ensure inclusion inside the +``sdist`` (``.tar.gz`` archive), you may also need to modify the +inclusion rules in your ``MANIFEST.in``: + +.. code-block:: text + + global-include *.pyi + global-include *.typed diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst deleted file mode 100644 index 1b4f7132a058..000000000000 --- a/docs/source/introduction.rst +++ /dev/null @@ -1,22 +0,0 @@ -Introduction -============ - -Mypy is a static type checker for Python. If you sprinkle your code -with type annotations using the Python 3 function annotation syntax, -mypy can type check you code and find common bugs. Mypy is a static -analyzer, or a lint-like tool: type annotations are just hints and are -not enforced when running you program. You run your program with a -standard Python interpreter, and the annotations are treated basically -as comments. - -Mypy has a powerful but easy-to-use type system with modern features -such as type inference, generics, function types, tuple types and -union types. You can also always escape to dynamic typing -- mypy's -take on static typing doesn't really restrict what you can do in your -programs, but it will make your programs easier to debug, maintain and -understand. - -This document is a short introduction to mypy. It will get you started -writing statically typed code. Knowledge of Python 3.x and some kind -of a statically typed object-oriented language such as Java are -assumed. diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 692055e5e5ef..54693cddf953 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -1,17 +1,23 @@ Kinds of types ============== -User-defined types -****************** +We've mostly restricted ourselves to built-in types until now. This +section introduces several additional kinds of types. You are likely +to need at least some of them to type check any non-trivial programs. -Each class is also a type. Any instance of a subclass is also -compatible with all superclasses. All values are compatible with the -``object`` type (and also the ``Any`` type). +Class types +*********** + +Every class is also a valid type. Any instance of a subclass is also +compatible with all superclasses -- it follows that every value is compatible +with the :py:class:`object` type (and incidentally also the ``Any`` type, discussed +below). Mypy analyzes the bodies of classes to determine which methods and +attributes are available in instances. This example uses subclassing: .. code-block:: python class A: - def f(self) -> int: # Type of self inferred (A) + def f(self) -> int: # Type of self inferred (A) return 2 class B(A): @@ -20,55 +26,138 @@ compatible with all superclasses. All values are compatible with the def g(self) -> int: return 4 - a = B() # type: A # OK (explicit type for a; override type inference) - print(a.f()) # 3 - a.g() # Type check error: A has no method g + def foo(a: A) -> None: + print(a.f()) # 3 + a.g() # Error: "A" has no attribute "g" + + foo(B()) # OK (B is a subclass of A) The Any type ************ -A value with the ``Any`` type is dynamically typed. Any operations are -permitted on the value, and the operations are checked at runtime, -similar to normal Python code. If you do not define a function return -value or argument types, these default to ``Any``. Also, a function -without an explicit return type is dynamically typed. The body of a -dynamically typed function is not checked statically. +A value with the ``Any`` type is dynamically typed. Mypy doesn't know +anything about the possible runtime types of such value. Any +operations are permitted on the value, and the operations are only checked +at runtime. You can use ``Any`` as an "escape hatch" when you can't use +a more precise type for some reason. + +``Any`` is compatible with every other type, and vice versa. You can freely +assign a value of type ``Any`` to a variable with a more precise type: + +.. code-block:: python + + a: Any = None + s: str = '' + a = 2 # OK (assign "int" to "Any") + s = a # OK (assign "Any" to "str") + +Declared (and inferred) types are ignored (or *erased*) at runtime. They are +basically treated as comments, and thus the above code does not +generate a runtime error, even though ``s`` gets an ``int`` value when +the program is run, while the declared type of ``s`` is actually +``str``! You need to be careful with ``Any`` types, since they let you +lie to mypy, and this could easily hide bugs. -``Any`` is compatible with every other type, and vice versa. No -implicit type check is inserted when assigning a value of type ``Any`` -to a variable with a more precise type: +If you do not define a function return value or argument types, these +default to ``Any``: .. code-block:: python - a, s = Undefined(Any), Undefined(str) - a = 2 # OK - s = a # OK + def show_heading(s) -> None: + print('=== ' + s + ' ===') # No static type checking, as s has type Any + + show_heading(1) # OK (runtime error only; mypy won't generate an error) + +You should give a statically typed function an explicit ``None`` +return type even if it doesn't return a value, as this lets mypy catch +additional type errors: + +.. code-block:: python + + def wait(t: float): # Implicit Any return value + print('Waiting...') + time.sleep(t) + + if wait(2) > 1: # Mypy doesn't catch this error! + ... -Declared (and inferred) types are erased at runtime (they are -basically treated as comments), and thus the above code does not -generate a runtime error. +If we had used an explicit ``None`` return type, mypy would have caught +the error: + +.. code-block:: python + + def wait(t: float) -> None: + print('Waiting...') + time.sleep(t) + + if wait(2) > 1: # Error: can't compare None and int + ... + +The ``Any`` type is discussed in more detail in section :ref:`dynamic-typing`. + +.. note:: + + A function without any types in the signature is dynamically + typed. The body of a dynamically typed function is not checked + statically, and local variables have implicit ``Any`` types. + This makes it easier to migrate legacy Python code to mypy, as + mypy won't complain about dynamically typed functions. + +.. _tuple-types: Tuple types *********** -The type ``Tuple[T1, ..., Tn]`` represents a tuple with the item types ``T1``, |...|, ``Tn``: +The type ``tuple[T1, ..., Tn]`` represents a tuple with the item types ``T1``, ..., ``Tn``: .. code-block:: python - def f(t: Tuple[int, str]) -> None: + # Use `typing.Tuple` in Python 3.8 and earlier + def f(t: tuple[int, str]) -> None: t = 1, 'foo' # OK t = 'foo', 1 # Type check error +A tuple type of this kind has exactly a specific number of items (2 in +the above example). Tuples can also be used as immutable, +varying-length sequences. You can use the type ``tuple[T, ...]`` (with +a literal ``...`` -- it's part of the syntax) for this +purpose. Example: + +.. code-block:: python + + def print_squared(t: tuple[int, ...]) -> None: + for n in t: + print(n, n ** 2) + + print_squared(()) # OK + print_squared((1, 3, 5)) # OK + print_squared([1, 2]) # Error: only a tuple is valid + +.. note:: + + Usually it's a better idea to use ``Sequence[T]`` instead of ``tuple[T, ...]``, as + :py:class:`~collections.abc.Sequence` is also compatible with lists and other non-tuple sequences. + +.. note:: + + ``tuple[...]`` is valid as a base class in Python 3.6 and later, and + always in stub files. In earlier Python versions you can sometimes work around this + limitation by using a named tuple as a base class (see section :ref:`named-tuples`). + +.. _callable-types: + Callable types (and lambdas) **************************** You can pass around function objects and bound methods in statically -typed code. The type of a function that accepts arguments ``A1``, |...|, ``An`` -and returns ``Rt`` is ``Function[[A1, ..., An], Rt]``. Example: +typed code. The type of a function that accepts arguments ``A1``, ..., ``An`` +and returns ``Rt`` is ``Callable[[A1, ..., An], Rt]``. Example: .. code-block:: python - def twice(i: int, next: Function[[int], int]) -> int: + from collections.abc import Callable + + def twice(i: int, next: Callable[[int], int]) -> int: return next(next(i)) def add(i: int) -> int: @@ -76,36 +165,88 @@ and returns ``Rt`` is ``Function[[A1, ..., An], Rt]``. Example: print(twice(3, add)) # 5 +.. note:: + + Import :py:data:`Callable[...] ` from ``typing`` instead + of ``collections.abc`` if you use Python 3.8 or earlier. + +You can only have positional arguments, and only ones without default +values, in callable types. These cover the vast majority of uses of +callable types, but sometimes this isn't quite enough. Mypy recognizes +a special form ``Callable[..., T]`` (with a literal ``...``) which can +be used in less typical cases. It is compatible with arbitrary +callable objects that return a type compatible with ``T``, independent +of the number, types or kinds of arguments. Mypy lets you call such +callable values with arbitrary arguments, without any checking -- in +this respect they are treated similar to a ``(*args: Any, **kwargs: +Any)`` function signature. Example: + +.. code-block:: python + + from collections.abc import Callable + + def arbitrary_call(f: Callable[..., int]) -> int: + return f('x') + f(y=2) # OK + + arbitrary_call(ord) # No static error, but fails at runtime + arbitrary_call(open) # Error: does not return an int + arbitrary_call(1) # Error: 'int' is not callable + +In situations where more precise or complex types of callbacks are +necessary one can use flexible :ref:`callback protocols `. Lambdas are also supported. The lambda argument and return value types cannot be given explicitly; they are always inferred based on context using bidirectional type inference: .. code-block:: python - l = map(lambda x: x + 1, [1, 2, 3]) # infer x as int and l as List[int] + l = map(lambda x: x + 1, [1, 2, 3]) # Infer x as int and l as list[int] If you want to give the argument or return value types explicitly, use an ordinary, perhaps nested function definition. +Callables can also be used against type objects, matching their +``__init__`` or ``__new__`` signature: + +.. code-block:: python + + from collections.abc import Callable + + class C: + def __init__(self, app: str) -> None: + pass + + CallableType = Callable[[str], C] + + def class_or_callable(arg: CallableType) -> None: + inst = arg("my_app") + reveal_type(inst) # Revealed type is "C" + +This is useful if you want ``arg`` to be either a ``Callable`` returning an +instance of ``C`` or the type of ``C`` itself. This also works with +:ref:`callback protocols `. + + .. _union-types: +.. _alternative_union_syntax: Union types *********** Python functions often accept values of two or more different -types. You can use overloading to model this in statically typed code, -but union types can make code like this easier to write. +types. You can use :ref:`overloading ` to +represent this, but union types are often more convenient. -Use the ``Union[T1, ..., Tn]`` type constructor to construct a union -type. For example, the type ``Union[int, str]`` is compatible with -both integers and strings. You can use an ``isinstance()`` check to -narrow down the type to a specific type: +Use ``T1 | ... | Tn`` to construct a union +type. For example, if an argument has type ``int | str``, both +integers and strings are valid argument values. -.. code-block:: python +You can use an :py:func:`isinstance` check to narrow down a union type to a +more specific type: - from typing import Union +.. code-block:: python - def f(x: Union[int, str]) -> None: + def f(x: int | str) -> None: x + 1 # Error: str + int is not valid if isinstance(x, int): # Here type of x is int. @@ -118,43 +259,476 @@ narrow down the type to a specific type: f('x') # OK f(1.1) # Error -Class name forward references -***************************** +.. note:: + + Operations are valid for union types only if they are valid for *every* + union item. This is why it's often necessary to use an :py:func:`isinstance` + check to first narrow down a union type to a non-union type. This also + means that it's recommended to avoid union types as function return types, + since the caller may have to use :py:func:`isinstance` before doing anything + interesting with the value. -Python does not allow references to a class object before the class is -defined. Thus this code is does not work as expected: +Python 3.9 and older only partially support this syntax. Instead, you can +use the legacy ``Union[T1, ..., Tn]`` type constructor. Example: .. code-block:: python - def f(x: A) -> None: # Error: Name A not defined - .... + from typing import Union - class A: + def f(x: Union[int, str]) -> None: ... -In cases like these you can enter the type as a string literal — this -is a *forward reference*: +It is also possible to use the new syntax with versions of Python where it +isn't supported by the runtime with some limitations, if you use +``from __future__ import annotations`` (see :ref:`runtime_troubles`): .. code-block:: python - def f(x: 'A') -> None: # OK + from __future__ import annotations + + def f(x: int | str) -> None: # OK on Python 3.7 and later ... - class A: +.. _strict_optional: + +Optional types and the None type +******************************** + +You can use ``T | None`` to define a type variant that allows ``None`` values, +such as ``int | None``. This is called an *optional type*: + +.. code-block:: python + + def strlen(s: str) -> int | None: + if not s: + return None # OK + return len(s) + + def strlen_invalid(s: str) -> int: + if not s: + return None # Error: None not compatible with int + return len(s) + +To support Python 3.9 and earlier, you can use the :py:data:`~typing.Optional` +type modifier instead, such as ``Optional[int]`` (``Optional[X]`` is +the preferred shorthand for ``Union[X, None]``): + +.. code-block:: python + + from typing import Optional + + def strlen(s: str) -> Optional[int]: ... -Of course, instead of using a string literal type, you could move the -function definition after the class definition. This is not always -desirable or even possible, though. +Most operations will not be allowed on unguarded ``None`` or *optional* values +(values with an optional type): + +.. code-block:: python + + def my_inc(x: int | None) -> int: + return x + 1 # Error: Cannot add None and int + +Instead, an explicit ``None`` check is required. Mypy has +powerful type inference that lets you use regular Python +idioms to guard against ``None`` values. For example, mypy +recognizes ``is None`` checks: + +.. code-block:: python + + def my_inc(x: int | None) -> int: + if x is None: + return 0 + else: + # The inferred type of x is just int here. + return x + 1 + +Mypy will infer the type of ``x`` to be ``int`` in the else block due to the +check against ``None`` in the if condition. + +Other supported checks for guarding against a ``None`` value include +``if x is not None``, ``if x`` and ``if not x``. Additionally, mypy understands +``None`` checks within logical expressions: + +.. code-block:: python + + def concat(x: str | None, y: str | None) -> str | None: + if x is not None and y is not None: + # Both x and y are not None here + return x + y + else: + return None + +Sometimes mypy doesn't realize that a value is never ``None``. This notably +happens when a class instance can exist in a partially defined state, +where some attribute is initialized to ``None`` during object +construction, but a method assumes that the attribute is no longer ``None``. Mypy +will complain about the possible ``None`` value. You can use +``assert x is not None`` to work around this in the method: + +.. code-block:: python + + class Resource: + path: str | None = None + + def initialize(self, path: str) -> None: + self.path = path + + def read(self) -> str: + # We require that the object has been initialized. + assert self.path is not None + with open(self.path) as f: # OK + return f.read() + + r = Resource() + r.initialize('/foo/bar') + r.read() + +When initializing a variable as ``None``, ``None`` is usually an +empty place-holder value, and the actual value has a different type. +This is why you need to annotate an attribute in cases like the class +``Resource`` above: + +.. code-block:: python + + class Resource: + path: str | None = None + ... + +This also works for attributes defined within methods: -Any type can be entered as a string literal, and youn can combine -string-literal types with non-string-literal types freely: +.. code-block:: python + + class Counter: + def __init__(self) -> None: + self.count: int | None = None + +Often it's easier to not use any initial value for an attribute. +This way you don't need to use an optional type and can avoid ``assert ... is not None`` +checks. No initial value is needed if you annotate an attribute in the class body: + +.. code-block:: python + + class Container: + items: list[str] # No initial value + +Mypy generally uses the first assignment to a variable to +infer the type of the variable. However, if you assign both a ``None`` +value and a non-``None`` value in the same scope, mypy can usually do +the right thing without an annotation: .. code-block:: python - a = Undefined(List['A']) # OK - n = Undefined('int') # OK, though not useful + def f(i: int) -> None: + n = None # Inferred type 'int | None' because of the assignment below + if i > 0: + n = i + ... + +Sometimes you may get the error "Cannot determine type of ". In this +case you should add an explicit ``... | None`` annotation. + +.. note:: + + ``None`` is a type with only one value, ``None``. ``None`` is also used + as the return type for functions that don't return a value, i.e. functions + that implicitly return ``None``. + +.. note:: + + The Python interpreter internally uses the name ``NoneType`` for + the type of ``None``, but ``None`` is always used in type + annotations. The latter is shorter and reads better. (``NoneType`` + is available as :py:data:`types.NoneType` on Python 3.10+, but is + not exposed at all on earlier versions of Python.) + +.. note:: + + The type ``Optional[T]`` *does not* mean a function parameter with a default value. + It simply means that ``None`` is a valid argument value. This is + a common confusion because ``None`` is a common default value for parameters, + and parameters with default values are sometimes called *optional* parameters + (or arguments). + +.. _type-aliases: + +Type aliases +************ + +In certain situations, type names may end up being long and painful to type, +especially if they are used frequently: + +.. code-block:: python + + def f() -> list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]]: + ... + +When cases like this arise, you can define a type alias by simply +assigning the type to a variable (this is an *implicit type alias*): + +.. code-block:: python + + AliasType = list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]] + + # Now we can use AliasType in place of the full name: + + def f() -> AliasType: + ... + +.. note:: + + A type alias does not create a new type. It's just a shorthand notation for + another type -- it's equivalent to the target type except for + :ref:`generic aliases `. + +Python 3.12 introduced the ``type`` statement for defining *explicit type aliases*. +Explicit type aliases are unambiguous and can also improve readability by +making the intent clear: + +.. code-block:: python + + type AliasType = list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]] + + # Now we can use AliasType in place of the full name: + + def f() -> AliasType: + ... + +There can be confusion about exactly when an assignment defines an implicit type alias -- +for example, when the alias contains forward references, invalid types, or violates some other +restrictions on type alias declarations. Because the +distinction between an unannotated variable and a type alias is implicit, +ambiguous or incorrect type alias declarations default to defining +a normal variable instead of a type alias. + +Aliases defined using the ``type`` statement have these properties, which +distinguish them from implicit type aliases: + +* The definition may contain forward references without having to use string + literal escaping, since it is evaluated lazily. +* The alias can be used in type annotations, type arguments, and casts, but + it can't be used in contexts which require a class object. For example, it's + not valid as a base class and it can't be used to construct instances. + +There is also use an older syntax for defining explicit type aliases, which was +introduced in Python 3.10 (:pep:`613`): + +.. code-block:: python + + from typing import TypeAlias # "from typing_extensions" in Python 3.9 and earlier + + AliasType: TypeAlias = list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]] + +.. _named-tuples: + +Named tuples +************ + +Mypy recognizes named tuples and can type check code that defines or +uses them. In this example, we can detect code trying to access a +missing attribute: + +.. code-block:: python + + Point = namedtuple('Point', ['x', 'y']) + p = Point(x=1, y=2) + print(p.z) # Error: Point has no attribute 'z' + +If you use :py:func:`namedtuple ` to define your named tuple, all the items +are assumed to have ``Any`` types. That is, mypy doesn't know anything +about item types. You can use :py:class:`~typing.NamedTuple` to also define +item types: + +.. code-block:: python + + from typing import NamedTuple + + Point = NamedTuple('Point', [('x', int), + ('y', int)]) + p = Point(x=1, y='x') # Argument has incompatible type "str"; expected "int" + +Python 3.6 introduced an alternative, class-based syntax for named tuples with types: + +.. code-block:: python + + from typing import NamedTuple + + class Point(NamedTuple): + x: int + y: int + + p = Point(x=1, y='x') # Argument has incompatible type "str"; expected "int" + +.. note:: + + You can use the raw ``NamedTuple`` "pseudo-class" in type annotations + if any ``NamedTuple`` object is valid. + + For example, it can be useful for deserialization: + + .. code-block:: python + + def deserialize_named_tuple(arg: NamedTuple) -> Dict[str, Any]: + return arg._asdict() + + Point = namedtuple('Point', ['x', 'y']) + Person = NamedTuple('Person', [('name', str), ('age', int)]) + + deserialize_named_tuple(Point(x=1, y=2)) # ok + deserialize_named_tuple(Person(name='Nikita', age=18)) # ok + + # Error: Argument 1 to "deserialize_named_tuple" has incompatible type + # "Tuple[int, int]"; expected "NamedTuple" + deserialize_named_tuple((1, 2)) + + Note that this behavior is highly experimental, non-standard, + and may not be supported by other type checkers and IDEs. + +.. _type-of-class: + +The type of class objects +************************* + +(Freely after :pep:`PEP 484: The type of class objects +<484#the-type-of-class-objects>`.) + +Sometimes you want to talk about class objects that inherit from a +given class. This can be spelled as ``type[C]`` (or, on Python 3.8 and lower, +:py:class:`typing.Type[C] `) where ``C`` is a +class. In other words, when ``C`` is the name of a class, using ``C`` +to annotate an argument declares that the argument is an instance of +``C`` (or of a subclass of ``C``), but using ``type[C]`` as an +argument annotation declares that the argument is a class object +deriving from ``C`` (or ``C`` itself). + +For example, assume the following classes: + +.. code-block:: python + + class User: + # Defines fields like name, email + + class BasicUser(User): + def upgrade(self): + """Upgrade to Pro""" + + class ProUser(User): + def pay(self): + """Pay bill""" + +Note that ``ProUser`` doesn't inherit from ``BasicUser``. + +Here's a function that creates an instance of one of these classes if +you pass it the right class object: + +.. code-block:: python + + def new_user(user_class): + user = user_class() + # (Here we could write the user object to a database) + return user + +How would we annotate this function? Without the ability to parameterize ``type``, the best we +could do would be: + +.. code-block:: python + + def new_user(user_class: type) -> User: + # Same implementation as before + +This seems reasonable, except that in the following example, mypy +doesn't see that the ``buyer`` variable has type ``ProUser``: + +.. code-block:: python + + buyer = new_user(ProUser) + buyer.pay() # Rejected, not a method on User + +However, using the ``type[C]`` syntax and a type variable with an upper bound (see +:ref:`type-variable-upper-bound`) we can do better (Python 3.12 syntax): + +.. code-block:: python + + def new_user[U: User](user_class: type[U]) -> U: + # Same implementation as before + +Here is the example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + U = TypeVar('U', bound=User) + + def new_user(user_class: type[U]) -> U: + # Same implementation as before + +Now mypy will infer the correct type of the result when we call +``new_user()`` with a specific subclass of ``User``: + +.. code-block:: python + + beginner = new_user(BasicUser) # Inferred type is BasicUser + beginner.upgrade() # OK + +.. note:: + + The value corresponding to ``type[C]`` must be an actual class + object that's a subtype of ``C``. Its constructor must be + compatible with the constructor of ``C``. If ``C`` is a type + variable, its upper bound must be a class object. + +For more details about ``type[]`` and :py:class:`typing.Type[] `, see :pep:`PEP 484: The type of +class objects <484#the-type-of-class-objects>`. + +.. _generators: + +Generators +********** + +A basic generator that only yields values can be succinctly annotated as having a return +type of either :py:class:`Iterator[YieldType] ` or :py:class:`Iterable[YieldType] `. For example: + +.. code-block:: python + + def squares(n: int) -> Iterator[int]: + for i in range(n): + yield i * i + +A good rule of thumb is to annotate functions with the most specific return +type possible. However, you should also take care to avoid leaking implementation +details into a function's public API. In keeping with these two principles, prefer +:py:class:`Iterator[YieldType] ` over +:py:class:`Iterable[YieldType] ` as the return-type annotation for a +generator function, as it lets mypy know that users are able to call :py:func:`next` on +the object returned by the function. Nonetheless, bear in mind that ``Iterable`` may +sometimes be the better option, if you consider it an implementation detail that +``next()`` can be called on the object returned by your function. + +If you want your generator to accept values via the :py:meth:`~generator.send` method or return +a value, on the other hand, you should use the +:py:class:`Generator[YieldType, SendType, ReturnType] ` generic type instead of +either ``Iterator`` or ``Iterable``. For example: + +.. code-block:: python + + def echo_round() -> Generator[int, float, str]: + sent = yield 0 + while sent >= 0: + sent = yield round(sent) + return 'Done' + +Note that unlike many other generics in the typing module, the ``SendType`` of +:py:class:`~typing.Generator` behaves contravariantly, not covariantly or invariantly. + +If you do not plan on receiving or returning values, then set the ``SendType`` +or ``ReturnType`` to ``None``, as appropriate. For example, we could have +annotated the first example as the following: + +.. code-block:: python - class A: pass + def squares(n: int) -> Generator[int, None, None]: + for i in range(n): + yield i * i -String literal types are never needed in ``# type:`` comments. +This is slightly different from using ``Iterator[int]`` or ``Iterable[int]``, +since generators have :py:meth:`~generator.close`, :py:meth:`~generator.send`, and :py:meth:`~generator.throw` methods that +generic iterators and iterables don't. If you plan to call these methods on the returned +generator, use the :py:class:`~typing.Generator` type instead of :py:class:`~typing.Iterator` or :py:class:`~typing.Iterable`. diff --git a/docs/source/literal_types.rst b/docs/source/literal_types.rst new file mode 100644 index 000000000000..877ab5de9087 --- /dev/null +++ b/docs/source/literal_types.rst @@ -0,0 +1,521 @@ +Literal types and Enums +======================= + +.. _literal_types: + +Literal types +------------- + +Literal types let you indicate that an expression is equal to some specific +primitive value. For example, if we annotate a variable with type ``Literal["foo"]``, +mypy will understand that variable is not only of type ``str``, but is also +equal to specifically the string ``"foo"``. + +This feature is primarily useful when annotating functions that behave +differently based on the exact value the caller provides. For example, +suppose we have a function ``fetch_data(...)`` that returns ``bytes`` if the +first argument is ``True``, and ``str`` if it's ``False``. We can construct a +precise type signature for this function using ``Literal[...]`` and overloads: + +.. code-block:: python + + from typing import overload, Union, Literal + + # The first two overloads use Literal[...] so we can + # have precise return types: + + @overload + def fetch_data(raw: Literal[True]) -> bytes: ... + @overload + def fetch_data(raw: Literal[False]) -> str: ... + + # The last overload is a fallback in case the caller + # provides a regular bool: + + @overload + def fetch_data(raw: bool) -> Union[bytes, str]: ... + + def fetch_data(raw: bool) -> Union[bytes, str]: + # Implementation is omitted + ... + + reveal_type(fetch_data(True)) # Revealed type is "bytes" + reveal_type(fetch_data(False)) # Revealed type is "str" + + # Variables declared without annotations will continue to have an + # inferred type of 'bool'. + + variable = True + reveal_type(fetch_data(variable)) # Revealed type is "Union[bytes, str]" + +.. note:: + + The examples in this page import ``Literal`` as well as ``Final`` and + ``TypedDict`` from the ``typing`` module. These types were added to + ``typing`` in Python 3.8, but are also available for use in Python + 3.4 - 3.7 via the ``typing_extensions`` package. + +Parameterizing Literals +*********************** + +Literal types may contain one or more literal bools, ints, strs, bytes, and +enum values. However, literal types **cannot** contain arbitrary expressions: +types like ``Literal[my_string.trim()]``, ``Literal[x > 3]``, or ``Literal[3j + 4]`` +are all illegal. + +Literals containing two or more values are equivalent to the union of those values. +So, ``Literal[-3, b"foo", MyEnum.A]`` is equivalent to +``Union[Literal[-3], Literal[b"foo"], Literal[MyEnum.A]]``. This makes writing more +complex types involving literals a little more convenient. + +Literal types may also contain ``None``. Mypy will treat ``Literal[None]`` as being +equivalent to just ``None``. This means that ``Literal[4, None]``, +``Literal[4] | None``, and ``Optional[Literal[4]]`` are all equivalent. + +Literals may also contain aliases to other literal types. For example, the +following program is legal: + +.. code-block:: python + + PrimaryColors = Literal["red", "blue", "yellow"] + SecondaryColors = Literal["purple", "green", "orange"] + AllowedColors = Literal[PrimaryColors, SecondaryColors] + + def paint(color: AllowedColors) -> None: ... + + paint("red") # Type checks! + paint("turquoise") # Does not type check + +Literals may not contain any other kind of type or expression. This means doing +``Literal[my_instance]``, ``Literal[Any]``, ``Literal[3.14]``, or +``Literal[{"foo": 2, "bar": 5}]`` are all illegal. + +Declaring literal variables +*************************** + +You must explicitly add an annotation to a variable to declare that it has +a literal type: + +.. code-block:: python + + a: Literal[19] = 19 + reveal_type(a) # Revealed type is "Literal[19]" + +In order to preserve backwards-compatibility, variables without this annotation +are **not** assumed to be literals: + +.. code-block:: python + + b = 19 + reveal_type(b) # Revealed type is "int" + +If you find repeating the value of the variable in the type hint to be tedious, +you can instead change the variable to be ``Final`` (see :ref:`final_attrs`): + +.. code-block:: python + + from typing import Final, Literal + + def expects_literal(x: Literal[19]) -> None: pass + + c: Final = 19 + + reveal_type(c) # Revealed type is "Literal[19]?" + expects_literal(c) # ...and this type checks! + +If you do not provide an explicit type in the ``Final``, the type of ``c`` becomes +*context-sensitive*: mypy will basically try "substituting" the original assigned +value whenever it's used before performing type checking. This is why the revealed +type of ``c`` is ``Literal[19]?``: the question mark at the end reflects this +context-sensitive nature. + +For example, mypy will type check the above program almost as if it were written like so: + +.. code-block:: python + + from typing import Final, Literal + + def expects_literal(x: Literal[19]) -> None: pass + + reveal_type(19) + expects_literal(19) + +This means that while changing a variable to be ``Final`` is not quite the same thing +as adding an explicit ``Literal[...]`` annotation, it often leads to the same effect +in practice. + +The main cases where the behavior of context-sensitive vs true literal types differ are +when you try using those types in places that are not explicitly expecting a ``Literal[...]``. +For example, compare and contrast what happens when you try appending these types to a list: + +.. code-block:: python + + from typing import Final, Literal + + a: Final = 19 + b: Literal[19] = 19 + + # Mypy will choose to infer list[int] here. + list_of_ints = [] + list_of_ints.append(a) + reveal_type(list_of_ints) # Revealed type is "list[int]" + + # But if the variable you're appending is an explicit Literal, mypy + # will infer list[Literal[19]]. + list_of_lits = [] + list_of_lits.append(b) + reveal_type(list_of_lits) # Revealed type is "list[Literal[19]]" + + +Intelligent indexing +******************** + +We can use Literal types to more precisely index into structured heterogeneous +types such as tuples, NamedTuples, and TypedDicts. This feature is known as +*intelligent indexing*. + +For example, when we index into a tuple using some int, the inferred type is +normally the union of the tuple item types. However, if we want just the type +corresponding to some particular index, we can use Literal types like so: + +.. code-block:: python + + from typing import TypedDict + + tup = ("foo", 3.4) + + # Indexing with an int literal gives us the exact type for that index + reveal_type(tup[0]) # Revealed type is "str" + + # But what if we want the index to be a variable? Normally mypy won't + # know exactly what the index is and so will return a less precise type: + int_index = 0 + reveal_type(tup[int_index]) # Revealed type is "Union[str, float]" + + # But if we use either Literal types or a Final int, we can gain back + # the precision we originally had: + lit_index: Literal[0] = 0 + fin_index: Final = 0 + reveal_type(tup[lit_index]) # Revealed type is "str" + reveal_type(tup[fin_index]) # Revealed type is "str" + + # We can do the same thing with with TypedDict and str keys: + class MyDict(TypedDict): + name: str + main_id: int + backup_id: int + + d: MyDict = {"name": "Saanvi", "main_id": 111, "backup_id": 222} + name_key: Final = "name" + reveal_type(d[name_key]) # Revealed type is "str" + + # You can also index using unions of literals + id_key: Literal["main_id", "backup_id"] + reveal_type(d[id_key]) # Revealed type is "int" + +.. _tagged_unions: + +Tagged unions +************* + +When you have a union of types, you can normally discriminate between each type +in the union by using ``isinstance`` checks. For example, if you had a variable ``x`` of +type ``Union[int, str]``, you could write some code that runs only if ``x`` is an int +by doing ``if isinstance(x, int): ...``. + +However, it is not always possible or convenient to do this. For example, it is not +possible to use ``isinstance`` to distinguish between two different TypedDicts since +at runtime, your variable will simply be just a dict. + +Instead, what you can do is *label* or *tag* your TypedDicts with a distinct Literal +type. Then, you can discriminate between each kind of TypedDict by checking the label: + +.. code-block:: python + + from typing import Literal, TypedDict, Union + + class NewJobEvent(TypedDict): + tag: Literal["new-job"] + job_name: str + config_file_path: str + + class CancelJobEvent(TypedDict): + tag: Literal["cancel-job"] + job_id: int + + Event = Union[NewJobEvent, CancelJobEvent] + + def process_event(event: Event) -> None: + # Since we made sure both TypedDicts have a key named 'tag', it's + # safe to do 'event["tag"]'. This expression normally has the type + # Literal["new-job", "cancel-job"], but the check below will narrow + # the type to either Literal["new-job"] or Literal["cancel-job"]. + # + # This in turns narrows the type of 'event' to either NewJobEvent + # or CancelJobEvent. + if event["tag"] == "new-job": + print(event["job_name"]) + else: + print(event["job_id"]) + +While this feature is mostly useful when working with TypedDicts, you can also +use the same technique with regular objects, tuples, or namedtuples. + +Similarly, tags do not need to be specifically str Literals: they can be any type +you can normally narrow within ``if`` statements and the like. For example, you +could have your tags be int or Enum Literals or even regular classes you narrow +using ``isinstance()`` (Python 3.12 syntax): + +.. code-block:: python + + class Wrapper[T]: + def __init__(self, inner: T) -> None: + self.inner = inner + + def process(w: Wrapper[int] | Wrapper[str]) -> None: + # Doing `if isinstance(w, Wrapper[int])` does not work: isinstance requires + # that the second argument always be an *erased* type, with no generics. + # This is because generics are a typing-only concept and do not exist at + # runtime in a way `isinstance` can always check. + # + # However, we can side-step this by checking the type of `w.inner` to + # narrow `w` itself: + if isinstance(w.inner, int): + reveal_type(w) # Revealed type is "Wrapper[int]" + else: + reveal_type(w) # Revealed type is "Wrapper[str]" + +This feature is sometimes called "sum types" or "discriminated union types" +in other programming languages. + +Exhaustiveness checking +*********************** + +You may want to check that some code covers all possible +``Literal`` or ``Enum`` cases. Example: + +.. code-block:: python + + from typing import Literal + + PossibleValues = Literal['one', 'two'] + + def validate(x: PossibleValues) -> bool: + if x == 'one': + return True + elif x == 'two': + return False + raise ValueError(f'Invalid value: {x}') + + assert validate('one') is True + assert validate('two') is False + +In the code above, it's easy to make a mistake. You can +add a new literal value to ``PossibleValues`` but forget +to handle it in the ``validate`` function: + +.. code-block:: python + + PossibleValues = Literal['one', 'two', 'three'] + +Mypy won't catch that ``'three'`` is not covered. If you want mypy to +perform an exhaustiveness check, you need to update your code to use an +``assert_never()`` check: + +.. code-block:: python + + from typing import Literal, NoReturn + from typing_extensions import assert_never + + PossibleValues = Literal['one', 'two'] + + def validate(x: PossibleValues) -> bool: + if x == 'one': + return True + elif x == 'two': + return False + assert_never(x) + +Now if you add a new value to ``PossibleValues`` but don't update ``validate``, +mypy will spot the error: + +.. code-block:: python + + PossibleValues = Literal['one', 'two', 'three'] + + def validate(x: PossibleValues) -> bool: + if x == 'one': + return True + elif x == 'two': + return False + # Error: Argument 1 to "assert_never" has incompatible type "Literal['three']"; + # expected "NoReturn" + assert_never(x) + +If runtime checking against unexpected values is not needed, you can +leave out the ``assert_never`` call in the above example, and mypy +will still generate an error about function ``validate`` returning +without a value: + +.. code-block:: python + + PossibleValues = Literal['one', 'two', 'three'] + + # Error: Missing return statement + def validate(x: PossibleValues) -> bool: + if x == 'one': + return True + elif x == 'two': + return False + +Exhaustiveness checking is also supported for match statements (Python 3.10 and later): + +.. code-block:: python + + def validate(x: PossibleValues) -> bool: + match x: + case 'one': + return True + case 'two': + return False + assert_never(x) + + +Limitations +*********** + +Mypy will not understand expressions that use variables of type ``Literal[..]`` +on a deep level. For example, if you have a variable ``a`` of type ``Literal[3]`` +and another variable ``b`` of type ``Literal[5]``, mypy will infer that +``a + b`` has type ``int``, **not** type ``Literal[8]``. + +The basic rule is that literal types are treated as just regular subtypes of +whatever type the parameter has. For example, ``Literal[3]`` is treated as a +subtype of ``int`` and so will inherit all of ``int``'s methods directly. This +means that ``Literal[3].__add__`` accepts the same arguments and has the same +return type as ``int.__add__``. + + +Enums +----- + +Mypy has special support for :py:class:`enum.Enum` and its subclasses: +:py:class:`enum.IntEnum`, :py:class:`enum.Flag`, :py:class:`enum.IntFlag`, +and :py:class:`enum.StrEnum`. + +.. code-block:: python + + from enum import Enum + + class Direction(Enum): + up = 'up' + down = 'down' + + reveal_type(Direction.up) # Revealed type is "Literal[Direction.up]?" + reveal_type(Direction.down) # Revealed type is "Literal[Direction.down]?" + +You can use enums to annotate types as you would expect: + +.. code-block:: python + + class Movement: + def __init__(self, direction: Direction, speed: float) -> None: + self.direction = direction + self.speed = speed + + Movement(Direction.up, 5.0) # ok + Movement('up', 5.0) # E: Argument 1 to "Movement" has incompatible type "str"; expected "Direction" + +Exhaustiveness checking +*********************** + +Similar to ``Literal`` types, ``Enum`` supports exhaustiveness checking. +Let's start with a definition: + +.. code-block:: python + + from enum import Enum + from typing import NoReturn + from typing_extensions import assert_never + + class Direction(Enum): + up = 'up' + down = 'down' + +Now, let's use an exhaustiveness check: + +.. code-block:: python + + def choose_direction(direction: Direction) -> None: + if direction is Direction.up: + reveal_type(direction) # N: Revealed type is "Literal[Direction.up]" + print('Going up!') + return + elif direction is Direction.down: + print('Down') + return + # This line is never reached + assert_never(direction) + +If we forget to handle one of the cases, mypy will generate an error: + +.. code-block:: python + + def choose_direction(direction: Direction) -> None: + if direction == Direction.up: + print('Going up!') + return + assert_never(direction) # E: Argument 1 to "assert_never" has incompatible type "Direction"; expected "NoReturn" + +Exhaustiveness checking is also supported for match statements (Python 3.10 and later). + +Extra Enum checks +***************** + +Mypy also tries to support special features of ``Enum`` +the same way Python's runtime does: + +- Any ``Enum`` class with values is implicitly :ref:`final `. + This is what happens in CPython: + + .. code-block:: python + + >>> class AllDirection(Direction): + ... left = 'left' + ... right = 'right' + Traceback (most recent call last): + ... + TypeError: AllDirection: cannot extend enumeration 'Direction' + + Mypy also catches this error: + + .. code-block:: python + + class AllDirection(Direction): # E: Cannot inherit from final class "Direction" + left = 'left' + right = 'right' + +- All ``Enum`` fields are implicitly ``final`` as well. + + .. code-block:: python + + Direction.up = '^' # E: Cannot assign to final attribute "up" + +- All field names are checked to be unique. + + .. code-block:: python + + class Some(Enum): + x = 1 + x = 2 # E: Attempted to reuse member name "x" in Enum definition "Some" + +- Base classes have no conflicts and mixin types are correct. + + .. code-block:: python + + class WrongEnum(str, int, enum.Enum): + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "int" + ... + + class MixinAfterEnum(enum.Enum, Mixin): # E: No base classes are allowed after "enum.Enum" + ... diff --git a/docs/source/metaclasses.rst b/docs/source/metaclasses.rst new file mode 100644 index 000000000000..a3ee25f16054 --- /dev/null +++ b/docs/source/metaclasses.rst @@ -0,0 +1,90 @@ +.. _metaclasses: + +Metaclasses +=========== + +A :ref:`metaclass ` is a class that describes +the construction and behavior of other classes, similarly to how classes +describe the construction and behavior of objects. +The default metaclass is :py:class:`type`, but it's possible to use other metaclasses. +Metaclasses allows one to create "a different kind of class", such as +:py:class:`~enum.Enum`\s, :py:class:`~typing.NamedTuple`\s and singletons. + +Mypy has some special understanding of :py:class:`~abc.ABCMeta` and ``EnumMeta``. + +.. _defining: + +Defining a metaclass +******************** + +.. code-block:: python + + class M(type): + pass + + class A(metaclass=M): + pass + +.. _examples: + +Metaclass usage example +*********************** + +Mypy supports the lookup of attributes in the metaclass: + +.. code-block:: python + + from typing import ClassVar, Self + + class M(type): + count: ClassVar[int] = 0 + + def make(cls) -> Self: + M.count += 1 + return cls() + + class A(metaclass=M): + pass + + a: A = A.make() # make() is looked up at M; the result is an object of type A + print(A.count) + + class B(A): + pass + + b: B = B.make() # metaclasses are inherited + print(B.count + " objects were created") # Error: Unsupported operand types for + ("int" and "str") + +.. note:: + In Python 3.10 and earlier, ``Self`` is available in ``typing_extensions``. + +.. _limitations: + +Gotchas and limitations of metaclass support +******************************************** + +Note that metaclasses pose some requirements on the inheritance structure, +so it's better not to combine metaclasses and class hierarchies: + +.. code-block:: python + + class M1(type): pass + class M2(type): pass + + class A1(metaclass=M1): pass + class A2(metaclass=M2): pass + + class B1(A1, metaclass=M2): pass # Mypy Error: metaclass conflict + # At runtime the above definition raises an exception + # TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + + class B12(A1, A2): pass # Mypy Error: metaclass conflict + + # This can be solved via a common metaclass subtype: + class CorrectMeta(M1, M2): pass + class B2(A1, A2, metaclass=CorrectMeta): pass # OK, runtime is also OK + +* Mypy does not understand dynamically-computed metaclasses, + such as ``class A(metaclass=f()): ...`` +* Mypy does not and cannot understand arbitrary metaclass code. +* Mypy only recognizes subclasses of :py:class:`type` as potential metaclasses. diff --git a/docs/source/more_types.rst b/docs/source/more_types.rst new file mode 100644 index 000000000000..cbf40d5dcaa5 --- /dev/null +++ b/docs/source/more_types.rst @@ -0,0 +1,1006 @@ +More types +========== + +This section introduces a few additional kinds of types, including :py:data:`~typing.NoReturn`, +:py:class:`~typing.NewType`, and types for async code. It also discusses +how to give functions more precise types using overloads. All of these are only +situationally useful, so feel free to skip this section and come back when you +have a need for some of them. + +Here's a quick summary of what's covered here: + +* :py:data:`~typing.NoReturn` lets you tell mypy that a function never returns normally. + +* :py:class:`~typing.NewType` lets you define a variant of a type that is treated as a + separate type by mypy but is identical to the original type at runtime. + For example, you can have ``UserId`` as a variant of ``int`` that is + just an ``int`` at runtime. + +* :py:func:`@overload ` lets you define a function that can accept multiple distinct + signatures. This is useful if you need to encode a relationship between the + arguments and the return type that would be difficult to express normally. + +* Async types let you type check programs using ``async`` and ``await``. + +.. _noreturn: + +The NoReturn type +***************** + +Mypy provides support for functions that never return. For +example, a function that unconditionally raises an exception: + +.. code-block:: python + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + +Mypy will ensure that functions annotated as returning :py:data:`~typing.NoReturn` +truly never return, either implicitly or explicitly. Mypy will also +recognize that the code after calls to such functions is unreachable +and will behave accordingly: + +.. code-block:: python + + def f(x: int) -> int: + if x == 0: + return x + stop() + return 'whatever works' # No error in an unreachable block + +In earlier Python versions you need to install ``typing_extensions`` using +pip to use :py:data:`~typing.NoReturn` in your code. Python 3 command line: + +.. code-block:: text + + python3 -m pip install --upgrade typing-extensions + +.. _newtypes: + +NewTypes +******** + +There are situations where you may want to avoid programming errors by +creating simple derived classes that are only used to distinguish +certain values from base class instances. Example: + +.. code-block:: python + + class UserId(int): + pass + + def get_by_user_id(user_id: UserId): + ... + +However, this approach introduces some runtime overhead. To avoid this, the typing +module provides a helper object :py:class:`~typing.NewType` that creates simple unique types with +almost zero runtime overhead. Mypy will treat the statement +``Derived = NewType('Derived', Base)`` as being roughly equivalent to the following +definition: + +.. code-block:: python + + class Derived(Base): + def __init__(self, _x: Base) -> None: + ... + +However, at runtime, ``NewType('Derived', Base)`` will return a dummy callable that +simply returns its argument: + +.. code-block:: python + + def Derived(_x): + return _x + +Mypy will require explicit casts from ``int`` where ``UserId`` is expected, while +implicitly casting from ``UserId`` where ``int`` is expected. Examples: + +.. code-block:: python + + from typing import NewType + + UserId = NewType('UserId', int) + + def name_by_id(user_id: UserId) -> str: + ... + + UserId('user') # Fails type check + + name_by_id(42) # Fails type check + name_by_id(UserId(42)) # OK + + num: int = UserId(5) + 1 + +:py:class:`~typing.NewType` accepts exactly two arguments. The first argument must be a string literal +containing the name of the new type and must equal the name of the variable to which the new +type is assigned. The second argument must be a properly subclassable class, i.e., +not a type construct like a :ref:`union type `, etc. + +The callable returned by :py:class:`~typing.NewType` accepts only one argument; this is equivalent to +supporting only one constructor accepting an instance of the base class (see above). +Example: + +.. code-block:: python + + from typing import NewType + + class PacketId: + def __init__(self, major: int, minor: int) -> None: + self._major = major + self._minor = minor + + TcpPacketId = NewType('TcpPacketId', PacketId) + + packet = PacketId(100, 100) + tcp_packet = TcpPacketId(packet) # OK + + tcp_packet = TcpPacketId(127, 0) # Fails in type checker and at runtime + +You cannot use :py:func:`isinstance` or :py:func:`issubclass` on the object returned by +:py:class:`~typing.NewType`, nor can you subclass an object returned by :py:class:`~typing.NewType`. + +.. note:: + + Unlike type aliases, :py:class:`~typing.NewType` will create an entirely new and + unique type when used. The intended purpose of :py:class:`~typing.NewType` is to help you + detect cases where you accidentally mixed together the old base type and the + new derived type. + + For example, the following will successfully typecheck when using type + aliases: + + .. code-block:: python + + UserId = int + + def name_by_id(user_id: UserId) -> str: + ... + + name_by_id(3) # ints and UserId are synonymous + + But a similar example using :py:class:`~typing.NewType` will not typecheck: + + .. code-block:: python + + from typing import NewType + + UserId = NewType('UserId', int) + + def name_by_id(user_id: UserId) -> str: + ... + + name_by_id(3) # int is not the same as UserId + +.. _function-overloading: + +Function overloading +******************** + +Sometimes the arguments and types in a function depend on each other +in ways that can't be captured with a :ref:`union types `. For example, suppose +we want to write a function that can accept x-y coordinates. If we pass +in just a single x-y coordinate, we return a ``ClickEvent`` object. However, +if we pass in two x-y coordinates, we return a ``DragEvent`` object. + +Our first attempt at writing this function might look like this: + +.. code-block:: python + + def mouse_event(x1: int, + y1: int, + x2: int | None = None, + y2: int | None = None) -> ClickEvent | DragEvent: + if x2 is None and y2 is None: + return ClickEvent(x1, y1) + elif x2 is not None and y2 is not None: + return DragEvent(x1, y1, x2, y2) + else: + raise TypeError("Bad arguments") + +While this function signature works, it's too loose: it implies ``mouse_event`` +could return either object regardless of the number of arguments +we pass in. It also does not prohibit a caller from passing in the wrong +number of ints: mypy would treat calls like ``mouse_event(1, 2, 20)`` as being +valid, for example. + +We can do better by using :pep:`overloading <484#function-method-overloading>` +which lets us give the same function multiple type annotations (signatures) +to more accurately describe the function's behavior: + +.. code-block:: python + + from typing import overload + + # Overload *variants* for 'mouse_event'. + # These variants give extra information to the type checker. + # They are ignored at runtime. + + @overload + def mouse_event(x1: int, y1: int) -> ClickEvent: ... + @overload + def mouse_event(x1: int, y1: int, x2: int, y2: int) -> DragEvent: ... + + # The actual *implementation* of 'mouse_event'. + # The implementation contains the actual runtime logic. + # + # It may or may not have type hints. If it does, mypy + # will check the body of the implementation against the + # type hints. + # + # Mypy will also check and make sure the signature is + # consistent with the provided variants. + + def mouse_event(x1: int, + y1: int, + x2: int | None = None, + y2: int | None = None) -> ClickEvent | DragEvent: + if x2 is None and y2 is None: + return ClickEvent(x1, y1) + elif x2 is not None and y2 is not None: + return DragEvent(x1, y1, x2, y2) + else: + raise TypeError("Bad arguments") + +This allows mypy to understand calls to ``mouse_event`` much more precisely. +For example, mypy will understand that ``mouse_event(5, 25)`` will +always have a return type of ``ClickEvent`` and will report errors for +calls like ``mouse_event(5, 25, 2)``. + +As another example, suppose we want to write a custom container class that +implements the :py:meth:`__getitem__ ` method (``[]`` bracket indexing). If this +method receives an integer we return a single item. If it receives a +``slice``, we return a :py:class:`~collections.abc.Sequence` of items. + +We can precisely encode this relationship between the argument and the +return type by using overloads like so (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Sequence + from typing import overload + + class MyList[T](Sequence[T]): + @overload + def __getitem__(self, index: int) -> T: ... + + @overload + def __getitem__(self, index: slice) -> Sequence[T]: ... + + def __getitem__(self, index: int | slice) -> T | Sequence[T]: + if isinstance(index, int): + # Return a T here + elif isinstance(index, slice): + # Return a sequence of Ts here + else: + raise TypeError(...) + +Here is the same example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Sequence + from typing import TypeVar, overload + + T = TypeVar('T') + + class MyList(Sequence[T]): + @overload + def __getitem__(self, index: int) -> T: ... + + @overload + def __getitem__(self, index: slice) -> Sequence[T]: ... + + def __getitem__(self, index: int | slice) -> T | Sequence[T]: + if isinstance(index, int): + # Return a T here + elif isinstance(index, slice): + # Return a sequence of Ts here + else: + raise TypeError(...) + +.. note:: + + If you just need to constrain a type variable to certain types or + subtypes, you can use a :ref:`value restriction + `. + +The default values of a function's arguments don't affect its signature -- only +the absence or presence of a default value does. So in order to reduce +redundancy, it's possible to replace default values in overload definitions with +``...`` as a placeholder: + +.. code-block:: python + + from typing import overload + + class M: ... + + @overload + def get_model(model_or_pk: M, flag: bool = ...) -> M: ... + @overload + def get_model(model_or_pk: int, flag: bool = ...) -> M | None: ... + + def get_model(model_or_pk: int | M, flag: bool = True) -> M | None: + ... + + +Runtime behavior +---------------- + +An overloaded function must consist of two or more overload *variants* +followed by an *implementation*. The variants and the implementations +must be adjacent in the code: think of them as one indivisible unit. + +The variant bodies must all be empty; only the implementation is allowed +to contain code. This is because at runtime, the variants are completely +ignored: they're overridden by the final implementation function. + +This means that an overloaded function is still an ordinary Python +function! There is no automatic dispatch handling and you must manually +handle the different types in the implementation (e.g. by using +``if`` statements and :py:func:`isinstance ` checks). + +If you are adding an overload within a stub file, the implementation +function should be omitted: stubs do not contain runtime logic. + +.. note:: + + While we can leave the variant body empty using the ``pass`` keyword, + the more common convention is to instead use the ellipsis (``...``) literal. + +Type checking calls to overloads +-------------------------------- + +When you call an overloaded function, mypy will infer the correct return +type by picking the best matching variant, after taking into consideration +both the argument types and arity. However, a call is never type +checked against the implementation. This is why mypy will report calls +like ``mouse_event(5, 25, 3)`` as being invalid even though it matches the +implementation signature. + +If there are multiple equally good matching variants, mypy will select +the variant that was defined first. For example, consider the following +program: + +.. code-block:: python + + # For Python 3.8 and below you must use `typing.List` instead of `list`. e.g. + # from typing import List + from typing import overload + + @overload + def summarize(data: list[int]) -> float: ... + + @overload + def summarize(data: list[str]) -> str: ... + + def summarize(data): + if not data: + return 0.0 + elif isinstance(data[0], int): + # Do int specific code + else: + # Do str-specific code + + # What is the type of 'output'? float or str? + output = summarize([]) + +The ``summarize([])`` call matches both variants: an empty list could +be either a ``list[int]`` or a ``list[str]``. In this case, mypy +will break the tie by picking the first matching variant: ``output`` +will have an inferred type of ``float``. The implementor is responsible +for making sure ``summarize`` breaks ties in the same way at runtime. + +However, there are two exceptions to the "pick the first match" rule. +First, if multiple variants match due to an argument being of type +``Any``, mypy will make the inferred type also be ``Any``: + +.. code-block:: python + + dynamic_var: Any = some_dynamic_function() + + # output2 is of type 'Any' + output2 = summarize(dynamic_var) + +Second, if multiple variants match due to one or more of the arguments +being a union, mypy will make the inferred type be the union of the +matching variant returns: + +.. code-block:: python + + some_list: list[int] | list[str] + + # output3 is of type 'float | str' + output3 = summarize(some_list) + +.. note:: + + Due to the "pick the first match" rule, changing the order of your + overload variants can change how mypy type checks your program. + + To minimize potential issues, we recommend that you: + + 1. Make sure your overload variants are listed in the same order as + the runtime checks (e.g. :py:func:`isinstance ` checks) in your implementation. + 2. Order your variants and runtime checks from most to least specific. + (See the following section for an example). + +Type checking the variants +-------------------------- + +Mypy will perform several checks on your overload variant definitions +to ensure they behave as expected. First, mypy will check and make sure +that no overload variant is shadowing a subsequent one. For example, +consider the following function which adds together two ``Expression`` +objects, and contains a special-case to handle receiving two ``Literal`` +types: + +.. code-block:: python + + from typing import overload + + class Expression: + # ...snip... + + class Literal(Expression): + # ...snip... + + # Warning -- the first overload variant shadows the second! + + @overload + def add(left: Expression, right: Expression) -> Expression: ... + + @overload + def add(left: Literal, right: Literal) -> Literal: ... + + def add(left: Expression, right: Expression) -> Expression: + # ...snip... + +While this code snippet is technically type-safe, it does contain an +anti-pattern: the second variant will never be selected! If we try calling +``add(Literal(3), Literal(4))``, mypy will always pick the first variant +and evaluate the function call to be of type ``Expression``, not ``Literal``. +This is because ``Literal`` is a subtype of ``Expression``, which means +the "pick the first match" rule will always halt after considering the +first overload. + +Because having an overload variant that can never be matched is almost +certainly a mistake, mypy will report an error. To fix the error, we can +either 1) delete the second overload or 2) swap the order of the overloads: + +.. code-block:: python + + # Everything is ok now -- the variants are correctly ordered + # from most to least specific. + + @overload + def add(left: Literal, right: Literal) -> Literal: ... + + @overload + def add(left: Expression, right: Expression) -> Expression: ... + + def add(left: Expression, right: Expression) -> Expression: + # ...snip... + +Mypy will also type check the different variants and flag any overloads +that have inherently unsafely overlapping variants. For example, consider +the following unsafe overload definition: + +.. code-block:: python + + from typing import overload + + @overload + def unsafe_func(x: int) -> int: ... + + @overload + def unsafe_func(x: object) -> str: ... + + def unsafe_func(x: object) -> int | str: + if isinstance(x, int): + return 42 + else: + return "some string" + +On the surface, this function definition appears to be fine. However, it will +result in a discrepancy between the inferred type and the actual runtime type +when we try using it like so: + +.. code-block:: python + + some_obj: object = 42 + unsafe_func(some_obj) + " danger danger" # Type checks, yet crashes at runtime! + +Since ``some_obj`` is of type :py:class:`object`, mypy will decide that ``unsafe_func`` +must return something of type ``str`` and concludes the above will type check. +But in reality, ``unsafe_func`` will return an int, causing the code to crash +at runtime! + +To prevent these kinds of issues, mypy will detect and prohibit inherently unsafely +overlapping overloads on a best-effort basis. Two variants are considered unsafely +overlapping when both of the following are true: + +1. All of the arguments of the first variant are potentially compatible with the second. +2. The return type of the first variant is *not* compatible with (e.g. is not a + subtype of) the second. + +So in this example, the ``int`` argument in the first variant is a subtype of +the ``object`` argument in the second, yet the ``int`` return type is not a subtype of +``str``. Both conditions are true, so mypy will correctly flag ``unsafe_func`` as +being unsafe. + +Note that in cases where you ignore the overlapping overload error, mypy will usually +still infer the types you expect at callsites. + +However, mypy will not detect *all* unsafe uses of overloads. For example, +suppose we modify the above snippet so it calls ``summarize`` instead of +``unsafe_func``: + +.. code-block:: python + + some_list: list[str] = [] + summarize(some_list) + "danger danger" # Type safe, yet crashes at runtime! + +We run into a similar issue here. This program type checks if we look just at the +annotations on the overloads. But since ``summarize(...)`` is designed to be biased +towards returning a float when it receives an empty list, this program will actually +crash during runtime. + +The reason mypy does not flag definitions like ``summarize`` as being potentially +unsafe is because if it did, it would be extremely difficult to write a safe +overload. For example, suppose we define an overload with two variants that accept +types ``A`` and ``B`` respectively. Even if those two types were completely unrelated, +the user could still potentially trigger a runtime error similar to the ones above by +passing in a value of some third type ``C`` that inherits from both ``A`` and ``B``. + +Thankfully, these types of situations are relatively rare. What this does mean, +however, is that you should exercise caution when designing or using an overloaded +function that can potentially receive values that are an instance of two seemingly +unrelated types. + + +Type checking the implementation +-------------------------------- + +The body of an implementation is type-checked against the +type hints provided on the implementation. For example, in the +``MyList`` example up above, the code in the body is checked with +argument list ``index: int | slice`` and a return type of +``T | Sequence[T]``. If there are no annotations on the +implementation, then the body is not type checked. If you want to +force mypy to check the body anyways, use the :option:`--check-untyped-defs ` +flag (:ref:`more details here `). + +The variants must also also be compatible with the implementation +type hints. In the ``MyList`` example, mypy will check that the +parameter type ``int`` and the return type ``T`` are compatible with +``int | slice`` and ``T | Sequence`` for the +first variant. For the second variant it verifies the parameter +type ``slice`` and the return type ``Sequence[T]`` are compatible +with ``int | slice`` and ``T | Sequence``. + +.. note:: + + The overload semantics documented above are new as of mypy 0.620. + + Previously, mypy used to perform type erasure on all overload variants. For + example, the ``summarize`` example from the previous section used to be + illegal because ``list[str]`` and ``list[int]`` both erased to just ``list[Any]``. + This restriction was removed in mypy 0.620. + + Mypy also previously used to select the best matching variant using a different + algorithm. If this algorithm failed to find a match, it would default to returning + ``Any``. The new algorithm uses the "pick the first match" rule and will fall back + to returning ``Any`` only if the input arguments also contain ``Any``. + + +Conditional overloads +--------------------- + +Sometimes it is useful to define overloads conditionally. +Common use cases include types that are unavailable at runtime or that +only exist in a certain Python version. All existing overload rules still apply. +For example, there must be at least two overloads. + +.. note:: + + Mypy can only infer a limited number of conditions. + Supported ones currently include :py:data:`~typing.TYPE_CHECKING`, ``MYPY``, + :ref:`version_and_platform_checks`, :option:`--always-true `, + and :option:`--always-false ` values. + +.. code-block:: python + + from typing import TYPE_CHECKING, Any, overload + + if TYPE_CHECKING: + class A: ... + class B: ... + + + if TYPE_CHECKING: + @overload + def func(var: A) -> A: ... + + @overload + def func(var: B) -> B: ... + + def func(var: Any) -> Any: + return var + + + reveal_type(func(A())) # Revealed type is "A" + +.. code-block:: python + + # flags: --python-version 3.10 + import sys + from typing import Any, overload + + class A: ... + class B: ... + class C: ... + class D: ... + + + if sys.version_info < (3, 7): + @overload + def func(var: A) -> A: ... + + elif sys.version_info >= (3, 10): + @overload + def func(var: B) -> B: ... + + else: + @overload + def func(var: C) -> C: ... + + @overload + def func(var: D) -> D: ... + + def func(var: Any) -> Any: + return var + + + reveal_type(func(B())) # Revealed type is "B" + reveal_type(func(C())) # No overload variant of "func" matches argument type "C" + # Possible overload variants: + # def func(var: B) -> B + # def func(var: D) -> D + # Revealed type is "Any" + + +.. note:: + + In the last example, mypy is executed with + :option:`--python-version 3.10 `. + Therefore, the condition ``sys.version_info >= (3, 10)`` will match and + the overload for ``B`` will be added. + The overloads for ``A`` and ``C`` are ignored! + The overload for ``D`` is not defined conditionally and thus is also added. + +When mypy cannot infer a condition to be always ``True`` or always ``False``, +an error is emitted. + +.. code-block:: python + + from typing import Any, overload + + class A: ... + class B: ... + + + def g(bool_var: bool) -> None: + if bool_var: # Condition can't be inferred, unable to merge overloads + @overload + def func(var: A) -> A: ... + + @overload + def func(var: B) -> B: ... + + def func(var: Any) -> Any: ... + + reveal_type(func(A())) # Revealed type is "Any" + + +.. _advanced_self: + +Advanced uses of self-types +*************************** + +Normally, mypy doesn't require annotations for the first arguments of instance and +class methods. However, they may be needed to have more precise static typing +for certain programming patterns. + +Restricted methods in generic classes +------------------------------------- + +In generic classes some methods may be allowed to be called only +for certain values of type arguments (Python 3.12 syntax): + +.. code-block:: python + + class Tag[T]: + item: T + + def uppercase_item(self: Tag[str]) -> str: + return self.item.upper() + + def label(ti: Tag[int], ts: Tag[str]) -> None: + ti.uppercase_item() # E: Invalid self argument "Tag[int]" to attribute function + # "uppercase_item" with type "Callable[[Tag[str]], str]" + ts.uppercase_item() # This is OK + +This pattern also allows matching on nested types in situations where the type +argument is itself generic (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Sequence + + class Storage[T]: + def __init__(self, content: T) -> None: + self._content = content + + def first_chunk[S](self: Storage[Sequence[S]]) -> S: + return self._content[0] + + page: Storage[list[str]] + page.first_chunk() # OK, type is "str" + + Storage(0).first_chunk() # Error: Invalid self argument "Storage[int]" to attribute function + # "first_chunk" with type "Callable[[Storage[Sequence[S]]], S]" + +Finally, one can use overloads on self-type to express precise types of +some tricky methods (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Callable + from typing import overload + + class Tag[T]: + @overload + def export(self: Tag[str]) -> str: ... + @overload + def export(self, converter: Callable[[T], str]) -> str: ... + + def export(self, converter=None): + if isinstance(self.item, str): + return self.item + return converter(self.item) + +In particular, an :py:meth:`~object.__init__` method overloaded on self-type +may be useful to annotate generic class constructors where type arguments +depend on constructor parameters in a non-trivial way, see e.g. :py:class:`~subprocess.Popen`. + +Mixin classes +------------- + +Using host class protocol as a self-type in mixin methods allows +more code re-usability for static typing of mixin classes. For example, +one can define a protocol that defines common functionality for +host classes instead of adding required abstract methods to every mixin: + +.. code-block:: python + + class Lockable(Protocol): + @property + def lock(self) -> Lock: ... + + class AtomicCloseMixin: + def atomic_close(self: Lockable) -> int: + with self.lock: + # perform actions + + class AtomicOpenMixin: + def atomic_open(self: Lockable) -> int: + with self.lock: + # perform actions + + class File(AtomicCloseMixin, AtomicOpenMixin): + def __init__(self) -> None: + self.lock = Lock() + + class Bad(AtomicCloseMixin): + pass + + f = File() + b: Bad + f.atomic_close() # OK + b.atomic_close() # Error: Invalid self type for "atomic_close" + +Note that the explicit self-type is *required* to be a protocol whenever it +is not a supertype of the current class. In this case mypy will check the validity +of the self-type only at the call site. + +Precise typing of alternative constructors +------------------------------------------ + +Some classes may define alternative constructors. If these +classes are generic, self-type allows giving them precise +signatures (Python 3.12 syntax): + +.. code-block:: python + + from typing import Self + + class Base[T]: + def __init__(self, item: T) -> None: + self.item = item + + @classmethod + def make_pair(cls, item: T) -> tuple[Self, Self]: + return cls(item), cls(item) + + class Sub[T](Base[T]): + ... + + pair = Sub.make_pair('yes') # Type is "tuple[Sub[str], Sub[str]]" + bad = Sub[int].make_pair('no') # Error: Argument 1 to "make_pair" of "Base" + # has incompatible type "str"; expected "int" + +.. _async-and-await: + +Typing async/await +****************** + +Mypy lets you type coroutines that use the ``async/await`` syntax. +For more information regarding coroutines, see :pep:`492` and the +`asyncio documentation `_. + +Functions defined using ``async def`` are typed similar to normal functions. +The return type annotation should be the same as the type of the value you +expect to get back when ``await``-ing the coroutine. + +.. code-block:: python + + import asyncio + + async def format_string(tag: str, count: int) -> str: + return f'T-minus {count} ({tag})' + + async def countdown(tag: str, count: int) -> str: + while count > 0: + my_str = await format_string(tag, count) # type is inferred to be str + print(my_str) + await asyncio.sleep(0.1) + count -= 1 + return "Blastoff!" + + asyncio.run(countdown("Millennium Falcon", 5)) + +The result of calling an ``async def`` function *without awaiting* will +automatically be inferred to be a value of type +:py:class:`Coroutine[Any, Any, T] `, which is a subtype of +:py:class:`Awaitable[T] `: + +.. code-block:: python + + my_coroutine = countdown("Millennium Falcon", 5) + reveal_type(my_coroutine) # Revealed type is "typing.Coroutine[Any, Any, builtins.str]" + +.. _async-iterators: + +Asynchronous iterators +---------------------- + +If you have an asynchronous iterator, you can use the +:py:class:`~collections.abc.AsyncIterator` type in your annotations: + +.. code-block:: python + + from collections.abc import AsyncIterator + from typing import Optional + import asyncio + + class arange: + def __init__(self, start: int, stop: int, step: int) -> None: + self.start = start + self.stop = stop + self.step = step + self.count = start - step + + def __aiter__(self) -> AsyncIterator[int]: + return self + + async def __anext__(self) -> int: + self.count += self.step + if self.count == self.stop: + raise StopAsyncIteration + else: + return self.count + + async def run_countdown(tag: str, countdown: AsyncIterator[int]) -> str: + async for i in countdown: + print(f'T-minus {i} ({tag})') + await asyncio.sleep(0.1) + return "Blastoff!" + + asyncio.run(run_countdown("Serenity", arange(5, 0, -1))) + +Async generators (introduced in :pep:`525`) are an easy way to create +async iterators: + +.. code-block:: python + + from collections.abc import AsyncGenerator + from typing import Optional + import asyncio + + # Could also type this as returning AsyncIterator[int] + async def arange(start: int, stop: int, step: int) -> AsyncGenerator[int, None]: + current = start + while (step > 0 and current < stop) or (step < 0 and current > stop): + yield current + current += step + + asyncio.run(run_countdown("Battlestar Galactica", arange(5, 0, -1))) + +One common confusion is that the presence of a ``yield`` statement in an +``async def`` function has an effect on the type of the function: + +.. code-block:: python + + from collections.abc import AsyncIterator + + async def arange(stop: int) -> AsyncIterator[int]: + # When called, arange gives you an async iterator + # Equivalent to Callable[[int], AsyncIterator[int]] + i = 0 + while i < stop: + yield i + i += 1 + + async def coroutine(stop: int) -> AsyncIterator[int]: + # When called, coroutine gives you something you can await to get an async iterator + # Equivalent to Callable[[int], Coroutine[Any, Any, AsyncIterator[int]]] + return arange(stop) + + async def main() -> None: + reveal_type(arange(5)) # Revealed type is "typing.AsyncIterator[builtins.int]" + reveal_type(coroutine(5)) # Revealed type is "typing.Coroutine[Any, Any, typing.AsyncIterator[builtins.int]]" + + await arange(5) # Error: Incompatible types in "await" (actual type "AsyncIterator[int]", expected type "Awaitable[Any]") + reveal_type(await coroutine(5)) # Revealed type is "typing.AsyncIterator[builtins.int]" + +This can sometimes come up when trying to define base classes, Protocols or overloads: + +.. code-block:: python + + from collections.abc import AsyncIterator + from typing import Protocol, overload + + class LauncherIncorrect(Protocol): + # Because launch does not have yield, this has type + # Callable[[], Coroutine[Any, Any, AsyncIterator[int]]] + # instead of + # Callable[[], AsyncIterator[int]] + async def launch(self) -> AsyncIterator[int]: + raise NotImplementedError + + class LauncherCorrect(Protocol): + def launch(self) -> AsyncIterator[int]: + raise NotImplementedError + + class LauncherAlsoCorrect(Protocol): + async def launch(self) -> AsyncIterator[int]: + raise NotImplementedError + if False: + yield 0 + + # The type of the overloads is independent of the implementation. + # In particular, their type is not affected by whether or not the + # implementation contains a `yield`. + # Use of `def`` makes it clear the type is Callable[..., AsyncIterator[int]], + # whereas with `async def` it would be Callable[..., Coroutine[Any, Any, AsyncIterator[int]]] + @overload + def launch(*, count: int = ...) -> AsyncIterator[int]: ... + @overload + def launch(*, time: float = ...) -> AsyncIterator[int]: ... + + async def launch(*, count: int = 0, time: float = 0) -> AsyncIterator[int]: + # The implementation of launch is an async generator and contains a yield + yield 0 diff --git a/docs/source/mypy_daemon.rst b/docs/source/mypy_daemon.rst new file mode 100644 index 000000000000..6c511e14eb95 --- /dev/null +++ b/docs/source/mypy_daemon.rst @@ -0,0 +1,380 @@ +.. _mypy_daemon: + +.. program:: dmypy + +Mypy daemon (mypy server) +========================= + +Instead of running mypy as a command-line tool, you can also run it as +a long-running daemon (server) process and use a command-line client to +send type-checking requests to the server. This way mypy can perform type +checking much faster, since program state cached from previous runs is kept +in memory and doesn't have to be read from the file system on each run. +The server also uses finer-grained dependency tracking to reduce the amount +of work that needs to be done. + +If you have a large codebase to check, running mypy using the mypy +daemon can be *10 or more times faster* than the regular command-line +``mypy`` tool, especially if your workflow involves running mypy +repeatedly after small edits -- which is often a good idea, as this way +you'll find errors sooner. + +.. note:: + + The command-line interface of mypy daemon may change in future mypy + releases. + +.. note:: + + Each mypy daemon process supports one user and one set of source files, + and it can only process one type checking request at a time. You can + run multiple mypy daemon processes to type check multiple repositories. + + +Basic usage +*********** + +The client utility ``dmypy`` is used to control the mypy daemon. +Use ``dmypy run -- `` to type check a set of files +(or directories). This will launch the daemon if it is not running. +You can use almost arbitrary mypy flags after ``--``. The daemon +will always run on the current host. Example:: + + dmypy run -- prog.py pkg/*.py + +``dmypy run`` will automatically restart the daemon if the +configuration or mypy version changes. + +The initial run will process all the code and may take a while to +finish, but subsequent runs will be quick, especially if you've only +changed a few files. (You can use :ref:`remote caching ` +to speed up the initial run. The speedup can be significant if +you have a large codebase.) + +.. note:: + + Mypy 0.780 added support for following imports in dmypy (enabled by + default). This functionality is still experimental. You can use + ``--follow-imports=skip`` or ``--follow-imports=error`` to fall + back to the stable functionality. See :ref:`follow-imports` for + details on how these work. + +.. note:: + + The mypy daemon requires ``--local-partial-types`` and automatically enables it. + + +Daemon client commands +********************** + +While ``dmypy run`` is sufficient for most uses, some workflows +(ones using :ref:`remote caching `, perhaps), +require more precise control over the lifetime of the daemon process: + +* ``dmypy stop`` stops the daemon. + +* ``dmypy start -- `` starts the daemon but does not check any files. + You can use almost arbitrary mypy flags after ``--``. + +* ``dmypy restart -- `` restarts the daemon. The flags are the same + as with ``dmypy start``. This is equivalent to a stop command followed + by a start. + +* Use ``dmypy run --timeout SECONDS -- `` (or + ``start`` or ``restart``) to automatically + shut down the daemon after inactivity. By default, the daemon runs + until it's explicitly stopped. + +* ``dmypy check `` checks a set of files using an already + running daemon. + +* ``dmypy recheck`` checks the same set of files as the most recent + ``check`` or ``recheck`` command. (You can also use the :option:`--update` + and :option:`--remove` options to alter the set of files, and to define + which files should be processed.) + +* ``dmypy status`` checks whether a daemon is running. It prints a + diagnostic and exits with ``0`` if there is a running daemon. + +Use ``dmypy --help`` for help on additional commands and command-line +options not discussed here, and ``dmypy --help`` for help on +command-specific options. + +Additional daemon flags +*********************** + +.. option:: --status-file FILE + + Use ``FILE`` as the status file for storing daemon runtime state. This is + normally a JSON file that contains information about daemon process and + connection. The default path is ``.dmypy.json`` in the current working + directory. + +.. option:: --log-file FILE + + Direct daemon stdout/stderr to ``FILE``. This is useful for debugging daemon + crashes, since the server traceback is not always printed by the client. + This is available for the ``start``, ``restart``, and ``run`` commands. + +.. option:: --timeout TIMEOUT + + Automatically shut down server after ``TIMEOUT`` seconds of inactivity. + This is available for the ``start``, ``restart``, and ``run`` commands. + +.. option:: --update FILE + + Re-check ``FILE``, or add it to the set of files being + checked (and check it). This option may be repeated, and it's only available for + the ``recheck`` command. By default, mypy finds and checks all files changed + since the previous run and files that depend on them. However, if you use this option + (and/or :option:`--remove`), mypy assumes that only the explicitly + specified files have changed. This is only useful to + speed up mypy if you type check a very large number of files, and use an + external, fast file system watcher, such as `watchman`_ or + `watchdog`_, to determine which files got edited or deleted. + *Note:* This option is never required and is only available for + performance tuning. + +.. option:: --remove FILE + + Remove ``FILE`` from the set of files being checked. This option may be + repeated. This is only available for the + ``recheck`` command. See :option:`--update` above for when this may be useful. + *Note:* This option is never required and is only available for performance + tuning. + +.. option:: --fswatcher-dump-file FILE + + Collect information about the current internal file state. This is + only available for the ``status`` command. This will dump JSON to + ``FILE`` in the format ``{path: [modification_time, size, + content_hash]}``. This is useful for debugging the built-in file + system watcher. *Note:* This is an internal flag and the format may + change. + +.. option:: --perf-stats-file FILE + + Write performance profiling information to ``FILE``. This is only available + for the ``check``, ``recheck``, and ``run`` commands. + +.. option:: --export-types + + Store all expression types in memory for future use. This is useful to speed + up future calls to ``dmypy inspect`` (but uses more memory). Only valid for + ``check``, ``recheck``, and ``run`` command. + +Static inference of annotations +******************************* + +The mypy daemon supports (as an experimental feature) statically inferring +draft function and method type annotations. Use ``dmypy suggest FUNCTION`` to +generate a draft signature in the format +``(param_type_1, param_type_2, ...) -> ret_type`` (types are included for all +arguments, including keyword-only arguments, ``*args`` and ``**kwargs``). + +This is a low-level feature intended to be used by editor integrations, +IDEs, and other tools (for example, the `mypy plugin for PyCharm`_), +to automatically add annotations to source files, or to propose function +signatures. + +In this example, the function ``format_id()`` has no annotation: + +.. code-block:: python + + def format_id(user): + return f"User: {user}" + + root = format_id(0) + +``dmypy suggest`` uses call sites, return statements, and other heuristics (such as +looking for signatures in base classes) to infer that ``format_id()`` accepts +an ``int`` argument and returns a ``str``. Use ``dmypy suggest module.format_id`` to +print the suggested signature for the function. + +More generally, the target function may be specified in two ways: + +* By its fully qualified name, i.e. ``[package.]module.[class.]function``. + +* By its location in a source file, i.e. ``/path/to/file.py:line``. The path can be + absolute or relative, and ``line`` can refer to any line number within + the function body. + +This command can also be used to find a more precise alternative for an existing, +imprecise annotation with some ``Any`` types. + +The following flags customize various aspects of the ``dmypy suggest`` +command. + +.. option:: --json + + Output the signature as JSON, so that `PyAnnotate`_ can read it and add + the signature to the source file. Here is what the JSON looks like: + + .. code-block:: python + + [{"func_name": "example.format_id", + "line": 1, + "path": "/absolute/path/to/example.py", + "samples": 0, + "signature": {"arg_types": ["int"], "return_type": "str"}}] + +.. option:: --no-errors + + Only produce suggestions that cause no errors in the checked code. By default, + mypy will try to find the most precise type, even if it causes some type errors. + +.. option:: --no-any + + Only produce suggestions that don't contain ``Any`` types. By default mypy + proposes the most precise signature found, even if it contains ``Any`` types. + +.. option:: --flex-any FRACTION + + Only allow some fraction of types in the suggested signature to be ``Any`` types. + The fraction ranges from ``0`` (same as ``--no-any``) to ``1``. + +.. option:: --callsites + + Only find call sites for a given function instead of suggesting a type. + This will produce a list with line numbers and types of actual + arguments for each call: ``/path/to/file.py:line: (arg_type_1, arg_type_2, ...)``. + +.. option:: --use-fixme NAME + + Use a dummy name instead of plain ``Any`` for types that cannot + be inferred. This may be useful to emphasize to a user that a given type + couldn't be inferred and needs to be entered manually. + +.. option:: --max-guesses NUMBER + + Set the maximum number of types to try for a function (default: ``64``). + +Statically inspect expressions +****************************** + +The daemon allows to get declared or inferred type of an expression (or other +information about an expression, such as known attributes or definition location) +using ``dmypy inspect LOCATION`` command. The location of the expression should be +specified in the format ``path/to/file.py:line:column[:end_line:end_column]``. +Both line and column are 1-based. Both start and end position are inclusive. +These rules match how mypy prints the error location in error messages. + +If a span is given (i.e. all 4 numbers), then only an exactly matching expression +is inspected. If only a position is given (i.e. 2 numbers, line and column), mypy +will inspect all *expressions*, that include this position, starting from the +innermost one. + +Consider this Python code snippet: + +.. code-block:: python + + def foo(x: int, longer_name: str) -> None: + x + longer_name + +Here to find the type of ``x`` one needs to call ``dmypy inspect src.py:2:5:2:5`` +or ``dmypy inspect src.py:2:5``. While for ``longer_name`` one needs to call +``dmypy inspect src.py:3:5:3:15`` or, for example, ``dmypy inspect src.py:3:10``. +Please note that this command is only valid after daemon had a successful type +check (without parse errors), so that types are populated, e.g. using +``dmypy check``. In case where multiple expressions match the provided location, +their types are returned separated by a newline. + +Important note: it is recommended to check files with :option:`--export-types` +since otherwise most inspections will not work without :option:`--force-reload`. + +.. option:: --show INSPECTION + + What kind of inspection to run for expression(s) found. Currently the supported + inspections are: + + * ``type`` (default): Show the best known type of a given expression. + * ``attrs``: Show which attributes are valid for an expression (e.g. for + auto-completion). Format is ``{"Base1": ["name_1", "name_2", ...]; "Base2": ...}``. + Names are sorted by method resolution order. If expression refers to a module, + then module attributes will be under key like ``""``. + * ``definition`` (experimental): Show the definition location for a name + expression or member expression. Format is ``path/to/file.py:line:column:Symbol``. + If multiple definitions are found (e.g. for a Union attribute), they are + separated by comma. + +.. option:: --verbose + + Increase verbosity of types string representation (can be repeated). + For example, this will print fully qualified names of instance types (like + ``"builtins.str"``), instead of just a short name (like ``"str"``). + +.. option:: --limit NUM + + If the location is given as ``line:column``, this will cause daemon to + return only at most ``NUM`` inspections of innermost expressions. + Value of 0 means no limit (this is the default). For example, if one calls + ``dmypy inspect src.py:4:10 --limit=1`` with this code + + .. code-block:: python + + def foo(x: int) -> str: .. + def bar(x: str) -> None: ... + baz: int + bar(foo(baz)) + + This will output just one type ``"int"`` (for ``baz`` name expression). + While without the limit option, it would output all three types: ``"int"``, + ``"str"``, and ``"None"``. + +.. option:: --include-span + + With this option on, the daemon will prepend each inspection result with + the full span of corresponding expression, formatted as ``1:2:1:4 -> "int"``. + This may be useful in case multiple expressions match a location. + +.. option:: --include-kind + + With this option on, the daemon will prepend each inspection result with + the kind of corresponding expression, formatted as ``NameExpr -> "int"``. + If both this option and :option:`--include-span` are on, the kind will + appear first, for example ``NameExpr:1:2:1:4 -> "int"``. + +.. option:: --include-object-attrs + + This will make the daemon include attributes of ``object`` (excluded by + default) in case of an ``atts`` inspection. + +.. option:: --union-attrs + + Include attributes valid for some of possible expression types (by default + an intersection is returned). This is useful for union types of type variables + with values. For example, with this code: + + .. code-block:: python + + from typing import Union + + class A: + x: int + z: int + class B: + y: int + z: int + var: Union[A, B] + var + + The command ``dmypy inspect --show attrs src.py:10:1`` will return + ``{"A": ["z"], "B": ["z"]}``, while with ``--union-attrs`` it will return + ``{"A": ["x", "z"], "B": ["y", "z"]}``. + +.. option:: --force-reload + + Force re-parsing and re-type-checking file before inspection. By default + this is done only when needed (for example file was not loaded from cache + or daemon was initially run without ``--export-types`` mypy option), + since reloading may be slow (up to few seconds for very large files). + +.. TODO: Add similar section about find usages when added, and then move + this to a separate file. + + +.. _watchman: https://facebook.github.io/watchman/ +.. _watchdog: https://pypi.org/project/watchdog/ +.. _PyAnnotate: https://github.com/dropbox/pyannotate +.. _mypy plugin for PyCharm: https://github.com/dropbox/mypy-PyCharm-plugin diff --git a/docs/source/mypy_light.svg b/docs/source/mypy_light.svg new file mode 100644 index 000000000000..4eaf65dbf344 --- /dev/null +++ b/docs/source/mypy_light.svg @@ -0,0 +1,99 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + diff --git a/docs/source/planned_features.rst b/docs/source/planned_features.rst deleted file mode 100644 index 5a4c59d547d0..000000000000 --- a/docs/source/planned_features.rst +++ /dev/null @@ -1,64 +0,0 @@ -Planned features -================ - -This section introduces some language features that are still work in -progress. - -Type checking of None ---------------------- - -Currently, ``None`` is a valid value for each type, similar to -``null`` or ``NULL`` in many languages. However, it is likely that -this decision will be reversed, and types do not include ``None`` -default. The ``Optional`` type modifier would be used to define a type -variant that includes ``None``, such as ``Optional[int]``: - -.. code-block:: python - - def f() -> Optional[int]: - return None # OK - - def g() -> int: - ... - return None # Error: None not compatible with int - -Also, most operations would not be supported on ``None`` values: - -.. code-block:: python - - def f(x: Optional[int]) -> int: - return x + 1 # Error: Cannot add None and int - -Instead, an explicit ``None`` check would be required. This would -benefit from more powerful type inference: - -.. code-block:: python - - def f(x: Optional[int]) -> int: - if x is None: - return 0 - else: - # The inferred type of x is just int here. - return x + 1 - -We would infer the type of ``x`` to be int in the else block due to the -check against ``None`` in the if condition. - -More general type inference ---------------------------- - -It may be useful to support type inference also for variables defined -in multiple locations in an if/else statement, even if the initializer -types are different: - -.. code-block:: python - - if x: - y = None # First definition of y - else: - y = 'a' # Second definition of y - -In the above example, both of the assignments would be used in type -inference, and the type of ``y`` would be ``str``. However, it is not -obvious whether this would be generally desirable in more complex -cases. diff --git a/docs/source/protocols.rst b/docs/source/protocols.rst new file mode 100644 index 000000000000..ed8d94f62ef1 --- /dev/null +++ b/docs/source/protocols.rst @@ -0,0 +1,582 @@ +.. _protocol-types: + +Protocols and structural subtyping +================================== + +The Python type system supports two ways of deciding whether two objects are +compatible as types: nominal subtyping and structural subtyping. + +*Nominal* subtyping is strictly based on the class hierarchy. If class ``Dog`` +inherits class ``Animal``, it's a subtype of ``Animal``. Instances of ``Dog`` +can be used when ``Animal`` instances are expected. This form of subtyping +is what Python's type system predominantly uses: it's easy to +understand and produces clear and concise error messages, and matches how the +native :py:func:`isinstance ` check works -- based on class +hierarchy. + +*Structural* subtyping is based on the operations that can be performed with an +object. Class ``Dog`` is a structural subtype of class ``Animal`` if the former +has all attributes and methods of the latter, and with compatible types. + +Structural subtyping can be seen as a static equivalent of duck typing, which is +well known to Python programmers. See :pep:`544` for the detailed specification +of protocols and structural subtyping in Python. + +.. _predefined_protocols: + +Predefined protocols +******************** + +The :py:mod:`collections.abc`, :py:mod:`typing` and other stdlib modules define +various protocol classes that correspond to common Python protocols, such as +:py:class:`Iterable[T] `. If a class +defines a suitable :py:meth:`__iter__ ` method, mypy understands that it +implements the iterable protocol and is compatible with :py:class:`Iterable[T] `. +For example, ``IntList`` below is iterable, over ``int`` values: + +.. code-block:: python + + from __future__ import annotations + + from collections.abc import Iterator, Iterable + + class IntList: + def __init__(self, value: int, next: IntList | None) -> None: + self.value = value + self.next = next + + def __iter__(self) -> Iterator[int]: + current = self + while current: + yield current.value + current = current.next + + def print_numbered(items: Iterable[int]) -> None: + for n, x in enumerate(items): + print(n + 1, x) + + x = IntList(3, IntList(5, None)) + print_numbered(x) # OK + print_numbered([4, 5]) # Also OK + +:ref:`predefined_protocols_reference` lists various protocols defined in +:py:mod:`collections.abc` and :py:mod:`typing` and the signatures of the corresponding methods +you need to define to implement each protocol. + +.. note:: + ``typing`` also contains deprecated aliases to protocols and ABCs defined in + :py:mod:`collections.abc`, such as :py:class:`Iterable[T] `. + These are only necessary in Python 3.8 and earlier, since the protocols in + ``collections.abc`` didn't yet support subscripting (``[]``) in Python 3.8, + but the aliases in ``typing`` have always supported + subscripting. In Python 3.9 and later, the aliases in ``typing`` don't provide + any extra functionality. + +Simple user-defined protocols +***************************** + +You can define your own protocol class by inheriting the special ``Protocol`` +class: + +.. code-block:: python + + from collections.abc import Iterable + from typing import Protocol + + class SupportsClose(Protocol): + # Empty method body (explicit '...') + def close(self) -> None: ... + + class Resource: # No SupportsClose base class! + + def close(self) -> None: + self.resource.release() + + # ... other methods ... + + def close_all(items: Iterable[SupportsClose]) -> None: + for item in items: + item.close() + + close_all([Resource(), open('some/file')]) # OK + +``Resource`` is a subtype of the ``SupportsClose`` protocol since it defines +a compatible ``close`` method. Regular file objects returned by :py:func:`open` are +similarly compatible with the protocol, as they support ``close()``. + +Defining subprotocols and subclassing protocols +*********************************************** + +You can also define subprotocols. Existing protocols can be extended +and merged using multiple inheritance. Example: + +.. code-block:: python + + # ... continuing from the previous example + + class SupportsRead(Protocol): + def read(self, amount: int) -> bytes: ... + + class TaggedReadableResource(SupportsClose, SupportsRead, Protocol): + label: str + + class AdvancedResource(Resource): + def __init__(self, label: str) -> None: + self.label = label + + def read(self, amount: int) -> bytes: + # some implementation + ... + + resource: TaggedReadableResource + resource = AdvancedResource('handle with care') # OK + +Note that inheriting from an existing protocol does not automatically +turn the subclass into a protocol -- it just creates a regular +(non-protocol) class or ABC that implements the given protocol (or +protocols). The ``Protocol`` base class must always be explicitly +present if you are defining a protocol: + +.. code-block:: python + + class NotAProtocol(SupportsClose): # This is NOT a protocol + new_attr: int + + class Concrete: + new_attr: int = 0 + + def close(self) -> None: + ... + + # Error: nominal subtyping used by default + x: NotAProtocol = Concrete() # Error! + +You can also include default implementations of methods in +protocols. If you explicitly subclass these protocols you can inherit +these default implementations. + +Explicitly including a protocol as a +base class is also a way of documenting that your class implements a +particular protocol, and it forces mypy to verify that your class +implementation is actually compatible with the protocol. In particular, +omitting a value for an attribute or a method body will make it implicitly +abstract: + +.. code-block:: python + + class SomeProto(Protocol): + attr: int # Note, no right hand side + def method(self) -> str: ... # Literally just ... here + + class ExplicitSubclass(SomeProto): + pass + + ExplicitSubclass() # error: Cannot instantiate abstract class 'ExplicitSubclass' + # with abstract attributes 'attr' and 'method' + +Similarly, explicitly assigning to a protocol instance can be a way to ask the +type checker to verify that your class implements a protocol: + +.. code-block:: python + + _proto: SomeProto = cast(ExplicitSubclass, None) + +Invariance of protocol attributes +********************************* + +A common issue with protocols is that protocol attributes are invariant. +For example: + +.. code-block:: python + + class Box(Protocol): + content: object + + class IntBox: + content: int + + def takes_box(box: Box) -> None: ... + + takes_box(IntBox()) # error: Argument 1 to "takes_box" has incompatible type "IntBox"; expected "Box" + # note: Following member(s) of "IntBox" have conflicts: + # note: content: expected "object", got "int" + +This is because ``Box`` defines ``content`` as a mutable attribute. +Here's why this is problematic: + +.. code-block:: python + + def takes_box_evil(box: Box) -> None: + box.content = "asdf" # This is bad, since box.content is supposed to be an object + + my_int_box = IntBox() + takes_box_evil(my_int_box) + my_int_box.content + 1 # Oops, TypeError! + +This can be fixed by declaring ``content`` to be read-only in the ``Box`` +protocol using ``@property``: + +.. code-block:: python + + class Box(Protocol): + @property + def content(self) -> object: ... + + class IntBox: + content: int + + def takes_box(box: Box) -> None: ... + + takes_box(IntBox(42)) # OK + +Recursive protocols +******************* + +Protocols can be recursive (self-referential) and mutually +recursive. This is useful for declaring abstract recursive collections +such as trees and linked lists: + +.. code-block:: python + + from __future__ import annotations + + from typing import Protocol + + class TreeLike(Protocol): + value: int + + @property + def left(self) -> TreeLike | None: ... + + @property + def right(self) -> TreeLike | None: ... + + class SimpleTree: + def __init__(self, value: int) -> None: + self.value = value + self.left: SimpleTree | None = None + self.right: SimpleTree | None = None + + root: TreeLike = SimpleTree(0) # OK + +Using isinstance() with protocols +********************************* + +You can use a protocol class with :py:func:`isinstance` if you decorate it +with the ``@runtime_checkable`` class decorator. The decorator adds +rudimentary support for runtime structural checks: + +.. code-block:: python + + from typing import Protocol, runtime_checkable + + @runtime_checkable + class Portable(Protocol): + handles: int + + class Mug: + def __init__(self) -> None: + self.handles = 1 + + def use(handles: int) -> None: ... + + mug = Mug() + if isinstance(mug, Portable): # Works at runtime! + use(mug.handles) + +:py:func:`isinstance` also works with the :ref:`predefined protocols ` +in :py:mod:`typing` such as :py:class:`~typing.Iterable`. + +.. warning:: + :py:func:`isinstance` with protocols is not completely safe at runtime. + For example, signatures of methods are not checked. The runtime + implementation only checks that all protocol members exist, + not that they have the correct type. :py:func:`issubclass` with protocols + will only check for the existence of methods. + +.. note:: + :py:func:`isinstance` with protocols can also be surprisingly slow. + In many cases, you're better served by using :py:func:`hasattr` to + check for the presence of attributes. + +.. _callback_protocols: + +Callback protocols +****************** + +Protocols can be used to define flexible callback types that are hard +(or even impossible) to express using the +:py:class:`Callable[...] ` syntax, +such as variadic, overloaded, and complex generic callbacks. They are defined with a +special :py:meth:`__call__ ` member: + +.. code-block:: python + + from collections.abc import Iterable + from typing import Optional, Protocol + + class Combiner(Protocol): + def __call__(self, *vals: bytes, maxlen: int | None = None) -> list[bytes]: ... + + def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes: + for item in data: + ... + + def good_cb(*vals: bytes, maxlen: int | None = None) -> list[bytes]: + ... + def bad_cb(*vals: bytes, maxitems: int | None) -> list[bytes]: + ... + + batch_proc([], good_cb) # OK + batch_proc([], bad_cb) # Error! Argument 2 has incompatible type because of + # different name and kind in the callback + +Callback protocols and :py:class:`~collections.abc.Callable` types can be used mostly interchangeably. +Parameter names in :py:meth:`__call__ ` methods must be identical, unless +the parameters are positional-only. Example (using the legacy syntax for generic functions): + +.. code-block:: python + + from collections.abc import Callable + from typing import Protocol, TypeVar + + T = TypeVar('T') + + class Copy(Protocol): + # '/' marks the end of positional-only parameters + def __call__(self, origin: T, /) -> T: ... + + copy_a: Callable[[T], T] + copy_b: Copy + + copy_a = copy_b # OK + copy_b = copy_a # Also OK + +.. _predefined_protocols_reference: + +Predefined protocol reference +***************************** + +Iteration protocols +................... + +The iteration protocols are useful in many contexts. For example, they allow +iteration of objects in for loops. + +collections.abc.Iterable[T] +--------------------------- + +The :ref:`example above ` has a simple implementation of an +:py:meth:`__iter__ ` method. + +.. code-block:: python + + def __iter__(self) -> Iterator[T] + +See also :py:class:`~collections.abc.Iterable`. + +collections.abc.Iterator[T] +--------------------------- + +.. code-block:: python + + def __next__(self) -> T + def __iter__(self) -> Iterator[T] + +See also :py:class:`~collections.abc.Iterator`. + +Collection protocols +.................... + +Many of these are implemented by built-in container types such as +:py:class:`list` and :py:class:`dict`, and these are also useful for user-defined +collection objects. + +collections.abc.Sized +--------------------- + +This is a type for objects that support :py:func:`len(x) `. + +.. code-block:: python + + def __len__(self) -> int + +See also :py:class:`~collections.abc.Sized`. + +collections.abc.Container[T] +---------------------------- + +This is a type for objects that support the ``in`` operator. + +.. code-block:: python + + def __contains__(self, x: object) -> bool + +See also :py:class:`~collections.abc.Container`. + +collections.abc.Collection[T] +----------------------------- + +.. code-block:: python + + def __len__(self) -> int + def __iter__(self) -> Iterator[T] + def __contains__(self, x: object) -> bool + +See also :py:class:`~collections.abc.Collection`. + +One-off protocols +................. + +These protocols are typically only useful with a single standard +library function or class. + +collections.abc.Reversible[T] +----------------------------- + +This is a type for objects that support :py:func:`reversed(x) `. + +.. code-block:: python + + def __reversed__(self) -> Iterator[T] + +See also :py:class:`~collections.abc.Reversible`. + +typing.SupportsAbs[T] +--------------------- + +This is a type for objects that support :py:func:`abs(x) `. ``T`` is the type of +value returned by :py:func:`abs(x) `. + +.. code-block:: python + + def __abs__(self) -> T + +See also :py:class:`~typing.SupportsAbs`. + +typing.SupportsBytes +-------------------- + +This is a type for objects that support :py:class:`bytes(x) `. + +.. code-block:: python + + def __bytes__(self) -> bytes + +See also :py:class:`~typing.SupportsBytes`. + +.. _supports-int-etc: + +typing.SupportsComplex +---------------------- + +This is a type for objects that support :py:class:`complex(x) `. Note that no arithmetic operations +are supported. + +.. code-block:: python + + def __complex__(self) -> complex + +See also :py:class:`~typing.SupportsComplex`. + +typing.SupportsFloat +-------------------- + +This is a type for objects that support :py:class:`float(x) `. Note that no arithmetic operations +are supported. + +.. code-block:: python + + def __float__(self) -> float + +See also :py:class:`~typing.SupportsFloat`. + +typing.SupportsInt +------------------ + +This is a type for objects that support :py:class:`int(x) `. Note that no arithmetic operations +are supported. + +.. code-block:: python + + def __int__(self) -> int + +See also :py:class:`~typing.SupportsInt`. + +typing.SupportsRound[T] +----------------------- + +This is a type for objects that support :py:func:`round(x) `. + +.. code-block:: python + + def __round__(self) -> T + +See also :py:class:`~typing.SupportsRound`. + +Async protocols +............... + +These protocols can be useful in async code. See :ref:`async-and-await` +for more information. + +collections.abc.Awaitable[T] +---------------------------- + +.. code-block:: python + + def __await__(self) -> Generator[Any, None, T] + +See also :py:class:`~collections.abc.Awaitable`. + +collections.abc.AsyncIterable[T] +-------------------------------- + +.. code-block:: python + + def __aiter__(self) -> AsyncIterator[T] + +See also :py:class:`~collections.abc.AsyncIterable`. + +collections.abc.AsyncIterator[T] +-------------------------------- + +.. code-block:: python + + def __anext__(self) -> Awaitable[T] + def __aiter__(self) -> AsyncIterator[T] + +See also :py:class:`~collections.abc.AsyncIterator`. + +Context manager protocols +......................... + +There are two protocols for context managers -- one for regular context +managers and one for async ones. These allow defining objects that can +be used in ``with`` and ``async with`` statements. + +contextlib.AbstractContextManager[T] +------------------------------------ + +.. code-block:: python + + def __enter__(self) -> T + def __exit__(self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None) -> bool | None + +See also :py:class:`~contextlib.AbstractContextManager`. + +contextlib.AbstractAsyncContextManager[T] +----------------------------------------- + +.. code-block:: python + + def __aenter__(self) -> Awaitable[T] + def __aexit__(self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None) -> Awaitable[bool | None] + +See also :py:class:`~contextlib.AbstractAsyncContextManager`. diff --git a/docs/source/revision_history.rst b/docs/source/revision_history.rst deleted file mode 100644 index 889daa5ef985..000000000000 --- a/docs/source/revision_history.rst +++ /dev/null @@ -1,16 +0,0 @@ -Revision history -================ - -List of major changes to this document: - -- Oct 26 2014: Major restructuring. Split the HTML documentation into - multiple pages. - -- Sep 15 2014: Migrated docs to Sphinx. - -- Aug 25 2014: Don't discuss native semantics. There is only Python - semantics. - -- Jul 2 2013: Rewrite to use new syntax. Shift focus to discussing - Python semantics. Add more content, including short discussions of - :ref:`generic-functions` and :ref:`union-types`. diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst new file mode 100644 index 000000000000..ff042b395e99 --- /dev/null +++ b/docs/source/running_mypy.rst @@ -0,0 +1,569 @@ +.. _running-mypy: + +Running mypy and managing imports +================================= + +The :ref:`getting-started` page should have already introduced you +to the basics of how to run mypy -- pass in the files and directories +you want to type check via the command line:: + + $ mypy foo.py bar.py some_directory + +This page discusses in more detail how exactly to specify what files +you want mypy to type check, how mypy discovers imported modules, +and recommendations on how to handle any issues you may encounter +along the way. + +If you are interested in learning about how to configure the +actual way mypy type checks your code, see our +:ref:`command-line` guide. + + +.. _specifying-code-to-be-checked: + +Specifying code to be checked +***************************** + +Mypy lets you specify what files it should type check in several different ways. + +1. First, you can pass in paths to Python files and directories you + want to type check. For example:: + + $ mypy file_1.py foo/file_2.py file_3.pyi some/directory + + The above command tells mypy it should type check all of the provided + files together. In addition, mypy will recursively type check the + entire contents of any provided directories. + + For more details about how exactly this is done, see + :ref:`Mapping file paths to modules `. + +2. Second, you can use the :option:`-m ` flag (long form: :option:`--module `) to + specify a module name to be type checked. The name of a module + is identical to the name you would use to import that module + within a Python program. For example, running:: + + $ mypy -m html.parser + + ...will type check the module ``html.parser`` (this happens to be + a library stub). + + Mypy will use an algorithm very similar to the one Python uses to + find where modules and imports are located on the file system. + For more details, see :ref:`finding-imports`. + +3. Third, you can use the :option:`-p ` (long form: :option:`--package `) flag to + specify a package to be (recursively) type checked. This flag + is almost identical to the :option:`-m ` flag except that if you give it + a package name, mypy will recursively type check all submodules + and subpackages of that package. For example, running:: + + $ mypy -p html + + ...will type check the entire ``html`` package (of library stubs). + In contrast, if we had used the :option:`-m ` flag, mypy would have type + checked just ``html``'s ``__init__.py`` file and anything imported + from there. + + Note that we can specify multiple packages and modules on the + command line. For example:: + + $ mypy --package p.a --package p.b --module c + +4. Fourth, you can also instruct mypy to directly type check small + strings as programs by using the :option:`-c ` (long form: :option:`--command `) + flag. For example:: + + $ mypy -c 'x = [1, 2]; print(x())' + + ...will type check the above string as a mini-program (and in this case, + will report that ``list[int]`` is not callable). + +You can also use the :confval:`files` option in your :file:`mypy.ini` file to specify which +files to check, in which case you can simply run ``mypy`` with no arguments. + + +Reading a list of files from a file +*********************************** + +Finally, any command-line argument starting with ``@`` reads additional +command-line arguments from the file following the ``@`` character. +This is primarily useful if you have a file containing a list of files +that you want to be type-checked: instead of using shell syntax like:: + + $ mypy $(cat file_of_files.txt) + +you can use this instead:: + + $ mypy @file_of_files.txt + +This file can technically also contain any command line flag, not +just file paths. However, if you want to configure many different +flags, the recommended approach is to use a +:ref:`configuration file ` instead. + + +.. _mapping-paths-to-modules: + +Mapping file paths to modules +***************************** + +One of the main ways you can tell mypy what to type check +is by providing mypy a list of paths. For example:: + + $ mypy file_1.py foo/file_2.py file_3.pyi some/directory + +This section describes how exactly mypy maps the provided paths +to modules to type check. + +- Mypy will check all paths provided that correspond to files. + +- Mypy will recursively discover and check all files ending in ``.py`` or + ``.pyi`` in directory paths provided, after accounting for + :option:`--exclude `. + +- For each file to be checked, mypy will attempt to associate the file (e.g. + ``project/foo/bar/baz.py``) with a fully qualified module name (e.g. + ``foo.bar.baz``). The directory the package is in (``project``) is then + added to mypy's module search paths. + +How mypy determines fully qualified module names depends on if the options +:option:`--no-namespace-packages ` and +:option:`--explicit-package-bases ` are set. + +1. If :option:`--no-namespace-packages ` is set, + mypy will rely solely upon the presence of ``__init__.py[i]`` files to + determine the fully qualified module name. That is, mypy will crawl up the + directory tree for as long as it continues to find ``__init__.py`` (or + ``__init__.pyi``) files. + + For example, if your directory tree consists of ``pkg/subpkg/mod.py``, mypy + would require ``pkg/__init__.py`` and ``pkg/subpkg/__init__.py`` to exist in + order correctly associate ``mod.py`` with ``pkg.subpkg.mod`` + +2. The default case. If :option:`--namespace-packages ` is on, but :option:`--explicit-package-bases ` is off, mypy will allow for the possibility that + directories without ``__init__.py[i]`` are packages. Specifically, mypy will + look at all parent directories of the file and use the location of the + highest ``__init__.py[i]`` in the directory tree to determine the top-level + package. + + For example, say your directory tree consists solely of ``pkg/__init__.py`` + and ``pkg/a/b/c/d/mod.py``. When determining ``mod.py``'s fully qualified + module name, mypy will look at ``pkg/__init__.py`` and conclude that the + associated module name is ``pkg.a.b.c.d.mod``. + +3. You'll notice that the above case still relies on ``__init__.py``. If + you can't put an ``__init__.py`` in your top-level package, but still wish to + pass paths (as opposed to packages or modules using the ``-p`` or ``-m`` + flags), :option:`--explicit-package-bases ` + provides a solution. + + With :option:`--explicit-package-bases `, mypy + will locate the nearest parent directory that is a member of the ``MYPYPATH`` + environment variable, the :confval:`mypy_path` config or is the current + working directory. Mypy will then use the relative path to determine the + fully qualified module name. + + For example, say your directory tree consists solely of + ``src/namespace_pkg/mod.py``. If you run the following command, mypy + will correctly associate ``mod.py`` with ``namespace_pkg.mod``:: + + $ MYPYPATH=src mypy --namespace-packages --explicit-package-bases . + +If you pass a file not ending in ``.py[i]``, the module name assumed is +``__main__`` (matching the behavior of the Python interpreter), unless +:option:`--scripts-are-modules ` is passed. + +Passing :option:`-v ` will show you the files and associated module +names that mypy will check. + + +How mypy handles imports +************************ + +When mypy encounters an ``import`` statement, it will first +:ref:`attempt to locate ` that module +or type stubs for that module in the file system. Mypy will then +type check the imported module. There are three different outcomes +of this process: + +1. Mypy is unable to follow the import: the module either does not + exist, or is a third party library that does not use type hints. + +2. Mypy is able to follow and type check the import, but you did + not want mypy to type check that module at all. + +3. Mypy is able to successfully both follow and type check the + module, and you want mypy to type check that module. + +The third outcome is what mypy will do in the ideal case. The following +sections will discuss what to do in the other two cases. + +.. _ignore-missing-imports: +.. _fix-missing-imports: + +Missing imports +*************** + +When you import a module, mypy may report that it is unable to follow +the import. This can cause errors that look like the following: + +.. code-block:: text + + main.py:1: error: Skipping analyzing 'django': module is installed, but missing library stubs or py.typed marker + main.py:2: error: Library stubs not installed for "requests" + main.py:3: error: Cannot find implementation or library stub for module named "this_module_does_not_exist" + +If you get any of these errors on an import, mypy will assume the type of that +module is ``Any``, the dynamic type. This means attempting to access any +attribute of the module will automatically succeed: + +.. code-block:: python + + # Error: Cannot find implementation or library stub for module named 'does_not_exist' + import does_not_exist + + # But this type checks, and x will have type 'Any' + x = does_not_exist.foobar() + +This can result in mypy failing to warn you about errors in your code. Since +operations on ``Any`` result in ``Any``, these dynamic types can propagate +through your code, making type checking less effective. See +:ref:`dynamic-typing` for more information. + +The next sections describe what each of these errors means and recommended next steps; scroll to +the section that matches your error. + + +Missing library stubs or py.typed marker +---------------------------------------- + +If you are getting a ``Skipping analyzing X: module is installed, but missing library stubs or py.typed marker``, +error, this means mypy was able to find the module you were importing, but no +corresponding type hints. + +Mypy will not try inferring the types of any 3rd party libraries you have installed +unless they either have declared themselves to be +:ref:`PEP 561 compliant stub package ` (e.g. with a ``py.typed`` file) or have registered +themselves on `typeshed `_, the repository +of types for the standard library and some 3rd party libraries. + +If you are getting this error, try to obtain type hints for the library you're using: + +1. Upgrading the version of the library you're using, in case a newer version + has started to include type hints. + +2. Searching to see if there is a :ref:`PEP 561 compliant stub package ` + corresponding to your third party library. Stub packages let you install + type hints independently from the library itself. + + For example, if you want type hints for the ``django`` library, you can + install the `django-stubs `_ package. + +3. :ref:`Writing your own stub files ` containing type hints for + the library. You can point mypy at your type hints either by passing + them in via the command line, by using the :confval:`files` or :confval:`mypy_path` + config file options, or by + adding the location to the ``MYPYPATH`` environment variable. + + These stub files do not need to be complete! A good strategy is to use + :ref:`stubgen `, a program that comes bundled with mypy, to generate a first + rough draft of the stubs. You can then iterate on just the parts of the + library you need. + + If you want to share your work, you can try contributing your stubs back + to the library -- see our documentation on creating + :ref:`PEP 561 compliant packages `. + +4. Force mypy to analyze the library as best as it can (as if the library provided + a ``py.typed`` file), despite it likely missing any type annotations. In general, + the quality of type checking will be poor and mypy may have issues when + analyzing code not designed to be type checked. + + You can do this via setting the + :option:`--follow-untyped-imports ` + command line flag or :confval:`follow_untyped_imports` config file option to True. + This option can be specified on a per-module basis as well:: + + # mypy.ini + [mypy-untyped_package.*] + follow_untyped_imports = True + + # pyproject.toml + [[tool.mypy.overrides]] + module = ["untyped_package.*"] + follow_untyped_imports = true + +If you are unable to find any existing type hints nor have time to write your +own, you can instead *suppress* the errors. + +All this will do is make mypy stop reporting an error on the line containing the +import: the imported module will continue to be of type ``Any``, and mypy may +not catch errors in its use. + +1. To suppress a *single* missing import error, add a ``# type: ignore`` at the end of the + line containing the import. + +2. To suppress *all* missing import errors from a single library, add + a per-module section to your :ref:`mypy config file ` setting + :confval:`ignore_missing_imports` to True for that library. For example, + suppose your codebase + makes heavy use of an (untyped) library named ``foobar``. You can silence + all import errors associated with that library and that library alone by + adding the following section to your config file:: + + # mypy.ini + [mypy-foobar.*] + ignore_missing_imports = True + + # pyproject.toml + [[tool.mypy.overrides]] + module = ["foobar.*"] + ignore_missing_imports = true + + Note: this option is equivalent to adding a ``# type: ignore`` to every + import of ``foobar`` in your codebase. For more information, see the + documentation about configuring + :ref:`import discovery ` in config files. + The ``.*`` after ``foobar`` will ignore imports of ``foobar`` modules + and subpackages in addition to the ``foobar`` top-level package namespace. + +3. To suppress *all* missing import errors for *all* untyped libraries + in your codebase, use :option:`--disable-error-code=import-untyped `. + See :ref:`code-import-untyped` for more details on this error code. + + You can also set :confval:`disable_error_code`, like so:: + + # mypy.ini + [mypy] + disable_error_code = import-untyped + + # pyproject.toml + [tool.mypy] + disable_error_code = ["import-untyped"] + + You can also set the :option:`--ignore-missing-imports ` + command line flag or set the :confval:`ignore_missing_imports` config file + option to True in the *global* section of your mypy config file. We + recommend avoiding ``--ignore-missing-imports`` if possible: it's equivalent + to adding a ``# type: ignore`` to all unresolved imports in your codebase. + + +Library stubs not installed +--------------------------- + +If mypy can't find stubs for a third-party library, and it knows that stubs exist for +the library, you will get a message like this: + +.. code-block:: text + + main.py:1: error: Library stubs not installed for "yaml" + main.py:1: note: Hint: "python3 -m pip install types-PyYAML" + main.py:1: note: (or run "mypy --install-types" to install all missing stub packages) + +You can resolve the issue by running the suggested pip commands. +If you're running mypy in CI, you can ensure the presence of any stub packages +you need the same as you would any other test dependency, e.g. by adding them to +the appropriate ``requirements.txt`` file. + +Alternatively, add the :option:`--install-types ` +to your mypy command to install all known missing stubs: + +.. code-block:: text + + mypy --install-types + +This is slower than explicitly installing stubs, since it effectively +runs mypy twice -- the first time to find the missing stubs, and +the second time to type check your code properly after mypy has +installed the stubs. It also can make controlling stub versions harder, +resulting in less reproducible type checking. + +By default, :option:`--install-types ` shows a confirmation prompt. +Use :option:`--non-interactive ` to install all suggested +stub packages without asking for confirmation *and* type check your code: + +If you've already installed the relevant third-party libraries in an environment +other than the one mypy is running in, you can use :option:`--python-executable +` flag to point to the Python executable for that +environment, and mypy will find packages installed for that Python executable. + +If you've installed the relevant stub packages and are still getting this error, +see the :ref:`section below `. + +.. _missing-type-hints-for-third-party-library: + +Cannot find implementation or library stub +------------------------------------------ + +If you are getting a ``Cannot find implementation or library stub for module`` +error, this means mypy was not able to find the module you are trying to +import, whether it comes bundled with type hints or not. If you are getting +this error, try: + +1. Making sure your import does not contain a typo. + +2. If the module is a third party library, making sure that mypy is able + to find the interpreter containing the installed library. + + For example, if you are running your code in a virtualenv, make sure + to install and use mypy within the virtualenv. Alternatively, if you + want to use a globally installed mypy, set the + :option:`--python-executable ` command + line flag to point the Python interpreter containing your installed + third party packages. + + You can confirm that you are running mypy from the environment you expect + by running it like ``python -m mypy ...``. You can confirm that you are + installing into the environment you expect by running pip like + ``python -m pip ...``. + +3. Reading the :ref:`finding-imports` section below to make sure you + understand how exactly mypy searches for and finds modules and modify + how you're invoking mypy accordingly. + +4. Directly specifying the directory containing the module you want to + type check from the command line, by using the :confval:`mypy_path` + or :confval:`files` config file options, + or by using the ``MYPYPATH`` environment variable. + + Note: if the module you are trying to import is actually a *submodule* of + some package, you should specify the directory containing the *entire* package. + For example, suppose you are trying to add the module ``foo.bar.baz`` + which is located at ``~/foo-project/src/foo/bar/baz.py``. In this case, + you must run ``mypy ~/foo-project/src`` (or set the ``MYPYPATH`` to + ``~/foo-project/src``). + +.. _finding-imports: + +How imports are found +********************* + +When mypy encounters an ``import`` statement or receives module +names from the command line via the :option:`--module ` or :option:`--package ` +flags, mypy tries to find the module on the file system similar +to the way Python finds it. However, there are some differences. + +First, mypy has its own search path. +This is computed from the following items: + +- The ``MYPYPATH`` environment variable + (a list of directories, colon-separated on UNIX systems, semicolon-separated on Windows). +- The :confval:`mypy_path` config file option. +- The directories containing the sources given on the command line + (see :ref:`Mapping file paths to modules `). +- The installed packages marked as safe for type checking (see + :ref:`PEP 561 support `) +- The relevant directories of the + `typeshed `_ repo. + +.. note:: + + You cannot point to a stub-only package (:pep:`561`) via the ``MYPYPATH``, it must be + installed (see :ref:`PEP 561 support `) + +Second, mypy searches for stub files in addition to regular Python files +and packages. +The rules for searching for a module ``foo`` are as follows: + +- The search looks in each of the directories in the search path + (see above) until a match is found. +- If a package named ``foo`` is found (i.e. a directory + ``foo`` containing an ``__init__.py`` or ``__init__.pyi`` file) + that's a match. +- If a stub file named ``foo.pyi`` is found, that's a match. +- If a Python module named ``foo.py`` is found, that's a match. + +These matches are tried in order, so that if multiple matches are found +in the same directory on the search path +(e.g. a package and a Python file, or a stub file and a Python file) +the first one in the above list wins. + +In particular, if a Python file and a stub file are both present in the +same directory on the search path, only the stub file is used. +(However, if the files are in different directories, the one found +in the earlier directory is used.) + +Setting :confval:`mypy_path`/``MYPYPATH`` is mostly useful in the case +where you want to try running mypy against multiple distinct +sets of files that happen to share some common dependencies. + +For example, if you have multiple projects that happen to be +using the same set of work-in-progress stubs, it could be +convenient to just have your ``MYPYPATH`` point to a single +directory containing the stubs. + +.. _follow-imports: + +Following imports +***************** + +Mypy is designed to :ref:`doggedly follow all imports `, +even if the imported module is not a file you explicitly wanted mypy to check. + +For example, suppose we have two modules ``mycode.foo`` and ``mycode.bar``: +the former has type hints and the latter does not. We run +:option:`mypy -m mycode.foo ` and mypy discovers that ``mycode.foo`` imports +``mycode.bar``. + +How do we want mypy to type check ``mycode.bar``? Mypy's behaviour here is +configurable -- although we **strongly recommend** using the default -- +by using the :option:`--follow-imports ` flag. This flag +accepts one of four string values: + +- ``normal`` (the default, recommended) follows all imports normally and + type checks all top level code (as well as the bodies of all + functions and methods with at least one type annotation in + the signature). + +- ``silent`` behaves in the same way as ``normal`` but will + additionally *suppress* any error messages. + +- ``skip`` will *not* follow imports and instead will silently + replace the module (and *anything imported from it*) with an + object of type ``Any``. + +- ``error`` behaves in the same way as ``skip`` but is not quite as + silent -- it will flag the import as an error, like this:: + + main.py:1: note: Import of "mycode.bar" ignored + main.py:1: note: (Using --follow-imports=error, module not passed on command line) + +If you are starting a new codebase and plan on using type hints from +the start, we **recommend** you use either :option:`--follow-imports=normal ` +(the default) or :option:`--follow-imports=error `. Either option will help +make sure you are not skipping checking any part of your codebase by +accident. + +If you are planning on adding type hints to a large, existing code base, +we recommend you start by trying to make your entire codebase (including +files that do not use type hints) pass under :option:`--follow-imports=normal `. +This is usually not too difficult to do: mypy is designed to report as +few error messages as possible when it is looking at unannotated code. + +Only if doing this is intractable, try passing mypy just the files +you want to type check and using :option:`--follow-imports=silent `. +Even if mypy is unable to perfectly type check a file, it can still glean some +useful information by parsing it (for example, understanding what methods +a given object has). See :ref:`existing-code` for more recommendations. + +Adjusting import following behaviour is often most useful when restricted to +specific modules. This can be accomplished by setting a per-module +:confval:`follow_imports` config option. + +.. warning:: + + We do not recommend using ``follow_imports=skip`` unless you're really sure + you know what you are doing. This option greatly restricts the analysis mypy + can perform and you will lose a lot of the benefits of type checking. + + This is especially true at the global level. Setting a per-module + ``follow_imports=skip`` for a specific problematic module can be + useful without causing too much harm. + +.. note:: + + If you're looking to resolve import errors related to libraries, try following + the advice in :ref:`fix-missing-imports` before messing with ``follow_imports``. diff --git a/docs/source/runtime_troubles.rst b/docs/source/runtime_troubles.rst new file mode 100644 index 000000000000..d039db30f3fa --- /dev/null +++ b/docs/source/runtime_troubles.rst @@ -0,0 +1,363 @@ +.. _runtime_troubles: + +Annotation issues at runtime +============================ + +Idiomatic use of type annotations can sometimes run up against what a given +version of Python considers legal code. This section describes these scenarios +and explains how to get your code running again. Generally speaking, we have +three tools at our disposal: + +* Use of ``from __future__ import annotations`` (:pep:`563`) + (this behaviour may eventually be made the default in a future Python version) +* Use of string literal types or type comments +* Use of ``typing.TYPE_CHECKING`` + +We provide a description of these before moving onto discussion of specific +problems you may encounter. + +.. _string-literal-types: + +String literal types and type comments +-------------------------------------- + +Mypy lets you add type annotations using the (now deprecated) ``# type:`` +type comment syntax. These were required with Python versions older than 3.6, +since they didn't support type annotations on variables. Example: + +.. code-block:: python + + a = 1 # type: int + + def f(x): # type: (int) -> int + return x + 1 + + # Alternative type comment syntax for functions with many arguments + def send_email( + address, # type: Union[str, List[str]] + sender, # type: str + cc, # type: Optional[List[str]] + subject='', + body=None # type: List[str] + ): + # type: (...) -> bool + +Type comments can't cause runtime errors because comments are not evaluated by +Python. + +In a similar way, using string literal types sidesteps the problem of +annotations that would cause runtime errors. + +Any type can be entered as a string literal, and you can combine +string-literal types with non-string-literal types freely: + +.. code-block:: python + + def f(a: list['A']) -> None: ... # OK, prevents NameError since A is defined later + def g(n: 'int') -> None: ... # Also OK, though not useful + + class A: pass + +String literal types are never needed in ``# type:`` comments and :ref:`stub files `. + +String literal types must be defined (or imported) later *in the same module*. +They cannot be used to leave cross-module references unresolved. (For dealing +with import cycles, see :ref:`import-cycles`.) + +.. _future-annotations: + +Future annotations import (PEP 563) +----------------------------------- + +Many of the issues described here are caused by Python trying to evaluate +annotations. Future Python versions (potentially Python 3.14) will by default no +longer attempt to evaluate function and variable annotations. This behaviour is +made available in Python 3.7 and later through the use of +``from __future__ import annotations``. + +This can be thought of as automatic string literal-ification of all function and +variable annotations. Note that function and variable annotations are still +required to be valid Python syntax. For more details, see :pep:`563`. + +.. note:: + + Even with the ``__future__`` import, there are some scenarios that could + still require string literals or result in errors, typically involving use + of forward references or generics in: + + * :ref:`type aliases ` not defined using the ``type`` statement; + * :ref:`type narrowing `; + * type definitions (see :py:class:`~typing.TypeVar`, :py:class:`~typing.NewType`, :py:class:`~typing.NamedTuple`); + * base classes. + + .. code-block:: python + + # base class example + from __future__ import annotations + + class A(tuple['B', 'C']): ... # String literal types needed here + class B: ... + class C: ... + +.. warning:: + + Some libraries may have use cases for dynamic evaluation of annotations, for + instance, through use of ``typing.get_type_hints`` or ``eval``. If your + annotation would raise an error when evaluated (say by using :pep:`604` + syntax with Python 3.9), you may need to be careful when using such + libraries. + +.. _typing-type-checking: + +typing.TYPE_CHECKING +-------------------- + +The :py:mod:`typing` module defines a :py:data:`~typing.TYPE_CHECKING` constant +that is ``False`` at runtime but treated as ``True`` while type checking. + +Since code inside ``if TYPE_CHECKING:`` is not executed at runtime, it provides +a convenient way to tell mypy something without the code being evaluated at +runtime. This is most useful for resolving :ref:`import cycles `. + +.. _forward-references: + +Class name forward references +----------------------------- + +Python does not allow references to a class object before the class is +defined (aka forward reference). Thus this code does not work as expected: + +.. code-block:: python + + def f(x: A) -> None: ... # NameError: name "A" is not defined + class A: ... + +Starting from Python 3.7, you can add ``from __future__ import annotations`` to +resolve this, as discussed earlier: + +.. code-block:: python + + from __future__ import annotations + + def f(x: A) -> None: ... # OK + class A: ... + +For Python 3.6 and below, you can enter the type as a string literal or type comment: + +.. code-block:: python + + def f(x: 'A') -> None: ... # OK + + # Also OK + def g(x): # type: (A) -> None + ... + + class A: ... + +Of course, instead of using future annotations import or string literal types, +you could move the function definition after the class definition. This is not +always desirable or even possible, though. + +.. _import-cycles: + +Import cycles +------------- + +An import cycle occurs where module A imports module B and module B +imports module A (perhaps indirectly, e.g. ``A -> B -> C -> A``). +Sometimes in order to add type annotations you have to add extra +imports to a module and those imports cause cycles that didn't exist +before. This can lead to errors at runtime like: + +.. code-block:: text + + ImportError: cannot import name 'b' from partially initialized module 'A' (most likely due to a circular import) + +If those cycles do become a problem when running your program, there's a trick: +if the import is only needed for type annotations and you're using a) the +:ref:`future annotations import`, or b) string literals or type +comments for the relevant annotations, you can write the imports inside ``if +TYPE_CHECKING:`` so that they are not executed at runtime. Example: + +File ``foo.py``: + +.. code-block:: python + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + import bar + + def listify(arg: 'bar.BarClass') -> 'list[bar.BarClass]': + return [arg] + +File ``bar.py``: + +.. code-block:: python + + from foo import listify + + class BarClass: + def listifyme(self) -> 'list[BarClass]': + return listify(self) + +.. _not-generic-runtime: + +Using classes that are generic in stubs but not at runtime +---------------------------------------------------------- + +Some classes are declared as :ref:`generic` in stubs, but not +at runtime. + +In Python 3.8 and earlier, there are several examples within the standard library, +for instance, :py:class:`os.PathLike` and :py:class:`queue.Queue`. Subscripting +such a class will result in a runtime error: + +.. code-block:: python + + from queue import Queue + + class Tasks(Queue[str]): # TypeError: 'type' object is not subscriptable + ... + + results: Queue[int] = Queue() # TypeError: 'type' object is not subscriptable + +To avoid errors from use of these generics in annotations, just use the +:ref:`future annotations import` (or string literals or type +comments for Python 3.6 and below). + +To avoid errors when inheriting from these classes, things are a little more +complicated and you need to use :ref:`typing.TYPE_CHECKING +`: + +.. code-block:: python + + from typing import TYPE_CHECKING + from queue import Queue + + if TYPE_CHECKING: + BaseQueue = Queue[str] # this is only processed by mypy + else: + BaseQueue = Queue # this is not seen by mypy but will be executed at runtime + + class Tasks(BaseQueue): # OK + ... + + task_queue: Tasks + reveal_type(task_queue.get()) # Reveals str + +If your subclass is also generic, you can use the following (using the +legacy syntax for generic classes): + +.. code-block:: python + + from typing import TYPE_CHECKING, TypeVar, Generic + from queue import Queue + + _T = TypeVar("_T") + if TYPE_CHECKING: + class _MyQueueBase(Queue[_T]): pass + else: + class _MyQueueBase(Generic[_T], Queue): pass + + class MyQueue(_MyQueueBase[_T]): pass + + task_queue: MyQueue[str] + reveal_type(task_queue.get()) # Reveals str + +In Python 3.9 and later, we can just inherit directly from ``Queue[str]`` or ``Queue[T]`` +since its :py:class:`queue.Queue` implements :py:meth:`~object.__class_getitem__`, so +the class object can be subscripted at runtime. You may still encounter issues (even if +you use a recent Python version) when subclassing generic classes defined in third-party +libraries if types are generic only in stubs. + +Using types defined in stubs but not at runtime +----------------------------------------------- + +Sometimes stubs that you're using may define types you wish to re-use that do +not exist at runtime. Importing these types naively will cause your code to fail +at runtime with ``ImportError`` or ``ModuleNotFoundError``. Similar to previous +sections, these can be dealt with by using :ref:`typing.TYPE_CHECKING +`: + +.. code-block:: python + + from __future__ import annotations + from typing import TYPE_CHECKING + if TYPE_CHECKING: + from _typeshed import SupportsRichComparison + + def f(x: SupportsRichComparison) -> None + +The ``from __future__ import annotations`` is required to avoid +a ``NameError`` when using the imported symbol. +For more information and caveats, see the section on +:ref:`future annotations `. + +.. _generic-builtins: + +Using generic builtins +---------------------- + +Starting with Python 3.9 (:pep:`585`), the type objects of many collections in +the standard library support subscription at runtime. This means that you no +longer have to import the equivalents from :py:mod:`typing`; you can simply use +the built-in collections or those from :py:mod:`collections.abc`: + +.. code-block:: python + + from collections.abc import Sequence + x: list[str] + y: dict[int, str] + z: Sequence[str] = x + +There is limited support for using this syntax in Python 3.7 and later as well: +if you use ``from __future__ import annotations``, mypy will understand this +syntax in annotations. However, since this will not be supported by the Python +interpreter at runtime, make sure you're aware of the caveats mentioned in the +notes at :ref:`future annotations import`. + +Using X | Y syntax for Unions +----------------------------- + +Starting with Python 3.10 (:pep:`604`), you can spell union types as +``x: int | str``, instead of ``x: typing.Union[int, str]``. + +There is limited support for using this syntax in Python 3.7 and later as well: +if you use ``from __future__ import annotations``, mypy will understand this +syntax in annotations, string literal types, type comments and stub files. +However, since this will not be supported by the Python interpreter at runtime +(if evaluated, ``int | str`` will raise ``TypeError: unsupported operand type(s) +for |: 'type' and 'type'``), make sure you're aware of the caveats mentioned in +the notes at :ref:`future annotations import`. + +Using new additions to the typing module +---------------------------------------- + +You may find yourself wanting to use features added to the :py:mod:`typing` +module in earlier versions of Python than the addition, for example, using any +of ``Literal``, ``Protocol``, ``TypedDict`` with Python 3.6. + +The easiest way to do this is to install and use the ``typing_extensions`` +package from PyPI for the relevant imports, for example: + +.. code-block:: python + + from typing_extensions import Literal + x: Literal["open", "close"] + +If you don't want to rely on ``typing_extensions`` being installed on newer +Pythons, you could alternatively use: + +.. code-block:: python + + import sys + if sys.version_info >= (3, 8): + from typing import Literal + else: + from typing_extensions import Literal + + x: Literal["open", "close"] + +This plays nicely well with following :pep:`508` dependency specification: +``typing_extensions; python_version<"3.8"`` diff --git a/docs/source/stubgen.rst b/docs/source/stubgen.rst new file mode 100644 index 000000000000..c9e52956379a --- /dev/null +++ b/docs/source/stubgen.rst @@ -0,0 +1,199 @@ +.. _stubgen: + +.. program:: stubgen + +Automatic stub generation (stubgen) +=================================== + +A stub file (see :pep:`484`) contains only type hints for the public +interface of a module, with empty function bodies. Mypy can use a stub +file instead of the real implementation to provide type information +for the module. They are useful for third-party modules whose authors +have not yet added type hints (and when no stubs are available in +typeshed) and C extension modules (which mypy can't directly process). + +Mypy includes the ``stubgen`` tool that can automatically generate +stub files (``.pyi`` files) for Python modules and C extension modules. +For example, consider this source file: + +.. code-block:: python + + from other_module import dynamic + + BORDER_WIDTH = 15 + + class Window: + parent = dynamic() + def __init__(self, width, height): + self.width = width + self.height = height + + def create_empty() -> Window: + return Window(0, 0) + +Stubgen can generate this stub file based on the above file: + +.. code-block:: python + + from typing import Any + + BORDER_WIDTH: int = ... + + class Window: + parent: Any = ... + width: Any = ... + height: Any = ... + def __init__(self, width, height) -> None: ... + + def create_empty() -> Window: ... + +Stubgen generates *draft* stubs. The auto-generated stub files often +require some manual updates, and most types will default to ``Any``. +The stubs will be much more useful if you add more precise type annotations, +at least for the most commonly used functionality. + +The rest of this section documents the command line interface of stubgen. +Run :option:`stubgen --help` for a quick summary of options. + +.. note:: + + The command-line flags may change between releases. + +Specifying what to stub +*********************** + +You can give stubgen paths of the source files for which you want to +generate stubs:: + + $ stubgen foo.py bar.py + +This generates stubs ``out/foo.pyi`` and ``out/bar.pyi``. The default +output directory ``out`` can be overridden with :option:`-o DIR <-o>`. + +You can also pass directories, and stubgen will recursively search +them for any ``.py`` files and generate stubs for all of them:: + + $ stubgen my_pkg_dir + +Alternatively, you can give module or package names using the +:option:`-m` or :option:`-p` options:: + + $ stubgen -m foo -m bar -p my_pkg_dir + +Details of the options: + +.. option:: -m MODULE, --module MODULE + + Generate a stub file for the given module. This flag may be repeated + multiple times. + + Stubgen *will not* recursively generate stubs for any submodules of + the provided module. + +.. option:: -p PACKAGE, --package PACKAGE + + Generate stubs for the given package. This flag maybe repeated + multiple times. + + Stubgen *will* recursively generate stubs for all submodules of + the provided package. This flag is identical to :option:`--module` apart from + this behavior. + +.. note:: + + You can't mix paths and :option:`-m`/:option:`-p` options in the same stubgen + invocation. + +Stubgen applies heuristics to avoid generating stubs for submodules +that include tests or vendored third-party packages. + +Specifying how to generate stubs +******************************** + +By default stubgen will try to import the target modules and packages. +This allows stubgen to use runtime introspection to generate stubs for C +extension modules and to improve the quality of the generated +stubs. By default, stubgen will also use mypy to perform light-weight +semantic analysis of any Python modules. Use the following flags to +alter the default behavior: + +.. option:: --no-import + + Don't try to import modules. Instead only use mypy's normal search mechanism to find + sources. This does not support C extension modules. This flag also disables + runtime introspection functionality, which mypy uses to find the value of + ``__all__``. As result the set of exported imported names in stubs may be + incomplete. This flag is generally only useful when importing a module causes + unwanted side effects, such as the running of tests. Stubgen tries to skip test + modules even without this option, but this does not always work. + +.. option:: --no-analysis + + Don't perform semantic analysis of source files. This may generate + worse stubs -- in particular, some module, class, and function aliases may + be represented as variables with the ``Any`` type. This is generally only + useful if semantic analysis causes a critical mypy error. Does not apply to + C extension modules. Incompatible with :option:`--inspect-mode`. + +.. option:: --inspect-mode + + Import and inspect modules instead of parsing source code. This is the default + behavior for C modules and pyc-only packages. The flag is useful to force + inspection for pure Python modules that make use of dynamically generated + members that would otherwise be omitted when using the default behavior of + code parsing. Implies :option:`--no-analysis` as analysis requires source + code. + +.. option:: --doc-dir PATH + + Try to infer better signatures by parsing .rst documentation in ``PATH``. + This may result in better stubs, but currently it only works for C extension + modules. + +Additional flags +**************** + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --ignore-errors + + If an exception was raised during stub generation, continue to process any + remaining modules instead of immediately failing with an error. + +.. option:: --include-private + + Include definitions that are considered private in stubs (with names such + as ``_foo`` with single leading underscore and no trailing underscores). + +.. option:: --export-less + + Don't export all names imported from other modules within the same package. + Instead, only export imported names that are not referenced in the module + that contains the import. + +.. option:: --include-docstrings + + Include docstrings in stubs. This will add docstrings to Python function and + classes stubs and to C extension function stubs. + +.. option:: --search-path PATH + + Specify module search directories, separated by colons (only used if + :option:`--no-import` is given). + +.. option:: -o PATH, --output PATH + + Change the output directory. By default the stubs are written in the + ``./out`` directory. The output directory will be created if it doesn't + exist. Existing stubs in the output directory will be overwritten without + warning. + +.. option:: -v, --verbose + + Produce more verbose output. + +.. option:: -q, --quiet + + Produce less verbose output. diff --git a/docs/source/stubs.rst b/docs/source/stubs.rst new file mode 100644 index 000000000000..c0a3f8b88111 --- /dev/null +++ b/docs/source/stubs.rst @@ -0,0 +1,132 @@ +.. _stub-files: + +Stub files +========== + +A *stub file* is a file containing a skeleton of the public interface +of that Python module, including classes, variables, functions -- and +most importantly, their types. + +Mypy uses stub files stored in the +`typeshed `_ repository to determine +the types of standard library and third-party library functions, classes, +and other definitions. You can also create your own stubs that will be +used to type check your code. + +Creating a stub +*************** + +Here is an overview of how to create a stub file: + +* Write a stub file for the library (or an arbitrary module) and store it as + a ``.pyi`` file in the same directory as the library module. +* Alternatively, put your stubs (``.pyi`` files) in a directory + reserved for stubs (e.g., :file:`myproject/stubs`). In this case you + have to set the environment variable ``MYPYPATH`` to refer to the + directory. For example:: + + $ export MYPYPATH=~/work/myproject/stubs + +Use the normal Python file name conventions for modules, e.g. :file:`csv.pyi` +for module ``csv``. Use a subdirectory with :file:`__init__.pyi` for packages. Note +that :pep:`561` stub-only packages must be installed, and may not be pointed +at through the ``MYPYPATH`` (see :ref:`PEP 561 support `). + +If a directory contains both a ``.py`` and a ``.pyi`` file for the +same module, the ``.pyi`` file takes precedence. This way you can +easily add annotations for a module even if you don't want to modify +the source code. This can be useful, for example, if you use 3rd party +open source libraries in your program (and there are no stubs in +typeshed yet). + +That's it! + +Now you can access the module in mypy programs and type check +code that uses the library. If you write a stub for a library module, +consider making it available for other programmers that use mypy +by contributing it back to the typeshed repo. + +Mypy also ships with two tools for making it easier to create and maintain +stubs: :ref:`stubgen` and :ref:`stubtest`. + +The following sections explain the kinds of type annotations you can use +in your programs and stub files. + +.. note:: + + You may be tempted to point ``MYPYPATH`` to the standard library or + to the :file:`site-packages` directory where your 3rd party packages + are installed. This is almost always a bad idea -- you will likely + get tons of error messages about code you didn't write and that + mypy can't analyze all that well yet, and in the worst case + scenario mypy may crash due to some construct in a 3rd party + package that it didn't expect. + +Stub file syntax +**************** + +Stub files are written in normal Python syntax, but generally +leaving out runtime logic like variable initializers, function bodies, +and default arguments. + +If it is not possible to completely leave out some piece of runtime +logic, the recommended convention is to replace or elide them with ellipsis +expressions (``...``). Each ellipsis below is literally written in the +stub file as three dots: + +.. code-block:: python + + # Variables with annotations do not need to be assigned a value. + # So by convention, we omit them in the stub file. + x: int + + # Function bodies cannot be completely removed. By convention, + # we replace them with `...` instead of the `pass` statement. + def func_1(code: str) -> int: ... + + # We can do the same with default arguments. + def func_2(a: int, b: int = ...) -> int: ... + +.. note:: + + The ellipsis ``...`` is also used with a different meaning in + :ref:`callable types ` and :ref:`tuple types + `. + +Using stub file syntax at runtime +********************************* + +You may also occasionally need to elide actual logic in regular +Python code -- for example, when writing methods in +:ref:`overload variants ` or +:ref:`custom protocols `. + +The recommended style is to use ellipses to do so, just like in +stub files. It is also considered stylistically acceptable to +throw a :py:exc:`NotImplementedError` in cases where the user of the +code may accidentally call functions with no actual logic. + +You can also elide default arguments as long as the function body +also contains no runtime logic: the function body only contains +a single ellipsis, the pass statement, or a ``raise NotImplementedError()``. +It is also acceptable for the function body to contain a docstring. +For example: + +.. code-block:: python + + from typing import Protocol + + class Resource(Protocol): + def ok_1(self, foo: list[str] = ...) -> None: ... + + def ok_2(self, foo: list[str] = ...) -> None: + raise NotImplementedError() + + def ok_3(self, foo: list[str] = ...) -> None: + """Some docstring""" + pass + + # Error: Incompatible default for argument "foo" (default has + # type "ellipsis", argument has type "list[str]") + def not_ok(self, foo: list[str] = ...) -> None: + print(foo) diff --git a/docs/source/stubtest.rst b/docs/source/stubtest.rst new file mode 100644 index 000000000000..59889252f056 --- /dev/null +++ b/docs/source/stubtest.rst @@ -0,0 +1,162 @@ +.. _stubtest: + +.. program:: stubtest + +Automatic stub testing (stubtest) +================================= + +Stub files are files containing type annotations. See +`PEP 484 `_ +for more motivation and details. + +A common problem with stub files is that they tend to diverge from the +actual implementation. Mypy includes the ``stubtest`` tool that can +automatically check for discrepancies between the stubs and the +implementation at runtime. + +What stubtest does and does not do +********************************** + +Stubtest will import your code and introspect your code objects at runtime, for +example, by using the capabilities of the :py:mod:`inspect` module. Stubtest +will then analyse the stub files, and compare the two, pointing out things that +differ between stubs and the implementation at runtime. + +It's important to be aware of the limitations of this comparison. Stubtest will +not make any attempt to statically analyse your actual code and relies only on +dynamic runtime introspection (in particular, this approach means stubtest works +well with extension modules). However, this means that stubtest has limited +visibility; for instance, it cannot tell if a return type of a function is +accurately typed in the stubs. + +For clarity, here are some additional things stubtest can't do: + +* Type check your code -- use ``mypy`` instead +* Generate stubs -- use ``stubgen`` or ``pyright --createstub`` instead +* Generate stubs based on running your application or test suite -- use ``monkeytype`` instead +* Apply stubs to code to produce inline types -- use ``retype`` or ``libcst`` instead + +In summary, stubtest works very well for ensuring basic consistency between +stubs and implementation or to check for stub completeness. It's used to +test Python's official collection of library stubs, +`typeshed `_. + +.. warning:: + + stubtest will import and execute Python code from the packages it checks. + +Example +******* + +Here's a quick example of what stubtest can do: + +.. code-block:: shell + + $ python3 -m pip install mypy + + $ cat library.py + x = "hello, stubtest" + + def foo(x=None): + print(x) + + $ cat library.pyi + x: int + + def foo(x: int) -> None: ... + + $ python3 -m mypy.stubtest library + error: library.foo is inconsistent, runtime argument "x" has a default value but stub argument does not + Stub: at line 3 + def (x: builtins.int) + Runtime: in file ~/library.py:3 + def (x=None) + + error: library.x variable differs from runtime type Literal['hello, stubtest'] + Stub: at line 1 + builtins.int + Runtime: + 'hello, stubtest' + + +Usage +***** + +Running stubtest can be as simple as ``stubtest module_to_check``. +Run :option:`stubtest --help` for a quick summary of options. + +Stubtest must be able to import the code to be checked, so make sure that mypy +is installed in the same environment as the library to be tested. In some +cases, setting ``PYTHONPATH`` can help stubtest find the code to import. + +Similarly, stubtest must be able to find the stubs to be checked. Stubtest +respects the ``MYPYPATH`` environment variable -- consider using this if you +receive a complaint along the lines of "failed to find stubs". + +Note that stubtest requires mypy to be able to analyse stubs. If mypy is unable +to analyse stubs, you may get an error on the lines of "not checking stubs due +to mypy build errors". In this case, you will need to mitigate those errors +before stubtest will run. Despite potential overlap in errors here, stubtest is +not intended as a substitute for running mypy directly. + +If you wish to ignore some of stubtest's complaints, stubtest supports a +pretty handy allowlist system. + +The rest of this section documents the command line interface of stubtest. + +.. option:: --concise + + Makes stubtest's output more concise, one line per error + +.. option:: --ignore-missing-stub + + Ignore errors for stub missing things that are present at runtime + +.. option:: --ignore-positional-only + + Ignore errors for whether an argument should or shouldn't be positional-only + +.. option:: --allowlist FILE + + Use file as an allowlist. Can be passed multiple times to combine multiple + allowlists. Allowlists can be created with --generate-allowlist. Allowlists + support regular expressions. + + The presence of an entry in the allowlist means stubtest will not generate + any errors for the corresponding definition. + +.. option:: --generate-allowlist + + Print an allowlist (to stdout) to be used with --allowlist + + When introducing stubtest to an existing project, this is an easy way to + silence all existing errors. + +.. option:: --ignore-unused-allowlist + + Ignore unused allowlist entries + + Without this option enabled, the default is for stubtest to complain if an + allowlist entry is not necessary for stubtest to pass successfully. + + Note if an allowlist entry is a regex that matches the empty string, + stubtest will never consider it unused. For example, to get + `--ignore-unused-allowlist` behaviour for a single allowlist entry like + ``foo.bar`` you could add an allowlist entry ``(foo\.bar)?``. + This can be useful when an error only occurs on a specific platform. + +.. option:: --mypy-config-file FILE + + Use specified mypy config file to determine mypy plugins and mypy path + +.. option:: --custom-typeshed-dir DIR + + Use the custom typeshed in DIR + +.. option:: --check-typeshed + + Check all stdlib modules in typeshed + +.. option:: --help + + Show a help message :-) diff --git a/docs/source/supported_python_features.rst b/docs/source/supported_python_features.rst index 068dd058881a..77d15a42f75c 100644 --- a/docs/source/supported_python_features.rst +++ b/docs/source/supported_python_features.rst @@ -1,22 +1,20 @@ -Supported Python features and modules -===================================== +Supported Python features +========================= -Lists of supported Python features and standard library modules are -maintained in the mypy wiki: +A list of unsupported Python features is maintained in the mypy wiki: -- `Supported Python features `_ -- `Supported Python modules `_ +- `Unsupported Python features `_ Runtime definition of methods and functions ******************************************* -By default, mypy will not let you redefine functions or methods, and -you can't add functions to a class or module outside its definition -- -but only if this is visible to the type checker. This only affects -static checking, as mypy performs no additional type checking at -runtime. You can easily work around this. For example, you can use -dynamically typed code or values with ``Any`` types, or you can use -``setattr`` or other introspection features. However, you need to be -careful if you decide to do this. If used indiscriminately, you may -have difficulty using static typing effectively, since the type -checker cannot see functions defined at runtime. +By default, mypy will complain if you add a function to a class +or module outside its definition -- but only if this is visible to the +type checker. This only affects static checking, as mypy performs no +additional type checking at runtime. You can easily work around +this. For example, you can use dynamically typed code or values with +``Any`` types, or you can use :py:func:`setattr` or other introspection +features. However, you need to be careful if you decide to do this. If +used indiscriminately, you may have difficulty using static typing +effectively, since the type checker cannot see functions defined at +runtime. diff --git a/docs/source/type_inference_and_annotations.rst b/docs/source/type_inference_and_annotations.rst index 410bc19ecea6..318ca4cd9160 100644 --- a/docs/source/type_inference_and_annotations.rst +++ b/docs/source/type_inference_and_annotations.rst @@ -1,161 +1,296 @@ +.. _type-inference-and-annotations: + Type inference and type annotations =================================== Type inference ************** -The initial assignment defines a variable. If you do not explicitly -specify the type of the variable, mypy infers the type based on the -static type of the value expression: +For most variables, if you do not explicitly specify its type, mypy will +infer the correct type based on what is initially assigned to the variable. .. code-block:: python - i = 1 # Infer type int for i - l = [1, 2] # Infer type List[int] for l + # Mypy will infer the type of these variables, despite no annotations + i = 1 + reveal_type(i) # Revealed type is "builtins.int" + l = [1, 2] + reveal_type(l) # Revealed type is "builtins.list[builtins.int]" -Type inference is bidirectional and takes context into account. For -example, the following is valid: -.. code-block:: python +.. note:: + + Note that mypy will not use type inference in dynamically typed functions + (those without a function type annotation) — every local variable type + defaults to ``Any`` in such functions. For more details, see :ref:`dynamic-typing`. - def f(l: List[object]) -> None: - l = [1, 2] # Infer type List[object] for [1, 2] + .. code-block:: python -In an assignment, the type context is determined by the assignment -target. In this case this is ``l``, which has the type -``List[object]``. The value expression ``[1, 2]`` is type checked in -this context and given the type ``List[object]``. In the previous -example we introduced a new variable ``l``, and here the type context -was empty. + def untyped_function(): + i = 1 + reveal_type(i) # Revealed type is "Any" + # 'reveal_type' always outputs 'Any' in unchecked functions -Note that the following is not valid, since ``List[int]`` is not -compatible with ``List[object]``: +.. _explicit-var-types: + +Explicit types for variables +**************************** + +You can override the inferred type of a variable by using a +variable type annotation: .. code-block:: python - def f(l: List[object], k: List[int]) -> None: - l = k # Type check error: incompatible types in assignment + x: int | str = 1 -The reason why the above assignment is disallowed is that allowing the -assignment could result in non-int values stored in a list of ``int``: +Without the type annotation, the type of ``x`` would be just ``int``. We +use an annotation to give it a more general type ``int | str`` (this +type means that the value can be either an ``int`` or a ``str``). + +The best way to think about this is that the type annotation sets the type of +the variable, not the type of the expression. For instance, mypy will complain +about the following code: .. code-block:: python - def f(l: List[object], k: List[int]) -> None: - l = k - l.append('x') - print(k[-1]) # Ouch; a string in List[int] + x: int | str = 1.1 # error: Incompatible types in assignment + # (expression has type "float", variable has type "int | str") -You can still run the above program; it prints ``x``. This illustrates -the fact that static types are used during type checking, but they do -not affect the runtime behavior of programs. You can run programs with -type check failures, which is often very handy when performing a large -refactoring. Thus you can always 'work around' the type system, and it -doesn't really limit what you can do in your program. +.. note:: + + To explicitly override the type of an expression you can use + :py:func:`cast(\, \) `. + See :ref:`casts` for details. + +Note that you can explicitly declare the type of a variable without +giving it an initial value: + +.. code-block:: python + + # We only unpack two values, so there's no right-hand side value + # for mypy to infer the type of "cs" from: + a, b, *cs = 1, 2 # error: Need type annotation for "cs" -Type inference is not used in dynamically typed functions (those -without an explicit return type) — every local variable type defaults -to ``Any``, which is discussed later. + rs: list[int] # no assignment! + p, q, *rs = 1, 2 # OK Explicit types for collections ****************************** The type checker cannot always infer the type of a list or a dictionary. This often arises when creating an empty list or -dictionary and assigning it to a new variable without an explicit -variable type. In these cases you can give the type explicitly using -the type name as a constructor: +dictionary and assigning it to a new variable that doesn't have an explicit +variable type. Here is an example where mypy can't infer the type +without some help: .. code-block:: python - l = List[int]() # Create empty list with type List[int] - d = Dict[str, int]() # Create empty dictionary (str -> int) + l = [] # Error: Need type annotation for "l" -Similarly, you can also give an explicit type when creating an empty set: +In these cases you can give the type explicitly using a type annotation: .. code-block:: python - s = Set[int]() + l: list[int] = [] # Create empty list of int + d: dict[str, int] = {} # Create empty dictionary (str -> int) -Explicit types for variables -**************************** +.. note:: + + Using type arguments (e.g. ``list[int]``) on builtin collections like + :py:class:`list`, :py:class:`dict`, :py:class:`tuple`, and :py:class:`set` + only works in Python 3.9 and later. For Python 3.8 and earlier, you must use + :py:class:`~typing.List` (e.g. ``List[int]``), :py:class:`~typing.Dict`, and + so on. + + +Compatibility of container types +******************************** + +A quick note: container types can sometimes be unintuitive. We'll discuss this +more in :ref:`variance`. For example, the following program generates a mypy error, +because mypy treats ``list[int]`` as incompatible with ``list[object]``: .. code-block:: python - s = Undefined(str) # Declare type of x to be str. - s = 'x' # OK - s = 1 # Type check error + def f(l: list[object], k: list[int]) -> None: + l = k # error: Incompatible types in assignment -The Undefined call evaluates to a special ``Undefined`` object that -raises an exception on any operation: +The reason why the above assignment is disallowed is that allowing the +assignment could result in non-int values stored in a list of ``int``: .. code-block:: python - s = Undefined(str) - if s: # Runtime error: undefined value - print('hello') + def f(l: list[object], k: list[int]) -> None: + l = k + l.append('x') + print(k[-1]) # Ouch; a string in list[int] + +Other container types like :py:class:`dict` and :py:class:`set` behave similarly. + +You can still run the above program; it prints ``x``. This illustrates the fact +that static types do not affect the runtime behavior of programs. You can run +programs with type check failures, which is often very handy when performing a +large refactoring. Thus you can always 'work around' the type system, and it +doesn't really limit what you can do in your program. -You can also override the inferred type of a variable by using a -special comment after an assignment statement: +Context in type inference +************************* + +Type inference is *bidirectional* and takes context into account. + +Mypy will take into account the type of the variable on the left-hand side +of an assignment when inferring the type of the expression on the right-hand +side. For example, the following will type check: .. code-block:: python - x = [] # type: List[int] + def f(l: list[object]) -> None: + l = [1, 2] # Infer type list[object] for [1, 2], not list[int] + + +The value expression ``[1, 2]`` is type checked with the additional +context that it is being assigned to a variable of type ``list[object]``. +This is used to infer the type of the *expression* as ``list[object]``. -Here the ``# type:`` comment applies both to the assignment target, in -this case ``x``, and also the initializer expression, via context. The -above code is equivalent to this: +Declared argument types are also used for type context. In this program +mypy knows that the empty list ``[]`` should have type ``list[int]`` based +on the declared type of ``arg`` in ``foo``: .. code-block:: python - x = List[int]() + def foo(arg: list[int]) -> None: + print('Items:', ''.join(str(a) for a in arg)) -The type checker infers the value of a variable from the initializer, -and if it is an empty collection such as ``[]``, the type is not -well-defined. You can declare the collection type using one of the -above syntax alternatives. + foo([]) # OK -Declaring multiple variable types at a time -******************************************* +However, context only works within a single statement. Here mypy requires +an annotation for the empty list, since the context would only be available +in the following statement: -You can declare more than a single variable at a time. In order to -nicely work with multiple assignment, you must give each variable a -type separately: +.. code-block:: python + + def foo(arg: list[int]) -> None: + print('Items:', ', '.join(arg)) + + a = [] # Error: Need type annotation for "a" + foo(a) + +Working around the issue is easy by adding a type annotation: + +.. code-block:: Python + + ... + a: list[int] = [] # OK + foo(a) + +.. _silencing-type-errors: + +Silencing type errors +********************* + +You might want to disable type checking on specific lines, or within specific +files in your codebase. To do that, you can use a ``# type: ignore`` comment. + +For example, say in its latest update, the web framework you use can now take an +integer argument to ``run()``, which starts it on localhost on that port. +Like so: .. code-block:: python - n, s = Undefined(int), Undefined(str) # Declare an integer and a string - i, found = 0, False # type: int, bool + # Starting app on http://localhost:8000 + app.run(8000) + +However, the devs forgot to update their type annotations for +``run``, so mypy still thinks ``run`` only expects ``str`` types. +This would give you the following error: + +.. code-block:: text + + error: Argument 1 to "run" of "A" has incompatible type "int"; expected "str" -When using the latter form, you can optinally use parentheses around -the types, assignment targets and assigned expression: +If you cannot directly fix the web framework yourself, you can temporarily +disable type checking on that line, by adding a ``# type: ignore``: .. code-block:: python - i, found = 0, False # type: (int, bool) # OK - (i, found) = 0, False # type: int, bool # OK - i, found = (0, False) # type: int, bool # OK - (i, found) = (0, False) # type: (int, bool) # OK + # Starting app on http://localhost:8000 + app.run(8000) # type: ignore -Starred expressions -****************************** +This will suppress any mypy errors that would have raised on that specific line. + +You should probably add some more information on the ``# type: ignore`` comment, +to explain why the ignore was added in the first place. This could be a link to +an issue on the repository responsible for the type stubs, or it could be a +short explanation of the bug. To do that, use this format: + +.. code-block:: python -In most cases, mypy can infer the type of starred expressions from the -right-hand side of an assignment, but not always: + # Starting app on http://localhost:8000 + app.run(8000) # type: ignore # `run()` in v2.0 accepts an `int`, as a port + +Type ignore error codes +----------------------- + +By default, mypy displays an error code for each error: + +.. code-block:: text + + error: "str" has no attribute "trim" [attr-defined] + + +It is possible to add a specific error-code in your ignore comment (e.g. +``# type: ignore[attr-defined]``) to clarify what's being silenced. You can +find more information about error codes :ref:`here `. + +Other ways to silence errors +---------------------------- + +You can get mypy to silence errors about a specific variable by dynamically +typing it with ``Any``. See :ref:`dynamic-typing` for more information. .. code-block:: python - a, *bs = 1, 2, 3 # OK - p, q, *rs = 1, 2 # Error: Type of cs cannot be inferred + from typing import Any + + def f(x: Any, y: str) -> None: + x = 'hello' + x += 1 # OK -On first line, the type of ``bs`` is inferred to be -``List[int]``. However, on the second line, mypy cannot infer the type -of ``rs``, because there is no right-hand side value for ``rs`` to -infer the type from. In cases like these, the starred expression needs -to be annotated with a starred type: +You can ignore all mypy errors in a file by adding a +``# mypy: ignore-errors`` at the top of the file: .. code-block:: python - p, q, *rs = 1, 2 # type: int, int, *List[int] + # mypy: ignore-errors + # This is a test file, skipping type checking in it. + import unittest + ... + +You can also specify per-module configuration options in your :ref:`config-file`. +For example: + +.. code-block:: ini + + # Don't report errors in the 'package_to_fix_later' package + [mypy-package_to_fix_later.*] + ignore_errors = True + + # Disable specific error codes in the 'tests' package + # Also don't require type annotations + [mypy-tests.*] + disable_error_code = var-annotated, has-type + allow_untyped_defs = True + + # Silence import errors from the 'library_missing_types' package + [mypy-library_missing_types.*] + ignore_missing_imports = True + +Finally, adding a ``@typing.no_type_check`` decorator to a class, method or +function causes mypy to avoid type checking that class, method or function +and to treat it as not having any type annotations. + +.. code-block:: python -Here, the type of ``rs`` is set to ``List[int]``. + @typing.no_type_check + def foo() -> str: + return 12345 # No error! diff --git a/docs/source/type_narrowing.rst b/docs/source/type_narrowing.rst new file mode 100644 index 000000000000..697a1519a603 --- /dev/null +++ b/docs/source/type_narrowing.rst @@ -0,0 +1,557 @@ +.. _type-narrowing: + +Type narrowing +============== + +This section is dedicated to several type narrowing +techniques which are supported by mypy. + +Type narrowing is when you convince a type checker that a broader type is actually more specific, for instance, that an object of type ``Shape`` is actually of the narrower type ``Square``. + + +Type narrowing expressions +-------------------------- + +The simplest way to narrow a type is to use one of the supported expressions: + +- :py:func:`isinstance` like in :code:`isinstance(obj, float)` will narrow ``obj`` to have ``float`` type +- :py:func:`issubclass` like in :code:`issubclass(cls, MyClass)` will narrow ``cls`` to be ``Type[MyClass]`` +- :py:class:`type` like in :code:`type(obj) is int` will narrow ``obj`` to have ``int`` type +- :py:func:`callable` like in :code:`callable(obj)` will narrow object to callable type +- :code:`obj is not None` will narrow object to its :ref:`non-optional form ` + +Type narrowing is contextual. For example, based on the condition, mypy will narrow an expression only within an ``if`` branch: + +.. code-block:: python + + def function(arg: object): + if isinstance(arg, int): + # Type is narrowed within the ``if`` branch only + reveal_type(arg) # Revealed type: "builtins.int" + elif isinstance(arg, str) or isinstance(arg, bool): + # Type is narrowed differently within this ``elif`` branch: + reveal_type(arg) # Revealed type: "builtins.str | builtins.bool" + + # Subsequent narrowing operations will narrow the type further + if isinstance(arg, bool): + reveal_type(arg) # Revealed type: "builtins.bool" + + # Back outside of the ``if`` statement, the type isn't narrowed: + reveal_type(arg) # Revealed type: "builtins.object" + +Mypy understands the implications ``return`` or exception raising can have +for what type an object could be: + +.. code-block:: python + + def function(arg: int | str): + if isinstance(arg, int): + return + + # `arg` can't be `int` at this point: + reveal_type(arg) # Revealed type: "builtins.str" + +We can also use ``assert`` to narrow types in the same context: + +.. code-block:: python + + def function(arg: Any): + assert isinstance(arg, int) + reveal_type(arg) # Revealed type: "builtins.int" + +.. note:: + + With :option:`--warn-unreachable ` + narrowing types to some impossible state will be treated as an error. + + .. code-block:: python + + def function(arg: int): + # error: Subclass of "int" and "str" cannot exist: + # would have incompatible method signatures + assert isinstance(arg, str) + + # error: Statement is unreachable + print("so mypy concludes the assert will always trigger") + + Without ``--warn-unreachable`` mypy will simply not check code it deems to be + unreachable. See :ref:`unreachable` for more information. + + .. code-block:: python + + x: int = 1 + assert isinstance(x, str) + reveal_type(x) # Revealed type is "builtins.int" + print(x + '!') # Typechecks with `mypy`, but fails in runtime. + + +issubclass +~~~~~~~~~~ + +Mypy can also use :py:func:`issubclass` +for better type inference when working with types and metaclasses: + +.. code-block:: python + + class MyCalcMeta(type): + @classmethod + def calc(cls) -> int: + ... + + def f(o: object) -> None: + t = type(o) # We must use a variable here + reveal_type(t) # Revealed type is "builtins.type" + + if issubclass(t, MyCalcMeta): # `issubclass(type(o), MyCalcMeta)` won't work + reveal_type(t) # Revealed type is "Type[MyCalcMeta]" + t.calc() # Okay + +callable +~~~~~~~~ + +Mypy knows what types are callable and which ones are not during type checking. +So, we know what ``callable()`` will return. For example: + +.. code-block:: python + + from collections.abc import Callable + + x: Callable[[], int] + + if callable(x): + reveal_type(x) # N: Revealed type is "def () -> builtins.int" + else: + ... # Will never be executed and will raise error with `--warn-unreachable` + +The ``callable`` function can even split union types into +callable and non-callable parts: + +.. code-block:: python + + from collections.abc import Callable + + x: int | Callable[[], int] + + if callable(x): + reveal_type(x) # N: Revealed type is "def () -> builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + +.. _casts: + +Casts +----- + +Mypy supports type casts that are usually used to coerce a statically +typed value to a subtype. Unlike languages such as Java or C#, +however, mypy casts are only used as hints for the type checker, and they +don't perform a runtime type check. Use the function :py:func:`~typing.cast` +to perform a cast: + +.. code-block:: python + + from typing import cast + + o: object = [1] + x = cast(list[int], o) # OK + y = cast(list[str], o) # OK (cast performs no actual runtime check) + +To support runtime checking of casts such as the above, we'd have to check +the types of all list items, which would be very inefficient for large lists. +Casts are used to silence spurious +type checker warnings and give the type checker a little help when it can't +quite understand what is going on. + +.. note:: + + You can use an assertion if you want to perform an actual runtime check: + + .. code-block:: python + + def foo(o: object) -> None: + print(o + 5) # Error: can't add 'object' and 'int' + assert isinstance(o, int) + print(o + 5) # OK: type of 'o' is 'int' here + +You don't need a cast for expressions with type ``Any``, or when +assigning to a variable with type ``Any``, as was explained earlier. +You can also use ``Any`` as the cast target type -- this lets you perform +any operations on the result. For example: + +.. code-block:: python + + from typing import cast, Any + + x = 1 + x.whatever() # Type check error + y = cast(Any, x) + y.whatever() # Type check OK (runtime error) + + +.. _type-guards: + +User-Defined Type Guards +------------------------ + +Mypy supports User-Defined Type Guards (:pep:`647`). + +A type guard is a way for programs to influence conditional +type narrowing employed by a type checker based on runtime checks. + +Basically, a ``TypeGuard`` is a "smart" alias for a ``bool`` type. +Let's have a look at the regular ``bool`` example: + +.. code-block:: python + + def is_str_list(val: list[object]) -> bool: + """Determines whether all objects in the list are strings""" + return all(isinstance(x, str) for x in val) + + def func1(val: list[object]) -> None: + if is_str_list(val): + reveal_type(val) # Reveals list[object] + print(" ".join(val)) # Error: incompatible type + +The same example with ``TypeGuard``: + +.. code-block:: python + + from typing import TypeGuard # use `typing_extensions` for Python 3.9 and below + + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: + """Determines whether all objects in the list are strings""" + return all(isinstance(x, str) for x in val) + + def func1(val: list[object]) -> None: + if is_str_list(val): + reveal_type(val) # list[str] + print(" ".join(val)) # ok + +How does it work? ``TypeGuard`` narrows the first function argument (``val``) +to the type specified as the first type parameter (``list[str]``). + +.. note:: + + Narrowing is + `not strict `_. + For example, you can narrow ``str`` to ``int``: + + .. code-block:: python + + def f(value: str) -> TypeGuard[int]: + return True + + Note: since strict narrowing is not enforced, it's easy + to break type safety. + + However, there are many ways a determined or uninformed developer can + subvert type safety -- most commonly by using cast or Any. + If a Python developer takes the time to learn about and implement + user-defined type guards within their code, + it is safe to assume that they are interested in type safety + and will not write their type guard functions in a way + that will undermine type safety or produce nonsensical results. + +Generic TypeGuards +~~~~~~~~~~~~~~~~~~ + +``TypeGuard`` can also work with generic types (Python 3.12 syntax): + +.. code-block:: python + + from typing import TypeGuard # use `typing_extensions` for `python<3.10` + + def is_two_element_tuple[T](val: tuple[T, ...]) -> TypeGuard[tuple[T, T]]: + return len(val) == 2 + + def func(names: tuple[str, ...]): + if is_two_element_tuple(names): + reveal_type(names) # tuple[str, str] + else: + reveal_type(names) # tuple[str, ...] + +TypeGuards with parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Type guard functions can accept extra arguments (Python 3.12 syntax): + +.. code-block:: python + + from typing import TypeGuard # use `typing_extensions` for `python<3.10` + + def is_set_of[T](val: set[Any], type: type[T]) -> TypeGuard[set[T]]: + return all(isinstance(x, type) for x in val) + + items: set[Any] + if is_set_of(items, str): + reveal_type(items) # set[str] + +TypeGuards as methods +~~~~~~~~~~~~~~~~~~~~~ + +A method can also serve as a ``TypeGuard``: + +.. code-block:: python + + class StrValidator: + def is_valid(self, instance: object) -> TypeGuard[str]: + return isinstance(instance, str) + + def func(to_validate: object) -> None: + if StrValidator().is_valid(to_validate): + reveal_type(to_validate) # Revealed type is "builtins.str" + +.. note:: + + Note, that ``TypeGuard`` + `does not narrow `_ + types of ``self`` or ``cls`` implicit arguments. + + If narrowing of ``self`` or ``cls`` is required, + the value can be passed as an explicit argument to a type guard function: + + .. code-block:: python + + class Parent: + def method(self) -> None: + reveal_type(self) # Revealed type is "Parent" + if is_child(self): + reveal_type(self) # Revealed type is "Child" + + class Child(Parent): + ... + + def is_child(instance: Parent) -> TypeGuard[Child]: + return isinstance(instance, Child) + +Assignment expressions as TypeGuards +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes you might need to create a new variable and narrow it +to some specific type at the same time. +This can be achieved by using ``TypeGuard`` together +with `:= operator `_. + +.. code-block:: python + + from typing import TypeGuard # use `typing_extensions` for `python<3.10` + + def is_float(a: object) -> TypeGuard[float]: + return isinstance(a, float) + + def main(a: object) -> None: + if is_float(x := a): + reveal_type(x) # N: Revealed type is 'builtins.float' + reveal_type(a) # N: Revealed type is 'builtins.object' + reveal_type(x) # N: Revealed type is 'builtins.object' + reveal_type(a) # N: Revealed type is 'builtins.object' + +What happens here? + +1. We create a new variable ``x`` and assign a value of ``a`` to it +2. We run ``is_float()`` type guard on ``x`` +3. It narrows ``x`` to be ``float`` in the ``if`` context and does not touch ``a`` + +.. note:: + + The same will work with ``isinstance(x := a, float)`` as well. + +Limitations +----------- + +Mypy's analysis is limited to individual symbols and it will not track +relationships between symbols. For example, in the following code +it's easy to deduce that if :code:`a` is None then :code:`b` must not be, +therefore :code:`a or b` will always be an instance of :code:`C`, +but Mypy will not be able to tell that: + +.. code-block:: python + + class C: + pass + + def f(a: C | None, b: C | None) -> C: + if a is not None or b is not None: + return a or b # Incompatible return value type (got "C | None", expected "C") + return C() + +Tracking these sort of cross-variable conditions in a type checker would add significant complexity +and performance overhead. + +You can use an ``assert`` to convince the type checker, override it with a :ref:`cast ` +or rewrite the function to be slightly more verbose: + +.. code-block:: python + + def f(a: C | None, b: C | None) -> C: + if a is not None: + return a + elif b is not None: + return b + return C() + + +.. _typeis: + +TypeIs +------ + +Mypy supports TypeIs (:pep:`742`). + +A `TypeIs narrowing function `_ +allows you to define custom type checks that can narrow the type of a variable +in `both the if and else `_ +branches of a conditional, similar to how the built-in isinstance() function works. + +TypeIs is new in Python 3.13 — for use in older Python versions, use the backport +from `typing_extensions `_ + +Consider the following example using TypeIs: + +.. code-block:: python + + from typing import TypeIs + + def is_str(x: object) -> TypeIs[str]: + return isinstance(x, str) + + def process(x: int | str) -> None: + if is_str(x): + reveal_type(x) # Revealed type is 'str' + print(x.upper()) # Valid: x is str + else: + reveal_type(x) # Revealed type is 'int' + print(x + 1) # Valid: x is int + +In this example, the function is_str is a type narrowing function +that returns TypeIs[str]. When used in an if statement, x is narrowed +to str in the if branch and to int in the else branch. + +Key points: + + +- The function must accept at least one positional argument. + +- The return type is annotated as ``TypeIs[T]``, where ``T`` is the type you + want to narrow to. + +- The function must return a ``bool`` value. + +- In the ``if`` branch (when the function returns ``True``), the type of the + argument is narrowed to the intersection of its original type and ``T``. + +- In the ``else`` branch (when the function returns ``False``), the type of + the argument is narrowed to the intersection of its original type and the + complement of ``T``. + + +TypeIs vs TypeGuard +~~~~~~~~~~~~~~~~~~~ + +While both TypeIs and TypeGuard allow you to define custom type narrowing +functions, they differ in important ways: + +- **Type narrowing behavior**: TypeIs narrows the type in both the if and else branches, + whereas TypeGuard narrows only in the if branch. + +- **Compatibility requirement**: TypeIs requires that the narrowed type T be + compatible with the input type of the function. TypeGuard does not have this restriction. + +- **Type inference**: With TypeIs, the type checker may infer a more precise type by + combining existing type information with T. + +Here's an example demonstrating the behavior with TypeGuard: + +.. code-block:: python + + from typing import TypeGuard, reveal_type + + def is_str(x: object) -> TypeGuard[str]: + return isinstance(x, str) + + def process(x: int | str) -> None: + if is_str(x): + reveal_type(x) # Revealed type is "builtins.str" + print(x.upper()) # ok: x is str + else: + reveal_type(x) # Revealed type is "Union[builtins.int, builtins.str]" + print(x + 1) # ERROR: Unsupported operand types for + ("str" and "int") [operator] + +Generic TypeIs +~~~~~~~~~~~~~~ + +``TypeIs`` functions can also work with generic types: + +.. code-block:: python + + from typing import TypeVar, TypeIs + + T = TypeVar('T') + + def is_two_element_tuple(val: tuple[T, ...]) -> TypeIs[tuple[T, T]]: + return len(val) == 2 + + def process(names: tuple[str, ...]) -> None: + if is_two_element_tuple(names): + reveal_type(names) # Revealed type is 'tuple[str, str]' + else: + reveal_type(names) # Revealed type is 'tuple[str, ...]' + + +TypeIs with Additional Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +TypeIs functions can accept additional parameters beyond the first. +The type narrowing applies only to the first argument. + +.. code-block:: python + + from typing import Any, TypeVar, reveal_type, TypeIs + + T = TypeVar('T') + + def is_instance_of(val: Any, typ: type[T]) -> TypeIs[T]: + return isinstance(val, typ) + + def process(x: Any) -> None: + if is_instance_of(x, int): + reveal_type(x) # Revealed type is 'int' + print(x + 1) # ok + else: + reveal_type(x) # Revealed type is 'Any' + +TypeIs in Methods +~~~~~~~~~~~~~~~~~ + +A method can also serve as a ``TypeIs`` function. Note that in instance or +class methods, the type narrowing applies to the second parameter +(after ``self`` or ``cls``). + +.. code-block:: python + + class Validator: + def is_valid(self, instance: object) -> TypeIs[str]: + return isinstance(instance, str) + + def process(self, to_validate: object) -> None: + if Validator().is_valid(to_validate): + reveal_type(to_validate) # Revealed type is 'str' + print(to_validate.upper()) # ok: to_validate is str + + +Assignment Expressions with TypeIs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can use the assignment expression operator ``:=`` with ``TypeIs`` to create a new variable and narrow its type simultaneously. + +.. code-block:: python + + from typing import TypeIs, reveal_type + + def is_float(x: object) -> TypeIs[float]: + return isinstance(x, float) + + def main(a: object) -> None: + if is_float(x := a): + reveal_type(x) # Revealed type is 'float' + # x is narrowed to float in this block + print(x + 1.0) diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst new file mode 100644 index 000000000000..bbb10a12abe8 --- /dev/null +++ b/docs/source/typed_dict.rst @@ -0,0 +1,328 @@ +.. _typeddict: + +TypedDict +********* + +Python programs often use dictionaries with string keys to represent objects. +``TypedDict`` lets you give precise types for dictionaries that represent +objects with a fixed schema, such as ``{'id': 1, 'items': ['x']}``. + +Here is a typical example: + +.. code-block:: python + + movie = {'name': 'Blade Runner', 'year': 1982} + +Only a fixed set of string keys is expected (``'name'`` and +``'year'`` above), and each key has an independent value type (``str`` +for ``'name'`` and ``int`` for ``'year'`` above). We've previously +seen the ``dict[K, V]`` type, which lets you declare uniform +dictionary types, where every value has the same type, and arbitrary keys +are supported. This is clearly not a good fit for +``movie`` above. Instead, you can use a ``TypedDict`` to give a precise +type for objects like ``movie``, where the type of each +dictionary value depends on the key: + +.. code-block:: python + + from typing import TypedDict + + Movie = TypedDict('Movie', {'name': str, 'year': int}) + + movie: Movie = {'name': 'Blade Runner', 'year': 1982} + +``Movie`` is a ``TypedDict`` type with two items: ``'name'`` (with type ``str``) +and ``'year'`` (with type ``int``). Note that we used an explicit type +annotation for the ``movie`` variable. This type annotation is +important -- without it, mypy will try to infer a regular, uniform +:py:class:`dict` type for ``movie``, which is not what we want here. + +.. note:: + + If you pass a ``TypedDict`` object as an argument to a function, no + type annotation is usually necessary since mypy can infer the + desired type based on the declared argument type. Also, if an + assignment target has been previously defined, and it has a + ``TypedDict`` type, mypy will treat the assigned value as a ``TypedDict``, + not :py:class:`dict`. + +Now mypy will recognize these as valid: + +.. code-block:: python + + name = movie['name'] # Okay; type of name is str + year = movie['year'] # Okay; type of year is int + +Mypy will detect an invalid key as an error: + +.. code-block:: python + + director = movie['director'] # Error: 'director' is not a valid key + +Mypy will also reject a runtime-computed expression as a key, as +it can't verify that it's a valid key. You can only use string +literals as ``TypedDict`` keys. + +The ``TypedDict`` type object can also act as a constructor. It +returns a normal :py:class:`dict` object at runtime -- a ``TypedDict`` does +not define a new runtime type: + +.. code-block:: python + + toy_story = Movie(name='Toy Story', year=1995) + +This is equivalent to just constructing a dictionary directly using +``{ ... }`` or ``dict(key=value, ...)``. The constructor form is +sometimes convenient, since it can be used without a type annotation, +and it also makes the type of the object explicit. + +Like all types, ``TypedDict``\s can be used as components to build +arbitrarily complex types. For example, you can define nested +``TypedDict``\s and containers with ``TypedDict`` items. +Unlike most other types, mypy uses structural compatibility checking +(or structural subtyping) with ``TypedDict``\s. A ``TypedDict`` object with +extra items is compatible with (a subtype of) a narrower +``TypedDict``, assuming item types are compatible (*totality* also affects +subtyping, as discussed below). + +A ``TypedDict`` object is not a subtype of the regular ``dict[...]`` +type (and vice versa), since :py:class:`dict` allows arbitrary keys to be +added and removed, unlike ``TypedDict``. However, any ``TypedDict`` object is +a subtype of (that is, compatible with) ``Mapping[str, object]``, since +:py:class:`~collections.abc.Mapping` only provides read-only access to the dictionary items: + +.. code-block:: python + + def print_typed_dict(obj: Mapping[str, object]) -> None: + for key, value in obj.items(): + print(f'{key}: {value}') + + print_typed_dict(Movie(name='Toy Story', year=1995)) # OK + +.. note:: + + Unless you are on Python 3.8 or newer (where ``TypedDict`` is available in + standard library :py:mod:`typing` module) you need to install ``typing_extensions`` + using pip to use ``TypedDict``: + + .. code-block:: text + + python3 -m pip install --upgrade typing-extensions + +Totality +-------- + +By default mypy ensures that a ``TypedDict`` object has all the specified +keys. This will be flagged as an error: + +.. code-block:: python + + # Error: 'year' missing + toy_story: Movie = {'name': 'Toy Story'} + +Sometimes you want to allow keys to be left out when creating a +``TypedDict`` object. You can provide the ``total=False`` argument to +``TypedDict(...)`` to achieve this: + +.. code-block:: python + + GuiOptions = TypedDict( + 'GuiOptions', {'language': str, 'color': str}, total=False) + options: GuiOptions = {} # Okay + options['language'] = 'en' + +You may need to use :py:meth:`~dict.get` to access items of a partial (non-total) +``TypedDict``, since indexing using ``[]`` could fail at runtime. +However, mypy still lets use ``[]`` with a partial ``TypedDict`` -- you +just need to be careful with it, as it could result in a :py:exc:`KeyError`. +Requiring :py:meth:`~dict.get` everywhere would be too cumbersome. (Note that you +are free to use :py:meth:`~dict.get` with total ``TypedDict``\s as well.) + +Keys that aren't required are shown with a ``?`` in error messages: + +.. code-block:: python + + # Revealed type is "TypedDict('GuiOptions', {'language'?: builtins.str, + # 'color'?: builtins.str})" + reveal_type(options) + +Totality also affects structural compatibility. You can't use a partial +``TypedDict`` when a total one is expected. Also, a total ``TypedDict`` is not +valid when a partial one is expected. + +Supported operations +-------------------- + +``TypedDict`` objects support a subset of dictionary operations and methods. +You must use string literals as keys when calling most of the methods, +as otherwise mypy won't be able to check that the key is valid. List +of supported operations: + +* Anything included in :py:class:`~collections.abc.Mapping`: + + * ``d[key]`` + * ``key in d`` + * ``len(d)`` + * ``for key in d`` (iteration) + * :py:meth:`d.get(key[, default]) ` + * :py:meth:`d.keys() ` + * :py:meth:`d.values() ` + * :py:meth:`d.items() ` + +* :py:meth:`d.copy() ` +* :py:meth:`d.setdefault(key, default) ` +* :py:meth:`d1.update(d2) ` +* :py:meth:`d.pop(key[, default]) ` (partial ``TypedDict``\s only) +* ``del d[key]`` (partial ``TypedDict``\s only) + +.. note:: + + :py:meth:`~dict.clear` and :py:meth:`~dict.popitem` are not supported since they are unsafe + -- they could delete required ``TypedDict`` items that are not visible to + mypy because of structural subtyping. + +Class-based syntax +------------------ + +An alternative, class-based syntax to define a ``TypedDict`` is supported +in Python 3.6 and later: + +.. code-block:: python + + from typing import TypedDict # "from typing_extensions" in Python 3.7 and earlier + + class Movie(TypedDict): + name: str + year: int + +The above definition is equivalent to the original ``Movie`` +definition. It doesn't actually define a real class. This syntax also +supports a form of inheritance -- subclasses can define additional +items. However, this is primarily a notational shortcut. Since mypy +uses structural compatibility with ``TypedDict``\s, inheritance is not +required for compatibility. Here is an example of inheritance: + +.. code-block:: python + + class Movie(TypedDict): + name: str + year: int + + class BookBasedMovie(Movie): + based_on: str + +Now ``BookBasedMovie`` has keys ``name``, ``year`` and ``based_on``. + +Mixing required and non-required items +-------------------------------------- + +In addition to allowing reuse across ``TypedDict`` types, inheritance also allows +you to mix required and non-required (using ``total=False``) items +in a single ``TypedDict``. Example: + +.. code-block:: python + + class MovieBase(TypedDict): + name: str + year: int + + class Movie(MovieBase, total=False): + based_on: str + +Now ``Movie`` has required keys ``name`` and ``year``, while ``based_on`` +can be left out when constructing an object. A ``TypedDict`` with a mix of required +and non-required keys, such as ``Movie`` above, will only be compatible with +another ``TypedDict`` if all required keys in the other ``TypedDict`` are required keys in the +first ``TypedDict``, and all non-required keys of the other ``TypedDict`` are also non-required keys +in the first ``TypedDict``. + +Read-only items +--------------- + +You can use ``typing.ReadOnly``, introduced in Python 3.13, or +``typing_extensions.ReadOnly`` to mark TypedDict items as read-only (:pep:`705`): + +.. code-block:: python + + from typing import TypedDict + + # Or "from typing ..." on Python 3.13+ + from typing_extensions import ReadOnly + + class Movie(TypedDict): + name: ReadOnly[str] + num_watched: int + + m: Movie = {"name": "Jaws", "num_watched": 1} + m["name"] = "The Godfather" # Error: "name" is read-only + m["num_watched"] += 1 # OK + +A TypedDict with a mutable item can be assigned to a TypedDict +with a corresponding read-only item, and the type of the item can +vary :ref:`covariantly `: + +.. code-block:: python + + class Entry(TypedDict): + name: ReadOnly[str | None] + year: ReadOnly[int] + + class Movie(TypedDict): + name: str + year: int + + def process_entry(i: Entry) -> None: ... + + m: Movie = {"name": "Jaws", "year": 1975} + process_entry(m) # OK + +Unions of TypedDicts +-------------------- + +Since TypedDicts are really just regular dicts at runtime, it is not possible to +use ``isinstance`` checks to distinguish between different variants of a Union of +TypedDict in the same way you can with regular objects. + +Instead, you can use the :ref:`tagged union pattern `. The referenced +section of the docs has a full description with an example, but in short, you will +need to give each TypedDict the same key where each value has a unique +:ref:`Literal type `. Then, check that key to distinguish +between your TypedDicts. + +Inline TypedDict types +---------------------- + +.. note:: + + This is an experimental (non-standard) feature. Use + ``--enable-incomplete-feature=InlineTypedDict`` to enable. + +Sometimes you may want to define a complex nested JSON schema, or annotate +a one-off function that returns a TypedDict. In such cases it may be convenient +to use inline TypedDict syntax. For example: + +.. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + + class Response(TypedDict): + status: int + msg: str + # Using inline syntax here avoids defining two additional TypedDicts. + content: {"items": list[{"key": str, "value": str}]} + +Inline TypedDicts can also by used as targets of type aliases, but due to +ambiguity with a regular variables it is only allowed for (newer) explicit +type alias forms: + +.. code-block:: python + + from typing import TypeAlias + + X = {"a": int, "b": int} # creates a variable with type dict[str, type[int]] + Y: TypeAlias = {"a": int, "b": int} # creates a type alias + type Z = {"a": int, "b": int} # same as above (Python 3.12+ only) + +Also, due to incompatibility with runtime type-checking it is strongly recommended +to *not* use inline syntax in union types. diff --git a/lib-python/3.2/base64.py b/lib-python/3.2/base64.py deleted file mode 100644 index bf5fb1df59f8..000000000000 --- a/lib-python/3.2/base64.py +++ /dev/null @@ -1,411 +0,0 @@ -#! /usr/bin/env python3 - -"""RFC 3548: Base16, Base32, Base64 Data Encodings""" - -# Modified 04-Oct-1995 by Jack Jansen to use binascii module -# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support -# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere - -import re -import struct -import binascii - -from typing import Dict, List, AnyStr, IO - - -__all__ = [ - # Legacy interface exports traditional RFC 1521 Base64 encodings - 'encode', 'decode', 'encodebytes', 'decodebytes', - # Generalized interface for other encodings - 'b64encode', 'b64decode', 'b32encode', 'b32decode', - 'b16encode', 'b16decode', - # Standard Base64 encoding - 'standard_b64encode', 'standard_b64decode', - # Some common Base64 alternatives. As referenced by RFC 3458, see thread - # starting at: - # - # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html - 'urlsafe_b64encode', 'urlsafe_b64decode', - ] - - -bytes_types = (bytes, bytearray) # Types acceptable as binary data - - -def _translate(s: bytes, altchars: Dict[AnyStr, bytes]) -> bytes: - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - translation = bytearray(range(256)) - for k, v in altchars.items(): - translation[ord(k)] = v[0] - return s.translate(translation) - - - -# Base64 encoding/decoding uses binascii - -def b64encode(s: bytes, altchars: bytes = None) -> bytes: - """Encode a byte string using Base64. - - s is the byte string to encode. Optional altchars must be a byte - string of length 2 which specifies an alternative alphabet for the - '+' and '/' characters. This allows an application to - e.g. generate url or filesystem safe Base64 strings. - - The encoded byte string is returned. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - # Strip off the trailing newline - encoded = binascii.b2a_base64(s)[:-1] - if altchars is not None: - if not isinstance(altchars, bytes_types): - raise TypeError("expected bytes, not %s" - % altchars.__class__.__name__) - assert len(altchars) == 2, repr(altchars) - return _translate(encoded, {'+': altchars[0:1], '/': altchars[1:2]}) - return encoded - - -def b64decode(s: bytes, altchars: bytes = None, - validate: bool = False) -> bytes: - """Decode a Base64 encoded byte string. - - s is the byte string to decode. Optional altchars must be a - string of length 2 which specifies the alternative alphabet used - instead of the '+' and '/' characters. - - The decoded string is returned. A binascii.Error is raised if s is - incorrectly padded. - - If validate is False (the default), non-base64-alphabet characters are - discarded prior to the padding check. If validate is True, - non-base64-alphabet characters in the input result in a binascii.Error. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - if altchars is not None: - if not isinstance(altchars, bytes_types): - raise TypeError("expected bytes, not %s" - % altchars.__class__.__name__) - assert len(altchars) == 2, repr(altchars) - s = _translate(s, {chr(altchars[0]): b'+', chr(altchars[1]): b'/'}) - if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s): - raise binascii.Error('Non-base64 digit found') - return binascii.a2b_base64(s) - - -def standard_b64encode(s: bytes) -> bytes: - """Encode a byte string using the standard Base64 alphabet. - - s is the byte string to encode. The encoded byte string is returned. - """ - return b64encode(s) - -def standard_b64decode(s: bytes) -> bytes: - """Decode a byte string encoded with the standard Base64 alphabet. - - s is the byte string to decode. The decoded byte string is - returned. binascii.Error is raised if the input is incorrectly - padded or if there are non-alphabet characters present in the - input. - """ - return b64decode(s) - -def urlsafe_b64encode(s: bytes) -> bytes: - """Encode a byte string using a url-safe Base64 alphabet. - - s is the byte string to encode. The encoded byte string is - returned. The alphabet uses '-' instead of '+' and '_' instead of - '/'. - """ - return b64encode(s, b'-_') - -def urlsafe_b64decode(s: bytes) -> bytes: - """Decode a byte string encoded with the standard Base64 alphabet. - - s is the byte string to decode. The decoded byte string is - returned. binascii.Error is raised if the input is incorrectly - padded or if there are non-alphabet characters present in the - input. - - The alphabet uses '-' instead of '+' and '_' instead of '/'. - """ - return b64decode(s, b'-_') - - - -# Base32 encoding/decoding must be done in Python -_b32alphabet = { - 0: b'A', 9: b'J', 18: b'S', 27: b'3', - 1: b'B', 10: b'K', 19: b'T', 28: b'4', - 2: b'C', 11: b'L', 20: b'U', 29: b'5', - 3: b'D', 12: b'M', 21: b'V', 30: b'6', - 4: b'E', 13: b'N', 22: b'W', 31: b'7', - 5: b'F', 14: b'O', 23: b'X', - 6: b'G', 15: b'P', 24: b'Y', - 7: b'H', 16: b'Q', 25: b'Z', - 8: b'I', 17: b'R', 26: b'2', - } - -_b32tab = [v[0] for k, v in sorted(_b32alphabet.items())] -_b32rev = dict([(v[0], k) for k, v in _b32alphabet.items()]) - - -def b32encode(s: bytes) -> bytes: - """Encode a byte string using Base32. - - s is the byte string to encode. The encoded byte string is returned. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - quanta, leftover = divmod(len(s), 5) - # Pad the last quantum with zero bits if necessary - if leftover: - s = s + bytes(5 - leftover) # Don't use += ! - quanta += 1 - encoded = bytes() - for i in range(quanta): - # c1 and c2 are 16 bits wide, c3 is 8 bits wide. The intent of this - # code is to process the 40 bits in units of 5 bits. So we take the 1 - # leftover bit of c1 and tack it onto c2. Then we take the 2 leftover - # bits of c2 and tack them onto c3. The shifts and masks are intended - # to give us values of exactly 5 bits in width. - c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5]) # type: (int, int, int) - c2 += (c1 & 1) << 16 # 17 bits wide - c3 += (c2 & 3) << 8 # 10 bits wide - encoded += bytes([_b32tab[c1 >> 11], # bits 1 - 5 - _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10 - _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15 - _b32tab[c2 >> 12], # bits 16 - 20 (1 - 5) - _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10) - _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15) - _b32tab[c3 >> 5], # bits 31 - 35 (1 - 5) - _b32tab[c3 & 0x1f], # bits 36 - 40 (1 - 5) - ]) - # Adjust for any leftover partial quanta - if leftover == 1: - return encoded[:-6] + b'======' - elif leftover == 2: - return encoded[:-4] + b'====' - elif leftover == 3: - return encoded[:-3] + b'===' - elif leftover == 4: - return encoded[:-1] + b'=' - return encoded - - -def b32decode(s: bytes, casefold: bool = False, map01: bytes = None) -> bytes: - """Decode a Base32 encoded byte string. - - s is the byte string to decode. Optional casefold is a flag - specifying whether a lowercase alphabet is acceptable as input. - For security purposes, the default is False. - - RFC 3548 allows for optional mapping of the digit 0 (zero) to the - letter O (oh), and for optional mapping of the digit 1 (one) to - either the letter I (eye) or letter L (el). The optional argument - map01 when not None, specifies which letter the digit 1 should be - mapped to (when map01 is not None, the digit 0 is always mapped to - the letter O). For security purposes the default is None, so that - 0 and 1 are not allowed in the input. - - The decoded byte string is returned. binascii.Error is raised if - the input is incorrectly padded or if there are non-alphabet - characters present in the input. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - quanta, leftover = divmod(len(s), 8) - if leftover: - raise binascii.Error('Incorrect padding') - # Handle section 2.4 zero and one mapping. The flag map01 will be either - # False, or the character to map the digit 1 (one) to. It should be - # either L (el) or I (eye). - if map01 is not None: - if not isinstance(map01, bytes_types): - raise TypeError("expected bytes, not %s" % map01.__class__.__name__) - assert len(map01) == 1, repr(map01) - s = _translate(s, {b'0': b'O', b'1': map01}) - if casefold: - s = s.upper() - # Strip off pad characters from the right. We need to count the pad - # characters because this will tell us how many null bytes to remove from - # the end of the decoded string. - padchars = 0 - mo = re.search(b'(?P[=]*)$', s) - if mo: - padchars = len(mo.group('pad')) - if padchars > 0: - s = s[:-padchars] - # Now decode the full quanta - parts = List[bytes]() - acc = 0 - shift = 35 - for c in s: - val = _b32rev.get(c) - if val is None: - raise TypeError('Non-base32 digit found') - acc += _b32rev[c] << shift - shift -= 5 - if shift < 0: - parts.append(binascii.unhexlify(bytes('%010x' % acc, "ascii"))) - acc = 0 - shift = 35 - # Process the last, partial quanta - last = binascii.unhexlify(bytes('%010x' % acc, "ascii")) - if padchars == 0: - last = b'' # No characters - elif padchars == 1: - last = last[:-1] - elif padchars == 3: - last = last[:-2] - elif padchars == 4: - last = last[:-3] - elif padchars == 6: - last = last[:-4] - else: - raise binascii.Error('Incorrect padding') - parts.append(last) - return b''.join(parts) - - - -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns -# lowercase. The RFC also recommends against accepting input case -# insensitively. -def b16encode(s: bytes) -> bytes: - """Encode a byte string using Base16. - - s is the byte string to encode. The encoded byte string is returned. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - return binascii.hexlify(s).upper() - - -def b16decode(s: bytes, casefold: bool = False) -> bytes: - """Decode a Base16 encoded byte string. - - s is the byte string to decode. Optional casefold is a flag - specifying whether a lowercase alphabet is acceptable as input. - For security purposes, the default is False. - - The decoded byte string is returned. binascii.Error is raised if - s were incorrectly padded or if there are non-alphabet characters - present in the string. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - if casefold: - s = s.upper() - if re.search(b'[^0-9A-F]', s): - raise binascii.Error('Non-base16 digit found') - return binascii.unhexlify(s) - - - -# Legacy interface. This code could be cleaned up since I don't believe -# binascii has any line length limitations. It just doesn't seem worth it -# though. The files should be opened in binary mode. - -MAXLINESIZE = 76 # Excluding the CRLF -MAXBINSIZE = (MAXLINESIZE//4)*3 - -def encode(input: IO[bytes], output: IO[bytes]) -> None: - """Encode a file; input and output are binary files.""" - while True: - s = input.read(MAXBINSIZE) - if not s: - break - while len(s) < MAXBINSIZE: - ns = input.read(MAXBINSIZE-len(s)) - if not ns: - break - s += ns - line = binascii.b2a_base64(s) - output.write(line) - - -def decode(input: IO[bytes], output: IO[bytes]) -> None: - """Decode a file; input and output are binary files.""" - while True: - line = input.readline() - if not line: - break - s = binascii.a2b_base64(line) - output.write(s) - - -def encodebytes(s: bytes) -> bytes: - """Encode a bytestring into a bytestring containing multiple lines - of base-64 data.""" - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - pieces = List[bytes]() - for i in range(0, len(s), MAXBINSIZE): - chunk = s[i : i + MAXBINSIZE] - pieces.append(binascii.b2a_base64(chunk)) - return b"".join(pieces) - -def encodestring(s: bytes) -> bytes: - """Legacy alias of encodebytes().""" - import warnings - warnings.warn("encodestring() is a deprecated alias, use encodebytes()", - DeprecationWarning, 2) - return encodebytes(s) - - -def decodebytes(s: bytes) -> bytes: - """Decode a bytestring of base-64 data into a bytestring.""" - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - return binascii.a2b_base64(s) - -def decodestring(s: bytes) -> bytes: - """Legacy alias of decodebytes().""" - import warnings - warnings.warn("decodestring() is a deprecated alias, use decodebytes()", - DeprecationWarning, 2) - return decodebytes(s) - - -# Usable as a script... -def main() -> None: - """Small main program""" - import sys, getopt - try: - opts, args = getopt.getopt(sys.argv[1:], 'deut') - except getopt.error as msg: - sys.stdout = sys.stderr - print(msg) - print("""usage: %s [-d|-e|-u|-t] [file|-] - -d, -u: decode - -e: encode (default) - -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0]) - sys.exit(2) - func = encode - for o, a in opts: - if o == '-e': func = encode - if o == '-d': func = decode - if o == '-u': func = decode - if o == '-t': test(); return - if args and args[0] != '-': - with open(args[0], 'rb') as f: - func(f, sys.stdout.buffer) - else: - func(sys.stdin.buffer, sys.stdout.buffer) - - -def test() -> None: - s0 = b"Aladdin:open sesame" - print(repr(s0)) - s1 = encodebytes(s0) - print(repr(s1)) - s2 = decodebytes(s1) - print(repr(s2)) - assert s0 == s2 - - -if __name__ == '__main__': - main() diff --git a/lib-python/3.2/fnmatch.py b/lib-python/3.2/fnmatch.py deleted file mode 100644 index e699b998ebf5..000000000000 --- a/lib-python/3.2/fnmatch.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Filename matching with shell patterns. - -fnmatch(FILENAME, PATTERN) matches according to the local convention. -fnmatchcase(FILENAME, PATTERN) always takes case in account. - -The functions operate by translating the pattern into a regular -expression. They cache the compiled regular expressions for speed. - -The function translate(PATTERN) returns a regular expression -corresponding to PATTERN. (It does not compile it.) -""" -import os -import posixpath -import re -import functools - -from typing import Iterable, List, AnyStr, Any, Function, Match - -__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] - -def fnmatch(name: AnyStr, pat: AnyStr) -> bool: - """Test whether FILENAME matches PATTERN. - - Patterns are Unix shell style: - - * matches everything - ? matches any single character - [seq] matches any character in seq - [!seq] matches any char not in seq - - An initial period in FILENAME is not special. - Both FILENAME and PATTERN are first case-normalized - if the operating system requires it. - If you don't want this, use fnmatchcase(FILENAME, PATTERN). - """ - name = os.path.normcase(name) - pat = os.path.normcase(pat) - return fnmatchcase(name, pat) - -@functools.lru_cache(maxsize=250) -def _compile_pattern(pat: AnyStr, - is_bytes: bool = False) -> Function[[AnyStr], - Match[AnyStr]]: - if isinstance(pat, bytes): - pat_str = str(pat, 'ISO-8859-1') - res_str = translate(pat_str) - res = bytes(res_str, 'ISO-8859-1') - else: - res = translate(pat) - return re.compile(res).match - -def filter(names: Iterable[AnyStr], pat: AnyStr) -> List[AnyStr]: - """Return the subset of the list NAMES that match PAT.""" - result = List[AnyStr]() - pat = os.path.normcase(pat) - match = _compile_pattern(pat, isinstance(pat, bytes)) - if os.path is posixpath: - # normcase on posix is NOP. Optimize it away from the loop. - for name in names: - if match(name): - result.append(name) - else: - for name in names: - if match(os.path.normcase(name)): - result.append(name) - return result - -def fnmatchcase(name: AnyStr, pat: AnyStr) -> bool: - """Test whether FILENAME matches PATTERN, including case. - - This is a version of fnmatch() which doesn't case-normalize - its arguments. - """ - match = _compile_pattern(pat, isinstance(pat, bytes)) - return match(name) is not None - -def translate(pat: str) -> str: - """Translate a shell PATTERN to a regular expression. - - There is no way to quote meta-characters. - """ - - i, n = 0, len(pat) - res = '' - while i < n: - c = pat[i] - i = i+1 - if c == '*': - res = res + '.*' - elif c == '?': - res = res + '.' - elif c == '[': - j = i - if j < n and pat[j] == '!': - j = j+1 - if j < n and pat[j] == ']': - j = j+1 - while j < n and pat[j] != ']': - j = j+1 - if j >= n: - res = res + '\\[' - else: - stuff = pat[i:j].replace('\\','\\\\') - i = j+1 - if stuff[0] == '!': - stuff = '^' + stuff[1:] - elif stuff[0] == '^': - stuff = '\\' + stuff - res = '%s[%s]' % (res, stuff) - else: - res = res + re.escape(c) - return res + '\Z(?ms)' diff --git a/lib-python/3.2/genericpath.py b/lib-python/3.2/genericpath.py deleted file mode 100644 index bd1fddf750bf..000000000000 --- a/lib-python/3.2/genericpath.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Path operations common to more than one OS -Do not use directly. The OS specific modules import the appropriate -functions from this module themselves. -""" -import os -import stat - -from typing import ( - Any as Any_, List as List_, AnyStr as AnyStr_, Tuple as Tuple_ -) - -__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile'] - - -# Does a path exist? -# This is false for dangling symbolic links on systems that support them. -def exists(path: AnyStr_) -> bool: - """Test whether a path exists. Returns False for broken symbolic links""" - try: - os.stat(path) - except os.error: - return False - return True - - -# This follows symbolic links, so both islink() and isdir() can be true -# for the same path ono systems that support symlinks -def isfile(path: AnyStr_) -> bool: - """Test whether a path is a regular file""" - try: - st = os.stat(path) - except os.error: - return False - return stat.S_ISREG(st.st_mode) - - -# Is a path a directory? -# This follows symbolic links, so both islink() and isdir() -# can be true for the same path on systems that support symlinks -def isdir(s: AnyStr_) -> bool: - """Return true if the pathname refers to an existing directory.""" - try: - st = os.stat(s) - except os.error: - return False - return stat.S_ISDIR(st.st_mode) - - -def getsize(filename: AnyStr_) -> int: - """Return the size of a file, reported by os.stat().""" - return os.stat(filename).st_size - - -def getmtime(filename: AnyStr_) -> float: - """Return the last modification time of a file, reported by os.stat().""" - return os.stat(filename).st_mtime - - -def getatime(filename: AnyStr_) -> float: - """Return the last access time of a file, reported by os.stat().""" - return os.stat(filename).st_atime - - -def getctime(filename: AnyStr_) -> float: - """Return the metadata change time of a file, reported by os.stat().""" - return os.stat(filename).st_ctime - - -# Return the longest prefix of all list elements. -def commonprefix(m: List_[Any_]) -> Any_: - "Given a list of pathnames, returns the longest common leading component" - if not m: return '' - s1 = min(m) - s2 = max(m) - for i, c in enumerate(s1): - if c != s2[i]: - return s1[:i] - return s1 - - -# Split a path in root and extension. -# The extension is everything starting at the last dot in the last -# pathname component; the root is everything before that. -# It is always true that root + ext == p. - -# Generic implementation of splitext, to be parametrized with -# the separators -def _splitext(p: AnyStr_, sep: AnyStr_, altsep: AnyStr_, - extsep: AnyStr_) -> Tuple_[AnyStr_, AnyStr_]: - """Split the extension from a pathname. - - Extension is everything from the last dot to the end, ignoring - leading dots. Returns "(root, ext)"; ext may be empty.""" - # NOTE: This code must work for text and bytes strings. - - sepIndex = p.rfind(sep) - if altsep: - altsepIndex = p.rfind(altsep) - sepIndex = max(sepIndex, altsepIndex) - - dotIndex = p.rfind(extsep) - if dotIndex > sepIndex: - # skip all leading dots - filenameIndex = sepIndex + 1 - while filenameIndex < dotIndex: - if p[filenameIndex:filenameIndex+1] != extsep: - return p[:dotIndex], p[dotIndex:] - filenameIndex += 1 - - return p, p[:0] diff --git a/lib-python/3.2/getopt.py b/lib-python/3.2/getopt.py deleted file mode 100644 index 86977e6d8e20..000000000000 --- a/lib-python/3.2/getopt.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Parser for command line options. - -This module helps scripts to parse the command line arguments in -sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software -may be used as well via an optional third argument. This module -provides two functions and an exception: - -getopt() -- Parse command line options -gnu_getopt() -- Like getopt(), but allow option and non-option arguments -to be intermixed. -GetoptError -- exception (class) raised with 'opt' attribute, which is the -option involved with the exception. -""" - -# Long option support added by Lars Wirzenius . -# -# Gerrit Holl moved the string-based exceptions -# to class-based exceptions. -# -# Peter Åstrand added gnu_getopt(). -# -# TODO for gnu_getopt(): -# -# - GNU getopt_long_only mechanism -# - allow the caller to specify ordering -# - RETURN_IN_ORDER option -# - GNU extension with '-' as first character of option string -# - optional arguments, specified by double colons -# - a option string with a W followed by semicolon should -# treat "-W foo" as "--foo" - -__all__ = ["GetoptError","error","getopt","gnu_getopt"] - -import os - -from typing import List, Tuple, Iterable - -class GetoptError(Exception): - opt = '' - msg = '' - def __init__(self, msg: str, opt: str = '') -> None: - self.msg = msg - self.opt = opt - Exception.__init__(self, msg, opt) - - def __str__(self) -> str: - return self.msg - -error = GetoptError # backward compatibility - -def getopt(args: List[str], shortopts: str, - longopts: Iterable[str] = []) -> Tuple[List[Tuple[str, str]], - List[str]]: - """getopt(args, options[, long_options]) -> opts, args - - Parses command line options and parameter list. args is the - argument list to be parsed, without the leading reference to the - running program. Typically, this means "sys.argv[1:]". shortopts - is the string of option letters that the script wants to - recognize, with options that require an argument followed by a - colon (i.e., the same format that Unix getopt() uses). If - specified, longopts is a list of strings with the names of the - long options which should be supported. The leading '--' - characters should not be included in the option name. Options - which require an argument should be followed by an equal sign - ('='). - - The return value consists of two elements: the first is a list of - (option, value) pairs; the second is the list of program arguments - left after the option list was stripped (this is a trailing slice - of the first argument). Each option-and-value pair returned has - the option as its first element, prefixed with a hyphen (e.g., - '-x'), and the option argument as its second element, or an empty - string if the option has no argument. The options occur in the - list in the same order in which they were found, thus allowing - multiple occurrences. Long and short options may be mixed. - - """ - - opts = List[Tuple[str, str]]() - if isinstance(longopts, str): - longopts = [longopts] - else: - longopts = list(longopts) - while args and args[0].startswith('-') and args[0] != '-': - if args[0] == '--': - args = args[1:] - break - if args[0].startswith('--'): - opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) - else: - opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) - - return opts, args - -def gnu_getopt(args: List[str], shortopts: str, - longopts: Iterable[str] = []) -> Tuple[List[Tuple[str, str]], - List[str]]: - """getopt(args, options[, long_options]) -> opts, args - - This function works like getopt(), except that GNU style scanning - mode is used by default. This means that option and non-option - arguments may be intermixed. The getopt() function stops - processing options as soon as a non-option argument is - encountered. - - If the first character of the option string is `+', or if the - environment variable POSIXLY_CORRECT is set, then option - processing stops as soon as a non-option argument is encountered. - - """ - - opts = List[Tuple[str, str]]() - prog_args = List[str]() - if isinstance(longopts, str): - longopts = [longopts] - else: - longopts = list(longopts) - - # Allow options after non-option arguments? - if shortopts.startswith('+'): - shortopts = shortopts[1:] - all_options_first = True - elif os.environ.get("POSIXLY_CORRECT"): - all_options_first = True - else: - all_options_first = False - - while args: - if args[0] == '--': - prog_args += args[1:] - break - - if args[0][:2] == '--': - opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) - elif args[0][:1] == '-' and args[0] != '-': - opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) - else: - if all_options_first: - prog_args += args - break - else: - prog_args.append(args[0]) - args = args[1:] - - return opts, prog_args - -def do_longs(opts: List[Tuple[str, str]], opt: str, - longopts: List[str], - args: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]: - try: - i = opt.index('=') - except ValueError: - optarg = None # type: str - else: - opt, optarg = opt[:i], opt[i+1:] - - has_arg, opt = long_has_args(opt, longopts) - if has_arg: - if optarg is None: - if not args: - raise GetoptError('option --%s requires argument' % opt, opt) - optarg, args = args[0], args[1:] - elif optarg is not None: - raise GetoptError('option --%s must not have an argument' % opt, opt) - opts.append(('--' + opt, optarg or '')) - return opts, args - -# Return: -# has_arg? -# full option name -def long_has_args(opt: str, longopts: List[str]) -> Tuple[bool, str]: - possibilities = [o for o in longopts if o.startswith(opt)] - if not possibilities: - raise GetoptError('option --%s not recognized' % opt, opt) - # Is there an exact match? - if opt in possibilities: - return False, opt - elif opt + '=' in possibilities: - return True, opt - # No exact match, so better be unique. - if len(possibilities) > 1: - # XXX since possibilities contains all valid continuations, might be - # nice to work them into the error msg - raise GetoptError('option --%s not a unique prefix' % opt, opt) - assert len(possibilities) == 1 - unique_match = possibilities[0] - has_arg = unique_match.endswith('=') - if has_arg: - unique_match = unique_match[:-1] - return has_arg, unique_match - -def do_shorts(opts: List[Tuple[str, str]], optstring: str, - shortopts: str, args: List[str]) -> Tuple[List[Tuple[str, str]], - List[str]]: - while optstring != '': - opt, optstring = optstring[0], optstring[1:] - if short_has_arg(opt, shortopts): - if optstring == '': - if not args: - raise GetoptError('option -%s requires argument' % opt, - opt) - optstring, args = args[0], args[1:] - optarg, optstring = optstring, '' - else: - optarg = '' - opts.append(('-' + opt, optarg)) - return opts, args - -def short_has_arg(opt: str, shortopts: str) -> bool: - for i in range(len(shortopts)): - if opt == shortopts[i] != ':': - return shortopts.startswith(':', i+1) - raise GetoptError('option -%s not recognized' % opt, opt) - -if __name__ == '__main__': - import sys - print(getopt(sys.argv[1:], "a:b", ["alpha=", "beta"])) diff --git a/lib-python/3.2/glob.py b/lib-python/3.2/glob.py deleted file mode 100644 index 0f3d5f5d9a09..000000000000 --- a/lib-python/3.2/glob.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Filename globbing utility.""" - -import os -import re -import fnmatch - -from typing import List, Iterator, Iterable, Any, AnyStr - -__all__ = ["glob", "iglob"] - -def glob(pathname: AnyStr) -> List[AnyStr]: - """Return a list of paths matching a pathname pattern. - - The pattern may contain simple shell-style wildcards a la fnmatch. - - """ - return list(iglob(pathname)) - -def iglob(pathname: AnyStr) -> Iterator[AnyStr]: - """Return an iterator which yields the paths matching a pathname pattern. - - The pattern may contain simple shell-style wildcards a la fnmatch. - - """ - if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname - return - dirname, basename = os.path.split(pathname) - if not dirname: - for name in glob1(None, basename): - yield name - return - if has_magic(dirname): - dirs = iglob(dirname) # type: Iterable[AnyStr] - else: - dirs = [dirname] - if has_magic(basename): - glob_in_dir = glob1 # type: Any - else: - glob_in_dir = glob0 - for dirname in dirs: - for name in glob_in_dir(dirname, basename): - yield os.path.join(dirname, name) - -# These 2 helper functions non-recursively glob inside a literal directory. -# They return a list of basenames. `glob1` accepts a pattern while `glob0` -# takes a literal basename (so it only has to check for its existence). - -def glob1(dirname: AnyStr, pattern: AnyStr) -> List[AnyStr]: - if not dirname: - if isinstance(pattern, bytes): - dirname = bytes(os.curdir, 'ASCII') - else: - dirname = os.curdir - try: - names = os.listdir(dirname) - except os.error: - return [] - if pattern[0] != '.': - names = [x for x in names if x[0] != '.'] - return fnmatch.filter(names, pattern) - -def glob0(dirname: AnyStr, basename: AnyStr) -> List[AnyStr]: - if basename == '': - # `os.path.split()` returns an empty basename for paths ending with a - # directory separator. 'q*x/' should match only directories. - if os.path.isdir(dirname): - return [basename] - else: - if os.path.lexists(os.path.join(dirname, basename)): - return [basename] - return [] - - -magic_check = re.compile('[*?[]') -magic_check_bytes = re.compile(b'[*?[]') - -def has_magic(s: AnyStr) -> bool: - if isinstance(s, bytes): - match = magic_check_bytes.search(s) - else: - match = magic_check.search(s) - return match is not None diff --git a/lib-python/3.2/incomplete/logging/__init__.py b/lib-python/3.2/incomplete/logging/__init__.py deleted file mode 100644 index aa861eb956f2..000000000000 --- a/lib-python/3.2/incomplete/logging/__init__.py +++ /dev/null @@ -1,1873 +0,0 @@ -# Copyright 2001-2010 by Vinay Sajip. All Rights Reserved. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose and without fee is hereby granted, -# provided that the above copyright notice appear in all copies and that -# both that copyright notice and this permission notice appear in -# supporting documentation, and that the name of Vinay Sajip -# not be used in advertising or publicity pertaining to distribution -# of the software without specific, written prior permission. -# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING -# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL -# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR -# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER -# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -""" -Logging package for Python. Based on PEP 282 and comments thereto in -comp.lang.python, and influenced by Apache's log4j system. - -Copyright (C) 2001-2011 Vinay Sajip. All Rights Reserved. - -To use, simply 'import logging' and log away! -""" - -import sys, os, time, io, traceback, warnings, weakref -from string import Template - -__all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', - 'FATAL', 'FileHandler', 'Filter', 'Formatter', 'Handler', 'INFO', - 'LogRecord', 'Logger', 'LoggerAdapter', 'NOTSET', 'NullHandler', - 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig', - 'captureWarnings', 'critical', 'debug', 'disable', 'error', - 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', - 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning', - 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort'] - -import codecs - -import _thread as thread -import threading - -__author__ = "Vinay Sajip " -__status__ = "production" -__version__ = "0.5.1.2" -__date__ = "07 February 2010" - -#--------------------------------------------------------------------------- -# Miscellaneous module data -#--------------------------------------------------------------------------- - -# -# _srcfile is used when walking the stack to check when we've got the first -# caller stack frame. -# -if hasattr(sys, 'frozen'): #support for py2exe - _srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:]) -else: - _srcfile = __file__ -_srcfile = os.path.normcase(_srcfile) - -# next bit filched from 1.5.2's inspect.py -def _currentframe(): - """Return the frame object for the caller's stack frame.""" - try: - raise Exception - except: - return sys.exc_info()[2].tb_frame.f_back -currentframe = _currentframe - -if hasattr(sys, '_getframe'): currentframe = lambda: sys._getframe(3) -# done filching - -# _srcfile is only used in conjunction with sys._getframe(). -# To provide compatibility with older versions of Python, set _srcfile -# to None if _getframe() is not available; this value will prevent -# findCaller() from being called. -#if not hasattr(sys, "_getframe"): -# _srcfile = None - -# -#_startTime is used as the base when calculating the relative time of events -# -_startTime = time.time() - -# -#raiseExceptions is used to see if exceptions during handling should be -#propagated -# -raiseExceptions = 1 - -# -# If you don't want threading information in the log, set this to zero -# -logThreads = 1 - -# -# If you don't want multiprocessing information in the log, set this to zero -# -logMultiprocessing = 1 - -# -# If you don't want process information in the log, set this to zero -# -logProcesses = 1 - -#--------------------------------------------------------------------------- -# Level related stuff -#--------------------------------------------------------------------------- -# -# Default levels and level names, these can be replaced with any positive set -# of values having corresponding names. There is a pseudo-level, NOTSET, which -# is only really there as a lower limit for user-defined levels. Handlers and -# loggers are initialized with NOTSET so that they will log all messages, even -# at user-defined levels. -# - -CRITICAL = 50 -FATAL = CRITICAL -ERROR = 40 -WARNING = 30 -WARN = WARNING -INFO = 20 -DEBUG = 10 -NOTSET = 0 - -_levelNames = { - CRITICAL : 'CRITICAL', - ERROR : 'ERROR', - WARNING : 'WARNING', - INFO : 'INFO', - DEBUG : 'DEBUG', - NOTSET : 'NOTSET', - 'CRITICAL' : CRITICAL, - 'ERROR' : ERROR, - 'WARN' : WARNING, - 'WARNING' : WARNING, - 'INFO' : INFO, - 'DEBUG' : DEBUG, - 'NOTSET' : NOTSET, -} - -def getLevelName(level): - """ - Return the textual representation of logging level 'level'. - - If the level is one of the predefined levels (CRITICAL, ERROR, WARNING, - INFO, DEBUG) then you get the corresponding string. If you have - associated levels with names using addLevelName then the name you have - associated with 'level' is returned. - - If a numeric value corresponding to one of the defined levels is passed - in, the corresponding string representation is returned. - - Otherwise, the string "Level %s" % level is returned. - """ - return _levelNames.get(level, ("Level %s" % level)) - -def addLevelName(level, levelName): - """ - Associate 'levelName' with 'level'. - - This is used when converting levels to text during message formatting. - """ - _acquireLock() - try: #unlikely to cause an exception, but you never know... - _levelNames[level] = levelName - _levelNames[levelName] = level - finally: - _releaseLock() - -def _checkLevel(level): - if isinstance(level, int): - rv = level - elif str(level) == level: - if level not in _levelNames: - raise ValueError("Unknown level: %r" % level) - rv = _levelNames[level] - else: - raise TypeError("Level not an integer or a valid string: %r" % level) - return rv - -#--------------------------------------------------------------------------- -# Thread-related stuff -#--------------------------------------------------------------------------- - -# -#_lock is used to serialize access to shared data structures in this module. -#This needs to be an RLock because fileConfig() creates and configures -#Handlers, and so might arbitrary user threads. Since Handler code updates the -#shared dictionary _handlers, it needs to acquire the lock. But if configuring, -#the lock would already have been acquired - so we need an RLock. -#The same argument applies to Loggers and Manager.loggerDict. -# -if thread: - _lock = threading.RLock() -else: - _lock = None - - -def _acquireLock(): - """ - Acquire the module-level lock for serializing access to shared data. - - This should be released with _releaseLock(). - """ - if _lock: - _lock.acquire() - -def _releaseLock(): - """ - Release the module-level lock acquired by calling _acquireLock(). - """ - if _lock: - _lock.release() - -#--------------------------------------------------------------------------- -# The logging record -#--------------------------------------------------------------------------- - -class LogRecord(object): - """ - A LogRecord instance represents an event being logged. - - LogRecord instances are created every time something is logged. They - contain all the information pertinent to the event being logged. The - main information passed in is in msg and args, which are combined - using str(msg) % args to create the message field of the record. The - record also includes information such as when the record was created, - the source line where the logging call was made, and any exception - information to be logged. - """ - def __init__(self, name, level, pathname, lineno, - msg, args, exc_info, func=None, sinfo=None, **kwargs): - """ - Initialize a logging record with interesting information. - """ - ct = time.time() - self.name = name - self.msg = msg - # - # The following statement allows passing of a dictionary as a sole - # argument, so that you can do something like - # logging.debug("a %(a)d b %(b)s", {'a':1, 'b':2}) - # Suggested by Stefan Behnel. - # Note that without the test for args[0], we get a problem because - # during formatting, we test to see if the arg is present using - # 'if self.args:'. If the event being logged is e.g. 'Value is %d' - # and if the passed arg fails 'if self.args:' then no formatting - # is done. For example, logger.warn('Value is %d', 0) would log - # 'Value is %d' instead of 'Value is 0'. - # For the use case of passing a dictionary, this should not be a - # problem. - if args and len(args) == 1 and isinstance(args[0], dict) and args[0]: - args = args[0] - self.args = args - self.levelname = getLevelName(level) - self.levelno = level - self.pathname = pathname - try: - self.filename = os.path.basename(pathname) - self.module = os.path.splitext(self.filename)[0] - except (TypeError, ValueError, AttributeError): - self.filename = pathname - self.module = "Unknown module" - self.exc_info = exc_info - self.exc_text = None # used to cache the traceback text - self.stack_info = sinfo - self.lineno = lineno - self.funcName = func - self.created = ct - self.msecs = (ct - int(ct)) * 1000 - self.relativeCreated = (self.created - _startTime) * 1000 - if logThreads and thread: - self.thread = thread.get_ident() - self.threadName = threading.current_thread().name - else: - self.thread = None - self.threadName = None - if not logMultiprocessing: - self.processName = None - else: - self.processName = 'MainProcess' - mp = sys.modules.get('multiprocessing') - if mp is not None: - # Errors may occur if multiprocessing has not finished loading - # yet - e.g. if a custom import hook causes third-party code - # to run when multiprocessing calls import. See issue 8200 - # for an example - try: - self.processName = mp.current_process().name - except Exception: - pass - if logProcesses and hasattr(os, 'getpid'): - self.process = os.getpid() - else: - self.process = None - - def __str__(self): - return ''%(self.name, self.levelno, - self.pathname, self.lineno, self.msg) - - def getMessage(self): - """ - Return the message for this LogRecord. - - Return the message for this LogRecord after merging any user-supplied - arguments with the message. - """ - msg = str(self.msg) - if self.args: - msg = msg % self.args - return msg - -# -# Determine which class to use when instantiating log records. -# -_logRecordFactory = LogRecord - -def setLogRecordFactory(factory): - """ - Set the factory to be used when instantiating a log record. - - :param factory: A callable which will be called to instantiate - a log record. - """ - global _logRecordFactory - _logRecordFactory = factory - -def getLogRecordFactory(): - """ - Return the factory to be used when instantiating a log record. - """ - - return _logRecordFactory - -def makeLogRecord(dict): - """ - Make a LogRecord whose attributes are defined by the specified dictionary, - This function is useful for converting a logging event received over - a socket connection (which is sent as a dictionary) into a LogRecord - instance. - """ - rv = _logRecordFactory(None, None, "", 0, "", (), None, None) - rv.__dict__.update(dict) - return rv - -#--------------------------------------------------------------------------- -# Formatter classes and functions -#--------------------------------------------------------------------------- - -class PercentStyle(object): - - default_format = '%(message)s' - asctime_format = '%(asctime)s' - asctime_search = '%(asctime)' - - def __init__(self, fmt): - self._fmt = fmt or self.default_format - - def usesTime(self): - return self._fmt.find(self.asctime_search) >= 0 - - def format(self, record): - return self._fmt % record.__dict__ - -class StrFormatStyle(PercentStyle): - default_format = '{message}' - asctime_format = '{asctime}' - asctime_search = '{asctime' - - def format(self, record): - return self._fmt.format(**record.__dict__) - - -class StringTemplateStyle(PercentStyle): - default_format = '${message}' - asctime_format = '${asctime}' - asctime_search = '${asctime}' - - def __init__(self, fmt): - self._fmt = fmt or self.default_format - self._tpl = Template(self._fmt) - - def usesTime(self): - fmt = self._fmt - return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0 - - def format(self, record): - return self._tpl.substitute(**record.__dict__) - -_STYLES = { - '%': PercentStyle, - '{': StrFormatStyle, - '$': StringTemplateStyle -} - -class Formatter(object): - """ - Formatter instances are used to convert a LogRecord to text. - - Formatters need to know how a LogRecord is constructed. They are - responsible for converting a LogRecord to (usually) a string which can - be interpreted by either a human or an external system. The base Formatter - allows a formatting string to be specified. If none is supplied, the - default value of "%s(message)" is used. - - The Formatter can be initialized with a format string which makes use of - knowledge of the LogRecord attributes - e.g. the default value mentioned - above makes use of the fact that the user's message and arguments are pre- - formatted into a LogRecord's message attribute. Currently, the useful - attributes in a LogRecord are described by: - - %(name)s Name of the logger (logging channel) - %(levelno)s Numeric logging level for the message (DEBUG, INFO, - WARNING, ERROR, CRITICAL) - %(levelname)s Text logging level for the message ("DEBUG", "INFO", - "WARNING", "ERROR", "CRITICAL") - %(pathname)s Full pathname of the source file where the logging - call was issued (if available) - %(filename)s Filename portion of pathname - %(module)s Module (name portion of filename) - %(lineno)d Source line number where the logging call was issued - (if available) - %(funcName)s Function name - %(created)f Time when the LogRecord was created (time.time() - return value) - %(asctime)s Textual time when the LogRecord was created - %(msecs)d Millisecond portion of the creation time - %(relativeCreated)d Time in milliseconds when the LogRecord was created, - relative to the time the logging module was loaded - (typically at application startup time) - %(thread)d Thread ID (if available) - %(threadName)s Thread name (if available) - %(process)d Process ID (if available) - %(message)s The result of record.getMessage(), computed just as - the record is emitted - """ - - converter = time.localtime - - def __init__(self, fmt=None, datefmt=None, style='%'): - """ - Initialize the formatter with specified format strings. - - Initialize the formatter either with the specified format string, or a - default as described above. Allow for specialized date formatting with - the optional datefmt argument (if omitted, you get the ISO8601 format). - - Use a style parameter of '%', '{' or '$' to specify that you want to - use one of %-formatting, :meth:`str.format` (``{}``) formatting or - :class:`string.Template` formatting in your format string. - - .. versionchanged: 3.2 - Added the ``style`` parameter. - """ - if style not in _STYLES: - raise ValueError('Style must be one of: %s' % ','.join( - _STYLES.keys())) - self._style = _STYLES[style](fmt) - self._fmt = self._style._fmt - self.datefmt = datefmt - - def formatTime(self, record, datefmt=None): - """ - Return the creation time of the specified LogRecord as formatted text. - - This method should be called from format() by a formatter which - wants to make use of a formatted time. This method can be overridden - in formatters to provide for any specific requirement, but the - basic behaviour is as follows: if datefmt (a string) is specified, - it is used with time.strftime() to format the creation time of the - record. Otherwise, the ISO8601 format is used. The resulting - string is returned. This function uses a user-configurable function - to convert the creation time to a tuple. By default, time.localtime() - is used; to change this for a particular formatter instance, set the - 'converter' attribute to a function with the same signature as - time.localtime() or time.gmtime(). To change it for all formatters, - for example if you want all logging times to be shown in GMT, - set the 'converter' attribute in the Formatter class. - """ - ct = self.converter(record.created) - if datefmt: - s = time.strftime(datefmt, ct) - else: - t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - s = "%s,%03d" % (t, record.msecs) # the use of % here is internal - return s - - def formatException(self, ei): - """ - Format and return the specified exception information as a string. - - This default implementation just uses - traceback.print_exception() - """ - sio = io.StringIO() - tb = ei[2] - # See issues #9427, #1553375. Commented out for now. - #if getattr(self, 'fullstack', False): - # traceback.print_stack(tb.tb_frame.f_back, file=sio) - traceback.print_exception(ei[0], ei[1], tb, None, sio) - s = sio.getvalue() - sio.close() - if s[-1:] == "\n": - s = s[:-1] - return s - - def usesTime(self): - """ - Check if the format uses the creation time of the record. - """ - return self._style.usesTime() - - def formatMessage(self, record): - return self._style.format(record) - - def formatStack(self, stack_info): - """ - This method is provided as an extension point for specialized - formatting of stack information. - - The input data is a string as returned from a call to - :func:`traceback.print_stack`, but with the last trailing newline - removed. - - The base implementation just returns the value passed in. - """ - return stack_info - - def format(self, record): - """ - Format the specified record as text. - - The record's attribute dictionary is used as the operand to a - string formatting operation which yields the returned string. - Before formatting the dictionary, a couple of preparatory steps - are carried out. The message attribute of the record is computed - using LogRecord.getMessage(). If the formatting string uses the - time (as determined by a call to usesTime(), formatTime() is - called to format the event time. If there is exception information, - it is formatted using formatException() and appended to the message. - """ - record.message = record.getMessage() - if self.usesTime(): - record.asctime = self.formatTime(record, self.datefmt) - s = self.formatMessage(record) - if record.exc_info: - # Cache the traceback text to avoid converting it multiple times - # (it's constant anyway) - if not record.exc_text: - record.exc_text = self.formatException(record.exc_info) - if record.exc_text: - if s[-1:] != "\n": - s = s + "\n" - s = s + record.exc_text - if record.stack_info: - if s[-1:] != "\n": - s = s + "\n" - s = s + self.formatStack(record.stack_info) - return s - -# -# The default formatter to use when no other is specified -# -_defaultFormatter = Formatter() - -class BufferingFormatter(object): - """ - A formatter suitable for formatting a number of records. - """ - def __init__(self, linefmt=None): - """ - Optionally specify a formatter which will be used to format each - individual record. - """ - if linefmt: - self.linefmt = linefmt - else: - self.linefmt = _defaultFormatter - - def formatHeader(self, records): - """ - Return the header string for the specified records. - """ - return "" - - def formatFooter(self, records): - """ - Return the footer string for the specified records. - """ - return "" - - def format(self, records): - """ - Format the specified records and return the result as a string. - """ - rv = "" - if len(records) > 0: - rv = rv + self.formatHeader(records) - for record in records: - rv = rv + self.linefmt.format(record) - rv = rv + self.formatFooter(records) - return rv - -#--------------------------------------------------------------------------- -# Filter classes and functions -#--------------------------------------------------------------------------- - -class Filter(object): - """ - Filter instances are used to perform arbitrary filtering of LogRecords. - - Loggers and Handlers can optionally use Filter instances to filter - records as desired. The base filter class only allows events which are - below a certain point in the logger hierarchy. For example, a filter - initialized with "A.B" will allow events logged by loggers "A.B", - "A.B.C", "A.B.C.D", "A.B.D" etc. but not "A.BB", "B.A.B" etc. If - initialized with the empty string, all events are passed. - """ - def __init__(self, name=''): - """ - Initialize a filter. - - Initialize with the name of the logger which, together with its - children, will have its events allowed through the filter. If no - name is specified, allow every event. - """ - self.name = name - self.nlen = len(name) - - def filter(self, record): - """ - Determine if the specified record is to be logged. - - Is the specified record to be logged? Returns 0 for no, nonzero for - yes. If deemed appropriate, the record may be modified in-place. - """ - if self.nlen == 0: - return 1 - elif self.name == record.name: - return 1 - elif record.name.find(self.name, 0, self.nlen) != 0: - return 0 - return (record.name[self.nlen] == ".") - -class Filterer(object): - """ - A base class for loggers and handlers which allows them to share - common code. - """ - def __init__(self): - """ - Initialize the list of filters to be an empty list. - """ - self.filters = [] - - def addFilter(self, filter): - """ - Add the specified filter to this handler. - """ - if not (filter in self.filters): - self.filters.append(filter) - - def removeFilter(self, filter): - """ - Remove the specified filter from this handler. - """ - if filter in self.filters: - self.filters.remove(filter) - - def filter(self, record): - """ - Determine if a record is loggable by consulting all the filters. - - The default is to allow the record to be logged; any filter can veto - this and the record is then dropped. Returns a zero value if a record - is to be dropped, else non-zero. - - .. versionchanged: 3.2 - - Allow filters to be just callables. - """ - rv = 1 - for f in self.filters: - if hasattr(f, 'filter'): - result = f.filter(record) - else: - result = f(record) # assume callable - will raise if not - if not result: - rv = 0 - break - return rv - -#--------------------------------------------------------------------------- -# Handler classes and functions -#--------------------------------------------------------------------------- - -any _handlers = weakref.WeakValueDictionary() #map of handler names to handlers -any _handlerList = [] # added to allow handlers to be removed in reverse of order initialized - -def _removeHandlerRef(wr): - """ - Remove a handler reference from the internal cleanup list. - """ - # This function can be called during module teardown, when globals are - # set to None. If _acquireLock is None, assume this is the case and do - # nothing. - if _acquireLock is not None: - _acquireLock() - try: - if wr in _handlerList: - _handlerList.remove(wr) - finally: - _releaseLock() - -def _addHandlerRef(handler): - """ - Add a handler to the internal cleanup list using a weak reference. - """ - _acquireLock() - try: - _handlerList.append(weakref.ref(handler, _removeHandlerRef)) - finally: - _releaseLock() - -class Handler(Filterer): - """ - Handler instances dispatch logging events to specific destinations. - - The base handler class. Acts as a placeholder which defines the Handler - interface. Handlers can optionally use Formatter instances to format - records as desired. By default, no formatter is specified; in this case, - the 'raw' message as determined by record.message is logged. - """ - def __init__(self, level=NOTSET): - """ - Initializes the instance - basically setting the formatter to None - and the filter list to empty. - """ - Filterer.__init__(self) - self._name = None - self.level = _checkLevel(level) - self.formatter = None - # Add the handler to the global _handlerList (for cleanup on shutdown) - _addHandlerRef(self) - self.createLock() - - def get_name(self): - return self._name - - def set_name(self, name): - _acquireLock() - try: - if self._name in _handlers: - del _handlers[self._name] - self._name = name - if name: - _handlers[name] = self - finally: - _releaseLock() - - #name = property(get_name, set_name) - - def createLock(self): - """ - Acquire a thread lock for serializing access to the underlying I/O. - """ - if thread: - self.lock = threading.RLock() - else: - self.lock = None - - def acquire(self): - """ - Acquire the I/O thread lock. - """ - if self.lock: - self.lock.acquire() - - def release(self): - """ - Release the I/O thread lock. - """ - if self.lock: - self.lock.release() - - def setLevel(self, level): - """ - Set the logging level of this handler. - """ - self.level = _checkLevel(level) - - def format(self, record): - """ - Format the specified record. - - If a formatter is set, use it. Otherwise, use the default formatter - for the module. - """ - if self.formatter: - fmt = self.formatter - else: - fmt = _defaultFormatter - return fmt.format(record) - - def emit(self, record): - """ - Do whatever it takes to actually log the specified logging record. - - This version is intended to be implemented by subclasses and so - raises a NotImplementedError. - """ - raise NotImplementedError('emit must be implemented ' - 'by Handler subclasses') - - def handle(self, record): - """ - Conditionally emit the specified logging record. - - Emission depends on filters which may have been added to the handler. - Wrap the actual emission of the record with acquisition/release of - the I/O thread lock. Returns whether the filter passed the record for - emission. - """ - rv = self.filter(record) - if rv: - self.acquire() - try: - self.emit(record) - finally: - self.release() - return rv - - def setFormatter(self, fmt): - """ - Set the formatter for this handler. - """ - self.formatter = fmt - - def flush(self): - """ - Ensure all logging output has been flushed. - - This version does nothing and is intended to be implemented by - subclasses. - """ - pass - - def close(self): - """ - Tidy up any resources used by the handler. - - This version removes the handler from an internal map of handlers, - _handlers, which is used for handler lookup by name. Subclasses - should ensure that this gets called from overridden close() - methods. - """ - #get the module data lock, as we're updating a shared structure. - _acquireLock() - try: #unlikely to raise an exception, but you never know... - if self._name and self._name in _handlers: - del _handlers[self._name] - finally: - _releaseLock() - - def handleError(self, record): - """ - Handle errors which occur during an emit() call. - - This method should be called from handlers when an exception is - encountered during an emit() call. If raiseExceptions is false, - exceptions get silently ignored. This is what is mostly wanted - for a logging system - most users will not care about errors in - the logging system, they are more interested in application errors. - You could, however, replace this with a custom handler if you wish. - The record which was being processed is passed in to this method. - """ - if raiseExceptions and sys.stderr: # see issue 13807 - ei = sys.exc_info() - try: - traceback.print_exception(ei[0], ei[1], ei[2], - None, sys.stderr) - sys.stderr.write('Logged from file %s, line %s\n' % ( - record.filename, record.lineno)) - except IOError: - pass # see issue 5971 - finally: - ei = None - -class StreamHandler(Handler): - """ - A handler class which writes logging records, appropriately formatted, - to a stream. Note that this class does not close the stream, as - sys.stdout or sys.stderr may be used. - """ - - terminator = '\n' - - def __init__(self, stream=None): - """ - Initialize the handler. - - If stream is not specified, sys.stderr is used. - """ - Handler.__init__(self) - if stream is None: - stream = sys.stderr - self.stream = stream - - def flush(self): - """ - Flushes the stream. - """ - if self.stream and hasattr(self.stream, "flush"): - self.stream.flush() - - def emit(self, record): - """ - Emit a record. - - If a formatter is specified, it is used to format the record. - The record is then written to the stream with a trailing newline. If - exception information is present, it is formatted using - traceback.print_exception and appended to the stream. If the stream - has an 'encoding' attribute, it is used to determine how to do the - output to the stream. - """ - try: - msg = self.format(record) - stream = self.stream - stream.write(msg) - stream.write(self.terminator) - self.flush() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.handleError(record) - -class FileHandler(StreamHandler): - """ - A handler class which writes formatted logging records to disk files. - """ - def __init__(self, filename, mode='a', encoding=None, delay=0): - """ - Open the specified file and use it as the stream for logging. - """ - #keep the absolute path, otherwise derived classes which use this - #may come a cropper when the current directory changes - if codecs is None: - encoding = None - self.baseFilename = os.path.abspath(filename) - self.mode = mode - self.encoding = encoding - if delay: - #We don't open the stream, but we still need to call the - #Handler constructor to set level, formatter, lock etc. - Handler.__init__(self) - self.stream = None - else: - StreamHandler.__init__(self, self._open()) - - def close(self): - """ - Closes the stream. - """ - if self.stream: - self.flush() - if hasattr(self.stream, "close"): - self.stream.close() - StreamHandler.close(self) - self.stream = None - - def _open(self): - """ - Open the current base file with the (original) mode and encoding. - Return the resulting stream. - """ - if self.encoding is None: - stream = open(self.baseFilename, self.mode) - else: - stream = codecs.open(self.baseFilename, self.mode, self.encoding) - return stream - - def emit(self, record): - """ - Emit a record. - - If the stream was not opened because 'delay' was specified in the - constructor, open it before calling the superclass's emit. - """ - if self.stream is None: - self.stream = self._open() - StreamHandler.emit(self, record) - -class _StderrHandler(StreamHandler): - """ - This class is like a StreamHandler using sys.stderr, but always uses - whatever sys.stderr is currently set to rather than the value of - sys.stderr at handler construction time. - """ - def __init__(self, level=NOTSET): - """ - Initialize the handler. - """ - Handler.__init__(self, level) - - #@property - #def stream(self): - # return sys.stderr - - -_defaultLastResort = _StderrHandler(WARNING) -lastResort = _defaultLastResort - -#--------------------------------------------------------------------------- -# Manager classes and functions -#--------------------------------------------------------------------------- - -class PlaceHolder(object): - """ - PlaceHolder instances are used in the Manager logger hierarchy to take - the place of nodes for which no loggers have been defined. This class is - intended for internal use only and not as part of the public API. - """ - def __init__(self, alogger): - """ - Initialize with the specified logger being a child of this placeholder. - """ - self.loggerMap = { alogger : None } - - def append(self, alogger): - """ - Add the specified logger as a child of this placeholder. - """ - if alogger not in self.loggerMap: - self.loggerMap[alogger] = None - -# -# Determine which class to use when instantiating loggers. -# -any _loggerClass = None - -def setLoggerClass(klass): - """ - Set the class to be used when instantiating a logger. The class should - define __init__() such that only a name argument is required, and the - __init__() should call Logger.__init__() - """ - if klass != Logger: - if not issubclass(klass, Logger): - raise TypeError("logger not derived from logging.Logger: " - + klass.__name__) - global _loggerClass - _loggerClass = klass - -def getLoggerClass(): - """ - Return the class to be used when instantiating a logger. - """ - - return _loggerClass - -class Manager(object): - """ - There is [under normal circumstances] just one Manager instance, which - holds the hierarchy of loggers. - """ - def __init__(self, rootnode): - """ - Initialize the manager with the root node of the logger hierarchy. - """ - self.root = rootnode - self.disable = 0 - self.emittedNoHandlerWarning = False - self.loggerDict = {} - self.loggerClass = None - self.logRecordFactory = None - - def getLogger(self, name): - """ - Get a logger with the specified name (channel name), creating it - if it doesn't yet exist. This name is a dot-separated hierarchical - name, such as "a", "a.b", "a.b.c" or similar. - - If a PlaceHolder existed for the specified name [i.e. the logger - didn't exist but a child of it did], replace it with the created - logger and fix up the parent/child references which pointed to the - placeholder to now point to the logger. - """ - rv = None - if not isinstance(name, str): - raise TypeError('A logger name must be a string') - _acquireLock() - try: - if name in self.loggerDict: - rv = self.loggerDict[name] - if isinstance(rv, PlaceHolder): - ph = rv - rv = (self.loggerClass or _loggerClass)(name) - rv.manager = self - self.loggerDict[name] = rv - self._fixupChildren(ph, rv) - self._fixupParents(rv) - else: - rv = (self.loggerClass or _loggerClass)(name) - rv.manager = self - self.loggerDict[name] = rv - self._fixupParents(rv) - finally: - _releaseLock() - return rv - - def setLoggerClass(self, klass): - """ - Set the class to be used when instantiating a logger with this Manager. - """ - if klass != Logger: - if not issubclass(klass, Logger): - raise TypeError("logger not derived from logging.Logger: " - + klass.__name__) - self.loggerClass = klass - - def setLogRecordFactory(self, factory): - """ - Set the factory to be used when instantiating a log record with this - Manager. - """ - self.logRecordFactory = factory - - def _fixupParents(self, alogger): - """ - Ensure that there are either loggers or placeholders all the way - from the specified logger to the root of the logger hierarchy. - """ - name = alogger.name - i = name.rfind(".") - rv = None - while (i > 0) and not rv: - substr = name[:i] - if substr not in self.loggerDict: - self.loggerDict[substr] = PlaceHolder(alogger) - else: - obj = self.loggerDict[substr] - if isinstance(obj, Logger): - rv = obj - else: - assert isinstance(obj, PlaceHolder) - obj.append(alogger) - i = name.rfind(".", 0, i - 1) - if not rv: - rv = self.root - alogger.parent = rv - - def _fixupChildren(self, ph, alogger): - """ - Ensure that children of the placeholder ph are connected to the - specified logger. - """ - name = alogger.name - namelen = len(name) - for c in ph.loggerMap.keys(): - #The if means ... if not c.parent.name.startswith(nm) - if c.parent.name[:namelen] != name: - alogger.parent = c.parent - c.parent = alogger - -#--------------------------------------------------------------------------- -# Logger classes and functions -#--------------------------------------------------------------------------- - -class Logger(Filterer): - """ - Instances of the Logger class represent a single logging channel. A - "logging channel" indicates an area of an application. Exactly how an - "area" is defined is up to the application developer. Since an - application can have any number of areas, logging channels are identified - by a unique string. Application areas can be nested (e.g. an area - of "input processing" might include sub-areas "read CSV files", "read - XLS files" and "read Gnumeric files"). To cater for this natural nesting, - channel names are organized into a namespace hierarchy where levels are - separated by periods, much like the Java or Python package namespace. So - in the instance given above, channel names might be "input" for the upper - level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels. - There is no arbitrary limit to the depth of nesting. - """ - - any root - any manager - - def __init__(self, name, level=NOTSET): - """ - Initialize the logger with a name and an optional level. - """ - Filterer.__init__(self) - self.name = name - self.level = _checkLevel(level) - self.parent = None - self.propagate = 1 - self.handlers = [] - self.disabled = 0 - - def setLevel(self, level): - """ - Set the logging level of this logger. - """ - self.level = _checkLevel(level) - - def debug(self, msg, *args, **kwargs): - """ - Log 'msg % args' with severity 'DEBUG'. - - To pass exception information, use the keyword argument exc_info with - a true value, e.g. - - logger.debug("Houston, we have a %s", "thorny problem", exc_info=1) - """ - if self.isEnabledFor(DEBUG): - self._log(DEBUG, msg, args, **kwargs) - - def info(self, msg, *args, **kwargs): - """ - Log 'msg % args' with severity 'INFO'. - - To pass exception information, use the keyword argument exc_info with - a true value, e.g. - - logger.info("Houston, we have a %s", "interesting problem", exc_info=1) - """ - if self.isEnabledFor(INFO): - self._log(INFO, msg, args, **kwargs) - - def warning(self, msg, *args, **kwargs): - """ - Log 'msg % args' with severity 'WARNING'. - - To pass exception information, use the keyword argument exc_info with - a true value, e.g. - - logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1) - """ - if self.isEnabledFor(WARNING): - self._log(WARNING, msg, args, **kwargs) - - warn = warning - - def error(self, msg, *args, **kwargs): - """ - Log 'msg % args' with severity 'ERROR'. - - To pass exception information, use the keyword argument exc_info with - a true value, e.g. - - logger.error("Houston, we have a %s", "major problem", exc_info=1) - """ - if self.isEnabledFor(ERROR): - self._log(ERROR, msg, args, **kwargs) - - def exception(self, msg, *args, **kwargs): - """ - Convenience method for logging an ERROR with exception information. - """ - kwargs['exc_info'] = True - self.error(msg, *args, **kwargs) - - def critical(self, msg, *args, **kwargs): - """ - Log 'msg % args' with severity 'CRITICAL'. - - To pass exception information, use the keyword argument exc_info with - a true value, e.g. - - logger.critical("Houston, we have a %s", "major disaster", exc_info=1) - """ - if self.isEnabledFor(CRITICAL): - self._log(CRITICAL, msg, args, **kwargs) - - fatal = critical - - def log(self, level, msg, *args, **kwargs): - """ - Log 'msg % args' with the integer severity 'level'. - - To pass exception information, use the keyword argument exc_info with - a true value, e.g. - - logger.log(level, "We have a %s", "mysterious problem", exc_info=1) - """ - if not isinstance(level, int): - if raiseExceptions: - raise TypeError("level must be an integer") - else: - return - if self.isEnabledFor(level): - self._log(level, msg, args, **kwargs) - - def findCaller(self, stack_info=False): - """ - Find the stack frame of the caller so that we can note the source - file name, line number and function name. - """ - f = currentframe() - #On some versions of IronPython, currentframe() returns None if - #IronPython isn't run with -X:Frames. - if f is not None: - f = f.f_back - rv = "(unknown file)", 0, "(unknown function)", None - while hasattr(f, "f_code"): - co = f.f_code - filename = os.path.normcase(co.co_filename) - if filename == _srcfile: - f = f.f_back - continue - sinfo = None - if stack_info: - sio = io.StringIO() - sio.write('Stack (most recent call last):\n') - traceback.print_stack(f, file=sio) - sinfo = sio.getvalue() - if sinfo[-1] == '\n': - sinfo = sinfo[:-1] - sio.close() - rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) - break - return rv - - def makeRecord(self, name, level, fn, lno, msg, args, exc_info, - func=None, extra=None, sinfo=None): - """ - A factory method which can be overridden in subclasses to create - specialized LogRecords. - """ - rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func, - sinfo) - if extra is not None: - for key in extra: - if (key in ["message", "asctime"]) or (key in rv.__dict__): - raise KeyError("Attempt to overwrite %r in LogRecord" % key) - rv.__dict__[key] = extra[key] - return rv - - def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): - """ - Low-level logging routine which creates a LogRecord and then calls - all the handlers of this logger to handle the record. - """ - sinfo = None - if _srcfile: - #IronPython doesn't track Python frames, so findCaller throws an - #exception on some versions of IronPython. We trap it here so that - #IronPython can use logging. - try: - fn, lno, func, sinfo = self.findCaller(stack_info) - except ValueError: - fn, lno, func = "(unknown file)", 0, "(unknown function)" - else: - fn, lno, func = "(unknown file)", 0, "(unknown function)" - if exc_info: - if not isinstance(exc_info, tuple): - exc_info = sys.exc_info() - record = self.makeRecord(self.name, level, fn, lno, msg, args, - exc_info, func, extra, sinfo) - self.handle(record) - - def handle(self, record): - """ - Call the handlers for the specified record. - - This method is used for unpickled records received from a socket, as - well as those created locally. Logger-level filtering is applied. - """ - if (not self.disabled) and self.filter(record): - self.callHandlers(record) - - def addHandler(self, hdlr): - """ - Add the specified handler to this logger. - """ - _acquireLock() - try: - if not (hdlr in self.handlers): - self.handlers.append(hdlr) - finally: - _releaseLock() - - def removeHandler(self, hdlr): - """ - Remove the specified handler from this logger. - """ - _acquireLock() - try: - if hdlr in self.handlers: - self.handlers.remove(hdlr) - finally: - _releaseLock() - - def hasHandlers(self): - """ - See if this logger has any handlers configured. - - Loop through all handlers for this logger and its parents in the - logger hierarchy. Return True if a handler was found, else False. - Stop searching up the hierarchy whenever a logger with the "propagate" - attribute set to zero is found - that will be the last logger which - is checked for the existence of handlers. - """ - c = self - rv = False - while c: - if c.handlers: - rv = True - break - if not c.propagate: - break - else: - c = c.parent - return rv - - def callHandlers(self, record): - """ - Pass a record to all relevant handlers. - - Loop through all handlers for this logger and its parents in the - logger hierarchy. If no handler was found, output a one-off error - message to sys.stderr. Stop searching up the hierarchy whenever a - logger with the "propagate" attribute set to zero is found - that - will be the last logger whose handlers are called. - """ - c = self - found = 0 - while c: - for hdlr in c.handlers: - found = found + 1 - if record.levelno >= hdlr.level: - hdlr.handle(record) - if not c.propagate: - c = None #break out - else: - c = c.parent - if (found == 0): - if lastResort: - if record.levelno >= lastResort.level: - lastResort.handle(record) - elif raiseExceptions and not self.manager.emittedNoHandlerWarning: - sys.stderr.write("No handlers could be found for logger" - " \"%s\"\n" % self.name) - self.manager.emittedNoHandlerWarning = True - - def getEffectiveLevel(self): - """ - Get the effective level for this logger. - - Loop through this logger and its parents in the logger hierarchy, - looking for a non-zero logging level. Return the first one found. - """ - logger = self - while logger: - if logger.level: - return logger.level - logger = logger.parent - return NOTSET - - def isEnabledFor(self, level): - """ - Is this logger enabled for level 'level'? - """ - if self.manager.disable >= level: - return 0 - return level >= self.getEffectiveLevel() - - def getChild(self, suffix): - """ - Get a logger which is a descendant to this one. - - This is a convenience method, such that - - logging.getLogger('abc').getChild('def.ghi') - - is the same as - - logging.getLogger('abc.def.ghi') - - It's useful, for example, when the parent logger is named using - __name__ rather than a literal string. - """ - if self.root is not self: - suffix = '.'.join((self.name, suffix)) - return self.manager.getLogger(suffix) - -class RootLogger(Logger): - """ - A root logger is not that different to any other logger, except that - it must have a logging level and there is only one instance of it in - the hierarchy. - """ - def __init__(self, level): - """ - Initialize the logger with the name "root". - """ - Logger.__init__(self, "root", level) - -_loggerClass = Logger - -class LoggerAdapter(object): - """ - An adapter for loggers which makes it easier to specify contextual - information in logging output. - """ - - def __init__(self, logger, extra): - """ - Initialize the adapter with a logger and a dict-like object which - provides contextual information. This constructor signature allows - easy stacking of LoggerAdapters, if so desired. - - You can effectively pass keyword arguments as shown in the - following example: - - adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) - """ - self.logger = logger - self.extra = extra - - def process(self, msg, kwargs): - """ - Process the logging message and keyword arguments passed in to - a logging call to insert contextual information. You can either - manipulate the message itself, the keyword args or both. Return - the message and kwargs modified (or not) to suit your needs. - - Normally, you'll only need to override this one method in a - LoggerAdapter subclass for your specific needs. - """ - kwargs["extra"] = self.extra - return msg, kwargs - - # - # Boilerplate convenience methods - # - def debug(self, msg, *args, **kwargs): - """ - Delegate a debug call to the underlying logger. - """ - self.log(DEBUG, msg, *args, **kwargs) - - def info(self, msg, *args, **kwargs): - """ - Delegate an info call to the underlying logger. - """ - self.log(INFO, msg, *args, **kwargs) - - def warning(self, msg, *args, **kwargs): - """ - Delegate a warning call to the underlying logger. - """ - self.log(WARNING, msg, *args, **kwargs) - - warn = warning - - def error(self, msg, *args, **kwargs): - """ - Delegate an error call to the underlying logger. - """ - self.log(ERROR, msg, *args, **kwargs) - - def exception(self, msg, *args, **kwargs): - """ - Delegate an exception call to the underlying logger. - """ - kwargs["exc_info"] = 1 - self.log(ERROR, msg, *args, **kwargs) - - def critical(self, msg, *args, **kwargs): - """ - Delegate a critical call to the underlying logger. - """ - self.log(CRITICAL, msg, *args, **kwargs) - - def log(self, level, msg, *args, **kwargs): - """ - Delegate a log call to the underlying logger, after adding - contextual information from this adapter instance. - """ - if self.isEnabledFor(level): - msg, kwargs = self.process(msg, kwargs) - self.logger._log(level, msg, args, **kwargs) - - def isEnabledFor(self, level): - """ - Is this logger enabled for level 'level'? - """ - if self.logger.manager.disable >= level: - return False - return level >= self.getEffectiveLevel() - - def setLevel(self, level): - """ - Set the specified level on the underlying logger. - """ - self.logger.setLevel(level) - - def getEffectiveLevel(self): - """ - Get the effective level for the underlying logger. - """ - return self.logger.getEffectiveLevel() - - def hasHandlers(self): - """ - See if the underlying logger has any handlers. - """ - return self.logger.hasHandlers() - -root = RootLogger(WARNING) -Logger.root = root -Logger.manager = Manager(Logger.root) - -#--------------------------------------------------------------------------- -# Configuration classes and functions -#--------------------------------------------------------------------------- - -BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" - -def basicConfig(**kwargs): - """ - Do basic configuration for the logging system. - - This function does nothing if the root logger already has handlers - configured. It is a convenience method intended for use by simple scripts - to do one-shot configuration of the logging package. - - The default behaviour is to create a StreamHandler which writes to - sys.stderr, set a formatter using the BASIC_FORMAT format string, and - add the handler to the root logger. - - A number of optional keyword arguments may be specified, which can alter - the default behaviour. - - filename Specifies that a FileHandler be created, using the specified - filename, rather than a StreamHandler. - filemode Specifies the mode to open the file, if filename is specified - (if filemode is unspecified, it defaults to 'a'). - format Use the specified format string for the handler. - datefmt Use the specified date/time format. - style If a format string is specified, use this to specify the - type of format string (possible values '%', '{', '$', for - %-formatting, :meth:`str.format` and :class:`string.Template` - - defaults to '%'). - level Set the root logger level to the specified level. - stream Use the specified stream to initialize the StreamHandler. Note - that this argument is incompatible with 'filename' - if both - are present, 'stream' is ignored. - - Note that you could specify a stream created using open(filename, mode) - rather than passing the filename and mode in. However, it should be - remembered that StreamHandler does not close its stream (since it may be - using sys.stdout or sys.stderr), whereas FileHandler closes its stream - when the handler is closed. - - .. versionchanged: 3.2 - Added the ``style`` parameter. - """ - # Add thread safety in case someone mistakenly calls - # basicConfig() from multiple threads - _acquireLock() - try: - if len(root.handlers) == 0: - filename = kwargs.get("filename") - if filename: - mode = kwargs.get("filemode", 'a') - hdlr = FileHandler(filename, mode) - else: - stream = kwargs.get("stream") - hdlr = StreamHandler(stream) - fs = kwargs.get("format", BASIC_FORMAT) - dfs = kwargs.get("datefmt", None) - style = kwargs.get("style", '%') - fmt = Formatter(fs, dfs, style) - hdlr.setFormatter(fmt) - root.addHandler(hdlr) - level = kwargs.get("level") - if level is not None: - root.setLevel(level) - finally: - _releaseLock() - -#--------------------------------------------------------------------------- -# Utility functions at module level. -# Basically delegate everything to the root logger. -#--------------------------------------------------------------------------- - -def getLogger(name=None): - """ - Return a logger with the specified name, creating it if necessary. - - If no name is specified, return the root logger. - """ - if name: - return Logger.manager.getLogger(name) - else: - return root - -def critical(msg, *args, **kwargs): - """ - Log a message with severity 'CRITICAL' on the root logger. If the logger - has no handlers, call basicConfig() to add a console handler with a - pre-defined format. - """ - if len(root.handlers) == 0: - basicConfig() - root.critical(msg, *args, **kwargs) - -fatal = critical - -def error(msg, *args, **kwargs): - """ - Log a message with severity 'ERROR' on the root logger. If the logger has - no handlers, call basicConfig() to add a console handler with a pre-defined - format. - """ - if len(root.handlers) == 0: - basicConfig() - root.error(msg, *args, **kwargs) - -def exception(msg, *args, **kwargs): - """ - Log a message with severity 'ERROR' on the root logger, with exception - information. If the logger has no handlers, basicConfig() is called to add - a console handler with a pre-defined format. - """ - kwargs['exc_info'] = True - error(msg, *args, **kwargs) - -def warning(msg, *args, **kwargs): - """ - Log a message with severity 'WARNING' on the root logger. If the logger has - no handlers, call basicConfig() to add a console handler with a pre-defined - format. - """ - if len(root.handlers) == 0: - basicConfig() - root.warning(msg, *args, **kwargs) - -warn = warning - -def info(msg, *args, **kwargs): - """ - Log a message with severity 'INFO' on the root logger. If the logger has - no handlers, call basicConfig() to add a console handler with a pre-defined - format. - """ - if len(root.handlers) == 0: - basicConfig() - root.info(msg, *args, **kwargs) - -def debug(msg, *args, **kwargs): - """ - Log a message with severity 'DEBUG' on the root logger. If the logger has - no handlers, call basicConfig() to add a console handler with a pre-defined - format. - """ - if len(root.handlers) == 0: - basicConfig() - root.debug(msg, *args, **kwargs) - -def log(level, msg, *args, **kwargs): - """ - Log 'msg % args' with the integer severity 'level' on the root logger. If - the logger has no handlers, call basicConfig() to add a console handler - with a pre-defined format. - """ - if len(root.handlers) == 0: - basicConfig() - root.log(level, msg, *args, **kwargs) - -def disable(level): - """ - Disable all logging calls of severity 'level' and below. - """ - root.manager.disable = level - -def shutdown(handlerList=_handlerList): - """ - Perform any cleanup actions in the logging system (e.g. flushing - buffers). - - Should be called at application exit. - """ - for wr in reversed(handlerList[:]): - #errors might occur, for example, if files are locked - #we just ignore them if raiseExceptions is not set - try: - h = wr() - if h: - try: - h.acquire() - h.flush() - h.close() - except (IOError, ValueError): - # Ignore errors which might be caused - # because handlers have been closed but - # references to them are still around at - # application exit. - pass - finally: - h.release() - except: - if raiseExceptions: - raise - #else, swallow - -#Let's try and shutdown automatically on application exit... -import atexit -atexit.register(shutdown) - -# Null handler - -class NullHandler(Handler): - """ - This handler does nothing. It's intended to be used to avoid the - "No handlers could be found for logger XXX" one-off warning. This is - important for library code, which may contain code to log events. If a user - of the library does not configure logging, the one-off warning might be - produced; to avoid this, the library developer simply needs to instantiate - a NullHandler and add it to the top-level logger of the library module or - package. - """ - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - -# Warnings integration - -any _warnings_showwarning = None - -def _showwarning(message, category, filename, lineno, file=None, line=None): - """ - Implementation of showwarnings which redirects to logging, which will first - check to see if the file parameter is None. If a file is specified, it will - delegate to the original warnings implementation of showwarning. Otherwise, - it will call warnings.formatwarning and will log the resulting string to a - warnings logger named "py.warnings" with level logging.WARNING. - """ - if file is not None: - if _warnings_showwarning is not None: - _warnings_showwarning(message, category, filename, lineno, file, line) - else: - s = warnings.formatwarning(message, category, filename, lineno, line) - logger = getLogger("py.warnings") - if not logger.handlers: - logger.addHandler(NullHandler()) - logger.warning("%s", s) - -def captureWarnings(capture): - """ - If capture is true, redirect all warnings to the logging package. - If capture is False, ensure that warnings are not redirected to logging - but to their original destinations. - """ - global _warnings_showwarning - if capture: - if _warnings_showwarning is None: - _warnings_showwarning = warnings.showwarning - warnings.showwarning = _showwarning - else: - if _warnings_showwarning is not None: - warnings.showwarning = _warnings_showwarning - _warnings_showwarning = None diff --git a/lib-python/3.2/incomplete/urllib/parse.py b/lib-python/3.2/incomplete/urllib/parse.py deleted file mode 100644 index 4fa65c40582b..000000000000 --- a/lib-python/3.2/incomplete/urllib/parse.py +++ /dev/null @@ -1,980 +0,0 @@ -"""Parse (absolute and relative) URLs. - -urlparse module is based upon the following RFC specifications. - -RFC 3986 (STD66): "Uniform Resource Identifiers" by T. Berners-Lee, R. Fielding -and L. Masinter, January 2005. - -RFC 2732 : "Format for Literal IPv6 Addresses in URLs" by R.Hinden, B.Carpenter -and L.Masinter, December 1999. - -RFC 2396: "Uniform Resource Identifiers (URI)": Generic Syntax by T. -Berners-Lee, R. Fielding, and L. Masinter, August 1998. - -RFC 2368: "The mailto URL scheme", by P.Hoffman , L Masinter, J. Zawinski, July 1998. - -RFC 1808: "Relative Uniform Resource Locators", by R. Fielding, UC Irvine, June -1995. - -RFC 1738: "Uniform Resource Locators (URL)" by T. Berners-Lee, L. Masinter, M. -McCahill, December 1994 - -RFC 3986 is considered the current standard and any future changes to -urlparse module should conform with it. The urlparse module is -currently not entirely compliant with this RFC due to defacto -scenarios for parsing, and for backward compatibility purposes, some -parsing quirks from older RFCs are retained. The testcases in -test_urlparse.py provides a good indicator of parsing behavior. -""" - -import sys -import collections - -__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", - "urlsplit", "urlunsplit", "urlencode", "parse_qs", - "parse_qsl", "quote", "quote_plus", "quote_from_bytes", - "unquote", "unquote_plus", "unquote_to_bytes"] - -# A classification of schemes ('' means apply by default) -uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', - 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', '', 'sftp', - 'svn', 'svn+ssh'] -uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', - 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', - 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh'] -non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', - 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] -uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', - 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', '', 'sftp'] -uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] -uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', - 'nntp', 'wais', 'https', 'shttp', 'snews', - 'file', 'prospero', ''] - -# Characters valid in scheme names -scheme_chars = ('abcdefghijklmnopqrstuvwxyz' - 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - '0123456789' - '+-.') - -# XXX: Consider replacing with functools.lru_cache -MAX_CACHE_SIZE = 20 -_parse_cache = {} - -def clear_cache(): - """Clear the parse cache and the quoters cache.""" - _parse_cache.clear() - _safe_quoters.clear() - - -# Helpers for bytes handling -# For 3.2, we deliberately require applications that -# handle improperly quoted URLs to do their own -# decoding and encoding. If valid use cases are -# presented, we may relax this by using latin-1 -# decoding internally for 3.3 -_implicit_encoding = 'ascii' -_implicit_errors = 'strict' - -def _noop(obj): - return obj - -def _encode_result(obj, encoding=_implicit_encoding, - errors=_implicit_errors): - return obj.encode(encoding, errors) - -def _decode_args(args, encoding=_implicit_encoding, - errors=_implicit_errors): - return tuple(x.decode(encoding, errors) if x else '' for x in args) - -def _coerce_args(*args): - # Invokes decode if necessary to create str args - # and returns the coerced inputs along with - # an appropriate result coercion function - # - noop for str inputs - # - encoding function otherwise - str_input = isinstance(args[0], str) - for arg in args[1:]: - # We special-case the empty string to support the - # "scheme=''" default argument to some functions - if arg and isinstance(arg, str) != str_input: - raise TypeError("Cannot mix str and non-str arguments") - if str_input: - return args + (_noop,) - return _decode_args(args) + (_encode_result,) - -# Result objects are more helpful than simple tuples -class _ResultMixinStr(object): - """Standard approach to encoding parsed results from str to bytes""" - __slots__ = () - - def encode(self, encoding='ascii', errors='strict'): - return self._encoded_counterpart(*(x.encode(encoding, errors) for x in self)) - - -class _ResultMixinBytes(object): - """Standard approach to decoding parsed results from bytes to str""" - __slots__ = () - - def decode(self, encoding='ascii', errors='strict'): - return self._decoded_counterpart(*(x.decode(encoding, errors) for x in self)) - - -class _NetlocResultMixinBase(object): - """Shared methods for the parsed result objects containing a netloc element""" - __slots__ = () - - @property - def username(self): - return self._userinfo[0] - - @property - def password(self): - return self._userinfo[1] - - @property - def hostname(self): - hostname = self._hostinfo[0] - if not hostname: - hostname = None - elif hostname is not None: - hostname = hostname.lower() - return hostname - - @property - def port(self): - port = self._hostinfo[1] - if port is not None: - port = int(port, 10) - return port - - -class _NetlocResultMixinStr(_NetlocResultMixinBase, _ResultMixinStr): - __slots__ = () - - @property - def _userinfo(self): - netloc = self.netloc - userinfo, have_info, hostinfo = netloc.rpartition('@') - if have_info: - username, have_password, password = userinfo.partition(':') - if not have_password: - password = None - else: - username = password = None - return username, password - - @property - def _hostinfo(self): - netloc = self.netloc - _, _, hostinfo = netloc.rpartition('@') - _, have_open_br, bracketed = hostinfo.partition('[') - if have_open_br: - hostname, _, port = bracketed.partition(']') - _, have_port, port = port.partition(':') - else: - hostname, have_port, port = hostinfo.partition(':') - if not have_port: - port = None - return hostname, port - - -class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes): - __slots__ = () - - @property - def _userinfo(self): - netloc = self.netloc - userinfo, have_info, hostinfo = netloc.rpartition(b'@') - if have_info: - username, have_password, password = userinfo.partition(b':') - if not have_password: - password = None - else: - username = password = None - return username, password - - @property - def _hostinfo(self): - netloc = self.netloc - _, _, hostinfo = netloc.rpartition(b'@') - _, have_open_br, bracketed = hostinfo.partition(b'[') - if have_open_br: - hostname, _, port = bracketed.partition(b']') - _, have_port, port = port.partition(b':') - else: - hostname, have_port, port = hostinfo.partition(b':') - if not have_port: - port = None - return hostname, port - - -from collections import namedtuple - -_DefragResultBase = namedtuple('DefragResult', 'url fragment') -_SplitResultBase = namedtuple('SplitResult', 'scheme netloc path query fragment') -_ParseResultBase = namedtuple('ParseResult', 'scheme netloc path params query fragment') - -# For backwards compatibility, alias _NetlocResultMixinStr -# ResultBase is no longer part of the documented API, but it is -# retained since deprecating it isn't worth the hassle -ResultBase = _NetlocResultMixinStr - -# Structured result objects for string data -class DefragResult(_DefragResultBase, _ResultMixinStr): - __slots__ = () - def geturl(self): - if self.fragment: - return self.url + '#' + self.fragment - else: - return self.url - -class SplitResult(_SplitResultBase, _NetlocResultMixinStr): - __slots__ = () - def geturl(self): - return urlunsplit(self) - -class ParseResult(_ParseResultBase, _NetlocResultMixinStr): - __slots__ = () - def geturl(self): - return urlunparse(self) - -# Structured result objects for bytes data -class DefragResultBytes(_DefragResultBase, _ResultMixinBytes): - __slots__ = () - def geturl(self): - if self.fragment: - return self.url + b'#' + self.fragment - else: - return self.url - -class SplitResultBytes(_SplitResultBase, _NetlocResultMixinBytes): - __slots__ = () - def geturl(self): - return urlunsplit(self) - -class ParseResultBytes(_ParseResultBase, _NetlocResultMixinBytes): - __slots__ = () - def geturl(self): - return urlunparse(self) - -# Set up the encode/decode result pairs -def _fix_result_transcoding(): - _result_pairs = ( - (DefragResult, DefragResultBytes), - (SplitResult, SplitResultBytes), - (ParseResult, ParseResultBytes), - ) - for _decoded, _encoded in _result_pairs: - _decoded._encoded_counterpart = _encoded - _encoded._decoded_counterpart = _decoded - -_fix_result_transcoding() -del _fix_result_transcoding - -def urlparse(url, scheme='', allow_fragments=True): - """Parse a URL into 6 components: - :///;?# - Return a 6-tuple: (scheme, netloc, path, params, query, fragment). - Note that we don't break the components up in smaller bits - (e.g. netloc is a single string) and we don't expand % escapes.""" - url, scheme, _coerce_result = _coerce_args(url, scheme) - tuple = urlsplit(url, scheme, allow_fragments) - scheme, netloc, url, query, fragment = tuple - if scheme in uses_params and ';' in url: - url, params = _splitparams(url) - else: - params = '' - result = ParseResult(scheme, netloc, url, params, query, fragment) - return _coerce_result(result) - -def _splitparams(url): - if '/' in url: - i = url.find(';', url.rfind('/')) - if i < 0: - return url, '' - else: - i = url.find(';') - return url[:i], url[i+1:] - -def _splitnetloc(url, start=0): - delim = len(url) # position of end of domain part of url, default is end - for c in '/?#': # look for delimiters; the order is NOT important - wdelim = url.find(c, start) # find first of this delim - if wdelim >= 0: # if found - delim = min(delim, wdelim) # use earliest delim position - return url[start:delim], url[delim:] # return (domain, rest) - -def urlsplit(url, scheme='', allow_fragments=True): - """Parse a URL into 5 components: - :///?# - Return a 5-tuple: (scheme, netloc, path, query, fragment). - Note that we don't break the components up in smaller bits - (e.g. netloc is a single string) and we don't expand % escapes.""" - url, scheme, _coerce_result = _coerce_args(url, scheme) - allow_fragments = bool(allow_fragments) - key = url, scheme, allow_fragments, type(url), type(scheme) - cached = _parse_cache.get(key, None) - if cached: - return _coerce_result(cached) - if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth - clear_cache() - netloc = query = fragment = '' - i = url.find(':') - if i > 0: - if url[:i] == 'http': # optimize the common case - scheme = url[:i].lower() - url = url[i+1:] - if url[:2] == '//': - netloc, url = _splitnetloc(url, 2) - if (('[' in netloc and ']' not in netloc) or - (']' in netloc and '[' not in netloc)): - raise ValueError("Invalid IPv6 URL") - if allow_fragments and '#' in url: - url, fragment = url.split('#', 1) - if '?' in url: - url, query = url.split('?', 1) - v = SplitResult(scheme, netloc, url, query, fragment) - _parse_cache[key] = v - return _coerce_result(v) - for c in url[:i]: - if c not in scheme_chars: - break - else: - try: - # make sure "url" is not actually a port number (in which case - # "scheme" is really part of the path - _testportnum = int(url[i+1:]) - except ValueError: - scheme, url = url[:i].lower(), url[i+1:] - - if url[:2] == '//': - netloc, url = _splitnetloc(url, 2) - if (('[' in netloc and ']' not in netloc) or - (']' in netloc and '[' not in netloc)): - raise ValueError("Invalid IPv6 URL") - if allow_fragments and scheme in uses_fragment and '#' in url: - url, fragment = url.split('#', 1) - if scheme in uses_query and '?' in url: - url, query = url.split('?', 1) - v = SplitResult(scheme, netloc, url, query, fragment) - _parse_cache[key] = v - return _coerce_result(v) - -def urlunparse(components): - """Put a parsed URL back together again. This may result in a - slightly different, but equivalent URL, if the URL that was parsed - originally had redundant delimiters, e.g. a ? with an empty query - (the draft states that these are equivalent).""" - scheme, netloc, url, params, query, fragment, _coerce_result = ( - _coerce_args(*components)) - if params: - url = "%s;%s" % (url, params) - return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment))) - -def urlunsplit(components): - """Combine the elements of a tuple as returned by urlsplit() into a - complete URL as a string. The data argument can be any five-item iterable. - This may result in a slightly different, but equivalent URL, if the URL that - was parsed originally had unnecessary delimiters (for example, a ? with an - empty query; the RFC states that these are equivalent).""" - scheme, netloc, url, query, fragment, _coerce_result = ( - _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): - if url and url[:1] != '/': url = '/' + url - url = '//' + (netloc or '') + url - if scheme: - url = scheme + ':' + url - if query: - url = url + '?' + query - if fragment: - url = url + '#' + fragment - return _coerce_result(url) - -def urljoin(base, url, allow_fragments=True): - """Join a base URL and a possibly relative URL to form an absolute - interpretation of the latter.""" - if not base: - return url - if not url: - return base - base, url, _coerce_result = _coerce_args(base, url) - bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) - scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) - if scheme != bscheme or scheme not in uses_relative: - return _coerce_result(url) - if scheme in uses_netloc: - if netloc: - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) - netloc = bnetloc - if path[:1] == '/': - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) - if not path and not params: - path = bpath - params = bparams - if not query: - query = bquery - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) - segments = bpath.split('/')[:-1] + path.split('/') - # XXX The stuff below is bogus in various ways... - if segments[-1] == '.': - segments[-1] = '' - while '.' in segments: - segments.remove('.') - while 1: - i = 1 - n = len(segments) - 1 - while i < n: - if (segments[i] == '..' - and segments[i-1] not in ('', '..')): - del segments[i-1:i+1] - break - i = i+1 - else: - break - if segments == ['', '..']: - segments[-1] = '' - elif len(segments) >= 2 and segments[-1] == '..': - segments[-2:] = [''] - return _coerce_result(urlunparse((scheme, netloc, '/'.join(segments), - params, query, fragment))) - -def urldefrag(url): - """Removes any existing fragment from URL. - - Returns a tuple of the defragmented URL and the fragment. If - the URL contained no fragments, the second element is the - empty string. - """ - url, _coerce_result = _coerce_args(url) - if '#' in url: - s, n, p, a, q, frag = urlparse(url) - defrag = urlunparse((s, n, p, a, q, '')) - else: - frag = '' - defrag = url - return _coerce_result(DefragResult(defrag, frag)) - -def unquote_to_bytes(string): - """unquote_to_bytes('abc%20def') -> b'abc def'.""" - # Note: strings are encoded as UTF-8. This is only an issue if it contains - # unescaped non-ASCII characters, which URIs should not. - if not string: - # Is it a string-like object? - string.split - return b'' - if isinstance(string, str): - string = string.encode('utf-8') - res = string.split(b'%') - if len(res) == 1: - return string - string = res[0] - for item in res[1:]: - try: - string += bytes([int(item[:2], 16)]) + item[2:] - except ValueError: - string += b'%' + item - return string - -def unquote(string, encoding='utf-8', errors='replace'): - """Replace %xx escapes by their single-character equivalent. The optional - encoding and errors parameters specify how to decode percent-encoded - sequences into Unicode characters, as accepted by the bytes.decode() - method. - By default, percent-encoded sequences are decoded with UTF-8, and invalid - sequences are replaced by a placeholder character. - - unquote('abc%20def') -> 'abc def'. - """ - if string == '': - return string - res = string.split('%') - if len(res) == 1: - return string - if encoding is None: - encoding = 'utf-8' - if errors is None: - errors = 'replace' - # pct_sequence: contiguous sequence of percent-encoded bytes, decoded - pct_sequence = b'' - string = res[0] - for item in res[1:]: - try: - if not item: - raise ValueError - pct_sequence += bytes.fromhex(item[:2]) - rest = item[2:] - if not rest: - # This segment was just a single percent-encoded character. - # May be part of a sequence of code units, so delay decoding. - # (Stored in pct_sequence). - continue - except ValueError: - rest = '%' + item - # Encountered non-percent-encoded characters. Flush the current - # pct_sequence. - string += pct_sequence.decode(encoding, errors) + rest - pct_sequence = b'' - if pct_sequence: - # Flush the final pct_sequence - string += pct_sequence.decode(encoding, errors) - return string - -def parse_qs(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): - """Parse a query given as a string argument. - - Arguments: - - qs: percent-encoded query string to be parsed - - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. - """ - dict = {} - pairs = parse_qsl(qs, keep_blank_values, strict_parsing, - encoding=encoding, errors=errors) - for name, value in pairs: - if name in dict: - dict[name].append(value) - else: - dict[name] = [value] - return dict - -def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): - """Parse a query given as a string argument. - - Arguments: - - qs: percent-encoded query string to be parsed - - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. - - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. - - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. - - Returns a list, as G-d intended. - """ - qs, _coerce_result = _coerce_args(qs) - pairs = [] - for s1 in qs.split('&'): - for s2 in s1.split(';'): - pairs.append(s2) - r = [] - for name_value in pairs: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: - raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = unquote(name, encoding=encoding, errors=errors) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = unquote(value, encoding=encoding, errors=errors) - value = _coerce_result(value) - r.append((name, value)) - return r - -def unquote_plus(string, encoding='utf-8', errors='replace'): - """Like unquote(), but also replace plus signs by spaces, as required for - unquoting HTML form values. - - unquote_plus('%7e/abc+def') -> '~/abc def' - """ - string = string.replace('+', ' ') - return unquote(string, encoding, errors) - -_ALWAYS_SAFE = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - b'abcdefghijklmnopqrstuvwxyz' - b'0123456789' - b'_.-') -_ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) -_safe_quoters = {} - -class Quoter(collections.defaultdict): - """A mapping from bytes (in range(0,256)) to strings. - - String values are percent-encoded byte values, unless the key < 128, and - in the "safe" set (either the specified safe set, or default set). - """ - # Keeps a cache internally, using defaultdict, for efficiency (lookups - # of cached keys don't call Python code at all). - def __init__(self, safe): - """safe: bytes object.""" - self.safe = _ALWAYS_SAFE.union(safe) - - def __repr__(self): - # Without this, will just display as a defaultdict - return "" % dict(self) - - def __missing__(self, b): - # Handle a cache miss. Store quoted string in cache and return. - res = chr(b) if b in self.safe else '%{:02X}'.format(b) - self[b] = res - return res - -def quote(string, safe='/', encoding=None, errors=None): - """quote('abc def') -> 'abc%20def' - - Each part of a URL, e.g. the path info, the query, etc., has a - different set of reserved characters that must be quoted. - - RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists - the following reserved characters. - - reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | - "$" | "," - - Each of these characters is reserved in some component of a URL, - but not necessarily in all of them. - - By default, the quote function is intended for quoting the path - section of a URL. Thus, it will not encode '/'. This character - is reserved, but in typical usage the quote function is being - called on a path where the existing slash characters are used as - reserved characters. - - string and safe may be either str or bytes objects. encoding must - not be specified if string is a str. - - The optional encoding and errors parameters specify how to deal with - non-ASCII characters, as accepted by the str.encode method. - By default, encoding='utf-8' (characters are encoded with UTF-8), and - errors='strict' (unsupported characters raise a UnicodeEncodeError). - """ - if isinstance(string, str): - if not string: - return string - if encoding is None: - encoding = 'utf-8' - if errors is None: - errors = 'strict' - string = string.encode(encoding, errors) - else: - if encoding is not None: - raise TypeError("quote() doesn't support 'encoding' for bytes") - if errors is not None: - raise TypeError("quote() doesn't support 'errors' for bytes") - return quote_from_bytes(string, safe) - -def quote_plus(string, safe='', encoding=None, errors=None): - """Like quote(), but also replace ' ' with '+', as required for quoting - HTML form values. Plus signs in the original string are escaped unless - they are included in safe. It also does not have safe default to '/'. - """ - # Check if ' ' in string, where string may either be a str or bytes. If - # there are no spaces, the regular quote will produce the right answer. - if ((isinstance(string, str) and ' ' not in string) or - (isinstance(string, bytes) and b' ' not in string)): - return quote(string, safe, encoding, errors) - if isinstance(safe, str): - space = ' ' - else: - space = b' ' - string = quote(string, safe + space, encoding, errors) - return string.replace(' ', '+') - -def quote_from_bytes(bs, safe='/'): - """Like quote(), but accepts a bytes object rather than a str, and does - not perform string-to-bytes encoding. It always returns an ASCII string. - quote_from_bytes(b'abc def\xab') -> 'abc%20def%AB' - """ - if not isinstance(bs, (bytes, bytearray)): - raise TypeError("quote_from_bytes() expected bytes") - if not bs: - return '' - if isinstance(safe, str): - # Normalize 'safe' by converting to bytes and removing non-ASCII chars - safe = safe.encode('ascii', 'ignore') - else: - safe = bytes([c for c in safe if c < 128]) - if not bs.rstrip(_ALWAYS_SAFE_BYTES + safe): - return bs.decode() - try: - quoter = _safe_quoters[safe] - except KeyError: - _safe_quoters[safe] = quoter = Quoter(safe).__getitem__ - return ''.join([quoter(char) for char in bs]) - -def urlencode(query, doseq=False, safe='', encoding=None, errors=None): - """Encode a sequence of two-element tuples or dictionary into a URL query string. - - If any values in the query arg are sequences and doseq is true, each - sequence element is converted to a separate parameter. - - If the query arg is a sequence of two-element tuples, the order of the - parameters in the output will match the order of parameters in the - input. - - The query arg may be either a string or a bytes type. When query arg is a - string, the safe, encoding and error parameters are sent the quote_plus for - encoding. - """ - - if hasattr(query, "items"): - query = query.items() - else: - # It's a bother at times that strings and string-like objects are - # sequences. - try: - # non-sequence items should not work with len() - # non-empty strings will fail this - if len(query) and not isinstance(query[0], tuple): - raise TypeError - # Zero-length sequences of all types will get here and succeed, - # but that's a minor nit. Since the original implementation - # allowed empty dicts that type of behavior probably should be - # preserved for consistency - except TypeError: - ty, va, tb = sys.exc_info() - raise TypeError("not a valid non-string sequence " - "or mapping object").with_traceback(tb) - - l = [] - if not doseq: - for k, v in query: - if isinstance(k, bytes): - k = quote_plus(k, safe) - else: - k = quote_plus(str(k), safe, encoding, errors) - - if isinstance(v, bytes): - v = quote_plus(v, safe) - else: - v = quote_plus(str(v), safe, encoding, errors) - l.append(k + '=' + v) - else: - for k, v in query: - if isinstance(k, bytes): - k = quote_plus(k, safe) - else: - k = quote_plus(str(k), safe, encoding, errors) - - if isinstance(v, bytes): - v = quote_plus(v, safe) - l.append(k + '=' + v) - elif isinstance(v, str): - v = quote_plus(v, safe, encoding, errors) - l.append(k + '=' + v) - else: - try: - # Is this a sufficient test for sequence-ness? - x = len(v) - except TypeError: - # not a sequence - v = quote_plus(str(v), safe, encoding, errors) - l.append(k + '=' + v) - else: - # loop over the sequence - for elt in v: - if isinstance(elt, bytes): - elt = quote_plus(elt, safe) - else: - elt = quote_plus(str(elt), safe, encoding, errors) - l.append(k + '=' + elt) - return '&'.join(l) - -# Utilities to parse URLs (most of these return None for missing parts): -# unwrap('') --> 'type://host/path' -# splittype('type:opaquestring') --> 'type', 'opaquestring' -# splithost('//host[:port]/path') --> 'host[:port]', '/path' -# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]' -# splitpasswd('user:passwd') -> 'user', 'passwd' -# splitport('host:port') --> 'host', 'port' -# splitquery('/path?query') --> '/path', 'query' -# splittag('/path#tag') --> '/path', 'tag' -# splitattr('/path;attr1=value1;attr2=value2;...') -> -# '/path', ['attr1=value1', 'attr2=value2', ...] -# splitvalue('attr=value') --> 'attr', 'value' -# urllib.parse.unquote('abc%20def') -> 'abc def' -# quote('abc def') -> 'abc%20def') - -def to_bytes(url): - """to_bytes(u"URL") --> 'URL'.""" - # Most URL schemes require ASCII. If that changes, the conversion - # can be relaxed. - # XXX get rid of to_bytes() - if isinstance(url, str): - try: - url = url.encode("ASCII").decode() - except UnicodeError: - raise UnicodeError("URL " + repr(url) + - " contains non-ASCII characters") - return url - -def unwrap(url): - """unwrap('') --> 'type://host/path'.""" - url = str(url).strip() - if url[:1] == '<' and url[-1:] == '>': - url = url[1:-1].strip() - if url[:4] == 'URL:': url = url[4:].strip() - return url - -_typeprog = None -def splittype(url): - """splittype('type:opaquestring') --> 'type', 'opaquestring'.""" - global _typeprog - if _typeprog is None: - import re - _typeprog = re.compile('^([^/:]+):') - - match = _typeprog.match(url) - if match: - scheme = match.group(1) - return scheme.lower(), url[len(scheme) + 1:] - return None, url - -_hostprog = None -def splithost(url): - """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" - global _hostprog - if _hostprog is None: - import re - _hostprog = re.compile('^//([^/?]*)(.*)$') - - match = _hostprog.match(url) - if match: - host_port = match.group(1) - path = match.group(2) - if path and not path.startswith('/'): - path = '/' + path - return host_port, path - return None, url - -_userprog = None -def splituser(host): - """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" - global _userprog - if _userprog is None: - import re - _userprog = re.compile('^(.*)@(.*)$') - - match = _userprog.match(host) - if match: return match.group(1, 2) - return None, host - -_passwdprog = None -def splitpasswd(user): - """splitpasswd('user:passwd') -> 'user', 'passwd'.""" - global _passwdprog - if _passwdprog is None: - import re - _passwdprog = re.compile('^([^:]*):(.*)$',re.S) - - match = _passwdprog.match(user) - if match: return match.group(1, 2) - return user, None - -# splittag('/path#tag') --> '/path', 'tag' -_portprog = None -def splitport(host): - """splitport('host:port') --> 'host', 'port'.""" - global _portprog - if _portprog is None: - import re - _portprog = re.compile('^(.*):([0-9]+)$') - - match = _portprog.match(host) - if match: return match.group(1, 2) - return host, None - -_nportprog = None -def splitnport(host, defport=-1): - """Split host and port, returning numeric port. - Return given default port if no ':' found; defaults to -1. - Return numerical port if a valid number are found after ':'. - Return None if ':' but not a valid number.""" - global _nportprog - if _nportprog is None: - import re - _nportprog = re.compile('^(.*):(.*)$') - - match = _nportprog.match(host) - if match: - host, port = match.group(1, 2) - try: - if not port: raise ValueError("no digits") - nport = int(port) - except ValueError: - nport = None - return host, nport - return host, defport - -_queryprog = None -def splitquery(url): - """splitquery('/path?query') --> '/path', 'query'.""" - global _queryprog - if _queryprog is None: - import re - _queryprog = re.compile('^(.*)\?([^?]*)$') - - match = _queryprog.match(url) - if match: return match.group(1, 2) - return url, None - -_tagprog = None -def splittag(url): - """splittag('/path#tag') --> '/path', 'tag'.""" - global _tagprog - if _tagprog is None: - import re - _tagprog = re.compile('^(.*)#([^#]*)$') - - match = _tagprog.match(url) - if match: return match.group(1, 2) - return url, None - -def splitattr(url): - """splitattr('/path;attr1=value1;attr2=value2;...') -> - '/path', ['attr1=value1', 'attr2=value2', ...].""" - words = url.split(';') - return words[0], words[1:] - -_valueprog = None -def splitvalue(attr): - """splitvalue('attr=value') --> 'attr', 'value'.""" - global _valueprog - if _valueprog is None: - import re - _valueprog = re.compile('^([^=]*)=(.*)$') - - match = _valueprog.match(attr) - if match: return match.group(1, 2) - return attr, None diff --git a/lib-python/3.2/posixpath.py b/lib-python/3.2/posixpath.py deleted file mode 100644 index f0f14a5dd5d3..000000000000 --- a/lib-python/3.2/posixpath.py +++ /dev/null @@ -1,467 +0,0 @@ -"""Common operations on Posix pathnames. - -Instead of importing this module directly, import os and refer to -this module as os.path. The "os.path" name is an alias for this -module on Posix systems; on other systems (e.g. Mac, Windows), -os.path provides the same operations in a manner specific to that -platform, and is an alias to another module (e.g. macpath, ntpath). - -Some of this can actually be useful on non-Posix systems too, e.g. -for manipulation of the pathname component of URLs. -""" - -import os -import sys -import stat -import genericpath -from genericpath import * - -from typing import ( - Tuple, BinaryIO, TextIO, Pattern, AnyStr, List, Set, Any, Union, Undefined -) - -__all__ = ["normcase","isabs","join","splitdrive","split","splitext", - "basename","dirname","commonprefix","getsize","getmtime", - "getatime","getctime","islink","exists","lexists","isdir","isfile", - "ismount", "expanduser","expandvars","normpath","abspath", - "samefile","sameopenfile","samestat", - "curdir","pardir","sep","pathsep","defpath","altsep","extsep", - "devnull","realpath","supports_unicode_filenames","relpath"] - -# Strings representing various path-related bits and pieces. -# These are primarily for export; internally, they are hardcoded. -curdir = '.' -pardir = '..' -extsep = '.' -sep = '/' -pathsep = ':' -defpath = ':/bin:/usr/bin' -altsep = None # type: str -devnull = '/dev/null' - -def _get_sep(path: AnyStr) -> AnyStr: - if isinstance(path, bytes): - return b'/' - else: - return '/' - -# Normalize the case of a pathname. Trivial in Posix, string.lower on Mac. -# On MS-DOS this may also turn slashes into backslashes; however, other -# normalizations (such as optimizing '../' away) are not allowed -# (another function should be defined to do that). - -def normcase(s: AnyStr) -> AnyStr: - """Normalize case of pathname. Has no effect under Posix""" - # TODO: on Mac OS X, this should really return s.lower(). - if not isinstance(s, (bytes, str)): - raise TypeError("normcase() argument must be str or bytes, " - "not '{}'".format(s.__class__.__name__)) - return s - - -# Return whether a path is absolute. -# Trivial in Posix, harder on the Mac or MS-DOS. - -def isabs(s: AnyStr) -> bool: - """Test whether a path is absolute""" - sep = _get_sep(s) - return s.startswith(sep) - - -# Join pathnames. -# Ignore the previous parts if a part is absolute. -# Insert a '/' unless the first part is empty or already ends in '/'. - -def join(a: AnyStr, *p: AnyStr) -> AnyStr: - """Join two or more pathname components, inserting '/' as needed. - If any component is an absolute path, all previous path components - will be discarded.""" - sep = _get_sep(a) - path = a - for b in p: - if b.startswith(sep): - path = b - elif not path or path.endswith(sep): - path += b - else: - path += sep + b - return path - - -# Split a path in head (everything up to the last '/') and tail (the -# rest). If the path ends in '/', tail will be empty. If there is no -# '/' in the path, head will be empty. -# Trailing '/'es are stripped from head unless it is the root. - -def split(p: AnyStr) -> Tuple[AnyStr, AnyStr]: - """Split a pathname. Returns tuple "(head, tail)" where "tail" is - everything after the final slash. Either part may be empty.""" - sep = _get_sep(p) - i = p.rfind(sep) + 1 - head, tail = p[:i], p[i:] - if head and head != sep*len(head): - head = head.rstrip(sep) - return head, tail - - -# Split a path in root and extension. -# The extension is everything starting at the last dot in the last -# pathname component; the root is everything before that. -# It is always true that root + ext == p. - -def splitext(p: AnyStr) -> Tuple[AnyStr, AnyStr]: - if isinstance(p, bytes): - sep = b'/' - extsep = b'.' - else: - sep = '/' - extsep = '.' - return genericpath._splitext(p, sep, None, extsep) -splitext.__doc__ = genericpath._splitext.__doc__ - -# Split a pathname into a drive specification and the rest of the -# path. Useful on DOS/Windows/NT; on Unix, the drive is always empty. - -def splitdrive(p: AnyStr) -> Tuple[AnyStr, AnyStr]: - """Split a pathname into drive and path. On Posix, drive is always - empty.""" - return p[:0], p - - -# Return the tail (basename) part of a path, same as split(path)[1]. - -def basename(p: AnyStr) -> AnyStr: - """Returns the final component of a pathname""" - sep = _get_sep(p) - i = p.rfind(sep) + 1 - return p[i:] - - -# Return the head (dirname) part of a path, same as split(path)[0]. - -def dirname(p: AnyStr) -> AnyStr: - """Returns the directory component of a pathname""" - sep = _get_sep(p) - i = p.rfind(sep) + 1 - head = p[:i] - if head and head != sep*len(head): - head = head.rstrip(sep) - return head - - -# Is a path a symbolic link? -# This will always return false on systems where os.lstat doesn't exist. - -def islink(path: AnyStr) -> bool: - """Test whether a path is a symbolic link""" - try: - st = os.lstat(path) - except (os.error, AttributeError): - return False - return stat.S_ISLNK(st.st_mode) - -# Being true for dangling symbolic links is also useful. - -def lexists(path: AnyStr) -> bool: - """Test whether a path exists. Returns True for broken symbolic links""" - try: - os.lstat(path) - except os.error: - return False - return True - - -# Are two filenames really pointing to the same file? - -def samefile(f1: AnyStr, f2: AnyStr) -> bool: - """Test whether two pathnames reference the same actual file""" - s1 = os.stat(f1) - s2 = os.stat(f2) - return samestat(s1, s2) - - -# Are two open files really referencing the same file? -# (Not necessarily the same file descriptor!) - -def sameopenfile(fp1: int, fp2: int) -> bool: - """Test whether two open file objects reference the same file""" - s1 = os.fstat(fp1) - s2 = os.fstat(fp2) - return samestat(s1, s2) - - -# Are two stat buffers (obtained from stat, fstat or lstat) -# describing the same file? - -def samestat(s1: os.stat_result, s2: os.stat_result) -> bool: - """Test whether two stat buffers reference the same file""" - return s1.st_ino == s2.st_ino and \ - s1.st_dev == s2.st_dev - - -# Is a path a mount point? -# (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) - -def ismount(path: AnyStr) -> bool: - """Test whether a path is a mount point""" - if islink(path): - # A symlink can never be a mount point - return False - try: - s1 = os.lstat(path) - if isinstance(path, bytes): - parent = join(path, b'..') - else: - parent = join(path, '..') - s2 = os.lstat(parent) - except os.error: - return False # It doesn't exist -- so not a mount point :-) - dev1 = s1.st_dev - dev2 = s2.st_dev - if dev1 != dev2: - return True # path/.. on a different device as path - ino1 = s1.st_ino - ino2 = s2.st_ino - if ino1 == ino2: - return True # path/.. is the same i-node as path - return False - - -# Expand paths beginning with '~' or '~user'. -# '~' means $HOME; '~user' means that user's home directory. -# If the path doesn't begin with '~', or if the user or $HOME is unknown, -# the path is returned unchanged (leaving error reporting to whatever -# function is called with the expanded path as argument). -# See also module 'glob' for expansion of *, ? and [...] in pathnames. -# (A function should also be defined to do full *sh-style environment -# variable expansion.) - -def expanduser(path: AnyStr) -> AnyStr: - """Expand ~ and ~user constructions. If user or $HOME is unknown, - do nothing.""" - if isinstance(path, bytes): - tilde = b'~' - else: - tilde = '~' - if not path.startswith(tilde): - return path - sep = _get_sep(path) - i = path.find(sep, 1) - if i < 0: - i = len(path) - userhome = Undefined(Union[str, bytes]) - if i == 1: - if 'HOME' not in os.environ: - import pwd - userhome = pwd.getpwuid(os.getuid()).pw_dir - else: - userhome = os.environ['HOME'] - else: - import pwd - name = Undefined(Union[str, bytes]) - name = path[1:i] - if isinstance(name, bytes): - name = str(name, 'ASCII') - try: - pwent = pwd.getpwnam(name) - except KeyError: - return path - userhome = pwent.pw_dir - if isinstance(path, bytes): - userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root - - -# Expand paths containing shell variable substitutions. -# This expands the forms $variable and ${variable} only. -# Non-existent variables are left unchanged. - -_varprog = None # type: Pattern[str] -_varprogb = None # type: Pattern[bytes] - -def expandvars(path: AnyStr) -> AnyStr: - """Expand shell variables of form $var and ${var}. Unknown variables - are left unchanged.""" - global _varprog, _varprogb - if isinstance(path, bytes): - if b'$' not in path: - return path - if not _varprogb: - import re - _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprogb.search - start = b'{' - end = b'}' - else: - if '$' not in path: - return path - if not _varprog: - import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprog.search - start = '{' - end = '}' - i = 0 - while True: - m = search(path, i) - if not m: - break - i, j = m.span(0) - name = Undefined(Union[str, bytes]) - name = m.group(1) - if name.startswith(start) and name.endswith(end): - name = name[1:-1] - if isinstance(name, bytes): - name = str(name, 'ASCII') - if name in os.environ: - tail = path[j:] - value = Undefined(Union[str, bytes]) - value = os.environ[name] - if isinstance(path, bytes): - value = value.encode('ASCII') - path = path[:i] + value - i = len(path) - path += tail - else: - i = j - return path - - -# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. -# It should be understood that this may change the meaning of the path -# if it contains symbolic links! - -def normpath(path: AnyStr) -> AnyStr: - """Normalize path, eliminating double slashes, etc.""" - if isinstance(path, bytes): - sep = b'/' - empty = b'' - dot = b'.' - dotdot = b'..' - else: - sep = '/' - empty = '' - dot = '.' - dotdot = '..' - if path == empty: - return dot - initial_slashes = path.startswith(sep) # type: int - # POSIX allows one or two initial slashes, but treats three or more - # as single slash. - if (initial_slashes and - path.startswith(sep*2) and not path.startswith(sep*3)): - initial_slashes = 2 - comps = path.split(sep) - new_comps = List[AnyStr]() - for comp in comps: - if comp in (empty, dot): - continue - if (comp != dotdot or (not initial_slashes and not new_comps) or - (new_comps and new_comps[-1] == dotdot)): - new_comps.append(comp) - elif new_comps: - new_comps.pop() - comps = new_comps - path = sep.join(comps) - if initial_slashes: - path = sep*initial_slashes + path - return path or dot - - -def abspath(path: AnyStr) -> AnyStr: - """Return an absolute path.""" - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) - return normpath(path) - - -# Return a canonical path (i.e. the absolute location of a file on the -# filesystem). - -def realpath(filename: AnyStr) -> AnyStr: - """Return the canonical path of the specified filename, eliminating any -symbolic links encountered in the path.""" - if isinstance(filename, bytes): - sep = b'/' - empty = b'' - else: - sep = '/' - empty = '' - if isabs(filename): - bits = [sep] + filename.split(sep)[1:] - else: - bits = [empty] + filename.split(sep) - - for i in range(2, len(bits)+1): - component = join(*bits[0:i]) - # Resolve symbolic links. - if islink(component): - resolved = _resolve_link(component) - if resolved is None: - # Infinite loop -- return original component + rest of the path - return abspath(join(*([component] + bits[i:]))) - else: - newpath = join(*([resolved] + bits[i:])) - return realpath(newpath) - - return abspath(filename) - - -def _resolve_link(path: AnyStr) -> AnyStr: - """Internal helper function. Takes a path and follows symlinks - until we either arrive at something that isn't a symlink, or - encounter a path we've seen before (meaning that there's a loop). - """ - paths_seen = Set[AnyStr]() - while islink(path): - if path in paths_seen: - # Already seen this path, so we must have a symlink loop - return None - paths_seen.add(path) - # Resolve where the link points to - resolved = os.readlink(path) - if not isabs(resolved): - dir = dirname(path) - path = normpath(join(dir, resolved)) - else: - path = normpath(resolved) - return path - -supports_unicode_filenames = (sys.platform == 'darwin') - -def relpath(path: AnyStr, start: AnyStr = None) -> AnyStr: - """Return a relative version of a path""" - - if not path: - raise ValueError("no path specified") - - if isinstance(path, bytes): - curdir = b'.' - sep = b'/' - pardir = b'..' - else: - curdir = '.' - sep = '/' - pardir = '..' - - if start is None: - start = curdir - - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] - - # Work out how much of the filepath is shared by start and path. - i = len(commonprefix([start_list, path_list])) - - rel_list = [pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return curdir - return join(*rel_list) diff --git a/lib-python/3.2/pprint.py b/lib-python/3.2/pprint.py deleted file mode 100644 index b5b9a2ef95c0..000000000000 --- a/lib-python/3.2/pprint.py +++ /dev/null @@ -1,380 +0,0 @@ -# Author: Fred L. Drake, Jr. -# fdrake@acm.org -# -# This is a simple little module I wrote to make life easier. I didn't -# see anything quite like it in the library, though I may have overlooked -# something. I wrote this when I was trying to read some heavily nested -# tuples with fairly non-descriptive content. This is modeled very much -# after Lisp/Scheme - style pretty-printing of lists. If you find it -# useful, thank small children who sleep at night. - -"""Support to pretty-print lists, tuples, & dictionaries recursively. - -Very simple, but useful, especially in debugging data structures. - -Classes -------- - -PrettyPrinter() - Handle pretty-printing operations onto a stream using a configured - set of formatting parameters. - -Functions ---------- - -pformat() - Format a Python object into a pretty-printed representation. - -pprint() - Pretty-print a Python object to a stream [default is sys.stdout]. - -saferepr() - Generate a 'standard' repr()-like value, but protect against recursive - data structures. - -""" - -import sys as _sys -from collections import OrderedDict as _OrderedDict -from io import StringIO as _StringIO - -from typing import Any, Tuple, Dict, TextIO, cast, List - -__all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", - "PrettyPrinter"] - -# cache these for faster access: -_commajoin = ", ".join -_id = id -_len = len -_type = type - - -def pprint(object: object, stream: TextIO = None, indent: int = 1, - width: int = 80, depth: int = None) -> None: - """Pretty-print a Python object to a stream [default is sys.stdout].""" - printer = PrettyPrinter( - stream=stream, indent=indent, width=width, depth=depth) - printer.pprint(object) - -def pformat(object: object, indent: int = 1, width: int = 80, - depth: int = None) -> str: - """Format a Python object into a pretty-printed representation.""" - return PrettyPrinter(indent=indent, width=width, depth=depth).pformat(object) - -def saferepr(object: object) -> str: - """Version of repr() which can handle recursive data structures.""" - return _safe_repr(object, {}, None, 0)[0] - -def isreadable(object: object) -> bool: - """Determine if saferepr(object) is readable by eval().""" - return _safe_repr(object, {}, None, 0)[1] - -def isrecursive(object: object) -> bool: - """Determine if object requires a recursive representation.""" - return _safe_repr(object, {}, None, 0)[2] - -class _safe_key: - """Helper function for key functions when sorting unorderable objects. - - The wrapped-object will fallback to an Py2.x style comparison for - unorderable types (sorting first comparing the type name and then by - the obj ids). Does not work recursively, so dict.items() must have - _safe_key applied to both the key and the value. - - """ - - __slots__ = ['obj'] - - def __init__(self, obj: Any) -> None: - self.obj = obj - - def __lt__(self, other: Any) -> Any: - rv = self.obj.__lt__(other.obj) # type: Any - if rv is NotImplemented: - rv = (str(type(self.obj)), id(self.obj)) < \ - (str(type(other.obj)), id(other.obj)) - return rv - -def _safe_tuple(t: Tuple[Any, Any]) -> Tuple[_safe_key, _safe_key]: - "Helper function for comparing 2-tuples" - return _safe_key(t[0]), _safe_key(t[1]) - -class PrettyPrinter: - def __init__(self, indent: int = 1, width: int = 80, depth: int = None, - stream: TextIO = None) -> None: - """Handle pretty printing operations onto a stream using a set of - configured parameters. - - indent - Number of spaces to indent for each level of nesting. - - width - Attempted maximum number of columns in the output. - - depth - The maximum depth to print out nested structures. - - stream - The desired output stream. If omitted (or false), the standard - output stream available at construction will be used. - - """ - indent = int(indent) - width = int(width) - assert indent >= 0, "indent must be >= 0" - assert depth is None or depth > 0, "depth must be > 0" - assert width, "width must be != 0" - self._depth = depth - self._indent_per_level = indent - self._width = width - if stream is not None: - self._stream = stream - else: - self._stream = _sys.stdout - - def pprint(self, object: object) -> None: - self._format(object, self._stream, 0, 0, {}, 0) - self._stream.write("\n") - - def pformat(self, object: object) -> str: - sio = _StringIO() - self._format(object, sio, 0, 0, {}, 0) - return sio.getvalue() - - def isrecursive(self, object: object) -> int: - return self.format(object, {}, 0, 0)[2] - - def isreadable(self, object: object) -> int: - s, readable, recursive = self.format(object, {}, 0, 0) - return readable and not recursive - - def _format(self, object: object, stream: TextIO, indent: int, - allowance: int, context: Dict[int, int], level: int) -> None: - level = level + 1 - objid = _id(object) - if objid in context: - stream.write(_recursion(object)) - self._recursive = True - self._readable = False - return - rep = self._repr(object, context, level - 1) - typ = _type(object) - sepLines = _len(rep) > (self._width - 1 - indent - allowance) - write = stream.write - - if self._depth and level > self._depth: - write(rep) - return - - if sepLines: - r = getattr(typ, "__repr__", None) - if isinstance(object, dict): - write('{') - if self._indent_per_level > 1: - write((self._indent_per_level - 1) * ' ') - length = _len(object) - if length: - context[objid] = 1 - indent = indent + self._indent_per_level - if issubclass(typ, _OrderedDict): - items = list(object.items()) - else: - items = sorted(object.items(), key=_safe_tuple) - key, ent = items[0] - rep = self._repr(key, context, level) - write(rep) - write(': ') - self._format(ent, stream, indent + _len(rep) + 2, - allowance + 1, context, level) - if length > 1: - for key, ent in items[1:]: - rep = self._repr(key, context, level) - write(',\n%s%s: ' % (' '*indent, rep)) - self._format(ent, stream, indent + _len(rep) + 2, - allowance + 1, context, level) - indent = indent - self._indent_per_level - del context[objid] - write('}') - return - - if ((issubclass(typ, list) and r is list.__repr__) or - (issubclass(typ, tuple) and r is tuple.__repr__) or - (issubclass(typ, set) and r is set.__repr__) or - (issubclass(typ, frozenset) and r is frozenset.__repr__) - ): - anyobj = Any(object) # TODO Collection? - length = _len(anyobj) - if issubclass(typ, list): - write('[') - endchar = ']' - lst = anyobj - elif issubclass(typ, set): - if not length: - write('set()') - return - write('{') - endchar = '}' - lst = sorted(anyobj, key=_safe_key) - elif issubclass(typ, frozenset): - if not length: - write('frozenset()') - return - write('frozenset({') - endchar = '})' - lst = sorted(anyobj, key=_safe_key) - indent += 10 - else: - write('(') - endchar = ')' - lst = list(anyobj) - if self._indent_per_level > 1: - write((self._indent_per_level - 1) * ' ') - if length: - context[objid] = 1 - indent = indent + self._indent_per_level - self._format(lst[0], stream, indent, allowance + 1, - context, level) - if length > 1: - for ent in lst[1:]: - write(',\n' + ' '*indent) - self._format(ent, stream, indent, - allowance + 1, context, level) - indent = indent - self._indent_per_level - del context[objid] - if issubclass(typ, tuple) and length == 1: - write(',') - write(endchar) - return - - write(rep) - - def _repr(self, object: object, context: Dict[int, int], - level: int) -> str: - repr, readable, recursive = self.format(object, context.copy(), - self._depth, level) - if not readable: - self._readable = False - if recursive: - self._recursive = True - return repr - - def format(self, object: object, context: Dict[int, int], - maxlevels: int, level: int) -> Tuple[str, int, int]: - """Format object for a specific context, returning a string - and flags indicating whether the representation is 'readable' - and whether the object represents a recursive construct. - """ - return _safe_repr(object, context, maxlevels, level) - - -# Return triple (repr_string, isreadable, isrecursive). - -def _safe_repr(object: object, context: Dict[int, int], - maxlevels: int, level: int) -> Tuple[str, bool, bool]: - typ = _type(object) - if typ is str: - s = cast(str, object) - if 'locale' not in _sys.modules: - return repr(object), True, False - if "'" in s and '"' not in s: - closure = '"' - quotes = {'"': '\\"'} - else: - closure = "'" - quotes = {"'": "\\'"} - qget = quotes.get - sio = _StringIO() - write = sio.write - for char in s: - if char.isalpha(): - write(char) - else: - write(qget(char, repr(char)[1:-1])) - return ("%s%s%s" % (closure, sio.getvalue(), closure)), True, False - - r = getattr(typ, "__repr__", None) - if issubclass(typ, dict) and r is dict.__repr__: - if not object: - return "{}", True, False - objid = _id(object) - if maxlevels and level >= maxlevels: - return "{...}", False, objid in context - if objid in context: - return _recursion(object), False, True - context[objid] = 1 - readable = True - recursive = False - components = List[str]() - append = components.append - level += 1 - saferepr = _safe_repr - items = sorted((cast(dict, object)).items(), key=_safe_tuple) - for k, v in items: - krepr, kreadable, krecur = saferepr(k, context, maxlevels, level) - vrepr, vreadable, vrecur = saferepr(v, context, maxlevels, level) - append("%s: %s" % (krepr, vrepr)) - readable = readable and kreadable and vreadable - if krecur or vrecur: - recursive = True - del context[objid] - return "{%s}" % _commajoin(components), readable, recursive - - if (issubclass(typ, list) and r is list.__repr__) or \ - (issubclass(typ, tuple) and r is tuple.__repr__): - anyobj = Any(object) # TODO Sequence? - if issubclass(typ, list): - if not object: - return "[]", True, False - format = "[%s]" - elif _len(anyobj) == 1: - format = "(%s,)" - else: - if not object: - return "()", True, False - format = "(%s)" - objid = _id(object) - if maxlevels and level >= maxlevels: - return format % "...", False, objid in context - if objid in context: - return _recursion(object), False, True - context[objid] = 1 - readable = True - recursive = False - components = [] - append = components.append - level += 1 - for o in anyobj: - orepr, oreadable, orecur = _safe_repr(o, context, maxlevels, level) - append(orepr) - if not oreadable: - readable = False - if orecur: - recursive = True - del context[objid] - return format % _commajoin(components), readable, recursive - - rep = repr(object) - return rep, bool(rep and not rep.startswith('<')), False - - -def _recursion(object: object) -> str: - return ("" - % (_type(object).__name__, _id(object))) - - -def _perfcheck(object: object = None) -> None: - import time - if object is None: - object = [("string", (1, 2), [3, 4], {5: 6, 7: 8})] * 100000 - p = PrettyPrinter() - t1 = time.time() - _safe_repr(object, {}, None, 0) - t2 = time.time() - p.pformat(object) - t3 = time.time() - print("_safe_repr:", t2 - t1) - print("pformat:", t3 - t2) - -if __name__ == "__main__": - _perfcheck() diff --git a/lib-python/3.2/random.py b/lib-python/3.2/random.py deleted file mode 100644 index fca36696d513..000000000000 --- a/lib-python/3.2/random.py +++ /dev/null @@ -1,743 +0,0 @@ -"""Random variable generators. - - integers - -------- - uniform within range - - sequences - --------- - pick random element - pick random sample - generate random permutation - - distributions on the real line: - ------------------------------ - uniform - triangular - normal (Gaussian) - lognormal - negative exponential - gamma - beta - pareto - Weibull - - distributions on the circle (angles 0 to 2pi) - --------------------------------------------- - circular uniform - von Mises - -General notes on the underlying Mersenne Twister core generator: - -* The period is 2**19937-1. -* It is one of the most extensively tested generators in existence. -* The random() method is implemented in C, executes in a single Python step, - and is, therefore, threadsafe. - -""" - -from warnings import warn as _warn -from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethodType -from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil -from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin -from os import urandom as _urandom -from collections import Set as _Set, Sequence as _Sequence -from hashlib import sha512 as _sha512 - -from typing import ( - Any, typevar, Iterable, Sequence, List, Function, Set, cast, SupportsInt, Union -) - -__all__ = ["Random","seed","random","uniform","randint","choice","sample", - "randrange","shuffle","normalvariate","lognormvariate", - "expovariate","vonmisesvariate","gammavariate","triangular", - "gauss","betavariate","paretovariate","weibullvariate", - "getstate","setstate", "getrandbits", - "SystemRandom"] - -NV_MAGICCONST = 4 * _exp(-0.5)/_sqrt(2.0) -TWOPI = 2.0*_pi -LOG4 = _log(4.0) -SG_MAGICCONST = 1.0 + _log(4.5) -BPF = 53 # Number of bits in a float -RECIP_BPF = 2**-BPF # type: float - - -# Translated by Guido van Rossum from C source provided by -# Adrian Baddeley. Adapted by Raymond Hettinger for use with -# the Mersenne Twister and os.urandom() core generators. - -import _random - -T = typevar('T') - -class Random(_random.Random): - """Random number generator base class used by bound module functions. - - Used to instantiate instances of Random to get generators that don't - share state. - - Class Random can also be subclassed if you want to use a different basic - generator of your own devising: in that case, override the following - methods: random(), seed(), getstate(), and setstate(). - Optionally, implement a getrandbits() method so that randrange() - can cover arbitrarily large ranges. - - """ - - VERSION = 3 # used by getstate/setstate - gauss_next = 0.0 - - def __init__(self, x: object = None) -> None: - """Initialize an instance. - - Optional argument x controls seeding, as for Random.seed(). - """ - - self.seed(x) - self.gauss_next = None - - def seed(self, a: Any = None, version: int = 2) -> None: - """Initialize internal state from hashable object. - - None or no argument seeds from current time or from an operating - system specific randomness source if available. - - For version 2 (the default), all of the bits are used if *a *is a str, - bytes, or bytearray. For version 1, the hash() of *a* is used instead. - - If *a* is an int, all bits are used. - - """ - - if a is None: - try: - a = int.from_bytes(_urandom(32), 'big') - except NotImplementedError: - import time - a = int(time.time() * 256) # use fractional seconds - - if version == 2: - if isinstance(a, (str, bytes, bytearray)): - if isinstance(a, str): - a = a.encode() - a += _sha512(a).digest() - a = int.from_bytes(a, 'big') - - super().seed(a) - self.gauss_next = None - - def getstate(self) -> tuple: - """Return internal state; can be passed to setstate() later.""" - return self.VERSION, super().getstate(), self.gauss_next - - def setstate(self, state: tuple) -> None: - """Restore internal state from object returned by getstate().""" - version = state[0] - if version == 3: - version, internalstate, self.gauss_next = state - super().setstate(internalstate) - elif version == 2: - version, internalstate, self.gauss_next = state - # In version 2, the state was saved as signed ints, which causes - # inconsistencies between 32/64-bit systems. The state is - # really unsigned 32-bit ints, so we convert negative ints from - # version 2 to positive longs for version 3. - try: - internalstate = tuple(x % (2**32) for x in internalstate) - except ValueError as e: - raise TypeError() - super().setstate(internalstate) - else: - raise ValueError("state with version %s passed to " - "Random.setstate() of version %s" % - (version, self.VERSION)) - -## ---- Methods below this point do not need to be overridden when -## ---- subclassing for the purpose of using a different core generator. - -## -------------------- pickle support ------------------- - - def __getstate__(self) -> object: # for pickle - return self.getstate() - - def __setstate__(self, state: Any) -> None: # for pickle - self.setstate(state) - - def __reduce__(self) -> object: - return self.__class__, (), self.getstate() - -## -------------------- integer methods ------------------- - - def randrange(self, start: SupportsInt, stop: SupportsInt = None, - step: int = 1, int: Function[[SupportsInt], - int] = int) -> int: - """Choose a random item from range(start, stop[, step]). - - This fixes the problem with randint() which includes the - endpoint; in Python this is usually not what you want. - - Do not supply the 'int' argument. - """ - - # This code is a bit messy to make it fast for the - # common case while still doing adequate error checking. - istart = int(start) - if istart != start: - raise ValueError("non-integer arg 1 for randrange()") - if stop is None: - if istart > 0: - return self._randbelow(istart) - raise ValueError("empty range for randrange()") - - # stop argument supplied. - istop = int(stop) - if istop != stop: - raise ValueError("non-integer stop for randrange()") - width = istop - istart - if step == 1 and width > 0: - return istart + self._randbelow(width) - if step == 1: - raise ValueError("empty range for randrange() (%d,%d, %d)" % (istart, istop, width)) - - # Non-unit step argument supplied. - istep = int(step) - if istep != step: - raise ValueError("non-integer step for randrange()") - if istep > 0: - n = (width + istep - 1) // istep - elif istep < 0: - n = (width + istep + 1) // istep - else: - raise ValueError("zero step for randrange()") - - if n <= 0: - raise ValueError("empty range for randrange()") - - return istart + istep*self._randbelow(n) - - def randint(self, a: int, b: int) -> int: - """Return random integer in range [a, b], including both end points. - """ - - return self.randrange(a, b+1) - - def _randbelow(self, n: int, int: Function[[float], int] = int, - maxsize: int = 1< int: - "Return a random int in the range [0,n). Raises ValueError if n==0." - - getrandbits = self.getrandbits - # Only call self.getrandbits if the original random() builtin method - # has not been overridden or if a new getrandbits() was supplied. - if type(self.random) is BuiltinMethod or type(getrandbits) is Method: - k = n.bit_length() # don't use (n-1) here because n can be 1 - r = getrandbits(k) # 0 <= r < 2**k - while r >= n: - r = getrandbits(k) - return r - # There's an overriden random() method but no new getrandbits() method, - # so we can only use random() from here. - random = self.random - if n >= maxsize: - _warn("Underlying random() generator does not supply \n" - "enough bits to choose from a population range this large.\n" - "To remove the range limitation, add a getrandbits() method.") - return int(random() * n) - rem = maxsize % n - limit = (maxsize - rem) / maxsize # int(limit * maxsize) % n == 0 - s = random() - while s >= limit: - s = random() - return int(s*maxsize) % n - -## -------------------- sequence methods ------------------- - - def choice(self, seq: Sequence[T]) -> T: - """Choose a random element from a non-empty sequence.""" - try: - i = self._randbelow(len(seq)) - except ValueError: - raise IndexError('Cannot choose from an empty sequence') - return seq[i] - - def shuffle(self, x: List[T], - random: Function[[], float] = None, - int: Function[[float], int] = int) -> None: - """x, random=random.random -> shuffle list x in place; return None. - - Optional arg random is a 0-argument function returning a random - float in [0.0, 1.0); by default, the standard random.random. - """ - - randbelow = self._randbelow - for i in reversed(range(1, len(x))): - # pick an element in x[:i+1] with which to exchange x[i] - j = randbelow(i+1) if random is None else int(random() * (i+1)) - x[i], x[j] = x[j], x[i] - - def sample(self, population: Union[_Set[T], _Sequence[T]], k: int) -> List[T]: - """Chooses k unique random elements from a population sequence or set. - - Returns a new list containing elements from the population while - leaving the original population unchanged. The resulting list is - in selection order so that all sub-slices will also be valid random - samples. This allows raffle winners (the sample) to be partitioned - into grand prize and second place winners (the subslices). - - Members of the population need not be hashable or unique. If the - population contains repeats, then each occurrence is a possible - selection in the sample. - - To choose a sample in a range of integers, use range as an argument. - This is especially fast and space efficient for sampling from a - large population: sample(range(10000000), 60) - """ - - # Sampling without replacement entails tracking either potential - # selections (the pool) in a list or previous selections in a set. - - # When the number of selections is small compared to the - # population, then tracking selections is efficient, requiring - # only a small set and an occasional reselection. For - # a larger number of selections, the pool tracking method is - # preferred since the list takes less space than the - # set and it doesn't suffer from frequent reselections. - - if isinstance(population, _Set): - population = list(population) - if not isinstance(population, _Sequence): - raise TypeError("Population must be a sequence or set. For dicts, use list(d).") - randbelow = self._randbelow - n = len(population) - if not (0 <= k and k <= n): - raise ValueError("Sample larger than population") - result = [cast(T, None)] * k - setsize = 21 # size of a small set minus size of an empty list - if k > 5: - setsize += 4 ** _ceil(_log(k * 3, 4)) # table size for big sets - if n <= setsize: - # An n-length list is smaller than a k-length set - pool = list(population) - for i in range(k): # invariant: non-selected at [0,n-i) - j = randbelow(n-i) - result[i] = pool[j] - pool[j] = pool[n-i-1] # move non-selected item into vacancy - else: - selected = Set[int]() - selected_add = selected.add - for i in range(k): - j = randbelow(n) - while j in selected: - j = randbelow(n) - selected_add(j) - result[i] = population[j] - return result - -## -------------------- real-valued distributions ------------------- - -## -------------------- uniform distribution ------------------- - - def uniform(self, a: float, b: float) -> float: - "Get a random number in the range [a, b) or [a, b] depending on rounding." - return a + (b-a) * self.random() - -## -------------------- triangular -------------------- - - def triangular(self, low: float = 0.0, high: float = 1.0, - mode: float = None) -> float: - """Triangular distribution. - - Continuous distribution bounded by given lower and upper limits, - and having a given mode value in-between. - - http://en.wikipedia.org/wiki/Triangular_distribution - - """ - u = self.random() - c = 0.5 if mode is None else (mode - low) / (high - low) - if u > c: - u = 1.0 - u - c = 1.0 - c - low, high = high, low - return low + (high - low) * (u * c) ** 0.5 - -## -------------------- normal distribution -------------------- - - def normalvariate(self, mu: float, sigma: float) -> float: - """Normal distribution. - - mu is the mean, and sigma is the standard deviation. - - """ - # mu = mean, sigma = standard deviation - - # Uses Kinderman and Monahan method. Reference: Kinderman, - # A.J. and Monahan, J.F., "Computer generation of random - # variables using the ratio of uniform deviates", ACM Trans - # Math Software, 3, (1977), pp257-260. - - random = self.random - while 1: - u1 = random() - u2 = 1.0 - random() - z = NV_MAGICCONST*(u1-0.5)/u2 - zz = z*z/4.0 - if zz <= -_log(u2): - break - return mu + z*sigma - -## -------------------- lognormal distribution -------------------- - - def lognormvariate(self, mu: float, sigma: float) -> float: - """Log normal distribution. - - If you take the natural logarithm of this distribution, you'll get a - normal distribution with mean mu and standard deviation sigma. - mu can have any value, and sigma must be greater than zero. - - """ - return _exp(self.normalvariate(mu, sigma)) - -## -------------------- exponential distribution -------------------- - - def expovariate(self, lambd: float) -> float: - """Exponential distribution. - - lambd is 1.0 divided by the desired mean. It should be - nonzero. (The parameter would be called "lambda", but that is - a reserved word in Python.) Returned values range from 0 to - positive infinity if lambd is positive, and from negative - infinity to 0 if lambd is negative. - - """ - # lambd: rate lambd = 1/mean - # ('lambda' is a Python reserved word) - - # we use 1-random() instead of random() to preclude the - # possibility of taking the log of zero. - return -_log(1.0 - self.random())/lambd - -## -------------------- von Mises distribution -------------------- - - def vonmisesvariate(self, mu: float, kappa: float) -> float: - """Circular data distribution. - - mu is the mean angle, expressed in radians between 0 and 2*pi, and - kappa is the concentration parameter, which must be greater than or - equal to zero. If kappa is equal to zero, this distribution reduces - to a uniform random angle over the range 0 to 2*pi. - - """ - # mu: mean angle (in radians between 0 and 2*pi) - # kappa: concentration parameter kappa (>= 0) - # if kappa = 0 generate uniform random angle - - # Based upon an algorithm published in: Fisher, N.I., - # "Statistical Analysis of Circular Data", Cambridge - # University Press, 1993. - - # Thanks to Magnus Kessler for a correction to the - # implementation of step 4. - - random = self.random - if kappa <= 1e-6: - return TWOPI * random() - - a = 1.0 + _sqrt(1.0 + 4.0 * kappa * kappa) - b = (a - _sqrt(2.0 * a))/(2.0 * kappa) - r = (1.0 + b * b)/(2.0 * b) - - while 1: - u1 = random() - - z = _cos(_pi * u1) - f = (1.0 + r * z)/(r + z) - c = kappa * (r - f) - - u2 = random() - - if u2 < c * (2.0 - c) or u2 <= c * _exp(1.0 - c): - break - - u3 = random() - if u3 > 0.5: - theta = (mu % TWOPI) + _acos(f) - else: - theta = (mu % TWOPI) - _acos(f) - - return theta - -## -------------------- gamma distribution -------------------- - - def gammavariate(self, alpha: float, beta: float) -> float: - """Gamma distribution. Not the gamma function! - - Conditions on the parameters are alpha > 0 and beta > 0. - - The probability distribution function is: - - x ** (alpha - 1) * math.exp(-x / beta) - pdf(x) = -------------------------------------- - math.gamma(alpha) * beta ** alpha - - """ - - # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 - - # Warning: a few older sources define the gamma distribution in terms - # of alpha > -1.0 - if alpha <= 0.0 or beta <= 0.0: - raise ValueError('gammavariate: alpha and beta must be > 0.0') - - random = self.random - if alpha > 1.0: - - # Uses R.C.H. Cheng, "The generation of Gamma - # variables with non-integral shape parameters", - # Applied Statistics, (1977), 26, No. 1, p71-74 - - ainv = _sqrt(2.0 * alpha - 1.0) - bbb = alpha - LOG4 - ccc = alpha + ainv - - while 1: - u1 = random() - if not (1e-7 < u1 and u1 < .9999999): - continue - u2 = 1.0 - random() - v = _log(u1/(1.0-u1))/ainv - x = alpha*_exp(v) - z = u1*u1*u2 - r = bbb+ccc*v-x - if r + SG_MAGICCONST - 4.5*z >= 0.0 or r >= _log(z): - return x * beta - - elif alpha == 1.0: - # expovariate(1) - u = random() - while u <= 1e-7: - u = random() - return -_log(u) * beta - - else: # alpha is between 0 and 1 (exclusive) - - # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle - - while 1: - u = random() - b = (_e + alpha)/_e - p = b*u - if p <= 1.0: - x = p ** (1.0/alpha) - else: - x = -_log((b-p)/alpha) - u1 = random() - if p > 1.0: - if u1 <= x ** (alpha - 1.0): - break - elif u1 <= _exp(-x): - break - return x * beta - -## -------------------- Gauss (faster alternative) -------------------- - - def gauss(self, mu: float, sigma: float) -> float: - """Gaussian distribution. - - mu is the mean, and sigma is the standard deviation. This is - slightly faster than the normalvariate() function. - - Not thread-safe without a lock around calls. - - """ - - # When x and y are two variables from [0, 1), uniformly - # distributed, then - # - # cos(2*pi*x)*sqrt(-2*log(1-y)) - # sin(2*pi*x)*sqrt(-2*log(1-y)) - # - # are two *independent* variables with normal distribution - # (mu = 0, sigma = 1). - # (Lambert Meertens) - # (corrected version; bug discovered by Mike Miller, fixed by LM) - - # Multithreading note: When two threads call this function - # simultaneously, it is possible that they will receive the - # same return value. The window is very small though. To - # avoid this, you have to use a lock around all calls. (I - # didn't want to slow this down in the serial case by using a - # lock here.) - - random = self.random - z = self.gauss_next - self.gauss_next = None - if z is None: - x2pi = random() * TWOPI - g2rad = _sqrt(-2.0 * _log(1.0 - random())) - z = _cos(x2pi) * g2rad - self.gauss_next = _sin(x2pi) * g2rad - - return mu + z*sigma - -## -------------------- beta -------------------- -## See -## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html -## for Ivan Frohne's insightful analysis of why the original implementation: -## -## def betavariate(self, alpha, beta): -## # Discrete Event Simulation in C, pp 87-88. -## -## y = self.expovariate(alpha) -## z = self.expovariate(1.0/beta) -## return z/(y+z) -## -## was dead wrong, and how it probably got that way. - - def betavariate(self, alpha: float, beta: float) -> 'float': - """Beta distribution. - - Conditions on the parameters are alpha > 0 and beta > 0. - Returned values range between 0 and 1. - - """ - - # This version due to Janne Sinkkonen, and matches all the std - # texts (e.g., Knuth Vol 2 Ed 3 pg 134 "the beta distribution"). - y = self.gammavariate(alpha, 1.) - if y == 0: - return 0.0 - else: - return y / (y + self.gammavariate(beta, 1.)) - -## -------------------- Pareto -------------------- - - def paretovariate(self, alpha: float) -> float: - """Pareto distribution. alpha is the shape parameter.""" - # Jain, pg. 495 - - u = 1.0 - self.random() - return 1.0 / u ** (1.0/alpha) - -## -------------------- Weibull -------------------- - - def weibullvariate(self, alpha: float, beta: float) -> float: - """Weibull distribution. - - alpha is the scale parameter and beta is the shape parameter. - - """ - # Jain, pg. 499; bug fix courtesy Bill Arms - - u = 1.0 - self.random() - return alpha * (-_log(u)) ** (1.0/beta) - -## --------------- Operating System Random Source ------------------ - -class SystemRandom(Random): - """Alternate random number generator using sources provided - by the operating system (such as /dev/urandom on Unix or - CryptGenRandom on Windows). - - Not available on all systems (see os.urandom() for details). - """ - - def random(self) -> float: - """Get the next random number in the range [0.0, 1.0).""" - return (int.from_bytes(_urandom(7), 'big') >> 3) * RECIP_BPF - - def getrandbits(self, k: int) -> int: - """getrandbits(k) -> x. Generates a long int with k random bits.""" - if k <= 0: - raise ValueError('number of bits must be greater than zero') - if k != int(k): - raise TypeError('number of bits should be an integer') - numbytes = (k + 7) // 8 # bits / 8 and rounded up - x = int.from_bytes(_urandom(numbytes), 'big') - return x >> (numbytes * 8 - k) # trim excess bits - - def seed(self, a: object = None, version: int = None) -> None: - "Stub method. Not used for a system random number generator." - return - - def _notimplemented(self, *args: Any, **kwds: Any) -> Any: - "Method should not be called for a system random number generator." - raise NotImplementedError('System entropy source does not have state.') - getstate = setstate = _notimplemented - -# Create one instance, seeded from current time, and export its methods -# as module-level functions. The functions share state across all uses -#(both in the user's code and in the Python libraries), but that's fine -# for most programs and is easier for the casual user than making them -# instantiate their own Random() instance. - -_inst = Random() -seed = _inst.seed -random = _inst.random -uniform = _inst.uniform -triangular = _inst.triangular -randint = _inst.randint -choice = _inst.choice -randrange = _inst.randrange -sample = _inst.sample -shuffle = _inst.shuffle -normalvariate = _inst.normalvariate -lognormvariate = _inst.lognormvariate -expovariate = _inst.expovariate -vonmisesvariate = _inst.vonmisesvariate -gammavariate = _inst.gammavariate -gauss = _inst.gauss -betavariate = _inst.betavariate -paretovariate = _inst.paretovariate -weibullvariate = _inst.weibullvariate -getstate = _inst.getstate -setstate = _inst.setstate -getrandbits = _inst.getrandbits - -## -------------------- test program -------------------- - -def _test_generator(n: int, func: Any, args: tuple) -> None: - import time - print(n, 'times', func.__name__) - total = 0.0 - sqsum = 0.0 - smallest = 1e10 - largest = -1e10 - t0 = time.time() - for i in range(n): - x = func(*args) # type: float - total += x - sqsum = sqsum + x*x - smallest = min(x, smallest) - largest = max(x, largest) - t1 = time.time() - print(round(t1-t0, 3), 'sec,', end=' ') - avg = total/n - stddev = _sqrt(sqsum/n - avg*avg) - print('avg %g, stddev %g, min %g, max %g' % \ - (avg, stddev, smallest, largest)) - - -def _test(N: int = 2000) -> None: - _test_generator(N, random, ()) - _test_generator(N, normalvariate, (0.0, 1.0)) - _test_generator(N, lognormvariate, (0.0, 1.0)) - _test_generator(N, vonmisesvariate, (0.0, 1.0)) - _test_generator(N, gammavariate, (0.01, 1.0)) - _test_generator(N, gammavariate, (0.1, 1.0)) - _test_generator(N, gammavariate, (0.1, 2.0)) - _test_generator(N, gammavariate, (0.5, 1.0)) - _test_generator(N, gammavariate, (0.9, 1.0)) - _test_generator(N, gammavariate, (1.0, 1.0)) - _test_generator(N, gammavariate, (2.0, 1.0)) - _test_generator(N, gammavariate, (20.0, 1.0)) - _test_generator(N, gammavariate, (200.0, 1.0)) - _test_generator(N, gauss, (0.0, 1.0)) - _test_generator(N, betavariate, (3.0, 3.0)) - _test_generator(N, triangular, (0.0, 1.0, 1.0/3.0)) - -if __name__ == '__main__': - _test() diff --git a/lib-python/3.2/shutil.py b/lib-python/3.2/shutil.py deleted file mode 100644 index 4e138a71b942..000000000000 --- a/lib-python/3.2/shutil.py +++ /dev/null @@ -1,790 +0,0 @@ -"""Utility functions for copying and archiving files and directory trees. - -XXX The functions here don't copy the resource fork or other metadata on Mac. - -""" - -import os -import sys -import stat -from os.path import abspath -import fnmatch -import collections -import errno -import tarfile -import builtins - -from typing import ( - Any, AnyStr, IO, List, Iterable, Function, Tuple, Dict, Sequence, cast, - Traceback -) - -try: - import bz2 - _BZ2_SUPPORTED = True -except ImportError: - _BZ2_SUPPORTED = False - -try: - from pwd import getpwnam as _getpwnam - getpwnam = _getpwnam -except ImportError: - getpwnam = None - -try: - from grp import getgrnam as _getgrnam - getgrnam = _getgrnam -except ImportError: - getgrnam = None - -__all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", - "copytree", "move", "rmtree", "Error", "SpecialFileError", - "ExecError", "make_archive", "get_archive_formats", - "register_archive_format", "unregister_archive_format", - "get_unpack_formats", "register_unpack_format", - "unregister_unpack_format", "unpack_archive", "ignore_patterns"] - -class Error(EnvironmentError): - pass - -class SpecialFileError(EnvironmentError): - """Raised when trying to do a kind of operation (e.g. copying) which is - not supported on a special file (e.g. a named pipe)""" - -class ExecError(EnvironmentError): - """Raised when a command could not be executed""" - -class ReadError(EnvironmentError): - """Raised when an archive cannot be read""" - -class RegistryError(Exception): - """Raised when a registery operation with the archiving - and unpacking registeries fails""" - - -try: - _WindowsError = WindowsError # type: type -except NameError: - _WindowsError = None - - -# Function aliases to be patched in test cases -rename = os.rename -open = builtins.open - - -def copyfileobj(fsrc: IO[AnyStr], fdst: IO[AnyStr], - length: int = 16*1024) -> None: - """copy data from file-like object fsrc to file-like object fdst""" - while 1: - buf = fsrc.read(length) - if not buf: - break - fdst.write(buf) - -def _samefile(src: str, dst: str) -> bool: - # Macintosh, Unix. - if hasattr(os.path, 'samefile'): - try: - return os.path.samefile(src, dst) - except OSError: - return False - - # All other platforms: check for same pathname. - return (os.path.normcase(os.path.abspath(src)) == - os.path.normcase(os.path.abspath(dst))) - -def copyfile(src: str, dst: str) -> None: - """Copy data from src to dst""" - if _samefile(src, dst): - raise Error("`%s` and `%s` are the same file" % (src, dst)) - - for fn in [src, dst]: - try: - st = os.stat(fn) - except OSError: - # File most likely does not exist - pass - else: - # XXX What about other special files? (sockets, devices...) - if stat.S_ISFIFO(st.st_mode): - raise SpecialFileError("`%s` is a named pipe" % fn) - - with open(src, 'rb') as fsrc: - with open(dst, 'wb') as fdst: - copyfileobj(fsrc, fdst) - -def copymode(src: str, dst: str) -> None: - """Copy mode bits from src to dst""" - if hasattr(os, 'chmod'): - st = os.stat(src) - mode = stat.S_IMODE(st.st_mode) - os.chmod(dst, mode) - -def copystat(src: str, dst: str) -> None: - """Copy all stat info (mode bits, atime, mtime, flags) from src to dst""" - st = os.stat(src) - mode = stat.S_IMODE(st.st_mode) - if hasattr(os, 'utime'): - os.utime(dst, (st.st_atime, st.st_mtime)) - if hasattr(os, 'chmod'): - os.chmod(dst, mode) - if hasattr(os, 'chflags') and hasattr(st, 'st_flags'): - try: - os.chflags(dst, st.st_flags) - except OSError as why: - if (not hasattr(errno, 'EOPNOTSUPP') or - why.errno != errno.EOPNOTSUPP): - raise - -def copy(src: str, dst: str) -> None: - """Copy data and mode bits ("cp src dst"). - - The destination may be a directory. - - """ - if os.path.isdir(dst): - dst = os.path.join(dst, os.path.basename(src)) - copyfile(src, dst) - copymode(src, dst) - -def copy2(src: str, dst: str) -> None: - """Copy data and all stat info ("cp -p src dst"). - - The destination may be a directory. - - """ - if os.path.isdir(dst): - dst = os.path.join(dst, os.path.basename(src)) - copyfile(src, dst) - copystat(src, dst) - -def ignore_patterns(*patterns: str) -> Function[[str, List[str]], - Iterable[str]]: - """Function that can be used as copytree() ignore parameter. - - Patterns is a sequence of glob-style patterns - that are used to exclude files""" - def _ignore_patterns(path: str, names: List[str]) -> Iterable[str]: - ignored_names = List[str]() - for pattern in patterns: - ignored_names.extend(fnmatch.filter(names, pattern)) - return set(ignored_names) - return _ignore_patterns - -def copytree(src: str, dst: str, symlinks: bool = False, - ignore: Function[[str, List[str]], Iterable[str]] = None, - copy_function: Function[[str, str], None] = copy2, - ignore_dangling_symlinks: bool = False) -> None: - """Recursively copy a directory tree. - - The destination directory must not already exist. - If exception(s) occur, an Error is raised with a list of reasons. - - If the optional symlinks flag is true, symbolic links in the - source tree result in symbolic links in the destination tree; if - it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't - exist, an exception will be added in the list of errors raised in - an Error exception at the end of the copy process. - - You can set the optional ignore_dangling_symlinks flag to true if you - want to silence this exception. Notice that this has no effect on - platforms that don't support os.symlink. - - The optional ignore argument is a callable. If given, it - is called with the `src` parameter, which is the directory - being visited by copytree(), and `names` which is the list of - `src` contents, as returned by os.listdir(): - - callable(src, names) -> ignored_names - - Since copytree() is called recursively, the callable will be - called once for each directory that is copied. It returns a - list of names relative to the `src` directory that should - not be copied. - - The optional copy_function argument is a callable that will be used - to copy each file. It will be called with the source path and the - destination path as arguments. By default, copy2() is used, but any - function that supports the same signature (like copy()) can be used. - - """ - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - os.makedirs(dst) - errors = List[Tuple[str, str, str]]() - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if os.path.islink(srcname): - linkto = os.readlink(srcname) - if symlinks: - os.symlink(linkto, dstname) - else: - # ignore dangling symlink if the flag is on - if not os.path.exists(linkto) and ignore_dangling_symlinks: - continue - # otherwise let the copy occurs. copy2 will raise an error - copy_function(srcname, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks, ignore, copy_function) - else: - # Will raise a SpecialFileError for unsupported file types - copy_function(srcname, dstname) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error as err: - errors.extend(err.args[0]) - except EnvironmentError as why: - errors.append((srcname, dstname, str(why))) - try: - copystat(src, dst) - except OSError as why: - if _WindowsError is not None and isinstance(why, _WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.append((src, dst, str(why))) - if errors: - raise Error(errors) - -def rmtree(path: str, ignore_errors: bool = False, - onerror: Function[[Any, str, Tuple[type, BaseException, Traceback]], - None] = None) -> None: - """Recursively delete a directory tree. - - If ignore_errors is set, errors are ignored; otherwise, if onerror - is set, it is called to handle the error with arguments (func, - path, exc_info) where func is os.listdir, os.remove, or os.rmdir; - path is the argument to that function that caused it to fail; and - exc_info is a tuple returned by sys.exc_info(). If ignore_errors - is false and onerror is None, an exception is raised. - - """ - if ignore_errors: - def _onerror(x: Any, y: str, - z: Tuple[type, BaseException, Traceback]) -> None: - pass - onerror = _onerror - elif onerror is None: - def __onerror(x: Any, y: str, - z: Tuple[type, BaseException, Traceback]) -> None: - raise - onerror = __onerror - try: - if os.path.islink(path): - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError: - onerror(os.path.islink, path, sys.exc_info()) - # can't continue even if onerror hook returns - return - names = List[str]() - try: - names = os.listdir(path) - except os.error as err: - onerror(os.listdir, path, sys.exc_info()) - for name in names: - fullname = os.path.join(path, name) - try: - mode = os.lstat(fullname).st_mode - except os.error: - mode = 0 - if stat.S_ISDIR(mode): - rmtree(fullname, ignore_errors, onerror) - else: - try: - os.remove(fullname) - except os.error as err: - onerror(os.remove, fullname, sys.exc_info()) - try: - os.rmdir(path) - except os.error: - onerror(os.rmdir, path, sys.exc_info()) - - -def _basename(path: str) -> str: - # A basename() variant which first strips the trailing slash, if present. - # Thus we always get the last component of the path, even for directories. - return os.path.basename(path.rstrip(os.path.sep)) - -def move(src: str, dst: str) -> None: - """Recursively move a file or directory to another location. This is - similar to the Unix "mv" command. - - If the destination is a directory or a symlink to a directory, the source - is moved inside the directory. The destination path must not already - exist. - - If the destination already exists but is not a directory, it may be - overwritten depending on os.rename() semantics. - - If the destination is on our current filesystem, then rename() is used. - Otherwise, src is copied to the destination and then removed. - A lot more could be done here... A look at a mv.c shows a lot of - the issues this implementation glosses over. - - """ - real_dst = dst - if os.path.isdir(dst): - if _samefile(src, dst): - # We might be on a case insensitive filesystem, - # perform the rename anyway. - os.rename(src, dst) - return - - real_dst = os.path.join(dst, _basename(src)) - if os.path.exists(real_dst): - raise Error("Destination path '%s' already exists" % real_dst) - try: - os.rename(src, real_dst) - except OSError as exc: - if os.path.isdir(src): - if _destinsrc(src, dst): - raise Error("Cannot move a directory '%s' into itself '%s'." % (src, dst)) - copytree(src, real_dst, symlinks=True) - rmtree(src) - else: - copy2(src, real_dst) - os.unlink(src) - -def _destinsrc(src: str, dst: str) -> bool: - src = abspath(src) - dst = abspath(dst) - if not src.endswith(os.path.sep): - src += os.path.sep - if not dst.endswith(os.path.sep): - dst += os.path.sep - return dst.startswith(src) - -def _get_gid(name: str) -> int: - """Returns a gid, given a group name.""" - if getgrnam is None or name is None: - return None - try: - result = getgrnam(name) - except KeyError: - result = None - if result is not None: - return result.gr_gid - return None - -def _get_uid(name: str) -> int: - """Returns an uid, given a user name.""" - if getpwnam is None or name is None: - return None - try: - result = getpwnam(name) - except KeyError: - result = None - if result is not None: - return result.pw_uid - return None - -def _make_tarball(base_name: str, base_dir: str, compress: str = "gzip", - verbose: bool = False, dry_run: bool = False, - owner: str = None, group: str = None, - logger: Any = None) -> str: - """Create a (possibly compressed) tar file from all the files under - 'base_dir'. - - 'compress' must be "gzip" (the default), "bzip2", or None. - - 'owner' and 'group' can be used to define an owner and a group for the - archive that is being built. If not provided, the current owner and group - will be used. - - The output tar file will be named 'base_name' + ".tar", possibly plus - the appropriate compression extension (".gz", or ".bz2"). - - Returns the output filename. - """ - tar_compression = {'gzip': 'gz', None: ''} - compress_ext = {'gzip': '.gz'} - - if _BZ2_SUPPORTED: - tar_compression['bzip2'] = 'bz2' - compress_ext['bzip2'] = '.bz2' - - # flags for compression program, each element of list will be an argument - if compress is not None and compress not in compress_ext.keys(): - raise ValueError("bad value for 'compress', or compression format not " - "supported : {0}".format(compress)) - - archive_name = base_name + '.tar' + compress_ext.get(compress, '') - archive_dir = os.path.dirname(archive_name) - - if not os.path.exists(archive_dir): - if logger is not None: - logger.info("creating %s", archive_dir) - if not dry_run: - os.makedirs(archive_dir) - - # creating the tarball - if logger is not None: - logger.info('Creating tar archive') - - uid = _get_uid(owner) - gid = _get_gid(group) - - def _set_uid_gid(tarinfo): - if gid is not None: - tarinfo.gid = gid - tarinfo.gname = group - if uid is not None: - tarinfo.uid = uid - tarinfo.uname = owner - return tarinfo - - if not dry_run: - tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) - try: - tar.add(base_dir, filter=_set_uid_gid) - finally: - tar.close() - - return archive_name - -def _call_external_zip(base_dir: str, zip_filename: str, verbose: bool = False, - dry_run: bool = False) -> None: - # XXX see if we want to keep an external call here - if verbose: - zipoptions = "-r" - else: - zipoptions = "-rq" - from distutils.errors import DistutilsExecError - from distutils.spawn import spawn - try: - spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run) - except DistutilsExecError: - # XXX really should distinguish between "couldn't find - # external 'zip' command" and "zip failed". - raise ExecError(("unable to create zip file '%s': " - "could neither import the 'zipfile' module nor " - "find a standalone zip utility") % zip_filename) - -def _make_zipfile(base_name: str, base_dir: str, verbose: bool = False, - dry_run: bool = False, logger: Any = None) -> str: - """Create a zip file from all the files under 'base_dir'. - - The output zip file will be named 'base_name' + ".zip". Uses either the - "zipfile" Python module (if available) or the InfoZIP "zip" utility - (if installed and found on the default search path). If neither tool is - available, raises ExecError. Returns the name of the output zip - file. - """ - zip_filename = base_name + ".zip" - archive_dir = os.path.dirname(base_name) - - if not os.path.exists(archive_dir): - if logger is not None: - logger.info("creating %s", archive_dir) - if not dry_run: - os.makedirs(archive_dir) - - # If zipfile module is not available, try spawning an external 'zip' - # command. - try: - import zipfile - except ImportError: - zipfile = None - - if zipfile is None: - _call_external_zip(base_dir, zip_filename, verbose, dry_run) - else: - if logger is not None: - logger.info("creating '%s' and adding '%s' to it", - zip_filename, base_dir) - - if not dry_run: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_DEFLATED) - - for dirpath, dirnames, filenames in os.walk(base_dir): - for name in filenames: - path = os.path.normpath(os.path.join(dirpath, name)) - if os.path.isfile(path): - zip.write(path, path) - if logger is not None: - logger.info("adding '%s'", path) - zip.close() - - return zip_filename - -_ARCHIVE_FORMATS = { - 'gztar': (_make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"), - 'tar': (_make_tarball, [('compress', None)], "uncompressed tar file"), - 'zip': (_make_zipfile, [],"ZIP file") - } # type: Dict[str, Tuple[Any, Sequence[Tuple[str, str]], str]] - -if _BZ2_SUPPORTED: - _ARCHIVE_FORMATS['bztar'] = (_make_tarball, [('compress', 'bzip2')], - "bzip2'ed tar-file") - -def get_archive_formats() -> List[Tuple[str, str]]: - """Returns a list of supported formats for archiving and unarchiving. - - Each element of the returned sequence is a tuple (name, description) - """ - formats = [(name, registry[2]) for name, registry in - _ARCHIVE_FORMATS.items()] - formats.sort() - return formats - -def register_archive_format(name: str, function: Any, - extra_args: Sequence[Tuple[str, Any]] = None, - description: str = '') -> None: - """Registers an archive format. - - name is the name of the format. function is the callable that will be - used to create archives. If provided, extra_args is a sequence of - (name, value) tuples that will be passed as arguments to the callable. - description can be provided to describe the format, and will be returned - by the get_archive_formats() function. - """ - if extra_args is None: - extra_args = [] - if not callable(function): - raise TypeError('The %s object is not callable' % function) - if not isinstance(extra_args, (tuple, list)): - raise TypeError('extra_args needs to be a sequence') - for element in extra_args: - if not isinstance(element, (tuple, list)) or len(cast(tuple, element)) !=2 : - raise TypeError('extra_args elements are : (arg_name, value)') - - _ARCHIVE_FORMATS[name] = (function, extra_args, description) - -def unregister_archive_format(name: str) -> None: - del _ARCHIVE_FORMATS[name] - -def make_archive(base_name: str, format: str, root_dir: str = None, - base_dir: str = None, verbose: bool = False, - dry_run: bool = False, owner: str = None, - group: str = None, logger: Any = None) -> str: - """Create an archive file (eg. zip or tar). - - 'base_name' is the name of the file to create, minus any format-specific - extension; 'format' is the archive format: one of "zip", "tar", "bztar" - or "gztar". - - 'root_dir' is a directory that will be the root directory of the - archive; ie. we typically chdir into 'root_dir' before creating the - archive. 'base_dir' is the directory where we start archiving from; - ie. 'base_dir' will be the common prefix of all files and - directories in the archive. 'root_dir' and 'base_dir' both default - to the current directory. Returns the name of the archive file. - - 'owner' and 'group' are used when creating a tar archive. By default, - uses the current owner and group. - """ - save_cwd = os.getcwd() - if root_dir is not None: - if logger is not None: - logger.debug("changing into '%s'", root_dir) - base_name = os.path.abspath(base_name) - if not dry_run: - os.chdir(root_dir) - - if base_dir is None: - base_dir = os.curdir - - kwargs = {'dry_run': dry_run, 'logger': logger} - - try: - format_info = _ARCHIVE_FORMATS[format] - except KeyError: - raise ValueError("unknown archive format '%s'" % format) - - func = format_info[0] - for arg, val in format_info[1]: - kwargs[arg] = val - - if format != 'zip': - kwargs['owner'] = owner - kwargs['group'] = group - - try: - filename = func(base_name, base_dir, **kwargs) - finally: - if root_dir is not None: - if logger is not None: - logger.debug("changing back to '%s'", save_cwd) - os.chdir(save_cwd) - - return filename - - -def get_unpack_formats() -> List[Tuple[str, List[str], str]]: - """Returns a list of supported formats for unpacking. - - Each element of the returned sequence is a tuple - (name, extensions, description) - """ - formats = [(name, info[0], info[3]) for name, info in - _UNPACK_FORMATS.items()] - formats.sort() - return formats - -def _check_unpack_options(extensions: List[str], function: Any, - extra_args: Sequence[Tuple[str, Any]]) -> None: - """Checks what gets registered as an unpacker.""" - # first make sure no other unpacker is registered for this extension - existing_extensions = Dict[str, str]() - for name, info in _UNPACK_FORMATS.items(): - for ext in info[0]: - existing_extensions[ext] = name - - for extension in extensions: - if extension in existing_extensions: - msg = '%s is already registered for "%s"' - raise RegistryError(msg % (extension, - existing_extensions[extension])) - - if not callable(function): - raise TypeError('The registered function must be a callable') - - -def register_unpack_format(name: str, extensions: List[str], function: Any, - extra_args: Sequence[Tuple[str, Any]] = None, - description: str = '') -> None: - """Registers an unpack format. - - `name` is the name of the format. `extensions` is a list of extensions - corresponding to the format. - - `function` is the callable that will be - used to unpack archives. The callable will receive archives to unpack. - If it's unable to handle an archive, it needs to raise a ReadError - exception. - - If provided, `extra_args` is a sequence of - (name, value) tuples that will be passed as arguments to the callable. - description can be provided to describe the format, and will be returned - by the get_unpack_formats() function. - """ - if extra_args is None: - extra_args = [] - _check_unpack_options(extensions, function, extra_args) - _UNPACK_FORMATS[name] = extensions, function, extra_args, description - -def unregister_unpack_format(name: str) -> None: - """Removes the pack format from the registery.""" - del _UNPACK_FORMATS[name] - -def _ensure_directory(path: str) -> None: - """Ensure that the parent directory of `path` exists""" - dirname = os.path.dirname(path) - if not os.path.isdir(dirname): - os.makedirs(dirname) - -def _unpack_zipfile(filename: str, extract_dir: str) -> None: - """Unpack zip `filename` to `extract_dir` - """ - try: - import zipfile - except ImportError: - raise ReadError('zlib not supported, cannot unpack this archive.') - - if not zipfile.is_zipfile(filename): - raise ReadError("%s is not a zip file" % filename) - - zip = zipfile.ZipFile(filename) - try: - for info in zip.infolist(): - name = info.filename - - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - - target = os.path.join(extract_dir, *name.split('/')) - if not target: - continue - - _ensure_directory(target) - if not name.endswith('/'): - # file - data = zip.read(info.filename) - f = open(target,'wb') - try: - f.write(data) - finally: - f.close() - del data - finally: - zip.close() - -def _unpack_tarfile(filename: str, extract_dir: str) -> None: - """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir` - """ - try: - tarobj = tarfile.open(filename) - except tarfile.TarError: - raise ReadError( - "%s is not a compressed or uncompressed tar file" % filename) - try: - tarobj.extractall(extract_dir) - finally: - tarobj.close() - -_UNPACK_FORMATS = { - 'gztar': (['.tar.gz', '.tgz'], _unpack_tarfile, [], "gzip'ed tar-file"), - 'tar': (['.tar'], _unpack_tarfile, [], "uncompressed tar file"), - 'zip': (['.zip'], _unpack_zipfile, [], "ZIP file") - } # type: Dict[str, Tuple[List[str], Any, Sequence[Tuple[str, Any]], str]] - -if _BZ2_SUPPORTED: - _UNPACK_FORMATS['bztar'] = (['.bz2'], _unpack_tarfile, [], - "bzip2'ed tar-file") - -def _find_unpack_format(filename: str) -> str: - for name, info in _UNPACK_FORMATS.items(): - for extension in info[0]: - if filename.endswith(extension): - return name - return None - -def unpack_archive(filename: str, extract_dir: str = None, - format: str = None) -> None: - """Unpack an archive. - - `filename` is the name of the archive. - - `extract_dir` is the name of the target directory, where the archive - is unpacked. If not provided, the current working directory is used. - - `format` is the archive format: one of "zip", "tar", or "gztar". Or any - other registered format. If not provided, unpack_archive will use the - filename extension and see if an unpacker was registered for that - extension. - - In case none is found, a ValueError is raised. - """ - if extract_dir is None: - extract_dir = os.getcwd() - - if format is not None: - try: - format_info = _UNPACK_FORMATS[format] - except KeyError: - raise ValueError("Unknown unpack format '{0}'".format(format)) - - func = format_info[1] - func(filename, extract_dir, **dict(format_info[2])) - else: - # we need to look at the registered unpackers supported extensions - format = _find_unpack_format(filename) - if format is None: - raise ReadError("Unknown archive format '{0}'".format(filename)) - - func = _UNPACK_FORMATS[format][1] - kwargs = dict(_UNPACK_FORMATS[format][2]) - func(filename, extract_dir, **kwargs) diff --git a/lib-python/3.2/subprocess.py b/lib-python/3.2/subprocess.py deleted file mode 100644 index d2529dba68f0..000000000000 --- a/lib-python/3.2/subprocess.py +++ /dev/null @@ -1,1702 +0,0 @@ -# subprocess - Subprocesses with accessible I/O streams -# -# For more information about this module, see PEP 324. -# -# Copyright (c) 2003-2005 by Peter Astrand -# -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -r"""subprocess - Subprocesses with accessible I/O streams - -This module allows you to spawn processes, connect to their -input/output/error pipes, and obtain their return codes. This module -intends to replace several other, older modules and functions, like: - -os.system -os.spawn* - -Information about how the subprocess module can be used to replace these -modules and functions can be found below. - - - -Using the subprocess module -=========================== -This module defines one class called Popen: - -class Popen(args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=True, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0, - restore_signals=True, start_new_session=False, pass_fds=()): - - -Arguments are: - -args should be a string, or a sequence of program arguments. The -program to execute is normally the first item in the args sequence or -string, but can be explicitly set by using the executable argument. - -On POSIX, with shell=False (default): In this case, the Popen class -uses os.execvp() to execute the child program. args should normally -be a sequence. A string will be treated as a sequence with the string -as the only item (the program to execute). - -On POSIX, with shell=True: If args is a string, it specifies the -command string to execute through the shell. If args is a sequence, -the first item specifies the command string, and any additional items -will be treated as additional shell arguments. - -On Windows: the Popen class uses CreateProcess() to execute the child -program, which operates on strings. If args is a sequence, it will be -converted to a string using the list2cmdline method. Please note that -not all MS Windows applications interpret the command line the same -way: The list2cmdline is designed for applications using the same -rules as the MS C runtime. - -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). - -stdin, stdout and stderr specify the executed programs' standard -input, standard output and standard error file handles, respectively. -Valid values are PIPE, an existing file descriptor (a positive -integer), an existing file object, and None. PIPE indicates that a -new pipe to the child should be created. With None, no redirection -will occur; the child's file handles will be inherited from the -parent. Additionally, stderr can be STDOUT, which indicates that the -stderr data from the applications should be captured into the same -file handle as for stdout. - -On POSIX, if preexec_fn is set to a callable object, this object will be -called in the child process just before the child is executed. The use -of preexec_fn is not thread safe, using it in the presence of threads -could lead to a deadlock in the child process before the new executable -is executed. - -If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. The default for close_fds -varies by platform: Always true on POSIX. True when stdin/stdout/stderr -are None on Windows, false otherwise. - -pass_fds is an optional sequence of file descriptors to keep open between the -parent and child. Providing any pass_fds implicitly sets close_fds to true. - -if shell is true, the specified command will be executed through the -shell. - -If cwd is not None, the current directory will be changed to cwd -before the child is executed. - -On POSIX, if restore_signals is True all signals that Python sets to -SIG_IGN are restored to SIG_DFL in the child process before the exec. -Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This -parameter does nothing on Windows. - -On POSIX, if start_new_session is True, the setsid() system call will be made -in the child process prior to executing the command. - -If env is not None, it defines the environment variables for the new -process. - -If universal_newlines is true, the file objects stdout and stderr are -opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the old Macintosh convention or -'\r\n', the Windows convention. All of these external representations -are seen as '\n' by the Python program. Note: This feature is only -available if Python is built with universal newline support (the -default). Also, the newlines attribute of the file objects stdout, -stdin and stderr are not updated by the communicate() method. - -The startupinfo and creationflags, if given, will be passed to the -underlying CreateProcess() function. They can specify things such as -appearance of the main window and priority for the new process. -(Windows only) - - -This module also defines some shortcut functions: - -call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> retcode = subprocess.call(["ls", "-l"]) - -check_call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete. If the - exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> subprocess.check_call(["ls", "-l"]) - 0 - -getstatusoutput(cmd): - Return (status, output) of executing cmd in a shell. - - Execute the string 'cmd' in a shell with os.popen() and return a 2-tuple - (status, output). cmd is actually run as '{ cmd ; } 2>&1', so that the - returned output will contain output or error messages. A trailing newline - is stripped from the output. The exit status for the command can be - interpreted according to the rules for the C function wait(). Example: - - >>> subprocess.getstatusoutput('ls /bin/ls') - (0, '/bin/ls') - >>> subprocess.getstatusoutput('cat /bin/junk') - (256, 'cat: /bin/junk: No such file or directory') - >>> subprocess.getstatusoutput('/bin/junk') - (256, 'sh: /bin/junk: not found') - -getoutput(cmd): - Return output (stdout or stderr) of executing cmd in a shell. - - Like getstatusoutput(), except the exit status is ignored and the return - value is a string containing the command's output. Example: - - >>> subprocess.getoutput('ls /bin/ls') - '/bin/ls' - -check_output(*popenargs, **kwargs): - Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> output = subprocess.check_output(["ls", "-l", "/dev/null"]) - - -Exceptions ----------- -Exceptions raised in the child process, before the new program has -started to execute, will be re-raised in the parent. Additionally, -the exception object will have one extra attribute called -'child_traceback', which is a string containing traceback information -from the childs point of view. - -The most common exception raised is OSError. This occurs, for -example, when trying to execute a non-existent file. Applications -should prepare for OSErrors. - -A ValueError will be raised if Popen is called with invalid arguments. - -check_call() and check_output() will raise CalledProcessError, if the -called process returns a non-zero return code. - - -Security --------- -Unlike some other popen functions, this implementation will never call -/bin/sh implicitly. This means that all characters, including shell -metacharacters, can safely be passed to child processes. - - -Popen objects -============= -Instances of the Popen class have the following methods: - -poll() - Check if child process has terminated. Returns returncode - attribute. - -wait() - Wait for child process to terminate. Returns returncode attribute. - -communicate(input=None) - Interact with process: Send data to stdin. Read data from stdout - and stderr, until end-of-file is reached. Wait for process to - terminate. The optional input argument should be a string to be - sent to the child process, or None, if no data should be sent to - the child. - - communicate() returns a tuple (stdout, stderr). - - Note: The data read is buffered in memory, so do not use this - method if the data size is large or unlimited. - -The following attributes are also available: - -stdin - If the stdin argument is PIPE, this attribute is a file object - that provides input to the child process. Otherwise, it is None. - -stdout - If the stdout argument is PIPE, this attribute is a file object - that provides output from the child process. Otherwise, it is - None. - -stderr - If the stderr argument is PIPE, this attribute is file object that - provides error output from the child process. Otherwise, it is - None. - -pid - The process ID of the child process. - -returncode - The child return code. A None value indicates that the process - hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (POSIX only). - - -Replacing older functions with the subprocess module -==================================================== -In this section, "a ==> b" means that b can be used as a replacement -for a. - -Note: All functions in this section fail (more or less) silently if -the executed program cannot be found; this module raises an OSError -exception. - -In the following examples, we assume that the subprocess module is -imported with "from subprocess import *". - - -Replacing /bin/sh shell backquote ---------------------------------- -output=`mycmd myarg` -==> -output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0] - - -Replacing shell pipe line -------------------------- -output=`dmesg | grep hda` -==> -p1 = Popen(["dmesg"], stdout=PIPE) -p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) -output = p2.communicate()[0] - - -Replacing os.system() ---------------------- -sts = os.system("mycmd" + " myarg") -==> -p = Popen("mycmd" + " myarg", shell=True) -pid, sts = os.waitpid(p.pid, 0) - -Note: - -* Calling the program through the shell is usually not required. - -* It's easier to look at the returncode attribute than the - exitstatus. - -A more real-world example would look like this: - -try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print("Child was terminated by signal", -retcode, file=sys.stderr) - else: - print("Child returned", retcode, file=sys.stderr) -except OSError as e: - print("Execution failed:", e, file=sys.stderr) - - -Replacing os.spawn* -------------------- -P_NOWAIT example: - -pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") -==> -pid = Popen(["/bin/mycmd", "myarg"]).pid - - -P_WAIT example: - -retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") -==> -retcode = call(["/bin/mycmd", "myarg"]) - - -Vector example: - -os.spawnvp(os.P_NOWAIT, path, args) -==> -Popen([path] + args[1:]) - - -Environment example: - -os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) -==> -Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) -""" - -import sys -mswindows = (sys.platform == "win32") - -import io -import os -import traceback -import gc -import signal -import builtins -import warnings -import errno - -from typing import ( - Any, Tuple, List, Sequence, Undefined, Function, Mapping, cast, Set, Dict, - IO, TextIO, Traceback, AnyStr -) - -# Exception classes used by this module. -class CalledProcessError(Exception): - """This exception is raised when a process run by check_call() or - check_output() returns a non-zero exit status. - The exit status will be stored in the returncode attribute; - check_output() will also store the output in the output attribute. - """ - def __init__(self, returncode: int, cmd: str, output: Any = None) -> None: - self.returncode = returncode - self.cmd = cmd - self.output = output - def __str__(self) -> str: - return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) - - -if mswindows: - import threading - import msvcrt - import _subprocess - class STARTUPINFO: - dwFlags = 0 - hStdInput = Any(None) - hStdOutput = Any(None) - hStdError = Any(None) - wShowWindow = 0 - class pywintypes: - error = IOError -else: - import select - _has_poll = hasattr(select, 'poll') - import fcntl - import pickle - - try: - import _posixsubprocess - have_posixsubprocess = True - except ImportError: - have_posixsubprocess = False - warnings.warn("The _posixsubprocess module is not being used. " - "Child process reliability may suffer if your " - "program uses threads.", RuntimeWarning) - - # When select or poll has indicated that the file is writable, - # we can write up to _PIPE_BUF bytes without risk of blocking. - # POSIX defines PIPE_BUF as >= 512. - _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) # type: int - - _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1) # type: int - - def _set_cloexec(fd: int, cloexec: bool) -> None: - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC) - - if have_posixsubprocess: - _create_pipe = _posixsubprocess.cloexec_pipe - else: - def __create_pipe() -> Tuple[int, int]: - fds = os.pipe() - _set_cloexec(fds[0], True) - _set_cloexec(fds[1], True) - return fds - _create_pipe = __create_pipe - -__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", - "getoutput", "check_output", "CalledProcessError"] - -if mswindows: - from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, - STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, - STD_ERROR_HANDLE, SW_HIDE, - STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) - - __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", - "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", - "STD_ERROR_HANDLE", "SW_HIDE", - "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except: - MAXFD = 256 - -# This lists holds Popen instances for which the underlying process had not -# exited at the time its __del__ method got called: those processes are wait()ed -# for synchronously from _cleanup() when a new Popen object is created, to avoid -# zombie processes. -_active = List['Popen']() - -def _cleanup() -> None: - for inst in _active[:]: - res = inst._internal_poll(_deadstate=sys.maxsize) - if res is not None: - try: - _active.remove(inst) - except ValueError: - # This can happen if two threads create a new Popen instance. - # It's harmless that it was already removed, so ignore. - pass - -PIPE = -1 -STDOUT = -2 - - -def _eintr_retry_call(func: Any, *args: Any) -> Any: - while True: - try: - return func(*args) - except (OSError, IOError) as e: - if e.errno == errno.EINTR: - continue - raise - - -def call(*popenargs: Any, **kwargs: Any) -> int: - """Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - """ - return Popen(*popenargs, **kwargs).wait() - - -def check_call(*popenargs: Any, **kwargs: Any) -> int: - """Run command with arguments. Wait for command to complete. If - the exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - """ - retcode = call(*popenargs, **kwargs) - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return 0 - - -def check_output(*popenargs: Any, **kwargs: Any) -> bytes: - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - b'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - kwargs['stdout'] = PIPE - process = Popen(*popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - - -def list2cmdline(seq: Sequence[str]) -> str: - """ - Translate a sequence of arguments into a command line - string, using the same rules as the MS C runtime: - - 1) Arguments are delimited by white space, which is either a - space or a tab. - - 2) A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - - 3) A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - - 4) Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - - 5) If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - """ - - # See - # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx - # or search http://msdn.microsoft.com for - # "Parsing C++ Command-Line Arguments" - result = List[str]() - needquote = False - for arg in seq: - bs_buf = List[str]() - - # Add a space to separate this argument from the others - if result: - result.append(' ') - - needquote = (" " in arg) or ("\t" in arg) or not arg - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - # Don't know if we need to double yet. - bs_buf.append(c) - elif c == '"': - # Double backslashes. - result.append('\\' * len(bs_buf)*2) - bs_buf = [] - result.append('\\"') - else: - # Normal char - if bs_buf: - result.extend(bs_buf) - bs_buf = [] - result.append(c) - - # Add remaining backslashes, if any. - if bs_buf: - result.extend(bs_buf) - - if needquote: - result.extend(bs_buf) - result.append('"') - - return ''.join(result) - - -# Various tools for executing commands and looking at their output and status. -# -# NB This only works (and is only relevant) for POSIX. - -def getstatusoutput(cmd: str) -> Tuple[int, str]: - """Return (status, output) of executing cmd in a shell. - - Execute the string 'cmd' in a shell with os.popen() and return a 2-tuple - (status, output). cmd is actually run as '{ cmd ; } 2>&1', so that the - returned output will contain output or error messages. A trailing newline - is stripped from the output. The exit status for the command can be - interpreted according to the rules for the C function wait(). Example: - - >>> import subprocess - >>> subprocess.getstatusoutput('ls /bin/ls') - (0, '/bin/ls') - >>> subprocess.getstatusoutput('cat /bin/junk') - (256, 'cat: /bin/junk: No such file or directory') - >>> subprocess.getstatusoutput('/bin/junk') - (256, 'sh: /bin/junk: not found') - """ - pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r') - text = pipe.read() - sts = pipe.close() - if sts is None: sts = 0 - if text[-1:] == '\n': text = text[:-1] - return sts, text - - -def getoutput(cmd: str) -> str: - """Return output (stdout or stderr) of executing cmd in a shell. - - Like getstatusoutput(), except the exit status is ignored and the return - value is a string containing the command's output. Example: - - >>> import subprocess - >>> subprocess.getoutput('ls /bin/ls') - '/bin/ls' - """ - return getstatusoutput(cmd)[1] - - -_PLATFORM_DEFAULT_CLOSE_FDS = object() - - -class Popen(object): - def __init__(self, args: Sequence[Any], bufsize: int = 0, - executable: str = None, stdin: Any = None, - stdout: Any = None, stderr: Any = None, - preexec_fn: Function[[], Any] = None, - close_fds: Any = _PLATFORM_DEFAULT_CLOSE_FDS, - shell: int = False, cwd: str = None, - env: Mapping[str, str] = None, - universal_newlines: int = False, - startupinfo: 'STARTUPINFO' = None, creationflags: int = 0, - restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Any = ()) -> None: - """Create new Popen instance.""" - _cleanup() - - self._child_created = False - if bufsize is None: - bufsize = 0 # Restore default - if not isinstance(bufsize, int): - raise TypeError("bufsize must be an integer") - - if mswindows: - if preexec_fn is not None: - raise ValueError("preexec_fn is not supported on Windows " - "platforms") - any_stdio_set = (stdin is not None or stdout is not None or - stderr is not None) - if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: - if any_stdio_set: - close_fds = False - else: - close_fds = True - elif close_fds and any_stdio_set: - raise ValueError( - "close_fds is not supported on Windows platforms" - " if you redirect stdin/stdout/stderr") - else: - # POSIX - if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: - close_fds = True - if pass_fds and not close_fds: - warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) - close_fds = True - if startupinfo is not None: - raise ValueError("startupinfo is only supported on Windows " - "platforms") - if creationflags != 0: - raise ValueError("creationflags is only supported on Windows " - "platforms") - - self.stdin = None # type: IO[Any] - self.stdout = None # type: IO[Any] - self.stderr = None # type: IO[Any] - self.pid = None # type: int - self.returncode = None # type: int - self.universal_newlines = universal_newlines - - # Input and output objects. The general principle is like - # this: - # - # Parent Child - # ------ ----- - # p2cwrite ---stdin---> p2cread - # c2pread <--stdout--- c2pwrite - # errread <--stderr--- errwrite - # - # On POSIX, the child objects are file descriptors. On - # Windows, these are Windows file handles. The parent objects - # are file descriptors on both platforms. The parent objects - # are -1 when not using PIPEs. The child objects are -1 - # when not redirecting. - - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) - - # We wrap OS handles *before* launching the child, otherwise a - # quickly terminating child could make our fds unwrappable - # (see #8458). - - if mswindows: - if p2cwrite != -1: - p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread != -1: - c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread != -1: - errread = msvcrt.open_osfhandle(errread.Detach(), 0) - - if p2cwrite != -1: - self.stdin = io.open(p2cwrite, 'wb', bufsize) - if self.universal_newlines: - self.stdin = io.TextIOWrapper(self.stdin, write_through=True) - if c2pread != -1: - self.stdout = io.open(c2pread, 'rb', bufsize) - if universal_newlines: - self.stdout = io.TextIOWrapper(self.stdout) - if errread != -1: - self.stderr = io.open(errread, 'rb', bufsize) - if universal_newlines: - self.stderr = io.TextIOWrapper(self.stderr) - - try: - self._execute_child(args, executable, preexec_fn, close_fds, - pass_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite, - restore_signals, start_new_session) - except: - # Cleanup if the child failed starting - for f in filter(None, [self.stdin, self.stdout, self.stderr]): - try: - f.close() - except EnvironmentError: - # Ignore EBADF or other errors - pass - raise - - - def _translate_newlines(self, data: bytes, encoding: str) -> str: - data = data.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - return data.decode(encoding) - - def __enter__(self) -> 'Popen': - return self - - def __exit__(self, type: type, value: BaseException, - traceback: Traceback) -> bool: - if self.stdout: - self.stdout.close() - if self.stderr: - self.stderr.close() - if self.stdin: - self.stdin.close() - # Wait for the process to terminate, to avoid zombies. - self.wait() - - def __del__(self, _maxsize: int = sys.maxsize, - _active: List['Popen'] = _active) -> None: - # If __init__ hasn't had a chance to execute (e.g. if it - # was passed an undeclared keyword argument), we don't - # have a _child_created attribute at all. - if not getattr(self, '_child_created', False): - # We didn't get to successfully create a child process. - return - # In case the child hasn't been waited on, check if it's done. - self._internal_poll(_deadstate=_maxsize) - if self.returncode is None and _active is not None: - # Child is still running, keep us alive until we can wait on it. - _active.append(self) - - - def communicate(self, input: Any = None) -> Tuple[Any, Any]: - """Interact with process: Send data to stdin. Read data from - stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data - should be sent to the child. - - communicate() returns a tuple (stdout, stderr).""" - - # Optimization: If we are only using one pipe, or no pipe at - # all, using select() or threads is unnecessary. - if [self.stdin, self.stdout, self.stderr].count(None) >= 2: - stdout = None # type: IO[Any] - stderr = None # type: IO[Any] - if self.stdin: - if input: - try: - self.stdin.write(input) - except IOError as e: - if e.errno != errno.EPIPE and e.errno != errno.EINVAL: - raise - self.stdin.close() - elif self.stdout: - stdout = _eintr_retry_call(self.stdout.read) - self.stdout.close() - elif self.stderr: - stderr = _eintr_retry_call(self.stderr.read) - self.stderr.close() - self.wait() - return (stdout, stderr) - - return self._communicate(input) - - - def poll(self) -> int: - return self._internal_poll() - - - if mswindows: - # - # Windows methods - # - def _get_handles(self, stdin: Any, stdout: Any, - stderr: Any) -> Tuple[Any, Any, Any, Any, Any, Any]: - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - if stdin is None and stdout is None and stderr is None: - return (-1, -1, -1, -1, -1, -1) - - p2cread, p2cwrite = -1, -1 # type: (Any, Any) - c2pread, c2pwrite = -1, -1 # type: (Any, Any) - errread, errwrite = -1, -1 # type: (Any, Any) - - if stdin is None: - p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) - if p2cread is None: - p2cread, _ = _subprocess.CreatePipe(None, 0) - elif stdin == PIPE: - p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdin, int): - p2cread = msvcrt.get_osfhandle(stdin) - else: - # Assuming file-like object - p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) - - if stdout is None: - c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) - if c2pwrite is None: - _, c2pwrite = _subprocess.CreatePipe(None, 0) - elif stdout == PIPE: - c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdout, int): - c2pwrite = msvcrt.get_osfhandle(stdout) - else: - # Assuming file-like object - c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) - - if stderr is None: - errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) - if errwrite is None: - _, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == PIPE: - errread, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = msvcrt.get_osfhandle(stderr) - else: - # Assuming file-like object - errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - - def _make_inheritable(self, handle: _subprocess.Handle) -> int: - """Return a duplicate of handle, which is inheritable""" - return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(), - handle, _subprocess.GetCurrentProcess(), 0, 1, - _subprocess.DUPLICATE_SAME_ACCESS) - - - def _find_w9xpopen(self) -> str: - """Find and return absolut path to w9xpopen.exe""" - w9xpopen = os.path.join( - os.path.dirname(_subprocess.GetModuleFileName(0)), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - # Eeek - file-not-found - possibly an embedding - # situation - see if we can locate it in sys.exec_prefix - w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - raise RuntimeError("Cannot locate w9xpopen.exe, which is " - "needed for Popen to work with your " - "shell or platform.") - return w9xpopen - - - def _execute_child(self, args: Sequence[str], executable: str, - preexec_fn: Function[[], Any], close_fds: Any, - pass_fds: Any, cwd: str, env: Mapping[str, str], - universal_newlines: int, - startupinfo: STARTUPINFO, creationflags: int, - shell: int, - p2cread: Any, p2cwrite: Any, - c2pread: Any, c2pwrite: Any, - errread: Any, errwrite: Any, - restore_signals: bool, - start_new_session: bool) -> None: - """Execute program (MS Windows version)""" - - assert not pass_fds, "pass_fds not supported on Windows." - - if not isinstance(args, str): - args = list2cmdline(args) - - # Process startup details - if startupinfo is None: - startupinfo = STARTUPINFO() - if -1 not in (p2cread, c2pwrite, errwrite): - startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES - startupinfo.hStdInput = p2cread - startupinfo.hStdOutput = c2pwrite - startupinfo.hStdError = errwrite - - if shell: - startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", "cmd.exe") - args = '{} /c "{}"'.format (comspec, args) - if (_subprocess.GetVersion() >= 0x80000000 or - os.path.basename(comspec).lower() == "command.com"): - # Win9x, or using command.com on NT. We need to - # use the w9xpopen intermediate program. For more - # information, see KB Q150956 - # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) - w9xpopen = self._find_w9xpopen() - args = '"%s" %s' % (w9xpopen, args) - # Not passing CREATE_NEW_CONSOLE has been known to - # cause random failures on win9x. Specifically a - # dialog: "Your program accessed mem currently in - # use at xxx" and a hopeful warning about the - # stability of your system. Cost is Ctrl+C won't - # kill children. - creationflags |= _subprocess.CREATE_NEW_CONSOLE - - # Start the process - try: - hp, ht, pid, tid = _subprocess.CreateProcess(executable, - cast(str, args), - # no special security - None, None, - int(not close_fds), - creationflags, - env, - cwd, - startupinfo) - except pywintypes.error as e: - # Translate pywintypes.error to WindowsError, which is - # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or similar), but - # how can this be done from Python? - raise WindowsError(*e.args) - finally: - # Child is launched. Close the parent's copy of those pipe - # handles that only the child should have open. You need - # to make sure that no handles to the write end of the - # output pipe are maintained in this process or else the - # pipe will not close when the child process exits and the - # ReadFile will hang. - if p2cread != -1: - p2cread.Close() - if c2pwrite != -1: - c2pwrite.Close() - if errwrite != -1: - errwrite.Close() - - # Retain the process handle, but close the thread handle - self._child_created = True - self._handle = hp - self.pid = pid - ht.Close() - - def _internal_poll(self, _deadstate: int = None) -> int: - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it can only refer to objects - in its local scope. - - """ - return self._internal_poll_win(_deadstate) - - from _subprocess import Handle - - def _internal_poll_win(self, _deadstate: int = None, - _WaitForSingleObject: Function[[Handle, int], int] = - _subprocess.WaitForSingleObject, - _WAIT_OBJECT_0: int = _subprocess.WAIT_OBJECT_0, - _GetExitCodeProcess: Function[[Handle], int] = - _subprocess.GetExitCodeProcess) -> int: - if self.returncode is None: - if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: - self.returncode = _GetExitCodeProcess(self._handle) - return self.returncode - - - def wait(self) -> int: - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - _subprocess.WaitForSingleObject(self._handle, - _subprocess.INFINITE) - self.returncode = _subprocess.GetExitCodeProcess(self._handle) - return self.returncode - - - def _readerthread(self, fh: IO[AnyStr], buffer: List[AnyStr]) -> None: - buffer.append(fh.read()) - fh.close() - - - def _communicate(self, input: Any) -> Tuple[Any, Any]: - stdout = Any(None) # Return - stderr = Any(None) # Return - - if self.stdout: - stdout = [] - stdout_thread = threading.Thread(target=self._readerthread, - args=(self.stdout, stdout)) - stdout_thread.daemon = True - stdout_thread.start() - if self.stderr: - stderr = [] - stderr_thread = threading.Thread(target=self._readerthread, - args=(self.stderr, stderr)) - stderr_thread.daemon = True - stderr_thread.start() - - if self.stdin: - if input is not None: - try: - self.stdin.write(input) - except IOError as e: - if e.errno != errno.EPIPE: - raise - self.stdin.close() - - if self.stdout: - stdout_thread.join() - if self.stderr: - stderr_thread.join() - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = stdout[0] - if stderr is not None: - stderr = stderr[0] - - self.wait() - return (stdout, stderr) - - def send_signal(self, sig: int) -> None: - """Send a signal to the process - """ - if sig == signal.SIGTERM: - self.terminate() - elif sig == signal.CTRL_C_EVENT: - os.kill(self.pid, signal.CTRL_C_EVENT) - elif sig == signal.CTRL_BREAK_EVENT: - os.kill(self.pid, signal.CTRL_BREAK_EVENT) - else: - raise ValueError("Unsupported signal: {}".format(sig)) - - def terminate(self) -> None: - """Terminates the process - """ - _subprocess.TerminateProcess(self._handle, 1) - - def kill(self) -> None: - """Terminates the process - """ - self.terminate() - - else: - # - # POSIX methods - # - def _get_handles(self, stdin: Any, stdout: Any, - stderr: Any) -> Tuple[Any, Any, Any, Any, Any, Any]: - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - p2cread, p2cwrite = -1, -1 - c2pread, c2pwrite = -1, -1 - errread, errwrite = -1, -1 - - if stdin is None: - pass - elif stdin == PIPE: - p2cread, p2cwrite = _create_pipe() - elif isinstance(stdin, int): - p2cread = stdin - else: - # Assuming file-like object - p2cread = stdin.fileno() - - if stdout is None: - pass - elif stdout == PIPE: - c2pread, c2pwrite = _create_pipe() - elif isinstance(stdout, int): - c2pwrite = stdout - else: - # Assuming file-like object - c2pwrite = stdout.fileno() - - if stderr is None: - pass - elif stderr == PIPE: - errread, errwrite = _create_pipe() - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = stderr - else: - # Assuming file-like object - errwrite = stderr.fileno() - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - - def _close_fds(self, fds_to_keep: Set[int]) -> None: - start_fd = 3 - for fd in sorted(fds_to_keep): - if fd >= start_fd: - os.closerange(start_fd, fd) - start_fd = fd + 1 - if start_fd <= MAXFD: - os.closerange(start_fd, MAXFD) - - - def _execute_child(self, args: Sequence[str], executable: str, - preexec_fn: Function[[], Any], close_fds: Any, - pass_fds: Any, cwd: str, env: Mapping[str, str], - universal_newlines: int, - startupinfo: 'STARTUPINFO', creationflags: int, - shell: int, - p2cread: Any, p2cwrite: Any, - c2pread: Any, c2pwrite: Any, - errread: Any, errwrite: Any, - restore_signals: bool, - start_new_session: bool) -> None: - """Execute program (POSIX version)""" - - if isinstance(args, str): - args = [args] - else: - args = list(args) - - if shell: - args = ["/bin/sh", "-c"] + args - if executable: - args[0] = executable - - if executable is None: - executable = args[0] - - # For transferring possible exec failure from child to parent. - # Data format: "exception name:hex errno:description" - # Pickle is not used; it is complex and involves memory allocation. - errpipe_read, errpipe_write = _create_pipe() - try: - try: - - if have_posixsubprocess: - # We must avoid complex work that could involve - # malloc or free in the child process to avoid - # potential deadlocks, thus we do all this here. - # and pass it to fork_exec() - - if env is not None: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] - else: - env_list = None # Use execv instead of execve. - executable_enc = os.fsencode(executable) - if os.path.dirname(executable_enc): - executable_list = (executable_enc,) # type: tuple - else: - # This matches the behavior of os._execvpe(). - executable_list = tuple( - os.path.join(os.fsencode(dir), executable_enc) - for dir in os.get_exec_path(env)) - fds_to_keep = set(pass_fds) - fds_to_keep.add(errpipe_write) - self.pid = _posixsubprocess.fork_exec( - args, executable_list, - close_fds, sorted(fds_to_keep), cwd, env_list, - p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite, - errpipe_read, errpipe_write, - restore_signals, start_new_session, preexec_fn) - self._child_created = True - else: - # Pure Python implementation: It is not thread safe. - # This implementation may deadlock in the child if your - # parent process has any other threads running. - - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. See issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite != -1: - os.close(p2cwrite) - if c2pread != -1: - os.close(c2pread) - if errread != -1: - os.close(errread) - os.close(errpipe_read) - - # When duping fds, if there arises a situation - # where one of the fds is either 0, 1 or 2, it - # is possible that it is overwritten (#12607). - if c2pwrite == 0: - c2pwrite = os.dup(c2pwrite) - if errwrite == 0 or errwrite == 1: - errwrite = os.dup(errwrite) - - # Dup fds for child - def _dup2(a: int, b: int) -> None: - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - _set_cloexec(a, False) - elif a != -1: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = Set[int]() - for fd in [p2cread, c2pwrite, errwrite]: - if fd > 2 and fd not in closed: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - fds_to_keep = set(pass_fds) - fds_to_keep.add(errpipe_write) - self._close_fds(fds_to_keep) - - - if cwd is not None: - os.chdir(cwd) - - # This is a copy of Python/pythonrun.c - # _Py_RestoreSignals(). If that were exposed - # as a sys._py_restoresignals func it would be - # better.. but this pure python implementation - # isn't likely to be used much anymore. - if restore_signals: - signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') - for sig in signals: - if hasattr(signal, sig): - signal.signal(getattr(signal, sig), - signal.SIG_DFL) - - if start_new_session and hasattr(os, 'setsid'): - os.setsid() - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except: - try: - exc_type, exc_value = sys.exc_info()[:2] - if isinstance(exc_value, OSError): - errno_num = exc_value.errno - else: - errno_num = 0 - message = '%s:%x:%s' % (exc_type.__name__, - errno_num, exc_value) - messageb = message.encode(errors="surrogatepass") - os.write(errpipe_write, messageb) - except Exception: - # We MUST not allow anything odd happening - # above to prevent us from exiting below. - pass - - # This exitcode won't be reported to applications - # so it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() - finally: - # be sure the FD is closed no matter what - os.close(errpipe_write) - - if p2cread != -1 and p2cwrite != -1: - os.close(p2cread) - if c2pwrite != -1 and c2pread != -1: - os.close(c2pwrite) - if errwrite != -1 and errread != -1: - os.close(errwrite) - - # Wait for exec to fail or succeed; possibly raising an - # exception (limited in size) - data = bytearray() - while True: - part = _eintr_retry_call(os.read, errpipe_read, 50000) - data += part - if not part or len(data) > 50000: - break - finally: - # be sure the FD is closed no matter what - os.close(errpipe_read) - - if data: - try: - _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError as e: - if e.errno != errno.ECHILD: - raise - try: - (exception_name, hex_errno, - err_msg_b) = bytes(data).split(b':', 2) - except ValueError: - print('Bad exception data:', repr(data)) - exception_name = b'RuntimeError' - hex_errno = b'0' - err_msg_b = b'Unknown' - child_exception_type = getattr( - builtins, exception_name.decode('ascii'), - RuntimeError) - for fd in (p2cwrite, c2pread, errread): - if fd != -1: - os.close(fd) - err_msg = err_msg_b.decode(errors="surrogatepass") - if issubclass(child_exception_type, OSError) and hex_errno: - errno_num = int(hex_errno, 16) - if errno_num != 0: - err_msg = os.strerror(errno_num) - if errno_num == errno.ENOENT: - err_msg += ': ' + repr(args[0]) - raise child_exception_type(errno_num, err_msg) - raise child_exception_type(err_msg) - - - def _handle_exitstatus( - self, sts: int, - _WIFSIGNALED: Function[[int], bool] = os.WIFSIGNALED, - _WTERMSIG: Function[[int], bool] = os.WTERMSIG, - _WIFEXITED: Function[[int], bool] = os.WIFEXITED, - _WEXITSTATUS: Function[[int], bool] = os.WEXITSTATUS) -> None: - # This method is called (indirectly) by __del__, so it cannot - # refer to anything outside of its local scope.""" - if _WIFSIGNALED(sts): - self.returncode = -_WTERMSIG(sts) - elif _WIFEXITED(sts): - self.returncode = _WEXITSTATUS(sts) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - - - def _internal_poll(self, _deadstate: int = None) -> int: - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it cannot reference anything - outside of the local scope (nor can any methods it calls). - - """ - return self._internal_poll_posix(_deadstate) - - def _internal_poll_posix(self, _deadstate: int = None, - _waitpid: Function[[int, int], - Tuple[int, int]] = os.waitpid, - _WNOHANG: int = os.WNOHANG, - _os_error: Any = os.error) -> int: - if self.returncode is None: - try: - pid, sts = _waitpid(self.pid, _WNOHANG) - if pid == self.pid: - self._handle_exitstatus(sts) - except _os_error: - if _deadstate is not None: - self.returncode = _deadstate - return self.returncode - - - def wait(self) -> int: - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - try: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError as e: - if e.errno != errno.ECHILD: - raise - # This happens if SIGCLD is set to be ignored or waiting - # for child processes has otherwise been disabled for our - # process. This child is dead, we can't get the status. - sts = 0 - self._handle_exitstatus(sts) - return self.returncode - - - def _communicate(self, input: Any) -> Tuple[Any, Any]: - if self.stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self.stdin.flush() - if not input: - self.stdin.close() - - if _has_poll: - stdout, stderr = self._communicate_with_poll(input) - else: - stdout, stderr = self._communicate_with_select(input) - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout2 = b''.join(stdout) - else: - stdout2 = None - if stderr is not None: - stderr2 = b''.join(stderr) - else: - stderr2 = None - - # Translate newlines, if requested. - # This also turns bytes into strings. - stdout3 = Any(stdout2) - stderr3 = Any(stderr2) - if self.universal_newlines: - if stdout is not None: - stdout3 = self._translate_newlines( - stdout2, cast(TextIO, self.stdout).encoding) - if stderr is not None: - stderr3 = self._translate_newlines( - stderr2, cast(TextIO, self.stderr).encoding) - - self.wait() - return (stdout3, stderr3) - - - def _communicate_with_poll(self, input: Any) -> Tuple[List[bytes], - List[bytes]]: - stdout = None # type: List[bytes] # Return - stderr = None # type: List[bytes] # Return - fd2file = Dict[int, Any]() - fd2output = Dict[int, List[bytes]]() - - poller = select.poll() - def register_and_append(file_obj: IO[Any], eventmask: int) -> None: - poller.register(file_obj.fileno(), eventmask) - fd2file[file_obj.fileno()] = file_obj - - def close_unregister_and_remove(fd: int) -> None: - poller.unregister(fd) - fd2file[fd].close() - fd2file.pop(fd) - - if self.stdin and input: - register_and_append(self.stdin, select.POLLOUT) - - select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI - if self.stdout: - register_and_append(self.stdout, select_POLLIN_POLLPRI) - fd2output[self.stdout.fileno()] = stdout = [] - if self.stderr: - register_and_append(self.stderr, select_POLLIN_POLLPRI) - fd2output[self.stderr.fileno()] = stderr = [] - - input_offset = 0 - while fd2file: - try: - ready = poller.poll() - except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise - - # XXX Rewrite these to use non-blocking I/O on the - # file objects; they are no longer using C stdio! - - for fd, mode in ready: - if mode & select.POLLOUT: - chunk = input[input_offset : input_offset + _PIPE_BUF] - try: - input_offset += os.write(fd, chunk) - except OSError as e: - if e.errno == errno.EPIPE: - close_unregister_and_remove(fd) - else: - raise - else: - if input_offset >= len(input): - close_unregister_and_remove(fd) - elif mode & select_POLLIN_POLLPRI: - data = os.read(fd, 4096) - if not data: - close_unregister_and_remove(fd) - fd2output[fd].append(data) - else: - # Ignore hang up or errors. - close_unregister_and_remove(fd) - - return (stdout, stderr) - - - def _communicate_with_select(self, input: Any) -> Tuple[List[bytes], - List[bytes]]: - read_set = List[IO[Any]]() - write_set = List[IO[Any]]() - stdout = None # type: List[bytes] # Return - stderr = None # type: List[bytes] # Return - - if self.stdin and input: - write_set.append(self.stdin) - if self.stdout: - read_set.append(self.stdout) - stdout = [] - if self.stderr: - read_set.append(self.stderr) - stderr = [] - - input_offset = 0 - while read_set or write_set: - try: - rlist, wlist, xlist = select.select(read_set, write_set, []) - except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise - - # XXX Rewrite these to use non-blocking I/O on the - # file objects; they are no longer using C stdio! - - if self.stdin in wlist: - chunk = input[input_offset : input_offset + _PIPE_BUF] - try: - bytes_written = os.write(self.stdin.fileno(), chunk) - except OSError as oe: - if oe.errno == errno.EPIPE: - self.stdin.close() - write_set.remove(self.stdin) - else: - raise - else: - input_offset += bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) - - if self.stdout in rlist: - data = os.read(self.stdout.fileno(), 1024) - if not data: - self.stdout.close() - read_set.remove(self.stdout) - stdout.append(data) - - if self.stderr in rlist: - data = os.read(self.stderr.fileno(), 1024) - if not data: - self.stderr.close() - read_set.remove(self.stderr) - stderr.append(data) - - return (stdout, stderr) - - - def send_signal(self, sig: int) -> None: - """Send a signal to the process - """ - os.kill(self.pid, sig) - - def terminate(self) -> None: - """Terminate the process with SIGTERM - """ - self.send_signal(signal.SIGTERM) - - def kill(self) -> None: - """Kill the process with SIGKILL - """ - self.send_signal(signal.SIGKILL) - - -def _demo_posix() -> None: - # - # Example 1: Simple redirection: Get process list - # - plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print("Process list:") - print(plist) - - # - # Example 2: Change uid before executing child - # - if os.getuid() == 0: - p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) - p.wait() - - # - # Example 3: Connecting several subprocesses - # - print("Looking for 'hda'...") - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print(repr(p2.communicate()[0])) - - # - # Example 4: Catch execution error - # - print() - print("Trying a weird file...") - try: - print(Popen(["/this/path/does/not/exist"]).communicate()) - except OSError as e: - if e.errno == errno.ENOENT: - print("The file didn't exist. I thought so...") - else: - print("Error", e.errno) - else: - print("Gosh. No error.", file=sys.stderr) - - -def _demo_windows() -> None: - # - # Example 1: Connecting several subprocesses - # - print("Looking for 'PROMPT' in set output...") - p1 = Popen("set", stdout=PIPE, shell=True) - p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print(repr(p2.communicate()[0])) - - # - # Example 2: Simple execution of program - # - print("Executing calc...") - p = Popen("calc") - p.wait() - - -if __name__ == "__main__": - if mswindows: - _demo_windows() - else: - _demo_posix() diff --git a/lib-python/3.2/tempfile.py b/lib-python/3.2/tempfile.py deleted file mode 100644 index cec2329ba491..000000000000 --- a/lib-python/3.2/tempfile.py +++ /dev/null @@ -1,718 +0,0 @@ -"""Temporary files. - -This module provides generic, low- and high-level interfaces for -creating temporary files and directories. The interfaces listed -as "safe" just below can be used without fear of race conditions. -Those listed as "unsafe" cannot, and are provided for backward -compatibility only. - -This module also provides some data items to the user: - - TMP_MAX - maximum number of names that will be tried before - giving up. - template - the default prefix for all temporary names. - You may change this to control the default prefix. - tempdir - If this is set to a string before the first use of - any routine from this module, it will be considered as - another candidate location to store temporary files. -""" - -__all__ = [ - "NamedTemporaryFile", "TemporaryFile", # high level safe interfaces - "SpooledTemporaryFile", "TemporaryDirectory", - "mkstemp", "mkdtemp", # low level safe interfaces - "mktemp", # deprecated unsafe interface - "TMP_MAX", "gettempprefix", # constants - "tempdir", "gettempdir" - ] - - -# Imports. - -import warnings as _warnings -import sys as _sys -import io as _io -import os as _os -import errno as _errno -from random import Random as _Random - -from typing import ( - Any as _Any, Function as _Function, Iterator as _Iterator, - Undefined as _Undefined, List as _List, Tuple as _Tuple, Dict as _Dict, - Iterable as _Iterable, IO as _IO, ducktype as _ducktype, - Traceback as _Traceback -) - -try: - import fcntl as _fcntl -except ImportError: - def _set_cloexec(fd: int) -> None: - pass -else: - def _set_cloexec(fd: int) -> None: - try: - flags = _fcntl.fcntl(fd, _fcntl.F_GETFD, 0) - except IOError: - pass - else: - # flags read successfully, modify - flags |= _fcntl.FD_CLOEXEC - _fcntl.fcntl(fd, _fcntl.F_SETFD, flags) - - -try: - import _thread - _allocate_lock = _thread.allocate_lock # type: _Function[[], _Any] -except ImportError: - import _dummy_thread - _allocate_lock = _dummy_thread.allocate_lock - -_text_openflags = _os.O_RDWR | _os.O_CREAT | _os.O_EXCL -if hasattr(_os, 'O_NOINHERIT'): - _text_openflags |= _os.O_NOINHERIT -if hasattr(_os, 'O_NOFOLLOW'): - _text_openflags |= _os.O_NOFOLLOW - -_bin_openflags = _text_openflags -if hasattr(_os, 'O_BINARY'): - _bin_openflags |= _os.O_BINARY - -if hasattr(_os, 'TMP_MAX'): - TMP_MAX = _os.TMP_MAX -else: - TMP_MAX = 10000 - -template = "tmp" - -# Internal routines. - -_once_lock = _allocate_lock() - -if hasattr(_os, "lstat"): - _stat = _os.lstat # type: _Function[[str], object] -elif hasattr(_os, "stat"): - _stat = _os.stat -else: - # Fallback. All we need is something that raises os.error if the - # file doesn't exist. - def __stat(fn: str) -> object: - try: - f = open(fn) - except IOError: - raise _os.error() - f.close() - _stat = __stat - -def _exists(fn: str) -> bool: - try: - _stat(fn) - except _os.error: - return False - else: - return True - -class _RandomNameSequence(_Iterator[str]): - """An instance of _RandomNameSequence generates an endless - sequence of unpredictable strings which can safely be incorporated - into file names. Each string is six characters long. Multiple - threads can safely use the same instance at the same time. - - _RandomNameSequence is an iterator.""" - - characters = "abcdefghijklmnopqrstuvwxyz0123456789_" - - @property - def rng(self) -> _Random: - cur_pid = _os.getpid() - if cur_pid != getattr(self, '_rng_pid', None): - self._rng = _Random() - self._rng_pid = cur_pid - return self._rng - - def __iter__(self) -> _Iterator[str]: - return self - - def __next__(self) -> str: - c = self.characters - choose = self.rng.choice - letters = [choose(c) for dummy in "123456"] - return ''.join(letters) - -def _candidate_tempdir_list() -> _List[str]: - """Generate a list of candidate temporary directories which - _get_default_tempdir will try.""" - - dirlist = [] # type: _List[str] - - # First, try the environment. - for envname in 'TMPDIR', 'TEMP', 'TMP': - dirname = _os.getenv(envname) - if dirname: dirlist.append(dirname) - - # Failing that, try OS-specific locations. - if _os.name == 'nt': - dirlist.extend([ r'c:\temp', r'c:\tmp', r'\temp', r'\tmp' ]) - else: - dirlist.extend([ '/tmp', '/var/tmp', '/usr/tmp' ]) - - # As a last resort, the current directory. - try: - dirlist.append(_os.getcwd()) - except (AttributeError, _os.error): - dirlist.append(_os.curdir) - - return dirlist - -def _get_default_tempdir() -> str: - """Calculate the default directory to use for temporary files. - This routine should be called exactly once. - - We determine whether or not a candidate temp dir is usable by - trying to create and write to a file in that directory. If this - is successful, the test file is deleted. To prevent denial of - service, the name of the test file must be randomized.""" - - namer = _RandomNameSequence() - dirlist = _candidate_tempdir_list() - - for dir in dirlist: - if dir != _os.curdir: - dir = _os.path.normcase(_os.path.abspath(dir)) - # Try only a few names per directory. - for seq in range(100): - name = next(namer) - filename = _os.path.join(dir, name) - try: - fd = _os.open(filename, _bin_openflags, 0o600) - fp = _io.open(fd, 'wb') - fp.write(b'blat') - fp.close() - _os.unlink(filename) - fp = fd = None - return dir - except (OSError, IOError) as e: - if e.args[0] != _errno.EEXIST: - break # no point trying more names in this directory - pass - raise IOError(_errno.ENOENT, - "No usable temporary directory found in %s" % dirlist) - -_name_sequence = None # type: _RandomNameSequence - -def _get_candidate_names() -> _RandomNameSequence: - """Common setup sequence for all user-callable interfaces.""" - - global _name_sequence - if _name_sequence is None: - _once_lock.acquire() - try: - if _name_sequence is None: - _name_sequence = _RandomNameSequence() - finally: - _once_lock.release() - return _name_sequence - - -def _mkstemp_inner(dir: str, pre: str, suf: str, - flags: int) -> _Tuple[int, str]: - """Code common to mkstemp, TemporaryFile, and NamedTemporaryFile.""" - - names = _get_candidate_names() - - for seq in range(TMP_MAX): - name = next(names) - file = _os.path.join(dir, pre + name + suf) - try: - fd = _os.open(file, flags, 0o600) - _set_cloexec(fd) - return (fd, _os.path.abspath(file)) - except OSError as e: - if e.errno == _errno.EEXIST: - continue # try again - raise - - raise IOError(_errno.EEXIST, "No usable temporary file name found") - - -# User visible interfaces. - -def gettempprefix() -> str: - """Accessor for tempdir.template.""" - return template - -tempdir = None # type: str - -def gettempdir() -> str: - """Accessor for tempfile.tempdir.""" - global tempdir - if tempdir is None: - _once_lock.acquire() - try: - if tempdir is None: - tempdir = _get_default_tempdir() - finally: - _once_lock.release() - return tempdir - -def mkstemp(suffix: str = "", prefix: str = template, dir: str = None, - text: bool = False) -> _Tuple[int, str]: - """User-callable function to create and return a unique temporary - file. The return value is a pair (fd, name) where fd is the - file descriptor returned by os.open, and name is the filename. - - If 'suffix' is specified, the file name will end with that suffix, - otherwise there will be no suffix. - - If 'prefix' is specified, the file name will begin with that prefix, - otherwise a default prefix is used. - - If 'dir' is specified, the file will be created in that directory, - otherwise a default directory is used. - - If 'text' is specified and true, the file is opened in text - mode. Else (the default) the file is opened in binary mode. On - some operating systems, this makes no difference. - - The file is readable and writable only by the creating user ID. - If the operating system uses permission bits to indicate whether a - file is executable, the file is executable by no one. The file - descriptor is not inherited by children of this process. - - Caller is responsible for deleting the file when done with it. - """ - - if dir is None: - dir = gettempdir() - - if text: - flags = _text_openflags - else: - flags = _bin_openflags - - return _mkstemp_inner(dir, prefix, suffix, flags) - - -def mkdtemp(suffix: str = "", prefix: str = template, dir: str = None) -> str: - """User-callable function to create and return a unique temporary - directory. The return value is the pathname of the directory. - - Arguments are as for mkstemp, except that the 'text' argument is - not accepted. - - The directory is readable, writable, and searchable only by the - creating user. - - Caller is responsible for deleting the directory when done with it. - """ - - if dir is None: - dir = gettempdir() - - names = _get_candidate_names() - - for seq in range(TMP_MAX): - name = next(names) - file = _os.path.join(dir, prefix + name + suffix) - try: - _os.mkdir(file, 0o700) - return file - except OSError as e: - if e.errno == _errno.EEXIST: - continue # try again - raise - - raise IOError(_errno.EEXIST, "No usable temporary directory name found") - -def mktemp(suffix: str = "", prefix: str = template, dir: str = None) -> str: - """User-callable function to return a unique temporary file name. The - file is not created. - - Arguments are as for mkstemp, except that the 'text' argument is - not accepted. - - This function is unsafe and should not be used. The file name - refers to a file that did not exist at some point, but by the time - you get around to creating it, someone else may have beaten you to - the punch. - """ - -## from warnings import warn as _warn -## _warn("mktemp is a potential security risk to your program", -## RuntimeWarning, stacklevel=2) - - if dir is None: - dir = gettempdir() - - names = _get_candidate_names() - for seq in range(TMP_MAX): - name = next(names) - file = _os.path.join(dir, prefix + name + suffix) - if not _exists(file): - return file - - raise IOError(_errno.EEXIST, "No usable temporary filename found") - - -@_ducktype(_IO[_Any]) -class _TemporaryFileWrapper: - """Temporary file wrapper - - This class provides a wrapper around files opened for - temporary use. In particular, it seeks to automatically - remove the file when it is no longer needed. - """ - - def __init__(self, file: _IO[_Any], name: str, - delete: bool = True) -> None: - self.file = file - self.name = name - self.close_called = False - self.delete = delete - - if _os.name != 'nt': - # Cache the unlinker so we don't get spurious errors at - # shutdown when the module-level "os" is None'd out. Note - # that this must be referenced as self.unlink, because the - # name TemporaryFileWrapper may also get None'd out before - # __del__ is called. - self.unlink = _os.unlink - - def __getattr__(self, name: str) -> _Any: - # Attribute lookups are delegated to the underlying file - # and cached for non-numeric results - # (i.e. methods are cached, closed and friends are not) - file = _Any(self).__dict__['file'] # type: _IO[_Any] - a = getattr(file, name) - if not isinstance(a, int): - setattr(self, name, a) - return a - - # The underlying __enter__ method returns the wrong object - # (self.file) so override it to return the wrapper - def __enter__(self) -> '_TemporaryFileWrapper': - self.file.__enter__() - return self - - # iter() doesn't use __getattr__ to find the __iter__ method - def __iter__(self) -> _Iterator[_Any]: - return iter(self.file) - - # NT provides delete-on-close as a primitive, so we don't need - # the wrapper to do anything special. We still use it so that - # file.name is useful (i.e. not "(fdopen)") with NamedTemporaryFile. - if _os.name != 'nt': - def close(self) -> None: - if not self.close_called: - self.close_called = True - self.file.close() - if self.delete: - self.unlink(self.name) - - def __del__(self) -> None: - self.close() - - # Need to trap __exit__ as well to ensure the file gets - # deleted when used in a with statement - def __exit__(self, exc: type, value: BaseException, - tb: _Traceback) -> bool: - result = self.file.__exit__(exc, value, tb) - self.close() - return result - else: - def __exit__(self, exc: type, value: BaseException, - tb: _Traceback) -> bool: - self.file.__exit__(exc, value, tb) - - -def NamedTemporaryFile(mode: str = 'w+b', buffering: int = -1, - encoding: str = None, newline: str = None, - suffix: str = "", prefix: str = template, - dir: str = None, delete: bool = True) -> _IO[_Any]: - """Create and return a temporary file. - Arguments: - 'prefix', 'suffix', 'dir' -- as for mkstemp. - 'mode' -- the mode argument to io.open (default "w+b"). - 'buffering' -- the buffer size argument to io.open (default -1). - 'encoding' -- the encoding argument to io.open (default None) - 'newline' -- the newline argument to io.open (default None) - 'delete' -- whether the file is deleted on close (default True). - The file is created as mkstemp() would do it. - - Returns an object with a file-like interface; the name of the file - is accessible as file.name. The file will be automatically deleted - when it is closed unless the 'delete' argument is set to False. - """ - - if dir is None: - dir = gettempdir() - - flags = _bin_openflags - - # Setting O_TEMPORARY in the flags causes the OS to delete - # the file when it is closed. This is only supported by Windows. - if _os.name == 'nt' and delete: - flags |= _os.O_TEMPORARY - - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags) - file = _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding) - - return _TemporaryFileWrapper(file, name, delete) - -if _os.name != 'posix' or _sys.platform == 'cygwin': - # On non-POSIX and Cygwin systems, assume that we cannot unlink a file - # while it is open. - TemporaryFile = NamedTemporaryFile - -else: - def _TemporaryFile(mode: str = 'w+b', buffering: int = -1, - encoding: str = None, newline: str = None, - suffix: str = "", prefix: str = template, - dir: str = None, delete: bool = True) -> _IO[_Any]: - """Create and return a temporary file. - Arguments: - 'prefix', 'suffix', 'dir' -- as for mkstemp. - 'mode' -- the mode argument to io.open (default "w+b"). - 'buffering' -- the buffer size argument to io.open (default -1). - 'encoding' -- the encoding argument to io.open (default None) - 'newline' -- the newline argument to io.open (default None) - The file is created as mkstemp() would do it. - - Returns an object with a file-like interface. The file has no - name, and will cease to exist when it is closed. - """ - - if dir is None: - dir = gettempdir() - - flags = _bin_openflags - - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags) - try: - _os.unlink(name) - return _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding) - except: - _os.close(fd) - raise - TemporaryFile = _TemporaryFile - -class SpooledTemporaryFile: - """Temporary file wrapper, specialized to switch from - StringIO to a real file when it exceeds a certain size or - when a fileno is needed. - """ - _rolled = False - _file = _Undefined # type: _Any # BytesIO, StringIO or TemporaryFile - - def __init__(self, max_size: int = 0, mode: str = 'w+b', - buffering: int = -1, encoding: str = None, - newline: str = None, suffix: str = "", - prefix: str = template, dir: str = None) -> None: - if 'b' in mode: - self._file = _io.BytesIO() - else: - # Setting newline="\n" avoids newline translation; - # this is important because otherwise on Windows we'd - # hget double newline translation upon rollover(). - self._file = _io.StringIO(newline="\n") - self._max_size = max_size - self._rolled = False - self._TemporaryFileArgs = { - 'mode': mode, 'buffering': buffering, - 'suffix': suffix, 'prefix': prefix, - 'encoding': encoding, 'newline': newline, - 'dir': dir} # type: _Dict[str, _Any] - - def _check(self, file: _IO[_Any]) -> None: - if self._rolled: return - max_size = self._max_size - if max_size and file.tell() > max_size: - self.rollover() - - def rollover(self) -> None: - if self._rolled: return - file = self._file - newfile = self._file = TemporaryFile(**self._TemporaryFileArgs) - self._TemporaryFileArgs = None - - newfile.write(file.getvalue()) - newfile.seek(file.tell(), 0) - - self._rolled = True - - # The method caching trick from NamedTemporaryFile - # won't work here, because _file may change from a - # _StringIO instance to a real file. So we list - # all the methods directly. - - # Context management protocol - def __enter__(self) -> 'SpooledTemporaryFile': - if self._file.closed: - raise ValueError("Cannot enter context with closed file") - return self - - def __exit__(self, exc: type, value: BaseException, - tb: _Traceback) -> bool: - self._file.close() - - # file protocol - def __iter__(self) -> _Iterable[_Any]: - return self._file.__iter__() - - def close(self) -> None: - self._file.close() - - @property - def closed(self) -> bool: - return self._file.closed - - @property - def encoding(self) -> str: - return self._file.encoding - - def fileno(self) -> int: - self.rollover() - return self._file.fileno() - - def flush(self) -> None: - self._file.flush() - - def isatty(self) -> bool: - return self._file.isatty() - - @property - def mode(self) -> str: - return self._file.mode - - @property - def name(self) -> str: - return self._file.name - - @property - def newlines(self) -> _Any: - return self._file.newlines - - #def next(self): - # return self._file.next - - def read(self, n: int = -1) -> _Any: - return self._file.read(n) - - def readline(self, limit: int = -1) -> _Any: - return self._file.readline(limit) - - def readlines(self, *args) -> _List[_Any]: - return self._file.readlines(*args) - - def seek(self, offset: int, whence: int = 0) -> None: - self._file.seek(offset, whence) - - @property - def softspace(self) -> bool: - return self._file.softspace - - def tell(self) -> int: - return self._file.tell() - - def truncate(self) -> None: - self._file.truncate() - - def write(self, s: _Any) -> int: - file = self._file # type: _IO[_Any] - rv = file.write(s) - self._check(file) - return rv - - def writelines(self, iterable: _Iterable[_Any]) -> None: - file = self._file # type: _IO[_Any] - file.writelines(iterable) - self._check(file) - - #def xreadlines(self, *args) -> _Any: - # return self._file.xreadlines(*args) - - -class TemporaryDirectory(object): - """Create and return a temporary directory. This has the same - behavior as mkdtemp but can be used as a context manager. For - example: - - with TemporaryDirectory() as tmpdir: - ... - - Upon exiting the context, the directory and everthing contained - in it are removed. - """ - - def __init__(self, suffix: str = "", prefix: str = template, - dir: str = None) -> None: - self._closed = False - self.name = None # type: str # Handle mkdtemp throwing an exception - self.name = mkdtemp(suffix, prefix, dir) - - # XXX (ncoghlan): The following code attempts to make - # this class tolerant of the module nulling out process - # that happens during CPython interpreter shutdown - # Alas, it doesn't actually manage it. See issue #10188 - self._listdir = _os.listdir - self._path_join = _os.path.join - self._isdir = _os.path.isdir - self._islink = _os.path.islink - self._remove = _os.remove - self._rmdir = _os.rmdir - self._os_error = _os.error - self._warn = _warnings.warn - - def __repr__(self) -> str: - return "<{} {!r}>".format(self.__class__.__name__, self.name) - - def __enter__(self) -> str: - return self.name - - def cleanup(self, _warn: bool = False) -> None: - if self.name and not self._closed: - try: - self._rmtree(self.name) - except (TypeError, AttributeError) as ex: - # Issue #10188: Emit a warning on stderr - # if the directory could not be cleaned - # up due to missing globals - if "None" not in str(ex): - raise - print("ERROR: {!r} while cleaning up {!r}".format(ex, self,), - file=_sys.stderr) - return - self._closed = True - if _warn: - self._warn("Implicitly cleaning up {!r}".format(self), - ResourceWarning) - - def __exit__(self, exc: type, value: BaseException, - tb: _Traceback) -> bool: - self.cleanup() - - def __del__(self) -> None: - # Issue a ResourceWarning if implicit cleanup needed - self.cleanup(_warn=True) - - def _rmtree(self, path: str) -> None: - # Essentially a stripped down version of shutil.rmtree. We can't - # use globals because they may be None'ed out at shutdown. - for name in self._listdir(path): - fullname = self._path_join(path, name) - try: - isdir = self._isdir(fullname) and not self._islink(fullname) - except self._os_error: - isdir = False - if isdir: - self._rmtree(fullname) - else: - try: - self._remove(fullname) - except self._os_error: - pass - try: - self._rmdir(path) - except self._os_error: - pass diff --git a/lib-python/3.2/test/randv2_32.pck b/lib-python/3.2/test/randv2_32.pck deleted file mode 100644 index 587ab241091e..000000000000 --- a/lib-python/3.2/test/randv2_32.pck +++ /dev/null @@ -1,633 +0,0 @@ -crandom -Random -p0 -(tRp1 -(I2 -(I-2147483648 -I-845974985 -I-1294090086 -I1193659239 -I-1849481736 -I-946579732 -I-34406770 -I1749049471 -I1997774682 -I1432026457 -I1288127073 -I-943175655 -I-1718073964 -I339993548 -I-1045260575 -I582505037 -I-1555108250 -I-1114765620 -I1578648750 -I-350384412 -I-20845848 -I-288255314 -I738790953 -I1901249641 -I1999324672 -I-277361068 -I-1515885839 -I2061761596 -I-809068089 -I1287981136 -I258129492 -I-6303745 -I-765148337 -I1090344911 -I1653434703 -I-1242923628 -I1639171313 -I-1870042660 -I-1655014050 -I345609048 -I2093410138 -I1963263374 -I-2122098342 -I1336859961 -I-810942729 -I945857753 -I2103049942 -I623922684 -I1418349549 -I690877342 -I754973107 -I-1605111847 -I1607137813 -I-1704917131 -I1317536428 -I1714882872 -I-1665385120 -I1823694397 -I-1790836866 -I-1696724812 -I-603979847 -I-498599394 -I-341265291 -I927388804 -I1778562135 -I1716895781 -I1023198122 -I1726145967 -I941955525 -I1240148950 -I-1929634545 -I-1288147083 -I-519318335 -I754559777 -I-707571958 -I374604022 -I420424061 -I-1095443486 -I1621934944 -I-1220502522 -I-140049608 -I-918917122 -I304341024 -I-1637446057 -I-353934485 -I1973436235 -I433380241 -I-686759465 -I-2111563154 -I-573422032 -I804304541 -I1513063483 -I1417381689 -I-804778729 -I211756408 -I544537322 -I890881641 -I150378374 -I1765739392 -I1011604116 -I584889095 -I1400520554 -I413747808 -I-1741992587 -I-1882421574 -I-1373001903 -I-1885348538 -I903819480 -I1083220038 -I-1318105424 -I1740421404 -I1693089625 -I775965557 -I1319608037 -I-2127475785 -I-367562895 -I-1416273451 -I1693000327 -I-1217438421 -I834405522 -I-128287275 -I864057548 -I-973917356 -I7304111 -I1712253182 -I1353897741 -I672982288 -I1778575559 -I-403058377 -I-38540378 -I-1393713496 -I13193171 -I1127196200 -I205176472 -I-2104790506 -I299985416 -I1403541685 -I-1018270667 -I-1980677490 -I-1182625797 -I1637015181 -I-1795357414 -I1514413405 -I-924516237 -I-1841873650 -I-1014591269 -I1576616065 -I-1319103135 -I-120847840 -I2062259778 -I-9285070 -I1160890300 -I-575137313 -I-1509108275 -I46701926 -I-287560914 -I-256824960 -I577558250 -I900598310 -I944607867 -I2121154920 -I-1170505192 -I-1347170575 -I77247778 -I-1899015765 -I1234103327 -I1027053658 -I1934632322 -I-792031234 -I1147322536 -I1290655117 -I1002059715 -I1325898538 -I896029793 -I-790940694 -I-980470721 -I-1922648255 -I-951672814 -I291543943 -I1158740218 -I-1959023736 -I-1977185236 -I1527900076 -I514104195 -I-814154113 -I-593157883 -I-1023704660 -I1285688377 -I-2117525386 -I768954360 -I-38676846 -I-799848659 -I-1305517259 -I-1938213641 -I-462146758 -I-1663302892 -I1899591069 -I-22935388 -I-275856976 -I-443736893 -I-739441156 -I93862068 -I-838105669 -I1735629845 -I-817484206 -I280814555 -I1753547179 -I1811123479 -I1974543632 -I-48447465 -I-642694345 -I-531149613 -I518698953 -I-221642627 -I-686519187 -I776644303 -I257774400 -I-1499134857 -I-1055273455 -I-237023943 -I1981752330 -I-917671662 -I-372905983 -I1588058420 -I1171936660 -I-1730977121 -I1360028989 -I1769469287 -I1910709542 -I-852692959 -I1396944667 -I-1723999155 -I-310975435 -I-1965453954 -I-1636858570 -I2005650794 -I680293715 -I1355629386 -I844514684 -I-1909152807 -I-808646074 -I1936510018 -I1134413810 -I-143411047 -I-1478436304 -I1394969244 -I-1170110660 -I1963112086 -I-1518351049 -I-1506287443 -I-455023090 -I-855366028 -I-1746785568 -I933990882 -I-703625141 -I-285036872 -I188277905 -I1471578620 -I-981382835 -I-586974220 -I945619758 -I1608778444 -I-1708548066 -I-1897629320 -I-42617810 -I-836840790 -I539154487 -I-235706962 -I332074418 -I-575700589 -I1534608003 -I632116560 -I-1819760653 -I642052958 -I-722391771 -I-1104719475 -I-1196847084 -I582413973 -I1563394876 -I642007944 -I108989456 -I361625014 -I677308625 -I-1806529496 -I-959050708 -I-1858251070 -I-216069832 -I701624579 -I501238033 -I12287030 -I1895107107 -I2089098638 -I-874806230 -I1236279203 -I563718890 -I-544352489 -I-1879707498 -I1767583393 -I-1776604656 -I-693294301 -I-88882831 -I169303357 -I1299196152 -I-1122791089 -I-379157172 -I1934671851 -I1575736961 -I-19573174 -I-1401511009 -I9305167 -I-1115174467 -I1670735537 -I1226436501 -I-2004524535 -I1767463878 -I-1722855079 -I-559413926 -I1529810851 -I1201272087 -I-1297130971 -I-1188149982 -I1396557188 -I-370358342 -I-1006619702 -I1600942463 -I906087130 -I-76991909 -I2069580179 -I-1674195181 -I-2098404729 -I-940972459 -I-573399187 -I-1930386277 -I-721311199 -I-647834744 -I1452181671 -I688681916 -I1812793731 -I1704380620 -I-1389615179 -I866287837 -I-1435265007 -I388400782 -I-147986600 -I-1613598851 -I-1040347408 -I782063323 -I-239282031 -I-575966722 -I-1865208174 -I-481365146 -I579572803 -I-1239481494 -I335361280 -I-429722947 -I1881772789 -I1908103808 -I1653690013 -I-1668588344 -I1933787953 -I-2033480609 -I22162797 -I-1516527040 -I-461232482 -I-16201372 -I-2043092030 -I114990337 -I-1524090084 -I1456374020 -I458606440 -I-1928083218 -I227773125 -I-1129028159 -I1678689 -I1575896907 -I-1792935220 -I-151387575 -I64084088 -I-95737215 -I1337335688 -I-1963466345 -I1243315130 -I-1798518411 -I-546013212 -I-607065396 -I1219824160 -I1715218469 -I-1368163783 -I1701552913 -I-381114888 -I1068821717 -I266062971 -I-2066513172 -I1767407229 -I-780936414 -I-705413443 -I-1256268847 -I1646874149 -I1107690353 -I839133072 -I67001749 -I860763503 -I884880613 -I91977084 -I755371933 -I420745153 -I-578480690 -I-1520193551 -I1011369331 -I-99754575 -I-733141064 -I-500598588 -I1081124271 -I-1341266575 -I921002612 -I-848852487 -I-1904467341 -I-1294256973 -I-94074714 -I-1778758498 -I-1401188547 -I2101830578 -I2058864877 -I-272875991 -I-1375854779 -I-1332937870 -I619425525 -I-1034529639 -I-36454393 -I-2030499985 -I-1637127500 -I-1408110287 -I-2108625749 -I-961007436 -I1475654951 -I-791946251 -I1667792115 -I1818978830 -I1897980514 -I1959546477 -I-74478911 -I-508643347 -I461594399 -I538802715 -I-2094970071 -I-2076660253 -I1091358944 -I1944029246 -I-343957436 -I-1915845022 -I1237620188 -I1144125174 -I1522190520 -I-670252952 -I-19469226 -I675626510 -I758750096 -I909724354 -I-1846259652 -I544669343 -I445182495 -I-821519930 -I-1124279685 -I-1668995122 -I1653284793 -I-678555151 -I-687513207 -I1558259445 -I-1978866839 -I1558835601 -I1732138472 -I-1904793363 -I620020296 -I1562597874 -I1942617227 -I-549632552 -I721603795 -I417978456 -I-1355281522 -I-538065208 -I-1079523196 -I187375699 -I449064972 -I1018083947 -I1632388882 -I-493269866 -I92769041 -I1477146750 -I1782708404 -I444873376 -I1085851104 -I-6823272 -I-1302251853 -I1602050688 -I-1042187824 -I287161745 -I-1972094479 -I103271491 -I2131619773 -I-2064115870 -I766815498 -I990861458 -I-1664407378 -I1083746756 -I-1018331904 -I-677315687 -I-951670647 -I-952356874 -I451460609 -I-818615564 -I851439508 -I656362634 -I-1351240485 -I823378078 -I1985597385 -I597757740 -I-1512303057 -I1590872798 -I1108424213 -I818850898 -I-1368594306 -I-201107761 -I1793370378 -I1247597611 -I-1594326264 -I-601653890 -I427642759 -I248322113 -I-292545338 -I1708985870 -I1917042771 -I429354503 -I-478470329 -I793960014 -I369939133 -I1728189157 -I-518963626 -I-278523974 -I-1877289696 -I-2088617658 -I-1367940049 -I-62295925 -I197975119 -I-252900777 -I803430539 -I485759441 -I-528283480 -I-1287443963 -I-478617444 -I-861906946 -I-649095555 -I-893184337 -I2050571322 -I803433133 -I1629574571 -I1649720417 -I-2050225209 -I1208598977 -I720314344 -I-615166251 -I-835077127 -I-1405372429 -I995698064 -I148123240 -I-943016676 -I-594609622 -I-1381596711 -I1017195301 -I-1268893013 -I-1815985179 -I-1393570351 -I-870027364 -I-476064472 -I185582645 -I569863326 -I1098584267 -I-1599147006 -I-485054391 -I-852098365 -I1477320135 -I222316762 -I-1515583064 -I-935051367 -I393383063 -I819617226 -I722921837 -I-1241806499 -I-1358566385 -I1666813591 -I1333875114 -I-1663688317 -I-47254623 -I-885800726 -I307388991 -I-1219459496 -I1374870300 -I2132047877 -I-1385624198 -I-245139206 -I1015139214 -I-926198559 -I1969798868 -I-1950480619 -I-559193432 -I-1256446518 -I-1983476981 -I790179655 -I1004289659 -I1541827617 -I1555805575 -I501127333 -I-1123446797 -I-453230915 -I2035104883 -I1296122398 -I-1843698604 -I-715464588 -I337143971 -I-1972119192 -I606777909 -I726977302 -I-1149501872 -I-1963733522 -I-1797504644 -I624 -tp2 -Ntp3 -b. \ No newline at end of file diff --git a/lib-python/3.2/test/randv2_64.pck b/lib-python/3.2/test/randv2_64.pck deleted file mode 100644 index 090dd6fd1968..000000000000 --- a/lib-python/3.2/test/randv2_64.pck +++ /dev/null @@ -1,633 +0,0 @@ -crandom -Random -p0 -(tRp1 -(I2 -(I2147483648 -I1812115682 -I2741755497 -I1028055730 -I809166036 -I2773628650 -I62321950 -I535290043 -I349877800 -I976167039 -I2490696940 -I3631326955 -I2107991114 -I2941205793 -I3199611605 -I1871971556 -I1456108540 -I2984591044 -I140836801 -I4203227310 -I3652722980 -I4031971234 -I555769760 -I697301296 -I2347638880 -I3302335858 -I320255162 -I2553586608 -I1570224361 -I2838780912 -I2315834918 -I2351348158 -I3545433015 -I2292018579 -I1177569331 -I758497559 -I2913311175 -I1014948880 -I1793619243 -I3982451053 -I3850988342 -I2393984324 -I1583100093 -I3144742543 -I3655047493 -I3507532385 -I3094515442 -I350042434 -I2455294844 -I1038739312 -I313809152 -I189433072 -I1653165452 -I4186650593 -I19281455 -I2589680619 -I4145931590 -I4283266118 -I636283172 -I943618337 -I3170184633 -I2308766231 -I634615159 -I538152647 -I2079576891 -I1029442616 -I3410689412 -I1370292761 -I1071718978 -I2139496322 -I1876699543 -I3485866187 -I3157490130 -I1633105386 -I1453253160 -I3841322080 -I3789608924 -I4110770792 -I95083673 -I931354627 -I2065389591 -I3448339827 -I3348204577 -I3263528560 -I2411324590 -I4003055026 -I1869670093 -I2737231843 -I4150701155 -I2689667621 -I2993263224 -I3239890140 -I1191430483 -I1214399779 -I3623428533 -I1817058866 -I3052274451 -I326030082 -I1505129312 -I2306812262 -I1349150363 -I1099127895 -I2543465574 -I2396380193 -I503926466 -I1607109730 -I3451716817 -I58037114 -I4290081119 -I947517597 -I3083440186 -I520522630 -I2948962496 -I4184319574 -I2957636335 -I668374201 -I2325446473 -I472785314 -I3791932366 -I573017189 -I2185725379 -I1262251492 -I3525089379 -I2951262653 -I1305347305 -I940958122 -I3343754566 -I359371744 -I3874044973 -I396897232 -I147188248 -I716683703 -I4013880315 -I1133359586 -I1794612249 -I3480815192 -I3988787804 -I1729355809 -I573408542 -I1419310934 -I1770030447 -I3552845567 -I1693976502 -I1271189893 -I2298236738 -I2049219027 -I3464198070 -I1233574082 -I1007451781 -I1838253750 -I687096593 -I1131375603 -I1223013895 -I1490478435 -I339265439 -I4232792659 -I491538536 -I2816256769 -I1044097522 -I2566227049 -I748762793 -I1511830494 -I3593259822 -I4121279213 -I3735541309 -I3609794797 -I1939942331 -I377570434 -I1437957554 -I1831285696 -I55062811 -I2046783110 -I1303902283 -I1838349877 -I420993556 -I1256392560 -I2795216506 -I2783687924 -I3322303169 -I512794749 -I308405826 -I517164429 -I3320436022 -I1328403632 -I2269184746 -I3729522810 -I3304314450 -I2238756124 -I1690581361 -I3813277532 -I4119706879 -I2659447875 -I388818978 -I2064580814 -I1586227676 -I2627522685 -I2017792269 -I547928109 -I859107450 -I1062238929 -I858886237 -I3795783146 -I4173914756 -I3835915965 -I3329504821 -I3494579904 -I838863205 -I3399734724 -I4247387481 -I3618414834 -I2984433798 -I2165205561 -I4260685684 -I3045904244 -I3450093836 -I3597307595 -I3215851166 -I3162801328 -I2558283799 -I950068105 -I1829664117 -I3108542987 -I2378860527 -I790023460 -I280087750 -I1171478018 -I2333653728 -I3976932140 -I896746152 -I1802494195 -I1232873794 -I2749440836 -I2032037296 -I2012091682 -I1296131034 -I3892133385 -I908161334 -I2296791795 -I548169794 -I696265 -I893156828 -I426904709 -I3565374535 -I2655906825 -I2792178515 -I2406814632 -I4038847579 -I3123934642 -I2197503004 -I3535032597 -I2266216689 -I2117613462 -I1787448518 -I1875089416 -I2037165384 -I1140676321 -I3606296464 -I3229138231 -I2458267132 -I1874651171 -I3331900867 -I1000557654 -I1432861701 -I473636323 -I2691783927 -I1871437447 -I1328016401 -I4118690062 -I449467602 -I681789035 -I864889442 -I1200888928 -I75769445 -I4008690037 -I2464577667 -I4167795823 -I3070097648 -I2579174882 -I1216886568 -I3810116343 -I2249507485 -I3266903480 -I3671233480 -I100191658 -I3087121334 -I365063087 -I3821275176 -I2165052848 -I1282465245 -I3601570637 -I3132413236 -I2780570459 -I3222142917 -I3129794692 -I2611590811 -I947031677 -I2991908938 -I750997949 -I3632575131 -I1632014461 -I2846484755 -I2347261779 -I2903959448 -I1397316686 -I1904578392 -I774649578 -I3164598558 -I2429587609 -I738244516 -I1563304975 -I1399317414 -I1021316297 -I3187933234 -I2126780757 -I4011907847 -I4095169219 -I3358010054 -I2729978247 -I3736811646 -I3009656410 -I2893043637 -I4027447385 -I1239610110 -I1488806900 -I2674866844 -I442876374 -I2853687260 -I2785921005 -I3151378528 -I1180567 -I2803146964 -I982221759 -I2192919417 -I3087026181 -I2480838002 -I738452921 -I687986185 -I3049371676 -I3636492954 -I3468311299 -I2379621102 -I788988633 -I1643210601 -I2983998168 -I2492730801 -I2586048705 -I604073029 -I4121082815 -I1496476928 -I2972357110 -I2663116968 -I2642628592 -I2116052039 -I487186279 -I2577680328 -I3974766614 -I730776636 -I3842528855 -I1929093695 -I44626622 -I3989908833 -I1695426222 -I3675479382 -I3051784964 -I1514876613 -I1254036595 -I2420450649 -I3034377361 -I2332990590 -I1535175126 -I185834384 -I1107372900 -I1707278185 -I1286285295 -I3332574225 -I2785672437 -I883170645 -I2005666473 -I3403131327 -I4122021352 -I1464032858 -I3702576112 -I260554598 -I1837731650 -I2594435345 -I75771049 -I2012484289 -I3058649775 -I29979703 -I3861335335 -I2506495152 -I3786448704 -I442947790 -I2582724774 -I4291336243 -I2568189843 -I1923072690 -I1121589611 -I837696302 -I3284631720 -I3865021324 -I3576453165 -I2559531629 -I1459231762 -I3506550036 -I3754420159 -I2622000757 -I124228596 -I1084328605 -I1692830753 -I547273558 -I674282621 -I655259103 -I3188629610 -I490502174 -I2081001293 -I3191330704 -I4109943593 -I1859948504 -I3163806460 -I508833168 -I1256371033 -I2709253790 -I2068956572 -I3092842814 -I3913926529 -I2039638759 -I981982529 -I536094190 -I368855295 -I51993975 -I1597480732 -I4058175522 -I2155896702 -I3196251991 -I1081913893 -I3952353788 -I3545548108 -I2370669647 -I2206572308 -I2576392991 -I1732303374 -I1153136290 -I537641955 -I1738691747 -I3232854186 -I2539632206 -I2829760278 -I3058187853 -I1202425792 -I3762361970 -I2863949342 -I2640635867 -I376638744 -I1857679757 -I330798087 -I1457400505 -I1135610046 -I606400715 -I1859536026 -I509811335 -I529772308 -I2579273244 -I1890382004 -I3959908876 -I2612335971 -I2834052227 -I1434475986 -I3684202717 -I4015011345 -I582567852 -I3689969571 -I3934753460 -I3034960691 -I208573292 -I4004113742 -I3992904842 -I2587153719 -I3529179079 -I1565424987 -I779130678 -I1048582935 -I3213591622 -I3607793434 -I3951254937 -I2047811901 -I7508850 -I248544605 -I4210090324 -I2331490884 -I70057213 -I776474945 -I1345528889 -I3290403612 -I1664955269 -I1533143116 -I545003424 -I4141564478 -I1257326139 -I868843601 -I2337603029 -I1918131449 -I1843439523 -I1125519035 -I673340118 -I421408852 -I1520454906 -I1804722630 -I3621254196 -I2329968000 -I39464672 -I430583134 -I294026512 -I53978525 -I2892276105 -I1418863764 -I3419054451 -I1391595797 -I3544981798 -I4191780858 -I825672357 -I2972000844 -I1571305069 -I4231982845 -I3611916419 -I3045163168 -I2982349733 -I278572141 -I4215338078 -I839860504 -I1819151779 -I1412347479 -I1386770353 -I3914589491 -I3783104977 -I4124296733 -I830546258 -I89825624 -I4110601328 -I2545483429 -I300600527 -I516641158 -I3693021034 -I2852912854 -I3240039868 -I4167407959 -I1479557946 -I3621188804 -I1391590944 -I3578441128 -I1227055556 -I406898396 -I3064054983 -I25835338 -I402664165 -I4097682779 -I2106728012 -I203613622 -I3045467686 -I1381726438 -I3798670110 -I1342314961 -I3552497361 -I535913619 -I2625787583 -I1606574307 -I1101269630 -I1950513752 -I1121355862 -I3586816903 -I438529984 -I2473182121 -I1229997203 -I405445940 -I1695535315 -I427014336 -I3916768430 -I392298359 -I1884642868 -I1244730821 -I741058080 -I567479957 -I3527621168 -I3191971011 -I3267069104 -I4108668146 -I1520795587 -I166581006 -I473794477 -I1562126550 -I929843010 -I889533294 -I1266556608 -I874518650 -I3520162092 -I3013765049 -I4220231414 -I547246449 -I3998093769 -I3737193746 -I3872944207 -I793651876 -I2606384318 -I875991012 -I1394836334 -I4102011644 -I854380426 -I2618666767 -I2568302000 -I1995512132 -I229491093 -I2673500286 -I3364550739 -I3836923416 -I243656987 -I3944388983 -I4064949677 -I1416956378 -I1703244487 -I3990798829 -I2023425781 -I3926702214 -I1229015501 -I3174247824 -I624 -tp2 -Ntp3 -b. \ No newline at end of file diff --git a/lib-python/3.2/test/randv3.pck b/lib-python/3.2/test/randv3.pck deleted file mode 100644 index 09fc38b1a876..000000000000 --- a/lib-python/3.2/test/randv3.pck +++ /dev/null @@ -1,633 +0,0 @@ -crandom -Random -p0 -(tRp1 -(I3 -(L2147483648L -L994081831L -L2806287265L -L2228999830L -L3396498069L -L2956805457L -L3273927761L -L920726507L -L1862624492L -L2921292485L -L1779526843L -L2469105503L -L251696293L -L1254390717L -L779197080L -L3165356830L -L2007365218L -L1870028812L -L2896519363L -L1855578438L -L979518416L -L3481710246L -L3191861507L -L3993006593L -L2967971479L -L3353342753L -L3576782572L -L339685558L -L2367675732L -L116208555L -L1220054437L -L486597056L -L1912115141L -L1037044792L -L4096904723L -L3409146175L -L3701651227L -L315824610L -L4138604583L -L1385764892L -L191878900L -L2320582219L -L3420677494L -L2776503169L -L1148247403L -L829555069L -L902064012L -L2934642741L -L2477108577L -L2583928217L -L1658612579L -L2865447913L -L129147346L -L3691171887L -L1569328110L -L1372860143L -L1054139183L -L1617707080L -L69020592L -L3810271603L -L1853953416L -L3499803073L -L1027545027L -L3229043605L -L250848720L -L3324932626L -L3537002962L -L2494323345L -L3238103962L -L4147541579L -L3636348186L -L3025455083L -L2678771977L -L584700256L -L3461826909L -L854511420L -L943463552L -L3609239025L -L3977577989L -L253070090L -L777394544L -L2144086567L -L1092947992L -L854327284L -L2222750082L -L360183510L -L1312466483L -L3227531091L -L2235022500L -L3013060530L -L2541091298L -L3480126342L -L1839762775L -L2632608190L -L1108889403L -L3045050923L -L731513126L -L3505436788L -L3062762017L -L1667392680L -L1354126500L -L1143573930L -L2816645702L -L2100356873L -L2817679106L -L1210746010L -L2409915248L -L2910119964L -L2309001420L -L220351824L -L3667352871L -L3993148590L -L2886160232L -L4239393701L -L1189270581L -L3067985541L -L147374573L -L2355164869L -L3696013550L -L4227037846L -L1905112743L -L3312843689L -L2930678266L -L1828795355L -L76933594L -L3987100796L -L1288361435L -L3464529151L -L965498079L -L1444623093L -L1372893415L -L1536235597L -L1341994850L -L963594758L -L2115295754L -L982098685L -L1053433904L -L2078469844L -L3059765792L -L1753606181L -L2130171254L -L567588194L -L529629426L -L3621523534L -L3027576564L -L1176438083L -L4096287858L -L1168574683L -L1425058962L -L1429631655L -L2902106759L -L761900641L -L1329183956L -L1947050932L -L447490289L -L3282516276L -L200037389L -L921868197L -L3331403999L -L4088760249L -L2188326318L -L288401961L -L1360802675L -L314302808L -L3314639210L -L3749821203L -L2286081570L -L2768939062L -L3200541016L -L2133495482L -L385029880L -L4217232202L -L3171617231L -L1660846653L -L2459987621L -L2691776124L -L4225030408L -L3595396773L -L1103680661L -L539064057L -L1492841101L -L166195394L -L757973658L -L533893054L -L2784879594L -L1021821883L -L2350548162L -L176852116L -L3503166025L -L148079914L -L1633466236L -L2773090165L -L1162846701L -L3575737795L -L1624178239L -L2454894710L -L3014691938L -L526355679L -L1870824081L -L3362425857L -L3907566665L -L3462563184L -L2229112004L -L4203735748L -L1557442481L -L924133999L -L1906634214L -L880459727L -L4065895870L -L141426254L -L1258450159L -L3243115027L -L1574958840L -L313939294L -L3055664260L -L3459714255L -L531778790L -L509505506L -L1620227491L -L2675554942L -L2516509560L -L3797299887L -L237135890L -L3203142213L -L1087745310L -L1897151854L -L3936590041L -L132765167L -L2385908063L -L1360600289L -L3574567769L -L2752788114L -L2644228966L -L2377705183L -L601277909L -L4046480498L -L324401408L -L3279931760L -L2227059377L -L1538827493L -L4220532064L -L478044564L -L2917117761L -L635492832L -L2319763261L -L795944206L -L1820473234L -L1673151409L -L1404095402L -L1661067505L -L3217106938L -L2406310683L -L1931309248L -L2458622868L -L3323670524L -L3266852755L -L240083943L -L3168387397L -L607722198L -L1256837690L -L3608124913L -L4244969357L -L1289959293L -L519750328L -L3229482463L -L1105196988L -L1832684479L -L3761037224L -L2363631822L -L3297957711L -L572766355L -L1195822137L -L2239207981L -L2034241203L -L163540514L -L288160255L -L716403680L -L4019439143L -L1536281935L -L2345100458L -L2786059178L -L2822232109L -L987025395L -L3061166559L -L490422513L -L2551030115L -L2638707620L -L1344728502L -L714108911L -L2831719700L -L2188615369L -L373509061L -L1351077504L -L3136217056L -L783521095L -L2554949468L -L2662499550L -L1203826951L -L1379632388L -L1918858985L -L607465976L -L1980450237L -L3540079211L -L3397813410L -L2913309266L -L2289572621L -L4133935327L -L4166227663L -L3371801704L -L3065474909L -L3580562343L -L3832172378L -L2556130719L -L310473705L -L3734014346L -L2490413810L -L347233056L -L526668037L -L1158393656L -L544329703L -L2150085419L -L3914038146L -L1060237586L -L4159394837L -L113205121L -L309966775L -L4098784465L -L3635222960L -L2417516569L -L2089579233L -L1725807541L -L2728122526L -L2365836523L -L2504078522L -L1443946869L -L2384171411L -L997046534L -L3249131657L -L1699875986L -L3618097146L -L1716038224L -L2629818607L -L2929217876L -L1367250314L -L1726434951L -L1388496325L -L2107602181L -L2822366842L -L3052979190L -L3796798633L -L1543813381L -L959000121L -L1363845999L -L2952528150L -L874184932L -L1888387194L -L2328695295L -L3442959855L -L841805947L -L1087739275L -L3230005434L -L3045399265L -L1161817318L -L2898673139L -L860011094L -L940539782L -L1297818080L -L4243941623L -L1577613033L -L4204131887L -L3819057225L -L1969439558L -L3297963932L -L241874069L -L3517033453L -L2295345664L -L1098911422L -L886955008L -L1477397621L -L4279347332L -L3616558791L -L2384411957L -L742537731L -L764221540L -L2871698900L -L3530636393L -L691256644L -L758730966L -L1717773090L -L2751856377L -L3188484000L -L3767469670L -L1623863053L -L3533236793L -L4099284176L -L723921107L -L310594036L -L223978745L -L2266565776L -L201843303L -L2969968546L -L3351170888L -L3465113624L -L2712246712L -L1521383057L -L2384461798L -L216357551L -L2167301975L -L3144653194L -L2781220155L -L3620747666L -L95971265L -L4255400243L -L59999757L -L4174273472L -L3974511524L -L1007123950L -L3112477628L -L806461512L -L3148074008L -L528352882L -L2545979588L -L2562281969L -L3010249477L -L1886331611L -L3210656433L -L1034099976L -L2906893579L -L1197048779L -L1870004401L -L3898300490L -L2686856402L -L3975723478L -L613043532L -L2565674353L -L3760045310L -L3468984376L -L4126258L -L303855424L -L3988963552L -L276256796L -L544071807L -L1023872062L -L1747461519L -L1975571260L -L4033766958L -L2946555557L -L1492957796L -L958271685L -L46480515L -L907760635L -L1306626357L -L819652378L -L1172300279L -L1116851319L -L495601075L -L1157715330L -L534220108L -L377320028L -L1672286106L -L2066219284L -L1842386355L -L2546059464L -L1839457336L -L3476194446L -L3050550028L -L594705582L -L1905813535L -L1813033412L -L2700858157L -L169067972L -L4252889045L -L1921944555L -L497671474L -L210143935L -L2688398489L -L325158375L -L3450846447L -L891760597L -L712802536L -L1132557436L -L1417044075L -L1639889660L -L1746379970L -L1478741647L -L2817563486L -L2573612532L -L4266444457L -L2911601615L -L804745411L -L2207254652L -L1189140646L -L3829725111L -L3637367348L -L1944731747L -L2193440343L -L1430195413L -L1173515229L -L1582618217L -L2070767037L -L247908936L -L1460675439L -L556001596L -L327629335L -L1036133876L -L4228129605L -L999174048L -L3635804039L -L1416550481L -L1270540269L -L4280743815L -L39607659L -L1552540623L -L2762294062L -L504137289L -L4117044239L -L1417130225L -L1342970056L -L1755716449L -L1169447322L -L2731401356L -L2319976745L -L2869221479L -L23972655L -L2251495389L -L1429860878L -L3728135992L -L4241432973L -L3698275076L -L216416432L -L4040046960L -L246077176L -L894675685L -L3932282259L -L3097205100L -L2128818650L -L1319010656L -L1601974009L -L2552960957L -L3554016055L -L4209395641L -L2013340102L -L3370447801L -L2307272002L -L1795091354L -L202109401L -L988345070L -L2514870758L -L1132726850L -L582746224L -L3112305421L -L1843020683L -L3600189223L -L1101349165L -L4211905855L -L2866677581L -L2881621130L -L4165324109L -L4238773191L -L3635649550L -L2670481044L -L2996248219L -L1676992480L -L3473067050L -L4205793699L -L4019490897L -L1579990481L -L1899617990L -L1136347713L -L1802842268L -L3591752960L -L1197308739L -L433629786L -L4032142790L -L3148041979L -L3312138845L -L3896860449L -L3298182567L -L907605170L -L1658664067L -L2682980313L -L2523523173L -L1208722103L -L3808530363L -L1079003946L -L4282402864L -L2041010073L -L2667555071L -L688018180L -L1405121012L -L4167994076L -L3504695336L -L1923944749L -L1143598790L -L3936268898L -L3606243846L -L1017420080L -L4026211169L -L596529763L -L1844259624L -L2840216282L -L2673807759L -L3407202575L -L2737971083L -L4075423068L -L3684057432L -L3146627241L -L599650513L -L69773114L -L1257035919L -L807485291L -L2376230687L -L3036593147L -L2642411658L -L106080044L -L2199622729L -L291834511L -L2697611361L -L11689733L -L625123952L -L3226023062L -L3229663265L -L753059444L -L2843610189L -L624L -tp2 -Ntp3 -b. \ No newline at end of file diff --git a/lib-python/3.2/test/subprocessdata/fd_status.py b/lib-python/3.2/test/subprocessdata/fd_status.py deleted file mode 100644 index 1f61e13a3456..000000000000 --- a/lib-python/3.2/test/subprocessdata/fd_status.py +++ /dev/null @@ -1,24 +0,0 @@ -"""When called as a script, print a comma-separated list of the open -file descriptors on stdout.""" - -import errno -import os - -try: - _MAXFD = os.sysconf("SC_OPEN_MAX") -except: - _MAXFD = 256 - -if __name__ == "__main__": - fds = [] - for fd in range(0, _MAXFD): - try: - st = os.fstat(fd) - except OSError as e: - if e.errno == errno.EBADF: - continue - raise - # Ignore Solaris door files - if st.st_mode & 0xF000 != 0xd000: - fds.append(fd) - print(','.join(map(str, fds))) diff --git a/lib-python/3.2/test/subprocessdata/input_reader.py b/lib-python/3.2/test/subprocessdata/input_reader.py deleted file mode 100644 index 1dc3191ad183..000000000000 --- a/lib-python/3.2/test/subprocessdata/input_reader.py +++ /dev/null @@ -1,7 +0,0 @@ -"""When called as a script, consumes the input""" - -import sys - -if __name__ == "__main__": - for line in sys.stdin: - pass diff --git a/lib-python/3.2/test/subprocessdata/qcat.py b/lib-python/3.2/test/subprocessdata/qcat.py deleted file mode 100644 index fe6f9db25c97..000000000000 --- a/lib-python/3.2/test/subprocessdata/qcat.py +++ /dev/null @@ -1,7 +0,0 @@ -"""When ran as a script, simulates cat with no arguments.""" - -import sys - -if __name__ == "__main__": - for line in sys.stdin: - sys.stdout.write(line) diff --git a/lib-python/3.2/test/subprocessdata/qgrep.py b/lib-python/3.2/test/subprocessdata/qgrep.py deleted file mode 100644 index 69906379a9b3..000000000000 --- a/lib-python/3.2/test/subprocessdata/qgrep.py +++ /dev/null @@ -1,10 +0,0 @@ -"""When called with a single argument, simulated fgrep with a single -argument and no options.""" - -import sys - -if __name__ == "__main__": - pattern = sys.argv[1] - for line in sys.stdin: - if pattern in line: - sys.stdout.write(line) diff --git a/lib-python/3.2/test/subprocessdata/sigchild_ignore.py b/lib-python/3.2/test/subprocessdata/sigchild_ignore.py deleted file mode 100644 index 6072aece28a8..000000000000 --- a/lib-python/3.2/test/subprocessdata/sigchild_ignore.py +++ /dev/null @@ -1,6 +0,0 @@ -import signal, subprocess, sys -# On Linux this causes os.waitpid to fail with OSError as the OS has already -# reaped our child process. The wait() passing the OSError on to the caller -# and causing us to exit with an error is what we are testing against. -signal.signal(signal.SIGCHLD, signal.SIG_IGN) -subprocess.Popen([sys.executable, '-c', 'print("albatross")']).wait() diff --git a/lib-python/3.2/test/support.py b/lib-python/3.2/test/support.py deleted file mode 100644 index b7af14aeeb2d..000000000000 --- a/lib-python/3.2/test/support.py +++ /dev/null @@ -1,1602 +0,0 @@ -"""Supporting definitions for the Python regression tests.""" - -if __name__ != 'test.support': - raise ImportError('support must be imported from the test package') - -import contextlib -import errno -import functools -import gc -import socket -import sys -import os -import platform -import shutil -import warnings -import unittest -import importlib -import collections -import re -import subprocess -import imp -import time -import sysconfig -import fnmatch -import logging.handlers - -import _thread, threading -from typing import Any, Dict -#try: -# import multiprocessing.process -#except ImportError: -# multiprocessing = None - - -__all__ = [ - "Error", "TestFailed", "ResourceDenied", "import_module", - "verbose", "use_resources", "max_memuse", "record_original_stdout", - "get_original_stdout", "unload", "unlink", "rmtree", "forget", - "is_resource_enabled", "requires", "requires_mac_ver", - "find_unused_port", "bind_port", - "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", - "findfile", "sortdict", "check_syntax_error", "open_urlresource", - "check_warnings", "CleanImport", "EnvironmentVarGuard", - "TransientResource", "captured_output", "captured_stdout", - "captured_stdin", "captured_stderr", - "time_out", "socket_peer_reset", "ioerror_peer_reset", - "run_with_locale", 'temp_umask', "transient_internet", - "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", - "run_unittest", "run_doctest", "threading_setup", "threading_cleanup", - "reap_children", "cpython_only", "check_impl_detail", "get_attribute", - "swap_item", "swap_attr", "requires_IEEE_754", - "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink", - "import_fresh_module", "failfast", - ] - -class Error(Exception): - """Base class for regression test exceptions.""" - -class TestFailed(Error): - """Test failed.""" - -class ResourceDenied(unittest.SkipTest): - """Test skipped because it requested a disallowed resource. - - This is raised when a test calls requires() for a resource that - has not be enabled. It is used to distinguish between expected - and unexpected skips. - """ - -@contextlib.contextmanager -def _ignore_deprecated_imports(ignore=True): - """Context manager to suppress package and module deprecation - warnings when importing them. - - If ignore is False, this context manager has no effect.""" - if ignore: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", ".+ (module|package)", - DeprecationWarning) - yield None - else: - yield None - - -def import_module(name, deprecated=False): - """Import and return the module to be tested, raising SkipTest if - it is not available. - - If deprecated is True, any module or package deprecation messages - will be suppressed.""" - with _ignore_deprecated_imports(deprecated): - try: - return importlib.import_module(name) - except ImportError as msg: - raise unittest.SkipTest(str(msg)) - - -def _save_and_remove_module(name, orig_modules): - """Helper function to save and remove a module from sys.modules - - Raise ImportError if the module can't be imported.""" - # try to import the module and raise an error if it can't be imported - if name not in sys.modules: - __import__(name) - del sys.modules[name] - for modname in list(sys.modules): - if modname == name or modname.startswith(name + '.'): - orig_modules[modname] = sys.modules[modname] - del sys.modules[modname] - -def _save_and_block_module(name, orig_modules): - """Helper function to save and block a module in sys.modules - - Return True if the module was in sys.modules, False otherwise.""" - saved = True - try: - orig_modules[name] = sys.modules[name] - except KeyError: - saved = False - sys.modules[name] = None - return saved - - -def import_fresh_module(name, fresh=(), blocked=(), deprecated=False): - """Imports and returns a module, deliberately bypassing the sys.modules cache - and importing a fresh copy of the module. Once the import is complete, - the sys.modules cache is restored to its original state. - - Modules named in fresh are also imported anew if needed by the import. - If one of these modules can't be imported, None is returned. - - Importing of modules named in blocked is prevented while the fresh import - takes place. - - If deprecated is True, any module or package deprecation messages - will be suppressed.""" - # NOTE: test_heapq, test_json and test_warnings include extra sanity checks - # to make sure that this utility function is working as expected - with _ignore_deprecated_imports(deprecated): - # Keep track of modules saved for later restoration as well - # as those which just need a blocking entry removed - orig_modules = {} - names_to_remove = [] - _save_and_remove_module(name, orig_modules) - try: - for fresh_name in fresh: - _save_and_remove_module(fresh_name, orig_modules) - for blocked_name in blocked: - if not _save_and_block_module(blocked_name, orig_modules): - names_to_remove.append(blocked_name) - fresh_module = importlib.import_module(name) - except ImportError: - fresh_module = None - finally: - for orig_name, module in orig_modules.items(): - sys.modules[orig_name] = module - for name_to_remove in names_to_remove: - del sys.modules[name_to_remove] - return fresh_module - - -def get_attribute(obj, name): - """Get an attribute, raising SkipTest if AttributeError is raised.""" - try: - attribute = getattr(obj, name) - except AttributeError: - raise unittest.SkipTest("module %s has no attribute %s" % ( - obj.__name__, name)) - else: - return attribute - -verbose = 1 # Flag set to 0 by regrtest.py -use_resources = None # type: Any # Flag set to [] by regrtest.py -max_memuse = 0 # Disable bigmem tests (they will still be run with - # small sizes, to make sure they work.) -real_max_memuse = 0 -failfast = False -match_tests = None # type: Any - -# _original_stdout is meant to hold stdout at the time regrtest began. -# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. -# The point is to have some flavor of stdout the user can actually see. -_original_stdout = None # type: 'Any' -def record_original_stdout(stdout): - global _original_stdout - _original_stdout = stdout - -def get_original_stdout(): - return _original_stdout or sys.stdout - -def unload(name): - try: - del sys.modules[name] - except KeyError: - pass - -def unlink(filename): - try: - os.unlink(filename) - except OSError as error: - # The filename need not exist. - if error.errno not in (errno.ENOENT, errno.ENOTDIR): - raise - -def rmtree(path): - try: - shutil.rmtree(path) - except OSError as error: - # Unix returns ENOENT, Windows returns ESRCH. - if error.errno not in (errno.ENOENT, errno.ESRCH): - raise - -def make_legacy_pyc(source): - """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location. - - The choice of .pyc or .pyo extension is done based on the __debug__ flag - value. - - :param source: The file system path to the source file. The source file - does not need to exist, however the PEP 3147 pyc file must exist. - :return: The file system path to the legacy pyc file. - """ - pyc_file = imp.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - if __debug__: - ch = 'c' - else: - ch = 'o' - legacy_pyc = os.path.join(up_one, source + ch) - os.rename(pyc_file, legacy_pyc) - return legacy_pyc - -def forget(modname): - """'Forget' a module was ever imported. - - This removes the module from sys.modules and deletes any PEP 3147 or - legacy .pyc and .pyo files. - """ - unload(modname) - for dirname in sys.path: - source = os.path.join(dirname, modname + '.py') - # It doesn't matter if they exist or not, unlink all possible - # combinations of PEP 3147 and legacy pyc and pyo files. - unlink(source + 'c') - unlink(source + 'o') - unlink(imp.cache_from_source(source, debug_override=True)) - unlink(imp.cache_from_source(source, debug_override=False)) - -# On some platforms, should not run gui test even if it is allowed -# in `use_resources'. -#if sys.platform.startswith('win'): - #import ctypes - #import ctypes.wintypes - #def _is_gui_available(): - # UOI_FLAGS = 1 - # WSF_VISIBLE = 0x0001 - # class USEROBJECTFLAGS(ctypes.Structure): - # _fields_ = [("fInherit", ctypes.wintypes.BOOL), - # ("fReserved", ctypes.wintypes.BOOL), - # ("dwFlags", ctypes.wintypes.DWORD)] - # dll = ctypes.windll.user32 - # h = dll.GetProcessWindowStation() - # if not h: - # raise ctypes.WinError() - # uof = USEROBJECTFLAGS() - # needed = ctypes.wintypes.DWORD() - # res = dll.GetUserObjectInformationW(h, - # UOI_FLAGS, - # ctypes.byref(uof), - # ctypes.sizeof(uof), - # ctypes.byref(needed)) - # if not res: - # raise ctypes.WinError() - # return bool(uof.dwFlags & WSF_VISIBLE) -#else: -def _is_gui_available(): - return True - -def is_resource_enabled(resource): - """Test whether a resource is enabled. Known resources are set by - regrtest.py.""" - return use_resources is not None and resource in use_resources - -def requires(resource, msg=None): - """Raise ResourceDenied if the specified resource is not available. - - If the caller's module is __main__ then automatically return True. The - possibility of False being returned occurs when regrtest.py is - executing. - """ - if resource == 'gui' and not _is_gui_available(): - raise unittest.SkipTest("Cannot use the 'gui' resource") - # see if the caller's module is __main__ - if so, treat as if - # the resource was set - if sys._getframe(1).f_globals.get("__name__") == "__main__": - return - if not is_resource_enabled(resource): - if msg is None: - msg = "Use of the `%s' resource not enabled" % resource - raise ResourceDenied(msg) - -def requires_mac_ver(*min_version): - """Decorator raising SkipTest if the OS is Mac OS X and the OS X - version if less than min_version. - - For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version - is lesser than 10.5. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kw): - if sys.platform == 'darwin': - version_txt = platform.mac_ver()[0] - try: - version = tuple(map(int, version_txt.split('.'))) - except ValueError: - pass - else: - if version < min_version: - min_version_txt = '.'.join(map(str, min_version)) - raise unittest.SkipTest( - "Mac OS X %s or higher required, not %s" - % (min_version_txt, version_txt)) - return func(*args, **kw) - wrapper.min_version = min_version - return wrapper - return decorator - -HOST = 'localhost' - -def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): - """Returns an unused port that should be suitable for binding. This is - achieved by creating a temporary socket with the same family and type as - the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to - the specified host address (defaults to 0.0.0.0) with the port set to 0, - eliciting an unused ephemeral port from the OS. The temporary socket is - then closed and deleted, and the ephemeral port is returned. - - Either this method or bind_port() should be used for any tests where a - server socket needs to be bound to a particular port for the duration of - the test. Which one to use depends on whether the calling code is creating - a python socket, or if an unused port needs to be provided in a constructor - or passed to an external program (i.e. the -accept argument to openssl's - s_server mode). Always prefer bind_port() over find_unused_port() where - possible. Hard coded ports should *NEVER* be used. As soon as a server - socket is bound to a hard coded port, the ability to run multiple instances - of the test simultaneously on the same host is compromised, which makes the - test a ticking time bomb in a buildbot environment. On Unix buildbots, this - may simply manifest as a failed test, which can be recovered from without - intervention in most cases, but on Windows, the entire python process can - completely and utterly wedge, requiring someone to log in to the buildbot - and manually kill the affected process. - - (This is easy to reproduce on Windows, unfortunately, and can be traced to - the SO_REUSEADDR socket option having different semantics on Windows versus - Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, - listen and then accept connections on identical host/ports. An EADDRINUSE - socket.error will be raised at some point (depending on the platform and - the order bind and listen were called on each socket). - - However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE - will ever be raised when attempting to bind two identical host/ports. When - accept() is called on each socket, the second caller's process will steal - the port from the first caller, leaving them both in an awkwardly wedged - state where they'll no longer respond to any signals or graceful kills, and - must be forcibly killed via OpenProcess()/TerminateProcess(). - - The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option - instead of SO_REUSEADDR, which effectively affords the same semantics as - SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open - Source world compared to Windows ones, this is a common mistake. A quick - look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when - openssl.exe is called with the 's_server' option, for example. See - http://bugs.python.org/issue2550 for more info. The following site also - has a very thorough description about the implications of both REUSEADDR - and EXCLUSIVEADDRUSE on Windows: - http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx) - - XXX: although this approach is a vast improvement on previous attempts to - elicit unused ports, it rests heavily on the assumption that the ephemeral - port returned to us by the OS won't immediately be dished back out to some - other process when we close and delete our temporary socket but before our - calling code has a chance to bind the returned port. We can deal with this - issue if/when we come across it. - """ - - tempsock = socket.socket(family, socktype) - port = bind_port(tempsock) - tempsock.close() - #del tempsock - return port - -def bind_port(sock, host=HOST): - """Bind the socket to a free port and return the port number. Relies on - ephemeral ports in order to ensure we are using an unbound port. This is - important as many tests may be running simultaneously, especially in a - buildbot environment. This method raises an exception if the sock.family - is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR - or SO_REUSEPORT set on it. Tests should *never* set these socket options - for TCP/IP sockets. The only case for setting these options is testing - multicasting via multiple UDP sockets. - - Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e. - on Windows), it will be set on the socket. This will prevent anyone else - from bind()'ing to our host/port for the duration of the test. - """ - - if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM: - if hasattr(socket, 'SO_REUSEADDR'): - if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: - raise TestFailed("tests should never set the SO_REUSEADDR " \ - "socket option on TCP/IP sockets!") - if hasattr(socket, 'SO_REUSEPORT'): - if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: - raise TestFailed("tests should never set the SO_REUSEPORT " \ - "socket option on TCP/IP sockets!") - if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) - - sock.bind((host, 0)) - port = sock.getsockname()[1] - return port - -FUZZ = 1e-6 - -def fcmp(x, y): # fuzzy comparison function - if isinstance(x, float) or isinstance(y, float): - try: - fuzz = (abs(x) + abs(y)) * FUZZ - if abs(x-y) <= fuzz: - return 0 - except: - pass - elif type(x) == type(y) and isinstance(x, (tuple, list)): - for i in range(min(len(x), len(y))): - outcome = fcmp(x[i], y[i]) - if outcome != 0: - return outcome - return (len(x) > len(y)) - (len(x) < len(y)) - return (x > y) - (x < y) - -# decorator for skipping tests on non-IEEE 754 platforms -requires_IEEE_754 = unittest.skipUnless( - (Any(float)).__getformat__("double").startswith("IEEE"), - "test requires IEEE 754 doubles") - -is_jython = sys.platform.startswith('java') - -TESTFN = '' -# Filename used for testing -if os.name == 'java': - # Jython disallows @ in module names - TESTFN = '$test' -else: - TESTFN = '@test' - -# Disambiguate TESTFN for parallel testing, while letting it remain a valid -# module name. -TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid()) - - -# TESTFN_UNICODE is a non-ascii filename -TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f" -if sys.platform == 'darwin': - # In Mac OS X's VFS API file names are, by definition, canonically - # decomposed Unicode, encoded using UTF-8. See QA1173: - # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html - import unicodedata - TESTFN_UNICODE = unicodedata.normalize('NFD', TESTFN_UNICODE) -TESTFN_ENCODING = sys.getfilesystemencoding() - -# TESTFN_UNENCODABLE is a filename (str type) that should *not* be able to be -# encoded by the filesystem encoding (in strict mode). It can be None if we -# cannot generate such filename. -TESTFN_UNENCODABLE = None # type: Any -if os.name in ('nt', 'ce'): - # skip win32s (0) or Windows 9x/ME (1) - if sys.getwindowsversion().platform >= 2: - # Different kinds of characters from various languages to minimize the - # probability that the whole name is encodable to MBCS (issue #9819) - TESTFN_UNENCODABLE = TESTFN + "-\u5171\u0141\u2661\u0363\uDC80" - try: - TESTFN_UNENCODABLE.encode(TESTFN_ENCODING) - except UnicodeEncodeError: - pass - else: - print('WARNING: The filename %r CAN be encoded by the filesystem encoding (%s). ' - 'Unicode filename tests may not be effective' - % (TESTFN_UNENCODABLE, TESTFN_ENCODING)) - TESTFN_UNENCODABLE = None -# Mac OS X denies unencodable filenames (invalid utf-8) -elif sys.platform != 'darwin': - try: - # ascii and utf-8 cannot encode the byte 0xff - b'\xff'.decode(TESTFN_ENCODING) - except UnicodeDecodeError: - # 0xff will be encoded using the surrogate character u+DCFF - TESTFN_UNENCODABLE = TESTFN \ - + b'-\xff'.decode(TESTFN_ENCODING, 'surrogateescape') - else: - # File system encoding (eg. ISO-8859-* encodings) can encode - # the byte 0xff. Skip some unicode filename tests. - pass - -# Save the initial cwd -SAVEDCWD = os.getcwd() - -@contextlib.contextmanager -def temp_cwd(name='tempcwd', quiet=False, path=None): - """ - Context manager that temporarily changes the CWD. - - An existing path may be provided as *path*, in which case this - function makes no changes to the file system. - - Otherwise, the new CWD is created in the current directory and it's - named *name*. If *quiet* is False (default) and it's not possible to - create or change the CWD, an error is raised. If it's True, only a - warning is raised and the original CWD is used. - """ - saved_dir = os.getcwd() - is_temporary = False - if path is None: - path = name - try: - os.mkdir(name) - is_temporary = True - except OSError: - if not quiet: - raise - warnings.warn('tests may fail, unable to create temp CWD ' + name, - RuntimeWarning, stacklevel=3) - try: - os.chdir(path) - except OSError: - if not quiet: - raise - warnings.warn('tests may fail, unable to change the CWD to ' + name, - RuntimeWarning, stacklevel=3) - try: - yield os.getcwd() - finally: - os.chdir(saved_dir) - if is_temporary: - rmtree(name) - - -@contextlib.contextmanager -def temp_umask(umask): - """Context manager that temporarily sets the process umask.""" - oldmask = os.umask(umask) - try: - yield None - finally: - os.umask(oldmask) - - -def findfile(file, here=__file__, subdir=None): - """Try to find a file on sys.path and the working directory. If it is not - found the argument passed to the function is returned (this does not - necessarily signal failure; could still be the legitimate path).""" - if os.path.isabs(file): - return file - if subdir is not None: - file = os.path.join(subdir, file) - path = sys.path - path = [os.path.dirname(here)] + path - for dn in path: - fn = os.path.join(dn, file) - if os.path.exists(fn): return fn - return file - -def sortdict(dict): - "Like repr(dict), but in sorted order." - items = sorted(dict.items()) - reprpairs = ["%r: %r" % pair for pair in items] - withcommas = ", ".join(reprpairs) - return "{%s}" % withcommas - -def make_bad_fd(): - """ - Create an invalid file descriptor by opening and closing a file and return - its fd. - """ - file = open(TESTFN, "wb") - try: - return file.fileno() - finally: - file.close() - unlink(TESTFN) - -def check_syntax_error(testcase, statement): - raise NotImplementedError('no compile built-in') - #testcase.assertRaises(SyntaxError, compile, statement, - # '', 'exec') - -def open_urlresource(url, *args, **kw): - from urllib import request, parse - - check = kw.pop('check', None) - - filename = parse.urlparse(url)[2].split('/')[-1] # '/': it's URL! - - fn = os.path.join(os.path.dirname(__file__), "data", filename) - - def check_valid_file(fn): - f = open(fn, *args, **kw) - if check is None: - return f - elif check(f): - f.seek(0) - return f - f.close() - - if os.path.exists(fn): - f = check_valid_file(fn) - if f is not None: - return f - unlink(fn) - - # Verify the requirement before downloading the file - requires('urlfetch') - - print('\tfetching %s ...' % url, file=get_original_stdout()) - f = request.urlopen(url, timeout=15) - try: - with open(fn, "wb") as out: - s = f.read() - while s: - out.write(s) - s = f.read() - finally: - f.close() - - f = check_valid_file(fn) - if f is not None: - return f - raise TestFailed('invalid resource "%s"' % fn) - - -class WarningsRecorder(object): - """Convenience wrapper for the warnings list returned on - entry to the warnings.catch_warnings() context manager. - """ - def __init__(self, warnings_list): - self._warnings = warnings_list - self._last = 0 - - def __getattr__(self, attr): - if len(self._warnings) > self._last: - return getattr(self._warnings[-1], attr) - elif attr in warnings.WarningMessage._WARNING_DETAILS: - return None - raise AttributeError("%r has no attribute %r" % (self, attr)) - - #@property - #def warnings(self): - # return self._warnings[self._last:] - - def reset(self): - self._last = len(self._warnings) - - -def _filterwarnings(filters, quiet=False): - """Catch the warnings, then check if all the expected - warnings have been raised and re-raise unexpected warnings. - If 'quiet' is True, only re-raise the unexpected warnings. - """ - # Clear the warning registry of the calling module - # in order to re-raise the warnings. - frame = sys._getframe(2) - registry = frame.f_globals.get('__warningregistry__') - if registry: - registry.clear() - with warnings.catch_warnings(record=True) as w: - # Set filter "always" to record all warnings. Because - # test_warnings swap the module, we need to look up in - # the sys.modules dictionary. - sys.modules['warnings'].simplefilter("always") - yield WarningsRecorder(w) - # Filter the recorded warnings - reraise = list(w) - missing = [] - for msg, cat in filters: - seen = False - for w in reraise[:]: - warning = w.message - # Filter out the matching messages - if (re.match(msg, str(warning), re.I) and - issubclass(warning.__class__, cat)): - seen = True - reraise.remove(w) - if not seen and not quiet: - # This filter caught nothing - missing.append((msg, cat.__name__)) - if reraise: - raise AssertionError("unhandled warning %s" % reraise[0]) - if missing: - raise AssertionError("filter (%r, %s) did not catch any warning" % - missing[0]) - - -@contextlib.contextmanager -def check_warnings(*filters, **kwargs): - """Context manager to silence warnings. - - Accept 2-tuples as positional arguments: - ("message regexp", WarningCategory) - - Optional argument: - - if 'quiet' is True, it does not fail if a filter catches nothing - (default True without argument, - default False if some filters are defined) - - Without argument, it defaults to: - check_warnings(("", Warning), quiet=True) - """ - quiet = kwargs.get('quiet') - if not filters: - filters = (("", Warning),) - # Preserve backward compatibility - if quiet is None: - quiet = True - return _filterwarnings(filters, quiet) - - -class CleanImport(object): - """Context manager to force import to return a new module reference. - - This is useful for testing module-level behaviours, such as - the emission of a DeprecationWarning on import. - - Use like this: - - with CleanImport("foo"): - importlib.import_module("foo") # new reference - """ - - def __init__(self, *module_names): - self.original_modules = sys.modules.copy() - for module_name in module_names: - if module_name in sys.modules: - module = sys.modules[module_name] - # It is possible that module_name is just an alias for - # another module (e.g. stub for modules renamed in 3.x). - # In that case, we also need delete the real module to clear - # the import cache. - if module.__name__ != module_name: - del sys.modules[module.__name__] - del sys.modules[module_name] - - def __enter__(self): - return self - - def __exit__(self, *ignore_exc): - sys.modules.update(self.original_modules) - - -class EnvironmentVarGuard(dict): - - """Class to help protect the environment variable properly. Can be used as - a context manager.""" - - def __init__(self): - self._environ = os.environ - self._changed = {} - - def __getitem__(self, envvar): - return self._environ[envvar] - - def __setitem__(self, envvar, value): - # Remember the initial value on the first access - if envvar not in self._changed: - self._changed[envvar] = self._environ.get(envvar) - self._environ[envvar] = value - - def __delitem__(self, envvar): - # Remember the initial value on the first access - if envvar not in self._changed: - self._changed[envvar] = self._environ.get(envvar) - if envvar in self._environ: - del self._environ[envvar] - - def keys(self): - return self._environ.keys() - - def __iter__(self): - return iter(self._environ) - - def __len__(self): - return len(self._environ) - - def set(self, envvar, value): - self[envvar] = value - - def unset(self, envvar): - del self[envvar] - - def __enter__(self): - return self - - def __exit__(self, *ignore_exc): - for k, v in self._changed.items(): - if v is None: - if k in self._environ: - del self._environ[k] - else: - self._environ[k] = v - os.environ = self._environ - - -class DirsOnSysPath(object): - """Context manager to temporarily add directories to sys.path. - - This makes a copy of sys.path, appends any directories given - as positional arguments, then reverts sys.path to the copied - settings when the context ends. - - Note that *all* sys.path modifications in the body of the - context manager, including replacement of the object, - will be reverted at the end of the block. - """ - - def __init__(self, *paths): - self.original_value = sys.path[:] - self.original_object = sys.path - sys.path.extend(paths) - - def __enter__(self): - return self - - def __exit__(self, *ignore_exc): - sys.path = self.original_object - sys.path[:] = self.original_value - - -class TransientResource(object): - - """Raise ResourceDenied if an exception is raised while the context manager - is in effect that matches the specified exception and attributes.""" - - def __init__(self, exc, **kwargs): - self.exc = exc - self.attrs = kwargs - - def __enter__(self): - return self - - def __exit__(self, type_=None, value=None, traceback=None): - """If type_ is a subclass of self.exc and value has attributes matching - self.attrs, raise ResourceDenied. Otherwise let the exception - propagate (if any).""" - if type_ is not None and issubclass(self.exc, type_): - for attr, attr_value in self.attrs.items(): - if not hasattr(value, attr): - break - if getattr(value, attr) != attr_value: - break - else: - raise ResourceDenied("an optional resource is not available") - -# Context managers that raise ResourceDenied when various issues -# with the Internet connection manifest themselves as exceptions. -# XXX deprecate these and use transient_internet() instead -time_out = TransientResource(IOError, errno=errno.ETIMEDOUT) -socket_peer_reset = TransientResource(socket.error, errno=errno.ECONNRESET) -ioerror_peer_reset = TransientResource(IOError, errno=errno.ECONNRESET) - - -@contextlib.contextmanager -def transient_internet(resource_name, *, timeout=30.0, errnos=()): - """Return a context manager that raises ResourceDenied when various issues - with the Internet connection manifest themselves as exceptions.""" - default_errnos = [ - ('ECONNREFUSED', 111), - ('ECONNRESET', 104), - ('EHOSTUNREACH', 113), - ('ENETUNREACH', 101), - ('ETIMEDOUT', 110), - ] - default_gai_errnos = [ - ('EAI_AGAIN', -3), - ('EAI_FAIL', -4), - ('EAI_NONAME', -2), - ('EAI_NODATA', -5), - # Encountered when trying to resolve IPv6-only hostnames - ('WSANO_DATA', 11004), - ] - - denied = ResourceDenied("Resource '%s' is not available" % resource_name) - captured_errnos = errnos - gai_errnos = [] - if not captured_errnos: - captured_errnos = [getattr(errno, name, num) - for name, num in default_errnos] - gai_errnos = [getattr(socket, name, num) - for name, num in default_gai_errnos] - - def filter_error(err): - n = getattr(err, 'errno', None) - if (isinstance(err, socket.timeout) or - (isinstance(err, socket.gaierror) and n in gai_errnos) or - n in captured_errnos): - if not verbose: - sys.stderr.write(denied.args[0] + "\n") - raise denied from err - - old_timeout = socket.getdefaulttimeout() - try: - if timeout is not None: - socket.setdefaulttimeout(timeout) - yield None - except IOError as err: - # urllib can wrap original socket errors multiple times (!), we must - # unwrap to get at the original error. - while True: - a = err.args - if len(a) >= 1 and isinstance(a[0], IOError): - err = a[0] - # The error can also be wrapped as args[1]: - # except socket.error as msg: - # raise IOError('socket error', msg).with_traceback(sys.exc_info()[2]) - elif len(a) >= 2 and isinstance(a[1], IOError): - err = a[1] - else: - break - filter_error(err) - raise - # XXX should we catch generic exceptions and look for their - # __cause__ or __context__? - finally: - socket.setdefaulttimeout(old_timeout) - - -@contextlib.contextmanager -def captured_output(stream_name): - """Return a context manager used by captured_stdout/stdin/stderr - that temporarily replaces the sys stream *stream_name* with a StringIO.""" - import io - orig_stdout = getattr(sys, stream_name) - setattr(sys, stream_name, io.StringIO()) - try: - yield getattr(sys, stream_name) - finally: - setattr(sys, stream_name, orig_stdout) - -def captured_stdout(): - """Capture the output of sys.stdout: - - with captured_stdout() as s: - print("hello") - self.assertEqual(s.getvalue(), "hello") - """ - return captured_output("stdout") - -def captured_stderr(): - return captured_output("stderr") - -def captured_stdin(): - return captured_output("stdin") - - -def gc_collect(): - """Force as many objects as possible to be collected. - - In non-CPython implementations of Python, this is needed because timely - deallocation is not guaranteed by the garbage collector. (Even in CPython - this can be the case in case of reference cycles.) This means that __del__ - methods may be called later than expected and weakrefs may remain alive for - longer than expected. This function tries its best to force all garbage - objects to disappear. - """ - gc.collect() - if is_jython: - time.sleep(0.1) - gc.collect() - gc.collect() - - -def python_is_optimized(): - """Find if Python was built with optimizations.""" - cflags = sysconfig.get_config_var('PY_CFLAGS') or '' - final_opt = "" - for opt in cflags.split(): - if opt.startswith('-O'): - final_opt = opt - return final_opt and final_opt != '-O0' - - -#======================================================================= -# Decorator for running a function in a different locale, correctly resetting -# it afterwards. - -def run_with_locale(catstr, *locales): - def decorator(func): - def inner(*args, **kwds): - try: - import locale - category = getattr(locale, catstr) - orig_locale = locale.setlocale(category) - except AttributeError: - # if the test author gives us an invalid category string - raise - except: - # cannot retrieve original locale, so do nothing - locale = orig_locale = None - else: - for loc in locales: - try: - locale.setlocale(category, loc) - break - except: - pass - - # now run the function, resetting the locale on exceptions - try: - return func(*args, **kwds) - finally: - if locale and orig_locale: - locale.setlocale(category, orig_locale) - inner.__name__ = func.__name__ - inner.__doc__ = func.__doc__ - return inner - return decorator - -#======================================================================= -# Big-memory-test support. Separate from 'resources' because memory use -# should be configurable. - -# Some handy shorthands. Note that these are used for byte-limits as well -# as size-limits, in the various bigmem tests -_1M = 1024*1024 -_1G = 1024 * _1M -_2G = 2 * _1G -_4G = 4 * _1G - -MAX_Py_ssize_t = sys.maxsize - -def set_memlimit(limit): - global max_memuse - global real_max_memuse - sizes = { - 'k': 1024, - 'm': _1M, - 'g': _1G, - 't': 1024*_1G, - } - m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit, - re.IGNORECASE | re.VERBOSE) - if m is None: - raise ValueError('Invalid memory limit %r' % (limit,)) - memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) - real_max_memuse = memlimit - if memlimit > MAX_Py_ssize_t: - memlimit = MAX_Py_ssize_t - if memlimit < _2G - 1: - raise ValueError('Memory limit %r too low to be useful' % (limit,)) - max_memuse = memlimit - -def _memory_watchdog(start_evt, finish_evt, period=10.0): - """A function which periodically watches the process' memory consumption - and prints it out. - """ - # XXX: because of the GIL, and because the very long operations tested - # in most bigmem tests are uninterruptible, the loop below gets woken up - # much less often than expected. - # The polling code should be rewritten in raw C, without holding the GIL, - # and push results onto an anonymous pipe. - try: - page_size = os.sysconf('SC_PAGESIZE') - except (ValueError, AttributeError): - try: - page_size = os.sysconf('SC_PAGE_SIZE') - except (ValueError, AttributeError): - page_size = 4096 - procfile = '/proc/{pid}/statm'.format(pid=os.getpid()) - try: - f = open(procfile, 'rb') - except IOError as e: - warnings.warn('/proc not available for stats: {}'.format(e), - RuntimeWarning) - sys.stderr.flush() - return - with f: - start_evt.set() - old_data = -1 - while not finish_evt.wait(period): - f.seek(0) - statm = f.read().decode('ascii') - data = int(statm.split()[5]) - if data != old_data: - old_data = data - print(" ... process data size: {data:.1f}G" - .format(data=data * page_size / (1024 ** 3))) - -def bigmemtest(size, memuse, dry_run=True): - """Decorator for bigmem tests. - - 'minsize' is the minimum useful size for the test (in arbitrary, - test-interpreted units.) 'memuse' is the number of 'bytes per size' for - the test, or a good estimate of it. - - if 'dry_run' is False, it means the test doesn't support dummy runs - when -M is not specified. - """ - def decorator(f): - def wrapper(self): - size = wrapper.size - memuse = wrapper.memuse - if not real_max_memuse: - maxsize = 5147 - else: - maxsize = size - - if ((real_max_memuse or not dry_run) - and real_max_memuse < maxsize * memuse): - raise unittest.SkipTest( - "not enough memory: %.1fG minimum needed" - % (size * memuse / (1024 ** 3))) - - if real_max_memuse and verbose and threading: - print() - print(" ... expected peak memory use: {peak:.1f}G" - .format(peak=size * memuse / (1024 ** 3))) - sys.stdout.flush() - start_evt = threading.Event() - finish_evt = threading.Event() - t = threading.Thread(target=_memory_watchdog, - args=(start_evt, finish_evt, 0.5)) - t.daemon = True - t.start() - start_evt.set() - else: - t = None - - try: - return f(self, maxsize) - finally: - if t: - finish_evt.set() - t.join() - - wrapper.size = size - wrapper.memuse = memuse - return wrapper - return decorator - -def bigaddrspacetest(f): - """Decorator for tests that fill the address space.""" - def wrapper(self): - if max_memuse < MAX_Py_ssize_t: - if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31: - raise unittest.SkipTest( - "not enough memory: try a 32-bit build instead") - else: - raise unittest.SkipTest( - "not enough memory: %.1fG minimum needed" - % (MAX_Py_ssize_t / (1024 ** 3))) - else: - return f(self) - return wrapper - -#======================================================================= -# unittest integration. - -class BasicTestRunner: - def run(self, test): - result = unittest.TestResult() - test(result) - return result - -def _id(obj): - return obj - -def requires_resource(resource): - if resource == 'gui' and not _is_gui_available(): - return unittest.skip("resource 'gui' is not available") - if is_resource_enabled(resource): - return _id - else: - return unittest.skip("resource {0!r} is not enabled".format(resource)) - -def cpython_only(test): - """ - Decorator for tests only applicable on CPython. - """ - return impl_detail(cpython=True)(test) - -def impl_detail(msg=None, **guards): - if check_impl_detail(**guards): - return _id - if msg is None: - guardnames, default = _parse_guards(guards) - if default: - msg = "implementation detail not available on {0}" - else: - msg = "implementation detail specific to {0}" - guardnames = sorted(guardnames.keys()) - msg = msg.format(' or '.join(guardnames)) - return unittest.skip(msg) - -def _parse_guards(guards): - # Returns a tuple ({platform_name: run_me}, default_value) - if not guards: - return ({'cpython': True}, False) - is_true = list(guards.values())[0] - assert list(guards.values()) == [is_true] * len(guards) # all True or all False - return (guards, not is_true) - -# Use the following check to guard CPython's implementation-specific tests -- -# or to run them only on the implementation(s) guarded by the arguments. -def check_impl_detail(**guards): - """This function returns True or False depending on the host platform. - Examples: - if check_impl_detail(): # only on CPython (default) - if check_impl_detail(jython=True): # only on Jython - if check_impl_detail(cpython=False): # everywhere except on CPython - """ - guards, default = _parse_guards(guards) - return guards.get(platform.python_implementation().lower(), default) - - -def _filter_suite(suite, pred): - """Recursively filter test cases in a suite based on a predicate.""" - newtests = [] - for test in suite._tests: - if isinstance(test, unittest.TestSuite): - _filter_suite(test, pred) - newtests.append(test) - else: - if pred(test): - newtests.append(test) - suite._tests = newtests - - -def _run_suite(suite): - """Run tests from a unittest.TestSuite-derived class.""" - if verbose: - runner = unittest.TextTestRunner(sys.stdout, verbosity=2, - failfast=failfast) - else: - runner = BasicTestRunner() - - result = runner.run(suite) - if not result.wasSuccessful(): - if len(result.errors) == 1 and not result.failures: - err = result.errors[0][1] - elif len(result.failures) == 1 and not result.errors: - err = result.failures[0][1] - else: - err = "multiple errors occurred" - if not verbose: err += "; run in verbose mode for details" - raise TestFailed(err) - - -def run_unittest(*classes): - """Run tests from unittest.TestCase-derived classes.""" - valid_types = (unittest.TestSuite, unittest.TestCase) - suite = unittest.TestSuite() - for cls in classes: - if isinstance(cls, str): - if cls in sys.modules: - suite.addTest(unittest.findTestCases(sys.modules[cls])) - else: - raise ValueError("str arguments must be keys in sys.modules") - elif isinstance(cls, valid_types): - suite.addTest(cls) - else: - suite.addTest(unittest.makeSuite(cls)) - def case_pred(test): - if match_tests is None: - return True - for name in test.id().split("."): - if fnmatch.fnmatchcase(name, match_tests): - return True - return False - _filter_suite(suite, case_pred) - _run_suite(suite) - - -#======================================================================= -# doctest driver. - -def run_doctest(module, verbosity=None): - """Run doctest on the given module. Return (#failures, #tests). - - If optional argument verbosity is not specified (or is None), pass - support's belief about verbosity on to doctest. Else doctest's - usual behavior is used (it searches sys.argv for -v). - """ - - import doctest - - if verbosity is None: - verbosity = verbose - else: - verbosity = None - - f, t = doctest.testmod(module, verbose=verbosity) - if f: - raise TestFailed("%d of %d doctests failed" % (f, t)) - if verbose: - print('doctest (%s) ... %d tests with zero failures' % - (module.__name__, t)) - return f, t - - -#======================================================================= -# Support for saving and restoring the imported modules. - -def modules_setup(): - return sys.modules.copy(), - -def modules_cleanup(oldmodules): - # Encoders/decoders are registered permanently within the internal - # codec cache. If we destroy the corresponding modules their - # globals will be set to None which will trip up the cached functions. - encodings = [(k, v) for k, v in sys.modules.items() - if k.startswith('encodings.')] - sys.modules.clear() - sys.modules.update(encodings) - # XXX: This kind of problem can affect more than just encodings. In particular - # extension modules (such as _ssl) don't cope with reloading properly. - # Really, test modules should be cleaning out the test specific modules they - # know they added (ala test_runpy) rather than relying on this function (as - # test_importhooks and test_pkg do currently). - # Implicitly imported *real* modules should be left alone (see issue 10556). - sys.modules.update(oldmodules) - -#======================================================================= -# Threading support to prevent reporting refleaks when running regrtest.py -R - -# NOTE: we use thread._count() rather than threading.enumerate() (or the -# moral equivalent thereof) because a threading.Thread object is still alive -# until its __bootstrap() method has returned, even after it has been -# unregistered from the threading module. -# thread._count(), on the other hand, only gets decremented *after* the -# __bootstrap() method has returned, which gives us reliable reference counts -# at the end of a test run. - -def threading_setup(): - if _thread: - return _thread._count(), threading._dangling.copy() - else: - return 1, () - -def threading_cleanup(*original_values): - if not _thread: - return - _MAX_COUNT = 10 - for count in range(_MAX_COUNT): - values = _thread._count(), threading._dangling - if values == original_values: - break - time.sleep(0.1) - gc_collect() - # XXX print a warning in case of failure? - -def reap_threads(func): - """Use this function when threads are being used. This will - ensure that the threads are cleaned up even when the test fails. - If threading is unavailable this function does nothing. - """ - if not _thread: - return func - - @functools.wraps(func) - def decorator(*args): - key = threading_setup() - try: - return func(*args) - finally: - threading_cleanup(*key) - return decorator - -def reap_children(): - """Use this function at the end of test_main() whenever sub-processes - are started. This will help ensure that no extra children (zombies) - stick around to hog resources and create problems when looking - for refleaks. - """ - - # Reap all our dead child processes so we don't leave zombies around. - # These hog resources and might be causing some of the buildbots to die. - if hasattr(os, 'waitpid'): - any_process = -1 - while True: - try: - # This will raise an exception on Windows. That's ok. - pid, status = os.waitpid(any_process, os.WNOHANG) - if pid == 0: - break - except: - break - -@contextlib.contextmanager -def swap_attr(obj, attr, new_val): - """Temporary swap out an attribute with a new object. - - Usage: - with swap_attr(obj, "attr", 5): - ... - - This will set obj.attr to 5 for the duration of the with: block, - restoring the old value at the end of the block. If `attr` doesn't - exist on `obj`, it will be created and then deleted at the end of the - block. - """ - if hasattr(obj, attr): - real_val = getattr(obj, attr) - setattr(obj, attr, new_val) - try: - yield None - finally: - setattr(obj, attr, real_val) - else: - setattr(obj, attr, new_val) - try: - yield None - finally: - delattr(obj, attr) - -@contextlib.contextmanager -def swap_item(obj, item, new_val): - """Temporary swap out an item with a new object. - - Usage: - with swap_item(obj, "item", 5): - ... - - This will set obj["item"] to 5 for the duration of the with: block, - restoring the old value at the end of the block. If `item` doesn't - exist on `obj`, it will be created and then deleted at the end of the - block. - """ - if item in obj: - real_val = obj[item] - obj[item] = new_val - try: - yield None - finally: - obj[item] = real_val - else: - obj[item] = new_val - try: - yield None - finally: - del obj[item] - -def strip_python_stderr(stderr): - """Strip the stderr of a Python process from potential debug output - emitted by the interpreter. - - This will typically be run on the result of the communicate() method - of a subprocess.Popen object. - """ - stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip() - return stderr - -def args_from_interpreter_flags(): - """Return a list of command-line arguments reproducing the current - settings in sys.flags.""" - flag_opt_map = { - 'bytes_warning': 'b', - 'dont_write_bytecode': 'B', - 'hash_randomization': 'R', - 'ignore_environment': 'E', - 'no_user_site': 's', - 'no_site': 'S', - 'optimize': 'O', - 'verbose': 'v', - } - args = [] - for flag, opt in flag_opt_map.items(): - v = getattr(sys.flags, flag) - if v > 0: - args.append('-' + opt * v) - return args - -#============================================================ -# Support for assertions about logging. -#============================================================ - -class TestHandler(logging.handlers.BufferingHandler): - def __init__(self, matcher): - # BufferingHandler takes a "capacity" argument - # so as to know when to flush. As we're overriding - # shouldFlush anyway, we can set a capacity of zero. - # You can call flush() manually to clear out the - # buffer. - logging.handlers.BufferingHandler.__init__(self, 0) - self.matcher = matcher - - def shouldFlush(self, record): - return False - - def emit(self, record): - self.format(record) - self.buffer.append(record.__dict__) - - def matches(self, **kwargs): - """ - Look for a saved dict whose keys/values match the supplied arguments. - """ - result = False - for d in self.buffer: - if self.matcher.matches(d, **kwargs): - result = True - break - return result - -class Matcher(object): - - _partial_matches = ('msg', 'message') - - def matches(self, d, **kwargs): - """ - Try to match a single dict with the supplied arguments. - - Keys whose values are strings and which are in self._partial_matches - will be checked for partial (i.e. substring) matches. You can extend - this scheme to (for example) do regular expression matching, etc. - """ - result = True - for k in kwargs: - v = kwargs[k] - dv = d.get(k) - if not self.match_value(k, dv, v): - result = False - break - return result - - def match_value(self, k, dv, v): - """ - Try to match a single stored value (dv) with a supplied value (v). - """ - if type(v) != type(dv): - result = False - elif type(dv) is not str or k not in self._partial_matches: - result = (v == dv) - else: - result = dv.find(v) >= 0 - return result - - -_can_symlink = None # type: Any -def can_symlink(): - global _can_symlink - if _can_symlink is not None: - return _can_symlink - symlink_path = TESTFN + "can_symlink" - try: - os.symlink(TESTFN, symlink_path) - can = True - except (OSError, NotImplementedError, AttributeError): - can = False - else: - os.remove(symlink_path) - _can_symlink = can - return can - -def skip_unless_symlink(test): - """Skip decorator for tests that require functional symlink""" - ok = can_symlink() - msg = "Requires functional symlink implementation" - if ok: - return test - else: - return unittest.skip(msg)(test) - -def patch(test_instance, object_to_patch, attr_name, new_value): - """Override 'object_to_patch'.'attr_name' with 'new_value'. - - Also, add a cleanup procedure to 'test_instance' to restore - 'object_to_patch' value for 'attr_name'. - The 'attr_name' should be a valid attribute for 'object_to_patch'. - - """ - # check that 'attr_name' is a real attribute for 'object_to_patch' - # will raise AttributeError if it does not exist - getattr(object_to_patch, attr_name) - - # keep a copy of the old value - attr_is_local = False - try: - old_value = object_to_patch.__dict__[attr_name] - except (AttributeError, KeyError): - old_value = getattr(object_to_patch, attr_name, None) - else: - attr_is_local = True - - # restore the value when the test is done - def cleanup(): - if attr_is_local: - setattr(object_to_patch, attr_name, old_value) - else: - delattr(object_to_patch, attr_name) - - test_instance.addCleanup(cleanup) - - # actually override the attribute - setattr(object_to_patch, attr_name, new_value) diff --git a/lib-python/3.2/test/test_base64.py b/lib-python/3.2/test/test_base64.py deleted file mode 100644 index 0ddd8b1a1ac6..000000000000 --- a/lib-python/3.2/test/test_base64.py +++ /dev/null @@ -1,267 +0,0 @@ -import unittest -from test import support -import base64 -import binascii -import sys -import subprocess - -from typing import Any - - - -class LegacyBase64TestCase(unittest.TestCase): - def test_encodebytes(self) -> None: - eq = self.assertEqual - eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") - eq(base64.encodebytes(b"a"), b"YQ==\n") - eq(base64.encodebytes(b"ab"), b"YWI=\n") - eq(base64.encodebytes(b"abc"), b"YWJj\n") - eq(base64.encodebytes(b""), b"") - eq(base64.encodebytes(b"abcdefghijklmnopqrstuvwxyz" - b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"0123456789!@#0^&*();:<>,. []{}"), - b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" - b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT" - b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n") - self.assertRaises(TypeError, base64.encodebytes, "") - - def test_decodebytes(self) -> None: - eq = self.assertEqual - eq(base64.decodebytes(b"d3d3LnB5dGhvbi5vcmc=\n"), b"www.python.org") - eq(base64.decodebytes(b"YQ==\n"), b"a") - eq(base64.decodebytes(b"YWI=\n"), b"ab") - eq(base64.decodebytes(b"YWJj\n"), b"abc") - eq(base64.decodebytes(b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" - b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT" - b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n"), - b"abcdefghijklmnopqrstuvwxyz" - b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"0123456789!@#0^&*();:<>,. []{}") - eq(base64.decodebytes(b''), b'') - self.assertRaises(TypeError, base64.decodebytes, "") - - def test_encode(self) -> None: - eq = self.assertEqual - from io import BytesIO - infp = BytesIO(b'abcdefghijklmnopqrstuvwxyz' - b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - b'0123456789!@#0^&*();:<>,. []{}') - outfp = BytesIO() - base64.encode(infp, outfp) - eq(outfp.getvalue(), - b'YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE' - b'RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT' - b'Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n') - - def test_decode(self) -> None: - from io import BytesIO - infp = BytesIO(b'd3d3LnB5dGhvbi5vcmc=') - outfp = BytesIO() - base64.decode(infp, outfp) - self.assertEqual(outfp.getvalue(), b'www.python.org') - - -class BaseXYTestCase(unittest.TestCase): - def test_b64encode(self) -> None: - eq = self.assertEqual - # Test default alphabet - eq(base64.b64encode(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=") - eq(base64.b64encode(b'\x00'), b'AA==') - eq(base64.b64encode(b"a"), b"YQ==") - eq(base64.b64encode(b"ab"), b"YWI=") - eq(base64.b64encode(b"abc"), b"YWJj") - eq(base64.b64encode(b""), b"") - eq(base64.b64encode(b"abcdefghijklmnopqrstuvwxyz" - b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"0123456789!@#0^&*();:<>,. []{}"), - b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" - b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" - b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") - # Test with arbitrary alternative characters - eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=b'*$'), b'01a*b$cd') - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.b64encode, "") - self.assertRaises(TypeError, base64.b64encode, b"", altchars="") - # Test standard alphabet - eq(base64.standard_b64encode(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=") - eq(base64.standard_b64encode(b"a"), b"YQ==") - eq(base64.standard_b64encode(b"ab"), b"YWI=") - eq(base64.standard_b64encode(b"abc"), b"YWJj") - eq(base64.standard_b64encode(b""), b"") - eq(base64.standard_b64encode(b"abcdefghijklmnopqrstuvwxyz" - b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"0123456789!@#0^&*();:<>,. []{}"), - b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" - b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" - b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.standard_b64encode, "") - self.assertRaises(TypeError, base64.standard_b64encode, b"", altchars="") - # Test with 'URL safe' alternative characters - eq(base64.urlsafe_b64encode(b'\xd3V\xbeo\xf7\x1d'), b'01a-b_cd') - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.urlsafe_b64encode, "") - - def test_b64decode(self) -> None: - eq = self.assertEqual - eq(base64.b64decode(b"d3d3LnB5dGhvbi5vcmc="), b"www.python.org") - eq(base64.b64decode(b'AA=='), b'\x00') - eq(base64.b64decode(b"YQ=="), b"a") - eq(base64.b64decode(b"YWI="), b"ab") - eq(base64.b64decode(b"YWJj"), b"abc") - eq(base64.b64decode(b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" - b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT" - b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ=="), - b"abcdefghijklmnopqrstuvwxyz" - b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"0123456789!@#0^&*();:<>,. []{}") - eq(base64.b64decode(b''), b'') - # Test with arbitrary alternative characters - eq(base64.b64decode(b'01a*b$cd', altchars=b'*$'), b'\xd3V\xbeo\xf7\x1d') - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.b64decode, "") - self.assertRaises(TypeError, base64.b64decode, b"", altchars="") - # Test standard alphabet - eq(base64.standard_b64decode(b"d3d3LnB5dGhvbi5vcmc="), b"www.python.org") - eq(base64.standard_b64decode(b"YQ=="), b"a") - eq(base64.standard_b64decode(b"YWI="), b"ab") - eq(base64.standard_b64decode(b"YWJj"), b"abc") - eq(base64.standard_b64decode(b""), b"") - eq(base64.standard_b64decode(b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" - b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" - b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ=="), - b"abcdefghijklmnopqrstuvwxyz" - b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"0123456789!@#0^&*();:<>,. []{}") - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.standard_b64decode, "") - self.assertRaises(TypeError, base64.standard_b64decode, b"", altchars="") - # Test with 'URL safe' alternative characters - eq(base64.urlsafe_b64decode(b'01a-b_cd'), b'\xd3V\xbeo\xf7\x1d') - self.assertRaises(TypeError, base64.urlsafe_b64decode, "") - - def test_b64decode_padding_error(self) -> None: - self.assertRaises(binascii.Error, base64.b64decode, b'abc') - - def test_b64decode_invalid_chars(self) -> None: - # issue 1466065: Test some invalid characters. - tests = ((b'%3d==', b'\xdd'), - (b'$3d==', b'\xdd'), - (b'[==', b''), - (b'YW]3=', b'am'), - (b'3{d==', b'\xdd'), - (b'3d}==', b'\xdd'), - (b'@@', b''), - (b'!', b''), - (b'YWJj\nYWI=', b'abcab')) - for bstr, res in tests: - self.assertEqual(base64.b64decode(bstr), res) - with self.assertRaises(binascii.Error): - base64.b64decode(bstr, validate=True) - - def test_b32encode(self) -> None: - eq = self.assertEqual - eq(base64.b32encode(b''), b'') - eq(base64.b32encode(b'\x00'), b'AA======') - eq(base64.b32encode(b'a'), b'ME======') - eq(base64.b32encode(b'ab'), b'MFRA====') - eq(base64.b32encode(b'abc'), b'MFRGG===') - eq(base64.b32encode(b'abcd'), b'MFRGGZA=') - eq(base64.b32encode(b'abcde'), b'MFRGGZDF') - self.assertRaises(TypeError, base64.b32encode, "") - - def test_b32decode(self) -> None: - eq = self.assertEqual - eq(base64.b32decode(b''), b'') - eq(base64.b32decode(b'AA======'), b'\x00') - eq(base64.b32decode(b'ME======'), b'a') - eq(base64.b32decode(b'MFRA===='), b'ab') - eq(base64.b32decode(b'MFRGG==='), b'abc') - eq(base64.b32decode(b'MFRGGZA='), b'abcd') - eq(base64.b32decode(b'MFRGGZDF'), b'abcde') - self.assertRaises(TypeError, base64.b32decode, "") - - def test_b32decode_casefold(self) -> None: - eq = self.assertEqual - eq(base64.b32decode(b'', True), b'') - eq(base64.b32decode(b'ME======', True), b'a') - eq(base64.b32decode(b'MFRA====', True), b'ab') - eq(base64.b32decode(b'MFRGG===', True), b'abc') - eq(base64.b32decode(b'MFRGGZA=', True), b'abcd') - eq(base64.b32decode(b'MFRGGZDF', True), b'abcde') - # Lower cases - eq(base64.b32decode(b'me======', True), b'a') - eq(base64.b32decode(b'mfra====', True), b'ab') - eq(base64.b32decode(b'mfrgg===', True), b'abc') - eq(base64.b32decode(b'mfrggza=', True), b'abcd') - eq(base64.b32decode(b'mfrggzdf', True), b'abcde') - # Expected exceptions - self.assertRaises(TypeError, base64.b32decode, b'me======') - # Mapping zero and one - eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode(b'M1023456', map01=b'L'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode(b'M1023456', map01=b'I'), b'b\x1d\xad\xf3\xbe') - self.assertRaises(TypeError, base64.b32decode, b"", map01="") - - def test_b32decode_error(self) -> None: - self.assertRaises(binascii.Error, base64.b32decode, b'abc') - self.assertRaises(binascii.Error, base64.b32decode, b'ABCDEF==') - - def test_b16encode(self) -> None: - eq = self.assertEqual - eq(base64.b16encode(b'\x01\x02\xab\xcd\xef'), b'0102ABCDEF') - eq(base64.b16encode(b'\x00'), b'00') - self.assertRaises(TypeError, base64.b16encode, "") - - def test_b16decode(self) -> None: - eq = self.assertEqual - eq(base64.b16decode(b'0102ABCDEF'), b'\x01\x02\xab\xcd\xef') - eq(base64.b16decode(b'00'), b'\x00') - # Lower case is not allowed without a flag - self.assertRaises(binascii.Error, base64.b16decode, b'0102abcdef') - # Case fold - eq(base64.b16decode(b'0102abcdef', True), b'\x01\x02\xab\xcd\xef') - self.assertRaises(TypeError, base64.b16decode, "") - - def test_ErrorHeritage(self) -> None: - self.assertTrue(issubclass(binascii.Error, ValueError)) - - - -class TestMain(unittest.TestCase): - def get_output(self, *args: str, **options: Any) -> Any: - args = [sys.executable, '-m', 'base64'] + list(args) - return subprocess.check_output(args, **options) - - def test_encode_decode(self) -> None: - output = self.get_output('-t') - self.assertSequenceEqual(output.splitlines(), [ - b"b'Aladdin:open sesame'", - br"b'QWxhZGRpbjpvcGVuIHNlc2FtZQ==\n'", - b"b'Aladdin:open sesame'", - ]) - - def test_encode_file(self) -> None: - with open(support.TESTFN, 'wb') as fp: - fp.write(b'a\xffb\n') - - output = self.get_output('-e', support.TESTFN) - self.assertEqual(output.rstrip(), b'Yf9iCg==') - - with open(support.TESTFN, 'rb') as fp: - output = self.get_output('-e', stdin=fp) - self.assertEqual(output.rstrip(), b'Yf9iCg==') - - def test_decode(self) -> None: - with open(support.TESTFN, 'wb') as fp: - fp.write(b'Yf9iCg==') - output = self.get_output('-d', support.TESTFN) - self.assertEqual(output.rstrip(), b'a\xffb') - - - -def test_main() -> None: - support.run_unittest(__name__) - -if __name__ == '__main__': - test_main() diff --git a/lib-python/3.2/test/test_fnmatch.py b/lib-python/3.2/test/test_fnmatch.py deleted file mode 100644 index 096513521f22..000000000000 --- a/lib-python/3.2/test/test_fnmatch.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Test cases for the fnmatch module.""" - -from test import support -import unittest - -from fnmatch import fnmatch, fnmatchcase, translate, filter - -from typing import Any, AnyStr, Function - -class FnmatchTestCase(unittest.TestCase): - - def check_match(self, filename: AnyStr, pattern: AnyStr, - should_match: int = 1, - fn: Any = fnmatch) -> None: # see #270 - if should_match: - self.assertTrue(fn(filename, pattern), - "expected %r to match pattern %r" - % (filename, pattern)) - else: - self.assertTrue(not fn(filename, pattern), - "expected %r not to match pattern %r" - % (filename, pattern)) - - def test_fnmatch(self) -> None: - check = self.check_match - check('abc', 'abc') - check('abc', '?*?') - check('abc', '???*') - check('abc', '*???') - check('abc', '???') - check('abc', '*') - check('abc', 'ab[cd]') - check('abc', 'ab[!de]') - check('abc', 'ab[de]', 0) - check('a', '??', 0) - check('a', 'b', 0) - - # these test that '\' is handled correctly in character sets; - # see SF bug #409651 - check('\\', r'[\]') - check('a', r'[!\]') - check('\\', r'[!\]', 0) - - # test that filenames with newlines in them are handled correctly. - # http://bugs.python.org/issue6665 - check('foo\nbar', 'foo*') - check('foo\nbar\n', 'foo*') - check('\nfoo', 'foo*', False) - check('\n', '*') - - def test_mix_bytes_str(self) -> None: - self.assertRaises(TypeError, fnmatch, 'test', b'*') - self.assertRaises(TypeError, fnmatch, b'test', '*') - self.assertRaises(TypeError, fnmatchcase, 'test', b'*') - self.assertRaises(TypeError, fnmatchcase, b'test', '*') - - def test_fnmatchcase(self) -> None: - check = self.check_match - check('AbC', 'abc', 0, fnmatchcase) - check('abc', 'AbC', 0, fnmatchcase) - - def test_bytes(self) -> None: - self.check_match(b'test', b'te*') - self.check_match(b'test\xff', b'te*\xff') - self.check_match(b'foo\nbar', b'foo*') - -class TranslateTestCase(unittest.TestCase): - - def test_translate(self) -> None: - self.assertEqual(translate('*'), '.*\Z(?ms)') - self.assertEqual(translate('?'), '.\Z(?ms)') - self.assertEqual(translate('a?b*'), 'a.b.*\Z(?ms)') - self.assertEqual(translate('[abc]'), '[abc]\Z(?ms)') - self.assertEqual(translate('[]]'), '[]]\Z(?ms)') - self.assertEqual(translate('[!x]'), '[^x]\Z(?ms)') - self.assertEqual(translate('[^x]'), '[\\^x]\Z(?ms)') - self.assertEqual(translate('[x'), '\\[x\Z(?ms)') - - -class FilterTestCase(unittest.TestCase): - - def test_filter(self) -> None: - self.assertEqual(filter(['a', 'b'], 'a'), ['a']) - - -def test_main() -> None: - support.run_unittest(FnmatchTestCase, - TranslateTestCase, - FilterTestCase) - - -if __name__ == "__main__": - test_main() diff --git a/lib-python/3.2/test/test_genericpath.py b/lib-python/3.2/test/test_genericpath.py deleted file mode 100644 index b34d7ef2b1f5..000000000000 --- a/lib-python/3.2/test/test_genericpath.py +++ /dev/null @@ -1,313 +0,0 @@ -""" -Tests common to genericpath, macpath, ntpath and posixpath -""" - -import unittest -from test import support -import os - -import genericpath -import imp -imp.reload(genericpath) # Make sure we are using the local copy - -import sys -from typing import Any, List - - -def safe_rmdir(dirname: str) -> None: - try: - os.rmdir(dirname) - except OSError: - pass - - -class GenericTest(unittest.TestCase): - # The path module to be tested - pathmodule = genericpath # type: Any - common_attributes = ['commonprefix', 'getsize', 'getatime', 'getctime', - 'getmtime', 'exists', 'isdir', 'isfile'] - attributes = List[str]() - - def test_no_argument(self) -> None: - for attr in self.common_attributes + self.attributes: - with self.assertRaises(TypeError): - getattr(self.pathmodule, attr)() - self.fail("{}.{}() did not raise a TypeError" - .format(self.pathmodule.__name__, attr)) - - def test_commonprefix(self) -> None: - commonprefix = self.pathmodule.commonprefix - self.assertEqual( - commonprefix([]), - "" - ) - self.assertEqual( - commonprefix(["/home/swenson/spam", "/home/swen/spam"]), - "/home/swen" - ) - self.assertEqual( - commonprefix(["/home/swen/spam", "/home/swen/eggs"]), - "/home/swen/" - ) - self.assertEqual( - commonprefix(["/home/swen/spam", "/home/swen/spam"]), - "/home/swen/spam" - ) - self.assertEqual( - commonprefix(["home:swenson:spam", "home:swen:spam"]), - "home:swen" - ) - self.assertEqual( - commonprefix([":home:swen:spam", ":home:swen:eggs"]), - ":home:swen:" - ) - self.assertEqual( - commonprefix([":home:swen:spam", ":home:swen:spam"]), - ":home:swen:spam" - ) - - self.assertEqual( - commonprefix([b"/home/swenson/spam", b"/home/swen/spam"]), - b"/home/swen" - ) - self.assertEqual( - commonprefix([b"/home/swen/spam", b"/home/swen/eggs"]), - b"/home/swen/" - ) - self.assertEqual( - commonprefix([b"/home/swen/spam", b"/home/swen/spam"]), - b"/home/swen/spam" - ) - self.assertEqual( - commonprefix([b"home:swenson:spam", b"home:swen:spam"]), - b"home:swen" - ) - self.assertEqual( - commonprefix([b":home:swen:spam", b":home:swen:eggs"]), - b":home:swen:" - ) - self.assertEqual( - commonprefix([b":home:swen:spam", b":home:swen:spam"]), - b":home:swen:spam" - ) - - testlist = ['', 'abc', 'Xbcd', 'Xb', 'XY', 'abcd', - 'aXc', 'abd', 'ab', 'aX', 'abcX'] - for s1 in testlist: - for s2 in testlist: - p = commonprefix([s1, s2]) - self.assertTrue(s1.startswith(p)) - self.assertTrue(s2.startswith(p)) - if s1 != s2: - n = len(p) - self.assertNotEqual(s1[n:n+1], s2[n:n+1]) - - def test_getsize(self) -> None: - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertEqual(self.pathmodule.getsize(support.TESTFN), 3) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) - - def test_time(self) -> None: - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - f = open(support.TESTFN, "ab") - f.write(b"bar") - f.close() - f = open(support.TESTFN, "rb") - d = f.read() - f.close() - self.assertEqual(d, b"foobar") - - self.assertLessEqual( - self.pathmodule.getctime(support.TESTFN), - self.pathmodule.getmtime(support.TESTFN) - ) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) - - def test_exists(self) -> None: - self.assertIs(self.pathmodule.exists(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.exists(support.TESTFN), True) - if not self.pathmodule == genericpath: - self.assertIs(self.pathmodule.lexists(support.TESTFN), - True) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) - - def test_isdir(self) -> None: - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isdir(support.TESTFN), True) - os.rmdir(support.TESTFN) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) - - def test_isfile(self) -> None: - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isfile(support.TESTFN), True) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - os.rmdir(support.TESTFN) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) - - -# Following TestCase is not supposed to be run from test_genericpath. -# It is inherited by other test modules (macpath, ntpath, posixpath). - -class CommonTest(GenericTest): - # The path module to be tested - pathmodule = None # type: Any - common_attributes = GenericTest.common_attributes + [ - # Properties - 'curdir', 'pardir', 'extsep', 'sep', - 'pathsep', 'defpath', 'altsep', 'devnull', - # Methods - 'normcase', 'splitdrive', 'expandvars', 'normpath', 'abspath', - 'join', 'split', 'splitext', 'isabs', 'basename', 'dirname', - 'lexists', 'islink', 'ismount', 'expanduser', 'normpath', 'realpath', - ] - - def test_normcase(self) -> None: - normcase = self.pathmodule.normcase - # check that normcase() is idempotent - for p in ["FoO/./BaR", b"FoO/./BaR"]: - p = normcase(p) - self.assertEqual(p, normcase(p)) - - self.assertEqual(normcase(''), '') - self.assertEqual(normcase(b''), b'') - - # check that normcase raises a TypeError for invalid types - for path in (None, True, 0, 2.5, [], bytearray(b''), {'o','o'}): - self.assertRaises(TypeError, normcase, path) - - def test_splitdrive(self) -> None: - # splitdrive for non-NT paths - splitdrive = self.pathmodule.splitdrive - self.assertEqual(splitdrive("/foo/bar"), ("", "/foo/bar")) - self.assertEqual(splitdrive("foo:bar"), ("", "foo:bar")) - self.assertEqual(splitdrive(":foo:bar"), ("", ":foo:bar")) - - self.assertEqual(splitdrive(b"/foo/bar"), (b"", b"/foo/bar")) - self.assertEqual(splitdrive(b"foo:bar"), (b"", b"foo:bar")) - self.assertEqual(splitdrive(b":foo:bar"), (b"", b":foo:bar")) - - def test_expandvars(self) -> None: - if self.pathmodule.__name__ == 'macpath': - self.skipTest('macpath.expandvars is a stub') - expandvars = self.pathmodule.expandvars - with support.EnvironmentVarGuard() as env: - env.clear() - env["foo"] = "bar" - env["{foo"] = "baz1" - env["{foo}"] = "baz2" - self.assertEqual(expandvars("foo"), "foo") - self.assertEqual(expandvars("$foo bar"), "bar bar") - self.assertEqual(expandvars("${foo}bar"), "barbar") - self.assertEqual(expandvars("$[foo]bar"), "$[foo]bar") - self.assertEqual(expandvars("$bar bar"), "$bar bar") - self.assertEqual(expandvars("$?bar"), "$?bar") - self.assertEqual(expandvars("${foo}bar"), "barbar") - self.assertEqual(expandvars("$foo}bar"), "bar}bar") - self.assertEqual(expandvars("${foo"), "${foo") - self.assertEqual(expandvars("${{foo}}"), "baz1}") - self.assertEqual(expandvars("$foo$foo"), "barbar") - self.assertEqual(expandvars("$bar$bar"), "$bar$bar") - - self.assertEqual(expandvars(b"foo"), b"foo") - self.assertEqual(expandvars(b"$foo bar"), b"bar bar") - self.assertEqual(expandvars(b"${foo}bar"), b"barbar") - self.assertEqual(expandvars(b"$[foo]bar"), b"$[foo]bar") - self.assertEqual(expandvars(b"$bar bar"), b"$bar bar") - self.assertEqual(expandvars(b"$?bar"), b"$?bar") - self.assertEqual(expandvars(b"${foo}bar"), b"barbar") - self.assertEqual(expandvars(b"$foo}bar"), b"bar}bar") - self.assertEqual(expandvars(b"${foo"), b"${foo") - self.assertEqual(expandvars(b"${{foo}}"), b"baz1}") - self.assertEqual(expandvars(b"$foo$foo"), b"barbar") - self.assertEqual(expandvars(b"$bar$bar"), b"$bar$bar") - - def test_abspath(self) -> None: - self.assertIn("foo", self.pathmodule.abspath("foo")) - self.assertIn(b"foo", self.pathmodule.abspath(b"foo")) - - # Abspath returns bytes when the arg is bytes - for path in (b'', b'foo', b'f\xf2\xf2', b'/foo', b'C:\\'): - self.assertIsInstance(self.pathmodule.abspath(path), bytes) - - def test_realpath(self) -> None: - self.assertIn("foo", self.pathmodule.realpath("foo")) - self.assertIn(b"foo", self.pathmodule.realpath(b"foo")) - - def test_normpath_issue5827(self) -> None: - # Make sure normpath preserves unicode - for path in ('', '.', '/', '\\', '///foo/.//bar//'): - self.assertIsInstance(self.pathmodule.normpath(path), str) - - def test_abspath_issue3426(self) -> None: - # Check that abspath returns unicode when the arg is unicode - # with both ASCII and non-ASCII cwds. - abspath = self.pathmodule.abspath - for path in ('', 'fuu', 'f\xf9\xf9', '/fuu', 'U:\\'): - self.assertIsInstance(abspath(path), str) - - unicwd = '\xe7w\xf0' - try: - fsencoding = support.TESTFN_ENCODING or "ascii" - unicwd.encode(fsencoding) - except (AttributeError, UnicodeEncodeError): - # FS encoding is probably ASCII - pass - else: - with support.temp_cwd(unicwd): - for path in ('', 'fuu', 'f\xf9\xf9', '/fuu', 'U:\\'): - self.assertIsInstance(abspath(path), str) - - @unittest.skipIf(sys.platform == 'darwin', - "Mac OS X denies the creation of a directory with an invalid utf8 name") - def test_nonascii_abspath(self) -> None: - # Test non-ASCII, non-UTF8 bytes in the path. - with support.temp_cwd(b'\xe7w\xf0'): - self.test_abspath() - - -def test_main() -> None: - support.run_unittest(GenericTest) - - -if __name__=="__main__": - test_main() diff --git a/lib-python/3.2/test/test_getopt.py b/lib-python/3.2/test/test_getopt.py deleted file mode 100644 index 3a92aa82cac6..000000000000 --- a/lib-python/3.2/test/test_getopt.py +++ /dev/null @@ -1,190 +0,0 @@ -# test_getopt.py -# David Goodger 2000-08-19 - -from test.support import verbose, run_doctest, run_unittest, EnvironmentVarGuard -import unittest - -import getopt - -from typing import Any - -sentinel = object() - -class GetoptTests(unittest.TestCase): - def setUp(self) -> None: - self.env = EnvironmentVarGuard() - if "POSIXLY_CORRECT" in self.env: - del self.env["POSIXLY_CORRECT"] - - def tearDown(self) -> None: - self.env.__exit__() - del self.env - - def assertError(self, *args: Any, **kwargs: Any) -> None: - # JLe: work around mypy bug #229 - Any(self.assertRaises)(getopt.GetoptError, *args, **kwargs) - - def test_short_has_arg(self) -> None: - self.assertTrue(getopt.short_has_arg('a', 'a:')) - self.assertFalse(getopt.short_has_arg('a', 'a')) - self.assertError(getopt.short_has_arg, 'a', 'b') - - def test_long_has_args(self) -> None: - has_arg, option = getopt.long_has_args('abc', ['abc=']) - self.assertTrue(has_arg) - self.assertEqual(option, 'abc') - - has_arg, option = getopt.long_has_args('abc', ['abc']) - self.assertFalse(has_arg) - self.assertEqual(option, 'abc') - - has_arg, option = getopt.long_has_args('abc', ['abcd']) - self.assertFalse(has_arg) - self.assertEqual(option, 'abcd') - - self.assertError(getopt.long_has_args, 'abc', ['def']) - self.assertError(getopt.long_has_args, 'abc', []) - self.assertError(getopt.long_has_args, 'abc', ['abcd','abcde']) - - def test_do_shorts(self) -> None: - opts, args = getopt.do_shorts([], 'a', 'a', []) - self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, []) - - opts, args = getopt.do_shorts([], 'a1', 'a:', []) - self.assertEqual(opts, [('-a', '1')]) - self.assertEqual(args, []) - - #opts, args = getopt.do_shorts([], 'a=1', 'a:', []) - #self.assertEqual(opts, [('-a', '1')]) - #self.assertEqual(args, []) - - opts, args = getopt.do_shorts([], 'a', 'a:', ['1']) - self.assertEqual(opts, [('-a', '1')]) - self.assertEqual(args, []) - - opts, args = getopt.do_shorts([], 'a', 'a:', ['1', '2']) - self.assertEqual(opts, [('-a', '1')]) - self.assertEqual(args, ['2']) - - self.assertError(getopt.do_shorts, [], 'a1', 'a', []) - self.assertError(getopt.do_shorts, [], 'a', 'a:', []) - - def test_do_longs(self) -> None: - opts, args = getopt.do_longs([], 'abc', ['abc'], []) - self.assertEqual(opts, [('--abc', '')]) - self.assertEqual(args, []) - - opts, args = getopt.do_longs([], 'abc=1', ['abc='], []) - self.assertEqual(opts, [('--abc', '1')]) - self.assertEqual(args, []) - - opts, args = getopt.do_longs([], 'abc=1', ['abcd='], []) - self.assertEqual(opts, [('--abcd', '1')]) - self.assertEqual(args, []) - - opts, args = getopt.do_longs([], 'abc', ['ab', 'abc', 'abcd'], []) - self.assertEqual(opts, [('--abc', '')]) - self.assertEqual(args, []) - - # Much like the preceding, except with a non-alpha character ("-") in - # option name that precedes "="; failed in - # http://python.org/sf/126863 - opts, args = getopt.do_longs([], 'foo=42', ['foo-bar', 'foo=',], []) - self.assertEqual(opts, [('--foo', '42')]) - self.assertEqual(args, []) - - self.assertError(getopt.do_longs, [], 'abc=1', ['abc'], []) - self.assertError(getopt.do_longs, [], 'abc', ['abc='], []) - - def test_getopt(self) -> None: - # note: the empty string between '-a' and '--beta' is significant: - # it simulates an empty string option argument ('-a ""') on the - # command line. - cmdline = ['-a', '1', '-b', '--alpha=2', '--beta', '-a', '3', '-a', - '', '--beta', 'arg1', 'arg2'] - - opts, args = getopt.getopt(cmdline, 'a:b', ['alpha=', 'beta']) - self.assertEqual(opts, [('-a', '1'), ('-b', ''), - ('--alpha', '2'), ('--beta', ''), - ('-a', '3'), ('-a', ''), ('--beta', '')]) - # Note ambiguity of ('-b', '') and ('-a', '') above. This must be - # accounted for in the code that calls getopt(). - self.assertEqual(args, ['arg1', 'arg2']) - - self.assertError(getopt.getopt, cmdline, 'a:b', ['alpha', 'beta']) - - def test_gnu_getopt(self) -> None: - # Test handling of GNU style scanning mode. - cmdline = ['-a', 'arg1', '-b', '1', '--alpha', '--beta=2'] - - # GNU style - opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) - self.assertEqual(args, ['arg1']) - self.assertEqual(opts, [('-a', ''), ('-b', '1'), - ('--alpha', ''), ('--beta', '2')]) - - # recognize "-" as an argument - opts, args = getopt.gnu_getopt(['-a', '-', '-b', '-'], 'ab:', []) - self.assertEqual(args, ['-']) - self.assertEqual(opts, [('-a', ''), ('-b', '-')]) - - # Posix style via + - opts, args = getopt.gnu_getopt(cmdline, '+ab:', ['alpha', 'beta=']) - self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) - - # Posix style via POSIXLY_CORRECT - self.env["POSIXLY_CORRECT"] = "1" - opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) - self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) - - def test_libref_examples(self) -> None: - s = """ - Examples from the Library Reference: Doc/lib/libgetopt.tex - - An example using only Unix style options: - - - >>> import getopt - >>> args = '-a -b -cfoo -d bar a1 a2'.split() - >>> args - ['-a', '-b', '-cfoo', '-d', 'bar', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'abc:d:') - >>> optlist - [('-a', ''), ('-b', ''), ('-c', 'foo'), ('-d', 'bar')] - >>> args - ['a1', 'a2'] - - Using long option names is equally easy: - - - >>> s = '--condition=foo --testing --output-file abc.def -x a1 a2' - >>> args = s.split() - >>> args - ['--condition=foo', '--testing', '--output-file', 'abc.def', '-x', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'x', [ - ... 'condition=', 'output-file=', 'testing']) - >>> optlist - [('--condition', 'foo'), ('--testing', ''), ('--output-file', 'abc.def'), ('-x', '')] - >>> args - ['a1', 'a2'] - """ - - import types - m = types.ModuleType("libreftest", s) - run_doctest(m, verbose) - - def test_issue4629(self) -> None: - longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) - self.assertEqual(longopts, [('--help', '')]) - longopts, shortopts = getopt.getopt(['--help=x'], '', ['help=']) - self.assertEqual(longopts, [('--help', 'x')]) - self.assertRaises(getopt.GetoptError, getopt.getopt, ['--help='], '', ['help']) - -def test_main() -> None: - run_unittest(GetoptTests) - -if __name__ == "__main__": - test_main() diff --git a/lib-python/3.2/test/test_glob.py b/lib-python/3.2/test/test_glob.py deleted file mode 100644 index b5975edda891..000000000000 --- a/lib-python/3.2/test/test_glob.py +++ /dev/null @@ -1,122 +0,0 @@ -import unittest -from test.support import run_unittest, TESTFN, skip_unless_symlink, can_symlink -import glob -import os -import shutil - -from typing import typevar, Iterable, List - -T = typevar('T') - -class GlobTests(unittest.TestCase): - - tempdir = '' - - # JLe: work around mypy issue #231 - def norm(self, first: str, *parts: str) -> str: - return os.path.normpath(os.path.join(self.tempdir, first, *parts)) - - def mktemp(self, *parts: str) -> None: - filename = self.norm(*parts) - base, file = os.path.split(filename) - if not os.path.exists(base): - os.makedirs(base) - f = open(filename, 'w') - f.close() - - def setUp(self) -> None: - self.tempdir = TESTFN+"_dir" - self.mktemp('a', 'D') - self.mktemp('aab', 'F') - self.mktemp('aaa', 'zzzF') - self.mktemp('ZZZ') - self.mktemp('a', 'bcd', 'EF') - self.mktemp('a', 'bcd', 'efg', 'ha') - if can_symlink(): - os.symlink(self.norm('broken'), self.norm('sym1')) - os.symlink(self.norm('broken'), self.norm('sym2')) - - def tearDown(self) -> None: - shutil.rmtree(self.tempdir) - - def glob(self, *parts: str) -> List[str]: - if len(parts) == 1: - pattern = parts[0] - else: - pattern = os.path.join(*parts) - p = os.path.join(self.tempdir, pattern) - res = glob.glob(p) - self.assertEqual(list(glob.iglob(p)), res) - return res - - def assertSequencesEqual_noorder(self, l1: Iterable[T], - l2: Iterable[T]) -> None: - self.assertEqual(set(l1), set(l2)) - - def test_glob_literal(self) -> None: - eq = self.assertSequencesEqual_noorder - eq(self.glob('a'), [self.norm('a')]) - eq(self.glob('a', 'D'), [self.norm('a', 'D')]) - eq(self.glob('aab'), [self.norm('aab')]) - eq(self.glob('zymurgy'), List[str]()) # JLe: work around #230 - - # test return types are unicode, but only if os.listdir - # returns unicode filenames - uniset = set([str]) - tmp = os.listdir('.') - if set(type(x) for x in tmp) == uniset: - u1 = glob.glob('*') - u2 = glob.glob('./*') - self.assertEqual(set(type(r) for r in u1), uniset) - self.assertEqual(set(type(r) for r in u2), uniset) - - def test_glob_one_directory(self) -> None: - eq = self.assertSequencesEqual_noorder - eq(self.glob('a*'), map(self.norm, ['a', 'aab', 'aaa'])) - eq(self.glob('*a'), map(self.norm, ['a', 'aaa'])) - eq(self.glob('aa?'), map(self.norm, ['aaa', 'aab'])) - eq(self.glob('aa[ab]'), map(self.norm, ['aaa', 'aab'])) - eq(self.glob('*q'), List[str]()) - - def test_glob_nested_directory(self) -> None: - eq = self.assertSequencesEqual_noorder - if os.path.normcase("abCD") == "abCD": - # case-sensitive filesystem - eq(self.glob('a', 'bcd', 'E*'), [self.norm('a', 'bcd', 'EF')]) - else: - # case insensitive filesystem - eq(self.glob('a', 'bcd', 'E*'), [self.norm('a', 'bcd', 'EF'), - self.norm('a', 'bcd', 'efg')]) - eq(self.glob('a', 'bcd', '*g'), [self.norm('a', 'bcd', 'efg')]) - - def test_glob_directory_names(self) -> None: - eq = self.assertSequencesEqual_noorder - eq(self.glob('*', 'D'), [self.norm('a', 'D')]) - eq(self.glob('*', '*a'), List[str]()) - eq(self.glob('a', '*', '*', '*a'), - [self.norm('a', 'bcd', 'efg', 'ha')]) - eq(self.glob('?a?', '*F'), map(self.norm, [os.path.join('aaa', 'zzzF'), - os.path.join('aab', 'F')])) - - def test_glob_directory_with_trailing_slash(self) -> None: - # We are verifying that when there is wildcard pattern which - # ends with os.sep doesn't blow up. - res = glob.glob(self.tempdir + '*' + os.sep) - self.assertEqual(len(res), 1) - # either of these results are reasonable - self.assertIn(res[0], [self.tempdir, self.tempdir + os.sep]) - - @skip_unless_symlink - def test_glob_broken_symlinks(self) -> None: - eq = self.assertSequencesEqual_noorder - eq(self.glob('sym*'), [self.norm('sym1'), self.norm('sym2')]) - eq(self.glob('sym1'), [self.norm('sym1')]) - eq(self.glob('sym2'), [self.norm('sym2')]) - - -def test_main() -> None: - run_unittest(GlobTests) - - -if __name__ == "__main__": - test_main() diff --git a/lib-python/3.2/test/test_posixpath.py b/lib-python/3.2/test/test_posixpath.py deleted file mode 100644 index b3da34fba213..000000000000 --- a/lib-python/3.2/test/test_posixpath.py +++ /dev/null @@ -1,531 +0,0 @@ -import unittest -from test import support, test_genericpath - -import posixpath -import genericpath - -import imp -imp.reload(posixpath) # Make sure we are using the local copy -imp.reload(genericpath) - -import os -import sys -from posixpath import realpath, abspath, dirname, basename - -import posix -from typing import Any, typevar, Function - -T = typevar('T') - -# An absolute path to a temporary filename for testing. We can't rely on TESTFN -# being an absolute path, so we need this. - -ABSTFN = abspath(support.TESTFN) - -def skip_if_ABSTFN_contains_backslash( - test: Function[[T], None]) -> Function[[T], None]: - """ - On Windows, posixpath.abspath still returns paths with backslashes - instead of posix forward slashes. If this is the case, several tests - fail, so skip them. - """ - found_backslash = '\\' in ABSTFN - msg = "ABSTFN is not a posix path - tests fail" - return [test, unittest.skip(msg)(test)][found_backslash] - -def safe_rmdir(dirname: str) -> None: - try: - os.rmdir(dirname) - except OSError: - pass - -class PosixPathTest(unittest.TestCase): - - def setUp(self) -> None: - self.tearDown() - - def tearDown(self) -> None: - for suffix in ["", "1", "2"]: - support.unlink(support.TESTFN + suffix) - safe_rmdir(support.TESTFN + suffix) - - def test_join(self) -> None: - self.assertEqual(posixpath.join("/foo", "bar", "/bar", "baz"), - "/bar/baz") - self.assertEqual(posixpath.join("/foo", "bar", "baz"), "/foo/bar/baz") - self.assertEqual(posixpath.join("/foo/", "bar/", "baz/"), - "/foo/bar/baz/") - - self.assertEqual(posixpath.join(b"/foo", b"bar", b"/bar", b"baz"), - b"/bar/baz") - self.assertEqual(posixpath.join(b"/foo", b"bar", b"baz"), - b"/foo/bar/baz") - self.assertEqual(posixpath.join(b"/foo/", b"bar/", b"baz/"), - b"/foo/bar/baz/") - - self.assertRaises(TypeError, posixpath.join, b"bytes", "str") - self.assertRaises(TypeError, posixpath.join, "str", b"bytes") - - def test_split(self) -> None: - self.assertEqual(posixpath.split("/foo/bar"), ("/foo", "bar")) - self.assertEqual(posixpath.split("/"), ("/", "")) - self.assertEqual(posixpath.split("foo"), ("", "foo")) - self.assertEqual(posixpath.split("////foo"), ("////", "foo")) - self.assertEqual(posixpath.split("//foo//bar"), ("//foo", "bar")) - - self.assertEqual(posixpath.split(b"/foo/bar"), (b"/foo", b"bar")) - self.assertEqual(posixpath.split(b"/"), (b"/", b"")) - self.assertEqual(posixpath.split(b"foo"), (b"", b"foo")) - self.assertEqual(posixpath.split(b"////foo"), (b"////", b"foo")) - self.assertEqual(posixpath.split(b"//foo//bar"), (b"//foo", b"bar")) - - def splitextTest(self, path: str, filename: str, ext: str) -> None: - self.assertEqual(posixpath.splitext(path), (filename, ext)) - self.assertEqual(posixpath.splitext("/" + path), ("/" + filename, ext)) - self.assertEqual(posixpath.splitext("abc/" + path), - ("abc/" + filename, ext)) - self.assertEqual(posixpath.splitext("abc.def/" + path), - ("abc.def/" + filename, ext)) - self.assertEqual(posixpath.splitext("/abc.def/" + path), - ("/abc.def/" + filename, ext)) - self.assertEqual(posixpath.splitext(path + "/"), - (filename + ext + "/", "")) - - pathb = bytes(path, "ASCII") - filenameb = bytes(filename, "ASCII") - extb = bytes(ext, "ASCII") - - self.assertEqual(posixpath.splitext(pathb), (filenameb, extb)) - self.assertEqual(posixpath.splitext(b"/" + pathb), - (b"/" + filenameb, extb)) - self.assertEqual(posixpath.splitext(b"abc/" + pathb), - (b"abc/" + filenameb, extb)) - self.assertEqual(posixpath.splitext(b"abc.def/" + pathb), - (b"abc.def/" + filenameb, extb)) - self.assertEqual(posixpath.splitext(b"/abc.def/" + pathb), - (b"/abc.def/" + filenameb, extb)) - self.assertEqual(posixpath.splitext(pathb + b"/"), - (filenameb + extb + b"/", b"")) - - def test_splitext(self) -> None: - self.splitextTest("foo.bar", "foo", ".bar") - self.splitextTest("foo.boo.bar", "foo.boo", ".bar") - self.splitextTest("foo.boo.biff.bar", "foo.boo.biff", ".bar") - self.splitextTest(".csh.rc", ".csh", ".rc") - self.splitextTest("nodots", "nodots", "") - self.splitextTest(".cshrc", ".cshrc", "") - self.splitextTest("...manydots", "...manydots", "") - self.splitextTest("...manydots.ext", "...manydots", ".ext") - self.splitextTest(".", ".", "") - self.splitextTest("..", "..", "") - self.splitextTest("........", "........", "") - self.splitextTest("", "", "") - - def test_isabs(self) -> None: - self.assertIs(posixpath.isabs(""), False) - self.assertIs(posixpath.isabs("/"), True) - self.assertIs(posixpath.isabs("/foo"), True) - self.assertIs(posixpath.isabs("/foo/bar"), True) - self.assertIs(posixpath.isabs("foo/bar"), False) - - self.assertIs(posixpath.isabs(b""), False) - self.assertIs(posixpath.isabs(b"/"), True) - self.assertIs(posixpath.isabs(b"/foo"), True) - self.assertIs(posixpath.isabs(b"/foo/bar"), True) - self.assertIs(posixpath.isabs(b"foo/bar"), False) - - def test_basename(self) -> None: - self.assertEqual(posixpath.basename("/foo/bar"), "bar") - self.assertEqual(posixpath.basename("/"), "") - self.assertEqual(posixpath.basename("foo"), "foo") - self.assertEqual(posixpath.basename("////foo"), "foo") - self.assertEqual(posixpath.basename("//foo//bar"), "bar") - - self.assertEqual(posixpath.basename(b"/foo/bar"), b"bar") - self.assertEqual(posixpath.basename(b"/"), b"") - self.assertEqual(posixpath.basename(b"foo"), b"foo") - self.assertEqual(posixpath.basename(b"////foo"), b"foo") - self.assertEqual(posixpath.basename(b"//foo//bar"), b"bar") - - def test_dirname(self) -> None: - self.assertEqual(posixpath.dirname("/foo/bar"), "/foo") - self.assertEqual(posixpath.dirname("/"), "/") - self.assertEqual(posixpath.dirname("foo"), "") - self.assertEqual(posixpath.dirname("////foo"), "////") - self.assertEqual(posixpath.dirname("//foo//bar"), "//foo") - - self.assertEqual(posixpath.dirname(b"/foo/bar"), b"/foo") - self.assertEqual(posixpath.dirname(b"/"), b"/") - self.assertEqual(posixpath.dirname(b"foo"), b"") - self.assertEqual(posixpath.dirname(b"////foo"), b"////") - self.assertEqual(posixpath.dirname(b"//foo//bar"), b"//foo") - - def test_islink(self) -> None: - self.assertIs(posixpath.islink(support.TESTFN + "1"), False) - self.assertIs(posixpath.lexists(support.TESTFN + "2"), False) - f = open(support.TESTFN + "1", "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(posixpath.islink(support.TESTFN + "1"), False) - if support.can_symlink(): - os.symlink(support.TESTFN + "1", support.TESTFN + "2") - self.assertIs(posixpath.islink(support.TESTFN + "2"), True) - os.remove(support.TESTFN + "1") - self.assertIs(posixpath.islink(support.TESTFN + "2"), True) - self.assertIs(posixpath.exists(support.TESTFN + "2"), False) - self.assertIs(posixpath.lexists(support.TESTFN + "2"), True) - finally: - if not f.closed: - f.close() - - @staticmethod - def _create_file(filename: str) -> None: - with open(filename, 'wb') as f: - f.write(b'foo') - - def test_samefile(self) -> None: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - self.assertTrue(posixpath.samefile(test_fn, test_fn)) - self.assertRaises(TypeError, posixpath.samefile) - - @unittest.skipIf( - sys.platform.startswith('win'), - "posixpath.samefile does not work on links in Windows") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - def test_samefile_on_links(self) -> None: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) - - os.symlink(test_fn1, test_fn2) - self.assertTrue(posixpath.samefile(test_fn1, test_fn2)) - os.remove(test_fn2) - - self._create_file(test_fn2) - self.assertFalse(posixpath.samefile(test_fn1, test_fn2)) - - - def test_samestat(self) -> None: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - test_fns = [test_fn]*2 - stats = map(os.stat, test_fns) - self.assertTrue(posixpath.samestat(*stats)) - - @unittest.skipIf( - sys.platform.startswith('win'), - "posixpath.samestat does not work on links in Windows") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - def test_samestat_on_links(self) -> None: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) - test_fns = [test_fn1, test_fn2] - Any(os.symlink)(*test_fns) - stats = map(os.stat, test_fns) - self.assertTrue(posixpath.samestat(*stats)) - os.remove(test_fn2) - - self._create_file(test_fn2) - stats = map(os.stat, test_fns) - self.assertFalse(posixpath.samestat(*stats)) - - self.assertRaises(TypeError, posixpath.samestat) - - def test_ismount(self) -> None: - self.assertIs(posixpath.ismount("/"), True) - self.assertIs(posixpath.ismount(b"/"), True) - - def test_ismount_non_existent(self) -> None: - # Non-existent mountpoint. - self.assertIs(posixpath.ismount(ABSTFN), False) - try: - os.mkdir(ABSTFN) - self.assertIs(posixpath.ismount(ABSTFN), False) - finally: - safe_rmdir(ABSTFN) - - @unittest.skipUnless(support.can_symlink(), - "Test requires symlink support") - def test_ismount_symlinks(self) -> None: - # Symlinks are never mountpoints. - try: - os.symlink("/", ABSTFN) - self.assertIs(posixpath.ismount(ABSTFN), False) - finally: - os.unlink(ABSTFN) - - @unittest.skipIf(posix is None, "Test requires posix module") - def test_ismount_different_device(self) -> None: - # Simulate the path being on a different device from its parent by - # mocking out st_dev. - save_lstat = os.lstat - def fake_lstat(path): - st_ino = 0 - st_dev = 0 - if path == ABSTFN: - st_dev = 1 - st_ino = 1 - return posix.stat_result((0, st_ino, st_dev, 0, 0, 0, 0, 0, 0, 0)) - try: - setattr(os, 'lstat', fake_lstat) # mypy: can't modify os directly - self.assertIs(posixpath.ismount(ABSTFN), True) - finally: - setattr(os, 'lstat', save_lstat) - - def test_expanduser(self) -> None: - self.assertEqual(posixpath.expanduser("foo"), "foo") - self.assertEqual(posixpath.expanduser(b"foo"), b"foo") - try: - import pwd - except ImportError: - pass - else: - self.assertIsInstance(posixpath.expanduser("~/"), str) - self.assertIsInstance(posixpath.expanduser(b"~/"), bytes) - # if home directory == root directory, this test makes no sense - if posixpath.expanduser("~") != '/': - self.assertEqual( - posixpath.expanduser("~") + "/", - posixpath.expanduser("~/") - ) - self.assertEqual( - posixpath.expanduser(b"~") + b"/", - posixpath.expanduser(b"~/") - ) - self.assertIsInstance(posixpath.expanduser("~root/"), str) - self.assertIsInstance(posixpath.expanduser("~foo/"), str) - self.assertIsInstance(posixpath.expanduser(b"~root/"), bytes) - self.assertIsInstance(posixpath.expanduser(b"~foo/"), bytes) - - with support.EnvironmentVarGuard() as env: - env['HOME'] = '/' - self.assertEqual(posixpath.expanduser("~"), "/") - # expanduser should fall back to using the password database - del env['HOME'] - home = pwd.getpwuid(os.getuid()).pw_dir - self.assertEqual(posixpath.expanduser("~"), home) - - def test_normpath(self) -> None: - self.assertEqual(posixpath.normpath(""), ".") - self.assertEqual(posixpath.normpath("/"), "/") - self.assertEqual(posixpath.normpath("//"), "//") - self.assertEqual(posixpath.normpath("///"), "/") - self.assertEqual(posixpath.normpath("///foo/.//bar//"), "/foo/bar") - self.assertEqual(posixpath.normpath("///foo/.//bar//.//..//.//baz"), - "/foo/baz") - self.assertEqual(posixpath.normpath("///..//./foo/.//bar"), "/foo/bar") - - self.assertEqual(posixpath.normpath(b""), b".") - self.assertEqual(posixpath.normpath(b"/"), b"/") - self.assertEqual(posixpath.normpath(b"//"), b"//") - self.assertEqual(posixpath.normpath(b"///"), b"/") - self.assertEqual(posixpath.normpath(b"///foo/.//bar//"), b"/foo/bar") - self.assertEqual(posixpath.normpath(b"///foo/.//bar//.//..//.//baz"), - b"/foo/baz") - self.assertEqual(posixpath.normpath(b"///..//./foo/.//bar"), - b"/foo/bar") - - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - @skip_if_ABSTFN_contains_backslash - def test_realpath_basic(self) -> None: - # Basic operation. - try: - os.symlink(ABSTFN+"1", ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") - finally: - support.unlink(ABSTFN) - - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - @skip_if_ABSTFN_contains_backslash - def test_realpath_relative(self) -> None: - try: - os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") - finally: - support.unlink(ABSTFN) - - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - @skip_if_ABSTFN_contains_backslash - def test_realpath_symlink_loops(self) -> None: - # Bug #930024, return the path unchanged if we get into an infinite - # symlink loop. - try: - old_path = abspath('.') - os.symlink(ABSTFN, ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN) - - os.symlink(ABSTFN+"1", ABSTFN+"2") - os.symlink(ABSTFN+"2", ABSTFN+"1") - self.assertEqual(realpath(ABSTFN+"1"), ABSTFN+"1") - self.assertEqual(realpath(ABSTFN+"2"), ABSTFN+"2") - - # Test using relative path as well. - os.chdir(dirname(ABSTFN)) - self.assertEqual(realpath(basename(ABSTFN)), ABSTFN) - finally: - os.chdir(old_path) - support.unlink(ABSTFN) - support.unlink(ABSTFN+"1") - support.unlink(ABSTFN+"2") - - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_parents(self) -> None: - # We also need to resolve any symlinks in the parents of a relative - # path passed to realpath. E.g.: current working directory is - # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call - # realpath("a"). This should return /usr/share/doc/a/. - try: - old_path = abspath('.') - os.mkdir(ABSTFN) - os.mkdir(ABSTFN + "/y") - os.symlink(ABSTFN + "/y", ABSTFN + "/k") - - os.chdir(ABSTFN + "/k") - self.assertEqual(realpath("a"), ABSTFN + "/y/a") - finally: - os.chdir(old_path) - support.unlink(ABSTFN + "/k") - safe_rmdir(ABSTFN + "/y") - safe_rmdir(ABSTFN) - - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_before_normalizing(self) -> None: - # Bug #990669: Symbolic links should be resolved before we - # normalize the path. E.g.: if we have directories 'a', 'k' and 'y' - # in the following hierarchy: - # a/k/y - # - # and a symbolic link 'link-y' pointing to 'y' in directory 'a', - # then realpath("link-y/..") should return 'k', not 'a'. - try: - old_path = abspath('.') - os.mkdir(ABSTFN) - os.mkdir(ABSTFN + "/k") - os.mkdir(ABSTFN + "/k/y") - os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y") - - # Absolute path. - self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k") - # Relative path. - os.chdir(dirname(ABSTFN)) - self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."), - ABSTFN + "/k") - finally: - os.chdir(old_path) - support.unlink(ABSTFN + "/link-y") - safe_rmdir(ABSTFN + "/k/y") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) - - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_first(self) -> None: - # Bug #1213894: The first component of the path, if not absolute, - # must be resolved too. - - try: - old_path = abspath('.') - os.mkdir(ABSTFN) - os.mkdir(ABSTFN + "/k") - os.symlink(ABSTFN, ABSTFN + "link") - os.chdir(dirname(ABSTFN)) - - base = basename(ABSTFN) - self.assertEqual(realpath(base + "link"), ABSTFN) - self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") - finally: - os.chdir(old_path) - support.unlink(ABSTFN + "link") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) - - def test_relpath(self) -> None: - real_getcwd = os.getcwd - # mypy: can't modify os directly - setattr(os, 'getcwd', lambda: r"/home/user/bar") - try: - curdir = os.path.split(os.getcwd())[-1] - self.assertRaises(ValueError, posixpath.relpath, "") - self.assertEqual(posixpath.relpath("a"), "a") - self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") - self.assertEqual(posixpath.relpath("a/b"), "a/b") - self.assertEqual(posixpath.relpath("../a/b"), "../a/b") - self.assertEqual(posixpath.relpath("a", "../b"), "../"+curdir+"/a") - self.assertEqual(posixpath.relpath("a/b", "../c"), - "../"+curdir+"/a/b") - self.assertEqual(posixpath.relpath("a", "b/c"), "../../a") - self.assertEqual(posixpath.relpath("a", "a"), ".") - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x/y/z"), '../../../foo/bar/bat') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/foo/bar"), 'bat') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/"), 'foo/bar/bat') - self.assertEqual(posixpath.relpath("/", "/foo/bar/bat"), '../../..') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x"), '../foo/bar/bat') - self.assertEqual(posixpath.relpath("/x", "/foo/bar/bat"), '../../../x') - self.assertEqual(posixpath.relpath("/", "/"), '.') - self.assertEqual(posixpath.relpath("/a", "/a"), '.') - self.assertEqual(posixpath.relpath("/a/b", "/a/b"), '.') - finally: - setattr(os, 'getcwd', real_getcwd) - - def test_relpath_bytes(self) -> None: - real_getcwdb = os.getcwdb - # mypy: can't modify os directly - setattr(os, 'getcwdb', lambda: br"/home/user/bar") - try: - curdir = os.path.split(os.getcwdb())[-1] - self.assertRaises(ValueError, posixpath.relpath, b"") - self.assertEqual(posixpath.relpath(b"a"), b"a") - self.assertEqual(posixpath.relpath(posixpath.abspath(b"a")), b"a") - self.assertEqual(posixpath.relpath(b"a/b"), b"a/b") - self.assertEqual(posixpath.relpath(b"../a/b"), b"../a/b") - self.assertEqual(posixpath.relpath(b"a", b"../b"), - b"../"+curdir+b"/a") - self.assertEqual(posixpath.relpath(b"a/b", b"../c"), - b"../"+curdir+b"/a/b") - self.assertEqual(posixpath.relpath(b"a", b"b/c"), b"../../a") - self.assertEqual(posixpath.relpath(b"a", b"a"), b".") - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x/y/z"), b'../../../foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/foo/bar"), b'bat') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/"), b'foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/", b"/foo/bar/bat"), b'../../..') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x"), b'../foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/x", b"/foo/bar/bat"), b'../../../x') - self.assertEqual(posixpath.relpath(b"/", b"/"), b'.') - self.assertEqual(posixpath.relpath(b"/a", b"/a"), b'.') - self.assertEqual(posixpath.relpath(b"/a/b", b"/a/b"), b'.') - - self.assertRaises(TypeError, posixpath.relpath, b"bytes", "str") - self.assertRaises(TypeError, posixpath.relpath, "str", b"bytes") - finally: - setattr(os, 'getcwdb', real_getcwdb) - - def test_sameopenfile(self) -> None: - fname = support.TESTFN + "1" - with open(fname, "wb") as a, open(fname, "wb") as b: - self.assertTrue(posixpath.sameopenfile(a.fileno(), b.fileno())) - - -class PosixCommonTest(test_genericpath.CommonTest): - pathmodule = posixpath - attributes = ['relpath', 'samefile', 'sameopenfile', 'samestat'] - - -def test_main() -> None: - support.run_unittest(PosixPathTest, PosixCommonTest) - - -if __name__=="__main__": - test_main() diff --git a/lib-python/3.2/test/test_pprint.py b/lib-python/3.2/test/test_pprint.py deleted file mode 100644 index 69398584cd65..000000000000 --- a/lib-python/3.2/test/test_pprint.py +++ /dev/null @@ -1,488 +0,0 @@ -import pprint -import test.support -import unittest -import test.test_set -import random -import collections -import itertools - -from typing import List, Any, Dict, Tuple, cast, Undefined, Function - -# list, tuple and dict subclasses that do or don't overwrite __repr__ -class list2(list): - pass - -class list3(list): - def __repr__(self) -> str: - return list.__repr__(self) - -class tuple2(tuple): - pass - -class tuple3(tuple): - def __repr__(self) -> str: - return tuple.__repr__(self) - -class dict2(dict): - pass - -class dict3(dict): - def __repr__(self) -> str: - return dict.__repr__(self) - -class Unorderable: - def __repr__(self) -> str: - return str(id(self)) - -class QueryTestCase(unittest.TestCase): - - def setUp(self) -> None: - self.a = List[Any](range(100)) - self.b = List[Any](range(200)) - self.a[-12] = self.b - - def test_basic(self) -> None: - # Verify .isrecursive() and .isreadable() w/o recursion - pp = pprint.PrettyPrinter() - for safe in (2, 2.0, complex(0.0, 2.0), "abc", [3], (2,2), {3: 3}, "yaddayadda", - self.a, self.b): - # module-level convenience functions - self.assertFalse(pprint.isrecursive(safe), - "expected not isrecursive for %r" % (safe,)) - self.assertTrue(pprint.isreadable(safe), - "expected isreadable for %r" % (safe,)) - # PrettyPrinter methods - self.assertFalse(pp.isrecursive(safe), - "expected not isrecursive for %r" % (safe,)) - self.assertTrue(pp.isreadable(safe), - "expected isreadable for %r" % (safe,)) - - def test_knotted(self) -> None: - # Verify .isrecursive() and .isreadable() w/ recursion - # Tie a knot. - self.b[67] = self.a - # Messy dict. - self.d = Dict[int, dict]() - self.d[0] = self.d[1] = self.d[2] = self.d - - pp = pprint.PrettyPrinter() - - for icky in self.a, self.b, self.d, (self.d, self.d): - self.assertTrue(pprint.isrecursive(icky), "expected isrecursive") - self.assertFalse(pprint.isreadable(icky), "expected not isreadable") - self.assertTrue(pp.isrecursive(icky), "expected isrecursive") - self.assertFalse(pp.isreadable(icky), "expected not isreadable") - - # Break the cycles. - self.d.clear() - del self.a[:] - del self.b[:] - - for safe in self.a, self.b, self.d, (self.d, self.d): - # module-level convenience functions - self.assertFalse(pprint.isrecursive(safe), - "expected not isrecursive for %r" % (safe,)) - self.assertTrue(pprint.isreadable(safe), - "expected isreadable for %r" % (safe,)) - # PrettyPrinter methods - self.assertFalse(pp.isrecursive(safe), - "expected not isrecursive for %r" % (safe,)) - self.assertTrue(pp.isreadable(safe), - "expected isreadable for %r" % (safe,)) - - def test_unreadable(self) -> None: - # Not recursive but not readable anyway - pp = pprint.PrettyPrinter() - for unreadable in type(3), pprint, pprint.isrecursive: - # module-level convenience functions - self.assertFalse(pprint.isrecursive(unreadable), - "expected not isrecursive for %r" % (unreadable,)) - self.assertFalse(pprint.isreadable(unreadable), - "expected not isreadable for %r" % (unreadable,)) - # PrettyPrinter methods - self.assertFalse(pp.isrecursive(unreadable), - "expected not isrecursive for %r" % (unreadable,)) - self.assertFalse(pp.isreadable(unreadable), - "expected not isreadable for %r" % (unreadable,)) - - def test_same_as_repr(self) -> None: - # Simple objects, small containers and classes that overwrite __repr__ - # For those the result should be the same as repr(). - # Ahem. The docs don't say anything about that -- this appears to - # be testing an implementation quirk. Starting in Python 2.5, it's - # not true for dicts: pprint always sorts dicts by key now; before, - # it sorted a dict display if and only if the display required - # multiple lines. For that reason, dicts with more than one element - # aren't tested here. - for simple in (0, 0, complex(0.0), 0.0, "", b"", - (), tuple2(), tuple3(), - [], list2(), list3(), - {}, dict2(), dict3(), - self.assertTrue, pprint, - -6, -6, complex(-6.,-6.), -1.5, "x", b"x", (3,), [3], {3: 6}, - (1,2), [3,4], {5: 6}, - tuple2((1,2)), tuple3((1,2)), tuple3(range(100)), - [3,4], list2(Any([3,4])), list3(Any([3,4])), list3(Any(range(100))), - dict2(Any({5: 6})), dict3(Any({5: 6})), # JLe: work around mypy issue #233 - range(10, -11, -1) - ): - native = repr(simple) - for function in "pformat", "saferepr": - f = getattr(pprint, function) - got = f(simple) - self.assertEqual(native, got, - "expected %s got %s from pprint.%s" % - (native, got, function)) - - def test_basic_line_wrap(self) -> None: - # verify basic line-wrapping operation - o = {'RPM_cal': 0, - 'RPM_cal2': 48059, - 'Speed_cal': 0, - 'controldesk_runtime_us': 0, - 'main_code_runtime_us': 0, - 'read_io_runtime_us': 0, - 'write_io_runtime_us': 43690} - exp = """\ -{'RPM_cal': 0, - 'RPM_cal2': 48059, - 'Speed_cal': 0, - 'controldesk_runtime_us': 0, - 'main_code_runtime_us': 0, - 'read_io_runtime_us': 0, - 'write_io_runtime_us': 43690}""" - # JLe: work around mypy issue #232 - for type in List[Any]([dict, dict2]): - self.assertEqual(pprint.pformat(type(o)), exp) - - o2 = range(100) - exp = '[%s]' % ',\n '.join(map(str, o2)) - for type in List[Any]([list, list2]): - self.assertEqual(pprint.pformat(type(o2)), exp) - - o3 = tuple(range(100)) - exp = '(%s)' % ',\n '.join(map(str, o3)) - for type in List[Any]([tuple, tuple2]): - self.assertEqual(pprint.pformat(type(o3)), exp) - - # indent parameter - o4 = range(100) - exp = '[ %s]' % ',\n '.join(map(str, o4)) - for type in List[Any]([list, list2]): - self.assertEqual(pprint.pformat(type(o4), indent=4), exp) - - def test_nested_indentations(self) -> None: - o1 = list(range(10)) - o2 = {'first':1, 'second':2, 'third':3} - o = [o1, o2] - expected = """\ -[ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - { 'first': 1, - 'second': 2, - 'third': 3}]""" - self.assertEqual(pprint.pformat(o, indent=4, width=42), expected) - - def test_sorted_dict(self) -> None: - # Starting in Python 2.5, pprint sorts dict displays by key regardless - # of how small the dictionary may be. - # Before the change, on 32-bit Windows pformat() gave order - # 'a', 'c', 'b' here, so this test failed. - d = {'a': 1, 'b': 1, 'c': 1} - self.assertEqual(pprint.pformat(d), "{'a': 1, 'b': 1, 'c': 1}") - self.assertEqual(pprint.pformat([d, d]), - "[{'a': 1, 'b': 1, 'c': 1}, {'a': 1, 'b': 1, 'c': 1}]") - - # The next one is kind of goofy. The sorted order depends on the - # alphabetic order of type names: "int" < "str" < "tuple". Before - # Python 2.5, this was in the test_same_as_repr() test. It's worth - # keeping around for now because it's one of few tests of pprint - # against a crazy mix of types. - self.assertEqual(pprint.pformat({"xy\tab\n": (3,), 5: [[]], (): {}}), - r"{5: [[]], 'xy\tab\n': (3,), (): {}}") - - def test_ordered_dict(self) -> None: - words = 'the quick brown fox jumped over a lazy dog'.split() - d = collections.OrderedDict(zip(words, itertools.count())) - self.assertEqual(pprint.pformat(d), -"""\ -{'the': 0, - 'quick': 1, - 'brown': 2, - 'fox': 3, - 'jumped': 4, - 'over': 5, - 'a': 6, - 'lazy': 7, - 'dog': 8}""") - def test_subclassing(self) -> None: - o = {'names with spaces': 'should be presented using repr()', - 'others.should.not.be': 'like.this'} - exp = """\ -{'names with spaces': 'should be presented using repr()', - others.should.not.be: like.this}""" - self.assertEqual(DottedPrettyPrinter().pformat(o), exp) - - @test.support.cpython_only - def test_set_reprs(self) -> None: - # This test creates a complex arrangement of frozensets and - # compares the pretty-printed repr against a string hard-coded in - # the test. The hard-coded repr depends on the sort order of - # frozensets. - # - # However, as the docs point out: "Since sets only define - # partial ordering (subset relationships), the output of the - # list.sort() method is undefined for lists of sets." - # - # In a nutshell, the test assumes frozenset({0}) will always - # sort before frozenset({1}), but: - # - # >>> frozenset({0}) < frozenset({1}) - # False - # >>> frozenset({1}) < frozenset({0}) - # False - # - # Consequently, this test is fragile and - # implementation-dependent. Small changes to Python's sort - # algorithm cause the test to fail when it should pass. - - self.assertEqual(pprint.pformat(set()), 'set()') - self.assertEqual(pprint.pformat(set(range(3))), '{0, 1, 2}') - self.assertEqual(pprint.pformat(frozenset()), 'frozenset()') - self.assertEqual(pprint.pformat(frozenset(range(3))), 'frozenset({0, 1, 2})') - cube_repr_tgt = """\ -{frozenset(): frozenset({frozenset({2}), frozenset({0}), frozenset({1})}), - frozenset({0}): frozenset({frozenset(), - frozenset({0, 2}), - frozenset({0, 1})}), - frozenset({1}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 1})}), - frozenset({2}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 2})}), - frozenset({1, 2}): frozenset({frozenset({2}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 2}): frozenset({frozenset({2}), - frozenset({0}), - frozenset({0, 1, 2})}), - frozenset({0, 1}): frozenset({frozenset({0}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 1, 2}): frozenset({frozenset({1, 2}), - frozenset({0, 2}), - frozenset({0, 1})})}""" - cube = test.test_set.cube(3) - self.assertEqual(pprint.pformat(cube), cube_repr_tgt) - cubo_repr_tgt = """\ -{frozenset({frozenset({0, 2}), frozenset({0})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1}), frozenset({1})}): frozenset({frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})})}), - frozenset({frozenset({1, 2}), frozenset({1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({1, 2}), frozenset({2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({0})}): frozenset({frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset()}): frozenset({frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({0}), frozenset({0, 1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({1, 2}), frozenset({0, 1, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})})})}""" - - cubo = test.test_set.linegraph(cube) - self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt) - - def test_depth(self) -> None: - nested_tuple = (1, (2, (3, (4, (5, 6))))) - nested_dict = {1: {2: {3: {4: {5: {6: 6}}}}}} - nested_list = [1, [2, [3, [4, [5, [6, []]]]]]] - self.assertEqual(pprint.pformat(nested_tuple), repr(nested_tuple)) - self.assertEqual(pprint.pformat(nested_dict), repr(nested_dict)) - self.assertEqual(pprint.pformat(nested_list), repr(nested_list)) - - lv1_tuple = '(1, (...))' - lv1_dict = '{1: {...}}' - lv1_list = '[1, [...]]' - self.assertEqual(pprint.pformat(nested_tuple, depth=1), lv1_tuple) - self.assertEqual(pprint.pformat(nested_dict, depth=1), lv1_dict) - self.assertEqual(pprint.pformat(nested_list, depth=1), lv1_list) - - def test_sort_unorderable_values(self) -> None: - # Issue 3976: sorted pprints fail for unorderable values. - n = 20 - keys = [Unorderable() for i in range(n)] - random.shuffle(keys) - skeys = sorted(keys, key=id) - clean = Undefined(Function[[str], str]) - clean = lambda s: s.replace(' ', '').replace('\n','') - - self.assertEqual(clean(pprint.pformat(set(keys))), - '{' + ','.join(map(repr, skeys)) + '}') - self.assertEqual(clean(pprint.pformat(frozenset(keys))), - 'frozenset({' + ','.join(map(repr, skeys)) + '})') - self.assertEqual(clean(pprint.pformat(dict.fromkeys(keys))), - '{' + ','.join('%r:None' % k for k in skeys) + '}') - -class DottedPrettyPrinter(pprint.PrettyPrinter): - - def format(self, object: object, context: Dict[int, Any], maxlevels: int, - level: int) -> Tuple[str, int, int]: - if isinstance(object, str): - if ' ' in object: - return repr(object), 1, 0 - else: - return object, 0, 0 - else: - return pprint.PrettyPrinter.format( - self, object, context, maxlevels, level) - - -def test_main() -> None: - test.support.run_unittest(QueryTestCase) - - -if __name__ == "__main__": - test_main() diff --git a/lib-python/3.2/test/test_random.py b/lib-python/3.2/test/test_random.py deleted file mode 100644 index d2ed7d6eb586..000000000000 --- a/lib-python/3.2/test/test_random.py +++ /dev/null @@ -1,533 +0,0 @@ -#!/usr/bin/env python3 - -import unittest -import random -import time -import pickle -import warnings -from math import log, exp, pi, fsum, sin -from test import support - -from typing import Undefined, Any, Dict, List, Function, Generic, typevar - -RT = typevar('RT', values=(random.Random, random.SystemRandom)) - -class TestBasicOps(unittest.TestCase, Generic[RT]): - # Superclass with tests common to all generators. - # Subclasses must arrange for self.gen to retrieve the Random instance - # to be tested. - - gen = Undefined(RT) # Either Random or SystemRandom - - def randomlist(self, n: int) -> List[float]: - """Helper function to make a list of random numbers""" - return [self.gen.random() for i in range(n)] - - def test_autoseed(self) -> None: - self.gen.seed() - state1 = self.gen.getstate() - time.sleep(0.1) - self.gen.seed() # diffent seeds at different times - state2 = self.gen.getstate() - self.assertNotEqual(state1, state2) - - def test_saverestore(self) -> None: - N = 1000 - self.gen.seed() - state = self.gen.getstate() - randseq = self.randomlist(N) - self.gen.setstate(state) # should regenerate the same sequence - self.assertEqual(randseq, self.randomlist(N)) - - def test_seedargs(self) -> None: - for arg in [None, 0, 0, 1, 1, -1, -1, 10**20, -(10**20), - 3.14, complex(1., 2.), 'a', tuple('abc')]: - self.gen.seed(arg) - for arg in [list(range(3)), {'one': 1}]: - self.assertRaises(TypeError, self.gen.seed, arg) - self.assertRaises(TypeError, self.gen.seed, 1, 2, 3, 4) - self.assertRaises(TypeError, type(self.gen), []) - - def test_choice(self) -> None: - choice = self.gen.choice - with self.assertRaises(IndexError): - choice([]) - self.assertEqual(choice([50]), 50) - self.assertIn(choice([25, 75]), [25, 75]) - - def test_sample(self) -> None: - # For the entire allowable range of 0 <= k <= N, validate that - # the sample is of the correct length and contains only unique items - N = 100 - population = range(N) - for k in range(N+1): - s = self.gen.sample(population, k) - self.assertEqual(len(s), k) - uniq = set(s) - self.assertEqual(len(uniq), k) - self.assertTrue(uniq <= set(population)) - self.assertEqual(self.gen.sample([], 0), []) # test edge case N==k==0 - - def test_sample_distribution(self) -> None: - # For the entire allowable range of 0 <= k <= N, validate that - # sample generates all possible permutations - n = 5 - pop = range(n) - trials = 10000 # large num prevents false negatives without slowing normal case - def factorial(n: int) -> int: - if n == 0: - return 1 - return n * factorial(n - 1) - for k in range(n): - expected = factorial(n) // factorial(n-k) - perms = Dict[tuple, object]() - for i in range(trials): - perms[tuple(self.gen.sample(pop, k))] = None - if len(perms) == expected: - break - else: - self.fail() - - def test_sample_inputs(self) -> None: - # SF bug #801342 -- population can be any iterable defining __len__() - self.gen.sample(set(range(20)), 2) - self.gen.sample(range(20), 2) - self.gen.sample(range(20), 2) - self.gen.sample(str('abcdefghijklmnopqrst'), 2) - self.gen.sample(tuple('abcdefghijklmnopqrst'), 2) - - def test_sample_on_dicts(self) -> None: - self.assertRaises(TypeError, self.gen.sample, dict.fromkeys('abcdef'), 2) - - def test_gauss(self) -> None: - # Ensure that the seed() method initializes all the hidden state. In - # particular, through 2.2.1 it failed to reset a piece of state used - # by (and only by) the .gauss() method. - - for seed in 1, 12, 123, 1234, 12345, 123456, 654321: - self.gen.seed(seed) - x1 = self.gen.random() - y1 = self.gen.gauss(0, 1) - - self.gen.seed(seed) - x2 = self.gen.random() - y2 = self.gen.gauss(0, 1) - - self.assertEqual(x1, x2) - self.assertEqual(y1, y2) - - def test_pickling(self) -> None: - state = pickle.dumps(self.gen) - origseq = [self.gen.random() for i in range(10)] - newgen = pickle.loads(state) - restoredseq = [newgen.random() for i in range(10)] - self.assertEqual(origseq, restoredseq) - - def test_bug_1727780(self) -> None: - # verify that version-2-pickles can be loaded - # fine, whether they are created on 32-bit or 64-bit - # platforms, and that version-3-pickles load fine. - files = [("randv2_32.pck", 780), - ("randv2_64.pck", 866), - ("randv3.pck", 343)] - for file, value in files: - f = open(support.findfile(file),"rb") - r = pickle.load(f) - f.close() - self.assertEqual(int(r.random()*1000), value) - - def test_bug_9025(self) -> None: - # Had problem with an uneven distribution in int(n*random()) - # Verify the fix by checking that distributions fall within expectations. - n = 100000 - randrange = self.gen.randrange - k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) - self.assertTrue(0.30 < k/n and k/n < .37, (k/n)) - -class SystemRandom_TestBasicOps(TestBasicOps[random.SystemRandom]): - gen = random.SystemRandom() - - def test_autoseed(self) -> None: - # Doesn't need to do anything except not fail - self.gen.seed() - - def test_saverestore(self) -> None: - self.assertRaises(NotImplementedError, self.gen.getstate) - self.assertRaises(NotImplementedError, self.gen.setstate, None) - - def test_seedargs(self) -> None: - # Doesn't need to do anything except not fail - self.gen.seed(100) - - def test_gauss(self) -> None: - self.gen.gauss_next = None - self.gen.seed(100) - self.assertEqual(self.gen.gauss_next, None) - - def test_pickling(self) -> None: - self.assertRaises(NotImplementedError, pickle.dumps, self.gen) - - def test_53_bits_per_float(self) -> None: - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 # type: int - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) - - def test_bigrand(self) -> None: - # The randrange routine should build-up the required number of bits - # in stages so that all bit positions are active. - span = 2 ** 500 # type: int - cum = 0 - for i in range(100): - r = self.gen.randrange(span) - self.assertTrue(0 <= r < span) - cum |= r - self.assertEqual(cum, span-1) - - def test_bigrand_ranges(self) -> None: - for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: - start = self.gen.randrange(2 ** i) - stop = self.gen.randrange(2 ** (i-2)) - if stop <= start: - return - self.assertTrue(start <= self.gen.randrange(start, stop) < stop) - - def test_rangelimits(self) -> None: - for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: - self.assertEqual(set(range(start,stop)), - set([self.gen.randrange(start,stop) for i in range(100)])) - - def test_genrandbits(self) -> None: - # Verify ranges - for k in range(1, 1000): - self.assertTrue(0 <= self.gen.getrandbits(k) < 2**k) - - # Verify all bits active - getbits = self.gen.getrandbits - for span in [1, 2, 3, 4, 31, 32, 32, 52, 53, 54, 119, 127, 128, 129]: - cum = 0 - for i in range(100): - cum |= getbits(span) - self.assertEqual(cum, 2**span-1) - - # Verify argument checking - self.assertRaises(TypeError, self.gen.getrandbits) - self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) - self.assertRaises(ValueError, self.gen.getrandbits, 0) - self.assertRaises(ValueError, self.gen.getrandbits, -1) - self.assertRaises(TypeError, self.gen.getrandbits, 10.1) - - def test_randbelow_logic(self, _log: Function[[float, float], float] = log, - int: Function[[float], int] = int) -> None: - # check bitcount transition points: 2**i and 2**(i+1)-1 - # show that: k = int(1.001 + _log(n, 2)) - # is equal to or one greater than the number of bits in n - for i in range(1, 1000): - n = 1 << i # check an exact power of two - numbits = i+1 - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) - self.assertEqual(n, 2**(k-1)) - - n += n - 1 # check 1 below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertIn(k, [numbits, numbits+1]) - self.assertTrue(2**k > n > 2**(k-2)) - - n -= n >> 15 # check a little farther below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) # note the stronger assertion - self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion - - -class MersenneTwister_TestBasicOps(TestBasicOps[random.Random]): - gen = random.Random() - - def test_guaranteed_stable(self) -> None: - # These sequences are guaranteed to stay the same across versions of python - self.gen.seed(3456147, version=1) - self.assertEqual([self.gen.random().hex() for i in range(4)], - ['0x1.ac362300d90d2p-1', '0x1.9d16f74365005p-1', - '0x1.1ebb4352e4c4dp-1', '0x1.1a7422abf9c11p-1']) - self.gen.seed("the quick brown fox", version=2) - self.assertEqual([self.gen.random().hex() for i in range(4)], - ['0x1.1239ddfb11b7cp-3', '0x1.b3cbb5c51b120p-4', - '0x1.8c4f55116b60fp-1', '0x1.63eb525174a27p-1']) - - def test_setstate_first_arg(self) -> None: - self.assertRaises(ValueError, self.gen.setstate, (1, None, None)) - - def test_setstate_middle_arg(self) -> None: - # Wrong type, s/b tuple - self.assertRaises(TypeError, self.gen.setstate, (2, None, None)) - # Wrong length, s/b 625 - self.assertRaises(ValueError, self.gen.setstate, (2, (1,2,3), None)) - # Wrong type, s/b tuple of 625 ints - self.assertRaises(TypeError, self.gen.setstate, (2, tuple(['a',]*625), None)) - # Last element s/b an int also - self.assertRaises(TypeError, self.gen.setstate, (2, Any((0,))*624+('a',), None)) - - def test_referenceImplementation(self) -> None: - # Compare the python implementation with results from the original - # code. Create 2000 53-bit precision random floats. Compare only - # the last ten entries to show that the independent implementations - # are tracking. Here is the main() function needed to create the - # list of expected random numbers: - # void main(void){ - # int i; - # unsigned long init[4]={61731, 24903, 614, 42143}, length=4; - # init_by_array(init, length); - # for (i=0; i<2000; i++) { - # printf("%.15f ", genrand_res53()); - # if (i%5==4) printf("\n"); - # } - # } - expected = [0.45839803073713259, - 0.86057815201978782, - 0.92848331726782152, - 0.35932681119782461, - 0.081823493762449573, - 0.14332226470169329, - 0.084297823823520024, - 0.53814864671831453, - 0.089215024911993401, - 0.78486196105372907] - - self.gen.seed(61731 + (24903<<32) + (614<<64) + (42143<<96)) - actual = self.randomlist(2000)[-10:] - for a, e in zip(actual, expected): - self.assertAlmostEqual(a,e,places=14) - - def test_strong_reference_implementation(self) -> None: - # Like test_referenceImplementation, but checks for exact bit-level - # equality. This should pass on any box where C double contains - # at least 53 bits of precision (the underlying algorithm suffers - # no rounding errors -- all results are exact). - from math import ldexp - - expected = [0x0eab3258d2231f, - 0x1b89db315277a5, - 0x1db622a5518016, - 0x0b7f9af0d575bf, - 0x029e4c4db82240, - 0x04961892f5d673, - 0x02b291598e4589, - 0x11388382c15694, - 0x02dad977c9e1fe, - 0x191d96d4d334c6] - self.gen.seed(61731 + (24903<<32) + (614<<64) + (42143<<96)) - actual = self.randomlist(2000)[-10:] - for a, e in zip(actual, expected): - self.assertEqual(int(ldexp(a, 53)), e) - - def test_long_seed(self) -> None: - # This is most interesting to run in debug mode, just to make sure - # nothing blows up. Under the covers, a dynamically resized array - # is allocated, consuming space proportional to the number of bits - # in the seed. Unfortunately, that's a quadratic-time algorithm, - # so don't make this horribly big. - seed = (1 << (10000 * 8)) - 1 # about 10K bytes - self.gen.seed(seed) - - def test_53_bits_per_float(self) -> None: - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 # type: int - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) - - def test_bigrand(self) -> None: - # The randrange routine should build-up the required number of bits - # in stages so that all bit positions are active. - span = 2 ** 500 # type: int - cum = 0 - for i in range(100): - r = self.gen.randrange(span) - self.assertTrue(0 <= r < span) - cum |= r - self.assertEqual(cum, span-1) - - def test_bigrand_ranges(self) -> None: - for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: - start = self.gen.randrange(2 ** i) - stop = self.gen.randrange(2 ** (i-2)) - if stop <= start: - return - self.assertTrue(start <= self.gen.randrange(start, stop) < stop) - - def test_rangelimits(self) -> None: - for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: - self.assertEqual(set(range(start,stop)), - set([self.gen.randrange(start,stop) for i in range(100)])) - - def test_genrandbits(self) -> None: - # Verify cross-platform repeatability - self.gen.seed(1234567) - self.assertEqual(self.gen.getrandbits(100), - 97904845777343510404718956115) - # Verify ranges - for k in range(1, 1000): - self.assertTrue(0 <= self.gen.getrandbits(k) < 2**k) - - # Verify all bits active - getbits = self.gen.getrandbits - for span in [1, 2, 3, 4, 31, 32, 32, 52, 53, 54, 119, 127, 128, 129]: - cum = 0 - for i in range(100): - cum |= getbits(span) - self.assertEqual(cum, 2**span-1) - - # Verify argument checking - self.assertRaises(TypeError, self.gen.getrandbits) - self.assertRaises(TypeError, self.gen.getrandbits, 'a') - self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) - self.assertRaises(ValueError, self.gen.getrandbits, 0) - self.assertRaises(ValueError, self.gen.getrandbits, -1) - - def test_randbelow_logic(self, - _log: Function[[int, float], float] = log, - int: Function[[float], int] = int) -> None: - # check bitcount transition points: 2**i and 2**(i+1)-1 - # show that: k = int(1.001 + _log(n, 2)) - # is equal to or one greater than the number of bits in n - for i in range(1, 1000): - n = 1 << i # check an exact power of two - numbits = i+1 - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) - self.assertEqual(n, 2**(k-1)) - - n += n - 1 # check 1 below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertIn(k, [numbits, numbits+1]) - self.assertTrue(2**k > n > 2**(k-2)) - - n -= n >> 15 # check a little farther below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) # note the stronger assertion - self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion - - def test_randrange_bug_1590891(self) -> None: - start = 1000000000000 - stop = -100000000000000000000 - step = -200 - x = self.gen.randrange(start, stop, step) - self.assertTrue(stop < x <= start) - self.assertEqual((x+stop)%step, 0) - -def gamma(z: float, sqrt2pi: float = (2.0*pi)**0.5) -> float: - # Reflection to right half of complex plane - if z < 0.5: - return pi / sin(pi*z) / gamma(1.0-z) - # Lanczos approximation with g=7 - az = z + (7.0 - 0.5) - return az ** (z-0.5) / exp(az) * sqrt2pi * fsum([ - 0.9999999999995183, - 676.5203681218835 / z, - -1259.139216722289 / (z+1.0), - 771.3234287757674 / (z+2.0), - -176.6150291498386 / (z+3.0), - 12.50734324009056 / (z+4.0), - -0.1385710331296526 / (z+5.0), - 0.9934937113930748e-05 / (z+6.0), - 0.1659470187408462e-06 / (z+7.0), - ]) - -class TestDistributions(unittest.TestCase): - def test_zeroinputs(self) -> None: - # Verify that distributions can handle a series of zero inputs' - g = random.Random() - x = [g.random() for i in range(50)] + [0.0]*5 - def patch() -> None: - setattr(g, 'random', x[:].pop) - patch(); g.uniform(1.0,10.0) - patch(); g.paretovariate(1.0) - patch(); g.expovariate(1.0) - patch(); g.weibullvariate(1.0, 1.0) - patch(); g.normalvariate(0.0, 1.0) - patch(); g.gauss(0.0, 1.0) - patch(); g.lognormvariate(0.0, 1.0) - patch(); g.vonmisesvariate(0.0, 1.0) - patch(); g.gammavariate(0.01, 1.0) - patch(); g.gammavariate(1.0, 1.0) - patch(); g.gammavariate(200.0, 1.0) - patch(); g.betavariate(3.0, 3.0) - patch(); g.triangular(0.0, 1.0, 1.0/3.0) - - def test_avg_std(self) -> None: - # Use integration to test distribution average and standard deviation. - # Only works for distributions which do not consume variates in pairs - g = random.Random() - N = 5000 - x = [i/float(N) for i in range(1,N)] - variate = Undefined(Any) - for variate, args, mu, sigmasqrd in [ - (g.uniform, (1.0,10.0), (10.0+1.0)/2, (10.0-1.0)**2/12), - (g.triangular, (0.0, 1.0, 1.0/3.0), 4.0/9.0, 7.0/9.0/18.0), - (g.expovariate, (1.5,), 1/1.5, 1/1.5**2), - (g.paretovariate, (5.0,), 5.0/(5.0-1), - 5.0/((5.0-1)**2*(5.0-2))), - (g.weibullvariate, (1.0, 3.0), gamma(1+1/3.0), - gamma(1+2/3.0)-gamma(1+1/3.0)**2) ]: - setattr(g, 'random', x[:].pop) - y = List[float]() - for i in range(len(x)): - try: - y.append(variate(*args)) - except IndexError: - pass - s1 = s2 = 0.0 - for e in y: - s1 += e - s2 += (e - mu) ** 2 - N = len(y) - self.assertAlmostEqual(s1/N, mu, places=2) - self.assertAlmostEqual(s2/(N-1), sigmasqrd, places=2) - -class TestModule(unittest.TestCase): - def testMagicConstants(self) -> None: - self.assertAlmostEqual(random.NV_MAGICCONST, 1.71552776992141) - self.assertAlmostEqual(random.TWOPI, 6.28318530718) - self.assertAlmostEqual(random.LOG4, 1.38629436111989) - self.assertAlmostEqual(random.SG_MAGICCONST, 2.50407739677627) - - def test__all__(self) -> None: - # tests validity but not completeness of the __all__ list - self.assertTrue(set(random.__all__) <= set(dir(random))) - - def test_random_subclass_with_kwargs(self) -> None: - # SF bug #1486663 -- this used to erroneously raise a TypeError - class Subclass(random.Random): - def __init__(self, newarg: object = None) -> None: - random.Random.__init__(self) - Subclass(newarg=1) - - -def test_main(verbose: bool = None) -> None: - testclasses = [MersenneTwister_TestBasicOps, - TestDistributions, - TestModule] - - try: - random.SystemRandom().random() - except NotImplementedError: - pass - else: - testclasses.append(SystemRandom_TestBasicOps) - - support.run_unittest(*testclasses) - - # verify reference counting - import sys - if verbose and hasattr(sys, "gettotalrefcount"): - counts = [None] * 5 # type: List[int] - for i in range(len(counts)): - support.run_unittest(*testclasses) - counts[i] = sys.gettotalrefcount() - print(counts) - -if __name__ == "__main__": - test_main(verbose=True) diff --git a/lib-python/3.2/test/test_set.py b/lib-python/3.2/test/test_set.py deleted file mode 100644 index d207a7f2e924..000000000000 --- a/lib-python/3.2/test/test_set.py +++ /dev/null @@ -1,1884 +0,0 @@ -import unittest -from test import support -import gc -import weakref -import operator -import copy -import pickle -from random import randrange, shuffle -import sys -import warnings -import collections -from typing import Set, Any, Undefined - -class PassThru(Exception): - pass - -def check_pass_thru(): - raise PassThru - yield 1 - -class BadCmp: - def __hash__(self): - return 1 - def __eq__(self, other): - raise RuntimeError - -class ReprWrapper: - 'Used to test self-referential repr() calls' - def __repr__(self): - return repr(self.value) - -#class HashCountingInt(int): -# 'int-like object that counts the number of times __hash__ is called' -# def __init__(self, *args): -# self.hash_count = 0 -# def __hash__(self): -# self.hash_count += 1 -# return int.__hash__(self) - -class TestJointOps(unittest.TestCase): - # Tests common to both set and frozenset - - def setUp(self): - self.word = word = 'simsalabim' - self.otherword = 'madagascar' - self.letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - self.s = self.thetype(word) - self.d = dict.fromkeys(word) - - def test_new_or_init(self): - self.assertRaises(TypeError, self.thetype, [], 2) - self.assertRaises(TypeError, set().__init__, a=1) - - def test_uniquification(self): - actual = sorted(self.s) - expected = sorted(self.d) - self.assertEqual(actual, expected) - self.assertRaises(PassThru, self.thetype, check_pass_thru()) - self.assertRaises(TypeError, self.thetype, [[]]) - - def test_len(self): - self.assertEqual(len(self.s), len(self.d)) - - def test_contains(self): - for c in self.letters: - self.assertEqual(c in self.s, c in self.d) - self.assertRaises(TypeError, self.s.__contains__, [[]]) - s = self.thetype([frozenset(self.letters)]) - self.assertIn(self.thetype(self.letters), s) - - def test_union(self): - u = self.s.union(self.otherword) - for c in self.letters: - self.assertEqual(c in u, c in self.d or c in self.otherword) - self.assertEqual(self.s, self.thetype(self.word)) - self.assertEqual(type(u), self.basetype) - self.assertRaises(PassThru, self.s.union, check_pass_thru()) - self.assertRaises(TypeError, self.s.union, [[]]) - for C in set, frozenset, dict.fromkeys, str, list, tuple: - self.assertEqual(self.thetype('abcba').union(C('cdc')), set('abcd')) - self.assertEqual(self.thetype('abcba').union(C('efgfe')), set('abcefg')) - self.assertEqual(self.thetype('abcba').union(C('ccb')), set('abc')) - self.assertEqual(self.thetype('abcba').union(C('ef')), set('abcef')) - self.assertEqual(self.thetype('abcba').union(C('ef'), C('fg')), set('abcefg')) - - # Issue #6573 - x = self.thetype() - self.assertEqual(x.union(set([1]), x, set([2])), self.thetype([1, 2])) - - def test_or(self): - i = self.s.union(self.otherword) - self.assertEqual(self.s | set(self.otherword), i) - self.assertEqual(self.s | frozenset(self.otherword), i) - try: - self.s | self.otherword - except TypeError: - pass - else: - self.fail("s|t did not screen-out general iterables") - - def test_intersection(self): - i = self.s.intersection(self.otherword) - for c in self.letters: - self.assertEqual(c in i, c in self.d and c in self.otherword) - self.assertEqual(self.s, self.thetype(self.word)) - self.assertEqual(type(i), self.basetype) - self.assertRaises(PassThru, self.s.intersection, check_pass_thru()) - for C in set, frozenset, dict.fromkeys, str, list, tuple: - self.assertEqual(self.thetype('abcba').intersection(C('cdc')), set('cc')) - self.assertEqual(self.thetype('abcba').intersection(C('efgfe')), set('')) - self.assertEqual(self.thetype('abcba').intersection(C('ccb')), set('bc')) - self.assertEqual(self.thetype('abcba').intersection(C('ef')), set('')) - self.assertEqual(self.thetype('abcba').intersection(C('cbcf'), C('bag')), set('b')) - s = self.thetype('abcba') - z = s.intersection() - if self.thetype == frozenset(): - self.assertEqual(id(s), id(z)) - else: - self.assertNotEqual(id(s), id(z)) - - def test_isdisjoint(self): - def f(s1, s2): - 'Pure python equivalent of isdisjoint()' - return not set(s1).intersection(s2) - for larg in '', 'a', 'ab', 'abc', 'ababac', 'cdc', 'cc', 'efgfe', 'ccb', 'ef': - s1 = self.thetype(larg) - for rarg in '', 'a', 'ab', 'abc', 'ababac', 'cdc', 'cc', 'efgfe', 'ccb', 'ef': - for C in set, frozenset, dict.fromkeys, str, list, tuple: - s2 = C(rarg) - actual = s1.isdisjoint(s2) - expected = f(s1, s2) - self.assertEqual(actual, expected) - self.assertTrue(actual is True or actual is False) - - def test_and(self): - i = self.s.intersection(self.otherword) - self.assertEqual(self.s & set(self.otherword), i) - self.assertEqual(self.s & frozenset(self.otherword), i) - try: - self.s & self.otherword - except TypeError: - pass - else: - self.fail("s&t did not screen-out general iterables") - - def test_difference(self): - i = self.s.difference(self.otherword) - for c in self.letters: - self.assertEqual(c in i, c in self.d and c not in self.otherword) - self.assertEqual(self.s, self.thetype(self.word)) - self.assertEqual(type(i), self.basetype) - self.assertRaises(PassThru, self.s.difference, check_pass_thru()) - self.assertRaises(TypeError, self.s.difference, [[]]) - for C in set, frozenset, dict.fromkeys, str, list, tuple: - self.assertEqual(self.thetype('abcba').difference(C('cdc')), set('ab')) - self.assertEqual(self.thetype('abcba').difference(C('efgfe')), set('abc')) - self.assertEqual(self.thetype('abcba').difference(C('ccb')), set('a')) - self.assertEqual(self.thetype('abcba').difference(C('ef')), set('abc')) - self.assertEqual(self.thetype('abcba').difference(), set('abc')) - self.assertEqual(self.thetype('abcba').difference(C('a'), C('b')), set('c')) - - def test_sub(self): - i = self.s.difference(self.otherword) - self.assertEqual(self.s - set(self.otherword), i) - self.assertEqual(self.s - frozenset(self.otherword), i) - try: - self.s - self.otherword - except TypeError: - pass - else: - self.fail("s-t did not screen-out general iterables") - - def test_symmetric_difference(self): - i = self.s.symmetric_difference(self.otherword) - for c in self.letters: - self.assertEqual(c in i, (c in self.d) ^ (c in self.otherword)) - self.assertEqual(self.s, self.thetype(self.word)) - self.assertEqual(type(i), self.basetype) - self.assertRaises(PassThru, self.s.symmetric_difference, check_pass_thru()) - self.assertRaises(TypeError, self.s.symmetric_difference, [[]]) - for C in set, frozenset, dict.fromkeys, str, list, tuple: - self.assertEqual(self.thetype('abcba').symmetric_difference(C('cdc')), set('abd')) - self.assertEqual(self.thetype('abcba').symmetric_difference(C('efgfe')), set('abcefg')) - self.assertEqual(self.thetype('abcba').symmetric_difference(C('ccb')), set('a')) - self.assertEqual(self.thetype('abcba').symmetric_difference(C('ef')), set('abcef')) - - def test_xor(self): - i = self.s.symmetric_difference(self.otherword) - self.assertEqual(self.s ^ set(self.otherword), i) - self.assertEqual(self.s ^ frozenset(self.otherword), i) - try: - self.s ^ self.otherword - except TypeError: - pass - else: - self.fail("s^t did not screen-out general iterables") - - def test_equality(self): - self.assertEqual(self.s, set(self.word)) - self.assertEqual(self.s, frozenset(self.word)) - self.assertEqual(self.s == self.word, False) - self.assertNotEqual(self.s, set(self.otherword)) - self.assertNotEqual(self.s, frozenset(self.otherword)) - self.assertEqual(self.s != self.word, True) - - def test_setOfFrozensets(self): - t = map(frozenset, ['abcdef', 'bcd', 'bdcb', 'fed', 'fedccba']) - s = self.thetype(t) - self.assertEqual(len(s), 3) - - def test_sub_and_super(self): - p, q, r = map(self.thetype, ['ab', 'abcde', 'def']) - self.assertTrue(p < q) - self.assertTrue(p <= q) - self.assertTrue(q <= q) - self.assertTrue(q > p) - self.assertTrue(q >= p) - self.assertFalse(q < r) - self.assertFalse(q <= r) - self.assertFalse(q > r) - self.assertFalse(q >= r) - self.assertTrue(set('a').issubset('abc')) - self.assertTrue(set('abc').issuperset('a')) - self.assertFalse(set('a').issubset('cbs')) - self.assertFalse(set('cbs').issuperset('a')) - - def test_pickling(self): - for i in range(pickle.HIGHEST_PROTOCOL + 1): - p = pickle.dumps(self.s, i) - dup = pickle.loads(p) - self.assertEqual(self.s, dup, "%s != %s" % (self.s, dup)) - if type(self.s) not in (set, frozenset): - self.s.x = 10 - p = pickle.dumps(self.s) - dup = pickle.loads(p) - self.assertEqual(self.s.x, dup.x) - - def test_deepcopy(self): - class Tracer: - def __init__(self, value): - self.value = value - def __hash__(self): - return self.value - def __deepcopy__(self, memo=None): - return Tracer(self.value + 1) - t = Tracer(10) - s = self.thetype([t]) - dup = copy.deepcopy(s) - self.assertNotEqual(id(s), id(dup)) - for elem in dup: - newt = elem - self.assertNotEqual(id(t), id(newt)) - self.assertEqual(t.value + 1, newt.value) - - def test_gc(self): - # Create a nest of cycles to exercise overall ref count check - class A: - pass - s = set(A() for i in range(1000)) - for elem in s: - elem.cycle = s - elem.sub = elem - elem.set = set([elem]) - - def test_subclass_with_custom_hash(self): - raise NotImplementedError() # runtime computed base class below - # Bug #1257731 - class H: # (self.thetype): - def __hash__(self): - return int(id(self) & 0x7fffffff) - s=H() - f=set() - f.add(s) - self.assertIn(s, f) - f.remove(s) - f.add(s) - f.discard(s) - - def test_badcmp(self): - s = self.thetype([BadCmp()]) - # Detect comparison errors during insertion and lookup - self.assertRaises(RuntimeError, self.thetype, [BadCmp(), BadCmp()]) - self.assertRaises(RuntimeError, s.__contains__, BadCmp()) - # Detect errors during mutating operations - if hasattr(s, 'add'): - self.assertRaises(RuntimeError, s.add, BadCmp()) - self.assertRaises(RuntimeError, s.discard, BadCmp()) - self.assertRaises(RuntimeError, s.remove, BadCmp()) - - def test_cyclical_repr(self): - w = ReprWrapper() - s = self.thetype([w]) - w.value = s - if self.thetype == set: - self.assertEqual(repr(s), '{set(...)}') - else: - name = repr(s).partition('(')[0] # strip class name - self.assertEqual(repr(s), '%s({%s(...)})' % (name, name)) - - def test_cyclical_print(self): - w = ReprWrapper() - s = self.thetype([w]) - w.value = s - fo = open(support.TESTFN, "w") - try: - fo.write(str(s)) - fo.close() - fo = open(support.TESTFN, "r") - self.assertEqual(fo.read(), repr(s)) - finally: - fo.close() - support.unlink(support.TESTFN) - - def test_do_not_rehash_dict_keys(self): - raise NotImplementedError() # cannot subclass int - n = 10 - d = None # dict.fromkeys(map(HashCountingInt, range(n))) - self.assertEqual(sum(elem.hash_count for elem in d), n) - s = self.thetype(d) - self.assertEqual(sum(elem.hash_count for elem in d), n) - s.difference(d) - self.assertEqual(sum(elem.hash_count for elem in d), n) - if hasattr(s, 'symmetric_difference_update'): - s.symmetric_difference_update(d) - self.assertEqual(sum(elem.hash_count for elem in d), n) - d2 = dict.fromkeys(set(d)) - self.assertEqual(sum(elem.hash_count for elem in d), n) - d3 = dict.fromkeys(frozenset(d)) - self.assertEqual(sum(elem.hash_count for elem in d), n) - d3 = dict.fromkeys(frozenset(d), 123) - self.assertEqual(sum(elem.hash_count for elem in d), n) - self.assertEqual(d3, dict.fromkeys(d, 123)) - - def test_container_iterator(self): - # Bug #3680: tp_traverse was not implemented for set iterator object - class C(object): - pass - obj = C() - ref = weakref.ref(obj) - container = set([obj, 1]) - obj.x = iter(container) - obj = None - container = None - gc.collect() - self.assertTrue(ref() is None, "Cycle was not collected") - -class TestSet(TestJointOps): - thetype = set - basetype = set - - def test_init(self): - s = self.thetype() - s.__init__(self.word) - self.assertEqual(s, set(self.word)) - s.__init__(self.otherword) - self.assertEqual(s, set(self.otherword)) - self.assertRaises(TypeError, s.__init__, s, 2); - self.assertRaises(TypeError, s.__init__, 1) - - def test_constructor_identity(self): - s = self.thetype(range(3)) - t = self.thetype(s) - self.assertNotEqual(id(s), id(t)) - - def test_set_literal(self): - raise NotImplementedError() - #s = set([1,2,3]) - #t = {1,2,3} - #self.assertEqual(s, t) - - def test_hash(self): - self.assertRaises(TypeError, hash, self.s) - - def test_clear(self): - self.s.clear() - self.assertEqual(self.s, set()) - self.assertEqual(len(self.s), 0) - - def test_copy(self): - dup = self.s.copy() - self.assertEqual(self.s, dup) - self.assertNotEqual(id(self.s), id(dup)) - self.assertEqual(type(dup), self.basetype) - - def test_add(self): - self.s.add('Q') - self.assertIn('Q', self.s) - dup = self.s.copy() - self.s.add('Q') - self.assertEqual(self.s, dup) - self.assertRaises(TypeError, self.s.add, []) - - def test_remove(self): - self.s.remove('a') - self.assertNotIn('a', self.s) - self.assertRaises(KeyError, self.s.remove, 'Q') - self.assertRaises(TypeError, self.s.remove, []) - s = self.thetype([frozenset(self.word)]) - self.assertIn(self.thetype(self.word), s) - s.remove(self.thetype(self.word)) - self.assertNotIn(self.thetype(self.word), s) - self.assertRaises(KeyError, self.s.remove, self.thetype(self.word)) - - def test_remove_keyerror_unpacking(self): - # bug: www.python.org/sf/1576657 - for v1 in ['Q', (1,)]: - try: - self.s.remove(v1) - except KeyError as e: - v2 = e.args[0] - self.assertEqual(v1, v2) - else: - self.fail() - - def test_remove_keyerror_set(self): - key = self.thetype([3, 4]) - try: - self.s.remove(key) - except KeyError as e: - self.assertTrue(e.args[0] is key, - "KeyError should be {0}, not {1}".format(key, - e.args[0])) - else: - self.fail() - - def test_discard(self): - self.s.discard('a') - self.assertNotIn('a', self.s) - self.s.discard('Q') - self.assertRaises(TypeError, self.s.discard, []) - s = self.thetype([frozenset(self.word)]) - self.assertIn(self.thetype(self.word), s) - s.discard(self.thetype(self.word)) - self.assertNotIn(self.thetype(self.word), s) - s.discard(self.thetype(self.word)) - - def test_pop(self): - for i in range(len(self.s)): - elem = self.s.pop() - self.assertNotIn(elem, self.s) - self.assertRaises(KeyError, self.s.pop) - - def test_update(self): - retval = self.s.update(self.otherword) - self.assertEqual(retval, None) - for c in (self.word + self.otherword): - self.assertIn(c, self.s) - self.assertRaises(PassThru, self.s.update, check_pass_thru()) - self.assertRaises(TypeError, self.s.update, [[]]) - for p, q in (('cdc', 'abcd'), ('efgfe', 'abcefg'), ('ccb', 'abc'), ('ef', 'abcef')): - for C in set, frozenset, dict.fromkeys, str, list, tuple: - s = self.thetype('abcba') - self.assertEqual(s.update(C(p)), None) - self.assertEqual(s, set(q)) - for p in ('cdc', 'efgfe', 'ccb', 'ef', 'abcda'): - q = 'ahi' - for C in set, frozenset, dict.fromkeys, str, list, tuple: - s = self.thetype('abcba') - self.assertEqual(s.update(C(p), C(q)), None) - self.assertEqual(s, set(s) | set(p) | set(q)) - - def test_ior(self): - self.s |= set(self.otherword) - for c in (self.word + self.otherword): - self.assertIn(c, self.s) - - def test_intersection_update(self): - retval = self.s.intersection_update(self.otherword) - self.assertEqual(retval, None) - for c in (self.word + self.otherword): - if c in self.otherword and c in self.word: - self.assertIn(c, self.s) - else: - self.assertNotIn(c, self.s) - self.assertRaises(PassThru, self.s.intersection_update, check_pass_thru()) - self.assertRaises(TypeError, self.s.intersection_update, [[]]) - for p, q in (('cdc', 'c'), ('efgfe', ''), ('ccb', 'bc'), ('ef', '')): - for C in set, frozenset, dict.fromkeys, str, list, tuple: - s = self.thetype('abcba') - self.assertEqual(s.intersection_update(C(p)), None) - self.assertEqual(s, set(q)) - ss = 'abcba' - s = self.thetype(ss) - t = 'cbc' - self.assertEqual(s.intersection_update(C(p), C(t)), None) - self.assertEqual(s, set('abcba')&set(p)&set(t)) - - def test_iand(self): - self.s &= set(self.otherword) - for c in (self.word + self.otherword): - if c in self.otherword and c in self.word: - self.assertIn(c, self.s) - else: - self.assertNotIn(c, self.s) - - def test_difference_update(self): - retval = self.s.difference_update(self.otherword) - self.assertEqual(retval, None) - for c in (self.word + self.otherword): - if c in self.word and c not in self.otherword: - self.assertIn(c, self.s) - else: - self.assertNotIn(c, self.s) - self.assertRaises(PassThru, self.s.difference_update, check_pass_thru()) - self.assertRaises(TypeError, self.s.difference_update, [[]]) - self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]]) - for p, q in (('cdc', 'ab'), ('efgfe', 'abc'), ('ccb', 'a'), ('ef', 'abc')): - for C in set, frozenset, dict.fromkeys, str, list, tuple: - s = self.thetype('abcba') - self.assertEqual(s.difference_update(C(p)), None) - self.assertEqual(s, set(q)) - - s = self.thetype('abcdefghih') - s.difference_update() - self.assertEqual(s, self.thetype('abcdefghih')) - - s = self.thetype('abcdefghih') - s.difference_update(C('aba')) - self.assertEqual(s, self.thetype('cdefghih')) - - s = self.thetype('abcdefghih') - s.difference_update(C('cdc'), C('aba')) - self.assertEqual(s, self.thetype('efghih')) - - def test_isub(self): - self.s -= set(self.otherword) - for c in (self.word + self.otherword): - if c in self.word and c not in self.otherword: - self.assertIn(c, self.s) - else: - self.assertNotIn(c, self.s) - - def test_symmetric_difference_update(self): - retval = self.s.symmetric_difference_update(self.otherword) - self.assertEqual(retval, None) - for c in (self.word + self.otherword): - if (c in self.word) ^ (c in self.otherword): - self.assertIn(c, self.s) - else: - self.assertNotIn(c, self.s) - self.assertRaises(PassThru, self.s.symmetric_difference_update, check_pass_thru()) - self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]]) - for p, q in (('cdc', 'abd'), ('efgfe', 'abcefg'), ('ccb', 'a'), ('ef', 'abcef')): - for C in set, frozenset, dict.fromkeys, str, list, tuple: - s = self.thetype('abcba') - self.assertEqual(s.symmetric_difference_update(C(p)), None) - self.assertEqual(s, set(q)) - - def test_ixor(self): - self.s ^= set(self.otherword) - for c in (self.word + self.otherword): - if (c in self.word) ^ (c in self.otherword): - self.assertIn(c, self.s) - else: - self.assertNotIn(c, self.s) - - def test_inplace_on_self(self): - t = self.s.copy() - t |= t - self.assertEqual(t, self.s) - t &= t - self.assertEqual(t, self.s) - t -= t - self.assertEqual(t, self.thetype()) - t = self.s.copy() - t ^= t - self.assertEqual(t, self.thetype()) - - def test_weakref(self): - s = self.thetype('gallahad') - p = weakref.proxy(s) - self.assertEqual(str(p), str(s)) - s = None - self.assertRaises(ReferenceError, str, p) - - def test_rich_compare(self): - class TestRichSetCompare: - def __gt__(self, some_set): - self.gt_called = True - return False - def __lt__(self, some_set): - self.lt_called = True - return False - def __ge__(self, some_set): - self.ge_called = True - return False - def __le__(self, some_set): - self.le_called = True - return False - - # This first tries the builtin rich set comparison, which doesn't know - # how to handle the custom object. Upon returning NotImplemented, the - # corresponding comparison on the right object is invoked. - myset = {1, 2, 3} - - myobj = TestRichSetCompare() - myset < myobj - self.assertTrue(myobj.gt_called) - - myobj = TestRichSetCompare() - myset > myobj - self.assertTrue(myobj.lt_called) - - myobj = TestRichSetCompare() - myset <= myobj - self.assertTrue(myobj.ge_called) - - myobj = TestRichSetCompare() - myset >= myobj - self.assertTrue(myobj.le_called) - - # C API test only available in a debug build - if hasattr(set, "test_c_api"): - def test_c_api(self): - self.assertEqual(set().test_c_api(), True) - -class SetSubclass(set): - pass - -class TestSetSubclass(TestSet): - thetype = SetSubclass - basetype = set - -class SetSubclassWithKeywordArgs(set): - def __init__(self, iterable=[], newarg=None): - set.__init__(self, iterable) - -class TestSetSubclassWithKeywordArgs(TestSet): - - def test_keywords_in_subclass(self): - 'SF bug #1486663 -- this used to erroneously raise a TypeError' - SetSubclassWithKeywordArgs(newarg=1) - -class TestFrozenSet(TestJointOps): - thetype = frozenset - basetype = frozenset - - def test_init(self): - s = self.thetype(self.word) - s.__init__(self.otherword) - self.assertEqual(s, set(self.word)) - - def test_singleton_empty_frozenset(self): - f = frozenset() - efs = [frozenset(), frozenset([]), frozenset(()), frozenset(''), - frozenset(), frozenset([]), frozenset(()), frozenset(''), - frozenset(range(0)), frozenset(frozenset()), - frozenset(f), f] - # All of the empty frozensets should have just one id() - self.assertEqual(len(set(map(id, efs))), 1) - - def test_constructor_identity(self): - s = self.thetype(range(3)) - t = self.thetype(s) - self.assertEqual(id(s), id(t)) - - def test_hash(self): - self.assertEqual(hash(self.thetype('abcdeb')), - hash(self.thetype('ebecda'))) - - # make sure that all permutations give the same hash value - n = 100 - seq = [randrange(n) for i in range(n)] - results = set() - for i in range(200): - shuffle(seq) - results.add(hash(self.thetype(seq))) - self.assertEqual(len(results), 1) - - def test_copy(self): - dup = self.s.copy() - self.assertEqual(id(self.s), id(dup)) - - def test_frozen_as_dictkey(self): - seq = list(range(10)) + list('abcdefg') + ['apple'] - key1 = self.thetype(seq) - key2 = self.thetype(reversed(seq)) - self.assertEqual(key1, key2) - self.assertNotEqual(id(key1), id(key2)) - d = {} - d[key1] = 42 - self.assertEqual(d[key2], 42) - - def test_hash_caching(self): - f = self.thetype('abcdcda') - self.assertEqual(hash(f), hash(f)) - - def test_hash_effectiveness(self): - n = 13 - hashvalues = set() - addhashvalue = hashvalues.add - elemmasks = [(i+1, 1<=": "issuperset", - } - - reverse = {"==": "==", - "!=": "!=", - "<": ">", - ">": "<", - "<=": ">=", - ">=": "<=", - } - - def test_issubset(self): - raise NotImplementedError() # eval not supported below - x = self.left - y = self.right - for case in "!=", "==", "<", "<=", ">", ">=": - expected = case in self.cases - # Test the binary infix spelling. - result = None ## eval("x" + case + "y", locals()) - self.assertEqual(result, expected) - # Test the "friendly" method-name spelling, if one exists. - if case in TestSubsets.case2method: - method = getattr(x, TestSubsets.case2method[case]) - result = method(y) - self.assertEqual(result, expected) - - # Now do the same for the operands reversed. - rcase = TestSubsets.reverse[case] - result = None ## eval("y" + rcase + "x", locals()) - self.assertEqual(result, expected) - if rcase in TestSubsets.case2method: - method = getattr(y, TestSubsets.case2method[rcase]) - result = method(x) - self.assertEqual(result, expected) -#------------------------------------------------------------------------------ - -class TestSubsetEqualEmpty(TestSubsets): - left = set() # type: Any - right = set() # type: Any - name = "both empty" - cases = "==", "<=", ">=" - -#------------------------------------------------------------------------------ - -class TestSubsetEqualNonEmpty(TestSubsets): - left = set([1, 2]) - right = set([1, 2]) - name = "equal pair" - cases = "==", "<=", ">=" - -#------------------------------------------------------------------------------ - -class TestSubsetEmptyNonEmpty(TestSubsets): - left = set() # type: Any - right = set([1, 2]) - name = "one empty, one non-empty" - cases = "!=", "<", "<=" - -#------------------------------------------------------------------------------ - -class TestSubsetPartial(TestSubsets): - left = set([1]) - right = set([1, 2]) - name = "one a non-empty proper subset of other" - cases = "!=", "<", "<=" - -#------------------------------------------------------------------------------ - -class TestSubsetNonOverlap(TestSubsets): - left = set([1]) - right = set([2]) - name = "neither empty, neither contains" - cases = "!=" - -#============================================================================== - -class TestOnlySetsInBinaryOps(unittest.TestCase): - - def test_eq_ne(self): - # Unlike the others, this is testing that == and != *are* allowed. - self.assertEqual(self.other == self.set, False) - self.assertEqual(self.set == self.other, False) - self.assertEqual(self.other != self.set, True) - self.assertEqual(self.set != self.other, True) - - def test_ge_gt_le_lt(self): - self.assertRaises(TypeError, lambda: self.set < self.other) - self.assertRaises(TypeError, lambda: self.set <= self.other) - self.assertRaises(TypeError, lambda: self.set > self.other) - self.assertRaises(TypeError, lambda: self.set >= self.other) - - self.assertRaises(TypeError, lambda: self.other < self.set) - self.assertRaises(TypeError, lambda: self.other <= self.set) - self.assertRaises(TypeError, lambda: self.other > self.set) - self.assertRaises(TypeError, lambda: self.other >= self.set) - - def test_update_operator(self): - try: - self.set |= self.other - except TypeError: - pass - else: - self.fail("expected TypeError") - - def test_update(self): - if self.otherIsIterable: - self.set.update(self.other) - else: - self.assertRaises(TypeError, self.set.update, self.other) - - def test_union(self): - self.assertRaises(TypeError, lambda: self.set | self.other) - self.assertRaises(TypeError, lambda: self.other | self.set) - if self.otherIsIterable: - self.set.union(self.other) - else: - self.assertRaises(TypeError, self.set.union, self.other) - - def test_intersection_update_operator(self): - try: - self.set &= self.other - except TypeError: - pass - else: - self.fail("expected TypeError") - - def test_intersection_update(self): - if self.otherIsIterable: - self.set.intersection_update(self.other) - else: - self.assertRaises(TypeError, - self.set.intersection_update, - self.other) - - def test_intersection(self): - self.assertRaises(TypeError, lambda: self.set & self.other) - self.assertRaises(TypeError, lambda: self.other & self.set) - if self.otherIsIterable: - self.set.intersection(self.other) - else: - self.assertRaises(TypeError, self.set.intersection, self.other) - - def test_sym_difference_update_operator(self): - try: - self.set ^= self.other - except TypeError: - pass - else: - self.fail("expected TypeError") - - def test_sym_difference_update(self): - if self.otherIsIterable: - self.set.symmetric_difference_update(self.other) - else: - self.assertRaises(TypeError, - self.set.symmetric_difference_update, - self.other) - - def test_sym_difference(self): - self.assertRaises(TypeError, lambda: self.set ^ self.other) - self.assertRaises(TypeError, lambda: self.other ^ self.set) - if self.otherIsIterable: - self.set.symmetric_difference(self.other) - else: - self.assertRaises(TypeError, self.set.symmetric_difference, self.other) - - def test_difference_update_operator(self): - try: - self.set -= self.other - except TypeError: - pass - else: - self.fail("expected TypeError") - - def test_difference_update(self): - if self.otherIsIterable: - self.set.difference_update(self.other) - else: - self.assertRaises(TypeError, - self.set.difference_update, - self.other) - - def test_difference(self): - self.assertRaises(TypeError, lambda: self.set - self.other) - self.assertRaises(TypeError, lambda: self.other - self.set) - if self.otherIsIterable: - self.set.difference(self.other) - else: - self.assertRaises(TypeError, self.set.difference, self.other) - -#------------------------------------------------------------------------------ - -class TestOnlySetsNumeric(TestOnlySetsInBinaryOps): - def setUp(self): - self.set = set((1, 2, 3)) - self.other = 19 - self.otherIsIterable = False - -#------------------------------------------------------------------------------ - -class TestOnlySetsDict(TestOnlySetsInBinaryOps): - def setUp(self): - self.set = set((1, 2, 3)) - self.other = {1:2, 3:4} - self.otherIsIterable = True - -#------------------------------------------------------------------------------ - -class TestOnlySetsOperator(TestOnlySetsInBinaryOps): - def setUp(self): - self.set = set((1, 2, 3)) - self.other = operator.add - self.otherIsIterable = False - -#------------------------------------------------------------------------------ - -class TestOnlySetsTuple(TestOnlySetsInBinaryOps): - def setUp(self): - self.set = set((1, 2, 3)) - self.other = (2, 4, 6) - self.otherIsIterable = True - -#------------------------------------------------------------------------------ - -class TestOnlySetsString(TestOnlySetsInBinaryOps): - def setUp(self): - self.set = set((1, 2, 3)) - self.other = 'abc' - self.otherIsIterable = True - -#------------------------------------------------------------------------------ - -class TestOnlySetsGenerator(TestOnlySetsInBinaryOps): - def setUp(self): - def gen(): - for i in range(0, 10, 2): - yield i - self.set = set((1, 2, 3)) - self.other = gen() - self.otherIsIterable = True - -#============================================================================== - -class TestCopying(unittest.TestCase): - - def test_copy(self): - dup = self.set.copy() - dup_list = sorted(dup, key=repr) - set_list = sorted(self.set, key=repr) - self.assertEqual(len(dup_list), len(set_list)) - for i in range(len(dup_list)): - self.assertTrue(dup_list[i] is set_list[i]) - - def test_deep_copy(self): - dup = copy.deepcopy(self.set) - ##print type(dup), repr(dup) - dup_list = sorted(dup, key=repr) - set_list = sorted(self.set, key=repr) - self.assertEqual(len(dup_list), len(set_list)) - for i in range(len(dup_list)): - self.assertEqual(dup_list[i], set_list[i]) - -#------------------------------------------------------------------------------ - -class TestCopyingEmpty(TestCopying): - def setUp(self): - self.set = set() - -#------------------------------------------------------------------------------ - -class TestCopyingSingleton(TestCopying): - def setUp(self): - self.set = set(["hello"]) - -#------------------------------------------------------------------------------ - -class TestCopyingTriple(TestCopying): - def setUp(self): - self.set = set(["zero", 0, None]) - -#------------------------------------------------------------------------------ - -class TestCopyingTuple(TestCopying): - def setUp(self): - self.set = set([(1, 2)]) - -#------------------------------------------------------------------------------ - -class TestCopyingNested(TestCopying): - def setUp(self): - self.set = set([((1, 2), (3, 4))]) - -#============================================================================== - -class TestIdentities(unittest.TestCase): - def setUp(self): - self.a = set('abracadabra') - self.b = set('alacazam') - - def test_binopsVsSubsets(self): - a, b = self.a, self.b - self.assertTrue(a - b < a) - self.assertTrue(b - a < b) - self.assertTrue(a & b < a) - self.assertTrue(a & b < b) - self.assertTrue(a | b > a) - self.assertTrue(a | b > b) - self.assertTrue(a ^ b < a | b) - - def test_commutativity(self): - a, b = self.a, self.b - self.assertEqual(a&b, b&a) - self.assertEqual(a|b, b|a) - self.assertEqual(a^b, b^a) - if a != b: - self.assertNotEqual(a-b, b-a) - - def test_summations(self): - # check that sums of parts equal the whole - a, b = self.a, self.b - self.assertEqual((a-b)|(a&b)|(b-a), a|b) - self.assertEqual((a&b)|(a^b), a|b) - self.assertEqual(a|(b-a), a|b) - self.assertEqual((a-b)|b, a|b) - self.assertEqual((a-b)|(a&b), a) - self.assertEqual((b-a)|(a&b), b) - self.assertEqual((a-b)|(b-a), a^b) - - def test_exclusion(self): - # check that inverse operations show non-overlap - a, b, zero = self.a, self.b, set() - self.assertEqual((a-b)&b, zero) - self.assertEqual((b-a)&a, zero) - self.assertEqual((a&b)&(a^b), zero) - -# Tests derived from test_itertools.py ======================================= - -def R(seqn): - 'Regular generator' - for i in seqn: - yield i - -class G: - 'Sequence using __getitem__' - def __init__(self, seqn): - self.seqn = seqn - def __getitem__(self, i): - return self.seqn[i] - -class I: - 'Sequence using iterator protocol' - def __init__(self, seqn): - self.seqn = seqn - self.i = 0 - def __iter__(self): - return self - def __next__(self): - if self.i >= len(self.seqn): raise StopIteration - v = self.seqn[self.i] - self.i += 1 - return v - -class Ig: - 'Sequence using iterator protocol defined with a generator' - def __init__(self, seqn): - self.seqn = seqn - self.i = 0 - def __iter__(self): - for val in self.seqn: - yield val - -class X: - 'Missing __getitem__ and __iter__' - def __init__(self, seqn): - self.seqn = seqn - self.i = 0 - def __next__(self): - if self.i >= len(self.seqn): raise StopIteration - v = self.seqn[self.i] - self.i += 1 - return v - -class N: - 'Iterator missing __next__()' - def __init__(self, seqn): - self.seqn = seqn - self.i = 0 - def __iter__(self): - return self - -class E: - 'Test propagation of exceptions' - def __init__(self, seqn): - self.seqn = seqn - self.i = 0 - def __iter__(self): - return self - def __next__(self): - 3 // 0 - -class S: - 'Test immediate stop' - def __init__(self, seqn): - pass - def __iter__(self): - return self - def __next__(self): - raise StopIteration - -from itertools import chain -def L(seqn): - 'Test multiple tiers of iterators' - return chain(map(lambda x:x, R(Ig(G(seqn))))) - -class TestVariousIteratorArgs(unittest.TestCase): - - def test_constructor(self): - for cons in (set, frozenset): - for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): - for g in (G, I, Ig, S, L, R): - self.assertEqual(sorted(cons(g(s)), key=repr), sorted(g(s), key=repr)) - self.assertRaises(TypeError, cons , X(s)) - self.assertRaises(TypeError, cons , N(s)) - self.assertRaises(ZeroDivisionError, cons , E(s)) - - def test_inline_methods(self): - s = set('november') - for data in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5), 'december'): - for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint): - for g in (G, I, Ig, L, R): - expected = meth(data) - actual = meth(G(data)) - if isinstance(expected, bool): - self.assertEqual(actual, expected) - else: - self.assertEqual(sorted(actual, key=repr), sorted(expected, key=repr)) - self.assertRaises(TypeError, meth, X(s)) - self.assertRaises(TypeError, meth, N(s)) - self.assertRaises(ZeroDivisionError, meth, E(s)) - - def test_inplace_methods(self): - for data in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5), 'december'): - for methname in ('update', 'intersection_update', - 'difference_update', 'symmetric_difference_update'): - for g in (G, I, Ig, S, L, R): - s = set('january') - t = s.copy() - getattr(s, methname)(list(g(data))) - getattr(t, methname)(g(data)) - self.assertEqual(sorted(s, key=repr), sorted(t, key=repr)) - - self.assertRaises(TypeError, getattr(set('january'), methname), X(data)) - self.assertRaises(TypeError, getattr(set('january'), methname), N(data)) - self.assertRaises(ZeroDivisionError, getattr(set('january'), methname), E(data)) - -be_bad = set2 = dict2 = Undefined(Any) - -class bad_eq: - def __eq__(self, other): - if be_bad: - set2.clear() - raise ZeroDivisionError - return self is other - def __hash__(self): - return 0 - -class bad_dict_clear: - def __eq__(self, other): - if be_bad: - dict2.clear() - return self is other - def __hash__(self): - return 0 - -class TestWeirdBugs(unittest.TestCase): - def test_8420_set_merge(self): - # This used to segfault - global be_bad, set2, dict2 - be_bad = False - set1 = {bad_eq()} - set2 = {bad_eq() for i in range(75)} - be_bad = True - self.assertRaises(ZeroDivisionError, set1.update, set2) - - be_bad = False - set1 = {bad_dict_clear()} - dict2 = {bad_dict_clear(): None} - be_bad = True - set1.symmetric_difference_update(dict2) - -# Application tests (based on David Eppstein's graph recipes ==================================== - -def powerset(U): - """Generates all subsets of a set or sequence U.""" - U = iter(U) - try: - x = frozenset([next(U)]) - for S in powerset(U): - yield S - yield S | x - except StopIteration: - yield frozenset() - -def cube(n): - """Graph of n-dimensional hypercube.""" - singletons = [frozenset([x]) for x in range(n)] - return dict([(x, frozenset([x^s for s in singletons])) - for x in powerset(range(n))]) - -def linegraph(G): - """Graph, the vertices of which are edges of G, - with two vertices being adjacent iff the corresponding - edges share a vertex.""" - L = {} - for x in G: - for y in G[x]: - nx = [frozenset([x,z]) for z in G[x] if z != y] - ny = [frozenset([y,z]) for z in G[y] if z != x] - L[frozenset([x,y])] = frozenset(nx+ny) - return L - -def faces(G): - 'Return a set of faces in G. Where a face is a set of vertices on that face' - # currently limited to triangles,squares, and pentagons - f = set() - for v1, edges in G.items(): - for v2 in edges: - for v3 in G[v2]: - if v1 == v3: - continue - if v1 in G[v3]: - f.add(frozenset([v1, v2, v3])) - else: - for v4 in G[v3]: - if v4 == v2: - continue - if v1 in G[v4]: - f.add(frozenset([v1, v2, v3, v4])) - else: - for v5 in G[v4]: - if v5 == v3 or v5 == v2: - continue - if v1 in G[v5]: - f.add(frozenset([v1, v2, v3, v4, v5])) - return f - - -class TestGraphs(unittest.TestCase): - - def test_cube(self): - - g = cube(3) # vert --> {v1, v2, v3} - vertices1 = set(g) - self.assertEqual(len(vertices1), 8) # eight vertices - for edge in g.values(): - self.assertEqual(len(edge), 3) # each vertex connects to three edges - vertices2 = set() - for edges in g.values(): - for v in edges: - vertices2.add(v) - self.assertEqual(vertices1, vertices2) # edge vertices in original set - - cubefaces = faces(g) - self.assertEqual(len(cubefaces), 6) # six faces - for face in cubefaces: - self.assertEqual(len(face), 4) # each face is a square - - def test_cuboctahedron(self): - - # http://en.wikipedia.org/wiki/Cuboctahedron - # 8 triangular faces and 6 square faces - # 12 indentical vertices each connecting a triangle and square - - g = cube(3) - cuboctahedron = linegraph(g) # V( --> {V1, V2, V3, V4} - self.assertEqual(len(cuboctahedron), 12)# twelve vertices - - vertices = set(cuboctahedron) - for edges in cuboctahedron.values(): - self.assertEqual(len(edges), 4) # each vertex connects to four other vertices - othervertices = set(edge for edges in cuboctahedron.values() for edge in edges) - self.assertEqual(vertices, othervertices) # edge vertices in original set - - cubofaces = faces(cuboctahedron) - facesizes = collections.defaultdict(int) - for face in cubofaces: - facesizes[len(face)] += 1 - self.assertEqual(facesizes[3], 8) # eight triangular faces - self.assertEqual(facesizes[4], 6) # six square faces - - for vertex in cuboctahedron: - edge = vertex # Cuboctahedron vertices are edges in Cube - self.assertEqual(len(edge), 2) # Two cube vertices define an edge - for cubevert in edge: - self.assertIn(cubevert, g) - - -#============================================================================== - -def test_main(verbose=None): - test_classes = ( - TestSet, - TestSetSubclass, - TestSetSubclassWithKeywordArgs, - TestFrozenSet, - TestFrozenSetSubclass, - TestSetOfSets, - TestExceptionPropagation, - TestBasicOpsEmpty, - TestBasicOpsSingleton, - TestBasicOpsTuple, - TestBasicOpsTriple, - TestBasicOpsString, - TestBasicOpsBytes, - TestBasicOpsMixedStringBytes, - TestBinaryOps, - TestUpdateOps, - TestMutate, - TestSubsetEqualEmpty, - TestSubsetEqualNonEmpty, - TestSubsetEmptyNonEmpty, - TestSubsetPartial, - TestSubsetNonOverlap, - TestOnlySetsNumeric, - TestOnlySetsDict, - TestOnlySetsOperator, - TestOnlySetsTuple, - TestOnlySetsString, - TestOnlySetsGenerator, - TestCopyingEmpty, - TestCopyingSingleton, - TestCopyingTriple, - TestCopyingTuple, - TestCopyingNested, - TestIdentities, - TestVariousIteratorArgs, - TestGraphs, - TestWeirdBugs, - ) - - support.run_unittest(*test_classes) - - # verify reference counting - if verbose and hasattr(sys, "gettotalrefcount"): - import gc - counts = [None] * 5 - for i in range(len(counts)): - support.run_unittest(*test_classes) - gc.collect() - counts[i] = sys.gettotalrefcount() - print(counts) - -if __name__ == "__main__": - test_main(verbose=True) diff --git a/lib-python/3.2/test/test_shutil.py b/lib-python/3.2/test/test_shutil.py deleted file mode 100644 index af0935ecdcd6..000000000000 --- a/lib-python/3.2/test/test_shutil.py +++ /dev/null @@ -1,981 +0,0 @@ -# Copyright (C) 2003 Python Software Foundation - -import unittest -import shutil -import tempfile -import sys -import stat -import os -import os.path -import functools -from test import support -from test.support import TESTFN -from os.path import splitdrive -from distutils.spawn import find_executable, spawn -from shutil import (_make_tarball, _make_zipfile, make_archive, - register_archive_format, unregister_archive_format, - get_archive_formats, Error, unpack_archive, - register_unpack_format, RegistryError, - unregister_unpack_format, get_unpack_formats) -import tarfile -import warnings - -from test import support -from test.support import check_warnings, captured_stdout - -from typing import ( - Any, Function, Tuple, List, Sequence, BinaryIO, overload, Traceback, IO, - ducktype -) - -import bz2 -BZ2_SUPPORTED = True - -TESTFN2 = TESTFN + "2" - -import grp -import pwd -UID_GID_SUPPORT = True - -import zlib - -import zipfile -ZIP_SUPPORT = True - -def _fake_rename(*args: Any, **kwargs: Any) -> None: - # Pretend the destination path is on a different filesystem. - raise OSError() - -def mock_rename(func: Any) -> Any: - @functools.wraps(func) - def wrap(*args: Any, **kwargs: Any) -> Any: - try: - builtin_rename = shutil.rename - shutil.rename = Any(_fake_rename) - return func(*args, **kwargs) - finally: - shutil.rename = Any(builtin_rename) - return wrap - -class TestShutil(unittest.TestCase): - - def setUp(self) -> None: - super().setUp() - self.tempdirs = List[str]() - - def tearDown(self) -> None: - super().tearDown() - while self.tempdirs: - d = self.tempdirs.pop() - shutil.rmtree(d, os.name in ('nt', 'cygwin')) - - @overload - def write_file(self, path: str, content: str = 'xxx') -> None: - """Writes a file in the given path. - - - path can be a string or a sequence. - """ - f = open(path, 'w') - try: - f.write(content) - finally: - f.close() - - @overload - def write_file(self, path: Sequence[str], content: str = 'xxx') -> None: - # JLe: work around mypy issue #238 - self.write_file(os.path.join(*list(path)), content) - - def mkdtemp(self) -> str: - """Create a temporary directory that will be cleaned up. - - Returns the path of the directory. - """ - d = tempfile.mkdtemp() - self.tempdirs.append(d) - return d - - def test_rmtree_errors(self) -> None: - # filename is guaranteed not to exist - filename = tempfile.mktemp() - self.assertRaises(OSError, shutil.rmtree, filename) - - # See bug #1071513 for why we don't run this on cygwin - # and bug #1076467 for why we don't run this as root. - if (hasattr(os, 'chmod') and sys.platform[:6] != 'cygwin' - and not (hasattr(os, 'geteuid') and os.geteuid() == 0)): - def test_on_error(self) -> None: - self.errorState = 0 - os.mkdir(TESTFN) - self.childpath = os.path.join(TESTFN, 'a') - f = open(self.childpath, 'w') - f.close() - old_dir_mode = os.stat(TESTFN).st_mode - old_child_mode = os.stat(self.childpath).st_mode - # Make unwritable. - os.chmod(self.childpath, stat.S_IREAD) - os.chmod(TESTFN, stat.S_IREAD) - - shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror) - # Test whether onerror has actually been called. - self.assertEqual(self.errorState, 2, - "Expected call to onerror function did not happen.") - - # Make writable again. - os.chmod(TESTFN, old_dir_mode) - os.chmod(self.childpath, old_child_mode) - - # Clean up. - shutil.rmtree(TESTFN) - - def check_args_to_onerror(self, func: Function[[str], Any], arg: str, - exc: Tuple[type, BaseException, - Traceback]) -> None: - # test_rmtree_errors deliberately runs rmtree - # on a directory that is chmod 400, which will fail. - # This function is run when shutil.rmtree fails. - # 99.9% of the time it initially fails to remove - # a file in the directory, so the first time through - # func is os.remove. - # However, some Linux machines running ZFS on - # FUSE experienced a failure earlier in the process - # at os.listdir. The first failure may legally - # be either. - if self.errorState == 0: - if func is os.remove: - self.assertEqual(arg, self.childpath) - else: - self.assertIs(func, os.listdir, - "func must be either os.remove or os.listdir") - self.assertEqual(arg, TESTFN) - self.assertTrue(issubclass(exc[0], OSError)) - self.errorState = 1 - else: - self.assertEqual(func, os.rmdir) - self.assertEqual(arg, TESTFN) - self.assertTrue(issubclass(exc[0], OSError)) - self.errorState = 2 - - def test_rmtree_dont_delete_file(self) -> None: - # When called on a file instead of a directory, don't delete it. - handle, path = tempfile.mkstemp() - os.fdopen(handle).close() - self.assertRaises(OSError, shutil.rmtree, path) - os.remove(path) - - def _write_data(self, path: str, data: str) -> None: - f = open(path, "w") - f.write(data) - f.close() - - def test_copytree_simple(self) -> None: - - def read_data(path: str) -> str: - f = open(path) - data = f.read() - f.close() - return data - - src_dir = tempfile.mkdtemp() - dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') - self._write_data(os.path.join(src_dir, 'test.txt'), '123') - os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') - - try: - shutil.copytree(src_dir, dst_dir) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) - self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir', - 'test.txt'))) - actual = read_data(os.path.join(dst_dir, 'test.txt')) - self.assertEqual(actual, '123') - actual = read_data(os.path.join(dst_dir, 'test_dir', 'test.txt')) - self.assertEqual(actual, '456') - finally: - for path in ( - os.path.join(src_dir, 'test.txt'), - os.path.join(dst_dir, 'test.txt'), - os.path.join(src_dir, 'test_dir', 'test.txt'), - os.path.join(dst_dir, 'test_dir', 'test.txt'), - ): - if os.path.exists(path): - os.remove(path) - for path in (src_dir, - os.path.dirname(dst_dir) - ): - if os.path.exists(path): - shutil.rmtree(path) - - def test_copytree_with_exclude(self) -> None: - - def read_data(path: str) -> str: - f = open(path) - data = f.read() - f.close() - return data - - # creating data - join = os.path.join - exists = os.path.exists - src_dir = tempfile.mkdtemp() - try: - dst_dir = join(tempfile.mkdtemp(), 'destination') - self._write_data(join(src_dir, 'test.txt'), '123') - self._write_data(join(src_dir, 'test.tmp'), '123') - os.mkdir(join(src_dir, 'test_dir')) - self._write_data(join(src_dir, 'test_dir', 'test.txt'), '456') - os.mkdir(join(src_dir, 'test_dir2')) - self._write_data(join(src_dir, 'test_dir2', 'test.txt'), '456') - os.mkdir(join(src_dir, 'test_dir2', 'subdir')) - os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - self._write_data(join(src_dir, 'test_dir2', 'subdir', 'test.txt'), - '456') - self._write_data(join(src_dir, 'test_dir2', 'subdir2', 'test.py'), - '456') - - - # testing glob-like patterns - try: - patterns = shutil.ignore_patterns('*.tmp', 'test_dir2') - shutil.copytree(src_dir, dst_dir, ignore=patterns) - # checking the result: some elements should not be copied - self.assertTrue(exists(join(dst_dir, 'test.txt'))) - self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2'))) - finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) - try: - patterns = shutil.ignore_patterns('*.tmp', 'subdir*') - shutil.copytree(src_dir, dst_dir, ignore=patterns) - # checking the result: some elements should not be copied - self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir'))) - finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) - - # testing callable-style - try: - def _filter(src: str, names: Sequence[str]) -> List[str]: - res = List[str]() - for name in names: - path = os.path.join(src, name) - - if (os.path.isdir(path) and - path.split()[-1] == 'subdir'): - res.append(name) - elif os.path.splitext(path)[-1] in ('.py'): - res.append(name) - return res - - shutil.copytree(src_dir, dst_dir, ignore=_filter) - - # checking the result: some elements should not be copied - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2', - 'test.py'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir'))) - - finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) - finally: - shutil.rmtree(src_dir) - shutil.rmtree(os.path.dirname(dst_dir)) - - @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') - def test_dont_copy_file_onto_link_to_itself(self) -> None: - # Temporarily disable test on Windows. - if os.name == 'nt': - return - # bug 851123. - os.mkdir(TESTFN) - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - try: - with open(src, 'w') as f: - f.write('cheddar') - os.link(src, dst) - self.assertRaises(shutil.Error, shutil.copyfile, src, dst) - with open(src, 'r') as f: - self.assertEqual(f.read(), 'cheddar') - os.remove(dst) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - @support.skip_unless_symlink - def test_dont_copy_file_onto_symlink_to_itself(self) -> None: - # bug 851123. - os.mkdir(TESTFN) - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - try: - with open(src, 'w') as f: - f.write('cheddar') - # Using `src` here would mean we end up with a symlink pointing - # to TESTFN/TESTFN/cheese, while it should point at - # TESTFN/cheese. - os.symlink('cheese', dst) - self.assertRaises(shutil.Error, shutil.copyfile, src, dst) - with open(src, 'r') as f: - self.assertEqual(f.read(), 'cheddar') - os.remove(dst) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - @support.skip_unless_symlink - def test_rmtree_on_symlink(self) -> None: - # bug 1669. - os.mkdir(TESTFN) - try: - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - os.mkdir(src) - os.symlink(src, dst) - self.assertRaises(OSError, shutil.rmtree, dst) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - if hasattr(os, "mkfifo"): - # Issue #3002: copyfile and copytree block indefinitely on named pipes - def test_copyfile_named_pipe(self) -> None: - os.mkfifo(TESTFN) - try: - self.assertRaises(shutil.SpecialFileError, - shutil.copyfile, TESTFN, TESTFN2) - self.assertRaises(shutil.SpecialFileError, - shutil.copyfile, __file__, TESTFN) - finally: - os.remove(TESTFN) - - @support.skip_unless_symlink - def test_copytree_named_pipe(self) -> None: - os.mkdir(TESTFN) - try: - subdir = os.path.join(TESTFN, "subdir") - os.mkdir(subdir) - pipe = os.path.join(subdir, "mypipe") - os.mkfifo(pipe) - try: - shutil.copytree(TESTFN, TESTFN2) - except shutil.Error as e: - errors = e.args[0] - self.assertEqual(len(errors), 1) - src, dst, error_msg = errors[0] - self.assertEqual("`%s` is a named pipe" % pipe, error_msg) - else: - self.fail("shutil.Error should have been raised") - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - shutil.rmtree(TESTFN2, ignore_errors=True) - - def test_copytree_special_func(self) -> None: - - src_dir = self.mkdtemp() - dst_dir = os.path.join(self.mkdtemp(), 'destination') - self._write_data(os.path.join(src_dir, 'test.txt'), '123') - os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') - - copied = List[Tuple[str, str]]() - def _copy(src: str, dst: str) -> None: - copied.append((src, dst)) - - shutil.copytree(src_dir, dst_dir, copy_function=_copy) - self.assertEqual(len(copied), 2) - - @support.skip_unless_symlink - def test_copytree_dangling_symlinks(self) -> None: - - # a dangling symlink raises an error at the end - src_dir = self.mkdtemp() - dst_dir = os.path.join(self.mkdtemp(), 'destination') - os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt')) - os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') - self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) - - # a dangling symlink is ignored with the proper flag - dst_dir = os.path.join(self.mkdtemp(), 'destination2') - shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True) - self.assertNotIn('test.txt', os.listdir(dst_dir)) - - # a dangling symlink is copied if symlinks=True - dst_dir = os.path.join(self.mkdtemp(), 'destination3') - shutil.copytree(src_dir, dst_dir, symlinks=True) - self.assertIn('test.txt', os.listdir(dst_dir)) - - def _copy_file(self, - method: Function[[str, str], None]) -> Tuple[str, str]: - fname = 'test.txt' - tmpdir = self.mkdtemp() - self.write_file([tmpdir, fname]) - file1 = os.path.join(tmpdir, fname) - tmpdir2 = self.mkdtemp() - method(file1, tmpdir2) - file2 = os.path.join(tmpdir2, fname) - return (file1, file2) - - @unittest.skipUnless(hasattr(os, 'chmod'), 'requires os.chmod') - def test_copy(self) -> None: - # Ensure that the copied file exists and has the same mode bits. - file1, file2 = self._copy_file(shutil.copy) - self.assertTrue(os.path.exists(file2)) - self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode) - - @unittest.skipUnless(hasattr(os, 'chmod'), 'requires os.chmod') - @unittest.skipUnless(hasattr(os, 'utime'), 'requires os.utime') - def test_copy2(self) -> None: - # Ensure that the copied file exists and has the same mode and - # modification time bits. - file1, file2 = self._copy_file(shutil.copy2) - self.assertTrue(os.path.exists(file2)) - file1_stat = os.stat(file1) - file2_stat = os.stat(file2) - self.assertEqual(file1_stat.st_mode, file2_stat.st_mode) - for attr in 'st_atime', 'st_mtime': - # The modification times may be truncated in the new file. - self.assertLessEqual(getattr(file1_stat, attr), - getattr(file2_stat, attr) + 1) - if hasattr(os, 'chflags') and hasattr(file1_stat, 'st_flags'): - self.assertEqual(getattr(file1_stat, 'st_flags'), - getattr(file2_stat, 'st_flags')) - - @unittest.skipUnless(zlib, "requires zlib") - def test_make_tarball(self) -> None: - # creating something to tar - tmpdir = self.mkdtemp() - self.write_file([tmpdir, 'file1'], 'xxx') - self.write_file([tmpdir, 'file2'], 'xxx') - os.mkdir(os.path.join(tmpdir, 'sub')) - self.write_file([tmpdir, 'sub', 'file3'], 'xxx') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - unittest.skipUnless(splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0], - "source and target should be on same drive") - - base_name = os.path.join(tmpdir2, 'archive') - - # working with relative paths to avoid tar warnings - old_dir = os.getcwd() - os.chdir(tmpdir) - try: - _make_tarball(splitdrive(base_name)[1], '.') - finally: - os.chdir(old_dir) - - # check if the compressed tarball was created - tarball = base_name + '.tar.gz' - self.assertTrue(os.path.exists(tarball)) - - # trying an uncompressed one - base_name = os.path.join(tmpdir2, 'archive') - old_dir = os.getcwd() - os.chdir(tmpdir) - try: - _make_tarball(splitdrive(base_name)[1], '.', compress=None) - finally: - os.chdir(old_dir) - tarball = base_name + '.tar' - self.assertTrue(os.path.exists(tarball)) - - def _tarinfo(self, path: str) -> tuple: - tar = tarfile.open(path) - try: - names = tar.getnames() - names.sort() - return tuple(names) - finally: - tar.close() - - def _create_files(self) -> Tuple[str, str, str]: - # creating something to tar - tmpdir = self.mkdtemp() - dist = os.path.join(tmpdir, 'dist') - os.mkdir(dist) - self.write_file([dist, 'file1'], 'xxx') - self.write_file([dist, 'file2'], 'xxx') - os.mkdir(os.path.join(dist, 'sub')) - self.write_file([dist, 'sub', 'file3'], 'xxx') - os.mkdir(os.path.join(dist, 'sub2')) - tmpdir2 = self.mkdtemp() - base_name = os.path.join(tmpdir2, 'archive') - return tmpdir, tmpdir2, base_name - - @unittest.skipUnless(zlib, "Requires zlib") - @unittest.skipUnless(find_executable('tar') and find_executable('gzip'), - 'Need the tar command to run') - def test_tarfile_vs_tar(self) -> None: - tmpdir, tmpdir2, base_name = self._create_files() - old_dir = os.getcwd() - os.chdir(tmpdir) - try: - _make_tarball(base_name, 'dist') - finally: - os.chdir(old_dir) - - # check if the compressed tarball was created - tarball = base_name + '.tar.gz' - self.assertTrue(os.path.exists(tarball)) - - # now create another tarball using `tar` - tarball2 = os.path.join(tmpdir, 'archive2.tar.gz') - tar_cmd = ['tar', '-cf', 'archive2.tar', 'dist'] - gzip_cmd = ['gzip', '-f9', 'archive2.tar'] - old_dir = os.getcwd() - os.chdir(tmpdir) - try: - with captured_stdout() as s: - spawn(tar_cmd) - spawn(gzip_cmd) - finally: - os.chdir(old_dir) - - self.assertTrue(os.path.exists(tarball2)) - # let's compare both tarballs - self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2)) - - # trying an uncompressed one - base_name = os.path.join(tmpdir2, 'archive') - old_dir = os.getcwd() - os.chdir(tmpdir) - try: - _make_tarball(base_name, 'dist', compress=None) - finally: - os.chdir(old_dir) - tarball = base_name + '.tar' - self.assertTrue(os.path.exists(tarball)) - - # now for a dry_run - base_name = os.path.join(tmpdir2, 'archive') - old_dir = os.getcwd() - os.chdir(tmpdir) - try: - _make_tarball(base_name, 'dist', compress=None, dry_run=True) - finally: - os.chdir(old_dir) - tarball = base_name + '.tar' - self.assertTrue(os.path.exists(tarball)) - - @unittest.skipUnless(zlib, "Requires zlib") - @unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run') - def test_make_zipfile(self) -> None: - # creating something to tar - tmpdir = self.mkdtemp() - self.write_file([tmpdir, 'file1'], 'xxx') - self.write_file([tmpdir, 'file2'], 'xxx') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - base_name = os.path.join(tmpdir2, 'archive') - _make_zipfile(base_name, tmpdir) - - # check if the compressed tarball was created - tarball = base_name + '.zip' - self.assertTrue(os.path.exists(tarball)) - - - def test_make_archive(self) -> None: - tmpdir = self.mkdtemp() - base_name = os.path.join(tmpdir, 'archive') - self.assertRaises(ValueError, make_archive, base_name, 'xxx') - - @unittest.skipUnless(zlib, "Requires zlib") - def test_make_archive_owner_group(self) -> None: - # testing make_archive with owner and group, with various combinations - # this works even if there's not gid/uid support - if UID_GID_SUPPORT: - group = grp.getgrgid(0).gr_name - owner = pwd.getpwuid(0).pw_name - else: - group = owner = 'root' - - base_dir, root_dir, base_name = self._create_files() - base_name = os.path.join(self.mkdtemp() , 'archive') - res = make_archive(base_name, 'zip', root_dir, base_dir, owner=owner, - group=group) - self.assertTrue(os.path.exists(res)) - - res = make_archive(base_name, 'zip', root_dir, base_dir) - self.assertTrue(os.path.exists(res)) - - res = make_archive(base_name, 'tar', root_dir, base_dir, - owner=owner, group=group) - self.assertTrue(os.path.exists(res)) - - res = make_archive(base_name, 'tar', root_dir, base_dir, - owner='kjhkjhkjg', group='oihohoh') - self.assertTrue(os.path.exists(res)) - - - @unittest.skipUnless(zlib, "Requires zlib") - @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") - def test_tarfile_root_owner(self) -> None: - tmpdir, tmpdir2, base_name = self._create_files() - old_dir = os.getcwd() - os.chdir(tmpdir) - group = grp.getgrgid(0).gr_name - owner = pwd.getpwuid(0).pw_name - try: - archive_name = _make_tarball(base_name, 'dist', compress=None, - owner=owner, group=group) - finally: - os.chdir(old_dir) - - # check if the compressed tarball was created - self.assertTrue(os.path.exists(archive_name)) - - # now checks the rights - archive = tarfile.open(archive_name) - try: - for member in archive.getmembers(): - self.assertEqual(member.uid, 0) - self.assertEqual(member.gid, 0) - finally: - archive.close() - - def test_make_archive_cwd(self) -> None: - current_dir = os.getcwd() - def _breaks(*args: Any, **kw: Any) -> None: - raise RuntimeError() - - register_archive_format('xxx', _breaks, [], 'xxx file') - try: - try: - make_archive('xxx', 'xxx', root_dir=self.mkdtemp()) - except Exception: - pass - self.assertEqual(os.getcwd(), current_dir) - finally: - unregister_archive_format('xxx') - - def test_register_archive_format(self) -> None: - - self.assertRaises(TypeError, register_archive_format, 'xxx', 1) - self.assertRaises(TypeError, register_archive_format, 'xxx', - lambda: 1/0, - 1) - self.assertRaises(TypeError, register_archive_format, 'xxx', - lambda: 1/0, - [(1, 2), (1, 2, 3)]) - - register_archive_format('xxx', lambda: 1/0, [('x', 2)], 'xxx file') - formats = [name for name, params in get_archive_formats()] - self.assertIn('xxx', formats) - - unregister_archive_format('xxx') - formats = [name for name, params in get_archive_formats()] - self.assertNotIn('xxx', formats) - - def _compare_dirs(self, dir1: str, dir2: str) -> List[str]: - # check that dir1 and dir2 are equivalent, - # return the diff - diff = List[str]() - for root, dirs, files in os.walk(dir1): - for file_ in files: - path = os.path.join(root, file_) - target_path = os.path.join(dir2, os.path.split(path)[-1]) - if not os.path.exists(target_path): - diff.append(file_) - return diff - - @unittest.skipUnless(zlib, "Requires zlib") - def test_unpack_archive(self) -> None: - formats = ['tar', 'gztar', 'zip'] - if BZ2_SUPPORTED: - formats.append('bztar') - - for format in formats: - tmpdir = self.mkdtemp() - base_dir, root_dir, base_name = self._create_files() - tmpdir2 = self.mkdtemp() - filename = make_archive(base_name, format, root_dir, base_dir) - - # let's try to unpack it now - unpack_archive(filename, tmpdir2) - diff = self._compare_dirs(tmpdir, tmpdir2) - self.assertEqual(diff, []) - - # and again, this time with the format specified - tmpdir3 = self.mkdtemp() - unpack_archive(filename, tmpdir3, format=format) - diff = self._compare_dirs(tmpdir, tmpdir3) - self.assertEqual(diff, []) - self.assertRaises(shutil.ReadError, unpack_archive, TESTFN) - self.assertRaises(ValueError, unpack_archive, TESTFN, format='xxx') - - def test_unpack_registery(self) -> None: - - formats = get_unpack_formats() - - def _boo(filename: str, extract_dir: str, extra: int) -> None: - self.assertEqual(extra, 1) - self.assertEqual(filename, 'stuff.boo') - self.assertEqual(extract_dir, 'xx') - - register_unpack_format('Boo', ['.boo', '.b2'], _boo, [('extra', 1)]) - unpack_archive('stuff.boo', 'xx') - - # trying to register a .boo unpacker again - self.assertRaises(RegistryError, register_unpack_format, 'Boo2', - ['.boo'], _boo) - - # should work now - unregister_unpack_format('Boo') - register_unpack_format('Boo2', ['.boo'], _boo) - self.assertIn(('Boo2', ['.boo'], ''), get_unpack_formats()) - self.assertNotIn(('Boo', ['.boo'], ''), get_unpack_formats()) - - # let's leave a clean state - unregister_unpack_format('Boo2') - self.assertEqual(get_unpack_formats(), formats) - - -class TestMove(unittest.TestCase): - - def setUp(self) -> None: - filename = "foo" - self.src_dir = tempfile.mkdtemp() - self.dst_dir = tempfile.mkdtemp() - self.src_file = os.path.join(self.src_dir, filename) - self.dst_file = os.path.join(self.dst_dir, filename) - with open(self.src_file, "wb") as f: - f.write(b"spam") - - def tearDown(self) -> None: - for d in (self.src_dir, self.dst_dir): - try: - if d: - shutil.rmtree(d) - except: - pass - - def _check_move_file(self, src: str, dst: str, real_dst: str) -> None: - with open(src, "rb") as f: - contents = f.read() - shutil.move(src, dst) - with open(real_dst, "rb") as f: - self.assertEqual(contents, f.read()) - self.assertFalse(os.path.exists(src)) - - def _check_move_dir(self, src: str, dst: str, real_dst: str) -> None: - contents = sorted(os.listdir(src)) - shutil.move(src, dst) - self.assertEqual(contents, sorted(os.listdir(real_dst))) - self.assertFalse(os.path.exists(src)) - - def test_move_file(self) -> None: - # Move a file to another location on the same filesystem. - self._check_move_file(self.src_file, self.dst_file, self.dst_file) - - def test_move_file_to_dir(self) -> None: - # Move a file inside an existing dir on the same filesystem. - self._check_move_file(self.src_file, self.dst_dir, self.dst_file) - - @mock_rename - def test_move_file_other_fs(self) -> None: - # Move a file to an existing dir on another filesystem. - self.test_move_file() - - @mock_rename - def test_move_file_to_dir_other_fs(self) -> None: - # Move a file to another location on another filesystem. - self.test_move_file_to_dir() - - def test_move_dir(self) -> None: - # Move a dir to another location on the same filesystem. - dst_dir = tempfile.mktemp() - try: - self._check_move_dir(self.src_dir, dst_dir, dst_dir) - finally: - try: - shutil.rmtree(dst_dir) - except: - pass - - @mock_rename - def test_move_dir_other_fs(self) -> None: - # Move a dir to another location on another filesystem. - self.test_move_dir() - - def test_move_dir_to_dir(self) -> None: - # Move a dir inside an existing dir on the same filesystem. - self._check_move_dir(self.src_dir, self.dst_dir, - os.path.join(self.dst_dir, os.path.basename(self.src_dir))) - - @mock_rename - def test_move_dir_to_dir_other_fs(self) -> None: - # Move a dir inside an existing dir on another filesystem. - self.test_move_dir_to_dir() - - def test_existing_file_inside_dest_dir(self) -> None: - # A file with the same name inside the destination dir already exists. - with open(self.dst_file, "wb"): - pass - self.assertRaises(shutil.Error, shutil.move, self.src_file, self.dst_dir) - - def test_dont_move_dir_in_itself(self) -> None: - # Moving a dir inside itself raises an Error. - dst = os.path.join(self.src_dir, "bar") - self.assertRaises(shutil.Error, shutil.move, self.src_dir, dst) - - def test_destinsrc_false_negative(self) -> None: - os.mkdir(TESTFN) - try: - for src, dst in [('srcdir', 'srcdir/dest')]: - src = os.path.join(TESTFN, src) - dst = os.path.join(TESTFN, dst) - self.assertTrue(shutil._destinsrc(src, dst), - msg='_destinsrc() wrongly concluded that ' - 'dst (%s) is not in src (%s)' % (dst, src)) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - def test_destinsrc_false_positive(self) -> None: - os.mkdir(TESTFN) - try: - for src, dst in [('srcdir', 'src/dest'), ('srcdir', 'srcdir.new')]: - src = os.path.join(TESTFN, src) - dst = os.path.join(TESTFN, dst) - self.assertFalse(shutil._destinsrc(src, dst), - msg='_destinsrc() wrongly concluded that ' - 'dst (%s) is in src (%s)' % (dst, src)) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - -class TestCopyFile(unittest.TestCase): - - _delete = False - - @ducktype(IO[str]) - class Faux(object): - _entered = False - _exited_with = None # type: tuple - _raised = False - def __init__(self, raise_in_exit: bool = False, - suppress_at_exit: bool = True) -> None: - self._raise_in_exit = raise_in_exit - self._suppress_at_exit = suppress_at_exit - def read(self, *args: Any) -> str: - return '' - def __enter__(self) -> None: - self._entered = True - def __exit__(self, exc_type: type, exc_val: BaseException, - exc_tb: Traceback) -> bool: - self._exited_with = exc_type, exc_val, exc_tb - if self._raise_in_exit: - self._raised = True - raise IOError("Cannot close") - return self._suppress_at_exit - - def tearDown(self) -> None: - shutil.open = open - - def _set_shutil_open(self, func: Any) -> None: - shutil.open = func - self._delete = True - - def test_w_source_open_fails(self) -> None: - def _open(filename: str, mode: str= 'r') -> BinaryIO: - if filename == 'srcfile': - raise IOError('Cannot open "srcfile"') - assert 0 # shouldn't reach here. - - self._set_shutil_open(_open) - - self.assertRaises(IOError, shutil.copyfile, 'srcfile', 'destfile') - - def test_w_dest_open_fails(self) -> None: - - srcfile = TestCopyFile.Faux() - - def _open(filename: str, mode: str = 'r') -> TestCopyFile.Faux: - if filename == 'srcfile': - return srcfile - if filename == 'destfile': - raise IOError('Cannot open "destfile"') - assert 0 # shouldn't reach here. - - self._set_shutil_open(_open) - - shutil.copyfile('srcfile', 'destfile') - self.assertTrue(srcfile._entered) - self.assertTrue(srcfile._exited_with[0] is IOError) - self.assertEqual(srcfile._exited_with[1].args, - ('Cannot open "destfile"',)) - - def test_w_dest_close_fails(self) -> None: - - srcfile = TestCopyFile.Faux() - destfile = TestCopyFile.Faux(True) - - def _open(filename: str, mode: str = 'r') -> TestCopyFile.Faux: - if filename == 'srcfile': - return srcfile - if filename == 'destfile': - return destfile - assert 0 # shouldn't reach here. - - self._set_shutil_open(_open) - - shutil.copyfile('srcfile', 'destfile') - self.assertTrue(srcfile._entered) - self.assertTrue(destfile._entered) - self.assertTrue(destfile._raised) - self.assertTrue(srcfile._exited_with[0] is IOError) - self.assertEqual(srcfile._exited_with[1].args, - ('Cannot close',)) - - def test_w_source_close_fails(self) -> None: - - srcfile = TestCopyFile.Faux(True) - destfile = TestCopyFile.Faux() - - def _open(filename: str, mode: str= 'r') -> TestCopyFile.Faux: - if filename == 'srcfile': - return srcfile - if filename == 'destfile': - return destfile - assert 0 # shouldn't reach here. - - self._set_shutil_open(_open) - - self.assertRaises(IOError, - shutil.copyfile, 'srcfile', 'destfile') - self.assertTrue(srcfile._entered) - self.assertTrue(destfile._entered) - self.assertFalse(destfile._raised) - self.assertTrue(srcfile._exited_with[0] is None) - self.assertTrue(srcfile._raised) - - def test_move_dir_caseinsensitive(self) -> None: - # Renames a folder to the same name - # but a different case. - - self.src_dir = tempfile.mkdtemp() - dst_dir = os.path.join( - os.path.dirname(self.src_dir), - os.path.basename(self.src_dir).upper()) - self.assertNotEqual(self.src_dir, dst_dir) - - try: - shutil.move(self.src_dir, dst_dir) - self.assertTrue(os.path.isdir(dst_dir)) - finally: - if os.path.exists(dst_dir): - os.rmdir(dst_dir) - - - -def test_main() -> None: - support.run_unittest(TestShutil, TestMove, TestCopyFile) - -if __name__ == '__main__': - test_main() diff --git a/lib-python/3.2/test/test_subprocess.py b/lib-python/3.2/test/test_subprocess.py deleted file mode 100644 index cfd23c2054e3..000000000000 --- a/lib-python/3.2/test/test_subprocess.py +++ /dev/null @@ -1,1764 +0,0 @@ -import unittest -from test import support -import subprocess -import sys -import signal -import io -import os -import errno -import tempfile -import time -import re -import sysconfig -import warnings -import select -import shutil -import gc - -import resource - -from typing import Any, Dict, Function, Iterable, List, Set, Tuple - -mswindows = (sys.platform == "win32") - -# -# Depends on the following external programs: Python -# - -if mswindows: - SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' - 'os.O_BINARY);') -else: - SETBINARY = '' - - -try: - mkstemp = tempfile.mkstemp -except AttributeError: - # tempfile.mkstemp is not available - def _mkstemp() -> Tuple[int, str]: - """Replacement for mkstemp, calling mktemp.""" - fname = tempfile.mktemp() - return os.open(fname, os.O_RDWR|os.O_CREAT), fname - mkstemp = Any(_mkstemp) - - -class BaseTestCase(unittest.TestCase): - def setUp(self) -> None: - # Try to minimize the number of children we have so this test - # doesn't crash on some buildbots (Alphas in particular). - support.reap_children() - - def tearDown(self) -> None: - for inst in subprocess._active: - inst.wait() - subprocess._cleanup() - self.assertFalse(subprocess._active, "subprocess._active not empty") - - def assertStderrEqual(self, stderr: bytes, expected: bytes, - msg: object = None) -> None: - # In a debug build, stuff like "[6580 refs]" is printed to stderr at - # shutdown time. That frustrates tests trying to check stderr produced - # from a spawned Python process. - actual = support.strip_python_stderr(stderr) - self.assertEqual(actual, expected, msg) - - -class ProcessTestCase(BaseTestCase): - - def test_call_seq(self) -> None: - # call() function with sequence argument - rc = subprocess.call([sys.executable, "-c", - "import sys; sys.exit(47)"]) - self.assertEqual(rc, 47) - - def test_check_call_zero(self) -> None: - # check_call() function with zero return code - rc = subprocess.check_call([sys.executable, "-c", - "import sys; sys.exit(0)"]) - self.assertEqual(rc, 0) - - def test_check_call_nonzero(self) -> None: - # check_call() function with non-zero return code - with self.assertRaises(subprocess.CalledProcessError) as c: - subprocess.check_call([sys.executable, "-c", - "import sys; sys.exit(47)"]) - self.assertEqual(c.exception.returncode, 47) - - def test_check_output(self) -> None: - # check_output() function with zero return code - output = subprocess.check_output( - [sys.executable, "-c", "print('BDFL')"]) - self.assertIn(b'BDFL', Any(output)) # see #39 - - def test_check_output_nonzero(self) -> None: - # check_call() function with non-zero return code - with self.assertRaises(subprocess.CalledProcessError) as c: - subprocess.check_output( - [sys.executable, "-c", "import sys; sys.exit(5)"]) - self.assertEqual(c.exception.returncode, 5) - - def test_check_output_stderr(self) -> None: - # check_output() function stderr redirected to stdout - output = subprocess.check_output( - [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], - stderr=subprocess.STDOUT) - self.assertIn(b'BDFL', Any(output)) # see #39 - - def test_check_output_stdout_arg(self) -> None: - # check_output() function stderr redirected to stdout - with self.assertRaises(ValueError) as c: - output = subprocess.check_output( - [sys.executable, "-c", "print('will not be run')"], - stdout=sys.stdout) - self.fail("Expected ValueError when stdout arg supplied.") - self.assertIn('stdout', c.exception.args[0]) - - def test_call_kwargs(self) -> None: - # call() function with keyword args - newenv = os.environ.copy() - newenv["FRUIT"] = "banana" - rc = subprocess.call([sys.executable, "-c", - 'import sys, os;' - 'sys.exit(os.getenv("FRUIT")=="banana")'], - env=newenv) - self.assertEqual(rc, 1) - - def test_invalid_args(self) -> None: - # Popen() called with invalid arguments should raise TypeError - # but Popen.__del__ should not complain (issue #12085) - with support.captured_stderr() as s: - self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) - argcount = subprocess.Popen.__init__.__code__.co_argcount - too_many_args = [0] * (argcount + 1) - self.assertRaises(TypeError, subprocess.Popen, *too_many_args) - self.assertEqual(s.getvalue(), '') - - def test_stdin_none(self) -> None: - # .stdin is None when not redirected - p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - p.wait() - self.assertEqual(p.stdin, None) - - def test_stdout_none(self) -> None: - # .stdout is None when not redirected - p = subprocess.Popen([sys.executable, "-c", - 'print(" this bit of output is from a ' - 'test of stdout in a different ' - 'process ...")'], - stdin=subprocess.PIPE, stderr=subprocess.PIPE) - self.addCleanup(p.stdin.close) - self.addCleanup(p.stderr.close) - p.wait() - self.assertEqual(p.stdout, None) - - def test_stderr_none(self) -> None: - # .stderr is None when not redirected - p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stdin.close) - p.wait() - self.assertEqual(p.stderr, None) - - def test_executable_with_cwd(self) -> None: - python_dir = os.path.dirname(os.path.realpath(sys.executable)) - p = subprocess.Popen(["somethingyoudonthave", "-c", - "import sys; sys.exit(47)"], - executable=sys.executable, cwd=python_dir) - p.wait() - self.assertEqual(p.returncode, 47) - - @unittest.skipIf(sysconfig.is_python_build(), - "need an installed Python. See #7774") - def test_executable_without_cwd(self) -> None: - # For a normal installation, it should work without 'cwd' - # argument. For test runs in the build directory, see #7774. - p = subprocess.Popen(["somethingyoudonthave", "-c", - "import sys; sys.exit(47)"], - executable=sys.executable) - p.wait() - self.assertEqual(p.returncode, 47) - - def test_stdin_pipe(self) -> None: - # stdin redirection - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.exit(sys.stdin.read() == "pear")'], - stdin=subprocess.PIPE) - p.stdin.write(b"pear") - p.stdin.close() - p.wait() - self.assertEqual(p.returncode, 1) - - def test_stdin_filedes(self) -> None: - # stdin is set to open file descriptor - tf = tempfile.TemporaryFile() - self.addCleanup(tf.close) - d = tf.fileno() - os.write(d, b"pear") - os.lseek(d, 0, 0) - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.exit(sys.stdin.read() == "pear")'], - stdin=d) - p.wait() - self.assertEqual(p.returncode, 1) - - def test_stdin_fileobj(self) -> None: - # stdin is set to open file object - tf = tempfile.TemporaryFile() - self.addCleanup(tf.close) - tf.write(b"pear") - tf.seek(0) - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.exit(sys.stdin.read() == "pear")'], - stdin=tf) - p.wait() - self.assertEqual(p.returncode, 1) - - def test_stdout_pipe(self) -> None: - # stdout redirection - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stdout.write("orange")'], - stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read(), b"orange") - - def test_stdout_filedes(self) -> None: - # stdout is set to open file descriptor - tf = tempfile.TemporaryFile() - self.addCleanup(tf.close) - d = tf.fileno() - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stdout.write("orange")'], - stdout=d) - p.wait() - os.lseek(d, 0, 0) - self.assertEqual(os.read(d, 1024), b"orange") - - def test_stdout_fileobj(self) -> None: - # stdout is set to open file object - tf = tempfile.TemporaryFile() - self.addCleanup(tf.close) - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stdout.write("orange")'], - stdout=tf) - p.wait() - tf.seek(0) - self.assertEqual(tf.read(), b"orange") - - def test_stderr_pipe(self) -> None: - # stderr redirection - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stderr.write("strawberry")'], - stderr=subprocess.PIPE) - self.addCleanup(p.stderr.close) - self.assertStderrEqual(p.stderr.read(), b"strawberry") - - def test_stderr_filedes(self) -> None: - # stderr is set to open file descriptor - tf = tempfile.TemporaryFile() - self.addCleanup(tf.close) - d = tf.fileno() - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stderr.write("strawberry")'], - stderr=d) - p.wait() - os.lseek(d, 0, 0) - self.assertStderrEqual(os.read(d, 1024), b"strawberry") - - def test_stderr_fileobj(self) -> None: - # stderr is set to open file object - tf = tempfile.TemporaryFile() - self.addCleanup(tf.close) - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stderr.write("strawberry")'], - stderr=tf) - p.wait() - tf.seek(0) - self.assertStderrEqual(tf.read(), b"strawberry") - - def test_stdout_stderr_pipe(self) -> None: - # capture stdout and stderr to the same pipe - p = subprocess.Popen([sys.executable, "-c", - 'import sys;' - 'sys.stdout.write("apple");' - 'sys.stdout.flush();' - 'sys.stderr.write("orange")'], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - self.addCleanup(p.stdout.close) - self.assertStderrEqual(p.stdout.read(), b"appleorange") - - def test_stdout_stderr_file(self) -> None: - # capture stdout and stderr to the same open file - tf = tempfile.TemporaryFile() - self.addCleanup(tf.close) - p = subprocess.Popen([sys.executable, "-c", - 'import sys;' - 'sys.stdout.write("apple");' - 'sys.stdout.flush();' - 'sys.stderr.write("orange")'], - stdout=tf, - stderr=tf) - p.wait() - tf.seek(0) - self.assertStderrEqual(tf.read(), b"appleorange") - - def test_stdout_filedes_of_stdout(self) -> None: - # stdout is set to 1 (#1531862). - cmd = r"import sys, os; sys.exit(os.write(sys.stdout.fileno(), b'.\n'))" - rc = subprocess.call([sys.executable, "-c", cmd], stdout=1) - self.assertEqual(rc, 2) - - def test_cwd(self) -> None: - tmpdir = tempfile.gettempdir() - # We cannot use os.path.realpath to canonicalize the path, - # since it doesn't expand Tru64 {memb} strings. See bug 1063571. - cwd = os.getcwd() - os.chdir(tmpdir) - tmpdir = os.getcwd() - os.chdir(cwd) - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' - 'sys.stdout.write(os.getcwd())'], - stdout=subprocess.PIPE, - cwd=tmpdir) - self.addCleanup(p.stdout.close) - normcase = os.path.normcase - self.assertEqual(normcase(p.stdout.read().decode("utf-8")), - normcase(tmpdir)) - - def test_env(self) -> None: - newenv = os.environ.copy() - newenv["FRUIT"] = "orange" - with subprocess.Popen([sys.executable, "-c", - 'import sys,os;' - 'sys.stdout.write(os.getenv("FRUIT"))'], - stdout=subprocess.PIPE, - env=newenv) as p: - stdout, stderr = p.communicate() - self.assertEqual(stdout, b"orange") - - # Windows requires at least the SYSTEMROOT environment variable to start - # Python - @unittest.skipIf(sys.platform == 'win32', - 'cannot test an empty env on Windows') - @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') is not None, - 'the python library cannot be loaded ' - 'with an empty environment') - def test_empty_env(self) -> None: - with subprocess.Popen([sys.executable, "-c", - 'import os; ' - 'print(list(os.environ.keys()))'], - stdout=subprocess.PIPE, - env={}) as p: - stdout, stderr = p.communicate() - self.assertIn(stdout.strip(), - [b"[]", - # Mac OS X adds __CF_USER_TEXT_ENCODING variable to an empty - # environment - b"['__CF_USER_TEXT_ENCODING']"]) - - def test_communicate_stdin(self) -> None: - p = subprocess.Popen([sys.executable, "-c", - 'import sys;' - 'sys.exit(sys.stdin.read() == "pear")'], - stdin=subprocess.PIPE) - p.communicate(b"pear") - self.assertEqual(p.returncode, 1) - - def test_communicate_stdout(self) -> None: - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stdout.write("pineapple")'], - stdout=subprocess.PIPE) - (stdout, stderr) = p.communicate() - self.assertEqual(stdout, b"pineapple") - self.assertEqual(stderr, None) - - def test_communicate_stderr(self) -> None: - p = subprocess.Popen([sys.executable, "-c", - 'import sys; sys.stderr.write("pineapple")'], - stderr=subprocess.PIPE) - (stdout, stderr) = p.communicate() - self.assertEqual(stdout, None) - self.assertStderrEqual(stderr, b"pineapple") - - def test_communicate(self) -> None: - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' - 'sys.stderr.write("pineapple");' - 'sys.stdout.write(sys.stdin.read())'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - (stdout, stderr) = p.communicate(b"banana") - self.assertEqual(stdout, b"banana") - self.assertStderrEqual(stderr, b"pineapple") - - # Test for the fd leak reported in http://bugs.python.org/issue2791. - def test_communicate_pipe_fd_leak(self) -> None: - for stdin_pipe in (False, True): - for stdout_pipe in (False, True): - for stderr_pipe in (False, True): - options = Dict[str, Any]() - if stdin_pipe: - options['stdin'] = subprocess.PIPE - if stdout_pipe: - options['stdout'] = subprocess.PIPE - if stderr_pipe: - options['stderr'] = subprocess.PIPE - if not options: - continue - p = subprocess.Popen([sys.executable, "-c", "pass"], **options) - p.communicate() - if p.stdin is not None: - self.assertTrue(p.stdin.closed) - if p.stdout is not None: - self.assertTrue(p.stdout.closed) - if p.stderr is not None: - self.assertTrue(p.stderr.closed) - - def test_communicate_returns(self) -> None: - # communicate() should return None if no redirection is active - p = subprocess.Popen([sys.executable, "-c", - "import sys; sys.exit(47)"]) - (stdout, stderr) = p.communicate() - self.assertEqual(stdout, None) - self.assertEqual(stderr, None) - - def test_communicate_pipe_buf(self) -> None: - # communicate() with writes larger than pipe_buf - # This test will probably deadlock rather than fail, if - # communicate() does not work properly. - x, y = os.pipe() - if mswindows: - pipe_buf = 512 - else: - pipe_buf = os.fpathconf(x, "PC_PIPE_BUF") - os.close(x) - os.close(y) - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' - 'sys.stdout.write(sys.stdin.read(47));' - 'sys.stderr.write("xyz"*%d);' - 'sys.stdout.write(sys.stdin.read())' % pipe_buf], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - string_to_write = b"abc"*pipe_buf - (stdout, stderr) = p.communicate(string_to_write) - self.assertEqual(stdout, string_to_write) - - def test_writes_before_communicate(self) -> None: - # stdin.write before communicate() - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' - 'sys.stdout.write(sys.stdin.read())'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - p.stdin.write(b"banana") - (stdout, stderr) = p.communicate(b"split") - self.assertEqual(stdout, b"bananasplit") - self.assertStderrEqual(stderr, b"") - - def test_universal_newlines(self) -> None: - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' + SETBINARY + - 'sys.stdout.write(sys.stdin.readline());' - 'sys.stdout.flush();' - 'sys.stdout.write("line2\\n");' - 'sys.stdout.flush();' - 'sys.stdout.write(sys.stdin.read());' - 'sys.stdout.flush();' - 'sys.stdout.write("line4\\n");' - 'sys.stdout.flush();' - 'sys.stdout.write("line5\\r\\n");' - 'sys.stdout.flush();' - 'sys.stdout.write("line6\\r");' - 'sys.stdout.flush();' - 'sys.stdout.write("\\nline7");' - 'sys.stdout.flush();' - 'sys.stdout.write("\\nline8");'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=1) - p.stdin.write("line1\n") - self.assertEqual(p.stdout.readline(), "line1\n") - p.stdin.write("line3\n") - p.stdin.close() - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.readline(), - "line2\n") - self.assertEqual(p.stdout.read(6), - "line3\n") - self.assertEqual(p.stdout.read(), - "line4\nline5\nline6\nline7\nline8") - - def test_universal_newlines_communicate(self) -> None: - # universal newlines through communicate() - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' + SETBINARY + - 'sys.stdout.write("line2\\n");' - 'sys.stdout.flush();' - 'sys.stdout.write("line4\\n");' - 'sys.stdout.flush();' - 'sys.stdout.write("line5\\r\\n");' - 'sys.stdout.flush();' - 'sys.stdout.write("line6\\r");' - 'sys.stdout.flush();' - 'sys.stdout.write("\\nline7");' - 'sys.stdout.flush();' - 'sys.stdout.write("\\nline8");'], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=1) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - # BUG: can't give a non-empty stdin because it breaks both the - # select- and poll-based communicate() implementations. - (stdout, stderr) = p.communicate() - self.assertEqual(stdout, - "line2\nline4\nline5\nline6\nline7\nline8") - - def test_universal_newlines_communicate_stdin(self) -> None: - # universal newlines through communicate(), with only stdin - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' + SETBINARY + '''\nif True: - s = sys.stdin.readline() - assert s == "line1\\n", repr(s) - s = sys.stdin.read() - assert s == "line3\\n", repr(s) - '''], - stdin=subprocess.PIPE, - universal_newlines=1) - (stdout, stderr) = p.communicate("line1\nline3\n") - self.assertEqual(p.returncode, 0) - - def test_no_leaking(self) -> None: - # Make sure we leak no resources - if not mswindows: - max_handles = 1026 # too much for most UNIX systems - else: - max_handles = 2050 # too much for (at least some) Windows setups - handles = List[int]() - tmpdir = tempfile.mkdtemp() - try: - for i in range(max_handles): - try: - tmpfile = os.path.join(tmpdir, support.TESTFN) - handles.append(os.open(tmpfile, os.O_WRONLY|os.O_CREAT)) - except OSError as e: - if e.errno != errno.EMFILE: - raise - break - else: - self.skipTest("failed to reach the file descriptor limit " - "(tried %d)" % max_handles) - # Close a couple of them (should be enough for a subprocess) - for i in range(10): - os.close(handles.pop()) - # Loop creating some subprocesses. If one of them leaks some fds, - # the next loop iteration will fail by reaching the max fd limit. - for i in range(15): - p = subprocess.Popen([sys.executable, "-c", - "import sys;" - "sys.stdout.write(sys.stdin.read())"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - data = p.communicate(b"lime")[0] - self.assertEqual(data, b"lime") - finally: - for h in handles: - os.close(h) - shutil.rmtree(tmpdir) - - def test_list2cmdline(self) -> None: - self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), - '"a b c" d e') - self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), - 'ab\\"c \\ d') - self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), - 'ab\\"c " \\\\" d') - self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), - 'a\\\\\\b "de fg" h') - self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), - 'a\\\\\\"b c d') - self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), - '"a\\\\b c" d e') - self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), - '"a\\\\b\\ c" d e') - self.assertEqual(subprocess.list2cmdline(['ab', '']), - 'ab ""') - - - def test_poll(self) -> None: - p = subprocess.Popen([sys.executable, - "-c", "import time; time.sleep(1)"]) - count = 0 - while p.poll() is None: - time.sleep(0.1) - count += 1 - # We expect that the poll loop probably went around about 10 times, - # but, based on system scheduling we can't control, it's possible - # poll() never returned None. It "should be" very rare that it - # didn't go around at least twice. - self.assertGreaterEqual(count, 2) - # Subsequent invocations should just return the returncode - self.assertEqual(p.poll(), 0) - - - def test_wait(self) -> None: - p = subprocess.Popen([sys.executable, - "-c", "import time; time.sleep(2)"]) - self.assertEqual(p.wait(), 0) - # Subsequent invocations should just return the returncode - self.assertEqual(p.wait(), 0) - - - def test_invalid_bufsize(self) -> None: - # an invalid type of the bufsize argument should raise - # TypeError. - with self.assertRaises(TypeError): - subprocess.Popen([sys.executable, "-c", "pass"], Any("orange")) - - def test_bufsize_is_none(self) -> None: - # bufsize=None should be the same as bufsize=0. - p = subprocess.Popen([sys.executable, "-c", "pass"], None) - self.assertEqual(p.wait(), 0) - # Again with keyword arg - p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None) - self.assertEqual(p.wait(), 0) - - def test_leaking_fds_on_error(self) -> None: - # see bug #5179: Popen leaks file descriptors to PIPEs if - # the child fails to execute; this will eventually exhaust - # the maximum number of open fds. 1024 seems a very common - # value for that limit, but Windows has 2048, so we loop - # 1024 times (each call leaked two fds). - for i in range(1024): - # Windows raises IOError. Others raise OSError. - with self.assertRaises(EnvironmentError) as c: - subprocess.Popen(['nonexisting_i_hope'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - # ignore errors that indicate the command was not found - if c.exception.errno not in (errno.ENOENT, errno.EACCES): - raise c.exception - - def test_issue8780(self) -> None: - # Ensure that stdout is inherited from the parent - # if stdout=PIPE is not used - code = ';'.join([ - 'import subprocess, sys', - 'retcode = subprocess.call(' - "[sys.executable, '-c', 'print(\"Hello World!\")'])", - 'assert retcode == 0']) - output = subprocess.check_output([sys.executable, '-c', code]) - self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) - - def test_handles_closed_on_exception(self) -> None: - # If CreateProcess exits with an error, ensure the - # duplicate output handles are released - ifhandle, ifname = mkstemp() - ofhandle, ofname = mkstemp() - efhandle, efname = mkstemp() - try: - subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, - stderr=efhandle) - except OSError: - os.close(ifhandle) - os.remove(ifname) - os.close(ofhandle) - os.remove(ofname) - os.close(efhandle) - os.remove(efname) - self.assertFalse(os.path.exists(ifname)) - self.assertFalse(os.path.exists(ofname)) - self.assertFalse(os.path.exists(efname)) - - def test_communicate_epipe(self) -> None: - # Issue 10963: communicate() should hide EPIPE - p = subprocess.Popen([sys.executable, "-c", 'pass'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - p.communicate(b"x" * 2**20) - - def test_communicate_epipe_only_stdin(self) -> None: - # Issue 10963: communicate() should hide EPIPE - p = subprocess.Popen([sys.executable, "-c", 'pass'], - stdin=subprocess.PIPE) - self.addCleanup(p.stdin.close) - time.sleep(2) - p.communicate(b"x" * 2**20) - - @unittest.skipUnless(hasattr(signal, 'SIGALRM'), - "Requires signal.SIGALRM") - def test_communicate_eintr(self) -> None: - # Issue #12493: communicate() should handle EINTR - def handler(signum, frame): - pass - old_handler = signal.signal(signal.SIGALRM, handler) - self.addCleanup(signal.signal, signal.SIGALRM, old_handler) - - # the process is running for 2 seconds - args = [sys.executable, "-c", 'import time; time.sleep(2)'] - for stream in ('stdout', 'stderr'): - kw = Dict[str, Any]({stream: subprocess.PIPE}) - with subprocess.Popen(args, **kw) as process: - signal.alarm(1) - # communicate() will be interrupted by SIGALRM - process.communicate() - - -# context manager -class _SuppressCoreFiles(object): - """Try to prevent core files from being created.""" - old_limit = None # type: Tuple[int, int] - - def __enter__(self) -> None: - """Try to save previous ulimit, then set it to (0, 0).""" - if resource is not None: - try: - self.old_limit = resource.getrlimit(resource.RLIMIT_CORE) - resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) - except (ValueError, resource.error): - pass - - if sys.platform == 'darwin': - # Check if the 'Crash Reporter' on OSX was configured - # in 'Developer' mode and warn that it will get triggered - # when it is. - # - # This assumes that this context manager is used in tests - # that might trigger the next manager. - value = subprocess.Popen(['/usr/bin/defaults', 'read', - 'com.apple.CrashReporter', 'DialogType'], - stdout=subprocess.PIPE).communicate()[0] - if value.strip() == b'developer': - print("this tests triggers the Crash Reporter, " - "that is intentional", end='') - sys.stdout.flush() - - def __exit__(self, *args: Any) -> None: - """Return core file behavior to default.""" - if self.old_limit is None: - return - if resource is not None: - try: - resource.setrlimit(resource.RLIMIT_CORE, self.old_limit) - except (ValueError, resource.error): - pass - - -@unittest.skipIf(mswindows, "POSIX specific tests") -class POSIXProcessTestCase(BaseTestCase): - - def test_exceptions(self) -> None: - nonexistent_dir = "/_this/pa.th/does/not/exist" - try: - os.chdir(nonexistent_dir) - except OSError as e: - # This avoids hard coding the errno value or the OS perror() - # string and instead capture the exception that we want to see - # below for comparison. - desired_exception = e - desired_exception.strerror += ': ' + repr(sys.executable) - else: - self.fail("chdir to nonexistant directory %s succeeded." % - nonexistent_dir) - - # Error in the child re-raised in the parent. - try: - p = subprocess.Popen([sys.executable, "-c", ""], - cwd=nonexistent_dir) - except OSError as e: - # Test that the child process chdir failure actually makes - # it up to the parent process as the correct exception. - self.assertEqual(desired_exception.errno, e.errno) - self.assertEqual(desired_exception.strerror, e.strerror) - else: - self.fail("Expected OSError: %s" % desired_exception) - - def test_restore_signals(self) -> None: - # Code coverage for both values of restore_signals to make sure it - # at least does not blow up. - # A test for behavior would be complex. Contributions welcome. - subprocess.call([sys.executable, "-c", ""], restore_signals=True) - subprocess.call([sys.executable, "-c", ""], restore_signals=False) - - def test_start_new_session(self) -> None: - # For code coverage of calling setsid(). We don't care if we get an - # EPERM error from it depending on the test execution environment, that - # still indicates that it was called. - try: - output = subprocess.check_output( - [sys.executable, "-c", - "import os; print(os.getpgid(os.getpid()))"], - start_new_session=True) - except OSError as e: - if e.errno != errno.EPERM: - raise - else: - parent_pgid = os.getpgid(os.getpid()) - child_pgid = int(output) - self.assertNotEqual(parent_pgid, child_pgid) - - def test_run_abort(self) -> None: - # returncode handles signal termination - with _SuppressCoreFiles(): - p = subprocess.Popen([sys.executable, "-c", - 'import os; os.abort()']) - p.wait() - self.assertEqual(-p.returncode, signal.SIGABRT) - - def test_preexec(self) -> None: - # DISCLAIMER: Setting environment variables is *not* a good use - # of a preexec_fn. This is merely a test. - p = subprocess.Popen([sys.executable, "-c", - 'import sys,os;' - 'sys.stdout.write(os.getenv("FRUIT"))'], - stdout=subprocess.PIPE, - preexec_fn=lambda: os.putenv("FRUIT", "apple")) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read(), b"apple") - - def test_preexec_exception(self) -> None: - def raise_it(): - raise ValueError("What if two swallows carried a coconut?") - try: - p = subprocess.Popen([sys.executable, "-c", ""], - preexec_fn=raise_it) - except RuntimeError as e: - self.assertTrue( - subprocess._posixsubprocess, - "Expected a ValueError from the preexec_fn") - except ValueError as e2: - self.assertIn("coconut", e2.args[0]) - else: - self.fail("Exception raised by preexec_fn did not make it " - "to the parent process.") - - def test_preexec_gc_module_failure(self) -> None: - # This tests the code that disables garbage collection if the child - # process will execute any Python. - def raise_runtime_error(): - raise RuntimeError("this shouldn't escape") - enabled = gc.isenabled() - orig_gc_disable = gc.disable - orig_gc_isenabled = gc.isenabled - try: - gc.disable() - self.assertFalse(gc.isenabled()) - subprocess.call([sys.executable, '-c', ''], - preexec_fn=lambda: None) - self.assertFalse(gc.isenabled(), - "Popen enabled gc when it shouldn't.") - - gc.enable() - self.assertTrue(gc.isenabled()) - subprocess.call([sys.executable, '-c', ''], - preexec_fn=lambda: None) - self.assertTrue(gc.isenabled(), "Popen left gc disabled.") - - setattr(gc, 'disable', raise_runtime_error) - self.assertRaises(RuntimeError, subprocess.Popen, - [sys.executable, '-c', ''], - preexec_fn=lambda: None) - - del gc.isenabled # force an AttributeError - self.assertRaises(AttributeError, subprocess.Popen, - [sys.executable, '-c', ''], - preexec_fn=lambda: None) - finally: - setattr(gc, 'disable', orig_gc_disable) - setattr(gc, 'isenabled', orig_gc_isenabled) - if not enabled: - gc.disable() - - def test_args_string(self) -> None: - # args is a string - fd, fname = mkstemp() - # reopen in text mode - with open(fd, "w", errors=Any("surrogateescape")) as fobj: # see #260 - fobj.write("#!/bin/sh\n") - fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % - sys.executable) - os.chmod(fname, 0o700) - p = subprocess.Popen(fname) - p.wait() - os.remove(fname) - self.assertEqual(p.returncode, 47) - - def test_invalid_args(self) -> None: - # invalid arguments should raise ValueError - self.assertRaises(ValueError, subprocess.call, - [sys.executable, "-c", - "import sys; sys.exit(47)"], - startupinfo=47) - self.assertRaises(ValueError, subprocess.call, - [sys.executable, "-c", - "import sys; sys.exit(47)"], - creationflags=47) - - def test_shell_sequence(self) -> None: - # Run command through the shell (sequence) - newenv = os.environ.copy() - newenv["FRUIT"] = "apple" - p = subprocess.Popen(["echo $FRUIT"], shell=1, - stdout=subprocess.PIPE, - env=newenv) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") - - def test_shell_string(self) -> None: - # Run command through the shell (string) - newenv = os.environ.copy() - newenv["FRUIT"] = "apple" - p = subprocess.Popen("echo $FRUIT", shell=1, - stdout=subprocess.PIPE, - env=newenv) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") - - def test_call_string(self) -> None: - # call() function with string argument on UNIX - fd, fname = mkstemp() - # reopen in text mode - with open(fd, "w", errors=Any("surrogateescape")) as fobj: # see #260 - fobj.write("#!/bin/sh\n") - fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % - sys.executable) - os.chmod(fname, 0o700) - rc = subprocess.call(fname) - os.remove(fname) - self.assertEqual(rc, 47) - - def test_specific_shell(self) -> None: - # Issue #9265: Incorrect name passed as arg[0]. - shells = List[str]() - for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: - for name in ['bash', 'ksh']: - sh = os.path.join(prefix, name) - if os.path.isfile(sh): - shells.append(sh) - if not shells: # Will probably work for any shell but csh. - self.skipTest("bash or ksh required for this test") - sh = '/bin/sh' - if os.path.isfile(sh) and not os.path.islink(sh): - # Test will fail if /bin/sh is a symlink to csh. - shells.append(sh) - for sh in shells: - p = subprocess.Popen("echo $0", executable=sh, shell=True, - stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) - - def _kill_process(self, method: str, *args: Any) -> subprocess.Popen: - # Do not inherit file handles from the parent. - # It should fix failures on some platforms. - p = subprocess.Popen([sys.executable, "-c", """if 1: - import sys, time - sys.stdout.write('x\\n') - sys.stdout.flush() - time.sleep(30) - """], - close_fds=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - # Wait for the interpreter to be completely initialized before - # sending any signal. - p.stdout.read(1) - getattr(p, method)(*args) - return p - - def test_send_signal(self) -> None: - p = self._kill_process('send_signal', signal.SIGINT) - _, stderr = p.communicate() - self.assertIn(b'KeyboardInterrupt', stderr) - self.assertNotEqual(p.wait(), 0) - - def test_kill(self) -> None: - p = self._kill_process('kill') - _, stderr = p.communicate() - self.assertStderrEqual(stderr, b'') - self.assertEqual(p.wait(), -signal.SIGKILL) - - def test_terminate(self) -> None: - p = self._kill_process('terminate') - _, stderr = p.communicate() - self.assertStderrEqual(stderr, b'') - self.assertEqual(p.wait(), -signal.SIGTERM) - - def check_close_std_fds(self, fds: Iterable[int]) -> None: - # Issue #9905: test that subprocess pipes still work properly with - # some standard fds closed - stdin = 0 - newfds = List[int]() - for a in fds: - b = os.dup(a) - newfds.append(b) - if a == 0: - stdin = b - try: - for fd in fds: - os.close(fd) - out, err = subprocess.Popen([sys.executable, "-c", - 'import sys;' - 'sys.stdout.write("apple");' - 'sys.stdout.flush();' - 'sys.stderr.write("orange")'], - stdin=stdin, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - err = support.strip_python_stderr(err) - self.assertEqual((out, err), (b'apple', b'orange')) - finally: - for b, a in zip(newfds, fds): - os.dup2(b, a) - for b in newfds: - os.close(b) - - def test_close_fd_0(self) -> None: - self.check_close_std_fds([0]) - - def test_close_fd_1(self) -> None: - self.check_close_std_fds([1]) - - def test_close_fd_2(self) -> None: - self.check_close_std_fds([2]) - - def test_close_fds_0_1(self) -> None: - self.check_close_std_fds([0, 1]) - - def test_close_fds_0_2(self) -> None: - self.check_close_std_fds([0, 2]) - - def test_close_fds_1_2(self) -> None: - self.check_close_std_fds([1, 2]) - - def test_close_fds_0_1_2(self) -> None: - # Issue #10806: test that subprocess pipes still work properly with - # all standard fds closed. - self.check_close_std_fds([0, 1, 2]) - - def test_remapping_std_fds(self) -> None: - # open up some temporary files - temps = [mkstemp() for i in range(3)] - try: - temp_fds = [fd for fd, fname in temps] - - # unlink the files -- we won't need to reopen them - for fd, fname in temps: - os.unlink(fname) - - # write some data to what will become stdin, and rewind - os.write(temp_fds[1], b"STDIN") - os.lseek(temp_fds[1], 0, 0) - - # move the standard file descriptors out of the way - saved_fds = [os.dup(fd) for fd in range(3)] - try: - # duplicate the file objects over the standard fd's - for fd, temp_fd in enumerate(temp_fds): - os.dup2(temp_fd, fd) - - # now use those files in the "wrong" order, so that subprocess - # has to rearrange them in the child - p = subprocess.Popen([sys.executable, "-c", - 'import sys; got = sys.stdin.read();' - 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], - stdin=temp_fds[1], - stdout=temp_fds[2], - stderr=temp_fds[0]) - p.wait() - finally: - # restore the original fd's underneath sys.stdin, etc. - for std, saved in enumerate(saved_fds): - os.dup2(saved, std) - os.close(saved) - - for fd in temp_fds: - os.lseek(fd, 0, 0) - - out = os.read(temp_fds[2], 1024) - err = support.strip_python_stderr(os.read(temp_fds[0], 1024)) - self.assertEqual(out, b"got STDIN") - self.assertEqual(err, b"err") - - finally: - for fd in temp_fds: - os.close(fd) - - def check_swap_fds(self, stdin_no: int, stdout_no: int, - stderr_no: int) -> None: - # open up some temporary files - temps = [mkstemp() for i in range(3)] - temp_fds = [fd for fd, fname in temps] - try: - # unlink the files -- we won't need to reopen them - for fd, fname in temps: - os.unlink(fname) - - # save a copy of the standard file descriptors - saved_fds = [os.dup(fd) for fd in range(3)] - try: - # duplicate the temp files over the standard fd's 0, 1, 2 - for fd, temp_fd in enumerate(temp_fds): - os.dup2(temp_fd, fd) - - # write some data to what will become stdin, and rewind - os.write(stdin_no, b"STDIN") - os.lseek(stdin_no, 0, 0) - - # now use those files in the given order, so that subprocess - # has to rearrange them in the child - p = subprocess.Popen([sys.executable, "-c", - 'import sys; got = sys.stdin.read();' - 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], - stdin=stdin_no, - stdout=stdout_no, - stderr=stderr_no) - p.wait() - - for fd in temp_fds: - os.lseek(fd, 0, 0) - - out = os.read(stdout_no, 1024) - err = support.strip_python_stderr(os.read(stderr_no, 1024)) - finally: - for std, saved in enumerate(saved_fds): - os.dup2(saved, std) - os.close(saved) - - self.assertEqual(out, b"got STDIN") - self.assertEqual(err, b"err") - - finally: - for fd in temp_fds: - os.close(fd) - - # When duping fds, if there arises a situation where one of the fds is - # either 0, 1 or 2, it is possible that it is overwritten (#12607). - # This tests all combinations of this. - def test_swap_fds(self) -> None: - self.check_swap_fds(0, 1, 2) - self.check_swap_fds(0, 2, 1) - self.check_swap_fds(1, 0, 2) - self.check_swap_fds(1, 2, 0) - self.check_swap_fds(2, 0, 1) - self.check_swap_fds(2, 1, 0) - - def test_surrogates_error_message(self) -> None: - def prepare() -> None: - raise ValueError("surrogate:\uDCff") - - try: - subprocess.call( - [sys.executable, "-c", "pass"], - preexec_fn=prepare) - except ValueError as err: - # Pure Python implementations keeps the message - self.assertIsNone(subprocess._posixsubprocess) - self.assertEqual(str(err), "surrogate:\uDCff") - except RuntimeError as err2: - # _posixsubprocess uses a default message - self.assertIsNotNone(subprocess._posixsubprocess) - self.assertEqual(str(err2), "Exception occurred in preexec_fn.") - else: - self.fail("Expected ValueError or RuntimeError") - - def test_undecodable_env(self) -> None: - for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): - # test str with surrogates - script = "import os; print(ascii(os.getenv(%s)))" % repr(key) - env = os.environ.copy() - env[key] = value - # Use C locale to get ascii for the locale encoding to force - # surrogate-escaping of \xFF in the child process; otherwise it can - # be decoded as-is if the default locale is latin-1. - env['LC_ALL'] = 'C' - stdout = subprocess.check_output( - [sys.executable, "-c", script], - env=env) - stdout = stdout.rstrip(b'\n\r') - self.assertEqual(stdout.decode('ascii'), ascii(value)) - - # test bytes - keyb = key.encode("ascii", "surrogateescape") - valueb = value.encode("ascii", "surrogateescape") - script = "import os; print(ascii(os.getenvb(%s)))" % repr(keyb) - envb = Dict[Any, Any](os.environ.copy().items()) - envb[keyb] = valueb - stdout = subprocess.check_output( - [sys.executable, "-c", script], - env=envb) - stdout = stdout.rstrip(b'\n\r') - self.assertEqual(stdout.decode('ascii'), ascii(valueb)) - - def test_bytes_program(self) -> None: - abs_program = os.fsencode(sys.executable) - path, programs = os.path.split(sys.executable) - program = os.fsencode(programs) - - # absolute bytes path - exitcode = subprocess.call([abs_program, "-c", "pass"]) - self.assertEqual(exitcode, 0) - - # bytes program, unicode PATH - env = os.environ.copy() - env["PATH"] = path - exitcode = subprocess.call([program, "-c", "pass"], env=env) - self.assertEqual(exitcode, 0) - - # bytes program, bytes PATH - envb = os.environb.copy() - envb[b"PATH"] = os.fsencode(path) - exitcode = subprocess.call([program, "-c", "pass"], env=envb) - self.assertEqual(exitcode, 0) - - def test_pipe_cloexec(self) -> None: - sleeper = support.findfile("input_reader.py", subdir="subprocessdata") - fd_status = support.findfile("fd_status.py", subdir="subprocessdata") - - p1 = subprocess.Popen([sys.executable, sleeper], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, close_fds=False) - - self.addCleanup(p1.communicate, b'') - - p2 = subprocess.Popen([sys.executable, fd_status], - stdout=subprocess.PIPE, close_fds=False) - - output, error = p2.communicate() - result_fds = set(map(int, output.split(b','))) - unwanted_fds = set([p1.stdin.fileno(), p1.stdout.fileno(), - p1.stderr.fileno()]) - - self.assertFalse(result_fds & unwanted_fds, - "Expected no fds from %r to be open in child, " - "found %r" % - (unwanted_fds, result_fds & unwanted_fds)) - - def test_pipe_cloexec_real_tools(self) -> None: - qcat = support.findfile("qcat.py", subdir="subprocessdata") - qgrep = support.findfile("qgrep.py", subdir="subprocessdata") - - subdata = b'zxcvbn' - data = subdata * 4 + b'\n' - - p1 = subprocess.Popen([sys.executable, qcat], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - close_fds=False) - - p2 = subprocess.Popen([sys.executable, qgrep, subdata], - stdin=p1.stdout, stdout=subprocess.PIPE, - close_fds=False) - - self.addCleanup(p1.wait) - self.addCleanup(p2.wait) - def kill_p1() -> None: - #try: - p1.terminate() - #except ProcessLookupError: - # pass - def kill_p2() -> None: - #try: - p2.terminate() - #except ProcessLookupError: - # pass - self.addCleanup(kill_p1) - self.addCleanup(kill_p2) - - p1.stdin.write(data) - p1.stdin.close() - - readfiles, ignored1, ignored2 = select.select([p2.stdout], [], [], 10) - - self.assertTrue(readfiles, "The child hung") - self.assertEqual(p2.stdout.read(), data) - - p1.stdout.close() - p2.stdout.close() - - def test_close_fds(self) -> None: - fd_status = support.findfile("fd_status.py", subdir="subprocessdata") - - fds = os.pipe() - self.addCleanup(os.close, fds[0]) - self.addCleanup(os.close, fds[1]) - - open_fds = set([fds[0], fds[1]]) - # add a bunch more fds - for _ in range(9): - fd = os.open("/dev/null", os.O_RDONLY) - self.addCleanup(os.close, fd) - open_fds.add(fd) - - p = subprocess.Popen([sys.executable, fd_status], - stdout=subprocess.PIPE, close_fds=False) - output, ignored = p.communicate() - remaining_fds = set(map(int, output.split(b','))) - - self.assertEqual(remaining_fds & open_fds, open_fds, - "Some fds were closed") - - p = subprocess.Popen([sys.executable, fd_status], - stdout=subprocess.PIPE, close_fds=True) - output, ignored = p.communicate() - remaining_fds = set(map(int, output.split(b','))) - - self.assertFalse(remaining_fds & open_fds, - "Some fds were left open") - self.assertIn(1, remaining_fds, "Subprocess failed") - - # Keep some of the fd's we opened open in the subprocess. - # This tests _posixsubprocess.c's proper handling of fds_to_keep. - fds_to_keep = set(open_fds.pop() for _ in range(8)) - p = subprocess.Popen([sys.executable, fd_status], - stdout=subprocess.PIPE, close_fds=True, - pass_fds=()) - output, ignored = p.communicate() - remaining_fds = set(map(int, output.split(b','))) - - self.assertFalse(remaining_fds & fds_to_keep & open_fds, - "Some fds not in pass_fds were left open") - self.assertIn(1, remaining_fds, "Subprocess failed") - - # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file - # descriptor of a pipe closed in the parent process is valid in the - # child process according to fstat(), but the mode of the file - # descriptor is invalid, and read or write raise an error. - @support.requires_mac_ver(10, 5) - def test_pass_fds(self) -> None: - fd_status = support.findfile("fd_status.py", subdir="subprocessdata") - - open_fds = Set[int]() - - for x in range(5): - fds = os.pipe() - self.addCleanup(os.close, fds[0]) - self.addCleanup(os.close, fds[1]) - open_fds.update([fds[0], fds[1]]) - - for fd in open_fds: - p = subprocess.Popen([sys.executable, fd_status], - stdout=subprocess.PIPE, close_fds=True, - pass_fds=(fd, )) - output, ignored = p.communicate() - - remaining_fds = set(map(int, output.split(b','))) - to_be_closed = open_fds - {fd} - - self.assertIn(fd, remaining_fds, "fd to be passed not passed") - self.assertFalse(remaining_fds & to_be_closed, - "fd to be closed passed") - - # pass_fds overrides close_fds with a warning. - with self.assertWarns(RuntimeWarning) as context: - self.assertFalse(subprocess.call( - [sys.executable, "-c", "import sys; sys.exit(0)"], - close_fds=False, pass_fds=(fd, ))) - self.assertIn('overriding close_fds', str(context.warning)) - - def test_stdout_stdin_are_single_inout_fd(self) -> None: - with io.open(os.devnull, "r+") as inout: - p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], - stdout=inout, stdin=inout) - p.wait() - - def test_stdout_stderr_are_single_inout_fd(self) -> None: - with io.open(os.devnull, "r+") as inout: - p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], - stdout=inout, stderr=inout) - p.wait() - - def test_stderr_stdin_are_single_inout_fd(self) -> None: - with io.open(os.devnull, "r+") as inout: - p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], - stderr=inout, stdin=inout) - p.wait() - - def test_wait_when_sigchild_ignored(self) -> None: - # NOTE: sigchild_ignore.py may not be an effective test on all OSes. - sigchild_ignore = support.findfile("sigchild_ignore.py", - subdir="subprocessdata") - p = subprocess.Popen([sys.executable, sigchild_ignore], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" - " non-zero with this error:\n%s" % - stderr.decode('utf8')) - - def test_select_unbuffered(self) -> None: - # Issue #11459: bufsize=0 should really set the pipes as - # unbuffered (and therefore let select() work properly). - select = support.import_module("select") - p = subprocess.Popen([sys.executable, "-c", - 'import sys;' - 'sys.stdout.write("apple")'], - stdout=subprocess.PIPE, - bufsize=0) - f = p.stdout - self.addCleanup(f.close) - try: - self.assertEqual(f.read(4), b"appl") - self.assertIn(f, select.select([f], [], [], 0.0)[0]) - finally: - p.wait() - - def test_zombie_fast_process_del(self) -> None: - # Issue #12650: on Unix, if Popen.__del__() was called before the - # process exited, it wouldn't be added to subprocess._active, and would - # remain a zombie. - # spawn a Popen, and delete its reference before it exits - p = subprocess.Popen([sys.executable, "-c", - 'import sys, time;' - 'time.sleep(0.2)'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - ident = id(p) - pid = p.pid - del p - # check that p is in the active processes list - self.assertIn(ident, [id(o) for o in subprocess._active]) - - def test_leak_fast_process_del_killed(self) -> None: - # Issue #12650: on Unix, if Popen.__del__() was called before the - # process exited, and the process got killed by a signal, it would never - # be removed from subprocess._active, which triggered a FD and memory - # leak. - # spawn a Popen, delete its reference and kill it - p = subprocess.Popen([sys.executable, "-c", - 'import time;' - 'time.sleep(3)'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - ident = id(p) - pid = p.pid - del p - os.kill(pid, signal.SIGKILL) - # check that p is in the active processes list - self.assertIn(ident, [id(o) for o in subprocess._active]) - - # let some time for the process to exit, and create a new Popen: this - # should trigger the wait() of p - time.sleep(0.2) - with self.assertRaises(EnvironmentError) as c: - with subprocess.Popen(['nonexisting_i_hope'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: - pass - # p should have been wait()ed on, and removed from the _active list - self.assertRaises(OSError, os.waitpid, pid, 0) - self.assertNotIn(ident, [id(o) for o in subprocess._active]) - - -@unittest.skipUnless(mswindows, "Windows specific tests") -class Win32ProcessTestCase(BaseTestCase): - - def test_startupinfo(self) -> None: - # startupinfo argument - # We uses hardcoded constants, because we do not want to - # depend on win32all. - STARTF_USESHOWWINDOW = 1 - SW_MAXIMIZE = 3 - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags = STARTF_USESHOWWINDOW - startupinfo.wShowWindow = SW_MAXIMIZE - # Since Python is a console process, it won't be affected - # by wShowWindow, but the argument should be silently - # ignored - subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], - startupinfo=startupinfo) - - def test_creationflags(self) -> None: - # creationflags argument - CREATE_NEW_CONSOLE = 16 - sys.stderr.write(" a DOS box should flash briefly ...\n") - subprocess.call(sys.executable + - ' -c "import time; time.sleep(0.25)"', - creationflags=CREATE_NEW_CONSOLE) - - def test_invalid_args(self) -> None: - # invalid arguments should raise ValueError - self.assertRaises(ValueError, subprocess.call, - [sys.executable, "-c", - "import sys; sys.exit(47)"], - preexec_fn=lambda: 1) - self.assertRaises(ValueError, subprocess.call, - [sys.executable, "-c", - "import sys; sys.exit(47)"], - stdout=subprocess.PIPE, - close_fds=True) - - def test_close_fds(self) -> None: - # close file descriptors - rc = subprocess.call([sys.executable, "-c", - "import sys; sys.exit(47)"], - close_fds=True) - self.assertEqual(rc, 47) - - def test_shell_sequence(self) -> None: - # Run command through the shell (sequence) - newenv = os.environ.copy() - newenv["FRUIT"] = "physalis" - p = subprocess.Popen(["set"], shell=1, - stdout=subprocess.PIPE, - env=newenv) - self.addCleanup(p.stdout.close) - self.assertIn(b"physalis", p.stdout.read()) - - def test_shell_string(self) -> None: - # Run command through the shell (string) - newenv = os.environ.copy() - newenv["FRUIT"] = "physalis" - p = subprocess.Popen("set", shell=1, - stdout=subprocess.PIPE, - env=newenv) - self.addCleanup(p.stdout.close) - self.assertIn(b"physalis", p.stdout.read()) - - def test_call_string(self) -> None: - # call() function with string argument on Windows - rc = subprocess.call(sys.executable + - ' -c "import sys; sys.exit(47)"') - self.assertEqual(rc, 47) - - def _kill_process(self, method: str, *args: Any) -> None: - # Some win32 buildbot raises EOFError if stdin is inherited - p = subprocess.Popen([sys.executable, "-c", """if 1: - import sys, time - sys.stdout.write('x\\n') - sys.stdout.flush() - time.sleep(30) - """], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - # Wait for the interpreter to be completely initialized before - # sending any signal. - p.stdout.read(1) - getattr(p, method)(*args) - _, stderr = p.communicate() - self.assertStderrEqual(stderr, b'') - returncode = p.wait() - self.assertNotEqual(returncode, 0) - - def test_send_signal(self) -> None: - self._kill_process('send_signal', signal.SIGTERM) - - def test_kill(self) -> None: - self._kill_process('kill') - - def test_terminate(self) -> None: - self._kill_process('terminate') - - -# The module says: -# "NB This only works (and is only relevant) for UNIX." -# -# Actually, getoutput should work on any platform with an os.popen, but -# I'll take the comment as given, and skip this suite. -@unittest.skipUnless(os.name == 'posix', "only relevant for UNIX") -class CommandTests(unittest.TestCase): - def test_getoutput(self) -> None: - self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') - self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), - (0, 'xyzzy')) - - # we use mkdtemp in the next line to create an empty directory - # under our exclusive control; from that, we can invent a pathname - # that we _know_ won't exist. This is guaranteed to fail. - dir = None # type: str - try: - dir = tempfile.mkdtemp() - name = os.path.join(dir, "foo") - - status, output = subprocess.getstatusoutput('cat ' + name) - self.assertNotEqual(status, 0) - finally: - if dir is not None: - os.rmdir(dir) - - -@unittest.skipUnless(getattr(subprocess, '_has_poll', False), - "poll system call not supported") -class ProcessTestCaseNoPoll(ProcessTestCase): - def setUp(self) -> None: - subprocess._has_poll = False - ProcessTestCase.setUp(self) - - def tearDown(self) -> None: - subprocess._has_poll = True - ProcessTestCase.tearDown(self) - - -#@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False), -# "_posixsubprocess extension module not found.") -#class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase): -# @classmethod -# def setUpClass(cls): -# global subprocess -# assert subprocess._posixsubprocess -# # Reimport subprocess while forcing _posixsubprocess to not exist. -# with support.check_warnings(('.*_posixsubprocess .* not being used.*', -# RuntimeWarning)): -# subprocess = support.import_fresh_module( -# 'subprocess', blocked=['_posixsubprocess']) -# assert not subprocess._posixsubprocess -# -# @classmethod -# def tearDownClass(cls): -# global subprocess -# # Reimport subprocess as it should be, restoring order to the universe#. -# subprocess = support.import_fresh_module('subprocess') -# assert subprocess._posixsubprocess - - -class HelperFunctionTests(unittest.TestCase): - @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") - def test_eintr_retry_call(self) -> None: - record_calls = List[Any]() - def fake_os_func(*args: Any) -> tuple: - record_calls.append(args) - if len(record_calls) == 2: - raise OSError(errno.EINTR, "fake interrupted system call") - return tuple(reversed(args)) - - self.assertEqual((999, 256), - subprocess._eintr_retry_call(fake_os_func, 256, 999)) - self.assertEqual([(256, 999)], record_calls) - # This time there will be an EINTR so it will loop once. - self.assertEqual((666,), - subprocess._eintr_retry_call(fake_os_func, 666)) - self.assertEqual([(256, 999), (666,), (666,)], record_calls) - - -@unittest.skipUnless(mswindows, "Windows-specific tests") -class CommandsWithSpaces (BaseTestCase): - - def setUp(self) -> None: - super().setUp() - f, fname = mkstemp(".py", "te st") - self.fname = fname.lower () - os.write(f, b"import sys;" - b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" - ) - os.close(f) - - def tearDown(self) -> None: - os.remove(self.fname) - super().tearDown() - - def with_spaces(self, *args: Any, **kwargs: Any) -> None: - kwargs['stdout'] = subprocess.PIPE - p = subprocess.Popen(*args, **kwargs) - self.addCleanup(p.stdout.close) - self.assertEqual( - p.stdout.read ().decode("mbcs"), - "2 [%r, 'ab cd']" % self.fname - ) - - def test_shell_string_with_spaces(self) -> None: - # call() function with string argument with spaces on Windows - self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, - "ab cd"), shell=1) - - def test_shell_sequence_with_spaces(self) -> None: - # call() function with sequence argument with spaces on Windows - self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) - - def test_noshell_string_with_spaces(self) -> None: - # call() function with string argument with spaces on Windows - self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, - "ab cd")) - - def test_noshell_sequence_with_spaces(self) -> None: - # call() function with sequence argument with spaces on Windows - self.with_spaces([sys.executable, self.fname, "ab cd"]) - - -class ContextManagerTests(BaseTestCase): - - def test_pipe(self) -> None: - with subprocess.Popen([sys.executable, "-c", - "import sys;" - "sys.stdout.write('stdout');" - "sys.stderr.write('stderr');"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: - self.assertEqual(proc.stdout.read(), b"stdout") - self.assertStderrEqual(proc.stderr.read(), b"stderr") - - self.assertTrue(proc.stdout.closed) - self.assertTrue(proc.stderr.closed) - - def test_returncode(self) -> None: - with subprocess.Popen([sys.executable, "-c", - "import sys; sys.exit(100)"]) as proc: - pass - # __exit__ calls wait(), so the returncode should be set - self.assertEqual(proc.returncode, 100) - - def test_communicate_stdin(self) -> None: - with subprocess.Popen([sys.executable, "-c", - "import sys;" - "sys.exit(sys.stdin.read() == 'context')"], - stdin=subprocess.PIPE) as proc: - proc.communicate(b"context") - self.assertEqual(proc.returncode, 1) - - def test_invalid_args(self) -> None: - with self.assertRaises(EnvironmentError) as c: - with subprocess.Popen(['nonexisting_i_hope'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: - pass - - if c.exception.errno != errno.ENOENT: # ignore "no such file" - raise c.exception - - -def test_main(): - unit_tests = (ProcessTestCase, - POSIXProcessTestCase, - Win32ProcessTestCase, - #ProcessTestCasePOSIXPurePython, - CommandTests, - ProcessTestCaseNoPoll, - HelperFunctionTests, - CommandsWithSpaces, - ContextManagerTests, - ) - - support.run_unittest(*unit_tests) - support.reap_children() - -if __name__ == "__main__": - unittest.main() diff --git a/lib-python/3.2/test/test_tempfile.py b/lib-python/3.2/test/test_tempfile.py deleted file mode 100644 index 2b3926623bea..000000000000 --- a/lib-python/3.2/test/test_tempfile.py +++ /dev/null @@ -1,1122 +0,0 @@ -# tempfile.py unit tests. -import tempfile -import os -import signal -import sys -import re -import warnings - -import unittest -from test import support - -from typing import Any, AnyStr, List, Dict, IO - - -if hasattr(os, 'stat'): - import stat - has_stat = 1 -else: - has_stat = 0 - -has_textmode = (tempfile._text_openflags != tempfile._bin_openflags) -has_spawnl = hasattr(os, 'spawnl') - -# TEST_FILES may need to be tweaked for systems depending on the maximum -# number of files that can be opened at one time (see ulimit -n) -if sys.platform in ('openbsd3', 'openbsd4'): - TEST_FILES = 48 -else: - TEST_FILES = 100 - -# This is organized as one test for each chunk of code in tempfile.py, -# in order of their appearance in the file. Testing which requires -# threads is not done here. - -# Common functionality. -class TC(unittest.TestCase): - - str_check = re.compile(r"[a-zA-Z0-9_-]{6}$") - - def setUp(self) -> None: - self._warnings_manager = support.check_warnings() - self._warnings_manager.__enter__() - warnings.filterwarnings("ignore", category=RuntimeWarning, - message="mktemp", module=__name__) - - def tearDown(self) -> None: - self._warnings_manager.__exit__(None, None, None) - - - def failOnException(self, what: str, ei: tuple = None) -> None: - if ei is None: - ei = sys.exc_info() - self.fail("%s raised %s: %s" % (what, ei[0], ei[1])) - - def nameCheck(self, name: str, dir: str, pre: str, suf: str) -> None: - (ndir, nbase) = os.path.split(name) - npre = nbase[:len(pre)] - nsuf = nbase[len(nbase)-len(suf):] - - # check for equality of the absolute paths! - self.assertEqual(os.path.abspath(ndir), os.path.abspath(dir), - "file '%s' not in directory '%s'" % (name, dir)) - self.assertEqual(npre, pre, - "file '%s' does not begin with '%s'" % (nbase, pre)) - self.assertEqual(nsuf, suf, - "file '%s' does not end with '%s'" % (nbase, suf)) - - nbase = nbase[len(pre):len(nbase)-len(suf)] - self.assertTrue(self.str_check.match(nbase), - "random string '%s' does not match /^[a-zA-Z0-9_-]{6}$/" - % nbase) - -test_classes = [] # type: List[type] - -class test_exports(TC): - def test_exports(self) -> None: - # There are no surprising symbols in the tempfile module - dict = tempfile.__dict__ - - expected = { - "NamedTemporaryFile" : 1, - "TemporaryFile" : 1, - "mkstemp" : 1, - "mkdtemp" : 1, - "mktemp" : 1, - "TMP_MAX" : 1, - "gettempprefix" : 1, - "gettempdir" : 1, - "tempdir" : 1, - "template" : 1, - "SpooledTemporaryFile" : 1, - "TemporaryDirectory" : 1, - } - - unexp = List[str]() - for key in dict: - if key[0] != '_' and key not in expected: - unexp.append(key) - self.assertTrue(len(unexp) == 0, - "unexpected keys: %s" % unexp) - -test_classes.append(test_exports) - - -class test__RandomNameSequence(TC): - """Test the internal iterator object _RandomNameSequence.""" - - def setUp(self) -> None: - self.r = tempfile._RandomNameSequence() - super().setUp() - - def test_get_six_char_str(self) -> None: - # _RandomNameSequence returns a six-character string - s = next(self.r) - self.nameCheck(s, '', '', '') - - def test_many(self) -> None: - # _RandomNameSequence returns no duplicate strings (stochastic) - - dict = Dict[str, int]() - r = self.r - for i in range(TEST_FILES): - s = next(r) - self.nameCheck(s, '', '', '') - self.assertNotIn(s, dict) - dict[s] = 1 - - def supports_iter(self) -> None: - # _RandomNameSequence supports the iterator protocol - - i = 0 - r = self.r - try: - for s in r: - i += 1 - if i == 20: - break - except: - self.failOnException("iteration") - - @unittest.skipUnless(hasattr(os, 'fork'), - "os.fork is required for this test") - def test_process_awareness(self) -> None: - # ensure that the random source differs between - # child and parent. - read_fd, write_fd = os.pipe() - pid = None # type: int - try: - pid = os.fork() - if not pid: - os.close(read_fd) - os.write(write_fd, next(self.r).encode("ascii")) - os.close(write_fd) - # bypass the normal exit handlers- leave those to - # the parent. - os._exit(0) - parent_value = next(self.r) - child_value = os.read(read_fd, len(parent_value)).decode("ascii") - finally: - if pid: - # best effort to ensure the process can't bleed out - # via any bugs above - try: - os.kill(pid, signal.SIGKILL) - except EnvironmentError: - pass - os.close(read_fd) - os.close(write_fd) - self.assertNotEqual(child_value, parent_value) - - -test_classes.append(test__RandomNameSequence) - - -class test__candidate_tempdir_list(TC): - """Test the internal function _candidate_tempdir_list.""" - - def test_nonempty_list(self) -> None: - # _candidate_tempdir_list returns a nonempty list of strings - - cand = tempfile._candidate_tempdir_list() - - self.assertFalse(len(cand) == 0) - for c in cand: - self.assertIsInstance(c, str) - - def test_wanted_dirs(self) -> None: - # _candidate_tempdir_list contains the expected directories - - # Make sure the interesting environment variables are all set. - with support.EnvironmentVarGuard() as env: - for envname in 'TMPDIR', 'TEMP', 'TMP': - dirname = os.getenv(envname) - if not dirname: - env[envname] = os.path.abspath(envname) - - cand = tempfile._candidate_tempdir_list() - - for envname in 'TMPDIR', 'TEMP', 'TMP': - dirname = os.getenv(envname) - if not dirname: raise ValueError - self.assertIn(dirname, cand) - - try: - dirname = os.getcwd() - except (AttributeError, os.error): - dirname = os.curdir - - self.assertIn(dirname, cand) - - # Not practical to try to verify the presence of OS-specific - # paths in this list. - -test_classes.append(test__candidate_tempdir_list) - - -# We test _get_default_tempdir by testing gettempdir. - - -class test__get_candidate_names(TC): - """Test the internal function _get_candidate_names.""" - - def test_retval(self) -> None: - # _get_candidate_names returns a _RandomNameSequence object - obj = tempfile._get_candidate_names() - self.assertIsInstance(obj, tempfile._RandomNameSequence) - - def test_same_thing(self) -> None: - # _get_candidate_names always returns the same object - a = tempfile._get_candidate_names() - b = tempfile._get_candidate_names() - - self.assertTrue(a is b) - -test_classes.append(test__get_candidate_names) - - -class test__mkstemp_inner(TC): - """Test the internal function _mkstemp_inner.""" - - class mkstemped: - _bflags = tempfile._bin_openflags - _tflags = tempfile._text_openflags - - def __init__(self, dir: str, pre: str, suf: str, bin: int) -> None: - if bin: flags = self._bflags - else: flags = self._tflags - - (self.fd, self.name) = tempfile._mkstemp_inner(dir, pre, suf, flags) - - self._close = os.close - self._unlink = os.unlink - - def write(self, str: bytes) -> None: - os.write(self.fd, str) - - def __del__(self) -> None: - self._close(self.fd) - self._unlink(self.name) - - def do_create(self, dir: str = None, pre: str = "", suf: str= "", - bin: int = 1) -> mkstemped: - if dir is None: - dir = tempfile.gettempdir() - try: - file = test__mkstemp_inner.mkstemped(dir, pre, suf, bin) # see #259 - except: - self.failOnException("_mkstemp_inner") - - self.nameCheck(file.name, dir, pre, suf) - return file - - def test_basic(self) -> None: - # _mkstemp_inner can create files - self.do_create().write(b"blat") - self.do_create(pre="a").write(b"blat") - self.do_create(suf="b").write(b"blat") - self.do_create(pre="a", suf="b").write(b"blat") - self.do_create(pre="aa", suf=".txt").write(b"blat") - - def test_basic_many(self) -> None: - # _mkstemp_inner can create many files (stochastic) - extant = List[Any](range(TEST_FILES)) - for i in extant: - extant[i] = self.do_create(pre="aa") - - def test_choose_directory(self) -> None: - # _mkstemp_inner can create files in a user-selected directory - dir = tempfile.mkdtemp() - try: - self.do_create(dir=dir).write(b"blat") - finally: - os.rmdir(dir) - - def test_file_mode(self) -> None: - # _mkstemp_inner creates files with the proper mode - if not has_stat: - return # ugh, can't use SkipTest. - - file = self.do_create() - mode = stat.S_IMODE(os.stat(file.name).st_mode) - expected = 0o600 - if sys.platform in ('win32', 'os2emx'): - # There's no distinction among 'user', 'group' and 'world'; - # replicate the 'user' bits. - user = expected >> 6 - expected = user * (1 + 8 + 64) - self.assertEqual(mode, expected) - - def test_noinherit(self) -> None: - # _mkstemp_inner file handles are not inherited by child processes - if not has_spawnl: - return # ugh, can't use SkipTest. - - if support.verbose: - v="v" - else: - v="q" - - file = self.do_create() - fd = "%d" % file.fd - - try: - me = __file__ # type: str - except NameError: - me = sys.argv[0] - - # We have to exec something, so that FD_CLOEXEC will take - # effect. The core of this test is therefore in - # tf_inherit_check.py, which see. - tester = os.path.join(os.path.dirname(os.path.abspath(me)), - "tf_inherit_check.py") - - # On Windows a spawn* /path/ with embedded spaces shouldn't be quoted, - # but an arg with embedded spaces should be decorated with double - # quotes on each end - if sys.platform in ('win32',): - decorated = '"%s"' % sys.executable - tester = '"%s"' % tester - else: - decorated = sys.executable - - retval = os.spawnl(os.P_WAIT, sys.executable, decorated, tester, v, fd) - self.assertFalse(retval < 0, - "child process caught fatal signal %d" % -retval) - self.assertFalse(retval > 0, "child process reports failure %d"%retval) - - def test_textmode(self) -> None: - # _mkstemp_inner can create files in text mode - if not has_textmode: - return # ugh, can't use SkipTest. - - # A text file is truncated at the first Ctrl+Z byte - f = self.do_create(bin=0) - f.write(b"blat\x1a") - f.write(b"extra\n") - os.lseek(f.fd, 0, os.SEEK_SET) - self.assertEqual(os.read(f.fd, 20), b"blat") - -test_classes.append(test__mkstemp_inner) - - -class test_gettempprefix(TC): - """Test gettempprefix().""" - - def test_sane_template(self) -> None: - # gettempprefix returns a nonempty prefix string - p = tempfile.gettempprefix() - - self.assertIsInstance(p, str) - self.assertTrue(len(p) > 0) - - def test_usable_template(self) -> None: - # gettempprefix returns a usable prefix string - - # Create a temp directory, avoiding use of the prefix. - # Then attempt to create a file whose name is - # prefix + 'xxxxxx.xxx' in that directory. - p = tempfile.gettempprefix() + "xxxxxx.xxx" - d = tempfile.mkdtemp(prefix="") - try: - p = os.path.join(d, p) - try: - fd = os.open(p, os.O_RDWR | os.O_CREAT) - except: - self.failOnException("os.open") - os.close(fd) - os.unlink(p) - finally: - os.rmdir(d) - -test_classes.append(test_gettempprefix) - - -class test_gettempdir(TC): - """Test gettempdir().""" - - def test_directory_exists(self) -> None: - # gettempdir returns a directory which exists - - dir = tempfile.gettempdir() - self.assertTrue(os.path.isabs(dir) or dir == os.curdir, - "%s is not an absolute path" % dir) - self.assertTrue(os.path.isdir(dir), - "%s is not a directory" % dir) - - def test_directory_writable(self) -> None: - # gettempdir returns a directory writable by the user - - # sneaky: just instantiate a NamedTemporaryFile, which - # defaults to writing into the directory returned by - # gettempdir. - try: - file = tempfile.NamedTemporaryFile() - file.write(b"blat") - file.close() - except: - self.failOnException("create file in %s" % tempfile.gettempdir()) - - def test_same_thing(self) -> None: - # gettempdir always returns the same object - a = tempfile.gettempdir() - b = tempfile.gettempdir() - - self.assertTrue(a is b) - -test_classes.append(test_gettempdir) - - -class test_mkstemp(TC): - """Test mkstemp().""" - - def do_create(self, dir: str = None, pre: str = "", suf: str = "") -> None: - if dir is None: - dir = tempfile.gettempdir() - try: - (fd, name) = tempfile.mkstemp(dir=dir, prefix=pre, suffix=suf) - (ndir, nbase) = os.path.split(name) - adir = os.path.abspath(dir) - self.assertEqual(adir, ndir, - "Directory '%s' incorrectly returned as '%s'" % (adir, ndir)) - except: - self.failOnException("mkstemp") - - try: - self.nameCheck(name, dir, pre, suf) - finally: - os.close(fd) - os.unlink(name) - - def test_basic(self) -> None: - # mkstemp can create files - self.do_create() - self.do_create(pre="a") - self.do_create(suf="b") - self.do_create(pre="a", suf="b") - self.do_create(pre="aa", suf=".txt") - self.do_create(dir=".") - - def test_choose_directory(self) -> None: - # mkstemp can create directories in a user-selected directory - dir = tempfile.mkdtemp() - try: - self.do_create(dir=dir) - finally: - os.rmdir(dir) - -test_classes.append(test_mkstemp) - - -class test_mkdtemp(TC): - """Test mkdtemp().""" - - def do_create(self, dir: str = None, pre: str = "", suf: str = "") -> str: - if dir is None: - dir = tempfile.gettempdir() - try: - name = tempfile.mkdtemp(dir=dir, prefix=pre, suffix=suf) - except: - self.failOnException("mkdtemp") - - try: - self.nameCheck(name, dir, pre, suf) - return name - except: - os.rmdir(name) - raise - - def test_basic(self) -> None: - # mkdtemp can create directories - os.rmdir(self.do_create()) - os.rmdir(self.do_create(pre="a")) - os.rmdir(self.do_create(suf="b")) - os.rmdir(self.do_create(pre="a", suf="b")) - os.rmdir(self.do_create(pre="aa", suf=".txt")) - - def test_basic_many(self) -> None: - # mkdtemp can create many directories (stochastic) - extant = List[Any](range(TEST_FILES)) - try: - for i in extant: - extant[i] = self.do_create(pre="aa") - finally: - for i in extant: - if(isinstance(i, str)): - os.rmdir(i) - - def test_choose_directory(self) -> None: - # mkdtemp can create directories in a user-selected directory - dir = tempfile.mkdtemp() - try: - os.rmdir(self.do_create(dir=dir)) - finally: - os.rmdir(dir) - - def test_mode(self) -> None: - # mkdtemp creates directories with the proper mode - if not has_stat: - return # ugh, can't use SkipTest. - - dir = self.do_create() - try: - mode = stat.S_IMODE(os.stat(dir).st_mode) - mode &= 0o777 # Mask off sticky bits inherited from /tmp - expected = 0o700 - if sys.platform in ('win32', 'os2emx'): - # There's no distinction among 'user', 'group' and 'world'; - # replicate the 'user' bits. - user = expected >> 6 - expected = user * (1 + 8 + 64) - self.assertEqual(mode, expected) - finally: - os.rmdir(dir) - -test_classes.append(test_mkdtemp) - - -class test_mktemp(TC): - """Test mktemp().""" - - # For safety, all use of mktemp must occur in a private directory. - # We must also suppress the RuntimeWarning it generates. - def setUp(self) -> None: - self.dir = tempfile.mkdtemp() - super().setUp() - - def tearDown(self) -> None: - if self.dir: - os.rmdir(self.dir) - self.dir = None - super().tearDown() - - class mktemped: - def _unlink(self, path: str) -> None: - os.unlink(path) - - _bflags = tempfile._bin_openflags - - def __init__(self, dir: str, pre: str, suf: str) -> None: - self.name = tempfile.mktemp(dir=dir, prefix=pre, suffix=suf) - # Create the file. This will raise an exception if it's - # mysteriously appeared in the meanwhile. - os.close(os.open(self.name, self._bflags, 0o600)) - - def __del__(self) -> None: - self._unlink(self.name) - - def do_create(self, pre: str = "", suf: str = "") -> mktemped: - try: - file = test_mktemp.mktemped(self.dir, pre, suf) # see #259 - except: - self.failOnException("mktemp") - - self.nameCheck(file.name, self.dir, pre, suf) - return file - - def test_basic(self) -> None: - # mktemp can choose usable file names - self.do_create() - self.do_create(pre="a") - self.do_create(suf="b") - self.do_create(pre="a", suf="b") - self.do_create(pre="aa", suf=".txt") - - def test_many(self) -> None: - # mktemp can choose many usable file names (stochastic) - extant = List[Any](range(TEST_FILES)) - for i in extant: - extant[i] = self.do_create(pre="aa") - -## def test_warning(self): -## # mktemp issues a warning when used -## warnings.filterwarnings("error", -## category=RuntimeWarning, -## message="mktemp") -## self.assertRaises(RuntimeWarning, -## tempfile.mktemp, dir=self.dir) - -test_classes.append(test_mktemp) - - -# We test _TemporaryFileWrapper by testing NamedTemporaryFile. - - -class test_NamedTemporaryFile(TC): - """Test NamedTemporaryFile().""" - - def do_create(self, dir: str = None, pre: str = "", suf: str = "", - delete: bool = True) -> IO[Any]: - if dir is None: - dir = tempfile.gettempdir() - try: - file = tempfile.NamedTemporaryFile(dir=dir, prefix=pre, suffix=suf, - delete=delete) - except: - self.failOnException("NamedTemporaryFile") - - self.nameCheck(file.name, dir, pre, suf) - return file - - - def test_basic(self) -> None: - # NamedTemporaryFile can create files - self.do_create() - self.do_create(pre="a") - self.do_create(suf="b") - self.do_create(pre="a", suf="b") - self.do_create(pre="aa", suf=".txt") - - def test_creates_named(self) -> None: - # NamedTemporaryFile creates files with names - f = tempfile.NamedTemporaryFile() - self.assertTrue(os.path.exists(f.name), - "NamedTemporaryFile %s does not exist" % f.name) - - def test_del_on_close(self) -> None: - # A NamedTemporaryFile is deleted when closed - dir = tempfile.mkdtemp() - try: - f = tempfile.NamedTemporaryFile(dir=dir) - f.write(b'blat') - f.close() - self.assertFalse(os.path.exists(f.name), - "NamedTemporaryFile %s exists after close" % f.name) - finally: - os.rmdir(dir) - - def test_dis_del_on_close(self) -> None: - # Tests that delete-on-close can be disabled - dir = tempfile.mkdtemp() - tmp = None # type: str - try: - f = tempfile.NamedTemporaryFile(dir=dir, delete=False) - tmp = f.name - f.write(b'blat') - f.close() - self.assertTrue(os.path.exists(f.name), - "NamedTemporaryFile %s missing after close" % f.name) - finally: - if tmp is not None: - os.unlink(tmp) - os.rmdir(dir) - - def test_multiple_close(self) -> None: - # A NamedTemporaryFile can be closed many times without error - f = tempfile.NamedTemporaryFile() - f.write(b'abc\n') - f.close() - try: - f.close() - f.close() - except: - self.failOnException("close") - - def test_context_manager(self) -> None: - # A NamedTemporaryFile can be used as a context manager - with tempfile.NamedTemporaryFile() as f: - self.assertTrue(os.path.exists(f.name)) - self.assertFalse(os.path.exists(f.name)) - def use_closed(): - with f: - pass - self.assertRaises(ValueError, use_closed) - - # How to test the mode and bufsize parameters? - -test_classes.append(test_NamedTemporaryFile) - -class test_SpooledTemporaryFile(TC): - """Test SpooledTemporaryFile().""" - - def do_create(self, max_size: int = 0, dir: str = None, pre: str = "", - suf: str = "") -> tempfile.SpooledTemporaryFile: - if dir is None: - dir = tempfile.gettempdir() - try: - file = tempfile.SpooledTemporaryFile(max_size=max_size, dir=dir, prefix=pre, suffix=suf) - except: - self.failOnException("SpooledTemporaryFile") - - return file - - - def test_basic(self) -> None: - # SpooledTemporaryFile can create files - f = self.do_create() - self.assertFalse(f._rolled) - f = self.do_create(max_size=100, pre="a", suf=".txt") - self.assertFalse(f._rolled) - - def test_del_on_close(self) -> None: - # A SpooledTemporaryFile is deleted when closed - dir = tempfile.mkdtemp() - try: - f = tempfile.SpooledTemporaryFile(max_size=10, dir=dir) - self.assertFalse(f._rolled) - f.write(b'blat ' * 5) - self.assertTrue(f._rolled) - filename = f.name - f.close() - self.assertFalse(isinstance(filename, str) and os.path.exists(filename), - "SpooledTemporaryFile %s exists after close" % filename) - finally: - os.rmdir(dir) - - def test_rewrite_small(self) -> None: - # A SpooledTemporaryFile can be written to multiple within the max_size - f = self.do_create(max_size=30) - self.assertFalse(f._rolled) - for i in range(5): - f.seek(0, 0) - f.write(b'x' * 20) - self.assertFalse(f._rolled) - - def test_write_sequential(self) -> None: - # A SpooledTemporaryFile should hold exactly max_size bytes, and roll - # over afterward - f = self.do_create(max_size=30) - self.assertFalse(f._rolled) - f.write(b'x' * 20) - self.assertFalse(f._rolled) - f.write(b'x' * 10) - self.assertFalse(f._rolled) - f.write(b'x') - self.assertTrue(f._rolled) - - def test_writelines(self) -> None: - # Verify writelines with a SpooledTemporaryFile - f = self.do_create() - f.writelines([b'x', b'y', b'z']) - f.seek(0) - buf = f.read() - self.assertEqual(buf, b'xyz') - - def test_writelines_sequential(self) -> None: - # A SpooledTemporaryFile should hold exactly max_size bytes, and roll - # over afterward - f = self.do_create(max_size=35) - f.writelines([b'x' * 20, b'x' * 10, b'x' * 5]) - self.assertFalse(f._rolled) - f.write(b'x') - self.assertTrue(f._rolled) - - def test_sparse(self) -> None: - # A SpooledTemporaryFile that is written late in the file will extend - # when that occurs - f = self.do_create(max_size=30) - self.assertFalse(f._rolled) - f.seek(100, 0) - self.assertFalse(f._rolled) - f.write(b'x') - self.assertTrue(f._rolled) - - def test_fileno(self) -> None: - # A SpooledTemporaryFile should roll over to a real file on fileno() - f = self.do_create(max_size=30) - self.assertFalse(f._rolled) - self.assertTrue(f.fileno() > 0) - self.assertTrue(f._rolled) - - def test_multiple_close_before_rollover(self) -> None: - # A SpooledTemporaryFile can be closed many times without error - f = tempfile.SpooledTemporaryFile() - f.write(b'abc\n') - self.assertFalse(f._rolled) - f.close() - try: - f.close() - f.close() - except: - self.failOnException("close") - - def test_multiple_close_after_rollover(self) -> None: - # A SpooledTemporaryFile can be closed many times without error - f = tempfile.SpooledTemporaryFile(max_size=1) - f.write(b'abc\n') - self.assertTrue(f._rolled) - f.close() - try: - f.close() - f.close() - except: - self.failOnException("close") - - def test_bound_methods(self) -> None: - # It should be OK to steal a bound method from a SpooledTemporaryFile - # and use it independently; when the file rolls over, those bound - # methods should continue to function - f = self.do_create(max_size=30) - read = f.read - write = f.write - seek = f.seek - - write(b"a" * 35) - write(b"b" * 35) - seek(0, 0) - self.assertEqual(read(70), b'a'*35 + b'b'*35) - - def test_text_mode(self) -> None: - # Creating a SpooledTemporaryFile with a text mode should produce - # a file object reading and writing (Unicode) text strings. - f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10) - f.write("abc\n") - f.seek(0) - self.assertEqual(f.read(), "abc\n") - f.write("def\n") - f.seek(0) - self.assertEqual(f.read(), "abc\ndef\n") - f.write("xyzzy\n") - f.seek(0) - self.assertEqual(f.read(), "abc\ndef\nxyzzy\n") - # Check that Ctrl+Z doesn't truncate the file - f.write("foo\x1abar\n") - f.seek(0) - self.assertEqual(f.read(), "abc\ndef\nxyzzy\nfoo\x1abar\n") - - def test_text_newline_and_encoding(self) -> None: - f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, - newline='', encoding='utf-8') - f.write("\u039B\r\n") - f.seek(0) - self.assertEqual(f.read(), "\u039B\r\n") - self.assertFalse(f._rolled) - - f.write("\u039B" * 20 + "\r\n") - f.seek(0) - self.assertEqual(f.read(), "\u039B\r\n" + ("\u039B" * 20) + "\r\n") - self.assertTrue(f._rolled) - - def test_context_manager_before_rollover(self) -> None: - # A SpooledTemporaryFile can be used as a context manager - with tempfile.SpooledTemporaryFile(max_size=1) as f: - self.assertFalse(f._rolled) - self.assertFalse(f.closed) - self.assertTrue(f.closed) - def use_closed(): - with f: - pass - self.assertRaises(ValueError, use_closed) - - def test_context_manager_during_rollover(self) -> None: - # A SpooledTemporaryFile can be used as a context manager - with tempfile.SpooledTemporaryFile(max_size=1) as f: - self.assertFalse(f._rolled) - f.write(b'abc\n') - f.flush() - self.assertTrue(f._rolled) - self.assertFalse(f.closed) - self.assertTrue(f.closed) - def use_closed(): - with f: - pass - self.assertRaises(ValueError, use_closed) - - def test_context_manager_after_rollover(self) -> None: - # A SpooledTemporaryFile can be used as a context manager - f = tempfile.SpooledTemporaryFile(max_size=1) - f.write(b'abc\n') - f.flush() - self.assertTrue(f._rolled) - with f: - self.assertFalse(f.closed) - self.assertTrue(f.closed) - def use_closed(): - with f: - pass - self.assertRaises(ValueError, use_closed) - - -test_classes.append(test_SpooledTemporaryFile) - - -class test_TemporaryFile(TC): - """Test TemporaryFile().""" - - def test_basic(self) -> None: - # TemporaryFile can create files - # No point in testing the name params - the file has no name. - try: - tempfile.TemporaryFile() - except: - self.failOnException("TemporaryFile") - - def test_has_no_name(self) -> None: - # TemporaryFile creates files with no names (on this system) - dir = tempfile.mkdtemp() - f = tempfile.TemporaryFile(dir=dir) - f.write(b'blat') - - # Sneaky: because this file has no name, it should not prevent - # us from removing the directory it was created in. - try: - os.rmdir(dir) - except: - ei = sys.exc_info() - # cleanup - f.close() - os.rmdir(dir) - self.failOnException("rmdir", ei) - - def test_multiple_close(self) -> None: - # A TemporaryFile can be closed many times without error - f = tempfile.TemporaryFile() - f.write(b'abc\n') - f.close() - try: - f.close() - f.close() - except: - self.failOnException("close") - - # How to test the mode and bufsize parameters? - def test_mode_and_encoding(self) -> None: - - def roundtrip(input: AnyStr, *args: Any, **kwargs: Any) -> None: - with tempfile.TemporaryFile(*args, **kwargs) as fileobj: - fileobj.write(input) - fileobj.seek(0) - self.assertEqual(input, fileobj.read()) - - roundtrip(b"1234", "w+b") - roundtrip("abdc\n", "w+") - roundtrip("\u039B", "w+", encoding="utf-16") - roundtrip("foo\r\n", "w+", newline="") - - -if tempfile.NamedTemporaryFile is not tempfile.TemporaryFile: - test_classes.append(test_TemporaryFile) - - -# Helper for test_del_on_shutdown -class NulledModules: - def __init__(self, *modules: Any) -> None: - self.refs = [mod.__dict__ for mod in modules] - self.contents = [ref.copy() for ref in self.refs] - - def __enter__(self) -> None: - for d in self.refs: - for key in d: - d[key] = None - - def __exit__(self, *exc_info: Any) -> None: - for d, c in zip(self.refs, self.contents): - d.clear() - d.update(c) - -class test_TemporaryDirectory(TC): - """Test TemporaryDirectory().""" - - def do_create(self, dir: str = None, pre: str = "", suf: str = "", - recurse: int = 1) -> tempfile.TemporaryDirectory: - if dir is None: - dir = tempfile.gettempdir() - try: - tmp = tempfile.TemporaryDirectory(dir=dir, prefix=pre, suffix=suf) - except: - self.failOnException("TemporaryDirectory") - self.nameCheck(tmp.name, dir, pre, suf) - # Create a subdirectory and some files - if recurse: - self.do_create(tmp.name, pre, suf, recurse-1) - with open(os.path.join(tmp.name, "test.txt"), "wb") as f: - f.write(b"Hello world!") - return tmp - - def test_mkdtemp_failure(self) -> None: - # Check no additional exception if mkdtemp fails - # Previously would raise AttributeError instead - # (noted as part of Issue #10188) - with tempfile.TemporaryDirectory() as nonexistent: - pass - with self.assertRaises(os.error): - tempfile.TemporaryDirectory(dir=nonexistent) - - def test_explicit_cleanup(self) -> None: - # A TemporaryDirectory is deleted when cleaned up - dir = tempfile.mkdtemp() - try: - d = self.do_create(dir=dir) - self.assertTrue(os.path.exists(d.name), - "TemporaryDirectory %s does not exist" % d.name) - d.cleanup() - self.assertFalse(os.path.exists(d.name), - "TemporaryDirectory %s exists after cleanup" % d.name) - finally: - os.rmdir(dir) - - @support.skip_unless_symlink - def test_cleanup_with_symlink_to_a_directory(self) -> None: - # cleanup() should not follow symlinks to directories (issue #12464) - d1 = self.do_create() - d2 = self.do_create() - - # Symlink d1/foo -> d2 - os.symlink(d2.name, os.path.join(d1.name, "foo")) - - # This call to cleanup() should not follow the "foo" symlink - d1.cleanup() - - self.assertFalse(os.path.exists(d1.name), - "TemporaryDirectory %s exists after cleanup" % d1.name) - self.assertTrue(os.path.exists(d2.name), - "Directory pointed to by a symlink was deleted") - self.assertEqual(os.listdir(d2.name), ['test.txt'], - "Contents of the directory pointed to by a symlink " - "were deleted") - d2.cleanup() - - @support.cpython_only - def test_del_on_collection(self) -> None: - # A TemporaryDirectory is deleted when garbage collected - dir = tempfile.mkdtemp() - try: - d = self.do_create(dir=dir) - name = d.name - del d # Rely on refcounting to invoke __del__ - self.assertFalse(os.path.exists(name), - "TemporaryDirectory %s exists after __del__" % name) - finally: - os.rmdir(dir) - - @unittest.expectedFailure # See issue #10188 - def test_del_on_shutdown(self) -> None: - # A TemporaryDirectory may be cleaned up during shutdown - # Make sure it works with the relevant modules nulled out - with self.do_create() as dir: - d = self.do_create(dir=dir) - # Mimic the nulling out of modules that - # occurs during system shutdown - modules = [os, os.path] - if has_stat: - modules.append(stat) - # Currently broken, so suppress the warning - # that is otherwise emitted on stdout - with support.captured_stderr() as err: - with NulledModules(*modules): - d.cleanup() - # Currently broken, so stop spurious exception by - # indicating the object has already been closed - d._closed = True - # And this assert will fail, as expected by the - # unittest decorator... - self.assertFalse(os.path.exists(d.name), - "TemporaryDirectory %s exists after cleanup" % d.name) - - def test_warnings_on_cleanup(self) -> None: - # Two kinds of warning on shutdown - # Issue 10888: may write to stderr if modules are nulled out - # ResourceWarning will be triggered by __del__ - with self.do_create() as dir: - if os.sep != '\\': - # Embed a backslash in order to make sure string escaping - # in the displayed error message is dealt with correctly - suffix = '\\check_backslash_handling' - else: - suffix = '' - d = self.do_create(dir=dir, suf=suffix) - - #Check for the Issue 10888 message - modules = [os, os.path] - if has_stat: - modules.append(stat) - with support.captured_stderr() as err: - with NulledModules(*modules): - d.cleanup() - message = err.getvalue().replace('\\\\', '\\') - self.assertIn("while cleaning up", message) - self.assertIn(d.name, message) - - # Check for the resource warning - with support.check_warnings(('Implicitly', ResourceWarning), quiet=False): - warnings.filterwarnings("always", category=ResourceWarning) - d.__del__() - self.assertFalse(os.path.exists(d.name), - "TemporaryDirectory %s exists after __del__" % d.name) - - def test_multiple_close(self) -> None: - # Can be cleaned-up many times without error - d = self.do_create() - d.cleanup() - try: - d.cleanup() - d.cleanup() - except: - self.failOnException("cleanup") - - def test_context_manager(self) -> None: - # Can be used as a context manager - d = self.do_create() - with d as name: - self.assertTrue(os.path.exists(name)) - self.assertEqual(name, d.name) - self.assertFalse(os.path.exists(name)) - - -test_classes.append(test_TemporaryDirectory) - -def test_main() -> None: - support.run_unittest(*test_classes) - -if __name__ == "__main__": - test_main() diff --git a/lib-python/3.2/test/test_textwrap.py b/lib-python/3.2/test/test_textwrap.py deleted file mode 100644 index d5eda6f2ee6c..000000000000 --- a/lib-python/3.2/test/test_textwrap.py +++ /dev/null @@ -1,601 +0,0 @@ -# -# Test suite for the textwrap module. -# -# Original tests written by Greg Ward . -# Converted to PyUnit by Peter Hansen . -# Currently maintained by Greg Ward. -# -# $Id$ -# - -import unittest -from test import support - -from typing import Any, List, Sequence, Undefined - -from textwrap import TextWrapper, wrap, fill, dedent - - -class BaseTestCase(unittest.TestCase): - '''Parent class with utility methods for textwrap tests.''' - - wrapper = Undefined(TextWrapper) - - def show(self, textin: Sequence[str]) -> str: - if isinstance(textin, list): - results = List[str]() - for i in range(len(textin)): - results.append(" %d: %r" % (i, textin[i])) - result = '\n'.join(results) - elif isinstance(textin, str): - result = " %s\n" % repr(textin) - return result - - - def check(self, result: Sequence[str], expect: Sequence[str]) -> None: - self.assertEqual(result, expect, - 'expected:\n%s\nbut got:\n%s' % ( - self.show(expect), self.show(result))) - - def check_wrap(self, text: str, width: int, expect: Sequence[str], - **kwargs: Any) -> None: - result = wrap(text, width, **kwargs) - self.check(result, expect) - - def check_split(self, text: str, expect: Sequence[str]) -> None: - result = self.wrapper._split(text) - self.assertEqual(result, expect, - "\nexpected %r\n" - "but got %r" % (expect, result)) - - -class WrapTestCase(BaseTestCase): - - def setUp(self) -> None: - self.wrapper = TextWrapper(width=45) - - def test_simple(self) -> None: - # Simple case: just words, spaces, and a bit of punctuation - - text = "Hello there, how are you this fine day? I'm glad to hear it!" - - self.check_wrap(text, 12, - ["Hello there,", - "how are you", - "this fine", - "day? I'm", - "glad to hear", - "it!"]) - self.check_wrap(text, 42, - ["Hello there, how are you this fine day?", - "I'm glad to hear it!"]) - self.check_wrap(text, 80, [text]) - - - def test_whitespace(self) -> None: - # Whitespace munging and end-of-sentence detection - - text = """\ -This is a paragraph that already has -line breaks. But some of its lines are much longer than the others, -so it needs to be wrapped. -Some lines are \ttabbed too. -What a mess! -""" - - expect = ["This is a paragraph that already has line", - "breaks. But some of its lines are much", - "longer than the others, so it needs to be", - "wrapped. Some lines are tabbed too. What a", - "mess!"] - - wrapper = TextWrapper(45, fix_sentence_endings=True) - result = wrapper.wrap(text) - self.check(result, expect) - - results = wrapper.fill(text) - self.check(results, '\n'.join(expect)) - - def test_fix_sentence_endings(self) -> None: - wrapper = TextWrapper(60, fix_sentence_endings=True) - - # SF #847346: ensure that fix_sentence_endings=True does the - # right thing even on input short enough that it doesn't need to - # be wrapped. - text = "A short line. Note the single space." - expect = ["A short line. Note the single space."] - self.check(wrapper.wrap(text), expect) - - # Test some of the hairy end cases that _fix_sentence_endings() - # is supposed to handle (the easy stuff is tested in - # test_whitespace() above). - text = "Well, Doctor? What do you think?" - expect = ["Well, Doctor? What do you think?"] - self.check(wrapper.wrap(text), expect) - - text = "Well, Doctor?\nWhat do you think?" - self.check(wrapper.wrap(text), expect) - - text = 'I say, chaps! Anyone for "tennis?"\nHmmph!' - expect = ['I say, chaps! Anyone for "tennis?" Hmmph!'] - self.check(wrapper.wrap(text), expect) - - wrapper.width = 20 - expect = ['I say, chaps!', 'Anyone for "tennis?"', 'Hmmph!'] - self.check(wrapper.wrap(text), expect) - - text = 'And she said, "Go to hell!"\nCan you believe that?' - expect = ['And she said, "Go to', - 'hell!" Can you', - 'believe that?'] - self.check(wrapper.wrap(text), expect) - - wrapper.width = 60 - expect = ['And she said, "Go to hell!" Can you believe that?'] - self.check(wrapper.wrap(text), expect) - - text = 'File stdio.h is nice.' - expect = ['File stdio.h is nice.'] - self.check(wrapper.wrap(text), expect) - - def test_wrap_short(self) -> None: - # Wrapping to make short lines longer - - text = "This is a\nshort paragraph." - - self.check_wrap(text, 20, ["This is a short", - "paragraph."]) - self.check_wrap(text, 40, ["This is a short paragraph."]) - - - def test_wrap_short_1line(self) -> None: - # Test endcases - - text = "This is a short line." - - self.check_wrap(text, 30, ["This is a short line."]) - self.check_wrap(text, 30, ["(1) This is a short line."], - initial_indent="(1) ") - - - def test_hyphenated(self) -> None: - # Test breaking hyphenated words - - text = ("this-is-a-useful-feature-for-" - "reformatting-posts-from-tim-peters'ly") - - self.check_wrap(text, 40, - ["this-is-a-useful-feature-for-", - "reformatting-posts-from-tim-peters'ly"]) - self.check_wrap(text, 41, - ["this-is-a-useful-feature-for-", - "reformatting-posts-from-tim-peters'ly"]) - self.check_wrap(text, 42, - ["this-is-a-useful-feature-for-reformatting-", - "posts-from-tim-peters'ly"]) - - def test_hyphenated_numbers(self) -> None: - # Test that hyphenated numbers (eg. dates) are not broken like words. - text = ("Python 1.0.0 was released on 1994-01-26. Python 1.0.1 was\n" - "released on 1994-02-15.") - - self.check_wrap(text, 30, ['Python 1.0.0 was released on', - '1994-01-26. Python 1.0.1 was', - 'released on 1994-02-15.']) - self.check_wrap(text, 40, ['Python 1.0.0 was released on 1994-01-26.', - 'Python 1.0.1 was released on 1994-02-15.']) - - text = "I do all my shopping at 7-11." - self.check_wrap(text, 25, ["I do all my shopping at", - "7-11."]) - self.check_wrap(text, 27, ["I do all my shopping at", - "7-11."]) - self.check_wrap(text, 29, ["I do all my shopping at 7-11."]) - - def test_em_dash(self) -> None: - # Test text with em-dashes - text = "Em-dashes should be written -- thus." - self.check_wrap(text, 25, - ["Em-dashes should be", - "written -- thus."]) - - # Probe the boundaries of the properly written em-dash, - # ie. " -- ". - self.check_wrap(text, 29, - ["Em-dashes should be written", - "-- thus."]) - expect = ["Em-dashes should be written --", - "thus."] - self.check_wrap(text, 30, expect) - self.check_wrap(text, 35, expect) - self.check_wrap(text, 36, - ["Em-dashes should be written -- thus."]) - - # The improperly written em-dash is handled too, because - # it's adjacent to non-whitespace on both sides. - text = "You can also do--this or even---this." - expect = ["You can also do", - "--this or even", - "---this."] - self.check_wrap(text, 15, expect) - self.check_wrap(text, 16, expect) - expect = ["You can also do--", - "this or even---", - "this."] - self.check_wrap(text, 17, expect) - self.check_wrap(text, 19, expect) - expect = ["You can also do--this or even", - "---this."] - self.check_wrap(text, 29, expect) - self.check_wrap(text, 31, expect) - expect = ["You can also do--this or even---", - "this."] - self.check_wrap(text, 32, expect) - self.check_wrap(text, 35, expect) - - # All of the above behaviour could be deduced by probing the - # _split() method. - text = "Here's an -- em-dash and--here's another---and another!" - expect = ["Here's", " ", "an", " ", "--", " ", "em-", "dash", " ", - "and", "--", "here's", " ", "another", "---", - "and", " ", "another!"] - self.check_split(text, expect) - - text = "and then--bam!--he was gone" - expect = ["and", " ", "then", "--", "bam!", "--", - "he", " ", "was", " ", "gone"] - self.check_split(text, expect) - - - def test_unix_options (self) -> None: - # Test that Unix-style command-line options are wrapped correctly. - # Both Optik (OptionParser) and Docutils rely on this behaviour! - - text = "You should use the -n option, or --dry-run in its long form." - self.check_wrap(text, 20, - ["You should use the", - "-n option, or --dry-", - "run in its long", - "form."]) - self.check_wrap(text, 21, - ["You should use the -n", - "option, or --dry-run", - "in its long form."]) - expect = ["You should use the -n option, or", - "--dry-run in its long form."] - self.check_wrap(text, 32, expect) - self.check_wrap(text, 34, expect) - self.check_wrap(text, 35, expect) - self.check_wrap(text, 38, expect) - expect = ["You should use the -n option, or --dry-", - "run in its long form."] - self.check_wrap(text, 39, expect) - self.check_wrap(text, 41, expect) - expect = ["You should use the -n option, or --dry-run", - "in its long form."] - self.check_wrap(text, 42, expect) - - # Again, all of the above can be deduced from _split(). - text = "the -n option, or --dry-run or --dryrun" - expect = ["the", " ", "-n", " ", "option,", " ", "or", " ", - "--dry-", "run", " ", "or", " ", "--dryrun"] - self.check_split(text, expect) - - def test_funky_hyphens (self) -> None: - # Screwy edge cases cooked up by David Goodger. All reported - # in SF bug #596434. - self.check_split("what the--hey!", ["what", " ", "the", "--", "hey!"]) - self.check_split("what the--", ["what", " ", "the--"]) - self.check_split("what the--.", ["what", " ", "the--."]) - self.check_split("--text--.", ["--text--."]) - - # When I first read bug #596434, this is what I thought David - # was talking about. I was wrong; these have always worked - # fine. The real problem is tested in test_funky_parens() - # below... - self.check_split("--option", ["--option"]) - self.check_split("--option-opt", ["--option-", "opt"]) - self.check_split("foo --option-opt bar", - ["foo", " ", "--option-", "opt", " ", "bar"]) - - def test_punct_hyphens(self) -> None: - # Oh bother, SF #965425 found another problem with hyphens -- - # hyphenated words in single quotes weren't handled correctly. - # In fact, the bug is that *any* punctuation around a hyphenated - # word was handled incorrectly, except for a leading "--", which - # was special-cased for Optik and Docutils. So test a variety - # of styles of punctuation around a hyphenated word. - # (Actually this is based on an Optik bug report, #813077). - self.check_split("the 'wibble-wobble' widget", - ['the', ' ', "'wibble-", "wobble'", ' ', 'widget']) - self.check_split('the "wibble-wobble" widget', - ['the', ' ', '"wibble-', 'wobble"', ' ', 'widget']) - self.check_split("the (wibble-wobble) widget", - ['the', ' ', "(wibble-", "wobble)", ' ', 'widget']) - self.check_split("the ['wibble-wobble'] widget", - ['the', ' ', "['wibble-", "wobble']", ' ', 'widget']) - - def test_funky_parens (self) -> None: - # Second part of SF bug #596434: long option strings inside - # parentheses. - self.check_split("foo (--option) bar", - ["foo", " ", "(--option)", " ", "bar"]) - - # Related stuff -- make sure parens work in simpler contexts. - self.check_split("foo (bar) baz", - ["foo", " ", "(bar)", " ", "baz"]) - self.check_split("blah (ding dong), wubba", - ["blah", " ", "(ding", " ", "dong),", - " ", "wubba"]) - - def test_initial_whitespace(self) -> None: - # SF bug #622849 reported inconsistent handling of leading - # whitespace; let's test that a bit, shall we? - text = " This is a sentence with leading whitespace." - self.check_wrap(text, 50, - [" This is a sentence with leading whitespace."]) - self.check_wrap(text, 30, - [" This is a sentence with", "leading whitespace."]) - - def test_no_drop_whitespace(self) -> None: - # SF patch #1581073 - text = " This is a sentence with much whitespace." - self.check_wrap(text, 10, - [" This is a", " ", "sentence ", - "with ", "much white", "space."], - drop_whitespace=False) - - def test_split(self) -> None: - # Ensure that the standard _split() method works as advertised - # in the comments - - text = "Hello there -- you goof-ball, use the -b option!" - - result = self.wrapper._split(text) - self.check(result, - ["Hello", " ", "there", " ", "--", " ", "you", " ", "goof-", - "ball,", " ", "use", " ", "the", " ", "-b", " ", "option!"]) - - def test_break_on_hyphens(self) -> None: - # Ensure that the break_on_hyphens attributes work - text = "yaba daba-doo" - self.check_wrap(text, 10, ["yaba daba-", "doo"], - break_on_hyphens=True) - self.check_wrap(text, 10, ["yaba", "daba-doo"], - break_on_hyphens=False) - - def test_bad_width(self) -> None: - # Ensure that width <= 0 is caught. - text = "Whatever, it doesn't matter." - self.assertRaises(ValueError, wrap, text, 0) - self.assertRaises(ValueError, wrap, text, -1) - - def test_no_split_at_umlaut(self) -> None: - text = "Die Empf\xe4nger-Auswahl" - self.check_wrap(text, 13, ["Die", "Empf\xe4nger-", "Auswahl"]) - - def test_umlaut_followed_by_dash(self) -> None: - text = "aa \xe4\xe4-\xe4\xe4" - self.check_wrap(text, 7, ["aa \xe4\xe4-", "\xe4\xe4"]) - - -class LongWordTestCase (BaseTestCase): - def setUp(self) -> None: - self.wrapper = TextWrapper() - self.text = '''\ -Did you say "supercalifragilisticexpialidocious?" -How *do* you spell that odd word, anyways? -''' - - def test_break_long(self) -> None: - # Wrap text with long words and lots of punctuation - - self.check_wrap(self.text, 30, - ['Did you say "supercalifragilis', - 'ticexpialidocious?" How *do*', - 'you spell that odd word,', - 'anyways?']) - self.check_wrap(self.text, 50, - ['Did you say "supercalifragilisticexpialidocious?"', - 'How *do* you spell that odd word, anyways?']) - - # SF bug 797650. Prevent an infinite loop by making sure that at - # least one character gets split off on every pass. - self.check_wrap('-'*10+'hello', 10, - ['----------', - ' h', - ' e', - ' l', - ' l', - ' o'], - subsequent_indent = ' '*15) - - # bug 1146. Prevent a long word to be wrongly wrapped when the - # preceding word is exactly one character shorter than the width - self.check_wrap(self.text, 12, - ['Did you say ', - '"supercalifr', - 'agilisticexp', - 'ialidocious?', - '" How *do*', - 'you spell', - 'that odd', - 'word,', - 'anyways?']) - - def test_nobreak_long(self) -> None: - # Test with break_long_words disabled - self.wrapper.break_long_words = False - self.wrapper.width = 30 - expect = ['Did you say', - '"supercalifragilisticexpialidocious?"', - 'How *do* you spell that odd', - 'word, anyways?' - ] - result = self.wrapper.wrap(self.text) - self.check(result, expect) - - # Same thing with kwargs passed to standalone wrap() function. - result = wrap(self.text, width=30, break_long_words=0) - self.check(result, expect) - - -class IndentTestCases(BaseTestCase): - - # called before each test method - def setUp(self) -> None: - self.text = '''\ -This paragraph will be filled, first without any indentation, -and then with some (including a hanging indent).''' - - - def test_fill(self) -> None: - # Test the fill() method - - expect = '''\ -This paragraph will be filled, first -without any indentation, and then with -some (including a hanging indent).''' - - result = fill(self.text, 40) - self.check(result, expect) - - - def test_initial_indent(self) -> None: - # Test initial_indent parameter - - expect = [" This paragraph will be filled,", - "first without any indentation, and then", - "with some (including a hanging indent)."] - result = wrap(self.text, 40, initial_indent=" ") - self.check(result, expect) - - expects = "\n".join(expect) - results = fill(self.text, 40, initial_indent=" ") - self.check(results, expects) - - - def test_subsequent_indent(self) -> None: - # Test subsequent_indent parameter - - expect = '''\ - * This paragraph will be filled, first - without any indentation, and then - with some (including a hanging - indent).''' - - result = fill(self.text, 40, - initial_indent=" * ", subsequent_indent=" ") - self.check(result, expect) - - -# Despite the similar names, DedentTestCase is *not* the inverse -# of IndentTestCase! -class DedentTestCase(unittest.TestCase): - - def assertUnchanged(self, text: str) -> None: - """assert that dedent() has no effect on 'text'""" - self.assertEqual(text, dedent(text)) - - def test_dedent_nomargin(self) -> None: - # No lines indented. - text = "Hello there.\nHow are you?\nOh good, I'm glad." - self.assertUnchanged(text) - - # Similar, with a blank line. - text = "Hello there.\n\nBoo!" - self.assertUnchanged(text) - - # Some lines indented, but overall margin is still zero. - text = "Hello there.\n This is indented." - self.assertUnchanged(text) - - # Again, add a blank line. - text = "Hello there.\n\n Boo!\n" - self.assertUnchanged(text) - - def test_dedent_even(self) -> None: - # All lines indented by two spaces. - text = " Hello there.\n How are ya?\n Oh good." - expect = "Hello there.\nHow are ya?\nOh good." - self.assertEqual(expect, dedent(text)) - - # Same, with blank lines. - text = " Hello there.\n\n How are ya?\n Oh good.\n" - expect = "Hello there.\n\nHow are ya?\nOh good.\n" - self.assertEqual(expect, dedent(text)) - - # Now indent one of the blank lines. - text = " Hello there.\n \n How are ya?\n Oh good.\n" - expect = "Hello there.\n\nHow are ya?\nOh good.\n" - self.assertEqual(expect, dedent(text)) - - def test_dedent_uneven(self) -> None: - # Lines indented unevenly. - text = '''\ - def foo(): - while 1: - return foo - ''' - expect = '''\ -def foo(): - while 1: - return foo -''' - self.assertEqual(expect, dedent(text)) - - # Uneven indentation with a blank line. - text = " Foo\n Bar\n\n Baz\n" - expect = "Foo\n Bar\n\n Baz\n" - self.assertEqual(expect, dedent(text)) - - # Uneven indentation with a whitespace-only line. - text = " Foo\n Bar\n \n Baz\n" - expect = "Foo\n Bar\n\n Baz\n" - self.assertEqual(expect, dedent(text)) - - # dedent() should not mangle internal tabs - def test_dedent_preserve_internal_tabs(self) -> None: - text = " hello\tthere\n how are\tyou?" - expect = "hello\tthere\nhow are\tyou?" - self.assertEqual(expect, dedent(text)) - - # make sure that it preserves tabs when it's not making any - # changes at all - self.assertEqual(expect, dedent(expect)) - - # dedent() should not mangle tabs in the margin (i.e. - # tabs and spaces both count as margin, but are *not* - # considered equivalent) - def test_dedent_preserve_margin_tabs(self) -> None: - text = " hello there\n\thow are you?" - self.assertUnchanged(text) - - # same effect even if we have 8 spaces - text = " hello there\n\thow are you?" - self.assertUnchanged(text) - - # dedent() only removes whitespace that can be uniformly removed! - text = "\thello there\n\thow are you?" - expect = "hello there\nhow are you?" - self.assertEqual(expect, dedent(text)) - - text = " \thello there\n \thow are you?" - self.assertEqual(expect, dedent(text)) - - text = " \t hello there\n \t how are you?" - self.assertEqual(expect, dedent(text)) - - text = " \thello there\n \t how are you?" - expect = "hello there\n how are you?" - self.assertEqual(expect, dedent(text)) - - -def test_main() -> None: - support.run_unittest(WrapTestCase, - LongWordTestCase, - IndentTestCases, - DedentTestCase) - -if __name__ == '__main__': - test_main() diff --git a/lib-python/3.2/test/tf_inherit_check.py b/lib-python/3.2/test/tf_inherit_check.py deleted file mode 100644 index 92ebd95e5236..000000000000 --- a/lib-python/3.2/test/tf_inherit_check.py +++ /dev/null @@ -1,25 +0,0 @@ -# Helper script for test_tempfile.py. argv[2] is the number of a file -# descriptor which should _not_ be open. Check this by attempting to -# write to it -- if we succeed, something is wrong. - -import sys -import os - -verbose = (sys.argv[1] == 'v') -try: - fd = int(sys.argv[2]) - - try: - os.write(fd, b"blat") - except os.error: - # Success -- could not write to fd. - sys.exit(0) - else: - if verbose: - sys.stderr.write("fd %d is open in child" % fd) - sys.exit(1) - -except Exception: - if verbose: - raise - sys.exit(1) diff --git a/lib-python/3.2/textwrap.py b/lib-python/3.2/textwrap.py deleted file mode 100644 index 473098090f00..000000000000 --- a/lib-python/3.2/textwrap.py +++ /dev/null @@ -1,391 +0,0 @@ -"""Text wrapping and filling. -""" - -# Copyright (C) 1999-2001 Gregory P. Ward. -# Copyright (C) 2002, 2003 Python Software Foundation. -# Written by Greg Ward - -import string, re - -from typing import Dict, List, Any - -__all__ = ['TextWrapper', 'wrap', 'fill', 'dedent'] - -# Hardcode the recognized whitespace characters to the US-ASCII -# whitespace characters. The main reason for doing this is that in -# ISO-8859-1, 0xa0 is non-breaking whitespace, so in certain locales -# that character winds up in string.whitespace. Respecting -# string.whitespace in those cases would 1) make textwrap treat 0xa0 the -# same as any other whitespace char, which is clearly wrong (it's a -# *non-breaking* space), 2) possibly cause problems with Unicode, -# since 0xa0 is not in range(128). -_whitespace = '\t\n\x0b\x0c\r ' - -class TextWrapper: - """ - Object for wrapping/filling text. The public interface consists of - the wrap() and fill() methods; the other methods are just there for - subclasses to override in order to tweak the default behaviour. - If you want to completely replace the main wrapping algorithm, - you'll probably have to override _wrap_chunks(). - - Several instance attributes control various aspects of wrapping: - width (default: 70) - the maximum width of wrapped lines (unless break_long_words - is false) - initial_indent (default: "") - string that will be prepended to the first line of wrapped - output. Counts towards the line's width. - subsequent_indent (default: "") - string that will be prepended to all lines save the first - of wrapped output; also counts towards each line's width. - expand_tabs (default: true) - Expand tabs in input text to spaces before further processing. - Each tab will become 1 .. 8 spaces, depending on its position in - its line. If false, each tab is treated as a single character. - replace_whitespace (default: true) - Replace all whitespace characters in the input text by spaces - after tab expansion. Note that if expand_tabs is false and - replace_whitespace is true, every tab will be converted to a - single space! - fix_sentence_endings (default: false) - Ensure that sentence-ending punctuation is always followed - by two spaces. Off by default because the algorithm is - (unavoidably) imperfect. - break_long_words (default: true) - Break words longer than 'width'. If false, those words will not - be broken, and some lines might be longer than 'width'. - break_on_hyphens (default: true) - Allow breaking hyphenated words. If true, wrapping will occur - preferably on whitespaces and right after hyphens part of - compound words. - drop_whitespace (default: true) - Drop leading and trailing whitespace from lines. - """ - - unicode_whitespace_trans = Dict[int, int]() - uspace = ord(' ') - for x in _whitespace: - unicode_whitespace_trans[ord(x)] = uspace - - # This funky little regex is just the trick for splitting - # text up into word-wrappable chunks. E.g. - # "Hello there -- you goof-ball, use the -b option!" - # splits into - # Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option! - # (after stripping out empty strings). - wordsep_re = re.compile( - r'(\s+|' # any whitespace - r'[^\s\w]*\w+[^0-9\W]-(?=\w+[^0-9\W])|' # hyphenated words - r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash - - # This less funky little regex just split on recognized spaces. E.g. - # "Hello there -- you goof-ball, use the -b option!" - # splits into - # Hello/ /there/ /--/ /you/ /goof-ball,/ /use/ /the/ /-b/ /option!/ - wordsep_simple_re = re.compile(r'(\s+)') - - # XXX this is not locale- or charset-aware -- string.lowercase - # is US-ASCII only (and therefore English-only) - sentence_end_re = re.compile(r'[a-z]' # lowercase letter - r'[\.\!\?]' # sentence-ending punct. - r'[\"\']?' # optional end-of-quote - r'\Z') # end of chunk - - - def __init__(self, - width: int = 70, - initial_indent: str = "", - subsequent_indent: str = "", - expand_tabs: bool = True, - replace_whitespace: bool = True, - fix_sentence_endings: bool = False, - break_long_words: bool = True, - drop_whitespace: bool = True, - break_on_hyphens: bool = True) -> None: - self.width = width - self.initial_indent = initial_indent - self.subsequent_indent = subsequent_indent - self.expand_tabs = expand_tabs - self.replace_whitespace = replace_whitespace - self.fix_sentence_endings = fix_sentence_endings - self.break_long_words = break_long_words - self.drop_whitespace = drop_whitespace - self.break_on_hyphens = break_on_hyphens - - - # -- Private methods ----------------------------------------------- - # (possibly useful for subclasses to override) - - def _munge_whitespace(self, text: str) -> str: - """_munge_whitespace(text : string) -> string - - Munge whitespace in text: expand tabs and convert all other - whitespace characters to spaces. Eg. " foo\tbar\n\nbaz" - becomes " foo bar baz". - """ - if self.expand_tabs: - text = text.expandtabs() - if self.replace_whitespace: - text = text.translate(self.unicode_whitespace_trans) - return text - - - def _split(self, text: str) -> List[str]: - """_split(text : string) -> [string] - - Split the text to wrap into indivisible chunks. Chunks are - not quite the same as words; see _wrap_chunks() for full - details. As an example, the text - Look, goof-ball -- use the -b option! - breaks into the following chunks: - 'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ', - 'use', ' ', 'the', ' ', '-b', ' ', 'option!' - if break_on_hyphens is True, or in: - 'Look,', ' ', 'goof-ball', ' ', '--', ' ', - 'use', ' ', 'the', ' ', '-b', ' ', option!' - otherwise. - """ - if self.break_on_hyphens is True: - chunks = self.wordsep_re.split(text) - else: - chunks = self.wordsep_simple_re.split(text) - chunks = [c for c in chunks if c] - return chunks - - def _fix_sentence_endings(self, chunks: List[str]) -> None: - """_fix_sentence_endings(chunks : [string]) - - Correct for sentence endings buried in 'chunks'. Eg. when the - original text contains "... foo.\nBar ...", munge_whitespace() - and split() will convert that to [..., "foo.", " ", "Bar", ...] - which has one too few spaces; this method simply changes the one - space to two. - """ - i = 0 - patsearch = self.sentence_end_re.search - while i < len(chunks)-1: - if chunks[i+1] == " " and patsearch(chunks[i]): - chunks[i+1] = " " - i += 2 - else: - i += 1 - - def _handle_long_word(self, reversed_chunks: List[str], - cur_line: List[str], cur_len: int, - width: int) -> None: - """_handle_long_word(chunks : [string], - cur_line : [string], - cur_len : int, width : int) - - Handle a chunk of text (most likely a word, not whitespace) that - is too long to fit in any line. - """ - # Figure out when indent is larger than the specified width, and make - # sure at least one character is stripped off on every pass - if width < 1: - space_left = 1 - else: - space_left = width - cur_len - - # If we're allowed to break long words, then do so: put as much - # of the next chunk onto the current line as will fit. - if self.break_long_words: - cur_line.append(reversed_chunks[-1][:space_left]) - reversed_chunks[-1] = reversed_chunks[-1][space_left:] - - # Otherwise, we have to preserve the long word intact. Only add - # it to the current line if there's nothing already there -- - # that minimizes how much we violate the width constraint. - elif not cur_line: - cur_line.append(reversed_chunks.pop()) - - # If we're not allowed to break long words, and there's already - # text on the current line, do nothing. Next time through the - # main loop of _wrap_chunks(), we'll wind up here again, but - # cur_len will be zero, so the next line will be entirely - # devoted to the long word that we can't handle right now. - - def _wrap_chunks(self, chunks: List[str]) -> List[str]: - """_wrap_chunks(chunks : [string]) -> [string] - - Wrap a sequence of text chunks and return a list of lines of - length 'self.width' or less. (If 'break_long_words' is false, - some lines may be longer than this.) Chunks correspond roughly - to words and the whitespace between them: each chunk is - indivisible (modulo 'break_long_words'), but a line break can - come between any two chunks. Chunks should not have internal - whitespace; ie. a chunk is either all whitespace or a "word". - Whitespace chunks will be removed from the beginning and end of - lines, but apart from that whitespace is preserved. - """ - lines = List[str]() - if self.width <= 0: - raise ValueError("invalid width %r (must be > 0)" % self.width) - - # Arrange in reverse order so items can be efficiently popped - # from a stack of chucks. - chunks.reverse() - - while chunks: - - # Start the list of chunks that will make up the current line. - # cur_len is just the length of all the chunks in cur_line. - cur_line = List[str]() - cur_len = 0 - - # Figure out which static string will prefix this line. - if lines: - indent = self.subsequent_indent - else: - indent = self.initial_indent - - # Maximum width for this line. - width = self.width - len(indent) - - # First chunk on line is whitespace -- drop it, unless this - # is the very beginning of the text (ie. no lines started yet). - if self.drop_whitespace and chunks[-1].strip() == '' and lines: - del chunks[-1] - - while chunks: - l = len(chunks[-1]) - - # Can at least squeeze this chunk onto the current line. - if cur_len + l <= width: - cur_line.append(chunks.pop()) - cur_len += l - - # Nope, this line is full. - else: - break - - # The current line is full, and the next chunk is too big to - # fit on *any* line (not just this one). - if chunks and len(chunks[-1]) > width: - self._handle_long_word(chunks, cur_line, cur_len, width) - - # If the last chunk on this line is all whitespace, drop it. - if self.drop_whitespace and cur_line and cur_line[-1].strip() == '': - del cur_line[-1] - - # Convert current line back to a string and store it in list - # of all lines (return value). - if cur_line: - lines.append(indent + ''.join(cur_line)) - - return lines - - - # -- Public interface ---------------------------------------------- - - def wrap(self, text: str) -> List[str]: - """wrap(text : string) -> [string] - - Reformat the single paragraph in 'text' so it fits in lines of - no more than 'self.width' columns, and return a list of wrapped - lines. Tabs in 'text' are expanded with string.expandtabs(), - and all other whitespace characters (including newline) are - converted to space. - """ - text = self._munge_whitespace(text) - chunks = self._split(text) - if self.fix_sentence_endings: - self._fix_sentence_endings(chunks) - return self._wrap_chunks(chunks) - - def fill(self, text: str) -> str: - """fill(text : string) -> string - - Reformat the single paragraph in 'text' to fit in lines of no - more than 'self.width' columns, and return a new string - containing the entire wrapped paragraph. - """ - return "\n".join(self.wrap(text)) - - -# -- Convenience interface --------------------------------------------- - -def wrap(text: str, width: int = 70, **kwargs: Any) -> List[str]: - """Wrap a single paragraph of text, returning a list of wrapped lines. - - Reformat the single paragraph in 'text' so it fits in lines of no - more than 'width' columns, and return a list of wrapped lines. By - default, tabs in 'text' are expanded with string.expandtabs(), and - all other whitespace characters (including newline) are converted to - space. See TextWrapper class for available keyword args to customize - wrapping behaviour. - """ - w = TextWrapper(width=width, **kwargs) - return w.wrap(text) - -def fill(text: str, width: int = 70, **kwargs: Any) -> str: - """Fill a single paragraph of text, returning a new string. - - Reformat the single paragraph in 'text' to fit in lines of no more - than 'width' columns, and return a new string containing the entire - wrapped paragraph. As with wrap(), tabs are expanded and other - whitespace characters converted to space. See TextWrapper class for - available keyword args to customize wrapping behaviour. - """ - w = TextWrapper(width=width, **kwargs) - return w.fill(text) - - -# -- Loosely related functionality ------------------------------------- - -_whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE) -_leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE) - -def dedent(text: str) -> str: - """Remove any common leading whitespace from every line in `text`. - - This can be used to make triple-quoted strings line up with the left - edge of the display, while still presenting them in the source code - in indented form. - - Note that tabs and spaces are both treated as whitespace, but they - are not equal: the lines " hello" and "\thello" are - considered to have no common leading whitespace. (This behaviour is - new in Python 2.5; older versions of this module incorrectly - expanded tabs before searching for common leading whitespace.) - """ - # Look for the longest leading string of spaces and tabs common to - # all lines. - margin = None # type: str - text = _whitespace_only_re.sub('', text) - indents = _leading_whitespace_re.findall(text) - for indent in indents: - if margin is None: - margin = indent - - # Current line more deeply indented than previous winner: - # no change (previous winner is still on top). - elif indent.startswith(margin): - pass - - # Current line consistent with and no deeper than previous winner: - # it's the new winner. - elif margin.startswith(indent): - margin = indent - - # Current line and previous winner have no common whitespace: - # there is no margin. - else: - margin = "" - break - - # sanity check (testing/debugging only) - if 0 and margin: - for line in text.split("\n"): - assert not line or line.startswith(margin), \ - "line = %r, margin = %r" % (line, margin) - - if margin: - text = re.sub(r'(?m)^' + margin, '', text) - return text - -if __name__ == "__main__": - #print dedent("\tfoo\n\tbar") - #print dedent(" \thello there\n \t how are you?") - print(dedent("Hello there.\n This is indented.")) diff --git a/lib-typing/2.7/setup.py b/lib-typing/2.7/setup.py deleted file mode 100644 index 18c34d84be64..000000000000 --- a/lib-typing/2.7/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -"""setup.py for Python 2.x typing module""" - -import glob -import os -import os.path -import sys - -from distutils.core import setup - -if sys.version_info >= (3, 0, 0): - sys.stderr.write("ERROR: You need Python 2.x to install this module.\n") - exit(1) - -version = '0.0.1.dev1' -description = 'typing (Python 2.x)' -long_description = ''' -typing (Python 2.x) -=================== - -This module is part of mypy, a static type checker for Python. -'''.lstrip() - -classifiers = [ - 'Development Status :: 2 - Pre-Alpha', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: POSIX', - 'Programming Language :: Python :: 2.7', - 'Topic :: Software Development', -] - -setup(name='typing', - version=version, - description=description, - long_description=long_description, - author='Jukka Lehtosalo', - author_email='jukka.lehtosalo@iki.fi', - url='http://www.mypy-lang.org/', - license='MIT License', - platforms=['POSIX'], - py_modules=['typing'], - classifiers=classifiers, - ) diff --git a/lib-typing/2.7/test_typing.py b/lib-typing/2.7/test_typing.py deleted file mode 100644 index e7be0bb42a61..000000000000 --- a/lib-typing/2.7/test_typing.py +++ /dev/null @@ -1,824 +0,0 @@ -from __future__ import with_statement -from abc import abstractmethod, ABCMeta -import unittest - -from typing import ( - List, Dict, Set, Tuple, Pattern, BytesPattern, Match, BytesMatch, Any, - Function, Generic, AbstractGeneric, Protocol, Sized, Iterable, Iterator, - Sequence, AbstractSet, Mapping, BinaryIO, TextIO, SupportsInt, SupportsFloat, - SupportsAbs, Reversible, Undefined, cast, forwardref, overload, typevar -) - - -class TestTyping(unittest.TestCase): - def test_List(self): - self.assertIs(List[int], list) - self.assertIs(List[unicode], list) - - def test_Dict(self): - self.assertIs(Dict[int, unicode], dict) - self.assertIs(Dict[unicode, int], dict) - - def test_Set(self): - self.assertIs(Set[int], set) - self.assertIs(Set[unicode], set) - - def test_Tuple(self): - self.assertIs(Tuple[int], tuple) - self.assertIs(Tuple[unicode, int], tuple) - self.assertIs(Tuple[unicode, int, bool], tuple) - - def test_Pattern(self): - import re - self.assertIs(type(re.compile(u'')), Pattern) - self.assertIs(type(re.compile('')), BytesPattern) - # Note that actually Pattern is the same as BytesPattern, which is - # a bit awkward. - - def test_Match(self): - import re - self.assertIs(type(re.match(u'', u'')), Match) - self.assertIs(type(re.match('', '')), BytesMatch) - # Note that actually Match is the same as BytesMatch, which is - # a bit awkward. - - def test_Any(self): - o = object() - self.assertIs(Any(o), o) - s = u'x' - self.assertIs(Any(s), s) - - def test_Function(self): - # Just check that we can call Function. Don't care about return value. - Function[[], int] - Function[[int], None] - Function[[int, unicode], bool] - - def test_cast(self): - o = object() - # cast performs no runtime checks! - self.assertIs(cast(int, o), o) - s = u'x' - self.assertIs(cast(unicode, s), s) - self.assertIs(cast(u'xyz', s), s) - # cast does not check type validity; anything goes. - self.assertIs(cast(o, s), s) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_simple_overload(self): - @overload - def f(x): return x + u'string' - @overload - def f(x): return x + 1 - - self.assertEqual(f(u'x'), u'xstring') - self.assertEqual(f(1), 2) - - @overload - def g(x): return u'integer' - @overload - def g(x): return u'string' - - self.assertEqual(g(u'x'), u'string') - self.assertEqual(g(1), u'integer') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overload_with_three_variants(self): - @overload - def f(x): return u'string' - @overload - def f(x): return u'integer' - @overload - def f(x): return u'floating' - - self.assertEqual(f(u'x'), u'string') - self.assertEqual(f(1), u'integer') - self.assertEqual(f(1.0), u'floating') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overload_with_two_args(self): - @overload - def f(x, y): return (1, x, y) - @overload - def f(x, y): return (2, x, y) - - self.assertEqual(f(u'x', u'y'), (1, u'x', u'y')) - self.assertEqual(f(u'z', 3), (2, u'z', 3)) - - @overload - def g(x, y): return 1 - @overload - def g(x, y): return 2 - - self.assertEqual(g(u'x', u'y'), 1) - self.assertEqual(g(u'x', 1), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_global_overload(self): - self.assertEqual(global_overload(u'x'), u's') - self.assertEqual(global_overload(1), u'i') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_partial_overload_annotation(self): - @overload - def f(x, y): return 1 - @overload - def f(x, y): return 2 - - self.assertEqual(f(u'x', object()), 1) - self.assertEqual(f(object(), 1), 2) - - @overload - def method_overload(self, x): - return u's' - - @overload - def method_overload(self, x): - return u'i' - - @unittest.skip("overloads not supported in 2.7 yet") - def test_method_overload(self): - self.assertEqual(self.method_overload(u'x'), u's') - self.assertEqual(self.method_overload(1), u'i') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overload_with_any_type(self): - @overload - def f(x, y): return 1 - @overload - def f(x, y): return 2 - - self.assertEqual(f((), 0), 1) - self.assertEqual(f((), ()), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overload_with_type_alias(self): - @overload - def f(x): return 1 - @overload - def f(x): return 2 - @overload - def f(x): return 3 - @overload - def f(x): return 4 - - self.assertEqual(f([]), 1) - self.assertEqual(f({}), 2) - self.assertEqual(f(()), 3) - self.assertEqual(f((1, u'x')), 3) - self.assertEqual(f(1), 4) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_call_overload_with_invalid_arg_count(self): - @overload - def f(x): return 1 - @overload - def f(x): return 2 - - msg1 = ur'f\(\) takes exactly 1 argument \(%d given\)' - msg2 = ur'f\(\) takes exactly 1 positional argument \(%d given\)' - - with self.assertRaisesRegex(TypeError, msg1 % 0): - f() - with self.assertRaisesRegex(TypeError, msg2 % 2): - f(1, 2) - with self.assertRaisesRegex(TypeError, msg2 % 3): - f(u'x', u'y', u'z') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overload_with_variable_argument_counts(self): - @overload - def f(): return None - @overload - def f(x): return x - - self.assertEqual(f(), None) - self.assertEqual(f(1), 1) - - @overload - def g(x): return x + 1 - @overload - def g(): return None - - self.assertEqual(g(), None) - self.assertEqual(g(1), 2) - - msg = ur'g\(\) takes no arguments \(2 given\)' - with self.assertRaisesRegex(TypeError, msg): - g(1, 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overload_dispatch_order(self): - class A(object): pass - class B(A): pass - class C(B): pass - - @overload - def f(x): return u'B' - @overload - def f(x): return u'A' - - self.assertEqual(f(B()), u'B') - self.assertEqual(f(A()), u'A') - - @overload - def g(x): return u'C' - @overload - def g(x): return u'B' - @overload - def g(x): return u'A' - - self.assertEqual(g(C()), u'C') - self.assertEqual(g(B()), u'B') - self.assertEqual(g(A()), u'A') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_name_of_overloaded_function(self): - @overload - def f(x): return 1 - @overload - def f(x): return 2 - - self.assertEqual(f.__name__, u'f') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overloaded_function_as_str(self): - @overload - def f(x): return 1 - @overload - def f(x): return 2 - - self.assertRegex(unicode(f), u'^$') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overload_with_default_arg_values(self): - @overload - def f(x=u'x'): return x + u'!' - @overload - def f(y): return y + 1 - - self.assertEqual(f(), u'x!') - self.assertEqual(f(u'y'), u'y!') - self.assertEqual(f(3), 4) - - @overload - def g(a, x=u'x', y=u'z'): return 1 - @overload - def g(a, x=1): return 2 - - self.assertEqual(g(1), 1) - self.assertEqual(g(u'x'), 2) - self.assertEqual(g(1, u'XX'), 1) - self.assertEqual(g(1, 2), 2) - self.assertEqual(g(1, u'XX', u'YY'), 1) - - with self.assertRaises(TypeError): - g(1, u'x', 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_no_matching_overload(self): - @overload - def f(x): return 1 - @overload - def f(x): return 2 - - # Fall back to the last overload variant if no annotation matches. - self.assertEqual(f(()), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_function_type_dispatch_in_overload(self): - @overload - def f(x): return 1 - @overload - def f(x): return 2 - - self.assertEqual(f(ord), 1) - self.assertEqual(f(unicode.find), 1) - self.assertEqual(f(u'x'.find), 1) - self.assertEqual(f(unicode), 1) - self.assertEqual(f(TestTyping), 1) - self.assertEqual(f(self.test_function_type_dispatch_in_overload), 1) - self.assertEqual(f(self.assertEqual), 1) - self.assertEqual(f(TestTyping.assertEqual), 1) - - class A(object): - def __call__(self): pass - - self.assertEqual(f(A()), 1) - - self.assertEqual(f(1), 2) - self.assertEqual(f(object()), 2) - - def test_typevar(self): - t = typevar(u't') - self.assertEqual(t.name, u't') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_typevar_in_overload(self): - t = typevar(u't') - - @overload - def f(x, y): return 1 - @overload - def f(x, y): return 2 - - self.assertEqual(f((), u'x'), 1) - self.assertEqual(f((), 1.1), 2) - - def test_simple_generic_class(self): - t = typevar(u't') - - class C(Generic[t]): - pass - - self.assertIs(C[int], C) - self.assertIsInstance(C(), C) - self.assertIsInstance(C[int](), C) - - def test_generic_class_with_two_typeargs(self): - t = typevar(u't') - u = typevar(u'u') - - class C(Generic[t, u]): - pass - - self.assertIs(C[int, unicode], C) - self.assertIsInstance(C(), C) - self.assertIsInstance(C[int, unicode](), C) - - def test_abstract_generic_class(self): - t = typevar(u't') - class C(AbstractGeneric[t]): - pass - class D(object): - pass - self.assertIs(C[int], C) - self.assertNotIsInstance(D(), C) - C.register(D) - self.assertIsInstance(D(), C) - - def test_sequence(self): - self.assertIs(Sequence[int], Sequence) - - self.assertIsInstance([], Sequence) - self.assertIsInstance((), Sequence) - self.assertIsInstance(u'', Sequence) - self.assertIsInstance('', Sequence) - self.assertIsInstance(xrange(5), Sequence) - self.assertNotIsInstance({}, Sequence) - - def test_abstract_set(self): - self.assertIs(AbstractSet[int], AbstractSet) - - self.assertIsInstance(set(), AbstractSet) - self.assertIsInstance(frozenset(), AbstractSet) - - def test_mapping(self): - self.assertIs(Mapping[int, unicode], Mapping) - self.assertIsInstance({}, Mapping) - - def test_io_types(self): - self.assertIsInstance(BinaryIO, type) - self.assertIsInstance(TextIO, type) - - def test_supports_int(self): - self.assertIsInstance(1, SupportsInt) - self.assertIsInstance(1.1, SupportsInt) - self.assertNotIsInstance(u'', SupportsInt) - self.assertNotIsInstance('', SupportsInt) - self.assertNotIsInstance((), SupportsInt) - - def test_supports_float(self): - self.assertIsInstance(1.1, SupportsFloat) - self.assertIsInstance(1, SupportsFloat) - self.assertNotIsInstance(u'', SupportsFloat) - self.assertNotIsInstance('', SupportsFloat) - self.assertNotIsInstance((), SupportsFloat) - - def test_supports_abs(self): - self.assertIsInstance(1.1, SupportsAbs) - self.assertIsInstance(1, SupportsAbs) - self.assertNotIsInstance(u'', SupportsAbs) - self.assertNotIsInstance((), SupportsAbs) - - def test_reversible(self): - self.assertIsInstance([], Reversible) - self.assertIsInstance(xrange(1), Reversible) - self.assertNotIsInstance((), Reversible) - self.assertNotIsInstance(u'', Reversible) - - def test_simple_protocol(self): - class P(Protocol): - def f(self): pass - - class A(object): - # Conforms to P - def f(self): pass - def g(self): pass - - class B(object): - # Does not conform to P - def g(self): pass - - self.assertIsInstance(A(), P) - self.assertNotIsInstance(B(), P) - - self.assertTrue(issubclass(A, P)) - self.assertFalse(issubclass(B, P)) - self.assertTrue(issubclass(P, Protocol)) - self.assertTrue(issubclass(Protocol, Protocol)) - self.assertTrue(issubclass(A, Protocol)) - - def test_issubclass_of_protocol(self): - class A(object): pass - self.assertTrue(issubclass(A, Protocol)) - - def test_protocol_with_two_attrs(self): - class P(Protocol): - def __int__(self): pass - x = 0 - - class A(object): - # Conforms to P; attribute values don't need to be similar - __int__ = 0 - def x(self): pass - def f(self): pass # Extra method - - class B(object): - # Does not conform to P - __int__ = 0 - class C(object): - # Does not conform to P - x = 0 - - self.assertIsInstance(A(), P) - self.assertNotIsInstance(B(), P) - self.assertNotIsInstance(C(), P) - - def test_protocol_inheritance(self): - class P(Protocol): - def f(self): pass - class PP(P, Protocol): - def g(self): pass - - class A(object): - # Conforms to P but not PP - def f(self): pass - class B(object): - # Conforms to P and PP - def f(self): pass - def g(self): pass - class C(object): - # Conforms to neither P nor PP - def g(self): pass - - self.assertIsInstance(A(), P) - self.assertIsInstance(B(), P) - self.assertIsInstance(B(), PP) - self.assertNotIsInstance(A(), PP) - self.assertNotIsInstance(C(), PP) - - def test_builtin_class_and_protocol(self): - class P(Protocol): - def __add__(self): pass - - self.assertIsInstance(u'', P) - self.assertIsInstance([], P) - self.assertIsInstance(1, P) - self.assertNotIsInstance({}, P) - - self.assertTrue(issubclass(unicode, P)) - self.assertFalse(issubclass(dict, P)) - - def test_generic_protocol(self): - t = typevar(u't') - class P(Protocol[t]): - x = 1 - class A(object): - x = 2 - self.assertIsInstance(A(), P) - - def test_indexing_in_protocol(self): - class P(Protocol): - def __getitem__(self): pass - class A(object): - def __getitem__(self): pass - class B(object): - pass - self.assertIsInstance(A(), P) - self.assertNotIsInstance(B(), P) - - def test_sized(self): - self.assertIsInstance([], Sized) - self.assertIsInstance((), Sized) - self.assertIsInstance(u'', Sized) - self.assertIsInstance('', Sized) - self.assertIsInstance({}, Sized) - self.assertIsInstance(set(), Sized) - self.assertIsInstance(xrange(5), Sized) - self.assertNotIsInstance(1, Sized) - - class A(object): - def __len__(self): pass - - self.assertIsInstance(A(), Sized) - - def test_iterable(self): - self.assertIsInstance([], Iterable) - class A(object): - def __iter__(self): pass - def g(self): pass - self.assertIsInstance(A(), Iterable) - self.assertNotIsInstance(1, Iterable) - - def test_iterator(self): - self.assertIsInstance(iter(u''), Iterator) - self.assertIsInstance(iter([]), Iterator) - self.assertIsInstance(iter({}), Iterator) - self.assertNotIsInstance([], Iterator) - - class A(object): - def __iter__(self): pass - def next(self): pass - self.assertIsInstance(A(), Iterator) - - class B(object): - def __iter__(self): pass - self.assertNotIsInstance(B(), Iterator) - - class C(object): - def next(self): pass - self.assertNotIsInstance(C(), Iterator) - - def test_class_inheritance_and_protocols(self): - class A(object): - def __iter__(self): pass - class B(A): - def next(self): pass - self.assertIsInstance(B(), Iterator) - self.assertNotIsInstance(A(), Iterator) - - def test_class_multiple_inheritance_and_protocols(self): - class A(object): - def __iter__(self): pass - class B(object): - def next(self): pass - class C(A, B): pass - self.assertIsInstance(C(), Iterator) - self.assertNotIsInstance(A(), Iterator) - self.assertNotIsInstance(B(), Iterator) - - def test_multiple_protocol_inheritance(self): - class P(Protocol): - x = 1 - class P2(Protocol): - y = 1 - class P3(P, P2, Protocol): pass - - class A(object): - x = 1 - y = 1 - class B(object): - x = 1 - class C(object): - y = 1 - - self.assertIsInstance(A(), P3) - self.assertNotIsInstance(B(), P3) - self.assertNotIsInstance(C(), P3) - - def test_protocol_docstrings(self): - class P(Protocol): - u"""blah""" - def f(self): pass - class A(object): - def f(self): pass - self.assertIsInstance(A(), P) - - def test_forward_ref_in_annotation(self): - A = forwardref(u'A') - def f(a): - return a - self.assertEqual(A.name, u'A') - class A(object): pass - - def test_string_literal_in_annotation(self): - def f(a): - return a + u'x' - def f(a): - return list(a) - - def test_undefined(self): - self.assertEqual(unicode(Undefined), u'') - with self.assertRaises(AttributeError): - Undefined.x = 1 - with self.assertRaises(AttributeError): - Undefined.x - with self.assertRaises(TypeError): - if Undefined == 0: pass - with self.assertRaises(TypeError): - if Undefined != 0: pass - with self.assertRaises(TypeError): - hash(Undefined) - with self.assertRaises(TypeError): - if Undefined: pass - with self.assertRaises(TypeError): - if not Undefined: pass - - @unittest.skip("overloads not supported in 2.7 yet") - def test_simple_string_literal_in_overload(self): - @overload - def f(a): return u's' - @overload - def f(a): return u'i' - - self.assertEqual(f(u''), u's') - self.assertEqual(f(2), u'i') - - @unittest.skip("overloads not supported in 2.7 yet") - def test_module_ref_string_literal_in_overload(self): - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(Dummy()), 1) - self.assertEqual(f(2), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_module_ref_string_literal_in_overload(self): - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(Dummy()), 1) - self.assertEqual(f(2), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_local_ref_string_literal_in_overload(self): - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - class C(object): pass - self.assertEqual(f(C()), 1) - self.assertEqual(f(2), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_any_string_literal_in_overload(self): - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(object()), 1) - self.assertEqual(f(None), 1) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_generic_type_string_literal_in_overload(self): - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f([]), 1) - self.assertEqual(f(()), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_tuple_type_string_literal_in_overload(self): - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(()), 1) - self.assertEqual(f([]), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_function_type_string_literal_in_overload(self): - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(ord), 1) - self.assertEqual(f([]), 2) - - @unittest.skip("overloads not supported in 2.7 yet") - def test_forward_ref_in_overload(self): - A = forwardref(u'A') - - @overload - def f(a): return 1 - @overload - def f(a): return 2 - - class A(object): pass - - self.assertEqual(f(A()), 1) - self.assertEqual(f(object()), 2) - - def test_construct_class_with_abstract_method(self): - t = typevar(u't') - - class A(AbstractGeneric[t]): - @abstractmethod - def f(self): pass - - class B(A): - def f(self): pass - - with self.assertRaises(TypeError): - A() - B() - - def test_protocol_with_abstract_method(self): - class A(Protocol): - @abstractmethod - def f(self): pass - - with self.assertRaises(TypeError): - A() # No implementation for abstract method. - - def test_protocol_inheritance(self): - class A(Protocol): - def f(self): return 1 - class B(A): pass - - self.assertEqual(B().f(), 1) - - class C(A): pass - # B is not a protocol since it doesn't explicitly subclass Protocol. - self.assertNotIsInstance(C(), B) - - def test_protocol_inheritance_with_abstract_method(self): - class A(Protocol): - @abstractmethod - def f(self): pass - class B(A): - pass - - with self.assertRaises(TypeError): - B() # No implementation for abstract method. - class C(A): - def f(self): pass - C() - - @unittest.skip("overloads not supported in 2.7 yet") - def test_overloaded_abstract_method(self): - class A(): - __metaclass__ = ABCMeta - @abstractmethod - @overload - def f(self, x): pass - @abstractmethod - @overload - def f(self, x): pass - - with self.assertRaises(TypeError): - A() - - class B(): - __metaclass__ = ABCMeta - @overload - @abstractmethod - def f(self, x): pass - - @overload - @abstractmethod - def f(self, x): pass - - with self.assertRaises(TypeError): - B() - - class C(B): - @overload - def f(self, x): - return 1 - - @overload - def f(self, x): - return u'x' - - self.assertEqual(C().f(2), 1) - self.assertEqual(C().f(None), u'x') - - -@overload -def global_overload(x): - return u's' - - -@overload -def global_overload(x): - return u'i' - - -class Dummy(object): - u"""Dummy class defined in module scope""" - - -if __name__ == u'__main__': - unittest.main() diff --git a/lib-typing/2.7/typing.py b/lib-typing/2.7/typing.py deleted file mode 100644 index 8180c34d0975..000000000000 --- a/lib-typing/2.7/typing.py +++ /dev/null @@ -1,550 +0,0 @@ -u"""Static type checking helpers""" - -from abc import ABCMeta, abstractmethod, abstractproperty -import inspect -import sys -import re - - -__all__ = [ - # Type system related - 'AbstractGeneric', - 'AbstractGenericMeta', - 'Any', - 'AnyStr', - 'Dict', - 'Function', - 'Generic', - 'GenericMeta', - 'IO', - 'List', - 'Match', - 'Pattern', - 'Protocol', - 'Set', - 'Tuple', - 'Undefined', - 'Union', - 'cast', - 'forwardref', - 'overload', - 'typevar', - # Protocols and abstract base classes - 'Container', - 'Iterable', - 'Iterator', - 'Sequence', - 'Sized', - 'AbstractSet', - 'Mapping', - 'BinaryIO', - 'TextIO', -] - - -class GenericMeta(type): - """Metaclass for generic classes that support indexing by types.""" - - def __getitem__(self, args): - # Just ignore args; they are for compile-time checks only. - return self - - -class Generic(object): - __metaclass__ = GenericMeta - """Base class for generic classes.""" - - -class AbstractGenericMeta(ABCMeta): - """Metaclass for abstract generic classes that support type indexing. - - This is used for both protocols and ordinary abstract classes. - """ - - def __new__(mcls, name, bases, namespace): - cls = ABCMeta.__new__(mcls, name, bases, namespace) - # 'Protocol' must be an explicit base class in order for a class to - # be a protocol. - cls._is_protocol = name == u'Protocol' or Protocol in bases - return cls - - def __getitem__(self, args): - # Just ignore args; they are for compile-time checks only. - return self - - -class Protocol(object): - __metaclass__ = AbstractGenericMeta - """Base class for protocol classes.""" - - @classmethod - def __subclasshook__(cls, c): - if not cls._is_protocol: - # No structural checks since this isn't a protocol. - return NotImplemented - - if cls is Protocol: - # Every class is a subclass of the empty protocol. - return True - - # Find all attributes defined in the protocol. - attrs = cls._get_protocol_attrs() - - for attr in attrs: - if not any(attr in d.__dict__ for d in c.__mro__): - return NotImplemented - return True - - @classmethod - def _get_protocol_attrs(cls): - # Get all Protocol base classes. - protocol_bases = [] - for c in cls.__mro__: - if getattr(c, '_is_protocol', False) and c.__name__ != 'Protocol': - protocol_bases.append(c) - - # Get attributes included in protocol. - attrs = set() - for base in protocol_bases: - for attr in base.__dict__.keys(): - # Include attributes not defined in any non-protocol bases. - for c in cls.__mro__: - if (c is not base and attr in c.__dict__ and - not getattr(c, '_is_protocol', False)): - break - else: - if (not attr.startswith(u'_abc_') and - attr != '__abstractmethods__' and - attr != '_is_protocol' and - attr != '__dict__' and - attr != '_get_protocol_attrs' and - attr != '__module__'): - attrs.add(attr) - - return attrs - - -class AbstractGeneric(object): - __metaclass__ = AbstractGenericMeta - """Base class for abstract generic classes.""" - - -class TypeAlias(object): - """Class for defining generic aliases for library types.""" - - def __init__(self, target_type): - self.target_type = target_type - - def __getitem__(self, typeargs): - return self.target_type - - -# Define aliases for built-in types that support indexing. -List = TypeAlias(list) -Dict = TypeAlias(dict) -Set = TypeAlias(set) -Tuple = TypeAlias(tuple) -Function = TypeAlias(callable) -Pattern = TypeAlias(type(re.compile(''))) -Match = TypeAlias(type(re.match('', ''))) - - -def union(x): return x - -Union = TypeAlias(union) - - -class typevar(object): - def __init__(self, name, values=None): - self.name = name - self.values = values - - -# Predefined type variables. -AnyStr = typevar('AnyStr', values=(str, unicode)) - - -class forwardref(object): - def __init__(self, name): - self.name = name - - -def Any(x): - """The Any type; can also be used to cast a value to type Any.""" - return x - - -def cast(type, object): - """Cast a value to a type. - - This only affects static checking; simply return object at runtime. - """ - return object - - -def overload(func): - """Function decorator for defining overloaded functions.""" - frame = sys._getframe(1) - locals = frame.f_locals - # See if there is a previous overload variant available. Also verify - # that the existing function really is overloaded: otherwise, replace - # the definition. The latter is actually important if we want to reload - # a library module such as genericpath with a custom one that uses - # overloading in the implementation. - if func.__name__ in locals and hasattr(locals[func.__name__], 'dispatch'): - orig_func = locals[func.__name__] - - def wrapper(*args, **kwargs): - ret, ok = orig_func.dispatch(*args, **kwargs) - if ok: - return ret - return func(*args, **kwargs) - wrapper.isoverload = True - wrapper.dispatch = make_dispatcher(func, orig_func.dispatch) - wrapper.next = orig_func - wrapper.__name__ = func.__name__ - if hasattr(func, '__isabstractmethod__'): - # Note that we can't reliably check that abstractmethod is - # used consistently across overload variants, so we let a - # static checker do it. - wrapper.__isabstractmethod__ = func.__isabstractmethod__ - return wrapper - else: - # Return the initial overload variant. - func.isoverload = True - func.dispatch = make_dispatcher(func) - func.next = None - return func - - -def is_erased_type(t): - return t is Any or isinstance(t, typevar) - - -def make_dispatcher(func, previous=None): - """Create argument dispatcher for an overloaded function. - - Also handle chaining of multiple overload variants. - """ - (args, varargs, varkw, defaults) = inspect.getargspec(func) - - argtypes = [] - for arg in args: - ann = None # annotations.get(arg) - if isinstance(ann, forwardref): - ann = ann.name - if is_erased_type(ann): - ann = None - elif isinstance(ann, unicode): - # The annotation is a string => evaluate it lazily when the - # overloaded function is first called. - frame = sys._getframe(2) - t = [None] - ann_str = ann - - def check(x): - if not t[0]: - # Evaluate string in the context of the overload caller. - t[0] = eval(ann_str, frame.f_globals, frame.f_locals) - if is_erased_type(t[0]): - # Anything goes. - t[0] = object - if isinstance(t[0], type): - return isinstance(x, t[0]) - else: - return t[0](x) - ann = check - argtypes.append(ann) - - maxargs = len(argtypes) - minargs = maxargs - if defaults: - minargs = len(argtypes) - len(defaults) - - def dispatch(*args, **kwargs): - if previous: - ret, ok = previous(*args, **kwargs) - if ok: - return ret, ok - - nargs = len(args) - if nargs < minargs or nargs > maxargs: - # Invalid argument count. - return None, False - - for i in xrange(nargs): - argtype = argtypes[i] - if argtype: - if isinstance(argtype, type): - if not isinstance(args[i], argtype): - break - else: - if not argtype(args[i]): - break - else: - return func(*args, **kwargs), True - return None, False - return dispatch - - -class Undefined(object): - """Class that represents an undefined value with a specified type. - - At runtime the name Undefined is bound to an instance of this - class. The intent is that any operation on an Undefined object - raises an exception, including use in a boolean context. Some - operations cannot be disallowed: Undefined can be used as an - operand of 'is', and it can be assigned to variables and stored in - containers. - - 'Undefined' makes it possible to declare the static type of a - variable even if there is no useful default value to initialize it - with: - - from typing import Undefined - x = Undefined(int) - y = Undefined # type: int - - The latter form can be used if efficiency is of utmost importance, - since it saves a call operation and potentially additional - operations needed to evaluate a type expression. Undefined(x) - just evaluates to Undefined, ignoring the argument value. - """ - - def __repr__(self): - return '' - - def __setattr__(self, attr, value): - raise AttributeError("'Undefined' object has no attribute '%s'" % attr) - - def __eq__(self, other): - raise TypeError("'Undefined' object cannot be compared") - - def __ne__(self, other): - raise TypeError("'Undefined' object cannot be compared") - - def __call__(self, type): - return self - - def __nonzero__(self): - raise TypeError("'Undefined' object is not valid as a boolean") - - def __hash__(self): - raise TypeError("'Undefined' object is not hashable") - - -Undefined = Undefined() - - -# Abstract classes - - -T = typevar('T') -KT = typevar('KT') -VT = typevar('VT') - - -class SupportsInt(Protocol): - @abstractmethod - def __int__(self): pass - - -class SupportsFloat(Protocol): - @abstractmethod - def __float__(self): pass - - -class SupportsAbs(Protocol[T]): - @abstractmethod - def __abs__(self): pass - - -class Reversible(Protocol[T]): - @abstractmethod - def __reversed__(self): pass - - -class Sized(Protocol): - @abstractmethod - def __len__(self): pass - - -class Container(Protocol[T]): - @abstractmethod - def __contains__(self, x): pass - - -class Iterable(Protocol[T]): - @abstractmethod - def __iter__(self): pass - - -class Iterator(Iterable[T], Protocol[T]): - @abstractmethod - def next(self): pass - - -class Sequence(Sized, Iterable[T], Container[T], AbstractGeneric[T]): - @abstractmethod - def __getitem__(self, i): pass - - @abstractmethod - def __getitem__(self, s): pass - - @abstractmethod - def __reversed__(self, s): pass - - @abstractmethod - def index(self, x): pass - - @abstractmethod - def count(self, x): pass - - -for t in list, tuple, unicode, str, xrange: - Sequence.register(t) - - -class AbstractSet(Sized, Iterable[T], AbstractGeneric[T]): - @abstractmethod - def __contains__(self, x): pass - @abstractmethod - def __and__(self, s): pass - @abstractmethod - def __or__(self, s): pass - @abstractmethod - def __sub__(self, s): pass - @abstractmethod - def __xor__(self, s): pass - @abstractmethod - def isdisjoint(self, s): pass - - -for t in set, frozenset, type({}.keys()), type({}.items()): - AbstractSet.register(t) - - -class Mapping(Sized, Iterable[KT], AbstractGeneric[KT, VT]): - @abstractmethod - def __getitem__(self, k): pass - @abstractmethod - def __setitem__(self, k, v): pass - @abstractmethod - def __delitem__(self, v): pass - @abstractmethod - def __contains__(self, o): pass - - @abstractmethod - def clear(self): pass - @abstractmethod - def copy(self): pass - @abstractmethod - def get(self, k): pass - @abstractmethod - def get(self, k, default): pass - @abstractmethod - def pop(self, k): pass - @abstractmethod - def pop(self, k, default): pass - @abstractmethod - def popitem(self): pass - @abstractmethod - def setdefault(self, k): pass - @abstractmethod - def setdefault(self, k, default): pass - - @abstractmethod - def update(self, m): pass - @abstractmethod - def update(self, m): pass - - @abstractmethod - def keys(self): pass - @abstractmethod - def values(self): pass - @abstractmethod - def items(self): pass - - -# TODO Consider more types: os.environ, etc. However, these add dependencies. -Mapping.register(dict) - - -# Note that the BinaryIO and TextIO classes must be in sync with typing module stubs. - - -class IO(AbstractGeneric[AnyStr]): - @abstractproperty - def mode(self): pass - @abstractproperty - def name(self): pass - @abstractmethod - def close(self): pass - @abstractmethod - def closed(self): pass - @abstractmethod - def fileno(self): pass - @abstractmethod - def flush(self): pass - @abstractmethod - def isatty(self): pass - @abstractmethod - def read(self, n=-1): pass - @abstractmethod - def readable(self): pass - @abstractmethod - def readline(self, limit=-1): pass - @abstractmethod - def readlines(self, hint=-1): pass - @abstractmethod - def seek(self, offset, whence=0): pass - @abstractmethod - def seekable(self): pass - @abstractmethod - def tell(self): pass - @abstractmethod - def truncate(self, size=None): pass - @abstractmethod - def writable(self): pass - @abstractmethod - def write(self, s): pass - @abstractmethod - def writelines(self, lines): pass - - @abstractmethod - def __enter__(self): pass - @abstractmethod - def __exit__(self, type, value, traceback): pass - - -class BinaryIO(IO[str]): - @overload - @abstractmethod - def write(self, s): pass - @overload - @abstractmethod - def write(self, s): pass - - @abstractmethod - def __enter__(self): pass - - -class TextIO(IO[unicode]): - @abstractproperty - def buffer(self): pass - @abstractproperty - def encoding(self): pass - @abstractproperty - def errors(self): pass - @abstractproperty - def line_buffering(self): pass - @abstractproperty - def newlines(self): pass - @abstractmethod - def __enter__(self): pass - - -# TODO Register TextIO/BinaryIO as the base class of file-like types. - - -del t diff --git a/lib-typing/3.2/test_typing.py b/lib-typing/3.2/test_typing.py deleted file mode 100644 index 130b77eb388a..000000000000 --- a/lib-typing/3.2/test_typing.py +++ /dev/null @@ -1,831 +0,0 @@ -from abc import abstractmethod, ABCMeta -import unittest - -from typing import ( - List, Dict, Set, Tuple, Pattern, Match, Any, Function, Generic, - AbstractGeneric, Protocol, Sized, Iterable, Iterator, Sequence, - AbstractSet, Mapping, BinaryIO, TextIO, SupportsInt, SupportsFloat, - SupportsAbs, SupportsRound, Reversible, Undefined, AnyStr, builtinclass, - cast, disjointclass, ducktype, forwardref, overload, typevar, - NamedTuple -) - - -class TestTyping(unittest.TestCase): - def test_List(self): - self.assertIs(List[int], list) - self.assertIs(List[str], list) - - def test_Dict(self): - self.assertIs(Dict[int, str], dict) - self.assertIs(Dict[str, int], dict) - - def test_Set(self): - self.assertIs(Set[int], set) - self.assertIs(Set[str], set) - - def test_Tuple(self): - self.assertIs(Tuple[int], tuple) - self.assertIs(Tuple[str, int], tuple) - self.assertIs(Tuple[str, int, bool], tuple) - - def test_Pattern(self): - import re - self.assertIs(type(re.compile('')), Pattern.target_type) - self.assertIs(type(re.compile(b'')), Pattern.target_type) - - def test_Match(self): - import re - self.assertIs(type(re.match('', '')), Match.target_type) - self.assertIs(type(re.match(b'', b'')), Match.target_type) - - def test_Any(self): - o = object() - self.assertIs(Any(o), o) - s = 'x' - self.assertIs(Any(s), s) - - def test_Function(self): - # Just check that we can call Function. Don't care about return value. - Function[[], int] - Function[[int], None] - Function[[int, str], bool] - - def test_cast(self): - o = object() - # cast performs no runtime checks! - self.assertIs(cast(int, o), o) - s = 'x' - self.assertIs(cast(str, s), s) - self.assertIs(cast('xyz', s), s) - # cast does not check type validity; anything goes. - self.assertIs(cast(o, s), s) - - def test_simple_overload(self): - @overload - def f(x:str) -> str: return x + 'string' - @overload - def f(x:int) -> str: return x + 1 - - self.assertEqual(f('x'), 'xstring') - self.assertEqual(f(1), 2) - - @overload - def g(x:int) -> str: return 'integer' - @overload - def g(x:str) -> str: return 'string' - - self.assertEqual(g('x'), 'string') - self.assertEqual(g(1), 'integer') - - def test_overload_with_three_variants(self): - @overload - def f(x:str) -> str: return 'string' - @overload - def f(x:int) -> str: return 'integer' - @overload - def f(x:float) -> str: return 'floating' - - self.assertEqual(f('x'), 'string') - self.assertEqual(f(1), 'integer') - self.assertEqual(f(1.0), 'floating') - - def test_overload_with_two_args(self): - @overload - def f(x:str, y:str) -> int: return (1, x, y) - @overload - def f(x:str, y:int) -> int: return (2, x, y) - - self.assertEqual(f('x', 'y'), (1, 'x', 'y')) - self.assertEqual(f('z', 3), (2, 'z', 3)) - - @overload - def g(x:str, y:str) -> int: return 1 - @overload - def g(x:int, y:str) -> int: return 2 - - self.assertEqual(g('x', 'y'), 1) - self.assertEqual(g('x', 1), 2) - - def test_global_overload(self): - self.assertEqual(global_overload('x'), 's') - self.assertEqual(global_overload(1), 'i') - - def test_partial_overload_annotation(self): - @overload - def f(x:str, y): return 1 - @overload - def f(x, y:int): return 2 - - self.assertEqual(f('x', object()), 1) - self.assertEqual(f(object(), 1), 2) - - @overload - def method_overload(self, x:str) -> str: - return 's' - - @overload - def method_overload(self, x:int) -> str: - return 'i' - - def test_method_overload(self): - self.assertEqual(self.method_overload('x'), 's') - self.assertEqual(self.method_overload(1), 'i') - - def test_overload_with_any_type(self): - @overload - def f(x:Any, y:int): return 1 - @overload - def f(x:Any, y:Any): return 2 - - self.assertEqual(f((), 0), 1) - self.assertEqual(f((), ()), 2) - - def test_overload_with_type_alias(self): - @overload - def f(x:List[int]): return 1 - @overload - def f(x:Dict[int, str]): return 2 - @overload - def f(x:Tuple[int, str]): return 3 - @overload - def f(x): return 4 - - self.assertEqual(f([]), 1) - self.assertEqual(f({}), 2) - self.assertEqual(f(()), 3) - self.assertEqual(f((1, 'x')), 3) - self.assertEqual(f(1), 4) - - def test_call_overload_with_invalid_arg_count(self): - @overload - def f(x:int): return 1 - @overload - def f(x:Any): return 2 - - msg1 = r'f\(\) takes exactly 1 argument \(%d given\)' - msg2 = r'f\(\) takes exactly 1 positional argument \(%d given\)' - - with self.assertRaisesRegex(TypeError, msg1 % 0): - f() - with self.assertRaisesRegex(TypeError, msg2 % 2): - f(1, 2) - with self.assertRaisesRegex(TypeError, msg2 % 3): - f('x', 'y', 'z') - - def test_overload_with_variable_argument_counts(self): - @overload - def f(): return None - @overload - def f(x): return x - - self.assertEqual(f(), None) - self.assertEqual(f(1), 1) - - @overload - def g(x): return x + 1 - @overload - def g(): return None - - self.assertEqual(g(), None) - self.assertEqual(g(1), 2) - - msg = r'g\(\) takes no arguments \(2 given\)' - with self.assertRaisesRegex(TypeError, msg): - g(1, 2) - - def test_overload_dispatch_order(self): - class A: pass - class B(A): pass - class C(B): pass - - @overload - def f(x:B): return 'B' - @overload - def f(x:A): return 'A' - - self.assertEqual(f(B()), 'B') - self.assertEqual(f(A()), 'A') - - @overload - def g(x:C): return 'C' - @overload - def g(x:B): return 'B' - @overload - def g(x:A): return 'A' - - self.assertEqual(g(C()), 'C') - self.assertEqual(g(B()), 'B') - self.assertEqual(g(A()), 'A') - - def test_name_of_overloaded_function(self): - @overload - def f(x:str): return 1 - @overload - def f(x): return 2 - - self.assertEqual(f.__name__, 'f') - - def test_overloaded_function_as_str(self): - @overload - def f(x:str): return 1 - @overload - def f(x): return 2 - - self.assertRegex(str(f), '^$') - - def test_overload_with_default_arg_values(self): - @overload - def f(x:str='x'): return x + '!' - @overload - def f(y): return y + 1 - - self.assertEqual(f(), 'x!') - self.assertEqual(f('y'), 'y!') - self.assertEqual(f(3), 4) - - @overload - def g(a:int, x:str='x', y:str='z'): return 1 - @overload - def g(a, x=1): return 2 - - self.assertEqual(g(1), 1) - self.assertEqual(g('x'), 2) - self.assertEqual(g(1, 'XX'), 1) - self.assertEqual(g(1, 2), 2) - self.assertEqual(g(1, 'XX', 'YY'), 1) - - with self.assertRaises(TypeError): - g(1, 'x', 2) - - def test_no_matching_overload(self): - @overload - def f(x:str): return 1 - @overload - def f(x:int): return 2 - - # Fall back to the last overload variant if no annotation matches. - self.assertEqual(f(()), 2) - - def test_function_type_dispatch_in_overload(self): - @overload - def f(x:Function[[], str]): return 1 - @overload - def f(x): return 2 - - self.assertEqual(f(ord), 1) - self.assertEqual(f(str.find), 1) - self.assertEqual(f('x'.find), 1) - self.assertEqual(f(str), 1) - self.assertEqual(f(TestTyping), 1) - self.assertEqual(f(self.test_function_type_dispatch_in_overload), 1) - self.assertEqual(f(self.assertEqual), 1) - self.assertEqual(f(TestTyping.assertEqual), 1) - - class A: - def __call__(self): pass - - self.assertEqual(f(A()), 1) - - self.assertEqual(f(1), 2) - self.assertEqual(f(object()), 2) - - def test_typevar(self): - t = typevar('t') - self.assertEqual(t.name, 't') - self.assertIsNone(t.values) - - def test_typevar_values(self): - t = typevar('t', values=(int, str)) - self.assertEqual(t.name, 't') - self.assertEqual(t.values, (int, str)) - - def test_predefined_typevars(self): - self.assertEqual(AnyStr.name, 'AnyStr') - self.assertEqual(AnyStr.values, (str, bytes)) - - def test_typevar_in_overload(self): - t = typevar('t') - - @overload - def f(x:t, y:str): return 1 - @overload - def f(x, y): return 2 - - self.assertEqual(f((), 'x'), 1) - self.assertEqual(f((), 1.1), 2) - - def test_simple_generic_class(self): - t = typevar('t') - - class C(Generic[t]): - pass - - self.assertIs(C[int], C) - self.assertIsInstance(C(), C) - self.assertIsInstance(C[int](), C) - - def test_generic_class_with_two_typeargs(self): - t = typevar('t') - u = typevar('u') - - class C(Generic[t, u]): - pass - - self.assertIs(C[int, str], C) - self.assertIsInstance(C(), C) - self.assertIsInstance(C[int, str](), C) - - def test_abstract_generic_class(self): - t = typevar('t') - class C(AbstractGeneric[t]): - pass - class D: - pass - self.assertIs(C[int], C) - self.assertNotIsInstance(D(), C) - C.register(D) - self.assertIsInstance(D(), C) - - def test_sequence(self): - self.assertIs(Sequence[int], Sequence) - - self.assertIsInstance([], Sequence) - self.assertIsInstance((), Sequence) - self.assertIsInstance('', Sequence) - self.assertIsInstance(b'', Sequence) - self.assertIsInstance(range(5), Sequence) - self.assertNotIsInstance({}, Sequence) - - def test_abstract_set(self): - self.assertIs(AbstractSet[int], AbstractSet) - - self.assertIsInstance(set(), AbstractSet) - self.assertIsInstance(frozenset(), AbstractSet) - self.assertIsInstance({}.keys(), AbstractSet) - self.assertIsInstance({}.items(), AbstractSet) - # This is consistent with collections.Set. - self.assertNotIsInstance({}.values(), AbstractSet) - - def test_mapping(self): - self.assertIs(Mapping[int, str], Mapping) - self.assertIsInstance({}, Mapping) - - def test_io_types(self): - self.assertIsInstance(BinaryIO, type) - self.assertIsInstance(TextIO, type) - - def test_supports_int(self): - self.assertIsInstance(1, SupportsInt) - self.assertIsInstance(1.1, SupportsInt) - self.assertNotIsInstance('', SupportsInt) - self.assertNotIsInstance(b'', SupportsInt) - self.assertNotIsInstance((), SupportsInt) - - def test_supports_float(self): - self.assertIsInstance(1.1, SupportsFloat) - self.assertIsInstance(1, SupportsFloat) - self.assertNotIsInstance('', SupportsFloat) - self.assertNotIsInstance(b'', SupportsFloat) - self.assertNotIsInstance((), SupportsFloat) - - def test_supports_abs(self): - self.assertIsInstance(1.1, SupportsAbs) - self.assertIsInstance(1, SupportsAbs) - self.assertNotIsInstance('', SupportsAbs) - self.assertNotIsInstance((), SupportsAbs) - - def test_supports_round(self): - self.assertIsInstance(1.1, SupportsRound) - self.assertIsInstance(1, SupportsRound) - self.assertNotIsInstance('', SupportsRound) - self.assertNotIsInstance((), SupportsRound) - - def test_reversible(self): - self.assertIsInstance([], Reversible) - self.assertIsInstance(range(1), Reversible) - self.assertNotIsInstance((), Reversible) - self.assertNotIsInstance('', Reversible) - - def test_simple_protocol(self): - class P(Protocol): - def f(self): pass - - class A: - # Conforms to P - def f(self): pass - def g(self): pass - - class B: - # Does not conform to P - def g(self): pass - - self.assertIsInstance(A(), P) - self.assertNotIsInstance(B(), P) - - self.assertTrue(issubclass(A, P)) - self.assertFalse(issubclass(B, P)) - self.assertTrue(issubclass(P, Protocol)) - self.assertTrue(issubclass(Protocol, Protocol)) - self.assertTrue(issubclass(A, Protocol)) - - def test_issubclass_of_protocol(self): - class A: pass - self.assertTrue(issubclass(A, Protocol)) - - def test_protocol_with_two_attrs(self): - class P(Protocol): - def __int__(self): pass - x = 0 - - class A: - # Conforms to P; attribute values don't need to be similar - __int__ = 0 - def x(self): pass - def f(self): pass # Extra method - - class B: - # Does not conform to P - __int__ = 0 - class C: - # Does not conform to P - x = 0 - - self.assertIsInstance(A(), P) - self.assertNotIsInstance(B(), P) - self.assertNotIsInstance(C(), P) - - def test_protocol_inheritance(self): - class P(Protocol): - def f(self): pass - class PP(P, Protocol): - def g(self): pass - - class A: - # Conforms to P but not PP - def f(self): pass - class B: - # Conforms to P and PP - def f(self): pass - def g(self): pass - class C: - # Conforms to neither P nor PP - def g(self): pass - - self.assertIsInstance(A(), P) - self.assertIsInstance(B(), P) - self.assertIsInstance(B(), PP) - self.assertNotIsInstance(A(), PP) - self.assertNotIsInstance(C(), PP) - - def test_builtin_class_and_protocol(self): - class P(Protocol): - def __add__(self): pass - - self.assertIsInstance('', P) - self.assertIsInstance([], P) - self.assertIsInstance(1, P) - self.assertNotIsInstance({}, P) - - self.assertTrue(issubclass(str, P)) - self.assertFalse(issubclass(dict, P)) - - def test_generic_protocol(self): - t = typevar('t') - class P(Protocol[t]): - x = 1 - class A: - x = 2 - self.assertIsInstance(A(), P) - - def test_indexing_in_protocol(self): - class P(Protocol): - def __getitem__(self): pass - class A: - def __getitem__(self): pass - class B: - pass - self.assertIsInstance(A(), P) - self.assertNotIsInstance(B(), P) - - def test_sized(self): - self.assertIsInstance([], Sized) - self.assertIsInstance((), Sized) - self.assertIsInstance('', Sized) - self.assertIsInstance(b'', Sized) - self.assertIsInstance({}, Sized) - self.assertIsInstance(set(), Sized) - self.assertIsInstance(range(5), Sized) - self.assertNotIsInstance(1, Sized) - - class A: - def __len__(self): pass - - self.assertIsInstance(A(), Sized) - - def test_iterable(self): - self.assertIsInstance([], Iterable) - class A: - def __iter__(self): pass - def g(self): pass - self.assertIsInstance(A(), Iterable) - self.assertNotIsInstance(1, Iterable) - - def test_iterator(self): - self.assertIsInstance(iter(''), Iterator) - self.assertIsInstance(iter([]), Iterator) - self.assertIsInstance(iter({}), Iterator) - self.assertNotIsInstance([], Iterator) - - class A: - def __iter__(self): pass - def __next__(self): pass - self.assertIsInstance(A(), Iterator) - - class B: - def __iter__(self): pass - self.assertNotIsInstance(B(), Iterator) - - class C: - def __next__(self): pass - self.assertNotIsInstance(C(), Iterator) - - def test_class_inheritance_and_protocols(self): - class A: - def __iter__(self): pass - class B(A): - def __next__(self): pass - self.assertIsInstance(B(), Iterator) - self.assertNotIsInstance(A(), Iterator) - - def test_class_multiple_inheritance_and_protocols(self): - class A: - def __iter__(self): pass - class B: - def __next__(self): pass - class C(A, B): pass - self.assertIsInstance(C(), Iterator) - self.assertNotIsInstance(A(), Iterator) - self.assertNotIsInstance(B(), Iterator) - - def test_multiple_protocol_inheritance(self): - class P(Protocol): - x = 1 - class P2(Protocol): - y = 1 - class P3(P, P2, Protocol): pass - - class A: - x = 1 - y = 1 - class B: - x = 1 - class C: - y = 1 - - self.assertIsInstance(A(), P3) - self.assertNotIsInstance(B(), P3) - self.assertNotIsInstance(C(), P3) - - def test_protocol_docstrings(self): - class P(Protocol): - """blah""" - def f(self): pass - class A: - def f(self): pass - self.assertIsInstance(A(), P) - - def test_forward_ref_in_annotation(self): - A = forwardref('A') - def f(a:A) -> A: - return a - self.assertEqual(A.name, 'A') - class A: pass - - def test_string_literal_in_annotation(self): - def f(a:'str') -> 'str': - return a + 'x' - def f(a:'Iterable[int]') -> 'List[int]': - return list(a) - - def test_undefined(self): - self.assertEqual(str(Undefined), '') - with self.assertRaises(AttributeError): - Undefined.x = 1 - with self.assertRaises(AttributeError): - Undefined.x - with self.assertRaises(TypeError): - if Undefined == 0: pass - with self.assertRaises(TypeError): - if Undefined != 0: pass - with self.assertRaises(TypeError): - hash(Undefined) - with self.assertRaises(TypeError): - if Undefined: pass - with self.assertRaises(TypeError): - if not Undefined: pass - - def test_simple_string_literal_in_overload(self): - @overload - def f(a:'str') -> str: return 's' - @overload - def f(a:'int') -> str: return 'i' - - self.assertEqual(f(''), 's') - self.assertEqual(f(2), 'i') - - def test_module_ref_string_literal_in_overload(self): - @overload - def f(a:'Dummy'): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(Dummy()), 1) - self.assertEqual(f(2), 2) - - def test_module_ref_string_literal_in_overload(self): - @overload - def f(a:'Dummy'): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(Dummy()), 1) - self.assertEqual(f(2), 2) - - def test_local_ref_string_literal_in_overload(self): - @overload - def f(a:'C'): return 1 - @overload - def f(a): return 2 - - class C: pass - self.assertEqual(f(C()), 1) - self.assertEqual(f(2), 2) - - def test_any_string_literal_in_overload(self): - @overload - def f(a:'Any'): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(object()), 1) - self.assertEqual(f(None), 1) - - def test_generic_type_string_literal_in_overload(self): - @overload - def f(a:'List[int]'): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f([]), 1) - self.assertEqual(f(()), 2) - - def test_tuple_type_string_literal_in_overload(self): - @overload - def f(a:'Tuple[int]'): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(()), 1) - self.assertEqual(f([]), 2) - - def test_function_type_string_literal_in_overload(self): - @overload - def f(a:'Function[[], int]'): return 1 - @overload - def f(a): return 2 - - self.assertEqual(f(ord), 1) - self.assertEqual(f([]), 2) - - def test_forward_ref_in_overload(self): - A = forwardref('A') - - @overload - def f(a:A): return 1 - @overload - def f(a): return 2 - - class A: pass - - self.assertEqual(f(A()), 1) - self.assertEqual(f(object()), 2) - - def test_construct_class_with_abstract_method(self): - t = typevar('t') - - class A(AbstractGeneric[t]): - @abstractmethod - def f(self): pass - - class B(A): - def f(self): pass - - with self.assertRaises(TypeError): - A() - B() - - def test_protocol_with_abstract_method(self): - class A(Protocol): - @abstractmethod - def f(self): pass - - with self.assertRaises(TypeError): - A() # No implementation for abstract method. - - def test_protocol_inheritance(self): - class A(Protocol): - def f(self): return 1 - class B(A): pass - - self.assertEqual(B().f(), 1) - - class C(A): pass - # B is not a protocol since it doesn't explicitly subclass Protocol. - self.assertNotIsInstance(C(), B) - - def test_protocol_inheritance_with_abstract_method(self): - class A(Protocol): - @abstractmethod - def f(self): pass - class B(A): - pass - - with self.assertRaises(TypeError): - B() # No implementation for abstract method. - class C(A): - def f(self): pass - C() - - def test_overloaded_abstract_method(self): - class A(metaclass=ABCMeta): - @abstractmethod - @overload - def f(self, x:int): pass - @abstractmethod - @overload - def f(self, x): pass - - with self.assertRaises(TypeError): - A() - - class B(metaclass=ABCMeta): - @overload - @abstractmethod - def f(self, x:int) -> int: pass - - @overload - @abstractmethod - def f(self, x) -> None: pass - - with self.assertRaises(TypeError): - B() - - class C(B): - @overload - def f(self, x:int) -> int: - return 1 - - @overload - def f(self, x): - return 'x' - - self.assertEqual(C().f(2), 1) - self.assertEqual(C().f(None), 'x') - - def test_builtinclass(self): - class A: pass - self.assertIs(builtinclass(int), int) - self.assertIs(builtinclass(A), A) - - def test_ducktype(self): - class A: pass - self.assertIs(ducktype(str)(A), A) - - def test_disjointclass(self): - class A: pass - self.assertIs(disjointclass(str)(A), A) - self.assertIs(disjointclass('str')(A), A) - - def test_NamedTuple(self): - n = NamedTuple('n', [('a', int), ('b', str)]) - assert n(a=1, b='x') == (1, 'x') - assert n(1, 'x')[1] == 'x' - - -@overload -def global_overload(x:str) -> str: - return 's' - - -@overload -def global_overload(x:int) -> str: - return 'i' - - -class Dummy: - """Dummy class defined in module scope""" - - -if __name__ == '__main__': - unittest.main() diff --git a/lib-typing/3.2/typing.py b/lib-typing/3.2/typing.py deleted file mode 100644 index 48d6b3456a51..000000000000 --- a/lib-typing/3.2/typing.py +++ /dev/null @@ -1,592 +0,0 @@ -"""Static type checking helpers""" - -from abc import ABCMeta, abstractmethod, abstractproperty -import collections -import inspect -import sys -import re - - -__all__ = [ - # Type system related - 'AbstractGeneric', - 'AbstractGenericMeta', - 'Any', - 'AnyStr', - 'Dict', - 'Function', - 'Generic', - 'GenericMeta', - 'IO', - 'List', - 'Match', - 'NamedTuple', - 'Pattern', - 'Protocol', - 'Set', - 'Tuple', - 'Undefined', - 'Union', - 'cast', - 'forwardref', - 'overload', - 'typevar', - # Protocols and abstract base classes - 'Container', - 'Iterable', - 'Iterator', - 'Sequence', - 'Sized', - 'AbstractSet', - 'Mapping', - 'BinaryIO', - 'TextIO', -] - - -def builtinclass(cls): - """Mark a class as a built-in/extension class for type checking.""" - return cls - - -def ducktype(type): - """Return a duck type declaration decorator. - - The decorator only affects type checking. - """ - def decorator(cls): - return cls - return decorator - - -def disjointclass(type): - """Return a disjoint class declaration decorator. - - The decorator only affects type checking. - """ - def decorator(cls): - return cls - return decorator - - -class GenericMeta(type): - """Metaclass for generic classes that support indexing by types.""" - - def __getitem__(self, args): - # Just ignore args; they are for compile-time checks only. - return self - - -class Generic(metaclass=GenericMeta): - """Base class for generic classes.""" - - -class AbstractGenericMeta(ABCMeta): - """Metaclass for abstract generic classes that support type indexing. - - This is used for both protocols and ordinary abstract classes. - """ - - def __new__(mcls, name, bases, namespace): - cls = super().__new__(mcls, name, bases, namespace) - # 'Protocol' must be an explicit base class in order for a class to - # be a protocol. - cls._is_protocol = name == 'Protocol' or Protocol in bases - return cls - - def __getitem__(self, args): - # Just ignore args; they are for compile-time checks only. - return self - - -class Protocol(metaclass=AbstractGenericMeta): - """Base class for protocol classes.""" - - @classmethod - def __subclasshook__(cls, c): - if not cls._is_protocol: - # No structural checks since this isn't a protocol. - return NotImplemented - - if cls is Protocol: - # Every class is a subclass of the empty protocol. - return True - - # Find all attributes defined in the protocol. - attrs = cls._get_protocol_attrs() - - for attr in attrs: - if not any(attr in d.__dict__ for d in c.__mro__): - return NotImplemented - return True - - @classmethod - def _get_protocol_attrs(cls): - # Get all Protocol base classes. - protocol_bases = [] - for c in cls.__mro__: - if getattr(c, '_is_protocol', False) and c.__name__ != 'Protocol': - protocol_bases.append(c) - - # Get attributes included in protocol. - attrs = set() - for base in protocol_bases: - for attr in base.__dict__.keys(): - # Include attributes not defined in any non-protocol bases. - for c in cls.__mro__: - if (c is not base and attr in c.__dict__ and - not getattr(c, '_is_protocol', False)): - break - else: - if (not attr.startswith('_abc_') and - attr != '__abstractmethods__' and - attr != '_is_protocol' and - attr != '__dict__' and - attr != '_get_protocol_attrs' and - attr != '__module__'): - attrs.add(attr) - - return attrs - - -class AbstractGeneric(metaclass=AbstractGenericMeta): - """Base class for abstract generic classes.""" - - -class TypeAlias: - """Class for defining generic aliases for library types.""" - - def __init__(self, target_type): - self.target_type = target_type - - def __getitem__(self, typeargs): - return self.target_type - - -Traceback = object() # TODO proper type object - - -# Define aliases for built-in types that support indexing. -List = TypeAlias(list) -Dict = TypeAlias(dict) -Set = TypeAlias(set) -Tuple = TypeAlias(tuple) -Function = TypeAlias(callable) -Pattern = TypeAlias(type(re.compile(''))) -Match = TypeAlias(type(re.match('', ''))) - - -def union(x): return x - - -Union = TypeAlias(union) - - -def NamedTuple(typename, fields): - return collections.namedtuple(typename, - (name for name, type in fields)) - - -class typevar: - def __init__(self, name, *, values=None): - self.name = name - self.values = values - - -# Predefined type variables. -AnyStr = typevar('AnyStr', values=(str, bytes)) - - -class forwardref: - def __init__(self, name): - self.name = name - - -def Any(x): - """The Any type; can also be used to cast a value to type Any.""" - return x - -def cast(type, object): - """Cast a value to a type. - - This only affects static checking; simply return object at runtime. - """ - return object - - -def overload(func): - """Function decorator for defining overloaded functions.""" - frame = sys._getframe(1) - locals = frame.f_locals - # See if there is a previous overload variant available. Also verify - # that the existing function really is overloaded: otherwise, replace - # the definition. The latter is actually important if we want to reload - # a library module such as genericpath with a custom one that uses - # overloading in the implementation. - if func.__name__ in locals and hasattr(locals[func.__name__], 'dispatch'): - orig_func = locals[func.__name__] - - def wrapper(*args, **kwargs): - ret, ok = orig_func.dispatch(*args, **kwargs) - if ok: - return ret - return func(*args, **kwargs) - wrapper.isoverload = True - wrapper.dispatch = make_dispatcher(func, orig_func.dispatch) - wrapper.next = orig_func - wrapper.__name__ = func.__name__ - if hasattr(func, '__isabstractmethod__'): - # Note that we can't reliably check that abstractmethod is - # used consistently across overload variants, so we let a - # static checker do it. - wrapper.__isabstractmethod__ = func.__isabstractmethod__ - return wrapper - else: - # Return the initial overload variant. - func.isoverload = True - func.dispatch = make_dispatcher(func) - func.next = None - return func - - -def is_erased_type(t): - return t is Any or isinstance(t, typevar) - - -def make_dispatcher(func, previous=None): - """Create argument dispatcher for an overloaded function. - - Also handle chaining of multiple overload variants. - """ - (args, varargs, varkw, defaults, - kwonlyargs, kwonlydefaults, annotations) = inspect.getfullargspec(func) - - argtypes = [] - for arg in args: - ann = annotations.get(arg) - if isinstance(ann, forwardref): - ann = ann.name - if is_erased_type(ann): - ann = None - elif isinstance(ann, str): - # The annotation is a string => evaluate it lazily when the - # overloaded function is first called. - frame = sys._getframe(2) - t = [None] - ann_str = ann - def check(x): - if not t[0]: - # Evaluate string in the context of the overload caller. - t[0] = eval(ann_str, frame.f_globals, frame.f_locals) - if is_erased_type(t[0]): - # Anything goes. - t[0] = object - if isinstance(t[0], type): - return isinstance(x, t[0]) - else: - return t[0](x) - ann = check - argtypes.append(ann) - - maxargs = len(argtypes) - minargs = maxargs - if defaults: - minargs = len(argtypes) - len(defaults) - - def dispatch(*args, **kwargs): - if previous: - ret, ok = previous(*args, **kwargs) - if ok: - return ret, ok - - nargs = len(args) - if nargs < minargs or nargs > maxargs: - # Invalid argument count. - return None, False - - for i in range(nargs): - argtype = argtypes[i] - if argtype: - if isinstance(argtype, type): - if not isinstance(args[i], argtype): - break - else: - if not argtype(args[i]): - break - else: - return func(*args, **kwargs), True - return None, False - return dispatch - - -class Undefined: - """Class that represents an undefined value with a specified type. - - At runtime the name Undefined is bound to an instance of this - class. The intent is that any operation on an Undefined object - raises an exception, including use in a boolean context. Some - operations cannot be disallowed: Undefined can be used as an - operand of 'is', and it can be assigned to variables and stored in - containers. - - 'Undefined' makes it possible to declare the static type of a - variable even if there is no useful default value to initialize it - with: - - from typing import Undefined - x = Undefined(int) - y = Undefined # type: int - - The latter form can be used if efficiency is of utmost importance, - since it saves a call operation and potentially additional - operations needed to evaluate a type expression. Undefined(x) - just evaluates to Undefined, ignoring the argument value. - """ - - def __repr__(self): - return '' - - def __setattr__(self, attr, value): - raise AttributeError("'Undefined' object has no attribute '%s'" % attr) - - def __eq__(self, other): - raise TypeError("'Undefined' object cannot be compared") - - def __call__(self, type): - return self - - def __bool__(self): - raise TypeError("'Undefined' object is not valid as a boolean") - - -Undefined = Undefined() - - -# Abstract classes - - -T = typevar('T') -KT = typevar('KT') -VT = typevar('VT') - - -class SupportsInt(Protocol): - @abstractmethod - def __int__(self) -> int: pass - - -class SupportsFloat(Protocol): - @abstractmethod - def __float__(self) -> float: pass - - -class SupportsAbs(Protocol[T]): - @abstractmethod - def __abs__(self) -> T: pass - - -class SupportsRound(Protocol[T]): - @abstractmethod - def __round__(self, ndigits: int = 0) -> T: pass - - -class Reversible(Protocol[T]): - @abstractmethod - def __reversed__(self) -> 'Iterator[T]': pass - - -class Sized(Protocol): - @abstractmethod - def __len__(self) -> int: pass - - -class Container(Protocol[T]): - @abstractmethod - def __contains__(self, x) -> bool: pass - - -class Iterable(Protocol[T]): - @abstractmethod - def __iter__(self) -> 'Iterator[T]': pass - - -class Iterator(Iterable[T], Protocol[T]): - @abstractmethod - def __next__(self) -> T: pass - - -class Sequence(Sized, Iterable[T], Container[T], AbstractGeneric[T]): - @abstractmethod - @overload - def __getitem__(self, i: int) -> T: pass - - @abstractmethod - @overload - def __getitem__(self, s: slice) -> 'Sequence[T]': pass - - @abstractmethod - def __reversed__(self, s: slice) -> Iterator[T]: pass - - @abstractmethod - def index(self, x) -> int: pass - - @abstractmethod - def count(self, x) -> int: pass - - -for t in list, tuple, str, bytes, range: - Sequence.register(t) - - -class AbstractSet(Sized, Iterable[T], AbstractGeneric[T]): - @abstractmethod - def __contains__(self, x: object) -> bool: pass - @abstractmethod - def __and__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass - @abstractmethod - def __or__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass - @abstractmethod - def __sub__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass - @abstractmethod - def __xor__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass - @abstractmethod - def isdisjoint(self, s: 'AbstractSet[T]') -> bool: pass - - -for t in set, frozenset, type({}.keys()), type({}.items()): - AbstractSet.register(t) - - -class Mapping(Sized, Iterable[KT], AbstractGeneric[KT, VT]): - @abstractmethod - def __getitem__(self, k: KT) -> VT: pass - @abstractmethod - def __setitem__(self, k: KT, v: VT) -> None: pass - @abstractmethod - def __delitem__(self, v: KT) -> None: pass - @abstractmethod - def __contains__(self, o: object) -> bool: pass - - @abstractmethod - def clear(self) -> None: pass - @abstractmethod - def copy(self) -> 'Mapping[KT, VT]': pass - @overload - @abstractmethod - def get(self, k: KT) -> VT: pass - @overload - @abstractmethod - def get(self, k: KT, default: VT) -> VT: pass - @overload - @abstractmethod - def pop(self, k: KT) -> VT: pass - @overload - @abstractmethod - def pop(self, k: KT, default: VT) -> VT: pass - @abstractmethod - def popitem(self) -> Tuple[KT, VT]: pass - @overload - @abstractmethod - def setdefault(self, k: KT) -> VT: pass - @overload - @abstractmethod - def setdefault(self, k: KT, default: VT) -> VT: pass - - @overload - @abstractmethod - def update(self, m: 'Mapping[KT, VT]') -> None: pass - @overload - @abstractmethod - def update(self, m: Iterable[Tuple[KT, VT]]) -> None: pass - - @abstractmethod - def keys(self) -> AbstractSet[KT]: pass - @abstractmethod - def values(self) -> AbstractSet[VT]: pass - @abstractmethod - def items(self) -> AbstractSet[Tuple[KT, VT]]: pass - - -# TODO Consider more types: os.environ, etc. However, these add dependencies. -Mapping.register(dict) - - -# Note that the BinaryIO and TextIO classes must be in sync with typing module -# stubs. - - -class IO(AbstractGeneric[AnyStr]): - @abstractproperty - def mode(self) -> str: pass - @abstractproperty - def name(self) -> str: pass - @abstractmethod - def close(self) -> None: pass - @abstractmethod - def closed(self) -> bool: pass - @abstractmethod - def fileno(self) -> int: pass - @abstractmethod - def flush(self) -> None: pass - @abstractmethod - def isatty(self) -> bool: pass - @abstractmethod - def read(self, n: int = -1) -> AnyStr: pass - @abstractmethod - def readable(self) -> bool: pass - @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: pass - @abstractmethod - def readlines(self, hint: int = -1) -> List[AnyStr]: pass - @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: pass - @abstractmethod - def seekable(self) -> bool: pass - @abstractmethod - def tell(self) -> int: pass - @abstractmethod - def truncate(self, size: int = None) -> int: pass - @abstractmethod - def writable(self) -> bool: pass - @abstractmethod - def write(self, s: AnyStr) -> int: pass - @abstractmethod - def writelines(self, lines: List[AnyStr]) -> None: pass - - @abstractmethod - def __enter__(self) -> 'IO[AnyStr]': pass - @abstractmethod - def __exit__(self, type, value, traceback) -> None: pass - - -class BinaryIO(IO[bytes]): - @overload - @abstractmethod - def write(self, s: bytes) -> int: pass - @overload - @abstractmethod - def write(self, s: bytearray) -> int: pass - - @abstractmethod - def __enter__(self) -> 'BinaryIO': pass - - -class TextIO(IO[str]): - @abstractproperty - def buffer(self) -> BinaryIO: pass - @abstractproperty - def encoding(self) -> str: pass - @abstractproperty - def errors(self) -> str: pass - @abstractproperty - def line_buffering(self) -> bool: pass - @abstractproperty - def newlines(self) -> Any: pass - @abstractmethod - def __enter__(self) -> 'TextIO': pass - - -# TODO Register IO/TextIO/BinaryIO as the base class of file-like types. - - -del t diff --git a/misc/actions_stubs.py b/misc/actions_stubs.py deleted file mode 100644 index 978af7187ffe..000000000000 --- a/misc/actions_stubs.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -import os -import shutil -from typing import Tuple, Any -try: - import click -except ImportError: - print("You need the module \'click\'") - exit(1) - -base_path = os.getcwd() - -# I don't know how to set callables with different args -def apply_all(func: Any, directory: str, extension: str, - to_extension: str='', exclude: Tuple[str]=('',), - recursive: bool=True, debug: bool=False) -> None: - excluded = [x+extension for x in exclude] if exclude else [] - for p, d, files in os.walk(os.path.join(base_path,directory)): - for f in files: - if "{}".format(f) in excluded: - continue - inner_path = os.path.join(p,f) - if not inner_path.endswith(extension): - continue - if to_extension: - new_path = "{}{}".format(inner_path[:-len(extension)],to_extension) - func(inner_path,new_path) - else: - func(inner_path) - if not recursive: - break - -def confirm(resp: bool=False, **kargs) -> bool: - kargs['rest'] = "to this {f2}/*{e2}".format(**kargs) if kargs.get('f2') else '' - prompt = "{act} all files {rec}matching this expression {f1}/*{e1} {rest}".format(**kargs) - prompt.format(**kargs) - prompt = "{} [{}]|{}: ".format(prompt, 'Y' if resp else 'N', 'n' if resp else 'y') - while True: - ans = input(prompt).lower() - if not ans: - return resp - if ans not in ['y','n']: - print( 'Please, enter (y) or (n).') - continue - if ans == 'y': - return True - else: - return False - -actions = ['cp', 'mv', 'rm'] -@click.command(context_settings=dict(help_option_names=['-h', '--help'])) -@click.option('--action', '-a', type=click.Choice(actions), required=True, help="What do I have to do :-)") -@click.option('--dir', '-d', 'directory', default='stubs', help="Directory to start search!") -@click.option('--ext', '-e', 'extension', default='.py', help="Extension \"from\" will be applied the action. Default .py") -@click.option('--to', '-t', 'to_extension', default='.pyi', help="Extension \"to\" will be applied the action if can. Default .pyi") -@click.option('--exclude', '-x', multiple=True, default=('__init__',), help="For every appear, will ignore this files. (can set multiples times)") -@click.option('--not-recursive', '-n', default=True, is_flag=True, help="Set if don't want to walk recursively.") -def main(action: str, directory: str, extension: str, to_extension: str, - exclude: Tuple[str], not_recursive: bool) -> None: - """ - This script helps to copy/move/remove files based on their extension. - - The three actions will ask you for confirmation. - - Examples (by default the script search in stubs directory): - - - Change extension of all stubs from .py to .pyi: - - python -a mv - - - Revert the previous action. - - python -a mv -e .pyi -t .py - - - If you want to ignore "awesome.py" files. - - python -a [cp|mv|rm] -x awesome - - - If you want to ignore "awesome.py" and "__init__.py" files. - - python -a [cp|mv|rm] -x awesome -x __init__ - - - If you want to remove all ".todo" files in "todo" directory, but not recursively: - - python -a rm -e .todo -d todo -r - - """ - if action not in actions: - print("Your action have to be one of this: {}".format(', '.join(actions))) - return - - rec = "[Recursively] " if not_recursive else '' - if not extension.startswith('.'): - extension = ".{}".format(extension) - if not to_extension.startswith('.'): - to_extension = ".{}".format(to_extension) - if directory.endswith('/'): - directory = directory[:-1] - if action == 'cp': - if confirm(act='Copy',rec=rec, f1=directory, e1=extension, f2=directory, e2=to_extension): - apply_all(shutil.copy, directory, extension, to_extension, exclude, not_recursive) - elif action == 'rm': - if confirm(act='Remove',rec=rec, f1=directory, e1=extension): - apply_all(os.remove, directory, extension, exclude=exclude, recursive=not_recursive) - elif action == 'mv': - if confirm(act='Move',rec=rec, f1=directory, e1=extension, f2=directory, e2=to_extension): - apply_all(shutil.move, directory, extension, to_extension, exclude, not_recursive) - - -if __name__ == '__main__': - main() diff --git a/misc/analyze_cache.py b/misc/analyze_cache.py new file mode 100644 index 000000000000..33205f5132fc --- /dev/null +++ b/misc/analyze_cache.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python + +from __future__ import annotations + +import json +import os +import os.path +from collections import Counter +from typing import Any, Dict, Final, Iterable +from typing_extensions import TypeAlias as _TypeAlias + +ROOT: Final = ".mypy_cache/3.5" + +JsonDict: _TypeAlias = Dict[str, Any] + + +class CacheData: + def __init__( + self, + filename: str, + data_json: JsonDict, + meta_json: JsonDict, + data_size: int, + meta_size: int, + ) -> None: + self.filename = filename + self.data = data_json + self.meta = meta_json + self.data_size = data_size + self.meta_size = meta_size + + @property + def total_size(self) -> int: + return self.data_size + self.meta_size + + +def extract_classes(chunks: Iterable[CacheData]) -> Iterable[JsonDict]: + def extract(chunks: Iterable[JsonDict]) -> Iterable[JsonDict]: + for chunk in chunks: + if isinstance(chunk, dict): + yield chunk + yield from extract(chunk.values()) + elif isinstance(chunk, list): + yield from extract(chunk) + + yield from extract([chunk.data for chunk in chunks]) + + +def load_json(data_path: str, meta_path: str) -> CacheData: + with open(data_path) as ds: + data_json = json.load(ds) + + with open(meta_path) as ms: + meta_json = json.load(ms) + + data_size = os.path.getsize(data_path) + meta_size = os.path.getsize(meta_path) + + return CacheData( + data_path.replace(".data.json", ".*.json"), data_json, meta_json, data_size, meta_size + ) + + +def get_files(root: str) -> Iterable[CacheData]: + for dirpath, dirnames, filenames in os.walk(root): + for filename in filenames: + if filename.endswith(".data.json"): + meta_filename = filename.replace(".data.json", ".meta.json") + yield load_json( + os.path.join(dirpath, filename), os.path.join(dirpath, meta_filename) + ) + + +def pluck(name: str, chunks: Iterable[JsonDict]) -> Iterable[JsonDict]: + return (chunk for chunk in chunks if chunk[".class"] == name) + + +def report_counter(counter: Counter[str], amount: int | None = None) -> None: + for name, count in counter.most_common(amount): + print(f" {count: <8} {name}") + print() + + +def report_most_common(chunks: list[JsonDict], amount: int | None = None) -> None: + report_counter(Counter(str(chunk) for chunk in chunks), amount) + + +def compress(chunk: JsonDict) -> JsonDict: + cache: dict[int, JsonDict] = {} + counter = 0 + + def helper(chunk: JsonDict) -> JsonDict: + nonlocal counter + if not isinstance(chunk, dict): + return chunk + + if len(chunk) <= 2: + return chunk + id = hash(str(chunk)) + + if id in cache: + return cache[id] + else: + cache[id] = {".id": counter} + chunk[".cache_id"] = counter + counter += 1 + + for name in sorted(chunk.keys()): + value = chunk[name] + if isinstance(value, list): + chunk[name] = [helper(child) for child in value] + elif isinstance(value, dict): + chunk[name] = helper(value) + + return chunk + + out = helper(chunk) + return out + + +def decompress(chunk: JsonDict) -> JsonDict: + cache: dict[int, JsonDict] = {} + + def helper(chunk: JsonDict) -> JsonDict: + if not isinstance(chunk, dict): + return chunk + if ".id" in chunk: + return cache[chunk[".id"]] + + counter = None + if ".cache_id" in chunk: + counter = chunk[".cache_id"] + del chunk[".cache_id"] + + for name in sorted(chunk.keys()): + value = chunk[name] + if isinstance(value, list): + chunk[name] = [helper(child) for child in value] + elif isinstance(value, dict): + chunk[name] = helper(value) + + if counter is not None: + cache[counter] = chunk + + return chunk + + return helper(chunk) + + +def main() -> None: + json_chunks = list(get_files(ROOT)) + class_chunks = list(extract_classes(json_chunks)) + + total_size = sum(chunk.total_size for chunk in json_chunks) + print(f"Total cache size: {total_size / (1024 * 1024):.3f} megabytes") + print() + + class_name_counter = Counter(chunk[".class"] for chunk in class_chunks) + print("Most commonly used classes:") + report_counter(class_name_counter) + + print("Most common literal chunks:") + report_most_common(class_chunks, 15) + + build = None + for chunk in json_chunks: + if "build.*.json" in chunk.filename: + build = chunk + break + assert build is not None + original = json.dumps(build.data, sort_keys=True) + print(f"Size of build.data.json, in kilobytes: {len(original) / 1024:.3f}") + + build.data = compress(build.data) + compressed = json.dumps(build.data, sort_keys=True) + print(f"Size of compressed build.data.json, in kilobytes: {len(compressed) / 1024:.3f}") + + build.data = decompress(build.data) + decompressed = json.dumps(build.data, sort_keys=True) + print(f"Size of decompressed build.data.json, in kilobytes: {len(decompressed) / 1024:.3f}") + + print("Lossless conversion back", original == decompressed) + + """var_chunks = list(pluck("Var", class_chunks)) + report_most_common(var_chunks, 20) + print() + + #for var in var_chunks: + # if var['fullname'] == 'self' and not (isinstance(var['type'], dict) and var['type']['.class'] == 'AnyType'): + # print(var) + #argument_chunks = list(pluck("Argument", class_chunks)) + + symbol_table_node_chunks = list(pluck("SymbolTableNode", class_chunks)) + report_most_common(symbol_table_node_chunks, 20) + + print() + print("Most common") + report_most_common(class_chunks, 20) + print()""" + + +if __name__ == "__main__": + main() diff --git a/misc/apply-cache-diff.py b/misc/apply-cache-diff.py new file mode 100644 index 000000000000..8ede9766bd06 --- /dev/null +++ b/misc/apply-cache-diff.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Script for applying a cache diff. + +With some infrastructure, this can allow for distributing small cache diffs to users in +many cases instead of full cache artifacts. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore +from mypy.util import json_dumps, json_loads + + +def make_cache(input_dir: str, sqlite: bool) -> MetadataStore: + if sqlite: + return SqliteMetadataStore(input_dir) + else: + return FilesystemMetadataStore(input_dir) + + +def apply_diff(cache_dir: str, diff_file: str, sqlite: bool = False) -> None: + cache = make_cache(cache_dir, sqlite) + with open(diff_file, "rb") as f: + diff = json_loads(f.read()) + + old_deps = json_loads(cache.read("@deps.meta.json")) + + for file, data in diff.items(): + if data is None: + cache.remove(file) + else: + cache.write(file, data) + if file.endswith(".meta.json") and "@deps" not in file: + meta = json_loads(data) + old_deps["snapshot"][meta["id"]] = meta["hash"] + + cache.write("@deps.meta.json", json_dumps(old_deps)) + + cache.commit() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--sqlite", action="store_true", default=False, help="Use a sqlite cache") + parser.add_argument("cache_dir", help="Directory for the cache") + parser.add_argument("diff", help="Cache diff file") + args = parser.parse_args() + + apply_diff(args.cache_dir, args.diff, args.sqlite) + + +if __name__ == "__main__": + main() diff --git a/misc/build-debug-python.sh b/misc/build-debug-python.sh new file mode 100755 index 000000000000..8dd1bff4c9ed --- /dev/null +++ b/misc/build-debug-python.sh @@ -0,0 +1,34 @@ +#!/bin/bash -eux + +# Build a debug build of python, install it, and create a venv for it +# This is mainly intended for use in our github actions builds but it can work +# locally. (Though it unfortunately uses brew on OS X to deal with openssl +# nonsense.) +# Usage: build-debug-python.sh +# +# Running it locally might look something like: mkdir -p ~/tmp/cpython-debug && cd ~/tmp/cpython-debug && ~/src/mypy/misc/build-debug-python.sh 3.6.6 ~/tmp/cpython-debug ~/src/mypy/env-debug + +VERSION=$1 +PREFIX=$2 +VENV=$3 +if [[ -f $PREFIX/bin/python3 ]]; then + exit +fi + +CPPFLAGS="" +LDFLAGS="" +if [[ $(uname) == Darwin ]]; then + brew install openssl xz + CPPFLAGS="-I$(brew --prefix openssl)/include" + LDFLAGS="-L$(brew --prefix openssl)/lib" +fi + +curl -O https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tgz +tar zxf Python-$VERSION.tgz +cd Python-$VERSION +CPPFLAGS="$CPPFLAGS" LDFLAGS="$LDFLAGS" ./configure CFLAGS="-DPy_DEBUG -DPy_TRACE_REFS -DPYMALLOC_DEBUG" --with-pydebug --prefix=$PREFIX --with-trace-refs +make -j4 +make install +$PREFIX/bin/python3 -m pip install virtualenv +$PREFIX/bin/python3 -m virtualenv $VENV +ln -s python3-config $PREFIX/bin/python-config diff --git a/misc/build_wheel.py b/misc/build_wheel.py new file mode 100644 index 000000000000..4389c80a14db --- /dev/null +++ b/misc/build_wheel.py @@ -0,0 +1,12 @@ +""" +The main GitHub workflow where wheels are built: +https://github.com/mypyc/mypy_mypyc-wheels/blob/master/.github/workflows/build.yml + +The script that builds wheels: +https://github.com/mypyc/mypy_mypyc-wheels/blob/master/build_wheel.py + +That script is a light wrapper around cibuildwheel. Now that cibuildwheel has native configuration +and better support for local builds, we could probably replace the script. +""" + +raise ImportError("This script has been moved back to https://github.com/mypyc/mypy_mypyc-wheels") diff --git a/misc/cherry-pick-typeshed.py b/misc/cherry-pick-typeshed.py new file mode 100644 index 000000000000..7e3b8b56e65f --- /dev/null +++ b/misc/cherry-pick-typeshed.py @@ -0,0 +1,70 @@ +"""Cherry-pick a commit from typeshed. + +Usage: + + python3 misc/cherry-pick-typeshed.py --typeshed-dir dir hash +""" + +from __future__ import annotations + +import argparse +import os.path +import re +import subprocess +import sys +import tempfile + + +def parse_commit_title(diff: str) -> str: + m = re.search("\n ([^ ].*)", diff) + assert m is not None, "Could not parse diff" + return m.group(1) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--typeshed-dir", help="location of typeshed", metavar="dir", required=True + ) + parser.add_argument("commit", help="typeshed commit hash to cherry-pick") + args = parser.parse_args() + typeshed_dir = args.typeshed_dir + commit = args.commit + + if not os.path.isdir(typeshed_dir): + sys.exit(f"error: {typeshed_dir} does not exist") + if not re.match("[0-9a-fA-F]+$", commit): + sys.exit(f"error: Invalid commit {commit!r}") + + if not os.path.exists("mypy") or not os.path.exists("mypyc"): + sys.exit("error: This script must be run at the mypy repository root directory") + + with tempfile.TemporaryDirectory() as d: + diff_file = os.path.join(d, "diff") + out = subprocess.run( + ["git", "show", commit], capture_output=True, text=True, check=True, cwd=typeshed_dir + ) + with open(diff_file, "w") as f: + f.write(out.stdout) + subprocess.run( + [ + "git", + "apply", + "--index", + "--directory=mypy/typeshed", + "--exclude=**/tests/**", + "--exclude=**/test_cases/**", + diff_file, + ], + check=True, + ) + + title = parse_commit_title(out.stdout) + subprocess.run(["git", "commit", "-m", f"Typeshed cherry-pick: {title}"], check=True) + + print() + print(f"Cherry-picked commit {commit} from {typeshed_dir}") + + +if __name__ == "__main__": + main() diff --git a/misc/clean-mypyc.sh b/misc/clean-mypyc.sh new file mode 100755 index 000000000000..8b29a7d09d68 --- /dev/null +++ b/misc/clean-mypyc.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "Cleaning C/C++ build artifacts..." +(cd mypyc/lib-rt; make clean) +(cd mypyc/external/googletest/make; make clean) diff --git a/misc/convert-cache.py b/misc/convert-cache.py new file mode 100755 index 000000000000..2a8a9579c11b --- /dev/null +++ b/misc/convert-cache.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Script for converting between cache formats. + +We support a filesystem tree based cache and a sqlite based cache. +See mypy/metastore.py for details. +""" + +from __future__ import annotations + +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import argparse + +from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--to-sqlite", + action="store_true", + default=False, + help="Convert to a sqlite cache (default: convert from)", + ) + parser.add_argument( + "--output_dir", + action="store", + default=None, + help="Output cache location (default: same as input)", + ) + parser.add_argument("input_dir", help="Input directory for the cache") + args = parser.parse_args() + + input_dir = args.input_dir + output_dir = args.output_dir or input_dir + assert os.path.isdir(output_dir), f"{output_dir} is not a directory" + if args.to_sqlite: + input: MetadataStore = FilesystemMetadataStore(input_dir) + output: MetadataStore = SqliteMetadataStore(output_dir) + else: + fnam = os.path.join(input_dir, "cache.db") + msg = f"{fnam} does not exist" + if not re.match(r"[0-9]+\.[0-9]+$", os.path.basename(input_dir)): + msg += f" (are you missing Python version at the end, e.g. {input_dir}/3.11)" + assert os.path.isfile(fnam), msg + input, output = SqliteMetadataStore(input_dir), FilesystemMetadataStore(output_dir) + + for s in input.list_all(): + if s.endswith(".json"): + assert output.write( + s, input.read(s), input.getmtime(s) + ), f"Failed to write cache file {s}!" + output.commit() + + +if __name__ == "__main__": + main() diff --git a/misc/diff-cache.py b/misc/diff-cache.py new file mode 100644 index 000000000000..8441caf81304 --- /dev/null +++ b/misc/diff-cache.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Produce a diff between mypy caches. + +With some infrastructure, this can allow for distributing small cache diffs to users in +many cases instead of full cache artifacts. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from collections import defaultdict +from typing import Any + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore +from mypy.util import json_dumps, json_loads + + +def make_cache(input_dir: str, sqlite: bool) -> MetadataStore: + if sqlite: + return SqliteMetadataStore(input_dir) + else: + return FilesystemMetadataStore(input_dir) + + +def merge_deps(all: dict[str, set[str]], new: dict[str, set[str]]) -> None: + for k, v in new.items(): + all.setdefault(k, set()).update(v) + + +def load(cache: MetadataStore, s: str) -> Any: + data = cache.read(s) + obj = json_loads(data) + if s.endswith(".meta.json"): + # For meta files, zero out the mtimes and sort the + # dependencies to avoid spurious conflicts + obj["mtime"] = 0 + obj["data_mtime"] = 0 + if "dependencies" in obj: + all_deps = obj["dependencies"] + obj["suppressed"] + num_deps = len(obj["dependencies"]) + thing = list(zip(all_deps, obj["dep_prios"], obj["dep_lines"])) + + def unzip(x: Any) -> Any: + return zip(*x) if x else ((), (), ()) + + obj["dependencies"], prios1, lines1 = unzip(sorted(thing[:num_deps])) + obj["suppressed"], prios2, lines2 = unzip(sorted(thing[num_deps:])) + obj["dep_prios"] = prios1 + prios2 + obj["dep_lines"] = lines1 + lines2 + if s.endswith(".deps.json"): + # For deps files, sort the deps to avoid spurious mismatches + for v in obj.values(): + v.sort() + return obj + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--verbose", action="store_true", default=False, help="Increase verbosity") + parser.add_argument("--sqlite", action="store_true", default=False, help="Use a sqlite cache") + parser.add_argument("input_dir1", help="Input directory for the cache") + parser.add_argument("input_dir2", help="Input directory for the cache") + parser.add_argument("output", help="Output file") + args = parser.parse_args() + + cache1 = make_cache(args.input_dir1, args.sqlite) + cache2 = make_cache(args.input_dir2, args.sqlite) + + type_misses: dict[str, int] = defaultdict(int) + type_hits: dict[str, int] = defaultdict(int) + + updates: dict[str, bytes | None] = {} + + deps1: dict[str, set[str]] = {} + deps2: dict[str, set[str]] = {} + + misses = hits = 0 + cache1_all = list(cache1.list_all()) + for s in cache1_all: + obj1 = load(cache1, s) + try: + obj2 = load(cache2, s) + except FileNotFoundError: + obj2 = None + + typ = s.split(".")[-2] + if obj1 != obj2: + misses += 1 + type_misses[typ] += 1 + + # Collect the dependencies instead of including them directly in the diff + # so we can produce a much smaller direct diff of them. + if ".deps." not in s: + if obj2 is not None: + updates[s] = json_dumps(obj2) + else: + updates[s] = None + elif obj2: + merge_deps(deps1, obj1) + merge_deps(deps2, obj2) + else: + hits += 1 + type_hits[typ] += 1 + + cache1_all_set = set(cache1_all) + for s in cache2.list_all(): + if s not in cache1_all_set: + updates[s] = cache2.read(s) + + # Compute what deps have been added and merge them all into the + # @root deps file. + new_deps = {k: deps1.get(k, set()) - deps2.get(k, set()) for k in deps2} + new_deps = {k: v for k, v in new_deps.items() if v} + try: + root_deps = load(cache1, "@root.deps.json") + except FileNotFoundError: + root_deps = {} + merge_deps(new_deps, root_deps) + + new_deps_json = {k: list(v) for k, v in new_deps.items() if v} + updates["@root.deps.json"] = json_dumps(new_deps_json) + + # Drop updates to deps.meta.json for size reasons. The diff + # applier will manually fix it up. + updates.pop("./@deps.meta.json", None) + updates.pop("@deps.meta.json", None) + + ### + + print("Generated incremental cache:", hits, "hits,", misses, "misses") + if args.verbose: + print("hits", type_hits) + print("misses", type_misses) + + with open(args.output, "wb") as f: + f.write(json_dumps(updates)) + + +if __name__ == "__main__": + main() diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile new file mode 100644 index 000000000000..3327f9e38815 --- /dev/null +++ b/misc/docker/Dockerfile @@ -0,0 +1,12 @@ +FROM ubuntu:latest + +WORKDIR /mypy + +RUN apt-get update +RUN apt-get install -y python3 python3-pip clang + +COPY mypy-requirements.txt . +COPY test-requirements.txt . +COPY build-requirements.txt . + +RUN pip3 install -r test-requirements.txt diff --git a/misc/docker/README.md b/misc/docker/README.md new file mode 100644 index 000000000000..839f9761cb03 --- /dev/null +++ b/misc/docker/README.md @@ -0,0 +1,101 @@ +Running mypy and mypyc tests in a Docker container +================================================== + +This directory contains scripts for running mypy and mypyc tests in a +Linux Docker container. This allows running Linux tests on a different +operating system that supports Docker, or running tests in an +isolated, predictable environment on a Linux host operating system. + +Why use Docker? +--------------- + +Mypyc tests can be significantly faster in a Docker container than +running natively on macOS. + +Also, if it's inconvient to install the necessary dependencies on the +host operating system, or there are issues getting some tests to pass +on the host operating system, using a container can be an easy +workaround. + +Prerequisites +------------- + +First install Docker. On macOS, both Docker Desktop (proprietary, but +with a free of charge subscription for some use cases) and Colima (MIT +license) should work as runtimes. + +You may have to explicitly start the runtime first. Colima example +(replace '8' with the number of CPU cores you have): + +``` +$ colima start -c 8 + +``` + +How to run tests +---------------- + +You need to build the container with all necessary dependencies before +you can run tests: + +``` +$ python3 misc/docker/build.py +``` + +This creates a `mypy-test` Docker container that you can use to run +tests. + +You may need to run the script as root: + +``` +$ sudo python3 misc/docker/build.py +``` + +If you have a stale container which isn't up-to-date, use `--no-cache` +`--pull` to force rebuilding everything: + +``` +$ python3 misc/docker/build.py --no-cache --pull +``` + +Now you can run tests by using the `misc/docker/run.sh` script. Give +it the pytest command line you want to run as arguments. For example, +you can run mypyc tests like this: + +``` +$ misc/docker/run.sh pytest mypyc +``` + +You can also use `-k `, `-n0`, `-q`, etc. + +Again, you may need to run `run.sh` as root: + +``` +$ sudo misc/docker/run.sh pytest mypyc +``` + +You can also use `runtests.py` in the container. Example: + +``` +$ misc/docker/run.sh ./runtests.py self lint +``` + +Notes +----- + +File system changes within the container are not visible to the host +system. You can't use the container to format code using Black, for +example. + +On a mac, you may want to give additional CPU to the VM used to run +the container. The default allocation may be way too low (e.g. 2 CPU +cores). For example, use the `-c` option when starting the VM if you +use Colima: + +``` +$ colima start -c 8 +``` + +Giving access to all available CPUs to the Linux VM tends to provide +the best performance. This is not needed on a Linux host, since the +container is not run in a VM. diff --git a/misc/docker/build.py b/misc/docker/build.py new file mode 100644 index 000000000000..2103be3f110f --- /dev/null +++ b/misc/docker/build.py @@ -0,0 +1,46 @@ +"""Build a "mypy-test" Linux Docker container for running mypy/mypyc tests. + +This allows running Linux tests under a non-Linux operating system. Mypyc +tests can also run much faster under Linux that the host OS. + +NOTE: You may need to run this as root (using sudo). + +Run with "--no-cache" to force reinstallation of mypy dependencies. +Run with "--pull" to force update of the Linux (Ubuntu) base image. + +After you've built the container, use "run.sh" to run tests. Example: + + misc/docker/run.sh pytest mypyc/ +""" + +import argparse +import os +import subprocess +import sys + + +def main() -> None: + parser = argparse.ArgumentParser( + description="""Build a 'mypy-test' Docker container for running mypy/mypyc tests. You may + need to run this as root (using sudo).""" + ) + parser.add_argument("--no-cache", action="store_true", help="Force rebuilding") + parser.add_argument("--pull", action="store_true", help="Force pulling fresh Linux base image") + args = parser.parse_args() + + dockerdir = os.path.dirname(os.path.abspath(__file__)) + dockerfile = os.path.join(dockerdir, "Dockerfile") + rootdir = os.path.join(dockerdir, "..", "..") + + cmdline = ["docker", "build", "-t", "mypy-test", "-f", dockerfile] + if args.no_cache: + cmdline.append("--no-cache") + if args.pull: + cmdline.append("--pull") + cmdline.append(rootdir) + result = subprocess.run(cmdline) + sys.exit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/misc/docker/run-wrapper.sh b/misc/docker/run-wrapper.sh new file mode 100755 index 000000000000..77e77d99af34 --- /dev/null +++ b/misc/docker/run-wrapper.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Internal wrapper script used to run commands in a container + +# Copy all the files we need from the mypy repo directory shared with +# the host to a local directory. Accessing files using a shared +# directory on a mac can be *very* slow. +echo "copying files to the container..." +cp -R /repo/{mypy,mypyc,test-data,misc} . +cp /repo/{pytest.ini,conftest.py,runtests.py,pyproject.toml,setup.cfg} . +cp /repo/{mypy_self_check.ini,mypy_bootstrap.ini} . + +# Run the wrapped command +"$@" diff --git a/misc/docker/run.sh b/misc/docker/run.sh new file mode 100755 index 000000000000..c8fc0e510e8e --- /dev/null +++ b/misc/docker/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Run mypy or mypyc tests in a Docker container that was built using misc/docker/build.py. +# +# Usage: misc/docker/run.sh ... +# +# For example, run mypyc tests like this: +# +# misc/docker/run.sh pytest mypyc +# +# NOTE: You may need to run this as root (using sudo). + +SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +MYPY_DIR="$SCRIPT_DIR/../.." + +docker run -ti --rm -v "$MYPY_DIR:/repo" mypy-test /repo/misc/docker/run-wrapper.sh "$@" diff --git a/misc/dump-ast.py b/misc/dump-ast.py new file mode 100755 index 000000000000..7fdf905bae0b --- /dev/null +++ b/misc/dump-ast.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +""" +Parse source files and print the abstract syntax trees. +""" + +from __future__ import annotations + +import argparse +import sys + +from mypy import defaults +from mypy.errors import CompileError, Errors +from mypy.options import Options +from mypy.parse import parse + + +def dump(fname: str, python_version: tuple[int, int], quiet: bool = False) -> None: + options = Options() + options.python_version = python_version + with open(fname, "rb") as f: + s = f.read() + tree = parse(s, fname, None, errors=Errors(options), options=options) + if not quiet: + print(tree) + + +def main() -> None: + # Parse a file and dump the AST (or display errors). + parser = argparse.ArgumentParser( + description="Parse source files and print the abstract syntax tree (AST)." + ) + parser.add_argument("--quiet", action="store_true", help="do not print AST") + parser.add_argument("FILE", nargs="*", help="files to parse") + args = parser.parse_args() + + status = 0 + for fname in args.FILE: + try: + dump(fname, defaults.PYTHON3_VERSION, args.quiet) + except CompileError as e: + for msg in e.messages: + sys.stderr.write("%s\n" % msg) + status = 1 + sys.exit(status) + + +if __name__ == "__main__": + main() diff --git a/misc/find_type.py b/misc/find_type.py new file mode 100755 index 000000000000..0031c72aea9f --- /dev/null +++ b/misc/find_type.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Usage: find_type.py FILENAME START_LINE START_COL END_LINE END_COL MYPY_AND_ARGS +# Prints out the type of the expression in the given location if the mypy run +# succeeds cleanly. Otherwise, prints out the errors encountered. +# Note: this only works on expressions, and not assignment targets. +# Note: MYPY_AND_ARGS is should be the remainder of argv, not a single +# spaces-included argument. +# NOTE: Line numbers are 1-based; column numbers are 0-based. +# +# +# Example vim usage: +# function RevealType() +# " Set this to the command you use to run mypy on your project. Include the mypy invocation. +# let mypycmd = 'python3 -m mypy mypy --incremental' +# let [startline, startcol] = getpos("'<")[1:2] +# let [endline, endcol] = getpos("'>")[1:2] +# " Convert to 0-based column offsets +# let startcol = startcol - 1 +# " Change this line to point to the find_type.py script. +# execute '!python3 /path/to/mypy/misc/find_type.py % ' . startline . ' ' . startcol . ' ' . endline . ' ' . endcol . ' ' . mypycmd +# endfunction +# vnoremap t :call RevealType() +# +# For an Emacs example, see misc/macs.el. + +from __future__ import annotations + +import os.path +import re +import subprocess +import sys +import tempfile + +REVEAL_TYPE_START = "reveal_type(" +REVEAL_TYPE_END = ")" + + +def update_line(line: str, s: str, pos: int) -> str: + return line[:pos] + s + line[pos:] + + +def run_mypy(mypy_and_args: list[str], filename: str, tmp_name: str) -> str: + proc = subprocess.run( + mypy_and_args + ["--shadow-file", filename, tmp_name], stdout=subprocess.PIPE + ) + assert isinstance( + proc.stdout, bytes + ) # Guaranteed to be true because we called run with universal_newlines=False + return proc.stdout.decode(encoding="utf-8") + + +def get_revealed_type(line: str, relevant_file: str, relevant_line: int) -> str | None: + m = re.match(r'(.+?):(\d+): note: Revealed type is "(.*)"$', line) + if m and int(m.group(2)) == relevant_line and os.path.samefile(relevant_file, m.group(1)): + return m.group(3) + else: + return None + + +def process_output(output: str, filename: str, start_line: int) -> tuple[str | None, bool]: + error_found = False + for line in output.splitlines(): + t = get_revealed_type(line, filename, start_line) + if t: + return t, error_found + elif "error:" in line: + error_found = True + return None, True # finding no reveal_type is an error + + +def main() -> None: + filename, start_line_str, start_col_str, end_line_str, end_col_str, *mypy_and_args = sys.argv[ + 1: + ] + start_line = int(start_line_str) + start_col = int(start_col_str) + end_line = int(end_line_str) + end_col = int(end_col_str) + with open(filename) as f: + lines = f.readlines() + lines[end_line - 1] = update_line( + lines[end_line - 1], REVEAL_TYPE_END, end_col + ) # insert after end_col + lines[start_line - 1] = update_line(lines[start_line - 1], REVEAL_TYPE_START, start_col) + with tempfile.NamedTemporaryFile(mode="w", prefix="mypy") as tmp_f: + tmp_f.writelines(lines) + tmp_f.flush() + + output = run_mypy(mypy_and_args, filename, tmp_f.name) + revealed_type, error = process_output(output, filename, start_line) + if revealed_type: + print(revealed_type) + if error: + print(output) + exit(int(error)) + + +if __name__ == "__main__": + main() diff --git a/misc/gen_blog_post_html.py b/misc/gen_blog_post_html.py new file mode 100644 index 000000000000..7170696d5d09 --- /dev/null +++ b/misc/gen_blog_post_html.py @@ -0,0 +1,171 @@ +"""Converter from CHANGELOG.md (Markdown) to HTML suitable for a mypy blog post. + +How to use: + +1. Write release notes in CHANGELOG.md. +2. Make sure the heading for the next release is of form `## Mypy X.Y`. +2. Run `misc/gen_blog_post_html.py X.Y > target.html`. +4. Manually inspect and tweak the result. + +Notes: + +* There are some fragile assumptions. Double check the output. +""" + +import argparse +import html +import os +import re +import sys + + +def format_lists(h: str) -> str: + a = h.splitlines() + r = [] + i = 0 + bullets = ("- ", "* ", " * ") + while i < len(a): + if a[i].startswith(bullets): + r.append("

    ") + while i < len(a) and a[i].startswith(bullets): + r.append("
  • %s" % a[i][2:].lstrip()) + i += 1 + r.append("
") + else: + r.append(a[i]) + i += 1 + return "\n".join(r) + + +def format_code(h: str) -> str: + a = h.splitlines() + r = [] + i = 0 + while i < len(a): + if a[i].startswith(" ") or a[i].startswith("```"): + indent = a[i].startswith(" ") + if not indent: + i += 1 + r.append("
")
+            while i < len(a) and (
+                (indent and a[i].startswith("    ")) or (not indent and not a[i].startswith("```"))
+            ):
+                # Undo > and <
+                line = a[i].replace(">", ">").replace("<", "<")
+                if not indent:
+                    line = "    " + line
+                r.append(html.escape(line))
+                i += 1
+            r.append("
") + if not indent and a[i].startswith("```"): + i += 1 + else: + r.append(a[i]) + i += 1 + return "\n".join(r) + + +def convert(src: str) -> str: + h = src + + # Replace < and >. + h = re.sub(r"<", "<", h) + h = re.sub(r">", ">", h) + + # Title + h = re.sub(r"^## (Mypy [0-9.]+)", r"

\1 Released

", h, flags=re.MULTILINE) + + # Subheadings + h = re.sub(r"\n#### ([A-Z`].*)\n", r"\n

\1

\n", h) + + # Sub-subheadings + h = re.sub(r"\n\*\*([A-Z_`].*)\*\*\n", r"\n

\1

\n", h) + h = re.sub(r"\n`\*\*([A-Z_`].*)\*\*\n", r"\n

`\1

\n", h) + + # Translate `**` + h = re.sub(r"`\*\*`", "**", h) + + # Paragraphs + h = re.sub(r"\n([A-Z])", r"\n

\1", h) + + # Bullet lists + h = format_lists(h) + + # Code blocks + h = format_code(h) + + # Code fragments + h = re.sub(r"`([^`]+)`", r"\1", h) + + # Remove **** noise + h = re.sub(r"\*\*\*\*", "", h) + + # Bold text + h = re.sub(r"\*\*([A-Za-z].*?)\*\*", r" \1", h) + + # Emphasized text + h = re.sub(r" \*([A-Za-z].*?)\*", r" \1", h) + + # Remove redundant PR links to avoid double links (they will be generated below) + h = re.sub(r"\[(#[0-9]+)\]\(https://github.com/python/mypy/pull/[0-9]+/?\)", r"\1", h) + + # Issue and PR links + h = re.sub(r"\((#[0-9]+)\) +\(([^)]+)\)", r"(\2, \1)", h) + h = re.sub( + r"fixes #([0-9]+)", + r'fixes issue \1', + h, + ) + h = re.sub(r"#([0-9]+)", r'PR \1', h) + h = re.sub(r"\) \(PR", ", PR", h) + + # Markdown links + h = re.sub(r"\[([^]]*)\]\(([^)]*)\)", r'\1', h) + + # Add random links in case they are missing + h = re.sub( + r"contributors to typeshed:", + 'contributors to typeshed:', + h, + ) + + # Add missing top-level HTML tags + h = '\n\n\n' + h + "\n" + + return h + + +def extract_version(src: str, version: str) -> str: + a = src.splitlines() + i = 0 + heading = f"## Mypy {version}" + while i < len(a): + if a[i].strip() == heading: + break + i += 1 + else: + raise RuntimeError(f"Can't find heading {heading!r}") + j = i + 1 + while not a[j].startswith("## "): + j += 1 + return "\n".join(a[i:j]) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate HTML release blog post based on CHANGELOG.md and write to stdout." + ) + parser.add_argument("version", help="mypy version, in form X.Y or X.Y.Z") + args = parser.parse_args() + version: str = args.version + if not re.match(r"[0-9]+(\.[0-9]+)+$", version): + sys.exit(f"error: Version must be of form X.Y or X.Y.Z, not {version!r}") + changelog_path = os.path.join(os.path.dirname(__file__), os.path.pardir, "CHANGELOG.md") + src = open(changelog_path).read() + src = extract_version(src, version) + dst = convert(src) + sys.stdout.write(dst) + + +if __name__ == "__main__": + main() diff --git a/misc/generate_changelog.py b/misc/generate_changelog.py new file mode 100644 index 000000000000..ebab6c569152 --- /dev/null +++ b/misc/generate_changelog.py @@ -0,0 +1,200 @@ +"""Generate the changelog for a mypy release.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from dataclasses import dataclass + + +def find_all_release_branches() -> list[tuple[int, int]]: + result = subprocess.run(["git", "branch", "-r"], text=True, capture_output=True, check=True) + versions = [] + for line in result.stdout.splitlines(): + line = line.strip() + if m := re.match(r"origin/release-([0-9]+)\.([0-9]+)$", line): + major = int(m.group(1)) + minor = int(m.group(2)) + versions.append((major, minor)) + return versions + + +def git_merge_base(rev1: str, rev2: str) -> str: + result = subprocess.run( + ["git", "merge-base", rev1, rev2], text=True, capture_output=True, check=True + ) + return result.stdout.strip() + + +@dataclass +class CommitInfo: + commit: str + author: str + title: str + pr_number: int | None + + +def normalize_author(author: str) -> str: + # Some ad-hoc rules to get more consistent author names. + if author == "AlexWaygood": + return "Alex Waygood" + elif author == "jhance": + return "Jared Hance" + return author + + +def git_commit_log(rev1: str, rev2: str) -> list[CommitInfo]: + result = subprocess.run( + ["git", "log", "--pretty=%H\t%an\t%s", f"{rev1}..{rev2}"], + text=True, + capture_output=True, + check=True, + ) + commits = [] + for line in result.stdout.splitlines(): + commit, author, title = line.strip().split("\t", 2) + pr_number = None + if m := re.match(r".*\(#([0-9]+)\) *$", title): + pr_number = int(m.group(1)) + title = re.sub(r" *\(#[0-9]+\) *$", "", title) + + author = normalize_author(author) + entry = CommitInfo(commit, author, title, pr_number) + commits.append(entry) + return commits + + +def filter_omitted_commits(commits: list[CommitInfo]) -> list[CommitInfo]: + result = [] + for c in commits: + title = c.title + keep = True + if title.startswith("Sync typeshed"): + # Typeshed syncs aren't mentioned in release notes + keep = False + if title.startswith( + ( + "Revert sum literal integer change", + "Remove use of LiteralString in builtins", + "Revert typeshed ctypes change", + ) + ): + # These are generated by a typeshed sync. + keep = False + if re.search(r"(bump|update).*version.*\+dev", title.lower()): + # Version number updates aren't mentioned + keep = False + if "pre-commit autoupdate" in title: + keep = False + if title.startswith(("Update commit hashes", "Update hashes")): + # Internal tool change + keep = False + if keep: + result.append(c) + return result + + +def normalize_title(title: str) -> str: + # We sometimes add a title prefix when cherry-picking commits to a + # release branch. Attempt to remove these prefixes so that we can + # match them to the corresponding master branch. + if m := re.match(r"\[release [0-9.]+\] *", title, flags=re.I): + title = title.replace(m.group(0), "") + return title + + +def filter_out_commits_from_old_release_branch( + new_commits: list[CommitInfo], old_commits: list[CommitInfo] +) -> list[CommitInfo]: + old_titles = {normalize_title(commit.title) for commit in old_commits} + result = [] + for commit in new_commits: + drop = False + if normalize_title(commit.title) in old_titles: + drop = True + if normalize_title(f"{commit.title} (#{commit.pr_number})") in old_titles: + drop = True + if not drop: + result.append(commit) + else: + print(f'NOTE: Drop "{commit.title}", since it was in previous release branch') + return result + + +def find_changes_between_releases(old_branch: str, new_branch: str) -> list[CommitInfo]: + merge_base = git_merge_base(old_branch, new_branch) + print(f"Merge base: {merge_base}") + new_commits = git_commit_log(merge_base, new_branch) + old_commits = git_commit_log(merge_base, old_branch) + + # Filter out some commits that won't be mentioned in release notes. + new_commits = filter_omitted_commits(new_commits) + + # Filter out commits cherry-picked to old branch. + new_commits = filter_out_commits_from_old_release_branch(new_commits, old_commits) + + return new_commits + + +def format_changelog_entry(c: CommitInfo) -> str: + """ + s = f" * {c.commit[:9]} - {c.title}" + if c.pr_number: + s += f" (#{c.pr_number})" + s += f" ({c.author})" + """ + s = f" * {c.title} ({c.author}" + if c.pr_number: + s += f", PR [{c.pr_number}](https://github.com/python/mypy/pull/{c.pr_number})" + s += ")" + + return s + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("version", help="target mypy version (form X.Y)") + parser.add_argument("--local", action="store_true") + args = parser.parse_args() + version: str = args.version + local: bool = args.local + + if not re.match(r"[0-9]+\.[0-9]+$", version): + sys.exit(f"error: Release must be of form X.Y (not {version!r})") + major, minor = (int(component) for component in version.split(".")) + + if not local: + print("Running 'git fetch' to fetch all release branches...") + subprocess.run(["git", "fetch"], check=True) + + if minor > 0: + prev_major = major + prev_minor = minor - 1 + else: + # For a x.0 release, the previous release is the most recent (x-1).y release. + all_releases = sorted(find_all_release_branches()) + if (major, minor) not in all_releases: + sys.exit(f"error: Can't find release branch for {major}.{minor} at origin") + for i in reversed(range(len(all_releases))): + if all_releases[i][0] == major - 1: + prev_major, prev_minor = all_releases[i] + break + else: + sys.exit("error: Could not determine previous release") + print(f"Generating changelog for {major}.{minor}") + print(f"Previous release was {prev_major}.{prev_minor}") + + new_branch = f"origin/release-{major}.{minor}" + old_branch = f"origin/release-{prev_major}.{prev_minor}" + + changes = find_changes_between_releases(old_branch, new_branch) + + print() + for c in changes: + print(format_changelog_entry(c)) + + +if __name__ == "__main__": + main() diff --git a/misc/incremental_checker.py b/misc/incremental_checker.py new file mode 100755 index 000000000000..4e42aef333bb --- /dev/null +++ b/misc/incremental_checker.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +""" +This file compares the output and runtime of running normal vs incremental mode +on the history of any arbitrary git repo as a way of performing a sanity check +to make sure incremental mode is working correctly and efficiently. + +It does so by first running mypy without incremental mode on the specified range +of commits to find the expected result, then rewinds back to the first commit and +re-runs mypy on the commits with incremental mode enabled to make sure it returns +the same results. + +This script will download and test the official mypy repo by default. Running: + + python3 misc/incremental_checker.py last 30 + +is equivalent to running + + python3 misc/incremental_checker.py last 30 \\ + --repo_url https://github.com/python/mypy.git \\ + --file-path mypy + +You can chose to run this script against a specific commit id or against the +last n commits. + +To run this script against the last 30 commits: + + python3 misc/incremental_checker.py last 30 + +To run this script starting from the commit id 2a432b: + + python3 misc/incremental_checker.py commit 2a432b +""" + +from __future__ import annotations + +import base64 +import json +import os +import random +import re +import shutil +import subprocess +import sys +import textwrap +import time +from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter +from typing import Any, Dict, Final +from typing_extensions import TypeAlias as _TypeAlias + +CACHE_PATH: Final = ".incremental_checker_cache.json" +MYPY_REPO_URL: Final = "https://github.com/python/mypy.git" +MYPY_TARGET_FILE: Final = "mypy" +DAEMON_CMD: Final = ["python3", "-m", "mypy.dmypy"] + +JsonDict: _TypeAlias = Dict[str, Any] + + +def print_offset(text: str, indent_length: int = 4) -> None: + print() + print(textwrap.indent(text, " " * indent_length)) + print() + + +def delete_folder(folder_path: str) -> None: + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + + +def execute(command: list[str], fail_on_error: bool = True) -> tuple[str, str, int]: + proc = subprocess.Popen( + " ".join(command), stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True + ) + stdout_bytes, stderr_bytes = proc.communicate() + stdout, stderr = stdout_bytes.decode("utf-8"), stderr_bytes.decode("utf-8") + if fail_on_error and proc.returncode != 0: + print("EXECUTED COMMAND:", repr(command)) + print("RETURN CODE:", proc.returncode) + print() + print("STDOUT:") + print_offset(stdout) + print("STDERR:") + print_offset(stderr) + raise RuntimeError("Unexpected error from external tool.") + return stdout, stderr, proc.returncode + + +def ensure_environment_is_ready(mypy_path: str, temp_repo_path: str, mypy_cache_path: str) -> None: + os.chdir(mypy_path) + delete_folder(temp_repo_path) + delete_folder(mypy_cache_path) + + +def initialize_repo(repo_url: str, temp_repo_path: str, branch: str) -> None: + print(f"Cloning repo {repo_url} to {temp_repo_path}") + execute(["git", "clone", repo_url, temp_repo_path]) + if branch is not None: + print(f"Checking out branch {branch}") + execute(["git", "-C", temp_repo_path, "checkout", branch]) + + +def get_commits(repo_folder_path: str, commit_range: str) -> list[tuple[str, str]]: + raw_data, _stderr, _errcode = execute( + ["git", "-C", repo_folder_path, "log", "--reverse", "--oneline", commit_range] + ) + output = [] + for line in raw_data.strip().split("\n"): + commit_id, _, message = line.partition(" ") + output.append((commit_id, message)) + return output + + +def get_commits_starting_at(repo_folder_path: str, start_commit: str) -> list[tuple[str, str]]: + print(f"Fetching commits starting at {start_commit}") + return get_commits(repo_folder_path, f"{start_commit}^..HEAD") + + +def get_nth_commit(repo_folder_path: str, n: int) -> tuple[str, str]: + print(f"Fetching last {n} commits (or all, if there are fewer commits than n)") + return get_commits(repo_folder_path, f"-{n}")[0] + + +def run_mypy( + target_file_path: str | None, + mypy_cache_path: str, + mypy_script: str | None, + *, + incremental: bool = False, + daemon: bool = False, + verbose: bool = False, +) -> tuple[float, str, dict[str, Any]]: + """Runs mypy against `target_file_path` and returns what mypy prints to stdout as a string. + + If `incremental` is set to True, this function will use store and retrieve all caching data + inside `mypy_cache_path`. If `verbose` is set to True, this function will pass the "-v -v" + flags to mypy to make it output debugging information. + + If `daemon` is True, we use daemon mode; the daemon must be started and stopped by the caller. + """ + stats: dict[str, Any] = {} + if daemon: + command = DAEMON_CMD + ["check", "-v"] + else: + if mypy_script is None: + command = ["python3", "-m", "mypy"] + else: + command = [mypy_script] + command.extend(["--cache-dir", mypy_cache_path]) + if incremental: + command.append("--incremental") + if verbose: + command.extend(["-v", "-v"]) + if target_file_path is not None: + command.append(target_file_path) + start = time.time() + output, stderr, _ = execute(command, False) + if stderr != "": + output = stderr + else: + if daemon: + output, stats = filter_daemon_stats(output) + runtime = time.time() - start + return runtime, output, stats + + +def filter_daemon_stats(output: str) -> tuple[str, dict[str, Any]]: + stats: dict[str, Any] = {} + lines = output.splitlines() + output_lines = [] + for line in lines: + m = re.match(r"(\w+)\s+:\s+(.*)", line) + if m: + key, value = m.groups() + stats[key] = value + else: + output_lines.append(line) + if output_lines: + output_lines.append("\n") + return "\n".join(output_lines), stats + + +def start_daemon(mypy_cache_path: str) -> None: + cmd = DAEMON_CMD + [ + "restart", + "--log-file", + "./@incr-chk-logs", + "--", + "--cache-dir", + mypy_cache_path, + ] + execute(cmd) + + +def stop_daemon() -> None: + execute(DAEMON_CMD + ["stop"]) + + +def load_cache(incremental_cache_path: str = CACHE_PATH) -> JsonDict: + if os.path.exists(incremental_cache_path): + with open(incremental_cache_path) as stream: + cache = json.load(stream) + assert isinstance(cache, dict) + return cache + else: + return {} + + +def save_cache(cache: JsonDict, incremental_cache_path: str = CACHE_PATH) -> None: + with open(incremental_cache_path, "w") as stream: + json.dump(cache, stream, indent=2) + + +def set_expected( + commits: list[tuple[str, str]], + cache: JsonDict, + temp_repo_path: str, + target_file_path: str | None, + mypy_cache_path: str, + mypy_script: str | None, +) -> None: + """Populates the given `cache` with the expected results for all of the given `commits`. + + This function runs mypy on the `target_file_path` inside the `temp_repo_path`, and stores + the result in the `cache`. + + If `cache` already contains results for a particular commit, this function will + skip evaluating that commit and move on to the next.""" + for commit_id, message in commits: + if commit_id in cache: + print(f'Skipping commit (already cached): {commit_id}: "{message}"') + else: + print(f'Caching expected output for commit {commit_id}: "{message}"') + execute(["git", "-C", temp_repo_path, "checkout", commit_id]) + runtime, output, stats = run_mypy( + target_file_path, mypy_cache_path, mypy_script, incremental=False + ) + cache[commit_id] = {"runtime": runtime, "output": output} + if output == "": + print(f" Clean output ({runtime:.3f} sec)") + else: + print(f" Output ({runtime:.3f} sec)") + print_offset(output, 8) + print() + + +def test_incremental( + commits: list[tuple[str, str]], + cache: JsonDict, + temp_repo_path: str, + target_file_path: str | None, + mypy_cache_path: str, + *, + mypy_script: str | None = None, + daemon: bool = False, + exit_on_error: bool = False, +) -> None: + """Runs incremental mode on all `commits` to verify the output matches the expected output. + + This function runs mypy on the `target_file_path` inside the `temp_repo_path`. The + expected output must be stored inside of the given `cache`. + """ + print("Note: first commit is evaluated twice to warm up cache") + commits = [commits[0]] + commits + overall_stats: dict[str, float] = {} + for commit_id, message in commits: + print(f'Now testing commit {commit_id}: "{message}"') + execute(["git", "-C", temp_repo_path, "checkout", commit_id]) + runtime, output, stats = run_mypy( + target_file_path, mypy_cache_path, mypy_script, incremental=True, daemon=daemon + ) + relevant_stats = combine_stats(overall_stats, stats) + expected_runtime: float = cache[commit_id]["runtime"] + expected_output: str = cache[commit_id]["output"] + if output != expected_output: + print(" Output does not match expected result!") + print(f" Expected output ({expected_runtime:.3f} sec):") + print_offset(expected_output, 8) + print(f" Actual output: ({runtime:.3f} sec):") + print_offset(output, 8) + if exit_on_error: + break + else: + print(" Output matches expected result!") + print(f" Incremental: {runtime:.3f} sec") + print(f" Original: {expected_runtime:.3f} sec") + if relevant_stats: + print(f" Stats: {relevant_stats}") + if overall_stats: + print("Overall stats:", overall_stats) + + +def combine_stats(overall_stats: dict[str, float], new_stats: dict[str, Any]) -> dict[str, float]: + INTERESTING_KEYS = ["build_time", "gc_time"] + # For now, we only support float keys + relevant_stats: dict[str, float] = {} + for key in INTERESTING_KEYS: + if key in new_stats: + value = float(new_stats[key]) + relevant_stats[key] = value + overall_stats[key] = overall_stats.get(key, 0.0) + value + return relevant_stats + + +def cleanup(temp_repo_path: str, mypy_cache_path: str) -> None: + delete_folder(temp_repo_path) + delete_folder(mypy_cache_path) + + +def test_repo( + target_repo_url: str, + temp_repo_path: str, + target_file_path: str | None, + mypy_path: str, + incremental_cache_path: str, + mypy_cache_path: str, + range_type: str, + range_start: str, + branch: str, + params: Namespace, +) -> None: + """Tests incremental mode against the repo specified in `target_repo_url`. + + This algorithm runs in five main stages: + + 1. Clones `target_repo_url` into the `temp_repo_path` folder locally, + checking out the specified `branch` if applicable. + 2. Examines the repo's history to get the list of all commits to + to test incremental mode on. + 3. Runs mypy WITHOUT incremental mode against the `target_file_path` (which is + assumed to be located inside the `temp_repo_path`), testing each commit + discovered in stage two. + - If the results of running mypy WITHOUT incremental mode on a + particular commit are already cached inside the `incremental_cache_path`, + skip that commit to save time. + - Cache the results after finishing. + 4. Rewind back to the first commit, and run mypy WITH incremental mode + against the `target_file_path` commit-by-commit, and compare to the expected + results found in stage 3. + 5. Delete all unnecessary temp files. + """ + # Stage 1: Clone repo and get ready to being testing + ensure_environment_is_ready(mypy_path, temp_repo_path, mypy_cache_path) + initialize_repo(target_repo_url, temp_repo_path, branch) + + # Stage 2: Get all commits we want to test + if range_type == "last": + start_commit = get_nth_commit(temp_repo_path, int(range_start))[0] + elif range_type == "commit": + start_commit = range_start + else: + raise RuntimeError(f"Invalid option: {range_type}") + commits = get_commits_starting_at(temp_repo_path, start_commit) + if params.limit: + commits = commits[: params.limit] + if params.sample: + seed = params.seed or base64.urlsafe_b64encode(os.urandom(15)).decode("ascii") + random.seed(seed) + commits = random.sample(commits, params.sample) + print("Sampled down to %d commits using random seed %s" % (len(commits), seed)) + + # Stage 3: Find and cache expected results for each commit (without incremental mode) + cache = load_cache(incremental_cache_path) + set_expected( + commits, + cache, + temp_repo_path, + target_file_path, + mypy_cache_path, + mypy_script=params.mypy_script, + ) + save_cache(cache, incremental_cache_path) + + # Stage 4: Rewind and re-run mypy (with incremental mode enabled) + if params.daemon: + print("Starting daemon") + start_daemon(mypy_cache_path) + test_incremental( + commits, + cache, + temp_repo_path, + target_file_path, + mypy_cache_path, + mypy_script=params.mypy_script, + daemon=params.daemon, + exit_on_error=params.exit_on_error, + ) + + # Stage 5: Remove temp files, stop daemon + if not params.keep_temporary_files: + cleanup(temp_repo_path, mypy_cache_path) + if params.daemon: + print("Stopping daemon") + stop_daemon() + + +def main() -> None: + help_factory: Any = lambda prog: RawDescriptionHelpFormatter(prog=prog, max_help_position=32) + parser = ArgumentParser( + prog="incremental_checker", description=__doc__, formatter_class=help_factory + ) + + parser.add_argument( + "range_type", + metavar="START_TYPE", + choices=["last", "commit"], + help="must be one of 'last' or 'commit'", + ) + parser.add_argument( + "range_start", + metavar="COMMIT_ID_OR_NUMBER", + help="the commit id to start from, or the number of commits to move back (see above)", + ) + parser.add_argument( + "-r", + "--repo_url", + default=MYPY_REPO_URL, + metavar="URL", + help="the repo to clone and run tests on", + ) + parser.add_argument( + "-f", + "--file-path", + default=MYPY_TARGET_FILE, + metavar="FILE", + help="the name of the file or directory to typecheck", + ) + parser.add_argument( + "-x", "--exit-on-error", action="store_true", help="Exits as soon as an error occurs" + ) + parser.add_argument( + "--keep-temporary-files", action="store_true", help="Keep temporary files on exit" + ) + parser.add_argument( + "--cache-path", + default=CACHE_PATH, + metavar="DIR", + help="sets a custom location to store cache data", + ) + parser.add_argument( + "--branch", + default=None, + metavar="NAME", + help="check out and test a custom branch uses the default if not specified", + ) + parser.add_argument("--sample", type=int, help="use a random sample of size SAMPLE") + parser.add_argument("--seed", type=str, help="random seed") + parser.add_argument( + "--limit", type=int, help="maximum number of commits to use (default until end)" + ) + parser.add_argument("--mypy-script", type=str, help="alternate mypy script to run") + parser.add_argument( + "--daemon", + action="store_true", + help="use mypy daemon instead of incremental (highly experimental)", + ) + + if len(sys.argv[1:]) == 0: + parser.print_help() + parser.exit() + + params = parser.parse_args(sys.argv[1:]) + + # Make all paths absolute so we avoid having to worry about being in the right folder + + # The path to this specific script (incremental_checker.py). + script_path = os.path.abspath(sys.argv[0]) + + # The path to the mypy repo. + mypy_path = os.path.abspath(os.path.dirname(os.path.dirname(script_path))) + + # The folder the cloned repo will reside in. + temp_repo_path = os.path.abspath(os.path.join(mypy_path, "tmp_repo")) + + # The particular file or package to typecheck inside the repo. + if params.file_path: + target_file_path = os.path.abspath(os.path.join(temp_repo_path, params.file_path)) + else: + # Allow `-f ''` to clear target_file_path. + target_file_path = None + + # The path to where the incremental checker cache data is stored. + incremental_cache_path = os.path.abspath(params.cache_path) + + # The path to store the mypy incremental mode cache data + mypy_cache_path = os.path.abspath(os.path.join(mypy_path, "misc", ".mypy_cache")) + + print(f"Assuming mypy is located at {mypy_path}") + print(f"Temp repo will be cloned at {temp_repo_path}") + print(f"Testing file/dir located at {target_file_path}") + print(f"Using cache data located at {incremental_cache_path}") + print() + + test_repo( + params.repo_url, + temp_repo_path, + target_file_path, + mypy_path, + incremental_cache_path, + mypy_cache_path, + params.range_type, + params.range_start, + params.branch, + params, + ) + + +if __name__ == "__main__": + main() diff --git a/misc/macs.el b/misc/macs.el new file mode 100644 index 000000000000..f4cf6702b989 --- /dev/null +++ b/misc/macs.el @@ -0,0 +1,22 @@ +; Example Emacs integration; shows type of expression in region. + +(defun mypy-show-region () + "Show type of variable at point." + (interactive) + (let ((here (region-beginning)) + (there (region-end)) + (filename (buffer-file-name))) + (let ((hereline (line-number-at-pos here)) + (herecol (save-excursion (goto-char here) (current-column))) + (thereline (line-number-at-pos there)) + (therecol (save-excursion (goto-char there) (current-column)))) + (shell-command + (format "cd ~/src/mypy; python3 ./misc/find_type.py %s %s %s %s %s python3 -m mypy -i mypy" + filename hereline herecol thereline therecol) + ) + ) + ) + ) + +; I like to bind this to ^X-t. +(global-set-key "\C-xt" 'mypy-show-region) diff --git a/misc/perf_checker.py b/misc/perf_checker.py new file mode 100644 index 000000000000..20c313e61af9 --- /dev/null +++ b/misc/perf_checker.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import os +import shutil +import statistics +import subprocess +import textwrap +import time +from typing import Callable + + +class Command: + def __init__(self, setup: Callable[[], None], command: Callable[[], None]) -> None: + self.setup = setup + self.command = command + + +def print_offset(text: str, indent_length: int = 4) -> None: + print() + print(textwrap.indent(text, " " * indent_length)) + print() + + +def delete_folder(folder_path: str) -> None: + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + + +def execute(command: list[str]) -> None: + proc = subprocess.Popen( + " ".join(command), stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True + ) + stdout_bytes, stderr_bytes = proc.communicate() + stdout, stderr = stdout_bytes.decode("utf-8"), stderr_bytes.decode("utf-8") + if proc.returncode != 0: + print("EXECUTED COMMAND:", repr(command)) + print("RETURN CODE:", proc.returncode) + print() + print("STDOUT:") + print_offset(stdout) + print("STDERR:") + print_offset(stderr) + raise RuntimeError("Unexpected error from external tool.") + + +def trial(num_trials: int, command: Command) -> list[float]: + trials = [] + for i in range(num_trials): + command.setup() + start = time.time() + command.command() + delta = time.time() - start + trials.append(delta) + return trials + + +def report(name: str, times: list[float]) -> None: + print(f"{name}:") + print(f" Times: {times}") + print(f" Mean: {statistics.mean(times)}") + print(f" Stdev: {statistics.stdev(times)}") + print() + + +def main() -> None: + trials = 3 + + print("Testing baseline") + baseline = trial( + trials, Command(lambda: None, lambda: execute(["python3", "-m", "mypy", "mypy"])) + ) + report("Baseline", baseline) + + print("Testing cold cache") + cold_cache = trial( + trials, + Command( + lambda: delete_folder(".mypy_cache"), + lambda: execute(["python3", "-m", "mypy", "-i", "mypy"]), + ), + ) + report("Cold cache", cold_cache) + + print("Testing warm cache") + execute(["python3", "-m", "mypy", "-i", "mypy"]) + warm_cache = trial( + trials, Command(lambda: None, lambda: execute(["python3", "-m", "mypy", "-i", "mypy"])) + ) + report("Warm cache", warm_cache) + + +if __name__ == "__main__": + main() diff --git a/misc/perf_compare.py b/misc/perf_compare.py new file mode 100644 index 000000000000..be05bb6ddc32 --- /dev/null +++ b/misc/perf_compare.py @@ -0,0 +1,146 @@ +"""Compare performance of mypyc-compiled mypy between one or more commits/branches. + +Simple usage: + + python misc/perf_compare.py my-branch master ... + +What this does: + + * Create a temp clone of the mypy repo for each target commit to measure + * Checkout a target commit in each of the clones + * Compile mypyc in each of the clones *in parallel* + * Create another temp clone of the mypy repo as the code to check + * Self check with each of the compiled mypys N times + * Report the average runtimes and relative performance + * Remove the temp clones +""" + +from __future__ import annotations + +import argparse +import glob +import os +import random +import shutil +import statistics +import subprocess +import sys +import threading +import time + + +def heading(s: str) -> None: + print() + print(f"=== {s} ===") + print() + + +def build_mypy(target_dir: str) -> None: + env = os.environ.copy() + env["CC"] = "clang" + env["MYPYC_OPT_LEVEL"] = "2" + cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"] + subprocess.run(cmd, env=env, check=True, cwd=target_dir) + + +def clone(target_dir: str, commit: str | None) -> None: + heading(f"Cloning mypy to {target_dir}") + repo_dir = os.getcwd() + if os.path.isdir(target_dir): + print(f"{target_dir} exists: deleting") + shutil.rmtree(target_dir) + subprocess.run(["git", "clone", repo_dir, target_dir], check=True) + if commit: + subprocess.run(["git", "checkout", commit], check=True, cwd=target_dir) + + +def run_benchmark(compiled_dir: str, check_dir: str) -> float: + cache_dir = os.path.join(compiled_dir, ".mypy_cache") + if os.path.isdir(cache_dir): + shutil.rmtree(cache_dir) + env = os.environ.copy() + env["PYTHONPATH"] = os.path.abspath(compiled_dir) + abschk = os.path.abspath(check_dir) + cmd = [ + sys.executable, + "-m", + "mypy", + "--config-file", + os.path.join(abschk, "mypy_self_check.ini"), + ] + cmd += glob.glob(os.path.join(abschk, "mypy/*.py")) + cmd += glob.glob(os.path.join(abschk, "mypy/*/*.py")) + t0 = time.time() + # Ignore errors, since some commits being measured may generate additional errors. + subprocess.run(cmd, cwd=compiled_dir, env=env) + return time.time() - t0 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("commit", nargs="+") + args = parser.parse_args() + commits = args.commit + num_runs = 16 + + if not (os.path.isdir(".git") and os.path.isdir("mypyc")): + sys.exit("error: Run this the mypy repo root") + + build_threads = [] + target_dirs = [] + for i, commit in enumerate(commits): + target_dir = f"mypy.{i}.tmpdir" + target_dirs.append(target_dir) + clone(target_dir, commit) + t = threading.Thread(target=lambda: build_mypy(target_dir)) + t.start() + build_threads.append(t) + + self_check_dir = "mypy.self.tmpdir" + clone(self_check_dir, commits[0]) + + heading("Compiling mypy") + print("(This will take a while...)") + + for t in build_threads: + t.join() + + print(f"Finished compiling mypy ({len(commits)} builds)") + + heading("Performing measurements") + + results: dict[str, list[float]] = {} + for n in range(num_runs): + if n == 0: + print("Warmup...") + else: + print(f"Run {n}/{num_runs - 1}...") + items = list(enumerate(commits)) + random.shuffle(items) + for i, commit in items: + tt = run_benchmark(target_dirs[i], self_check_dir) + # Don't record the first warm-up run + if n > 0: + print(f"{commit}: t={tt:.3f}s") + results.setdefault(commit, []).append(tt) + + print() + heading("Results") + first = -1.0 + for commit in commits: + tt = statistics.mean(results[commit]) + if first < 0: + delta = "0.0%" + first = tt + else: + d = (tt / first) - 1 + delta = f"{d:+.1%}" + print(f"{commit:<25} {tt:.3f}s ({delta})") + + shutil.rmtree(self_check_dir) + for target_dir in target_dirs: + shutil.rmtree(target_dir) + + +if __name__ == "__main__": + main() diff --git a/misc/remove-eol-whitespace.sh b/misc/remove-eol-whitespace.sh deleted file mode 100644 index 3da6b9de64a5..000000000000 --- a/misc/remove-eol-whitespace.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# Remove trailing whitespace from all non-binary files in a git repo. - -# From https://gist.github.com/dpaluy/3690668; originally from here: -# http://unix.stackexchange.com/questions/36233/how-to-skip-file-in-sed-if-it-contains-regex/36240#36240 - -git grep -I --name-only -z -e '' | xargs -0 sed -i -e 's/[ \t]\+\(\r\?\)$/\1/' diff --git a/misc/sync-typeshed.py b/misc/sync-typeshed.py new file mode 100644 index 000000000000..22023234710e --- /dev/null +++ b/misc/sync-typeshed.py @@ -0,0 +1,228 @@ +"""Sync stdlib stubs (and a few other files) from typeshed. + +Usage: + + python3 misc/sync-typeshed.py [--commit hash] [--typeshed-dir dir] + +By default, sync to the latest typeshed commit. +""" + +from __future__ import annotations + +import argparse +import functools +import glob +import os +import re +import shutil +import subprocess +import sys +import tempfile +import textwrap +from collections.abc import Mapping + +import requests + + +def check_state() -> None: + if not os.path.isfile("pyproject.toml") or not os.path.isdir("mypy"): + sys.exit("error: The current working directory must be the mypy repository root") + out = subprocess.check_output(["git", "status", "-s", os.path.join("mypy", "typeshed")]) + if out: + # If there are local changes under mypy/typeshed, they would be lost. + sys.exit('error: Output of "git status -s mypy/typeshed" must be empty') + + +def update_typeshed(typeshed_dir: str, commit: str | None) -> str: + """Update contents of local typeshed copy. + + We maintain our own separate mypy_extensions stubs, since it's + treated specially by mypy and we make assumptions about what's there. + We don't sync mypy_extensions stubs here -- this is done manually. + + Return the normalized typeshed commit hash. + """ + assert os.path.isdir(os.path.join(typeshed_dir, "stdlib")) + if commit: + subprocess.run(["git", "checkout", commit], check=True, cwd=typeshed_dir) + commit = git_head_commit(typeshed_dir) + + stdlib_dir = os.path.join("mypy", "typeshed", "stdlib") + # Remove existing stubs. + shutil.rmtree(stdlib_dir) + # Copy new stdlib stubs. + shutil.copytree( + os.path.join(typeshed_dir, "stdlib"), stdlib_dir, ignore=shutil.ignore_patterns("@tests") + ) + shutil.copy(os.path.join(typeshed_dir, "LICENSE"), os.path.join("mypy", "typeshed")) + return commit + + +def git_head_commit(repo: str) -> str: + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo).decode("ascii") + return commit.strip() + + +@functools.cache +def get_github_api_headers() -> Mapping[str, str]: + headers = {"Accept": "application/vnd.github.v3+json"} + secret = os.environ.get("GITHUB_TOKEN") + if secret is not None: + headers["Authorization"] = ( + f"token {secret}" if secret.startswith("ghp") else f"Bearer {secret}" + ) + return headers + + +@functools.cache +def get_origin_owner() -> str: + output = subprocess.check_output(["git", "remote", "get-url", "origin"], text=True).strip() + match = re.match( + r"(git@github.com:|https://github.com/)(?P[^/]+)/(?P[^/\s]+)", output + ) + assert match is not None, f"Couldn't identify origin's owner: {output!r}" + assert ( + match.group("repo").removesuffix(".git") == "mypy" + ), f'Unexpected repo: {match.group("repo")!r}' + return match.group("owner") + + +def create_or_update_pull_request(*, title: str, body: str, branch_name: str) -> None: + fork_owner = get_origin_owner() + + with requests.post( + "https://api.github.com/repos/python/mypy/pulls", + json={ + "title": title, + "body": body, + "head": f"{fork_owner}:{branch_name}", + "base": "master", + }, + headers=get_github_api_headers(), + ) as response: + resp_json = response.json() + if response.status_code == 422 and any( + "A pull request already exists" in e.get("message", "") + for e in resp_json.get("errors", []) + ): + # Find the existing PR + with requests.get( + "https://api.github.com/repos/python/mypy/pulls", + params={"state": "open", "head": f"{fork_owner}:{branch_name}", "base": "master"}, + headers=get_github_api_headers(), + ) as response: + response.raise_for_status() + resp_json = response.json() + assert len(resp_json) >= 1 + pr_number = resp_json[0]["number"] + # Update the PR's title and body + with requests.patch( + f"https://api.github.com/repos/python/mypy/pulls/{pr_number}", + json={"title": title, "body": body}, + headers=get_github_api_headers(), + ) as response: + response.raise_for_status() + return + response.raise_for_status() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--commit", + default=None, + help="Typeshed commit (default to latest main if using a repository clone)", + ) + parser.add_argument( + "--typeshed-dir", + default=None, + help="Location of typeshed (default to a temporary repository clone)", + ) + parser.add_argument( + "--make-pr", + action="store_true", + help="Whether to make a PR with the changes (default to no)", + ) + args = parser.parse_args() + + check_state() + + if args.make_pr: + if os.environ.get("GITHUB_TOKEN") is None: + raise ValueError("GITHUB_TOKEN environment variable must be set") + + with tempfile.TemporaryDirectory() as tmpdir: + # Stash patches before checking out a new branch + typeshed_patches = os.path.join("misc", "typeshed_patches") + tmp_patches = os.path.join(tmpdir, "typeshed_patches") + shutil.copytree(typeshed_patches, tmp_patches) + + branch_name = "mypybot/sync-typeshed" + subprocess.run(["git", "checkout", "-B", branch_name, "origin/master"], check=True) + + # Copy the stashed patches back + shutil.rmtree(typeshed_patches, ignore_errors=True) + shutil.copytree(tmp_patches, typeshed_patches) + if subprocess.run(["git", "diff", "--quiet", "--exit-code"], check=False).returncode != 0: + subprocess.run(["git", "commit", "-am", "Update typeshed patches"], check=True) + + if not args.typeshed_dir: + tmp_typeshed = os.path.join(tmpdir, "typeshed") + os.makedirs(tmp_typeshed) + # Clone typeshed repo if no directory given. + print(f"Cloning typeshed in {tmp_typeshed}...") + subprocess.run( + ["git", "clone", "https://github.com/python/typeshed.git"], + check=True, + cwd=tmp_typeshed, + ) + repo = os.path.join(tmp_typeshed, "typeshed") + commit = update_typeshed(repo, args.commit) + else: + commit = update_typeshed(args.typeshed_dir, args.commit) + + assert commit + + # Create a commit + message = textwrap.dedent( + f"""\ + Sync typeshed + + Source commit: + https://github.com/python/typeshed/commit/{commit} + """ + ) + subprocess.run(["git", "add", "--all", os.path.join("mypy", "typeshed")], check=True) + subprocess.run(["git", "commit", "-m", message], check=True) + print("Created typeshed sync commit.") + + patches = sorted(glob.glob(os.path.join(typeshed_patches, "*.patch"))) + for patch in patches: + cmd = ["git", "am", "--3way", patch] + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"\n\nFailed to apply patch {patch}\n" + "1. Resolve the conflict, `git add --update`, then run `git am --continue`\n" + "2. Run `git format-patch -1 -o misc/typeshed_patches ` " + "to update the patch file.\n" + "3. Re-run sync-typeshed.py" + ) from e + + print(f"Applied patch {patch}") + + if args.make_pr: + subprocess.run(["git", "push", "--force", "origin", branch_name], check=True) + print("Pushed commit.") + + warning = "Note that you will need to close and re-open the PR in order to trigger CI." + + create_or_update_pull_request( + title="Sync typeshed", body=message + "\n" + warning, branch_name=branch_name + ) + print("Created PR.") + + +if __name__ == "__main__": + main() diff --git a/misc/test-stubgenc.sh b/misc/test-stubgenc.sh new file mode 100755 index 000000000000..ad66722628d8 --- /dev/null +++ b/misc/test-stubgenc.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e +set -x + +cd "$(dirname "$0")/.." + +# Install dependencies, demo project and mypy +python -m pip install -r test-requirements.txt +python -m pip install ./test-data/pybind11_fixtures +python -m pip install . + +EXIT=0 + +# performs the stubgenc test +# first argument is the test result folder +# everything else is passed to stubgen as its arguments +function stubgenc_test() { + # Remove expected stubs and generate new inplace + STUBGEN_OUTPUT_FOLDER=./test-data/pybind11_fixtures/$1 + rm -rf "${STUBGEN_OUTPUT_FOLDER:?}" + + stubgen -o "$STUBGEN_OUTPUT_FOLDER" "${@:2}" + + # Check if generated stubs can actually be type checked by mypy + if ! mypy "$STUBGEN_OUTPUT_FOLDER"; + then + echo "Stubgen test failed, because generated stubs failed to type check." + EXIT=1 + fi + + # Compare generated stubs to expected ones + if ! git diff --exit-code "$STUBGEN_OUTPUT_FOLDER"; + then + echo "Stubgen test failed, because generated stubs differ from expected outputs." + EXIT=1 + fi +} + +# create stubs without docstrings +stubgenc_test expected_stubs_no_docs -p pybind11_fixtures +# create stubs with docstrings +stubgenc_test expected_stubs_with_docs -p pybind11_fixtures --include-docstrings + +exit $EXIT diff --git a/misc/trigger_wheel_build.sh b/misc/trigger_wheel_build.sh new file mode 100755 index 000000000000..c914a6e7cf86 --- /dev/null +++ b/misc/trigger_wheel_build.sh @@ -0,0 +1,23 @@ +#!/bin/bash -eux + +# Trigger a build of mypyc compiled mypy wheels by updating the mypy +# submodule in the git repo that drives those builds. + +# $WHEELS_PUSH_TOKEN is stored in Github Settings and is an API token +# for the mypy-build-bot account. + +git config --global user.email "nobody" +git config --global user.name "mypy wheels autopush" + +COMMIT=$(git rev-parse HEAD) +pip install -r mypy-requirements.txt +V=$(python3 -m mypy --version) +V=$(echo "$V" | cut -d" " -f2) + +git clone --depth 1 https://${WHEELS_PUSH_TOKEN}@github.com/mypyc/mypy_mypyc-wheels.git build +cd build +echo $COMMIT > mypy_commit +git commit -am "Build wheels for mypy $V" +git tag v$V +# Push a tag, but no need to push the change to master +git push --tags diff --git a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch new file mode 100644 index 000000000000..91e255242ee9 --- /dev/null +++ b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch @@ -0,0 +1,197 @@ +From b4259edd94188f9e4cc77a22e768eea183a32053 Mon Sep 17 00:00:00 2001 +From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> +Date: Mon, 26 Sep 2022 12:55:07 -0700 +Subject: [PATCH] Remove use of LiteralString in builtins (#13743) + +--- + mypy/typeshed/stdlib/builtins.pyi | 100 +----------------------------- + 1 file changed, 1 insertion(+), 99 deletions(-) + +diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi +index 63c53a5f6..d55042b56 100644 +--- a/mypy/typeshed/stdlib/builtins.pyi ++++ b/mypy/typeshed/stdlib/builtins.pyi +@@ -63,7 +63,6 @@ from typing import ( # noqa: Y022 + from typing_extensions import ( # noqa: Y023 + Concatenate, + Literal, +- LiteralString, + ParamSpec, + Self, + TypeAlias, +@@ -438,31 +437,16 @@ class str(Sequence[str]): + def __new__(cls, object: object = ...) -> Self: ... + @overload + def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... +- @overload +- def capitalize(self: LiteralString) -> LiteralString: ... +- @overload + def capitalize(self) -> str: ... # type: ignore[misc] +- @overload +- def casefold(self: LiteralString) -> LiteralString: ... +- @overload + def casefold(self) -> str: ... # type: ignore[misc] +- @overload +- def center(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... +- @overload + def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] + def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... + def endswith( + self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> bool: ... +- @overload +- def expandtabs(self: LiteralString, tabsize: SupportsIndex = 8) -> LiteralString: ... +- @overload + def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] + def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... +- @overload +- def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... +- @overload + def format(self, *args: object, **kwargs: object) -> str: ... + def format_map(self, mapping: _FormatMapMapping, /) -> str: ... + def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... +@@ -478,99 +462,35 @@ class str(Sequence[str]): + def isspace(self) -> bool: ... + def istitle(self) -> bool: ... + def isupper(self) -> bool: ... +- @overload +- def join(self: LiteralString, iterable: Iterable[LiteralString], /) -> LiteralString: ... +- @overload + def join(self, iterable: Iterable[str], /) -> str: ... # type: ignore[misc] +- @overload +- def ljust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... +- @overload + def ljust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] +- @overload +- def lower(self: LiteralString) -> LiteralString: ... +- @overload + def lower(self) -> str: ... # type: ignore[misc] +- @overload +- def lstrip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... +- @overload + def lstrip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] +- @overload +- def partition(self: LiteralString, sep: LiteralString, /) -> tuple[LiteralString, LiteralString, LiteralString]: ... +- @overload + def partition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] + if sys.version_info >= (3, 13): +- @overload +- def replace( +- self: LiteralString, old: LiteralString, new: LiteralString, /, count: SupportsIndex = -1 +- ) -> LiteralString: ... +- @overload + def replace(self, old: str, new: str, /, count: SupportsIndex = -1) -> str: ... # type: ignore[misc] + else: +- @overload +- def replace( +- self: LiteralString, old: LiteralString, new: LiteralString, count: SupportsIndex = -1, / +- ) -> LiteralString: ... +- @overload + def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ... # type: ignore[misc] + if sys.version_info >= (3, 9): +- @overload +- def removeprefix(self: LiteralString, prefix: LiteralString, /) -> LiteralString: ... +- @overload + def removeprefix(self, prefix: str, /) -> str: ... # type: ignore[misc] +- @overload +- def removesuffix(self: LiteralString, suffix: LiteralString, /) -> LiteralString: ... +- @overload + def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] + + def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... +- @overload +- def rjust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... +- @overload + def rjust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] +- @overload +- def rpartition(self: LiteralString, sep: LiteralString, /) -> tuple[LiteralString, LiteralString, LiteralString]: ... +- @overload + def rpartition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] +- @overload +- def rsplit(self: LiteralString, sep: LiteralString | None = None, maxsplit: SupportsIndex = -1) -> list[LiteralString]: ... +- @overload + def rsplit(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] +- @overload +- def rstrip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... +- @overload + def rstrip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] +- @overload +- def split(self: LiteralString, sep: LiteralString | None = None, maxsplit: SupportsIndex = -1) -> list[LiteralString]: ... +- @overload + def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] +- @overload +- def splitlines(self: LiteralString, keepends: bool = False) -> list[LiteralString]: ... +- @overload + def splitlines(self, keepends: bool = False) -> list[str]: ... # type: ignore[misc] + def startswith( + self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> bool: ... +- @overload +- def strip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... +- @overload + def strip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] +- @overload +- def swapcase(self: LiteralString) -> LiteralString: ... +- @overload + def swapcase(self) -> str: ... # type: ignore[misc] +- @overload +- def title(self: LiteralString) -> LiteralString: ... +- @overload + def title(self) -> str: ... # type: ignore[misc] + def translate(self, table: _TranslateTable, /) -> str: ... +- @overload +- def upper(self: LiteralString) -> LiteralString: ... +- @overload + def upper(self) -> str: ... # type: ignore[misc] +- @overload +- def zfill(self: LiteralString, width: SupportsIndex, /) -> LiteralString: ... +- @overload + def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] + @staticmethod + @overload +@@ -581,39 +501,21 @@ class str(Sequence[str]): + @staticmethod + @overload + def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... +- @overload +- def __add__(self: LiteralString, value: LiteralString, /) -> LiteralString: ... +- @overload + def __add__(self, value: str, /) -> str: ... # type: ignore[misc] + # Incompatible with Sequence.__contains__ + def __contains__(self, key: str, /) -> bool: ... # type: ignore[override] + def __eq__(self, value: object, /) -> bool: ... + def __ge__(self, value: str, /) -> bool: ... +- @overload +- def __getitem__(self: LiteralString, key: SupportsIndex | slice, /) -> LiteralString: ... +- @overload +- def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... # type: ignore[misc] ++ def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... + def __gt__(self, value: str, /) -> bool: ... + def __hash__(self) -> int: ... +- @overload +- def __iter__(self: LiteralString) -> Iterator[LiteralString]: ... +- @overload + def __iter__(self) -> Iterator[str]: ... # type: ignore[misc] + def __le__(self, value: str, /) -> bool: ... + def __len__(self) -> int: ... + def __lt__(self, value: str, /) -> bool: ... +- @overload +- def __mod__(self: LiteralString, value: LiteralString | tuple[LiteralString, ...], /) -> LiteralString: ... +- @overload + def __mod__(self, value: Any, /) -> str: ... +- @overload +- def __mul__(self: LiteralString, value: SupportsIndex, /) -> LiteralString: ... +- @overload + def __mul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] + def __ne__(self, value: object, /) -> bool: ... +- @overload +- def __rmul__(self: LiteralString, value: SupportsIndex, /) -> LiteralString: ... +- @overload + def __rmul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] + def __getnewargs__(self) -> tuple[str]: ... + +-- +2.47.0 + diff --git a/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch b/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch new file mode 100644 index 000000000000..331628af1424 --- /dev/null +++ b/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch @@ -0,0 +1,36 @@ +From 58c6a6ab863c1c38e95ccafaf13792ed9c00e499 Mon Sep 17 00:00:00 2001 +From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> +Date: Sat, 29 Oct 2022 12:47:21 -0700 +Subject: [PATCH] Revert sum literal integer change (#13961) + +This is allegedly causing large performance problems, see 13821 + +typeshed/8231 had zero hits on mypy_primer, so it's not the worst thing +to undo. Patching this in typeshed also feels weird, since there's a +more general soundness issue. If a typevar has a bound or constraint, we +might not want to solve it to a Literal. + +If we can confirm the performance regression or fix the unsoundness +within mypy, I might pursue upstreaming this in typeshed. + +(Reminder: add this to the sync_typeshed script once merged) +--- + mypy/typeshed/stdlib/builtins.pyi | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi +index ea9f8c894..a6065cc67 100644 +--- a/mypy/typeshed/stdlib/builtins.pyi ++++ b/mypy/typeshed/stdlib/builtins.pyi +@@ -1653,7 +1653,7 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit + # without creating many false-positive errors (see #7578). + # Instead, we special-case the most common examples of this: bool and literal integers. + @overload +-def sum(iterable: Iterable[bool | _LiteralInteger], /, start: int = 0) -> int: ... ++def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ... + @overload + def sum(iterable: Iterable[_SupportsSumNoDefaultT], /) -> _SupportsSumNoDefaultT | Literal[0]: ... + @overload +-- +2.46.0 + diff --git a/misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch b/misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch new file mode 100644 index 000000000000..27066bf3c25b --- /dev/null +++ b/misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch @@ -0,0 +1,32 @@ +From 61a490091d7c941780919660dc4fdfa88ae6474a Mon Sep 17 00:00:00 2001 +From: AlexWaygood +Date: Mon, 1 May 2023 20:34:55 +0100 +Subject: [PATCH] Revert typeshed ctypes change Since the plugin provides + superior type checking: + https://github.com/python/mypy/pull/13987#issuecomment-1310863427 A manual + cherry-pick of e437cdf. + +--- + mypy/typeshed/stdlib/_ctypes.pyi | 6 +----- + 1 file changed, 1 insertion(+), 5 deletions(-) + +diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi +index 60bbc51d9..cf9cb81a4 100644 +--- a/mypy/typeshed/stdlib/_ctypes.pyi ++++ b/mypy/typeshed/stdlib/_ctypes.pyi +@@ -169,11 +169,7 @@ class Array(_CData, Generic[_CT]): + def _type_(self) -> type[_CT]: ... + @_type_.setter + def _type_(self, value: type[_CT]) -> None: ... +- # Note: only available if _CT == c_char +- @property +- def raw(self) -> bytes: ... +- @raw.setter +- def raw(self, value: ReadableBuffer) -> None: ... ++ raw: bytes # Note: only available if _CT == c_char + value: Any # Note: bytes if _CT == c_char, str if _CT == c_wchar, unavailable otherwise + # TODO These methods cannot be annotated correctly at the moment. + # All of these "Any"s stand for the array's element type, but it's not possible to use _CT +-- +2.39.3 (Apple Git-146) + diff --git a/misc/upload-pypi.py b/misc/upload-pypi.py new file mode 100644 index 000000000000..9d8827c5e46c --- /dev/null +++ b/misc/upload-pypi.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Upload mypy packages to PyPI. + +You must first tag the release, use `git push --tags` and wait for the wheel build in CI to complete. + +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import re +import shutil +import subprocess +import tarfile +import tempfile +import venv +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Iterator +from urllib.request import urlopen + +BASE = "https://api.github.com/repos" +REPO = "mypyc/mypy_mypyc-wheels" + + +def is_whl_or_tar(name: str) -> bool: + return name.endswith(".tar.gz") or name.endswith(".whl") + + +def item_ok_for_pypi(name: str) -> bool: + if not is_whl_or_tar(name): + return False + + if name.endswith(".tar.gz"): + name = name[:-7] + if name.endswith(".whl"): + name = name[:-4] + + if name.endswith("wasm32"): + return False + + return True + + +def get_release_for_tag(tag: str) -> dict[str, Any]: + with urlopen(f"{BASE}/{REPO}/releases/tags/{tag}") as f: + data = json.load(f) + assert isinstance(data, dict) + assert data["tag_name"] == tag + return data + + +def download_asset(asset: dict[str, Any], dst: Path) -> Path: + name = asset["name"] + assert isinstance(name, str) + download_url = asset["browser_download_url"] + assert is_whl_or_tar(name) + with urlopen(download_url) as src_file: + with open(dst / name, "wb") as dst_file: + shutil.copyfileobj(src_file, dst_file) + return dst / name + + +def download_all_release_assets(release: dict[str, Any], dst: Path) -> None: + print("Downloading assets...") + with ThreadPoolExecutor() as e: + for asset in e.map(lambda asset: download_asset(asset, dst), release["assets"]): + print(f"Downloaded {asset}") + + +def check_sdist(dist: Path, version: str) -> None: + tarfiles = list(dist.glob("*.tar.gz")) + assert len(tarfiles) == 1 + sdist = tarfiles[0] + assert version in sdist.name + with tarfile.open(sdist) as f: + version_py = f.extractfile(f"{sdist.name[:-len('.tar.gz')]}/mypy/version.py") + assert version_py is not None + version_py_contents = version_py.read().decode("utf-8") + + # strip a git hash from our version, if necessary, since that's not present in version.py + match = re.match(r"(.*\+dev).*$", version) + hashless_version = match.group(1) if match else version + + assert ( + f'"{hashless_version}"' in version_py_contents + ), "Version does not match version.py in sdist" + + +def spot_check_dist(dist: Path, version: str) -> None: + items = [item for item in dist.iterdir() if item_ok_for_pypi(item.name)] + assert len(items) > 10 + assert all(version in item.name for item in items) + assert any(item.name.endswith("py3-none-any.whl") for item in items) + + +@contextlib.contextmanager +def tmp_twine() -> Iterator[Path]: + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_venv_dir = Path(tmp_dir) / "venv" + venv.create(tmp_venv_dir, with_pip=True) + pip_exe = tmp_venv_dir / "bin" / "pip" + subprocess.check_call([pip_exe, "install", "twine"]) + yield tmp_venv_dir / "bin" / "twine" + + +def upload_dist(dist: Path, dry_run: bool = True) -> None: + with tmp_twine() as twine: + files = [item for item in dist.iterdir() if item_ok_for_pypi(item.name)] + cmd: list[Any] = [twine, "upload"] + cmd += files + if dry_run: + print("[dry run] " + " ".join(map(str, cmd))) + else: + print(" ".join(map(str, cmd))) + subprocess.check_call(cmd) + + +def upload_to_pypi(version: str, dry_run: bool = True) -> None: + assert re.match(r"v?[1-9]\.[0-9]+\.[0-9](\+\S+)?$", version) + if "dev" in version: + assert dry_run, "Must use --dry-run with dev versions of mypy" + if version.startswith("v"): + version = version[1:] + + target_dir = tempfile.mkdtemp() + dist = Path(target_dir) / "dist" + dist.mkdir() + print(f"Temporary target directory: {target_dir}") + + release = get_release_for_tag(f"v{version}") + download_all_release_assets(release, dist) + + spot_check_dist(dist, version) + check_sdist(dist, version) + upload_dist(dist, dry_run) + print("<< All done! >>") + + +def main() -> None: + parser = argparse.ArgumentParser(description="PyPI mypy package uploader") + parser.add_argument( + "--dry-run", action="store_true", default=False, help="Don't actually upload packages" + ) + parser.add_argument("version", help="mypy version to release") + args = parser.parse_args() + + upload_to_pypi(args.version, args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/mypy-requirements.txt b/mypy-requirements.txt new file mode 100644 index 000000000000..8d41a3fc7003 --- /dev/null +++ b/mypy-requirements.txt @@ -0,0 +1,5 @@ +# NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml +# and the pins in setup.py +typing_extensions>=4.6.0 +mypy_extensions>=1.0.0 +tomli>=1.1.0; python_version<'3.11' diff --git a/mypy/__main__.py b/mypy/__main__.py new file mode 100644 index 000000000000..049553cd1b44 --- /dev/null +++ b/mypy/__main__.py @@ -0,0 +1,37 @@ +"""Mypy type checker command line tool.""" + +from __future__ import annotations + +import os +import sys +import traceback + +from mypy.main import main, process_options +from mypy.util import FancyFormatter + + +def console_entry() -> None: + try: + main() + sys.stdout.flush() + sys.stderr.flush() + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(2) + except KeyboardInterrupt: + _, options = process_options(args=sys.argv[1:]) + if options.show_traceback: + sys.stdout.write(traceback.format_exc()) + formatter = FancyFormatter(sys.stdout, sys.stderr, False) + msg = "Interrupted\n" + sys.stdout.write(formatter.style(msg, color="red", bold=True)) + sys.stdout.flush() + sys.stderr.flush() + sys.exit(2) + + +if __name__ == "__main__": + console_entry() diff --git a/mypy/api.py b/mypy/api.py new file mode 100644 index 000000000000..e2179dba30ca --- /dev/null +++ b/mypy/api.py @@ -0,0 +1,94 @@ +"""This module makes it possible to use mypy as part of a Python application. + +Since mypy still changes, the API was kept utterly simple and non-intrusive. +It just mimics command line activation without starting a new interpreter. +So the normal docs about the mypy command line apply. +Changes in the command line version of mypy will be immediately usable. + +Just import this module and then call the 'run' function with a parameter of +type List[str], containing what normally would have been the command line +arguments to mypy. + +Function 'run' returns a Tuple[str, str, int], namely +(, , ), +in which is what mypy normally writes to sys.stdout, + is what mypy normally writes to sys.stderr and exit_status is +the exit status mypy normally returns to the operating system. + +Any pretty formatting is left to the caller. + +The 'run_dmypy' function is similar, but instead mimics invocation of +dmypy. Note that run_dmypy is not thread-safe and modifies sys.stdout +and sys.stderr during its invocation. + +Note that these APIs don't support incremental generation of error +messages. + +Trivial example of code using this module: + +import sys +from mypy import api + +result = api.run(sys.argv[1:]) + +if result[0]: + print('\nType checking report:\n') + print(result[0]) # stdout + +if result[1]: + print('\nError report:\n') + print(result[1]) # stderr + +print('\nExit status:', result[2]) + +""" + +from __future__ import annotations + +import sys +from io import StringIO +from typing import Callable, TextIO + + +def _run(main_wrapper: Callable[[TextIO, TextIO], None]) -> tuple[str, str, int]: + stdout = StringIO() + stderr = StringIO() + + try: + main_wrapper(stdout, stderr) + exit_status = 0 + except SystemExit as system_exit: + assert isinstance(system_exit.code, int) + exit_status = system_exit.code + + return stdout.getvalue(), stderr.getvalue(), exit_status + + +def run(args: list[str]) -> tuple[str, str, int]: + # Lazy import to avoid needing to import all of mypy to call run_dmypy + from mypy.main import main + + return _run( + lambda stdout, stderr: main(args=args, stdout=stdout, stderr=stderr, clean_exit=True) + ) + + +def run_dmypy(args: list[str]) -> tuple[str, str, int]: + from mypy.dmypy.client import main + + # A bunch of effort has been put into threading stdout and stderr + # through the main API to avoid the threadsafety problems of + # modifying sys.stdout/sys.stderr, but that hasn't been done for + # the dmypy client, so we just do the non-threadsafe thing. + def f(stdout: TextIO, stderr: TextIO) -> None: + old_stdout = sys.stdout + old_stderr = sys.stderr + try: + sys.stdout = stdout + sys.stderr = stderr + main(args) + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + return _run(f) diff --git a/mypy/applytype.py b/mypy/applytype.py index ff9e66caa19c..e88947cc6430 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -1,64 +1,303 @@ -from typing import List, Dict +from __future__ import annotations + +from typing import Callable, Iterable, Sequence import mypy.subtypes +from mypy.erasetype import erase_typevars from mypy.expandtype import expand_type -from mypy.types import Type, Callable, AnyType -from mypy.messages import MessageBuilder -from mypy.nodes import Context +from mypy.nodes import Context, TypeInfo +from mypy.type_visitor import TypeTranslator +from mypy.typeops import get_all_type_vars +from mypy.types import ( + AnyType, + CallableType, + Instance, + Parameters, + ParamSpecFlavor, + ParamSpecType, + PartialType, + ProperType, + Type, + TypeAliasType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UninhabitedType, + UnpackType, + get_proper_type, + remove_dups, +) + +def get_target_type( + tvar: TypeVarLikeType, + type: Type, + callable: CallableType, + report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None], + context: Context, + skip_unsatisfied: bool, +) -> Type | None: + p_type = get_proper_type(type) + if isinstance(p_type, UninhabitedType) and tvar.has_default(): + return tvar.default + if isinstance(tvar, ParamSpecType): + return type + if isinstance(tvar, TypeVarTupleType): + return type + assert isinstance(tvar, TypeVarType) + values = tvar.values + if values: + if isinstance(p_type, AnyType): + return type + if isinstance(p_type, TypeVarType) and p_type.values: + # Allow substituting T1 for T if every allowed value of T1 + # is also a legal value of T. + if all(any(mypy.subtypes.is_same_type(v, v1) for v in values) for v1 in p_type.values): + return type + matching = [] + for value in values: + if mypy.subtypes.is_subtype(type, value): + matching.append(value) + if matching: + best = matching[0] + # If there are more than one matching value, we select the narrowest + for match in matching[1:]: + if mypy.subtypes.is_subtype(match, best): + best = match + return best + if skip_unsatisfied: + return None + report_incompatible_typevar_value(callable, type, tvar.name, context) + else: + upper_bound = tvar.upper_bound + if tvar.name == "Self": + # Internally constructed Self-types contain class type variables in upper bound, + # so we need to erase them to avoid false positives. This is safe because we do + # not support type variables in upper bounds of user defined types. + upper_bound = erase_typevars(upper_bound) + if not mypy.subtypes.is_subtype(type, upper_bound): + if skip_unsatisfied: + return None + report_incompatible_typevar_value(callable, type, tvar.name, context) + return type -def apply_generic_arguments(callable: Callable, types: List[Type], - msg: MessageBuilder, context: Context) -> Type: + +def apply_generic_arguments( + callable: CallableType, + orig_types: Sequence[Type | None], + report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None], + context: Context, + skip_unsatisfied: bool = False, +) -> CallableType: """Apply generic type arguments to a callable type. For example, applying [int] to 'def [T] (T) -> T' results in - 'def [-1:int] (int) -> int'. Here '[-1:int]' is an implicit bound type - variable. + 'def (int) -> int'. Note that each type can be None; in this case, it will not be applied. + + If `skip_unsatisfied` is True, then just skip the types that don't satisfy type variable + bound or constraints, instead of giving an error. """ tvars = callable.variables - if len(tvars) != len(types): - msg.incompatible_type_application(len(tvars), len(types), context) - return AnyType() - + assert len(orig_types) <= len(tvars) # Check that inferred type variable values are compatible with allowed - # values. Also, promote subtype values to allowed values. - types = types[:] - for i, type in enumerate(types): - values = callable.variables[i].values - if values and type: - if isinstance(type, AnyType): - continue - for value in values: - if mypy.subtypes.is_subtype(type, value): - types[i] = value - break - else: - msg.incompatible_typevar_value(callable, i + 1, type, context) - + # values and bounds. Also, promote subtype values to allowed values. # Create a map from type variable id to target type. - id_to_type = {} # type: Dict[int, Type] - for i, tv in enumerate(tvars): - if types[i]: - id_to_type[tv.id] = types[i] + id_to_type: dict[TypeVarId, Type] = {} + + for tvar, type in zip(tvars, orig_types): + assert not isinstance(type, PartialType), "Internal error: must never apply partial type" + if type is None: + continue + + target_type = get_target_type( + tvar, type, callable, report_incompatible_typevar_value, context, skip_unsatisfied + ) + if target_type is not None: + id_to_type[tvar.id] = target_type + + # TODO: validate arg_kinds/arg_names for ParamSpec and TypeVarTuple replacements, + # not just type variable bounds above. + param_spec = callable.param_spec() + if param_spec is not None: + nt = id_to_type.get(param_spec.id) + if nt is not None: + # ParamSpec expansion is special-cased, so we need to always expand callable + # as a whole, not expanding arguments individually. + callable = expand_type(callable, id_to_type) + assert isinstance(callable, CallableType) + return callable.copy_modified( + variables=[tv for tv in tvars if tv.id not in id_to_type] + ) # Apply arguments to argument types. - arg_types = [expand_type(at, id_to_type) for at in callable.arg_types] + var_arg = callable.var_arg() + if var_arg is not None and isinstance(var_arg.typ, UnpackType): + # Same as for ParamSpec, callable with variadic types needs to be expanded as a whole. + callable = expand_type(callable, id_to_type) + assert isinstance(callable, CallableType) + return callable.copy_modified(variables=[tv for tv in tvars if tv.id not in id_to_type]) + else: + callable = callable.copy_modified( + arg_types=[expand_type(at, id_to_type) for at in callable.arg_types] + ) - bound_vars = [(tv.id, id_to_type[tv.id]) - for tv in tvars - if tv.id in id_to_type] + # Apply arguments to TypeGuard and TypeIs if any. + if callable.type_guard is not None: + type_guard = expand_type(callable.type_guard, id_to_type) + else: + type_guard = None + if callable.type_is is not None: + type_is = expand_type(callable.type_is, id_to_type) + else: + type_is = None # The callable may retain some type vars if only some were applied. - remaining_tvars = [tv for tv in tvars if tv.id not in id_to_type] - - return Callable(arg_types, - callable.arg_kinds, - callable.arg_names, - expand_type(callable.ret_type, id_to_type), - callable.fallback, - callable.name, - remaining_tvars, - callable.bound_vars + bound_vars, - callable.line, callable.repr) + # TODO: move apply_poly() logic here when new inference + # becomes universally used (i.e. in all passes + in unification). + # With this new logic we can actually *add* some new free variables. + remaining_tvars: list[TypeVarLikeType] = [] + for tv in tvars: + if tv.id in id_to_type: + continue + if not tv.has_default(): + remaining_tvars.append(tv) + continue + # TypeVarLike isn't in id_to_type mapping. + # Only expand the TypeVar default here. + typ = expand_type(tv, id_to_type) + assert isinstance(typ, TypeVarLikeType) + remaining_tvars.append(typ) + + return callable.copy_modified( + ret_type=expand_type(callable.ret_type, id_to_type), + variables=remaining_tvars, + type_guard=type_guard, + type_is=type_is, + ) + + +def apply_poly(tp: CallableType, poly_tvars: Sequence[TypeVarLikeType]) -> CallableType | None: + """Make free type variables generic in the type if possible. + + This will translate the type `tp` while trying to create valid bindings for + type variables `poly_tvars` while traversing the type. This follows the same rules + as we do during semantic analysis phase, examples: + * Callable[Callable[[T], T], T] -> def [T] (def (T) -> T) -> T + * Callable[[], Callable[[T], T]] -> def () -> def [T] (T -> T) + * List[T] -> None (not possible) + """ + try: + return tp.copy_modified( + arg_types=[t.accept(PolyTranslator(poly_tvars)) for t in tp.arg_types], + ret_type=tp.ret_type.accept(PolyTranslator(poly_tvars)), + variables=[], + ) + except PolyTranslationError: + return None + + +class PolyTranslationError(Exception): + pass + + +class PolyTranslator(TypeTranslator): + """Make free type variables generic in the type if possible. + + See docstring for apply_poly() for details. + """ + + def __init__( + self, + poly_tvars: Iterable[TypeVarLikeType], + bound_tvars: frozenset[TypeVarLikeType] = frozenset(), + seen_aliases: frozenset[TypeInfo] = frozenset(), + ) -> None: + super().__init__() + self.poly_tvars = set(poly_tvars) + # This is a simplified version of TypeVarScope used during semantic analysis. + self.bound_tvars = bound_tvars + self.seen_aliases = seen_aliases + + def collect_vars(self, t: CallableType | Parameters) -> list[TypeVarLikeType]: + found_vars = [] + for arg in t.arg_types: + for tv in get_all_type_vars(arg): + if isinstance(tv, ParamSpecType): + normalized: TypeVarLikeType = tv.copy_modified( + flavor=ParamSpecFlavor.BARE, prefix=Parameters([], [], []) + ) + else: + normalized = tv + if normalized in self.poly_tvars and normalized not in self.bound_tvars: + found_vars.append(normalized) + return remove_dups(found_vars) + + def visit_callable_type(self, t: CallableType) -> Type: + found_vars = self.collect_vars(t) + self.bound_tvars |= set(found_vars) + result = super().visit_callable_type(t) + self.bound_tvars -= set(found_vars) + + assert isinstance(result, ProperType) and isinstance(result, CallableType) + result.variables = list(result.variables) + found_vars + return result + + def visit_type_var(self, t: TypeVarType) -> Type: + if t in self.poly_tvars and t not in self.bound_tvars: + raise PolyTranslationError() + return super().visit_type_var(t) + + def visit_param_spec(self, t: ParamSpecType) -> Type: + if t in self.poly_tvars and t not in self.bound_tvars: + raise PolyTranslationError() + return super().visit_param_spec(t) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: + if t in self.poly_tvars and t not in self.bound_tvars: + raise PolyTranslationError() + return super().visit_type_var_tuple(t) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + if not t.args: + return t.copy_modified() + if not t.is_recursive: + return get_proper_type(t).accept(self) + # We can't handle polymorphic application for recursive generic aliases + # without risking an infinite recursion, just give up for now. + raise PolyTranslationError() + + def visit_instance(self, t: Instance) -> Type: + if t.type.has_param_spec_type: + # We need this special-casing to preserve the possibility to store a + # generic function in an instance type. Things like + # forall T . Foo[[x: T], T] + # are not really expressible in current type system, but this looks like + # a useful feature, so let's keep it. + param_spec_index = next( + i for (i, tv) in enumerate(t.type.defn.type_vars) if isinstance(tv, ParamSpecType) + ) + p = get_proper_type(t.args[param_spec_index]) + if isinstance(p, Parameters): + found_vars = self.collect_vars(p) + self.bound_tvars |= set(found_vars) + new_args = [a.accept(self) for a in t.args] + self.bound_tvars -= set(found_vars) + + repl = new_args[param_spec_index] + assert isinstance(repl, ProperType) and isinstance(repl, Parameters) + repl.variables = list(repl.variables) + list(found_vars) + return t.copy_modified(args=new_args) + # There is the same problem with callback protocols as with aliases + # (callback protocols are essentially more flexible aliases to callables). + if t.args and t.type.is_protocol and t.type.protocol_members == ["__call__"]: + if t.type in self.seen_aliases: + raise PolyTranslationError() + call = mypy.subtypes.find_member("__call__", t, t, is_operator=True) + assert call is not None + return call.accept( + PolyTranslator(self.poly_tvars, self.bound_tvars, self.seen_aliases | {t.type}) + ) + return super().visit_instance(t) diff --git a/mypy/argmap.py b/mypy/argmap.py new file mode 100644 index 000000000000..e6700c9f1092 --- /dev/null +++ b/mypy/argmap.py @@ -0,0 +1,268 @@ +"""Utilities for mapping between actual and formal arguments (and their types).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable, Sequence + +from mypy import nodes +from mypy.maptype import map_instance_to_supertype +from mypy.types import ( + AnyType, + Instance, + ParamSpecType, + TupleType, + Type, + TypedDictType, + TypeOfAny, + TypeVarTupleType, + UnpackType, + get_proper_type, +) + +if TYPE_CHECKING: + from mypy.infer import ArgumentInferContext + + +def map_actuals_to_formals( + actual_kinds: list[nodes.ArgKind], + actual_names: Sequence[str | None] | None, + formal_kinds: list[nodes.ArgKind], + formal_names: Sequence[str | None], + actual_arg_type: Callable[[int], Type], +) -> list[list[int]]: + """Calculate mapping between actual (caller) args and formals. + + The result contains a list of caller argument indexes mapping to each + callee argument index, indexed by callee index. + + The caller_arg_type argument should evaluate to the type of the actual + argument type with the given index. + """ + nformals = len(formal_kinds) + formal_to_actual: list[list[int]] = [[] for i in range(nformals)] + ambiguous_actual_kwargs: list[int] = [] + fi = 0 + for ai, actual_kind in enumerate(actual_kinds): + if actual_kind == nodes.ARG_POS: + if fi < nformals: + if not formal_kinds[fi].is_star(): + formal_to_actual[fi].append(ai) + fi += 1 + elif formal_kinds[fi] == nodes.ARG_STAR: + formal_to_actual[fi].append(ai) + elif actual_kind == nodes.ARG_STAR: + # We need to know the actual type to map varargs. + actualt = get_proper_type(actual_arg_type(ai)) + if isinstance(actualt, TupleType): + # A tuple actual maps to a fixed number of formals. + for _ in range(len(actualt.items)): + if fi < nformals: + if formal_kinds[fi] != nodes.ARG_STAR2: + formal_to_actual[fi].append(ai) + else: + break + if formal_kinds[fi] != nodes.ARG_STAR: + fi += 1 + else: + # Assume that it is an iterable (if it isn't, there will be + # an error later). + while fi < nformals: + if formal_kinds[fi].is_named(star=True): + break + else: + formal_to_actual[fi].append(ai) + if formal_kinds[fi] == nodes.ARG_STAR: + break + fi += 1 + elif actual_kind.is_named(): + assert actual_names is not None, "Internal error: named kinds without names given" + name = actual_names[ai] + if name in formal_names: + formal_to_actual[formal_names.index(name)].append(ai) + elif nodes.ARG_STAR2 in formal_kinds: + formal_to_actual[formal_kinds.index(nodes.ARG_STAR2)].append(ai) + else: + assert actual_kind == nodes.ARG_STAR2 + actualt = get_proper_type(actual_arg_type(ai)) + if isinstance(actualt, TypedDictType): + for name in actualt.items: + if name in formal_names: + formal_to_actual[formal_names.index(name)].append(ai) + elif nodes.ARG_STAR2 in formal_kinds: + formal_to_actual[formal_kinds.index(nodes.ARG_STAR2)].append(ai) + else: + # We don't exactly know which **kwargs are provided by the + # caller, so we'll defer until all the other unambiguous + # actuals have been processed + ambiguous_actual_kwargs.append(ai) + + if ambiguous_actual_kwargs: + # Assume the ambiguous kwargs will fill the remaining arguments. + # + # TODO: If there are also tuple varargs, we might be missing some potential + # matches if the tuple was short enough to not match everything. + unmatched_formals = [ + fi + for fi in range(nformals) + if ( + formal_names[fi] + and ( + not formal_to_actual[fi] + or actual_kinds[formal_to_actual[fi][0]] == nodes.ARG_STAR + ) + and formal_kinds[fi] != nodes.ARG_STAR + ) + or formal_kinds[fi] == nodes.ARG_STAR2 + ] + for ai in ambiguous_actual_kwargs: + for fi in unmatched_formals: + formal_to_actual[fi].append(ai) + + return formal_to_actual + + +def map_formals_to_actuals( + actual_kinds: list[nodes.ArgKind], + actual_names: Sequence[str | None] | None, + formal_kinds: list[nodes.ArgKind], + formal_names: list[str | None], + actual_arg_type: Callable[[int], Type], +) -> list[list[int]]: + """Calculate the reverse mapping of map_actuals_to_formals.""" + formal_to_actual = map_actuals_to_formals( + actual_kinds, actual_names, formal_kinds, formal_names, actual_arg_type + ) + # Now reverse the mapping. + actual_to_formal: list[list[int]] = [[] for _ in actual_kinds] + for formal, actuals in enumerate(formal_to_actual): + for actual in actuals: + actual_to_formal[actual].append(formal) + return actual_to_formal + + +class ArgTypeExpander: + """Utility class for mapping actual argument types to formal arguments. + + One of the main responsibilities is to expand caller tuple *args and TypedDict + **kwargs, and to keep track of which tuple/TypedDict items have already been + consumed. + + Example: + + def f(x: int, *args: str) -> None: ... + f(*(1, 'x', 1.1)) + + We'd call expand_actual_type three times: + + 1. The first call would provide 'int' as the actual type of 'x' (from '1'). + 2. The second call would provide 'str' as one of the actual types for '*args'. + 2. The third call would provide 'float' as one of the actual types for '*args'. + + A single instance can process all the arguments for a single call. Each call + needs a separate instance since instances have per-call state. + """ + + def __init__(self, context: ArgumentInferContext) -> None: + # Next tuple *args index to use. + self.tuple_index = 0 + # Keyword arguments in TypedDict **kwargs used. + self.kwargs_used: set[str] = set() + # Type context for `*` and `**` arg kinds. + self.context = context + + def expand_actual_type( + self, + actual_type: Type, + actual_kind: nodes.ArgKind, + formal_name: str | None, + formal_kind: nodes.ArgKind, + allow_unpack: bool = False, + ) -> Type: + """Return the actual (caller) type(s) of a formal argument with the given kinds. + + If the actual argument is a tuple *args, return the next individual tuple item that + maps to the formal arg. + + If the actual argument is a TypedDict **kwargs, return the next matching typed dict + value type based on formal argument name and kind. + + This is supposed to be called for each formal, in order. Call multiple times per + formal if multiple actuals map to a formal. + """ + original_actual = actual_type + actual_type = get_proper_type(actual_type) + if actual_kind == nodes.ARG_STAR: + if isinstance(actual_type, TypeVarTupleType): + # This code path is hit when *Ts is passed to a callable and various + # special-handling didn't catch this. The best thing we can do is to use + # the upper bound. + actual_type = get_proper_type(actual_type.upper_bound) + if isinstance(actual_type, Instance) and actual_type.args: + from mypy.subtypes import is_subtype + + if is_subtype(actual_type, self.context.iterable_type): + return map_instance_to_supertype( + actual_type, self.context.iterable_type.type + ).args[0] + else: + # We cannot properly unpack anything other + # than `Iterable` type with `*`. + # Just return `Any`, other parts of code would raise + # a different error for improper use. + return AnyType(TypeOfAny.from_error) + elif isinstance(actual_type, TupleType): + # Get the next tuple item of a tuple *arg. + if self.tuple_index >= len(actual_type.items): + # Exhausted a tuple -- continue to the next *args. + self.tuple_index = 1 + else: + self.tuple_index += 1 + item = actual_type.items[self.tuple_index - 1] + if isinstance(item, UnpackType) and not allow_unpack: + # An upack item that doesn't have special handling, use upper bound as above. + unpacked = get_proper_type(item.type) + if isinstance(unpacked, TypeVarTupleType): + fallback = get_proper_type(unpacked.upper_bound) + else: + fallback = unpacked + assert ( + isinstance(fallback, Instance) + and fallback.type.fullname == "builtins.tuple" + ) + item = fallback.args[0] + return item + elif isinstance(actual_type, ParamSpecType): + # ParamSpec is valid in *args but it can't be unpacked. + return actual_type + else: + return AnyType(TypeOfAny.from_error) + elif actual_kind == nodes.ARG_STAR2: + from mypy.subtypes import is_subtype + + if isinstance(actual_type, TypedDictType): + if formal_kind != nodes.ARG_STAR2 and formal_name in actual_type.items: + # Lookup type based on keyword argument name. + assert formal_name is not None + else: + # Pick an arbitrary item if no specified keyword is expected. + formal_name = (set(actual_type.items.keys()) - self.kwargs_used).pop() + self.kwargs_used.add(formal_name) + return actual_type.items[formal_name] + elif ( + isinstance(actual_type, Instance) + and len(actual_type.args) > 1 + and is_subtype(actual_type, self.context.mapping_type) + ): + # Only `Mapping` type can be unpacked with `**`. + # Other types will produce an error somewhere else. + return map_instance_to_supertype(actual_type, self.context.mapping_type.type).args[ + 1 + ] + elif isinstance(actual_type, ParamSpecType): + # ParamSpec is valid in **kwargs but it can't be unpacked. + return actual_type + else: + return AnyType(TypeOfAny.from_error) + else: + # No translation for other kinds -- 1:1 mapping. + return original_actual diff --git a/mypy/binder.py b/mypy/binder.py new file mode 100644 index 000000000000..52ae9774e6d4 --- /dev/null +++ b/mypy/binder.py @@ -0,0 +1,561 @@ +from __future__ import annotations + +from collections import defaultdict +from contextlib import contextmanager +from typing import DefaultDict, Iterator, List, NamedTuple, Optional, Tuple, Union +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.erasetype import remove_instance_last_known_values +from mypy.join import join_simple +from mypy.literals import Key, literal, literal_hash, subkeys +from mypy.nodes import Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, TypeInfo, Var +from mypy.subtypes import is_same_type, is_subtype +from mypy.types import ( + AnyType, + Instance, + NoneType, + PartialType, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeType, + UnionType, + UnpackType, + find_unpack_in_list, + get_proper_type, +) +from mypy.typevars import fill_typevars_with_any + +BindableExpression: _TypeAlias = Union[IndexExpr, MemberExpr, NameExpr] + + +class CurrentType(NamedTuple): + type: Type + from_assignment: bool + + +class Frame: + """A Frame represents a specific point in the execution of a program. + It carries information about the current types of expressions at + that point, arising either from assignments to those expressions + or the result of isinstance checks. It also records whether it is + possible to reach that point at all. + + This information is not copied into a new Frame when it is pushed + onto the stack, so a given Frame only has information about types + that were assigned in that frame. + """ + + def __init__(self, id: int, conditional_frame: bool = False) -> None: + self.id = id + self.types: dict[Key, CurrentType] = {} + self.unreachable = False + self.conditional_frame = conditional_frame + self.suppress_unreachable_warnings = False + + def __repr__(self) -> str: + return f"Frame({self.id}, {self.types}, {self.unreachable}, {self.conditional_frame})" + + +Assigns = DefaultDict[Expression, List[Tuple[Type, Optional[Type]]]] + + +class ConditionalTypeBinder: + """Keep track of conditional types of variables. + + NB: Variables are tracked by literal expression, so it is possible + to confuse the binder; for example, + + ``` + class A: + a: Union[int, str] = None + x = A() + lst = [x] + reveal_type(x.a) # Union[int, str] + x.a = 1 + reveal_type(x.a) # int + reveal_type(lst[0].a) # Union[int, str] + lst[0].a = 'a' + reveal_type(x.a) # int + reveal_type(lst[0].a) # str + ``` + """ + + # Stored assignments for situations with tuple/list lvalue and rvalue of union type. + # This maps an expression to a list of bound types for every item in the union type. + type_assignments: Assigns | None = None + + def __init__(self) -> None: + self.next_id = 1 + + # The stack of frames currently used. These map + # literal_hash(expr) -- literals like 'foo.bar' -- + # to types. The last element of this list is the + # top-most, current frame. Each earlier element + # records the state as of when that frame was last + # on top of the stack. + self.frames = [Frame(self._get_id())] + + # For frames higher in the stack, we record the set of + # Frames that can escape there, either by falling off + # the end of the frame or by a loop control construct + # or raised exception. The last element of self.frames + # has no corresponding element in this list. + self.options_on_return: list[list[Frame]] = [] + + # Maps literal_hash(expr) to get_declaration(expr) + # for every expr stored in the binder + self.declarations: dict[Key, Type | None] = {} + # Set of other keys to invalidate if a key is changed, e.g. x -> {x.a, x[0]} + # Whenever a new key (e.g. x.a.b) is added, we update this + self.dependencies: dict[Key, set[Key]] = {} + + # Whether the last pop changed the newly top frame on exit + self.last_pop_changed = False + + self.try_frames: set[int] = set() + self.break_frames: list[int] = [] + self.continue_frames: list[int] = [] + + def _get_id(self) -> int: + self.next_id += 1 + return self.next_id + + def _add_dependencies(self, key: Key, value: Key | None = None) -> None: + if value is None: + value = key + else: + self.dependencies.setdefault(key, set()).add(value) + for elt in subkeys(key): + self._add_dependencies(elt, value) + + def push_frame(self, conditional_frame: bool = False) -> Frame: + """Push a new frame into the binder.""" + f = Frame(self._get_id(), conditional_frame) + self.frames.append(f) + self.options_on_return.append([]) + return f + + def _put(self, key: Key, type: Type, from_assignment: bool, index: int = -1) -> None: + self.frames[index].types[key] = CurrentType(type, from_assignment) + + def _get(self, key: Key, index: int = -1) -> CurrentType | None: + if index < 0: + index += len(self.frames) + for i in range(index, -1, -1): + if key in self.frames[i].types: + return self.frames[i].types[key] + return None + + def put(self, expr: Expression, typ: Type, *, from_assignment: bool = True) -> None: + if not isinstance(expr, (IndexExpr, MemberExpr, NameExpr)): + return + if not literal(expr): + return + key = literal_hash(expr) + assert key is not None, "Internal error: binder tried to put non-literal" + if key not in self.declarations: + self.declarations[key] = get_declaration(expr) + self._add_dependencies(key) + self._put(key, typ, from_assignment) + + def unreachable(self) -> None: + self.frames[-1].unreachable = True + + def suppress_unreachable_warnings(self) -> None: + self.frames[-1].suppress_unreachable_warnings = True + + def get(self, expr: Expression) -> Type | None: + key = literal_hash(expr) + assert key is not None, "Internal error: binder tried to get non-literal" + found = self._get(key) + if found is None: + return None + return found.type + + def is_unreachable(self) -> bool: + # TODO: Copy the value of unreachable into new frames to avoid + # this traversal on every statement? + return any(f.unreachable for f in self.frames) + + def is_unreachable_warning_suppressed(self) -> bool: + return any(f.suppress_unreachable_warnings for f in self.frames) + + def cleanse(self, expr: Expression) -> None: + """Remove all references to a Node from the binder.""" + key = literal_hash(expr) + assert key is not None, "Internal error: binder tried cleanse non-literal" + self._cleanse_key(key) + + def _cleanse_key(self, key: Key) -> None: + """Remove all references to a key from the binder.""" + for frame in self.frames: + if key in frame.types: + del frame.types[key] + + def update_from_options(self, frames: list[Frame]) -> bool: + """Update the frame to reflect that each key will be updated + as in one of the frames. Return whether any item changes. + + If a key is declared as AnyType, only update it if all the + options are the same. + """ + all_reachable = all(not f.unreachable for f in frames) + frames = [f for f in frames if not f.unreachable] + changed = False + keys = {key for f in frames for key in f.types} + + for key in keys: + current_value = self._get(key) + resulting_values = [f.types.get(key, current_value) for f in frames] + if any(x is None for x in resulting_values): + # We didn't know anything about key before + # (current_value must be None), and we still don't + # know anything about key in at least one possible frame. + continue + + if all_reachable and all( + x is not None and not x.from_assignment for x in resulting_values + ): + # Do not synthesize a new type if we encountered a conditional block + # (if, while or match-case) without assignments. + # See check-isinstance.test::testNoneCheckDoesNotMakeTypeVarOptional + # This is a safe assumption: the fact that we checked something with `is` + # or `isinstance` does not change the type of the value. + continue + + current_type = resulting_values[0] + assert current_type is not None + type = current_type.type + declaration_type = get_proper_type(self.declarations.get(key)) + if isinstance(declaration_type, AnyType): + # At this point resulting values can't contain None, see continue above + if not all( + t is not None and is_same_type(type, t.type) for t in resulting_values[1:] + ): + type = AnyType(TypeOfAny.from_another_any, source_any=declaration_type) + else: + for other in resulting_values[1:]: + assert other is not None + type = join_simple(self.declarations[key], type, other.type) + # Try simplifying resulting type for unions involving variadic tuples. + # Technically, everything is still valid without this step, but if we do + # not do this, this may create long unions after exiting an if check like: + # x: tuple[int, ...] + # if len(x) < 10: + # ... + # We want the type of x to be tuple[int, ...] after this block (if it is + # still equivalent to such type). + if isinstance(type, UnionType): + type = collapse_variadic_union(type) + if isinstance(type, ProperType) and isinstance(type, UnionType): + # Simplify away any extra Any's that were added to the declared + # type when popping a frame. + simplified = UnionType.make_union( + [t for t in type.items if not isinstance(get_proper_type(t), AnyType)] + ) + if simplified == self.declarations[key]: + type = simplified + if current_value is None or not is_same_type(type, current_value[0]): + self._put(key, type, from_assignment=True) + changed = True + + self.frames[-1].unreachable = not frames + + return changed + + def pop_frame(self, can_skip: bool, fall_through: int) -> Frame: + """Pop a frame and return it. + + See frame_context() for documentation of fall_through. + """ + + if fall_through > 0: + self.allow_jump(-fall_through) + + result = self.frames.pop() + options = self.options_on_return.pop() + + if can_skip: + options.insert(0, self.frames[-1]) + + self.last_pop_changed = self.update_from_options(options) + + return result + + @contextmanager + def accumulate_type_assignments(self) -> Iterator[Assigns]: + """Push a new map to collect assigned types in multiassign from union. + + If this map is not None, actual binding is deferred until all items in + the union are processed (a union of collected items is later bound + manually by the caller). + """ + old_assignments = None + if self.type_assignments is not None: + old_assignments = self.type_assignments + self.type_assignments = defaultdict(list) + yield self.type_assignments + self.type_assignments = old_assignments + + def assign_type( + self, expr: Expression, type: Type, declared_type: Type | None, restrict_any: bool = False + ) -> None: + # We should erase last known value in binder, because if we are using it, + # it means that the target is not final, and therefore can't hold a literal. + type = remove_instance_last_known_values(type) + + if self.type_assignments is not None: + # We are in a multiassign from union, defer the actual binding, + # just collect the types. + self.type_assignments[expr].append((type, declared_type)) + return + if not isinstance(expr, (IndexExpr, MemberExpr, NameExpr)): + return + if not literal(expr): + return + self.invalidate_dependencies(expr) + + if declared_type is None: + # Not sure why this happens. It seems to mainly happen in + # member initialization. + return + if not is_subtype(type, declared_type): + # Pretty sure this is only happens when there's a type error. + + # Ideally this function wouldn't be called if the + # expression has a type error, though -- do other kinds of + # errors cause this function to get called at invalid + # times? + return + + p_declared = get_proper_type(declared_type) + p_type = get_proper_type(type) + enclosing_type = get_proper_type(self.most_recent_enclosing_type(expr, type)) + if isinstance(enclosing_type, AnyType) and not restrict_any: + # If x is Any and y is int, after x = y we do not infer that x is int. + # This could be changed. + # Instead, since we narrowed type from Any in a recent frame (probably an + # isinstance check), but now it is reassigned, we broaden back + # to Any (which is the most recent enclosing type) + self.put(expr, enclosing_type) + # As a special case, when assigning Any to a variable with a + # declared Optional type that has been narrowed to None, + # replace all the Nones in the declared Union type with Any. + # This overrides the normal behavior of ignoring Any assignments to variables + # in order to prevent false positives. + # (See discussion in #3526) + elif ( + isinstance(p_type, AnyType) + and isinstance(p_declared, UnionType) + and any(isinstance(get_proper_type(item), NoneType) for item in p_declared.items) + and isinstance( + get_proper_type(self.most_recent_enclosing_type(expr, NoneType())), NoneType + ) + ): + # Replace any Nones in the union type with Any + new_items = [ + type if isinstance(get_proper_type(item), NoneType) else item + for item in p_declared.items + ] + self.put(expr, UnionType(new_items)) + elif isinstance(p_type, AnyType) and not ( + isinstance(p_declared, UnionType) + and any(isinstance(get_proper_type(item), AnyType) for item in p_declared.items) + ): + # Assigning an Any value doesn't affect the type to avoid false negatives, unless + # there is an Any item in a declared union type. + self.put(expr, declared_type) + else: + self.put(expr, type) + + for i in self.try_frames: + # XXX This should probably not copy the entire frame, but + # just copy this variable into a single stored frame. + self.allow_jump(i) + + def invalidate_dependencies(self, expr: BindableExpression) -> None: + """Invalidate knowledge of types that include expr, but not expr itself. + + For example, when expr is foo.bar, invalidate foo.bar.baz. + + It is overly conservative: it invalidates globally, including + in code paths unreachable from here. + """ + key = literal_hash(expr) + assert key is not None + for dep in self.dependencies.get(key, set()): + self._cleanse_key(dep) + + def most_recent_enclosing_type(self, expr: BindableExpression, type: Type) -> Type | None: + type = get_proper_type(type) + if isinstance(type, AnyType): + return get_declaration(expr) + key = literal_hash(expr) + assert key is not None + enclosers = [get_declaration(expr)] + [ + f.types[key].type + for f in self.frames + if key in f.types and is_subtype(type, f.types[key][0]) + ] + return enclosers[-1] + + def allow_jump(self, index: int) -> None: + # self.frames and self.options_on_return have different lengths + # so make sure the index is positive + if index < 0: + index += len(self.options_on_return) + frame = Frame(self._get_id()) + for f in self.frames[index + 1 :]: + frame.types.update(f.types) + if f.unreachable: + frame.unreachable = True + self.options_on_return[index].append(frame) + + def handle_break(self) -> None: + self.allow_jump(self.break_frames[-1]) + self.unreachable() + + def handle_continue(self) -> None: + self.allow_jump(self.continue_frames[-1]) + self.unreachable() + + @contextmanager + def frame_context( + self, + *, + can_skip: bool, + fall_through: int = 1, + break_frame: int = 0, + continue_frame: int = 0, + conditional_frame: bool = False, + try_frame: bool = False, + ) -> Iterator[Frame]: + """Return a context manager that pushes/pops frames on enter/exit. + + If can_skip is True, control flow is allowed to bypass the + newly-created frame. + + If fall_through > 0, then it will allow control flow that + falls off the end of the frame to escape to its ancestor + `fall_through` levels higher. Otherwise control flow ends + at the end of the frame. + + If break_frame > 0, then 'break' statements within this frame + will jump out to the frame break_frame levels higher than the + frame created by this call to frame_context. Similarly for + continue_frame and 'continue' statements. + + If try_frame is true, then execution is allowed to jump at any + point within the newly created frame (or its descendants) to + its parent (i.e., to the frame that was on top before this + call to frame_context). + + After the context manager exits, self.last_pop_changed indicates + whether any types changed in the newly-topmost frame as a result + of popping this frame. + """ + assert len(self.frames) > 1 + + if break_frame: + self.break_frames.append(len(self.frames) - break_frame) + if continue_frame: + self.continue_frames.append(len(self.frames) - continue_frame) + if try_frame: + self.try_frames.add(len(self.frames) - 1) + + new_frame = self.push_frame(conditional_frame) + if try_frame: + # An exception may occur immediately + self.allow_jump(-1) + yield new_frame + self.pop_frame(can_skip, fall_through) + + if break_frame: + self.break_frames.pop() + if continue_frame: + self.continue_frames.pop() + if try_frame: + self.try_frames.remove(len(self.frames) - 1) + + @contextmanager + def top_frame_context(self) -> Iterator[Frame]: + """A variant of frame_context for use at the top level of + a namespace (module, function, or class). + """ + assert len(self.frames) == 1 + yield self.push_frame() + self.pop_frame(True, 0) + assert len(self.frames) == 1 + + +def get_declaration(expr: BindableExpression) -> Type | None: + if isinstance(expr, RefExpr): + if isinstance(expr.node, Var): + type = expr.node.type + if not isinstance(get_proper_type(type), PartialType): + return type + elif isinstance(expr.node, TypeInfo): + return TypeType(fill_typevars_with_any(expr.node)) + return None + + +def collapse_variadic_union(typ: UnionType) -> Type: + """Simplify a union involving variadic tuple if possible. + + This will collapse a type like e.g. + tuple[X, Z] | tuple[X, Y, Z] | tuple[X, Y, Y, *tuple[Y, ...], Z] + back to + tuple[X, *tuple[Y, ...], Z] + which is equivalent, but much simpler form of the same type. + """ + tuple_items = [] + other_items = [] + for t in typ.items: + p_t = get_proper_type(t) + if isinstance(p_t, TupleType): + tuple_items.append(p_t) + else: + other_items.append(t) + if len(tuple_items) <= 1: + # This type cannot be simplified further. + return typ + tuple_items = sorted(tuple_items, key=lambda t: len(t.items)) + first = tuple_items[0] + last = tuple_items[-1] + unpack_index = find_unpack_in_list(last.items) + if unpack_index is None: + return typ + unpack = last.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if not isinstance(unpacked, Instance): + return typ + assert unpacked.type.fullname == "builtins.tuple" + suffix = last.items[unpack_index + 1 :] + + # Check that first item matches the expected pattern and infer prefix. + if len(first.items) < len(suffix): + return typ + if suffix and first.items[-len(suffix) :] != suffix: + return typ + if suffix: + prefix = first.items[: -len(suffix)] + else: + prefix = first.items + + # Check that all middle types match the expected pattern as well. + arg = unpacked.args[0] + for i, it in enumerate(tuple_items[1:-1]): + if it.items != prefix + [arg] * (i + 1) + suffix: + return typ + + # Check the last item (the one with unpack), and choose an appropriate simplified type. + if last.items != prefix + [arg] * (len(typ.items) - 1) + [unpack] + suffix: + return typ + if len(first.items) == 0: + simplified: Type = unpacked.copy_modified() + else: + simplified = TupleType(prefix + [unpack] + suffix, fallback=last.partial_fallback) + return UnionType.make_union([simplified] + other_items) diff --git a/mypy/bogus_type.py b/mypy/bogus_type.py new file mode 100644 index 000000000000..1a61abac9732 --- /dev/null +++ b/mypy/bogus_type.py @@ -0,0 +1,27 @@ +"""A Bogus[T] type alias for marking when we subvert the type system + +We need this for compiling with mypyc, which inserts runtime +typechecks that cause problems when we subvert the type system. So +when compiling with mypyc, we turn those places into Any, while +keeping the types around for normal typechecks. + +Since this causes the runtime types to be Any, this is best used +in places where efficient access to properties is not important. +For those cases some other technique should be used. +""" + +from __future__ import annotations + +from typing import Any, TypeVar + +from mypy_extensions import FlexibleAlias + +T = TypeVar("T") + +# This won't ever be true at runtime, but we consider it true during +# mypyc compilations. +MYPYC = False +if MYPYC: + Bogus = FlexibleAlias[T, Any] +else: + Bogus = FlexibleAlias[T, T] diff --git a/mypy/build.py b/mypy/build.py index 138090f525ac..40dd73313335 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1,7 +1,7 @@ -"""Facilities to build mypy programs and modules they depend on. +"""Facilities to analyze entire programs, including imported modules. -Parse, analyze and translate the source files of a program in the correct -order (based on file dependencies), and collect the results. +Parse and analyze the source files of a program in the correct order +(based on file dependencies), and collect the results. This module only directs a build, which is performed in multiple passes per file. The individual passes are implemented in separate modules. @@ -9,910 +9,3551 @@ The function build() is the main interface to this module. """ -import os -import os.path -import shlex -import subprocess -import sys -from os.path import dirname, basename +# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file -from typing import Undefined, Dict, List, Tuple, cast, Set, Union +from __future__ import annotations -from mypy.types import Type -from mypy.nodes import MypyFile, Node, Import, ImportFrom, ImportAll -from mypy.nodes import SymbolTableNode, MODULE_REF -from mypy.semanal import SemanticAnalyzer, FirstPass, ThirdPass +import collections +import contextlib +import errno +import gc +import json +import os +import platform +import re +import stat +import sys +import time +import types +from typing import ( + TYPE_CHECKING, + AbstractSet, + Any, + Callable, + ClassVar, + Dict, + Final, + Iterator, + Mapping, + NamedTuple, + NoReturn, + Sequence, + TextIO, +) +from typing_extensions import TypeAlias as _TypeAlias, TypedDict + +import mypy.semanal_main from mypy.checker import TypeChecker -from mypy.errors import Errors, CompileError -from mypy import parse -from mypy import stats +from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter +from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error +from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort +from mypy.indirection import TypeIndirectionVisitor +from mypy.messages import MessageBuilder +from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable, TypeInfo +from mypy.partially_defined import PossiblyUndefinedVariableVisitor +from mypy.semanal import SemanticAnalyzer +from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis +from mypy.util import ( + DecodeError, + decode_python_encoding, + get_mypy_comments, + hash_digest, + is_stub_package_file, + is_sub_path_normabs, + is_typeshed_file, + module_prefix, + read_py_file, + time_ref, + time_spent_us, +) + +if TYPE_CHECKING: + from mypy.report import Reports # Avoid unconditional slow import + +from mypy import errorcodes as codes +from mypy.config_parser import parse_mypy_comments +from mypy.fixup import fixup_module +from mypy.freetree import free_tree +from mypy.fscache import FileSystemCache +from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore +from mypy.modulefinder import ( + BuildSource as BuildSource, + BuildSourceSet as BuildSourceSet, + FindModuleCache, + ModuleNotFoundReason, + ModuleSearchResult, + SearchPaths, + compute_search_paths, +) +from mypy.nodes import Expression +from mypy.options import Options +from mypy.parse import parse +from mypy.plugin import ChainedPlugin, Plugin, ReportConfigContext +from mypy.plugins.default import DefaultPlugin +from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor +from mypy.stats import dump_type_stats +from mypy.stubinfo import is_module_from_legacy_bundled_package, stub_distribution_name +from mypy.types import Type +from mypy.typestate import reset_global_state, type_state +from mypy.util import json_dumps, json_loads +from mypy.version import __version__ + +# Switch to True to produce debug output related to fine-grained incremental +# mode only that is useful during development. This produces only a subset of +# output compared to --verbose output. We use a global flag to enable this so +# that it's easy to enable this when running tests. +DEBUG_FINE_GRAINED: Final = False + +# These modules are special and should always come from typeshed. +CORE_BUILTIN_MODULES: Final = { + "builtins", + "typing", + "types", + "typing_extensions", + "mypy_extensions", + "_typeshed", + "_collections_abc", + "collections", + "collections.abc", + "sys", + "abc", +} + + +Graph: _TypeAlias = Dict[str, "State"] + + +# TODO: Get rid of BuildResult. We might as well return a BuildManager. +class BuildResult: + """The result of a successful build. + Attributes: + manager: The build manager. + files: Dictionary from module name to related AST node. + types: Dictionary from parse tree node to its inferred type. + used_cache: Whether the build took advantage of a pre-existing cache + errors: List of error messages. + """ -debug = False + def __init__(self, manager: BuildManager, graph: Graph) -> None: + self.manager = manager + self.graph = graph + self.files = manager.modules + self.types = manager.all_types # Non-empty if export_types True in options + self.used_cache = manager.cache_enabled + self.errors: list[str] = [] # Filled in by build if desired + + +def build( + sources: list[BuildSource], + options: Options, + alt_lib_path: str | None = None, + flush_errors: Callable[[str | None, list[str], bool], None] | None = None, + fscache: FileSystemCache | None = None, + stdout: TextIO | None = None, + stderr: TextIO | None = None, + extra_plugins: Sequence[Plugin] | None = None, +) -> BuildResult: + """Analyze a program. + A single call to build performs parsing, semantic analysis and optionally + type checking for the program *and* all imported modules, recursively. -# Build targets (for selecting compiler passes) -SEMANTIC_ANALYSIS = 0 # Semantic analysis only -TYPE_CHECK = 1 # Type check + Return BuildResult if successful or only non-blocking errors were found; + otherwise raise CompileError. + If a flush_errors callback is provided, all error messages will be + passed to it and the errors and messages fields of BuildResult and + CompileError (respectively) will be empty. Otherwise those fields will + report any error messages. -# Build flags -DO_NOT_RUN = 'do-not-run' # Only type check, don't run the program -VERBOSE = 'verbose' # More verbose messages (for troubleshooting) -MODULE = 'module' # Build/run module as a script -TEST_BUILTINS = 'test-builtins' # Use stub builtins to speed up tests + Args: + sources: list of sources to build + options: build options + alt_lib_path: an additional directory for looking up library modules + (takes precedence over other directories) + flush_errors: optional function to flush errors after a file is processed + fscache: optionally a file-system cacher -# State ids. These describe the states a source file / module can be in a -# build. + """ + # If we were not given a flush_errors, we use one that will populate those + # fields for callers that want the traditional API. + messages = [] -# We aren't processing this source file yet (no associated state object). -UNSEEN_STATE = 0 -# The source file has a state object, but we haven't done anything with it yet. -UNPROCESSED_STATE = 1 -# We've parsed the source file. -PARSED_STATE = 2 -# We've done the first two passes of semantic analysis. -PARTIAL_SEMANTIC_ANALYSIS_STATE = 3 -# We've semantically analyzed the source file. -SEMANTICALLY_ANALYSED_STATE = 4 -# We've type checked the source file (and all its dependencies). -TYPE_CHECKED_STATE = 5 + def default_flush_errors( + filename: str | None, new_messages: list[str], is_serious: bool + ) -> None: + messages.extend(new_messages) + flush_errors = flush_errors or default_flush_errors + stdout = stdout or sys.stdout + stderr = stderr or sys.stderr + extra_plugins = extra_plugins or [] -final_state = TYPE_CHECKED_STATE + try: + result = _build( + sources, options, alt_lib_path, flush_errors, fscache, stdout, stderr, extra_plugins + ) + result.errors = messages + return result + except CompileError as e: + # CompileErrors raised from an errors object carry all of the + # messages that have not been reported out by error streaming. + # Patch it up to contain either none or all none of the messages, + # depending on whether we are flushing errors. + serious = not e.use_stdout + flush_errors(None, e.messages, serious) + e.messages = messages + raise + + +def _build( + sources: list[BuildSource], + options: Options, + alt_lib_path: str | None, + flush_errors: Callable[[str | None, list[str], bool], None], + fscache: FileSystemCache | None, + stdout: TextIO, + stderr: TextIO, + extra_plugins: Sequence[Plugin], +) -> BuildResult: + if platform.python_implementation() == "CPython": + # This seems the most reasonable place to tune garbage collection. + gc.set_threshold(150 * 1000) + + data_dir = default_data_dir() + fscache = fscache or FileSystemCache() + + search_paths = compute_search_paths(sources, options, data_dir, alt_lib_path) + + reports = None + if options.report_dirs: + # Import lazily to avoid slowing down startup. + from mypy.report import Reports + + reports = Reports(data_dir, options.report_dirs) + + source_set = BuildSourceSet(sources) + cached_read = fscache.read + errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read)) + plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins) + + # Add catch-all .gitignore to cache dir if we created it + cache_dir_existed = os.path.isdir(options.cache_dir) + + # Construct a build manager object to hold state during the build. + # + # Ignore current directory prefix in error messages. + manager = BuildManager( + data_dir, + search_paths, + ignore_prefix=os.getcwd(), + source_set=source_set, + reports=reports, + options=options, + version_id=__version__, + plugin=plugin, + plugins_snapshot=snapshot, + errors=errors, + error_formatter=None if options.output is None else OUTPUT_CHOICES.get(options.output), + flush_errors=flush_errors, + fscache=fscache, + stdout=stdout, + stderr=stderr, + ) + if manager.verbosity() >= 2: + manager.trace(repr(options)) + + reset_global_state() + try: + graph = dispatch(sources, manager, stdout) + if not options.fine_grained_incremental: + type_state.reset_all_subtype_caches() + if options.timing_stats is not None: + dump_timing_stats(options.timing_stats, graph) + if options.line_checking_stats is not None: + dump_line_checking_stats(options.line_checking_stats, graph) + return BuildResult(manager, graph) + finally: + t0 = time.time() + manager.metastore.commit() + manager.add_stats(cache_commit_time=time.time() - t0) + manager.log( + "Build finished in %.3f seconds with %d modules, and %d errors" + % ( + time.time() - manager.start_time, + len(manager.modules), + manager.errors.num_messages(), + ) + ) + manager.dump_stats() + if reports is not None: + # Finish the HTML or XML reports even if CompileError was raised. + reports.finish() + if not cache_dir_existed and os.path.isdir(options.cache_dir): + add_catch_all_gitignore(options.cache_dir) + exclude_from_backups(options.cache_dir) + if os.path.isdir(options.cache_dir): + record_missing_stub_packages(options.cache_dir, manager.missing_stub_packages) + + +def default_data_dir() -> str: + """Returns directory containing typeshed directory.""" + return os.path.dirname(__file__) + + +def normpath(path: str, options: Options) -> str: + """Convert path to absolute; but to relative in bazel mode. + + (Bazel's distributed cache doesn't like filesystem metadata to + end up in output files.) + """ + # TODO: Could we always use relpath? (A worry in non-bazel + # mode would be that a moved file may change its full module + # name without changing its size, mtime or hash.) + if options.bazel: + return os.path.relpath(path) + else: + return os.path.abspath(path) + + +class CacheMeta(NamedTuple): + id: str + path: str + mtime: int + size: int + hash: str + dependencies: list[str] # names of imported modules + data_mtime: int # mtime of data_json + data_json: str # path of .data.json + suppressed: list[str] # dependencies that weren't imported + options: dict[str, object] | None # build options + # dep_prios and dep_lines are in parallel with dependencies + suppressed + dep_prios: list[int] + dep_lines: list[int] + interface_hash: str # hash representing the public interface + version_id: str # mypy version for cache invalidation + ignore_all: bool # if errors were ignored + plugin_data: Any # config data from plugins + + +# NOTE: dependencies + suppressed == all reachable imports; +# suppressed contains those reachable imports that were prevented by +# silent mode or simply not found. + + +# Metadata for the fine-grained dependencies file associated with a module. +class FgDepMeta(TypedDict): + path: str + mtime: int + + +def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: + """Build a CacheMeta object from a json metadata dictionary + + Args: + meta: JSON metadata read from the metadata cache file + data_json: Path to the .data.json file containing the AST trees + """ + sentinel: Any = None # Values to be validated by the caller + return CacheMeta( + meta.get("id", sentinel), + meta.get("path", sentinel), + int(meta["mtime"]) if "mtime" in meta else sentinel, + meta.get("size", sentinel), + meta.get("hash", sentinel), + meta.get("dependencies", []), + int(meta["data_mtime"]) if "data_mtime" in meta else sentinel, + data_json, + meta.get("suppressed", []), + meta.get("options"), + meta.get("dep_prios", []), + meta.get("dep_lines", []), + meta.get("interface_hash", ""), + meta.get("version_id", sentinel), + meta.get("ignore_all", True), + meta.get("plugin_data", None), + ) + + +# Priorities used for imports. (Here, top-level includes inside a class.) +# These are used to determine a more predictable order in which the +# nodes in an import cycle are processed. +PRI_HIGH: Final = 5 # top-level "from X import blah" +PRI_MED: Final = 10 # top-level "import X" +PRI_LOW: Final = 20 # either form inside a function +PRI_MYPY: Final = 25 # inside "if MYPY" or "if TYPE_CHECKING" +PRI_INDIRECT: Final = 30 # an indirect dependency +PRI_ALL: Final = 99 # include all priorities + + +def import_priority(imp: ImportBase, toplevel_priority: int) -> int: + """Compute import priority from an import node.""" + if not imp.is_top_level: + # Inside a function + return PRI_LOW + if imp.is_mypy_only: + # Inside "if MYPY" or "if typing.TYPE_CHECKING" + return max(PRI_MYPY, toplevel_priority) + # A regular import; priority determined by argument. + return toplevel_priority + + +def load_plugins_from_config( + options: Options, errors: Errors, stdout: TextIO +) -> tuple[list[Plugin], dict[str, str]]: + """Load all configured plugins. + + Return a list of all the loaded plugins from the config file. + The second return value is a snapshot of versions/hashes of loaded user + plugins (for cache validation). + """ + import importlib + + snapshot: dict[str, str] = {} + + if not options.config_file: + return [], snapshot + + line = find_config_file_line_number(options.config_file, "mypy", "plugins") + if line == -1: + line = 1 # We need to pick some line number that doesn't look too confusing + + def plugin_error(message: str) -> NoReturn: + errors.report(line, 0, message) + errors.raise_error(use_stdout=False) + + custom_plugins: list[Plugin] = [] + errors.set_file(options.config_file, None, options) + for plugin_path in options.plugins: + func_name = "plugin" + plugin_dir: str | None = None + if ":" in os.path.basename(plugin_path): + plugin_path, func_name = plugin_path.rsplit(":", 1) + if plugin_path.endswith(".py"): + # Plugin paths can be relative to the config file location. + plugin_path = os.path.join(os.path.dirname(options.config_file), plugin_path) + if not os.path.isfile(plugin_path): + plugin_error(f'Can\'t find plugin "{plugin_path}"') + # Use an absolute path to avoid populating the cache entry + # for 'tmp' during tests, since it will be different in + # different tests. + plugin_dir = os.path.abspath(os.path.dirname(plugin_path)) + fnam = os.path.basename(plugin_path) + module_name = fnam[:-3] + sys.path.insert(0, plugin_dir) + elif re.search(r"[\\/]", plugin_path): + fnam = os.path.basename(plugin_path) + plugin_error(f'Plugin "{fnam}" does not have a .py extension') + else: + module_name = plugin_path + try: + module = importlib.import_module(module_name) + except Exception as exc: + plugin_error(f'Error importing plugin "{plugin_path}": {exc}') + finally: + if plugin_dir is not None: + assert sys.path[0] == plugin_dir + del sys.path[0] + + if not hasattr(module, func_name): + plugin_error( + 'Plugin "{}" does not define entry point function "{}"'.format( + plugin_path, func_name + ) + ) -def earlier_state(s: int, t: int) -> bool: - return s < t + try: + plugin_type = getattr(module, func_name)(__version__) + except Exception: + print(f"Error calling the plugin(version) entry point of {plugin_path}\n", file=stdout) + raise # Propagate to display traceback + + if not isinstance(plugin_type, type): + plugin_error( + 'Type object expected as the return value of "plugin"; got {!r} (in {})'.format( + plugin_type, plugin_path + ) + ) + if not issubclass(plugin_type, Plugin): + plugin_error( + 'Return value of "plugin" must be a subclass of "mypy.plugin.Plugin" ' + "(in {})".format(plugin_path) + ) + try: + custom_plugins.append(plugin_type(options)) + snapshot[module_name] = take_module_snapshot(module) + except Exception: + print(f"Error constructing plugin instance of {plugin_type.__name__}\n", file=stdout) + raise # Propagate to display traceback + return custom_plugins, snapshot -class BuildResult: - """The result of a successful build. - Attributes: - files: Dictionary from module name to related AST node. - types: Dictionary from parse tree node to its inferred type. +def load_plugins( + options: Options, errors: Errors, stdout: TextIO, extra_plugins: Sequence[Plugin] +) -> tuple[Plugin, dict[str, str]]: + """Load all configured plugins. + + Return a plugin that encapsulates all plugins chained together. Always + at least include the default plugin (it's last in the chain). + The second return value is a snapshot of versions/hashes of loaded user + plugins (for cache validation). """ + custom_plugins, snapshot = load_plugins_from_config(options, errors, stdout) - def __init__(self, files: Dict[str, MypyFile], - types: Dict[Node, Type]) -> None: - self.files = files - self.types = types - - -def build(program_path: str, - target: int, - module: str = None, - program_text: str = None, - alt_lib_path: str = None, - bin_dir: str = None, - output_dir: str = None, - pyversion: int = 3, - custom_typing_module: str = None, - html_report_dir: str = None, - flags: List[str] = None, - python_path: bool = False) -> BuildResult: - """Build a mypy program. + custom_plugins += extra_plugins - A single call to build performs parsing, semantic analysis and optionally - type checking and other build passes for the program *and* all imported - modules, recursively. - - Return BuildResult if successful; otherwise raise CompileError. - - Arguments: - program_path: the path to the main source file (if module argument is - given, this can be None => will be looked up) - target: select passes to perform (a build target constant, e.g. C) - Optional arguments: - module: name of the initial module; __main__ by default - program_text: the main source file contents; if omitted, read from file - alt_lib_dir: an additional directory for looking up library modules - (takes precedence over other directories) - bin_dir: directory containing the mypy script, used for finding data - directories; if omitted, use '.' as the data directory - output_dir: directory where the output (Python) is stored - pyversion: Python version (2 for 2.x or 3 for 3.x) - custom_typing_module: if not None, use this module id as an alias for typing - flags: list of build options (e.g. COMPILE_ONLY) - """ - flags = flags or [] - module = module or '__main__' - - data_dir = default_data_dir(bin_dir) - - # Determine the default module search path. - lib_path = default_lib_path(data_dir, target, pyversion, python_path) - - if TEST_BUILTINS in flags: - # Use stub builtins (to speed up test cases and to make them easier to - # debug). - lib_path.insert(0, os.path.join('mypy', 'test', 'data', 'lib-stub')) - elif program_path: - # Include directory of the program file in the module search path. - lib_path.insert( - 0, remove_cwd_prefix_from_path(dirname(program_path))) - else: - # Building/running a module. - lib_path.insert(0, os.getcwd()) + default_plugin: Plugin = DefaultPlugin(options) + if not custom_plugins: + return default_plugin, snapshot - # If provided, insert the caller-supplied extra module path to the - # beginning (highest priority) of the search path. - if alt_lib_path: - lib_path.insert(0, alt_lib_path) + # Custom plugins take precedence over the default plugin. + return ChainedPlugin(options, custom_plugins + [default_plugin]), snapshot - # Construct a build manager object that performs all the stages of the - # build in the correct order. - # - # Ignore current directory prefix in error messages. - manager = BuildManager(data_dir, lib_path, target, output_dir, - pyversion=pyversion, flags=flags, - ignore_prefix=os.getcwd(), - custom_typing_module=custom_typing_module, - html_report_dir=html_report_dir) - - program_path = program_path or lookup_program(module, lib_path) - if program_text is None: - program_text = read_program(program_path) - - # Construct information that describes the initial file. __main__ is the - # implicit module id and the import context is empty initially ([]). - info = StateInfo(program_path, module, [], manager) - # Perform the build by sending the file as new file (UnprocessedFile is the - # initial state of all files) to the manager. The manager will process the - # file and all dependant modules recursively. - result = manager.process(UnprocessedFile(info, program_text)) - if 'html-report' in flags: - stats.generate_html_index(html_report_dir) - return result - - -def default_data_dir(bin_dir: str) -> str: - if not bin_dir: - # Default to current directory. - return '' - base = os.path.basename(bin_dir) - dir = os.path.dirname(bin_dir) - if (sys.platform == 'win32' and base.lower() == 'scripts' - and not os.path.isdir(os.path.join(dir, 'stubs'))): - # Installed, on Windows. - return os.path.join(dir, 'Lib', 'mypy') - elif base == 'scripts': - # Assume that we have a repo check out or unpacked source tarball. - return os.path.dirname(bin_dir) - elif base == 'bin': - # Installed to somewhere (can be under /usr/local or anywhere). - return os.path.join(dir, 'lib', 'mypy') - elif base == 'python3': - # Assume we installed python3 with brew on os x - return os.path.join(os.path.dirname(dir), 'lib', 'mypy') - else: - # Don't know where to find the data files! - raise RuntimeError("Broken installation: can't determine base dir") - - -def default_lib_path(data_dir: str, target: int, pyversion: int, - python_path:bool) -> List[str]: - """Return default standard library search paths.""" - # IDEA: Make this more portable. - path = List[str]() - - # Add MYPYPATH environment variable to library path, if defined. - path_env = os.getenv('MYPYPATH') - if path_env is not None: - path[:0] = path_env.split(os.pathsep) - - # Add library stubs directory. By convention, they are stored in the - # stubs/x.y directory of the mypy installation. - version_dir = '3.2' - if pyversion < 3: - version_dir = '2.7' - path.append(os.path.join(data_dir, 'stubs', version_dir)) - path.append(os.path.join(data_dir, 'stubs-auto', version_dir)) - #Add py3.3 and 3.4 stubs - if sys.version_info.major == 3: - versions = ['3.' + str(x) for x in range(3, sys.version_info.minor + 1)] - for v in versions: - path.append(os.path.join(data_dir, 'stubs', v)) - - # Add fallback path that can be used if we have a broken installation. - if sys.platform != 'win32': - path.append('/usr/local/lib/mypy') - - # Contents of Python's sys.path go last, to prefer the stubs - if python_path: - path.extend(sys.path) - - return path - - -def lookup_program(module: str, lib_path: List[str]) -> str: - # Modules are .py and not .pyi - path = find_module(module, lib_path) - if path: - return path + +def take_module_snapshot(module: types.ModuleType) -> str: + """Take plugin module snapshot by recording its version and hash. + + We record _both_ hash and the version to detect more possible changes + (e.g. if there is a change in modules imported by a plugin). + """ + if hasattr(module, "__file__"): + assert module.__file__ is not None + with open(module.__file__, "rb") as f: + digest = hash_digest(f.read()) else: - raise CompileError([ - "mypy: can't find module '{}'".format(module)]) + digest = "unknown" + ver = getattr(module, "__version__", "none") + return f"{ver}:{digest}" + +def find_config_file_line_number(path: str, section: str, setting_name: str) -> int: + """Return the approximate location of setting_name within mypy config file. -def read_program(path: str) -> str: + Return -1 if can't determine the line unambiguously. + """ + in_desired_section = False try: - f = open(path) - text = f.read() - f.close() - except IOError as ioerr: - raise CompileError([ - "mypy: can't read file '{}': {}".format(path, ioerr.strerror)]) - return text + results = [] + with open(path, encoding="UTF-8") as f: + for i, line in enumerate(f): + line = line.strip() + if line.startswith("[") and line.endswith("]"): + current_section = line[1:-1].strip() + in_desired_section = current_section == section + elif in_desired_section and re.match(rf"{setting_name}\s*=", line): + results.append(i + 1) + if len(results) == 1: + return results[0] + except OSError: + pass + return -1 class BuildManager: - """This is the central class for building a mypy program. + """This class holds shared state for building a mypy program. - It coordinates parsing, import processing, semantic analysis and - type checking. It manages state objects that actually perform the - build steps. + It is used to coordinate parsing, import processing, semantic + analysis and type checking. The actual build steps are carried + out by dispatch(). Attributes: data_dir: Mypy data directory (contains stubs) - target: Build target; selects which passes to perform - lib_path: Library path for looking up modules + search_paths: SearchPaths instance indicating where to look for modules + modules: Mapping of module ID to MypyFile (shared by the passes) semantic_analyzer: Semantic analyzer, pass 2 - semantic_analyzer_pass3: - Semantic analyzer, pass 3 - type_checker: Type checker - errors: Used for reporting all errors - output_dir: Store output files here (Python) - pyversion: Python version (2 or 3) - flags: Build options - states: States of all individual files that are being - processed. Each file in a build is always represented - by a single state object (after it has been encountered - for the first time). This is the only place where - states are stored. - module_files: Map from module name to source file path. There is a - 1:1 mapping between modules and source files. - module_deps: Cache for module dependencies (direct or indirect). - Item (m, n) indicates whether m depends on n (directly - or indirectly). + all_types: Map {Expression: Type} from all modules (enabled by export_types) + options: Build options missing_modules: Set of modules that could not be imported encountered so far + stale_modules: Set of modules that needed to be rechecked (only used by tests) + fg_deps_meta: Metadata for fine-grained dependencies caches associated with modules + fg_deps: A fine-grained dependency map + version_id: The current mypy version (based on commit id when possible) + plugin: Active mypy plugin(s) + plugins_snapshot: + Snapshot of currently active user plugins (versions and hashes) + old_plugins_snapshot: + Plugins snapshot from previous incremental run (or None in + non-incremental mode and if cache was not found) + errors: Used for reporting all errors + flush_errors: A function for processing errors after each SCC + cache_enabled: Whether cache is being read. This is set based on options, + but is disabled if fine-grained cache loading fails + and after an initial fine-grained load. This doesn't + determine whether we write cache files or not. + quickstart_state: + A cache of filename -> mtime/size/hash info used to avoid + needing to hash source files when using a cache with mismatching mtimes + stats: Dict with various instrumentation numbers, it is used + not only for debugging, but also required for correctness, + in particular to check consistency of the fine-grained dependency cache. + fscache: A file system cacher + ast_cache: AST cache to speed up mypy daemon """ - def __init__(self, data_dir: str, - lib_path: List[str], - target: int, - output_dir: str, - pyversion: int, - flags: List[str], - ignore_prefix: str, - custom_typing_module: str, - html_report_dir: str) -> None: + def __init__( + self, + data_dir: str, + search_paths: SearchPaths, + ignore_prefix: str, + source_set: BuildSourceSet, + reports: Reports | None, + options: Options, + version_id: str, + plugin: Plugin, + plugins_snapshot: dict[str, str], + errors: Errors, + flush_errors: Callable[[str | None, list[str], bool], None], + fscache: FileSystemCache, + stdout: TextIO, + stderr: TextIO, + error_formatter: ErrorFormatter | None = None, + ) -> None: + self.stats: dict[str, Any] = {} # Values are ints or floats + self.stdout = stdout + self.stderr = stderr + self.start_time = time.time() self.data_dir = data_dir - self.errors = Errors() + self.errors = errors self.errors.set_ignore_prefix(ignore_prefix) - self.lib_path = lib_path - self.target = target - self.output_dir = output_dir - self.pyversion = pyversion - self.flags = flags - self.custom_typing_module = custom_typing_module - self.html_report_dir = html_report_dir - self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, - pyversion=pyversion) - self.semantic_analyzer_pass3 = ThirdPass(self.errors) - self.type_checker = TypeChecker(self.errors, - self.semantic_analyzer.modules, - self.pyversion) - self.states = List[State]() - self.module_files = Dict[str, str]() - self.module_deps = Dict[Tuple[str, str], bool]() - self.missing_modules = Set[str]() - - def process(self, initial_state: 'UnprocessedFile') -> BuildResult: - """Perform a build. - - The argument is a state that represents the main program - file. This method should only be called once per a build - manager object. The return values are identical to the return - values of the build function. - """ - self.states.append(initial_state) - - # Process states in a loop until all files (states) have been - # semantically analyzed or type checked (depending on target). - # - # We type check all files before the rest of the passes so that we can - # report errors and fail as quickly as possible. - while True: - # Find the next state that has all its dependencies met. - next = self.next_available_state() - if not next: - trace('done') - break - - # Potentially output some debug information. - trace('next {} ({})'.format(next.path, next.state())) - - # Set the import context for reporting error messages correctly. - self.errors.set_import_context(next.import_context) - # Process the state. The process method is reponsible for adding a - # new state object representing the new state of the file. - next.process() - - # Raise exception if the build failed. The build can fail for - # various reasons, such as parse error, semantic analysis error, - # etc. - if self.errors.is_blockers(): - self.errors.raise_error() - - # If there were no errors, all files should have been fully processed. - for s in self.states: - assert s.state() == final_state, ( - '{} still unprocessed in state {}'.format(s.path, s.state())) - - if self.errors.is_errors(): - self.errors.raise_error() + self.error_formatter = error_formatter + self.search_paths = search_paths + self.source_set = source_set + self.reports = reports + self.options = options + self.version_id = version_id + self.modules: dict[str, MypyFile] = {} + self.missing_modules: set[str] = set() + self.fg_deps_meta: dict[str, FgDepMeta] = {} + # fg_deps holds the dependencies of every module that has been + # processed. We store this in BuildManager so that we can compute + # dependencies as we go, which allows us to free ASTs and type information, + # saving a ton of memory on net. + self.fg_deps: dict[str, set[str]] = {} + # Always convert the plugin to a ChainedPlugin so that it can be manipulated if needed + if not isinstance(plugin, ChainedPlugin): + plugin = ChainedPlugin(options, [plugin]) + self.plugin = plugin + # Set of namespaces (module or class) that are being populated during semantic + # analysis and may have missing definitions. + self.incomplete_namespaces: set[str] = set() + self.semantic_analyzer = SemanticAnalyzer( + self.modules, + self.missing_modules, + self.incomplete_namespaces, + self.errors, + self.plugin, + ) + self.all_types: dict[Expression, Type] = {} # Enabled by export_types + self.indirection_detector = TypeIndirectionVisitor() + self.stale_modules: set[str] = set() + self.rechecked_modules: set[str] = set() + self.flush_errors = flush_errors + has_reporters = reports is not None and reports.reporters + self.cache_enabled = ( + options.incremental + and (not options.fine_grained_incremental or options.use_fine_grained_cache) + and not has_reporters + ) + self.fscache = fscache + self.find_module_cache = FindModuleCache( + self.search_paths, self.fscache, self.options, source_set=self.source_set + ) + for module in CORE_BUILTIN_MODULES: + if options.use_builtins_fixtures: + continue + path = self.find_module_cache.find_module(module, fast_path=True) + if not isinstance(path, str): + raise CompileError( + [f"Failed to find builtin module {module}, perhaps typeshed is broken?"] + ) + if is_typeshed_file(options.abs_custom_typeshed_dir, path) or is_stub_package_file( + path + ): + continue - # Collect a list of all files. - trees = List[MypyFile]() - for state in self.states: - trees.append((cast('ParsedFile', state)).tree) - - # Perform any additional passes after type checking for all the files. - self.final_passes(trees, self.type_checker.type_map) - - return BuildResult(self.semantic_analyzer.modules, - self.type_checker.type_map) - - def next_available_state(self) -> 'State': - """Find a ready state (one that has all its dependencies met).""" - i = len(self.states) - 1 - while i >= 0: - if self.states[i].is_ready(): - num_incomplete = self.states[i].num_incomplete_deps() - if num_incomplete == 0: - # This is perfect; no need to look for the best match. - return self.states[i] - i -= 1 - return None + raise CompileError( + [ + f'mypy: "{os.path.relpath(path)}" shadows library module "{module}"', + f'note: A user-defined top-level module with name "{module}" is not supported', + ] + ) + + self.metastore = create_metastore(options) + + # a mapping from source files to their corresponding shadow files + # for efficient lookup + self.shadow_map: dict[str, str] = {} + if self.options.shadow_file is not None: + self.shadow_map = dict(self.options.shadow_file) + # a mapping from each file being typechecked to its possible shadow file + self.shadow_equivalence_map: dict[str, str | None] = {} + self.plugin = plugin + self.plugins_snapshot = plugins_snapshot + self.old_plugins_snapshot = read_plugins_snapshot(self) + self.quickstart_state = read_quickstart_file(options, self.stdout) + # Fine grained targets (module top levels and top level functions) processed by + # the semantic analyzer, used only for testing. Currently used only by the new + # semantic analyzer. Tuple of module and target name. + self.processed_targets: list[tuple[str, str]] = [] + # Missing stub packages encountered. + self.missing_stub_packages: set[str] = set() + # Cache for mypy ASTs that have completed semantic analysis + # pass 1. When multiple files are added to the build in a + # single daemon increment, only one of the files gets added + # per step and the others are discarded. This gets repeated + # until all the files have been added. This means that a + # new file can be processed O(n**2) times. This cache + # avoids most of this redundant work. + self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {} + + def dump_stats(self) -> None: + if self.options.dump_build_stats: + print("Stats:") + for key, value in sorted(self.stats_summary().items()): + print(f"{key + ':':24}{value}") + + def use_fine_grained_cache(self) -> bool: + return self.cache_enabled and self.options.use_fine_grained_cache + + def maybe_swap_for_shadow_path(self, path: str) -> str: + if not self.shadow_map: + return path + + path = normpath(path, self.options) + + previously_checked = path in self.shadow_equivalence_map + if not previously_checked: + for source, shadow in self.shadow_map.items(): + if self.fscache.samefile(path, source): + self.shadow_equivalence_map[path] = shadow + break + else: + self.shadow_equivalence_map[path] = None - def has_module(self, name: str) -> bool: - """Have we seen a module yet?""" - return name in self.module_files + shadow_file = self.shadow_equivalence_map.get(path) + return shadow_file if shadow_file else path - def file_state(self, path: str) -> int: - """Return the state of a source file. + def get_stat(self, path: str) -> os.stat_result | None: + return self.fscache.stat_or_none(self.maybe_swap_for_shadow_path(path)) - In particular, return UNSEEN_STATE if the file has no associated - state. + def getmtime(self, path: str) -> int: + """Return a file's mtime; but 0 in bazel mode. - This function does not consider any dependencies. + (Bazel's distributed cache doesn't like filesystem metadata to + end up in output files.) """ - for s in self.states: - if s.path == path: - return s.state() - return UNSEEN_STATE + if self.options.bazel: + return 0 + else: + return int(self.metastore.getmtime(path)) - def module_state(self, name: str) -> int: - """Return the state of a module. + def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, int]]: + """Find all reachable import statements in a file. - In particular, return UNSEEN_STATE if the file has no associated - state. + Return list of tuples (priority, module id, import line number) + for all modules imported in file; lower numbers == higher priority. - This considers also module dependencies. + Can generate blocking errors on bogus relative imports. """ - if not self.has_module(name): - return UNSEEN_STATE - state = final_state - fs = self.file_state(self.module_files[name]) - if earlier_state(fs, state): - state = fs - return state - - def is_dep(self, m1: str, m2: str, done: Set[str] = None) -> bool: - """Does m1 import m2 directly or indirectly?""" - # Have we computed this previously? - dep = self.module_deps.get((m1, m2)) - if dep is not None: - return dep - - if not done: - done = set([m1]) - - # m1 depends on m2 iff one of the deps of m1 depends on m2. - st = self.lookup_state(m1) - for m in st.dependencies: - if m in done: - continue - done.add(m) - # Cache this dependency. - self.module_deps[m1, m] = True - # Search recursively. - if m == m2 or self.is_dep(m, m2, done): - # Yes! Mark it in the cache. - self.module_deps[m1, m2] = True - return True - # No dependency. Mark it in the cache. - self.module_deps[m1, m2] = False - return False - - def lookup_state(self, module: str) -> 'State': - for state in self.states: - if state.id == module: - return state - raise RuntimeError('%s not found' % str) - def all_imported_modules_in_file(self, - file: MypyFile) -> List[Tuple[str, int]]: - """Find all reachable import statements in a file. - - Return list of tuples (module id, import line number) for all modules - imported in file. - """ - def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: + def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: """Function to correct for relative imports.""" - file_id = file.fullname() + file_id = file.fullname rel = imp.relative if rel == 0: return imp.id - if os.path.basename(file.path) == '__init__.py': + if os.path.basename(file.path).startswith("__init__."): rel -= 1 if rel != 0: file_id = ".".join(file_id.split(".")[:-rel]) new_id = file_id + "." + imp.id if imp.id else file_id + if not new_id: + self.errors.set_file(file.path, file.name, self.options) + self.errors.report( + imp.line, 0, "No parent module -- cannot perform relative import", blocker=True + ) + return new_id - res = List[Tuple[str, int]]() + res: list[tuple[int, str, int]] = [] for imp in file.imports: if not imp.is_unreachable: if isinstance(imp, Import): + pri = import_priority(imp, PRI_MED) + ancestor_pri = import_priority(imp, PRI_LOW) for id, _ in imp.ids: - res.append((id, imp.line)) + res.append((pri, id, imp.line)) + ancestor_parts = id.split(".")[:-1] + ancestors = [] + for part in ancestor_parts: + ancestors.append(part) + res.append((ancestor_pri, ".".join(ancestors), imp.line)) elif isinstance(imp, ImportFrom): cur_id = correct_rel_imp(imp) - res.append((cur_id, imp.line)) + all_are_submodules = True # Also add any imported names that are submodules. + pri = import_priority(imp, PRI_MED) for name, __ in imp.names: - sub_id = cur_id + '.' + name + sub_id = cur_id + "." + name if self.is_module(sub_id): - res.append((sub_id, imp.line)) + res.append((pri, sub_id, imp.line)) + else: + all_are_submodules = False + # Add cur_id as a dependency, even if all of the + # imports are submodules. Processing import from will try + # to look through cur_id, so we should depend on it. + # As a workaround for for some bugs in cycle handling (#4498), + # if all of the imports are submodules, do the import at a lower + # priority. + pri = import_priority(imp, PRI_HIGH if not all_are_submodules else PRI_LOW) + res.append((pri, cur_id, imp.line)) elif isinstance(imp, ImportAll): - res.append((correct_rel_imp(imp), imp.line)) + pri = import_priority(imp, PRI_HIGH) + res.append((pri, correct_rel_imp(imp), imp.line)) + + # Sort such that module (e.g. foo.bar.baz) comes before its ancestors (e.g. foo + # and foo.bar) so that, if FindModuleCache finds the target module in a + # package marked with py.typed underneath a namespace package installed in + # site-packages, (gasp), that cache's knowledge of the ancestors + # (aka FindModuleCache.ns_ancestors) can be primed when it is asked to find + # the parent. + res.sort(key=lambda x: -x[1].count(".")) return res def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" - return find_module(id, self.lib_path) is not None + return find_module_simple(id, self) is not None - def final_passes(self, files: List[MypyFile], - types: Dict[Node, Type]) -> None: - """Perform the code generation passes for type checked files.""" - if self.target in [SEMANTIC_ANALYSIS, TYPE_CHECK]: - pass # Nothing to do. - else: - raise RuntimeError('Unsupported target %d' % self.target) + def parse_file( + self, id: str, path: str, source: str, ignore_errors: bool, options: Options + ) -> MypyFile: + """Parse the source of a file with the given name. + + Raise CompileError if there is a parse error. + """ + t0 = time.time() + if ignore_errors: + self.errors.ignored_files.add(path) + tree = parse(source, path, id, self.errors, options=options) + tree._fullname = id + self.add_stats( + files_parsed=1, + modules_parsed=int(not tree.is_stub), + stubs_parsed=int(tree.is_stub), + parse_time=time.time() - t0, + ) + + if self.errors.is_blockers(): + self.log("Bailing due to parse errors") + self.errors.raise_error() - def get_python_out_path(self, f: MypyFile) -> str: - if f.fullname() == '__main__': - return os.path.join(self.output_dir, basename(f.path)) + self.errors.set_file_ignored_lines(path, tree.ignored_lines, ignore_errors) + return tree + + def load_fine_grained_deps(self, id: str) -> dict[str, set[str]]: + t0 = time.time() + if id in self.fg_deps_meta: + # TODO: Assert deps file wasn't changed. + deps = json_loads(self.metastore.read(self.fg_deps_meta[id]["path"])) else: - components = f.fullname().split('.') - if os.path.basename(f.path) == '__init__.py': - components.append('__init__.py') + deps = {} + val = {k: set(v) for k, v in deps.items()} + self.add_stats(load_fg_deps_time=time.time() - t0) + return val + + def report_file( + self, file: MypyFile, type_map: dict[Expression, Type], options: Options + ) -> None: + if self.reports is not None and self.source_set.is_source(file): + self.reports.file(file, self.modules, type_map, options) + + def verbosity(self) -> int: + return self.options.verbosity + + def log(self, *message: str) -> None: + if self.verbosity() >= 1: + if message: + print("LOG: ", *message, file=self.stderr) + else: + print(file=self.stderr) + self.stderr.flush() + + def log_fine_grained(self, *message: str) -> None: + import mypy.build + + if self.verbosity() >= 1: + self.log("fine-grained:", *message) + elif mypy.build.DEBUG_FINE_GRAINED: + # Output log in a simplified format that is quick to browse. + if message: + print(*message, file=self.stderr) + else: + print(file=self.stderr) + self.stderr.flush() + + def trace(self, *message: str) -> None: + if self.verbosity() >= 2: + print("TRACE:", *message, file=self.stderr) + self.stderr.flush() + + def add_stats(self, **kwds: Any) -> None: + for key, value in kwds.items(): + if key in self.stats: + self.stats[key] += value else: - components[-1] += '.py' - return os.path.join(self.output_dir, *components) + self.stats[key] = value + + def stats_summary(self) -> Mapping[str, object]: + return self.stats + + +def deps_to_json(x: dict[str, set[str]]) -> bytes: + return json_dumps({k: list(v) for k, v in x.items()}) + + +# File for storing metadata about all the fine-grained dependency caches +DEPS_META_FILE: Final = "@deps.meta.json" +# File for storing fine-grained dependencies that didn't a parent in the build +DEPS_ROOT_FILE: Final = "@root.deps.json" + +# The name of the fake module used to store fine-grained dependencies that +# have no other place to go. +FAKE_ROOT_MODULE: Final = "@root" + + +def write_deps_cache( + rdeps: dict[str, dict[str, set[str]]], manager: BuildManager, graph: Graph +) -> None: + """Write cache files for fine-grained dependencies. + + Serialize fine-grained dependencies map for fine grained mode. + + Dependencies on some module 'm' is stored in the dependency cache + file m.deps.json. This entails some spooky action at a distance: + if module 'n' depends on 'm', that produces entries in m.deps.json. + When there is a dependency on a module that does not exist in the + build, it is stored with its first existing parent module. If no + such module exists, it is stored with the fake module FAKE_ROOT_MODULE. + + This means that the validity of the fine-grained dependency caches + are a global property, so we store validity checking information for + fine-grained dependencies in a global cache file: + * We take a snapshot of current sources to later check consistency + between the fine-grained dependency cache and module cache metadata + * We store the mtime of all of the dependency files to verify they + haven't changed + """ + metastore = manager.metastore - def log(self, message: str) -> None: - if VERBOSE in self.flags: - print('LOG: %s' % message) + error = False + fg_deps_meta = manager.fg_deps_meta.copy() -def remove_cwd_prefix_from_path(p: str) -> str: - """Remove current working directory prefix from p, if present. + for id in rdeps: + if id != FAKE_ROOT_MODULE: + _, _, deps_json = get_cache_names(id, graph[id].xpath, manager.options) + else: + deps_json = DEPS_ROOT_FILE + assert deps_json + manager.log("Writing deps cache", deps_json) + if not manager.metastore.write(deps_json, deps_to_json(rdeps[id])): + manager.log(f"Error writing fine-grained deps JSON file {deps_json}") + error = True + else: + fg_deps_meta[id] = {"path": deps_json, "mtime": manager.getmtime(deps_json)} + + meta_snapshot: dict[str, str] = {} + for id, st in graph.items(): + # If we didn't parse a file (so it doesn't have a + # source_hash), then it must be a module with a fresh cache, + # so use the hash from that. + if st.source_hash: + hash = st.source_hash + else: + assert st.meta, "Module must be either parsed or cached" + hash = st.meta.hash + meta_snapshot[id] = hash + + meta = {"snapshot": meta_snapshot, "deps_meta": fg_deps_meta} + + if not metastore.write(DEPS_META_FILE, json_dumps(meta)): + manager.log(f"Error writing fine-grained deps meta JSON file {DEPS_META_FILE}") + error = True + + if error: + manager.errors.set_file(_cache_dir_prefix(manager.options), None, manager.options) + manager.errors.report(0, 0, "Error writing fine-grained dependencies cache", blocker=True) - If the result would be empty, return '.' instead. + +def invert_deps(deps: dict[str, set[str]], graph: Graph) -> dict[str, dict[str, set[str]]]: + """Splits fine-grained dependencies based on the module of the trigger. + + Returns a dictionary from module ids to all dependencies on that + module. Dependencies not associated with a module in the build will be + associated with the nearest parent module that is in the build, or the + fake module FAKE_ROOT_MODULE if none are. """ - cur = os.getcwd() - # Add separator to the end of the path, unless one is already present. - if basename(cur) != '': - cur += os.sep - # Remove current directory prefix from the path, if present. - if p.startswith(cur): - p = p[len(cur):] - # Avoid returning an empty path; replace that with '.'. - if p == '': - p = '.' - return p - - -def is_stub(path: str) -> bool: - """Does path refer to a stubs file? - - Currently check if there is a 'stubs' directory component somewhere - in the path. + # Lazy import to speed up startup + from mypy.server.target import trigger_to_target + + # Prepopulate the map for all the modules that have been processed, + # so that we always generate files for processed modules (even if + # there aren't any dependencies to them.) + rdeps: dict[str, dict[str, set[str]]] = {id: {} for id, st in graph.items() if st.tree} + for trigger, targets in deps.items(): + module = module_prefix(graph, trigger_to_target(trigger)) + if not module or not graph[module].tree: + module = FAKE_ROOT_MODULE + + mod_rdeps = rdeps.setdefault(module, {}) + mod_rdeps.setdefault(trigger, set()).update(targets) + + return rdeps + + +def generate_deps_for_cache(manager: BuildManager, graph: Graph) -> dict[str, dict[str, set[str]]]: + """Generate fine-grained dependencies into a form suitable for serializing. + + This does a couple things: + 1. Splits fine-grained deps based on the module of the trigger + 2. For each module we generated fine-grained deps for, load any previous + deps and merge them in. + + Returns a dictionary from module ids to all dependencies on that + module. Dependencies not associated with a module in the build will be + associated with the nearest parent module that is in the build, or the + fake module FAKE_ROOT_MODULE if none are. """ - # TODO more precise check - dirname, basename = os.path.split(path) - if basename == '': - return False + from mypy.server.deps import merge_dependencies # Lazy import to speed up startup + + # Split the dependencies out into based on the module that is depended on. + rdeps = invert_deps(manager.fg_deps, graph) + + # We can't just clobber existing dependency information, so we + # load the deps for every module we've generated new dependencies + # to and merge the new deps into them. + for module, mdeps in rdeps.items(): + old_deps = manager.load_fine_grained_deps(module) + merge_dependencies(old_deps, mdeps) + + return rdeps + + +PLUGIN_SNAPSHOT_FILE: Final = "@plugins_snapshot.json" + + +def write_plugins_snapshot(manager: BuildManager) -> None: + """Write snapshot of versions and hashes of currently active plugins.""" + snapshot = json_dumps(manager.plugins_snapshot) + if ( + not manager.metastore.write(PLUGIN_SNAPSHOT_FILE, snapshot) + and manager.options.cache_dir != os.devnull + ): + manager.errors.set_file(_cache_dir_prefix(manager.options), None, manager.options) + manager.errors.report(0, 0, "Error writing plugins snapshot", blocker=True) + + +def read_plugins_snapshot(manager: BuildManager) -> dict[str, str] | None: + """Read cached snapshot of versions and hashes of plugins from previous run.""" + snapshot = _load_json_file( + PLUGIN_SNAPSHOT_FILE, + manager, + log_success="Plugins snapshot ", + log_error="Could not load plugins snapshot: ", + ) + if snapshot is None: + return None + if not isinstance(snapshot, dict): + manager.log(f"Could not load plugins snapshot: cache is not a dict: {type(snapshot)}") + return None + return snapshot + + +def read_quickstart_file( + options: Options, stdout: TextIO +) -> dict[str, tuple[float, int, str]] | None: + quickstart: dict[str, tuple[float, int, str]] | None = None + if options.quickstart_file: + # This is very "best effort". If the file is missing or malformed, + # just ignore it. + raw_quickstart: dict[str, Any] = {} + try: + with open(options.quickstart_file, "rb") as f: + raw_quickstart = json_loads(f.read()) + + quickstart = {} + for file, (x, y, z) in raw_quickstart.items(): + quickstart[file] = (x, y, z) + except Exception as e: + print(f"Warning: Failed to load quickstart file: {str(e)}\n", file=stdout) + return quickstart + + +def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta] | None: + """Read and validate the fine-grained dependencies cache. + + See the write_deps_cache documentation for more information on + the details of the cache. + + Returns None if the cache was invalid in some way. + """ + deps_meta = _load_json_file( + DEPS_META_FILE, + manager, + log_success="Deps meta ", + log_error="Could not load fine-grained dependency metadata: ", + ) + if deps_meta is None: + return None + meta_snapshot = deps_meta["snapshot"] + # Take a snapshot of the source hashes from all of the metas we found. + # (Including the ones we rejected because they were out of date.) + # We use this to verify that they match up with the proto_deps. + current_meta_snapshot = { + id: st.meta_source_hash for id, st in graph.items() if st.meta_source_hash is not None + } + + common = set(meta_snapshot.keys()) & set(current_meta_snapshot.keys()) + if any(meta_snapshot[id] != current_meta_snapshot[id] for id in common): + # TODO: invalidate also if options changed (like --strict-optional)? + manager.log("Fine-grained dependencies cache inconsistent, ignoring") + return None + + module_deps_metas = deps_meta["deps_meta"] + assert isinstance(module_deps_metas, dict) + if not manager.options.skip_cache_mtime_checks: + for meta in module_deps_metas.values(): + try: + matched = manager.getmtime(meta["path"]) == meta["mtime"] + except FileNotFoundError: + matched = False + if not matched: + manager.log(f"Invalid or missing fine-grained deps cache: {meta['path']}") + return None + + return module_deps_metas + + +def _load_json_file( + file: str, manager: BuildManager, log_success: str, log_error: str +) -> dict[str, Any] | None: + """A simple helper to read a JSON file with logging.""" + t0 = time.time() + try: + data = manager.metastore.read(file) + except OSError: + manager.log(log_error + file) + return None + manager.add_stats(metastore_read_time=time.time() - t0) + # Only bother to compute the log message if we are logging it, since it could be big + if manager.verbosity() >= 2: + manager.trace(log_success + data.rstrip().decode()) + try: + t1 = time.time() + result = json_loads(data) + manager.add_stats(data_json_load_time=time.time() - t1) + except json.JSONDecodeError: + manager.errors.set_file(file, None, manager.options) + manager.errors.report( + -1, + -1, + "Error reading JSON file;" + " you likely have a bad cache.\n" + "Try removing the {cache_dir} directory" + " and run mypy again.".format(cache_dir=manager.options.cache_dir), + blocker=True, + ) + return None else: - stubnames = ['stubs', 'stubs-auto'] - return (basename in stubnames) or is_stub(dirname) - - -class StateInfo: - """Description of a source file that is being built.""" - - def __init__(self, path: str, id: str, - import_context: List[Tuple[str, int]], - manager: BuildManager) -> None: - """Initialize state information. - - Arguments: - path: Path to the file - id: Module id, such as 'os.path' or '__main__' (for the main - program file) - import_context: - The import trail that caused this module to be - imported (path, line) tuples - manager: The manager that manages this build - """ - self.path = path - self.id = id - self.import_context = import_context - self.manager = manager + assert isinstance(result, dict) + return result -class State: - """Abstract base class for build states. +def _cache_dir_prefix(options: Options) -> str: + """Get current cache directory (or file if id is given).""" + if options.bazel: + # This is needed so the cache map works. + return os.curdir + cache_dir = options.cache_dir + pyversion = options.python_version + base = os.path.join(cache_dir, "%d.%d" % pyversion) + return base + + +def add_catch_all_gitignore(target_dir: str) -> None: + """Add catch-all .gitignore to an existing directory. - There is always at most one state per source file. + No-op if the .gitignore already exists. """ + gitignore = os.path.join(target_dir, ".gitignore") + try: + with open(gitignore, "x") as f: + print("# Automatically created by mypy", file=f) + print("*", file=f) + except FileExistsError: + pass - # The StateInfo attributes are duplicated here for convenience. - path = Undefined(str) - id = Undefined(str) - import_context = Undefined(List[Tuple[str, int]]) - manager = Undefined(BuildManager) - # Modules that this file directly depends on (in no particular order). - dependencies = Undefined(List[str]) - - def __init__(self, info: StateInfo) -> None: - self.path = info.path - self.id = info.id - self.import_context = info.import_context - self.manager = info.manager - self.dependencies = [] - def info(self) -> StateInfo: - return StateInfo(self.path, self.id, self.import_context, self.manager) +def exclude_from_backups(target_dir: str) -> None: + """Exclude the directory from various archives and backups supporting CACHEDIR.TAG. + + If the CACHEDIR.TAG file exists the function is a no-op. + """ + cachedir_tag = os.path.join(target_dir, "CACHEDIR.TAG") + try: + with open(cachedir_tag, "x") as f: + f.write( + """Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag automatically created by mypy. +# For information about cache directory tags see https://bford.info/cachedir/ +""" + ) + except FileExistsError: + pass - def process(self) -> None: - raise RuntimeError('Not implemented') - def is_ready(self) -> bool: - """Return True if all dependencies are at least in the same state - as this object (but not in the initial state). - """ - for module in self.dependencies: - state = self.manager.module_state(module) - if earlier_state(state, - self.state()) or state == UNPROCESSED_STATE: - return False - return True +def create_metastore(options: Options) -> MetadataStore: + """Create the appropriate metadata store.""" + if options.sqlite_cache: + mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options)) + else: + mds = FilesystemMetadataStore(_cache_dir_prefix(options)) + return mds - def num_incomplete_deps(self) -> int: - """Return the number of dependencies that are ready but incomplete.""" - return 0 # Does not matter in this state - def state(self) -> int: - raise RuntimeError('Not implemented') +def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str | None]: + """Return the file names for the cache files. - def switch_state(self, state_object: 'State') -> None: - """Called by state objects to replace the state of the file. + Args: + id: module ID + path: module path + cache_dir: cache directory + pyversion: Python version (major, minor) - Also notify the manager. - """ - for i in range(len(self.manager.states)): - if self.manager.states[i].path == state_object.path: - self.manager.states[i] = state_object - return - raise RuntimeError('State for {} not found'.format(state_object.path)) + Returns: + A tuple with the file names to be used for the meta JSON, the + data JSON, and the fine-grained deps JSON, respectively. + """ + if options.cache_map: + pair = options.cache_map.get(normpath(path, options)) + else: + pair = None + if pair is not None: + # The cache map paths were specified relative to the base directory, + # but the filesystem metastore APIs operates relative to the cache + # prefix directory. + # Solve this by rewriting the paths as relative to the root dir. + # This only makes sense when using the filesystem backed cache. + root = _cache_dir_prefix(options) + return (os.path.relpath(pair[0], root), os.path.relpath(pair[1], root), None) + prefix = os.path.join(*id.split(".")) + is_package = os.path.basename(path).startswith("__init__.py") + if is_package: + prefix = os.path.join(prefix, "__init__") + + deps_json = None + if options.cache_fine_grained: + deps_json = prefix + ".deps.json" + return (prefix + ".meta.json", prefix + ".data.json", deps_json) + + +def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | None: + """Find cache data for a module. + + Args: + id: module ID + path: module path + manager: the build manager (for pyversion, log/trace, and build options) + + Returns: + A CacheMeta instance if the cache data was found and appears + valid; otherwise None. + """ + # TODO: May need to take more build options into account + meta_json, data_json, _ = get_cache_names(id, path, manager.options) + manager.trace(f"Looking for {id} at {meta_json}") + t0 = time.time() + meta = _load_json_file( + meta_json, manager, log_success=f"Meta {id} ", log_error=f"Could not load cache for {id}: " + ) + t1 = time.time() + if meta is None: + return None + if not isinstance(meta, dict): + manager.log(f"Could not load cache for {id}: meta cache is not a dict: {repr(meta)}") + return None + m = cache_meta_from_dict(meta, data_json) + t2 = time.time() + manager.add_stats( + load_meta_time=t2 - t0, load_meta_load_time=t1 - t0, load_meta_from_dict_time=t2 - t1 + ) - def errors(self) -> Errors: - return self.manager.errors + # Don't check for path match, that is dealt with in validate_meta(). + # + # TODO: these `type: ignore`s wouldn't be necessary + # if the type annotations for CacheMeta were more accurate + # (all of these attributes can be `None`) + if ( + m.id != id + or m.mtime is None # type: ignore[redundant-expr] + or m.size is None # type: ignore[redundant-expr] + or m.dependencies is None # type: ignore[redundant-expr] + or m.data_mtime is None + ): + manager.log(f"Metadata abandoned for {id}: attributes are missing") + return None - def semantic_analyzer(self) -> SemanticAnalyzer: - return self.manager.semantic_analyzer + # Ignore cache if generated by an older mypy version. + if ( + (m.version_id != manager.version_id and not manager.options.skip_version_check) + or m.options is None + or len(m.dependencies) + len(m.suppressed) != len(m.dep_prios) + or len(m.dependencies) + len(m.suppressed) != len(m.dep_lines) + ): + manager.log(f"Metadata abandoned for {id}: new attributes are missing") + return None - def semantic_analyzer_pass3(self) -> ThirdPass: - return self.manager.semantic_analyzer_pass3 + # Ignore cache if (relevant) options aren't the same. + # Note that it's fine to mutilate cached_options since it's only used here. + cached_options = m.options + current_options = manager.options.clone_for_module(id).select_options_affecting_cache() + if manager.options.skip_version_check: + # When we're lax about version we're also lax about platform. + cached_options["platform"] = current_options["platform"] + if "debug_cache" in cached_options: + # Older versions included debug_cache, but it's silly to compare it. + del cached_options["debug_cache"] + if cached_options != current_options: + manager.log(f"Metadata abandoned for {id}: options differ") + if manager.options.verbosity >= 2: + for key in sorted(set(cached_options) | set(current_options)): + if cached_options.get(key) != current_options.get(key): + manager.trace( + " {}: {} != {}".format( + key, cached_options.get(key), current_options.get(key) + ) + ) + return None + if manager.old_plugins_snapshot and manager.plugins_snapshot: + # Check if plugins are still the same. + if manager.plugins_snapshot != manager.old_plugins_snapshot: + manager.log(f"Metadata abandoned for {id}: plugins differ") + return None + # So that plugins can return data with tuples in it without + # things silently always invalidating modules, we round-trip + # the config data. This isn't beautiful. + plugin_data = json_loads( + json_dumps(manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=True))) + ) + if m.plugin_data != plugin_data: + manager.log(f"Metadata abandoned for {id}: plugin configuration differs") + return None - def type_checker(self) -> TypeChecker: - return self.manager.type_checker - - def fail(self, path: str, line: int, msg: str, blocker: bool = True) -> None: - """Report an error in the build (e.g. if could not find a module).""" - self.errors().set_file(path) - self.errors().report(line, msg, blocker=blocker) - - -class UnprocessedFile(State): - def __init__(self, info: StateInfo, program_text: str) -> None: - super().__init__(info) - self.program_text = program_text - trace('waiting {}'.format(info.path)) - - # Add surrounding package(s) as dependencies. - for p in super_packages(self.id): - if not self.import_module(p): - # Could not find a module. Typically the reason is a misspelled - # module name, or the module has not been installed. - self.fail(self.path, 1, "No module named '{}'".format(p)) - self.dependencies.append(p) - - def process(self) -> None: - """Parse the file, store global names and advance to the next state.""" - tree = self.parse(self.program_text, self.path) - - # Store the parsed module in the shared module symbol table. - self.manager.semantic_analyzer.modules[self.id] = tree - - if '.' in self.id: - # Include module in the symbol table of the enclosing package. - c = self.id.split('.') - p = '.'.join(c[:-1]) - sem_anal = self.manager.semantic_analyzer - sem_anal.modules[p].names[c[-1]] = SymbolTableNode( - MODULE_REF, tree, p) - - if self.id != 'builtins': - # The builtins module is imported implicitly in every program (it - # contains definitions of int, print etc.). - trace('import builtins') - if not self.import_module('builtins'): - self.fail(self.path, 1, 'Could not find builtins') - - # Do the first pass of semantic analysis: add top-level definitions in - # the file to the symbol table. We must do this before processing imports, - # since this may mark some import statements as unreachable. - first = FirstPass(self.semantic_analyzer()) - first.analyze(tree, self.path, self.id) - - # Add all directly imported modules to be processed (however they are - # not processed yet, just waiting to be processed). - for id, line in self.manager.all_imported_modules_in_file(tree): - self.errors().push_import_context(self.path, line) + manager.add_stats(fresh_metas=1) + return m + + +def validate_meta( + meta: CacheMeta | None, id: str, path: str | None, ignore_all: bool, manager: BuildManager +) -> CacheMeta | None: + """Checks whether the cached AST of this module can be used. + + Returns: + None, if the cached AST is unusable. + Original meta, if mtime/size matched. + Meta with mtime updated to match source file, if hash/size matched but mtime/path didn't. + """ + # This requires two steps. The first one is obvious: we check that the module source file + # contents is the same as it was when the cache data file was created. The second one is not + # too obvious: we check that the cache data file mtime has not changed; it is needed because + # we use cache data file mtime to propagate information about changes in the dependencies. + + if meta is None: + manager.log(f"Metadata not found for {id}") + return None + + if meta.ignore_all and not ignore_all: + manager.log(f"Metadata abandoned for {id}: errors were previously ignored") + return None + + t0 = time.time() + bazel = manager.options.bazel + assert path is not None, "Internal error: meta was provided without a path" + if not manager.options.skip_cache_mtime_checks: + # Check data_json; assume if its mtime matches it's good. + try: + data_mtime = manager.getmtime(meta.data_json) + except OSError: + manager.log(f"Metadata abandoned for {id}: failed to stat data_json") + return None + if data_mtime != meta.data_mtime: + manager.log(f"Metadata abandoned for {id}: data cache is modified") + return None + + if bazel: + # Normalize path under bazel to make sure it isn't absolute + path = normpath(path, manager.options) + + st = manager.get_stat(path) + if st is None: + return None + if not stat.S_ISDIR(st.st_mode) and not stat.S_ISREG(st.st_mode): + manager.log(f"Metadata abandoned for {id}: file or directory {path} does not exist") + return None + + manager.add_stats(validate_stat_time=time.time() - t0) + + # When we are using a fine-grained cache, we want our initial + # build() to load all of the cache information and then do a + # fine-grained incremental update to catch anything that has + # changed since the cache was generated. We *don't* want to do a + # coarse-grained incremental rebuild, so we accept the cache + # metadata even if it doesn't match the source file. + # + # We still *do* the mtime/hash checks, however, to enable + # fine-grained mode to take advantage of the mtime-updating + # optimization when mtimes differ but hashes match. There is + # essentially no extra time cost to computing the hash here, since + # it will be cached and will be needed for finding changed files + # later anyways. + fine_grained_cache = manager.use_fine_grained_cache() + + size = st.st_size + # Bazel ensures the cache is valid. + if size != meta.size and not bazel and not fine_grained_cache: + manager.log(f"Metadata abandoned for {id}: file {path} has different size") + return None + + # Bazel ensures the cache is valid. + mtime = 0 if bazel else int(st.st_mtime) + if not bazel and (mtime != meta.mtime or path != meta.path): + if manager.quickstart_state and path in manager.quickstart_state: + # If the mtime and the size of the file recorded in the quickstart dump matches + # what we see on disk, we know (assume) that the hash matches the quickstart + # data as well. If that hash matches the hash in the metadata, then we know + # the file is up to date even though the mtime is wrong, without needing to hash it. + qmtime, qsize, qhash = manager.quickstart_state[path] + if int(qmtime) == mtime and qsize == size and qhash == meta.hash: + manager.log(f"Metadata fresh (by quickstart) for {id}: file {path}") + meta = meta._replace(mtime=mtime, path=path) + return meta + + t0 = time.time() + try: + # dir means it is a namespace package + if stat.S_ISDIR(st.st_mode): + source_hash = "" + else: + source_hash = manager.fscache.hash_digest(path) + except (OSError, UnicodeDecodeError, DecodeError): + return None + manager.add_stats(validate_hash_time=time.time() - t0) + if source_hash != meta.hash: + if fine_grained_cache: + manager.log(f"Using stale metadata for {id}: file {path}") + return meta + else: + manager.log(f"Metadata abandoned for {id}: file {path} has different hash") + return None + else: + t0 = time.time() + # Optimization: update mtime and path (otherwise, this mismatch will reappear). + meta = meta._replace(mtime=mtime, path=path) + # Construct a dict we can pass to json.dumps() (compare to write_cache()). + meta_dict = { + "id": id, + "path": path, + "mtime": mtime, + "size": size, + "hash": source_hash, + "data_mtime": meta.data_mtime, + "dependencies": meta.dependencies, + "suppressed": meta.suppressed, + "options": (manager.options.clone_for_module(id).select_options_affecting_cache()), + "dep_prios": meta.dep_prios, + "dep_lines": meta.dep_lines, + "interface_hash": meta.interface_hash, + "version_id": manager.version_id, + "ignore_all": meta.ignore_all, + "plugin_data": meta.plugin_data, + } + meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) + meta_json, _, _ = get_cache_names(id, path, manager.options) + manager.log( + "Updating mtime for {}: file {}, meta {}, mtime {}".format( + id, path, meta_json, meta.mtime + ) + ) + t1 = time.time() + manager.metastore.write(meta_json, meta_bytes) # Ignore errors, just an optimization. + manager.add_stats(validate_update_time=time.time() - t1, validate_munging_time=t1 - t0) + return meta + + # It's a match on (id, path, size, hash, mtime). + manager.log(f"Metadata fresh for {id}: file {path}") + return meta + + +def compute_hash(text: str) -> str: + # We use a crypto hash instead of the builtin hash(...) function + # because the output of hash(...) can differ between runs due to + # hash randomization (enabled by default in Python 3.3). See the + # note in + # https://docs.python.org/3/reference/datamodel.html#object.__hash__. + return hash_digest(text.encode("utf-8")) + + +def write_cache( + id: str, + path: str, + tree: MypyFile, + dependencies: list[str], + suppressed: list[str], + dep_prios: list[int], + dep_lines: list[int], + old_interface_hash: str, + source_hash: str, + ignore_all: bool, + manager: BuildManager, +) -> tuple[str, CacheMeta | None]: + """Write cache files for a module. + + Note that this mypy's behavior is still correct when any given + write_cache() call is replaced with a no-op, so error handling + code that bails without writing anything is okay. + + Args: + id: module ID + path: module path + tree: the fully checked module data + dependencies: module IDs on which this module depends + suppressed: module IDs which were suppressed as dependencies + dep_prios: priorities (parallel array to dependencies) + dep_lines: import line locations (parallel array to dependencies) + old_interface_hash: the hash from the previous version of the data cache file + source_hash: the hash of the source code + ignore_all: the ignore_all flag for this module + manager: the build manager (for pyversion, log/trace) + + Returns: + A tuple containing the interface hash and CacheMeta + corresponding to the metadata that was written (the latter may + be None if the cache could not be written). + """ + metastore = manager.metastore + # For Bazel we use relative paths and zero mtimes. + bazel = manager.options.bazel + + # Obtain file paths. + meta_json, data_json, _ = get_cache_names(id, path, manager.options) + manager.log(f"Writing {id} {path} {meta_json} {data_json}") + + # Update tree.path so that in bazel mode it's made relative (since + # sometimes paths leak out). + if bazel: + tree.path = path + + # Serialize data and analyze interface + data = tree.serialize() + data_bytes = json_dumps(data, manager.options.debug_cache) + interface_hash = hash_digest(data_bytes) + + plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) + + # Obtain and set up metadata + st = manager.get_stat(path) + if st is None: + manager.log(f"Cannot get stat for {path}") + # Remove apparently-invalid cache files. + # (This is purely an optimization.) + for filename in [data_json, meta_json]: try: - res = self.import_module(id) - finally: - self.errors().pop_import_context() - if not res: - if id == '': - # Must be from a relative import. - self.fail(self.path, line, - "No parent module -- cannot perform relative import".format(id), - blocker=True) - else: - self.fail(self.path, line, "No module named '{}'".format(id), blocker=False) - self.manager.missing_modules.add(id) + os.remove(filename) + except OSError: + pass + # Still return the interface hash we computed. + return interface_hash, None + + # Write data cache file, if applicable + # Note that for Bazel we don't record the data file's mtime. + if old_interface_hash == interface_hash: + manager.trace(f"Interface for {id} is unchanged") + else: + manager.trace(f"Interface for {id} has changed") + if not metastore.write(data_json, data_bytes): + # Most likely the error is the replace() call + # (see https://github.com/python/mypy/issues/3215). + manager.log(f"Error writing data JSON file {data_json}") + # Let's continue without writing the meta file. Analysis: + # If the replace failed, we've changed nothing except left + # behind an extraneous temporary file; if the replace + # worked but the getmtime() call failed, the meta file + # will be considered invalid on the next run because the + # data_mtime field won't match the data file's mtime. + # Both have the effect of slowing down the next run a + # little bit due to an out-of-date cache file. + return interface_hash, None + + try: + data_mtime = manager.getmtime(data_json) + except OSError: + manager.log(f"Error in os.stat({data_json!r}), skipping cache write") + return interface_hash, None + + mtime = 0 if bazel else int(st.st_mtime) + size = st.st_size + # Note that the options we store in the cache are the options as + # specified by the command line/config file and *don't* reflect + # updates made by inline config directives in the file. This is + # important, or otherwise the options would never match when + # verifying the cache. + options = manager.options.clone_for_module(id) + assert source_hash is not None + meta = { + "id": id, + "path": path, + "mtime": mtime, + "size": size, + "hash": source_hash, + "data_mtime": data_mtime, + "dependencies": dependencies, + "suppressed": suppressed, + "options": options.select_options_affecting_cache(), + "dep_prios": dep_prios, + "dep_lines": dep_lines, + "interface_hash": interface_hash, + "version_id": manager.version_id, + "ignore_all": ignore_all, + "plugin_data": plugin_data, + } + + # Write meta cache file + meta_str = json_dumps(meta, manager.options.debug_cache) + if not metastore.write(meta_json, meta_str): + # Most likely the error is the replace() call + # (see https://github.com/python/mypy/issues/3215). + # The next run will simply find the cache entry out of date. + manager.log(f"Error writing meta JSON file {meta_json}") + + return interface_hash, cache_meta_from_dict(meta, data_json) + + +def delete_cache(id: str, path: str, manager: BuildManager) -> None: + """Delete cache files for a module. + + The cache files for a module are deleted when mypy finds errors there. + This avoids inconsistent states with cache files from different mypy runs, + see #4043 for an example. + """ + # We don't delete .deps files on errors, since the dependencies + # are mostly generated from other files and the metadata is + # tracked separately. + meta_path, data_path, _ = get_cache_names(id, path, manager.options) + cache_paths = [meta_path, data_path] + manager.log(f"Deleting {id} {path} {' '.join(x for x in cache_paths if x)}") + + for filename in cache_paths: + try: + manager.metastore.remove(filename) + except OSError as e: + if e.errno != errno.ENOENT: + manager.log(f"Error deleting cache file {filename}: {e.strerror}") + + +"""Dependency manager. + +Design +====== + +Ideally +------- + +A. Collapse cycles (each SCC -- strongly connected component -- + becomes one "supernode"). + +B. Topologically sort nodes based on dependencies. + +C. Process from leaves towards roots. + +Wrinkles +-------- + +a. Need to parse source modules to determine dependencies. + +b. Processing order for modules within an SCC. + +c. Must order mtimes of files to decide whether to re-process; depends + on clock never resetting. + +d. from P import M; checks filesystem whether module P.M exists in + filesystem. + +e. Race conditions, where somebody modifies a file while we're + processing. Solved by using a FileSystemCache. + + +Steps +----- + +1. For each explicitly given module find the source file location. + +2. For each such module load and check the cache metadata, and decide + whether it's valid. + +3. Now recursively (or iteratively) find dependencies and add those to + the graph: + + - for cached nodes use the list of dependencies from the cache + metadata (this will be valid even if we later end up re-parsing + the same source); + + - for uncached nodes parse the file and process all imports found, + taking care of (a) above. + +Step 3 should also address (d) above. - # Initialize module symbol table, which was populated by the semantic - # analyzer. - tree.names = self.semantic_analyzer().globals +Once step 3 terminates we have the entire dependency graph, and for +each module we've either loaded the cache metadata or parsed the +source code. (However, we may still need to parse those modules for +which we have cache metadata but that depend, directly or indirectly, +on at least one module for which the cache metadata is stale.) - # Replace this state object with a parsed state in BuildManager. - self.switch_state(ParsedFile(self.info(), tree)) +Now we can execute steps A-C from the first section. Finding SCCs for +step A shouldn't be hard; there's a recipe here: +https://code.activestate.com/recipes/578507/. There's also a plethora +of topsort recipes, e.g. https://code.activestate.com/recipes/577413/. - def import_module(self, id: str) -> bool: - """Schedule a module to be processed. +For single nodes, processing is simple. If the node was cached, we +deserialize the cache data and fix up cross-references. Otherwise, we +do semantic analysis followed by type checking. We also handle (c) +above; if a module has valid cache data *but* any of its +dependencies was processed from source, then the module should be +processed from source. - Add an unprocessed state object corresponding to the module to the - manager, or do nothing if the module already has a state object. +A relatively simple optimization (outside SCCs) we might do in the +future is as follows: if a node's cache data is valid, but one or more +of its dependencies are out of date so we have to re-parse the node +from source, once we have fully type-checked the node, we can decide +whether its symbol table actually changed compared to the cache data +(by reading the cache data and comparing it to the data we would be +writing). If there is no change we can declare the node up to date, +and any node that depends (and for which we have cached data, and +whose other dependencies are up to date) on it won't need to be +re-parsed from source. + +Import cycles +------------- + +Finally we have to decide how to handle (c), import cycles. Here +we'll need a modified version of the original state machine +(build.py), but we only need to do this per SCC, and we won't have to +deal with changes to the list of nodes while we're processing it. + +If all nodes in the SCC have valid cache metadata and all dependencies +outside the SCC are still valid, we can proceed as follows: + + 1. Load cache data for all nodes in the SCC. + + 2. Fix up cross-references for all nodes in the SCC. + +Otherwise, the simplest (but potentially slow) way to proceed is to +invalidate all cache data in the SCC and re-parse all nodes in the SCC +from source. We can do this as follows: + + 1. Parse source for all nodes in the SCC. + + 2. Semantic analysis for all nodes in the SCC. + + 3. Type check all nodes in the SCC. + +(If there are more passes the process is the same -- each pass should +be done for all nodes before starting the next pass for any nodes in +the SCC.) + +We could process the nodes in the SCC in any order. For sentimental +reasons, I've decided to process them in the reverse order in which we +encountered them when originally constructing the graph. That's how +the old build.py deals with cycles, and at least this reproduces the +previous implementation more accurately. + +Can we do better than re-parsing all nodes in the SCC when any of its +dependencies are out of date? It's doubtful. The optimization +mentioned at the end of the previous section would require re-parsing +and type-checking a node and then comparing its symbol table to the +cached data; but because the node is part of a cycle we can't +technically type-check it until the semantic analysis of all other +nodes in the cycle has completed. (This is an important issue because +Dropbox has a very large cycle in production code. But I'd like to +deal with it later.) + +Additional wrinkles +------------------- + +During implementation more wrinkles were found. + +- When a submodule of a package (e.g. x.y) is encountered, the parent + package (e.g. x) must also be loaded, but it is not strictly a + dependency. See State.add_ancestors() below. +""" + + +class ModuleNotFound(Exception): + """Control flow exception to signal that a module was not found.""" + + +class State: + """The state for a module. + + The source is only used for the -c command line option; in that + case path is None. Otherwise source is None and path isn't. + """ + + manager: BuildManager + order_counter: ClassVar[int] = 0 + order: int # Order in which modules were encountered + id: str # Fully qualified module name + path: str | None = None # Path to module source + abspath: str | None = None # Absolute path to module source + xpath: str # Path or '' + source: str | None = None # Module source code + source_hash: str | None = None # Hash calculated based on the source code + meta_source_hash: str | None = None # Hash of the source given in the meta, if any + meta: CacheMeta | None = None + data: str | None = None + tree: MypyFile | None = None + # We keep both a list and set of dependencies. A set because it makes it efficient to + # prevent duplicates and the list because I am afraid of changing the order of + # iteration over dependencies. + # They should be managed with add_dependency and suppress_dependency. + dependencies: list[str] # Modules directly imported by the module + dependencies_set: set[str] # The same but as a set for deduplication purposes + suppressed: list[str] # Suppressed/missing dependencies + suppressed_set: set[str] # Suppressed/missing dependencies + priorities: dict[str, int] + + # Map each dependency to the line number where it is first imported + dep_line_map: dict[str, int] + + # Parent package, its parent, etc. + ancestors: list[str] | None = None + + # List of (path, line number) tuples giving context for import + import_context: list[tuple[str, int]] + + # The State from which this module was imported, if any + caller_state: State | None = None + + # If caller_state is set, the line number in the caller where the import occurred + caller_line = 0 + + # If True, indicate that the public interface of this module is unchanged + externally_same = True + + # Contains a hash of the public interface in incremental mode + interface_hash: str = "" + + # Options, specialized for this file + options: Options + + # Whether to ignore all errors + ignore_all = False + + # Whether the module has an error or any of its dependencies have one. + transitive_error = False + + # Errors reported before semantic analysis, to allow fine-grained + # mode to keep reporting them. + early_errors: list[ErrorInfo] + + # Type checker used for checking this file. Use type_checker() for + # access and to construct this on demand. + _type_checker: TypeChecker | None = None + + fine_grained_deps_loaded = False + + # Cumulative time spent on this file, in microseconds (for profiling stats) + time_spent_us: int = 0 + + # Per-line type-checking time (cumulative time spent type-checking expressions + # on a given source code line). + per_line_checking_time_ns: dict[int, int] + + def __init__( + self, + id: str | None, + path: str | None, + source: str | None, + manager: BuildManager, + caller_state: State | None = None, + caller_line: int = 0, + ancestor_for: State | None = None, + root_source: bool = False, + # If `temporary` is True, this State is being created to just + # quickly parse/load the tree, without an intention to further + # process it. With this flag, any changes to external state as well + # as error reporting should be avoided. + temporary: bool = False, + ) -> None: + if not temporary: + assert id or path or source is not None, "Neither id, path nor source given" + self.manager = manager + State.order_counter += 1 + self.order = State.order_counter + self.caller_state = caller_state + self.caller_line = caller_line + if caller_state: + self.import_context = caller_state.import_context.copy() + self.import_context.append((caller_state.xpath, caller_line)) + else: + self.import_context = [] + self.id = id or "__main__" + self.options = manager.options.clone_for_module(self.id) + self.early_errors = [] + self._type_checker = None + if not path and source is None: + assert id is not None + try: + path, follow_imports = find_module_and_diagnose( + manager, + id, + self.options, + caller_state, + caller_line, + ancestor_for, + root_source, + skip_diagnose=temporary, + ) + except ModuleNotFound: + if not temporary: + manager.missing_modules.add(id) + raise + if follow_imports == "silent": + self.ignore_all = True + elif path and is_silent_import_module(manager, path) and not root_source: + self.ignore_all = True + self.path = path + if path: + self.abspath = os.path.abspath(path) + self.xpath = path or "" + if path and source is None and self.manager.cache_enabled: + self.meta = find_cache_meta(self.id, path, manager) + # TODO: Get mtime if not cached. + if self.meta is not None: + self.interface_hash = self.meta.interface_hash + self.meta_source_hash = self.meta.hash + if path and source is None and self.manager.fscache.isdir(path): + source = "" + self.source = source + self.add_ancestors() + self.per_line_checking_time_ns = collections.defaultdict(int) + t0 = time.time() + self.meta = validate_meta(self.meta, self.id, self.path, self.ignore_all, manager) + self.manager.add_stats(validate_meta_time=time.time() - t0) + if self.meta: + # Make copies, since we may modify these and want to + # compare them to the originals later. + self.dependencies = list(self.meta.dependencies) + self.dependencies_set = set(self.dependencies) + self.suppressed = list(self.meta.suppressed) + self.suppressed_set = set(self.suppressed) + all_deps = self.dependencies + self.suppressed + assert len(all_deps) == len(self.meta.dep_prios) + self.priorities = {id: pri for id, pri in zip(all_deps, self.meta.dep_prios)} + assert len(all_deps) == len(self.meta.dep_lines) + self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)} + if temporary: + self.load_tree(temporary=True) + if not manager.use_fine_grained_cache(): + # Special case: if there were a previously missing package imported here + # and it is not present, then we need to re-calculate dependencies. + # This is to support patterns like this: + # from missing_package import missing_module # type: ignore + # At first mypy doesn't know that `missing_module` is a module + # (it may be a variable, a class, or a function), so it is not added to + # suppressed dependencies. Therefore, when the package with module is added, + # we need to re-calculate dependencies. + # NOTE: see comment below for why we skip this in fine grained mode. + if exist_added_packages(self.suppressed, manager, self.options): + self.parse_file() # This is safe because the cache is anyway stale. + self.compute_dependencies() + else: + # When doing a fine-grained cache load, pretend we only + # know about modules that have cache information and defer + # handling new modules until the fine-grained update. + if manager.use_fine_grained_cache(): + manager.log(f"Deferring module to fine-grained update {path} ({id})") + raise ModuleNotFound + + # Parse the file (and then some) to get the dependencies. + self.parse_file(temporary=temporary) + self.compute_dependencies() + + @property + def xmeta(self) -> CacheMeta: + assert self.meta, "missing meta on allegedly fresh module" + return self.meta + + def add_ancestors(self) -> None: + if self.path is not None: + _, name = os.path.split(self.path) + base, _ = os.path.splitext(name) + if "." in base: + # This is just a weird filename, don't add anything + self.ancestors = [] + return + # All parent packages are new ancestors. + ancestors = [] + parent = self.id + while "." in parent: + parent, _ = parent.rsplit(".", 1) + ancestors.append(parent) + self.ancestors = ancestors + + def is_fresh(self) -> bool: + """Return whether the cache data for this file is fresh.""" + # NOTE: self.dependencies may differ from + # self.meta.dependencies when a dependency is dropped due to + # suppression by silent mode. However when a suppressed + # dependency is added back we find out later in the process. + return ( + self.meta is not None + and self.is_interface_fresh() + and self.dependencies == self.meta.dependencies + ) + + def is_interface_fresh(self) -> bool: + return self.externally_same + + def mark_as_rechecked(self) -> None: + """Marks this module as having been fully re-analyzed by the type-checker.""" + self.manager.rechecked_modules.add(self.id) + + def mark_interface_stale(self, *, on_errors: bool = False) -> None: + """Marks this module as having a stale public interface, and discards the cache data.""" + self.externally_same = False + if not on_errors: + self.manager.stale_modules.add(self.id) + + def check_blockers(self) -> None: + """Raise CompileError if a blocking error is detected.""" + if self.manager.errors.is_blockers(): + self.manager.log("Bailing due to blocking errors") + self.manager.errors.raise_error() + + @contextlib.contextmanager + def wrap_context(self, check_blockers: bool = True) -> Iterator[None]: + """Temporarily change the error import context to match this state. + + Also report an internal error if an unexpected exception was raised + and raise an exception on a blocking error, unless + check_blockers is False. Skipping blocking error reporting is used + in the semantic analyzer so that we can report all blocking errors + for a file (across multiple targets) to maintain backward + compatibility. """ - if self.manager.has_module(id): - # Do nothing:f already being compiled. - return True + save_import_context = self.manager.errors.import_context() + self.manager.errors.set_import_context(self.import_context) + try: + yield + except CompileError: + raise + except Exception as err: + report_internal_error( + err, + self.path, + 0, + self.manager.errors, + self.options, + self.manager.stdout, + self.manager.stderr, + ) + self.manager.errors.set_import_context(save_import_context) + # TODO: Move this away once we've removed the old semantic analyzer? + if check_blockers: + self.check_blockers() + + def load_fine_grained_deps(self) -> dict[str, set[str]]: + return self.manager.load_fine_grained_deps(self.id) + + def load_tree(self, temporary: bool = False) -> None: + assert ( + self.meta is not None + ), "Internal error: this method must be called only for cached modules" + + data = _load_json_file( + self.meta.data_json, self.manager, "Load tree ", "Could not load tree: " + ) + if data is None: + return + + t0 = time.time() + # TODO: Assert data file wasn't changed. + self.tree = MypyFile.deserialize(data) + t1 = time.time() + self.manager.add_stats(deserialize_time=t1 - t0) + if not temporary: + self.manager.modules[self.id] = self.tree + self.manager.add_stats(fresh_trees=1) + + def fix_cross_refs(self) -> None: + assert self.tree is not None, "Internal error: method must be called on parsed file only" + # We need to set allow_missing when doing a fine grained cache + # load because we need to gracefully handle missing modules. + fixup_module(self.tree, self.manager.modules, self.options.use_fine_grained_cache) + + # Methods for processing modules from source code. + + def parse_file(self, *, temporary: bool = False) -> None: + """Parse file and run first pass of semantic analysis. + + Everything done here is local to the file. Don't depend on imported + modules in any way. Also record module dependencies based on imports. + """ + if self.tree is not None: + # The file was already parsed (in __init__()). + return - path, text = read_module_source_from_file(id, self.manager.lib_path) - if text is not None: - info = StateInfo(path, id, self.errors().import_context(), - self.manager) - self.manager.states.append(UnprocessedFile(info, text)) - self.manager.module_files[id] = path - return True + manager = self.manager + + # Can we reuse a previously parsed AST? This avoids redundant work in daemon. + cached = self.id in manager.ast_cache + modules = manager.modules + if not cached: + manager.log(f"Parsing {self.xpath} ({self.id})") else: - return False + manager.log(f"Using cached AST for {self.xpath} ({self.id})") + + t0 = time_ref() + + with self.wrap_context(): + source = self.source + self.source = None # We won't need it again. + if self.path and source is None: + try: + path = manager.maybe_swap_for_shadow_path(self.path) + source = decode_python_encoding(manager.fscache.read(path)) + self.source_hash = manager.fscache.hash_digest(path) + except OSError as ioerr: + # ioerr.strerror differs for os.stat failures between Windows and + # other systems, but os.strerror(ioerr.errno) does not, so we use that. + # (We want the error messages to be platform-independent so that the + # tests have predictable output.) + assert ioerr.errno is not None + raise CompileError( + [ + "mypy: can't read file '{}': {}".format( + self.path.replace(os.getcwd() + os.sep, ""), + os.strerror(ioerr.errno), + ) + ], + module_with_blocker=self.id, + ) from ioerr + except (UnicodeDecodeError, DecodeError) as decodeerr: + if self.path.endswith(".pyd"): + err = f"mypy: stubgen does not support .pyd files: '{self.path}'" + else: + err = f"mypy: can't decode file '{self.path}': {str(decodeerr)}" + raise CompileError([err], module_with_blocker=self.id) from decodeerr + elif self.path and self.manager.fscache.isdir(self.path): + source = "" + self.source_hash = "" + else: + assert source is not None + self.source_hash = compute_hash(source) + + self.parse_inline_configuration(source) + if not cached: + self.tree = manager.parse_file( + self.id, + self.xpath, + source, + ignore_errors=self.ignore_all or self.options.ignore_errors, + options=self.options, + ) - def parse(self, source_text: str, fnam: str) -> MypyFile: - """Parse the source of a file with the given name. + else: + # Reuse a cached AST + self.tree = manager.ast_cache[self.id][0] + manager.errors.set_file_ignored_lines( + self.xpath, + self.tree.ignored_lines, + self.ignore_all or self.options.ignore_errors, + ) + + self.time_spent_us += time_spent_us(t0) + + if not cached: + # Make a copy of any errors produced during parse time so that + # fine-grained mode can repeat them when the module is + # reprocessed. + self.early_errors = list(manager.errors.error_info_map.get(self.xpath, [])) + else: + self.early_errors = manager.ast_cache[self.id][1] - Raise CompileError if there is a parse error. - """ - num_errs = self.errors().num_messages() - tree = parse.parse(source_text, fnam, self.errors(), - pyversion=self.manager.pyversion, - custom_typing_module=self.manager.custom_typing_module) - tree._fullname = self.id - if self.errors().num_messages() != num_errs: - self.errors().raise_error() - return tree + if not temporary: + modules[self.id] = self.tree + + if not cached: + self.semantic_analysis_pass1() + + if not temporary: + self.check_blockers() - def state(self) -> int: - return UNPROCESSED_STATE + manager.ast_cache[self.id] = (self.tree, self.early_errors) + def parse_inline_configuration(self, source: str) -> None: + """Check for inline mypy: options directive and parse them.""" + flags = get_mypy_comments(source) + if flags: + changes, config_errors = parse_mypy_comments(flags, self.options) + self.options = self.options.apply_changes(changes) + self.manager.errors.set_file(self.xpath, self.id, self.options) + for lineno, error in config_errors: + self.manager.errors.report(lineno, 0, error) -class ParsedFile(State): - tree = Undefined(MypyFile) + def semantic_analysis_pass1(self) -> None: + """Perform pass 1 of semantic analysis, which happens immediately after parsing. - def __init__(self, info: StateInfo, tree: MypyFile) -> None: - super().__init__(info) - self.tree = tree + This pass can't assume that any other modules have been processed yet. + """ + options = self.options + assert self.tree is not None - # Build a list all directly imported moules (dependencies). - imp = List[str]() - for id, line in self.manager.all_imported_modules_in_file(tree): - # Omit missing modules, as otherwise we could not type check - # programs with missing modules. - if not id in self.manager.missing_modules and id != self.id: - imp.append(id) - if self.id != 'builtins': - imp.append('builtins') + t0 = time_ref() - if imp != []: - trace('{} dependencies: {}'.format(info.path, imp)) + # Do the first pass of semantic analysis: analyze the reachability + # of blocks and import statements. We must do this before + # processing imports, since this may mark some import statements as + # unreachable. + # + # TODO: This should not be considered as a semantic analysis + # pass -- it's an independent pass. + analyzer = SemanticAnalyzerPreAnalysis() + with self.wrap_context(): + analyzer.visit_file(self.tree, self.xpath, self.id, options) + self.manager.errors.set_skipped_lines(self.xpath, self.tree.skipped_lines) + # TODO: Do this while constructing the AST? + self.tree.names = SymbolTable() + if not self.tree.is_stub: + # Always perform some low-key variable renaming + self.tree.accept(LimitedVariableRenameVisitor()) + if options.allow_redefinition: + # Perform more renaming across the AST to allow variable redefinitions + self.tree.accept(VariableRenameVisitor()) + self.time_spent_us += time_spent_us(t0) + + def add_dependency(self, dep: str) -> None: + if dep not in self.dependencies_set: + self.dependencies.append(dep) + self.dependencies_set.add(dep) + if dep in self.suppressed_set: + self.suppressed.remove(dep) + self.suppressed_set.remove(dep) + + def suppress_dependency(self, dep: str) -> None: + if dep in self.dependencies_set: + self.dependencies.remove(dep) + self.dependencies_set.remove(dep) + if dep not in self.suppressed_set: + self.suppressed.append(dep) + self.suppressed_set.add(dep) + + def compute_dependencies(self) -> None: + """Compute a module's dependencies after parsing it. + + This is used when we parse a file that we didn't have + up-to-date cache information for. When we have an up-to-date + cache, we just use the cached info. + """ + manager = self.manager + assert self.tree is not None - # Record the dependencies. Note that the dependencies list also - # contains any superpackages and we must preserve them (e.g. os for - # os.path). - self.dependencies.extend(imp) + # Compute (direct) dependencies. + # Add all direct imports (this is why we needed the first pass). + # Also keep track of each dependency's source line. + # Missing dependencies will be moved from dependencies to + # suppressed when they fail to be loaded in load_graph. - def process(self) -> None: - """Semantically analyze file and advance to the next state.""" - self.semantic_analyzer().visit_file(self.tree, self.tree.path) - self.switch_state(PartiallySemanticallyAnalyzedFile(self.info(), - self.tree)) + self.dependencies = [] + self.dependencies_set = set() + self.suppressed = [] + self.suppressed_set = set() + self.priorities = {} # id -> priority + self.dep_line_map = {} # id -> line + dep_entries = manager.all_imported_modules_in_file( + self.tree + ) + self.manager.plugin.get_additional_deps(self.tree) + for pri, id, line in dep_entries: + self.priorities[id] = min(pri, self.priorities.get(id, PRI_ALL)) + if id == self.id: + continue + self.add_dependency(id) + if id not in self.dep_line_map: + self.dep_line_map[id] = line + # Every module implicitly depends on builtins. + if self.id != "builtins": + self.add_dependency("builtins") + + self.check_blockers() # Can fail due to bogus relative imports + + def type_check_first_pass(self) -> None: + if self.options.semantic_analysis_only: + return + t0 = time_ref() + with self.wrap_context(): + self.type_checker().check_first_pass() + self.time_spent_us += time_spent_us(t0) + + def type_checker(self) -> TypeChecker: + if not self._type_checker: + assert self.tree is not None, "Internal error: must be called on parsed file only" + manager = self.manager + self._type_checker = TypeChecker( + manager.errors, + manager.modules, + self.options, + self.tree, + self.xpath, + manager.plugin, + self.per_line_checking_time_ns, + ) + return self._type_checker + + def type_map(self) -> dict[Expression, Type]: + # We can extract the master type map directly since at this + # point no temporary type maps can be active. + assert len(self.type_checker()._type_maps) == 1 + return self.type_checker()._type_maps[0] + + def type_check_second_pass(self) -> bool: + if self.options.semantic_analysis_only: + return False + t0 = time_ref() + with self.wrap_context(): + result = self.type_checker().check_second_pass() + self.time_spent_us += time_spent_us(t0) + return result + + def detect_possibly_undefined_vars(self) -> None: + assert self.tree is not None, "Internal error: method must be called on parsed file only" + if self.tree.is_stub: + # We skip stub files because they aren't actually executed. + return + manager = self.manager + manager.errors.set_file(self.xpath, self.tree.fullname, options=self.options) + if manager.errors.is_error_code_enabled( + codes.POSSIBLY_UNDEFINED + ) or manager.errors.is_error_code_enabled(codes.USED_BEFORE_DEF): + self.tree.accept( + PossiblyUndefinedVariableVisitor( + MessageBuilder(manager.errors, manager.modules), + self.type_map(), + self.options, + self.tree.names, + ) + ) + + def finish_passes(self) -> None: + assert self.tree is not None, "Internal error: method must be called on parsed file only" + manager = self.manager + if self.options.semantic_analysis_only: + return + t0 = time_ref() + with self.wrap_context(): + # Some tests (and tools) want to look at the set of all types. + options = manager.options + if options.export_types: + manager.all_types.update(self.type_map()) + + # We should always patch indirect dependencies, even in full (non-incremental) builds, + # because the cache still may be written, and it must be correct. + # TODO: find a more robust way to traverse *all* relevant types? + expr_types = set(self.type_map().values()) + symbol_types = set() + for _, sym, _ in self.tree.local_definitions(): + if sym.type is not None: + symbol_types.add(sym.type) + if isinstance(sym.node, TypeInfo): + # TypeInfo symbols have some extra relevant types. + symbol_types.update(sym.node.bases) + if sym.node.metaclass_type: + symbol_types.add(sym.node.metaclass_type) + if sym.node.typeddict_type: + symbol_types.add(sym.node.typeddict_type) + if sym.node.tuple_type: + symbol_types.add(sym.node.tuple_type) + self._patch_indirect_dependencies( + self.type_checker().module_refs, expr_types | symbol_types + ) + + if self.options.dump_inference_stats: + dump_type_stats( + self.tree, + self.xpath, + modules=self.manager.modules, + inferred=True, + typemap=self.type_map(), + ) + manager.report_file(self.tree, self.type_map(), self.options) + + self.update_fine_grained_deps(self.manager.fg_deps) + + if manager.options.export_ref_info: + write_undocumented_ref_info( + self, manager.metastore, manager.options, self.type_map() + ) + + self.free_state() + if not manager.options.fine_grained_incremental and not manager.options.preserve_asts: + free_tree(self.tree) + self.time_spent_us += time_spent_us(t0) + + def free_state(self) -> None: + if self._type_checker: + self._type_checker.reset() + self._type_checker = None + + def _patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: + assert None not in types + valid = self.valid_references() + + encountered = self.manager.indirection_detector.find_modules(types) | module_refs + extra = encountered - valid + + for dep in sorted(extra): + if dep not in self.manager.modules: + continue + if dep not in self.suppressed_set and dep not in self.manager.missing_modules: + self.add_dependency(dep) + self.priorities[dep] = PRI_INDIRECT + elif dep not in self.suppressed_set and dep in self.manager.missing_modules: + self.suppress_dependency(dep) + + def compute_fine_grained_deps(self) -> dict[str, set[str]]: + assert self.tree is not None + if self.id in ("builtins", "typing", "types", "sys", "_typeshed"): + # We don't track changes to core parts of typeshed -- the + # assumption is that they are only changed as part of mypy + # updates, which will invalidate everything anyway. These + # will always be processed in the initial non-fine-grained + # build. Other modules may be brought in as a result of an + # fine-grained increment, and we may need these + # dependencies then to handle cyclic imports. + return {} + from mypy.server.deps import get_dependencies # Lazy import to speed up startup + + return get_dependencies( + target=self.tree, + type_map=self.type_map(), + python_version=self.options.python_version, + options=self.manager.options, + ) + + def update_fine_grained_deps(self, deps: dict[str, set[str]]) -> None: + options = self.manager.options + if options.cache_fine_grained or options.fine_grained_incremental: + from mypy.server.deps import merge_dependencies # Lazy import to speed up startup + + merge_dependencies(self.compute_fine_grained_deps(), deps) + type_state.update_protocol_deps(deps) + + def valid_references(self) -> set[str]: + assert self.ancestors is not None + valid_refs = set(self.dependencies + self.suppressed + self.ancestors) + valid_refs.add(self.id) + + if "os" in valid_refs: + valid_refs.add("os.path") + + return valid_refs + + def write_cache(self) -> None: + assert self.tree is not None, "Internal error: method must be called on parsed file only" + # We don't support writing cache files in fine-grained incremental mode. + if ( + not self.path + or self.options.cache_dir == os.devnull + or self.options.fine_grained_incremental + ): + if self.options.debug_serialize: + try: + self.tree.serialize() + except Exception: + print(f"Error serializing {self.id}", file=self.manager.stdout) + raise # Propagate to display traceback + return + is_errors = self.transitive_error + if is_errors: + delete_cache(self.id, self.path, self.manager) + self.meta = None + self.mark_interface_stale(on_errors=True) + return + dep_prios = self.dependency_priorities() + dep_lines = self.dependency_lines() + assert self.source_hash is not None + assert len(set(self.dependencies)) == len( + self.dependencies + ), f"Duplicates in dependencies list for {self.id} ({self.dependencies})" + new_interface_hash, self.meta = write_cache( + self.id, + self.path, + self.tree, + list(self.dependencies), + list(self.suppressed), + dep_prios, + dep_lines, + self.interface_hash, + self.source_hash, + self.ignore_all, + self.manager, + ) + if new_interface_hash == self.interface_hash: + self.manager.log(f"Cached module {self.id} has same interface") + else: + self.manager.log(f"Cached module {self.id} has changed interface") + self.mark_interface_stale() + self.interface_hash = new_interface_hash - def num_incomplete_deps(self) -> int: - """Return the number of dependencies that are incomplete. + def verify_dependencies(self, suppressed_only: bool = False) -> None: + """Report errors for import targets in modules that don't exist. - Here complete means that their state is *later* than this module. - Cyclic dependencies are omitted to break cycles forcibly (and somewhat - arbitrarily). + If suppressed_only is set, only check suppressed dependencies. """ - incomplete = 0 - for module in self.dependencies: - state = self.manager.module_state(module) - if (not earlier_state(self.state(), state) and - not self.manager.is_dep(module, self.id)): - incomplete += 1 - return incomplete - - def state(self) -> int: - return PARSED_STATE - - -class PartiallySemanticallyAnalyzedFile(ParsedFile): - def process(self) -> None: - """Perform final pass of semantic analysis and advance state.""" - self.semantic_analyzer_pass3().visit_file(self.tree, self.tree.path) - if 'dump-type-stats' in self.manager.flags: - stats.dump_type_stats(self.tree, self.tree.path) - self.switch_state(SemanticallyAnalyzedFile(self.info(), self.tree)) - - def state(self) -> int: - return PARTIAL_SEMANTIC_ANALYSIS_STATE - - -class SemanticallyAnalyzedFile(ParsedFile): - def process(self) -> None: - """Type check file and advance to the next state.""" - if self.manager.target >= TYPE_CHECK: - self.type_checker().visit_file(self.tree, self.tree.path) - if 'dump-infer-stats' in self.manager.flags: - stats.dump_type_stats(self.tree, self.tree.path, inferred=True, - typemap=self.manager.type_checker.type_map) - elif 'html-report' in self.manager.flags: - stats.generate_html_report( - self.tree, self.tree.path, - type_map=self.manager.type_checker.type_map, - output_dir=self.manager.html_report_dir) - - # FIX remove from active state list to speed up processing - - self.switch_state(TypeCheckedFile(self.info(), self.tree)) - - def state(self) -> int: - return SEMANTICALLY_ANALYSED_STATE - - -class TypeCheckedFile(SemanticallyAnalyzedFile): - def process(self) -> None: - """Finished, so cannot process.""" - raise RuntimeError('Cannot process TypeCheckedFile') - - def is_ready(self) -> bool: - """Finished, so cannot ever become ready.""" - return False + manager = self.manager + assert self.ancestors is not None + if suppressed_only: + all_deps = self.suppressed + else: + # Strip out indirect dependencies. See comment in build.load_graph(). + dependencies = [ + dep for dep in self.dependencies if self.priorities.get(dep) != PRI_INDIRECT + ] + all_deps = dependencies + self.suppressed + self.ancestors + for dep in all_deps: + if dep in manager.modules: + continue + options = manager.options.clone_for_module(dep) + if options.ignore_missing_imports: + continue + line = self.dep_line_map.get(dep, 1) + try: + if dep in self.ancestors: + state: State | None = None + ancestor: State | None = self + else: + state, ancestor = self, None + # Called just for its side effects of producing diagnostics. + find_module_and_diagnose( + manager, + dep, + options, + caller_state=state, + caller_line=line, + ancestor_for=ancestor, + ) + except (ModuleNotFound, CompileError): + # Swallow up any ModuleNotFounds or CompilerErrors while generating + # a diagnostic. CompileErrors may get generated in + # fine-grained mode when an __init__.py is deleted, if a module + # that was in that package has targets reprocessed before + # it is renamed. + pass + + def dependency_priorities(self) -> list[int]: + return [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies + self.suppressed] + + def dependency_lines(self) -> list[int]: + return [self.dep_line_map.get(dep, 1) for dep in self.dependencies + self.suppressed] + + def generate_unused_ignore_notes(self) -> None: + if ( + self.options.warn_unused_ignores + or codes.UNUSED_IGNORE in self.options.enabled_error_codes + ) and codes.UNUSED_IGNORE not in self.options.disabled_error_codes: + # If this file was initially loaded from the cache, it may have suppressed + # dependencies due to imports with ignores on them. We need to generate + # those errors to avoid spuriously flagging them as unused ignores. + if self.meta: + self.verify_dependencies(suppressed_only=True) + self.manager.errors.generate_unused_ignore_errors(self.xpath) + + def generate_ignore_without_code_notes(self) -> None: + if self.manager.errors.is_error_code_enabled(codes.IGNORE_WITHOUT_CODE): + self.manager.errors.generate_ignore_without_code_errors( + self.xpath, self.options.warn_unused_ignores + ) + + +# Module import and diagnostic glue + + +def find_module_and_diagnose( + manager: BuildManager, + id: str, + options: Options, + caller_state: State | None = None, + caller_line: int = 0, + ancestor_for: State | None = None, + root_source: bool = False, + skip_diagnose: bool = False, +) -> tuple[str, str]: + """Find a module by name, respecting follow_imports and producing diagnostics. + + If the module is not found, then the ModuleNotFound exception is raised. + + Args: + id: module to find + options: the options for the module being loaded + caller_state: the state of the importing module, if applicable + caller_line: the line number of the import + ancestor_for: the child module this is an ancestor of, if applicable + root_source: whether this source was specified on the command line + skip_diagnose: skip any error diagnosis and reporting (but ModuleNotFound is + still raised if the module is missing) + + The specified value of follow_imports for a module can be overridden + if the module is specified on the command line or if it is a stub, + so we compute and return the "effective" follow_imports of the module. + + Returns a tuple containing (file path, target's effective follow_imports setting) + """ + result = find_module_with_reason(id, manager) + if isinstance(result, str): + # For non-stubs, look at options.follow_imports: + # - normal (default) -> fully analyze + # - silent -> analyze but silence errors + # - skip -> don't analyze, make the type Any + follow_imports = options.follow_imports + if ( + root_source # Honor top-level modules + or ( + result.endswith(".pyi") # Stubs are always normal + and not options.follow_imports_for_stubs # except when they aren't + ) + or id in CORE_BUILTIN_MODULES # core is always normal + ): + follow_imports = "normal" + if skip_diagnose: + pass + elif follow_imports == "silent": + # Still import it, but silence non-blocker errors. + manager.log(f"Silencing {result} ({id})") + elif follow_imports == "skip" or follow_imports == "error": + # In 'error' mode, produce special error messages. + if id not in manager.missing_modules: + manager.log(f"Skipping {result} ({id})") + if follow_imports == "error": + if ancestor_for: + skipping_ancestor(manager, id, result, ancestor_for) + else: + skipping_module(manager, caller_line, caller_state, id, result) + raise ModuleNotFound + if is_silent_import_module(manager, result) and not root_source: + follow_imports = "silent" + return (result, follow_imports) + else: + # Could not find a module. Typically the reason is a + # misspelled module name, missing stub, module not in + # search path or the module has not been installed. + + ignore_missing_imports = options.ignore_missing_imports + + # Don't honor a global (not per-module) ignore_missing_imports + # setting for modules that used to have bundled stubs, as + # otherwise updating mypy can silently result in new false + # negatives. (Unless there are stubs but they are incomplete.) + global_ignore_missing_imports = manager.options.ignore_missing_imports + if ( + is_module_from_legacy_bundled_package(id) + and global_ignore_missing_imports + and not options.ignore_missing_imports_per_module + and result is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + ): + ignore_missing_imports = False + + if skip_diagnose: + raise ModuleNotFound + if caller_state: + if not (ignore_missing_imports or in_partial_package(id, manager)): + module_not_found(manager, caller_line, caller_state, id, result) + raise ModuleNotFound + elif root_source: + # If we can't find a root source it's always fatal. + # TODO: This might hide non-fatal errors from + # root sources processed earlier. + raise CompileError([f"mypy: can't find module '{id}'"]) + else: + raise ModuleNotFound + + +def exist_added_packages(suppressed: list[str], manager: BuildManager, options: Options) -> bool: + """Find if there are any newly added packages that were previously suppressed. - def state(self) -> int: - return TYPE_CHECKED_STATE + Exclude everything not in build for follow-imports=skip. + """ + for dep in suppressed: + if dep in manager.source_set.source_modules: + # We don't need to add any special logic for this. If a module + # is added to build, importers will be invalidated by normal mechanism. + continue + path = find_module_simple(dep, manager) + if not path: + continue + if options.follow_imports == "skip" and ( + not path.endswith(".pyi") or options.follow_imports_for_stubs + ): + continue + if "__init__.py" in path: + # It is better to have a bit lenient test, this will only slightly reduce + # performance, while having a too strict test may affect correctness. + return True + return False -def trace(s): - if debug: - print(s) +def find_module_simple(id: str, manager: BuildManager) -> str | None: + """Find a filesystem path for module `id` or `None` if not found.""" + t0 = time.time() + x = manager.find_module_cache.find_module(id, fast_path=True) + manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1) + if isinstance(x, ModuleNotFoundReason): + return None + return x -def read_module_source_from_file(id: str, - lib_path: List[str]) -> Tuple[str, str]: - """Find and read the source file of a module. +def find_module_with_reason(id: str, manager: BuildManager) -> ModuleSearchResult: + """Find a filesystem path for module `id` or the reason it can't be found.""" + t0 = time.time() + x = manager.find_module_cache.find_module(id, fast_path=False) + manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1) + return x - Return a pair (path, file contents). Return (None, None) if the module - could not be found or read. - Arguments: - id: module name, a string of form 'foo' or 'foo.bar' - lib_path: library search path +def in_partial_package(id: str, manager: BuildManager) -> bool: + """Check if a missing module can potentially be a part of a package. + + This checks if there is any existing parent __init__.pyi stub that + defines a module-level __getattr__ (a.k.a. partial stub package). """ - path = find_module(id, lib_path) - if path is not None: - text = '' - try: - f = open(path) + while "." in id: + parent, _ = id.rsplit(".", 1) + if parent in manager.modules: + parent_mod: MypyFile | None = manager.modules[parent] + else: + # Parent is not in build, try quickly if we can find it. try: - text = f.read() - finally: - f.close() - except IOError: - return None, None - return path, text + parent_st = State( + id=parent, path=None, source=None, manager=manager, temporary=True + ) + except (ModuleNotFound, CompileError): + parent_mod = None + else: + parent_mod = parent_st.tree + if parent_mod is not None: + # Bail out soon, complete subpackage found + return parent_mod.is_partial_stub_package + id = parent + return False + + +def module_not_found( + manager: BuildManager, + line: int, + caller_state: State, + target: str, + reason: ModuleNotFoundReason, +) -> None: + errors = manager.errors + save_import_context = errors.import_context() + errors.set_import_context(caller_state.import_context) + errors.set_file(caller_state.xpath, caller_state.id, caller_state.options) + if target == "builtins": + errors.report( + line, 0, "Cannot find 'builtins' module. Typeshed appears broken!", blocker=True + ) + errors.raise_error() else: - return None, None - - -def find_module(id: str, lib_path: List[str]) -> str: - """Return the path of the module source file, or None if not found.""" - extensions = ['.pyi', '.py'] - for pathitem in lib_path: - for extension in extensions: - comp = id.split('.') - path = os.path.join(pathitem, os.sep.join(comp[:-1]), comp[-1] + extension) - text = '' - if not os.path.isfile(path): - path = os.path.join(pathitem, os.sep.join(comp), '__init__.py') - if os.path.isfile(path) and verify_module(id, path): - return path - return None - - -def verify_module(id: str, path: str) -> bool: - """Check that all packages containing id have a __init__ file.""" - if path.endswith('__init__.py'): - path = dirname(path) - for i in range(id.count('.')): - path = dirname(path) - if not os.path.isfile(os.path.join(path, '__init__.py')): - return False - return True + daemon = manager.options.fine_grained_incremental + msg, notes = reason.error_message_templates(daemon) + if reason == ModuleNotFoundReason.NOT_FOUND: + code = codes.IMPORT_NOT_FOUND + elif ( + reason == ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS + or reason == ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + ): + code = codes.IMPORT_UNTYPED + else: + code = codes.IMPORT + errors.report(line, 0, msg.format(module=target), code=code) + + dist = stub_distribution_name(target) + for note in notes: + if "{stub_dist}" in note: + assert dist is not None + note = note.format(stub_dist=dist) + errors.report(line, 0, note, severity="note", only_once=True, code=code) + if reason is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: + assert dist is not None + manager.missing_stub_packages.add(dist) + errors.set_import_context(save_import_context) + + +def skipping_module( + manager: BuildManager, line: int, caller_state: State | None, id: str, path: str +) -> None: + """Produce an error for an import ignored due to --follow_imports=error""" + assert caller_state, (id, path) + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(caller_state.import_context) + manager.errors.set_file(caller_state.xpath, caller_state.id, manager.options) + manager.errors.report(line, 0, f'Import of "{id}" ignored', severity="error") + manager.errors.report( + line, + 0, + "(Using --follow-imports=error, module not passed on command line)", + severity="note", + only_once=True, + ) + manager.errors.set_import_context(save_import_context) + + +def skipping_ancestor(manager: BuildManager, id: str, path: str, ancestor_for: State) -> None: + """Produce an error for an ancestor ignored due to --follow_imports=error""" + # TODO: Read the path (the __init__.py file) and return + # immediately if it's empty or only contains comments. + # But beware, some package may be the ancestor of many modules, + # so we'd need to cache the decision. + manager.errors.set_import_context([]) + manager.errors.set_file(ancestor_for.xpath, ancestor_for.id, manager.options) + manager.errors.report( + -1, -1, f'Ancestor package "{id}" ignored', severity="error", only_once=True + ) + manager.errors.report( + -1, + -1, + "(Using --follow-imports=error, submodule passed on command line)", + severity="note", + only_once=True, + ) + + +def log_configuration(manager: BuildManager, sources: list[BuildSource]) -> None: + """Output useful configuration information to LOG and TRACE""" + + config_file = manager.options.config_file + if config_file: + config_file = os.path.abspath(config_file) + + manager.log() + configuration_vars = [ + ("Mypy Version", __version__), + ("Config File", (config_file or "Default")), + ("Configured Executable", manager.options.python_executable or "None"), + ("Current Executable", sys.executable), + ("Cache Dir", manager.options.cache_dir), + ("Compiled", str(not __file__.endswith(".py"))), + ("Exclude", manager.options.exclude), + ] + + for conf_name, conf_value in configuration_vars: + manager.log(f"{conf_name + ':':24}{conf_value}") + + for source in sources: + manager.log(f"{'Found source:':24}{source}") + + # Complete list of searched paths can get very long, put them under TRACE + for path_type, paths in manager.search_paths.asdict().items(): + if not paths: + manager.trace(f"No {path_type}") + continue + + manager.trace(f"{path_type}:") + + for pth in paths: + manager.trace(f" {pth}") + + +# The driver + + +def dispatch(sources: list[BuildSource], manager: BuildManager, stdout: TextIO) -> Graph: + log_configuration(manager, sources) + + t0 = time.time() + graph = load_graph(sources, manager) + + # This is a kind of unfortunate hack to work around some of fine-grained's + # fragility: if we have loaded less than 50% of the specified files from + # cache in fine-grained cache mode, load the graph again honestly. + # In this case, we just turn the cache off entirely, so we don't need + # to worry about some files being loaded and some from cache and so + # that fine-grained mode never *writes* to the cache. + if manager.use_fine_grained_cache() and len(graph) < 0.50 * len(sources): + manager.log("Redoing load_graph without cache because too much was missing") + manager.cache_enabled = False + graph = load_graph(sources, manager) + + t1 = time.time() + manager.add_stats( + graph_size=len(graph), + stubs_found=sum(g.path is not None and g.path.endswith(".pyi") for g in graph.values()), + graph_load_time=(t1 - t0), + fm_cache_size=len(manager.find_module_cache.results), + ) + if not graph: + print("Nothing to do?!", file=stdout) + return graph + manager.log(f"Loaded graph with {len(graph)} nodes ({t1 - t0:.3f} sec)") + if manager.options.dump_graph: + dump_graph(graph, stdout) + return graph + + # Fine grained dependencies that didn't have an associated module in the build + # are serialized separately, so we read them after we load the graph. + # We need to read them both for running in daemon mode and if we are generating + # a fine-grained cache (so that we can properly update them incrementally). + # The `read_deps_cache` will also validate + # the deps cache against the loaded individual cache files. + if manager.options.cache_fine_grained or manager.use_fine_grained_cache(): + t2 = time.time() + fg_deps_meta = read_deps_cache(manager, graph) + manager.add_stats(load_fg_deps_time=time.time() - t2) + if fg_deps_meta is not None: + manager.fg_deps_meta = fg_deps_meta + elif manager.stats.get("fresh_metas", 0) > 0: + # Clear the stats so we don't infinite loop because of positive fresh_metas + manager.stats.clear() + # There were some cache files read, but no fine-grained dependencies loaded. + manager.log("Error reading fine-grained dependencies cache -- aborting cache load") + manager.cache_enabled = False + manager.log("Falling back to full run -- reloading graph...") + return dispatch(sources, manager, stdout) + + # If we are loading a fine-grained incremental mode cache, we + # don't want to do a real incremental reprocess of the + # graph---we'll handle it all later. + if not manager.use_fine_grained_cache(): + process_graph(graph, manager) + # Update plugins snapshot. + write_plugins_snapshot(manager) + manager.old_plugins_snapshot = manager.plugins_snapshot + if manager.options.cache_fine_grained or manager.options.fine_grained_incremental: + # If we are running a daemon or are going to write cache for further fine grained use, + # then we need to collect fine grained protocol dependencies. + # Since these are a global property of the program, they are calculated after we + # processed the whole graph. + type_state.add_all_protocol_deps(manager.fg_deps) + if not manager.options.fine_grained_incremental: + rdeps = generate_deps_for_cache(manager, graph) + write_deps_cache(rdeps, manager, graph) + + if manager.options.dump_deps: + # This speeds up startup a little when not using the daemon mode. + from mypy.server.deps import dump_all_dependencies + + dump_all_dependencies( + manager.modules, manager.all_types, manager.options.python_version, manager.options + ) + + return graph + + +class NodeInfo: + """Some info about a node in the graph of SCCs.""" + + def __init__(self, index: int, scc: list[str]) -> None: + self.node_id = "n%d" % index + self.scc = scc + self.sizes: dict[str, int] = {} # mod -> size in bytes + self.deps: dict[str, int] = {} # node_id -> pri + + def dumps(self) -> str: + """Convert to JSON string.""" + total_size = sum(self.sizes.values()) + return "[{}, {}, {},\n {},\n {}]".format( + json.dumps(self.node_id), + json.dumps(total_size), + json.dumps(self.scc), + json.dumps(self.sizes), + json.dumps(self.deps), + ) + + +def dump_timing_stats(path: str, graph: Graph) -> None: + """Dump timing stats for each file in the given graph.""" + with open(path, "w") as f: + for id in sorted(graph): + f.write(f"{id} {graph[id].time_spent_us}\n") + + +def dump_line_checking_stats(path: str, graph: Graph) -> None: + """Dump per-line expression type checking stats.""" + with open(path, "w") as f: + for id in sorted(graph): + if not graph[id].per_line_checking_time_ns: + continue + f.write(f"{id}:\n") + for line in sorted(graph[id].per_line_checking_time_ns): + line_time = graph[id].per_line_checking_time_ns[line] + f.write(f"{line:>5} {line_time/1000:8.1f}\n") + + +def dump_graph(graph: Graph, stdout: TextIO | None = None) -> None: + """Dump the graph as a JSON string to stdout. + + This copies some of the work by process_graph() + (sorted_components() and order_ascc()). + """ + stdout = stdout or sys.stdout + nodes = [] + sccs = sorted_components(graph) + for i, ascc in enumerate(sccs): + scc = order_ascc(graph, ascc) + node = NodeInfo(i, scc) + nodes.append(node) + inv_nodes = {} # module -> node_id + for node in nodes: + for mod in node.scc: + inv_nodes[mod] = node.node_id + for node in nodes: + for mod in node.scc: + state = graph[mod] + size = 0 + if state.path: + try: + size = os.path.getsize(state.path) + except OSError: + pass + node.sizes[mod] = size + for dep in state.dependencies: + if dep in state.priorities: + pri = state.priorities[dep] + if dep in inv_nodes: + dep_id = inv_nodes[dep] + if dep_id != node.node_id and ( + dep_id not in node.deps or pri < node.deps[dep_id] + ): + node.deps[dep_id] = pri + print("[" + ",\n ".join(node.dumps() for node in nodes) + "\n]", file=stdout) + + +def load_graph( + sources: list[BuildSource], + manager: BuildManager, + old_graph: Graph | None = None, + new_modules: list[State] | None = None, +) -> Graph: + """Given some source files, load the full dependency graph. + + If an old_graph is passed in, it is used as the starting point and + modified during graph loading. + + If a new_modules is passed in, any modules that are loaded are + added to the list. This is an argument and not a return value + so that the caller can access it even if load_graph fails. + + As this may need to parse files, this can raise CompileError in case + there are syntax errors. + """ + graph: Graph = old_graph if old_graph is not None else {} -def super_packages(id: str) -> List[str]: - """Return the surrounding packages of a module, e.g. ['os'] for os.path.""" - c = id.split('.') - res = [] # type: List[str] - for i in range(1, len(c)): - res.append('.'.join(c[:i])) + # The deque is used to implement breadth-first traversal. + # TODO: Consider whether to go depth-first instead. This may + # affect the order in which we process files within import cycles. + new = new_modules if new_modules is not None else [] + entry_points: set[str] = set() + # Seed the graph with the initial root sources. + for bs in sources: + try: + st = State( + id=bs.module, + path=bs.path, + source=bs.text, + manager=manager, + root_source=not bs.followed, + ) + except ModuleNotFound: + continue + if st.id in graph: + manager.errors.set_file(st.xpath, st.id, manager.options) + manager.errors.report( + -1, + -1, + f'Duplicate module named "{st.id}" (also at "{graph[st.id].xpath}")', + blocker=True, + ) + manager.errors.report( + -1, + -1, + "See https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules " + "for more info", + severity="note", + ) + manager.errors.report( + -1, + -1, + "Common resolutions include: a) using `--exclude` to avoid checking one of them, " + "b) adding `__init__.py` somewhere, c) using `--explicit-package-bases` or " + "adjusting MYPYPATH", + severity="note", + ) + + manager.errors.raise_error() + graph[st.id] = st + new.append(st) + entry_points.add(bs.module) + + # Note: Running this each time could be slow in the daemon. If it's a problem, we + # can do more work to maintain this incrementally. + seen_files = {st.abspath: st for st in graph.values() if st.path} + + # Collect dependencies. We go breadth-first. + # More nodes might get added to new as we go, but that's fine. + for st in new: + assert st.ancestors is not None + # Strip out indirect dependencies. These will be dealt with + # when they show up as direct dependencies, and there's a + # scenario where they hurt: + # - Suppose A imports B and B imports C. + # - Suppose on the next round: + # - C is deleted; + # - B is updated to remove the dependency on C; + # - A is unchanged. + # - In this case A's cached *direct* dependencies are still valid + # (since direct dependencies reflect the imports found in the source) + # but A's cached *indirect* dependency on C is wrong. + dependencies = [dep for dep in st.dependencies if st.priorities.get(dep) != PRI_INDIRECT] + if not manager.use_fine_grained_cache(): + # TODO: Ideally we could skip here modules that appeared in st.suppressed + # because they are not in build with `follow-imports=skip`. + # This way we could avoid overhead of cloning options in `State.__init__()` + # below to get the option value. This is quite minor performance loss however. + added = [dep for dep in st.suppressed if find_module_simple(dep, manager)] + else: + # During initial loading we don't care about newly added modules, + # they will be taken care of during fine grained update. See also + # comment about this in `State.__init__()`. + added = [] + for dep in st.ancestors + dependencies + st.suppressed: + ignored = dep in st.suppressed_set and dep not in entry_points + if ignored and dep not in added: + manager.missing_modules.add(dep) + elif dep not in graph: + try: + if dep in st.ancestors: + # TODO: Why not 'if dep not in st.dependencies' ? + # Ancestors don't have import context. + newst = State( + id=dep, path=None, source=None, manager=manager, ancestor_for=st + ) + else: + newst = State( + id=dep, + path=None, + source=None, + manager=manager, + caller_state=st, + caller_line=st.dep_line_map.get(dep, 1), + ) + except ModuleNotFound: + if dep in st.dependencies_set: + st.suppress_dependency(dep) + else: + if newst.path: + newst_path = os.path.abspath(newst.path) + + if newst_path in seen_files: + manager.errors.report( + -1, + 0, + "Source file found twice under different module names: " + '"{}" and "{}"'.format(seen_files[newst_path].id, newst.id), + blocker=True, + ) + manager.errors.report( + -1, + 0, + "See https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules " + "for more info", + severity="note", + ) + manager.errors.report( + -1, + 0, + "Common resolutions include: a) adding `__init__.py` somewhere, " + "b) using `--explicit-package-bases` or adjusting MYPYPATH", + severity="note", + ) + manager.errors.raise_error() + + seen_files[newst_path] = newst + + assert newst.id not in graph, newst.id + graph[newst.id] = newst + new.append(newst) + if dep in graph and dep in st.suppressed_set: + # Previously suppressed file is now visible + st.add_dependency(dep) + manager.plugin.set_modules(manager.modules) + return graph + + +def process_graph(graph: Graph, manager: BuildManager) -> None: + """Process everything in dependency order.""" + sccs = sorted_components(graph) + manager.log("Found %d SCCs; largest has %d nodes" % (len(sccs), max(len(scc) for scc in sccs))) + + fresh_scc_queue: list[list[str]] = [] + + # We're processing SCCs from leaves (those without further + # dependencies) to roots (those from which everything else can be + # reached). + for ascc in sccs: + # Order the SCC's nodes using a heuristic. + # Note that ascc is a set, and scc is a list. + scc = order_ascc(graph, ascc) + # Make the order of the SCC that includes 'builtins' and 'typing', + # among other things, predictable. Various things may break if + # the order changes. + if "builtins" in ascc: + scc = sorted(scc, reverse=True) + # If builtins is in the list, move it last. (This is a bit of + # a hack, but it's necessary because the builtins module is + # part of a small cycle involving at least {builtins, abc, + # typing}. Of these, builtins must be processed last or else + # some builtin objects will be incompletely processed.) + scc.remove("builtins") + scc.append("builtins") + if manager.options.verbosity >= 2: + for id in scc: + manager.trace( + f"Priorities for {id}:", + " ".join( + "%s:%d" % (x, graph[id].priorities[x]) + for x in graph[id].dependencies + if x in ascc and x in graph[id].priorities + ), + ) + # Because the SCCs are presented in topological sort order, we + # don't need to look at dependencies recursively for staleness + # -- the immediate dependencies are sufficient. + stale_scc = {id for id in scc if not graph[id].is_fresh()} + fresh = not stale_scc + deps = set() + for id in scc: + deps.update(graph[id].dependencies) + deps -= ascc + stale_deps = {id for id in deps if id in graph and not graph[id].is_interface_fresh()} + fresh = fresh and not stale_deps + undeps = set() + if fresh: + # Check if any dependencies that were suppressed according + # to the cache have been added back in this run. + # NOTE: Newly suppressed dependencies are handled by is_fresh(). + for id in scc: + undeps.update(graph[id].suppressed) + undeps &= graph.keys() + if undeps: + fresh = False + if fresh: + # All cache files are fresh. Check that no dependency's + # cache file is newer than any scc node's cache file. + oldest_in_scc = min(graph[id].xmeta.data_mtime for id in scc) + viable = {id for id in stale_deps if graph[id].meta is not None} + newest_in_deps = ( + 0 if not viable else max(graph[dep].xmeta.data_mtime for dep in viable) + ) + if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. + all_ids = sorted(ascc | viable, key=lambda id: graph[id].xmeta.data_mtime) + for id in all_ids: + if id in scc: + if graph[id].xmeta.data_mtime < newest_in_deps: + key = "*id:" + else: + key = "id:" + else: + if graph[id].xmeta.data_mtime > oldest_in_scc: + key = "+dep:" + else: + key = "dep:" + manager.trace(" %5s %.0f %s" % (key, graph[id].xmeta.data_mtime, id)) + # If equal, give the benefit of the doubt, due to 1-sec time granularity + # (on some platforms). + if oldest_in_scc < newest_in_deps: + fresh = False + fresh_msg = f"out of date by {newest_in_deps - oldest_in_scc:.0f} seconds" + else: + fresh_msg = "fresh" + elif undeps: + fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})" + elif stale_scc: + fresh_msg = "inherently stale" + if stale_scc != ascc: + fresh_msg += f" ({' '.join(sorted(stale_scc))})" + if stale_deps: + fresh_msg += f" with stale deps ({' '.join(sorted(stale_deps))})" + else: + fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})" + + # Initialize transitive_error for all SCC members from union + # of transitive_error of dependencies. + if any(graph[dep].transitive_error for dep in deps if dep in graph): + for id in scc: + graph[id].transitive_error = True + + scc_str = " ".join(scc) + if fresh: + manager.trace(f"Queuing {fresh_msg} SCC ({scc_str})") + fresh_scc_queue.append(scc) + else: + if fresh_scc_queue: + manager.log(f"Processing {len(fresh_scc_queue)} queued fresh SCCs") + # Defer processing fresh SCCs until we actually run into a stale SCC + # and need the earlier modules to be loaded. + # + # Note that `process_graph` may end with us not having processed every + # single fresh SCC. This is intentional -- we don't need those modules + # loaded if there are no more stale SCCs to be rechecked. + # + # Also note we shouldn't have to worry about transitive_error here, + # since modules with transitive errors aren't written to the cache, + # and if any dependencies were changed, this SCC would be stale. + # (Also, in quick_and_dirty mode we don't care about transitive errors.) + # + # TODO: see if it's possible to determine if we need to process only a + # _subset_ of the past SCCs instead of having to process them all. + for prev_scc in fresh_scc_queue: + process_fresh_modules(graph, prev_scc, manager) + fresh_scc_queue = [] + size = len(scc) + if size == 1: + manager.log(f"Processing SCC singleton ({scc_str}) as {fresh_msg}") + else: + manager.log("Processing SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg)) + process_stale_scc(graph, scc, manager) + + sccs_left = len(fresh_scc_queue) + nodes_left = sum(len(scc) for scc in fresh_scc_queue) + manager.add_stats(sccs_left=sccs_left, nodes_left=nodes_left) + if sccs_left: + manager.log( + "{} fresh SCCs ({} nodes) left in queue (and will remain unprocessed)".format( + sccs_left, nodes_left + ) + ) + manager.trace(str(fresh_scc_queue)) + else: + manager.log("No fresh SCCs left in queue") + + +def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> list[str]: + """Come up with the ideal processing order within an SCC. + + Using the priorities assigned by all_imported_modules_in_file(), + try to reduce the cycle to a DAG, by omitting arcs representing + dependencies of lower priority. + + In the simplest case, if we have A <--> B where A has a top-level + "import B" (medium priority) but B only has the reverse "import A" + inside a function (low priority), we turn the cycle into a DAG by + dropping the B --> A arc, which leaves only A --> B. + + If all arcs have the same priority, we fall back to sorting by + reverse global order (the order in which modules were first + encountered). + + The algorithm is recursive, as follows: when as arcs of different + priorities are present, drop all arcs of the lowest priority, + identify SCCs in the resulting graph, and apply the algorithm to + each SCC thus found. The recursion is bounded because at each + recursion the spread in priorities is (at least) one less. + + In practice there are only a few priority levels (less than a + dozen) and in the worst case we just carry out the same algorithm + for finding SCCs N times. Thus the complexity is no worse than + the complexity of the original SCC-finding algorithm -- see + strongly_connected_components() below for a reference. + """ + if len(ascc) == 1: + return list(ascc) + pri_spread = set() + for id in ascc: + state = graph[id] + for dep in state.dependencies: + if dep in ascc: + pri = state.priorities.get(dep, PRI_HIGH) + if pri < pri_max: + pri_spread.add(pri) + if len(pri_spread) == 1: + # Filtered dependencies are uniform -- order by global order. + return sorted(ascc, key=lambda id: -graph[id].order) + pri_max = max(pri_spread) + sccs = sorted_components(graph, ascc, pri_max) + # The recursion is bounded by the len(pri_spread) check above. + return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] + + +def process_fresh_modules(graph: Graph, modules: list[str], manager: BuildManager) -> None: + """Process the modules in one group of modules from their cached data. + + This can be used to process an SCC of modules + This involves loading the tree from JSON and then doing various cleanups. + """ + t0 = time.time() + for id in modules: + graph[id].load_tree() + t1 = time.time() + for id in modules: + graph[id].fix_cross_refs() + t2 = time.time() + manager.add_stats(process_fresh_time=t2 - t0, load_tree_time=t1 - t0) + + +def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> None: + """Process the modules in one SCC from source code. + + Exception: If quick_and_dirty is set, use the cache for fresh modules. + """ + stale = scc + for id in stale: + # We may already have parsed the module, or not. + # If the former, parse_file() is a no-op. + graph[id].parse_file() + if "typing" in scc: + # For historical reasons we need to manually add typing aliases + # for built-in generic collections, see docstring of + # SemanticAnalyzerPass2.add_builtin_aliases for details. + typing_mod = graph["typing"].tree + assert typing_mod, "The typing module was not parsed" + mypy.semanal_main.semantic_analysis_for_scc(graph, scc, manager.errors) + + # Track what modules aren't yet done so we can finish them as soon + # as possible, saving memory. + unfinished_modules = set(stale) + for id in stale: + graph[id].type_check_first_pass() + if not graph[id].type_checker().deferred_nodes: + unfinished_modules.discard(id) + graph[id].detect_possibly_undefined_vars() + graph[id].finish_passes() + + while unfinished_modules: + for id in stale: + if id not in unfinished_modules: + continue + if not graph[id].type_check_second_pass(): + unfinished_modules.discard(id) + graph[id].detect_possibly_undefined_vars() + graph[id].finish_passes() + for id in stale: + graph[id].generate_unused_ignore_notes() + graph[id].generate_ignore_without_code_notes() + if any(manager.errors.is_errors_for_file(graph[id].xpath) for id in stale): + for id in stale: + graph[id].transitive_error = True + for id in stale: + if graph[id].xpath not in manager.errors.ignored_files: + errors = manager.errors.file_messages( + graph[id].xpath, formatter=manager.error_formatter + ) + manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) + graph[id].write_cache() + graph[id].mark_as_rechecked() + + +def sorted_components( + graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_ALL +) -> list[AbstractSet[str]]: + """Return the graph's SCCs, topologically sorted by dependencies. + + The sort order is from leaves (nodes without dependencies) to + roots (nodes on which no other nodes depend). + + This works for a subset of the full dependency graph too; + dependencies that aren't present in graph.keys() are ignored. + """ + # Compute SCCs. + if vertices is None: + vertices = set(graph) + edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} + sccs = list(strongly_connected_components(vertices, edges)) + # Topsort. + res = [] + for ready in topsort(prepare_sccs(sccs, edges)): + # Sort the sets in ready by reversed smallest State.order. Examples: + # + # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get + # [{y}, {x}]. + # + # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, + # c.order == 2, d.order == 4, the sort keys become [1, 2] + # and the result is [{c, d}, {a, b}]. + res.extend(sorted(ready, key=lambda scc: -min(graph[id].order for id in scc))) return res -def make_parent_dirs(path: str) -> None: - parent = os.path.dirname(path) - try: - os.makedirs(parent) - except OSError: - pass +def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> list[str]: + """Filter dependencies for id with pri < pri_max.""" + if id not in vertices: + return [] + state = graph[id] + return [ + dep + for dep in state.dependencies + if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max + ] + + +def missing_stubs_file(cache_dir: str) -> str: + return os.path.join(cache_dir, "missing_stubs") + + +def record_missing_stub_packages(cache_dir: str, missing_stub_packages: set[str]) -> None: + """Write a file containing missing stub packages. + + This allows a subsequent "mypy --install-types" run (without other arguments) + to install missing stub packages. + """ + fnam = missing_stubs_file(cache_dir) + if missing_stub_packages: + with open(fnam, "w") as f: + for pkg in sorted(missing_stub_packages): + f.write(f"{pkg}\n") + else: + if os.path.isfile(fnam): + os.remove(fnam) + + +def is_silent_import_module(manager: BuildManager, path: str) -> bool: + if manager.options.no_silence_site_packages: + return False + # Silence errors in site-package dirs and typeshed + if any(is_sub_path_normabs(path, dir) for dir in manager.search_paths.package_path): + return True + return any(is_sub_path_normabs(path, dir) for dir in manager.search_paths.typeshed_path) + + +def write_undocumented_ref_info( + state: State, metastore: MetadataStore, options: Options, type_map: dict[Expression, Type] +) -> None: + # This exports some dependency information in a rather ad-hoc fashion, which + # can be helpful for some tools. This is all highly experimental and could be + # removed at any time. + + from mypy.refinfo import get_undocumented_ref_info_json + + if not state.tree: + # We need a full AST for this. + return + + _, data_file, _ = get_cache_names(state.id, state.xpath, options) + ref_info_file = ".".join(data_file.split(".")[:-2]) + ".refs.json" + assert not ref_info_file.startswith(".") + + deps_json = get_undocumented_ref_info_json(state.tree, type_map) + metastore.write(ref_info_file, json_dumps(deps_json)) diff --git a/mypy/checker.py b/mypy/checker.py index 570ae7e2d2a0..2edcaa6bc5c5 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1,702 +1,1834 @@ """Mypy type checker.""" -import itertools +from __future__ import annotations -from typing import Undefined, Any, Dict, Set, List, cast, overload, Tuple, Function, typevar, Union +import itertools +from collections import defaultdict +from contextlib import ExitStack, contextmanager +from typing import ( + AbstractSet, + Callable, + Dict, + Final, + Generic, + Iterable, + Iterator, + Mapping, + NamedTuple, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + cast, + overload, +) +from typing_extensions import TypeAlias as _TypeAlias -from mypy.errors import Errors -from mypy.nodes import ( - SymbolTable, Node, MypyFile, VarDef, LDEF, Var, - OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, - ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, - TupleExpr, ListExpr, ParenExpr, ExpressionStmt, ReturnStmt, IfStmt, - WhileStmt, OperatorAssignmentStmt, YieldStmt, WithStmt, AssertStmt, - RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, - BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, SuperExpr, - TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, - Context, ListComprehension, ConditionalExpr, GeneratorExpr, - Decorator, SetExpr, PassStmt, TypeVarExpr, UndefinedExpr, PrintStmt, - LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, - YieldFromExpr, YieldFromStmt, NamedTupleExpr, SetComprehension, - DictionaryComprehension, ComplexExpr +import mypy.checkexpr +from mypy import errorcodes as codes, join, message_registry, nodes, operators +from mypy.binder import ConditionalTypeBinder, Frame, get_declaration +from mypy.checkmember import ( + MemberContext, + analyze_decorator_or_funcbase_access, + analyze_descriptor_access, + analyze_member_access, + type_object_type, ) -from mypy.nodes import function_type, method_type -from mypy import nodes -from mypy.types import ( - Type, AnyType, Callable, Void, FunctionLike, Overloaded, TupleType, - Instance, NoneTyp, UnboundType, ErrorType, TypeTranslator, strip_type, UnionType +from mypy.checkpattern import PatternChecker +from mypy.constraints import SUPERTYPE_OF +from mypy.erasetype import erase_type, erase_typevars, remove_instance_last_known_values +from mypy.errorcodes import TYPE_VAR, UNUSED_AWAITABLE, UNUSED_COROUTINE, ErrorCode +from mypy.errors import Errors, ErrorWatcher, report_internal_error +from mypy.expandtype import expand_self_type, expand_type, expand_type_by_instance +from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash +from mypy.maptype import map_instance_to_supertype +from mypy.meet import is_overlapping_erased_types, is_overlapping_types, meet_types +from mypy.message_registry import ErrorMessage +from mypy.messages import ( + SUGGESTED_TEST_FIXTURES, + MessageBuilder, + append_invariance_notes, + append_union_note, + format_type, + format_type_bare, + format_type_distinctly, + make_inferred_type_note, + pretty_seq, ) -from mypy.sametypes import is_same_type -from mypy.messages import MessageBuilder -import mypy.checkexpr -from mypy import messages +from mypy.mro import MroError, calculate_mro +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + ARG_STAR, + CONTRAVARIANT, + COVARIANT, + FUNC_NO_INFO, + GDEF, + IMPLICITLY_ABSTRACT, + INVARIANT, + IS_ABSTRACT, + LDEF, + LITERAL_TYPE, + MDEF, + NOT_ABSTRACT, + AssertStmt, + AssignmentExpr, + AssignmentStmt, + Block, + BreakStmt, + BytesExpr, + CallExpr, + ClassDef, + ComparisonExpr, + Context, + ContinueStmt, + Decorator, + DelStmt, + EllipsisExpr, + Expression, + ExpressionStmt, + FloatExpr, + ForStmt, + FuncBase, + FuncDef, + FuncItem, + IfStmt, + Import, + ImportAll, + ImportBase, + ImportFrom, + IndexExpr, + IntExpr, + LambdaExpr, + ListExpr, + Lvalue, + MatchStmt, + MemberExpr, + MypyFile, + NameExpr, + Node, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + PassStmt, + PromoteExpr, + RaiseStmt, + RefExpr, + ReturnStmt, + StarExpr, + Statement, + StrExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TempNode, + TryStmt, + TupleExpr, + TypeAlias, + TypeAliasStmt, + TypeInfo, + TypeVarExpr, + UnaryExpr, + Var, + WhileStmt, + WithStmt, + YieldExpr, + is_final_node, +) +from mypy.operators import flip_ops, int_op_to_method, neg_ops +from mypy.options import PRECISE_TUPLE_TYPES, Options +from mypy.patterns import AsPattern, StarredPattern +from mypy.plugin import CheckerPluginInterface, Plugin +from mypy.plugins import dataclasses as dataclasses_plugin +from mypy.scope import Scope +from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name +from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS +from mypy.sharedparse import BINARY_MAGIC_METHODS +from mypy.state import state from mypy.subtypes import ( - is_subtype, is_equivalent, is_proper_subtype, - is_more_precise, restrict_subtype_away + find_member, + infer_class_variances, + is_callable_compatible, + is_equivalent, + is_more_precise, + is_proper_subtype, + is_same_type, + is_subtype, + restrict_subtype_away, + unify_generic_callable, ) -from mypy.maptype import map_instance_to_supertype -from mypy.semanal import self_type, set_callable_name, refers_to_fullname -from mypy.erasetype import erase_typevars -from mypy.expandtype import expand_type_by_instance, expand_type -from mypy.visitor import NodeVisitor -from mypy.join import join_simple, join_types +from mypy.traverser import TraverserVisitor, all_return_statements, has_return_statement from mypy.treetransform import TransformVisitor -from mypy.meet import meet_simple, meet_simple_away, nearest_builtin_ancestor, is_overlapping_types - - -# Kinds of isinstance checks. -ISINSTANCE_OVERLAPPING = 0 -ISINSTANCE_ALWAYS_TRUE = 1 -ISINSTANCE_ALWAYS_FALSE = 2 - -T = typevar('T') - - -def min_with_None_large(x: T, y: T) -> T: - """Return min(x, y) but with a < None for all variables a that are not None""" - if x is None: - return y - return min(x, x if y is None else y) - - -class Frame(Dict[Any, Type]): - pass - - -class Key(AnyType): - pass - - -class ConditionalTypeBinder: - """Keep track of conditional types of variables.""" - - def __init__(self) -> None: - self.frames = List[Frame]() - # The first frame is special: it's the declared types of variables. - self.frames.append(Frame()) - self.dependencies = Dict[Key, Set[Key]]() # Set of other keys to invalidate if a key - # is changed - self._added_dependencies = Set[Key]() # Set of keys with dependencies added already - - self.frames_on_escape = Dict[int, List[Frame]]() - - self.try_frames = Set[int]() - self.loop_frames = List[int]() - - def _add_dependencies(self, key: Key, value: Key = None) -> None: - if value is None: - value = key - if value in self._added_dependencies: - return - self._added_dependencies.add(value) - if isinstance(key, tuple): - key = cast(Any, key) # XXX sad - if key != value: - self.dependencies[key] = Set[Key]() - self.dependencies.setdefault(key, Set[Key]()).add(value) - for elt in cast(Any, key): - self._add_dependencies(elt, value) - - def push_frame(self) -> Frame: - d = Frame() - self.frames.append(d) - return d - - def _push(self, key: Key, type: Type, index: int=-1) -> None: - self._add_dependencies(key) - self.frames[index][key] = type - - def _get(self, key: Key, index: int=-1) -> Type: - if index < 0: - index += len(self.frames) - for i in range(index, -1, -1): - if key in self.frames[i]: - return self.frames[i][key] - return None - - def push(self, expr: Node, type: Type) -> None: - if not expr.literal: - return - key = expr.literal_hash - self.frames[0][key] = self.get_declaration(expr) - self._push(key, type) - - def get(self, expr: Node) -> Type: - return self._get(expr.literal_hash) - - def update_from_options(self, frames: List[Frame]) -> bool: - """Update the frame to reflect that each key will be updated - as in one of the frames. Return whether any item changes.""" - - changed = False - keys = set(key for f in frames for key in f) - - for key in keys: - current_value = self._get(key) - resulting_values = [f.get(key, current_value) for f in frames] - if any(x is None for x in resulting_values): - continue - - type = resulting_values[0] - for other in resulting_values[1:]: - type = join_simple(self.frames[0][key], type, other) - if not is_same_type(type, current_value): - self._push(key, type) - changed = True - - return changed - - def update_expand(self, frame: Frame, index: int = -1) -> bool: - """Update frame to include another one, if that other one is larger than the current value. - - Return whether anything changed.""" - result = False - - for key in frame: - old_type = self._get(key, index) - if old_type is None: - continue - replacement = join_simple(self.frames[0][key], old_type, frame[key]) - - if not is_same_type(replacement, old_type): - self._push(key, replacement, index) - result = True - return result - - def pop_frame(self, canskip=True, fallthrough=False) -> Tuple[bool, Frame]: - """Pop a frame. - - If canskip, then allow types to skip all the inner frame - blocks. - - If fallthrough, then allow types to escape from the inner - frame to the resulting frame. - - Return whether the newly innermost frame was modified since it - was last on top, and what it would be if the block had run to - completion. - """ - result = self.frames.pop() - - options = self.frames_on_escape.get(len(self.frames) - 1, []) - if canskip: - options.append(self.frames[-1]) - if fallthrough: - options.append(result) - - changed = self.update_from_options(options) - - return (changed, result) - - def get_declaration(self, expr: Any) -> Type: - if hasattr(expr, 'node') and isinstance(expr.node, Var): - return expr.node.type - else: - return self.frames[0].get(expr.literal_hash) - - def assign_type(self, expr: Node, type: Type) -> None: - if not expr.literal: - return - self.invalidate_dependencies(expr) - - declared_type = self.get_declaration(expr) - - if declared_type is None: - # Not sure why this happens. It seems to mainly happen in - # member initialization. - return - if not is_subtype(type, declared_type): - # Pretty sure this is only happens when there's a type error. +from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type, make_optional_type +from mypy.typeops import ( + bind_self, + coerce_to_literal, + custom_special_method, + erase_def_to_union_or_bound, + erase_to_bound, + erase_to_union_or_bound, + false_only, + fixup_partial_type, + function_type, + is_literal_type_like, + is_singleton_type, + make_simplified_union, + map_type_from_supertype, + true_only, + try_expanding_sum_type_to_union, + try_getting_int_literals_from_type, + try_getting_str_literals, + try_getting_str_literals_from_type, + tuple_fallback, +) +from mypy.types import ( + ANY_STRATEGY, + MYPYC_NATIVE_INT_NAMES, + OVERLOAD_NAMES, + AnyType, + BoolTypeQuery, + CallableType, + DeletedType, + ErasedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeGuardedType, + TypeOfAny, + TypeTranslator, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + find_unpack_in_list, + flatten_nested_unions, + get_proper_type, + get_proper_types, + is_literal_type, + is_named_instance, +) +from mypy.types_utils import is_overlapping_none, remove_optional, store_argument_type, strip_type +from mypy.typetraverser import TypeTraverserVisitor +from mypy.typevars import fill_typevars, fill_typevars_with_any, has_no_typevars +from mypy.util import is_dunder, is_sunder +from mypy.visitor import NodeVisitor - # Ideally this function wouldn't be called if the - # expression has a type error, though -- do other kinds of - # errors cause this function to get called at invalid - # times? +T = TypeVar("T") - return +DEFAULT_LAST_PASS: Final = 1 # Pass numbers start at 0 - # If x is Any and y is int, after x = y we do not infer that x is int. - # This could be changed. +# Maximum length of fixed tuple types inferred when narrowing from variadic tuples. +MAX_PRECISE_TUPLE_SIZE: Final = 8 - if isinstance(self.most_recent_enclosing_type(expr, type), AnyType): - pass - elif isinstance(type, AnyType): - self.push(expr, declared_type) - else: - self.push(expr, type) +DeferredNodeType: _TypeAlias = Union[FuncDef, LambdaExpr, OverloadedFuncDef, Decorator] +FineGrainedDeferredNodeType: _TypeAlias = Union[FuncDef, MypyFile, OverloadedFuncDef] - for i in self.try_frames: - # XXX This should probably not copy the entire frame, but - # just copy this variable into a single stored frame. - self.allow_jump(i) - def invalidate_dependencies(self, expr: Node) -> None: - """Invalidate knowledge of types that include expr, but not expr itself. +# A node which is postponed to be processed during the next pass. +# In normal mode one can defer functions and methods (also decorated and/or overloaded) +# and lambda expressions. Nested functions can't be deferred -- only top-level functions +# and methods of classes not defined within a function can be deferred. +class DeferredNode(NamedTuple): + node: DeferredNodeType + # And its TypeInfo (for semantic analysis self type handling + active_typeinfo: TypeInfo | None - For example, when expr is foo.bar, invalidate foo.bar.baz and - foo.bar[0]. - It is overly conservative: it invalidates globally, including - in code paths unreachable from here. - """ - for dep in self.dependencies.get(expr.literal_hash, Set[Key]()): - for f in self.frames: - if dep in f: - del f[dep] +# Same as above, but for fine-grained mode targets. Only top-level functions/methods +# and module top levels are allowed as such. +class FineGrainedDeferredNode(NamedTuple): + node: FineGrainedDeferredNodeType + active_typeinfo: TypeInfo | None - def most_recent_enclosing_type(self, expr: Node, type: Type) -> Type: - if isinstance(type, AnyType): - return self.get_declaration(expr) - key = expr.literal_hash - enclosers = ([self.get_declaration(expr)] + - [f[key] for f in self.frames - if key in f and is_subtype(type, f[key])]) - return enclosers[-1] - def allow_jump(self, index: int) -> None: - new_frame = Frame() - for f in self.frames[index + 1:]: - for k in f: - new_frame[k] = f[k] +# Data structure returned by find_isinstance_check representing +# information learned from the truth or falsehood of a condition. The +# dict maps nodes representing expressions like 'a[0].x' to their +# refined types under the assumption that the condition has a +# particular truth value. A value of None means that the condition can +# never have that truth value. - self.frames_on_escape.setdefault(index, []).append(new_frame) +# NB: The keys of this dict are nodes in the original source program, +# which are compared by reference equality--effectively, being *the +# same* expression of the program, not just two identical expressions +# (such as two references to the same variable). TODO: it would +# probably be better to have the dict keyed by the nodes' literal_hash +# field instead. +TypeMap: _TypeAlias = Optional[Dict[Expression, Type]] - def push_loop_frame(self): - self.loop_frames.append(len(self.frames) - 1) - def pop_loop_frame(self): - self.loop_frames.pop() +# An object that represents either a precise type or a type with an upper bound; +# it is important for correct type inference with isinstance. +class TypeRange(NamedTuple): + item: Type + is_upper_bound: bool # False => precise type -def meet_frames(*frames: Frame) -> Frame: - answer = Frame() - for f in frames: - for key in f: - if key in answer: - answer[key] = meet_simple(answer[key], f[key]) - else: - answer[key] = f[key] - return answer +# Keeps track of partial types in a single scope. In fine-grained incremental +# mode partial types initially defined at the top level cannot be completed in +# a function, and we use the 'is_function' attribute to enforce this. +class PartialTypeScope(NamedTuple): + map: dict[Var, Context] + is_function: bool + is_local: bool -class TypeChecker(NodeVisitor[Type]): +class TypeChecker(NodeVisitor[None], CheckerPluginInterface): """Mypy type checker. - Type check mypy source files that have been semantically analysed. + Type check mypy source files that have been semantically analyzed. + + You must create a separate instance for each source file. """ - # Target Python major version - pyversion = 3 - # Error message reporting - errors = Undefined(Errors) - # SymbolNode table for the whole program - symtable = Undefined(SymbolTable) + # Are we type checking a stub? + is_stub = False + # Error message reporter + errors: Errors # Utility for generating messages - msg = Undefined(MessageBuilder) - # Types of type checked nodes - type_map = Undefined(Dict[Node, Type]) + msg: MessageBuilder + # Types of type checked nodes. The first item is the "master" type + # map that will store the final, exported types. Additional items + # are temporary type maps used during type inference, and these + # will be eventually popped and either discarded or merged into + # the master type map. + # + # Avoid accessing this directly, but prefer the lookup_type(), + # has_type() etc. helpers instead. + _type_maps: list[dict[Expression, Type]] # Helper for managing conditional types - binder = Undefined(ConditionalTypeBinder) + binder: ConditionalTypeBinder # Helper for type checking expressions - expr_checker = Undefined('mypy.checkexpr.ExpressionChecker') + expr_checker: mypy.checkexpr.ExpressionChecker + + pattern_checker: PatternChecker + tscope: Scope + scope: CheckerScope # Stack of function return types - return_types = Undefined(List[Type]) - # Type context for type inference - type_context = Undefined(List[Type]) + return_types: list[Type] # Flags; true for dynamically typed functions - dynamic_funcs = Undefined(List[bool]) - # Stack of functions being type checked - function_stack = Undefined(List[FuncItem]) - # Set to True on return/break/raise, False on blocks that can block any of them - breaking_out = False - - globals = Undefined(SymbolTable) - locals = Undefined(SymbolTable) - modules = Undefined(Dict[str, MypyFile]) - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile], - pyversion: int = 3) -> None: + dynamic_funcs: list[bool] + # Stack of collections of variables with partial types + partial_types: list[PartialTypeScope] + # Vars for which partial type errors are already reported + # (to avoid logically duplicate errors with different error context). + partial_reported: set[Var] + globals: SymbolTable + modules: dict[str, MypyFile] + # Nodes that couldn't be checked because some types weren't available. We'll run + # another pass and try these again. + deferred_nodes: list[DeferredNode] + # Type checking pass number (0 = first pass) + pass_num = 0 + # Last pass number to take + last_pass = DEFAULT_LAST_PASS + # Have we deferred the current function? If yes, don't infer additional + # types during this pass within the function. + current_node_deferred = False + # Is this file a typeshed stub? + is_typeshed_stub = False + options: Options + # Used for collecting inferred attribute types so that they can be checked + # for consistency. + inferred_attribute_types: dict[Var, Type] | None = None + # Don't infer partial None types if we are processing assignment from Union + no_partial_types: bool = False + + # The set of all dependencies (suppressed or not) that this module accesses, either + # directly or indirectly. + module_refs: set[str] + + # A map from variable nodes to a snapshot of the frame ids of the + # frames that were active when the variable was declared. This can + # be used to determine nearest common ancestor frame of a variable's + # declaration and the current frame, which lets us determine if it + # was declared in a different branch of the same `if` statement + # (if that frame is a conditional_frame). + var_decl_frames: dict[Var, set[int]] + + # Plugin that provides special type checking rules for specific library + # functions such as open(), etc. + plugin: Plugin + + def __init__( + self, + errors: Errors, + modules: dict[str, MypyFile], + options: Options, + tree: MypyFile, + path: str, + plugin: Plugin, + per_line_checking_time_ns: dict[int, int], + ) -> None: """Construct a type checker. - Use errors to report type check errors. Assume symtable has been - populated by the semantic analyzer. + Use errors to report type check errors. """ - self.expr_checker self.errors = errors self.modules = modules - self.pyversion = pyversion - self.msg = MessageBuilder(errors) - self.type_map = {} + self.options = options + self.tree = tree + self.path = path + self.msg = MessageBuilder(errors, modules) + self.plugin = plugin + self.tscope = Scope() + self.scope = CheckerScope(tree) self.binder = ConditionalTypeBinder() - self.binder.push_frame() - self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) + self.globals = tree.names self.return_types = [] - self.type_context = [] self.dynamic_funcs = [] - self.function_stack = [] - - def visit_file(self, file_node: MypyFile, path: str) -> None: - """Type check a mypy file with the given path.""" - self.errors.set_file(path) - self.globals = file_node.names - self.locals = None + self.partial_types = [] + self.partial_reported = set() + self.var_decl_frames = {} + self.deferred_nodes = [] + self._type_maps = [{}] + self.module_refs = set() + self.pass_num = 0 + self.current_node_deferred = False + self.is_stub = tree.is_stub + self.is_typeshed_stub = tree.is_typeshed_file(options) + self.inferred_attribute_types = None + + # If True, process function definitions. If False, don't. This is used + # for processing module top levels in fine-grained incremental mode. + self.recurse_into_functions = True + # This internal flag is used to track whether we a currently type-checking + # a final declaration (assignment), so that some errors should be suppressed. + # Should not be set manually, use get_final_context/enter_final_context instead. + # NOTE: we use the context manager to avoid "threading" an additional `is_final_def` + # argument through various `checker` and `checkmember` functions. + self._is_final_def = False + + # This flag is set when we run type-check or attribute access check for the purpose + # of giving a note on possibly missing "await". It is used to avoid infinite recursion. + self.checking_missing_await = False + + # While this is True, allow passing an abstract class where Type[T] is expected. + # although this is technically unsafe, this is desirable in some context, for + # example when type-checking class decorators. + self.allow_abstract_call = False + + # Child checker objects for specific AST node types + self.expr_checker = mypy.checkexpr.ExpressionChecker( + self, self.msg, self.plugin, per_line_checking_time_ns + ) + self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) + + @property + def type_context(self) -> list[Type | None]: + return self.expr_checker.type_context + + def reset(self) -> None: + """Cleanup stale state that might be left over from a typechecking run. + + This allows us to reuse TypeChecker objects in fine-grained + incremental mode. + """ + # TODO: verify this is still actually worth it over creating new checkers + self.partial_reported.clear() + self.module_refs.clear() + self.binder = ConditionalTypeBinder() + self._type_maps[1:] = [] + self._type_maps[0].clear() + self.temp_type_map = None + self.expr_checker.reset() + + assert self.inferred_attribute_types is None + assert self.partial_types == [] + assert self.deferred_nodes == [] + assert len(self.scope.stack) == 1 + assert self.partial_types == [] + + def check_first_pass(self) -> None: + """Type check the entire file, but defer functions with unresolved references. + + Unresolved references are forward references to variables + whose types haven't been inferred yet. They may occur later + in the same file or in a different file that's being processed + later (usually due to an import cycle). + + Deferred functions will be processed by check_second_pass(). + """ + self.recurse_into_functions = True + with state.strict_optional_set(self.options.strict_optional): + self.errors.set_file( + self.path, self.tree.fullname, scope=self.tscope, options=self.options + ) + with self.tscope.module_scope(self.tree.fullname): + with self.enter_partial_types(), self.binder.top_frame_context(): + for d in self.tree.defs: + if self.binder.is_unreachable(): + if not self.should_report_unreachable_issues(): + break + if not self.is_noop_for_reachability(d): + self.msg.unreachable_statement(d) + break + else: + self.accept(d) + + assert not self.current_node_deferred + + all_ = self.globals.get("__all__") + if all_ is not None and all_.type is not None: + all_node = all_.node + assert all_node is not None + seq_str = self.named_generic_type( + "typing.Sequence", [self.named_type("builtins.str")] + ) + if not is_subtype(all_.type, seq_str): + str_seq_s, all_s = format_type_distinctly( + seq_str, all_.type, options=self.options + ) + self.fail( + message_registry.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), all_node + ) - for d in file_node.defs: - self.accept(d) + def check_second_pass( + self, todo: Sequence[DeferredNode | FineGrainedDeferredNode] | None = None + ) -> bool: + """Run second or following pass of type checking. - def accept(self, node: Node, type_context: Type = None) -> Type: - """Type check a node in the given type context.""" - self.type_context.append(type_context) - typ = node.accept(self) - self.type_context.pop() - self.store_type(node, typ) - if self.is_dynamic_function(): - return AnyType() + This goes through deferred nodes, returning True if there were any. + """ + self.recurse_into_functions = True + with state.strict_optional_set(self.options.strict_optional): + if not todo and not self.deferred_nodes: + return False + self.errors.set_file( + self.path, self.tree.fullname, scope=self.tscope, options=self.options + ) + with self.tscope.module_scope(self.tree.fullname): + self.pass_num += 1 + if not todo: + todo = self.deferred_nodes + else: + assert not self.deferred_nodes + self.deferred_nodes = [] + done: set[DeferredNodeType | FineGrainedDeferredNodeType] = set() + for node, active_typeinfo in todo: + if node in done: + continue + # This is useful for debugging: + # print("XXX in pass %d, class %s, function %s" % + # (self.pass_num, type_name, node.fullname or node.name)) + done.add(node) + with ExitStack() as stack: + if active_typeinfo: + stack.enter_context(self.tscope.class_scope(active_typeinfo)) + stack.enter_context(self.scope.push_class(active_typeinfo)) + self.check_partial(node) + return True + + def check_partial(self, node: DeferredNodeType | FineGrainedDeferredNodeType) -> None: + if isinstance(node, MypyFile): + self.check_top_level(node) else: - return typ - - def accept_in_frame(self, node: Node, type_context: Type = None, - repeat_till_fixed: bool = False) -> Type: - """Type check a node in the given type context in a new frame of inferred types.""" - while True: - self.binder.push_frame() - answer = self.accept(node, type_context) - changed, _ = self.binder.pop_frame(True, True) - self.breaking_out = False - if not repeat_till_fixed or not changed: - break + self.recurse_into_functions = True + with self.binder.top_frame_context(): + if isinstance(node, LambdaExpr): + self.expr_checker.accept(node) + else: + self.accept(node) + + def check_top_level(self, node: MypyFile) -> None: + """Check only the top-level of a module, skipping function definitions.""" + self.recurse_into_functions = False + with self.enter_partial_types(): + with self.binder.top_frame_context(): + for d in node.defs: + d.accept(self) + + assert not self.current_node_deferred + # TODO: Handle __all__ + + def defer_node(self, node: DeferredNodeType, enclosing_class: TypeInfo | None) -> None: + """Defer a node for processing during next type-checking pass. + + Args: + node: function/method being deferred + enclosing_class: for methods, the class where the method is defined + NOTE: this can't handle nested functions/methods. + """ + # We don't freeze the entire scope since only top-level functions and methods + # can be deferred. Only module/class level scope information is needed. + # Module-level scope information is preserved in the TypeChecker instance. + self.deferred_nodes.append(DeferredNode(node, enclosing_class)) + + def handle_cannot_determine_type(self, name: str, context: Context) -> None: + node = self.scope.top_non_lambda_function() + if self.pass_num < self.last_pass and isinstance(node, FuncDef): + # Don't report an error yet. Just defer. Note that we don't defer + # lambdas because they are coupled to the surrounding function + # through the binder and the inferred type of the lambda, so it + # would get messy. + enclosing_class = self.scope.enclosing_class() + self.defer_node(node, enclosing_class) + # Set a marker so that we won't infer additional types in this + # function. Any inferred types could be bogus, because there's at + # least one type that we don't know. + self.current_node_deferred = True + else: + self.msg.cannot_determine_type(name, context) - return answer + def accept(self, stmt: Statement) -> None: + """Type check a node in the given type context.""" + try: + stmt.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, stmt.line, self.errors, self.options) + + def accept_loop( + self, + body: Statement, + else_body: Statement | None = None, + *, + exit_condition: Expression | None = None, + ) -> None: + """Repeatedly type check a loop body until the frame doesn't change. + If exit_condition is set, assume it must be False on exit from the loop. + + Then check the else_body. + """ + # The outer frame accumulates the results of all iterations + with self.binder.frame_context(can_skip=False, conditional_frame=True): + while True: + with self.binder.frame_context(can_skip=True, break_frame=2, continue_frame=1): + self.accept(body) + if not self.binder.last_pop_changed: + break + if exit_condition: + _, else_map = self.find_isinstance_check(exit_condition) + self.push_type_map(else_map) + if else_body: + self.accept(else_body) # # Definitions # - def visit_var_def(self, defn: VarDef) -> Type: - """Type check a variable definition. - - It can be of any kind: local, member or global. - """ - # Type check initializer. - if defn.init: - # There is an initializer. - if defn.items[0].type: - # Explicit types. - if len(defn.items) == 1: - self.check_simple_assignment(defn.items[0].type, - defn.init, defn.init) - else: - # Multiple assignment. - lv = List[Node]() - for v in defn.items: - lv.append(self.temp_node(v.type, v)) - self.check_multi_assignment(lv, defn.init, defn.init) - else: - init_type = self.accept(defn.init) - if defn.kind == LDEF and not defn.is_top_level: - # Infer local variable type if there is an initializer - # except if the definition is at the top level (outside a - # function). - self.infer_local_variable_type(defn.items, init_type, defn) - else: - # No initializer - if (defn.kind == LDEF and not defn.items[0].type and - not defn.is_top_level and not self.is_dynamic_function()): - self.fail(messages.NEED_ANNOTATION_FOR_VAR, defn) - - def infer_local_variable_type(self, x, y, z): - # TODO - raise RuntimeError('Not implemented') + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + if not self.recurse_into_functions: + return + with self.tscope.function_scope(defn): + self._visit_overloaded_func_def(defn) - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: + def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: num_abstract = 0 + if not defn.items: + # In this case we have already complained about none of these being + # valid overloads. + return + if len(defn.items) == 1: + self.fail(message_registry.MULTIPLE_OVERLOADS_REQUIRED, defn) + + if defn.is_property: + # HACK: Infer the type of the property. + assert isinstance(defn.items[0], Decorator) + self.visit_decorator(defn.items[0]) for fdef in defn.items: - self.check_func_item(fdef.func, name=fdef.func.name()) - if fdef.func.is_abstract: + assert isinstance(fdef, Decorator) + if defn.is_property: + self.check_func_item(fdef.func, name=fdef.func.name, allow_empty=True) + else: + # Perform full check for real overloads to infer type of all decorated + # overload variants. + self.visit_decorator_inner(fdef, allow_empty=True) + if fdef.func.abstract_status in (IS_ABSTRACT, IMPLICITLY_ABSTRACT): num_abstract += 1 if num_abstract not in (0, len(defn.items)): - self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) + self.fail(message_registry.INCONSISTENT_ABSTRACT_OVERLOAD, defn) + if defn.impl: + defn.impl.accept(self) + if not defn.is_property: + self.check_overlapping_overloads(defn) + if defn.type is None: + item_types = [] + for item in defn.items: + assert isinstance(item, Decorator) + item_type = self.extract_callable_type(item.var.type, item) + if item_type is not None: + item_types.append(item_type) + if item_types: + defn.type = Overloaded(item_types) + # Check override validity after we analyzed current definition. if defn.info: - self.check_method_override(defn) + found_method_base_classes = self.check_method_override(defn) + if ( + defn.is_explicit_override + and not found_method_base_classes + and found_method_base_classes is not None + ): + self.msg.no_overridable_method(defn.name, defn) + self.check_explicit_override_decorator(defn, found_method_base_classes, defn.impl) self.check_inplace_operator_method(defn) - self.check_overlapping_overloads(defn) + + def extract_callable_type(self, inner_type: Type | None, ctx: Context) -> CallableType | None: + """Get type as seen by an overload item caller.""" + inner_type = get_proper_type(inner_type) + outer_type: CallableType | None = None + if inner_type is not None and not isinstance(inner_type, AnyType): + if isinstance(inner_type, TypeVarLikeType): + inner_type = get_proper_type(inner_type.upper_bound) + if isinstance(inner_type, TypeType): + inner_type = get_proper_type( + self.expr_checker.analyze_type_type_callee(inner_type.item, ctx) + ) + + if isinstance(inner_type, CallableType): + outer_type = inner_type + elif isinstance(inner_type, Instance): + inner_call = get_proper_type( + analyze_member_access( + name="__call__", + typ=inner_type, + context=ctx, + is_lvalue=False, + is_super=False, + is_operator=True, + msg=self.msg, + original_type=inner_type, + chk=self, + ) + ) + if isinstance(inner_call, CallableType): + outer_type = inner_call + elif isinstance(inner_type, UnionType): + union_type = make_simplified_union(inner_type.items) + if isinstance(union_type, UnionType): + items = [] + for item in union_type.items: + callable_item = self.extract_callable_type(item, ctx) + if callable_item is None: + break + items.append(callable_item) + else: + joined_type = get_proper_type(join.join_type_list(items)) + if isinstance(joined_type, CallableType): + outer_type = joined_type + else: + return self.extract_callable_type(union_type, ctx) + if outer_type is None: + self.msg.not_callable(inner_type, ctx) + return outer_type def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: + # At this point we should have set the impl already, and all remaining + # items are decorators + + if self.msg.errors.file in self.msg.errors.ignored_files or ( + self.is_typeshed_stub and self.options.test_env + ): + # This is a little hacky, however, the quadratic check here is really expensive, this + # method has no side effects, so we should skip it if we aren't going to report + # anything. In some other places we swallow errors in stubs, but this error is very + # useful for stubs! + return + + # Compute some info about the implementation (if it exists) for use below + impl_type: CallableType | None = None + if defn.impl: + if isinstance(defn.impl, FuncDef): + inner_type: Type | None = defn.impl.type + elif isinstance(defn.impl, Decorator): + inner_type = defn.impl.var.type + else: + assert False, "Impl isn't the right type" + + # This can happen if we've got an overload with a different + # decorator or if the implementation is untyped -- we gave up on the types. + impl_type = self.extract_callable_type(inner_type, defn.impl) + + is_descriptor_get = defn.info and defn.name == "__get__" for i, item in enumerate(defn.items): - for j, item2 in enumerate(defn.items[i + 1:]): - # TODO overloads involving decorators - sig1 = self.function_type(item.func) - sig2 = self.function_type(item2.func) - if is_unsafe_overlapping_signatures(sig1, sig2): - self.msg.overloaded_signatures_overlap(i + 1, j + 2, - item.func) - - def visit_func_def(self, defn: FuncDef) -> Type: + assert isinstance(item, Decorator) + sig1 = self.extract_callable_type(item.var.type, item) + if sig1 is None: + continue + + for j, item2 in enumerate(defn.items[i + 1 :]): + assert isinstance(item2, Decorator) + sig2 = self.extract_callable_type(item2.var.type, item2) + if sig2 is None: + continue + + if not are_argument_counts_overlapping(sig1, sig2): + continue + + if overload_can_never_match(sig1, sig2): + self.msg.overloaded_signature_will_never_match(i + 1, i + j + 2, item2.func) + elif not is_descriptor_get: + # Note: we force mypy to check overload signatures in strict-optional mode + # so we don't incorrectly report errors when a user tries typing an overload + # that happens to have a 'if the argument is None' fallback. + # + # For example, the following is fine in strict-optional mode but would throw + # the unsafe overlap error when strict-optional is disabled: + # + # @overload + # def foo(x: None) -> int: ... + # @overload + # def foo(x: str) -> str: ... + # + # See Python 2's map function for a concrete example of this kind of overload. + current_class = self.scope.active_class() + type_vars = current_class.defn.type_vars if current_class else [] + with state.strict_optional_set(True): + if is_unsafe_overlapping_overload_signatures(sig1, sig2, type_vars): + flip_note = ( + j == 0 + and not is_unsafe_overlapping_overload_signatures( + sig2, sig1, type_vars + ) + and not overload_can_never_match(sig2, sig1) + ) + self.msg.overloaded_signatures_overlap( + i + 1, i + j + 2, flip_note, item.func + ) + + if impl_type is not None: + assert defn.impl is not None + + # This is what we want from implementation, it should accept all arguments + # of an overload, but the return types should go the opposite way. + if is_callable_compatible( + impl_type, + sig1, + is_compat=is_subtype, + is_proper_subtype=False, + is_compat_return=lambda l, r: is_subtype(r, l), + ): + continue + # If the above check didn't work, we repeat some key steps in + # is_callable_compatible() to give a better error message. + + # We perform a unification step that's very similar to what + # 'is_callable_compatible' does -- the only difference is that + # we check and see if the impl_type's return value is a + # *supertype* of the overload alternative, not a *subtype*. + # + # This is to match the direction the implementation's return + # needs to be compatible in. + if impl_type.variables: + impl: CallableType | None = unify_generic_callable( + # Normalize both before unifying + impl_type.with_unpacked_kwargs(), + sig1.with_unpacked_kwargs(), + ignore_return=False, + return_constraint_direction=SUPERTYPE_OF, + ) + if impl is None: + self.msg.overloaded_signatures_typevar_specific(i + 1, defn.impl) + continue + else: + impl = impl_type + + # Prevent extra noise from inconsistent use of @classmethod by copying + # the first arg from the method being checked against. + if sig1.arg_types and defn.info: + impl = impl.copy_modified(arg_types=[sig1.arg_types[0]] + impl.arg_types[1:]) + + # Is the overload alternative's arguments subtypes of the implementation's? + if not is_callable_compatible( + impl, sig1, is_compat=is_subtype, is_proper_subtype=False, ignore_return=True + ): + self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl) + + # Is the overload alternative's return type a subtype of the implementation's? + if not ( + is_subtype(sig1.ret_type, impl.ret_type) + or is_subtype(impl.ret_type, sig1.ret_type) + ): + self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl) + + # Here's the scoop about generators and coroutines. + # + # There are two kinds of generators: classic generators (functions + # with `yield` or `yield from` in the body) and coroutines + # (functions declared with `async def`). The latter are specified + # in PEP 492 and only available in Python >= 3.5. + # + # Classic generators can be parameterized with three types: + # - ty is the Yield type (the type of y in `yield y`) + # - tc is the type reCeived by yield (the type of c in `c = yield`). + # - tr is the Return type (the type of r in `return r`) + # + # A classic generator must define a return type that's either + # `Generator[ty, tc, tr]`, Iterator[ty], or Iterable[ty] (or + # object or Any). If tc/tr are not given, both are None. + # + # A coroutine must define a return type corresponding to tr; the + # other two are unconstrained. The "external" return type (seen + # by the caller) is Awaitable[tr]. + # + # In addition, there's the synthetic type AwaitableGenerator: it + # inherits from both Awaitable and Generator and can be used both + # in `yield from` and in `await`. This type is set automatically + # for functions decorated with `@types.coroutine` or + # `@asyncio.coroutine`. Its single parameter corresponds to tr. + # + # PEP 525 adds a new type, the asynchronous generator, which was + # first released in Python 3.6. Async generators are `async def` + # functions that can also `yield` values. They can be parameterized + # with two types, ty and tc, because they cannot return a value. + # + # There are several useful methods, each taking a type t and a + # flag c indicating whether it's for a generator or coroutine: + # + # - is_generator_return_type(t, c) returns whether t is a Generator, + # Iterator, Iterable (if not c), or Awaitable (if c), or + # AwaitableGenerator (regardless of c). + # - is_async_generator_return_type(t) returns whether t is an + # AsyncGenerator. + # - get_generator_yield_type(t, c) returns ty. + # - get_generator_receive_type(t, c) returns tc. + # - get_generator_return_type(t, c) returns tr. + + def is_generator_return_type(self, typ: Type, is_coroutine: bool) -> bool: + """Is `typ` a valid type for a generator/coroutine? + + True if `typ` is a *supertype* of Generator or Awaitable. + Also true it it's *exactly* AwaitableGenerator (modulo type parameters). + """ + typ = get_proper_type(typ) + if is_coroutine: + # This means we're in Python 3.5 or later. + at = self.named_generic_type("typing.Awaitable", [AnyType(TypeOfAny.special_form)]) + if is_subtype(at, typ): + return True + else: + any_type = AnyType(TypeOfAny.special_form) + gt = self.named_generic_type("typing.Generator", [any_type, any_type, any_type]) + if is_subtype(gt, typ): + return True + return isinstance(typ, Instance) and typ.type.fullname == "typing.AwaitableGenerator" + + def is_async_generator_return_type(self, typ: Type) -> bool: + """Is `typ` a valid type for an async generator? + + True if `typ` is a supertype of AsyncGenerator. + """ + try: + any_type = AnyType(TypeOfAny.special_form) + agt = self.named_generic_type("typing.AsyncGenerator", [any_type, any_type]) + except KeyError: + # we're running on a version of typing that doesn't have AsyncGenerator yet + return False + return is_subtype(agt, typ) + + def get_generator_yield_type(self, return_type: Type, is_coroutine: bool) -> Type: + """Given the declared return type of a generator (t), return the type it yields (ty).""" + return_type = get_proper_type(return_type) + + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + elif isinstance(return_type, UnionType): + return make_simplified_union( + [self.get_generator_yield_type(item, is_coroutine) for item in return_type.items] + ) + elif not self.is_generator_return_type( + return_type, is_coroutine + ) and not self.is_async_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or + # Awaitable) return type, anything is permissible. + return AnyType(TypeOfAny.from_error) + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType(TypeOfAny.from_error) + elif return_type.type.fullname == "typing.Awaitable": + # Awaitable: ty is Any. + return AnyType(TypeOfAny.special_form) + elif return_type.args: + # AwaitableGenerator, Generator, AsyncGenerator, Iterator, or Iterable; ty is args[0]. + ret_type = return_type.args[0] + # TODO not best fix, better have dedicated yield token + return ret_type + else: + # If the function's declared supertype of Generator has no type + # parameters (i.e. is `object`), then the yielded values can't + # be accessed so any type is acceptable. IOW, ty is Any. + # (However, see https://github.com/python/mypy/issues/1933) + return AnyType(TypeOfAny.special_form) + + def get_generator_receive_type(self, return_type: Type, is_coroutine: bool) -> Type: + """Given a declared generator return type (t), return the type its yield receives (tc).""" + return_type = get_proper_type(return_type) + + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + elif isinstance(return_type, UnionType): + return make_simplified_union( + [self.get_generator_receive_type(item, is_coroutine) for item in return_type.items] + ) + elif not self.is_generator_return_type( + return_type, is_coroutine + ) and not self.is_async_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or + # Awaitable) return type, anything is permissible. + return AnyType(TypeOfAny.from_error) + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType(TypeOfAny.from_error) + elif return_type.type.fullname == "typing.Awaitable": + # Awaitable, AwaitableGenerator: tc is Any. + return AnyType(TypeOfAny.special_form) + elif ( + return_type.type.fullname in ("typing.Generator", "typing.AwaitableGenerator") + and len(return_type.args) >= 3 + ): + # Generator: tc is args[1]. + return return_type.args[1] + elif return_type.type.fullname == "typing.AsyncGenerator" and len(return_type.args) >= 2: + return return_type.args[1] + else: + # `return_type` is a supertype of Generator, so callers won't be able to send it + # values. IOW, tc is None. + return NoneType() + + def get_coroutine_return_type(self, return_type: Type) -> Type: + return_type = get_proper_type(return_type) + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + assert isinstance(return_type, Instance), "Should only be called on coroutine functions." + # Note: return type is the 3rd type parameter of Coroutine. + return return_type.args[2] + + def get_generator_return_type(self, return_type: Type, is_coroutine: bool) -> Type: + """Given the declared return type of a generator (t), return the type it returns (tr).""" + return_type = get_proper_type(return_type) + + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + elif isinstance(return_type, UnionType): + return make_simplified_union( + [self.get_generator_return_type(item, is_coroutine) for item in return_type.items] + ) + elif not self.is_generator_return_type(return_type, is_coroutine): + # If the function doesn't have a proper Generator (or + # Awaitable) return type, anything is permissible. + return AnyType(TypeOfAny.from_error) + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType(TypeOfAny.from_error) + elif return_type.type.fullname == "typing.Awaitable" and len(return_type.args) == 1: + # Awaitable: tr is args[0]. + return return_type.args[0] + elif ( + return_type.type.fullname in ("typing.Generator", "typing.AwaitableGenerator") + and len(return_type.args) >= 3 + ): + # AwaitableGenerator, Generator: tr is args[2]. + return return_type.args[2] + else: + # We have a supertype of Generator (Iterator, Iterable, object) + # Treat `Iterator[X]` as a shorthand for `Generator[X, Any, None]`. + return NoneType() + + def visit_func_def(self, defn: FuncDef) -> None: + if not self.recurse_into_functions: + return + with self.tscope.function_scope(defn): + self._visit_func_def(defn) + + def _visit_func_def(self, defn: FuncDef) -> None: """Type check a function definition.""" - self.check_func_item(defn, name=defn.name()) + self.check_func_item(defn, name=defn.name) if defn.info: - self.check_method_override(defn) + if not defn.is_overload and not defn.is_decorated: + # If the definition is the implementation for an + # overload, the legality of the override has already + # been typechecked, and decorated methods will be + # checked when the decorator is. + found_method_base_classes = self.check_method_override(defn) + self.check_explicit_override_decorator(defn, found_method_base_classes) self.check_inplace_operator_method(defn) if defn.original_def: - if not is_same_type(self.function_type(defn), - self.function_type(defn.original_def)): - self.msg.incompatible_conditional_function_def(defn) - - def check_func_item(self, defn: FuncItem, - type_override: Callable = None, - name: str = None) -> Type: + # Override previous definition. + new_type = self.function_type(defn) + self.check_func_def_override(defn, new_type) + + def check_func_item( + self, + defn: FuncItem, + type_override: CallableType | None = None, + name: str | None = None, + allow_empty: bool = False, + ) -> None: """Type check a function. If type_override is provided, use it as the function type. """ - # We may be checking a function definition or an anonymous function. In - # the first case, set up another reference with the precise type. - fdef = None # type: FuncDef - if isinstance(defn, FuncDef): - fdef = defn - - self.function_stack.append(defn) - self.dynamic_funcs.append(defn.type is None and not type_override) - - if fdef: - self.errors.push_function(fdef.name()) - - typ = self.function_type(defn) - if type_override: - typ = type_override - if isinstance(typ, Callable): - self.check_func_def(defn, typ, name) - else: - raise RuntimeError('Not supported') - - if fdef: - self.errors.pop_function() + self.dynamic_funcs.append(defn.is_dynamic() and not type_override) + + with self.enter_partial_types(is_function=True): + typ = self.function_type(defn) + if type_override: + typ = type_override.copy_modified(line=typ.line, column=typ.column) + if isinstance(typ, CallableType): + with self.enter_attribute_inference_context(): + self.check_func_def(defn, typ, name, allow_empty) + else: + raise RuntimeError("Not supported") self.dynamic_funcs.pop() - self.function_stack.pop() - - def check_func_def(self, defn: FuncItem, typ: Callable, name: str) -> None: + self.current_node_deferred = False + + if name == "__exit__": + self.check__exit__return_type(defn) + # TODO: the following logic should move to the dataclasses plugin + # https://github.com/python/mypy/issues/15515 + if name == "__post_init__": + if dataclasses_plugin.is_processed_dataclass(defn.info): + dataclasses_plugin.check_post_init(self, defn, defn.info) + + def check_func_def_override(self, defn: FuncDef, new_type: FunctionLike) -> None: + assert defn.original_def is not None + if isinstance(defn.original_def, FuncDef): + # Function definition overrides function definition. + old_type = self.function_type(defn.original_def) + if not is_same_type(new_type, old_type): + self.msg.incompatible_conditional_function_def(defn, old_type, new_type) + else: + # Function definition overrides a variable initialized via assignment or a + # decorated function. + orig_type = defn.original_def.type + if orig_type is None: + # If other branch is unreachable, we don't type check it and so we might + # not have a type for the original definition + return + if isinstance(orig_type, PartialType): + if orig_type.type is None: + # Ah this is a partial type. Give it the type of the function. + orig_def = defn.original_def + if isinstance(orig_def, Decorator): + var = orig_def.var + else: + var = orig_def + partial_types = self.find_partial_types(var) + if partial_types is not None: + var.type = new_type + del partial_types[var] + else: + # Trying to redefine something like partial empty list as function. + self.fail(message_registry.INCOMPATIBLE_REDEFINITION, defn) + else: + name_expr = NameExpr(defn.name) + name_expr.node = defn.original_def + self.binder.assign_type(name_expr, new_type, orig_type) + self.check_subtype( + new_type, + orig_type, + defn, + message_registry.INCOMPATIBLE_REDEFINITION, + "redefinition with type", + "original type", + ) + + @contextmanager + def enter_attribute_inference_context(self) -> Iterator[None]: + old_types = self.inferred_attribute_types + self.inferred_attribute_types = {} + yield None + self.inferred_attribute_types = old_types + + def check_func_def( + self, defn: FuncItem, typ: CallableType, name: str | None, allow_empty: bool = False + ) -> None: """Type check a function definition.""" # Expand type variables with value restrictions to ordinary types. - for item, typ in self.expand_typevars(defn, typ): + self.check_typevar_defaults(typ.variables) + expanded = self.expand_typevars(defn, typ) + original_typ = typ + for item, typ in expanded: old_binder = self.binder self.binder = ConditionalTypeBinder() - self.binder.push_frame() - defn.expanded.append(item) - - # We may be checking a function definition or an anonymous - # function. In the first case, set up another reference with the - # precise type. - if isinstance(item, FuncDef): - fdef = item - else: - fdef = None - - self.enter() - - if fdef: - # Check if __init__ has an invalid, non-None return type. - if (fdef.info and fdef.name() == '__init__' and - not isinstance(typ.ret_type, Void) and - not self.dynamic_funcs[-1]): - self.fail(messages.INIT_MUST_NOT_HAVE_RETURN_TYPE, - item.type) - - if name in nodes.reverse_op_method_set: - self.check_reverse_op_method(item, typ, name) - - # Push return type. - self.return_types.append(typ.ret_type) - - # Store argument types. - nargs = len(item.args) - for i in range(len(typ.arg_types)): - arg_type = typ.arg_types[i] - if typ.arg_kinds[i] == nodes.ARG_STAR: - arg_type = self.named_generic_type('builtins.list', - [arg_type]) - elif typ.arg_kinds[i] == nodes.ARG_STAR2: - arg_type = self.named_generic_type('builtins.dict', - [self.str_type(), - arg_type]) - item.args[i].type = arg_type - - # Type check initialization expressions. - for j in range(len(item.init)): - if item.init[j]: - self.accept(item.init[j]) - - # Clear out the default assignments from the binder - self.binder.pop_frame() - self.binder.push_frame() + with self.binder.top_frame_context(): + defn.expanded.append(item) + + # We may be checking a function definition or an anonymous + # function. In the first case, set up another reference with the + # precise type. + if isinstance(item, FuncDef): + fdef = item + # Check if __init__ has an invalid return type. + if ( + fdef.info + and fdef.name in ("__init__", "__init_subclass__") + and not isinstance( + get_proper_type(typ.ret_type), (NoneType, UninhabitedType) + ) + and not self.dynamic_funcs[-1] + ): + self.fail( + message_registry.MUST_HAVE_NONE_RETURN_TYPE.format(fdef.name), item + ) + + # Check validity of __new__ signature + if fdef.info and fdef.name == "__new__": + self.check___new___signature(fdef, typ) + + self.check_for_missing_annotations(fdef) + if self.options.disallow_any_unimported: + if fdef.type and isinstance(fdef.type, CallableType): + ret_type = fdef.type.ret_type + if has_any_from_unimported_type(ret_type): + self.msg.unimported_type_becomes_any("Return type", ret_type, fdef) + for idx, arg_type in enumerate(fdef.type.arg_types): + if has_any_from_unimported_type(arg_type): + prefix = f'Argument {idx + 1} to "{fdef.name}"' + self.msg.unimported_type_becomes_any(prefix, arg_type, fdef) + check_for_explicit_any( + fdef.type, self.options, self.is_typeshed_stub, self.msg, context=fdef + ) + + if name: # Special method names + if defn.info and self.is_reverse_op_method(name): + self.check_reverse_op_method(item, typ, name, defn) + elif name in ("__getattr__", "__getattribute__"): + self.check_getattr_method(typ, defn, name) + elif name == "__setattr__": + self.check_setattr_method(typ, defn) + + # Refuse contravariant return type variable + if isinstance(typ.ret_type, TypeVarType): + if typ.ret_type.variance == CONTRAVARIANT: + self.fail( + message_registry.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, typ.ret_type + ) + self.check_unbound_return_typevar(typ) + elif ( + isinstance(original_typ.ret_type, TypeVarType) and original_typ.ret_type.values + ): + # Since type vars with values are expanded, the return type is changed + # to a raw value. This is a hack to get it back. + self.check_unbound_return_typevar(original_typ) + + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if defn.is_async_generator: + if not self.is_async_generator_return_type(typ.ret_type): + self.fail( + message_registry.INVALID_RETURN_TYPE_FOR_ASYNC_GENERATOR, typ + ) + else: + if not self.is_generator_return_type(typ.ret_type, defn.is_coroutine): + self.fail(message_registry.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + # Fix the type if decorated with `@types.coroutine` or `@asyncio.coroutine`. + if defn.is_awaitable_coroutine: + # Update the return type to AwaitableGenerator. + # (This doesn't exist in typing.py, only in typing.pyi.) + t = typ.ret_type + c = defn.is_coroutine + ty = self.get_generator_yield_type(t, c) + tc = self.get_generator_receive_type(t, c) + if c: + tr = self.get_coroutine_return_type(t) + else: + tr = self.get_generator_return_type(t, c) + ret_type = self.named_generic_type( + "typing.AwaitableGenerator", [ty, tc, tr, t] + ) + typ = typ.copy_modified(ret_type=ret_type) + defn.type = typ + + # Push return type. + self.return_types.append(typ.ret_type) + + with self.scope.push_function(defn): + # We temporary push the definition to get the self type as + # visible from *inside* of this function/method. + ref_type: Type | None = self.scope.active_self_type() + + if typ.type_is: + arg_index = 0 + # For methods and classmethods, we want the second parameter + if ref_type is not None and (not defn.is_static or defn.name == "__new__"): + arg_index = 1 + if arg_index < len(typ.arg_types) and not is_subtype( + typ.type_is, typ.arg_types[arg_index] + ): + self.fail( + message_registry.NARROWED_TYPE_NOT_SUBTYPE.format( + format_type(typ.type_is, self.options), + format_type(typ.arg_types[arg_index], self.options), + ), + item, + ) + + # Store argument types. + for i in range(len(typ.arg_types)): + arg_type = typ.arg_types[i] + if ( + isinstance(defn, FuncDef) + and ref_type is not None + and i == 0 + and (not defn.is_static or defn.name == "__new__") + and typ.arg_kinds[0] not in [nodes.ARG_STAR, nodes.ARG_STAR2] + ): + if defn.is_class or defn.name == "__new__": + ref_type = mypy.types.TypeType.make_normalized(ref_type) + if not is_same_type(arg_type, ref_type): + # This level of erasure matches the one in checkmember.check_self_arg(), + # better keep these two checks consistent. + erased = get_proper_type(erase_typevars(erase_to_bound(arg_type))) + if not is_subtype(ref_type, erased, ignore_type_params=True): + if ( + isinstance(erased, Instance) + and erased.type.is_protocol + or isinstance(erased, TypeType) + and isinstance(erased.item, Instance) + and erased.item.type.is_protocol + ): + # We allow the explicit self-type to be not a supertype of + # the current class if it is a protocol. For such cases + # the consistency check will be performed at call sites. + msg = None + elif typ.arg_names[i] in {"self", "cls"}: + msg = message_registry.ERASED_SELF_TYPE_NOT_SUPERTYPE.format( + erased.str_with_options(self.options), + ref_type.str_with_options(self.options), + ) + else: + msg = message_registry.MISSING_OR_INVALID_SELF_TYPE + if msg: + self.fail(msg, defn) + elif isinstance(arg_type, TypeVarType): + # Refuse covariant parameter type variables + # TODO: check recursively for inner type variables + if ( + arg_type.variance == COVARIANT + and defn.name not in ("__init__", "__new__", "__post_init__") + and not is_private(defn.name) # private methods are not inherited + ): + ctx: Context = arg_type + if ctx.line < 0: + ctx = typ + self.fail(message_registry.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, ctx) + # Need to store arguments again for the expanded item. + store_argument_type(item, i, typ, self.named_generic_type) + + # Type check initialization expressions. + body_is_trivial = is_trivial_body(defn.body) + self.check_default_args(item, body_is_trivial) + # Type check body in a new scope. - self.accept_in_frame(item.body) + with self.binder.top_frame_context(): + # Copy some type narrowings from an outer function when it seems safe enough + # (i.e. we can't find an assignment that might change the type of the + # variable afterwards). + new_frame: Frame | None = None + for frame in old_binder.frames: + for key, narrowed_type in frame.types.items(): + key_var = extract_var_from_literal_hash(key) + if key_var is not None and not self.is_var_redefined_in_outer_context( + key_var, defn.line + ): + # It seems safe to propagate the type narrowing to a nested scope. + if new_frame is None: + new_frame = self.binder.push_frame() + new_frame.types[key] = narrowed_type + self.binder.declarations[key] = old_binder.declarations[key] + with self.scope.push_function(defn): + # We suppress reachability warnings for empty generator functions + # (return; yield) which have a "yield" that's unreachable by definition + # since it's only there to promote the function into a generator function. + # + # We also suppress reachability warnings when we use TypeVars with value + # restrictions: we only want to report a warning if a certain statement is + # marked as being suppressed in *all* of the expansions, but we currently + # have no good way of doing this. + # + # TODO: Find a way of working around this limitation + if _is_empty_generator_function(item) or len(expanded) >= 2: + self.binder.suppress_unreachable_warnings() + self.accept(item.body) + unreachable = self.binder.is_unreachable() + if new_frame is not None: + self.binder.pop_frame(True, 0) + + if not unreachable: + if defn.is_generator or is_named_instance( + self.return_types[-1], "typing.AwaitableGenerator" + ): + return_type = self.get_generator_return_type( + self.return_types[-1], defn.is_coroutine + ) + elif defn.is_coroutine: + return_type = self.get_coroutine_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + return_type = get_proper_type(return_type) + + allow_empty = allow_empty or self.options.allow_empty_bodies + + show_error = ( + not body_is_trivial + or + # Allow empty bodies for abstract methods, overloads, in tests and stubs. + ( + not allow_empty + and not ( + isinstance(defn, FuncDef) and defn.abstract_status != NOT_ABSTRACT + ) + and not self.is_stub + ) + ) + + # Ignore plugin generated methods, these usually don't need any bodies. + if defn.info is not FUNC_NO_INFO and ( + defn.name not in defn.info.names or defn.info.names[defn.name].plugin_generated + ): + show_error = False + + # Ignore also definitions that appear in `if TYPE_CHECKING: ...` blocks. + # These can't be called at runtime anyway (similar to plugin-generated). + if isinstance(defn, FuncDef) and defn.is_mypy_only: + show_error = False + + # We want to minimize the fallout from checking empty bodies + # that was absent in many mypy versions. + if body_is_trivial and is_subtype(NoneType(), return_type): + show_error = False + + may_be_abstract = ( + body_is_trivial + and defn.info is not FUNC_NO_INFO + and defn.info.metaclass_type is not None + and defn.info.metaclass_type.type.has_base("abc.ABCMeta") + ) + + if self.options.warn_no_return: + if ( + not self.current_node_deferred + and not isinstance(return_type, (NoneType, AnyType)) + and show_error + ): + # Control flow fell off the end of a function that was + # declared to return a non-None type. + if isinstance(return_type, UninhabitedType): + # This is a NoReturn function + msg = message_registry.INVALID_IMPLICIT_RETURN + else: + msg = message_registry.MISSING_RETURN_STATEMENT + if body_is_trivial: + msg = msg._replace(code=codes.EMPTY_BODY) + self.fail(msg, defn) + if may_be_abstract: + self.note(message_registry.EMPTY_BODY_ABSTRACT, defn) + elif show_error: + msg = message_registry.INCOMPATIBLE_RETURN_VALUE_TYPE + if body_is_trivial: + msg = msg._replace(code=codes.EMPTY_BODY) + # similar to code in check_return_stmt + if ( + not self.check_subtype( + subtype_label="implicitly returns", + subtype=NoneType(), + supertype_label="expected", + supertype=return_type, + context=defn, + msg=msg, + ) + and may_be_abstract + ): + self.note(message_registry.EMPTY_BODY_ABSTRACT, defn) self.return_types.pop() - self.leave() self.binder = old_binder - def check_reverse_op_method(self, defn: FuncItem, typ: Callable, - method: str) -> None: + def is_var_redefined_in_outer_context(self, v: Var, after_line: int) -> bool: + """Can the variable be assigned to at module top level or outer function? + + Note that this doesn't do a full CFG analysis but uses a line number based + heuristic that isn't correct in some (rare) cases. + """ + outers = self.tscope.outer_functions() + if not outers: + # Top-level function -- outer context is top level, and we can't reason about + # globals + return True + for outer in outers: + if isinstance(outer, FuncDef): + if find_last_var_assignment_line(outer.body, v) >= after_line: + return True + return False + + def check_unbound_return_typevar(self, typ: CallableType) -> None: + """Fails when the return typevar is not defined in arguments.""" + if isinstance(typ.ret_type, TypeVarType) and typ.ret_type in typ.variables: + arg_type_visitor = CollectArgTypeVarTypes() + for argtype in typ.arg_types: + argtype.accept(arg_type_visitor) + + if typ.ret_type not in arg_type_visitor.arg_types: + self.fail(message_registry.UNBOUND_TYPEVAR, typ.ret_type, code=TYPE_VAR) + upper_bound = get_proper_type(typ.ret_type.upper_bound) + if not ( + isinstance(upper_bound, Instance) + and upper_bound.type.fullname == "builtins.object" + ): + self.note( + "Consider using the upper bound " + f"{format_type(typ.ret_type.upper_bound, self.options)} instead", + context=typ.ret_type, + ) + + def check_default_args(self, item: FuncItem, body_is_trivial: bool) -> None: + for arg in item.arguments: + if arg.initializer is None: + continue + if body_is_trivial and isinstance(arg.initializer, EllipsisExpr): + continue + name = arg.variable.name + msg = "Incompatible default for " + if name.startswith("__tuple_arg_"): + msg += f"tuple argument {name[12:]}" + else: + msg += f'argument "{name}"' + if ( + not self.options.implicit_optional + and isinstance(arg.initializer, NameExpr) + and arg.initializer.fullname == "builtins.None" + ): + notes = [ + "PEP 484 prohibits implicit Optional. " + "Accordingly, mypy has changed its default to no_implicit_optional=True", + "Use https://github.com/hauntsaninja/no_implicit_optional to automatically " + "upgrade your codebase", + ] + else: + notes = None + self.check_simple_assignment( + arg.variable.type, + arg.initializer, + context=arg.initializer, + msg=ErrorMessage(msg, code=codes.ASSIGNMENT), + lvalue_name="argument", + rvalue_name="default", + notes=notes, + ) + + def is_forward_op_method(self, method_name: str) -> bool: + return method_name in operators.reverse_op_methods + + def is_reverse_op_method(self, method_name: str) -> bool: + return method_name in operators.reverse_op_method_set + + def check_for_missing_annotations(self, fdef: FuncItem) -> None: + # Check for functions with unspecified/not fully specified types. + def is_unannotated_any(t: Type) -> bool: + if not isinstance(t, ProperType): + return False + return isinstance(t, AnyType) and t.type_of_any == TypeOfAny.unannotated + + has_explicit_annotation = isinstance(fdef.type, CallableType) and any( + not is_unannotated_any(t) for t in fdef.type.arg_types + [fdef.type.ret_type] + ) + + show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub + check_incomplete_defs = self.options.disallow_incomplete_defs and has_explicit_annotation + if show_untyped and (self.options.disallow_untyped_defs or check_incomplete_defs): + if fdef.type is None and self.options.disallow_untyped_defs: + if not fdef.arguments or ( + len(fdef.arguments) == 1 + and (fdef.arg_names[0] == "self" or fdef.arg_names[0] == "cls") + ): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + if not has_return_statement(fdef) and not fdef.is_generator: + self.note( + 'Use "-> None" if function does not return a value', + fdef, + code=codes.NO_UNTYPED_DEF, + ) + else: + self.fail(message_registry.FUNCTION_TYPE_EXPECTED, fdef) + elif isinstance(fdef.type, CallableType): + ret_type = get_proper_type(fdef.type.ret_type) + if is_unannotated_any(ret_type): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + elif fdef.is_generator: + if is_unannotated_any( + self.get_generator_return_type(ret_type, fdef.is_coroutine) + ): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + elif fdef.is_coroutine and isinstance(ret_type, Instance): + if is_unannotated_any(self.get_coroutine_return_type(ret_type)): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + if any(is_unannotated_any(t) for t in fdef.type.arg_types): + self.fail(message_registry.ARGUMENT_TYPE_EXPECTED, fdef) + + def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None: + self_type = fill_typevars_with_any(fdef.info) + bound_type = bind_self(typ, self_type, is_classmethod=True) + # Check that __new__ (after binding cls) returns an instance + # type (or any). + if fdef.info.is_metaclass(): + # This is a metaclass, so it must return a new unrelated type. + self.check_subtype( + bound_type.ret_type, + self.type_type(), + fdef, + message_registry.INVALID_NEW_TYPE, + "returns", + "but must return a subtype of", + ) + elif not isinstance( + get_proper_type(bound_type.ret_type), (AnyType, Instance, TupleType, UninhabitedType) + ): + self.fail( + message_registry.NON_INSTANCE_NEW_TYPE.format( + format_type(bound_type.ret_type, self.options) + ), + fdef, + ) + else: + # And that it returns a subtype of the class + self.check_subtype( + bound_type.ret_type, + self_type, + fdef, + message_registry.INVALID_NEW_TYPE, + "returns", + "but must return a subtype of", + ) + + def check_reverse_op_method( + self, defn: FuncItem, reverse_type: CallableType, reverse_name: str, context: Context + ) -> None: """Check a reverse operator method such as __radd__.""" + # Decides whether it's worth calling check_overlapping_op_methods(). + + # This used to check for some very obscure scenario. It now + # just decides whether it's worth calling + # check_overlapping_op_methods(). + + assert defn.info + + # First check for a valid signature + method_type = CallableType( + [AnyType(TypeOfAny.special_form), AnyType(TypeOfAny.special_form)], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + AnyType(TypeOfAny.special_form), + self.named_type("builtins.function"), + ) + if not is_subtype(reverse_type, method_type): + self.msg.invalid_signature(reverse_type, context) + return - # If the argument of a reverse operator method such as __radd__ - # does not define the corresponding non-reverse method such as __add__ - # the return type of __radd__ may not reliably represent the value of - # the corresponding operation even in a fully statically typed program. - # - # This example illustrates the issue: - # - # class A: pass - # class B: - # def __radd__(self, x: A) -> int: # Note that A does not define - # return 1 # __add__! - # class C(A): - # def __add__(self, x: Any) -> str: return 'x' - # a = Undefined(A) - # a = C() - # a + B() # Result would be 'x', even though static type seems to - # # be int! - - if method in ('__eq__', '__ne__'): + if reverse_name in ("__eq__", "__ne__"): # These are defined for all objects => can't cause trouble. return # With 'Any' or 'object' return type we are happy, since any possible # return value is valid. - ret_type = typ.ret_type + ret_type = get_proper_type(reverse_type.ret_type) if isinstance(ret_type, AnyType): return if isinstance(ret_type, Instance): - if ret_type.type.fullname() == 'builtins.object': - return - # Plausibly the method could have too few arguments, which would result - # in an error elsewhere. - if len(typ.arg_types) <= 2: - # TODO check self argument kind - - # Check for the issue described above. - arg_type = typ.arg_types[1] - other_method = nodes.normal_from_reverse_op[method] - fail = False - if isinstance(arg_type, Instance): - if not arg_type.type.has_readable_member(other_method): - fail = True - elif isinstance(arg_type, AnyType): - self.msg.reverse_operator_method_with_any_arg_must_return_any( - method, defn) - return - elif isinstance(arg_type, UnionType): - if not arg_type.has_readable_member(other_method): - fail = True - else: - fail = True - if fail: - self.msg.invalid_reverse_operator_signature( - method, other_method, defn) + if ret_type.type.fullname == "builtins.object": return - - typ2 = self.expr_checker.analyse_external_member_access( - other_method, arg_type, defn) - self.check_overlapping_op_methods( - typ, method, defn.info, - typ2, other_method, cast(Instance, arg_type), - defn) - - def check_overlapping_op_methods(self, - reverse_type: Callable, - reverse_name: str, - reverse_class: TypeInfo, - forward_type: Type, - forward_name: str, - forward_base: Instance, - context: Context) -> None: + if reverse_type.arg_kinds[0] == ARG_STAR: + reverse_type = reverse_type.copy_modified( + arg_types=[reverse_type.arg_types[0]] * 2, + arg_kinds=[ARG_POS] * 2, + arg_names=[reverse_type.arg_names[0], "_"], + ) + assert len(reverse_type.arg_types) >= 2 + + forward_name = operators.normal_from_reverse_op[reverse_name] + forward_inst = get_proper_type(reverse_type.arg_types[1]) + if isinstance(forward_inst, TypeVarType): + forward_inst = get_proper_type(forward_inst.upper_bound) + elif isinstance(forward_inst, TupleType): + forward_inst = tuple_fallback(forward_inst) + elif isinstance(forward_inst, (FunctionLike, TypedDictType, LiteralType)): + forward_inst = forward_inst.fallback + if isinstance(forward_inst, TypeType): + item = forward_inst.item + if isinstance(item, Instance): + opt_meta = item.type.metaclass_type + if opt_meta is not None: + forward_inst = opt_meta + + def has_readable_member(typ: UnionType | Instance, name: str) -> bool: + # TODO: Deal with attributes of TupleType etc. + if isinstance(typ, Instance): + return typ.type.has_readable_member(name) + return all( + (isinstance(x, UnionType) and has_readable_member(x, name)) + or (isinstance(x, Instance) and x.type.has_readable_member(name)) + for x in get_proper_types(typ.relevant_items()) + ) + + if not ( + isinstance(forward_inst, (Instance, UnionType)) + and has_readable_member(forward_inst, forward_name) + ): + return + forward_base = reverse_type.arg_types[1] + forward_type = self.expr_checker.analyze_external_member_access( + forward_name, forward_base, context=defn + ) + self.check_overlapping_op_methods( + reverse_type, + reverse_name, + defn.info, + forward_type, + forward_name, + forward_base, + context=defn, + ) + + def check_overlapping_op_methods( + self, + reverse_type: CallableType, + reverse_name: str, + reverse_class: TypeInfo, + forward_type: Type, + forward_name: str, + forward_base: Type, + context: Context, + ) -> None: """Check for overlapping method and reverse method signatures. - Assume reverse method has valid argument count and kinds. + This function assumes that: + + - The reverse method has valid argument count and kinds. + - If the reverse operator method accepts some argument of type + X, the forward operator method also belong to class X. + + For example, if we have the reverse operator `A.__radd__(B)`, then the + corresponding forward operator must have the type `B.__add__(...)`. """ - # Reverse operator method that overlaps unsafely with the - # forward operator method can result in type unsafety. This is - # similar to overlapping overload variants. + # Note: Suppose we have two operator methods "A.__rOP__(B) -> R1" and + # "B.__OP__(C) -> R2". We check if these two methods are unsafely overlapping + # by using the following algorithm: # - # This example illustrates the issue: + # 1. Rewrite "B.__OP__(C) -> R1" to "temp1(B, C) -> R1" # - # class X: pass - # class A: - # def __add__(self, x: X) -> int: - # if isinstance(x, X): - # return 1 - # return NotImplemented - # class B: - # def __radd__(self, x: A) -> str: return 'x' - # class C(X, B): pass - # b = Undefined(B) - # b = C() - # A() + b # Result is 1, even though static type seems to be str! + # 2. Rewrite "A.__rOP__(B) -> R2" to "temp2(B, A) -> R2" # - # The reason for the problem is that B and X are overlapping - # types, and the return types are different. Also, if the type - # of x in __radd__ would not be A, the methods could be - # non-overlapping. - - if isinstance(forward_type, Callable): - # TODO check argument kinds - if len(forward_type.arg_types) < 1: - # Not a valid operator method -- can't succeed anyway. - return + # 3. Treat temp1 and temp2 as if they were both variants in the same + # overloaded function. (This mirrors how the Python runtime calls + # operator methods: we first try __OP__, then __rOP__.) + # + # If the first signature is unsafely overlapping with the second, + # report an error. + # + # 4. However, if temp1 shadows temp2 (e.g. the __rOP__ method can never + # be called), do NOT report an error. + # + # This behavior deviates from how we handle overloads -- many of the + # modules in typeshed seem to define __OP__ methods that shadow the + # corresponding __rOP__ method. + # + # Note: we do not attempt to handle unsafe overlaps related to multiple + # inheritance. (This is consistent with how we handle overloads: we also + # do not try checking unsafe overlaps due to multiple inheritance there.) + + for forward_item in flatten_nested_unions([forward_type]): + forward_item = get_proper_type(forward_item) + if isinstance(forward_item, CallableType): + if self.is_unsafe_overlapping_op(forward_item, forward_base, reverse_type): + self.msg.operator_method_signatures_overlap( + reverse_class, reverse_name, forward_base, forward_name, context + ) + elif isinstance(forward_item, Overloaded): + for item in forward_item.items: + if self.is_unsafe_overlapping_op(item, forward_base, reverse_type): + self.msg.operator_method_signatures_overlap( + reverse_class, reverse_name, forward_base, forward_name, context + ) + elif not isinstance(forward_item, AnyType): + self.msg.forward_operator_not_callable(forward_name, context) + + def is_unsafe_overlapping_op( + self, forward_item: CallableType, forward_base: Type, reverse_type: CallableType + ) -> bool: + # TODO: check argument kinds? + if len(forward_item.arg_types) < 1: + # Not a valid operator method -- can't succeed anyway. + return False - # Construct normalized function signatures corresponding to the - # operator methods. The first argument is the left operand and the - # second operand is the right argument -- we switch the order of - # the arguments of the reverse method. - forward_tweaked = Callable([forward_base, - forward_type.arg_types[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - forward_type.ret_type, - forward_type.fallback, - name=forward_type.name) - reverse_args = reverse_type.arg_types - reverse_tweaked = Callable([reverse_args[1], reverse_args[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - reverse_type.ret_type, - fallback=self.named_type('builtins.function'), - name=reverse_type.name) - - if is_unsafe_overlapping_signatures(forward_tweaked, - reverse_tweaked): - self.msg.operator_method_signatures_overlap( - reverse_class.name(), reverse_name, - forward_base.type.name(), forward_name, context) - elif isinstance(forward_type, Overloaded): - for item in forward_type.items(): - self.check_overlapping_op_methods( - reverse_type, reverse_name, reverse_class, - item, forward_name, forward_base, context) + # Erase the type if necessary to make sure we don't have a single + # TypeVar in forward_tweaked. (Having a function signature containing + # just a single TypeVar can lead to unpredictable behavior.) + forward_base_erased = forward_base + if isinstance(forward_base, TypeVarType): + forward_base_erased = erase_to_bound(forward_base) + + # Construct normalized function signatures corresponding to the + # operator methods. The first argument is the left operand and the + # second operand is the right argument -- we switch the order of + # the arguments of the reverse method. + + # TODO: this manipulation is dangerous if callables are generic. + # Shuffling arguments between callables can create meaningless types. + forward_tweaked = forward_item.copy_modified( + arg_types=[forward_base_erased, forward_item.arg_types[0]], + arg_kinds=[nodes.ARG_POS] * 2, + arg_names=[None] * 2, + ) + reverse_tweaked = reverse_type.copy_modified( + arg_types=[reverse_type.arg_types[1], reverse_type.arg_types[0]], + arg_kinds=[nodes.ARG_POS] * 2, + arg_names=[None] * 2, + ) + + reverse_base_erased = reverse_type.arg_types[0] + if isinstance(reverse_base_erased, TypeVarType): + reverse_base_erased = erase_to_bound(reverse_base_erased) + + if is_same_type(reverse_base_erased, forward_base_erased): + return False + elif is_subtype(reverse_base_erased, forward_base_erased): + first = reverse_tweaked + second = forward_tweaked else: - # TODO what about this? - assert False, 'Forward operator method type is not Callable' + first = forward_tweaked + second = reverse_tweaked + + current_class = self.scope.active_class() + type_vars = current_class.defn.type_vars if current_class else [] + return is_unsafe_overlapping_overload_signatures( + first, second, type_vars, partial_only=False + ) def check_inplace_operator_method(self, defn: FuncBase) -> None: """Check an inplace operator method such as __iadd__. They cannot arbitrarily overlap with __add__. """ - method = defn.name() - if method not in nodes.inplace_operator_methods: + method = defn.name + if method not in operators.inplace_operator_methods: return - typ = self.method_type(defn) + typ = bind_self(self.function_type(defn)) cls = defn.info - other_method = '__' + method[3:] + other_method = "__" + method[3:] if cls.has_readable_member(other_method): - instance = self_type(cls) - typ2 = self.expr_checker.analyse_external_member_access( - other_method, instance, defn) + instance = fill_typevars(cls) + typ2 = get_proper_type( + self.expr_checker.analyze_external_member_access(other_method, instance, defn) + ) fail = False if isinstance(typ2, FunctionLike): if not is_more_general_arg_prefix(typ, typ2): @@ -707,275 +1839,1927 @@ def check_inplace_operator_method(self, defn: FuncBase) -> None: if fail: self.msg.signatures_incompatible(method, other_method, defn) - def expand_typevars(self, defn: FuncItem, - typ: Callable) -> List[Tuple[FuncItem, Callable]]: + def check_getattr_method(self, typ: Type, context: Context, name: str) -> None: + if len(self.scope.stack) == 1: + # module scope + if name == "__getattribute__": + self.fail(message_registry.MODULE_LEVEL_GETATTRIBUTE, context) + return + # __getattr__ is fine at the module level as of Python 3.7 (PEP 562). We could + # show an error for Python < 3.7, but that would be annoying in code that supports + # both 3.7 and older versions. + method_type = CallableType( + [self.named_type("builtins.str")], + [nodes.ARG_POS], + [None], + AnyType(TypeOfAny.special_form), + self.named_type("builtins.function"), + ) + elif self.scope.active_class(): + method_type = CallableType( + [AnyType(TypeOfAny.special_form), self.named_type("builtins.str")], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + AnyType(TypeOfAny.special_form), + self.named_type("builtins.function"), + ) + else: + return + if not is_subtype(typ, method_type): + self.msg.invalid_signature_for_special_method(typ, context, name) + + def check_setattr_method(self, typ: Type, context: Context) -> None: + if not self.scope.active_class(): + return + method_type = CallableType( + [ + AnyType(TypeOfAny.special_form), + self.named_type("builtins.str"), + AnyType(TypeOfAny.special_form), + ], + [nodes.ARG_POS, nodes.ARG_POS, nodes.ARG_POS], + [None, None, None], + NoneType(), + self.named_type("builtins.function"), + ) + if not is_subtype(typ, method_type): + self.msg.invalid_signature_for_special_method(typ, context, "__setattr__") + + def check_slots_definition(self, typ: Type, context: Context) -> None: + """Check the type of __slots__.""" + str_type = self.named_type("builtins.str") + expected_type = UnionType( + [str_type, self.named_generic_type("typing.Iterable", [str_type])] + ) + self.check_subtype( + typ, + expected_type, + context, + message_registry.INVALID_TYPE_FOR_SLOTS, + "actual type", + "expected type", + code=codes.ASSIGNMENT, + ) + + def check_match_args(self, var: Var, typ: Type, context: Context) -> None: + """Check that __match_args__ contains literal strings""" + if not self.scope.active_class(): + return + typ = get_proper_type(typ) + if not isinstance(typ, TupleType) or not all( + is_string_literal(item) for item in typ.items + ): + self.msg.note( + "__match_args__ must be a tuple containing string literals for checking " + "of match statements to work", + context, + code=codes.LITERAL_REQ, + ) + + def expand_typevars( + self, defn: FuncItem, typ: CallableType + ) -> list[tuple[FuncItem, CallableType]]: # TODO use generator - subst = List[List[Tuple[int, Type]]]() - tvars = typ.variables or [] - tvars = tvars[:] + subst: list[list[tuple[TypeVarId, Type]]] = [] + tvars = list(typ.variables) or [] if defn.info: # Class type variables tvars += defn.info.defn.type_vars or [] for tvar in tvars: - if tvar.values: - subst.append([(tvar.id, value) - for value in tvar.values]) - if subst: - result = List[Tuple[FuncItem, Callable]]() + if isinstance(tvar, TypeVarType) and tvar.values: + subst.append([(tvar.id, value) for value in tvar.values]) + # Make a copy of the function to check for each combination of + # value restricted type variables. (Except when running mypyc, + # where we need one canonical version of the function.) + if subst and not (self.options.mypyc or self.options.inspections): + result: list[tuple[FuncItem, CallableType]] = [] for substitutions in itertools.product(*subst): mapping = dict(substitutions) - expanded = cast(Callable, expand_type(typ, mapping)) - result.append((expand_func(defn, mapping), expanded)) + result.append((expand_func(defn, mapping), expand_type(typ, mapping))) return result else: return [(defn, typ)] - def check_method_override(self, defn: FuncBase) -> None: - """Check if function definition is compatible with base classes.""" + def check_explicit_override_decorator( + self, + defn: FuncDef | OverloadedFuncDef, + found_method_base_classes: list[TypeInfo] | None, + context: Context | None = None, + ) -> None: + plugin_generated = False + if defn.info and (node := defn.info.get(defn.name)) and node.plugin_generated: + # Do not report issues for plugin generated nodes, + # they can't realistically use `@override` for their methods. + plugin_generated = True + + if ( + not plugin_generated + and found_method_base_classes + and not defn.is_explicit_override + and defn.name not in ("__init__", "__new__") + and not is_private(defn.name) + ): + self.msg.explicit_override_decorator_missing( + defn.name, found_method_base_classes[0].fullname, context or defn + ) + + def check_method_override( + self, defn: FuncDef | OverloadedFuncDef | Decorator + ) -> list[TypeInfo] | None: + """Check if function definition is compatible with base classes. + + This may defer the method if a signature is not available in at least one base class. + Return ``None`` if that happens. + + Return a list of base classes which contain an attribute with the method name. + """ # Check against definitions in base classes. + check_override_compatibility = defn.name not in ( + "__init__", + "__new__", + "__init_subclass__", + "__post_init__", + ) and (self.options.check_untyped_defs or not defn.is_dynamic()) + found_method_base_classes: list[TypeInfo] = [] for base in defn.info.mro[1:]: - self.check_method_or_accessor_override_for_base(defn, base) - - def check_method_or_accessor_override_for_base(self, defn: FuncBase, - base: TypeInfo) -> None: - """Check if method definition is compatible with a base class.""" + result = self.check_method_or_accessor_override_for_base( + defn, base, check_override_compatibility + ) + if result is None: + # Node was deferred, we will have another attempt later. + return None + if result: + found_method_base_classes.append(base) + return found_method_base_classes + + def check_method_or_accessor_override_for_base( + self, + defn: FuncDef | OverloadedFuncDef | Decorator, + base: TypeInfo, + check_override_compatibility: bool, + ) -> bool | None: + """Check if method definition is compatible with a base class. + + Return ``None`` if the node was deferred because one of the corresponding + superclass nodes is not ready. + + Return ``True`` if an attribute with the method name was found in the base class. + """ + found_base_method = False if base: - name = defn.name() - if name != '__init__': - # Check method override (__init__ is special). - self.check_method_override_for_base_with_name(defn, name, base) - if name in nodes.inplace_operator_methods: + name = defn.name + base_attr = base.names.get(name) + if base_attr: + # First, check if we override a final (always an error, even with Any types). + if is_final_node(base_attr.node) and not is_private(name): + self.msg.cant_override_final(name, base.name, defn) + # Second, final can't override anything writeable independently of types. + if defn.is_final: + self.check_if_final_var_override_writable(name, base_attr.node, defn) + found_base_method = True + if check_override_compatibility: + # Check compatibility of the override signature + # (__init__, __new__, __init_subclass__ are special). + if self.check_method_override_for_base_with_name(defn, name, base): + return None + if name in operators.inplace_operator_methods: # Figure out the name of the corresponding operator method. - method = '__' + name[3:] - # An inplace overator method such as __iadd__ might not be + method = "__" + name[3:] + # An inplace operator method such as __iadd__ might not be # always introduced safely if a base class defined __add__. # TODO can't come up with an example where this is # necessary; now it's "just in case" - self.check_method_override_for_base_with_name(defn, method, - base) + if self.check_method_override_for_base_with_name(defn, method, base): + return None + return found_base_method def check_method_override_for_base_with_name( - self, defn: FuncBase, name: str, base: TypeInfo) -> None: + self, defn: FuncDef | OverloadedFuncDef | Decorator, name: str, base: TypeInfo + ) -> bool: + """Check if overriding an attribute `name` of `base` with `defn` is valid. + + Return True if the supertype node was not analysed yet, and `defn` was deferred. + """ base_attr = base.names.get(name) if base_attr: # The name of the method is defined in the base class. + # Point errors at the 'def' line (important for backward compatibility + # of type ignores). + if not isinstance(defn, Decorator): + context = defn + else: + context = defn.func + # Construct the type of the overriding method. - typ = self.method_type(defn) + # TODO: this logic is much less complete than similar one in checkmember.py + if isinstance(defn, (FuncDef, OverloadedFuncDef)): + typ: Type = self.function_type(defn) + override_class_or_static = defn.is_class or defn.is_static + override_class = defn.is_class + else: + assert defn.var.is_ready + assert defn.var.type is not None + typ = defn.var.type + override_class_or_static = defn.func.is_class or defn.func.is_static + override_class = defn.func.is_class + typ = get_proper_type(typ) + if isinstance(typ, FunctionLike) and not is_static(context): + typ = bind_self(typ, self.scope.active_self_type(), is_classmethod=override_class) # Map the overridden method type to subtype context so that # it can be checked for compatibility. - original_type = base_attr.type - if original_type is None and isinstance(base_attr.node, - FuncDef): - original_type = self.function_type(cast(FuncDef, - base_attr.node)) + original_type = get_proper_type(base_attr.type) + original_node = base_attr.node + # `original_type` can be partial if (e.g.) it is originally an + # instance variable from an `__init__` block that becomes deferred. + if original_type is None or isinstance(original_type, PartialType): + if self.pass_num < self.last_pass: + # If there are passes left, defer this node until next pass, + # otherwise try reconstructing the method type from available information. + self.defer_node(defn, defn.info) + return True + elif isinstance(original_node, (FuncDef, OverloadedFuncDef)): + original_type = self.function_type(original_node) + elif isinstance(original_node, Decorator): + original_type = self.function_type(original_node.func) + elif isinstance(original_node, Var): + # Super type can define method as an attribute. + # See https://github.com/python/mypy/issues/10134 + + # We also check that sometimes `original_node.type` is None. + # This is the case when we use something like `__hash__ = None`. + if original_node.type is not None: + original_type = get_proper_type(original_node.type) + else: + original_type = NoneType() + else: + # Will always fail to typecheck below, since we know the node is a method + original_type = NoneType() + if isinstance(original_node, (FuncDef, OverloadedFuncDef)): + original_class_or_static = original_node.is_class or original_node.is_static + elif isinstance(original_node, Decorator): + fdef = original_node.func + original_class_or_static = fdef.is_class or fdef.is_static + else: + original_class_or_static = False # a variable can't be class or static + if isinstance(original_type, FunctionLike): - original = map_type_from_supertype( - method_type(original_type), - defn.info, base) + original_type = self.bind_and_map_method(base_attr, original_type, defn.info, base) + if original_node and is_property(original_node): + original_type = get_property_type(original_type) + + if is_property(defn): + inner: FunctionLike | None + if isinstance(typ, FunctionLike): + inner = typ + else: + inner = self.extract_callable_type(typ, context) + if inner is not None: + typ = inner + typ = get_property_type(typ) + if ( + isinstance(original_node, Var) + and not original_node.is_final + and (not original_node.is_property or original_node.is_settable_property) + and isinstance(defn, Decorator) + ): + # We only give an error where no other similar errors will be given. + if not isinstance(original_type, AnyType): + self.msg.fail( + "Cannot override writeable attribute with read-only property", + # Give an error on function line to match old behaviour. + defn.func, + code=codes.OVERRIDE, + ) + + if isinstance(original_type, AnyType) or isinstance(typ, AnyType): + pass + elif isinstance(original_type, FunctionLike) and isinstance(typ, FunctionLike): # Check that the types are compatible. - # TODO overloaded signatures - self.check_override(cast(FunctionLike, typ), - cast(FunctionLike, original), - defn.name(), - name, - base.name(), - defn) + ok = self.check_override( + typ, + original_type, + defn.name, + name, + base.name, + original_class_or_static, + override_class_or_static, + context, + ) + # Check if this override is covariant. + if ( + ok + and original_node + and codes.MUTABLE_OVERRIDE in self.options.enabled_error_codes + and self.is_writable_attribute(original_node) + and not is_subtype(original_type, typ, ignore_pos_arg_names=True) + ): + base_str, override_str = format_type_distinctly( + original_type, typ, options=self.options + ) + msg = message_registry.COVARIANT_OVERRIDE_OF_MUTABLE_ATTRIBUTE.with_additional_msg( + f' (base class "{base.name}" defined the type as {base_str},' + f" override has type {override_str})" + ) + self.fail(msg, context) + elif isinstance(original_type, UnionType) and any( + is_subtype(typ, orig_typ, ignore_pos_arg_names=True) + for orig_typ in original_type.items + ): + # This method is a subtype of at least one union variant. + if ( + original_node + and codes.MUTABLE_OVERRIDE in self.options.enabled_error_codes + and self.is_writable_attribute(original_node) + ): + # Covariant override of mutable attribute. + base_str, override_str = format_type_distinctly( + original_type, typ, options=self.options + ) + msg = message_registry.COVARIANT_OVERRIDE_OF_MUTABLE_ATTRIBUTE.with_additional_msg( + f' (base class "{base.name}" defined the type as {base_str},' + f" override has type {override_str})" + ) + self.fail(msg, context) + elif is_equivalent(original_type, typ): + # Assume invariance for a non-callable attribute here. Note + # that this doesn't affect read-only properties which can have + # covariant overrides. + pass + elif ( + original_node + and not self.is_writable_attribute(original_node) + and is_subtype(typ, original_type) + ): + # If the attribute is read-only, allow covariance + pass else: - assert original_type is not None self.msg.signature_incompatible_with_supertype( - defn.name(), name, base.name(), defn) + defn.name, name, base.name, context, original=original_type, override=typ + ) + return False - def check_override(self, override: FunctionLike, original: FunctionLike, - name: str, name_in_super: str, supertype: str, - node: Context) -> None: + def bind_and_map_method( + self, sym: SymbolTableNode, typ: FunctionLike, sub_info: TypeInfo, super_info: TypeInfo + ) -> FunctionLike: + """Bind self-type and map type variables for a method. + + Arguments: + sym: a symbol that points to method definition + typ: method type on the definition + sub_info: class where the method is used + super_info: class where the method was defined + """ + if isinstance(sym.node, (FuncDef, OverloadedFuncDef, Decorator)) and not is_static( + sym.node + ): + if isinstance(sym.node, Decorator): + is_class_method = sym.node.func.is_class + else: + is_class_method = sym.node.is_class + + mapped_typ = cast(FunctionLike, map_type_from_supertype(typ, sub_info, super_info)) + active_self_type = self.scope.active_self_type() + if isinstance(mapped_typ, Overloaded) and active_self_type: + # If we have an overload, filter to overloads that match the self type. + # This avoids false positives for concrete subclasses of generic classes, + # see testSelfTypeOverrideCompatibility for an example. + filtered_items = [] + for item in mapped_typ.items: + if not item.arg_types: + filtered_items.append(item) + item_arg = item.arg_types[0] + if isinstance(item_arg, TypeVarType): + item_arg = item_arg.upper_bound + if is_subtype(active_self_type, item_arg): + filtered_items.append(item) + # If we don't have any filtered_items, maybe it's always a valid override + # of the superclass? However if you get to that point you're in murky type + # territory anyway, so we just preserve the type and have the behaviour match + # that of older versions of mypy. + if filtered_items: + mapped_typ = Overloaded(filtered_items) + + return bind_self(mapped_typ, active_self_type, is_class_method) + else: + return cast(FunctionLike, map_type_from_supertype(typ, sub_info, super_info)) + + def get_op_other_domain(self, tp: FunctionLike) -> Type | None: + if isinstance(tp, CallableType): + if tp.arg_kinds and tp.arg_kinds[0] == ARG_POS: + # For generic methods, domain comparison is tricky, as a first + # approximation erase all remaining type variables. + return erase_typevars(tp.arg_types[0], {v.id for v in tp.variables}) + return None + elif isinstance(tp, Overloaded): + raw_items = [self.get_op_other_domain(it) for it in tp.items] + items = [it for it in raw_items if it] + if items: + return make_simplified_union(items) + return None + else: + assert False, "Need to check all FunctionLike subtypes here" + + def check_override( + self, + override: FunctionLike, + original: FunctionLike, + name: str, + name_in_super: str, + supertype: str, + original_class_or_static: bool, + override_class_or_static: bool, + node: Context, + ) -> bool: """Check a method override with given signatures. Arguments: - override: The signature of the overriding method. - original: The signature of the original supertype method. - name: The name of the subtype. This and the next argument are - only used for generating error messages. - supertype: The name of the supertype. - """ - if (isinstance(override, Overloaded) or - isinstance(original, Overloaded) or - len(cast(Callable, override).arg_types) != - len(cast(Callable, original).arg_types) or - cast(Callable, override).min_args != - cast(Callable, original).min_args): - # Use boolean variable to clarify code. - fail = False - if not is_subtype(override, original): + override: The signature of the overriding method. + original: The signature of the original supertype method. + name: The name of the overriding method. + Used primarily for generating error messages. + name_in_super: The name of the overridden in the superclass. + Used for generating error messages only. + supertype: The name of the supertype. + original_class_or_static: Indicates whether the original method (from the superclass) + is either a class method or a static method. + override_class_or_static: Indicates whether the overriding method (from the subclass) + is either a class method or a static method. + node: Context node. + """ + # Use boolean variable to clarify code. + fail = False + op_method_wider_note = False + if not is_subtype(override, original, ignore_pos_arg_names=True): + fail = True + elif isinstance(override, Overloaded) and self.is_forward_op_method(name): + # Operator method overrides cannot extend the domain, as + # this could be unsafe with reverse operator methods. + original_domain = self.get_op_other_domain(original) + override_domain = self.get_op_other_domain(override) + if ( + original_domain + and override_domain + and not is_subtype(override_domain, original_domain) + ): fail = True - elif (not isinstance(original, Overloaded) and - isinstance(override, Overloaded) and - name in nodes.reverse_op_methods.keys()): - # Operator method overrides cannot introduce overloading, as - # this could be unsafe with reverse operator methods. + op_method_wider_note = True + if isinstance(override, FunctionLike): + if original_class_or_static and not override_class_or_static: fail = True - if fail: + elif isinstance(original, CallableType) and isinstance(override, CallableType): + if original.type_guard is not None and override.type_guard is None: + fail = True + if original.type_is is not None and override.type_is is None: + fail = True + + if is_private(name): + fail = False + + if fail: + emitted_msg = False + + offset_arguments = isinstance(override, CallableType) and override.unpack_kwargs + # Normalize signatures, so we get better diagnostics. + if isinstance(override, (CallableType, Overloaded)): + override = override.with_unpacked_kwargs() + if isinstance(original, (CallableType, Overloaded)): + original = original.with_unpacked_kwargs() + + if ( + isinstance(override, CallableType) + and isinstance(original, CallableType) + and len(override.arg_types) == len(original.arg_types) + and override.min_args == original.min_args + ): + # Give more detailed messages for the common case of both + # signatures having the same number of arguments and no + # overloads. + + # override might have its own generic function type + # variables. If an argument or return type of override + # does not have the correct subtyping relationship + # with the original type even after these variables + # are erased, then it is definitely an incompatibility. + + override_ids = override.type_var_ids() + type_name = None + if isinstance(override.definition, FuncDef): + type_name = override.definition.info.name + + def erase_override(t: Type) -> Type: + return erase_typevars(t, ids_to_erase=override_ids) + + for i, (sub_kind, super_kind) in enumerate( + zip(override.arg_kinds, original.arg_kinds) + ): + if sub_kind.is_positional() and super_kind.is_positional(): + override_arg_type = override.arg_types[i] + original_arg_type = original.arg_types[i] + elif sub_kind.is_named() and super_kind.is_named() and not offset_arguments: + arg_name = override.arg_names[i] + if arg_name in original.arg_names: + override_arg_type = override.arg_types[i] + original_i = original.arg_names.index(arg_name) + original_arg_type = original.arg_types[original_i] + else: + continue + else: + continue + if not is_subtype(original_arg_type, erase_override(override_arg_type)): + context: Context = node + if isinstance(node, FuncDef) and not node.is_property: + arg_node = node.arguments[i + len(override.bound_args)] + if arg_node.line != -1: + context = arg_node + self.msg.argument_incompatible_with_supertype( + i + 1, + name, + type_name, + name_in_super, + original_arg_type, + supertype, + context, + secondary_context=node, + ) + emitted_msg = True + + if not is_subtype(erase_override(override.ret_type), original.ret_type): + self.msg.return_type_incompatible_with_supertype( + name, name_in_super, supertype, original.ret_type, override.ret_type, node + ) + emitted_msg = True + elif isinstance(override, Overloaded) and isinstance(original, Overloaded): + # Give a more detailed message in the case where the user is trying to + # override an overload, and the subclass's overload is plausible, except + # that the order of the variants are wrong. + # + # For example, if the parent defines the overload f(int) -> int and f(str) -> str + # (in that order), and if the child swaps the two and does f(str) -> str and + # f(int) -> int + order = [] + for child_variant in override.items: + for i, parent_variant in enumerate(original.items): + if is_subtype(child_variant, parent_variant): + order.append(i) + break + + if len(order) == len(original.items) and order != sorted(order): + self.msg.overload_signature_incompatible_with_supertype( + name, name_in_super, supertype, node + ) + emitted_msg = True + + if not emitted_msg: + # Fall back to generic incompatibility message. self.msg.signature_incompatible_with_supertype( - name, name_in_super, supertype, node) + name, name_in_super, supertype, node, original=original, override=override + ) + if op_method_wider_note: + self.note( + "Overloaded operator methods can't have wider argument types in overrides", + node, + code=codes.OVERRIDE, + ) + return not fail + + def check__exit__return_type(self, defn: FuncItem) -> None: + """Generate error if the return type of __exit__ is problematic. + + If __exit__ always returns False but the return type is declared + as bool, mypy thinks that a with statement may "swallow" + exceptions even though this is not the case, resulting in + invalid reachability inference. + """ + if not defn.type or not isinstance(defn.type, CallableType): return - else: - # Give more detailed messages for the common case of both - # signatures having the same number of arguments and no - # overloads. - coverride = cast(Callable, override) - coriginal = cast(Callable, original) + ret_type = get_proper_type(defn.type.ret_type) + if not has_bool_item(ret_type): + return - for i in range(len(coverride.arg_types)): - if not is_subtype(coriginal.arg_types[i], - coverride.arg_types[i]): - self.msg.argument_incompatible_with_supertype( - i + 1, name, name_in_super, supertype, node) + returns = all_return_statements(defn) + if not returns: + return - if not is_subtype(coverride.ret_type, coriginal.ret_type): - self.msg.return_type_incompatible_with_supertype( - name, name_in_super, supertype, node) + if all( + isinstance(ret.expr, NameExpr) and ret.expr.fullname == "builtins.False" + for ret in returns + ): + self.msg.incorrect__exit__return(defn) - def visit_class_def(self, defn: ClassDef) -> Type: + def visit_class_def(self, defn: ClassDef) -> None: """Type check a class definition.""" typ = defn.info - self.errors.push_type(defn.name) - old_binder = self.binder - self.binder = ConditionalTypeBinder() - self.binder.push_frame() - self.accept(defn.defs) - self.binder = old_binder - self.check_multiple_inheritance(typ) - self.errors.pop_type() + for base in typ.mro[1:]: + if base.is_final: + self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) + with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.top_frame_context(): + with self.scope.push_class(defn.info): + self.accept(defn.defs) + self.binder = old_binder + if not (defn.info.typeddict_type or defn.info.tuple_type or defn.info.is_enum): + # If it is not a normal class (not a special form) check class keywords. + self.check_init_subclass(defn) + if not defn.has_incompatible_baseclass: + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) + self.check_metaclass_compatibility(typ) + self.check_final_deletable(typ) + + if defn.decorators: + sig: Type = type_object_type(defn.info, self.named_type) + # Decorators are applied in reverse order. + for decorator in reversed(defn.decorators): + if isinstance(decorator, CallExpr) and isinstance( + decorator.analyzed, PromoteExpr + ): + # _promote is a special type checking related construct. + continue + + dec = self.expr_checker.accept(decorator) + temp = self.temp_node(sig, context=decorator) + fullname = None + if isinstance(decorator, RefExpr): + fullname = decorator.fullname or None + + # TODO: Figure out how to have clearer error messages. + # (e.g. "class decorator must be a function that accepts a type." + old_allow_abstract_call = self.allow_abstract_call + self.allow_abstract_call = True + sig, _ = self.expr_checker.check_call( + dec, [temp], [nodes.ARG_POS], defn, callable_name=fullname + ) + self.allow_abstract_call = old_allow_abstract_call + # TODO: Apply the sig to the actual TypeInfo so we can handle decorators + # that completely swap out the type. (e.g. Callable[[Type[A]], Type[B]]) + if typ.defn.type_vars and typ.defn.type_args is None: + for base_inst in typ.bases: + for base_tvar, base_decl_tvar in zip( + base_inst.args, base_inst.type.defn.type_vars + ): + if ( + isinstance(base_tvar, TypeVarType) + and base_tvar.variance != INVARIANT + and isinstance(base_decl_tvar, TypeVarType) + and base_decl_tvar.variance != base_tvar.variance + ): + self.fail( + f'Variance of TypeVar "{base_tvar.name}" incompatible ' + "with variance in parent type", + context=defn, + code=codes.TYPE_VAR, + ) + if typ.defn.type_vars: + self.check_typevar_defaults(typ.defn.type_vars) + + if typ.is_protocol and typ.defn.type_vars: + self.check_protocol_variance(defn) + if not defn.has_incompatible_baseclass and defn.info.is_enum: + self.check_enum(defn) + infer_class_variances(defn.info) + + def check_final_deletable(self, typ: TypeInfo) -> None: + # These checks are only for mypyc. Only perform some checks that are easier + # to implement here than in mypyc. + for attr in typ.deletable_attributes: + node = typ.names.get(attr) + if node and isinstance(node.node, Var) and node.node.is_final: + self.fail(message_registry.CANNOT_MAKE_DELETABLE_FINAL, node.node) + + def check_init_subclass(self, defn: ClassDef) -> None: + """Check that keywords in a class definition are valid arguments for __init_subclass__(). + + In this example: + 1 class Base: + 2 def __init_subclass__(cls, thing: int): + 3 pass + 4 class Child(Base, thing=5): + 5 def __init_subclass__(cls): + 6 pass + 7 Child() + + Base.__init_subclass__(thing=5) is called at line 4. This is what we simulate here. + Child.__init_subclass__ is never called. + """ + if defn.info.metaclass_type and defn.info.metaclass_type.type.fullname not in ( + "builtins.type", + "abc.ABCMeta", + ): + # We can't safely check situations when both __init_subclass__ and a custom + # metaclass are present. + return + # At runtime, only Base.__init_subclass__ will be called, so + # we skip the current class itself. + for base in defn.info.mro[1:]: + if "__init_subclass__" not in base.names: + continue + name_expr = NameExpr(defn.name) + name_expr.node = base + callee = MemberExpr(name_expr, "__init_subclass__") + args = list(defn.keywords.values()) + arg_names: list[str | None] = list(defn.keywords.keys()) + # 'metaclass' keyword is consumed by the rest of the type machinery, + # and is never passed to __init_subclass__ implementations + if "metaclass" in arg_names: + idx = arg_names.index("metaclass") + arg_names.pop(idx) + args.pop(idx) + arg_kinds = [ARG_NAMED] * len(args) + call_expr = CallExpr(callee, args, arg_kinds, arg_names) + call_expr.line = defn.line + call_expr.column = defn.column + call_expr.end_line = defn.end_line + self.expr_checker.accept(call_expr, allow_none_return=True, always_allow_any=True) + # We are only interested in the first Base having __init_subclass__, + # all other bases have already been checked. + break + + def check_typevar_defaults(self, tvars: Sequence[TypeVarLikeType]) -> None: + for tv in tvars: + if not (isinstance(tv, TypeVarType) and tv.has_default()): + continue + if not is_subtype(tv.default, tv.upper_bound): + self.fail("TypeVar default must be a subtype of the bound type", tv) + if tv.values and not any(tv.default == value for value in tv.values): + self.fail("TypeVar default must be one of the constraint types", tv) + + def check_enum(self, defn: ClassDef) -> None: + assert defn.info.is_enum + if defn.info.fullname not in ENUM_BASES and "__members__" in defn.info.names: + sym = defn.info.names["__members__"] + if isinstance(sym.node, Var) and sym.node.has_explicit_value: + # `__members__` will always be overwritten by `Enum` and is considered + # read-only so we disallow assigning a value to it + self.fail(message_registry.ENUM_MEMBERS_ATTR_WILL_BE_OVERRIDEN, sym.node) + for base in defn.info.mro[1:-1]: # we don't need self and `object` + if base.is_enum and base.fullname not in ENUM_BASES: + self.check_final_enum(defn, base) + + if self.is_stub and self.tree.fullname not in {"enum", "_typeshed"}: + if not defn.info.enum_members: + self.fail( + f'Detected enum "{defn.info.fullname}" in a type stub with zero members. ' + "There is a chance this is due to a recent change in the semantics of " + "enum membership. If so, use `member = value` to mark an enum member, " + "instead of `member: type`", + defn, + ) + self.note( + "See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members", + defn, + ) + + self.check_enum_bases(defn) + self.check_enum_new(defn) + + def check_final_enum(self, defn: ClassDef, base: TypeInfo) -> None: + for sym in base.names.values(): + if self.is_final_enum_value(sym): + self.fail(f'Cannot extend enum with existing members: "{base.name}"', defn) + break - def check_multiple_inheritance(self, typ: TypeInfo) -> None: - """Check for multiple inheritance related errors.""" + def is_final_enum_value(self, sym: SymbolTableNode) -> bool: + if isinstance(sym.node, (FuncBase, Decorator)): + return False # A method is fine + if not isinstance(sym.node, Var): + return True # Can be a class or anything else + + # Now, only `Var` is left, we need to check: + # 1. Private name like in `__prop = 1` + # 2. Dunder name like `__hash__ = some_hasher` + # 3. Sunder name like `_order_ = 'a, b, c'` + # 4. If it is a method / descriptor like in `method = classmethod(func)` + if ( + is_private(sym.node.name) + or is_dunder(sym.node.name) + or is_sunder(sym.node.name) + # TODO: make sure that `x = @class/staticmethod(func)` + # and `x = property(prop)` both work correctly. + # Now they are incorrectly counted as enum members. + or isinstance(get_proper_type(sym.node.type), FunctionLike) + ): + return False - if len(typ.bases) <= 1: + return self.is_stub or sym.node.has_explicit_value + + def check_enum_bases(self, defn: ClassDef) -> None: + """ + Non-enum mixins cannot appear after enum bases; this is disallowed at runtime: + + class Foo: ... + class Bar(enum.Enum, Foo): ... + + But any number of enum mixins can appear in a class definition + (even if multiple enum bases define __new__). So this is fine: + + class Foo(enum.Enum): + def __new__(cls, val): ... + class Bar(enum.Enum): + def __new__(cls, val): ... + class Baz(int, Foo, Bar, enum.Flag): ... + """ + enum_base: Instance | None = None + for base in defn.info.bases: + if enum_base is None and base.type.is_enum: + enum_base = base + continue + elif enum_base is not None and not base.type.is_enum: + self.fail( + f'No non-enum mixin classes are allowed after "{enum_base.str_with_options(self.options)}"', + defn, + ) + break + + def check_enum_new(self, defn: ClassDef) -> None: + def has_new_method(info: TypeInfo) -> bool: + new_method = info.get("__new__") + return bool( + new_method + and new_method.node + and new_method.node.fullname != "builtins.object.__new__" + ) + + has_new = False + for base in defn.info.bases: + candidate = False + + if base.type.is_enum: + # If we have an `Enum`, then we need to check all its bases. + candidate = any(not b.is_enum and has_new_method(b) for b in base.type.mro[1:-1]) + else: + candidate = has_new_method(base.type) + + if candidate and has_new: + self.fail( + "Only a single data type mixin is allowed for Enum subtypes, " + 'found extra "{}"'.format(base.str_with_options(self.options)), + defn, + ) + elif candidate: + has_new = True + + def check_protocol_variance(self, defn: ClassDef) -> None: + """Check that protocol definition is compatible with declared + variances of type variables. + + Note that we also prohibit declaring protocol classes as invariant + if they are actually covariant/contravariant, since this may break + transitivity of subtyping, see PEP 544. + """ + if defn.type_args is not None: + # Using new-style syntax (PEP 695), so variance will be inferred + return + info = defn.info + object_type = Instance(info.mro[-1], []) + tvars = info.defn.type_vars + for i, tvar in enumerate(tvars): + if not isinstance(tvar, TypeVarType): + # Variance of TypeVarTuple and ParamSpec is underspecified by PEPs. + continue + up_args: list[Type] = [ + object_type if i == j else AnyType(TypeOfAny.special_form) + for j, _ in enumerate(tvars) + ] + down_args: list[Type] = [ + UninhabitedType() if i == j else AnyType(TypeOfAny.special_form) + for j, _ in enumerate(tvars) + ] + up, down = Instance(info, up_args), Instance(info, down_args) + # TODO: add advanced variance checks for recursive protocols + if is_subtype(down, up, ignore_declared_variance=True): + expected = COVARIANT + elif is_subtype(up, down, ignore_declared_variance=True): + expected = CONTRAVARIANT + else: + expected = INVARIANT + if expected != tvar.variance: + self.msg.bad_proto_variance(tvar.variance, tvar.name, expected, defn) + + def check_multiple_inheritance(self, typ: TypeInfo) -> None: + """Check for multiple inheritance related errors.""" + if len(typ.bases) <= 1: # No multiple inheritance. return # Verify that inherited attributes are compatible. mro = typ.mro[1:] for i, base in enumerate(mro): - for name in base.names: - for base2 in mro[i + 1:]: + # Attributes defined in both the type and base are skipped. + # Normal checks for attribute compatibility should catch any problems elsewhere. + non_overridden_attrs = base.names.keys() - typ.names.keys() + for name in non_overridden_attrs: + if is_private(name): + continue + for base2 in mro[i + 1 :]: # We only need to check compatibility of attributes from classes not # in a subclass relationship. For subclasses, normal (single inheritance) # checks suffice (these are implemented elsewhere). - if name in base2.names and not base2 in base.mro: + if name in base2.names and base2 not in base.mro: self.check_compatibility(name, base, base2, typ) - # Verify that base class layouts are compatible. - builtin_bases = [nearest_builtin_ancestor(base.type) - for base in typ.bases] - for base1 in builtin_bases: - for base2 in builtin_bases: - if not (base1 in base2.mro or base2 in base1.mro): - self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) - # Verify that no disjointclass constraints are violated. - for base in typ.mro: - for disjoint in base.disjointclass_decls: - if disjoint in typ.mro: - self.msg.disjointness_violation(base, disjoint, typ) - - def check_compatibility(self, name: str, base1: TypeInfo, - base2: TypeInfo, ctx: Context) -> None: - if name == '__init__': - # __init__ can be incompatible -- it's a special case. + + def determine_type_of_member(self, sym: SymbolTableNode) -> Type | None: + if sym.type is not None: + return sym.type + if isinstance(sym.node, FuncBase): + return self.function_type(sym.node) + if isinstance(sym.node, TypeInfo): + if sym.node.typeddict_type: + # We special-case TypedDict, because they don't define any constructor. + return self.expr_checker.typeddict_callable(sym.node) + else: + return type_object_type(sym.node, self.named_type) + if isinstance(sym.node, TypeVarExpr): + # Use of TypeVars is rejected in an expression/runtime context, so + # we don't need to check supertype compatibility for them. + return AnyType(TypeOfAny.special_form) + if isinstance(sym.node, TypeAlias): + with self.msg.filter_errors(): + # Suppress any errors, they will be given when analyzing the corresponding node. + # Here we may have incorrect options and location context. + return self.expr_checker.alias_type_in_runtime_context(sym.node, ctx=sym.node) + # TODO: handle more node kinds here. + return None + + def check_compatibility( + self, name: str, base1: TypeInfo, base2: TypeInfo, ctx: TypeInfo + ) -> None: + """Check if attribute name in base1 is compatible with base2 in multiple inheritance. + + Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have + a direct subclass relationship (i.e., the compatibility requirement only derives from + multiple inheritance). + + This check verifies that a definition taken from base1 (and mapped to the current + class ctx), is type compatible with the definition taken from base2 (also mapped), so + that unsafe subclassing like this can be detected: + class A(Generic[T]): + def foo(self, x: T) -> None: ... + + class B: + def foo(self, x: str) -> None: ... + + class C(B, A[int]): ... # this is unsafe because... + + x: A[int] = C() + x.foo # ...runtime type is (str) -> None, while static type is (int) -> None + """ + if name in ("__init__", "__new__", "__init_subclass__"): + # __init__ and friends can be incompatible -- it's a special case. return - first = base1[name] - second = base2[name] - first_type = first.type - second_type = second.type - if (isinstance(first_type, FunctionLike) and - isinstance(second_type, FunctionLike)): - # Method override - first_sig = method_type(cast(FunctionLike, first_type)) - second_sig = method_type(cast(FunctionLike, second_type)) - # TODO Can we relax the equivalency requirement? - ok = is_equivalent(first_sig, second_sig) - else: + first = base1.names[name] + second = base2.names[name] + first_type = get_proper_type(self.determine_type_of_member(first)) + second_type = get_proper_type(self.determine_type_of_member(second)) + + # TODO: use more principled logic to decide is_subtype() vs is_equivalent(). + # We should rely on mutability of superclass node, not on types being Callable. + + # start with the special case that Instance can be a subtype of FunctionLike + call = None + if isinstance(first_type, Instance): + call = find_member("__call__", first_type, first_type, is_operator=True) + if call and isinstance(second_type, FunctionLike): + second_sig = self.bind_and_map_method(second, second_type, ctx, base2) + ok = is_subtype(call, second_sig, ignore_pos_arg_names=True) + elif isinstance(first_type, FunctionLike) and isinstance(second_type, FunctionLike): + if first_type.is_type_obj() and second_type.is_type_obj(): + # For class objects only check the subtype relationship of the classes, + # since we allow incompatible overrides of '__init__'/'__new__' + ok = is_subtype( + left=fill_typevars_with_any(first_type.type_object()), + right=fill_typevars_with_any(second_type.type_object()), + ) + else: + # First bind/map method types when necessary. + first_sig = self.bind_and_map_method(first, first_type, ctx, base1) + second_sig = self.bind_and_map_method(second, second_type, ctx, base2) + ok = is_subtype(first_sig, second_sig, ignore_pos_arg_names=True) + elif first_type and second_type: + if isinstance(first.node, Var): + first_type = expand_self_type(first.node, first_type, fill_typevars(ctx)) + if isinstance(second.node, Var): + second_type = expand_self_type(second.node, second_type, fill_typevars(ctx)) ok = is_equivalent(first_type, second_type) + if not ok: + second_node = base2[name].node + if ( + isinstance(second_type, FunctionLike) + and second_node is not None + and is_property(second_node) + ): + second_type = get_property_type(second_type) + ok = is_subtype(first_type, second_type) + else: + if first_type is None: + self.msg.cannot_determine_type_in_base(name, base1.name, ctx) + if second_type is None: + self.msg.cannot_determine_type_in_base(name, base2.name, ctx) + ok = True + # Final attributes can never be overridden, but can override + # non-final read-only attributes. + if is_final_node(second.node) and not is_private(name): + self.msg.cant_override_final(name, base2.name, ctx) + if is_final_node(first.node): + self.check_if_final_var_override_writable(name, second.node, ctx) + # Some attributes like __slots__ and __deletable__ are special, and the type can + # vary across class hierarchy. + if isinstance(second.node, Var) and second.node.allow_incompatible_override: + ok = True if not ok: - self.msg.base_class_definitions_incompatible(name, base1, base2, - ctx) + self.msg.base_class_definitions_incompatible(name, base1, base2, ctx) + + def check_metaclass_compatibility(self, typ: TypeInfo) -> None: + """Ensures that metaclasses of all parent types are compatible.""" + if ( + typ.is_metaclass() + or typ.is_protocol + or typ.is_named_tuple + or typ.is_enum + or typ.typeddict_type is not None + ): + return # Reasonable exceptions from this check + + metaclasses = [ + entry.metaclass_type + for entry in typ.mro[1:-1] + if entry.metaclass_type + and not is_named_instance(entry.metaclass_type, "builtins.type") + ] + if not metaclasses: + return + if typ.metaclass_type is not None and all( + is_subtype(typ.metaclass_type, meta) for meta in metaclasses + ): + return + self.fail( + "Metaclass conflict: the metaclass of a derived class must be " + "a (non-strict) subclass of the metaclasses of all its bases", + typ, + ) + + def visit_import_from(self, node: ImportFrom) -> None: + for name, _ in node.names: + if (sym := self.globals.get(name)) is not None: + self.warn_deprecated(sym.node, node) + self.check_import(node) + + def visit_import_all(self, node: ImportAll) -> None: + self.check_import(node) + + def visit_import(self, node: Import) -> None: + self.check_import(node) + + def check_import(self, node: ImportBase) -> None: + for assign in node.assignments: + lvalue = assign.lvalues[0] + lvalue_type, _, __ = self.check_lvalue(lvalue) + if lvalue_type is None: + # TODO: This is broken. + lvalue_type = AnyType(TypeOfAny.special_form) + assert isinstance(assign.rvalue, NameExpr) + message = message_registry.INCOMPATIBLE_IMPORT_OF.format(assign.rvalue.name) + self.check_simple_assignment( + lvalue_type, + assign.rvalue, + node, + msg=message, + lvalue_name="local name", + rvalue_name="imported name", + ) # # Statements # - def visit_block(self, b: Block) -> Type: + def visit_block(self, b: Block) -> None: if b.is_unreachable: - return None + # This block was marked as being unreachable during semantic analysis. + # It turns out any blocks marked in this way are *intentionally* marked + # as unreachable -- so we don't display an error. + self.binder.unreachable() + return for s in b.body: - self.accept(s) - if self.breaking_out: - break + if self.binder.is_unreachable(): + if not self.should_report_unreachable_issues(): + break + if not self.is_noop_for_reachability(s): + self.msg.unreachable_statement(s) + break + else: + self.accept(s) + + def should_report_unreachable_issues(self) -> bool: + return ( + self.in_checked_function() + and self.options.warn_unreachable + and not self.current_node_deferred + and not self.binder.is_unreachable_warning_suppressed() + ) + + def is_noop_for_reachability(self, s: Statement) -> bool: + """Returns 'true' if the given statement either throws an error of some kind + or is a no-op. + + We use this function while handling the '--warn-unreachable' flag. When + that flag is present, we normally report an error on any unreachable statement. + But if that statement is just something like a 'pass' or a just-in-case 'assert False', + reporting an error would be annoying. + """ + if isinstance(s, AssertStmt) and is_false_literal(s.expr): + return True + elif isinstance(s, (RaiseStmt, PassStmt)): + return True + elif isinstance(s, ExpressionStmt): + if isinstance(s.expr, EllipsisExpr): + return True + elif isinstance(s.expr, CallExpr): + with self.expr_checker.msg.filter_errors(): + typ = get_proper_type( + self.expr_checker.accept( + s.expr, allow_none_return=True, always_allow_any=True + ) + ) - def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: + if isinstance(typ, UninhabitedType): + return True + return False + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: """Type check an assignment statement. Handle all kinds of assignment statements (simple, indexed, multiple). """ - self.check_assignment(s.lvalues[-1], s.rvalue, s.type == None) + + # Avoid type checking type aliases in stubs to avoid false + # positives about modern type syntax available in stubs such + # as X | Y. + if not (s.is_alias_def and self.is_stub): + with self.enter_final_context(s.is_final_def): + self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None, s.new_syntax) + + if s.is_alias_def: + self.check_type_alias_rvalue(s) + + if ( + s.type is not None + and self.options.disallow_any_unimported + and has_any_from_unimported_type(s.type) + ): + if isinstance(s.lvalues[-1], TupleExpr): + # This is a multiple assignment. Instead of figuring out which type is problematic, + # give a generic error message. + self.msg.unimported_type_becomes_any( + "A type on this line", AnyType(TypeOfAny.special_form), s + ) + else: + self.msg.unimported_type_becomes_any("Type of variable", s.type, s) + check_for_explicit_any(s.type, self.options, self.is_typeshed_stub, self.msg, context=s) if len(s.lvalues) > 1: # Chained assignment (e.g. x = y = ...). # Make sure that rvalue type will not be reinferred. - rvalue = self.temp_node(self.type_map[s.rvalue], s) + if not self.has_type(s.rvalue): + self.expr_checker.accept(s.rvalue) + rvalue = self.temp_node(self.lookup_type(s.rvalue), s) for lv in s.lvalues[:-1]: - self.check_assignment(lv, rvalue, s.type == None) - - def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: - """Type check a single assignment: lvalue = rvalue - """ - if isinstance(lvalue, ParenExpr): - self.check_assignment(lvalue.expr, rvalue, infer_lvalue_type) - elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - ltuple = cast(Union[TupleExpr, ListExpr], lvalue) - rvalue = self.remove_parens(rvalue) - - self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, infer_lvalue_type) + with self.enter_final_context(s.is_final_def): + self.check_assignment(lv, rvalue, s.type is None) + + self.check_final(s) + if ( + s.is_final_def + and s.type + and not has_no_typevars(s.type) + and self.scope.active_class() is not None + ): + self.fail(message_registry.DEPENDENT_FINAL_IN_CLASS_BODY, s) + + if s.unanalyzed_type and not self.in_checked_function(): + self.msg.annotation_in_unchecked_function(context=s) + + def check_type_alias_rvalue(self, s: AssignmentStmt) -> None: + with self.msg.filter_errors(): + alias_type = self.expr_checker.accept(s.rvalue) + self.store_type(s.lvalues[-1], alias_type) + + def check_assignment( + self, + lvalue: Lvalue, + rvalue: Expression, + infer_lvalue_type: bool = True, + new_syntax: bool = False, + ) -> None: + """Type check a single assignment: lvalue = rvalue.""" + if isinstance(lvalue, (TupleExpr, ListExpr)): + self.check_assignment_to_multiple_lvalues( + lvalue.items, rvalue, rvalue, infer_lvalue_type + ) else: + self.try_infer_partial_generic_type_from_assignment(lvalue, rvalue, "=") lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) + # If we're assigning to __getattr__ or similar methods, check that the signature is + # valid. + if isinstance(lvalue, NameExpr) and lvalue.node: + name = lvalue.node.name + if name in ("__setattr__", "__getattribute__", "__getattr__"): + # If an explicit type is given, use that. + if lvalue_type: + signature = lvalue_type + else: + signature = self.expr_checker.accept(rvalue) + if signature: + if name == "__setattr__": + self.check_setattr_method(signature, lvalue) + else: + self.check_getattr_method(signature, lvalue, name) + + if name == "__slots__": + typ = lvalue_type or self.expr_checker.accept(rvalue) + self.check_slots_definition(typ, lvalue) + if name == "__match_args__" and inferred is not None: + typ = self.expr_checker.accept(rvalue) + self.check_match_args(inferred, typ, lvalue) + if name == "__post_init__": + active_class = self.scope.active_class() + if active_class and dataclasses_plugin.is_processed_dataclass(active_class): + self.fail(message_registry.DATACLASS_POST_INIT_MUST_BE_A_FUNCTION, rvalue) + + # Defer PartialType's super type checking. + if ( + isinstance(lvalue, RefExpr) + and not (isinstance(lvalue_type, PartialType) and lvalue_type.type is None) + and not (isinstance(lvalue, NameExpr) and lvalue.name == "__match_args__") + ): + if self.check_compatibility_all_supers(lvalue, lvalue_type, rvalue): + # We hit an error on this line; don't check for any others + return + + if isinstance(lvalue, MemberExpr) and lvalue.name == "__match_args__": + self.fail(message_registry.CANNOT_MODIFY_MATCH_ARGS, lvalue) if lvalue_type: - rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) + if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: + # Try to infer a proper type for a variable with a partial None type. + rvalue_type = self.expr_checker.accept(rvalue) + if isinstance(get_proper_type(rvalue_type), NoneType): + # This doesn't actually provide any additional information -- multiple + # None initializers preserve the partial None type. + return + + var = lvalue_type.var + if is_valid_inferred_type(rvalue_type, is_lvalue_final=var.is_final): + partial_types = self.find_partial_types(var) + if partial_types is not None: + if not self.current_node_deferred: + # Partial type can't be final, so strip any literal values. + rvalue_type = remove_instance_last_known_values(rvalue_type) + inferred_type = make_simplified_union([rvalue_type, NoneType()]) + self.set_inferred_type(var, lvalue, inferred_type) + else: + var.type = None + del partial_types[var] + lvalue_type = var.type + else: + # Try to infer a partial type. No need to check the return value, as + # an error will be reported elsewhere. + self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) + # Handle None PartialType's super type checking here, after it's resolved. + if isinstance(lvalue, RefExpr) and self.check_compatibility_all_supers( + lvalue, lvalue_type, rvalue + ): + # We hit an error on this line; don't check for any others + return + elif ( + is_literal_none(rvalue) + and isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and lvalue.node.is_initialized_in_class + and not new_syntax + ): + # Allow None's to be assigned to class variables with non-Optional types. + rvalue_type = lvalue_type + elif ( + isinstance(lvalue, MemberExpr) and lvalue.kind is None + ): # Ignore member access to modules + instance_type = self.expr_checker.accept(lvalue.expr) + rvalue_type, lvalue_type, infer_lvalue_type = self.check_member_assignment( + instance_type, lvalue_type, rvalue, context=rvalue + ) + else: + # Hacky special case for assigning a literal None + # to a variable defined in a previous if + # branch. When we detect this, we'll go back and + # make the type optional. This is somewhat + # unpleasant, and a generalization of this would + # be an improvement! + if ( + is_literal_none(rvalue) + and isinstance(lvalue, NameExpr) + and lvalue.kind == LDEF + and isinstance(lvalue.node, Var) + and lvalue.node.type + and lvalue.node in self.var_decl_frames + and not isinstance(get_proper_type(lvalue_type), AnyType) + ): + decl_frame_map = self.var_decl_frames[lvalue.node] + # Check if the nearest common ancestor frame for the definition site + # and the current site is the enclosing frame of an if/elif/else block. + has_if_ancestor = False + for frame in reversed(self.binder.frames): + if frame.id in decl_frame_map: + has_if_ancestor = frame.conditional_frame + break + if has_if_ancestor: + lvalue_type = make_optional_type(lvalue_type) + self.set_inferred_type(lvalue.node, lvalue, lvalue_type) + + rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, context=rvalue) + + # Special case: only non-abstract non-protocol classes can be assigned to + # variables with explicit type Type[A], where A is protocol or abstract. + p_rvalue_type = get_proper_type(rvalue_type) + p_lvalue_type = get_proper_type(lvalue_type) + if ( + isinstance(p_rvalue_type, FunctionLike) + and p_rvalue_type.is_type_obj() + and ( + p_rvalue_type.type_object().is_abstract + or p_rvalue_type.type_object().is_protocol + ) + and isinstance(p_lvalue_type, TypeType) + and isinstance(p_lvalue_type.item, Instance) + and ( + p_lvalue_type.item.type.is_abstract or p_lvalue_type.item.type.is_protocol + ) + ): + self.msg.concrete_only_assign(p_lvalue_type, rvalue) + return + if rvalue_type and infer_lvalue_type and not isinstance(lvalue_type, PartialType): + # Don't use type binder for definitions of special forms, like named tuples. + if not (isinstance(lvalue, NameExpr) and lvalue.is_special_form): + self.binder.assign_type(lvalue, rvalue_type, lvalue_type, False) + if ( + isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and lvalue.node.is_inferred + and lvalue.node.is_index_var + and lvalue_type is not None + ): + lvalue.node.type = remove_instance_last_known_values(lvalue_type) - if rvalue_type and infer_lvalue_type: - self.binder.assign_type(lvalue, rvalue_type) elif index_lvalue: - self.check_indexed_assignment(index_lvalue, rvalue, rvalue) + self.check_indexed_assignment(index_lvalue, rvalue, lvalue) if inferred: - self.infer_variable_type(inferred, lvalue, self.accept(rvalue), - rvalue) + type_context = self.get_variable_type_context(inferred) + rvalue_type = self.expr_checker.accept(rvalue, type_context=type_context) + if not ( + inferred.is_final + or inferred.is_index_var + or (isinstance(lvalue, NameExpr) and lvalue.name == "__match_args__") + ): + rvalue_type = remove_instance_last_known_values(rvalue_type) + self.infer_variable_type(inferred, lvalue, rvalue_type, rvalue) + self.check_assignment_to_slots(lvalue) + + # (type, operator) tuples for augmented assignments supported with partial types + partial_type_augmented_ops: Final = {("builtins.list", "+"), ("builtins.set", "|")} + + def get_variable_type_context(self, inferred: Var) -> Type | None: + type_contexts = [] + if inferred.info: + for base in inferred.info.mro[1:]: + base_type, base_node = self.lvalue_type_from_base(inferred, base) + if ( + base_type + and not (isinstance(base_node, Var) and base_node.invalid_partial_type) + and not isinstance(base_type, PartialType) + ): + type_contexts.append(base_type) + # Use most derived supertype as type context if available. + if not type_contexts: + return None + candidate = type_contexts[0] + for other in type_contexts: + if is_proper_subtype(other, candidate): + candidate = other + elif not is_subtype(candidate, other): + # Multiple incompatible candidates, cannot use any of them as context. + return None + return candidate + + def try_infer_partial_generic_type_from_assignment( + self, lvalue: Lvalue, rvalue: Expression, op: str + ) -> None: + """Try to infer a precise type for partial generic type from assignment. + + 'op' is '=' for normal assignment and a binary operator ('+', ...) for + augmented assignment. + + Example where this happens: + + x = [] + if foo(): + x = [1] # Infer List[int] as type of 'x' + """ + var = None + if ( + isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and isinstance(lvalue.node.type, PartialType) + ): + var = lvalue.node + elif isinstance(lvalue, MemberExpr): + var = self.expr_checker.get_partial_self_var(lvalue) + if var is not None: + typ = var.type + assert isinstance(typ, PartialType) + if typ.type is None: + return + # Return if this is an unsupported augmented assignment. + if op != "=" and (typ.type.fullname, op) not in self.partial_type_augmented_ops: + return + # TODO: some logic here duplicates the None partial type counterpart + # inlined in check_assignment(), see #8043. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + rvalue_type = self.expr_checker.accept(rvalue) + rvalue_type = get_proper_type(rvalue_type) + if isinstance(rvalue_type, Instance): + if rvalue_type.type == typ.type and is_valid_inferred_type(rvalue_type): + var.type = rvalue_type + del partial_types[var] + elif isinstance(rvalue_type, AnyType): + var.type = fill_typevars_with_any(typ.type) + del partial_types[var] + + def check_compatibility_all_supers( + self, lvalue: RefExpr, lvalue_type: Type | None, rvalue: Expression + ) -> bool: + lvalue_node = lvalue.node + # Check if we are a class variable with at least one base class + if ( + isinstance(lvalue_node, Var) + and lvalue.kind in (MDEF, None) + and len(lvalue_node.info.bases) > 0 # None for Vars defined via self + ): + for base in lvalue_node.info.mro[1:]: + tnode = base.names.get(lvalue_node.name) + if tnode is not None: + if not self.check_compatibility_classvar_super(lvalue_node, base, tnode.node): + # Show only one error per variable + break + + if not self.check_compatibility_final_super(lvalue_node, base, tnode.node): + # Show only one error per variable + break + + direct_bases = lvalue_node.info.direct_base_classes() + last_immediate_base = direct_bases[-1] if direct_bases else None + + for base in lvalue_node.info.mro[1:]: + # The type of "__slots__" and some other attributes usually doesn't need to + # be compatible with a base class. We'll still check the type of "__slots__" + # against "object" as an exception. + if lvalue_node.allow_incompatible_override and not ( + lvalue_node.name == "__slots__" and base.fullname == "builtins.object" + ): + continue + + if is_private(lvalue_node.name): + continue + + base_type, base_node = self.lvalue_type_from_base(lvalue_node, base) + if isinstance(base_type, PartialType): + base_type = None + + if base_type: + assert base_node is not None + if not self.check_compatibility_super( + lvalue, lvalue_type, rvalue, base, base_type, base_node + ): + # Only show one error per variable; even if other + # base classes are also incompatible + return True + if base is last_immediate_base: + # At this point, the attribute was found to be compatible with all + # immediate parents. + break + return False + + def check_compatibility_super( + self, + lvalue: RefExpr, + lvalue_type: Type | None, + rvalue: Expression, + base: TypeInfo, + base_type: Type, + base_node: Node, + ) -> bool: + lvalue_node = lvalue.node + assert isinstance(lvalue_node, Var) + + # Do not check whether the rvalue is compatible if the + # lvalue had a type defined; this is handled by other + # parts, and all we have to worry about in that case is + # that lvalue is compatible with the base class. + compare_node = None + if lvalue_type: + compare_type = lvalue_type + compare_node = lvalue.node + else: + compare_type = self.expr_checker.accept(rvalue, base_type) + if isinstance(rvalue, NameExpr): + compare_node = rvalue.node + if isinstance(compare_node, Decorator): + compare_node = compare_node.func + + base_type = get_proper_type(base_type) + compare_type = get_proper_type(compare_type) + if compare_type: + if isinstance(base_type, CallableType) and isinstance(compare_type, CallableType): + base_static = is_node_static(base_node) + compare_static = is_node_static(compare_node) + + # In case compare_static is unknown, also check + # if 'definition' is set. The most common case for + # this is with TempNode(), where we lose all + # information about the real rvalue node (but only get + # the rvalue type) + if compare_static is None and compare_type.definition: + compare_static = is_node_static(compare_type.definition) + + # Compare against False, as is_node_static can return None + if base_static is False and compare_static is False: + # Class-level function objects and classmethods become bound + # methods: the former to the instance, the latter to the + # class + base_type = bind_self(base_type, self.scope.active_self_type()) + compare_type = bind_self(compare_type, self.scope.active_self_type()) + + # If we are a static method, ensure to also tell the + # lvalue it now contains a static method + if base_static and compare_static: + lvalue_node.is_staticmethod = True + + ok = self.check_subtype( + compare_type, + base_type, + rvalue, + message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + "expression has type", + f'base class "{base.name}" defined the type as', + ) + if ( + ok + and codes.MUTABLE_OVERRIDE in self.options.enabled_error_codes + and self.is_writable_attribute(base_node) + ): + ok = self.check_subtype( + base_type, + compare_type, + rvalue, + message_registry.COVARIANT_OVERRIDE_OF_MUTABLE_ATTRIBUTE, + f'base class "{base.name}" defined the type as', + "expression has type", + ) + return ok + return True + + def lvalue_type_from_base( + self, expr_node: Var, base: TypeInfo + ) -> tuple[Type | None, Node | None]: + """For a NameExpr that is part of a class, walk all base classes and try + to find the first class that defines a Type for the same name.""" + expr_name = expr_node.name + base_var = base.names.get(expr_name) + + if base_var: + base_node = base_var.node + base_type = base_var.type + if isinstance(base_node, Var) and base_type is not None: + base_type = expand_self_type(base_node, base_type, fill_typevars(expr_node.info)) + if isinstance(base_node, Decorator): + base_node = base_node.func + base_type = base_node.type + + if base_type: + if not has_no_typevars(base_type): + self_type = self.scope.active_self_type() + assert self_type is not None, "Internal error: base lookup outside class" + if isinstance(self_type, TupleType): + instance = tuple_fallback(self_type) + else: + instance = self_type + itype = map_instance_to_supertype(instance, base) + base_type = expand_type_by_instance(base_type, itype) + + base_type = get_proper_type(base_type) + if isinstance(base_type, CallableType) and isinstance(base_node, FuncDef): + # If we are a property, return the Type of the return + # value, not the Callable + if base_node.is_property: + base_type = get_proper_type(base_type.ret_type) + if isinstance(base_type, FunctionLike) and isinstance( + base_node, OverloadedFuncDef + ): + # Same for properties with setter + if base_node.is_property: + base_type = base_type.items[0].ret_type + + return base_type, base_node + + return None, None + + def check_compatibility_classvar_super( + self, node: Var, base: TypeInfo, base_node: Node | None + ) -> bool: + if not isinstance(base_node, Var): + return True + if node.is_classvar and not base_node.is_classvar: + self.fail(message_registry.CANNOT_OVERRIDE_INSTANCE_VAR.format(base.name), node) + return False + elif not node.is_classvar and base_node.is_classvar: + self.fail(message_registry.CANNOT_OVERRIDE_CLASS_VAR.format(base.name), node) + return False + return True + + def check_compatibility_final_super( + self, node: Var, base: TypeInfo, base_node: Node | None + ) -> bool: + """Check if an assignment overrides a final attribute in a base class. + + This only checks situations where either a node in base class is not a variable + but a final method, or where override is explicitly declared as final. + In these cases we give a more detailed error message. In addition, we check that + a final variable doesn't override writeable attribute, which is not safe. + + Other situations are checked in `check_final()`. + """ + if not isinstance(base_node, (Var, FuncBase, Decorator)): + return True + if is_private(node.name): + return True + if base_node.is_final and (node.is_final or not isinstance(base_node, Var)): + # Give this error only for explicit override attempt with `Final`, or + # if we are overriding a final method with variable. + # Other override attempts will be flagged as assignment to constant + # in `check_final()`. + self.msg.cant_override_final(node.name, base.name, node) + return False + if node.is_final: + if base.fullname in ENUM_BASES or node.name in ENUM_SPECIAL_PROPS: + return True + self.check_if_final_var_override_writable(node.name, base_node, node) + return True + + def check_if_final_var_override_writable( + self, name: str, base_node: Node | None, ctx: Context + ) -> None: + """Check that a final variable doesn't override writeable attribute. + + This is done to prevent situations like this: + class C: + attr = 1 + class D(C): + attr: Final = 2 - def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, context: Context, - infer_lvalue_type: bool = True) -> None: - if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): + x: C = D() + x.attr = 3 # Oops! + """ + writable = True + if base_node: + writable = self.is_writable_attribute(base_node) + if writable: + self.msg.final_cant_override_writable(name, ctx) + + def get_final_context(self) -> bool: + """Check whether we a currently checking a final declaration.""" + return self._is_final_def + + @contextmanager + def enter_final_context(self, is_final_def: bool) -> Iterator[None]: + """Store whether the current checked assignment is a final declaration.""" + old_ctx = self._is_final_def + self._is_final_def = is_final_def + try: + yield + finally: + self._is_final_def = old_ctx + + def check_final(self, s: AssignmentStmt | OperatorAssignmentStmt | AssignmentExpr) -> None: + """Check if this assignment does not assign to a final attribute. + + This function performs the check only for name assignments at module + and class scope. The assignments to `obj.attr` and `Cls.attr` are checked + in checkmember.py. + """ + if isinstance(s, AssignmentStmt): + lvs = self.flatten_lvalues(s.lvalues) + elif isinstance(s, AssignmentExpr): + lvs = [s.target] + else: + lvs = [s.lvalue] + is_final_decl = s.is_final_def if isinstance(s, AssignmentStmt) else False + if is_final_decl and self.scope.active_class(): + lv = lvs[0] + assert isinstance(lv, RefExpr) + if lv.node is not None: + assert isinstance(lv.node, Var) + if ( + lv.node.final_unset_in_class + and not lv.node.final_set_in_init + and not self.is_stub # It is OK to skip initializer in stub files. + and + # Avoid extra error messages, if there is no type in Final[...], + # then we already reported the error about missing r.h.s. + isinstance(s, AssignmentStmt) + and s.type is not None + ): + self.msg.final_without_value(s) + for lv in lvs: + if isinstance(lv, RefExpr) and isinstance(lv.node, Var): + name = lv.node.name + cls = self.scope.active_class() + if cls is not None: + # These additional checks exist to give more error messages + # even if the final attribute was overridden with a new symbol + # (which is itself an error)... + for base in cls.mro[1:]: + sym = base.names.get(name) + # We only give this error if base node is variable, + # overriding final method will be caught in + # `check_compatibility_final_super()`. + if sym and isinstance(sym.node, Var): + if sym.node.is_final and not is_final_decl: + self.msg.cant_assign_to_final(name, sym.node.info is None, s) + # ...but only once + break + if lv.node.is_final and not is_final_decl: + self.msg.cant_assign_to_final(name, lv.node.info is None, s) + + def check_assignment_to_slots(self, lvalue: Lvalue) -> None: + if not isinstance(lvalue, MemberExpr): + return + + inst = get_proper_type(self.expr_checker.accept(lvalue.expr)) + if not isinstance(inst, Instance): + return + if inst.type.slots is None: + return # Slots do not exist, we can allow any assignment + if lvalue.name in inst.type.slots: + return # We are assigning to an existing slot + for base_info in inst.type.mro[:-1]: + if base_info.names.get("__setattr__") is not None: + # When type has `__setattr__` defined, + # we can assign any dynamic value. + # We exclude object, because it always has `__setattr__`. + return + + definition = inst.type.get(lvalue.name) + if definition is None: + # We don't want to duplicate + # `"SomeType" has no attribute "some_attr"` + # error twice. + return + if self.is_assignable_slot(lvalue, definition.type): + return + + self.fail( + message_registry.NAME_NOT_IN_SLOTS.format(lvalue.name, inst.type.fullname), lvalue + ) + + def is_assignable_slot(self, lvalue: Lvalue, typ: Type | None) -> bool: + if getattr(lvalue, "node", None): + return False # This is a definition + + typ = get_proper_type(typ) + if typ is None or isinstance(typ, AnyType): + return True # Any can be literally anything, like `@propery` + if isinstance(typ, Instance): + # When working with instances, we need to know if they contain + # `__set__` special method. Like `@property` does. + # This makes assigning to properties possible, + # even without extra slot spec. + return typ.type.get("__set__") is not None + if isinstance(typ, FunctionLike): + return True # Can be a property, or some other magic + if isinstance(typ, UnionType): + return all(self.is_assignable_slot(lvalue, u) for u in typ.items) + return False + + def flatten_rvalues(self, rvalues: list[Expression]) -> list[Expression]: + """Flatten expression list by expanding those * items that have tuple type. + + For each regular type item in the tuple type use a TempNode(), for an Unpack + item use a corresponding StarExpr(TempNode()). + """ + new_rvalues = [] + for rv in rvalues: + if not isinstance(rv, StarExpr): + new_rvalues.append(rv) + continue + typ = get_proper_type(self.expr_checker.accept(rv.expr)) + if not isinstance(typ, TupleType): + new_rvalues.append(rv) + continue + for t in typ.items: + if not isinstance(t, UnpackType): + new_rvalues.append(TempNode(t)) + else: + unpacked = get_proper_type(t.type) + if isinstance(unpacked, TypeVarTupleType): + fallback = unpacked.upper_bound + else: + assert ( + isinstance(unpacked, Instance) + and unpacked.type.fullname == "builtins.tuple" + ) + fallback = unpacked + new_rvalues.append(StarExpr(TempNode(fallback))) + return new_rvalues + + def check_assignment_to_multiple_lvalues( + self, + lvalues: list[Lvalue], + rvalue: Expression, + context: Context, + infer_lvalue_type: bool = True, + ) -> None: + if isinstance(rvalue, (TupleExpr, ListExpr)): # Recursively go into Tuple or List expression rhs instead of - # using the type of rhs, because this allowed more fine grained + # using the type of rhs, because this allows more fine-grained # control in cases like: a, b = [int, str] where rhs would get # type List[object] - - rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items + rvalues: list[Expression] = [] + iterable_type: Type | None = None + last_idx: int | None = None + for idx_rval, rval in enumerate(self.flatten_rvalues(rvalue.items)): + if isinstance(rval, StarExpr): + typs = get_proper_type(self.expr_checker.accept(rval.expr)) + if self.type_is_iterable(typs) and isinstance(typs, Instance): + if iterable_type is not None and iterable_type != self.iterable_item_type( + typs, rvalue + ): + self.fail(message_registry.CONTIGUOUS_ITERABLE_EXPECTED, context) + else: + if last_idx is None or last_idx + 1 == idx_rval: + rvalues.append(rval) + last_idx = idx_rval + iterable_type = self.iterable_item_type(typs, rvalue) + else: + self.fail(message_registry.CONTIGUOUS_ITERABLE_EXPECTED, context) + else: + self.fail(message_registry.ITERABLE_TYPE_EXPECTED.format(typs), context) + else: + rvalues.append(rval) + iterable_start: int | None = None + iterable_end: int | None = None + for i, rval in enumerate(rvalues): + if isinstance(rval, StarExpr): + typs = get_proper_type(self.expr_checker.accept(rval.expr)) + if self.type_is_iterable(typs) and isinstance(typs, Instance): + if iterable_start is None: + iterable_start = i + iterable_end = i + if ( + iterable_start is not None + and iterable_end is not None + and iterable_type is not None + ): + iterable_num = iterable_end - iterable_start + 1 + rvalue_needed = len(lvalues) - (len(rvalues) - iterable_num) + if rvalue_needed > 0: + rvalues = ( + rvalues[0:iterable_start] + + [TempNode(iterable_type) for i in range(rvalue_needed)] + + rvalues[iterable_end + 1 :] + ) if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): - star_index = next((i for i, lv in enumerate(lvalues) if - isinstance(lv, StarExpr)), len(lvalues)) + star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) + ) left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index+1:] + star_lv = ( + cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + ) + right_lvs = lvalues[star_index + 1 :] left_rvs, star_rvs, right_rvs = self.split_around_star( - rvalues, star_index, len(lvalues)) + rvalues, star_index, len(lvalues) + ) lr_pairs = list(zip(left_lvs, left_rvs)) if star_lv: rv_list = ListExpr(star_rvs) - rv_list.set_line(rvalue.get_line()) - lr_pairs.append( (star_lv.expr, rv_list) ) + rv_list.set_line(rvalue) + lr_pairs.append((star_lv.expr, rv_list)) lr_pairs.extend(zip(right_lvs, right_rvs)) for lv, rv in lr_pairs: @@ -983,99 +3767,285 @@ def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node else: self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) - def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, - context: Context) -> bool: + def check_rvalue_count_in_assignment( + self, + lvalues: list[Lvalue], + rvalue_count: int, + context: Context, + rvalue_unpack: int | None = None, + ) -> bool: + if rvalue_unpack is not None: + if not any(isinstance(e, StarExpr) for e in lvalues): + self.fail("Variadic tuple unpacking requires a star target", context) + return False + if len(lvalues) > rvalue_count: + self.fail(message_registry.TOO_MANY_TARGETS_FOR_VARIADIC_UNPACK, context) + return False + left_star_index = next(i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)) + left_prefix = left_star_index + left_suffix = len(lvalues) - left_star_index - 1 + right_prefix = rvalue_unpack + right_suffix = rvalue_count - rvalue_unpack - 1 + if left_suffix > right_suffix or left_prefix > right_prefix: + # Case of asymmetric unpack like: + # rv: tuple[int, *Ts, int, int] + # x, y, *xs, z = rv + # it is technically valid, but is tricky to reason about. + # TODO: support this (at least if the r.h.s. unpack is a homogeneous tuple). + self.fail(message_registry.TOO_MANY_TARGETS_FOR_VARIADIC_UNPACK, context) + return True if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): - if len(lvalues)-1 > rvalue_count: - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues)-1, context) + if len(lvalues) - 1 > rvalue_count: + self.msg.wrong_number_values_to_unpack(rvalue_count, len(lvalues) - 1, context) return False elif rvalue_count != len(lvalues): - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues), context) + self.msg.wrong_number_values_to_unpack(rvalue_count, len(lvalues), context) return False return True - def remove_parens(self, node: Node) -> Node: - if isinstance(node, ParenExpr): - return self.remove_parens(node.expr) - else: - return node - - def check_multi_assignment(self, lvalues: List[Node], - rvalue: Node, - context: Context, - infer_lvalue_type: bool = True, - msg: str = None) -> None: - """Check the assignment of one rvalue to a number of lvalues - for example from a ListExpr or TupleExpr. - """ - - if not msg: - msg = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT - - # First handle case where rvalue is of form Undefined, ... - rvalue_type = get_undefined_tuple(rvalue, self.named_type('builtins.tuple')) - undefined_rvalue = True - if not rvalue_type: - # Infer the type of an ordinary rvalue expression. - rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant - undefined_rvalue = False + def check_multi_assignment( + self, + lvalues: list[Lvalue], + rvalue: Expression, + context: Context, + infer_lvalue_type: bool = True, + rv_type: Type | None = None, + undefined_rvalue: bool = False, + ) -> None: + """Check the assignment of one rvalue to a number of lvalues.""" + + # Infer the type of an ordinary rvalue expression. + # TODO: maybe elsewhere; redundant. + rvalue_type = get_proper_type(rv_type or self.expr_checker.accept(rvalue)) + + if isinstance(rvalue_type, TypeVarLikeType): + rvalue_type = get_proper_type(rvalue_type.upper_bound) + + if isinstance(rvalue_type, UnionType): + # If this is an Optional type in non-strict Optional code, unwrap it. + relevant_items = rvalue_type.relevant_items() + if len(relevant_items) == 1: + rvalue_type = get_proper_type(relevant_items[0]) + + if ( + isinstance(rvalue_type, TupleType) + and find_unpack_in_list(rvalue_type.items) is not None + ): + # Normalize for consistent handling with "old-style" homogeneous tuples. + rvalue_type = expand_type(rvalue_type, {}) if isinstance(rvalue_type, AnyType): for lv in lvalues: if isinstance(lv, StarExpr): lv = lv.expr - self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) + temp_node = self.temp_node( + AnyType(TypeOfAny.from_another_any, source_any=rvalue_type), context + ) + self.check_assignment(lv, temp_node, infer_lvalue_type) elif isinstance(rvalue_type, TupleType): - self.check_multi_assignment_from_tuple(lvalues, rvalue, cast(TupleType, rvalue_type), - context, undefined_rvalue, infer_lvalue_type) + self.check_multi_assignment_from_tuple( + lvalues, rvalue, rvalue_type, context, undefined_rvalue, infer_lvalue_type + ) + elif isinstance(rvalue_type, UnionType): + self.check_multi_assignment_from_union( + lvalues, rvalue, rvalue_type, context, infer_lvalue_type + ) + elif isinstance(rvalue_type, Instance) and rvalue_type.type.fullname == "builtins.str": + self.msg.unpacking_strings_disallowed(context) else: - self.check_multi_assignment_from_iterable(lvalues, rvalue_type, - context, infer_lvalue_type) - - def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, - rvalue_type: TupleType, context: Context, - undefined_rvalue: bool, infer_lvalue_type: bool=True) -> None: - if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): - star_index = next((i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues)) + self.check_multi_assignment_from_iterable( + lvalues, rvalue_type, context, infer_lvalue_type + ) + + def check_multi_assignment_from_union( + self, + lvalues: list[Expression], + rvalue: Expression, + rvalue_type: UnionType, + context: Context, + infer_lvalue_type: bool, + ) -> None: + """Check assignment to multiple lvalue targets when rvalue type is a Union[...]. + For example: + + t: Union[Tuple[int, int], Tuple[str, str]] + x, y = t + reveal_type(x) # Union[int, str] + + The idea in this case is to process the assignment for every item of the union. + Important note: the types are collected in two places, 'union_types' contains + inferred types for first assignments, 'assignments' contains the narrowed types + for binder. + """ + self.no_partial_types = True + transposed: tuple[list[Type], ...] = tuple([] for _ in self.flatten_lvalues(lvalues)) + # Notify binder that we want to defer bindings and instead collect types. + with self.binder.accumulate_type_assignments() as assignments: + for item in rvalue_type.items: + # Type check the assignment separately for each union item and collect + # the inferred lvalue types for each union item. + self.check_multi_assignment( + lvalues, + rvalue, + context, + infer_lvalue_type=infer_lvalue_type, + rv_type=item, + undefined_rvalue=True, + ) + for t, lv in zip(transposed, self.flatten_lvalues(lvalues)): + # We can access _type_maps directly since temporary type maps are + # only created within expressions. + t.append(self._type_maps[0].pop(lv, AnyType(TypeOfAny.special_form))) + union_types = tuple(make_simplified_union(col) for col in transposed) + for expr, items in assignments.items(): + # Bind a union of types collected in 'assignments' to every expression. + if isinstance(expr, StarExpr): + expr = expr.expr + + # TODO: See todo in binder.py, ConditionalTypeBinder.assign_type + # It's unclear why the 'declared_type' param is sometimes 'None' + clean_items: list[tuple[Type, Type]] = [] + for type, declared_type in items: + assert declared_type is not None + clean_items.append((type, declared_type)) + + types, declared_types = zip(*clean_items) + self.binder.assign_type( + expr, + make_simplified_union(list(types)), + make_simplified_union(list(declared_types)), + False, + ) + for union, lv in zip(union_types, self.flatten_lvalues(lvalues)): + # Properly store the inferred types. + _1, _2, inferred = self.check_lvalue(lv) + if inferred: + self.set_inferred_type(inferred, lv, union) + else: + self.store_type(lv, union) + self.no_partial_types = False + + def flatten_lvalues(self, lvalues: list[Expression]) -> list[Expression]: + res: list[Expression] = [] + for lv in lvalues: + if isinstance(lv, (TupleExpr, ListExpr)): + res.extend(self.flatten_lvalues(lv.items)) + if isinstance(lv, StarExpr): + # Unwrap StarExpr, since it is unwrapped by other helpers. + lv = lv.expr + res.append(lv) + return res + + def check_multi_assignment_from_tuple( + self, + lvalues: list[Lvalue], + rvalue: Expression, + rvalue_type: TupleType, + context: Context, + undefined_rvalue: bool, + infer_lvalue_type: bool = True, + ) -> None: + rvalue_unpack = find_unpack_in_list(rvalue_type.items) + if self.check_rvalue_count_in_assignment( + lvalues, len(rvalue_type.items), context, rvalue_unpack=rvalue_unpack + ): + star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) + ) left_lvs = lvalues[:star_index] star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index+1:] + right_lvs = lvalues[star_index + 1 :] if not undefined_rvalue: # Infer rvalue again, now in the correct type context. lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) - rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) + reinferred_rvalue_type = get_proper_type( + self.expr_checker.accept(rvalue, lvalue_type) + ) + + if isinstance(reinferred_rvalue_type, TypeVarLikeType): + reinferred_rvalue_type = get_proper_type(reinferred_rvalue_type.upper_bound) + if isinstance(reinferred_rvalue_type, UnionType): + # If this is an Optional type in non-strict Optional code, unwrap it. + relevant_items = reinferred_rvalue_type.relevant_items() + if len(relevant_items) == 1: + reinferred_rvalue_type = get_proper_type(relevant_items[0]) + if isinstance(reinferred_rvalue_type, UnionType): + self.check_multi_assignment_from_union( + lvalues, rvalue, reinferred_rvalue_type, context, infer_lvalue_type + ) + return + if isinstance(reinferred_rvalue_type, AnyType): + # We can get Any if the current node is + # deferred. Doing more inference in deferred nodes + # is hard, so give up for now. We can also get + # here if reinferring types above changes the + # inferred return type for an overloaded function + # to be ambiguous. + return + assert isinstance(reinferred_rvalue_type, TupleType) + rvalue_type = reinferred_rvalue_type left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) + rvalue_type.items, star_index, len(lvalues) + ) for lv, rv_type in zip(left_lvs, left_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) if star_lv: - nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] - list_expr = ListExpr(nodes) - list_expr.set_line(context.get_line()) + list_expr = ListExpr( + [ + ( + self.temp_node(rv_type, context) + if not isinstance(rv_type, UnpackType) + else StarExpr(self.temp_node(rv_type.type, context)) + ) + for rv_type in star_rv_types + ] + ) + list_expr.set_line(context) self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) for lv, rv_type in zip(right_lvs, right_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - - def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: - star_index = next((i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues)) + else: + # Store meaningful Any types for lvalues, errors are already given + # by check_rvalue_count_in_assignment() + if infer_lvalue_type: + for lv in lvalues: + if ( + isinstance(lv, NameExpr) + and isinstance(lv.node, Var) + and lv.node.type is None + ): + lv.node.type = AnyType(TypeOfAny.from_error) + elif isinstance(lv, StarExpr): + if ( + isinstance(lv.expr, NameExpr) + and isinstance(lv.expr.node, Var) + and lv.expr.node.type is None + ): + lv.expr.node.type = self.named_generic_type( + "builtins.list", [AnyType(TypeOfAny.from_error)] + ) + + def lvalue_type_for_inference(self, lvalues: list[Lvalue], rvalue_type: TupleType) -> Type: + star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) + ) left_lvs = lvalues[:star_index] star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index+1:] + right_lvs = lvalues[star_index + 1 :] left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) + rvalue_type.items, star_index, len(lvalues) + ) - type_parameters = [] # type: List[Type] + type_parameters: list[Type] = [] - def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: + def append_types_for_inference(lvs: list[Expression], rv_types: list[Type]) -> None: for lv, rv_type in zip(lvs, rv_types): sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) - if sub_lvalue_type: + if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): type_parameters.append(sub_lvalue_type) else: # index lvalue # TODO Figure out more precise type context, probably @@ -1086,7 +4056,7 @@ def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: if star_lv: sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) - if sub_lvalue_type: + if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) else: # index lvalue # TODO Figure out more precise type context, probably @@ -1095,10 +4065,11 @@ def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: append_types_for_inference(right_lvs, right_rv_types) - return TupleType(type_parameters, self.named_type('builtins.tuple')) + return TupleType(type_parameters, self.named_type("builtins.tuple")) - def split_around_star(self, items: List[T], star_index: int, - length: int) -> Tuple[List[T], List[T], List[T]]: + def split_around_star( + self, items: list[T], star_index: int, length: int + ) -> tuple[list[T], list[T], list[T]]: """Splits a list of items in three to match another list of length 'length' that contains a starred expression at 'star_index' in the following way: @@ -1106,64 +4077,87 @@ def split_around_star(self, items: List[T], star_index: int, returns in: ([1,2], [3,4,5], [6,7]) """ nr_right_of_star = length - star_index - 1 - right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) + right_index = -nr_right_of_star if nr_right_of_star != 0 else len(items) left = items[:star_index] star = items[star_index:right_index] right = items[right_index:] - return (left, star, right) + return left, star, right def type_is_iterable(self, type: Type) -> bool: - return (is_subtype(type, self.named_generic_type('typing.Iterable', - [AnyType()])) and - isinstance(type, Instance)) - - def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, - context: Context, infer_lvalue_type: bool=True) -> None: - if self.type_is_iterable(rvalue_type): - item_type = self.iterable_item_type(cast(Instance,rvalue_type)) + type = get_proper_type(type) + if isinstance(type, FunctionLike) and type.is_type_obj(): + type = type.fallback + return is_subtype( + type, self.named_generic_type("typing.Iterable", [AnyType(TypeOfAny.special_form)]) + ) + + def check_multi_assignment_from_iterable( + self, + lvalues: list[Lvalue], + rvalue_type: Type, + context: Context, + infer_lvalue_type: bool = True, + ) -> None: + rvalue_type = get_proper_type(rvalue_type) + if self.type_is_iterable(rvalue_type) and isinstance( + rvalue_type, (Instance, CallableType, TypeType, Overloaded) + ): + item_type = self.iterable_item_type(rvalue_type, context) for lv in lvalues: if isinstance(lv, StarExpr): - self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), infer_lvalue_type) + items_type = self.named_generic_type("builtins.list", [item_type]) + self.check_assignment( + lv.expr, self.temp_node(items_type, context), infer_lvalue_type + ) else: - self.check_assignment(lv, self.temp_node(item_type, context), infer_lvalue_type) + self.check_assignment( + lv, self.temp_node(item_type, context), infer_lvalue_type + ) else: self.msg.type_not_iterable(rvalue_type, context) - def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: - lvalue_type = None # type: Type - index_lvalue = None # type: IndexExpr - inferred = None # type: Var + def check_lvalue(self, lvalue: Lvalue) -> tuple[Type | None, IndexExpr | None, Var | None]: + lvalue_type = None + index_lvalue = None + inferred = None - if self.is_definition(lvalue): + if self.is_definition(lvalue) and ( + not isinstance(lvalue, NameExpr) or isinstance(lvalue.node, Var) + ): if isinstance(lvalue, NameExpr): - inferred = cast(Var, lvalue.node) + assert isinstance(lvalue.node, Var) + inferred = lvalue.node else: - m = cast(MemberExpr, lvalue) - self.accept(m.expr) - inferred = m.def_var + assert isinstance(lvalue, MemberExpr) + self.expr_checker.accept(lvalue.expr) + inferred = lvalue.def_var elif isinstance(lvalue, IndexExpr): index_lvalue = lvalue elif isinstance(lvalue, MemberExpr): - lvalue_type = self.expr_checker.analyse_ordinary_member_access(lvalue, - True) + lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, True) self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, NameExpr): - lvalue_type = self.expr_checker.analyse_ref_expr(lvalue) + lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - lv = cast(Union[TupleExpr, ListExpr], lvalue) - types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] - lvalue_type = TupleType(types, self.named_type('builtins.tuple')) - elif isinstance(lvalue, ParenExpr): - return self.check_lvalue(lvalue.expr) + elif isinstance(lvalue, (TupleExpr, ListExpr)): + types = [ + self.check_lvalue(sub_expr)[0] or + # This type will be used as a context for further inference of rvalue, + # we put Uninhabited if there is no information available from lvalue. + UninhabitedType() + for sub_expr in lvalue.items + ] + lvalue_type = TupleType(types, self.named_type("builtins.tuple")) + elif isinstance(lvalue, StarExpr): + lvalue_type, _, _ = self.check_lvalue(lvalue.expr) else: - lvalue_type = self.accept(lvalue) + lvalue_type = self.expr_checker.accept(lvalue) return lvalue_type, index_lvalue, inferred - def is_definition(self, s: Node) -> bool: + def is_definition(self, s: Lvalue) -> bool: if isinstance(s, NameExpr): - if s.is_def: + if s.is_inferred_def: return True # If the node type is not defined, this must the first assignment # that we process => this is a definition, even though the semantic @@ -1174,18 +4168,36 @@ def is_definition(self, s: Node) -> bool: if isinstance(node, Var): return node.type is None elif isinstance(s, MemberExpr): - return s.is_def + return s.is_inferred_def return False - def infer_variable_type(self, name: Var, lvalue: Node, - init_type: Type, context: Context) -> None: + def infer_variable_type( + self, name: Var, lvalue: Lvalue, init_type: Type, context: Context + ) -> None: """Infer the type of initialized variables from initializer type.""" - if isinstance(init_type, Void): - self.check_not_void(init_type, context) - elif not self.is_valid_inferred_type(init_type): - # We cannot use the type of the initialization expression for type - # inference (it's not specific enough). - self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + if isinstance(init_type, DeletedType): + self.msg.deleted_as_rvalue(init_type, context) + elif ( + not is_valid_inferred_type(init_type, is_lvalue_final=name.is_final) + and not self.no_partial_types + ): + # We cannot use the type of the initialization expression for full type + # inference (it's not specific enough), but we might be able to give + # partial type which will be made more specific later. A partial type + # gets generated in assignment like 'x = []' where item type is not known. + if not self.infer_partial_type(name, lvalue, init_type): + self.msg.need_annotation_for_var(name, context, self.options.python_version) + self.set_inference_error_fallback_type(name, lvalue, init_type) + elif ( + isinstance(lvalue, MemberExpr) + and self.inferred_attribute_types is not None + and lvalue.def_var + and lvalue.def_var in self.inferred_attribute_types + and not is_same_type(self.inferred_attribute_types[lvalue.def_var], init_type) + ): + # Multiple, inconsistent types inferred for an attribute. + self.msg.need_annotation_for_var(name, context, self.options.python_version) + name.type = AnyType(TypeOfAny.from_error) else: # Infer type of the target. @@ -1194,851 +4206,3854 @@ def infer_variable_type(self, name: Var, lvalue: Node, self.set_inferred_type(name, lvalue, init_type) - def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: + def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool: + init_type = get_proper_type(init_type) + if isinstance(init_type, NoneType): + partial_type = PartialType(None, name) + elif isinstance(init_type, Instance): + fullname = init_type.type.fullname + is_ref = isinstance(lvalue, RefExpr) + if ( + is_ref + and ( + fullname == "builtins.list" + or fullname == "builtins.set" + or fullname == "builtins.dict" + or fullname == "collections.OrderedDict" + ) + and all( + isinstance(t, (NoneType, UninhabitedType)) + for t in get_proper_types(init_type.args) + ) + ): + partial_type = PartialType(init_type.type, name) + elif is_ref and fullname == "collections.defaultdict": + arg0 = get_proper_type(init_type.args[0]) + arg1 = get_proper_type(init_type.args[1]) + if isinstance( + arg0, (NoneType, UninhabitedType) + ) and self.is_valid_defaultdict_partial_value_type(arg1): + arg1 = erase_type(arg1) + assert isinstance(arg1, Instance) + partial_type = PartialType(init_type.type, name, arg1) + else: + return False + else: + return False + else: + return False + self.set_inferred_type(name, lvalue, partial_type) + self.partial_types[-1].map[name] = lvalue + return True + + def is_valid_defaultdict_partial_value_type(self, t: ProperType) -> bool: + """Check if t can be used as the basis for a partial defaultdict value type. + + Examples: + + * t is 'int' --> True + * t is 'list[Never]' --> True + * t is 'dict[...]' --> False (only generic types with a single type + argument supported) + """ + if not isinstance(t, Instance): + return False + if len(t.args) == 0: + return True + if len(t.args) == 1: + arg = get_proper_type(t.args[0]) + if self.options.old_type_inference: + # Allow leaked TypeVars for legacy inference logic. + allowed = isinstance(arg, (UninhabitedType, NoneType, TypeVarType)) + else: + allowed = isinstance(arg, (UninhabitedType, NoneType)) + if allowed: + return True + return False + + def set_inferred_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: """Store inferred variable type. Store the type to both the variable node and the expression node that refers to the variable (lvalue). If var is None, do nothing. """ - if var: + if var and not self.current_node_deferred: var.type = type + var.is_inferred = True + if var not in self.var_decl_frames: + # Used for the hack to improve optional type inference in conditionals + self.var_decl_frames[var] = {frame.id for frame in self.binder.frames} + if isinstance(lvalue, MemberExpr) and self.inferred_attribute_types is not None: + # Store inferred attribute type so that we can check consistency afterwards. + if lvalue.def_var is not None: + self.inferred_attribute_types[lvalue.def_var] = type self.store_type(lvalue, type) - def is_valid_inferred_type(self, typ: Type) -> bool: - """Is an inferred type invalid? + def set_inference_error_fallback_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: + """Store best known type for variable if type inference failed. + + If a program ignores error on type inference error, the variable should get some + inferred type so that if can used later on in the program. Example: - Examples include the None type or a type with a None component. + x = [] # type: ignore + x.append(1) # Should be ok! + + We implement this here by giving x a valid type (replacing inferred Never with Any). """ - if is_same_type(typ, NoneTyp()): - return False - elif isinstance(typ, Instance): - for arg in typ.args: - if not self.is_valid_inferred_type(arg): - return False - elif isinstance(typ, TupleType): - for item in typ.items: - if not self.is_valid_inferred_type(item): - return False - return True + fallback = self.inference_error_fallback_type(type) + self.set_inferred_type(var, lvalue, fallback) - def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: - if expr.literal >= LITERAL_TYPE: - restriction = self.binder.get(expr) - if restriction: - ans = meet_simple(known_type, restriction) - return ans - return known_type - - def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, - context: Node, - msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT) -> Type: - """Checks the assignment of rvalue to a lvalue of type lvalue_type.""" - if refers_to_fullname(rvalue, 'typing.Undefined'): - # The rvalue is just 'Undefined'; this is always valid. - # Infer the type of 'Undefined' from the lvalue type. - self.store_type(rvalue, lvalue_type) - return None + def inference_error_fallback_type(self, type: Type) -> Type: + fallback = type.accept(SetNothingToAny()) + # Type variables may leak from inference, see https://github.com/python/mypy/issues/5738, + # we therefore need to erase them. + return erase_typevars(fallback) + + def simple_rvalue(self, rvalue: Expression) -> bool: + """Returns True for expressions for which inferred type should not depend on context. + + Note that this function can still return False for some expressions where inferred type + does not depend on context. It only exists for performance optimizations. + """ + if isinstance(rvalue, (IntExpr, StrExpr, BytesExpr, FloatExpr, RefExpr)): + return True + if isinstance(rvalue, CallExpr): + if isinstance(rvalue.callee, RefExpr) and isinstance(rvalue.callee.node, FuncBase): + typ = rvalue.callee.node.type + if isinstance(typ, CallableType): + return not typ.variables + elif isinstance(typ, Overloaded): + return not any(item.variables for item in typ.items) + return False + + def check_simple_assignment( + self, + lvalue_type: Type | None, + rvalue: Expression, + context: Context, + msg: ErrorMessage = message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + lvalue_name: str = "variable", + rvalue_name: str = "expression", + *, + notes: list[str] | None = None, + ) -> Type: + if self.is_stub and isinstance(rvalue, EllipsisExpr): + # '...' is always a valid initializer in a stub. + return AnyType(TypeOfAny.special_form) else: - rvalue_type = self.accept(rvalue, lvalue_type) - self.check_subtype(rvalue_type, lvalue_type, context, msg, - 'expression has type', 'variable has type') + always_allow_any = lvalue_type is not None and not isinstance( + get_proper_type(lvalue_type), AnyType + ) + rvalue_type = self.expr_checker.accept( + rvalue, lvalue_type, always_allow_any=always_allow_any + ) + if ( + isinstance(get_proper_type(lvalue_type), UnionType) + # Skip literal types, as they have special logic (for better errors). + and not isinstance(get_proper_type(rvalue_type), LiteralType) + and not self.simple_rvalue(rvalue) + ): + # Try re-inferring r.h.s. in empty context, and use that if it + # results in a narrower type. We don't do this always because this + # may cause some perf impact, plus we want to partially preserve + # the old behavior. This helps with various practical examples, see + # e.g. testOptionalTypeNarrowedByGenericCall. + with self.msg.filter_errors() as local_errors, self.local_type_map() as type_map: + alt_rvalue_type = self.expr_checker.accept( + rvalue, None, always_allow_any=always_allow_any + ) + if ( + not local_errors.has_new_errors() + # Skip Any type, since it is special cased in binder. + and not isinstance(get_proper_type(alt_rvalue_type), AnyType) + and is_valid_inferred_type(alt_rvalue_type) + and is_proper_subtype(alt_rvalue_type, rvalue_type) + ): + rvalue_type = alt_rvalue_type + self.store_types(type_map) + if isinstance(rvalue_type, DeletedType): + self.msg.deleted_as_rvalue(rvalue_type, context) + if isinstance(lvalue_type, DeletedType): + self.msg.deleted_as_lvalue(lvalue_type, context) + elif lvalue_type: + self.check_subtype( + # Preserve original aliases for error messages when possible. + rvalue_type, + lvalue_type, + context, + msg, + f"{rvalue_name} has type", + f"{lvalue_name} has type", + notes=notes, + ) return rvalue_type - def check_indexed_assignment(self, lvalue: IndexExpr, - rvalue: Node, context: Context) -> None: + def check_member_assignment( + self, instance_type: Type, attribute_type: Type, rvalue: Expression, context: Context + ) -> tuple[Type, Type, bool]: + """Type member assignment. + + This defers to check_simple_assignment, unless the member expression + is a descriptor, in which case this checks descriptor semantics as well. + + Return the inferred rvalue_type, inferred lvalue_type, and whether to use the binder + for this assignment. + + Note: this method exists here and not in checkmember.py, because we need to take + care about interaction between binder and __set__(). + """ + instance_type = get_proper_type(instance_type) + attribute_type = get_proper_type(attribute_type) + # Descriptors don't participate in class-attribute access + if (isinstance(instance_type, FunctionLike) and instance_type.is_type_obj()) or isinstance( + instance_type, TypeType + ): + rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context) + return rvalue_type, attribute_type, True + + if not isinstance(attribute_type, Instance): + # TODO: support __set__() for union types. + rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context) + return rvalue_type, attribute_type, True + + mx = MemberContext( + is_lvalue=False, + is_super=False, + is_operator=False, + original_type=instance_type, + context=context, + self_type=None, + msg=self.msg, + chk=self, + ) + get_type = analyze_descriptor_access(attribute_type, mx, assignment=True) + if not attribute_type.type.has_readable_member("__set__"): + # If there is no __set__, we type-check that the assigned value matches + # the return type of __get__. This doesn't match the python semantics, + # (which allow you to override the descriptor with any value), but preserves + # the type of accessing the attribute (even after the override). + rvalue_type = self.check_simple_assignment(get_type, rvalue, context) + return rvalue_type, get_type, True + + dunder_set = attribute_type.type.get_method("__set__") + if dunder_set is None: + self.fail( + message_registry.DESCRIPTOR_SET_NOT_CALLABLE.format( + attribute_type.str_with_options(self.options) + ), + context, + ) + return AnyType(TypeOfAny.from_error), get_type, False + + bound_method = analyze_decorator_or_funcbase_access( + defn=dunder_set, + itype=attribute_type, + info=attribute_type.type, + self_type=attribute_type, + name="__set__", + mx=mx, + ) + typ = map_instance_to_supertype(attribute_type, dunder_set.info) + dunder_set_type = expand_type_by_instance(bound_method, typ) + + callable_name = self.expr_checker.method_fullname(attribute_type, "__set__") + dunder_set_type = self.expr_checker.transform_callee_type( + callable_name, + dunder_set_type, + [TempNode(instance_type, context=context), rvalue], + [nodes.ARG_POS, nodes.ARG_POS], + context, + object_type=attribute_type, + ) + + # For non-overloaded setters, the result should be type-checked like a regular assignment. + # Hence, we first only try to infer the type by using the rvalue as type context. + type_context = rvalue + with self.msg.filter_errors(): + _, inferred_dunder_set_type = self.expr_checker.check_call( + dunder_set_type, + [TempNode(instance_type, context=context), type_context], + [nodes.ARG_POS, nodes.ARG_POS], + context, + object_type=attribute_type, + callable_name=callable_name, + ) + + # And now we in fact type check the call, to show errors related to wrong arguments + # count, etc., replacing the type context for non-overloaded setters only. + inferred_dunder_set_type = get_proper_type(inferred_dunder_set_type) + if isinstance(inferred_dunder_set_type, CallableType): + type_context = TempNode(AnyType(TypeOfAny.special_form), context=context) + self.expr_checker.check_call( + dunder_set_type, + [TempNode(instance_type, context=context), type_context], + [nodes.ARG_POS, nodes.ARG_POS], + context, + object_type=attribute_type, + callable_name=callable_name, + ) + + # Search for possible deprecations: + mx.chk.check_deprecated(dunder_set, mx.context) + mx.chk.warn_deprecated_overload_item( + dunder_set, mx.context, target=inferred_dunder_set_type, selftype=attribute_type + ) + + # In the following cases, a message already will have been recorded in check_call. + if (not isinstance(inferred_dunder_set_type, CallableType)) or ( + len(inferred_dunder_set_type.arg_types) < 2 + ): + return AnyType(TypeOfAny.from_error), get_type, False + + set_type = inferred_dunder_set_type.arg_types[1] + # Special case: if the rvalue_type is a subtype of both '__get__' and '__set__' types, + # and '__get__' type is narrower than '__set__', then we invoke the binder to narrow type + # by this assignment. Technically, this is not safe, but in practice this is + # what a user expects. + rvalue_type = self.check_simple_assignment(set_type, rvalue, context) + infer = is_subtype(rvalue_type, get_type) and is_subtype(get_type, set_type) + return rvalue_type if infer else set_type, get_type, infer + + def check_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Expression, context: Context + ) -> None: """Type check indexed assignment base[index] = rvalue. The lvalue argument is the base[index] expression. """ - basetype = self.accept(lvalue.base) - method_type = self.expr_checker.analyse_external_member_access( - '__setitem__', basetype, context) + self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) + basetype = get_proper_type(self.expr_checker.accept(lvalue.base)) + method_type = self.expr_checker.analyze_external_member_access( + "__setitem__", basetype, lvalue + ) + lvalue.method_type = method_type - self.expr_checker.check_call(method_type, [lvalue.index, rvalue], - [nodes.ARG_POS, nodes.ARG_POS], - context) + res_type, _ = self.expr_checker.check_method_call( + "__setitem__", + basetype, + method_type, + [lvalue.index, rvalue], + [nodes.ARG_POS, nodes.ARG_POS], + context, + ) + res_type = get_proper_type(res_type) + if isinstance(res_type, UninhabitedType) and not res_type.ambiguous: + self.binder.unreachable() + + def try_infer_partial_type_from_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Expression + ) -> None: + # TODO: Should we share some of this with try_infer_partial_type? + var = None + if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): + var = lvalue.base.node + elif isinstance(lvalue.base, MemberExpr): + var = self.expr_checker.get_partial_self_var(lvalue.base) + if isinstance(var, Var): + if isinstance(var.type, PartialType): + type_type = var.type.type + if type_type is None: + return # The partial type is None. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + typename = type_type.fullname + if ( + typename == "builtins.dict" + or typename == "collections.OrderedDict" + or typename == "collections.defaultdict" + ): + # TODO: Don't infer things twice. + key_type = self.expr_checker.accept(lvalue.index) + value_type = self.expr_checker.accept(rvalue) + if ( + is_valid_inferred_type(key_type) + and is_valid_inferred_type(value_type) + and not self.current_node_deferred + and not ( + typename == "collections.defaultdict" + and var.type.value_type is not None + and not is_equivalent(value_type, var.type.value_type) + ) + ): + var.type = self.named_generic_type(typename, [key_type, value_type]) + del partial_types[var] + + def type_requires_usage(self, typ: Type) -> tuple[str, ErrorCode] | None: + """Some types require usage in all cases. The classic example is + an unused coroutine. - def visit_expression_stmt(self, s: ExpressionStmt) -> Type: - self.accept(s.expr) + In the case that it does require usage, returns a note to attach + to the error message. + """ + proper_type = get_proper_type(typ) + if isinstance(proper_type, Instance): + # We use different error codes for generic awaitable vs coroutine. + # Coroutines are on by default, whereas generic awaitables are not. + if proper_type.type.fullname == "typing.Coroutine": + return ("Are you missing an await?", UNUSED_COROUTINE) + if proper_type.type.get("__await__") is not None: + return ("Are you missing an await?", UNUSED_AWAITABLE) + return None - def visit_return_stmt(self, s: ReturnStmt) -> Type: + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + expr_type = self.expr_checker.accept(s.expr, allow_none_return=True, always_allow_any=True) + error_note_and_code = self.type_requires_usage(expr_type) + if error_note_and_code: + error_note, code = error_note_and_code + self.fail( + message_registry.TYPE_MUST_BE_USED.format(format_type(expr_type, self.options)), + s, + code=code, + ) + self.note(error_note, s, code=code) + + def visit_return_stmt(self, s: ReturnStmt) -> None: """Type check a return statement.""" - self.breaking_out = True - if self.is_within_function(): + self.check_return_stmt(s) + self.binder.unreachable() + + def check_return_stmt(self, s: ReturnStmt) -> None: + defn = self.scope.top_function() + if defn is not None: + if defn.is_generator: + return_type = self.get_generator_return_type( + self.return_types[-1], defn.is_coroutine + ) + elif defn.is_coroutine: + return_type = self.get_coroutine_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + return_type = get_proper_type(return_type) + + is_lambda = isinstance(self.scope.top_function(), LambdaExpr) + if isinstance(return_type, UninhabitedType): + # Avoid extra error messages for failed inference in lambdas + if not is_lambda and not return_type.ambiguous: + self.fail(message_registry.NO_RETURN_EXPECTED, s) + return + if s.expr: + declared_none_return = isinstance(return_type, NoneType) + declared_any_return = isinstance(return_type, AnyType) + + # This controls whether or not we allow a function call that + # returns None as the expression of this return statement. + # E.g. `return f()` for some `f` that returns None. We allow + # this only if we're in a lambda or in a function that returns + # `None` or `Any`. + allow_none_func_call = is_lambda or declared_none_return or declared_any_return + # Return with a value. - typ = self.accept(s.expr, self.return_types[-1]) + typ = get_proper_type( + self.expr_checker.accept( + s.expr, return_type, allow_none_return=allow_none_func_call + ) + ) + # Treat NotImplemented as having type Any, consistent with its + # definition in typeshed prior to python/typeshed#4222. + if ( + isinstance(typ, Instance) + and typ.type.fullname == "builtins._NotImplementedType" + ): + typ = AnyType(TypeOfAny.special_form) + + if defn.is_async_generator: + self.fail(message_registry.RETURN_IN_ASYNC_GENERATOR, s) + return # Returning a value of type Any is always fine. - if not isinstance(typ, AnyType): - if isinstance(self.return_types[-1], Void): - # FuncExpr (lambda) may have a Void return. - # Function returning a value of type None may have a Void return. - if (not isinstance(self.function_stack[-1], FuncExpr) and - not isinstance(typ, NoneTyp)): - self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) - else: - if self.function_stack[-1].is_coroutine: # Something similar will be needed to mix return and yield - # If the function is a coroutine, wrap the return type in a Future - typ = self.wrap_generic_type(cast(Instance,typ), cast(Instance,self.return_types[-1]), 'asyncio.futures.Future', s) - self.check_subtype( - typ, self.return_types[-1], s, - messages.INCOMPATIBLE_RETURN_VALUE_TYPE - + ": expected {}, got {}".format(self.return_types[-1], typ) + if isinstance(typ, AnyType): + # (Unless you asked to be warned in that case, and the + # function is not declared to return Any) + if ( + self.options.warn_return_any + and not self.current_node_deferred + and not is_proper_subtype(AnyType(TypeOfAny.special_form), return_type) + and not ( + defn.name in BINARY_MAGIC_METHODS + and is_literal_not_implemented(s.expr) ) + and not ( + isinstance(return_type, Instance) + and return_type.type.fullname == "builtins.object" + ) + and not is_lambda + ): + self.msg.incorrectly_returning_any(return_type, s) + return + + # Disallow return expressions in functions declared to return + # None, subject to two exceptions below. + if declared_none_return: + # Lambdas are allowed to have None returns. + # Functions returning a value of type None are allowed to have a None return. + if is_lambda or isinstance(typ, NoneType): + return + self.fail(message_registry.NO_RETURN_VALUE_EXPECTED, s) + else: + self.check_subtype( + subtype_label="got", + subtype=typ, + supertype_label="expected", + supertype=return_type, + context=s.expr, + outer_context=s, + msg=message_registry.INCOMPATIBLE_RETURN_VALUE_TYPE, + ) else: - # Return without a value. It's valid in a generator and coroutine function. - if not self.function_stack[-1].is_generator and not self.function_stack[-1].is_coroutine: - if (not isinstance(self.return_types[-1], Void) and - not self.is_dynamic_function()): - self.fail(messages.RETURN_VALUE_EXPECTED, s) - - def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: str, context: Context) -> Type: - n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, check_type) - if n_diff == 1: - return self.named_generic_type(check_type, [typ]) - elif n_diff == 0 or n_diff > 1: - self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE - + ": expected {}, got {}".format(rtyp, typ), context) - return typ - return typ - - def count_nested_types(self, typ: Instance, check_type: str) -> int: - c = 0 - while is_subtype(typ, self.named_type(check_type)): - c += 1 - typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), self.lookup_typeinfo(check_type)) - if typ.args: - typ = cast(Instance,typ.args[0]) - else: - return c - return c - - def visit_yield_stmt(self, s: YieldStmt) -> Type: - return_type = self.return_types[-1] - if isinstance(return_type, Instance): - if return_type.type.fullname() != 'typing.Iterator': - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD, s) - return None - expected_item_type = return_type.args[0] - elif isinstance(return_type, AnyType): - expected_item_type = AnyType() - else: - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD, s) - return None - if s.expr is None: - actual_item_type = Void() # type: Type - else: - actual_item_type = self.accept(s.expr, expected_item_type) - self.check_subtype(actual_item_type, expected_item_type, s, - messages.INCOMPATIBLE_TYPES_IN_YIELD, - 'actual type', 'expected type') - - def visit_yield_from_stmt(self, s: YieldFromStmt) -> Type: - return_type = self.return_types[-1] - type_func = self.accept(s.expr, return_type) - if isinstance(type_func, Instance): - if type_func.type.fullname() == 'asyncio.futures.Future': - # if is a Future, in stmt don't need to do nothing - # because the type Future[Some] jus matters to the main loop - # that python executes, in statement we shouldn't get the Future, - # is just for async purposes. - self.function_stack[-1].is_coroutine = True # Set the function as coroutine - elif is_subtype(type_func, self.named_type('typing.Iterable')): - # If it's and Iterable-Like, let's check the types. - # Maybe just check if have __iter__? (like in analyse_iterable) - self.check_iterable_yield_from(s) - else: - self.msg.yield_from_invalid_operand_type(type_func, s) - elif isinstance(type_func, AnyType): - self.check_iterable_yield_from(s) - else: - self.msg.yield_from_invalid_operand_type(type_func, s) - - def check_iterable_yield_from(self, s: YieldFromStmt) -> Type: - """ - Check that return type is super type of Iterable (Maybe just check if have __iter__?) - and compare it with the type of the expression - """ - expected_item_type = self.return_types[-1] - if isinstance(expected_item_type, Instance): - if not is_subtype(expected_item_type, self.named_type('typing.Iterable')): - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD_FROM, s) - return None - elif expected_item_type.args: - expected_item_type = map_instance_to_supertype(expected_item_type, self.lookup_typeinfo('typing.Iterable')) - expected_item_type = expected_item_type.args[0] # Take the item inside the iterator - elif isinstance(expected_item_type, AnyType): - expected_item_type = AnyType() - else: - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD_FROM, s) - return None - if s.expr is None: - actual_item_type = Void() # type: Type - else: - actual_item_type = self.accept(s.expr, expected_item_type) - if hasattr(actual_item_type, 'args') and cast(Instance,actual_item_type).args: - actual_item_type = map_instance_to_supertype(cast(Instance,actual_item_type), self.lookup_typeinfo('typing.Iterable')) - actual_item_type = actual_item_type.args[0] # Take the item inside the iterator - self.check_subtype(actual_item_type, expected_item_type, s, - messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, - 'actual type', 'expected type') - - def visit_if_stmt(self, s: IfStmt) -> Type: + # Empty returns are valid in Generators with Any typed returns, but not in + # coroutines. + if ( + defn.is_generator + and not defn.is_coroutine + and isinstance(return_type, AnyType) + ): + return + + if isinstance(return_type, (NoneType, AnyType)): + return + + if self.in_checked_function(): + self.fail(message_registry.RETURN_VALUE_EXPECTED, s) + + def visit_if_stmt(self, s: IfStmt) -> None: """Type check an if statement.""" - broken = True - ending_frames = List[Frame]() - clauses_frame = self.binder.push_frame() - for e, b in zip(s.expr, s.body): - t = self.accept(e) - self.check_not_void(t, e) - var, type, elsetype, kind = find_isinstance_check(e, self.type_map) - if kind == ISINSTANCE_ALWAYS_FALSE: - # XXX should issue a warning? - pass - else: - # Only type check body if the if condition can be true. - self.binder.push_frame() - if var: - self.binder.push(var, type) - self.accept(b) - _, frame = self.binder.pop_frame() - self.binder.allow_jump(len(self.binder.frames) - 1) - if not self.breaking_out: - broken = False - ending_frames.append(meet_frames(clauses_frame, frame)) - - self.breaking_out = False - - if var: - self.binder.push(var, elsetype) - if kind == ISINSTANCE_ALWAYS_TRUE: - # The condition is always true => remaining elif/else blocks - # can never be reached. - - # Might also want to issue a warning - # print("Warning: isinstance always true") - if broken: - self.binder.pop_frame() - self.breaking_out = True - return None - break - else: - if s.else_body: - self.accept(s.else_body) + # This frame records the knowledge from previous if/elif clauses not being taken. + # Fall-through to the original frame is handled explicitly in each block. + with self.binder.frame_context(can_skip=False, conditional_frame=True, fall_through=0): + for e, b in zip(s.expr, s.body): + t = get_proper_type(self.expr_checker.accept(e)) - if self.breaking_out and broken: - self.binder.pop_frame() - return None + if isinstance(t, DeletedType): + self.msg.deleted_as_rvalue(t, s) - if not self.breaking_out: - ending_frames.append(clauses_frame) + if_map, else_map = self.find_isinstance_check(e) - self.breaking_out = False - else: - ending_frames.append(clauses_frame) + # XXX Issue a warning if condition is always False? + with self.binder.frame_context(can_skip=True, fall_through=2): + self.push_type_map(if_map, from_assignment=False) + self.accept(b) + + # XXX Issue a warning if condition is always True? + self.push_type_map(else_map, from_assignment=False) - self.binder.pop_frame() - self.binder.update_from_options(ending_frames) + with self.binder.frame_context(can_skip=False, fall_through=2): + if s.else_body: + self.accept(s.else_body) - def visit_while_stmt(self, s: WhileStmt) -> Type: + def visit_while_stmt(self, s: WhileStmt) -> None: """Type check a while statement.""" - self.binder.push_frame() - self.binder.push_loop_frame() - self.accept_in_frame(IfStmt([s.expr], [s.body], None), - repeat_till_fixed=True) - self.binder.pop_loop_frame() - if s.else_body: - self.accept(s.else_body) - self.binder.pop_frame(False, True) + if_stmt = IfStmt([s.expr], [s.body], None) + if_stmt.set_line(s) + self.accept_loop(if_stmt, s.else_body, exit_condition=s.expr) - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> Type: + def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> None: """Type check an operator assignment statement, e.g. x += 1.""" - lvalue_type = self.accept(s.lvalue) - method = infer_operator_assignment_method(lvalue_type, s.op) - rvalue_type, method_type = self.expr_checker.check_op( - method, lvalue_type, s.rvalue, s) - - if isinstance(s.lvalue, IndexExpr): - lv = cast(IndexExpr, s.lvalue) - self.check_indexed_assignment(lv, s.rvalue, s.rvalue) + self.try_infer_partial_generic_type_from_assignment(s.lvalue, s.rvalue, s.op) + if isinstance(s.lvalue, MemberExpr): + # Special case, some additional errors may be given for + # assignments to read-only or final attributes. + lvalue_type = self.expr_checker.visit_member_expr(s.lvalue, True) else: + lvalue_type = self.expr_checker.accept(s.lvalue) + inplace, method = infer_operator_assignment_method(lvalue_type, s.op) + if inplace: + # There is __ifoo__, treat as x = x.__ifoo__(y) + rvalue_type, method_type = self.expr_checker.check_op(method, lvalue_type, s.rvalue, s) + if isinstance(inst := get_proper_type(lvalue_type), Instance) and isinstance( + defn := inst.type.get_method(method), OverloadedFuncDef + ): + for item in defn.items: + if ( + isinstance(item, Decorator) + and isinstance(typ := item.func.type, CallableType) + and (bind_self(typ) == method_type) + ): + self.warn_deprecated(item.func, s) if not is_subtype(rvalue_type, lvalue_type): self.msg.incompatible_operator_assignment(s.op, s) - - def visit_assert_stmt(self, s: AssertStmt) -> Type: - self.accept(s.expr) - - def visit_raise_stmt(self, s: RaiseStmt) -> Type: + else: + # There is no __ifoo__, treat as x = x y + expr = OpExpr(s.op, s.lvalue, s.rvalue) + expr.set_line(s) + self.check_assignment( + lvalue=s.lvalue, rvalue=expr, infer_lvalue_type=True, new_syntax=False + ) + self.check_final(s) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + self.expr_checker.accept(s.expr) + + if isinstance(s.expr, TupleExpr) and len(s.expr.items) > 0: + self.fail(message_registry.MALFORMED_ASSERT, s) + + # If this is asserting some isinstance check, bind that type in the following code + true_map, else_map = self.find_isinstance_check(s.expr) + if s.msg is not None: + self.expr_checker.analyze_cond_branch(else_map, s.msg, None) + self.push_type_map(true_map) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: """Type check a raise statement.""" - self.breaking_out = True if s.expr: self.type_check_raise(s.expr, s) if s.from_expr: - self.type_check_raise(s.from_expr, s) + self.type_check_raise(s.from_expr, s, optional=True) + self.binder.unreachable() - def type_check_raise(self, e: Node, s: RaiseStmt) -> None: - typ = self.accept(e) - if isinstance(typ, FunctionLike): - if typ.is_type_obj(): - # Cases like "raise/from ExceptionClass". - typeinfo = typ.type_object() - base = self.lookup_typeinfo('builtins.BaseException') - if base in typeinfo.mro: - # Good! - return None - # Else fall back to the check below (which will fail). - self.check_subtype(typ, - self.named_type('builtins.BaseException'), s, - messages.INVALID_EXCEPTION) + def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) -> None: + typ = get_proper_type(self.expr_checker.accept(e)) + if isinstance(typ, DeletedType): + self.msg.deleted_as_rvalue(typ, e) + return - def visit_try_stmt(self, s: TryStmt) -> Type: - """Type check a try statement.""" - completed_frames = List[Frame]() - self.binder.push_frame() - self.binder.try_frames.add(len(self.binder.frames) - 2) - self.accept(s.body) - self.binder.try_frames.remove(len(self.binder.frames) - 2) - if s.else_body: - self.accept(s.else_body) - changed, frame_on_completion = self.binder.pop_frame() - completed_frames.append(frame_on_completion) - - for i in range(len(s.handlers)): - if s.types[i]: - t = self.exception_type(s.types[i]) - if s.vars[i]: - self.check_assignment(s.vars[i], - self.temp_node(t, s.vars[i])) - self.binder.push_frame() - self.accept(s.handlers[i]) - changed, frame_on_completion = self.binder.pop_frame() - completed_frames.append(frame_on_completion) - if s.else_body: - self.binder.push_frame() - self.accept(s.else_body) - changed, frame_on_completion = self.binder.pop_frame() - completed_frames.append(frame_on_completion) + exc_type = self.named_type("builtins.BaseException") + expected_type_items = [exc_type, TypeType(exc_type)] + if optional: + # This is used for `x` part in a case like `raise e from x`, + # where we allow `raise e from None`. + expected_type_items.append(NoneType()) - self.binder.update_from_options(completed_frames) + self.check_subtype( + typ, UnionType.make_union(expected_type_items), s, message_registry.INVALID_EXCEPTION + ) + + if isinstance(typ, FunctionLike): + # https://github.com/python/mypy/issues/11089 + self.expr_checker.check_call(typ, [], [], e) + + if isinstance(typ, Instance) and typ.type.fullname == "builtins._NotImplementedType": + self.fail( + message_registry.INVALID_EXCEPTION.with_additional_msg( + '; did you mean "NotImplementedError"?' + ), + s, + ) + + def visit_try_stmt(self, s: TryStmt) -> None: + """Type check a try statement.""" + # Our enclosing frame will get the result if the try/except falls through. + # This one gets all possible states after the try block exited abnormally + # (by exception, return, break, etc.) + with self.binder.frame_context(can_skip=False, fall_through=0): + # Not only might the body of the try statement exit + # abnormally, but so might an exception handler or else + # clause. The finally clause runs in *all* cases, so we + # need an outer try frame to catch all intermediate states + # in case an exception is raised during an except or else + # clause. As an optimization, only create the outer try + # frame when there actually is a finally clause. + self.visit_try_without_finally(s, try_frame=bool(s.finally_body)) + if s.finally_body: + # First we check finally_body is type safe on all abnormal exit paths + self.accept(s.finally_body) if s.finally_body: - self.accept(s.finally_body) - - def exception_type(self, n: Node) -> Type: - if isinstance(n, ParenExpr): - # Multiple exception types (...). - unwrapped = self.expr_checker.strip_parens(n) - if isinstance(unwrapped, TupleExpr): - t = None # type: Type - for item in unwrapped.items: - tt = self.exception_type(item) - if t: - t = join_types(t, tt) - else: - t = tt - return t - else: - # A single exception type; should evaluate to a type object type. - type = self.accept(n) - return self.check_exception_type(type, n) - self.fail('Unsupported exception', n) - return AnyType() + # Then we try again for the more restricted set of options + # that can fall through. (Why do we need to check the + # finally clause twice? Depending on whether the finally + # clause was reached by the try clause falling off the end + # or exiting abnormally, after completing the finally clause + # either flow will continue to after the entire try statement + # or the exception/return/etc. will be processed and control + # flow will escape. We need to check that the finally clause + # type checks in both contexts, but only the resulting types + # from the latter context affect the type state in the code + # that follows the try statement.) + if not self.binder.is_unreachable(): + self.accept(s.finally_body) + + def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: + """Type check a try statement, ignoring the finally block. + + On entry, the top frame should receive all flow that exits the + try block abnormally (i.e., such that the else block does not + execute), and its parent should receive all flow that exits + the try block normally. + """ + # This frame will run the else block if the try fell through. + # In that case, control flow continues to the parent of what + # was the top frame on entry. + with self.binder.frame_context(can_skip=False, fall_through=2, try_frame=try_frame): + # This frame receives exit via exception, and runs exception handlers + with self.binder.frame_context(can_skip=False, conditional_frame=True, fall_through=2): + # Finally, the body of the try statement + with self.binder.frame_context(can_skip=False, fall_through=2, try_frame=True): + self.accept(s.body) + for i in range(len(s.handlers)): + with self.binder.frame_context(can_skip=True, fall_through=4): + typ = s.types[i] + if typ: + t = self.check_except_handler_test(typ, s.is_star) + var = s.vars[i] + if var: + # To support local variables, we make this a definition line, + # causing assignment to set the variable's type. + var.is_inferred_def = True + self.check_assignment(var, self.temp_node(t, var)) + self.accept(s.handlers[i]) + var = s.vars[i] + if var: + # Exception variables are deleted. + # Unfortunately, this doesn't let us detect usage before the + # try/except block. + source = var.name + if isinstance(var.node, Var): + var.node.type = DeletedType(source=source) + self.binder.cleanse(var) + if s.else_body: + self.accept(s.else_body) - @overload - def check_exception_type(self, type: FunctionLike, - context: Context) -> Type: - item = type.items()[0] - ret = item.ret_type - if (is_subtype(ret, self.named_type('builtins.BaseException')) - and item.is_type_obj()): - return ret - else: - self.fail(messages.INVALID_EXCEPTION_TYPE, context) - return AnyType() + def check_except_handler_test(self, n: Expression, is_star: bool) -> Type: + """Type check an exception handler test clause.""" + typ = self.expr_checker.accept(n) - @overload - def check_exception_type(self, type: AnyType, context: Context) -> Type: - return AnyType() + all_types: list[Type] = [] + test_types = self.get_types_from_except_handler(typ, n) - @overload - def check_exception_type(self, type: Type, context: Context) -> Type: - self.fail(messages.INVALID_EXCEPTION_TYPE, context) - return AnyType() + for ttype in get_proper_types(test_types): + if isinstance(ttype, AnyType): + all_types.append(ttype) + continue + + if isinstance(ttype, FunctionLike): + item = ttype.items[0] + if not item.is_type_obj(): + self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) + return self.default_exception_type(is_star) + exc_type = erase_typevars(item.ret_type) + elif isinstance(ttype, TypeType): + exc_type = ttype.item + else: + self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) + return self.default_exception_type(is_star) + + if not is_subtype(exc_type, self.named_type("builtins.BaseException")): + self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) + return self.default_exception_type(is_star) + + all_types.append(exc_type) + + if is_star: + new_all_types: list[Type] = [] + for typ in all_types: + if is_proper_subtype(typ, self.named_type("builtins.BaseExceptionGroup")): + self.fail(message_registry.INVALID_EXCEPTION_GROUP, n) + new_all_types.append(AnyType(TypeOfAny.from_error)) + else: + new_all_types.append(typ) + return self.wrap_exception_group(new_all_types) + return make_simplified_union(all_types) + + def default_exception_type(self, is_star: bool) -> Type: + """Exception type to return in case of a previous type error.""" + any_type = AnyType(TypeOfAny.from_error) + if is_star: + return self.named_generic_type("builtins.ExceptionGroup", [any_type]) + return any_type + + def wrap_exception_group(self, types: Sequence[Type]) -> Type: + """Transform except* variable type into an appropriate exception group.""" + arg = make_simplified_union(types) + if is_subtype(arg, self.named_type("builtins.Exception")): + base = "builtins.ExceptionGroup" + else: + base = "builtins.BaseExceptionGroup" + return self.named_generic_type(base, [arg]) + + def get_types_from_except_handler(self, typ: Type, n: Expression) -> list[Type]: + """Helper for check_except_handler_test to retrieve handler types.""" + typ = get_proper_type(typ) + if isinstance(typ, TupleType): + return typ.items + elif isinstance(typ, UnionType): + return [ + union_typ + for item in typ.relevant_items() + for union_typ in self.get_types_from_except_handler(item, n) + ] + elif is_named_instance(typ, "builtins.tuple"): + # variadic tuple + return [typ.args[0]] + else: + return [typ] - def visit_for_stmt(self, s: ForStmt) -> Type: + def visit_for_stmt(self, s: ForStmt) -> None: """Type check a for statement.""" - item_type = self.analyse_iterable_item_type(s.expr) - self.analyse_index_variables(s.index, item_type, s) - self.binder.push_frame() - self.binder.push_loop_frame() - self.accept_in_frame(s.body, repeat_till_fixed=True) - self.binder.pop_loop_frame() - if s.else_body: - self.accept(s.else_body) - self.binder.pop_frame(False, True) - - def analyse_iterable_item_type(self, expr: Node) -> Type: - """Analyse iterable expression and return iterator item type.""" - iterable = self.accept(expr) - - self.check_not_void(iterable, expr) - if isinstance(iterable, TupleType): - joined = NoneTyp() # type: Type - for item in iterable.items: - joined = join_types(joined, item) - if isinstance(joined, ErrorType): - self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) - return AnyType() - return joined + if s.is_async: + iterator_type, item_type = self.analyze_async_iterable_item_type(s.expr) + else: + iterator_type, item_type = self.analyze_iterable_item_type(s.expr) + s.inferred_item_type = item_type + s.inferred_iterator_type = iterator_type + self.analyze_index_variables(s.index, item_type, s.index_type is None, s) + self.accept_loop(s.body, s.else_body) + + def analyze_async_iterable_item_type(self, expr: Expression) -> tuple[Type, Type]: + """Analyse async iterable expression and return iterator and iterator item types.""" + echk = self.expr_checker + iterable = echk.accept(expr) + iterator = echk.check_method_call_by_name("__aiter__", iterable, [], [], expr)[0] + awaitable = echk.check_method_call_by_name("__anext__", iterator, [], [], expr)[0] + item_type = echk.check_awaitable_expr( + awaitable, expr, message_registry.INCOMPATIBLE_TYPES_IN_ASYNC_FOR + ) + return iterator, item_type + + def analyze_iterable_item_type(self, expr: Expression) -> tuple[Type, Type]: + """Analyse iterable expression and return iterator and iterator item types.""" + iterator, iterable = self.analyze_iterable_item_type_without_expression( + self.expr_checker.accept(expr), context=expr + ) + int_type = self.analyze_range_native_int_type(expr) + if int_type: + return iterator, int_type + return iterator, iterable + + def analyze_iterable_item_type_without_expression( + self, type: Type, context: Context + ) -> tuple[Type, Type]: + """Analyse iterable type and return iterator and iterator item types.""" + echk = self.expr_checker + iterable: Type + iterable = get_proper_type(type) + iterator = echk.check_method_call_by_name("__iter__", iterable, [], [], context)[0] + + if ( + isinstance(iterable, TupleType) + and iterable.partial_fallback.type.fullname == "builtins.tuple" + ): + return iterator, tuple_fallback(iterable).args[0] else: # Non-tuple iterable. - self.check_subtype(iterable, - self.named_generic_type('typing.Iterable', - [AnyType()]), - expr, messages.ITERABLE_EXPECTED) - - echk = self.expr_checker - method = echk.analyse_external_member_access('__iter__', iterable, - expr) - iterator = echk.check_call(method, [], [], expr)[0] - if self.pyversion >= 3: - nextmethod = '__next__' - else: - nextmethod = 'next' - method = echk.analyse_external_member_access(nextmethod, iterator, - expr) - return echk.check_call(method, [], [], expr)[0] + iterable = echk.check_method_call_by_name("__next__", iterator, [], [], context)[0] + return iterator, iterable + + def analyze_range_native_int_type(self, expr: Expression) -> Type | None: + """Try to infer native int item type from arguments to range(...). + + For example, return i64 if the expression is "range(0, i64(n))". + + Return None if unsuccessful. + """ + if ( + isinstance(expr, CallExpr) + and isinstance(expr.callee, RefExpr) + and expr.callee.fullname == "builtins.range" + and 1 <= len(expr.args) <= 3 + and all(kind == ARG_POS for kind in expr.arg_kinds) + ): + native_int: Type | None = None + ok = True + for arg in expr.args: + argt = get_proper_type(self.lookup_type(arg)) + if isinstance(argt, Instance) and argt.type.fullname in MYPYC_NATIVE_INT_NAMES: + if native_int is None: + native_int = argt + elif argt != native_int: + ok = False + if ok and native_int: + return native_int + return None + + def analyze_container_item_type(self, typ: Type) -> Type | None: + """Check if a type is a nominal container of a union of such. + + Return the corresponding container item type. + """ + typ = get_proper_type(typ) + if isinstance(typ, UnionType): + types: list[Type] = [] + for item in typ.items: + c_type = self.analyze_container_item_type(item) + if c_type: + types.append(c_type) + return UnionType.make_union(types) + if isinstance(typ, Instance) and typ.type.has_base("typing.Container"): + supertype = self.named_type("typing.Container").type + super_instance = map_instance_to_supertype(typ, supertype) + assert len(super_instance.args) == 1 + return super_instance.args[0] + if isinstance(typ, TupleType): + return self.analyze_container_item_type(tuple_fallback(typ)) + return None - def analyse_index_variables(self, index: Node, item_type: Type, - context: Context) -> None: + def analyze_index_variables( + self, index: Expression, item_type: Type, infer_lvalue_type: bool, context: Context + ) -> None: """Type check or infer for loop or list comprehension index vars.""" - self.check_assignment(index, self.temp_node(item_type, context)) + self.check_assignment(index, self.temp_node(item_type, context), infer_lvalue_type) - def visit_del_stmt(self, s: DelStmt) -> Type: + def visit_del_stmt(self, s: DelStmt) -> None: if isinstance(s.expr, IndexExpr): - e = cast(IndexExpr, s.expr) # Cast - m = MemberExpr(e.base, '__delitem__') + e = s.expr + m = MemberExpr(e.base, "__delitem__") m.line = s.line + m.column = s.column c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) c.line = s.line - return c.accept(self) + c.column = s.column + self.expr_checker.accept(c, allow_none_return=True) else: - s.expr.accept(self) - return None - - def visit_decorator(self, e: Decorator) -> Type: - e.func.accept(self) - sig = self.function_type(e.func) # type: Type - # Process decorators from the inside out. - for i in range(len(e.decorators)): - n = len(e.decorators) - 1 - i - dec = self.accept(e.decorators[n]) - temp = self.temp_node(sig) - sig, t2 = self.expr_checker.check_call(dec, [temp], - [nodes.ARG_POS], e) - sig = cast(FunctionLike, sig) + s.expr.accept(self.expr_checker) + for elt in flatten(s.expr): + if isinstance(elt, NameExpr): + self.binder.assign_type( + elt, DeletedType(source=elt.name), get_declaration(elt), False + ) + + def visit_decorator(self, e: Decorator) -> None: + for d in e.decorators: + if isinstance(d, RefExpr): + if d.fullname == "typing.no_type_check": + e.var.type = AnyType(TypeOfAny.special_form) + e.var.is_ready = True + return + self.visit_decorator_inner(e) + + def visit_decorator_inner(self, e: Decorator, allow_empty: bool = False) -> None: + if self.recurse_into_functions: + with self.tscope.function_scope(e.func): + self.check_func_item(e.func, name=e.func.name, allow_empty=allow_empty) + + # Process decorators from the inside out to determine decorated signature, which + # may be different from the declared signature. + sig: Type = self.function_type(e.func) + for d in reversed(e.decorators): + if refers_to_fullname(d, OVERLOAD_NAMES): + if not allow_empty: + self.fail(message_registry.MULTIPLE_OVERLOADS_REQUIRED, e) + continue + dec = self.expr_checker.accept(d) + temp = self.temp_node(sig, context=e) + fullname = None + if isinstance(d, RefExpr): + fullname = d.fullname or None + # if this is a expression like @b.a where b is an object, get the type of b + # so we can pass it the method hook in the plugins + object_type: Type | None = None + if fullname is None and isinstance(d, MemberExpr) and self.has_type(d.expr): + object_type = self.lookup_type(d.expr) + fullname = self.expr_checker.method_fullname(object_type, d.name) + self.check_for_untyped_decorator(e.func, dec, d) + sig, t2 = self.expr_checker.check_call( + dec, [temp], [nodes.ARG_POS], e, callable_name=fullname, object_type=object_type + ) + self.check_untyped_after_decorator(sig, e.func) sig = set_callable_name(sig, e.func) e.var.type = sig e.var.is_ready = True + if e.func.is_property: + if isinstance(sig, CallableType): + if len([k for k in sig.arg_kinds if k.is_required()]) > 1: + self.msg.fail("Too many arguments for property", e) + self.check_incompatible_property_override(e) + # For overloaded functions we already checked override for overload as a whole. + if allow_empty: + return + if e.func.info and not e.func.is_dynamic() and not e.is_overload: + found_method_base_classes = self.check_method_override(e) + if ( + e.func.is_explicit_override + and not found_method_base_classes + and found_method_base_classes is not None + ): + self.msg.no_overridable_method(e.func.name, e.func) + self.check_explicit_override_decorator(e.func, found_method_base_classes) + + if e.func.info and e.func.name in ("__init__", "__new__"): + if e.type and not isinstance(get_proper_type(e.type), (FunctionLike, AnyType)): + self.fail(message_registry.BAD_CONSTRUCTOR_TYPE, e) + + if e.func.original_def and isinstance(sig, FunctionLike): + # Function definition overrides function definition. + self.check_func_def_override(e.func, sig) + + def check_for_untyped_decorator( + self, func: FuncDef, dec_type: Type, dec_expr: Expression + ) -> None: + if ( + self.options.disallow_untyped_decorators + and is_typed_callable(func.type) + and is_untyped_decorator(dec_type) + ): + self.msg.typed_function_untyped_decorator(func.name, dec_expr) + + def check_incompatible_property_override(self, e: Decorator) -> None: + if not e.var.is_settable_property and e.func.info: + name = e.func.name + for base in e.func.info.mro[1:]: + base_attr = base.names.get(name) + if not base_attr: + continue + if ( + isinstance(base_attr.node, OverloadedFuncDef) + and base_attr.node.is_property + and cast(Decorator, base_attr.node.items[0]).var.is_settable_property + ): + self.fail(message_registry.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) + + def visit_with_stmt(self, s: WithStmt) -> None: + exceptions_maybe_suppressed = False + for expr, target in zip(s.expr, s.target): + if s.is_async: + exit_ret_type = self.check_async_with_item(expr, target, s.unanalyzed_type is None) + else: + exit_ret_type = self.check_with_item(expr, target, s.unanalyzed_type is None) - def visit_with_stmt(self, s: WithStmt) -> Type: - echk = self.expr_checker - for expr, name in zip(s.expr, s.name): - ctx = self.accept(expr) - enter = echk.analyse_external_member_access('__enter__', ctx, expr) - obj = echk.check_call(enter, [], [], expr)[0] - if name: - self.check_assignment(name, self.temp_node(obj, expr)) - exit = echk.analyse_external_member_access('__exit__', ctx, expr) - arg = self.temp_node(AnyType(), expr) - echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) - self.accept(s.body) - - def visit_print_stmt(self, s: PrintStmt) -> Type: - for arg in s.args: - self.accept(arg) + # Based on the return type, determine if this context manager 'swallows' + # exceptions or not. We determine this using a heuristic based on the + # return type of the __exit__ method -- see the discussion in + # https://github.com/python/mypy/issues/7214 and the section about context managers + # in https://github.com/python/typeshed/blob/main/CONTRIBUTING.md#conventions + # for more details. - # - # Expressions - # + exit_ret_type = get_proper_type(exit_ret_type) + if is_literal_type(exit_ret_type, "builtins.bool", False): + continue - def visit_name_expr(self, e: NameExpr) -> Type: - return self.expr_checker.visit_name_expr(e) - - def visit_paren_expr(self, e: ParenExpr) -> Type: - return self.expr_checker.visit_paren_expr(e) - - def visit_call_expr(self, e: CallExpr) -> Type: - result = self.expr_checker.visit_call_expr(e) - self.breaking_out = False - return result - - def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: - # result = self.expr_checker.visit_yield_from_expr(e) - result = self.accept(e.expr) - result_instance = cast(Instance, result) - if result_instance.type.fullname() == "asyncio.futures.Future": - self.function_stack[-1].is_coroutine = True # Set the function as coroutine - result = result_instance.args[0] # Set the return type as the type inside - elif is_subtype(result, self.named_type('typing.Iterable')): - # TODO - # Check return type Iterator[Some] - # Maybe set result like in the Future - pass + if is_literal_type(exit_ret_type, "builtins.bool", True) or ( + isinstance(exit_ret_type, Instance) + and exit_ret_type.type.fullname == "builtins.bool" + and state.strict_optional + ): + # Note: if strict-optional is disabled, this bool instance + # could actually be an Optional[bool]. + exceptions_maybe_suppressed = True + + if exceptions_maybe_suppressed: + # Treat this 'with' block in the same way we'd treat a 'try: BODY; except: pass' + # block. This means control flow can continue after the 'with' even if the 'with' + # block immediately returns. + with self.binder.frame_context(can_skip=True, try_frame=True): + self.accept(s.body) else: - # self.msg.yield_from_invalid_operand_type(e.expr, e) - self.msg.yield_from_invalid_operand_type(e.expr.accept(self), e) - return result - - def visit_member_expr(self, e: MemberExpr) -> Type: - return self.expr_checker.visit_member_expr(e) - - def visit_break_stmt(self, s: BreakStmt) -> Type: - self.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1] - 1) - return None + self.accept(s.body) - def visit_continue_stmt(self, s: ContinueStmt) -> Type: - self.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1]) - return None - - def visit_int_expr(self, e: IntExpr) -> Type: - return self.expr_checker.visit_int_expr(e) - - def visit_str_expr(self, e: StrExpr) -> Type: - return self.expr_checker.visit_str_expr(e) + def check_untyped_after_decorator(self, typ: Type, func: FuncDef) -> None: + if not self.options.disallow_any_decorated or self.is_stub: + return - def visit_bytes_expr(self, e: BytesExpr) -> Type: - return self.expr_checker.visit_bytes_expr(e) + if mypy.checkexpr.has_any_type(typ): + self.msg.untyped_decorated_function(typ, func) - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - return self.expr_checker.visit_unicode_expr(e) + def check_async_with_item( + self, expr: Expression, target: Expression | None, infer_lvalue_type: bool + ) -> Type: + echk = self.expr_checker + ctx = echk.accept(expr) + obj = echk.check_method_call_by_name("__aenter__", ctx, [], [], expr)[0] + obj = echk.check_awaitable_expr( + obj, expr, message_registry.INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AENTER + ) + if target: + self.check_assignment(target, self.temp_node(obj, expr), infer_lvalue_type) + arg = self.temp_node(AnyType(TypeOfAny.special_form), expr) + res, _ = echk.check_method_call_by_name( + "__aexit__", ctx, [arg] * 3, [nodes.ARG_POS] * 3, expr + ) + return echk.check_awaitable_expr( + res, expr, message_registry.INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AEXIT + ) + + def check_with_item( + self, expr: Expression, target: Expression | None, infer_lvalue_type: bool + ) -> Type: + echk = self.expr_checker + ctx = echk.accept(expr) + obj = echk.check_method_call_by_name("__enter__", ctx, [], [], expr)[0] + if target: + self.check_assignment(target, self.temp_node(obj, expr), infer_lvalue_type) + arg = self.temp_node(AnyType(TypeOfAny.special_form), expr) + res, _ = echk.check_method_call_by_name( + "__exit__", ctx, [arg] * 3, [nodes.ARG_POS] * 3, expr + ) + return res + + def visit_break_stmt(self, s: BreakStmt) -> None: + self.binder.handle_break() + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + self.binder.handle_continue() + return + + def visit_match_stmt(self, s: MatchStmt) -> None: + named_subject: Expression + if isinstance(s.subject, CallExpr): + # Create a dummy subject expression to handle cases where a match statement's subject + # is not a literal value. This lets us correctly narrow types and check exhaustivity + # This is hack! + id = s.subject.callee.fullname if isinstance(s.subject.callee, RefExpr) else "" + name = "dummy-match-" + id + v = Var(name) + named_subject = NameExpr(name) + named_subject.node = v + else: + named_subject = s.subject + + with self.binder.frame_context(can_skip=False, fall_through=0): + subject_type = get_proper_type(self.expr_checker.accept(s.subject)) + + if isinstance(subject_type, DeletedType): + self.msg.deleted_as_rvalue(subject_type, s) + + # We infer types of patterns twice. The first pass is used + # to infer the types of capture variables. The type of a + # capture variable may depend on multiple patterns (it + # will be a union of all capture types). This pass ignores + # guard expressions. + pattern_types = [self.pattern_checker.accept(p, subject_type) for p in s.patterns] + type_maps: list[TypeMap] = [t.captures for t in pattern_types] + inferred_types = self.infer_variable_types_from_type_maps(type_maps) + + # The second pass narrows down the types and type checks bodies. + for p, g, b in zip(s.patterns, s.guards, s.bodies): + current_subject_type = self.expr_checker.narrow_type_from_binder( + named_subject, subject_type + ) + pattern_type = self.pattern_checker.accept(p, current_subject_type) + with self.binder.frame_context(can_skip=True, fall_through=2): + if b.is_unreachable or isinstance( + get_proper_type(pattern_type.type), UninhabitedType + ): + self.push_type_map(None, from_assignment=False) + else_map: TypeMap = {} + else: + pattern_map, else_map = conditional_types_to_typemaps( + named_subject, pattern_type.type, pattern_type.rest_type + ) + self.remove_capture_conflicts(pattern_type.captures, inferred_types) + self.push_type_map(pattern_map, from_assignment=False) + if pattern_map: + for expr, typ in pattern_map.items(): + self.push_type_map( + self._get_recursive_sub_patterns_map(expr, typ), + from_assignment=False, + ) + self.push_type_map(pattern_type.captures, from_assignment=False) + if g is not None: + with self.binder.frame_context(can_skip=False, fall_through=3): + gt = get_proper_type(self.expr_checker.accept(g)) + + if isinstance(gt, DeletedType): + self.msg.deleted_as_rvalue(gt, s) + + guard_map, guard_else_map = self.find_isinstance_check(g) + else_map = or_conditional_maps(else_map, guard_else_map) + + # If the guard narrowed the subject, copy the narrowed types over + if isinstance(p, AsPattern): + case_target = p.pattern or p.name + if isinstance(case_target, NameExpr): + for type_map in (guard_map, else_map): + if not type_map: + continue + for expr in list(type_map): + if not ( + isinstance(expr, NameExpr) + and expr.fullname == case_target.fullname + ): + continue + type_map[named_subject] = type_map[expr] + + self.push_type_map(guard_map, from_assignment=False) + self.accept(b) + else: + self.accept(b) + self.push_type_map(else_map, from_assignment=False) + + # This is needed due to a quirk in frame_context. Without it types will stay narrowed + # after the match. + with self.binder.frame_context(can_skip=False, fall_through=2): + pass + + def _get_recursive_sub_patterns_map( + self, expr: Expression, typ: Type + ) -> dict[Expression, Type]: + sub_patterns_map: dict[Expression, Type] = {} + typ_ = get_proper_type(typ) + if isinstance(expr, TupleExpr) and isinstance(typ_, TupleType): + # When matching a tuple expression with a sequence pattern, narrow individual tuple items + assert len(expr.items) == len(typ_.items) + for item_expr, item_typ in zip(expr.items, typ_.items): + sub_patterns_map[item_expr] = item_typ + sub_patterns_map.update(self._get_recursive_sub_patterns_map(item_expr, item_typ)) + + return sub_patterns_map + + def infer_variable_types_from_type_maps(self, type_maps: list[TypeMap]) -> dict[Var, Type]: + all_captures: dict[Var, list[tuple[NameExpr, Type]]] = defaultdict(list) + for tm in type_maps: + if tm is not None: + for expr, typ in tm.items(): + if isinstance(expr, NameExpr): + node = expr.node + assert isinstance(node, Var) + all_captures[node].append((expr, typ)) + + inferred_types: dict[Var, Type] = {} + for var, captures in all_captures.items(): + already_exists = False + types: list[Type] = [] + for expr, typ in captures: + types.append(typ) + + previous_type, _, _ = self.check_lvalue(expr) + if previous_type is not None: + already_exists = True + if self.check_subtype( + typ, + previous_type, + expr, + msg=message_registry.INCOMPATIBLE_TYPES_IN_CAPTURE, + subtype_label="pattern captures type", + supertype_label="variable has type", + ): + inferred_types[var] = previous_type + + if not already_exists: + new_type = UnionType.make_union(types) + # Infer the union type at the first occurrence + first_occurrence, _ = captures[0] + inferred_types[var] = new_type + self.infer_variable_type(var, first_occurrence, new_type, first_occurrence) + return inferred_types + + def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var, Type]) -> None: + if type_map: + for expr, typ in list(type_map.items()): + if isinstance(expr, NameExpr): + node = expr.node + assert isinstance(node, Var) + if node not in inferred_types or not is_subtype(typ, inferred_types[node]): + del type_map[expr] + + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + if o.alias_node: + self.check_typevar_defaults(o.alias_node.alias_tvars) + + with self.msg.filter_errors(): + self.expr_checker.accept(o.value) + + def make_fake_typeinfo( + self, + curr_module_fullname: str, + class_gen_name: str, + class_short_name: str, + bases: list[Instance], + ) -> tuple[ClassDef, TypeInfo]: + # Build the fake ClassDef and TypeInfo together. + # The ClassDef is full of lies and doesn't actually contain a body. + # Use format_bare to generate a nice name for error messages. + # We skip fully filling out a handful of TypeInfo fields because they + # should be irrelevant for a generated type like this: + # is_protocol, protocol_members, is_abstract + cdef = ClassDef(class_short_name, Block([])) + cdef.fullname = curr_module_fullname + "." + class_gen_name + info = TypeInfo(SymbolTable(), cdef, curr_module_fullname) + cdef.info = info + info.bases = bases + calculate_mro(info) + info.metaclass_type = info.calculate_metaclass_type() + return cdef, info + + def intersect_instances( + self, instances: tuple[Instance, Instance], errors: list[tuple[str, str]] + ) -> Instance | None: + """Try creating an ad-hoc intersection of the given instances. + + Note that this function does *not* try and create a full-fledged + intersection type. Instead, it returns an instance of a new ad-hoc + subclass of the given instances. + + This is mainly useful when you need a way of representing some + theoretical subclass of the instances the user may be trying to use + the generated intersection can serve as a placeholder. + + This function will create a fresh subclass every time you call it, + even if you pass in the exact same arguments. So this means calling + `self.intersect_intersection([inst_1, inst_2], ctx)` twice will result + in instances of two distinct subclasses of inst_1 and inst_2. + + This is by design: we want each ad-hoc intersection to be unique since + they're supposed represent some other unknown subclass. + + Returns None if creating the subclass is impossible (e.g. due to + MRO errors or incompatible signatures). If we do successfully create + a subclass, its TypeInfo will automatically be added to the global scope. + """ + curr_module = self.scope.stack[0] + assert isinstance(curr_module, MypyFile) + + # First, retry narrowing while allowing promotions (they are disabled by default + # for isinstance() checks, etc). This way we will still type-check branches like + # x: complex = 1 + # if isinstance(x, int): + # ... + left, right = instances + if is_proper_subtype(left, right, ignore_promotions=False): + return left + if is_proper_subtype(right, left, ignore_promotions=False): + return right + + def _get_base_classes(instances_: tuple[Instance, Instance]) -> list[Instance]: + base_classes_ = [] + for inst in instances_: + if inst.type.is_intersection: + expanded = inst.type.bases + else: + expanded = [inst] + + for expanded_inst in expanded: + base_classes_.append(expanded_inst) + return base_classes_ + + def _make_fake_typeinfo_and_full_name( + base_classes_: list[Instance], curr_module_: MypyFile + ) -> tuple[TypeInfo, str]: + names_list = pretty_seq([x.type.name for x in base_classes_], "and") + short_name = f"" + full_name_ = gen_unique_name(short_name, curr_module_.names) + cdef, info_ = self.make_fake_typeinfo( + curr_module_.fullname, full_name_, short_name, base_classes_ + ) + return info_, full_name_ + + base_classes = _get_base_classes(instances) + # We use the pretty_names_list for error messages but can't + # use it for the real name that goes into the symbol table + # because it can have dots in it. + pretty_names_list = pretty_seq( + format_type_distinctly(*base_classes, options=self.options, bare=True), "and" + ) + + new_errors = [] + for base in base_classes: + if base.type.is_final: + new_errors.append((pretty_names_list, f'"{base.type.name}" is final')) + if new_errors: + errors.extend(new_errors) + return None + + try: + info, full_name = _make_fake_typeinfo_and_full_name(base_classes, curr_module) + with self.msg.filter_errors() as local_errors: + self.check_multiple_inheritance(info) + if local_errors.has_new_errors(): + # "class A(B, C)" unsafe, now check "class A(C, B)": + base_classes = _get_base_classes(instances[::-1]) + info, full_name = _make_fake_typeinfo_and_full_name(base_classes, curr_module) + with self.msg.filter_errors() as local_errors: + self.check_multiple_inheritance(info) + info.is_intersection = True + except MroError: + errors.append((pretty_names_list, "would have inconsistent method resolution order")) + return None + if local_errors.has_new_errors(): + errors.append((pretty_names_list, "would have incompatible method signatures")) + return None + + curr_module.names[full_name] = SymbolTableNode(GDEF, info) + return Instance(info, [], extra_attrs=instances[0].extra_attrs or instances[1].extra_attrs) + + def intersect_instance_callable(self, typ: Instance, callable_type: CallableType) -> Instance: + """Creates a fake type that represents the intersection of an Instance and a CallableType. + + It operates by creating a bare-minimum dummy TypeInfo that + subclasses type and adds a __call__ method matching callable_type. + """ + + # In order for this to work in incremental mode, the type we generate needs to + # have a valid fullname and a corresponding entry in a symbol table. We generate + # a unique name inside the symbol table of the current module. + cur_module = self.scope.stack[0] + assert isinstance(cur_module, MypyFile) + gen_name = gen_unique_name(f"", cur_module.names) + + # Synthesize a fake TypeInfo + short_name = format_type_bare(typ, self.options) + cdef, info = self.make_fake_typeinfo(cur_module.fullname, gen_name, short_name, [typ]) + + # Build up a fake FuncDef so we can populate the symbol table. + func_def = FuncDef("__call__", [], Block([]), callable_type) + func_def._fullname = cdef.fullname + ".__call__" + func_def.info = info + info.names["__call__"] = SymbolTableNode(MDEF, func_def) + + cur_module.names[gen_name] = SymbolTableNode(GDEF, info) + + return Instance(info, [], extra_attrs=typ.extra_attrs) + + def make_fake_callable(self, typ: Instance) -> Instance: + """Produce a new type that makes type Callable with a generic callable type.""" + + fallback = self.named_type("builtins.function") + callable_type = CallableType( + [AnyType(TypeOfAny.explicit), AnyType(TypeOfAny.explicit)], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=AnyType(TypeOfAny.explicit), + fallback=fallback, + is_ellipsis_args=True, + ) + + return self.intersect_instance_callable(typ, callable_type) + + def partition_by_callable( + self, typ: Type, unsound_partition: bool + ) -> tuple[list[Type], list[Type]]: + """Partitions a type into callable subtypes and uncallable subtypes. + + Thus, given: + `callables, uncallables = partition_by_callable(type)` + + If we assert `callable(type)` then `type` has type Union[*callables], and + If we assert `not callable(type)` then `type` has type Union[*uncallables] + + If unsound_partition is set, assume that anything that is not + clearly callable is in fact not callable. Otherwise we generate a + new subtype that *is* callable. + + Guaranteed to not return [], []. + """ + typ = get_proper_type(typ) + + if isinstance(typ, (FunctionLike, TypeType)): + return [typ], [] + + if isinstance(typ, AnyType): + return [typ], [typ] + + if isinstance(typ, NoneType): + return [], [typ] + + if isinstance(typ, UnionType): + callables = [] + uncallables = [] + for subtype in typ.items: + # Use unsound_partition when handling unions in order to + # allow the expected type discrimination. + subcallables, subuncallables = self.partition_by_callable( + subtype, unsound_partition=True + ) + callables.extend(subcallables) + uncallables.extend(subuncallables) + return callables, uncallables + + if isinstance(typ, TypeVarType): + # We could do better probably? + # Refine the type variable's bound as our type in the case that + # callable() is true. This unfortunately loses the information that + # the type is a type variable in that branch. + # This matches what is done for isinstance, but it may be possible to + # do better. + # If it is possible for the false branch to execute, return the original + # type to avoid losing type information. + callables, uncallables = self.partition_by_callable( + erase_to_union_or_bound(typ), unsound_partition + ) + uncallables = [typ] if uncallables else [] + return callables, uncallables + + # A TupleType is callable if its fallback is, but needs special handling + # when we dummy up a new type. + ityp = typ + if isinstance(typ, TupleType): + ityp = tuple_fallback(typ) + + if isinstance(ityp, Instance): + method = ityp.type.get_method("__call__") + if method and method.type: + callables, uncallables = self.partition_by_callable( + method.type, unsound_partition=False + ) + if callables and not uncallables: + # Only consider the type callable if its __call__ method is + # definitely callable. + return [typ], [] + + if not unsound_partition: + fake = self.make_fake_callable(ityp) + if isinstance(typ, TupleType): + fake.type.tuple_type = TupleType(typ.items, fake) + return [fake.type.tuple_type], [typ] + return [fake], [typ] + + if unsound_partition: + return [], [typ] + else: + # We don't know how properly make the type callable. + return [typ], [typ] + + def conditional_callable_type_map( + self, expr: Expression, current_type: Type | None + ) -> tuple[TypeMap, TypeMap]: + """Takes in an expression and the current type of the expression. + + Returns a 2-tuple: The first element is a map from the expression to + the restricted type if it were callable. The second element is a + map from the expression to the type it would hold if it weren't + callable. + """ + if not current_type: + return {}, {} + + if isinstance(get_proper_type(current_type), AnyType): + return {}, {} + + callables, uncallables = self.partition_by_callable(current_type, unsound_partition=False) + + if callables and uncallables: + callable_map = {expr: UnionType.make_union(callables)} if callables else None + uncallable_map = {expr: UnionType.make_union(uncallables)} if uncallables else None + return callable_map, uncallable_map + + elif callables: + return {}, None + + return None, {} + + def conditional_types_for_iterable( + self, item_type: Type, iterable_type: Type + ) -> tuple[Type | None, Type | None]: + """ + Narrows the type of `iterable_type` based on the type of `item_type`. + For now, we only support narrowing unions of TypedDicts based on left operand being literal string(s). + """ + if_types: list[Type] = [] + else_types: list[Type] = [] + + iterable_type = get_proper_type(iterable_type) + if isinstance(iterable_type, UnionType): + possible_iterable_types = get_proper_types(iterable_type.relevant_items()) + else: + possible_iterable_types = [iterable_type] + + item_str_literals = try_getting_str_literals_from_type(item_type) + + for possible_iterable_type in possible_iterable_types: + if item_str_literals and isinstance(possible_iterable_type, TypedDictType): + for key in item_str_literals: + if key in possible_iterable_type.required_keys: + if_types.append(possible_iterable_type) + elif ( + key in possible_iterable_type.items or not possible_iterable_type.is_final + ): + if_types.append(possible_iterable_type) + else_types.append(possible_iterable_type) + else: + else_types.append(possible_iterable_type) + else: + if_types.append(possible_iterable_type) + else_types.append(possible_iterable_type) + + return ( + UnionType.make_union(if_types) if if_types else None, + UnionType.make_union(else_types) if else_types else None, + ) + + def _is_truthy_type(self, t: ProperType) -> bool: + return ( + ( + isinstance(t, Instance) + and bool(t.type) + and not t.type.has_readable_member("__bool__") + and not t.type.has_readable_member("__len__") + and t.type.fullname != "builtins.object" + ) + or isinstance(t, FunctionLike) + or ( + isinstance(t, UnionType) + and all(self._is_truthy_type(t) for t in get_proper_types(t.items)) + ) + ) + + def check_for_truthy_type(self, t: Type, expr: Expression) -> None: + """ + Check if a type can have a truthy value. + + Used in checks like:: - def visit_float_expr(self, e: FloatExpr) -> Type: - return self.expr_checker.visit_float_expr(e) + if x: # <--- - def visit_complex_expr(self, e: ComplexExpr) -> Type: - return self.expr_checker.visit_complex_expr(e) + not x # <--- + """ + if not state.strict_optional: + return # if everything can be None, all bets are off - def visit_op_expr(self, e: OpExpr) -> Type: - return self.expr_checker.visit_op_expr(e) + t = get_proper_type(t) + if not self._is_truthy_type(t): + return - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - return self.expr_checker.visit_comparison_expr(e) + def format_expr_type() -> str: + typ = format_type(t, self.options) + if isinstance(expr, MemberExpr): + return f'Member "{expr.name}" has type {typ}' + elif isinstance(expr, RefExpr) and expr.fullname: + return f'"{expr.fullname}" has type {typ}' + elif isinstance(expr, CallExpr): + if isinstance(expr.callee, MemberExpr): + return f'"{expr.callee.name}" returns {typ}' + elif isinstance(expr.callee, RefExpr) and expr.callee.fullname: + return f'"{expr.callee.fullname}" returns {typ}' + return f"Call returns {typ}" + else: + return f"Expression has type {typ}" - def visit_unary_expr(self, e: UnaryExpr) -> Type: - return self.expr_checker.visit_unary_expr(e) + def get_expr_name() -> str: + if isinstance(expr, (NameExpr, MemberExpr)): + return f'"{expr.name}"' + else: + # return type if expr has no name + return format_type(t, self.options) + + if isinstance(t, FunctionLike): + self.fail(message_registry.FUNCTION_ALWAYS_TRUE.format(get_expr_name()), expr) + elif isinstance(t, UnionType): + self.fail(message_registry.TYPE_ALWAYS_TRUE_UNIONTYPE.format(format_expr_type()), expr) + elif isinstance(t, Instance) and t.type.fullname == "typing.Iterable": + _, info = self.make_fake_typeinfo("typing", "Collection", "Collection", []) + self.fail( + message_registry.ITERABLE_ALWAYS_TRUE.format( + format_expr_type(), format_type(Instance(info, t.args), self.options) + ), + expr, + ) + else: + self.fail(message_registry.TYPE_ALWAYS_TRUE.format(format_expr_type()), expr) + + def find_type_equals_check( + self, node: ComparisonExpr, expr_indices: list[int] + ) -> tuple[TypeMap, TypeMap]: + """Narrow types based on any checks of the type ``type(x) == T`` - def visit_index_expr(self, e: IndexExpr) -> Type: - return self.expr_checker.visit_index_expr(e) + Args: + node: The node that might contain the comparison + expr_indices: The list of indices of expressions in ``node`` that are being + compared + """ - def visit_cast_expr(self, e: CastExpr) -> Type: - return self.expr_checker.visit_cast_expr(e) + def is_type_call(expr: CallExpr) -> bool: + """Is expr a call to type with one argument?""" + return refers_to_fullname(expr.callee, "builtins.type") and len(expr.args) == 1 + + # exprs that are being passed into type + exprs_in_type_calls: list[Expression] = [] + # type that is being compared to type(expr) + type_being_compared: list[TypeRange] | None = None + # whether the type being compared to is final + is_final = False + + for index in expr_indices: + expr = node.operands[index] + + if isinstance(expr, CallExpr) and is_type_call(expr): + exprs_in_type_calls.append(expr.args[0]) + else: + current_type = self.get_isinstance_type(expr) + if current_type is None: + continue + if type_being_compared is not None: + # It doesn't really make sense to have several types being + # compared to the output of type (like type(x) == int == str) + # because whether that's true is solely dependent on what the + # types being compared are, so we don't try to narrow types any + # further because we can't really get any information about the + # type of x from that check + return {}, {} + else: + if isinstance(expr, RefExpr) and isinstance(expr.node, TypeInfo): + is_final = expr.node.is_final + type_being_compared = current_type + + if not exprs_in_type_calls: + return {}, {} + + if_maps: list[TypeMap] = [] + else_maps: list[TypeMap] = [] + for expr in exprs_in_type_calls: + current_if_type, current_else_type = self.conditional_types_with_intersection( + self.lookup_type(expr), type_being_compared, expr + ) + current_if_map, current_else_map = conditional_types_to_typemaps( + expr, current_if_type, current_else_type + ) + if_maps.append(current_if_map) + else_maps.append(current_else_map) + + def combine_maps(list_maps: list[TypeMap]) -> TypeMap: + """Combine all typemaps in list_maps into one typemap""" + result_map = {} + for d in list_maps: + if d is not None: + result_map.update(d) + return result_map + + if_map = combine_maps(if_maps) + # type(x) == T is only true when x has the same type as T, meaning + # that it can be false if x is an instance of a subclass of T. That means + # we can't do any narrowing in the else case unless T is final, in which + # case T can't be subclassed + if is_final: + else_map = combine_maps(else_maps) + else: + else_map = {} + return if_map, else_map - def visit_super_expr(self, e: SuperExpr) -> Type: - return self.expr_checker.visit_super_expr(e) + def find_isinstance_check( + self, node: Expression, *, in_boolean_context: bool = True + ) -> tuple[TypeMap, TypeMap]: + """Find any isinstance checks (within a chain of ands). Includes + implicit and explicit checks for None and calls to callable. + Also includes TypeGuard and TypeIs functions. - def visit_type_application(self, e: TypeApplication) -> Type: - return self.expr_checker.visit_type_application(e) + Return value is a map of variables to their types if the condition + is true and a map of variables to their types if the condition is false. - def visit_type_var_expr(self, e: TypeVarExpr) -> Type: - # TODO: Perhaps return a special type used for type variables only? - return AnyType() + If either of the values in the tuple is None, then that particular + branch can never occur. + + If `in_boolean_context=True` is passed, it means that we handle + a walrus expression. We treat rhs values + in expressions like `(a := A())` specially: + for example, some errors are suppressed. + + May return {}, {}. + Can return None, None in situations involving NoReturn. + """ + if_map, else_map = self.find_isinstance_check_helper( + node, in_boolean_context=in_boolean_context + ) + new_if_map = self.propagate_up_typemap_info(if_map) + new_else_map = self.propagate_up_typemap_info(else_map) + return new_if_map, new_else_map + + def find_isinstance_check_helper( + self, node: Expression, *, in_boolean_context: bool = True + ) -> tuple[TypeMap, TypeMap]: + if is_true_literal(node): + return {}, None + if is_false_literal(node): + return None, {} + + if isinstance(node, CallExpr) and len(node.args) != 0: + expr = collapse_walrus(node.args[0]) + if refers_to_fullname(node.callee, "builtins.isinstance"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + if literal(expr) == LITERAL_TYPE: + return conditional_types_to_typemaps( + expr, + *self.conditional_types_with_intersection( + self.lookup_type(expr), self.get_isinstance_type(node.args[1]), expr + ), + ) + elif refers_to_fullname(node.callee, "builtins.issubclass"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + if literal(expr) == LITERAL_TYPE: + return self.infer_issubclass_maps(node, expr) + elif refers_to_fullname(node.callee, "builtins.callable"): + if len(node.args) != 1: # the error will be reported elsewhere + return {}, {} + if literal(expr) == LITERAL_TYPE: + vartype = self.lookup_type(expr) + return self.conditional_callable_type_map(expr, vartype) + elif refers_to_fullname(node.callee, "builtins.hasattr"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + attr = try_getting_str_literals(node.args[1], self.lookup_type(node.args[1])) + if literal(expr) == LITERAL_TYPE and attr and len(attr) == 1: + return self.hasattr_type_maps(expr, self.lookup_type(expr), attr[0]) + elif isinstance(node.callee, RefExpr): + if node.callee.type_guard is not None or node.callee.type_is is not None: + # TODO: Follow *args, **kwargs + if node.arg_kinds[0] != nodes.ARG_POS: + # the first argument might be used as a kwarg + called_type = get_proper_type(self.lookup_type(node.callee)) + + # TODO: there are some more cases in check_call() to handle. + if isinstance(called_type, Instance): + call = find_member( + "__call__", called_type, called_type, is_operator=True + ) + if call is not None: + called_type = get_proper_type(call) + + # *assuming* the overloaded function is correct, there's a couple cases: + # 1) The first argument has different names, but is pos-only. We don't + # care about this case, the argument must be passed positionally. + # 2) The first argument allows keyword reference, therefore must be the + # same between overloads. + if isinstance(called_type, (CallableType, Overloaded)): + name = called_type.items[0].arg_names[0] + if name in node.arg_names: + idx = node.arg_names.index(name) + # we want the idx-th variable to be narrowed + expr = collapse_walrus(node.args[idx]) + else: + kind = ( + "guard" if node.callee.type_guard is not None else "narrower" + ) + self.fail( + message_registry.TYPE_GUARD_POS_ARG_REQUIRED.format(kind), node + ) + return {}, {} + if literal(expr) == LITERAL_TYPE: + # Note: we wrap the target type, so that we can special case later. + # Namely, for isinstance() we use a normal meet, while TypeGuard is + # considered "always right" (i.e. even if the types are not overlapping). + # Also note that a care must be taken to unwrap this back at read places + # where we use this to narrow down declared type. + if node.callee.type_guard is not None: + return {expr: TypeGuardedType(node.callee.type_guard)}, {} + else: + assert node.callee.type_is is not None + return conditional_types_to_typemaps( + expr, + *self.conditional_types_with_intersection( + self.lookup_type(expr), + [TypeRange(node.callee.type_is, is_upper_bound=False)], + expr, + ), + ) + elif isinstance(node, ComparisonExpr): + return self.comparison_type_narrowing_helper(node) + elif isinstance(node, AssignmentExpr): + if_map: dict[Expression, Type] | None + else_map: dict[Expression, Type] | None + if_map = {} + else_map = {} + + if_assignment_map, else_assignment_map = self.find_isinstance_check(node.target) + + if if_assignment_map is not None: + if_map.update(if_assignment_map) + if else_assignment_map is not None: + else_map.update(else_assignment_map) + + if_condition_map, else_condition_map = self.find_isinstance_check( + node.value, in_boolean_context=False + ) + + if if_condition_map is not None: + if_map.update(if_condition_map) + if else_condition_map is not None: + else_map.update(else_condition_map) + + return ( + (None if if_assignment_map is None or if_condition_map is None else if_map), + (None if else_assignment_map is None or else_condition_map is None else else_map), + ) + elif isinstance(node, OpExpr) and node.op == "and": + left_if_vars, left_else_vars = self.find_isinstance_check(node.left) + right_if_vars, right_else_vars = self.find_isinstance_check(node.right) + + # (e1 and e2) is true if both e1 and e2 are true, + # and false if at least one of e1 and e2 is false. + return ( + and_conditional_maps(left_if_vars, right_if_vars), + # Note that if left else type is Any, we can't add any additional + # types to it, since the right maps were computed assuming + # the left is True, which may be not the case in the else branch. + or_conditional_maps(left_else_vars, right_else_vars, coalesce_any=True), + ) + elif isinstance(node, OpExpr) and node.op == "or": + left_if_vars, left_else_vars = self.find_isinstance_check(node.left) + right_if_vars, right_else_vars = self.find_isinstance_check(node.right) + + # (e1 or e2) is true if at least one of e1 or e2 is true, + # and false if both e1 and e2 are false. + return ( + or_conditional_maps(left_if_vars, right_if_vars), + and_conditional_maps(left_else_vars, right_else_vars), + ) + elif isinstance(node, UnaryExpr) and node.op == "not": + left, right = self.find_isinstance_check(node.expr) + return right, left + elif ( + literal(node) == LITERAL_TYPE + and self.has_type(node) + and self.can_be_narrowed_with_len(self.lookup_type(node)) + # Only translate `if x` to `if len(x) > 0` when possible. + and not custom_special_method(self.lookup_type(node), "__bool__") + and self.options.strict_optional + ): + # Combine a `len(x) > 0` check with the default logic below. + yes_type, no_type = self.narrow_with_len(self.lookup_type(node), ">", 0) + if yes_type is not None: + yes_type = true_only(yes_type) + else: + yes_type = UninhabitedType() + if no_type is not None: + no_type = false_only(no_type) + else: + no_type = UninhabitedType() + if_map = {node: yes_type} if not isinstance(yes_type, UninhabitedType) else None + else_map = {node: no_type} if not isinstance(no_type, UninhabitedType) else None + return if_map, else_map + + # Restrict the type of the variable to True-ish/False-ish in the if and else branches + # respectively + original_vartype = self.lookup_type(node) + if in_boolean_context: + # We don't check `:=` values in expressions like `(a := A())`, + # because they produce two error messages. + self.check_for_truthy_type(original_vartype, node) + vartype = try_expanding_sum_type_to_union(original_vartype, "builtins.bool") + + if_type = true_only(vartype) + else_type = false_only(vartype) + if_map = {node: if_type} if not isinstance(if_type, UninhabitedType) else None + else_map = {node: else_type} if not isinstance(else_type, UninhabitedType) else None + return if_map, else_map + + def comparison_type_narrowing_helper(self, node: ComparisonExpr) -> tuple[TypeMap, TypeMap]: + """Infer type narrowing from a comparison expression.""" + # Step 1: Obtain the types of each operand and whether or not we can + # narrow their types. (For example, we shouldn't try narrowing the + # types of literal string or enum expressions). + + operands = [collapse_walrus(x) for x in node.operands] + operand_types = [] + narrowable_operand_index_to_hash = {} + for i, expr in enumerate(operands): + if not self.has_type(expr): + return {}, {} + expr_type = self.lookup_type(expr) + operand_types.append(expr_type) + + if ( + literal(expr) == LITERAL_TYPE + and not is_literal_none(expr) + and not self.is_literal_enum(expr) + ): + h = literal_hash(expr) + if h is not None: + narrowable_operand_index_to_hash[i] = h + + # Step 2: Group operands chained by either the 'is' or '==' operands + # together. For all other operands, we keep them in groups of size 2. + # So the expression: + # + # x0 == x1 == x2 < x3 < x4 is x5 is x6 is not x7 is not x8 + # + # ...is converted into the simplified operator list: + # + # [("==", [0, 1, 2]), ("<", [2, 3]), ("<", [3, 4]), + # ("is", [4, 5, 6]), ("is not", [6, 7]), ("is not", [7, 8])] + # + # We group identity/equality expressions so we can propagate information + # we discover about one operand across the entire chain. We don't bother + # handling 'is not' and '!=' chains in a special way: those are very rare + # in practice. + + simplified_operator_list = group_comparison_operands( + node.pairwise(), narrowable_operand_index_to_hash, {"==", "is"} + ) + + # Step 3: Analyze each group and infer more precise type maps for each + # assignable operand, if possible. We combine these type maps together + # in the final step. + + partial_type_maps = [] + for operator, expr_indices in simplified_operator_list: + if operator in {"is", "is not", "==", "!="}: + if_map, else_map = self.equality_type_narrowing_helper( + node, + operator, + operands, + operand_types, + expr_indices, + narrowable_operand_index_to_hash, + ) + elif operator in {"in", "not in"}: + assert len(expr_indices) == 2 + left_index, right_index = expr_indices + item_type = operand_types[left_index] + iterable_type = operand_types[right_index] + + if_map, else_map = {}, {} + + if left_index in narrowable_operand_index_to_hash: + # We only try and narrow away 'None' for now + if is_overlapping_none(item_type): + collection_item_type = get_proper_type(builtin_item_type(iterable_type)) + if ( + collection_item_type is not None + and not is_overlapping_none(collection_item_type) + and not ( + isinstance(collection_item_type, Instance) + and collection_item_type.type.fullname == "builtins.object" + ) + and is_overlapping_erased_types(item_type, collection_item_type) + ): + if_map[operands[left_index]] = remove_optional(item_type) + + if right_index in narrowable_operand_index_to_hash: + if_type, else_type = self.conditional_types_for_iterable( + item_type, iterable_type + ) + expr = operands[right_index] + if if_type is None: + if_map = None + else: + if_map[expr] = if_type + if else_type is None: + else_map = None + else: + else_map[expr] = else_type - def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: - # TODO: Perhaps return a type object type? - return AnyType() + else: + if_map = {} + else_map = {} + + if operator in {"is not", "!=", "not in"}: + if_map, else_map = else_map, if_map + + partial_type_maps.append((if_map, else_map)) + + # If we have found non-trivial restrictions from the regular comparisons, + # then return soon. Otherwise try to infer restrictions involving `len(x)`. + # TODO: support regular and len() narrowing in the same chain. + if any(m != ({}, {}) for m in partial_type_maps): + return reduce_conditional_maps(partial_type_maps) + else: + # Use meet for `and` maps to get correct results for chained checks + # like `if 1 < len(x) < 4: ...` + return reduce_conditional_maps(self.find_tuple_len_narrowing(node), use_meet=True) + + def equality_type_narrowing_helper( + self, + node: ComparisonExpr, + operator: str, + operands: list[Expression], + operand_types: list[Type], + expr_indices: list[int], + narrowable_operand_index_to_hash: dict[int, tuple[Key, ...]], + ) -> tuple[TypeMap, TypeMap]: + """Calculate type maps for '==', '!=', 'is' or 'is not' expression.""" + # is_valid_target: + # Controls which types we're allowed to narrow exprs to. Note that + # we cannot use 'is_literal_type_like' in both cases since doing + # 'x = 10000 + 1; x is 10001' is not always True in all Python + # implementations. + # + # coerce_only_in_literal_context: + # If true, coerce types into literal types only if one or more of + # the provided exprs contains an explicit Literal type. This could + # technically be set to any arbitrary value, but it seems being liberal + # with narrowing when using 'is' and conservative when using '==' seems + # to break the least amount of real-world code. + # + # should_narrow_by_identity: + # Set to 'false' only if the user defines custom __eq__ or __ne__ methods + # that could cause identity-based narrowing to produce invalid results. + if operator in {"is", "is not"}: + is_valid_target: Callable[[Type], bool] = is_singleton_type + coerce_only_in_literal_context = False + should_narrow_by_identity = True + else: + + def is_exactly_literal_type(t: Type) -> bool: + return isinstance(get_proper_type(t), LiteralType) + + def has_no_custom_eq_checks(t: Type) -> bool: + return not custom_special_method( + t, "__eq__", check_all=False + ) and not custom_special_method(t, "__ne__", check_all=False) + + is_valid_target = is_exactly_literal_type + coerce_only_in_literal_context = True + + expr_types = [operand_types[i] for i in expr_indices] + should_narrow_by_identity = all( + map(has_no_custom_eq_checks, expr_types) + ) and not is_ambiguous_mix_of_enums(expr_types) + + if_map: TypeMap = {} + else_map: TypeMap = {} + if should_narrow_by_identity: + if_map, else_map = self.refine_identity_comparison_expression( + operands, + operand_types, + expr_indices, + narrowable_operand_index_to_hash.keys(), + is_valid_target, + coerce_only_in_literal_context, + ) + + if if_map == {} and else_map == {}: + if_map, else_map = self.refine_away_none_in_comparison( + operands, operand_types, expr_indices, narrowable_operand_index_to_hash.keys() + ) + + # If we haven't been able to narrow types yet, we might be dealing with a + # explicit type(x) == some_type check + if if_map == {} and else_map == {}: + if_map, else_map = self.find_type_equals_check(node, expr_indices) + return if_map, else_map + + def propagate_up_typemap_info(self, new_types: TypeMap) -> TypeMap: + """Attempts refining parent expressions of any MemberExpr or IndexExprs in new_types. + + Specifically, this function accepts two mappings of expression to original types: + the original mapping (existing_types), and a new mapping (new_types) intended to + update the original. + + This function iterates through new_types and attempts to use the information to try + refining any parent types that happen to be unions. + + For example, suppose there are two types "A = Tuple[int, int]" and "B = Tuple[str, str]". + Next, suppose that 'new_types' specifies the expression 'foo[0]' has a refined type + of 'int' and that 'foo' was previously deduced to be of type Union[A, B]. + + Then, this function will observe that since A[0] is an int and B[0] is not, the type of + 'foo' can be further refined from Union[A, B] into just B. + + We perform this kind of "parent narrowing" for member lookup expressions and indexing + expressions into tuples, namedtuples, and typeddicts. We repeat this narrowing + recursively if the parent is also a "lookup expression". So for example, if we have + the expression "foo['bar'].baz[0]", we'd potentially end up refining types for the + expressions "foo", "foo['bar']", and "foo['bar'].baz". + + We return the newly refined map. This map is guaranteed to be a superset of 'new_types'. + """ + if new_types is None: + return None + output_map = {} + for expr, expr_type in new_types.items(): + # The original inferred type should always be present in the output map, of course + output_map[expr] = expr_type + + # Next, try using this information to refine the parent types, if applicable. + new_mapping = self.refine_parent_types(expr, expr_type) + for parent_expr, proposed_parent_type in new_mapping.items(): + # We don't try inferring anything if we've already inferred something for + # the parent expression. + # TODO: Consider picking the narrower type instead of always discarding this? + if parent_expr in new_types: + continue + output_map[parent_expr] = proposed_parent_type + return output_map + + def refine_parent_types(self, expr: Expression, expr_type: Type) -> Mapping[Expression, Type]: + """Checks if the given expr is a 'lookup operation' into a union and iteratively refines + the parent types based on the 'expr_type'. + + For example, if 'expr' is an expression like 'a.b.c.d', we'll potentially return refined + types for expressions 'a', 'a.b', and 'a.b.c'. + + For more details about what a 'lookup operation' is and how we use the expr_type to refine + the parent types of lookup_expr, see the docstring in 'propagate_up_typemap_info'. + """ + output: dict[Expression, Type] = {} + + # Note: parent_expr and parent_type are progressively refined as we crawl up the + # parent lookup chain. + while True: + # First, check if this expression is one that's attempting to + # "lookup" some key in the parent type. If so, save the parent type + # and create function that will try replaying the same lookup + # operation against arbitrary types. + if isinstance(expr, MemberExpr): + parent_expr = collapse_walrus(expr.expr) + parent_type = self.lookup_type_or_none(parent_expr) + member_name = expr.name + + def replay_lookup(new_parent_type: ProperType) -> Type | None: + with self.msg.filter_errors() as w: + member_type = analyze_member_access( + name=member_name, + typ=new_parent_type, + context=parent_expr, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, + original_type=new_parent_type, + chk=self, + in_literal_context=False, + ) + if w.has_new_errors(): + return None + else: + return member_type + + elif isinstance(expr, IndexExpr): + parent_expr = collapse_walrus(expr.base) + parent_type = self.lookup_type_or_none(parent_expr) + + index_type = self.lookup_type_or_none(expr.index) + if index_type is None: + return output + + str_literals = try_getting_str_literals_from_type(index_type) + if str_literals is not None: + # Refactoring these two indexing replay functions is surprisingly + # tricky -- see https://github.com/python/mypy/pull/7917, which + # was blocked by https://github.com/mypyc/mypyc/issues/586 + def replay_lookup(new_parent_type: ProperType) -> Type | None: + if not isinstance(new_parent_type, TypedDictType): + return None + try: + assert str_literals is not None + member_types = [new_parent_type.items[key] for key in str_literals] + except KeyError: + return None + return make_simplified_union(member_types) + + else: + int_literals = try_getting_int_literals_from_type(index_type) + if int_literals is not None: + + def replay_lookup(new_parent_type: ProperType) -> Type | None: + if not isinstance(new_parent_type, TupleType): + return None + try: + assert int_literals is not None + member_types = [new_parent_type.items[key] for key in int_literals] + except IndexError: + return None + return make_simplified_union(member_types) + + else: + return output + else: + return output + + # If we somehow didn't previously derive the parent type, abort completely + # with what we have so far: something went wrong at an earlier stage. + if parent_type is None: + return output + + # We currently only try refining the parent type if it's a Union. + # If not, there's no point in trying to refine any further parents + # since we have no further information we can use to refine the lookup + # chain, so we end early as an optimization. + parent_type = get_proper_type(parent_type) + if not isinstance(parent_type, UnionType): + return output + + # Take each element in the parent union and replay the original lookup procedure + # to figure out which parents are compatible. + new_parent_types = [] + for item in flatten_nested_unions(parent_type.items): + member_type = replay_lookup(get_proper_type(item)) + if member_type is None: + # We were unable to obtain the member type. So, we give up on refining this + # parent type entirely and abort. + return output + + if is_overlapping_types(member_type, expr_type): + new_parent_types.append(item) + + # If none of the parent types overlap (if we derived an empty union), something + # went wrong. We should never hit this case, but deriving the uninhabited type or + # reporting an error both seem unhelpful. So we abort. + if not new_parent_types: + return output + + expr = parent_expr + expr_type = output[parent_expr] = make_simplified_union(new_parent_types) + + def refine_identity_comparison_expression( + self, + operands: list[Expression], + operand_types: list[Type], + chain_indices: list[int], + narrowable_operand_indices: AbstractSet[int], + is_valid_target: Callable[[ProperType], bool], + coerce_only_in_literal_context: bool, + ) -> tuple[TypeMap, TypeMap]: + """Produce conditional type maps refining expressions by an identity/equality comparison. + + The 'operands' and 'operand_types' lists should be the full list of operands used + in the overall comparison expression. The 'chain_indices' list is the list of indices + actually used within this identity comparison chain. + + So if we have the expression: + + a <= b is c is d <= e + + ...then 'operands' and 'operand_types' would be lists of length 5 and 'chain_indices' + would be the list [1, 2, 3]. + + The 'narrowable_operand_indices' parameter is the set of all indices we are allowed + to refine the types of: that is, all operands that will potentially be a part of + the output TypeMaps. + + Although this function could theoretically try setting the types of the operands + in the chains to the meet, doing that causes too many issues in real-world code. + Instead, we use 'is_valid_target' to identify which of the given chain types + we could plausibly use as the refined type for the expressions in the chain. + + Similarly, 'coerce_only_in_literal_context' controls whether we should try coercing + expressions in the chain to a Literal type. Performing this coercion is sometimes + too aggressive of a narrowing, depending on context. + """ + should_coerce = True + if coerce_only_in_literal_context: + + def should_coerce_inner(typ: Type) -> bool: + typ = get_proper_type(typ) + return is_literal_type_like(typ) or ( + isinstance(typ, Instance) and typ.type.is_enum + ) + + should_coerce = any(should_coerce_inner(operand_types[i]) for i in chain_indices) + + target: Type | None = None + possible_target_indices = [] + for i in chain_indices: + expr_type = operand_types[i] + if should_coerce: + expr_type = coerce_to_literal(expr_type) + if not is_valid_target(get_proper_type(expr_type)): + continue + if target and not is_same_type(target, expr_type): + # We have multiple disjoint target types. So the 'if' branch + # must be unreachable. + return None, {} + target = expr_type + possible_target_indices.append(i) + + # There's nothing we can currently infer if none of the operands are valid targets, + # so we end early and infer nothing. + if target is None: + return {}, {} + + # If possible, use an unassignable expression as the target. + # We skip refining the type of the target below, so ideally we'd + # want to pick an expression we were going to skip anyways. + singleton_index = -1 + for i in possible_target_indices: + if i not in narrowable_operand_indices: + singleton_index = i + + # But if none of the possible singletons are unassignable ones, we give up + # and arbitrarily pick the last item, mostly because other parts of the + # type narrowing logic bias towards picking the rightmost item and it'd be + # nice to stay consistent. + # + # That said, it shouldn't matter which index we pick. For example, suppose we + # have this if statement, where 'x' and 'y' both have singleton types: + # + # if x is y: + # reveal_type(x) + # reveal_type(y) + # else: + # reveal_type(x) + # reveal_type(y) + # + # At this point, 'x' and 'y' *must* have the same singleton type: we would have + # ended early in the first for-loop in this function if they weren't. + # + # So, we should always get the same result in the 'if' case no matter which + # index we pick. And while we do end up getting different results in the 'else' + # case depending on the index (e.g. if we pick 'y', then its type stays the same + # while 'x' is narrowed to ''), this distinction is also moot: mypy + # currently will just mark the whole branch as unreachable if either operand is + # narrowed to . + if singleton_index == -1: + singleton_index = possible_target_indices[-1] + + sum_type_name = None + target = get_proper_type(target) + if isinstance(target, LiteralType) and ( + target.is_enum_literal() or isinstance(target.value, bool) + ): + sum_type_name = target.fallback.type.fullname + + target_type = [TypeRange(target, is_upper_bound=False)] + + partial_type_maps = [] + for i in chain_indices: + # If we try refining a type against itself, conditional_type_map + # will end up assuming that the 'else' branch is unreachable. This is + # typically not what we want: generally the user will intend for the + # target type to be some fixed 'sentinel' value and will want to refine + # the other exprs against this one instead. + if i == singleton_index: + continue - def visit_list_expr(self, e: ListExpr) -> Type: - return self.expr_checker.visit_list_expr(e) + # Naturally, we can't refine operands which are not permitted to be refined. + if i not in narrowable_operand_indices: + continue - def visit_set_expr(self, e: SetExpr) -> Type: - return self.expr_checker.visit_set_expr(e) + expr = operands[i] + expr_type = coerce_to_literal(operand_types[i]) - def visit_tuple_expr(self, e: TupleExpr) -> Type: - return self.expr_checker.visit_tuple_expr(e) + if sum_type_name is not None: + expr_type = try_expanding_sum_type_to_union(expr_type, sum_type_name) - def visit_dict_expr(self, e: DictExpr) -> Type: - return self.expr_checker.visit_dict_expr(e) + # We intentionally use 'conditional_types' directly here instead of + # 'self.conditional_types_with_intersection': we only compute ad-hoc + # intersections when working with pure instances. + types = conditional_types(expr_type, target_type) + partial_type_maps.append(conditional_types_to_typemaps(expr, *types)) - def visit_slice_expr(self, e: SliceExpr) -> Type: - return self.expr_checker.visit_slice_expr(e) + return reduce_conditional_maps(partial_type_maps) - def visit_func_expr(self, e: FuncExpr) -> Type: - return self.expr_checker.visit_func_expr(e) + def refine_away_none_in_comparison( + self, + operands: list[Expression], + operand_types: list[Type], + chain_indices: list[int], + narrowable_operand_indices: AbstractSet[int], + ) -> tuple[TypeMap, TypeMap]: + """Produces conditional type maps refining away None in an identity/equality chain. - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.expr_checker.visit_list_comprehension(e) + For more details about what the different arguments mean, see the + docstring of 'refine_identity_comparison_expression' up above. + """ - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.expr_checker.visit_set_comprehension(e) + non_optional_types = [] + for i in chain_indices: + typ = operand_types[i] + if not is_overlapping_none(typ): + non_optional_types.append(typ) + + if_map, else_map = {}, {} + + if not non_optional_types or (len(non_optional_types) != len(chain_indices)): + + # Narrow e.g. `Optional[A] == "x"` or `Optional[A] is "x"` to `A` (which may be + # convenient but is strictly not type-safe): + for i in narrowable_operand_indices: + expr_type = operand_types[i] + if not is_overlapping_none(expr_type): + continue + if any(is_overlapping_erased_types(expr_type, t) for t in non_optional_types): + if_map[operands[i]] = remove_optional(expr_type) + + # Narrow e.g. `Optional[A] != None` to `A` (which is stricter than the above step and + # so type-safe but less convenient, because e.g. `Optional[A] == None` still results + # in `Optional[A]`): + if any(isinstance(get_proper_type(ot), NoneType) for ot in operand_types): + for i in narrowable_operand_indices: + expr_type = operand_types[i] + if is_overlapping_none(expr_type): + else_map[operands[i]] = remove_optional(expr_type) + + return if_map, else_map + + def is_len_of_tuple(self, expr: Expression) -> bool: + """Is this expression a `len(x)` call where x is a tuple or union of tuples?""" + if not isinstance(expr, CallExpr): + return False + if not refers_to_fullname(expr.callee, "builtins.len"): + return False + if len(expr.args) != 1: + return False + expr = expr.args[0] + if literal(expr) != LITERAL_TYPE: + return False + if not self.has_type(expr): + return False + return self.can_be_narrowed_with_len(self.lookup_type(expr)) - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.expr_checker.visit_generator_expr(e) + def can_be_narrowed_with_len(self, typ: Type) -> bool: + """Is this a type that can benefit from length check type restrictions? - def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: - return self.expr_checker.visit_dictionary_comprehension(e) + Currently supported types are TupleTypes, Instances of builtins.tuple, and + unions involving such types. + """ + if custom_special_method(typ, "__len__"): + # If user overrides builtin behavior, we can't do anything. + return False + p_typ = get_proper_type(typ) + # Note: we are conservative about tuple subclasses, because some code may rely on + # the fact that tuple_type of fallback TypeInfo matches the original TupleType. + if isinstance(p_typ, TupleType): + if any(isinstance(t, UnpackType) for t in p_typ.items): + return p_typ.partial_fallback.type.fullname == "builtins.tuple" + return True + if isinstance(p_typ, Instance): + return p_typ.type.has_base("builtins.tuple") + if isinstance(p_typ, UnionType): + return any(self.can_be_narrowed_with_len(t) for t in p_typ.items) + return False - def visit_undefined_expr(self, e: UndefinedExpr) -> Type: - return self.expr_checker.visit_undefined_expr(e) + def literal_int_expr(self, expr: Expression) -> int | None: + """Is this expression an int literal, or a reference to an int constant? - def visit_temp_node(self, e: TempNode) -> Type: - return e.type + If yes, return the corresponding int value, otherwise return None. + """ + if not self.has_type(expr): + return None + expr_type = self.lookup_type(expr) + expr_type = coerce_to_literal(expr_type) + proper_type = get_proper_type(expr_type) + if not isinstance(proper_type, LiteralType): + return None + if not isinstance(proper_type.value, int): + return None + return proper_type.value + + def find_tuple_len_narrowing(self, node: ComparisonExpr) -> list[tuple[TypeMap, TypeMap]]: + """Top-level logic to find type restrictions from a length check on tuples. + + We try to detect `if` checks like the following: + x: tuple[int, int] | tuple[int, int, int] + y: tuple[int, int] | tuple[int, int, int] + if len(x) == len(y) == 2: + a, b = x # OK + c, d = y # OK + + z: tuple[int, ...] + if 1 < len(z) < 4: + x = z # OK + and report corresponding type restrictions to the binder. + """ + # First step: group consecutive `is` and `==` comparisons together. + # This is essentially a simplified version of group_comparison_operands(), + # tuned to the len()-like checks. Note that we don't propagate indirect + # restrictions like e.g. `len(x) > foo() > 1` yet, since it is tricky. + # TODO: propagate indirect len() comparison restrictions. + chained = [] + last_group = set() + for op, left, right in node.pairwise(): + if isinstance(left, AssignmentExpr): + left = left.value + if isinstance(right, AssignmentExpr): + right = right.value + if op in ("is", "=="): + last_group.add(left) + last_group.add(right) + else: + if last_group: + chained.append(("==", list(last_group))) + last_group = set() + if op in {"is not", "!=", "<", "<=", ">", ">="}: + chained.append((op, [left, right])) + if last_group: + chained.append(("==", list(last_group))) + + # Second step: infer type restrictions from each group found above. + type_maps = [] + for op, items in chained: + # TODO: support unions of literal types as len() comparison targets. + if not any(self.literal_int_expr(it) is not None for it in items): + continue + if not any(self.is_len_of_tuple(it) for it in items): + continue - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - return self.expr_checker.visit_conditional_expr(e) + # At this step we know there is at least one len(x) and one literal in the group. + if op in ("is", "=="): + literal_values = set() + tuples = [] + for it in items: + lit = self.literal_int_expr(it) + if lit is not None: + literal_values.add(lit) + continue + if self.is_len_of_tuple(it): + assert isinstance(it, CallExpr) + tuples.append(it.args[0]) + if len(literal_values) > 1: + # More than one different literal value found, like 1 == len(x) == 2, + # so the corresponding branch is unreachable. + return [(None, {})] + size = literal_values.pop() + if size > MAX_PRECISE_TUPLE_SIZE: + # Avoid creating huge tuples from checks like if len(x) == 300. + continue + for tpl in tuples: + yes_type, no_type = self.narrow_with_len(self.lookup_type(tpl), op, size) + yes_map = None if yes_type is None else {tpl: yes_type} + no_map = None if no_type is None else {tpl: no_type} + type_maps.append((yes_map, no_map)) + else: + left, right = items + if self.is_len_of_tuple(right): + # Normalize `1 < len(x)` and similar as `len(x) > 1`. + left, right = right, left + op = flip_ops.get(op, op) + r_size = self.literal_int_expr(right) + assert r_size is not None + if r_size > MAX_PRECISE_TUPLE_SIZE: + # Avoid creating huge unions from checks like if len(x) > 300. + continue + assert isinstance(left, CallExpr) + yes_type, no_type = self.narrow_with_len( + self.lookup_type(left.args[0]), op, r_size + ) + yes_map = None if yes_type is None else {left.args[0]: yes_type} + no_map = None if no_type is None else {left.args[0]: no_type} + type_maps.append((yes_map, no_map)) + return type_maps + + def narrow_with_len(self, typ: Type, op: str, size: int) -> tuple[Type | None, Type | None]: + """Dispatch tuple type narrowing logic depending on the kind of type we got.""" + typ = get_proper_type(typ) + if isinstance(typ, TupleType): + return self.refine_tuple_type_with_len(typ, op, size) + elif isinstance(typ, Instance): + return self.refine_instance_type_with_len(typ, op, size) + elif isinstance(typ, UnionType): + yes_types = [] + no_types = [] + other_types = [] + for t in typ.items: + if not self.can_be_narrowed_with_len(t): + other_types.append(t) + continue + yt, nt = self.narrow_with_len(t, op, size) + if yt is not None: + yes_types.append(yt) + if nt is not None: + no_types.append(nt) + yes_types += other_types + no_types += other_types + if yes_types: + yes_type = make_simplified_union(yes_types) + else: + yes_type = None + if no_types: + no_type = make_simplified_union(no_types) + else: + no_type = None + return yes_type, no_type + else: + assert False, "Unsupported type for len narrowing" + + def refine_tuple_type_with_len( + self, typ: TupleType, op: str, size: int + ) -> tuple[Type | None, Type | None]: + """Narrow a TupleType using length restrictions.""" + unpack_index = find_unpack_in_list(typ.items) + if unpack_index is None: + # For fixed length tuple situation is trivial, it is either reachable or not, + # depending on the current length, expected length, and the comparison op. + method = int_op_to_method[op] + if method(typ.length(), size): + return typ, None + return None, typ + unpack = typ.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if isinstance(unpacked, TypeVarTupleType): + # For tuples involving TypeVarTuple unpack we can't do much except + # inferring reachability, and recording the restrictions on TypeVarTuple + # for further "manual" use elsewhere. + min_len = typ.length() - 1 + unpacked.min_len + if op in ("==", "is"): + if min_len <= size: + return typ, typ + return None, typ + elif op in ("<", "<="): + if op == "<=": + size += 1 + if min_len < size: + prefix = typ.items[:unpack_index] + suffix = typ.items[unpack_index + 1 :] + # TODO: also record max_len to avoid false negatives? + unpack = UnpackType(unpacked.copy_modified(min_len=size - typ.length() + 1)) + return typ, typ.copy_modified(items=prefix + [unpack] + suffix) + return None, typ + else: + yes_type, no_type = self.refine_tuple_type_with_len(typ, neg_ops[op], size) + return no_type, yes_type + # Homogeneous variadic item is the case where we are most flexible. Essentially, + # we adjust the variadic item by "eating away" from it to satisfy the restriction. + assert isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + min_len = typ.length() - 1 + arg = unpacked.args[0] + prefix = typ.items[:unpack_index] + suffix = typ.items[unpack_index + 1 :] + if op in ("==", "is"): + if min_len <= size: + # TODO: return fixed union + prefixed variadic tuple for no_type? + return typ.copy_modified(items=prefix + [arg] * (size - min_len) + suffix), typ + return None, typ + elif op in ("<", "<="): + if op == "<=": + size += 1 + if min_len < size: + # Note: there is some ambiguity w.r.t. to where to put the additional + # items: before or after the unpack. However, such types are equivalent, + # so we always put them before for consistency. + no_type = typ.copy_modified( + items=prefix + [arg] * (size - min_len) + [unpack] + suffix + ) + yes_items = [] + for n in range(size - min_len): + yes_items.append(typ.copy_modified(items=prefix + [arg] * n + suffix)) + return UnionType.make_union(yes_items, typ.line, typ.column), no_type + return None, typ + else: + yes_type, no_type = self.refine_tuple_type_with_len(typ, neg_ops[op], size) + return no_type, yes_type + + def refine_instance_type_with_len( + self, typ: Instance, op: str, size: int + ) -> tuple[Type | None, Type | None]: + """Narrow a homogeneous tuple using length restrictions.""" + base = map_instance_to_supertype(typ, self.lookup_typeinfo("builtins.tuple")) + arg = base.args[0] + # Again, we are conservative about subclasses until we gain more confidence. + allow_precise = ( + PRECISE_TUPLE_TYPES in self.options.enable_incomplete_feature + ) and typ.type.fullname == "builtins.tuple" + if op in ("==", "is"): + # TODO: return fixed union + prefixed variadic tuple for no_type? + return TupleType(items=[arg] * size, fallback=typ), typ + elif op in ("<", "<="): + if op == "<=": + size += 1 + if allow_precise: + unpack = UnpackType(self.named_generic_type("builtins.tuple", [arg])) + no_type: Type | None = TupleType(items=[arg] * size + [unpack], fallback=typ) + else: + no_type = typ + if allow_precise: + items = [] + for n in range(size): + items.append(TupleType([arg] * n, fallback=typ)) + yes_type: Type | None = UnionType.make_union(items, typ.line, typ.column) + else: + yes_type = typ + return yes_type, no_type + else: + yes_type, no_type = self.refine_instance_type_with_len(typ, neg_ops[op], size) + return no_type, yes_type # # Helpers # + @overload + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: str, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, + code: ErrorCode | None = None, + outer_context: Context | None = None, + ) -> bool: ... - def check_subtype(self, subtype: Type, supertype: Type, context: Context, - msg: str = messages.INCOMPATIBLE_TYPES, - subtype_label: str = None, - supertype_label: str = None) -> None: - """Generate an error if the subtype is not compatible with - supertype.""" - if not is_subtype(subtype, supertype): - if isinstance(subtype, Void): - self.msg.does_not_return_value(subtype, context) - else: - extra_info = [] # type: List[str] - if subtype_label is not None: - extra_info.append(subtype_label + ' ' + self.msg.format(subtype, verbose=True)) - if supertype_label is not None: - extra_info.append(supertype_label + ' ' + self.msg.format(supertype, - verbose=True)) - if extra_info: - msg += ' (' + ', '.join(extra_info) + ')' - self.fail(msg, context) + @overload + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: ErrorMessage, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, + outer_context: Context | None = None, + ) -> bool: ... + + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: str | ErrorMessage, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, + code: ErrorCode | None = None, + outer_context: Context | None = None, + ) -> bool: + """Generate an error if the subtype is not compatible with supertype.""" + if is_subtype(subtype, supertype, options=self.options): + return True + + if isinstance(msg, str): + msg = ErrorMessage(msg, code=code) + + if self.msg.prefer_simple_messages(): + self.fail(msg, context) # Fast path -- skip all fancy logic + return False + + orig_subtype = subtype + subtype = get_proper_type(subtype) + orig_supertype = supertype + supertype = get_proper_type(supertype) + if self.msg.try_report_long_tuple_assignment_error( + subtype, supertype, context, msg, subtype_label, supertype_label + ): + return False + extra_info: list[str] = [] + note_msg = "" + notes = notes or [] + if subtype_label is not None or supertype_label is not None: + subtype_str, supertype_str = format_type_distinctly( + orig_subtype, orig_supertype, options=self.options + ) + if subtype_label is not None: + extra_info.append(subtype_label + " " + subtype_str) + if supertype_label is not None: + extra_info.append(supertype_label + " " + supertype_str) + note_msg = make_inferred_type_note( + outer_context or context, subtype, supertype, supertype_str + ) + if isinstance(subtype, Instance) and isinstance(supertype, Instance): + notes = append_invariance_notes(notes, subtype, supertype) + if isinstance(subtype, UnionType) and isinstance(supertype, UnionType): + notes = append_union_note(notes, subtype, supertype, self.options) + if extra_info: + msg = msg.with_additional_msg(" (" + ", ".join(extra_info) + ")") + + self.fail(msg, context) + for note in notes: + self.msg.note(note, context, code=msg.code) + if note_msg: + self.note(note_msg, context, code=msg.code) + self.msg.maybe_note_concatenate_pos_args(subtype, supertype, context, code=msg.code) + if ( + isinstance(supertype, Instance) + and supertype.type.is_protocol + and isinstance(subtype, (CallableType, Instance, TupleType, TypedDictType)) + ): + self.msg.report_protocol_problems(subtype, supertype, context, code=msg.code) + if isinstance(supertype, CallableType) and isinstance(subtype, Instance): + call = find_member("__call__", subtype, subtype, is_operator=True) + if call: + self.msg.note_call(subtype, call, context, code=msg.code) + if isinstance(subtype, (CallableType, Overloaded)) and isinstance(supertype, Instance): + if supertype.type.is_protocol and "__call__" in supertype.type.protocol_members: + call = find_member("__call__", supertype, subtype, is_operator=True) + assert call is not None + if not is_subtype(subtype, call, options=self.options): + self.msg.note_call(supertype, call, context, code=msg.code) + self.check_possible_missing_await(subtype, supertype, context, code=msg.code) + return False + + def get_precise_awaitable_type(self, typ: Type, local_errors: ErrorWatcher) -> Type | None: + """If type implements Awaitable[X] with non-Any X, return X. + + In all other cases return None. This method must be called in context + of local_errors. + """ + if isinstance(get_proper_type(typ), PartialType): + # Partial types are special, ignore them here. + return None + try: + aw_type = self.expr_checker.check_awaitable_expr( + typ, Context(), "", ignore_binder=True + ) + except KeyError: + # This is a hack to speed up tests by not including Awaitable in all typing stubs. + return None + if local_errors.has_new_errors(): + return None + if isinstance(get_proper_type(aw_type), (AnyType, UnboundType)): + return None + return aw_type + + @contextmanager + def checking_await_set(self) -> Iterator[None]: + self.checking_missing_await = True + try: + yield + finally: + self.checking_missing_await = False + + def check_possible_missing_await( + self, subtype: Type, supertype: Type, context: Context, code: ErrorCode | None + ) -> None: + """Check if the given type becomes a subtype when awaited.""" + if self.checking_missing_await: + # Avoid infinite recursion. + return + with self.checking_await_set(), self.msg.filter_errors() as local_errors: + aw_type = self.get_precise_awaitable_type(subtype, local_errors) + if aw_type is None: + return + if not self.check_subtype( + aw_type, supertype, context, msg=message_registry.INCOMPATIBLE_TYPES + ): + return + self.msg.possible_missing_await(context, code) def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no - type arguments. For example, named_type('builtins.object') - produces the object type. + """Return an instance type with given name and implicit Any type args. + + For example, named_type('builtins.object') produces the 'object' type. """ # Assume that the name refers to a type. sym = self.lookup_qualified(name) - return Instance(cast(TypeInfo, sym.node), []) - - def named_generic_type(self, name: str, args: List[Type]) -> Instance: + node = sym.node + if isinstance(node, TypeAlias): + assert isinstance(node.target, Instance) # type: ignore[misc] + node = node.target.type + assert isinstance(node, TypeInfo) + any_type = AnyType(TypeOfAny.from_omitted_generics) + return Instance(node, [any_type] * len(node.defn.type_vars)) + + def named_generic_type(self, name: str, args: list[Type]) -> Instance: """Return an instance with the given name and type arguments. Assume that the number of arguments is correct. Assume that the name refers to a compatible generic type. """ - return Instance(self.lookup_typeinfo(name), args) + info = self.lookup_typeinfo(name) + args = [remove_instance_last_known_values(arg) for arg in args] + # TODO: assert len(args) == len(info.defn.type_vars) + return Instance(info, args) def lookup_typeinfo(self, fullname: str) -> TypeInfo: # Assume that the name refers to a class. sym = self.lookup_qualified(fullname) - return cast(TypeInfo, sym.node) + node = sym.node + assert isinstance(node, TypeInfo) + return node def type_type(self) -> Instance: """Return instance type 'type'.""" - return self.named_type('builtins.type') - - def object_type(self) -> Instance: - """Return instance type 'object'.""" - return self.named_type('builtins.object') - - def bool_type(self) -> Instance: - """Return instance type 'bool'.""" - return self.named_type('builtins.bool') + return self.named_type("builtins.type") def str_type(self) -> Instance: """Return instance type 'str'.""" - return self.named_type('builtins.str') + return self.named_type("builtins.str") - def check_type_equivalency(self, t1: Type, t2: Type, node: Context, - msg: str = messages.INCOMPATIBLE_TYPES) -> None: - """Generate an error if the types are not equivalent. The - dynamic type is equivalent with all types. - """ - if not is_equivalent(t1, t2): - self.fail(msg, node) - - def store_type(self, node: Node, typ: Type) -> None: + def store_type(self, node: Expression, typ: Type) -> None: """Store the type of a node in the type map.""" - self.type_map[node] = typ + self._type_maps[-1][node] = typ - def is_dynamic_function(self) -> bool: - return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] + def has_type(self, node: Expression) -> bool: + return any(node in m for m in reversed(self._type_maps)) - def lookup(self, name: str, kind: int) -> SymbolTableNode: - """Look up a definition from the symbol table with the given name. - TODO remove kind argument + def lookup_type_or_none(self, node: Expression) -> Type | None: + for m in reversed(self._type_maps): + if node in m: + return m[node] + return None + + def lookup_type(self, node: Expression) -> Type: + for m in reversed(self._type_maps): + t = m.get(node) + if t is not None: + return t + raise KeyError(node) + + def store_types(self, d: dict[Expression, Type]) -> None: + self._type_maps[-1].update(d) + + @contextmanager + def local_type_map(self) -> Iterator[dict[Expression, Type]]: + """Store inferred types into a temporary type map (returned). + + This can be used to perform type checking "experiments" without + affecting exported types (which are used by mypyc). + """ + temp_type_map: dict[Expression, Type] = {} + self._type_maps.append(temp_type_map) + yield temp_type_map + self._type_maps.pop() + + def in_checked_function(self) -> bool: + """Should we type-check the current function? + + - Yes if --check-untyped-defs is set. + - Yes outside functions. + - Yes in annotated functions. + - No otherwise. """ - if self.locals is not None and name in self.locals: - return self.locals[name] - elif name in self.globals: + return ( + self.options.check_untyped_defs or not self.dynamic_funcs or not self.dynamic_funcs[-1] + ) + + def lookup(self, name: str) -> SymbolTableNode: + """Look up a definition from the symbol table with the given name.""" + if name in self.globals: return self.globals[name] else: - b = self.globals.get('__builtins__', None) + b = self.globals.get("__builtins__", None) if b: - table = cast(MypyFile, b.node).names + assert isinstance(b.node, MypyFile) + table = b.node.names if name in table: return table[name] - raise KeyError('Failed lookup: {}'.format(name)) + raise KeyError(f"Failed lookup: {name}") def lookup_qualified(self, name: str) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, GDEF) # FIX kind + if "." not in name: + return self.lookup(name) else: - parts = name.split('.') + parts = name.split(".") n = self.modules[parts[0]] for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names.get(parts[i], None).node) - return n.names[parts[-1]] + sym = n.names.get(parts[i]) + assert sym is not None, "Internal error: attempted lookup of unknown name" + assert isinstance(sym.node, MypyFile) + n = sym.node + last = parts[-1] + if last in n.names: + return n.names[last] + elif len(parts) == 2 and parts[0] in ("builtins", "typing"): + fullname = ".".join(parts) + if fullname in SUGGESTED_TEST_FIXTURES: + suggestion = ", e.g. add '[{} fixtures/{}]' to your test".format( + parts[0], SUGGESTED_TEST_FIXTURES[fullname] + ) + else: + suggestion = "" + raise KeyError( + "Could not find builtin symbol '{}' (If you are running a " + "test case, use a fixture that " + "defines this symbol{})".format(last, suggestion) + ) + else: + msg = "Failed qualified lookup: '{}' (fullname = '{}')." + raise KeyError(msg.format(last, name)) - def enter(self) -> None: - self.locals = SymbolTable() + @contextmanager + def enter_partial_types( + self, *, is_function: bool = False, is_class: bool = False + ) -> Iterator[None]: + """Enter a new scope for collecting partial types. + + Also report errors for (some) variables which still have partial + types, i.e. we couldn't infer a complete type. + """ + is_local = (self.partial_types and self.partial_types[-1].is_local) or is_function + self.partial_types.append(PartialTypeScope({}, is_function, is_local)) + yield + + # Don't complain about not being able to infer partials if it is + # at the toplevel (with allow_untyped_globals) or if it is in an + # untyped function being checked with check_untyped_defs. + permissive = (self.options.allow_untyped_globals and not is_local) or ( + self.options.check_untyped_defs and self.dynamic_funcs and self.dynamic_funcs[-1] + ) + + partial_types, _, _ = self.partial_types.pop() + if not self.current_node_deferred: + for var, context in partial_types.items(): + # If we require local partial types, there are a few exceptions where + # we fall back to inferring just "None" as the type from a None initializer: + # + # 1. If all happens within a single function this is acceptable, since only + # the topmost function is a separate target in fine-grained incremental mode. + # We primarily want to avoid "splitting" partial types across targets. + # + # 2. A None initializer in the class body if the attribute is defined in a base + # class is fine, since the attribute is already defined and it's currently okay + # to vary the type of an attribute covariantly. The None type will still be + # checked for compatibility with base classes elsewhere. Without this exception + # mypy could require an annotation for an attribute that already has been + # declared in a base class, which would be bad. + allow_none = ( + not self.options.local_partial_types + or is_function + or (is_class and self.is_defined_in_base_class(var)) + ) + if ( + allow_none + and isinstance(var.type, PartialType) + and var.type.type is None + and not permissive + ): + var.type = NoneType() + else: + if var not in self.partial_reported and not permissive: + self.msg.need_annotation_for_var(var, context, self.options.python_version) + self.partial_reported.add(var) + if var.type: + fixed = fixup_partial_type(var.type) + var.invalid_partial_type = fixed != var.type + var.type = fixed + + def handle_partial_var_type( + self, typ: PartialType, is_lvalue: bool, node: Var, context: Context + ) -> Type: + """Handle a reference to a partial type through a var. + + (Used by checkexpr and checkmember.) + """ + in_scope, is_local, partial_types = self.find_partial_types_in_all_scopes(node) + if typ.type is None and in_scope: + # 'None' partial type. It has a well-defined type. In an lvalue context + # we want to preserve the knowledge of it being a partial type. + if not is_lvalue: + return NoneType() + else: + return typ + else: + if partial_types is not None and not self.current_node_deferred: + if in_scope: + context = partial_types[node] + if is_local or not self.options.allow_untyped_globals: + self.msg.need_annotation_for_var( + node, context, self.options.python_version + ) + self.partial_reported.add(node) + else: + # Defer the node -- we might get a better type in the outer scope + self.handle_cannot_determine_type(node.name, context) + return fixup_partial_type(typ) - def leave(self) -> None: - self.locals = None + def is_defined_in_base_class(self, var: Var) -> bool: + if not var.info: + return False + return var.info.fallback_to_any or any( + base.get(var.name) is not None for base in var.info.mro[1:] + ) - def is_within_function(self) -> bool: - """Are we currently type checking within a function? + def find_partial_types(self, var: Var) -> dict[Var, Context] | None: + """Look for an active partial type scope containing variable. - I.e. not at class body or at the top level. + A scope is active if assignments in the current context can refine a partial + type originally defined in the scope. This is affected by the local_partial_types + configuration option. """ - return self.return_types != [] + in_scope, _, partial_types = self.find_partial_types_in_all_scopes(var) + if in_scope: + return partial_types + return None - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if the type is Void.""" - if isinstance(typ, Void): - self.msg.does_not_return_value(typ, context) + def find_partial_types_in_all_scopes( + self, var: Var + ) -> tuple[bool, bool, dict[Var, Context] | None]: + """Look for partial type scope containing variable. - def temp_node(self, t: Type, context: Context = None) -> Node: + Return tuple (is the scope active, is the scope a local scope, scope). + """ + for scope in reversed(self.partial_types): + if var in scope.map: + # All scopes within the outermost function are active. Scopes out of + # the outermost function are inactive to allow local reasoning (important + # for fine-grained incremental mode). + disallow_other_scopes = self.options.local_partial_types + + if isinstance(var.type, PartialType) and var.type.type is not None and var.info: + # This is an ugly hack to make partial generic self attributes behave + # as if --local-partial-types is always on (because it used to be like this). + disallow_other_scopes = True + + scope_active = ( + not disallow_other_scopes or scope.is_local == self.partial_types[-1].is_local + ) + return scope_active, scope.is_local, scope.map + return False, False, None + + def temp_node(self, t: Type, context: Context | None = None) -> TempNode: """Create a temporary node with the given, fixed type.""" - temp = TempNode(t) - if context: - temp.set_line(context.get_line()) - return temp + return TempNode(t, context=context) - def fail(self, msg: str, context: Context) -> None: + def fail( + self, msg: str | ErrorMessage, context: Context, *, code: ErrorCode | None = None + ) -> None: """Produce an error message.""" - self.msg.fail(msg, context) - - def iterable_item_type(self, instance: Instance) -> Type: - iterable = map_instance_to_supertype( - instance, - self.lookup_typeinfo('typing.Iterable')) - return iterable.args[0] + if isinstance(msg, ErrorMessage): + self.msg.fail(msg.value, context, code=msg.code) + return + self.msg.fail(msg, context, code=code) + + def note( + self, + msg: str | ErrorMessage, + context: Context, + offset: int = 0, + *, + code: ErrorCode | None = None, + ) -> None: + """Produce a note.""" + if isinstance(msg, ErrorMessage): + self.msg.note(msg.value, context, code=msg.code) + return + self.msg.note(msg, context, offset=offset, code=code) + + def iterable_item_type( + self, it: Instance | CallableType | TypeType | Overloaded, context: Context + ) -> Type: + if isinstance(it, Instance): + iterable = map_instance_to_supertype(it, self.lookup_typeinfo("typing.Iterable")) + item_type = iterable.args[0] + if not isinstance(get_proper_type(item_type), AnyType): + # This relies on 'map_instance_to_supertype' returning 'Iterable[Any]' + # in case there is no explicit base class. + return item_type + # Try also structural typing. + return self.analyze_iterable_item_type_without_expression(it, context)[1] def function_type(self, func: FuncBase) -> FunctionLike: - return function_type(func, self.named_type('builtins.function')) + return function_type(func, self.named_type("builtins.function")) + + def push_type_map(self, type_map: TypeMap, *, from_assignment: bool = True) -> None: + if type_map is None: + self.binder.unreachable() + else: + for expr, type in type_map.items(): + self.binder.put(expr, type, from_assignment=from_assignment) + + def infer_issubclass_maps(self, node: CallExpr, expr: Expression) -> tuple[TypeMap, TypeMap]: + """Infer type restrictions for an expression in issubclass call.""" + vartype = self.lookup_type(expr) + type = self.get_isinstance_type(node.args[1]) + if isinstance(vartype, TypeVarType): + vartype = vartype.upper_bound + vartype = get_proper_type(vartype) + if isinstance(vartype, UnionType): + union_list = [] + for t in get_proper_types(vartype.items): + if isinstance(t, TypeType): + union_list.append(t.item) + else: + # This is an error that should be reported earlier + # if we reach here, we refuse to do any type inference. + return {}, {} + vartype = UnionType(union_list) + elif isinstance(vartype, TypeType): + vartype = vartype.item + elif isinstance(vartype, Instance) and vartype.type.is_metaclass(): + vartype = self.named_type("builtins.object") + else: + # Any other object whose type we don't know precisely + # for example, Any or a custom metaclass. + return {}, {} # unknown type + yes_type, no_type = self.conditional_types_with_intersection(vartype, type, expr) + yes_map, no_map = conditional_types_to_typemaps(expr, yes_type, no_type) + yes_map, no_map = map(convert_to_typetype, (yes_map, no_map)) + return yes_map, no_map + + @overload + def conditional_types_with_intersection( + self, + expr_type: Type, + type_ranges: list[TypeRange] | None, + ctx: Context, + default: None = None, + ) -> tuple[Type | None, Type | None]: ... + + @overload + def conditional_types_with_intersection( + self, expr_type: Type, type_ranges: list[TypeRange] | None, ctx: Context, default: Type + ) -> tuple[Type, Type]: ... + + def conditional_types_with_intersection( + self, + expr_type: Type, + type_ranges: list[TypeRange] | None, + ctx: Context, + default: Type | None = None, + ) -> tuple[Type | None, Type | None]: + initial_types = conditional_types(expr_type, type_ranges, default) + # For some reason, doing "yes_map, no_map = conditional_types_to_typemaps(...)" + # doesn't work: mypyc will decide that 'yes_map' is of type None if we try. + yes_type: Type | None = initial_types[0] + no_type: Type | None = initial_types[1] + + if not isinstance(get_proper_type(yes_type), UninhabitedType) or type_ranges is None: + return yes_type, no_type + + # If conditional_types was unable to successfully narrow the expr_type + # using the type_ranges and concluded if-branch is unreachable, we try + # computing it again using a different algorithm that tries to generate + # an ad-hoc intersection between the expr_type and the type_ranges. + proper_type = get_proper_type(expr_type) + if isinstance(proper_type, UnionType): + possible_expr_types = get_proper_types(proper_type.relevant_items()) + else: + possible_expr_types = [proper_type] + + possible_target_types = [] + for tr in type_ranges: + item = get_proper_type(tr.item) + if isinstance(item, (Instance, NoneType)): + possible_target_types.append(item) + if not possible_target_types: + return yes_type, no_type + + out = [] + errors: list[tuple[str, str]] = [] + for v in possible_expr_types: + if not isinstance(v, Instance): + return yes_type, no_type + for t in possible_target_types: + if isinstance(t, NoneType): + errors.append((f'"{v.type.name}" and "NoneType"', '"NoneType" is final')) + continue + intersection = self.intersect_instances((v, t), errors) + if intersection is None: + continue + out.append(intersection) + if not out: + # Only report errors if no element in the union worked. + if self.should_report_unreachable_issues(): + for types, reason in errors: + self.msg.impossible_intersection(types, reason, ctx) + return UninhabitedType(), expr_type + new_yes_type = make_simplified_union(out) + return new_yes_type, expr_type + + def is_writable_attribute(self, node: Node) -> bool: + """Check if an attribute is writable""" + if isinstance(node, Var): + if node.is_property and not node.is_settable_property: + return False + return True + elif isinstance(node, OverloadedFuncDef) and node.is_property: + first_item = node.items[0] + assert isinstance(first_item, Decorator) + return first_item.var.is_settable_property + return False - def method_type(self, func: FuncBase) -> FunctionLike: - return method_type(func, self.named_type('builtins.function')) + def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: + if isinstance(expr, OpExpr) and expr.op == "|": + left = self.get_isinstance_type(expr.left) + if left is None and is_literal_none(expr.left): + left = [TypeRange(NoneType(), is_upper_bound=False)] + right = self.get_isinstance_type(expr.right) + if right is None and is_literal_none(expr.right): + right = [TypeRange(NoneType(), is_upper_bound=False)] + if left is None or right is None: + return None + return left + right + all_types = get_proper_types(flatten_types(self.lookup_type(expr))) + types: list[TypeRange] = [] + for typ in all_types: + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + # Type variables may be present -- erase them, which is the best + # we can do (outside disallowing them here). + erased_type = erase_typevars(typ.items[0].ret_type) + types.append(TypeRange(erased_type, is_upper_bound=False)) + elif isinstance(typ, TypeType): + # Type[A] means "any type that is a subtype of A" rather than "precisely type A" + # we indicate this by setting is_upper_bound flag + is_upper_bound = True + if isinstance(typ.item, NoneType): + # except for Type[None], because "'NoneType' is not an acceptable base type" + is_upper_bound = False + types.append(TypeRange(typ.item, is_upper_bound=is_upper_bound)) + elif isinstance(typ, Instance) and typ.type.fullname == "builtins.type": + object_type = Instance(typ.type.mro[-1], []) + types.append(TypeRange(object_type, is_upper_bound=True)) + elif isinstance(typ, Instance) and typ.type.fullname == "types.UnionType" and typ.args: + types.append(TypeRange(UnionType(typ.args), is_upper_bound=False)) + elif isinstance(typ, AnyType): + types.append(TypeRange(typ, is_upper_bound=False)) + else: # we didn't see an actual type, but rather a variable with unknown value + return None + if not types: + # this can happen if someone has empty tuple as 2nd argument to isinstance + # strictly speaking, we should return UninhabitedType but for simplicity we will simply + # refuse to do any type inference for now + return None + return types + def is_literal_enum(self, n: Expression) -> bool: + """Returns true if this expression (with the given type context) is an Enum literal. -def map_type_from_supertype(typ: Type, sub_info: TypeInfo, - super_info: TypeInfo) -> Type: - """Map type variables in a type defined in a supertype context to be valid - in the subtype context. Assume that the result is unique; if more than - one type is possible, return one of the alternatives. + For example, if we had an enum: - For example, assume + class Foo(Enum): + A = 1 + B = 2 - . class D(Generic[S]) ... - . class C(D[E[T]], Generic[T]) ... + ...and if the expression 'Foo' referred to that enum within the current type context, + then the expression 'Foo.A' would be a literal enum. However, if we did 'a = Foo.A', + then the variable 'a' would *not* be a literal enum. - Now S in the context of D would be mapped to E[T] in the context of C. - """ - # Create the type of self in subtype, of form t[a1, ...]. - inst_type = self_type(sub_info) - if isinstance(inst_type, TupleType): - inst_type = inst_type.fallback - # Map the type of self to supertype. This gets us a description of the - # supertype type variables in terms of subtype variables, i.e. t[t1, ...] - # so that any type variables in tN are to be interpreted in subtype - # context. - inst_type = map_instance_to_supertype(inst_type, super_info) - # Finally expand the type variables in type with those in the previously - # constructed type. Note that both type and inst_type may have type - # variables, but in type they are interpreterd in supertype context while - # in inst_type they are interpreted in subtype context. This works even if - # the names of type variables in supertype and subtype overlap. - return expand_type_by_instance(typ, inst_type) - - -def get_undefined_tuple(rvalue: Node, tuple_type: Instance) -> Type: - """Get tuple type corresponding to a tuple of Undefined values. - - The type is Tuple[Any, ...]. If rvalue is not of the right form, return - None. - """ - if isinstance(rvalue, TupleExpr): - for item in rvalue.items: - if not refers_to_fullname(item, 'typing.Undefined'): - break + We occasionally special-case expressions like 'Foo.A' and treat them as a single primitive + unit for the same reasons we sometimes treat 'True', 'False', or 'None' as a single + primitive unit. + """ + if not isinstance(n, MemberExpr) or not isinstance(n.expr, NameExpr): + return False + + parent_type = self.lookup_type_or_none(n.expr) + member_type = self.lookup_type_or_none(n) + if member_type is None or parent_type is None: + return False + + parent_type = get_proper_type(parent_type) + member_type = get_proper_type(coerce_to_literal(member_type)) + if not isinstance(parent_type, FunctionLike) or not isinstance(member_type, LiteralType): + return False + + if not parent_type.is_type_obj(): + return False + + return ( + member_type.is_enum_literal() + and member_type.fallback.type == parent_type.type_object() + ) + + def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: + """Inject an extra attribute with Any type using fallbacks.""" + orig_typ = typ + typ = get_proper_type(typ) + any_type = AnyType(TypeOfAny.unannotated) + if isinstance(typ, Instance): + result = typ.copy_with_extra_attr(name, any_type) + # For instances, we erase the possible module name, so that restrictions + # become anonymous types.ModuleType instances, allowing hasattr() to + # have effect on modules. + assert result.extra_attrs is not None + result.extra_attrs.mod_name = None + return result + if isinstance(typ, TupleType): + fallback = typ.partial_fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, CallableType): + fallback = typ.fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, TypeType) and isinstance(typ.item, Instance): + return TypeType.make_normalized(self.add_any_attribute_to_type(typ.item, name)) + if isinstance(typ, TypeVarType): + return typ.copy_modified( + upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name), + values=[self.add_any_attribute_to_type(v, name) for v in typ.values], + ) + if isinstance(typ, UnionType): + with_attr, without_attr = self.partition_union_by_attr(typ, name) + return make_simplified_union( + with_attr + [self.add_any_attribute_to_type(typ, name) for typ in without_attr] + ) + return orig_typ + + def hasattr_type_maps( + self, expr: Expression, source_type: Type, name: str + ) -> tuple[TypeMap, TypeMap]: + """Simple support for hasattr() checks. + + Essentially the logic is following: + * In the if branch, keep types that already has a valid attribute as is, + for other inject an attribute with `Any` type. + * In the else branch, remove types that already have a valid attribute, + while keeping the rest. + """ + if self.has_valid_attribute(source_type, name): + return {expr: source_type}, {} + + source_type = get_proper_type(source_type) + if isinstance(source_type, UnionType): + _, without_attr = self.partition_union_by_attr(source_type, name) + yes_map = {expr: self.add_any_attribute_to_type(source_type, name)} + return yes_map, {expr: make_simplified_union(without_attr)} + + type_with_attr = self.add_any_attribute_to_type(source_type, name) + if type_with_attr != source_type: + return {expr: type_with_attr}, {} + return {}, {} + + def partition_union_by_attr( + self, source_type: UnionType, name: str + ) -> tuple[list[Type], list[Type]]: + with_attr = [] + without_attr = [] + for item in source_type.items: + if self.has_valid_attribute(item, name): + with_attr.append(item) + else: + without_attr.append(item) + return with_attr, without_attr + + def has_valid_attribute(self, typ: Type, name: str) -> bool: + p_typ = get_proper_type(typ) + if isinstance(p_typ, AnyType): + return False + if isinstance(p_typ, Instance) and p_typ.extra_attrs and p_typ.extra_attrs.mod_name: + # Presence of module_symbol_table means this check will skip ModuleType.__getattr__ + module_symbol_table = p_typ.type.names else: - return TupleType([AnyType()] * len(rvalue.items), tuple_type) - return None + module_symbol_table = None + with self.msg.filter_errors() as watcher: + analyze_member_access( + name, + typ, + TempNode(AnyType(TypeOfAny.special_form)), + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, + original_type=typ, + chk=self, + # This is not a real attribute lookup so don't mess with deferring nodes. + no_deferral=True, + module_symbol_table=module_symbol_table, + ) + return not watcher.has_new_errors() + + def get_expression_type(self, node: Expression, type_context: Type | None = None) -> Type: + return self.expr_checker.accept(node, type_context=type_context) + + def check_deprecated(self, node: Node | None, context: Context) -> None: + """Warn if deprecated and not directly imported with a `from` statement.""" + if isinstance(node, Decorator): + node = node.func + if isinstance(node, (FuncDef, OverloadedFuncDef, TypeInfo)) and ( + node.deprecated is not None + ): + for imp in self.tree.imports: + if isinstance(imp, ImportFrom) and any(node.name == n[0] for n in imp.names): + break + else: + self.warn_deprecated(node, context) + + def warn_deprecated(self, node: Node | None, context: Context) -> None: + """Warn if deprecated.""" + if isinstance(node, Decorator): + node = node.func + if ( + isinstance(node, (FuncDef, OverloadedFuncDef, TypeInfo)) + and ((deprecated := node.deprecated) is not None) + and not self.is_typeshed_stub + ): + warn = self.msg.note if self.options.report_deprecated_as_note else self.msg.fail + warn(deprecated, context, code=codes.DEPRECATED) + + def warn_deprecated_overload_item( + self, node: Node | None, context: Context, *, target: Type, selftype: Type | None = None + ) -> None: + """Warn if the overload item corresponding to the given callable is deprecated.""" + target = get_proper_type(target) + if isinstance(node, OverloadedFuncDef) and isinstance(target, CallableType): + for item in node.items: + if isinstance(item, Decorator) and isinstance( + candidate := item.func.type, CallableType + ): + if selftype is not None: + candidate = bind_self(candidate, selftype) + if candidate == target: + self.warn_deprecated(item.func, context) + + +class CollectArgTypeVarTypes(TypeTraverserVisitor): + """Collects the non-nested argument types in a set.""" + + def __init__(self) -> None: + self.arg_types: set[TypeVarType] = set() + + def visit_type_var(self, t: TypeVarType) -> None: + self.arg_types.add(t) + + +@overload +def conditional_types( + current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: None = None +) -> tuple[Type | None, Type | None]: ... + + +@overload +def conditional_types( + current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type +) -> tuple[Type, Type]: ... + + +def conditional_types( + current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type | None = None +) -> tuple[Type | None, Type | None]: + """Takes in the current type and a proposed type of an expression. + + Returns a 2-tuple: The first element is the proposed type, if the expression + can be the proposed type. The second element is the type it would hold + if it was not the proposed type, if any. UninhabitedType means unreachable. + None means no new information can be inferred. If default is set it is returned + instead.""" + if proposed_type_ranges: + if len(proposed_type_ranges) == 1: + target = proposed_type_ranges[0].item + target = get_proper_type(target) + if isinstance(target, LiteralType) and ( + target.is_enum_literal() or isinstance(target.value, bool) + ): + enum_name = target.fallback.type.fullname + current_type = try_expanding_sum_type_to_union(current_type, enum_name) + proposed_items = [type_range.item for type_range in proposed_type_ranges] + proposed_type = make_simplified_union(proposed_items) + if isinstance(proposed_type, AnyType): + # We don't really know much about the proposed type, so we shouldn't + # attempt to narrow anything. Instead, we broaden the expr to Any to + # avoid false positives + return proposed_type, default + elif not any( + type_range.is_upper_bound for type_range in proposed_type_ranges + ) and is_proper_subtype(current_type, proposed_type, ignore_promotions=True): + # Expression is always of one of the types in proposed_type_ranges + return default, UninhabitedType() + elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): + # Expression is never of any type in proposed_type_ranges + return UninhabitedType(), default + else: + # we can only restrict when the type is precise, not bounded + proposed_precise_type = UnionType.make_union( + [ + type_range.item + for type_range in proposed_type_ranges + if not type_range.is_upper_bound + ] + ) + remaining_type = restrict_subtype_away(current_type, proposed_precise_type) + return proposed_type, remaining_type + else: + # An isinstance check, but we don't understand the type + return current_type, default + + +def conditional_types_to_typemaps( + expr: Expression, yes_type: Type | None, no_type: Type | None +) -> tuple[TypeMap, TypeMap]: + expr = collapse_walrus(expr) + maps: list[TypeMap] = [] + for typ in (yes_type, no_type): + proper_type = get_proper_type(typ) + if isinstance(proper_type, UninhabitedType): + maps.append(None) + elif proper_type is None: + maps.append({}) + else: + assert typ is not None + maps.append({expr: typ}) + + return cast(Tuple[TypeMap, TypeMap], tuple(maps)) + + +def gen_unique_name(base: str, table: SymbolTable) -> str: + """Generate a name that does not appear in table by appending numbers to base.""" + if base not in table: + return base + i = 1 + while base + str(i) in table: + i += 1 + return base + str(i) + + +def is_true_literal(n: Expression) -> bool: + """Returns true if this expression is the 'True' literal/keyword.""" + return refers_to_fullname(n, "builtins.True") or isinstance(n, IntExpr) and n.value != 0 + + +def is_false_literal(n: Expression) -> bool: + """Returns true if this expression is the 'False' literal/keyword.""" + return refers_to_fullname(n, "builtins.False") or isinstance(n, IntExpr) and n.value == 0 -def find_isinstance_check(node: Node, - type_map: Dict[Node, Type]) -> Tuple[Node, Type, Type, int]: - """Check if node is an isinstance(variable, type) check. +def is_literal_none(n: Expression) -> bool: + """Returns true if this expression is the 'None' literal/keyword.""" + return isinstance(n, NameExpr) and n.fullname == "builtins.None" - If successful, return tuple (variable, target-type, else-type, - kind); otherwise, return (None, AnyType, AnyType, -1). - When successful, the kind takes one of these values: +def is_literal_not_implemented(n: Expression) -> bool: + return isinstance(n, NameExpr) and n.fullname == "builtins.NotImplemented" - ISINSTANCE_OVERLAPPING: The type of variable and the target type are - partially overlapping => the test result can be True or False. - ISINSTANCE_ALWAYS_TRUE: The target type at least as general as the - variable type => the test is always True. - ISINSTANCE_ALWAYS_FALSE: The target type and the variable type are not - overlapping => the test is always False. + +def _is_empty_generator_function(func: FuncItem) -> bool: """ - if isinstance(node, CallExpr): - if refers_to_fullname(node.callee, 'builtins.isinstance'): - expr = node.args[0] - if expr.literal == LITERAL_TYPE: - type = get_isinstance_type(node.args[1], type_map) - if type: - vartype = type_map[expr] - kind = ISINSTANCE_OVERLAPPING - elsetype = vartype - if vartype: - if is_proper_subtype(vartype, type): - kind = ISINSTANCE_ALWAYS_TRUE - elsetype = None - elif not is_overlapping_types(vartype, type): - kind = ISINSTANCE_ALWAYS_FALSE - else: - elsetype = restrict_subtype_away(vartype, type) - return expr, type, elsetype, kind - # Not a supported isinstance check - return None, AnyType(), AnyType(), -1 - - -def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: - type = type_map[node] - if isinstance(type, FunctionLike): - if type.is_type_obj(): - # Type variables may be present -- erase them, which is the best - # we can do (outside disallowing them here). - return erase_typevars(type.items()[0].ret_type) + Checks whether a function's body is 'return; yield' (the yield being added only + to promote the function into a generator function). + """ + body = func.body.body + return ( + len(body) == 2 + and isinstance(ret_stmt := body[0], ReturnStmt) + and (ret_stmt.expr is None or is_literal_none(ret_stmt.expr)) + and isinstance(expr_stmt := body[1], ExpressionStmt) + and isinstance(yield_expr := expr_stmt.expr, YieldExpr) + and (yield_expr.expr is None or is_literal_none(yield_expr.expr)) + ) + + +def builtin_item_type(tp: Type) -> Type | None: + """Get the item type of a builtin container. + + If 'tp' is not one of the built containers (these includes NamedTuple and TypedDict) + or if the container is not parameterized (like List or List[Any]) + return None. This function is used to narrow optional types in situations like this: + + x: Optional[int] + if x in (1, 2, 3): + x + 42 # OK + + Note: this is only OK for built-in containers, where we know the behavior + of __contains__. + """ + tp = get_proper_type(tp) + + if isinstance(tp, Instance): + if tp.type.fullname in [ + "builtins.list", + "builtins.tuple", + "builtins.dict", + "builtins.set", + "builtins.frozenset", + "_collections_abc.dict_keys", + "typing.KeysView", + ]: + if not tp.args: + # TODO: fix tuple in lib-stub/builtins.pyi (it should be generic). + return None + if not isinstance(get_proper_type(tp.args[0]), AnyType): + return tp.args[0] + elif isinstance(tp, TupleType): + normalized_items = [] + for it in tp.items: + # This use case is probably rare, but not handling unpacks here can cause crashes. + if isinstance(it, UnpackType): + unpacked = get_proper_type(it.type) + if isinstance(unpacked, TypeVarTupleType): + unpacked = get_proper_type(unpacked.upper_bound) + assert ( + isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + ) + normalized_items.append(unpacked.args[0]) + else: + normalized_items.append(it) + if all(not isinstance(it, AnyType) for it in get_proper_types(normalized_items)): + return make_simplified_union(normalized_items) # this type is not externally visible + elif isinstance(tp, TypedDictType): + # TypedDict always has non-optional string keys. Find the key type from the Mapping + # base class. + for base in tp.fallback.type.mro: + if base.fullname == "typing.Mapping": + return map_instance_to_supertype(tp.fallback, base).args[0] + assert False, "No Mapping base class found for TypedDict fallback" return None -def expand_node(defn: Node, map: Dict[int, Type]) -> Node: - visitor = TypeTransformVisitor(map) - return defn.accept(visitor) +def and_conditional_maps(m1: TypeMap, m2: TypeMap, use_meet: bool = False) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 and e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None or m2 is None: + # One of the conditions can never be true. + return None + # Both conditions can be true; combine the information. Anything + # we learn from either conditions' truth is valid. If the same + # expression's type is refined by both conditions, we somewhat + # arbitrarily give precedence to m2 unless m1 value is Any. + # In the future, we could use an intersection type or meet_types(). + result = m2.copy() + m2_keys = {literal_hash(n2) for n2 in m2} + for n1 in m1: + if literal_hash(n1) not in m2_keys or isinstance(get_proper_type(m1[n1]), AnyType): + result[n1] = m1[n1] + if use_meet: + # For now, meet common keys only if specifically requested. + # This is currently used for tuple types narrowing, where having + # a precise result is important. + for n1 in m1: + for n2 in m2: + if literal_hash(n1) == literal_hash(n2): + result[n1] = meet_types(m1[n1], m2[n2]) + return result + + +def or_conditional_maps(m1: TypeMap, m2: TypeMap, coalesce_any: bool = False) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 or e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. If coalesce_any is True, consider Any a supertype when + joining restrictions. + """ + + if m1 is None: + return m2 + if m2 is None: + return m1 + # Both conditions can be true. Combine information about + # expressions whose type is refined by both conditions. (We do not + # learn anything about expressions whose type is refined by only + # one condition.) + result: dict[Expression, Type] = {} + for n1 in m1: + for n2 in m2: + if literal_hash(n1) == literal_hash(n2): + if coalesce_any and isinstance(get_proper_type(m1[n1]), AnyType): + result[n1] = m1[n1] + else: + result[n1] = make_simplified_union([m1[n1], m2[n2]]) + return result + + +def reduce_conditional_maps( + type_maps: list[tuple[TypeMap, TypeMap]], use_meet: bool = False +) -> tuple[TypeMap, TypeMap]: + """Reduces a list containing pairs of if/else TypeMaps into a single pair. + + We "and" together all of the if TypeMaps and "or" together the else TypeMaps. So + for example, if we had the input: + + [ + ({x: TypeIfX, shared: TypeIfShared1}, {x: TypeElseX, shared: TypeElseShared1}), + ({y: TypeIfY, shared: TypeIfShared2}, {y: TypeElseY, shared: TypeElseShared2}), + ] + + ...we'd return the output: + ( + {x: TypeIfX, y: TypeIfY, shared: PseudoIntersection[TypeIfShared1, TypeIfShared2]}, + {shared: Union[TypeElseShared1, TypeElseShared2]}, + ) -def expand_func(defn: FuncItem, map: Dict[int, Type]) -> FuncItem: - return cast(FuncItem, expand_node(defn, map)) + ...where "PseudoIntersection[X, Y] == Y" because mypy actually doesn't understand intersections + yet, so we settle for just arbitrarily picking the right expr's type. + + We only retain the shared expression in the 'else' case because we don't actually know + whether x was refined or y was refined -- only just that one of the two was refined. + """ + if len(type_maps) == 0: + return {}, {} + elif len(type_maps) == 1: + return type_maps[0] + else: + final_if_map, final_else_map = type_maps[0] + for if_map, else_map in type_maps[1:]: + final_if_map = and_conditional_maps(final_if_map, if_map, use_meet=use_meet) + final_else_map = or_conditional_maps(final_else_map, else_map) + + return final_if_map, final_else_map + + +def convert_to_typetype(type_map: TypeMap) -> TypeMap: + converted_type_map: dict[Expression, Type] = {} + if type_map is None: + return None + for expr, typ in type_map.items(): + t = typ + if isinstance(t, TypeVarType): + t = t.upper_bound + # TODO: should we only allow unions of instances as per PEP 484? + if not isinstance(get_proper_type(t), (UnionType, Instance, NoneType)): + # unknown type; error was likely reported earlier + return {} + converted_type_map[expr] = TypeType.make_normalized(typ) + return converted_type_map + + +def flatten(t: Expression) -> list[Expression]: + """Flatten a nested sequence of tuples/lists into one list of nodes.""" + if isinstance(t, (TupleExpr, ListExpr)): + return [b for a in t.items for b in flatten(a)] + elif isinstance(t, StarExpr): + return flatten(t.expr) + else: + return [t] + + +def flatten_types(t: Type) -> list[Type]: + """Flatten a nested sequence of tuples into one list of nodes.""" + t = get_proper_type(t) + if isinstance(t, TupleType): + return [b for a in t.items for b in flatten_types(a)] + elif is_named_instance(t, "builtins.tuple"): + return [t.args[0]] + else: + return [t] + + +def expand_func(defn: FuncItem, map: dict[TypeVarId, Type]) -> FuncItem: + visitor = TypeTransformVisitor(map) + ret = visitor.node(defn) + assert isinstance(ret, FuncItem) + return ret class TypeTransformVisitor(TransformVisitor): - def __init__(self, map: Dict[int, Type]) -> None: + def __init__(self, map: dict[TypeVarId, Type]) -> None: super().__init__() self.map = map @@ -2046,101 +8061,775 @@ def type(self, type: Type) -> Type: return expand_type(type, self.map) -def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: - """Check if two signatures may be unsafely overlapping. +def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool: + """Can a single call match both t and s, based just on positional argument counts?""" + min_args = max(t.min_args, s.min_args) + max_args = min(t.max_possible_positional_args(), s.max_possible_positional_args()) + return min_args <= max_args + + +def expand_callable_variants(c: CallableType) -> list[CallableType]: + """Expand a generic callable using all combinations of type variables' values/bounds.""" + for tv in c.variables: + # We need to expand self-type before other variables, because this is the only + # type variable that can have other type variables in the upper bound. + if tv.id.is_self(): + c = expand_type(c, {tv.id: tv.upper_bound}).copy_modified( + variables=[v for v in c.variables if not v.id.is_self()] + ) + break + + if not c.is_generic(): + # Fast path. + return [c] + + tvar_values = [] + for tvar in c.variables: + if isinstance(tvar, TypeVarType) and tvar.values: + tvar_values.append(tvar.values) + else: + tvar_values.append([tvar.upper_bound]) + + variants = [] + for combination in itertools.product(*tvar_values): + tvar_map = {tv.id: subst for (tv, subst) in zip(c.variables, combination)} + variants.append(expand_type(c, tvar_map).copy_modified(variables=[])) + return variants + + +def is_unsafe_overlapping_overload_signatures( + signature: CallableType, + other: CallableType, + class_type_vars: list[TypeVarLikeType], + partial_only: bool = True, +) -> bool: + """Check if two overloaded signatures are unsafely overlapping or partially overlapping. + + We consider two functions 's' and 't' to be unsafely overlapping if three + conditions hold: + + 1. s's parameters are partially overlapping with t's. i.e. there are calls that are + valid for both signatures. + 2. for these common calls, some of t's parameters types are wider that s's. + 3. s's return type is NOT a subset of t's. + + Note that we use subset rather than subtype relationship in these checks because: + * Overload selection happens at runtime, not statically. + * This results in more lenient behavior. + This can cause false negatives (e.g. if overloaded function returns an externally + visible attribute with invariant type), but such situations are rare. In general, + overloads in Python are generally unsafe, so we intentionally try to avoid giving + non-actionable errors (see more details in comments below). + + Assumes that 'signature' appears earlier in the list of overload + alternatives then 'other' and that their argument counts are overlapping. + """ + # Try detaching callables from the containing class so that all TypeVars + # are treated as being free, i.e. the signature is as seen from inside the class, + # where "self" is not yet bound to anything. + signature = detach_callable(signature, class_type_vars) + other = detach_callable(other, class_type_vars) + + # Note: We repeat this check twice in both directions compensate for slight + # asymmetries in 'is_callable_compatible'. + + for sig_variant in expand_callable_variants(signature): + for other_variant in expand_callable_variants(other): + # Using only expanded callables may cause false negatives, we can add + # more variants (e.g. using inference between callables) in the future. + if is_subset_no_promote(sig_variant.ret_type, other_variant.ret_type): + continue + if not ( + is_callable_compatible( + sig_variant, + other_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=False, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(l, r), + allow_partial_overlap=True, + ) + or is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=True, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(r, l), + allow_partial_overlap=True, + ) + ): + continue + # Using the same `allow_partial_overlap` flag as before, can cause false + # negatives in case where star argument is used in a catch-all fallback overload. + # But again, practicality beats purity here. + if not partial_only or not is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_subset_no_promote, + check_args_covariantly=True, + is_proper_subtype=False, + ignore_return=True, + allow_partial_overlap=True, + ): + return True + return False + + +def detach_callable(typ: CallableType, class_type_vars: list[TypeVarLikeType]) -> CallableType: + """Ensures that the callable's type variables are 'detached' and independent of the context. - Two signatures s and t are overlapping if both can be valid for the same - statically typed values and the return types are incompatible. + A callable normally keeps track of the type variables it uses within its 'variables' field. + However, if the callable is from a method and that method is using a class type variable, + the callable will not keep track of that type variable since it belongs to the class. + """ + if not class_type_vars: + # Fast path, nothing to update. + return typ + return typ.copy_modified(variables=list(typ.variables) + class_type_vars) + + +def overload_can_never_match(signature: CallableType, other: CallableType) -> bool: + """Check if the 'other' method can never be matched due to 'signature'. - Assume calls are first checked against 'signature', then against 'other'. - Thus if 'signature' is more general than 'other', there is no unsafe - overlapping. + This can happen if signature's parameters are all strictly broader then + other's parameters. - TODO If argument types vary covariantly, the return type may vary - covariantly as well. + Assumes that both signatures have overlapping argument counts. """ - if isinstance(signature, Callable): - if isinstance(other, Callable): - # TODO varargs - # TODO keyword args - # TODO erasure - # TODO allow to vary covariantly - # Check if the argument counts are overlapping. - min_args = max(signature.min_args, other.min_args) - max_args = min(len(signature.arg_types), len(other.arg_types)) - if min_args > max_args: - # Argument counts are not overlapping. - return False - # Signatures are overlapping iff if they are overlapping for the - # smallest common argument count. - for i in range(min_args): - t1 = signature.arg_types[i] - t2 = other.arg_types[i] - if not is_overlapping_types(t1, t2): - return False - # All arguments types for the smallest common argument count are - # overlapping => the signature is overlapping. The overlapping is - # safe if the return types are identical. - if is_same_type(signature.ret_type, other.ret_type): - return False - # If the first signature has more general argument types, the - # latter will never be called - if is_more_general_arg_prefix(signature, other): - return False - return not is_more_precise_signature(signature, other) - return True + # The extra erasure is needed to prevent spurious errors + # in situations where an `Any` overload is used as a fallback + # for an overload with type variables. The spurious error appears + # because the type variables turn into `Any` during unification in + # the below subtype check and (surprisingly?) `is_proper_subtype(Any, Any)` + # returns `True`. + # TODO: find a cleaner solution instead of this ad-hoc erasure. + exp_signature = expand_type( + signature, {tvar.id: erase_def_to_union_or_bound(tvar) for tvar in signature.variables} + ) + return is_callable_compatible( + exp_signature, other, is_compat=is_more_precise, is_proper_subtype=True, ignore_return=True + ) def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: """Does t have wider arguments than s?""" # TODO should an overload with additional items be allowed to be more # general than one with fewer items (or just one item)? - # TODO check argument kinds - if isinstance(t, Callable): - if isinstance(s, Callable): - return all(is_proper_subtype(args, argt) - for argt, args in zip(t.arg_types, s.arg_types)) + if isinstance(t, CallableType): + if isinstance(s, CallableType): + return is_callable_compatible( + t, s, is_compat=is_proper_subtype, is_proper_subtype=True, ignore_return=True + ) elif isinstance(t, FunctionLike): if isinstance(s, FunctionLike): - if len(t.items()) == len(s.items()): - return all(is_same_arg_prefix(items, itemt) - for items, itemt in zip(t.items(), s.items())) + if len(t.items) == len(s.items): + return all( + is_same_arg_prefix(items, itemt) for items, itemt in zip(t.items, s.items) + ) return False -def is_same_arg_prefix(t: Callable, s: Callable) -> bool: - # TODO check argument kinds - return all(is_same_type(argt, args) - for argt, args in zip(t.arg_types, s.arg_types)) +def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: + return is_callable_compatible( + t, + s, + is_compat=is_same_type, + is_proper_subtype=True, + ignore_return=True, + check_args_covariantly=True, + ignore_pos_arg_names=True, + ) + + +def infer_operator_assignment_method(typ: Type, operator: str) -> tuple[bool, str]: + """Determine if operator assignment on given value type is in-place, and the method name. + + For example, if operator is '+', return (True, '__iadd__') or (False, '__add__') + depending on which method is supported by the type. + """ + typ = get_proper_type(typ) + method = operators.op_methods[operator] + existing_method = None + if isinstance(typ, Instance): + existing_method = _find_inplace_method(typ, method, operator) + elif isinstance(typ, TypedDictType): + existing_method = _find_inplace_method(typ.fallback, method, operator) + + if existing_method is not None: + return True, existing_method + return False, method + + +def _find_inplace_method(inst: Instance, method: str, operator: str) -> str | None: + if operator in operators.ops_with_inplace_method: + inplace_method = "__i" + method[2:] + if inst.type.has_readable_member(inplace_method): + return inplace_method + return None + + +def is_valid_inferred_type(typ: Type, is_lvalue_final: bool = False) -> bool: + """Is an inferred type valid and needs no further refinement? + + Examples of invalid types include the None type (when we are not assigning + None to a final lvalue) or List[]. + + When not doing strict Optional checking, all types containing None are + invalid. When doing strict Optional checking, only None and types that are + incompletely defined (i.e. contain UninhabitedType) are invalid. + """ + proper_type = get_proper_type(typ) + if isinstance(proper_type, NoneType): + # If the lvalue is final, we may immediately infer NoneType when the + # initializer is None. + # + # If not, we want to defer making this decision. The final inferred + # type could either be NoneType or an Optional type, depending on + # the context. This resolution happens in leave_partial_types when + # we pop a partial types scope. + return is_lvalue_final + elif isinstance(proper_type, UninhabitedType): + return False + return not typ.accept(InvalidInferredTypes()) + + +class InvalidInferredTypes(BoolTypeQuery): + """Find type components that are not valid for an inferred type. + + These include type, and any uninhabited types resulting from failed + (ambiguous) type inference. + """ + + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return t.ambiguous + + def visit_erased_type(self, t: ErasedType) -> bool: + # This can happen inside a lambda. + return True + + def visit_type_var(self, t: TypeVarType) -> bool: + # This is needed to prevent leaking into partial types during + # multi-step type inference. + return t.id.is_meta_var() + + +class SetNothingToAny(TypeTranslator): + """Replace all ambiguous Uninhabited types with Any (to avoid spurious extra errors).""" + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if t.ambiguous: + return AnyType(TypeOfAny.from_error) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # Target of the alias cannot be an ambiguous UninhabitedType, so we just + # replace the arguments. + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + +def is_node_static(node: Node | None) -> bool | None: + """Find out if a node describes a static function method.""" + + if isinstance(node, FuncDef): + return node.is_static + + if isinstance(node, Var): + return node.is_staticmethod + + return None + + +class CheckerScope: + # We keep two stacks combined, to maintain the relative order + stack: list[TypeInfo | FuncItem | MypyFile] + def __init__(self, module: MypyFile) -> None: + self.stack = [module] -def is_more_precise_signature(t: Callable, s: Callable) -> bool: - """Is t more precise than s? + def top_function(self) -> FuncItem | None: + for e in reversed(self.stack): + if isinstance(e, FuncItem): + return e + return None + + def top_non_lambda_function(self) -> FuncItem | None: + for e in reversed(self.stack): + if isinstance(e, FuncItem) and not isinstance(e, LambdaExpr): + return e + return None + + def active_class(self) -> TypeInfo | None: + if isinstance(self.stack[-1], TypeInfo): + return self.stack[-1] + return None + + def enclosing_class(self) -> TypeInfo | None: + """Is there a class *directly* enclosing this function?""" + top = self.top_function() + assert top, "This method must be called from inside a function" + index = self.stack.index(top) + assert index, "CheckerScope stack must always start with a module" + enclosing = self.stack[index - 1] + if isinstance(enclosing, TypeInfo): + return enclosing + return None + + def active_self_type(self) -> Instance | TupleType | None: + """An instance or tuple type representing the current class. + + This returns None unless we are in class body or in a method. + In particular, inside a function nested in method this returns None. + """ + info = self.active_class() + if not info and self.top_function(): + info = self.enclosing_class() + if info: + return fill_typevars(info) + return None + + @contextmanager + def push_function(self, item: FuncItem) -> Iterator[None]: + self.stack.append(item) + yield + self.stack.pop() + + @contextmanager + def push_class(self, info: TypeInfo) -> Iterator[None]: + self.stack.append(info) + yield + self.stack.pop() + + +TKey = TypeVar("TKey") +TValue = TypeVar("TValue") + + +class DisjointDict(Generic[TKey, TValue]): + """An variation of the union-find algorithm/data structure where instead of keeping + track of just disjoint sets, we keep track of disjoint dicts -- keep track of multiple + Set[Key] -> Set[Value] mappings, where each mapping's keys are guaranteed to be disjoint. + + This data structure is currently used exclusively by 'group_comparison_operands' below + to merge chains of '==' and 'is' comparisons when two or more chains use the same expression + in best-case O(n), where n is the number of operands. + + Specifically, the `add_mapping()` function and `items()` functions will take on average + O(k + v) and O(n) respectively, where k and v are the number of keys and values we're adding + for a given chain. Note that k <= n and v <= n. + + We hit these average/best-case scenarios for most user code: e.g. when the user has just + a single chain like 'a == b == c == d == ...' or multiple disjoint chains like + 'a==b < c==d < e==f < ...'. (Note that a naive iterative merging would be O(n^2) for + the latter case). + + In comparison, this data structure will make 'group_comparison_operands' have a worst-case + runtime of O(n*log(n)): 'add_mapping()' and 'items()' are worst-case O(k*log(n) + v) and + O(k*log(n)) respectively. This happens only in the rare case where the user keeps repeatedly + making disjoint mappings before merging them in a way that persistently dodges the path + compression optimization in '_lookup_root_id', which would end up constructing a single + tree of height log_2(n). This makes root lookups no longer amoritized constant time when we + finally call 'items()'. + """ + + def __init__(self) -> None: + # Each key maps to a unique ID + self._key_to_id: dict[TKey, int] = {} + + # Each id points to the parent id, forming a forest of upwards-pointing trees. If the + # current id already is the root, it points to itself. We gradually flatten these trees + # as we perform root lookups: eventually all nodes point directly to its root. + self._id_to_parent_id: dict[int, int] = {} + + # Each root id in turn maps to the set of values. + self._root_id_to_values: dict[int, set[TValue]] = {} + + def add_mapping(self, keys: set[TKey], values: set[TValue]) -> None: + """Adds a 'Set[TKey] -> Set[TValue]' mapping. If there already exists a mapping + containing one or more of the given keys, we merge the input mapping with the old one. + + Note that the given set of keys must be non-empty -- otherwise, nothing happens. + """ + if not keys: + return + + subtree_roots = [self._lookup_or_make_root_id(key) for key in keys] + new_root = subtree_roots[0] + + root_values = self._root_id_to_values[new_root] + root_values.update(values) + for subtree_root in subtree_roots[1:]: + if subtree_root == new_root or subtree_root not in self._root_id_to_values: + continue + self._id_to_parent_id[subtree_root] = new_root + root_values.update(self._root_id_to_values.pop(subtree_root)) + + def items(self) -> list[tuple[set[TKey], set[TValue]]]: + """Returns all disjoint mappings in key-value pairs.""" + root_id_to_keys: dict[int, set[TKey]] = {} + for key in self._key_to_id: + root_id = self._lookup_root_id(key) + if root_id not in root_id_to_keys: + root_id_to_keys[root_id] = set() + root_id_to_keys[root_id].add(key) + + output = [] + for root_id, keys in root_id_to_keys.items(): + output.append((keys, self._root_id_to_values[root_id])) + + return output + + def _lookup_or_make_root_id(self, key: TKey) -> int: + if key in self._key_to_id: + return self._lookup_root_id(key) + else: + new_id = len(self._key_to_id) + self._key_to_id[key] = new_id + self._id_to_parent_id[new_id] = new_id + self._root_id_to_values[new_id] = set() + return new_id + + def _lookup_root_id(self, key: TKey) -> int: + i = self._key_to_id[key] + while i != self._id_to_parent_id[i]: + # Optimization: make keys directly point to their grandparents to speed up + # future traversals. This prevents degenerate trees of height n from forming. + new_parent = self._id_to_parent_id[self._id_to_parent_id[i]] + self._id_to_parent_id[i] = new_parent + i = new_parent + return i + + +def group_comparison_operands( + pairwise_comparisons: Iterable[tuple[str, Expression, Expression]], + operand_to_literal_hash: Mapping[int, Key], + operators_to_group: set[str], +) -> list[tuple[str, list[int]]]: + """Group a series of comparison operands together chained by any operand + in the 'operators_to_group' set. All other pairwise operands are kept in + groups of size 2. + + For example, suppose we have the input comparison expression: + + x0 == x1 == x2 < x3 < x4 is x5 is x6 is not x7 is not x8 + + If we get these expressions in a pairwise way (e.g. by calling ComparisionExpr's + 'pairwise()' method), we get the following as input: + + [('==', x0, x1), ('==', x1, x2), ('<', x2, x3), ('<', x3, x4), + ('is', x4, x5), ('is', x5, x6), ('is not', x6, x7), ('is not', x7, x8)] + + If `operators_to_group` is the set {'==', 'is'}, this function will produce + the following "simplified operator list": + + [("==", [0, 1, 2]), ("<", [2, 3]), ("<", [3, 4]), + ("is", [4, 5, 6]), ("is not", [6, 7]), ("is not", [7, 8])] + + Note that (a) we yield *indices* to the operands rather then the operand + expressions themselves and that (b) operands used in a consecutive chain + of '==' or 'is' are grouped together. + + If two of these chains happen to contain operands with the same underlying + literal hash (e.g. are assignable and correspond to the same expression), + we combine those chains together. For example, if we had: + + same == x < y == same + + ...and if 'operand_to_literal_hash' contained the same values for the indices + 0 and 3, we'd produce the following output: + + [("==", [0, 1, 2, 3]), ("<", [1, 2])] + + But if the 'operand_to_literal_hash' did *not* contain an entry, we'd instead + default to returning: - A signature t is more precise than s if all argument types and the return - type of t are more precise than the corresponding types in s. + [("==", [0, 1]), ("<", [1, 2]), ("==", [2, 3])] - Assume that the argument kinds and names are compatible, and that the - argument counts are overlapping. + This function is currently only used to assist with type-narrowing refinements + and is extracted out to a helper function so we can unit test it. """ - # TODO generic function types - # Only consider the common prefix of argument types. - for argt, args in zip(t.arg_types, s.arg_types): - if not is_more_precise(argt, args): + groups: dict[str, DisjointDict[Key, int]] = {op: DisjointDict() for op in operators_to_group} + + simplified_operator_list: list[tuple[str, list[int]]] = [] + last_operator: str | None = None + current_indices: set[int] = set() + current_hashes: set[Key] = set() + for i, (operator, left_expr, right_expr) in enumerate(pairwise_comparisons): + if last_operator is None: + last_operator = operator + + if current_indices and (operator != last_operator or operator not in operators_to_group): + # If some of the operands in the chain are assignable, defer adding it: we might + # end up needing to merge it with other chains that appear later. + if not current_hashes: + simplified_operator_list.append((last_operator, sorted(current_indices))) + else: + groups[last_operator].add_mapping(current_hashes, current_indices) + last_operator = operator + current_indices = set() + current_hashes = set() + + # Note: 'i' corresponds to the left operand index, so 'i + 1' is the + # right operand. + current_indices.add(i) + current_indices.add(i + 1) + + # We only ever want to combine operands/combine chains for these operators + if operator in operators_to_group: + left_hash = operand_to_literal_hash.get(i) + if left_hash is not None: + current_hashes.add(left_hash) + right_hash = operand_to_literal_hash.get(i + 1) + if right_hash is not None: + current_hashes.add(right_hash) + + if last_operator is not None: + if not current_hashes: + simplified_operator_list.append((last_operator, sorted(current_indices))) + else: + groups[last_operator].add_mapping(current_hashes, current_indices) + + # Now that we know which chains happen to contain the same underlying expressions + # and can be merged together, add in this info back to the output. + for operator, disjoint_dict in groups.items(): + for keys, indices in disjoint_dict.items(): + simplified_operator_list.append((operator, sorted(indices))) + + # For stability, reorder list by the first operand index to appear + simplified_operator_list.sort(key=lambda item: item[1][0]) + return simplified_operator_list + + +def is_typed_callable(c: Type | None) -> bool: + c = get_proper_type(c) + if not c or not isinstance(c, CallableType): + return False + return not all( + isinstance(t, AnyType) and t.type_of_any == TypeOfAny.unannotated + for t in get_proper_types(c.arg_types + [c.ret_type]) + ) + + +def is_untyped_decorator(typ: Type | None) -> bool: + typ = get_proper_type(typ) + if not typ: + return True + elif isinstance(typ, CallableType): + return not is_typed_callable(typ) + elif isinstance(typ, Instance): + method = typ.type.get_method("__call__") + if method: + if isinstance(method, Decorator): + return is_untyped_decorator(method.func.type) or is_untyped_decorator( + method.var.type + ) + + if isinstance(method.type, Overloaded): + return any(is_untyped_decorator(item) for item in method.type.items) + else: + return not is_typed_callable(method.type) + else: return False - return is_more_precise(t.ret_type, s.ret_type) + elif isinstance(typ, Overloaded): + return any(is_untyped_decorator(item) for item in typ.items) + return True + + +def is_static(func: FuncBase | Decorator) -> bool: + if isinstance(func, Decorator): + return is_static(func.func) + elif isinstance(func, FuncBase): + return func.is_static + assert False, f"Unexpected func type: {type(func)}" + + +def is_property(defn: SymbolNode) -> bool: + if isinstance(defn, Decorator): + return defn.func.is_property + if isinstance(defn, OverloadedFuncDef): + if defn.items and isinstance(defn.items[0], Decorator): + return defn.items[0].func.is_property + return False + + +def get_property_type(t: ProperType) -> ProperType: + if isinstance(t, CallableType): + return get_proper_type(t.ret_type) + if isinstance(t, Overloaded): + return get_proper_type(t.items[0].ret_type) + return t + + +def is_subset_no_promote(left: Type, right: Type) -> bool: + return is_subtype(left, right, ignore_promotions=True, always_covariant=True) + + +def is_overlapping_types_for_overload(left: Type, right: Type) -> bool: + # Note that among other effects 'overlap_for_overloads' flag will effectively + # ignore possible overlap between type variables and None. This is technically + # unsafe, but unsafety is tiny and this prevents some common use cases like: + # @overload + # def foo(x: None) -> None: .. + # @overload + # def foo(x: T) -> Foo[T]: ... + return is_overlapping_types( + left, + right, + ignore_promotions=True, + prohibit_none_typevar_overlap=True, + overlap_for_overloads=True, + ) + +def is_private(node_name: str) -> bool: + """Check if node is private to class definition.""" + return node_name.startswith("__") and not node_name.endswith("__") -def infer_operator_assignment_method(type: Type, operator: str) -> str: - """Return the method used for operator assignment for given value type. - For example, if operator is '+', return '__iadd__' or '__add__' depending - on which method is supported by the type. +def is_string_literal(typ: Type) -> bool: + strs = try_getting_str_literals_from_type(typ) + return strs is not None and len(strs) == 1 + + +def has_bool_item(typ: ProperType) -> bool: + """Return True if type is 'bool' or a union with a 'bool' item.""" + if is_named_instance(typ, "builtins.bool"): + return True + if isinstance(typ, UnionType): + return any(is_named_instance(item, "builtins.bool") for item in typ.items) + return False + + +def collapse_walrus(e: Expression) -> Expression: + """If an expression is an AssignmentExpr, pull out the assignment target. + + We don't make any attempt to pull out all the targets in code like `x := (y := z)`. + We could support narrowing those if that sort of code turns out to be common. + """ + if isinstance(e, AssignmentExpr): + return e.target + return e + + +def find_last_var_assignment_line(n: Node, v: Var) -> int: + """Find the highest line number of a potential assignment to variable within node. + + This supports local and global variables. + + Return -1 if no assignment was found. """ - method = nodes.op_methods[operator] - if isinstance(type, Instance): - if operator in nodes.ops_with_inplace_method: - inplace = '__i' + method[2:] - if type.type.has_readable_member(inplace): - method = inplace - return method + visitor = VarAssignVisitor(v) + n.accept(visitor) + return visitor.last_line + + +class VarAssignVisitor(TraverserVisitor): + def __init__(self, v: Var) -> None: + self.last_line = -1 + self.lvalue = False + self.var_node = v + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.lvalue = True + for lv in s.lvalues: + lv.accept(self) + self.lvalue = False + + def visit_name_expr(self, e: NameExpr) -> None: + if self.lvalue and e.node is self.var_node: + self.last_line = max(self.last_line, e.line) + + def visit_member_expr(self, e: MemberExpr) -> None: + old_lvalue = self.lvalue + self.lvalue = False + super().visit_member_expr(e) + self.lvalue = old_lvalue + + def visit_index_expr(self, e: IndexExpr) -> None: + old_lvalue = self.lvalue + self.lvalue = False + super().visit_index_expr(e) + self.lvalue = old_lvalue + + def visit_with_stmt(self, s: WithStmt) -> None: + self.lvalue = True + for lv in s.target: + if lv is not None: + lv.accept(self) + self.lvalue = False + s.body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.lvalue = True + s.index.accept(self) + self.lvalue = False + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_assignment_expr(self, e: AssignmentExpr) -> None: + self.lvalue = True + e.target.accept(self) + self.lvalue = False + e.value.accept(self) + + def visit_as_pattern(self, p: AsPattern) -> None: + if p.pattern is not None: + p.pattern.accept(self) + if p.name is not None: + self.lvalue = True + p.name.accept(self) + self.lvalue = False + + def visit_starred_pattern(self, p: StarredPattern) -> None: + if p.capture is not None: + self.lvalue = True + p.capture.accept(self) + self.lvalue = False + + +def is_ambiguous_mix_of_enums(types: list[Type]) -> bool: + """Do types have IntEnum/StrEnum types that are potentially overlapping with other types? + + If True, we shouldn't attempt type narrowing based on enum values, as it gets + too ambiguous. + + For example, return True if there's an 'int' type together with an IntEnum literal. + However, IntEnum together with a literal of the same IntEnum type is not ambiguous. + """ + # We need these things for this to be ambiguous: + # (1) an IntEnum or StrEnum type + # (2) either a different IntEnum/StrEnum type or a non-enum type ("") + # + # It would be slightly more correct to calculate this separately for IntEnum and + # StrEnum related types, as an IntEnum can't be confused with a StrEnum. + return len(_ambiguous_enum_variants(types)) > 1 + + +def _ambiguous_enum_variants(types: list[Type]) -> set[str]: + result = set() + for t in types: + t = get_proper_type(t) + if isinstance(t, UnionType): + result.update(_ambiguous_enum_variants(t.items)) + elif isinstance(t, Instance): + if t.last_known_value: + result.update(_ambiguous_enum_variants([t.last_known_value])) + elif t.type.is_enum and any( + base.fullname in ("enum.IntEnum", "enum.StrEnum") for base in t.type.mro + ): + result.add(t.type.fullname) + elif not t.type.is_enum: + # These might compare equal to IntEnum/StrEnum types (e.g. Decimal), so + # let's be conservative + result.add("") + elif isinstance(t, LiteralType): + result.update(_ambiguous_enum_variants([t.fallback])) + elif isinstance(t, NoneType): + pass + else: + result.add("") + return result diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index d9c76487ccc3..34004b5e3b48 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1,372 +1,2205 @@ """Expression type checker. This file is conceptually part of TypeChecker.""" -from typing import Undefined, cast, List, Tuple, Dict, Function, Union +from __future__ import annotations -from mypy.types import ( - Type, AnyType, Callable, Overloaded, NoneTyp, Void, TypeVarDef, - TupleType, Instance, TypeVar, TypeTranslator, ErasedType, FunctionLike, UnionType +import enum +import itertools +import time +from collections import defaultdict +from contextlib import contextmanager +from typing import Callable, ClassVar, Final, Iterable, Iterator, List, Optional, Sequence, cast +from typing_extensions import TypeAlias as _TypeAlias, assert_never, overload + +import mypy.checker +import mypy.errorcodes as codes +from mypy import applytype, erasetype, join, message_registry, nodes, operators, types +from mypy.argmap import ArgTypeExpander, map_actuals_to_formals, map_formals_to_actuals +from mypy.checkmember import analyze_member_access, freeze_all_type_vars, type_object_type +from mypy.checkstrformat import StringFormatterChecker +from mypy.erasetype import erase_type, remove_instance_last_known_values, replace_meta_vars +from mypy.errors import ErrorWatcher, report_internal_error +from mypy.expandtype import ( + expand_type, + expand_type_by_instance, + freshen_all_functions_type_vars, + freshen_function_type_vars, ) +from mypy.infer import ArgumentInferContext, infer_function_type_arguments, infer_type_arguments +from mypy.literals import literal +from mypy.maptype import map_instance_to_supertype +from mypy.meet import is_overlapping_types, narrow_declared_type +from mypy.message_registry import ErrorMessage +from mypy.messages import MessageBuilder, format_type from mypy.nodes import ( - NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, - Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, - OpExpr, UnaryExpr, IndexExpr, CastExpr, TypeApplication, ListExpr, - TupleExpr, DictExpr, FuncExpr, SuperExpr, ParenExpr, SliceExpr, Context, - ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, - UndefinedExpr, ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, - DictionaryComprehension, ComplexExpr, LITERAL_TYPE + ARG_NAMED, + ARG_POS, + ARG_STAR, + ARG_STAR2, + IMPLICITLY_ABSTRACT, + LAMBDA_NAME, + LITERAL_TYPE, + REVEAL_LOCALS, + REVEAL_TYPE, + ArgKind, + AssertTypeExpr, + AssignmentExpr, + AwaitExpr, + BytesExpr, + CallExpr, + CastExpr, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + Context, + Decorator, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, + Expression, + FloatExpr, + FuncDef, + GeneratorExpr, + IndexExpr, + IntExpr, + LambdaExpr, + ListComprehension, + ListExpr, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + NewTypeExpr, + OpExpr, + OverloadedFuncDef, + ParamSpecExpr, + PlaceholderNode, + PromoteExpr, + RefExpr, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + StrExpr, + SuperExpr, + SymbolNode, + TempNode, + TupleExpr, + TypeAlias, + TypeAliasExpr, + TypeApplication, + TypedDictExpr, + TypeInfo, + TypeVarExpr, + TypeVarTupleExpr, + UnaryExpr, + Var, + YieldExpr, + YieldFromExpr, ) -from mypy.errors import Errors -from mypy.nodes import function_type, method_type -from mypy import nodes -import mypy.checker -from mypy import types -from mypy.sametypes import is_same_type -from mypy.replacetvars import replace_func_type_vars, replace_type_vars -from mypy.messages import MessageBuilder -from mypy import messages -from mypy.infer import infer_type_arguments, infer_function_type_arguments -from mypy import join -from mypy.expandtype import expand_type, expand_caller_var_args -from mypy.subtypes import is_subtype, is_more_precise -from mypy import applytype -from mypy import erasetype -from mypy.checkmember import analyse_member_access, type_object_type -from mypy.semanal import self_type -from mypy.constraints import get_actual_type -from mypy.checkstrformat import StringFormatterChecker +from mypy.options import PRECISE_TUPLE_TYPES +from mypy.plugin import ( + FunctionContext, + FunctionSigContext, + MethodContext, + MethodSigContext, + Plugin, +) +from mypy.semanal_enum import ENUM_BASES +from mypy.state import state +from mypy.subtypes import ( + find_member, + is_equivalent, + is_same_type, + is_subtype, + non_method_protocol_members, +) +from mypy.traverser import has_await_expression +from mypy.typeanal import ( + check_for_explicit_any, + fix_instance, + has_any_from_unimported_type, + instantiate_type_alias, + make_optional_type, + set_any_tvars, + validate_instance, +) +from mypy.typeops import ( + bind_self, + callable_type, + custom_special_method, + erase_to_union_or_bound, + false_only, + fixup_partial_type, + function_type, + get_all_type_vars, + get_type_vars, + is_literal_type_like, + make_simplified_union, + simple_literal_type, + true_only, + try_expanding_sum_type_to_union, + try_getting_str_literals, + tuple_fallback, +) +from mypy.types import ( + LITERAL_TYPE_NAMES, + TUPLE_LIKE_INSTANCE_NAMES, + AnyType, + CallableType, + DeletedType, + ErasedType, + ExtraAttrs, + FunctionLike, + Instance, + LiteralType, + LiteralValue, + NoneType, + Overloaded, + Parameters, + ParamSpecFlavor, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + find_unpack_in_list, + flatten_nested_tuples, + flatten_nested_unions, + get_proper_type, + get_proper_types, + has_recursive_types, + has_type_vars, + is_named_instance, + split_with_prefix_and_suffix, +) +from mypy.types_utils import ( + is_generic_instance, + is_overlapping_none, + is_self_type_like, + remove_optional, +) +from mypy.typestate import type_state +from mypy.typevars import fill_typevars +from mypy.util import split_module_names +from mypy.visitor import ExpressionVisitor + +# Type of callback user for checking individual function arguments. See +# check_args() below for details. +ArgChecker: _TypeAlias = Callable[ + [Type, Type, ArgKind, Type, int, int, CallableType, Optional[Type], Context, Context], None +] + +# Maximum nesting level for math union in overloads, setting this to large values +# may cause performance issues. The reason is that although union math algorithm we use +# nicely captures most corner cases, its worst case complexity is exponential, +# see https://github.com/python/mypy/pull/5255#discussion_r196896335 for discussion. +MAX_UNIONS: Final = 5 + + +# Types considered safe for comparisons with --strict-equality due to known behaviour of __eq__. +# NOTE: All these types are subtypes of AbstractSet. +OVERLAPPING_TYPES_ALLOWLIST: Final = [ + "builtins.set", + "builtins.frozenset", + "typing.KeysView", + "typing.ItemsView", + "builtins._dict_keys", + "builtins._dict_items", + "_collections_abc.dict_keys", + "_collections_abc.dict_items", +] +OVERLAPPING_BYTES_ALLOWLIST: Final = { + "builtins.bytes", + "builtins.bytearray", + "builtins.memoryview", +} + + +class TooManyUnions(Exception): + """Indicates that we need to stop splitting unions in an attempt + to match an overload in order to save performance. + """ -class ExpressionChecker: +def allow_fast_container_literal(t: Type) -> bool: + if isinstance(t, TypeAliasType) and t.is_recursive: + return False + t = get_proper_type(t) + return isinstance(t, Instance) or ( + isinstance(t, TupleType) and all(allow_fast_container_literal(it) for it in t.items) + ) + + +def extract_refexpr_names(expr: RefExpr) -> set[str]: + """Recursively extracts all module references from a reference expression. + + Note that currently, the only two subclasses of RefExpr are NameExpr and + MemberExpr.""" + output: set[str] = set() + while isinstance(expr.node, MypyFile) or expr.fullname: + if isinstance(expr.node, MypyFile) and expr.fullname: + # If it's None, something's wrong (perhaps due to an + # import cycle or a suppressed error). For now we just + # skip it. + output.add(expr.fullname) + + if isinstance(expr, NameExpr): + is_suppressed_import = isinstance(expr.node, Var) and expr.node.is_suppressed_import + if isinstance(expr.node, TypeInfo): + # Reference to a class or a nested class + output.update(split_module_names(expr.node.module_name)) + elif "." in expr.fullname and not is_suppressed_import: + # Everything else (that is not a silenced import within a class) + output.add(expr.fullname.rsplit(".", 1)[0]) + break + elif isinstance(expr, MemberExpr): + if isinstance(expr.expr, RefExpr): + expr = expr.expr + else: + break + else: + raise AssertionError(f"Unknown RefExpr subclass: {type(expr)}") + return output + + +class Finished(Exception): + """Raised if we can terminate overload argument check early (no match).""" + + +@enum.unique +class UseReverse(enum.Enum): + """Used in `visit_op_expr` to enable or disable reverse method checks.""" + + DEFAULT = 0 + ALWAYS = 1 + NEVER = 2 + + +USE_REVERSE_DEFAULT: Final = UseReverse.DEFAULT +USE_REVERSE_ALWAYS: Final = UseReverse.ALWAYS +USE_REVERSE_NEVER: Final = UseReverse.NEVER + + +class ExpressionChecker(ExpressionVisitor[Type]): """Expression type checker. This class works closely together with checker.TypeChecker. """ # Some services are provided by a TypeChecker instance. - chk = Undefined('mypy.checker.TypeChecker') + chk: mypy.checker.TypeChecker # This is shared with TypeChecker, but stored also here for convenience. - msg = Undefined(MessageBuilder) - - strfrm_checker = Undefined('mypy.checkstrformat.StringFormatterChecker') - - def __init__(self, - chk: 'mypy.checker.TypeChecker', - msg: MessageBuilder) -> None: + msg: MessageBuilder + # Type context for type inference + type_context: list[Type | None] + + # cache resolved types in some cases + resolved_type: dict[Expression, ProperType] + + strfrm_checker: StringFormatterChecker + plugin: Plugin + + def __init__( + self, + chk: mypy.checker.TypeChecker, + msg: MessageBuilder, + plugin: Plugin, + per_line_checking_time_ns: dict[int, int], + ) -> None: """Construct an expression type checker.""" self.chk = chk self.msg = msg - self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) + self.plugin = plugin + self.per_line_checking_time_ns = per_line_checking_time_ns + self.collect_line_checking_stats = chk.options.line_checking_stats is not None + # Are we already visiting some expression? This is used to avoid double counting + # time for nested expressions. + self.in_expression = False + self.type_context = [None] + + # Temporary overrides for expression types. This is currently + # used by the union math in overloads. + # TODO: refactor this to use a pattern similar to one in + # multiassign_from_union, or maybe even combine the two? + self.type_overrides: dict[Expression, Type] = {} + self.strfrm_checker = StringFormatterChecker(self, self.chk, self.msg) + + self.resolved_type = {} + + # Callee in a call expression is in some sense both runtime context and + # type context, because we support things like C[int](...). Store information + # on whether current expression is a callee, to give better error messages + # related to type context. + self.is_callee = False + type_state.infer_polymorphic = not self.chk.options.old_type_inference + + def reset(self) -> None: + self.resolved_type = {} def visit_name_expr(self, e: NameExpr) -> Type: """Type check a name expression. It can be of any kind: local, member or global. """ - result = self.analyse_ref_expr(e) - return self.chk.narrow_type_from_binder(e, result) - - def analyse_ref_expr(self, e: RefExpr) -> Type: - result = Undefined(Type) + self.chk.module_refs.update(extract_refexpr_names(e)) + result = self.analyze_ref_expr(e) + narrowed = self.narrow_type_from_binder(e, result) + self.chk.check_deprecated(e.node, e) + return narrowed + + def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: + result: Type | None = None node = e.node + + if isinstance(e, NameExpr) and e.is_special_form: + # A special form definition, nothing to check here. + return AnyType(TypeOfAny.special_form) + if isinstance(node, Var): # Variable reference. - result = self.analyse_var_ref(node, e) + result = self.analyze_var_ref(node, e) + if isinstance(result, PartialType): + result = self.chk.handle_partial_var_type(result, lvalue, node, e) elif isinstance(node, FuncDef): # Reference to a global function. - result = function_type(node, self.named_type('builtins.function')) + result = function_type(node, self.named_type("builtins.function")) elif isinstance(node, OverloadedFuncDef): - result = node.type + if node.type is None: + if self.chk.in_checked_function() and node.items: + self.chk.handle_cannot_determine_type(node.name, e) + result = AnyType(TypeOfAny.from_error) + else: + result = node.type elif isinstance(node, TypeInfo): # Reference to a type object. - result = type_object_type(node, self.named_type) + if node.typeddict_type: + # We special-case TypedDict, because they don't define any constructor. + result = self.typeddict_callable(node) + elif node.fullname == "types.NoneType": + # We special case NoneType, because its stub definition is not related to None. + result = TypeType(NoneType()) + else: + result = type_object_type(node, self.named_type) + if isinstance(result, CallableType) and isinstance( # type: ignore[misc] + result.ret_type, Instance + ): + # We need to set correct line and column + # TODO: always do this in type_object_type by passing the original context + result.ret_type.line = e.line + result.ret_type.column = e.column + if is_type_type_context(self.type_context[-1]): + # This is the type in a type[] expression, so substitute type + # variables with Any. + result = erasetype.erase_typevars(result) elif isinstance(node, MypyFile): # Reference to a module object. - result = self.named_type('builtins.module') + result = self.module_type(node) elif isinstance(node, Decorator): - result = self.analyse_var_ref(node.var, e) + result = self.analyze_var_ref(node.var, e) + elif isinstance(node, TypeAlias): + # Something that refers to a type alias appears in runtime context. + # Note that we suppress bogus errors for alias redefinitions, + # they are already reported in semanal.py. + result = self.alias_type_in_runtime_context( + node, ctx=e, alias_definition=e.is_alias_rvalue or lvalue + ) + elif isinstance(node, TypeVarExpr): + return self.named_type("typing.TypeVar") + elif isinstance(node, (ParamSpecExpr, TypeVarTupleExpr)): + result = self.object_type() else: + if isinstance(node, PlaceholderNode): + assert False, f"PlaceholderNode {node.fullname!r} leaked to checker" # Unknown reference; use any type implicitly to avoid # generating extra type errors. - result = AnyType() + result = AnyType(TypeOfAny.from_error) + assert result is not None return result - def analyse_var_ref(self, var: Var, context: Context) -> Type: - if not var.type: - if not var.is_ready: - self.msg.cannot_determine_type(var.name(), context) - # Implicit 'Any' type. - return AnyType() + def analyze_var_ref(self, var: Var, context: Context) -> Type: + if var.type: + var_type = get_proper_type(var.type) + if isinstance(var_type, Instance): + if var.fullname == "typing.Any": + # The typeshed type is 'object'; give a more useful type in runtime context + return self.named_type("typing._SpecialForm") + if self.is_literal_context() and var_type.last_known_value is not None: + return var_type.last_known_value + if var.name in {"True", "False"}: + return self.infer_literal_expr_type(var.name == "True", "builtins.bool") + return var.type else: - # Look up local type of variable with type (inferred or explicit). - val = self.chk.binder.get(var) - if val is None: - return var.type + if not var.is_ready and self.chk.in_checked_function(): + self.chk.handle_cannot_determine_type(var.name, context) + # Implicit 'Any' type. + return AnyType(TypeOfAny.special_form) + + def module_type(self, node: MypyFile) -> Instance: + try: + result = self.named_type("types.ModuleType") + except KeyError: + # In test cases might 'types' may not be available. + # Fall back to a dummy 'object' type instead to + # avoid a crash. + result = self.named_type("builtins.object") + module_attrs = {} + immutable = set() + for name, n in node.names.items(): + if not n.module_public: + continue + if isinstance(n.node, Var) and n.node.is_final: + immutable.add(name) + typ = self.chk.determine_type_of_member(n) + if typ: + module_attrs[name] = typ else: - return val + # TODO: what to do about nested module references? + # They are non-trivial because there may be import cycles. + module_attrs[name] = AnyType(TypeOfAny.special_form) + result.extra_attrs = ExtraAttrs(module_attrs, immutable, node.fullname) + return result - def visit_call_expr(self, e: CallExpr) -> Type: + def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: """Type check a call expression.""" if e.analyzed: + if isinstance(e.analyzed, NamedTupleExpr) and not e.analyzed.is_typed: + # Type check the arguments, but ignore the results. This relies + # on the typeshed stubs to type check the arguments. + self.visit_call_expr_inner(e) # It's really a special form that only looks like a call. - return self.accept(e.analyzed) - self.accept(e.callee) - # Access callee type directly, since accept may return the Any type - # even if the type is known (in a dynamically typed function). This - # way we get a more precise callee in dynamically typed functions. - callee_type = self.chk.type_map[e.callee] - return self.check_call_expr_with_callee_type(callee_type, e) - - def check_call_expr_with_callee_type(self, callee_type: Type, - e: CallExpr) -> Type: - """Type check call expression. + return self.accept(e.analyzed, self.type_context[-1]) + return self.visit_call_expr_inner(e, allow_none_return=allow_none_return) - The given callee type overrides the type of the callee - expression. + def refers_to_typeddict(self, base: Expression) -> bool: + if not isinstance(base, RefExpr): + return False + if isinstance(base.node, TypeInfo) and base.node.typeddict_type is not None: + # Direct reference. + return True + return isinstance(base.node, TypeAlias) and isinstance( + get_proper_type(base.node.target), TypedDictType + ) + + def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> Type: + if ( + self.refers_to_typeddict(e.callee) + or isinstance(e.callee, IndexExpr) + and self.refers_to_typeddict(e.callee.base) + ): + typeddict_callable = get_proper_type(self.accept(e.callee, is_callee=True)) + if isinstance(typeddict_callable, CallableType): + typeddict_type = get_proper_type(typeddict_callable.ret_type) + assert isinstance(typeddict_type, TypedDictType) + return self.check_typeddict_call( + typeddict_type, e.arg_kinds, e.arg_names, e.args, e, typeddict_callable + ) + if ( + isinstance(e.callee, NameExpr) + and e.callee.name in ("isinstance", "issubclass") + and len(e.args) == 2 + ): + for typ in mypy.checker.flatten(e.args[1]): + node = None + if isinstance(typ, NameExpr): + try: + node = self.chk.lookup_qualified(typ.name) + except KeyError: + # Undefined names should already be reported in semantic analysis. + pass + if is_expr_literal_type(typ): + self.msg.cannot_use_function_with_type(e.callee.name, "Literal", e) + continue + if node and isinstance(node.node, TypeAlias): + target = get_proper_type(node.node.target) + if isinstance(target, AnyType): + self.msg.cannot_use_function_with_type(e.callee.name, "Any", e) + continue + if isinstance(target, NoneType): + continue + if ( + isinstance(typ, IndexExpr) + and isinstance(typ.analyzed, (TypeApplication, TypeAliasExpr)) + ) or ( + isinstance(typ, NameExpr) + and node + and isinstance(node.node, TypeAlias) + and not node.node.no_args + and not ( + isinstance(union_target := get_proper_type(node.node.target), UnionType) + and union_target.uses_pep604_syntax + ) + ): + self.msg.type_arguments_not_allowed(e) + if isinstance(typ, RefExpr) and isinstance(typ.node, TypeInfo): + if typ.node.typeddict_type: + self.msg.cannot_use_function_with_type(e.callee.name, "TypedDict", e) + elif typ.node.is_newtype: + self.msg.cannot_use_function_with_type(e.callee.name, "NewType", e) + self.try_infer_partial_type(e) + type_context = None + if isinstance(e.callee, LambdaExpr): + formal_to_actual = map_actuals_to_formals( + e.arg_kinds, + e.arg_names, + e.callee.arg_kinds, + e.callee.arg_names, + lambda i: self.accept(e.args[i]), + ) + + arg_types = [ + join.join_type_list([self.accept(e.args[j]) for j in formal_to_actual[i]]) + for i in range(len(e.callee.arg_kinds)) + ] + type_context = CallableType( + arg_types, + e.callee.arg_kinds, + e.callee.arg_names, + ret_type=self.object_type(), + fallback=self.named_type("builtins.function"), + ) + callee_type = get_proper_type( + self.accept(e.callee, type_context, always_allow_any=True, is_callee=True) + ) + + # Figure out the full name of the callee for plugin lookup. + object_type = None + member = None + fullname = None + if isinstance(e.callee, RefExpr): + # There are two special cases where plugins might act: + # * A "static" reference/alias to a class or function; + # get_function_hook() will be invoked for these. + fullname = e.callee.fullname or None + if isinstance(e.callee.node, TypeAlias): + target = get_proper_type(e.callee.node.target) + if isinstance(target, Instance): + fullname = target.type.fullname + # * Call to a method on object that has a full name (see + # method_fullname() for details on supported objects); + # get_method_hook() and get_method_signature_hook() will + # be invoked for these. + if ( + not fullname + and isinstance(e.callee, MemberExpr) + and self.chk.has_type(e.callee.expr) + ): + member = e.callee.name + object_type = self.chk.lookup_type(e.callee.expr) + + if ( + self.chk.options.disallow_untyped_calls + and self.chk.in_checked_function() + and isinstance(callee_type, CallableType) + and callee_type.implicit + and callee_type.name != LAMBDA_NAME + ): + if fullname is None and member is not None: + assert object_type is not None + fullname = self.method_fullname(object_type, member) + if not fullname or not any( + fullname == p or fullname.startswith(f"{p}.") + for p in self.chk.options.untyped_calls_exclude + ): + self.msg.untyped_function_call(callee_type, e) + + ret_type = self.check_call_expr_with_callee_type( + callee_type, e, fullname, object_type, member + ) + if isinstance(e.callee, RefExpr) and len(e.args) == 2: + if e.callee.fullname in ("builtins.isinstance", "builtins.issubclass"): + self.check_runtime_protocol_test(e) + if e.callee.fullname == "builtins.issubclass": + self.check_protocol_issubclass(e) + if isinstance(e.callee, MemberExpr) and e.callee.name == "format": + self.check_str_format_call(e) + ret_type = get_proper_type(ret_type) + if isinstance(ret_type, UnionType): + ret_type = make_simplified_union(ret_type.items) + if isinstance(ret_type, UninhabitedType) and not ret_type.ambiguous: + self.chk.binder.unreachable() + # Warn on calls to functions that always return None. The check + # of ret_type is both a common-case optimization and prevents reporting + # the error in dynamic functions (where it will be Any). + if ( + not allow_none_return + and isinstance(ret_type, NoneType) + and self.always_returns_none(e.callee) + ): + self.chk.msg.does_not_return_value(callee_type, e) + return AnyType(TypeOfAny.from_error) + return ret_type + + def check_str_format_call(self, e: CallExpr) -> None: + """More precise type checking for str.format() calls on literals.""" + assert isinstance(e.callee, MemberExpr) + format_value = None + if isinstance(e.callee.expr, StrExpr): + format_value = e.callee.expr.value + elif self.chk.has_type(e.callee.expr): + typ = get_proper_type(self.chk.lookup_type(e.callee.expr)) + if ( + isinstance(typ, Instance) + and typ.type.is_enum + and isinstance(typ.last_known_value, LiteralType) + and isinstance(typ.last_known_value.value, str) + ): + value_type = typ.type.names[typ.last_known_value.value].type + if isinstance(value_type, Type): + typ = get_proper_type(value_type) + base_typ = try_getting_literal(typ) + if isinstance(base_typ, LiteralType) and isinstance(base_typ.value, str): + format_value = base_typ.value + if format_value is not None: + self.strfrm_checker.check_str_format_call(e, format_value) + + def method_fullname(self, object_type: Type, method_name: str) -> str | None: + """Convert a method name to a fully qualified name, based on the type of the object that + it is invoked on. Return `None` if the name of `object_type` cannot be determined. """ - return self.check_call(callee_type, e.args, e.arg_kinds, e, - e.arg_names, callable_node=e.callee)[0] - - def check_call(self, callee: Type, args: List[Node], - arg_kinds: List[int], context: Context, - arg_names: List[str] = None, - callable_node: Node = None, - arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: - """Type check a call. + object_type = get_proper_type(object_type) + + if isinstance(object_type, CallableType) and object_type.is_type_obj(): + # For class method calls, object_type is a callable representing the class object. + # We "unwrap" it to a regular type, as the class/instance method difference doesn't + # affect the fully qualified name. + object_type = get_proper_type(object_type.ret_type) + elif isinstance(object_type, TypeType): + object_type = object_type.item + + type_name = None + if isinstance(object_type, Instance): + type_name = object_type.type.fullname + elif isinstance(object_type, (TypedDictType, LiteralType)): + info = object_type.fallback.type.get_containing_type_info(method_name) + type_name = info.fullname if info is not None else None + elif isinstance(object_type, TupleType): + type_name = tuple_fallback(object_type).type.fullname + + if type_name: + return f"{type_name}.{method_name}" + else: + return None - Also infer type arguments if the callee is a generic function. + def always_returns_none(self, node: Expression) -> bool: + """Check if `node` refers to something explicitly annotated as only returning None.""" + if isinstance(node, RefExpr): + if self.defn_returns_none(node.node): + return True + if isinstance(node, MemberExpr) and node.node is None: # instance or class attribute + typ = get_proper_type(self.chk.lookup_type(node.expr)) + if isinstance(typ, Instance): + info = typ.type + elif isinstance(typ, CallableType) and typ.is_type_obj(): + ret_type = get_proper_type(typ.ret_type) + if isinstance(ret_type, Instance): + info = ret_type.type + else: + return False + else: + return False + sym = info.get(node.name) + if sym and self.defn_returns_none(sym.node): + return True + return False + + def defn_returns_none(self, defn: SymbolNode | None) -> bool: + """Check if `defn` can _only_ return None.""" + if isinstance(defn, FuncDef): + return isinstance(defn.type, CallableType) and isinstance( + get_proper_type(defn.type.ret_type), NoneType + ) + if isinstance(defn, OverloadedFuncDef): + return all(self.defn_returns_none(item) for item in defn.items) + if isinstance(defn, Var): + typ = get_proper_type(defn.type) + if ( + not defn.is_inferred + and isinstance(typ, CallableType) + and isinstance(get_proper_type(typ.ret_type), NoneType) + ): + return True + if isinstance(typ, Instance): + sym = typ.type.get("__call__") + if sym and self.defn_returns_none(sym.node): + return True + return False + + def check_runtime_protocol_test(self, e: CallExpr) -> None: + for expr in mypy.checker.flatten(e.args[1]): + tp = get_proper_type(self.chk.lookup_type(expr)) + if ( + isinstance(tp, FunctionLike) + and tp.is_type_obj() + and tp.type_object().is_protocol + and not tp.type_object().runtime_protocol + ): + self.chk.fail(message_registry.RUNTIME_PROTOCOL_EXPECTED, e) + + def check_protocol_issubclass(self, e: CallExpr) -> None: + for expr in mypy.checker.flatten(e.args[1]): + tp = get_proper_type(self.chk.lookup_type(expr)) + if isinstance(tp, FunctionLike) and tp.is_type_obj() and tp.type_object().is_protocol: + attr_members = non_method_protocol_members(tp.type_object()) + if attr_members: + self.chk.msg.report_non_method_protocol(tp.type_object(), attr_members, e) + + def check_typeddict_call( + self, + callee: TypedDictType, + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None], + args: list[Expression], + context: Context, + orig_callee: Type | None, + ) -> Type: + if args and all(ak in (ARG_NAMED, ARG_STAR2) for ak in arg_kinds): + # ex: Point(x=42, y=1337, **extras) + # This is a bit ugly, but this is a price for supporting all possible syntax + # variants for TypedDict constructors. + kwargs = zip([StrExpr(n) if n is not None else None for n in arg_names], args) + result = self.validate_typeddict_kwargs(kwargs=kwargs, callee=callee) + if result is not None: + validated_kwargs, always_present_keys = result + return self.check_typeddict_call_with_kwargs( + callee, validated_kwargs, context, orig_callee, always_present_keys + ) + return AnyType(TypeOfAny.from_error) + + if len(args) == 1 and arg_kinds[0] == ARG_POS: + unique_arg = args[0] + if isinstance(unique_arg, DictExpr): + # ex: Point({'x': 42, 'y': 1337, **extras}) + return self.check_typeddict_call_with_dict( + callee, unique_arg.items, context, orig_callee + ) + if isinstance(unique_arg, CallExpr) and isinstance(unique_arg.analyzed, DictExpr): + # ex: Point(dict(x=42, y=1337, **extras)) + return self.check_typeddict_call_with_dict( + callee, unique_arg.analyzed.items, context, orig_callee + ) + + if not args: + # ex: EmptyDict() + return self.check_typeddict_call_with_kwargs(callee, {}, context, orig_callee, set()) + + self.chk.fail(message_registry.INVALID_TYPEDDICT_ARGS, context) + return AnyType(TypeOfAny.from_error) + + def validate_typeddict_kwargs( + self, kwargs: Iterable[tuple[Expression | None, Expression]], callee: TypedDictType + ) -> tuple[dict[str, list[Expression]], set[str]] | None: + # All (actual or mapped from ** unpacks) expressions that can match given key. + result = defaultdict(list) + # Keys that are guaranteed to be present no matter what (e.g. for all items of a union) + always_present_keys = set() + # Indicates latest encountered ** unpack among items. + last_star_found = None + + for item_name_expr, item_arg in kwargs: + if item_name_expr: + key_type = self.accept(item_name_expr) + values = try_getting_str_literals(item_name_expr, key_type) + literal_value = None + if values and len(values) == 1: + literal_value = values[0] + if literal_value is None: + key_context = item_name_expr or item_arg + self.chk.fail( + message_registry.TYPEDDICT_KEY_MUST_BE_STRING_LITERAL, + key_context, + code=codes.LITERAL_REQ, + ) + return None + else: + # A directly present key unconditionally shadows all previously found + # values from ** items. + # TODO: for duplicate keys, type-check all values. + result[literal_value] = [item_arg] + always_present_keys.add(literal_value) + else: + last_star_found = item_arg + if not self.validate_star_typeddict_item( + item_arg, callee, result, always_present_keys + ): + return None + if self.chk.options.extra_checks and last_star_found is not None: + absent_keys = [] + for key in callee.items: + if key not in callee.required_keys and key not in result: + absent_keys.append(key) + if absent_keys: + # Having an optional key not explicitly declared by a ** unpacked + # TypedDict is unsafe, it may be an (incompatible) subtype at runtime. + # TODO: catch the cases where a declared key is overridden by a subsequent + # ** item without it (and not again overriden with complete ** item). + self.msg.non_required_keys_absent_with_star(absent_keys, last_star_found) + return result, always_present_keys + + def validate_star_typeddict_item( + self, + item_arg: Expression, + callee: TypedDictType, + result: dict[str, list[Expression]], + always_present_keys: set[str], + ) -> bool: + """Update keys/expressions from a ** expression in TypedDict constructor. + + Note `result` and `always_present_keys` are updated in place. Return true if the + expression `item_arg` may valid in `callee` TypedDict context. + """ + inferred = get_proper_type(self.accept(item_arg, type_context=callee)) + possible_tds = [] + if isinstance(inferred, TypedDictType): + possible_tds = [inferred] + elif isinstance(inferred, UnionType): + for item in get_proper_types(inferred.relevant_items()): + if isinstance(item, TypedDictType): + possible_tds.append(item) + elif not self.valid_unpack_fallback_item(item): + self.msg.unsupported_target_for_star_typeddict(item, item_arg) + return False + elif not self.valid_unpack_fallback_item(inferred): + self.msg.unsupported_target_for_star_typeddict(inferred, item_arg) + return False + all_keys: set[str] = set() + for td in possible_tds: + all_keys |= td.items.keys() + for key in all_keys: + arg = TempNode( + UnionType.make_union([td.items[key] for td in possible_tds if key in td.items]) + ) + arg.set_line(item_arg) + if all(key in td.required_keys for td in possible_tds): + always_present_keys.add(key) + # Always present keys override previously found values. This is done + # to support use cases like `Config({**defaults, **overrides})`, where + # some `overrides` types are narrower that types in `defaults`, and + # former are too wide for `Config`. + if result[key]: + first = result[key][0] + if not isinstance(first, TempNode): + # We must always preserve any non-synthetic values, so that + # we will accept them even if they are shadowed. + result[key] = [first, arg] + else: + result[key] = [arg] + else: + result[key] = [arg] + else: + # If this key is not required at least in some item of a union + # it may not shadow previous item, so we need to type check both. + result[key].append(arg) + return True - Return (result type, inferred callee type). + def valid_unpack_fallback_item(self, typ: ProperType) -> bool: + if isinstance(typ, AnyType): + return True + if not isinstance(typ, Instance) or not typ.type.has_base("typing.Mapping"): + return False + mapped = map_instance_to_supertype(typ, self.chk.lookup_typeinfo("typing.Mapping")) + return all(isinstance(a, AnyType) for a in get_proper_types(mapped.args)) + + def match_typeddict_call_with_dict( + self, + callee: TypedDictType, + kwargs: list[tuple[Expression | None, Expression]], + context: Context, + ) -> bool: + result = self.validate_typeddict_kwargs(kwargs=kwargs, callee=callee) + if result is not None: + validated_kwargs, _ = result + return callee.required_keys <= set(validated_kwargs.keys()) <= set(callee.items.keys()) + else: + return False - Arguments: - callee: type of the called value - args: actual argument expressions - arg_kinds: contains nodes.ARG_* constant for each argument in args - describing whether the argument is positional, *arg, etc. - arg_names: names of arguments (optional) - callable_node: associate the inferred callable type to this node, - if specified - arg_messages: TODO + def check_typeddict_call_with_dict( + self, + callee: TypedDictType, + kwargs: list[tuple[Expression | None, Expression]], + context: Context, + orig_callee: Type | None, + ) -> Type: + result = self.validate_typeddict_kwargs(kwargs=kwargs, callee=callee) + if result is not None: + validated_kwargs, always_present_keys = result + return self.check_typeddict_call_with_kwargs( + callee, + kwargs=validated_kwargs, + context=context, + orig_callee=orig_callee, + always_present_keys=always_present_keys, + ) + else: + return AnyType(TypeOfAny.from_error) + + def typeddict_callable(self, info: TypeInfo) -> CallableType: + """Construct a reasonable type for a TypedDict type in runtime context. + + If it appears as a callee, it will be special-cased anyway, e.g. it is + also allowed to accept a single positional argument if it is a dict literal. + + Note it is not safe to move this to type_object_type() since it will crash + on plugin-generated TypedDicts, that may not have the special_alias. """ - arg_messages = arg_messages or self.msg - is_var_arg = nodes.ARG_STAR in arg_kinds - if isinstance(callee, Callable): - if callee.is_type_obj(): - t = callee.type_object() - if callee.is_type_obj() and callee.type_object().is_abstract: - type = callee.type_object() - self.msg.cannot_instantiate_abstract_class( - callee.type_object().name(), type.abstract_attributes, - context) + assert info.special_alias is not None + target = info.special_alias.target + assert isinstance(target, ProperType) and isinstance(target, TypedDictType) + expected_types = list(target.items.values()) + kinds = [ArgKind.ARG_NAMED] * len(expected_types) + names = list(target.items.keys()) + return CallableType( + expected_types, + kinds, + names, + target, + self.named_type("builtins.type"), + variables=info.defn.type_vars, + ) + + def typeddict_callable_from_context(self, callee: TypedDictType) -> CallableType: + return CallableType( + list(callee.items.values()), + [ + ArgKind.ARG_NAMED if name in callee.required_keys else ArgKind.ARG_NAMED_OPT + for name in callee.items + ], + list(callee.items.keys()), + callee, + self.named_type("builtins.type"), + ) + + def check_typeddict_call_with_kwargs( + self, + callee: TypedDictType, + kwargs: dict[str, list[Expression]], + context: Context, + orig_callee: Type | None, + always_present_keys: set[str], + ) -> Type: + actual_keys = kwargs.keys() + if callee.to_be_mutated: + assigned_readonly_keys = actual_keys & callee.readonly_keys + if assigned_readonly_keys: + self.msg.readonly_keys_mutated(assigned_readonly_keys, context=context) + if not ( + callee.required_keys <= always_present_keys and actual_keys <= callee.items.keys() + ): + if not (actual_keys <= callee.items.keys()): + self.msg.unexpected_typeddict_keys( + callee, + expected_keys=[ + key + for key in callee.items.keys() + if key in callee.required_keys or key in actual_keys + ], + actual_keys=list(actual_keys), + context=context, + ) + if not (callee.required_keys <= always_present_keys): + self.msg.unexpected_typeddict_keys( + callee, + expected_keys=[ + key for key in callee.items.keys() if key in callee.required_keys + ], + actual_keys=[ + key for key in always_present_keys if key in callee.required_keys + ], + context=context, + ) + if callee.required_keys > actual_keys: + # found_set is a sub-set of the required_keys + # This means we're missing some keys and as such, we can't + # properly type the object + return AnyType(TypeOfAny.from_error) + + orig_callee = get_proper_type(orig_callee) + if isinstance(orig_callee, CallableType): + infer_callee = orig_callee + else: + # Try reconstructing from type context. + if callee.fallback.type.special_alias is not None: + infer_callee = self.typeddict_callable(callee.fallback.type) + else: + # Likely a TypedDict type generated by a plugin. + infer_callee = self.typeddict_callable_from_context(callee) + + # We don't show any errors, just infer types in a generic TypedDict type, + # a custom error message will be given below, if there are errors. + with self.msg.filter_errors(), self.chk.local_type_map(): + orig_ret_type, _ = self.check_callable_call( + infer_callee, + # We use first expression for each key to infer type variables of a generic + # TypedDict. This is a bit arbitrary, but in most cases will work better than + # trying to infer a union or a join. + [args[0] for args in kwargs.values()], + [ArgKind.ARG_NAMED] * len(kwargs), + context, + list(kwargs.keys()), + None, + None, + None, + ) + + ret_type = get_proper_type(orig_ret_type) + if not isinstance(ret_type, TypedDictType): + # If something went really wrong, type-check call with original type, + # this may give a better error message. + ret_type = callee + + for item_name, item_expected_type in ret_type.items.items(): + if item_name in kwargs: + item_values = kwargs[item_name] + for item_value in item_values: + self.chk.check_simple_assignment( + lvalue_type=item_expected_type, + rvalue=item_value, + context=item_value, + msg=ErrorMessage( + message_registry.INCOMPATIBLE_TYPES.value, code=codes.TYPEDDICT_ITEM + ), + lvalue_name=f'TypedDict item "{item_name}"', + rvalue_name="expression", + ) + + return orig_ret_type + + def get_partial_self_var(self, expr: MemberExpr) -> Var | None: + """Get variable node for a partial self attribute. + + If the expression is not a self attribute, or attribute is not variable, + or variable is not partial, return None. + """ + if not ( + isinstance(expr.expr, NameExpr) + and isinstance(expr.expr.node, Var) + and expr.expr.node.is_self + ): + # Not a self.attr expression. + return None + info = self.chk.scope.enclosing_class() + if not info or expr.name not in info.names: + # Don't mess with partial types in superclasses. + return None + sym = info.names[expr.name] + if isinstance(sym.node, Var) and isinstance(sym.node.type, PartialType): + return sym.node + return None + + # Types and methods that can be used to infer partial types. + item_args: ClassVar[dict[str, list[str]]] = { + "builtins.list": ["append"], + "builtins.set": ["add", "discard"], + } + container_args: ClassVar[dict[str, dict[str, list[str]]]] = { + "builtins.list": {"extend": ["builtins.list"]}, + "builtins.dict": {"update": ["builtins.dict"]}, + "collections.OrderedDict": {"update": ["builtins.dict"]}, + "builtins.set": {"update": ["builtins.set", "builtins.list"]}, + } + + def try_infer_partial_type(self, e: CallExpr) -> None: + """Try to make partial type precise from a call.""" + if not isinstance(e.callee, MemberExpr): + return + callee = e.callee + if isinstance(callee.expr, RefExpr): + # Call a method with a RefExpr callee, such as 'x.method(...)'. + ret = self.get_partial_var(callee.expr) + if ret is None: + return + var, partial_types = ret + typ = self.try_infer_partial_value_type_from_call(e, callee.name, var) + # Var may be deleted from partial_types in try_infer_partial_value_type_from_call + if typ is not None and var in partial_types: + var.type = typ + del partial_types[var] + elif isinstance(callee.expr, IndexExpr) and isinstance(callee.expr.base, RefExpr): + # Call 'x[y].method(...)'; may infer type of 'x' if it's a partial defaultdict. + if callee.expr.analyzed is not None: + return # A special form + base = callee.expr.base + index = callee.expr.index + ret = self.get_partial_var(base) + if ret is None: + return + var, partial_types = ret + partial_type = get_partial_instance_type(var.type) + if partial_type is None or partial_type.value_type is None: + return + value_type = self.try_infer_partial_value_type_from_call(e, callee.name, var) + if value_type is not None: + # Infer key type. + key_type = self.accept(index) + if mypy.checker.is_valid_inferred_type(key_type): + # Store inferred partial type. + assert partial_type.type is not None + typename = partial_type.type.fullname + var.type = self.chk.named_generic_type(typename, [key_type, value_type]) + del partial_types[var] + + def get_partial_var(self, ref: RefExpr) -> tuple[Var, dict[Var, Context]] | None: + var = ref.node + if var is None and isinstance(ref, MemberExpr): + var = self.get_partial_self_var(ref) + if not isinstance(var, Var): + return None + partial_types = self.chk.find_partial_types(var) + if partial_types is None: + return None + return var, partial_types + def try_infer_partial_value_type_from_call( + self, e: CallExpr, methodname: str, var: Var + ) -> Instance | None: + """Try to make partial type precise from a call such as 'x.append(y)'.""" + if self.chk.current_node_deferred: + return None + partial_type = get_partial_instance_type(var.type) + if partial_type is None: + return None + if partial_type.value_type: + typename = partial_type.value_type.type.fullname + else: + assert partial_type.type is not None + typename = partial_type.type.fullname + # Sometimes we can infer a full type for a partial List, Dict or Set type. + # TODO: Don't infer argument expression twice. + if ( + typename in self.item_args + and methodname in self.item_args[typename] + and e.arg_kinds == [ARG_POS] + ): + item_type = self.accept(e.args[0]) + if mypy.checker.is_valid_inferred_type(item_type): + return self.chk.named_generic_type(typename, [item_type]) + elif ( + typename in self.container_args + and methodname in self.container_args[typename] + and e.arg_kinds == [ARG_POS] + ): + arg_type = get_proper_type(self.accept(e.args[0])) + if isinstance(arg_type, Instance): + arg_typename = arg_type.type.fullname + if arg_typename in self.container_args[typename][methodname]: + if all( + mypy.checker.is_valid_inferred_type(item_type) + for item_type in arg_type.args + ): + return self.chk.named_generic_type(typename, list(arg_type.args)) + elif isinstance(arg_type, AnyType): + return self.chk.named_type(typename) + + return None + + def apply_function_plugin( + self, + callee: CallableType, + arg_kinds: list[ArgKind], + arg_types: list[Type], + arg_names: Sequence[str | None] | None, + formal_to_actual: list[list[int]], + args: list[Expression], + fullname: str, + object_type: Type | None, + context: Context, + ) -> Type: + """Use special case logic to infer the return type of a specific named function/method. + + Caller must ensure that a plugin hook exists. There are two different cases: + + - If object_type is None, the caller must ensure that a function hook exists + for fullname. + - If object_type is not None, the caller must ensure that a method hook exists + for fullname. + + Return the inferred return type. + """ + num_formals = len(callee.arg_types) + formal_arg_types: list[list[Type]] = [[] for _ in range(num_formals)] + formal_arg_exprs: list[list[Expression]] = [[] for _ in range(num_formals)] + formal_arg_names: list[list[str | None]] = [[] for _ in range(num_formals)] + formal_arg_kinds: list[list[ArgKind]] = [[] for _ in range(num_formals)] + for formal, actuals in enumerate(formal_to_actual): + for actual in actuals: + formal_arg_types[formal].append(arg_types[actual]) + formal_arg_exprs[formal].append(args[actual]) + if arg_names: + formal_arg_names[formal].append(arg_names[actual]) + else: + formal_arg_names[formal].append(None) + formal_arg_kinds[formal].append(arg_kinds[actual]) + + if object_type is None: + # Apply function plugin + callback = self.plugin.get_function_hook(fullname) + assert callback is not None # Assume that caller ensures this + return callback( + FunctionContext( + arg_types=formal_arg_types, + arg_kinds=formal_arg_kinds, + callee_arg_names=callee.arg_names, + arg_names=formal_arg_names, + default_return_type=callee.ret_type, + args=formal_arg_exprs, + context=context, + api=self.chk, + ) + ) + else: + # Apply method plugin + method_callback = self.plugin.get_method_hook(fullname) + assert method_callback is not None # Assume that caller ensures this + object_type = get_proper_type(object_type) + return method_callback( + MethodContext( + type=object_type, + arg_types=formal_arg_types, + arg_kinds=formal_arg_kinds, + callee_arg_names=callee.arg_names, + arg_names=formal_arg_names, + default_return_type=callee.ret_type, + args=formal_arg_exprs, + context=context, + api=self.chk, + ) + ) + + def apply_signature_hook( + self, + callee: FunctionLike, + args: list[Expression], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + hook: Callable[[list[list[Expression]], CallableType], FunctionLike], + ) -> FunctionLike: + """Helper to apply a signature hook for either a function or method""" + if isinstance(callee, CallableType): + num_formals = len(callee.arg_kinds) formal_to_actual = map_actuals_to_formals( - arg_kinds, arg_names, - callee.arg_kinds, callee.arg_names, - lambda i: self.accept(args[i])) + arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: self.accept(args[i]), + ) + formal_arg_exprs: list[list[Expression]] = [[] for _ in range(num_formals)] + for formal, actuals in enumerate(formal_to_actual): + for actual in actuals: + formal_arg_exprs[formal].append(args[actual]) + return hook(formal_arg_exprs, callee) + else: + assert isinstance(callee, Overloaded) + items = [] + for item in callee.items: + adjusted = self.apply_signature_hook(item, args, arg_kinds, arg_names, hook) + assert isinstance(adjusted, CallableType) + items.append(adjusted) + return Overloaded(items) + + def apply_function_signature_hook( + self, + callee: FunctionLike, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + arg_names: Sequence[str | None] | None, + signature_hook: Callable[[FunctionSigContext], FunctionLike], + ) -> FunctionLike: + """Apply a plugin hook that may infer a more precise signature for a function.""" + return self.apply_signature_hook( + callee, + args, + arg_kinds, + arg_names, + (lambda args, sig: signature_hook(FunctionSigContext(args, sig, context, self.chk))), + ) + + def apply_method_signature_hook( + self, + callee: FunctionLike, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + arg_names: Sequence[str | None] | None, + object_type: Type, + signature_hook: Callable[[MethodSigContext], FunctionLike], + ) -> FunctionLike: + """Apply a plugin hook that may infer a more precise signature for a method.""" + pobject_type = get_proper_type(object_type) + return self.apply_signature_hook( + callee, + args, + arg_kinds, + arg_names, + ( + lambda args, sig: signature_hook( + MethodSigContext(pobject_type, args, sig, context, self.chk) + ) + ), + ) + + def transform_callee_type( + self, + callable_name: str | None, + callee: Type, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + arg_names: Sequence[str | None] | None = None, + object_type: Type | None = None, + ) -> Type: + """Attempt to determine a more accurate signature for a method call. + + This is done by looking up and applying a method signature hook (if one exists for the + given method name). + + If no matching method signature hook is found, callee is returned unmodified. The same + happens if the arguments refer to a non-method callable (this is allowed so that the code + calling transform_callee_type needs to perform fewer boilerplate checks). + + Note: this method is *not* called automatically as part of check_call, because in some + cases check_call is called multiple times while checking a single call (for example when + dealing with overloads). Instead, this method needs to be called explicitly + (if appropriate) before the signature is passed to check_call. + """ + callee = get_proper_type(callee) + if callable_name is not None and isinstance(callee, FunctionLike): + if object_type is not None: + method_sig_hook = self.plugin.get_method_signature_hook(callable_name) + if method_sig_hook: + return self.apply_method_signature_hook( + callee, args, arg_kinds, context, arg_names, object_type, method_sig_hook + ) + else: + function_sig_hook = self.plugin.get_function_signature_hook(callable_name) + if function_sig_hook: + return self.apply_function_signature_hook( + callee, args, arg_kinds, context, arg_names, function_sig_hook + ) + + return callee + + def is_generic_decorator_overload_call( + self, callee_type: CallableType, args: list[Expression] + ) -> Overloaded | None: + """Check if this looks like an application of a generic function to overload argument.""" + assert callee_type.variables + if len(callee_type.arg_types) != 1 or len(args) != 1: + # TODO: can we handle more general cases? + return None + if not isinstance(get_proper_type(callee_type.arg_types[0]), CallableType): + return None + if not isinstance(get_proper_type(callee_type.ret_type), CallableType): + return None + with self.chk.local_type_map(): + with self.msg.filter_errors(): + arg_type = get_proper_type(self.accept(args[0], type_context=None)) + if isinstance(arg_type, Overloaded): + return arg_type + return None + + def handle_decorator_overload_call( + self, callee_type: CallableType, overloaded: Overloaded, ctx: Context + ) -> tuple[Type, Type] | None: + """Type-check application of a generic callable to an overload. + + We check call on each individual overload item, and then combine results into a new + overload. This function should be only used if callee_type takes and returns a Callable. + """ + result = [] + inferred_args = [] + for item in overloaded.items: + arg = TempNode(typ=item) + with self.msg.filter_errors() as err: + item_result, inferred_arg = self.check_call(callee_type, [arg], [ARG_POS], ctx) + if err.has_new_errors(): + # This overload doesn't match. + continue + p_item_result = get_proper_type(item_result) + if not isinstance(p_item_result, CallableType): + continue + p_inferred_arg = get_proper_type(inferred_arg) + if not isinstance(p_inferred_arg, CallableType): + continue + inferred_args.append(p_inferred_arg) + result.append(p_item_result) + if not result or not inferred_args: + # None of the overload matched (or overload was initially malformed). + return None + return Overloaded(result), Overloaded(inferred_args) + + def check_call_expr_with_callee_type( + self, + callee_type: Type, + e: CallExpr, + callable_name: str | None, + object_type: Type | None, + member: str | None = None, + ) -> Type: + """Type check call expression. - if callee.is_generic(): - callee = self.infer_function_type_arguments_using_context( - callee, context) - callee = self.infer_function_type_arguments( - callee, args, arg_kinds, formal_to_actual, context) + The callee_type should be used as the type of callee expression. In particular, + in case of a union type this can be a particular item of the union, so that we can + apply plugin hooks to each item. - arg_types = self.infer_arg_types_in_context2( - callee, args, arg_kinds, formal_to_actual) + The 'member', 'callable_name' and 'object_type' are only used to call plugin hooks. + If 'callable_name' is None but 'member' is not None (member call), try constructing + 'callable_name' using 'object_type' (the base type on which the method is called), + for example 'typing.Mapping.get'. + """ + if callable_name is None and member is not None: + assert object_type is not None + callable_name = self.method_fullname(object_type, member) + object_type = get_proper_type(object_type) + if callable_name: + # Try to refine the call signature using plugin hooks before checking the call. + callee_type = self.transform_callee_type( + callable_name, callee_type, e.args, e.arg_kinds, e, e.arg_names, object_type + ) + # Unions are special-cased to allow plugins to act on each item in the union. + elif member is not None and isinstance(object_type, UnionType): + return self.check_union_call_expr(e, object_type, member) + ret_type, callee_type = self.check_call( + callee_type, + e.args, + e.arg_kinds, + e, + e.arg_names, + callable_node=e.callee, + callable_name=callable_name, + object_type=object_type, + ) + proper_callee = get_proper_type(callee_type) + if isinstance(e.callee, (NameExpr, MemberExpr)): + self.chk.warn_deprecated_overload_item(e.callee.node, e, target=callee_type) + if isinstance(e.callee, RefExpr) and isinstance(proper_callee, CallableType): + # Cache it for find_isinstance_check() + if proper_callee.type_guard is not None: + e.callee.type_guard = proper_callee.type_guard + if proper_callee.type_is is not None: + e.callee.type_is = proper_callee.type_is + return ret_type + + def check_union_call_expr(self, e: CallExpr, object_type: UnionType, member: str) -> Type: + """Type check calling a member expression where the base type is a union.""" + res: list[Type] = [] + for typ in object_type.relevant_items(): + # Member access errors are already reported when visiting the member expression. + with self.msg.filter_errors(): + item = analyze_member_access( + member, + typ, + e, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, + original_type=object_type, + chk=self.chk, + in_literal_context=self.is_literal_context(), + self_type=typ, + ) + narrowed = self.narrow_type_from_binder(e.callee, item, skip_non_overlapping=True) + if narrowed is None: + continue + callable_name = self.method_fullname(typ, member) + item_object_type = typ if callable_name else None + res.append( + self.check_call_expr_with_callee_type(narrowed, e, callable_name, item_object_type) + ) + return make_simplified_union(res) + + def check_call( + self, + callee: Type, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + arg_names: Sequence[str | None] | None = None, + callable_node: Expression | None = None, + callable_name: str | None = None, + object_type: Type | None = None, + original_type: Type | None = None, + ) -> tuple[Type, Type]: + """Type check a call. - self.check_argument_count(callee, arg_types, arg_kinds, - arg_names, formal_to_actual, context) + Also infer type arguments if the callee is a generic function. - self.check_argument_types(arg_types, arg_kinds, callee, - formal_to_actual, context, - messages=arg_messages) - if callable_node: - # Store the inferred callable type. - self.chk.store_type(callable_node, callee) - return callee.ret_type, callee + Return (result type, inferred callee type). + + Arguments: + callee: type of the called value + args: actual argument expressions + arg_kinds: contains nodes.ARG_* constant for each argument in args + describing whether the argument is positional, *arg, etc. + context: current expression context, used for inference. + arg_names: names of arguments (optional) + callable_node: associate the inferred callable type to this node, + if specified + callable_name: Fully-qualified name of the function/method to call, + or None if unavailable (examples: 'builtins.open', 'typing.Mapping.get') + object_type: If callable_name refers to a method, the type of the object + on which the method is being called + """ + callee = get_proper_type(callee) + + if isinstance(callee, CallableType): + if callee.variables: + overloaded = self.is_generic_decorator_overload_call(callee, args) + if overloaded is not None: + # Special casing for inline application of generic callables to overloads. + # Supporting general case would be tricky, but this should cover 95% of cases. + overloaded_result = self.handle_decorator_overload_call( + callee, overloaded, context + ) + if overloaded_result is not None: + return overloaded_result + + return self.check_callable_call( + callee, + args, + arg_kinds, + context, + arg_names, + callable_node, + callable_name, + object_type, + ) elif isinstance(callee, Overloaded): - # Type check arguments in empty context. They will be checked again - # later in a context derived from the signature; these types are - # only used to pick a signature variant. - self.msg.disable_errors() - arg_types = self.infer_arg_types_in_context(None, args) - self.msg.enable_errors() - - target = self.overload_call_target(arg_types, is_var_arg, - callee, context, - messages=arg_messages) - return self.check_call(target, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - elif isinstance(callee, AnyType) or self.chk.is_dynamic_function(): - self.infer_arg_types_in_context(None, args) - return AnyType(), AnyType() + return self.check_overload_call( + callee, args, arg_kinds, arg_names, callable_name, object_type, context + ) + elif isinstance(callee, AnyType) or not self.chk.in_checked_function(): + return self.check_any_type_call(args, callee) elif isinstance(callee, UnionType): - self.msg.disable_type_names += 1 - results = [self.check_call(subtype, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - for subtype in callee.items] - self.msg.disable_type_names -= 1 - return (UnionType.make_simplified_union([res[0] for res in results]), - callee) + return self.check_union_call(callee, args, arg_kinds, arg_names, context) + elif isinstance(callee, Instance): + call_function = analyze_member_access( + "__call__", + callee, + context, + is_lvalue=False, + is_super=False, + is_operator=True, + msg=self.msg, + original_type=original_type or callee, + chk=self.chk, + in_literal_context=self.is_literal_context(), + ) + callable_name = callee.type.fullname + ".__call__" + # Apply method signature hook, if one exists + call_function = self.transform_callee_type( + callable_name, call_function, args, arg_kinds, context, arg_names, callee + ) + result = self.check_call( + call_function, + args, + arg_kinds, + context, + arg_names, + callable_node, + callable_name, + callee, + ) + if callable_node: + # check_call() stored "call_function" as the type, which is incorrect. + # Override the type. + self.chk.store_type(callable_node, callee) + return result + elif isinstance(callee, TypeVarType): + return self.check_call( + callee.upper_bound, args, arg_kinds, context, arg_names, callable_node + ) + elif isinstance(callee, TypeType): + item = self.analyze_type_type_callee(callee.item, context) + return self.check_call(item, args, arg_kinds, context, arg_names, callable_node) + elif isinstance(callee, TupleType): + return self.check_call( + tuple_fallback(callee), + args, + arg_kinds, + context, + arg_names, + callable_node, + callable_name, + object_type, + original_type=callee, + ) else: - return self.msg.not_callable(callee, context), AnyType() - - def infer_arg_types_in_context(self, callee: Callable, - args: List[Node]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. + return self.msg.not_callable(callee, context), AnyType(TypeOfAny.from_error) + + def check_callable_call( + self, + callee: CallableType, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + arg_names: Sequence[str | None] | None, + callable_node: Expression | None, + callable_name: str | None, + object_type: Type | None, + ) -> tuple[Type, Type]: + """Type check a call that targets a callable value. + + See the docstring of check_call for more information. """ - # TODO Always called with callee as None, i.e. empty context. - res = [] # type: List[Type] - - fixed = len(args) - if callee: - fixed = min(fixed, callee.max_fixed_args()) - - arg_type = None # type: Type - ctx = None # type: Type - for i, arg in enumerate(args): - if i < fixed: - if callee and i < len(callee.arg_types): - ctx = callee.arg_types[i] - arg_type = self.accept(arg, ctx) - else: - if callee and callee.is_var_arg: - arg_type = self.accept(arg, callee.arg_types[-1]) + # Always unpack **kwargs before checking a call. + callee = callee.with_unpacked_kwargs().with_normalized_var_args() + if callable_name is None and callee.name: + callable_name = callee.name + ret_type = get_proper_type(callee.ret_type) + if callee.is_type_obj() and isinstance(ret_type, Instance): + callable_name = ret_type.type.fullname + if isinstance(callable_node, RefExpr) and callable_node.fullname in ENUM_BASES: + # An Enum() call that failed SemanticAnalyzerPass2.check_enum_call(). + return callee.ret_type, callee + + if ( + callee.is_type_obj() + and callee.type_object().is_protocol + # Exception for Type[...] + and not callee.from_type_type + ): + self.chk.fail( + message_registry.CANNOT_INSTANTIATE_PROTOCOL.format(callee.type_object().name), + context, + ) + elif ( + callee.is_type_obj() + and callee.type_object().is_abstract + # Exception for Type[...] + and not callee.from_type_type + and not callee.type_object().fallback_to_any + ): + type = callee.type_object() + # Determine whether the implicitly abstract attributes are functions with + # None-compatible return types. + abstract_attributes: dict[str, bool] = {} + for attr_name, abstract_status in type.abstract_attributes: + if abstract_status == IMPLICITLY_ABSTRACT: + abstract_attributes[attr_name] = self.can_return_none(type, attr_name) else: - arg_type = self.accept(arg) - if isinstance(arg_type, ErasedType): - res.append(NoneTyp()) + abstract_attributes[attr_name] = False + self.msg.cannot_instantiate_abstract_class( + callee.type_object().name, abstract_attributes, context + ) + + var_arg = callee.var_arg() + if var_arg and isinstance(var_arg.typ, UnpackType): + # It is hard to support multiple variadic unpacks (except for old-style *args: int), + # fail gracefully to avoid crashes later. + seen_unpack = False + for arg, arg_kind in zip(args, arg_kinds): + if arg_kind != ARG_STAR: + continue + arg_type = get_proper_type(self.accept(arg)) + if not isinstance(arg_type, TupleType) or any( + isinstance(t, UnpackType) for t in arg_type.items + ): + if seen_unpack: + self.msg.fail( + "Passing multiple variadic unpacks in a call is not supported", + context, + code=codes.CALL_ARG, + ) + return AnyType(TypeOfAny.from_error), callee + seen_unpack = True + + formal_to_actual = map_actuals_to_formals( + arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: self.accept(args[i]), + ) + + # This is tricky: return type may contain its own type variables, like in + # def [S] (S) -> def [T] (T) -> tuple[S, T], so we need to update their ids + # to avoid possible id clashes if this call itself appears in a generic + # function body. + ret_type = get_proper_type(callee.ret_type) + if isinstance(ret_type, CallableType) and ret_type.variables: + fresh_ret_type = freshen_all_functions_type_vars(callee.ret_type) + freeze_all_type_vars(fresh_ret_type) + callee = callee.copy_modified(ret_type=fresh_ret_type) + + if callee.is_generic(): + need_refresh = any( + isinstance(v, (ParamSpecType, TypeVarTupleType)) for v in callee.variables + ) + callee = freshen_function_type_vars(callee) + callee = self.infer_function_type_arguments_using_context(callee, context) + if need_refresh: + # Argument kinds etc. may have changed due to + # ParamSpec or TypeVarTuple variables being replaced with an arbitrary + # number of arguments; recalculate actual-to-formal map + formal_to_actual = map_actuals_to_formals( + arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: self.accept(args[i]), + ) + callee = self.infer_function_type_arguments( + callee, args, arg_kinds, arg_names, formal_to_actual, need_refresh, context + ) + if need_refresh: + formal_to_actual = map_actuals_to_formals( + arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: self.accept(args[i]), + ) + + param_spec = callee.param_spec() + if ( + param_spec is not None + and arg_kinds == [ARG_STAR, ARG_STAR2] + and len(formal_to_actual) == 2 + ): + arg1 = self.accept(args[0]) + arg2 = self.accept(args[1]) + if ( + isinstance(arg1, ParamSpecType) + and isinstance(arg2, ParamSpecType) + and arg1.flavor == ParamSpecFlavor.ARGS + and arg2.flavor == ParamSpecFlavor.KWARGS + and arg1.id == arg2.id == param_spec.id + ): + return callee.ret_type, callee + + arg_types = self.infer_arg_types_in_context(callee, args, arg_kinds, formal_to_actual) + + self.check_argument_count( + callee, + arg_types, + arg_kinds, + arg_names, + formal_to_actual, + context, + object_type, + callable_name, + ) + + self.check_argument_types( + arg_types, arg_kinds, args, callee, formal_to_actual, context, object_type=object_type + ) + + if ( + callee.is_type_obj() + and (len(arg_types) == 1) + and is_equivalent(callee.ret_type, self.named_type("builtins.type")) + ): + callee = callee.copy_modified(ret_type=TypeType.make_normalized(arg_types[0])) + + if callable_node: + # Store the inferred callable type. + self.chk.store_type(callable_node, callee) + + if callable_name and ( + (object_type is None and self.plugin.get_function_hook(callable_name)) + or (object_type is not None and self.plugin.get_method_hook(callable_name)) + ): + new_ret_type = self.apply_function_plugin( + callee, + arg_kinds, + arg_types, + arg_names, + formal_to_actual, + args, + callable_name, + object_type, + context, + ) + callee = callee.copy_modified(ret_type=new_ret_type) + return callee.ret_type, callee + + def can_return_none(self, type: TypeInfo, attr_name: str) -> bool: + """Is the given attribute a method with a None-compatible return type? + + Overloads are only checked if there is an implementation. + """ + if not state.strict_optional: + # If strict-optional is not set, is_subtype(NoneType(), T) is always True. + # So, we cannot do anything useful here in that case. + return False + for base in type.mro: + symnode = base.names.get(attr_name) + if symnode is None: + continue + node = symnode.node + if isinstance(node, OverloadedFuncDef): + node = node.impl + if isinstance(node, Decorator): + node = node.func + if isinstance(node, FuncDef): + if node.type is not None: + assert isinstance(node.type, CallableType) + return is_subtype(NoneType(), node.type.ret_type) + return False + + def analyze_type_type_callee(self, item: ProperType, context: Context) -> Type: + """Analyze the callee X in X(...) where X is Type[item]. + + Return a Y that we can pass to check_call(Y, ...). + """ + if isinstance(item, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=item) + if isinstance(item, Instance): + res = type_object_type(item.type, self.named_type) + if isinstance(res, CallableType): + res = res.copy_modified(from_type_type=True) + expanded = expand_type_by_instance(res, item) + if isinstance(expanded, CallableType): + # Callee of the form Type[...] should never be generic, only + # proper class objects can be. + expanded = expanded.copy_modified(variables=[]) + return expanded + if isinstance(item, UnionType): + return UnionType( + [ + self.analyze_type_type_callee(get_proper_type(tp), context) + for tp in item.relevant_items() + ], + item.line, + ) + if isinstance(item, TypeVarType): + # Pretend we're calling the typevar's upper bound, + # i.e. its constructor (a poor approximation for reality, + # but better than AnyType...), but replace the return type + # with typevar. + callee = self.analyze_type_type_callee(get_proper_type(item.upper_bound), context) + callee = get_proper_type(callee) + if isinstance(callee, CallableType): + callee = callee.copy_modified(ret_type=item) + elif isinstance(callee, Overloaded): + callee = Overloaded([c.copy_modified(ret_type=item) for c in callee.items]) + return callee + # We support Type of namedtuples but not of tuples in general + if isinstance(item, TupleType) and tuple_fallback(item).type.fullname != "builtins.tuple": + return self.analyze_type_type_callee(tuple_fallback(item), context) + if isinstance(item, TypedDictType): + return self.typeddict_callable_from_context(item) + + self.msg.unsupported_type_type(item, context) + return AnyType(TypeOfAny.from_error) + + def infer_arg_types_in_empty_context(self, args: list[Expression]) -> list[Type]: + """Infer argument expression types in an empty context. + + In short, we basically recurse on each argument without considering + in what context the argument was called. + """ + res: list[Type] = [] + + for arg in args: + arg_type = self.accept(arg) + if has_erased_component(arg_type): + res.append(NoneType()) else: res.append(arg_type) return res - def infer_arg_types_in_context2( - self, callee: Callable, args: List[Node], arg_kinds: List[int], - formal_to_actual: List[List[int]]) -> List[Type]: + def infer_more_unions_for_recursive_type(self, type_context: Type) -> bool: + """Adjust type inference of unions if type context has a recursive type. + + Return the old state. The caller must assign it to type_state.infer_unions + afterwards. + + This is a hack to better support inference for recursive types. + + Note: This is performance-sensitive and must not be a context manager + until mypyc supports them better. + """ + old = type_state.infer_unions + if has_recursive_types(type_context): + type_state.infer_unions = True + return old + + def infer_arg_types_in_context( + self, + callee: CallableType, + args: list[Expression], + arg_kinds: list[ArgKind], + formal_to_actual: list[list[int]], + ) -> list[Type]: """Infer argument expression types using a callable type as context. For example, if callee argument 2 has type List[int], infer the - argument exprsession with List[int] type context. + argument expression with List[int] type context. Returns the inferred types of *actual arguments*. """ - res = [None] * len(args) # type: List[Type] + res: list[Type | None] = [None] * len(args) for i, actuals in enumerate(formal_to_actual): for ai in actuals: - if arg_kinds[ai] != nodes.ARG_STAR: - res[ai] = self.accept(args[ai], callee.arg_types[i]) + if not arg_kinds[ai].is_star(): + arg_type = callee.arg_types[i] + # When the outer context for a function call is known to be recursive, + # we solve type constraints inferred from arguments using unions instead + # of joins. This is a bit arbitrary, but in practice it works for most + # cases. A cleaner alternative would be to switch to single bin type + # inference, but this is a lot of work. + old = self.infer_more_unions_for_recursive_type(arg_type) + res[ai] = self.accept(args[ai], arg_type) + # We need to manually restore union inference state, ugh. + type_state.infer_unions = old # Fill in the rest of the argument types. for i, t in enumerate(res): if not t: res[i] = self.accept(args[i]) - return res + assert all(tp is not None for tp in res) + return cast(List[Type], res) def infer_function_type_arguments_using_context( - self, callable: Callable, error_context: Context) -> Callable: + self, callable: CallableType, error_context: Context + ) -> CallableType: """Unify callable return type to type context to infer type vars. For example, if the return type is set[t] where 't' is a type variable of callable, and if the context is set[int], return callable modified by substituting 't' with 'int'. """ - ctx = self.chk.type_context[-1] + ctx = self.type_context[-1] if not ctx: return callable - # The return type may have references to function type variables that + # The return type may have references to type metavariables that # we are inferring right now. We must consider them as indeterminate # and they are not potential results; thus we replace them with the # special ErasedType type. On the other hand, class type variables are # valid results. - erased_ctx = replace_func_type_vars(ctx, ErasedType()) + erased_ctx = replace_meta_vars(ctx, ErasedType()) ret_type = callable.ret_type - if isinstance(ret_type, TypeVar): - if ret_type.values or (not isinstance(ctx, Instance) or - not cast(Instance, ctx).args): - # The return type is a type variable. If it has values, we can't easily restrict - # type inference to conform to the valid values. If it's unrestricted, we could - # infer a too general type for the type variable if we use context, and this could - # result in confusing and spurious type errors elsewhere. - # - # Give up and just use function arguments for type inference. As an exception, - # if the context is a generic instance type, actually use it as context, as - # this *seems* to usually be the reasonable thing to do. - # - # See also github issues #462 and #360. - ret_type = NoneTyp() - args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) - # Only substite non-None and non-erased types. - new_args = [] # type: List[Type] + if is_overlapping_none(ret_type) and is_overlapping_none(ctx): + # If both the context and the return type are optional, unwrap the optional, + # since in 99% cases this is what a user expects. In other words, we replace + # Optional[T] <: Optional[int] + # with + # T <: int + # while the former would infer T <: Optional[int]. + ret_type = remove_optional(ret_type) + erased_ctx = remove_optional(erased_ctx) + # + # TODO: Instead of this hack and the one below, we need to use outer and + # inner contexts at the same time. This is however not easy because of two + # reasons: + # * We need to support constraints like [1 <: 2, 2 <: X], i.e. with variables + # on both sides. (This is not too hard.) + # * We need to update all the inference "infrastructure", so that all + # variables in an expression are inferred at the same time. + # (And this is hard, also we need to be careful with lambdas that require + # two passes.) + if isinstance(ret_type, TypeVarType): + # Another special case: the return type is a type variable. If it's unrestricted, + # we could infer a too general type for the type variable if we use context, + # and this could result in confusing and spurious type errors elsewhere. + # + # So we give up and just use function arguments for type inference, with just two + # exceptions: + # + # 1. If the context is a generic instance type, actually use it as context, as + # this *seems* to usually be the reasonable thing to do. + # + # See also github issues #462 and #360. + # + # 2. If the context is some literal type, we want to "propagate" that information + # down so that we infer a more precise type for literal expressions. For example, + # the expression `3` normally has an inferred type of `builtins.int`: but if it's + # in a literal context like below, we want it to infer `Literal[3]` instead. + # + # def expects_literal(x: Literal[3]) -> None: pass + # def identity(x: T) -> T: return x + # + # expects_literal(identity(3)) # Should type-check + # TODO: we may want to add similar exception if all arguments are lambdas, since + # in this case external context is almost everything we have. + if not is_generic_instance(ctx) and not is_literal_type_like(ctx): + return callable.copy_modified() + args = infer_type_arguments( + callable.variables, ret_type, erased_ctx, skip_unsatisfied=True + ) + # Only substitute non-Uninhabited and non-erased types. + new_args: list[Type | None] = [] for arg in args: - if isinstance(arg, NoneTyp) or has_erased_component(arg): + if has_uninhabited_component(arg) or has_erased_component(arg): new_args.append(None) else: new_args.append(arg) - return cast(Callable, self.apply_generic_arguments(callable, new_args, - error_context)) - - def infer_function_type_arguments(self, callee_type: Callable, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - context: Context) -> Callable: + # Don't show errors after we have only used the outer context for inference. + # We will use argument context to infer more variables. + return self.apply_generic_arguments( + callable, new_args, error_context, skip_unsatisfied=True + ) + + def infer_function_type_arguments( + self, + callee_type: CallableType, + args: list[Expression], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + formal_to_actual: list[list[int]], + need_refresh: bool, + context: Context, + ) -> CallableType: """Infer the type arguments for a generic callee type. Infer based on the types of arguments. - Return a derived callable type that has the arguments applied (and - stored as implicit type arguments). + Return a derived callable type that has the arguments applied. """ - if not self.chk.is_dynamic_function(): + if self.chk.in_checked_function(): # Disable type errors during type inference. There may be errors # due to partial available context information at this time, but # these errors can be safely ignored as the arguments will be # inferred again later. - self.msg.disable_errors() - - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - self.msg.enable_errors() + with self.msg.filter_errors(): + arg_types = self.infer_arg_types_in_context( + callee_type, args, arg_kinds, formal_to_actual + ) arg_pass_nums = self.get_arg_infer_passes( - callee_type.arg_types, formal_to_actual, len(args)) + callee_type, args, arg_types, formal_to_actual, len(args) + ) - pass1_args = [] # type: List[Type] + pass1_args: list[Type | None] = [] for i, arg in enumerate(arg_types): if arg_pass_nums[i] > 1: pass1_args.append(None) else: pass1_args.append(arg) - inferred_args = infer_function_type_arguments( - callee_type, pass1_args, arg_kinds, formal_to_actual) # type: List[Type] + inferred_args, _ = infer_function_type_arguments( + callee_type, + pass1_args, + arg_kinds, + arg_names, + formal_to_actual, + context=self.argument_infer_context(), + strict=self.chk.in_checked_function(), + ) if 2 in arg_pass_nums: # Second pass of type inference. - (callee_type, - inferred_args) = self.infer_function_type_arguments_pass2( - callee_type, args, arg_kinds, formal_to_actual, - inferred_args, context) + (callee_type, inferred_args) = self.infer_function_type_arguments_pass2( + callee_type, + args, + arg_kinds, + arg_names, + formal_to_actual, + inferred_args, + need_refresh, + context, + ) + + if ( + callee_type.special_sig == "dict" + and len(inferred_args) == 2 + and (ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds) + ): + # HACK: Infer str key type for dict(...) with keyword args. The type system + # can't represent this so we special case it, as this is a pretty common + # thing. This doesn't quite work with all possible subclasses of dict + # if they shuffle type variables around, as we assume that there is a 1-1 + # correspondence with dict type variables. This is a marginal issue and + # a little tricky to fix so it's left unfixed for now. + first_arg = get_proper_type(inferred_args[0]) + if isinstance(first_arg, (NoneType, UninhabitedType)): + inferred_args[0] = self.named_type("builtins.str") + elif not first_arg or not is_subtype(self.named_type("builtins.str"), first_arg): + self.chk.fail(message_registry.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, context) + + if not self.chk.options.old_type_inference and any( + a is None + or isinstance(get_proper_type(a), UninhabitedType) + or set(get_type_vars(a)) & set(callee_type.variables) + for a in inferred_args + ): + if need_refresh: + # Technically we need to refresh formal_to_actual after *each* inference pass, + # since each pass can expand ParamSpec or TypeVarTuple. Although such situations + # are very rare, not doing this can cause crashes. + formal_to_actual = map_actuals_to_formals( + arg_kinds, + arg_names, + callee_type.arg_kinds, + callee_type.arg_names, + lambda a: self.accept(args[a]), + ) + # If the regular two-phase inference didn't work, try inferring type + # variables while allowing for polymorphic solutions, i.e. for solutions + # potentially involving free variables. + # TODO: support the similar inference for return type context. + poly_inferred_args, free_vars = infer_function_type_arguments( + callee_type, + arg_types, + arg_kinds, + arg_names, + formal_to_actual, + context=self.argument_infer_context(), + strict=self.chk.in_checked_function(), + allow_polymorphic=True, + ) + poly_callee_type = self.apply_generic_arguments( + callee_type, poly_inferred_args, context + ) + # Try applying inferred polymorphic type if possible, e.g. Callable[[T], T] can + # be interpreted as def [T] (T) -> T, but dict[T, T] cannot be expressed. + applied = applytype.apply_poly(poly_callee_type, free_vars) + if applied is not None and all( + a is not None and not isinstance(get_proper_type(a), UninhabitedType) + for a in poly_inferred_args + ): + freeze_all_type_vars(applied) + return applied + # If it didn't work, erase free variables as uninhabited, to avoid confusing errors. + unknown = UninhabitedType() + unknown.ambiguous = True + inferred_args = [ + ( + expand_type( + a, {v.id: unknown for v in list(callee_type.variables) + free_vars} + ) + if a is not None + else None + ) + for a in poly_inferred_args + ] else: # In dynamically typed functions use implicit 'Any' types for # type variables. - inferred_args = [AnyType()] * len(callee_type.variables) - return self.apply_inferred_arguments(callee_type, inferred_args, - context) + inferred_args = [AnyType(TypeOfAny.unannotated)] * len(callee_type.variables) + return self.apply_inferred_arguments(callee_type, inferred_args, context) def infer_function_type_arguments_pass2( - self, callee_type: Callable, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - inferred_args: List[Type], - context: Context) -> Tuple[Callable, List[Type]]: + self, + callee_type: CallableType, + args: list[Expression], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + formal_to_actual: list[list[int]], + old_inferred_args: Sequence[Type | None], + need_refresh: bool, + context: Context, + ) -> tuple[CallableType, list[Type | None]]: """Perform second pass of generic function type argument inference. - The second pass is needed for arguments with types such as Function[[T], S], + The second pass is needed for arguments with types such as Callable[[T], S], where both T and S are type variables, when the actual argument is a lambda with inferred types. The idea is to infer the type variable T in the first pass (based on the types of other arguments). This lets @@ -378,23 +2211,50 @@ def infer_function_type_arguments_pass2( # None or erased types in inferred types mean that there was not enough # information to infer the argument. Replace them with None values so # that they are not applied yet below. - for i, arg in enumerate(inferred_args): - if isinstance(arg, NoneTyp) or isinstance(arg, ErasedType): + inferred_args = list(old_inferred_args) + for i, arg in enumerate(get_proper_types(inferred_args)): + if isinstance(arg, (NoneType, UninhabitedType)) or has_erased_component(arg): inferred_args[i] = None - - callee_type = cast(Callable, self.apply_generic_arguments( - callee_type, inferred_args, context)) - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - inferred_args = infer_function_type_arguments( - callee_type, arg_types, arg_kinds, formal_to_actual) + callee_type = self.apply_generic_arguments(callee_type, inferred_args, context) + if need_refresh: + formal_to_actual = map_actuals_to_formals( + arg_kinds, + arg_names, + callee_type.arg_kinds, + callee_type.arg_names, + lambda a: self.accept(args[a]), + ) + + # Same as during first pass, disable type errors (we still have partial context). + with self.msg.filter_errors(): + arg_types = self.infer_arg_types_in_context( + callee_type, args, arg_kinds, formal_to_actual + ) + + inferred_args, _ = infer_function_type_arguments( + callee_type, + arg_types, + arg_kinds, + arg_names, + formal_to_actual, + context=self.argument_infer_context(), + ) return callee_type, inferred_args - def get_arg_infer_passes(self, arg_types: List[Type], - formal_to_actual: List[List[int]], - num_actuals: int) -> List[int]: + def argument_infer_context(self) -> ArgumentInferContext: + return ArgumentInferContext( + self.chk.named_type("typing.Mapping"), self.chk.named_type("typing.Iterable") + ) + + def get_arg_infer_passes( + self, + callee: CallableType, + args: list[Expression], + arg_types: list[Type], + formal_to_actual: list[list[int]], + num_actuals: int, + ) -> list[int]: """Return pass numbers for args for two-pass argument type inference. For each actual, the pass number is either 1 (first pass) or 2 (second @@ -404,15 +2264,39 @@ def get_arg_infer_passes(self, arg_types: List[Type], lambdas more effectively. """ res = [1] * num_actuals - for i, arg in enumerate(arg_types): - if arg.accept(ArgInferSecondPassQuery()): + for i, arg in enumerate(callee.arg_types): + skip_param_spec = False + p_formal = get_proper_type(callee.arg_types[i]) + if isinstance(p_formal, CallableType) and p_formal.param_spec(): + for j in formal_to_actual[i]: + p_actual = get_proper_type(arg_types[j]) + # This is an exception from the usual logic where we put generic Callable + # arguments in the second pass. If we have a non-generic actual, it is + # likely to infer good constraints, for example if we have: + # def run(Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... + # def test(x: int, y: int) -> int: ... + # run(test, 1, 2) + # we will use `test` for inference, since it will allow to infer also + # argument *names* for P <: [x: int, y: int]. + if isinstance(p_actual, Instance): + call_method = find_member("__call__", p_actual, p_actual, is_operator=True) + if call_method is not None: + p_actual = get_proper_type(call_method) + if ( + isinstance(p_actual, CallableType) + and not p_actual.variables + and not isinstance(args[j], LambdaExpr) + ): + skip_param_spec = True + break + if not skip_param_spec and arg.accept(ArgInferSecondPassQuery()): for j in formal_to_actual[i]: res[j] = 2 return res - def apply_inferred_arguments(self, callee_type: Callable, - inferred_args: List[Type], - context: Context) -> Callable: + def apply_inferred_arguments( + self, callee_type: CallableType, inferred_args: Sequence[Type | None], context: Context + ) -> CallableType: """Apply inferred values of type arguments to a generic function. Inferred_args contains the values of function type arguments. @@ -421,318 +2305,1212 @@ def apply_inferred_arguments(self, callee_type: Callable, # case assume that all variables have type Any to avoid extra # bogus error messages. for i, inferred_type in enumerate(inferred_args): - if not inferred_type: + if not inferred_type or has_erased_component(inferred_type): # Could not infer a non-trivial type for a type variable. - self.msg.could_not_infer_type_arguments( - callee_type, i + 1, context) - inferred_args = [AnyType()] * len(inferred_args) + self.msg.could_not_infer_type_arguments(callee_type, i + 1, context) + inferred_args = [AnyType(TypeOfAny.from_error)] * len(inferred_args) # Apply the inferred types to the function type. In this case the - # return type must be Callable, since we give the right number of type + # return type must be CallableType, since we give the right number of type # arguments. - return cast(Callable, self.apply_generic_arguments(callee_type, - inferred_args, context)) + return self.apply_generic_arguments(callee_type, inferred_args, context) + + def check_argument_count( + self, + callee: CallableType, + actual_types: list[Type], + actual_kinds: list[ArgKind], + actual_names: Sequence[str | None] | None, + formal_to_actual: list[list[int]], + context: Context | None, + object_type: Type | None = None, + callable_name: str | None = None, + ) -> bool: + """Check that there is a value for all required arguments to a function. + + Also check that there are no duplicate values for arguments. Report found errors + using 'messages' if it's not None. If 'messages' is given, 'context' must also be given. + + Return False if there were any errors. Otherwise return True + """ + if context is None: + # Avoid "is None" checks + context = TempNode(AnyType(TypeOfAny.special_form)) - def check_argument_count(self, callee: Callable, actual_types: List[Type], - actual_kinds: List[int], actual_names: List[str], - formal_to_actual: List[List[int]], - context: Context) -> None: - """Check that the number of arguments to a function are valid. + # TODO(jukka): We could return as soon as we find an error if messages is None. - Also check that there are no duplicate values for arguments. + # Collect dict of all actual arguments matched to formal arguments, with occurrence count + all_actuals: dict[int, int] = {} + for actuals in formal_to_actual: + for a in actuals: + all_actuals[a] = all_actuals.get(a, 0) + 1 + + ok, is_unexpected_arg_error = self.check_for_extra_actual_arguments( + callee, actual_types, actual_kinds, actual_names, all_actuals, context + ) + + # Check for too many or few values for formals. + for i, kind in enumerate(callee.arg_kinds): + if kind.is_required() and not formal_to_actual[i] and not is_unexpected_arg_error: + # No actual for a mandatory formal + if kind.is_positional(): + self.msg.too_few_arguments(callee, context, actual_names) + if object_type and callable_name and "." in callable_name: + self.missing_classvar_callable_note(object_type, callable_name, context) + else: + argname = callee.arg_names[i] or "?" + self.msg.missing_named_argument(callee, context, argname) + ok = False + elif not kind.is_star() and is_duplicate_mapping( + formal_to_actual[i], actual_types, actual_kinds + ): + if self.chk.in_checked_function() or isinstance( + get_proper_type(actual_types[formal_to_actual[i][0]]), TupleType + ): + self.msg.duplicate_argument_value(callee, i, context) + ok = False + elif ( + kind.is_named() + and formal_to_actual[i] + and actual_kinds[formal_to_actual[i][0]] not in [nodes.ARG_NAMED, nodes.ARG_STAR2] + ): + # Positional argument when expecting a keyword argument. + self.msg.too_many_positional_arguments(callee, context) + ok = False + elif ( + callee.param_spec() is not None + and not formal_to_actual[i] + and callee.special_sig != "partial" + ): + self.msg.too_few_arguments(callee, context, actual_names) + ok = False + return ok + + def check_for_extra_actual_arguments( + self, + callee: CallableType, + actual_types: list[Type], + actual_kinds: list[ArgKind], + actual_names: Sequence[str | None] | None, + all_actuals: dict[int, int], + context: Context, + ) -> tuple[bool, bool]: + """Check for extra actual arguments. + + Return tuple (was everything ok, + was there an extra keyword argument error [used to avoid duplicate errors]). """ - formal_kinds = callee.arg_kinds - # Collect list of all actual arguments matched to formal arguments. - all_actuals = [] # type: List[int] - for actuals in formal_to_actual: - all_actuals.extend(actuals) + is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors + ok = True # False if we've found any error - is_error = False # Keep track of errors to avoid duplicate errors. for i, kind in enumerate(actual_kinds): - if i not in all_actuals and ( - kind != nodes.ARG_STAR or - not is_empty_tuple(actual_types[i])): + if ( + i not in all_actuals + and + # We accept the other iterables than tuple (including Any) + # as star arguments because they could be empty, resulting no arguments. + (kind != nodes.ARG_STAR or is_non_empty_tuple(actual_types[i])) + and + # Accept all types for double-starred arguments, because they could be empty + # dictionaries and we can't tell it from their types + kind != nodes.ARG_STAR2 + ): # Extra actual: not matched by a formal argument. + ok = False if kind != nodes.ARG_NAMED: self.msg.too_many_arguments(callee, context) else: - self.msg.unexpected_keyword_argument( - callee, actual_names[i], context) - is_error = True - elif kind == nodes.ARG_STAR and ( - nodes.ARG_STAR not in formal_kinds): - actual_type = actual_types[i] - if isinstance(actual_type, TupleType): - if all_actuals.count(i) < len(actual_type.items): - # Too many tuple items as some did not match. - self.msg.too_many_arguments(callee, context) - # *args can be applied even if the function takes a fixed + assert actual_names, "Internal error: named kinds without names given" + act_name = actual_names[i] + assert act_name is not None + act_type = actual_types[i] + self.msg.unexpected_keyword_argument(callee, act_name, act_type, context) + is_unexpected_arg_error = True + elif ( + kind == nodes.ARG_STAR and nodes.ARG_STAR not in callee.arg_kinds + ) or kind == nodes.ARG_STAR2: + actual_type = get_proper_type(actual_types[i]) + if isinstance(actual_type, (TupleType, TypedDictType)): + if all_actuals.get(i, 0) < len(actual_type.items): + # Too many tuple/dict items as some did not match. + if kind != nodes.ARG_STAR2 or not isinstance(actual_type, TypedDictType): + self.msg.too_many_arguments(callee, context) + else: + self.msg.too_many_arguments_from_typed_dict( + callee, actual_type, context + ) + is_unexpected_arg_error = True + ok = False + # *args/**kwargs can be applied even if the function takes a fixed # number of positional arguments. This may succeed at runtime. - for i, kind in enumerate(formal_kinds): - if kind == nodes.ARG_POS and (not formal_to_actual[i] and - not is_error): - # No actual for a mandatory positional formal. - self.msg.too_few_arguments(callee, context) - elif kind in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED] and is_duplicate_mapping( - formal_to_actual[i], actual_kinds): - self.msg.duplicate_argument_value(callee, i, context) - elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and - actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): - # Positional argument when expecting a keyword argument. - self.msg.too_many_positional_arguments(callee, context) - - def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], - callee: Callable, - formal_to_actual: List[List[int]], - context: Context, - messages: MessageBuilder = None) -> None: + return ok, is_unexpected_arg_error + + def missing_classvar_callable_note( + self, object_type: Type, callable_name: str, context: Context + ) -> None: + if isinstance(object_type, ProperType) and isinstance(object_type, Instance): + _, var_name = callable_name.rsplit(".", maxsplit=1) + node = object_type.type.get(var_name) + if node is not None and isinstance(node.node, Var): + if not node.node.is_inferred and not node.node.is_classvar: + self.msg.note( + f'"{var_name}" is considered instance variable,' + " to make it class variable use ClassVar[...]", + context, + ) + + def check_argument_types( + self, + arg_types: list[Type], + arg_kinds: list[ArgKind], + args: list[Expression], + callee: CallableType, + formal_to_actual: list[list[int]], + context: Context, + check_arg: ArgChecker | None = None, + object_type: Type | None = None, + ) -> None: """Check argument types against a callable type. Report errors if the argument types are not compatible. + + The check_call docstring describes some of the arguments. """ - messages = messages or self.msg + check_arg = check_arg or self.check_arg # Keep track of consumed tuple *arg items. - tuple_counter = [0] + mapper = ArgTypeExpander(self.argument_infer_context()) + + for arg_type, arg_kind in zip(arg_types, arg_kinds): + arg_type = get_proper_type(arg_type) + if arg_kind == nodes.ARG_STAR and not self.is_valid_var_arg(arg_type): + self.msg.invalid_var_arg(arg_type, context) + if arg_kind == nodes.ARG_STAR2 and not self.is_valid_keyword_var_arg(arg_type): + is_mapping = is_subtype( + arg_type, self.chk.named_type("_typeshed.SupportsKeysAndGetItem") + ) + self.msg.invalid_keyword_var_arg(arg_type, is_mapping, context) + for i, actuals in enumerate(formal_to_actual): - for actual in actuals: - arg_type = arg_types[actual] + orig_callee_arg_type = get_proper_type(callee.arg_types[i]) + + # Checking the case that we have more than one item but the first argument + # is an unpack, so this would be something like: + # [Tuple[Unpack[Ts]], int] + # + # In this case we have to check everything together, we do this by re-unifying + # the suffices to the tuple, e.g. a single actual like + # Tuple[Unpack[Ts], int] + expanded_tuple = False + actual_kinds = [arg_kinds[a] for a in actuals] + if len(actuals) > 1: + p_actual_type = get_proper_type(arg_types[actuals[0]]) + if ( + isinstance(p_actual_type, TupleType) + and len(p_actual_type.items) == 1 + and isinstance(p_actual_type.items[0], UnpackType) + and actual_kinds == [nodes.ARG_STAR] + [nodes.ARG_POS] * (len(actuals) - 1) + ): + actual_types = [p_actual_type.items[0]] + [arg_types[a] for a in actuals[1:]] + if isinstance(orig_callee_arg_type, UnpackType): + p_callee_type = get_proper_type(orig_callee_arg_type.type) + if isinstance(p_callee_type, TupleType): + assert p_callee_type.items + callee_arg_types = p_callee_type.items + callee_arg_kinds = [nodes.ARG_STAR] + [nodes.ARG_POS] * ( + len(p_callee_type.items) - 1 + ) + expanded_tuple = True + + if not expanded_tuple: + actual_types = [arg_types[a] for a in actuals] + if isinstance(orig_callee_arg_type, UnpackType): + unpacked_type = get_proper_type(orig_callee_arg_type.type) + if isinstance(unpacked_type, TupleType): + inner_unpack_index = find_unpack_in_list(unpacked_type.items) + if inner_unpack_index is None: + callee_arg_types = unpacked_type.items + callee_arg_kinds = [ARG_POS] * len(actuals) + else: + inner_unpack = unpacked_type.items[inner_unpack_index] + assert isinstance(inner_unpack, UnpackType) + inner_unpacked_type = get_proper_type(inner_unpack.type) + if isinstance(inner_unpacked_type, TypeVarTupleType): + # This branch mimics the expanded_tuple case above but for + # the case where caller passed a single * unpacked tuple argument. + callee_arg_types = unpacked_type.items + callee_arg_kinds = [ + ARG_POS if i != inner_unpack_index else ARG_STAR + for i in range(len(unpacked_type.items)) + ] + else: + # We assume heterogeneous tuples are desugared earlier. + assert isinstance(inner_unpacked_type, Instance) + assert inner_unpacked_type.type.fullname == "builtins.tuple" + callee_arg_types = ( + unpacked_type.items[:inner_unpack_index] + + [inner_unpacked_type.args[0]] + * (len(actuals) - len(unpacked_type.items) + 1) + + unpacked_type.items[inner_unpack_index + 1 :] + ) + callee_arg_kinds = [ARG_POS] * len(actuals) + elif isinstance(unpacked_type, TypeVarTupleType): + callee_arg_types = [orig_callee_arg_type] + callee_arg_kinds = [ARG_STAR] + else: + assert isinstance(unpacked_type, Instance) + assert unpacked_type.type.fullname == "builtins.tuple" + callee_arg_types = [unpacked_type.args[0]] * len(actuals) + callee_arg_kinds = [ARG_POS] * len(actuals) + else: + callee_arg_types = [orig_callee_arg_type] * len(actuals) + callee_arg_kinds = [callee.arg_kinds[i]] * len(actuals) + + assert len(actual_types) == len(actuals) == len(actual_kinds) + + if len(callee_arg_types) != len(actual_types): + if len(actual_types) > len(callee_arg_types): + self.chk.msg.too_many_arguments(callee, context) + else: + self.chk.msg.too_few_arguments(callee, context, None) + continue + + assert len(callee_arg_types) == len(actual_types) + assert len(callee_arg_types) == len(callee_arg_kinds) + for actual, actual_type, actual_kind, callee_arg_type, callee_arg_kind in zip( + actuals, actual_types, actual_kinds, callee_arg_types, callee_arg_kinds + ): + if actual_type is None: + continue # Some kind of error was already reported. # Check that a *arg is valid as varargs. - if (arg_kinds[actual] == nodes.ARG_STAR and - not self.is_valid_var_arg(arg_type)): - messages.invalid_var_arg(arg_type, context) - if (arg_kinds[actual] == nodes.ARG_STAR2 and - not self.is_valid_keyword_var_arg(arg_type)): - messages.invalid_keyword_var_arg(arg_type, context) - # Get the type of an inidividual actual argument (for *args - # and **args this is the item type, not the collection type). - actual_type = get_actual_type(arg_type, arg_kinds[actual], - tuple_counter) - self.check_arg(actual_type, arg_type, - callee.arg_types[i], - actual + 1, callee, context, messages) - - # There may be some remaining tuple varargs items that haven't - # been checked yet. Handle them. - if (callee.arg_kinds[i] == nodes.ARG_STAR and - arg_kinds[actual] == nodes.ARG_STAR and - isinstance(arg_types[actual], TupleType)): - tuplet = cast(TupleType, arg_types[actual]) - while tuple_counter[0] < len(tuplet.items): - actual_type = get_actual_type(arg_type, - arg_kinds[actual], - tuple_counter) - self.check_arg(actual_type, arg_type, - callee.arg_types[i], - actual + 1, callee, context, messages) - - def check_arg(self, caller_type: Type, original_caller_type: Type, - callee_type: Type, n: int, callee: Callable, - context: Context, messages: MessageBuilder) -> None: + expanded_actual = mapper.expand_actual_type( + actual_type, + actual_kind, + callee.arg_names[i], + callee_arg_kind, + allow_unpack=isinstance(callee_arg_type, UnpackType), + ) + check_arg( + expanded_actual, + actual_type, + actual_kind, + callee_arg_type, + actual + 1, + i + 1, + callee, + object_type, + args[actual], + context, + ) + + def check_arg( + self, + caller_type: Type, + original_caller_type: Type, + caller_kind: ArgKind, + callee_type: Type, + n: int, + m: int, + callee: CallableType, + object_type: Type | None, + context: Context, + outer_context: Context, + ) -> None: """Check the type of a single argument in a call.""" - if isinstance(caller_type, Void): - messages.does_not_return_value(caller_type, context) - elif not is_subtype(caller_type, callee_type): - messages.incompatible_argument(n, callee, original_caller_type, - context) - - def overload_call_target(self, arg_types: List[Type], is_var_arg: bool, - overload: Overloaded, context: Context, - messages: MessageBuilder = None) -> Type: - """Infer the correct overload item to call with given argument types. - - The return value may be Callable or AnyType (if an unique item - could not be determined). If is_var_arg is True, the caller - uses varargs. + caller_type = get_proper_type(caller_type) + original_caller_type = get_proper_type(original_caller_type) + callee_type = get_proper_type(callee_type) + + if isinstance(caller_type, DeletedType): + self.msg.deleted_as_rvalue(caller_type, context) + # Only non-abstract non-protocol class can be given where Type[...] is expected... + elif self.has_abstract_type_part(caller_type, callee_type): + self.msg.concrete_only_call(callee_type, context) + elif not is_subtype(caller_type, callee_type, options=self.chk.options): + code = self.msg.incompatible_argument( + n, + m, + callee, + original_caller_type, + caller_kind, + object_type=object_type, + context=context, + outer_context=outer_context, + ) + self.msg.incompatible_argument_note( + original_caller_type, callee_type, context, code=code + ) + if not self.msg.prefer_simple_messages(): + self.chk.check_possible_missing_await(caller_type, callee_type, context, code) + + def check_overload_call( + self, + callee: Overloaded, + args: list[Expression], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + callable_name: str | None, + object_type: Type | None, + context: Context, + ) -> tuple[Type, Type]: + """Checks a call to an overloaded function.""" + # Normalize unpacked kwargs before checking the call. + callee = callee.with_unpacked_kwargs() + arg_types = self.infer_arg_types_in_empty_context(args) + # Step 1: Filter call targets to remove ones where the argument counts don't match + plausible_targets = self.plausible_overload_call_targets( + arg_types, arg_kinds, arg_names, callee + ) + + # Step 2: If the arguments contain a union, we try performing union math first, + # instead of picking the first matching overload. + # This is because picking the first overload often ends up being too greedy: + # for example, when we have a fallback alternative that accepts an unrestricted + # typevar. See https://github.com/python/mypy/issues/4063 for related discussion. + erased_targets: list[CallableType] | None = None + unioned_result: tuple[Type, Type] | None = None + + # Determine whether we need to encourage union math. This should be generally safe, + # as union math infers better results in the vast majority of cases, but it is very + # computationally intensive. + none_type_var_overlap = self.possible_none_type_var_overlap(arg_types, plausible_targets) + union_interrupted = False # did we try all union combinations? + if any(self.real_union(arg) for arg in arg_types): + try: + with self.msg.filter_errors(): + unioned_return = self.union_overload_result( + plausible_targets, + args, + arg_types, + arg_kinds, + arg_names, + callable_name, + object_type, + none_type_var_overlap, + context, + ) + except TooManyUnions: + union_interrupted = True + else: + # Record if we succeeded. Next we need to see if maybe normal procedure + # gives a narrower type. + if unioned_return: + returns, inferred_types = zip(*unioned_return) + # Note that we use `combine_function_signatures` instead of just returning + # a union of inferred callables because for example a call + # Union[int -> int, str -> str](Union[int, str]) is invalid and + # we don't want to introduce internal inconsistencies. + unioned_result = ( + make_simplified_union(list(returns), context.line, context.column), + self.combine_function_signatures(get_proper_types(inferred_types)), + ) + + # Step 3: We try checking each branch one-by-one. + inferred_result = self.infer_overload_return_type( + plausible_targets, + args, + arg_types, + arg_kinds, + arg_names, + callable_name, + object_type, + context, + ) + # If any of checks succeed, stop early. + if inferred_result is not None and unioned_result is not None: + # Both unioned and direct checks succeeded, choose the more precise type. + if ( + is_subtype(inferred_result[0], unioned_result[0]) + and not isinstance(get_proper_type(inferred_result[0]), AnyType) + and not none_type_var_overlap + ): + return inferred_result + return unioned_result + elif unioned_result is not None: + return unioned_result + elif inferred_result is not None: + return inferred_result + + # Step 4: Failure. At this point, we know there is no match. We fall back to trying + # to find a somewhat plausible overload target using the erased types + # so we can produce a nice error message. + # + # For example, suppose the user passes a value of type 'List[str]' into an + # overload with signatures f(x: int) -> int and f(x: List[int]) -> List[int]. + # + # Neither alternative matches, but we can guess the user probably wants the + # second one. + erased_targets = self.overload_erased_call_targets( + plausible_targets, arg_types, arg_kinds, arg_names, args, context + ) + + # Step 5: We try and infer a second-best alternative if possible. If not, fall back + # to using 'Any'. + if len(erased_targets) > 0: + # Pick the first plausible erased target as the fallback + # TODO: Adjust the error message here to make it clear there was no match. + # In order to do this, we need to find a clean way of associating + # a note with whatever error message 'self.check_call' will generate. + # In particular, the note's line and column numbers need to be the same + # as the error's. + target: Type = erased_targets[0] + else: + # There was no plausible match: give up + target = AnyType(TypeOfAny.from_error) + if not is_operator_method(callable_name): + code = None + else: + code = codes.OPERATOR + self.msg.no_variant_matches_arguments(callee, arg_types, context, code=code) + + result = self.check_call( + target, + args, + arg_kinds, + context, + arg_names, + callable_name=callable_name, + object_type=object_type, + ) + # Do not show the extra error if the union math was forced. + if union_interrupted and not none_type_var_overlap: + self.chk.fail(message_registry.TOO_MANY_UNION_COMBINATIONS, context) + return result + + def plausible_overload_call_targets( + self, + arg_types: list[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + overload: Overloaded, + ) -> list[CallableType]: + """Returns all overload call targets that having matching argument counts. + + If the given args contains a star-arg (*arg or **kwarg argument, except for + ParamSpec), this method will ensure all star-arg overloads appear at the start + of the list, instead of their usual location. + + The only exception is if the starred argument is something like a Tuple or a + NamedTuple, which has a definitive "shape". If so, we don't move the corresponding + alternative to the front since we can infer a more precise match using the original + order.""" + + def has_shape(typ: Type) -> bool: + typ = get_proper_type(typ) + return isinstance(typ, (TupleType, TypedDictType)) or ( + isinstance(typ, Instance) and typ.type.is_named_tuple + ) + + matches: list[CallableType] = [] + star_matches: list[CallableType] = [] + + args_have_var_arg = False + args_have_kw_arg = False + for kind, typ in zip(arg_kinds, arg_types): + if kind == ARG_STAR and not has_shape(typ): + args_have_var_arg = True + if kind == ARG_STAR2 and not has_shape(typ): + args_have_kw_arg = True + + for typ in overload.items: + formal_to_actual = map_actuals_to_formals( + arg_kinds, arg_names, typ.arg_kinds, typ.arg_names, lambda i: arg_types[i] + ) + with self.msg.filter_errors(): + if typ.param_spec() is not None: + # ParamSpec can be expanded in a lot of different ways. We may try + # to expand it here instead, but picking an impossible overload + # is safe: it will be filtered out later. + # Unlike other var-args signatures, ParamSpec produces essentially + # a fixed signature, so there's no need to push them to the top. + matches.append(typ) + elif self.check_argument_count( + typ, arg_types, arg_kinds, arg_names, formal_to_actual, None + ): + if args_have_var_arg and typ.is_var_arg: + star_matches.append(typ) + elif args_have_kw_arg and typ.is_kw_arg: + star_matches.append(typ) + else: + matches.append(typ) + + return star_matches + matches + + def infer_overload_return_type( + self, + plausible_targets: list[CallableType], + args: list[Expression], + arg_types: list[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + callable_name: str | None, + object_type: Type | None, + context: Context, + ) -> tuple[Type, Type] | None: + """Attempts to find the first matching callable from the given list. + + If a match is found, returns a tuple containing the result type and the inferred + callee type. (This tuple is meant to be eventually returned by check_call.) + If multiple targets match due to ambiguous Any parameters, returns (AnyType, AnyType). + If no targets match, returns None. + + Assumes all of the given targets have argument counts compatible with the caller. """ - messages = messages or self.msg - # TODO also consider argument names and kinds - # TODO for overlapping signatures we should try to get a more precise - # result than 'Any' - match = [] # type: List[Callable] - for typ in overload.items(): - if self.matches_signature_erased(arg_types, is_var_arg, typ): - if (match and not is_same_type(match[-1].ret_type, - typ.ret_type) and - not mypy.checker.is_more_precise_signature( - match[-1], typ)): - # Ambiguous return type. Either the function overload is - # overlapping (which results in an error elsewhere) or the - # caller has provided some Any argument types; in - # either case can only infer the type to be Any, as it is - # not an error to use Any types in calls. - # - # Overlapping overload items are fine if the items are - # covariant in both argument types and return types with - # respect to type precision. - return AnyType() + + matches: list[CallableType] = [] + return_types: list[Type] = [] + inferred_types: list[Type] = [] + args_contain_any = any(map(has_any_type, arg_types)) + type_maps: list[dict[Expression, Type]] = [] + + for typ in plausible_targets: + assert self.msg is self.chk.msg + with self.msg.filter_errors() as w: + with self.chk.local_type_map() as m: + ret_type, infer_type = self.check_call( + callee=typ, + args=args, + arg_kinds=arg_kinds, + arg_names=arg_names, + context=context, + callable_name=callable_name, + object_type=object_type, + ) + is_match = not w.has_new_errors() + if is_match: + # Return early if possible; otherwise record info, so we can + # check for ambiguity due to 'Any' below. + if not args_contain_any: + self.chk.store_types(m) + return ret_type, infer_type + p_infer_type = get_proper_type(infer_type) + if isinstance(p_infer_type, CallableType): + # Prefer inferred types if possible, this will avoid false triggers for + # Any-ambiguity caused by arguments with Any passed to generic overloads. + matches.append(p_infer_type) else: - match.append(typ) - if not match: - messages.no_variant_matches_arguments(overload, context) - return AnyType() - else: - if len(match) == 1: - return match[0] + matches.append(typ) + return_types.append(ret_type) + inferred_types.append(infer_type) + type_maps.append(m) + + if not matches: + return None + elif any_causes_overload_ambiguity(matches, return_types, arg_types, arg_kinds, arg_names): + # An argument of type or containing the type 'Any' caused ambiguity. + # We try returning a precise type if we can. If not, we give up and just return 'Any'. + if all_same_types(return_types): + self.chk.store_types(type_maps[0]) + return return_types[0], inferred_types[0] + elif all_same_types([erase_type(typ) for typ in return_types]): + self.chk.store_types(type_maps[0]) + return erase_type(return_types[0]), erase_type(inferred_types[0]) else: - # More than one signature matches. Pick the first *non-erased* - # matching signature, or default to the first one if none - # match. - for m in match: - if self.match_signature_types(arg_types, is_var_arg, m): - return m - return match[0] - - def matches_signature_erased(self, arg_types: List[Type], is_var_arg: bool, - callee: Callable) -> bool: - """Determine whether arguments could match the signature at runtime. - - If is_var_arg is True, the caller uses varargs. This is used for - overload resolution. + return self.check_call( + callee=AnyType(TypeOfAny.special_form), + args=args, + arg_kinds=arg_kinds, + arg_names=arg_names, + context=context, + callable_name=callable_name, + object_type=object_type, + ) + else: + # Success! No ambiguity; return the first match. + self.chk.store_types(type_maps[0]) + return return_types[0], inferred_types[0] + + def overload_erased_call_targets( + self, + plausible_targets: list[CallableType], + arg_types: list[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + args: list[Expression], + context: Context, + ) -> list[CallableType]: + """Returns a list of all targets that match the caller after erasing types. + + Assumes all of the given targets have argument counts compatible with the caller. + """ + matches: list[CallableType] = [] + for typ in plausible_targets: + if self.erased_signature_similarity( + arg_types, arg_kinds, arg_names, args, typ, context + ): + matches.append(typ) + return matches + + def possible_none_type_var_overlap( + self, arg_types: list[Type], plausible_targets: list[CallableType] + ) -> bool: + """Heuristic to determine whether we need to try forcing union math. + + This is needed to avoid greedy type variable match in situations like this: + @overload + def foo(x: None) -> None: ... + @overload + def foo(x: T) -> list[T]: ... + + x: int | None + foo(x) + we want this call to infer list[int] | None, not list[int | None]. """ - if not is_valid_argc(len(arg_types), False, callee): + if not plausible_targets or not arg_types: + return False + has_optional_arg = False + for arg_type in get_proper_types(arg_types): + if not isinstance(arg_type, UnionType): + continue + for item in get_proper_types(arg_type.items): + if isinstance(item, NoneType): + has_optional_arg = True + break + if not has_optional_arg: return False - if is_var_arg: - if not self.is_valid_var_arg(arg_types[-1]): - return False - arg_types, rest = expand_caller_var_args(arg_types, - callee.max_fixed_args()) - - # Fixed function arguments. - func_fixed = callee.max_fixed_args() - for i in range(min(len(arg_types), func_fixed)): - # Use is_more_precise rather than is_subtype because it ignores ducktype - # declarations. This is important since ducktype declarations are ignored - # when performing runtime type checking. - if not is_compatible_overload_arg(arg_types[i], callee.arg_types[i]): - return False - # Function varargs. - if callee.is_var_arg: - for i in range(func_fixed, len(arg_types)): - # See above for why we use is_more_precise. - if not is_compatible_overload_arg(arg_types[i], - callee.arg_types[func_fixed]): - return False - return True + min_prefix = min(len(c.arg_types) for c in plausible_targets) + for i in range(min_prefix): + if any( + isinstance(get_proper_type(c.arg_types[i]), NoneType) for c in plausible_targets + ) and any( + isinstance(get_proper_type(c.arg_types[i]), TypeVarType) for c in plausible_targets + ): + return True + return False + + def union_overload_result( + self, + plausible_targets: list[CallableType], + args: list[Expression], + arg_types: list[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + callable_name: str | None, + object_type: Type | None, + none_type_var_overlap: bool, + context: Context, + level: int = 0, + ) -> list[tuple[Type, Type]] | None: + """Accepts a list of overload signatures and attempts to match calls by destructuring + the first union. + + Return a list of (, ) if call succeeds for every + item of the desctructured union. Returns None if there is no match. + """ + # Step 1: If we are already too deep, then stop immediately. Otherwise mypy might + # hang for long time because of a weird overload call. The caller will get + # the exception and generate an appropriate note message, if needed. + if level >= MAX_UNIONS: + raise TooManyUnions + + # Step 2: Find position of the first union in arguments. Return the normal inferred + # type if no more unions left. + for idx, typ in enumerate(arg_types): + if self.real_union(typ): + break + else: + # No unions in args, just fall back to normal inference + with self.type_overrides_set(args, arg_types): + res = self.infer_overload_return_type( + plausible_targets, + args, + arg_types, + arg_kinds, + arg_names, + callable_name, + object_type, + context, + ) + if res is not None: + return [res] + return None - def match_signature_types(self, arg_types: List[Type], is_var_arg: bool, - callee: Callable) -> bool: - """Determine whether arguments types match the signature. + # Step 3: Try a direct match before splitting to avoid unnecessary union splits + # and save performance. + if not none_type_var_overlap: + with self.type_overrides_set(args, arg_types): + direct = self.infer_overload_return_type( + plausible_targets, + args, + arg_types, + arg_kinds, + arg_names, + callable_name, + object_type, + context, + ) + if direct is not None and not isinstance( + get_proper_type(direct[0]), (UnionType, AnyType) + ): + # We only return non-unions soon, to avoid greedy match. + return [direct] + + # Step 4: Split the first remaining union type in arguments into items and + # try to match each item individually (recursive). + first_union = get_proper_type(arg_types[idx]) + assert isinstance(first_union, UnionType) + res_items = [] + for item in first_union.relevant_items(): + new_arg_types = arg_types.copy() + new_arg_types[idx] = item + sub_result = self.union_overload_result( + plausible_targets, + args, + new_arg_types, + arg_kinds, + arg_names, + callable_name, + object_type, + none_type_var_overlap, + context, + level + 1, + ) + if sub_result is not None: + res_items.extend(sub_result) + else: + # Some item doesn't match, return soon. + return None + + # Step 5: If splitting succeeded, then filter out duplicate items before returning. + seen: set[tuple[Type, Type]] = set() + result = [] + for pair in res_items: + if pair not in seen: + seen.add(pair) + result.append(pair) + return result - If is_var_arg is True, the caller uses varargs. Assume that argument - counts are compatible. + def real_union(self, typ: Type) -> bool: + typ = get_proper_type(typ) + return isinstance(typ, UnionType) and len(typ.relevant_items()) > 1 + + @contextmanager + def type_overrides_set( + self, exprs: Sequence[Expression], overrides: Sequence[Type] + ) -> Iterator[None]: + """Set _temporary_ type overrides for given expressions.""" + assert len(exprs) == len(overrides) + for expr, typ in zip(exprs, overrides): + self.type_overrides[expr] = typ + try: + yield + finally: + for expr in exprs: + del self.type_overrides[expr] + + def combine_function_signatures(self, types: list[ProperType]) -> AnyType | CallableType: + """Accepts a list of function signatures and attempts to combine them together into a + new CallableType consisting of the union of all of the given arguments and return types. + + If there is at least one non-callable type, return Any (this can happen if there is + an ambiguity because of Any in arguments). """ - if is_var_arg: - arg_types, rest = expand_caller_var_args(arg_types, - callee.max_fixed_args()) - - # Fixed function arguments. - func_fixed = callee.max_fixed_args() - for i in range(min(len(arg_types), func_fixed)): - if not is_subtype(arg_types[i], callee.arg_types[i]): + assert types, "Trying to merge no callables" + if not all(isinstance(c, CallableType) for c in types): + return AnyType(TypeOfAny.special_form) + callables = cast("list[CallableType]", types) + if len(callables) == 1: + return callables[0] + + # Note: we are assuming here that if a user uses some TypeVar 'T' in + # two different functions, they meant for that TypeVar to mean the + # same thing. + # + # This function will make sure that all instances of that TypeVar 'T' + # refer to the same underlying TypeVarType objects to simplify the union-ing + # logic below. + # + # (If the user did *not* mean for 'T' to be consistently bound to the + # same type in their overloads, well, their code is probably too + # confusing and ought to be re-written anyways.) + callables, variables = merge_typevars_in_callables_by_name(callables) + + new_args: list[list[Type]] = [[] for _ in range(len(callables[0].arg_types))] + new_kinds = list(callables[0].arg_kinds) + new_returns: list[Type] = [] + + too_complex = False + for target in callables: + # We fall back to Callable[..., Union[]] if the functions do not have + # the exact same signature. The only exception is if one arg is optional and + # the other is positional: in that case, we continue unioning (and expect a + # positional arg). + # TODO: Enhance the merging logic to handle a wider variety of signatures. + if len(new_kinds) != len(target.arg_kinds): + too_complex = True + break + for i, (new_kind, target_kind) in enumerate(zip(new_kinds, target.arg_kinds)): + if new_kind == target_kind: + continue + elif new_kind.is_positional() and target_kind.is_positional(): + new_kinds[i] = ARG_POS + else: + too_complex = True + break + + if too_complex: + break # outer loop + + for i, arg in enumerate(target.arg_types): + new_args[i].append(arg) + new_returns.append(target.ret_type) + + union_return = make_simplified_union(new_returns) + if too_complex: + any = AnyType(TypeOfAny.special_form) + return callables[0].copy_modified( + arg_types=[any, any], + arg_kinds=[ARG_STAR, ARG_STAR2], + arg_names=[None, None], + ret_type=union_return, + variables=variables, + implicit=True, + ) + + final_args = [] + for args_list in new_args: + new_type = make_simplified_union(args_list) + final_args.append(new_type) + + return callables[0].copy_modified( + arg_types=final_args, + arg_kinds=new_kinds, + ret_type=union_return, + variables=variables, + implicit=True, + ) + + def erased_signature_similarity( + self, + arg_types: list[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + args: list[Expression], + callee: CallableType, + context: Context, + ) -> bool: + """Determine whether arguments could match the signature at runtime, after + erasing types.""" + formal_to_actual = map_actuals_to_formals( + arg_kinds, arg_names, callee.arg_kinds, callee.arg_names, lambda i: arg_types[i] + ) + + with self.msg.filter_errors(): + if not self.check_argument_count( + callee, arg_types, arg_kinds, arg_names, formal_to_actual, None + ): + # Too few or many arguments -> no match. return False - # Function varargs. - if callee.is_var_arg: - for i in range(func_fixed, len(arg_types)): - if not is_subtype(arg_types[i], - callee.arg_types[func_fixed]): - return False - return True - def apply_generic_arguments(self, callable: Callable, types: List[Type], - context: Context) -> Type: - """Simple wrapper around mypy.applytype.apply_generic_arguments.""" - return applytype.apply_generic_arguments(callable, types, self.msg, context) - - def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], - context: Context) -> Type: - items = [] # type: List[Callable] - for item in overload.items(): - applied = self.apply_generic_arguments(item, types, context) - if isinstance(applied, Callable): - items.append(applied) - else: - # There was an error. - return AnyType() - return Overloaded(items) + def check_arg( + caller_type: Type, + original_ccaller_type: Type, + caller_kind: ArgKind, + callee_type: Type, + n: int, + m: int, + callee: CallableType, + object_type: Type | None, + context: Context, + outer_context: Context, + ) -> None: + if not arg_approximate_similarity(caller_type, callee_type): + # No match -- exit early since none of the remaining work can change + # the result. + raise Finished + + try: + self.check_argument_types( + arg_types, + arg_kinds, + args, + callee, + formal_to_actual, + context=context, + check_arg=check_arg, + ) + return True + except Finished: + return False - def visit_member_expr(self, e: MemberExpr) -> Type: + def apply_generic_arguments( + self, + callable: CallableType, + types: Sequence[Type | None], + context: Context, + skip_unsatisfied: bool = False, + ) -> CallableType: + """Simple wrapper around mypy.applytype.apply_generic_arguments.""" + return applytype.apply_generic_arguments( + callable, + types, + self.msg.incompatible_typevar_value, + context, + skip_unsatisfied=skip_unsatisfied, + ) + + def check_any_type_call(self, args: list[Expression], callee: Type) -> tuple[Type, Type]: + self.infer_arg_types_in_empty_context(args) + callee = get_proper_type(callee) + if isinstance(callee, AnyType): + return ( + AnyType(TypeOfAny.from_another_any, source_any=callee), + AnyType(TypeOfAny.from_another_any, source_any=callee), + ) + else: + return AnyType(TypeOfAny.special_form), AnyType(TypeOfAny.special_form) + + def check_union_call( + self, + callee: UnionType, + args: list[Expression], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + context: Context, + ) -> tuple[Type, Type]: + with self.msg.disable_type_names(): + results = [ + self.check_call(subtype, args, arg_kinds, context, arg_names) + for subtype in callee.relevant_items() + ] + + return (make_simplified_union([res[0] for res in results]), callee) + + def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type: """Visit member expression (of form e.id).""" - result = self.analyse_ordinary_member_access(e, False) - return self.chk.narrow_type_from_binder(e, result) + self.chk.module_refs.update(extract_refexpr_names(e)) + result = self.analyze_ordinary_member_access(e, is_lvalue) + narrowed = self.narrow_type_from_binder(e, result) + self.chk.warn_deprecated(e.node, e) + return narrowed - def analyse_ordinary_member_access(self, e: MemberExpr, - is_lvalue: bool) -> Type: + def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type: """Analyse member expression or member lvalue.""" if e.kind is not None: # This is a reference to a module attribute. - return self.analyse_ref_expr(e) + return self.analyze_ref_expr(e) else: # This is a reference to a non-module attribute. - return analyse_member_access(e.name, self.accept(e.expr), e, - is_lvalue, False, - self.named_type, self.msg) - - def analyse_external_member_access(self, member: str, base_type: Type, - context: Context) -> Type: + original_type = self.accept(e.expr, is_callee=self.is_callee) + base = e.expr + module_symbol_table = None + + if isinstance(base, RefExpr) and isinstance(base.node, MypyFile): + module_symbol_table = base.node.names + if isinstance(base, RefExpr) and isinstance(base.node, Var): + # This is needed to special case self-types, so we don't need to track + # these flags separately in checkmember.py. + is_self = base.node.is_self or base.node.is_cls + else: + is_self = False + + member_type = analyze_member_access( + e.name, + original_type, + e, + is_lvalue=is_lvalue, + is_super=False, + is_operator=False, + msg=self.msg, + original_type=original_type, + chk=self.chk, + in_literal_context=self.is_literal_context(), + module_symbol_table=module_symbol_table, + is_self=is_self, + ) + + return member_type + + def analyze_external_member_access( + self, member: str, base_type: Type, context: Context + ) -> Type: """Analyse member access that is external, i.e. it cannot refer to private definitions. Return the result type. """ # TODO remove; no private definitions in mypy - return analyse_member_access(member, base_type, context, False, False, - self.named_type, self.msg) + return analyze_member_access( + member, + base_type, + context, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, + original_type=base_type, + chk=self.chk, + in_literal_context=self.is_literal_context(), + ) + + def is_literal_context(self) -> bool: + return is_literal_type_like(self.type_context[-1]) + + def infer_literal_expr_type(self, value: LiteralValue, fallback_name: str) -> Type: + """Analyzes the given literal expression and determines if we should be + inferring an Instance type, a Literal[...] type, or an Instance that + remembers the original literal. We... + + 1. ...Infer a normal Instance in most circumstances. + + 2. ...Infer a Literal[...] if we're in a literal context. For example, if we + were analyzing the "3" in "foo(3)" where "foo" has a signature of + "def foo(Literal[3]) -> None", we'd want to infer that the "3" has a + type of Literal[3] instead of Instance. + + 3. ...Infer an Instance that remembers the original Literal if we're declaring + a Final variable with an inferred type -- for example, "bar" in "bar: Final = 3" + would be assigned an Instance that remembers it originated from a '3'. See + the comments in Instance's constructor for more details. + """ + typ = self.named_type(fallback_name) + if self.is_literal_context(): + return LiteralType(value=value, fallback=typ) + else: + return typ.copy_modified( + last_known_value=LiteralType( + value=value, fallback=typ, line=typ.line, column=typ.column + ) + ) + + def concat_tuples(self, left: TupleType, right: TupleType) -> TupleType: + """Concatenate two fixed length tuples.""" + assert not (find_unpack_in_list(left.items) and find_unpack_in_list(right.items)) + return TupleType( + items=left.items + right.items, fallback=self.named_type("builtins.tuple") + ) def visit_int_expr(self, e: IntExpr) -> Type: """Type check an integer literal (trivial).""" - return self.named_type('builtins.int') + return self.infer_literal_expr_type(e.value, "builtins.int") def visit_str_expr(self, e: StrExpr) -> Type: """Type check a string literal (trivial).""" - return self.named_type('builtins.str') + return self.infer_literal_expr_type(e.value, "builtins.str") def visit_bytes_expr(self, e: BytesExpr) -> Type: """Type check a bytes literal (trivial).""" - return self.named_type('builtins.bytes') - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - """Type check a unicode literal (trivial).""" - return self.named_type('builtins.unicode') + return self.infer_literal_expr_type(e.value, "builtins.bytes") def visit_float_expr(self, e: FloatExpr) -> Type: """Type check a float literal (trivial).""" - return self.named_type('builtins.float') + return self.named_type("builtins.float") def visit_complex_expr(self, e: ComplexExpr) -> Type: """Type check a complex literal.""" - return self.named_type('builtins.complex') + return self.named_type("builtins.complex") + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + """Type check '...'.""" + return self.named_type("builtins.ellipsis") def visit_op_expr(self, e: OpExpr) -> Type: """Type check a binary operator expression.""" - if e.op == 'and' or e.op == 'or': + if e.analyzed: + # It's actually a type expression X | Y. + return self.accept(e.analyzed) + if e.op == "and" or e.op == "or": return self.check_boolean_op(e, e) - if e.op == '*' and isinstance(e.left, ListExpr): + if e.op == "*" and isinstance(e.left, ListExpr): # Expressions of form [...] * e get special type inference. return self.check_list_multiply(e) - if e.op == '%' and isinstance(e.left, StrExpr): - return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) + if e.op == "%": + if isinstance(e.left, BytesExpr): + return self.strfrm_checker.check_str_interpolation(e.left, e.right) + if isinstance(e.left, StrExpr): + return self.strfrm_checker.check_str_interpolation(e.left, e.right) left_type = self.accept(e.left) - if e.op in nodes.op_methods: - method = self.get_operator_method(e.op) - result, method_type = self.check_op(method, left_type, e.right, e, - allow_reverse=True) + proper_left_type = get_proper_type(left_type) + if isinstance(proper_left_type, TupleType) and e.op == "+": + left_add_method = proper_left_type.partial_fallback.type.get("__add__") + if left_add_method and left_add_method.fullname == "builtins.tuple.__add__": + proper_right_type = get_proper_type(self.accept(e.right)) + if isinstance(proper_right_type, TupleType): + right_radd_method = proper_right_type.partial_fallback.type.get("__radd__") + if right_radd_method is None: + # One cannot have two variadic items in the same tuple. + if ( + find_unpack_in_list(proper_left_type.items) is None + or find_unpack_in_list(proper_right_type.items) is None + ): + return self.concat_tuples(proper_left_type, proper_right_type) + elif ( + PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature + and isinstance(proper_right_type, Instance) + and self.chk.type_is_iterable(proper_right_type) + ): + # Handle tuple[X, Y] + tuple[Z, ...] = tuple[X, Y, *tuple[Z, ...]]. + right_radd_method = proper_right_type.type.get("__radd__") + if ( + right_radd_method is None + and proper_left_type.partial_fallback.type.fullname == "builtins.tuple" + and find_unpack_in_list(proper_left_type.items) is None + ): + item_type = self.chk.iterable_item_type(proper_right_type, e) + mapped = self.chk.named_generic_type("builtins.tuple", [item_type]) + return proper_left_type.copy_modified( + items=proper_left_type.items + [UnpackType(mapped)] + ) + + use_reverse: UseReverse = USE_REVERSE_DEFAULT + if e.op == "|": + if is_named_instance(proper_left_type, "builtins.dict"): + # This is a special case for `dict | TypedDict`. + # 1. Find `dict | TypedDict` case + # 2. Switch `dict.__or__` to `TypedDict.__ror__` (the same from both runtime and typing perspective) + proper_right_type = get_proper_type(self.accept(e.right)) + if isinstance(proper_right_type, TypedDictType): + use_reverse = USE_REVERSE_ALWAYS + if isinstance(proper_left_type, TypedDictType): + # This is the reverse case: `TypedDict | dict`, + # simply do not allow the reverse checking: + # do not call `__dict__.__ror__`. + proper_right_type = get_proper_type(self.accept(e.right)) + if is_named_instance(proper_right_type, "builtins.dict"): + use_reverse = USE_REVERSE_NEVER + + if PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature: + # Handle tuple[X, ...] + tuple[Y, Z] = tuple[*tuple[X, ...], Y, Z]. + if ( + e.op == "+" + and isinstance(proper_left_type, Instance) + and proper_left_type.type.fullname == "builtins.tuple" + ): + proper_right_type = get_proper_type(self.accept(e.right)) + if ( + isinstance(proper_right_type, TupleType) + and proper_right_type.partial_fallback.type.fullname == "builtins.tuple" + and find_unpack_in_list(proper_right_type.items) is None + ): + return proper_right_type.copy_modified( + items=[UnpackType(proper_left_type)] + proper_right_type.items + ) + + if e.op in operators.op_methods: + method = operators.op_methods[e.op] + if use_reverse is UseReverse.DEFAULT or use_reverse is UseReverse.NEVER: + result, method_type = self.check_op( + method, + base_type=left_type, + arg=e.right, + context=e, + allow_reverse=use_reverse is UseReverse.DEFAULT, + ) + elif use_reverse is UseReverse.ALWAYS: + result, method_type = self.check_op( + # The reverse operator here gives better error messages: + operators.reverse_op_methods[method], + base_type=self.accept(e.right), + arg=e.left, + context=e, + allow_reverse=False, + ) + else: + assert_never(use_reverse) e.method_type = method_type return result else: - raise RuntimeError('Unknown operator {}'.format(e.op)) - - def strip_parens(self, node: Node) -> Node: - if isinstance(node, ParenExpr): - return self.strip_parens(node.expr) - else: - return node + raise RuntimeError(f"Unknown operator {e.op}") def visit_comparison_expr(self, e: ComparisonExpr) -> Type: """Type check a comparison expression. @@ -740,132 +3518,679 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type: Comparison expressions are type checked consecutive-pair-wise That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' """ - result = None # type: mypy.types.Type + result: Type | None = None + sub_result: Type # Check each consecutive operand pair and their operator for left, right, operator in zip(e.operands, e.operands[1:], e.operators): left_type = self.accept(left) - method_type = None # type: mypy.types.Type - - if operator == 'in' or operator == 'not in': - right_type = self.accept(right) # TODO only evaluate if needed - - local_errors = self.msg.copy() - sub_result, method_type = self.check_op_local('__contains__', right_type, - left, e, local_errors) - if (local_errors.is_errors() and - # is_valid_var_arg is True for any Iterable - self.is_valid_var_arg(right_type)): - itertype = self.chk.analyse_iterable_item_type(right) - method_type = Callable([left_type], - [nodes.ARG_POS], - [None], - self.chk.bool_type(), - self.named_type('builtins.function')) - sub_result = self.chk.bool_type() - if not is_subtype(left_type, itertype): - self.msg.unsupported_operand_types('in', left_type, right_type, e) - else: - self.msg.add_errors(local_errors) - if operator == 'not in': - sub_result = self.chk.bool_type() - elif operator in nodes.op_methods: - method = self.get_operator_method(operator) - sub_result, method_type = self.check_op(method, left_type, right, e, - allow_reverse=True) - - elif operator == 'is' or operator == 'is not': - sub_result = self.chk.bool_type() - method_type = None + if operator == "in" or operator == "not in": + # This case covers both iterables and containers, which have different meanings. + # For a container, the in operator calls the __contains__ method. + # For an iterable, the in operator iterates over the iterable, and compares each item one-by-one. + # We allow `in` for a union of containers and iterables as long as at least one of them matches the + # type of the left operand, as the operation will simply return False if the union's container/iterator + # type doesn't match the left operand. + + # If the right operand has partial type, look it up without triggering + # a "Need type annotation ..." message, as it would be noise. + right_type = self.find_partial_type_ref_fast_path(right) + if right_type is None: + right_type = self.accept(right) # Validate the right operand + + right_type = get_proper_type(right_type) + item_types: Sequence[Type] = [right_type] + if isinstance(right_type, UnionType): + item_types = list(right_type.relevant_items()) + + sub_result = self.bool_type() + + container_types: list[Type] = [] + iterable_types: list[Type] = [] + failed_out = False + encountered_partial_type = False + + for item_type in item_types: + # Keep track of whether we get type check errors (these won't be reported, they + # are just to verify whether something is valid typing wise). + with self.msg.filter_errors(save_filtered_errors=True) as container_errors: + _, method_type = self.check_method_call_by_name( + method="__contains__", + base_type=item_type, + args=[left], + arg_kinds=[ARG_POS], + context=e, + original_type=right_type, + ) + # Container item type for strict type overlap checks. Note: we need to only + # check for nominal type, because a usual "Unsupported operands for in" + # will be reported for types incompatible with __contains__(). + # See testCustomContainsCheckStrictEquality for an example. + cont_type = self.chk.analyze_container_item_type(item_type) + + if isinstance(item_type, PartialType): + # We don't really know if this is an error or not, so just shut up. + encountered_partial_type = True + pass + elif ( + container_errors.has_new_errors() + and + # is_valid_var_arg is True for any Iterable + self.is_valid_var_arg(item_type) + ): + # it's not a container, but it is an iterable + with self.msg.filter_errors(save_filtered_errors=True) as iterable_errors: + _, itertype = self.chk.analyze_iterable_item_type_without_expression( + item_type, e + ) + if iterable_errors.has_new_errors(): + self.msg.add_errors(iterable_errors.filtered_errors()) + failed_out = True + else: + method_type = CallableType( + [left_type], + [nodes.ARG_POS], + [None], + self.bool_type(), + self.named_type("builtins.function"), + ) + e.method_types.append(method_type) + iterable_types.append(itertype) + elif not container_errors.has_new_errors() and cont_type: + container_types.append(cont_type) + e.method_types.append(method_type) + else: + self.msg.add_errors(container_errors.filtered_errors()) + failed_out = True + + if not encountered_partial_type and not failed_out: + iterable_type = UnionType.make_union(iterable_types) + if not is_subtype(left_type, iterable_type): + if not container_types: + self.msg.unsupported_operand_types("in", left_type, right_type, e) + else: + container_type = UnionType.make_union(container_types) + if self.dangerous_comparison( + left_type, + container_type, + original_container=right_type, + prefer_literal=False, + ): + self.msg.dangerous_comparison( + left_type, container_type, "container", e + ) + + elif operator in operators.op_methods: + method = operators.op_methods[operator] + + with ErrorWatcher(self.msg.errors) as w: + sub_result, method_type = self.check_op( + method, left_type, right, e, allow_reverse=True + ) + e.method_types.append(method_type) + + # Only show dangerous overlap if there are no other errors. See + # testCustomEqCheckStrictEquality for an example. + if not w.has_new_errors() and operator in ("==", "!="): + right_type = self.accept(right) + if self.dangerous_comparison(left_type, right_type): + # Show the most specific literal types possible + left_type = try_getting_literal(left_type) + right_type = try_getting_literal(right_type) + self.msg.dangerous_comparison(left_type, right_type, "equality", e) + + elif operator == "is" or operator == "is not": + right_type = self.accept(right) # validate the right operand + sub_result = self.bool_type() + if self.dangerous_comparison(left_type, right_type): + # Show the most specific literal types possible + left_type = try_getting_literal(left_type) + right_type = try_getting_literal(right_type) + self.msg.dangerous_comparison(left_type, right_type, "identity", e) + e.method_types.append(None) else: - raise RuntimeError('Unknown comparison operator {}'.format(operator)) - - e.method_types.append(method_type) + raise RuntimeError(f"Unknown comparison operator {operator}") # Determine type of boolean-and of result and sub_result - if result == None: + if result is None: result = sub_result else: - # TODO: check on void needed? - self.check_not_void(sub_result, e) result = join.join_types(result, sub_result) + assert result is not None return result - def get_operator_method(self, op: str) -> str: - if op == '/' and self.chk.pyversion == 2: - # TODO also check for "from __future__ import division" - return '__div__' - else: - return nodes.op_methods[op] + def find_partial_type_ref_fast_path(self, expr: Expression) -> Type | None: + """If expression has a partial generic type, return it without additional checks. - def check_op_local(self, method: str, base_type: Type, arg: Node, - context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. + In particular, this does not generate an error about a missing annotation. - Return tuple (result type, inferred operator method type). + Otherwise, return None. """ - method_type = analyse_member_access(method, base_type, context, False, False, - self.named_type, local_errors) - return self.check_call(method_type, [arg], [nodes.ARG_POS], - context, arg_messages=local_errors) - - def check_op(self, method: str, base_type: Type, arg: Node, - context: Context, - allow_reverse: bool = False) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. + if not isinstance(expr, RefExpr): + return None + if isinstance(expr.node, Var): + result = self.analyze_var_ref(expr.node, expr) + if isinstance(result, PartialType) and result.type is not None: + self.chk.store_type(expr, fixup_partial_type(result)) + return result + return None + + def dangerous_comparison( + self, + left: Type, + right: Type, + *, + original_container: Type | None = None, + seen_types: set[tuple[Type, Type]] | None = None, + prefer_literal: bool = True, + ) -> bool: + """Check for dangerous non-overlapping comparisons like 42 == 'no'. + + The original_container is the original container type for 'in' checks + (and None for equality checks). + + Rules: + * X and None are overlapping even in strict-optional mode. This is to allow + 'assert x is not None' for x defined as 'x = None # type: str' in class body + (otherwise mypy itself would have couple dozen errors because of this). + * Optional[X] and Optional[Y] are non-overlapping if X and Y are + non-overlapping, although technically None is overlap, it is most + likely an error. + * Any overlaps with everything, i.e. always safe. + * Special case: b'abc' in b'cde' is safe. + """ + if not self.chk.options.strict_equality: + return False - Return tuple (result type, inferred operator method type). + if seen_types is None: + seen_types = set() + if (left, right) in seen_types: + return False + seen_types.add((left, right)) + + left, right = get_proper_types((left, right)) + + # We suppress the error if there is a custom __eq__() method on either + # side. User defined (or even standard library) classes can define this + # to return True for comparisons between non-overlapping types. + if custom_special_method(left, "__eq__") or custom_special_method(right, "__eq__"): + return False + + if prefer_literal: + # Also flag non-overlapping literals in situations like: + # x: Literal['a', 'b'] + # if x == 'c': + # ... + left = try_getting_literal(left) + right = try_getting_literal(right) + + if self.chk.binder.is_unreachable_warning_suppressed(): + # We are inside a function that contains type variables with value restrictions in + # its signature. In this case we just suppress all strict-equality checks to avoid + # false positives for code like: + # + # T = TypeVar('T', str, int) + # def f(x: T) -> T: + # if x == 0: + # ... + # return x + # + # TODO: find a way of disabling the check only for types resulted from the expansion. + return False + if isinstance(left, NoneType) or isinstance(right, NoneType): + return False + if isinstance(left, UnionType) and isinstance(right, UnionType): + left = remove_optional(left) + right = remove_optional(right) + left, right = get_proper_types((left, right)) + if ( + original_container + and has_bytes_component(original_container) + and has_bytes_component(left) + ): + # We need to special case bytes and bytearray, because 97 in b'abc', b'a' in b'abc', + # b'a' in bytearray(b'abc') etc. all return True (and we want to show the error only + # if the check can _never_ be True). + return False + if isinstance(left, Instance) and isinstance(right, Instance): + # Special case some builtin implementations of AbstractSet. + left_name = left.type.fullname + right_name = right.type.fullname + if ( + left_name in OVERLAPPING_TYPES_ALLOWLIST + and right_name in OVERLAPPING_TYPES_ALLOWLIST + ): + abstract_set = self.chk.lookup_typeinfo("typing.AbstractSet") + left = map_instance_to_supertype(left, abstract_set) + right = map_instance_to_supertype(right, abstract_set) + return self.dangerous_comparison( + left.args[0], right.args[0], seen_types=seen_types + ) + elif left.type.has_base("typing.Mapping") and right.type.has_base("typing.Mapping"): + # Similar to above: Mapping ignores the classes, it just compares items. + abstract_map = self.chk.lookup_typeinfo("typing.Mapping") + left = map_instance_to_supertype(left, abstract_map) + right = map_instance_to_supertype(right, abstract_map) + return self.dangerous_comparison( + left.args[0], right.args[0], seen_types=seen_types + ) or self.dangerous_comparison(left.args[1], right.args[1], seen_types=seen_types) + elif left_name in ("builtins.list", "builtins.tuple") and right_name == left_name: + return self.dangerous_comparison( + left.args[0], right.args[0], seen_types=seen_types + ) + elif left_name in OVERLAPPING_BYTES_ALLOWLIST and right_name in ( + OVERLAPPING_BYTES_ALLOWLIST + ): + return False + if isinstance(left, LiteralType) and isinstance(right, LiteralType): + if isinstance(left.value, bool) and isinstance(right.value, bool): + # Comparing different booleans is not dangerous. + return False + return not is_overlapping_types(left, right, ignore_promotions=False) + + def check_method_call_by_name( + self, + method: str, + base_type: Type, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + original_type: Type | None = None, + ) -> tuple[Type, Type]: + """Type check a call to a named method on an object. + + Return tuple (result type, inferred method type). The 'original_type' + is used for error messages. """ - # Use a local error storage for errors related to invalid argument - # type (but NOT other errors). This error may need to be suppressed - # for operators which support __rX methods. - local_errors = self.msg.copy() - if not allow_reverse or self.has_member(base_type, method): - result = self.check_op_local(method, base_type, arg, context, - local_errors) - if allow_reverse: - arg_type = self.chk.type_map[arg] - if isinstance(arg_type, AnyType): - # If the right operand has type Any, we can't make any - # conjectures about the type of the result, since the - # operand could have a __r method that returns anything. - result = AnyType(), result[1] - success = not local_errors.is_errors() - else: - result = AnyType(), AnyType() - success = False - if success or not allow_reverse or isinstance(base_type, AnyType): - # We were able to call the normal variant of the operator method, - # or there was some problem not related to argument type - # validity, or the operator has no __rX method. In any case, we - # don't need to consider the __rX method. - self.msg.add_errors(local_errors) - return result + original_type = original_type or base_type + # Unions are special-cased to allow plugins to act on each element of the union. + base_type = get_proper_type(base_type) + if isinstance(base_type, UnionType): + return self.check_union_method_call_by_name( + method, base_type, args, arg_kinds, context, original_type + ) + + method_type = analyze_member_access( + method, + base_type, + context, + is_lvalue=False, + is_super=False, + is_operator=True, + msg=self.msg, + original_type=original_type, + self_type=base_type, + chk=self.chk, + in_literal_context=self.is_literal_context(), + ) + return self.check_method_call(method, base_type, method_type, args, arg_kinds, context) + + def check_union_method_call_by_name( + self, + method: str, + base_type: UnionType, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + original_type: Type | None = None, + ) -> tuple[Type, Type]: + """Type check a call to a named method on an object with union type. + + This essentially checks the call using check_method_call_by_name() for each + union item and unions the result. We do this to allow plugins to act on + individual union items. + """ + res: list[Type] = [] + meth_res: list[Type] = [] + for typ in base_type.relevant_items(): + # Format error messages consistently with + # mypy.checkmember.analyze_union_member_access(). + with self.msg.disable_type_names(): + item, meth_item = self.check_method_call_by_name( + method, typ, args, arg_kinds, context, original_type + ) + res.append(item) + meth_res.append(meth_item) + return make_simplified_union(res), make_simplified_union(meth_res) + + def check_method_call( + self, + method_name: str, + base_type: Type, + method_type: Type, + args: list[Expression], + arg_kinds: list[ArgKind], + context: Context, + ) -> tuple[Type, Type]: + """Type check a call to a method with the given name and type on an object. + + Return tuple (result type, inferred method type). + """ + callable_name = self.method_fullname(base_type, method_name) + object_type = base_type if callable_name is not None else None + + # Try to refine the method signature using plugin hooks before checking the call. + method_type = self.transform_callee_type( + callable_name, method_type, args, arg_kinds, context, object_type=object_type + ) + + return self.check_call( + method_type, + args, + arg_kinds, + context, + callable_name=callable_name, + object_type=base_type, + ) + + def check_op_reversible( + self, + op_name: str, + left_type: Type, + left_expr: Expression, + right_type: Type, + right_expr: Expression, + context: Context, + ) -> tuple[Type, Type]: + def lookup_operator(op_name: str, base_type: Type) -> Type | None: + """Looks up the given operator and returns the corresponding type, + if it exists.""" + + # This check is an important performance optimization, + # even though it is mostly a subset of + # analyze_member_access. + # TODO: Find a way to remove this call without performance implications. + if not self.has_member(base_type, op_name): + return None + + with self.msg.filter_errors() as w: + member = analyze_member_access( + name=op_name, + typ=base_type, + is_lvalue=False, + is_super=False, + is_operator=True, + original_type=base_type, + context=context, + msg=self.msg, + chk=self.chk, + in_literal_context=self.is_literal_context(), + ) + return None if w.has_new_errors() else member + + def lookup_definer(typ: Instance, attr_name: str) -> str | None: + """Returns the name of the class that contains the actual definition of attr_name. + + So if class A defines foo and class B subclasses A, running + 'get_class_defined_in(B, "foo")` would return the full name of A. + + However, if B were to override and redefine foo, that method call would + return the full name of B instead. + + If the attr name is not present in the given class or its MRO, returns None. + """ + for cls in typ.type.mro: + if cls.names.get(attr_name): + return cls.fullname + return None + + left_type = get_proper_type(left_type) + right_type = get_proper_type(right_type) + + # If either the LHS or the RHS are Any, we can't really concluding anything + # about the operation since the Any type may or may not define an + # __op__ or __rop__ method. So, we punt and return Any instead. + + if isinstance(left_type, AnyType): + any_type = AnyType(TypeOfAny.from_another_any, source_any=left_type) + return any_type, any_type + if isinstance(right_type, AnyType): + any_type = AnyType(TypeOfAny.from_another_any, source_any=right_type) + return any_type, any_type + + # STEP 1: + # We start by getting the __op__ and __rop__ methods, if they exist. + + rev_op_name = operators.reverse_op_methods[op_name] + + left_op = lookup_operator(op_name, left_type) + right_op = lookup_operator(rev_op_name, right_type) + + # STEP 2a: + # We figure out in which order Python will call the operator methods. As it + # turns out, it's not as simple as just trying to call __op__ first and + # __rop__ second. + # + # We store the determined order inside the 'variants_raw' variable, + # which records tuples containing the method, base type, and the argument. + + if op_name in operators.op_methods_that_shortcut and is_same_type(left_type, right_type): + # When we do "A() + A()", for example, Python will only call the __add__ method, + # never the __radd__ method. + # + # This is the case even if the __add__ method is completely missing and the __radd__ + # method is defined. + + variants_raw = [(op_name, left_op, left_type, right_expr)] + elif ( + is_subtype(right_type, left_type) + and isinstance(left_type, Instance) + and isinstance(right_type, Instance) + and not ( + left_type.type.alt_promote is not None + and left_type.type.alt_promote.type is right_type.type + ) + and lookup_definer(left_type, op_name) != lookup_definer(right_type, rev_op_name) + ): + # When we do "A() + B()" where B is a subclass of A, we'll actually try calling + # B's __radd__ method first, but ONLY if B explicitly defines or overrides the + # __radd__ method. + # + # This mechanism lets subclasses "refine" the expected outcome of the operation, even + # if they're located on the RHS. + # + # As a special case, the alt_promote check makes sure that we don't use the + # __radd__ method of int if the LHS is a native int type. + + variants_raw = [ + (rev_op_name, right_op, right_type, left_expr), + (op_name, left_op, left_type, right_expr), + ] else: - # Calling the operator method was unsuccessful. Try the __rX - # method of the other operand instead. - rmethod = self.get_reverse_op_method(method) - arg_type = self.accept(arg) - if self.has_member(arg_type, rmethod): - method_type = self.analyse_external_member_access( - rmethod, arg_type, context) - temp = TempNode(base_type) - return self.check_call(method_type, [temp], [nodes.ARG_POS], - context) + # In all other cases, we do the usual thing and call __add__ first and + # __radd__ second when doing "A() + B()". + + variants_raw = [ + (op_name, left_op, left_type, right_expr), + (rev_op_name, right_op, right_type, left_expr), + ] + + # STEP 3: + # We now filter out all non-existent operators. The 'variants' list contains + # all operator methods that are actually present, in the order that Python + # attempts to invoke them. + + variants = [(na, op, obj, arg) for (na, op, obj, arg) in variants_raw if op is not None] + + # STEP 4: + # We now try invoking each one. If an operation succeeds, end early and return + # the corresponding result. Otherwise, return the result and errors associated + # with the first entry. + + errors = [] + results = [] + for name, method, obj, arg in variants: + with self.msg.filter_errors(save_filtered_errors=True) as local_errors: + result = self.check_method_call(op_name, obj, method, [arg], [ARG_POS], context) + if local_errors.has_new_errors(): + errors.append(local_errors.filtered_errors()) + results.append(result) + else: + if isinstance(obj, Instance) and isinstance( + defn := obj.type.get_method(name), OverloadedFuncDef + ): + for item in defn.items: + if ( + isinstance(item, Decorator) + and isinstance(typ := item.func.type, CallableType) + and bind_self(typ) == result[1] + ): + self.chk.check_deprecated(item.func, context) + return result + + # We finish invoking above operators and no early return happens. Therefore, + # we check if either the LHS or the RHS is Instance and fallbacks to Any, + # if so, we also return Any + if (isinstance(left_type, Instance) and left_type.type.fallback_to_any) or ( + isinstance(right_type, Instance) and right_type.type.fallback_to_any + ): + any_type = AnyType(TypeOfAny.special_form) + return any_type, any_type + + # STEP 4b: + # Sometimes, the variants list is empty. In that case, we fall-back to attempting to + # call the __op__ method (even though it's missing). + + if not variants: + with self.msg.filter_errors(save_filtered_errors=True) as local_errors: + result = self.check_method_call_by_name( + op_name, left_type, [right_expr], [ARG_POS], context + ) + + if local_errors.has_new_errors(): + errors.append(local_errors.filtered_errors()) + results.append(result) else: - # No __rX method either. Do deferred type checking to produce - # error message that we may have missed previously. - # TODO Fix type checking an expression more than once. - return self.check_op_local(method, base_type, arg, context, - self.msg) - - def get_reverse_op_method(self, method: str) -> str: - if method == '__div__' and self.chk.pyversion == 2: - return '__rdiv__' + # In theory, we should never enter this case, but it seems + # we sometimes do, when dealing with Type[...]? E.g. see + # check-classes.testTypeTypeComparisonWorks. + # + # This is probably related to the TODO in lookup_operator(...) + # up above. + # + # TODO: Remove this extra case + return result + + self.msg.add_errors(errors[0]) + if len(results) == 1: + return results[0] + else: + error_any = AnyType(TypeOfAny.from_error) + result = error_any, error_any + return result + + def check_op( + self, + method: str, + base_type: Type, + arg: Expression, + context: Context, + allow_reverse: bool = False, + ) -> tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + + if allow_reverse: + left_variants = [base_type] + base_type = get_proper_type(base_type) + if isinstance(base_type, UnionType): + left_variants = list(flatten_nested_unions(base_type.relevant_items())) + right_type = self.accept(arg) + + # Step 1: We first try leaving the right arguments alone and destructure + # just the left ones. (Mypy can sometimes perform some more precise inference + # if we leave the right operands a union -- see testOperatorWithEmptyListAndSum.) + all_results = [] + all_inferred = [] + + with self.msg.filter_errors() as local_errors: + for left_possible_type in left_variants: + result, inferred = self.check_op_reversible( + op_name=method, + left_type=left_possible_type, + left_expr=TempNode(left_possible_type, context=context), + right_type=right_type, + right_expr=arg, + context=context, + ) + all_results.append(result) + all_inferred.append(inferred) + + if not local_errors.has_new_errors(): + results_final = make_simplified_union(all_results) + inferred_final = make_simplified_union(all_inferred) + return results_final, inferred_final + + # Step 2: If that fails, we try again but also destructure the right argument. + # This is also necessary to make certain edge cases work -- see + # testOperatorDoubleUnionInterwovenUnionAdd, for example. + + # Note: We want to pass in the original 'arg' for 'left_expr' and 'right_expr' + # whenever possible so that plugins and similar things can introspect on the original + # node if possible. + # + # We don't do the same for the base expression because it could lead to weird + # type inference errors -- e.g. see 'testOperatorDoubleUnionSum'. + # TODO: Can we use `type_overrides_set()` here? + right_variants = [(right_type, arg)] + right_type = get_proper_type(right_type) + if isinstance(right_type, UnionType): + right_variants = [ + (item, TempNode(item, context=context)) + for item in flatten_nested_unions(right_type.relevant_items()) + ] + + all_results = [] + all_inferred = [] + + with self.msg.filter_errors(save_filtered_errors=True) as local_errors: + for left_possible_type in left_variants: + for right_possible_type, right_expr in right_variants: + result, inferred = self.check_op_reversible( + op_name=method, + left_type=left_possible_type, + left_expr=TempNode(left_possible_type, context=context), + right_type=right_possible_type, + right_expr=right_expr, + context=context, + ) + all_results.append(result) + all_inferred.append(inferred) + + if local_errors.has_new_errors(): + self.msg.add_errors(local_errors.filtered_errors()) + # Point any notes to the same location as an existing message. + err = local_errors.filtered_errors()[-1] + recent_context = TempNode(NoneType()) + recent_context.line = err.line + recent_context.column = err.column + if len(left_variants) >= 2 and len(right_variants) >= 2: + self.msg.warn_both_operands_are_from_unions(recent_context) + elif len(left_variants) >= 2: + self.msg.warn_operand_was_from_union("Left", base_type, context=recent_context) + elif len(right_variants) >= 2: + self.msg.warn_operand_was_from_union( + "Right", right_type, context=recent_context + ) + + # See the comment in 'check_overload_call' for more details on why + # we call 'combine_function_signature' instead of just unioning the inferred + # callable types. + results_final = make_simplified_union(all_results) + inferred_final = self.combine_function_signatures(get_proper_types(all_inferred)) + return results_final, inferred_final else: - return nodes.reverse_op_methods[method] + return self.check_method_call_by_name( + method=method, + base_type=base_type, + args=[arg], + arg_kinds=[ARG_POS], + context=context, + ) def check_boolean_op(self, e: OpExpr, context: Context) -> Type: """Type check a boolean operation ('and' or 'or').""" @@ -876,14 +4201,75 @@ def check_boolean_op(self, e: OpExpr, context: Context) -> Type: # the left operand. We also use the left operand type to guide the type # inference of the right operand so that expressions such as # '[1] or []' are inferred correctly. - ctx = self.chk.type_context[-1] + ctx = self.type_context[-1] left_type = self.accept(e.left, ctx) - right_type = self.accept(e.right, left_type) - - self.check_not_void(left_type, context) - self.check_not_void(right_type, context) - - return join.join_types(left_type, right_type) + expanded_left_type = try_expanding_sum_type_to_union( + self.accept(e.left, ctx), "builtins.bool" + ) + + assert e.op in ("and", "or") # Checked by visit_op_expr + + if e.right_always: + left_map: mypy.checker.TypeMap = None + right_map: mypy.checker.TypeMap = {} + elif e.right_unreachable: + left_map, right_map = {}, None + elif e.op == "and": + right_map, left_map = self.chk.find_isinstance_check(e.left) + elif e.op == "or": + left_map, right_map = self.chk.find_isinstance_check(e.left) + + # If left_map is None then we know mypy considers the left expression + # to be redundant. + if ( + codes.REDUNDANT_EXPR in self.chk.options.enabled_error_codes + and left_map is None + # don't report an error if it's intentional + and not e.right_always + ): + self.msg.redundant_left_operand(e.op, e.left) + + if ( + self.chk.should_report_unreachable_issues() + and right_map is None + # don't report an error if it's intentional + and not e.right_unreachable + ): + self.msg.unreachable_right_operand(e.op, e.right) + + # If right_map is None then we know mypy considers the right branch + # to be unreachable and therefore any errors found in the right branch + # should be suppressed. + with self.msg.filter_errors(filter_errors=right_map is None): + right_type = self.analyze_cond_branch(right_map, e.right, expanded_left_type) + + if left_map is None and right_map is None: + return UninhabitedType() + + if right_map is None: + # The boolean expression is statically known to be the left value + assert left_map is not None + return left_type + if left_map is None: + # The boolean expression is statically known to be the right value + assert right_map is not None + return right_type + + if e.op == "and": + restricted_left_type = false_only(expanded_left_type) + result_is_left = not expanded_left_type.can_be_true + elif e.op == "or": + restricted_left_type = true_only(expanded_left_type) + result_is_left = not expanded_left_type.can_be_false + + if isinstance(restricted_left_type, UninhabitedType): + # The left operand can never be the result + return right_type + elif result_is_left: + # The left operand is always the result + return left_type + else: + return make_simplified_union([restricted_left_type, right_type]) def check_list_multiply(self, e: OpExpr) -> Type: """Type check an expression of form '[...] * e'. @@ -891,38 +4277,37 @@ def check_list_multiply(self, e: OpExpr) -> Type: Type inference is special-cased for this common construct. """ right_type = self.accept(e.right) - if is_subtype(right_type, self.named_type('builtins.int')): + if is_subtype(right_type, self.named_type("builtins.int")): # Special case: [...] * . Use the type context of the # OpExpr, since the multiplication does not affect the type. - left_type = self.accept(e.left, context=self.chk.type_context[-1]) + left_type = self.accept(e.left, type_context=self.type_context[-1]) else: left_type = self.accept(e.left) - result, method_type = self.check_op('__mul__', left_type, e.right, e) + result, method_type = self.check_op("__mul__", left_type, e.right, e) e.method_type = method_type return result + def visit_assignment_expr(self, e: AssignmentExpr) -> Type: + value = self.accept(e.value) + self.chk.check_assignment(e.target, e.value) + self.chk.check_final(e) + if not has_uninhabited_component(value): + # TODO: can we get rid of this extra store_type()? + # Usually, check_assignment() already stores the lvalue type correctly. + self.chk.store_type(e.target, value) + self.find_partial_type_ref_fast_path(e.target) + return value + def visit_unary_expr(self, e: UnaryExpr) -> Type: """Type check an unary operation ('not', '-', '+' or '~').""" operand_type = self.accept(e.expr) op = e.op - if op == 'not': - self.check_not_void(operand_type, e) - result = self.chk.bool_type() # type: Type - elif op == '-': - method_type = self.analyse_external_member_access('__neg__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - elif op == '+': - method_type = self.analyse_external_member_access('__pos__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type + if op == "not": + result: Type = self.bool_type() + self.chk.check_for_truthy_type(operand_type, e.expr) else: - assert op == '~', "unhandled unary operator" - method_type = self.analyse_external_member_access('__invert__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) + method = operators.unary_op_methods[op] + result, method_type = self.check_method_call_by_name(method, operand_type, [], [], e) e.method_type = method_type return result @@ -932,308 +4317,1604 @@ def visit_index_expr(self, e: IndexExpr) -> Type: It may also represent type application. """ result = self.visit_index_expr_helper(e) - return self.chk.narrow_type_from_binder(e, result) + result = self.narrow_type_from_binder(e, result) + p_result = get_proper_type(result) + if ( + self.is_literal_context() + and isinstance(p_result, Instance) + and p_result.last_known_value is not None + ): + result = p_result.last_known_value + return result def visit_index_expr_helper(self, e: IndexExpr) -> Type: if e.analyzed: # It's actually a type application. return self.accept(e.analyzed) left_type = self.accept(e.base) - if isinstance(left_type, TupleType): - # Special case for tuples. They support indexing only by integer - # literals. - index = self.strip_parens(e.index) - ok = False - if isinstance(index, IntExpr): - n = index.value - ok = True - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - n = len(left_type.items) - operand.value - ok = True - if ok: - if n >= 0 and n < len(left_type.items): - return left_type.items[n] - else: - self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) - return AnyType() + return self.visit_index_with_type(left_type, e) + + def visit_index_with_type( + self, left_type: Type, e: IndexExpr, original_type: ProperType | None = None + ) -> Type: + """Analyze type of an index expression for a given type of base expression. + + The 'original_type' is used for error messages (currently used for union types). + """ + index = e.index + left_type = get_proper_type(left_type) + + # Visit the index, just to make sure we have a type for it available + self.accept(index) + + if isinstance(left_type, TupleType) and any( + isinstance(it, UnpackType) for it in left_type.items + ): + # Normalize variadic tuples for consistency. + left_type = expand_type(left_type, {}) + + if isinstance(left_type, UnionType): + original_type = original_type or left_type + # Don't combine literal types, since we may need them for type narrowing. + return make_simplified_union( + [ + self.visit_index_with_type(typ, e, original_type) + for typ in left_type.relevant_items() + ], + contract_literals=False, + ) + elif isinstance(left_type, TupleType) and self.chk.in_checked_function(): + # Special case for tuples. They return a more specific type when + # indexed by an integer literal. + if isinstance(index, SliceExpr): + return self.visit_tuple_slice_helper(left_type, index) + + ns = self.try_getting_int_literals(index) + if ns is not None: + out = [] + for n in ns: + item = self.visit_tuple_index_helper(left_type, n) + if item is not None: + out.append(item) + else: + self.chk.fail(message_registry.TUPLE_INDEX_OUT_OF_RANGE, e) + if any(isinstance(t, UnpackType) for t in left_type.items): + min_len = self.min_tuple_length(left_type) + self.chk.note(f"Variadic tuple can have length {min_len}", e) + return AnyType(TypeOfAny.from_error) + return make_simplified_union(out) else: - self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) - return AnyType() + return self.nonliteral_tuple_index_helper(left_type, index) + elif isinstance(left_type, TypedDictType): + return self.visit_typeddict_index_expr(left_type, e.index)[0] + elif isinstance(left_type, FunctionLike) and left_type.is_type_obj(): + if left_type.type_object().is_enum: + return self.visit_enum_index_expr(left_type.type_object(), e.index, e) + elif self.chk.options.python_version >= (3, 9) and ( + left_type.type_object().type_vars + or left_type.type_object().fullname == "builtins.type" + ): + return self.named_type("types.GenericAlias") + + if isinstance(left_type, TypeVarType) and not self.has_member( + left_type.upper_bound, "__getitem__" + ): + return self.visit_index_with_type(left_type.upper_bound, e, original_type) + elif isinstance(left_type, Instance) and left_type.type.fullname == "typing._SpecialForm": + # Allow special forms to be indexed and used to create union types + return self.named_type("typing._SpecialForm") else: - result, method_type = self.check_op('__getitem__', left_type, - e.index, e) + result, method_type = self.check_method_call_by_name( + "__getitem__", left_type, [e.index], [ARG_POS], e, original_type=original_type + ) e.method_type = method_type return result + def min_tuple_length(self, left: TupleType) -> int: + unpack_index = find_unpack_in_list(left.items) + if unpack_index is None: + return left.length() + unpack = left.items[unpack_index] + assert isinstance(unpack, UnpackType) + if isinstance(unpack.type, TypeVarTupleType): + return left.length() - 1 + unpack.type.min_len + return left.length() - 1 + + def visit_tuple_index_helper(self, left: TupleType, n: int) -> Type | None: + unpack_index = find_unpack_in_list(left.items) + if unpack_index is None: + if n < 0: + n += len(left.items) + if 0 <= n < len(left.items): + return left.items[n] + return None + unpack = left.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if isinstance(unpacked, TypeVarTupleType): + # Usually we say that TypeVarTuple can't be split, be in case of + # indexing it seems benign to just return the upper bound item, similar + # to what we do when indexing a regular TypeVar. + bound = get_proper_type(unpacked.upper_bound) + assert isinstance(bound, Instance) + assert bound.type.fullname == "builtins.tuple" + middle = bound.args[0] + else: + assert isinstance(unpacked, Instance) + assert unpacked.type.fullname == "builtins.tuple" + middle = unpacked.args[0] + + extra_items = self.min_tuple_length(left) - left.length() + 1 + if n >= 0: + if n >= self.min_tuple_length(left): + # For tuple[int, *tuple[str, ...], int] we allow either index 0 or 1, + # since variadic item may have zero items. + return None + if n < unpack_index: + return left.items[n] + return UnionType.make_union( + [middle] + + left.items[unpack_index + 1 : max(n - extra_items + 2, unpack_index + 1)], + left.line, + left.column, + ) + n += self.min_tuple_length(left) + if n < 0: + # Similar to above, we only allow -1, and -2 for tuple[int, *tuple[str, ...], int] + return None + if n >= unpack_index + extra_items: + return left.items[n - extra_items + 1] + return UnionType.make_union( + left.items[min(n, unpack_index) : unpack_index] + [middle], left.line, left.column + ) + + def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr) -> Type: + begin: Sequence[int | None] = [None] + end: Sequence[int | None] = [None] + stride: Sequence[int | None] = [None] + + if slic.begin_index: + begin_raw = self.try_getting_int_literals(slic.begin_index) + if begin_raw is None: + return self.nonliteral_tuple_index_helper(left_type, slic) + begin = begin_raw + + if slic.end_index: + end_raw = self.try_getting_int_literals(slic.end_index) + if end_raw is None: + return self.nonliteral_tuple_index_helper(left_type, slic) + end = end_raw + + if slic.stride: + stride_raw = self.try_getting_int_literals(slic.stride) + if stride_raw is None: + return self.nonliteral_tuple_index_helper(left_type, slic) + stride = stride_raw + + items: list[Type] = [] + for b, e, s in itertools.product(begin, end, stride): + item = left_type.slice(b, e, s, fallback=self.named_type("builtins.tuple")) + if item is None: + self.chk.fail(message_registry.AMBIGUOUS_SLICE_OF_VARIADIC_TUPLE, slic) + return AnyType(TypeOfAny.from_error) + items.append(item) + return make_simplified_union(items) + + def try_getting_int_literals(self, index: Expression) -> list[int] | None: + """If the given expression or type corresponds to an int literal + or a union of int literals, returns a list of the underlying ints. + Otherwise, returns None. + + Specifically, this function is guaranteed to return a list with + one or more ints if one the following is true: + + 1. 'expr' is a IntExpr or a UnaryExpr backed by an IntExpr + 2. 'typ' is a LiteralType containing an int + 3. 'typ' is a UnionType containing only LiteralType of ints + """ + if isinstance(index, IntExpr): + return [index.value] + elif isinstance(index, UnaryExpr): + if index.op == "-": + operand = index.expr + if isinstance(operand, IntExpr): + return [-1 * operand.value] + if index.op == "+": + operand = index.expr + if isinstance(operand, IntExpr): + return [operand.value] + typ = get_proper_type(self.accept(index)) + if isinstance(typ, Instance) and typ.last_known_value is not None: + typ = typ.last_known_value + if isinstance(typ, LiteralType) and isinstance(typ.value, int): + return [typ.value] + if isinstance(typ, UnionType): + out = [] + for item in get_proper_types(typ.items): + if isinstance(item, LiteralType) and isinstance(item.value, int): + out.append(item.value) + else: + return None + return out + return None + + def nonliteral_tuple_index_helper(self, left_type: TupleType, index: Expression) -> Type: + self.check_method_call_by_name("__getitem__", left_type, [index], [ARG_POS], context=index) + # We could return the return type from above, but unions are often better than the join + union = self.union_tuple_fallback_item(left_type) + if isinstance(index, SliceExpr): + return self.chk.named_generic_type("builtins.tuple", [union]) + return union + + def union_tuple_fallback_item(self, left_type: TupleType) -> Type: + # TODO: this duplicates logic in typeops.tuple_fallback(). + items = [] + for item in left_type.items: + if isinstance(item, UnpackType): + unpacked_type = get_proper_type(item.type) + if isinstance(unpacked_type, TypeVarTupleType): + unpacked_type = get_proper_type(unpacked_type.upper_bound) + if ( + isinstance(unpacked_type, Instance) + and unpacked_type.type.fullname == "builtins.tuple" + ): + items.append(unpacked_type.args[0]) + else: + raise NotImplementedError + else: + items.append(item) + return make_simplified_union(items) + + def visit_typeddict_index_expr( + self, td_type: TypedDictType, index: Expression, setitem: bool = False + ) -> tuple[Type, set[str]]: + if isinstance(index, StrExpr): + key_names = [index.value] + else: + typ = get_proper_type(self.accept(index)) + if isinstance(typ, UnionType): + key_types: list[Type] = list(typ.items) + else: + key_types = [typ] + + key_names = [] + for key_type in get_proper_types(key_types): + if isinstance(key_type, Instance) and key_type.last_known_value is not None: + key_type = key_type.last_known_value + + if ( + isinstance(key_type, LiteralType) + and isinstance(key_type.value, str) + and key_type.fallback.type.fullname != "builtins.bytes" + ): + key_names.append(key_type.value) + else: + self.msg.typeddict_key_must_be_string_literal(td_type, index) + return AnyType(TypeOfAny.from_error), set() + + value_types = [] + for key_name in key_names: + value_type = td_type.items.get(key_name) + if value_type is None: + self.msg.typeddict_key_not_found(td_type, key_name, index, setitem) + return AnyType(TypeOfAny.from_error), set() + else: + value_types.append(value_type) + return make_simplified_union(value_types), set(key_names) + + def visit_enum_index_expr( + self, enum_type: TypeInfo, index: Expression, context: Context + ) -> Type: + string_type: Type = self.named_type("builtins.str") + self.chk.check_subtype( + self.accept(index), + string_type, + context, + "Enum index should be a string", + "actual index type", + ) + return Instance(enum_type, []) + def visit_cast_expr(self, expr: CastExpr) -> Type: """Type check a cast expression.""" - source_type = self.accept(expr.expr) + source_type = self.accept( + expr.expr, + type_context=AnyType(TypeOfAny.special_form), + allow_none_return=True, + always_allow_any=True, + ) target_type = expr.type - if not self.is_valid_cast(source_type, target_type): - self.msg.invalid_cast(target_type, source_type, expr) + options = self.chk.options + if ( + options.warn_redundant_casts + and not isinstance(get_proper_type(target_type), AnyType) + and source_type == target_type + ): + self.msg.redundant_cast(target_type, expr) + if options.disallow_any_unimported and has_any_from_unimported_type(target_type): + self.msg.unimported_type_becomes_any("Target type of cast", target_type, expr) + check_for_explicit_any( + target_type, self.chk.options, self.chk.is_typeshed_stub, self.msg, context=expr + ) return target_type - def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: - """Is a cast from source_type to target_type meaningful?""" - return (isinstance(target_type, AnyType) or - (not isinstance(source_type, Void) and - not isinstance(target_type, Void))) + def visit_assert_type_expr(self, expr: AssertTypeExpr) -> Type: + source_type = self.accept( + expr.expr, + type_context=self.type_context[-1], + allow_none_return=True, + always_allow_any=True, + ) + if self.chk.current_node_deferred: + return source_type + + target_type = expr.type + proper_source_type = get_proper_type(source_type) + if ( + isinstance(proper_source_type, mypy.types.Instance) + and proper_source_type.last_known_value is not None + ): + source_type = proper_source_type.last_known_value + if not is_same_type(source_type, target_type): + if not self.chk.in_checked_function(): + self.msg.note( + '"assert_type" expects everything to be "Any" in unchecked functions', + expr.expr, + ) + self.msg.assert_type_fail(source_type, target_type, expr) + return source_type + + def visit_reveal_expr(self, expr: RevealExpr) -> Type: + """Type check a reveal_type expression.""" + if expr.kind == REVEAL_TYPE: + assert expr.expr is not None + revealed_type = self.accept( + expr.expr, type_context=self.type_context[-1], allow_none_return=True + ) + if not self.chk.current_node_deferred: + self.msg.reveal_type(revealed_type, expr.expr) + if not self.chk.in_checked_function(): + self.msg.note( + "'reveal_type' always outputs 'Any' in unchecked functions", expr.expr + ) + self.check_reveal_imported(expr) + return revealed_type + else: + # REVEAL_LOCALS + if not self.chk.current_node_deferred: + # the RevealExpr contains a local_nodes attribute, + # calculated at semantic analysis time. Use it to pull out the + # corresponding subset of variables in self.chk.type_map + names_to_types = ( + {var_node.name: var_node.type for var_node in expr.local_nodes} + if expr.local_nodes is not None + else {} + ) + + self.msg.reveal_locals(names_to_types, expr) + self.check_reveal_imported(expr) + return NoneType() + + def check_reveal_imported(self, expr: RevealExpr) -> None: + if codes.UNIMPORTED_REVEAL not in self.chk.options.enabled_error_codes: + return + + name = "" + if expr.kind == REVEAL_LOCALS: + name = "reveal_locals" + elif expr.kind == REVEAL_TYPE and not expr.is_imported: + name = "reveal_type" + else: + return + + self.chk.fail(f'Name "{name}" is not defined', expr, code=codes.UNIMPORTED_REVEAL) + if name == "reveal_type": + module = ( + "typing" if self.chk.options.python_version >= (3, 11) else "typing_extensions" + ) + hint = ( + 'Did you forget to import it from "{module}"?' + ' (Suggestion: "from {module} import {name}")' + ).format(module=module, name=name) + self.chk.note(hint, expr, code=codes.UNIMPORTED_REVEAL) def visit_type_application(self, tapp: TypeApplication) -> Type: - """Type check a type application (expr[type, ...]).""" - expr_type = self.accept(tapp.expr) - if isinstance(expr_type, Callable): - new_type = self.apply_generic_arguments(expr_type, - tapp.types, tapp) - elif isinstance(expr_type, Overloaded): - overload = expr_type - # Only target items with the right number of generic type args. - items = [c for c in overload.items() - if len(c.variables) == len(tapp.types)] - new_type = self.apply_generic_arguments2(Overloaded(items), - tapp.types, tapp) + """Type check a type application (expr[type, ...]). + + There are two different options here, depending on whether expr refers + to a type alias or directly to a generic class. In the first case we need + to use a dedicated function typeanal.instantiate_type_alias(). This + is due to slight differences in how type arguments are applied and checked. + """ + if isinstance(tapp.expr, RefExpr) and isinstance(tapp.expr.node, TypeAlias): + if tapp.expr.node.python_3_12_type_alias: + return self.type_alias_type_type() + # Subscription of a (generic) alias in runtime context, expand the alias. + item = instantiate_type_alias( + tapp.expr.node, + tapp.types, + self.chk.fail, + tapp.expr.node.no_args, + tapp, + self.chk.options, + ) + item = get_proper_type(item) + if isinstance(item, Instance): + tp = type_object_type(item.type, self.named_type) + return self.apply_type_arguments_to_callable(tp, item.args, tapp) + elif isinstance(item, TupleType) and item.partial_fallback.type.is_named_tuple: + tp = type_object_type(item.partial_fallback.type, self.named_type) + return self.apply_type_arguments_to_callable(tp, item.partial_fallback.args, tapp) + elif isinstance(item, TypedDictType): + return self.typeddict_callable_from_context(item) + else: + self.chk.fail(message_registry.ONLY_CLASS_APPLICATION, tapp) + return AnyType(TypeOfAny.from_error) + # Type application of a normal generic class in runtime context. + # This is typically used as `x = G[int]()`. + tp = get_proper_type(self.accept(tapp.expr)) + if isinstance(tp, (CallableType, Overloaded)): + if not tp.is_type_obj(): + self.chk.fail(message_registry.ONLY_CLASS_APPLICATION, tapp) + return self.apply_type_arguments_to_callable(tp, tapp.types, tapp) + if isinstance(tp, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=tp) + return AnyType(TypeOfAny.special_form) + + def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: + """Right hand side of a type alias definition. + + It has the same type as if the alias itself was used in a runtime context. + For example, here: + + A = reveal_type(List[T]) + reveal_type(A) + + both `reveal_type` instances will reveal the same type `def (...) -> builtins.list[Any]`. + Note that type variables are implicitly substituted with `Any`. + """ + return self.alias_type_in_runtime_context(alias.node, ctx=alias, alias_definition=True) + + def alias_type_in_runtime_context( + self, alias: TypeAlias, *, ctx: Context, alias_definition: bool = False + ) -> Type: + """Get type of a type alias (could be generic) in a runtime expression. + + Note that this function can be called only if the alias appears _not_ + as a target of type application, which is treated separately in the + visit_type_application method. Some examples where this method is called are + casts and instantiation: + + class LongName(Generic[T]): ... + A = LongName[int] + + x = A() + y = cast(A, ...) + """ + if alias.python_3_12_type_alias: + return self.type_alias_type_type() + if isinstance(alias.target, Instance) and alias.target.invalid: # type: ignore[misc] + # An invalid alias, error already has been reported + return AnyType(TypeOfAny.from_error) + # If this is a generic alias, we set all variables to `Any`. + # For example: + # A = List[Tuple[T, T]] + # x = A() <- same as List[Tuple[Any, Any]], see PEP 484. + disallow_any = self.chk.options.disallow_any_generics and self.is_callee + item = get_proper_type( + set_any_tvars( + alias, + [], + ctx.line, + ctx.column, + self.chk.options, + disallow_any=disallow_any, + fail=self.msg.fail, + ) + ) + if isinstance(item, Instance): + # Normally we get a callable type (or overloaded) with .is_type_obj() true + # representing the class's constructor + tp = type_object_type(item.type, self.named_type) + if alias.no_args: + return tp + return self.apply_type_arguments_to_callable(tp, item.args, ctx) + elif ( + isinstance(item, TupleType) + and + # Tuple[str, int]() fails at runtime, only named tuples and subclasses work. + tuple_fallback(item).type.fullname != "builtins.tuple" + ): + return type_object_type(tuple_fallback(item).type, self.named_type) + elif isinstance(item, TypedDictType): + return self.typeddict_callable_from_context(item) + elif isinstance(item, NoneType): + return TypeType(item, line=item.line, column=item.column) + elif isinstance(item, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=item) + elif ( + isinstance(item, UnionType) + and item.uses_pep604_syntax + and self.chk.options.python_version >= (3, 10) + ): + return self.chk.named_generic_type("types.UnionType", item.items) else: - self.chk.fail(messages.INVALID_TYPE_APPLICATION_TARGET_TYPE, tapp) - new_type = AnyType() - self.chk.type_map[tapp.expr] = new_type - return new_type + if alias_definition: + return AnyType(TypeOfAny.special_form) + # The _SpecialForm type can be used in some runtime contexts (e.g. it may have __or__). + return self.named_type("typing._SpecialForm") + + def split_for_callable( + self, t: CallableType, args: Sequence[Type], ctx: Context + ) -> list[Type]: + """Handle directly applying type arguments to a variadic Callable. + + This is needed in situations where e.g. variadic class object appears in + runtime context. For example: + class C(Generic[T, Unpack[Ts]]): ... + x = C[int, str]() + + We simply group the arguments that need to go into Ts variable into a TupleType, + similar to how it is done in other places using split_with_prefix_and_suffix(). + """ + if t.is_type_obj(): + # Type arguments must map to class type variables, ignoring constructor vars. + vars = t.type_object().defn.type_vars + else: + vars = list(t.variables) + args = flatten_nested_tuples(args) + + # TODO: this logic is duplicated with semanal_typeargs. + for tv, arg in zip(t.variables, args): + if isinstance(tv, ParamSpecType): + if not isinstance( + get_proper_type(arg), (Parameters, ParamSpecType, AnyType, UnboundType) + ): + self.chk.fail( + "Can only replace ParamSpec with a parameter types list or" + f" another ParamSpec, got {format_type(arg, self.chk.options)}", + ctx, + ) + return [AnyType(TypeOfAny.from_error)] * len(vars) + + if not vars or not any(isinstance(v, TypeVarTupleType) for v in vars): + return list(args) + # TODO: in future we may want to support type application to variadic functions. + assert t.is_type_obj() + info = t.type_object() + # We reuse the logic from semanal phase to reduce code duplication. + fake = Instance(info, args, line=ctx.line, column=ctx.column) + # This code can be only called either from checking a type application, or from + # checking a type alias (after the caller handles no_args aliases), so we know it + # was initially an IndexExpr, and we allow empty tuple type arguments. + if not validate_instance(fake, self.chk.fail, empty_tuple_index=True): + fix_instance( + fake, self.chk.fail, self.chk.note, disallow_any=False, options=self.chk.options + ) + args = list(fake.args) + + prefix = next(i for (i, v) in enumerate(vars) if isinstance(v, TypeVarTupleType)) + suffix = len(vars) - prefix - 1 + tvt = vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + start, middle, end = split_with_prefix_and_suffix(tuple(args), prefix, suffix) + return list(start) + [TupleType(list(middle), tvt.tuple_fallback)] + list(end) + + def apply_type_arguments_to_callable( + self, tp: Type, args: Sequence[Type], ctx: Context + ) -> Type: + """Apply type arguments to a generic callable type coming from a type object. + + This will first perform type arguments count checks, report the + error as needed, and return the correct kind of Any. As a special + case this returns Any for non-callable types, because if type object type + is not callable, then an error should be already reported. + """ + tp = get_proper_type(tp) + + if isinstance(tp, CallableType): + if tp.is_type_obj(): + # If we have a class object in runtime context, then the available type + # variables are those of the class, we don't include additional variables + # of the constructor. So that with + # class C(Generic[T]): + # def __init__(self, f: Callable[[S], T], x: S) -> None + # C[int] is valid + # C[int, str] is invalid (although C as a callable has 2 type variables) + # Note: various logic below and in applytype.py relies on the fact that + # class type variables appear *before* constructor variables. + type_vars = tp.type_object().defn.type_vars + else: + type_vars = list(tp.variables) + min_arg_count = sum(not v.has_default() for v in type_vars) + has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in type_vars) + if ( + len(args) < min_arg_count or len(args) > len(type_vars) + ) and not has_type_var_tuple: + if tp.is_type_obj() and tp.type_object().fullname == "builtins.tuple": + # e.g. expression tuple[X, Y] + # - want the type of the expression i.e. a function with that as its return type + # - tp is type of tuple (note it won't have params as we are only called + # with generic callable type) + # - tuple[X, Y]() takes a single arg that is a tuple containing an X and a Y + return CallableType( + [TupleType(list(args), self.chk.named_type("tuple"))], + [ARG_POS], + [None], + TupleType(list(args), self.chk.named_type("tuple")), + tp.fallback, + name="tuple", + definition=tp.definition, + bound_args=tp.bound_args, + ) + self.msg.incompatible_type_application( + min_arg_count, len(type_vars), len(args), ctx + ) + return AnyType(TypeOfAny.from_error) + return self.apply_generic_arguments(tp, self.split_for_callable(tp, args, ctx), ctx) + if isinstance(tp, Overloaded): + for it in tp.items: + if tp.is_type_obj(): + # Same as above. + type_vars = tp.type_object().defn.type_vars + else: + type_vars = list(it.variables) + min_arg_count = sum(not v.has_default() for v in type_vars) + has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in type_vars) + if ( + len(args) < min_arg_count or len(args) > len(type_vars) + ) and not has_type_var_tuple: + self.msg.incompatible_type_application( + min_arg_count, len(type_vars), len(args), ctx + ) + return AnyType(TypeOfAny.from_error) + return Overloaded( + [ + self.apply_generic_arguments(it, self.split_for_callable(it, args, ctx), ctx) + for it in tp.items + ] + ) + return AnyType(TypeOfAny.special_form) def visit_list_expr(self, e: ListExpr) -> Type: """Type check a list expression [...].""" - return self.check_list_or_set_expr(e.items, 'builtins.list', '', - e) + return self.check_lst_expr(e, "builtins.list", "") def visit_set_expr(self, e: SetExpr) -> Type: - return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) + return self.check_lst_expr(e, "builtins.set", "") + + def fast_container_type( + self, e: ListExpr | SetExpr | TupleExpr, container_fullname: str + ) -> Type | None: + """ + Fast path to determine the type of a list or set literal, + based on the list of entries. This mostly impacts large + module-level constant definitions. + + Limitations: + - no active type context + - no star expressions + - the joined type of all entries must be an Instance or Tuple type + """ + ctx = self.type_context[-1] + if ctx: + return None + rt = self.resolved_type.get(e, None) + if rt is not None: + return rt if isinstance(rt, Instance) else None + values: list[Type] = [] + for item in e.items: + if isinstance(item, StarExpr): + # fallback to slow path + self.resolved_type[e] = NoneType() + return None + values.append(self.accept(item)) + vt = join.join_type_list(values) + if not allow_fast_container_literal(vt): + self.resolved_type[e] = NoneType() + return None + ct = self.chk.named_generic_type(container_fullname, [vt]) + self.resolved_type[e] = ct + return ct + + def check_lst_expr(self, e: ListExpr | SetExpr | TupleExpr, fullname: str, tag: str) -> Type: + # fast path + t = self.fast_container_type(e, fullname) + if t: + return t - def check_list_or_set_expr(self, items: List[Node], fullname: str, - tag: str, context: Context) -> Type: # Translate into type checking a generic function call. - tv = TypeVar('T', -1, [], self.chk.object_type()) - constructor = Callable([tv], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type(fullname, - [tv]), - self.named_type('builtins.function'), - tag, - [TypeVarDef('T', -1, None, self.chk.object_type())]) - return self.check_call(constructor, - items, - [nodes.ARG_POS] * len(items), context)[0] + # Used for list and set expressions, as well as for tuples + # containing star expressions that don't refer to a + # Tuple. (Note: "lst" stands for list-set-tuple. :-) + tv = TypeVarType( + "T", + "T", + id=TypeVarId(-1, namespace=""), + values=[], + upper_bound=self.object_type(), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + constructor = CallableType( + [tv], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type(fullname, [tv]), + self.named_type("builtins.function"), + name=tag, + variables=[tv], + ) + out = self.check_call( + constructor, + [(i.expr if isinstance(i, StarExpr) else i) for i in e.items], + [(nodes.ARG_STAR if isinstance(i, StarExpr) else nodes.ARG_POS) for i in e.items], + e, + )[0] + return remove_instance_last_known_values(out) + + def tuple_context_matches(self, expr: TupleExpr, ctx: TupleType) -> bool: + ctx_unpack_index = find_unpack_in_list(ctx.items) + if ctx_unpack_index is None: + # For fixed tuples accept everything that can possibly match, even if this + # requires all star items to be empty. + return len([e for e in expr.items if not isinstance(e, StarExpr)]) <= len(ctx.items) + # For variadic context, the only easy case is when structure matches exactly. + # TODO: try using tuple type context in more cases. + if len([e for e in expr.items if isinstance(e, StarExpr)]) != 1: + return False + expr_star_index = next(i for i, lv in enumerate(expr.items) if isinstance(lv, StarExpr)) + return len(expr.items) == len(ctx.items) and ctx_unpack_index == expr_star_index def visit_tuple_expr(self, e: TupleExpr) -> Type: """Type check a tuple expression.""" - ctx = None # type: TupleType # Try to determine type context for type inference. - if isinstance(self.chk.type_context[-1], TupleType): - t = cast(TupleType, self.chk.type_context[-1]) - if len(t.items) == len(e.items): - ctx = t - # Infer item types. - items = [] # type: List[Type] + type_context = get_proper_type(self.type_context[-1]) + type_context_items = None + if isinstance(type_context, UnionType): + tuples_in_context = [ + t + for t in get_proper_types(type_context.items) + if (isinstance(t, TupleType) and self.tuple_context_matches(e, t)) + or is_named_instance(t, TUPLE_LIKE_INSTANCE_NAMES) + ] + if len(tuples_in_context) == 1: + type_context = tuples_in_context[0] + else: + # There are either no relevant tuples in the Union, or there is + # more than one. Either way, we can't decide on a context. + pass + + if isinstance(type_context, TupleType) and self.tuple_context_matches(e, type_context): + type_context_items = type_context.items + elif type_context and is_named_instance(type_context, TUPLE_LIKE_INSTANCE_NAMES): + assert isinstance(type_context, Instance) + if type_context.args: + type_context_items = [type_context.args[0]] * len(e.items) + # NOTE: it's possible for the context to have a different + # number of items than e. In that case we use those context + # items that match a position in e, and we'll worry about type + # mismatches later. + + unpack_in_context = False + if type_context_items is not None: + unpack_in_context = find_unpack_in_list(type_context_items) is not None + seen_unpack_in_items = False + allow_precise_tuples = ( + unpack_in_context or PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature + ) + + # Infer item types. Give up if there's a star expression + # that's not a Tuple. + items: list[Type] = [] + j = 0 # Index into type_context_items; irrelevant if type_context_items is none for i in range(len(e.items)): item = e.items[i] - tt = Undefined # type: Type - if not ctx: - tt = self.accept(item) + if isinstance(item, StarExpr): + # Special handling for star expressions. + # TODO: If there's a context, and item.expr is a + # TupleExpr, flatten it, so we can benefit from the + # context? Counterargument: Why would anyone write + # (1, *(2, 3)) instead of (1, 2, 3) except in a test? + if unpack_in_context: + # Note: this logic depends on full structure match in tuple_context_matches(). + assert type_context_items + ctx_item = type_context_items[j] + assert isinstance(ctx_item, UnpackType) + ctx = ctx_item.type + else: + ctx = None + tt = self.accept(item.expr, ctx) + tt = get_proper_type(tt) + if isinstance(tt, TupleType): + if find_unpack_in_list(tt.items) is not None: + if seen_unpack_in_items: + # Multiple unpack items are not allowed in tuples, + # fall back to instance type. + return self.check_lst_expr(e, "builtins.tuple", "") + else: + seen_unpack_in_items = True + items.extend(tt.items) + # Note: this logic depends on full structure match in tuple_context_matches(). + if unpack_in_context: + j += 1 + else: + # If there is an unpack in expressions, but not in context, this will + # result in an error later, just do something predictable here. + j += len(tt.items) + else: + if allow_precise_tuples and not seen_unpack_in_items: + # Handle (x, *y, z), where y is e.g. tuple[Y, ...]. + if isinstance(tt, Instance) and self.chk.type_is_iterable(tt): + item_type = self.chk.iterable_item_type(tt, e) + mapped = self.chk.named_generic_type("builtins.tuple", [item_type]) + items.append(UnpackType(mapped)) + seen_unpack_in_items = True + continue + # A star expression that's not a Tuple. + # Treat the whole thing as a variable-length tuple. + return self.check_lst_expr(e, "builtins.tuple", "") else: - tt = self.accept(item, ctx.items[i]) - self.check_not_void(tt, e) - items.append(tt) - return TupleType(items, self.named_type('builtins.tuple')) + if not type_context_items or j >= len(type_context_items): + tt = self.accept(item) + else: + tt = self.accept(item, type_context_items[j]) + j += 1 + items.append(tt) + # This is a partial fallback item type. A precise type will be calculated on demand. + fallback_item = AnyType(TypeOfAny.special_form) + result: ProperType = TupleType( + items, self.chk.named_generic_type("builtins.tuple", [fallback_item]) + ) + if seen_unpack_in_items: + # Return already normalized tuple type just in case. + result = expand_type(result, {}) + return result - def visit_dict_expr(self, e: DictExpr) -> Type: - # Translate into type checking a generic function call. - tv1 = TypeVar('KT', -1, [], self.chk.object_type()) - tv2 = TypeVar('VT', -2, [], self.chk.object_type()) - constructor = Undefined(Callable) - # The callable type represents a function like this: - # - # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... - constructor = Callable([TupleType([tv1, tv2], self.named_type('builtins.tuple'))], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type('builtins.dict', - [tv1, tv2]), - self.named_type('builtins.function'), - '', - [TypeVarDef('KT', -1, None, self.chk.object_type()), - TypeVarDef('VT', -2, None, self.chk.object_type())]) - # Synthesize function arguments. - args = List[Node]() + def fast_dict_type(self, e: DictExpr) -> Type | None: + """ + Fast path to determine the type of a dict literal, + based on the list of entries. This mostly impacts large + module-level constant definitions. + + Limitations: + - no active type context + - only supported star expressions are other dict instances + - the joined types of all keys and values must be Instance or Tuple types + """ + ctx = self.type_context[-1] + if ctx: + return None + rt = self.resolved_type.get(e, None) + if rt is not None: + return rt if isinstance(rt, Instance) else None + keys: list[Type] = [] + values: list[Type] = [] + stargs: tuple[Type, Type] | None = None for key, value in e.items: - args.append(TupleExpr([key, value])) - return self.check_call(constructor, - args, - [nodes.ARG_POS] * len(args), e)[0] + if key is None: + st = get_proper_type(self.accept(value)) + if ( + isinstance(st, Instance) + and st.type.fullname == "builtins.dict" + and len(st.args) == 2 + ): + stargs = (st.args[0], st.args[1]) + else: + self.resolved_type[e] = NoneType() + return None + else: + keys.append(self.accept(key)) + values.append(self.accept(value)) + kt = join.join_type_list(keys) + vt = join.join_type_list(values) + if not (allow_fast_container_literal(kt) and allow_fast_container_literal(vt)): + self.resolved_type[e] = NoneType() + return None + if stargs and (stargs[0] != kt or stargs[1] != vt): + self.resolved_type[e] = NoneType() + return None + dt = self.chk.named_generic_type("builtins.dict", [kt, vt]) + self.resolved_type[e] = dt + return dt + + def check_typeddict_literal_in_context( + self, e: DictExpr, typeddict_context: TypedDictType + ) -> Type: + orig_ret_type = self.check_typeddict_call_with_dict( + callee=typeddict_context, kwargs=e.items, context=e, orig_callee=None + ) + ret_type = get_proper_type(orig_ret_type) + if isinstance(ret_type, TypedDictType): + return ret_type.copy_modified() + return typeddict_context.copy_modified() + + def visit_dict_expr(self, e: DictExpr) -> Type: + """Type check a dict expression. - def visit_func_expr(self, e: FuncExpr) -> Type: + Translate it into a call to dict(), with provisions for **expr. + """ + # if the dict literal doesn't match TypedDict, check_typeddict_call_with_dict reports + # an error, but returns the TypedDict type that matches the literal it found + # that would cause a second error when that TypedDict type is returned upstream + # to avoid the second error, we always return TypedDict type that was requested + typeddict_contexts = self.find_typeddict_context(self.type_context[-1], e) + if typeddict_contexts: + if len(typeddict_contexts) == 1: + return self.check_typeddict_literal_in_context(e, typeddict_contexts[0]) + # Multiple items union, check if at least one of them matches cleanly. + for typeddict_context in typeddict_contexts: + with self.msg.filter_errors() as err, self.chk.local_type_map() as tmap: + ret_type = self.check_typeddict_literal_in_context(e, typeddict_context) + if err.has_new_errors(): + continue + self.chk.store_types(tmap) + return ret_type + # No item matched without an error, so we can't unambiguously choose the item. + self.msg.typeddict_context_ambiguous(typeddict_contexts, e) + + # fast path attempt + dt = self.fast_dict_type(e) + if dt: + return dt + + # Define type variables (used in constructors below). + kt = TypeVarType( + "KT", + "KT", + id=TypeVarId(-1, namespace=""), + values=[], + upper_bound=self.object_type(), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + vt = TypeVarType( + "VT", + "VT", + id=TypeVarId(-2, namespace=""), + values=[], + upper_bound=self.object_type(), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + + # Collect function arguments, watching out for **expr. + args: list[Expression] = [] + expected_types: list[Type] = [] + for key, value in e.items: + if key is None: + args.append(value) + expected_types.append( + self.chk.named_generic_type("_typeshed.SupportsKeysAndGetItem", [kt, vt]) + ) + else: + tup = TupleExpr([key, value]) + if key.line >= 0: + tup.line = key.line + tup.column = key.column + else: + tup.line = value.line + tup.column = value.column + tup.end_line = value.end_line + tup.end_column = value.end_column + args.append(tup) + expected_types.append(TupleType([kt, vt], self.named_type("builtins.tuple"))) + + # The callable type represents a function like this (except we adjust for **expr): + # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... + constructor = CallableType( + expected_types, + [nodes.ARG_POS] * len(expected_types), + [None] * len(expected_types), + self.chk.named_generic_type("builtins.dict", [kt, vt]), + self.named_type("builtins.function"), + name="", + variables=[kt, vt], + ) + return self.check_call(constructor, args, [nodes.ARG_POS] * len(args), e)[0] + + def find_typeddict_context( + self, context: Type | None, dict_expr: DictExpr + ) -> list[TypedDictType]: + context = get_proper_type(context) + if isinstance(context, TypedDictType): + return [context] + elif isinstance(context, UnionType): + items = [] + for item in context.items: + item_contexts = self.find_typeddict_context(item, dict_expr) + for item_context in item_contexts: + if self.match_typeddict_call_with_dict( + item_context, dict_expr.items, dict_expr + ): + items.append(item_context) + return items + # No TypedDict type in context. + return [] + + def visit_lambda_expr(self, e: LambdaExpr) -> Type: """Type check lambda expression.""" - inferred_type = self.infer_lambda_type_using_context(e) + self.chk.check_default_args(e, body_is_trivial=False) + inferred_type, type_override = self.infer_lambda_type_using_context(e) if not inferred_type: - # No useful type context. - ret_type = e.expr().accept(self.chk) - if not e.args: - # Form 'lambda: e'; just use the inferred return type. - return Callable([], [], [], ret_type, self.named_type('builtins.function')) - else: - # TODO: Consider reporting an error. However, this is fine if - # we are just doing the first pass in contextual type - # inference. - return AnyType() + self.chk.return_types.append(AnyType(TypeOfAny.special_form)) + # Type check everything in the body except for the final return + # statement (it can contain tuple unpacking before return). + with self.chk.binder.frame_context( + can_skip=True, fall_through=0 + ), self.chk.scope.push_function(e): + # Lambdas can have more than one element in body, + # when we add "fictional" AssigmentStatement nodes, like in: + # `lambda (a, b): a` + for stmt in e.body.body[:-1]: + stmt.accept(self.chk) + # Only type check the return expression, not the return statement. + # There's no useful type context. + ret_type = self.accept(e.expr(), allow_none_return=True) + fallback = self.named_type("builtins.function") + self.chk.return_types.pop() + return callable_type(e, fallback, ret_type) else: # Type context available. - self.chk.check_func_item(e, type_override=inferred_type) - ret_type = self.chk.type_map[e.expr()] + self.chk.return_types.append(inferred_type.ret_type) + with self.chk.tscope.function_scope(e): + self.chk.check_func_item(e, type_override=type_override) + if not self.chk.has_type(e.expr()): + # TODO: return expression must be accepted before exiting function scope. + with self.chk.binder.frame_context(can_skip=True, fall_through=0): + self.accept(e.expr(), allow_none_return=True) + ret_type = self.chk.lookup_type(e.expr()) + self.chk.return_types.pop() return replace_callable_return_type(inferred_type, ret_type) - def infer_lambda_type_using_context(self, e: FuncExpr) -> Callable: + def infer_lambda_type_using_context( + self, e: LambdaExpr + ) -> tuple[CallableType | None, CallableType | None]: """Try to infer lambda expression type using context. Return None if could not infer type. + The second item in the return type is the type_override parameter for check_func_item. """ # TODO also accept 'Any' context - ctx = self.chk.type_context[-1] - if not ctx or not isinstance(ctx, Callable): - return None + ctx = get_proper_type(self.type_context[-1]) + + if isinstance(ctx, UnionType): + callables = [ + t for t in get_proper_types(ctx.relevant_items()) if isinstance(t, CallableType) + ] + if len(callables) == 1: + ctx = callables[0] + + if not ctx or not isinstance(ctx, CallableType): + return None, None # The context may have function type variables in it. We replace them # since these are the type variables we are ultimately trying to infer; # they must be considered as indeterminate. We use ErasedType since it # does not affect type inference results (it is for purposes like this # only). - ctx = replace_func_type_vars(ctx, ErasedType()) - - callable_ctx = cast(Callable, ctx) - - if callable_ctx.arg_kinds != e.arg_kinds: + if not self.chk.options.old_type_inference: + # With new type inference we can preserve argument types even if they + # are generic, since new inference algorithm can handle constraints + # like S <: T (we still erase return type since it's ultimately unknown). + extra_vars = [] + for arg in ctx.arg_types: + meta_vars = [tv for tv in get_all_type_vars(arg) if tv.id.is_meta_var()] + extra_vars.extend([tv for tv in meta_vars if tv not in extra_vars]) + callable_ctx = ctx.copy_modified( + ret_type=replace_meta_vars(ctx.ret_type, ErasedType()), + variables=list(ctx.variables) + extra_vars, + ) + else: + erased_ctx = replace_meta_vars(ctx, ErasedType()) + assert isinstance(erased_ctx, ProperType) and isinstance(erased_ctx, CallableType) + callable_ctx = erased_ctx + + # The callable_ctx may have a fallback of builtins.type if the context + # is a constructor -- but this fallback doesn't make sense for lambdas. + callable_ctx = callable_ctx.copy_modified(fallback=self.named_type("builtins.function")) + + if callable_ctx.type_guard is not None or callable_ctx.type_is is not None: + # Lambda's return type cannot be treated as a `TypeGuard`, + # because it is implicit. And `TypeGuard`s must be explicit. + # See https://github.com/python/mypy/issues/9927 + return None, None + + arg_kinds = [arg.kind for arg in e.arguments] + + if callable_ctx.is_ellipsis_args or ctx.param_spec() is not None: + # Fill in Any arguments to match the arguments of the lambda. + callable_ctx = callable_ctx.copy_modified( + is_ellipsis_args=False, + arg_types=[AnyType(TypeOfAny.special_form)] * len(arg_kinds), + arg_kinds=arg_kinds, + arg_names=e.arg_names.copy(), + ) + + if ARG_STAR in arg_kinds or ARG_STAR2 in arg_kinds: + # TODO treat this case appropriately + return callable_ctx, None + + if callable_ctx.arg_kinds != arg_kinds: # Incompatible context; cannot use it to infer types. - self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) - return None + self.chk.fail(message_registry.CANNOT_INFER_LAMBDA_TYPE, e) + return None, None - return callable_ctx + # Type of lambda must have correct argument names, to prevent false + # negatives when lambdas appear in `ParamSpec` context. + return callable_ctx.copy_modified(arg_names=e.arg_names), callable_ctx def visit_super_expr(self, e: SuperExpr) -> Type: """Type check a super expression (non-lvalue).""" - t = self.analyse_super(e, False) - return t - - def analyse_super(self, e: SuperExpr, is_lvalue: bool) -> Type: - """Type check a super expression.""" - if e.info and e.info.bases: - # TODO fix multiple inheritance etc - return analyse_member_access(e.name, self_type(e.info), e, - is_lvalue, True, - self.named_type, self.msg, - e.info.mro[1]) + + # We have an expression like super(T, var).member + + # First compute the types of T and var + types = self._super_arg_types(e) + if isinstance(types, tuple): + type_type, instance_type = types else: - # Invalid super. This has been reported by the semantic analyser. - return AnyType() + return types + + # Now get the MRO + type_info = type_info_from_type(type_type) + if type_info is None: + self.chk.fail(message_registry.UNSUPPORTED_ARG_1_FOR_SUPER, e) + return AnyType(TypeOfAny.from_error) + + instance_info = type_info_from_type(instance_type) + if instance_info is None: + self.chk.fail(message_registry.UNSUPPORTED_ARG_2_FOR_SUPER, e) + return AnyType(TypeOfAny.from_error) + + mro = instance_info.mro + + # The base is the first MRO entry *after* type_info that has a member + # with the right name + index = None + if type_info in mro: + index = mro.index(type_info) + else: + method = self.chk.scope.top_function() + # Mypy explicitly allows supertype upper bounds (and no upper bound at all) + # for annotating self-types. However, if such an annotation is used for + # checking super() we will still get an error. So to be consistent, we also + # allow such imprecise annotations for use with super(), where we fall back + # to the current class MRO instead. This works only from inside a method. + if method is not None and is_self_type_like( + instance_type, is_classmethod=method.is_class + ): + if e.info and type_info in e.info.mro: + mro = e.info.mro + index = mro.index(type_info) + if index is None: + if ( + instance_info.is_protocol + and instance_info != type_info + and not type_info.is_protocol + ): + # A special case for mixins, in this case super() should point + # directly to the host protocol, this is not safe, since the real MRO + # is not known yet for mixin, but this feature is more like an escape hatch. + index = -1 + else: + self.chk.fail(message_registry.SUPER_ARG_2_NOT_INSTANCE_OF_ARG_1, e) + return AnyType(TypeOfAny.from_error) + + if len(mro) == index + 1: + self.chk.fail(message_registry.TARGET_CLASS_HAS_NO_BASE_CLASS, e) + return AnyType(TypeOfAny.from_error) + + for base in mro[index + 1 :]: + if e.name in base.names or base == mro[-1]: + if e.info and e.info.fallback_to_any and base == mro[-1]: + # There's an undefined base class, and we're at the end of the + # chain. That's not an error. + return AnyType(TypeOfAny.special_form) + + return analyze_member_access( + name=e.name, + typ=instance_type, + is_lvalue=False, + is_super=True, + is_operator=False, + original_type=instance_type, + override_info=base, + context=e, + msg=self.msg, + chk=self.chk, + in_literal_context=self.is_literal_context(), + ) + + assert False, "unreachable" + + def _super_arg_types(self, e: SuperExpr) -> Type | tuple[Type, Type]: + """ + Computes the types of the type and instance expressions in super(T, instance), or the + implicit ones for zero-argument super() expressions. Returns a single type for the whole + super expression when possible (for errors, anys), otherwise the pair of computed types. + """ - def visit_paren_expr(self, e: ParenExpr) -> Type: - """Type check a parenthesised expression.""" - return self.accept(e.expr, self.chk.type_context[-1]) + if not self.chk.in_checked_function(): + return AnyType(TypeOfAny.unannotated) + elif len(e.call.args) == 0: + if not e.info: + # This has already been reported by the semantic analyzer. + return AnyType(TypeOfAny.from_error) + elif self.chk.scope.active_class(): + self.chk.fail(message_registry.SUPER_OUTSIDE_OF_METHOD_NOT_SUPPORTED, e) + return AnyType(TypeOfAny.from_error) + + # Zero-argument super() is like super(, ) + current_type = fill_typevars(e.info) + type_type: ProperType = TypeType(current_type) + + # Use the type of the self argument, in case it was annotated + method = self.chk.scope.top_function() + assert method is not None + if method.arguments: + instance_type: Type = method.arguments[0].variable.type or current_type + else: + self.chk.fail(message_registry.SUPER_ENCLOSING_POSITIONAL_ARGS_REQUIRED, e) + return AnyType(TypeOfAny.from_error) + elif ARG_STAR in e.call.arg_kinds: + self.chk.fail(message_registry.SUPER_VARARGS_NOT_SUPPORTED, e) + return AnyType(TypeOfAny.from_error) + elif set(e.call.arg_kinds) != {ARG_POS}: + self.chk.fail(message_registry.SUPER_POSITIONAL_ARGS_REQUIRED, e) + return AnyType(TypeOfAny.from_error) + elif len(e.call.args) == 1: + self.chk.fail(message_registry.SUPER_WITH_SINGLE_ARG_NOT_SUPPORTED, e) + return AnyType(TypeOfAny.from_error) + elif len(e.call.args) == 2: + type_type = get_proper_type(self.accept(e.call.args[0])) + instance_type = self.accept(e.call.args[1]) + else: + self.chk.fail(message_registry.TOO_MANY_ARGS_FOR_SUPER, e) + return AnyType(TypeOfAny.from_error) + + # Imprecisely assume that the type is the current class + if isinstance(type_type, AnyType): + if e.info: + type_type = TypeType(fill_typevars(e.info)) + else: + return AnyType(TypeOfAny.from_another_any, source_any=type_type) + elif isinstance(type_type, TypeType): + type_item = type_type.item + if isinstance(type_item, AnyType): + if e.info: + type_type = TypeType(fill_typevars(e.info)) + else: + return AnyType(TypeOfAny.from_another_any, source_any=type_item) + + if not isinstance(type_type, TypeType) and not ( + isinstance(type_type, FunctionLike) and type_type.is_type_obj() + ): + self.msg.first_argument_for_super_must_be_type(type_type, e) + return AnyType(TypeOfAny.from_error) + + # Imprecisely assume that the instance is of the current class + instance_type = get_proper_type(instance_type) + if isinstance(instance_type, AnyType): + if e.info: + instance_type = fill_typevars(e.info) + else: + return AnyType(TypeOfAny.from_another_any, source_any=instance_type) + elif isinstance(instance_type, TypeType): + instance_item = instance_type.item + if isinstance(instance_item, AnyType): + if e.info: + instance_type = TypeType(fill_typevars(e.info)) + else: + return AnyType(TypeOfAny.from_another_any, source_any=instance_item) + + return type_type, instance_type def visit_slice_expr(self, e: SliceExpr) -> Type: + try: + supports_index = self.chk.named_type("typing_extensions.SupportsIndex") + except KeyError: + supports_index = self.chk.named_type("builtins.int") # thanks, fixture life + expected = make_optional_type(supports_index) + type_args = [] for index in [e.begin_index, e.end_index, e.stride]: if index: t = self.accept(index) - self.chk.check_subtype(t, self.named_type('builtins.int'), - index, messages.INVALID_SLICE_INDEX) - return self.named_type('builtins.slice') + self.chk.check_subtype(t, expected, index, message_registry.INVALID_SLICE_INDEX) + type_args.append(t) + else: + type_args.append(NoneType()) + return self.chk.named_generic_type("builtins.slice", type_args) def visit_list_comprehension(self, e: ListComprehension) -> Type: return self.check_generator_or_comprehension( - e.generator, 'builtins.list', '') + e.generator, "builtins.list", "" + ) def visit_set_comprehension(self, e: SetComprehension) -> Type: return self.check_generator_or_comprehension( - e.generator, 'builtins.set', '') + e.generator, "builtins.set", "" + ) def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.check_generator_or_comprehension(e, 'typing.Iterator', - '') - - def check_generator_or_comprehension(self, gen: GeneratorExpr, - type_name: str, - id_for_messages: str) -> Type: + # If any of the comprehensions use async for, the expression will return an async generator + # object, or await is used anywhere but in the leftmost sequence. + if ( + any(e.is_async) + or has_await_expression(e.left_expr) + or any(has_await_expression(sequence) for sequence in e.sequences[1:]) + or any(has_await_expression(cond) for condlist in e.condlists for cond in condlist) + ): + typ = "typing.AsyncGenerator" + # received type is always None in async generator expressions + additional_args: list[Type] = [NoneType()] + else: + typ = "typing.Generator" + # received type and returned type are None + additional_args = [NoneType(), NoneType()] + return self.check_generator_or_comprehension( + e, typ, "", additional_args=additional_args + ) + + def check_generator_or_comprehension( + self, + gen: GeneratorExpr, + type_name: str, + id_for_messages: str, + additional_args: list[Type] | None = None, + ) -> Type: """Type check a generator expression or a list comprehension.""" - self.check_for_comp(gen) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - tv = TypeVar('T', -1, [], self.chk.object_type()) - constructor = Callable([tv], - [nodes.ARG_POS], - [None], - self.chk.named_generic_type(type_name, [tv]), - self.chk.named_type('builtins.function'), - id_for_messages, - [TypeVarDef('T', -1, None, self.chk.object_type())]) - return self.check_call(constructor, - [gen.left_expr], [nodes.ARG_POS], gen)[0] - - def visit_dictionary_comprehension(self, e: DictionaryComprehension): + additional_args = additional_args or [] + with self.chk.binder.frame_context(can_skip=True, fall_through=0): + self.check_for_comp(gen) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + tv = TypeVarType( + "T", + "T", + id=TypeVarId(-1, namespace=""), + values=[], + upper_bound=self.object_type(), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + tv_list: list[Type] = [tv] + constructor = CallableType( + tv_list, + [nodes.ARG_POS], + [None], + self.chk.named_generic_type(type_name, tv_list + additional_args), + self.chk.named_type("builtins.function"), + name=id_for_messages, + variables=[tv], + ) + return self.check_call(constructor, [gen.left_expr], [nodes.ARG_POS], gen)[0] + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: """Type check a dictionary comprehension.""" - self.check_for_comp(e) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - key_tv = TypeVar('KT', -1, [], self.chk.object_type()) - value_tv = TypeVar('VT', -2, [], self.chk.object_type()) - constructor = Callable([key_tv, value_tv], - [nodes.ARG_POS, nodes.ARG_POS], - [None, None], - self.chk.named_generic_type('builtins.dict', [key_tv, value_tv]), - self.chk.named_type('builtins.function'), - '', - [TypeVarDef('KT', -1, None, self.chk.object_type()), - TypeVarDef('VT', -2, None, self.chk.object_type())]) - return self.check_call(constructor, - [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] - - def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: + with self.chk.binder.frame_context(can_skip=True, fall_through=0): + self.check_for_comp(e) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + ktdef = TypeVarType( + "KT", + "KT", + id=TypeVarId(-1, namespace=""), + values=[], + upper_bound=self.object_type(), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + vtdef = TypeVarType( + "VT", + "VT", + id=TypeVarId(-2, namespace=""), + values=[], + upper_bound=self.object_type(), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + constructor = CallableType( + [ktdef, vtdef], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + self.chk.named_generic_type("builtins.dict", [ktdef, vtdef]), + self.chk.named_type("builtins.function"), + name="", + variables=[ktdef, vtdef], + ) + return self.check_call( + constructor, [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e + )[0] + + def check_for_comp(self, e: GeneratorExpr | DictionaryComprehension) -> None: """Check the for_comp part of comprehensions. That is the part from 'for': ... for x in y if z + + Note: This adds the type information derived from the condlists to the current binder. """ - self.chk.binder.push_frame() - for index, sequence, conditions in zip(e.indices, e.sequences, - e.condlists): - sequence_type = self.chk.analyse_iterable_item_type(sequence) - self.chk.analyse_index_variables(index, sequence_type, e) + for index, sequence, conditions, is_async in zip( + e.indices, e.sequences, e.condlists, e.is_async + ): + if is_async: + _, sequence_type = self.chk.analyze_async_iterable_item_type(sequence) + else: + _, sequence_type = self.chk.analyze_iterable_item_type(sequence) + self.chk.analyze_index_variables(index, sequence_type, True, e) for condition in conditions: self.accept(condition) - self.chk.binder.pop_frame() - def visit_undefined_expr(self, e: UndefinedExpr) -> Type: - return e.type + # values are only part of the comprehension when all conditions are true + true_map, false_map = self.chk.find_isinstance_check(condition) + + if true_map: + self.chk.push_type_map(true_map) + + if codes.REDUNDANT_EXPR in self.chk.options.enabled_error_codes: + if true_map is None: + self.msg.redundant_condition_in_comprehension(False, condition) + elif false_map is None: + self.msg.redundant_condition_in_comprehension(True, condition) + + def visit_conditional_expr(self, e: ConditionalExpr, allow_none_return: bool = False) -> Type: + self.accept(e.cond) + ctx = self.type_context[-1] + + # Gain type information from isinstance if it is there + # but only for the current expression + if_map, else_map = self.chk.find_isinstance_check(e.cond) + if codes.REDUNDANT_EXPR in self.chk.options.enabled_error_codes: + if if_map is None: + self.msg.redundant_condition_in_if(False, e.cond) + elif else_map is None: + self.msg.redundant_condition_in_if(True, e.cond) + + if_type = self.analyze_cond_branch( + if_map, e.if_expr, context=ctx, allow_none_return=allow_none_return + ) + + # we want to keep the narrowest value of if_type for union'ing the branches + # however, it would be silly to pass a literal as a type context. Pass the + # underlying fallback type instead. + if_type_fallback = simple_literal_type(get_proper_type(if_type)) or if_type + + # Analyze the right branch using full type context and store the type + full_context_else_type = self.analyze_cond_branch( + else_map, e.else_expr, context=ctx, allow_none_return=allow_none_return + ) + + if not mypy.checker.is_valid_inferred_type(if_type): + # Analyze the right branch disregarding the left branch. + else_type = full_context_else_type + # we want to keep the narrowest value of else_type for union'ing the branches + # however, it would be silly to pass a literal as a type context. Pass the + # underlying fallback type instead. + else_type_fallback = simple_literal_type(get_proper_type(else_type)) or else_type + + # If it would make a difference, re-analyze the left + # branch using the right branch's type as context. + if ctx is None or not is_equivalent(else_type_fallback, ctx): + # TODO: If it's possible that the previous analysis of + # the left branch produced errors that are avoided + # using this context, suppress those errors. + if_type = self.analyze_cond_branch( + if_map, + e.if_expr, + context=else_type_fallback, + allow_none_return=allow_none_return, + ) + + elif if_type_fallback == ctx: + # There is no point re-running the analysis if if_type is equal to ctx. + # That would be an exact duplicate of the work we just did. + # This optimization is particularly important to avoid exponential blowup with nested + # if/else expressions: https://github.com/python/mypy/issues/9591 + # TODO: would checking for is_proper_subtype also work and cover more cases? + else_type = full_context_else_type + else: + # Analyze the right branch in the context of the left + # branch's type. + else_type = self.analyze_cond_branch( + else_map, + e.else_expr, + context=if_type_fallback, + allow_none_return=allow_none_return, + ) + + # In most cases using if_type as a context for right branch gives better inferred types. + # This is however not the case for literal types, so use the full context instead. + if is_literal_type_like(full_context_else_type) and not is_literal_type_like(else_type): + else_type = full_context_else_type + + res: Type = make_simplified_union([if_type, else_type]) + if has_uninhabited_component(res) and not isinstance( + get_proper_type(self.type_context[-1]), UnionType + ): + # In rare cases with empty collections join may give a better result. + alternative = join.join_types(if_type, else_type) + p_alt = get_proper_type(alternative) + if not isinstance(p_alt, Instance) or p_alt.type.fullname != "builtins.object": + res = alternative + return res - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - cond_type = self.accept(e.cond) - self.check_not_void(cond_type, e) - if_type = self.accept(e.if_expr) - else_type = self.accept(e.else_expr, context=if_type) - return join.join_types(if_type, else_type) + def analyze_cond_branch( + self, + map: dict[Expression, Type] | None, + node: Expression, + context: Type | None, + allow_none_return: bool = False, + ) -> Type: + with self.chk.binder.frame_context(can_skip=True, fall_through=0): + if map is None: + # We still need to type check node, in case we want to + # process it for isinstance checks later + self.accept(node, type_context=context, allow_none_return=allow_none_return) + return UninhabitedType() + self.chk.push_type_map(map) + return self.accept(node, type_context=context, allow_none_return=allow_none_return) # # Helpers # - def accept(self, node: Node, context: Type = None) -> Type: - """Type check a node. Alias for TypeChecker.accept.""" - return self.chk.accept(node, context) - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if type is Void.""" - self.chk.check_not_void(typ, context) - - def is_boolean(self, typ: Type) -> bool: - """Is type compatible with bool?""" - return is_subtype(typ, self.chk.bool_type()) + def accept( + self, + node: Expression, + type_context: Type | None = None, + allow_none_return: bool = False, + always_allow_any: bool = False, + is_callee: bool = False, + ) -> Type: + """Type check a node in the given type context. If allow_none_return + is True and this expression is a call, allow it to return None. This + applies only to this expression and not any subexpressions. + """ + if node in self.type_overrides: + # This branch is very fast, there is no point timing it. + return self.type_overrides[node] + # We don't use context manager here to get most precise data (and avoid overhead). + record_time = False + if self.collect_line_checking_stats and not self.in_expression: + t0 = time.perf_counter_ns() + self.in_expression = True + record_time = True + self.type_context.append(type_context) + old_is_callee = self.is_callee + self.is_callee = is_callee + try: + if allow_none_return and isinstance(node, CallExpr): + typ = self.visit_call_expr(node, allow_none_return=True) + elif allow_none_return and isinstance(node, YieldFromExpr): + typ = self.visit_yield_from_expr(node, allow_none_return=True) + elif allow_none_return and isinstance(node, ConditionalExpr): + typ = self.visit_conditional_expr(node, allow_none_return=True) + elif allow_none_return and isinstance(node, AwaitExpr): + typ = self.visit_await_expr(node, allow_none_return=True) + else: + typ = node.accept(self) + except Exception as err: + report_internal_error( + err, self.chk.errors.file, node.line, self.chk.errors, self.chk.options + ) + self.is_callee = old_is_callee + self.type_context.pop() + assert typ is not None + self.chk.store_type(node, typ) + + if ( + self.chk.options.disallow_any_expr + and not always_allow_any + and not self.chk.is_stub + and self.chk.in_checked_function() + and has_any_type(typ) + and not self.chk.current_node_deferred + ): + self.msg.disallowed_any_type(typ, node) + + if not self.chk.in_checked_function() or self.chk.current_node_deferred: + result: Type = AnyType(TypeOfAny.unannotated) + else: + result = typ + if record_time: + self.per_line_checking_time_ns[node.line] += time.perf_counter_ns() - t0 + self.in_expression = False + return result def named_type(self, name: str) -> Instance: """Return an instance type with type given by the name and no type @@ -1241,217 +5922,708 @@ def named_type(self, name: str) -> Instance: """ return self.chk.named_type(name) + def type_alias_type_type(self) -> Instance: + """Returns a `typing.TypeAliasType` or `typing_extensions.TypeAliasType`.""" + if self.chk.options.python_version >= (3, 12): + return self.named_type("typing.TypeAliasType") + return self.named_type("typing_extensions.TypeAliasType") + def is_valid_var_arg(self, typ: Type) -> bool: """Is a type valid as a *args argument?""" - return (isinstance(typ, TupleType) or - is_subtype(typ, self.chk.named_generic_type('typing.Iterable', - [AnyType()])) or - isinstance(typ, AnyType)) + typ = get_proper_type(typ) + return isinstance(typ, (TupleType, AnyType, ParamSpecType, UnpackType)) or is_subtype( + typ, self.chk.named_generic_type("typing.Iterable", [AnyType(TypeOfAny.special_form)]) + ) def is_valid_keyword_var_arg(self, typ: Type) -> bool: """Is a type valid as a **kwargs argument?""" - return is_subtype(typ, self.chk.named_generic_type( - 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) - - def has_non_method(self, typ: Type, member: str) -> bool: - """Does type have a member variable / property with the given name?""" - if isinstance(typ, Instance): - return (not typ.type.has_method(member) and - typ.type.has_readable_member(member)) - else: - return False + return ( + is_subtype( + typ, + self.chk.named_generic_type( + "_typeshed.SupportsKeysAndGetItem", + [self.named_type("builtins.str"), AnyType(TypeOfAny.special_form)], + ), + ) + or is_subtype( + typ, + self.chk.named_generic_type( + "_typeshed.SupportsKeysAndGetItem", [UninhabitedType(), UninhabitedType()] + ), + ) + or isinstance(typ, ParamSpecType) + ) def has_member(self, typ: Type, member: str) -> bool: """Does type have member with the given name?""" - # TODO TupleType => also consider tuple attributes + # TODO: refactor this to use checkmember.analyze_member_access, otherwise + # these two should be carefully kept in sync. + # This is much faster than analyze_member_access, though, and so using + # it first as a filter is important for performance. + typ = get_proper_type(typ) + + if isinstance(typ, TypeVarType): + typ = get_proper_type(typ.upper_bound) + if isinstance(typ, TupleType): + typ = tuple_fallback(typ) + if isinstance(typ, LiteralType): + typ = typ.fallback if isinstance(typ, Instance): return typ.type.has_readable_member(member) + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + return typ.fallback.type.has_readable_member(member) elif isinstance(typ, AnyType): return True elif isinstance(typ, UnionType): - result = all(self.has_member(x, member) for x in typ.items) + result = all(self.has_member(x, member) for x in typ.relevant_items()) return result + elif isinstance(typ, TypeType): + # Type[Union[X, ...]] is always normalized to Union[Type[X], ...], + # so we don't need to care about unions here. + item = typ.item + if isinstance(item, TypeVarType): + item = get_proper_type(item.upper_bound) + if isinstance(item, TupleType): + item = tuple_fallback(item) + if isinstance(item, Instance) and item.type.metaclass_type is not None: + return self.has_member(item.type.metaclass_type, member) + if isinstance(item, AnyType): + return True + return False else: return False - def unwrap_list(self, a: List[Node]) -> List[Node]: - """Unwrap parentheses from a list of expression nodes.""" - r = List[Node]() - for n in a: - r.append(self.strip_parens(n)) - return r + def not_ready_callback(self, name: str, context: Context) -> None: + """Called when we can't infer the type of a variable because it's not ready yet. - -def is_valid_argc(nargs: int, is_var_arg: bool, callable: Callable) -> bool: - """Return a boolean indicating whether a call expression has a - (potentially) compatible number of arguments for calling a function. - Varargs at caller are not checked. - """ - if is_var_arg: - if callable.is_var_arg: - return True + Either defer type checking of the enclosing function to the next + pass or report an error. + """ + self.chk.handle_cannot_determine_type(name, context) + + def visit_yield_expr(self, e: YieldExpr) -> Type: + return_type = self.chk.return_types[-1] + expected_item_type = self.chk.get_generator_yield_type(return_type, False) + if e.expr is None: + if ( + not isinstance(get_proper_type(expected_item_type), (NoneType, AnyType)) + and self.chk.in_checked_function() + ): + self.chk.fail(message_registry.YIELD_VALUE_EXPECTED, e) else: - return nargs - 1 <= callable.max_fixed_args() - elif callable.is_var_arg: - return nargs >= callable.min_args - else: - # Neither has varargs. - return nargs <= len(callable.arg_types) and nargs >= callable.min_args - - -def map_actuals_to_formals(caller_kinds: List[int], - caller_names: List[str], - callee_kinds: List[int], - callee_names: List[str], - caller_arg_type: Function[[int], - Type]) -> List[List[int]]: - """Calculate mapping between actual (caller) args and formals. - - The result contains a list of caller argument indexes mapping to each - callee argument index, indexed by callee index. - - The caller_arg_type argument should evaluate to the type of the actual - argument type with the given index. - """ - ncallee = len(callee_kinds) - map = [None] * ncallee # type: List[List[int]] - for i in range(ncallee): - map[i] = [] - j = 0 - for i, kind in enumerate(caller_kinds): - if kind == nodes.ARG_POS: - if j < ncallee: - if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED]: - map[j].append(i) - j += 1 - elif callee_kinds[j] == nodes.ARG_STAR: - map[j].append(i) - elif kind == nodes.ARG_STAR: - # We need to to know the actual type to map varargs. - argt = caller_arg_type(i) - if isinstance(argt, TupleType): - # A tuple actual maps to a fixed number of formals. - for k in range(len(argt.items)): - if j < ncallee: - if callee_kinds[j] != nodes.ARG_STAR2: - map[j].append(i) - else: - raise NotImplementedError() - j += 1 - else: - # Assume that it is an iterable (if it isn't, there will be - # an error later). - while j < ncallee: - if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): - break - else: - map[j].append(i) - j += 1 - elif kind == nodes.ARG_NAMED: - name = caller_names[i] - if name in callee_names: - map[callee_names.index(name)].append(i) - elif nodes.ARG_STAR2 in callee_kinds: - map[callee_kinds.index(nodes.ARG_STAR2)].append(i) + actual_item_type = self.accept(e.expr, expected_item_type) + self.chk.check_subtype( + actual_item_type, + expected_item_type, + e, + message_registry.INCOMPATIBLE_TYPES_IN_YIELD, + "actual type", + "expected type", + ) + return self.chk.get_generator_receive_type(return_type, False) + + def visit_await_expr(self, e: AwaitExpr, allow_none_return: bool = False) -> Type: + expected_type = self.type_context[-1] + if expected_type is not None: + expected_type = self.chk.named_generic_type("typing.Awaitable", [expected_type]) + actual_type = get_proper_type(self.accept(e.expr, expected_type)) + if isinstance(actual_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=actual_type) + ret = self.check_awaitable_expr( + actual_type, e, message_registry.INCOMPATIBLE_TYPES_IN_AWAIT + ) + if not allow_none_return and isinstance(get_proper_type(ret), NoneType): + self.chk.msg.does_not_return_value(None, e) + return ret + + def check_awaitable_expr( + self, t: Type, ctx: Context, msg: str | ErrorMessage, ignore_binder: bool = False + ) -> Type: + """Check the argument to `await` and extract the type of value. + + Also used by `async for` and `async with`. + """ + if not self.chk.check_subtype( + t, self.named_type("typing.Awaitable"), ctx, msg, "actual type", "expected type" + ): + return AnyType(TypeOfAny.special_form) else: - assert kind == nodes.ARG_STAR2 - for j in range(ncallee): - # TODO tuple varargs complicate this - no_certain_match = ( - not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) - if ((callee_names[j] and no_certain_match) - or callee_kinds[j] == nodes.ARG_STAR2): - map[j].append(i) - return map + generator = self.check_method_call_by_name("__await__", t, [], [], ctx)[0] + ret_type = self.chk.get_generator_return_type(generator, False) + ret_type = get_proper_type(ret_type) + if ( + not ignore_binder + and isinstance(ret_type, UninhabitedType) + and not ret_type.ambiguous + ): + self.chk.binder.unreachable() + return ret_type + + def visit_yield_from_expr(self, e: YieldFromExpr, allow_none_return: bool = False) -> Type: + # NOTE: Whether `yield from` accepts an `async def` decorated + # with `@types.coroutine` (or `@asyncio.coroutine`) depends on + # whether the generator containing the `yield from` is itself + # thus decorated. But it accepts a generator regardless of + # how it's decorated. + return_type = self.chk.return_types[-1] + # TODO: What should the context for the sub-expression be? + # If the containing function has type Generator[X, Y, ...], + # the context should be Generator[X, Y, T], where T is the + # context of the 'yield from' itself (but it isn't known). + subexpr_type = get_proper_type(self.accept(e.expr)) + + # Check that the expr is an instance of Iterable and get the type of the iterator produced + # by __iter__. + if isinstance(subexpr_type, AnyType): + iter_type: Type = AnyType(TypeOfAny.from_another_any, source_any=subexpr_type) + elif self.chk.type_is_iterable(subexpr_type): + if is_async_def(subexpr_type) and not has_coroutine_decorator(return_type): + self.chk.msg.yield_from_invalid_operand_type(subexpr_type, e) + + any_type = AnyType(TypeOfAny.special_form) + generic_generator_type = self.chk.named_generic_type( + "typing.Generator", [any_type, any_type, any_type] + ) + iter_type, _ = self.check_method_call_by_name( + "__iter__", subexpr_type, [], [], context=generic_generator_type + ) + else: + if not (is_async_def(subexpr_type) and has_coroutine_decorator(return_type)): + self.chk.msg.yield_from_invalid_operand_type(subexpr_type, e) + iter_type = AnyType(TypeOfAny.from_error) + else: + iter_type = self.check_awaitable_expr( + subexpr_type, e, message_registry.INCOMPATIBLE_TYPES_IN_YIELD_FROM + ) + + # Check that the iterator's item type matches the type yielded by the Generator function + # containing this `yield from` expression. + expected_item_type = self.chk.get_generator_yield_type(return_type, False) + actual_item_type = self.chk.get_generator_yield_type(iter_type, False) + + self.chk.check_subtype( + actual_item_type, + expected_item_type, + e, + message_registry.INCOMPATIBLE_TYPES_IN_YIELD_FROM, + "actual type", + "expected type", + ) + + # Determine the type of the entire yield from expression. + iter_type = get_proper_type(iter_type) + expr_type = self.chk.get_generator_return_type(iter_type, is_coroutine=False) + + if not allow_none_return and isinstance(get_proper_type(expr_type), NoneType): + self.chk.msg.does_not_return_value(None, e) + return expr_type + + def visit_temp_node(self, e: TempNode) -> Type: + return e.type + def visit_type_var_expr(self, e: TypeVarExpr) -> Type: + p_default = get_proper_type(e.default) + if not ( + isinstance(p_default, AnyType) + and p_default.type_of_any == TypeOfAny.from_omitted_generics + ): + if not is_subtype(p_default, e.upper_bound): + self.chk.fail("TypeVar default must be a subtype of the bound type", e) + if e.values and not any(p_default == value for value in e.values): + self.chk.fail("TypeVar default must be one of the constraint types", e) + return AnyType(TypeOfAny.special_form) + + def visit_paramspec_expr(self, e: ParamSpecExpr) -> Type: + return AnyType(TypeOfAny.special_form) + + def visit_type_var_tuple_expr(self, e: TypeVarTupleExpr) -> Type: + return AnyType(TypeOfAny.special_form) + + def visit_newtype_expr(self, e: NewTypeExpr) -> Type: + return AnyType(TypeOfAny.special_form) + + def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: + tuple_type = e.info.tuple_type + if tuple_type: + if self.chk.options.disallow_any_unimported and has_any_from_unimported_type( + tuple_type + ): + self.msg.unimported_type_becomes_any("NamedTuple type", tuple_type, e) + check_for_explicit_any( + tuple_type, self.chk.options, self.chk.is_typeshed_stub, self.msg, context=e + ) + return AnyType(TypeOfAny.special_form) + + def visit_enum_call_expr(self, e: EnumCallExpr) -> Type: + for name, value in zip(e.items, e.values): + if value is not None: + typ = self.accept(value) + if not isinstance(get_proper_type(typ), AnyType): + var = e.info.names[name].node + if isinstance(var, Var): + # Inline TypeChecker.set_inferred_type(), + # without the lvalue. (This doesn't really do + # much, since the value attribute is defined + # to have type Any in the typeshed stub.) + var.type = typ + var.is_inferred = True + return AnyType(TypeOfAny.special_form) + + def visit_typeddict_expr(self, e: TypedDictExpr) -> Type: + return AnyType(TypeOfAny.special_form) + + def visit__promote_expr(self, e: PromoteExpr) -> Type: + return e.type + + def visit_star_expr(self, e: StarExpr) -> Type: + # TODO: should this ever be called (see e.g. mypyc visitor)? + return self.accept(e.expr) -def is_empty_tuple(t: Type) -> bool: - return isinstance(t, TupleType) and not cast(TupleType, t).items + def object_type(self) -> Instance: + """Return instance type 'object'.""" + return self.named_type("builtins.object") + def bool_type(self) -> Instance: + """Return instance type 'bool'.""" + return self.named_type("builtins.bool") -def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: - # Multiple actuals can map to the same formal only if they both come from - # varargs (*args and **kwargs); in this case at runtime it is possible that - # there are no duplicates. We need to allow this, as the convention - # f(..., *args, **kwargs) is common enough. - return len(mapping) > 1 and not ( - len(mapping) == 2 and - actual_kinds[mapping[0]] == nodes.ARG_STAR and - actual_kinds[mapping[1]] == nodes.ARG_STAR2) + @overload + def narrow_type_from_binder(self, expr: Expression, known_type: Type) -> Type: ... + @overload + def narrow_type_from_binder( + self, expr: Expression, known_type: Type, skip_non_overlapping: bool + ) -> Type | None: ... -def replace_callable_return_type(c: Callable, new_ret_type: Type) -> Callable: + def narrow_type_from_binder( + self, expr: Expression, known_type: Type, skip_non_overlapping: bool = False + ) -> Type | None: + """Narrow down a known type of expression using information in conditional type binder. + + If 'skip_non_overlapping' is True, return None if the type and restriction are + non-overlapping. + """ + if literal(expr) >= LITERAL_TYPE: + restriction = self.chk.binder.get(expr) + # If the current node is deferred, some variables may get Any types that they + # otherwise wouldn't have. We don't want to narrow down these since it may + # produce invalid inferred Optional[Any] types, at least. + if restriction and not ( + isinstance(get_proper_type(known_type), AnyType) and self.chk.current_node_deferred + ): + # Note: this call should match the one in narrow_declared_type(). + if skip_non_overlapping and not is_overlapping_types( + known_type, restriction, prohibit_none_typevar_overlap=True + ): + return None + return narrow_declared_type(known_type, restriction) + return known_type + + def has_abstract_type_part(self, caller_type: ProperType, callee_type: ProperType) -> bool: + # TODO: support other possible types here + if isinstance(caller_type, TupleType) and isinstance(callee_type, TupleType): + return any( + self.has_abstract_type(get_proper_type(caller), get_proper_type(callee)) + for caller, callee in zip(caller_type.items, callee_type.items) + ) + return self.has_abstract_type(caller_type, callee_type) + + def has_abstract_type(self, caller_type: ProperType, callee_type: ProperType) -> bool: + return ( + isinstance(caller_type, FunctionLike) + and isinstance(callee_type, TypeType) + and caller_type.is_type_obj() + and (caller_type.type_object().is_abstract or caller_type.type_object().is_protocol) + and isinstance(callee_type.item, Instance) + and (callee_type.item.type.is_abstract or callee_type.item.type.is_protocol) + and not self.chk.allow_abstract_call + ) + + +def has_any_type(t: Type, ignore_in_type_obj: bool = False) -> bool: + """Whether t contains an Any type""" + return t.accept(HasAnyType(ignore_in_type_obj)) + + +class HasAnyType(types.BoolTypeQuery): + def __init__(self, ignore_in_type_obj: bool) -> None: + super().__init__(types.ANY_STRATEGY) + self.ignore_in_type_obj = ignore_in_type_obj + + def visit_any(self, t: AnyType) -> bool: + return t.type_of_any != TypeOfAny.special_form # special forms are not real Any types + + def visit_callable_type(self, t: CallableType) -> bool: + if self.ignore_in_type_obj and t.is_type_obj(): + return False + return super().visit_callable_type(t) + + def visit_type_var(self, t: TypeVarType) -> bool: + default = [t.default] if t.has_default() else [] + return self.query_types([t.upper_bound, *default] + t.values) + + def visit_param_spec(self, t: ParamSpecType) -> bool: + default = [t.default] if t.has_default() else [] + return self.query_types([t.upper_bound, *default]) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool: + default = [t.default] if t.has_default() else [] + return self.query_types([t.upper_bound, *default]) + + +def has_coroutine_decorator(t: Type) -> bool: + """Whether t came from a function decorated with `@coroutine`.""" + t = get_proper_type(t) + return isinstance(t, Instance) and t.type.fullname == "typing.AwaitableGenerator" + + +def is_async_def(t: Type) -> bool: + """Whether t came from a function defined using `async def`.""" + # In check_func_def(), when we see a function decorated with + # `@typing.coroutine` or `@async.coroutine`, we change the + # return type to typing.AwaitableGenerator[...], so that its + # type is compatible with either Generator or Awaitable. + # But for the check here we need to know whether the original + # function (before decoration) was an `async def`. The + # AwaitableGenerator type conveniently preserves the original + # type as its 4th parameter (3rd when using 0-origin indexing + # :-), so that we can recover that information here. + # (We really need to see whether the original, undecorated + # function was an `async def`, which is orthogonal to its + # decorations.) + t = get_proper_type(t) + if ( + isinstance(t, Instance) + and t.type.fullname == "typing.AwaitableGenerator" + and len(t.args) >= 4 + ): + t = get_proper_type(t.args[3]) + return isinstance(t, Instance) and t.type.fullname == "typing.Coroutine" + + +def is_non_empty_tuple(t: Type) -> bool: + t = get_proper_type(t) + return isinstance(t, TupleType) and bool(t.items) + + +def is_duplicate_mapping( + mapping: list[int], actual_types: list[Type], actual_kinds: list[ArgKind] +) -> bool: + return ( + len(mapping) > 1 + # Multiple actuals can map to the same formal if they both come from + # varargs (*args and **kwargs); in this case at runtime it is possible + # that here are no duplicates. We need to allow this, as the convention + # f(..., *args, **kwargs) is common enough. + and not ( + len(mapping) == 2 + and actual_kinds[mapping[0]] == nodes.ARG_STAR + and actual_kinds[mapping[1]] == nodes.ARG_STAR2 + ) + # Multiple actuals can map to the same formal if there are multiple + # **kwargs which cannot be mapped with certainty (non-TypedDict + # **kwargs). + and not all( + actual_kinds[m] == nodes.ARG_STAR2 + and not isinstance(get_proper_type(actual_types[m]), TypedDictType) + for m in mapping + ) + ) + + +def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: """Return a copy of a callable type with a different return type.""" - return Callable(c.arg_types, - c.arg_kinds, - c.arg_names, - new_ret_type, - c.fallback, - c.name, - c.variables, - c.bound_vars, - c.line) - - -class ArgInferSecondPassQuery(types.TypeQuery): + return c.copy_modified(ret_type=new_ret_type) + + +class ArgInferSecondPassQuery(types.BoolTypeQuery): """Query whether an argument type should be inferred in the second pass. The result is True if the type has a type variable in a callable return - type anywhere. For example, the result for Function[[], T] is True if t is + type anywhere. For example, the result for Callable[[], T] is True if t is a type variable. """ + def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) + super().__init__(types.ANY_STRATEGY) + + def visit_callable_type(self, t: CallableType) -> bool: + # TODO: we need to check only for type variables of original callable. + return self.query_types(t.arg_types) or has_type_vars(t) - def visit_callable(self, t: Callable) -> bool: - return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) + +def has_erased_component(t: Type | None) -> bool: + return t is not None and t.accept(HasErasedComponentsQuery()) -class HasTypeVarQuery(types.TypeQuery): - """Visitor for querying whether a type has a type variable component.""" +class HasErasedComponentsQuery(types.BoolTypeQuery): + """Visitor for querying whether a type has an erased component.""" + def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) + super().__init__(types.ANY_STRATEGY) - def visit_type_var(self, t: TypeVar) -> bool: + def visit_erased_type(self, t: ErasedType) -> bool: return True -def has_erased_component(t: Type) -> bool: - return t is not None and t.accept(HasErasedComponentsQuery()) +def has_uninhabited_component(t: Type | None) -> bool: + return t is not None and t.accept(HasUninhabitedComponentsQuery()) -class HasErasedComponentsQuery(types.TypeQuery): - """Visitor for querying whether a type has an erased component.""" +class HasUninhabitedComponentsQuery(types.BoolTypeQuery): + """Visitor for querying whether a type has an UninhabitedType component.""" + def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) + super().__init__(types.ANY_STRATEGY) - def visit_erased_type(self, t: ErasedType) -> bool: + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: return True -def is_compatible_overload_arg(actual: Type, formal: Type) -> bool: - if (isinstance(actual, NoneTyp) or isinstance(actual, AnyType) or - isinstance(formal, AnyType) or isinstance(formal, TypeVar) or - isinstance(formal, Callable)): - # These could match anything at runtime. +def arg_approximate_similarity(actual: Type, formal: Type) -> bool: + """Return if caller argument (actual) is roughly compatible with signature arg (formal). + + This function is deliberately loose and will report two types are similar + as long as their "shapes" are plausibly the same. + + This is useful when we're doing error reporting: for example, if we're trying + to select an overload alternative and there's no exact match, we can use + this function to help us identify which alternative the user might have + *meant* to match. + """ + actual = get_proper_type(actual) + formal = get_proper_type(formal) + + # Erase typevars: we'll consider them all to have the same "shape". + if isinstance(actual, TypeVarType): + actual = erase_to_union_or_bound(actual) + if isinstance(formal, TypeVarType): + formal = erase_to_union_or_bound(formal) + + # Callable or Type[...]-ish types + def is_typetype_like(typ: ProperType) -> bool: + return ( + isinstance(typ, TypeType) + or (isinstance(typ, FunctionLike) and typ.is_type_obj()) + or (isinstance(typ, Instance) and typ.type.fullname == "builtins.type") + ) + + if isinstance(formal, CallableType): + if isinstance(actual, (CallableType, Overloaded, TypeType)): + return True + if is_typetype_like(actual) and is_typetype_like(formal): return True + + # Unions if isinstance(actual, UnionType): - return any(is_compatible_overload_arg(item, formal) - for item in actual.items) + return any(arg_approximate_similarity(item, formal) for item in actual.relevant_items()) if isinstance(formal, UnionType): - return any(is_compatible_overload_arg(actual, item) - for item in formal.items) + return any(arg_approximate_similarity(actual, item) for item in formal.relevant_items()) + + # TypedDicts + if isinstance(actual, TypedDictType): + if isinstance(formal, TypedDictType): + return True + return arg_approximate_similarity(actual.fallback, formal) + + # Instances + # For instances, we mostly defer to the existing is_subtype check. if isinstance(formal, Instance): - if isinstance(actual, Callable): + if isinstance(actual, CallableType): actual = actual.fallback if isinstance(actual, Overloaded): - actual = actual.items()[0].fallback + actual = actual.items[0].fallback if isinstance(actual, TupleType): - actual = actual.fallback - if isinstance(actual, Instance): - return formal.type in actual.type.mro - else: - return False - # Fall back to a conservative equality check for the remaining kinds of type. - return is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) + actual = tuple_fallback(actual) + if isinstance(actual, Instance) and formal.type in actual.type.mro: + # Try performing a quick check as an optimization + return True + + # Fall back to a standard subtype check for the remaining kinds of type. + return is_subtype(erasetype.erase_type(actual), erasetype.erase_type(formal)) + + +def any_causes_overload_ambiguity( + items: list[CallableType], + return_types: list[Type], + arg_types: list[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, +) -> bool: + """May an argument containing 'Any' cause ambiguous result type on call to overloaded function? + + Note that this sometimes returns True even if there is no ambiguity, since a correct + implementation would be complex (and the call would be imprecisely typed due to Any + types anyway). + + Args: + items: Overload items matching the actual arguments + arg_types: Actual argument types + arg_kinds: Actual argument kinds + arg_names: Actual argument names + """ + if all_same_types(return_types): + return False + + actual_to_formal = [ + map_formals_to_actuals( + arg_kinds, arg_names, item.arg_kinds, item.arg_names, lambda i: arg_types[i] + ) + for item in items + ] + + for arg_idx, arg_type in enumerate(arg_types): + # We ignore Anys in type object callables as ambiguity + # creators, since that can lead to falsely claiming ambiguity + # for overloads between Type and Callable. + if has_any_type(arg_type, ignore_in_type_obj=True): + matching_formals_unfiltered = [ + (item_idx, lookup[arg_idx]) + for item_idx, lookup in enumerate(actual_to_formal) + if lookup[arg_idx] + ] + + matching_returns = [] + matching_formals = [] + for item_idx, formals in matching_formals_unfiltered: + matched_callable = items[item_idx] + matching_returns.append(matched_callable.ret_type) + + # Note: if an actual maps to multiple formals of differing types within + # a single callable, then we know at least one of those formals must be + # a different type then the formal(s) in some other callable. + # So it's safe to just append everything to the same list. + for formal in formals: + matching_formals.append(matched_callable.arg_types[formal]) + if not all_same_types(matching_formals) and not all_same_types(matching_returns): + # Any maps to multiple different types, and the return types of these items differ. + return True + return False + + +def all_same_types(types: list[Type]) -> bool: + if not types: + return True + return all(is_same_type(t, types[0]) for t in types[1:]) + + +def merge_typevars_in_callables_by_name( + callables: Sequence[CallableType], +) -> tuple[list[CallableType], list[TypeVarType]]: + """Takes all the typevars present in the callables and 'combines' the ones with the same name. + + For example, suppose we have two callables with signatures "f(x: T, y: S) -> T" and + "f(x: List[Tuple[T, S]]) -> Tuple[T, S]". Both callables use typevars named "T" and + "S", but we treat them as distinct, unrelated typevars. (E.g. they could both have + distinct ids.) + + If we pass in both callables into this function, it returns a list containing two + new callables that are identical in signature, but use the same underlying TypeVarType + for T and S. + + This is useful if we want to take the output lists and "merge" them into one callable + in some way -- for example, when unioning together overloads. + + Returns both the new list of callables and a list of all distinct TypeVarType objects used. + """ + output: list[CallableType] = [] + unique_typevars: dict[str, TypeVarType] = {} + variables: list[TypeVarType] = [] + + for target in callables: + if target.is_generic(): + target = freshen_function_type_vars(target) + + rename = {} # Dict[TypeVarId, TypeVar] + for tv in target.variables: + name = tv.fullname + if name not in unique_typevars: + # TODO: support ParamSpecType and TypeVarTuple. + if isinstance(tv, (ParamSpecType, TypeVarTupleType)): + continue + assert isinstance(tv, TypeVarType) + unique_typevars[name] = tv + variables.append(tv) + rename[tv.id] = unique_typevars[name] + + target = expand_type(target, rename) + output.append(target) + + return output, variables + + +def try_getting_literal(typ: Type) -> ProperType: + """If possible, get a more precise literal type for a given type.""" + typ = get_proper_type(typ) + if isinstance(typ, Instance) and typ.last_known_value is not None: + return typ.last_known_value + return typ + + +def is_expr_literal_type(node: Expression) -> bool: + """Returns 'true' if the given node is a Literal""" + if isinstance(node, IndexExpr): + base = node.base + return isinstance(base, RefExpr) and base.fullname in LITERAL_TYPE_NAMES + if isinstance(node, NameExpr): + underlying = node.node + return isinstance(underlying, TypeAlias) and isinstance( + get_proper_type(underlying.target), LiteralType + ) + return False + + +def has_bytes_component(typ: Type) -> bool: + """Is this one of builtin byte types, or a union that contains it?""" + typ = get_proper_type(typ) + byte_types = {"builtins.bytes", "builtins.bytearray"} + if isinstance(typ, UnionType): + return any(has_bytes_component(t) for t in typ.items) + if isinstance(typ, Instance) and typ.type.fullname in byte_types: + return True + return False + + +def type_info_from_type(typ: Type) -> TypeInfo | None: + """Gets the TypeInfo for a type, indirecting through things like type variables and tuples.""" + typ = get_proper_type(typ) + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + return typ.type_object() + if isinstance(typ, TypeType): + typ = typ.item + if isinstance(typ, TypeVarType): + typ = get_proper_type(typ.upper_bound) + if isinstance(typ, TupleType): + typ = tuple_fallback(typ) + if isinstance(typ, Instance): + return typ.type + + # A complicated type. Too tricky, give up. + # TODO: Do something more clever here. + return None + + +def is_operator_method(fullname: str | None) -> bool: + if not fullname: + return False + short_name = fullname.split(".")[-1] + return ( + short_name in operators.op_methods.values() + or short_name in operators.reverse_op_methods.values() + or short_name in operators.unary_op_methods.values() + ) + + +def get_partial_instance_type(t: Type | None) -> PartialType | None: + if t is None or not isinstance(t, PartialType) or t.type is None: + return None + return t + + +def is_type_type_context(context: Type | None) -> bool: + context = get_proper_type(context) + if isinstance(context, TypeType): + return True + if isinstance(context, UnionType): + return any(is_type_type_context(item) for item in context.items) + return False diff --git a/mypy/checkmember.py b/mypy/checkmember.py index c89d45a4c902..50e54ca30460 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1,158 +1,915 @@ """Type checking of attribute access""" -from typing import cast, Function, List +from __future__ import annotations -from mypy.types import ( - Type, Instance, AnyType, TupleType, Callable, FunctionLike, TypeVarDef, - Overloaded, TypeVar, TypeTranslator, UnionType +from typing import TYPE_CHECKING, Callable, Sequence, cast + +from mypy import meet, message_registry, subtypes +from mypy.erasetype import erase_typevars +from mypy.expandtype import ( + expand_self_type, + expand_type_by_instance, + freshen_all_functions_type_vars, ) -from mypy.nodes import TypeInfo, FuncBase, Var, FuncDef, SymbolNode, Context -from mypy.nodes import ARG_POS, function_type, Decorator -from mypy.messages import MessageBuilder from mypy.maptype import map_instance_to_supertype -from mypy.expandtype import expand_type_by_instance -from mypy.nodes import method_type -from mypy.semanal import self_type -from mypy import messages -from mypy import subtypes +from mypy.messages import MessageBuilder +from mypy.nodes import ( + ARG_POS, + ARG_STAR, + ARG_STAR2, + EXCLUDED_ENUM_ATTRIBUTES, + SYMBOL_FUNCBASE_TYPES, + Context, + Decorator, + FuncBase, + FuncDef, + IndexExpr, + MypyFile, + NameExpr, + OverloadedFuncDef, + SymbolNode, + SymbolTable, + TempNode, + TypeAlias, + TypeInfo, + TypeVarExpr, + Var, + is_final_node, +) +from mypy.plugin import AttributeContext +from mypy.typeops import ( + bind_self, + class_callable, + erase_to_bound, + function_type, + get_type_vars, + make_simplified_union, + supported_self_type, + tuple_fallback, + type_object_type_from_function, +) +from mypy.types import ( + AnyType, + CallableType, + DeletedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnionType, + get_proper_type, +) +from mypy.typetraverser import TypeTraverserVisitor + +if TYPE_CHECKING: # import for forward declaration only + import mypy.checker +from mypy import state -def analyse_member_access(name: str, typ: Type, node: Context, is_lvalue: bool, - is_super: bool, - builtin_type: Function[[str], Instance], - msg: MessageBuilder, override_info: TypeInfo = None, - report_type: Type = None) -> Type: - """Analyse attribute access. + +class MemberContext: + """Information and objects needed to type check attribute access. + + Look at the docstring of analyze_member_access for more information. + """ + + def __init__( + self, + *, + is_lvalue: bool, + is_super: bool, + is_operator: bool, + original_type: Type, + context: Context, + msg: MessageBuilder, + chk: mypy.checker.TypeChecker, + self_type: Type | None, + module_symbol_table: SymbolTable | None = None, + no_deferral: bool = False, + is_self: bool = False, + ) -> None: + self.is_lvalue = is_lvalue + self.is_super = is_super + self.is_operator = is_operator + self.original_type = original_type + self.self_type = self_type or original_type + self.context = context # Error context + self.msg = msg + self.chk = chk + self.module_symbol_table = module_symbol_table + self.no_deferral = no_deferral + self.is_self = is_self + + def named_type(self, name: str) -> Instance: + return self.chk.named_type(name) + + def not_ready_callback(self, name: str, context: Context) -> None: + self.chk.handle_cannot_determine_type(name, context) + + def copy_modified( + self, + *, + messages: MessageBuilder | None = None, + self_type: Type | None = None, + is_lvalue: bool | None = None, + original_type: Type | None = None, + ) -> MemberContext: + mx = MemberContext( + is_lvalue=self.is_lvalue, + is_super=self.is_super, + is_operator=self.is_operator, + original_type=self.original_type, + context=self.context, + msg=self.msg, + chk=self.chk, + self_type=self.self_type, + module_symbol_table=self.module_symbol_table, + no_deferral=self.no_deferral, + ) + if messages is not None: + mx.msg = messages + if self_type is not None: + mx.self_type = self_type + if is_lvalue is not None: + mx.is_lvalue = is_lvalue + if original_type is not None: + mx.original_type = original_type + return mx + + +def analyze_member_access( + name: str, + typ: Type, + context: Context, + *, + is_lvalue: bool, + is_super: bool, + is_operator: bool, + msg: MessageBuilder, + original_type: Type, + chk: mypy.checker.TypeChecker, + override_info: TypeInfo | None = None, + in_literal_context: bool = False, + self_type: Type | None = None, + module_symbol_table: SymbolTable | None = None, + no_deferral: bool = False, + is_self: bool = False, +) -> Type: + """Return the type of attribute 'name' of 'typ'. + + The actual implementation is in '_analyze_member_access' and this docstring + also applies to it. This is a general operation that supports various different variations: - 1. lvalue or non-lvalue access (i.e. setter or getter access) - 2. supertype access (when using super(); is_super == True and - override_info should refer to the supertype) + 1. lvalue or non-lvalue access (setter or getter access) + 2. supertype access when using super() (is_super == True and + 'override_info' should refer to the supertype) + + 'original_type' is the most precise inferred or declared type of the base object + that we have available. When looking for an attribute of 'typ', we may perform + recursive calls targeting the fallback type, and 'typ' may become some supertype + of 'original_type'. 'original_type' is always preserved as the 'typ' type used in + the initial, non-recursive call. The 'self_type' is a component of 'original_type' + to which generic self should be bound (a narrower type that has a fallback to instance). + Currently this is used only for union types. + + 'module_symbol_table' is passed to this function if 'typ' is actually a module + and we want to keep track of the available attributes of the module (since they + are not available via the type object directly) """ - report_type = report_type or typ + mx = MemberContext( + is_lvalue=is_lvalue, + is_super=is_super, + is_operator=is_operator, + original_type=original_type, + context=context, + msg=msg, + chk=chk, + self_type=self_type, + module_symbol_table=module_symbol_table, + no_deferral=no_deferral, + is_self=is_self, + ) + result = _analyze_member_access(name, typ, mx, override_info) + possible_literal = get_proper_type(result) + if ( + in_literal_context + and isinstance(possible_literal, Instance) + and possible_literal.last_known_value is not None + ): + return possible_literal.last_known_value + else: + return result + + +def _analyze_member_access( + name: str, typ: Type, mx: MemberContext, override_info: TypeInfo | None = None +) -> Type: + # TODO: This and following functions share some logic with subtypes.find_member; + # consider refactoring. + typ = get_proper_type(typ) if isinstance(typ, Instance): - if name == '__init__' and not is_super: - # Accessing __init__ in statically typed code would compromise - # type safety unless used via super(). - msg.fail(messages.CANNOT_ACCESS_INIT, node) - return AnyType() - - # The base object has an instance type. - - info = typ.type - if override_info: - info = override_info - - # Look up the member. First look up the method dictionary. - method = info.get_method(name) - if method: - if is_lvalue: - msg.cant_assign_to_method(node) - typ = map_instance_to_supertype(typ, method.info) - return expand_type_by_instance( - method_type(method, builtin_type('builtins.function')), typ) - else: - # Not a method. - return analyse_member_var_access(name, typ, info, node, - is_lvalue, is_super, msg, - report_type=report_type) + return analyze_instance_member_access(name, typ, mx, override_info) elif isinstance(typ, AnyType): # The base object has dynamic type. - return AnyType() + return AnyType(TypeOfAny.from_another_any, source_any=typ) elif isinstance(typ, UnionType): - # The base object has dynamic type. - msg.disable_type_names += 1 - results = [analyse_member_access(name, subtype, node, is_lvalue, - is_super, builtin_type, msg) - for subtype in typ.items] - msg.disable_type_names -= 1 - return UnionType.make_simplified_union(results) + return analyze_union_member_access(name, typ, mx) + elif isinstance(typ, FunctionLike) and typ.is_type_obj(): + return analyze_type_callable_member_access(name, typ, mx) + elif isinstance(typ, TypeType): + return analyze_type_type_member_access(name, typ, mx, override_info) elif isinstance(typ, TupleType): # Actually look up from the fallback instance type. - return analyse_member_access(name, typ.fallback, node, is_lvalue, - is_super, builtin_type, msg) - elif (isinstance(typ, FunctionLike) and - cast(FunctionLike, typ).is_type_obj()): - # Class attribute. - # TODO super? - sig = cast(FunctionLike, typ) - itype = cast(Instance, sig.items()[0].ret_type) - result = analyse_class_attribute_access(itype, name, node, is_lvalue, builtin_type, msg) - if result: - return result + return _analyze_member_access(name, tuple_fallback(typ), mx, override_info) + elif isinstance(typ, (LiteralType, FunctionLike)): + # Actually look up from the fallback instance type. + return _analyze_member_access(name, typ.fallback, mx, override_info) + elif isinstance(typ, TypedDictType): + return analyze_typeddict_access(name, typ, mx, override_info) + elif isinstance(typ, NoneType): + return analyze_none_member_access(name, typ, mx) + elif isinstance(typ, TypeVarLikeType): + if isinstance(typ, TypeVarType) and typ.values: + return _analyze_member_access( + name, make_simplified_union(typ.values), mx, override_info + ) + return _analyze_member_access(name, typ.upper_bound, mx, override_info) + elif isinstance(typ, DeletedType): + mx.msg.deleted_as_rvalue(typ, mx.context) + return AnyType(TypeOfAny.from_error) + return report_missing_attribute(mx.original_type, typ, name, mx) + + +def may_be_awaitable_attribute( + name: str, typ: Type, mx: MemberContext, override_info: TypeInfo | None = None +) -> bool: + """Check if the given type has the attribute when awaited.""" + if mx.chk.checking_missing_await: + # Avoid infinite recursion. + return False + with mx.chk.checking_await_set(), mx.msg.filter_errors() as local_errors: + aw_type = mx.chk.get_precise_awaitable_type(typ, local_errors) + if aw_type is None: + return False + _ = _analyze_member_access(name, aw_type, mx, override_info) + return not local_errors.has_new_errors() + + +def report_missing_attribute( + original_type: Type, + typ: Type, + name: str, + mx: MemberContext, + override_info: TypeInfo | None = None, +) -> Type: + error_code = mx.msg.has_no_attr(original_type, typ, name, mx.context, mx.module_symbol_table) + if not mx.msg.prefer_simple_messages(): + if may_be_awaitable_attribute(name, typ, mx, override_info): + mx.msg.possible_missing_await(mx.context, error_code) + return AnyType(TypeOfAny.from_error) + + +# The several functions that follow implement analyze_member_access for various +# types and aren't documented individually. + + +def analyze_instance_member_access( + name: str, typ: Instance, mx: MemberContext, override_info: TypeInfo | None +) -> Type: + if name == "__init__" and not mx.is_super: + # Accessing __init__ in statically typed code would compromise + # type safety unless used via super(). + mx.msg.fail(message_registry.CANNOT_ACCESS_INIT, mx.context) + return AnyType(TypeOfAny.from_error) + + # The base object has an instance type. + + info = typ.type + if override_info: + info = override_info + + if ( + state.find_occurrences + and info.name == state.find_occurrences[0] + and name == state.find_occurrences[1] + ): + mx.msg.note("Occurrence of '{}.{}'".format(*state.find_occurrences), mx.context) + + # Look up the member. First look up the method dictionary. + method = info.get_method(name) + if method and not isinstance(method, Decorator): + if mx.is_super: + validate_super_call(method, mx) + + if method.is_property: + assert isinstance(method, OverloadedFuncDef) + getter = method.items[0] + assert isinstance(getter, Decorator) + if mx.is_lvalue and (len(items := method.items) > 1): + mx.chk.warn_deprecated(items[1], mx.context) + return analyze_var(name, getter.var, typ, info, mx) + + if mx.is_lvalue: + mx.msg.cant_assign_to_method(mx.context) + if not isinstance(method, OverloadedFuncDef): + signature = function_type(method, mx.named_type("builtins.function")) + else: + if method.type is None: + # Overloads may be not ready if they are decorated. Handle this in same + # manner as we would handle a regular decorated function: defer if possible. + if not mx.no_deferral and method.items: + mx.not_ready_callback(method.name, mx.context) + return AnyType(TypeOfAny.special_form) + assert isinstance(method.type, Overloaded) + signature = method.type + signature = freshen_all_functions_type_vars(signature) + if not method.is_static: + # TODO: use proper treatment of special methods on unions instead + # of this hack here and below (i.e. mx.self_type). + dispatched_type = meet.meet_types(mx.original_type, typ) + signature = check_self_arg( + signature, dispatched_type, method.is_class, mx.context, name, mx.msg + ) + signature = bind_self(signature, mx.self_type, is_classmethod=method.is_class) + # TODO: should we skip these steps for static methods as well? + # Since generic static methods should not be allowed. + typ = map_instance_to_supertype(typ, method.info) + member_type = expand_type_by_instance(signature, typ) + freeze_all_type_vars(member_type) + return member_type + else: + # Not a method. + return analyze_member_var_access(name, typ, info, mx) + + +def validate_super_call(node: FuncBase, mx: MemberContext) -> None: + unsafe_super = False + if isinstance(node, FuncDef) and node.is_trivial_body: + unsafe_super = True + impl = node + elif isinstance(node, OverloadedFuncDef): + if node.impl: + impl = node.impl if isinstance(node.impl, FuncDef) else node.impl.func + unsafe_super = impl.is_trivial_body + if unsafe_super: + mx.msg.unsafe_super(node.name, node.info.name, mx.context) + + +def analyze_type_callable_member_access(name: str, typ: FunctionLike, mx: MemberContext) -> Type: + # Class attribute. + # TODO super? + ret_type = typ.items[0].ret_type + assert isinstance(ret_type, ProperType) + if isinstance(ret_type, TupleType): + ret_type = tuple_fallback(ret_type) + if isinstance(ret_type, TypedDictType): + ret_type = ret_type.fallback + if isinstance(ret_type, Instance): + if not mx.is_operator: + # When Python sees an operator (eg `3 == 4`), it automatically translates that + # into something like `int.__eq__(3, 4)` instead of `(3).__eq__(4)` as an + # optimization. + # + # While it normally it doesn't matter which of the two versions are used, it + # does cause inconsistencies when working with classes. For example, translating + # `int == int` to `int.__eq__(int)` would not work since `int.__eq__` is meant to + # compare two int _instances_. What we really want is `type(int).__eq__`, which + # is meant to compare two types or classes. + # + # This check makes sure that when we encounter an operator, we skip looking up + # the corresponding method in the current instance to avoid this edge case. + # See https://github.com/python/mypy/pull/1787 for more info. + # TODO: do not rely on same type variables being present in all constructor overloads. + result = analyze_class_attribute_access( + ret_type, name, mx, original_vars=typ.items[0].variables, mcs_fallback=typ.fallback + ) + if result: + return result # Look up from the 'type' type. - return analyse_member_access(name, sig.fallback, node, is_lvalue, is_super, - builtin_type, msg, report_type=report_type) - elif isinstance(typ, FunctionLike): - # Look up from the 'function' type. - return analyse_member_access(name, typ.fallback, node, is_lvalue, is_super, - builtin_type, msg, report_type=report_type) - return msg.has_no_attr(report_type, name, node) - - -def analyse_member_var_access(name: str, itype: Instance, info: TypeInfo, - node: Context, is_lvalue: bool, is_super: bool, - msg: MessageBuilder, - report_type: Type = None) -> Type: + return _analyze_member_access(name, typ.fallback, mx) + else: + assert False, f"Unexpected type {ret_type!r}" + + +def analyze_type_type_member_access( + name: str, typ: TypeType, mx: MemberContext, override_info: TypeInfo | None +) -> Type: + # Similar to analyze_type_callable_attribute_access. + item = None + fallback = mx.named_type("builtins.type") + if isinstance(typ.item, Instance): + item = typ.item + elif isinstance(typ.item, AnyType): + with mx.msg.filter_errors(): + return _analyze_member_access(name, fallback, mx, override_info) + elif isinstance(typ.item, TypeVarType): + upper_bound = get_proper_type(typ.item.upper_bound) + if isinstance(upper_bound, Instance): + item = upper_bound + elif isinstance(upper_bound, UnionType): + return _analyze_member_access( + name, + TypeType.make_normalized(upper_bound, line=typ.line, column=typ.column), + mx, + override_info, + ) + elif isinstance(upper_bound, TupleType): + item = tuple_fallback(upper_bound) + elif isinstance(upper_bound, AnyType): + with mx.msg.filter_errors(): + return _analyze_member_access(name, fallback, mx, override_info) + elif isinstance(typ.item, TupleType): + item = tuple_fallback(typ.item) + elif isinstance(typ.item, FunctionLike) and typ.item.is_type_obj(): + item = typ.item.fallback + elif isinstance(typ.item, TypeType): + # Access member on metaclass object via Type[Type[C]] + if isinstance(typ.item.item, Instance): + item = typ.item.item.type.metaclass_type + ignore_messages = False + + if item is not None: + fallback = item.type.metaclass_type or fallback + + if item and not mx.is_operator: + # See comment above for why operators are skipped + result = analyze_class_attribute_access( + item, name, mx, mcs_fallback=fallback, override_info=override_info + ) + if result: + if not (isinstance(get_proper_type(result), AnyType) and item.type.fallback_to_any): + return result + else: + # We don't want errors on metaclass lookup for classes with Any fallback + ignore_messages = True + + with mx.msg.filter_errors(filter_errors=ignore_messages): + return _analyze_member_access(name, fallback, mx, override_info) + + +def analyze_union_member_access(name: str, typ: UnionType, mx: MemberContext) -> Type: + with mx.msg.disable_type_names(): + results = [] + for subtype in typ.relevant_items(): + # Self types should be bound to every individual item of a union. + item_mx = mx.copy_modified(self_type=subtype) + results.append(_analyze_member_access(name, subtype, item_mx)) + return make_simplified_union(results) + + +def analyze_none_member_access(name: str, typ: NoneType, mx: MemberContext) -> Type: + if name == "__bool__": + literal_false = LiteralType(False, fallback=mx.named_type("builtins.bool")) + return CallableType( + arg_types=[], + arg_kinds=[], + arg_names=[], + ret_type=literal_false, + fallback=mx.named_type("builtins.function"), + ) + else: + return _analyze_member_access(name, mx.named_type("builtins.object"), mx) + + +def analyze_member_var_access( + name: str, itype: Instance, info: TypeInfo, mx: MemberContext +) -> Type: """Analyse attribute access that does not target a method. - This is logically part of analyse_member_access and the arguments are - similar. + This is logically part of analyze_member_access and the arguments are similar. + + original_type is the type of E in the expression E.var """ # It was not a method. Try looking up a variable. - v = lookup_member_var_or_accessor(info, name, is_lvalue) + v = lookup_member_var_or_accessor(info, name, mx.is_lvalue) + + mx.chk.warn_deprecated(v, mx.context) vv = v if isinstance(vv, Decorator): # The associated Var node of a decorator contains the type. v = vv.var + if mx.is_super: + validate_super_call(vv.func, mx) + + if isinstance(vv, TypeInfo): + # If the associated variable is a TypeInfo synthesize a Var node for + # the purposes of type checking. This enables us to type check things + # like accessing class attributes on an inner class. + v = Var(name, type=type_object_type(vv, mx.named_type)) + v.info = info + + if isinstance(vv, TypeAlias): + # Similar to the above TypeInfo case, we allow using + # qualified type aliases in runtime context if it refers to an + # instance type. For example: + # class C: + # A = List[int] + # x = C.A() <- this is OK + typ = mx.chk.expr_checker.alias_type_in_runtime_context( + vv, ctx=mx.context, alias_definition=mx.is_lvalue + ) + v = Var(name, type=typ) + v.info = info if isinstance(v, Var): - # Found a member variable. - var = v - itype = map_instance_to_supertype(itype, var.info) - if var.type: - t = expand_type_by_instance(var.type, itype) - if var.is_initialized_in_class and isinstance(t, FunctionLike): - if is_lvalue: - if var.is_property: - msg.read_only_property(name, info, node) - else: - msg.cant_assign_to_method(node) - - if not var.is_staticmethod: - # Class-level function objects and classmethods become bound - # methods: the former to the instance, the latter to the - # class. - functype = cast(FunctionLike, t) - check_method_type(functype, itype, node, msg) - signature = method_type(functype) - if var.is_property: - # A property cannot have an overloaded type => the cast - # is fine. - return cast(Callable, signature).ret_type - else: - return signature - return t - else: - if not var.is_ready: - msg.cannot_determine_type(var.name(), node) - # Implicit 'Any' type. - return AnyType() + implicit = info[name].implicit + + # An assignment to final attribute is always an error, + # independently of types. + if mx.is_lvalue and not mx.chk.get_final_context(): + check_final_member(name, info, mx.msg, mx.context) + + return analyze_var(name, v, itype, info, mx, implicit=implicit) elif isinstance(v, FuncDef): assert False, "Did not expect a function" + elif isinstance(v, MypyFile): + mx.chk.module_refs.add(v.fullname) + return mx.chk.expr_checker.module_type(v) + elif ( + not v + and name not in ["__getattr__", "__setattr__", "__getattribute__"] + and not mx.is_operator + and mx.module_symbol_table is None + ): + # Above we skip ModuleType.__getattr__ etc. if we have a + # module symbol table, since the symbol table allows precise + # checking. + if not mx.is_lvalue: + for method_name in ("__getattribute__", "__getattr__"): + method = info.get_method(method_name) + + # __getattribute__ is defined on builtins.object and returns Any, so without + # the guard this search will always find object.__getattribute__ and conclude + # that the attribute exists + if method and method.info.fullname != "builtins.object": + bound_method = analyze_decorator_or_funcbase_access( + defn=method, + itype=itype, + info=info, + self_type=mx.self_type, + name=method_name, + mx=mx, + ) + typ = map_instance_to_supertype(itype, method.info) + getattr_type = get_proper_type(expand_type_by_instance(bound_method, typ)) + if isinstance(getattr_type, CallableType): + result = getattr_type.ret_type + else: + result = getattr_type + + # Call the attribute hook before returning. + fullname = f"{method.info.fullname}.{name}" + hook = mx.chk.plugin.get_attribute_hook(fullname) + if hook: + result = hook( + AttributeContext( + get_proper_type(mx.original_type), + result, + mx.is_lvalue, + mx.context, + mx.chk, + ) + ) + return result + else: + setattr_meth = info.get_method("__setattr__") + if setattr_meth and setattr_meth.info.fullname != "builtins.object": + bound_type = analyze_decorator_or_funcbase_access( + defn=setattr_meth, + itype=itype, + info=info, + self_type=mx.self_type, + name=name, + mx=mx.copy_modified(is_lvalue=False), + ) + typ = map_instance_to_supertype(itype, setattr_meth.info) + setattr_type = get_proper_type(expand_type_by_instance(bound_type, typ)) + if isinstance(setattr_type, CallableType) and len(setattr_type.arg_types) > 0: + return setattr_type.arg_types[-1] + + if itype.type.fallback_to_any: + return AnyType(TypeOfAny.special_form) # Could not find the member. - if is_super: - msg.undefined_in_superclass(name, node) - return AnyType() + if itype.extra_attrs and name in itype.extra_attrs.attrs: + # For modules use direct symbol table lookup. + if not itype.extra_attrs.mod_name: + return itype.extra_attrs.attrs[name] + + if mx.is_super: + mx.msg.undefined_in_superclass(name, mx.context) + return AnyType(TypeOfAny.from_error) else: - return msg.has_no_attr(report_type or itype, name, node) + ret = report_missing_attribute(mx.original_type, itype, name, mx) + # Avoid paying double jeopardy if we can't find the member due to --no-implicit-reexport + if ( + mx.module_symbol_table is not None + and name in mx.module_symbol_table + and not mx.module_symbol_table[name].module_public + ): + v = mx.module_symbol_table[name].node + e = NameExpr(name) + e.set_line(mx.context) + e.node = v + return mx.chk.expr_checker.analyze_ref_expr(e, lvalue=mx.is_lvalue) + return ret -def lookup_member_var_or_accessor(info: TypeInfo, name: str, - is_lvalue: bool) -> SymbolNode: +def check_final_member(name: str, info: TypeInfo, msg: MessageBuilder, ctx: Context) -> None: + """Give an error if the name being assigned was declared as final.""" + for base in info.mro: + sym = base.names.get(name) + if sym and is_final_node(sym.node): + msg.cant_assign_to_final(name, attr_assign=True, ctx=ctx) + + +def analyze_descriptor_access( + descriptor_type: Type, mx: MemberContext, *, assignment: bool = False +) -> Type: + """Type check descriptor access. + + Arguments: + descriptor_type: The type of the descriptor attribute being accessed + (the type of ``f`` in ``a.f`` when ``f`` is a descriptor). + mx: The current member access context. + Return: + The return type of the appropriate ``__get__`` overload for the descriptor. + """ + instance_type = get_proper_type(mx.self_type) + orig_descriptor_type = descriptor_type + descriptor_type = get_proper_type(descriptor_type) + + if isinstance(descriptor_type, UnionType): + # Map the access over union types + return make_simplified_union( + [analyze_descriptor_access(typ, mx) for typ in descriptor_type.items] + ) + elif not isinstance(descriptor_type, Instance): + return orig_descriptor_type + + if not descriptor_type.type.has_readable_member("__get__"): + return orig_descriptor_type + + dunder_get = descriptor_type.type.get_method("__get__") + if dunder_get is None: + mx.msg.fail( + message_registry.DESCRIPTOR_GET_NOT_CALLABLE.format( + descriptor_type.str_with_options(mx.msg.options) + ), + mx.context, + ) + return AnyType(TypeOfAny.from_error) + + bound_method = analyze_decorator_or_funcbase_access( + defn=dunder_get, + itype=descriptor_type, + info=descriptor_type.type, + self_type=descriptor_type, + name="__get__", + mx=mx, + ) + + typ = map_instance_to_supertype(descriptor_type, dunder_get.info) + dunder_get_type = expand_type_by_instance(bound_method, typ) + + if isinstance(instance_type, FunctionLike) and instance_type.is_type_obj(): + owner_type = instance_type.items[0].ret_type + instance_type = NoneType() + elif isinstance(instance_type, TypeType): + owner_type = instance_type.item + instance_type = NoneType() + else: + owner_type = instance_type + + callable_name = mx.chk.expr_checker.method_fullname(descriptor_type, "__get__") + dunder_get_type = mx.chk.expr_checker.transform_callee_type( + callable_name, + dunder_get_type, + [ + TempNode(instance_type, context=mx.context), + TempNode(TypeType.make_normalized(owner_type), context=mx.context), + ], + [ARG_POS, ARG_POS], + mx.context, + object_type=descriptor_type, + ) + + _, inferred_dunder_get_type = mx.chk.expr_checker.check_call( + dunder_get_type, + [ + TempNode(instance_type, context=mx.context), + TempNode(TypeType.make_normalized(owner_type), context=mx.context), + ], + [ARG_POS, ARG_POS], + mx.context, + object_type=descriptor_type, + callable_name=callable_name, + ) + + if not assignment: + mx.chk.check_deprecated(dunder_get, mx.context) + mx.chk.warn_deprecated_overload_item( + dunder_get, mx.context, target=inferred_dunder_get_type, selftype=descriptor_type + ) + + inferred_dunder_get_type = get_proper_type(inferred_dunder_get_type) + if isinstance(inferred_dunder_get_type, AnyType): + # check_call failed, and will have reported an error + return inferred_dunder_get_type + + if not isinstance(inferred_dunder_get_type, CallableType): + mx.msg.fail( + message_registry.DESCRIPTOR_GET_NOT_CALLABLE.format( + descriptor_type.str_with_options(mx.msg.options) + ), + mx.context, + ) + return AnyType(TypeOfAny.from_error) + + return inferred_dunder_get_type.ret_type + + +def is_instance_var(var: Var) -> bool: + """Return if var is an instance variable according to PEP 526.""" + return ( + # check the type_info node is the var (not a decorated function, etc.) + var.name in var.info.names + and var.info.names[var.name].node is var + and not var.is_classvar + # variables without annotations are treated as classvar + and not var.is_inferred + ) + + +def analyze_var( + name: str, + var: Var, + itype: Instance, + info: TypeInfo, + mx: MemberContext, + *, + implicit: bool = False, +) -> Type: + """Analyze access to an attribute via a Var node. + + This is conceptually part of analyze_member_access and the arguments are similar. + itype is the instance type in which attribute should be looked up + original_type is the type of E in the expression E.var + if implicit is True, the original Var was created as an assignment to self + """ + # Found a member variable. + original_itype = itype + itype = map_instance_to_supertype(itype, var.info) + typ = var.type + if typ: + if isinstance(typ, PartialType): + return mx.chk.handle_partial_var_type(typ, mx.is_lvalue, var, mx.context) + if mx.is_lvalue and var.is_property and not var.is_settable_property: + # TODO allow setting attributes in subclass (although it is probably an error) + mx.msg.read_only_property(name, itype.type, mx.context) + if mx.is_lvalue and var.is_classvar: + mx.msg.cant_assign_to_classvar(name, mx.context) + t = freshen_all_functions_type_vars(typ) + t = expand_self_type_if_needed(t, mx, var, original_itype) + t = expand_type_by_instance(t, itype) + freeze_all_type_vars(t) + result = t + typ = get_proper_type(typ) + + call_type: ProperType | None = None + if var.is_initialized_in_class and (not is_instance_var(var) or mx.is_operator): + if isinstance(typ, FunctionLike) and not typ.is_type_obj(): + call_type = typ + elif var.is_property: + call_type = get_proper_type(_analyze_member_access("__call__", typ, mx)) + else: + call_type = typ + + if isinstance(call_type, FunctionLike) and not call_type.is_type_obj(): + if mx.is_lvalue: + if var.is_property: + if not var.is_settable_property: + mx.msg.read_only_property(name, itype.type, mx.context) + else: + mx.msg.cant_assign_to_method(mx.context) + + if not var.is_staticmethod: + # Class-level function objects and classmethods become bound methods: + # the former to the instance, the latter to the class. + functype: FunctionLike = call_type + # Use meet to narrow original_type to the dispatched type. + # For example, assume + # * A.f: Callable[[A1], None] where A1 <: A (maybe A1 == A) + # * B.f: Callable[[B1], None] where B1 <: B (maybe B1 == B) + # * x: Union[A1, B1] + # In `x.f`, when checking `x` against A1 we assume x is compatible with A + # and similarly for B1 when checking against B + dispatched_type = meet.meet_types(mx.original_type, itype) + signature = freshen_all_functions_type_vars(functype) + bound = get_proper_type(expand_self_type(var, signature, mx.original_type)) + assert isinstance(bound, FunctionLike) + signature = bound + signature = check_self_arg( + signature, dispatched_type, var.is_classmethod, mx.context, name, mx.msg + ) + signature = bind_self(signature, mx.self_type, var.is_classmethod) + expanded_signature = expand_type_by_instance(signature, itype) + freeze_all_type_vars(expanded_signature) + if var.is_property: + # A property cannot have an overloaded type => the cast is fine. + assert isinstance(expanded_signature, CallableType) + result = expanded_signature.ret_type + else: + result = expanded_signature + else: + if not var.is_ready and not mx.no_deferral: + mx.not_ready_callback(var.name, mx.context) + # Implicit 'Any' type. + result = AnyType(TypeOfAny.special_form) + fullname = f"{var.info.fullname}.{name}" + hook = mx.chk.plugin.get_attribute_hook(fullname) + if result and not mx.is_lvalue and not implicit: + result = analyze_descriptor_access(result, mx) + if hook: + result = hook( + AttributeContext( + get_proper_type(mx.original_type), result, mx.is_lvalue, mx.context, mx.chk + ) + ) + return result + + +def expand_self_type_if_needed( + t: Type, mx: MemberContext, var: Var, itype: Instance, is_class: bool = False +) -> Type: + """Expand special Self type in a backwards compatible manner. + + This should ensure that mixing old-style and new-style self-types work + seamlessly. Also, re-bind new style self-types in subclasses if needed. + """ + original = get_proper_type(mx.self_type) + if not (mx.is_self or mx.is_super): + repl = mx.self_type + if is_class: + if isinstance(original, TypeType): + repl = original.item + elif isinstance(original, CallableType): + # Problematic access errors should have been already reported. + repl = erase_typevars(original.ret_type) + else: + repl = itype + return expand_self_type(var, t, repl) + elif supported_self_type( + # Support compatibility with plain old style T -> T and Type[T] -> T only. + get_proper_type(mx.self_type), + allow_instances=False, + allow_callable=False, + ): + repl = mx.self_type + if is_class and isinstance(original, TypeType): + repl = original.item + return expand_self_type(var, t, repl) + elif ( + mx.is_self + and itype.type != var.info + # If an attribute with Self-type was defined in a supertype, we need to + # rebind the Self type variable to Self type variable of current class... + and itype.type.self_type is not None + # ...unless `self` has an explicit non-trivial annotation. + and itype == mx.chk.scope.active_self_type() + ): + return expand_self_type(var, t, itype.type.self_type) + else: + return t + + +def freeze_all_type_vars(member_type: Type) -> None: + member_type.accept(FreezeTypeVarsVisitor()) + + +class FreezeTypeVarsVisitor(TypeTraverserVisitor): + def visit_callable_type(self, t: CallableType) -> None: + for v in t.variables: + v.id.meta_level = 0 + super().visit_callable_type(t) + + +def lookup_member_var_or_accessor(info: TypeInfo, name: str, is_lvalue: bool) -> SymbolNode | None: """Find the attribute/accessor node that refers to a member of a type.""" # TODO handle lvalues node = info.get(name) @@ -162,147 +919,503 @@ def lookup_member_var_or_accessor(info: TypeInfo, name: str, return None -def check_method_type(functype: FunctionLike, itype: Instance, - context: Context, msg: MessageBuilder) -> None: - for item in functype.items(): - if not item.arg_types or item.arg_kinds[0] != ARG_POS: - # No positional first (self) argument. - msg.invalid_method_type(item, context) +def check_self_arg( + functype: FunctionLike, + dispatched_arg_type: Type, + is_classmethod: bool, + context: Context, + name: str, + msg: MessageBuilder, +) -> FunctionLike: + """Check that an instance has a valid type for a method with annotated 'self'. + + For example if the method is defined as: + class A: + def f(self: S) -> T: ... + then for 'x.f' we check that meet(type(x), A) <: S. If the method is overloaded, we + select only overloads items that satisfy this requirement. If there are no matching + overloads, an error is generated. + + Note: dispatched_arg_type uses a meet to select a relevant item in case if the + original type of 'x' is a union. This is done because several special methods + treat union types in ad-hoc manner, so we can't use MemberContext.self_type yet. + """ + items = functype.items + if not items: + return functype + new_items = [] + if is_classmethod: + dispatched_arg_type = TypeType.make_normalized(dispatched_arg_type) + + for item in items: + if not item.arg_types or item.arg_kinds[0] not in (ARG_POS, ARG_STAR): + # No positional first (self) argument (*args is okay). + msg.no_formal_self(name, item, context) + # This is pretty bad, so just return the original signature if + # there is at least one such error. + return functype else: - # Check that self argument has type 'Any' or valid instance type. - selfarg = item.arg_types[0] - if not subtypes.is_equivalent(selfarg, itype): - msg.invalid_method_type(item, context) - - -def analyse_class_attribute_access(itype: Instance, - name: str, - context: Context, - is_lvalue: bool, - builtin_type: Function[[str], Instance], - msg: MessageBuilder) -> Type: - node = itype.type.get(name) + selfarg = get_proper_type(item.arg_types[0]) + # This level of erasure matches the one in checker.check_func_def(), + # better keep these two checks consistent. + if subtypes.is_subtype(dispatched_arg_type, erase_typevars(erase_to_bound(selfarg))): + new_items.append(item) + elif isinstance(selfarg, ParamSpecType): + # TODO: This is not always right. What's the most reasonable thing to do here? + new_items.append(item) + elif isinstance(selfarg, TypeVarTupleType): + raise NotImplementedError + if not new_items: + # Choose first item for the message (it may be not very helpful for overloads). + msg.incompatible_self_argument( + name, dispatched_arg_type, items[0], is_classmethod, context + ) + return functype + if len(new_items) == 1: + return new_items[0] + return Overloaded(new_items) + + +def analyze_class_attribute_access( + itype: Instance, + name: str, + mx: MemberContext, + *, + mcs_fallback: Instance, + override_info: TypeInfo | None = None, + original_vars: Sequence[TypeVarLikeType] | None = None, +) -> Type | None: + """Analyze access to an attribute on a class object. + + itype is the return type of the class object callable, original_type is the type + of E in the expression E.var, original_vars are type variables of the class callable + (for generic classes). + """ + info = itype.type + if override_info: + info = override_info + + fullname = f"{info.fullname}.{name}" + hook = mx.chk.plugin.get_class_attribute_hook(fullname) + + node = info.get(name) if not node: + if itype.extra_attrs and name in itype.extra_attrs.attrs: + # For modules use direct symbol table lookup. + if not itype.extra_attrs.mod_name: + return itype.extra_attrs.attrs[name] + if info.fallback_to_any or info.meta_fallback_to_any: + return apply_class_attr_hook(mx, hook, AnyType(TypeOfAny.special_form)) return None + if ( + isinstance(node.node, Var) + and not node.node.is_classvar + and not hook + and mcs_fallback.type.get(name) + ): + # If the same attribute is declared on the metaclass and the class but with different types, + # and the attribute on the class is not a ClassVar, + # the type of the attribute on the metaclass should take priority + # over the type of the attribute on the class, + # when the attribute is being accessed from the class object itself. + # + # Return `None` here to signify that the name should be looked up + # on the class object itself rather than the instance. + return None + + mx.chk.warn_deprecated(node.node, mx.context) + is_decorated = isinstance(node.node, Decorator) - is_method = is_decorated or isinstance(node.node, FuncDef) - if is_lvalue: + is_method = is_decorated or isinstance(node.node, FuncBase) + if mx.is_lvalue: if is_method: - msg.cant_assign_to_method(context) + mx.msg.cant_assign_to_method(mx.context) if isinstance(node.node, TypeInfo): - msg.fail(messages.CANNOT_ASSIGN_TO_TYPE, context) + mx.msg.fail(message_registry.CANNOT_ASSIGN_TO_TYPE, mx.context) + + # Refuse class attribute access if slot defined + if info.slots and name in info.slots: + mx.msg.fail(message_registry.CLASS_VAR_CONFLICTS_SLOTS.format(name), mx.context) + + # If a final attribute was declared on `self` in `__init__`, then it + # can't be accessed on the class object. + if node.implicit and isinstance(node.node, Var) and node.node.is_final: + mx.msg.fail( + message_registry.CANNOT_ACCESS_FINAL_INSTANCE_ATTR.format(node.node.name), mx.context + ) + + # An assignment to final attribute on class object is also always an error, + # independently of types. + if mx.is_lvalue and not mx.chk.get_final_context(): + check_final_member(name, info, mx.msg, mx.context) + + if info.is_enum and not (mx.is_lvalue or is_decorated or is_method): + enum_class_attribute_type = analyze_enum_class_attribute_access(itype, name, mx) + if enum_class_attribute_type: + return apply_class_attr_hook(mx, hook, enum_class_attribute_type) t = node.type if t: - is_classmethod = is_decorated and cast(Decorator, node.node).func.is_class - return add_class_tvars(t, itype.type, is_classmethod, builtin_type) + if isinstance(t, PartialType): + symnode = node.node + assert isinstance(symnode, Var) + return apply_class_attr_hook( + mx, hook, mx.chk.handle_partial_var_type(t, mx.is_lvalue, symnode, mx.context) + ) + + # Find the class where method/variable was defined. + if isinstance(node.node, Decorator): + super_info: TypeInfo | None = node.node.var.info + elif isinstance(node.node, (Var, SYMBOL_FUNCBASE_TYPES)): + super_info = node.node.info + else: + super_info = None + + # Map the type to how it would look as a defining class. For example: + # class C(Generic[T]): ... + # class D(C[Tuple[T, S]]): ... + # D[int, str].method() + # Here itype is D[int, str], isuper is C[Tuple[int, str]]. + if not super_info: + isuper = None + else: + isuper = map_instance_to_supertype(itype, super_info) + + if isinstance(node.node, Var): + assert isuper is not None + # Check if original variable type has type variables. For example: + # class C(Generic[T]): + # x: T + # C.x # Error, ambiguous access + # C[int].x # Also an error, since C[int] is same as C at runtime + # Exception is Self type wrapped in ClassVar, that is safe. + def_vars = set(node.node.info.defn.type_vars) + if not node.node.is_classvar and node.node.info.self_type: + def_vars.add(node.node.info.self_type) + typ_vars = set(get_type_vars(t)) + if def_vars & typ_vars: + # Exception: access on Type[...], including first argument of class methods is OK. + if not isinstance(get_proper_type(mx.original_type), TypeType) or node.implicit: + if node.node.is_classvar: + message = message_registry.GENERIC_CLASS_VAR_ACCESS + else: + message = message_registry.GENERIC_INSTANCE_VAR_CLASS_ACCESS + mx.msg.fail(message, mx.context) + t = expand_self_type_if_needed(t, mx, node.node, itype, is_class=True) + # Erase non-mapped variables, but keep mapped ones, even if there is an error. + # In the above example this means that we infer following types: + # C.x -> Any + # C[int].x -> int + t = erase_typevars(expand_type_by_instance(t, isuper), {tv.id for tv in def_vars}) + + is_classmethod = (is_decorated and cast(Decorator, node.node).func.is_class) or ( + isinstance(node.node, FuncBase) and node.node.is_class + ) + is_staticmethod = (is_decorated and cast(Decorator, node.node).func.is_static) or ( + isinstance(node.node, FuncBase) and node.node.is_static + ) + t = get_proper_type(t) + if isinstance(t, FunctionLike) and is_classmethod: + t = check_self_arg(t, mx.self_type, False, mx.context, name, mx.msg) + result = add_class_tvars( + t, isuper, is_classmethod, is_staticmethod, mx.self_type, original_vars=original_vars + ) + if not mx.is_lvalue: + result = analyze_descriptor_access(result, mx) + + return apply_class_attr_hook(mx, hook, result) + elif isinstance(node.node, Var): + mx.not_ready_callback(name, mx.context) + return AnyType(TypeOfAny.special_form) + + if isinstance(node.node, TypeVarExpr): + mx.msg.fail( + message_registry.CANNOT_USE_TYPEVAR_AS_EXPRESSION.format(info.name, name), mx.context + ) + return AnyType(TypeOfAny.from_error) if isinstance(node.node, TypeInfo): - return type_object_type(cast(TypeInfo, node.node), builtin_type) + return type_object_type(node.node, mx.named_type) + + if isinstance(node.node, MypyFile): + # Reference to a module object. + return mx.named_type("types.ModuleType") + + if isinstance(node.node, TypeAlias): + return mx.chk.expr_checker.alias_type_in_runtime_context( + node.node, ctx=mx.context, alias_definition=mx.is_lvalue + ) + + if is_decorated: + assert isinstance(node.node, Decorator) + if node.node.type: + return apply_class_attr_hook(mx, hook, node.node.type) + else: + mx.not_ready_callback(name, mx.context) + return AnyType(TypeOfAny.from_error) + else: + assert isinstance(node.node, FuncBase) + typ = function_type(node.node, mx.named_type("builtins.function")) + # Note: if we are accessing class method on class object, the cls argument is bound. + # Annotated and/or explicit class methods go through other code paths above, for + # unannotated implicit class methods we do this here. + if node.node.is_class: + typ = bind_self(typ, is_classmethod=True) + return apply_class_attr_hook(mx, hook, typ) + + +def apply_class_attr_hook( + mx: MemberContext, hook: Callable[[AttributeContext], Type] | None, result: Type +) -> Type | None: + if hook: + result = hook( + AttributeContext( + get_proper_type(mx.original_type), result, mx.is_lvalue, mx.context, mx.chk + ) + ) + return result + + +def analyze_enum_class_attribute_access( + itype: Instance, name: str, mx: MemberContext +) -> Type | None: + # Skip these since Enum will remove it + if name in EXCLUDED_ENUM_ATTRIBUTES: + return report_missing_attribute(mx.original_type, itype, name, mx) + # Dunders and private names are not Enum members + if name.startswith("__") and name.replace("_", "") != "": + return None + + node = itype.type.get(name) + if node and node.type: + proper = get_proper_type(node.type) + # Support `A = nonmember(1)` function call and decorator. + if ( + isinstance(proper, Instance) + and proper.type.fullname == "enum.nonmember" + and proper.args + ): + return proper.args[0] - return function_type(cast(FuncBase, node.node), builtin_type('builtins.function')) + enum_literal = LiteralType(name, fallback=itype) + return itype.copy_modified(last_known_value=enum_literal) + + +def analyze_typeddict_access( + name: str, typ: TypedDictType, mx: MemberContext, override_info: TypeInfo | None +) -> Type: + if name == "__setitem__": + if isinstance(mx.context, IndexExpr): + # Since we can get this during `a['key'] = ...` + # it is safe to assume that the context is `IndexExpr`. + item_type, key_names = mx.chk.expr_checker.visit_typeddict_index_expr( + typ, mx.context.index, setitem=True + ) + assigned_readonly_keys = typ.readonly_keys & key_names + if assigned_readonly_keys: + mx.msg.readonly_keys_mutated(assigned_readonly_keys, context=mx.context) + else: + # It can also be `a.__setitem__(...)` direct call. + # In this case `item_type` can be `Any`, + # because we don't have args available yet. + # TODO: check in `default` plugin that `__setitem__` is correct. + item_type = AnyType(TypeOfAny.implementation_artifact) + return CallableType( + arg_types=[mx.chk.named_type("builtins.str"), item_type], + arg_kinds=[ARG_POS, ARG_POS], + arg_names=[None, None], + ret_type=NoneType(), + fallback=mx.chk.named_type("builtins.function"), + name=name, + ) + elif name == "__delitem__": + return CallableType( + arg_types=[mx.chk.named_type("builtins.str")], + arg_kinds=[ARG_POS], + arg_names=[None], + ret_type=NoneType(), + fallback=mx.chk.named_type("builtins.function"), + name=name, + ) + return _analyze_member_access(name, typ.fallback, mx, override_info) -def add_class_tvars(t: Type, info: TypeInfo, is_classmethod: bool, - builtin_type: Function[[str], Instance]) -> Type: - if isinstance(t, Callable): - # TODO: Should we propagate type variable values? - vars = [TypeVarDef(n, i + 1, None, builtin_type('builtins.object')) - for i, n in enumerate(info.type_vars)] - arg_types = t.arg_types - arg_kinds = t.arg_kinds - arg_names = t.arg_names +def add_class_tvars( + t: ProperType, + isuper: Instance | None, + is_classmethod: bool, + is_staticmethod: bool, + original_type: Type, + original_vars: Sequence[TypeVarLikeType] | None = None, +) -> Type: + """Instantiate type variables during analyze_class_attribute_access, + e.g T and Q in the following: + + class A(Generic[T]): + @classmethod + def foo(cls: Type[Q]) -> Tuple[T, Q]: ... + + class B(A[str]): pass + B.foo() + + Args: + t: Declared type of the method (or property) + isuper: Current instance mapped to the superclass where method was defined, this + is usually done by map_instance_to_supertype() + is_classmethod: True if this method is decorated with @classmethod + is_staticmethod: True if this method is decorated with @staticmethod + original_type: The value of the type B in the expression B.foo() or the corresponding + component in case of a union (this is used to bind the self-types) + original_vars: Type variables of the class callable on which the method was accessed + Returns: + Expanded method type with added type variables (when needed). + """ + # TODO: verify consistency between Q and T + + # We add class type variables if the class method is accessed on class object + # without applied type arguments, this matches the behavior of __init__(). + # For example (continuing the example in docstring): + # A # The type of callable is def [T] () -> A[T], _not_ def () -> A[Any] + # A[int] # The type of callable is def () -> A[int] + # and + # A.foo # The type is generic def [T] () -> Tuple[T, A[T]] + # A[int].foo # The type is non-generic def () -> Tuple[int, A[int]] + # + # This behaviour is useful for defining alternative constructors for generic classes. + # To achieve such behaviour, we add the class type variables that are still free + # (i.e. appear in the return type of the class object on which the method was accessed). + if isinstance(t, CallableType): + tvars = original_vars if original_vars is not None else [] + t = freshen_all_functions_type_vars(t) if is_classmethod: - arg_types = arg_types[1:] - arg_kinds = arg_kinds[1:] - arg_names = arg_names[1:] - return Callable(arg_types, - arg_kinds, - arg_names, - t.ret_type, - t.fallback, - t.name, - vars + t.variables, - t.bound_vars, - t.line, None) + t = bind_self(t, original_type, is_classmethod=True) + if is_classmethod or is_staticmethod: + assert isuper is not None + t = expand_type_by_instance(t, isuper) + freeze_all_type_vars(t) + return t.copy_modified(variables=list(tvars) + list(t.variables)) elif isinstance(t, Overloaded): - return Overloaded([cast(Callable, add_class_tvars(i, info, is_classmethod, builtin_type)) - for i in t.items()]) + return Overloaded( + [ + cast( + CallableType, + add_class_tvars( + item, + isuper, + is_classmethod, + is_staticmethod, + original_type, + original_vars=original_vars, + ), + ) + for item in t.items + ] + ) + if isuper is not None: + t = expand_type_by_instance(t, isuper) return t -def type_object_type(info: TypeInfo, builtin_type: Function[[str], Instance]) -> Type: +def type_object_type(info: TypeInfo, named_type: Callable[[str], Instance]) -> ProperType: """Return the type of a type object. - For a generic type G with type variables T and S the type is of form + For a generic type G with type variables T and S the type is generally of form - def [T, S](...) -> G[T, S], + Callable[..., G[T, S]] - where ... are argument types for the __init__ method (without the self argument). + where ... are argument types for the __init__/__new__ method (without the self + argument). Also, the fallback type will be 'type' instead of 'function'. """ - init_method = info.get_method('__init__') - if not init_method: + + # We take the type from whichever of __init__ and __new__ is first + # in the MRO, preferring __init__ if there is a tie. + init_method = info.get("__init__") + new_method = info.get("__new__") + if not init_method or not is_valid_constructor(init_method.node): # Must be an invalid class definition. - return AnyType() - else: - # Construct callable type based on signature of __init__. Adjust - # return type and insert type arguments. - init_type = method_type(init_method, builtin_type('builtins.function')) - if isinstance(init_type, Callable): - return class_callable(init_type, info, builtin_type('builtins.type')) - else: - # Overloaded __init__. - items = [] # type: List[Callable] - for it in cast(Overloaded, init_type).items(): - items.append(class_callable(it, info, builtin_type('builtins.type'))) - return Overloaded(items) + return AnyType(TypeOfAny.from_error) + # There *should* always be a __new__ method except the test stubs + # lack it, so just copy init_method in that situation + new_method = new_method or init_method + if not is_valid_constructor(new_method.node): + # Must be an invalid class definition. + return AnyType(TypeOfAny.from_error) + # The two is_valid_constructor() checks ensure this. + assert isinstance(new_method.node, (SYMBOL_FUNCBASE_TYPES, Decorator)) + assert isinstance(init_method.node, (SYMBOL_FUNCBASE_TYPES, Decorator)) -def class_callable(init_type: Callable, info: TypeInfo, type_type: Instance) -> Callable: - """Create a type object type based on the signature of __init__.""" - variables = [] # type: List[TypeVarDef] - for i, tvar in enumerate(info.defn.type_vars): - variables.append(TypeVarDef(tvar.name, i + 1, tvar.values, tvar.upper_bound)) + init_index = info.mro.index(init_method.node.info) + new_index = info.mro.index(new_method.node.info) - initvars = init_type.variables - variables.extend(initvars) + fallback = info.metaclass_type or named_type("builtins.type") + if init_index < new_index: + method: FuncBase | Decorator = init_method.node + is_new = False + elif init_index > new_index: + method = new_method.node + is_new = True + else: + if init_method.node.info.fullname == "builtins.object": + # Both are defined by object. But if we've got a bogus + # base class, we can't know for sure, so check for that. + if info.fallback_to_any: + # Construct a universal callable as the prototype. + any_type = AnyType(TypeOfAny.special_form) + sig = CallableType( + arg_types=[any_type, any_type], + arg_kinds=[ARG_STAR, ARG_STAR2], + arg_names=["_args", "_kwds"], + ret_type=any_type, + fallback=named_type("builtins.function"), + ) + return class_callable(sig, info, fallback, None, is_new=False) - c = Callable(init_type.arg_types, - init_type.arg_kinds, - init_type.arg_names, - self_type(info), - type_type, - None, - variables).with_name('"{}"'.format(info.name())) - return convert_class_tvars_to_func_tvars(c, len(initvars)) + # Otherwise prefer __init__ in a tie. It isn't clear that this + # is the right thing, but __new__ caused problems with + # typeshed (#5647). + method = init_method.node + is_new = False + # Construct callable type based on signature of __init__. Adjust + # return type and insert type arguments. + if isinstance(method, FuncBase): + t = function_type(method, fallback) + else: + assert isinstance(method.type, ProperType) + assert isinstance(method.type, FunctionLike) # is_valid_constructor() ensures this + t = method.type + return type_object_type_from_function(t, info, method.info, fallback, is_new) -def convert_class_tvars_to_func_tvars(callable: Callable, - num_func_tvars: int) -> Callable: - return cast(Callable, callable.accept(TvarTranslator(num_func_tvars))) +def analyze_decorator_or_funcbase_access( + defn: Decorator | FuncBase, + itype: Instance, + info: TypeInfo, + self_type: Type | None, + name: str, + mx: MemberContext, +) -> Type: + """Analyzes the type behind method access. + The function itself can possibly be decorated. + See: https://github.com/python/mypy/issues/10409 + """ + if isinstance(defn, Decorator): + return analyze_var(name, defn.var, itype, info, mx) + return bind_self( + function_type(defn, mx.chk.named_type("builtins.function")), original_type=self_type + ) -class TvarTranslator(TypeTranslator): - def __init__(self, num_func_tvars: int) -> None: - super().__init__() - self.num_func_tvars = num_func_tvars - def visit_type_var(self, t: TypeVar) -> Type: - if t.id < 0: - return t - else: - return TypeVar(t.name, -t.id - self.num_func_tvars, t.values, t.upper_bound) - - def translate_variables(self, - variables: List[TypeVarDef]) -> List[TypeVarDef]: - if not variables: - return variables - items = [] # type: List[TypeVarDef] - for v in variables: - if v.id > 0: - items.append(TypeVarDef(v.name, -v.id - self.num_func_tvars, - v.values, v.upper_bound)) - else: - items.append(v) - return items +def is_valid_constructor(n: SymbolNode | None) -> bool: + """Does this node represents a valid constructor method? + + This includes normal functions, overloaded functions, and decorators + that return a callable type. + """ + if isinstance(n, FuncBase): + return True + if isinstance(n, Decorator): + return isinstance(get_proper_type(n.type), FunctionLike) + return False diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py new file mode 100644 index 000000000000..43f42039b199 --- /dev/null +++ b/mypy/checkpattern.py @@ -0,0 +1,823 @@ +"""Pattern checker. This file is conceptually part of TypeChecker.""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Final, NamedTuple + +import mypy.checker +from mypy import message_registry +from mypy.checkmember import analyze_member_access +from mypy.expandtype import expand_type_by_instance +from mypy.join import join_types +from mypy.literals import literal_hash +from mypy.maptype import map_instance_to_supertype +from mypy.meet import narrow_declared_type +from mypy.messages import MessageBuilder +from mypy.nodes import ARG_POS, Context, Expression, NameExpr, TypeAlias, TypeInfo, Var +from mypy.options import Options +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + Pattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, +) +from mypy.plugin import Plugin +from mypy.subtypes import is_subtype +from mypy.typeops import ( + coerce_to_literal, + make_simplified_union, + try_getting_str_literals_from_type, + tuple_fallback, +) +from mypy.types import ( + AnyType, + Instance, + LiteralType, + NoneType, + ProperType, + TupleType, + Type, + TypedDictType, + TypeOfAny, + TypeVarTupleType, + TypeVarType, + UninhabitedType, + UnionType, + UnpackType, + find_unpack_in_list, + get_proper_type, + split_with_prefix_and_suffix, +) +from mypy.typevars import fill_typevars +from mypy.visitor import PatternVisitor + +self_match_type_names: Final = [ + "builtins.bool", + "builtins.bytearray", + "builtins.bytes", + "builtins.dict", + "builtins.float", + "builtins.frozenset", + "builtins.int", + "builtins.list", + "builtins.set", + "builtins.str", + "builtins.tuple", +] + +non_sequence_match_type_names: Final = ["builtins.str", "builtins.bytes", "builtins.bytearray"] + + +# For every Pattern a PatternType can be calculated. This requires recursively calculating +# the PatternTypes of the sub-patterns first. +# Using the data in the PatternType the match subject and captured names can be narrowed/inferred. +class PatternType(NamedTuple): + type: Type # The type the match subject can be narrowed to + rest_type: Type # The remaining type if the pattern didn't match + captures: dict[Expression, Type] # The variables captured by the pattern + + +class PatternChecker(PatternVisitor[PatternType]): + """Pattern checker. + + This class checks if a pattern can match a type, what the type can be narrowed to, and what + type capture patterns should be inferred as. + """ + + # Some services are provided by a TypeChecker instance. + chk: mypy.checker.TypeChecker + # This is shared with TypeChecker, but stored also here for convenience. + msg: MessageBuilder + # Currently unused + plugin: Plugin + # The expression being matched against the pattern + subject: Expression + + subject_type: Type + # Type of the subject to check the (sub)pattern against + type_context: list[Type] + # Types that match against self instead of their __match_args__ if used as a class pattern + # Filled in from self_match_type_names + self_match_types: list[Type] + # Types that are sequences, but don't match sequence patterns. Filled in from + # non_sequence_match_type_names + non_sequence_match_types: list[Type] + + options: Options + + def __init__( + self, chk: mypy.checker.TypeChecker, msg: MessageBuilder, plugin: Plugin, options: Options + ) -> None: + self.chk = chk + self.msg = msg + self.plugin = plugin + + self.type_context = [] + self.self_match_types = self.generate_types_from_names(self_match_type_names) + self.non_sequence_match_types = self.generate_types_from_names( + non_sequence_match_type_names + ) + self.options = options + + def accept(self, o: Pattern, type_context: Type) -> PatternType: + self.type_context.append(type_context) + result = o.accept(self) + self.type_context.pop() + + return result + + def visit_as_pattern(self, o: AsPattern) -> PatternType: + current_type = self.type_context[-1] + if o.pattern is not None: + pattern_type = self.accept(o.pattern, current_type) + typ, rest_type, type_map = pattern_type + else: + typ, rest_type, type_map = current_type, UninhabitedType(), {} + + if not is_uninhabited(typ) and o.name is not None: + typ, _ = self.chk.conditional_types_with_intersection( + current_type, [get_type_range(typ)], o, default=current_type + ) + if not is_uninhabited(typ): + type_map[o.name] = typ + + return PatternType(typ, rest_type, type_map) + + def visit_or_pattern(self, o: OrPattern) -> PatternType: + current_type = self.type_context[-1] + + # + # Check all the subpatterns + # + pattern_types = [] + for pattern in o.patterns: + pattern_type = self.accept(pattern, current_type) + pattern_types.append(pattern_type) + if not is_uninhabited(pattern_type.type): + current_type = pattern_type.rest_type + + # + # Collect the final type + # + types = [] + for pattern_type in pattern_types: + if not is_uninhabited(pattern_type.type): + types.append(pattern_type.type) + + # + # Check the capture types + # + capture_types: dict[Var, list[tuple[Expression, Type]]] = defaultdict(list) + # Collect captures from the first subpattern + for expr, typ in pattern_types[0].captures.items(): + node = get_var(expr) + capture_types[node].append((expr, typ)) + + # Check if other subpatterns capture the same names + for i, pattern_type in enumerate(pattern_types[1:]): + vars = {get_var(expr) for expr, _ in pattern_type.captures.items()} + if capture_types.keys() != vars: + self.msg.fail(message_registry.OR_PATTERN_ALTERNATIVE_NAMES, o.patterns[i]) + for expr, typ in pattern_type.captures.items(): + node = get_var(expr) + capture_types[node].append((expr, typ)) + + captures: dict[Expression, Type] = {} + for capture_list in capture_types.values(): + typ = UninhabitedType() + for _, other in capture_list: + typ = join_types(typ, other) + + captures[capture_list[0][0]] = typ + + union_type = make_simplified_union(types) + return PatternType(union_type, current_type, captures) + + def visit_value_pattern(self, o: ValuePattern) -> PatternType: + current_type = self.type_context[-1] + typ = self.chk.expr_checker.accept(o.expr) + typ = coerce_to_literal(typ) + narrowed_type, rest_type = self.chk.conditional_types_with_intersection( + current_type, [get_type_range(typ)], o, default=get_proper_type(typ) + ) + if not isinstance(get_proper_type(narrowed_type), (LiteralType, UninhabitedType)): + return PatternType(narrowed_type, UnionType.make_union([narrowed_type, rest_type]), {}) + return PatternType(narrowed_type, rest_type, {}) + + def visit_singleton_pattern(self, o: SingletonPattern) -> PatternType: + current_type = self.type_context[-1] + value: bool | None = o.value + if isinstance(value, bool): + typ = self.chk.expr_checker.infer_literal_expr_type(value, "builtins.bool") + elif value is None: + typ = NoneType() + else: + assert False + + narrowed_type, rest_type = self.chk.conditional_types_with_intersection( + current_type, [get_type_range(typ)], o, default=current_type + ) + return PatternType(narrowed_type, rest_type, {}) + + def visit_sequence_pattern(self, o: SequencePattern) -> PatternType: + # + # check for existence of a starred pattern + # + current_type = get_proper_type(self.type_context[-1]) + if not self.can_match_sequence(current_type): + return self.early_non_match() + star_positions = [i for i, p in enumerate(o.patterns) if isinstance(p, StarredPattern)] + star_position: int | None = None + if len(star_positions) == 1: + star_position = star_positions[0] + elif len(star_positions) >= 2: + assert False, "Parser should prevent multiple starred patterns" + required_patterns = len(o.patterns) + if star_position is not None: + required_patterns -= 1 + + # + # get inner types of original type + # + unpack_index = None + if isinstance(current_type, TupleType): + inner_types = current_type.items + unpack_index = find_unpack_in_list(inner_types) + if unpack_index is None: + size_diff = len(inner_types) - required_patterns + if size_diff < 0: + return self.early_non_match() + elif size_diff > 0 and star_position is None: + return self.early_non_match() + else: + normalized_inner_types = [] + for it in inner_types: + # Unfortunately, it is not possible to "split" the TypeVarTuple + # into individual items, so we just use its upper bound for the whole + # analysis instead. + if isinstance(it, UnpackType) and isinstance(it.type, TypeVarTupleType): + it = UnpackType(it.type.upper_bound) + normalized_inner_types.append(it) + inner_types = normalized_inner_types + current_type = current_type.copy_modified(items=normalized_inner_types) + if len(inner_types) - 1 > required_patterns and star_position is None: + return self.early_non_match() + else: + inner_type = self.get_sequence_type(current_type, o) + if inner_type is None: + inner_type = self.chk.named_type("builtins.object") + inner_types = [inner_type] * len(o.patterns) + + # + # match inner patterns + # + contracted_new_inner_types: list[Type] = [] + contracted_rest_inner_types: list[Type] = [] + captures: dict[Expression, Type] = {} + + contracted_inner_types = self.contract_starred_pattern_types( + inner_types, star_position, required_patterns + ) + for p, t in zip(o.patterns, contracted_inner_types): + pattern_type = self.accept(p, t) + typ, rest, type_map = pattern_type + contracted_new_inner_types.append(typ) + contracted_rest_inner_types.append(rest) + self.update_type_map(captures, type_map) + + new_inner_types = self.expand_starred_pattern_types( + contracted_new_inner_types, star_position, len(inner_types), unpack_index is not None + ) + rest_inner_types = self.expand_starred_pattern_types( + contracted_rest_inner_types, star_position, len(inner_types), unpack_index is not None + ) + + # + # Calculate new type + # + new_type: Type + rest_type: Type = current_type + if isinstance(current_type, TupleType) and unpack_index is None: + narrowed_inner_types = [] + inner_rest_types = [] + for inner_type, new_inner_type in zip(inner_types, new_inner_types): + (narrowed_inner_type, inner_rest_type) = ( + self.chk.conditional_types_with_intersection( + inner_type, [get_type_range(new_inner_type)], o, default=inner_type + ) + ) + narrowed_inner_types.append(narrowed_inner_type) + inner_rest_types.append(inner_rest_type) + if all(not is_uninhabited(typ) for typ in narrowed_inner_types): + new_type = TupleType(narrowed_inner_types, current_type.partial_fallback) + else: + new_type = UninhabitedType() + + if all(is_uninhabited(typ) for typ in inner_rest_types): + # All subpatterns always match, so we can apply negative narrowing + rest_type = TupleType(rest_inner_types, current_type.partial_fallback) + elif sum(not is_uninhabited(typ) for typ in inner_rest_types) == 1: + # Exactly one subpattern may conditionally match, the rest always match. + # We can apply negative narrowing to this one position. + rest_type = TupleType( + [ + curr if is_uninhabited(rest) else rest + for curr, rest in zip(inner_types, inner_rest_types) + ], + current_type.partial_fallback, + ) + elif isinstance(current_type, TupleType): + # For variadic tuples it is too tricky to match individual items like for fixed + # tuples, so we instead try to narrow the entire type. + # TODO: use more precise narrowing when possible (e.g. for identical shapes). + new_tuple_type = TupleType(new_inner_types, current_type.partial_fallback) + new_type, rest_type = self.chk.conditional_types_with_intersection( + new_tuple_type, [get_type_range(current_type)], o, default=new_tuple_type + ) + else: + new_inner_type = UninhabitedType() + for typ in new_inner_types: + new_inner_type = join_types(new_inner_type, typ) + if isinstance(current_type, TypeVarType): + new_bound = self.narrow_sequence_child(current_type.upper_bound, new_inner_type, o) + new_type = current_type.copy_modified(upper_bound=new_bound) + else: + new_type = self.narrow_sequence_child(current_type, new_inner_type, o) + return PatternType(new_type, rest_type, captures) + + def get_sequence_type(self, t: Type, context: Context) -> Type | None: + t = get_proper_type(t) + if isinstance(t, AnyType): + return AnyType(TypeOfAny.from_another_any, t) + if isinstance(t, UnionType): + items = [self.get_sequence_type(item, context) for item in t.items] + not_none_items = [item for item in items if item is not None] + if not_none_items: + return make_simplified_union(not_none_items) + else: + return None + + if self.chk.type_is_iterable(t) and isinstance(t, (Instance, TupleType)): + if isinstance(t, TupleType): + t = tuple_fallback(t) + return self.chk.iterable_item_type(t, context) + else: + return None + + def contract_starred_pattern_types( + self, types: list[Type], star_pos: int | None, num_patterns: int + ) -> list[Type]: + """ + Contracts a list of types in a sequence pattern depending on the position of a starred + capture pattern. + + For example if the sequence pattern [a, *b, c] is matched against types [bool, int, str, + bytes] the contracted types are [bool, Union[int, str], bytes]. + + If star_pos in None the types are returned unchanged. + """ + unpack_index = find_unpack_in_list(types) + if unpack_index is not None: + # Variadic tuples require "re-shaping" to match the requested pattern. + unpack = types[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + # This should be guaranteed by the normalization in the caller. + assert isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + if star_pos is None: + missing = num_patterns - len(types) + 1 + new_types = types[:unpack_index] + new_types += [unpacked.args[0]] * missing + new_types += types[unpack_index + 1 :] + return new_types + prefix, middle, suffix = split_with_prefix_and_suffix( + tuple([UnpackType(unpacked) if isinstance(t, UnpackType) else t for t in types]), + star_pos, + num_patterns - star_pos, + ) + new_middle = [] + for m in middle: + # The existing code expects the star item type, rather than the type of + # the whole tuple "slice". + if isinstance(m, UnpackType): + new_middle.append(unpacked.args[0]) + else: + new_middle.append(m) + return list(prefix) + [make_simplified_union(new_middle)] + list(suffix) + else: + if star_pos is None: + return types + new_types = types[:star_pos] + star_length = len(types) - num_patterns + new_types.append(make_simplified_union(types[star_pos : star_pos + star_length])) + new_types += types[star_pos + star_length :] + return new_types + + def expand_starred_pattern_types( + self, types: list[Type], star_pos: int | None, num_types: int, original_unpack: bool + ) -> list[Type]: + """Undoes the contraction done by contract_starred_pattern_types. + + For example if the sequence pattern is [a, *b, c] and types [bool, int, str] are extended + to length 4 the result is [bool, int, int, str]. + """ + if star_pos is None: + return types + if original_unpack: + # In the case where original tuple type has an unpack item, it is not practical + # to coerce pattern type back to the original shape (and may not even be possible), + # so we only restore the type of the star item. + res = [] + for i, t in enumerate(types): + if i != star_pos: + res.append(t) + else: + res.append(UnpackType(self.chk.named_generic_type("builtins.tuple", [t]))) + return res + new_types = types[:star_pos] + star_length = num_types - len(types) + 1 + new_types += [types[star_pos]] * star_length + new_types += types[star_pos + 1 :] + + return new_types + + def narrow_sequence_child(self, outer_type: Type, inner_type: Type, ctx: Context) -> Type: + new_type = self.construct_sequence_child(outer_type, inner_type) + if is_subtype(new_type, outer_type): + new_type, _ = self.chk.conditional_types_with_intersection( + outer_type, [get_type_range(new_type)], ctx, default=outer_type + ) + else: + new_type = outer_type + return new_type + + def visit_starred_pattern(self, o: StarredPattern) -> PatternType: + captures: dict[Expression, Type] = {} + if o.capture is not None: + list_type = self.chk.named_generic_type("builtins.list", [self.type_context[-1]]) + captures[o.capture] = list_type + return PatternType(self.type_context[-1], UninhabitedType(), captures) + + def visit_mapping_pattern(self, o: MappingPattern) -> PatternType: + current_type = get_proper_type(self.type_context[-1]) + can_match = True + captures: dict[Expression, Type] = {} + for key, value in zip(o.keys, o.values): + inner_type = self.get_mapping_item_type(o, current_type, key) + if inner_type is None: + can_match = False + inner_type = self.chk.named_type("builtins.object") + pattern_type = self.accept(value, inner_type) + if is_uninhabited(pattern_type.type): + can_match = False + else: + self.update_type_map(captures, pattern_type.captures) + + if o.rest is not None: + mapping = self.chk.named_type("typing.Mapping") + if is_subtype(current_type, mapping) and isinstance(current_type, Instance): + mapping_inst = map_instance_to_supertype(current_type, mapping.type) + dict_typeinfo = self.chk.lookup_typeinfo("builtins.dict") + rest_type = Instance(dict_typeinfo, mapping_inst.args) + else: + object_type = self.chk.named_type("builtins.object") + rest_type = self.chk.named_generic_type( + "builtins.dict", [object_type, object_type] + ) + + captures[o.rest] = rest_type + + if can_match: + # We can't narrow the type here, as Mapping key is invariant. + new_type = self.type_context[-1] + else: + new_type = UninhabitedType() + return PatternType(new_type, current_type, captures) + + def get_mapping_item_type( + self, pattern: MappingPattern, mapping_type: Type, key: Expression + ) -> Type | None: + mapping_type = get_proper_type(mapping_type) + if isinstance(mapping_type, TypedDictType): + with self.msg.filter_errors() as local_errors: + result: Type | None = self.chk.expr_checker.visit_typeddict_index_expr( + mapping_type, key + )[0] + has_local_errors = local_errors.has_new_errors() + # If we can't determine the type statically fall back to treating it as a normal + # mapping + if has_local_errors: + with self.msg.filter_errors() as local_errors: + result = self.get_simple_mapping_item_type(pattern, mapping_type, key) + + if local_errors.has_new_errors(): + result = None + else: + with self.msg.filter_errors(): + result = self.get_simple_mapping_item_type(pattern, mapping_type, key) + return result + + def get_simple_mapping_item_type( + self, pattern: MappingPattern, mapping_type: Type, key: Expression + ) -> Type: + result, _ = self.chk.expr_checker.check_method_call_by_name( + "__getitem__", mapping_type, [key], [ARG_POS], pattern + ) + return result + + def visit_class_pattern(self, o: ClassPattern) -> PatternType: + current_type = get_proper_type(self.type_context[-1]) + + # + # Check class type + # + type_info = o.class_ref.node + if type_info is None: + return PatternType(AnyType(TypeOfAny.from_error), AnyType(TypeOfAny.from_error), {}) + if isinstance(type_info, TypeAlias) and not type_info.no_args: + self.msg.fail(message_registry.CLASS_PATTERN_GENERIC_TYPE_ALIAS, o) + return self.early_non_match() + if isinstance(type_info, TypeInfo): + any_type = AnyType(TypeOfAny.implementation_artifact) + args: list[Type] = [] + for tv in type_info.defn.type_vars: + if isinstance(tv, TypeVarTupleType): + args.append( + UnpackType(self.chk.named_generic_type("builtins.tuple", [any_type])) + ) + else: + args.append(any_type) + typ: Type = Instance(type_info, args) + elif isinstance(type_info, TypeAlias): + typ = type_info.target + elif ( + isinstance(type_info, Var) + and type_info.type is not None + and isinstance(get_proper_type(type_info.type), AnyType) + ): + typ = type_info.type + else: + if isinstance(type_info, Var) and type_info.type is not None: + name = type_info.type.str_with_options(self.options) + else: + name = type_info.name + self.msg.fail(message_registry.CLASS_PATTERN_TYPE_REQUIRED.format(name), o) + return self.early_non_match() + + new_type, rest_type = self.chk.conditional_types_with_intersection( + current_type, [get_type_range(typ)], o, default=current_type + ) + if is_uninhabited(new_type): + return self.early_non_match() + # TODO: Do I need this? + narrowed_type = narrow_declared_type(current_type, new_type) + + # + # Convert positional to keyword patterns + # + keyword_pairs: list[tuple[str | None, Pattern]] = [] + match_arg_set: set[str] = set() + + captures: dict[Expression, Type] = {} + + if len(o.positionals) != 0: + if self.should_self_match(typ): + if len(o.positionals) > 1: + self.msg.fail(message_registry.CLASS_PATTERN_TOO_MANY_POSITIONAL_ARGS, o) + pattern_type = self.accept(o.positionals[0], narrowed_type) + if not is_uninhabited(pattern_type.type): + return PatternType( + pattern_type.type, + join_types(rest_type, pattern_type.rest_type), + pattern_type.captures, + ) + captures = pattern_type.captures + else: + with self.msg.filter_errors() as local_errors: + match_args_type = analyze_member_access( + "__match_args__", + typ, + o, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, + original_type=typ, + chk=self.chk, + ) + has_local_errors = local_errors.has_new_errors() + if has_local_errors: + self.msg.fail( + message_registry.MISSING_MATCH_ARGS.format( + typ.str_with_options(self.options) + ), + o, + ) + return self.early_non_match() + + proper_match_args_type = get_proper_type(match_args_type) + if isinstance(proper_match_args_type, TupleType): + match_arg_names = get_match_arg_names(proper_match_args_type) + + if len(o.positionals) > len(match_arg_names): + self.msg.fail(message_registry.CLASS_PATTERN_TOO_MANY_POSITIONAL_ARGS, o) + return self.early_non_match() + else: + match_arg_names = [None] * len(o.positionals) + + for arg_name, pos in zip(match_arg_names, o.positionals): + keyword_pairs.append((arg_name, pos)) + if arg_name is not None: + match_arg_set.add(arg_name) + + # + # Check for duplicate patterns + # + keyword_arg_set = set() + has_duplicates = False + for key, value in zip(o.keyword_keys, o.keyword_values): + keyword_pairs.append((key, value)) + if key in match_arg_set: + self.msg.fail( + message_registry.CLASS_PATTERN_KEYWORD_MATCHES_POSITIONAL.format(key), value + ) + has_duplicates = True + elif key in keyword_arg_set: + self.msg.fail( + message_registry.CLASS_PATTERN_DUPLICATE_KEYWORD_PATTERN.format(key), value + ) + has_duplicates = True + keyword_arg_set.add(key) + + if has_duplicates: + return self.early_non_match() + + # + # Check keyword patterns + # + can_match = True + for keyword, pattern in keyword_pairs: + key_type: Type | None = None + with self.msg.filter_errors() as local_errors: + if keyword is not None: + key_type = analyze_member_access( + keyword, + narrowed_type, + pattern, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, + original_type=new_type, + chk=self.chk, + ) + else: + key_type = AnyType(TypeOfAny.from_error) + has_local_errors = local_errors.has_new_errors() + if has_local_errors or key_type is None: + key_type = AnyType(TypeOfAny.from_error) + self.msg.fail( + message_registry.CLASS_PATTERN_UNKNOWN_KEYWORD.format( + typ.str_with_options(self.options), keyword + ), + pattern, + ) + + inner_type, inner_rest_type, inner_captures = self.accept(pattern, key_type) + if is_uninhabited(inner_type): + can_match = False + else: + self.update_type_map(captures, inner_captures) + if not is_uninhabited(inner_rest_type): + rest_type = current_type + + if not can_match: + new_type = UninhabitedType() + return PatternType(new_type, rest_type, captures) + + def should_self_match(self, typ: Type) -> bool: + typ = get_proper_type(typ) + if isinstance(typ, Instance) and typ.type.get("__match_args__") is not None: + # Named tuples and other subtypes of builtins that define __match_args__ + # should not self match. + return False + for other in self.self_match_types: + if is_subtype(typ, other): + return True + return False + + def can_match_sequence(self, typ: ProperType) -> bool: + if isinstance(typ, UnionType): + return any(self.can_match_sequence(get_proper_type(item)) for item in typ.items) + for other in self.non_sequence_match_types: + # We have to ignore promotions, as memoryview should match, but bytes, + # which it can be promoted to, shouldn't + if is_subtype(typ, other, ignore_promotions=True): + return False + sequence = self.chk.named_type("typing.Sequence") + # If the static type is more general than sequence the actual type could still match + return is_subtype(typ, sequence) or is_subtype(sequence, typ) + + def generate_types_from_names(self, type_names: list[str]) -> list[Type]: + types: list[Type] = [] + for name in type_names: + try: + types.append(self.chk.named_type(name)) + except KeyError as e: + # Some built in types are not defined in all test cases + if not name.startswith("builtins."): + raise e + return types + + def update_type_map( + self, original_type_map: dict[Expression, Type], extra_type_map: dict[Expression, Type] + ) -> None: + # Calculating this would not be needed if TypeMap directly used literal hashes instead of + # expressions, as suggested in the TODO above it's definition + already_captured = {literal_hash(expr) for expr in original_type_map} + for expr, typ in extra_type_map.items(): + if literal_hash(expr) in already_captured: + node = get_var(expr) + self.msg.fail( + message_registry.MULTIPLE_ASSIGNMENTS_IN_PATTERN.format(node.name), expr + ) + else: + original_type_map[expr] = typ + + def construct_sequence_child(self, outer_type: Type, inner_type: Type) -> Type: + """ + If outer_type is a child class of typing.Sequence returns a new instance of + outer_type, that is a Sequence of inner_type. If outer_type is not a child class of + typing.Sequence just returns a Sequence of inner_type + + For example: + construct_sequence_child(List[int], str) = List[str] + + TODO: this doesn't make sense. For example if one has class S(Sequence[int], Generic[T]) + or class T(Sequence[Tuple[T, T]]), there is no way any of those can map to Sequence[str]. + """ + proper_type = get_proper_type(outer_type) + if isinstance(proper_type, UnionType): + types = [ + self.construct_sequence_child(item, inner_type) + for item in proper_type.items + if self.can_match_sequence(get_proper_type(item)) + ] + return make_simplified_union(types) + sequence = self.chk.named_generic_type("typing.Sequence", [inner_type]) + if is_subtype(outer_type, self.chk.named_type("typing.Sequence")): + proper_type = get_proper_type(outer_type) + if isinstance(proper_type, TupleType): + proper_type = tuple_fallback(proper_type) + assert isinstance(proper_type, Instance) + empty_type = fill_typevars(proper_type.type) + partial_type = expand_type_by_instance(empty_type, sequence) + return expand_type_by_instance(partial_type, proper_type) + else: + return sequence + + def early_non_match(self) -> PatternType: + return PatternType(UninhabitedType(), self.type_context[-1], {}) + + +def get_match_arg_names(typ: TupleType) -> list[str | None]: + args: list[str | None] = [] + for item in typ.items: + values = try_getting_str_literals_from_type(item) + if values is None or len(values) != 1: + args.append(None) + else: + args.append(values[0]) + return args + + +def get_var(expr: Expression) -> Var: + """ + Warning: this in only true for expressions captured by a match statement. + Don't call it from anywhere else + """ + assert isinstance(expr, NameExpr) + node = expr.node + assert isinstance(node, Var) + return node + + +def get_type_range(typ: Type) -> mypy.checker.TypeRange: + typ = get_proper_type(typ) + if ( + isinstance(typ, Instance) + and typ.last_known_value + and isinstance(typ.last_known_value.value, bool) + ): + typ = typ.last_known_value + return mypy.checker.TypeRange(typ, is_upper_bound=False) + + +def is_uninhabited(typ: Type) -> bool: + return isinstance(get_proper_type(typ), UninhabitedType) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index 3459bef31665..dd42fe7755a0 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -1,93 +1,714 @@ -"""Expression type checker. This file is conceptually part of ExpressionChecker and TypeChecker.""" +""" +Format expression type checker. -import re +This file is conceptually part of ExpressionChecker and TypeChecker. Main functionality +is located in StringFormatterChecker.check_str_format_call() for '{}'.format(), and in +StringFormatterChecker.check_str_interpolation() for printf-style % interpolation. -from typing import Undefined, cast, List, Tuple, Dict, Function +Note that although at runtime format strings are parsed using custom parsers, +here we use a regexp-based approach. This way we 99% match runtime behaviour while keeping +implementation simple. +""" -from mypy.types import ( - Type, AnyType, TupleType, Instance, UnionType -) +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Callable, Dict, Final, Match, Pattern, Tuple, Union, cast +from typing_extensions import TypeAlias as _TypeAlias + +import mypy.errorcodes as codes +from mypy.errors import Errors from mypy.nodes import ( - Node, StrExpr, TupleExpr, DictExpr, Context + ARG_NAMED, + ARG_POS, + ARG_STAR, + ARG_STAR2, + BytesExpr, + CallExpr, + Context, + DictExpr, + Expression, + ExpressionStmt, + IndexExpr, + IntExpr, + MemberExpr, + MypyFile, + NameExpr, + Node, + StarExpr, + StrExpr, + TempNode, + TupleExpr, +) +from mypy.types import ( + AnyType, + Instance, + LiteralType, + TupleType, + Type, + TypeOfAny, + TypeVarTupleType, + TypeVarType, + UnionType, + UnpackType, + find_unpack_in_list, + get_proper_type, + get_proper_types, ) -import mypy.checker -from mypy import messages + +if TYPE_CHECKING: + # break import cycle only needed for mypy + import mypy.checker + import mypy.checkexpr + +from mypy import message_registry +from mypy.maptype import map_instance_to_supertype from mypy.messages import MessageBuilder +from mypy.parse import parse +from mypy.subtypes import is_subtype +from mypy.typeops import custom_special_method + +FormatStringExpr: _TypeAlias = Union[StrExpr, BytesExpr] +Checkers: _TypeAlias = Tuple[Callable[[Expression], None], Callable[[Type], bool]] +MatchMap: _TypeAlias = Dict[Tuple[int, int], Match[str]] # span -> match + + +def compile_format_re() -> Pattern[str]: + """Construct regexp to match format conversion specifiers in % interpolation. + + See https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting + The regexp is intentionally a bit wider to report better errors. + """ + key_re = r"(\((?P[^)]*)\))?" # (optional) parenthesised sequence of characters. + flags_re = r"(?P[#0\-+ ]*)" # (optional) sequence of flags. + width_re = r"(?P[1-9][0-9]*|\*)?" # (optional) minimum field width (* or numbers). + precision_re = r"(?:\.(?P\*|[0-9]+)?)?" # (optional) . followed by * of numbers. + length_mod_re = r"[hlL]?" # (optional) length modifier (unused). + type_re = r"(?P.)?" # conversion type. + format_re = "%" + key_re + flags_re + width_re + precision_re + length_mod_re + type_re + return re.compile(format_re) + + +def compile_new_format_re(custom_spec: bool) -> Pattern[str]: + """Construct regexps to match format conversion specifiers in str.format() calls. + + See After https://docs.python.org/3/library/string.html#formatspec for + specifications. The regexps are intentionally wider, to report better errors, + instead of just not matching. + """ + + # Field (optional) is an integer/identifier possibly followed by several .attr and [index]. + field = r"(?P(?P[^.[!:]*)([^:!]+)?)" + + # Conversion (optional) is ! followed by one of letters for forced repr(), str(), or ascii(). + conversion = r"(?P![^:])?" + + # Format specification (optional) follows its own mini-language: + if not custom_spec: + # Fill and align is valid for all builtin types. + fill_align = r"(?P.?[<>=^])?" + # Number formatting options are only valid for int, float, complex, and Decimal, + # except if only width is given (it is valid for all types). + # This contains sign, flags (sign, # and/or 0), width, grouping (_ or ,) and precision. + num_spec = r"(?P[+\- ]?#?0?)(?P\d+)?[_,]?(?P\.\d+)?" + # The last element is type. + conv_type = r"(?P.)?" # only some are supported, but we want to give a better error + format_spec = r"(?P:" + fill_align + num_spec + conv_type + r")?" + else: + # Custom types can define their own form_spec using __format__(). + format_spec = r"(?P:.*)?" + + return re.compile(field + conversion + format_spec) + + +FORMAT_RE: Final = compile_format_re() +FORMAT_RE_NEW: Final = compile_new_format_re(False) +FORMAT_RE_NEW_CUSTOM: Final = compile_new_format_re(True) +DUMMY_FIELD_NAME: Final = "__dummy_name__" + +# Types that require either int or float. +NUMERIC_TYPES_OLD: Final = {"d", "i", "o", "u", "x", "X", "e", "E", "f", "F", "g", "G"} +NUMERIC_TYPES_NEW: Final = {"b", "d", "o", "e", "E", "f", "F", "g", "G", "n", "x", "X", "%"} + +# These types accept _only_ int. +REQUIRE_INT_OLD: Final = {"o", "x", "X"} +REQUIRE_INT_NEW: Final = {"b", "d", "o", "x", "X"} + +# These types fall back to SupportsFloat with % (other fall back to SupportsInt) +FLOAT_TYPES: Final = {"e", "E", "f", "F", "g", "G"} class ConversionSpecifier: - def __init__(self, key: str, flags: str, width: str, precision: str, type: str) -> None: - self.key = key - self.flags = flags - self.width = width - self.precision = precision - self.type = type + def __init__( + self, match: Match[str], start_pos: int = -1, non_standard_format_spec: bool = False + ) -> None: + self.whole_seq = match.group() + self.start_pos = start_pos + + m_dict = match.groupdict() + self.key = m_dict.get("key") - def has_key(self): - return self.key != None + # Replace unmatched optional groups with empty matches (for convenience). + self.conv_type = m_dict.get("type", "") + self.flags = m_dict.get("flags", "") + self.width = m_dict.get("width", "") + self.precision = m_dict.get("precision", "") - def has_star(self): - return self.width == '*' or self.precision == '*' + # Used only for str.format() calls (it may be custom for types with __format__()). + self.format_spec = m_dict.get("format_spec") + self.non_standard_format_spec = non_standard_format_spec + # Used only for str.format() calls. + self.conversion = m_dict.get("conversion") + # Full formatted expression (i.e. key plus following attributes and/or indexes). + # Used only for str.format() calls. + self.field = m_dict.get("field") + + def has_key(self) -> bool: + return self.key is not None + + def has_star(self) -> bool: + return self.width == "*" or self.precision == "*" + + +def parse_conversion_specifiers(format_str: str) -> list[ConversionSpecifier]: + """Parse c-printf-style format string into list of conversion specifiers.""" + specifiers: list[ConversionSpecifier] = [] + for m in re.finditer(FORMAT_RE, format_str): + specifiers.append(ConversionSpecifier(m, start_pos=m.start())) + return specifiers + + +def parse_format_value( + format_value: str, ctx: Context, msg: MessageBuilder, nested: bool = False +) -> list[ConversionSpecifier] | None: + """Parse format string into list of conversion specifiers. + + The specifiers may be nested (two levels maximum), in this case they are ordered as + '{0:{1}}, {2:{3}{4}}'. Return None in case of an error. + """ + top_targets = find_non_escaped_targets(format_value, ctx, msg) + if top_targets is None: + return None + + result: list[ConversionSpecifier] = [] + for target, start_pos in top_targets: + match = FORMAT_RE_NEW.fullmatch(target) + if match: + conv_spec = ConversionSpecifier(match, start_pos=start_pos) + else: + custom_match = FORMAT_RE_NEW_CUSTOM.fullmatch(target) + if custom_match: + conv_spec = ConversionSpecifier( + custom_match, start_pos=start_pos, non_standard_format_spec=True + ) + else: + msg.fail( + "Invalid conversion specifier in format string", + ctx, + code=codes.STRING_FORMATTING, + ) + return None + + if conv_spec.key and ("{" in conv_spec.key or "}" in conv_spec.key): + msg.fail("Conversion value must not contain { or }", ctx, code=codes.STRING_FORMATTING) + return None + result.append(conv_spec) + + # Parse nested conversions that are allowed in format specifier. + if ( + conv_spec.format_spec + and conv_spec.non_standard_format_spec + and ("{" in conv_spec.format_spec or "}" in conv_spec.format_spec) + ): + if nested: + msg.fail( + "Formatting nesting must be at most two levels deep", + ctx, + code=codes.STRING_FORMATTING, + ) + return None + sub_conv_specs = parse_format_value(conv_spec.format_spec, ctx, msg, nested=True) + if sub_conv_specs is None: + return None + result.extend(sub_conv_specs) + return result + + +def find_non_escaped_targets( + format_value: str, ctx: Context, msg: MessageBuilder +) -> list[tuple[str, int]] | None: + """Return list of raw (un-parsed) format specifiers in format string. + + Format specifiers don't include enclosing braces. We don't use regexp for + this because they don't work well with nested/repeated patterns + (both greedy and non-greedy), and these are heavily used internally for + representation of f-strings. + + Return None in case of an error. + """ + result = [] + next_spec = "" + pos = 0 + nesting = 0 + while pos < len(format_value): + c = format_value[pos] + if not nesting: + # Skip any paired '{{' and '}}', enter nesting on '{', report error on '}'. + if c == "{": + if pos < len(format_value) - 1 and format_value[pos + 1] == "{": + pos += 1 + else: + nesting = 1 + if c == "}": + if pos < len(format_value) - 1 and format_value[pos + 1] == "}": + pos += 1 + else: + msg.fail( + "Invalid conversion specifier in format string: unexpected }", + ctx, + code=codes.STRING_FORMATTING, + ) + return None + else: + # Adjust nesting level, then either continue adding chars or move on. + if c == "{": + nesting += 1 + if c == "}": + nesting -= 1 + if nesting: + next_spec += c + else: + result.append((next_spec, pos - len(next_spec))) + next_spec = "" + pos += 1 + if nesting: + msg.fail( + "Invalid conversion specifier in format string: unmatched {", + ctx, + code=codes.STRING_FORMATTING, + ) + return None + return result class StringFormatterChecker: - """String interplation/formatter type checker. + """String interpolation/formatter type checker. This class works closely together with checker.ExpressionChecker. """ # Some services are provided by a TypeChecker instance. - chk = Undefined('mypy.checker.TypeChecker') + chk: mypy.checker.TypeChecker # This is shared with TypeChecker, but stored also here for convenience. - msg = Undefined(MessageBuilder) + msg: MessageBuilder # Some services are provided by a ExpressionChecker instance. - exprchk = Undefined('mypy.checkexpr.ExpressionChecker') + exprchk: mypy.checkexpr.ExpressionChecker - def __init__(self, - exprchk: 'mypy.checkexpr.ExpressionChecker', - chk: 'mypy.checker.TypeChecker', - msg: MessageBuilder) -> None: + def __init__( + self, + exprchk: mypy.checkexpr.ExpressionChecker, + chk: mypy.checker.TypeChecker, + msg: MessageBuilder, + ) -> None: """Construct an expression type checker.""" self.chk = chk self.exprchk = exprchk self.msg = msg - def check_str_interpolation(self, str: StrExpr, replacements: Node) -> Type: + def check_str_format_call(self, call: CallExpr, format_value: str) -> None: + """Perform more precise checks for str.format() calls when possible. + + Currently the checks are performed for: + * Actual string literals + * Literal types with string values + * Final names with string values + + The checks that we currently perform: + * Check generic validity (e.g. unmatched { or }, and {} in invalid positions) + * Check consistency of specifiers' auto-numbering + * Verify that replacements can be found for all conversion specifiers, + and all arguments were used + * Non-standard format specs are only allowed for types with custom __format__ + * Type check replacements with accessors applied (if any). + * Verify that specifier type is known and matches replacement type + * Perform special checks for some specifier types: + - 'c' requires a single character string + - 's' must not accept bytes + - non-empty flags are only allowed for numeric types + """ + conv_specs = parse_format_value(format_value, call, self.msg) + if conv_specs is None: + return + if not self.auto_generate_keys(conv_specs, call): + return + self.check_specs_in_format_call(call, conv_specs, format_value) + + def check_specs_in_format_call( + self, call: CallExpr, specs: list[ConversionSpecifier], format_value: str + ) -> None: + """Perform pairwise checks for conversion specifiers vs their replacements. + + The core logic for format checking is implemented in this method. + """ + assert all(s.key for s in specs), "Keys must be auto-generated first!" + replacements = self.find_replacements_in_call(call, [cast(str, s.key) for s in specs]) + assert len(replacements) == len(specs) + for spec, repl in zip(specs, replacements): + repl = self.apply_field_accessors(spec, repl, ctx=call) + actual_type = repl.type if isinstance(repl, TempNode) else self.chk.lookup_type(repl) + assert actual_type is not None + + # Special case custom formatting. + if ( + spec.format_spec + and spec.non_standard_format_spec + and + # Exclude "dynamic" specifiers (i.e. containing nested formatting). + not ("{" in spec.format_spec or "}" in spec.format_spec) + ): + if ( + not custom_special_method(actual_type, "__format__", check_all=True) + or spec.conversion + ): + # TODO: add support for some custom specs like datetime? + self.msg.fail( + f'Unrecognized format specification "{spec.format_spec[1:]}"', + call, + code=codes.STRING_FORMATTING, + ) + continue + # Adjust expected and actual types. + if not spec.conv_type: + expected_type: Type | None = AnyType(TypeOfAny.special_form) + else: + assert isinstance(call.callee, MemberExpr) + if isinstance(call.callee.expr, StrExpr): + format_str = call.callee.expr + else: + format_str = StrExpr(format_value) + expected_type = self.conversion_type( + spec.conv_type, call, format_str, format_call=True + ) + if spec.conversion is not None: + # If the explicit conversion is given, then explicit conversion is called _first_. + if spec.conversion[1] not in "rsa": + self.msg.fail( + ( + f'Invalid conversion type "{spec.conversion[1]}", ' + f'must be one of "r", "s" or "a"' + ), + call, + code=codes.STRING_FORMATTING, + ) + actual_type = self.named_type("builtins.str") + + # Perform the checks for given types. + if expected_type is None: + continue + + a_type = get_proper_type(actual_type) + actual_items = ( + get_proper_types(a_type.items) if isinstance(a_type, UnionType) else [a_type] + ) + for a_type in actual_items: + if custom_special_method(a_type, "__format__"): + continue + self.check_placeholder_type(a_type, expected_type, call) + self.perform_special_format_checks(spec, call, repl, a_type, expected_type) + + def perform_special_format_checks( + self, + spec: ConversionSpecifier, + call: CallExpr, + repl: Expression, + actual_type: Type, + expected_type: Type, + ) -> None: + # TODO: try refactoring to combine this logic with % formatting. + if spec.conv_type == "c": + if isinstance(repl, (StrExpr, BytesExpr)) and len(repl.value) != 1: + self.msg.requires_int_or_char(call, format_call=True) + c_typ = get_proper_type(self.chk.lookup_type(repl)) + if isinstance(c_typ, Instance) and c_typ.last_known_value: + c_typ = c_typ.last_known_value + if isinstance(c_typ, LiteralType) and isinstance(c_typ.value, str): + if len(c_typ.value) != 1: + self.msg.requires_int_or_char(call, format_call=True) + if (not spec.conv_type or spec.conv_type == "s") and not spec.conversion: + if has_type_component(actual_type, "builtins.bytes") and not custom_special_method( + actual_type, "__str__" + ): + self.msg.fail( + 'If x = b\'abc\' then f"{x}" or "{}".format(x) produces "b\'abc\'", ' + 'not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). ' + "Otherwise, decode the bytes", + call, + code=codes.STR_BYTES_PY3, + ) + if spec.flags: + numeric_types = UnionType( + [self.named_type("builtins.int"), self.named_type("builtins.float")] + ) + if ( + spec.conv_type + and spec.conv_type not in NUMERIC_TYPES_NEW + or not spec.conv_type + and not is_subtype(actual_type, numeric_types) + and not custom_special_method(actual_type, "__format__") + ): + self.msg.fail( + "Numeric flags are only allowed for numeric types", + call, + code=codes.STRING_FORMATTING, + ) + + def find_replacements_in_call(self, call: CallExpr, keys: list[str]) -> list[Expression]: + """Find replacement expression for every specifier in str.format() call. + + In case of an error use TempNode(AnyType). + """ + result: list[Expression] = [] + used: set[Expression] = set() + for key in keys: + if key.isdecimal(): + expr = self.get_expr_by_position(int(key), call) + if not expr: + self.msg.fail( + f"Cannot find replacement for positional format specifier {key}", + call, + code=codes.STRING_FORMATTING, + ) + expr = TempNode(AnyType(TypeOfAny.from_error)) + else: + expr = self.get_expr_by_name(key, call) + if not expr: + self.msg.fail( + f'Cannot find replacement for named format specifier "{key}"', + call, + code=codes.STRING_FORMATTING, + ) + expr = TempNode(AnyType(TypeOfAny.from_error)) + result.append(expr) + if not isinstance(expr, TempNode): + used.add(expr) + # Strictly speaking not using all replacements is not a type error, but most likely + # a typo in user code, so we show an error like we do for % formatting. + total_explicit = len([kind for kind in call.arg_kinds if kind in (ARG_POS, ARG_NAMED)]) + if len(used) < total_explicit: + self.msg.too_many_string_formatting_arguments(call) + return result + + def get_expr_by_position(self, pos: int, call: CallExpr) -> Expression | None: + """Get positional replacement expression from '{0}, {1}'.format(x, y, ...) call. + + If the type is from *args, return TempNode(). Return None in case of + an error. + """ + pos_args = [arg for arg, kind in zip(call.args, call.arg_kinds) if kind == ARG_POS] + if pos < len(pos_args): + return pos_args[pos] + star_args = [arg for arg, kind in zip(call.args, call.arg_kinds) if kind == ARG_STAR] + if not star_args: + return None + + # Fall back to *args when present in call. + star_arg = star_args[0] + varargs_type = get_proper_type(self.chk.lookup_type(star_arg)) + if not isinstance(varargs_type, Instance) or not varargs_type.type.has_base( + "typing.Sequence" + ): + # Error should be already reported. + return TempNode(AnyType(TypeOfAny.special_form)) + iter_info = self.chk.named_generic_type( + "typing.Sequence", [AnyType(TypeOfAny.special_form)] + ).type + return TempNode(map_instance_to_supertype(varargs_type, iter_info).args[0]) + + def get_expr_by_name(self, key: str, call: CallExpr) -> Expression | None: + """Get named replacement expression from '{name}'.format(name=...) call. + + If the type is from **kwargs, return TempNode(). Return None in case of + an error. + """ + named_args = [ + arg + for arg, kind, name in zip(call.args, call.arg_kinds, call.arg_names) + if kind == ARG_NAMED and name == key + ] + if named_args: + return named_args[0] + star_args_2 = [arg for arg, kind in zip(call.args, call.arg_kinds) if kind == ARG_STAR2] + if not star_args_2: + return None + star_arg_2 = star_args_2[0] + kwargs_type = get_proper_type(self.chk.lookup_type(star_arg_2)) + if not isinstance(kwargs_type, Instance) or not kwargs_type.type.has_base( + "typing.Mapping" + ): + # Error should be already reported. + return TempNode(AnyType(TypeOfAny.special_form)) + any_type = AnyType(TypeOfAny.special_form) + mapping_info = self.chk.named_generic_type("typing.Mapping", [any_type, any_type]).type + return TempNode(map_instance_to_supertype(kwargs_type, mapping_info).args[1]) + + def auto_generate_keys(self, all_specs: list[ConversionSpecifier], ctx: Context) -> bool: + """Translate '{} {name} {}' to '{0} {name} {1}'. + + Return True if generation was successful, otherwise report an error and return false. + """ + some_defined = any(s.key and s.key.isdecimal() for s in all_specs) + all_defined = all(bool(s.key) for s in all_specs) + if some_defined and not all_defined: + self.msg.fail( + "Cannot combine automatic field numbering and manual field specification", + ctx, + code=codes.STRING_FORMATTING, + ) + return False + if all_defined: + return True + next_index = 0 + for spec in all_specs: + if not spec.key: + str_index = str(next_index) + spec.key = str_index + # Update also the full field (i.e. turn {.x} into {0.x}). + if not spec.field: + spec.field = str_index + else: + spec.field = str_index + spec.field + next_index += 1 + return True + + def apply_field_accessors( + self, spec: ConversionSpecifier, repl: Expression, ctx: Context + ) -> Expression: + """Transform and validate expr in '{.attr[item]}'.format(expr) into expr.attr['item']. + + If validation fails, return TempNode(AnyType). + """ + assert spec.key, "Keys must be auto-generated first!" + if spec.field == spec.key: + return repl + assert spec.field + + temp_errors = Errors(self.chk.options) + dummy = DUMMY_FIELD_NAME + spec.field[len(spec.key) :] + temp_ast: Node = parse( + dummy, fnam="", module=None, options=self.chk.options, errors=temp_errors + ) + if temp_errors.is_errors(): + self.msg.fail( + f'Syntax error in format specifier "{spec.field}"', + ctx, + code=codes.STRING_FORMATTING, + ) + return TempNode(AnyType(TypeOfAny.from_error)) + + # These asserts are guaranteed by the original regexp. + assert isinstance(temp_ast, MypyFile) + temp_ast = temp_ast.defs[0] + assert isinstance(temp_ast, ExpressionStmt) + temp_ast = temp_ast.expr + if not self.validate_and_transform_accessors(temp_ast, repl, spec, ctx=ctx): + return TempNode(AnyType(TypeOfAny.from_error)) + + # Check if there are any other errors (like missing members). + # TODO: fix column to point to actual start of the format specifier _within_ string. + temp_ast.line = ctx.line + temp_ast.column = ctx.column + self.exprchk.accept(temp_ast) + return temp_ast + + def validate_and_transform_accessors( + self, + temp_ast: Expression, + original_repl: Expression, + spec: ConversionSpecifier, + ctx: Context, + ) -> bool: + """Validate and transform (in-place) format field accessors. + + On error, report it and return False. The transformations include replacing the dummy + variable with actual replacement expression and translating any name expressions in an + index into strings, so that this will work: + + class User(TypedDict): + name: str + id: int + u: User + '{[id]:d} -> {[name]}'.format(u) + """ + if not isinstance(temp_ast, (MemberExpr, IndexExpr)): + self.msg.fail( + "Only index and member expressions are allowed in" + ' format field accessors; got "{}"'.format(spec.field), + ctx, + code=codes.STRING_FORMATTING, + ) + return False + if isinstance(temp_ast, MemberExpr): + node = temp_ast.expr + else: + node = temp_ast.base + if not isinstance(temp_ast.index, (NameExpr, IntExpr)): + assert spec.key, "Call this method only after auto-generating keys!" + assert spec.field + self.msg.fail( + 'Invalid index expression in format field accessor "{}"'.format( + spec.field[len(spec.key) :] + ), + ctx, + code=codes.STRING_FORMATTING, + ) + return False + if isinstance(temp_ast.index, NameExpr): + temp_ast.index = StrExpr(temp_ast.index.name) + if isinstance(node, NameExpr) and node.name == DUMMY_FIELD_NAME: + # Replace it with the actual replacement expression. + assert isinstance(temp_ast, (IndexExpr, MemberExpr)) # XXX: this is redundant + if isinstance(temp_ast, IndexExpr): + temp_ast.base = original_repl + else: + temp_ast.expr = original_repl + return True + node.line = ctx.line + node.column = ctx.column + return self.validate_and_transform_accessors( + node, original_repl=original_repl, spec=spec, ctx=ctx + ) + + # TODO: In Python 3, the bytes formatting has a more restricted set of options + # compared to string formatting. + def check_str_interpolation(self, expr: FormatStringExpr, replacements: Expression) -> Type: """Check the types of the 'replacements' in a string interpolation - expression: str % replacements + expression: str % replacements. """ - replacements = self.exprchk.strip_parens(replacements) - specifiers = self.parse_conversion_specifiers(str.value) - has_mapping_keys = self.analyse_conversion_specifiers(specifiers, str) - if has_mapping_keys == None: - pass # Error was reported + self.exprchk.accept(expr) + specifiers = parse_conversion_specifiers(expr.value) + has_mapping_keys = self.analyze_conversion_specifiers(specifiers, expr) + if has_mapping_keys is None: + pass # Error was reported elif has_mapping_keys: - self.check_mapping_str_interpolation(specifiers, replacements) + self.check_mapping_str_interpolation(specifiers, replacements, expr) else: - self.check_simple_str_interpolation(specifiers, replacements) - return self.named_type('builtins.str') - - def parse_conversion_specifiers(self, format: str) -> List[ConversionSpecifier]: - key_regex = r'(\((\w*)\))?' # (optional) parenthesised sequence of characters - flags_regex = r'([#0\-+ ]*)' # (optional) sequence of flags - width_regex = r'(\*|[1-9][0-9]*)?' # (optional) minimum field width (* or numbers) - precision_regex = r'(?:\.(\*|[0-9]+))?' # (optional) . followed by * of numbers - length_mod_regex = r'[hlL]?' # (optional) length modifier (unused) - type_regex = r'(.)?' # conversion type - regex = ('%' + key_regex + flags_regex + width_regex + - precision_regex + length_mod_regex + type_regex) - specifiers = [] # type: List[ConversionSpecifier] - for parens_key, key, flags, width, precision, type in re.findall(regex, format): - if parens_key == '': - key = None - specifiers.append(ConversionSpecifier(key, flags, width, precision, type)) - return specifiers - - def analyse_conversion_specifiers(self, specifiers: List[ConversionSpecifier], - context: Context) -> bool: + self.check_simple_str_interpolation(specifiers, replacements, expr) + + if isinstance(expr, BytesExpr): + return self.named_type("builtins.bytes") + elif isinstance(expr, StrExpr): + return self.named_type("builtins.str") + else: + assert False + + def analyze_conversion_specifiers( + self, specifiers: list[ConversionSpecifier], context: Context + ) -> bool | None: has_star = any(specifier.has_star() for specifier in specifiers) has_key = any(specifier.has_key() for specifier in specifiers) - all_have_keys = all(specifier.has_key() for specifier in specifiers) + all_have_keys = all( + specifier.has_key() or specifier.conv_type == "%" for specifier in specifiers + ) if has_key and has_star: self.msg.string_interpolation_with_star_and_key(context) @@ -97,30 +718,71 @@ def analyse_conversion_specifiers(self, specifiers: List[ConversionSpecifier], return None return has_key - def check_simple_str_interpolation(self, specifiers: List[ConversionSpecifier], - replacements: Node) -> None: - checkers = self.build_replacement_checkers(specifiers, replacements) - if checkers == None: + def check_simple_str_interpolation( + self, + specifiers: list[ConversionSpecifier], + replacements: Expression, + expr: FormatStringExpr, + ) -> None: + """Check % string interpolation with positional specifiers '%s, %d' % ('yes, 42').""" + checkers = self.build_replacement_checkers(specifiers, replacements, expr) + if checkers is None: return - rhs_type = self.accept(replacements) - rep_types = [] # type: List[Type] + rhs_type = get_proper_type(self.accept(replacements)) + rep_types: list[Type] = [] if isinstance(rhs_type, TupleType): rep_types = rhs_type.items + unpack_index = find_unpack_in_list(rep_types) + if unpack_index is not None: + # TODO: we should probably warn about potentially short tuple. + # However, without special-casing for tuple(f(i) for in other_tuple) + # this causes false positive on mypy self-check in report.py. + extras = max(0, len(checkers) - len(rep_types) + 1) + unpacked = rep_types[unpack_index] + assert isinstance(unpacked, UnpackType) + unpacked = get_proper_type(unpacked.type) + if isinstance(unpacked, TypeVarTupleType): + unpacked = get_proper_type(unpacked.upper_bound) + assert ( + isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + ) + unpack_items = [unpacked.args[0]] * extras + rep_types = rep_types[:unpack_index] + unpack_items + rep_types[unpack_index + 1 :] elif isinstance(rhs_type, AnyType): return + elif isinstance(rhs_type, Instance) and rhs_type.type.fullname == "builtins.tuple": + # Assume that an arbitrary-length tuple has the right number of items. + rep_types = [rhs_type.args[0]] * len(checkers) + elif isinstance(rhs_type, UnionType): + for typ in rhs_type.relevant_items(): + temp_node = TempNode(typ) + temp_node.line = replacements.line + self.check_simple_str_interpolation(specifiers, temp_node, expr) + return else: rep_types = [rhs_type] if len(checkers) > len(rep_types): - self.msg.too_few_string_formatting_arguments(replacements) + # Only check the fix-length Tuple type. Other Iterable types would skip. + if is_subtype(rhs_type, self.chk.named_type("typing.Iterable")) and not isinstance( + rhs_type, TupleType + ): + return + else: + self.msg.too_few_string_formatting_arguments(replacements) elif len(checkers) < len(rep_types): self.msg.too_many_string_formatting_arguments(replacements) else: if len(checkers) == 1: check_node, check_type = checkers[0] - check_node(replacements) - elif isinstance(replacements, TupleExpr): + if isinstance(rhs_type, TupleType) and len(rhs_type.items) == 1: + check_type(rhs_type.items[0]) + else: + check_node(replacements) + elif isinstance(replacements, TupleExpr) and not any( + isinstance(item, StarExpr) for item in replacements.items + ): for checks, rep_node in zip(checkers, replacements.items): check_node, check_type = checks check_node(rep_node) @@ -129,148 +791,287 @@ def check_simple_str_interpolation(self, specifiers: List[ConversionSpecifier], check_node, check_type = checks check_type(rep_type) - def check_mapping_str_interpolation(self, specifiers: List[ConversionSpecifier], - replacements: Node) -> None: - dict_with_only_str_literal_keys = (isinstance(replacements, DictExpr) and - all(isinstance(self.exprchk.strip_parens(k), StrExpr) - for k, v in cast(DictExpr, replacements).items)) - if dict_with_only_str_literal_keys: - mapping = {} # type: Dict[str, Type] - for k, v in cast(DictExpr, replacements).items: - key_str = cast(StrExpr, k).value + def check_mapping_str_interpolation( + self, + specifiers: list[ConversionSpecifier], + replacements: Expression, + expr: FormatStringExpr, + ) -> None: + """Check % string interpolation with names specifiers '%(name)s' % {'name': 'John'}.""" + if isinstance(replacements, DictExpr) and all( + isinstance(k, (StrExpr, BytesExpr)) for k, v in replacements.items + ): + mapping: dict[str, Type] = {} + for k, v in replacements.items: + if isinstance(expr, BytesExpr): + # Special case: for bytes formatting keys must be bytes. + if not isinstance(k, BytesExpr): + self.msg.fail( + "Dictionary keys in bytes formatting must be bytes, not strings", + expr, + code=codes.STRING_FORMATTING, + ) + key_str = cast(FormatStringExpr, k).value mapping[key_str] = self.accept(v) for specifier in specifiers: + if specifier.conv_type == "%": + # %% is allowed in mappings, no checking is required + continue + assert specifier.key is not None if specifier.key not in mapping: self.msg.key_not_in_mapping(specifier.key, replacements) return rep_type = mapping[specifier.key] - expected_type = self.conversion_type(specifier.type, replacements) - if expected_type == None: + assert specifier.conv_type is not None + expected_type = self.conversion_type(specifier.conv_type, replacements, expr) + if expected_type is None: return - self.chk.check_subtype(rep_type, expected_type, replacements, - messages.INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION, - 'expression has type', 'placeholder with key \'%s\' has type' % specifier.key) + self.chk.check_subtype( + rep_type, + expected_type, + replacements, + message_registry.INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION, + "expression has type", + f"placeholder with key '{specifier.key}' has type", + code=codes.STRING_FORMATTING, + ) + if specifier.conv_type == "s": + self.check_s_special_cases(expr, rep_type, expr) else: rep_type = self.accept(replacements) - dict_type = self.chk.named_generic_type('builtins.dict', - [AnyType(), AnyType()]) - self.chk.check_subtype(rep_type, dict_type, replacements, messages.FORMAT_REQUIRES_MAPPING, - 'expression has type', 'expected type for mapping is') - - def build_replacement_checkers(self, specifiers: List[ConversionSpecifier], - context: Context) -> List[ Tuple[ Function[[Node], None], - Function[[Type], None] ] ]: - checkers = [] # type: List[ Tuple[ Function[[Node], None], Function[[Type], None] ] ] + dict_type = self.build_dict_type(expr) + self.chk.check_subtype( + rep_type, + dict_type, + replacements, + message_registry.FORMAT_REQUIRES_MAPPING, + "expression has type", + "expected type for mapping is", + code=codes.STRING_FORMATTING, + ) + + def build_dict_type(self, expr: FormatStringExpr) -> Type: + """Build expected mapping type for right operand in % formatting.""" + any_type = AnyType(TypeOfAny.special_form) + if isinstance(expr, BytesExpr): + bytes_type = self.chk.named_generic_type("builtins.bytes", []) + return self.chk.named_generic_type( + "_typeshed.SupportsKeysAndGetItem", [bytes_type, any_type] + ) + elif isinstance(expr, StrExpr): + str_type = self.chk.named_generic_type("builtins.str", []) + return self.chk.named_generic_type( + "_typeshed.SupportsKeysAndGetItem", [str_type, any_type] + ) + else: + assert False, "Unreachable" + + def build_replacement_checkers( + self, specifiers: list[ConversionSpecifier], context: Context, expr: FormatStringExpr + ) -> list[Checkers] | None: + checkers: list[Checkers] = [] for specifier in specifiers: - checker = self.replacement_checkers(specifier, context) - if checker == None: + checker = self.replacement_checkers(specifier, context, expr) + if checker is None: return None checkers.extend(checker) return checkers - def replacement_checkers(self, specifier: ConversionSpecifier, - context: Context) -> List[ Tuple[ Function[[Node], None], - Function[[Type], None] ] ]: + def replacement_checkers( + self, specifier: ConversionSpecifier, context: Context, expr: FormatStringExpr + ) -> list[Checkers] | None: """Returns a list of tuples of two functions that check whether a replacement is - of the right type for the specifier. The first functions take a node and checks + of the right type for the specifier. The first function takes a node and checks its type in the right type context. The second function just checks a type. """ - checkers = [] # type: List[ Tuple[ Function[[Node], None], Function[[Type], None] ] ] + checkers: list[Checkers] = [] - if specifier.width == '*': + if specifier.width == "*": checkers.append(self.checkers_for_star(context)) - if specifier.precision == '*': + if specifier.precision == "*": checkers.append(self.checkers_for_star(context)) - if specifier.type == 'c': - c = self.checkers_for_c_type(specifier.type, context) - if c == None: + + if specifier.conv_type == "c": + c = self.checkers_for_c_type(specifier.conv_type, context, expr) + if c is None: return None checkers.append(c) - elif specifier.type != '%': - c = self.checkers_for_regular_type(specifier.type, context) - if c == None: + elif specifier.conv_type is not None and specifier.conv_type != "%": + c = self.checkers_for_regular_type(specifier.conv_type, context, expr) + if c is None: return None checkers.append(c) return checkers - def checkers_for_star(self, context: Context) -> Tuple[ Function[[Node], None], - Function[[Type], None] ]: + def checkers_for_star(self, context: Context) -> Checkers: """Returns a tuple of check functions that check whether, respectively, - a node or a type is compatible with a star in a conversion specifier + a node or a type is compatible with a star in a conversion specifier. """ - expected = self.named_type('builtins.int') + expected = self.named_type("builtins.int") - def check_type(type: Type = None) -> None: - expected = self.named_type('builtins.int') - self.chk.check_subtype(type, expected, context, '* wants int') + def check_type(type: Type) -> bool: + expected = self.named_type("builtins.int") + return self.chk.check_subtype( + type, expected, context, "* wants int", code=codes.STRING_FORMATTING + ) - def check_node(node: Node) -> None: - type = self.accept(node, expected) + def check_expr(expr: Expression) -> None: + type = self.accept(expr, expected) check_type(type) - return check_node, check_type + return check_expr, check_type - def checkers_for_regular_type(self, type: str, context: Context) -> Tuple[ Function[[Node], None], - Function[[Type], None] ]: + def check_placeholder_type(self, typ: Type, expected_type: Type, context: Context) -> bool: + return self.chk.check_subtype( + typ, + expected_type, + context, + message_registry.INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION, + "expression has type", + "placeholder has type", + code=codes.STRING_FORMATTING, + ) + + def checkers_for_regular_type( + self, conv_type: str, context: Context, expr: FormatStringExpr + ) -> Checkers | None: """Returns a tuple of check functions that check whether, respectively, - a node or a type is compatible with 'type'. Return None in case of an + a node or a type is compatible with 'type'. Return None in case of an error. """ - expected_type = self.conversion_type(type, context) - if expected_type == None: + expected_type = self.conversion_type(conv_type, context, expr) + if expected_type is None: return None - def check_type(type: Type = None) -> None: - self.chk.check_subtype(type, expected_type, context, - messages.INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION, - 'expression has type', 'placeholder has type') + def check_type(typ: Type) -> bool: + assert expected_type is not None + ret = self.check_placeholder_type(typ, expected_type, context) + if ret and conv_type == "s": + ret = self.check_s_special_cases(expr, typ, context) + return ret - def check_node(node: Node) -> None: - type = self.accept(node, expected_type) + def check_expr(expr: Expression) -> None: + type = self.accept(expr, expected_type) check_type(type) - return check_node, check_type + return check_expr, check_type + + def check_s_special_cases(self, expr: FormatStringExpr, typ: Type, context: Context) -> bool: + """Additional special cases for %s in bytes vs string context.""" + if isinstance(expr, StrExpr): + # Couple special cases for string formatting. + if has_type_component(typ, "builtins.bytes"): + self.msg.fail( + 'If x = b\'abc\' then "%s" % x produces "b\'abc\'", not "abc". ' + 'If this is desired behavior use "%r" % x. Otherwise, decode the bytes', + context, + code=codes.STR_BYTES_PY3, + ) + return False + if isinstance(expr, BytesExpr): + # A special case for bytes formatting: b'%s' actually requires bytes on Python 3. + if has_type_component(typ, "builtins.str"): + self.msg.fail( + "On Python 3 b'%s' requires bytes, not string", + context, + code=codes.STRING_FORMATTING, + ) + return False + return True - def checkers_for_c_type(self, type: str, context: Context) -> Tuple[ Function[[Node], None], - Function[[Type], None] ]: + def checkers_for_c_type( + self, type: str, context: Context, format_expr: FormatStringExpr + ) -> Checkers | None: """Returns a tuple of check functions that check whether, respectively, - a node or a type is compatible with 'type' that is a character type + a node or a type is compatible with 'type' that is a character type. """ - expected_type = self.conversion_type(type, context) - if expected_type == None: + expected_type = self.conversion_type(type, context, format_expr) + if expected_type is None: return None - def check_type(type: Type = None) -> None: - self.chk.check_subtype(type, expected_type, context, - messages.INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION, - 'expression has type', 'placeholder has type') + def check_type(type: Type) -> bool: + assert expected_type is not None + if isinstance(format_expr, BytesExpr): + err_msg = '"%c" requires an integer in range(256) or a single byte' + else: + err_msg = '"%c" requires int or char' + return self.chk.check_subtype( + type, + expected_type, + context, + err_msg, + "expression has type", + code=codes.STRING_FORMATTING, + ) - def check_node(node: Node) -> None: + def check_expr(expr: Expression) -> None: """int, or str with length 1""" - type = self.accept(node, expected_type) - if isinstance(node, StrExpr) and len(cast(StrExpr, node).value) != 1: - self.msg.requires_int_or_char(context) - check_type(type) + type = self.accept(expr, expected_type) + # We need further check with expr to make sure that + # it has exact one char or one single byte. + if check_type(type): + # Python 3 doesn't support b'%c' % str + if ( + isinstance(format_expr, BytesExpr) + and isinstance(expr, BytesExpr) + and len(expr.value) != 1 + ): + self.msg.requires_int_or_single_byte(context) + elif isinstance(expr, (StrExpr, BytesExpr)) and len(expr.value) != 1: + self.msg.requires_int_or_char(context) - return check_node, check_type + return check_expr, check_type - def conversion_type(self, p: str, context: Context) -> Type: - """Return the type that is accepted for a string interpolation - conversion specifier type. + def conversion_type( + self, p: str, context: Context, expr: FormatStringExpr, format_call: bool = False + ) -> Type | None: + """Return the type that is accepted for a string interpolation conversion specifier type. Note that both Python's float (e.g. %f) and integer (e.g. %d) specifier types accept both float and integers. + + The 'format_call' argument indicates whether this type came from % interpolation or from + a str.format() call, the meaning of few formatting types are different. """ - if p in ['s', 'r']: - return AnyType() - elif p in ['d', 'i', 'o', 'u', 'x', 'X', - 'e', 'E', 'f', 'F', 'g', 'G']: - return UnionType([self.named_type('builtins.int'), - self.named_type('builtins.float')]) - elif p in ['c']: - return UnionType([self.named_type('builtins.int'), - self.named_type('builtins.float'), - self.named_type('builtins.str')]) + NUMERIC_TYPES = NUMERIC_TYPES_NEW if format_call else NUMERIC_TYPES_OLD + INT_TYPES = REQUIRE_INT_NEW if format_call else REQUIRE_INT_OLD + if p == "b" and not format_call: + if not isinstance(expr, BytesExpr): + self.msg.fail( + 'Format character "b" is only supported on bytes patterns', + context, + code=codes.STRING_FORMATTING, + ) + return None + return self.named_type("builtins.bytes") + elif p == "a": + # TODO: return type object? + return AnyType(TypeOfAny.special_form) + elif p in ["s", "r"]: + return AnyType(TypeOfAny.special_form) + elif p in NUMERIC_TYPES: + if p in INT_TYPES: + numeric_types = [self.named_type("builtins.int")] + else: + numeric_types = [ + self.named_type("builtins.int"), + self.named_type("builtins.float"), + ] + if not format_call: + if p in FLOAT_TYPES: + numeric_types.append(self.named_type("typing.SupportsFloat")) + else: + numeric_types.append(self.named_type("typing.SupportsInt")) + return UnionType.make_union(numeric_types) + elif p in ["c"]: + if isinstance(expr, BytesExpr): + return UnionType( + [self.named_type("builtins.int"), self.named_type("builtins.bytes")] + ) + else: + return UnionType( + [self.named_type("builtins.int"), self.named_type("builtins.str")] + ) else: self.msg.unsupported_placeholder(p, context) return None @@ -285,6 +1086,26 @@ def named_type(self, name: str) -> Instance: """ return self.chk.named_type(name) - def accept(self, node: Node, context: Type = None) -> Type: + def accept(self, expr: Expression, context: Type | None = None) -> Type: """Type check a node. Alias for TypeChecker.accept.""" - return self.chk.accept(node, context) + return self.chk.expr_checker.accept(expr, context) + + +def has_type_component(typ: Type, fullname: str) -> bool: + """Is this a specific instance type, or a union that contains it? + + We use this ad-hoc function instead of a proper visitor or subtype check + because some str vs bytes errors are strictly speaking not runtime errors, + but rather highly counter-intuitive behavior. This is similar to what is used for + --strict-equality. + """ + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return typ.type.has_base(fullname) + elif isinstance(typ, TypeVarType): + return has_type_component(typ.upper_bound, fullname) or any( + has_type_component(v, fullname) for v in typ.values + ) + elif isinstance(typ, UnionType): + return any(has_type_component(t, fullname) for t in typ.relevant_items()) + return False diff --git a/mypy/codec/README b/mypy/codec/README deleted file mode 100644 index 040bbe918461..000000000000 --- a/mypy/codec/README +++ /dev/null @@ -1,40 +0,0 @@ -mypy.codec readme -================= - - -What's this? ------------- - -mypy.codec defines an encoding that strips type annotations from -function definitions in Python 2, and leaves them in Python 3, so that -code with these annotations can be executed in either environment. - -mypy.codec is derived from pyxl (https://github.com/dropbox/pyxl). - - -Using the codec ---------------- - -To register the codec, add this to the beginning of each module that -can act as an entry point to your program: - - from mypy.codec import register - -Now you can use Python 3 function annotations in any module imported -after registering the codec, even in Python 2: - - # coding: mypy - - import typing - - def inc(x: int) -> int: - return x + 1 - - -Running tests -------------- - -Unit tests are run by py.test. This requires that you have the pytest -package installed (e.g. sudo pip install pytest). Run tests like this: - - $ py.test mypy/codec diff --git a/mypy/codec/mypy_codec.py b/mypy/codec/mypy_codec.py deleted file mode 100644 index 020ec76dcbe3..000000000000 --- a/mypy/codec/mypy_codec.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import absolute_import - -import codecs -import traceback -from encodings import utf_8 -from io import BytesIO - -from .tokenizer import mypy_tokenize, mypy_untokenize - - -def mypy_transform(stream): - try: - output = mypy_untokenize(mypy_tokenize(stream.readline)) - except Exception as ex: - print(ex) - traceback.print_exc() - raise - - return output - - -def mypy_transform_string(text): - stream = BytesIO(text) - return mypy_transform(stream) - - -def mypy_decode(input, errors='strict'): - return utf_8.decode(mypy_transform_string(input), errors) - - -class MyPyIncrementalDecoder(utf_8.IncrementalDecoder): - def decode(self, input, final=False): - self.buffer += input - if final: - buff = self.buffer - self.buffer = '' - return super(MyPyIncrementalDecoder, self).decode( - mypy_transform_string(buff), final=True) - - -class MyPyStreamReader(utf_8.StreamReader): - def __init__(self, *args, **kwargs): - codecs.StreamReader.__init__(self, *args, **kwargs) - self.stream = BytesIO(mypy_transform(self.stream)) diff --git a/mypy/codec/pytokenize.py b/mypy/codec/pytokenize.py deleted file mode 100644 index 395ced2cf353..000000000000 --- a/mypy/codec/pytokenize.py +++ /dev/null @@ -1,485 +0,0 @@ -"""Tokenization help for Python programs. - -generate_tokens(readline) is a generator that breaks a stream of -text into Python tokens. It accepts a readline-like method which is called -repeatedly to get the next line of input (or "" for EOF). It generates -5-tuples with these members: - - the token type (see token.py) - the token (a string) - the starting (row, column) indices of the token (a 2-tuple of ints) - the ending (row, column) indices of the token (a 2-tuple of ints) - the original line (string) - -It is designed to match the working of the Python tokenizer exactly, except -that it produces COMMENT tokens for comments and gives type OP for all -operators - -Older entry points - tokenize_loop(readline, tokeneater) - tokenize(readline, tokeneater=printtoken) -are the same, except instead of generating tokens, tokeneater is a callback -function to which the 5 fields described above are passed as 5 arguments, -each time a new token is found. - - -This file was taken from the python 2.7.4 library and modified for use by -the Pyxl decoder. Changes made: - - When it encounters an unexpected EOF, the tokenizer does not raise an - exception, and instead yields an errortoken if appropriate. - - When it encounters an unexpected dedent, the tokenizer does not - raise an exception. - - The Untokenizer class was heavily modified. - - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013 Python Software Foundation; All Rights Reserved" are retained -in Python alone or in any derivative version prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. -""" - -from __future__ import print_function - -__author__ = 'Ka-Ping Yee ' -__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' - 'Skip Montanaro, Raymond Hettinger') - -import string -import re -from token import * - -import token -x = None -__all__ = [x for x in dir(token) if not x.startswith("_")] -__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"] -del x -del token - -COMMENT = N_TOKENS -tok_name[COMMENT] = 'COMMENT' -NL = N_TOKENS + 1 -tok_name[NL] = 'NL' -N_TOKENS += 2 - - -def group(*choices): return '(' + '|'.join(choices) + ')' - - -def any(*choices): return group(*choices) + '*' - - -def maybe(*choices): return group(*choices) + '?' - -Whitespace = r'[ \f\t]*' -Comment = r'#[^\r\n]*' -Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'[a-zA-Z_]\w*' - -Hexnumber = r'0[xX][\da-fA-F]+[lL]?' -Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?' -Binnumber = r'0[bB][01]+[lL]?' -Decnumber = r'[1-9]\d*[lL]?' -Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent -Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') -Number = group(Imagnumber, Floatnumber, Intnumber) - -# Tail end of ' string. -Single = r"[^'\\]*(?:\\.[^'\\]*)*'" -# Tail end of " string. -Double = r'[^"\\]*(?:\\.[^"\\]*)*"' -# Tail end of ''' string. -Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" -# Tail end of """ string. -Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') -# Single-line ' or " string. -String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') - -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", - r"//=?", - r"[+\-*/%&|^=<>]=?", - r"~") - -Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,`@]') -Funny = group(Operator, Bracket, Special) - -PlainToken = group(Number, Funny, String, Name) -Token = Ignore + PlainToken - -# First (or only) line of ' or " string. -ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + - group("'", r'\\\r?\n'), - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + - group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) -PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) - -tokenprog, pseudoprog, single3prog, double3prog = map( - re.compile, (Token, PseudoToken, Single3, Double3)) -endprogs = {"'": re.compile(Single), '"': re.compile(Double), - "'''": single3prog, '"""': double3prog, - "r'''": single3prog, 'r"""': double3prog, - "u'''": single3prog, 'u"""': double3prog, - "ur'''": single3prog, 'ur"""': double3prog, - "R'''": single3prog, 'R"""': double3prog, - "U'''": single3prog, 'U"""': double3prog, - "uR'''": single3prog, 'uR"""': double3prog, - "Ur'''": single3prog, 'Ur"""': double3prog, - "UR'''": single3prog, 'UR"""': double3prog, - "b'''": single3prog, 'b"""': double3prog, - "br'''": single3prog, 'br"""': double3prog, - "B'''": single3prog, 'B"""': double3prog, - "bR'''": single3prog, 'bR"""': double3prog, - "Br'''": single3prog, 'Br"""': double3prog, - "BR'''": single3prog, 'BR"""': double3prog, - 'r': None, 'R': None, 'u': None, 'U': None, - 'b': None, 'B': None} - -triple_quoted = {} -for t in ("'''", '"""', - "r'''", 'r"""', "R'''", 'R"""', - "u'''", 'u"""', "U'''", 'U"""', - "ur'''", 'ur"""', "Ur'''", 'Ur"""', - "uR'''", 'uR"""', "UR'''", 'UR"""', - "b'''", 'b"""', "B'''", 'B"""', - "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""'): - triple_quoted[t] = t -single_quoted = {} -for t in ("'", '"', - "r'", 'r"', "R'", 'R"', - "u'", 'u"', "U'", 'U"', - "ur'", 'ur"', "Ur'", 'Ur"', - "uR'", 'uR"', "UR'", 'UR"', - "b'", 'b"', "B'", 'B"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"'): - single_quoted[t] = t - -tabsize = 8 - - -class TokenError(Exception): pass - - -class StopTokenizing(Exception): pass - - -def printtoken(type, token, srow_scol, erow_ecol, line): # for testing - srow, scol = srow_scol - erow, ecol = erow_ecol - print("%d,%d-%d,%d:\t%s\t%s" % - (srow, scol, erow, ecol, tok_name[type], repr(token))) - - -def tokenize(readline, tokeneater=printtoken): - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). - - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). - """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - - -# backwards compatible interface -def tokenize_loop(readline, tokeneater): - for token_info in generate_tokens(readline): - tokeneater(*token_info) - - -class Untokenizer: - - # PYXL MODIFICATION: This entire class. - - def __init__(self, row=None, col=None): - self.tokens = [] - self.prev_row = row - self.prev_col = col - - def add_whitespace(self, start): - row, col = start - assert row >= self.prev_row, "row (%r) should be >= prev_row (%r)" % (row, self.prev_row) - row_offset = row - self.prev_row - if row_offset: - self.tokens.append("\n" * row_offset) - col_offset = col - self.prev_col - if col_offset: - self.tokens.append(" " * col_offset) - - def feed(self, t): - assert len(t) == 5 - tok_type, token, start, end, line = t - if (self.prev_row is None): - self.prev_row, self.prev_col = start - self.add_whitespace(start) - self.tokens.append(token) - self.prev_row, self.prev_col = end - if tok_type in (NEWLINE, NL): - self.prev_row += 1 - self.prev_col = 0 - - def finish(self): - return "".join(self.tokens) - - def untokenize(self, iterable): - for t in iterable: - self.feed(t) - return self.finish() - - -def untokenize(iterable): - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited intput: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tok in generate_tokens(readline)] - assert t1 == t2 - """ - ut = Untokenizer() - return ut.untokenize(iterable) - - -def generate_tokens(readline): - """ - The generate_tokens() generator requires one argment, readline, which - must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline - can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple (srow, scol) of ints specifying the row and - column where the token begins in the source; a 2-tuple (erow, ecol) of - ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the - logical line; continuation lines are included. - """ - lnum = parenlev = continued = 0 - namechars, numchars = string.ascii_letters + '_', '0123456789' - contstr, needcont = '', 0 - contline = None - indents = [0] - - while 1: # loop over lines in stream - try: - line = readline() - except StopIteration: - line = '' - lnum += 1 - pos, max = 0, len(line) - - if contstr: # continued string - if not line: - # PYXL MODIFICATION: instead of raising an error here, we - # return the remainder of the file as an errortoken. - yield (ERRORTOKEN, contstr, - strstart, (lnum, 0), contline + line) - contstr, needcont = '', 0 - contline = None - return - endmatch = endprog.match(line) - if endmatch: - pos = end = endmatch.end(0) - yield (STRING, contstr + line[:end], - strstart, (lnum, end), contline + line) - contstr, needcont = '', 0 - contline = None - elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield (ERRORTOKEN, contstr + line, - strstart, (lnum, len(line)), contline) - contstr = '' - contline = None - continue - else: - contstr = contstr + line - contline = contline + line - continue - - elif parenlev == 0 and not continued: # new statement - if not line: break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == ' ': - column += 1 - elif line[pos] == '\t': - column = (column // tabsize + 1) * tabsize - elif line[pos] == '\f': - column = 0 - else: - break - pos += 1 - if pos == max: - break - - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], - (lnum, pos), (lnum, len(line)), line) - continue - - if column > indents[-1]: # count indents or dedents - indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - while column < indents[-1]: - if column not in indents: - # PYXL MODIFICATION: instead of raising an error here, we - # emit an empty dedent token, which has no effect on - # the decoded file. - pass - indents = indents[:-1] - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) - - else: # continued statement - if not line: - # PYXL MODIFICATION: instead of raising an error here, we - # return as if successful. - return - continued = 0 - - while pos < max: - pseudomatch = pseudoprog.match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - if start == end: - continue - token, initial = line[start:end], line[start] - - if initial in numchars or \ - (initial == '.' and token != '.'): # ordinary number - yield (NUMBER, token, spos, epos, line) - elif initial in '\r\n': - yield (NL if parenlev > 0 else NEWLINE, - token, spos, epos, line) - elif initial == '#': - assert not token.endswith("\n") - yield (COMMENT, token, spos, epos, line) - elif token in triple_quoted: - endprog = endprogs[token] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - yield (STRING, token, spos, (lnum, pos), line) - else: - strstart = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: - if token[-1] == '\n': # continued string - strstart = (lnum, start) - endprog = (endprogs[initial] or endprogs[token[1]] or - endprogs[token[2]]) - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - yield (STRING, token, spos, epos, line) - elif initial in namechars: # ordinary name - yield (NAME, token, spos, epos, line) - elif initial == '\\': # continued stmt - continued = 1 - else: - if initial in '([{': - parenlev += 1 - elif initial in ')]}': - parenlev -= 1 - yield (OP, token, spos, epos, line) - else: - yield (ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos + 1), line) - pos += 1 - - for indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, '', (lnum, 0), (lnum, 0), '') - yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') - -if __name__ == '__main__': # testing - import sys - if len(sys.argv) > 1: - tokenize(open(sys.argv[1]).readline) - else: - tokenize(sys.stdin.readline) diff --git a/mypy/codec/register.py b/mypy/codec/register.py deleted file mode 100644 index 5d8812371209..000000000000 --- a/mypy/codec/register.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import absolute_import - -import codecs -import encodings -import sys - - -def search_function(encoding): - if encoding != 'mypy': - return None - # Assume utf8 encoding - utf8 = encodings.search_function('utf8') - if sys.version_info[0] == 3: # Python 3 - return utf8 - else: # Python 2 - from .mypy_codec import mypy_decode, MyPyIncrementalDecoder, MyPyStreamReader - return codecs.CodecInfo(name='mypy', - encode=utf8.encode, - decode=mypy_decode, - incrementalencoder=utf8.incrementalencoder, - incrementaldecoder=MyPyIncrementalDecoder, - streamreader=MyPyStreamReader, - streamwriter=utf8.streamwriter) - -codecs.register(search_function) - - -def main(): - fn = sys.argv[1] - with open(fn) as fp: - data = fp.read() - print(codecs.decode(data, 'mypy')) - -if __name__ == '__main__': - main() diff --git a/mypy/codec/test/test_function_translation.py b/mypy/codec/test/test_function_translation.py deleted file mode 100644 index d308ca10f90a..000000000000 --- a/mypy/codec/test/test_function_translation.py +++ /dev/null @@ -1,87 +0,0 @@ -import codecs - -from unittest import TestCase - -from mypy.codec import register - -# The test functions that will be translated. -# even indices are input functions, odd indices are expected output -test_function_examples = [ -# test a function without annotations -'''\ -def f(x): - x = {'a': x} - return x['a'] -''', -'''\ -def f(x): - x = {'a': x} - return x['a'] -''', - -# test parameter type annotations -'''\ -def f(x: int, y: str = 'abc'): - return x -''', -'''\ -def f(x , y = 'abc'): - return x -''', - -# test return type annotations -'''\ -def f(x, y) -> str: - return x -''', -'''\ -def f(x, y) : - return x -''', - -# test newlines in param list -'''\ -def f(x: int, - y: str) -> str: - return x -''', -'''\ -def f(x , - y ) : - return x -''', - -# test newlines in return type annotation -'''\ -def f(x: int, y: str='abc') -> Tuple[int, - str]: - return x, y -''', -'''\ -def f(x , y ='abc')\\ - : - return x, y -''', - - -# test unrelated continuations -'''\ -x = 1 + \ - 2 -''', -'''\ -x = 1 + \ - 2 -''', - -] - - -class TestFunctionTranslation(TestCase): - - def test_all_functions(self): - for i in range(0, len(test_function_examples), 2): - func_translated = codecs.decode(test_function_examples[i], 'mypy') - #print repr(func_translated) - #print repr(test_function_examples[i + 1]) - self.assertEqual(func_translated, test_function_examples[i + 1]) diff --git a/mypy/codec/tokenizer.py b/mypy/codec/tokenizer.py deleted file mode 100644 index d248e89bb8e8..000000000000 --- a/mypy/codec/tokenizer.py +++ /dev/null @@ -1,122 +0,0 @@ -from __future__ import absolute_import - -from . import pytokenize as tokenize - - -def get_end_pos(start_pos, tvalue): - row, col = start_pos - for c in tvalue: - if c == '\n': - col = 0 - row += 1 - else: - col += 1 - return (row, col) - - -def mypy_untokenize(tokens): - parts = [] - prev_row = 1 - prev_col = 0 - - for token in tokens: - ttype, tvalue, tstart, tend, tline = token - row, col = tstart - - row_offset = row - prev_row - - assert row_offset >= 0, 'Unexpected jump in rows on line:%d: %s' % (row, tline) - - # Add whitespace - if row_offset > 0: - if ttype == tokenize.ENDMARKER: # don't add a continuation at the end of a file - parts.append('\n' * row_offset) - else: - parts.append("\\\n" * row_offset) - - if row_offset: - col_offset = col - else: - col_offset = col - prev_col - assert col_offset >= 0 - if col_offset > 0: - parts.append(" " * col_offset) - - parts.append(tvalue) - prev_row, prev_col = tend - - if ttype in (tokenize.NL, tokenize.NEWLINE): - prev_row += 1 - prev_col = 0 - - return ''.join(parts) - - -def mypy_tokenize(readline): - return transform_tokens(tokenize.generate_tokens(readline)) - - -def transform_tokens(tokens): - # state variables - in_a_def_statement = False # true if we're between a 'def' and associated ':' - def_depth = -1 # bracket depth of last def, for reference - - bracket_depth = 0 # bracket nesting depth of current token - - while 1: - try: - token = tokens.next() - except (StopIteration, tokenize.TokenError): - break - - tvalue = token[1] - - bracket_depth += bracket_delta(token) - - if in_a_def_statement: - if tvalue == ':' and bracket_depth > def_depth: - # param type annotation - newlines, token = scan_until(['=', ',', ')'], tokens) - tvalue = token[1] - for tok in newlines: - yield tok - bracket_depth += bracket_delta(token) - if tvalue == ')' and bracket_depth == def_depth: - # return type annotation - # TODO assert next are -> if there's an annotation at all - yield token - newlines, token = scan_until([':'], tokens) - # don't explicitly yield the newlines here, untokenize can handle continuations - in_a_def_statement = False - elif tvalue == 'def': - in_a_def_statement = True - def_depth = bracket_depth - - yield token - - -def scan_until(match_list, tokens): - """Steps through the tokens iterator until it finds a token whose value is in match_list at the - original bracket depth. Returns the token that matches, plus a list of other tokens that should - be kept (currently only newlines). - """ - bracket_depth = 0 - token = tokens.next() - to_keep = [] - while token[1] not in match_list or bracket_depth != 0: - bracket_depth += bracket_delta(token) - if token[0] in (tokenize.NL, tokenize.NEWLINE): - # keep newlines - to_keep.append(token) - token = tokens.next() - return to_keep, token - - -def bracket_delta(token): - """Returns +/-1 if the current token increases/decreases bracket nesting depth, 0 otherwise.""" - if token[0] == tokenize.OP and token[1] in ['{', '(', '[']: - return 1 - elif token[0] == tokenize.OP and token[1] in ['}', ')', ']']: - return -1 - else: - return 0 diff --git a/mypy/config_parser.py b/mypy/config_parser.py new file mode 100644 index 000000000000..9fa99333a42a --- /dev/null +++ b/mypy/config_parser.py @@ -0,0 +1,665 @@ +from __future__ import annotations + +import argparse +import configparser +import glob as fileglob +import os +import re +import sys +from io import StringIO + +from mypy.errorcodes import error_codes + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +from typing import ( + Any, + Callable, + Dict, + Final, + Iterable, + List, + Mapping, + MutableMapping, + Sequence, + TextIO, + Tuple, + Union, +) +from typing_extensions import TypeAlias as _TypeAlias + +from mypy import defaults +from mypy.options import PER_MODULE_OPTIONS, Options + +_CONFIG_VALUE_TYPES: _TypeAlias = Union[ + str, bool, int, float, Dict[str, str], List[str], Tuple[int, int] +] +_INI_PARSER_CALLABLE: _TypeAlias = Callable[[Any], _CONFIG_VALUE_TYPES] + + +def parse_version(v: str | float) -> tuple[int, int]: + m = re.match(r"\A(\d)\.(\d+)\Z", str(v)) + if not m: + raise argparse.ArgumentTypeError(f"Invalid python version '{v}' (expected format: 'x.y')") + major, minor = int(m.group(1)), int(m.group(2)) + if major == 2 and minor == 7: + pass # Error raised elsewhere + elif major == 3: + if minor < defaults.PYTHON3_VERSION_MIN[1]: + msg = "Python 3.{} is not supported (must be {}.{} or higher)".format( + minor, *defaults.PYTHON3_VERSION_MIN + ) + + if isinstance(v, float): + msg += ". You may need to put quotes around your Python version" + + raise argparse.ArgumentTypeError(msg) + else: + raise argparse.ArgumentTypeError( + f"Python major version '{major}' out of range (must be 3)" + ) + return major, minor + + +def try_split(v: str | Sequence[str], split_regex: str = "[,]") -> list[str]: + """Split and trim a str or list of str into a list of str""" + if isinstance(v, str): + return [p.strip() for p in re.split(split_regex, v)] + + return [p.strip() for p in v] + + +def validate_codes(codes: list[str]) -> list[str]: + invalid_codes = set(codes) - set(error_codes.keys()) + if invalid_codes: + raise argparse.ArgumentTypeError( + f"Invalid error code(s): {', '.join(sorted(invalid_codes))}" + ) + return codes + + +def validate_package_allow_list(allow_list: list[str]) -> list[str]: + for p in allow_list: + msg = f"Invalid allow list entry: {p}" + if "*" in p: + raise argparse.ArgumentTypeError( + f"{msg} (entries are already prefixes so must not contain *)" + ) + if "\\" in p or "/" in p: + raise argparse.ArgumentTypeError( + f"{msg} (entries must be packages like foo.bar not directories or files)" + ) + return allow_list + + +def expand_path(path: str) -> str: + """Expand the user home directory and any environment variables contained within + the provided path. + """ + + return os.path.expandvars(os.path.expanduser(path)) + + +def str_or_array_as_list(v: str | Sequence[str]) -> list[str]: + if isinstance(v, str): + return [v.strip()] if v.strip() else [] + return [p.strip() for p in v if p.strip()] + + +def split_and_match_files_list(paths: Sequence[str]) -> list[str]: + """Take a list of files/directories (with support for globbing through the glob library). + + Where a path/glob matches no file, we still include the raw path in the resulting list. + + Returns a list of file paths + """ + expanded_paths = [] + + for path in paths: + path = expand_path(path.strip()) + globbed_files = fileglob.glob(path, recursive=True) + if globbed_files: + expanded_paths.extend(globbed_files) + else: + expanded_paths.append(path) + + return expanded_paths + + +def split_and_match_files(paths: str) -> list[str]: + """Take a string representing a list of files/directories (with support for globbing + through the glob library). + + Where a path/glob matches no file, we still include the raw path in the resulting list. + + Returns a list of file paths + """ + + return split_and_match_files_list(paths.split(",")) + + +def check_follow_imports(choice: str) -> str: + choices = ["normal", "silent", "skip", "error"] + if choice not in choices: + raise argparse.ArgumentTypeError( + "invalid choice '{}' (choose from {})".format( + choice, ", ".join(f"'{x}'" for x in choices) + ) + ) + return choice + + +def check_junit_format(choice: str) -> str: + choices = ["global", "per_file"] + if choice not in choices: + raise argparse.ArgumentTypeError( + "invalid choice '{}' (choose from {})".format( + choice, ", ".join(f"'{x}'" for x in choices) + ) + ) + return choice + + +def split_commas(value: str) -> list[str]: + # Uses a bit smarter technique to allow last trailing comma + # and to remove last `""` item from the split. + items = value.split(",") + if items and items[-1] == "": + items.pop(-1) + return items + + +# For most options, the type of the default value set in options.py is +# sufficient, and we don't have to do anything here. This table +# exists to specify types for values initialized to None or container +# types. +ini_config_types: Final[dict[str, _INI_PARSER_CALLABLE]] = { + "python_version": parse_version, + "custom_typing_module": str, + "custom_typeshed_dir": expand_path, + "mypy_path": lambda s: [expand_path(p.strip()) for p in re.split("[,:]", s)], + "files": split_and_match_files, + "quickstart_file": expand_path, + "junit_xml": expand_path, + "junit_format": check_junit_format, + "follow_imports": check_follow_imports, + "no_site_packages": bool, + "plugins": lambda s: [p.strip() for p in split_commas(s)], + "always_true": lambda s: [p.strip() for p in split_commas(s)], + "always_false": lambda s: [p.strip() for p in split_commas(s)], + "untyped_calls_exclude": lambda s: validate_package_allow_list( + [p.strip() for p in split_commas(s)] + ), + "enable_incomplete_feature": lambda s: [p.strip() for p in split_commas(s)], + "disable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), + "enable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), + "package_root": lambda s: [p.strip() for p in split_commas(s)], + "cache_dir": expand_path, + "python_executable": expand_path, + "strict": bool, + "exclude": lambda s: [s.strip()], + "packages": try_split, + "modules": try_split, +} + +# Reuse the ini_config_types and overwrite the diff +toml_config_types: Final[dict[str, _INI_PARSER_CALLABLE]] = ini_config_types.copy() +toml_config_types.update( + { + "python_version": parse_version, + "mypy_path": lambda s: [expand_path(p) for p in try_split(s, "[,:]")], + "files": lambda s: split_and_match_files_list(try_split(s)), + "junit_format": lambda s: check_junit_format(str(s)), + "follow_imports": lambda s: check_follow_imports(str(s)), + "plugins": try_split, + "always_true": try_split, + "always_false": try_split, + "untyped_calls_exclude": lambda s: validate_package_allow_list(try_split(s)), + "enable_incomplete_feature": try_split, + "disable_error_code": lambda s: validate_codes(try_split(s)), + "enable_error_code": lambda s: validate_codes(try_split(s)), + "package_root": try_split, + "exclude": str_or_array_as_list, + "packages": try_split, + "modules": try_split, + } +) + + +def parse_config_file( + options: Options, + set_strict_flags: Callable[[], None], + filename: str | None, + stdout: TextIO | None = None, + stderr: TextIO | None = None, +) -> None: + """Parse a config file into an Options object. + + Errors are written to stderr but are not fatal. + + If filename is None, fall back to default config files. + """ + stdout = stdout or sys.stdout + stderr = stderr or sys.stderr + + if filename is not None: + config_files: tuple[str, ...] = (filename,) + else: + config_files_iter: Iterable[str] = map(os.path.expanduser, defaults.CONFIG_FILES) + config_files = tuple(config_files_iter) + + config_parser = configparser.RawConfigParser() + + for config_file in config_files: + if not os.path.exists(config_file): + continue + try: + if is_toml(config_file): + with open(config_file, "rb") as f: + toml_data = tomllib.load(f) + # Filter down to just mypy relevant toml keys + toml_data = toml_data.get("tool", {}) + if "mypy" not in toml_data: + continue + toml_data = {"mypy": toml_data["mypy"]} + parser: MutableMapping[str, Any] = destructure_overrides(toml_data) + config_types = toml_config_types + else: + config_parser.read(config_file) + parser = config_parser + config_types = ini_config_types + except (tomllib.TOMLDecodeError, configparser.Error, ConfigTOMLValueError) as err: + print(f"{config_file}: {err}", file=stderr) + else: + if config_file in defaults.SHARED_CONFIG_FILES and "mypy" not in parser: + continue + file_read = config_file + options.config_file = file_read + break + else: + return + + os.environ["MYPY_CONFIG_FILE_DIR"] = os.path.dirname(os.path.abspath(config_file)) + + if "mypy" not in parser: + if filename or file_read not in defaults.SHARED_CONFIG_FILES: + print(f"{file_read}: No [mypy] section in config file", file=stderr) + else: + section = parser["mypy"] + prefix = f"{file_read}: [mypy]: " + updates, report_dirs = parse_section( + prefix, options, set_strict_flags, section, config_types, stderr + ) + for k, v in updates.items(): + setattr(options, k, v) + options.report_dirs.update(report_dirs) + + for name, section in parser.items(): + if name.startswith("mypy-"): + prefix = get_prefix(file_read, name) + updates, report_dirs = parse_section( + prefix, options, set_strict_flags, section, config_types, stderr + ) + if report_dirs: + print( + prefix, + "Per-module sections should not specify reports ({})".format( + ", ".join(s + "_report" for s in sorted(report_dirs)) + ), + file=stderr, + ) + if set(updates) - PER_MODULE_OPTIONS: + print( + prefix, + "Per-module sections should only specify per-module flags ({})".format( + ", ".join(sorted(set(updates) - PER_MODULE_OPTIONS)) + ), + file=stderr, + ) + updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS} + + globs = name[5:] + for glob in globs.split(","): + # For backwards compatibility, replace (back)slashes with dots. + glob = glob.replace(os.sep, ".") + if os.altsep: + glob = glob.replace(os.altsep, ".") + + if any(c in glob for c in "?[]!") or any( + "*" in x and x != "*" for x in glob.split(".") + ): + print( + prefix, + "Patterns must be fully-qualified module names, optionally " + "with '*' in some components (e.g spam.*.eggs.*)", + file=stderr, + ) + else: + options.per_module_options[glob] = updates + + +def get_prefix(file_read: str, name: str) -> str: + if is_toml(file_read): + module_name_str = 'module = "%s"' % "-".join(name.split("-")[1:]) + else: + module_name_str = name + + return f"{file_read}: [{module_name_str}]:" + + +def is_toml(filename: str) -> bool: + return filename.lower().endswith(".toml") + + +def destructure_overrides(toml_data: dict[str, Any]) -> dict[str, Any]: + """Take the new [[tool.mypy.overrides]] section array in the pyproject.toml file, + and convert it back to a flatter structure that the existing config_parser can handle. + + E.g. the following pyproject.toml file: + + [[tool.mypy.overrides]] + module = [ + "a.b", + "b.*" + ] + disallow_untyped_defs = true + + [[tool.mypy.overrides]] + module = 'c' + disallow_untyped_defs = false + + Would map to the following config dict that it would have gotten from parsing an equivalent + ini file: + + { + "mypy-a.b": { + disallow_untyped_defs = true, + }, + "mypy-b.*": { + disallow_untyped_defs = true, + }, + "mypy-c": { + disallow_untyped_defs: false, + }, + } + """ + if "overrides" not in toml_data["mypy"]: + return toml_data + + if not isinstance(toml_data["mypy"]["overrides"], list): + raise ConfigTOMLValueError( + "tool.mypy.overrides sections must be an array. Please make " + "sure you are using double brackets like so: [[tool.mypy.overrides]]" + ) + + result = toml_data.copy() + for override in result["mypy"]["overrides"]: + if "module" not in override: + raise ConfigTOMLValueError( + "toml config file contains a [[tool.mypy.overrides]] " + "section, but no module to override was specified." + ) + + if isinstance(override["module"], str): + modules = [override["module"]] + elif isinstance(override["module"], list): + modules = override["module"] + else: + raise ConfigTOMLValueError( + "toml config file contains a [[tool.mypy.overrides]] " + "section with a module value that is not a string or a list of " + "strings" + ) + + for module in modules: + module_overrides = override.copy() + del module_overrides["module"] + old_config_name = f"mypy-{module}" + if old_config_name not in result: + result[old_config_name] = module_overrides + else: + for new_key, new_value in module_overrides.items(): + if ( + new_key in result[old_config_name] + and result[old_config_name][new_key] != new_value + ): + raise ConfigTOMLValueError( + "toml config file contains " + "[[tool.mypy.overrides]] sections with conflicting " + f"values. Module '{module}' has two different values for '{new_key}'" + ) + result[old_config_name][new_key] = new_value + + del result["mypy"]["overrides"] + return result + + +def parse_section( + prefix: str, + template: Options, + set_strict_flags: Callable[[], None], + section: Mapping[str, Any], + config_types: dict[str, Any], + stderr: TextIO = sys.stderr, +) -> tuple[dict[str, object], dict[str, str]]: + """Parse one section of a config file. + + Returns a dict of option values encountered, and a dict of report directories. + """ + results: dict[str, object] = {} + report_dirs: dict[str, str] = {} + + # Because these fields exist on Options, without proactive checking, we would accept them + # and crash later + invalid_options = { + "enabled_error_codes": "enable_error_code", + "disabled_error_codes": "disable_error_code", + } + + for key in section: + invert = False + options_key = key + if key in config_types: + ct = config_types[key] + elif key in invalid_options: + print( + f"{prefix}Unrecognized option: {key} = {section[key]}" + f" (did you mean {invalid_options[key]}?)", + file=stderr, + ) + continue + else: + dv = getattr(template, key, None) + if dv is None: + if key.endswith("_report"): + report_type = key[:-7].replace("_", "-") + if report_type in defaults.REPORTER_NAMES: + report_dirs[report_type] = str(section[key]) + else: + print(f"{prefix}Unrecognized report type: {key}", file=stderr) + continue + if key.startswith("x_"): + pass # Don't complain about `x_blah` flags + elif key.startswith("no_") and hasattr(template, key[3:]): + options_key = key[3:] + invert = True + elif key.startswith("allow") and hasattr(template, "dis" + key): + options_key = "dis" + key + invert = True + elif key.startswith("disallow") and hasattr(template, key[3:]): + options_key = key[3:] + invert = True + elif key.startswith("show_") and hasattr(template, "hide_" + key[5:]): + options_key = "hide_" + key[5:] + invert = True + elif key == "strict": + pass # Special handling below + else: + print(f"{prefix}Unrecognized option: {key} = {section[key]}", file=stderr) + if invert: + dv = getattr(template, options_key, None) + else: + continue + ct = type(dv) + v: Any = None + try: + if ct is bool: + if isinstance(section, dict): + v = convert_to_boolean(section.get(key)) + else: + v = section.getboolean(key) # type: ignore[attr-defined] # Until better stub + if invert: + v = not v + elif callable(ct): + if invert: + print(f"{prefix}Can not invert non-boolean key {options_key}", file=stderr) + continue + try: + v = ct(section.get(key)) + except argparse.ArgumentTypeError as err: + print(f"{prefix}{key}: {err}", file=stderr) + continue + else: + print(f"{prefix}Don't know what type {key} should have", file=stderr) + continue + except ValueError as err: + print(f"{prefix}{key}: {err}", file=stderr) + continue + if key == "strict": + if v: + set_strict_flags() + continue + results[options_key] = v + + # These two flags act as per-module overrides, so store the empty defaults. + if "disable_error_code" not in results: + results["disable_error_code"] = [] + if "enable_error_code" not in results: + results["enable_error_code"] = [] + + return results, report_dirs + + +def convert_to_boolean(value: Any | None) -> bool: + """Return a boolean value translating from other types if necessary.""" + if isinstance(value, bool): + return value + if not isinstance(value, str): + value = str(value) + if value.lower() not in configparser.RawConfigParser.BOOLEAN_STATES: + raise ValueError(f"Not a boolean: {value}") + return configparser.RawConfigParser.BOOLEAN_STATES[value.lower()] + + +def split_directive(s: str) -> tuple[list[str], list[str]]: + """Split s on commas, except during quoted sections. + + Returns the parts and a list of error messages.""" + parts = [] + cur: list[str] = [] + errors = [] + i = 0 + while i < len(s): + if s[i] == ",": + parts.append("".join(cur).strip()) + cur = [] + elif s[i] == '"': + i += 1 + while i < len(s) and s[i] != '"': + cur.append(s[i]) + i += 1 + if i == len(s): + errors.append("Unterminated quote in configuration comment") + cur.clear() + else: + cur.append(s[i]) + i += 1 + if cur: + parts.append("".join(cur).strip()) + + return parts, errors + + +def mypy_comments_to_config_map(line: str, template: Options) -> tuple[dict[str, str], list[str]]: + """Rewrite the mypy comment syntax into ini file syntax.""" + options = {} + entries, errors = split_directive(line) + for entry in entries: + if "=" not in entry: + name = entry + value = None + else: + name, value = (x.strip() for x in entry.split("=", 1)) + + name = name.replace("-", "_") + if value is None: + value = "True" + options[name] = value + + return options, errors + + +def parse_mypy_comments( + args: list[tuple[int, str]], template: Options +) -> tuple[dict[str, object], list[tuple[int, str]]]: + """Parse a collection of inline mypy: configuration comments. + + Returns a dictionary of options to be applied and a list of error messages + generated. + """ + + errors: list[tuple[int, str]] = [] + sections = {} + + for lineno, line in args: + # In order to easily match the behavior for bools, we abuse configparser. + # Oddly, the only way to get the SectionProxy object with the getboolean + # method is to create a config parser. + parser = configparser.RawConfigParser() + options, parse_errors = mypy_comments_to_config_map(line, template) + parser["dummy"] = options + errors.extend((lineno, x) for x in parse_errors) + + stderr = StringIO() + strict_found = False + + def set_strict_flags() -> None: + nonlocal strict_found + strict_found = True + + new_sections, reports = parse_section( + "", template, set_strict_flags, parser["dummy"], ini_config_types, stderr=stderr + ) + errors.extend((lineno, x) for x in stderr.getvalue().strip().split("\n") if x) + if reports: + errors.append((lineno, "Reports not supported in inline configuration")) + if strict_found: + errors.append( + ( + lineno, + 'Setting "strict" not supported in inline configuration: specify it in ' + "a configuration file instead, or set individual inline flags " + '(see "mypy -h" for the list of flags enabled in strict mode)', + ) + ) + + sections.update(new_sections) + + return sections, errors + + +def get_config_module_names(filename: str | None, modules: list[str]) -> str: + if not filename or not modules: + return "" + + if not is_toml(filename): + return ", ".join(f"[mypy-{module}]" for module in modules) + + return "module = ['%s']" % ("', '".join(sorted(modules))) + + +class ConfigTOMLValueError(ValueError): + pass diff --git a/mypy/constant_fold.py b/mypy/constant_fold.py new file mode 100644 index 000000000000..4582b2a7396d --- /dev/null +++ b/mypy/constant_fold.py @@ -0,0 +1,187 @@ +"""Constant folding of expressions. + +For example, 3 + 5 can be constant folded into 8. +""" + +from __future__ import annotations + +from typing import Final, Union + +from mypy.nodes import ( + ComplexExpr, + Expression, + FloatExpr, + IntExpr, + NameExpr, + OpExpr, + StrExpr, + UnaryExpr, + Var, +) + +# All possible result types of constant folding +ConstantValue = Union[int, bool, float, complex, str] +CONST_TYPES: Final = (int, bool, float, complex, str) + + +def constant_fold_expr(expr: Expression, cur_mod_id: str) -> ConstantValue | None: + """Return the constant value of an expression for supported operations. + + Among other things, support int arithmetic and string + concatenation. For example, the expression 3 + 5 has the constant + value 8. + + Also bind simple references to final constants defined in the + current module (cur_mod_id). Binding to references is best effort + -- we don't bind references to other modules. Mypyc trusts these + to be correct in compiled modules, so that it can replace a + constant expression (or a reference to one) with the statically + computed value. We don't want to infer constant values based on + stubs, in particular, as these might not match the implementation + (due to version skew, for example). + + Return None if unsuccessful. + """ + if isinstance(expr, IntExpr): + return expr.value + if isinstance(expr, StrExpr): + return expr.value + if isinstance(expr, FloatExpr): + return expr.value + if isinstance(expr, ComplexExpr): + return expr.value + elif isinstance(expr, NameExpr): + if expr.name == "True": + return True + elif expr.name == "False": + return False + node = expr.node + if ( + isinstance(node, Var) + and node.is_final + and node.fullname.rsplit(".", 1)[0] == cur_mod_id + ): + value = node.final_value + if isinstance(value, (CONST_TYPES)): + return value + elif isinstance(expr, OpExpr): + left = constant_fold_expr(expr.left, cur_mod_id) + right = constant_fold_expr(expr.right, cur_mod_id) + if left is not None and right is not None: + return constant_fold_binary_op(expr.op, left, right) + elif isinstance(expr, UnaryExpr): + value = constant_fold_expr(expr.expr, cur_mod_id) + if value is not None: + return constant_fold_unary_op(expr.op, value) + return None + + +def constant_fold_binary_op( + op: str, left: ConstantValue, right: ConstantValue +) -> ConstantValue | None: + if isinstance(left, int) and isinstance(right, int): + return constant_fold_binary_int_op(op, left, right) + + # Float and mixed int/float arithmetic. + if isinstance(left, float) and isinstance(right, float): + return constant_fold_binary_float_op(op, left, right) + elif isinstance(left, float) and isinstance(right, int): + return constant_fold_binary_float_op(op, left, right) + elif isinstance(left, int) and isinstance(right, float): + return constant_fold_binary_float_op(op, left, right) + + # String concatenation and multiplication. + if op == "+" and isinstance(left, str) and isinstance(right, str): + return left + right + elif op == "*" and isinstance(left, str) and isinstance(right, int): + return left * right + elif op == "*" and isinstance(left, int) and isinstance(right, str): + return left * right + + # Complex construction. + if op == "+" and isinstance(left, (int, float)) and isinstance(right, complex): + return left + right + elif op == "+" and isinstance(left, complex) and isinstance(right, (int, float)): + return left + right + elif op == "-" and isinstance(left, (int, float)) and isinstance(right, complex): + return left - right + elif op == "-" and isinstance(left, complex) and isinstance(right, (int, float)): + return left - right + + return None + + +def constant_fold_binary_int_op(op: str, left: int, right: int) -> int | float | None: + if op == "+": + return left + right + if op == "-": + return left - right + elif op == "*": + return left * right + elif op == "/": + if right != 0: + return left / right + elif op == "//": + if right != 0: + return left // right + elif op == "%": + if right != 0: + return left % right + elif op == "&": + return left & right + elif op == "|": + return left | right + elif op == "^": + return left ^ right + elif op == "<<": + if right >= 0: + return left << right + elif op == ">>": + if right >= 0: + return left >> right + elif op == "**": + if right >= 0: + ret = left**right + assert isinstance(ret, int) + return ret + return None + + +def constant_fold_binary_float_op(op: str, left: int | float, right: int | float) -> float | None: + assert not (isinstance(left, int) and isinstance(right, int)), (op, left, right) + if op == "+": + return left + right + elif op == "-": + return left - right + elif op == "*": + return left * right + elif op == "/": + if right != 0: + return left / right + elif op == "//": + if right != 0: + return left // right + elif op == "%": + if right != 0: + return left % right + elif op == "**": + if (left < 0 and isinstance(right, int)) or left > 0: + try: + ret = left**right + except OverflowError: + return None + else: + assert isinstance(ret, float), ret + return ret + + return None + + +def constant_fold_unary_op(op: str, value: ConstantValue) -> int | float | None: + if op == "-" and isinstance(value, (int, float)): + return -value + elif op == "~" and isinstance(value, int): + return ~value + elif op == "+" and isinstance(value, (int, float)): + return value + return None diff --git a/mypy/constraints.py b/mypy/constraints.py index 159707a47e7e..5c815bf2af65 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1,19 +1,70 @@ """Type inference constraints.""" -from typing import List, cast, Undefined +from __future__ import annotations +from typing import TYPE_CHECKING, Final, Iterable, List, Sequence + +import mypy.subtypes +import mypy.typeops +from mypy.argmap import ArgTypeExpander +from mypy.erasetype import erase_typevars +from mypy.maptype import map_instance_to_supertype +from mypy.nodes import ( + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + CONTRAVARIANT, + COVARIANT, + ArgKind, + TypeInfo, +) from mypy.types import ( - Callable, Type, TypeVisitor, UnboundType, AnyType, Void, NoneTyp, TypeVar, - Instance, TupleType, UnionType, Overloaded, ErasedType, is_named_instance + TUPLE_LIKE_INSTANCE_NAMES, + AnyType, + CallableType, + DeletedType, + ErasedType, + Instance, + LiteralType, + NoneType, + NormalizedCallableType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeQuery, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + find_unpack_in_list, + get_proper_type, + has_recursive_types, + has_type_vars, + is_named_instance, + split_with_prefix_and_suffix, ) -from mypy.expandtype import expand_caller_var_args -from mypy.maptype import map_instance_to_supertype -from mypy import nodes -import mypy.subtypes +from mypy.types_utils import is_union_with_any +from mypy.typestate import type_state +if TYPE_CHECKING: + from mypy.infer import ArgumentInferContext -SUBTYPE_OF = 0 # type: int -SUPERTYPE_OF = 1 # type: int +SUBTYPE_OF: Final = 0 +SUPERTYPE_OF: Final = 1 class Constraint: @@ -22,91 +73,217 @@ class Constraint: It can be either T <: type or T :> type (T is a type variable). """ - type_var = 0 # Type variable id - op = 0 # SUBTYPE_OF or SUPERTYPE_OF + type_var: TypeVarId + op = 0 # SUBTYPE_OF or SUPERTYPE_OF + target: Type - target = Undefined(Type) + def __init__(self, type_var: TypeVarLikeType, op: int, target: Type) -> None: + self.type_var = type_var.id + self.op = op + # TODO: should we add "assert not isinstance(target, UnpackType)"? + # UnpackType is a synthetic type, and is never valid as a constraint target. + self.target = target + self.origin_type_var = type_var + # These are additional type variables that should be solved for together with type_var. + # TODO: A cleaner solution may be to modify the return type of infer_constraints() + # to include these instead, but this is a rather big refactoring. + self.extra_tvars: list[TypeVarLikeType] = [] def __repr__(self) -> str: - op_str = '<:' + op_str = "<:" if self.op == SUPERTYPE_OF: - op_str = ':>' - return '{} {} {}'.format(self.type_var, op_str, self.target) + op_str = ":>" + return f"{self.type_var} {op_str} {self.target}" - def __init__(self, type_var: int, op: int, target: Type) -> None: - self.type_var = type_var - self.op = op - self.target = target + def __hash__(self) -> int: + return hash((self.type_var, self.op, self.target)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Constraint): + return False + return (self.type_var, self.op, self.target) == (other.type_var, other.op, other.target) def infer_constraints_for_callable( - callee: Callable, arg_types: List[Type], arg_kinds: List[int], - formal_to_actual: List[List[int]]) -> List[Constraint]: + callee: CallableType, + arg_types: Sequence[Type | None], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + formal_to_actual: list[list[int]], + context: ArgumentInferContext, +) -> list[Constraint]: """Infer type variable constraints for a callable and actual arguments. Return a list of constraints. """ + constraints: list[Constraint] = [] + mapper = ArgTypeExpander(context) - constraints = [] # type: List[Constraint] - tuple_counter = [0] + param_spec = callee.param_spec() + param_spec_arg_types = [] + param_spec_arg_names = [] + param_spec_arg_kinds = [] + incomplete_star_mapping = False for i, actuals in enumerate(formal_to_actual): for actual in actuals: - actual_type = get_actual_type(arg_types[actual], arg_kinds[actual], - tuple_counter) - c = infer_constraints(callee.arg_types[i], actual_type, - SUPERTYPE_OF) - constraints.extend(c) + if actual is None and callee.arg_kinds[i] in (ARG_STAR, ARG_STAR2): + # We can't use arguments to infer ParamSpec constraint, if only some + # are present in the current inference pass. + incomplete_star_mapping = True + break - return constraints + for i, actuals in enumerate(formal_to_actual): + if isinstance(callee.arg_types[i], UnpackType): + unpack_type = callee.arg_types[i] + assert isinstance(unpack_type, UnpackType) + # In this case we are binding all the actuals to *args, + # and we want a constraint that the typevar tuple being unpacked + # is equal to a type list of all the actuals. + actual_types = [] -def get_actual_type(arg_type: Type, kind: int, - tuple_counter: List[int]) -> Type: - """Return the type of an actual argument with the given kind. + unpacked_type = get_proper_type(unpack_type.type) + if isinstance(unpacked_type, TypeVarTupleType): + tuple_instance = unpacked_type.tuple_fallback + elif isinstance(unpacked_type, TupleType): + tuple_instance = unpacked_type.partial_fallback + else: + assert False, "mypy bug: unhandled constraint inference case" - If the argument is a *arg, return the individual argument item. - """ + for actual in actuals: + actual_arg_type = arg_types[actual] + if actual_arg_type is None: + continue - if kind == nodes.ARG_STAR: - if isinstance(arg_type, Instance): - if arg_type.type.fullname() == 'builtins.list': - # List *arg. - return arg_type.args[0] - elif arg_type.args: - # TODO try to map type arguments to Iterable - return arg_type.args[0] + expanded_actual = mapper.expand_actual_type( + actual_arg_type, + arg_kinds[actual], + callee.arg_names[i], + callee.arg_kinds[i], + allow_unpack=True, + ) + + if arg_kinds[actual] != ARG_STAR or isinstance( + get_proper_type(actual_arg_type), TupleType + ): + actual_types.append(expanded_actual) + else: + # If we are expanding an iterable inside * actual, append a homogeneous item instead + actual_types.append( + UnpackType(tuple_instance.copy_modified(args=[expanded_actual])) + ) + + if isinstance(unpacked_type, TypeVarTupleType): + constraints.append( + Constraint( + unpacked_type, + SUPERTYPE_OF, + TupleType(actual_types, unpacked_type.tuple_fallback), + ) + ) + elif isinstance(unpacked_type, TupleType): + # Prefixes get converted to positional args, so technically the only case we + # should have here is like Tuple[Unpack[Ts], Y1, Y2, Y3]. If this turns out + # not to hold we can always handle the prefixes too. + inner_unpack = unpacked_type.items[0] + assert isinstance(inner_unpack, UnpackType) + inner_unpacked_type = get_proper_type(inner_unpack.type) + suffix_len = len(unpacked_type.items) - 1 + if isinstance(inner_unpacked_type, TypeVarTupleType): + # Variadic item can be either *Ts... + constraints.append( + Constraint( + inner_unpacked_type, + SUPERTYPE_OF, + TupleType( + actual_types[:-suffix_len], inner_unpacked_type.tuple_fallback + ), + ) + ) + else: + # ...or it can be a homogeneous tuple. + assert ( + isinstance(inner_unpacked_type, Instance) + and inner_unpacked_type.type.fullname == "builtins.tuple" + ) + for at in actual_types[:-suffix_len]: + constraints.extend( + infer_constraints(inner_unpacked_type.args[0], at, SUPERTYPE_OF) + ) + # Now handle the suffix (if any). + if suffix_len: + for tt, at in zip(unpacked_type.items[1:], actual_types[-suffix_len:]): + constraints.extend(infer_constraints(tt, at, SUPERTYPE_OF)) else: - return AnyType() - elif isinstance(arg_type, TupleType): - # Get the next tuple item of a tuple *arg. - tuplet = cast(TupleType, arg_type) - tuple_counter[0] += 1 - return tuplet.items[tuple_counter[0] - 1] - else: - return AnyType() - elif kind == nodes.ARG_STAR2: - if isinstance(arg_type, Instance) and ( - (cast(Instance, arg_type)).type.fullname() == 'builtins.dict'): - # Dict **arg. TODO more general (Mapping) - return (cast(Instance, arg_type)).args[1] + assert False, "mypy bug: unhandled constraint inference case" else: - return AnyType() - else: - # No translation for other kinds. - return arg_type + for actual in actuals: + actual_arg_type = arg_types[actual] + if actual_arg_type is None: + continue + + if param_spec and callee.arg_kinds[i] in (ARG_STAR, ARG_STAR2): + # If actual arguments are mapped to ParamSpec type, we can't infer individual + # constraints, instead store them and infer single constraint at the end. + # It is impossible to map actual kind to formal kind, so use some heuristic. + # This inference is used as a fallback, so relying on heuristic should be OK. + if not incomplete_star_mapping: + param_spec_arg_types.append( + mapper.expand_actual_type( + actual_arg_type, arg_kinds[actual], None, arg_kinds[actual] + ) + ) + actual_kind = arg_kinds[actual] + param_spec_arg_kinds.append( + ARG_POS if actual_kind not in (ARG_STAR, ARG_STAR2) else actual_kind + ) + param_spec_arg_names.append(arg_names[actual] if arg_names else None) + else: + actual_type = mapper.expand_actual_type( + actual_arg_type, + arg_kinds[actual], + callee.arg_names[i], + callee.arg_kinds[i], + ) + c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) + constraints.extend(c) + if ( + param_spec + and not any(c.type_var == param_spec.id for c in constraints) + and not incomplete_star_mapping + ): + # Use ParamSpec constraint from arguments only if there are no other constraints, + # since as explained above it is quite ad-hoc. + constraints.append( + Constraint( + param_spec, + SUPERTYPE_OF, + Parameters( + arg_types=param_spec_arg_types, + arg_kinds=param_spec_arg_kinds, + arg_names=param_spec_arg_names, + imprecise_arg_kinds=True, + ), + ) + ) + if any(isinstance(v, ParamSpecType) for v in callee.variables): + # As a perf optimization filter imprecise constraints only when we can have them. + constraints = filter_imprecise_kinds(constraints) + return constraints -def infer_constraints(template: Type, actual: Type, - direction: int) -> List[Constraint]: +def infer_constraints( + template: Type, actual: Type, direction: int, skip_neg_op: bool = False +) -> list[Constraint]: """Infer type constraints. Match a template type, which may contain type variable references, recursively against a type which does not contain (the same) type variable references. The result is a list of type constrains of form 'T is a supertype/subtype of x', where T is a type variable - present in the the template and x is a type without reference to - type variables present in the template. + present in the template and x is a type without reference to type + variables present in the template. Assume T and S are type variables. Now the following results can be calculated (read as '(template, actual) --> result'): @@ -117,161 +294,1051 @@ def infer_constraints(template: Type, actual: Type, ((T, S), (X, Y)) --> T :> X and S :> Y (X[T], Any) --> T <: Any and T :> Any - The constraints are represented as Constraint objects. + The constraints are represented as Constraint objects. If skip_neg_op == True, + then skip adding reverse (polymorphic) constraints (since this is already a call + to infer such constraints). + """ + if any( + get_proper_type(template) == get_proper_type(t) + and get_proper_type(actual) == get_proper_type(a) + for (t, a) in reversed(type_state.inferring) + ): + return [] + if has_recursive_types(template) or isinstance(get_proper_type(template), Instance): + # This case requires special care because it may cause infinite recursion. + # Note that we include Instances because the may be recursive as str(Sequence[str]). + if not has_type_vars(template): + # Return early on an empty branch. + return [] + type_state.inferring.append((template, actual)) + res = _infer_constraints(template, actual, direction, skip_neg_op) + type_state.inferring.pop() + return res + return _infer_constraints(template, actual, direction, skip_neg_op) + + +def _infer_constraints( + template: Type, actual: Type, direction: int, skip_neg_op: bool +) -> list[Constraint]: + orig_template = template + template = get_proper_type(template) + actual = get_proper_type(actual) + + # Type inference shouldn't be affected by whether union types have been simplified. + # We however keep any ErasedType items, so that the caller will see it when using + # checkexpr.has_erased_component(). + if isinstance(template, UnionType): + template = mypy.typeops.make_simplified_union(template.items, keep_erased=True) + if isinstance(actual, UnionType): + actual = mypy.typeops.make_simplified_union(actual.items, keep_erased=True) + + # Ignore Any types from the type suggestion engine to avoid them + # causing us to infer Any in situations where a better job could + # be done otherwise. (This can produce false positives but that + # doesn't really matter because it is all heuristic anyway.) + if isinstance(actual, AnyType) and actual.type_of_any == TypeOfAny.suggestion_engine: + return [] + + # If the template is simply a type variable, emit a Constraint directly. + # We need to handle this case before handling Unions for two reasons: + # 1. "T <: Union[U1, U2]" is not equivalent to "T <: U1 or T <: U2", + # because T can itself be a union (notably, Union[U1, U2] itself). + # 2. "T :> Union[U1, U2]" is logically equivalent to "T :> U1 and + # T :> U2", but they are not equivalent to the constraint solver, + # which never introduces new Union types (it uses join() instead). + if isinstance(template, TypeVarType): + return [Constraint(template, direction, actual)] + + if ( + isinstance(actual, TypeVarType) + and not actual.id.is_meta_var() + and direction == SUPERTYPE_OF + ): + # Unless template is also a type variable (or a union that contains one), using the upper + # bound for inference will usually give better result for actual that is a type variable. + if not isinstance(template, UnionType) or not any( + isinstance(t, TypeVarType) for t in template.items + ): + actual = get_proper_type(actual.upper_bound) + + # Now handle the case of either template or actual being a Union. + # For a Union to be a subtype of another type, every item of the Union + # must be a subtype of that type, so concatenate the constraints. + if direction == SUBTYPE_OF and isinstance(template, UnionType): + res = [] + for t_item in template.items: + res.extend(infer_constraints(t_item, actual, direction)) + return res + if direction == SUPERTYPE_OF and isinstance(actual, UnionType): + res = [] + for a_item in actual.items: + res.extend(infer_constraints(orig_template, a_item, direction)) + return res + + # Now the potential subtype is known not to be a Union or a type + # variable that we are solving for. In that case, for a Union to + # be a supertype of the potential subtype, some item of the Union + # must be a supertype of it. + if direction == SUBTYPE_OF and isinstance(actual, UnionType): + # If some of items is not a complete type, disregard that. + items = simplify_away_incomplete_types(actual.items) + # We infer constraints eagerly -- try to find constraints for a type + # variable if possible. This seems to help with some real-world + # use cases. + return any_constraints( + [infer_constraints_if_possible(template, a_item, direction) for a_item in items], + eager=True, + ) + if direction == SUPERTYPE_OF and isinstance(template, UnionType): + # When the template is a union, we are okay with leaving some + # type variables indeterminate. This helps with some special + # cases, though this isn't very principled. + result = any_constraints( + [ + infer_constraints_if_possible(t_item, actual, direction) + for t_item in template.items + ], + eager=False, + ) + if result: + return result + elif has_recursive_types(template) and not has_recursive_types(actual): + return handle_recursive_union(template, actual, direction) + return [] + + # Remaining cases are handled by ConstraintBuilderVisitor. + return template.accept(ConstraintBuilderVisitor(actual, direction, skip_neg_op)) + + +def infer_constraints_if_possible( + template: Type, actual: Type, direction: int +) -> list[Constraint] | None: + """Like infer_constraints, but return None if the input relation is + known to be unsatisfiable, for example if template=List[T] and actual=int. + (In this case infer_constraints would return [], just like it would for + an automatically satisfied relation like template=List[T] and actual=object.) + """ + if direction == SUBTYPE_OF and not mypy.subtypes.is_subtype(erase_typevars(template), actual): + return None + if direction == SUPERTYPE_OF and not mypy.subtypes.is_subtype( + actual, erase_typevars(template) + ): + return None + if ( + direction == SUPERTYPE_OF + and isinstance(template, TypeVarType) + and not mypy.subtypes.is_subtype(actual, erase_typevars(template.upper_bound)) + ): + # This is not caught by the above branch because of the erase_typevars() call, + # that would return 'Any' for a type variable. + return None + return infer_constraints(template, actual, direction) + + +def select_trivial(options: Sequence[list[Constraint] | None]) -> list[list[Constraint]]: + """Select only those lists where each item is a constraint against Any.""" + res = [] + for option in options: + if option is None: + continue + if all(isinstance(get_proper_type(c.target), AnyType) for c in option): + res.append(option) + return res + + +def merge_with_any(constraint: Constraint) -> Constraint: + """Transform a constraint target into a union with given Any type.""" + target = constraint.target + if is_union_with_any(target): + # Do not produce redundant unions. + return constraint + # TODO: if we will support multiple sources Any, use this here instead. + any_type = AnyType(TypeOfAny.implementation_artifact) + return Constraint( + constraint.origin_type_var, + constraint.op, + UnionType.make_union([target, any_type], target.line, target.column), + ) + + +def handle_recursive_union(template: UnionType, actual: Type, direction: int) -> list[Constraint]: + # This is a hack to special-case things like Union[T, Inst[T]] in recursive types. Although + # it is quite arbitrary, it is a relatively common pattern, so we should handle it well. + # This function may be called when inferring against such union resulted in different + # constraints for each item. Normally we give up in such case, but here we instead split + # the union in two parts, and try inferring sequentially. + non_type_var_items = [t for t in template.items if not isinstance(t, TypeVarType)] + type_var_items = [t for t in template.items if isinstance(t, TypeVarType)] + return infer_constraints( + UnionType.make_union(non_type_var_items), actual, direction + ) or infer_constraints(UnionType.make_union(type_var_items), actual, direction) + + +def any_constraints(options: list[list[Constraint] | None], eager: bool) -> list[Constraint]: + """Deduce what we can from a collection of constraint lists. + + It's a given that at least one of the lists must be satisfied. A + None element in the list of options represents an unsatisfiable + constraint and is ignored. Ignore empty constraint lists if eager + is true -- they are always trivially satisfiable. + """ + if eager: + valid_options = [option for option in options if option] + else: + valid_options = [option for option in options if option is not None] + + if not valid_options: + return [] + + if len(valid_options) == 1: + return valid_options[0] + + if all(is_same_constraints(valid_options[0], c) for c in valid_options[1:]): + # Multiple sets of constraints that are all the same. Just pick any one of them. + return valid_options[0] + + if all(is_similar_constraints(valid_options[0], c) for c in valid_options[1:]): + # All options have same structure. In this case we can merge-in trivial + # options (i.e. those that only have Any) and try again. + # TODO: More generally, if a given (variable, direction) pair appears in + # every option, combine the bounds with meet/join always, not just for Any. + trivial_options = select_trivial(valid_options) + if trivial_options and len(trivial_options) < len(valid_options): + merged_options = [] + for option in valid_options: + if option in trivial_options: + continue + if option is not None: + merged_option: list[Constraint] | None = [merge_with_any(c) for c in option] + else: + merged_option = None + merged_options.append(merged_option) + return any_constraints(list(merged_options), eager) + + # If normal logic didn't work, try excluding trivially unsatisfiable constraint (due to + # upper bounds) from each option, and comparing them again. + filtered_options = [filter_satisfiable(o) for o in options] + if filtered_options != options: + return any_constraints(filtered_options, eager=eager) + + # Otherwise, there are either no valid options or multiple, inconsistent valid + # options. Give up and deduce nothing. + return [] + + +def filter_satisfiable(option: list[Constraint] | None) -> list[Constraint] | None: + """Keep only constraints that can possibly be satisfied. + + Currently, we filter out constraints where target is not a subtype of the upper bound. + Since those can be never satisfied. We may add more cases in future if it improves type + inference. + """ + if not option: + return option + satisfiable = [] + for c in option: + if isinstance(c.origin_type_var, TypeVarType) and c.origin_type_var.values: + if any( + mypy.subtypes.is_subtype(c.target, value) for value in c.origin_type_var.values + ): + satisfiable.append(c) + elif mypy.subtypes.is_subtype(c.target, c.origin_type_var.upper_bound): + satisfiable.append(c) + if not satisfiable: + return None + return satisfiable + + +def is_same_constraints(x: list[Constraint], y: list[Constraint]) -> bool: + for c1 in x: + if not any(is_same_constraint(c1, c2) for c2 in y): + return False + for c1 in y: + if not any(is_same_constraint(c1, c2) for c2 in x): + return False + return True + + +def is_same_constraint(c1: Constraint, c2: Constraint) -> bool: + # Ignore direction when comparing constraints against Any. + skip_op_check = isinstance(get_proper_type(c1.target), AnyType) and isinstance( + get_proper_type(c2.target), AnyType + ) + return ( + c1.type_var == c2.type_var + and (c1.op == c2.op or skip_op_check) + and mypy.subtypes.is_same_type(c1.target, c2.target) + ) + + +def is_similar_constraints(x: list[Constraint], y: list[Constraint]) -> bool: + """Check that two lists of constraints have similar structure. + + This means that each list has same type variable plus direction pairs (i.e we + ignore the target). Except for constraints where target is Any type, there + we ignore direction as well. + """ + return _is_similar_constraints(x, y) and _is_similar_constraints(y, x) + + +def _is_similar_constraints(x: list[Constraint], y: list[Constraint]) -> bool: + """Check that every constraint in the first list has a similar one in the second. + + See docstring above for definition of similarity. """ + for c1 in x: + has_similar = False + for c2 in y: + # Ignore direction when either constraint is against Any. + skip_op_check = isinstance(get_proper_type(c1.target), AnyType) or isinstance( + get_proper_type(c2.target), AnyType + ) + if c1.type_var == c2.type_var and (c1.op == c2.op or skip_op_check): + has_similar = True + break + if not has_similar: + return False + return True - return template.accept(ConstraintBuilderVisitor(actual, direction)) + +def simplify_away_incomplete_types(types: Iterable[Type]) -> list[Type]: + complete = [typ for typ in types if is_complete_type(typ)] + if complete: + return complete + else: + return list(types) + + +def is_complete_type(typ: Type) -> bool: + """Is a type complete? + + A complete doesn't have uninhabited type components or (when not in strict + optional mode) None components. + """ + return typ.accept(CompleteTypeVisitor()) + + +class CompleteTypeVisitor(TypeQuery[bool]): + def __init__(self) -> None: + super().__init__(all) + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return False class ConstraintBuilderVisitor(TypeVisitor[List[Constraint]]): """Visitor class for inferring type constraints.""" # The type that is compared against a template - actual = Undefined(Type) + # TODO: The value may be None. Is that actually correct? + actual: ProperType - def __init__(self, actual: Type, direction: int) -> None: + def __init__(self, actual: ProperType, direction: int, skip_neg_op: bool) -> None: # Direction must be SUBTYPE_OF or SUPERTYPE_OF. self.actual = actual self.direction = direction + # Whether to skip polymorphic inference (involves inference in opposite direction) + # this is used to prevent infinite recursion when both template and actual are + # generic callables. + self.skip_neg_op = skip_neg_op # Trivial leaf types - def visit_unbound_type(self, template: UnboundType) -> List[Constraint]: + def visit_unbound_type(self, template: UnboundType) -> list[Constraint]: return [] - def visit_any(self, template: AnyType) -> List[Constraint]: + def visit_any(self, template: AnyType) -> list[Constraint]: return [] - def visit_void(self, template: Void) -> List[Constraint]: + def visit_none_type(self, template: NoneType) -> list[Constraint]: return [] - def visit_none_type(self, template: NoneTyp) -> List[Constraint]: + def visit_uninhabited_type(self, template: UninhabitedType) -> list[Constraint]: return [] - def visit_erased_type(self, template: ErasedType) -> List[Constraint]: + def visit_erased_type(self, template: ErasedType) -> list[Constraint]: return [] + def visit_deleted_type(self, template: DeletedType) -> list[Constraint]: + return [] + + def visit_literal_type(self, template: LiteralType) -> list[Constraint]: + return [] + + # Errors + + def visit_partial_type(self, template: PartialType) -> list[Constraint]: + # We can't do anything useful with a partial type here. + assert False, "Internal error" + # Non-trivial leaf type - def visit_type_var(self, template: TypeVar) -> List[Constraint]: - return [Constraint(template.id, self.direction, self.actual)] + def visit_type_var(self, template: TypeVarType) -> list[Constraint]: + assert False, ( + "Unexpected TypeVarType in ConstraintBuilderVisitor" + " (should have been handled in infer_constraints)" + ) + + def visit_param_spec(self, template: ParamSpecType) -> list[Constraint]: + # Can't infer ParamSpecs from component values (only via Callable[P, T]). + return [] + + def visit_type_var_tuple(self, template: TypeVarTupleType) -> list[Constraint]: + raise NotImplementedError + + def visit_unpack_type(self, template: UnpackType) -> list[Constraint]: + raise RuntimeError("Mypy bug: unpack should be handled at a higher level.") + + def visit_parameters(self, template: Parameters) -> list[Constraint]: + # Constraining Any against C[P] turns into infer_against_any([P], Any) + if isinstance(self.actual, AnyType): + return self.infer_against_any(template.arg_types, self.actual) + if type_state.infer_polymorphic and isinstance(self.actual, Parameters): + # For polymorphic inference we need to be able to infer secondary constraints + # in situations like [x: T] <: P <: [x: int]. + return infer_callable_arguments_constraints(template, self.actual, self.direction) + if type_state.infer_polymorphic and isinstance(self.actual, ParamSpecType): + # Similar for [x: T] <: Q <: Concatenate[int, P]. + return infer_callable_arguments_constraints( + template, self.actual.prefix, self.direction + ) + # There also may be unpatched types after a user error, simply ignore them. + return [] # Non-leaf types - def visit_instance(self, template: Instance) -> List[Constraint]: - actual = self.actual - res = [] # type: List[Constraint] + def visit_instance(self, template: Instance) -> list[Constraint]: + original_actual = actual = self.actual + res: list[Constraint] = [] + if isinstance(actual, (CallableType, Overloaded)) and template.type.is_protocol: + if "__call__" in template.type.protocol_members: + # Special case: a generic callback protocol + if not any(template == t for t in template.type.inferring): + template.type.inferring.append(template) + call = mypy.subtypes.find_member( + "__call__", template, actual, is_operator=True + ) + assert call is not None + if mypy.subtypes.is_subtype(actual, erase_typevars(call)): + subres = infer_constraints(call, actual, self.direction) + res.extend(subres) + template.type.inferring.pop() + if isinstance(actual, CallableType) and actual.fallback is not None: + if actual.is_type_obj() and template.type.is_protocol: + ret_type = get_proper_type(actual.ret_type) + if isinstance(ret_type, TupleType): + ret_type = mypy.typeops.tuple_fallback(ret_type) + if isinstance(ret_type, Instance): + if self.direction == SUBTYPE_OF: + subtype = template + else: + subtype = ret_type + res.extend( + self.infer_constraints_from_protocol_members( + ret_type, template, subtype, template, class_obj=True + ) + ) + actual = actual.fallback + if isinstance(actual, TypeType) and template.type.is_protocol: + if isinstance(actual.item, Instance): + if self.direction == SUBTYPE_OF: + subtype = template + else: + subtype = actual.item + res.extend( + self.infer_constraints_from_protocol_members( + actual.item, template, subtype, template, class_obj=True + ) + ) + if self.direction == SUPERTYPE_OF: + # Infer constraints for Type[T] via metaclass of T when it makes sense. + a_item = actual.item + if isinstance(a_item, TypeVarType): + a_item = get_proper_type(a_item.upper_bound) + if isinstance(a_item, Instance) and a_item.type.metaclass_type: + res.extend( + self.infer_constraints_from_protocol_members( + a_item.type.metaclass_type, template, actual, template + ) + ) + + if isinstance(actual, Overloaded) and actual.fallback is not None: + actual = actual.fallback + if isinstance(actual, TypedDictType): + actual = actual.as_anonymous().fallback + if isinstance(actual, LiteralType): + actual = actual.fallback if isinstance(actual, Instance): - instance = cast(Instance, actual) - if (self.direction == SUBTYPE_OF and - template.type.has_base(instance.type.fullname())): + instance = actual + erased = erase_typevars(template) + assert isinstance(erased, Instance) # type: ignore[misc] + # We always try nominal inference if possible, + # it is much faster than the structural one. + if self.direction == SUBTYPE_OF and template.type.has_base(instance.type.fullname): mapped = map_instance_to_supertype(template, instance.type) - for i in range(len(instance.args)): - # The constraints for generic type parameters are - # invariant. Include the default constraint and its - # negation to achieve the effect. - cb = infer_constraints(mapped.args[i], instance.args[i], - self.direction) - res.extend(cb) - res.extend(negate_constraints(cb)) + tvars = mapped.type.defn.type_vars + + if instance.type.has_type_var_tuple_type: + # Variadic types need special handling to map each type argument to + # the correct corresponding type variable. + assert instance.type.type_var_tuple_prefix is not None + assert instance.type.type_var_tuple_suffix is not None + prefix_len = instance.type.type_var_tuple_prefix + suffix_len = instance.type.type_var_tuple_suffix + tvt = instance.type.defn.type_vars[prefix_len] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + i_prefix, i_middle, i_suffix = split_with_prefix_and_suffix( + instance.args, prefix_len, suffix_len + ) + m_prefix, m_middle, m_suffix = split_with_prefix_and_suffix( + mapped.args, prefix_len, suffix_len + ) + instance_args = i_prefix + (TupleType(list(i_middle), fallback),) + i_suffix + mapped_args = m_prefix + (TupleType(list(m_middle), fallback),) + m_suffix + else: + mapped_args = mapped.args + instance_args = instance.args + + # N.B: We use zip instead of indexing because the lengths might have + # mismatches during daemon reprocessing. + for tvar, mapped_arg, instance_arg in zip(tvars, mapped_args, instance_args): + if isinstance(tvar, TypeVarType): + # The constraints for generic type parameters depend on variance. + # Include constraints from both directions if invariant. + if tvar.variance != CONTRAVARIANT: + res.extend(infer_constraints(mapped_arg, instance_arg, self.direction)) + if tvar.variance != COVARIANT: + res.extend( + infer_constraints(mapped_arg, instance_arg, neg_op(self.direction)) + ) + elif isinstance(tvar, ParamSpecType) and isinstance(mapped_arg, ParamSpecType): + prefix = mapped_arg.prefix + if isinstance(instance_arg, Parameters): + # No such thing as variance for ParamSpecs, consider them invariant + # TODO: constraints between prefixes using + # infer_callable_arguments_constraints() + suffix: Type = instance_arg.copy_modified( + instance_arg.arg_types[len(prefix.arg_types) :], + instance_arg.arg_kinds[len(prefix.arg_kinds) :], + instance_arg.arg_names[len(prefix.arg_names) :], + ) + res.append(Constraint(mapped_arg, SUBTYPE_OF, suffix)) + res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix)) + elif isinstance(instance_arg, ParamSpecType): + suffix = instance_arg.copy_modified( + prefix=Parameters( + instance_arg.prefix.arg_types[len(prefix.arg_types) :], + instance_arg.prefix.arg_kinds[len(prefix.arg_kinds) :], + instance_arg.prefix.arg_names[len(prefix.arg_names) :], + ) + ) + res.append(Constraint(mapped_arg, SUBTYPE_OF, suffix)) + res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix)) + elif isinstance(tvar, TypeVarTupleType): + # Handle variadic type variables covariantly for consistency. + res.extend(infer_constraints(mapped_arg, instance_arg, self.direction)) + return res - elif (self.direction == SUPERTYPE_OF and - instance.type.has_base(template.type.fullname())): + elif self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname): mapped = map_instance_to_supertype(instance, template.type) - for j in range(len(template.args)): - # The constraints for generic type parameters are - # invariant. - cb = infer_constraints(template.args[j], mapped.args[j], - self.direction) - res.extend(cb) - res.extend(negate_constraints(cb)) + tvars = template.type.defn.type_vars + if template.type.has_type_var_tuple_type: + # Variadic types need special handling to map each type argument to + # the correct corresponding type variable. + assert template.type.type_var_tuple_prefix is not None + assert template.type.type_var_tuple_suffix is not None + prefix_len = template.type.type_var_tuple_prefix + suffix_len = template.type.type_var_tuple_suffix + tvt = template.type.defn.type_vars[prefix_len] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix( + template.args, prefix_len, suffix_len + ) + m_prefix, m_middle, m_suffix = split_with_prefix_and_suffix( + mapped.args, prefix_len, suffix_len + ) + template_args = t_prefix + (TupleType(list(t_middle), fallback),) + t_suffix + mapped_args = m_prefix + (TupleType(list(m_middle), fallback),) + m_suffix + else: + mapped_args = mapped.args + template_args = template.args + # N.B: We use zip instead of indexing because the lengths might have + # mismatches during daemon reprocessing. + for tvar, mapped_arg, template_arg in zip(tvars, mapped_args, template_args): + if isinstance(tvar, TypeVarType): + # The constraints for generic type parameters depend on variance. + # Include constraints from both directions if invariant. + if tvar.variance != CONTRAVARIANT: + res.extend(infer_constraints(template_arg, mapped_arg, self.direction)) + if tvar.variance != COVARIANT: + res.extend( + infer_constraints(template_arg, mapped_arg, neg_op(self.direction)) + ) + elif isinstance(tvar, ParamSpecType) and isinstance( + template_arg, ParamSpecType + ): + prefix = template_arg.prefix + if isinstance(mapped_arg, Parameters): + # No such thing as variance for ParamSpecs, consider them invariant + # TODO: constraints between prefixes using + # infer_callable_arguments_constraints() + suffix = mapped_arg.copy_modified( + mapped_arg.arg_types[len(prefix.arg_types) :], + mapped_arg.arg_kinds[len(prefix.arg_kinds) :], + mapped_arg.arg_names[len(prefix.arg_names) :], + ) + res.append(Constraint(template_arg, SUBTYPE_OF, suffix)) + res.append(Constraint(template_arg, SUPERTYPE_OF, suffix)) + elif isinstance(mapped_arg, ParamSpecType): + suffix = mapped_arg.copy_modified( + prefix=Parameters( + mapped_arg.prefix.arg_types[len(prefix.arg_types) :], + mapped_arg.prefix.arg_kinds[len(prefix.arg_kinds) :], + mapped_arg.prefix.arg_names[len(prefix.arg_names) :], + ) + ) + res.append(Constraint(template_arg, SUBTYPE_OF, suffix)) + res.append(Constraint(template_arg, SUPERTYPE_OF, suffix)) + elif isinstance(tvar, TypeVarTupleType): + # Consider variadic type variables to be invariant. + res.extend(infer_constraints(template_arg, mapped_arg, SUBTYPE_OF)) + res.extend(infer_constraints(template_arg, mapped_arg, SUPERTYPE_OF)) return res + if ( + template.type.is_protocol + and self.direction == SUPERTYPE_OF + and + # We avoid infinite recursion for structural subtypes by checking + # whether this type already appeared in the inference chain. + # This is a conservative way to break the inference cycles. + # It never produces any "false" constraints but gives up soon + # on purely structural inference cycles, see #3829. + # Note that we use is_protocol_implementation instead of is_subtype + # because some type may be considered a subtype of a protocol + # due to _promote, but still not implement the protocol. + not any(template == t for t in reversed(template.type.inferring)) + and mypy.subtypes.is_protocol_implementation(instance, erased, skip=["__call__"]) + ): + template.type.inferring.append(template) + res.extend( + self.infer_constraints_from_protocol_members( + instance, template, original_actual, template + ) + ) + template.type.inferring.pop() + return res + elif ( + instance.type.is_protocol + and self.direction == SUBTYPE_OF + and + # We avoid infinite recursion for structural subtypes also here. + not any(instance == i for i in reversed(instance.type.inferring)) + and mypy.subtypes.is_protocol_implementation(erased, instance, skip=["__call__"]) + ): + instance.type.inferring.append(instance) + res.extend( + self.infer_constraints_from_protocol_members( + instance, template, template, instance + ) + ) + instance.type.inferring.pop() + return res + if res: + return res + if isinstance(actual, AnyType): - # IDEA: Include both ways, i.e. add negation as well? - return self.infer_against_any(template.args) - if (isinstance(actual, TupleType) and - (is_named_instance(template, 'typing.Iterable') or - is_named_instance(template, 'typing.Sequence') or - is_named_instance(template, 'typing.Reversible')) - and self.direction == SUPERTYPE_OF): - actual = cast(TupleType, actual) + return self.infer_against_any(template.args, actual) + if ( + isinstance(actual, TupleType) + and is_named_instance(template, TUPLE_LIKE_INSTANCE_NAMES) + and self.direction == SUPERTYPE_OF + ): for item in actual.items: + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, TypeVarTupleType): + # Cannot infer anything for T from [T, ...] <: *Ts + continue + assert ( + isinstance(unpacked, Instance) + and unpacked.type.fullname == "builtins.tuple" + ) + item = unpacked.args[0] cb = infer_constraints(template.args[0], item, SUPERTYPE_OF) res.extend(cb) return res + elif isinstance(actual, TupleType) and self.direction == SUPERTYPE_OF: + return infer_constraints(template, mypy.typeops.tuple_fallback(actual), self.direction) + elif isinstance(actual, TypeVarType): + if not actual.values and not actual.id.is_meta_var(): + return infer_constraints(template, actual.upper_bound, self.direction) + return [] + elif isinstance(actual, ParamSpecType): + return infer_constraints(template, actual.upper_bound, self.direction) + elif isinstance(actual, TypeVarTupleType): + raise NotImplementedError else: return [] - def visit_callable(self, template: Callable) -> List[Constraint]: - if isinstance(self.actual, Callable): - cactual = cast(Callable, self.actual) - # FIX verify argument counts - # FIX what if one of the functions is generic - res = [] # type: List[Constraint] - for i in range(len(template.arg_types)): - # Negate constraints due function argument type contravariance. - res.extend(negate_constraints(infer_constraints( - template.arg_types[i], cactual.arg_types[i], - self.direction))) - res.extend(infer_constraints(template.ret_type, cactual.ret_type, - self.direction)) + def infer_constraints_from_protocol_members( + self, + instance: Instance, + template: Instance, + subtype: Type, + protocol: Instance, + class_obj: bool = False, + ) -> list[Constraint]: + """Infer constraints for situations where either 'template' or 'instance' is a protocol. + + The 'protocol' is the one of two that is an instance of protocol type, 'subtype' + is the type used to bind self during inference. Currently, we just infer constrains for + every protocol member type (both ways for settable members). + """ + res = [] + for member in protocol.type.protocol_members: + inst = mypy.subtypes.find_member(member, instance, subtype, class_obj=class_obj) + temp = mypy.subtypes.find_member(member, template, subtype) + if inst is None or temp is None: + if member == "__call__": + continue + return [] # See #11020 + # The above is safe since at this point we know that 'instance' is a subtype + # of (erased) 'template', therefore it defines all protocol members + res.extend(infer_constraints(temp, inst, self.direction)) + if mypy.subtypes.IS_SETTABLE in mypy.subtypes.get_member_flags(member, protocol): + # Settable members are invariant, add opposite constraints + res.extend(infer_constraints(temp, inst, neg_op(self.direction))) + return res + + def visit_callable_type(self, template: CallableType) -> list[Constraint]: + # Normalize callables before matching against each other. + # Note that non-normalized callables can be created in annotations + # using e.g. callback protocols. + # TODO: check that callables match? Ideally we should not infer constraints + # callables that can never be subtypes of one another in given direction. + template = template.with_unpacked_kwargs() + extra_tvars = False + if isinstance(self.actual, CallableType): + res: list[Constraint] = [] + cactual = self.actual.with_unpacked_kwargs() + param_spec = template.param_spec() + + template_ret_type, cactual_ret_type = template.ret_type, cactual.ret_type + if template.type_guard is not None and cactual.type_guard is not None: + template_ret_type = template.type_guard + cactual_ret_type = cactual.type_guard + + if template.type_is is not None and cactual.type_is is not None: + template_ret_type = template.type_is + cactual_ret_type = cactual.type_is + + res.extend(infer_constraints(template_ret_type, cactual_ret_type, self.direction)) + + if param_spec is None: + # TODO: Erase template variables if it is generic? + if ( + type_state.infer_polymorphic + and cactual.variables + and not self.skip_neg_op + # Technically, the correct inferred type for application of e.g. + # Callable[..., T] -> Callable[..., T] (with literal ellipsis), to a generic + # like U -> U, should be Callable[..., Any], but if U is a self-type, we can + # allow it to leak, to be later bound to self. A bunch of existing code + # depends on this old behaviour. + and not any(tv.id.is_self() for tv in cactual.variables) + ): + # If the actual callable is generic, infer constraints in the opposite + # direction, and indicate to the solver there are extra type variables + # to solve for (see more details in mypy/solve.py). + res.extend( + infer_constraints( + cactual, template, neg_op(self.direction), skip_neg_op=True + ) + ) + extra_tvars = True + + # We can't infer constraints from arguments if the template is Callable[..., T] + # (with literal '...'). + if not template.is_ellipsis_args: + unpack_present = find_unpack_in_list(template.arg_types) + # When both ParamSpec and TypeVarTuple are present, things become messy + # quickly. For now, we only allow ParamSpec to "capture" TypeVarTuple, + # but not vice versa. + # TODO: infer more from prefixes when possible. + if unpack_present is not None and not cactual.param_spec(): + # We need to re-normalize args to the form they appear in tuples, + # for callables we always pack the suffix inside another tuple. + unpack = template.arg_types[unpack_present] + assert isinstance(unpack, UnpackType) + tuple_type = get_tuple_fallback_from_unpack(unpack) + template_types = repack_callable_args(template, tuple_type) + actual_types = repack_callable_args(cactual, tuple_type) + # Now we can use the same general helper as for tuple types. + unpack_constraints = build_constraints_for_simple_unpack( + template_types, actual_types, neg_op(self.direction) + ) + res.extend(unpack_constraints) + else: + # TODO: do we need some special-casing when unpack is present in actual + # callable but not in template callable? + res.extend( + infer_callable_arguments_constraints(template, cactual, self.direction) + ) + else: + prefix = param_spec.prefix + prefix_len = len(prefix.arg_types) + cactual_ps = cactual.param_spec() + + if type_state.infer_polymorphic and cactual.variables and not self.skip_neg_op: + # Similar logic to the branch above. + res.extend( + infer_constraints( + cactual, template, neg_op(self.direction), skip_neg_op=True + ) + ) + extra_tvars = True + + # Compare prefixes as well + cactual_prefix = cactual.copy_modified( + arg_types=cactual.arg_types[:prefix_len], + arg_kinds=cactual.arg_kinds[:prefix_len], + arg_names=cactual.arg_names[:prefix_len], + ) + res.extend( + infer_callable_arguments_constraints(prefix, cactual_prefix, self.direction) + ) + + param_spec_target: Type | None = None + if not cactual_ps: + max_prefix_len = len([k for k in cactual.arg_kinds if k in (ARG_POS, ARG_OPT)]) + prefix_len = min(prefix_len, max_prefix_len) + param_spec_target = Parameters( + arg_types=cactual.arg_types[prefix_len:], + arg_kinds=cactual.arg_kinds[prefix_len:], + arg_names=cactual.arg_names[prefix_len:], + variables=cactual.variables if not type_state.infer_polymorphic else [], + imprecise_arg_kinds=cactual.imprecise_arg_kinds, + ) + else: + if len(param_spec.prefix.arg_types) <= len(cactual_ps.prefix.arg_types): + param_spec_target = cactual_ps.copy_modified( + prefix=Parameters( + arg_types=cactual_ps.prefix.arg_types[prefix_len:], + arg_kinds=cactual_ps.prefix.arg_kinds[prefix_len:], + arg_names=cactual_ps.prefix.arg_names[prefix_len:], + imprecise_arg_kinds=cactual_ps.prefix.imprecise_arg_kinds, + ) + ) + if param_spec_target is not None: + res.append(Constraint(param_spec, self.direction, param_spec_target)) + if extra_tvars: + for c in res: + c.extra_tvars += cactual.variables return res elif isinstance(self.actual, AnyType): - # FIX what if generic - res = self.infer_against_any(template.arg_types) - res.extend(infer_constraints(template.ret_type, AnyType(), - self.direction)) + param_spec = template.param_spec() + any_type = AnyType(TypeOfAny.from_another_any, source_any=self.actual) + if param_spec is None: + # FIX what if generic + res = self.infer_against_any(template.arg_types, self.actual) + else: + res = [ + Constraint( + param_spec, + SUBTYPE_OF, + Parameters([any_type, any_type], [ARG_STAR, ARG_STAR2], [None, None]), + ) + ] + res.extend(infer_constraints(template.ret_type, any_type, self.direction)) return res elif isinstance(self.actual, Overloaded): - return self.infer_against_overloaded(cast(Overloaded, self.actual), - template) + return self.infer_against_overloaded(self.actual, template) + elif isinstance(self.actual, TypeType): + return infer_constraints(template.ret_type, self.actual.item, self.direction) + elif isinstance(self.actual, Instance): + # Instances with __call__ method defined are considered structural + # subtypes of Callable with a compatible signature. + call = mypy.subtypes.find_member( + "__call__", self.actual, self.actual, is_operator=True + ) + if call: + return infer_constraints(template, call, self.direction) + else: + return [] else: return [] - def infer_against_overloaded(self, overloaded: Overloaded, - template: Callable) -> List[Constraint]: + def infer_against_overloaded( + self, overloaded: Overloaded, template: CallableType + ) -> list[Constraint]: # Create constraints by matching an overloaded type against a template. # This is tricky to do in general. We cheat by only matching against - # the first overload item, and by only matching the return type. This + # the first overload item that is callable compatible. This # seems to work somewhat well, but we should really use a more # reliable technique. item = find_matching_overload_item(overloaded, template) - return infer_constraints(template.ret_type, item.ret_type, - self.direction) + return infer_constraints(template, item, self.direction) - def visit_tuple_type(self, template: TupleType) -> List[Constraint]: + def visit_tuple_type(self, template: TupleType) -> list[Constraint]: actual = self.actual - if (isinstance(actual, TupleType) and - len((cast(TupleType, actual)).items) == len(template.items)): - res = [] # type: List[Constraint] - for i in range(len(template.items)): - res.extend(infer_constraints(template.items[i], - cast(TupleType, actual).items[i], - self.direction)) + unpack_index = find_unpack_in_list(template.items) + is_varlength_tuple = ( + isinstance(actual, Instance) and actual.type.fullname == "builtins.tuple" + ) + + if isinstance(actual, TupleType) or is_varlength_tuple: + res: list[Constraint] = [] + if unpack_index is not None: + if is_varlength_tuple: + # Variadic tuple can be only a supertype of a tuple type, but even if + # direction is opposite, inferring something may give better error messages. + unpack_type = template.items[unpack_index] + assert isinstance(unpack_type, UnpackType) + unpacked_type = get_proper_type(unpack_type.type) + if isinstance(unpacked_type, TypeVarTupleType): + res = [ + Constraint(type_var=unpacked_type, op=self.direction, target=actual) + ] + else: + assert ( + isinstance(unpacked_type, Instance) + and unpacked_type.type.fullname == "builtins.tuple" + ) + res = infer_constraints(unpacked_type, actual, self.direction) + assert isinstance(actual, Instance) # ensured by is_varlength_tuple == True + for i, ti in enumerate(template.items): + if i == unpack_index: + # This one we just handled above. + continue + # For Tuple[T, *Ts, S] <: tuple[X, ...] infer also T <: X and S <: X. + res.extend(infer_constraints(ti, actual.args[0], self.direction)) + return res + else: + assert isinstance(actual, TupleType) + unpack_constraints = build_constraints_for_simple_unpack( + template.items, actual.items, self.direction + ) + actual_items: tuple[Type, ...] = () + template_items: tuple[Type, ...] = () + res.extend(unpack_constraints) + elif isinstance(actual, TupleType): + a_unpack_index = find_unpack_in_list(actual.items) + if a_unpack_index is not None: + # The case where template tuple doesn't have an unpack, but actual tuple + # has an unpack. We can infer something if actual unpack is a variadic tuple. + # Tuple[T, S, U] <: tuple[X, *tuple[Y, ...], Z] => T <: X, S <: Y, U <: Z. + a_unpack = actual.items[a_unpack_index] + assert isinstance(a_unpack, UnpackType) + a_unpacked = get_proper_type(a_unpack.type) + if len(actual.items) + 1 <= len(template.items): + a_prefix_len = a_unpack_index + a_suffix_len = len(actual.items) - a_unpack_index - 1 + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix( + tuple(template.items), a_prefix_len, a_suffix_len + ) + actual_items = tuple(actual.items[:a_prefix_len]) + if a_suffix_len: + actual_items += tuple(actual.items[-a_suffix_len:]) + template_items = t_prefix + t_suffix + if isinstance(a_unpacked, Instance): + assert a_unpacked.type.fullname == "builtins.tuple" + for tm in t_middle: + res.extend( + infer_constraints(tm, a_unpacked.args[0], self.direction) + ) + else: + actual_items = () + template_items = () + else: + actual_items = tuple(actual.items) + template_items = tuple(template.items) + else: + return res + + # Cases above will return if actual wasn't a TupleType. + assert isinstance(actual, TupleType) + if len(actual_items) == len(template_items): + if ( + actual.partial_fallback.type.is_named_tuple + and template.partial_fallback.type.is_named_tuple + ): + # For named tuples using just the fallbacks usually gives better results. + return res + infer_constraints( + template.partial_fallback, actual.partial_fallback, self.direction + ) + for i in range(len(template_items)): + res.extend( + infer_constraints(template_items[i], actual_items[i], self.direction) + ) return res elif isinstance(actual, AnyType): - return self.infer_against_any(template.items) + return self.infer_against_any(template.items, actual) else: return [] - def visit_union_type(self, template: UnionType) -> List[Constraint]: - res = [] # type: List[Constraint] - for item in template.items: - res.extend(infer_constraints(item, self.actual, self.direction)) - return res + def visit_typeddict_type(self, template: TypedDictType) -> list[Constraint]: + actual = self.actual + if isinstance(actual, TypedDictType): + res: list[Constraint] = [] + # NOTE: Non-matching keys are ignored. Compatibility is checked + # elsewhere so this shouldn't be unsafe. + for item_name, template_item_type, actual_item_type in template.zip(actual): + res.extend(infer_constraints(template_item_type, actual_item_type, self.direction)) + return res + elif isinstance(actual, AnyType): + return self.infer_against_any(template.items.values(), actual) + else: + return [] - def infer_against_any(self, types: List[Type]) -> List[Constraint]: - res = [] # type: List[Constraint] + def visit_union_type(self, template: UnionType) -> list[Constraint]: + assert False, ( + "Unexpected UnionType in ConstraintBuilderVisitor" + " (should have been handled in infer_constraints)" + ) + + def visit_type_alias_type(self, template: TypeAliasType) -> list[Constraint]: + assert False, f"This should be never called, got {template}" + + def infer_against_any(self, types: Iterable[Type], any_type: AnyType) -> list[Constraint]: + res: list[Constraint] = [] for t in types: - res.extend(infer_constraints(t, AnyType(), self.direction)) + if isinstance(t, UnpackType): + if isinstance(t.type, TypeVarTupleType): + res.append(Constraint(t.type, self.direction, any_type)) + else: + unpacked = get_proper_type(t.type) + assert isinstance(unpacked, Instance) + res.extend(infer_constraints(unpacked, any_type, self.direction)) + else: + # Note that we ignore variance and simply always use the + # original direction. This is because for Any targets direction is + # irrelevant in most cases, see e.g. is_same_constraint(). + res.extend(infer_constraints(t, any_type, self.direction)) return res + def visit_overloaded(self, template: Overloaded) -> list[Constraint]: + if isinstance(self.actual, CallableType): + items = find_matching_overload_items(template, self.actual) + else: + items = template.items + res: list[Constraint] = [] + for t in items: + res.extend(infer_constraints(t, self.actual, self.direction)) + return res -def negate_constraints(constraints: List[Constraint]) -> List[Constraint]: - res = [] # type: List[Constraint] - for c in constraints: - res.append(Constraint(c.type_var, neg_op(c.op), c.target)) - return res + def visit_type_type(self, template: TypeType) -> list[Constraint]: + if isinstance(self.actual, CallableType): + return infer_constraints(template.item, self.actual.ret_type, self.direction) + elif isinstance(self.actual, Overloaded): + return infer_constraints(template.item, self.actual.items[0].ret_type, self.direction) + elif isinstance(self.actual, TypeType): + return infer_constraints(template.item, self.actual.item, self.direction) + elif isinstance(self.actual, AnyType): + return infer_constraints(template.item, self.actual, self.direction) + else: + return [] def neg_op(op: int) -> int: @@ -282,17 +1349,292 @@ def neg_op(op: int) -> int: elif op == SUPERTYPE_OF: return SUBTYPE_OF else: - raise ValueError('Invalid operator {}'.format(op)) + raise ValueError(f"Invalid operator {op}") -def find_matching_overload_item(overloaded: Overloaded, template: Callable) -> Callable: +def find_matching_overload_item(overloaded: Overloaded, template: CallableType) -> CallableType: """Disambiguate overload item against a template.""" - items = overloaded.items() + items = overloaded.items for item in items: # Return type may be indeterminate in the template, so ignore it when performing a # subtype check. - if mypy.subtypes.is_callable_subtype(item, template, ignore_return=True): + if mypy.subtypes.is_callable_compatible( + item, + template, + is_compat=mypy.subtypes.is_subtype, + is_proper_subtype=False, + ignore_return=True, + ): return item # Fall back to the first item if we can't find a match. This is totally arbitrary -- # maybe we should just bail out at this point. return items[0] + + +def find_matching_overload_items( + overloaded: Overloaded, template: CallableType +) -> list[CallableType]: + """Like find_matching_overload_item, but return all matches, not just the first.""" + items = overloaded.items + res = [] + for item in items: + # Return type may be indeterminate in the template, so ignore it when performing a + # subtype check. + if mypy.subtypes.is_callable_compatible( + item, + template, + is_compat=mypy.subtypes.is_subtype, + is_proper_subtype=False, + ignore_return=True, + ): + res.append(item) + if not res: + # Falling back to all items if we can't find a match is pretty arbitrary, but + # it maintains backward compatibility. + res = items.copy() + return res + + +def get_tuple_fallback_from_unpack(unpack: UnpackType) -> TypeInfo: + """Get builtins.tuple type from available types to construct homogeneous tuples.""" + tp = get_proper_type(unpack.type) + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + return tp.type + if isinstance(tp, TypeVarTupleType): + return tp.tuple_fallback.type + if isinstance(tp, TupleType): + for base in tp.partial_fallback.type.mro: + if base.fullname == "builtins.tuple": + return base + assert False, "Invalid unpack type" + + +def repack_callable_args(callable: CallableType, tuple_type: TypeInfo) -> list[Type]: + """Present callable with star unpack in a normalized form. + + Since positional arguments cannot follow star argument, they are packed in a suffix, + while prefix is represented as individual positional args. We want to put all in a single + list with unpack in the middle, and prefix/suffix on the sides (as they would appear + in e.g. a TupleType). + """ + if ARG_STAR not in callable.arg_kinds: + return callable.arg_types + star_index = callable.arg_kinds.index(ARG_STAR) + arg_types = callable.arg_types[:star_index] + star_type = callable.arg_types[star_index] + suffix_types = [] + if not isinstance(star_type, UnpackType): + # Re-normalize *args: X -> *args: *tuple[X, ...] + star_type = UnpackType(Instance(tuple_type, [star_type])) + else: + tp = get_proper_type(star_type.type) + if isinstance(tp, TupleType): + assert isinstance(tp.items[0], UnpackType) + star_type = tp.items[0] + suffix_types = tp.items[1:] + return arg_types + [star_type] + suffix_types + + +def build_constraints_for_simple_unpack( + template_args: list[Type], actual_args: list[Type], direction: int +) -> list[Constraint]: + """Infer constraints between two lists of types with variadic items. + + This function is only supposed to be called when a variadic item is present in templates. + If there is no variadic item the actuals, we simply use split_with_prefix_and_suffix() + and infer prefix <: prefix, suffix <: suffix, variadic <: middle. If there is a variadic + item in the actuals we need to be more careful, only common prefix/suffix can generate + constraints, also we can only infer constraints for variadic template item, if template + prefix/suffix are shorter that actual ones, otherwise there may be partial overlap + between variadic items, for example if template prefix is longer: + + templates: T1, T2, Ts, Ts, Ts, ... + actuals: A1, As, As, As, ... + + Note: this function can only be called for builtin variadic constructors: Tuple and Callable. + For instances, you should first find correct type argument mapping. + """ + template_unpack = find_unpack_in_list(template_args) + assert template_unpack is not None + template_prefix = template_unpack + template_suffix = len(template_args) - template_prefix - 1 + + t_unpack = None + res = [] + + actual_unpack = find_unpack_in_list(actual_args) + if actual_unpack is None: + t_unpack = template_args[template_unpack] + if template_prefix + template_suffix > len(actual_args): + # These can't be subtypes of each-other, return fast. + assert isinstance(t_unpack, UnpackType) + if isinstance(t_unpack.type, TypeVarTupleType): + # Set TypeVarTuple to empty to improve error messages. + return [ + Constraint( + t_unpack.type, direction, TupleType([], t_unpack.type.tuple_fallback) + ) + ] + else: + return [] + common_prefix = template_prefix + common_suffix = template_suffix + else: + actual_prefix = actual_unpack + actual_suffix = len(actual_args) - actual_prefix - 1 + common_prefix = min(template_prefix, actual_prefix) + common_suffix = min(template_suffix, actual_suffix) + if actual_prefix >= template_prefix and actual_suffix >= template_suffix: + # This is the only case where we can guarantee there will be no partial overlap + # (note however partial overlap is OK for variadic tuples, it is handled below). + t_unpack = template_args[template_unpack] + + # Handle constraints from prefixes/suffixes first. + start, middle, end = split_with_prefix_and_suffix( + tuple(actual_args), common_prefix, common_suffix + ) + for t, a in zip(template_args[:common_prefix], start): + res.extend(infer_constraints(t, a, direction)) + if common_suffix: + for t, a in zip(template_args[-common_suffix:], end): + res.extend(infer_constraints(t, a, direction)) + + if t_unpack is not None: + # Add constraint(s) for variadic item when possible. + assert isinstance(t_unpack, UnpackType) + tp = get_proper_type(t_unpack.type) + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + # Homogeneous case *tuple[T, ...] <: [X, Y, Z, ...]. + for a in middle: + # TODO: should we use union instead of join here? + if not isinstance(a, UnpackType): + res.extend(infer_constraints(tp.args[0], a, direction)) + else: + a_tp = get_proper_type(a.type) + # This is the case *tuple[T, ...] <: *tuple[A, ...]. + if isinstance(a_tp, Instance) and a_tp.type.fullname == "builtins.tuple": + res.extend(infer_constraints(tp.args[0], a_tp.args[0], direction)) + elif isinstance(tp, TypeVarTupleType): + res.append(Constraint(tp, direction, TupleType(list(middle), tp.tuple_fallback))) + elif actual_unpack is not None: + # A special case for a variadic tuple unpack, we simply infer T <: X from + # Tuple[..., *tuple[T, ...], ...] <: Tuple[..., *tuple[X, ...], ...]. + actual_unpack_type = actual_args[actual_unpack] + assert isinstance(actual_unpack_type, UnpackType) + a_unpacked = get_proper_type(actual_unpack_type.type) + if isinstance(a_unpacked, Instance) and a_unpacked.type.fullname == "builtins.tuple": + t_unpack = template_args[template_unpack] + assert isinstance(t_unpack, UnpackType) + tp = get_proper_type(t_unpack.type) + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + res.extend(infer_constraints(tp.args[0], a_unpacked.args[0], direction)) + return res + + +def infer_directed_arg_constraints(left: Type, right: Type, direction: int) -> list[Constraint]: + """Infer constraints between two arguments using direction between original callables.""" + if isinstance(left, (ParamSpecType, UnpackType)) or isinstance( + right, (ParamSpecType, UnpackType) + ): + # This avoids bogus constraints like T <: P.args + # TODO: can we infer something useful for *T vs P? + return [] + if direction == SUBTYPE_OF: + # We invert direction to account for argument contravariance. + return infer_constraints(left, right, neg_op(direction)) + else: + return infer_constraints(right, left, neg_op(direction)) + + +def infer_callable_arguments_constraints( + template: NormalizedCallableType | Parameters, + actual: NormalizedCallableType | Parameters, + direction: int, +) -> list[Constraint]: + """Infer constraints between argument types of two callables. + + This function essentially extracts four steps from are_parameters_compatible() in + subtypes.py that involve subtype checks between argument types. We keep the argument + matching logic, but ignore various strictness flags present there, and checks that + do not involve subtyping. Then in place of every subtype check we put an infer_constraints() + call for the same types. + """ + res = [] + if direction == SUBTYPE_OF: + left, right = template, actual + else: + left, right = actual, template + left_star = left.var_arg() + left_star2 = left.kw_arg() + right_star = right.var_arg() + right_star2 = right.kw_arg() + + # Numbering of steps below matches the one in are_parameters_compatible() for convenience. + # Phase 1a: compare star vs star arguments. + if left_star is not None and right_star is not None: + res.extend(infer_directed_arg_constraints(left_star.typ, right_star.typ, direction)) + if left_star2 is not None and right_star2 is not None: + res.extend(infer_directed_arg_constraints(left_star2.typ, right_star2.typ, direction)) + + # Phase 1b: compare left args with corresponding non-star right arguments. + for right_arg in right.formal_arguments(): + left_arg = mypy.typeops.callable_corresponding_argument(left, right_arg) + if left_arg is None: + continue + res.extend(infer_directed_arg_constraints(left_arg.typ, right_arg.typ, direction)) + + # Phase 1c: compare left args with right *args. + if right_star is not None: + right_by_position = right.try_synthesizing_arg_from_vararg(None) + assert right_by_position is not None + i = right_star.pos + assert i is not None + while i < len(left.arg_kinds) and left.arg_kinds[i].is_positional(): + left_by_position = left.argument_by_position(i) + assert left_by_position is not None + res.extend( + infer_directed_arg_constraints( + left_by_position.typ, right_by_position.typ, direction + ) + ) + i += 1 + + # Phase 1d: compare left args with right **kwargs. + if right_star2 is not None: + right_names = {name for name in right.arg_names if name is not None} + left_only_names = set() + for name, kind in zip(left.arg_names, left.arg_kinds): + if name is None or kind.is_star() or name in right_names: + continue + left_only_names.add(name) + + right_by_name = right.try_synthesizing_arg_from_kwarg(None) + assert right_by_name is not None + for name in left_only_names: + left_by_name = left.argument_by_name(name) + assert left_by_name is not None + res.extend( + infer_directed_arg_constraints(left_by_name.typ, right_by_name.typ, direction) + ) + return res + + +def filter_imprecise_kinds(cs: list[Constraint]) -> list[Constraint]: + """For each ParamSpec remove all imprecise constraints, if at least one precise available.""" + have_precise = set() + for c in cs: + if not isinstance(c.origin_type_var, ParamSpecType): + continue + if ( + isinstance(c.target, ParamSpecType) + or isinstance(c.target, Parameters) + and not c.target.imprecise_arg_kinds + ): + have_precise.add(c.type_var) + new_cs = [] + for c in cs: + if not isinstance(c.origin_type_var, ParamSpecType) or c.type_var not in have_precise: + new_cs.append(c) + if not isinstance(c.target, Parameters) or not c.target.imprecise_arg_kinds: + new_cs.append(c) + return new_cs diff --git a/mypy/copytype.py b/mypy/copytype.py new file mode 100644 index 000000000000..ecb1a89759b6 --- /dev/null +++ b/mypy/copytype.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from typing import Any, cast + +from mypy.types import ( + AnyType, + CallableType, + DeletedType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + TypeAliasType, + TypedDictType, + TypeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, +) + +# type_visitor needs to be imported after types +from mypy.type_visitor import TypeVisitor # ruff: isort: skip + + +def copy_type(t: ProperType) -> ProperType: + """Create a shallow copy of a type. + + This can be used to mutate the copy with truthiness information. + + Classes compiled with mypyc don't support copy.copy(), so we need + a custom implementation. + """ + return t.accept(TypeShallowCopier()) + + +class TypeShallowCopier(TypeVisitor[ProperType]): + def visit_unbound_type(self, t: UnboundType) -> ProperType: + return t + + def visit_any(self, t: AnyType) -> ProperType: + return self.copy_common(t, AnyType(t.type_of_any, t.source_any, t.missing_import_name)) + + def visit_none_type(self, t: NoneType) -> ProperType: + return self.copy_common(t, NoneType()) + + def visit_uninhabited_type(self, t: UninhabitedType) -> ProperType: + dup = UninhabitedType() + dup.ambiguous = t.ambiguous + return self.copy_common(t, dup) + + def visit_erased_type(self, t: ErasedType) -> ProperType: + return self.copy_common(t, ErasedType()) + + def visit_deleted_type(self, t: DeletedType) -> ProperType: + return self.copy_common(t, DeletedType(t.source)) + + def visit_instance(self, t: Instance) -> ProperType: + dup = Instance(t.type, t.args, last_known_value=t.last_known_value) + dup.invalid = t.invalid + return self.copy_common(t, dup) + + def visit_type_var(self, t: TypeVarType) -> ProperType: + return self.copy_common(t, t.copy_modified()) + + def visit_param_spec(self, t: ParamSpecType) -> ProperType: + dup = ParamSpecType( + t.name, t.fullname, t.id, t.flavor, t.upper_bound, t.default, prefix=t.prefix + ) + return self.copy_common(t, dup) + + def visit_parameters(self, t: Parameters) -> ProperType: + dup = Parameters( + t.arg_types, + t.arg_kinds, + t.arg_names, + variables=t.variables, + is_ellipsis_args=t.is_ellipsis_args, + ) + return self.copy_common(t, dup) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> ProperType: + dup = TypeVarTupleType( + t.name, t.fullname, t.id, t.upper_bound, t.tuple_fallback, t.default + ) + return self.copy_common(t, dup) + + def visit_unpack_type(self, t: UnpackType) -> ProperType: + dup = UnpackType(t.type) + return self.copy_common(t, dup) + + def visit_partial_type(self, t: PartialType) -> ProperType: + return self.copy_common(t, PartialType(t.type, t.var, t.value_type)) + + def visit_callable_type(self, t: CallableType) -> ProperType: + return self.copy_common(t, t.copy_modified()) + + def visit_tuple_type(self, t: TupleType) -> ProperType: + return self.copy_common(t, TupleType(t.items, t.partial_fallback, implicit=t.implicit)) + + def visit_typeddict_type(self, t: TypedDictType) -> ProperType: + return self.copy_common( + t, TypedDictType(t.items, t.required_keys, t.readonly_keys, t.fallback) + ) + + def visit_literal_type(self, t: LiteralType) -> ProperType: + return self.copy_common(t, LiteralType(value=t.value, fallback=t.fallback)) + + def visit_union_type(self, t: UnionType) -> ProperType: + return self.copy_common(t, UnionType(t.items)) + + def visit_overloaded(self, t: Overloaded) -> ProperType: + return self.copy_common(t, Overloaded(items=t.items)) + + def visit_type_type(self, t: TypeType) -> ProperType: + # Use cast since the type annotations in TypeType are imprecise. + return self.copy_common(t, TypeType(cast(Any, t.item))) + + def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: + assert False, "only ProperTypes supported" + + def copy_common(self, t: ProperType, t2: ProperType) -> ProperType: + t2.line = t.line + t2.column = t.column + t2.can_be_false = t.can_be_false + t2.can_be_true = t.can_be_true + return t2 diff --git a/mypy/defaults.py b/mypy/defaults.py new file mode 100644 index 000000000000..2bbae23d7e2d --- /dev/null +++ b/mypy/defaults.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import os +from typing import Final + +# Earliest fully supported Python 3.x version. Used as the default Python +# version in tests. Mypy wheels should be built starting with this version, +# and CI tests should be run on this version (and later versions). +PYTHON3_VERSION: Final = (3, 8) + +# Earliest Python 3.x version supported via --python-version 3.x. To run +# mypy, at least version PYTHON3_VERSION is needed. +PYTHON3_VERSION_MIN: Final = (3, 8) # Keep in sync with typeshed's python support + +CACHE_DIR: Final = ".mypy_cache" +CONFIG_FILE: Final = ["mypy.ini", ".mypy.ini"] +PYPROJECT_CONFIG_FILES: Final = ["pyproject.toml"] +SHARED_CONFIG_FILES: Final = ["setup.cfg"] +USER_CONFIG_FILES: Final = ["~/.config/mypy/config", "~/.mypy.ini"] +if os.environ.get("XDG_CONFIG_HOME"): + USER_CONFIG_FILES.insert(0, os.path.join(os.environ["XDG_CONFIG_HOME"], "mypy/config")) + +CONFIG_FILES: Final = ( + CONFIG_FILE + PYPROJECT_CONFIG_FILES + SHARED_CONFIG_FILES + USER_CONFIG_FILES +) + +# This must include all reporters defined in mypy.report. This is defined here +# to make reporter names available without importing mypy.report -- this speeds +# up startup. +REPORTER_NAMES: Final = [ + "linecount", + "any-exprs", + "linecoverage", + "memory-xml", + "cobertura-xml", + "xml", + "xslt-html", + "xslt-txt", + "html", + "txt", + "lineprecision", +] + +# Threshold after which we sometimes filter out most errors to avoid very +# verbose output. The default is to show all errors. +MANY_ERRORS_THRESHOLD: Final = -1 diff --git a/lib-python/3.2/incomplete/urllib/__init__.py b/mypy/dmypy/__init__.py similarity index 100% rename from lib-python/3.2/incomplete/urllib/__init__.py rename to mypy/dmypy/__init__.py diff --git a/mypy/dmypy/__main__.py b/mypy/dmypy/__main__.py new file mode 100644 index 000000000000..5441b9f8e8fa --- /dev/null +++ b/mypy/dmypy/__main__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +from mypy.dmypy.client import console_entry + +if __name__ == "__main__": + console_entry() diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py new file mode 100644 index 000000000000..9f0751e93609 --- /dev/null +++ b/mypy/dmypy/client.py @@ -0,0 +1,749 @@ +"""Client for mypy daemon mode. + +This manages a daemon process which keeps useful state in memory +rather than having to read it back from disk on each run. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import pickle +import sys +import time +import traceback +from typing import Any, Callable, Mapping, NoReturn + +from mypy.dmypy_os import alive, kill +from mypy.dmypy_util import DEFAULT_STATUS_FILE, receive, send +from mypy.ipc import IPCClient, IPCException +from mypy.util import check_python_version, get_terminal_width, should_force_color +from mypy.version import __version__ + +# Argument parser. Subparsers are tied to action functions by the +# @action(subparse) decorator. + + +class AugmentedHelpFormatter(argparse.RawDescriptionHelpFormatter): + def __init__(self, prog: str) -> None: + super().__init__(prog=prog, max_help_position=30) + + +parser = argparse.ArgumentParser( + prog="dmypy", description="Client for mypy daemon mode", fromfile_prefix_chars="@" +) +parser.set_defaults(action=None) +parser.add_argument( + "--status-file", default=DEFAULT_STATUS_FILE, help="status file to retrieve daemon details" +) +parser.add_argument( + "-V", + "--version", + action="version", + version="%(prog)s " + __version__, + help="Show program's version number and exit", +) +subparsers = parser.add_subparsers() + +start_parser = p = subparsers.add_parser("start", help="Start daemon") +p.add_argument("--log-file", metavar="FILE", type=str, help="Direct daemon stdout/stderr to FILE") +p.add_argument( + "--timeout", metavar="TIMEOUT", type=int, help="Server shutdown timeout (in seconds)" +) +p.add_argument( + "flags", metavar="FLAG", nargs="*", type=str, help="Regular mypy flags (precede with --)" +) + +restart_parser = p = subparsers.add_parser( + "restart", help="Restart daemon (stop or kill followed by start)" +) +p.add_argument("--log-file", metavar="FILE", type=str, help="Direct daemon stdout/stderr to FILE") +p.add_argument( + "--timeout", metavar="TIMEOUT", type=int, help="Server shutdown timeout (in seconds)" +) +p.add_argument( + "flags", metavar="FLAG", nargs="*", type=str, help="Regular mypy flags (precede with --)" +) + +status_parser = p = subparsers.add_parser("status", help="Show daemon status") +p.add_argument("-v", "--verbose", action="store_true", help="Print detailed status") +p.add_argument("--fswatcher-dump-file", help="Collect information about the current file state") + +stop_parser = p = subparsers.add_parser("stop", help="Stop daemon (asks it politely to go away)") + +kill_parser = p = subparsers.add_parser("kill", help="Kill daemon (kills the process)") + +check_parser = p = subparsers.add_parser( + "check", formatter_class=AugmentedHelpFormatter, help="Check some files (requires daemon)" +) +p.add_argument("-v", "--verbose", action="store_true", help="Print detailed status") +p.add_argument("-q", "--quiet", action="store_true", help=argparse.SUPPRESS) # Deprecated +p.add_argument("--junit-xml", help="Write junit.xml to the given file") +p.add_argument("--perf-stats-file", help="write performance information to the given file") +p.add_argument("files", metavar="FILE", nargs="+", help="File (or directory) to check") +p.add_argument( + "--export-types", + action="store_true", + help="Store types of all expressions in a shared location (useful for inspections)", +) + +run_parser = p = subparsers.add_parser( + "run", + formatter_class=AugmentedHelpFormatter, + help="Check some files, [re]starting daemon if necessary", +) +p.add_argument("-v", "--verbose", action="store_true", help="Print detailed status") +p.add_argument("--junit-xml", help="Write junit.xml to the given file") +p.add_argument("--perf-stats-file", help="write performance information to the given file") +p.add_argument( + "--timeout", metavar="TIMEOUT", type=int, help="Server shutdown timeout (in seconds)" +) +p.add_argument("--log-file", metavar="FILE", type=str, help="Direct daemon stdout/stderr to FILE") +p.add_argument( + "--export-types", + action="store_true", + help="Store types of all expressions in a shared location (useful for inspections)", +) +p.add_argument( + "flags", + metavar="ARG", + nargs="*", + type=str, + help="Regular mypy flags and files (precede with --)", +) + +recheck_parser = p = subparsers.add_parser( + "recheck", + formatter_class=AugmentedHelpFormatter, + help="Re-check the previous list of files, with optional modifications (requires daemon)", +) +p.add_argument("-v", "--verbose", action="store_true", help="Print detailed status") +p.add_argument("-q", "--quiet", action="store_true", help=argparse.SUPPRESS) # Deprecated +p.add_argument("--junit-xml", help="Write junit.xml to the given file") +p.add_argument("--perf-stats-file", help="write performance information to the given file") +p.add_argument( + "--export-types", + action="store_true", + help="Store types of all expressions in a shared location (useful for inspections)", +) +p.add_argument( + "--update", + metavar="FILE", + nargs="*", + help="Files in the run to add or check again (default: all from previous run)", +) +p.add_argument("--remove", metavar="FILE", nargs="*", help="Files to remove from the run") + +suggest_parser = p = subparsers.add_parser( + "suggest", help="Suggest a signature or show call sites for a specific function" +) +p.add_argument( + "function", + metavar="FUNCTION", + type=str, + help="Function specified as '[package.]module.[class.]function'", +) +p.add_argument( + "--json", + action="store_true", + help="Produce json that pyannotate can use to apply a suggestion", +) +p.add_argument( + "--no-errors", action="store_true", help="Only produce suggestions that cause no errors" +) +p.add_argument( + "--no-any", action="store_true", help="Only produce suggestions that don't contain Any" +) +p.add_argument( + "--flex-any", + type=float, + help="Allow anys in types if they go above a certain score (scores are from 0-1)", +) +p.add_argument( + "--callsites", action="store_true", help="Find callsites instead of suggesting a type" +) +p.add_argument( + "--use-fixme", + metavar="NAME", + type=str, + help="A dummy name to use instead of Any for types that can't be inferred", +) +p.add_argument( + "--max-guesses", + type=int, + help="Set the maximum number of types to try for a function (default 64)", +) + +inspect_parser = p = subparsers.add_parser( + "inspect", help="Locate and statically inspect expression(s)" +) +p.add_argument( + "location", + metavar="LOCATION", + type=str, + help="Location specified as path/to/file.py:line:column[:end_line:end_column]." + " If position is given (i.e. only line and column), this will return all" + " enclosing expressions", +) +p.add_argument( + "--show", + metavar="INSPECTION", + type=str, + default="type", + choices=["type", "attrs", "definition"], + help="What kind of inspection to run", +) +p.add_argument( + "--verbose", + "-v", + action="count", + default=0, + help="Increase verbosity of the type string representation (can be repeated)", +) +p.add_argument( + "--limit", + metavar="NUM", + type=int, + default=0, + help="Return at most NUM innermost expressions (if position is given); 0 means no limit", +) +p.add_argument( + "--include-span", + action="store_true", + help="Prepend each inspection result with the span of corresponding expression" + ' (e.g. 1:2:3:4:"int")', +) +p.add_argument( + "--include-kind", + action="store_true", + help="Prepend each inspection result with the kind of corresponding expression" + ' (e.g. NameExpr:"int")', +) +p.add_argument( + "--include-object-attrs", + action="store_true", + help='Include attributes of "object" in "attrs" inspection', +) +p.add_argument( + "--union-attrs", + action="store_true", + help="Include attributes valid for some of possible expression types" + " (by default an intersection is returned)", +) +p.add_argument( + "--force-reload", + action="store_true", + help="Re-parse and re-type-check file before inspection (may be slow)", +) + +hang_parser = p = subparsers.add_parser("hang", help="Hang for 100 seconds") + +daemon_parser = p = subparsers.add_parser("daemon", help="Run daemon in foreground") +p.add_argument( + "--timeout", metavar="TIMEOUT", type=int, help="Server shutdown timeout (in seconds)" +) +p.add_argument("--log-file", metavar="FILE", type=str, help="Direct daemon stdout/stderr to FILE") +p.add_argument( + "flags", metavar="FLAG", nargs="*", type=str, help="Regular mypy flags (precede with --)" +) +p.add_argument("--options-data", help=argparse.SUPPRESS) +help_parser = p = subparsers.add_parser("help") + +del p + + +class BadStatus(Exception): + """Exception raised when there is something wrong with the status file. + + For example: + - No status file found + - Status file malformed + - Process whose pid is in the status file does not exist + """ + + +def main(argv: list[str]) -> None: + """The code is top-down.""" + check_python_version("dmypy") + args = parser.parse_args(argv) + if not args.action: + parser.print_usage() + else: + try: + args.action(args) + except BadStatus as err: + fail(err.args[0]) + except Exception: + # We do this explicitly to avoid exceptions percolating up + # through mypy.api invocations + traceback.print_exc() + sys.exit(2) + + +def fail(msg: str) -> NoReturn: + print(msg, file=sys.stderr) + sys.exit(2) + + +ActionFunction = Callable[[argparse.Namespace], None] + + +def action(subparser: argparse.ArgumentParser) -> Callable[[ActionFunction], ActionFunction]: + """Decorator to tie an action function to a subparser.""" + + def register(func: ActionFunction) -> ActionFunction: + subparser.set_defaults(action=func) + return func + + return register + + +# Action functions (run in client from command line). + + +@action(start_parser) +def do_start(args: argparse.Namespace) -> None: + """Start daemon (it must not already be running). + + This is where mypy flags are set from the command line. + + Setting flags is a bit awkward; you have to use e.g.: + + dmypy start -- --strict + + since we don't want to duplicate mypy's huge list of flags. + """ + try: + get_status(args.status_file) + except BadStatus: + # Bad or missing status file or dead process; good to start. + pass + else: + fail("Daemon is still alive") + start_server(args) + + +@action(restart_parser) +def do_restart(args: argparse.Namespace) -> None: + """Restart daemon (it may or may not be running; but not hanging). + + We first try to stop it politely if it's running. This also sets + mypy flags from the command line (see do_start()). + """ + restart_server(args) + + +def restart_server(args: argparse.Namespace, allow_sources: bool = False) -> None: + """Restart daemon (it may or may not be running; but not hanging).""" + try: + do_stop(args) + except BadStatus: + # Bad or missing status file or dead process; good to start. + pass + start_server(args, allow_sources) + + +def start_server(args: argparse.Namespace, allow_sources: bool = False) -> None: + """Start the server from command arguments and wait for it.""" + # Lazy import so this import doesn't slow down other commands. + from mypy.dmypy_server import daemonize, process_start_options + + start_options = process_start_options(args.flags, allow_sources) + if daemonize(start_options, args.status_file, timeout=args.timeout, log_file=args.log_file): + sys.exit(2) + wait_for_server(args.status_file) + + +def wait_for_server(status_file: str, timeout: float = 5.0) -> None: + """Wait until the server is up. + + Exit if it doesn't happen within the timeout. + """ + endtime = time.time() + timeout + while time.time() < endtime: + try: + data = read_status(status_file) + except BadStatus: + # If the file isn't there yet, retry later. + time.sleep(0.1) + continue + # If the file's content is bogus or the process is dead, fail. + check_status(data) + print("Daemon started") + return + fail("Timed out waiting for daemon to start") + + +@action(run_parser) +def do_run(args: argparse.Namespace) -> None: + """Do a check, starting (or restarting) the daemon as necessary + + Restarts the daemon if the running daemon reports that it is + required (due to a configuration change, for example). + + Setting flags is a bit awkward; you have to use e.g.: + + dmypy run -- --strict a.py b.py ... + + since we don't want to duplicate mypy's huge list of flags. + (The -- is only necessary if flags are specified.) + """ + if not is_running(args.status_file): + # Bad or missing status file or dead process; good to start. + start_server(args, allow_sources=True) + t0 = time.time() + response = request( + args.status_file, + "run", + version=__version__, + args=args.flags, + export_types=args.export_types, + ) + # If the daemon signals that a restart is necessary, do it + if "restart" in response: + print(f"Restarting: {response['restart']}") + restart_server(args, allow_sources=True) + response = request( + args.status_file, + "run", + version=__version__, + args=args.flags, + export_types=args.export_types, + ) + + t1 = time.time() + response["roundtrip_time"] = t1 - t0 + check_output(response, args.verbose, args.junit_xml, args.perf_stats_file) + + +@action(status_parser) +def do_status(args: argparse.Namespace) -> None: + """Print daemon status. + + This verifies that it is responsive to requests. + """ + status = read_status(args.status_file) + if args.verbose: + show_stats(status) + # Both check_status() and request() may raise BadStatus, + # which will be handled by main(). + check_status(status) + response = request( + args.status_file, "status", fswatcher_dump_file=args.fswatcher_dump_file, timeout=5 + ) + if args.verbose or "error" in response: + show_stats(response) + if "error" in response: + fail(f"Daemon is stuck; consider {sys.argv[0]} kill") + print("Daemon is up and running") + + +@action(stop_parser) +def do_stop(args: argparse.Namespace) -> None: + """Stop daemon via a 'stop' request.""" + # May raise BadStatus, which will be handled by main(). + response = request(args.status_file, "stop", timeout=5) + if "error" in response: + show_stats(response) + fail(f"Daemon is stuck; consider {sys.argv[0]} kill") + else: + print("Daemon stopped") + + +@action(kill_parser) +def do_kill(args: argparse.Namespace) -> None: + """Kill daemon process with SIGKILL.""" + pid, _ = get_status(args.status_file) + try: + kill(pid) + except OSError as err: + fail(str(err)) + else: + print("Daemon killed") + + +@action(check_parser) +def do_check(args: argparse.Namespace) -> None: + """Ask the daemon to check a list of files.""" + t0 = time.time() + response = request(args.status_file, "check", files=args.files, export_types=args.export_types) + t1 = time.time() + response["roundtrip_time"] = t1 - t0 + check_output(response, args.verbose, args.junit_xml, args.perf_stats_file) + + +@action(recheck_parser) +def do_recheck(args: argparse.Namespace) -> None: + """Ask the daemon to recheck the previous list of files, with optional modifications. + + If at least one of --remove or --update is given, the server will + update the list of files to check accordingly and assume that any other files + are unchanged. If none of these flags are given, the server will call stat() + on each file last checked to determine its status. + + Files given in --update ought to exist. Files given in --remove need not exist; + if they don't they will be ignored. + The lists may be empty but oughtn't contain duplicates or overlap. + + NOTE: The list of files is lost when the daemon is restarted. + """ + t0 = time.time() + if args.remove is not None or args.update is not None: + response = request( + args.status_file, + "recheck", + export_types=args.export_types, + remove=args.remove, + update=args.update, + ) + else: + response = request(args.status_file, "recheck", export_types=args.export_types) + t1 = time.time() + response["roundtrip_time"] = t1 - t0 + check_output(response, args.verbose, args.junit_xml, args.perf_stats_file) + + +@action(suggest_parser) +def do_suggest(args: argparse.Namespace) -> None: + """Ask the daemon for a suggested signature. + + This just prints whatever the daemon reports as output. + For now it may be closer to a list of call sites. + """ + response = request( + args.status_file, + "suggest", + function=args.function, + json=args.json, + callsites=args.callsites, + no_errors=args.no_errors, + no_any=args.no_any, + flex_any=args.flex_any, + use_fixme=args.use_fixme, + max_guesses=args.max_guesses, + ) + check_output(response, verbose=False, junit_xml=None, perf_stats_file=None) + + +@action(inspect_parser) +def do_inspect(args: argparse.Namespace) -> None: + """Ask daemon to print the type of an expression.""" + response = request( + args.status_file, + "inspect", + show=args.show, + location=args.location, + verbosity=args.verbose, + limit=args.limit, + include_span=args.include_span, + include_kind=args.include_kind, + include_object_attrs=args.include_object_attrs, + union_attrs=args.union_attrs, + force_reload=args.force_reload, + ) + check_output(response, verbose=False, junit_xml=None, perf_stats_file=None) + + +def check_output( + response: dict[str, Any], verbose: bool, junit_xml: str | None, perf_stats_file: str | None +) -> None: + """Print the output from a check or recheck command. + + Call sys.exit() unless the status code is zero. + """ + if "error" in response: + fail(response["error"]) + try: + out, err, status_code = response["out"], response["err"], response["status"] + except KeyError: + fail(f"Response: {str(response)}") + sys.stdout.write(out) + sys.stdout.flush() + sys.stderr.write(err) + sys.stderr.flush() + if verbose: + show_stats(response) + if junit_xml: + # Lazy import so this import doesn't slow things down when not writing junit + from mypy.util import write_junit_xml + + messages = (out + err).splitlines() + write_junit_xml( + response["roundtrip_time"], + bool(err), + {None: messages} if messages else {}, + junit_xml, + response["python_version"], + response["platform"], + ) + if perf_stats_file: + telemetry = response.get("stats", {}) + with open(perf_stats_file, "w") as f: + json.dump(telemetry, f) + + if status_code: + sys.exit(status_code) + + +def show_stats(response: Mapping[str, object]) -> None: + for key, value in sorted(response.items()): + if key in ("out", "err", "stdout", "stderr"): + # Special case text output to display just 40 characters of text + value = repr(value)[1:-1] + if len(value) > 50: + value = f"{value[:40]} ... {len(value)-40} more characters" + print("%-24s: %s" % (key, value)) + continue + print("%-24s: %10s" % (key, "%.3f" % value if isinstance(value, float) else value)) + + +@action(hang_parser) +def do_hang(args: argparse.Namespace) -> None: + """Hang for 100 seconds, as a debug hack.""" + print(request(args.status_file, "hang", timeout=1)) + + +@action(daemon_parser) +def do_daemon(args: argparse.Namespace) -> None: + """Serve requests in the foreground.""" + # Lazy import so this import doesn't slow down other commands. + from mypy.dmypy_server import Server, process_start_options + + if args.log_file: + sys.stdout = sys.stderr = open(args.log_file, "a", buffering=1) + fd = sys.stdout.fileno() + os.dup2(fd, 2) + os.dup2(fd, 1) + + if args.options_data: + from mypy.options import Options + + options_dict = pickle.loads(base64.b64decode(args.options_data)) + options_obj = Options() + options = options_obj.apply_changes(options_dict) + else: + options = process_start_options(args.flags, allow_sources=False) + + Server(options, args.status_file, timeout=args.timeout).serve() + + +@action(help_parser) +def do_help(args: argparse.Namespace) -> None: + """Print full help (same as dmypy --help).""" + parser.print_help() + + +# Client-side infrastructure. + + +def request( + status_file: str, command: str, *, timeout: int | None = None, **kwds: object +) -> dict[str, Any]: + """Send a request to the daemon. + + Return the JSON dict with the response. + + Raise BadStatus if there is something wrong with the status file + or if the process whose pid is in the status file has died. + + Return {'error': } if an IPC operation or receive() + raised OSError. This covers cases such as connection refused or + closed prematurely as well as invalid JSON received. + """ + response: dict[str, str] = {} + args = dict(kwds) + args["command"] = command + # Tell the server whether this request was initiated from a human-facing terminal, + # so that it can format the type checking output accordingly. + args["is_tty"] = sys.stdout.isatty() or should_force_color() + args["terminal_width"] = get_terminal_width() + _, name = get_status(status_file) + try: + with IPCClient(name, timeout) as client: + send(client, args) + + final = False + while not final: + response = receive(client) + final = bool(response.pop("final", False)) + # Display debugging output written to stdout/stderr in the server process for convenience. + # This should not be confused with "out" and "err" fields in the response. + # Those fields hold the output of the "check" command, and are handled in check_output(). + stdout = response.pop("stdout", None) + if stdout: + sys.stdout.write(stdout) + stderr = response.pop("stderr", None) + if stderr: + sys.stderr.write(stderr) + except (OSError, IPCException) as err: + return {"error": str(err)} + # TODO: Other errors, e.g. ValueError, UnicodeError + + return response + + +def get_status(status_file: str) -> tuple[int, str]: + """Read status file and check if the process is alive. + + Return (pid, connection_name) on success. + + Raise BadStatus if something's wrong. + """ + data = read_status(status_file) + return check_status(data) + + +def check_status(data: dict[str, Any]) -> tuple[int, str]: + """Check if the process is alive. + + Return (pid, connection_name) on success. + + Raise BadStatus if something's wrong. + """ + if "pid" not in data: + raise BadStatus("Invalid status file (no pid field)") + pid = data["pid"] + if not isinstance(pid, int): + raise BadStatus("pid field is not an int") + if not alive(pid): + raise BadStatus("Daemon has died") + if "connection_name" not in data: + raise BadStatus("Invalid status file (no connection_name field)") + connection_name = data["connection_name"] + if not isinstance(connection_name, str): + raise BadStatus("connection_name field is not a string") + return pid, connection_name + + +def read_status(status_file: str) -> dict[str, object]: + """Read status file. + + Raise BadStatus if the status file doesn't exist or contains + invalid JSON or the JSON is not a dict. + """ + if not os.path.isfile(status_file): + raise BadStatus("No status file found") + with open(status_file) as f: + try: + data = json.load(f) + except Exception as e: + raise BadStatus("Malformed status file (not JSON)") from e + if not isinstance(data, dict): + raise BadStatus("Invalid status file (not a dict)") + return data + + +def is_running(status_file: str) -> bool: + """Check if the server is running cleanly""" + try: + get_status(status_file) + except BadStatus: + return False + return True + + +# Run main(). +def console_entry() -> None: + main(sys.argv[1:]) diff --git a/mypy/dmypy_os.py b/mypy/dmypy_os.py new file mode 100644 index 000000000000..63c3e4c88979 --- /dev/null +++ b/mypy/dmypy_os.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import sys +from typing import Any, Callable + +if sys.platform == "win32": + import ctypes + import subprocess + from ctypes.wintypes import DWORD, HANDLE + + PROCESS_QUERY_LIMITED_INFORMATION = ctypes.c_ulong(0x1000) + + kernel32 = ctypes.windll.kernel32 + OpenProcess: Callable[[DWORD, int, int], HANDLE] = kernel32.OpenProcess + GetExitCodeProcess: Callable[[HANDLE, Any], int] = kernel32.GetExitCodeProcess +else: + import os + import signal + + +def alive(pid: int) -> bool: + """Is the process alive?""" + if sys.platform == "win32": + # why can't anything be easy... + status = DWORD() + handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) + GetExitCodeProcess(handle, ctypes.byref(status)) + return status.value == 259 # STILL_ACTIVE + else: + try: + os.kill(pid, 0) + except OSError: + return False + return True + + +def kill(pid: int) -> None: + """Kill the process.""" + if sys.platform == "win32": + subprocess.check_output(f"taskkill /pid {pid} /f /t") + else: + os.kill(pid, signal.SIGKILL) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py new file mode 100644 index 000000000000..70cfaa5b2fb9 --- /dev/null +++ b/mypy/dmypy_server.py @@ -0,0 +1,1117 @@ +"""Server for mypy daemon mode. + +This implements a daemon process which keeps useful state in memory +to enable fine-grained incremental reprocessing of changes. +""" + +from __future__ import annotations + +import argparse +import base64 +import io +import json +import os +import pickle +import subprocess +import sys +import time +import traceback +from contextlib import redirect_stderr, redirect_stdout +from typing import AbstractSet, Any, Callable, Final, List, Sequence, Tuple +from typing_extensions import TypeAlias as _TypeAlias + +import mypy.build +import mypy.errors +import mypy.main +from mypy.dmypy_util import WriteToConn, receive, send +from mypy.find_sources import InvalidSourceList, create_source_list +from mypy.fscache import FileSystemCache +from mypy.fswatcher import FileData, FileSystemWatcher +from mypy.inspections import InspectionEngine +from mypy.ipc import IPCServer +from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, compute_search_paths +from mypy.options import Options +from mypy.server.update import FineGrainedBuildManager, refresh_suppressed_submodules +from mypy.suggestions import SuggestionEngine, SuggestionFailure +from mypy.typestate import reset_global_state +from mypy.util import FancyFormatter, count_stats +from mypy.version import __version__ + +MEM_PROFILE: Final = False # If True, dump memory profile after initialization + +if sys.platform == "win32": + from subprocess import STARTUPINFO + + def daemonize( + options: Options, status_file: str, timeout: int | None = None, log_file: str | None = None + ) -> int: + """Create the daemon process via "dmypy daemon" and pass options via command line + + When creating the daemon grandchild, we create it in a new console, which is + started hidden. We cannot use DETACHED_PROCESS since it will cause console windows + to pop up when starting. See + https://github.com/python/cpython/pull/4150#issuecomment-340215696 + for more on why we can't have nice things. + + It also pickles the options to be unpickled by mypy. + """ + command = [sys.executable, "-m", "mypy.dmypy", "--status-file", status_file, "daemon"] + pickled_options = pickle.dumps(options.snapshot()) + command.append(f'--options-data="{base64.b64encode(pickled_options).decode()}"') + if timeout: + command.append(f"--timeout={timeout}") + if log_file: + command.append(f"--log-file={log_file}") + info = STARTUPINFO() + info.dwFlags = 0x1 # STARTF_USESHOWWINDOW aka use wShowWindow's value + info.wShowWindow = 0 # SW_HIDE aka make the window invisible + try: + subprocess.Popen(command, creationflags=0x10, startupinfo=info) # CREATE_NEW_CONSOLE + return 0 + except subprocess.CalledProcessError as e: + return e.returncode + +else: + + def _daemonize_cb(func: Callable[[], None], log_file: str | None = None) -> int: + """Arrange to call func() in a grandchild of the current process. + + Return 0 for success, exit status for failure, negative if + subprocess killed by signal. + """ + # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python + sys.stdout.flush() + sys.stderr.flush() + pid = os.fork() + if pid: + # Parent process: wait for child in case things go bad there. + npid, sts = os.waitpid(pid, 0) + sig = sts & 0xFF + if sig: + print("Child killed by signal", sig) + return -sig + sts = sts >> 8 + if sts: + print("Child exit status", sts) + return sts + # Child process: do a bunch of UNIX stuff and then fork a grandchild. + try: + os.setsid() # Detach controlling terminal + os.umask(0o27) + devnull = os.open("/dev/null", os.O_RDWR) + os.dup2(devnull, 0) + os.dup2(devnull, 1) + os.dup2(devnull, 2) + os.close(devnull) + pid = os.fork() + if pid: + # Child is done, exit to parent. + os._exit(0) + # Grandchild: run the server. + if log_file: + sys.stdout = sys.stderr = open(log_file, "a", buffering=1) + fd = sys.stdout.fileno() + os.dup2(fd, 2) + os.dup2(fd, 1) + func() + finally: + # Make sure we never get back into the caller. + os._exit(1) + + def daemonize( + options: Options, status_file: str, timeout: int | None = None, log_file: str | None = None + ) -> int: + """Run the mypy daemon in a grandchild of the current process + + Return 0 for success, exit status for failure, negative if + subprocess killed by signal. + """ + return _daemonize_cb(Server(options, status_file, timeout).serve, log_file) + + +# Server code. + +CONNECTION_NAME: Final = "dmypy" + + +def process_start_options(flags: list[str], allow_sources: bool) -> Options: + _, options = mypy.main.process_options( + ["-i"] + flags, require_targets=False, server_options=True + ) + if options.report_dirs: + print("dmypy: Ignoring report generation settings. Start/restart cannot generate reports.") + if options.junit_xml: + print( + "dmypy: Ignoring report generation settings. " + "Start/restart does not support --junit-xml. Pass it to check/recheck instead" + ) + options.junit_xml = None + if not options.incremental: + sys.exit("dmypy: start/restart should not disable incremental mode") + if options.follow_imports not in ("skip", "error", "normal"): + sys.exit("dmypy: follow-imports=silent not supported") + return options + + +def ignore_suppressed_imports(module: str) -> bool: + """Can we skip looking for newly unsuppressed imports to module?""" + # Various submodules of 'encodings' can be suppressed, since it + # uses module-level '__getattr__'. Skip them since there are many + # of them, and following imports to them is kind of pointless. + return module.startswith("encodings.") + + +ModulePathPair: _TypeAlias = Tuple[str, str] +ModulePathPairs: _TypeAlias = List[ModulePathPair] +ChangesAndRemovals: _TypeAlias = Tuple[ModulePathPairs, ModulePathPairs] + + +class Server: + # NOTE: the instance is constructed in the parent process but + # serve() is called in the grandchild (by daemonize()). + + def __init__(self, options: Options, status_file: str, timeout: int | None = None) -> None: + """Initialize the server with the desired mypy flags.""" + self.options = options + # Snapshot the options info before we muck with it, to detect changes + self.options_snapshot = options.snapshot() + self.timeout = timeout + self.fine_grained_manager: FineGrainedBuildManager | None = None + + if os.path.isfile(status_file): + os.unlink(status_file) + + self.fscache = FileSystemCache() + + options.raise_exceptions = True + options.incremental = True + options.fine_grained_incremental = True + options.show_traceback = True + if options.use_fine_grained_cache: + # Using fine_grained_cache implies generating and caring + # about the fine grained cache + options.cache_fine_grained = True + else: + options.cache_dir = os.devnull + # Fine-grained incremental doesn't support general partial types + # (details in https://github.com/python/mypy/issues/4492) + options.local_partial_types = True + self.status_file = status_file + + # Since the object is created in the parent process we can check + # the output terminal options here. + self.formatter = FancyFormatter(sys.stdout, sys.stderr, options.hide_error_codes) + + def _response_metadata(self) -> dict[str, str]: + py_version = f"{self.options.python_version[0]}_{self.options.python_version[1]}" + return {"platform": self.options.platform, "python_version": py_version} + + def serve(self) -> None: + """Serve requests, synchronously (no thread or fork).""" + + command = None + server = IPCServer(CONNECTION_NAME, self.timeout) + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + try: + with open(self.status_file, "w") as f: + json.dump({"pid": os.getpid(), "connection_name": server.connection_name}, f) + f.write("\n") # I like my JSON with a trailing newline + while True: + with server: + data = receive(server) + sys.stdout = WriteToConn(server, "stdout", sys.stdout.isatty()) + sys.stderr = WriteToConn(server, "stderr", sys.stderr.isatty()) + resp: dict[str, Any] = {} + if "command" not in data: + resp = {"error": "No command found in request"} + else: + command = data["command"] + if not isinstance(command, str): + resp = {"error": "Command is not a string"} + else: + command = data.pop("command") + try: + resp = self.run_command(command, data) + except Exception: + # If we are crashing, report the crash to the client + tb = traceback.format_exception(*sys.exc_info()) + resp = {"error": "Daemon crashed!\n" + "".join(tb)} + resp.update(self._response_metadata()) + resp["final"] = True + send(server, resp) + raise + resp["final"] = True + try: + resp.update(self._response_metadata()) + send(server, resp) + except OSError: + pass # Maybe the client hung up + if command == "stop": + reset_global_state() + sys.exit(0) + finally: + # Revert stdout/stderr so we can see any errors. + sys.stdout = orig_stdout + sys.stderr = orig_stderr + + # If the final command is something other than a clean + # stop, remove the status file. (We can't just + # simplify the logic and always remove the file, since + # that could cause us to remove a future server's + # status file.) + if command != "stop": + os.unlink(self.status_file) + try: + server.cleanup() # try to remove the socket dir on Linux + except OSError: + pass + exc_info = sys.exc_info() + if exc_info[0] and exc_info[0] is not SystemExit: + traceback.print_exception(*exc_info) + + def run_command(self, command: str, data: dict[str, object]) -> dict[str, object]: + """Run a specific command from the registry.""" + key = "cmd_" + command + method = getattr(self.__class__, key, None) + if method is None: + return {"error": f"Unrecognized command '{command}'"} + else: + if command not in {"check", "recheck", "run"}: + # Only the above commands use some error formatting. + del data["is_tty"] + del data["terminal_width"] + ret = method(self, **data) + assert isinstance(ret, dict) + return ret + + # Command functions (run in the server via RPC). + + def cmd_status(self, fswatcher_dump_file: str | None = None) -> dict[str, object]: + """Return daemon status.""" + res: dict[str, object] = {} + res.update(get_meminfo()) + if fswatcher_dump_file: + data = self.fswatcher.dump_file_data() if hasattr(self, "fswatcher") else {} + # Using .dumps and then writing was noticeably faster than using dump + s = json.dumps(data) + with open(fswatcher_dump_file, "w") as f: + f.write(s) + return res + + def cmd_stop(self) -> dict[str, object]: + """Stop daemon.""" + # We need to remove the status file *before* we complete the + # RPC. Otherwise a race condition exists where a subsequent + # command can see a status file from a dying server and think + # it is a live one. + os.unlink(self.status_file) + return {} + + def cmd_run( + self, + version: str, + args: Sequence[str], + export_types: bool, + is_tty: bool, + terminal_width: int, + ) -> dict[str, object]: + """Check a list of files, triggering a restart if needed.""" + stderr = io.StringIO() + stdout = io.StringIO() + try: + # Process options can exit on improper arguments, so we need to catch that and + # capture stderr so the client can report it + with redirect_stderr(stderr): + with redirect_stdout(stdout): + sources, options = mypy.main.process_options( + ["-i"] + list(args), + require_targets=True, + server_options=True, + fscache=self.fscache, + program="mypy-daemon", + header=argparse.SUPPRESS, + ) + # Signal that we need to restart if the options have changed + if not options.compare_stable(self.options_snapshot): + return {"restart": "configuration changed"} + if __version__ != version: + return {"restart": "mypy version changed"} + if self.fine_grained_manager: + manager = self.fine_grained_manager.manager + start_plugins_snapshot = manager.plugins_snapshot + _, current_plugins_snapshot = mypy.build.load_plugins( + options, manager.errors, sys.stdout, extra_plugins=() + ) + if current_plugins_snapshot != start_plugins_snapshot: + return {"restart": "plugins changed"} + except InvalidSourceList as err: + return {"out": "", "err": str(err), "status": 2} + except SystemExit as e: + return {"out": stdout.getvalue(), "err": stderr.getvalue(), "status": e.code} + return self.check(sources, export_types, is_tty, terminal_width) + + def cmd_check( + self, files: Sequence[str], export_types: bool, is_tty: bool, terminal_width: int + ) -> dict[str, object]: + """Check a list of files.""" + try: + sources = create_source_list(files, self.options, self.fscache) + except InvalidSourceList as err: + return {"out": "", "err": str(err), "status": 2} + return self.check(sources, export_types, is_tty, terminal_width) + + def cmd_recheck( + self, + is_tty: bool, + terminal_width: int, + export_types: bool, + remove: list[str] | None = None, + update: list[str] | None = None, + ) -> dict[str, object]: + """Check the same list of files we checked most recently. + + If remove/update is given, they modify the previous list; + if all are None, stat() is called for each file in the previous list. + """ + t0 = time.time() + if not self.fine_grained_manager: + return {"error": "Command 'recheck' is only valid after a 'check' command"} + sources = self.previous_sources + if remove: + removals = set(remove) + sources = [s for s in sources if s.path and s.path not in removals] + if update: + # Sort list of file updates by extension, so *.pyi files are first. + update.sort(key=lambda f: os.path.splitext(f)[1], reverse=True) + + known = {s.path for s in sources if s.path} + added = [p for p in update if p not in known] + try: + added_sources = create_source_list(added, self.options, self.fscache) + except InvalidSourceList as err: + return {"out": "", "err": str(err), "status": 2} + sources = sources + added_sources # Make a copy! + t1 = time.time() + manager = self.fine_grained_manager.manager + manager.log(f"fine-grained increment: cmd_recheck: {t1 - t0:.3f}s") + old_export_types = self.options.export_types + self.options.export_types = self.options.export_types or export_types + if not self.following_imports(): + messages = self.fine_grained_increment( + sources, remove, update, explicit_export_types=export_types + ) + else: + assert remove is None and update is None + messages = self.fine_grained_increment_follow_imports( + sources, explicit_export_types=export_types + ) + res = self.increment_output(messages, sources, is_tty, terminal_width) + self.flush_caches() + self.update_stats(res) + self.options.export_types = old_export_types + return res + + def check( + self, sources: list[BuildSource], export_types: bool, is_tty: bool, terminal_width: int + ) -> dict[str, Any]: + """Check using fine-grained incremental mode. + + If is_tty is True format the output nicely with colors and summary line + (unless disabled in self.options). Also pass the terminal_width to formatter. + """ + old_export_types = self.options.export_types + self.options.export_types = self.options.export_types or export_types + if not self.fine_grained_manager: + res = self.initialize_fine_grained(sources, is_tty, terminal_width) + else: + if not self.following_imports(): + messages = self.fine_grained_increment(sources, explicit_export_types=export_types) + else: + messages = self.fine_grained_increment_follow_imports( + sources, explicit_export_types=export_types + ) + res = self.increment_output(messages, sources, is_tty, terminal_width) + self.flush_caches() + self.update_stats(res) + self.options.export_types = old_export_types + return res + + def flush_caches(self) -> None: + self.fscache.flush() + if self.fine_grained_manager: + self.fine_grained_manager.flush_cache() + + def update_stats(self, res: dict[str, Any]) -> None: + if self.fine_grained_manager: + manager = self.fine_grained_manager.manager + manager.dump_stats() + res["stats"] = manager.stats + manager.stats = {} + + def following_imports(self) -> bool: + """Are we following imports?""" + # TODO: What about silent? + return self.options.follow_imports == "normal" + + def initialize_fine_grained( + self, sources: list[BuildSource], is_tty: bool, terminal_width: int + ) -> dict[str, Any]: + self.fswatcher = FileSystemWatcher(self.fscache) + t0 = time.time() + self.update_sources(sources) + t1 = time.time() + try: + result = mypy.build.build(sources=sources, options=self.options, fscache=self.fscache) + except mypy.errors.CompileError as e: + output = "".join(s + "\n" for s in e.messages) + if e.use_stdout: + out, err = output, "" + else: + out, err = "", output + return {"out": out, "err": err, "status": 2} + messages = result.errors + self.fine_grained_manager = FineGrainedBuildManager(result) + + original_sources_len = len(sources) + if self.following_imports(): + sources = find_all_sources_in_build(self.fine_grained_manager.graph, sources) + self.update_sources(sources) + + self.previous_sources = sources + + # If we are using the fine-grained cache, build hasn't actually done + # the typechecking on the updated files yet. + # Run a fine-grained update starting from the cached data + if result.used_cache: + t2 = time.time() + # Pull times and hashes out of the saved_cache and stick them into + # the fswatcher, so we pick up the changes. + for state in self.fine_grained_manager.graph.values(): + meta = state.meta + if meta is None: + continue + assert state.path is not None + self.fswatcher.set_file_data( + state.path, + FileData(st_mtime=float(meta.mtime), st_size=meta.size, hash=meta.hash), + ) + + changed, removed = self.find_changed(sources) + changed += self.find_added_suppressed( + self.fine_grained_manager.graph, + set(), + self.fine_grained_manager.manager.search_paths, + ) + + # Find anything that has had its dependency list change + for state in self.fine_grained_manager.graph.values(): + if not state.is_fresh(): + assert state.path is not None + changed.append((state.id, state.path)) + + t3 = time.time() + # Run an update + messages = self.fine_grained_manager.update(changed, removed) + + if self.following_imports(): + # We need to do another update to any new files found by following imports. + messages = self.fine_grained_increment_follow_imports(sources) + + t4 = time.time() + self.fine_grained_manager.manager.add_stats( + update_sources_time=t1 - t0, + build_time=t2 - t1, + find_changes_time=t3 - t2, + fg_update_time=t4 - t3, + files_changed=len(removed) + len(changed), + ) + + else: + # Stores the initial state of sources as a side effect. + self.fswatcher.find_changed() + + if MEM_PROFILE: + from mypy.memprofile import print_memory_profile + + print_memory_profile(run_gc=False) + + __, n_notes, __ = count_stats(messages) + status = 1 if messages and n_notes < len(messages) else 0 + # We use explicit sources length to match the logic in non-incremental mode. + messages = self.pretty_messages(messages, original_sources_len, is_tty, terminal_width) + return {"out": "".join(s + "\n" for s in messages), "err": "", "status": status} + + def fine_grained_increment( + self, + sources: list[BuildSource], + remove: list[str] | None = None, + update: list[str] | None = None, + explicit_export_types: bool = False, + ) -> list[str]: + """Perform a fine-grained type checking increment. + + If remove and update are None, determine changed paths by using + fswatcher. Otherwise, assume that only these files have changes. + + Args: + sources: sources passed on the command line + remove: paths of files that have been removed + update: paths of files that have been changed or created + explicit_export_types: --export-type was passed in a check command + (as opposite to being set in dmypy start) + """ + assert self.fine_grained_manager is not None + manager = self.fine_grained_manager.manager + + t0 = time.time() + if remove is None and update is None: + # Use the fswatcher to determine which files were changed + # (updated or added) or removed. + self.update_sources(sources) + changed, removed = self.find_changed(sources) + else: + # Use the remove/update lists to update fswatcher. + # This avoids calling stat() for unchanged files. + changed, removed = self.update_changed(sources, remove or [], update or []) + if explicit_export_types: + # If --export-types is given, we need to force full re-checking of all + # explicitly passed files, since we need to visit each expression. + add_all_sources_to_changed(sources, changed) + changed += self.find_added_suppressed( + self.fine_grained_manager.graph, set(), manager.search_paths + ) + manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) + t1 = time.time() + manager.log(f"fine-grained increment: find_changed: {t1 - t0:.3f}s") + messages = self.fine_grained_manager.update(changed, removed) + t2 = time.time() + manager.log(f"fine-grained increment: update: {t2 - t1:.3f}s") + manager.add_stats( + find_changes_time=t1 - t0, + fg_update_time=t2 - t1, + files_changed=len(removed) + len(changed), + ) + + self.previous_sources = sources + return messages + + def fine_grained_increment_follow_imports( + self, sources: list[BuildSource], explicit_export_types: bool = False + ) -> list[str]: + """Like fine_grained_increment, but follow imports.""" + t0 = time.time() + + # TODO: Support file events + + assert self.fine_grained_manager is not None + fine_grained_manager = self.fine_grained_manager + graph = fine_grained_manager.graph + manager = fine_grained_manager.manager + + orig_modules = list(graph.keys()) + + self.update_sources(sources) + changed_paths = self.fswatcher.find_changed() + manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) + + t1 = time.time() + manager.log(f"fine-grained increment: find_changed: {t1 - t0:.3f}s") + + seen = {source.module for source in sources} + + # Find changed modules reachable from roots (or in roots) already in graph. + changed, new_files = self.find_reachable_changed_modules( + sources, graph, seen, changed_paths + ) + # Same as in fine_grained_increment(). + self.add_explicitly_new(sources, changed) + if explicit_export_types: + # Same as in fine_grained_increment(). + add_all_sources_to_changed(sources, changed) + sources.extend(new_files) + + # Process changes directly reachable from roots. + messages = fine_grained_manager.update(changed, [], followed=True) + + # Follow deps from changed modules (still within graph). + worklist = changed.copy() + while worklist: + module = worklist.pop() + if module[0] not in graph: + continue + sources2 = self.direct_imports(module, graph) + # Filter anything already seen before. This prevents + # infinite looping if there are any self edges. (Self + # edges are maybe a bug, but...) + sources2 = [source for source in sources2 if source.module not in seen] + changed, new_files = self.find_reachable_changed_modules( + sources2, graph, seen, changed_paths + ) + self.update_sources(new_files) + messages = fine_grained_manager.update(changed, [], followed=True) + worklist.extend(changed) + + t2 = time.time() + + def refresh_file(module: str, path: str) -> list[str]: + return fine_grained_manager.update([(module, path)], [], followed=True) + + for module_id, state in list(graph.items()): + new_messages = refresh_suppressed_submodules( + module_id, state.path, fine_grained_manager.deps, graph, self.fscache, refresh_file + ) + if new_messages is not None: + messages = new_messages + + t3 = time.time() + + # There may be new files that became available, currently treated as + # suppressed imports. Process them. + while True: + new_unsuppressed = self.find_added_suppressed(graph, seen, manager.search_paths) + if not new_unsuppressed: + break + new_files = [BuildSource(mod[1], mod[0], followed=True) for mod in new_unsuppressed] + sources.extend(new_files) + self.update_sources(new_files) + messages = fine_grained_manager.update(new_unsuppressed, [], followed=True) + + for module_id, path in new_unsuppressed: + new_messages = refresh_suppressed_submodules( + module_id, path, fine_grained_manager.deps, graph, self.fscache, refresh_file + ) + if new_messages is not None: + messages = new_messages + + t4 = time.time() + + # Find all original modules in graph that were not reached -- they are deleted. + to_delete = [] + for module_id in orig_modules: + if module_id not in graph: + continue + if module_id not in seen: + module_path = graph[module_id].path + assert module_path is not None + to_delete.append((module_id, module_path)) + if to_delete: + messages = fine_grained_manager.update([], to_delete) + + fix_module_deps(graph) + + self.previous_sources = find_all_sources_in_build(graph) + self.update_sources(self.previous_sources) + + # Store current file state as side effect + self.fswatcher.find_changed() + + t5 = time.time() + + manager.log(f"fine-grained increment: update: {t5 - t1:.3f}s") + manager.add_stats( + find_changes_time=t1 - t0, + fg_update_time=t2 - t1, + refresh_suppressed_time=t3 - t2, + find_added_supressed_time=t4 - t3, + cleanup_time=t5 - t4, + ) + + return messages + + def find_reachable_changed_modules( + self, + roots: list[BuildSource], + graph: mypy.build.Graph, + seen: set[str], + changed_paths: AbstractSet[str], + ) -> tuple[list[tuple[str, str]], list[BuildSource]]: + """Follow imports within graph from given sources until hitting changed modules. + + If we find a changed module, we can't continue following imports as the imports + may have changed. + + Args: + roots: modules where to start search from + graph: module graph to use for the search + seen: modules we've seen before that won't be visited (mutated here!!) + changed_paths: which paths have changed (stop search here and return any found) + + Return (encountered reachable changed modules, + unchanged files not in sources_set traversed). + """ + changed = [] + new_files = [] + worklist = roots.copy() + seen.update(source.module for source in worklist) + while worklist: + nxt = worklist.pop() + if nxt.module not in seen: + seen.add(nxt.module) + new_files.append(nxt) + if nxt.path in changed_paths: + assert nxt.path is not None # TODO + changed.append((nxt.module, nxt.path)) + elif nxt.module in graph: + state = graph[nxt.module] + for dep in state.dependencies: + if dep not in seen: + seen.add(dep) + worklist.append(BuildSource(graph[dep].path, graph[dep].id, followed=True)) + return changed, new_files + + def direct_imports( + self, module: tuple[str, str], graph: mypy.build.Graph + ) -> list[BuildSource]: + """Return the direct imports of module not included in seen.""" + state = graph[module[0]] + return [BuildSource(graph[dep].path, dep, followed=True) for dep in state.dependencies] + + def find_added_suppressed( + self, graph: mypy.build.Graph, seen: set[str], search_paths: SearchPaths + ) -> list[tuple[str, str]]: + """Find suppressed modules that have been added (and not included in seen). + + Args: + seen: reachable modules we've seen before (mutated here!!) + + Return suppressed, added modules. + """ + all_suppressed = set() + for state in graph.values(): + all_suppressed |= state.suppressed_set + + # Filter out things that shouldn't actually be considered suppressed. + # + # TODO: Figure out why these are treated as suppressed + all_suppressed = { + module + for module in all_suppressed + if module not in graph and not ignore_suppressed_imports(module) + } + + # Optimization: skip top-level packages that are obviously not + # there, to avoid calling the relatively slow find_module() + # below too many times. + packages = {module.split(".", 1)[0] for module in all_suppressed} + packages = filter_out_missing_top_level_packages(packages, search_paths, self.fscache) + + # TODO: Namespace packages + + finder = FindModuleCache(search_paths, self.fscache, self.options) + + found = [] + + for module in all_suppressed: + top_level_pkg = module.split(".", 1)[0] + if top_level_pkg not in packages: + # Fast path: non-existent top-level package + continue + result = finder.find_module(module, fast_path=True) + if isinstance(result, str) and module not in seen: + # When not following imports, we only follow imports to .pyi files. + if not self.following_imports() and not result.endswith(".pyi"): + continue + found.append((module, result)) + seen.add(module) + + return found + + def increment_output( + self, messages: list[str], sources: list[BuildSource], is_tty: bool, terminal_width: int + ) -> dict[str, Any]: + status = 1 if messages else 0 + messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) + return {"out": "".join(s + "\n" for s in messages), "err": "", "status": status} + + def pretty_messages( + self, + messages: list[str], + n_sources: int, + is_tty: bool = False, + terminal_width: int | None = None, + ) -> list[str]: + use_color = self.options.color_output and is_tty + fit_width = self.options.pretty and is_tty + if fit_width: + messages = self.formatter.fit_in_terminal( + messages, fixed_terminal_width=terminal_width + ) + if self.options.error_summary: + summary: str | None = None + n_errors, n_notes, n_files = count_stats(messages) + if n_errors: + summary = self.formatter.format_error( + n_errors, n_files, n_sources, use_color=use_color + ) + elif not messages or n_notes == len(messages): + summary = self.formatter.format_success(n_sources, use_color) + if summary: + # Create new list to avoid appending multiple summaries on successive runs. + messages = messages + [summary] + if use_color: + messages = [self.formatter.colorize(m) for m in messages] + return messages + + def update_sources(self, sources: list[BuildSource]) -> None: + paths = [source.path for source in sources if source.path is not None] + if self.following_imports(): + # Filter out directories (used for namespace packages). + paths = [path for path in paths if self.fscache.isfile(path)] + self.fswatcher.add_watched_paths(paths) + + def update_changed( + self, sources: list[BuildSource], remove: list[str], update: list[str] + ) -> ChangesAndRemovals: + changed_paths = self.fswatcher.update_changed(remove, update) + return self._find_changed(sources, changed_paths) + + def find_changed(self, sources: list[BuildSource]) -> ChangesAndRemovals: + changed_paths = self.fswatcher.find_changed() + return self._find_changed(sources, changed_paths) + + def _find_changed( + self, sources: list[BuildSource], changed_paths: AbstractSet[str] + ) -> ChangesAndRemovals: + # Find anything that has been added or modified + changed = [ + (source.module, source.path) + for source in sources + if source.path and source.path in changed_paths + ] + + # Now find anything that has been removed from the build + modules = {source.module for source in sources} + omitted = [source for source in self.previous_sources if source.module not in modules] + removed = [] + for source in omitted: + path = source.path + assert path + removed.append((source.module, path)) + + self.add_explicitly_new(sources, changed) + + # Find anything that has had its module path change because of added or removed __init__s + last = {s.path: s.module for s in self.previous_sources} + for s in sources: + assert s.path + if s.path in last and last[s.path] != s.module: + # Mark it as removed from its old name and changed at its new name + removed.append((last[s.path], s.path)) + changed.append((s.module, s.path)) + + return changed, removed + + def add_explicitly_new( + self, sources: list[BuildSource], changed: list[tuple[str, str]] + ) -> None: + # Always add modules that were (re-)added, since they may be detected as not changed by + # fswatcher (if they were actually not changed), but they may still need to be checked + # in case they had errors before they were deleted from sources on previous runs. + previous_modules = {source.module for source in self.previous_sources} + changed_set = set(changed) + changed.extend( + [ + (source.module, source.path) + for source in sources + if source.path + and source.module not in previous_modules + and (source.module, source.path) not in changed_set + ] + ) + + def cmd_inspect( + self, + show: str, + location: str, + verbosity: int = 0, + limit: int = 0, + include_span: bool = False, + include_kind: bool = False, + include_object_attrs: bool = False, + union_attrs: bool = False, + force_reload: bool = False, + ) -> dict[str, object]: + """Locate and inspect expression(s).""" + if not self.fine_grained_manager: + return { + "error": 'Command "inspect" is only valid after a "check" command' + " (that produces no parse errors)" + } + engine = InspectionEngine( + self.fine_grained_manager, + verbosity=verbosity, + limit=limit, + include_span=include_span, + include_kind=include_kind, + include_object_attrs=include_object_attrs, + union_attrs=union_attrs, + force_reload=force_reload, + ) + old_inspections = self.options.inspections + self.options.inspections = True + try: + if show == "type": + result = engine.get_type(location) + elif show == "attrs": + result = engine.get_attrs(location) + elif show == "definition": + result = engine.get_definition(location) + else: + assert False, "Unknown inspection kind" + finally: + self.options.inspections = old_inspections + if "out" in result: + assert isinstance(result["out"], str) + result["out"] += "\n" + return result + + def cmd_suggest(self, function: str, callsites: bool, **kwargs: Any) -> dict[str, object]: + """Suggest a signature for a function.""" + if not self.fine_grained_manager: + return { + "error": "Command 'suggest' is only valid after a 'check' command" + " (that produces no parse errors)" + } + engine = SuggestionEngine(self.fine_grained_manager, **kwargs) + try: + if callsites: + out = engine.suggest_callsites(function) + else: + out = engine.suggest(function) + except SuggestionFailure as err: + return {"error": str(err)} + else: + if not out: + out = "No suggestions\n" + elif not out.endswith("\n"): + out += "\n" + return {"out": out, "err": "", "status": 0} + finally: + self.flush_caches() + + def cmd_hang(self) -> dict[str, object]: + """Hang for 100 seconds, as a debug hack.""" + time.sleep(100) + return {} + + +# Misc utilities. + + +MiB: Final = 2**20 + + +def get_meminfo() -> dict[str, Any]: + res: dict[str, Any] = {} + try: + import psutil + except ImportError: + res["memory_psutil_missing"] = ( + "psutil not found, run pip install mypy[dmypy] " + "to install the needed components for dmypy" + ) + else: + process = psutil.Process() + meminfo = process.memory_info() + res["memory_rss_mib"] = meminfo.rss / MiB + res["memory_vms_mib"] = meminfo.vms / MiB + if sys.platform == "win32": + res["memory_maxrss_mib"] = meminfo.peak_wset / MiB + else: + # See https://stackoverflow.com/questions/938733/total-memory-used-by-python-process + import resource # Since it doesn't exist on Windows. + + rusage = resource.getrusage(resource.RUSAGE_SELF) + if sys.platform == "darwin": + factor = 1 + else: + factor = 1024 # Linux + res["memory_maxrss_mib"] = rusage.ru_maxrss * factor / MiB + return res + + +def find_all_sources_in_build( + graph: mypy.build.Graph, extra: Sequence[BuildSource] = () +) -> list[BuildSource]: + result = list(extra) + seen = {source.module for source in result} + for module, state in graph.items(): + if module not in seen: + result.append(BuildSource(state.path, module)) + return result + + +def add_all_sources_to_changed(sources: list[BuildSource], changed: list[tuple[str, str]]) -> None: + """Add all (explicit) sources to the list changed files in place. + + Use this when re-processing of unchanged files is needed (e.g. for + the purpose of exporting types for inspections). + """ + changed_set = set(changed) + changed.extend( + [ + (bs.module, bs.path) + for bs in sources + if bs.path and (bs.module, bs.path) not in changed_set + ] + ) + + +def fix_module_deps(graph: mypy.build.Graph) -> None: + """After an incremental update, update module dependencies to reflect the new state. + + This can make some suppressed dependencies non-suppressed, and vice versa (if modules + have been added to or removed from the build). + """ + for state in graph.values(): + new_suppressed = [] + new_dependencies = [] + for dep in state.dependencies + state.suppressed: + if dep in graph: + new_dependencies.append(dep) + else: + new_suppressed.append(dep) + state.dependencies = new_dependencies + state.dependencies_set = set(new_dependencies) + state.suppressed = new_suppressed + state.suppressed_set = set(new_suppressed) + + +def filter_out_missing_top_level_packages( + packages: set[str], search_paths: SearchPaths, fscache: FileSystemCache +) -> set[str]: + """Quickly filter out obviously missing top-level packages. + + Return packages with entries that can't be found removed. + + This is approximate: some packages that aren't actually valid may be + included. However, all potentially valid packages must be returned. + """ + # Start with a empty set and add all potential top-level packages. + found = set() + paths = ( + search_paths.python_path + + search_paths.mypy_path + + search_paths.package_path + + search_paths.typeshed_path + ) + for p in paths: + try: + entries = fscache.listdir(p) + except Exception: + entries = [] + for entry in entries: + # The code is hand-optimized for mypyc since this may be somewhat + # performance-critical. + if entry.endswith(".py"): + entry = entry[:-3] + elif entry.endswith(".pyi"): + entry = entry[:-4] + elif entry.endswith("-stubs"): + # Possible PEP 561 stub package + entry = entry[:-6] + if entry in packages: + found.add(entry) + return found diff --git a/mypy/dmypy_util.py b/mypy/dmypy_util.py new file mode 100644 index 000000000000..9b21d78ce599 --- /dev/null +++ b/mypy/dmypy_util.py @@ -0,0 +1,116 @@ +"""Shared code between dmypy.py and dmypy_server.py. + +This should be pretty lightweight and not depend on other mypy code (other than ipc). +""" + +from __future__ import annotations + +import io +import json +from types import TracebackType +from typing import Any, Final, Iterable, Iterator, TextIO + +from mypy.ipc import IPCBase + +DEFAULT_STATUS_FILE: Final = ".dmypy.json" + + +def receive(connection: IPCBase) -> Any: + """Receive single JSON data frame from a connection. + + Raise OSError if the data received is not valid JSON or if it is + not a dict. + """ + bdata = connection.read() + if not bdata: + raise OSError("No data received") + try: + data = json.loads(bdata) + except Exception as e: + raise OSError("Data received is not valid JSON") from e + if not isinstance(data, dict): + raise OSError(f"Data received is not a dict ({type(data)})") + return data + + +def send(connection: IPCBase, data: Any) -> None: + """Send data to a connection encoded and framed. + + The data must be JSON-serializable. We assume that a single send call is a + single frame to be sent on the connect. + """ + connection.write(json.dumps(data)) + + +class WriteToConn(TextIO): + """Helper class to write to a connection instead of standard output.""" + + def __init__(self, server: IPCBase, output_key: str, isatty: bool) -> None: + self.server = server + self.output_key = output_key + self._isatty = isatty + + def __enter__(self) -> TextIO: + return self + + def __exit__( + self, + t: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + pass + + def __iter__(self) -> Iterator[str]: + raise io.UnsupportedOperation + + def __next__(self) -> str: + raise io.UnsupportedOperation + + def close(self) -> None: + pass + + def fileno(self) -> int: + raise OSError + + def flush(self) -> None: + pass + + def isatty(self) -> bool: + return self._isatty + + def read(self, n: int = 0) -> str: + raise io.UnsupportedOperation + + def readable(self) -> bool: + return False + + def readline(self, limit: int = 0) -> str: + raise io.UnsupportedOperation + + def readlines(self, hint: int = 0) -> list[str]: + raise io.UnsupportedOperation + + def seek(self, offset: int, whence: int = 0) -> int: + raise io.UnsupportedOperation + + def seekable(self) -> bool: + return False + + def tell(self) -> int: + raise io.UnsupportedOperation + + def truncate(self, size: int | None = 0) -> int: + raise io.UnsupportedOperation + + def write(self, output: str) -> int: + resp: dict[str, Any] = {self.output_key: output} + send(self.server, resp) + return len(output) + + def writable(self) -> bool: + return True + + def writelines(self, lines: Iterable[str]) -> None: + for s in lines: + self.write(s) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 1f0a2b7ba67f..222e7f2a6d7a 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -1,104 +1,270 @@ -import typing +from __future__ import annotations +from typing import Callable, Container, cast + +from mypy.nodes import ARG_STAR, ARG_STAR2 from mypy.types import ( - Type, TypeVisitor, UnboundType, ErrorType, AnyType, Void, NoneTyp, - Instance, TypeVar, Callable, TupleType, UnionType, Overloaded, ErasedType, - TypeTranslator, TypeList + AnyType, + CallableType, + DeletedType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeTranslator, + TypeType, + TypeVarId, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, + get_proper_types, ) +from mypy.typevartuples import erased_vars -def erase_type(typ: Type) -> Type: +def erase_type(typ: Type) -> ProperType: """Erase any type variables from a type. - Also replace tuple types with the corresponding concrete types. Replace - callable types with empty callable types. + Also replace tuple types with the corresponding concrete types. Examples: A -> A B[X] -> B[Any] Tuple[A, B] -> tuple - Function[...] -> Function[[], None] + Callable[[A1, A2, ...], R] -> Callable[..., Any] + Type[X] -> Type[Any] """ - + typ = get_proper_type(typ) return typ.accept(EraseTypeVisitor()) -class EraseTypeVisitor(TypeVisitor[Type]): - def visit_unbound_type(self, t: UnboundType) -> Type: - assert False, 'Not supported' +class EraseTypeVisitor(TypeVisitor[ProperType]): + def visit_unbound_type(self, t: UnboundType) -> ProperType: + # TODO: replace with an assert after UnboundType can't leak from semantic analysis. + return AnyType(TypeOfAny.from_error) - def visit_error_type(self, t: ErrorType) -> Type: + def visit_any(self, t: AnyType) -> ProperType: return t - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: + def visit_none_type(self, t: NoneType) -> ProperType: return t - def visit_void(self, t: Void) -> Type: + def visit_uninhabited_type(self, t: UninhabitedType) -> ProperType: return t - def visit_none_type(self, t: NoneTyp) -> Type: + def visit_erased_type(self, t: ErasedType) -> ProperType: return t - def visit_erased_type(self, t: ErasedType) -> Type: + def visit_partial_type(self, t: PartialType) -> ProperType: # Should not get here. - raise RuntimeError() + raise RuntimeError("Cannot erase partial types") - def visit_instance(self, t: Instance) -> Type: - return Instance(t.type, [AnyType()] * len(t.args), t.line, - t.repr) + def visit_deleted_type(self, t: DeletedType) -> ProperType: + return t + + def visit_instance(self, t: Instance) -> ProperType: + args = erased_vars(t.type.defn.type_vars, TypeOfAny.special_form) + return Instance(t.type, args, t.line) + + def visit_type_var(self, t: TypeVarType) -> ProperType: + return AnyType(TypeOfAny.special_form) + + def visit_param_spec(self, t: ParamSpecType) -> ProperType: + return AnyType(TypeOfAny.special_form) - def visit_type_var(self, t: TypeVar) -> Type: - return AnyType() + def visit_parameters(self, t: Parameters) -> ProperType: + raise RuntimeError("Parameters should have been bound to a class") - def visit_callable(self, t: Callable) -> Type: + def visit_type_var_tuple(self, t: TypeVarTupleType) -> ProperType: + # Likely, we can never get here because of aggressive erasure of types that + # can contain this, but better still return a valid replacement. + return t.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)]) + + def visit_unpack_type(self, t: UnpackType) -> ProperType: + return AnyType(TypeOfAny.special_form) + + def visit_callable_type(self, t: CallableType) -> ProperType: # We must preserve the fallback type for overload resolution to work. - return Callable([], [], [], Void(), t.fallback) + any_type = AnyType(TypeOfAny.special_form) + return CallableType( + arg_types=[any_type, any_type], + arg_kinds=[ARG_STAR, ARG_STAR2], + arg_names=[None, None], + ret_type=any_type, + fallback=t.fallback, + is_ellipsis_args=True, + implicit=True, + ) + + def visit_overloaded(self, t: Overloaded) -> ProperType: + return t.fallback.accept(self) + + def visit_tuple_type(self, t: TupleType) -> ProperType: + return t.partial_fallback.accept(self) + + def visit_typeddict_type(self, t: TypedDictType) -> ProperType: + return t.fallback.accept(self) + + def visit_literal_type(self, t: LiteralType) -> ProperType: + # The fallback for literal types should always be either + # something like int or str, or an enum class -- types that + # don't contain any TypeVars. So there's no need to visit it. + return t - def visit_overloaded(self, t: Overloaded) -> Type: - return t.items()[0].accept(self) + def visit_union_type(self, t: UnionType) -> ProperType: + erased_items = [erase_type(item) for item in t.items] + from mypy.typeops import make_simplified_union - def visit_tuple_type(self, t: TupleType) -> Type: - return t.fallback + return make_simplified_union(erased_items) - def visit_union_type(self, t: UnionType) -> Type: - return AnyType() # XXX: return underlying type if only one? + def visit_type_type(self, t: TypeType) -> ProperType: + return TypeType.make_normalized(t.item.accept(self), line=t.line) + def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: + raise RuntimeError("Type aliases should be expanded before accepting this visitor") -def erase_generic_types(t: Type) -> Type: - """Remove generic type arguments and type variables from a type. - Replace all types A[...] with simply A, and all type variables - with 'Any'. +def erase_typevars(t: Type, ids_to_erase: Container[TypeVarId] | None = None) -> Type: + """Replace all type variables in a type with any, + or just the ones in the provided collection. """ - if t: - return t.accept(GenericTypeEraser()) - else: - return None + def erase_id(id: TypeVarId) -> bool: + if ids_to_erase is None: + return True + return id in ids_to_erase + + return t.accept(TypeVarEraser(erase_id, AnyType(TypeOfAny.special_form))) + + +def replace_meta_vars(t: Type, target_type: Type) -> Type: + """Replace unification variables in a type with the target type.""" + return t.accept(TypeVarEraser(lambda id: id.is_meta_var(), target_type)) -class GenericTypeEraser(TypeTranslator): +class TypeVarEraser(TypeTranslator): """Implementation of type erasure""" - # FIX: What about generic function types? + def __init__(self, erase_id: Callable[[TypeVarId], bool], replacement: Type) -> None: + super().__init__() + self.erase_id = erase_id + self.replacement = replacement - def visit_type_var(self, t: TypeVar) -> Type: - return AnyType() + def visit_type_var(self, t: TypeVarType) -> Type: + if self.erase_id(t.id): + return self.replacement + return t + # TODO: below two methods duplicate some logic with expand_type(). + # In fact, we may want to refactor this whole visitor to use expand_type(). def visit_instance(self, t: Instance) -> Type: - return Instance(t.type, [], t.line) + result = super().visit_instance(t) + assert isinstance(result, ProperType) and isinstance(result, Instance) + if t.type.fullname == "builtins.tuple": + # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] + arg = result.args[0] + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + return unpacked + return result + + def visit_tuple_type(self, t: TupleType) -> Type: + result = super().visit_tuple_type(t) + assert isinstance(result, ProperType) and isinstance(result, TupleType) + if len(result.items) == 1: + # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] + item = result.items[0] + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + if result.partial_fallback.type.fullname != "builtins.tuple": + # If it is a subtype (like named tuple) we need to preserve it, + # this essentially mimics the logic in tuple_fallback(). + return result.partial_fallback.accept(self) + return unpacked + return result + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: + if self.erase_id(t.id): + return t.tuple_fallback.copy_modified(args=[self.replacement]) + return t + def visit_param_spec(self, t: ParamSpecType) -> Type: + if self.erase_id(t.id): + return self.replacement + return t -def erase_typevars(t: Type) -> Type: - """Replace all type variables in a type with any.""" - return t.accept(TypeVarEraser()) + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # Type alias target can't contain bound type variables (not bound by the type + # alias itself), so it is safe to just erase the arguments. + return t.copy_modified(args=[a.accept(self) for a in t.args]) -class TypeVarEraser(TypeTranslator): - """Implementation of type erasure""" +def remove_instance_last_known_values(t: Type) -> Type: + return t.accept(LastKnownValueEraser()) - def visit_type_var(self, t: TypeVar) -> Type: - return AnyType() + +class LastKnownValueEraser(TypeTranslator): + """Removes the Literal[...] type that may be associated with any + Instance types.""" + + def visit_instance(self, t: Instance) -> Type: + if not t.last_known_value and not t.args: + return t + return t.copy_modified(args=[a.accept(self) for a in t.args], last_known_value=None) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # Type aliases can't contain literal values, because they are + # always constructed as explicit types. + return t + + def visit_union_type(self, t: UnionType) -> Type: + new = cast(UnionType, super().visit_union_type(t)) + # Erasure can result in many duplicate items; merge them. + # Call make_simplified_union only on lists of instance types + # that all have the same fullname, to avoid simplifying too + # much. + instances = [item for item in new.items if isinstance(get_proper_type(item), Instance)] + # Avoid merge in simple cases such as optional types. + if len(instances) > 1: + instances_by_name: dict[str, list[Instance]] = {} + p_new_items = get_proper_types(new.items) + for p_item in p_new_items: + if isinstance(p_item, Instance) and not p_item.args: + instances_by_name.setdefault(p_item.type.fullname, []).append(p_item) + merged: list[Type] = [] + for item in new.items: + orig_item = item + item = get_proper_type(item) + if isinstance(item, Instance) and not item.args: + types = instances_by_name.get(item.type.fullname) + if types is not None: + if len(types) == 1: + merged.append(item) + else: + from mypy.typeops import make_simplified_union + + merged.append(make_simplified_union(types)) + del instances_by_name[item.type.fullname] + else: + merged.append(orig_item) + return UnionType.make_union(merged) + return new diff --git a/mypy/error_formatter.py b/mypy/error_formatter.py new file mode 100644 index 000000000000..ffc6b6747596 --- /dev/null +++ b/mypy/error_formatter.py @@ -0,0 +1,37 @@ +"""Defines the different custom formats in which mypy can output.""" + +import json +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from mypy.errors import MypyError + + +class ErrorFormatter(ABC): + """Base class to define how errors are formatted before being printed.""" + + @abstractmethod + def report_error(self, error: "MypyError") -> str: + raise NotImplementedError + + +class JSONFormatter(ErrorFormatter): + """Formatter for basic JSON output format.""" + + def report_error(self, error: "MypyError") -> str: + """Prints out the errors as simple, static JSON lines.""" + return json.dumps( + { + "file": error.file_path, + "line": error.line, + "column": error.column, + "message": error.message, + "hint": None if len(error.hints) == 0 else "\n".join(error.hints), + "code": None if error.errorcode is None else error.errorcode.code, + "severity": error.severity, + } + ) + + +OUTPUT_CHOICES = {"json": JSONFormatter()} diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py new file mode 100644 index 000000000000..5736be5c143e --- /dev/null +++ b/mypy/errorcodes.py @@ -0,0 +1,315 @@ +"""Classification of possible errors mypy can detect. + +These can be used for filtering specific errors. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Final + +from mypy_extensions import mypyc_attr + +error_codes: dict[str, ErrorCode] = {} +sub_code_map: dict[str, set[str]] = defaultdict(set) + + +@mypyc_attr(allow_interpreted_subclasses=True) +class ErrorCode: + def __init__( + self, + code: str, + description: str, + category: str, + default_enabled: bool = True, + sub_code_of: ErrorCode | None = None, + ) -> None: + self.code = code + self.description = description + self.category = category + self.default_enabled = default_enabled + self.sub_code_of = sub_code_of + if sub_code_of is not None: + assert sub_code_of.sub_code_of is None, "Nested subcategories are not supported" + sub_code_map[sub_code_of.code].add(code) + error_codes[code] = self + + def __str__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ErrorCode): + return False + return self.code == other.code + + def __hash__(self) -> int: + return hash((self.code,)) + + +ATTR_DEFINED: Final = ErrorCode("attr-defined", "Check that attribute exists", "General") +NAME_DEFINED: Final = ErrorCode("name-defined", "Check that name is defined", "General") +CALL_ARG: Final[ErrorCode] = ErrorCode( + "call-arg", "Check number, names and kinds of arguments in calls", "General" +) +ARG_TYPE: Final = ErrorCode("arg-type", "Check argument types in calls", "General") +CALL_OVERLOAD: Final = ErrorCode( + "call-overload", "Check that an overload variant matches arguments", "General" +) +VALID_TYPE: Final[ErrorCode] = ErrorCode( + "valid-type", "Check that type (annotation) is valid", "General" +) +VAR_ANNOTATED: Final = ErrorCode( + "var-annotated", "Require variable annotation if type can't be inferred", "General" +) +OVERRIDE: Final = ErrorCode( + "override", "Check that method override is compatible with base class", "General" +) +RETURN: Final[ErrorCode] = ErrorCode( + "return", "Check that function always returns a value", "General" +) +RETURN_VALUE: Final[ErrorCode] = ErrorCode( + "return-value", "Check that return value is compatible with signature", "General" +) +ASSIGNMENT: Final[ErrorCode] = ErrorCode( + "assignment", "Check that assigned value is compatible with target", "General" +) +METHOD_ASSIGN: Final[ErrorCode] = ErrorCode( + "method-assign", + "Check that assignment target is not a method", + "General", + sub_code_of=ASSIGNMENT, +) +TYPE_ARG: Final = ErrorCode("type-arg", "Check that generic type arguments are present", "General") +TYPE_VAR: Final = ErrorCode("type-var", "Check that type variable values are valid", "General") +UNION_ATTR: Final = ErrorCode( + "union-attr", "Check that attribute exists in each item of a union", "General" +) +INDEX: Final = ErrorCode("index", "Check indexing operations", "General") +OPERATOR: Final = ErrorCode("operator", "Check that operator is valid for operands", "General") +LIST_ITEM: Final = ErrorCode( + "list-item", "Check list items in a list expression [item, ...]", "General" +) +DICT_ITEM: Final = ErrorCode( + "dict-item", "Check dict items in a dict expression {key: value, ...}", "General" +) +TYPEDDICT_ITEM: Final = ErrorCode( + "typeddict-item", "Check items when constructing TypedDict", "General" +) +TYPEDDICT_UNKNOWN_KEY: Final = ErrorCode( + "typeddict-unknown-key", + "Check unknown keys when constructing TypedDict", + "General", + sub_code_of=TYPEDDICT_ITEM, +) +HAS_TYPE: Final = ErrorCode( + "has-type", "Check that type of reference can be determined", "General" +) +IMPORT: Final = ErrorCode( + "import", "Require that imported module can be found or has stubs", "General" +) +IMPORT_NOT_FOUND: Final = ErrorCode( + "import-not-found", "Require that imported module can be found", "General", sub_code_of=IMPORT +) +IMPORT_UNTYPED: Final = ErrorCode( + "import-untyped", "Require that imported module has stubs", "General", sub_code_of=IMPORT +) +NO_REDEF: Final = ErrorCode("no-redef", "Check that each name is defined once", "General") +FUNC_RETURNS_VALUE: Final = ErrorCode( + "func-returns-value", "Check that called function returns a value in value context", "General" +) +ABSTRACT: Final = ErrorCode( + "abstract", "Prevent instantiation of classes with abstract attributes", "General" +) +TYPE_ABSTRACT: Final = ErrorCode( + "type-abstract", "Require only concrete classes where Type[...] is expected", "General" +) +VALID_NEWTYPE: Final = ErrorCode( + "valid-newtype", "Check that argument 2 to NewType is valid", "General" +) +STRING_FORMATTING: Final = ErrorCode( + "str-format", "Check that string formatting/interpolation is type-safe", "General" +) +STR_BYTES_PY3: Final = ErrorCode( + "str-bytes-safe", "Warn about implicit coercions related to bytes and string types", "General" +) +EXIT_RETURN: Final = ErrorCode( + "exit-return", "Warn about too general return type for '__exit__'", "General" +) +LITERAL_REQ: Final = ErrorCode("literal-required", "Check that value is a literal", "General") +UNUSED_COROUTINE: Final = ErrorCode( + "unused-coroutine", "Ensure that all coroutines are used", "General" +) +# TODO: why do we need the explicit type here? Without it mypyc CI builds fail with +# mypy/message_registry.py:37: error: Cannot determine type of "EMPTY_BODY" [has-type] +EMPTY_BODY: Final[ErrorCode] = ErrorCode( + "empty-body", + "A dedicated error code to opt out return errors for empty/trivial bodies", + "General", +) +SAFE_SUPER: Final = ErrorCode( + "safe-super", "Warn about calls to abstract methods with empty/trivial bodies", "General" +) +TOP_LEVEL_AWAIT: Final = ErrorCode( + "top-level-await", "Warn about top level await expressions", "General" +) +AWAIT_NOT_ASYNC: Final = ErrorCode( + "await-not-async", 'Warn about "await" outside coroutine ("async def")', "General" +) +# These error codes aren't enabled by default. +NO_UNTYPED_DEF: Final[ErrorCode] = ErrorCode( + "no-untyped-def", "Check that every function has an annotation", "General" +) +NO_UNTYPED_CALL: Final = ErrorCode( + "no-untyped-call", + "Disallow calling functions without type annotations from annotated functions", + "General", +) +REDUNDANT_CAST: Final = ErrorCode( + "redundant-cast", "Check that cast changes type of expression", "General" +) +ASSERT_TYPE: Final = ErrorCode("assert-type", "Check that assert_type() call succeeds", "General") +COMPARISON_OVERLAP: Final = ErrorCode( + "comparison-overlap", "Check that types in comparisons and 'in' expressions overlap", "General" +) +NO_ANY_UNIMPORTED: Final = ErrorCode( + "no-any-unimported", 'Reject "Any" types from unfollowed imports', "General" +) +NO_ANY_RETURN: Final = ErrorCode( + "no-any-return", + 'Reject returning value with "Any" type if return type is not "Any"', + "General", +) +UNREACHABLE: Final = ErrorCode( + "unreachable", "Warn about unreachable statements or expressions", "General" +) +ANNOTATION_UNCHECKED = ErrorCode( + "annotation-unchecked", "Notify about type annotations in unchecked functions", "General" +) +TYPEDDICT_READONLY_MUTATED = ErrorCode( + "typeddict-readonly-mutated", "TypedDict's ReadOnly key is mutated", "General" +) +POSSIBLY_UNDEFINED: Final[ErrorCode] = ErrorCode( + "possibly-undefined", + "Warn about variables that are defined only in some execution paths", + "General", + default_enabled=False, +) +REDUNDANT_EXPR: Final = ErrorCode( + "redundant-expr", "Warn about redundant expressions", "General", default_enabled=False +) +TRUTHY_BOOL: Final[ErrorCode] = ErrorCode( + "truthy-bool", + "Warn about expressions that could always evaluate to true in boolean contexts", + "General", + default_enabled=False, +) +TRUTHY_FUNCTION: Final[ErrorCode] = ErrorCode( + "truthy-function", + "Warn about function that always evaluate to true in boolean contexts", + "General", +) +TRUTHY_ITERABLE: Final[ErrorCode] = ErrorCode( + "truthy-iterable", + "Warn about Iterable expressions that could always evaluate to true in boolean contexts", + "General", + default_enabled=False, +) +NAME_MATCH: Final = ErrorCode( + "name-match", "Check that type definition has consistent naming", "General" +) +NO_OVERLOAD_IMPL: Final = ErrorCode( + "no-overload-impl", + "Check that overloaded functions outside stub files have an implementation", + "General", +) +IGNORE_WITHOUT_CODE: Final = ErrorCode( + "ignore-without-code", + "Warn about '# type: ignore' comments which do not have error codes", + "General", + default_enabled=False, +) +UNUSED_AWAITABLE: Final = ErrorCode( + "unused-awaitable", + "Ensure that all awaitable values are used", + "General", + default_enabled=False, +) +REDUNDANT_SELF_TYPE = ErrorCode( + "redundant-self", + "Warn about redundant Self type annotations on method first argument", + "General", + default_enabled=False, +) +USED_BEFORE_DEF: Final[ErrorCode] = ErrorCode( + "used-before-def", "Warn about variables that are used before they are defined", "General" +) +UNUSED_IGNORE: Final = ErrorCode( + "unused-ignore", "Ensure that all type ignores are used", "General", default_enabled=False +) +EXPLICIT_OVERRIDE_REQUIRED: Final = ErrorCode( + "explicit-override", + "Require @override decorator if method is overriding a base class method", + "General", + default_enabled=False, +) +UNIMPORTED_REVEAL: Final = ErrorCode( + "unimported-reveal", + "Require explicit import from typing or typing_extensions for reveal_type", + "General", + default_enabled=False, +) +MUTABLE_OVERRIDE: Final[ErrorCode] = ErrorCode( + "mutable-override", + "Reject covariant overrides for mutable attributes", + "General", + default_enabled=False, +) + +# Syntax errors are often blocking. +SYNTAX: Final[ErrorCode] = ErrorCode("syntax", "Report syntax errors", "General") + +# This is an internal marker code for a whole-file ignore. It is not intended to +# be user-visible. +FILE: Final = ErrorCode("file", "Internal marker for a whole file being ignored", "General") +del error_codes[FILE.code] + +# This is a catch-all for remaining uncategorized errors. +MISC: Final[ErrorCode] = ErrorCode("misc", "Miscellaneous other checks", "General") + +OVERLOAD_CANNOT_MATCH: Final[ErrorCode] = ErrorCode( + "overload-cannot-match", + "Warn if an @overload signature can never be matched", + "General", + sub_code_of=MISC, +) + + +OVERLOAD_OVERLAP: Final[ErrorCode] = ErrorCode( + "overload-overlap", + "Warn if multiple @overload variants overlap in unsafe ways", + "General", + sub_code_of=MISC, +) + +PROPERTY_DECORATOR = ErrorCode( + "prop-decorator", + "Decorators on top of @property are not supported", + "General", + sub_code_of=MISC, +) + +NARROWED_TYPE_NOT_SUBTYPE: Final[ErrorCode] = ErrorCode( + "narrowed-type-not-subtype", + "Warn if a TypeIs function's narrowed type is not a subtype of the original type", + "General", +) + +DEPRECATED: Final = ErrorCode( + "deprecated", + "Warn when importing or using deprecated (overloaded) functions, methods or classes", + "General", + default_enabled=False, +) + +# This copy will not include any error codes defined later in the plugins. +mypy_error_codes = error_codes.copy() diff --git a/mypy/errors.py b/mypy/errors.py index 4d22b9284b1d..1b3f485d19c0 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -1,10 +1,50 @@ -import os -import os.path - -from typing import Undefined, Tuple, List, typevar, Sequence, Any, Function - +from __future__ import annotations -T = typevar('T') +import os.path +import sys +import traceback +from collections import defaultdict +from typing import Callable, Final, Iterable, NoReturn, Optional, TextIO, Tuple, TypeVar +from typing_extensions import Literal, TypeAlias as _TypeAlias + +from mypy import errorcodes as codes +from mypy.error_formatter import ErrorFormatter +from mypy.errorcodes import IMPORT, IMPORT_NOT_FOUND, IMPORT_UNTYPED, ErrorCode, mypy_error_codes +from mypy.message_registry import ErrorMessage +from mypy.options import Options +from mypy.scope import Scope +from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file +from mypy.version import __version__ as mypy_version + +T = TypeVar("T") + +# Show error codes for some note-level messages (these usually appear alone +# and not as a comment for a previous error-level message). +SHOW_NOTE_CODES: Final = {codes.ANNOTATION_UNCHECKED, codes.DEPRECATED} + +# Do not add notes with links to error code docs to errors with these codes. +# We can tweak this set as we get more experience about what is helpful and what is not. +HIDE_LINK_CODES: Final = { + # This is a generic error code, so it has no useful docs + codes.MISC, + # These are trivial and have some custom notes (e.g. for list being invariant) + codes.ASSIGNMENT, + codes.ARG_TYPE, + codes.RETURN_VALUE, + # Undefined name/attribute errors are self-explanatory + codes.ATTR_DEFINED, + codes.NAME_DEFINED, + # Overrides have a custom link to docs + codes.OVERRIDE, +} + +allowed_duplicates: Final = ["@overload", "Got:", "Expected:"] + +BASE_RTD_URL: Final = "https://mypy.rtfd.io/en/stable/_refs.html#code" + +# Keep track of the original error code when the error code of a message is changed. +# This is used to give notes about out-of-date "type: ignore" comments. +original_error_codes: Final = {codes.LITERAL_REQ: codes.MISC, codes.TYPE_ABSTRACT: codes.MISC} class ErrorInfo: @@ -12,35 +52,169 @@ class ErrorInfo: # Description of a sequence of imports that refer to the source file # related to this error. Each item is a (path, line number) tuple. - import_ctx = Undefined(List[Tuple[str, int]]) + import_ctx: list[tuple[str, int]] + + # The path to source file that was the source of this error. + file = "" - # The source file that was the source of this error. - file = '' + # The fully-qualified id of the source module for this error. + module: str | None = None # The name of the type in which this error is located at. - type = '' # Unqualified, may be None + type: str | None = "" # Unqualified, may be None # The name of the function or member in which this error is located at. - function_or_member = '' # Unqualified, may be None + function_or_member: str | None = "" # Unqualified, may be None # The line number related to this error within file. - line = 0 # -1 if unknown + line = 0 # -1 if unknown + + # The column number related to this error with file. + column = 0 # -1 if unknown + + # The end line number related to this error within file. + end_line = 0 # -1 if unknown + + # The end column number related to this error with file. + end_column = 0 # -1 if unknown + + # Either 'error' or 'note' + severity = "" # The error message. - message = '' + message = "" - # If True, we should halt build after the file that generated this error. - blocker = True + # The error code. + code: ErrorCode | None = None - def __init__(self, import_ctx: List[Tuple[str, int]], file: str, typ: str, - function_or_member: str, line: int, message: str, blocker: bool) -> None: + # If True, we should halt build after the file that generated this error. + blocker = False + + # Only report this particular messages once per program. + only_once = False + + # Do not remove duplicate copies of this message (ignored if only_once is True). + allow_dups = False + + # Actual origin of the error message as tuple (path, line number, end line number) + # If end line number is unknown, use line number. + origin: tuple[str, Iterable[int]] + + # Fine-grained incremental target where this was reported + target: str | None = None + + # If True, don't show this message in output, but still record the error (needed + # by mypy daemon) + hidden = False + + def __init__( + self, + import_ctx: list[tuple[str, int]], + *, + file: str, + module: str | None, + typ: str | None, + function_or_member: str | None, + line: int, + column: int, + end_line: int, + end_column: int, + severity: str, + message: str, + code: ErrorCode | None, + blocker: bool, + only_once: bool, + allow_dups: bool, + origin: tuple[str, Iterable[int]] | None = None, + target: str | None = None, + priority: int = 0, + ) -> None: self.import_ctx = import_ctx self.file = file + self.module = module self.type = typ self.function_or_member = function_or_member self.line = line + self.column = column + self.end_line = end_line + self.end_column = end_column + self.severity = severity self.message = message + self.code = code self.blocker = blocker + self.only_once = only_once + self.allow_dups = allow_dups + self.origin = origin or (file, [line]) + self.target = target + self.priority = priority + + +# Type used internally to represent errors: +# (path, line, column, end_line, end_column, severity, message, allow_dups, code) +ErrorTuple: _TypeAlias = Tuple[ + Optional[str], int, int, int, int, str, str, bool, Optional[ErrorCode] +] + + +class ErrorWatcher: + """Context manager that can be used to keep track of new errors recorded + around a given operation. + + Errors maintain a stack of such watchers. The handler is called starting + at the top of the stack, and is propagated down the stack unless filtered + out by one of the ErrorWatcher instances. + """ + + def __init__( + self, + errors: Errors, + *, + filter_errors: bool | Callable[[str, ErrorInfo], bool] = False, + save_filtered_errors: bool = False, + ) -> None: + self.errors = errors + self._has_new_errors = False + self._filter = filter_errors + self._filtered: list[ErrorInfo] | None = [] if save_filtered_errors else None + + def __enter__(self) -> ErrorWatcher: + self.errors._watchers.append(self) + return self + + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> Literal[False]: + last = self.errors._watchers.pop() + assert last == self + return False + + def on_error(self, file: str, info: ErrorInfo) -> bool: + """Handler called when a new error is recorded. + + The default implementation just sets the has_new_errors flag + + Return True to filter out the error, preventing it from being seen by other + ErrorWatcher further down the stack and from being recorded by Errors + """ + if info.code == codes.DEPRECATED: + return False + + self._has_new_errors = True + if isinstance(self._filter, bool): + should_filter = self._filter + elif callable(self._filter): + should_filter = self._filter(file, info) + else: + raise AssertionError(f"invalid error filter: {type(self._filter)}") + if should_filter and self._filtered is not None: + self._filtered.append(info) + + return should_filter + + def has_new_errors(self) -> bool: + return self._has_new_errors + + def filtered_errors(self) -> list[ErrorInfo]: + assert self._filtered is not None + return self._filtered class Errors: @@ -50,192 +224,875 @@ class Errors: current error context (nested imports). """ - # List of generated error messages. - error_info = Undefined(List[ErrorInfo]) + # Map from files to generated error messages. Is an OrderedDict so + # that it can be used to order messages based on the order the + # files were processed. + error_info_map: dict[str, list[ErrorInfo]] - # Current error context. - # Import context, as a list of (path, line) pairs. - import_ctx = Undefined(List[Tuple[str, int]]) + # optimization for legacy codebases with many files with errors + has_blockers: set[str] - # Path name prefix that is removed from all paths, if set. - ignore_prefix = None # type: str - - # Path to current file. - file = None # type: str + # Files that we have reported the errors for + flushed_files: set[str] - # Stack of short names of currents types (or None). - type_name = Undefined(List[str]) + # Current error context: nested import context/stack, as a list of (path, line) pairs. + import_ctx: list[tuple[str, int]] - # Stack of short names of current functions or members (or None). - function_or_member = Undefined(List[str]) + # Path name prefix that is removed from all paths, if set. + ignore_prefix: str | None = None - def __init__(self) -> None: - self.error_info = [] + # Path to current file. + file: str = "" + + # Ignore some errors on these lines of each file + # (path -> line -> error-codes) + ignored_lines: dict[str, dict[int, list[str]]] + + # Lines that were skipped during semantic analysis e.g. due to ALWAYS_FALSE, MYPY_FALSE, + # or platform/version checks. Those lines would not be type-checked. + skipped_lines: dict[str, set[int]] + + # Lines on which an error was actually ignored. + used_ignored_lines: dict[str, dict[int, list[str]]] + + # Files where all errors should be ignored. + ignored_files: set[str] + + # Collection of reported only_once messages. + only_once_messages: set[str] + + # Set to True to show "In function "foo":" messages. + show_error_context: bool = False + + # Set to True to show column numbers in error messages. + show_column_numbers: bool = False + + # Set to True to show end line and end column in error messages. + # Ths implies `show_column_numbers`. + show_error_end: bool = False + + # Set to True to show absolute file paths in error messages. + show_absolute_path: bool = False + + # State for keeping track of the current fine-grained incremental mode target. + # (See mypy.server.update for more about targets.) + # Current module id. + target_module: str | None = None + scope: Scope | None = None + + # Have we seen an import-related error so far? If yes, we filter out other messages + # in some cases to avoid reporting huge numbers of errors. + seen_import_error = False + + _watchers: list[ErrorWatcher] = [] + + def __init__( + self, + options: Options, + *, + read_source: Callable[[str], list[str] | None] | None = None, + hide_error_codes: bool | None = None, + ) -> None: + self.options = options + self.hide_error_codes = ( + hide_error_codes if hide_error_codes is not None else options.hide_error_codes + ) + # We use fscache to read source code when showing snippets. + self.read_source = read_source + self.initialize() + + def initialize(self) -> None: + self.error_info_map = {} + self.flushed_files = set() self.import_ctx = [] - self.type_name = [None] self.function_or_member = [None] - - def copy(self) -> 'Errors': - new = Errors() - new.file = self.file - new.import_ctx = self.import_ctx[:] - new.type_name = self.type_name[:] - new.function_or_member = self.function_or_member[:] - return new + self.ignored_lines = {} + self.skipped_lines = {} + self.used_ignored_lines = defaultdict(lambda: defaultdict(list)) + self.ignored_files = set() + self.only_once_messages = set() + self.has_blockers = set() + self.scope = None + self.target_module = None + self.seen_import_error = False + + def reset(self) -> None: + self.initialize() def set_ignore_prefix(self, prefix: str) -> None: """Set path prefix that will be removed from all paths.""" prefix = os.path.normpath(prefix) # Add separator to the end, if not given. - if os.path.basename(prefix) != '': + if os.path.basename(prefix) != "": prefix += os.sep self.ignore_prefix = prefix - def set_file(self, file: str) -> None: - """Set the path of the current file.""" - file = os.path.normpath(file) - self.file = remove_path_prefix(file, self.ignore_prefix) - - def push_function(self, name: str) -> None: - """Set the current function or member short name (it can be None).""" - self.function_or_member.append(name) + def simplify_path(self, file: str) -> str: + if self.options.show_absolute_path: + return os.path.abspath(file) + else: + file = os.path.normpath(file) + return remove_path_prefix(file, self.ignore_prefix) + + def set_file( + self, file: str, module: str | None, options: Options, scope: Scope | None = None + ) -> None: + """Set the path and module id of the current file.""" + # The path will be simplified later, in render_messages. That way + # * 'file' is always a key that uniquely identifies a source file + # that mypy read (simplified paths might not be unique); and + # * we only have to simplify in one place, while still supporting + # reporting errors for files other than the one currently being + # processed. + self.file = file + self.target_module = module + self.scope = scope + self.options = options - def pop_function(self) -> None: - self.function_or_member.pop() + def set_file_ignored_lines( + self, file: str, ignored_lines: dict[int, list[str]], ignore_all: bool = False + ) -> None: + self.ignored_lines[file] = ignored_lines + if ignore_all: + self.ignored_files.add(file) - def push_type(self, name: str) -> None: - """Set the short name of the current type (it can be None).""" - self.type_name.append(name) + def set_skipped_lines(self, file: str, skipped_lines: set[int]) -> None: + self.skipped_lines[file] = skipped_lines - def pop_type(self) -> None: - self.type_name.pop() + def current_target(self) -> str | None: + """Retrieves the current target from the associated scope. - def push_import_context(self, path: str, line: int) -> None: - """Add a (file, line) tuple to the import context.""" - self.import_ctx.append((os.path.normpath(path), line)) + If there is no associated scope, use the target module.""" + if self.scope is not None: + return self.scope.current_target() + return self.target_module - def pop_import_context(self) -> None: - """Remove the topmost item from the import context.""" - self.import_ctx.pop() + def current_module(self) -> str | None: + return self.target_module - def import_context(self) -> List[Tuple[str, int]]: + def import_context(self) -> list[tuple[str, int]]: """Return a copy of the import context.""" - return self.import_ctx[:] + return self.import_ctx.copy() - def set_import_context(self, ctx: List[Tuple[str, int]]) -> None: + def set_import_context(self, ctx: list[tuple[str, int]]) -> None: """Replace the entire import context with a new value.""" - self.import_ctx = ctx[:] - - def report(self, line: int, message: str, blocker: bool = True) -> None: - """Report message at the given line using the current error context.""" - type = self.type_name[-1] - if len(self.function_or_member) > 2: - type = None # Omit type context if nested function - info = ErrorInfo(self.import_context(), self.file, type, - self.function_or_member[-1], line, message, - blocker) - self.error_info.append(info) + self.import_ctx = ctx.copy() + + def report( + self, + line: int, + column: int | None, + message: str, + code: ErrorCode | None = None, + *, + blocker: bool = False, + severity: str = "error", + file: str | None = None, + only_once: bool = False, + allow_dups: bool = False, + origin_span: Iterable[int] | None = None, + offset: int = 0, + end_line: int | None = None, + end_column: int | None = None, + ) -> None: + """Report message at the given line using the current error context. + + Args: + line: line number of error + column: column number of error + message: message to report + code: error code (defaults to 'misc'; not shown for notes) + blocker: if True, don't continue analysis after this error + severity: 'error' or 'note' + file: if non-None, override current file as context + only_once: if True, only report this exact message once per build + allow_dups: if True, allow duplicate copies of this message (ignored if only_once) + origin_span: if non-None, override current context as origin + (type: ignores have effect here) + end_line: if non-None, override current context as end + """ + if self.scope: + type = self.scope.current_type_name() + if self.scope.ignored > 0: + type = None # Omit type context if nested function + function = self.scope.current_function_name() + else: + type = None + function = None + + if column is None: + column = -1 + if end_column is None: + if column == -1: + end_column = -1 + else: + end_column = column + 1 + + if file is None: + file = self.file + if offset: + message = " " * offset + message + + if origin_span is None: + origin_span = [line] + + if end_line is None: + end_line = line + + code = code or (codes.MISC if not blocker else None) + + info = ErrorInfo( + import_ctx=self.import_context(), + file=file, + module=self.current_module(), + typ=type, + function_or_member=function, + line=line, + column=column, + end_line=end_line, + end_column=end_column, + severity=severity, + message=message, + code=code, + blocker=blocker, + only_once=only_once, + allow_dups=allow_dups, + origin=(self.file, origin_span), + target=self.current_target(), + ) + self.add_error_info(info) + + def _add_error_info(self, file: str, info: ErrorInfo) -> None: + assert file not in self.flushed_files + # process the stack of ErrorWatchers before modifying any internal state + # in case we need to filter out the error entirely + if self._filter_error(file, info): + return + if file not in self.error_info_map: + self.error_info_map[file] = [] + self.error_info_map[file].append(info) + if info.blocker: + self.has_blockers.add(file) + if info.code in (IMPORT, IMPORT_UNTYPED, IMPORT_NOT_FOUND): + self.seen_import_error = True + + def _filter_error(self, file: str, info: ErrorInfo) -> bool: + """ + process ErrorWatcher stack from top to bottom, + stopping early if error needs to be filtered out + """ + i = len(self._watchers) + while i > 0: + i -= 1 + w = self._watchers[i] + if w.on_error(file, info): + return True + return False + + def add_error_info(self, info: ErrorInfo) -> None: + file, lines = info.origin + # process the stack of ErrorWatchers before modifying any internal state + # in case we need to filter out the error entirely + # NB: we need to do this both here and in _add_error_info, otherwise we + # might incorrectly update the sets of ignored or only_once messages + if self._filter_error(file, info): + return + if not info.blocker: # Blockers cannot be ignored + if file in self.ignored_lines: + # Check each line in this context for "type: ignore" comments. + # line == end_line for most nodes, so we only loop once. + for scope_line in lines: + if self.is_ignored_error(scope_line, info, self.ignored_lines[file]): + # Annotation requests us to ignore all errors on this line. + self.used_ignored_lines[file][scope_line].append( + (info.code or codes.MISC).code + ) + return + if file in self.ignored_files: + return + if info.only_once: + if info.message in self.only_once_messages: + return + self.only_once_messages.add(info.message) + if ( + self.seen_import_error + and info.code not in (IMPORT, IMPORT_UNTYPED, IMPORT_NOT_FOUND) + and self.has_many_errors() + ): + # Missing stubs can easily cause thousands of errors about + # Any types, especially when upgrading to mypy 0.900, + # which no longer bundles third-party library stubs. Avoid + # showing too many errors to make it easier to see + # import-related errors. + info.hidden = True + self.report_hidden_errors(info) + self._add_error_info(file, info) + ignored_codes = self.ignored_lines.get(file, {}).get(info.line, []) + if ignored_codes and info.code: + # Something is ignored on the line, but not this error, so maybe the error + # code is incorrect. + msg = f'Error code "{info.code.code}" not covered by "type: ignore" comment' + if info.code in original_error_codes: + # If there seems to be a "type: ignore" with a stale error + # code, report a more specific note. + old_code = original_error_codes[info.code].code + if old_code in ignored_codes: + msg = ( + f'Error code changed to {info.code.code}; "type: ignore" comment ' + + "may be out of date" + ) + note = ErrorInfo( + import_ctx=info.import_ctx, + file=info.file, + module=info.module, + typ=info.type, + function_or_member=info.function_or_member, + line=info.line, + column=info.column, + end_line=info.end_line, + end_column=info.end_column, + severity="note", + message=msg, + code=None, + blocker=False, + only_once=False, + allow_dups=False, + ) + self._add_error_info(file, note) + if ( + self.options.show_error_code_links + and not self.options.hide_error_codes + and info.code is not None + and info.code not in HIDE_LINK_CODES + and info.code.code in mypy_error_codes + ): + message = f"See {BASE_RTD_URL}-{info.code.code} for more info" + if message in self.only_once_messages: + return + self.only_once_messages.add(message) + info = ErrorInfo( + import_ctx=info.import_ctx, + file=info.file, + module=info.module, + typ=info.type, + function_or_member=info.function_or_member, + line=info.line, + column=info.column, + end_line=info.end_line, + end_column=info.end_column, + severity="note", + message=message, + code=info.code, + blocker=False, + only_once=True, + allow_dups=False, + priority=20, + ) + self._add_error_info(file, info) + + def has_many_errors(self) -> bool: + if self.options.many_errors_threshold < 0: + return False + if len(self.error_info_map) >= self.options.many_errors_threshold: + return True + if ( + sum(len(errors) for errors in self.error_info_map.values()) + >= self.options.many_errors_threshold + ): + return True + return False + + def report_hidden_errors(self, info: ErrorInfo) -> None: + message = ( + "(Skipping most remaining errors due to unresolved imports or missing stubs; " + + "fix these first)" + ) + if message in self.only_once_messages: + return + self.only_once_messages.add(message) + new_info = ErrorInfo( + import_ctx=info.import_ctx, + file=info.file, + module=info.module, + typ=None, + function_or_member=None, + line=info.line, + column=info.column, + end_line=info.end_line, + end_column=info.end_column, + severity="note", + message=message, + code=None, + blocker=False, + only_once=True, + allow_dups=False, + origin=info.origin, + target=info.target, + ) + self._add_error_info(info.origin[0], new_info) + + def is_ignored_error(self, line: int, info: ErrorInfo, ignores: dict[int, list[str]]) -> bool: + if info.blocker: + # Blocking errors can never be ignored + return False + if info.code and not self.is_error_code_enabled(info.code): + return True + if line not in ignores: + return False + if not ignores[line]: + # Empty list means that we ignore all errors + return True + if info.code and self.is_error_code_enabled(info.code): + return ( + info.code.code in ignores[line] + or info.code.sub_code_of is not None + and info.code.sub_code_of.code in ignores[line] + ) + return False + + def is_error_code_enabled(self, error_code: ErrorCode) -> bool: + if self.options: + current_mod_disabled = self.options.disabled_error_codes + current_mod_enabled = self.options.enabled_error_codes + else: + current_mod_disabled = set() + current_mod_enabled = set() + + if error_code in current_mod_disabled: + return False + elif error_code in current_mod_enabled: + return True + elif error_code.sub_code_of is not None and error_code.sub_code_of in current_mod_disabled: + return False + else: + return error_code.default_enabled + + def clear_errors_in_targets(self, path: str, targets: set[str]) -> None: + """Remove errors in specific fine-grained targets within a file.""" + if path in self.error_info_map: + new_errors = [] + has_blocker = False + for info in self.error_info_map[path]: + if info.target not in targets: + new_errors.append(info) + has_blocker |= info.blocker + elif info.only_once: + self.only_once_messages.remove(info.message) + self.error_info_map[path] = new_errors + if not has_blocker and path in self.has_blockers: + self.has_blockers.remove(path) + + def generate_unused_ignore_errors(self, file: str) -> None: + if ( + is_typeshed_file(self.options.abs_custom_typeshed_dir if self.options else None, file) + or file in self.ignored_files + ): + return + ignored_lines = self.ignored_lines[file] + used_ignored_lines = self.used_ignored_lines[file] + for line, ignored_codes in ignored_lines.items(): + if line in self.skipped_lines[file]: + continue + if codes.UNUSED_IGNORE.code in ignored_codes: + continue + used_ignored_codes = used_ignored_lines[line] + unused_ignored_codes = set(ignored_codes) - set(used_ignored_codes) + # `ignore` is used + if not ignored_codes and used_ignored_codes: + continue + # All codes appearing in `ignore[...]` are used + if ignored_codes and not unused_ignored_codes: + continue + # Display detail only when `ignore[...]` specifies more than one error code + unused_codes_message = "" + if len(ignored_codes) > 1 and unused_ignored_codes: + unused_codes_message = f"[{', '.join(sorted(unused_ignored_codes))}]" + message = f'Unused "type: ignore{unused_codes_message}" comment' + for unused in unused_ignored_codes: + narrower = set(used_ignored_codes) & codes.sub_code_map[unused] + if narrower: + message += f", use narrower [{', '.join(narrower)}] instead of [{unused}] code" + # Don't use report since add_error_info will ignore the error! + info = ErrorInfo( + import_ctx=self.import_context(), + file=file, + module=self.current_module(), + typ=None, + function_or_member=None, + line=line, + column=-1, + end_line=line, + end_column=-1, + severity="error", + message=message, + code=codes.UNUSED_IGNORE, + blocker=False, + only_once=False, + allow_dups=False, + ) + self._add_error_info(file, info) + + def generate_ignore_without_code_errors( + self, file: str, is_warning_unused_ignores: bool + ) -> None: + if ( + is_typeshed_file(self.options.abs_custom_typeshed_dir if self.options else None, file) + or file in self.ignored_files + ): + return + + used_ignored_lines = self.used_ignored_lines[file] + + # If the whole file is ignored, ignore it. + if used_ignored_lines: + _, used_codes = min(used_ignored_lines.items()) + if codes.FILE.code in used_codes: + return + + for line, ignored_codes in self.ignored_lines[file].items(): + if ignored_codes: + continue + + # If the ignore is itself unused and that would be warned about, let + # that error stand alone + if is_warning_unused_ignores and not used_ignored_lines[line]: + continue + + codes_hint = "" + ignored_codes = sorted(set(used_ignored_lines[line])) + if ignored_codes: + codes_hint = f' (consider "type: ignore[{", ".join(ignored_codes)}]" instead)' + + message = f'"type: ignore" comment without error code{codes_hint}' + # Don't use report since add_error_info will ignore the error! + info = ErrorInfo( + import_ctx=self.import_context(), + file=file, + module=self.current_module(), + typ=None, + function_or_member=None, + line=line, + column=-1, + end_line=line, + end_column=-1, + severity="error", + message=message, + code=codes.IGNORE_WITHOUT_CODE, + blocker=False, + only_once=False, + allow_dups=False, + ) + self._add_error_info(file, info) def num_messages(self) -> int: """Return the number of generated messages.""" - return len(self.error_info) + return sum(len(x) for x in self.error_info_map.values()) def is_errors(self) -> bool: - """Are there any generated errors?""" - return bool(self.error_info) + """Are there any generated messages?""" + return bool(self.error_info_map) def is_blockers(self) -> bool: """Are the any errors that are blockers?""" - return any(err for err in self.error_info if err.blocker) + return bool(self.has_blockers) - def raise_error(self) -> None: + def blocker_module(self) -> str | None: + """Return the module with a blocking error, or None if not possible.""" + for path in self.has_blockers: + for err in self.error_info_map[path]: + if err.blocker: + return err.module + return None + + def is_errors_for_file(self, file: str) -> bool: + """Are there any errors for the given file?""" + return file in self.error_info_map and file not in self.ignored_files + + def prefer_simple_messages(self) -> bool: + """Should we generate simple/fast error messages? + + Return True if errors are not shown to user, i.e. errors are ignored + or they are collected for internal use only. + + If True, we should prefer to generate a simple message quickly. + All normal errors should still be reported. + """ + if self.file in self.ignored_files: + # Errors ignored, so no point generating fancy messages + return True + for _watcher in self._watchers: + if _watcher._filter is True and _watcher._filtered is None: + # Errors are filtered + return True + return False + + def raise_error(self, use_stdout: bool = True) -> NoReturn: """Raise a CompileError with the generated messages. Render the messages suitable for displaying. """ - raise CompileError(self.messages()) - - def messages(self) -> List[str]: + # self.new_messages() will format all messages that haven't already + # been returned from a file_messages() call. + raise CompileError( + self.new_messages(), use_stdout=use_stdout, module_with_blocker=self.blocker_module() + ) + + def format_messages( + self, error_tuples: list[ErrorTuple], source_lines: list[str] | None + ) -> list[str]: """Return a string list that represents the error messages. - Use a form suitable for displaying to the user. + Use a form suitable for displaying to the user. If self.pretty + is True also append a relevant trimmed source code line (only for + severity 'error'). """ - a = [] # type: List[str] - errors = self.render_messages(self.sort_messages(self.error_info)) - errors = self.remove_duplicates(errors) - for file, line, message in errors: - s = '' + a: list[str] = [] + for ( + file, + line, + column, + end_line, + end_column, + severity, + message, + allow_dups, + code, + ) in error_tuples: + s = "" if file is not None: - if line is not None and line >= 0: - s = '{}, line {}: {}'.format(file, line, message) + if self.options.show_column_numbers and line >= 0 and column >= 0: + srcloc = f"{file}:{line}:{1 + column}" + if self.options.show_error_end and end_line >= 0 and end_column >= 0: + srcloc += f":{end_line}:{end_column}" + elif line >= 0: + srcloc = f"{file}:{line}" else: - s = '{}: {}'.format(file, message) + srcloc = file + s = f"{srcloc}: {severity}: {message}" else: s = message + if ( + not self.hide_error_codes + and code + and (severity != "note" or code in SHOW_NOTE_CODES) + ): + # If note has an error code, it is related to a previous error. Avoid + # displaying duplicate error codes. + s = f"{s} [{code.code}]" a.append(s) + if self.options.pretty: + # Add source code fragment and a location marker. + if severity == "error" and source_lines and line > 0: + source_line = source_lines[line - 1] + source_line_expanded = source_line.expandtabs() + if column < 0: + # Something went wrong, take first non-empty column. + column = len(source_line) - len(source_line.lstrip()) + + # Shifts column after tab expansion + column = len(source_line[:column].expandtabs()) + end_column = len(source_line[:end_column].expandtabs()) + + # Note, currently coloring uses the offset to detect source snippets, + # so these offsets should not be arbitrary. + a.append(" " * DEFAULT_SOURCE_OFFSET + source_line_expanded) + marker = "^" + if end_line == line and end_column > column: + marker = f'^{"~" * (end_column - column - 1)}' + a.append(" " * (DEFAULT_SOURCE_OFFSET + column) + marker) return a - def render_messages(self, errors: List[ErrorInfo]) -> List[Tuple[str, int, - str]]: - """Translate the messages into a sequence of tuples. + def file_messages(self, path: str, formatter: ErrorFormatter | None = None) -> list[str]: + """Return a string list of new error messages from a given file. - Each tuple is of form (path, line, message. The rendered - sequence includes information about error contexts. The path - item may be None. If the line item is negative, the line - number is not defined for the tuple. + Use a form suitable for displaying to the user. """ - result = [] # type: List[Tuple[str, int, str]] # (path, line, message) + if path not in self.error_info_map: + return [] + + error_info = self.error_info_map[path] + error_info = [info for info in error_info if not info.hidden] + error_tuples = self.render_messages(self.sort_messages(error_info)) + error_tuples = self.remove_duplicates(error_tuples) + + if formatter is not None: + errors = create_errors(error_tuples) + return [formatter.report_error(err) for err in errors] + + self.flushed_files.add(path) + source_lines = None + if self.options.pretty and self.read_source: + # Find shadow file mapping and read source lines if a shadow file exists for the given path. + # If shadow file mapping is not found, read source lines + mapped_path = self.find_shadow_file_mapping(path) + if mapped_path: + source_lines = self.read_source(mapped_path) + else: + source_lines = self.read_source(path) + return self.format_messages(error_tuples, source_lines) + + def find_shadow_file_mapping(self, path: str) -> str | None: + """Return the shadow file path for a given source file path or None.""" + if self.options.shadow_file is None: + return None - prev_import_context = [] # type: List[Tuple[str, int]] - prev_function_or_member = None # type: str - prev_type = None # type: str + for i in self.options.shadow_file: + if i[0] == path: + return i[1] + return None + + def new_messages(self) -> list[str]: + """Return a string list of new error messages. + + Use a form suitable for displaying to the user. + Errors from different files are ordered based on the order in which + they first generated an error. + """ + msgs = [] + for path in self.error_info_map.keys(): + if path not in self.flushed_files: + msgs.extend(self.file_messages(path)) + return msgs + + def targets(self) -> set[str]: + """Return a set of all targets that contain errors.""" + # TODO: Make sure that either target is always defined or that not being defined + # is okay for fine-grained incremental checking. + return { + info.target for errs in self.error_info_map.values() for info in errs if info.target + } + + def render_messages(self, errors: list[ErrorInfo]) -> list[ErrorTuple]: + """Translate the messages into a sequence of tuples. + + Each tuple is of form (path, line, col, severity, message, allow_dups, code). + The rendered sequence includes information about error contexts. + The path item may be None. If the line item is negative, the + line number is not defined for the tuple. + """ + result: list[ErrorTuple] = [] + prev_import_context: list[tuple[str, int]] = [] + prev_function_or_member: str | None = None + prev_type: str | None = None for e in errors: # Report module import context, if different from previous message. - if e.import_ctx != prev_import_context: + if not self.options.show_error_context: + pass + elif e.import_ctx != prev_import_context: last = len(e.import_ctx) - 1 i = last while i >= 0: path, line = e.import_ctx[i] - fmt = 'In module imported in {}, line {}' + fmt = "{}:{}: note: In module imported here" if i < last: - fmt = ' in {}, line {}' + fmt = "{}:{}: note: ... from here" if i > 0: - fmt += ',' + fmt += "," else: - fmt += ':' + fmt += ":" # Remove prefix to ignore from path (if present) to # simplify path. path = remove_path_prefix(path, self.ignore_prefix) - result.append((None, -1, fmt.format(path, line))) + result.append( + (None, -1, -1, -1, -1, "note", fmt.format(path, line), e.allow_dups, None) + ) i -= 1 + file = self.simplify_path(e.file) + # Report context within a source file. - if (e.function_or_member != prev_function_or_member or - e.type != prev_type): + if not self.options.show_error_context: + pass + elif e.function_or_member != prev_function_or_member or e.type != prev_type: if e.function_or_member is None: if e.type is None: - result.append((e.file, -1, 'At top level:')) + result.append( + (file, -1, -1, -1, -1, "note", "At top level:", e.allow_dups, None) + ) else: - result.append((e.file, -1, 'In class "{}":'.format( - e.type))) + result.append( + ( + file, + -1, + -1, + -1, + -1, + "note", + f'In class "{e.type}":', + e.allow_dups, + None, + ) + ) else: if e.type is None: - result.append((e.file, -1, - 'In function "{}":'.format( - e.function_or_member))) + result.append( + ( + file, + -1, + -1, + -1, + -1, + "note", + f'In function "{e.function_or_member}":', + e.allow_dups, + None, + ) + ) else: - result.append((e.file, -1, - 'In member "{}" of class "{}":'.format( - e.function_or_member, e.type))) + result.append( + ( + file, + -1, + -1, + -1, + -1, + "note", + 'In member "{}" of class "{}":'.format( + e.function_or_member, e.type + ), + e.allow_dups, + None, + ) + ) elif e.type != prev_type: if e.type is None: - result.append((e.file, -1, 'At top level:')) + result.append( + (file, -1, -1, -1, -1, "note", "At top level:", e.allow_dups, None) + ) else: - result.append((e.file, -1, - 'In class "{}":'.format(e.type))) - - result.append((e.file, e.line, e.message)) + result.append( + (file, -1, -1, -1, -1, "note", f'In class "{e.type}":', e.allow_dups, None) + ) + + if isinstance(e.message, ErrorMessage): + result.append( + ( + file, + e.line, + e.column, + e.end_line, + e.end_column, + e.severity, + e.message.value, + e.allow_dups, + e.code, + ) + ) + else: + result.append( + ( + file, + e.line, + e.column, + e.end_line, + e.end_column, + e.severity, + e.message, + e.allow_dups, + e.code, + ) + ) prev_import_context = e.import_ctx prev_function_or_member = e.function_or_member @@ -243,43 +1100,89 @@ def render_messages(self, errors: List[ErrorInfo]) -> List[Tuple[str, int, return result - def sort_messages(self, errors: List[ErrorInfo]) -> List[ErrorInfo]: + def sort_messages(self, errors: list[ErrorInfo]) -> list[ErrorInfo]: """Sort an array of error messages locally by line number. - I.e., sort a run of consecutive messages with the same file + I.e., sort a run of consecutive messages with the same context by line number, but otherwise retain the general ordering of the messages. """ - result = [] # type: List[ErrorInfo] + result: list[ErrorInfo] = [] i = 0 while i < len(errors): i0 = i # Find neighbouring errors with the same context and file. - while (i + 1 < len(errors) and - errors[i + 1].import_ctx == errors[i].import_ctx and - errors[i + 1].file == errors[i].file): + while ( + i + 1 < len(errors) + and errors[i + 1].import_ctx == errors[i].import_ctx + and errors[i + 1].file == errors[i].file + ): i += 1 i += 1 - # Sort the errors specific to a file according to line number. - a = stable_sort(errors[i0:i], lambda x: x.line) + # Sort the errors specific to a file according to line number and column. + a = sorted(errors[i0:i], key=lambda x: (x.line, x.column)) + a = self.sort_within_context(a) result.extend(a) return result - def remove_duplicates(self, errors: List[Tuple[str, int, str]] - ) -> List[Tuple[str, int, str]]: + def sort_within_context(self, errors: list[ErrorInfo]) -> list[ErrorInfo]: + """For the same location decide which messages to show first/last. + + Currently, we only compare within the same error code, to decide the + order of various additional notes. + """ + result = [] + i = 0 + while i < len(errors): + i0 = i + # Find neighbouring errors with the same position and error code. + while ( + i + 1 < len(errors) + and errors[i + 1].line == errors[i].line + and errors[i + 1].column == errors[i].column + and errors[i + 1].end_line == errors[i].end_line + and errors[i + 1].end_column == errors[i].end_column + and errors[i + 1].code == errors[i].code + ): + i += 1 + i += 1 + + # Sort the messages specific to a given error by priority. + a = sorted(errors[i0:i], key=lambda x: x.priority) + result.extend(a) + return result + + def remove_duplicates(self, errors: list[ErrorTuple]) -> list[ErrorTuple]: """Remove duplicates from a sorted error list.""" - res = [] # type: List[Tuple[str, int, str]] + res: list[ErrorTuple] = [] i = 0 while i < len(errors): dup = False + # Use slightly special formatting for member conflicts reporting. + conflicts_notes = False j = i - 1 - while (j >= 0 and errors[j][0] == errors[i][0] and - errors[j][1] == errors[i][1]): - if errors[j] == errors[i]: - dup = True - break - j -= 1 + # Find duplicates, unless duplicates are allowed. + if not errors[i][7]: + while j >= 0 and errors[j][0] == errors[i][0]: + if errors[j][6].strip() == "Got:": + conflicts_notes = True + j -= 1 + j = i - 1 + while j >= 0 and errors[j][0] == errors[i][0] and errors[j][1] == errors[i][1]: + if ( + errors[j][5] == errors[i][5] + and + # Allow duplicate notes in overload conflicts reporting. + not ( + (errors[i][5] == "note" and errors[i][6].strip() in allowed_duplicates) + or (errors[i][6].strip().startswith("def ") and conflicts_notes) + ) + and errors[j][6] == errors[i][6] + ): # ignore column + dup = True + break + j -= 1 if not dup: res.append(errors[i]) i += 1 @@ -291,37 +1194,167 @@ class CompileError(Exception): It can be a parse, semantic analysis, type check or other compilation-related error. - """ - messages = Undefined(List[str]) - - def __init__(self, messages: List[str]) -> None: - super().__init__() - self.messages = messages + CompileErrors raised from an errors object carry all of the + messages that have not been reported out by error streaming. + This is patched up by build.build to contain either all error + messages (if errors were streamed) or none (if they were not). + """ -def stable_sort(a: Sequence[T], key: Function[[T], Any]) -> List[T]: - """Perform a stable sort of a sequence. + messages: list[str] + use_stdout = False + # Can be set in case there was a module with a blocking error + module_with_blocker: str | None = None - If the original sequence has a[n] == a[n+m] (when comparing using - the comparison function f), in the sorted sequence item a[n] will - be at an earlier index than a[n + m]. - """ - # TODO use sorted with key (need support for keyword arguments) - l = List[Tuple[Any, int, T]]() - for i, x in enumerate(a): - l.append((key(x), i, x)) - result = List[T]() - for k, j, y in sorted(l): - result.append(y) - return result + def __init__( + self, messages: list[str], use_stdout: bool = False, module_with_blocker: str | None = None + ) -> None: + super().__init__("\n".join(messages)) + self.messages = messages + self.use_stdout = use_stdout + self.module_with_blocker = module_with_blocker -def remove_path_prefix(path: str, prefix: str) -> str: +def remove_path_prefix(path: str, prefix: str | None) -> str: """If path starts with prefix, return copy of path with the prefix removed. Otherwise, return path. If path is None, return None. """ if prefix is not None and path.startswith(prefix): - return path[len(prefix):] + return path[len(prefix) :] else: return path + + +def report_internal_error( + err: Exception, + file: str | None, + line: int, + errors: Errors, + options: Options, + stdout: TextIO | None = None, + stderr: TextIO | None = None, +) -> NoReturn: + """Report internal error and exit. + + This optionally starts pdb or shows a traceback. + """ + stdout = stdout or sys.stdout + stderr = stderr or sys.stderr + # Dump out errors so far, they often provide a clue. + # But catch unexpected errors rendering them. + try: + for msg in errors.new_messages(): + print(msg) + except Exception as e: + print("Failed to dump errors:", repr(e), file=stderr) + + # Compute file:line prefix for official-looking error messages. + if file: + if line: + prefix = f"{file}:{line}: " + else: + prefix = f"{file}: " + else: + prefix = "" + + # Print "INTERNAL ERROR" message. + print( + f"{prefix}error: INTERNAL ERROR --", + "Please try using mypy master on GitHub:\n" + "https://mypy.readthedocs.io/en/stable/common_issues.html" + "#using-a-development-mypy-build", + file=stderr, + ) + if options.show_traceback: + print("Please report a bug at https://github.com/python/mypy/issues", file=stderr) + else: + print( + "If this issue continues with mypy master, " + "please report a bug at https://github.com/python/mypy/issues", + file=stderr, + ) + print(f"version: {mypy_version}", file=stderr) + + # If requested, drop into pdb. This overrides show_tb. + if options.pdb: + print("Dropping into pdb", file=stderr) + import pdb + + pdb.post_mortem(sys.exc_info()[2]) + + # If requested, print traceback, else print note explaining how to get one. + if options.raise_exceptions: + raise err + if not options.show_traceback: + if not options.pdb: + print( + "{}: note: please use --show-traceback to print a traceback " + "when reporting a bug".format(prefix), + file=stderr, + ) + else: + tb = traceback.extract_stack()[:-2] + tb2 = traceback.extract_tb(sys.exc_info()[2]) + print("Traceback (most recent call last):") + for s in traceback.format_list(tb + tb2): + print(s.rstrip("\n")) + print(f"{type(err).__name__}: {err}", file=stdout) + print(f"{prefix}: note: use --pdb to drop into pdb", file=stderr) + + # Exit. The caller has nothing more to say. + # We use exit code 2 to signal that this is no ordinary error. + raise SystemExit(2) + + +class MypyError: + def __init__( + self, + file_path: str, + line: int, + column: int, + message: str, + errorcode: ErrorCode | None, + severity: Literal["error", "note"], + ) -> None: + self.file_path = file_path + self.line = line + self.column = column + self.message = message + self.errorcode = errorcode + self.severity = severity + self.hints: list[str] = [] + + +# (file_path, line, column) +_ErrorLocation = Tuple[str, int, int] + + +def create_errors(error_tuples: list[ErrorTuple]) -> list[MypyError]: + errors: list[MypyError] = [] + latest_error_at_location: dict[_ErrorLocation, MypyError] = {} + + for error_tuple in error_tuples: + file_path, line, column, _, _, severity, message, _, errorcode = error_tuple + if file_path is None: + continue + + assert severity in ("error", "note") + if severity == "note": + error_location = (file_path, line, column) + error = latest_error_at_location.get(error_location) + if error is None: + # This is purely a note, with no error correlated to it + error = MypyError(file_path, line, column, message, errorcode, severity="note") + errors.append(error) + continue + + error.hints.append(message) + + else: + error = MypyError(file_path, line, column, message, errorcode, severity="error") + errors.append(error) + error_location = (file_path, line, column) + latest_error_at_location[error_location] = error + + return errors diff --git a/mypy/evalexpr.py b/mypy/evalexpr.py new file mode 100644 index 000000000000..e39c5840d47a --- /dev/null +++ b/mypy/evalexpr.py @@ -0,0 +1,205 @@ +""" + +Evaluate an expression. + +Used by stubtest; in a separate file because things break if we don't +put it in a mypyc-compiled file. + +""" + +import ast +from typing import Final + +import mypy.nodes +from mypy.visitor import ExpressionVisitor + +UNKNOWN = object() + + +class _NodeEvaluator(ExpressionVisitor[object]): + def visit_int_expr(self, o: mypy.nodes.IntExpr) -> int: + return o.value + + def visit_str_expr(self, o: mypy.nodes.StrExpr) -> str: + return o.value + + def visit_bytes_expr(self, o: mypy.nodes.BytesExpr) -> object: + # The value of a BytesExpr is a string created from the repr() + # of the bytes object. Get the original bytes back. + try: + return ast.literal_eval(f"b'{o.value}'") + except SyntaxError: + return ast.literal_eval(f'b"{o.value}"') + + def visit_float_expr(self, o: mypy.nodes.FloatExpr) -> float: + return o.value + + def visit_complex_expr(self, o: mypy.nodes.ComplexExpr) -> object: + return o.value + + def visit_ellipsis(self, o: mypy.nodes.EllipsisExpr) -> object: + return Ellipsis + + def visit_star_expr(self, o: mypy.nodes.StarExpr) -> object: + return UNKNOWN + + def visit_name_expr(self, o: mypy.nodes.NameExpr) -> object: + if o.name == "True": + return True + elif o.name == "False": + return False + elif o.name == "None": + return None + # TODO: Handle more names by figuring out a way to hook into the + # symbol table. + return UNKNOWN + + def visit_member_expr(self, o: mypy.nodes.MemberExpr) -> object: + return UNKNOWN + + def visit_yield_from_expr(self, o: mypy.nodes.YieldFromExpr) -> object: + return UNKNOWN + + def visit_yield_expr(self, o: mypy.nodes.YieldExpr) -> object: + return UNKNOWN + + def visit_call_expr(self, o: mypy.nodes.CallExpr) -> object: + return UNKNOWN + + def visit_op_expr(self, o: mypy.nodes.OpExpr) -> object: + return UNKNOWN + + def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> object: + return UNKNOWN + + def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> object: + return o.expr.accept(self) + + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> object: + return o.expr.accept(self) + + def visit_reveal_expr(self, o: mypy.nodes.RevealExpr) -> object: + return UNKNOWN + + def visit_super_expr(self, o: mypy.nodes.SuperExpr) -> object: + return UNKNOWN + + def visit_unary_expr(self, o: mypy.nodes.UnaryExpr) -> object: + operand = o.expr.accept(self) + if operand is UNKNOWN: + return UNKNOWN + if o.op == "-": + if isinstance(operand, (int, float, complex)): + return -operand + elif o.op == "+": + if isinstance(operand, (int, float, complex)): + return +operand + elif o.op == "~": + if isinstance(operand, int): + return ~operand + elif o.op == "not": + if isinstance(operand, (bool, int, float, str, bytes)): + return not operand + return UNKNOWN + + def visit_assignment_expr(self, o: mypy.nodes.AssignmentExpr) -> object: + return o.value.accept(self) + + def visit_list_expr(self, o: mypy.nodes.ListExpr) -> object: + items = [item.accept(self) for item in o.items] + if all(item is not UNKNOWN for item in items): + return items + return UNKNOWN + + def visit_dict_expr(self, o: mypy.nodes.DictExpr) -> object: + items = [ + (UNKNOWN if key is None else key.accept(self), value.accept(self)) + for key, value in o.items + ] + if all(key is not UNKNOWN and value is not None for key, value in items): + return dict(items) + return UNKNOWN + + def visit_tuple_expr(self, o: mypy.nodes.TupleExpr) -> object: + items = [item.accept(self) for item in o.items] + if all(item is not UNKNOWN for item in items): + return tuple(items) + return UNKNOWN + + def visit_set_expr(self, o: mypy.nodes.SetExpr) -> object: + items = [item.accept(self) for item in o.items] + if all(item is not UNKNOWN for item in items): + return set(items) + return UNKNOWN + + def visit_index_expr(self, o: mypy.nodes.IndexExpr) -> object: + return UNKNOWN + + def visit_type_application(self, o: mypy.nodes.TypeApplication) -> object: + return UNKNOWN + + def visit_lambda_expr(self, o: mypy.nodes.LambdaExpr) -> object: + return UNKNOWN + + def visit_list_comprehension(self, o: mypy.nodes.ListComprehension) -> object: + return UNKNOWN + + def visit_set_comprehension(self, o: mypy.nodes.SetComprehension) -> object: + return UNKNOWN + + def visit_dictionary_comprehension(self, o: mypy.nodes.DictionaryComprehension) -> object: + return UNKNOWN + + def visit_generator_expr(self, o: mypy.nodes.GeneratorExpr) -> object: + return UNKNOWN + + def visit_slice_expr(self, o: mypy.nodes.SliceExpr) -> object: + return UNKNOWN + + def visit_conditional_expr(self, o: mypy.nodes.ConditionalExpr) -> object: + return UNKNOWN + + def visit_type_var_expr(self, o: mypy.nodes.TypeVarExpr) -> object: + return UNKNOWN + + def visit_paramspec_expr(self, o: mypy.nodes.ParamSpecExpr) -> object: + return UNKNOWN + + def visit_type_var_tuple_expr(self, o: mypy.nodes.TypeVarTupleExpr) -> object: + return UNKNOWN + + def visit_type_alias_expr(self, o: mypy.nodes.TypeAliasExpr) -> object: + return UNKNOWN + + def visit_namedtuple_expr(self, o: mypy.nodes.NamedTupleExpr) -> object: + return UNKNOWN + + def visit_enum_call_expr(self, o: mypy.nodes.EnumCallExpr) -> object: + return UNKNOWN + + def visit_typeddict_expr(self, o: mypy.nodes.TypedDictExpr) -> object: + return UNKNOWN + + def visit_newtype_expr(self, o: mypy.nodes.NewTypeExpr) -> object: + return UNKNOWN + + def visit__promote_expr(self, o: mypy.nodes.PromoteExpr) -> object: + return UNKNOWN + + def visit_await_expr(self, o: mypy.nodes.AwaitExpr) -> object: + return UNKNOWN + + def visit_temp_node(self, o: mypy.nodes.TempNode) -> object: + return UNKNOWN + + +_evaluator: Final = _NodeEvaluator() + + +def evaluate_expression(expr: mypy.nodes.Expression) -> object: + """Evaluate an expression at runtime. + + Return the result of the expression, or UNKNOWN if the expression cannot be + evaluated. + """ + return expr.accept(_evaluator) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index c4f36f2132a3..b2040ec074c3 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,158 +1,561 @@ -from typing import Dict, Tuple, List, cast, Undefined +from __future__ import annotations +from typing import Final, Iterable, Mapping, Sequence, TypeVar, cast, overload + +from mypy.nodes import ARG_STAR, FakeInfo, Var +from mypy.state import state from mypy.types import ( - Type, Instance, Callable, TypeVisitor, UnboundType, ErrorType, AnyType, - Void, NoneTyp, TypeVar, Overloaded, TupleType, UnionType, ErasedType, TypeList + ANY_STRATEGY, + AnyType, + BoolTypeQuery, + CallableType, + DeletedType, + ErasedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecFlavor, + ParamSpecType, + PartialType, + ProperType, + TrivialSyntheticTypeTranslator, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + flatten_nested_unions, + get_proper_type, + split_with_prefix_and_suffix, ) +from mypy.typevartuples import split_with_instance +# Solving the import cycle: +import mypy.type_visitor # ruff: isort: skip -def expand_type(typ: Type, map: Dict[int, Type]) -> Type: - """Substitute any type variable references in a type with given values.""" +# WARNING: these functions should never (directly or indirectly) depend on +# is_subtype(), meet_types(), join_types() etc. +# TODO: add a static dependency test for this. - return typ.accept(ExpandTypeVisitor(map)) +@overload +def expand_type(typ: CallableType, env: Mapping[TypeVarId, Type]) -> CallableType: ... -def expand_type_by_instance(typ: Type, instance: Instance) -> Type: - """Substitute type variables in type using values from an Instance.""" - if instance.args == []: +@overload +def expand_type(typ: ProperType, env: Mapping[TypeVarId, Type]) -> ProperType: ... + + +@overload +def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: ... + + +def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: + """Substitute any type variable references in a type given by a type + environment. + """ + return typ.accept(ExpandTypeVisitor(env)) + + +@overload +def expand_type_by_instance(typ: CallableType, instance: Instance) -> CallableType: ... + + +@overload +def expand_type_by_instance(typ: ProperType, instance: Instance) -> ProperType: ... + + +@overload +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: ... + + +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: + """Substitute type variables in type using values from an Instance. + Type variables are considered to be bound by the class declaration.""" + if not instance.args and not instance.type.has_type_var_tuple_type: return typ else: - variables = {} # type: Dict[int, Type] - for i in range(len(instance.args)): - variables[i + 1] = instance.args[i] - typ = expand_type(typ, variables) - if isinstance(typ, Callable): - bounds = [] # type: List[Tuple[int, Type]] - for j in range(len(instance.args)): - bounds.append((j + 1, instance.args[j])) - typ = update_callable_implicit_bounds(cast(Callable, typ), bounds) + variables: dict[TypeVarId, Type] = {} + if instance.type.has_type_var_tuple_type: + assert instance.type.type_var_tuple_prefix is not None + assert instance.type.type_var_tuple_suffix is not None + + args_prefix, args_middle, args_suffix = split_with_instance(instance) + tvars_prefix, tvars_middle, tvars_suffix = split_with_prefix_and_suffix( + tuple(instance.type.defn.type_vars), + instance.type.type_var_tuple_prefix, + instance.type.type_var_tuple_suffix, + ) + tvar = tvars_middle[0] + assert isinstance(tvar, TypeVarTupleType) + variables = {tvar.id: TupleType(list(args_middle), tvar.tuple_fallback)} + instance_args = args_prefix + args_suffix + tvars = tvars_prefix + tvars_suffix else: - pass - return typ + tvars = tuple(instance.type.defn.type_vars) + instance_args = instance.args + + for binder, arg in zip(tvars, instance_args): + assert isinstance(binder, TypeVarLikeType) + variables[binder.id] = arg + + return expand_type(typ, variables) + + +F = TypeVar("F", bound=FunctionLike) + + +def freshen_function_type_vars(callee: F) -> F: + """Substitute fresh type variables for generic function type variables.""" + if isinstance(callee, CallableType): + if not callee.is_generic(): + return cast(F, callee) + tvs = [] + tvmap: dict[TypeVarId, Type] = {} + for v in callee.variables: + tv = v.new_unification_variable(v) + tvs.append(tv) + tvmap[v.id] = tv + fresh = expand_type(callee, tvmap).copy_modified(variables=tvs) + return cast(F, fresh) + else: + assert isinstance(callee, Overloaded) + fresh_overload = Overloaded([freshen_function_type_vars(item) for item in callee.items]) + return cast(F, fresh_overload) + + +class HasGenericCallable(BoolTypeQuery): + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + + def visit_callable_type(self, t: CallableType) -> bool: + return t.is_generic() or super().visit_callable_type(t) + +# Share a singleton since this is performance sensitive +has_generic_callable: Final = HasGenericCallable() -class ExpandTypeVisitor(TypeVisitor[Type]): + +T = TypeVar("T", bound=Type) + + +def freshen_all_functions_type_vars(t: T) -> T: + result: Type + has_generic_callable.reset() + if not t.accept(has_generic_callable): + return t # Fast path to avoid expensive freshening + else: + result = t.accept(FreshenCallableVisitor()) + assert isinstance(result, type(t)) + return result + + +class FreshenCallableVisitor(mypy.type_visitor.TypeTranslator): + def visit_callable_type(self, t: CallableType) -> Type: + result = super().visit_callable_type(t) + assert isinstance(result, ProperType) and isinstance(result, CallableType) + return freshen_function_type_vars(result) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # Same as for ExpandTypeVisitor + return t.copy_modified(args=[arg.accept(self) for arg in t.args]) + + +class ExpandTypeVisitor(TrivialSyntheticTypeTranslator): """Visitor that substitutes type variables with values.""" - variables = Undefined(Dict[int, Type]) # typevar id -> value + variables: Mapping[TypeVarId, Type] # TypeVar id -> TypeVar value - def __init__(self, variables: Dict[int, Type]) -> None: + def __init__(self, variables: Mapping[TypeVarId, Type]) -> None: + super().__init__() self.variables = variables + self.recursive_tvar_guard: dict[TypeVarId, Type | None] = {} def visit_unbound_type(self, t: UnboundType) -> Type: return t - def visit_error_type(self, t: ErrorType) -> Type: + def visit_any(self, t: AnyType) -> Type: return t - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: + def visit_none_type(self, t: NoneType) -> Type: return t - def visit_void(self, t: Void) -> Type: + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: return t - def visit_none_type(self, t: NoneTyp) -> Type: + def visit_deleted_type(self, t: DeletedType) -> Type: return t def visit_erased_type(self, t: ErasedType) -> Type: - # Should not get here. - raise RuntimeError() + # This may happen during type inference if some function argument + # type is a generic callable, and its erased form will appear in inferred + # constraints, then solver may check subtyping between them, which will trigger + # unify_generic_callables(), this is why we can get here. Another example is + # when inferring type of lambda in generic context, the lambda body contains + # a generic method in generic class. + return t def visit_instance(self, t: Instance) -> Type: - args = self.expand_types(t.args) - return Instance(t.type, args, t.line, t.repr) - - def visit_type_var(self, t: TypeVar) -> Type: + args = self.expand_types_with_unpack(list(t.args)) + + if isinstance(t.type, FakeInfo): + # The type checker expands function definitions and bodies + # if they depend on constrained type variables but the body + # might contain a tuple type comment (e.g., # type: (int, float)), + # in which case 't.type' is not yet available. + # + # See: https://github.com/python/mypy/issues/16649 + return t.copy_modified(args=args) + + if t.type.fullname == "builtins.tuple": + # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] + arg = args[0] + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + args = list(unpacked.args) + return t.copy_modified(args=args) + + def visit_type_var(self, t: TypeVarType) -> Type: + # Normally upper bounds can't contain other type variables, the only exception is + # special type variable Self`0 <: C[T, S], where C is the class where Self is used. + if t.id.is_self(): + t = t.copy_modified(upper_bound=t.upper_bound.accept(self)) repl = self.variables.get(t.id, t) - if isinstance(repl, Instance): - inst = cast(Instance, repl) - # Return copy of instance with type erasure flag on. - return Instance(inst.type, inst.args, inst.line, inst.repr, True) + if isinstance(repl, ProperType) and isinstance(repl, Instance): + # TODO: do we really need to do this? + # If I try to remove this special-casing ~40 tests fail on reveal_type(). + return repl.copy_modified(last_known_value=None) + if isinstance(repl, TypeVarType) and repl.has_default(): + if (tvar_id := repl.id) in self.recursive_tvar_guard: + return self.recursive_tvar_guard[tvar_id] or repl + self.recursive_tvar_guard[tvar_id] = None + repl = repl.accept(self) + if isinstance(repl, TypeVarType): + repl.default = repl.default.accept(self) + self.recursive_tvar_guard[tvar_id] = repl + return repl + + def visit_param_spec(self, t: ParamSpecType) -> Type: + # Set prefix to something empty, so we don't duplicate it below. + repl = self.variables.get(t.id, t.copy_modified(prefix=Parameters([], [], []))) + if isinstance(repl, ParamSpecType): + return repl.copy_modified( + flavor=t.flavor, + prefix=t.prefix.copy_modified( + arg_types=self.expand_types(t.prefix.arg_types) + repl.prefix.arg_types, + arg_kinds=t.prefix.arg_kinds + repl.prefix.arg_kinds, + arg_names=t.prefix.arg_names + repl.prefix.arg_names, + ), + ) + elif isinstance(repl, Parameters): + assert t.flavor == ParamSpecFlavor.BARE + return Parameters( + self.expand_types(t.prefix.arg_types) + repl.arg_types, + t.prefix.arg_kinds + repl.arg_kinds, + t.prefix.arg_names + repl.arg_names, + variables=[*t.prefix.variables, *repl.variables], + imprecise_arg_kinds=repl.imprecise_arg_kinds, + ) else: + # We could encode Any as trivial parameters etc., but it would be too verbose. + # TODO: assert this is a trivial type, like Any, Never, or object. return repl - def visit_callable(self, t: Callable) -> Type: - return Callable(self.expand_types(t.arg_types), - t.arg_kinds, - t.arg_names, - t.ret_type.accept(self), - t.fallback, - t.name, - t.variables, - self.expand_bound_vars(t.bound_vars), t.line, t.repr) + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: + # Sometimes solver may need to expand a type variable with (a copy of) itself + # (usually together with other TypeVars, but it is hard to filter out TypeVarTuples). + repl = self.variables.get(t.id, t) + if isinstance(repl, TypeVarTupleType): + return repl + elif isinstance(repl, ProperType) and isinstance(repl, (AnyType, UninhabitedType)): + # Some failed inference scenarios will try to set all type variables to Never. + # Instead of being picky and require all the callers to wrap them, + # do this here instead. + # Note: most cases when this happens are handled in expand unpack below, but + # in rare cases (e.g. ParamSpec containing Unpack star args) it may be skipped. + return t.tuple_fallback.copy_modified(args=[repl]) + raise NotImplementedError + + def visit_unpack_type(self, t: UnpackType) -> Type: + # It is impossible to reasonably implement visit_unpack_type, because + # unpacking inherently expands to something more like a list of types. + # + # Relevant sections that can call unpack should call expand_unpack() + # instead. + # However, if the item is a variadic tuple, we can simply carry it over. + # In particular, if we expand A[*tuple[T, ...]] with substitutions {T: str}, + # it is hard to assert this without getting proper type. Another important + # example is non-normalized types when called from semanal.py. + return UnpackType(t.type.accept(self)) + + def expand_unpack(self, t: UnpackType) -> list[Type]: + assert isinstance(t.type, TypeVarTupleType) + repl = get_proper_type(self.variables.get(t.type.id, t.type)) + if isinstance(repl, UnpackType): + repl = get_proper_type(repl.type) + if isinstance(repl, TupleType): + return repl.items + elif ( + isinstance(repl, Instance) + and repl.type.fullname == "builtins.tuple" + or isinstance(repl, TypeVarTupleType) + ): + return [UnpackType(typ=repl)] + elif isinstance(repl, (AnyType, UninhabitedType)): + # Replace *Ts = Any with *Ts = *tuple[Any, ...] and same for Never. + # These types may appear here as a result of user error or failed inference. + return [UnpackType(t.type.tuple_fallback.copy_modified(args=[repl]))] + else: + raise RuntimeError(f"Invalid type replacement to expand: {repl}") + + def visit_parameters(self, t: Parameters) -> Type: + return t.copy_modified(arg_types=self.expand_types(t.arg_types)) + + def interpolate_args_for_unpack(self, t: CallableType, var_arg: UnpackType) -> list[Type]: + star_index = t.arg_kinds.index(ARG_STAR) + prefix = self.expand_types(t.arg_types[:star_index]) + suffix = self.expand_types(t.arg_types[star_index + 1 :]) + + var_arg_type = get_proper_type(var_arg.type) + new_unpack: Type + if isinstance(var_arg_type, Instance): + # we have something like Unpack[Tuple[Any, ...]] + new_unpack = UnpackType(var_arg.type.accept(self)) + elif isinstance(var_arg_type, TupleType): + # We have something like Unpack[Tuple[Unpack[Ts], X1, X2]] + expanded_tuple = var_arg_type.accept(self) + assert isinstance(expanded_tuple, ProperType) and isinstance(expanded_tuple, TupleType) + expanded_items = expanded_tuple.items + fallback = var_arg_type.partial_fallback + new_unpack = UnpackType(TupleType(expanded_items, fallback)) + elif isinstance(var_arg_type, TypeVarTupleType): + # We have plain Unpack[Ts] + fallback = var_arg_type.tuple_fallback + expanded_items = self.expand_unpack(var_arg) + new_unpack = UnpackType(TupleType(expanded_items, fallback)) + else: + # We have invalid type in Unpack. This can happen when expanding aliases + # to Callable[[*Invalid], Ret] + new_unpack = AnyType(TypeOfAny.from_error, line=var_arg.line, column=var_arg.column) + return prefix + [new_unpack] + suffix + + def visit_callable_type(self, t: CallableType) -> CallableType: + param_spec = t.param_spec() + if param_spec is not None: + repl = self.variables.get(param_spec.id) + # If a ParamSpec in a callable type is substituted with a + # callable type, we can't use normal substitution logic, + # since ParamSpec is actually split into two components + # *P.args and **P.kwargs in the original type. Instead, we + # must expand both of them with all the argument types, + # kinds and names in the replacement. The return type in + # the replacement is ignored. + if isinstance(repl, Parameters): + # We need to expand both the types in the prefix and the ParamSpec itself + expanded = t.copy_modified( + arg_types=self.expand_types(t.arg_types[:-2]) + repl.arg_types, + arg_kinds=t.arg_kinds[:-2] + repl.arg_kinds, + arg_names=t.arg_names[:-2] + repl.arg_names, + ret_type=t.ret_type.accept(self), + type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), + type_is=(t.type_is.accept(self) if t.type_is is not None else None), + imprecise_arg_kinds=(t.imprecise_arg_kinds or repl.imprecise_arg_kinds), + variables=[*repl.variables, *t.variables], + ) + var_arg = expanded.var_arg() + if var_arg is not None and isinstance(var_arg.typ, UnpackType): + # Sometimes we get new unpacks after expanding ParamSpec. + expanded.normalize_trivial_unpack() + return expanded + elif isinstance(repl, ParamSpecType): + # We're substituting one ParamSpec for another; this can mean that the prefix + # changes, e.g. substitute Concatenate[int, P] in place of Q. + prefix = repl.prefix + clean_repl = repl.copy_modified(prefix=Parameters([], [], [])) + return t.copy_modified( + arg_types=self.expand_types(t.arg_types[:-2]) + + prefix.arg_types + + [ + clean_repl.with_flavor(ParamSpecFlavor.ARGS), + clean_repl.with_flavor(ParamSpecFlavor.KWARGS), + ], + arg_kinds=t.arg_kinds[:-2] + prefix.arg_kinds + t.arg_kinds[-2:], + arg_names=t.arg_names[:-2] + prefix.arg_names + t.arg_names[-2:], + ret_type=t.ret_type.accept(self), + from_concatenate=t.from_concatenate or bool(repl.prefix.arg_types), + imprecise_arg_kinds=(t.imprecise_arg_kinds or prefix.imprecise_arg_kinds), + ) + + var_arg = t.var_arg() + needs_normalization = False + if var_arg is not None and isinstance(var_arg.typ, UnpackType): + needs_normalization = True + arg_types = self.interpolate_args_for_unpack(t, var_arg.typ) + else: + arg_types = self.expand_types(t.arg_types) + expanded = t.copy_modified( + arg_types=arg_types, + ret_type=t.ret_type.accept(self), + type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), + type_is=(t.type_is.accept(self) if t.type_is is not None else None), + ) + if needs_normalization: + return expanded.with_normalized_var_args() + return expanded def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[Callable] - for item in t.items(): - items.append(cast(Callable, item.accept(self))) + items: list[CallableType] = [] + for item in t.items: + new_item = item.accept(self) + assert isinstance(new_item, ProperType) + assert isinstance(new_item, CallableType) + items.append(new_item) return Overloaded(items) + def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]: + """Expands a list of types that has an unpack.""" + items: list[Type] = [] + for item in typs: + if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): + items.extend(self.expand_unpack(item)) + else: + items.append(item.accept(self)) + return items + def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.expand_types(t.items), t.fallback, t.line, t.repr) + items = self.expand_types_with_unpack(t.items) + if len(items) == 1: + # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] + item = items[0] + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + if t.partial_fallback.type.fullname != "builtins.tuple": + # If it is a subtype (like named tuple) we need to preserve it, + # this essentially mimics the logic in tuple_fallback(). + return t.partial_fallback.accept(self) + return unpacked + fallback = t.partial_fallback.accept(self) + assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) + return t.copy_modified(items=items, fallback=fallback) + + def visit_typeddict_type(self, t: TypedDictType) -> Type: + if cached := self.get_cached(t): + return cached + fallback = t.fallback.accept(self) + assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) + result = t.copy_modified(item_types=self.expand_types(t.items.values()), fallback=fallback) + self.set_cached(t, result) + return result + + def visit_literal_type(self, t: LiteralType) -> Type: + # TODO: Verify this implementation is correct + return t def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.expand_types(t.items), t.line, t.repr) + # Use cache to avoid O(n**2) or worse expansion of types during translation + # (only for large unions, since caching adds overhead) + use_cache = len(t.items) > 3 + if use_cache and (cached := self.get_cached(t)): + return cached + + expanded = self.expand_types(t.items) + # After substituting for type variables in t.items, some resulting types + # might be subtypes of others, however calling make_simplified_union() + # can cause recursion, so we just remove strict duplicates. + simplified = UnionType.make_union( + remove_trivial(flatten_nested_unions(expanded)), t.line, t.column + ) + # This call to get_proper_type() is unfortunate but is required to preserve + # the invariant that ProperType will stay ProperType after applying expand_type(), + # otherwise a single item union of a type alias will break it. Note this should not + # cause infinite recursion since pathological aliases like A = Union[A, B] are + # banned at the semantic analysis level. + result = get_proper_type(simplified) + + if use_cache: + self.set_cached(t, result) + return result + + def visit_partial_type(self, t: PartialType) -> Type: + return t - def expand_types(self, types: List[Type]) -> List[Type]: - a = [] # type: List[Type] + def visit_type_type(self, t: TypeType) -> Type: + # TODO: Verify that the new item type is valid (instance or + # union of instances or Any). Sadly we can't report errors + # here yet. + item = t.item.accept(self) + return TypeType.make_normalized(item) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # Target of the type alias cannot contain type variables (not bound by the type + # alias itself), so we just expand the arguments. + args = self.expand_types_with_unpack(t.args) + # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? + return t.copy_modified(args=args) + + def expand_types(self, types: Iterable[Type]) -> list[Type]: + a: list[Type] = [] for t in types: a.append(t.accept(self)) return a - def expand_bound_vars( - self, types: List[Tuple[int, Type]]) -> List[Tuple[int, Type]]: - a = [] # type: List[Tuple[int, Type]] - for id, t in types: - a.append((id, t.accept(self))) - return a +@overload +def expand_self_type(var: Var, typ: ProperType, replacement: ProperType) -> ProperType: ... -def update_callable_implicit_bounds( - t: Callable, arg_types: List[Tuple[int, Type]]) -> Callable: - # FIX what if there are existing bounds? - return Callable(t.arg_types, - t.arg_kinds, - t.arg_names, - t.ret_type, - t.fallback, - t.name, - t.variables, - arg_types, t.line, t.repr) +@overload +def expand_self_type(var: Var, typ: Type, replacement: Type) -> Type: ... -def expand_caller_var_args(arg_types: List[Type], - fixed_argc: int) -> Tuple[List[Type], Type]: - """Expand the caller argument types in a varargs call. - Fixedargc is the maximum number of fixed arguments that the target - function accepts. +def expand_self_type(var: Var, typ: Type, replacement: Type) -> Type: + """Expand appearances of Self type in a variable type.""" + if var.info.self_type is not None and not var.is_property: + return expand_type(typ, {var.info.self_type.id: replacement}) + return typ - Return (fixed argument types, type of the rest of the arguments). Return - (None, None) if the last (vararg) argument had an invalid type. If the - vararg argument was not an array (nor dynamic), the last item in the - returned tuple is None. - """ - if isinstance(arg_types[-1], TupleType): - return arg_types[:-1] + (cast(TupleType, arg_types[-1])).items, None - else: - item_type = Undefined # type: Type - if isinstance(arg_types[-1], AnyType): - item_type = AnyType() - elif isinstance(arg_types[-1], Instance) and ( - cast(Instance, arg_types[-1]).type.fullname() == - 'builtins.list'): - # List. - item_type = (cast(Instance, arg_types[-1])).args[0] - else: - return None, None +def remove_trivial(types: Iterable[Type]) -> list[Type]: + """Make trivial simplifications on a list of types without calling is_subtype(). - if len(arg_types) > fixed_argc: - return arg_types[:-1], item_type - else: - return (arg_types[:-1] + - [item_type] * (fixed_argc - len(arg_types) + 1), item_type) + This makes following simplifications: + * Remove bottom types (taking into account strict optional setting) + * Remove everything else if there is an `object` + * Remove strict duplicate types + """ + removed_none = False + new_types = [] + all_types = set() + for t in types: + p_t = get_proper_type(t) + if isinstance(p_t, UninhabitedType): + continue + if isinstance(p_t, NoneType) and not state.strict_optional: + removed_none = True + continue + if isinstance(p_t, Instance) and p_t.type.fullname == "builtins.object": + return [p_t] + if p_t not in all_types: + new_types.append(t) + all_types.add(p_t) + if new_types: + return new_types + if removed_none: + return [NoneType()] + return [UninhabitedType()] diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py new file mode 100644 index 000000000000..506194a4b285 --- /dev/null +++ b/mypy/exprtotype.py @@ -0,0 +1,252 @@ +"""Translate an Expression to a Type value.""" + +from __future__ import annotations + +from typing import Callable + +from mypy.fastparse import parse_type_string +from mypy.nodes import ( + MISSING_FALLBACK, + BytesExpr, + CallExpr, + ComplexExpr, + Context, + DictExpr, + EllipsisExpr, + Expression, + FloatExpr, + IndexExpr, + IntExpr, + ListExpr, + MemberExpr, + NameExpr, + OpExpr, + RefExpr, + StarExpr, + StrExpr, + SymbolTableNode, + TupleExpr, + UnaryExpr, + get_member_expr_fullname, +) +from mypy.options import Options +from mypy.types import ( + ANNOTATED_TYPE_NAMES, + AnyType, + CallableArgument, + EllipsisType, + Instance, + ProperType, + RawExpressionType, + Type, + TypedDictType, + TypeList, + TypeOfAny, + UnboundType, + UnionType, + UnpackType, +) + + +class TypeTranslationError(Exception): + """Exception raised when an expression is not valid as a type.""" + + +def _extract_argument_name(expr: Expression) -> str | None: + if isinstance(expr, NameExpr) and expr.name == "None": + return None + elif isinstance(expr, StrExpr): + return expr.value + else: + raise TypeTranslationError() + + +def expr_to_unanalyzed_type( + expr: Expression, + options: Options, + allow_new_syntax: bool = False, + _parent: Expression | None = None, + allow_unpack: bool = False, + lookup_qualified: Callable[[str, Context], SymbolTableNode | None] | None = None, +) -> ProperType: + """Translate an expression to the corresponding type. + + The result is not semantically analyzed. It can be UnboundType or TypeList. + Raise TypeTranslationError if the expression cannot represent a type. + + If lookup_qualified is not provided, the expression is expected to be semantically + analyzed. + + If allow_new_syntax is True, allow all type syntax independent of the target + Python version (used in stubs). + + # TODO: a lot of code here is duplicated in fastparse.py, refactor this. + """ + # The `parent` parameter is used in recursive calls to provide context for + # understanding whether an CallableArgument is ok. + name: str | None = None + if isinstance(expr, NameExpr): + name = expr.name + if name == "True": + return RawExpressionType(True, "builtins.bool", line=expr.line, column=expr.column) + elif name == "False": + return RawExpressionType(False, "builtins.bool", line=expr.line, column=expr.column) + else: + return UnboundType(name, line=expr.line, column=expr.column) + elif isinstance(expr, MemberExpr): + fullname = get_member_expr_fullname(expr) + if fullname: + return UnboundType(fullname, line=expr.line, column=expr.column) + else: + raise TypeTranslationError() + elif isinstance(expr, IndexExpr): + base = expr_to_unanalyzed_type(expr.base, options, allow_new_syntax, expr) + if isinstance(base, UnboundType): + if base.args: + raise TypeTranslationError() + if isinstance(expr.index, TupleExpr): + args = expr.index.items + else: + args = [expr.index] + + if isinstance(expr.base, RefExpr): + # Check if the type is Annotated[...]. For this we need the fullname, + # which must be looked up if the expression hasn't been semantically analyzed. + base_fullname = None + if lookup_qualified is not None: + sym = lookup_qualified(base.name, expr) + if sym and sym.node: + base_fullname = sym.node.fullname + else: + base_fullname = expr.base.fullname + + if base_fullname is not None and base_fullname in ANNOTATED_TYPE_NAMES: + # TODO: this is not the optimal solution as we are basically getting rid + # of the Annotation definition and only returning the type information, + # losing all the annotations. + return expr_to_unanalyzed_type(args[0], options, allow_new_syntax, expr) + base.args = tuple( + expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr, allow_unpack=True) + for arg in args + ) + if not base.args: + base.empty_tuple_index = True + return base + else: + raise TypeTranslationError() + elif ( + isinstance(expr, OpExpr) + and expr.op == "|" + and ((options.python_version >= (3, 10)) or allow_new_syntax) + ): + return UnionType( + [ + expr_to_unanalyzed_type(expr.left, options, allow_new_syntax), + expr_to_unanalyzed_type(expr.right, options, allow_new_syntax), + ], + uses_pep604_syntax=True, + ) + elif isinstance(expr, CallExpr) and isinstance(_parent, ListExpr): + c = expr.callee + names = [] + # Go through the dotted member expr chain to get the full arg + # constructor name to look up + while True: + if isinstance(c, NameExpr): + names.append(c.name) + break + elif isinstance(c, MemberExpr): + names.append(c.name) + c = c.expr + else: + raise TypeTranslationError() + arg_const = ".".join(reversed(names)) + + # Go through the constructor args to get its name and type. + name = None + default_type = AnyType(TypeOfAny.unannotated) + typ: Type = default_type + for i, arg in enumerate(expr.args): + if expr.arg_names[i] is not None: + if expr.arg_names[i] == "name": + if name is not None: + # Two names + raise TypeTranslationError() + name = _extract_argument_name(arg) + continue + elif expr.arg_names[i] == "type": + if typ is not default_type: + # Two types + raise TypeTranslationError() + typ = expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr) + continue + else: + raise TypeTranslationError() + elif i == 0: + typ = expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr) + elif i == 1: + name = _extract_argument_name(arg) + else: + raise TypeTranslationError() + return CallableArgument(typ, name, arg_const, expr.line, expr.column) + elif isinstance(expr, ListExpr): + return TypeList( + [ + expr_to_unanalyzed_type(t, options, allow_new_syntax, expr, allow_unpack=True) + for t in expr.items + ], + line=expr.line, + column=expr.column, + ) + elif isinstance(expr, StrExpr): + return parse_type_string(expr.value, "builtins.str", expr.line, expr.column) + elif isinstance(expr, BytesExpr): + return parse_type_string(expr.value, "builtins.bytes", expr.line, expr.column) + elif isinstance(expr, UnaryExpr): + typ = expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax) + if isinstance(typ, RawExpressionType): + if isinstance(typ.literal_value, int): + if expr.op == "-": + typ.literal_value *= -1 + return typ + elif expr.op == "+": + return typ + raise TypeTranslationError() + elif isinstance(expr, IntExpr): + return RawExpressionType(expr.value, "builtins.int", line=expr.line, column=expr.column) + elif isinstance(expr, FloatExpr): + # Floats are not valid parameters for RawExpressionType , so we just + # pass in 'None' for now. We'll report the appropriate error at a later stage. + return RawExpressionType(None, "builtins.float", line=expr.line, column=expr.column) + elif isinstance(expr, ComplexExpr): + # Same thing as above with complex numbers. + return RawExpressionType(None, "builtins.complex", line=expr.line, column=expr.column) + elif isinstance(expr, EllipsisExpr): + return EllipsisType(expr.line) + elif allow_unpack and isinstance(expr, StarExpr): + return UnpackType( + expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True + ) + elif isinstance(expr, DictExpr): + if not expr.items: + raise TypeTranslationError() + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in expr.items: + if not isinstance(item_name, StrExpr): + if item_name is None: + extra_items_from.append( + expr_to_unanalyzed_type(value, options, allow_new_syntax, expr) + ) + continue + raise TypeTranslationError() + items[item_name.value] = expr_to_unanalyzed_type( + value, options, allow_new_syntax, expr + ) + result = TypedDictType( + items, set(), set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column + ) + result.extra_items_from = extra_items_from + return result + else: + raise TypeTranslationError() diff --git a/mypy/fastparse.py b/mypy/fastparse.py new file mode 100644 index 000000000000..a47ed9b536da --- /dev/null +++ b/mypy/fastparse.py @@ -0,0 +1,2253 @@ +from __future__ import annotations + +import copy +import re +import sys +import warnings +from typing import Any, Callable, Final, List, Optional, Sequence, TypeVar, Union, cast +from typing_extensions import Literal, overload + +from mypy import defaults, errorcodes as codes, message_registry +from mypy.errors import Errors +from mypy.message_registry import ErrorMessage +from mypy.nodes import ( + ARG_NAMED, + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + MISSING_FALLBACK, + PARAM_SPEC_KIND, + TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, + ArgKind, + Argument, + AssertStmt, + AssignmentExpr, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + BytesExpr, + CallExpr, + ClassDef, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + ContinueStmt, + Decorator, + DelStmt, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + Expression, + ExpressionStmt, + FloatExpr, + ForStmt, + FuncDef, + GeneratorExpr, + GlobalDecl, + IfStmt, + Import, + ImportAll, + ImportBase, + ImportFrom, + IndexExpr, + IntExpr, + LambdaExpr, + ListComprehension, + ListExpr, + MatchStmt, + MemberExpr, + MypyFile, + NameExpr, + Node, + NonlocalDecl, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + OverloadPart, + PassStmt, + RaiseStmt, + RefExpr, + ReturnStmt, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + Statement, + StrExpr, + SuperExpr, + TempNode, + TryStmt, + TupleExpr, + TypeAliasStmt, + TypeParam, + UnaryExpr, + Var, + WhileStmt, + WithStmt, + YieldExpr, + YieldFromExpr, + check_arg_names, +) +from mypy.options import Options +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, +) +from mypy.reachability import infer_reachability_of_if_statement, mark_block_unreachable +from mypy.sharedparse import argument_elide_name, special_function_elide_names +from mypy.traverser import TraverserVisitor +from mypy.types import ( + AnyType, + CallableArgument, + CallableType, + EllipsisType, + Instance, + ProperType, + RawExpressionType, + TupleType, + Type, + TypedDictType, + TypeList, + TypeOfAny, + UnboundType, + UnionType, + UnpackType, +) +from mypy.util import bytes_to_human_readable_repr, unnamed_function + +# pull this into a final variable to make mypyc be quiet about the +# the default argument warning +PY_MINOR_VERSION: Final = sys.version_info[1] + +import ast as ast3 + +# TODO: Index, ExtSlice are deprecated in 3.9. +from ast import AST, Attribute, Call, FunctionType, Index, Name, Starred, UAdd, UnaryOp, USub + + +def ast3_parse( + source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION +) -> AST: + return ast3.parse( + source, + filename, + mode, + type_comments=True, # This works the magic + feature_version=feature_version, + ) + + +NamedExpr = ast3.NamedExpr +Constant = ast3.Constant + +if sys.version_info >= (3, 10): + Match = ast3.Match + MatchValue = ast3.MatchValue + MatchSingleton = ast3.MatchSingleton + MatchSequence = ast3.MatchSequence + MatchStar = ast3.MatchStar + MatchMapping = ast3.MatchMapping + MatchClass = ast3.MatchClass + MatchAs = ast3.MatchAs + MatchOr = ast3.MatchOr + AstNode = Union[ast3.expr, ast3.stmt, ast3.pattern, ast3.ExceptHandler] +else: + Match = Any + MatchValue = Any + MatchSingleton = Any + MatchSequence = Any + MatchStar = Any + MatchMapping = Any + MatchClass = Any + MatchAs = Any + MatchOr = Any + AstNode = Union[ast3.expr, ast3.stmt, ast3.ExceptHandler] + +if sys.version_info >= (3, 11): + TryStar = ast3.TryStar +else: + TryStar = Any + +if sys.version_info >= (3, 12): + ast_TypeAlias = ast3.TypeAlias + ast_ParamSpec = ast3.ParamSpec + ast_TypeVar = ast3.TypeVar + ast_TypeVarTuple = ast3.TypeVarTuple +else: + ast_TypeAlias = Any + ast_ParamSpec = Any + ast_TypeVar = Any + ast_TypeVarTuple = Any + +N = TypeVar("N", bound=Node) + +# There is no way to create reasonable fallbacks at this stage, +# they must be patched later. +_dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) + +TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)") + + +def parse( + source: str | bytes, + fnam: str, + module: str | None, + errors: Errors, + options: Options | None = None, +) -> MypyFile: + """Parse a source file, without doing any semantic analysis. + + Return the parse tree. If errors is not provided, raise ParseError + on failure. Otherwise, use the errors object to report parse errors. + """ + ignore_errors = (options is not None and options.ignore_errors) or ( + fnam in errors.ignored_files + ) + # If errors are ignored, we can drop many function bodies to speed up type checking. + strip_function_bodies = ignore_errors and (options is None or not options.preserve_asts) + + if options is None: + options = Options() + errors.set_file(fnam, module, options=options) + is_stub_file = fnam.endswith(".pyi") + if is_stub_file: + feature_version = defaults.PYTHON3_VERSION[1] + if options.python_version[0] == 3 and options.python_version[1] > feature_version: + feature_version = options.python_version[1] + else: + assert options.python_version[0] >= 3 + feature_version = options.python_version[1] + try: + # Disable deprecation warnings about \u + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + ast = ast3_parse(source, fnam, "exec", feature_version=feature_version) + + tree = ASTConverter( + options=options, + is_stub=is_stub_file, + errors=errors, + strip_function_bodies=strip_function_bodies, + path=fnam, + ).visit(ast) + except SyntaxError as e: + # alias to please mypyc + is_py38_or_earlier = sys.version_info < (3, 9) + if is_py38_or_earlier and e.filename == "": + # In Python 3.8 and earlier, syntax errors in f-strings have lineno relative to the + # start of the f-string. This would be misleading, as mypy will report the error as the + # lineno within the file. + e.lineno = None + message = e.msg + if feature_version > sys.version_info.minor and message.startswith("invalid syntax"): + python_version_str = f"{options.python_version[0]}.{options.python_version[1]}" + message += f"; you likely need to run mypy using Python {python_version_str} or newer" + errors.report( + e.lineno if e.lineno is not None else -1, + e.offset, + message, + blocker=True, + code=codes.SYNTAX, + ) + tree = MypyFile([], [], False, {}) + + assert isinstance(tree, MypyFile) + return tree + + +def parse_type_ignore_tag(tag: str | None) -> list[str] | None: + """Parse optional "[code, ...]" tag after "# type: ignore". + + Return: + * [] if no tag was found (ignore all errors) + * list of ignored error codes if a tag was found + * None if the tag was invalid. + """ + if not tag or tag.strip() == "" or tag.strip().startswith("#"): + # No tag -- ignore all errors. + return [] + m = re.match(r"\s*\[([^]#]*)\]\s*(#.*)?$", tag) + if m is None: + # Invalid "# type: ignore" comment. + return None + return [code.strip() for code in m.group(1).split(",")] + + +def parse_type_comment( + type_comment: str, line: int, column: int, errors: Errors | None +) -> tuple[list[str] | None, ProperType | None]: + """Parse type portion of a type comment (+ optional type ignore). + + Return (ignore info, parsed type). + """ + try: + typ = ast3_parse(type_comment, "", "eval") + except SyntaxError: + if errors is not None: + stripped_type = type_comment.split("#", 2)[0].strip() + err_msg = message_registry.TYPE_COMMENT_SYNTAX_ERROR_VALUE.format(stripped_type) + errors.report(line, column, err_msg.value, blocker=True, code=err_msg.code) + return None, None + else: + raise + else: + extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment) + if extra_ignore: + tag: str | None = extra_ignore.group(1) + ignored: list[str] | None = parse_type_ignore_tag(tag) + if ignored is None: + if errors is not None: + errors.report( + line, column, message_registry.INVALID_TYPE_IGNORE.value, code=codes.SYNTAX + ) + else: + raise SyntaxError + else: + ignored = None + assert isinstance(typ, ast3.Expression) + converted = TypeConverter( + errors, line=line, override_column=column, is_evaluated=False + ).visit(typ.body) + return ignored, converted + + +def parse_type_string( + expr_string: str, expr_fallback_name: str, line: int, column: int +) -> ProperType: + """Parses a type that was originally present inside of an explicit string. + + For example, suppose we have the type `Foo["blah"]`. We should parse the + string expression "blah" using this function. + """ + try: + _, node = parse_type_comment(f"({expr_string})", line=line, column=column, errors=None) + if isinstance(node, (UnboundType, UnionType)) and node.original_str_expr is None: + node.original_str_expr = expr_string + node.original_str_fallback = expr_fallback_name + return node + else: + return RawExpressionType(expr_string, expr_fallback_name, line, column) + except (SyntaxError, ValueError): + # Note: the parser will raise a `ValueError` instead of a SyntaxError if + # the string happens to contain things like \x00. + return RawExpressionType(expr_string, expr_fallback_name, line, column) + + +def is_no_type_check_decorator(expr: ast3.expr) -> bool: + if isinstance(expr, Name): + return expr.id == "no_type_check" + elif isinstance(expr, Attribute): + if isinstance(expr.value, Name): + return expr.value.id == "typing" and expr.attr == "no_type_check" + return False + + +def find_disallowed_expression_in_annotation_scope(expr: ast3.expr | None) -> ast3.expr | None: + if expr is None: + return None + for node in ast3.walk(expr): + if isinstance(node, (ast3.Yield, ast3.YieldFrom, ast3.NamedExpr, ast3.Await)): + return node + return None + + +class ASTConverter: + def __init__( + self, + options: Options, + is_stub: bool, + errors: Errors, + *, + strip_function_bodies: bool, + path: str, + ) -> None: + # 'C' for class, 'D' for function signature, 'F' for function, 'L' for lambda + self.class_and_function_stack: list[Literal["C", "D", "F", "L"]] = [] + self.imports: list[ImportBase] = [] + + self.options = options + self.is_stub = is_stub + self.errors = errors + self.strip_function_bodies = strip_function_bodies + self.path = path + + self.type_ignores: dict[int, list[str]] = {} + + # Cache of visit_X methods keyed by type of visited object + self.visitor_cache: dict[type, Callable[[AST | None], Any]] = {} + + def note(self, msg: str, line: int, column: int) -> None: + self.errors.report(line, column, msg, severity="note", code=codes.SYNTAX) + + def fail(self, msg: ErrorMessage, line: int, column: int, blocker: bool = True) -> None: + if blocker or not self.options.ignore_errors: + # Make sure self.errors reflects any type ignores that we have parsed + self.errors.set_file_ignored_lines( + self.path, self.type_ignores, self.options.ignore_errors + ) + self.errors.report(line, column, msg.value, blocker=blocker, code=msg.code) + + def fail_merge_overload(self, node: IfStmt) -> None: + self.fail( + message_registry.FAILED_TO_MERGE_OVERLOADS, + line=node.line, + column=node.column, + blocker=False, + ) + + def visit(self, node: AST | None) -> Any: + if node is None: + return None + typeobj = type(node) + visitor = self.visitor_cache.get(typeobj) + if visitor is None: + method = "visit_" + node.__class__.__name__ + visitor = getattr(self, method) + self.visitor_cache[typeobj] = visitor + return visitor(node) + + def set_line(self, node: N, n: AstNode) -> N: + node.line = n.lineno + node.column = n.col_offset + node.end_line = getattr(n, "end_lineno", None) + node.end_column = getattr(n, "end_col_offset", None) + + return node + + def translate_opt_expr_list(self, l: Sequence[AST | None]) -> list[Expression | None]: + res: list[Expression | None] = [] + for e in l: + exp = self.visit(e) + res.append(exp) + return res + + def translate_expr_list(self, l: Sequence[AST]) -> list[Expression]: + return cast(List[Expression], self.translate_opt_expr_list(l)) + + def get_lineno(self, node: ast3.expr | ast3.stmt) -> int: + if ( + isinstance(node, (ast3.AsyncFunctionDef, ast3.ClassDef, ast3.FunctionDef)) + and node.decorator_list + ): + return node.decorator_list[0].lineno + return node.lineno + + def translate_stmt_list( + self, + stmts: Sequence[ast3.stmt], + *, + ismodule: bool = False, + can_strip: bool = False, + is_coroutine: bool = False, + ) -> list[Statement]: + # A "# type: ignore" comment before the first statement of a module + # ignores the whole module: + if ( + ismodule + and stmts + and self.type_ignores + and min(self.type_ignores) < self.get_lineno(stmts[0]) + ): + ignores = self.type_ignores[min(self.type_ignores)] + if ignores: + joined_ignores = ", ".join(ignores) + self.fail( + message_registry.TYPE_IGNORE_WITH_ERRCODE_ON_MODULE.format(joined_ignores), + line=min(self.type_ignores), + column=0, + blocker=False, + ) + self.errors.used_ignored_lines[self.errors.file][min(self.type_ignores)].append( + codes.FILE.code + ) + block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) + self.set_block_lines(block, stmts) + mark_block_unreachable(block) + return [block] + + stack = self.class_and_function_stack + # Fast case for stripping function bodies + if ( + can_strip + and self.strip_function_bodies + and len(stack) == 1 + and stack[0] == "F" + and not is_coroutine + ): + return [] + + res: list[Statement] = [] + for stmt in stmts: + node = self.visit(stmt) + res.append(node) + + # Slow case for stripping function bodies + if can_strip and self.strip_function_bodies: + if stack[-2:] == ["C", "F"]: + if is_possible_trivial_body(res): + can_strip = False + else: + # We only strip method bodies if they don't assign to an attribute, as + # this may define an attribute which has an externally visible effect. + visitor = FindAttributeAssign() + for s in res: + s.accept(visitor) + if visitor.found: + can_strip = False + break + + if can_strip and stack[-1] == "F" and is_coroutine: + # Yields inside an async function affect the return type and should not + # be stripped. + yield_visitor = FindYield() + for s in res: + s.accept(yield_visitor) + if yield_visitor.found: + can_strip = False + break + + if can_strip: + return [] + return res + + def translate_type_comment( + self, n: ast3.stmt | ast3.arg, type_comment: str | None + ) -> ProperType | None: + if type_comment is None: + return None + else: + lineno = n.lineno + extra_ignore, typ = parse_type_comment(type_comment, lineno, n.col_offset, self.errors) + if extra_ignore is not None: + self.type_ignores[lineno] = extra_ignore + return typ + + op_map: Final[dict[type[AST], str]] = { + ast3.Add: "+", + ast3.Sub: "-", + ast3.Mult: "*", + ast3.MatMult: "@", + ast3.Div: "/", + ast3.Mod: "%", + ast3.Pow: "**", + ast3.LShift: "<<", + ast3.RShift: ">>", + ast3.BitOr: "|", + ast3.BitXor: "^", + ast3.BitAnd: "&", + ast3.FloorDiv: "//", + } + + def from_operator(self, op: ast3.operator) -> str: + op_name = ASTConverter.op_map.get(type(op)) + if op_name is None: + raise RuntimeError("Unknown operator " + str(type(op))) + else: + return op_name + + comp_op_map: Final[dict[type[AST], str]] = { + ast3.Gt: ">", + ast3.Lt: "<", + ast3.Eq: "==", + ast3.GtE: ">=", + ast3.LtE: "<=", + ast3.NotEq: "!=", + ast3.Is: "is", + ast3.IsNot: "is not", + ast3.In: "in", + ast3.NotIn: "not in", + } + + def from_comp_operator(self, op: ast3.cmpop) -> str: + op_name = ASTConverter.comp_op_map.get(type(op)) + if op_name is None: + raise RuntimeError("Unknown comparison operator " + str(type(op))) + else: + return op_name + + def set_block_lines(self, b: Block, stmts: Sequence[ast3.stmt]) -> None: + first, last = stmts[0], stmts[-1] + b.line = first.lineno + b.column = first.col_offset + b.end_line = getattr(last, "end_lineno", None) + b.end_column = getattr(last, "end_col_offset", None) + if not b.body: + return + new_first = b.body[0] + if isinstance(new_first, (Decorator, OverloadedFuncDef)): + # Decorated function lines are different between Python versions. + # copy the normalization we do for them to block first lines. + b.line = new_first.line + b.column = new_first.column + + def as_block(self, stmts: list[ast3.stmt]) -> Block | None: + b = None + if stmts: + b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) + self.set_block_lines(b, stmts) + return b + + def as_required_block( + self, stmts: list[ast3.stmt], *, can_strip: bool = False, is_coroutine: bool = False + ) -> Block: + assert stmts # must be non-empty + b = Block( + self.fix_function_overloads( + self.translate_stmt_list(stmts, can_strip=can_strip, is_coroutine=is_coroutine) + ) + ) + self.set_block_lines(b, stmts) + return b + + def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]: + ret: list[Statement] = [] + current_overload: list[OverloadPart] = [] + current_overload_name: str | None = None + seen_unconditional_func_def = False + last_if_stmt: IfStmt | None = None + last_if_overload: Decorator | FuncDef | OverloadedFuncDef | None = None + last_if_stmt_overload_name: str | None = None + last_if_unknown_truth_value: IfStmt | None = None + skipped_if_stmts: list[IfStmt] = [] + for stmt in stmts: + if_overload_name: str | None = None + if_block_with_overload: Block | None = None + if_unknown_truth_value: IfStmt | None = None + if isinstance(stmt, IfStmt) and seen_unconditional_func_def is False: + # Check IfStmt block to determine if function overloads can be merged + if_overload_name = self._check_ifstmt_for_overloads(stmt, current_overload_name) + if if_overload_name is not None: + (if_block_with_overload, if_unknown_truth_value) = ( + self._get_executable_if_block_with_overloads(stmt) + ) + + if ( + current_overload_name is not None + and isinstance(stmt, (Decorator, FuncDef)) + and stmt.name == current_overload_name + ): + if last_if_stmt is not None: + skipped_if_stmts.append(last_if_stmt) + if last_if_overload is not None: + # Last stmt was an IfStmt with same overload name + # Add overloads to current_overload + if isinstance(last_if_overload, OverloadedFuncDef): + current_overload.extend(last_if_overload.items) + else: + current_overload.append(last_if_overload) + last_if_stmt, last_if_overload = None, None + if last_if_unknown_truth_value: + self.fail_merge_overload(last_if_unknown_truth_value) + last_if_unknown_truth_value = None + current_overload.append(stmt) + if isinstance(stmt, FuncDef): + seen_unconditional_func_def = True + elif ( + current_overload_name is not None + and isinstance(stmt, IfStmt) + and if_overload_name == current_overload_name + ): + # IfStmt only contains stmts relevant to current_overload. + # Check if stmts are reachable and add them to current_overload, + # otherwise skip IfStmt to allow subsequent overload + # or function definitions. + skipped_if_stmts.append(stmt) + if if_block_with_overload is None: + if if_unknown_truth_value is not None: + self.fail_merge_overload(if_unknown_truth_value) + continue + if last_if_overload is not None: + # Last stmt was an IfStmt with same overload name + # Add overloads to current_overload + if isinstance(last_if_overload, OverloadedFuncDef): + current_overload.extend(last_if_overload.items) + else: + current_overload.append(last_if_overload) + last_if_stmt, last_if_overload = None, None + if isinstance(if_block_with_overload.body[-1], OverloadedFuncDef): + skipped_if_stmts.extend(cast(List[IfStmt], if_block_with_overload.body[:-1])) + current_overload.extend(if_block_with_overload.body[-1].items) + else: + current_overload.append( + cast(Union[Decorator, FuncDef], if_block_with_overload.body[0]) + ) + else: + if last_if_stmt is not None: + ret.append(last_if_stmt) + last_if_stmt_overload_name = current_overload_name + last_if_stmt, last_if_overload = None, None + last_if_unknown_truth_value = None + + if current_overload and current_overload_name == last_if_stmt_overload_name: + # Remove last stmt (IfStmt) from ret if the overload names matched + # Only happens if no executable block had been found in IfStmt + popped = ret.pop() + assert isinstance(popped, IfStmt) + skipped_if_stmts.append(popped) + if current_overload and skipped_if_stmts: + # Add bare IfStmt (without overloads) to ret + # Required for mypy to be able to still check conditions + for if_stmt in skipped_if_stmts: + self._strip_contents_from_if_stmt(if_stmt) + ret.append(if_stmt) + skipped_if_stmts = [] + if len(current_overload) == 1: + ret.append(current_overload[0]) + elif len(current_overload) > 1: + ret.append(OverloadedFuncDef(current_overload)) + + # If we have multiple decorated functions named "_" next to each, we want to treat + # them as a series of regular FuncDefs instead of one OverloadedFuncDef because + # most of mypy/mypyc assumes that all the functions in an OverloadedFuncDef are + # related, but multiple underscore functions next to each other aren't necessarily + # related + seen_unconditional_func_def = False + if isinstance(stmt, Decorator) and not unnamed_function(stmt.name): + current_overload = [stmt] + current_overload_name = stmt.name + elif isinstance(stmt, IfStmt) and if_overload_name is not None: + current_overload = [] + current_overload_name = if_overload_name + last_if_stmt = stmt + last_if_stmt_overload_name = None + if if_block_with_overload is not None: + skipped_if_stmts.extend( + cast(List[IfStmt], if_block_with_overload.body[:-1]) + ) + last_if_overload = cast( + Union[Decorator, FuncDef, OverloadedFuncDef], + if_block_with_overload.body[-1], + ) + last_if_unknown_truth_value = if_unknown_truth_value + else: + current_overload = [] + current_overload_name = None + ret.append(stmt) + + if current_overload and skipped_if_stmts: + # Add bare IfStmt (without overloads) to ret + # Required for mypy to be able to still check conditions + for if_stmt in skipped_if_stmts: + self._strip_contents_from_if_stmt(if_stmt) + ret.append(if_stmt) + if len(current_overload) == 1: + ret.append(current_overload[0]) + elif len(current_overload) > 1: + ret.append(OverloadedFuncDef(current_overload)) + elif last_if_overload is not None: + ret.append(last_if_overload) + elif last_if_stmt is not None: + ret.append(last_if_stmt) + return ret + + def _check_ifstmt_for_overloads( + self, stmt: IfStmt, current_overload_name: str | None = None + ) -> str | None: + """Check if IfStmt contains only overloads with the same name. + Return overload_name if found, None otherwise. + """ + # Check that block only contains a single Decorator, FuncDef, or OverloadedFuncDef. + # Multiple overloads have already been merged as OverloadedFuncDef. + if not ( + len(stmt.body[0].body) == 1 + and ( + isinstance(stmt.body[0].body[0], (Decorator, OverloadedFuncDef)) + or current_overload_name is not None + and isinstance(stmt.body[0].body[0], FuncDef) + ) + or len(stmt.body[0].body) > 1 + and isinstance(stmt.body[0].body[-1], OverloadedFuncDef) + and all(self._is_stripped_if_stmt(if_stmt) for if_stmt in stmt.body[0].body[:-1]) + ): + return None + + overload_name = cast( + Union[Decorator, FuncDef, OverloadedFuncDef], stmt.body[0].body[-1] + ).name + if stmt.else_body is None: + return overload_name + + if len(stmt.else_body.body) == 1: + # For elif: else_body contains an IfStmt itself -> do a recursive check. + if ( + isinstance(stmt.else_body.body[0], (Decorator, FuncDef, OverloadedFuncDef)) + and stmt.else_body.body[0].name == overload_name + ): + return overload_name + if ( + isinstance(stmt.else_body.body[0], IfStmt) + and self._check_ifstmt_for_overloads(stmt.else_body.body[0], current_overload_name) + == overload_name + ): + return overload_name + + return None + + def _get_executable_if_block_with_overloads( + self, stmt: IfStmt + ) -> tuple[Block | None, IfStmt | None]: + """Return block from IfStmt that will get executed. + + Return + 0 -> A block if sure that alternative blocks are unreachable. + 1 -> An IfStmt if the reachability of it can't be inferred, + i.e. the truth value is unknown. + """ + infer_reachability_of_if_statement(stmt, self.options) + if stmt.else_body is None and stmt.body[0].is_unreachable is True: + # always False condition with no else + return None, None + if ( + stmt.else_body is None + or stmt.body[0].is_unreachable is False + and stmt.else_body.is_unreachable is False + ): + # The truth value is unknown, thus not conclusive + return None, stmt + if stmt.else_body.is_unreachable is True: + # else_body will be set unreachable if condition is always True + return stmt.body[0], None + if stmt.body[0].is_unreachable is True: + # body will be set unreachable if condition is always False + # else_body can contain an IfStmt itself (for elif) -> do a recursive check + if isinstance(stmt.else_body.body[0], IfStmt): + return self._get_executable_if_block_with_overloads(stmt.else_body.body[0]) + return stmt.else_body, None + return None, stmt + + def _strip_contents_from_if_stmt(self, stmt: IfStmt) -> None: + """Remove contents from IfStmt. + + Needed to still be able to check the conditions after the contents + have been merged with the surrounding function overloads. + """ + if len(stmt.body) == 1: + stmt.body[0].body = [] + if stmt.else_body and len(stmt.else_body.body) == 1: + if isinstance(stmt.else_body.body[0], IfStmt): + self._strip_contents_from_if_stmt(stmt.else_body.body[0]) + else: + stmt.else_body.body = [] + + def _is_stripped_if_stmt(self, stmt: Statement) -> bool: + """Check stmt to make sure it is a stripped IfStmt. + + See also: _strip_contents_from_if_stmt + """ + if not isinstance(stmt, IfStmt): + return False + + if not (len(stmt.body) == 1 and len(stmt.body[0].body) == 0): + # Body not empty + return False + + if not stmt.else_body or len(stmt.else_body.body) == 0: + # No or empty else_body + return True + + # For elif, IfStmt are stored recursively in else_body + return self._is_stripped_if_stmt(stmt.else_body.body[0]) + + def translate_module_id(self, id: str) -> str: + """Return the actual, internal module id for a source text id.""" + if id == self.options.custom_typing_module: + return "typing" + return id + + def visit_Module(self, mod: ast3.Module) -> MypyFile: + self.type_ignores = {} + for ti in mod.type_ignores: + parsed = parse_type_ignore_tag(ti.tag) + if parsed is not None: + self.type_ignores[ti.lineno] = parsed + else: + self.fail(message_registry.INVALID_TYPE_IGNORE, ti.lineno, -1, blocker=False) + + body = self.fix_function_overloads(self.translate_stmt_list(mod.body, ismodule=True)) + + ret = MypyFile(body, self.imports, False, ignored_lines=self.type_ignores) + ret.is_stub = self.is_stub + ret.path = self.path + return ret + + # --- stmt --- + # FunctionDef(identifier name, arguments args, + # stmt* body, expr* decorator_list, expr? returns, string? type_comment) + # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, + # arg? kwarg, expr* defaults) + def visit_FunctionDef(self, n: ast3.FunctionDef) -> FuncDef | Decorator: + return self.do_func_def(n) + + # AsyncFunctionDef(identifier name, arguments args, + # stmt* body, expr* decorator_list, expr? returns, string? type_comment) + def visit_AsyncFunctionDef(self, n: ast3.AsyncFunctionDef) -> FuncDef | Decorator: + return self.do_func_def(n, is_coroutine=True) + + def do_func_def( + self, n: ast3.FunctionDef | ast3.AsyncFunctionDef, is_coroutine: bool = False + ) -> FuncDef | Decorator: + """Helper shared between visit_FunctionDef and visit_AsyncFunctionDef.""" + self.class_and_function_stack.append("D") + no_type_check = bool( + n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list) + ) + + lineno = n.lineno + args = self.transform_args(n.args, lineno, no_type_check=no_type_check) + if special_function_elide_names(n.name): + for arg in args: + arg.pos_only = True + + arg_kinds = [arg.kind for arg in args] + arg_names = [None if arg.pos_only else arg.variable.name for arg in args] + # Type parameters, if using new syntax for generics (PEP 695) + explicit_type_params: list[TypeParam] | None = None + + arg_types: list[Type | None] = [] + if no_type_check: + arg_types = [None] * len(args) + return_type = None + elif n.type_comment is not None: + try: + func_type_ast = ast3_parse(n.type_comment, "", "func_type") + assert isinstance(func_type_ast, FunctionType) + # for ellipsis arg + if ( + len(func_type_ast.argtypes) == 1 + and isinstance(func_type_ast.argtypes[0], Constant) + and func_type_ast.argtypes[0].value is Ellipsis + ): + if n.returns: + # PEP 484 disallows both type annotations and type comments + self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) + arg_types = [ + ( + a.type_annotation + if a.type_annotation is not None + else AnyType(TypeOfAny.unannotated) + ) + for a in args + ] + else: + # PEP 484 disallows both type annotations and type comments + if n.returns or any(a.type_annotation is not None for a in args): + self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) + translated_args: list[Type] = TypeConverter( + self.errors, line=lineno, override_column=n.col_offset + ).translate_expr_list(func_type_ast.argtypes) + # Use a cast to work around `list` invariance + arg_types = cast(List[Optional[Type]], translated_args) + return_type = TypeConverter(self.errors, line=lineno).visit(func_type_ast.returns) + + # add implicit self type + in_method_scope = self.class_and_function_stack[-2:] == ["C", "D"] + if in_method_scope and len(arg_types) < len(args): + arg_types.insert(0, AnyType(TypeOfAny.special_form)) + except SyntaxError: + stripped_type = n.type_comment.split("#", 2)[0].strip() + err_msg = message_registry.TYPE_COMMENT_SYNTAX_ERROR_VALUE.format(stripped_type) + self.fail(err_msg, lineno, n.col_offset) + if n.type_comment and n.type_comment[0] not in ["(", "#"]: + self.note( + "Suggestion: wrap argument types in parentheses", lineno, n.col_offset + ) + arg_types = [AnyType(TypeOfAny.from_error)] * len(args) + return_type = AnyType(TypeOfAny.from_error) + else: + if sys.version_info >= (3, 12) and n.type_params: + explicit_type_params = self.translate_type_params(n.type_params) + + arg_types = [a.type_annotation for a in args] + return_type = TypeConverter( + self.errors, line=n.returns.lineno if n.returns else lineno + ).visit(n.returns) + + for arg, arg_type in zip(args, arg_types): + self.set_type_optional(arg_type, arg.initializer) + + func_type = None + if any(arg_types) or return_type: + if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types): + self.fail(message_registry.ELLIPSIS_WITH_OTHER_TYPEARGS, lineno, n.col_offset) + elif len(arg_types) > len(arg_kinds): + self.fail( + message_registry.TYPE_SIGNATURE_TOO_MANY_ARGS, + lineno, + n.col_offset, + blocker=False, + ) + elif len(arg_types) < len(arg_kinds): + self.fail( + message_registry.TYPE_SIGNATURE_TOO_FEW_ARGS, + lineno, + n.col_offset, + blocker=False, + ) + else: + func_type = CallableType( + [a if a is not None else AnyType(TypeOfAny.unannotated) for a in arg_types], + arg_kinds, + arg_names, + return_type if return_type is not None else AnyType(TypeOfAny.unannotated), + _dummy_fallback, + ) + + # End position is always the same. + end_line = getattr(n, "end_lineno", None) + end_column = getattr(n, "end_col_offset", None) + + self.class_and_function_stack.pop() + self.class_and_function_stack.append("F") + body = self.as_required_block(n.body, can_strip=True, is_coroutine=is_coroutine) + func_def = FuncDef(n.name, args, body, func_type, explicit_type_params) + if isinstance(func_def.type, CallableType): + # semanal.py does some in-place modifications we want to avoid + func_def.unanalyzed_type = func_def.type.copy_modified() + if is_coroutine: + func_def.is_coroutine = True + if func_type is not None: + func_type.definition = func_def + func_type.line = lineno + + if n.decorator_list: + # Set deco_line to the old pre-3.8 lineno, in order to keep + # existing "# type: ignore" comments working: + deco_line = n.decorator_list[0].lineno + + var = Var(func_def.name) + var.is_ready = False + var.set_line(lineno) + + func_def.is_decorated = True + func_def.deco_line = deco_line + func_def.set_line(lineno, n.col_offset, end_line, end_column) + + deco = Decorator(func_def, self.translate_expr_list(n.decorator_list), var) + first = n.decorator_list[0] + deco.set_line(first.lineno, first.col_offset, end_line, end_column) + retval: FuncDef | Decorator = deco + else: + # FuncDef overrides set_line -- can't use self.set_line + func_def.set_line(lineno, n.col_offset, end_line, end_column) + retval = func_def + if self.options.include_docstrings: + func_def.docstring = ast3.get_docstring(n, clean=False) + self.class_and_function_stack.pop() + return retval + + def set_type_optional(self, type: Type | None, initializer: Expression | None) -> None: + if not self.options.implicit_optional: + return + # Indicate that type should be wrapped in an Optional if arg is initialized to None. + optional = isinstance(initializer, NameExpr) and initializer.name == "None" + if isinstance(type, UnboundType): + type.optional = optional + + def transform_args( + self, args: ast3.arguments, line: int, no_type_check: bool = False + ) -> list[Argument]: + new_args = [] + names: list[ast3.arg] = [] + posonlyargs = getattr(args, "posonlyargs", cast(List[ast3.arg], [])) + args_args = posonlyargs + args.args + args_defaults = args.defaults + num_no_defaults = len(args_args) - len(args_defaults) + # positional arguments without defaults + for i, a in enumerate(args_args[:num_no_defaults]): + pos_only = i < len(posonlyargs) + new_args.append(self.make_argument(a, None, ARG_POS, no_type_check, pos_only)) + names.append(a) + + # positional arguments with defaults + for i, (a, d) in enumerate(zip(args_args[num_no_defaults:], args_defaults)): + pos_only = num_no_defaults + i < len(posonlyargs) + new_args.append(self.make_argument(a, d, ARG_OPT, no_type_check, pos_only)) + names.append(a) + + # *arg + if args.vararg is not None: + new_args.append(self.make_argument(args.vararg, None, ARG_STAR, no_type_check)) + names.append(args.vararg) + + # keyword-only arguments with defaults + for a, kd in zip(args.kwonlyargs, args.kw_defaults): + new_args.append( + self.make_argument( + a, kd, ARG_NAMED if kd is None else ARG_NAMED_OPT, no_type_check + ) + ) + names.append(a) + + # **kwarg + if args.kwarg is not None: + new_args.append(self.make_argument(args.kwarg, None, ARG_STAR2, no_type_check)) + names.append(args.kwarg) + + check_arg_names([arg.variable.name for arg in new_args], names, self.fail_arg) + + return new_args + + def make_argument( + self, + arg: ast3.arg, + default: ast3.expr | None, + kind: ArgKind, + no_type_check: bool, + pos_only: bool = False, + ) -> Argument: + if no_type_check: + arg_type = None + else: + annotation = arg.annotation + type_comment = arg.type_comment + if annotation is not None and type_comment is not None: + self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, arg.lineno, arg.col_offset) + arg_type = None + if annotation is not None: + arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation) + else: + arg_type = self.translate_type_comment(arg, type_comment) + if argument_elide_name(arg.arg): + pos_only = True + + argument = Argument(Var(arg.arg, arg_type), arg_type, self.visit(default), kind, pos_only) + argument.set_line( + arg.lineno, + arg.col_offset, + getattr(arg, "end_lineno", None), + getattr(arg, "end_col_offset", None), + ) + return argument + + def fail_arg(self, msg: str, arg: ast3.arg) -> None: + self.fail(ErrorMessage(msg), arg.lineno, arg.col_offset) + + # ClassDef(identifier name, + # expr* bases, + # keyword* keywords, + # stmt* body, + # expr* decorator_list) + def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: + self.class_and_function_stack.append("C") + keywords = [(kw.arg, self.visit(kw.value)) for kw in n.keywords if kw.arg] + + # Type parameters, if using new syntax for generics (PEP 695) + explicit_type_params: list[TypeParam] | None = None + + if sys.version_info >= (3, 12) and n.type_params: + explicit_type_params = self.translate_type_params(n.type_params) + + cdef = ClassDef( + n.name, + self.as_required_block(n.body), + None, + self.translate_expr_list(n.bases), + metaclass=dict(keywords).get("metaclass"), + keywords=keywords, + type_args=explicit_type_params, + ) + cdef.decorators = self.translate_expr_list(n.decorator_list) + # Set lines to match the old mypy 0.700 lines, in order to keep + # existing "# type: ignore" comments working: + cdef.line = n.lineno + cdef.deco_line = n.decorator_list[0].lineno if n.decorator_list else None + + if self.options.include_docstrings: + cdef.docstring = ast3.get_docstring(n, clean=False) + cdef.column = n.col_offset + cdef.end_line = getattr(n, "end_lineno", None) + cdef.end_column = getattr(n, "end_col_offset", None) + self.class_and_function_stack.pop() + return cdef + + def validate_type_param(self, type_param: ast_TypeVar) -> None: + incorrect_expr = find_disallowed_expression_in_annotation_scope(type_param.bound) + if incorrect_expr is None: + return + if isinstance(incorrect_expr, (ast3.Yield, ast3.YieldFrom)): + self.fail( + message_registry.TYPE_VAR_YIELD_EXPRESSION_IN_BOUND, + type_param.lineno, + type_param.col_offset, + ) + if isinstance(incorrect_expr, ast3.NamedExpr): + self.fail( + message_registry.TYPE_VAR_NAMED_EXPRESSION_IN_BOUND, + type_param.lineno, + type_param.col_offset, + ) + if isinstance(incorrect_expr, ast3.Await): + self.fail( + message_registry.TYPE_VAR_AWAIT_EXPRESSION_IN_BOUND, + type_param.lineno, + type_param.col_offset, + ) + + def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: + explicit_type_params = [] + for p in type_params: + bound: Type | None = None + values: list[Type] = [] + default: Type | None = None + if sys.version_info >= (3, 13): + default = TypeConverter(self.errors, line=p.lineno).visit(p.default_value) + if isinstance(p, ast_ParamSpec): # type: ignore[misc] + explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, [], default)) + elif isinstance(p, ast_TypeVarTuple): # type: ignore[misc] + explicit_type_params.append( + TypeParam(p.name, TYPE_VAR_TUPLE_KIND, None, [], default) + ) + else: + if isinstance(p.bound, ast3.Tuple): + if len(p.bound.elts) < 2: + self.fail( + message_registry.TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES, + p.lineno, + p.col_offset, + blocker=False, + ) + else: + conv = TypeConverter(self.errors, line=p.lineno) + values = [conv.visit(t) for t in p.bound.elts] + elif p.bound is not None: + self.validate_type_param(p) + bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) + explicit_type_params.append( + TypeParam(p.name, TYPE_VAR_KIND, bound, values, default) + ) + return explicit_type_params + + # Return(expr? value) + def visit_Return(self, n: ast3.Return) -> ReturnStmt: + node = ReturnStmt(self.visit(n.value)) + return self.set_line(node, n) + + # Delete(expr* targets) + def visit_Delete(self, n: ast3.Delete) -> DelStmt: + if len(n.targets) > 1: + tup = TupleExpr(self.translate_expr_list(n.targets)) + tup.set_line(n.lineno) + node = DelStmt(tup) + else: + node = DelStmt(self.visit(n.targets[0])) + return self.set_line(node, n) + + # Assign(expr* targets, expr? value, string? type_comment, expr? annotation) + def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt: + lvalues = self.translate_expr_list(n.targets) + rvalue = self.visit(n.value) + typ = self.translate_type_comment(n, n.type_comment) + s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False) + return self.set_line(s, n) + + # AnnAssign(expr target, expr annotation, expr? value, int simple) + def visit_AnnAssign(self, n: ast3.AnnAssign) -> AssignmentStmt: + line = n.lineno + if n.value is None: # always allow 'x: int' + rvalue: Expression = TempNode(AnyType(TypeOfAny.special_form), no_rhs=True) + rvalue.line = line + rvalue.column = n.col_offset + else: + rvalue = self.visit(n.value) + typ = TypeConverter(self.errors, line=line).visit(n.annotation) + assert typ is not None + typ.column = n.annotation.col_offset + s = AssignmentStmt([self.visit(n.target)], rvalue, type=typ, new_syntax=True) + return self.set_line(s, n) + + # AugAssign(expr target, operator op, expr value) + def visit_AugAssign(self, n: ast3.AugAssign) -> OperatorAssignmentStmt: + s = OperatorAssignmentStmt( + self.from_operator(n.op), self.visit(n.target), self.visit(n.value) + ) + return self.set_line(s, n) + + # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + def visit_For(self, n: ast3.For) -> ForStmt: + target_type = self.translate_type_comment(n, n.type_comment) + node = ForStmt( + self.visit(n.target), + self.visit(n.iter), + self.as_required_block(n.body), + self.as_block(n.orelse), + target_type, + ) + return self.set_line(node, n) + + # AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt: + target_type = self.translate_type_comment(n, n.type_comment) + node = ForStmt( + self.visit(n.target), + self.visit(n.iter), + self.as_required_block(n.body), + self.as_block(n.orelse), + target_type, + ) + node.is_async = True + return self.set_line(node, n) + + # While(expr test, stmt* body, stmt* orelse) + def visit_While(self, n: ast3.While) -> WhileStmt: + node = WhileStmt( + self.visit(n.test), self.as_required_block(n.body), self.as_block(n.orelse) + ) + return self.set_line(node, n) + + # If(expr test, stmt* body, stmt* orelse) + def visit_If(self, n: ast3.If) -> IfStmt: + node = IfStmt( + [self.visit(n.test)], [self.as_required_block(n.body)], self.as_block(n.orelse) + ) + return self.set_line(node, n) + + # With(withitem* items, stmt* body, string? type_comment) + def visit_With(self, n: ast3.With) -> WithStmt: + target_type = self.translate_type_comment(n, n.type_comment) + node = WithStmt( + [self.visit(i.context_expr) for i in n.items], + [self.visit(i.optional_vars) for i in n.items], + self.as_required_block(n.body), + target_type, + ) + return self.set_line(node, n) + + # AsyncWith(withitem* items, stmt* body, string? type_comment) + def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt: + target_type = self.translate_type_comment(n, n.type_comment) + s = WithStmt( + [self.visit(i.context_expr) for i in n.items], + [self.visit(i.optional_vars) for i in n.items], + self.as_required_block(n.body), + target_type, + ) + s.is_async = True + return self.set_line(s, n) + + # Raise(expr? exc, expr? cause) + def visit_Raise(self, n: ast3.Raise) -> RaiseStmt: + node = RaiseStmt(self.visit(n.exc), self.visit(n.cause)) + return self.set_line(node, n) + + # Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + def visit_Try(self, n: ast3.Try) -> TryStmt: + vs = [ + self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers + ] + types = [self.visit(h.type) for h in n.handlers] + handlers = [self.as_required_block(h.body) for h in n.handlers] + + node = TryStmt( + self.as_required_block(n.body), + vs, + types, + handlers, + self.as_block(n.orelse), + self.as_block(n.finalbody), + ) + return self.set_line(node, n) + + def visit_TryStar(self, n: TryStar) -> TryStmt: + vs = [ + self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers + ] + types = [self.visit(h.type) for h in n.handlers] + handlers = [self.as_required_block(h.body) for h in n.handlers] + + node = TryStmt( + self.as_required_block(n.body), + vs, + types, + handlers, + self.as_block(n.orelse), + self.as_block(n.finalbody), + ) + node.is_star = True + return self.set_line(node, n) + + # Assert(expr test, expr? msg) + def visit_Assert(self, n: ast3.Assert) -> AssertStmt: + node = AssertStmt(self.visit(n.test), self.visit(n.msg)) + return self.set_line(node, n) + + # Import(alias* names) + def visit_Import(self, n: ast3.Import) -> Import: + names: list[tuple[str, str | None]] = [] + for alias in n.names: + name = self.translate_module_id(alias.name) + asname = alias.asname + if asname is None and name != alias.name: + # if the module name has been translated (and it's not already + # an explicit import-as), make it an implicit import-as the + # original name + asname = alias.name + names.append((name, asname)) + i = Import(names) + self.imports.append(i) + return self.set_line(i, n) + + # ImportFrom(identifier? module, alias* names, int? level) + def visit_ImportFrom(self, n: ast3.ImportFrom) -> ImportBase: + assert n.level is not None + if len(n.names) == 1 and n.names[0].name == "*": + mod = n.module if n.module is not None else "" + i: ImportBase = ImportAll(mod, n.level) + else: + i = ImportFrom( + self.translate_module_id(n.module) if n.module is not None else "", + n.level, + [(a.name, a.asname) for a in n.names], + ) + self.imports.append(i) + return self.set_line(i, n) + + # Global(identifier* names) + def visit_Global(self, n: ast3.Global) -> GlobalDecl: + g = GlobalDecl(n.names) + return self.set_line(g, n) + + # Nonlocal(identifier* names) + def visit_Nonlocal(self, n: ast3.Nonlocal) -> NonlocalDecl: + d = NonlocalDecl(n.names) + return self.set_line(d, n) + + # Expr(expr value) + def visit_Expr(self, n: ast3.Expr) -> ExpressionStmt: + value = self.visit(n.value) + node = ExpressionStmt(value) + return self.set_line(node, n) + + # Pass + def visit_Pass(self, n: ast3.Pass) -> PassStmt: + s = PassStmt() + return self.set_line(s, n) + + # Break + def visit_Break(self, n: ast3.Break) -> BreakStmt: + s = BreakStmt() + return self.set_line(s, n) + + # Continue + def visit_Continue(self, n: ast3.Continue) -> ContinueStmt: + s = ContinueStmt() + return self.set_line(s, n) + + # --- expr --- + + def visit_NamedExpr(self, n: NamedExpr) -> AssignmentExpr: + s = AssignmentExpr(self.visit(n.target), self.visit(n.value)) + return self.set_line(s, n) + + # BoolOp(boolop op, expr* values) + def visit_BoolOp(self, n: ast3.BoolOp) -> OpExpr: + # mypy translates (1 and 2 and 3) as (1 and (2 and 3)) + assert len(n.values) >= 2 + op_node = n.op + if isinstance(op_node, ast3.And): + op = "and" + elif isinstance(op_node, ast3.Or): + op = "or" + else: + raise RuntimeError("unknown BoolOp " + str(type(n))) + + # potentially inefficient! + return self.group(op, self.translate_expr_list(n.values), n) + + def group(self, op: str, vals: list[Expression], n: ast3.expr) -> OpExpr: + if len(vals) == 2: + e = OpExpr(op, vals[0], vals[1]) + else: + e = OpExpr(op, vals[0], self.group(op, vals[1:], n)) + return self.set_line(e, n) + + # BinOp(expr left, operator op, expr right) + def visit_BinOp(self, n: ast3.BinOp) -> OpExpr: + op = self.from_operator(n.op) + + if op is None: + raise RuntimeError("cannot translate BinOp " + str(type(n.op))) + + e = OpExpr(op, self.visit(n.left), self.visit(n.right)) + return self.set_line(e, n) + + # UnaryOp(unaryop op, expr operand) + def visit_UnaryOp(self, n: ast3.UnaryOp) -> UnaryExpr: + op = None + if isinstance(n.op, ast3.Invert): + op = "~" + elif isinstance(n.op, ast3.Not): + op = "not" + elif isinstance(n.op, ast3.UAdd): + op = "+" + elif isinstance(n.op, ast3.USub): + op = "-" + + if op is None: + raise RuntimeError("cannot translate UnaryOp " + str(type(n.op))) + + e = UnaryExpr(op, self.visit(n.operand)) + return self.set_line(e, n) + + # Lambda(arguments args, expr body) + def visit_Lambda(self, n: ast3.Lambda) -> LambdaExpr: + body = ast3.Return(n.body) + body.lineno = n.body.lineno + body.col_offset = n.body.col_offset + + self.class_and_function_stack.append("L") + e = LambdaExpr(self.transform_args(n.args, n.lineno), self.as_required_block([body])) + self.class_and_function_stack.pop() + e.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line + return e + + # IfExp(expr test, expr body, expr orelse) + def visit_IfExp(self, n: ast3.IfExp) -> ConditionalExpr: + e = ConditionalExpr(self.visit(n.test), self.visit(n.body), self.visit(n.orelse)) + return self.set_line(e, n) + + # Dict(expr* keys, expr* values) + def visit_Dict(self, n: ast3.Dict) -> DictExpr: + e = DictExpr( + list(zip(self.translate_opt_expr_list(n.keys), self.translate_expr_list(n.values))) + ) + return self.set_line(e, n) + + # Set(expr* elts) + def visit_Set(self, n: ast3.Set) -> SetExpr: + e = SetExpr(self.translate_expr_list(n.elts)) + return self.set_line(e, n) + + # ListComp(expr elt, comprehension* generators) + def visit_ListComp(self, n: ast3.ListComp) -> ListComprehension: + e = ListComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n))) + return self.set_line(e, n) + + # SetComp(expr elt, comprehension* generators) + def visit_SetComp(self, n: ast3.SetComp) -> SetComprehension: + e = SetComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n))) + return self.set_line(e, n) + + # DictComp(expr key, expr value, comprehension* generators) + def visit_DictComp(self, n: ast3.DictComp) -> DictionaryComprehension: + targets = [self.visit(c.target) for c in n.generators] + iters = [self.visit(c.iter) for c in n.generators] + ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] + is_async = [bool(c.is_async) for c in n.generators] + e = DictionaryComprehension( + self.visit(n.key), self.visit(n.value), targets, iters, ifs_list, is_async + ) + return self.set_line(e, n) + + # GeneratorExp(expr elt, comprehension* generators) + def visit_GeneratorExp(self, n: ast3.GeneratorExp) -> GeneratorExpr: + targets = [self.visit(c.target) for c in n.generators] + iters = [self.visit(c.iter) for c in n.generators] + ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] + is_async = [bool(c.is_async) for c in n.generators] + e = GeneratorExpr(self.visit(n.elt), targets, iters, ifs_list, is_async) + return self.set_line(e, n) + + # Await(expr value) + def visit_Await(self, n: ast3.Await) -> AwaitExpr: + v = self.visit(n.value) + e = AwaitExpr(v) + return self.set_line(e, n) + + # Yield(expr? value) + def visit_Yield(self, n: ast3.Yield) -> YieldExpr: + e = YieldExpr(self.visit(n.value)) + return self.set_line(e, n) + + # YieldFrom(expr value) + def visit_YieldFrom(self, n: ast3.YieldFrom) -> YieldFromExpr: + e = YieldFromExpr(self.visit(n.value)) + return self.set_line(e, n) + + # Compare(expr left, cmpop* ops, expr* comparators) + def visit_Compare(self, n: ast3.Compare) -> ComparisonExpr: + operators = [self.from_comp_operator(o) for o in n.ops] + operands = self.translate_expr_list([n.left] + n.comparators) + e = ComparisonExpr(operators, operands) + return self.set_line(e, n) + + # Call(expr func, expr* args, keyword* keywords) + # keyword = (identifier? arg, expr value) + def visit_Call(self, n: Call) -> CallExpr: + args = n.args + keywords = n.keywords + keyword_names = [k.arg for k in keywords] + arg_types = self.translate_expr_list( + [a.value if isinstance(a, Starred) else a for a in args] + [k.value for k in keywords] + ) + arg_kinds = [ARG_STAR if type(a) is Starred else ARG_POS for a in args] + [ + ARG_STAR2 if arg is None else ARG_NAMED for arg in keyword_names + ] + e = CallExpr( + self.visit(n.func), + arg_types, + arg_kinds, + cast("List[Optional[str]]", [None] * len(args)) + keyword_names, + ) + return self.set_line(e, n) + + # Constant(object value) -- a constant, in Python 3.8. + def visit_Constant(self, n: Constant) -> Any: + val = n.value + e: Any = None + if val is None: + e = NameExpr("None") + elif isinstance(val, str): + e = StrExpr(val) + elif isinstance(val, bytes): + e = BytesExpr(bytes_to_human_readable_repr(val)) + elif isinstance(val, bool): # Must check before int! + e = NameExpr(str(val)) + elif isinstance(val, int): + e = IntExpr(val) + elif isinstance(val, float): + e = FloatExpr(val) + elif isinstance(val, complex): + e = ComplexExpr(val) + elif val is Ellipsis: + e = EllipsisExpr() + else: + raise RuntimeError("Constant not implemented for " + str(type(val))) + return self.set_line(e, n) + + # JoinedStr(expr* values) + def visit_JoinedStr(self, n: ast3.JoinedStr) -> Expression: + # Each of n.values is a str or FormattedValue; we just concatenate + # them all using ''.join. + empty_string = StrExpr("") + empty_string.set_line(n.lineno, n.col_offset) + strs_to_join = ListExpr(self.translate_expr_list(n.values)) + strs_to_join.set_line(empty_string) + # Don't make unnecessary join call if there is only one str to join + if len(strs_to_join.items) == 1: + return self.set_line(strs_to_join.items[0], n) + elif len(strs_to_join.items) > 1: + last = strs_to_join.items[-1] + if isinstance(last, StrExpr) and last.value == "": + # 3.12 can add an empty literal at the end. Delete it for consistency + # between Python versions. + del strs_to_join.items[-1:] + join_method = MemberExpr(empty_string, "join") + join_method.set_line(empty_string) + result_expression = CallExpr(join_method, [strs_to_join], [ARG_POS], [None]) + return self.set_line(result_expression, n) + + # FormattedValue(expr value) + def visit_FormattedValue(self, n: ast3.FormattedValue) -> Expression: + # A FormattedValue is a component of a JoinedStr, or it can exist + # on its own. We translate them to individual '{}'.format(value) + # calls. Format specifier and conversion information is passed along + # to allow mypyc to support f-strings with format specifiers and conversions. + val_exp = self.visit(n.value) + val_exp.set_line(n.lineno, n.col_offset) + conv_str = "" if n.conversion < 0 else "!" + chr(n.conversion) + format_string = StrExpr("{" + conv_str + ":{}}") + format_spec_exp = self.visit(n.format_spec) if n.format_spec is not None else StrExpr("") + format_string.set_line(n.lineno, n.col_offset) + format_method = MemberExpr(format_string, "format") + format_method.set_line(format_string) + result_expression = CallExpr( + format_method, [val_exp, format_spec_exp], [ARG_POS, ARG_POS], [None, None] + ) + return self.set_line(result_expression, n) + + # Attribute(expr value, identifier attr, expr_context ctx) + def visit_Attribute(self, n: Attribute) -> MemberExpr | SuperExpr: + value = n.value + member_expr = MemberExpr(self.visit(value), n.attr) + obj = member_expr.expr + if ( + isinstance(obj, CallExpr) + and isinstance(obj.callee, NameExpr) + and obj.callee.name == "super" + ): + e: MemberExpr | SuperExpr = SuperExpr(member_expr.name, obj) + else: + e = member_expr + return self.set_line(e, n) + + # Subscript(expr value, slice slice, expr_context ctx) + def visit_Subscript(self, n: ast3.Subscript) -> IndexExpr: + e = IndexExpr(self.visit(n.value), self.visit(n.slice)) + self.set_line(e, n) + # alias to please mypyc + is_py38_or_earlier = sys.version_info < (3, 9) + if isinstance(n.slice, ast3.Slice) or ( + is_py38_or_earlier and isinstance(n.slice, ast3.ExtSlice) + ): + # Before Python 3.9, Slice has no line/column in the raw ast. To avoid incompatibility + # visit_Slice doesn't set_line, even in Python 3.9 on. + # ExtSlice also has no line/column info. In Python 3.9 on, line/column is set for + # e.index when visiting n.slice. + e.index.line = e.line + e.index.column = e.column + return e + + # Starred(expr value, expr_context ctx) + def visit_Starred(self, n: Starred) -> StarExpr: + e = StarExpr(self.visit(n.value)) + return self.set_line(e, n) + + # Name(identifier id, expr_context ctx) + def visit_Name(self, n: Name) -> NameExpr: + e = NameExpr(n.id) + return self.set_line(e, n) + + # List(expr* elts, expr_context ctx) + def visit_List(self, n: ast3.List) -> ListExpr | TupleExpr: + expr_list: list[Expression] = [self.visit(e) for e in n.elts] + if isinstance(n.ctx, ast3.Store): + # [x, y] = z and (x, y) = z means exactly the same thing + e: ListExpr | TupleExpr = TupleExpr(expr_list) + else: + e = ListExpr(expr_list) + return self.set_line(e, n) + + # Tuple(expr* elts, expr_context ctx) + def visit_Tuple(self, n: ast3.Tuple) -> TupleExpr: + e = TupleExpr(self.translate_expr_list(n.elts)) + return self.set_line(e, n) + + # --- slice --- + + # Slice(expr? lower, expr? upper, expr? step) + def visit_Slice(self, n: ast3.Slice) -> SliceExpr: + return SliceExpr(self.visit(n.lower), self.visit(n.upper), self.visit(n.step)) + + # ExtSlice(slice* dims) + def visit_ExtSlice(self, n: ast3.ExtSlice) -> TupleExpr: + # cast for mypyc's benefit on Python 3.9 + return TupleExpr(self.translate_expr_list(cast(Any, n).dims)) + + # Index(expr value) + def visit_Index(self, n: Index) -> Node: + # cast for mypyc's benefit on Python 3.9 + value = self.visit(cast(Any, n).value) + assert isinstance(value, Node) + return value + + # Match(expr subject, match_case* cases) # python 3.10 and later + def visit_Match(self, n: Match) -> MatchStmt: + node = MatchStmt( + self.visit(n.subject), + [self.visit(c.pattern) for c in n.cases], + [self.visit(c.guard) for c in n.cases], + [self.as_required_block(c.body) for c in n.cases], + ) + return self.set_line(node, n) + + def visit_MatchValue(self, n: MatchValue) -> ValuePattern: + node = ValuePattern(self.visit(n.value)) + return self.set_line(node, n) + + def visit_MatchSingleton(self, n: MatchSingleton) -> SingletonPattern: + node = SingletonPattern(n.value) + return self.set_line(node, n) + + def visit_MatchSequence(self, n: MatchSequence) -> SequencePattern: + patterns = [self.visit(p) for p in n.patterns] + stars = [p for p in patterns if isinstance(p, StarredPattern)] + assert len(stars) < 2 + + node = SequencePattern(patterns) + return self.set_line(node, n) + + def visit_MatchStar(self, n: MatchStar) -> StarredPattern: + if n.name is None: + node = StarredPattern(None) + else: + name = self.set_line(NameExpr(n.name), n) + node = StarredPattern(name) + + return self.set_line(node, n) + + def visit_MatchMapping(self, n: MatchMapping) -> MappingPattern: + keys = [self.visit(k) for k in n.keys] + values = [self.visit(v) for v in n.patterns] + + if n.rest is None: + rest = None + else: + rest = NameExpr(n.rest) + + node = MappingPattern(keys, values, rest) + return self.set_line(node, n) + + def visit_MatchClass(self, n: MatchClass) -> ClassPattern: + class_ref = self.visit(n.cls) + assert isinstance(class_ref, RefExpr) + positionals = [self.visit(p) for p in n.patterns] + keyword_keys = n.kwd_attrs + keyword_values = [self.visit(p) for p in n.kwd_patterns] + + node = ClassPattern(class_ref, positionals, keyword_keys, keyword_values) + return self.set_line(node, n) + + # MatchAs(expr pattern, identifier name) + def visit_MatchAs(self, n: MatchAs) -> AsPattern: + if n.name is None: + name = None + else: + name = NameExpr(n.name) + name = self.set_line(name, n) + node = AsPattern(self.visit(n.pattern), name) + return self.set_line(node, n) + + # MatchOr(expr* pattern) + def visit_MatchOr(self, n: MatchOr) -> OrPattern: + node = OrPattern([self.visit(pattern) for pattern in n.patterns]) + return self.set_line(node, n) + + def validate_type_alias(self, n: ast_TypeAlias) -> None: + incorrect_expr = find_disallowed_expression_in_annotation_scope(n.value) + if incorrect_expr is None: + return + if isinstance(incorrect_expr, (ast3.Yield, ast3.YieldFrom)): + self.fail(message_registry.TYPE_ALIAS_WITH_YIELD_EXPRESSION, n.lineno, n.col_offset) + if isinstance(incorrect_expr, ast3.NamedExpr): + self.fail(message_registry.TYPE_ALIAS_WITH_NAMED_EXPRESSION, n.lineno, n.col_offset) + if isinstance(incorrect_expr, ast3.Await): + self.fail(message_registry.TYPE_ALIAS_WITH_AWAIT_EXPRESSION, n.lineno, n.col_offset) + + # TypeAlias(identifier name, type_param* type_params, expr value) + def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: + node: TypeAliasStmt | AssignmentStmt + type_params = self.translate_type_params(n.type_params) + self.validate_type_alias(n) + value = self.visit(n.value) + # Since the value is evaluated lazily, wrap the value inside a lambda. + # This helps mypyc. + ret = ReturnStmt(value) + self.set_line(ret, n.value) + value_func = LambdaExpr(body=Block([ret])) + self.set_line(value_func, n.value) + node = TypeAliasStmt(self.visit_Name(n.name), type_params, value_func) + return self.set_line(node, n) + + +class TypeConverter: + def __init__( + self, + errors: Errors | None, + line: int = -1, + override_column: int = -1, + is_evaluated: bool = True, + ) -> None: + self.errors = errors + self.line = line + self.override_column = override_column + self.node_stack: list[AST] = [] + self.is_evaluated = is_evaluated + + def convert_column(self, column: int) -> int: + """Apply column override if defined; otherwise return column. + + Column numbers are sometimes incorrect in the AST and the column + override can be used to work around that. + """ + if self.override_column < 0: + return column + else: + return self.override_column + + def invalid_type(self, node: AST, note: str | None = None) -> RawExpressionType: + """Constructs a type representing some expression that normally forms an invalid type. + For example, if we see a type hint that says "3 + 4", we would transform that + expression into a RawExpressionType. + + The semantic analysis layer will report an "Invalid type" error when it + encounters this type, along with the given note if one is provided. + + See RawExpressionType's docstring for more details on how it's used. + """ + return RawExpressionType( + None, "typing.Any", line=self.line, column=getattr(node, "col_offset", -1), note=note + ) + + @overload + def visit(self, node: ast3.expr) -> ProperType: ... + + @overload + def visit(self, node: AST | None) -> ProperType | None: ... + + def visit(self, node: AST | None) -> ProperType | None: + """Modified visit -- keep track of the stack of nodes""" + if node is None: + return None + self.node_stack.append(node) + try: + method = "visit_" + node.__class__.__name__ + visitor = getattr(self, method, None) + if visitor is not None: + typ = visitor(node) + assert isinstance(typ, ProperType) + return typ + else: + return self.invalid_type(node) + finally: + self.node_stack.pop() + + def parent(self) -> AST | None: + """Return the AST node above the one we are processing""" + if len(self.node_stack) < 2: + return None + return self.node_stack[-2] + + def fail(self, msg: ErrorMessage, line: int, column: int) -> None: + if self.errors: + self.errors.report(line, column, msg.value, blocker=True, code=msg.code) + + def note(self, msg: str, line: int, column: int) -> None: + if self.errors: + self.errors.report(line, column, msg, severity="note", code=codes.SYNTAX) + + def translate_expr_list(self, l: Sequence[ast3.expr]) -> list[Type]: + return [self.visit(e) for e in l] + + def visit_Call(self, e: Call) -> Type: + # Parse the arg constructor + f = e.func + constructor = stringify_name(f) + + if not isinstance(self.parent(), ast3.List): + note = None + if constructor: + note = "Suggestion: use {0}[...] instead of {0}(...)".format(constructor) + return self.invalid_type(e, note=note) + if not constructor: + self.fail(message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED, e.lineno, e.col_offset) + + name: str | None = None + default_type = AnyType(TypeOfAny.special_form) + typ: Type = default_type + for i, arg in enumerate(e.args): + if i == 0: + converted = self.visit(arg) + assert converted is not None + typ = converted + elif i == 1: + name = self._extract_argument_name(arg) + else: + self.fail(message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS, f.lineno, f.col_offset) + for k in e.keywords: + value = k.value + if k.arg == "name": + if name is not None: + self.fail( + message_registry.MULTIPLE_VALUES_FOR_NAME_KWARG.format(constructor), + f.lineno, + f.col_offset, + ) + name = self._extract_argument_name(value) + elif k.arg == "type": + if typ is not default_type: + self.fail( + message_registry.MULTIPLE_VALUES_FOR_TYPE_KWARG.format(constructor), + f.lineno, + f.col_offset, + ) + converted = self.visit(value) + assert converted is not None + typ = converted + else: + self.fail( + message_registry.ARG_CONSTRUCTOR_UNEXPECTED_ARG.format(k.arg), + value.lineno, + value.col_offset, + ) + return CallableArgument(typ, name, constructor, e.lineno, e.col_offset) + + def translate_argument_list(self, l: Sequence[ast3.expr]) -> TypeList: + return TypeList([self.visit(e) for e in l], line=self.line) + + def _extract_argument_name(self, n: ast3.expr) -> str | None: + if isinstance(n, Constant) and isinstance(n.value, str): + return n.value.strip() + elif isinstance(n, Constant) and n.value is None: + return None + self.fail( + message_registry.ARG_NAME_EXPECTED_STRING_LITERAL.format(type(n).__name__), + self.line, + 0, + ) + return None + + def visit_Name(self, n: Name) -> Type: + return UnboundType(n.id, line=self.line, column=self.convert_column(n.col_offset)) + + def visit_BinOp(self, n: ast3.BinOp) -> Type: + if not isinstance(n.op, ast3.BitOr): + return self.invalid_type(n) + + left = self.visit(n.left) + right = self.visit(n.right) + return UnionType( + [left, right], + line=self.line, + column=self.convert_column(n.col_offset), + is_evaluated=self.is_evaluated, + uses_pep604_syntax=True, + ) + + def visit_Constant(self, n: Constant) -> Type: + val = n.value + if val is None: + # None is a type. + return UnboundType("None", line=self.line) + if isinstance(val, str): + # Parse forward reference. + return parse_type_string(val, "builtins.str", self.line, n.col_offset) + if val is Ellipsis: + # '...' is valid in some types. + return EllipsisType(line=self.line) + if isinstance(val, bool): + # Special case for True/False. + return RawExpressionType(val, "builtins.bool", line=self.line) + if isinstance(val, (int, float, complex)): + return self.numeric_type(val, n) + if isinstance(val, bytes): + contents = bytes_to_human_readable_repr(val) + return RawExpressionType(contents, "builtins.bytes", self.line, column=n.col_offset) + # Everything else is invalid. + return self.invalid_type(n) + + # UnaryOp(op, operand) + def visit_UnaryOp(self, n: UnaryOp) -> Type: + # We support specifically Literal[-4], Literal[+4], and nothing else. + # For example, Literal[~6] or Literal[not False] is not supported. + typ = self.visit(n.operand) + if ( + isinstance(typ, RawExpressionType) + # Use type() because we do not want to allow bools. + and type(typ.literal_value) is int + ): + if isinstance(n.op, USub): + typ.literal_value *= -1 + return typ + if isinstance(n.op, UAdd): + return typ + return self.invalid_type(n) + + def numeric_type(self, value: object, n: AST) -> Type: + # The node's field has the type complex, but complex isn't *really* + # a parent of int and float, and this causes isinstance below + # to think that the complex branch is always picked. Avoid + # this by throwing away the type. + if isinstance(value, int): + numeric_value: int | None = value + type_name = "builtins.int" + else: + # Other kinds of numbers (floats, complex) are not valid parameters for + # RawExpressionType so we just pass in 'None' for now. We'll report the + # appropriate error at a later stage. + numeric_value = None + type_name = f"builtins.{type(value).__name__}" + return RawExpressionType( + numeric_value, type_name, line=self.line, column=getattr(n, "col_offset", -1) + ) + + def visit_Index(self, n: ast3.Index) -> Type: + # cast for mypyc's benefit on Python 3.9 + value = self.visit(cast(Any, n).value) + assert isinstance(value, Type) + return value + + def visit_Slice(self, n: ast3.Slice) -> Type: + return self.invalid_type(n, note="did you mean to use ',' instead of ':' ?") + + # Subscript(expr value, slice slice, expr_context ctx) # Python 3.8 and before + # Subscript(expr value, expr slice, expr_context ctx) # Python 3.9 and later + def visit_Subscript(self, n: ast3.Subscript) -> Type: + if sys.version_info >= (3, 9): # Really 3.9a5 or later + sliceval: Any = n.slice + # Python 3.8 or earlier use a different AST structure for subscripts + elif isinstance(n.slice, ast3.Index): + sliceval: Any = n.slice.value + elif isinstance(n.slice, ast3.Slice): + sliceval = copy.deepcopy(n.slice) # so we don't mutate passed AST + if getattr(sliceval, "col_offset", None) is None: + # Fix column information so that we get Python 3.9+ message order + sliceval.col_offset = sliceval.lower.col_offset + else: + assert isinstance(n.slice, ast3.ExtSlice) + dims = cast(List[ast3.expr], copy.deepcopy(n.slice.dims)) + for s in dims: + # These fields don't actually have a col_offset attribute but we add + # it manually. + if getattr(s, "col_offset", None) is None: + if isinstance(s, ast3.Index): + s.col_offset = s.value.col_offset + elif isinstance(s, ast3.Slice): + assert s.lower is not None + s.col_offset = s.lower.col_offset + sliceval = ast3.Tuple(dims, n.ctx) + + empty_tuple_index = False + if isinstance(sliceval, ast3.Tuple): + params = self.translate_expr_list(sliceval.elts) + if len(sliceval.elts) == 0: + empty_tuple_index = True + else: + params = [self.visit(sliceval)] + + value = self.visit(n.value) + if isinstance(value, UnboundType) and not value.args: + return UnboundType( + value.name, + params, + line=self.line, + column=value.column, + empty_tuple_index=empty_tuple_index, + ) + else: + return self.invalid_type(n) + + def visit_Tuple(self, n: ast3.Tuple) -> Type: + return TupleType( + self.translate_expr_list(n.elts), + _dummy_fallback, + implicit=True, + line=self.line, + column=self.convert_column(n.col_offset), + ) + + def visit_Dict(self, n: ast3.Dict) -> Type: + if not n.keys: + return self.invalid_type(n) + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in zip(n.keys, n.values): + if not isinstance(item_name, ast3.Constant) or not isinstance(item_name.value, str): + if item_name is None: + extra_items_from.append(self.visit(value)) + continue + return self.invalid_type(n) + items[item_name.value] = self.visit(value) + result = TypedDictType(items, set(), set(), _dummy_fallback, n.lineno, n.col_offset) + result.extra_items_from = extra_items_from + return result + + # Attribute(expr value, identifier attr, expr_context ctx) + def visit_Attribute(self, n: Attribute) -> Type: + before_dot = self.visit(n.value) + + if isinstance(before_dot, UnboundType) and not before_dot.args: + return UnboundType(f"{before_dot.name}.{n.attr}", line=self.line) + else: + return self.invalid_type(n) + + # Used for Callable[[X *Ys, Z], R] etc. + def visit_Starred(self, n: ast3.Starred) -> Type: + return UnpackType(self.visit(n.value), from_star_syntax=True) + + # List(expr* elts, expr_context ctx) + def visit_List(self, n: ast3.List) -> Type: + assert isinstance(n.ctx, ast3.Load) + result = self.translate_argument_list(n.elts) + return result + + +def stringify_name(n: AST) -> str | None: + if isinstance(n, Name): + return n.id + elif isinstance(n, Attribute): + sv = stringify_name(n.value) + if sv is not None: + return f"{sv}.{n.attr}" + return None # Can't do it. + + +class FindAttributeAssign(TraverserVisitor): + """Check if an AST contains attribute assignments (e.g. self.x = 0).""" + + def __init__(self) -> None: + self.lvalue = False + self.found = False + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.lvalue = True + for lv in s.lvalues: + lv.accept(self) + self.lvalue = False + + def visit_with_stmt(self, s: WithStmt) -> None: + self.lvalue = True + for lv in s.target: + if lv is not None: + lv.accept(self) + self.lvalue = False + s.body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.lvalue = True + s.index.accept(self) + self.lvalue = False + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + # No need to look inside these + pass + + def visit_call_expr(self, e: CallExpr) -> None: + # No need to look inside these + pass + + def visit_index_expr(self, e: IndexExpr) -> None: + # No need to look inside these + pass + + def visit_member_expr(self, e: MemberExpr) -> None: + if self.lvalue: + self.found = True + + +class FindYield(TraverserVisitor): + """Check if an AST contains yields or yield froms.""" + + def __init__(self) -> None: + self.found = False + + def visit_yield_expr(self, e: YieldExpr) -> None: + self.found = True + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + self.found = True + + +def is_possible_trivial_body(s: list[Statement]) -> bool: + """Could the statements form a "trivial" function body, such as 'pass'? + + This mimics mypy.semanal.is_trivial_body, but this runs before + semantic analysis so some checks must be conservative. + """ + l = len(s) + if l == 0: + return False + i = 0 + if isinstance(s[0], ExpressionStmt) and isinstance(s[0].expr, StrExpr): + # Skip docstring + i += 1 + if i == l: + return True + if l > i + 1: + return False + stmt = s[i] + return isinstance(stmt, (PassStmt, RaiseStmt)) or ( + isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr) + ) diff --git a/mypy/find_sources.py b/mypy/find_sources.py new file mode 100644 index 000000000000..3565fc4609cd --- /dev/null +++ b/mypy/find_sources.py @@ -0,0 +1,243 @@ +"""Routines for finding the sources that mypy will check""" + +from __future__ import annotations + +import functools +import os +from typing import Final, Sequence + +from mypy.fscache import FileSystemCache +from mypy.modulefinder import PYTHON_EXTENSIONS, BuildSource, matches_exclude, mypy_path +from mypy.options import Options + +PY_EXTENSIONS: Final = tuple(PYTHON_EXTENSIONS) + + +class InvalidSourceList(Exception): + """Exception indicating a problem in the list of sources given to mypy.""" + + +def create_source_list( + paths: Sequence[str], + options: Options, + fscache: FileSystemCache | None = None, + allow_empty_dir: bool = False, +) -> list[BuildSource]: + """From a list of source files/directories, makes a list of BuildSources. + + Raises InvalidSourceList on errors. + """ + fscache = fscache or FileSystemCache() + finder = SourceFinder(fscache, options) + + sources = [] + for path in paths: + path = os.path.normpath(path) + if path.endswith(PY_EXTENSIONS): + # Can raise InvalidSourceList if a directory doesn't have a valid module name. + name, base_dir = finder.crawl_up(path) + sources.append(BuildSource(path, name, None, base_dir)) + elif fscache.isdir(path): + sub_sources = finder.find_sources_in_dir(path) + if not sub_sources and not allow_empty_dir: + raise InvalidSourceList(f"There are no .py[i] files in directory '{path}'") + sources.extend(sub_sources) + else: + mod = os.path.basename(path) if options.scripts_are_modules else None + sources.append(BuildSource(path, mod, None)) + return sources + + +def keyfunc(name: str) -> tuple[bool, int, str]: + """Determines sort order for directory listing. + + The desirable properties are: + 1) foo < foo.pyi < foo.py + 2) __init__.py[i] < foo + """ + base, suffix = os.path.splitext(name) + for i, ext in enumerate(PY_EXTENSIONS): + if suffix == ext: + return (base != "__init__", i, base) + return (base != "__init__", -1, name) + + +def normalise_package_base(root: str) -> str: + if not root: + root = os.curdir + root = os.path.abspath(root) + if root.endswith(os.sep): + root = root[:-1] + return root + + +def get_explicit_package_bases(options: Options) -> list[str] | None: + """Returns explicit package bases to use if the option is enabled, or None if disabled. + + We currently use MYPYPATH and the current directory as the package bases. In the future, + when --namespace-packages is the default could also use the values passed with the + --package-root flag, see #9632. + + Values returned are normalised so we can use simple string comparisons in + SourceFinder.is_explicit_package_base + """ + if not options.explicit_package_bases: + return None + roots = mypy_path() + options.mypy_path + [os.getcwd()] + return [normalise_package_base(root) for root in roots] + + +class SourceFinder: + def __init__(self, fscache: FileSystemCache, options: Options) -> None: + self.fscache = fscache + self.explicit_package_bases = get_explicit_package_bases(options) + self.namespace_packages = options.namespace_packages + self.exclude = options.exclude + self.verbosity = options.verbosity + + def is_explicit_package_base(self, path: str) -> bool: + assert self.explicit_package_bases + return normalise_package_base(path) in self.explicit_package_bases + + def find_sources_in_dir(self, path: str) -> list[BuildSource]: + sources = [] + + seen: set[str] = set() + names = sorted(self.fscache.listdir(path), key=keyfunc) + for name in names: + # Skip certain names altogether + if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): + continue + subpath = os.path.join(path, name) + + if matches_exclude(subpath, self.exclude, self.fscache, self.verbosity >= 2): + continue + + if self.fscache.isdir(subpath): + sub_sources = self.find_sources_in_dir(subpath) + if sub_sources: + seen.add(name) + sources.extend(sub_sources) + else: + stem, suffix = os.path.splitext(name) + if stem not in seen and suffix in PY_EXTENSIONS: + seen.add(stem) + module, base_dir = self.crawl_up(subpath) + sources.append(BuildSource(subpath, module, None, base_dir)) + + return sources + + def crawl_up(self, path: str) -> tuple[str, str]: + """Given a .py[i] filename, return module and base directory. + + For example, given "xxx/yyy/foo/bar.py", we might return something like: + ("foo.bar", "xxx/yyy") + + If namespace packages is off, we crawl upwards until we find a directory without + an __init__.py + + If namespace packages is on, we crawl upwards until the nearest explicit base directory. + Failing that, we return one past the highest directory containing an __init__.py + + We won't crawl past directories with invalid package names. + The base directory returned is an absolute path. + """ + path = os.path.abspath(path) + parent, filename = os.path.split(path) + + module_name = strip_py(filename) or filename + + parent_module, base_dir = self.crawl_up_dir(parent) + if module_name == "__init__": + return parent_module, base_dir + + # Note that module_name might not actually be a valid identifier, but that's okay + # Ignoring this possibility sidesteps some search path confusion + module = module_join(parent_module, module_name) + return module, base_dir + + def crawl_up_dir(self, dir: str) -> tuple[str, str]: + return self._crawl_up_helper(dir) or ("", dir) + + @functools.lru_cache # noqa: B019 + def _crawl_up_helper(self, dir: str) -> tuple[str, str] | None: + """Given a directory, maybe returns module and base directory. + + We return a non-None value if we were able to find something clearly intended as a base + directory (as adjudicated by being an explicit base directory or by containing a package + with __init__.py). + + This distinction is necessary for namespace packages, so that we know when to treat + ourselves as a subpackage. + """ + # stop crawling if we're an explicit base directory + if self.explicit_package_bases is not None and self.is_explicit_package_base(dir): + return "", dir + + parent, name = os.path.split(dir) + if name.endswith("-stubs"): + name = name[:-6] # PEP-561 stub-only directory + + # recurse if there's an __init__.py + init_file = self.get_init_file(dir) + if init_file is not None: + if not name.isidentifier(): + # in most cases the directory name is invalid, we'll just stop crawling upwards + # but if there's an __init__.py in the directory, something is messed up + raise InvalidSourceList(f"{name} is not a valid Python package name") + # we're definitely a package, so we always return a non-None value + mod_prefix, base_dir = self.crawl_up_dir(parent) + return module_join(mod_prefix, name), base_dir + + # stop crawling if we're out of path components or our name is an invalid identifier + if not name or not parent or not name.isidentifier(): + return None + + # stop crawling if namespace packages is off (since we don't have an __init__.py) + if not self.namespace_packages: + return None + + # at this point: namespace packages is on, we don't have an __init__.py and we're not an + # explicit base directory + result = self._crawl_up_helper(parent) + if result is None: + # we're not an explicit base directory and we don't have an __init__.py + # and none of our parents are either, so return + return None + # one of our parents was an explicit base directory or had an __init__.py, so we're + # definitely a subpackage! chain our name to the module. + mod_prefix, base_dir = result + return module_join(mod_prefix, name), base_dir + + def get_init_file(self, dir: str) -> str | None: + """Check whether a directory contains a file named __init__.py[i]. + + If so, return the file's name (with dir prefixed). If not, return None. + + This prefers .pyi over .py (because of the ordering of PY_EXTENSIONS). + """ + for ext in PY_EXTENSIONS: + f = os.path.join(dir, "__init__" + ext) + if self.fscache.isfile(f): + return f + if ext == ".py" and self.fscache.init_under_package_root(f): + return f + return None + + +def module_join(parent: str, child: str) -> str: + """Join module ids, accounting for a possibly empty parent.""" + if parent: + return parent + "." + child + return child + + +def strip_py(arg: str) -> str | None: + """Strip a trailing .py or .pyi suffix. + + Return None if no such suffix is found. + """ + for ext in PY_EXTENSIONS: + if arg.endswith(ext): + return arg[: -len(ext)] + return None diff --git a/mypy/fixup.py b/mypy/fixup.py new file mode 100644 index 000000000000..f2b5bc17d32e --- /dev/null +++ b/mypy/fixup.py @@ -0,0 +1,431 @@ +"""Fix up various things after deserialization.""" + +from __future__ import annotations + +from typing import Any, Final + +from mypy.lookup import lookup_fully_qualified +from mypy.nodes import ( + Block, + ClassDef, + Decorator, + FuncDef, + MypyFile, + OverloadedFuncDef, + ParamSpecExpr, + SymbolTable, + TypeAlias, + TypeInfo, + TypeVarExpr, + TypeVarTupleExpr, + Var, +) +from mypy.types import ( + NOT_READY, + AnyType, + CallableType, + Instance, + LiteralType, + Overloaded, + Parameters, + ParamSpecType, + TupleType, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UnionType, + UnpackType, +) +from mypy.visitor import NodeVisitor + + +# N.B: we do a allow_missing fixup when fixing up a fine-grained +# incremental cache load (since there may be cross-refs into deleted +# modules) +def fixup_module(tree: MypyFile, modules: dict[str, MypyFile], allow_missing: bool) -> None: + node_fixer = NodeFixer(modules, allow_missing) + node_fixer.visit_symbol_table(tree.names, tree.fullname) + + +# TODO: Fix up .info when deserializing, i.e. much earlier. +class NodeFixer(NodeVisitor[None]): + current_info: TypeInfo | None = None + + def __init__(self, modules: dict[str, MypyFile], allow_missing: bool) -> None: + self.modules = modules + self.allow_missing = allow_missing + self.type_fixer = TypeFixer(self.modules, allow_missing) + + # NOTE: This method isn't (yet) part of the NodeVisitor API. + def visit_type_info(self, info: TypeInfo) -> None: + save_info = self.current_info + try: + self.current_info = info + if info.defn: + info.defn.accept(self) + if info.names: + self.visit_symbol_table(info.names, info.fullname) + if info.bases: + for base in info.bases: + base.accept(self.type_fixer) + if info._promote: + for p in info._promote: + p.accept(self.type_fixer) + if info.tuple_type: + info.tuple_type.accept(self.type_fixer) + info.update_tuple_type(info.tuple_type) + if info.special_alias: + info.special_alias.alias_tvars = list(info.defn.type_vars) + for i, t in enumerate(info.defn.type_vars): + if isinstance(t, TypeVarTupleType): + info.special_alias.tvar_tuple_index = i + if info.typeddict_type: + info.typeddict_type.accept(self.type_fixer) + info.update_typeddict_type(info.typeddict_type) + if info.special_alias: + info.special_alias.alias_tvars = list(info.defn.type_vars) + for i, t in enumerate(info.defn.type_vars): + if isinstance(t, TypeVarTupleType): + info.special_alias.tvar_tuple_index = i + if info.declared_metaclass: + info.declared_metaclass.accept(self.type_fixer) + if info.metaclass_type: + info.metaclass_type.accept(self.type_fixer) + if info.alt_promote: + info.alt_promote.accept(self.type_fixer) + instance = Instance(info, []) + # Hack: We may also need to add a backwards promotion (from int to native int), + # since it might not be serialized. + if instance not in info.alt_promote.type._promote: + info.alt_promote.type._promote.append(instance) + if info._mro_refs: + info.mro = [ + lookup_fully_qualified_typeinfo( + self.modules, name, allow_missing=self.allow_missing + ) + for name in info._mro_refs + ] + info._mro_refs = None + finally: + self.current_info = save_info + + # NOTE: This method *definitely* isn't part of the NodeVisitor API. + def visit_symbol_table(self, symtab: SymbolTable, table_fullname: str) -> None: + # Copy the items because we may mutate symtab. + for key, value in list(symtab.items()): + cross_ref = value.cross_ref + if cross_ref is not None: # Fix up cross-reference. + value.cross_ref = None + if cross_ref in self.modules: + value.node = self.modules[cross_ref] + else: + stnode = lookup_fully_qualified( + cross_ref, self.modules, raise_on_missing=not self.allow_missing + ) + if stnode is not None: + if stnode is value: + # The node seems to refer to itself, which can mean that + # the target is a deleted submodule of the current module, + # and thus lookup falls back to the symbol table of the parent + # package. Here's how this may happen: + # + # pkg/__init__.py: + # from pkg import sub + # + # Now if pkg.sub is deleted, the pkg.sub symbol table entry + # appears to refer to itself. Replace the entry with a + # placeholder to avoid a crash. We can't delete the entry, + # as it would stop dependency propagation. + value.node = Var(key + "@deleted") + else: + assert stnode.node is not None, (table_fullname + "." + key, cross_ref) + value.node = stnode.node + elif not self.allow_missing: + assert False, f"Could not find cross-ref {cross_ref}" + else: + # We have a missing crossref in allow missing mode, need to put something + value.node = missing_info(self.modules) + else: + if isinstance(value.node, TypeInfo): + # TypeInfo has no accept(). TODO: Add it? + self.visit_type_info(value.node) + elif value.node is not None: + value.node.accept(self) + else: + assert False, f"Unexpected empty node {key!r}: {value}" + + def visit_func_def(self, func: FuncDef) -> None: + if self.current_info is not None: + func.info = self.current_info + if func.type is not None: + func.type.accept(self.type_fixer) + + def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: + if self.current_info is not None: + o.info = self.current_info + if o.type: + o.type.accept(self.type_fixer) + for item in o.items: + item.accept(self) + if o.impl: + o.impl.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + if self.current_info is not None: + d.var.info = self.current_info + if d.func: + d.func.accept(self) + if d.var: + d.var.accept(self) + for node in d.decorators: + node.accept(self) + + def visit_class_def(self, c: ClassDef) -> None: + for v in c.type_vars: + v.accept(self.type_fixer) + + def visit_type_var_expr(self, tv: TypeVarExpr) -> None: + for value in tv.values: + value.accept(self.type_fixer) + tv.upper_bound.accept(self.type_fixer) + tv.default.accept(self.type_fixer) + + def visit_paramspec_expr(self, p: ParamSpecExpr) -> None: + p.upper_bound.accept(self.type_fixer) + p.default.accept(self.type_fixer) + + def visit_type_var_tuple_expr(self, tv: TypeVarTupleExpr) -> None: + tv.upper_bound.accept(self.type_fixer) + tv.tuple_fallback.accept(self.type_fixer) + tv.default.accept(self.type_fixer) + + def visit_var(self, v: Var) -> None: + if self.current_info is not None: + v.info = self.current_info + if v.type is not None: + v.type.accept(self.type_fixer) + + def visit_type_alias(self, a: TypeAlias) -> None: + a.target.accept(self.type_fixer) + for v in a.alias_tvars: + v.accept(self.type_fixer) + + +class TypeFixer(TypeVisitor[None]): + def __init__(self, modules: dict[str, MypyFile], allow_missing: bool) -> None: + self.modules = modules + self.allow_missing = allow_missing + + def visit_instance(self, inst: Instance) -> None: + # TODO: Combine Instances that are exactly the same? + type_ref = inst.type_ref + if type_ref is None: + return # We've already been here. + inst.type_ref = None + inst.type = lookup_fully_qualified_typeinfo( + self.modules, type_ref, allow_missing=self.allow_missing + ) + # TODO: Is this needed or redundant? + # Also fix up the bases, just in case. + for base in inst.type.bases: + if base.type is NOT_READY: + base.accept(self) + for a in inst.args: + a.accept(self) + if inst.last_known_value is not None: + inst.last_known_value.accept(self) + if inst.extra_attrs: + for v in inst.extra_attrs.attrs.values(): + v.accept(self) + + def visit_type_alias_type(self, t: TypeAliasType) -> None: + type_ref = t.type_ref + if type_ref is None: + return # We've already been here. + t.type_ref = None + t.alias = lookup_fully_qualified_alias( + self.modules, type_ref, allow_missing=self.allow_missing + ) + for a in t.args: + a.accept(self) + + def visit_any(self, o: Any) -> None: + pass # Nothing to descend into. + + def visit_callable_type(self, ct: CallableType) -> None: + if ct.fallback: + ct.fallback.accept(self) + for argt in ct.arg_types: + # argt may be None, e.g. for __self in NamedTuple constructors. + if argt is not None: + argt.accept(self) + if ct.ret_type is not None: + ct.ret_type.accept(self) + for v in ct.variables: + v.accept(self) + for arg in ct.bound_args: + if arg: + arg.accept(self) + if ct.type_guard is not None: + ct.type_guard.accept(self) + if ct.type_is is not None: + ct.type_is.accept(self) + + def visit_overloaded(self, t: Overloaded) -> None: + for ct in t.items: + ct.accept(self) + + def visit_erased_type(self, o: Any) -> None: + # This type should exist only temporarily during type inference + raise RuntimeError("Shouldn't get here", o) + + def visit_deleted_type(self, o: Any) -> None: + pass # Nothing to descend into. + + def visit_none_type(self, o: Any) -> None: + pass # Nothing to descend into. + + def visit_uninhabited_type(self, o: Any) -> None: + pass # Nothing to descend into. + + def visit_partial_type(self, o: Any) -> None: + raise RuntimeError("Shouldn't get here", o) + + def visit_tuple_type(self, tt: TupleType) -> None: + if tt.items: + for it in tt.items: + it.accept(self) + if tt.partial_fallback is not None: + tt.partial_fallback.accept(self) + + def visit_typeddict_type(self, tdt: TypedDictType) -> None: + if tdt.items: + for it in tdt.items.values(): + it.accept(self) + if tdt.fallback is not None: + if tdt.fallback.type_ref is not None: + if ( + lookup_fully_qualified( + tdt.fallback.type_ref, + self.modules, + raise_on_missing=not self.allow_missing, + ) + is None + ): + # We reject fake TypeInfos for TypedDict fallbacks because + # the latter are used in type checking and must be valid. + tdt.fallback.type_ref = "typing._TypedDict" + tdt.fallback.accept(self) + + def visit_literal_type(self, lt: LiteralType) -> None: + lt.fallback.accept(self) + + def visit_type_var(self, tvt: TypeVarType) -> None: + if tvt.values: + for vt in tvt.values: + vt.accept(self) + tvt.upper_bound.accept(self) + tvt.default.accept(self) + + def visit_param_spec(self, p: ParamSpecType) -> None: + p.upper_bound.accept(self) + p.default.accept(self) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + t.tuple_fallback.accept(self) + t.upper_bound.accept(self) + t.default.accept(self) + + def visit_unpack_type(self, u: UnpackType) -> None: + u.type.accept(self) + + def visit_parameters(self, p: Parameters) -> None: + for argt in p.arg_types: + if argt is not None: + argt.accept(self) + for var in p.variables: + var.accept(self) + + def visit_unbound_type(self, o: UnboundType) -> None: + for a in o.args: + a.accept(self) + + def visit_union_type(self, ut: UnionType) -> None: + if ut.items: + for it in ut.items: + it.accept(self) + + def visit_type_type(self, t: TypeType) -> None: + t.item.accept(self) + + +def lookup_fully_qualified_typeinfo( + modules: dict[str, MypyFile], name: str, *, allow_missing: bool +) -> TypeInfo: + stnode = lookup_fully_qualified(name, modules, raise_on_missing=not allow_missing) + node = stnode.node if stnode else None + if isinstance(node, TypeInfo): + return node + else: + # Looks like a missing TypeInfo during an initial daemon load, put something there + assert ( + allow_missing + ), "Should never get here in normal mode, got {}:{} instead of TypeInfo".format( + type(node).__name__, node.fullname if node else "" + ) + return missing_info(modules) + + +def lookup_fully_qualified_alias( + modules: dict[str, MypyFile], name: str, *, allow_missing: bool +) -> TypeAlias: + stnode = lookup_fully_qualified(name, modules, raise_on_missing=not allow_missing) + node = stnode.node if stnode else None + if isinstance(node, TypeAlias): + return node + elif isinstance(node, TypeInfo): + if node.special_alias: + # Already fixed up. + return node.special_alias + if node.tuple_type: + alias = TypeAlias.from_tuple_type(node) + elif node.typeddict_type: + alias = TypeAlias.from_typeddict_type(node) + else: + assert allow_missing + return missing_alias() + node.special_alias = alias + return alias + else: + # Looks like a missing TypeAlias during an initial daemon load, put something there + assert ( + allow_missing + ), "Should never get here in normal mode, got {}:{} instead of TypeAlias".format( + type(node).__name__, node.fullname if node else "" + ) + return missing_alias() + + +_SUGGESTION: Final = "" + + +def missing_info(modules: dict[str, MypyFile]) -> TypeInfo: + suggestion = _SUGGESTION.format("info") + dummy_def = ClassDef(suggestion, Block([])) + dummy_def.fullname = suggestion + + info = TypeInfo(SymbolTable(), dummy_def, "") + obj_type = lookup_fully_qualified_typeinfo(modules, "builtins.object", allow_missing=False) + info.bases = [Instance(obj_type, [])] + info.mro = [info, obj_type] + return info + + +def missing_alias() -> TypeAlias: + suggestion = _SUGGESTION.format("alias") + return TypeAlias(AnyType(TypeOfAny.special_form), suggestion, line=-1, column=-1) diff --git a/mypy/freetree.py b/mypy/freetree.py new file mode 100644 index 000000000000..75b89e2623ae --- /dev/null +++ b/mypy/freetree.py @@ -0,0 +1,23 @@ +"""Generic node traverser visitor""" + +from __future__ import annotations + +from mypy.nodes import Block, MypyFile +from mypy.traverser import TraverserVisitor + + +class TreeFreer(TraverserVisitor): + def visit_block(self, block: Block) -> None: + super().visit_block(block) + block.body.clear() + + +def free_tree(tree: MypyFile) -> None: + """Free all the ASTs associated with a module. + + This needs to be done recursively, since symbol tables contain + references to definitions, so those won't be freed but we want their + contents to be. + """ + tree.accept(TreeFreer()) + tree.defs.clear() diff --git a/mypy/fscache.py b/mypy/fscache.py new file mode 100644 index 000000000000..8251f4bd9488 --- /dev/null +++ b/mypy/fscache.py @@ -0,0 +1,302 @@ +"""Interface for accessing the file system with automatic caching. + +The idea is to cache the results of any file system state reads during +a single transaction. This has two main benefits: + +* This avoids redundant syscalls, as we won't perform the same OS + operations multiple times. + +* This makes it easier to reason about concurrent FS updates, as different + operations targeting the same paths can't report different state during + a transaction. + +Note that this only deals with reading state, not writing. + +Properties maintained by the API: + +* The contents of the file are always from the same or later time compared + to the reported mtime of the file, even if mtime is queried after reading + a file. + +* Repeating an operation produces the same result as the first one during + a transaction. + +* Call flush() to start a new transaction (flush the caches). + +The API is a bit limited. It's easy to add new cached operations, however. +You should perform all file system reads through the API to actually take +advantage of the benefits. +""" + +from __future__ import annotations + +import os +import stat + +from mypy_extensions import mypyc_attr + +from mypy.util import hash_digest + + +@mypyc_attr(allow_interpreted_subclasses=True) # for tests +class FileSystemCache: + def __init__(self) -> None: + # The package root is not flushed with the caches. + # It is set by set_package_root() below. + self.package_root: list[str] = [] + self.flush() + + def set_package_root(self, package_root: list[str]) -> None: + self.package_root = package_root + + def flush(self) -> None: + """Start another transaction and empty all caches.""" + self.stat_or_none_cache: dict[str, os.stat_result | None] = {} + + self.listdir_cache: dict[str, list[str]] = {} + self.listdir_error_cache: dict[str, OSError] = {} + self.isfile_case_cache: dict[str, bool] = {} + self.exists_case_cache: dict[str, bool] = {} + self.read_cache: dict[str, bytes] = {} + self.read_error_cache: dict[str, Exception] = {} + self.hash_cache: dict[str, str] = {} + self.fake_package_cache: set[str] = set() + + def stat_or_none(self, path: str) -> os.stat_result | None: + if path in self.stat_or_none_cache: + return self.stat_or_none_cache[path] + + st = None + try: + st = os.stat(path) + except OSError: + if self.init_under_package_root(path): + try: + st = self._fake_init(path) + except OSError: + pass + + self.stat_or_none_cache[path] = st + return st + + def init_under_package_root(self, path: str) -> bool: + """Is this path an __init__.py under a package root? + + This is used to detect packages that don't contain __init__.py + files, which is needed to support Bazel. The function should + only be called for non-existing files. + + It will return True if it refers to a __init__.py file that + Bazel would create, so that at runtime Python would think the + directory containing it is a package. For this to work you + must pass one or more package roots using the --package-root + flag. + + As an exceptional case, any directory that is a package root + itself will not be considered to contain a __init__.py file. + This is different from the rules Bazel itself applies, but is + necessary for mypy to properly distinguish packages from other + directories. + + See https://docs.bazel.build/versions/master/be/python.html, + where this behavior is described under legacy_create_init. + """ + if not self.package_root: + return False + dirname, basename = os.path.split(path) + if basename != "__init__.py": + return False + if not os.path.basename(dirname).isidentifier(): + # Can't put an __init__.py in a place that's not an identifier + return False + + st = self.stat_or_none(dirname) + if st is None: + return False + else: + if not stat.S_ISDIR(st.st_mode): + return False + ok = False + drive, path = os.path.splitdrive(path) # Ignore Windows drive name + if os.path.isabs(path): + path = os.path.relpath(path) + path = os.path.normpath(path) + for root in self.package_root: + if path.startswith(root): + if path == root + basename: + # A package root itself is never a package. + ok = False + break + else: + ok = True + return ok + + def _fake_init(self, path: str) -> os.stat_result: + """Prime the cache with a fake __init__.py file. + + This makes code that looks for path believe an empty file by + that name exists. Should only be called after + init_under_package_root() returns True. + """ + dirname, basename = os.path.split(path) + assert basename == "__init__.py", path + assert not os.path.exists(path), path # Not cached! + dirname = os.path.normpath(dirname) + st = os.stat(dirname) # May raise OSError + # Get stat result as a list so we can modify it. + seq: list[float] = list(st) + seq[stat.ST_MODE] = stat.S_IFREG | 0o444 + seq[stat.ST_INO] = 1 + seq[stat.ST_NLINK] = 1 + seq[stat.ST_SIZE] = 0 + st = os.stat_result(seq) + # Make listdir() and read() also pretend this file exists. + self.fake_package_cache.add(dirname) + return st + + def listdir(self, path: str) -> list[str]: + path = os.path.normpath(path) + if path in self.listdir_cache: + res = self.listdir_cache[path] + # Check the fake cache. + if path in self.fake_package_cache and "__init__.py" not in res: + res.append("__init__.py") # Updates the result as well as the cache + return res + if path in self.listdir_error_cache: + raise copy_os_error(self.listdir_error_cache[path]) + try: + results = os.listdir(path) + except OSError as err: + # Like above, take a copy to reduce memory use. + self.listdir_error_cache[path] = copy_os_error(err) + raise err + self.listdir_cache[path] = results + # Check the fake cache. + if path in self.fake_package_cache and "__init__.py" not in results: + results.append("__init__.py") + return results + + def isfile(self, path: str) -> bool: + st = self.stat_or_none(path) + if st is None: + return False + return stat.S_ISREG(st.st_mode) + + def isfile_case(self, path: str, prefix: str) -> bool: + """Return whether path exists and is a file. + + On case-insensitive filesystems (like Mac or Windows) this returns + False if the case of path's last component does not exactly match + the case found in the filesystem. + + We check also the case of other path components up to prefix. + For example, if path is 'user-stubs/pack/mod.pyi' and prefix is 'user-stubs', + we check that the case of 'pack' and 'mod.py' matches exactly, 'user-stubs' will be + case insensitive on case insensitive filesystems. + + The caller must ensure that prefix is a valid file system prefix of path. + """ + if not self.isfile(path): + # Fast path + return False + if path in self.isfile_case_cache: + return self.isfile_case_cache[path] + head, tail = os.path.split(path) + if not tail: + self.isfile_case_cache[path] = False + return False + try: + names = self.listdir(head) + # This allows one to check file name case sensitively in + # case-insensitive filesystems. + res = tail in names + except OSError: + res = False + if res: + # Also recursively check the other path components in case sensitive way. + res = self.exists_case(head, prefix) + self.isfile_case_cache[path] = res + return res + + def exists_case(self, path: str, prefix: str) -> bool: + """Return whether path exists - checking path components in case sensitive + fashion, up to prefix. + """ + if path in self.exists_case_cache: + return self.exists_case_cache[path] + head, tail = os.path.split(path) + if not head.startswith(prefix) or not tail: + # Only perform the check for paths under prefix. + self.exists_case_cache[path] = True + return True + try: + names = self.listdir(head) + # This allows one to check file name case sensitively in + # case-insensitive filesystems. + res = tail in names + except OSError: + res = False + if res: + # Also recursively check other path components. + res = self.exists_case(head, prefix) + self.exists_case_cache[path] = res + return res + + def isdir(self, path: str) -> bool: + st = self.stat_or_none(path) + if st is None: + return False + return stat.S_ISDIR(st.st_mode) + + def exists(self, path: str) -> bool: + st = self.stat_or_none(path) + return st is not None + + def read(self, path: str) -> bytes: + if path in self.read_cache: + return self.read_cache[path] + if path in self.read_error_cache: + raise self.read_error_cache[path] + + # Need to stat first so that the contents of file are from no + # earlier instant than the mtime reported by self.stat(). + self.stat_or_none(path) + + dirname, basename = os.path.split(path) + dirname = os.path.normpath(dirname) + # Check the fake cache. + if basename == "__init__.py" and dirname in self.fake_package_cache: + data = b"" + else: + try: + with open(path, "rb") as f: + data = f.read() + except OSError as err: + self.read_error_cache[path] = err + raise + + self.read_cache[path] = data + self.hash_cache[path] = hash_digest(data) + return data + + def hash_digest(self, path: str) -> str: + if path not in self.hash_cache: + self.read(path) + return self.hash_cache[path] + + def samefile(self, f1: str, f2: str) -> bool: + s1 = self.stat_or_none(f1) + s2 = self.stat_or_none(f2) + if s1 is None or s2 is None: + return False + return os.path.samestat(s1, s2) + + +def copy_os_error(e: OSError) -> OSError: + new = OSError(*e.args) + new.errno = e.errno + new.strerror = e.strerror + new.filename = e.filename + if e.filename2: + new.filename2 = e.filename2 + return new diff --git a/mypy/fswatcher.py b/mypy/fswatcher.py new file mode 100644 index 000000000000..97a62ca9f9f7 --- /dev/null +++ b/mypy/fswatcher.py @@ -0,0 +1,105 @@ +"""Watch parts of the file system for changes.""" + +from __future__ import annotations + +import os +from typing import AbstractSet, Iterable, NamedTuple + +from mypy.fscache import FileSystemCache + + +class FileData(NamedTuple): + st_mtime: float + st_size: int + hash: str + + +class FileSystemWatcher: + """Watcher for file system changes among specific paths. + + All file system access is performed using FileSystemCache. We + detect changed files by stat()ing them all and comparing hashes + of potentially changed files. If a file has both size and mtime + unmodified, the file is assumed to be unchanged. + + An important goal of this class is to make it easier to eventually + use file system events to detect file changes. + + Note: This class doesn't flush the file system cache. If you don't + manually flush it, changes won't be seen. + """ + + # TODO: Watching directories? + # TODO: Handle non-files + + def __init__(self, fs: FileSystemCache) -> None: + self.fs = fs + self._paths: set[str] = set() + self._file_data: dict[str, FileData | None] = {} + + def dump_file_data(self) -> dict[str, tuple[float, int, str]]: + return {k: v for k, v in self._file_data.items() if v is not None} + + def set_file_data(self, path: str, data: FileData) -> None: + self._file_data[path] = data + + def add_watched_paths(self, paths: Iterable[str]) -> None: + for path in paths: + if path not in self._paths: + # By storing None this path will get reported as changed by + # find_changed if it exists. + self._file_data[path] = None + self._paths |= set(paths) + + def remove_watched_paths(self, paths: Iterable[str]) -> None: + for path in paths: + if path in self._file_data: + del self._file_data[path] + self._paths -= set(paths) + + def _update(self, path: str, st: os.stat_result) -> None: + hash_digest = self.fs.hash_digest(path) + self._file_data[path] = FileData(st.st_mtime, st.st_size, hash_digest) + + def _find_changed(self, paths: Iterable[str]) -> AbstractSet[str]: + changed = set() + for path in paths: + old = self._file_data[path] + st = self.fs.stat_or_none(path) + if st is None: + if old is not None: + # File was deleted. + changed.add(path) + self._file_data[path] = None + else: + if old is None: + # File is new. + changed.add(path) + self._update(path, st) + # Round mtimes down, to match the mtimes we write to meta files + elif st.st_size != old.st_size or int(st.st_mtime) != int(old.st_mtime): + # Only look for changes if size or mtime has changed as an + # optimization, since calculating hash is expensive. + new_hash = self.fs.hash_digest(path) + self._update(path, st) + if st.st_size != old.st_size or new_hash != old.hash: + # Changed file. + changed.add(path) + return changed + + def find_changed(self) -> AbstractSet[str]: + """Return paths that have changes since the last call, in the watched set.""" + return self._find_changed(self._paths) + + def update_changed(self, remove: list[str], update: list[str]) -> AbstractSet[str]: + """Alternative to find_changed() given explicit changes. + + This only calls self.fs.stat() on added or updated files, not + on all files. It believes all other files are unchanged! + + Implies add_watched_paths() for add and update, and + remove_watched_paths() for remove. + """ + self.remove_watched_paths(remove) + self.add_watched_paths(update) + return self._find_changed(update) diff --git a/mypy/gclogger.py b/mypy/gclogger.py new file mode 100644 index 000000000000..d111e609223c --- /dev/null +++ b/mypy/gclogger.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import gc +import time +from typing import Mapping + + +class GcLogger: + """Context manager to log GC stats and overall time.""" + + def __enter__(self) -> GcLogger: + self.gc_start_time: float | None = None + self.gc_time = 0.0 + self.gc_calls = 0 + self.gc_collected = 0 + self.gc_uncollectable = 0 + gc.callbacks.append(self.gc_callback) + self.start_time = time.time() + return self + + def gc_callback(self, phase: str, info: Mapping[str, int]) -> None: + if phase == "start": + assert self.gc_start_time is None, "Start phase out of sequence" + self.gc_start_time = time.time() + elif phase == "stop": + assert self.gc_start_time is not None, "Stop phase out of sequence" + self.gc_calls += 1 + self.gc_time += time.time() - self.gc_start_time + self.gc_start_time = None + self.gc_collected += info["collected"] + self.gc_uncollectable += info["uncollectable"] + else: + assert False, f"Unrecognized gc phase ({phase!r})" + + def __exit__(self, *args: object) -> None: + while self.gc_callback in gc.callbacks: + gc.callbacks.remove(self.gc_callback) + + def get_stats(self) -> Mapping[str, float]: + end_time = time.time() + result = { + "gc_time": self.gc_time, + "gc_calls": self.gc_calls, + "gc_collected": self.gc_collected, + "gc_uncollectable": self.gc_uncollectable, + "build_time": end_time - self.start_time, + } + return result diff --git a/mypy/git.py b/mypy/git.py new file mode 100644 index 000000000000..1c63bf6471dc --- /dev/null +++ b/mypy/git.py @@ -0,0 +1,34 @@ +"""Git utilities.""" + +# Used also from setup.py, so don't pull in anything additional here (like mypy or typing): +from __future__ import annotations + +import os +import subprocess + + +def is_git_repo(dir: str) -> bool: + """Is the given directory version-controlled with git?""" + return os.path.exists(os.path.join(dir, ".git")) + + +def have_git() -> bool: + """Can we run the git executable?""" + try: + subprocess.check_output(["git", "--help"]) + return True + except subprocess.CalledProcessError: + return False + except OSError: + return False + + +def git_revision(dir: str) -> bytes: + """Get the SHA-1 of the HEAD of a git repository.""" + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dir).strip() + + +def is_dirty(dir: str) -> bool: + """Check whether a git repository has uncommitted changes.""" + output = subprocess.check_output(["git", "status", "-uno", "--porcelain"], cwd=dir) + return output.strip() != b"" diff --git a/mypy/graph_utils.py b/mypy/graph_utils.py new file mode 100644 index 000000000000..399301a6b0fd --- /dev/null +++ b/mypy/graph_utils.py @@ -0,0 +1,112 @@ +"""Helpers for manipulations with graphs.""" + +from __future__ import annotations + +from typing import AbstractSet, Iterable, Iterator, TypeVar + +T = TypeVar("T") + + +def strongly_connected_components( + vertices: AbstractSet[T], edges: dict[T, list[T]] +) -> Iterator[set[T]]: + """Compute Strongly Connected Components of a directed graph. + + Args: + vertices: the labels for the vertices + edges: for each vertex, gives the target vertices of its outgoing edges + + Returns: + An iterator yielding strongly connected components, each + represented as a set of vertices. Each input vertex will occur + exactly once; vertices not part of a SCC are returned as + singleton sets. + + From https://code.activestate.com/recipes/578507/. + """ + identified: set[T] = set() + stack: list[T] = [] + index: dict[T, int] = {} + boundaries: list[int] = [] + + def dfs(v: T) -> Iterator[set[T]]: + index[v] = len(stack) + stack.append(v) + boundaries.append(index[v]) + + for w in edges[v]: + if w not in index: + yield from dfs(w) + elif w not in identified: + while index[w] < boundaries[-1]: + boundaries.pop() + + if boundaries[-1] == index[v]: + boundaries.pop() + scc = set(stack[index[v] :]) + del stack[index[v] :] + identified.update(scc) + yield scc + + for v in vertices: + if v not in index: + yield from dfs(v) + + +def prepare_sccs( + sccs: list[set[T]], edges: dict[T, list[T]] +) -> dict[AbstractSet[T], set[AbstractSet[T]]]: + """Use original edges to organize SCCs in a graph by dependencies between them.""" + sccsmap = {v: frozenset(scc) for scc in sccs for v in scc} + data: dict[AbstractSet[T], set[AbstractSet[T]]] = {} + for scc in sccs: + deps: set[AbstractSet[T]] = set() + for v in scc: + deps.update(sccsmap[x] for x in edges[v]) + data[frozenset(scc)] = deps + return data + + +def topsort(data: dict[T, set[T]]) -> Iterable[set[T]]: + """Topological sort. + + Args: + data: A map from vertices to all vertices that it has an edge + connecting it to. NOTE: This data structure + is modified in place -- for normalization purposes, + self-dependencies are removed and entries representing + orphans are added. + + Returns: + An iterator yielding sets of vertices that have an equivalent + ordering. + + Example: + Suppose the input has the following structure: + + {A: {B, C}, B: {D}, C: {D}} + + This is normalized to: + + {A: {B, C}, B: {D}, C: {D}, D: {}} + + The algorithm will yield the following values: + + {D} + {B, C} + {A} + + From https://code.activestate.com/recipes/577413/. + """ + # TODO: Use a faster algorithm? + for k, v in data.items(): + v.discard(k) # Ignore self dependencies. + for item in set.union(*data.values()) - set(data.keys()): + data[item] = set() + while True: + ready = {item for item, dep in data.items() if not dep} + if not ready: + break + yield ready + data = {item: (dep - ready) for item, dep in data.items() if item not in ready} + assert not data, f"A cyclic dependency exists amongst {data!r}" diff --git a/mypy/indirection.py b/mypy/indirection.py new file mode 100644 index 000000000000..00356d7a4ddb --- /dev/null +++ b/mypy/indirection.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from typing import Iterable, Set + +import mypy.types as types +from mypy.types import TypeVisitor +from mypy.util import split_module_names + + +def extract_module_names(type_name: str | None) -> list[str]: + """Returns the module names of a fully qualified type name.""" + if type_name is not None: + # Discard the first one, which is just the qualified name of the type + possible_module_names = split_module_names(type_name) + return possible_module_names[1:] + else: + return [] + + +class TypeIndirectionVisitor(TypeVisitor[Set[str]]): + """Returns all module references within a particular type.""" + + def __init__(self) -> None: + self.cache: dict[types.Type, set[str]] = {} + self.seen_aliases: set[types.TypeAliasType] = set() + + def find_modules(self, typs: Iterable[types.Type]) -> set[str]: + self.seen_aliases.clear() + return self._visit(typs) + + def _visit(self, typ_or_typs: types.Type | Iterable[types.Type]) -> set[str]: + typs = [typ_or_typs] if isinstance(typ_or_typs, types.Type) else typ_or_typs + output: set[str] = set() + for typ in typs: + if isinstance(typ, types.TypeAliasType): + # Avoid infinite recursion for recursive type aliases. + if typ in self.seen_aliases: + continue + self.seen_aliases.add(typ) + if typ in self.cache: + modules = self.cache[typ] + else: + modules = typ.accept(self) + self.cache[typ] = set(modules) + output.update(modules) + return output + + def visit_unbound_type(self, t: types.UnboundType) -> set[str]: + return self._visit(t.args) + + def visit_any(self, t: types.AnyType) -> set[str]: + return set() + + def visit_none_type(self, t: types.NoneType) -> set[str]: + return set() + + def visit_uninhabited_type(self, t: types.UninhabitedType) -> set[str]: + return set() + + def visit_erased_type(self, t: types.ErasedType) -> set[str]: + return set() + + def visit_deleted_type(self, t: types.DeletedType) -> set[str]: + return set() + + def visit_type_var(self, t: types.TypeVarType) -> set[str]: + return self._visit(t.values) | self._visit(t.upper_bound) | self._visit(t.default) + + def visit_param_spec(self, t: types.ParamSpecType) -> set[str]: + return self._visit(t.upper_bound) | self._visit(t.default) + + def visit_type_var_tuple(self, t: types.TypeVarTupleType) -> set[str]: + return self._visit(t.upper_bound) | self._visit(t.default) + + def visit_unpack_type(self, t: types.UnpackType) -> set[str]: + return t.type.accept(self) + + def visit_parameters(self, t: types.Parameters) -> set[str]: + return self._visit(t.arg_types) + + def visit_instance(self, t: types.Instance) -> set[str]: + out = self._visit(t.args) + if t.type: + # Uses of a class depend on everything in the MRO, + # as changes to classes in the MRO can add types to methods, + # change property types, change the MRO itself, etc. + for s in t.type.mro: + out.update(split_module_names(s.module_name)) + if t.type.metaclass_type is not None: + out.update(split_module_names(t.type.metaclass_type.type.module_name)) + return out + + def visit_callable_type(self, t: types.CallableType) -> set[str]: + out = self._visit(t.arg_types) | self._visit(t.ret_type) + if t.definition is not None: + out.update(extract_module_names(t.definition.fullname)) + return out + + def visit_overloaded(self, t: types.Overloaded) -> set[str]: + return self._visit(t.items) | self._visit(t.fallback) + + def visit_tuple_type(self, t: types.TupleType) -> set[str]: + return self._visit(t.items) | self._visit(t.partial_fallback) + + def visit_typeddict_type(self, t: types.TypedDictType) -> set[str]: + return self._visit(t.items.values()) | self._visit(t.fallback) + + def visit_literal_type(self, t: types.LiteralType) -> set[str]: + return self._visit(t.fallback) + + def visit_union_type(self, t: types.UnionType) -> set[str]: + return self._visit(t.items) + + def visit_partial_type(self, t: types.PartialType) -> set[str]: + return set() + + def visit_type_type(self, t: types.TypeType) -> set[str]: + return self._visit(t.item) + + def visit_type_alias_type(self, t: types.TypeAliasType) -> set[str]: + return self._visit(types.get_proper_type(t)) diff --git a/mypy/infer.py b/mypy/infer.py index 70289ca68829..bcf0c95808ab 100644 --- a/mypy/infer.py +++ b/mypy/infer.py @@ -1,17 +1,44 @@ """Utilities for type argument inference.""" -from typing import List +from __future__ import annotations -from mypy.constraints import infer_constraints, infer_constraints_for_callable -from mypy.types import Type, Callable +from typing import NamedTuple, Sequence + +from mypy.constraints import ( + SUBTYPE_OF, + SUPERTYPE_OF, + infer_constraints, + infer_constraints_for_callable, +) +from mypy.nodes import ArgKind from mypy.solve import solve_constraints -from mypy.constraints import SUBTYPE_OF +from mypy.types import CallableType, Instance, Type, TypeVarLikeType + + +class ArgumentInferContext(NamedTuple): + """Type argument inference context. + + We need this because we pass around ``Mapping`` and ``Iterable`` types. + These types are only known by ``TypeChecker`` itself. + It is required for ``*`` and ``**`` argument inference. + + https://github.com/python/mypy/issues/11144 + """ + + mapping_type: Instance + iterable_type: Instance -def infer_function_type_arguments(callee_type: Callable, - arg_types: List[Type], - arg_kinds: List[int], - formal_to_actual: List[List[int]]) -> List[Type]: +def infer_function_type_arguments( + callee_type: CallableType, + arg_types: Sequence[Type | None], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None] | None, + formal_to_actual: list[list[int]], + context: ArgumentInferContext, + strict: bool = True, + allow_polymorphic: bool = False, +) -> tuple[list[Type | None], list[TypeVarLikeType]]: """Infer the type arguments of a generic function. Return an array of lower bound types for the type variables -1 (at @@ -20,22 +47,29 @@ def infer_function_type_arguments(callee_type: Callable, Arguments: callee_type: the target generic function - arg_types: argument types at the call site + arg_types: argument types at the call site (each optional; if None, + we are not considering this argument in the current pass) arg_kinds: nodes.ARG_* values for arg_types formal_to_actual: mapping from formal to actual variable indices """ # Infer constraints. constraints = infer_constraints_for_callable( - callee_type, arg_types, arg_kinds, formal_to_actual) + callee_type, arg_types, arg_kinds, arg_names, formal_to_actual, context + ) # Solve constraints. - type_vars = callee_type.type_var_ids() - return solve_constraints(type_vars, constraints) + type_vars = callee_type.variables + return solve_constraints(type_vars, constraints, strict, allow_polymorphic) -def infer_type_arguments(type_var_ids: List[int], - template: Type, actual: Type) -> List[Type]: +def infer_type_arguments( + type_vars: Sequence[TypeVarLikeType], + template: Type, + actual: Type, + is_supertype: bool = False, + skip_unsatisfied: bool = False, +) -> list[Type | None]: # Like infer_function_type_arguments, but only match a single type # against a generic type. - constraints = infer_constraints(template, actual, SUBTYPE_OF) - return solve_constraints(type_var_ids, constraints) + constraints = infer_constraints(template, actual, SUPERTYPE_OF if is_supertype else SUBTYPE_OF) + return solve_constraints(type_vars, constraints, skip_unsatisfied=skip_unsatisfied)[0] diff --git a/mypy/inspections.py b/mypy/inspections.py new file mode 100644 index 000000000000..0baf0896f7e5 --- /dev/null +++ b/mypy/inspections.py @@ -0,0 +1,626 @@ +from __future__ import annotations + +import os +from collections import defaultdict +from functools import cmp_to_key +from typing import Callable + +from mypy.build import State +from mypy.messages import format_type +from mypy.modulefinder import PYTHON_EXTENSIONS +from mypy.nodes import ( + LDEF, + Decorator, + Expression, + FuncBase, + MemberExpr, + MypyFile, + Node, + OverloadedFuncDef, + RefExpr, + SymbolNode, + TypeInfo, + Var, +) +from mypy.server.update import FineGrainedBuildManager +from mypy.traverser import ExtendedTraverserVisitor +from mypy.typeops import tuple_fallback +from mypy.types import ( + FunctionLike, + Instance, + LiteralType, + ProperType, + TupleType, + TypedDictType, + TypeVarType, + UnionType, + get_proper_type, +) +from mypy.typevars import fill_typevars_with_any + + +def node_starts_after(o: Node, line: int, column: int) -> bool: + return o.line > line or o.line == line and o.column > column + + +def node_ends_before(o: Node, line: int, column: int) -> bool: + # Unfortunately, end positions for some statements are a mess, + # e.g. overloaded functions, so we return False when we don't know. + if o.end_line is not None and o.end_column is not None: + if o.end_line < line or o.end_line == line and o.end_column < column: + return True + return False + + +def expr_span(expr: Expression) -> str: + """Format expression span as in mypy error messages.""" + return f"{expr.line}:{expr.column + 1}:{expr.end_line}:{expr.end_column}" + + +def get_instance_fallback(typ: ProperType) -> list[Instance]: + """Returns the Instance fallback for this type if one exists or None.""" + if isinstance(typ, Instance): + return [typ] + elif isinstance(typ, TupleType): + return [tuple_fallback(typ)] + elif isinstance(typ, TypedDictType): + return [typ.fallback] + elif isinstance(typ, FunctionLike): + return [typ.fallback] + elif isinstance(typ, LiteralType): + return [typ.fallback] + elif isinstance(typ, TypeVarType): + if typ.values: + res = [] + for t in typ.values: + res.extend(get_instance_fallback(get_proper_type(t))) + return res + return get_instance_fallback(get_proper_type(typ.upper_bound)) + elif isinstance(typ, UnionType): + res = [] + for t in typ.items: + res.extend(get_instance_fallback(get_proper_type(t))) + return res + return [] + + +def find_node(name: str, info: TypeInfo) -> Var | FuncBase | None: + """Find the node defining member 'name' in given TypeInfo.""" + # TODO: this code shares some logic with checkmember.py + method = info.get_method(name) + if method: + if isinstance(method, Decorator): + return method.var + if method.is_property: + assert isinstance(method, OverloadedFuncDef) + dec = method.items[0] + assert isinstance(dec, Decorator) + return dec.var + return method + else: + # don't have such method, maybe variable? + node = info.get(name) + v = node.node if node else None + if isinstance(v, Var): + return v + return None + + +def find_module_by_fullname(fullname: str, modules: dict[str, State]) -> State | None: + """Find module by a node fullname. + + This logic mimics the one we use in fixup, so should be good enough. + """ + head = fullname + # Special case: a module symbol is considered to be defined in itself, not in enclosing + # package, since this is what users want when clicking go to definition on a module. + if head in modules: + return modules[head] + while True: + if "." not in head: + return None + head, tail = head.rsplit(".", maxsplit=1) + mod = modules.get(head) + if mod is not None: + return mod + + +class SearchVisitor(ExtendedTraverserVisitor): + """Visitor looking for an expression whose span matches given one exactly.""" + + def __init__(self, line: int, column: int, end_line: int, end_column: int) -> None: + self.line = line + self.column = column + self.end_line = end_line + self.end_column = end_column + self.result: Expression | None = None + + def visit(self, o: Node) -> bool: + if node_starts_after(o, self.line, self.column): + return False + if node_ends_before(o, self.end_line, self.end_column): + return False + if ( + o.line == self.line + and o.end_line == self.end_line + and o.column == self.column + and o.end_column == self.end_column + ): + if isinstance(o, Expression): + self.result = o + return self.result is None + + +def find_by_location( + tree: MypyFile, line: int, column: int, end_line: int, end_column: int +) -> Expression | None: + """Find an expression matching given span, or None if not found.""" + if end_line < line: + raise ValueError('"end_line" must not be before "line"') + if end_line == line and end_column <= column: + raise ValueError('"end_column" must be after "column"') + visitor = SearchVisitor(line, column, end_line, end_column) + tree.accept(visitor) + return visitor.result + + +class SearchAllVisitor(ExtendedTraverserVisitor): + """Visitor looking for all expressions whose spans enclose given position.""" + + def __init__(self, line: int, column: int) -> None: + self.line = line + self.column = column + self.result: list[Expression] = [] + + def visit(self, o: Node) -> bool: + if node_starts_after(o, self.line, self.column): + return False + if node_ends_before(o, self.line, self.column): + return False + if isinstance(o, Expression): + self.result.append(o) + return True + + +def find_all_by_location(tree: MypyFile, line: int, column: int) -> list[Expression]: + """Find all expressions enclosing given position starting from innermost.""" + visitor = SearchAllVisitor(line, column) + tree.accept(visitor) + return list(reversed(visitor.result)) + + +class InspectionEngine: + """Engine for locating and statically inspecting expressions.""" + + def __init__( + self, + fg_manager: FineGrainedBuildManager, + *, + verbosity: int = 0, + limit: int = 0, + include_span: bool = False, + include_kind: bool = False, + include_object_attrs: bool = False, + union_attrs: bool = False, + force_reload: bool = False, + ) -> None: + self.fg_manager = fg_manager + self.verbosity = verbosity + self.limit = limit + self.include_span = include_span + self.include_kind = include_kind + self.include_object_attrs = include_object_attrs + self.union_attrs = union_attrs + self.force_reload = force_reload + # Module for which inspection was requested. + self.module: State | None = None + + def reload_module(self, state: State) -> None: + """Reload given module while temporary exporting types.""" + old = self.fg_manager.manager.options.export_types + self.fg_manager.manager.options.export_types = True + try: + self.fg_manager.flush_cache() + assert state.path is not None + self.fg_manager.update([(state.id, state.path)], []) + finally: + self.fg_manager.manager.options.export_types = old + + def expr_type(self, expression: Expression) -> tuple[str, bool]: + """Format type for an expression using current options. + + If type is known, second item returned is True. If type is not known, an error + message is returned instead, and second item returned is False. + """ + expr_type = self.fg_manager.manager.all_types.get(expression) + if expr_type is None: + return self.missing_type(expression), False + + type_str = format_type( + expr_type, self.fg_manager.manager.options, verbosity=self.verbosity + ) + return self.add_prefixes(type_str, expression), True + + def object_type(self) -> Instance: + builtins = self.fg_manager.graph["builtins"].tree + assert builtins is not None + object_node = builtins.names["object"].node + assert isinstance(object_node, TypeInfo) + return Instance(object_node, []) + + def collect_attrs(self, instances: list[Instance]) -> dict[TypeInfo, list[str]]: + """Collect attributes from all union/typevar variants.""" + + def item_attrs(attr_dict: dict[TypeInfo, list[str]]) -> set[str]: + attrs = set() + for base in attr_dict: + attrs |= set(attr_dict[base]) + return attrs + + def cmp_types(x: TypeInfo, y: TypeInfo) -> int: + if x in y.mro: + return 1 + if y in x.mro: + return -1 + return 0 + + # First gather all attributes for every union variant. + assert instances + all_attrs = [] + for instance in instances: + attrs = {} + mro = instance.type.mro + if not self.include_object_attrs: + mro = mro[:-1] + for base in mro: + attrs[base] = sorted(base.names) + all_attrs.append(attrs) + + # Find attributes valid for all variants in a union or type variable. + intersection = item_attrs(all_attrs[0]) + for item in all_attrs[1:]: + intersection &= item_attrs(item) + + # Combine attributes from all variants into a single dict while + # also removing invalid attributes (unless using --union-attrs). + combined_attrs = defaultdict(list) + for item in all_attrs: + for base in item: + if base in combined_attrs: + continue + for name in item[base]: + if self.union_attrs or name in intersection: + combined_attrs[base].append(name) + + # Sort bases by MRO, unrelated will appear in the order they appeared as union variants. + sorted_bases = sorted(combined_attrs.keys(), key=cmp_to_key(cmp_types)) + result = {} + for base in sorted_bases: + if not combined_attrs[base]: + # Skip bases where everytihng was filtered out. + continue + result[base] = combined_attrs[base] + return result + + def _fill_from_dict( + self, attrs_strs: list[str], attrs_dict: dict[TypeInfo, list[str]] + ) -> None: + for base in attrs_dict: + cls_name = base.name if self.verbosity < 1 else base.fullname + attrs = [f'"{attr}"' for attr in attrs_dict[base]] + attrs_strs.append(f'"{cls_name}": [{", ".join(attrs)}]') + + def expr_attrs(self, expression: Expression) -> tuple[str, bool]: + """Format attributes that are valid for a given expression. + + If expression type is not an Instance, try using fallback. Attributes are + returned as a JSON (ordered by MRO) that maps base class name to list of + attributes. Attributes may appear in multiple bases if overridden (we simply + follow usual mypy logic for creating new Vars etc). + """ + expr_type = self.fg_manager.manager.all_types.get(expression) + if expr_type is None: + return self.missing_type(expression), False + + expr_type = get_proper_type(expr_type) + instances = get_instance_fallback(expr_type) + if not instances: + # Everything is an object in Python. + instances = [self.object_type()] + + attrs_dict = self.collect_attrs(instances) + + # Special case: modules have names apart from those from ModuleType. + if isinstance(expression, RefExpr) and isinstance(expression.node, MypyFile): + node = expression.node + names = sorted(node.names) + if "__builtins__" in names: + # This is just to make tests stable. No one will really need ths name. + names.remove("__builtins__") + mod_dict = {f'"<{node.fullname}>"': [f'"{name}"' for name in names]} + else: + mod_dict = {} + + # Special case: for class callables, prepend with the class attributes. + # TODO: also handle cases when such callable appears in a union. + if isinstance(expr_type, FunctionLike) and expr_type.is_type_obj(): + template = fill_typevars_with_any(expr_type.type_object()) + class_dict = self.collect_attrs(get_instance_fallback(template)) + else: + class_dict = {} + + # We don't use JSON dump to be sure keys order is always preserved. + base_attrs = [] + if mod_dict: + for mod in mod_dict: + base_attrs.append(f'{mod}: [{", ".join(mod_dict[mod])}]') + self._fill_from_dict(base_attrs, class_dict) + self._fill_from_dict(base_attrs, attrs_dict) + return self.add_prefixes(f'{{{", ".join(base_attrs)}}}', expression), True + + def format_node(self, module: State, node: FuncBase | SymbolNode) -> str: + return f"{module.path}:{node.line}:{node.column + 1}:{node.name}" + + def collect_nodes(self, expression: RefExpr) -> list[FuncBase | SymbolNode]: + """Collect nodes that can be referred to by an expression. + + Note: it can be more than one for example in case of a union attribute. + """ + node: FuncBase | SymbolNode | None = expression.node + nodes: list[FuncBase | SymbolNode] + if node is None: + # Tricky case: instance attribute + if isinstance(expression, MemberExpr) and expression.kind is None: + base_type = self.fg_manager.manager.all_types.get(expression.expr) + if base_type is None: + return [] + + # Now we use the base type to figure out where the attribute is defined. + base_type = get_proper_type(base_type) + instances = get_instance_fallback(base_type) + nodes = [] + for instance in instances: + node = find_node(expression.name, instance.type) + if node: + nodes.append(node) + if not nodes: + # Try checking class namespace if attribute is on a class object. + if isinstance(base_type, FunctionLike) and base_type.is_type_obj(): + instances = get_instance_fallback( + fill_typevars_with_any(base_type.type_object()) + ) + for instance in instances: + node = find_node(expression.name, instance.type) + if node: + nodes.append(node) + else: + # Still no luck, give up. + return [] + else: + return [] + else: + # Easy case: a module-level definition + nodes = [node] + return nodes + + def modules_for_nodes( + self, nodes: list[FuncBase | SymbolNode], expression: RefExpr + ) -> tuple[dict[FuncBase | SymbolNode, State], bool]: + """Gather modules where given nodes where defined. + + Also check if they need to be refreshed (cached nodes may have + lines/columns missing). + """ + modules = {} + reload_needed = False + for node in nodes: + module = find_module_by_fullname(node.fullname, self.fg_manager.graph) + if not module: + if expression.kind == LDEF and self.module: + module = self.module + else: + continue + modules[node] = module + if not module.tree or module.tree.is_cache_skeleton or self.force_reload: + reload_needed |= not module.tree or module.tree.is_cache_skeleton + self.reload_module(module) + return modules, reload_needed + + def expression_def(self, expression: Expression) -> tuple[str, bool]: + """Find and format definition location for an expression. + + If it is not a RefExpr, it is effectively skipped by returning an + empty result. + """ + if not isinstance(expression, RefExpr): + # If there are no suitable matches at all, we return error later. + return "", True + + nodes = self.collect_nodes(expression) + + if not nodes: + return self.missing_node(expression), False + + modules, reload_needed = self.modules_for_nodes(nodes, expression) + if reload_needed: + # TODO: line/column are not stored in cache for vast majority of symbol nodes. + # Adding them will make thing faster, but will have visible memory impact. + nodes = self.collect_nodes(expression) + modules, reload_needed = self.modules_for_nodes(nodes, expression) + assert not reload_needed + + result = [] + for node in modules: + result.append(self.format_node(modules[node], node)) + + if not result: + return self.missing_node(expression), False + + return self.add_prefixes(", ".join(result), expression), True + + def missing_type(self, expression: Expression) -> str: + alt_suggestion = "" + if not self.force_reload: + alt_suggestion = " or try --force-reload" + return ( + f'No known type available for "{type(expression).__name__}"' + f" (maybe unreachable{alt_suggestion})" + ) + + def missing_node(self, expression: Expression) -> str: + return ( + f'Cannot find definition for "{type(expression).__name__}" at {expr_span(expression)}' + ) + + def add_prefixes(self, result: str, expression: Expression) -> str: + prefixes = [] + if self.include_kind: + prefixes.append(f"{type(expression).__name__}") + if self.include_span: + prefixes.append(expr_span(expression)) + if prefixes: + prefix = ":".join(prefixes) + " -> " + else: + prefix = "" + return prefix + result + + def run_inspection_by_exact_location( + self, + tree: MypyFile, + line: int, + column: int, + end_line: int, + end_column: int, + method: Callable[[Expression], tuple[str, bool]], + ) -> dict[str, object]: + """Get type of an expression matching a span. + + Type or error is returned as a standard daemon response dict. + """ + try: + expression = find_by_location(tree, line, column - 1, end_line, end_column) + except ValueError as err: + return {"error": str(err)} + + if expression is None: + span = f"{line}:{column}:{end_line}:{end_column}" + return {"out": f"Can't find expression at span {span}", "err": "", "status": 1} + + inspection_str, success = method(expression) + return {"out": inspection_str, "err": "", "status": 0 if success else 1} + + def run_inspection_by_position( + self, + tree: MypyFile, + line: int, + column: int, + method: Callable[[Expression], tuple[str, bool]], + ) -> dict[str, object]: + """Get types of all expressions enclosing a position. + + Types and/or errors are returned as a standard daemon response dict. + """ + expressions = find_all_by_location(tree, line, column - 1) + if not expressions: + position = f"{line}:{column}" + return { + "out": f"Can't find any expressions at position {position}", + "err": "", + "status": 1, + } + + inspection_strs = [] + status = 0 + for expression in expressions: + inspection_str, success = method(expression) + if not success: + status = 1 + if inspection_str: + inspection_strs.append(inspection_str) + if self.limit: + inspection_strs = inspection_strs[: self.limit] + return {"out": "\n".join(inspection_strs), "err": "", "status": status} + + def find_module(self, file: str) -> tuple[State | None, dict[str, object]]: + """Find module by path, or return a suitable error message. + + Note we don't use exceptions to simplify handling 1 vs 2 statuses. + """ + if not any(file.endswith(ext) for ext in PYTHON_EXTENSIONS): + return None, {"error": "Source file is not a Python file"} + + # We are using a bit slower but robust way to find a module by path, + # to be sure that namespace packages are handled properly. + abs_path = os.path.abspath(file) + state = next((s for s in self.fg_manager.graph.values() if s.abspath == abs_path), None) + self.module = state + return ( + state, + {"out": f"Unknown module: {file}", "err": "", "status": 1} if state is None else {}, + ) + + def run_inspection( + self, location: str, method: Callable[[Expression], tuple[str, bool]] + ) -> dict[str, object]: + """Top-level logic to inspect expression(s) at a location. + + This can be re-used by various simple inspections. + """ + try: + file, pos = parse_location(location) + except ValueError as err: + return {"error": str(err)} + + state, err_dict = self.find_module(file) + if state is None: + assert err_dict + return err_dict + + # Force reloading to load from cache, account for any edits, etc. + if not state.tree or state.tree.is_cache_skeleton or self.force_reload: + self.reload_module(state) + assert state.tree is not None + + if len(pos) == 4: + # Full span, return an exact match only. + line, column, end_line, end_column = pos + return self.run_inspection_by_exact_location( + state.tree, line, column, end_line, end_column, method + ) + assert len(pos) == 2 + # Inexact location, return all expressions. + line, column = pos + return self.run_inspection_by_position(state.tree, line, column, method) + + def get_type(self, location: str) -> dict[str, object]: + """Get types of expression(s) at a location.""" + return self.run_inspection(location, self.expr_type) + + def get_attrs(self, location: str) -> dict[str, object]: + """Get attributes of expression(s) at a location.""" + return self.run_inspection(location, self.expr_attrs) + + def get_definition(self, location: str) -> dict[str, object]: + """Get symbol definitions of expression(s) at a location.""" + result = self.run_inspection(location, self.expression_def) + if "out" in result and not result["out"]: + # None of the expressions found turns out to be a RefExpr. + _, location = location.split(":", maxsplit=1) + result["out"] = f"No name or member expressions at {location}" + result["status"] = 1 + return result + + +def parse_location(location: str) -> tuple[str, list[int]]: + if location.count(":") < 2: + raise ValueError("Format should be file:line:column[:end_line:end_column]") + parts = location.rsplit(":", maxsplit=2) + start, *rest = parts + # Note: we must allow drive prefix like `C:` on Windows. + if start.count(":") < 2: + return start, [int(p) for p in rest] + parts = start.rsplit(":", maxsplit=2) + start, *start_rest = parts + if start.count(":") < 2: + return start, [int(p) for p in start_rest + rest] + raise ValueError("Format should be file:line:column[:end_line:end_column]") diff --git a/mypy/ipc.py b/mypy/ipc.py new file mode 100644 index 000000000000..991f9ac56652 --- /dev/null +++ b/mypy/ipc.py @@ -0,0 +1,309 @@ +"""Cross platform abstractions for inter-process communication + +On Unix, this uses AF_UNIX sockets. +On Windows, this uses NamedPipes. +""" + +from __future__ import annotations + +import base64 +import codecs +import os +import shutil +import sys +import tempfile +from types import TracebackType +from typing import Callable, Final + +if sys.platform == "win32": + # This may be private, but it is needed for IPC on Windows, and is basically stable + import _winapi + import ctypes + + _IPCHandle = int + + kernel32 = ctypes.windll.kernel32 + DisconnectNamedPipe: Callable[[_IPCHandle], int] = kernel32.DisconnectNamedPipe + FlushFileBuffers: Callable[[_IPCHandle], int] = kernel32.FlushFileBuffers +else: + import socket + + _IPCHandle = socket.socket + + +class IPCException(Exception): + """Exception for IPC issues.""" + + +class IPCBase: + """Base class for communication between the dmypy client and server. + + This contains logic shared between the client and server, such as reading + and writing. + We want to be able to send multiple "messages" over a single connection and + to be able to separate the messages. We do this by encoding the messages + in an alphabet that does not contain spaces, then adding a space for + separation. The last framed message is also followed by a space. + """ + + connection: _IPCHandle + + def __init__(self, name: str, timeout: float | None) -> None: + self.name = name + self.timeout = timeout + self.buffer = bytearray() + + def frame_from_buffer(self) -> bytearray | None: + """Return a full frame from the bytes we have in the buffer.""" + space_pos = self.buffer.find(b" ") + if space_pos == -1: + return None + # We have a full frame + bdata = self.buffer[:space_pos] + self.buffer = self.buffer[space_pos + 1 :] + return bdata + + def read(self, size: int = 100000) -> str: + """Read bytes from an IPC connection until we have a full frame.""" + bdata: bytearray | None = bytearray() + if sys.platform == "win32": + while True: + # Check if we already have a message in the buffer before + # receiving any more data from the socket. + bdata = self.frame_from_buffer() + if bdata is not None: + break + + # Receive more data into the buffer. + ov, err = _winapi.ReadFile(self.connection, size, overlapped=True) + try: + if err == _winapi.ERROR_IO_PENDING: + timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE + res = _winapi.WaitForSingleObject(ov.event, timeout) + if res != _winapi.WAIT_OBJECT_0: + raise IPCException(f"Bad result from I/O wait: {res}") + except BaseException: + ov.cancel() + raise + _, err = ov.GetOverlappedResult(True) + more = ov.getbuffer() + if more: + self.buffer.extend(more) + bdata = self.frame_from_buffer() + if bdata is not None: + break + if err == 0: + # we are done! + break + elif err == _winapi.ERROR_MORE_DATA: + # read again + continue + elif err == _winapi.ERROR_OPERATION_ABORTED: + raise IPCException("ReadFile operation aborted.") + else: + while True: + # Check if we already have a message in the buffer before + # receiving any more data from the socket. + bdata = self.frame_from_buffer() + if bdata is not None: + break + + # Receive more data into the buffer. + more = self.connection.recv(size) + if not more: + # Connection closed + break + self.buffer.extend(more) + + if not bdata: + # Socket was empty and we didn't get any frame. + # This should only happen if the socket was closed. + return "" + return codecs.decode(bdata, "base64").decode("utf8") + + def write(self, data: str) -> None: + """Write to an IPC connection.""" + + # Frame the data by urlencoding it and separating by space. + encoded_data = codecs.encode(data.encode("utf8"), "base64") + b" " + + if sys.platform == "win32": + try: + ov, err = _winapi.WriteFile(self.connection, encoded_data, overlapped=True) + try: + if err == _winapi.ERROR_IO_PENDING: + timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE + res = _winapi.WaitForSingleObject(ov.event, timeout) + if res != _winapi.WAIT_OBJECT_0: + raise IPCException(f"Bad result from I/O wait: {res}") + elif err != 0: + raise IPCException(f"Failed writing to pipe with error: {err}") + except BaseException: + ov.cancel() + raise + bytes_written, err = ov.GetOverlappedResult(True) + assert err == 0, err + assert bytes_written == len(encoded_data) + except OSError as e: + raise IPCException(f"Failed to write with error: {e.winerror}") from e + else: + self.connection.sendall(encoded_data) + + def close(self) -> None: + if sys.platform == "win32": + if self.connection != _winapi.NULL: + _winapi.CloseHandle(self.connection) + else: + self.connection.close() + + +class IPCClient(IPCBase): + """The client side of an IPC connection.""" + + def __init__(self, name: str, timeout: float | None) -> None: + super().__init__(name, timeout) + if sys.platform == "win32": + timeout = int(self.timeout * 1000) if self.timeout else _winapi.NMPWAIT_WAIT_FOREVER + try: + _winapi.WaitNamedPipe(self.name, timeout) + except FileNotFoundError as e: + raise IPCException(f"The NamedPipe at {self.name} was not found.") from e + except OSError as e: + if e.winerror == _winapi.ERROR_SEM_TIMEOUT: + raise IPCException("Timed out waiting for connection.") from e + else: + raise + try: + self.connection = _winapi.CreateFile( + self.name, + _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, + 0, + _winapi.NULL, + _winapi.OPEN_EXISTING, + _winapi.FILE_FLAG_OVERLAPPED, + _winapi.NULL, + ) + except OSError as e: + if e.winerror == _winapi.ERROR_PIPE_BUSY: + raise IPCException("The connection is busy.") from e + else: + raise + _winapi.SetNamedPipeHandleState( + self.connection, _winapi.PIPE_READMODE_MESSAGE, None, None + ) + else: + self.connection = socket.socket(socket.AF_UNIX) + self.connection.settimeout(timeout) + self.connection.connect(name) + + def __enter__(self) -> IPCClient: + return self + + def __exit__( + self, + exc_ty: type[BaseException] | None = None, + exc_val: BaseException | None = None, + exc_tb: TracebackType | None = None, + ) -> None: + self.close() + + +class IPCServer(IPCBase): + BUFFER_SIZE: Final = 2**16 + + def __init__(self, name: str, timeout: float | None = None) -> None: + if sys.platform == "win32": + name = r"\\.\pipe\{}-{}.pipe".format( + name, base64.urlsafe_b64encode(os.urandom(6)).decode() + ) + else: + name = f"{name}.sock" + super().__init__(name, timeout) + if sys.platform == "win32": + self.connection = _winapi.CreateNamedPipe( + self.name, + _winapi.PIPE_ACCESS_DUPLEX + | _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE + | _winapi.FILE_FLAG_OVERLAPPED, + _winapi.PIPE_READMODE_MESSAGE + | _winapi.PIPE_TYPE_MESSAGE + | _winapi.PIPE_WAIT + | 0x8, # PIPE_REJECT_REMOTE_CLIENTS + 1, # one instance + self.BUFFER_SIZE, + self.BUFFER_SIZE, + _winapi.NMPWAIT_WAIT_FOREVER, + 0, # Use default security descriptor + ) + if self.connection == -1: # INVALID_HANDLE_VALUE + err = _winapi.GetLastError() + raise IPCException(f"Invalid handle to pipe: {err}") + else: + self.sock_directory = tempfile.mkdtemp() + sockfile = os.path.join(self.sock_directory, self.name) + self.sock = socket.socket(socket.AF_UNIX) + self.sock.bind(sockfile) + self.sock.listen(1) + if timeout is not None: + self.sock.settimeout(timeout) + + def __enter__(self) -> IPCServer: + if sys.platform == "win32": + # NOTE: It is theoretically possible that this will hang forever if the + # client never connects, though this can be "solved" by killing the server + try: + ov = _winapi.ConnectNamedPipe(self.connection, overlapped=True) + except OSError as e: + # Don't raise if the client already exists, or the client already connected + if e.winerror not in (_winapi.ERROR_PIPE_CONNECTED, _winapi.ERROR_NO_DATA): + raise + else: + try: + timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE + res = _winapi.WaitForSingleObject(ov.event, timeout) + assert res == _winapi.WAIT_OBJECT_0 + except BaseException: + ov.cancel() + _winapi.CloseHandle(self.connection) + raise + _, err = ov.GetOverlappedResult(True) + assert err == 0 + else: + try: + self.connection, _ = self.sock.accept() + except socket.timeout as e: + raise IPCException("The socket timed out") from e + return self + + def __exit__( + self, + exc_ty: type[BaseException] | None = None, + exc_val: BaseException | None = None, + exc_tb: TracebackType | None = None, + ) -> None: + if sys.platform == "win32": + try: + # Wait for the client to finish reading the last write before disconnecting + if not FlushFileBuffers(self.connection): + raise IPCException( + "Failed to flush NamedPipe buffer, maybe the client hung up?" + ) + finally: + DisconnectNamedPipe(self.connection) + else: + self.close() + + def cleanup(self) -> None: + if sys.platform == "win32": + self.close() + else: + shutil.rmtree(self.sock_directory) + + @property + def connection_name(self) -> str: + if sys.platform == "win32": + return self.name + else: + name = self.sock.getsockname() + assert isinstance(name, str) + return name diff --git a/mypy/join.py b/mypy/join.py index d3aa7849b4fd..2ada7479789b 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -1,250 +1,859 @@ """Calculation of the least upper bound types (joins).""" -from typing import cast, List +from __future__ import annotations +from typing import Sequence, overload + +import mypy.typeops +from mypy.expandtype import expand_type +from mypy.maptype import map_instance_to_supertype +from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT, VARIANCE_NOT_READY +from mypy.state import state +from mypy.subtypes import ( + SubtypeContext, + find_member, + is_equivalent, + is_proper_subtype, + is_protocol_implementation, + is_subtype, +) from mypy.types import ( - Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, - ErrorType, TypeVar, Callable, TupleType, ErasedType, TypeList, - UnionType, FunctionLike + AnyType, + CallableType, + DeletedType, + ErasedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + find_unpack_in_list, + get_proper_type, + get_proper_types, + split_with_prefix_and_suffix, ) -from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent -def join_simple(declaration: Type, s: Type, t: Type) -> Type: - """Return a simple least upper bound given the declared type.""" +class InstanceJoiner: + def __init__(self) -> None: + self.seen_instances: list[tuple[Instance, Instance]] = [] + + def join_instances(self, t: Instance, s: Instance) -> ProperType: + if (t, s) in self.seen_instances or (s, t) in self.seen_instances: + return object_from_instance(t) + + self.seen_instances.append((t, s)) + + # Calculate the join of two instance types + if t.type == s.type: + # Simplest case: join two types with the same base type (but + # potentially different arguments). + + # Combine type arguments. + args: list[Type] = [] + # N.B: We use zip instead of indexing because the lengths might have + # mismatches during daemon reprocessing. + if t.type.has_type_var_tuple_type: + # We handle joins of variadic instances by simply creating correct mapping + # for type arguments and compute the individual joins same as for regular + # instances. All the heavy lifting is done in the join of tuple types. + assert s.type.type_var_tuple_prefix is not None + assert s.type.type_var_tuple_suffix is not None + prefix = s.type.type_var_tuple_prefix + suffix = s.type.type_var_tuple_suffix + tvt = s.type.defn.type_vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + s_prefix, s_middle, s_suffix = split_with_prefix_and_suffix(s.args, prefix, suffix) + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix(t.args, prefix, suffix) + s_args = s_prefix + (TupleType(list(s_middle), fallback),) + s_suffix + t_args = t_prefix + (TupleType(list(t_middle), fallback),) + t_suffix + else: + t_args = t.args + s_args = s.args + for ta, sa, type_var in zip(t_args, s_args, t.type.defn.type_vars): + ta_proper = get_proper_type(ta) + sa_proper = get_proper_type(sa) + new_type: Type | None = None + if isinstance(ta_proper, AnyType): + new_type = AnyType(TypeOfAny.from_another_any, ta_proper) + elif isinstance(sa_proper, AnyType): + new_type = AnyType(TypeOfAny.from_another_any, sa_proper) + elif isinstance(type_var, TypeVarType): + if type_var.variance in (COVARIANT, VARIANCE_NOT_READY): + new_type = join_types(ta, sa, self) + if len(type_var.values) != 0 and new_type not in type_var.values: + self.seen_instances.pop() + return object_from_instance(t) + if not is_subtype(new_type, type_var.upper_bound): + self.seen_instances.pop() + return object_from_instance(t) + # TODO: contravariant case should use meet but pass seen instances as + # an argument to keep track of recursive checks. + elif type_var.variance in (INVARIANT, CONTRAVARIANT): + if isinstance(ta_proper, UninhabitedType) and ta_proper.ambiguous: + new_type = sa + elif isinstance(sa_proper, UninhabitedType) and sa_proper.ambiguous: + new_type = ta + elif not is_equivalent(ta, sa): + self.seen_instances.pop() + return object_from_instance(t) + else: + # If the types are different but equivalent, then an Any is involved + # so using a join in the contravariant case is also OK. + new_type = join_types(ta, sa, self) + elif isinstance(type_var, TypeVarTupleType): + new_type = get_proper_type(join_types(ta, sa, self)) + # Put the joined arguments back into instance in the normal form: + # a) Tuple[X, Y, Z] -> [X, Y, Z] + # b) tuple[X, ...] -> [*tuple[X, ...]] + if isinstance(new_type, Instance): + assert new_type.type.fullname == "builtins.tuple" + new_type = UnpackType(new_type) + else: + assert isinstance(new_type, TupleType) + args.extend(new_type.items) + continue + else: + # ParamSpec type variables behave the same, independent of variance + if not is_equivalent(ta, sa): + return get_proper_type(type_var.upper_bound) + new_type = join_types(ta, sa, self) + assert new_type is not None + args.append(new_type) + result: ProperType = Instance(t.type, args) + elif t.type.bases and is_proper_subtype( + t, s, subtype_context=SubtypeContext(ignore_type_params=True) + ): + result = self.join_instances_via_supertype(t, s) + else: + # Now t is not a subtype of s, and t != s. Now s could be a subtype + # of t; alternatively, we need to find a common supertype. This works + # in of the both cases. + result = self.join_instances_via_supertype(s, t) + + self.seen_instances.pop() + return result + + def join_instances_via_supertype(self, t: Instance, s: Instance) -> ProperType: + # Give preference to joins via duck typing relationship, so that + # join(int, float) == float, for example. + for p in t.type._promote: + if is_subtype(p, s): + return join_types(p, s, self) + for p in s.type._promote: + if is_subtype(p, t): + return join_types(t, p, self) + + # Compute the "best" supertype of t when joined with s. + # The definition of "best" may evolve; for now it is the one with + # the longest MRO. Ties are broken by using the earlier base. + best: ProperType | None = None + for base in t.type.bases: + mapped = map_instance_to_supertype(t, base.type) + res = self.join_instances(mapped, s) + if best is None or is_better(res, best): + best = res + assert best is not None + for promote in t.type._promote: + if isinstance(promote, Instance): + res = self.join_instances(promote, s) + if is_better(res, best): + best = res + return best + + +def join_simple(declaration: Type | None, s: Type, t: Type) -> ProperType: + """Return a simple least upper bound given the declared type. + + This function should be only used by binder, and should not recurse. + For all other uses, use `join_types()`. + """ + declaration = get_proper_type(declaration) + s = get_proper_type(s) + t = get_proper_type(t) + + if (s.can_be_true, s.can_be_false) != (t.can_be_true, t.can_be_false): + # if types are restricted in different ways, use the more general versions + s = mypy.typeops.true_or_false(s) + t = mypy.typeops.true_or_false(t) if isinstance(s, AnyType): return s - if isinstance(s, NoneTyp) and not isinstance(t, Void): - return t - if isinstance(s, ErasedType): return t - if is_subtype(s, t): + if is_proper_subtype(s, t, ignore_promotions=True): return t - if is_subtype(t, s): + if is_proper_subtype(t, s, ignore_promotions=True): return s if isinstance(declaration, UnionType): - return UnionType.make_simplified_union([s, t]) + return mypy.typeops.make_simplified_union([s, t]) - value = t.accept(TypeJoinVisitor(s)) + if isinstance(s, NoneType) and not isinstance(t, NoneType): + s, t = t, s + + if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): + s, t = t, s + + # Meets/joins require callable type normalization. + s, t = normalize_callables(s, t) - if value is None: - # XXX this code path probably should be avoided. - # It seems to happen when a line (x = y) is a type error, and - # it's not clear that assuming that x is arbitrary afterward - # is a good idea. - return declaration + if isinstance(s, UnionType) and not isinstance(t, UnionType): + s, t = t, s + value = t.accept(TypeJoinVisitor(s)) if declaration is None or is_subtype(value, declaration): return value return declaration -def join_types(s: Type, t: Type) -> Type: +def trivial_join(s: Type, t: Type) -> Type: + """Return one of types (expanded) if it is a supertype of other, otherwise top type.""" + if is_subtype(s, t): + return t + elif is_subtype(t, s): + return s + else: + return object_or_any_from_type(get_proper_type(t)) + + +@overload +def join_types( + s: ProperType, t: ProperType, instance_joiner: InstanceJoiner | None = None +) -> ProperType: ... + + +@overload +def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> Type: ... + + +def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> Type: """Return the least upper bound of s and t. For example, the join of 'int' and 'object' is 'object'. - - If the join does not exist, return an ErrorType instance. """ + if mypy.typeops.is_recursive_pair(s, t): + # This case can trigger an infinite recursion, general support for this will be + # tricky so we use a trivial join (like for protocols). + return trivial_join(s, t) + s = get_proper_type(s) + t = get_proper_type(t) + + if (s.can_be_true, s.can_be_false) != (t.can_be_true, t.can_be_false): + # if types are restricted in different ways, use the more general versions + s = mypy.typeops.true_or_false(s) + t = mypy.typeops.true_or_false(t) + + if isinstance(s, UnionType) and not isinstance(t, UnionType): + s, t = t, s if isinstance(s, AnyType): return s - if isinstance(s, NoneTyp) and not isinstance(t, Void): - return t - if isinstance(s, ErasedType): return t + if isinstance(s, NoneType) and not isinstance(t, NoneType): + s, t = t, s + + if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): + s, t = t, s + + # Meets/joins require callable type normalization. + s, t = normalize_callables(s, t) + # Use a visitor to handle non-trivial cases. - return t.accept(TypeJoinVisitor(s)) + return t.accept(TypeJoinVisitor(s, instance_joiner)) -class TypeJoinVisitor(TypeVisitor[Type]): - """Implementation of the least upper bound algorithm.""" +class TypeJoinVisitor(TypeVisitor[ProperType]): + """Implementation of the least upper bound algorithm. + + Attributes: + s: The other (left) type operand. + """ - def __init__(self, s: Type) -> None: + def __init__(self, s: ProperType, instance_joiner: InstanceJoiner | None = None) -> None: self.s = s + self.instance_joiner = instance_joiner - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - else: - return AnyType() + def visit_unbound_type(self, t: UnboundType) -> ProperType: + return AnyType(TypeOfAny.special_form) - def visit_union_type(self, t: UnionType) -> Type: - if is_subtype(self.s, t): + def visit_union_type(self, t: UnionType) -> ProperType: + if is_proper_subtype(self.s, t): return t else: - return UnionType(t.items + [self.s]) + return mypy.typeops.make_simplified_union([self.s, t]) - def visit_error_type(self, t: ErrorType) -> Type: + def visit_any(self, t: AnyType) -> ProperType: return t - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' + def visit_none_type(self, t: NoneType) -> ProperType: + if state.strict_optional: + if isinstance(self.s, (NoneType, UninhabitedType)): + return t + elif isinstance(self.s, (UnboundType, AnyType)): + return AnyType(TypeOfAny.special_form) + else: + return mypy.typeops.make_simplified_union([self.s, t]) + else: + return self.s - def visit_any(self, t: AnyType) -> Type: - return t + def visit_uninhabited_type(self, t: UninhabitedType) -> ProperType: + return self.s - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() + def visit_deleted_type(self, t: DeletedType) -> ProperType: + return self.s - def visit_none_type(self, t: NoneTyp) -> Type: - if not isinstance(self.s, Void): + def visit_erased_type(self, t: ErasedType) -> ProperType: + return self.s + + def visit_type_var(self, t: TypeVarType) -> ProperType: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: return self.s else: return self.default(self.s) - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s + def visit_param_spec(self, t: ParamSpecType) -> ProperType: + if self.s == t: + return t + return self.default(self.s) - def visit_type_var(self, t: TypeVar) -> Type: - if isinstance(self.s, TypeVar) and (cast(TypeVar, self.s)).id == t.id: - return self.s + def visit_type_var_tuple(self, t: TypeVarTupleType) -> ProperType: + if self.s == t: + return t + return self.default(self.s) + + def visit_unpack_type(self, t: UnpackType) -> UnpackType: + raise NotImplementedError + + def visit_parameters(self, t: Parameters) -> ProperType: + if isinstance(self.s, Parameters): + if len(t.arg_types) != len(self.s.arg_types): + return self.default(self.s) + from mypy.meet import meet_types + + return t.copy_modified( + arg_types=[ + meet_types(s_a, t_a) for s_a, t_a in zip(self.s.arg_types, t.arg_types) + ], + arg_names=combine_arg_names(self.s, t), + ) else: return self.default(self.s) - def visit_instance(self, t: Instance) -> Type: + def visit_instance(self, t: Instance) -> ProperType: if isinstance(self.s, Instance): - return join_instances(t, cast(Instance, self.s)) - elif t.type.fullname() == 'builtins.type' and is_subtype(self.s, t): - return t + if self.instance_joiner is None: + self.instance_joiner = InstanceJoiner() + nominal = self.instance_joiner.join_instances(t, self.s) + structural: Instance | None = None + if t.type.is_protocol and is_protocol_implementation(self.s, t): + structural = t + elif self.s.type.is_protocol and is_protocol_implementation(t, self.s): + structural = self.s + # Structural join is preferred in the case where we have found both + # structural and nominal and they have same MRO length (see two comments + # in join_instances_via_supertype). Otherwise, just return the nominal join. + if not structural or is_better(nominal, structural): + return nominal + return structural + elif isinstance(self.s, FunctionLike): + if t.type.is_protocol: + call = unpack_callback_protocol(t) + if call: + return join_types(call, self.s) + return join_types(t, self.s.fallback) + elif isinstance(self.s, TypeType): + return join_types(t, self.s) + elif isinstance(self.s, TypedDictType): + return join_types(t, self.s) + elif isinstance(self.s, TupleType): + return join_types(t, self.s) + elif isinstance(self.s, LiteralType): + return join_types(t, self.s) else: return self.default(self.s) - def visit_callable(self, t: Callable) -> Type: - if isinstance(self.s, Callable) and is_similar_callables( - t, cast(Callable, self.s)): - return combine_similar_callables(t, cast(Callable, self.s)) - elif t.is_type_obj() and is_subtype(self.s, t.fallback): - return t.fallback - elif (t.is_type_obj() and isinstance(self.s, Instance) and - cast(Instance, self.s).type == t.fallback): - return t.fallback + def visit_callable_type(self, t: CallableType) -> ProperType: + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + if is_equivalent(t, self.s): + return combine_similar_callables(t, self.s) + result = join_similar_callables(t, self.s) + # We set the from_type_type flag to suppress error when a collection of + # concrete class objects gets inferred as their common abstract superclass. + if not ( + (t.is_type_obj() and t.type_object().is_abstract) + or (self.s.is_type_obj() and self.s.type_object().is_abstract) + ): + result.from_type_type = True + if any( + isinstance(tp, (NoneType, UninhabitedType)) + for tp in get_proper_types(result.arg_types) + ): + # We don't want to return unusable Callable, attempt fallback instead. + return join_types(t.fallback, self.s) + return result + elif isinstance(self.s, Overloaded): + # Switch the order of arguments to that we'll get to visit_overloaded. + return join_types(t, self.s) + elif isinstance(self.s, Instance) and self.s.type.is_protocol: + call = unpack_callback_protocol(self.s) + if call: + return join_types(t, call) + return join_types(t.fallback, self.s) + + def visit_overloaded(self, t: Overloaded) -> ProperType: + # This is more complex than most other cases. Here are some + # examples that illustrate how this works. + # + # First let's define a concise notation: + # - Cn are callable types (for n in 1, 2, ...) + # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... + # - Callable[[T, ...], S] is written as [T, ...] -> S. + # + # We want some basic properties to hold (assume Cn are all + # unrelated via Any-similarity): + # + # join(Ov(C1, C2), C1) == C1 + # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) + # join(Ov(C1, C2), Ov(C1, C3)) == C1 + # join(Ov(C2, C2), C3) == join of fallback types + # + # The presence of Any types makes things more interesting. The join is the + # most general type we can get with respect to Any: + # + # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str + # + # We could use a simplification step that removes redundancies, but that's not + # implemented right now. Consider this example, where we get a redundancy: + # + # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == + # Ov([Any, int] -> Any, [Any, int] -> Any) + # + # TODO: Consider more cases of callable subtyping. + result: list[CallableType] = [] + s = self.s + if isinstance(s, FunctionLike): + # The interesting case where both types are function types. + for t_item in t.items: + for s_item in s.items: + if is_similar_callables(t_item, s_item): + if is_equivalent(t_item, s_item): + result.append(combine_similar_callables(t_item, s_item)) + elif is_subtype(t_item, s_item): + result.append(s_item) + if result: + # TODO: Simplify redundancies from the result. + if len(result) == 1: + return result[0] + else: + return Overloaded(result) + return join_types(t.fallback, s.fallback) + elif isinstance(s, Instance) and s.type.is_protocol: + call = unpack_callback_protocol(s) + if call: + return join_types(t, call) + return join_types(t.fallback, s) + + def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: + """Join two tuple types while handling variadic entries. + + This is surprisingly tricky, and we don't handle some tricky corner cases. + Most of the trickiness comes from the variadic tuple items like *tuple[X, ...] + since they can have arbitrary partial overlaps (while *Ts can't be split). + """ + s_unpack_index = find_unpack_in_list(s.items) + t_unpack_index = find_unpack_in_list(t.items) + if s_unpack_index is None and t_unpack_index is None: + if s.length() == t.length(): + items: list[Type] = [] + for i in range(t.length()): + items.append(join_types(t.items[i], s.items[i])) + return items + return None + if s_unpack_index is not None and t_unpack_index is not None: + # The most complex case: both tuples have an upack item. + s_unpack = s.items[s_unpack_index] + assert isinstance(s_unpack, UnpackType) + s_unpacked = get_proper_type(s_unpack.type) + t_unpack = t.items[t_unpack_index] + assert isinstance(t_unpack, UnpackType) + t_unpacked = get_proper_type(t_unpack.type) + if s.length() == t.length() and s_unpack_index == t_unpack_index: + # We can handle a case where arity is perfectly aligned, e.g. + # join(Tuple[X1, *tuple[Y1, ...], Z1], Tuple[X2, *tuple[Y2, ...], Z2]). + # We can essentially perform the join elementwise. + prefix_len = t_unpack_index + suffix_len = t.length() - t_unpack_index - 1 + items = [] + for si, ti in zip(s.items[:prefix_len], t.items[:prefix_len]): + items.append(join_types(si, ti)) + joined = join_types(s_unpacked, t_unpacked) + if isinstance(joined, TypeVarTupleType): + items.append(UnpackType(joined)) + elif isinstance(joined, Instance) and joined.type.fullname == "builtins.tuple": + items.append(UnpackType(joined)) + else: + if isinstance(t_unpacked, Instance): + assert t_unpacked.type.fullname == "builtins.tuple" + tuple_instance = t_unpacked + else: + assert isinstance(t_unpacked, TypeVarTupleType) + tuple_instance = t_unpacked.tuple_fallback + items.append( + UnpackType( + tuple_instance.copy_modified( + args=[object_from_instance(tuple_instance)] + ) + ) + ) + if suffix_len: + for si, ti in zip(s.items[-suffix_len:], t.items[-suffix_len:]): + items.append(join_types(si, ti)) + return items + if s.length() == 1 or t.length() == 1: + # Another case we can handle is when one of tuple is purely variadic + # (i.e. a non-normalized form of tuple[X, ...]), in this case the join + # will be again purely variadic. + if not (isinstance(s_unpacked, Instance) and isinstance(t_unpacked, Instance)): + return None + assert s_unpacked.type.fullname == "builtins.tuple" + assert t_unpacked.type.fullname == "builtins.tuple" + mid_joined = join_types(s_unpacked.args[0], t_unpacked.args[0]) + t_other = [a for i, a in enumerate(t.items) if i != t_unpack_index] + s_other = [a for i, a in enumerate(s.items) if i != s_unpack_index] + other_joined = join_type_list(s_other + t_other) + mid_joined = join_types(mid_joined, other_joined) + return [UnpackType(s_unpacked.copy_modified(args=[mid_joined]))] + # TODO: are there other case we can handle (e.g. both prefix/suffix are shorter)? + return None + if s_unpack_index is not None: + variadic = s + unpack_index = s_unpack_index + fixed = t + else: + assert t_unpack_index is not None + variadic = t + unpack_index = t_unpack_index + fixed = s + # Case where one tuple has variadic item and the other one doesn't. The join will + # be variadic, since fixed tuple is a subtype of variadic, but not vice versa. + unpack = variadic.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if not isinstance(unpacked, Instance): + return None + if fixed.length() < variadic.length() - 1: + # There are no non-trivial types that are supertype of both. + return None + prefix_len = unpack_index + suffix_len = variadic.length() - prefix_len - 1 + prefix, middle, suffix = split_with_prefix_and_suffix( + tuple(fixed.items), prefix_len, suffix_len + ) + items = [] + for fi, vi in zip(prefix, variadic.items[:prefix_len]): + items.append(join_types(fi, vi)) + mid_joined = join_type_list(list(middle)) + mid_joined = join_types(mid_joined, unpacked.args[0]) + items.append(UnpackType(unpacked.copy_modified(args=[mid_joined]))) + if suffix_len: + for fi, vi in zip(suffix, variadic.items[-suffix_len:]): + items.append(join_types(fi, vi)) + return items + + def visit_tuple_type(self, t: TupleType) -> ProperType: + # When given two fixed-length tuples: + # * If they have the same length, join their subtypes item-wise: + # Tuple[int, bool] + Tuple[bool, bool] becomes Tuple[int, bool] + # * If lengths do not match, return a variadic tuple: + # Tuple[bool, int] + Tuple[bool] becomes Tuple[int, ...] + # + # Otherwise, `t` is a fixed-length tuple but `self.s` is NOT: + # * Joining with a variadic tuple returns variadic tuple: + # Tuple[int, bool] + Tuple[bool, ...] becomes Tuple[int, ...] + # * Joining with any Sequence also returns a Sequence: + # Tuple[int, bool] + List[bool] becomes Sequence[int] + if isinstance(self.s, TupleType): + if self.instance_joiner is None: + self.instance_joiner = InstanceJoiner() + fallback = self.instance_joiner.join_instances( + mypy.typeops.tuple_fallback(self.s), mypy.typeops.tuple_fallback(t) + ) + assert isinstance(fallback, Instance) + items = self.join_tuples(self.s, t) + if items is not None: + return TupleType(items, fallback) + else: + # TODO: should this be a default fallback behaviour like for meet? + if is_proper_subtype(self.s, t): + return t + if is_proper_subtype(t, self.s): + return self.s + return fallback + else: + return join_types(self.s, mypy.typeops.tuple_fallback(t)) + + def visit_typeddict_type(self, t: TypedDictType) -> ProperType: + if isinstance(self.s, TypedDictType): + items = { + item_name: s_item_type + for (item_name, s_item_type, t_item_type) in self.s.zip(t) + if ( + is_equivalent(s_item_type, t_item_type) + and (item_name in t.required_keys) == (item_name in self.s.required_keys) + ) + } + fallback = self.s.create_anonymous_fallback() + all_keys = set(items.keys()) + # We need to filter by items.keys() since some required keys present in both t and + # self.s might be missing from the join if the types are incompatible. + required_keys = all_keys & t.required_keys & self.s.required_keys + # If one type has a key as readonly, we mark it as readonly for both: + readonly_keys = (t.readonly_keys | t.readonly_keys) & all_keys + return TypedDictType(items, required_keys, readonly_keys, fallback) + elif isinstance(self.s, Instance): + return join_types(self.s, t.fallback) else: return self.default(self.s) - def visit_tuple_type(self, t: TupleType) -> Type: - if (isinstance(self.s, TupleType) and - cast(TupleType, self.s).length() == t.length()): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.join(t.items[i], - (cast(TupleType, self.s)).items[i])) - # TODO: What if the fallback types are different? - return TupleType(items, t.fallback) + def visit_literal_type(self, t: LiteralType) -> ProperType: + if isinstance(self.s, LiteralType): + if t == self.s: + return t + if self.s.fallback.type.is_enum and t.fallback.type.is_enum: + return mypy.typeops.make_simplified_union([self.s, t]) + return join_types(self.s.fallback, t.fallback) + else: + return join_types(self.s, t.fallback) + + def visit_partial_type(self, t: PartialType) -> ProperType: + # We only have partial information so we can't decide the join result. We should + # never get here. + assert False, "Internal error" + + def visit_type_type(self, t: TypeType) -> ProperType: + if isinstance(self.s, TypeType): + return TypeType.make_normalized(join_types(t.item, self.s.item), line=t.line) + elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type": + return self.s else: return self.default(self.s) - def join(self, s: Type, t: Type) -> Type: - return join_types(s, t) + def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: + assert False, f"This should be never called, got {t}" - def default(self, typ: Type) -> Type: + def default(self, typ: Type) -> ProperType: + typ = get_proper_type(typ) if isinstance(typ, Instance): return object_from_instance(typ) elif isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() + return AnyType(TypeOfAny.special_form) elif isinstance(typ, TupleType): + return self.default(mypy.typeops.tuple_fallback(typ)) + elif isinstance(typ, TypedDictType): return self.default(typ.fallback) elif isinstance(typ, FunctionLike): return self.default(typ.fallback) - elif isinstance(typ, TypeVar): + elif isinstance(typ, TypeVarType): + return self.default(typ.upper_bound) + elif isinstance(typ, ParamSpecType): return self.default(typ.upper_bound) else: - return AnyType() + return AnyType(TypeOfAny.special_form) -def join_instances(t: Instance, s: Instance) -> Type: - """Calculate the join of two instance types. +def is_better(t: Type, s: Type) -> bool: + # Given two possible results from join_instances_via_supertype(), + # indicate whether t is the better one. + t = get_proper_type(t) + s = get_proper_type(s) - If allow_interfaces is True, also consider interface-type results for - non-interface types. + if isinstance(t, Instance): + if not isinstance(s, Instance): + return True + # Use len(mro) as a proxy for the better choice. + if len(t.type.mro) > len(s.type.mro): + return True + return False - Return ErrorType if the result is ambiguous. - """ - if t.type == s.type: - # Simplest case: join two types with the same base type (but - # potentially different arguments). - if is_subtype(t, s): - # Compatible; combine type arguments. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(join_types(t.args[i], s.args[i])) - return Instance(t.type, args) - else: - # Incompatible; return trivial result object. - return object_from_instance(t) - elif t.type.bases and is_subtype(t, s): - return join_instances_via_supertype(t, s) - else: - # Now t is not a subtype of s, and t != s. Now s could be a subtype - # of t; alternatively, we need to find a common supertype. This works - # in of the both cases. - return join_instances_via_supertype(s, t) - - -def join_instances_via_supertype(t: Instance, s: Instance) -> Type: - # Give preference to joins via duck typing relationship, so that - # join(int, float) == float, for example. - if t.type.ducktype and is_subtype(t.type.ducktype, s): - return join_types(t.type.ducktype, s) - elif s.type.ducktype and is_subtype(s.type.ducktype, t): - return join_types(t, s.type.ducktype) - res = s - mapped = map_instance_to_supertype(t, t.type.bases[0].type) - join = join_instances(mapped, res) - # If the join failed, fail. This is a defensive measure (this might - # never happen). - if isinstance(join, ErrorType): - return join - # Now the result must be an Instance, so the cast below cannot fail. - res = cast(Instance, join) - return res +def normalize_callables(s: ProperType, t: ProperType) -> tuple[ProperType, ProperType]: + if isinstance(s, (CallableType, Overloaded)): + s = s.with_unpacked_kwargs() + if isinstance(t, (CallableType, Overloaded)): + t = t.with_unpacked_kwargs() + return s, t -def is_similar_callables(t: Callable, s: Callable) -> bool: - """Return True if t and s are equivalent and have identical numbers of +def is_similar_callables(t: CallableType, s: CallableType) -> bool: + """Return True if t and s have identical numbers of arguments, default arguments and varargs. """ - - return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args - and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) - - -def combine_similar_callables(t: Callable, s: Callable) -> Callable: - arg_types = [] # type: List[Type] + return ( + len(t.arg_types) == len(s.arg_types) + and t.min_args == s.min_args + and t.is_var_arg == s.is_var_arg + ) + + +def update_callable_ids(c: CallableType, ids: list[TypeVarId]) -> CallableType: + tv_map = {} + tvs = [] + for tv, new_id in zip(c.variables, ids): + new_tv = tv.copy_modified(id=new_id) + tvs.append(new_tv) + tv_map[tv.id] = new_tv + return expand_type(c, tv_map).copy_modified(variables=tvs) + + +def match_generic_callables(t: CallableType, s: CallableType) -> tuple[CallableType, CallableType]: + # The case where we combine/join/meet similar callables, situation where both are generic + # requires special care. A more principled solution may involve unify_generic_callable(), + # but it would have two problems: + # * This adds risk of infinite recursion: e.g. join -> unification -> solver -> join + # * Using unification is an incorrect thing for meets, as it "widens" the types + # Finally, this effectively falls back to an old behaviour before namespaces were added to + # type variables, and it worked relatively well. + max_len = max(len(t.variables), len(s.variables)) + min_len = min(len(t.variables), len(s.variables)) + if min_len == 0: + return t, s + new_ids = [TypeVarId.new(meta_level=0) for _ in range(max_len)] + # Note: this relies on variables being in order they appear in function definition. + return update_callable_ids(t, new_ids), update_callable_ids(s, new_ids) + + +def join_similar_callables(t: CallableType, s: CallableType) -> CallableType: + t, s = match_generic_callables(t, s) + arg_types: list[Type] = [] + for i in range(len(t.arg_types)): + arg_types.append(safe_meet(t.arg_types[i], s.arg_types[i])) + # TODO in combine_similar_callables also applies here (names and kinds; user metaclasses) + # The fallback type can be either 'function', 'type', or some user-provided metaclass. + # The result should always use 'function' as a fallback if either operands are using it. + if t.fallback.type.fullname == "builtins.function": + fallback = t.fallback + else: + fallback = s.fallback + return t.copy_modified( + arg_types=arg_types, + arg_names=combine_arg_names(t, s), + ret_type=join_types(t.ret_type, s.ret_type), + fallback=fallback, + name=None, + ) + + +def safe_join(t: Type, s: Type) -> Type: + # This is a temporary solution to prevent crashes in combine_similar_callables() etc., + # until relevant TODOs on handling arg_kinds will be addressed there. + if not isinstance(t, UnpackType) and not isinstance(s, UnpackType): + return join_types(t, s) + if isinstance(t, UnpackType) and isinstance(s, UnpackType): + return UnpackType(join_types(t.type, s.type)) + return object_or_any_from_type(get_proper_type(t)) + + +def safe_meet(t: Type, s: Type) -> Type: + # Similar to above but for meet_types(). + from mypy.meet import meet_types + + if not isinstance(t, UnpackType) and not isinstance(s, UnpackType): + return meet_types(t, s) + if isinstance(t, UnpackType) and isinstance(s, UnpackType): + unpacked = get_proper_type(t.type) + if isinstance(unpacked, TypeVarTupleType): + fallback_type = unpacked.tuple_fallback.type + elif isinstance(unpacked, TupleType): + fallback_type = unpacked.partial_fallback.type + else: + assert isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + fallback_type = unpacked.type + res = meet_types(t.type, s.type) + if isinstance(res, UninhabitedType): + res = Instance(fallback_type, [res]) + return UnpackType(res) + return UninhabitedType() + + +def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: + t, s = match_generic_callables(t, s) + arg_types: list[Type] = [] for i in range(len(t.arg_types)): - arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) + arg_types.append(safe_join(t.arg_types[i], s.arg_types[i])) # TODO kinds and argument names - # The fallback type can be either 'function' or 'type'. The result should have 'type' as - # fallback only if both operands have it as 'type'. - if t.fallback.type.fullname() != 'builtins.type': + # TODO what should happen if one fallback is 'type' and the other is a user-provided metaclass? + # The fallback type can be either 'function', 'type', or some user-provided metaclass. + # The result should always use 'function' as a fallback if either operands are using it. + if t.fallback.type.fullname == "builtins.function": fallback = t.fallback else: fallback = s.fallback - return Callable(arg_types, - t.arg_kinds, - t.arg_names, - join_types(t.ret_type, s.ret_type), - fallback, - None, - t.variables) - return s + return t.copy_modified( + arg_types=arg_types, + arg_names=combine_arg_names(t, s), + ret_type=join_types(t.ret_type, s.ret_type), + fallback=fallback, + name=None, + ) + + +def combine_arg_names( + t: CallableType | Parameters, s: CallableType | Parameters +) -> list[str | None]: + """Produces a list of argument names compatible with both callables. + + For example, suppose 't' and 's' have the following signatures: + + - t: (a: int, b: str, X: str) -> None + - s: (a: int, b: str, Y: str) -> None + + This function would return ["a", "b", None]. This information + is then used above to compute the join of t and s, which results + in a signature of (a: int, b: str, str) -> None. + + Note that the third argument's name is omitted and 't' and 's' + are both valid subtypes of this inferred signature. + + Precondition: is_similar_types(t, s) is true. + """ + num_args = len(t.arg_types) + new_names = [] + for i in range(num_args): + t_name = t.arg_names[i] + s_name = s.arg_names[i] + if t_name == s_name or t.arg_kinds[i].is_named() or s.arg_kinds[i].is_named(): + new_names.append(t_name) + else: + new_names.append(None) + return new_names def object_from_instance(instance: Instance) -> Instance: @@ -252,3 +861,45 @@ def object_from_instance(instance: Instance) -> Instance: # Use the fact that 'object' is always the last class in the mro. res = Instance(instance.type.mro[-1], []) return res + + +def object_or_any_from_type(typ: ProperType) -> ProperType: + # Similar to object_from_instance() but tries hard for all types. + # TODO: find a better way to get object, or make this more reliable. + if isinstance(typ, Instance): + return object_from_instance(typ) + elif isinstance(typ, (CallableType, TypedDictType, LiteralType)): + return object_from_instance(typ.fallback) + elif isinstance(typ, TupleType): + return object_from_instance(typ.partial_fallback) + elif isinstance(typ, TypeType): + return object_or_any_from_type(typ.item) + elif isinstance(typ, TypeVarLikeType) and isinstance(typ.upper_bound, ProperType): + return object_or_any_from_type(typ.upper_bound) + elif isinstance(typ, UnionType): + for item in typ.items: + if isinstance(item, ProperType): + candidate = object_or_any_from_type(item) + if isinstance(candidate, Instance): + return candidate + elif isinstance(typ, UnpackType): + object_or_any_from_type(get_proper_type(typ.type)) + return AnyType(TypeOfAny.implementation_artifact) + + +def join_type_list(types: Sequence[Type]) -> Type: + if not types: + # This is a little arbitrary but reasonable. Any empty tuple should be compatible + # with all variable length tuples, and this makes it possible. + return UninhabitedType() + joined = types[0] + for t in types[1:]: + joined = join_types(joined, t) + return joined + + +def unpack_callback_protocol(t: Instance) -> ProperType | None: + assert t.type.is_protocol + if t.type.protocol_members == ["__call__"]: + return get_proper_type(find_member("__call__", t, t, is_operator=True)) + return None diff --git a/mypy/lex.py b/mypy/lex.py deleted file mode 100644 index 17fdb0f261f4..000000000000 --- a/mypy/lex.py +++ /dev/null @@ -1,816 +0,0 @@ -"""Lexical analyzer for mypy. - -Translate a string that represents a file or a compilation unit to a list of -tokens. - -This module can be run as a script (lex.py FILE). -""" - -import re - -from mypy.util import short_type -from typing import List, Undefined, Function, Dict, Any, Match, Pattern, Set - - -class Token: - """Base class for all tokens.""" - - def __init__(self, string: str, pre: str = '') -> None: - """Initialize a token. - - Arguments: - string: Token string in program text - pre: Space, comments etc. before token - """ - - self.string = string - self.pre = pre - self.line = 0 - - def __repr__(self) -> str: - """The representation is of form 'Keyword( if)'.""" - t = short_type(self) - return t + '(' + self.fix(self.pre) + self.fix(self.string) + ')' - - def rep(self) -> str: - return self.pre + self.string - - def fix(self, s: str) -> str: - """Replace common non-printable chars with escape sequences. - - Do not use repr() since we don't want do duplicate backslashes. - """ - return s.replace('\n', '\\n').replace('\t', '\\t').replace('\r', '\\r') - - -# Token classes - - -class Break(Token): - """Statement break (line break or semicolon)""" - - -class Indent(Token): - """Increase block indent level.""" - - -class Dedent(Token): - """Decrease block indent level.""" - - -class Eof(Token): - """End of file""" - - -class Keyword(Token): - """Reserved word (other than keyword operators; they use Op). - - Examples: if, class, while, def. - """ - - -class Name(Token): - """An alphanumeric identifier""" - - -class IntLit(Token): - """Integer literal""" - - -class StrLit(Token): - """String literal""" - - def parsed(self) -> str: - """Return the parsed contents of the literal.""" - return _parse_str_literal(self.string) - - -class BytesLit(Token): - """Bytes literal""" - - def parsed(self) -> str: - """Return the parsed contents of the literal.""" - return _parse_str_literal(self.string) - - -class UnicodeLit(Token): - """Unicode literal (Python 2.x)""" - - def parsed(self) -> str: - """Return the parsed contents of the literal.""" - return _parse_str_literal(self.string) - - -class FloatLit(Token): - """Float literal""" - - -class ComplexLit(Token): - """Complex literal""" - - -class Punct(Token): - """Punctuator (e.g. comma, '(' or '=')""" - - -class Colon(Token): - pass - - -class Op(Token): - """Operator (e.g. '+' or 'in')""" - - -class Bom(Token): - """Byte order mark (at the start of a file)""" - - -class LexError(Token): - """Lexer error token""" - - def __init__(self, string: str, type: int) -> None: - """Initialize token. - - The type argument is one of the error types below. - """ - super().__init__(string) - self.type = type - - -# Lexer error types -NUMERIC_LITERAL_ERROR = 0 -UNTERMINATED_STRING_LITERAL = 1 -INVALID_CHARACTER = 2 -NON_ASCII_CHARACTER_IN_COMMENT = 3 -NON_ASCII_CHARACTER_IN_STRING = 4 -INVALID_UTF8_SEQUENCE = 5 -INVALID_BACKSLASH = 6 -INVALID_DEDENT = 7 - -# Encoding contexts -STR_CONTEXT = 1 -COMMENT_CONTEXT = 2 - - -def lex(string: str, first_line: int = 1, pyversion: int = 3) -> List[Token]: - """Analyze string and return an array of token objects. - - The last token is always Eof. - """ - l = Lexer(pyversion) - l.lex(string, first_line) - return l.tok - - -# Reserved words (not including operators) -keywords_common = set([ - 'as', 'assert', 'break', 'class', 'continue', 'def', 'del', 'elif', - 'else', 'except', 'finally', 'from', 'for', 'global', 'if', 'import', - 'lambda', 'pass', 'raise', 'return', 'try', 'while', 'with', - 'yield']) - -# Reserved words specific for Python version 2 -# TODO (jukka): 'print' should be here, but it breaks the parsing of Python 2 -# builtins, since they also define the function 'print'. -keywords2 = set([]) # type: Set[str] - -# Reserved words specific for Python version 3 -keywords3 = set(['nonlocal']) - -# Alphabetical operators (reserved words) -alpha_operators = set(['in', 'is', 'not', 'and', 'or']) - -# String literal prefixes -str_prefixes = set(['r', 'b', 'br', 'u', 'ur']) - -# List of regular expressions that match non-alphabetical operators -operators = [re.compile('[-+*/<>.%&|^~]'), - re.compile('==|!=|<=|>=|\\*\\*|//|<<|>>')] - -# List of regular expressions that match punctuator tokens -punctuators = [re.compile('[=,()@]|(->)'), - re.compile('\\['), - re.compile(']'), - re.compile('([-+*/%&|^]|\\*\\*|//|<<|>>)=')] - - -# Source file encodings -DEFAULT_ENCODING = 0 -ASCII_ENCODING = 1 -LATIN1_ENCODING = 2 -UTF8_ENCODING = 3 - - -# Map single-character string escape sequences to corresponding characters. -escape_map = {'a': '\x07', - 'b': '\x08', - 'f': '\x0c', - 'n': '\x0a', - 'r': '\x0d', - 't': '\x09', - 'v': '\x0b', - '"': '"', - "'": "'"} - - -# Matches the optional prefix of a string literal, e.g. the 'r' in r"foo". -str_prefix_re = re.compile('[rRbBuU]*') - -# Matches an escape sequence in a string, e.g. \n or \x4F. -escape_re = re.compile( - "\\\\([abfnrtv'\"]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|[0-7]{1,3})") - - -def _parse_str_literal(string: str) -> str: - """Translate escape sequences in str literal to the corresponding chars. - - For example, \t is translated to the tab character (ascii 9). - - Return the translated contents of the literal. Also handle raw and - triple-quoted string literals. - """ - - prefix = str_prefix_re.match(string).group(0).lower() - s = string[len(prefix):] - if s.startswith("'''") or s.startswith('"""'): - return s[3:-3] - elif 'r' in prefix: - return s[1:-1].replace('\\' + s[0], s[0]) - else: - return escape_re.sub(lambda m: escape_repl(m, prefix), s[1:-1]) - - -def escape_repl(m: Match[str], prefix: str) -> str: - """Translate a string escape sequence, e.g. \t -> the tab character. - - Assume that the Match object is from escape_re. - """ - - seq = m.group(1) - if len(seq) == 1 and seq in escape_map: - # Single-character escape sequence, e.g. \n. - return escape_map[seq] - elif seq.startswith('x'): - # Hexadecimal sequence \xNN. - return chr(int(seq[1:], 16)) - elif seq.startswith('u'): - # Unicode sequence \uNNNN. - if 'b' not in prefix: - return chr(int(seq[1:], 16)) - else: - return '\\' + seq - else: - # Octal sequence. - ord = int(seq, 8) - if 'b' in prefix: - # Make sure code is no larger than 255 for bytes literals. - ord = ord % 256 - return chr(ord) - - -class Lexer: - """Lexical analyzer.""" - - i = 0 # Current string index (into s) - s = '' # The string being analyzed - line = 0 # Current line number - pre_whitespace = '' # Whitespace and comments before the next token - enc = DEFAULT_ENCODING # Encoding TODO implement properly - - # Generated tokens - tok = Undefined(List[Token]) - - # Table from byte character value to lexer method. E.g. entry at ord('0') - # contains the method lex_number(). - map = Undefined(List[Function[[], None]]) - - # Indent levels of currently open blocks, in spaces. - indents = Undefined(List[int]) - - # Open ('s, ['s and {'s without matching closing bracket; used for ignoring - # newlines within parentheses/brackets. - open_brackets = Undefined(List[str]) - - def __init__(self, pyversion: int = 3) -> None: - self.map = [self.unknown_character] * 256 - self.tok = [] - self.indents = [0] - self.open_brackets = [] - # Fill in the map from valid character codes to relevant lexer methods. - for seq, method in [('ABCDEFGHIJKLMNOPQRSTUVWXYZ', self.lex_name), - ('abcdefghijklmnopqrstuvwxyz_', self.lex_name), - ('0123456789', self.lex_number), - ('.', self.lex_number_or_dot), - (' ' + '\t' + '\x0c', self.lex_space), - ('"', self.lex_str_double), - ("'", self.lex_str_single), - ('\r' + '\n', self.lex_break), - (';', self.lex_semicolon), - (':', self.lex_colon), - ('#', self.lex_comment), - ('\\', self.lex_backslash), - ('([{', self.lex_open_bracket), - (')]}', self.lex_close_bracket), - ('-+*/<>%&|^~=!,@', self.lex_misc)]: - for c in seq: - self.map[ord(c)] = method - if pyversion == 2: - self.keywords = keywords_common | keywords2 - if pyversion == 3: - self.keywords = keywords_common | keywords3 - - def lex(self, s: str, first_line: int) -> None: - """Lexically analyze a string, storing the tokens at the tok list.""" - self.s = s - self.i = 0 - self.line = first_line - - if s.startswith('\xef\xbb\xbf'): - self.add_token(Bom(s[0:3])) - - # Parse initial indent; otherwise first-line indent would not generate - # an error. - self.lex_indent() - - # Make a local copy of map as a simple optimization. - map = self.map - - # Lex the file. Repeatedly call the lexer method for the current char. - while self.i < len(s): - # Get the character code of the next character to lex. - c = ord(s[self.i]) - # Dispatch to the relevant lexer method. This will consume some - # characters in the text, add a token to self.tok and increment - # self.i. - map[c]() - - # Append a break if there is no statement/block terminator at the end - # of input. - if len(self.tok) > 0 and (not isinstance(self.tok[-1], Break) and - not isinstance(self.tok[-1], Dedent)): - self.add_token(Break('')) - - # Attack any dangling comments/whitespace to a final Break token. - if self.tok and isinstance(self.tok[-1], Break): - self.tok[-1].string += self.pre_whitespace - self.pre_whitespace = '' - - # Close remaining open blocks with Dedent tokens. - self.lex_indent() - - self.add_token(Eof('')) - - def lex_number_or_dot(self) -> None: - """Analyse a token starting with a dot. - - It can be the member access operator or a float literal such as '.123'. - """ - if self.is_at_number(): - self.lex_number() - else: - self.lex_misc() - - number_exp = re.compile(r'[0-9]|\.[0-9]') - - def is_at_number(self) -> bool: - """Is the current location at a numeric literal?""" - return self.match(self.number_exp) != '' - - # Regexps used by lex_number - - # Decimal/hex/octal literal or integer complex literal - number_exp1 = re.compile('0[xXoO][0-9a-fA-F]+|[0-9]+') - # Float literal, e.g. '1.23' or '12e+34' or '1.2j' - number_exp2 = re.compile( - r'[0-9]*\.[0-9]*([eE][-+]?[0-9]+)?|[0-9]+[eE][-+]?[0-9]+') - # Complex literal, e.g. '3j' or '1.5e2J' - number_complex = re.compile( - r'([0-9]*\.[0-9]*([eE][-+]?[0-9]+)?|[0-9]+([eE][-+]?[0-9]+)?)[jJ]') - # These characters must not appear after a number literal. - name_char_exp = re.compile('[a-zA-Z0-9_]') - - def lex_number(self) -> None: - """Analyse an int or float literal. - - Assume that the current location points to one of them. - """ - s1 = self.match(self.number_exp1) - s2 = self.match(self.number_exp2) - sc = self.match(self.number_complex) - - maxlen = max(len(s1), len(s2), len(sc)) - if self.name_char_exp.match( - self.s[self.i + maxlen:self.i + maxlen + 1]) is not None: - # Error: alphanumeric character after number literal. - s3 = self.match(re.compile('[0-9][0-9a-zA-Z_]*')) - maxlen = max(maxlen, len(s3)) - self.add_token(LexError(' ' * maxlen, NUMERIC_LITERAL_ERROR)) - elif len(s1) == maxlen: - # Integer literal. - self.add_token(IntLit(s1)) - elif len(s2) == maxlen: - # Float literal. - self.add_token(FloatLit(s2)) - else: - # Complex literal. - self.add_token(ComplexLit(sc)) - - name_exp = re.compile('[a-zA-Z_][a-zA-Z0-9_]*') - - def lex_name(self) -> None: - """Analyse a name. - - A name can be an identifier, a keyword or an alphabetical operator. - Also deal with prefixed string literals such as r'...'. - """ - s = self.match(self.name_exp) - if s in self.keywords: - self.add_token(Keyword(s)) - elif s in alpha_operators: - self.add_token(Op(s)) - elif s in str_prefixes and self.match(re.compile('[a-z]+[\'"]')) != '': - self.lex_prefixed_str(s) - else: - self.add_token(Name(s)) - - # Regexps representing components of string literals - - # Initial part of a single-quoted literal, e.g. b'foo' or b'foo\\\n - str_exp_single = re.compile( - r"[a-z]*'([^'\\\r\n]|\\[^\r\n])*('|\\(\n|\r\n?))") - # Non-initial part of a multiline single-quoted literal, e.g. foo' - str_exp_single_multi = re.compile( - r"([^'\\\r\n]|\\[^\r\n])*('|\\(\n|\r\n?))") - # Initial part of a single-quoted raw literal, e.g. r'foo' or r'foo\\\n - str_exp_raw_single = re.compile( - r"[a-z]*'([^'\r\n\\]|\\'|\\[^\n\r])*('|\\(\n|\r\n?))") - # Non-initial part of a raw multiline single-quoted literal, e.g. foo' - str_exp_raw_single_multi = re.compile( - r"([^'\r\n]|'')*('|\\(\n|\r\n?))") - - # Start of a ''' literal, e.g. b''' - str_exp_single3 = re.compile("[a-z]*'''") - # End of a ''' literal, e.g. foo''' - str_exp_single3end = re.compile(r"[^\n\r]*?'''") - - # The following are similar to above (but use double quotes). - - str_exp_double = re.compile( - r'[a-z]*"([^"\\\r\n]|\\[^\r\n])*("|\\(\n|\r\n?))') - str_exp_double_multi = re.compile( - r'([^"\\\r\n]|\\[^\r\n])*("|\\(\n|\r\n?))') - str_exp_raw_double = re.compile( - r'[a-z]*"([^"\r\n\\]|\\"|\\[^\n\r])*("|\\(\n|\r\n?))') - str_exp_raw_double_multi = re.compile( - r'([^"\r\n]|"")*("|\\(\n|\r\n?))') - - str_exp_double3 = re.compile('[a-z]*"""') - str_exp_double3end = re.compile(r'[^\n\r]*?"""') - - def lex_str_single(self) -> None: - """Analyse single-quoted string literal""" - self.lex_str(self.str_exp_single, self.str_exp_single_multi, - self.str_exp_single3, self.str_exp_single3end) - - def lex_str_double(self) -> None: - """Analyse double-quoted string literal""" - self.lex_str(self.str_exp_double, self.str_exp_double_multi, - self.str_exp_double3, self.str_exp_double3end) - - def lex_prefixed_str(self, prefix: str) -> None: - """Analyse a string literal with a prefix, such as r'...'.""" - s = self.match(re.compile('[a-z]+[\'"]')) - if s.endswith("'"): - re1 = self.str_exp_single - re2 = self.str_exp_single_multi - if 'r' in prefix: - re1 = self.str_exp_raw_single - re2 = self.str_exp_raw_single_multi - self.lex_str(re1, re2, self.str_exp_single3, - self.str_exp_single3end, prefix) - else: - re1 = self.str_exp_double - re2 = self.str_exp_double_multi - if 'r' in prefix: - re1 = self.str_exp_raw_double - re2 = self.str_exp_raw_double_multi - self.lex_str(re1, re2, self.str_exp_double3, - self.str_exp_double3end, prefix) - - def lex_str(self, regex: Pattern[str], re2: Pattern[str], - re3: Pattern[str], re3end: Pattern[str], - prefix: str = '') -> None: - """Analyse a string literal described by regexps. - - Assume that the current location is at the beginning of the - literal. The arguments re3 and re3end describe the - corresponding triple-quoted literals. - """ - s3 = self.match(re3) - if s3 != '': - # Triple-quoted string literal. - self.lex_triple_quoted_str(re3end, prefix) - else: - # Single or double quoted string literal. - s = self.match(regex) - if s != '': - if s.endswith('\n') or s.endswith('\r'): - self.lex_multiline_string_literal(re2, s) - else: - self.verify_encoding(s, STR_CONTEXT) - if 'b' in prefix: - self.add_token(BytesLit(s)) - elif 'u' in prefix: - self.add_token(UnicodeLit(s)) - else: - self.add_token(StrLit(s)) - else: - # Unterminated string literal. - s = self.match(re.compile('[^\\n\\r]*')) - self.add_token(LexError(s, UNTERMINATED_STRING_LITERAL)) - - def lex_triple_quoted_str(self, re3end: Pattern[str], prefix: str) -> None: - line = self.line - ss = self.s[self.i:self.i + len(prefix) + 3] - self.i += len(prefix) + 3 - while True: - m = re3end.match(self.s, self.i) - if m is not None: - break - m = re.match('[^\\n\\r]*(\\n|\\r\\n?)', self.s[self.i:]) - if m is None: - self.add_special_token( - LexError(ss, UNTERMINATED_STRING_LITERAL), line, 0) - return - s = m.group(0) - ss += s - self.line += 1 - self.i += len(s) - lit = Undefined # type: Token - if 'b' in prefix: - lit = BytesLit(ss + m.group(0)) - elif 'u' in prefix: - lit = UnicodeLit(ss + m.group(0)) - else: - lit = StrLit(ss + m.group(0)) - self.add_special_token(lit, line, len(m.group(0))) - - def lex_multiline_string_literal(self, re_end: Pattern[str], - prefix: str) -> None: - """Analyze multiline single/double-quoted string literal. - - Use explicit \ for line continuation. - """ - line = self.line - self.i += len(prefix) - ss = prefix - while True: - m = self.match(re_end) - if m == '': - self.add_special_token( - LexError(ss, UNTERMINATED_STRING_LITERAL), line, 0) - return - ss += m - self.line += 1 - self.i += len(m) - if not m.endswith('\n') and not m.endswith('\r'): break - self.add_special_token(StrLit(ss), line, 0) # TODO bytes - - comment_exp = re.compile(r'#[^\n\r]*') - - def lex_comment(self) -> None: - """Analyze a comment.""" - s = self.match(self.comment_exp) - self.verify_encoding(s, COMMENT_CONTEXT) - self.add_pre_whitespace(s) - - backslash_exp = re.compile(r'\\(\n|\r\n?)') - - def lex_backslash(self) -> None: - s = self.match(self.backslash_exp) - if s != '': - self.add_pre_whitespace(s) - self.line += 1 - else: - self.add_token(LexError('\\', INVALID_BACKSLASH)) - - space_exp = re.compile(r'[ \t\x0c]*') - indent_exp = re.compile(r'[ \t]*[#\n\r]?') - - def lex_space(self) -> None: - """Analyze a run of whitespace characters (within a line, not indents). - - Only store them in self.pre_whitespace. - """ - s = self.match(self.space_exp) - self.add_pre_whitespace(s) - - comment_or_newline = '#' + '\n' + '\r' # type: str - - def lex_indent(self) -> None: - """Analyze whitespace chars at the beginning of a line (indents).""" - s = self.match(self.indent_exp) - if s != '' and s[-1] in self.comment_or_newline: - # Empty line (whitespace only or comment only). - self.add_pre_whitespace(s[:-1]) - if s[-1] == '#': - self.lex_comment() - else: - self.lex_break() - self.lex_indent() - return - indent = self.calc_indent(s) - if indent == self.indents[-1]: - # No change in indent: just whitespace. - self.add_pre_whitespace(s) - elif indent > self.indents[-1]: - # An increased indent (new block). - self.indents.append(indent) - self.add_token(Indent(s)) - else: - # Decreased indent (end of one or more blocks). - pre = self.pre_whitespace - self.pre_whitespace = '' - while indent < self.indents[-1]: - self.add_token(Dedent('')) - self.indents.pop() - self.pre_whitespace = pre - self.add_pre_whitespace(s) - if indent != self.indents[-1]: - # Error: indent level does not match a previous indent level. - self.add_token(LexError('', INVALID_DEDENT)) - - def calc_indent(self, s: str) -> int: - indent = 0 - for ch in s: - if ch == ' ': - indent += 1 - else: - # Tab: 8 spaces (rounded to a multiple of 8). - indent += 8 - indent % 8 - return indent - - break_exp = re.compile(r'\r\n|\r|\n|;') - - def lex_break(self) -> None: - """Analyse a line break.""" - s = self.match(self.break_exp) - last_tok = self.tok[-1] if self.tok else None - if isinstance(last_tok, Break): - was_semicolon = last_tok.string == ';' - last_tok.string += self.pre_whitespace + s - self.i += len(s) - self.line += 1 - self.pre_whitespace = '' - if was_semicolon: - self.lex_indent() - elif self.ignore_break(): - self.add_pre_whitespace(s) - self.line += 1 - else: - self.add_token(Break(s)) - self.line += 1 - self.lex_indent() - - def lex_semicolon(self) -> None: - self.add_token(Break(';')) - - def lex_colon(self) -> None: - self.add_token(Colon(':')) - - open_bracket_exp = re.compile('[[({]') - - def lex_open_bracket(self) -> None: - s = self.match(self.open_bracket_exp) - self.open_brackets.append(s) - self.add_token(Punct(s)) - - close_bracket_exp = re.compile('[])}]') - - open_bracket = {')': '(', ']': '[', '}': '{'} - - def lex_close_bracket(self) -> None: - s = self.match(self.close_bracket_exp) - if (self.open_brackets != [] - and self.open_bracket[s] == self.open_brackets[-1]): - self.open_brackets.pop() - self.add_token(Punct(s)) - - def lex_misc(self) -> None: - """Analyze a non-alphabetical operator or a punctuator.""" - s = '' - t = None # type: Any - for re_list, type in [(operators, Op), (punctuators, Punct)]: - for re in re_list: - s2 = self.match(re) - if len(s2) > len(s): - t = type - s = s2 - if s == '': - # Could not match any token; report an invalid character. This is - # reached at least if the current character is '!' not followed by - # '='. - self.add_token(LexError(self.s[self.i], INVALID_CHARACTER)) - else: - self.add_token(t(s)) - - def unknown_character(self) -> None: - """Report an unknown character as a lexical analysis error.""" - self.add_token(LexError(self.s[self.i], INVALID_CHARACTER)) - - # Utility methods - - def match(self, pattern: Pattern[str]) -> str: - """Try to match a regular expression at current location. - - If the argument regexp is matched at the current location, - return the matched string; otherwise return the empty string. - """ - m = pattern.match(self.s, self.i) - if m is not None: - return m.group(0) - else: - return '' - - def add_pre_whitespace(self, s: str) -> None: - """Record whitespace and comments before the next token. - - The accumulated whitespace/comments will be stored in the next token - and then it will be cleared. - - This is needed for pretty-printing the original source code while - preserving comments, indentation, whitespace etc. - """ - self.pre_whitespace += s - self.i += len(s) - - def add_token(self, tok: Token) -> None: - """Store a token. - - Update its line number and record preceding whitespace - characters and comments. - """ - if (tok.string == '' and not isinstance(tok, Eof) - and not isinstance(tok, Break) - and not isinstance(tok, LexError) - and not isinstance(tok, Dedent)): - raise ValueError('Empty token') - tok.pre = self.pre_whitespace - tok.line = self.line - self.tok.append(tok) - self.i += len(tok.string) - self.pre_whitespace = '' - - def add_special_token(self, tok: Token, line: int, skip: int) -> None: - """Like add_token, but caller sets the number of chars to skip.""" - if (tok.string == '' and not isinstance(tok, Eof) - and not isinstance(tok, Break) - and not isinstance(tok, LexError) - and not isinstance(tok, Dedent)): - raise ValueError('Empty token') - tok.pre = self.pre_whitespace - tok.line = line - self.tok.append(tok) - self.i += skip - self.pre_whitespace = '' - - def ignore_break(self) -> bool: - """If the next token is a break, can we ignore it?""" - if len(self.open_brackets) > 0 or len(self.tok) == 0: - # Ignore break after open ( [ or { or at the beginning of file. - return True - else: - # Ignore break after another break or dedent. - t = self.tok[-1] - return isinstance(t, Break) or isinstance(t, Dedent) - - def verify_encoding(self, string: str, context: int) -> None: - """Verify that token is encoded correctly (using the file encoding).""" - codec = None # type: str - if self.enc == ASCII_ENCODING: - codec = 'ascii' - elif self.enc in [UTF8_ENCODING, DEFAULT_ENCODING]: - codec = 'utf8' - if codec is not None: - try: - pass # FIX string.decode(codec) - except UnicodeDecodeError: - type = INVALID_UTF8_SEQUENCE - if self.enc == ASCII_ENCODING: - if context == STR_CONTEXT: - type = NON_ASCII_CHARACTER_IN_STRING - else: - type = NON_ASCII_CHARACTER_IN_COMMENT - self.add_token(LexError('', type)) - - -if __name__ == '__main__': - # Lexically analyze a file and dump the tokens to stdout. - import sys - if len(sys.argv) != 2: - print('Usage: lex.py FILE') - sys.exit(2) - fnam = sys.argv[1] - s = open(fnam).read() - for t in lex(s): - print(t) diff --git a/mypy/literals.py b/mypy/literals.py new file mode 100644 index 000000000000..cba5712644be --- /dev/null +++ b/mypy/literals.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +from typing import Any, Final, Iterable, Optional, Tuple +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.nodes import ( + LITERAL_NO, + LITERAL_TYPE, + LITERAL_YES, + AssertTypeExpr, + AssignmentExpr, + AwaitExpr, + BytesExpr, + CallExpr, + CastExpr, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, + Expression, + FloatExpr, + GeneratorExpr, + IndexExpr, + IntExpr, + LambdaExpr, + ListComprehension, + ListExpr, + MemberExpr, + NamedTupleExpr, + NameExpr, + NewTypeExpr, + OpExpr, + ParamSpecExpr, + PromoteExpr, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + StrExpr, + SuperExpr, + TempNode, + TupleExpr, + TypeAliasExpr, + TypeApplication, + TypedDictExpr, + TypeVarExpr, + TypeVarTupleExpr, + UnaryExpr, + Var, + YieldExpr, + YieldFromExpr, +) +from mypy.visitor import ExpressionVisitor + +# [Note Literals and literal_hash] +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Mypy uses the term "literal" to refer to any expression built out of +# the following: +# +# * Plain literal expressions, like `1` (integer, float, string, etc.) +# +# * Compound literal expressions, like `(lit1, lit2)` (list, dict, +# set, or tuple) +# +# * Operator expressions, like `lit1 + lit2` +# +# * Variable references, like `x` +# +# * Member references, like `lit.m` +# +# * Index expressions, like `lit[0]` +# +# A typical "literal" looks like `x[(i,j+1)].m`. +# +# An expression that is a literal has a `literal_hash`, with the +# following properties. +# +# * `literal_hash` is a Key: a tuple containing basic data types and +# possibly other Keys. So it can be used as a key in a dictionary +# that will be compared by value (as opposed to the Node itself, +# which is compared by identity). +# +# * Two expressions have equal `literal_hash`es if and only if they +# are syntactically equal expressions. (NB: Actually, we also +# identify as equal expressions like `3` and `3.0`; is this a good +# idea?) +# +# * The elements of `literal_hash` that are tuples are exactly the +# subexpressions of the original expression (e.g. the base and index +# of an index expression, or the operands of an operator expression). + + +def literal(e: Expression) -> int: + if isinstance(e, ComparisonExpr): + return min(literal(o) for o in e.operands) + + elif isinstance(e, OpExpr): + return min(literal(e.left), literal(e.right)) + + elif isinstance(e, (MemberExpr, UnaryExpr, StarExpr)): + return literal(e.expr) + + elif isinstance(e, AssignmentExpr): + return literal(e.target) + + elif isinstance(e, IndexExpr): + if literal(e.index) == LITERAL_YES: + return literal(e.base) + else: + return LITERAL_NO + + elif isinstance(e, NameExpr): + if isinstance(e.node, Var) and e.node.is_final and e.node.final_value is not None: + return LITERAL_YES + return LITERAL_TYPE + + if isinstance(e, (IntExpr, FloatExpr, ComplexExpr, StrExpr, BytesExpr)): + return LITERAL_YES + + if literal_hash(e): + return LITERAL_YES + + return LITERAL_NO + + +Key: _TypeAlias = Tuple[Any, ...] + + +def subkeys(key: Key) -> Iterable[Key]: + return [elt for elt in key if isinstance(elt, tuple)] + + +def literal_hash(e: Expression) -> Key | None: + return e.accept(_hasher) + + +def extract_var_from_literal_hash(key: Key) -> Var | None: + """If key refers to a Var node, return it. + + Return None otherwise. + """ + if len(key) == 2 and key[0] == "Var" and isinstance(key[1], Var): + return key[1] + return None + + +class _Hasher(ExpressionVisitor[Optional[Key]]): + def visit_int_expr(self, e: IntExpr) -> Key: + return ("Literal", e.value) + + def visit_str_expr(self, e: StrExpr) -> Key: + return ("Literal", e.value) + + def visit_bytes_expr(self, e: BytesExpr) -> Key: + return ("Literal", e.value) + + def visit_float_expr(self, e: FloatExpr) -> Key: + return ("Literal", e.value) + + def visit_complex_expr(self, e: ComplexExpr) -> Key: + return ("Literal", e.value) + + def visit_star_expr(self, e: StarExpr) -> Key: + return ("Star", literal_hash(e.expr)) + + def visit_name_expr(self, e: NameExpr) -> Key: + if isinstance(e.node, Var) and e.node.is_final and e.node.final_value is not None: + return ("Literal", e.node.final_value) + # N.B: We use the node itself as the key, and not the name, + # because using the name causes issues when there is shadowing + # (for example, in list comprehensions). + return ("Var", e.node) + + def visit_member_expr(self, e: MemberExpr) -> Key: + return ("Member", literal_hash(e.expr), e.name) + + def visit_op_expr(self, e: OpExpr) -> Key: + return ("Binary", e.op, literal_hash(e.left), literal_hash(e.right)) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Key: + rest: tuple[str | Key | None, ...] = tuple(e.operators) + rest += tuple(literal_hash(o) for o in e.operands) + return ("Comparison",) + rest + + def visit_unary_expr(self, e: UnaryExpr) -> Key: + return ("Unary", e.op, literal_hash(e.expr)) + + def seq_expr(self, e: ListExpr | TupleExpr | SetExpr, name: str) -> Key | None: + if all(literal(x) == LITERAL_YES for x in e.items): + rest: tuple[Key | None, ...] = tuple(literal_hash(x) for x in e.items) + return (name,) + rest + return None + + def visit_list_expr(self, e: ListExpr) -> Key | None: + return self.seq_expr(e, "List") + + def visit_dict_expr(self, e: DictExpr) -> Key | None: + if all(a and literal(a) == literal(b) == LITERAL_YES for a, b in e.items): + rest: tuple[Key | None, ...] = tuple( + (literal_hash(a) if a else None, literal_hash(b)) for a, b in e.items + ) + return ("Dict",) + rest + return None + + def visit_tuple_expr(self, e: TupleExpr) -> Key | None: + return self.seq_expr(e, "Tuple") + + def visit_set_expr(self, e: SetExpr) -> Key | None: + return self.seq_expr(e, "Set") + + def visit_index_expr(self, e: IndexExpr) -> Key | None: + if literal(e.index) == LITERAL_YES: + return ("Index", literal_hash(e.base), literal_hash(e.index)) + return None + + def visit_assignment_expr(self, e: AssignmentExpr) -> Key | None: + return literal_hash(e.target) + + def visit_call_expr(self, e: CallExpr) -> None: + return None + + def visit_slice_expr(self, e: SliceExpr) -> None: + return None + + def visit_cast_expr(self, e: CastExpr) -> None: + return None + + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: + return None + + def visit_conditional_expr(self, e: ConditionalExpr) -> None: + return None + + def visit_ellipsis(self, e: EllipsisExpr) -> None: + return None + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + return None + + def visit_yield_expr(self, e: YieldExpr) -> None: + return None + + def visit_reveal_expr(self, e: RevealExpr) -> None: + return None + + def visit_super_expr(self, e: SuperExpr) -> None: + return None + + def visit_type_application(self, e: TypeApplication) -> None: + return None + + def visit_lambda_expr(self, e: LambdaExpr) -> None: + return None + + def visit_list_comprehension(self, e: ListComprehension) -> None: + return None + + def visit_set_comprehension(self, e: SetComprehension) -> None: + return None + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> None: + return None + + def visit_generator_expr(self, e: GeneratorExpr) -> None: + return None + + def visit_type_var_expr(self, e: TypeVarExpr) -> None: + return None + + def visit_paramspec_expr(self, e: ParamSpecExpr) -> None: + return None + + def visit_type_var_tuple_expr(self, e: TypeVarTupleExpr) -> None: + return None + + def visit_type_alias_expr(self, e: TypeAliasExpr) -> None: + return None + + def visit_namedtuple_expr(self, e: NamedTupleExpr) -> None: + return None + + def visit_enum_call_expr(self, e: EnumCallExpr) -> None: + return None + + def visit_typeddict_expr(self, e: TypedDictExpr) -> None: + return None + + def visit_newtype_expr(self, e: NewTypeExpr) -> None: + return None + + def visit__promote_expr(self, e: PromoteExpr) -> None: + return None + + def visit_await_expr(self, e: AwaitExpr) -> None: + return None + + def visit_temp_node(self, e: TempNode) -> None: + return None + + +_hasher: Final = _Hasher() diff --git a/mypy/lookup.py b/mypy/lookup.py new file mode 100644 index 000000000000..8fc8cf8be3c2 --- /dev/null +++ b/mypy/lookup.py @@ -0,0 +1,61 @@ +""" +This is a module for various lookup functions: +functions that will find a semantic node by its name. +""" + +from __future__ import annotations + +from mypy.nodes import MypyFile, SymbolTableNode, TypeInfo + +# TODO: gradually move existing lookup functions to this module. + + +def lookup_fully_qualified( + name: str, modules: dict[str, MypyFile], *, raise_on_missing: bool = False +) -> SymbolTableNode | None: + """Find a symbol using it fully qualified name. + + The algorithm has two steps: first we try splitting the name on '.' to find + the module, then iteratively look for each next chunk after a '.' (e.g. for + nested classes). + + This function should *not* be used to find a module. Those should be looked + in the modules dictionary. + """ + head = name + rest = [] + # 1. Find a module tree in modules dictionary. + while True: + if "." not in head: + if raise_on_missing: + assert "." in head, f"Cannot find module for {name}" + return None + head, tail = head.rsplit(".", maxsplit=1) + rest.append(tail) + mod = modules.get(head) + if mod is not None: + break + names = mod.names + # 2. Find the symbol in the module tree. + if not rest: + # Looks like a module, don't use this to avoid confusions. + if raise_on_missing: + assert rest, f"Cannot find {name}, got a module symbol" + return None + while True: + key = rest.pop() + if key not in names: + if raise_on_missing: + assert key in names, f"Cannot find component {key!r} for {name!r}" + return None + stnode = names[key] + if not rest: + return stnode + node = stnode.node + # In fine-grained mode, could be a cross-reference to a deleted module + # or a Var made up for a missing module. + if not isinstance(node, TypeInfo): + if raise_on_missing: + assert node, f"Cannot find {name}" + return None + names = node.names diff --git a/mypy/main.py b/mypy/main.py new file mode 100644 index 000000000000..7032682c9fd0 --- /dev/null +++ b/mypy/main.py @@ -0,0 +1,1591 @@ +"""Mypy type checker command line tool.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import time +from collections import defaultdict +from gettext import gettext +from typing import IO, Any, Final, NoReturn, Sequence, TextIO + +from mypy import build, defaults, state, util +from mypy.config_parser import ( + get_config_module_names, + parse_config_file, + parse_version, + validate_package_allow_list, +) +from mypy.error_formatter import OUTPUT_CHOICES +from mypy.errors import CompileError +from mypy.find_sources import InvalidSourceList, create_source_list +from mypy.fscache import FileSystemCache +from mypy.modulefinder import ( + BuildSource, + FindModuleCache, + ModuleNotFoundReason, + SearchPaths, + get_search_dirs, + mypy_path, +) +from mypy.options import INCOMPLETE_FEATURES, BuildType, Options +from mypy.split_namespace import SplitNamespace +from mypy.version import __version__ + +orig_stat: Final = os.stat +MEM_PROFILE: Final = False # If True, dump memory profile + + +def stat_proxy(path: str) -> os.stat_result: + try: + st = orig_stat(path) + except OSError as err: + print(f"stat({path!r}) -> {err}") + raise + else: + print( + "stat(%r) -> (st_mode=%o, st_mtime=%d, st_size=%d)" + % (path, st.st_mode, st.st_mtime, st.st_size) + ) + return st + + +def main( + *, + args: list[str] | None = None, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, + clean_exit: bool = False, +) -> None: + """Main entry point to the type checker. + + Args: + args: Custom command-line arguments. If not given, sys.argv[1:] will + be used. + clean_exit: Don't hard kill the process on exit. This allows catching + SystemExit. + """ + util.check_python_version("mypy") + t0 = time.time() + # To log stat() calls: os.stat = stat_proxy + sys.setrecursionlimit(2**14) + if args is None: + args = sys.argv[1:] + + fscache = FileSystemCache() + sources, options = process_options(args, stdout=stdout, stderr=stderr, fscache=fscache) + if clean_exit: + options.fast_exit = False + + formatter = util.FancyFormatter( + stdout, stderr, options.hide_error_codes, hide_success=bool(options.output) + ) + + if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): + # Since --install-types performs user input, we want regular stdout and stderr. + fail("error: --install-types not supported in this mode of running mypy", stderr, options) + + if options.non_interactive and not options.install_types: + fail("error: --non-interactive is only supported with --install-types", stderr, options) + + if options.install_types and not options.incremental: + fail( + "error: --install-types not supported with incremental mode disabled", stderr, options + ) + + if options.install_types and options.python_executable is None: + fail( + "error: --install-types not supported without python executable or site packages", + stderr, + options, + ) + + if options.install_types and not sources: + install_types(formatter, options, non_interactive=options.non_interactive) + return + + res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) + + if options.non_interactive: + missing_pkgs = read_types_packages_to_install(options.cache_dir, after_run=True) + if missing_pkgs: + # Install missing type packages and rerun build. + install_types(formatter, options, after_run=True, non_interactive=True) + fscache.flush() + print() + res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) + show_messages(messages, stderr, formatter, options) + + if MEM_PROFILE: + from mypy.memprofile import print_memory_profile + + print_memory_profile() + + code = 0 + n_errors, n_notes, n_files = util.count_stats(messages) + if messages and n_notes < len(messages): + code = 2 if blockers else 1 + if options.error_summary: + if n_errors: + summary = formatter.format_error( + n_errors, n_files, len(sources), blockers=blockers, use_color=options.color_output + ) + stdout.write(summary + "\n") + # Only notes should also output success + elif not messages or n_notes == len(messages): + stdout.write(formatter.format_success(len(sources), options.color_output) + "\n") + stdout.flush() + + if options.install_types and not options.non_interactive: + result = install_types(formatter, options, after_run=True, non_interactive=False) + if result: + print() + print("note: Run mypy again for up-to-date results with installed types") + code = 2 + + if options.fast_exit: + # Exit without freeing objects -- it's faster. + # + # NOTE: We don't flush all open files on exit (or run other destructors)! + util.hard_exit(code) + elif code: + sys.exit(code) + + # HACK: keep res alive so that mypyc won't free it before the hard_exit + list([res]) # noqa: C410 + + +def run_build( + sources: list[BuildSource], + options: Options, + fscache: FileSystemCache, + t0: float, + stdout: TextIO, + stderr: TextIO, +) -> tuple[build.BuildResult | None, list[str], bool]: + formatter = util.FancyFormatter( + stdout, stderr, options.hide_error_codes, hide_success=bool(options.output) + ) + + messages = [] + messages_by_file = defaultdict(list) + + def flush_errors(filename: str | None, new_messages: list[str], serious: bool) -> None: + if options.pretty: + new_messages = formatter.fit_in_terminal(new_messages) + messages.extend(new_messages) + if new_messages: + messages_by_file[filename].extend(new_messages) + if options.non_interactive: + # Collect messages and possibly show them later. + return + f = stderr if serious else stdout + show_messages(new_messages, f, formatter, options) + + serious = False + blockers = False + res = None + try: + # Keep a dummy reference (res) for memory profiling afterwards, as otherwise + # the result could be freed. + res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) + except CompileError as e: + blockers = True + if not e.use_stdout: + serious = True + if ( + options.warn_unused_configs + and options.unused_configs + and not options.incremental + and not options.non_interactive + ): + print( + "Warning: unused section(s) in {}: {}".format( + options.config_file, + get_config_module_names( + options.config_file, + [ + glob + for glob in options.per_module_options.keys() + if glob in options.unused_configs + ], + ), + ), + file=stderr, + ) + maybe_write_junit_xml(time.time() - t0, serious, messages, messages_by_file, options) + return res, messages, blockers + + +def show_messages( + messages: list[str], f: TextIO, formatter: util.FancyFormatter, options: Options +) -> None: + for msg in messages: + if options.color_output: + msg = formatter.colorize(msg) + f.write(msg + "\n") + f.flush() + + +# Make the help output a little less jarring. +class AugmentedHelpFormatter(argparse.RawDescriptionHelpFormatter): + def __init__(self, prog: str) -> None: + super().__init__(prog=prog, max_help_position=28) + + def _fill_text(self, text: str, width: int, indent: str) -> str: + if "\n" in text: + # Assume we want to manually format the text + return super()._fill_text(text, width, indent) + else: + # Assume we want argparse to manage wrapping, indenting, and + # formatting the text for us. + return argparse.HelpFormatter._fill_text(self, text, width, indent) + + +# Define pairs of flag prefixes with inverse meaning. +flag_prefix_pairs: Final = [("allow", "disallow"), ("show", "hide")] +flag_prefix_map: Final[dict[str, str]] = {} +for a, b in flag_prefix_pairs: + flag_prefix_map[a] = b + flag_prefix_map[b] = a + + +def invert_flag_name(flag: str) -> str: + split = flag[2:].split("-", 1) + if len(split) == 2: + prefix, rest = split + if prefix in flag_prefix_map: + return f"--{flag_prefix_map[prefix]}-{rest}" + elif prefix == "no": + return f"--{rest}" + + return f"--no-{flag[2:]}" + + +class PythonExecutableInferenceError(Exception): + """Represents a failure to infer the version or executable while searching.""" + + +def python_executable_prefix(v: str) -> list[str]: + if sys.platform == "win32": + # on Windows, all Python executables are named `python`. To handle this, there + # is the `py` launcher, which can be passed a version e.g. `py -3.8`, and it will + # execute an installed Python 3.8 interpreter. See also: + # https://docs.python.org/3/using/windows.html#python-launcher-for-windows + return ["py", f"-{v}"] + else: + return [f"python{v}"] + + +def _python_executable_from_version(python_version: tuple[int, int]) -> str: + if sys.version_info[:2] == python_version: + return sys.executable + str_ver = ".".join(map(str, python_version)) + try: + sys_exe = ( + subprocess.check_output( + python_executable_prefix(str_ver) + ["-c", "import sys; print(sys.executable)"], + stderr=subprocess.STDOUT, + ) + .decode() + .strip() + ) + return sys_exe + except (subprocess.CalledProcessError, FileNotFoundError) as e: + raise PythonExecutableInferenceError( + "failed to find a Python executable matching version {}," + " perhaps try --python-executable, or --no-site-packages?".format(python_version) + ) from e + + +def infer_python_executable(options: Options, special_opts: argparse.Namespace) -> None: + """Infer the Python executable from the given version. + + This function mutates options based on special_opts to infer the correct Python executable + to use. + """ + # TODO: (ethanhs) Look at folding these checks and the site packages subprocess calls into + # one subprocess call for speed. + + # Use the command line specified executable, or fall back to one set in the + # config file. If an executable is not specified, infer it from the version + # (unless no_executable is set) + python_executable = special_opts.python_executable or options.python_executable + + if python_executable is None: + if not special_opts.no_executable and not options.no_site_packages: + python_executable = _python_executable_from_version(options.python_version) + options.python_executable = python_executable + + +HEADER: Final = """%(prog)s [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...]""" + + +DESCRIPTION: Final = """ +Mypy is a program that will type check your Python code. + +Pass in any files or folders you want to type check. Mypy will +recursively traverse any provided folders to find .py files: + + $ mypy my_program.py my_src_folder + +For more information on getting started, see: + +- https://mypy.readthedocs.io/en/stable/getting_started.html + +For more details on both running mypy and using the flags below, see: + +- https://mypy.readthedocs.io/en/stable/running_mypy.html +- https://mypy.readthedocs.io/en/stable/command_line.html + +You can also use a config file to configure mypy instead of using +command line flags. For more details, see: + +- https://mypy.readthedocs.io/en/stable/config_file.html +""" + +FOOTER: Final = """Environment variables: + Define MYPYPATH for additional module search path entries. + Define MYPY_CACHE_DIR to override configuration cache_dir path.""" + + +class CapturableArgumentParser(argparse.ArgumentParser): + """Override ArgumentParser methods that use sys.stdout/sys.stderr directly. + + This is needed because hijacking sys.std* is not thread-safe, + yet output must be captured to properly support mypy.api.run. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.stdout = kwargs.pop("stdout", sys.stdout) + self.stderr = kwargs.pop("stderr", sys.stderr) + super().__init__(*args, **kwargs) + + # ===================== + # Help-printing methods + # ===================== + def print_usage(self, file: IO[str] | None = None) -> None: + if file is None: + file = self.stdout + self._print_message(self.format_usage(), file) + + def print_help(self, file: IO[str] | None = None) -> None: + if file is None: + file = self.stdout + self._print_message(self.format_help(), file) + + def _print_message(self, message: str, file: IO[str] | None = None) -> None: + if message: + if file is None: + file = self.stderr + file.write(message) + + # =============== + # Exiting methods + # =============== + def exit(self, status: int = 0, message: str | None = None) -> NoReturn: + if message: + self._print_message(message, self.stderr) + sys.exit(status) + + def error(self, message: str) -> NoReturn: + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + self.print_usage(self.stderr) + args = {"prog": self.prog, "message": message} + self.exit(2, gettext("%(prog)s: error: %(message)s\n") % args) + + +class CapturableVersionAction(argparse.Action): + """Supplement CapturableArgumentParser to handle --version. + + This is nearly identical to argparse._VersionAction except, + like CapturableArgumentParser, it allows output to be captured. + + Another notable difference is that version is mandatory. + This allows removing a line in __call__ that falls back to parser.version + (which does not appear to exist). + """ + + def __init__( + self, + option_strings: Sequence[str], + version: str, + dest: str = argparse.SUPPRESS, + default: str = argparse.SUPPRESS, + help: str = "show program's version number and exit", + stdout: IO[str] | None = None, + ) -> None: + super().__init__( + option_strings=option_strings, dest=dest, default=default, nargs=0, help=help + ) + self.version = version + self.stdout = stdout or sys.stdout + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: str | Sequence[Any] | None, + option_string: str | None = None, + ) -> NoReturn: + formatter = parser._get_formatter() + formatter.add_text(self.version) + parser._print_message(formatter.format_help(), self.stdout) + parser.exit() + + +def process_options( + args: list[str], + stdout: TextIO | None = None, + stderr: TextIO | None = None, + require_targets: bool = True, + server_options: bool = False, + fscache: FileSystemCache | None = None, + program: str = "mypy", + header: str = HEADER, +) -> tuple[list[BuildSource], Options]: + """Parse command line arguments. + + If a FileSystemCache is passed in, and package_root options are given, + call fscache.set_package_root() to set the cache's package root. + """ + stdout = stdout or sys.stdout + stderr = stderr or sys.stderr + + parser = CapturableArgumentParser( + prog=program, + usage=header, + description=DESCRIPTION, + epilog=FOOTER, + fromfile_prefix_chars="@", + formatter_class=AugmentedHelpFormatter, + add_help=False, + stdout=stdout, + stderr=stderr, + ) + + strict_flag_names: list[str] = [] + strict_flag_assignments: list[tuple[str, bool]] = [] + + def add_invertible_flag( + flag: str, + *, + inverse: str | None = None, + default: bool, + dest: str | None = None, + help: str, + strict_flag: bool = False, + group: argparse._ActionsContainer | None = None, + ) -> None: + if inverse is None: + inverse = invert_flag_name(flag) + if group is None: + group = parser + + if help is not argparse.SUPPRESS: + help += f" (inverse: {inverse})" + + arg = group.add_argument( + flag, action="store_false" if default else "store_true", dest=dest, help=help + ) + dest = arg.dest + group.add_argument( + inverse, + action="store_true" if default else "store_false", + dest=dest, + help=argparse.SUPPRESS, + ) + if strict_flag: + assert dest is not None + strict_flag_names.append(flag) + strict_flag_assignments.append((dest, not default)) + + # Unless otherwise specified, arguments will be parsed directly onto an + # Options object. Options that require further processing should have + # their `dest` prefixed with `special-opts:`, which will cause them to be + # parsed into the separate special_opts namespace object. + + # Note: we have a style guide for formatting the mypy --help text. See + # https://github.com/python/mypy/wiki/Documentation-Conventions + + general_group = parser.add_argument_group(title="Optional arguments") + general_group.add_argument( + "-h", "--help", action="help", help="Show this help message and exit" + ) + general_group.add_argument( + "-v", "--verbose", action="count", dest="verbosity", help="More verbose messages" + ) + + compilation_status = "no" if __file__.endswith(".py") else "yes" + general_group.add_argument( + "-V", + "--version", + action=CapturableVersionAction, + version="%(prog)s " + __version__ + f" (compiled: {compilation_status})", + help="Show program's version number and exit", + stdout=stdout, + ) + + general_group.add_argument( + "-O", + "--output", + metavar="FORMAT", + help="Set a custom output format", + choices=OUTPUT_CHOICES, + ) + + config_group = parser.add_argument_group( + title="Config file", + description="Use a config file instead of command line arguments. " + "This is useful if you are using many flags or want " + "to set different options per each module.", + ) + config_group.add_argument( + "--config-file", + help=( + f"Configuration file, must have a [mypy] section " + f"(defaults to {', '.join(defaults.CONFIG_FILES)})" + ), + ) + add_invertible_flag( + "--warn-unused-configs", + default=False, + strict_flag=True, + help="Warn about unused '[mypy-]' or '[[tool.mypy.overrides]]' config sections", + group=config_group, + ) + + imports_group = parser.add_argument_group( + title="Import discovery", description="Configure how imports are discovered and followed." + ) + add_invertible_flag( + "--no-namespace-packages", + dest="namespace_packages", + default=True, + help="Disable support for namespace packages (PEP 420, __init__.py-less)", + group=imports_group, + ) + imports_group.add_argument( + "--ignore-missing-imports", + action="store_true", + help="Silently ignore imports of missing modules", + ) + imports_group.add_argument( + "--follow-untyped-imports", + action="store_true", + help="Typecheck modules without stubs or py.typed marker", + ) + imports_group.add_argument( + "--follow-imports", + choices=["normal", "silent", "skip", "error"], + default="normal", + help="How to treat imports (default normal)", + ) + imports_group.add_argument( + "--python-executable", + action="store", + metavar="EXECUTABLE", + help="Python executable used for finding PEP 561 compliant installed packages and stubs", + dest="special-opts:python_executable", + ) + imports_group.add_argument( + "--no-site-packages", + action="store_true", + dest="special-opts:no_executable", + help="Do not search for installed PEP 561 compliant packages", + ) + imports_group.add_argument( + "--no-silence-site-packages", + action="store_true", + help="Do not silence errors in PEP 561 compliant installed packages", + ) + + platform_group = parser.add_argument_group( + title="Platform configuration", + description="Type check code assuming it will be run under certain " + "runtime conditions. By default, mypy assumes your code " + "will be run using the same operating system and Python " + "version you are using to run mypy itself.", + ) + platform_group.add_argument( + "--python-version", + type=parse_version, + metavar="x.y", + help="Type check code assuming it will be running on Python x.y", + dest="special-opts:python_version", + ) + platform_group.add_argument( + "--platform", + action="store", + metavar="PLATFORM", + help="Type check special-cased code for the given OS platform (defaults to sys.platform)", + ) + platform_group.add_argument( + "--always-true", + metavar="NAME", + action="append", + default=[], + help="Additional variable to be considered True (may be repeated)", + ) + platform_group.add_argument( + "--always-false", + metavar="NAME", + action="append", + default=[], + help="Additional variable to be considered False (may be repeated)", + ) + + disallow_any_group = parser.add_argument_group( + title="Disallow dynamic typing", + description="Disallow the use of the dynamic 'Any' type under certain conditions.", + ) + disallow_any_group.add_argument( + "--disallow-any-expr", + default=False, + action="store_true", + help="Disallow all expressions that have type Any", + ) + disallow_any_group.add_argument( + "--disallow-any-decorated", + default=False, + action="store_true", + help="Disallow functions that have Any in their signature after decorator transformation", + ) + disallow_any_group.add_argument( + "--disallow-any-explicit", + default=False, + action="store_true", + help="Disallow explicit Any in type positions", + ) + add_invertible_flag( + "--disallow-any-generics", + default=False, + strict_flag=True, + help="Disallow usage of generic types that do not specify explicit type parameters", + group=disallow_any_group, + ) + add_invertible_flag( + "--disallow-any-unimported", + default=False, + help="Disallow Any types resulting from unfollowed imports", + group=disallow_any_group, + ) + add_invertible_flag( + "--disallow-subclassing-any", + default=False, + strict_flag=True, + help="Disallow subclassing values of type 'Any' when defining classes", + group=disallow_any_group, + ) + + untyped_group = parser.add_argument_group( + title="Untyped definitions and calls", + description="Configure how untyped definitions and calls are handled. " + "Note: by default, mypy ignores any untyped function definitions " + "and assumes any calls to such functions have a return " + "type of 'Any'.", + ) + add_invertible_flag( + "--disallow-untyped-calls", + default=False, + strict_flag=True, + help="Disallow calling functions without type annotations" + " from functions with type annotations", + group=untyped_group, + ) + untyped_group.add_argument( + "--untyped-calls-exclude", + metavar="MODULE", + action="append", + default=[], + help="Disable --disallow-untyped-calls for functions/methods coming" + " from specific package, module, or class", + ) + add_invertible_flag( + "--disallow-untyped-defs", + default=False, + strict_flag=True, + help="Disallow defining functions without type annotations" + " or with incomplete type annotations", + group=untyped_group, + ) + add_invertible_flag( + "--disallow-incomplete-defs", + default=False, + strict_flag=True, + help="Disallow defining functions with incomplete type annotations " + "(while still allowing entirely unannotated definitions)", + group=untyped_group, + ) + add_invertible_flag( + "--check-untyped-defs", + default=False, + strict_flag=True, + help="Type check the interior of functions without type annotations", + group=untyped_group, + ) + add_invertible_flag( + "--disallow-untyped-decorators", + default=False, + strict_flag=True, + help="Disallow decorating typed functions with untyped decorators", + group=untyped_group, + ) + + none_group = parser.add_argument_group( + title="None and Optional handling", + description="Adjust how values of type 'None' are handled. For more context on " + "how mypy handles values of type 'None', see: " + "https://mypy.readthedocs.io/en/stable/kinds_of_types.html#no-strict-optional", + ) + add_invertible_flag( + "--implicit-optional", + default=False, + help="Assume arguments with default values of None are Optional", + group=none_group, + ) + none_group.add_argument("--strict-optional", action="store_true", help=argparse.SUPPRESS) + none_group.add_argument( + "--no-strict-optional", + action="store_false", + dest="strict_optional", + help="Disable strict Optional checks (inverse: --strict-optional)", + ) + + add_invertible_flag( + "--force-uppercase-builtins", default=False, help=argparse.SUPPRESS, group=none_group + ) + + add_invertible_flag( + "--force-union-syntax", default=False, help=argparse.SUPPRESS, group=none_group + ) + + lint_group = parser.add_argument_group( + title="Configuring warnings", + description="Detect code that is sound but redundant or problematic.", + ) + add_invertible_flag( + "--warn-redundant-casts", + default=False, + strict_flag=True, + help="Warn about casting an expression to its inferred type", + group=lint_group, + ) + add_invertible_flag( + "--warn-unused-ignores", + default=False, + strict_flag=True, + help="Warn about unneeded '# type: ignore' comments", + group=lint_group, + ) + add_invertible_flag( + "--no-warn-no-return", + dest="warn_no_return", + default=True, + help="Do not warn about functions that end without returning", + group=lint_group, + ) + add_invertible_flag( + "--warn-return-any", + default=False, + strict_flag=True, + help="Warn about returning values of type Any from non-Any typed functions", + group=lint_group, + ) + add_invertible_flag( + "--warn-unreachable", + default=False, + strict_flag=False, + help="Warn about statements or expressions inferred to be unreachable", + group=lint_group, + ) + add_invertible_flag( + "--report-deprecated-as-note", + default=False, + strict_flag=False, + help="Report importing or using deprecated features as notes instead of errors", + group=lint_group, + ) + + # Note: this group is intentionally added here even though we don't add + # --strict to this group near the end. + # + # That way, this group will appear after the various strictness groups + # but before the remaining flags. + # We add `--strict` near the end so we don't accidentally miss any strictness + # flags that are added after this group. + strictness_group = parser.add_argument_group(title="Miscellaneous strictness flags") + + add_invertible_flag( + "--allow-untyped-globals", + default=False, + strict_flag=False, + help="Suppress toplevel errors caused by missing annotations", + group=strictness_group, + ) + + add_invertible_flag( + "--allow-redefinition", + default=False, + strict_flag=False, + help="Allow unconditional variable redefinition with a new type", + group=strictness_group, + ) + + add_invertible_flag( + "--no-implicit-reexport", + default=True, + strict_flag=True, + dest="implicit_reexport", + help="Treat imports as private unless aliased", + group=strictness_group, + ) + + add_invertible_flag( + "--strict-equality", + default=False, + strict_flag=True, + help="Prohibit equality, identity, and container checks for non-overlapping types", + group=strictness_group, + ) + + add_invertible_flag( + "--extra-checks", + default=False, + strict_flag=True, + help="Enable additional checks that are technically correct but may be impractical " + "in real code. For example, this prohibits partial overlap in TypedDict updates, " + "and makes arguments prepended via Concatenate positional-only", + group=strictness_group, + ) + + strict_help = "Strict mode; enables the following flags: {}".format( + ", ".join(strict_flag_names) + ) + strictness_group.add_argument( + "--strict", action="store_true", dest="special-opts:strict", help=strict_help + ) + + strictness_group.add_argument( + "--disable-error-code", + metavar="NAME", + action="append", + default=[], + help="Disable a specific error code", + ) + strictness_group.add_argument( + "--enable-error-code", + metavar="NAME", + action="append", + default=[], + help="Enable a specific error code", + ) + + error_group = parser.add_argument_group( + title="Configuring error messages", + description="Adjust the amount of detail shown in error messages.", + ) + add_invertible_flag( + "--show-error-context", + default=False, + dest="show_error_context", + help='Precede errors with "note:" messages explaining context', + group=error_group, + ) + add_invertible_flag( + "--show-column-numbers", + default=False, + help="Show column numbers in error messages", + group=error_group, + ) + add_invertible_flag( + "--show-error-end", + default=False, + help="Show end line/end column numbers in error messages." + " This implies --show-column-numbers", + group=error_group, + ) + add_invertible_flag( + "--hide-error-codes", + default=False, + help="Hide error codes in error messages", + group=error_group, + ) + add_invertible_flag( + "--show-error-code-links", + default=False, + help="Show links to error code documentation", + group=error_group, + ) + add_invertible_flag( + "--pretty", + default=False, + help="Use visually nicer output in error messages:" + " Use soft word wrap, show source code snippets," + " and show error location markers", + group=error_group, + ) + add_invertible_flag( + "--no-color-output", + dest="color_output", + default=True, + help="Do not colorize error messages", + group=error_group, + ) + add_invertible_flag( + "--no-error-summary", + dest="error_summary", + default=True, + help="Do not show error stats summary", + group=error_group, + ) + add_invertible_flag( + "--show-absolute-path", + default=False, + help="Show absolute paths to files", + group=error_group, + ) + error_group.add_argument( + "--soft-error-limit", + default=defaults.MANY_ERRORS_THRESHOLD, + type=int, + dest="many_errors_threshold", + help=argparse.SUPPRESS, + ) + + incremental_group = parser.add_argument_group( + title="Incremental mode", + description="Adjust how mypy incrementally type checks and caches modules. " + "Mypy caches type information about modules into a cache to " + "let you speed up future invocations of mypy. Also see " + "mypy's daemon mode: " + "mypy.readthedocs.io/en/stable/mypy_daemon.html#mypy-daemon", + ) + incremental_group.add_argument( + "-i", "--incremental", action="store_true", help=argparse.SUPPRESS + ) + incremental_group.add_argument( + "--no-incremental", + action="store_false", + dest="incremental", + help="Disable module cache (inverse: --incremental)", + ) + incremental_group.add_argument( + "--cache-dir", + action="store", + metavar="DIR", + help="Store module cache info in the given folder in incremental mode " + "(defaults to '{}')".format(defaults.CACHE_DIR), + ) + add_invertible_flag( + "--sqlite-cache", + default=False, + help="Use a sqlite database to store the cache", + group=incremental_group, + ) + incremental_group.add_argument( + "--cache-fine-grained", + action="store_true", + help="Include fine-grained dependency information in the cache for the mypy daemon", + ) + incremental_group.add_argument( + "--skip-version-check", + action="store_true", + help="Allow using cache written by older mypy version", + ) + incremental_group.add_argument( + "--skip-cache-mtime-checks", + action="store_true", + help="Skip cache internal consistency checks based on mtime", + ) + + internals_group = parser.add_argument_group( + title="Advanced options", description="Debug and customize mypy internals." + ) + internals_group.add_argument("--pdb", action="store_true", help="Invoke pdb on fatal error") + internals_group.add_argument( + "--show-traceback", "--tb", action="store_true", help="Show traceback on fatal error" + ) + internals_group.add_argument( + "--raise-exceptions", action="store_true", help="Raise exception on fatal error" + ) + internals_group.add_argument( + "--custom-typing-module", + metavar="MODULE", + dest="custom_typing_module", + help="Use a custom typing module", + ) + internals_group.add_argument( + "--old-type-inference", + action="store_true", + help="Disable new experimental type inference algorithm", + ) + # Deprecated reverse variant of the above. + internals_group.add_argument( + "--new-type-inference", action="store_true", help=argparse.SUPPRESS + ) + parser.add_argument( + "--enable-incomplete-feature", + action="append", + metavar="{" + ",".join(sorted(INCOMPLETE_FEATURES)) + "}", + help="Enable support of incomplete/experimental features for early preview", + ) + internals_group.add_argument( + "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR" + ) + add_invertible_flag( + "--warn-incomplete-stub", + default=False, + help="Warn if missing type annotation in typeshed, only relevant with" + " --disallow-untyped-defs or --disallow-incomplete-defs enabled", + group=internals_group, + ) + internals_group.add_argument( + "--shadow-file", + nargs=2, + metavar=("SOURCE_FILE", "SHADOW_FILE"), + dest="shadow_file", + action="append", + help="When encountering SOURCE_FILE, read and type check " + "the contents of SHADOW_FILE instead.", + ) + internals_group.add_argument("--fast-exit", action="store_true", help=argparse.SUPPRESS) + internals_group.add_argument( + "--no-fast-exit", action="store_false", dest="fast_exit", help=argparse.SUPPRESS + ) + # This flag is useful for mypy tests, where function bodies may be omitted. Plugin developers + # may want to use this as well in their tests. + add_invertible_flag( + "--allow-empty-bodies", default=False, help=argparse.SUPPRESS, group=internals_group + ) + # This undocumented feature exports limited line-level dependency information. + internals_group.add_argument("--export-ref-info", action="store_true", help=argparse.SUPPRESS) + + report_group = parser.add_argument_group( + title="Report generation", description="Generate a report in the specified format." + ) + for report_type in sorted(defaults.REPORTER_NAMES): + if report_type not in {"memory-xml"}: + report_group.add_argument( + f"--{report_type.replace('_', '-')}-report", + metavar="DIR", + dest=f"special-opts:{report_type}_report", + ) + + other_group = parser.add_argument_group(title="Miscellaneous") + other_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) + other_group.add_argument("--junit-xml", help="Write junit.xml to the given file") + imports_group.add_argument( + "--junit-format", + choices=["global", "per_file"], + default="global", + help="If --junit-xml is set, specifies format. global: single test with all errors; per_file: one test entry per file with failures", + ) + other_group.add_argument( + "--find-occurrences", + metavar="CLASS.MEMBER", + dest="special-opts:find_occurrences", + help="Print out all usages of a class member (experimental)", + ) + other_group.add_argument( + "--scripts-are-modules", + action="store_true", + help="Script x becomes module x instead of __main__", + ) + + add_invertible_flag( + "--install-types", + default=False, + strict_flag=False, + help="Install detected missing library stub packages using pip", + group=other_group, + ) + add_invertible_flag( + "--non-interactive", + default=False, + strict_flag=False, + help=( + "Install stubs without asking for confirmation and hide " + + "errors, with --install-types" + ), + group=other_group, + inverse="--interactive", + ) + + if server_options: + # TODO: This flag is superfluous; remove after a short transition (2018-03-16) + other_group.add_argument( + "--experimental", + action="store_true", + dest="fine_grained_incremental", + help="Enable fine-grained incremental mode", + ) + other_group.add_argument( + "--use-fine-grained-cache", + action="store_true", + help="Use the cache in fine-grained incremental mode", + ) + + # hidden options + parser.add_argument( + "--stats", action="store_true", dest="dump_type_stats", help=argparse.SUPPRESS + ) + parser.add_argument( + "--inferstats", action="store_true", dest="dump_inference_stats", help=argparse.SUPPRESS + ) + parser.add_argument("--dump-build-stats", action="store_true", help=argparse.SUPPRESS) + # Dump timing stats for each processed file into the given output file + parser.add_argument("--timing-stats", dest="timing_stats", help=argparse.SUPPRESS) + # Dump per line type checking timing stats for each processed file into the given + # output file. Only total time spent in each top level expression will be shown. + # Times are show in microseconds. + parser.add_argument( + "--line-checking-stats", dest="line_checking_stats", help=argparse.SUPPRESS + ) + # --debug-cache will disable any cache-related compressions/optimizations, + # which will make the cache writing process output pretty-printed JSON (which + # is easier to debug). + parser.add_argument("--debug-cache", action="store_true", help=argparse.SUPPRESS) + # --dump-deps will dump all fine-grained dependencies to stdout + parser.add_argument("--dump-deps", action="store_true", help=argparse.SUPPRESS) + # --dump-graph will dump the contents of the graph of SCCs and exit. + parser.add_argument("--dump-graph", action="store_true", help=argparse.SUPPRESS) + # --semantic-analysis-only does exactly that. + parser.add_argument("--semantic-analysis-only", action="store_true", help=argparse.SUPPRESS) + # Some tests use this to tell mypy that we are running a test. + parser.add_argument("--test-env", action="store_true", help=argparse.SUPPRESS) + # --local-partial-types disallows partial types spanning module top level and a function + # (implicitly defined in fine-grained incremental mode) + parser.add_argument("--local-partial-types", action="store_true", help=argparse.SUPPRESS) + # --logical-deps adds some more dependencies that are not semantically needed, but + # may be helpful to determine relative importance of classes and functions for overall + # type precision in a code base. It also _removes_ some deps, so this flag should be never + # used except for generating code stats. This also automatically enables --cache-fine-grained. + # NOTE: This is an experimental option that may be modified or removed at any time. + parser.add_argument("--logical-deps", action="store_true", help=argparse.SUPPRESS) + # --bazel changes some behaviors for use with Bazel (https://bazel.build). + parser.add_argument("--bazel", action="store_true", help=argparse.SUPPRESS) + # --package-root adds a directory below which directories are considered + # packages even without __init__.py. May be repeated. + parser.add_argument( + "--package-root", metavar="ROOT", action="append", default=[], help=argparse.SUPPRESS + ) + # --cache-map FILE ... gives a mapping from source files to cache files. + # Each triple of arguments is a source file, a cache meta file, and a cache data file. + # Modules not mentioned in the file will go through cache_dir. + # Must be followed by another flag or by '--' (and then only file args may follow). + parser.add_argument( + "--cache-map", nargs="+", dest="special-opts:cache_map", help=argparse.SUPPRESS + ) + # --debug-serialize will run tree.serialize() even if cache generation is disabled. + # Useful for mypy_primer to detect serialize errors earlier. + parser.add_argument("--debug-serialize", action="store_true", help=argparse.SUPPRESS) + + parser.add_argument( + "--disable-bytearray-promotion", action="store_true", help=argparse.SUPPRESS + ) + parser.add_argument( + "--disable-memoryview-promotion", action="store_true", help=argparse.SUPPRESS + ) + # This flag is deprecated, it has been moved to --extra-checks + parser.add_argument("--strict-concatenate", action="store_true", help=argparse.SUPPRESS) + + # options specifying code to check + code_group = parser.add_argument_group( + title="Running code", + description="Specify the code you want to type check. For more details, see " + "mypy.readthedocs.io/en/stable/running_mypy.html#running-mypy", + ) + add_invertible_flag( + "--explicit-package-bases", + default=False, + help="Use current directory and MYPYPATH to determine module names of files passed", + group=code_group, + ) + add_invertible_flag( + "--fast-module-lookup", default=False, help=argparse.SUPPRESS, group=code_group + ) + code_group.add_argument( + "--exclude", + action="append", + metavar="PATTERN", + default=[], + help=( + "Regular expression to match file names, directory names or paths which mypy should " + "ignore while recursively discovering files to check, e.g. --exclude '/setup\\.py$'. " + "May be specified more than once, eg. --exclude a --exclude b" + ), + ) + code_group.add_argument( + "-m", + "--module", + action="append", + metavar="MODULE", + default=[], + dest="special-opts:modules", + help="Type-check module; can repeat for more modules", + ) + code_group.add_argument( + "-p", + "--package", + action="append", + metavar="PACKAGE", + default=[], + dest="special-opts:packages", + help="Type-check package recursively; can be repeated", + ) + code_group.add_argument( + "-c", + "--command", + action="append", + metavar="PROGRAM_TEXT", + dest="special-opts:command", + help="Type-check program passed in as string", + ) + code_group.add_argument( + metavar="files", + nargs="*", + dest="special-opts:files", + help="Type-check given files or directories", + ) + + # Parse arguments once into a dummy namespace so we can get the + # filename for the config file and know if the user requested all strict options. + dummy = argparse.Namespace() + parser.parse_args(args, dummy) + config_file = dummy.config_file + # Don't explicitly test if "config_file is not None" for this check. + # This lets `--config-file=` (an empty string) be used to disable all config files. + if config_file and not os.path.exists(config_file): + parser.error(f"Cannot find config file '{config_file}'") + + options = Options() + strict_option_set = False + + def set_strict_flags() -> None: + nonlocal strict_option_set + strict_option_set = True + for dest, value in strict_flag_assignments: + setattr(options, dest, value) + + # Parse config file first, so command line can override. + parse_config_file(options, set_strict_flags, config_file, stdout, stderr) + + # Set strict flags before parsing (if strict mode enabled), so other command + # line options can override. + if getattr(dummy, "special-opts:strict"): + set_strict_flags() + + # Override cache_dir if provided in the environment + environ_cache_dir = os.getenv("MYPY_CACHE_DIR", "") + if environ_cache_dir.strip(): + options.cache_dir = environ_cache_dir + options.cache_dir = os.path.expanduser(options.cache_dir) + + # Parse command line for real, using a split namespace. + special_opts = argparse.Namespace() + parser.parse_args(args, SplitNamespace(options, special_opts, "special-opts:")) + + # The python_version is either the default, which can be overridden via a config file, + # or stored in special_opts and is passed via the command line. + options.python_version = special_opts.python_version or options.python_version + if options.python_version < (3,): + parser.error( + "Mypy no longer supports checking Python 2 code. " + "Consider pinning to mypy<0.980 if you need to check Python 2 code." + ) + try: + infer_python_executable(options, special_opts) + except PythonExecutableInferenceError as e: + parser.error(str(e)) + + if special_opts.no_executable or options.no_site_packages: + options.python_executable = None + + # Paths listed in the config file will be ignored if any paths, modules or packages + # are passed on the command line. + if not (special_opts.files or special_opts.packages or special_opts.modules): + if options.files: + special_opts.files = options.files + if options.packages: + special_opts.packages = options.packages + if options.modules: + special_opts.modules = options.modules + + # Check for invalid argument combinations. + if require_targets: + code_methods = sum( + bool(c) + for c in [ + special_opts.modules + special_opts.packages, + special_opts.command, + special_opts.files, + ] + ) + if code_methods == 0 and not options.install_types: + parser.error("Missing target module, package, files, or command.") + elif code_methods > 1: + parser.error("May only specify one of: module/package, files, or command.") + if options.explicit_package_bases and not options.namespace_packages: + parser.error( + "Can only use --explicit-package-bases with --namespace-packages, since otherwise " + "examining __init__.py's is sufficient to determine module names for files" + ) + + # Check for overlapping `--always-true` and `--always-false` flags. + overlap = set(options.always_true) & set(options.always_false) + if overlap: + parser.error( + "You can't make a variable always true and always false (%s)" + % ", ".join(sorted(overlap)) + ) + + validate_package_allow_list(options.untyped_calls_exclude) + + options.process_error_codes(error_callback=parser.error) + options.process_incomplete_features(error_callback=parser.error, warning_callback=print) + + # Compute absolute path for custom typeshed (if present). + if options.custom_typeshed_dir is not None: + options.abs_custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir) + + # Set build flags. + if special_opts.find_occurrences: + _find_occurrences = tuple(special_opts.find_occurrences.split(".")) + if len(_find_occurrences) < 2: + parser.error("Can only find occurrences of class members.") + if len(_find_occurrences) != 2: + parser.error("Can only find occurrences of non-nested class members.") + state.find_occurrences = _find_occurrences + + # Set reports. + for flag, val in vars(special_opts).items(): + if flag.endswith("_report") and val is not None: + report_type = flag[:-7].replace("_", "-") + report_dir = val + options.report_dirs[report_type] = report_dir + + # Process --package-root. + if options.package_root: + process_package_roots(fscache, parser, options) + + # Process --cache-map. + if special_opts.cache_map: + if options.sqlite_cache: + parser.error("--cache-map is incompatible with --sqlite-cache") + + process_cache_map(parser, special_opts, options) + + # An explicitly specified cache_fine_grained implies local_partial_types + # (because otherwise the cache is not compatible with dmypy) + if options.cache_fine_grained: + options.local_partial_types = True + + # Implicitly show column numbers if error location end is shown + if options.show_error_end: + options.show_column_numbers = True + + # Let logical_deps imply cache_fine_grained (otherwise the former is useless). + if options.logical_deps: + options.cache_fine_grained = True + + if options.new_type_inference: + print( + "Warning: --new-type-inference flag is deprecated;" + " new type inference algorithm is already enabled by default" + ) + + if options.strict_concatenate and not strict_option_set: + print("Warning: --strict-concatenate is deprecated; use --extra-checks instead") + + # Set target. + if special_opts.modules + special_opts.packages: + options.build_type = BuildType.MODULE + sys_path, _ = get_search_dirs(options.python_executable) + search_paths = SearchPaths( + (os.getcwd(),), tuple(mypy_path() + options.mypy_path), tuple(sys_path), () + ) + targets = [] + # TODO: use the same cache that the BuildManager will + cache = FindModuleCache(search_paths, fscache, options) + for p in special_opts.packages: + if os.sep in p or os.altsep and os.altsep in p: + fail(f"Package name '{p}' cannot have a slash in it.", stderr, options) + p_targets = cache.find_modules_recursive(p) + if not p_targets: + reason = cache.find_module(p) + if reason is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: + fail( + f"Package '{p}' cannot be type checked due to missing py.typed marker. See https://mypy.readthedocs.io/en/stable/installed_packages.html for more details", + stderr, + options, + ) + else: + fail(f"Can't find package '{p}'", stderr, options) + targets.extend(p_targets) + for m in special_opts.modules: + targets.append(BuildSource(None, m, None)) + return targets, options + elif special_opts.command: + options.build_type = BuildType.PROGRAM_TEXT + targets = [BuildSource(None, None, "\n".join(special_opts.command))] + return targets, options + else: + try: + targets = create_source_list(special_opts.files, options, fscache) + # Variable named e2 instead of e to work around mypyc bug #620 + # which causes issues when using the same variable to catch + # exceptions of different types. + except InvalidSourceList as e2: + fail(str(e2), stderr, options) + return targets, options + + +def process_package_roots( + fscache: FileSystemCache | None, parser: argparse.ArgumentParser, options: Options +) -> None: + """Validate and normalize package_root.""" + if fscache is None: + parser.error("--package-root does not work here (no fscache)") + assert fscache is not None # Since mypy doesn't know parser.error() raises. + # Do some stuff with drive letters to make Windows happy (esp. tests). + current_drive, _ = os.path.splitdrive(os.getcwd()) + dot = os.curdir + dotslash = os.curdir + os.sep + dotdotslash = os.pardir + os.sep + trivial_paths = {dot, dotslash} + package_root = [] + for root in options.package_root: + if os.path.isabs(root): + parser.error(f"Package root cannot be absolute: {root!r}") + drive, root = os.path.splitdrive(root) + if drive and drive != current_drive: + parser.error(f"Package root must be on current drive: {drive + root!r}") + # Empty package root is always okay. + if root: + root = os.path.relpath(root) # Normalize the heck out of it. + if not root.endswith(os.sep): + root = root + os.sep + if root.startswith(dotdotslash): + parser.error(f"Package root cannot be above current directory: {root!r}") + if root in trivial_paths: + root = "" + package_root.append(root) + options.package_root = package_root + # Pass the package root on the filesystem cache. + fscache.set_package_root(package_root) + + +def process_cache_map( + parser: argparse.ArgumentParser, special_opts: argparse.Namespace, options: Options +) -> None: + """Validate cache_map and copy into options.cache_map.""" + n = len(special_opts.cache_map) + if n % 3 != 0: + parser.error("--cache-map requires one or more triples (see source)") + for i in range(0, n, 3): + source, meta_file, data_file = special_opts.cache_map[i : i + 3] + if source in options.cache_map: + parser.error(f"Duplicate --cache-map source {source})") + if not source.endswith(".py") and not source.endswith(".pyi"): + parser.error(f"Invalid --cache-map source {source} (triple[0] must be *.py[i])") + if not meta_file.endswith(".meta.json"): + parser.error( + "Invalid --cache-map meta_file %s (triple[1] must be *.meta.json)" % meta_file + ) + if not data_file.endswith(".data.json"): + parser.error( + "Invalid --cache-map data_file %s (triple[2] must be *.data.json)" % data_file + ) + options.cache_map[source] = (meta_file, data_file) + + +def maybe_write_junit_xml( + td: float, + serious: bool, + all_messages: list[str], + messages_by_file: dict[str | None, list[str]], + options: Options, +) -> None: + if options.junit_xml: + py_version = f"{options.python_version[0]}_{options.python_version[1]}" + if options.junit_format == "global": + util.write_junit_xml( + td, + serious, + {None: all_messages} if all_messages else {}, + options.junit_xml, + py_version, + options.platform, + ) + else: + # per_file + util.write_junit_xml( + td, serious, messages_by_file, options.junit_xml, py_version, options.platform + ) + + +def fail(msg: str, stderr: TextIO, options: Options) -> NoReturn: + """Fail with a serious error.""" + stderr.write(f"{msg}\n") + maybe_write_junit_xml( + 0.0, serious=True, all_messages=[msg], messages_by_file={None: [msg]}, options=options + ) + sys.exit(2) + + +def read_types_packages_to_install(cache_dir: str, after_run: bool) -> list[str]: + if not os.path.isdir(cache_dir): + if not after_run: + sys.stderr.write( + "error: Can't determine which types to install with no files to check " + + "(and no cache from previous mypy run)\n" + ) + else: + sys.stderr.write("error: --install-types failed (no mypy cache directory)\n") + sys.exit(2) + fnam = build.missing_stubs_file(cache_dir) + if not os.path.isfile(fnam): + # No missing stubs. + return [] + with open(fnam) as f: + return [line.strip() for line in f] + + +def install_types( + formatter: util.FancyFormatter, + options: Options, + *, + after_run: bool = False, + non_interactive: bool = False, +) -> bool: + """Install stub packages using pip if some missing stubs were detected.""" + packages = read_types_packages_to_install(options.cache_dir, after_run) + if not packages: + # If there are no missing stubs, generate no output. + return False + if after_run and not non_interactive: + print() + print("Installing missing stub packages:") + assert options.python_executable, "Python executable required to install types" + cmd = [options.python_executable, "-m", "pip", "install"] + packages + print(formatter.style(" ".join(cmd), "none", bold=True)) + print() + if not non_interactive: + x = input("Install? [yN] ") + if not x.strip() or not x.lower().startswith("y"): + print(formatter.style("mypy: Skipping installation", "red", bold=True)) + sys.exit(2) + print() + subprocess.run(cmd) + return True diff --git a/mypy/maptype.py b/mypy/maptype.py index 73771313f2df..59ecb2bc9993 100644 --- a/mypy/maptype.py +++ b/mypy/maptype.py @@ -1,70 +1,68 @@ -from typing import Dict, List, cast +from __future__ import annotations -from mypy.expandtype import expand_type +from mypy.expandtype import expand_type_by_instance from mypy.nodes import TypeInfo -from mypy.types import Type, Instance, AnyType +from mypy.types import AnyType, Instance, TupleType, TypeOfAny, has_type_vars -def map_instance_to_supertype(instance: Instance, - supertype: TypeInfo) -> Instance: - """Map an Instance type, including the type arguments, to compatible - Instance of a specific supertype. +def map_instance_to_supertype(instance: Instance, superclass: TypeInfo) -> Instance: + """Produce a supertype of `instance` that is an Instance + of `superclass`, mapping type arguments up the chain of bases. - Assume that supertype is a supertype of instance.type. + If `superclass` is not a nominal superclass of `instance.type`, + then all type arguments are mapped to 'Any'. """ - if instance.type == supertype: + if instance.type == superclass: + # Fast path: `instance` already belongs to `superclass`. return instance - # Strip type variables away if the supertype has none. - if not supertype.type_vars: - return Instance(supertype, []) - - return map_instance_to_supertypes(instance, supertype)[0] - - -def map_instance_to_direct_supertype(instance: Instance, - supertype: TypeInfo) -> Instance: - typ = instance.type - - for base in typ.bases: - if base.type == supertype: - map = type_var_map(typ, instance.args) - return cast(Instance, expand_type(base, map)) - - # Relationship with the supertype not specified explicitly. Use AnyType - # type arguments implicitly. - # TODO Should this be an error instead? - return Instance(supertype, [AnyType()] * len(supertype.type_vars)) - - -def type_var_map(typ: TypeInfo, args: List[Type]) -> Dict[int, Type]: - if not args: - return None - else: - tvars = {} # type: Dict[int, Type] - for i in range(len(args)): - tvars[i + 1] = args[i] - return tvars - - -def map_instance_to_supertypes(instance: Instance, - supertype: TypeInfo) -> List[Instance]: + if superclass.fullname == "builtins.tuple" and instance.type.tuple_type: + if has_type_vars(instance.type.tuple_type): + # We special case mapping generic tuple types to tuple base, because for + # such tuples fallback can't be calculated before applying type arguments. + alias = instance.type.special_alias + assert alias is not None + if not alias._is_recursive: + # Unfortunately we can't support this for generic recursive tuples. + # If we skip this special casing we will fall back to tuple[Any, ...]. + tuple_type = expand_type_by_instance(instance.type.tuple_type, instance) + if isinstance(tuple_type, TupleType): + # Make the import here to avoid cyclic imports. + import mypy.typeops + + return mypy.typeops.tuple_fallback(tuple_type) + elif isinstance(tuple_type, Instance): + # This can happen after normalizing variadic tuples. + return tuple_type + + if not superclass.type_vars: + # Fast path: `superclass` has no type variables to map to. + return Instance(superclass, []) + + return map_instance_to_supertypes(instance, superclass)[0] + + +def map_instance_to_supertypes(instance: Instance, supertype: TypeInfo) -> list[Instance]: # FIX: Currently we should only have one supertype per interface, so no # need to return an array - result = [] # type: List[Instance] + result: list[Instance] = [] for path in class_derivation_paths(instance.type, supertype): types = [instance] for sup in path: - a = [] # type: List[Instance] + a: list[Instance] = [] for t in types: a.extend(map_instance_to_direct_supertypes(t, sup)) types = a result.extend(types) - return result + if result: + return result + else: + # Nothing. Presumably due to an error. Construct a dummy using Any. + any_type = AnyType(TypeOfAny.from_error) + return [Instance(supertype, [any_type] * len(supertype.type_vars))] -def class_derivation_paths(typ: TypeInfo, - supertype: TypeInfo) -> List[List[TypeInfo]]: +def class_derivation_paths(typ: TypeInfo, supertype: TypeInfo) -> list[list[TypeInfo]]: """Return an array of non-empty paths of direct base classes from type to supertype. Return [] if no such path could be found. @@ -74,33 +72,35 @@ def class_derivation_paths(typ: TypeInfo, """ # FIX: Currently we might only ever have a single path, so this could be # simplified - result = [] # type: List[List[TypeInfo]] + result: list[list[TypeInfo]] = [] for base in typ.bases: - if base.type == supertype: - result.append([base.type]) + btype = base.type + if btype == supertype: + result.append([btype]) else: # Try constructing a longer path via the base class. - for path in class_derivation_paths(base.type, supertype): - result.append([base.type] + path) + for path in class_derivation_paths(btype, supertype): + result.append([btype] + path) return result -def map_instance_to_direct_supertypes(instance: Instance, - supertype: TypeInfo) -> List[Instance]: +def map_instance_to_direct_supertypes(instance: Instance, supertype: TypeInfo) -> list[Instance]: # FIX: There should only be one supertypes, always. typ = instance.type - result = [] # type: List[Instance] + result: list[Instance] = [] for b in typ.bases: if b.type == supertype: - map = type_var_map(typ, instance.args) - result.append(cast(Instance, expand_type(b, map))) + t = expand_type_by_instance(b, instance) + assert isinstance(t, Instance) + result.append(t) if result: return result else: # Relationship with the supertype not specified explicitly. Use dynamic # type arguments implicitly. - return [Instance(supertype, [AnyType()] * len(supertype.type_vars))] + any_type = AnyType(TypeOfAny.unannotated) + return [Instance(supertype, [any_type] * len(supertype.type_vars))] diff --git a/mypy/meet.py b/mypy/meet.py index ffeb7b18ea7b..cbe3e99cdcd8 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1,207 +1,1196 @@ -from typing import cast, List +from __future__ import annotations -from mypy.join import is_similar_callables, combine_similar_callables +from typing import Callable + +from mypy import join +from mypy.erasetype import erase_type +from mypy.maptype import map_instance_to_supertype +from mypy.state import state +from mypy.subtypes import ( + are_parameters_compatible, + find_member, + is_callable_compatible, + is_equivalent, + is_proper_subtype, + is_same_type, + is_subtype, +) +from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback from mypy.types import ( - Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVar, - Instance, Callable, TupleType, ErasedType, TypeList, UnionType + MYPYC_NATIVE_INT_NAMES, + TUPLE_LIKE_INSTANCE_NAMES, + AnyType, + CallableType, + DeletedType, + ErasedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeGuardedType, + TypeOfAny, + TypeType, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + find_unpack_in_list, + get_proper_type, + get_proper_types, + split_with_prefix_and_suffix, ) -from mypy.sametypes import is_same_type -from mypy.subtypes import is_subtype -from mypy.nodes import TypeInfo # TODO Describe this module. -def meet_types(s: Type, t: Type) -> Type: +def trivial_meet(s: Type, t: Type) -> ProperType: + """Return one of types (expanded) if it is a subtype of other, otherwise bottom type.""" + if is_subtype(s, t): + return get_proper_type(s) + elif is_subtype(t, s): + return get_proper_type(t) + else: + if state.strict_optional: + return UninhabitedType() + else: + return NoneType() + + +def meet_types(s: Type, t: Type) -> ProperType: """Return the greatest lower bound of two types.""" + if is_recursive_pair(s, t): + # This case can trigger an infinite recursion, general support for this will be + # tricky, so we use a trivial meet (like for protocols). + return trivial_meet(s, t) + s = get_proper_type(s) + t = get_proper_type(t) + + if isinstance(s, Instance) and isinstance(t, Instance) and s.type == t.type: + # Code in checker.py should merge any extra_items where possible, so we + # should have only compatible extra_items here. We check this before + # the below subtype check, so that extra_attrs will not get erased. + if (s.extra_attrs or t.extra_attrs) and is_same_type(s, t): + if s.extra_attrs and t.extra_attrs: + if len(s.extra_attrs.attrs) > len(t.extra_attrs.attrs): + # Return the one that has more precise information. + return s + return t + if s.extra_attrs: + return s + return t + + if not isinstance(s, UnboundType) and not isinstance(t, UnboundType): + if is_proper_subtype(s, t, ignore_promotions=True): + return s + if is_proper_subtype(t, s, ignore_promotions=True): + return t + if isinstance(s, ErasedType): return s if isinstance(s, AnyType): return t if isinstance(s, UnionType) and not isinstance(t, UnionType): s, t = t, s + + # Meets/joins require callable type normalization. + s, t = join.normalize_callables(s, t) + return t.accept(TypeMeetVisitor(s)) -def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: - if s == t: - return s - if isinstance(s, UnionType): - return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) - elif not is_overlapping_types(s, t): - return Void() - else: - if default_right: - return t +def narrow_declared_type(declared: Type, narrowed: Type) -> Type: + """Return the declared type narrowed down to another type.""" + # TODO: check infinite recursion for aliases here. + if isinstance(narrowed, TypeGuardedType): # type: ignore[misc] + # A type guard forces the new type even if it doesn't overlap the old. + return narrowed.type_guard + + original_declared = declared + original_narrowed = narrowed + declared = get_proper_type(declared) + narrowed = get_proper_type(narrowed) + + if declared == narrowed: + return original_declared + if isinstance(declared, UnionType): + return make_simplified_union( + [ + narrow_declared_type(x, narrowed) + for x in declared.relevant_items() + # This (ugly) special-casing is needed to support checking + # branches like this: + # x: Union[float, complex] + # if isinstance(x, int): + # ... + if ( + is_overlapping_types(x, narrowed, ignore_promotions=True) + or is_subtype(narrowed, x, ignore_promotions=False) + ) + ] + ) + if is_enum_overlapping_union(declared, narrowed): + return original_narrowed + elif not is_overlapping_types(declared, narrowed, prohibit_none_typevar_overlap=True): + if state.strict_optional: + return UninhabitedType() else: - return s + return NoneType() + elif isinstance(narrowed, UnionType): + return make_simplified_union( + [narrow_declared_type(declared, x) for x in narrowed.relevant_items()] + ) + elif isinstance(narrowed, AnyType): + return original_narrowed + elif isinstance(narrowed, TypeVarType) and is_subtype(narrowed.upper_bound, declared): + return narrowed + elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType): + return TypeType.make_normalized(narrow_declared_type(declared.item, narrowed.item)) + elif ( + isinstance(declared, TypeType) + and isinstance(narrowed, Instance) + and narrowed.type.is_metaclass() + ): + # We'd need intersection types, so give up. + return original_declared + elif isinstance(declared, Instance): + if declared.type.alt_promote: + # Special case: low-level integer type can't be narrowed + return original_declared + if ( + isinstance(narrowed, Instance) + and narrowed.type.alt_promote + and narrowed.type.alt_promote.type is declared.type + ): + # Special case: 'int' can't be narrowed down to a native int type such as + # i64, since they have different runtime representations. + return original_declared + return meet_types(original_declared, original_narrowed) + elif isinstance(declared, (TupleType, TypeType, LiteralType)): + return meet_types(original_declared, original_narrowed) + elif isinstance(declared, TypedDictType) and isinstance(narrowed, Instance): + # Special case useful for selecting TypedDicts from unions using isinstance(x, dict). + if narrowed.type.fullname == "builtins.dict" and all( + isinstance(t, AnyType) for t in get_proper_types(narrowed.args) + ): + return original_declared + return meet_types(original_declared, original_narrowed) + return original_narrowed + + +def get_possible_variants(typ: Type) -> list[Type]: + """This function takes any "Union-like" type and returns a list of the available "options". + + Specifically, there are currently exactly three different types that can have + "variants" or are "union-like": + - Unions + - TypeVars with value restrictions + - Overloads -def meet_simple_away(s: Type, t: Type) -> Type: - if isinstance(s, UnionType): - return UnionType.make_simplified_union([x for x in s.items - if not is_subtype(x, t)]) - elif not isinstance(s, AnyType) and is_subtype(s, t): - return Void() + This function will return a list of each "option" present in those types. + + If this function receives any other type, we return a list containing just that + original type. (E.g. pretend the type was contained within a singleton union). + + The only current exceptions are regular TypeVars and ParamSpecs. For these "TypeVarLike"s, + we return a list containing that TypeVarLike's upper bound. + + This function is useful primarily when checking to see if two types are overlapping: + the algorithm to check if two unions are overlapping is fundamentally the same as + the algorithm for checking if two overloads are overlapping. + + Normalizing both kinds of types in the same way lets us reuse the same algorithm + for both. + """ + typ = get_proper_type(typ) + + if isinstance(typ, TypeVarType): + if len(typ.values) > 0: + return typ.values + else: + return [typ.upper_bound] + elif isinstance(typ, ParamSpecType): + # Extract 'object' from the final mro item + upper_bound = get_proper_type(typ.upper_bound) + if isinstance(upper_bound, Instance): + return [Instance(upper_bound.type.mro[-1], [])] + return [AnyType(TypeOfAny.implementation_artifact)] + elif isinstance(typ, TypeVarTupleType): + return [typ.upper_bound] + elif isinstance(typ, UnionType): + return list(typ.items) + elif isinstance(typ, Overloaded): + # Note: doing 'return typ.items()' makes mypy + # infer a too-specific return type of List[CallableType] + return list(typ.items) else: - return s + return [typ] -def is_overlapping_types(t: Type, s: Type) -> bool: - """Can a value of type t be a value of type s, or vice versa?""" - if isinstance(t, Instance): - if isinstance(s, Instance): - # If the classes are explicitly declared as disjoint, they can't - # overlap. - if t.type in s.type.disjoint_classes: - return False - - # Built-in classes in the mro affect whether two types can be - # overlapping. - # TODO Find the most distant ancestor with the same memory layout, - # since multiple inheritance seems possible if the memory - # layout is the same. - tbuiltin = nearest_builtin_ancestor(t.type) - sbuiltin = nearest_builtin_ancestor(s.type) - if not sbuiltin or not tbuiltin: - return True +def is_enum_overlapping_union(x: ProperType, y: ProperType) -> bool: + """Return True if x is an Enum, and y is an Union with at least one Literal from x""" + return ( + isinstance(x, Instance) + and x.type.is_enum + and isinstance(y, UnionType) + and any( + isinstance(p := get_proper_type(z), LiteralType) and x.type == p.fallback.type + for z in y.relevant_items() + ) + ) + + +def is_literal_in_union(x: ProperType, y: ProperType) -> bool: + """Return True if x is a Literal and y is an Union that includes x""" + return ( + isinstance(x, LiteralType) + and isinstance(y, UnionType) + and any(x == get_proper_type(z) for z in y.items) + ) + + +def is_object(t: ProperType) -> bool: + return isinstance(t, Instance) and t.type.fullname == "builtins.object" + + +def is_overlapping_types( + left: Type, + right: Type, + ignore_promotions: bool = False, + prohibit_none_typevar_overlap: bool = False, + overlap_for_overloads: bool = False, + seen_types: set[tuple[Type, Type]] | None = None, +) -> bool: + """Can a value of type 'left' also be of type 'right' or vice-versa? + + If 'ignore_promotions' is True, we ignore promotions while checking for overlaps. + If 'prohibit_none_typevar_overlap' is True, we disallow None from overlapping with + TypeVars (in both strict-optional and non-strict-optional mode). + If 'overlap_for_overloads' is True, we check for overlaps more strictly (to avoid false + positives), for example: None only overlaps with explicitly optional types, Any + doesn't overlap with anything except object, we don't ignore positional argument names. + """ + if isinstance(left, TypeGuardedType) or isinstance( # type: ignore[misc] + right, TypeGuardedType + ): + # A type guard forces the new type even if it doesn't overlap the old. + return True + + if seen_types is None: + seen_types = set() + if (left, right) in seen_types: + return True + if isinstance(left, TypeAliasType) and isinstance(right, TypeAliasType): + seen_types.add((left, right)) + + left, right = get_proper_types((left, right)) + + def _is_overlapping_types(left: Type, right: Type) -> bool: + """Encode the kind of overlapping check to perform. + + This function mostly exists, so we don't have to repeat keyword arguments everywhere. + """ + return is_overlapping_types( + left, + right, + ignore_promotions=ignore_promotions, + prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, + overlap_for_overloads=overlap_for_overloads, + seen_types=seen_types.copy(), + ) + + # We should never encounter this type. + if isinstance(left, PartialType) or isinstance(right, PartialType): + assert False, "Unexpectedly encountered partial type" + + # We should also never encounter these types, but it's possible a few + # have snuck through due to unrelated bugs. For now, we handle these + # in the same way we handle 'Any'. + # + # TODO: Replace these with an 'assert False' once we are more confident. + illegal_types = (UnboundType, ErasedType, DeletedType) + if isinstance(left, illegal_types) or isinstance(right, illegal_types): + return True + + # When running under non-strict optional mode, simplify away types of + # the form 'Union[A, B, C, None]' into just 'Union[A, B, C]'. - # If one is a base class of other, the types overlap, unless there - # is an explicit disjointclass constraint. - if tbuiltin in sbuiltin.mro or sbuiltin in tbuiltin.mro: + if not state.strict_optional: + if isinstance(left, UnionType): + left = UnionType.make_union(left.relevant_items()) + if isinstance(right, UnionType): + right = UnionType.make_union(right.relevant_items()) + left, right = get_proper_types((left, right)) + + # 'Any' may or may not be overlapping with the other type + if isinstance(left, AnyType) or isinstance(right, AnyType): + return not overlap_for_overloads or is_object(left) or is_object(right) + + # We check for complete overlaps next as a general-purpose failsafe. + # If this check fails, we start checking to see if there exists a + # *partial* overlap between types. + # + # These checks will also handle the NoneType and UninhabitedType cases for us. + + # enums are sometimes expanded into an Union of Literals + # when that happens we want to make sure we treat the two as overlapping + # and crucially, we want to do that *fast* in case the enum is large + # so we do it before expanding variants below to avoid O(n**2) behavior + if ( + is_enum_overlapping_union(left, right) + or is_enum_overlapping_union(right, left) + or is_literal_in_union(left, right) + or is_literal_in_union(right, left) + ): + return True + + def is_none_object_overlap(t1: Type, t2: Type) -> bool: + t1, t2 = get_proper_types((t1, t2)) + return ( + isinstance(t1, NoneType) + and isinstance(t2, Instance) + and t2.type.fullname == "builtins.object" + ) + + if overlap_for_overloads: + if is_none_object_overlap(left, right) or is_none_object_overlap(right, left): + return False + + def _is_subtype(left: Type, right: Type) -> bool: + if overlap_for_overloads: + return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) + else: + return is_subtype(left, right, ignore_promotions=ignore_promotions) + + if _is_subtype(left, right) or _is_subtype(right, left): + return True + + # See the docstring for 'get_possible_variants' for more info on what the + # following lines are doing. + + left_possible = get_possible_variants(left) + right_possible = get_possible_variants(right) + + # Now move on to checking multi-variant types like Unions. We also perform + # the same logic if either type happens to be a TypeVar/ParamSpec/TypeVarTuple. + # + # Handling the TypeVarLikes now lets us simulate having them bind to the corresponding + # type -- if we deferred these checks, the "return-early" logic of the other + # checks will prevent us from detecting certain overlaps. + # + # If both types are singleton variants (and are not TypeVarLikes), we've hit the base case: + # we skip these checks to avoid infinitely recursing. + + def is_none_typevarlike_overlap(t1: Type, t2: Type) -> bool: + t1, t2 = get_proper_types((t1, t2)) + return isinstance(t1, NoneType) and isinstance(t2, TypeVarLikeType) + + if prohibit_none_typevar_overlap: + if is_none_typevarlike_overlap(left, right) or is_none_typevarlike_overlap(right, left): + return False + + if ( + len(left_possible) > 1 + or len(right_possible) > 1 + or isinstance(left, TypeVarLikeType) + or isinstance(right, TypeVarLikeType) + ): + for l in left_possible: + for r in right_possible: + if _is_overlapping_types(l, r): + return True + return False + + # Now that we've finished handling TypeVarLikes, we're free to end early + # if one one of the types is None and we're running in strict-optional mode. + # (None only overlaps with None in strict-optional mode). + # + # We must perform this check after the TypeVarLike checks because + # a TypeVar could be bound to None, for example. + + if state.strict_optional and isinstance(left, NoneType) != isinstance(right, NoneType): + return False + + # Next, we handle single-variant types that may be inherently partially overlapping: + # + # - TypedDicts + # - Tuples + # + # If we cannot identify a partial overlap and end early, we degrade these two types + # into their 'Instance' fallbacks. + + if isinstance(left, TypedDictType) and isinstance(right, TypedDictType): + return are_typed_dicts_overlapping(left, right, _is_overlapping_types) + elif typed_dict_mapping_pair(left, right): + # Overlaps between TypedDicts and Mappings require dedicated logic. + return typed_dict_mapping_overlap(left, right, overlapping=_is_overlapping_types) + elif isinstance(left, TypedDictType): + left = left.fallback + elif isinstance(right, TypedDictType): + right = right.fallback + + if is_tuple(left) and is_tuple(right): + return are_tuples_overlapping(left, right, _is_overlapping_types) + elif isinstance(left, TupleType): + left = tuple_fallback(left) + elif isinstance(right, TupleType): + right = tuple_fallback(right) + + # Next, we handle single-variant types that cannot be inherently partially overlapping, + # but do require custom logic to inspect. + # + # As before, we degrade into 'Instance' whenever possible. + + if isinstance(left, TypeType) and isinstance(right, TypeType): + return _is_overlapping_types(left.item, right.item) + + def _type_object_overlap(left: Type, right: Type) -> bool: + """Special cases for type object types overlaps.""" + # TODO: these checks are a bit in gray area, adjust if they cause problems. + left, right = get_proper_types((left, right)) + # 1. Type[C] vs Callable[..., C] overlap even if the latter is not class object. + if isinstance(left, TypeType) and isinstance(right, CallableType): + return _is_overlapping_types(left.item, right.ret_type) + # 2. Type[C] vs Meta, where Meta is a metaclass for C. + if isinstance(left, TypeType) and isinstance(right, Instance): + if isinstance(left.item, Instance): + left_meta = left.item.type.metaclass_type + if left_meta is not None: + return _is_overlapping_types(left_meta, right) + # builtins.type (default metaclass) overlaps with all metaclasses + return right.type.has_base("builtins.type") + elif isinstance(left.item, AnyType): + return right.type.has_base("builtins.type") + # 3. Callable[..., C] vs Meta is considered below, when we switch to fallbacks. + return False + + if isinstance(left, TypeType) or isinstance(right, TypeType): + return _type_object_overlap(left, right) or _type_object_overlap(right, left) + + if isinstance(left, Parameters) and isinstance(right, Parameters): + return are_parameters_compatible( + left, + right, + is_compat=_is_overlapping_types, + is_proper_subtype=False, + ignore_pos_arg_names=not overlap_for_overloads, + allow_partial_overlap=True, + ) + # A `Parameters` does not overlap with anything else, however + if isinstance(left, Parameters) or isinstance(right, Parameters): + return False + + if isinstance(left, CallableType) and isinstance(right, CallableType): + return is_callable_compatible( + left, + right, + is_compat=_is_overlapping_types, + is_proper_subtype=False, + ignore_pos_arg_names=not overlap_for_overloads, + allow_partial_overlap=True, + ) + + call = None + other = None + if isinstance(left, CallableType) and isinstance(right, Instance): + call = find_member("__call__", right, right, is_operator=True) + other = left + if isinstance(right, CallableType) and isinstance(left, Instance): + call = find_member("__call__", left, left, is_operator=True) + other = right + if isinstance(get_proper_type(call), FunctionLike): + assert call is not None and other is not None + return _is_overlapping_types(call, other) + + if isinstance(left, CallableType): + left = left.fallback + if isinstance(right, CallableType): + right = right.fallback + + if isinstance(left, LiteralType) and isinstance(right, LiteralType): + if left.value == right.value: + # If values are the same, we still need to check if fallbacks are overlapping, + # this is done below. + left = left.fallback + right = right.fallback + else: + return False + elif isinstance(left, LiteralType): + left = left.fallback + elif isinstance(right, LiteralType): + right = right.fallback + + # Finally, we handle the case where left and right are instances. + + if isinstance(left, Instance) and isinstance(right, Instance): + # First we need to handle promotions and structural compatibility for instances + # that came as fallbacks, so simply call is_subtype() to avoid code duplication. + if _is_subtype(left, right) or _is_subtype(right, left): + return True + + if right.type.fullname == "builtins.int" and left.type.fullname in MYPYC_NATIVE_INT_NAMES: + return True + + # Two unrelated types cannot be partially overlapping: they're disjoint. + if left.type.has_base(right.type.fullname): + left = map_instance_to_supertype(left, right.type) + elif right.type.has_base(left.type.fullname): + right = map_instance_to_supertype(right, left.type) + else: + return False + + if len(left.args) == len(right.args): + # Note: we don't really care about variance here, since the overlapping check + # is symmetric and since we want to return 'True' even for partial overlaps. + # + # For example, suppose we have two types Wrapper[Parent] and Wrapper[Child]. + # It doesn't matter whether Wrapper is covariant or contravariant since + # either way, one of the two types will overlap with the other. + # + # Similarly, if Wrapper was invariant, the two types could still be partially + # overlapping -- what if Wrapper[Parent] happened to contain only instances of + # specifically Child? + # + # Or, to use a more concrete example, List[Union[A, B]] and List[Union[B, C]] + # would be considered partially overlapping since it's possible for both lists + # to contain only instances of B at runtime. + if all( + _is_overlapping_types(left_arg, right_arg) + for left_arg, right_arg in zip(left.args, right.args) + ): return True - return tbuiltin == sbuiltin - if isinstance(t, UnionType): - return any(is_overlapping_types(item, s) - for item in t.items) - if isinstance(s, UnionType): - return any(is_overlapping_types(t, item) - for item in s.items) - # We conservatively assume that non-instance, non-union types can overlap any other - # types. + + return False + + # We ought to have handled every case by now: we conclude the + # two types are not overlapping, either completely or partially. + # + # Note: it's unclear however, whether returning False is the right thing + # to do when inferring reachability -- see https://github.com/python/mypy/issues/5529 + + assert type(left) != type(right), f"{type(left)} vs {type(right)}" + return False + + +def is_overlapping_erased_types( + left: Type, right: Type, *, ignore_promotions: bool = False +) -> bool: + """The same as 'is_overlapping_erased_types', except the types are erased first.""" + return is_overlapping_types( + erase_type(left), + erase_type(right), + ignore_promotions=ignore_promotions, + prohibit_none_typevar_overlap=True, + ) + + +def are_typed_dicts_overlapping( + left: TypedDictType, right: TypedDictType, is_overlapping: Callable[[Type, Type], bool] +) -> bool: + """Returns 'true' if left and right are overlapping TypeDictTypes.""" + # All required keys in left are present and overlapping with something in right + for key in left.required_keys: + if key not in right.items: + return False + if not is_overlapping(left.items[key], right.items[key]): + return False + + # Repeat check in the other direction + for key in right.required_keys: + if key not in left.items: + return False + if not is_overlapping(left.items[key], right.items[key]): + return False + + # The presence of any additional optional keys does not affect whether the two + # TypedDicts are partially overlapping: the dicts would be overlapping if the + # keys happened to be missing. return True -def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: - for base in type.mro: - if base.defn.is_builtinclass: - return base - else: - return None - assert False, 'No built-in ancestor found for {}'.format(type.name()) +def are_tuples_overlapping( + left: Type, right: Type, is_overlapping: Callable[[Type, Type], bool] +) -> bool: + """Returns true if left and right are overlapping tuples.""" + left, right = get_proper_types((left, right)) + left = adjust_tuple(left, right) or left + right = adjust_tuple(right, left) or right + assert isinstance(left, TupleType), f"Type {left} is not a tuple" + assert isinstance(right, TupleType), f"Type {right} is not a tuple" + # This algorithm works well if only one tuple is variadic, if both are + # variadic we may get rare false negatives for overlapping prefix/suffix. + # Also, this ignores empty unpack case, but it is probably consistent with + # how we handle e.g. empty lists in overload overlaps. + # TODO: write a more robust algorithm for cases where both types are variadic. + left_unpack = find_unpack_in_list(left.items) + right_unpack = find_unpack_in_list(right.items) + if left_unpack is not None: + left = expand_tuple_if_possible(left, len(right.items)) + if right_unpack is not None: + right = expand_tuple_if_possible(right, len(left.items)) -class TypeMeetVisitor(TypeVisitor[Type]): - def __init__(self, s: Type) -> None: - self.s = s + if len(left.items) != len(right.items): + return False + return all(is_overlapping(l, r) for l, r in zip(left.items, right.items)) - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - elif isinstance(self.s, NoneTyp): - return self.s + +def expand_tuple_if_possible(tup: TupleType, target: int) -> TupleType: + if len(tup.items) > target + 1: + return tup + extra = target + 1 - len(tup.items) + new_items = [] + for it in tup.items: + if not isinstance(it, UnpackType): + new_items.append(it) + continue + unpacked = get_proper_type(it.type) + if isinstance(unpacked, TypeVarTupleType): + instance = unpacked.tuple_fallback else: - return AnyType() + # Nested non-variadic tuples should be normalized at this point. + assert isinstance(unpacked, Instance) + instance = unpacked + assert instance.type.fullname == "builtins.tuple" + new_items.extend([instance.args[0]] * extra) + return tup.copy_modified(items=new_items) - def visit_error_type(self, t: ErrorType) -> Type: - return t - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' +def adjust_tuple(left: ProperType, r: ProperType) -> TupleType | None: + """Find out if `left` is a Tuple[A, ...], and adjust its length to `right`""" + if isinstance(left, Instance) and left.type.fullname == "builtins.tuple": + n = r.length() if isinstance(r, TupleType) else 1 + return TupleType([left.args[0]] * n, left) + return None + + +def is_tuple(typ: Type) -> bool: + typ = get_proper_type(typ) + return isinstance(typ, TupleType) or ( + isinstance(typ, Instance) and typ.type.fullname == "builtins.tuple" + ) + + +class TypeMeetVisitor(TypeVisitor[ProperType]): + def __init__(self, s: ProperType) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> ProperType: + if isinstance(self.s, NoneType): + if state.strict_optional: + return UninhabitedType() + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return AnyType(TypeOfAny.special_form) - def visit_any(self, t: AnyType) -> Type: + def visit_any(self, t: AnyType) -> ProperType: return self.s - def visit_union_type(self, t: UnionType) -> Type: + def visit_union_type(self, t: UnionType) -> ProperType: if isinstance(self.s, UnionType): - meets = List[Type]() + meets: list[Type] = [] for x in t.items: for y in self.s.items: meets.append(meet_types(x, y)) else: - meets = [meet_types(x, self.s) - for x in t.items] - return UnionType.make_simplified_union(meets) + meets = [meet_types(x, self.s) for x in t.items] + return make_simplified_union(meets) - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t + def visit_none_type(self, t: NoneType) -> ProperType: + if state.strict_optional: + if isinstance(self.s, NoneType) or ( + isinstance(self.s, Instance) and self.s.type.fullname == "builtins.object" + ): + return t + else: + return UninhabitedType() else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> ProperType: + return t + + def visit_deleted_type(self, t: DeletedType) -> ProperType: + if isinstance(self.s, NoneType): + if state.strict_optional: + return t + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s else: - return ErrorType() + return t - def visit_erased_type(self, t: ErasedType) -> Type: + def visit_erased_type(self, t: ErasedType) -> ProperType: return self.s - def visit_type_var(self, t: TypeVar) -> Type: - if isinstance(self.s, TypeVar) and (cast(TypeVar, self.s)).id == t.id: + def visit_type_var(self, t: TypeVarType) -> ProperType: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_param_spec(self, t: ParamSpecType) -> ProperType: + if self.s == t: return self.s else: return self.default(self.s) - def visit_instance(self, t: Instance) -> Type: + def visit_type_var_tuple(self, t: TypeVarTupleType) -> ProperType: + if isinstance(self.s, TypeVarTupleType) and self.s.id == t.id: + return self.s if self.s.min_len > t.min_len else t + else: + return self.default(self.s) + + def visit_unpack_type(self, t: UnpackType) -> ProperType: + raise NotImplementedError + + def visit_parameters(self, t: Parameters) -> ProperType: + if isinstance(self.s, Parameters): + if len(t.arg_types) != len(self.s.arg_types): + return self.default(self.s) + from mypy.join import join_types + + return t.copy_modified( + arg_types=[join_types(s_a, t_a) for s_a, t_a in zip(self.s.arg_types, t.arg_types)] + ) + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> ProperType: if isinstance(self.s, Instance): - si = cast(Instance, self.s) - if t.type == si.type: - if is_subtype(t, self.s): + if t.type == self.s.type: + if is_subtype(t, self.s) or is_subtype(self.s, t): # Combine type arguments. We could have used join below # equivalently. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(self.meet(t.args[i], si.args[i])) + args: list[Type] = [] + # N.B: We use zip instead of indexing because the lengths might have + # mismatches during daemon reprocessing. + if t.type.has_type_var_tuple_type: + # We handle meet of variadic instances by simply creating correct mapping + # for type arguments and compute the individual meets same as for regular + # instances. All the heavy lifting is done in the meet of tuple types. + s = self.s + assert s.type.type_var_tuple_prefix is not None + assert s.type.type_var_tuple_suffix is not None + prefix = s.type.type_var_tuple_prefix + suffix = s.type.type_var_tuple_suffix + tvt = s.type.defn.type_vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + s_prefix, s_middle, s_suffix = split_with_prefix_and_suffix( + s.args, prefix, suffix + ) + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix( + t.args, prefix, suffix + ) + s_args = s_prefix + (TupleType(list(s_middle), fallback),) + s_suffix + t_args = t_prefix + (TupleType(list(t_middle), fallback),) + t_suffix + else: + t_args = t.args + s_args = self.s.args + for ta, sa, tv in zip(t_args, s_args, t.type.defn.type_vars): + meet = self.meet(ta, sa) + if isinstance(tv, TypeVarTupleType): + # Correctly unpack possible outcomes of meets of tuples: it can be + # either another tuple type or Never (normalized as *tuple[Never, ...]) + if isinstance(meet, TupleType): + args.extend(meet.items) + continue + else: + assert isinstance(meet, UninhabitedType) + meet = UnpackType(tv.tuple_fallback.copy_modified(args=[meet])) + args.append(meet) return Instance(t.type, args) else: - return NoneTyp() + if state.strict_optional: + return UninhabitedType() + else: + return NoneType() else: + alt_promote = t.type.alt_promote + if alt_promote and alt_promote.type is self.s.type: + return t + alt_promote = self.s.type.alt_promote + if alt_promote and alt_promote.type is t.type: + return self.s if is_subtype(t, self.s): return t elif is_subtype(self.s, t): # See also above comment. return self.s else: - return NoneTyp() - else: + if state.strict_optional: + return UninhabitedType() + else: + return NoneType() + elif isinstance(self.s, FunctionLike) and t.type.is_protocol: + call = join.unpack_callback_protocol(t) + if call: + return meet_types(call, self.s) + elif isinstance(self.s, FunctionLike) and self.s.is_type_obj() and t.type.is_metaclass(): + if is_subtype(self.s.fallback, t): + return self.s return self.default(self.s) + elif isinstance(self.s, TypeType): + return meet_types(t, self.s) + elif isinstance(self.s, TupleType): + return meet_types(t, self.s) + elif isinstance(self.s, LiteralType): + return meet_types(t, self.s) + elif isinstance(self.s, TypedDictType): + return meet_types(t, self.s) + return self.default(self.s) - def visit_callable(self, t: Callable) -> Type: - if isinstance(self.s, Callable) and is_similar_callables( - t, cast(Callable, self.s)): - return combine_similar_callables(t, cast(Callable, self.s)) - else: + def visit_callable_type(self, t: CallableType) -> ProperType: + if isinstance(self.s, CallableType) and join.is_similar_callables(t, self.s): + if is_equivalent(t, self.s): + return join.combine_similar_callables(t, self.s) + result = meet_similar_callables(t, self.s) + # We set the from_type_type flag to suppress error when a collection of + # concrete class objects gets inferred as their common abstract superclass. + if not ( + (t.is_type_obj() and t.type_object().is_abstract) + or (self.s.is_type_obj() and self.s.type_object().is_abstract) + ): + result.from_type_type = True + if isinstance(get_proper_type(result.ret_type), UninhabitedType): + # Return a plain None or instead of a weird function. + return self.default(self.s) + return result + elif isinstance(self.s, TypeType) and t.is_type_obj() and not t.is_generic(): + # In this case we are able to potentially produce a better meet. + res = meet_types(self.s.item, t.ret_type) + if not isinstance(res, (NoneType, UninhabitedType)): + return TypeType.make_normalized(res) return self.default(self.s) + elif isinstance(self.s, Instance) and self.s.type.is_protocol: + call = join.unpack_callback_protocol(self.s) + if call: + return meet_types(t, call) + return self.default(self.s) + + def visit_overloaded(self, t: Overloaded) -> ProperType: + # TODO: Implement a better algorithm that covers at least the same cases + # as TypeJoinVisitor.visit_overloaded(). + s = self.s + if isinstance(s, FunctionLike): + if s.items == t.items: + return Overloaded(t.items) + elif is_subtype(s, t): + return s + elif is_subtype(t, s): + return t + else: + return meet_types(t.fallback, s.fallback) + elif isinstance(self.s, Instance) and self.s.type.is_protocol: + call = join.unpack_callback_protocol(self.s) + if call: + return meet_types(t, call) + return meet_types(t.fallback, s) + + def meet_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: + """Meet two tuple types while handling variadic entries. + + This is surprisingly tricky, and we don't handle some tricky corner cases. + Most of the trickiness comes from the variadic tuple items like *tuple[X, ...] + since they can have arbitrary partial overlaps (while *Ts can't be split). This + function is roughly a mirror of join_tuples() w.r.t. to the fact that fixed + tuples are subtypes of variadic ones but not vice versa. + """ + s_unpack_index = find_unpack_in_list(s.items) + t_unpack_index = find_unpack_in_list(t.items) + if s_unpack_index is None and t_unpack_index is None: + if s.length() == t.length(): + items: list[Type] = [] + for i in range(t.length()): + items.append(self.meet(t.items[i], s.items[i])) + return items + return None + if s_unpack_index is not None and t_unpack_index is not None: + # The only simple case we can handle if both tuples are variadic + # is when their structure fully matches. Other cases are tricky because + # a variadic item is effectively a union of tuples of all length, thus + # potentially causing overlap between a suffix in `s` and a prefix + # in `t` (see how this is handled in is_subtype() for details). + # TODO: handle more cases (like when both prefix/suffix are shorter in s or t). + if s.length() == t.length() and s_unpack_index == t_unpack_index: + unpack_index = s_unpack_index + s_unpack = s.items[unpack_index] + assert isinstance(s_unpack, UnpackType) + s_unpacked = get_proper_type(s_unpack.type) + t_unpack = t.items[unpack_index] + assert isinstance(t_unpack, UnpackType) + t_unpacked = get_proper_type(t_unpack.type) + if not (isinstance(s_unpacked, Instance) and isinstance(t_unpacked, Instance)): + return None + meet = self.meet(s_unpacked, t_unpacked) + if not isinstance(meet, Instance): + return None + m_prefix: list[Type] = [] + for si, ti in zip(s.items[:unpack_index], t.items[:unpack_index]): + m_prefix.append(meet_types(si, ti)) + m_suffix: list[Type] = [] + for si, ti in zip(s.items[unpack_index + 1 :], t.items[unpack_index + 1 :]): + m_suffix.append(meet_types(si, ti)) + return m_prefix + [UnpackType(meet)] + m_suffix + return None + if s_unpack_index is not None: + variadic = s + unpack_index = s_unpack_index + fixed = t + else: + assert t_unpack_index is not None + variadic = t + unpack_index = t_unpack_index + fixed = s + # If one tuple is variadic one, and the other one is fixed, the meet will be fixed. + unpack = variadic.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if not isinstance(unpacked, Instance): + return None + if fixed.length() < variadic.length() - 1: + return None + prefix_len = unpack_index + suffix_len = variadic.length() - prefix_len - 1 + prefix, middle, suffix = split_with_prefix_and_suffix( + tuple(fixed.items), prefix_len, suffix_len + ) + items = [] + for fi, vi in zip(prefix, variadic.items[:prefix_len]): + items.append(self.meet(fi, vi)) + for mi in middle: + items.append(self.meet(mi, unpacked.args[0])) + if suffix_len: + for fi, vi in zip(suffix, variadic.items[-suffix_len:]): + items.append(self.meet(fi, vi)) + return items - def visit_tuple_type(self, t: TupleType) -> Type: - if isinstance(self.s, TupleType) and ( - cast(TupleType, self.s).length() == t.length()): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.meet(t.items[i], - (cast(TupleType, self.s)).items[i])) + def visit_tuple_type(self, t: TupleType) -> ProperType: + if isinstance(self.s, TupleType): + items = self.meet_tuples(self.s, t) + if items is None: + return self.default(self.s) # TODO: What if the fallbacks are different? - return TupleType(items, t.fallback) + return TupleType(items, tuple_fallback(t)) + elif isinstance(self.s, Instance): + # meet(Tuple[t1, t2, <...>], Tuple[s, ...]) == Tuple[meet(t1, s), meet(t2, s), <...>]. + if self.s.type.fullname in TUPLE_LIKE_INSTANCE_NAMES and self.s.args: + return t.copy_modified(items=[meet_types(it, self.s.args[0]) for it in t.items]) + elif is_proper_subtype(t, self.s): + # A named tuple that inherits from a normal class + return t + elif self.s.type.has_type_var_tuple_type and is_subtype(t, self.s): + # This is a bit ad-hoc but more principled handling is tricky, and this + # special case is important for type narrowing in binder to work. + return t + return self.default(self.s) + + def visit_typeddict_type(self, t: TypedDictType) -> ProperType: + if isinstance(self.s, TypedDictType): + for name, l, r in self.s.zip(t): + if not is_equivalent(l, r) or (name in t.required_keys) != ( + name in self.s.required_keys + ): + return self.default(self.s) + item_list: list[tuple[str, Type]] = [] + for item_name, s_item_type, t_item_type in self.s.zipall(t): + if s_item_type is not None: + item_list.append((item_name, s_item_type)) + else: + # at least one of s_item_type and t_item_type is not None + assert t_item_type is not None + item_list.append((item_name, t_item_type)) + items = dict(item_list) + fallback = self.s.create_anonymous_fallback() + required_keys = t.required_keys | self.s.required_keys + readonly_keys = t.readonly_keys | self.s.readonly_keys + return TypedDictType(items, required_keys, readonly_keys, fallback) + elif isinstance(self.s, Instance) and is_subtype(t, self.s): + return t else: return self.default(self.s) - def visit_intersection(self, t): - # TODO Obsolete; target overload types instead? - # Only support very rudimentary meets between intersection types. - if is_same_type(self.s, t): - return self.s + def visit_literal_type(self, t: LiteralType) -> ProperType: + if isinstance(self.s, LiteralType) and self.s == t: + return t + elif isinstance(self.s, Instance) and is_subtype(t.fallback, self.s): + return t + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> ProperType: + # We can't determine the meet of partial types. We should never get here. + assert False, "Internal error" + + def visit_type_type(self, t: TypeType) -> ProperType: + if isinstance(self.s, TypeType): + typ = self.meet(t.item, self.s.item) + if not isinstance(typ, NoneType): + typ = TypeType.make_normalized(typ, line=t.line) + return typ + elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type": + return t + elif isinstance(self.s, CallableType): + return self.meet(t, self.s) else: return self.default(self.s) - def meet(self, s, t): + def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: + assert False, f"This should be never called, got {t}" + + def meet(self, s: Type, t: Type) -> ProperType: return meet_types(s, t) - def default(self, typ): + def default(self, typ: Type) -> ProperType: if isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() + return AnyType(TypeOfAny.special_form) else: - return NoneTyp() + if state.strict_optional: + return UninhabitedType() + else: + return NoneType() + + +def meet_similar_callables(t: CallableType, s: CallableType) -> CallableType: + from mypy.join import match_generic_callables, safe_join + + t, s = match_generic_callables(t, s) + arg_types: list[Type] = [] + for i in range(len(t.arg_types)): + arg_types.append(safe_join(t.arg_types[i], s.arg_types[i])) + # TODO in combine_similar_callables also applies here (names and kinds) + # The fallback type can be either 'function' or 'type'. The result should have 'function' as + # fallback only if both operands have it as 'function'. + if t.fallback.type.fullname != "builtins.function": + fallback = t.fallback + else: + fallback = s.fallback + return t.copy_modified( + arg_types=arg_types, + ret_type=meet_types(t.ret_type, s.ret_type), + fallback=fallback, + name=None, + ) + + +def meet_type_list(types: list[Type]) -> Type: + if not types: + # This should probably be builtins.object but that is hard to get and + # it doesn't matter for any current users. + return AnyType(TypeOfAny.implementation_artifact) + met = types[0] + for t in types[1:]: + met = meet_types(met, t) + return met + + +def typed_dict_mapping_pair(left: Type, right: Type) -> bool: + """Is this a pair where one type is a TypedDict and another one is an instance of Mapping? + + This case requires a precise/principled consideration because there are two use cases + that push the boundary the opposite ways: we need to avoid spurious overlaps to avoid + false positives for overloads, but we also need to avoid spuriously non-overlapping types + to avoid false positives with --strict-equality. + """ + left, right = get_proper_types((left, right)) + assert not isinstance(left, TypedDictType) or not isinstance(right, TypedDictType) + + if isinstance(left, TypedDictType): + _, other = left, right + elif isinstance(right, TypedDictType): + _, other = right, left + else: + return False + return isinstance(other, Instance) and other.type.has_base("typing.Mapping") + + +def typed_dict_mapping_overlap( + left: Type, right: Type, overlapping: Callable[[Type, Type], bool] +) -> bool: + """Check if a TypedDict type is overlapping with a Mapping. + + The basic logic here consists of two rules: + + * A TypedDict with some required keys is overlapping with Mapping[str, ] + if and only if every key type is overlapping with . For example: + + - TypedDict(x=int, y=str) overlaps with Dict[str, Union[str, int]] + - TypedDict(x=int, y=str) doesn't overlap with Dict[str, int] + + Note that any additional non-required keys can't change the above result. + + * A TypedDict with no required keys overlaps with Mapping[str, ] if and + only if at least one of key types overlaps with . For example: + + - TypedDict(x=str, y=str, total=False) overlaps with Dict[str, str] + - TypedDict(x=str, y=str, total=False) doesn't overlap with Dict[str, int] + - TypedDict(x=int, y=str, total=False) overlaps with Dict[str, str] + + * A TypedDict with at least one ReadOnly[] key does not overlap + with Dict or MutableMapping, because they assume mutable data. + + As usual empty, dictionaries lie in a gray area. In general, List[str] and List[str] + are considered non-overlapping despite empty list belongs to both. However, List[int] + and List[Never] are considered overlapping. + + So here we follow the same logic: a TypedDict with no required keys is considered + non-overlapping with Mapping[str, ], but is considered overlapping with + Mapping[Never, Never]. This way we avoid false positives for overloads, and also + avoid false positives for comparisons like SomeTypedDict == {} under --strict-equality. + """ + left, right = get_proper_types((left, right)) + assert not isinstance(left, TypedDictType) or not isinstance(right, TypedDictType) + + if isinstance(left, TypedDictType): + assert isinstance(right, Instance) + typed, other = left, right + else: + assert isinstance(left, Instance) + assert isinstance(right, TypedDictType) + typed, other = right, left + + mutable_mapping = next( + (base for base in other.type.mro if base.fullname == "typing.MutableMapping"), None + ) + if mutable_mapping is not None and typed.readonly_keys: + return False + + mapping = next(base for base in other.type.mro if base.fullname == "typing.Mapping") + other = map_instance_to_supertype(other, mapping) + key_type, value_type = get_proper_types(other.args) + + # TODO: is there a cleaner way to get str_type here? + fallback = typed.as_anonymous().fallback + str_type = fallback.type.bases[0].args[0] # typing._TypedDict inherits Mapping[str, object] + + # Special case: a TypedDict with no required keys overlaps with an empty dict. + if isinstance(key_type, UninhabitedType) and isinstance(value_type, UninhabitedType): + return not typed.required_keys + + if typed.required_keys: + if not overlapping(key_type, str_type): + return False + return all(overlapping(typed.items[k], value_type) for k in typed.required_keys) + else: + if not overlapping(key_type, str_type): + return False + non_required = set(typed.items.keys()) - typed.required_keys + return any(overlapping(typed.items[k], value_type) for k in non_required) diff --git a/mypy/memprofile.py b/mypy/memprofile.py new file mode 100644 index 000000000000..48c0cb5ce022 --- /dev/null +++ b/mypy/memprofile.py @@ -0,0 +1,121 @@ +"""Utility for dumping memory usage stats. + +This is tailored to mypy and knows (a little) about which list objects are +owned by particular AST nodes, etc. +""" + +from __future__ import annotations + +import gc +import sys +from collections import defaultdict +from typing import Dict, Iterable, cast + +from mypy.nodes import FakeInfo, Node +from mypy.types import Type +from mypy.util import get_class_descriptors + + +def collect_memory_stats() -> tuple[dict[str, int], dict[str, int]]: + """Return stats about memory use. + + Return a tuple with these items: + - Dict from object kind to number of instances of that kind + - Dict from object kind to total bytes used by all instances of that kind + """ + objs = gc.get_objects() + find_recursive_objects(objs) + + inferred = {} + for obj in objs: + if type(obj) is FakeInfo: + # Processing these would cause a crash. + continue + n = type(obj).__name__ + if hasattr(obj, "__dict__"): + # Keep track of which class a particular __dict__ is associated with. + inferred[id(obj.__dict__)] = f"{n} (__dict__)" + if isinstance(obj, (Node, Type)): # type: ignore[misc] + if hasattr(obj, "__dict__"): + for x in obj.__dict__.values(): + if isinstance(x, list): + # Keep track of which node a list is associated with. + inferred[id(x)] = f"{n} (list)" + if isinstance(x, tuple): + # Keep track of which node a list is associated with. + inferred[id(x)] = f"{n} (tuple)" + + for k in get_class_descriptors(type(obj)): + x = getattr(obj, k, None) + if isinstance(x, list): + inferred[id(x)] = f"{n} (list)" + if isinstance(x, tuple): + inferred[id(x)] = f"{n} (tuple)" + + freqs: dict[str, int] = {} + memuse: dict[str, int] = {} + for obj in objs: + if id(obj) in inferred: + name = inferred[id(obj)] + else: + name = type(obj).__name__ + freqs[name] = freqs.get(name, 0) + 1 + memuse[name] = memuse.get(name, 0) + sys.getsizeof(obj) + + return freqs, memuse + + +def print_memory_profile(run_gc: bool = True) -> None: + if not sys.platform.startswith("win"): + import resource + + system_memuse = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + else: + system_memuse = -1 # TODO: Support this on Windows + if run_gc: + gc.collect() + freqs, memuse = collect_memory_stats() + print("%7s %7s %7s %s" % ("Freq", "Size(k)", "AvgSize", "Type")) + print("-------------------------------------------") + totalmem = 0 + i = 0 + for n, mem in sorted(memuse.items(), key=lambda x: -x[1]): + f = freqs[n] + if i < 50: + print("%7d %7d %7.0f %s" % (f, mem // 1024, mem / f, n)) + i += 1 + totalmem += mem + print() + print("Mem usage RSS ", system_memuse // 1024) + print("Total reachable ", totalmem // 1024) + + +def find_recursive_objects(objs: list[object]) -> None: + """Find additional objects referenced by objs and append them to objs. + + We use this since gc.get_objects() does not return objects without pointers + in them such as strings. + """ + seen = {id(o) for o in objs} + + def visit(o: object) -> None: + if id(o) not in seen: + objs.append(o) + seen.add(id(o)) + + for obj in objs.copy(): + if type(obj) is FakeInfo: + # Processing these would cause a crash. + continue + if type(obj) in (dict, defaultdict): + for key, val in cast(Dict[object, object], obj).items(): + visit(key) + visit(val) + if type(obj) in (list, tuple, set): + for x in cast(Iterable[object], obj): + visit(x) + if hasattr(obj, "__slots__"): + for base in type.mro(type(obj)): + for slot in getattr(base, "__slots__", ()): + if hasattr(obj, slot): + visit(getattr(obj, slot)) diff --git a/mypy/message_registry.py b/mypy/message_registry.py new file mode 100644 index 000000000000..346a677a8e85 --- /dev/null +++ b/mypy/message_registry.py @@ -0,0 +1,367 @@ +"""Message constants for generating error messages during type checking. + +Literal messages should be defined as constants in this module so they won't get out of sync +if used in more than one place, and so that they can be easily introspected. These messages are +ultimately consumed by messages.MessageBuilder.fail(). For more non-trivial message generation, +add a method to MessageBuilder and call this instead. +""" + +from __future__ import annotations + +from typing import Final, NamedTuple + +from mypy import errorcodes as codes + + +class ErrorMessage(NamedTuple): + value: str + code: codes.ErrorCode | None = None + + def format(self, *args: object, **kwargs: object) -> ErrorMessage: + return ErrorMessage(self.value.format(*args, **kwargs), code=self.code) + + def with_additional_msg(self, info: str) -> ErrorMessage: + return ErrorMessage(self.value + info, code=self.code) + + +# Invalid types +INVALID_TYPE_RAW_ENUM_VALUE: Final = ErrorMessage( + "Invalid type: try using Literal[{}.{}] instead?", codes.VALID_TYPE +) + +# Type checker error message constants +NO_RETURN_VALUE_EXPECTED: Final = ErrorMessage("No return value expected", codes.RETURN_VALUE) +MISSING_RETURN_STATEMENT: Final = ErrorMessage("Missing return statement", codes.RETURN) +EMPTY_BODY_ABSTRACT: Final = ErrorMessage( + "If the method is meant to be abstract, use @abc.abstractmethod", codes.EMPTY_BODY +) +INVALID_IMPLICIT_RETURN: Final = ErrorMessage("Implicit return in function which does not return") +INCOMPATIBLE_RETURN_VALUE_TYPE: Final = ErrorMessage( + "Incompatible return value type", codes.RETURN_VALUE +) +RETURN_VALUE_EXPECTED: Final = ErrorMessage("Return value expected", codes.RETURN_VALUE) +NO_RETURN_EXPECTED: Final = ErrorMessage("Return statement in function which does not return") +INVALID_EXCEPTION: Final = ErrorMessage("Exception must be derived from BaseException") +INVALID_EXCEPTION_TYPE: Final = ErrorMessage( + "Exception type must be derived from BaseException (or be a tuple of exception classes)" +) +INVALID_EXCEPTION_GROUP: Final = ErrorMessage( + "Exception type in except* cannot derive from BaseExceptionGroup" +) +RETURN_IN_ASYNC_GENERATOR: Final = ErrorMessage( + '"return" with value in async generator is not allowed' +) +INVALID_RETURN_TYPE_FOR_GENERATOR: Final = ErrorMessage( + 'The return type of a generator function should be "Generator" or one of its supertypes' +) +INVALID_RETURN_TYPE_FOR_ASYNC_GENERATOR: Final = ErrorMessage( + 'The return type of an async generator function should be "AsyncGenerator" or one of its ' + "supertypes" +) +YIELD_VALUE_EXPECTED: Final = ErrorMessage("Yield value expected") +INCOMPATIBLE_TYPES: Final = ErrorMessage("Incompatible types") +INCOMPATIBLE_TYPES_IN_ASSIGNMENT: Final = ErrorMessage( + "Incompatible types in assignment", code=codes.ASSIGNMENT +) +COVARIANT_OVERRIDE_OF_MUTABLE_ATTRIBUTE: Final = ErrorMessage( + "Covariant override of a mutable attribute", code=codes.MUTABLE_OVERRIDE +) +INCOMPATIBLE_TYPES_IN_AWAIT: Final = ErrorMessage('Incompatible types in "await"') +INCOMPATIBLE_REDEFINITION: Final = ErrorMessage("Incompatible redefinition") +INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AENTER: Final = ( + 'Incompatible types in "async with" for "__aenter__"' +) +INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AEXIT: Final = ( + 'Incompatible types in "async with" for "__aexit__"' +) +INCOMPATIBLE_TYPES_IN_ASYNC_FOR: Final = 'Incompatible types in "async for"' +INVALID_TYPE_FOR_SLOTS: Final = 'Invalid type for "__slots__"' + +ASYNC_FOR_OUTSIDE_COROUTINE: Final = '"async for" outside async function' +ASYNC_WITH_OUTSIDE_COROUTINE: Final = '"async with" outside async function' + +INCOMPATIBLE_TYPES_IN_YIELD: Final = ErrorMessage('Incompatible types in "yield"') +INCOMPATIBLE_TYPES_IN_YIELD_FROM: Final = ErrorMessage('Incompatible types in "yield from"') +INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION: Final = "Incompatible types in string interpolation" +INCOMPATIBLE_TYPES_IN_CAPTURE: Final = ErrorMessage("Incompatible types in capture pattern") +MUST_HAVE_NONE_RETURN_TYPE: Final = ErrorMessage('The return type of "{}" must be None') +TUPLE_INDEX_OUT_OF_RANGE: Final = ErrorMessage("Tuple index out of range") +AMBIGUOUS_SLICE_OF_VARIADIC_TUPLE: Final = ErrorMessage("Ambiguous slice of a variadic tuple") +TOO_MANY_TARGETS_FOR_VARIADIC_UNPACK: Final = ErrorMessage( + "Too many assignment targets for variadic unpack" +) +INVALID_SLICE_INDEX: Final = ErrorMessage("Slice index must be an integer, SupportsIndex or None") +CANNOT_INFER_LAMBDA_TYPE: Final = ErrorMessage("Cannot infer type of lambda") +CANNOT_ACCESS_INIT: Final = ( + 'Accessing "__init__" on an instance is unsound, since instance.__init__ could be from' + " an incompatible subclass" +) +NON_INSTANCE_NEW_TYPE: Final = ErrorMessage('"__new__" must return a class instance (got {})') +INVALID_NEW_TYPE: Final = ErrorMessage('Incompatible return type for "__new__"') +BAD_CONSTRUCTOR_TYPE: Final = ErrorMessage("Unsupported decorated constructor type") +CANNOT_ASSIGN_TO_METHOD: Final = "Cannot assign to a method" +CANNOT_ASSIGN_TO_TYPE: Final = "Cannot assign to a type" +INCONSISTENT_ABSTRACT_OVERLOAD: Final = ErrorMessage( + "Overloaded method has both abstract and non-abstract variants" +) +MULTIPLE_OVERLOADS_REQUIRED: Final = ErrorMessage("Single overload definition, multiple required") +READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE: Final = ErrorMessage( + "Read-only property cannot override read-write property" +) +FORMAT_REQUIRES_MAPPING: Final = "Format requires a mapping" +RETURN_TYPE_CANNOT_BE_CONTRAVARIANT: Final = ErrorMessage( + "Cannot use a contravariant type variable as return type" +) +FUNCTION_PARAMETER_CANNOT_BE_COVARIANT: Final = ErrorMessage( + "Cannot use a covariant type variable as a parameter" +) +INCOMPATIBLE_IMPORT_OF: Final = ErrorMessage('Incompatible import of "{}"', code=codes.ASSIGNMENT) +FUNCTION_TYPE_EXPECTED: Final = ErrorMessage( + "Function is missing a type annotation", codes.NO_UNTYPED_DEF +) +ONLY_CLASS_APPLICATION: Final = ErrorMessage( + "Type application is only supported for generic classes" +) +RETURN_TYPE_EXPECTED: Final = ErrorMessage( + "Function is missing a return type annotation", codes.NO_UNTYPED_DEF +) +ARGUMENT_TYPE_EXPECTED: Final = ErrorMessage( + "Function is missing a type annotation for one or more arguments", codes.NO_UNTYPED_DEF +) +KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE: Final = ErrorMessage( + 'Keyword argument only valid with "str" key type in call to "dict"' +) +ALL_MUST_BE_SEQ_STR: Final = ErrorMessage("Type of __all__ must be {}, not {}") +INVALID_TYPEDDICT_ARGS: Final = ErrorMessage( + "Expected keyword arguments, {...}, or dict(...) in TypedDict constructor" +) +TYPEDDICT_KEY_MUST_BE_STRING_LITERAL: Final = ErrorMessage( + "Expected TypedDict key to be string literal" +) +TYPEDDICT_OVERRIDE_MERGE: Final = 'Overwriting TypedDict field "{}" while merging' +MALFORMED_ASSERT: Final = ErrorMessage("Assertion is always true, perhaps remove parentheses?") +DUPLICATE_TYPE_SIGNATURES: Final = ErrorMessage("Function has duplicate type signatures") +DESCRIPTOR_SET_NOT_CALLABLE: Final = ErrorMessage("{}.__set__ is not callable") +DESCRIPTOR_GET_NOT_CALLABLE: Final = "{}.__get__ is not callable" +MODULE_LEVEL_GETATTRIBUTE: Final = ErrorMessage( + "__getattribute__ is not valid at the module level" +) +CLASS_VAR_CONFLICTS_SLOTS: Final = '"{}" in __slots__ conflicts with class variable access' +NAME_NOT_IN_SLOTS: Final = ErrorMessage( + 'Trying to assign name "{}" that is not in "__slots__" of type "{}"' +) +TYPE_ALWAYS_TRUE: Final = ErrorMessage( + "{} which does not implement __bool__ or __len__ " + "so it could always be true in boolean context", + code=codes.TRUTHY_BOOL, +) +TYPE_ALWAYS_TRUE_UNIONTYPE: Final = ErrorMessage( + "{} of which no members implement __bool__ or __len__ " + "so it could always be true in boolean context", + code=codes.TRUTHY_BOOL, +) +FUNCTION_ALWAYS_TRUE: Final = ErrorMessage( + "Function {} could always be true in boolean context", code=codes.TRUTHY_FUNCTION +) +ITERABLE_ALWAYS_TRUE: Final = ErrorMessage( + "{} which can always be true in boolean context. Consider using {} instead.", + code=codes.TRUTHY_ITERABLE, +) +NOT_CALLABLE: Final = "{} not callable" +TYPE_MUST_BE_USED: Final = "Value of type {} must be used" + +# Generic +GENERIC_INSTANCE_VAR_CLASS_ACCESS: Final = ( + "Access to generic instance variables via class is ambiguous" +) +GENERIC_CLASS_VAR_ACCESS: Final = "Access to generic class variables is ambiguous" +BARE_GENERIC: Final = "Missing type parameters for generic type {}" +IMPLICIT_GENERIC_ANY_BUILTIN: Final = ( + 'Implicit generic "Any". Use "{}" and specify generic parameters' +) +INVALID_UNPACK: Final = "{} cannot be unpacked (must be tuple or TypeVarTuple)" +INVALID_UNPACK_POSITION: Final = "Unpack is only valid in a variadic position" +INVALID_PARAM_SPEC_LOCATION: Final = "Invalid location for ParamSpec {}" +INVALID_PARAM_SPEC_LOCATION_NOTE: Final = ( + 'You can use ParamSpec as the first argument to Callable, e.g., "Callable[{}, int]"' +) + +# TypeVar +INCOMPATIBLE_TYPEVAR_VALUE: Final = 'Value of type variable "{}" of {} cannot be {}' +CANNOT_USE_TYPEVAR_AS_EXPRESSION: Final = 'Type variable "{}.{}" cannot be used as an expression' +INVALID_TYPEVAR_AS_TYPEARG: Final = 'Type variable "{}" not valid as type argument value for "{}"' +INVALID_TYPEVAR_ARG_BOUND: Final = 'Type argument {} of "{}" must be a subtype of {}' +INVALID_TYPEVAR_ARG_VALUE: Final = 'Invalid type argument value for "{}"' +TYPEVAR_VARIANCE_DEF: Final = 'TypeVar "{}" may only be a literal bool' +TYPEVAR_ARG_MUST_BE_TYPE: Final = '{} "{}" must be a type' +TYPEVAR_UNEXPECTED_ARGUMENT: Final = 'Unexpected argument to "TypeVar()"' +UNBOUND_TYPEVAR: Final = ( + "A function returning TypeVar should receive at least one argument containing the same TypeVar" +) +TYPE_PARAMETERS_SHOULD_BE_DECLARED: Final = ( + "All type parameters should be declared ({} not declared)" +) + +# Super +TOO_MANY_ARGS_FOR_SUPER: Final = ErrorMessage('Too many arguments for "super"') +SUPER_WITH_SINGLE_ARG_NOT_SUPPORTED: Final = ErrorMessage( + '"super" with a single argument not supported' +) +UNSUPPORTED_ARG_1_FOR_SUPER: Final = ErrorMessage('Unsupported argument 1 for "super"') +UNSUPPORTED_ARG_2_FOR_SUPER: Final = ErrorMessage('Unsupported argument 2 for "super"') +SUPER_VARARGS_NOT_SUPPORTED: Final = ErrorMessage('Varargs not supported with "super"') +SUPER_POSITIONAL_ARGS_REQUIRED: Final = ErrorMessage('"super" only accepts positional arguments') +SUPER_ARG_2_NOT_INSTANCE_OF_ARG_1: Final = ErrorMessage( + 'Argument 2 for "super" not an instance of argument 1' +) +TARGET_CLASS_HAS_NO_BASE_CLASS: Final = ErrorMessage("Target class has no base class") +SUPER_OUTSIDE_OF_METHOD_NOT_SUPPORTED: Final = ErrorMessage( + '"super()" outside of a method is not supported' +) +SUPER_ENCLOSING_POSITIONAL_ARGS_REQUIRED: Final = ErrorMessage( + '"super()" requires one or two positional arguments in enclosing function' +) + +# Self-type +MISSING_OR_INVALID_SELF_TYPE: Final = ErrorMessage( + "Self argument missing for a non-static method (or an invalid type for self)" +) +ERASED_SELF_TYPE_NOT_SUPERTYPE: Final = ErrorMessage( + 'The erased type of self "{}" is not a supertype of its class "{}"' +) + +# Final +CANNOT_INHERIT_FROM_FINAL: Final = ErrorMessage('Cannot inherit from final class "{}"') +DEPENDENT_FINAL_IN_CLASS_BODY: Final = ErrorMessage( + "Final name declared in class body cannot depend on type variables" +) +CANNOT_ACCESS_FINAL_INSTANCE_ATTR: Final = ( + 'Cannot access final instance attribute "{}" on class object' +) +CANNOT_MAKE_DELETABLE_FINAL: Final = ErrorMessage("Deletable attribute cannot be final") + +# Enum +ENUM_MEMBERS_ATTR_WILL_BE_OVERRIDEN: Final = ErrorMessage( + 'Assigned "__members__" will be overridden by "Enum" internally' +) + +# ClassVar +CANNOT_OVERRIDE_INSTANCE_VAR: Final = ErrorMessage( + 'Cannot override instance variable (previously declared on base class "{}") with class ' + "variable" +) +CANNOT_OVERRIDE_CLASS_VAR: Final = ErrorMessage( + 'Cannot override class variable (previously declared on base class "{}") with instance ' + "variable" +) +CLASS_VAR_WITH_TYPEVARS: Final = "ClassVar cannot contain type variables" +CLASS_VAR_WITH_GENERIC_SELF: Final = "ClassVar cannot contain Self type in generic classes" +CLASS_VAR_OUTSIDE_OF_CLASS: Final = "ClassVar can only be used for assignments in class body" + +# Protocol +RUNTIME_PROTOCOL_EXPECTED: Final = ErrorMessage( + "Only @runtime_checkable protocols can be used with instance and class checks" +) +CANNOT_INSTANTIATE_PROTOCOL: Final = ErrorMessage('Cannot instantiate protocol class "{}"') +TOO_MANY_UNION_COMBINATIONS: Final = ErrorMessage( + "Not all union combinations were tried because there are too many unions" +) + +CONTIGUOUS_ITERABLE_EXPECTED: Final = ErrorMessage("Contiguous iterable with same type expected") +ITERABLE_TYPE_EXPECTED: Final = ErrorMessage("Invalid type '{}' for *expr (iterable expected)") +TYPE_GUARD_POS_ARG_REQUIRED: Final = ErrorMessage("Type {} requires positional argument") + +# Match Statement +MISSING_MATCH_ARGS: Final = 'Class "{}" doesn\'t define "__match_args__"' +OR_PATTERN_ALTERNATIVE_NAMES: Final = "Alternative patterns bind different names" +CLASS_PATTERN_GENERIC_TYPE_ALIAS: Final = ( + "Class pattern class must not be a type alias with type parameters" +) +CLASS_PATTERN_TYPE_REQUIRED: Final = 'Expected type in class pattern; found "{}"' +CLASS_PATTERN_TOO_MANY_POSITIONAL_ARGS: Final = "Too many positional patterns for class pattern" +CLASS_PATTERN_KEYWORD_MATCHES_POSITIONAL: Final = ( + 'Keyword "{}" already matches a positional pattern' +) +CLASS_PATTERN_DUPLICATE_KEYWORD_PATTERN: Final = 'Duplicate keyword pattern "{}"' +CLASS_PATTERN_UNKNOWN_KEYWORD: Final = 'Class "{}" has no attribute "{}"' +CLASS_PATTERN_CLASS_OR_STATIC_METHOD: Final = "Cannot have both classmethod and staticmethod" +MULTIPLE_ASSIGNMENTS_IN_PATTERN: Final = 'Multiple assignments to name "{}" in pattern' +CANNOT_MODIFY_MATCH_ARGS: Final = 'Cannot assign to "__match_args__"' + +DATACLASS_FIELD_ALIAS_MUST_BE_LITERAL: Final = ( + '"alias" argument to dataclass field must be a string literal' +) +DATACLASS_POST_INIT_MUST_BE_A_FUNCTION: Final = '"__post_init__" method must be an instance method' + +# fastparse +FAILED_TO_MERGE_OVERLOADS: Final = ErrorMessage( + "Condition can't be inferred, unable to merge overloads" +) +TYPE_IGNORE_WITH_ERRCODE_ON_MODULE: Final = ErrorMessage( + "type ignore with error code is not supported for modules; " + 'use `# mypy: disable-error-code="{}"`', + codes.SYNTAX, +) +INVALID_TYPE_IGNORE: Final = ErrorMessage('Invalid "type: ignore" comment', codes.SYNTAX) +TYPE_COMMENT_SYNTAX_ERROR_VALUE: Final = ErrorMessage( + 'Syntax error in type comment "{}"', codes.SYNTAX +) +ELLIPSIS_WITH_OTHER_TYPEARGS: Final = ErrorMessage( + "Ellipses cannot accompany other argument types in function type signature", codes.SYNTAX +) +TYPE_SIGNATURE_TOO_MANY_ARGS: Final = ErrorMessage( + "Type signature has too many arguments", codes.SYNTAX +) +TYPE_SIGNATURE_TOO_FEW_ARGS: Final = ErrorMessage( + "Type signature has too few arguments", codes.SYNTAX +) +ARG_CONSTRUCTOR_NAME_EXPECTED: Final = ErrorMessage("Expected arg constructor name", codes.SYNTAX) +ARG_CONSTRUCTOR_TOO_MANY_ARGS: Final = ErrorMessage( + "Too many arguments for argument constructor", codes.SYNTAX +) +MULTIPLE_VALUES_FOR_NAME_KWARG: Final = ErrorMessage( + '"{}" gets multiple values for keyword argument "name"', codes.SYNTAX +) +MULTIPLE_VALUES_FOR_TYPE_KWARG: Final = ErrorMessage( + '"{}" gets multiple values for keyword argument "type"', codes.SYNTAX +) +ARG_CONSTRUCTOR_UNEXPECTED_ARG: Final = ErrorMessage( + 'Unexpected argument "{}" for argument constructor', codes.SYNTAX +) +ARG_NAME_EXPECTED_STRING_LITERAL: Final = ErrorMessage( + "Expected string literal for argument name, got {}", codes.SYNTAX +) +NARROWED_TYPE_NOT_SUBTYPE: Final = ErrorMessage( + "Narrowed type {} is not a subtype of input type {}", codes.NARROWED_TYPE_NOT_SUBTYPE +) +TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES: Final = ErrorMessage( + "Type variable must have at least two constrained types", codes.MISC +) + +TYPE_VAR_YIELD_EXPRESSION_IN_BOUND: Final = ErrorMessage( + "Yield expression cannot be used as a type variable bound", codes.SYNTAX +) + +TYPE_VAR_NAMED_EXPRESSION_IN_BOUND: Final = ErrorMessage( + "Named expression cannot be used as a type variable bound", codes.SYNTAX +) + +TYPE_VAR_AWAIT_EXPRESSION_IN_BOUND: Final = ErrorMessage( + "Await expression cannot be used as a type variable bound", codes.SYNTAX +) + +TYPE_VAR_GENERIC_CONSTRAINT_TYPE: Final = ErrorMessage( + "TypeVar constraint type cannot be parametrized by type variables", codes.MISC +) + +TYPE_ALIAS_WITH_YIELD_EXPRESSION: Final = ErrorMessage( + "Yield expression cannot be used within a type alias", codes.SYNTAX +) + +TYPE_ALIAS_WITH_NAMED_EXPRESSION: Final = ErrorMessage( + "Named expression cannot be used within a type alias", codes.SYNTAX +) + +TYPE_ALIAS_WITH_AWAIT_EXPRESSION: Final = ErrorMessage( + "Await expression cannot be used within a type alias", codes.SYNTAX +) diff --git a/mypy/messages.py b/mypy/messages.py index 4919f6651108..6b0760cd79c6 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1,70 +1,155 @@ -"""Facilities and constants for generating error messages during type checking. +"""Facilities for generating error messages during type checking. -The type checker itself does not deal with message string literals to -improve code clarity and to simplify localization (in the future).""" +Don't add any non-trivial message construction logic to the type +checker, as it can compromise clarity and make messages less +consistent. Add such logic to this module instead. Literal messages, including those +with format args, should be defined as constants in mypy.message_registry. -import re +Historically we tried to avoid all message string literals in the type +checker but we are moving away from this convention. +""" -from typing import Undefined, cast, List, Any, Sequence, Iterable +from __future__ import annotations -from mypy.errors import Errors -from mypy.types import ( - Type, Callable, Instance, TypeVar, TupleType, UnionType, Void, NoneTyp, AnyType, - Overloaded, FunctionLike -) +import difflib +import itertools +import re +from contextlib import contextmanager +from textwrap import dedent +from typing import Any, Callable, Collection, Final, Iterable, Iterator, List, Sequence, cast + +import mypy.typeops +from mypy import errorcodes as codes, message_registry +from mypy.erasetype import erase_type +from mypy.errorcodes import ErrorCode +from mypy.errors import ErrorInfo, Errors, ErrorWatcher from mypy.nodes import ( - TypeInfo, Context, op_methods, FuncDef, reverse_type_aliases + ARG_NAMED, + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + CONTRAVARIANT, + COVARIANT, + SYMBOL_FUNCBASE_TYPES, + ArgKind, + CallExpr, + ClassDef, + Context, + Expression, + FuncDef, + IndexExpr, + MypyFile, + NameExpr, + ReturnStmt, + StrExpr, + SymbolNode, + SymbolTable, + TypeInfo, + Var, + reverse_builtin_aliases, ) +from mypy.operators import op_methods, op_methods_to_symbols +from mypy.options import Options +from mypy.subtypes import ( + IS_CLASS_OR_STATIC, + IS_CLASSVAR, + IS_SETTABLE, + IS_VAR, + find_member, + get_member_flags, + is_same_type, + is_subtype, +) +from mypy.typeops import separate_union_literals +from mypy.types import ( + AnyType, + CallableType, + DeletedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeStrVisitor, + TypeType, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + flatten_nested_unions, + get_proper_type, + get_proper_types, +) +from mypy.typetraverser import TypeTraverserVisitor +from mypy.util import plural_s, unmangle + +TYPES_FOR_UNIMPORTED_HINTS: Final = { + "typing.Any", + "typing.Callable", + "typing.Dict", + "typing.Iterable", + "typing.Iterator", + "typing.List", + "typing.Optional", + "typing.Set", + "typing.Tuple", + "typing.TypeVar", + "typing.Union", + "typing.cast", +} + + +ARG_CONSTRUCTOR_NAMES: Final = { + ARG_POS: "Arg", + ARG_OPT: "DefaultArg", + ARG_NAMED: "NamedArg", + ARG_NAMED_OPT: "DefaultNamedArg", + ARG_STAR: "VarArg", + ARG_STAR2: "KwArg", +} + + +# Map from the full name of a missing definition to the test fixture (under +# test-data/unit/fixtures/) that provides the definition. This is used for +# generating better error messages when running mypy tests only. +SUGGESTED_TEST_FIXTURES: Final = { + "builtins.set": "set.pyi", + "builtins.tuple": "tuple.pyi", + "builtins.bool": "bool.pyi", + "builtins.Exception": "exception.pyi", + "builtins.BaseException": "exception.pyi", + "builtins.isinstance": "isinstancelist.pyi", + "builtins.property": "property.pyi", + "builtins.classmethod": "classmethod.pyi", + "typing._SpecialForm": "typing-medium.pyi", +} + +UNSUPPORTED_NUMBERS_TYPES: Final = { + "numbers.Number", + "numbers.Complex", + "numbers.Real", + "numbers.Rational", + "numbers.Integral", +} + +MAX_TUPLE_ITEMS = 10 +MAX_UNION_ITEMS = 10 -# Constants that represent simple type checker error message, i.e. messages -# that do not have any parameters. - -NO_RETURN_VALUE_EXPECTED = 'No return value expected' -INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' -RETURN_VALUE_EXPECTED = 'Return value expected' -BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' -BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' -BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' -BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' -BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' -INVALID_EXCEPTION = 'Exception must be derived from BaseException' -INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' -INVALID_RETURN_TYPE_FOR_YIELD = \ - 'Iterator function return type expected for "yield"' -INVALID_RETURN_TYPE_FOR_YIELD_FROM = \ - 'Iterable function return type expected for "yield from"' -INCOMPATIBLE_TYPES = 'Incompatible types' -INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' -INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' -INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' -INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' -INIT_MUST_NOT_HAVE_RETURN_TYPE = 'Cannot define return type for "__init__"' -GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ - 'Type of getter incompatible with setter' -TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must an integer literal' -TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' -TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' -INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' -INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' -INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' -INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' -INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' -NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' -ITERABLE_EXPECTED = 'Iterable expected' -INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' -INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' -INVALID_SLICE_INDEX = 'Slice index must be an integer or None' -CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' -CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' -CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' -CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' -CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' -INCONSISTENT_ABSTRACT_OVERLOAD = \ - 'Overloaded method has both abstract and non-abstract variants' -INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' -FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' - class MessageBuilder: """Helper class for reporting type checker error messages with parameters. @@ -78,629 +163,2909 @@ class MessageBuilder: # Report errors using this instance. It knows about the current file and # import context. - errors = Undefined(Errors) + errors: Errors - # Number of times errors have been disabled. - disable_count = 0 + modules: dict[str, MypyFile] # Hack to deduplicate error messages from union types - disable_type_names = 0 + _disable_type_names: list[bool] - def __init__(self, errors: Errors) -> None: + def __init__(self, errors: Errors, modules: dict[str, MypyFile]) -> None: self.errors = errors - self.disable_count = 0 - self.disable_type_names = 0 + self.options = errors.options + self.modules = modules + self._disable_type_names = [] # # Helpers # - def copy(self) -> 'MessageBuilder': - new = MessageBuilder(self.errors.copy()) - new.disable_count = self.disable_count - return new - - def add_errors(self, messages: 'MessageBuilder') -> None: + def filter_errors( + self, + *, + filter_errors: bool | Callable[[str, ErrorInfo], bool] = True, + save_filtered_errors: bool = False, + ) -> ErrorWatcher: + return ErrorWatcher( + self.errors, filter_errors=filter_errors, save_filtered_errors=save_filtered_errors + ) + + def add_errors(self, errors: list[ErrorInfo]) -> None: """Add errors in messages to this builder.""" - self.errors.error_info.extend(messages.errors.error_info) + for info in errors: + self.errors.add_error_info(info) - def disable_errors(self) -> None: - self.disable_count += 1 + @contextmanager + def disable_type_names(self) -> Iterator[None]: + self._disable_type_names.append(True) + try: + yield + finally: + self._disable_type_names.pop() - def enable_errors(self) -> None: - self.disable_count -= 1 + def are_type_names_disabled(self) -> bool: + return len(self._disable_type_names) > 0 and self._disable_type_names[-1] - def is_errors(self) -> bool: - return self.errors.is_errors() - - def fail(self, msg: str, context: Context) -> None: - """Report an error message (unless disabled).""" - if self.disable_count <= 0: - self.errors.report(context.get_line(), msg.strip()) + def prefer_simple_messages(self) -> bool: + """Should we generate simple/fast error messages? - def format(self, typ: Type, verbose: bool = False) -> str: - """Convert a type to a relatively short string that is - suitable for error messages. Mostly behave like format_simple - below, but never return an empty string. + If errors aren't shown to the user, we don't want to waste cyles producing + complex error messages. + """ + return self.errors.prefer_simple_messages() + + def report( + self, + msg: str, + context: Context | None, + severity: str, + *, + code: ErrorCode | None = None, + file: str | None = None, + origin: Context | None = None, + offset: int = 0, + allow_dups: bool = False, + secondary_context: Context | None = None, + ) -> None: + """Report an error or note (unless disabled). + + Note that context controls where error is reported, while origin controls + where # type: ignore comments have effect. """ - s = self.format_simple(typ) - if s != '': - # If format_simple returns a non-trivial result, use that. - return s - elif isinstance(typ, FunctionLike): - func = cast(FunctionLike, typ) - if func.is_type_obj(): - # The type of a type object type can be derived from the - # return type (this always works). - itype = cast(Instance, func.items()[0].ret_type) - result = self.format(itype) - if verbose: - # In some contexts we want to be explicit about the distinction - # between type X and the type of type object X. - result += ' (type object)' - return result - elif isinstance(func, Callable): - arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] - return_type = strip_quotes(self.format(func.ret_type)) - return 'Function[[{}], {}]'.format(", ".join(arg_types), return_type) - else: - # Use a simple representation for function types; proper - # function types may result in long and difficult-to-read - # error messages. - return 'overloaded function' - else: - # Default case; we simply have to return something meaningful here. - return 'object' - def format_simple(self, typ: Type) -> str: - """Convert simple types to string that is suitable for error messages. + def span_from_context(ctx: Context) -> Iterable[int]: + """This determines where a type: ignore for a given context has effect. - Return "" for complex types. Try to keep the length of the result - relatively short to avoid overly long error messages. + Current logic is a bit tricky, to keep as much backwards compatibility as + possible. We may reconsider this to always be a single line (or otherwise + simplify it) when we drop Python 3.7. - Examples: - builtins.int -> 'int' - Any type -> 'Any' - void -> None - function type -> "" (empty string) - """ - if isinstance(typ, Instance): - itype = cast(Instance, typ) - # Get the short name of the type. - base_str = itype.type.name() - if itype.args == []: - # No type arguments. Place the type name in quotes to avoid - # potential for confusion: otherwise, the type name could be - # interpreted as a normal word. - return '"{}"'.format(base_str) - elif itype.type.fullname() in reverse_type_aliases: - alias = reverse_type_aliases[itype.type.fullname()] - alias = alias.split('.')[-1] - items = [strip_quotes(self.format(arg)) for arg in itype.args] - return '{}[{}]'.format(alias, ', '.join(items)) - else: - # There are type arguments. Convert the arguments to strings - # (using format() instead of format_simple() to avoid empty - # strings). If the result is too long, replace arguments - # with [...]. - a = [] # type: List[str] - for arg in itype.args: - a.append(strip_quotes(self.format(arg))) - s = ', '.join(a) - if len((base_str + s)) < 25: - return '{}[{}]'.format(base_str, s) - else: - return '{}[...]'.format(base_str) - elif isinstance(typ, TypeVar): - # This is similar to non-generic instance types. - return '"{}"'.format((cast(TypeVar, typ)).name) - elif isinstance(typ, TupleType): - fallback = self.format_simple(typ.fallback) - if fallback != '"tuple"': - # Prefer the name of the fallback class (if not tuple), as it's more informative. - return fallback - items = [] - for t in (cast(TupleType, typ)).items: - items.append(strip_quotes(self.format(t))) - s = '"Tuple[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s + TODO: address this in follow up PR + """ + if isinstance(ctx, (ClassDef, FuncDef)): + return range(ctx.deco_line or ctx.line, ctx.line + 1) + elif not isinstance(ctx, Expression): + return [ctx.line] else: - return 'tuple(length {})'.format(len(items)) - elif isinstance(typ, UnionType): - items = [] - for t in (cast(UnionType, typ)).items: - items.append(strip_quotes(self.format(t))) - s = '"Union[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'union type ({} items)'.format(len(items)) - elif isinstance(typ, Void): - return 'None' - elif isinstance(typ, NoneTyp): - return 'None' - elif isinstance(typ, AnyType): - return '"Any"' - elif typ is None: - raise RuntimeError('Type is None') - else: - # No simple representation for this type that would convey very - # useful information. No need to mention the type explicitly in a - # message. - return '' + return range(ctx.line, (ctx.end_line or ctx.line) + 1) + + origin_span: Iterable[int] | None + if origin is not None: + origin_span = span_from_context(origin) + elif context is not None: + origin_span = span_from_context(context) + else: + origin_span = None + + if secondary_context is not None: + assert origin_span is not None + origin_span = itertools.chain(origin_span, span_from_context(secondary_context)) + + self.errors.report( + context.line if context else -1, + context.column if context else -1, + msg, + severity=severity, + file=file, + offset=offset, + origin_span=origin_span, + end_line=context.end_line if context else -1, + end_column=context.end_column if context else -1, + code=code, + allow_dups=allow_dups, + ) + + def fail( + self, + msg: str, + context: Context | None, + *, + code: ErrorCode | None = None, + file: str | None = None, + allow_dups: bool = False, + secondary_context: Context | None = None, + ) -> None: + """Report an error message (unless disabled).""" + self.report( + msg, + context, + "error", + code=code, + file=file, + allow_dups=allow_dups, + secondary_context=secondary_context, + ) + + def note( + self, + msg: str, + context: Context, + file: str | None = None, + origin: Context | None = None, + offset: int = 0, + allow_dups: bool = False, + *, + code: ErrorCode | None = None, + secondary_context: Context | None = None, + ) -> None: + """Report a note (unless disabled).""" + self.report( + msg, + context, + "note", + file=file, + origin=origin, + offset=offset, + allow_dups=allow_dups, + code=code, + secondary_context=secondary_context, + ) + + def note_multiline( + self, + messages: str, + context: Context, + file: str | None = None, + offset: int = 0, + allow_dups: bool = False, + code: ErrorCode | None = None, + *, + secondary_context: Context | None = None, + ) -> None: + """Report as many notes as lines in the message (unless disabled).""" + for msg in messages.splitlines(): + self.report( + msg, + context, + "note", + file=file, + offset=offset, + allow_dups=allow_dups, + code=code, + secondary_context=secondary_context, + ) # # Specific operations # - # The following operations are for genering specific error messages. They + # The following operations are for generating specific error messages. They # get some information as arguments, and they build an error message based # on them. - def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: + def has_no_attr( + self, + original_type: Type, + typ: Type, + member: str, + context: Context, + module_symbol_table: SymbolTable | None = None, + ) -> ErrorCode | None: """Report a missing or non-accessible member. - The type argument is the base type. If member corresponds to - an operator, use the corresponding operator name in the - messages. Return type Any. + original_type is the top-level type on which the error occurred. + typ is the actual type that is missing the member. These can be + different, e.g., in a union, original_type will be the union and typ + will be the specific item in the union that does not have the member + attribute. + + 'module_symbol_table' is passed to this function if the type for which we + are trying to get a member was originally a module. The SymbolTable allows + us to look up and suggests attributes of the module since they are not + directly available on original_type + + If member corresponds to an operator, use the corresponding operator + name in the messages. Return the error code that was produced, if any. """ - if (isinstance(typ, Instance) and - (cast(Instance, typ)).type.has_readable_member(member)): - self.fail('Member "{}" is not assignable'.format(member), context) - elif isinstance(typ, Void): - self.check_void(typ, context) - elif member == '__contains__': - self.fail('Unsupported right operand type for in ({})'.format( - self.format(typ)), context) + original_type = get_proper_type(original_type) + typ = get_proper_type(typ) + + if isinstance(original_type, Instance) and original_type.type.has_readable_member(member): + self.fail(f'Member "{member}" is not assignable', context) + return None + elif member == "__contains__": + self.fail( + f"Unsupported right operand type for in ({format_type(original_type, self.options)})", + context, + code=codes.OPERATOR, + ) + return codes.OPERATOR elif member in op_methods.values(): # Access to a binary operator member (e.g. _add). This case does # not handle indexing operations. for op, method in op_methods.items(): if method == member: - self.unsupported_left_operand(op, typ, context) - break - elif member == '__neg__': - self.fail('Unsupported operand type for unary - ({})'.format( - self.format(typ)), context) - elif member == '__pos__': - self.fail('Unsupported operand type for unary + ({})'.format( - self.format(typ)), context) - elif member == '__invert__': - self.fail('Unsupported operand type for ~ ({})'.format( - self.format(typ)), context) - elif member == '__getitem__': + self.unsupported_left_operand(op, original_type, context) + return codes.OPERATOR + elif member == "__neg__": + self.fail( + f"Unsupported operand type for unary - ({format_type(original_type, self.options)})", + context, + code=codes.OPERATOR, + ) + return codes.OPERATOR + elif member == "__pos__": + self.fail( + f"Unsupported operand type for unary + ({format_type(original_type, self.options)})", + context, + code=codes.OPERATOR, + ) + return codes.OPERATOR + elif member == "__invert__": + self.fail( + f"Unsupported operand type for ~ ({format_type(original_type, self.options)})", + context, + code=codes.OPERATOR, + ) + return codes.OPERATOR + elif member == "__getitem__": # Indexed get. - self.fail('Value of type {} is not indexable'.format( - self.format(typ)), context) - elif member == '__setitem__': + # TODO: Fix this consistently in format_type + if isinstance(original_type, FunctionLike) and original_type.is_type_obj(): + self.fail( + "The type {} is not generic and not indexable".format( + format_type(original_type, self.options) + ), + context, + ) + return None + else: + self.fail( + f"Value of type {format_type(original_type, self.options)} is not indexable", + context, + code=codes.INDEX, + ) + return codes.INDEX + elif member == "__setitem__": # Indexed set. - self.fail('Unsupported target for indexed assignment', context) + self.fail( + "Unsupported target for indexed assignment ({})".format( + format_type(original_type, self.options) + ), + context, + code=codes.INDEX, + ) + return codes.INDEX + elif member == "__call__": + if isinstance(original_type, Instance) and ( + original_type.type.fullname == "builtins.function" + ): + # "'function' not callable" is a confusing error message. + # Explain that the problem is that the type of the function is not known. + self.fail("Cannot call function of unknown type", context, code=codes.OPERATOR) + return codes.OPERATOR + else: + self.fail( + message_registry.NOT_CALLABLE.format(format_type(original_type, self.options)), + context, + code=codes.OPERATOR, + ) + return codes.OPERATOR else: # The non-special case: a missing ordinary attribute. - if not self.disable_type_names: - self.fail('{} has no attribute "{}"'.format(self.format(typ), - member), context) + extra = "" + if member == "__iter__": + extra = " (not iterable)" + elif member == "__aiter__": + extra = " (not async iterable)" + if not self.are_type_names_disabled(): + failed = False + if isinstance(original_type, Instance) and original_type.type.names: + if ( + module_symbol_table is not None + and member in module_symbol_table + and not module_symbol_table[member].module_public + ): + self.fail( + f"{format_type(original_type, self.options, module_names=True)} does not " + f'explicitly export attribute "{member}"', + context, + code=codes.ATTR_DEFINED, + ) + failed = True + else: + alternatives = set(original_type.type.names.keys()) + if module_symbol_table is not None: + alternatives |= { + k for k, v in module_symbol_table.items() if v.module_public + } + # Rare but possible, see e.g. testNewAnalyzerCyclicDefinitionCrossModule + alternatives.discard(member) + + matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] + matches.extend(best_matches(member, alternatives, n=3)) + if member == "__aiter__" and matches == ["__iter__"]: + matches = [] # Avoid misleading suggestion + if matches: + self.fail( + '{} has no attribute "{}"; maybe {}?{}'.format( + format_type(original_type, self.options), + member, + pretty_seq(matches, "or"), + extra, + ), + context, + code=codes.ATTR_DEFINED, + ) + failed = True + if not failed: + self.fail( + '{} has no attribute "{}"{}'.format( + format_type(original_type, self.options), member, extra + ), + context, + code=codes.ATTR_DEFINED, + ) + return codes.ATTR_DEFINED + elif isinstance(original_type, UnionType): + # The checker passes "object" in lieu of "None" for attribute + # checks, so we manually convert it back. + typ_format, orig_type_format = format_type_distinctly( + typ, original_type, options=self.options + ) + if typ_format == '"object"' and any( + type(item) == NoneType for item in original_type.items + ): + typ_format = '"None"' + self.fail( + 'Item {} of {} has no attribute "{}"{}'.format( + typ_format, orig_type_format, member, extra + ), + context, + code=codes.UNION_ATTR, + ) + return codes.UNION_ATTR + elif isinstance(original_type, TypeVarType): + bound = get_proper_type(original_type.upper_bound) + if isinstance(bound, UnionType): + typ_fmt, bound_fmt = format_type_distinctly(typ, bound, options=self.options) + original_type_fmt = format_type(original_type, self.options) + self.fail( + "Item {} of the upper bound {} of type variable {} has no " + 'attribute "{}"{}'.format( + typ_fmt, bound_fmt, original_type_fmt, member, extra + ), + context, + code=codes.UNION_ATTR, + ) + return codes.UNION_ATTR else: - self.fail('Some element of union has no attribute "{}"'.format( - member), context) - return AnyType() - - def unsupported_operand_types(self, op: str, left_type: Any, - right_type: Any, context: Context) -> None: + self.fail( + '{} has no attribute "{}"{}'.format( + format_type(original_type, self.options), member, extra + ), + context, + code=codes.ATTR_DEFINED, + ) + return codes.ATTR_DEFINED + return None + + def unsupported_operand_types( + self, + op: str, + left_type: Any, + right_type: Any, + context: Context, + *, + code: ErrorCode = codes.OPERATOR, + ) -> None: """Report unsupported operand types for a binary operation. Types can be Type objects or strings. """ - if isinstance(left_type, Void) or isinstance(right_type, Void): - self.check_void(left_type, context) - self.check_void(right_type, context) - return - left_str = '' + left_str = "" if isinstance(left_type, str): left_str = left_type else: - left_str = self.format(left_type) + left_str = format_type(left_type, self.options) - right_str = '' + right_str = "" if isinstance(right_type, str): right_str = right_type else: - right_str = self.format(right_type) + right_str = format_type(right_type, self.options) - if self.disable_type_names: - msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) + if self.are_type_names_disabled(): + msg = f"Unsupported operand types for {op} (likely involving Union)" else: - msg = 'Unsupported operand types for {} ({} and {})'.format( - op, left_str, right_str) - self.fail(msg, context) - - def unsupported_left_operand(self, op: str, typ: Type, - context: Context) -> None: - if not self.check_void(typ, context): - if self.disable_type_names: - msg = 'Unsupported left operand type for {} (some union)'.format(op) - else: - msg = 'Unsupported left operand type for {} ({})'.format( - op, self.format(typ)) - self.fail(msg, context) + msg = f"Unsupported operand types for {op} ({left_str} and {right_str})" + self.fail(msg, context, code=code) - def type_expected_as_right_operand_of_is(self, context: Context) -> None: - self.fail('Type expected as right operand of "is"', context) + def unsupported_left_operand(self, op: str, typ: Type, context: Context) -> None: + if self.are_type_names_disabled(): + msg = f"Unsupported left operand type for {op} (some union)" + else: + msg = f"Unsupported left operand type for {op} ({format_type(typ, self.options)})" + self.fail(msg, context, code=codes.OPERATOR) def not_callable(self, typ: Type, context: Context) -> Type: - self.fail('{} not callable'.format(self.format(typ)), context) - return AnyType() - - def incompatible_argument(self, n: int, callee: Callable, arg_type: Type, - context: Context) -> None: + self.fail(message_registry.NOT_CALLABLE.format(format_type(typ, self.options)), context) + return AnyType(TypeOfAny.from_error) + + def untyped_function_call(self, callee: CallableType, context: Context) -> Type: + name = callable_name(callee) or "(unknown)" + self.fail( + f"Call to untyped function {name} in typed context", + context, + code=codes.NO_UNTYPED_CALL, + ) + return AnyType(TypeOfAny.from_error) + + def incompatible_argument( + self, + n: int, + m: int, + callee: CallableType, + arg_type: Type, + arg_kind: ArgKind, + object_type: Type | None, + context: Context, + outer_context: Context, + ) -> ErrorCode | None: """Report an error about an incompatible argument type. The argument type is arg_type, argument number is n and the callee type is 'callee'. If the callee represents a method that corresponds to an operator, use the corresponding operator name in the messages. + + Return the error code that used for the argument (multiple error + codes are possible). """ - target = '' - if callee.name: - name = callee.name - base = extract_type(name) + arg_type = get_proper_type(arg_type) + + target = "" + callee_name = callable_name(callee) + if callee_name is not None: + name = callee_name + if callee.bound_args and callee.bound_args[0] is not None: + base = format_type(callee.bound_args[0], self.options) + else: + base = extract_type(name) - for op, method in op_methods.items(): - for variant in method, '__r' + method[2:]: - if name.startswith('"{}" of'.format(variant)): - if op == 'in' or variant != method: + for method, op in op_methods_to_symbols.items(): + for variant in method, "__r" + method[2:]: + # FIX: do not rely on textual formatting + if name.startswith(f'"{variant}" of'): + if op == "in" or variant != method: # Reversed order of base/argument. - self.unsupported_operand_types(op, arg_type, base, - context) + self.unsupported_operand_types( + op, arg_type, base, context, code=codes.OPERATOR + ) else: - self.unsupported_operand_types(op, base, arg_type, - context) - return + self.unsupported_operand_types( + op, base, arg_type, context, code=codes.OPERATOR + ) + return codes.OPERATOR if name.startswith('"__getitem__" of'): - self.invalid_index_type(arg_type, base, context) - return + self.invalid_index_type( + arg_type, callee.arg_types[n - 1], base, context, code=codes.INDEX + ) + return codes.INDEX if name.startswith('"__setitem__" of'): if n == 1: - self.invalid_index_type(arg_type, base, context) + self.invalid_index_type( + arg_type, callee.arg_types[n - 1], base, context, code=codes.INDEX + ) + return codes.INDEX else: - self.fail(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, context) - return + arg_type_str, callee_type_str = format_type_distinctly( + arg_type, callee.arg_types[n - 1], options=self.options + ) + info = ( + f" (expression has type {arg_type_str}, target has type {callee_type_str})" + ) + error_msg = ( + message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT.with_additional_msg(info) + ) + self.fail(error_msg.value, context, code=error_msg.code) + return error_msg.code + + target = f"to {name} " + + msg = "" + code = codes.MISC + notes: list[str] = [] + if callee_name == "": + name = callee_name[1:-1] + n -= 1 + actual_type_str, expected_type_str = format_type_distinctly( + arg_type, callee.arg_types[0], options=self.options + ) + msg = "{} item {} has incompatible type {}; expected {}".format( + name.title(), n, actual_type_str, expected_type_str + ) + code = codes.LIST_ITEM + elif callee_name == "" and isinstance( + get_proper_type(callee.arg_types[n - 1]), TupleType + ): + name = callee_name[1:-1] + n -= 1 + key_type, value_type = cast(TupleType, arg_type).items + expected_key_type, expected_value_type = cast(TupleType, callee.arg_types[n]).items + + # don't increase verbosity unless there is need to do so + if is_subtype(key_type, expected_key_type): + key_type_str = format_type(key_type, self.options) + expected_key_type_str = format_type(expected_key_type, self.options) + else: + key_type_str, expected_key_type_str = format_type_distinctly( + key_type, expected_key_type, options=self.options + ) + if is_subtype(value_type, expected_value_type): + value_type_str = format_type(value_type, self.options) + expected_value_type_str = format_type(expected_value_type, self.options) + else: + value_type_str, expected_value_type_str = format_type_distinctly( + value_type, expected_value_type, options=self.options + ) + + msg = "{} entry {} has incompatible type {}: {}; expected {}: {}".format( + name.title(), + n, + key_type_str, + value_type_str, + expected_key_type_str, + expected_value_type_str, + ) + code = codes.DICT_ITEM + elif callee_name == "": + value_type_str, expected_value_type_str = format_type_distinctly( + arg_type, callee.arg_types[n - 1], options=self.options + ) + msg = "Unpacked dict entry {} has incompatible type {}; expected {}".format( + n - 1, value_type_str, expected_value_type_str + ) + code = codes.DICT_ITEM + elif callee_name == "": + actual_type_str, expected_type_str = map( + strip_quotes, + format_type_distinctly(arg_type, callee.arg_types[0], options=self.options), + ) + msg = "List comprehension has incompatible type List[{}]; expected List[{}]".format( + actual_type_str, expected_type_str + ) + elif callee_name == "": + actual_type_str, expected_type_str = map( + strip_quotes, + format_type_distinctly(arg_type, callee.arg_types[0], options=self.options), + ) + msg = "Set comprehension has incompatible type Set[{}]; expected Set[{}]".format( + actual_type_str, expected_type_str + ) + elif callee_name == "": + actual_type_str, expected_type_str = format_type_distinctly( + arg_type, callee.arg_types[n - 1], options=self.options + ) + msg = ( + "{} expression in dictionary comprehension has incompatible type {}; " + "expected type {}" + ).format("Key" if n == 1 else "Value", actual_type_str, expected_type_str) + elif callee_name == "": + actual_type_str, expected_type_str = format_type_distinctly( + arg_type, callee.arg_types[0], options=self.options + ) + msg = "Generator has incompatible item type {}; expected {}".format( + actual_type_str, expected_type_str + ) + else: + if self.prefer_simple_messages(): + msg = "Argument has incompatible type" + else: + try: + expected_type = callee.arg_types[m - 1] + except IndexError: # Varargs callees + expected_type = callee.arg_types[-1] + arg_type_str, expected_type_str = format_type_distinctly( + arg_type, expected_type, bare=True, options=self.options + ) + if arg_kind == ARG_STAR: + arg_type_str = "*" + arg_type_str + elif arg_kind == ARG_STAR2: + arg_type_str = "**" + arg_type_str + + # For function calls with keyword arguments, display the argument name rather + # than the number. + arg_label = str(n) + if isinstance(outer_context, CallExpr) and len(outer_context.arg_names) >= n: + arg_name = outer_context.arg_names[n - 1] + if arg_name is not None: + arg_label = f'"{arg_name}"' + if ( + arg_kind == ARG_STAR2 + and isinstance(arg_type, TypedDictType) + and m <= len(callee.arg_names) + and callee.arg_names[m - 1] is not None + and callee.arg_kinds[m - 1] != ARG_STAR2 + ): + arg_name = callee.arg_names[m - 1] + assert arg_name is not None + arg_type_str, expected_type_str = format_type_distinctly( + arg_type.items[arg_name], expected_type, bare=True, options=self.options + ) + arg_label = f'"{arg_name}"' + if isinstance(outer_context, IndexExpr) and isinstance( + outer_context.index, StrExpr + ): + msg = 'Value of "{}" has incompatible type {}; expected {}'.format( + outer_context.index.value, + quote_type_string(arg_type_str), + quote_type_string(expected_type_str), + ) + else: + msg = "Argument {} {}has incompatible type {}; expected {}".format( + arg_label, + target, + quote_type_string(arg_type_str), + quote_type_string(expected_type_str), + ) + expected_type = get_proper_type(expected_type) + if isinstance(expected_type, UnionType): + expected_types = list(expected_type.items) + else: + expected_types = [expected_type] + for type in get_proper_types(expected_types): + if isinstance(arg_type, Instance) and isinstance(type, Instance): + notes = append_invariance_notes(notes, arg_type, type) + notes = append_numbers_notes(notes, arg_type, type) + object_type = get_proper_type(object_type) + if isinstance(object_type, TypedDictType): + code = codes.TYPEDDICT_ITEM + else: + code = codes.ARG_TYPE + self.fail(msg, context, code=code) + if notes: + for note_msg in notes: + self.note(note_msg, context, code=code) + return code + + def incompatible_argument_note( + self, + original_caller_type: ProperType, + callee_type: ProperType, + context: Context, + code: ErrorCode | None, + ) -> None: + if self.prefer_simple_messages(): + return + if isinstance( + original_caller_type, (Instance, TupleType, TypedDictType, TypeType, CallableType) + ): + if isinstance(callee_type, Instance) and callee_type.type.is_protocol: + self.report_protocol_problems( + original_caller_type, callee_type, context, code=code + ) + if isinstance(callee_type, UnionType): + for item in callee_type.items: + item = get_proper_type(item) + if isinstance(item, Instance) and item.type.is_protocol: + self.report_protocol_problems( + original_caller_type, item, context, code=code + ) + if isinstance(callee_type, CallableType) and isinstance(original_caller_type, Instance): + call = find_member( + "__call__", original_caller_type, original_caller_type, is_operator=True + ) + if call: + self.note_call(original_caller_type, call, context, code=code) + if isinstance(callee_type, Instance) and callee_type.type.is_protocol: + call = find_member("__call__", callee_type, callee_type, is_operator=True) + if call: + self.note_call(callee_type, call, context, code=code) + self.maybe_note_concatenate_pos_args(original_caller_type, callee_type, context, code) + + def maybe_note_concatenate_pos_args( + self, + original_caller_type: ProperType, + callee_type: ProperType, + context: Context, + code: ErrorCode | None = None, + ) -> None: + # pos-only vs positional can be confusing, with Concatenate + if ( + isinstance(callee_type, CallableType) + and isinstance(original_caller_type, CallableType) + and (original_caller_type.from_concatenate or callee_type.from_concatenate) + ): + names: list[str] = [] + for c, o in zip( + callee_type.formal_arguments(), original_caller_type.formal_arguments() + ): + if None in (c.pos, o.pos): + # non-positional + continue + if c.name != o.name and c.name is None and o.name is not None: + names.append(o.name) + + if names: + missing_arguments = '"' + '", "'.join(names) + '"' + self.note( + f'This is likely because "{original_caller_type.name}" has named arguments: ' + f"{missing_arguments}. Consider marking them positional-only", + context, + code=code, + ) + + def invalid_index_type( + self, + index_type: Type, + expected_type: Type, + base_str: str, + context: Context, + *, + code: ErrorCode, + ) -> None: + index_str, expected_str = format_type_distinctly( + index_type, expected_type, options=self.options + ) + self.fail( + "Invalid index type {} for {}; expected type {}".format( + index_str, base_str, expected_str + ), + context, + code=code, + ) + + def readonly_keys_mutated(self, keys: set[str], context: Context) -> None: + if len(keys) == 1: + suffix = "is" + else: + suffix = "are" + self.fail( + "ReadOnly {} TypedDict {} mutated".format(format_key_list(sorted(keys)), suffix), + code=codes.TYPEDDICT_READONLY_MUTATED, + context=context, + ) + + def too_few_arguments( + self, callee: CallableType, context: Context, argument_names: Sequence[str | None] | None + ) -> None: + if self.prefer_simple_messages(): + msg = "Too few arguments" + elif argument_names is not None: + num_positional_args = sum(k is None for k in argument_names) + arguments_left = callee.arg_names[num_positional_args : callee.min_args] + diff = [k for k in arguments_left if k not in argument_names] + if len(diff) == 1: + msg = "Missing positional argument" + else: + msg = "Missing positional arguments" + callee_name = callable_name(callee) + if callee_name is not None and diff and all(d is not None for d in diff): + args = '", "'.join(cast(List[str], diff)) + msg += f' "{args}" in call to {callee_name}' + else: + msg = "Too few arguments" + for_function(callee) - target = 'to {} '.format(name) - - msg = '' - if callee.name == '': - name = callee.name[1:-1] - msg = '{} item {} has incompatible type {}'.format( - name[0].upper() + name[1:], n, self.format_simple(arg_type)) - elif callee.name == '': - msg = 'List comprehension has incompatible type List[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = 'Set comprehension has incompatible type Set[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = '{} expression in dictionary comprehension has incompatible type {}; expected type {}'.format( - 'Key' if n == 1 else 'Value', - self.format(arg_type), - self.format(callee.arg_types[n-1])) - elif callee.name == '': - msg = 'Generator has incompatible item type {}'.format( - self.format_simple(arg_type)) else: - try: - expected_type = callee.arg_types[n - 1] - except IndexError: # Varargs callees - expected_type = callee.arg_types[-1] - msg = 'Argument {} {}has incompatible type {}; expected {}'.format( - n, target, self.format(arg_type), self.format(expected_type)) - self.fail(msg, context) + msg = "Too few arguments" + for_function(callee) + self.fail(msg, context, code=codes.CALL_ARG) - def invalid_index_type(self, index_type: Type, base_str: str, - context: Context) -> None: - self.fail('Invalid index type {} for {}'.format( - self.format(index_type), base_str), context) + def missing_named_argument(self, callee: CallableType, context: Context, name: str) -> None: + msg = f'Missing named argument "{name}"' + for_function(callee) + self.fail(msg, context, code=codes.CALL_ARG) - def invalid_argument_count(self, callee: Callable, num_args: int, - context: Context) -> None: - if num_args < len(callee.arg_types): - self.too_few_arguments(callee, context) + def too_many_arguments(self, callee: CallableType, context: Context) -> None: + if self.prefer_simple_messages(): + msg = "Too many arguments" + else: + msg = "Too many arguments" + for_function(callee) + self.fail(msg, context, code=codes.CALL_ARG) + self.maybe_note_about_special_args(callee, context) + + def too_many_arguments_from_typed_dict( + self, callee: CallableType, arg_type: TypedDictType, context: Context + ) -> None: + # Try to determine the name of the extra argument. + for key in arg_type.items: + if key not in callee.arg_names: + msg = f'Extra argument "{key}" from **args' + for_function(callee) + break else: self.too_many_arguments(callee, context) - - def too_few_arguments(self, callee: Callable, context: Context) -> None: - msg = 'Too few arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_arguments(self, callee: Callable, context: Context) -> None: - msg = 'Too many arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_positional_arguments(self, callee: Callable, - context: Context) -> None: - msg = 'Too many positional arguments' - if callee.name: - msg += ' for {}'.format(callee.name) + return self.fail(msg, context) - def unexpected_keyword_argument(self, callee: Callable, name: str, - context: Context) -> None: - msg = 'Unexpected keyword argument "{}"'.format(name) - if callee.name: - msg += ' for {}'.format(callee.name) + def too_many_positional_arguments(self, callee: CallableType, context: Context) -> None: + if self.prefer_simple_messages(): + msg = "Too many positional arguments" + else: + msg = "Too many positional arguments" + for_function(callee) self.fail(msg, context) + self.maybe_note_about_special_args(callee, context) - def duplicate_argument_value(self, callee: Callable, index: int, - context: Context) -> None: - self.fail('{} gets multiple values for keyword argument "{}"'. - format(capitalize(callable_name(callee)), - callee.arg_names[index]), context) + def maybe_note_about_special_args(self, callee: CallableType, context: Context) -> None: + if self.prefer_simple_messages(): + return + # https://github.com/python/mypy/issues/11309 + first_arg = callee.def_extras.get("first_arg") + if first_arg and first_arg not in {"self", "cls", "mcs"}: + self.note( + "Looks like the first special argument in a method " + 'is not named "self", "cls", or "mcs", ' + "maybe it is missing?", + context, + ) + + def unexpected_keyword_argument_for_function( + self, for_func: str, name: str, context: Context, *, matches: list[str] | None = None + ) -> None: + msg = f'Unexpected keyword argument "{name}"' + for_func + if matches: + msg += f"; did you mean {pretty_seq(matches, 'or')}?" + self.fail(msg, context, code=codes.CALL_ARG) + + def unexpected_keyword_argument( + self, callee: CallableType, name: str, arg_type: Type, context: Context + ) -> None: + # Suggest intended keyword, look for type match else fallback on any match. + matching_type_args = [] + not_matching_type_args = [] + for i, kwarg_type in enumerate(callee.arg_types): + callee_arg_name = callee.arg_names[i] + if callee_arg_name is not None and callee.arg_kinds[i] != ARG_STAR: + if is_subtype(arg_type, kwarg_type): + matching_type_args.append(callee_arg_name) + else: + not_matching_type_args.append(callee_arg_name) + matches = best_matches(name, matching_type_args, n=3) + if not matches: + matches = best_matches(name, not_matching_type_args, n=3) + self.unexpected_keyword_argument_for_function( + for_function(callee), name, context, matches=matches + ) + module = find_defining_module(self.modules, callee) + if module: + assert callee.definition is not None + fname = callable_name(callee) + if not fname: # an alias to function with a different name + fname = "Called function" + self.note( + f"{fname} defined here", + callee.definition, + file=module.path, + origin=context, + code=codes.CALL_ARG, + ) + + def duplicate_argument_value(self, callee: CallableType, index: int, context: Context) -> None: + self.fail( + '{} gets multiple values for keyword argument "{}"'.format( + callable_name(callee) or "Function", callee.arg_names[index] + ), + context, + ) + + def does_not_return_value(self, callee_type: Type | None, context: Context) -> None: + """Report an error about use of an unusable type.""" + callee_type = get_proper_type(callee_type) + callee_name = callable_name(callee_type) if isinstance(callee_type, FunctionLike) else None + name = callee_name or "Function" + message = f"{name} does not return a value (it only ever returns None)" + self.fail(message, context, code=codes.FUNC_RETURNS_VALUE) + + def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an rvalue.""" + if typ.source is None: + s = "" + else: + s = f' "{typ.source}"' + self.fail(f"Trying to read deleted variable{s}", context) - def does_not_return_value(self, void_type: Type, context: Context) -> None: - """Report an error about a void type in a non-void context. + def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an lvalue. - The first argument must be a void type. If the void type has a - source in it, report it in the error message. This allows - giving messages such as 'Foo does not return a value'. + Currently, this only occurs when trying to assign to an + exception variable outside the local except: blocks. """ - if (cast(Void, void_type)).source is None: - self.fail('Function does not return a value', context) + if typ.source is None: + s = "" else: - self.fail('{} does not return a value'.format( - capitalize((cast(Void, void_type)).source)), context) - - def no_variant_matches_arguments(self, overload: Overloaded, - context: Context) -> None: - if overload.name(): - self.fail('No overload variant of {} matches argument types' - .format(overload.name()), context) + s = f' "{typ.source}"' + self.fail(f"Assignment to variable{s} outside except: block", context) + + def no_variant_matches_arguments( + self, + overload: Overloaded, + arg_types: list[Type], + context: Context, + *, + code: ErrorCode | None = None, + ) -> None: + code = code or codes.CALL_OVERLOAD + name = callable_name(overload) + if name: + name_str = f" of {name}" else: - self.fail('No overload variant matches argument types', context) - - def function_variants_overlap(self, n1: int, n2: int, - context: Context) -> None: - self.fail('Function signature variants {} and {} overlap'.format( - n1 + 1, n2 + 1), context) - - def invalid_cast(self, target_type: Type, source_type: Type, - context: Context) -> None: - if not self.check_void(source_type, context): - self.fail('Cannot cast from {} to {}'.format( - self.format(source_type), self.format(target_type)), context) - - def wrong_number_values_to_unpack(self, provided: int, expected: int, context: Context) -> None: + name_str = "" + arg_types_str = ", ".join(format_type(arg, self.options) for arg in arg_types) + num_args = len(arg_types) + if num_args == 0: + self.fail( + f"All overload variants{name_str} require at least one argument", + context, + code=code, + ) + elif num_args == 1: + self.fail( + f"No overload variant{name_str} matches argument type {arg_types_str}", + context, + code=code, + ) + else: + self.fail( + f"No overload variant{name_str} matches argument types {arg_types_str}", + context, + code=code, + ) + + self.note(f"Possible overload variant{plural_s(len(overload.items))}:", context, code=code) + for item in overload.items: + self.note(pretty_callable(item, self.options), context, offset=4, code=code) + + def wrong_number_values_to_unpack( + self, provided: int, expected: int, context: Context + ) -> None: if provided < expected: if provided == 1: - self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), context) + self.fail(f"Need more than 1 value to unpack ({expected} expected)", context) else: - self.fail('Need more than {} values to unpack ({} expected)'.format(provided, expected), context) + self.fail( + f"Need more than {provided} values to unpack ({expected} expected)", context + ) elif provided > expected: - self.fail('Too many values to unpack ({} expected, {} provided)'.format(expected, provided), context) + self.fail( + f"Too many values to unpack ({expected} expected, {provided} provided)", context + ) + + def unpacking_strings_disallowed(self, context: Context) -> None: + self.fail("Unpacking a string is disallowed", context) def type_not_iterable(self, type: Type, context: Context) -> None: - self.fail('\'{}\' object is not iterable'.format(type), context) - - def incompatible_operator_assignment(self, op: str, - context: Context) -> None: - self.fail('Result type of {} incompatible in assignment'.format(op), - context) - - def incompatible_value_count_in_assignment(self, lvalue_count: int, - rvalue_count: int, - context: Context) -> None: - if rvalue_count < lvalue_count: - self.fail('Need {} values to assign'.format(lvalue_count), context) - elif rvalue_count > lvalue_count: - self.fail('Too many values to assign', context) - - def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, - context: Context) -> None: - self.fail('Type of "{}" incompatible with supertype "{}"'.format( - name, supertype.name), context) + self.fail(f"{format_type(type, self.options)} object is not iterable", context) + + def possible_missing_await(self, context: Context, code: ErrorCode | None) -> None: + self.note('Maybe you forgot to use "await"?', context, code=code) + + def incompatible_operator_assignment(self, op: str, context: Context) -> None: + self.fail(f"Result type of {op} incompatible in assignment", context) + + def overload_signature_incompatible_with_supertype( + self, name: str, name_in_super: str, supertype: str, context: Context + ) -> None: + target = self.override_target(name, name_in_super, supertype) + self.fail( + f'Signature of "{name}" incompatible with {target}', context, code=codes.OVERRIDE + ) + + note_template = 'Overload variants must be defined in the same order as they are in "{}"' + self.note(note_template.format(supertype), context, code=codes.OVERRIDE) def signature_incompatible_with_supertype( - self, name: str, name_in_super: str, supertype: str, - context: Context) -> None: + self, + name: str, + name_in_super: str, + supertype: str, + context: Context, + *, + original: ProperType, + override: ProperType, + ) -> None: + code = codes.OVERRIDE target = self.override_target(name, name_in_super, supertype) - self.fail('Signature of "{}" incompatible with {}'.format( - name, target), context) + self.fail(f'Signature of "{name}" incompatible with {target}', context, code=code) + + original_str, override_str = format_type_distinctly( + original, override, options=self.options, bare=True + ) + + INCLUDE_DECORATOR = True # Include @classmethod and @staticmethod decorators, if any + ALLOW_DUPS = True # Allow duplicate notes, needed when signatures are duplicates + ALIGN_OFFSET = 1 # One space, to account for the difference between error and note + OFFSET = 4 # Four spaces, so that notes will look like this: + # error: Signature of "f" incompatible with supertype "A" + # note: Superclass: + # note: def f(self) -> str + # note: Subclass: + # note: def f(self, x: str) -> None + self.note( + "Superclass:", context, offset=ALIGN_OFFSET + OFFSET, allow_dups=ALLOW_DUPS, code=code + ) + if isinstance(original, (CallableType, Overloaded)): + self.pretty_callable_or_overload( + original, + context, + offset=ALIGN_OFFSET + 2 * OFFSET, + add_class_or_static_decorator=INCLUDE_DECORATOR, + allow_dups=ALLOW_DUPS, + code=code, + ) + else: + self.note( + original_str, + context, + offset=ALIGN_OFFSET + 2 * OFFSET, + allow_dups=ALLOW_DUPS, + code=code, + ) + + self.note( + "Subclass:", context, offset=ALIGN_OFFSET + OFFSET, allow_dups=ALLOW_DUPS, code=code + ) + if isinstance(override, (CallableType, Overloaded)): + self.pretty_callable_or_overload( + override, + context, + offset=ALIGN_OFFSET + 2 * OFFSET, + add_class_or_static_decorator=INCLUDE_DECORATOR, + allow_dups=ALLOW_DUPS, + code=code, + ) + else: + self.note( + override_str, + context, + offset=ALIGN_OFFSET + 2 * OFFSET, + allow_dups=ALLOW_DUPS, + code=code, + ) + + def pretty_callable_or_overload( + self, + tp: CallableType | Overloaded, + context: Context, + *, + offset: int = 0, + add_class_or_static_decorator: bool = False, + allow_dups: bool = False, + code: ErrorCode | None = None, + ) -> None: + if isinstance(tp, CallableType): + if add_class_or_static_decorator: + decorator = pretty_class_or_static_decorator(tp) + if decorator is not None: + self.note(decorator, context, offset=offset, allow_dups=allow_dups, code=code) + self.note( + pretty_callable(tp, self.options), + context, + offset=offset, + allow_dups=allow_dups, + code=code, + ) + elif isinstance(tp, Overloaded): + self.pretty_overload( + tp, + context, + offset, + add_class_or_static_decorator=add_class_or_static_decorator, + allow_dups=allow_dups, + code=code, + ) def argument_incompatible_with_supertype( - self, arg_num: int, name: str, name_in_supertype: str, - supertype: str, context: Context) -> None: + self, + arg_num: int, + name: str, + type_name: str | None, + name_in_supertype: str, + arg_type_in_supertype: Type, + supertype: str, + context: Context, + secondary_context: Context, + ) -> None: target = self.override_target(name, name_in_supertype, supertype) - self.fail('Argument {} of "{}" incompatible with {}' - .format(arg_num, name, target), context) + arg_type_in_supertype_f = format_type_bare(arg_type_in_supertype, self.options) + self.fail( + 'Argument {} of "{}" is incompatible with {}; ' + 'supertype defines the argument type as "{}"'.format( + arg_num, name, target, arg_type_in_supertype_f + ), + context, + code=codes.OVERRIDE, + secondary_context=secondary_context, + ) + if name != "__post_init__": + # `__post_init__` is special, it can be incompatible by design. + # So, this note is misleading. + self.note( + "This violates the Liskov substitution principle", + context, + code=codes.OVERRIDE, + secondary_context=secondary_context, + ) + self.note( + "See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides", + context, + code=codes.OVERRIDE, + secondary_context=secondary_context, + ) + + if name == "__eq__" and type_name: + multiline_msg = self.comparison_method_example_msg(class_name=type_name) + self.note_multiline( + multiline_msg, context, code=codes.OVERRIDE, secondary_context=secondary_context + ) + + def comparison_method_example_msg(self, class_name: str) -> str: + return dedent( + """\ + It is recommended for "__eq__" to work with arbitrary objects, for example: + def __eq__(self, other: object) -> bool: + if not isinstance(other, {class_name}): + return NotImplemented + return + """.format( + class_name=class_name + ) + ) def return_type_incompatible_with_supertype( - self, name: str, name_in_supertype: str, supertype: str, - context: Context) -> None: + self, + name: str, + name_in_supertype: str, + supertype: str, + original: Type, + override: Type, + context: Context, + ) -> None: target = self.override_target(name, name_in_supertype, supertype) - self.fail('Return type of "{}" incompatible with {}' - .format(name, target), context) - - def override_target(self, name: str, name_in_super: str, - supertype: str) -> str: - target = 'supertype "{}"'.format(supertype) + override_str, original_str = format_type_distinctly( + override, original, options=self.options + ) + self.fail( + 'Return type {} of "{}" incompatible with return type {} in {}'.format( + override_str, name, original_str, target + ), + context, + code=codes.OVERRIDE, + ) + + original = get_proper_type(original) + override = get_proper_type(override) + if ( + isinstance(original, Instance) + and isinstance(override, Instance) + and override.type.fullname == "typing.AsyncIterator" + and original.type.fullname == "typing.Coroutine" + and len(original.args) == 3 + and original.args[2] == override + ): + self.note(f'Consider declaring "{name}" in {target} without "async"', context) + self.note( + "See https://mypy.readthedocs.io/en/stable/more_types.html#asynchronous-iterators", + context, + ) + + def override_target(self, name: str, name_in_super: str, supertype: str) -> str: + target = f'supertype "{supertype}"' if name_in_super != name: - target = '"{}" of {}'.format(name_in_super, target) + target = f'"{name_in_super}" of {target}' return target - def boolean_return_value_expected(self, method: str, - context: Context) -> None: - self.fail('Boolean return value expected for method "{}"'.format( - method), context) - - def incompatible_type_application(self, expected_arg_count: int, - actual_arg_count: int, - context: Context) -> None: - if expected_arg_count == 0: - self.fail('Type application targets a non-generic function', - context) - elif actual_arg_count > expected_arg_count: - self.fail('Type application has too many types ({} expected)' - .format(expected_arg_count), context) - else: - self.fail('Type application has too few types ({} expected)' - .format(expected_arg_count), context) - - def incompatible_array_item_type(self, typ: Type, index: int, - context: Context) -> None: - self.fail('Array item {} has incompatible type {}'.format( - index, self.format(typ)), context) - - def could_not_infer_type_arguments(self, callee_type: Callable, n: int, - context: Context) -> None: - if callee_type.name and n > 0: - self.fail('Cannot infer type argument {} of {}'.format( - n, callee_type.name), context) - else: - self.fail('Cannot infer function type argument', context) + def incompatible_type_application( + self, min_arg_count: int, max_arg_count: int, actual_arg_count: int, context: Context + ) -> None: + if max_arg_count == 0: + self.fail("Type application targets a non-generic function or class", context) + return - def invalid_var_arg(self, typ: Type, context: Context) -> None: - self.fail('List or tuple expected as variable arguments', context) + if min_arg_count == max_arg_count: + s = f"{max_arg_count} expected" + else: + s = f"expected between {min_arg_count} and {max_arg_count}" - def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: - if isinstance(typ, Instance) and ( - (cast(Instance, typ)).type.fullname() == 'builtins.dict'): - self.fail('Keywords must be strings', context) + if actual_arg_count > max_arg_count: + self.fail(f"Type application has too many types ({s})", context) + else: + self.fail(f"Type application has too few types ({s})", context) + + def could_not_infer_type_arguments( + self, callee_type: CallableType, n: int, context: Context + ) -> None: + callee_name = callable_name(callee_type) + if callee_name is not None and n > 0: + self.fail(f"Cannot infer type argument {n} of {callee_name}", context) + if callee_name == "": + # Invariance in key type causes more of these errors than we would want. + self.note( + "Try assigning the literal to a variable annotated as dict[, ]", + context, + ) else: - self.fail('Argument after ** must be a dictionary', - context) + self.fail("Cannot infer function type argument", context) - def incomplete_type_var_match(self, member: str, context: Context) -> None: - self.fail('"{}" has incomplete match to supertype type variable' - .format(member), context) + def invalid_var_arg(self, typ: Type, context: Context) -> None: + self.fail("Expected iterable as variadic argument", context) - def not_implemented(self, msg: str, context: Context) -> Type: - self.fail('Feature not implemented yet ({})'.format(msg), context) - return AnyType() + def invalid_keyword_var_arg(self, typ: Type, is_mapping: bool, context: Context) -> None: + typ = get_proper_type(typ) + if isinstance(typ, Instance) and is_mapping: + self.fail("Keywords must be strings", context) + else: + self.fail( + f"Argument after ** must be a mapping, not {format_type(typ, self.options)}", + context, + code=codes.ARG_TYPE, + ) def undefined_in_superclass(self, member: str, context: Context) -> None: - self.fail('"{}" undefined in superclass'.format(member), context) + self.fail(f'"{member}" undefined in superclass', context) - def check_void(self, typ: Type, context: Context) -> bool: - """If type is void, report an error such as '.. does not - return a value' and return True. Otherwise, return False. - """ - if isinstance(typ, Void): - self.does_not_return_value(typ, context) - return True + def variable_may_be_undefined(self, name: str, context: Context) -> None: + self.fail(f'Name "{name}" may be undefined', context, code=codes.POSSIBLY_UNDEFINED) + + def var_used_before_def(self, name: str, context: Context) -> None: + self.fail(f'Name "{name}" is used before definition', context, code=codes.USED_BEFORE_DEF) + + def first_argument_for_super_must_be_type(self, actual: Type, context: Context) -> None: + actual = get_proper_type(actual) + if isinstance(actual, Instance): + # Don't include type of instance, because it can look confusingly like a type + # object. + type_str = "a non-type instance" else: - return False + type_str = format_type(actual, self.options) + self.fail( + f'Argument 1 for "super" must be a type object; got {type_str}', + context, + code=codes.ARG_TYPE, + ) + + def unsafe_super(self, method: str, cls: str, ctx: Context) -> None: + self.fail( + f'Call to abstract method "{method}" of "{cls}" with trivial body via super() is unsafe', + ctx, + code=codes.SAFE_SUPER, + ) def too_few_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not enough arguments for format string', context) + self.fail("Not enough arguments for format string", context, code=codes.STRING_FORMATTING) def too_many_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not all arguments converted during string formatting', context) - - def incomplete_conversion_specifier_format(self, context: Context) -> None: - self.fail('Incomplete format', context) + self.fail( + "Not all arguments converted during string formatting", + context, + code=codes.STRING_FORMATTING, + ) def unsupported_placeholder(self, placeholder: str, context: Context) -> None: - self.fail('Unsupported format character \'%s\'' % placeholder, context) + self.fail( + f'Unsupported format character "{placeholder}"', context, code=codes.STRING_FORMATTING + ) def string_interpolation_with_star_and_key(self, context: Context) -> None: - self.fail('String interpolation contains both stars and mapping keys', context) - - def requires_int_or_char(self, context: Context) -> None: - self.fail('%c requires int or char', context) + self.fail( + "String interpolation contains both stars and mapping keys", + context, + code=codes.STRING_FORMATTING, + ) + + def requires_int_or_single_byte(self, context: Context, format_call: bool = False) -> None: + self.fail( + '"{}c" requires an integer in range(256) or a single byte'.format( + ":" if format_call else "%" + ), + context, + code=codes.STRING_FORMATTING, + ) + + def requires_int_or_char(self, context: Context, format_call: bool = False) -> None: + self.fail( + '"{}c" requires int or char'.format(":" if format_call else "%"), + context, + code=codes.STRING_FORMATTING, + ) def key_not_in_mapping(self, key: str, context: Context) -> None: - self.fail('Key \'%s\' not found in mapping' % key, context) + self.fail(f'Key "{key}" not found in mapping', context, code=codes.STRING_FORMATTING) def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: - self.fail('String interpolation mixes specifier with and without mapping keys', context) + self.fail( + "String interpolation mixes specifier with and without mapping keys", + context, + code=codes.STRING_FORMATTING, + ) def cannot_determine_type(self, name: str, context: Context) -> None: - self.fail("Cannot determine type of '%s'" % name, context) - - def invalid_method_type(self, sig: Callable, context: Context) -> None: - self.fail('Invalid method type', context) - - def incompatible_conditional_function_def(self, defn: FuncDef) -> None: - self.fail('All conditional function variants must have identical ' - 'signatures', defn) - - def cannot_instantiate_abstract_class(self, class_name: str, - abstract_attributes: List[str], - context: Context) -> None: - attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) - self.fail("Cannot instantiate abstract class '%s' with abstract " - "method%s %s" % (class_name, plural_s(abstract_attributes), - attrs), - context) - - def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, - base2: TypeInfo, - context: Context) -> None: - self.fail('Definition of "{}" in base class "{}" is incompatible ' - 'with definition in base class "{}"'.format( - name, base1.name(), base2.name()), context) + self.fail(f'Cannot determine type of "{name}"', context, code=codes.HAS_TYPE) + + def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: + self.fail(f'Cannot determine type of "{name}" in base class "{base}"', context) + + def no_formal_self(self, name: str, item: CallableType, context: Context) -> None: + type = format_type(item, self.options) + self.fail( + f'Attribute function "{name}" with type {type} does not accept self argument', context + ) + + def incompatible_self_argument( + self, name: str, arg: Type, sig: CallableType, is_classmethod: bool, context: Context + ) -> None: + kind = "class attribute function" if is_classmethod else "attribute function" + arg_type = format_type(arg, self.options) + sig_type = format_type(sig, self.options) + self.fail( + f'Invalid self argument {arg_type} to {kind} "{name}" with type {sig_type}', context + ) + + def incompatible_conditional_function_def( + self, defn: FuncDef, old_type: FunctionLike, new_type: FunctionLike + ) -> None: + self.fail("All conditional function variants must have identical signatures", defn) + if isinstance(old_type, (CallableType, Overloaded)) and isinstance( + new_type, (CallableType, Overloaded) + ): + self.note("Original:", defn) + self.pretty_callable_or_overload(old_type, defn, offset=4) + self.note("Redefinition:", defn) + self.pretty_callable_or_overload(new_type, defn, offset=4) + + def cannot_instantiate_abstract_class( + self, class_name: str, abstract_attributes: dict[str, bool], context: Context + ) -> None: + attrs = format_string_list([f'"{a}"' for a in abstract_attributes]) + self.fail( + f'Cannot instantiate abstract class "{class_name}" with abstract ' + f"attribute{plural_s(abstract_attributes)} {attrs}", + context, + code=codes.ABSTRACT, + ) + attrs_with_none = [ + f'"{a}"' + for a, implicit_and_can_return_none in abstract_attributes.items() + if implicit_and_can_return_none + ] + if not attrs_with_none: + return + if len(attrs_with_none) == 1: + note = ( + f"{attrs_with_none[0]} is implicitly abstract because it has an empty function " + "body. If it is not meant to be abstract, explicitly `return` or `return None`." + ) + else: + note = ( + "The following methods were marked implicitly abstract because they have empty " + f"function bodies: {format_string_list(attrs_with_none)}. " + "If they are not meant to be abstract, explicitly `return` or `return None`." + ) + self.note(note, context, code=codes.ABSTRACT) + + def base_class_definitions_incompatible( + self, name: str, base1: TypeInfo, base2: TypeInfo, context: Context + ) -> None: + self.fail( + 'Definition of "{}" in base class "{}" is incompatible ' + 'with definition in base class "{}"'.format(name, base1.name, base2.name), + context, + ) def cant_assign_to_method(self, context: Context) -> None: - self.fail(CANNOT_ASSIGN_TO_METHOD, context) - - def read_only_property(self, name: str, type: TypeInfo, - context: Context) -> None: - self.fail('Property "{}" defined in "{}" is read-only'.format( - name, type.name()), context) - - def incompatible_typevar_value(self, callee: Callable, index: int, - type: Type, context: Context) -> None: - self.fail('Type argument {} of {} has incompatible value {}'.format( - index, callable_name(callee), self.format(type)), context) - - def disjointness_violation(self, cls: TypeInfo, disjoint: TypeInfo, - context: Context) -> None: - self.fail('disjointclass constraint of class {} disallows {} as a ' - 'base class'.format(cls.name(), disjoint.name()), context) - - def overloaded_signatures_overlap(self, index1: int, index2: int, - context: Context) -> None: - self.fail('Overloaded function signatures {} and {} overlap with ' - 'incompatible return types'.format(index1, index2), context) - - def invalid_reverse_operator_signature(self, reverse: str, other: str, - context: Context) -> None: - self.fail('"Any" return type expected since argument to {} does not ' - 'support {}'.format(reverse, other), context) - - def reverse_operator_method_with_any_arg_must_return_any( - self, method: str, context: Context) -> None: - self.fail('"Any" return type expected since argument to {} has type ' - '"Any"'.format(method), context) + self.fail(message_registry.CANNOT_ASSIGN_TO_METHOD, context, code=codes.METHOD_ASSIGN) + + def cant_assign_to_classvar(self, name: str, context: Context) -> None: + self.fail(f'Cannot assign to class variable "{name}" via instance', context) + + def no_overridable_method(self, name: str, context: Context) -> None: + self.fail( + f'Method "{name}" is marked as an override, ' + "but no base method was found with this name", + context, + ) + + def explicit_override_decorator_missing( + self, name: str, base_name: str, context: Context + ) -> None: + self.fail( + f'Method "{name}" is not using @override ' + f'but is overriding a method in class "{base_name}"', + context, + code=codes.EXPLICIT_OVERRIDE_REQUIRED, + ) + + def final_cant_override_writable(self, name: str, ctx: Context) -> None: + self.fail(f'Cannot override writable attribute "{name}" with a final one', ctx) + + def cant_override_final(self, name: str, base_name: str, ctx: Context) -> None: + self.fail( + ( + f'Cannot override final attribute "{name}" ' + f'(previously declared in base class "{base_name}")' + ), + ctx, + ) + + def cant_assign_to_final(self, name: str, attr_assign: bool, ctx: Context) -> None: + """Warn about a prohibited assignment to a final attribute. + + Pass `attr_assign=True` if the assignment assigns to an attribute. + """ + kind = "attribute" if attr_assign else "name" + self.fail(f'Cannot assign to final {kind} "{unmangle(name)}"', ctx) + + def protocol_members_cant_be_final(self, ctx: Context) -> None: + self.fail("Protocol member cannot be final", ctx) + + def final_without_value(self, ctx: Context) -> None: + self.fail("Final name must be initialized with a value", ctx) + + def read_only_property(self, name: str, type: TypeInfo, context: Context) -> None: + self.fail(f'Property "{name}" defined in "{type.name}" is read-only', context) + + def incompatible_typevar_value( + self, callee: CallableType, typ: Type, typevar_name: str, context: Context + ) -> None: + self.fail( + message_registry.INCOMPATIBLE_TYPEVAR_VALUE.format( + typevar_name, callable_name(callee) or "function", format_type(typ, self.options) + ), + context, + code=codes.TYPE_VAR, + ) + + def dangerous_comparison(self, left: Type, right: Type, kind: str, ctx: Context) -> None: + left_str = "element" if kind == "container" else "left operand" + right_str = "container item" if kind == "container" else "right operand" + message = "Non-overlapping {} check ({} type: {}, {} type: {})" + left_typ, right_typ = format_type_distinctly(left, right, options=self.options) + self.fail( + message.format(kind, left_str, left_typ, right_str, right_typ), + ctx, + code=codes.COMPARISON_OVERLAP, + ) + + def overload_inconsistently_applies_decorator(self, decorator: str, context: Context) -> None: + self.fail( + f'Overload does not consistently use the "@{decorator}" ' + + "decorator on all function signatures.", + context, + ) + + def overloaded_signatures_overlap( + self, index1: int, index2: int, flip_note: bool, context: Context + ) -> None: + self.fail( + "Overloaded function signatures {} and {} overlap with " + "incompatible return types".format(index1, index2), + context, + code=codes.OVERLOAD_OVERLAP, + ) + if flip_note: + self.note( + "Flipping the order of overloads will fix this error", + context, + code=codes.OVERLOAD_OVERLAP, + ) + + def overloaded_signature_will_never_match( + self, index1: int, index2: int, context: Context + ) -> None: + self.fail( + "Overloaded function signature {index2} will never be matched: " + "signature {index1}'s parameter type(s) are the same or broader".format( + index1=index1, index2=index2 + ), + context, + code=codes.OVERLOAD_CANNOT_MATCH, + ) + + def overloaded_signatures_typevar_specific(self, index: int, context: Context) -> None: + self.fail( + f"Overloaded function implementation cannot satisfy signature {index} " + + "due to inconsistencies in how they use type variables", + context, + ) + + def overloaded_signatures_arg_specific(self, index: int, context: Context) -> None: + self.fail( + ( + f"Overloaded function implementation does not accept all possible arguments " + f"of signature {index}" + ), + context, + ) + + def overloaded_signatures_ret_specific(self, index: int, context: Context) -> None: + self.fail( + f"Overloaded function implementation cannot produce return type of signature {index}", + context, + ) + + def warn_both_operands_are_from_unions(self, context: Context) -> None: + self.note("Both left and right operands are unions", context, code=codes.OPERATOR) + + def warn_operand_was_from_union(self, side: str, original: Type, context: Context) -> None: + self.note( + f"{side} operand is of type {format_type(original, self.options)}", + context, + code=codes.OPERATOR, + ) def operator_method_signatures_overlap( - self, reverse_class: str, reverse_method: str, forward_class: str, - forward_method: str, context: Context) -> None: - self.fail('Signatures of "{}" of "{}" and "{}" of "{}" are unsafely ' - 'overlapping'.format(reverse_method, reverse_class, - forward_method, forward_class), context) - - def signatures_incompatible(self, method: str, other_method: str, - context: Context) -> None: - self.fail('Signatures of "{}" and "{}" are incompatible'.format( - method, other_method), context) + self, + reverse_class: TypeInfo, + reverse_method: str, + forward_class: Type, + forward_method: str, + context: Context, + ) -> None: + self.fail( + 'Signatures of "{}" of "{}" and "{}" of {} are unsafely overlapping'.format( + reverse_method, + reverse_class.name, + forward_method, + format_type(forward_class, self.options), + ), + context, + ) + + def forward_operator_not_callable(self, forward_method: str, context: Context) -> None: + self.fail(f'Forward operator "{forward_method}" is not callable', context) + + def signatures_incompatible(self, method: str, other_method: str, context: Context) -> None: + self.fail(f'Signatures of "{method}" and "{other_method}" are incompatible', context) def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: - text = self.format(expr) if self.format(expr) != 'object' else expr - self.fail('"yield from" can\'t be applied to {}'.format(text), context) - return AnyType() + text = ( + format_type(expr, self.options) + if format_type(expr, self.options) != "object" + else expr + ) + self.fail(f'"yield from" can\'t be applied to {text}', context) + return AnyType(TypeOfAny.from_error) + + def invalid_signature(self, func_type: Type, context: Context) -> None: + self.fail(f"Invalid signature {format_type(func_type, self.options)}", context) + + def invalid_signature_for_special_method( + self, func_type: Type, context: Context, method_name: str + ) -> None: + self.fail( + f'Invalid signature {format_type(func_type, self.options)} for "{method_name}"', + context, + ) + + def reveal_type(self, typ: Type, context: Context) -> None: + visitor = TypeStrVisitor(options=self.options) + self.note(f'Revealed type is "{typ.accept(visitor)}"', context) + + def reveal_locals(self, type_map: dict[str, Type | None], context: Context) -> None: + # To ensure that the output is predictable on Python < 3.6, + # use an ordered dictionary sorted by variable name + sorted_locals = dict(sorted(type_map.items(), key=lambda t: t[0])) + if sorted_locals: + self.note("Revealed local types are:", context) + for k, v in sorted_locals.items(): + visitor = TypeStrVisitor(options=self.options) + self.note(f" {k}: {v.accept(visitor) if v is not None else None}", context) + else: + self.note("There are no locals to reveal", context) + + def unsupported_type_type(self, item: Type, context: Context) -> None: + self.fail( + f'Cannot instantiate type "Type[{format_type_bare(item, self.options)}]"', context + ) + + def redundant_cast(self, typ: Type, context: Context) -> None: + self.fail( + f"Redundant cast to {format_type(typ, self.options)}", + context, + code=codes.REDUNDANT_CAST, + ) + + def assert_type_fail(self, source_type: Type, target_type: Type, context: Context) -> None: + (source, target) = format_type_distinctly(source_type, target_type, options=self.options) + self.fail(f"Expression is of type {source}, not {target}", context, code=codes.ASSERT_TYPE) + + def unimported_type_becomes_any(self, prefix: str, typ: Type, ctx: Context) -> None: + self.fail( + f"{prefix} becomes {format_type(typ, self.options)} due to an unfollowed import", + ctx, + code=codes.NO_ANY_UNIMPORTED, + ) + + def need_annotation_for_var( + self, node: SymbolNode, context: Context, python_version: tuple[int, int] | None = None + ) -> None: + hint = "" + pep604_supported = not python_version or python_version >= (3, 10) + # type to recommend the user adds + recommended_type = None + # Only gives hint if it's a variable declaration and the partial type is a builtin type + if python_version and isinstance(node, Var) and isinstance(node.type, PartialType): + type_dec = "" + if not node.type.type: + # partial None + if pep604_supported: + recommended_type = f"{type_dec} | None" + else: + recommended_type = f"Optional[{type_dec}]" + elif node.type.type.fullname in reverse_builtin_aliases: + # partial types other than partial None + alias = reverse_builtin_aliases[node.type.type.fullname] + alias = alias.split(".")[-1] + if alias == "Dict": + type_dec = f"{type_dec}, {type_dec}" + if self.options.use_lowercase_names(): + alias = alias.lower() + recommended_type = f"{alias}[{type_dec}]" + if recommended_type is not None: + hint = f' (hint: "{node.name}: {recommended_type} = ...")' + + self.fail( + f'Need type annotation for "{unmangle(node.name)}"{hint}', + context, + code=codes.VAR_ANNOTATED, + ) + + def explicit_any(self, ctx: Context) -> None: + self.fail('Explicit "Any" is not allowed', ctx) + + def unsupported_target_for_star_typeddict(self, typ: Type, ctx: Context) -> None: + self.fail( + "Unsupported type {} for ** expansion in TypedDict".format( + format_type(typ, self.options) + ), + ctx, + code=codes.TYPEDDICT_ITEM, + ) + + def non_required_keys_absent_with_star(self, keys: list[str], ctx: Context) -> None: + self.fail( + "Non-required {} not explicitly found in any ** item".format( + format_key_list(keys, short=True) + ), + ctx, + code=codes.TYPEDDICT_ITEM, + ) + + def unexpected_typeddict_keys( + self, + typ: TypedDictType, + expected_keys: list[str], + actual_keys: list[str], + context: Context, + ) -> None: + actual_set = set(actual_keys) + expected_set = set(expected_keys) + if not typ.is_anonymous(): + # Generate simpler messages for some common special cases. + # Use list comprehension instead of set operations to preserve order. + missing = [key for key in expected_keys if key not in actual_set] + if missing: + self.fail( + "Missing {} for TypedDict {}".format( + format_key_list(missing, short=True), format_type(typ, self.options) + ), + context, + code=codes.TYPEDDICT_ITEM, + ) + extra = [key for key in actual_keys if key not in expected_set] + if extra: + self.fail( + "Extra {} for TypedDict {}".format( + format_key_list(extra, short=True), format_type(typ, self.options) + ), + context, + code=codes.TYPEDDICT_UNKNOWN_KEY, + ) + if missing or extra: + # No need to check for further errors + return + found = format_key_list(actual_keys, short=True) + if not expected_keys: + self.fail(f"Unexpected TypedDict {found}", context) + return + expected = format_key_list(expected_keys) + if actual_keys and actual_set < expected_set: + found = f"only {found}" + self.fail(f"Expected {expected} but found {found}", context, code=codes.TYPEDDICT_ITEM) + + def typeddict_key_must_be_string_literal(self, typ: TypedDictType, context: Context) -> None: + self.fail( + "TypedDict key must be a string literal; expected one of {}".format( + format_item_name_list(typ.items.keys()) + ), + context, + code=codes.LITERAL_REQ, + ) + + def typeddict_key_not_found( + self, typ: TypedDictType, item_name: str, context: Context, setitem: bool = False + ) -> None: + """Handle error messages for TypedDicts that have unknown keys. + + Note, that we differentiate in between reading a value and setting a + value. + Setting a value on a TypedDict is an 'unknown-key' error, whereas + reading it is the more serious/general 'item' error. + """ + if typ.is_anonymous(): + self.fail( + '"{}" is not a valid TypedDict key; expected one of {}'.format( + item_name, format_item_name_list(typ.items.keys()) + ), + context, + ) + else: + err_code = codes.TYPEDDICT_UNKNOWN_KEY if setitem else codes.TYPEDDICT_ITEM + self.fail( + f'TypedDict {format_type(typ, self.options)} has no key "{item_name}"', + context, + code=err_code, + ) + matches = best_matches(item_name, typ.items.keys(), n=3) + if matches: + self.note( + "Did you mean {}?".format(pretty_seq(matches, "or")), context, code=err_code + ) + + def typeddict_context_ambiguous(self, types: list[TypedDictType], context: Context) -> None: + formatted_types = ", ".join(list(format_type_distinctly(*types, options=self.options))) + self.fail( + f"Type of TypedDict is ambiguous, none of ({formatted_types}) matches cleanly", context + ) + + def typeddict_key_cannot_be_deleted( + self, typ: TypedDictType, item_name: str, context: Context + ) -> None: + if typ.is_anonymous(): + self.fail(f'TypedDict key "{item_name}" cannot be deleted', context) + else: + self.fail( + f'Key "{item_name}" of TypedDict {format_type(typ, self.options)} cannot be deleted', + context, + ) + + def typeddict_setdefault_arguments_inconsistent( + self, default: Type, expected: Type, context: Context + ) -> None: + msg = 'Argument 2 to "setdefault" of "TypedDict" has incompatible type {}; expected {}' + self.fail( + msg.format(format_type(default, self.options), format_type(expected, self.options)), + context, + code=codes.TYPEDDICT_ITEM, + ) + + def type_arguments_not_allowed(self, context: Context) -> None: + self.fail("Parameterized generics cannot be used with class or instance checks", context) + + def disallowed_any_type(self, typ: Type, context: Context) -> None: + typ = get_proper_type(typ) + if isinstance(typ, AnyType): + message = 'Expression has type "Any"' + else: + message = f'Expression type contains "Any" (has type {format_type(typ, self.options)})' + self.fail(message, context) + + def incorrectly_returning_any(self, typ: Type, context: Context) -> None: + message = ( + f"Returning Any from function declared to return {format_type(typ, self.options)}" + ) + self.fail(message, context, code=codes.NO_ANY_RETURN) + + def incorrect__exit__return(self, context: Context) -> None: + self.fail( + '"bool" is invalid as return type for "__exit__" that always returns False', + context, + code=codes.EXIT_RETURN, + ) + self.note( + 'Use "typing_extensions.Literal[False]" as the return type or change it to "None"', + context, + code=codes.EXIT_RETURN, + ) + self.note( + 'If return type of "__exit__" implies that it may return True, ' + "the context manager may swallow exceptions", + context, + code=codes.EXIT_RETURN, + ) + + def untyped_decorated_function(self, typ: Type, context: Context) -> None: + typ = get_proper_type(typ) + if isinstance(typ, AnyType): + self.fail("Function is untyped after decorator transformation", context) + else: + self.fail( + f'Type of decorated function contains type "Any" ({format_type(typ, self.options)})', + context, + ) + + def typed_function_untyped_decorator(self, func_name: str, context: Context) -> None: + self.fail(f'Untyped decorator makes function "{func_name}" untyped', context) + + def bad_proto_variance( + self, actual: int, tvar_name: str, expected: int, context: Context + ) -> None: + msg = capitalize( + '{} type variable "{}" used in protocol where {} one is expected'.format( + variance_string(actual), tvar_name, variance_string(expected) + ) + ) + self.fail(msg, context) + + def concrete_only_assign(self, typ: Type, context: Context) -> None: + self.fail( + f"Can only assign concrete classes to a variable of type {format_type(typ, self.options)}", + context, + code=codes.TYPE_ABSTRACT, + ) + + def concrete_only_call(self, typ: Type, context: Context) -> None: + self.fail( + f"Only concrete class can be given where {format_type(typ, self.options)} is expected", + context, + code=codes.TYPE_ABSTRACT, + ) + + def cannot_use_function_with_type( + self, method_name: str, type_name: str, context: Context + ) -> None: + self.fail(f"Cannot use {method_name}() with {type_name} type", context) + + def report_non_method_protocol( + self, tp: TypeInfo, members: list[str], context: Context + ) -> None: + self.fail( + "Only protocols that don't have non-method members can be used with issubclass()", + context, + ) + if len(members) < 3: + attrs = ", ".join(members) + self.note(f'Protocol "{tp.name}" has non-method member(s): {attrs}', context) + + def note_call( + self, subtype: Type, call: Type, context: Context, *, code: ErrorCode | None + ) -> None: + self.note( + '"{}.__call__" has type {}'.format( + format_type_bare(subtype, self.options), + format_type(call, self.options, verbosity=1), + ), + context, + code=code, + ) + + def unreachable_statement(self, context: Context) -> None: + self.fail("Statement is unreachable", context, code=codes.UNREACHABLE) + + def redundant_left_operand(self, op_name: str, context: Context) -> None: + """Indicates that the left operand of a boolean expression is redundant: + it does not change the truth value of the entire condition as a whole. + 'op_name' should either be the string "and" or the string "or". + """ + self.redundant_expr(f'Left operand of "{op_name}"', op_name == "and", context) + + def unreachable_right_operand(self, op_name: str, context: Context) -> None: + """Indicates that the right operand of a boolean expression is redundant: + it does not change the truth value of the entire condition as a whole. + 'op_name' should either be the string "and" or the string "or". + """ + self.fail( + f'Right operand of "{op_name}" is never evaluated', context, code=codes.UNREACHABLE + ) + + def redundant_condition_in_comprehension(self, truthiness: bool, context: Context) -> None: + self.redundant_expr("If condition in comprehension", truthiness, context) + + def redundant_condition_in_if(self, truthiness: bool, context: Context) -> None: + self.redundant_expr("If condition", truthiness, context) + + def redundant_expr(self, description: str, truthiness: bool, context: Context) -> None: + self.fail( + f"{description} is always {str(truthiness).lower()}", + context, + code=codes.REDUNDANT_EXPR, + ) + + def impossible_intersection( + self, formatted_base_class_list: str, reason: str, context: Context + ) -> None: + template = "Subclass of {} cannot exist: {}" + self.fail( + template.format(formatted_base_class_list, reason), context, code=codes.UNREACHABLE + ) + + def tvar_without_default_type( + self, tvar_name: str, last_tvar_name_with_default: str, context: Context + ) -> None: + self.fail( + f'"{tvar_name}" cannot appear after "{last_tvar_name_with_default}" ' + "in type parameter list because it has no default type", + context, + ) + + def report_protocol_problems( + self, + subtype: Instance | TupleType | TypedDictType | TypeType | CallableType, + supertype: Instance, + context: Context, + *, + code: ErrorCode | None, + ) -> None: + """Report possible protocol conflicts between 'subtype' and 'supertype'. + + This includes missing members, incompatible types, and incompatible + attribute flags, such as settable vs read-only or class variable vs + instance variable. + """ + OFFSET = 4 # Four spaces, so that notes will look like this: + # note: 'Cls' is missing following 'Proto' members: + # note: method, attr + MAX_ITEMS = 2 # Maximum number of conflicts, missing members, and overloads shown + # List of special situations where we don't want to report additional problems + exclusions: dict[type, list[str]] = { + TypedDictType: ["typing.Mapping"], + TupleType: ["typing.Iterable", "typing.Sequence"], + } + if supertype.type.fullname in exclusions.get(type(subtype), []): + return + if any(isinstance(tp, UninhabitedType) for tp in get_proper_types(supertype.args)): + # We don't want to add notes for failed inference (e.g. Iterable[Never]). + # This will be only confusing a user even more. + return + + class_obj = False + is_module = False + skip = [] + if isinstance(subtype, TupleType): + if not isinstance(subtype.partial_fallback, Instance): + return + subtype = subtype.partial_fallback + elif isinstance(subtype, TypedDictType): + if not isinstance(subtype.fallback, Instance): + return + subtype = subtype.fallback + elif isinstance(subtype, TypeType): + if not isinstance(subtype.item, Instance): + return + class_obj = True + subtype = subtype.item + elif isinstance(subtype, CallableType): + if subtype.is_type_obj(): + ret_type = get_proper_type(subtype.ret_type) + if isinstance(ret_type, TupleType): + ret_type = ret_type.partial_fallback + if not isinstance(ret_type, Instance): + return + class_obj = True + subtype = ret_type + else: + subtype = subtype.fallback + skip = ["__call__"] + if subtype.extra_attrs and subtype.extra_attrs.mod_name: + is_module = True + + # Report missing members + missing = get_missing_protocol_members(subtype, supertype, skip=skip) + if ( + missing + and (len(missing) < len(supertype.type.protocol_members) or missing == ["__call__"]) + and len(missing) <= MAX_ITEMS + ): + if missing == ["__call__"] and class_obj: + self.note( + '"{}" has constructor incompatible with "__call__" of "{}"'.format( + subtype.type.name, supertype.type.name + ), + context, + code=code, + ) + else: + self.note( + '"{}" is missing following "{}" protocol member{}:'.format( + subtype.type.name, supertype.type.name, plural_s(missing) + ), + context, + code=code, + ) + self.note(", ".join(missing), context, offset=OFFSET, code=code) + elif len(missing) > MAX_ITEMS or len(missing) == len(supertype.type.protocol_members): + # This is an obviously wrong type: too many missing members + return + + # Report member type conflicts + conflict_types = get_conflict_protocol_types( + subtype, supertype, class_obj=class_obj, options=self.options + ) + if conflict_types and ( + not is_subtype(subtype, erase_type(supertype), options=self.options) + or not subtype.type.defn.type_vars + or not supertype.type.defn.type_vars + # Always show detailed message for ParamSpec + or subtype.type.has_param_spec_type + or supertype.type.has_param_spec_type + ): + type_name = format_type(subtype, self.options, module_names=True) + self.note(f"Following member(s) of {type_name} have conflicts:", context, code=code) + for name, got, exp in conflict_types[:MAX_ITEMS]: + exp = get_proper_type(exp) + got = get_proper_type(got) + if not isinstance(exp, (CallableType, Overloaded)) or not isinstance( + got, (CallableType, Overloaded) + ): + self.note( + "{}: expected {}, got {}".format( + name, *format_type_distinctly(exp, got, options=self.options) + ), + context, + offset=OFFSET, + code=code, + ) + else: + self.note("Expected:", context, offset=OFFSET, code=code) + if isinstance(exp, CallableType): + self.note( + pretty_callable(exp, self.options, skip_self=class_obj or is_module), + context, + offset=2 * OFFSET, + code=code, + ) + else: + assert isinstance(exp, Overloaded) + self.pretty_overload( + exp, context, 2 * OFFSET, code=code, skip_self=class_obj or is_module + ) + self.note("Got:", context, offset=OFFSET, code=code) + if isinstance(got, CallableType): + self.note( + pretty_callable(got, self.options, skip_self=class_obj or is_module), + context, + offset=2 * OFFSET, + code=code, + ) + else: + assert isinstance(got, Overloaded) + self.pretty_overload( + got, context, 2 * OFFSET, code=code, skip_self=class_obj or is_module + ) + self.print_more(conflict_types, context, OFFSET, MAX_ITEMS, code=code) + + # Report flag conflicts (i.e. settable vs read-only etc.) + conflict_flags = get_bad_protocol_flags(subtype, supertype, class_obj=class_obj) + for name, subflags, superflags in conflict_flags[:MAX_ITEMS]: + if not class_obj and IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: + self.note( + "Protocol member {}.{} expected instance variable, got class variable".format( + supertype.type.name, name + ), + context, + code=code, + ) + if not class_obj and IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: + self.note( + "Protocol member {}.{} expected class variable, got instance variable".format( + supertype.type.name, name + ), + context, + code=code, + ) + if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: + self.note( + "Protocol member {}.{} expected settable variable," + " got read-only attribute".format(supertype.type.name, name), + context, + code=code, + ) + if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: + self.note( + "Protocol member {}.{} expected class or static method".format( + supertype.type.name, name + ), + context, + code=code, + ) + if ( + class_obj + and IS_VAR in superflags + and (IS_VAR in subflags and IS_CLASSVAR not in subflags) + ): + self.note( + "Only class variables allowed for class object access on protocols," + ' {} is an instance variable of "{}"'.format(name, subtype.type.name), + context, + code=code, + ) + if class_obj and IS_CLASSVAR in superflags: + self.note( + "ClassVar protocol member {}.{} can never be matched by a class object".format( + supertype.type.name, name + ), + context, + code=code, + ) + self.print_more(conflict_flags, context, OFFSET, MAX_ITEMS, code=code) + + def pretty_overload( + self, + tp: Overloaded, + context: Context, + offset: int, + *, + add_class_or_static_decorator: bool = False, + allow_dups: bool = False, + code: ErrorCode | None = None, + skip_self: bool = False, + ) -> None: + for item in tp.items: + self.note("@overload", context, offset=offset, allow_dups=allow_dups, code=code) + + if add_class_or_static_decorator: + decorator = pretty_class_or_static_decorator(item) + if decorator is not None: + self.note(decorator, context, offset=offset, allow_dups=allow_dups, code=code) + + self.note( + pretty_callable(item, self.options, skip_self=skip_self), + context, + offset=offset, + allow_dups=allow_dups, + code=code, + ) + + def print_more( + self, + conflicts: Sequence[Any], + context: Context, + offset: int, + max_items: int, + *, + code: ErrorCode | None = None, + ) -> None: + if len(conflicts) > max_items: + self.note( + f"<{len(conflicts) - max_items} more conflict(s) not shown>", + context, + offset=offset, + code=code, + ) + + def try_report_long_tuple_assignment_error( + self, + subtype: ProperType, + supertype: ProperType, + context: Context, + msg: message_registry.ErrorMessage, + subtype_label: str | None = None, + supertype_label: str | None = None, + ) -> bool: + """Try to generate meaningful error message for very long tuple assignment + + Returns a bool: True when generating long tuple assignment error, + False when no such error reported + """ + if isinstance(subtype, TupleType): + if ( + len(subtype.items) > MAX_TUPLE_ITEMS + and isinstance(supertype, Instance) + and supertype.type.fullname == "builtins.tuple" + ): + lhs_type = supertype.args[0] + lhs_types = [lhs_type] * len(subtype.items) + self.generate_incompatible_tuple_error(lhs_types, subtype.items, context, msg) + return True + elif isinstance(supertype, TupleType) and ( + len(subtype.items) > MAX_TUPLE_ITEMS or len(supertype.items) > MAX_TUPLE_ITEMS + ): + if len(subtype.items) != len(supertype.items): + if supertype_label is not None and subtype_label is not None: + msg = msg.with_additional_msg( + " ({} {}, {} {})".format( + subtype_label, + self.format_long_tuple_type(subtype), + supertype_label, + self.format_long_tuple_type(supertype), + ) + ) + self.fail(msg.value, context, code=msg.code) + return True + self.generate_incompatible_tuple_error( + supertype.items, subtype.items, context, msg + ) + return True + return False + + def format_long_tuple_type(self, typ: TupleType) -> str: + """Format very long tuple type using an ellipsis notation""" + item_cnt = len(typ.items) + if item_cnt > MAX_TUPLE_ITEMS: + return "{}[{}, {}, ... <{} more items>]".format( + "tuple" if self.options.use_lowercase_names() else "Tuple", + format_type_bare(typ.items[0], self.options), + format_type_bare(typ.items[1], self.options), + str(item_cnt - 2), + ) + else: + return format_type_bare(typ, self.options) + + def generate_incompatible_tuple_error( + self, + lhs_types: list[Type], + rhs_types: list[Type], + context: Context, + msg: message_registry.ErrorMessage, + ) -> None: + """Generate error message for individual incompatible tuple pairs""" + error_cnt = 0 + notes: list[str] = [] + for i, (lhs_t, rhs_t) in enumerate(zip(lhs_types, rhs_types)): + if not is_subtype(lhs_t, rhs_t): + if error_cnt < 3: + notes.append( + "Expression tuple item {} has type {}; {} expected; ".format( + str(i), + format_type(rhs_t, self.options), + format_type(lhs_t, self.options), + ) + ) + error_cnt += 1 + + info = f" ({str(error_cnt)} tuple items are incompatible" + if error_cnt - 3 > 0: + info += f"; {str(error_cnt - 3)} items are omitted)" + else: + info += ")" + msg = msg.with_additional_msg(info) + self.fail(msg.value, context, code=msg.code) + for note in notes: + self.note(note, context, code=msg.code) + + def add_fixture_note(self, fullname: str, ctx: Context) -> None: + self.note(f'Maybe your test fixture does not define "{fullname}"?', ctx) + if fullname in SUGGESTED_TEST_FIXTURES: + self.note( + "Consider adding [builtins fixtures/{}] to your test description".format( + SUGGESTED_TEST_FIXTURES[fullname] + ), + ctx, + ) + + def annotation_in_unchecked_function(self, context: Context) -> None: + self.note( + "By default the bodies of untyped functions are not checked," + " consider using --check-untyped-defs", + context, + code=codes.ANNOTATION_UNCHECKED, + ) + + def type_parameters_should_be_declared(self, undeclared: list[str], context: Context) -> None: + names = ", ".join('"' + n + '"' for n in undeclared) + self.fail( + message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format(names), + context, + code=codes.VALID_TYPE, + ) + + +def quote_type_string(type_string: str) -> str: + """Quotes a type representation for use in messages.""" + no_quote_regex = r"^<(tuple|union): \d+ items>$" + if ( + type_string in ["Module", "overloaded function", ""] + or type_string.startswith("Module ") + or re.match(no_quote_regex, type_string) is not None + or type_string.endswith("?") + ): + # Messages are easier to read if these aren't quoted. We use a + # regex to match strings with variable contents. + return type_string + return f'"{type_string}"' + + +def format_callable_args( + arg_types: list[Type], + arg_kinds: list[ArgKind], + arg_names: list[str | None], + format: Callable[[Type], str], + verbosity: int, +) -> str: + """Format a bunch of Callable arguments into a string""" + arg_strings = [] + for arg_name, arg_type, arg_kind in zip(arg_names, arg_types, arg_kinds): + if arg_kind == ARG_POS and arg_name is None or verbosity == 0 and arg_kind.is_positional(): + arg_strings.append(format(arg_type)) + else: + constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] + if arg_kind.is_star() or arg_name is None: + arg_strings.append(f"{constructor}({format(arg_type)})") + else: + arg_strings.append(f"{constructor}({format(arg_type)}, {repr(arg_name)})") + + return ", ".join(arg_strings) + + +def format_type_inner( + typ: Type, + verbosity: int, + options: Options, + fullnames: set[str] | None, + module_names: bool = False, +) -> str: + """ + Convert a type to a relatively short string suitable for error messages. + + Args: + verbosity: a coarse grained control on the verbosity of the type + fullnames: a set of names that should be printed in full + """ + + def format(typ: Type) -> str: + return format_type_inner(typ, verbosity, options, fullnames) + + def format_list(types: Sequence[Type]) -> str: + return ", ".join(format(typ) for typ in types) + + def format_union_items(types: Sequence[Type]) -> list[str]: + formatted = [format(typ) for typ in types if format(typ) != "None"] + if len(formatted) > MAX_UNION_ITEMS and verbosity == 0: + more = len(formatted) - MAX_UNION_ITEMS // 2 + formatted = formatted[: MAX_UNION_ITEMS // 2] + else: + more = 0 + if more: + formatted.append(f"<{more} more items>") + if any(format(typ) == "None" for typ in types): + formatted.append("None") + return formatted + + def format_union(types: Sequence[Type]) -> str: + return " | ".join(format_union_items(types)) + + def format_literal_value(typ: LiteralType) -> str: + if typ.is_enum_literal(): + underlying_type = format(typ.fallback) + return f"{underlying_type}.{typ.value}" + else: + return typ.value_repr() + + if isinstance(typ, TypeAliasType) and typ.is_recursive: + if typ.alias is None: + type_str = "" + else: + if verbosity >= 2 or (fullnames and typ.alias.fullname in fullnames): + type_str = typ.alias.fullname + else: + type_str = typ.alias.name + if typ.args: + type_str += f"[{format_list(typ.args)}]" + return type_str + + # TODO: always mention type alias names in errors. + typ = get_proper_type(typ) + + if isinstance(typ, Instance): + itype = typ + # Get the short name of the type. + if itype.type.fullname == "types.ModuleType": + # Make some common error messages simpler and tidier. + base_str = "Module" + if itype.extra_attrs and itype.extra_attrs.mod_name and module_names: + return f'{base_str} "{itype.extra_attrs.mod_name}"' + return base_str + if itype.type.fullname == "typing._SpecialForm": + # This is not a real type but used for some typing-related constructs. + return "" + if itype.type.fullname in reverse_builtin_aliases and not options.use_lowercase_names(): + alias = reverse_builtin_aliases[itype.type.fullname] + base_str = alias.split(".")[-1] + elif verbosity >= 2 or (fullnames and itype.type.fullname in fullnames): + base_str = itype.type.fullname + else: + base_str = itype.type.name + if not itype.args: + if itype.type.has_type_var_tuple_type and len(itype.type.type_vars) == 1: + return base_str + "[()]" + # No type arguments, just return the type name + return base_str + elif itype.type.fullname == "builtins.tuple": + item_type_str = format(itype.args[0]) + return f"{'tuple' if options.use_lowercase_names() else 'Tuple'}[{item_type_str}, ...]" + else: + # There are type arguments. Convert the arguments to strings. + return f"{base_str}[{format_list(itype.args)}]" + elif isinstance(typ, UnpackType): + if options.use_star_unpack(): + return f"*{format(typ.type)}" + return f"Unpack[{format(typ.type)}]" + elif isinstance(typ, TypeVarType): + # This is similar to non-generic instance types. + fullname = scoped_type_var_name(typ) + if verbosity >= 2 or (fullnames and fullname in fullnames): + return fullname + return typ.name + elif isinstance(typ, TypeVarTupleType): + # This is similar to non-generic instance types. + fullname = scoped_type_var_name(typ) + if verbosity >= 2 or (fullnames and fullname in fullnames): + return fullname + return typ.name + elif isinstance(typ, ParamSpecType): + # Concatenate[..., P] + if typ.prefix.arg_types: + args = format_callable_args( + typ.prefix.arg_types, typ.prefix.arg_kinds, typ.prefix.arg_names, format, verbosity + ) + + return f"[{args}, **{typ.name_with_suffix()}]" + else: + # TODO: better disambiguate ParamSpec name clashes. + return typ.name_with_suffix() + elif isinstance(typ, TupleType): + # Prefer the name of the fallback class (if not tuple), as it's more informative. + if typ.partial_fallback.type.fullname != "builtins.tuple": + return format(typ.partial_fallback) + type_items = format_list(typ.items) or "()" + if options.use_lowercase_names(): + s = f"tuple[{type_items}]" + else: + s = f"Tuple[{type_items}]" + return s + elif isinstance(typ, TypedDictType): + # If the TypedDictType is named, return the name + if not typ.is_anonymous(): + return format(typ.fallback) + items = [] + for item_name, item_type in typ.items.items(): + modifier = "" + if item_name not in typ.required_keys: + modifier += "?" + if item_name in typ.readonly_keys: + modifier += "=" + items.append(f"{item_name!r}{modifier}: {format(item_type)}") + return f"TypedDict({{{', '.join(items)}}})" + elif isinstance(typ, LiteralType): + return f"Literal[{format_literal_value(typ)}]" + elif isinstance(typ, UnionType): + typ = get_proper_type(ignore_last_known_values(typ)) + if not isinstance(typ, UnionType): + return format(typ) + literal_items, union_items = separate_union_literals(typ) + + # Coalesce multiple Literal[] members. This also changes output order. + # If there's just one Literal item, retain the original ordering. + if len(literal_items) > 1: + literal_str = "Literal[{}]".format( + ", ".join(format_literal_value(t) for t in literal_items) + ) + + if len(union_items) == 1 and isinstance(get_proper_type(union_items[0]), NoneType): + return ( + f"{literal_str} | None" + if options.use_or_syntax() + else f"Optional[{literal_str}]" + ) + elif union_items: + return ( + f"{literal_str} | {format_union(union_items)}" + if options.use_or_syntax() + else f"Union[{', '.join(format_union_items(union_items))}, {literal_str}]" + ) + else: + return literal_str + else: + # Only print Union as Optional if the Optional wouldn't have to contain another Union + print_as_optional = ( + len(typ.items) - sum(isinstance(get_proper_type(t), NoneType) for t in typ.items) + == 1 + ) + if print_as_optional: + rest = [t for t in typ.items if not isinstance(get_proper_type(t), NoneType)] + return ( + f"{format(rest[0])} | None" + if options.use_or_syntax() + else f"Optional[{format(rest[0])}]" + ) + else: + s = ( + format_union(typ.items) + if options.use_or_syntax() + else f"Union[{', '.join(format_union_items(typ.items))}]" + ) + return s + elif isinstance(typ, NoneType): + return "None" + elif isinstance(typ, AnyType): + return "Any" + elif isinstance(typ, DeletedType): + return "" + elif isinstance(typ, UninhabitedType): + return "Never" + elif isinstance(typ, TypeType): + type_name = "type" if options.use_lowercase_names() else "Type" + return f"{type_name}[{format(typ.item)}]" + elif isinstance(typ, FunctionLike): + func = typ + if func.is_type_obj(): + # The type of a type object type can be derived from the + # return type (this always works). + return format(TypeType.make_normalized(erase_type(func.items[0].ret_type))) + elif isinstance(func, CallableType): + if func.type_guard is not None: + return_type = f"TypeGuard[{format(func.type_guard)}]" + elif func.type_is is not None: + return_type = f"TypeIs[{format(func.type_is)}]" + else: + return_type = format(func.ret_type) + if func.is_ellipsis_args: + return f"Callable[..., {return_type}]" + param_spec = func.param_spec() + if param_spec is not None: + return f"Callable[{format(param_spec)}, {return_type}]" + args = format_callable_args( + func.arg_types, func.arg_kinds, func.arg_names, format, verbosity + ) + return f"Callable[[{args}], {return_type}]" + else: + # Use a simple representation for function types; proper + # function types may result in long and difficult-to-read + # error messages. + return "overloaded function" + elif isinstance(typ, UnboundType): + return typ.accept(TypeStrVisitor(options=options)) + elif isinstance(typ, Parameters): + args = format_callable_args(typ.arg_types, typ.arg_kinds, typ.arg_names, format, verbosity) + return f"[{args}]" + elif typ is None: + raise RuntimeError("Type is None") + else: + # Default case; we simply have to return something meaningful here. + return "object" + + +def collect_all_named_types(t: Type) -> list[Type]: + """Return all instances/aliases/type variables that `t` contains (including `t`). + + This is similar to collect_all_inner_types from typeanal but only + returns instances and will recurse into fallbacks. + """ + visitor = CollectAllNamedTypesQuery() + t.accept(visitor) + return visitor.types + + +class CollectAllNamedTypesQuery(TypeTraverserVisitor): + def __init__(self) -> None: + self.types: list[Type] = [] + + def visit_instance(self, t: Instance) -> None: + self.types.append(t) + super().visit_instance(t) + + def visit_type_alias_type(self, t: TypeAliasType) -> None: + if t.alias and not t.is_recursive: + get_proper_type(t).accept(self) + else: + self.types.append(t) + super().visit_type_alias_type(t) + + def visit_type_var(self, t: TypeVarType) -> None: + self.types.append(t) + super().visit_type_var(t) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + self.types.append(t) + super().visit_type_var_tuple(t) + + def visit_param_spec(self, t: ParamSpecType) -> None: + self.types.append(t) + super().visit_param_spec(t) + + +def scoped_type_var_name(t: TypeVarLikeType) -> str: + if not t.id.namespace: + return t.name + # TODO: support rare cases when both TypeVar name and namespace suffix coincide. + *_, suffix = t.id.namespace.split(".") + return f"{t.name}@{suffix}" + + +def find_type_overlaps(*types: Type) -> set[str]: + """Return a set of fullnames that share a short name and appear in either type. + + This is used to ensure that distinct types with the same short name are printed + with their fullname. + """ + d: dict[str, set[str]] = {} + for type in types: + for t in collect_all_named_types(type): + if isinstance(t, ProperType) and isinstance(t, Instance): + d.setdefault(t.type.name, set()).add(t.type.fullname) + elif isinstance(t, TypeAliasType) and t.alias: + d.setdefault(t.alias.name, set()).add(t.alias.fullname) + else: + assert isinstance(t, TypeVarLikeType) + d.setdefault(t.name, set()).add(scoped_type_var_name(t)) + for shortname in d.keys(): + if f"typing.{shortname}" in TYPES_FOR_UNIMPORTED_HINTS: + d[shortname].add(f"typing.{shortname}") + + overlaps: set[str] = set() + for fullnames in d.values(): + if len(fullnames) > 1: + overlaps.update(fullnames) + return overlaps + + +def format_type( + typ: Type, options: Options, verbosity: int = 0, module_names: bool = False +) -> str: + """ + Convert a type to a relatively short string suitable for error messages. + + `verbosity` is a coarse-grained control on the verbosity of the type + + This function returns a string appropriate for unmodified use in error + messages; this means that it will be quoted in most cases. If + modification of the formatted string is required, callers should use + format_type_bare. + """ + return quote_type_string(format_type_bare(typ, options, verbosity, module_names)) + + +def format_type_bare( + typ: Type, options: Options, verbosity: int = 0, module_names: bool = False +) -> str: + """ + Convert a type to a relatively short string suitable for error messages. + + `verbosity` is a coarse-grained control on the verbosity of the type + `fullnames` specifies a set of names that should be printed in full + + This function will return an unquoted string. If a caller doesn't need to + perform post-processing on the string output, format_type should be used + instead. (The caller may want to use quote_type_string after + processing has happened, to maintain consistent quoting in messages.) + """ + return format_type_inner(typ, verbosity, options, find_type_overlaps(typ), module_names) + + +def format_type_distinctly(*types: Type, options: Options, bare: bool = False) -> tuple[str, ...]: + """Jointly format types to distinct strings. + + Increase the verbosity of the type strings until they become distinct + while also requiring that distinct types with the same short name are + formatted distinctly. + + By default, the returned strings are created using format_type() and will be + quoted accordingly. If ``bare`` is True, the returned strings will not + be quoted; callers who need to do post-processing of the strings before + quoting them (such as prepending * or **) should use this. + """ + overlapping = find_type_overlaps(*types) + for verbosity in range(2): + strs = [ + format_type_inner(type, verbosity=verbosity, options=options, fullnames=overlapping) + for type in types + ] + if len(set(strs)) == len(strs): + break + if bare: + return tuple(strs) + else: + return tuple(quote_type_string(s) for s in strs) + + +def pretty_class_or_static_decorator(tp: CallableType) -> str | None: + """Return @classmethod or @staticmethod, if any, for the given callable type.""" + if tp.definition is not None and isinstance(tp.definition, SYMBOL_FUNCBASE_TYPES): + if tp.definition.is_class: + return "@classmethod" + if tp.definition.is_static: + return "@staticmethod" + return None + + +def pretty_callable(tp: CallableType, options: Options, skip_self: bool = False) -> str: + """Return a nice easily-readable representation of a callable type. + For example: + def [T <: int] f(self, x: int, y: T) -> None + + If skip_self is True, print an actual callable type, as it would appear + when bound on an instance/class, rather than how it would appear in the + defining statement. + """ + s = "" + asterisk = False + slash = False + for i in range(len(tp.arg_types)): + if s: + s += ", " + if tp.arg_kinds[i].is_named() and not asterisk: + s += "*, " + asterisk = True + if tp.arg_kinds[i] == ARG_STAR: + s += "*" + asterisk = True + if tp.arg_kinds[i] == ARG_STAR2: + s += "**" + name = tp.arg_names[i] + if name: + s += name + ": " + type_str = format_type_bare(tp.arg_types[i], options) + if tp.arg_kinds[i] == ARG_STAR2 and tp.unpack_kwargs: + type_str = f"Unpack[{type_str}]" + s += type_str + if tp.arg_kinds[i].is_optional(): + s += " = ..." + if ( + not slash + and tp.arg_kinds[i].is_positional() + and name is None + and ( + i == len(tp.arg_types) - 1 + or (tp.arg_names[i + 1] is not None or not tp.arg_kinds[i + 1].is_positional()) + ) + ): + s += ", /" + slash = True + + # If we got a "special arg" (i.e: self, cls, etc...), prepend it to the arg list + if ( + isinstance(tp.definition, FuncDef) + and hasattr(tp.definition, "arguments") + and not tp.from_concatenate + ): + definition_arg_names = [arg.variable.name for arg in tp.definition.arguments] + if ( + len(definition_arg_names) > len(tp.arg_names) + and definition_arg_names[0] + and not skip_self + ): + if s: + s = ", " + s + s = definition_arg_names[0] + s + s = f"{tp.definition.name}({s})" + elif tp.name: + first_arg = tp.def_extras.get("first_arg") + if first_arg: + if s: + s = ", " + s + s = first_arg + s + s = f"{tp.name.split()[0]}({s})" # skip "of Class" part + else: + s = f"({s})" + + s += " -> " + if tp.type_guard is not None: + s += f"TypeGuard[{format_type_bare(tp.type_guard, options)}]" + elif tp.type_is is not None: + s += f"TypeIs[{format_type_bare(tp.type_is, options)}]" + else: + s += format_type_bare(tp.ret_type, options) + + if tp.variables: + tvars = [] + for tvar in tp.variables: + if isinstance(tvar, TypeVarType): + upper_bound = get_proper_type(tvar.upper_bound) + if not ( + isinstance(upper_bound, Instance) + and upper_bound.type.fullname == "builtins.object" + ): + tvars.append(f"{tvar.name}: {format_type_bare(upper_bound, options)}") + elif tvar.values: + tvars.append( + "{}: ({})".format( + tvar.name, + ", ".join([format_type_bare(tp, options) for tp in tvar.values]), + ) + ) + else: + tvars.append(tvar.name) + else: + # For other TypeVarLikeTypes, just use the repr + tvars.append(repr(tvar)) + s = f"[{', '.join(tvars)}] {s}" + return f"def {s}" + + +def variance_string(variance: int) -> str: + if variance == COVARIANT: + return "covariant" + elif variance == CONTRAVARIANT: + return "contravariant" + else: + return "invariant" + + +def get_missing_protocol_members(left: Instance, right: Instance, skip: list[str]) -> list[str]: + """Find all protocol members of 'right' that are not implemented + (i.e. completely missing) in 'left'. + """ + assert right.type.is_protocol + missing: list[str] = [] + for member in right.type.protocol_members: + if member in skip: + continue + if not find_member(member, left, left): + missing.append(member) + return missing + + +def get_conflict_protocol_types( + left: Instance, right: Instance, class_obj: bool = False, options: Options | None = None +) -> list[tuple[str, Type, Type]]: + """Find members that are defined in 'left' but have incompatible types. + Return them as a list of ('member', 'got', 'expected'). + """ + assert right.type.is_protocol + conflicts: list[tuple[str, Type, Type]] = [] + for member in right.type.protocol_members: + if member in ("__init__", "__new__"): + continue + supertype = find_member(member, right, left) + assert supertype is not None + subtype = mypy.typeops.get_protocol_member(left, member, class_obj) + if not subtype: + continue + is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True, options=options) + if IS_SETTABLE in get_member_flags(member, right): + is_compat = is_compat and is_subtype(supertype, subtype, options=options) + if not is_compat: + conflicts.append((member, subtype, supertype)) + return conflicts + + +def get_bad_protocol_flags( + left: Instance, right: Instance, class_obj: bool = False +) -> list[tuple[str, set[int], set[int]]]: + """Return all incompatible attribute flags for members that are present in both + 'left' and 'right'. + """ + assert right.type.is_protocol + all_flags: list[tuple[str, set[int], set[int]]] = [] + for member in right.type.protocol_members: + if find_member(member, left, left): + item = (member, get_member_flags(member, left), get_member_flags(member, right)) + all_flags.append(item) + bad_flags = [] + for name, subflags, superflags in all_flags: + if ( + IS_CLASSVAR in subflags + and IS_CLASSVAR not in superflags + and IS_SETTABLE in superflags + or IS_CLASSVAR in superflags + and IS_CLASSVAR not in subflags + or IS_SETTABLE in superflags + and IS_SETTABLE not in subflags + or IS_CLASS_OR_STATIC in superflags + and IS_CLASS_OR_STATIC not in subflags + or class_obj + and IS_VAR in superflags + and IS_CLASSVAR not in subflags + or class_obj + and IS_CLASSVAR in superflags + ): + bad_flags.append((name, subflags, superflags)) + return bad_flags def capitalize(s: str) -> str: """Capitalize the first character of a string.""" - if s == '': - return '' + if s == "": + return "" else: return s[0].upper() + s[1:] @@ -710,40 +3075,236 @@ def extract_type(name: str) -> str: the type portion in quotes (e.g. "y"). Otherwise, return the string unmodified. """ - name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) + name = re.sub('^"[a-zA-Z0-9_]+" of ', "", name) return name def strip_quotes(s: str) -> str: """Strip a double quote at the beginning and end of the string, if any.""" - s = re.sub('^"', '', s) - s = re.sub('"$', '', s) + s = re.sub('^"', "", s) + s = re.sub('"$', "", s) return s -def plural_s(s: Sequence[Any]) -> str: - if len(s) > 1: - return 's' +def format_string_list(lst: list[str]) -> str: + assert lst + if len(lst) == 1: + return lst[0] + elif len(lst) <= 5: + return f"{', '.join(lst[:-1])} and {lst[-1]}" else: - return '' + return "%s, ... and %s (%i methods suppressed)" % ( + ", ".join(lst[:2]), + lst[-1], + len(lst) - 3, + ) -def format_string_list(s: Iterable[str]) -> str: - l = list(s) - assert len(l) > 0 - if len(l) == 1: - return l[0] +def format_item_name_list(s: Iterable[str]) -> str: + lst = list(s) + if len(lst) <= 5: + return "(" + ", ".join([f'"{name}"' for name in lst]) + ")" else: - return '%s and %s' % (', '.join(l[:-1]), l[-1]) + return "(" + ", ".join([f'"{name}"' for name in lst[:5]]) + ", ...)" + + +def callable_name(type: FunctionLike) -> str | None: + name = type.get_name() + if name is not None and name[0] != "<": + return f'"{name}"'.replace(" of ", '" of "') + return name + + +def for_function(callee: CallableType) -> str: + name = callable_name(callee) + if name is not None: + return f" for {name}" + return "" -def callable_name(type: Callable) -> str: - if type.name: - return type.name +def wrong_type_arg_count(low: int, high: int, act: str, name: str) -> str: + if low == high: + s = f"{low} type arguments" + if low == 0: + s = "no type arguments" + elif low == 1: + s = "1 type argument" else: - return 'function' + s = f"between {low} and {high} type arguments" + if act == "0": + act = "none" + return f'"{name}" expects {s}, but {act} given' + + +def find_defining_module(modules: dict[str, MypyFile], typ: CallableType) -> MypyFile | None: + if not typ.definition: + return None + fullname = typ.definition.fullname + if "." in fullname: + for i in range(fullname.count(".")): + module_name = fullname.rsplit(".", i + 1)[0] + try: + return modules[module_name] + except KeyError: + pass + assert False, "Couldn't determine module from CallableType" + return None + + +# For hard-coding suggested missing member alternatives. +COMMON_MISTAKES: Final[dict[str, Sequence[str]]] = {"add": ("append", "extend")} + + +def _real_quick_ratio(a: str, b: str) -> float: + # this is an upper bound on difflib.SequenceMatcher.ratio + # similar to difflib.SequenceMatcher.real_quick_ratio, but faster since we don't instantiate + al = len(a) + bl = len(b) + return 2.0 * min(al, bl) / (al + bl) + + +def best_matches(current: str, options: Collection[str], n: int) -> list[str]: + if not current: + return [] + # narrow down options cheaply + options = [o for o in options if _real_quick_ratio(current, o) > 0.75] + if len(options) >= 50: + options = [o for o in options if abs(len(o) - len(current)) <= 1] + + ratios = {option: difflib.SequenceMatcher(a=current, b=option).ratio() for option in options} + options = [option for option, ratio in ratios.items() if ratio > 0.75] + return sorted(options, key=lambda v: (-ratios[v], v))[:n] + + +def pretty_seq(args: Sequence[str], conjunction: str) -> str: + quoted = ['"' + a + '"' for a in args] + if len(quoted) == 1: + return quoted[0] + if len(quoted) == 2: + return f"{quoted[0]} {conjunction} {quoted[1]}" + last_sep = ", " + conjunction + " " + return ", ".join(quoted[:-1]) + last_sep + quoted[-1] + + +def append_invariance_notes( + notes: list[str], arg_type: Instance, expected_type: Instance +) -> list[str]: + """Explain that the type is invariant and give notes for how to solve the issue.""" + invariant_type = "" + covariant_suggestion = "" + if ( + arg_type.type.fullname == "builtins.list" + and expected_type.type.fullname == "builtins.list" + and is_subtype(arg_type.args[0], expected_type.args[0]) + ): + invariant_type = "List" + covariant_suggestion = 'Consider using "Sequence" instead, which is covariant' + elif ( + arg_type.type.fullname == "builtins.dict" + and expected_type.type.fullname == "builtins.dict" + and is_same_type(arg_type.args[0], expected_type.args[0]) + and is_subtype(arg_type.args[1], expected_type.args[1]) + ): + invariant_type = "Dict" + covariant_suggestion = ( + 'Consider using "Mapping" instead, which is covariant in the value type' + ) + if invariant_type and covariant_suggestion: + notes.append( + f'"{invariant_type}" is invariant -- see ' + + "https://mypy.readthedocs.io/en/stable/common_issues.html#variance" + ) + notes.append(covariant_suggestion) + return notes + + +def append_union_note( + notes: list[str], arg_type: UnionType, expected_type: UnionType, options: Options +) -> list[str]: + """Point to specific union item(s) that may cause failure in subtype check.""" + non_matching = [] + items = flatten_nested_unions(arg_type.items) + if len(items) < MAX_UNION_ITEMS: + return notes + for item in items: + if not is_subtype(item, expected_type): + non_matching.append(item) + if non_matching: + types = ", ".join([format_type(typ, options) for typ in non_matching]) + notes.append(f"Item{plural_s(non_matching)} in the first union not in the second: {types}") + return notes + + +def append_numbers_notes( + notes: list[str], arg_type: Instance, expected_type: Instance +) -> list[str]: + """Explain if an unsupported type from "numbers" is used in a subtype check.""" + if expected_type.type.fullname in UNSUPPORTED_NUMBERS_TYPES: + notes.append('Types from "numbers" aren\'t supported for static type checking') + notes.append("See https://peps.python.org/pep-0484/#the-numeric-tower") + notes.append("Consider using a protocol instead, such as typing.SupportsFloat") + return notes + + +def make_inferred_type_note( + context: Context, subtype: Type, supertype: Type, supertype_str: str +) -> str: + """Explain that the user may have forgotten to type a variable. + + The user does not expect an error if the inferred container type is the same as the return + type of a function and the argument type(s) are a subtype of the argument type(s) of the + return type. This note suggests that they add a type annotation with the return type instead + of relying on the inferred type. + """ + subtype = get_proper_type(subtype) + supertype = get_proper_type(supertype) + if ( + isinstance(subtype, Instance) + and isinstance(supertype, Instance) + and subtype.type.fullname == supertype.type.fullname + and subtype.args + and supertype.args + and isinstance(context, ReturnStmt) + and isinstance(context.expr, NameExpr) + and isinstance(context.expr.node, Var) + and context.expr.node.is_inferred + ): + for subtype_arg, supertype_arg in zip(subtype.args, supertype.args): + if not is_subtype(subtype_arg, supertype_arg): + return "" + var_name = context.expr.name + return 'Perhaps you need a type annotation for "{}"? Suggestion: {}'.format( + var_name, supertype_str + ) + return "" + + +def format_key_list(keys: list[str], *, short: bool = False) -> str: + formatted_keys = [f'"{key}"' for key in keys] + td = "" if short else "TypedDict " + if len(keys) == 0: + return f"no {td}keys" + elif len(keys) == 1: + return f"{td}key {formatted_keys[0]}" + else: + return f"{td}keys ({', '.join(formatted_keys)})" -def temp_message_builder() -> MessageBuilder: - """Return a message builder usable for collecting errors locally.""" - return MessageBuilder(Errors()) +def ignore_last_known_values(t: UnionType) -> Type: + """This will avoid types like str | str in error messages. + + last_known_values are kept during union simplification, but may cause + weird formatting for e.g. tuples of literals. + """ + union_items: list[Type] = [] + seen_instances = set() + for item in t.items: + if isinstance(item, ProperType) and isinstance(item, Instance): + erased = item.copy_modified(last_known_value=None) + if erased in seen_instances: + continue + seen_instances.add(erased) + union_items.append(erased) + else: + union_items.append(item) + return UnionType.make_union(union_items, t.line, t.column) diff --git a/mypy/metastore.py b/mypy/metastore.py new file mode 100644 index 000000000000..ece397360e5b --- /dev/null +++ b/mypy/metastore.py @@ -0,0 +1,218 @@ +"""Interfaces for accessing metadata. + +We provide two implementations. + * The "classic" file system implementation, which uses a directory + structure of files. + * A hokey sqlite backed implementation, which basically simulates + the file system in an effort to work around poor file system performance + on OS X. +""" + +from __future__ import annotations + +import binascii +import os +import time +from abc import abstractmethod +from typing import TYPE_CHECKING, Any, Iterable + +if TYPE_CHECKING: + # We avoid importing sqlite3 unless we are using it so we can mostly work + # on semi-broken pythons that are missing it. + import sqlite3 + + +class MetadataStore: + """Generic interface for metadata storage.""" + + @abstractmethod + def getmtime(self, name: str) -> float: + """Read the mtime of a metadata entry. + + Raises FileNotFound if the entry does not exist. + """ + + @abstractmethod + def read(self, name: str) -> bytes: + """Read the contents of a metadata entry. + + Raises FileNotFound if the entry does not exist. + """ + + @abstractmethod + def write(self, name: str, data: bytes, mtime: float | None = None) -> bool: + """Write a metadata entry. + + If mtime is specified, set it as the mtime of the entry. Otherwise, + the current time is used. + + Returns True if the entry is successfully written, False otherwise. + """ + + @abstractmethod + def remove(self, name: str) -> None: + """Delete a metadata entry""" + + @abstractmethod + def commit(self) -> None: + """If the backing store requires a commit, do it. + + But N.B. that this is not *guaranteed* to do anything, and + there is no guarantee that changes are not made until it is + called. + """ + + @abstractmethod + def list_all(self) -> Iterable[str]: ... + + +def random_string() -> str: + return binascii.hexlify(os.urandom(8)).decode("ascii") + + +class FilesystemMetadataStore(MetadataStore): + def __init__(self, cache_dir_prefix: str) -> None: + # We check startswith instead of equality because the version + # will have already been appended by the time the cache dir is + # passed here. + if cache_dir_prefix.startswith(os.devnull): + self.cache_dir_prefix = None + else: + self.cache_dir_prefix = cache_dir_prefix + + def getmtime(self, name: str) -> float: + if not self.cache_dir_prefix: + raise FileNotFoundError() + + return int(os.path.getmtime(os.path.join(self.cache_dir_prefix, name))) + + def read(self, name: str) -> bytes: + assert os.path.normpath(name) != os.path.abspath(name), "Don't use absolute paths!" + + if not self.cache_dir_prefix: + raise FileNotFoundError() + + with open(os.path.join(self.cache_dir_prefix, name), "rb") as f: + return f.read() + + def write(self, name: str, data: bytes, mtime: float | None = None) -> bool: + assert os.path.normpath(name) != os.path.abspath(name), "Don't use absolute paths!" + + if not self.cache_dir_prefix: + return False + + path = os.path.join(self.cache_dir_prefix, name) + tmp_filename = path + "." + random_string() + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(tmp_filename, "wb") as f: + f.write(data) + os.replace(tmp_filename, path) + if mtime is not None: + os.utime(path, times=(mtime, mtime)) + + except OSError: + return False + return True + + def remove(self, name: str) -> None: + if not self.cache_dir_prefix: + raise FileNotFoundError() + + os.remove(os.path.join(self.cache_dir_prefix, name)) + + def commit(self) -> None: + pass + + def list_all(self) -> Iterable[str]: + if not self.cache_dir_prefix: + return + + for dir, _, files in os.walk(self.cache_dir_prefix): + dir = os.path.relpath(dir, self.cache_dir_prefix) + for file in files: + yield os.path.normpath(os.path.join(dir, file)) + + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS files2 ( + path TEXT UNIQUE NOT NULL, + mtime REAL, + data BLOB +); +CREATE INDEX IF NOT EXISTS path_idx on files2(path); +""" + + +def connect_db(db_file: str) -> sqlite3.Connection: + import sqlite3.dbapi2 + + db = sqlite3.dbapi2.connect(db_file) + db.executescript(SCHEMA) + return db + + +class SqliteMetadataStore(MetadataStore): + def __init__(self, cache_dir_prefix: str) -> None: + # We check startswith instead of equality because the version + # will have already been appended by the time the cache dir is + # passed here. + if cache_dir_prefix.startswith(os.devnull): + self.db = None + return + + os.makedirs(cache_dir_prefix, exist_ok=True) + self.db = connect_db(os.path.join(cache_dir_prefix, "cache.db")) + + def _query(self, name: str, field: str) -> Any: + # Raises FileNotFound for consistency with the file system version + if not self.db: + raise FileNotFoundError() + + cur = self.db.execute(f"SELECT {field} FROM files2 WHERE path = ?", (name,)) + results = cur.fetchall() + if not results: + raise FileNotFoundError() + assert len(results) == 1 + return results[0][0] + + def getmtime(self, name: str) -> float: + mtime = self._query(name, "mtime") + assert isinstance(mtime, float) + return mtime + + def read(self, name: str) -> bytes: + data = self._query(name, "data") + assert isinstance(data, bytes) + return data + + def write(self, name: str, data: bytes, mtime: float | None = None) -> bool: + import sqlite3 + + if not self.db: + return False + try: + if mtime is None: + mtime = time.time() + self.db.execute( + "INSERT OR REPLACE INTO files2(path, mtime, data) VALUES(?, ?, ?)", + (name, mtime, data), + ) + except sqlite3.OperationalError: + return False + return True + + def remove(self, name: str) -> None: + if not self.db: + raise FileNotFoundError() + + self.db.execute("DELETE FROM files2 WHERE path = ?", (name,)) + + def commit(self) -> None: + if self.db: + self.db.commit() + + def list_all(self) -> Iterable[str]: + if self.db: + for row in self.db.execute("SELECT path FROM files2"): + yield row[0] diff --git a/mypy/mixedtraverser.py b/mypy/mixedtraverser.py new file mode 100644 index 000000000000..dfde41859c67 --- /dev/null +++ b/mypy/mixedtraverser.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from mypy.nodes import ( + AssertTypeExpr, + AssignmentStmt, + CastExpr, + ClassDef, + ForStmt, + FuncItem, + NamedTupleExpr, + NewTypeExpr, + PromoteExpr, + TypeAliasExpr, + TypeApplication, + TypedDictExpr, + TypeVarExpr, + Var, + WithStmt, +) +from mypy.traverser import TraverserVisitor +from mypy.types import Type +from mypy.typetraverser import TypeTraverserVisitor + + +class MixedTraverserVisitor(TraverserVisitor, TypeTraverserVisitor): + """Recursive traversal of both Node and Type objects.""" + + def __init__(self) -> None: + self.in_type_alias_expr = False + + # Symbol nodes + + def visit_var(self, var: Var) -> None: + self.visit_optional_type(var.type) + + def visit_func(self, o: FuncItem) -> None: + super().visit_func(o) + self.visit_optional_type(o.type) + + def visit_class_def(self, o: ClassDef) -> None: + # TODO: Should we visit generated methods/variables as well, either here or in + # TraverserVisitor? + super().visit_class_def(o) + info = o.info + if info: + for base in info.bases: + base.accept(self) + + def visit_type_alias_expr(self, o: TypeAliasExpr) -> None: + super().visit_type_alias_expr(o) + self.in_type_alias_expr = True + o.node.target.accept(self) + self.in_type_alias_expr = False + + def visit_type_var_expr(self, o: TypeVarExpr) -> None: + super().visit_type_var_expr(o) + o.upper_bound.accept(self) + for value in o.values: + value.accept(self) + + def visit_typeddict_expr(self, o: TypedDictExpr) -> None: + super().visit_typeddict_expr(o) + self.visit_optional_type(o.info.typeddict_type) + + def visit_namedtuple_expr(self, o: NamedTupleExpr) -> None: + super().visit_namedtuple_expr(o) + assert o.info.tuple_type + o.info.tuple_type.accept(self) + + def visit__promote_expr(self, o: PromoteExpr) -> None: + super().visit__promote_expr(o) + o.type.accept(self) + + def visit_newtype_expr(self, o: NewTypeExpr) -> None: + super().visit_newtype_expr(o) + self.visit_optional_type(o.old_type) + + # Statements + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + super().visit_assignment_stmt(o) + self.visit_optional_type(o.type) + + def visit_for_stmt(self, o: ForStmt) -> None: + super().visit_for_stmt(o) + self.visit_optional_type(o.index_type) + + def visit_with_stmt(self, o: WithStmt) -> None: + super().visit_with_stmt(o) + for typ in o.analyzed_types: + typ.accept(self) + + # Expressions + + def visit_cast_expr(self, o: CastExpr) -> None: + super().visit_cast_expr(o) + o.type.accept(self) + + def visit_assert_type_expr(self, o: AssertTypeExpr) -> None: + super().visit_assert_type_expr(o) + o.type.accept(self) + + def visit_type_application(self, o: TypeApplication) -> None: + super().visit_type_application(o) + for t in o.types: + t.accept(self) + + # Helpers + + def visit_optional_type(self, t: Type | None) -> None: + if t: + t.accept(self) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py new file mode 100644 index 000000000000..fdd89837002f --- /dev/null +++ b/mypy/modulefinder.py @@ -0,0 +1,919 @@ +"""Low-level infrastructure to find modules. + +This builds on fscache.py; find_sources.py builds on top of this. +""" + +from __future__ import annotations + +import ast +import collections +import functools +import os +import re +import subprocess +import sys +from enum import Enum, unique +from typing import Dict, Final, List, Optional, Tuple, Union +from typing_extensions import TypeAlias as _TypeAlias + +from mypy import pyinfo +from mypy.errors import CompileError +from mypy.fscache import FileSystemCache +from mypy.nodes import MypyFile +from mypy.options import Options +from mypy.stubinfo import stub_distribution_name +from mypy.util import os_path_join + + +# Paths to be searched in find_module(). +class SearchPaths: + def __init__( + self, + python_path: tuple[str, ...], + mypy_path: tuple[str, ...], + package_path: tuple[str, ...], + typeshed_path: tuple[str, ...], + ) -> None: + # where user code is found + self.python_path = tuple(map(os.path.abspath, python_path)) + # from $MYPYPATH or config variable + self.mypy_path = tuple(map(os.path.abspath, mypy_path)) + # from get_site_packages_dirs() + self.package_path = tuple(map(os.path.abspath, package_path)) + # paths in typeshed + self.typeshed_path = tuple(map(os.path.abspath, typeshed_path)) + + def asdict(self) -> dict[str, tuple[str, ...]]: + return { + "python_path": self.python_path, + "mypy_path": self.mypy_path, + "package_path": self.package_path, + "typeshed_path": self.typeshed_path, + } + + +# Package dirs are a two-tuple of path to search and whether to verify the module +OnePackageDir = Tuple[str, bool] +PackageDirs = List[OnePackageDir] + +# Minimum and maximum Python versions for modules in stdlib as (major, minor) +StdlibVersions: _TypeAlias = Dict[str, Tuple[Tuple[int, int], Optional[Tuple[int, int]]]] + +PYTHON_EXTENSIONS: Final = [".pyi", ".py"] + + +# TODO: Consider adding more reasons here? +# E.g. if we deduce a module would likely be found if the user were +# to set the --namespace-packages flag. +@unique +class ModuleNotFoundReason(Enum): + # The module was not found: we found neither stubs nor a plausible code + # implementation (with or without a py.typed file). + NOT_FOUND = 0 + + # The implementation for this module plausibly exists (e.g. we + # found a matching folder or *.py file), but either the parent package + # did not contain a py.typed file or we were unable to find a + # corresponding *-stubs package. + FOUND_WITHOUT_TYPE_HINTS = 1 + + # The module was not found in the current working directory, but + # was able to be found in the parent directory. + WRONG_WORKING_DIRECTORY = 2 + + # Stub PyPI package (typically types-pkgname) known to exist but not installed. + APPROVED_STUBS_NOT_INSTALLED = 3 + + def error_message_templates(self, daemon: bool) -> tuple[str, list[str]]: + doc_link = "See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports" + if self is ModuleNotFoundReason.NOT_FOUND: + msg = 'Cannot find implementation or library stub for module named "{module}"' + notes = [doc_link] + elif self is ModuleNotFoundReason.WRONG_WORKING_DIRECTORY: + msg = 'Cannot find implementation or library stub for module named "{module}"' + notes = [ + "You may be running mypy in a subpackage, mypy should be run on the package root" + ] + elif self is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: + msg = ( + 'Skipping analyzing "{module}": module is installed, but missing library stubs ' + "or py.typed marker" + ) + notes = [doc_link] + elif self is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: + msg = 'Library stubs not installed for "{module}"' + notes = ['Hint: "python3 -m pip install {stub_dist}"'] + if not daemon: + notes.append( + '(or run "mypy --install-types" to install all missing stub packages)' + ) + notes.append(doc_link) + else: + assert False + return msg, notes + + +# If we found the module, returns the path to the module as a str. +# Otherwise, returns the reason why the module wasn't found. +ModuleSearchResult = Union[str, ModuleNotFoundReason] + + +class BuildSource: + """A single source file.""" + + def __init__( + self, + path: str | None, + module: str | None, + text: str | None = None, + base_dir: str | None = None, + followed: bool = False, + ) -> None: + self.path = path # File where it's found (e.g. 'xxx/yyy/foo/bar.py') + self.module = module or "__main__" # Module name (e.g. 'foo.bar') + self.text = text # Source code, if initially supplied, else None + self.base_dir = base_dir # Directory where the package is rooted (e.g. 'xxx/yyy') + self.followed = followed # Was this found by following imports? + + def __repr__(self) -> str: + return ( + "BuildSource(path={!r}, module={!r}, has_text={}, base_dir={!r}, followed={})".format( + self.path, self.module, self.text is not None, self.base_dir, self.followed + ) + ) + + +class BuildSourceSet: + """Helper to efficiently test a file's membership in a set of build sources.""" + + def __init__(self, sources: list[BuildSource]) -> None: + self.source_text_present = False + self.source_modules: dict[str, str] = {} + self.source_paths: set[str] = set() + + for source in sources: + if source.text is not None: + self.source_text_present = True + if source.path: + self.source_paths.add(source.path) + if source.module: + self.source_modules[source.module] = source.path or "" + + def is_source(self, file: MypyFile) -> bool: + return ( + (file.path and file.path in self.source_paths) + or file._fullname in self.source_modules + or self.source_text_present + ) + + +class FindModuleCache: + """Module finder with integrated cache. + + Module locations and some intermediate results are cached internally + and can be cleared with the clear() method. + + All file system accesses are performed through a FileSystemCache, + which is not ever cleared by this class. If necessary it must be + cleared by client code. + """ + + def __init__( + self, + search_paths: SearchPaths, + fscache: FileSystemCache | None, + options: Options | None, + stdlib_py_versions: StdlibVersions | None = None, + source_set: BuildSourceSet | None = None, + ) -> None: + self.search_paths = search_paths + self.source_set = source_set + self.fscache = fscache or FileSystemCache() + # Cache for get_toplevel_possibilities: + # search_paths -> (toplevel_id -> list(package_dirs)) + self.initial_components: dict[tuple[str, ...], dict[str, list[str]]] = {} + # Cache find_module: id -> result + self.results: dict[str, ModuleSearchResult] = {} + self.ns_ancestors: dict[str, str] = {} + self.options = options + custom_typeshed_dir = None + if options: + custom_typeshed_dir = options.custom_typeshed_dir + self.stdlib_py_versions = stdlib_py_versions or load_stdlib_py_versions( + custom_typeshed_dir + ) + + def clear(self) -> None: + self.results.clear() + self.initial_components.clear() + self.ns_ancestors.clear() + + def find_module_via_source_set(self, id: str) -> ModuleSearchResult | None: + """Fast path to find modules by looking through the input sources + + This is only used when --fast-module-lookup is passed on the command line.""" + if not self.source_set: + return None + + p = self.source_set.source_modules.get(id, None) + if p and self.fscache.isfile(p): + # We need to make sure we still have __init__.py all the way up + # otherwise we might have false positives compared to slow path + # in case of deletion of init files, which is covered by some tests. + # TODO: are there some combination of flags in which this check should be skipped? + d = os.path.dirname(p) + for _ in range(id.count(".")): + if not any( + self.fscache.isfile(os_path_join(d, "__init__" + x)) for x in PYTHON_EXTENSIONS + ): + return None + d = os.path.dirname(d) + return p + + idx = id.rfind(".") + if idx != -1: + # When we're looking for foo.bar.baz and can't find a matching module + # in the source set, look up for a foo.bar module. + parent = self.find_module_via_source_set(id[:idx]) + if parent is None or not isinstance(parent, str): + return None + + basename, ext = os.path.splitext(parent) + if not any(parent.endswith("__init__" + x) for x in PYTHON_EXTENSIONS) and ( + ext in PYTHON_EXTENSIONS and not self.fscache.isdir(basename) + ): + # If we do find such a *module* (and crucially, we don't want a package, + # hence the filtering out of __init__ files, and checking for the presence + # of a folder with a matching name), then we can be pretty confident that + # 'baz' will either be a top-level variable in foo.bar, or will not exist. + # + # Either way, spelunking in other search paths for another 'foo.bar.baz' + # module should be avoided because: + # 1. in the unlikely event that one were found, it's highly likely that + # it would be unrelated to the source being typechecked and therefore + # more likely to lead to erroneous results + # 2. as described in _find_module, in some cases the search itself could + # potentially waste significant amounts of time + return ModuleNotFoundReason.NOT_FOUND + return None + + def find_lib_path_dirs(self, id: str, lib_path: tuple[str, ...]) -> PackageDirs: + """Find which elements of a lib_path have the directory a module needs to exist.""" + components = id.split(".") + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + + dirs = [] + for pathitem in self.get_toplevel_possibilities(lib_path, components[0]): + # e.g., '/usr/lib/python3.4/foo/bar' + if dir_chain: + dir = os_path_join(pathitem, dir_chain) + else: + dir = pathitem + if self.fscache.isdir(dir): + dirs.append((dir, True)) + return dirs + + def get_toplevel_possibilities(self, lib_path: tuple[str, ...], id: str) -> list[str]: + """Find which elements of lib_path could contain a particular top-level module. + + In practice, almost all modules can be routed to the correct entry in + lib_path by looking at just the first component of the module name. + + We take advantage of this by enumerating the contents of all of the + directories on the lib_path and building a map of which entries in + the lib_path could contain each potential top-level module that appears. + """ + + if lib_path in self.initial_components: + return self.initial_components[lib_path].get(id, []) + + # Enumerate all the files in the directories on lib_path and produce the map + components: dict[str, list[str]] = {} + for dir in lib_path: + try: + contents = self.fscache.listdir(dir) + except OSError: + contents = [] + # False positives are fine for correctness here, since we will check + # precisely later, so we only look at the root of every filename without + # any concern for the exact details. + for name in contents: + name = os.path.splitext(name)[0] + components.setdefault(name, []).append(dir) + + self.initial_components[lib_path] = components + return components.get(id, []) + + def find_module(self, id: str, *, fast_path: bool = False) -> ModuleSearchResult: + """Return the path of the module source file or why it wasn't found. + + If fast_path is True, prioritize performance over generating detailed + error descriptions. + """ + if id not in self.results: + top_level = id.partition(".")[0] + use_typeshed = True + if id in self.stdlib_py_versions: + use_typeshed = self._typeshed_has_version(id) + elif top_level in self.stdlib_py_versions: + use_typeshed = self._typeshed_has_version(top_level) + self.results[id] = self._find_module(id, use_typeshed) + if ( + not (fast_path or (self.options is not None and self.options.fast_module_lookup)) + and self.results[id] is ModuleNotFoundReason.NOT_FOUND + and self._can_find_module_in_parent_dir(id) + ): + self.results[id] = ModuleNotFoundReason.WRONG_WORKING_DIRECTORY + return self.results[id] + + def _typeshed_has_version(self, module: str) -> bool: + if not self.options: + return True + version = typeshed_py_version(self.options) + min_version, max_version = self.stdlib_py_versions[module] + return version >= min_version and (max_version is None or version <= max_version) + + def _find_module_non_stub_helper( + self, id: str, pkg_dir: str + ) -> OnePackageDir | ModuleNotFoundReason: + plausible_match = False + dir_path = pkg_dir + components = id.split(".") + for index, component in enumerate(components): + dir_path = os_path_join(dir_path, component) + if self.fscache.isfile(os_path_join(dir_path, "py.typed")): + return os.path.join(pkg_dir, *components[:-1]), index == 0 + elif not plausible_match and ( + self.fscache.isdir(dir_path) or self.fscache.isfile(dir_path + ".py") + ): + plausible_match = True + # If this is not a directory then we can't traverse further into it + if not self.fscache.isdir(dir_path): + break + if plausible_match: + if self.options: + module_specific_options = self.options.clone_for_module(id) + if module_specific_options.follow_untyped_imports: + return os.path.join(pkg_dir, *components[:-1]), False + return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS + else: + return ModuleNotFoundReason.NOT_FOUND + + def _update_ns_ancestors(self, components: list[str], match: tuple[str, bool]) -> None: + path, verify = match + for i in range(1, len(components)): + pkg_id = ".".join(components[:-i]) + if pkg_id not in self.ns_ancestors and self.fscache.isdir(path): + self.ns_ancestors[pkg_id] = path + path = os.path.dirname(path) + + def _can_find_module_in_parent_dir(self, id: str) -> bool: + """Test if a module can be found by checking the parent directories + of the current working directory. + """ + working_dir = os.getcwd() + parent_search = FindModuleCache( + SearchPaths((), (), (), ()), + self.fscache, + self.options, + stdlib_py_versions=self.stdlib_py_versions, + ) + while any(is_init_file(file) for file in os.listdir(working_dir)): + working_dir = os.path.dirname(working_dir) + parent_search.search_paths = SearchPaths((working_dir,), (), (), ()) + if not isinstance(parent_search._find_module(id, False), ModuleNotFoundReason): + return True + return False + + def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: + fscache = self.fscache + + # Fast path for any modules in the current source set. + # This is particularly important when there are a large number of search + # paths which share the first (few) component(s) due to the use of namespace + # packages, for instance: + # foo/ + # company/ + # __init__.py + # foo/ + # bar/ + # company/ + # __init__.py + # bar/ + # baz/ + # company/ + # __init__.py + # baz/ + # + # mypy gets [foo/company/foo, bar/company/bar, baz/company/baz, ...] as input + # and computes [foo, bar, baz, ...] as the module search path. + # + # This would result in O(n) search for every import of company.*, leading to + # O(n**2) behavior in load_graph as such imports are unsurprisingly present + # at least once, and usually many more times than that, in each and every file + # being parsed. + # + # Thankfully, such cases are efficiently handled by looking up the module path + # via BuildSourceSet. + p = ( + self.find_module_via_source_set(id) + if (self.options is not None and self.options.fast_module_lookup) + else None + ) + if p: + return p + + # If we're looking for a module like 'foo.bar.baz', it's likely that most of the + # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover + # that only once and cache it for when we look for modules like 'foo.bar.blah' + # that will require the same subdirectory. + components = id.split(".") + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + + # We have two sets of folders so that we collect *all* stubs folders and + # put them in the front of the search path + third_party_inline_dirs: PackageDirs = [] + third_party_stubs_dirs: PackageDirs = [] + found_possible_third_party_missing_type_hints = False + # Third-party stub/typed packages + candidate_package_dirs = { + package_dir[0] + for component in (components[0], components[0] + "-stubs") + for package_dir in self.find_lib_path_dirs(component, self.search_paths.package_path) + } + for pkg_dir in self.search_paths.package_path: + if pkg_dir not in candidate_package_dirs: + continue + stub_name = components[0] + "-stubs" + stub_dir = os_path_join(pkg_dir, stub_name) + if fscache.isdir(stub_dir): + stub_typed_file = os_path_join(stub_dir, "py.typed") + stub_components = [stub_name] + components[1:] + path = os.path.join(pkg_dir, *stub_components[:-1]) + if fscache.isdir(path): + if fscache.isfile(stub_typed_file): + # Stub packages can have a py.typed file, which must include + # 'partial\n' to make the package partial + # Partial here means that mypy should look at the runtime + # package if installed. + if fscache.read(stub_typed_file).decode().strip() == "partial": + runtime_path = os_path_join(pkg_dir, dir_chain) + third_party_inline_dirs.append((runtime_path, True)) + # if the package is partial, we don't verify the module, as + # the partial stub package may not have a __init__.pyi + third_party_stubs_dirs.append((path, False)) + else: + # handle the edge case where people put a py.typed file + # in a stub package, but it isn't partial + third_party_stubs_dirs.append((path, True)) + else: + third_party_stubs_dirs.append((path, True)) + non_stub_match = self._find_module_non_stub_helper(id, pkg_dir) + if isinstance(non_stub_match, ModuleNotFoundReason): + if non_stub_match is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: + found_possible_third_party_missing_type_hints = True + else: + third_party_inline_dirs.append(non_stub_match) + self._update_ns_ancestors(components, non_stub_match) + + if self.options and self.options.use_builtins_fixtures: + # Everything should be in fixtures. + third_party_inline_dirs.clear() + third_party_stubs_dirs.clear() + found_possible_third_party_missing_type_hints = False + python_mypy_path = self.search_paths.mypy_path + self.search_paths.python_path + candidate_base_dirs = self.find_lib_path_dirs(id, python_mypy_path) + if use_typeshed: + # Search for stdlib stubs in typeshed before installed + # stubs to avoid picking up backports (dataclasses, for + # example) when the library is included in stdlib. + candidate_base_dirs += self.find_lib_path_dirs(id, self.search_paths.typeshed_path) + candidate_base_dirs += third_party_stubs_dirs + third_party_inline_dirs + + # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now + # contains just the subdirectories 'foo/bar' that actually exist under the + # elements of lib_path. This is probably much shorter than lib_path itself. + # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. + seplast = os.sep + components[-1] # so e.g. '/baz' + sepinit = os.sep + "__init__" + near_misses = [] # Collect near misses for namespace mode (see below). + for base_dir, verify in candidate_base_dirs: + base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + has_init = False + dir_prefix = base_dir + for _ in range(len(components) - 1): + dir_prefix = os.path.dirname(dir_prefix) + # Prefer package over module, i.e. baz/__init__.py* over baz.py*. + for extension in PYTHON_EXTENSIONS: + path = base_path + sepinit + extension + path_stubs = base_path + "-stubs" + sepinit + extension + if fscache.isfile_case(path, dir_prefix): + has_init = True + if verify and not verify_module(fscache, id, path, dir_prefix): + near_misses.append((path, dir_prefix)) + continue + return path + elif fscache.isfile_case(path_stubs, dir_prefix): + if verify and not verify_module(fscache, id, path_stubs, dir_prefix): + near_misses.append((path_stubs, dir_prefix)) + continue + return path_stubs + + # In namespace mode, register a potential namespace package + if self.options and self.options.namespace_packages: + if ( + not has_init + and fscache.exists_case(base_path, dir_prefix) + and not fscache.isfile_case(base_path, dir_prefix) + ): + near_misses.append((base_path, dir_prefix)) + + # No package, look for module. + for extension in PYTHON_EXTENSIONS: + path = base_path + extension + if fscache.isfile_case(path, dir_prefix): + if verify and not verify_module(fscache, id, path, dir_prefix): + near_misses.append((path, dir_prefix)) + continue + return path + + # In namespace mode, re-check those entries that had 'verify'. + # Assume search path entries xxx, yyy and zzz, and we're + # looking for foo.bar.baz. Suppose near_misses has: + # + # - xxx/foo/bar/baz.py + # - yyy/foo/bar/baz/__init__.py + # - zzz/foo/bar/baz.pyi + # + # If any of the foo directories has __init__.py[i], it wins. + # Else, we look for foo/bar/__init__.py[i], etc. If there are + # none, the first hit wins. Note that this does not take into + # account whether the lowest-level module is a file (baz.py), + # a package (baz/__init__.py), or a stub file (baz.pyi) -- for + # these the first one encountered along the search path wins. + # + # The helper function highest_init_level() returns an int that + # indicates the highest level at which a __init__.py[i] file + # is found; if no __init__ was found it returns 0, if we find + # only foo/bar/__init__.py it returns 1, and if we have + # foo/__init__.py it returns 2 (regardless of what's in + # foo/bar). It doesn't look higher than that. + if self.options and self.options.namespace_packages and near_misses: + levels = [ + highest_init_level(fscache, id, path, dir_prefix) + for path, dir_prefix in near_misses + ] + index = levels.index(max(levels)) + return near_misses[index][0] + + # Finally, we may be asked to produce an ancestor for an + # installed package with a py.typed marker that is a + # subpackage of a namespace package. We only fess up to these + # if we would otherwise return "not found". + ancestor = self.ns_ancestors.get(id) + if ancestor is not None: + return ancestor + + approved_dist_name = stub_distribution_name(id) + if approved_dist_name: + if len(components) == 1: + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + # If we're a missing submodule of an already installed approved stubs, we don't want to + # error with APPROVED_STUBS_NOT_INSTALLED, but rather want to return NOT_FOUND. + for i in range(1, len(components)): + parent_id = ".".join(components[:i]) + if stub_distribution_name(parent_id) == approved_dist_name: + break + else: + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + if self.find_module(parent_id) is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + return ModuleNotFoundReason.NOT_FOUND + + if found_possible_third_party_missing_type_hints: + return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS + return ModuleNotFoundReason.NOT_FOUND + + def find_modules_recursive(self, module: str) -> list[BuildSource]: + module_path = self.find_module(module, fast_path=True) + if isinstance(module_path, ModuleNotFoundReason): + return [] + sources = [BuildSource(module_path, module, None)] + + package_path = None + if is_init_file(module_path): + package_path = os.path.dirname(module_path) + elif self.fscache.isdir(module_path): + package_path = module_path + if package_path is None: + return sources + + # This logic closely mirrors that in find_sources. One small but important difference is + # that we do not sort names with keyfunc. The recursive call to find_modules_recursive + # calls find_module, which will handle the preference between packages, pyi and py. + # Another difference is it doesn't handle nested search paths / package roots. + + seen: set[str] = set() + names = sorted(self.fscache.listdir(package_path)) + for name in names: + # Skip certain names altogether + if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): + continue + subpath = os_path_join(package_path, name) + + if self.options and matches_exclude( + subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2 + ): + continue + + if self.fscache.isdir(subpath): + # Only recurse into packages + if (self.options and self.options.namespace_packages) or ( + self.fscache.isfile(os_path_join(subpath, "__init__.py")) + or self.fscache.isfile(os_path_join(subpath, "__init__.pyi")) + ): + seen.add(name) + sources.extend(self.find_modules_recursive(module + "." + name)) + else: + stem, suffix = os.path.splitext(name) + if stem == "__init__": + continue + if stem not in seen and "." not in stem and suffix in PYTHON_EXTENSIONS: + # (If we sorted names by keyfunc) we could probably just make the BuildSource + # ourselves, but this ensures compatibility with find_module / the cache + seen.add(stem) + sources.extend(self.find_modules_recursive(module + "." + stem)) + return sources + + +def matches_exclude( + subpath: str, excludes: list[str], fscache: FileSystemCache, verbose: bool +) -> bool: + if not excludes: + return False + subpath_str = os.path.relpath(subpath).replace(os.sep, "/") + if fscache.isdir(subpath): + subpath_str += "/" + for exclude in excludes: + if re.search(exclude, subpath_str): + if verbose: + print( + f"TRACE: Excluding {subpath_str} (matches pattern {exclude})", file=sys.stderr + ) + return True + return False + + +def is_init_file(path: str) -> bool: + return os.path.basename(path) in ("__init__.py", "__init__.pyi") + + +def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool: + """Check that all packages containing id have a __init__ file.""" + if is_init_file(path): + path = os.path.dirname(path) + for i in range(id.count(".")): + path = os.path.dirname(path) + if not any( + fscache.isfile_case(os_path_join(path, f"__init__{extension}"), prefix) + for extension in PYTHON_EXTENSIONS + ): + return False + return True + + +def highest_init_level(fscache: FileSystemCache, id: str, path: str, prefix: str) -> int: + """Compute the highest level where an __init__ file is found.""" + if is_init_file(path): + path = os.path.dirname(path) + level = 0 + for i in range(id.count(".")): + path = os.path.dirname(path) + if any( + fscache.isfile_case(os_path_join(path, f"__init__{extension}"), prefix) + for extension in PYTHON_EXTENSIONS + ): + level = i + 1 + return level + + +def mypy_path() -> list[str]: + path_env = os.getenv("MYPYPATH") + if not path_env: + return [] + return path_env.split(os.pathsep) + + +def default_lib_path( + data_dir: str, pyversion: tuple[int, int], custom_typeshed_dir: str | None +) -> list[str]: + """Return default standard library search paths. Guaranteed to be normalised.""" + + data_dir = os.path.abspath(data_dir) + path: list[str] = [] + + if custom_typeshed_dir: + custom_typeshed_dir = os.path.abspath(custom_typeshed_dir) + typeshed_dir = os.path.join(custom_typeshed_dir, "stdlib") + mypy_extensions_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-extensions") + versions_file = os.path.join(typeshed_dir, "VERSIONS") + if not os.path.isdir(typeshed_dir) or not os.path.isfile(versions_file): + print( + "error: --custom-typeshed-dir does not point to a valid typeshed ({})".format( + custom_typeshed_dir + ) + ) + sys.exit(2) + else: + auto = os.path.join(data_dir, "stubs-auto") + if os.path.isdir(auto): + data_dir = auto + typeshed_dir = os.path.join(data_dir, "typeshed", "stdlib") + mypy_extensions_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-extensions") + path.append(typeshed_dir) + + # Get mypy-extensions stubs from typeshed, since we treat it as an + # "internal" library, similar to typing and typing-extensions. + path.append(mypy_extensions_dir) + + # Add fallback path that can be used if we have a broken installation. + if sys.platform != "win32": + path.append("/usr/local/lib/mypy") + if not path: + print( + "Could not resolve typeshed subdirectories. Your mypy install is broken.\n" + "Python executable is located at {}.\nMypy located at {}".format( + sys.executable, data_dir + ), + file=sys.stderr, + ) + sys.exit(1) + return path + + +@functools.lru_cache(maxsize=None) +def get_search_dirs(python_executable: str | None) -> tuple[list[str], list[str]]: + """Find package directories for given python. Guaranteed to return absolute paths. + + This runs a subprocess call, which generates a list of the directories in sys.path. + To avoid repeatedly calling a subprocess (which can be slow!) we + lru_cache the results. + """ + + if python_executable is None: + return ([], []) + elif python_executable == sys.executable: + # Use running Python's package dirs + sys_path, site_packages = pyinfo.getsearchdirs() + else: + # Use subprocess to get the package directory of given Python + # executable + env = {**dict(os.environ), "PYTHONSAFEPATH": "1"} + try: + sys_path, site_packages = ast.literal_eval( + subprocess.check_output( + [python_executable, pyinfo.__file__, "getsearchdirs"], + env=env, + stderr=subprocess.PIPE, + ).decode() + ) + except subprocess.CalledProcessError as err: + print(err.stderr) + print(err.stdout) + raise + except OSError as err: + assert err.errno is not None + reason = os.strerror(err.errno) + raise CompileError( + [f"mypy: Invalid python executable '{python_executable}': {reason}"] + ) from err + return sys_path, site_packages + + +def compute_search_paths( + sources: list[BuildSource], options: Options, data_dir: str, alt_lib_path: str | None = None +) -> SearchPaths: + """Compute the search paths as specified in PEP 561. + + There are the following 4 members created: + - User code (from `sources`) + - MYPYPATH (set either via config or environment variable) + - installed package directories (which will later be split into stub-only and inline) + - typeshed + """ + # Determine the default module search path. + lib_path = collections.deque( + default_lib_path( + data_dir, options.python_version, custom_typeshed_dir=options.custom_typeshed_dir + ) + ) + + if options.use_builtins_fixtures: + # Use stub builtins (to speed up test cases and to make them easier to + # debug). This is a test-only feature, so assume our files are laid out + # as in the source tree. + # We also need to allow overriding where to look for it. Argh. + root_dir = os.getenv("MYPY_TEST_PREFIX", None) + if not root_dir: + root_dir = os.path.dirname(os.path.dirname(__file__)) + root_dir = os.path.abspath(root_dir) + lib_path.appendleft(os.path.join(root_dir, "test-data", "unit", "lib-stub")) + # alt_lib_path is used by some tests to bypass the normal lib_path mechanics. + # If we don't have one, grab directories of source files. + python_path: list[str] = [] + if not alt_lib_path: + for source in sources: + # Include directory of the program file in the module search path. + if source.base_dir: + dir = source.base_dir + if dir not in python_path: + python_path.append(dir) + + # Do this even if running as a file, for sanity (mainly because with + # multiple builds, there could be a mix of files/modules, so its easier + # to just define the semantics that we always add the current director + # to the lib_path + # TODO: Don't do this in some cases; for motivation see see + # https://github.com/python/mypy/issues/4195#issuecomment-341915031 + if options.bazel: + dir = "." + else: + dir = os.getcwd() + if dir not in lib_path: + python_path.insert(0, dir) + + # Start with a MYPYPATH environment variable at the front of the mypy_path, if defined. + mypypath = mypy_path() + + # Add a config-defined mypy path. + mypypath.extend(options.mypy_path) + + # If provided, insert the caller-supplied extra module path to the + # beginning (highest priority) of the search path. + if alt_lib_path: + mypypath.insert(0, alt_lib_path) + + sys_path, site_packages = get_search_dirs(options.python_executable) + # We only use site packages for this check + for site in site_packages: + assert site not in lib_path + if ( + site in mypypath + or any(p.startswith(site + os.path.sep) for p in mypypath) + or (os.path.altsep and any(p.startswith(site + os.path.altsep) for p in mypypath)) + ): + print(f"{site} is in the MYPYPATH. Please remove it.", file=sys.stderr) + print( + "See https://mypy.readthedocs.io/en/stable/running_mypy.html" + "#how-mypy-handles-imports for more info", + file=sys.stderr, + ) + sys.exit(1) + + return SearchPaths( + python_path=tuple(reversed(python_path)), + mypy_path=tuple(mypypath), + package_path=tuple(sys_path + site_packages), + typeshed_path=tuple(lib_path), + ) + + +def load_stdlib_py_versions(custom_typeshed_dir: str | None) -> StdlibVersions: + """Return dict with minimum and maximum Python versions of stdlib modules. + + The contents look like + {..., 'secrets': ((3, 6), None), 'symbol': ((2, 7), (3, 9)), ...} + + None means there is no maximum version. + """ + typeshed_dir = custom_typeshed_dir or os_path_join(os.path.dirname(__file__), "typeshed") + stdlib_dir = os_path_join(typeshed_dir, "stdlib") + result = {} + + versions_path = os_path_join(stdlib_dir, "VERSIONS") + assert os.path.isfile(versions_path), (custom_typeshed_dir, versions_path, __file__) + with open(versions_path) as f: + for line in f: + line = line.split("#")[0].strip() + if line == "": + continue + module, version_range = line.split(":") + versions = version_range.split("-") + min_version = parse_version(versions[0]) + max_version = ( + parse_version(versions[1]) if len(versions) >= 2 and versions[1].strip() else None + ) + result[module] = min_version, max_version + return result + + +def parse_version(version: str) -> tuple[int, int]: + major, minor = version.strip().split(".") + return int(major), int(minor) + + +def typeshed_py_version(options: Options) -> tuple[int, int]: + """Return Python version used for checking whether module supports typeshed.""" + # Typeshed no longer covers Python 3.x versions before 3.8, so 3.8 is + # the earliest we can support. + return max(options.python_version, (3, 8)) diff --git a/mypy/moduleinspect.py b/mypy/moduleinspect.py new file mode 100644 index 000000000000..35db2132f66c --- /dev/null +++ b/mypy/moduleinspect.py @@ -0,0 +1,184 @@ +"""Basic introspection of modules.""" + +from __future__ import annotations + +import importlib +import inspect +import os +import pkgutil +import queue +import sys +from multiprocessing import Queue, get_context +from types import ModuleType + + +class ModuleProperties: + # Note that all __init__ args must have default values + def __init__( + self, + name: str = "", + file: str | None = None, + path: list[str] | None = None, + all: list[str] | None = None, + is_c_module: bool = False, + subpackages: list[str] | None = None, + ) -> None: + self.name = name # __name__ attribute + self.file = file # __file__ attribute + self.path = path # __path__ attribute + self.all = all # __all__ attribute + self.is_c_module = is_c_module + self.subpackages = subpackages or [] + + +def is_c_module(module: ModuleType) -> bool: + if module.__dict__.get("__file__") is None: + # Could be a namespace package. These must be handled through + # introspection, since there is no source file. + return True + return os.path.splitext(module.__dict__["__file__"])[-1] in [".so", ".pyd", ".dll"] + + +def is_pyc_only(file: str | None) -> bool: + return bool(file and file.endswith(".pyc") and not os.path.exists(file[:-1])) + + +class InspectError(Exception): + pass + + +def get_package_properties(package_id: str) -> ModuleProperties: + """Use runtime introspection to get information about a module/package.""" + try: + package = importlib.import_module(package_id) + except BaseException as e: + raise InspectError(str(e)) from e + name = getattr(package, "__name__", package_id) + file = getattr(package, "__file__", None) + path: list[str] | None = getattr(package, "__path__", None) + if not isinstance(path, list): + path = None + pkg_all = getattr(package, "__all__", None) + if pkg_all is not None: + try: + pkg_all = list(pkg_all) + except Exception: + pkg_all = None + is_c = is_c_module(package) + + if path is None: + # Object has no path; this means it's either a module inside a package + # (and thus no sub-packages), or it could be a C extension package. + if is_c: + # This is a C extension module, now get the list of all sub-packages + # using the inspect module + subpackages = [ + package.__name__ + "." + name + for name, val in inspect.getmembers(package) + if inspect.ismodule(val) and val.__name__ == package.__name__ + "." + name + ] + else: + # It's a module inside a package. There's nothing else to walk/yield. + subpackages = [] + else: + all_packages = pkgutil.walk_packages( + path, prefix=package.__name__ + ".", onerror=lambda r: None + ) + subpackages = [qualified_name for importer, qualified_name, ispkg in all_packages] + return ModuleProperties( + name=name, file=file, path=path, all=pkg_all, is_c_module=is_c, subpackages=subpackages + ) + + +def worker(tasks: Queue[str], results: Queue[str | ModuleProperties], sys_path: list[str]) -> None: + """The main loop of a worker introspection process.""" + sys.path = sys_path + while True: + mod = tasks.get() + try: + prop = get_package_properties(mod) + except InspectError as e: + results.put(str(e)) + continue + results.put(prop) + + +class ModuleInspect: + """Perform runtime introspection of modules in a separate process. + + Reuse the process for multiple modules for efficiency. However, if there is an + error, retry using a fresh process to avoid cross-contamination of state between + modules. + + We use a separate process to isolate us from many side effects. For example, the + import of a module may kill the current process, and we want to recover from that. + + Always use in a with statement for proper clean-up: + + with ModuleInspect() as m: + p = m.get_package_properties('urllib.parse') + """ + + def __init__(self) -> None: + self._start() + + def _start(self) -> None: + if sys.platform == "linux": + ctx = get_context("forkserver") + else: + ctx = get_context("spawn") + self.tasks: Queue[str] = ctx.Queue() + self.results: Queue[ModuleProperties | str] = ctx.Queue() + self.proc = ctx.Process(target=worker, args=(self.tasks, self.results, sys.path)) + self.proc.start() + self.counter = 0 # Number of successful roundtrips + + def close(self) -> None: + """Free any resources used.""" + self.proc.terminate() + + def get_package_properties(self, package_id: str) -> ModuleProperties: + """Return some properties of a module/package using runtime introspection. + + Raise InspectError if the target couldn't be imported. + """ + self.tasks.put(package_id) + res = self._get_from_queue() + if res is None: + # The process died; recover and report error. + self._start() + raise InspectError(f"Process died when importing {package_id!r}") + if isinstance(res, str): + # Error importing module + if self.counter > 0: + # Also try with a fresh process. Maybe one of the previous imports has + # corrupted some global state. + self.close() + self._start() + return self.get_package_properties(package_id) + raise InspectError(res) + self.counter += 1 + return res + + def _get_from_queue(self) -> ModuleProperties | str | None: + """Get value from the queue. + + Return the value read from the queue, or None if the process unexpectedly died. + """ + max_iter = 600 + n = 0 + while True: + if n == max_iter: + raise RuntimeError("Timeout waiting for subprocess") + try: + return self.results.get(timeout=0.05) + except queue.Empty: + if not self.proc.is_alive(): + return None + n += 1 + + def __enter__(self) -> ModuleInspect: + return self + + def __exit__(self, *args: object) -> None: + self.close() diff --git a/mypy/mro.py b/mypy/mro.py new file mode 100644 index 000000000000..f34f3fa0c46d --- /dev/null +++ b/mypy/mro.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import TypeInfo +from mypy.types import Instance +from mypy.typestate import type_state + + +def calculate_mro(info: TypeInfo, obj_type: Callable[[], Instance] | None = None) -> None: + """Calculate and set mro (method resolution order). + + Raise MroError if cannot determine mro. + """ + mro = linearize_hierarchy(info, obj_type) + assert mro, f"Could not produce a MRO at all for {info}" + info.mro = mro + # The property of falling back to Any is inherited. + info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in info.mro) + type_state.reset_all_subtype_caches_for(info) + + +class MroError(Exception): + """Raised if a consistent mro cannot be determined for a class.""" + + +def linearize_hierarchy( + info: TypeInfo, obj_type: Callable[[], Instance] | None = None +) -> list[TypeInfo]: + # TODO describe + if info.mro: + return info.mro + bases = info.direct_base_classes() + if not bases and info.fullname != "builtins.object" and obj_type is not None: + # Probably an error, add a dummy `object` base class, + # otherwise MRO calculation may spuriously fail. + bases = [obj_type().type] + lin_bases = [] + for base in bases: + assert base is not None, f"Cannot linearize bases for {info.fullname} {bases}" + lin_bases.append(linearize_hierarchy(base, obj_type)) + lin_bases.append(bases) + return [info] + merge(lin_bases) + + +def merge(seqs: list[list[TypeInfo]]) -> list[TypeInfo]: + seqs = [s.copy() for s in seqs] + result: list[TypeInfo] = [] + while True: + seqs = [s for s in seqs if s] + if not seqs: + return result + for seq in seqs: + head = seq[0] + if not [s for s in seqs if head in s[1:]]: + break + else: + raise MroError() + result.append(head) + for s in seqs: + if s[0] is head: + del s[0] diff --git a/mypy/myunit.py b/mypy/myunit.py deleted file mode 100644 index 58215569dfeb..000000000000 --- a/mypy/myunit.py +++ /dev/null @@ -1,310 +0,0 @@ -import sys -import re -import time -import traceback - -from typing import List, Tuple, Any, Function, overload, Undefined - - -# TODO remove global state -is_verbose = False -is_quiet = False -patterns = List[str]() -times = List[Tuple[float, str]]() - - -class AssertionFailure(Exception): - """Exception used to signal skipped test cases.""" - def __init__(self, s: str = None) -> None: - if s: - super().__init__(s) - else: - super().__init__() - - -class SkipTestCaseException(Exception): pass - - -def assert_true(b: bool, msg: str = None) -> None: - if not b: - raise AssertionFailure(msg) - - -def assert_false(b: bool, msg: str = None) -> None: - if b: - raise AssertionFailure(msg) - - -def assert_equal(a: object, b: object, fmt: str = '{} != {}') -> None: - if a != b: - raise AssertionFailure(fmt.format(repr(a), repr(b))) - - -def assert_not_equal(a: object, b: object, fmt: str = '{} == {}') -> None: - if a == b: - raise AssertionFailure(fmt.format(repr(a), repr(b))) - - -def assert_raises(typ: type, *rest: Any) -> None: - """Usage: assert_raises(exception class[, message], function[, args]) - - Call function with the given arguments and expect an exception of the given - type. - - TODO use overloads for better type checking - """ - # Parse arguments. - msg = None # type: str - if isinstance(rest[0], str) or rest[0] is None: - msg = rest[0] - rest = rest[1:] - f = rest[0] - args = [] # type: List[Any] - if len(rest) > 1: - args = rest[1] - assert len(rest) <= 2 - - # Perform call and verify the exception. - try: - f(*args) - except Exception as e: - assert_type(typ, e) - if msg: - assert_equal(e.args[0], msg, 'Invalid message {}, expected {}') - else: - raise AssertionFailure('No exception raised') - - -def assert_type(typ: type, value: object) -> None: - if type(value) != typ: - raise AssertionFailure('Invalid type {}, expected {}'.format( - typename(type(value)), typename(typ))) - - -def fail() -> None: - raise AssertionFailure() - - -class TestCase: - def __init__(self, name: str, suite: 'Suite' = None, - func: Function[[], None] = None) -> None: - self.func = func - self.name = name - self.suite = suite - - def run(self) -> None: - if self.func: - self.func() - - def set_up(self) -> None: - if self.suite: - self.suite.set_up() - - def tear_down(self) -> None: - if self.suite: - self.suite.tear_down() - - -class Suite: - def __init__(self) -> None: - self.prefix = typename(type(self)) + '.' - # Each test case is either a TestCase object or (str, function). - self._test_cases = [] # type: List[Any] - self.init() - - def set_up(self) -> None: - pass - - def tear_down(self) -> None: - pass - - def init(self) -> None: - for m in dir(self): - if m.startswith('test'): - t = getattr(self, m) - if isinstance(t, Suite): - self.add_test((m + '.', t)) - else: - self.add_test(TestCase(m, self, getattr(self, m))) - - @overload - def add_test(self, test: TestCase) -> None: - self._test_cases.append(test) - - @overload - def add_test(self, test: Tuple[str, Function[[], None]]) -> None: - self._test_cases.append(test) - - @overload - def add_test(self, test: Tuple[str, 'Suite']) -> None: - self._test_cases.append(test) - - def cases(self) -> List[Any]: - return self._test_cases[:] - - def skip(self) -> None: - raise SkipTestCaseException() - - -def run_test(t: Suite, args: List[str] = None) -> None: - global patterns, is_verbose, is_quiet - if not args: - args = [] - is_verbose = False - is_quiet = False - patterns = [] - i = 0 - while i < len(args): - a = args[i] - if a == '-v': - is_verbose = True - elif a == '-q': - is_quiet = True - # Used for updating tests, but breaks clean code usage. - elif a in ('-u', '-i'): - pass - elif len(a) > 0 and a[0] != '-': - patterns.append(a) - else: - raise ValueError('Invalid arguments') - i += 1 - if len(patterns) == 0: - patterns.append('*') - - num_total, num_fail, num_skip = run_test_recursive(t, 0, 0, 0, '', 0) - - skip_msg = '' - if num_skip > 0: - skip_msg = ', {} skipped'.format(num_skip) - - if num_fail == 0: - if not is_quiet: - print('%d test cases run%s, all passed.' % (num_total, skip_msg)) - print('*** OK ***') - else: - sys.stderr.write('%d/%d test cases failed%s.\n' % (num_fail, - num_total, - skip_msg)) - sys.stderr.write('*** FAILURE ***\n') - sys.exit(1) - - -def run_test_recursive(test: Any, num_total: int, num_fail: int, num_skip: int, - prefix: str, depth: int) -> Tuple[int, int, int]: - """The first argument may be TestCase, Suite or (str, Suite).""" - if isinstance(test, TestCase): - name = prefix + test.name - for pattern in patterns: - if match_pattern(name, pattern): - match = True - break - else: - match = False - if match: - is_fail, is_skip = run_single_test(name, test) - if is_fail: num_fail += 1 - if is_skip: num_skip += 1 - num_total += 1 - else: - suite = Undefined # type: Suite - suite_prefix = '' - if isinstance(test, list) or isinstance(test, tuple): - suite = test[1] - suite_prefix = test[0] - else: - suite = test - suite_prefix = test.prefix - - for stest in suite.cases(): - new_prefix = prefix - if depth > 0: - new_prefix = prefix + suite_prefix - num_total, num_fail, num_skip = run_test_recursive( - stest, num_total, num_fail, num_skip, new_prefix, depth + 1) - return num_total, num_fail, num_skip - - -def run_single_test(name: str, test: Any) -> Tuple[bool, bool]: - if is_verbose: - sys.stderr.write(name) - sys.stderr.flush() - - time0 = time.time() - test.set_up() # FIX: check exceptions - exc_traceback = None # type: Any - try: - test.run() - except Exception: - exc_type, exc_value, exc_traceback = sys.exc_info() - test.tear_down() # FIX: check exceptions - times.append((time.time() - time0, name)) - - if exc_traceback: - if isinstance(exc_value, SkipTestCaseException): - if is_verbose: - sys.stderr.write(' (skipped)\n') - return False, True - else: - handle_failure(name, exc_type, exc_value, exc_traceback) - return True, False - elif is_verbose: - sys.stderr.write('\n') - - return False, False - - -def handle_failure(name, exc_type, exc_value, exc_traceback) -> None: - # Report failed test case. - if is_verbose: - sys.stderr.write('\n\n') - msg = '' - if exc_value.args and exc_value.args[0]: - msg = ': ' + str(exc_value) - else: - msg = '' - sys.stderr.write('Traceback (most recent call last):\n') - tb = traceback.format_tb(exc_traceback) - tb = clean_traceback(tb) - for s in tb: - sys.stderr.write(s) - exception = typename(exc_type) - sys.stderr.write('{}{}\n\n'.format(exception, msg)) - sys.stderr.write('{} failed\n\n'.format(name)) - - -def typename(t: type) -> str: - if '.' in str(t): - return str(t).split('.')[-1].rstrip("'>") - else: - return str(t)[8:-2] - - -def match_pattern(s: str, p: str) -> bool: - if len(p) == 0: - return len(s) == 0 - elif p[0] == '*': - if len(p) == 1: - return True - else: - for i in range(len(s) + 1): - if match_pattern(s[i:], p[1:]): - return True - return False - elif len(s) == 0: - return False - else: - return s[0] == p[0] and match_pattern(s[1:], p[1:]) - - -def clean_traceback(tb: List[str]) -> List[str]: - # Remove clutter from the traceback. - start = 0 - for i, s in enumerate(tb): - if '\n test.run()\n' in s or '\n self.func()\n' in s: - start = i + 1 - tb = tb[start:] - for f in ['assert_equal', 'assert_not_equal', 'assert_type', - 'assert_raises', 'assert_true']: - if tb != [] and ', in {}\n'.format(f) in tb[-1]: - tb = tb[:-1] - return tb diff --git a/mypy/noderepr.py b/mypy/noderepr.py deleted file mode 100644 index b3a704496157..000000000000 --- a/mypy/noderepr.py +++ /dev/null @@ -1,376 +0,0 @@ -"""Classes for storing the lexical token information of nodes. - -This is used for outputting the original source code represented by the nodes -(including original formatting and comments). - -Each node representation usually only contains tokens directly associated -with that node (terminals). All members are Tokens or lists of Tokens, -unless explicitly mentioned otherwise. - -If a representation has a Break token, the member name is br. -""" - -from typing import Any, List, Tuple, Undefined - -from mypy.lex import Token - - -class MypyFileRepr: - def __init__(self, eof): - self.eof = eof - - -class ImportRepr: - def __init__(self, import_tok: Any, components: List[List[Token]], - as_names: List[Tuple[Token, Token]], commas: List[Token], - br: Any) -> None: - self.import_tok = import_tok - self.components = components - self.as_names = as_names - self.commas = commas - self.br = br - - -class ImportFromRepr: - def __init__(self, - from_tok: Any, - rel_toks: Any, - components: List[Token], - import_tok: Any, - lparen: Any, - names: List[Tuple[List[Token], Token]], - rparen: Any, br: Any) -> None: - # Notes: - # - lparen, rparen, and rel_tok may be empty - # - in each names tuple, the first item contains tokens for - # 'name [as name]' and the second item is a comma or empty. - self.from_tok = from_tok - self.rel_toks = rel_toks - self.components = components - self.import_tok = import_tok - self.lparen = lparen - self.names = names - self.rparen = rparen - self.br = br - - -class FuncRepr: - def __init__(self, def_tok: Any, name: Any, args: 'FuncArgsRepr') -> None: - # Note: name may be empty. - self.def_tok = def_tok - self.name = name - self.args = args - - -class FuncArgsRepr: - """Representation of a set of function arguments.""" - def __init__(self, lseparator: Any, rseparator: Any, arg_names: Any, - commas: Any, assigns: Any, asterisk: Any) -> None: - # Lseparator and rseparator are '(' and ')', respectively. - self.lseparator = lseparator - self.rseparator = rseparator - self.arg_names = arg_names - self.commas = commas - self.assigns = assigns - self.asterisk = asterisk - - -class VarRepr: - def __init__(self, name: Any, comma: Any) -> None: - # Note_ comma may be empty. - self.name = name - self.comma = comma - - -class TypeDefRepr: - def __init__(self, class_tok: Any, name: Any, lparen: Any, commas: Any, - rparen: Any) -> None: - self.class_tok = class_tok - self.name = name - self.lparen = lparen - self.commas = commas - self.rparen = rparen - - -class VarDefRepr: - def __init__(self, assign: Any, br: Any) -> None: - # Note: assign may be empty. - self.assign = assign - self.br = br - - -class DecoratorRepr: - def __init__(self, ats: Any, brs: Any) -> None: - self.ats = ats - self.brs = brs - - -class BlockRepr: - def __init__(self, colon: Any, br: Any, indent: Any, dedent: Any) -> None: - self.colon = colon - self.br = br - self.indent = indent - self.dedent = dedent - - -class GlobalDeclRepr: - def __init__(self, global_tok: Any, names: List[Token], - commas: List[Token], br: Any) -> None: - self.global_tok = global_tok - self.names = names - self.commas = commas - self.br = br - - -class NonlocalDeclRepr: - def __init__(self, nonlocal_tok: Any, names: List[Token], - commas: List[Token], br: Any) -> None: - self.nonlocal_tok = nonlocal_tok - self.names = names - self.commas = commas - self.br = br - - -class ExpressionStmtRepr: - def __init__(self, br: Any) -> None: - self.br = br - - -class AssignmentStmtRepr: - def __init__(self, assigns: List[Token], br: Any) -> None: - self.assigns = assigns - self.br = br - - -class OperatorAssignmentStmtRepr: - def __init__(self, assign: Any, br: Any) -> None: - self.assign = assign - self.br = br - - -class WhileStmtRepr: - def __init__(self, while_tok: Any, else_tok: Any) -> None: - self.while_tok = while_tok - self.else_tok = else_tok - - -class ForStmtRepr: - def __init__(self, for_tok: Any, in_tok: Any, - else_tok: Any) -> None: - self.for_tok = for_tok - self.in_tok = in_tok - self.else_tok = else_tok - - -class SimpleStmtRepr: - """Representation for break, continue, pass, return and assert.""" - def __init__(self, keyword: Any, br: Any) -> None: - self.keyword = keyword - self.br = br - - -class IfStmtRepr: - def __init__(self, if_tok: Any, elif_toks: Any, else_tok: Any) -> None: - # Note: else_tok may be empty. - self.if_tok = if_tok - self.elif_toks = elif_toks - self.else_tok = else_tok - - -class RaiseStmtRepr: - def __init__(self, raise_tok: Any, from_tok: Any, br: Any) -> None: - self.raise_tok = raise_tok - self.from_tok = from_tok - self.br = br - - -class TryStmtRepr: - def __init__(self, try_tok: Any, except_toks: Any, name_toks: Any, - as_toks: Any, else_tok: Any, finally_tok: Any) -> None: - self.try_tok = try_tok - self.except_toks = except_toks - self.name_toks = name_toks - self.as_toks = as_toks - self.else_tok = else_tok - self.finally_tok = finally_tok - - -class WithStmtRepr: - def __init__(self, with_tok: Any, as_toks: Any, commas: Any) -> None: - self.with_tok = with_tok - self.as_toks = as_toks - self.commas = commas - - -class IntExprRepr: - def __init__(self, int: Any) -> None: - self.int = int - - -class StrExprRepr: - def __init__(self, string: List[Token]) -> None: - self.string = string - - -class FloatExprRepr: - def __init__(self, float: Any) -> None: - self.float = float - - -class ComplexExprRepr: - def __init__(self, complex: Any) -> None: - self.complex = complex - - -class ParenExprRepr: - def __init__(self, lparen: Any, rparen: Any) -> None: - self.lparen = lparen - self.rparen = rparen - - -class StarExprRepr: - def __init__(self, star: Any) -> None: - self.star = star - - -class NameExprRepr: - def __init__(self, id: Any) -> None: - self.id = id - - -class MemberExprRepr: - def __init__(self, dot: Any, name: Any) -> None: - self.dot = dot - self.name = name - -class ComparisonExprRepr: - def __init__(self, operators: List[Any]) -> None: - # List of tupples of (op, op2). - # Note: op2 may be empty; it is used for "is not" and "not in". - self.operators = operators - -class CallExprRepr: - def __init__(self, lparen: Any, commas: List[Token], star: Any, star2: Any, - keywords: List[List[Token]], rparen: Any) -> None: - # Asterisk may be empty. - self.lparen = lparen - self.commas = commas - self.star = star - self.star2 = star2 - self.keywords = keywords - self.rparen = rparen - - -class IndexExprRepr: - def __init__(self, lbracket: Any, rbracket: Any) -> None: - self.lbracket = lbracket - self.rbracket = rbracket - - -class SliceExprRepr: - def __init__(self, colon: Any, colon2: Any) -> None: - self.colon = colon - self.colon2 = colon2 - - -class UnaryExprRepr: - def __init__(self, op: Any) -> None: - self.op = op - - -class OpExprRepr: - def __init__(self, op: Any) -> None: - self.op = op - - -class CastExprRepr: - def __init__(self, lparen: Any, rparen: Any) -> None: - self.lparen = lparen - self.rparen = rparen - - -class FuncExprRepr: - def __init__(self, lambda_tok: Any, colon: Any, args: Any) -> None: - self.lambda_tok = lambda_tok - self.colon = colon - self.args = args - - -class SuperExprRepr: - def __init__(self, super_tok: Any, lparen: Any, rparen: Any, dot: Any, - name: Any) -> None: - self.super_tok = super_tok - self.lparen = lparen - self.rparen = rparen - self.dot = dot - self.name = name - - -class ListSetExprRepr: - # [...] or {...} - def __init__(self, lbracket: Any, commas: List[Token], rbracket: Any, - langle: Any, rangle: Any) -> None: - self.lbracket = lbracket - self.commas = commas - self.rbracket = rbracket - self.langle = langle - self.rangle = rangle - - -class TupleExprRepr: - def __init__(self, lparen: Any, commas: List[Token], rparen: Any) -> None: - # Note: lparen and rparen may be empty. - self.lparen = lparen - self.commas = commas - self.rparen = rparen - - -class DictExprRepr: - def __init__(self, lbrace: Any, colons: List[Token], commas: List[Token], - rbrace: Any, langle: Any, type_comma: Any, - rangle: Any) -> None: - self.lbrace = lbrace - self.colons = colons - self.commas = commas - self.rbrace = rbrace - self.langle = langle - self.type_comma = type_comma - self.rangle = rangle - - -class TypeApplicationRepr: - def __init__(self, langle: Any, commas: Any, rangle: Any) -> None: - self.langle = langle - self.commas = commas - self.rangle = rangle - - -class GeneratorExprRepr: - def __init__(self, for_toks: List[Token], in_toks: List[Token], - if_toklists: List[List[Token]]) -> None: - self.for_toks = for_toks - self.in_toks = in_toks - self.if_toklists = if_toklists - - -class ListComprehensionRepr: - def __init__(self, lbracket: Any, rbracket: Any) -> None: - self.lbracket = lbracket - self.rbracket = rbracket - - -class SetComprehensionRepr: - def __init__(self, lbrace: Any, rbrace: Any) -> None: - self.lbrace = lbrace - self.rbrace = rbrace - - -class DictionaryComprehensionRepr: - def __init__(self, lbrace: Any, colon: Token, for_toks: List[Token], in_toks: List[Token], - if_toklists: List[List[Token]], rbrace: Any) -> None: - self.lbrace = lbrace - self.rbrace = rbrace - self.colon = colon - self.for_toks = for_toks - self.in_toks = in_toks - self.if_toklists = if_toklists diff --git a/mypy/nodes.py b/mypy/nodes.py index afa912e4d917..9e26103e2f58 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1,140 +1,370 @@ """Abstract syntax tree node classes (i.e. parse tree).""" -import os -import re -from abc import abstractmethod, ABCMeta +from __future__ import annotations +import os +from abc import abstractmethod +from collections import defaultdict +from enum import Enum, unique from typing import ( - Any, overload, typevar, Undefined, List, Tuple, cast, Set, Dict + TYPE_CHECKING, + Any, + Callable, + Dict, + Final, + Iterator, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + cast, ) +from typing_extensions import TypeAlias as _TypeAlias, TypeGuard + +from mypy_extensions import trait -from mypy.lex import Token import mypy.strconv -from mypy.visitor import NodeVisitor -from mypy.util import dump_tagged, short_type +from mypy.options import Options +from mypy.util import is_typeshed_file, short_type +from mypy.visitor import ExpressionVisitor, NodeVisitor, StatementVisitor + +if TYPE_CHECKING: + from mypy.patterns import Pattern -class Context(metaclass=ABCMeta): +class Context: """Base type for objects that are valid as error message locations.""" - @abstractmethod - def get_line(self) -> int: pass + __slots__ = ("line", "column", "end_line", "end_column") + + def __init__(self, line: int = -1, column: int = -1) -> None: + self.line = line + self.column = column + self.end_line: int | None = None + self.end_column: int | None = None + + def set_line( + self, + target: Context | int, + column: int | None = None, + end_line: int | None = None, + end_column: int | None = None, + ) -> None: + """If target is a node, pull line (and column) information + into this node. If column is specified, this will override any column + information coming from a node. + """ + if isinstance(target, int): + self.line = target + else: + self.line = target.line + self.column = target.column + self.end_line = target.end_line + self.end_column = target.end_column + + if column is not None: + self.column = column + + if end_line is not None: + self.end_line = end_line + + if end_column is not None: + self.end_column = end_column -import mypy.types +if TYPE_CHECKING: + # break import cycle only needed for mypy + import mypy.types -T = typevar('T') +T = TypeVar("T") -# Variable kind constants +JsonDict: _TypeAlias = Dict[str, Any] + + +# Symbol table node kinds +# # TODO rename to use more descriptive names -LDEF = 0 # type: int -GDEF = 1 # type: int -MDEF = 2 # type: int -MODULE_REF = 3 # type: int -# Type variable declared using typevar(...) has kind UNBOUND_TVAR. It's not -# valid as a type. A type variable is valid as a type (kind TVAR) within -# (1) a generic class that uses the type variable as a type argument or -# (2) a generic function that refers to the type variable in its signature. -UNBOUND_TVAR = 4 # type: 'int' -TVAR = 5 # type: int - - -LITERAL_YES = 2 -LITERAL_TYPE = 1 -LITERAL_NO = 0 - -node_kinds = { - LDEF: 'Ldef', - GDEF: 'Gdef', - MDEF: 'Mdef', - MODULE_REF: 'ModuleRef', - UNBOUND_TVAR: 'UnboundTvar', - TVAR: 'Tvar', +LDEF: Final = 0 +GDEF: Final = 1 +MDEF: Final = 2 + +# Placeholder for a name imported via 'from ... import'. Second phase of +# semantic will replace this the actual imported reference. This is +# needed so that we can detect whether a name has been imported during +# XXX what? +UNBOUND_IMPORTED: Final = 3 + +# RevealExpr node kinds +REVEAL_TYPE: Final = 0 +REVEAL_LOCALS: Final = 1 + +LITERAL_YES: Final = 2 +LITERAL_TYPE: Final = 1 +LITERAL_NO: Final = 0 + +node_kinds: Final = {LDEF: "Ldef", GDEF: "Gdef", MDEF: "Mdef", UNBOUND_IMPORTED: "UnboundImported"} +inverse_node_kinds: Final = {_kind: _name for _name, _kind in node_kinds.items()} + + +implicit_module_attrs: Final = { + "__name__": "__builtins__.str", + "__doc__": None, # depends on Python version, see semanal.py + "__path__": None, # depends on if the module is a package + "__file__": "__builtins__.str", + "__package__": "__builtins__.str", + "__annotations__": None, # dict[str, Any] bounded in add_implicit_module_attrs() + "__spec__": None, # importlib.machinery.ModuleSpec bounded in add_implicit_module_attrs() } -implicit_module_attrs = ['__name__', '__doc__', '__file__'] +# These aliases exist because built-in class objects are not subscriptable. +# For example `list[int]` fails at runtime. Instead List[int] should be used. +type_aliases: Final = { + "typing.List": "builtins.list", + "typing.Dict": "builtins.dict", + "typing.Set": "builtins.set", + "typing.FrozenSet": "builtins.frozenset", + "typing.ChainMap": "collections.ChainMap", + "typing.Counter": "collections.Counter", + "typing.DefaultDict": "collections.defaultdict", + "typing.Deque": "collections.deque", + "typing.OrderedDict": "collections.OrderedDict", + # HACK: a lie in lieu of actual support for PEP 675 + "typing.LiteralString": "builtins.str", +} + +# This keeps track of the oldest supported Python version where the corresponding +# alias source is available. +type_aliases_source_versions: Final = {"typing.LiteralString": (3, 11)} +# This keeps track of aliases in `typing_extensions`, which we treat specially. +typing_extensions_aliases: Final = { + # See: https://github.com/python/mypy/issues/11528 + "typing_extensions.OrderedDict": "collections.OrderedDict", + # HACK: a lie in lieu of actual support for PEP 675 + "typing_extensions.LiteralString": "builtins.str", +} -type_aliases = { - 'typing.List': '__builtins__.list', - 'typing.Dict': '__builtins__.dict', - 'typing.Set': '__builtins__.set', +reverse_builtin_aliases: Final = { + "builtins.list": "typing.List", + "builtins.dict": "typing.Dict", + "builtins.set": "typing.Set", + "builtins.frozenset": "typing.FrozenSet", } -reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) - for alias, name in type_aliases.items()) +_nongen_builtins: Final = {"builtins.tuple": "typing.Tuple", "builtins.enumerate": ""} +_nongen_builtins.update((name, alias) for alias, name in type_aliases.items()) +# Drop OrderedDict from this for backward compatibility +del _nongen_builtins["collections.OrderedDict"] +# HACK: consequence of hackily treating LiteralString as an alias for str +del _nongen_builtins["builtins.str"] + + +def get_nongen_builtins(python_version: tuple[int, int]) -> dict[str, str]: + # After 3.9 with pep585 generic builtins are allowed + return _nongen_builtins if python_version < (3, 9) else {} + + +RUNTIME_PROTOCOL_DECOS: Final = ( + "typing.runtime_checkable", + "typing_extensions.runtime", + "typing_extensions.runtime_checkable", +) + +LAMBDA_NAME: Final = "" class Node(Context): """Common base class for all non-type parse tree nodes.""" - line = -1 - # Textual representation - repr = None # type: Any - - literal = LITERAL_NO - literal_hash = None # type: Any + __slots__ = () def __str__(self) -> str: - ans = self.accept(mypy.strconv.StrConv()) + ans = self.accept(mypy.strconv.StrConv(options=Options())) if ans is None: return repr(self) return ans - @overload - def set_line(self, tok: Token) -> 'Node': - self.line = tok.line - return self + def str_with_options(self, options: Options) -> str: + ans = self.accept(mypy.strconv.StrConv(options=options)) + assert ans + return ans - @overload - def set_line(self, line: int) -> 'Node': - self.line = line - return self + def accept(self, visitor: NodeVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) - def get_line(self) -> int: - # TODO this should be just 'line' - return self.line - def accept(self, visitor: NodeVisitor[T]) -> T: - raise RuntimeError('Not implemented') +@trait +class Statement(Node): + """A statement node.""" + + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) + + +@trait +class Expression(Node): + """An expression node.""" + + __slots__ = () + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) +class FakeExpression(Expression): + """A dummy expression. + + We need a dummy expression in one place, and can't instantiate Expression + because it is a trait and mypyc barfs. + """ + + __slots__ = () + + +# TODO: +# Lvalue = Union['NameExpr', 'MemberExpr', 'IndexExpr', 'SuperExpr', 'StarExpr' +# 'TupleExpr']; see #1783. +Lvalue: _TypeAlias = Expression + + +@trait class SymbolNode(Node): - # Nodes that can be stored in a symbol table. + """Nodes that can be stored in a symbol table.""" + + __slots__ = () - # TODO do not use methods for these + @property + @abstractmethod + def name(self) -> str: + pass + # Fully qualified name + @property @abstractmethod - def name(self) -> str: pass + def fullname(self) -> str: + pass @abstractmethod - def fullname(self) -> str: pass + def serialize(self) -> JsonDict: + pass + + @classmethod + def deserialize(cls, data: JsonDict) -> SymbolNode: + classname = data[".class"] + method = deserialize_map.get(classname) + if method is not None: + return method(data) + raise NotImplementedError(f"unexpected .class {classname}") + + +# Items: fullname, related symbol table node, surrounding type (if any) +Definition: _TypeAlias = Tuple[str, "SymbolTableNode", Optional["TypeInfo"]] class MypyFile(SymbolNode): """The abstract syntax tree of a single source file.""" - _name = None # type: str # Module name ('__main__' for initial file) - _fullname = None # type: str # Qualified module name - path = '' # Path to the file (None if not known) - defs = Undefined # type: List[Node] # Global definitions and statements - is_bom = False # Is there a UTF-8 BOM at the start? - names = Undefined('SymbolTable') - imports = Undefined(List['ImportBase']) # All import nodes within the file - - def __init__(self, defs: List[Node], imports: List['ImportBase'], - is_bom: bool = False) -> None: + __slots__ = ( + "_fullname", + "path", + "defs", + "alias_deps", + "is_bom", + "names", + "imports", + "ignored_lines", + "skipped_lines", + "is_stub", + "is_cache_skeleton", + "is_partial_stub_package", + "plugin_deps", + "future_import_flags", + "_is_typeshed_file", + ) + + __match_args__ = ("name", "path", "defs") + + # Fully qualified module name + _fullname: str + # Path to the file (empty string if not known) + path: str + # Top-level definitions and statements + defs: list[Statement] + # Type alias dependencies as mapping from target to set of alias full names + alias_deps: defaultdict[str, set[str]] + # Is there a UTF-8 BOM at the start? + is_bom: bool + names: SymbolTable + # All import nodes within the file (also ones within functions etc.) + imports: list[ImportBase] + # Lines on which to ignore certain errors when checking. + # If the value is empty, ignore all errors; otherwise, the list contains all + # error codes to ignore. + ignored_lines: dict[int, list[str]] + # Lines that were skipped during semantic analysis e.g. due to ALWAYS_FALSE, MYPY_FALSE, + # or platform/version checks. Those lines would not be type-checked. + skipped_lines: set[int] + # Is this file represented by a stub file (.pyi)? + is_stub: bool + # Is this loaded from the cache and thus missing the actual body of the file? + is_cache_skeleton: bool + # Does this represent an __init__.pyi stub with a module __getattr__ + # (i.e. a partial stub package), for such packages we suppress any missing + # module errors in addition to missing attribute errors. + is_partial_stub_package: bool + # Plugin-created dependencies + plugin_deps: dict[str, set[str]] + # Future imports defined in this file. Populated during semantic analysis. + future_import_flags: set[str] + _is_typeshed_file: bool | None + + def __init__( + self, + defs: list[Statement], + imports: list[ImportBase], + is_bom: bool = False, + ignored_lines: dict[int, list[str]] | None = None, + ) -> None: + super().__init__() self.defs = defs self.line = 1 # Dummy line number + self.column = 0 # Dummy column self.imports = imports self.is_bom = is_bom + self.alias_deps = defaultdict(set) + self.plugin_deps = {} + if ignored_lines: + self.ignored_lines = ignored_lines + else: + self.ignored_lines = {} + self.skipped_lines = set() + + self.path = "" + self.is_stub = False + self.is_cache_skeleton = False + self.is_partial_stub_package = False + self.future_import_flags = set() + self._is_typeshed_file = None + + def local_definitions(self) -> Iterator[Definition]: + """Return all definitions within the module (including nested). + This doesn't include imported definitions. + """ + return local_definitions(self.names, self.fullname) + + @property def name(self) -> str: - return self._name + return "" if not self._fullname else self._fullname.split(".")[-1] + @property def fullname(self) -> str: return self._fullname @@ -142,233 +372,615 @@ def accept(self, visitor: NodeVisitor[T]) -> T: return visitor.visit_mypy_file(self) def is_package_init_file(self) -> bool: - return not (self.path is None) and len(self.path) != 0 \ - and os.path.basename(self.path) == '__init__.py' - -class ImportBase(Node): + return len(self.path) != 0 and os.path.basename(self.path).startswith("__init__.") + + def is_future_flag_set(self, flag: str) -> bool: + return flag in self.future_import_flags + + def is_typeshed_file(self, options: Options) -> bool: + # Cache result since this is called a lot + if self._is_typeshed_file is None: + self._is_typeshed_file = is_typeshed_file(options.abs_custom_typeshed_dir, self.path) + return self._is_typeshed_file + + def serialize(self) -> JsonDict: + return { + ".class": "MypyFile", + "_fullname": self._fullname, + "names": self.names.serialize(self._fullname), + "is_stub": self.is_stub, + "path": self.path, + "is_partial_stub_package": self.is_partial_stub_package, + "future_import_flags": list(self.future_import_flags), + } + + @classmethod + def deserialize(cls, data: JsonDict) -> MypyFile: + assert data[".class"] == "MypyFile", data + tree = MypyFile([], []) + tree._fullname = data["_fullname"] + tree.names = SymbolTable.deserialize(data["names"]) + tree.is_stub = data["is_stub"] + tree.path = data["path"] + tree.is_partial_stub_package = data["is_partial_stub_package"] + tree.is_cache_skeleton = True + tree.future_import_flags = set(data["future_import_flags"]) + return tree + + +class ImportBase(Statement): """Base class for all import statements.""" - is_unreachable = False + + __slots__ = ("is_unreachable", "is_top_level", "is_mypy_only", "assignments") + + is_unreachable: bool # Set by semanal.SemanticAnalyzerPass1 if inside `if False` etc. + is_top_level: bool # Ditto if outside any class or def + is_mypy_only: bool # Ditto if inside `if TYPE_CHECKING` or `if MYPY` + + # If an import replaces existing definitions, we construct dummy assignment + # statements that assign the imported names to the names in the current scope, + # for type checking purposes. Example: + # + # x = 1 + # from m import x <-- add assignment representing "x = m.x" + assignments: list[AssignmentStmt] + + def __init__(self) -> None: + super().__init__() + self.assignments = [] + self.is_unreachable = False + self.is_top_level = False + self.is_mypy_only = False class Import(ImportBase): """import m [as n]""" - ids = Undefined(List[Tuple[str, str]]) # (module id, as id) + __slots__ = ("ids",) + + __match_args__ = ("ids",) - def __init__(self, ids: List[Tuple[str, str]]) -> None: + ids: list[tuple[str, str | None]] # (module id, as id) + + def __init__(self, ids: list[tuple[str, str | None]]) -> None: + super().__init__() self.ids = ids - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_import(self) class ImportFrom(ImportBase): - """from m import x, ...""" + """from m import x [as y], ...""" + + __slots__ = ("id", "names", "relative") + + __match_args__ = ("id", "names", "relative") - names = Undefined(List[Tuple[str, str]]) # Tuples (name, as name) + id: str + relative: int + names: list[tuple[str, str | None]] # Tuples (name, as name) - def __init__(self, id: str, relative: int, names: List[Tuple[str, str]]) -> None: + def __init__(self, id: str, relative: int, names: list[tuple[str, str | None]]) -> None: + super().__init__() self.id = id self.names = names self.relative = relative - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_import_from(self) class ImportAll(ImportBase): """from m import *""" + __slots__ = ("id", "relative") + + __match_args__ = ("id", "relative") + + id: str + relative: int + def __init__(self, id: str, relative: int) -> None: + super().__init__() self.id = id self.relative = relative - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_import_all(self) -class FuncBase(SymbolNode): - """Abstract base class for function-like nodes""" +FUNCBASE_FLAGS: Final = ["is_property", "is_class", "is_static", "is_final"] + + +class FuncBase(Node): + """Abstract base class for function-like nodes. + + N.B: Although this has SymbolNode subclasses (FuncDef, + OverloadedFuncDef), avoid calling isinstance(..., FuncBase) on + something that is typed as SymbolNode. This is to work around + mypy bug #3603, in which mypy doesn't understand multiple + inheritance very well, and will assume that a SymbolNode + cannot be a FuncBase. - # Type signature. This is usually Callable or Overloaded, but it can be something else for - # decorated functions/ - type = None # type: mypy.types.Type - # If method, reference to TypeInfo - info = None # type: TypeInfo + Instead, test against SYMBOL_FUNCBASE_TYPES, which enumerates + SymbolNode subclasses that are also FuncBase subclasses. + """ + + __slots__ = ( + "type", + "unanalyzed_type", + "info", + "is_property", + "is_class", # Uses "@classmethod" (explicit or implicit) + "is_static", # Uses "@staticmethod" (explicit or implicit) + "is_final", # Uses "@final" + "is_explicit_override", # Uses "@override" + "is_type_check_only", # Uses "@type_check_only" + "_fullname", + ) + + def __init__(self) -> None: + super().__init__() + # Type signature. This is usually CallableType or Overloaded, but it can be + # something else for decorated functions. + self.type: mypy.types.ProperType | None = None + # Original, not semantically analyzed type (used for reprocessing) + self.unanalyzed_type: mypy.types.ProperType | None = None + # If method, reference to TypeInfo + self.info = FUNC_NO_INFO + self.is_property = False + self.is_class = False + self.is_static = False + self.is_final = False + self.is_explicit_override = False + self.is_type_check_only = False + # Name with module prefix + self._fullname = "" + @property @abstractmethod - def name(self) -> str: pass + def name(self) -> str: + pass + @property def fullname(self) -> str: - return self.name() + return self._fullname + + +OverloadPart: _TypeAlias = Union["FuncDef", "Decorator"] - def is_method(self) -> bool: - return bool(self.info) +class OverloadedFuncDef(FuncBase, SymbolNode, Statement): + """A logical node representing all the variants of a multi-declaration function. -class OverloadedFuncDef(FuncBase): - """A logical node representing all the variants of an overloaded function. + A multi-declaration function is often an @overload, but can also be a + @property with a setter and a/or a deleter. This node has no explicit representation in the source program. Overloaded variants must be consecutive in the source file. """ - items = Undefined(List['Decorator']) - _fullname = None # type: str + __slots__ = ("items", "unanalyzed_items", "impl", "deprecated") - def __init__(self, items: List['Decorator']) -> None: + items: list[OverloadPart] + unanalyzed_items: list[OverloadPart] + impl: OverloadPart | None + deprecated: str | None + + def __init__(self, items: list[OverloadPart]) -> None: + super().__init__() self.items = items - self.set_line(items[0].line) + self.unanalyzed_items = items.copy() + self.impl = None + self.deprecated = None + if items: + # TODO: figure out how to reliably set end position (we don't know the impl here). + self.set_line(items[0].line, items[0].column) + @property def name(self) -> str: - return self.items[1].func.name() - - def fullname(self) -> str: - return self._fullname + if self.items: + return self.items[0].name + else: + # This may happen for malformed overload + assert self.impl is not None + return self.impl.name - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_overloaded_func_def(self) + def serialize(self) -> JsonDict: + return { + ".class": "OverloadedFuncDef", + "items": [i.serialize() for i in self.items], + "type": None if self.type is None else self.type.serialize(), + "fullname": self._fullname, + "impl": None if self.impl is None else self.impl.serialize(), + "flags": get_flags(self, FUNCBASE_FLAGS), + "deprecated": self.deprecated, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: + assert data[".class"] == "OverloadedFuncDef" + res = OverloadedFuncDef( + [cast(OverloadPart, SymbolNode.deserialize(d)) for d in data["items"]] + ) + if data.get("impl") is not None: + res.impl = cast(OverloadPart, SymbolNode.deserialize(data["impl"])) + # set line for empty overload items, as not set in __init__ + if len(res.items) > 0: + res.set_line(res.impl.line) + if data.get("type") is not None: + typ = mypy.types.deserialize_type(data["type"]) + assert isinstance(typ, mypy.types.ProperType) + res.type = typ + res._fullname = data["fullname"] + set_flags(res, data["flags"]) + res.deprecated = data["deprecated"] + # NOTE: res.info will be set in the fixup phase. + return res + + def is_dynamic(self) -> bool: + return all(item.is_dynamic() for item in self.items) + + +class Argument(Node): + """A single argument in a FuncItem.""" + + __slots__ = ("variable", "type_annotation", "initializer", "kind", "pos_only") + + __match_args__ = ("variable", "type_annotation", "initializer", "kind", "pos_only") + + def __init__( + self, + variable: Var, + type_annotation: mypy.types.Type | None, + initializer: Expression | None, + kind: ArgKind, + pos_only: bool = False, + ) -> None: + super().__init__() + self.variable = variable + self.type_annotation = type_annotation + self.initializer = initializer + self.kind = kind # must be an ARG_* constant + self.pos_only = pos_only + + def set_line( + self, + target: Context | int, + column: int | None = None, + end_line: int | None = None, + end_column: int | None = None, + ) -> None: + super().set_line(target, column, end_line, end_column) + + if self.initializer and self.initializer.line < 0: + self.initializer.set_line(self.line, self.column, self.end_line, self.end_column) + + self.variable.set_line(self.line, self.column, self.end_line, self.end_column) + + +# These specify the kind of a TypeParam +TYPE_VAR_KIND: Final = 0 +PARAM_SPEC_KIND: Final = 1 +TYPE_VAR_TUPLE_KIND: Final = 2 + + +class TypeParam: + __slots__ = ("name", "kind", "upper_bound", "values", "default") + + def __init__( + self, + name: str, + kind: int, + upper_bound: mypy.types.Type | None, + values: list[mypy.types.Type], + default: mypy.types.Type | None, + ) -> None: + self.name = name + self.kind = kind + self.upper_bound = upper_bound + self.values = values + self.default = default + + +FUNCITEM_FLAGS: Final = FUNCBASE_FLAGS + [ + "is_overload", + "is_generator", + "is_coroutine", + "is_async_generator", + "is_awaitable_coroutine", +] + class FuncItem(FuncBase): - args = Undefined(List['Var']) # Argument names - arg_kinds = Undefined(List[int]) # Kinds of arguments (ARG_*) - - # Initialization expessions for fixed args; None if no initialiser - init = Undefined(List['AssignmentStmt']) - min_args = 0 # Minimum number of arguments - max_pos = 0 # Maximum number of positional arguments, -1 if - # no explicit limit (*args not included) - body = Undefined('Block') - is_implicit = False # Implicit dynamic types? - is_overload = False # Is this an overload variant of function with - # more than one overload variant? - is_generator = False # Contains a yield statement? - is_coroutine = False # Contains @coroutine or yield from Future - is_static = False # Uses @staticmethod? - is_class = False # Uses @classmethod? - expanded = Undefined(List['FuncItem']) # Variants of function with type - # variables with values expanded - - def __init__(self, args: List['Var'], arg_kinds: List[int], - init: List[Node], body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - self.args = args - self.arg_kinds = arg_kinds - self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) - self.body = body + """Base class for nodes usable as overloaded function items.""" + + __slots__ = ( + "arguments", # Note that can be unset if deserialized (type is a lie!) + "arg_names", # Names of arguments + "arg_kinds", # Kinds of arguments + "min_args", # Minimum number of arguments + "max_pos", # Maximum number of positional arguments, -1 if no explicit + # limit (*args not included) + "type_args", # New-style type parameters (PEP 695) + "body", # Body of the function + "is_overload", # Is this an overload variant of function with more than + # one overload variant? + "is_generator", # Contains a yield statement? + "is_coroutine", # Defined using 'async def' syntax? + "is_async_generator", # Is an async def generator? + "is_awaitable_coroutine", # Decorated with '@{typing,asyncio}.coroutine'? + "expanded", # Variants of function with type variables with values expanded + ) + + __deletable__ = ("arguments", "max_pos", "min_args") + + def __init__( + self, + arguments: list[Argument] | None = None, + body: Block | None = None, + typ: mypy.types.FunctionLike | None = None, + type_args: list[TypeParam] | None = None, + ) -> None: + super().__init__() + self.arguments = arguments or [] + self.arg_names = [None if arg.pos_only else arg.variable.name for arg in self.arguments] + self.arg_kinds: list[ArgKind] = [arg.kind for arg in self.arguments] + self.max_pos: int = self.arg_kinds.count(ARG_POS) + self.arg_kinds.count(ARG_OPT) + self.type_args: list[TypeParam] | None = type_args + self.body: Block = body or Block([]) self.type = typ - self.expanded = [] + self.unanalyzed_type = typ + self.is_overload: bool = False + self.is_generator: bool = False + self.is_coroutine: bool = False + self.is_async_generator: bool = False + self.is_awaitable_coroutine: bool = False + self.expanded: list[FuncItem] = [] - i2 = List[AssignmentStmt]() self.min_args = 0 - for i in range(len(init)): - if init[i] is not None: - rvalue = init[i] - lvalue = NameExpr(args[i].name()).set_line(rvalue.line) - assign = AssignmentStmt([lvalue], rvalue) - assign.set_line(rvalue.line) - i2.append(assign) - else: - i2.append(None) - if i < self.max_fixed_argc(): - self.min_args = i + 1 - self.init = i2 + for i in range(len(self.arguments)): + if self.arguments[i] is None and i < self.max_fixed_argc(): + self.min_args = i + 1 def max_fixed_argc(self) -> int: return self.max_pos - @overload - def set_line(self, tok: Token) -> Node: - super().set_line(tok) - for n in self.args: - n.line = self.line - return self - - @overload - def set_line(self, tok: int) -> Node: - super().set_line(tok) - for n in self.args: - n.line = self.line - return self - - def init_expressions(self) -> List[Node]: - res = List[Node]() - for i in self.init: - if i is not None: - res.append(i.rvalue) - else: - res.append(None) - return res + def is_dynamic(self) -> bool: + return self.type is None + + +FUNCDEF_FLAGS: Final = FUNCITEM_FLAGS + [ + "is_decorated", + "is_conditional", + "is_trivial_body", + "is_mypy_only", +] + +# Abstract status of a function +NOT_ABSTRACT: Final = 0 +# Explicitly abstract (with @abstractmethod or overload without implementation) +IS_ABSTRACT: Final = 1 +# Implicitly abstract: used for functions with trivial bodies defined in Protocols +IMPLICITLY_ABSTRACT: Final = 2 -class FuncDef(FuncItem): +class FuncDef(FuncItem, SymbolNode, Statement): """Function definition. This is a non-lambda function defined using 'def'. """ - _fullname = None # type: str # Name with module prefix - is_decorated = False - is_conditional = False # Defined conditionally (within block)? - is_abstract = False - is_property = False - original_def = None # type: FuncDef # Original conditional definition - - def __init__(self, - name: str, # Function name - args: List['Var'], # Argument names - arg_kinds: List[int], # Arguments kinds (nodes.ARG_*) - init: List[Node], # Initializers (each may be None) - body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - super().__init__(args, arg_kinds, init, body, typ) + __slots__ = ( + "_name", + "is_decorated", + "is_conditional", + "abstract_status", + "original_def", + "deco_line", + "is_trivial_body", + "is_mypy_only", + # Present only when a function is decorated with @typing.dataclass_transform or similar + "dataclass_transform_spec", + "docstring", + "deprecated", + ) + + __match_args__ = ("name", "arguments", "type", "body") + + # Note that all __init__ args must have default values + def __init__( + self, + name: str = "", # Function name + arguments: list[Argument] | None = None, + body: Block | None = None, + typ: mypy.types.FunctionLike | None = None, + type_args: list[TypeParam] | None = None, + ) -> None: + super().__init__(arguments, body, typ, type_args) self._name = name + self.is_decorated = False + self.is_conditional = False # Defined conditionally (within block)? + self.abstract_status = NOT_ABSTRACT + # Is this an abstract method with trivial body? + # Such methods can't be called via super(). + self.is_trivial_body = False + # Original conditional definition + self.original_def: None | FuncDef | Var | Decorator = None + # Used for error reporting (to keep backward compatibility with pre-3.8) + self.deco_line: int | None = None + # Definitions that appear in if TYPE_CHECKING are marked with this flag. + self.is_mypy_only = False + self.dataclass_transform_spec: DataclassTransformSpec | None = None + self.docstring: str | None = None + self.deprecated: str | None = None + @property def name(self) -> str: return self._name - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_func_def(self) - def is_constructor(self) -> bool: - return self.info is not None and self._name == '__init__' - - def get_name(self) -> str: - """TODO merge with name()""" - return self._name - - -class Decorator(SymbolNode): + def serialize(self) -> JsonDict: + # We're deliberating omitting arguments and storing only arg_names and + # arg_kinds for space-saving reasons (arguments is not used in later + # stages of mypy). + # TODO: After a FuncDef is deserialized, the only time we use `arg_names` + # and `arg_kinds` is when `type` is None and we need to infer a type. Can + # we store the inferred type ahead of time? + return { + ".class": "FuncDef", + "name": self._name, + "fullname": self._fullname, + "arg_names": self.arg_names, + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "type": None if self.type is None else self.type.serialize(), + "flags": get_flags(self, FUNCDEF_FLAGS), + "abstract_status": self.abstract_status, + # TODO: Do we need expanded, original_def? + "dataclass_transform_spec": ( + None + if self.dataclass_transform_spec is None + else self.dataclass_transform_spec.serialize() + ), + "deprecated": self.deprecated, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> FuncDef: + assert data[".class"] == "FuncDef" + body = Block([]) + ret = FuncDef( + data["name"], + [], + body, + ( + None + if data["type"] is None + else cast(mypy.types.FunctionLike, mypy.types.deserialize_type(data["type"])) + ), + ) + ret._fullname = data["fullname"] + set_flags(ret, data["flags"]) + # NOTE: ret.info is set in the fixup phase. + ret.arg_names = data["arg_names"] + ret.arg_kinds = [ArgKind(x) for x in data["arg_kinds"]] + ret.abstract_status = data["abstract_status"] + ret.dataclass_transform_spec = ( + DataclassTransformSpec.deserialize(data["dataclass_transform_spec"]) + if data["dataclass_transform_spec"] is not None + else None + ) + ret.deprecated = data["deprecated"] + # Leave these uninitialized so that future uses will trigger an error + del ret.arguments + del ret.max_pos + del ret.min_args + return ret + + +# All types that are both SymbolNodes and FuncBases. See the FuncBase +# docstring for the rationale. +SYMBOL_FUNCBASE_TYPES = (OverloadedFuncDef, FuncDef) + + +class Decorator(SymbolNode, Statement): """A decorated function. A single Decorator object can include any number of function decorators. """ - func = Undefined(FuncDef) # Decorated function - decorators = Undefined(List[Node]) # Decorators, at least one - var = Undefined('Var') # Represents the decorated function obj - is_overload = False + __slots__ = ("func", "decorators", "original_decorators", "var", "is_overload") + + __match_args__ = ("decorators", "var", "func") - def __init__(self, func: FuncDef, decorators: List[Node], - var: 'Var') -> None: + func: FuncDef # Decorated function + decorators: list[Expression] # Decorators (may be empty) + # Some decorators are removed by semanal, keep the original here. + original_decorators: list[Expression] + # TODO: This is mostly used for the type; consider replacing with a 'type' attribute + var: Var # Represents the decorated function obj + is_overload: bool + + def __init__(self, func: FuncDef, decorators: list[Expression], var: Var) -> None: + super().__init__() self.func = func self.decorators = decorators + self.original_decorators = decorators.copy() self.var = var self.is_overload = False + @property def name(self) -> str: - return self.func.name() + return self.func.name + @property def fullname(self) -> str: - return self.func.fullname() + return self.func.fullname - def accept(self, visitor: NodeVisitor[T]) -> T: + @property + def is_final(self) -> bool: + return self.func.is_final + + @property + def info(self) -> TypeInfo: + return self.func.info + + @property + def type(self) -> mypy.types.Type | None: + return self.var.type + + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_decorator(self) + def serialize(self) -> JsonDict: + return { + ".class": "Decorator", + "func": self.func.serialize(), + "var": self.var.serialize(), + "is_overload": self.is_overload, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> Decorator: + assert data[".class"] == "Decorator" + dec = Decorator(FuncDef.deserialize(data["func"]), [], Var.deserialize(data["var"])) + dec.is_overload = data["is_overload"] + return dec + + def is_dynamic(self) -> bool: + return self.func.is_dynamic() + + +VAR_FLAGS: Final = [ + "is_self", + "is_cls", + "is_initialized_in_class", + "is_staticmethod", + "is_classmethod", + "is_property", + "is_settable_property", + "is_suppressed_import", + "is_classvar", + "is_abstract_var", + "is_final", + "is_index_var", + "final_unset_in_class", + "final_set_in_init", + "explicit_self_type", + "is_ready", + "is_inferred", + "invalid_partial_type", + "from_module_getattr", + "has_explicit_value", + "allow_incompatible_override", +] + class Var(SymbolNode): """A variable. @@ -376,507 +988,882 @@ class Var(SymbolNode): It can refer to global/local variable or a data attribute. """ - _name = None # type: str # Name without module prefix - _fullname = None # type: str # Name with module prefix - info = Undefined('TypeInfo') # Defining class (for member variables) - type = None # type: mypy.types.Type # Declared or inferred type, or None - is_self = False # Is this the first argument to an ordinary method - # (usually "self")? - is_ready = False # If inferred, is the inferred type available? - # Is this initialized explicitly to a non-None value in class body? - is_initialized_in_class = False - is_staticmethod = False - is_classmethod = False - is_property = False - - def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: - self._name = name - self.type = type + __slots__ = ( + "_name", + "_fullname", + "info", + "type", + "final_value", + "is_self", + "is_cls", + "is_ready", + "is_inferred", + "is_initialized_in_class", + "is_staticmethod", + "is_classmethod", + "is_property", + "is_settable_property", + "is_classvar", + "is_abstract_var", + "is_final", + "is_index_var", + "final_unset_in_class", + "final_set_in_init", + "is_suppressed_import", + "explicit_self_type", + "from_module_getattr", + "has_explicit_value", + "allow_incompatible_override", + "invalid_partial_type", + ) + + __match_args__ = ("name", "type", "final_value") + + def __init__(self, name: str, type: mypy.types.Type | None = None) -> None: + super().__init__() + self._name = name # Name without module prefix + # TODO: Should be Optional[str] + self._fullname = "" # Name with module prefix + # TODO: Should be Optional[TypeInfo] + self.info = VAR_NO_INFO + self.type: mypy.types.Type | None = type # Declared or inferred type, or None + # Is this the first argument to an ordinary method (usually "self")? self.is_self = False - self.is_ready = True + # Is this the first argument to a classmethod (typically "cls")? + self.is_cls = False + self.is_ready = True # If inferred, is the inferred type available? + self.is_inferred = self.type is None + # Is this initialized explicitly to a non-None value in class body? self.is_initialized_in_class = False + self.is_staticmethod = False + self.is_classmethod = False + self.is_property = False + self.is_settable_property = False + self.is_classvar = False + self.is_abstract_var = False + self.is_index_var = False + # Set to true when this variable refers to a module we were unable to + # parse for some reason (eg a silenced module) + self.is_suppressed_import = False + # Was this "variable" (rather a constant) defined as Final[...]? + self.is_final = False + # If constant value is a simple literal, + # store the literal value (unboxed) for the benefit of + # tools like mypyc. + self.final_value: int | float | complex | bool | str | None = None + # Where the value was set (only for class attributes) + self.final_unset_in_class = False + self.final_set_in_init = False + # This is True for a variable that was declared on self with an explicit type: + # class C: + # def __init__(self) -> None: + # self.x: int + # This case is important because this defines a new Var, even if there is one + # present in a superclass (without explicit type this doesn't create a new Var). + # See SemanticAnalyzer.analyze_member_lvalue() for details. + self.explicit_self_type = False + # If True, this is an implicit Var created due to module-level __getattr__. + self.from_module_getattr = False + # Var can be created with an explicit value `a = 1` or without one `a: int`, + # we need a way to tell which one is which. + self.has_explicit_value = False + # If True, subclasses can override this with an incompatible type. + self.allow_incompatible_override = False + # If True, this means we didn't manage to infer full type and fall back to + # something like list[Any]. We may decide to not use such types as context. + self.invalid_partial_type = False + @property def name(self) -> str: return self._name + @property def fullname(self) -> str: return self._fullname def accept(self, visitor: NodeVisitor[T]) -> T: return visitor.visit_var(self) - -class ClassDef(Node): + def serialize(self) -> JsonDict: + # TODO: Leave default values out? + # NOTE: Sometimes self.is_ready is False here, but we don't care. + data: JsonDict = { + ".class": "Var", + "name": self._name, + "fullname": self._fullname, + "type": None if self.type is None else self.type.serialize(), + "flags": get_flags(self, VAR_FLAGS), + } + if self.final_value is not None: + data["final_value"] = self.final_value + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> Var: + assert data[".class"] == "Var" + name = data["name"] + type = None if data["type"] is None else mypy.types.deserialize_type(data["type"]) + v = Var(name, type) + v.is_ready = False # Override True default set in __init__ + v._fullname = data["fullname"] + set_flags(v, data["flags"]) + v.final_value = data.get("final_value") + return v + + +class ClassDef(Statement): """Class definition""" - name = Undefined(str) # Name of the class without module prefix - fullname = None # type: str # Fully qualified name of the class - defs = Undefined('Block') - type_vars = Undefined(List['mypy.types.TypeVarDef']) - # Base classes (Instance or UnboundType). - base_types = Undefined(List['mypy.types.Type']) - info = None # type: TypeInfo # Related TypeInfo - metaclass = '' - decorators = Undefined(List[Node]) - # Built-in/extension class? (single implementation inheritance only) - is_builtinclass = False - - def __init__(self, name: str, defs: 'Block', - type_vars: List['mypy.types.TypeVarDef'] = None, - base_types: List['mypy.types.Type'] = None, - metaclass: str = None) -> None: - if not base_types: - base_types = [] + __slots__ = ( + "name", + "_fullname", + "defs", + "type_args", + "type_vars", + "base_type_exprs", + "removed_base_type_exprs", + "info", + "metaclass", + "decorators", + "keywords", + "analyzed", + "has_incompatible_baseclass", + "deco_line", + "docstring", + "removed_statements", + ) + + __match_args__ = ("name", "defs") + + name: str # Name of the class without module prefix + _fullname: str # Fully qualified name of the class + defs: Block + # New-style type parameters (PEP 695), unanalyzed + type_args: list[TypeParam] | None + # Semantically analyzed type parameters (all syntax variants) + type_vars: list[mypy.types.TypeVarLikeType] + # Base class expressions (not semantically analyzed -- can be arbitrary expressions) + base_type_exprs: list[Expression] + # Special base classes like Generic[...] get moved here during semantic analysis + removed_base_type_exprs: list[Expression] + info: TypeInfo # Related TypeInfo + metaclass: Expression | None + decorators: list[Expression] + keywords: dict[str, Expression] + analyzed: Expression | None + has_incompatible_baseclass: bool + # Used by special forms like NamedTuple and TypedDict to store invalid statements + removed_statements: list[Statement] + + def __init__( + self, + name: str, + defs: Block, + type_vars: list[mypy.types.TypeVarLikeType] | None = None, + base_type_exprs: list[Expression] | None = None, + metaclass: Expression | None = None, + keywords: list[tuple[str, Expression]] | None = None, + type_args: list[TypeParam] | None = None, + ) -> None: + super().__init__() self.name = name + self._fullname = "" self.defs = defs self.type_vars = type_vars or [] - self.base_types = base_types + self.type_args = type_args + self.base_type_exprs = base_type_exprs or [] + self.removed_base_type_exprs = [] + self.info = CLASSDEF_NO_INFO self.metaclass = metaclass self.decorators = [] + self.keywords = dict(keywords) if keywords else {} + self.analyzed = None + self.has_incompatible_baseclass = False + # Used for error reporting (to keep backwad compatibility with pre-3.8) + self.deco_line: int | None = None + self.docstring: str | None = None + self.removed_statements = [] - def accept(self, visitor: NodeVisitor[T]) -> T: + @property + def fullname(self) -> str: + return self._fullname + + @fullname.setter + def fullname(self, v: str) -> None: + self._fullname = v + + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_class_def(self) def is_generic(self) -> bool: return self.info.is_generic() + def serialize(self) -> JsonDict: + # Not serialized: defs, base_type_exprs, metaclass, decorators, + # analyzed (for named tuples etc.) + return { + ".class": "ClassDef", + "name": self.name, + "fullname": self.fullname, + "type_vars": [v.serialize() for v in self.type_vars], + } + + @classmethod + def deserialize(cls, data: JsonDict) -> ClassDef: + assert data[".class"] == "ClassDef" + res = ClassDef( + data["name"], + Block([]), + # https://github.com/python/mypy/issues/12257 + [ + cast(mypy.types.TypeVarLikeType, mypy.types.deserialize_type(v)) + for v in data["type_vars"] + ], + ) + res.fullname = data["fullname"] + return res -class VarDef(Node): - """Variable definition with explicit types""" - - items = Undefined(List[Var]) - kind = None # type: int # LDEF/GDEF/MDEF/... - init = Undefined(Node) # Expression or None - is_top_level = False # Is the definition at the top level (not within - # a function or a type)? - - def __init__(self, items: List[Var], is_top_level: bool, - init: Node = None) -> None: - self.items = items - self.is_top_level = is_top_level - self.init = init - - def info(self) -> 'TypeInfo': - return self.items[0].info - - @overload - def set_line(self, tok: Token) -> Node: - super().set_line(tok) - for n in self.items: - n.line = self.line - return self - - @overload - def set_line(self, tok: int) -> Node: - super().set_line(tok) - for n in self.items: - n.line = self.line - return self - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_var_def(self) +class GlobalDecl(Statement): + """Declaration global x, y, ...""" + __slots__ = ("names",) -class GlobalDecl(Node): - """Declaration global x, y, ...""" + __match_args__ = ("names",) - names = Undefined(List[str]) + names: list[str] - def __init__(self, names: List[str]) -> None: + def __init__(self, names: list[str]) -> None: + super().__init__() self.names = names - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_global_decl(self) -class NonlocalDecl(Node): +class NonlocalDecl(Statement): """Declaration nonlocal x, y, ...""" - names = Undefined(List[str]) + __slots__ = ("names",) + + __match_args__ = ("names",) - def __init__(self, names: List[str]) -> None: + names: list[str] + + def __init__(self, names: list[str]) -> None: + super().__init__() self.names = names - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_nonlocal_decl(self) -class Block(Node): - body = Undefined(List[Node]) - # True if we can determine that this block is not executed. For example, - # this applies to blocks that are protected by something like "if PY3:" - # when using Python 2. - is_unreachable = False +class Block(Statement): + __slots__ = ("body", "is_unreachable") - def __init__(self, body: List[Node]) -> None: - self.body = body + __match_args__ = ("body", "is_unreachable") - def accept(self, visitor: NodeVisitor[T]) -> T: + def __init__(self, body: list[Statement], *, is_unreachable: bool = False) -> None: + super().__init__() + self.body = body + # True if we can determine that this block is not executed during semantic + # analysis. For example, this applies to blocks that are protected by + # something like "if PY3:" when using Python 2. However, some code is + # only considered unreachable during type checking and this is not true + # in those cases. + self.is_unreachable = is_unreachable + + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_block(self) # Statements -class ExpressionStmt(Node): - """An expression as a statament, such as print(s).""" - expr = Undefined(Node) +class ExpressionStmt(Statement): + """An expression as a statement, such as print(s).""" + + __slots__ = ("expr",) - def __init__(self, expr: Node) -> None: + __match_args__ = ("expr",) + + expr: Expression + + def __init__(self, expr: Expression) -> None: + super().__init__() self.expr = expr - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_expression_stmt(self) -class AssignmentStmt(Node): - """Assignment statement +class AssignmentStmt(Statement): + """Assignment statement. The same node class is used for single assignment, multiple assignment (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments - that define new names, and assignments with explicit types (# type). + that define new names, and assignments with explicit types ("# type: t" + or "x: t [= ...]"). - An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr or - ParenExpr. + An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, or IndexExpr. """ - lvalues = Undefined(List[Node]) - rvalue = Undefined(Node) - type = None # type: mypy.types.Type # Declared type in a comment, - # may be None. - - def __init__(self, lvalues: List[Node], rvalue: Node, - type: 'mypy.types.Type' = None) -> None: + __slots__ = ( + "lvalues", + "rvalue", + "type", + "unanalyzed_type", + "new_syntax", + "is_alias_def", + "is_final_def", + "invalid_recursive_alias", + ) + + __match_args__ = ("lvalues", "rvalues", "type") + + lvalues: list[Lvalue] + # This is a TempNode if and only if no rvalue (x: t). + rvalue: Expression + # Declared type in a comment, may be None. + type: mypy.types.Type | None + # Original, not semantically analyzed type in annotation (used for reprocessing) + unanalyzed_type: mypy.types.Type | None + # This indicates usage of PEP 526 type annotation syntax in assignment. + new_syntax: bool + # Does this assignment define a type alias? + is_alias_def: bool + # Is this a final definition? + # Final attributes can't be re-assigned once set, and can't be overridden + # in a subclass. This flag is not set if an attempted declaration was found to + # be invalid during semantic analysis. It is still set to `True` if + # a final declaration overrides another final declaration (this is checked + # during type checking when MROs are known). + is_final_def: bool + # Stop further processing of this assignment, to prevent flipping back and forth + # during semantic analysis passes. + invalid_recursive_alias: bool + + def __init__( + self, + lvalues: list[Lvalue], + rvalue: Expression, + type: mypy.types.Type | None = None, + new_syntax: bool = False, + ) -> None: + super().__init__() self.lvalues = lvalues self.rvalue = rvalue self.type = type + self.unanalyzed_type = type + self.new_syntax = new_syntax + self.is_alias_def = False + self.is_final_def = False + self.invalid_recursive_alias = False - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_assignment_stmt(self) -class OperatorAssignmentStmt(Node): +class OperatorAssignmentStmt(Statement): """Operator assignment statement such as x += 1""" - op = '' - lvalue = Undefined(Node) - rvalue = Undefined(Node) + __slots__ = ("op", "lvalue", "rvalue") + + __match_args__ = ("lvalue", "op", "rvalue") - def __init__(self, op: str, lvalue: Node, rvalue: Node) -> None: + op: str # TODO: Enum? + lvalue: Lvalue + rvalue: Expression + + def __init__(self, op: str, lvalue: Lvalue, rvalue: Expression) -> None: + super().__init__() self.op = op self.lvalue = lvalue self.rvalue = rvalue - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_operator_assignment_stmt(self) -class WhileStmt(Node): - expr = Undefined(Node) - body = Undefined(Block) - else_body = Undefined(Block) +class WhileStmt(Statement): + __slots__ = ("expr", "body", "else_body") - def __init__(self, expr: Node, body: Block, else_body: Block) -> None: + __match_args__ = ("expr", "body", "else_body") + + expr: Expression + body: Block + else_body: Block | None + + def __init__(self, expr: Expression, body: Block, else_body: Block | None) -> None: + super().__init__() self.expr = expr self.body = body self.else_body = else_body - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_while_stmt(self) -class ForStmt(Node): +class ForStmt(Statement): + __slots__ = ( + "index", + "index_type", + "unanalyzed_index_type", + "inferred_item_type", + "inferred_iterator_type", + "expr", + "body", + "else_body", + "is_async", + ) + + __match_args__ = ("index", "index_type", "expr", "body", "else_body") + # Index variables - index = Undefined(Node) + index: Lvalue + # Type given by type comments for index, can be None + index_type: mypy.types.Type | None + # Original, not semantically analyzed type in annotation (used for reprocessing) + unanalyzed_index_type: mypy.types.Type | None + # Inferred iterable item type + inferred_item_type: mypy.types.Type | None + # Inferred iterator type + inferred_iterator_type: mypy.types.Type | None # Expression to iterate - expr = Undefined(Node) - body = Undefined(Block) - else_body = Undefined(Block) - - def __init__(self, index: Node, expr: Node, body: Block, - else_body: Block) -> None: + expr: Expression + body: Block + else_body: Block | None + is_async: bool # True if `async for ...` (PEP 492, Python 3.5) + + def __init__( + self, + index: Lvalue, + expr: Expression, + body: Block, + else_body: Block | None, + index_type: mypy.types.Type | None = None, + ) -> None: + super().__init__() self.index = index + self.index_type = index_type + self.unanalyzed_index_type = index_type + self.inferred_item_type = None + self.inferred_iterator_type = None self.expr = expr self.body = body self.else_body = else_body + self.is_async = False - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_for_stmt(self) -class ReturnStmt(Node): - expr = Undefined(Node) # Expression or None - - def __init__(self, expr: Node) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_return_stmt(self) +class ReturnStmt(Statement): + __slots__ = ("expr",) + __match_args__ = ("expr",) -class AssertStmt(Node): - expr = Undefined(Node) + expr: Expression | None - def __init__(self, expr: Node) -> None: + def __init__(self, expr: Expression | None) -> None: + super().__init__() self.expr = expr - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assert_stmt(self) + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_return_stmt(self) -class YieldStmt(Node): - expr = Undefined(Node) +class AssertStmt(Statement): + __slots__ = ("expr", "msg") - def __init__(self, expr: Node) -> None: - self.expr = expr + __match_args__ = ("expr", "msg") - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_stmt(self) + expr: Expression + msg: Expression | None + def __init__(self, expr: Expression, msg: Expression | None = None) -> None: + super().__init__() + self.expr = expr + self.msg = msg -class YieldFromStmt(Node): - expr = Undefined(Node) + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_assert_stmt(self) - def __init__(self, expr: Node) -> None: - self.expr = expr - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_from_stmt(self) +class DelStmt(Statement): + __slots__ = ("expr",) + __match_args__ = ("expr",) -class DelStmt(Node): - expr = Undefined(Node) + expr: Lvalue - def __init__(self, expr: Node) -> None: + def __init__(self, expr: Lvalue) -> None: + super().__init__() self.expr = expr - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_del_stmt(self) -class BreakStmt(Node): - def accept(self, visitor: NodeVisitor[T]) -> T: +class BreakStmt(Statement): + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_break_stmt(self) -class ContinueStmt(Node): - def accept(self, visitor: NodeVisitor[T]) -> T: +class ContinueStmt(Statement): + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_continue_stmt(self) -class PassStmt(Node): - def accept(self, visitor: NodeVisitor[T]) -> T: +class PassStmt(Statement): + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_pass_stmt(self) -class IfStmt(Node): - expr = Undefined(List[Node]) - body = Undefined(List[Block]) - else_body = Undefined(Block) +class IfStmt(Statement): + __slots__ = ("expr", "body", "else_body") - def __init__(self, expr: List[Node], body: List[Block], - else_body: Block) -> None: + __match_args__ = ("expr", "body", "else_body") + + expr: list[Expression] + body: list[Block] + else_body: Block | None + + def __init__(self, expr: list[Expression], body: list[Block], else_body: Block | None) -> None: + super().__init__() self.expr = expr self.body = body self.else_body = else_body - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_if_stmt(self) -class RaiseStmt(Node): - expr = Undefined(Node) - from_expr = Undefined(Node) +class RaiseStmt(Statement): + __slots__ = ("expr", "from_expr") + + __match_args__ = ("expr", "from_expr") - def __init__(self, expr: Node, from_expr: Node = None) -> None: + # Plain 'raise' is a valid statement. + expr: Expression | None + from_expr: Expression | None + + def __init__(self, expr: Expression | None, from_expr: Expression | None) -> None: + super().__init__() self.expr = expr self.from_expr = from_expr - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_raise_stmt(self) -class TryStmt(Node): - body = Undefined(Block) # Try body - types = Undefined(List[Node]) # Except type expressions - vars = Undefined(List['NameExpr']) # Except variable names - handlers = Undefined(List[Block]) # Except bodies - else_body = Undefined(Block) - finally_body = Undefined(Block) - - def __init__(self, body: Block, vars: List['NameExpr'], types: List[Node], - handlers: List[Block], else_body: Block, - finally_body: Block) -> None: +class TryStmt(Statement): + __slots__ = ("body", "types", "vars", "handlers", "else_body", "finally_body", "is_star") + + __match_args__ = ("body", "types", "vars", "handlers", "else_body", "finally_body", "is_star") + + body: Block # Try body + # Plain 'except:' also possible + types: list[Expression | None] # Except type expressions + vars: list[NameExpr | None] # Except variable names + handlers: list[Block] # Except bodies + else_body: Block | None + finally_body: Block | None + # Whether this is try ... except* (added in Python 3.11) + is_star: bool + + def __init__( + self, + body: Block, + vars: list[NameExpr | None], + types: list[Expression | None], + handlers: list[Block], + else_body: Block | None, + finally_body: Block | None, + ) -> None: + super().__init__() self.body = body self.vars = vars self.types = types self.handlers = handlers self.else_body = else_body self.finally_body = finally_body + self.is_star = False - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_try_stmt(self) -class WithStmt(Node): - expr = Undefined(List[Node]) - name = Undefined(List['NameExpr']) - body = Undefined(Block) - - def __init__(self, expr: List[Node], name: List['NameExpr'], - body: Block) -> None: +class WithStmt(Statement): + __slots__ = ("expr", "target", "unanalyzed_type", "analyzed_types", "body", "is_async") + + __match_args__ = ("expr", "target", "body") + + expr: list[Expression] + target: list[Lvalue | None] + # Type given by type comments for target, can be None + unanalyzed_type: mypy.types.Type | None + # Semantically analyzed types from type comment (TypeList type expanded) + analyzed_types: list[mypy.types.Type] + body: Block + is_async: bool # True if `async with ...` (PEP 492, Python 3.5) + + def __init__( + self, + expr: list[Expression], + target: list[Lvalue | None], + body: Block, + target_type: mypy.types.Type | None = None, + ) -> None: + super().__init__() self.expr = expr - self.name = name + self.target = target + self.unanalyzed_type = target_type + self.analyzed_types = [] self.body = body + self.is_async = False - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_with_stmt(self) -class PrintStmt(Node): - """Python 2 print statement""" +class MatchStmt(Statement): + __slots__ = ("subject", "patterns", "guards", "bodies") - args = Undefined(List[Node]) - newline = False + __match_args__ = ("subject", "patterns", "guards", "bodies") - def __init__(self, args: List[Node], newline: bool) -> None: - self.args = args - self.newline = newline + subject: Expression + patterns: list[Pattern] + guards: list[Expression | None] + bodies: list[Block] - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_print_stmt(self) + def __init__( + self, + subject: Expression, + patterns: list[Pattern], + guards: list[Expression | None], + bodies: list[Block], + ) -> None: + super().__init__() + assert len(patterns) == len(guards) == len(bodies) + self.subject = subject + self.patterns = patterns + self.guards = guards + self.bodies = bodies + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_match_stmt(self) + + +class TypeAliasStmt(Statement): + __slots__ = ("name", "type_args", "value", "invalid_recursive_alias", "alias_node") + + __match_args__ = ("name", "type_args", "value") + + name: NameExpr + type_args: list[TypeParam] + value: LambdaExpr # Return value will get translated into a type + invalid_recursive_alias: bool + alias_node: TypeAlias | None + + def __init__(self, name: NameExpr, type_args: list[TypeParam], value: LambdaExpr) -> None: + super().__init__() + self.name = name + self.type_args = type_args + self.value = value + self.invalid_recursive_alias = False + self.alias_node = None + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_type_alias_stmt(self) # Expressions -class IntExpr(Node): +class IntExpr(Expression): """Integer literal""" - value = 0 - literal = LITERAL_YES + __slots__ = ("value",) + + __match_args__ = ("value",) + + value: int # 0 by default def __init__(self, value: int) -> None: + super().__init__() self.value = value - self.literal_hash = value - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_int_expr(self) -class StrExpr(Node): +# How mypy uses StrExpr and BytesExpr: +# +# b'x' -> BytesExpr +# 'x', u'x' -> StrExpr + + +class StrExpr(Expression): """String literal""" - value = '' - literal = LITERAL_YES + __slots__ = ("value",) + + __match_args__ = ("value",) + + value: str # '' by default def __init__(self, value: str) -> None: + super().__init__() self.value = value - self.literal_hash = value - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_str_expr(self) -class BytesExpr(Node): - """Bytes literal""" - - value = '' # TODO use bytes - literal = LITERAL_YES +def is_StrExpr_list(seq: list[Expression]) -> TypeGuard[list[StrExpr]]: # noqa: N802 + return all(isinstance(item, StrExpr) for item in seq) - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_bytes_expr(self) +class BytesExpr(Expression): + """Bytes literal""" + __slots__ = ("value",) -class UnicodeExpr(Node): - """Unicode literal (Python 2.x)""" + __match_args__ = ("value",) - value = '' # TODO use bytes - literal = LITERAL_YES + # Note: we deliberately do NOT use bytes here because it ends up + # unnecessarily complicating a lot of the result logic. For example, + # we'd have to worry about converting the bytes into a format we can + # easily serialize/deserialize to and from JSON, would have to worry + # about turning the bytes into a human-readable representation in + # error messages... + # + # It's more convenient to just store the human-readable representation + # from the very start. + value: str def __init__(self, value: str) -> None: + super().__init__() self.value = value - self.literal_hash = value - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unicode_expr(self) + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_bytes_expr(self) -class FloatExpr(Node): +class FloatExpr(Expression): """Float literal""" - value = 0.0 - literal = LITERAL_YES + __slots__ = ("value",) + + __match_args__ = ("value",) + + value: float # 0.0 by default def __init__(self, value: float) -> None: + super().__init__() self.value = value - self.literal_hash = value - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_float_expr(self) -class ComplexExpr(Node): +class ComplexExpr(Expression): """Complex literal""" - value = 0.0j - literal = LITERAL_YES + __slots__ = ("value",) - def __init__(self, value: complex) -> None: + __match_args__ = ("value",) + + value: complex + + def __init__(self, value: complex) -> None: + super().__init__() self.value = value - self.literal_hash = value - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_complex_expr(self) -class ParenExpr(Node): - """Parenthesised expression""" +class EllipsisExpr(Expression): + """Ellipsis (...)""" - expr = Undefined(Node) + __slots__ = () - def __init__(self, expr: Node) -> None: - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Paren', expr.literal_hash,) + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_ellipsis(self) - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_paren_expr(self) - -class StarExpr(Node): +class StarExpr(Expression): """Star expression""" - expr = Undefined(Node) + __slots__ = ("expr", "valid") + + __match_args__ = ("expr", "valid") + + expr: Expression + valid: bool - def __init__(self, expr: Node) -> None: + def __init__(self, expr: Expression) -> None: + super().__init__() self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Star', expr.literal_hash,) # Whether this starred expression is used in a tuple/list and as lvalue self.valid = False - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_star_expr(self) -class RefExpr(Node): +class RefExpr(Expression): """Abstract base class for name-like constructs""" - kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) - node = Undefined(Node) # Var, FuncDef or TypeInfo that describes this - fullname = None # type: str # Fully qualified name (or name if not global) + __slots__ = ( + "kind", + "node", + "_fullname", + "is_new_def", + "is_inferred_def", + "is_alias_rvalue", + "type_guard", + "type_is", + ) + + def __init__(self) -> None: + super().__init__() + # LDEF/GDEF/MDEF/... (None if not available) + self.kind: int | None = None + # Var, FuncDef or TypeInfo that describes this + self.node: SymbolNode | None = None + # Fully qualified name (or name if not global) + self._fullname = "" + # Does this define a new name? + self.is_new_def = False + # Does this define a new name with inferred type? + # + # For members, after semantic analysis, this does not take base + # classes into consideration at all; the type checker deals with these. + self.is_inferred_def = False + # Is this expression appears as an rvalue of a valid type alias definition? + self.is_alias_rvalue = False + # Cache type guard from callable_type.type_guard + self.type_guard: mypy.types.Type | None = None + # And same for TypeIs + self.type_is: mypy.types.Type | None = None - # Does this define a new name with inferred type? - # - # For members, after semantic analysis, this does not take base - # classes into consideration at all; the type checker deals with these. - is_def = False + @property + def fullname(self) -> str: + return self._fullname + + @fullname.setter + def fullname(self, v: str) -> None: + self._fullname = v class NameExpr(RefExpr): @@ -885,534 +1872,986 @@ class NameExpr(RefExpr): This refers to a local name, global name or a module. """ - name = None # type: str # Name referred to (may be qualified) - info = Undefined('TypeInfo') # TypeInfo of class surrounding expression - # (may be None) - literal = LITERAL_TYPE + __slots__ = ("name", "is_special_form") - def __init__(self, name: str) -> None: - self.name = name - self.literal_hash = ('Var', name,) + __match_args__ = ("name", "node") - def type_node(self): - return cast('TypeInfo', self.node) + def __init__(self, name: str) -> None: + super().__init__() + self.name = name # Name referred to + # Is this a l.h.s. of a special form assignment like typed dict or type variable? + self.is_special_form = False - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_name_expr(self) + def serialize(self) -> JsonDict: + assert False, f"Serializing NameExpr: {self}" + class MemberExpr(RefExpr): """Member access expression x.y""" - expr = Undefined(Node) - name = None # type: str - # The variable node related to a definition. - def_var = None # type: Var - # Is this direct assignment to a data member (bypassing accessors)? - direct = False + __slots__ = ("expr", "name", "def_var") + + __match_args__ = ("expr", "name", "node") - def __init__(self, expr: Node, name: str, direct: bool = False) -> None: + def __init__(self, expr: Expression, name: str) -> None: + super().__init__() self.expr = expr self.name = name - self.direct = direct - self.literal = self.expr.literal - self.literal_hash = ('Member', expr.literal_hash, name, direct) + # The variable node related to a definition through 'self.x = '. + # The nodes of other kinds of member expressions are resolved during type checking. + self.def_var: Var | None = None - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_member_expr(self) # Kinds of arguments - -# Positional argument -ARG_POS = 0 # type: int -# Positional, optional argument (functions only, not calls) -ARG_OPT = 1 # type: int -# *arg argument -ARG_STAR = 2 # type: int -# Keyword argument x=y in call, or keyword-only function arg -ARG_NAMED = 3 # type: int -# **arg argument -ARG_STAR2 = 4 # type: int - - -class CallExpr(Node): +@unique +class ArgKind(Enum): + # Positional argument + ARG_POS = 0 + # Positional, optional argument (functions only, not calls) + ARG_OPT = 1 + # *arg argument + ARG_STAR = 2 + # Keyword argument x=y in call, or keyword-only function arg + ARG_NAMED = 3 + # **arg argument + ARG_STAR2 = 4 + # In an argument list, keyword-only and also optional + ARG_NAMED_OPT = 5 + + def is_positional(self, star: bool = False) -> bool: + return self == ARG_POS or self == ARG_OPT or (star and self == ARG_STAR) + + def is_named(self, star: bool = False) -> bool: + return self == ARG_NAMED or self == ARG_NAMED_OPT or (star and self == ARG_STAR2) + + def is_required(self) -> bool: + return self == ARG_POS or self == ARG_NAMED + + def is_optional(self) -> bool: + return self == ARG_OPT or self == ARG_NAMED_OPT + + def is_star(self) -> bool: + return self == ARG_STAR or self == ARG_STAR2 + + +ARG_POS: Final = ArgKind.ARG_POS +ARG_OPT: Final = ArgKind.ARG_OPT +ARG_STAR: Final = ArgKind.ARG_STAR +ARG_NAMED: Final = ArgKind.ARG_NAMED +ARG_STAR2: Final = ArgKind.ARG_STAR2 +ARG_NAMED_OPT: Final = ArgKind.ARG_NAMED_OPT + + +class CallExpr(Expression): """Call expression. This can also represent several special forms that are syntactically calls - such as cast(...) and Undefined(...). + such as cast(...) and None # type: .... """ - callee = Undefined(Node) - args = Undefined(List[Node]) - arg_kinds = Undefined(List[int]) # ARG_ constants - arg_names = Undefined(List[str]) # Each name can be None if not a keyword - # argument. - analyzed = Undefined(Node) # If not None, the node that represents - # the meaning of the CallExpr. For - # cast(...) this is a CastExpr. - - def __init__(self, callee: Node, args: List[Node], arg_kinds: List[int], - arg_names: List[str] = None, analyzed: Node = None) -> None: + __slots__ = ("callee", "args", "arg_kinds", "arg_names", "analyzed") + + __match_args__ = ("callee", "args", "arg_kinds", "arg_names") + + def __init__( + self, + callee: Expression, + args: list[Expression], + arg_kinds: list[ArgKind], + arg_names: list[str | None], + analyzed: Expression | None = None, + ) -> None: + super().__init__() if not arg_names: arg_names = [None] * len(args) + self.callee = callee self.args = args - self.arg_kinds = arg_kinds - self.arg_names = arg_names + self.arg_kinds = arg_kinds # ARG_ constants + # Each name can be None if not a keyword argument. + self.arg_names: list[str | None] = arg_names + # If not None, the node that represents the meaning of the CallExpr. For + # cast(...) this is a CastExpr. self.analyzed = analyzed - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_call_expr(self) -class YieldFromExpr(Node): - expr = Undefined(Node) +class YieldFromExpr(Expression): + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression - def __init__(self, expr: Node) -> None: + def __init__(self, expr: Expression) -> None: + super().__init__() self.expr = expr - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_yield_from_expr(self) -class IndexExpr(Node): +class YieldExpr(Expression): + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression | None + + def __init__(self, expr: Expression | None) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_yield_expr(self) + + +class IndexExpr(Expression): """Index expression x[y]. - Also wraps type application as a special form. + Also wraps type application such as List[int] as a special form. """ - base = Undefined(Node) - index = Undefined(Node) + __slots__ = ("base", "index", "method_type", "analyzed") + + __match_args__ = ("base", "index") + + base: Expression + index: Expression # Inferred __getitem__ method type - method_type = None # type: mypy.types.Type + method_type: mypy.types.Type | None # If not None, this is actually semantically a type application - # Class[type, ...]. - analyzed = Undefined('TypeApplication') + # Class[type, ...] or a type alias initializer. + analyzed: TypeApplication | TypeAliasExpr | None - def __init__(self, base: Node, index: Node) -> None: + def __init__(self, base: Expression, index: Expression) -> None: + super().__init__() self.base = base self.index = index + self.method_type = None self.analyzed = None - if self.index.literal == LITERAL_YES: - self.literal = self.base.literal - self.literal_hash = ('Member', base.literal_hash, - index.literal_hash) - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_index_expr(self) -class UnaryExpr(Node): +class UnaryExpr(Expression): """Unary operation""" - op = '' - expr = Undefined(Node) + __slots__ = ("op", "expr", "method_type") + + __match_args__ = ("op", "expr") + + op: str # TODO: Enum? + expr: Expression # Inferred operator method type - method_type = None # type: mypy.types.Type + method_type: mypy.types.Type | None - def __init__(self, op: str, expr: Node) -> None: + def __init__(self, op: str, expr: Expression) -> None: + super().__init__() self.op = op self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Unary', op, expr.literal_hash) + self.method_type = None - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_unary_expr(self) -# Map from binary operator id to related method name (in Python 3). -op_methods = { - '+': '__add__', - '-': '__sub__', - '*': '__mul__', - '/': '__truediv__', - '%': '__mod__', - '//': '__floordiv__', - '**': '__pow__', - '&': '__and__', - '|': '__or__', - '^': '__xor__', - '<<': '__lshift__', - '>>': '__rshift__', - '==': '__eq__', - '!=': '__ne__', - '<': '__lt__', - '>=': '__ge__', - '>': '__gt__', - '<=': '__le__', - 'in': '__contains__', -} +class AssignmentExpr(Expression): + """Assignment expressions in Python 3.8+, like "a := 2".""" -ops_with_inplace_method = { - '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} - -inplace_operator_methods = set( - '__i' + op_methods[op][2:] for op in ops_with_inplace_method) - -reverse_op_methods = { - '__add__': '__radd__', - '__sub__': '__rsub__', - '__mul__': '__rmul__', - '__truediv__': '__rtruediv__', - '__mod__': '__rmod__', - '__floordiv__': '__rfloordiv__', - '__pow__': '__rpow__', - '__and__': '__rand__', - '__or__': '__ror__', - '__xor__': '__rxor__', - '__lshift__': '__rlshift__', - '__rshift__': '__rrshift__', - '__eq__': '__eq__', - '__ne__': '__ne__', - '__lt__': '__gt__', - '__ge__': '__le__', - '__gt__': '__lt__', - '__le__': '__ge__', -} + __slots__ = ("target", "value") -normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) -reverse_op_method_set = set(reverse_op_methods.values()) + __match_args__ = ("target", "value") + def __init__(self, target: Expression, value: Expression) -> None: + super().__init__() + self.target = target + self.value = value -class OpExpr(Node): - """Binary operation (other than . or [] or comparison operators, - which have specific nodes).""" + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_assignment_expr(self) - op = '' - left = Undefined(Node) - right = Undefined(Node) - # Inferred type for the operator method type (when relevant). - method_type = None # type: mypy.types.Type - def __init__(self, op: str, left: Node, right: Node) -> None: +class OpExpr(Expression): + """Binary operation. + + The dot (.), [] and comparison operators have more specific nodes. + """ + + __slots__ = ( + "op", + "left", + "right", + "method_type", + "right_always", + "right_unreachable", + "analyzed", + ) + + __match_args__ = ("left", "op", "right") + + op: str # TODO: Enum? + left: Expression + right: Expression + # Inferred type for the operator method type (when relevant). + method_type: mypy.types.Type | None + # Per static analysis only: Is the right side going to be evaluated every time? + right_always: bool + # Per static analysis only: Is the right side unreachable? + right_unreachable: bool + # Used for expressions that represent a type "X | Y" in some contexts + analyzed: TypeAliasExpr | None + + def __init__( + self, op: str, left: Expression, right: Expression, analyzed: TypeAliasExpr | None = None + ) -> None: + super().__init__() self.op = op self.left = left self.right = right - self.literal = min(self.left.literal, self.right.literal) - self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) + self.method_type = None + self.right_always = False + self.right_unreachable = False + self.analyzed = analyzed - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_op_expr(self) -class ComparisonExpr(Node): +class ComparisonExpr(Expression): """Comparison expression (e.g. a < b > c < d).""" - operators = Undefined(List[str]) - operands = Undefined(List[Node]) + __slots__ = ("operators", "operands", "method_types") + + __match_args__ = ("operands", "operators") + + operators: list[str] + operands: list[Expression] # Inferred type for the operator methods (when relevant; None for 'is'). - method_types = Undefined(List["mypy.types.Type"]) + method_types: list[mypy.types.Type | None] - def __init__(self, operators: List[str], operands: List[Node]) -> None: + def __init__(self, operators: list[str], operands: list[Expression]) -> None: + super().__init__() self.operators = operators self.operands = operands self.method_types = [] - self.literal = min(o.literal for o in self.operands) - self.literal_hash = ( ('Comparison',) + tuple(operators) + - tuple(o.literal_hash for o in operands) ) - def accept(self, visitor: NodeVisitor[T]) -> T: + def pairwise(self) -> Iterator[tuple[str, Expression, Expression]]: + """If this comparison expr is "a < b is c == d", yields the sequence + ("<", a, b), ("is", b, c), ("==", c, d) + """ + for i, operator in enumerate(self.operators): + yield operator, self.operands[i], self.operands[i + 1] + + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_comparison_expr(self) -class SliceExpr(Node): +class SliceExpr(Expression): """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). This is only valid as index in index expressions. """ - begin_index = Undefined(Node) # May be None - end_index = Undefined(Node) # May be None - stride = Undefined(Node) # May be None + __slots__ = ("begin_index", "end_index", "stride") + + __match_args__ = ("begin_index", "end_index", "stride") - def __init__(self, begin_index: Node, end_index: Node, - stride: Node) -> None: + begin_index: Expression | None + end_index: Expression | None + stride: Expression | None + + def __init__( + self, + begin_index: Expression | None, + end_index: Expression | None, + stride: Expression | None, + ) -> None: + super().__init__() self.begin_index = begin_index self.end_index = end_index self.stride = stride - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_slice_expr(self) -class CastExpr(Node): +class CastExpr(Expression): """Cast expression cast(type, expr).""" - expr = Undefined(Node) - type = Undefined('mypy.types.Type') + __slots__ = ("expr", "type") + + __match_args__ = ("expr", "type") - def __init__(self, expr: Node, typ: 'mypy.types.Type') -> None: + expr: Expression + type: mypy.types.Type + + def __init__(self, expr: Expression, typ: mypy.types.Type) -> None: + super().__init__() self.expr = expr self.type = typ - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_cast_expr(self) -class SuperExpr(Node): +class AssertTypeExpr(Expression): + """Represents a typing.assert_type(expr, type) call.""" + + __slots__ = ("expr", "type") + + __match_args__ = ("expr", "type") + + expr: Expression + type: mypy.types.Type + + def __init__(self, expr: Expression, typ: mypy.types.Type) -> None: + super().__init__() + self.expr = expr + self.type = typ + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_assert_type_expr(self) + + +class RevealExpr(Expression): + """Reveal type expression reveal_type(expr) or reveal_locals() expression.""" + + __slots__ = ("expr", "kind", "local_nodes", "is_imported") + + __match_args__ = ("expr", "kind", "local_nodes", "is_imported") + + expr: Expression | None + kind: int + local_nodes: list[Var] | None + + def __init__( + self, + kind: int, + expr: Expression | None = None, + local_nodes: list[Var] | None = None, + is_imported: bool = False, + ) -> None: + super().__init__() + self.expr = expr + self.kind = kind + self.local_nodes = local_nodes + self.is_imported = is_imported + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_reveal_expr(self) + + +class SuperExpr(Expression): """Expression super().name""" - name = '' - info = Undefined('TypeInfo') # Type that contains this super expression + __slots__ = ("name", "info", "call") - def __init__(self, name: str) -> None: + __match_args__ = ("name", "call", "info") + + name: str + info: TypeInfo | None # Type that contains this super expression + call: CallExpr # The expression super(...) + + def __init__(self, name: str, call: CallExpr) -> None: + super().__init__() self.name = name + self.call = call + self.info = None - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_super_expr(self) -class FuncExpr(FuncItem): +class LambdaExpr(FuncItem, Expression): """Lambda expression""" + __match_args__ = ("arguments", "arg_names", "arg_kinds", "body") + + @property def name(self) -> str: - return '' + return LAMBDA_NAME - def expr(self) -> Node: + def expr(self) -> Expression: """Return the expression (the body) of the lambda.""" - ret = cast(ReturnStmt, self.body.body[0]) - return ret.expr + ret = self.body.body[-1] + assert isinstance(ret, ReturnStmt) + expr = ret.expr + assert expr is not None # lambda can't have empty body + return expr - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_expr(self) + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_lambda_expr(self) + + def is_dynamic(self) -> bool: + return False -class ListExpr(Node): +class ListExpr(Expression): """List literal expression [...].""" - items = Undefined(List[Node]) + __slots__ = ("items",) + + __match_args__ = ("items",) - def __init__(self, items: List[Node]) -> None: + items: list[Expression] + + def __init__(self, items: list[Expression]) -> None: + super().__init__() self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_list_expr(self) -class DictExpr(Node): +class DictExpr(Expression): """Dictionary literal expression {key: value, ...}.""" - items = Undefined(List[Tuple[Node, Node]]) + __slots__ = ("items",) - def __init__(self, items: List[Tuple[Node, Node]]) -> None: + __match_args__ = ("items",) + + items: list[tuple[Expression | None, Expression]] + + def __init__(self, items: list[tuple[Expression | None, Expression]]) -> None: + super().__init__() self.items = items - if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES - for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Dict',) + tuple((x[0].literal_hash, x[1].literal_hash) for x in items) - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_dict_expr(self) -class TupleExpr(Node): - """Tuple literal expression (..., ...)""" +class TupleExpr(Expression): + """Tuple literal expression (..., ...) + + Also lvalue sequences (..., ...) and [..., ...]""" + + __slots__ = ("items",) - items = Undefined(List[Node]) + __match_args__ = ("items",) - def __init__(self, items: List[Node]) -> None: + items: list[Expression] + + def __init__(self, items: list[Expression]) -> None: + super().__init__() self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_tuple_expr(self) -class SetExpr(Node): +class SetExpr(Expression): """Set literal expression {value, ...}.""" - items = Undefined(List[Node]) + __slots__ = ("items",) + + __match_args__ = ("items",) - def __init__(self, items: List[Node]) -> None: + items: list[Expression] + + def __init__(self, items: list[Expression]) -> None: + super().__init__() self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_set_expr(self) -class GeneratorExpr(Node): +class GeneratorExpr(Expression): """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" - left_expr = Undefined(Node) - sequences_expr = Undefined(List[Node]) - condlists = Undefined(List[List[Node]]) - indices = Undefined(List[Node]) - - def __init__(self, left_expr: Node, indices: List[Node], - sequences: List[Node], condlists: List[List[Node]]) -> None: + __slots__ = ("left_expr", "sequences", "condlists", "is_async", "indices") + + __match_args__ = ("left_expr", "indices", "sequences", "condlists") + + left_expr: Expression + sequences: list[Expression] + condlists: list[list[Expression]] + is_async: list[bool] + indices: list[Lvalue] + + def __init__( + self, + left_expr: Expression, + indices: list[Lvalue], + sequences: list[Expression], + condlists: list[list[Expression]], + is_async: list[bool], + ) -> None: + super().__init__() self.left_expr = left_expr self.sequences = sequences self.condlists = condlists self.indices = indices + self.is_async = is_async - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_generator_expr(self) -class ListComprehension(Node): +class ListComprehension(Expression): """List comprehension (e.g. [x + 1 for x in a])""" - generator = Undefined(GeneratorExpr) + __slots__ = ("generator",) + + __match_args__ = ("generator",) + + generator: GeneratorExpr def __init__(self, generator: GeneratorExpr) -> None: + super().__init__() self.generator = generator - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_list_comprehension(self) -class SetComprehension(Node): +class SetComprehension(Expression): """Set comprehension (e.g. {x + 1 for x in a})""" - generator = Undefined(GeneratorExpr) + __slots__ = ("generator",) + + __match_args__ = ("generator",) + + generator: GeneratorExpr def __init__(self, generator: GeneratorExpr) -> None: + super().__init__() self.generator = generator - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_set_comprehension(self) -class DictionaryComprehension(Node): +class DictionaryComprehension(Expression): """Dictionary comprehension (e.g. {k: v for k, v in a}""" - key = Undefined(Node) - value = Undefined(Node) - sequences_expr = Undefined(List[Node]) - condlists = Undefined(List[List[Node]]) - indices = Undefined(List[Node]) - - def __init__(self, key: Node, value: Node, indices: List[Node], - sequences: List[Node], condlists: List[List[Node]]) -> None: + __slots__ = ("key", "value", "sequences", "condlists", "is_async", "indices") + + __match_args__ = ("key", "value", "indices", "sequences", "condlists") + + key: Expression + value: Expression + sequences: list[Expression] + condlists: list[list[Expression]] + is_async: list[bool] + indices: list[Lvalue] + + def __init__( + self, + key: Expression, + value: Expression, + indices: list[Lvalue], + sequences: list[Expression], + condlists: list[list[Expression]], + is_async: list[bool], + ) -> None: + super().__init__() self.key = key self.value = value self.sequences = sequences self.condlists = condlists self.indices = indices + self.is_async = is_async - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_dictionary_comprehension(self) -class ConditionalExpr(Node): +class ConditionalExpr(Expression): """Conditional expression (e.g. x if y else z)""" - cond = Undefined(Node) - if_expr = Undefined(Node) - else_expr = Undefined(Node) + __slots__ = ("cond", "if_expr", "else_expr") - def __init__(self, cond: Node, if_expr: Node, else_expr: Node) -> None: + __match_args__ = ("if_expr", "cond", "else_expr") + + cond: Expression + if_expr: Expression + else_expr: Expression + + def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: + super().__init__() self.cond = cond self.if_expr = if_expr self.else_expr = else_expr - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_conditional_expr(self) -class UndefinedExpr(Node): - """Expression Undefined(type), used as an initializer. - - This is used to declare the type of a variable without initializing with - a proper value. For example: - - x = Undefined(List[int]) - """ - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_undefined_expr(self) +class TypeApplication(Expression): + """Type application expr[type, ...]""" + __slots__ = ("expr", "types") -class TypeApplication(Node): - """Type application expr[type, ...]""" + __match_args__ = ("expr", "types") - expr = Undefined(Node) - types = Undefined(List['mypy.types.Type']) + expr: Expression + types: list[mypy.types.Type] - def __init__(self, expr: Node, types: List['mypy.types.Type']) -> None: + def __init__(self, expr: Expression, types: list[mypy.types.Type]) -> None: + super().__init__() self.expr = expr self.types = types - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_type_application(self) -class TypeVarExpr(SymbolNode): - """Type variable expression typevar(...).""" +# Variance of a type variable. For example, T in the definition of +# List[T] is invariant, so List[int] is not a subtype of List[object], +# and also List[object] is not a subtype of List[int]. +# +# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of +# Iterable[object], but not vice versa. +# +# If T is contravariant in Foo[T], Foo[object] is a subtype of +# Foo[int], but not vice versa. +INVARIANT: Final = 0 +COVARIANT: Final = 1 +CONTRAVARIANT: Final = 2 +VARIANCE_NOT_READY: Final = 3 # Variance hasn't been inferred (using Python 3.12 syntax) - _name = '' - _fullname = '' - # Value restriction: only types in the list are valid as values. If the - # list is empty, there is no restriction. - values = Undefined(List['mypy.types.Type']) - def __init__(self, name: str, fullname: str, - values: List['mypy.types.Type']) -> None: +class TypeVarLikeExpr(SymbolNode, Expression): + """Base class for TypeVarExpr, ParamSpecExpr and TypeVarTupleExpr. + + Note that they are constructed by the semantic analyzer. + """ + + __slots__ = ("_name", "_fullname", "upper_bound", "default", "variance", "is_new_style") + + _name: str + _fullname: str + # Upper bound: only subtypes of upper_bound are valid as values. By default + # this is 'object', meaning no restriction. + upper_bound: mypy.types.Type + # Default: used to resolve the TypeVar if the default is not explicitly given. + # By default this is 'AnyType(TypeOfAny.from_omitted_generics)'. See PEP 696. + default: mypy.types.Type + # Variance of the type variable. Invariant is the default. + # TypeVar(..., covariant=True) defines a covariant type variable. + # TypeVar(..., contravariant=True) defines a contravariant type + # variable. + variance: int + + def __init__( + self, + name: str, + fullname: str, + upper_bound: mypy.types.Type, + default: mypy.types.Type, + variance: int = INVARIANT, + is_new_style: bool = False, + line: int = -1, + ) -> None: + super().__init__(line=line) self._name = name self._fullname = fullname - self.values = values + self.upper_bound = upper_bound + self.default = default + self.variance = variance + self.is_new_style = is_new_style + @property def name(self) -> str: return self._name + @property def fullname(self) -> str: return self._fullname - def accept(self, visitor: NodeVisitor[T]) -> T: + +class TypeVarExpr(TypeVarLikeExpr): + """Type variable expression TypeVar(...). + + This is also used to represent type variables in symbol tables. + + A type variable is not valid as a type unless bound in a TypeVarLikeScope. + That happens within: + + 1. a generic class that uses the type variable as a type argument or + 2. a generic function that refers to the type variable in its signature. + """ + + __slots__ = ("values",) + + __match_args__ = ("name", "values", "upper_bound", "default") + + # Value restriction: only types in the list are valid as values. If the + # list is empty, there is no restriction. + values: list[mypy.types.Type] + + def __init__( + self, + name: str, + fullname: str, + values: list[mypy.types.Type], + upper_bound: mypy.types.Type, + default: mypy.types.Type, + variance: int = INVARIANT, + is_new_style: bool = False, + line: int = -1, + ) -> None: + super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line) + self.values = values + + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_type_var_expr(self) + def serialize(self) -> JsonDict: + return { + ".class": "TypeVarExpr", + "name": self._name, + "fullname": self._fullname, + "values": [t.serialize() for t in self.values], + "upper_bound": self.upper_bound.serialize(), + "default": self.default.serialize(), + "variance": self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeVarExpr: + assert data[".class"] == "TypeVarExpr" + return TypeVarExpr( + data["name"], + data["fullname"], + [mypy.types.deserialize_type(v) for v in data["values"]], + mypy.types.deserialize_type(data["upper_bound"]), + mypy.types.deserialize_type(data["default"]), + data["variance"], + ) + + +class ParamSpecExpr(TypeVarLikeExpr): + __slots__ = () + + __match_args__ = ("name", "upper_bound", "default") + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_paramspec_expr(self) + + def serialize(self) -> JsonDict: + return { + ".class": "ParamSpecExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": self.upper_bound.serialize(), + "default": self.default.serialize(), + "variance": self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> ParamSpecExpr: + assert data[".class"] == "ParamSpecExpr" + return ParamSpecExpr( + data["name"], + data["fullname"], + mypy.types.deserialize_type(data["upper_bound"]), + mypy.types.deserialize_type(data["default"]), + data["variance"], + ) + + +class TypeVarTupleExpr(TypeVarLikeExpr): + """Type variable tuple expression TypeVarTuple(...).""" + + __slots__ = "tuple_fallback" + + tuple_fallback: mypy.types.Instance + + __match_args__ = ("name", "upper_bound", "default") + + def __init__( + self, + name: str, + fullname: str, + upper_bound: mypy.types.Type, + tuple_fallback: mypy.types.Instance, + default: mypy.types.Type, + variance: int = INVARIANT, + is_new_style: bool = False, + line: int = -1, + ) -> None: + super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line) + self.tuple_fallback = tuple_fallback + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_var_tuple_expr(self) + + def serialize(self) -> JsonDict: + return { + ".class": "TypeVarTupleExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": self.upper_bound.serialize(), + "tuple_fallback": self.tuple_fallback.serialize(), + "default": self.default.serialize(), + "variance": self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeVarTupleExpr: + assert data[".class"] == "TypeVarTupleExpr" + return TypeVarTupleExpr( + data["name"], + data["fullname"], + mypy.types.deserialize_type(data["upper_bound"]), + mypy.types.Instance.deserialize(data["tuple_fallback"]), + mypy.types.deserialize_type(data["default"]), + data["variance"], + ) + + +class TypeAliasExpr(Expression): + """Type alias expression (rvalue).""" + + __slots__ = ("node",) + + __match_args__ = ("node",) + + node: TypeAlias + + def __init__(self, node: TypeAlias) -> None: + super().__init__() + self.node = node + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_alias_expr(self) + + +class NamedTupleExpr(Expression): + """Named tuple expression namedtuple(...) or NamedTuple(...).""" -class NamedTupleExpr(Node): - """Named tuple expression namedtuple(...).""" + __slots__ = ("info", "is_typed") + + __match_args__ = ("info",) # The class representation of this named tuple (its tuple_type attribute contains # the tuple item types) - info = Undefined('TypeInfo') + info: TypeInfo + is_typed: bool # whether this class was created with typing(_extensions).NamedTuple - def __init__(self, info: 'TypeInfo') -> None: + def __init__(self, info: TypeInfo, is_typed: bool = False) -> None: + super().__init__() self.info = info + self.is_typed = is_typed - def accept(self, visitor: NodeVisitor[T]) -> T: + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_namedtuple_expr(self) -class DucktypeExpr(Node): - """Ducktype class decorator expression ducktype(...).""" +class TypedDictExpr(Expression): + """Typed dict expression TypedDict(...).""" + + __slots__ = ("info",) + + __match_args__ = ("info",) + + # The class representation of this typed dict + info: TypeInfo + + def __init__(self, info: TypeInfo) -> None: + super().__init__() + self.info = info - type = Undefined('mypy.types.Type') + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_typeddict_expr(self) - def __init__(self, type: 'mypy.types.Type') -> None: + +class EnumCallExpr(Expression): + """Named tuple expression Enum('name', 'val1 val2 ...').""" + + __slots__ = ("info", "items", "values") + + __match_args__ = ("info", "items", "values") + + # The class representation of this enumerated type + info: TypeInfo + # The item names (for debugging) + items: list[str] + values: list[Expression | None] + + def __init__(self, info: TypeInfo, items: list[str], values: list[Expression | None]) -> None: + super().__init__() + self.info = info + self.items = items + self.values = values + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_enum_call_expr(self) + + +class PromoteExpr(Expression): + """Ducktype class decorator expression _promote(...).""" + + __slots__ = ("type",) + + type: mypy.types.ProperType + + def __init__(self, type: mypy.types.ProperType) -> None: + super().__init__() self.type = type - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_ducktype_expr(self) + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit__promote_expr(self) -class DisjointclassExpr(Node): - """Disjoint class class decorator expression disjointclass(cls).""" +class NewTypeExpr(Expression): + """NewType expression NewType(...).""" - cls = Undefined(RefExpr) + __slots__ = ("name", "old_type", "info") - def __init__(self, cls: RefExpr) -> None: - self.cls = cls + __match_args__ = ("name", "old_type", "info") + + name: str + # The base type (the second argument to NewType) + old_type: mypy.types.Type | None + # The synthesized class representing the new type (inherits old_type) + info: TypeInfo | None + + def __init__( + self, name: str, old_type: mypy.types.Type | None, line: int, column: int + ) -> None: + super().__init__(line=line, column=column) + self.name = name + self.old_type = old_type + self.info = None + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_newtype_expr(self) - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_disjointclass_expr(self) +class AwaitExpr(Expression): + """Await expression (await ...).""" -class TempNode(Node): + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression + + def __init__(self, expr: Expression) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_await_expr(self) + + +# Constants + + +class TempNode(Expression): """Temporary dummy node used during type checking. This node is not present in the original program; it is just an artifact @@ -1420,91 +2859,394 @@ class TempNode(Node): some fixed type. """ - type = Undefined('mypy.types.Type') + __slots__ = ("type", "no_rhs") + + type: mypy.types.Type + # Is this TempNode used to indicate absence of a right hand side in an annotated assignment? + # (e.g. for 'x: int' the rvalue is TempNode(AnyType(TypeOfAny.special_form), no_rhs=True)) + no_rhs: bool - def __init__(self, typ: 'mypy.types.Type') -> None: + def __init__( + self, typ: mypy.types.Type, no_rhs: bool = False, *, context: Context | None = None + ) -> None: + """Construct a dummy node; optionally borrow line/column from context object.""" + super().__init__() self.type = typ + self.no_rhs = no_rhs + if context is not None: + self.line = context.line + self.column = context.column - def accept(self, visitor: NodeVisitor[T]) -> T: + def __repr__(self) -> str: + return "TempNode:%d(%s)" % (self.line, str(self.type)) + + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_temp_node(self) +# Special attributes not collected as protocol members by Python 3.12 +# See typing._SPECIAL_NAMES +EXCLUDED_PROTOCOL_ATTRIBUTES: Final = frozenset( + { + "__abstractmethods__", + "__annotations__", + "__dict__", + "__doc__", + "__init__", + "__module__", + "__new__", + "__slots__", + "__subclasshook__", + "__weakref__", + "__class_getitem__", # Since Python 3.9 + } +) + +# Attributes that can optionally be defined in the body of a subclass of +# enum.Enum but are removed from the class __dict__ by EnumMeta. +EXCLUDED_ENUM_ATTRIBUTES: Final = frozenset({"_ignore_", "_order_", "__order__"}) + + class TypeInfo(SymbolNode): - """Class representing the type structure of a single class. + """The type structure of a single class. + + Each TypeInfo corresponds one-to-one to a ClassDef, which + represents the AST of the class. - The corresponding ClassDef instance represents the parse tree of - the class. + In type-theory terms, this is a "type constructor", and if the + class is generic then it will be a type constructor of higher kind. + Where the class is used in an actual type, it's in the form of an + Instance, which amounts to a type application of the tycon to + the appropriate number of arguments. """ - _fullname = None # type: str # Fully qualified name - defn = Undefined(ClassDef) # Corresponding ClassDef + __slots__ = ( + "_fullname", + "module_name", + "defn", + "mro", + "_mro_refs", + "bad_mro", + "is_final", + "declared_metaclass", + "metaclass_type", + "names", + "is_abstract", + "is_protocol", + "runtime_protocol", + "abstract_attributes", + "deletable_attributes", + "slots", + "assuming", + "assuming_proper", + "inferring", + "is_enum", + "fallback_to_any", + "meta_fallback_to_any", + "type_vars", + "has_param_spec_type", + "bases", + "_promote", + "tuple_type", + "special_alias", + "is_named_tuple", + "typeddict_type", + "is_newtype", + "is_intersection", + "metadata", + "alt_promote", + "has_type_var_tuple_type", + "type_var_tuple_prefix", + "type_var_tuple_suffix", + "self_type", + "dataclass_transform_spec", + "is_type_check_only", + "deprecated", + ) + + _fullname: str # Fully qualified name + # Fully qualified name for the module this type was defined in. This + # information is also in the fullname, but is harder to extract in the + # case of nested class definitions. + module_name: str + defn: ClassDef # Corresponding ClassDef # Method Resolution Order: the order of looking up attributes. The first # value always to refers to this class. - mro = Undefined(List['TypeInfo']) - subtypes = Undefined(Set['TypeInfo']) # Direct subclasses encountered so far - names = Undefined('SymbolTable') # Names defined directly in this type - is_abstract = False # Does the class have any abstract attributes? - abstract_attributes = Undefined(List[str]) - # All classes in this build unit that are disjoint with this class. - disjoint_classes = Undefined(List['TypeInfo']) - # Targets of disjointclass declarations present in this class only (for - # generating error messages). - disjointclass_decls = Undefined(List['TypeInfo']) + mro: list[TypeInfo] + # Used to stash the names of the mro classes temporarily between + # deserialization and fixup. See deserialize() for why. + _mro_refs: list[str] | None + bad_mro: bool # Could not construct full MRO + is_final: bool + + declared_metaclass: mypy.types.Instance | None + metaclass_type: mypy.types.Instance | None + + names: SymbolTable # Names defined directly in this type + is_abstract: bool # Does the class have any abstract attributes? + is_protocol: bool # Is this a protocol class? + runtime_protocol: bool # Does this protocol support isinstance checks? + # List of names of abstract attributes together with their abstract status. + # The abstract status must be one of `NOT_ABSTRACT`, `IS_ABSTRACT`, `IMPLICITLY_ABSTRACT`. + abstract_attributes: list[tuple[str, int]] + deletable_attributes: list[str] # Used by mypyc only + # Does this type have concrete `__slots__` defined? + # If class does not have `__slots__` defined then it is `None`, + # if it has empty `__slots__` then it is an empty set. + slots: set[str] | None + + # The attributes 'assuming' and 'assuming_proper' represent structural subtype matrices. + # + # In languages with structural subtyping, one can keep a global subtype matrix like this: + # . A B C . + # A 1 0 0 + # B 1 1 1 + # C 1 0 1 + # . + # where 1 indicates that the type in corresponding row is a subtype of the type + # in corresponding column. This matrix typically starts filled with all 1's and + # a typechecker tries to "disprove" every subtyping relation using atomic (or nominal) types. + # However, we don't want to keep this huge global state. Instead, we keep the subtype + # information in the form of list of pairs (subtype, supertype) shared by all Instances + # with given supertype's TypeInfo. When we enter a subtype check we push a pair in this list + # thus assuming that we started with 1 in corresponding matrix element. Such algorithm allows + # to treat recursive and mutually recursive protocols and other kinds of complex situations. + # + # If concurrent/parallel type checking will be added in future, + # then there should be one matrix per thread/process to avoid false negatives + # during the type checking phase. + assuming: list[tuple[mypy.types.Instance, mypy.types.Instance]] + assuming_proper: list[tuple[mypy.types.Instance, mypy.types.Instance]] + # Ditto for temporary 'inferring' stack of recursive constraint inference. + # It contains Instances of protocol types that appeared as an argument to + # constraints.infer_constraints(). We need 'inferring' to avoid infinite recursion for + # recursive and mutually recursive protocols. + # + # We make 'assuming' and 'inferring' attributes here instead of passing they as kwargs, + # since this would require to pass them in many dozens of calls. In particular, + # there is a dependency infer_constraint -> is_subtype -> is_callable_subtype -> + # -> infer_constraints. + inferring: list[mypy.types.Instance] + # 'inferring' and 'assuming' can't be made sets, since we need to use + # is_same_type to correctly treat unions. + + # Classes inheriting from Enum shadow their true members with a __getattr__, so we + # have to treat them as a special case. + is_enum: bool + # If true, any unknown attributes should have type 'Any' instead + # of generating a type error. This would be true if there is a + # base class with type 'Any', but other use cases may be + # possible. This is similar to having __getattr__ that returns Any + # (and __setattr__), but without the __getattr__ method. + fallback_to_any: bool + + # Same as above but for cases where metaclass has type Any. This will suppress + # all attribute errors only for *class object* access. + meta_fallback_to_any: bool # Information related to type annotations. - # Generic type variable names - type_vars = Undefined(List[str]) + # Generic type variable names (full names) + type_vars: list[str] - # Direct base classes. - bases = Undefined(List['mypy.types.Instance']) + # Whether this class has a ParamSpec type variable + has_param_spec_type: bool - # Duck type compatibility (ducktype decorator) - ducktype = None # type: mypy.types.Type + # Direct base classes. + bases: list[mypy.types.Instance] + + # Another type which this type will be treated as a subtype of, + # even though it's not a subclass in Python. The non-standard + # `@_promote` decorator introduces this, and there are also + # several builtin examples, in particular `int` -> `float`. + _promote: list[mypy.types.ProperType] + + # This is used for promoting native integer types such as 'i64' to + # 'int'. (_promote is used for the other direction.) This only + # supports one-step promotions (e.g., i64 -> int, not + # i64 -> int -> float, and this isn't used to promote in joins. + # + # This results in some unintuitive results, such as that even + # though i64 is compatible with int and int is compatible with + # float, i64 is *not* compatible with float. + alt_promote: mypy.types.Instance | None # Representation of a Tuple[...] base class, if the class has any # (e.g., for named tuples). If this is not None, the actual Type # object used for this class is not an Instance but a TupleType; # the corresponding Instance is set as the fallback type of the # tuple type. - tuple_type = None # type: mypy.types.TupleType - - def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: + tuple_type: mypy.types.TupleType | None + + # Is this a named tuple type? + is_named_tuple: bool + + # If this class is defined by the TypedDict type constructor, + # then this is not None. + typeddict_type: mypy.types.TypedDictType | None + + # Is this a newtype type? + is_newtype: bool + + # Is this a synthesized intersection type? + is_intersection: bool + + # This is a dictionary that will be serialized and un-serialized as is. + # It is useful for plugins to add their data to save in the cache. + metadata: dict[str, JsonDict] + + # Store type alias representing this type (for named tuples and TypedDicts). + # Although definitions of these types are stored in symbol tables as TypeInfo, + # when a type analyzer will find them, it should construct a TupleType, or + # a TypedDict type. However, we can't use the plain types, since if the definition + # is recursive, this will create an actual recursive structure of types (i.e. as + # internal Python objects) causing infinite recursions everywhere during type checking. + # To overcome this, we create a TypeAlias node, that will point to these types. + # We store this node in the `special_alias` attribute, because it must be the same node + # in case we are doing multiple semantic analysis passes. + special_alias: TypeAlias | None + + # Shared type variable for typing.Self in this class (if used, otherwise None). + self_type: mypy.types.TypeVarType | None + + # Added if the corresponding class is directly decorated with `typing.dataclass_transform` + dataclass_transform_spec: DataclassTransformSpec | None + + # Is set to `True` when class is decorated with `@typing.type_check_only` + is_type_check_only: bool + + # The type's deprecation message (in case it is deprecated) + deprecated: str | None + + FLAGS: Final = [ + "is_abstract", + "is_enum", + "fallback_to_any", + "meta_fallback_to_any", + "is_named_tuple", + "is_newtype", + "is_protocol", + "runtime_protocol", + "is_final", + "is_intersection", + ] + + def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None: """Initialize a TypeInfo.""" + super().__init__() + self._fullname = defn.fullname self.names = names self.defn = defn - self.subtypes = set() - self.mro = [] + self.module_name = module_name self.type_vars = [] + self.has_param_spec_type = False + self.has_type_var_tuple_type = False self.bases = [] - self._fullname = defn.fullname + self.mro = [] + self._mro_refs = None + self.bad_mro = False + self.declared_metaclass = None + self.metaclass_type = None self.is_abstract = False self.abstract_attributes = [] - self.disjoint_classes = [] - self.disjointclass_decls = [] - if defn.type_vars: - for vd in defn.type_vars: + self.deletable_attributes = [] + self.slots = None + self.assuming = [] + self.assuming_proper = [] + self.inferring = [] + self.is_protocol = False + self.runtime_protocol = False + self.type_var_tuple_prefix: int | None = None + self.type_var_tuple_suffix: int | None = None + self.add_type_vars() + self.is_final = False + self.is_enum = False + self.fallback_to_any = False + self.meta_fallback_to_any = False + self._promote = [] + self.alt_promote = None + self.tuple_type = None + self.special_alias = None + self.is_named_tuple = False + self.typeddict_type = None + self.is_newtype = False + self.is_intersection = False + self.metadata = {} + self.self_type = None + self.dataclass_transform_spec = None + self.is_type_check_only = False + self.deprecated = None + + def add_type_vars(self) -> None: + self.has_type_var_tuple_type = False + if self.defn.type_vars: + for i, vd in enumerate(self.defn.type_vars): + if isinstance(vd, mypy.types.ParamSpecType): + self.has_param_spec_type = True + if isinstance(vd, mypy.types.TypeVarTupleType): + assert not self.has_type_var_tuple_type + self.has_type_var_tuple_type = True + self.type_var_tuple_prefix = i + self.type_var_tuple_suffix = len(self.defn.type_vars) - i - 1 self.type_vars.append(vd.name) + @property def name(self) -> str: """Short name.""" return self.defn.name + @property def fullname(self) -> str: return self._fullname def is_generic(self) -> bool: """Is the type generic (i.e. does it have type variables)?""" - return self.type_vars is not None and len(self.type_vars) > 0 + return len(self.type_vars) > 0 - def get(self, name: str) -> 'SymbolTableNode': + def get(self, name: str) -> SymbolTableNode | None: for cls in self.mro: n = cls.names.get(name) if n: return n return None - def __getitem__(self, name: str) -> 'SymbolTableNode': + def get_containing_type_info(self, name: str) -> TypeInfo | None: + for cls in self.mro: + if name in cls.names: + return cls + return None + + @property + def protocol_members(self) -> list[str]: + # Protocol members are names of all attributes/methods defined in a protocol + # and in all its supertypes (except for 'object'). + members: set[str] = set() + assert self.mro, "This property can be only accessed after MRO is (re-)calculated" + for base in self.mro[:-1]: # we skip "object" since everyone implements it + if base.is_protocol: + for name, node in base.names.items(): + if isinstance(node.node, (TypeAlias, TypeVarExpr, MypyFile)): + # These are auxiliary definitions (and type aliases are prohibited). + continue + if name in EXCLUDED_PROTOCOL_ATTRIBUTES: + continue + members.add(name) + return sorted(members) + + @property + def enum_members(self) -> list[str]: + return [ + name + for name, sym in self.names.items() + if ( + isinstance(sym.node, Var) + and name not in EXCLUDED_ENUM_ATTRIBUTES + and not name.startswith("__") + and sym.node.has_explicit_value + ) + ] + + def __getitem__(self, name: str) -> SymbolTableNode: n = self.get(name) if n: return n @@ -1512,57 +3254,51 @@ def __getitem__(self, name: str) -> 'SymbolTableNode': raise KeyError(name) def __repr__(self) -> str: - return '' % self.fullname() + return f"" - # IDEA: Refactor the has* methods to be more consistent and document - # them. + def __bool__(self) -> bool: + # We defined this here instead of just overriding it in + # FakeInfo so that mypyc can generate a direct call instead of + # using the generic bool handling. + return not isinstance(self, FakeInfo) def has_readable_member(self, name: str) -> bool: return self.get(name) is not None - def has_writable_member(self, name: str) -> bool: - return self.has_var(name) - - def has_var(self, name: str) -> bool: - return self.get_var(name) is not None - - def has_method(self, name: str) -> bool: - return self.get_method(name) is not None - - def get_var(self, name: str) -> Var: - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, Var): - return cast(Var, node) - else: - return None - return None - - def get_var_or_getter(self, name: str) -> SymbolNode: - # TODO getter - return self.get_var(name) - - def get_var_or_setter(self, name: str) -> SymbolNode: - # TODO setter - return self.get_var(name) - - def get_method(self, name: str) -> FuncBase: + def get_method(self, name: str) -> FuncBase | Decorator | None: for cls in self.mro: if name in cls.names: node = cls.names[name].node if isinstance(node, FuncBase): return node + elif isinstance(node, Decorator): # Two `if`s make `mypyc` happy + return node else: return None return None - def calculate_mro(self) -> None: - """Calculate and set mro (method resolution order). + def calculate_metaclass_type(self) -> mypy.types.Instance | None: + declared = self.declared_metaclass + if declared is not None and not declared.type.has_base("builtins.type"): + return declared + if self._fullname == "builtins.type": + return mypy.types.Instance(self, []) + candidates = [ + s.declared_metaclass + for s in self.mro + if s.declared_metaclass is not None and s.declared_metaclass.type is not None + ] + for c in candidates: + if all(other.type in c.type.mro for other in candidates): + return c + return None - Raise MroError if cannot determine mro. - """ - self.mro = linearize_hierarchy(self) + def is_metaclass(self) -> bool: + return ( + self.has_base("builtins.type") + or self.fullname == "abc.ABCMeta" + or self.fallback_to_any + ) def has_base(self, fullname: str) -> bool: """Return True if type has a base type with the specified name. @@ -1570,197 +3306,923 @@ def has_base(self, fullname: str) -> bool: This can be either via extension or via implementation. """ for cls in self.mro: - if cls.fullname() == fullname: + if cls.fullname == fullname: return True return False - def all_subtypes(self) -> 'Set[TypeInfo]': - """Return TypeInfos of all subtypes, including this type, as a set.""" - subtypes = set([self]) - for subt in self.subtypes: - for t in subt.all_subtypes(): - subtypes.add(t) - return subtypes - - def all_base_classes(self) -> 'List[TypeInfo]': - """Return a list of base classes, including indirect bases.""" - assert False - - def direct_base_classes(self) -> 'List[TypeInfo]': + def direct_base_classes(self) -> list[TypeInfo]: """Return a direct base classes. Omit base classes of other base classes. """ return [base.type for base in self.bases] + def update_tuple_type(self, typ: mypy.types.TupleType) -> None: + """Update tuple_type and special_alias as needed.""" + self.tuple_type = typ + alias = TypeAlias.from_tuple_type(self) + if not self.special_alias: + self.special_alias = alias + else: + self.special_alias.target = alias.target + + def update_typeddict_type(self, typ: mypy.types.TypedDictType) -> None: + """Update typeddict_type and special_alias as needed.""" + self.typeddict_type = typ + alias = TypeAlias.from_typeddict_type(self) + if not self.special_alias: + self.special_alias = alias + else: + self.special_alias.target = alias.target + def __str__(self) -> str: """Return a string representation of the type. This includes the most important information about the type. """ - base = None # type: str + options = Options() + return self.dump( + str_conv=mypy.strconv.StrConv(options=options), + type_str_conv=mypy.types.TypeStrVisitor(options=options), + ) + + def dump( + self, str_conv: mypy.strconv.StrConv, type_str_conv: mypy.types.TypeStrVisitor + ) -> str: + """Return a string dump of the contents of the TypeInfo.""" + + base: str = "" + + def type_str(typ: mypy.types.Type) -> str: + return typ.accept(type_str_conv) + + head = "TypeInfo" + str_conv.format_id(self) if self.bases: - base = 'Bases({})'.format(', '.join(str(base) - for base in self.bases)) - return dump_tagged(['Name({})'.format(self.fullname()), - base, - ('Names', sorted(self.names.keys()))], - 'TypeInfo') + base = f"Bases({', '.join(type_str(base) for base in self.bases)})" + mro = "Mro({})".format( + ", ".join(item.fullname + str_conv.format_id(item) for item in self.mro) + ) + names = [] + for name in sorted(self.names): + description = name + str_conv.format_id(self.names[name].node) + node = self.names[name].node + if isinstance(node, Var) and node.type: + description += f" ({type_str(node.type)})" + names.append(description) + items = [f"Name({self.fullname})", base, mro, ("Names", names)] + if self.declared_metaclass: + items.append(f"DeclaredMetaclass({type_str(self.declared_metaclass)})") + if self.metaclass_type: + items.append(f"MetaclassType({type_str(self.metaclass_type)})") + return mypy.strconv.dump_tagged(items, head, str_conv=str_conv) + + def serialize(self) -> JsonDict: + # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. + data = { + ".class": "TypeInfo", + "module_name": self.module_name, + "fullname": self.fullname, + "names": self.names.serialize(self.fullname), + "defn": self.defn.serialize(), + "abstract_attributes": self.abstract_attributes, + "type_vars": self.type_vars, + "has_param_spec_type": self.has_param_spec_type, + "bases": [b.serialize() for b in self.bases], + "mro": [c.fullname for c in self.mro], + "_promote": [p.serialize() for p in self._promote], + "alt_promote": None if self.alt_promote is None else self.alt_promote.serialize(), + "declared_metaclass": ( + None if self.declared_metaclass is None else self.declared_metaclass.serialize() + ), + "metaclass_type": ( + None if self.metaclass_type is None else self.metaclass_type.serialize() + ), + "tuple_type": None if self.tuple_type is None else self.tuple_type.serialize(), + "typeddict_type": ( + None if self.typeddict_type is None else self.typeddict_type.serialize() + ), + "flags": get_flags(self, TypeInfo.FLAGS), + "metadata": self.metadata, + "slots": sorted(self.slots) if self.slots is not None else None, + "deletable_attributes": self.deletable_attributes, + "self_type": self.self_type.serialize() if self.self_type is not None else None, + "dataclass_transform_spec": ( + self.dataclass_transform_spec.serialize() + if self.dataclass_transform_spec is not None + else None + ), + "deprecated": self.deprecated, + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeInfo: + names = SymbolTable.deserialize(data["names"]) + defn = ClassDef.deserialize(data["defn"]) + module_name = data["module_name"] + ti = TypeInfo(names, defn, module_name) + ti._fullname = data["fullname"] + # TODO: Is there a reason to reconstruct ti.subtypes? + ti.abstract_attributes = [(attr[0], attr[1]) for attr in data["abstract_attributes"]] + ti.type_vars = data["type_vars"] + ti.has_param_spec_type = data["has_param_spec_type"] + ti.bases = [mypy.types.Instance.deserialize(b) for b in data["bases"]] + _promote = [] + for p in data["_promote"]: + t = mypy.types.deserialize_type(p) + assert isinstance(t, mypy.types.ProperType) + _promote.append(t) + ti._promote = _promote + ti.alt_promote = ( + None + if data["alt_promote"] is None + else mypy.types.Instance.deserialize(data["alt_promote"]) + ) + ti.declared_metaclass = ( + None + if data["declared_metaclass"] is None + else mypy.types.Instance.deserialize(data["declared_metaclass"]) + ) + ti.metaclass_type = ( + None + if data["metaclass_type"] is None + else mypy.types.Instance.deserialize(data["metaclass_type"]) + ) + # NOTE: ti.mro will be set in the fixup phase based on these + # names. The reason we need to store the mro instead of just + # recomputing it from base classes has to do with a subtle + # point about fine-grained incremental: the cache files might + # not be loaded until after a class in the mro has changed its + # bases, which causes the mro to change. If we recomputed our + # mro, we would compute the *new* mro, which leaves us with no + # way to detect that the mro has changed! Thus we need to make + # sure to load the original mro so that once the class is + # rechecked, it can tell that the mro has changed. + ti._mro_refs = data["mro"] + ti.tuple_type = ( + None + if data["tuple_type"] is None + else mypy.types.TupleType.deserialize(data["tuple_type"]) + ) + ti.typeddict_type = ( + None + if data["typeddict_type"] is None + else mypy.types.TypedDictType.deserialize(data["typeddict_type"]) + ) + ti.metadata = data["metadata"] + ti.slots = set(data["slots"]) if data["slots"] is not None else None + ti.deletable_attributes = data["deletable_attributes"] + set_flags(ti, data["flags"]) + st = data["self_type"] + ti.self_type = mypy.types.TypeVarType.deserialize(st) if st is not None else None + if data.get("dataclass_transform_spec") is not None: + ti.dataclass_transform_spec = DataclassTransformSpec.deserialize( + data["dataclass_transform_spec"] + ) + ti.deprecated = data.get("deprecated") + return ti + + +class FakeInfo(TypeInfo): + __slots__ = ("msg",) + + # types.py defines a single instance of this class, called types.NOT_READY. + # This instance is used as a temporary placeholder in the process of de-serialization + # of 'Instance' types. The de-serialization happens in two steps: In the first step, + # Instance.type is set to NOT_READY. In the second step (in fixup.py) it is replaced by + # an actual TypeInfo. If you see the assertion error below, then most probably something + # went wrong during the second step and an 'Instance' that raised this error was not fixed. + # Note: + # 'None' is not used as a dummy value for two reasons: + # 1. This will require around 80-100 asserts to make 'mypy --strict-optional mypy' + # pass cleanly. + # 2. If NOT_READY value is accidentally used somewhere, it will be obvious where the value + # is from, whereas a 'None' value could come from anywhere. + # + # Additionally, this serves as a more general-purpose placeholder + # for missing TypeInfos in a number of places where the excuses + # for not being Optional are a little weaker. + # + # TypeInfo defines a __bool__ method that returns False for FakeInfo + # so that it can be conveniently tested against in the same way that it + # would be if things were properly optional. + def __init__(self, msg: str) -> None: + self.msg = msg + + def __getattribute__(self, attr: str) -> type: + # Handle __class__ so that isinstance still works... + if attr == "__class__": + return object.__getattribute__(self, attr) # type: ignore[no-any-return] + raise AssertionError(object.__getattribute__(self, "msg")) + + +VAR_NO_INFO: Final[TypeInfo] = FakeInfo("Var is lacking info") +CLASSDEF_NO_INFO: Final[TypeInfo] = FakeInfo("ClassDef is lacking info") +FUNC_NO_INFO: Final[TypeInfo] = FakeInfo("FuncBase for non-methods lack info") +MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") + + +class TypeAlias(SymbolNode): + """ + A symbol node representing a type alias. + + Type alias is a static concept, in contrast to variables with types + like Type[...]. Namely: + * type aliases + - can be used in type context (annotations) + - cannot be re-assigned + * variables with type Type[...] + - cannot be used in type context + - but can be re-assigned + + An alias can be defined only by an assignment to a name (not any other lvalues). + + Such assignment defines an alias by default. To define a variable, + an explicit Type[...] annotation is required. As an exception, + at non-global scope non-subscripted rvalue creates a variable even without + an annotation. This exception exists to accommodate the common use case of + class-valued attributes. See SemanticAnalyzerPass2.check_and_set_up_type_alias + for details. + + Aliases can be generic. We use bound type variables for generic aliases, similar + to classes. Essentially, type aliases work as macros that expand textually. + The definition and expansion rules are following: + + 1. An alias targeting a generic class without explicit variables act as + the given class (this doesn't apply to TypedDict, Tuple and Callable, which + are not proper classes but special type constructors): + + A = List + AA = List[Any] + + x: A # same as List[Any] + x: A[int] # same as List[int] + + x: AA # same as List[Any] + x: AA[int] # Error! + + C = Callable # Same as Callable[..., Any] + T = Tuple # Same as Tuple[Any, ...] + + 2. An alias using explicit type variables in its rvalue expects + replacements (type arguments) for these variables. If missing, they + are treated as Any, like for other generics: + + B = List[Tuple[T, T]] + + x: B # same as List[Tuple[Any, Any]] + x: B[int] # same as List[Tuple[int, int]] + + def f(x: B[T]) -> T: ... # without T, Any would be used here + + 3. An alias can be defined using another aliases. In the definition + rvalue the Any substitution doesn't happen for top level unsubscripted + generic classes: + + A = List + B = A # here A is expanded to List, _not_ List[Any], + # to match the Python runtime behaviour + x: B[int] # same as List[int] + C = List[A] # this expands to List[List[Any]] + + AA = List[T] + D = AA # here AA expands to List[Any] + x: D[int] # Error! + + Note: the fact that we support aliases like `A = List` means that the target + type will be initially an instance type with wrong number of type arguments. + Such instances are all fixed either during or after main semantic analysis passes. + We therefore store the difference between `List` and `List[Any]` rvalues (targets) + using the `no_args` flag. See also TypeAliasExpr.no_args. + + Meaning of other fields: + + target: The target type. For generic aliases contains bound type variables + as nested types (currently TypeVar and ParamSpec are supported). + _fullname: Qualified name of this type alias. This is used in particular + to track fine grained dependencies from aliases. + alias_tvars: Type variables used to define this alias. + normalized: Used to distinguish between `A = List`, and `A = list`. Both + are internally stored using `builtins.list` (because `typing.List` is + itself an alias), while the second cannot be subscripted because of + Python runtime limitation. + line and column: Line and column on the original alias definition. + eager: If True, immediately expand alias when referred to (useful for aliases + within functions that can't be looked up from the symbol table) + """ + + __slots__ = ( + "target", + "_fullname", + "alias_tvars", + "no_args", + "normalized", + "_is_recursive", + "eager", + "tvar_tuple_index", + "python_3_12_type_alias", + ) + + __match_args__ = ("name", "target", "alias_tvars", "no_args") + + def __init__( + self, + target: mypy.types.Type, + fullname: str, + line: int, + column: int, + *, + alias_tvars: list[mypy.types.TypeVarLikeType] | None = None, + no_args: bool = False, + normalized: bool = False, + eager: bool = False, + python_3_12_type_alias: bool = False, + ) -> None: + self._fullname = fullname + self.target = target + if alias_tvars is None: + alias_tvars = [] + self.alias_tvars = alias_tvars + self.no_args = no_args + self.normalized = normalized + # This attribute is manipulated by TypeAliasType. If non-None, + # it is the cached value. + self._is_recursive: bool | None = None + self.eager = eager + self.python_3_12_type_alias = python_3_12_type_alias + self.tvar_tuple_index = None + for i, t in enumerate(alias_tvars): + if isinstance(t, mypy.types.TypeVarTupleType): + self.tvar_tuple_index = i + super().__init__(line, column) + + @classmethod + def from_tuple_type(cls, info: TypeInfo) -> TypeAlias: + """Generate an alias to the tuple type described by a given TypeInfo. + + NOTE: this doesn't set type alias type variables (for generic tuple types), + they must be set by the caller (when fully analyzed). + """ + assert info.tuple_type + # TODO: is it possible to refactor this to set the correct type vars here? + return TypeAlias( + info.tuple_type.copy_modified( + # Create an Instance similar to fill_typevars(). + fallback=mypy.types.Instance( + info, mypy.types.type_vars_as_args(info.defn.type_vars) + ) + ), + info.fullname, + info.line, + info.column, + ) + + @classmethod + def from_typeddict_type(cls, info: TypeInfo) -> TypeAlias: + """Generate an alias to the TypedDict type described by a given TypeInfo. + + NOTE: this doesn't set type alias type variables (for generic TypedDicts), + they must be set by the caller (when fully analyzed). + """ + assert info.typeddict_type + # TODO: is it possible to refactor this to set the correct type vars here? + return TypeAlias( + info.typeddict_type.copy_modified( + # Create an Instance similar to fill_typevars(). + fallback=mypy.types.Instance( + info, mypy.types.type_vars_as_args(info.defn.type_vars) + ) + ), + info.fullname, + info.line, + info.column, + ) + + @property + def name(self) -> str: + return self._fullname.split(".")[-1] + + @property + def fullname(self) -> str: + return self._fullname + + @property + def has_param_spec_type(self) -> bool: + return any(isinstance(v, mypy.types.ParamSpecType) for v in self.alias_tvars) + + def serialize(self) -> JsonDict: + data: JsonDict = { + ".class": "TypeAlias", + "fullname": self._fullname, + "target": self.target.serialize(), + "alias_tvars": [v.serialize() for v in self.alias_tvars], + "no_args": self.no_args, + "normalized": self.normalized, + "line": self.line, + "column": self.column, + "python_3_12_type_alias": self.python_3_12_type_alias, + } + return data + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_alias(self) + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeAlias: + assert data[".class"] == "TypeAlias" + fullname = data["fullname"] + alias_tvars = [mypy.types.deserialize_type(v) for v in data["alias_tvars"]] + assert all(isinstance(t, mypy.types.TypeVarLikeType) for t in alias_tvars) + target = mypy.types.deserialize_type(data["target"]) + no_args = data["no_args"] + normalized = data["normalized"] + line = data["line"] + column = data["column"] + python_3_12_type_alias = data["python_3_12_type_alias"] + return cls( + target, + fullname, + line, + column, + alias_tvars=cast(List[mypy.types.TypeVarLikeType], alias_tvars), + no_args=no_args, + normalized=normalized, + python_3_12_type_alias=python_3_12_type_alias, + ) + + +class PlaceholderNode(SymbolNode): + """Temporary symbol node that will later become a real SymbolNode. + + These are only present during semantic analysis when using the new + semantic analyzer. These are created if some essential dependencies + of a definition are not yet complete. + + A typical use is for names imported from a module which is still + incomplete (within an import cycle): + + from m import f # Initially may create PlaceholderNode + + This is particularly important if the imported shadows a name from + an enclosing scope or builtins: + + from m import int # Placeholder avoids mixups with builtins.int + + Another case where this is useful is when there is another definition + or assignment: + + from m import f + def f() -> None: ... + + In the above example, the presence of PlaceholderNode allows us to + handle the second definition as a redefinition. + + They are also used to create PlaceholderType instances for types + that refer to incomplete types. Example: + + class C(Sequence[C]): ... + + We create a PlaceholderNode (with becomes_typeinfo=True) for C so + that the type C in Sequence[C] can be bound. + + Attributes: + + fullname: Full name of the PlaceholderNode. + node: AST node that contains the definition that caused this to + be created. This is useful for tracking order of incomplete definitions + and for debugging. + becomes_typeinfo: If True, this refers something that could later + become a TypeInfo. It can't be used with type variables, in + particular, as this would cause issues with class type variable + detection. + + The long-term purpose of placeholder nodes/types is to evolve into + something that can support general recursive types. + """ + + __slots__ = ("_fullname", "node", "becomes_typeinfo") + + def __init__( + self, fullname: str, node: Node, line: int, *, becomes_typeinfo: bool = False + ) -> None: + self._fullname = fullname + self.node = node + self.becomes_typeinfo = becomes_typeinfo + self.line = line + + @property + def name(self) -> str: + return self._fullname.split(".")[-1] + + @property + def fullname(self) -> str: + return self._fullname + + def serialize(self) -> JsonDict: + assert False, "PlaceholderNode can't be serialized" + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_placeholder_node(self) class SymbolTableNode: - # LDEF/GDEF/MDEF/UNBOUND_TVAR/TVAR/... - kind = None # type: int - # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, - # or None for a bound type variable). - node = Undefined(SymbolNode) - # Type variable id (for bound type variables only) - tvar_id = 0 - # Module id (e.g. "foo.bar") or None - mod_id = '' - # If None, fall back to type of node - type_override = Undefined('mypy.types.Type') - - def __init__(self, kind: int, node: SymbolNode, mod_id: str = None, - typ: 'mypy.types.Type' = None, tvar_id: int = 0) -> None: + """Description of a name binding in a symbol table. + + These are only used as values in module (global), function (local) + and class symbol tables (see SymbolTable). The name that is bound is + the key in SymbolTable. + + Symbol tables don't contain direct references to AST nodes primarily + because there can be multiple symbol table references to a single + AST node (due to imports and aliases), and different references can + behave differently. This class describes the unique properties of + each reference. + + The most fundamental attribute is 'node', which is the AST node that + the name refers to. + + The kind is usually one of LDEF, GDEF or MDEF, depending on the scope + of the definition. These three kinds can usually be used + interchangeably and the difference between local, global and class + scopes is mostly descriptive, with no semantic significance. + However, some tools that consume mypy ASTs may care about these so + they should be correct. + + Attributes: + node: AST node of definition. Among others, this can be one of + FuncDef, Var, TypeInfo, TypeVarExpr or MypyFile -- or None + for cross_ref that hasn't been fixed up yet. + kind: Kind of node. Possible values: + - LDEF: local definition + - GDEF: global (module-level) definition + - MDEF: class member definition + - UNBOUND_IMPORTED: temporary kind for imported names (we + don't know the final kind yet) + module_public: If False, this name won't be imported via + 'from import *'. This has no effect on names within + classes. + module_hidden: If True, the name will be never exported (needed for + stub files) + cross_ref: For deserialized MypyFile nodes, the referenced module + name; for other nodes, optionally the name of the referenced object. + implicit: Was this defined by assignment to self attribute? + plugin_generated: Was this symbol generated by a plugin? + (And therefore needs to be removed in aststrip.) + no_serialize: Do not serialize this node if True. This is used to prevent + keys in the cache that refer to modules on which this file does not + depend. Currently this can happen if there is a module not in build + used e.g. like this: + import a.b.c # type: ignore + This will add a submodule symbol to parent module `a` symbol table, + but `a.b` is _not_ added as its dependency. Therefore, we should + not serialize these symbols as they may not be found during fixup + phase, instead they will be re-added during subsequent patch parents + phase. + TODO: Refactor build.py to make dependency tracking more transparent + and/or refactor look-up functions to not require parent patching. + + NOTE: No other attributes should be added to this class unless they + are shared by all node kinds. + """ + + __slots__ = ( + "kind", + "node", + "module_public", + "module_hidden", + "cross_ref", + "implicit", + "plugin_generated", + "no_serialize", + ) + + def __init__( + self, + kind: int, + node: SymbolNode | None, + module_public: bool = True, + implicit: bool = False, + module_hidden: bool = False, + *, + plugin_generated: bool = False, + no_serialize: bool = False, + ) -> None: self.kind = kind self.node = node - self.type_override = typ - self.mod_id = mod_id - self.tvar_id = tvar_id + self.module_public = module_public + self.implicit = implicit + self.module_hidden = module_hidden + self.cross_ref: str | None = None + self.plugin_generated = plugin_generated + self.no_serialize = no_serialize @property - def fullname(self) -> str: + def fullname(self) -> str | None: if self.node is not None: - return self.node.fullname() + return self.node.fullname else: return None @property - def type(self) -> 'mypy.types.Type': - # IDEA: Get rid of the Any type. - node = self.node # type: Any - if self.type_override is not None: - return self.type_override - elif ((isinstance(node, Var) or isinstance(node, FuncDef)) - and node.type is not None): + def type(self) -> mypy.types.Type | None: + node = self.node + if isinstance(node, (Var, SYMBOL_FUNCBASE_TYPES)) and node.type is not None: return node.type elif isinstance(node, Decorator): - return (cast(Decorator, node)).var.type + return node.var.type else: return None + def copy(self) -> SymbolTableNode: + new = SymbolTableNode( + self.kind, self.node, self.module_public, self.implicit, self.module_hidden + ) + new.cross_ref = self.cross_ref + return new + def __str__(self) -> str: - s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) - if self.mod_id is not None: - s += ' ({})'.format(self.mod_id) + s = f"{node_kinds[self.kind]}/{short_type(self.node)}" + if isinstance(self.node, SymbolNode): + s += f" ({self.node.fullname})" # Include declared type of variables and functions. if self.type is not None: - s += ' : {}'.format(self.type) + s += f" : {self.type}" + if self.cross_ref: + s += f" cross_ref:{self.cross_ref}" return s + def serialize(self, prefix: str, name: str) -> JsonDict: + """Serialize a SymbolTableNode. + + Args: + prefix: full name of the containing module or class; or None + name: name of this object relative to the containing object + """ + data: JsonDict = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} + if self.module_hidden: + data["module_hidden"] = True + if not self.module_public: + data["module_public"] = False + if self.implicit: + data["implicit"] = True + if self.plugin_generated: + data["plugin_generated"] = True + if isinstance(self.node, MypyFile): + data["cross_ref"] = self.node.fullname + else: + assert self.node is not None, f"{prefix}:{name}" + if prefix is not None: + fullname = self.node.fullname + if ( + "." in fullname + and fullname != prefix + "." + name + and not (isinstance(self.node, Var) and self.node.from_module_getattr) + ): + assert not isinstance( + self.node, PlaceholderNode + ), f"Definition of {fullname} is unexpectedly incomplete" + data["cross_ref"] = fullname + return data + data["node"] = self.node.serialize() + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> SymbolTableNode: + assert data[".class"] == "SymbolTableNode" + kind = inverse_node_kinds[data["kind"]] + if "cross_ref" in data: + # This will be fixed up later. + stnode = SymbolTableNode(kind, None) + stnode.cross_ref = data["cross_ref"] + else: + assert "node" in data, data + node = SymbolNode.deserialize(data["node"]) + stnode = SymbolTableNode(kind, node) + if "module_hidden" in data: + stnode.module_hidden = data["module_hidden"] + if "module_public" in data: + stnode.module_public = data["module_public"] + if "implicit" in data: + stnode.implicit = data["implicit"] + if "plugin_generated" in data: + stnode.plugin_generated = data["plugin_generated"] + return stnode + class SymbolTable(Dict[str, SymbolTableNode]): + """Static representation of a namespace dictionary. + + This is used for module, class and function namespaces. + """ + + __slots__ = () + def __str__(self) -> str: - a = List[str]() + a: list[str] = [] for key, value in self.items(): # Filter out the implicit import of builtins. if isinstance(value, SymbolTableNode): - if (value.fullname != 'builtins' and - value.fullname.split('.')[-1] not in - implicit_module_attrs): - a.append(' ' + str(key) + ' : ' + str(value)) + if ( + value.fullname != "builtins" + and (value.fullname or "").split(".")[-1] not in implicit_module_attrs + ): + a.append(" " + str(key) + " : " + str(value)) else: - a.append(' ') + a.append(" ") a = sorted(a) - a.insert(0, 'SymbolTable(') - a[-1] += ')' - return '\n'.join(a) - - -def clean_up(s: str) -> str: - # TODO remove - return re.sub('.*::', '', s) - - -def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - if func.type: - assert isinstance(func.type, mypy.types.FunctionLike) - return cast(mypy.types.FunctionLike, func.type) - else: - # Implicit type signature with dynamic types. - # Overloaded functions always have a signature, so func must be an ordinary function. - fdef = cast(FuncDef, func) - name = func.name() - if name: - name = '"{}"'.format(name) - names = [] # type: List[str] - for arg in fdef.args: - names.append(arg.name()) - return mypy.types.Callable([mypy.types.AnyType()] * len(fdef.args), - fdef.arg_kinds, - names, - mypy.types.AnyType(), - fallback, - name) - - -@overload -def method_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - """Return the signature of a method (omit self).""" - return method_type(function_type(func, fallback)) - - -@overload -def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': - if isinstance(sig, mypy.types.Callable): - return method_callable(sig) - else: - osig = cast(mypy.types.Overloaded, sig) - items = List[mypy.types.Callable]() - for c in osig.items(): - items.append(method_callable(c)) - return mypy.types.Overloaded(items) - - -def method_callable(c: 'mypy.types.Callable') -> 'mypy.types.Callable': - return mypy.types.Callable(c.arg_types[1:], - c.arg_kinds[1:], - c.arg_names[1:], - c.ret_type, - c.fallback, - c.name, - c.variables, - c.bound_vars) - - -class MroError(Exception): - """Raised if a consistent mro cannot be determined for a class.""" - - -def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: - # TODO describe - if info.mro: - return info.mro - bases = info.direct_base_classes() - return [info] + merge([linearize_hierarchy(base) for base in bases] + - [bases]) - - -def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: - seqs = [s[:] for s in seqs] - result = List[TypeInfo]() - while True: - seqs = [s for s in seqs if s] - if not seqs: - return result - for seq in seqs: - head = seq[0] - if not [s for s in seqs if head in s[1:]]: + a.insert(0, "SymbolTable(") + a[-1] += ")" + return "\n".join(a) + + def copy(self) -> SymbolTable: + return SymbolTable([(key, node.copy()) for key, node in self.items()]) + + def serialize(self, fullname: str) -> JsonDict: + data: JsonDict = {".class": "SymbolTable"} + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzerPass2.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == "__builtins__" or value.no_serialize: + continue + data[key] = value.serialize(fullname, key) + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> SymbolTable: + assert data[".class"] == "SymbolTable" + st = SymbolTable() + for key, value in data.items(): + if key != ".class": + st[key] = SymbolTableNode.deserialize(value) + return st + + +class DataclassTransformSpec: + """Specifies how a dataclass-like transform should be applied. The fields here are based on the + parameters accepted by `typing.dataclass_transform`.""" + + __slots__ = ( + "eq_default", + "order_default", + "kw_only_default", + "frozen_default", + "field_specifiers", + ) + + def __init__( + self, + *, + eq_default: bool | None = None, + order_default: bool | None = None, + kw_only_default: bool | None = None, + field_specifiers: tuple[str, ...] | None = None, + # Specified outside of PEP 681: + # frozen_default was added to CPythonin https://github.com/python/cpython/pull/99958 citing + # positive discussion in typing-sig + frozen_default: bool | None = None, + ) -> None: + self.eq_default = eq_default if eq_default is not None else True + self.order_default = order_default if order_default is not None else False + self.kw_only_default = kw_only_default if kw_only_default is not None else False + self.frozen_default = frozen_default if frozen_default is not None else False + self.field_specifiers = field_specifiers if field_specifiers is not None else () + + def serialize(self) -> JsonDict: + return { + "eq_default": self.eq_default, + "order_default": self.order_default, + "kw_only_default": self.kw_only_default, + "frozen_default": self.frozen_default, + "field_specifiers": list(self.field_specifiers), + } + + @classmethod + def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: + return DataclassTransformSpec( + eq_default=data.get("eq_default"), + order_default=data.get("order_default"), + kw_only_default=data.get("kw_only_default"), + frozen_default=data.get("frozen_default"), + field_specifiers=tuple(data.get("field_specifiers", [])), + ) + + +def get_flags(node: Node, names: list[str]) -> list[str]: + return [name for name in names if getattr(node, name)] + + +def set_flags(node: Node, flags: list[str]) -> None: + for name in flags: + setattr(node, name, True) + + +def get_member_expr_fullname(expr: MemberExpr) -> str | None: + """Return the qualified name representation of a member expression. + + Return a string of form foo.bar, foo.bar.baz, or similar, or None if the + argument cannot be represented in this form. + """ + initial: str | None = None + if isinstance(expr.expr, NameExpr): + initial = expr.expr.name + elif isinstance(expr.expr, MemberExpr): + initial = get_member_expr_fullname(expr.expr) + if initial is None: + return None + return f"{initial}.{expr.name}" + + +deserialize_map: Final = { + key: obj.deserialize + for key, obj in globals().items() + if type(obj) is not FakeInfo + and isinstance(obj, type) + and issubclass(obj, SymbolNode) + and obj is not SymbolNode +} + + +def check_arg_kinds( + arg_kinds: list[ArgKind], nodes: list[T], fail: Callable[[str, T], None] +) -> None: + is_var_arg = False + is_kw_arg = False + seen_named = False + seen_opt = False + for kind, node in zip(arg_kinds, nodes): + if kind == ARG_POS: + if is_var_arg or is_kw_arg or seen_named or seen_opt: + fail( + "Required positional args may not appear after default, named or var args", + node, + ) break - else: - raise MroError() - result.append(head) - for s in seqs: - if s[0] is head: - del s[0] + elif kind == ARG_OPT: + if is_var_arg or is_kw_arg or seen_named: + fail("Positional default args may not appear after named or var args", node) + break + seen_opt = True + elif kind == ARG_STAR: + if is_var_arg or is_kw_arg or seen_named: + fail("Var args may not appear after named or var args", node) + break + is_var_arg = True + elif kind == ARG_NAMED or kind == ARG_NAMED_OPT: + seen_named = True + if is_kw_arg: + fail("A **kwargs argument must be the last argument", node) + break + elif kind == ARG_STAR2: + if is_kw_arg: + fail("You may only have one **kwargs argument", node) + break + is_kw_arg = True + + +def check_arg_names( + names: Sequence[str | None], + nodes: list[T], + fail: Callable[[str, T], None], + description: str = "function definition", +) -> None: + seen_names: set[str | None] = set() + for name, node in zip(names, nodes): + if name is not None and name in seen_names: + fail(f'Duplicate argument "{name}" in {description}', node) + break + seen_names.add(name) + + +def is_class_var(expr: NameExpr) -> bool: + """Return whether the expression is ClassVar[...]""" + if isinstance(expr.node, Var): + return expr.node.is_classvar + return False + + +def is_final_node(node: SymbolNode | None) -> bool: + """Check whether `node` corresponds to a final attribute.""" + return isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)) and node.is_final + + +def local_definitions( + names: SymbolTable, name_prefix: str, info: TypeInfo | None = None +) -> Iterator[Definition]: + """Iterate over local definitions (not imported) in a symbol table. + + Recursively iterate over class members and nested classes. + """ + # TODO: What should the name be? Or maybe remove it? + for name, symnode in names.items(): + shortname = name + if "-redef" in name: + # Restore original name from mangled name of multiply defined function + shortname = name.split("-redef")[0] + fullname = name_prefix + "." + shortname + node = symnode.node + if node and node.fullname == fullname: + yield fullname, symnode, info + if isinstance(node, TypeInfo): + yield from local_definitions(node.names, fullname, node) diff --git a/mypy/operators.py b/mypy/operators.py new file mode 100644 index 000000000000..d1f050b58fae --- /dev/null +++ b/mypy/operators.py @@ -0,0 +1,126 @@ +"""Information about Python operators""" + +from __future__ import annotations + +from typing import Final + +# Map from binary operator id to related method name (in Python 3). +op_methods: Final = { + "+": "__add__", + "-": "__sub__", + "*": "__mul__", + "/": "__truediv__", + "%": "__mod__", + "divmod": "__divmod__", + "//": "__floordiv__", + "**": "__pow__", + "@": "__matmul__", + "&": "__and__", + "|": "__or__", + "^": "__xor__", + "<<": "__lshift__", + ">>": "__rshift__", + "==": "__eq__", + "!=": "__ne__", + "<": "__lt__", + ">=": "__ge__", + ">": "__gt__", + "<=": "__le__", + "in": "__contains__", +} + +op_methods_to_symbols: Final = {v: k for (k, v) in op_methods.items()} + +ops_falling_back_to_cmp: Final = {"__ne__", "__eq__", "__lt__", "__le__", "__gt__", "__ge__"} + + +ops_with_inplace_method: Final = { + "+", + "-", + "*", + "/", + "%", + "//", + "**", + "@", + "&", + "|", + "^", + "<<", + ">>", +} + +inplace_operator_methods: Final = {"__i" + op_methods[op][2:] for op in ops_with_inplace_method} + +reverse_op_methods: Final = { + "__add__": "__radd__", + "__sub__": "__rsub__", + "__mul__": "__rmul__", + "__truediv__": "__rtruediv__", + "__mod__": "__rmod__", + "__divmod__": "__rdivmod__", + "__floordiv__": "__rfloordiv__", + "__pow__": "__rpow__", + "__matmul__": "__rmatmul__", + "__and__": "__rand__", + "__or__": "__ror__", + "__xor__": "__rxor__", + "__lshift__": "__rlshift__", + "__rshift__": "__rrshift__", + "__eq__": "__eq__", + "__ne__": "__ne__", + "__lt__": "__gt__", + "__ge__": "__le__", + "__gt__": "__lt__", + "__le__": "__ge__", +} + +reverse_op_method_names: Final = set(reverse_op_methods.values()) + +# Suppose we have some class A. When we do A() + A(), Python will only check +# the output of A().__add__(A()) and skip calling the __radd__ method entirely. +# This shortcut is used only for the following methods: +op_methods_that_shortcut: Final = { + "__add__", + "__sub__", + "__mul__", + "__truediv__", + "__mod__", + "__divmod__", + "__floordiv__", + "__pow__", + "__matmul__", + "__and__", + "__or__", + "__xor__", + "__lshift__", + "__rshift__", +} + +normal_from_reverse_op: Final = {m: n for n, m in reverse_op_methods.items()} +reverse_op_method_set: Final = set(reverse_op_methods.values()) + +unary_op_methods: Final = {"-": "__neg__", "+": "__pos__", "~": "__invert__"} + +int_op_to_method: Final = { + "==": int.__eq__, + "is": int.__eq__, + "<": int.__lt__, + "<=": int.__le__, + "!=": int.__ne__, + "is not": int.__ne__, + ">": int.__gt__, + ">=": int.__ge__, +} + +flip_ops: Final = {"<": ">", "<=": ">=", ">": "<", ">=": "<="} +neg_ops: Final = { + "==": "!=", + "!=": "==", + "is": "is not", + "is not": "is", + "<": ">=", + "<=": ">", + ">": "<=", + ">=": "<", +} diff --git a/mypy/options.py b/mypy/options.py new file mode 100644 index 000000000000..33a2c75d164e --- /dev/null +++ b/mypy/options.py @@ -0,0 +1,588 @@ +from __future__ import annotations + +import pprint +import re +import sys +import sysconfig +from typing import Any, Callable, Final, Mapping, Pattern + +from mypy import defaults +from mypy.errorcodes import ErrorCode, error_codes +from mypy.util import get_class_descriptors, replace_object_state + + +class BuildType: + STANDARD: Final = 0 + MODULE: Final = 1 + PROGRAM_TEXT: Final = 2 + + +PER_MODULE_OPTIONS: Final = { + # Please keep this list sorted + "allow_redefinition", + "allow_untyped_globals", + "always_false", + "always_true", + "check_untyped_defs", + "debug_cache", + "disable_error_code", + "disabled_error_codes", + "disallow_any_decorated", + "disallow_any_explicit", + "disallow_any_expr", + "disallow_any_generics", + "disallow_any_unimported", + "disallow_incomplete_defs", + "disallow_subclassing_any", + "disallow_untyped_calls", + "disallow_untyped_decorators", + "disallow_untyped_defs", + "enable_error_code", + "enabled_error_codes", + "extra_checks", + "follow_imports_for_stubs", + "follow_imports", + "follow_untyped_imports", + "ignore_errors", + "ignore_missing_imports", + "implicit_optional", + "implicit_reexport", + "local_partial_types", + "mypyc", + "strict_concatenate", + "strict_equality", + "strict_optional", + "warn_no_return", + "warn_return_any", + "warn_unreachable", + "warn_unused_ignores", +} + +OPTIONS_AFFECTING_CACHE: Final = ( + PER_MODULE_OPTIONS + | { + "platform", + "bazel", + "old_type_inference", + "plugins", + "disable_bytearray_promotion", + "disable_memoryview_promotion", + } +) - {"debug_cache"} + +# Features that are currently (or were recently) incomplete/experimental +TYPE_VAR_TUPLE: Final = "TypeVarTuple" +UNPACK: Final = "Unpack" +PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" +NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" +INLINE_TYPEDDICT: Final = "InlineTypedDict" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT)) +COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, NEW_GENERIC_SYNTAX)) + + +class Options: + """Options collected from flags.""" + + def __init__(self) -> None: + # Cache for clone_for_module() + self._per_module_cache: dict[str, Options] | None = None + + # -- build options -- + self.build_type = BuildType.STANDARD + self.python_version: tuple[int, int] = sys.version_info[:2] + # The executable used to search for PEP 561 packages. If this is None, + # then mypy does not search for PEP 561 packages. + self.python_executable: str | None = sys.executable + + # When cross compiling to emscripten, we need to rely on MACHDEP because + # sys.platform is the host build platform, not emscripten. + MACHDEP = sysconfig.get_config_var("MACHDEP") + if MACHDEP == "emscripten": + self.platform = MACHDEP + else: + self.platform = sys.platform + + self.custom_typing_module: str | None = None + self.custom_typeshed_dir: str | None = None + # The abspath() version of the above, we compute it once as an optimization. + self.abs_custom_typeshed_dir: str | None = None + self.mypy_path: list[str] = [] + self.report_dirs: dict[str, str] = {} + # Show errors in PEP 561 packages/site-packages modules + self.no_silence_site_packages = False + self.no_site_packages = False + self.ignore_missing_imports = False + # Is ignore_missing_imports set in a per-module section + self.ignore_missing_imports_per_module = False + # Typecheck modules without stubs or py.typed marker + self.follow_untyped_imports = False + self.follow_imports = "normal" # normal|silent|skip|error + # Whether to respect the follow_imports setting even for stub files. + # Intended to be used for disabling specific stubs. + self.follow_imports_for_stubs = False + # PEP 420 namespace packages + # This allows definitions of packages without __init__.py and allows packages to span + # multiple directories. This flag affects both import discovery and the association of + # input files/modules/packages to the relevant file and fully qualified module name. + self.namespace_packages = True + # Use current directory and MYPYPATH to determine fully qualified module names of files + # passed by automatically considering their subdirectories as packages. This is only + # relevant if namespace packages are enabled, since otherwise examining __init__.py's is + # sufficient to determine module names for files. As a possible alternative, add a single + # top-level __init__.py to your packages. + self.explicit_package_bases = False + # File names, directory names or subpaths to avoid checking + self.exclude: list[str] = [] + + # disallow_any options + self.disallow_any_generics = False + self.disallow_any_unimported = False + self.disallow_any_expr = False + self.disallow_any_decorated = False + self.disallow_any_explicit = False + + # Disallow calling untyped functions from typed ones + self.disallow_untyped_calls = False + + # Always allow untyped calls for function coming from modules/packages + # in this list (each item effectively acts as a prefix match) + self.untyped_calls_exclude: list[str] = [] + + # Disallow defining untyped (or incompletely typed) functions + self.disallow_untyped_defs = False + + # Disallow defining incompletely typed functions + self.disallow_incomplete_defs = False + + # Type check unannotated functions + self.check_untyped_defs = False + + # Disallow decorating typed functions with untyped decorators + self.disallow_untyped_decorators = False + + # Disallow subclassing values of type 'Any' + self.disallow_subclassing_any = False + + # Also check typeshed for missing annotations + self.warn_incomplete_stub = False + + # Warn about casting an expression to its inferred type + self.warn_redundant_casts = False + + # Warn about falling off the end of a function returning non-None + self.warn_no_return = True + + # Warn about returning objects of type Any when the function is + # declared with a precise type + self.warn_return_any = False + + # Report importing or using deprecated features as errors instead of notes. + self.report_deprecated_as_note = False + + # Warn about unused '# type: ignore' comments + self.warn_unused_ignores = False + + # Warn about unused '[mypy-]' or '[[tool.mypy.overrides]]' config sections + self.warn_unused_configs = False + + # Files in which to ignore all non-fatal errors + self.ignore_errors = False + + # Apply strict None checking + self.strict_optional = True + + # Show "note: In function "foo":" messages. + self.show_error_context = False + + # Use nicer output (when possible). + self.color_output = True + self.error_summary = True + + # Assume arguments with default values of None are Optional + self.implicit_optional = False + + # Don't re-export names unless they are imported with `from ... as ...` + self.implicit_reexport = True + + # Suppress toplevel errors caused by missing annotations + self.allow_untyped_globals = False + + # Allow variable to be redefined with an arbitrary type in the same block + # and the same nesting level as the initialization + self.allow_redefinition = False + + # Prohibit equality, identity, and container checks for non-overlapping types. + # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. + self.strict_equality = False + + # Deprecated, use extra_checks instead. + self.strict_concatenate = False + + # Enable additional checks that are technically correct but impractical. + self.extra_checks = False + + # Report an error for any branches inferred to be unreachable as a result of + # type analysis. + self.warn_unreachable = False + + # Variable names considered True + self.always_true: list[str] = [] + + # Variable names considered False + self.always_false: list[str] = [] + + # Error codes to disable + self.disable_error_code: list[str] = [] + self.disabled_error_codes: set[ErrorCode] = set() + + # Error codes to enable + self.enable_error_code: list[str] = [] + self.enabled_error_codes: set[ErrorCode] = set() + + # Use script name instead of __main__ + self.scripts_are_modules = False + + # Config file name + self.config_file: str | None = None + + # A filename containing a JSON mapping from filenames to + # mtime/size/hash arrays, used to avoid having to recalculate + # source hashes as often. + self.quickstart_file: str | None = None + + # A comma-separated list of files/directories for mypy to type check; + # supports globbing + self.files: list[str] | None = None + + # A list of packages for mypy to type check + self.packages: list[str] | None = None + + # A list of modules for mypy to type check + self.modules: list[str] | None = None + + # Write junit.xml to given file + self.junit_xml: str | None = None + + self.junit_format: str = "global" # global|per_file + + # Caching and incremental checking options + self.incremental = True + self.cache_dir = defaults.CACHE_DIR + self.sqlite_cache = False + self.debug_cache = False + self.skip_version_check = False + self.skip_cache_mtime_checks = False + self.fine_grained_incremental = False + # Include fine-grained dependencies in written cache files + self.cache_fine_grained = False + # Read cache files in fine-grained incremental mode (cache must include dependencies) + self.use_fine_grained_cache = False + + # Run tree.serialize() even if cache generation is disabled + self.debug_serialize = False + + # Tune certain behaviors when being used as a front-end to mypyc. Set per-module + # in modules being compiled. Not in the config file or command line. + self.mypyc = False + + # An internal flag to modify some type-checking logic while + # running inspections (e.g. don't expand function definitions). + # Not in the config file or command line. + self.inspections = False + + # Disable the memory optimization of freeing ASTs when + # possible. This isn't exposed as a command line option + # because it is intended for software integrating with + # mypy. (Like mypyc.) + self.preserve_asts = False + + # If True, function and class docstrings will be extracted and retained. + # This isn't exposed as a command line option + # because it is intended for software integrating with + # mypy. (Like stubgen.) + self.include_docstrings = False + + # Paths of user plugins + self.plugins: list[str] = [] + + # Per-module options (raw) + self.per_module_options: dict[str, dict[str, object]] = {} + self._glob_options: list[tuple[str, Pattern[str]]] = [] + self.unused_configs: set[str] = set() + + # -- development options -- + self.verbosity = 0 # More verbose messages (for troubleshooting) + self.pdb = False + self.show_traceback = False + self.raise_exceptions = False + self.dump_type_stats = False + self.dump_inference_stats = False + self.dump_build_stats = False + self.enable_incomplete_feature: list[str] = [] + self.timing_stats: str | None = None + self.line_checking_stats: str | None = None + + # -- test options -- + # Stop after the semantic analysis phase + self.semantic_analysis_only = False + + # Use stub builtins fixtures to speed up tests + self.use_builtins_fixtures = False + + # This should only be set when running certain mypy tests. + # Use this sparingly to avoid tests diverging from non-test behavior. + self.test_env = False + + # -- experimental options -- + self.shadow_file: list[list[str]] | None = None + self.show_column_numbers: bool = False + self.show_error_end: bool = False + self.hide_error_codes = False + self.show_error_code_links = False + # Use soft word wrap and show trimmed source snippets with error location markers. + self.pretty = False + self.dump_graph = False + self.dump_deps = False + self.logical_deps = False + # If True, partial types can't span a module top level and a function + self.local_partial_types = False + # Some behaviors are changed when using Bazel (https://bazel.build). + self.bazel = False + # If True, export inferred types for all expressions as BuildResult.types + self.export_types = False + # List of package roots -- directories under these are packages even + # if they don't have __init__.py. + self.package_root: list[str] = [] + self.cache_map: dict[str, tuple[str, str]] = {} + # Don't properly free objects on exit, just kill the current process. + self.fast_exit = True + # fast path for finding modules from source set + self.fast_module_lookup = False + # Allow empty function bodies even if it is not safe, used for testing only. + self.allow_empty_bodies = False + # Used to transform source code before parsing if not None + # TODO: Make the type precise (AnyStr -> AnyStr) + self.transform_source: Callable[[Any], Any] | None = None + # Print full path to each file in the report. + self.show_absolute_path: bool = False + # Install missing stub packages if True + self.install_types = False + # Install missing stub packages in non-interactive mode (don't prompt for + # confirmation, and don't show any errors) + self.non_interactive = False + # When we encounter errors that may cause many additional errors, + # skip most errors after this many messages have been reported. + # -1 means unlimited. + self.many_errors_threshold = defaults.MANY_ERRORS_THRESHOLD + # Disable new experimental type inference algorithm. + self.old_type_inference = False + # Deprecated reverse version of the above, do not use. + self.new_type_inference = False + # Export line-level, limited, fine-grained dependency information in cache data + # (undocumented feature). + self.export_ref_info = False + + self.disable_bytearray_promotion = False + self.disable_memoryview_promotion = False + self.force_uppercase_builtins = False + self.force_union_syntax = False + + # Sets custom output format + self.output: str | None = None + + def use_lowercase_names(self) -> bool: + if self.python_version >= (3, 9): + return not self.force_uppercase_builtins + return False + + def use_or_syntax(self) -> bool: + if self.python_version >= (3, 10): + return not self.force_union_syntax + return False + + def use_star_unpack(self) -> bool: + return self.python_version >= (3, 11) + + def snapshot(self) -> dict[str, object]: + """Produce a comparable snapshot of this Option""" + # Under mypyc, we don't have a __dict__, so we need to do worse things. + d = dict(getattr(self, "__dict__", ())) + for k in get_class_descriptors(Options): + if hasattr(self, k): + d[k] = getattr(self, k) + # Remove private attributes from snapshot + d = {k: v for k, v in d.items() if not k.startswith("_")} + return d + + def __repr__(self) -> str: + return f"Options({pprint.pformat(self.snapshot())})" + + def process_error_codes(self, *, error_callback: Callable[[str], Any]) -> None: + # Process `--enable-error-code` and `--disable-error-code` flags + disabled_codes = set(self.disable_error_code) + enabled_codes = set(self.enable_error_code) + + valid_error_codes = set(error_codes.keys()) + + invalid_codes = (enabled_codes | disabled_codes) - valid_error_codes + if invalid_codes: + error_callback(f"Invalid error code(s): {', '.join(sorted(invalid_codes))}") + + self.disabled_error_codes |= {error_codes[code] for code in disabled_codes} + self.enabled_error_codes |= {error_codes[code] for code in enabled_codes} + + # Enabling an error code always overrides disabling + self.disabled_error_codes -= self.enabled_error_codes + + def process_incomplete_features( + self, *, error_callback: Callable[[str], Any], warning_callback: Callable[[str], Any] + ) -> None: + # Validate incomplete features. + for feature in self.enable_incomplete_feature: + if feature not in INCOMPLETE_FEATURES | COMPLETE_FEATURES: + error_callback(f"Unknown incomplete feature: {feature}") + if feature in COMPLETE_FEATURES: + warning_callback(f"Warning: {feature} is already enabled by default") + + def apply_changes(self, changes: dict[str, object]) -> Options: + # Note: effects of this method *must* be idempotent. + new_options = Options() + # Under mypyc, we don't have a __dict__, so we need to do worse things. + replace_object_state(new_options, self, copy_dict=True) + for key, value in changes.items(): + setattr(new_options, key, value) + if changes.get("ignore_missing_imports"): + # This is the only option for which a per-module and a global + # option sometimes beheave differently. + new_options.ignore_missing_imports_per_module = True + + # These two act as overrides, so apply them when cloning. + # Similar to global codes enabling overrides disabling, so we start from latter. + new_options.disabled_error_codes = self.disabled_error_codes.copy() + new_options.enabled_error_codes = self.enabled_error_codes.copy() + for code_str in new_options.disable_error_code: + code = error_codes[code_str] + new_options.disabled_error_codes.add(code) + new_options.enabled_error_codes.discard(code) + for code_str in new_options.enable_error_code: + code = error_codes[code_str] + new_options.enabled_error_codes.add(code) + new_options.disabled_error_codes.discard(code) + + return new_options + + def compare_stable(self, other_snapshot: dict[str, object]) -> bool: + """Compare options in a way that is stable for snapshot() -> apply_changes() roundtrip. + + This is needed because apply_changes() has non-trivial effects for some flags, so + Options().apply_changes(options.snapshot()) may result in a (slightly) different object. + """ + return ( + Options().apply_changes(self.snapshot()).snapshot() + == Options().apply_changes(other_snapshot).snapshot() + ) + + def build_per_module_cache(self) -> None: + self._per_module_cache = {} + + # Config precedence is as follows: + # 1. Concrete section names: foo.bar.baz + # 2. "Unstructured" glob patterns: foo.*.baz, in the order + # they appear in the file (last wins) + # 3. "Well-structured" wildcard patterns: foo.bar.*, in specificity order. + + # Since structured configs inherit from structured configs above them in the hierarchy, + # we need to process per-module configs in a careful order. + # We have to process foo.* before foo.bar.* before foo.bar, + # and we need to apply *.bar to foo.bar but not to foo.bar.*. + # To do this, process all well-structured glob configs before non-glob configs and + # exploit the fact that foo.* sorts earlier ASCIIbetically (unicodebetically?) + # than foo.bar.*. + # (A section being "processed last" results in its config "winning".) + # Unstructured glob configs are stored and are all checked for each module. + unstructured_glob_keys = [k for k in self.per_module_options.keys() if "*" in k[:-1]] + structured_keys = [k for k in self.per_module_options.keys() if "*" not in k[:-1]] + wildcards = sorted(k for k in structured_keys if k.endswith(".*")) + concrete = [k for k in structured_keys if not k.endswith(".*")] + + for glob in unstructured_glob_keys: + self._glob_options.append((glob, self.compile_glob(glob))) + + # We (for ease of implementation) treat unstructured glob + # sections as used if any real modules use them or if any + # concrete config sections use them. This means we need to + # track which get used while constructing. + self.unused_configs = set(unstructured_glob_keys) + + for key in wildcards + concrete: + # Find what the options for this key would be, just based + # on inheriting from parent configs. + options = self.clone_for_module(key) + # And then update it with its per-module options. + self._per_module_cache[key] = options.apply_changes(self.per_module_options[key]) + + # Add the more structured sections into unused configs, since + # they only count as used if actually used by a real module. + self.unused_configs.update(structured_keys) + + def clone_for_module(self, module: str) -> Options: + """Create an Options object that incorporates per-module options. + + NOTE: Once this method is called all Options objects should be + considered read-only, else the caching might be incorrect. + """ + if self._per_module_cache is None: + self.build_per_module_cache() + assert self._per_module_cache is not None + + # If the module just directly has a config entry, use it. + if module in self._per_module_cache: + self.unused_configs.discard(module) + return self._per_module_cache[module] + + # If not, search for glob paths at all the parents. So if we are looking for + # options for foo.bar.baz, we search foo.bar.baz.*, foo.bar.*, foo.*, + # in that order, looking for an entry. + # This is technically quadratic in the length of the path, but module paths + # don't actually get all that long. + options = self + path = module.split(".") + for i in range(len(path), 0, -1): + key = ".".join(path[:i] + ["*"]) + if key in self._per_module_cache: + self.unused_configs.discard(key) + options = self._per_module_cache[key] + break + + # OK and *now* we need to look for unstructured glob matches. + # We only do this for concrete modules, not structured wildcards. + if not module.endswith(".*"): + for key, pattern in self._glob_options: + if pattern.match(module): + self.unused_configs.discard(key) + options = options.apply_changes(self.per_module_options[key]) + + # We could update the cache to directly point to modules once + # they have been looked up, but in testing this made things + # slower and not faster, so we don't bother. + + return options + + def compile_glob(self, s: str) -> Pattern[str]: + # Compile one of the glob patterns to a regex so that '.*' can + # match *zero or more* module sections. This means we compile + # '.*' into '(\..*)?'. + parts = s.split(".") + expr = re.escape(parts[0]) if parts[0] != "*" else ".*" + for part in parts[1:]: + expr += re.escape("." + part) if part != "*" else r"(\..*)?" + return re.compile(expr + "\\Z") + + def select_options_affecting_cache(self) -> Mapping[str, object]: + result: dict[str, object] = {} + for opt in OPTIONS_AFFECTING_CACHE: + val = getattr(self, opt) + if opt in ("disabled_error_codes", "enabled_error_codes"): + val = sorted([code.code for code in val]) + result[opt] = val + return result diff --git a/mypy/output.py b/mypy/output.py deleted file mode 100644 index fe2bd1a745d4..000000000000 --- a/mypy/output.py +++ /dev/null @@ -1,630 +0,0 @@ -"""Parse tree pretty printer.""" - -import re - -import typing - -from mypy import nodes -from mypy.visitor import NodeVisitor -from mypy.typerepr import CommonTypeRepr - - -class OutputVisitor(NodeVisitor): - """Parse tree Node visitor that outputs the original, formatted - source code. You can implement custom transformations by - subclassing this class. - """ - def __init__(self): - super().__init__() - self.result = [] # strings - self.line_number = 1 - # If True, omit the next character if it is a space - self.omit_next_space = False - # Number of spaces of indent right now - self.indent = 0 - # Number of spaces of extra indent to add when encountering a line - # break - self.extra_indent = 0 - self.block_depth = 0 - - def output(self): - """Return a string representation of the output.""" - return ''.join(self.result) - - def visit_mypy_file(self, o): - self.nodes(o.defs) - self.token(o.repr.eof) - - def visit_import(self, o): - r = o.repr - self.token(r.import_tok) - for i in range(len(r.components)): - self.tokens(r.components[i]) - if r.as_names[i]: - self.tokens(r.as_names[i]) - if i < len(r.commas): - self.token(r.commas[i]) - self.token(r.br) - - def visit_import_from(self, o): - self.output_import_from_or_all(o) - - def visit_import_all(self, o): - self.output_import_from_or_all(o) - - def output_import_from_or_all(self, o): - r = o.repr - self.token(r.from_tok) - self.tokens(r.rel_toks) - self.tokens(r.components) - self.token(r.import_tok) - self.token(r.lparen) - for misc, comma in r.names: - self.tokens(misc) - self.token(comma) - self.token(r.rparen) - self.token(r.br) - - def visit_class_def(self, o): - r = o.repr - self.tokens([r.class_tok, r.name]) - self.type_vars(o.type_vars) - self.token(r.lparen) - for i in range(len(o.base_types)): - if o.base_types[i].repr: - self.type(o.base_types[i]) - if i < len(r.commas): - self.token(r.commas[i]) - self.token(r.rparen) - self.node(o.defs) - - def type_vars(self, v): - # IDEA: Combine this with type_vars in TypeOutputVisitor. - if v and v.repr: - r = v.repr - self.token(r.langle) - for i in range(len(v.items)): - d = v.items[i] - self.token(d.repr.name) - self.token(d.repr.is_tok) - if d.bound: - self.type(d.bound) - if i < len(r.commas): - self.token(r.commas[i]) - self.token(r.rangle) - - def visit_func_def(self, o): - r = o.repr - - if r.def_tok: - self.token(r.def_tok) - else: - self.type(o.type.items()[0].ret_type) - - self.token(r.name) - - self.function_header(o, r.args, o.arg_kinds) - - self.node(o.body) - - def visit_overloaded_func_def(self, o): - for f in o.items: - f.accept(self) - - def function_header(self, o, arg_repr, arg_kinds, pre_args_func=None, - erase_type=False, strip_space_before_first_arg=False): - r = o.repr - - t = None - if o.type and not erase_type: - t = o.type - - init = o.init - - if t: - self.type_vars(t.variables) - - self.token(arg_repr.lseparator) - if pre_args_func: - pre_args_func() - asterisk = 0 - for i in range(len(arg_repr.arg_names)): - if t: - if t.arg_types[i].repr: - self.type(t.arg_types[i]) - if arg_kinds[i] in [nodes.ARG_STAR, nodes.ARG_STAR2]: - self.token(arg_repr.asterisk[asterisk]) - asterisk += 1 - if not erase_type: - self.token(arg_repr.arg_names[i]) - else: - n = arg_repr.arg_names[i].rep() - if i == 0 and strip_space_before_first_arg: - # Remove spaces before the first argument name. Generally - # spaces are only present after a type, and if we erase the - # type, we should also erase also the spaces. - n = re.sub(' +([a-zA-Z0-9_]+)$', '\\1', n) - self.string(n) - if i < len(arg_repr.assigns): - self.token(arg_repr.assigns[i]) - if init and i < len(init) and init[i]: - self.node(init[i].rvalue) - if i < len(arg_repr.commas): - self.token(arg_repr.commas[i]) - self.token(arg_repr.rseparator) - - def visit_var_def(self, o): - r = o.repr - if r: - for v in o.items: - self.type(v.type) - self.node(v) - self.token(r.assign) - self.node(o.init) - self.token(r.br) - - def visit_var(self, o): - r = o.repr - self.token(r.name) - self.token(r.comma) - - def visit_decorator(self, o): - for at, br, dec in zip(o.repr.ats, o.repr.brs, o.decorators): - self.token(at) - self.node(dec) - self.token(br) - self.node(o.func) - - # Statements - - def visit_block(self, o): - r = o.repr - self.tokens([r.colon, r.br, r.indent]) - self.block_depth += 1 - old_indent = self.indent - self.indent = len(r.indent.string) - self.nodes(o.body) - self.token(r.dedent) - self.indent = old_indent - self.block_depth -= 1 - - def visit_global_decl(self, o): - r = o.repr - self.token(r.global_tok) - for i in range(len(r.names)): - self.token(r.names[i]) - if i < len(r.commas): - self.token(r.commas[i]) - self.token(r.br) - - def visit_nonlocal_decl(self, o): - r = o.repr - self.token(r.nonlocal_tok) - for i in range(len(r.names)): - self.token(r.names[i]) - if i < len(r.commas): - self.token(r.commas[i]) - self.token(r.br) - - def visit_expression_stmt(self, o): - self.node(o.expr) - self.token(o.repr.br) - - def visit_assignment_stmt(self, o): - r = o.repr - i = 0 - for lv in o.lvalues: - self.node(lv) - self.token(r.assigns[i]) - i += 1 - self.node(o.rvalue) - self.token(r.br) - - def visit_operator_assignment_stmt(self, o): - r = o.repr - self.node(o.lvalue) - self.token(r.assign) - self.node(o.rvalue) - self.token(r.br) - - def visit_return_stmt(self, o): - self.simple_stmt(o, o.expr) - - def visit_assert_stmt(self, o): - self.simple_stmt(o, o.expr) - - def visit_yield_stmt(self, o): - self.simple_stmt(o, o.expr) - - def visit_yield_from_stmt(self, o): - self.simple_stmt(o, o.expr) - - def visit_del_stmt(self, o): - self.simple_stmt(o, o.expr) - - def visit_break_stmt(self, o): - self.simple_stmt(o) - - def visit_continue_stmt(self, o): - self.simple_stmt(o) - - def visit_pass_stmt(self, o): - self.simple_stmt(o) - - def simple_stmt(self, o, expr=None): - self.token(o.repr.keyword) - self.node(expr) - self.token(o.repr.br) - - def visit_raise_stmt(self, o): - self.token(o.repr.raise_tok) - self.node(o.expr) - if o.from_expr: - self.token(o.repr.from_tok) - self.node(o.from_expr) - self.token(o.repr.br) - - def visit_while_stmt(self, o): - self.token(o.repr.while_tok) - self.node(o.expr) - self.node(o.body) - if o.else_body: - self.token(o.repr.else_tok) - self.node(o.else_body) - - def visit_for_stmt(self, o): - r = o.repr - self.token(r.for_tok) - self.node(o.index) - self.token(r.in_tok) - self.node(o.expr) - - self.node(o.body) - if o.else_body: - self.token(r.else_tok) - self.node(o.else_body) - - def visit_if_stmt(self, o): - r = o.repr - self.token(r.if_tok) - self.node(o.expr[0]) - self.node(o.body[0]) - for i in range(1, len(o.expr)): - self.token(r.elif_toks[i - 1]) - self.node(o.expr[i]) - self.node(o.body[i]) - self.token(r.else_tok) - if o.else_body: - self.node(o.else_body) - - def visit_try_stmt(self, o): - r = o.repr - self.token(r.try_tok) - self.node(o.body) - for i in range(len(o.types)): - self.token(r.except_toks[i]) - self.node(o.types[i]) - self.token(r.as_toks[i]) - self.node(o.vars[i]) - self.node(o.handlers[i]) - if o.else_body: - self.token(r.else_tok) - self.node(o.else_body) - if o.finally_body: - self.token(r.finally_tok) - self.node(o.finally_body) - - def visit_with_stmt(self, o): - self.token(o.repr.with_tok) - for i in range(len(o.expr)): - self.node(o.expr[i]) - self.token(o.repr.as_toks[i]) - self.node(o.name[i]) - if i < len(o.repr.commas): - self.token(o.repr.commas[i]) - self.node(o.body) - - # Expressions - - def visit_int_expr(self, o): - self.token(o.repr.int) - - def visit_str_expr(self, o): - self.tokens(o.repr.string) - - def visit_bytes_expr(self, o): - self.tokens(o.repr.string) - - def visit_float_expr(self, o): - self.token(o.repr.float) - - def visit_complex_expr(self, o): - self.token(o.repr.complex) - - def visit_paren_expr(self, o): - self.token(o.repr.lparen) - self.node(o.expr) - self.token(o.repr.rparen) - - def visit_star_expr(self, o): - self.token(o.repr.star) - self.node(o.expr) - - def visit_name_expr(self, o): - # Supertype references may not have a representation. - if o.repr: - self.token(o.repr.id) - - def visit_member_expr(self, o): - self.node(o.expr) - self.token(o.repr.dot) - self.token(o.repr.name) - - def visit_index_expr(self, o): - self.node(o.base) - self.token(o.repr.lbracket) - self.node(o.index) - self.token(o.repr.rbracket) - - def visit_slice_expr(self, o): - self.node(o.begin_index) - self.token(o.repr.colon) - self.node(o.end_index) - self.token(o.repr.colon2) - self.node(o.stride) - - def visit_yield_from_expr(self, o): - o.expr.accept(self) - - def visit_call_expr(self, o): - r = o.repr - self.node(o.callee) - self.token(r.lparen) - nargs = len(o.args) - nkeyword = 0 - for i in range(nargs): - if o.arg_kinds[i] == nodes.ARG_STAR: - self.token(r.star) - elif o.arg_kinds[i] == nodes.ARG_STAR2: - self.token(r.star2) - elif o.arg_kinds[i] == nodes.ARG_NAMED: - self.tokens(r.keywords[nkeyword]) - nkeyword += 1 - self.node(o.args[i]) - if i < len(r.commas): - self.token(r.commas[i]) - self.token(r.rparen) - - def visit_op_expr(self, o): - self.node(o.left) - self.tokens([o.repr.op]) - self.node(o.right) - - def visit_comparison_expr(self, o): - self.node(o.operands[0]) - for ops, operand in zip(o.repr.operators, o.operands[1:]): - # ops = op, op2 - self.tokens(list(ops)) - self.node(operand) - - def visit_cast_expr(self, o): - self.token(o.repr.lparen) - self.type(o.type) - self.token(o.repr.rparen) - self.node(o.expr) - - def visit_super_expr(self, o): - r = o.repr - self.tokens([r.super_tok, r.lparen, r.rparen, r.dot, r.name]) - - def visit_unary_expr(self, o): - self.token(o.repr.op) - self.node(o.expr) - - def visit_list_expr(self, o): - r = o.repr - self.token(r.lbracket) - self.comma_list(o.items, r.commas) - self.token(r.rbracket) - - def visit_set_expr(self, o): - self.visit_list_expr(o) - - def visit_tuple_expr(self, o): - r = o.repr - self.token(r.lparen) - self.comma_list(o.items, r.commas) - self.token(r.rparen) - - def visit_dict_expr(self, o): - r = o.repr - self.token(r.lbrace) - i = 0 - for k, v in o.items: - self.node(k) - self.token(r.colons[i]) - self.node(v) - if i < len(r.commas): - self.token(r.commas[i]) - i += 1 - self.token(r.rbrace) - - def visit_func_expr(self, o): - r = o.repr - self.token(r.lambda_tok) - self.function_header(o, r.args, o.arg_kinds) - self.token(r.colon) - self.node(o.body.body[0].expr) - - def visit_type_application(self, o): - self.node(o.expr) - self.token(o.repr.langle) - self.type_list(o.types, o.repr.commas) - self.token(o.repr.rangle) - - def visit_generator_expr(self, o): - r = o.repr - self.node(o.left_expr) - for i in range(len(o.indices)): - self.token(r.for_toks[i]) - self.node(o.indices[i]) - self.token(r.in_toks[i]) - self.node(o.sequences[i]) - for cond, if_tok in zip(o.condlists[i], r.if_toklists[i]): - self.token(if_tok) - self.node(cond) - - def visit_list_comprehension(self, o): - self.token(o.repr.lbracket) - self.node(o.generator) - self.token(o.repr.rbracket) - - # Helpers - - def line(self): - return self.line_number - - def string(self, s): - """Output a string.""" - if self.omit_next_space: - if s.startswith(' '): - s = s[1:] - self.omit_next_space = False - self.line_number += s.count('\n') - if s != '': - s = s.replace('\n', '\n' + ' ' * self.extra_indent) - self.result.append(s) - - def token(self, t): - """Output a token.""" - self.string(t.rep()) - - def tokens(self, a): - """Output an array of tokens.""" - for t in a: - self.token(t) - - def node(self, n): - """Output a node.""" - if n: n.accept(self) - - def nodes(self, a): - """Output an array of nodes.""" - for n in a: - self.node(n) - - def comma_list(self, items, commas): - for i in range(len(items)): - self.node(items[i]) - if i < len(commas): - self.token(commas[i]) - - def type_list(self, items, commas): - for i in range(len(items)): - self.type(items[i]) - if i < len(commas): - self.token(commas[i]) - - def type(self, t): - """Output a type.""" - if t: - v = TypeOutputVisitor() - t.accept(v) - self.string(v.output()) - - def last_output_char(self): - if self.result and self.result[-1]: - return self.result[-1][-1] - else: - return '' - - -class TypeOutputVisitor: - """Type visitor that outputs source code.""" - def __init__(self): - self.result = [] # strings - - def output(self): - """Return a string representation of the output.""" - return ''.join(self.result) - - def visit_unbound_type(self, t): - self.visit_instance(t) - - def visit_any(self, t): - if t.repr: - self.token(t.repr.any_tok) - - def visit_void(self, t): - if t.repr: - self.token(t.repr.void) - - def visit_instance(self, t): - r = t.repr - if isinstance(r, CommonTypeRepr): - self.tokens(r.components) - self.token(r.langle) - self.comma_list(t.args, r.commas) - self.token(r.rangle) - else: - # List type t[]. - assert len(t.args) == 1 - self.comma_list(t.args, []) - self.tokens([r.lbracket, r.rbracket]) - - def visit_type_var(self, t): - self.token(t.repr.name) - - def visit_tuple_type(self, t): - r = t.repr - self.tokens(r.components) - self.token(r.langle) - self.comma_list(t.items, r.commas) - self.token(r.rangle) - - def visit_callable(self, t): - r = t.repr - self.tokens([r.func, r.langle]) - t.ret_type.accept(self) - self.token(r.lparen) - self.comma_list(t.arg_types, r.commas) - self.tokens([r.rparen, r.rangle]) - - def type_vars(self, v): - if v and v.repr: - r = v.repr - self.token(r.langle) - for i in range(len(v.items)): - d = v.items[i] - self.token(d.repr.name) - self.token(d.repr.is_tok) - if d.bound: - self.type(d.bound) - if i < len(r.commas): - self.token(r.commas[i]) - self.token(r.rangle) - - # Helpers - - def string(self, s): - """Output a string.""" - self.result.append(s) - - def token(self, t): - """Output a token.""" - self.result.append(t.rep()) - - def tokens(self, a): - """Output an array of tokens.""" - for t in a: - self.token(t) - - def type(self, n): - """Output a type.""" - if n: n.accept(self) - - def comma_list(self, items, commas): - for i in range(len(items)): - self.type(items[i]) - if i < len(commas): - self.token(commas[i]) diff --git a/mypy/parse.py b/mypy/parse.py index f06ab3d265d3..ee61760c0ac0 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,1880 +1,30 @@ -"""Mypy parser. +from __future__ import annotations -Constructs a parse tree (abstract syntax tree) based on a string -representing a source file. Performs only minimal semantic checks. -""" +from mypy.errors import Errors +from mypy.nodes import MypyFile +from mypy.options import Options -import re -from typing import Undefined, List, Tuple, Any, Set, cast, Union - -from mypy import lex -from mypy.lex import ( - Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, - UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit -) -import mypy.types -from mypy.nodes import ( - MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, - ClassDef, Decorator, Block, Var, VarDef, OperatorAssignmentStmt, - ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, - YieldStmt, DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, - WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, CastExpr, ParenExpr, - TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, - DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, - FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, - UnaryExpr, FuncExpr, TypeApplication, PrintStmt, ImportBase, ComparisonExpr, - StarExpr, YieldFromStmt, YieldFromExpr, NonlocalDecl, DictionaryComprehension, - SetComprehension, ComplexExpr -) -from mypy import nodes -from mypy import noderepr -from mypy.errors import Errors, CompileError -from mypy.types import Void, Type, Callable, AnyType, UnboundType -from mypy.parsetype import ( - parse_type, parse_types, parse_signature, TypeParseError -) - - -precedence = { - '**': 16, - '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) - '': 14, - '*': 13, '/': 13, '//': 13, '%': 13, - '+': 12, '-': 12, - '>>': 11, '<<': 11, - '&': 10, - '^': 9, - '|': 8, - '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, '*u': 7, # unary * for star expressions - 'not': 6, - 'and': 5, - 'or': 4, - '': 3, # conditional expression - '': 2, # list comprehension - ',': 1} - - -op_assign = set([ - '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', - '<<=']) - -op_comp = set([ - '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) - -none = Token('') # Empty token - - -def parse(s: str, fnam: str = None, errors: Errors = None, - pyversion: int = 3, custom_typing_module: str = None) -> MypyFile: +def parse( + source: str | bytes, + fnam: str, + module: str | None, + errors: Errors, + options: Options, + raise_on_error: bool = False, +) -> MypyFile: """Parse a source file, without doing any semantic analysis. Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. - The pyversion argument determines the Python syntax variant (2 for 2.x and - 3 for 3.x). - """ - parser = Parser(fnam, errors, pyversion, custom_typing_module) - tree = parser.parse(s) - tree.path = fnam - return tree - - -class Parser: - tok = Undefined(List[Token]) - ind = 0 - errors = Undefined(Errors) - raise_on_error = False - - # Are we currently parsing the body of a class definition? - is_class_body = False - # All import nodes encountered so far in this parse unit. - imports = Undefined(List[ImportBase]) - # Names imported from __future__. - future_options = Undefined(List[str]) - - def __init__(self, fnam: str, errors: Errors, pyversion: int, - custom_typing_module: str = None) -> None: - self.raise_on_error = errors is None - self.pyversion = pyversion - self.custom_typing_module = custom_typing_module - if errors is not None: - self.errors = errors - else: - self.errors = Errors() - if fnam is not None: - self.errors.set_file(fnam) - else: - self.errors.set_file('') - - def parse(self, s: str) -> MypyFile: - self.tok = lex.lex(s, pyversion=self.pyversion) - self.ind = 0 - self.imports = [] - self.future_options = [] - file = self.parse_file() - if self.raise_on_error and self.errors.is_errors(): - self.errors.raise_error() - return file - - def parse_file(self) -> MypyFile: - """Parse a mypy source file.""" - is_bom = self.parse_bom() - defs = self.parse_defs() - eof = self.expect_type(Eof) - node = MypyFile(defs, self.imports, is_bom) - self.set_repr(node, noderepr.MypyFileRepr(eof)) - return node - - # Parse the initial part - - def parse_bom(self) -> bool: - """Parse the optional byte order mark at the beginning of a file.""" - if isinstance(self.current(), Bom): - self.expect_type(Bom) - if isinstance(self.current(), Break): - self.expect_break() - return True - else: - return False - - def parse_import(self) -> Import: - import_tok = self.expect('import') - ids = List[Tuple[str, str]]() - id_toks = List[List[Token]]() - commas = List[Token]() - as_names = List[Tuple[Token, Token]]() - while True: - id, components = self.parse_qualified_name() - if id == self.custom_typing_module: - id = 'typing' - id_toks.append(components) - as_id = id - if self.current_str() == 'as': - as_tok = self.expect('as') - name_tok = self.expect_type(Name) - as_id = name_tok.string - as_names.append((as_tok, name_tok)) - else: - as_names.append(None) - ids.append((id, as_id)) - if self.current_str() != ',': - break - commas.append(self.expect(',')) - br = self.expect_break() - node = Import(ids) - self.imports.append(node) - self.set_repr(node, noderepr.ImportRepr(import_tok, id_toks, as_names, - commas, br)) - return node - - def parse_import_from(self) -> Node: - from_tok = self.expect('from') - - # Build the list of beginning relative tokens. - relative = 0 - rel_toks = List[Token]() - while self.current_str() == ".": - rel_toks.append(self.expect('.')) - relative += 1 - - # Parse qualified name to actually import from. - if self.current_str() == "import": - # Empty/defualt values. - name = "" - components = List[Token]() - else: - name, components = self.parse_qualified_name() - - if name == self.custom_typing_module: - name = 'typing' - - # Parse import list - import_tok = self.expect('import') - name_toks = List[Tuple[List[Token], Token]]() - lparen = none - rparen = none - node = None # type: ImportBase - if self.current_str() == '*': - # An import all from a module node: - name_toks.append(([self.skip()], none)) - node = ImportAll(name, relative) - else: - is_paren = self.current_str() == '(' - if is_paren: - lparen = self.expect('(') - targets = List[Tuple[str, str]]() - while True: - id, as_id, toks = self.parse_import_name() - if '%s.%s' % (name, id) == self.custom_typing_module: - if targets or self.current_str() == ',': - self.fail('You cannot import any other modules when you ' - 'import a custom typing module', - toks[0].line) - node = Import([('typing', as_id)]) - self.skip_until_break() - break - targets.append((id, as_id)) - if self.current_str() != ',': - name_toks.append((toks, none)) - break - name_toks.append((toks, self.expect(','))) - if is_paren and self.current_str() == ')': - break - if is_paren: - rparen = self.expect(')') - if node is None: - node = ImportFrom(name, relative, targets) - br = self.expect_break() - self.imports.append(node) - # TODO: Fix representation if there is a custom typing module import. - self.set_repr(node, noderepr.ImportFromRepr( - from_tok, rel_toks, components, import_tok, lparen, name_toks, rparen, br)) - if name == '__future__': - self.future_options.extend(target[0] for target in targets) - return node - - def parse_import_name(self) -> Tuple[str, str, List[Token]]: - tok = self.expect_type(Name) - name = tok.string - tokens = [tok] - if self.current_str() == 'as': - tokens.append(self.skip()) - as_name = self.expect_type(Name) - tokens.append(as_name) - return name, as_name.string, tokens - else: - return name, name, tokens - - def parse_qualified_name(self) -> Tuple[str, List[Token]]: - """Parse a name with an optional module qualifier. - - Return a tuple with the name as a string and a token array - containing all the components of the name. - """ - components = List[Token]() - tok = self.expect_type(Name) - n = tok.string - components.append(tok) - while self.current_str() == '.': - components.append(self.expect('.')) - tok = self.expect_type(Name) - n += '.' + tok.string - components.append(tok) - return n, components - - # Parsing global definitions - - def parse_defs(self) -> List[Node]: - defs = List[Node]() - while not self.eof(): - try: - defn = self.parse_statement() - if defn is not None: - if not self.try_combine_overloads(defn, defs): - defs.append(defn) - except ParseError: - pass - return defs - - def parse_class_def(self) -> ClassDef: - old_is_class_body = self.is_class_body - self.is_class_body = True - - type_tok = self.expect('class') - lparen = none - rparen = none - metaclass = None # type: str - - try: - commas, base_types = List[Token](), List[Type]() - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_type(name) - - if self.current_str() == '(': - lparen = self.skip() - while True: - if self.current_str() == ')': - break - if self.current_str() == 'metaclass': - metaclass = self.parse_metaclass() - break - base_types.append(self.parse_super_type()) - if self.current_str() != ',': - break - commas.append(self.skip()) - rparen = self.expect(')') - except ParseError: - pass - - defs, _ = self.parse_block() - - node = ClassDef(name, defs, None, base_types, metaclass=metaclass) - self.set_repr(node, noderepr.TypeDefRepr(type_tok, name_tok, - lparen, commas, rparen)) - return node - finally: - self.errors.pop_type() - self.is_class_body = old_is_class_body - - def parse_super_type(self) -> Type: - if (isinstance(self.current(), Name) and self.current_str() != 'void'): - return self.parse_type() - else: - self.parse_error() - - def parse_metaclass(self) -> str: - self.expect('metaclass') - self.expect('=') - return self.parse_qualified_name()[0] - - def parse_decorated_function_or_class(self) -> Node: - ats = List[Token]() - brs = List[Token]() - decorators = List[Node]() - while self.current_str() == '@': - ats.append(self.expect('@')) - decorators.append(self.parse_expression()) - brs.append(self.expect_break()) - if self.current_str() != 'class': - func = self.parse_function() - func.is_decorated = True - var = Var(func.name()) - # Types of decorated functions must always be inferred. - var.is_ready = False - var.set_line(decorators[0].line) - node = Decorator(func, decorators, var) - self.set_repr(node, noderepr.DecoratorRepr(ats, brs)) - return node - else: - cls = self.parse_class_def() - cls.decorators = decorators - return cls - - def parse_function(self) -> FuncDef: - def_tok = self.expect('def') - is_method = self.is_class_body - self.is_class_body = False - try: - (name, args, init, kinds, - typ, is_error, toks) = self.parse_function_header() - - body, comment_type = self.parse_block(allow_type=True) - if comment_type: - # The function has a # type: ... signature. - if typ: - self.errors.report( - def_tok.line, 'Function has duplicate type signatures') - sig = cast(Callable, comment_type) - if is_method: - self.check_argument_kinds(kinds, - [nodes.ARG_POS] + sig.arg_kinds, - def_tok.line) - # Add implicit 'self' argument to signature. - typ = Callable(List[Type]([AnyType()]) + sig.arg_types, - kinds, - [arg.name() for arg in args], - sig.ret_type, - None) - else: - self.check_argument_kinds(kinds, sig.arg_kinds, - def_tok.line) - typ = Callable(sig.arg_types, - kinds, - [arg.name() for arg in args], - sig.ret_type, - None) - - # If there was a serious error, we really cannot build a parse tree - # node. - if is_error: - return None - - node = FuncDef(name, args, kinds, init, body, typ) - name_tok, arg_reprs = toks - node.set_line(name_tok) - self.set_repr(node, noderepr.FuncRepr(def_tok, name_tok, - arg_reprs)) - return node - finally: - self.errors.pop_function() - self.is_class_body = is_method - - def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], - line: int) -> None: - """Check that * and ** arguments are consistent. - - Arguments: - funckinds: kinds of arguments in function definition - sigkinds: kinds of arguments in signature (after # type:) - """ - for kind, token in [(nodes.ARG_STAR, '*'), - (nodes.ARG_STAR2, '**')]: - if ((kind in funckinds and - sigkinds[funckinds.index(kind)] != kind) or - (funckinds.count(kind) != sigkinds.count(kind))): - self.fail( - "Inconsistent use of '{}' in function " - "signature".format(token), line) - - def parse_function_header(self) -> Tuple[str, List[Var], List[Node], - List[int], Callable, bool, - Tuple[Token, Any]]: - """Parse function header (a name followed by arguments) - - Returns a 7-tuple with the following items: - name - arguments - initializers - kinds - signature (annotation) - error flag (True if error) - (name token, representation of arguments) - """ - name_tok = none - name = '' - - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_function(name) - - (args, init, kinds, typ, arg_repr) = self.parse_args() - except ParseError: - if not isinstance(self.current(), Break): - self.ind -= 1 # Kludge: go back to the Break token - # Resynchronise parsing by going back over :, if present. - if isinstance(self.tok[self.ind - 1], Colon): - self.ind -= 1 - return (name, [], [], [], None, True, (name_tok, None)) - - return (name, args, init, kinds, typ, False, (name_tok, arg_repr)) - - def parse_args(self) -> Tuple[List[Var], List[Node], List[int], Callable, - noderepr.FuncArgsRepr]: - """Parse a function signature (...) [-> t].""" - lparen = self.expect('(') - - # Parse the argument list (everything within '(' and ')'). - (args, init, kinds, - has_inits, arg_names, - commas, asterisk, - assigns, arg_types) = self.parse_arg_list() - - rparen = self.expect(')') - - if self.current_str() == '->': - self.skip() - ret_type = self.parse_type() - else: - ret_type = None - - self.verify_argument_kinds(kinds, lparen.line) - - names = [] # type: List[str] - for arg in args: - names.append(arg.name()) - - annotation = self.build_func_annotation( - ret_type, arg_types, kinds, names, lparen.line) - - return (args, init, kinds, annotation, - noderepr.FuncArgsRepr(lparen, rparen, arg_names, commas, - assigns, asterisk)) - - def build_func_annotation(self, ret_type: Type, arg_types: List[Type], - kinds: List[int], names: List[str], - line: int, is_default_ret: bool = False) -> Callable: - # Are there any type annotations? - if ((ret_type and not is_default_ret) - or arg_types != [None] * len(arg_types)): - # Yes. Construct a type for the function signature. - return self.construct_function_type(arg_types, kinds, names, - ret_type, line) - else: - return None - - def parse_arg_list( - self, allow_signature: bool = True) -> Tuple[List[Var], List[Node], - List[int], bool, - List[Token], List[Token], - List[Token], List[Token], - List[Type]]: - """Parse function definition argument list. - - This includes everything between '(' and ')'). - - Return a 9-tuple with these items: - arguments, initializers, kinds, has inits, arg name tokens, - comma tokens, asterisk tokens, assignment tokens, argument types - """ - args = [] # type: List[Var] - kinds = [] # type: List[int] - names = [] # type: List[str] - init = [] # type: List[Node] - has_inits = False - arg_types = [] # type: List[Type] - - arg_names = [] # type: List[Token] - commas = [] # type: List[Token] - asterisk = [] # type: List[Token] - assigns = [] # type: List[Token] - - require_named = False - bare_asterisk_before = -1 - - if self.current_str() != ')' and self.current_str() != ':': - while self.current_str() != ')': - if self.current_str() == '*' and self.peek().string == ',': - if require_named: - # can only have one bare star, must be before any - # *args or **args - self.parse_error() - self.expect('*') - require_named = True - bare_asterisk_before = len(args) - elif self.current_str() in ['*', '**']: - if bare_asterisk_before == len(args): - # named arguments must follow bare * - self.parse_error() - asterisk.append(self.skip()) - isdict = asterisk[-1].string == '**' - name = self.expect_type(Name) - arg_names.append(name) - names.append(name.string) - var_arg = Var(name.string) - self.set_repr(var_arg, noderepr.VarRepr(name, none)) - args.append(var_arg) - init.append(None) - assigns.append(none) - if isdict: - kinds.append(nodes.ARG_STAR2) - else: - kinds.append(nodes.ARG_STAR) - arg_types.append(self.parse_arg_type(allow_signature)) - require_named = True - else: - name = self.expect_type(Name) - arg_names.append(name) - args.append(Var(name.string)) - arg_types.append(self.parse_arg_type(allow_signature)) - - if self.current_str() == '=': - assigns.append(self.expect('=')) - init.append(self.parse_expression(precedence[','])) - has_inits = True - if require_named: - kinds.append(nodes.ARG_NAMED) - else: - kinds.append(nodes.ARG_OPT) - else: - init.append(None) - assigns.append(none) - if require_named: - # required keyword-only argument - kinds.append(nodes.ARG_NAMED) - else: - kinds.append(nodes.ARG_POS) - - if self.current().string != ',': - break - commas.append(self.expect(',')) - - return (args, init, kinds, has_inits, arg_names, commas, asterisk, - assigns, arg_types) - - def parse_arg_type(self, allow_signature: bool) -> Type: - if self.current_str() == ':' and allow_signature: - self.skip() - return self.parse_type() - else: - return None - - def verify_argument_kinds(self, kinds: List[int], line: int) -> None: - found = Set[int]() - for i, kind in enumerate(kinds): - if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, - nodes.ARG_STAR, - nodes.ARG_STAR2]): - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: - self.fail('Invalid argument list', line) - found.add(kind) - - def construct_function_type(self, arg_types: List[Type], kinds: List[int], - names: List[str], ret_type: Type, - line: int) -> Callable: - # Complete the type annotation by replacing omitted types with 'Any'. - arg_types = arg_types[:] - for i in range(len(arg_types)): - if arg_types[i] is None: - arg_types[i] = AnyType() - if ret_type is None: - ret_type = AnyType() - return Callable(arg_types, kinds, names, ret_type, None, None, - None, [], line, None) - - # Parsing statements - - def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: - colon = self.expect(':') - if not isinstance(self.current(), Break): - # Block immediately after ':'. - node = Block([self.parse_statement()]).set_line(colon) - self.set_repr(node, noderepr.BlockRepr(colon, none, none, none)) - return cast(Block, node), None - else: - # Indented block. - br = self.expect_break() - type = self.parse_type_comment(br, signature=True) - indent = self.expect_indent() - stmt = [] # type: List[Node] - while (not isinstance(self.current(), Dedent) and - not isinstance(self.current(), Eof)): - try: - s = self.parse_statement() - if s is not None: - if not self.try_combine_overloads(s, stmt): - stmt.append(s) - except ParseError: - pass - dedent = none - if isinstance(self.current(), Dedent): - dedent = self.skip() - node = Block(stmt).set_line(colon) - self.set_repr(node, noderepr.BlockRepr(colon, br, indent, dedent)) - return cast(Block, node), type - - def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: - if isinstance(s, Decorator) and stmt: - fdef = cast(Decorator, s) - n = fdef.func.name() - if (isinstance(stmt[-1], Decorator) and - (cast(Decorator, stmt[-1])).func.name() == n): - stmt[-1] = OverloadedFuncDef([cast(Decorator, stmt[-1]), fdef]) - return True - elif (isinstance(stmt[-1], OverloadedFuncDef) and - (cast(OverloadedFuncDef, stmt[-1])).name() == n): - (cast(OverloadedFuncDef, stmt[-1])).items.append(fdef) - return True - return False - - def parse_statement(self) -> Node: - stmt = Undefined # type: Node - t = self.current() - ts = self.current_str() - if ts == 'if': - stmt = self.parse_if_stmt() - elif ts == 'def': - stmt = self.parse_function() - elif ts == 'while': - stmt = self.parse_while_stmt() - elif ts == 'return': - stmt = self.parse_return_stmt() - elif ts == 'for': - stmt = self.parse_for_stmt() - elif ts == 'try': - stmt = self.parse_try_stmt() - elif ts == 'break': - stmt = self.parse_break_stmt() - elif ts == 'continue': - stmt = self.parse_continue_stmt() - elif ts == 'pass': - stmt = self.parse_pass_stmt() - elif ts == 'raise': - stmt = self.parse_raise_stmt() - elif ts == 'import': - stmt = self.parse_import() - elif ts == 'from': - stmt = self.parse_import_from() - elif ts == 'class': - stmt = self.parse_class_def() - elif ts == 'global': - stmt = self.parse_global_decl() - elif ts == 'nonlocal' and self.pyversion >= 3: - stmt = self.parse_nonlocal_decl() - elif ts == 'assert': - stmt = self.parse_assert_stmt() - elif ts == 'yield': - stmt = self.parse_yield_stmt() - elif ts == 'del': - stmt = self.parse_del_stmt() - elif ts == 'with': - stmt = self.parse_with_stmt() - elif ts == '@': - stmt = self.parse_decorated_function_or_class() - elif ts == 'print' and (self.pyversion == 2 and - 'print_function' not in self.future_options): - stmt = self.parse_print_stmt() - else: - stmt = self.parse_expression_or_assignment() - if stmt is not None: - stmt.set_line(t) - return stmt - - def parse_expression_or_assignment(self) -> Node: - e = self.parse_expression(star_expr_allowed=True) - if self.current_str() == '=': - return self.parse_assignment(e) - elif self.current_str() in op_assign: - # Operator assignment statement. - op = self.current_str()[:-1] - assign = self.skip() - r = self.parse_expression() - br = self.expect_break() - node = OperatorAssignmentStmt(op, e, r) - self.set_repr(node, - noderepr.OperatorAssignmentStmtRepr(assign, br)) - return node - else: - # Expression statement. - br = self.expect_break() - expr = ExpressionStmt(e) - self.set_repr(expr, noderepr.ExpressionStmtRepr(br)) - return expr - - def parse_assignment(self, lv: Any) -> Node: - """Parse an assignment statement. - - Assume that lvalue has been parsed already, and the current token is =. - Also parse an optional '# type:' comment. - """ - assigns = [self.expect('=')] - lvalues = [lv] - - e = self.parse_expression(star_expr_allowed=True) - while self.current_str() == '=': - lvalues.append(e) - assigns.append(self.skip()) - e = self.parse_expression(star_expr_allowed=True) - br = self.expect_break() - - type = self.parse_type_comment(br, signature=False) - assignment = AssignmentStmt(lvalues, e, type) - self.set_repr(assignment, noderepr.AssignmentStmtRepr(assigns, br)) - return assignment - - def parse_return_stmt(self) -> ReturnStmt: - return_tok = self.expect('return') - expr = None # type: Node - if not isinstance(self.current(), Break): - expr = self.parse_expression() - if isinstance(expr, YieldFromExpr): # "yield from" expressions can't be returned. - return None - br = self.expect_break() - node = ReturnStmt(expr) - self.set_repr(node, noderepr.SimpleStmtRepr(return_tok, br)) - return node - - def parse_raise_stmt(self) -> RaiseStmt: - raise_tok = self.expect('raise') - expr = None # type: Node - from_expr = None # type: Node - from_tok = none - if not isinstance(self.current(), Break): - expr = self.parse_expression() - if self.current_str() == 'from': - from_tok = self.expect('from') - from_expr = self.parse_expression() - br = self.expect_break() - node = RaiseStmt(expr, from_expr) - self.set_repr(node, noderepr.RaiseStmtRepr(raise_tok, from_tok, br)) - return node - - def parse_assert_stmt(self) -> AssertStmt: - assert_tok = self.expect('assert') - expr = self.parse_expression() - br = self.expect_break() - node = AssertStmt(expr) - self.set_repr(node, noderepr.SimpleStmtRepr(assert_tok, br)) - return node - - def parse_yield_stmt(self) -> Union[YieldStmt, YieldFromStmt]: - yield_tok = self.expect('yield') - expr = None # type: Node - node = YieldStmt(expr) - if not isinstance(self.current(), Break): - if isinstance(self.current(), Keyword) and self.current_str() == "from": # Not go if it's not from - from_tok = self.expect("from") - expr = self.parse_expression() # Here comes when yield from is not assigned - node_from = YieldFromStmt(expr) - br = self.expect_break() - self.set_repr(node_from, noderepr.SimpleStmtRepr(yield_tok, br)) - return node_from # return here, we've gotted the type - else: - expr = self.parse_expression() - node = YieldStmt(expr) - br = self.expect_break() - self.set_repr(node, noderepr.SimpleStmtRepr(yield_tok, br)) - return node - - def parse_yield_from_expr(self) -> YieldFromExpr: - y_tok = self.expect("yield") - expr = None # type: Node - node = YieldFromExpr(expr) - if self.current_str() == "from": - f_tok = self.expect("from") - tok = self.parse_expression() # Here comes when yield from is assigned to a variable - node = YieldFromExpr(tok) - else: - # TODO - # Here comes the yield expression (ex: x = yield 3 ) - # tok = self.parse_expression() - # node = YieldExpr(tok) # Doesn't exist now - pass - return node - - def parse_del_stmt(self) -> DelStmt: - del_tok = self.expect('del') - expr = self.parse_expression() - br = self.expect_break() - node = DelStmt(expr) - self.set_repr(node, noderepr.SimpleStmtRepr(del_tok, br)) - return node - - def parse_break_stmt(self) -> BreakStmt: - break_tok = self.expect('break') - br = self.expect_break() - node = BreakStmt() - self.set_repr(node, noderepr.SimpleStmtRepr(break_tok, br)) - return node - - def parse_continue_stmt(self) -> ContinueStmt: - continue_tok = self.expect('continue') - br = self.expect_break() - node = ContinueStmt() - self.set_repr(node, noderepr.SimpleStmtRepr(continue_tok, br)) - return node - - def parse_pass_stmt(self) -> PassStmt: - pass_tok = self.expect('pass') - br = self.expect_break() - node = PassStmt() - self.set_repr(node, noderepr.SimpleStmtRepr(pass_tok, br)) - return node - - def parse_global_decl(self) -> GlobalDecl: - global_tok = self.expect('global') - name_toks, names, commas = self.parse_identifier_list() - br = self.expect_break() - node = GlobalDecl(names) - self.set_repr(node, noderepr.GlobalDeclRepr(global_tok, name_toks, - commas, br)) - return node - - def parse_nonlocal_decl(self) -> NonlocalDecl: - nonlocal_tok = self.expect('nonlocal') - name_toks, names, commas = self.parse_identifier_list() - br = self.expect_break() - node = NonlocalDecl(names) - self.set_repr(node, noderepr.NonlocalDeclRepr(nonlocal_tok, name_toks, - commas, br)) - return node - - def parse_identifier_list(self) -> Tuple[List[Token], List[str], List[Token]]: - names = List[str]() - name_toks = List[Token]() - commas = List[Token]() - while True: - n = self.expect_type(Name) - names.append(n.string) - name_toks.append(n) - if self.current_str() != ',': - break - commas.append(self.skip()) - return name_toks, names, commas - - def parse_while_stmt(self) -> WhileStmt: - is_error = False - while_tok = self.expect('while') - try: - expr = self.parse_expression() - except ParseError: - is_error = True - body, _ = self.parse_block() - if self.current_str() == 'else': - else_tok = self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - else_tok = none - if is_error is not None: - node = WhileStmt(expr, body, else_body) - self.set_repr(node, noderepr.WhileStmtRepr(while_tok, else_tok)) - return node - else: - return None - - def parse_for_stmt(self) -> ForStmt: - for_tok = self.expect('for') - index = self.parse_for_index_variables() - in_tok = self.expect('in') - expr = self.parse_expression() - - body, _ = self.parse_block() - - if self.current_str() == 'else': - else_tok = self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - else_tok = none - - node = ForStmt(index, expr, body, else_body) - self.set_repr(node, noderepr.ForStmtRepr(for_tok, in_tok, else_tok)) - return node - - def parse_for_index_variables(self) -> Node: - # Parse index variables of a 'for' statement. - index_items = List[Node]() - commas = List[Token]() - - while True: - v = self.parse_expression(precedence['in'], star_expr_allowed=True) # prevent parsing of for-stmt's 'in' - index_items.append(v) - if self.current_str() != ',': - commas.append(none) - break - commas.append(self.skip()) - - if len(index_items) == 1: - index = index_items[0] - else: - index = TupleExpr(index_items) - index.set_line(index_items[0].get_line()) - self.set_repr(index, noderepr.TupleExprRepr(none, commas, none)) - - return index - - def parse_if_stmt(self) -> IfStmt: - is_error = False - - if_tok = self.expect('if') - expr = List[Node]() - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - - body = [self.parse_block()[0]] - - elif_toks = List[Token]() - while self.current_str() == 'elif': - elif_toks.append(self.expect('elif')) - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - body.append(self.parse_block()[0]) - - if self.current_str() == 'else': - else_tok = self.expect('else') - else_body, _ = self.parse_block() - else: - else_tok = none - else_body = None - - if not is_error: - node = IfStmt(expr, body, else_body) - self.set_repr(node, noderepr.IfStmtRepr(if_tok, elif_toks, - else_tok)) - return node - else: - return None - - def parse_try_stmt(self) -> Node: - try_tok = self.expect('try') - body, _ = self.parse_block() - is_error = False - vars = List[NameExpr]() - types = List[Node]() - handlers = List[Block]() - except_toks, name_toks, as_toks, except_brs = (List[Token](), - List[Token](), - List[Token](), - List[Token]()) - while self.current_str() == 'except': - except_toks.append(self.expect('except')) - if not isinstance(self.current(), Colon): - try: - t = self.current() - types.append(self.parse_expression().set_line(t)) - if self.current_str() == 'as': - as_toks.append(self.expect('as')) - vars.append(self.parse_name_expr()) - else: - name_toks.append(none) - vars.append(None) - as_toks.append(none) - except ParseError: - is_error = True - else: - types.append(None) - vars.append(None) - as_toks.append(none) - handlers.append(self.parse_block()[0]) - if not is_error: - if self.current_str() == 'else': - else_tok = self.skip() - else_body, _ = self.parse_block() - else: - else_tok = none - else_body = None - if self.current_str() == 'finally': - finally_tok = self.expect('finally') - finally_body, _ = self.parse_block() - else: - finally_tok = none - finally_body = None - node = TryStmt(body, vars, types, handlers, else_body, - finally_body) - self.set_repr(node, noderepr.TryStmtRepr(try_tok, except_toks, - name_toks, as_toks, - else_tok, finally_tok)) - return node - else: - return None - - def parse_with_stmt(self) -> WithStmt: - with_tok = self.expect('with') - as_toks = List[Token]() - commas = List[Token]() - expr = List[Node]() - name = List[NameExpr]() - while True: - e = self.parse_expression(precedence[',']) - if self.current_str() == 'as': - as_toks.append(self.expect('as')) - n = self.parse_name_expr() - else: - as_toks.append(none) - n = None - expr.append(e) - name.append(n) - if self.current_str() != ',': - break - commas.append(self.expect(',')) - body, _ = self.parse_block() - node = WithStmt(expr, name, body) - self.set_repr(node, noderepr.WithStmtRepr(with_tok, as_toks, commas)) - return node - - def parse_print_stmt(self) -> PrintStmt: - self.expect('print') - args = List[Node]() - while not isinstance(self.current(), Break): - args.append(self.parse_expression(precedence[','])) - if self.current_str() == ',': - comma = True - self.skip() - else: - comma = False - break - self.expect_break() - return PrintStmt(args, newline=not comma) - - # Parsing expressions - - def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: - """Parse a subexpression within a specific precedence context.""" - expr = Undefined # type: Node - t = self.current() # Remember token for setting the line number. - - # Parse a "value" expression or unary operator expression and store - # that in expr. - s = self.current_str() - if s == '(': - # Parerenthesised expression or cast. - expr = self.parse_parentheses() - elif s == '[': - expr = self.parse_list_expr() - elif s in ['-', '+', 'not', '~']: - # Unary operation. - expr = self.parse_unary_expr() - elif s == 'lambda': - expr = self.parse_lambda_expr() - elif s == '{': - expr = self.parse_dict_or_set_expr() - elif s == '*' and star_expr_allowed: - expr = self.parse_star_expr() - else: - if isinstance(self.current(), Name): - # Name expression. - expr = self.parse_name_expr() - elif isinstance(self.current(), IntLit): - expr = self.parse_int_expr() - elif isinstance(self.current(), StrLit): - expr = self.parse_str_expr() - elif isinstance(self.current(), BytesLit): - expr = self.parse_bytes_literal() - elif isinstance(self.current(), UnicodeLit): - expr = self.parse_unicode_literal() - elif isinstance(self.current(), FloatLit): - expr = self.parse_float_expr() - elif isinstance(self.current(), ComplexLit): - expr = self.parse_complex_expr() - elif isinstance(t, Keyword) and s == "yield": - expr = self.parse_yield_from_expr() # The expression yield from and yield to assign - else: - # Invalid expression. - self.parse_error() - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type needs to - # deal with it separately. - if expr.line < 0: - expr.set_line(t) - - # Parse operations that require a left argument (stored in expr). - while True: - t = self.current() - s = self.current_str() - if s == '(': - # Call expression. - expr = self.parse_call_expr(expr) - elif s == '.': - # Member access expression. - expr = self.parse_member_expr(expr) - elif s == '[': - # Indexing expression. - expr = self.parse_index_expr(expr) - elif s == ',': - # The comma operator is used to build tuples. Comma also - # separates array items and function arguments, but in this - # case the precedence is too low to build a tuple. - if precedence[','] > prec: - expr = self.parse_tuple_expr(expr) - else: - break - elif s == 'for': - if precedence[''] > prec: - # List comprehension or generator expression. Parse as - # generator expression; it will be converted to list - # comprehension if needed elsewhere. - expr = self.parse_generator_expr(expr) - else: - break - elif s == 'if': - # Conditional expression. - if precedence[''] > prec: - expr = self.parse_conditional_expr(expr) - else: - break - else: - # Binary operation or a special case. - if isinstance(self.current(), Op): - op = self.current_str() - op_prec = precedence[op] - if op == 'not': - # Either "not in" or an error. - op_prec = precedence['in'] - if op_prec > prec: - if op in op_comp: - expr = self.parse_comparison_expr(expr, op_prec) - else: - expr = self.parse_bin_op_expr(expr, op_prec) - else: - # The operation cannot be associated with the - # current left operand due to the precedence - # context; let the caller handle it. - break - else: - # Not an operation that accepts a left argument; let the - # caller handle the rest. - break - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type - # needs to deal with it separately. - if expr.line < 0: - expr.set_line(t) - - return expr - - def parse_parentheses(self) -> Node: - lparen = self.skip() - if self.current_str() == ')': - # Empty tuple (). - expr = self.parse_empty_tuple_expr(lparen) # type: Node - else: - # Parenthesised expression. - expr = self.parse_expression(0, star_expr_allowed=True) - rparen = self.expect(')') - expr = ParenExpr(expr) - self.set_repr(expr, noderepr.ParenExprRepr(lparen, rparen)) - return expr - - def parse_star_expr(self) -> Node: - star = self.expect('*') - expr = self.parse_expression(precedence['*u']) - expr = StarExpr(expr) - if expr.line < 0: - expr.set_line(star) - self.set_repr(expr, noderepr.StarExprRepr(star)) - return expr - - def parse_empty_tuple_expr(self, lparen: Any) -> TupleExpr: - rparen = self.expect(')') - node = TupleExpr([]) - self.set_repr(node, noderepr.TupleExprRepr(lparen, [], rparen)) - return node - - def parse_list_expr(self) -> Node: - """Parse list literal or list comprehension.""" - items = List[Node]() - lbracket = self.expect('[') - commas = List[Token]() - while self.current_str() != ']' and not self.eol(): - items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) - if self.current_str() != ',': - break - commas.append(self.expect(',')) - if self.current_str() == 'for' and len(items) == 1: - items[0] = self.parse_generator_expr(items[0]) - rbracket = self.expect(']') - if len(items) == 1 and isinstance(items[0], GeneratorExpr): - list_comp = ListComprehension(cast(GeneratorExpr, items[0])) - self.set_repr(list_comp, noderepr.ListComprehensionRepr(lbracket, - rbracket)) - return list_comp - else: - expr = ListExpr(items) - self.set_repr(expr, noderepr.ListSetExprRepr(lbracket, commas, - rbracket, none, none)) - return expr - - def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: - indices, sequences, condlists, for_toks, in_toks, if_toklists = self.parse_comp_for() - - gen = GeneratorExpr(left_expr, indices, sequences, condlists) - gen.set_line(for_toks[0]) - self.set_repr(gen, noderepr.GeneratorExprRepr(for_toks, in_toks, if_toklists)) - return gen - - def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]], - List[Token], List[Token], List[List[Token]]]: - indices = List[Node]() - sequences = List[Node]() - for_toks = List[Token]() - in_toks = List[Token]() - if_toklists = List[List[Token]]() - condlists = List[List[Node]]() - while self.current_str() == 'for': - if_toks = List[Token]() - conds = List[Node]() - for_toks.append(self.expect('for')) - index = self.parse_for_index_variables() - indices.append(index) - in_toks.append(self.expect('in')) - sequence = self.parse_expression_list() - sequences.append(sequence) - while self.current_str() == 'if': - if_toks.append(self.skip()) - conds.append(self.parse_expression(precedence[''])) - if_toklists.append(if_toks) - condlists.append(conds) - - return indices, sequences, condlists, for_toks, in_toks, if_toklists - - def parse_expression_list(self) -> Node: - prec = precedence[''] - expr = self.parse_expression(prec) - if self.current_str() != ',': - return expr - else: - t = self.current() - return self.parse_tuple_expr(expr, prec).set_line(t) - - def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: - self.expect('if') - cond = self.parse_expression(precedence['']) - self.expect('else') - else_expr = self.parse_expression(precedence['']) - return ConditionalExpr(cond, left_expr, else_expr) - - def parse_dict_or_set_expr(self) -> Node: - items = List[Tuple[Node, Node]]() - lbrace = self.expect('{') - colons = List[Token]() - commas = List[Token]() - while self.current_str() != '}' and not self.eol(): - key = self.parse_expression(precedence['']) - if self.current_str() in [',', '}'] and items == []: - return self.parse_set_expr(key, lbrace) - elif self.current_str() == 'for' and items == []: - return self.parse_set_comprehension(key, lbrace) - elif self.current_str() != ':': - self.parse_error() - colons.append(self.expect(':')) - value = self.parse_expression(precedence['']) - if self.current_str() == 'for' and items == []: - return self.parse_dict_comprehension(key, value, lbrace, colons[0]) - items.append((key, value)) - if self.current_str() != ',': - break - commas.append(self.expect(',')) - rbrace = self.expect('}') - node = DictExpr(items) - self.set_repr(node, noderepr.DictExprRepr(lbrace, colons, commas, - rbrace, none, none, none)) - return node - - def parse_set_expr(self, first: Node, lbrace: Token) -> SetExpr: - items = [first] - commas = List[Token]() - while self.current_str() != '}' and not self.eol(): - commas.append(self.expect(',')) - if self.current_str() == '}': - break - items.append(self.parse_expression(precedence[','])) - rbrace = self.expect('}') - expr = SetExpr(items) - self.set_repr(expr, noderepr.ListSetExprRepr(lbrace, commas, - rbrace, none, none)) - return expr - - def parse_set_comprehension(self, expr: Node, lbrace: Token): - gen = self.parse_generator_expr(expr) - rbrace = self.expect('}') - set_comp = SetComprehension(gen) - self.set_repr(set_comp, noderepr.SetComprehensionRepr(lbrace, rbrace)) - return set_comp - - def parse_dict_comprehension(self, key: Node, value: Node, lbrace: Token, colon: Token) -> DictionaryComprehension: - indices, sequences, condlists, for_toks, in_toks, if_toklists = self.parse_comp_for() - dic = DictionaryComprehension(key, value, indices, sequences, condlists) - dic.set_line(colon) - rbrace = self.expect('}') - self.set_repr(dic, noderepr.DictionaryComprehensionRepr(lbrace, colon, for_toks, in_toks, if_toklists, rbrace)) - return dic - - def parse_tuple_expr(self, expr: Node, - prec: int = precedence[',']) -> TupleExpr: - items = [expr] - commas = List[Token]() - while True: - commas.append(self.expect(',')) - if (self.current_str() in [')', ']', '='] or - isinstance(self.current(), Break)): - break - items.append(self.parse_expression(prec, star_expr_allowed=True)) - if self.current_str() != ',': break - node = TupleExpr(items) - self.set_repr(node, noderepr.TupleExprRepr(none, commas, none)) - return node - - def parse_name_expr(self) -> NameExpr: - tok = self.expect_type(Name) - node = NameExpr(tok.string) - node.set_line(tok) - self.set_repr(node, noderepr.NameExprRepr(tok)) - return node - - def parse_int_expr(self) -> IntExpr: - tok = self.expect_type(IntLit) - s = tok.string - v = 0 - if len(s) > 2 and s[1] in 'xX': - v = int(s[2:], 16) - elif len(s) > 2 and s[1] in 'oO': - v = int(s[2:], 8) - else: - v = int(s) - node = IntExpr(v) - self.set_repr(node, noderepr.IntExprRepr(tok)) - return node - - def parse_str_expr(self) -> Node: - # XXX \uxxxx literals - tok = [self.expect_type(StrLit)] - value = (cast(StrLit, tok[0])).parsed() - while isinstance(self.current(), StrLit): - t = cast(StrLit, self.skip()) - tok.append(t) - value += t.parsed() - node = Undefined(Node) - if self.pyversion == 2 and 'unicode_literals' in self.future_options: - node = UnicodeExpr(value) - else: - node = StrExpr(value) - self.set_repr(node, noderepr.StrExprRepr(tok)) - return node - - def parse_bytes_literal(self) -> Node: - # XXX \uxxxx literals - tok = [self.expect_type(BytesLit)] - value = (cast(BytesLit, tok[0])).parsed() - while isinstance(self.current(), BytesLit): - t = cast(BytesLit, self.skip()) - tok.append(t) - value += t.parsed() - if self.pyversion >= 3: - node = BytesExpr(value) # type: Node - else: - node = StrExpr(value) - self.set_repr(node, noderepr.StrExprRepr(tok)) - return node - - def parse_unicode_literal(self) -> Node: - # XXX \uxxxx literals - tok = [self.expect_type(UnicodeLit)] - value = (cast(UnicodeLit, tok[0])).parsed() - while isinstance(self.current(), UnicodeLit): - t = cast(UnicodeLit, self.skip()) - tok.append(t) - value += t.parsed() - if self.pyversion >= 3: - # Python 3.3 supports u'...' as an alias of '...'. - node = StrExpr(value) # type: Node - else: - node = UnicodeExpr(value) - self.set_repr(node, noderepr.StrExprRepr(tok)) - return node - - def parse_float_expr(self) -> FloatExpr: - tok = self.expect_type(FloatLit) - node = FloatExpr(float(tok.string)) - self.set_repr(node, noderepr.FloatExprRepr(tok)) - return node - - def parse_complex_expr(self) -> ComplexExpr: - tok = self.expect_type(ComplexLit) - node = ComplexExpr(complex(tok.string)) - self.set_repr(node, noderepr.ComplexExprRepr(tok)) - return node - - def parse_call_expr(self, callee: Any) -> CallExpr: - lparen = self.expect('(') - (args, kinds, names, - commas, star, star2, assigns) = self.parse_arg_expr() - rparen = self.expect(')') - node = CallExpr(callee, args, kinds, names) - self.set_repr(node, noderepr.CallExprRepr(lparen, commas, star, star2, - assigns, rparen)) - return node - - def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str], - List[Token], Token, Token, - List[List[Token]]]: - """Parse arguments in a call expression (within '(' and ')'). - - Return a tuple with these items: - argument expressions - argument kinds - argument names (for named arguments; None for ordinary args) - comma tokens - * token (if any) - ** token (if any) - (assignment, name) tokens - """ - args = [] # type: List[Node] - kinds = [] # type: List[int] - names = [] # type: List[str] - star = none - star2 = none - commas = [] # type: List[Token] - keywords = [] # type: List[List[Token]] - var_arg = False - dict_arg = False - named_args = False - while self.current_str() != ')' and not self.eol() and not dict_arg: - if isinstance(self.current(), Name) and self.peek().string == '=': - # Named argument - name = self.expect_type(Name) - assign = self.expect('=') - kinds.append(nodes.ARG_NAMED) - names.append(name.string) - keywords.append([name, assign]) - named_args = True - elif (self.current_str() == '*' and not var_arg and not dict_arg - and not named_args): - # *args - var_arg = True - star = self.expect('*') - kinds.append(nodes.ARG_STAR) - names.append(None) - elif self.current_str() == '**': - # **kwargs - star2 = self.expect('**') - dict_arg = True - kinds.append(nodes.ARG_STAR2) - names.append(None) - elif not var_arg and not named_args: - # Ordinary argument - kinds.append(nodes.ARG_POS) - names.append(None) - else: - self.parse_error() - args.append(self.parse_expression(precedence[','])) - if self.current_str() != ',': - break - commas.append(self.expect(',')) - return args, kinds, names, commas, star, star2, keywords - - def parse_member_expr(self, expr: Any) -> Node: - dot = self.expect('.') - name = self.expect_type(Name) - node = Undefined(Node) - if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) - and expr.callee.name == 'super'): - # super() expression - node = SuperExpr(name.string) - self.set_repr(node, - noderepr.SuperExprRepr(expr.callee.repr.id, - expr.repr.lparen, - expr.repr.rparen, dot, name)) - else: - node = MemberExpr(expr, name.string) - self.set_repr(node, noderepr.MemberExprRepr(dot, name)) - return node - - def parse_index_expr(self, base: Any) -> IndexExpr: - lbracket = self.expect('[') - if self.current_str() != ':': - index = self.parse_expression(0) - else: - index = None - if self.current_str() == ':': - # Slice. - colon = self.expect(':') - if self.current_str() != ']' and self.current_str() != ':': - end_index = self.parse_expression(0) - else: - end_index = None - colon2 = none - stride = None # type: Node - if self.current_str() == ':': - colon2 = self.expect(':') - if self.current_str() != ']': - stride = self.parse_expression() - index = SliceExpr(index, end_index, stride).set_line(colon.line) - self.set_repr(index, noderepr.SliceExprRepr(colon, colon2)) - rbracket = self.expect(']') - node = IndexExpr(base, index) - self.set_repr(node, noderepr.IndexExprRepr(lbracket, rbracket)) - return node - - def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: - op = self.expect_type(Op) - op_str = op.string - if op_str == '~': - self.ind -= 1 - self.parse_error() - right = self.parse_expression(prec) - node = OpExpr(op_str, left, right) - self.set_repr(node, noderepr.OpExprRepr(op)) - return node - - def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: - operators = [] # type: List[Tuple[Token, Token]] - operators_str = [] # type: List[str] - operands = [left] - - while True: - op = self.expect_type(Op) - op2 = none - op_str = op.string - if op_str == 'not': - if self.current_str() == 'in': - op_str = 'not in' - op2 = self.skip() - else: - self.parse_error() - elif op_str == 'is' and self.current_str() == 'not': - op_str = 'is not' - op2 = self.skip() - - operators_str.append(op_str) - operators.append( (op, op2) ) - operand = self.parse_expression(prec) - operands.append(operand) - - # Continue if next token is a comparison operator - t = self.current() - s = self.current_str() - if s not in op_comp: - break - - node = ComparisonExpr(operators_str, operands) - self.set_repr(node, noderepr.ComparisonExprRepr(operators)) - return node - - - def parse_unary_expr(self) -> UnaryExpr: - op_tok = self.skip() - op = op_tok.string - if op == '-' or op == '+': - prec = precedence['-u'] - else: - prec = precedence[op] - expr = self.parse_expression(prec) - node = UnaryExpr(op, expr) - self.set_repr(node, noderepr.UnaryExprRepr(op_tok)) - return node - - def parse_lambda_expr(self) -> FuncExpr: - is_error = False - lambda_tok = self.expect('lambda') - - (args, init, kinds, has_inits, - arg_names, commas, asterisk, - assigns, arg_types) = self.parse_arg_list(allow_signature=False) - - names = List[str]() - for arg in args: - names.append(arg.name()) - - # Use 'object' as the placeholder return type; it will be inferred - # later. We can't use 'Any' since it could make type inference results - # less precise. - ret_type = UnboundType('__builtins__.object') - typ = self.build_func_annotation(ret_type, arg_types, kinds, names, - lambda_tok.line, is_default_ret=True) - - colon = self.expect(':') - - expr = self.parse_expression(precedence[',']) - - body = Block([ReturnStmt(expr).set_line(lambda_tok)]) - body.set_line(colon) - - node = FuncExpr(args, kinds, init, body, typ) - self.set_repr(node, - noderepr.FuncExprRepr( - lambda_tok, colon, - noderepr.FuncArgsRepr(none, none, arg_names, commas, - assigns, asterisk))) - return node - - # Helper methods - - def skip(self) -> Token: - self.ind += 1 - return self.tok[self.ind - 1] - - def expect(self, string: str) -> Token: - if self.current_str() == string: - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def expect_indent(self) -> Token: - if isinstance(self.current(), Indent): - return self.expect_type(Indent) - else: - self.fail('Expected an indented block', self.current().line) - return none - - def fail(self, msg: str, line: int) -> None: - self.errors.report(line, msg) - - def expect_type(self, typ: type) -> Token: - if isinstance(self.current(), typ): - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def expect_colon_and_break(self) -> Tuple[Token, Token]: - return self.expect_type(Colon), self.expect_type(Break) - - def expect_break(self) -> Token: - return self.expect_type(Break) - - def expect_end(self) -> Tuple[Token, Token]: - return self.expect('end'), self.expect_type(Break) - - def current(self) -> Token: - return self.tok[self.ind] - - def current_str(self) -> str: - return self.current().string - - def peek(self) -> Token: - return self.tok[self.ind + 1] - - def parse_error(self) -> None: - self.parse_error_at(self.current()) - raise ParseError() - - def parse_error_at(self, tok: Token, skip: bool = True) -> None: - msg = '' - if isinstance(tok, LexError): - msg = token_repr(tok) - msg = msg[0].upper() + msg[1:] - elif isinstance(tok, Indent) or isinstance(tok, Dedent): - msg = 'Inconsistent indentation' - else: - msg = 'Parse error before {}'.format(token_repr(tok)) - - self.errors.report(tok.line, msg) - - if skip: - self.skip_until_next_line() - - def skip_until_break(self) -> None: - n = 0 - while (not isinstance(self.current(), Break) - and not isinstance(self.current(), Eof)): - self.skip() - n += 1 - if isinstance(self.tok[self.ind - 1], Colon) and n > 1: - self.ind -= 1 - - def skip_until_next_line(self) -> None: - self.skip_until_break() - if isinstance(self.current(), Break): - self.skip() - - def eol(self) -> bool: - return isinstance(self.current(), Break) or self.eof() - - def eof(self) -> bool: - return isinstance(self.current(), Eof) - - # Type annotation related functionality - - def parse_type(self) -> Type: - line = self.current().line - try: - typ, self.ind = parse_type(self.tok, self.ind) - except TypeParseError as e: - self.parse_error_at(e.token) - raise ParseError() - return typ - - annotation_prefix_re = re.compile(r'#\s*type:') - - def parse_type_comment(self, token: Token, signature: bool) -> Type: - """Parse a '# type: ...' annotation. - - Return None if no annotation found. If signature is True, expect - a type signature of form (...) -> t. - """ - whitespace_or_comments = token.rep().strip() - if self.annotation_prefix_re.match(whitespace_or_comments): - type_as_str = whitespace_or_comments.split(':', 1)[1].strip() - tokens = lex.lex(type_as_str, token.line) - if len(tokens) < 2: - # Empty annotation (only Eof token) - self.errors.report(token.line, 'Empty type annotation') - return None - try: - if not signature: - type, index = parse_types(tokens, 0) - else: - type, index = parse_signature(tokens) - except TypeParseError as e: - self.parse_error_at(e.token, skip=False) - return None - if index < len(tokens) - 2: - self.parse_error_at(tokens[index], skip=False) - return None - return type - else: - return None - - # Representation management - - def set_repr(self, node: Node, repr: Any) -> None: - node.repr = repr - - def repr(self, node: Node) -> Any: - return node.repr - - def paren_repr(self, e: Node) -> Tuple[List[Token], List[Token]]: - """If e is a ParenExpr, return an array of left-paren tokens - (more that one if nested parens) and an array of corresponding - right-paren tokens. Otherwise, return [], []. - """ - if isinstance(e, ParenExpr): - lp, rp = self.paren_repr(e.expr) - lp.insert(0, self.repr(e).lparen) - rp.append(self.repr(e).rparen) - return lp, rp - else: - return [], [] - - -class ParseError(Exception): pass - - -def token_repr(tok: Token) -> str: - """Return a representation of a token for use in parse error messages.""" - if isinstance(tok, Break): - return 'end of line' - elif isinstance(tok, Eof): - return 'end of file' - elif isinstance(tok, Keyword) or isinstance(tok, Name): - return '"{}"'.format(tok.string) - elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): - return 'numeric literal' - elif isinstance(tok, StrLit): - return 'string literal' - elif (isinstance(tok, Punct) or isinstance(tok, Op) - or isinstance(tok, Colon)): - return tok.string - elif isinstance(tok, Bom): - return 'byte order mark' - elif isinstance(tok, Indent): - return 'indent' - elif isinstance(tok, Dedent): - return 'dedent' - else: - if isinstance(tok, LexError): - t = tok.type - if t == lex.NUMERIC_LITERAL_ERROR: - return 'invalid numeric literal' - elif t == lex.UNTERMINATED_STRING_LITERAL: - return 'unterminated string literal' - elif t == lex.INVALID_CHARACTER: - msg = 'unrecognized character' - if ord(tok.string) in range(33, 127): - msg += ' ' + tok.string - return msg - elif t == lex.INVALID_UTF8_SEQUENCE: - return 'invalid UTF-8 sequence' - elif t == lex.NON_ASCII_CHARACTER_IN_COMMENT: - return 'non-ASCII character in comment' - elif t == lex.NON_ASCII_CHARACTER_IN_STRING: - return 'non-ASCII character in string' - elif t == lex.INVALID_DEDENT: - return 'inconsistent indentation' - raise ValueError('Unknown token {}'.format(repr(tok))) - - -def unwrap_parens(node: Node) -> Node: - """Unwrap any outer parentheses in node. - - If the node is a parenthesised expression, recursively find the first - non-parenthesised subexpression and return that. Otherwise, return node. + The python_version (major, minor) option determines the Python syntax variant. """ - if isinstance(node, ParenExpr): - return unwrap_parens(node.expr) - else: - return node - + if options.transform_source is not None: + source = options.transform_source(source) + import mypy.fastparse -if __name__ == '__main__': - # Parse a file and dump the AST (or display errors). - import sys - if len(sys.argv) != 2: - print('Usage: parse.py FILE') - sys.exit(2) - fnam = sys.argv[1] - s = open(fnam).read() - errors = Errors() - try: - tree = parse(s, fnam) - print(tree) - except CompileError as e: - for msg in e.messages: - print(msg) + tree = mypy.fastparse.parse(source, fnam=fnam, module=module, errors=errors, options=options) + if raise_on_error and errors.is_errors(): + errors.raise_error() + return tree diff --git a/mypy/parsetype.py b/mypy/parsetype.py deleted file mode 100644 index 2ac76adb29c1..000000000000 --- a/mypy/parsetype.py +++ /dev/null @@ -1,234 +0,0 @@ -"""Type parser""" - -from typing import List, Tuple, Union, cast - -from mypy.types import ( - Type, UnboundType, TupleType, UnionType, TypeList, AnyType, Callable, StarType -) -from mypy.typerepr import CommonTypeRepr, ListTypeRepr -from mypy.lex import Token, Name, StrLit, Break, lex -from mypy import nodes - - -none = Token('') # Empty token - - -class TypeParseError(Exception): - def __init__(self, token: Token, index: int) -> None: - super().__init__() - self.token = token - self.index = index - - -def parse_type(tok: List[Token], index: int) -> Tuple[Type, int]: - """Parse a type. - - Return (type, index after type). - """ - - p = TypeParser(tok, index) - return p.parse_type(), p.index() - - -def parse_types(tok: List[Token], index: int) -> Tuple[Type, int]: - """Parse one or more types separated by commas (optional parentheses). - - Return (type, index after type). - """ - - p = TypeParser(tok, index) - return p.parse_types(), p.index() - - -class TypeParser: - def __init__(self, tok: List[Token], ind: int) -> None: - self.tok = tok - self.ind = ind - - def index(self) -> int: - return self.ind - - def parse_type(self) -> Type: - """Parse a type.""" - t = self.current_token() - if t.string == '(': - return self.parse_parens() - if isinstance(t, Name): - return self.parse_named_type() - elif t.string == '[': - return self.parse_type_list() - elif t.string == '*': - return self.parse_star_type() - elif isinstance(t, StrLit): - # Type escaped as string literal. - typestr = t.parsed() - line = t.line - self.skip() - try: - result = parse_str_as_type(typestr, line) - except TypeParseError as e: - raise TypeParseError(e.token, self.ind) - return result - else: - self.parse_error() - - def parse_parens(self) -> Type: - self.expect('(') - types = self.parse_types() - self.expect(')') - return types - - def parse_types(self) -> Type: - """ Parse either a single type or a comma separated - list of types as a tuple type. In the latter case, a - trailing comma is needed when the list contains only - a single type (and optional otherwise). - - int -> int - int, -> TupleType[int] - int, int, int -> TupleType[int, int, int] - """ - type = self.parse_type() - if self.current_token_str() == ',': - items = [type] - while self.current_token_str() == ',': - self.skip() - if self.current_token_str() == ')': - break - items.append(self.parse_type()) - type = TupleType(items, None, type.line) - return type - - def parse_type_list(self) -> TypeList: - """Parse type list [t, ...].""" - lbracket = self.expect('[') - commas = [] # type: List[Token] - items = [] # type: List[Type] - while self.current_token_str() != ']': - t = self.parse_type() - items.append(t) - if self.current_token_str() != ',': - break - commas.append(self.skip()) - rbracket = self.expect(']') - return TypeList(items, line=lbracket.line) - - def parse_named_type(self) -> Type: - line = self.current_token().line - name = '' - components = [] # type: List[Token] - - components.append(self.expect_type(Name)) - name += components[-1].string - - while self.current_token_str() == '.': - components.append(self.skip()) - t = self.expect_type(Name) - components.append(t) - name += '.' + t.string - - lbracket, rbracket = none, none - commas = [] # type: List[Token] - args = [] # type: List[Type] - if self.current_token_str() == '[': - lbracket = self.skip() - - while True: - typ = self.parse_type() - args.append(typ) - if self.current_token_str() != ',': - break - commas.append(self.skip()) - - rbracket = self.expect(']') - - typ = UnboundType(name, args, line, CommonTypeRepr(components, - lbracket, - commas, rbracket)) - return typ - - def parse_star_type(self) -> Type: - star = self.expect('*') - type = self.parse_type() - return StarType(type, star.line) - - # Helpers - - def skip(self) -> Token: - self.ind += 1 - return self.tok[self.ind - 1] - - def expect(self, string: str) -> Token: - if self.tok[self.ind].string == string: - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def expect_type(self, typ: type) -> Token: - if isinstance(self.current_token(), typ): - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def current_token(self) -> Token: - return self.tok[self.ind] - - def current_token_str(self) -> str: - return self.current_token().string - - def parse_error(self) -> None: - raise TypeParseError(self.tok[self.ind], self.ind) - - -def parse_str_as_type(typestr: str, line: int) -> Type: - """Parse a type represented as a string. - - Raise TypeParseError on parse error. - """ - - typestr = typestr.strip() - tokens = lex(typestr, line) - result, i = parse_type(tokens, 0) - if i < len(tokens) - 2: - raise TypeParseError(tokens[i], i) - return result - - -def parse_signature(tokens: List[Token]) -> Tuple[Callable, int]: - """Parse signature of form (argtype, ...) -> ... - - Return tuple (signature type, token index). - """ - i = 0 - if tokens[i].string != '(': - raise TypeParseError(tokens[i], i) - i += 1 - arg_types = List[Type]() - arg_kinds = List[int]() - while tokens[i].string != ')': - if tokens[i].string == '*': - arg_kinds.append(nodes.ARG_STAR) - i += 1 - elif tokens[i].string == '**': - arg_kinds.append(nodes.ARG_STAR2) - i += 1 - else: - arg_kinds.append(nodes.ARG_POS) - arg, i = parse_type(tokens, i) - arg_types.append(arg) - next = tokens[i].string - if next not in ',)': - raise TypeParseError(tokens[i], i) - if next == ',': - i += 1 - i += 1 - if tokens[i].string != '->': - raise TypeParseError(tokens[i], i) - i += 1 - ret_type, i = parse_type(tokens, i) - return Callable(arg_types, - arg_kinds, - [None] * len(arg_types), - ret_type, None), i diff --git a/mypy/partially_defined.py b/mypy/partially_defined.py new file mode 100644 index 000000000000..da0bb517189a --- /dev/null +++ b/mypy/partially_defined.py @@ -0,0 +1,680 @@ +from __future__ import annotations + +from enum import Enum + +from mypy import checker, errorcodes +from mypy.messages import MessageBuilder +from mypy.nodes import ( + AssertStmt, + AssignmentExpr, + AssignmentStmt, + BreakStmt, + ClassDef, + Context, + ContinueStmt, + DictionaryComprehension, + Expression, + ExpressionStmt, + ForStmt, + FuncDef, + FuncItem, + GeneratorExpr, + GlobalDecl, + IfStmt, + Import, + ImportFrom, + LambdaExpr, + ListExpr, + Lvalue, + MatchStmt, + MypyFile, + NameExpr, + NonlocalDecl, + RaiseStmt, + ReturnStmt, + StarExpr, + SymbolTable, + TryStmt, + TupleExpr, + TypeAliasStmt, + WhileStmt, + WithStmt, + implicit_module_attrs, +) +from mypy.options import Options +from mypy.patterns import AsPattern, StarredPattern +from mypy.reachability import ALWAYS_TRUE, infer_pattern_value +from mypy.traverser import ExtendedTraverserVisitor +from mypy.types import Type, UninhabitedType + + +class BranchState: + """BranchState contains information about variable definition at the end of a branching statement. + `if` and `match` are examples of branching statements. + + `may_be_defined` contains variables that were defined in only some branches. + `must_be_defined` contains variables that were defined in all branches. + """ + + def __init__( + self, + must_be_defined: set[str] | None = None, + may_be_defined: set[str] | None = None, + skipped: bool = False, + ) -> None: + if may_be_defined is None: + may_be_defined = set() + if must_be_defined is None: + must_be_defined = set() + + self.may_be_defined = set(may_be_defined) + self.must_be_defined = set(must_be_defined) + self.skipped = skipped + + def copy(self) -> BranchState: + return BranchState( + must_be_defined=set(self.must_be_defined), + may_be_defined=set(self.may_be_defined), + skipped=self.skipped, + ) + + +class BranchStatement: + def __init__(self, initial_state: BranchState | None = None) -> None: + if initial_state is None: + initial_state = BranchState() + self.initial_state = initial_state + self.branches: list[BranchState] = [ + BranchState( + must_be_defined=self.initial_state.must_be_defined, + may_be_defined=self.initial_state.may_be_defined, + ) + ] + + def copy(self) -> BranchStatement: + result = BranchStatement(self.initial_state) + result.branches = [b.copy() for b in self.branches] + return result + + def next_branch(self) -> None: + self.branches.append( + BranchState( + must_be_defined=self.initial_state.must_be_defined, + may_be_defined=self.initial_state.may_be_defined, + ) + ) + + def record_definition(self, name: str) -> None: + assert len(self.branches) > 0 + self.branches[-1].must_be_defined.add(name) + self.branches[-1].may_be_defined.discard(name) + + def delete_var(self, name: str) -> None: + assert len(self.branches) > 0 + self.branches[-1].must_be_defined.discard(name) + self.branches[-1].may_be_defined.discard(name) + + def record_nested_branch(self, state: BranchState) -> None: + assert len(self.branches) > 0 + current_branch = self.branches[-1] + if state.skipped: + current_branch.skipped = True + return + current_branch.must_be_defined.update(state.must_be_defined) + current_branch.may_be_defined.update(state.may_be_defined) + current_branch.may_be_defined.difference_update(current_branch.must_be_defined) + + def skip_branch(self) -> None: + assert len(self.branches) > 0 + self.branches[-1].skipped = True + + def is_possibly_undefined(self, name: str) -> bool: + assert len(self.branches) > 0 + return name in self.branches[-1].may_be_defined + + def is_undefined(self, name: str) -> bool: + assert len(self.branches) > 0 + branch = self.branches[-1] + return name not in branch.may_be_defined and name not in branch.must_be_defined + + def is_defined_in_a_branch(self, name: str) -> bool: + assert len(self.branches) > 0 + for b in self.branches: + if name in b.must_be_defined or name in b.may_be_defined: + return True + return False + + def done(self) -> BranchState: + # First, compute all vars, including skipped branches. We include skipped branches + # because our goal is to capture all variables that semantic analyzer would + # consider defined. + all_vars = set() + for b in self.branches: + all_vars.update(b.may_be_defined) + all_vars.update(b.must_be_defined) + # For the rest of the things, we only care about branches that weren't skipped. + non_skipped_branches = [b for b in self.branches if not b.skipped] + if non_skipped_branches: + must_be_defined = non_skipped_branches[0].must_be_defined + for b in non_skipped_branches[1:]: + must_be_defined.intersection_update(b.must_be_defined) + else: + must_be_defined = set() + # Everything that wasn't defined in all branches but was defined + # in at least one branch should be in `may_be_defined`! + may_be_defined = all_vars.difference(must_be_defined) + return BranchState( + must_be_defined=must_be_defined, + may_be_defined=may_be_defined, + skipped=len(non_skipped_branches) == 0, + ) + + +class ScopeType(Enum): + Global = 1 + Class = 2 + Func = 3 + Generator = 4 + + +class Scope: + def __init__(self, stmts: list[BranchStatement], scope_type: ScopeType) -> None: + self.branch_stmts: list[BranchStatement] = stmts + self.scope_type = scope_type + self.undefined_refs: dict[str, set[NameExpr]] = {} + + def copy(self) -> Scope: + result = Scope([s.copy() for s in self.branch_stmts], self.scope_type) + result.undefined_refs = self.undefined_refs.copy() + return result + + def record_undefined_ref(self, o: NameExpr) -> None: + if o.name not in self.undefined_refs: + self.undefined_refs[o.name] = set() + self.undefined_refs[o.name].add(o) + + def pop_undefined_ref(self, name: str) -> set[NameExpr]: + return self.undefined_refs.pop(name, set()) + + +class DefinedVariableTracker: + """DefinedVariableTracker manages the state and scope for the UndefinedVariablesVisitor.""" + + def __init__(self) -> None: + # There's always at least one scope. Within each scope, there's at least one "global" BranchingStatement. + self.scopes: list[Scope] = [Scope([BranchStatement()], ScopeType.Global)] + # disable_branch_skip is used to disable skipping a branch due to a return/raise/etc. This is useful + # in things like try/except/finally statements. + self.disable_branch_skip = False + + def copy(self) -> DefinedVariableTracker: + result = DefinedVariableTracker() + result.scopes = [s.copy() for s in self.scopes] + result.disable_branch_skip = self.disable_branch_skip + return result + + def _scope(self) -> Scope: + assert len(self.scopes) > 0 + return self.scopes[-1] + + def enter_scope(self, scope_type: ScopeType) -> None: + assert len(self._scope().branch_stmts) > 0 + initial_state = None + if scope_type == ScopeType.Generator: + # Generators are special because they inherit the outer scope. + initial_state = self._scope().branch_stmts[-1].branches[-1] + self.scopes.append(Scope([BranchStatement(initial_state)], scope_type)) + + def exit_scope(self) -> None: + self.scopes.pop() + + def in_scope(self, scope_type: ScopeType) -> bool: + return self._scope().scope_type == scope_type + + def start_branch_statement(self) -> None: + assert len(self._scope().branch_stmts) > 0 + self._scope().branch_stmts.append( + BranchStatement(self._scope().branch_stmts[-1].branches[-1]) + ) + + def next_branch(self) -> None: + assert len(self._scope().branch_stmts) > 1 + self._scope().branch_stmts[-1].next_branch() + + def end_branch_statement(self) -> None: + assert len(self._scope().branch_stmts) > 1 + result = self._scope().branch_stmts.pop().done() + self._scope().branch_stmts[-1].record_nested_branch(result) + + def skip_branch(self) -> None: + # Only skip branch if we're outside of "root" branch statement. + if len(self._scope().branch_stmts) > 1 and not self.disable_branch_skip: + self._scope().branch_stmts[-1].skip_branch() + + def record_definition(self, name: str) -> None: + assert len(self.scopes) > 0 + assert len(self.scopes[-1].branch_stmts) > 0 + self._scope().branch_stmts[-1].record_definition(name) + + def delete_var(self, name: str) -> None: + assert len(self.scopes) > 0 + assert len(self.scopes[-1].branch_stmts) > 0 + self._scope().branch_stmts[-1].delete_var(name) + + def record_undefined_ref(self, o: NameExpr) -> None: + """Records an undefined reference. These can later be retrieved via `pop_undefined_ref`.""" + assert len(self.scopes) > 0 + self._scope().record_undefined_ref(o) + + def pop_undefined_ref(self, name: str) -> set[NameExpr]: + """If name has previously been reported as undefined, the NameExpr that was called will be returned.""" + assert len(self.scopes) > 0 + return self._scope().pop_undefined_ref(name) + + def is_possibly_undefined(self, name: str) -> bool: + assert len(self._scope().branch_stmts) > 0 + # A variable is undefined if it's in a set of `may_be_defined` but not in `must_be_defined`. + return self._scope().branch_stmts[-1].is_possibly_undefined(name) + + def is_defined_in_different_branch(self, name: str) -> bool: + """This will return true if a variable is defined in a branch that's not the current branch.""" + assert len(self._scope().branch_stmts) > 0 + stmt = self._scope().branch_stmts[-1] + if not stmt.is_undefined(name): + return False + for stmt in self._scope().branch_stmts: + if stmt.is_defined_in_a_branch(name): + return True + return False + + def is_undefined(self, name: str) -> bool: + assert len(self._scope().branch_stmts) > 0 + return self._scope().branch_stmts[-1].is_undefined(name) + + +class Loop: + def __init__(self) -> None: + self.has_break = False + + +class PossiblyUndefinedVariableVisitor(ExtendedTraverserVisitor): + """Detects the following cases: + - A variable that's defined only part of the time. + - If a variable is used before definition + + An example of a partial definition: + if foo(): + x = 1 + print(x) # Error: "x" may be undefined. + + Example of a used before definition: + x = y + y: int = 2 + + Note that this code does not detect variables not defined in any of the branches -- that is + handled by the semantic analyzer. + """ + + def __init__( + self, + msg: MessageBuilder, + type_map: dict[Expression, Type], + options: Options, + names: SymbolTable, + ) -> None: + self.msg = msg + self.type_map = type_map + self.options = options + self.builtins = SymbolTable() + builtins_mod = names.get("__builtins__", None) + if builtins_mod: + assert isinstance(builtins_mod.node, MypyFile) + self.builtins = builtins_mod.node.names + self.loops: list[Loop] = [] + self.try_depth = 0 + self.tracker = DefinedVariableTracker() + for name in implicit_module_attrs: + self.tracker.record_definition(name) + + def var_used_before_def(self, name: str, context: Context) -> None: + if self.msg.errors.is_error_code_enabled(errorcodes.USED_BEFORE_DEF): + self.msg.var_used_before_def(name, context) + + def variable_may_be_undefined(self, name: str, context: Context) -> None: + if self.msg.errors.is_error_code_enabled(errorcodes.POSSIBLY_UNDEFINED): + self.msg.variable_may_be_undefined(name, context) + + def process_definition(self, name: str) -> None: + # Was this name previously used? If yes, it's a used-before-definition error. + if not self.tracker.in_scope(ScopeType.Class): + refs = self.tracker.pop_undefined_ref(name) + for ref in refs: + if self.loops: + self.variable_may_be_undefined(name, ref) + else: + self.var_used_before_def(name, ref) + else: + # Errors in class scopes are caught by the semantic analyzer. + pass + self.tracker.record_definition(name) + + def visit_global_decl(self, o: GlobalDecl) -> None: + for name in o.names: + self.process_definition(name) + super().visit_global_decl(o) + + def visit_nonlocal_decl(self, o: NonlocalDecl) -> None: + for name in o.names: + self.process_definition(name) + super().visit_nonlocal_decl(o) + + def process_lvalue(self, lvalue: Lvalue | None) -> None: + if isinstance(lvalue, NameExpr): + self.process_definition(lvalue.name) + elif isinstance(lvalue, StarExpr): + self.process_lvalue(lvalue.expr) + elif isinstance(lvalue, (ListExpr, TupleExpr)): + for item in lvalue.items: + self.process_lvalue(item) + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + for lvalue in o.lvalues: + self.process_lvalue(lvalue) + super().visit_assignment_stmt(o) + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + o.value.accept(self) + self.process_lvalue(o.target) + + def visit_if_stmt(self, o: IfStmt) -> None: + for e in o.expr: + e.accept(self) + self.tracker.start_branch_statement() + for b in o.body: + if b.is_unreachable: + continue + b.accept(self) + self.tracker.next_branch() + if o.else_body: + if not o.else_body.is_unreachable: + o.else_body.accept(self) + else: + self.tracker.skip_branch() + self.tracker.end_branch_statement() + + def visit_match_stmt(self, o: MatchStmt) -> None: + o.subject.accept(self) + self.tracker.start_branch_statement() + for i in range(len(o.patterns)): + pattern = o.patterns[i] + pattern.accept(self) + guard = o.guards[i] + if guard is not None: + guard.accept(self) + if not o.bodies[i].is_unreachable: + o.bodies[i].accept(self) + else: + self.tracker.skip_branch() + is_catchall = infer_pattern_value(pattern) == ALWAYS_TRUE + if not is_catchall: + self.tracker.next_branch() + self.tracker.end_branch_statement() + + def visit_func_def(self, o: FuncDef) -> None: + self.process_definition(o.name) + super().visit_func_def(o) + + def visit_func(self, o: FuncItem) -> None: + if o.is_dynamic() and not self.options.check_untyped_defs: + return + + args = o.arguments or [] + # Process initializers (defaults) outside the function scope. + for arg in args: + if arg.initializer is not None: + arg.initializer.accept(self) + + self.tracker.enter_scope(ScopeType.Func) + for arg in args: + self.process_definition(arg.variable.name) + super().visit_var(arg.variable) + o.body.accept(self) + self.tracker.exit_scope() + + def visit_generator_expr(self, o: GeneratorExpr) -> None: + self.tracker.enter_scope(ScopeType.Generator) + for idx in o.indices: + self.process_lvalue(idx) + super().visit_generator_expr(o) + self.tracker.exit_scope() + + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + self.tracker.enter_scope(ScopeType.Generator) + for idx in o.indices: + self.process_lvalue(idx) + super().visit_dictionary_comprehension(o) + self.tracker.exit_scope() + + def visit_for_stmt(self, o: ForStmt) -> None: + o.expr.accept(self) + self.process_lvalue(o.index) + o.index.accept(self) + self.tracker.start_branch_statement() + loop = Loop() + self.loops.append(loop) + o.body.accept(self) + self.tracker.next_branch() + self.tracker.end_branch_statement() + if o.else_body is not None: + # If the loop has a `break` inside, `else` is executed conditionally. + # If the loop doesn't have a `break` either the function will return or + # execute the `else`. + has_break = loop.has_break + if has_break: + self.tracker.start_branch_statement() + self.tracker.next_branch() + o.else_body.accept(self) + if has_break: + self.tracker.end_branch_statement() + self.loops.pop() + + def visit_return_stmt(self, o: ReturnStmt) -> None: + super().visit_return_stmt(o) + self.tracker.skip_branch() + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + self.tracker.enter_scope(ScopeType.Func) + super().visit_lambda_expr(o) + self.tracker.exit_scope() + + def visit_assert_stmt(self, o: AssertStmt) -> None: + super().visit_assert_stmt(o) + if checker.is_false_literal(o.expr): + self.tracker.skip_branch() + + def visit_raise_stmt(self, o: RaiseStmt) -> None: + super().visit_raise_stmt(o) + self.tracker.skip_branch() + + def visit_continue_stmt(self, o: ContinueStmt) -> None: + super().visit_continue_stmt(o) + self.tracker.skip_branch() + + def visit_break_stmt(self, o: BreakStmt) -> None: + super().visit_break_stmt(o) + if self.loops: + self.loops[-1].has_break = True + self.tracker.skip_branch() + + def visit_expression_stmt(self, o: ExpressionStmt) -> None: + if isinstance(self.type_map.get(o.expr, None), (UninhabitedType, type(None))): + self.tracker.skip_branch() + super().visit_expression_stmt(o) + + def visit_try_stmt(self, o: TryStmt) -> None: + """ + Note that finding undefined vars in `finally` requires different handling from + the rest of the code. In particular, we want to disallow skipping branches due to jump + statements in except/else clauses for finally but not for other cases. Imagine a case like: + def f() -> int: + try: + x = 1 + except: + # This jump statement needs to be handled differently depending on whether or + # not we're trying to process `finally` or not. + return 0 + finally: + # `x` may be undefined here. + pass + # `x` is always defined here. + return x + """ + self.try_depth += 1 + if o.finally_body is not None: + # In order to find undefined vars in `finally`, we need to + # process try/except with branch skipping disabled. However, for the rest of the code + # after finally, we need to process try/except with branch skipping enabled. + # Therefore, we need to process try/finally twice. + # Because processing is not idempotent, we should make a copy of the tracker. + old_tracker = self.tracker.copy() + self.tracker.disable_branch_skip = True + self.process_try_stmt(o) + self.tracker = old_tracker + self.process_try_stmt(o) + self.try_depth -= 1 + + def process_try_stmt(self, o: TryStmt) -> None: + """ + Processes try statement decomposing it into the following: + if ...: + body + else_body + elif ...: + except 1 + elif ...: + except 2 + else: + except n + finally + """ + self.tracker.start_branch_statement() + o.body.accept(self) + if o.else_body is not None: + o.else_body.accept(self) + if len(o.handlers) > 0: + assert len(o.handlers) == len(o.vars) == len(o.types) + for i in range(len(o.handlers)): + self.tracker.next_branch() + exc_type = o.types[i] + if exc_type is not None: + exc_type.accept(self) + var = o.vars[i] + if var is not None: + self.process_definition(var.name) + var.accept(self) + o.handlers[i].accept(self) + if var is not None: + self.tracker.delete_var(var.name) + self.tracker.end_branch_statement() + + if o.finally_body is not None: + o.finally_body.accept(self) + + def visit_while_stmt(self, o: WhileStmt) -> None: + o.expr.accept(self) + self.tracker.start_branch_statement() + loop = Loop() + self.loops.append(loop) + o.body.accept(self) + has_break = loop.has_break + if not checker.is_true_literal(o.expr): + # If this is a loop like `while True`, we can consider the body to be + # a single branch statement (we're guaranteed that the body is executed at least once). + # If not, call next_branch() to make all variables defined there conditional. + self.tracker.next_branch() + self.tracker.end_branch_statement() + if o.else_body is not None: + # If the loop has a `break` inside, `else` is executed conditionally. + # If the loop doesn't have a `break` either the function will return or + # execute the `else`. + if has_break: + self.tracker.start_branch_statement() + self.tracker.next_branch() + if o.else_body: + o.else_body.accept(self) + if has_break: + self.tracker.end_branch_statement() + self.loops.pop() + + def visit_as_pattern(self, o: AsPattern) -> None: + if o.name is not None: + self.process_lvalue(o.name) + super().visit_as_pattern(o) + + def visit_starred_pattern(self, o: StarredPattern) -> None: + if o.capture is not None: + self.process_lvalue(o.capture) + super().visit_starred_pattern(o) + + def visit_name_expr(self, o: NameExpr) -> None: + if o.name in self.builtins and self.tracker.in_scope(ScopeType.Global): + return + if self.tracker.is_possibly_undefined(o.name): + # A variable is only defined in some branches. + self.variable_may_be_undefined(o.name, o) + # We don't want to report the error on the same variable multiple times. + self.tracker.record_definition(o.name) + elif self.tracker.is_defined_in_different_branch(o.name): + # A variable is defined in one branch but used in a different branch. + if self.loops or self.try_depth > 0: + # If we're in a loop or in a try, we can't be sure that this variable + # is undefined. Report it as "may be undefined". + self.variable_may_be_undefined(o.name, o) + else: + self.var_used_before_def(o.name, o) + elif self.tracker.is_undefined(o.name): + # A variable is undefined. It could be due to two things: + # 1. A variable is just totally undefined + # 2. The variable is defined later in the code. + # Case (1) will be caught by semantic analyzer. Case (2) is a forward ref that should + # be caught by this visitor. Save the ref for later, so that if we see a definition, + # we know it's a used-before-definition scenario. + self.tracker.record_undefined_ref(o) + super().visit_name_expr(o) + + def visit_with_stmt(self, o: WithStmt) -> None: + for expr, idx in zip(o.expr, o.target): + expr.accept(self) + self.process_lvalue(idx) + o.body.accept(self) + + def visit_class_def(self, o: ClassDef) -> None: + self.process_definition(o.name) + self.tracker.enter_scope(ScopeType.Class) + super().visit_class_def(o) + self.tracker.exit_scope() + + def visit_import(self, o: Import) -> None: + for mod, alias in o.ids: + if alias is not None: + self.tracker.record_definition(alias) + else: + # When you do `import x.y`, only `x` becomes defined. + names = mod.split(".") + if names: + # `names` should always be nonempty, but we don't want mypy + # to crash on invalid code. + self.tracker.record_definition(names[0]) + super().visit_import(o) + + def visit_import_from(self, o: ImportFrom) -> None: + for mod, alias in o.names: + name = alias + if name is None: + name = mod + self.tracker.record_definition(name) + super().visit_import_from(o) + + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + # Type alias target may contain forward references + self.tracker.record_definition(o.name.name) diff --git a/mypy/patterns.py b/mypy/patterns.py new file mode 100644 index 000000000000..a01bf6acc876 --- /dev/null +++ b/mypy/patterns.py @@ -0,0 +1,150 @@ +"""Classes for representing match statement patterns.""" + +from __future__ import annotations + +from typing import TypeVar + +from mypy_extensions import trait + +from mypy.nodes import Expression, NameExpr, Node, RefExpr +from mypy.visitor import PatternVisitor + +T = TypeVar("T") + + +@trait +class Pattern(Node): + """A pattern node.""" + + __slots__ = () + + def accept(self, visitor: PatternVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) + + +class AsPattern(Pattern): + """The pattern as """ + + # The python ast, and therefore also our ast merges capture, wildcard and as patterns into one + # for easier handling. + # If pattern is None this is a capture pattern. If name and pattern are both none this is a + # wildcard pattern. + # Only name being None should not happen but also won't break anything. + pattern: Pattern | None + name: NameExpr | None + + def __init__(self, pattern: Pattern | None, name: NameExpr | None) -> None: + super().__init__() + self.pattern = pattern + self.name = name + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_as_pattern(self) + + +class OrPattern(Pattern): + """The pattern | | ...""" + + patterns: list[Pattern] + + def __init__(self, patterns: list[Pattern]) -> None: + super().__init__() + self.patterns = patterns + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_or_pattern(self) + + +class ValuePattern(Pattern): + """The pattern x.y (or x.y.z, ...)""" + + expr: Expression + + def __init__(self, expr: Expression) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_value_pattern(self) + + +class SingletonPattern(Pattern): + # This can be exactly True, False or None + value: bool | None + + def __init__(self, value: bool | None) -> None: + super().__init__() + self.value = value + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_singleton_pattern(self) + + +class SequencePattern(Pattern): + """The pattern [, ...]""" + + patterns: list[Pattern] + + def __init__(self, patterns: list[Pattern]) -> None: + super().__init__() + self.patterns = patterns + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_sequence_pattern(self) + + +class StarredPattern(Pattern): + # None corresponds to *_ in a list pattern. It will match multiple items but won't bind them to + # a name. + capture: NameExpr | None + + def __init__(self, capture: NameExpr | None) -> None: + super().__init__() + self.capture = capture + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_starred_pattern(self) + + +class MappingPattern(Pattern): + keys: list[Expression] + values: list[Pattern] + rest: NameExpr | None + + def __init__( + self, keys: list[Expression], values: list[Pattern], rest: NameExpr | None + ) -> None: + super().__init__() + assert len(keys) == len(values) + self.keys = keys + self.values = values + self.rest = rest + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_mapping_pattern(self) + + +class ClassPattern(Pattern): + """The pattern Cls(...)""" + + class_ref: RefExpr + positionals: list[Pattern] + keyword_keys: list[str] + keyword_values: list[Pattern] + + def __init__( + self, + class_ref: RefExpr, + positionals: list[Pattern], + keyword_keys: list[str], + keyword_values: list[Pattern], + ) -> None: + super().__init__() + assert len(keyword_keys) == len(keyword_values) + self.class_ref = class_ref + self.positionals = positionals + self.keyword_keys = keyword_keys + self.keyword_values = keyword_values + + def accept(self, visitor: PatternVisitor[T]) -> T: + return visitor.visit_class_pattern(self) diff --git a/mypy/plugin.py b/mypy/plugin.py new file mode 100644 index 000000000000..fcbbc32f6237 --- /dev/null +++ b/mypy/plugin.py @@ -0,0 +1,898 @@ +"""Plugin system for extending mypy. + +At large scale the plugin system works as following: + +* Plugins are collected from the corresponding mypy config file option + (either via paths to Python files, or installed Python modules) + and imported using importlib. + +* Every module should get an entry point function (called 'plugin' by default, + but may be overridden in the config file) that should accept a single string + argument that is a full mypy version (includes git commit hash for dev + versions) and return a subclass of mypy.plugins.Plugin. + +* All plugin class constructors should match the signature of mypy.plugin.Plugin + (i.e. should accept an mypy.options.Options object), and *must* call + super().__init__(). + +* At several steps during semantic analysis and type checking mypy calls + special `get_xxx` methods on user plugins with a single string argument that + is a fully qualified name (full name) of a relevant definition + (see mypy.plugin.Plugin method docstrings for details). + +* The plugins are called in the order they are passed in the config option. + Every plugin must decide whether to act on a given full name. The first + plugin that returns non-None object will be used. + +* The above decision should be made using the limited common API specified by + mypy.plugin.CommonPluginApi. + +* The callback returned by the plugin will be called with a larger context that + includes relevant current state (e.g. a default return type, or a default + attribute type) and a wider relevant API provider (e.g. + SemanticAnalyzerPluginInterface or CheckerPluginInterface). + +* The result of this is used for further processing. See various `XxxContext` + named tuples for details about which information is given to each hook. + +Plugin developers should ensure that their plugins work well in incremental and +daemon modes. In particular, plugins should not hold global state, and should +always call add_plugin_dependency() in plugin hooks called during semantic +analysis. See the method docstring for more details. + +There is no dedicated cache storage for plugins, but plugins can store +per-TypeInfo data in a special .metadata attribute that is serialized to the +mypy caches between incremental runs. To avoid collisions between plugins, they +are encouraged to store their state under a dedicated key coinciding with +plugin name in the metadata dictionary. Every value stored there must be +JSON-serializable. + +## Notes about the semantic analyzer + +Mypy 0.710 introduced a new semantic analyzer that changed how plugins are +expected to work in several notable ways (from mypy 0.730 the old semantic +analyzer is no longer available): + +1. The order of processing AST nodes in modules is different. The old semantic + analyzer processed modules in textual order, one module at a time. The new + semantic analyzer first processes the module top levels, including bodies of + any top-level classes and classes nested within classes. ("Top-level" here + means "not nested within a function/method".) Functions and methods are + processed only after module top levels have been finished. If there is an + import cycle, all module top levels in the cycle are processed before + processing any functions or methods. Each unit of processing (a module top + level or a function/method) is called a *target*. + + This also means that function signatures in the same module have not been + analyzed yet when analyzing the module top level. If you need access to + a function signature, you'll need to explicitly analyze the signature first + using `anal_type()`. + +2. Each target can be processed multiple times. This may happen if some forward + references are not ready yet, for example. This means that semantic analyzer + related plugin hooks can be called multiple times for the same full name. + These plugin methods must thus be idempotent. + +3. The `anal_type` API function returns None if some part of the type is not + available yet. If this happens, the current target being analyzed will be + *deferred*, which means that it will be processed again soon, in the hope + that additional dependencies will be available. This may happen if there are + forward references to types or inter-module references to types within an + import cycle. + + Note that if there is a circular definition, mypy may decide to stop + processing to avoid an infinite number of iterations. When this happens, + `anal_type` will generate an error and return an `AnyType` type object + during the final iteration (instead of None). + +4. There is a new API method `defer()`. This can be used to explicitly request + the current target to be reprocessed one more time. You don't need this + to call this if `anal_type` returns None, however. + +5. There is a new API property `final_iteration`, which is true once mypy + detected no progress during the previous iteration or if the maximum + semantic analysis iteration count has been reached. You must never + defer during the final iteration, as it will cause a crash. + +6. The `node` attribute of SymbolTableNode objects may contain a reference to + a PlaceholderNode object. This object means that this definition has not + been fully processed yet. If you encounter a PlaceholderNode, you should + defer unless it's the final iteration. If it's the final iteration, you + should generate an error message. It usually means that there's a cyclic + definition that cannot be resolved by mypy. PlaceholderNodes can only refer + to references inside an import cycle. If you are looking up things from + another module, such as the builtins, that is outside the current module or + import cycle, you can safely assume that you won't receive a placeholder. + +When testing your plugin, you should have a test case that forces a module top +level to be processed multiple times. The easiest way to do this is to include +a forward reference to a class in a top-level annotation. Example: + + c: C # Forward reference causes second analysis pass + class C: pass + +Note that a forward reference in a function signature won't trigger another +pass, since all functions are processed only after the top level has been fully +analyzed. +""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import Any, Callable, NamedTuple, TypeVar + +from mypy_extensions import mypyc_attr, trait + +from mypy.errorcodes import ErrorCode +from mypy.lookup import lookup_fully_qualified +from mypy.message_registry import ErrorMessage +from mypy.messages import MessageBuilder +from mypy.nodes import ( + ArgKind, + CallExpr, + ClassDef, + Context, + Expression, + MypyFile, + SymbolTableNode, + TypeInfo, +) +from mypy.options import Options +from mypy.tvar_scope import TypeVarLikeScope +from mypy.types import ( + CallableType, + FunctionLike, + Instance, + ProperType, + Type, + TypeList, + UnboundType, +) + + +@trait +class TypeAnalyzerPluginInterface: + """Interface for accessing semantic analyzer functionality in plugins. + + Methods docstrings contain only basic info. Look for corresponding implementation + docstrings in typeanal.py for more details. + """ + + # An options object. Note: these are the cloned options for the current file. + # This might be different from Plugin.options (that contains default/global options) + # if there are per-file options in the config. This applies to all other interfaces + # in this file. + options: Options + + @abstractmethod + def fail(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + """Emit an error message at given location.""" + raise NotImplementedError + + @abstractmethod + def named_type(self, name: str, args: list[Type]) -> Instance: + """Construct an instance of a builtin type with given name.""" + raise NotImplementedError + + @abstractmethod + def analyze_type(self, typ: Type) -> Type: + """Analyze an unbound type using the default mypy logic.""" + raise NotImplementedError + + @abstractmethod + def analyze_callable_args( + self, arglist: TypeList + ) -> tuple[list[Type], list[ArgKind], list[str | None]] | None: + """Find types, kinds, and names of arguments from extended callable syntax.""" + raise NotImplementedError + + +# A context for a hook that semantically analyzes an unbound type. +class AnalyzeTypeContext(NamedTuple): + type: UnboundType # Type to analyze + context: Context # Relevant location context (e.g. for error messages) + api: TypeAnalyzerPluginInterface + + +@mypyc_attr(allow_interpreted_subclasses=True) +class CommonPluginApi: + """ + A common plugin API (shared between semantic analysis and type checking phases) + that all plugin hooks get independently of the context. + """ + + # Global mypy options. + # Per-file options can be only accessed on various + # XxxPluginInterface classes. + options: Options + + @abstractmethod + def lookup_fully_qualified(self, fullname: str) -> SymbolTableNode | None: + """Lookup a symbol by its full name (including module). + + This lookup function available for all plugins. Return None if a name + is not found. This function doesn't support lookup from current scope. + Use SemanticAnalyzerPluginInterface.lookup_qualified() for this.""" + raise NotImplementedError + + +@trait +class CheckerPluginInterface: + """Interface for accessing type checker functionality in plugins. + + Methods docstrings contain only basic info. Look for corresponding implementation + docstrings in checker.py for more details. + """ + + msg: MessageBuilder + options: Options + path: str + + # Type context for type inference + @property + @abstractmethod + def type_context(self) -> list[Type | None]: + """Return the type context of the plugin""" + raise NotImplementedError + + @abstractmethod + def fail( + self, msg: str | ErrorMessage, ctx: Context, /, *, code: ErrorCode | None = None + ) -> None: + """Emit an error message at given location.""" + raise NotImplementedError + + @abstractmethod + def named_generic_type(self, name: str, args: list[Type]) -> Instance: + """Construct an instance of a generic type with given type arguments.""" + raise NotImplementedError + + @abstractmethod + def get_expression_type(self, node: Expression, type_context: Type | None = None) -> Type: + """Checks the type of the given expression.""" + raise NotImplementedError + + +@trait +class SemanticAnalyzerPluginInterface: + """Interface for accessing semantic analyzer functionality in plugins. + + Methods docstrings contain only basic info. Look for corresponding implementation + docstrings in semanal.py for more details. + + # TODO: clean-up lookup functions. + """ + + modules: dict[str, MypyFile] + # Options for current file. + options: Options + cur_mod_id: str + msg: MessageBuilder + + @abstractmethod + def named_type(self, fullname: str, args: list[Type] | None = None) -> Instance: + """Construct an instance of a builtin type with given type arguments.""" + raise NotImplementedError + + @abstractmethod + def builtin_type(self, fully_qualified_name: str) -> Instance: + """Legacy function -- use named_type() instead.""" + # NOTE: Do not delete this since many plugins may still use it. + raise NotImplementedError + + @abstractmethod + def named_type_or_none(self, fullname: str, args: list[Type] | None = None) -> Instance | None: + """Construct an instance of a type with given type arguments. + + Return None if a type could not be constructed for the qualified + type name. This is possible when the qualified name includes a + module name and the module has not been imported. + """ + raise NotImplementedError + + @abstractmethod + def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance, line: int) -> TypeInfo: + raise NotImplementedError + + @abstractmethod + def parse_bool(self, expr: Expression) -> bool | None: + """Parse True/False literals.""" + raise NotImplementedError + + @abstractmethod + def parse_str_literal(self, expr: Expression) -> str | None: + """Parse string literals.""" + + @abstractmethod + def fail( + self, + msg: str, + ctx: Context, + serious: bool = False, + *, + blocker: bool = False, + code: ErrorCode | None = None, + ) -> None: + """Emit an error message at given location.""" + raise NotImplementedError + + @abstractmethod + def anal_type( + self, + t: Type, + *, + tvar_scope: TypeVarLikeScope | None = None, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, + ) -> Type | None: + """Analyze an unbound type. + + Return None if some part of the type is not ready yet. In this + case the current target being analyzed will be deferred and + analyzed again. + """ + raise NotImplementedError + + @abstractmethod + def class_type(self, self_type: Type) -> Type: + """Generate type of first argument of class methods from type of self.""" + raise NotImplementedError + + @abstractmethod + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a symbol by its fully qualified name. + + Raise an error if not found. + """ + raise NotImplementedError + + @abstractmethod + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode | None: + """Lookup a symbol by its fully qualified name. + + Return None if not found. + """ + raise NotImplementedError + + @abstractmethod + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + """Lookup symbol using a name in current scope. + + This follows Python local->non-local->global->builtins rules. + """ + raise NotImplementedError + + @abstractmethod + def add_plugin_dependency(self, trigger: str, target: str | None = None) -> None: + """Specify semantic dependencies for generated methods/variables. + + If the symbol with full name given by trigger is found to be stale by mypy, + then the body of node with full name given by target will be re-checked. + By default, this is the node that is currently analyzed. + + For example, the dataclass plugin adds a generated __init__ method with + a signature that depends on types of attributes in ancestor classes. If any + attribute in an ancestor class gets stale (modified), we need to reprocess + the subclasses (and thus regenerate __init__ methods). + + This is used by fine-grained incremental mode (mypy daemon). See mypy/server/deps.py + for more details. + """ + raise NotImplementedError + + @abstractmethod + def add_symbol_table_node(self, name: str, stnode: SymbolTableNode) -> Any: + """Add node to global symbol table (or to nearest class if there is one).""" + raise NotImplementedError + + @abstractmethod + def qualified_name(self, n: str) -> str: + """Make qualified name using current module and enclosing class (if any).""" + raise NotImplementedError + + @abstractmethod + def defer(self) -> None: + """Call this to defer the processing of the current node. + + This will request an additional iteration of semantic analysis. + """ + raise NotImplementedError + + @property + @abstractmethod + def final_iteration(self) -> bool: + """Is this the final iteration of semantic analysis?""" + raise NotImplementedError + + @property + @abstractmethod + def is_stub_file(self) -> bool: + raise NotImplementedError + + @abstractmethod + def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Type | None: + raise NotImplementedError + + +# A context for querying for configuration data about a module for +# cache invalidation purposes. +class ReportConfigContext(NamedTuple): + id: str # Module name + path: str # Module file path + is_check: bool # Is this invocation for checking whether the config matches + + +# A context for a function signature hook that infers a better signature for a +# function. Note that argument types aren't available yet. If you need them, +# you have to use a method hook instead. +class FunctionSigContext(NamedTuple): + args: list[list[Expression]] # Actual expressions for each formal argument + default_signature: CallableType # Original signature of the method + context: Context # Relevant location context (e.g. for error messages) + api: CheckerPluginInterface + + +# A context for a function hook that infers the return type of a function with +# a special signature. +# +# A no-op callback would just return the inferred return type, but a useful +# callback at least sometimes can infer a more precise type. +class FunctionContext(NamedTuple): + arg_types: list[list[Type]] # List of actual caller types for each formal argument + arg_kinds: list[list[ArgKind]] # Ditto for argument kinds, see nodes.ARG_* constants + # Names of formal parameters from the callee definition, + # these will be sufficient in most cases. + callee_arg_names: list[str | None] + # Names of actual arguments in the call expression. For example, + # in a situation like this: + # def func(**kwargs) -> None: + # pass + # func(kw1=1, kw2=2) + # callee_arg_names will be ['kwargs'] and arg_names will be [['kw1', 'kw2']]. + arg_names: list[list[str | None]] + default_return_type: Type # Return type inferred from signature + args: list[list[Expression]] # Actual expressions for each formal argument + context: Context # Relevant location context (e.g. for error messages) + api: CheckerPluginInterface + + +# A context for a method signature hook that infers a better signature for a +# method. Note that argument types aren't available yet. If you need them, +# you have to use a method hook instead. +# TODO: document ProperType in the plugin changelog/update issue. +class MethodSigContext(NamedTuple): + type: ProperType # Base object type for method call + args: list[list[Expression]] # Actual expressions for each formal argument + default_signature: CallableType # Original signature of the method + context: Context # Relevant location context (e.g. for error messages) + api: CheckerPluginInterface + + +# A context for a method hook that infers the return type of a method with a +# special signature. +# +# This is very similar to FunctionContext (only differences are documented). +class MethodContext(NamedTuple): + type: ProperType # Base object type for method call + arg_types: list[list[Type]] # List of actual caller types for each formal argument + # see FunctionContext for details about names and kinds + arg_kinds: list[list[ArgKind]] + callee_arg_names: list[str | None] + arg_names: list[list[str | None]] + default_return_type: Type # Return type inferred by mypy + args: list[list[Expression]] # Lists of actual expressions for every formal argument + context: Context + api: CheckerPluginInterface + + +# A context for an attribute type hook that infers the type of an attribute. +class AttributeContext(NamedTuple): + type: ProperType # Type of object with attribute + default_attr_type: Type # Original attribute type + is_lvalue: bool # Whether the attribute is the target of an assignment + context: Context # Relevant location context (e.g. for error messages) + api: CheckerPluginInterface + + +# A context for a class hook that modifies the class definition. +class ClassDefContext(NamedTuple): + cls: ClassDef # The class definition + reason: Expression # The expression being applied (decorator, metaclass, base class) + api: SemanticAnalyzerPluginInterface + + +# A context for dynamic class definitions like +# Base = declarative_base() +class DynamicClassDefContext(NamedTuple): + call: CallExpr # The r.h.s. of dynamic class definition + name: str # The name this class is being assigned to + api: SemanticAnalyzerPluginInterface + + +@mypyc_attr(allow_interpreted_subclasses=True) +class Plugin(CommonPluginApi): + """Base class of all type checker plugins. + + This defines a no-op plugin. Subclasses can override some methods to + provide some actual functionality. + + All get_ methods are treated as pure functions (you should assume that + results might be cached). A plugin should return None from a get_ method + to give way to other plugins. + + Look at the comments of various *Context objects for additional information on + various hooks. + """ + + def __init__(self, options: Options) -> None: + self.options = options + self.python_version = options.python_version + # This can't be set in __init__ because it is executed too soon in build.py. + # Therefore, build.py *must* set it later before graph processing starts + # by calling set_modules(). + self._modules: dict[str, MypyFile] | None = None + + def set_modules(self, modules: dict[str, MypyFile]) -> None: + self._modules = modules + + def lookup_fully_qualified(self, fullname: str) -> SymbolTableNode | None: + assert self._modules is not None + return lookup_fully_qualified(fullname, self._modules) + + def report_config_data(self, ctx: ReportConfigContext) -> Any: + """Get representation of configuration data for a module. + + The data must be encodable as JSON and will be stored in the + cache metadata for the module. A mismatch between the cached + values and the returned will result in that module's cache + being invalidated and the module being rechecked. + + This can be called twice for each module, once after loading + the cache to check if it is valid and once while writing new + cache information. + + If is_check in the context is true, then the return of this + call will be checked against the cached version. Otherwise the + call is being made to determine what to put in the cache. This + can be used to allow consulting extra cache files in certain + complex situations. + + This can be used to incorporate external configuration information + that might require changes to typechecking. + """ + return None + + def get_additional_deps(self, file: MypyFile) -> list[tuple[int, str, int]]: + """Customize dependencies for a module. + + This hook allows adding in new dependencies for a module. It + is called after parsing a file but before analysis. This can + be useful if a library has dependencies that are dynamic based + on configuration information, for example. + + Returns a list of (priority, module name, line number) tuples. + + The line number can be -1 when there is not a known real line number. + + Priorities are defined in mypy.build (but maybe shouldn't be). + 10 is a good choice for priority. + """ + return [] + + def get_type_analyze_hook(self, fullname: str) -> Callable[[AnalyzeTypeContext], Type] | None: + """Customize behaviour of the type analyzer for given full names. + + This method is called during the semantic analysis pass whenever mypy sees an + unbound type. For example, while analysing this code: + + from lib import Special, Other + + var: Special + def func(x: Other[int]) -> None: + ... + + this method will be called with 'lib.Special', and then with 'lib.Other'. + The callback returned by plugin must return an analyzed type, + i.e. an instance of `mypy.types.Type`. + """ + return None + + def get_function_signature_hook( + self, fullname: str + ) -> Callable[[FunctionSigContext], FunctionLike] | None: + """Adjust the signature of a function. + + This method is called before type checking a function call. Plugin + may infer a better type for the function. + + from lib import Class, do_stuff + + do_stuff(42) + Class() + + This method will be called with 'lib.do_stuff' and then with 'lib.Class'. + """ + return None + + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + """Adjust the return type of a function call. + + This method is called after type checking a call. Plugin may adjust the return + type inferred by mypy, and/or emit some error messages. Note, this hook is also + called for class instantiation calls, so that in this example: + + from lib import Class, do_stuff + + do_stuff(42) + Class() + + This method will be called with 'lib.do_stuff' and then with 'lib.Class'. + """ + return None + + def get_method_signature_hook( + self, fullname: str + ) -> Callable[[MethodSigContext], FunctionLike] | None: + """Adjust the signature of a method. + + This method is called before type checking a method call. Plugin + may infer a better type for the method. The hook is also called for special + Python dunder methods except __init__ and __new__ (use get_function_hook to customize + class instantiation). This function is called with the method full name using + the class where it was _defined_. For example, in this code: + + from lib import Special + + class Base: + def method(self, arg: Any) -> Any: + ... + class Derived(Base): + ... + + var: Derived + var.method(42) + + x: Special + y = x[0] + + this method is called with '__main__.Base.method', and then with + 'lib.Special.__getitem__'. + """ + return None + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + """Adjust return type of a method call. + + This is the same as get_function_hook(), but is called with the + method full name (again, using the class where the method is defined). + """ + return None + + def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: + """Adjust type of an instance attribute. + + This method is called with attribute full name using the class of the instance where + the attribute was defined (or Var.info.fullname for generated attributes). + + For classes without __getattr__ or __getattribute__, this hook is only called for + names of fields/properties (but not methods) that exist in the instance MRO. + + For classes that implement __getattr__ or __getattribute__, this hook is called + for all fields/properties, including nonexistent ones (but still not methods). + + For example: + + class Base: + x: Any + def __getattr__(self, attr: str) -> Any: ... + + class Derived(Base): + ... + + var: Derived + var.x + var.y + + get_attribute_hook is called with '__main__.Base.x' and '__main__.Base.y'. + However, if we had not implemented __getattr__ on Base, you would only get + the callback for 'var.x'; 'var.y' would produce an error without calling the hook. + """ + return None + + def get_class_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: + """ + Adjust type of a class attribute. + + This method is called with attribute full name using the class where the attribute was + defined (or Var.info.fullname for generated attributes). + + For example: + + class Cls: + x: Any + + Cls.x + + get_class_attribute_hook is called with '__main__.Cls.x' as fullname. + """ + return None + + def get_class_decorator_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + """Update class definition for given class decorators. + + The plugin can modify a TypeInfo _in place_ (for example add some generated + methods to the symbol table). This hook is called after the class body was + semantically analyzed, but *there may still be placeholders* (typically + caused by forward references). + + NOTE: Usually get_class_decorator_hook_2 is the better option, since it + guarantees that there are no placeholders. + + The hook is called with full names of all class decorators. + + The hook can be called multiple times per class, so it must be + idempotent. + """ + return None + + def get_class_decorator_hook_2( + self, fullname: str + ) -> Callable[[ClassDefContext], bool] | None: + """Update class definition for given class decorators. + + Similar to get_class_decorator_hook, but this runs in a later pass when + placeholders have been resolved. + + The hook can return False if some base class hasn't been + processed yet using class hooks. It causes all class hooks + (that are run in this same pass) to be invoked another time for + the file(s) currently being processed. + + The hook can be called multiple times per class, so it must be + idempotent. + """ + return None + + def get_metaclass_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + """Update class definition for given declared metaclasses. + + Same as get_class_decorator_hook() but for metaclasses. Note: + this hook will be only called for explicit metaclasses, not for + inherited ones. + + TODO: probably it should also be called on inherited metaclasses. + """ + return None + + def get_base_class_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + """Update class definition for given base classes. + + Same as get_class_decorator_hook() but for base classes. Base classes + don't need to refer to TypeInfos, if a base class refers to a variable with + Any type, this hook will still be called. + """ + return None + + def get_customize_class_mro_hook( + self, fullname: str + ) -> Callable[[ClassDefContext], None] | None: + """Customize MRO for given classes. + + The plugin can modify the class MRO _in place_. This method is called + with the class full name before its body was semantically analyzed. + """ + return None + + def get_dynamic_class_hook( + self, fullname: str + ) -> Callable[[DynamicClassDefContext], None] | None: + """Semantically analyze a dynamic class definition. + + This plugin hook allows one to semantically analyze dynamic class definitions like: + + from lib import dynamic_class + + X = dynamic_class('X', []) + + For such definition, this hook will be called with 'lib.dynamic_class'. + The plugin should create the corresponding TypeInfo, and place it into a relevant + symbol table, e.g. using ctx.api.add_symbol_table_node(). + """ + return None + + +T = TypeVar("T") + + +class ChainedPlugin(Plugin): + """A plugin that represents a sequence of chained plugins. + + Each lookup method returns the hook for the first plugin that + reports a match. + + This class should not be subclassed -- use Plugin as the base class + for all plugins. + """ + + # TODO: Support caching of lookup results (through a LRU cache, for example). + + def __init__(self, options: Options, plugins: list[Plugin]) -> None: + """Initialize chained plugin. + + Assume that the child plugins aren't mutated (results may be cached). + """ + super().__init__(options) + self._plugins = plugins + + def set_modules(self, modules: dict[str, MypyFile]) -> None: + for plugin in self._plugins: + plugin.set_modules(modules) + + def report_config_data(self, ctx: ReportConfigContext) -> Any: + config_data = [plugin.report_config_data(ctx) for plugin in self._plugins] + return config_data if any(x is not None for x in config_data) else None + + def get_additional_deps(self, file: MypyFile) -> list[tuple[int, str, int]]: + deps = [] + for plugin in self._plugins: + deps.extend(plugin.get_additional_deps(file)) + return deps + + def get_type_analyze_hook(self, fullname: str) -> Callable[[AnalyzeTypeContext], Type] | None: + return self._find_hook(lambda plugin: plugin.get_type_analyze_hook(fullname)) + + def get_function_signature_hook( + self, fullname: str + ) -> Callable[[FunctionSigContext], FunctionLike] | None: + return self._find_hook(lambda plugin: plugin.get_function_signature_hook(fullname)) + + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + return self._find_hook(lambda plugin: plugin.get_function_hook(fullname)) + + def get_method_signature_hook( + self, fullname: str + ) -> Callable[[MethodSigContext], FunctionLike] | None: + return self._find_hook(lambda plugin: plugin.get_method_signature_hook(fullname)) + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + return self._find_hook(lambda plugin: plugin.get_method_hook(fullname)) + + def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: + return self._find_hook(lambda plugin: plugin.get_attribute_hook(fullname)) + + def get_class_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: + return self._find_hook(lambda plugin: plugin.get_class_attribute_hook(fullname)) + + def get_class_decorator_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + return self._find_hook(lambda plugin: plugin.get_class_decorator_hook(fullname)) + + def get_class_decorator_hook_2( + self, fullname: str + ) -> Callable[[ClassDefContext], bool] | None: + return self._find_hook(lambda plugin: plugin.get_class_decorator_hook_2(fullname)) + + def get_metaclass_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + return self._find_hook(lambda plugin: plugin.get_metaclass_hook(fullname)) + + def get_base_class_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + return self._find_hook(lambda plugin: plugin.get_base_class_hook(fullname)) + + def get_customize_class_mro_hook( + self, fullname: str + ) -> Callable[[ClassDefContext], None] | None: + return self._find_hook(lambda plugin: plugin.get_customize_class_mro_hook(fullname)) + + def get_dynamic_class_hook( + self, fullname: str + ) -> Callable[[DynamicClassDefContext], None] | None: + return self._find_hook(lambda plugin: plugin.get_dynamic_class_hook(fullname)) + + def _find_hook(self, lookup: Callable[[Plugin], T]) -> T | None: + for plugin in self._plugins: + hook = lookup(plugin) + if hook: + return hook + return None diff --git a/lib-python/3.2/test/__init__.py b/mypy/plugins/__init__.py similarity index 100% rename from lib-python/3.2/test/__init__.py rename to mypy/plugins/__init__.py diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py new file mode 100644 index 000000000000..b67a285af11d --- /dev/null +++ b/mypy/plugins/attrs.py @@ -0,0 +1,1182 @@ +"""Plugin for supporting the attrs library (http://www.attrs.org)""" + +from __future__ import annotations + +from collections import defaultdict +from functools import reduce +from typing import Final, Iterable, List, Mapping, cast +from typing_extensions import Literal + +import mypy.plugin # To avoid circular imports. +from mypy.applytype import apply_generic_arguments +from mypy.errorcodes import LITERAL_REQ +from mypy.expandtype import expand_type, expand_type_by_instance +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.meet import meet_types +from mypy.messages import format_type_bare +from mypy.nodes import ( + ARG_NAMED, + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + MDEF, + Argument, + AssignmentStmt, + CallExpr, + Context, + Decorator, + Expression, + FuncDef, + IndexExpr, + JsonDict, + LambdaExpr, + ListExpr, + MemberExpr, + NameExpr, + OverloadedFuncDef, + PlaceholderNode, + RefExpr, + SymbolTableNode, + TempNode, + TupleExpr, + TypeApplication, + TypeInfo, + TypeVarExpr, + Var, + is_class_var, +) +from mypy.plugin import SemanticAnalyzerPluginInterface +from mypy.plugins.common import ( + _get_argument, + _get_bool_argument, + _get_decorator_bool_argument, + add_attribute_to_class, + add_method_to_class, + deserialize_and_fixup_type, +) +from mypy.server.trigger import make_wildcard_trigger +from mypy.state import state +from mypy.typeops import get_type_vars, make_simplified_union, map_type_from_supertype +from mypy.types import ( + AnyType, + CallableType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarType, + UninhabitedType, + UnionType, + get_proper_type, +) +from mypy.typevars import fill_typevars +from mypy.util import unmangle + +# The names of the different functions that create classes or arguments. +attr_class_makers: Final = {"attr.s", "attr.attrs", "attr.attributes"} +attr_dataclass_makers: Final = {"attr.dataclass"} +attr_frozen_makers: Final = {"attr.frozen", "attrs.frozen"} +attr_define_makers: Final = {"attr.define", "attr.mutable", "attrs.define", "attrs.mutable"} +attr_attrib_makers: Final = {"attr.ib", "attr.attrib", "attr.attr", "attr.field", "attrs.field"} +attr_optional_converters: Final = {"attr.converters.optional", "attrs.converters.optional"} + +SELF_TVAR_NAME: Final = "_AT" +MAGIC_ATTR_NAME: Final = "__attrs_attrs__" +MAGIC_ATTR_CLS_NAME_TEMPLATE: Final = "__{}_AttrsAttributes__" # The tuple subclass pattern. +ATTRS_INIT_NAME: Final = "__attrs_init__" + + +class Converter: + """Holds information about a `converter=` argument""" + + def __init__(self, init_type: Type | None = None, ret_type: Type | None = None) -> None: + self.init_type = init_type + self.ret_type = ret_type + + +class Attribute: + """The value of an attr.ib() call.""" + + def __init__( + self, + name: str, + alias: str | None, + info: TypeInfo, + has_default: bool, + init: bool, + kw_only: bool, + converter: Converter | None, + context: Context, + init_type: Type | None, + ) -> None: + self.name = name + self.alias = alias + self.info = info + self.has_default = has_default + self.init = init + self.kw_only = kw_only + self.converter = converter + self.context = context + self.init_type = init_type + + def argument(self, ctx: mypy.plugin.ClassDefContext) -> Argument: + """Return this attribute as an argument to __init__.""" + assert self.init + init_type: Type | None = None + if self.converter: + if self.converter.init_type: + init_type = self.converter.init_type + if init_type and self.init_type and self.converter.ret_type: + # The converter return type should be the same type as the attribute type. + # Copy type vars from attr type to converter. + converter_vars = get_type_vars(self.converter.ret_type) + init_vars = get_type_vars(self.init_type) + if converter_vars and len(converter_vars) == len(init_vars): + variables = { + binder.id: arg for binder, arg in zip(converter_vars, init_vars) + } + init_type = expand_type(init_type, variables) + else: + ctx.api.fail("Cannot determine __init__ type from converter", self.context) + init_type = AnyType(TypeOfAny.from_error) + else: # There is no converter, the init type is the normal type. + init_type = self.init_type or self.info[self.name].type + + unannotated = False + if init_type is None: + unannotated = True + # Convert type not set to Any. + init_type = AnyType(TypeOfAny.unannotated) + else: + proper_type = get_proper_type(init_type) + if isinstance(proper_type, AnyType): + if proper_type.type_of_any == TypeOfAny.unannotated: + unannotated = True + + if unannotated and ctx.api.options.disallow_untyped_defs: + # This is a compromise. If you don't have a type here then the + # __init__ will be untyped. But since the __init__ is added it's + # pointing at the decorator. So instead we also show the error in the + # assignment, which is where you would fix the issue. + node = self.info[self.name].node + assert node is not None + ctx.api.msg.need_annotation_for_var(node, self.context) + + if self.kw_only: + arg_kind = ARG_NAMED_OPT if self.has_default else ARG_NAMED + else: + arg_kind = ARG_OPT if self.has_default else ARG_POS + + # Attrs removes leading underscores when creating the __init__ arguments. + name = self.alias or self.name.lstrip("_") + return Argument(Var(name, init_type), init_type, None, arg_kind) + + def serialize(self) -> JsonDict: + """Serialize this object so it can be saved and restored.""" + return { + "name": self.name, + "alias": self.alias, + "has_default": self.has_default, + "init": self.init, + "kw_only": self.kw_only, + "has_converter": self.converter is not None, + "converter_init_type": ( + self.converter.init_type.serialize() + if self.converter and self.converter.init_type + else None + ), + "context_line": self.context.line, + "context_column": self.context.column, + "init_type": self.init_type.serialize() if self.init_type else None, + } + + @classmethod + def deserialize( + cls, info: TypeInfo, data: JsonDict, api: SemanticAnalyzerPluginInterface + ) -> Attribute: + """Return the Attribute that was serialized.""" + raw_init_type = data["init_type"] + init_type = deserialize_and_fixup_type(raw_init_type, api) if raw_init_type else None + raw_converter_init_type = data["converter_init_type"] + converter_init_type = ( + deserialize_and_fixup_type(raw_converter_init_type, api) + if raw_converter_init_type + else None + ) + + return Attribute( + data["name"], + data["alias"], + info, + data["has_default"], + data["init"], + data["kw_only"], + Converter(converter_init_type) if data["has_converter"] else None, + Context(line=data["context_line"], column=data["context_column"]), + init_type, + ) + + def expand_typevar_from_subtype(self, sub_type: TypeInfo) -> None: + """Expands type vars in the context of a subtype when an attribute is inherited + from a generic super type.""" + if self.init_type: + self.init_type = map_type_from_supertype(self.init_type, sub_type, self.info) + else: + self.init_type = None + + +def _determine_eq_order(ctx: mypy.plugin.ClassDefContext) -> bool: + """ + Validate the combination of *cmp*, *eq*, and *order*. Derive the effective + value of order. + """ + cmp = _get_decorator_optional_bool_argument(ctx, "cmp") + eq = _get_decorator_optional_bool_argument(ctx, "eq") + order = _get_decorator_optional_bool_argument(ctx, "order") + + if cmp is not None and any((eq is not None, order is not None)): + ctx.api.fail('Don\'t mix "cmp" with "eq" and "order"', ctx.reason) + + # cmp takes precedence due to bw-compatibility. + if cmp is not None: + return cmp + + # If left None, equality is on and ordering mirrors equality. + if eq is None: + eq = True + + if order is None: + order = eq + + if eq is False and order is True: + ctx.api.fail("eq must be True if order is True", ctx.reason) + + return order + + +def _get_decorator_optional_bool_argument( + ctx: mypy.plugin.ClassDefContext, name: str, default: bool | None = None +) -> bool | None: + """Return the Optional[bool] argument for the decorator. + + This handles both @decorator(...) and @decorator. + """ + if isinstance(ctx.reason, CallExpr): + attr_value = _get_argument(ctx.reason, name) + if attr_value: + if isinstance(attr_value, NameExpr): + if attr_value.fullname == "builtins.True": + return True + if attr_value.fullname == "builtins.False": + return False + if attr_value.fullname == "builtins.None": + return None + ctx.api.fail( + f'"{name}" argument must be a True, False, or None literal', + ctx.reason, + code=LITERAL_REQ, + ) + return default + return default + else: + return default + + +def attr_tag_callback(ctx: mypy.plugin.ClassDefContext) -> None: + """Record that we have an attrs class in the main semantic analysis pass. + + The later pass implemented by attr_class_maker_callback will use this + to detect attrs classes in base classes. + """ + # The value is ignored, only the existence matters. + ctx.cls.info.metadata["attrs_tag"] = {} + + +def attr_class_maker_callback( + ctx: mypy.plugin.ClassDefContext, + auto_attribs_default: bool | None = False, + frozen_default: bool = False, + slots_default: bool = False, +) -> bool: + """Add necessary dunder methods to classes decorated with attr.s. + + attrs is a package that lets you define classes without writing dull boilerplate code. + + At a quick glance, the decorator searches the class body for assignments of `attr.ib`s (or + annotated variables if auto_attribs=True), then depending on how the decorator is called, + it will add an __init__ or all the compare methods. + For frozen=True it will turn the attrs into properties. + + Hashability will be set according to https://www.attrs.org/en/stable/hashing.html. + + See https://www.attrs.org/en/stable/how-does-it-work.html for information on how attrs works. + + If this returns False, some required metadata was not ready yet, and we need another + pass. + """ + with state.strict_optional_set(ctx.api.options.strict_optional): + # This hook is called during semantic analysis, but it uses a bunch of + # type-checking ops, so it needs the strict optional set properly. + return attr_class_maker_callback_impl( + ctx, auto_attribs_default, frozen_default, slots_default + ) + + +def attr_class_maker_callback_impl( + ctx: mypy.plugin.ClassDefContext, + auto_attribs_default: bool | None, + frozen_default: bool, + slots_default: bool, +) -> bool: + info = ctx.cls.info + + init = _get_decorator_bool_argument(ctx, "init", True) + frozen = _get_frozen(ctx, frozen_default) + order = _determine_eq_order(ctx) + slots = _get_decorator_bool_argument(ctx, "slots", slots_default) + + auto_attribs = _get_decorator_optional_bool_argument(ctx, "auto_attribs", auto_attribs_default) + kw_only = _get_decorator_bool_argument(ctx, "kw_only", False) + match_args = _get_decorator_bool_argument(ctx, "match_args", True) + + for super_info in ctx.cls.info.mro[1:-1]: + if "attrs_tag" in super_info.metadata and "attrs" not in super_info.metadata: + # Super class is not ready yet. Request another pass. + return False + + attributes = _analyze_class(ctx, auto_attribs, kw_only) + + # Check if attribute types are ready. + for attr in attributes: + node = info.get(attr.name) + if node is None: + # This name is likely blocked by some semantic analysis error that + # should have been reported already. + _add_empty_metadata(info) + return True + + _add_attrs_magic_attribute(ctx, [(attr.name, info[attr.name].type) for attr in attributes]) + if slots: + _add_slots(ctx, attributes) + if match_args and ctx.api.options.python_version[:2] >= (3, 10): + # `.__match_args__` is only added for python3.10+, but the argument + # exists for earlier versions as well. + _add_match_args(ctx, attributes) + + # Save the attributes so that subclasses can reuse them. + ctx.cls.info.metadata["attrs"] = { + "attributes": [attr.serialize() for attr in attributes], + "frozen": frozen, + } + + adder = MethodAdder(ctx) + # If __init__ is not being generated, attrs still generates it as __attrs_init__ instead. + _add_init(ctx, attributes, adder, "__init__" if init else ATTRS_INIT_NAME) + + if order: + _add_order(ctx, adder) + if frozen: + _make_frozen(ctx, attributes) + # Frozen classes are hashable by default, even if inheriting from non-frozen ones. + hashable: bool | None = _get_decorator_bool_argument( + ctx, "hash", True + ) and _get_decorator_bool_argument(ctx, "unsafe_hash", True) + else: + hashable = _get_decorator_optional_bool_argument(ctx, "unsafe_hash") + if hashable is None: # unspecified + hashable = _get_decorator_optional_bool_argument(ctx, "hash") + + eq = _get_decorator_optional_bool_argument(ctx, "eq") + has_own_hash = "__hash__" in ctx.cls.info.names + + if has_own_hash or (hashable is None and eq is False): + pass # Do nothing. + elif hashable: + # We copy the `__hash__` signature from `object` to make them hashable. + ctx.cls.info.names["__hash__"] = ctx.cls.info.mro[-1].names["__hash__"] + else: + _remove_hashability(ctx) + + return True + + +def _get_frozen(ctx: mypy.plugin.ClassDefContext, frozen_default: bool) -> bool: + """Return whether this class is frozen.""" + if _get_decorator_bool_argument(ctx, "frozen", frozen_default): + return True + # Subclasses of frozen classes are frozen so check that. + for super_info in ctx.cls.info.mro[1:-1]: + if "attrs" in super_info.metadata and super_info.metadata["attrs"]["frozen"]: + return True + return False + + +def _analyze_class( + ctx: mypy.plugin.ClassDefContext, auto_attribs: bool | None, kw_only: bool +) -> list[Attribute]: + """Analyze the class body of an attr maker, its parents, and return the Attributes found. + + auto_attribs=True means we'll generate attributes from type annotations also. + auto_attribs=None means we'll detect which mode to use. + kw_only=True means that all attributes created here will be keyword only args in __init__. + """ + own_attrs: dict[str, Attribute] = {} + if auto_attribs is None: + auto_attribs = _detect_auto_attribs(ctx) + + # Walk the body looking for assignments and decorators. + for stmt in ctx.cls.defs.body: + if isinstance(stmt, AssignmentStmt): + for attr in _attributes_from_assignment(ctx, stmt, auto_attribs, kw_only): + # When attrs are defined twice in the same body we want to use the 2nd definition + # in the 2nd location. So remove it from the OrderedDict. + # Unless it's auto_attribs in which case we want the 2nd definition in the + # 1st location. + if not auto_attribs and attr.name in own_attrs: + del own_attrs[attr.name] + own_attrs[attr.name] = attr + elif isinstance(stmt, Decorator): + _cleanup_decorator(stmt, own_attrs) + + for attribute in own_attrs.values(): + # Even though these look like class level assignments we want them to look like + # instance level assignments. + if attribute.name in ctx.cls.info.names: + node = ctx.cls.info.names[attribute.name].node + if isinstance(node, PlaceholderNode): + # This node is not ready yet. + continue + assert isinstance(node, Var) + node.is_initialized_in_class = False + + # Traverse the MRO and collect attributes from the parents. + taken_attr_names = set(own_attrs) + super_attrs = [] + for super_info in ctx.cls.info.mro[1:-1]: + if "attrs" in super_info.metadata: + # Each class depends on the set of attributes in its attrs ancestors. + ctx.api.add_plugin_dependency(make_wildcard_trigger(super_info.fullname)) + + for data in super_info.metadata["attrs"]["attributes"]: + # Only add an attribute if it hasn't been defined before. This + # allows for overwriting attribute definitions by subclassing. + if data["name"] not in taken_attr_names: + a = Attribute.deserialize(super_info, data, ctx.api) + a.expand_typevar_from_subtype(ctx.cls.info) + super_attrs.append(a) + taken_attr_names.add(a.name) + attributes = super_attrs + list(own_attrs.values()) + + # Check the init args for correct default-ness. Note: This has to be done after all the + # attributes for all classes have been read, because subclasses can override parents. + last_default = False + + for i, attribute in enumerate(attributes): + if not attribute.init: + continue + + if attribute.kw_only: + # Keyword-only attributes don't care whether they are default or not. + continue + + # If the issue comes from merging different classes, report it + # at the class definition point. + context = attribute.context if i >= len(super_attrs) else ctx.cls + + if not attribute.has_default and last_default: + ctx.api.fail("Non-default attributes not allowed after default attributes.", context) + last_default |= attribute.has_default + + return attributes + + +def _add_empty_metadata(info: TypeInfo) -> None: + """Add empty metadata to mark that we've finished processing this class.""" + info.metadata["attrs"] = {"attributes": [], "frozen": False} + + +def _detect_auto_attribs(ctx: mypy.plugin.ClassDefContext) -> bool: + """Return whether auto_attribs should be enabled or disabled. + + It's disabled if there are any unannotated attribs() + """ + for stmt in ctx.cls.defs.body: + if isinstance(stmt, AssignmentStmt): + for lvalue in stmt.lvalues: + lvalues, rvalues = _parse_assignments(lvalue, stmt) + + if len(lvalues) != len(rvalues): + # This means we have some assignment that isn't 1 to 1. + # It can't be an attrib. + continue + + for lhs, rvalue in zip(lvalues, rvalues): + # Check if the right hand side is a call to an attribute maker. + if ( + isinstance(rvalue, CallExpr) + and isinstance(rvalue.callee, RefExpr) + and rvalue.callee.fullname in attr_attrib_makers + and not stmt.new_syntax + ): + # This means we have an attrib without an annotation and so + # we can't do auto_attribs=True + return False + return True + + +def _attributes_from_assignment( + ctx: mypy.plugin.ClassDefContext, stmt: AssignmentStmt, auto_attribs: bool, kw_only: bool +) -> Iterable[Attribute]: + """Return Attribute objects that are created by this assignment. + + The assignments can look like this: + x = attr.ib() + x = y = attr.ib() + x, y = attr.ib(), attr.ib() + or if auto_attribs is enabled also like this: + x: type + x: type = default_value + x: type = attr.ib(...) + """ + for lvalue in stmt.lvalues: + lvalues, rvalues = _parse_assignments(lvalue, stmt) + + if len(lvalues) != len(rvalues): + # This means we have some assignment that isn't 1 to 1. + # It can't be an attrib. + continue + + for lhs, rvalue in zip(lvalues, rvalues): + # Check if the right hand side is a call to an attribute maker. + if ( + isinstance(rvalue, CallExpr) + and isinstance(rvalue.callee, RefExpr) + and rvalue.callee.fullname in attr_attrib_makers + ): + attr = _attribute_from_attrib_maker(ctx, auto_attribs, kw_only, lhs, rvalue, stmt) + if attr: + yield attr + elif auto_attribs and stmt.type and stmt.new_syntax and not is_class_var(lhs): + yield _attribute_from_auto_attrib(ctx, kw_only, lhs, rvalue, stmt) + + +def _cleanup_decorator(stmt: Decorator, attr_map: dict[str, Attribute]) -> None: + """Handle decorators in class bodies. + + `x.default` will set a default value on x + `x.validator` and `x.default` will get removed to avoid throwing a type error. + """ + remove_me = [] + for func_decorator in stmt.decorators: + if ( + isinstance(func_decorator, MemberExpr) + and isinstance(func_decorator.expr, NameExpr) + and func_decorator.expr.name in attr_map + ): + if func_decorator.name == "default": + attr_map[func_decorator.expr.name].has_default = True + + if func_decorator.name in ("default", "validator"): + # These are decorators on the attrib object that only exist during + # class creation time. In order to not trigger a type error later we + # just remove them. This might leave us with a Decorator with no + # decorators (Emperor's new clothes?) + # TODO: It would be nice to type-check these rather than remove them. + # default should be Callable[[], T] + # validator should be Callable[[Any, 'Attribute', T], Any] + # where T is the type of the attribute. + remove_me.append(func_decorator) + for dec in remove_me: + stmt.decorators.remove(dec) + + +def _attribute_from_auto_attrib( + ctx: mypy.plugin.ClassDefContext, + kw_only: bool, + lhs: NameExpr, + rvalue: Expression, + stmt: AssignmentStmt, +) -> Attribute: + """Return an Attribute for a new type assignment.""" + name = unmangle(lhs.name) + # `x: int` (without equal sign) assigns rvalue to TempNode(AnyType()) + has_rhs = not isinstance(rvalue, TempNode) + sym = ctx.cls.info.names.get(name) + init_type = sym.type if sym else None + return Attribute(name, None, ctx.cls.info, has_rhs, True, kw_only, None, stmt, init_type) + + +def _attribute_from_attrib_maker( + ctx: mypy.plugin.ClassDefContext, + auto_attribs: bool, + kw_only: bool, + lhs: NameExpr, + rvalue: CallExpr, + stmt: AssignmentStmt, +) -> Attribute | None: + """Return an Attribute from the assignment or None if you can't make one.""" + if auto_attribs and not stmt.new_syntax: + # auto_attribs requires an annotation on *every* attr.ib. + assert lhs.node is not None + ctx.api.msg.need_annotation_for_var(lhs.node, stmt) + return None + + if len(stmt.lvalues) > 1: + ctx.api.fail("Too many names for one attribute", stmt) + return None + + # This is the type that belongs in the __init__ method for this attrib. + init_type = stmt.type + + # Read all the arguments from the call. + init = _get_bool_argument(ctx, rvalue, "init", True) + # Note: If the class decorator says kw_only=True the attribute is ignored. + # See https://github.com/python-attrs/attrs/issues/481 for explanation. + kw_only |= _get_bool_argument(ctx, rvalue, "kw_only", False) + + # TODO: Check for attr.NOTHING + attr_has_default = bool(_get_argument(rvalue, "default")) + attr_has_factory = bool(_get_argument(rvalue, "factory")) + + if attr_has_default and attr_has_factory: + ctx.api.fail('Can\'t pass both "default" and "factory".', rvalue) + elif attr_has_factory: + attr_has_default = True + + # If the type isn't set through annotation but is passed through `type=` use that. + type_arg = _get_argument(rvalue, "type") + if type_arg and not init_type: + try: + un_type = expr_to_unanalyzed_type(type_arg, ctx.api.options, ctx.api.is_stub_file) + except TypeTranslationError: + ctx.api.fail("Invalid argument to type", type_arg) + else: + init_type = ctx.api.anal_type(un_type) + if init_type and isinstance(lhs.node, Var) and not lhs.node.type: + # If there is no annotation, add one. + lhs.node.type = init_type + lhs.is_inferred_def = False + + # Note: convert is deprecated but works the same as converter. + converter = _get_argument(rvalue, "converter") + convert = _get_argument(rvalue, "convert") + if convert and converter: + ctx.api.fail('Can\'t pass both "convert" and "converter".', rvalue) + elif convert: + ctx.api.fail("convert is deprecated, use converter", rvalue) + converter = convert + converter_info = _parse_converter(ctx, converter) + + # Custom alias might be defined: + alias = None + alias_expr = _get_argument(rvalue, "alias") + if alias_expr: + alias = ctx.api.parse_str_literal(alias_expr) + if alias is None: + ctx.api.fail( + '"alias" argument to attrs field must be a string literal', + rvalue, + code=LITERAL_REQ, + ) + name = unmangle(lhs.name) + return Attribute( + name, alias, ctx.cls.info, attr_has_default, init, kw_only, converter_info, stmt, init_type + ) + + +def _parse_converter( + ctx: mypy.plugin.ClassDefContext, converter_expr: Expression | None +) -> Converter | None: + """Return the Converter object from an Expression.""" + # TODO: Support complex converters, e.g. lambdas, calls, etc. + if not converter_expr: + return None + converter_info = Converter() + if ( + isinstance(converter_expr, CallExpr) + and isinstance(converter_expr.callee, RefExpr) + and converter_expr.callee.fullname in attr_optional_converters + and converter_expr.args + and converter_expr.args[0] + ): + # Special handling for attr.converters.optional(type) + # We extract the type and add make the init_args Optional in Attribute.argument + converter_expr = converter_expr.args[0] + is_attr_converters_optional = True + else: + is_attr_converters_optional = False + + converter_type: Type | None = None + if isinstance(converter_expr, RefExpr) and converter_expr.node: + if isinstance(converter_expr.node, FuncDef): + if converter_expr.node.type and isinstance(converter_expr.node.type, FunctionLike): + converter_type = converter_expr.node.type + else: # The converter is an unannotated function. + converter_info.init_type = AnyType(TypeOfAny.unannotated) + return converter_info + elif isinstance(converter_expr.node, OverloadedFuncDef) and is_valid_overloaded_converter( + converter_expr.node + ): + converter_type = converter_expr.node.type + elif isinstance(converter_expr.node, TypeInfo): + from mypy.checkmember import type_object_type # To avoid import cycle. + + converter_type = type_object_type(converter_expr.node, ctx.api.named_type) + elif ( + isinstance(converter_expr, IndexExpr) + and isinstance(converter_expr.analyzed, TypeApplication) + and isinstance(converter_expr.base, RefExpr) + and isinstance(converter_expr.base.node, TypeInfo) + ): + # The converter is a generic type. + from mypy.checkmember import type_object_type # To avoid import cycle. + + converter_type = type_object_type(converter_expr.base.node, ctx.api.named_type) + if isinstance(converter_type, CallableType): + converter_type = apply_generic_arguments( + converter_type, + converter_expr.analyzed.types, + ctx.api.msg.incompatible_typevar_value, + converter_type, + ) + else: + converter_type = None + + if isinstance(converter_expr, LambdaExpr): + # TODO: should we send a fail if converter_expr.min_args > 1? + converter_info.init_type = AnyType(TypeOfAny.unannotated) + return converter_info + + if not converter_type: + # Signal that we have an unsupported converter. + ctx.api.fail( + "Unsupported converter, only named functions, types and lambdas are currently " + "supported", + converter_expr, + ) + converter_info.init_type = AnyType(TypeOfAny.from_error) + return converter_info + + converter_type = get_proper_type(converter_type) + if isinstance(converter_type, CallableType) and converter_type.arg_types: + converter_info.init_type = converter_type.arg_types[0] + if not is_attr_converters_optional: + converter_info.ret_type = converter_type.ret_type + elif isinstance(converter_type, Overloaded): + types: list[Type] = [] + for item in converter_type.items: + # Walk the overloads looking for methods that can accept one argument. + num_arg_types = len(item.arg_types) + if not num_arg_types: + continue + if num_arg_types > 1 and any(kind == ARG_POS for kind in item.arg_kinds[1:]): + continue + types.append(item.arg_types[0]) + # Make a union of all the valid types. + if types: + converter_info.init_type = make_simplified_union(types) + + if is_attr_converters_optional and converter_info.init_type: + # If the converter was attr.converter.optional(type) then add None to + # the allowed init_type. + converter_info.init_type = UnionType.make_union([converter_info.init_type, NoneType()]) + + return converter_info + + +def is_valid_overloaded_converter(defn: OverloadedFuncDef) -> bool: + return all( + (not isinstance(item, Decorator) or isinstance(item.func.type, FunctionLike)) + for item in defn.items + ) + + +def _parse_assignments( + lvalue: Expression, stmt: AssignmentStmt +) -> tuple[list[NameExpr], list[Expression]]: + """Convert a possibly complex assignment expression into lists of lvalues and rvalues.""" + lvalues: list[NameExpr] = [] + rvalues: list[Expression] = [] + if isinstance(lvalue, (TupleExpr, ListExpr)): + if all(isinstance(item, NameExpr) for item in lvalue.items): + lvalues = cast(List[NameExpr], lvalue.items) + if isinstance(stmt.rvalue, (TupleExpr, ListExpr)): + rvalues = stmt.rvalue.items + elif isinstance(lvalue, NameExpr): + lvalues = [lvalue] + rvalues = [stmt.rvalue] + return lvalues, rvalues + + +def _add_order(ctx: mypy.plugin.ClassDefContext, adder: MethodAdder) -> None: + """Generate all the ordering methods for this class.""" + bool_type = ctx.api.named_type("builtins.bool") + object_type = ctx.api.named_type("builtins.object") + # Make the types be: + # AT = TypeVar('AT') + # def __lt__(self: AT, other: AT) -> bool + # This way comparisons with subclasses will work correctly. + fullname = f"{ctx.cls.info.fullname}.{SELF_TVAR_NAME}" + tvd = TypeVarType( + SELF_TVAR_NAME, + fullname, + # Namespace is patched per-method below. + id=TypeVarId(-1, namespace=""), + values=[], + upper_bound=object_type, + default=AnyType(TypeOfAny.from_omitted_generics), + ) + self_tvar_expr = TypeVarExpr( + SELF_TVAR_NAME, fullname, [], object_type, AnyType(TypeOfAny.from_omitted_generics) + ) + ctx.cls.info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr) + + for method in ["__lt__", "__le__", "__gt__", "__ge__"]: + namespace = f"{ctx.cls.info.fullname}.{method}" + tvd = tvd.copy_modified(id=TypeVarId(tvd.id.raw_id, namespace=namespace)) + args = [Argument(Var("other", tvd), tvd, None, ARG_POS)] + adder.add_method(method, args, bool_type, self_type=tvd, tvd=tvd) + + +def _make_frozen(ctx: mypy.plugin.ClassDefContext, attributes: list[Attribute]) -> None: + """Turn all the attributes into properties to simulate frozen classes.""" + for attribute in attributes: + if attribute.name in ctx.cls.info.names: + # This variable belongs to this class so we can modify it. + node = ctx.cls.info.names[attribute.name].node + if not isinstance(node, Var): + # The superclass attribute was overridden with a non-variable. + # No need to do anything here, override will be verified during + # type checking. + continue + node.is_property = True + else: + # This variable belongs to a super class so create new Var so we + # can modify it. + var = Var(attribute.name, attribute.init_type) + var.info = ctx.cls.info + var._fullname = f"{ctx.cls.info.fullname}.{var.name}" + ctx.cls.info.names[var.name] = SymbolTableNode(MDEF, var) + var.is_property = True + + +def _add_init( + ctx: mypy.plugin.ClassDefContext, + attributes: list[Attribute], + adder: MethodAdder, + method_name: Literal["__init__", "__attrs_init__"], +) -> None: + """Generate an __init__ method for the attributes and add it to the class.""" + # Convert attributes to arguments with kw_only arguments at the end of + # the argument list + pos_args = [] + kw_only_args = [] + sym_table = ctx.cls.info.names + for attribute in attributes: + if not attribute.init: + continue + if attribute.kw_only: + kw_only_args.append(attribute.argument(ctx)) + else: + pos_args.append(attribute.argument(ctx)) + + # If the attribute is Final, present in `__init__` and has + # no default, make sure it doesn't error later. + if not attribute.has_default and attribute.name in sym_table: + sym_node = sym_table[attribute.name].node + if isinstance(sym_node, Var) and sym_node.is_final: + sym_node.final_set_in_init = True + args = pos_args + kw_only_args + if all( + # We use getattr rather than instance checks because the variable.type + # might be wrapped into a Union or some other type, but even non-Any + # types reliably track the fact that the argument was not annotated. + getattr(arg.variable.type, "type_of_any", None) == TypeOfAny.unannotated + for arg in args + ): + # This workaround makes --disallow-incomplete-defs usable with attrs, + # but is definitely suboptimal as a long-term solution. + # See https://github.com/python/mypy/issues/5954 for discussion. + for a in args: + a.variable.type = AnyType(TypeOfAny.implementation_artifact) + a.type_annotation = AnyType(TypeOfAny.implementation_artifact) + adder.add_method(method_name, args, NoneType()) + + +def _add_attrs_magic_attribute( + ctx: mypy.plugin.ClassDefContext, attrs: list[tuple[str, Type | None]] +) -> None: + any_type = AnyType(TypeOfAny.explicit) + attributes_types: list[Type] = [ + ctx.api.named_type_or_none("attr.Attribute", [attr_type or any_type]) or any_type + for _, attr_type in attrs + ] + fallback_type = ctx.api.named_type( + "builtins.tuple", [ctx.api.named_type_or_none("attr.Attribute", [any_type]) or any_type] + ) + + attr_name = MAGIC_ATTR_CLS_NAME_TEMPLATE.format(ctx.cls.fullname.replace(".", "_")) + ti = ctx.api.basic_new_typeinfo(attr_name, fallback_type, 0) + for (name, _), attr_type in zip(attrs, attributes_types): + var = Var(name, attr_type) + var._fullname = name + var.is_property = True + proper_type = get_proper_type(attr_type) + if isinstance(proper_type, Instance): + var.info = proper_type.type + ti.names[name] = SymbolTableNode(MDEF, var, plugin_generated=True) + attributes_type = Instance(ti, []) + + # We need to stash the type of the magic attribute so it can be + # loaded on cached runs. + ctx.cls.info.names[attr_name] = SymbolTableNode(MDEF, ti, plugin_generated=True) + + add_attribute_to_class( + ctx.api, + ctx.cls, + MAGIC_ATTR_NAME, + TupleType(attributes_types, fallback=attributes_type), + fullname=f"{ctx.cls.fullname}.{MAGIC_ATTR_NAME}", + override_allow_incompatible=True, + is_classvar=True, + ) + + +def _add_slots(ctx: mypy.plugin.ClassDefContext, attributes: list[Attribute]) -> None: + if any(p.slots is None for p in ctx.cls.info.mro[1:-1]): + # At least one type in mro (excluding `self` and `object`) + # does not have concrete `__slots__` defined. Ignoring. + return + + # Unlike `@dataclasses.dataclass`, `__slots__` is rewritten here. + ctx.cls.info.slots = {attr.name for attr in attributes} + + # Also, inject `__slots__` attribute to class namespace: + slots_type = TupleType( + [ctx.api.named_type("builtins.str") for _ in attributes], + fallback=ctx.api.named_type("builtins.tuple"), + ) + add_attribute_to_class(api=ctx.api, cls=ctx.cls, name="__slots__", typ=slots_type) + + +def _add_match_args(ctx: mypy.plugin.ClassDefContext, attributes: list[Attribute]) -> None: + if ( + "__match_args__" not in ctx.cls.info.names + or ctx.cls.info.names["__match_args__"].plugin_generated + ): + str_type = ctx.api.named_type("builtins.str") + match_args = TupleType( + [ + str_type.copy_modified(last_known_value=LiteralType(attr.name, fallback=str_type)) + for attr in attributes + if not attr.kw_only and attr.init + ], + fallback=ctx.api.named_type("builtins.tuple"), + ) + add_attribute_to_class(api=ctx.api, cls=ctx.cls, name="__match_args__", typ=match_args) + + +def _remove_hashability(ctx: mypy.plugin.ClassDefContext) -> None: + """Remove hashability from a class.""" + add_attribute_to_class( + ctx.api, ctx.cls, "__hash__", NoneType(), is_classvar=True, overwrite_existing=True + ) + + +class MethodAdder: + """Helper to add methods to a TypeInfo. + + ctx: The ClassDefCtx we are using on which we will add methods. + """ + + # TODO: Combine this with the code build_namedtuple_typeinfo to support both. + + def __init__(self, ctx: mypy.plugin.ClassDefContext) -> None: + self.ctx = ctx + self.self_type = fill_typevars(ctx.cls.info) + + def add_method( + self, + method_name: str, + args: list[Argument], + ret_type: Type, + self_type: Type | None = None, + tvd: TypeVarType | None = None, + ) -> None: + """Add a method: def (self, ) -> ): ... to info. + + self_type: The type to use for the self argument or None to use the inferred self type. + tvd: If the method is generic these should be the type variables. + """ + self_type = self_type if self_type is not None else self.self_type + add_method_to_class( + self.ctx.api, self.ctx.cls, method_name, args, ret_type, self_type, tvd + ) + + +def _get_attrs_init_type(typ: Instance) -> CallableType | None: + """ + If `typ` refers to an attrs class, get the type of its initializer method. + """ + magic_attr = typ.type.get(MAGIC_ATTR_NAME) + if magic_attr is None or not magic_attr.plugin_generated: + return None + init_method = typ.type.get_method("__init__") or typ.type.get_method(ATTRS_INIT_NAME) + if not isinstance(init_method, FuncDef) or not isinstance(init_method.type, CallableType): + return None + return init_method.type + + +def _fail_not_attrs_class(ctx: mypy.plugin.FunctionSigContext, t: Type, parent_t: Type) -> None: + t_name = format_type_bare(t, ctx.api.options) + if parent_t is t: + msg = ( + f'Argument 1 to "evolve" has a variable type "{t_name}" not bound to an attrs class' + if isinstance(t, TypeVarType) + else f'Argument 1 to "evolve" has incompatible type "{t_name}"; expected an attrs class' + ) + else: + pt_name = format_type_bare(parent_t, ctx.api.options) + msg = ( + f'Argument 1 to "evolve" has type "{pt_name}" whose item "{t_name}" is not bound to an attrs class' + if isinstance(t, TypeVarType) + else f'Argument 1 to "evolve" has incompatible type "{pt_name}" whose item "{t_name}" is not an attrs class' + ) + + ctx.api.fail(msg, ctx.context) + + +def _get_expanded_attr_types( + ctx: mypy.plugin.FunctionSigContext, + typ: ProperType, + display_typ: ProperType, + parent_typ: ProperType, +) -> list[Mapping[str, Type]] | None: + """ + For a given type, determine what attrs classes it can be: for each class, return the field types. + For generic classes, the field types are expanded. + If the type contains Any or a non-attrs type, returns None; in the latter case, also reports an error. + """ + if isinstance(typ, AnyType): + return None + elif isinstance(typ, UnionType): + ret: list[Mapping[str, Type]] | None = [] + for item in typ.relevant_items(): + item = get_proper_type(item) + item_types = _get_expanded_attr_types(ctx, item, item, parent_typ) + if ret is not None and item_types is not None: + ret += item_types + else: + ret = None # but keep iterating to emit all errors + return ret + elif isinstance(typ, TypeVarType): + return _get_expanded_attr_types( + ctx, get_proper_type(typ.upper_bound), display_typ, parent_typ + ) + elif isinstance(typ, Instance): + init_func = _get_attrs_init_type(typ) + if init_func is None: + _fail_not_attrs_class(ctx, display_typ, parent_typ) + return None + init_func = expand_type_by_instance(init_func, typ) + # [1:] to skip the self argument of AttrClass.__init__ + field_names = cast(List[str], init_func.arg_names[1:]) + field_types = init_func.arg_types[1:] + return [dict(zip(field_names, field_types))] + else: + _fail_not_attrs_class(ctx, display_typ, parent_typ) + return None + + +def _meet_fields(types: list[Mapping[str, Type]]) -> Mapping[str, Type]: + """ + "Meet" the fields of a list of attrs classes, i.e. for each field, its new type will be the lower bound. + """ + field_to_types = defaultdict(list) + for fields in types: + for name, typ in fields.items(): + field_to_types[name].append(typ) + + return { + name: ( + get_proper_type(reduce(meet_types, f_types)) + if len(f_types) == len(types) + else UninhabitedType() + ) + for name, f_types in field_to_types.items() + } + + +def evolve_function_sig_callback(ctx: mypy.plugin.FunctionSigContext) -> CallableType: + """ + Generate a signature for the 'attr.evolve' function that's specific to the call site + and dependent on the type of the first argument. + """ + if len(ctx.args) != 2: + # Ideally the name and context should be callee's, but we don't have it in FunctionSigContext. + ctx.api.fail(f'"{ctx.default_signature.name}" has unexpected type annotation', ctx.context) + return ctx.default_signature + + if len(ctx.args[0]) != 1: + return ctx.default_signature # leave it to the type checker to complain + + inst_arg = ctx.args[0][0] + inst_type = get_proper_type(ctx.api.get_expression_type(inst_arg)) + inst_type_str = format_type_bare(inst_type, ctx.api.options) + + attr_types = _get_expanded_attr_types(ctx, inst_type, inst_type, inst_type) + if attr_types is None: + return ctx.default_signature + fields = _meet_fields(attr_types) + + return CallableType( + arg_names=["inst", *fields.keys()], + arg_kinds=[ARG_POS] + [ARG_NAMED_OPT] * len(fields), + arg_types=[inst_type, *fields.values()], + ret_type=inst_type, + fallback=ctx.default_signature.fallback, + name=f"{ctx.default_signature.name} of {inst_type_str}", + ) + + +def fields_function_sig_callback(ctx: mypy.plugin.FunctionSigContext) -> CallableType: + """Provide the signature for `attrs.fields`.""" + if len(ctx.args) != 1 or len(ctx.args[0]) != 1: + return ctx.default_signature + + proper_type = get_proper_type(ctx.api.get_expression_type(ctx.args[0][0])) + + # fields(Any) -> Any, fields(type[Any]) -> Any + if ( + isinstance(proper_type, AnyType) + or isinstance(proper_type, TypeType) + and isinstance(proper_type.item, AnyType) + ): + return ctx.default_signature + + cls = None + arg_types = ctx.default_signature.arg_types + + if isinstance(proper_type, TypeVarType): + inner = get_proper_type(proper_type.upper_bound) + if isinstance(inner, Instance): + # We need to work arg_types to compensate for the attrs stubs. + arg_types = [proper_type] + cls = inner.type + elif isinstance(proper_type, CallableType): + cls = proper_type.type_object() + + if cls is not None and MAGIC_ATTR_NAME in cls.names: + # This is a proper attrs class. + ret_type = cls.names[MAGIC_ATTR_NAME].type + assert ret_type is not None + return ctx.default_signature.copy_modified(arg_types=arg_types, ret_type=ret_type) + + return ctx.default_signature diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py new file mode 100644 index 000000000000..f0ff6f30a3b9 --- /dev/null +++ b/mypy/plugins/common.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +from typing import NamedTuple + +from mypy.argmap import map_actuals_to_formals +from mypy.fixup import TypeFixer +from mypy.nodes import ( + ARG_POS, + MDEF, + SYMBOL_FUNCBASE_TYPES, + Argument, + Block, + CallExpr, + ClassDef, + Decorator, + Expression, + FuncDef, + JsonDict, + NameExpr, + Node, + OverloadedFuncDef, + PassStmt, + RefExpr, + SymbolTableNode, + TypeInfo, + Var, +) +from mypy.plugin import CheckerPluginInterface, ClassDefContext, SemanticAnalyzerPluginInterface +from mypy.semanal_shared import ( + ALLOW_INCOMPATIBLE_OVERRIDE, + parse_bool, + require_bool_literal_argument, + set_callable_name, +) +from mypy.typeops import try_getting_str_literals as try_getting_str_literals +from mypy.types import ( + AnyType, + CallableType, + Instance, + LiteralType, + NoneType, + Overloaded, + Type, + TypeOfAny, + TypeType, + TypeVarType, + deserialize_type, + get_proper_type, +) +from mypy.types_utils import is_overlapping_none +from mypy.typevars import fill_typevars +from mypy.util import get_unique_redefinition_name + + +def _get_decorator_bool_argument(ctx: ClassDefContext, name: str, default: bool) -> bool: + """Return the bool argument for the decorator. + + This handles both @decorator(...) and @decorator. + """ + if isinstance(ctx.reason, CallExpr): + return _get_bool_argument(ctx, ctx.reason, name, default) + else: + return default + + +def _get_bool_argument(ctx: ClassDefContext, expr: CallExpr, name: str, default: bool) -> bool: + """Return the boolean value for an argument to a call or the + default if it's not found. + """ + attr_value = _get_argument(expr, name) + if attr_value: + return require_bool_literal_argument(ctx.api, attr_value, name, default) + return default + + +def _get_argument(call: CallExpr, name: str) -> Expression | None: + """Return the expression for the specific argument.""" + # To do this we use the CallableType of the callee to find the FormalArgument, + # then walk the actual CallExpr looking for the appropriate argument. + # + # Note: I'm not hard-coding the index so that in the future we can support other + # attrib and class makers. + callee_type = _get_callee_type(call) + if not callee_type: + return None + + argument = callee_type.argument_by_name(name) + if not argument: + return None + assert argument.name + + for i, (attr_name, attr_value) in enumerate(zip(call.arg_names, call.args)): + if argument.pos is not None and not attr_name and i == argument.pos: + return attr_value + if attr_name == argument.name: + return attr_value + + return None + + +def find_shallow_matching_overload_item(overload: Overloaded, call: CallExpr) -> CallableType: + """Perform limited lookup of a matching overload item. + + Full overload resolution is only supported during type checking, but plugins + sometimes need to resolve overloads. This can be used in some such use cases. + + Resolve overloads based on these things only: + + * Match using argument kinds and names + * If formal argument has type None, only accept the "None" expression in the callee + * If formal argument has type Literal[True] or Literal[False], only accept the + relevant bool literal + + Return the first matching overload item, or the last one if nothing matches. + """ + for item in overload.items[:-1]: + ok = True + mapped = map_actuals_to_formals( + call.arg_kinds, + call.arg_names, + item.arg_kinds, + item.arg_names, + lambda i: AnyType(TypeOfAny.special_form), + ) + + # Look for extra actuals + matched_actuals = set() + for actuals in mapped: + matched_actuals.update(actuals) + if any(i not in matched_actuals for i in range(len(call.args))): + ok = False + + for arg_type, kind, actuals in zip(item.arg_types, item.arg_kinds, mapped): + if kind.is_required() and not actuals: + # Missing required argument + ok = False + break + elif actuals: + args = [call.args[i] for i in actuals] + arg_type = get_proper_type(arg_type) + arg_none = any(isinstance(arg, NameExpr) and arg.name == "None" for arg in args) + if isinstance(arg_type, NoneType): + if not arg_none: + ok = False + break + elif ( + arg_none + and not is_overlapping_none(arg_type) + and not ( + isinstance(arg_type, Instance) + and arg_type.type.fullname == "builtins.object" + ) + and not isinstance(arg_type, AnyType) + ): + ok = False + break + elif isinstance(arg_type, LiteralType) and isinstance(arg_type.value, bool): + if not any(parse_bool(arg) == arg_type.value for arg in args): + ok = False + break + if ok: + return item + return overload.items[-1] + + +def _get_callee_type(call: CallExpr) -> CallableType | None: + """Return the type of the callee, regardless of its syntatic form.""" + + callee_node: Node | None = call.callee + + if isinstance(callee_node, RefExpr): + callee_node = callee_node.node + + # Some decorators may be using typing.dataclass_transform, which is itself a decorator, so we + # need to unwrap them to get at the true callee + if isinstance(callee_node, Decorator): + callee_node = callee_node.func + + if isinstance(callee_node, (Var, SYMBOL_FUNCBASE_TYPES)) and callee_node.type: + callee_node_type = get_proper_type(callee_node.type) + if isinstance(callee_node_type, Overloaded): + return find_shallow_matching_overload_item(callee_node_type, call) + elif isinstance(callee_node_type, CallableType): + return callee_node_type + + return None + + +def add_method( + ctx: ClassDefContext, + name: str, + args: list[Argument], + return_type: Type, + self_type: Type | None = None, + tvar_def: TypeVarType | None = None, + is_classmethod: bool = False, + is_staticmethod: bool = False, +) -> None: + """ + Adds a new method to a class. + Deprecated, use add_method_to_class() instead. + """ + add_method_to_class( + ctx.api, + ctx.cls, + name=name, + args=args, + return_type=return_type, + self_type=self_type, + tvar_def=tvar_def, + is_classmethod=is_classmethod, + is_staticmethod=is_staticmethod, + ) + + +class MethodSpec(NamedTuple): + """Represents a method signature to be added, except for `name`.""" + + args: list[Argument] + return_type: Type + self_type: Type | None = None + tvar_defs: list[TypeVarType] | None = None + + +def add_method_to_class( + api: SemanticAnalyzerPluginInterface | CheckerPluginInterface, + cls: ClassDef, + name: str, + # MethodSpec items kept for backward compatibility: + args: list[Argument], + return_type: Type, + self_type: Type | None = None, + tvar_def: list[TypeVarType] | TypeVarType | None = None, + is_classmethod: bool = False, + is_staticmethod: bool = False, +) -> FuncDef | Decorator: + """Adds a new method to a class definition.""" + _prepare_class_namespace(cls, name) + + if tvar_def is not None and not isinstance(tvar_def, list): + tvar_def = [tvar_def] + + func, sym = _add_method_by_spec( + api, + cls.info, + name, + MethodSpec(args=args, return_type=return_type, self_type=self_type, tvar_defs=tvar_def), + is_classmethod=is_classmethod, + is_staticmethod=is_staticmethod, + ) + cls.info.names[name] = sym + cls.info.defn.defs.body.append(func) + return func + + +def add_overloaded_method_to_class( + api: SemanticAnalyzerPluginInterface | CheckerPluginInterface, + cls: ClassDef, + name: str, + items: list[MethodSpec], + is_classmethod: bool = False, + is_staticmethod: bool = False, +) -> OverloadedFuncDef: + """Adds a new overloaded method to a class definition.""" + assert len(items) >= 2, "Overloads must contain at least two cases" + + # Save old definition, if it exists. + _prepare_class_namespace(cls, name) + + # Create function bodies for each passed method spec. + funcs: list[Decorator | FuncDef] = [] + for item in items: + func, _sym = _add_method_by_spec( + api, + cls.info, + name=name, + spec=item, + is_classmethod=is_classmethod, + is_staticmethod=is_staticmethod, + ) + if isinstance(func, FuncDef): + var = Var(func.name, func.type) + var.set_line(func.line) + func.is_decorated = True + func.deco_line = func.line + + deco = Decorator(func, [], var) + else: + deco = func + deco.is_overload = True + funcs.append(deco) + + # Create the final OverloadedFuncDef node: + overload_def = OverloadedFuncDef(funcs) + overload_def.info = cls.info + overload_def.is_class = is_classmethod + overload_def.is_static = is_staticmethod + sym = SymbolTableNode(MDEF, overload_def) + sym.plugin_generated = True + + cls.info.names[name] = sym + cls.info.defn.defs.body.append(overload_def) + return overload_def + + +def _prepare_class_namespace(cls: ClassDef, name: str) -> None: + info = cls.info + assert info + + # First remove any previously generated methods with the same name + # to avoid clashes and problems in the semantic analyzer. + if name in info.names: + sym = info.names[name] + if sym.plugin_generated and isinstance(sym.node, FuncDef): + cls.defs.body.remove(sym.node) + + # NOTE: we would like the plugin generated node to dominate, but we still + # need to keep any existing definitions so they get semantically analyzed. + if name in info.names: + # Get a nice unique name instead. + r_name = get_unique_redefinition_name(name, info.names) + info.names[r_name] = info.names[name] + + +def _add_method_by_spec( + api: SemanticAnalyzerPluginInterface | CheckerPluginInterface, + info: TypeInfo, + name: str, + spec: MethodSpec, + *, + is_classmethod: bool, + is_staticmethod: bool, +) -> tuple[FuncDef | Decorator, SymbolTableNode]: + args, return_type, self_type, tvar_defs = spec + + assert not ( + is_classmethod is True and is_staticmethod is True + ), "Can't add a new method that's both staticmethod and classmethod." + + if isinstance(api, SemanticAnalyzerPluginInterface): + function_type = api.named_type("builtins.function") + else: + function_type = api.named_generic_type("builtins.function", []) + + if is_classmethod: + self_type = self_type or TypeType(fill_typevars(info)) + first = [Argument(Var("_cls"), self_type, None, ARG_POS, True)] + elif is_staticmethod: + first = [] + else: + self_type = self_type or fill_typevars(info) + first = [Argument(Var("self"), self_type, None, ARG_POS)] + args = first + args + + arg_types, arg_names, arg_kinds = [], [], [] + for arg in args: + assert arg.type_annotation, "All arguments must be fully typed." + arg_types.append(arg.type_annotation) + arg_names.append(arg.variable.name) + arg_kinds.append(arg.kind) + + signature = CallableType(arg_types, arg_kinds, arg_names, return_type, function_type) + if tvar_defs: + signature.variables = tvar_defs + + func = FuncDef(name, args, Block([PassStmt()])) + func.info = info + func.type = set_callable_name(signature, func) + func.is_class = is_classmethod + func.is_static = is_staticmethod + func._fullname = info.fullname + "." + name + func.line = info.line + + # Add decorator for is_staticmethod. It's unnecessary for is_classmethod. + if is_staticmethod: + func.is_decorated = True + v = Var(name, func.type) + v.info = info + v._fullname = func._fullname + v.is_staticmethod = True + dec = Decorator(func, [], v) + dec.line = info.line + sym = SymbolTableNode(MDEF, dec) + sym.plugin_generated = True + return dec, sym + + sym = SymbolTableNode(MDEF, func) + sym.plugin_generated = True + return func, sym + + +def add_attribute_to_class( + api: SemanticAnalyzerPluginInterface, + cls: ClassDef, + name: str, + typ: Type, + final: bool = False, + no_serialize: bool = False, + override_allow_incompatible: bool = False, + fullname: str | None = None, + is_classvar: bool = False, + overwrite_existing: bool = False, +) -> Var: + """ + Adds a new attribute to a class definition. + This currently only generates the symbol table entry and no corresponding AssignmentStatement + """ + info = cls.info + + # NOTE: we would like the plugin generated node to dominate, but we still + # need to keep any existing definitions so they get semantically analyzed. + if name in info.names and not overwrite_existing: + # Get a nice unique name instead. + r_name = get_unique_redefinition_name(name, info.names) + info.names[r_name] = info.names[name] + + node = Var(name, typ) + node.info = info + node.is_final = final + node.is_classvar = is_classvar + if name in ALLOW_INCOMPATIBLE_OVERRIDE: + node.allow_incompatible_override = True + else: + node.allow_incompatible_override = override_allow_incompatible + + if fullname: + node._fullname = fullname + else: + node._fullname = info.fullname + "." + name + + info.names[name] = SymbolTableNode( + MDEF, node, plugin_generated=True, no_serialize=no_serialize + ) + return node + + +def deserialize_and_fixup_type(data: str | JsonDict, api: SemanticAnalyzerPluginInterface) -> Type: + typ = deserialize_type(data) + typ.accept(TypeFixer(api.modules, allow_missing=False)) + return typ diff --git a/mypy/plugins/ctypes.py b/mypy/plugins/ctypes.py new file mode 100644 index 000000000000..b6dbec13ce90 --- /dev/null +++ b/mypy/plugins/ctypes.py @@ -0,0 +1,245 @@ +"""Plugin to provide accurate types for some parts of the ctypes module.""" + +from __future__ import annotations + +# Fully qualified instead of "from mypy.plugin import ..." to avoid circular import problems. +import mypy.plugin +from mypy import nodes +from mypy.maptype import map_instance_to_supertype +from mypy.messages import format_type +from mypy.subtypes import is_subtype +from mypy.typeops import make_simplified_union +from mypy.types import ( + AnyType, + CallableType, + Instance, + NoneType, + ProperType, + Type, + TypeOfAny, + UnionType, + flatten_nested_unions, + get_proper_type, +) + + +def _find_simplecdata_base_arg( + tp: Instance, api: mypy.plugin.CheckerPluginInterface +) -> ProperType | None: + """Try to find a parametrized _SimpleCData in tp's bases and return its single type argument. + + None is returned if _SimpleCData appears nowhere in tp's (direct or indirect) bases. + """ + if tp.type.has_base("_ctypes._SimpleCData"): + simplecdata_base = map_instance_to_supertype( + tp, + api.named_generic_type("_ctypes._SimpleCData", [AnyType(TypeOfAny.special_form)]).type, + ) + assert len(simplecdata_base.args) == 1, "_SimpleCData takes exactly one type argument" + return get_proper_type(simplecdata_base.args[0]) + return None + + +def _autoconvertible_to_cdata(tp: Type, api: mypy.plugin.CheckerPluginInterface) -> Type: + """Get a type that is compatible with all types that can be implicitly converted to the given + CData type. + + Examples: + * c_int -> Union[c_int, int] + * c_char_p -> Union[c_char_p, bytes, int, NoneType] + * MyStructure -> MyStructure + """ + allowed_types = [] + # If tp is a union, we allow all types that are convertible to at least one of the union + # items. This is not quite correct - strictly speaking, only types convertible to *all* of the + # union items should be allowed. This may be worth changing in the future, but the more + # correct algorithm could be too strict to be useful. + for t in flatten_nested_unions([tp]): + t = get_proper_type(t) + # Every type can be converted from itself (obviously). + allowed_types.append(t) + if isinstance(t, Instance): + unboxed = _find_simplecdata_base_arg(t, api) + if unboxed is not None: + # If _SimpleCData appears in tp's (direct or indirect) bases, its type argument + # specifies the type's "unboxed" version, which can always be converted back to + # the original "boxed" type. + allowed_types.append(unboxed) + + if t.type.has_base("ctypes._PointerLike"): + # Pointer-like _SimpleCData subclasses can also be converted from + # an int or None. + allowed_types.append(api.named_generic_type("builtins.int", [])) + allowed_types.append(NoneType()) + + return make_simplified_union(allowed_types) + + +def _autounboxed_cdata(tp: Type) -> ProperType: + """Get the auto-unboxed version of a CData type, if applicable. + + For *direct* _SimpleCData subclasses, the only type argument of _SimpleCData in the bases list + is returned. + For all other CData types, including indirect _SimpleCData subclasses, tp is returned as-is. + """ + tp = get_proper_type(tp) + + if isinstance(tp, UnionType): + return make_simplified_union([_autounboxed_cdata(t) for t in tp.items]) + elif isinstance(tp, Instance): + for base in tp.type.bases: + if base.type.fullname == "_ctypes._SimpleCData": + # If tp has _SimpleCData as a direct base class, + # the auto-unboxed type is the single type argument of the _SimpleCData type. + assert len(base.args) == 1 + return get_proper_type(base.args[0]) + # If tp is not a concrete type, or if there is no _SimpleCData in the bases, + # the type is not auto-unboxed. + return tp + + +def _get_array_element_type(tp: Type) -> ProperType | None: + """Get the element type of the Array type tp, or None if not specified.""" + tp = get_proper_type(tp) + if isinstance(tp, Instance): + assert tp.type.fullname == "_ctypes.Array" + if len(tp.args) == 1: + return get_proper_type(tp.args[0]) + return None + + +def array_constructor_callback(ctx: mypy.plugin.FunctionContext) -> Type: + """Callback to provide an accurate signature for the ctypes.Array constructor.""" + # Extract the element type from the constructor's return type, i. e. the type of the array + # being constructed. + et = _get_array_element_type(ctx.default_return_type) + if et is not None: + allowed = _autoconvertible_to_cdata(et, ctx.api) + assert ( + len(ctx.arg_types) == 1 + ), "The stub of the ctypes.Array constructor should have a single vararg parameter" + for arg_num, (arg_kind, arg_type) in enumerate(zip(ctx.arg_kinds[0], ctx.arg_types[0]), 1): + if arg_kind == nodes.ARG_POS and not is_subtype(arg_type, allowed): + ctx.api.msg.fail( + "Array constructor argument {} of type {}" + " is not convertible to the array element type {}".format( + arg_num, + format_type(arg_type, ctx.api.options), + format_type(et, ctx.api.options), + ), + ctx.context, + ) + elif arg_kind == nodes.ARG_STAR: + ty = ctx.api.named_generic_type("typing.Iterable", [allowed]) + if not is_subtype(arg_type, ty): + it = ctx.api.named_generic_type("typing.Iterable", [et]) + ctx.api.msg.fail( + "Array constructor argument {} of type {}" + " is not convertible to the array element type {}".format( + arg_num, + format_type(arg_type, ctx.api.options), + format_type(it, ctx.api.options), + ), + ctx.context, + ) + + return ctx.default_return_type + + +def array_getitem_callback(ctx: mypy.plugin.MethodContext) -> Type: + """Callback to provide an accurate return type for ctypes.Array.__getitem__.""" + et = _get_array_element_type(ctx.type) + if et is not None: + unboxed = _autounboxed_cdata(et) + assert ( + len(ctx.arg_types) == 1 + ), "The stub of ctypes.Array.__getitem__ should have exactly one parameter" + assert ( + len(ctx.arg_types[0]) == 1 + ), "ctypes.Array.__getitem__'s parameter should not be variadic" + index_type = get_proper_type(ctx.arg_types[0][0]) + if isinstance(index_type, Instance): + if index_type.type.has_base("builtins.int"): + return unboxed + elif index_type.type.has_base("builtins.slice"): + return ctx.api.named_generic_type("builtins.list", [unboxed]) + return ctx.default_return_type + + +def array_setitem_callback(ctx: mypy.plugin.MethodSigContext) -> CallableType: + """Callback to provide an accurate signature for ctypes.Array.__setitem__.""" + et = _get_array_element_type(ctx.type) + if et is not None: + allowed = _autoconvertible_to_cdata(et, ctx.api) + assert len(ctx.default_signature.arg_types) == 2 + index_type = get_proper_type(ctx.default_signature.arg_types[0]) + if isinstance(index_type, Instance): + arg_type = None + if index_type.type.has_base("builtins.int"): + arg_type = allowed + elif index_type.type.has_base("builtins.slice"): + arg_type = ctx.api.named_generic_type("builtins.list", [allowed]) + if arg_type is not None: + # Note: arg_type can only be None if index_type is invalid, in which case we use + # the default signature and let mypy report an error about it. + return ctx.default_signature.copy_modified( + arg_types=ctx.default_signature.arg_types[:1] + [arg_type] + ) + return ctx.default_signature + + +def array_iter_callback(ctx: mypy.plugin.MethodContext) -> Type: + """Callback to provide an accurate return type for ctypes.Array.__iter__.""" + et = _get_array_element_type(ctx.type) + if et is not None: + unboxed = _autounboxed_cdata(et) + return ctx.api.named_generic_type("typing.Iterator", [unboxed]) + return ctx.default_return_type + + +def array_value_callback(ctx: mypy.plugin.AttributeContext) -> Type: + """Callback to provide an accurate type for ctypes.Array.value.""" + et = _get_array_element_type(ctx.type) + if et is not None: + types: list[Type] = [] + for tp in flatten_nested_unions([et]): + tp = get_proper_type(tp) + if isinstance(tp, AnyType): + types.append(AnyType(TypeOfAny.from_another_any, source_any=tp)) + elif isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_char": + types.append(ctx.api.named_generic_type("builtins.bytes", [])) + elif isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_wchar": + types.append(ctx.api.named_generic_type("builtins.str", [])) + else: + ctx.api.msg.fail( + 'Array attribute "value" is only available' + ' with element type "c_char" or "c_wchar", not {}'.format( + format_type(et, ctx.api.options) + ), + ctx.context, + ) + return make_simplified_union(types) + return ctx.default_attr_type + + +def array_raw_callback(ctx: mypy.plugin.AttributeContext) -> Type: + """Callback to provide an accurate type for ctypes.Array.raw.""" + et = _get_array_element_type(ctx.type) + if et is not None: + types: list[Type] = [] + for tp in flatten_nested_unions([et]): + tp = get_proper_type(tp) + if ( + isinstance(tp, AnyType) + or isinstance(tp, Instance) + and tp.type.fullname == "ctypes.c_char" + ): + types.append(ctx.api.named_generic_type("builtins.bytes", [])) + else: + ctx.api.msg.fail( + 'Array attribute "raw" is only available' + ' with element type "c_char", not {}'.format(format_type(et, ctx.api.options)), + ctx.context, + ) + return make_simplified_union(types) + return ctx.default_attr_type diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py new file mode 100644 index 000000000000..349eca7f0143 --- /dev/null +++ b/mypy/plugins/dataclasses.py @@ -0,0 +1,1127 @@ +"""Plugin that provides support for dataclasses.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, Iterator, Literal + +from mypy import errorcodes, message_registry +from mypy.expandtype import expand_type, expand_type_by_instance +from mypy.meet import meet_types +from mypy.messages import format_type_bare +from mypy.nodes import ( + ARG_NAMED, + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + MDEF, + Argument, + AssignmentStmt, + Block, + CallExpr, + ClassDef, + Context, + DataclassTransformSpec, + Decorator, + EllipsisExpr, + Expression, + FuncDef, + FuncItem, + IfStmt, + JsonDict, + NameExpr, + Node, + PlaceholderNode, + RefExpr, + Statement, + SymbolTableNode, + TempNode, + TypeAlias, + TypeInfo, + TypeVarExpr, + Var, +) +from mypy.plugin import ClassDefContext, FunctionSigContext, SemanticAnalyzerPluginInterface +from mypy.plugins.common import ( + _get_callee_type, + _get_decorator_bool_argument, + add_attribute_to_class, + add_method_to_class, + deserialize_and_fixup_type, +) +from mypy.semanal_shared import find_dataclass_transform_spec, require_bool_literal_argument +from mypy.server.trigger import make_wildcard_trigger +from mypy.state import state +from mypy.typeops import map_type_from_supertype, try_getting_literals_from_type +from mypy.types import ( + AnyType, + CallableType, + FunctionLike, + Instance, + LiteralType, + NoneType, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeVarId, + TypeVarType, + UninhabitedType, + UnionType, + get_proper_type, +) +from mypy.typevars import fill_typevars + +if TYPE_CHECKING: + from mypy.checker import TypeChecker + +# The set of decorators that generate dataclasses. +dataclass_makers: Final = {"dataclass", "dataclasses.dataclass"} + + +SELF_TVAR_NAME: Final = "_DT" +_TRANSFORM_SPEC_FOR_DATACLASSES: Final = DataclassTransformSpec( + eq_default=True, + order_default=False, + kw_only_default=False, + frozen_default=False, + field_specifiers=("dataclasses.Field", "dataclasses.field"), +) +_INTERNAL_REPLACE_SYM_NAME: Final = "__mypy-replace" +_INTERNAL_POST_INIT_SYM_NAME: Final = "__mypy-post_init" + + +class DataclassAttribute: + def __init__( + self, + name: str, + alias: str | None, + is_in_init: bool, + is_init_var: bool, + has_default: bool, + line: int, + column: int, + type: Type | None, + info: TypeInfo, + kw_only: bool, + is_neither_frozen_nor_nonfrozen: bool, + api: SemanticAnalyzerPluginInterface, + ) -> None: + self.name = name + self.alias = alias + self.is_in_init = is_in_init + self.is_init_var = is_init_var + self.has_default = has_default + self.line = line + self.column = column + self.type = type # Type as __init__ argument + self.info = info + self.kw_only = kw_only + self.is_neither_frozen_nor_nonfrozen = is_neither_frozen_nor_nonfrozen + self._api = api + + def to_argument( + self, current_info: TypeInfo, *, of: Literal["__init__", "replace", "__post_init__"] + ) -> Argument: + if of == "__init__": + arg_kind = ARG_POS + if self.kw_only and self.has_default: + arg_kind = ARG_NAMED_OPT + elif self.kw_only and not self.has_default: + arg_kind = ARG_NAMED + elif not self.kw_only and self.has_default: + arg_kind = ARG_OPT + elif of == "replace": + arg_kind = ARG_NAMED if self.is_init_var and not self.has_default else ARG_NAMED_OPT + elif of == "__post_init__": + # We always use `ARG_POS` without a default value, because it is practical. + # Consider this case: + # + # @dataclass + # class My: + # y: dataclasses.InitVar[str] = 'a' + # def __post_init__(self, y: str) -> None: ... + # + # We would be *required* to specify `y: str = ...` if default is added here. + # But, most people won't care about adding default values to `__post_init__`, + # because it is not designed to be called directly, and duplicating default values + # for the sake of type-checking is unpleasant. + arg_kind = ARG_POS + return Argument( + variable=self.to_var(current_info), + type_annotation=self.expand_type(current_info), + initializer=EllipsisExpr() if self.has_default else None, # Only used by stubgen + kind=arg_kind, + ) + + def expand_type(self, current_info: TypeInfo) -> Type | None: + if self.type is not None and self.info.self_type is not None: + # In general, it is not safe to call `expand_type()` during semantic analysis, + # however this plugin is called very late, so all types should be fully ready. + # Also, it is tricky to avoid eager expansion of Self types here (e.g. because + # we serialize attributes). + with state.strict_optional_set(self._api.options.strict_optional): + return expand_type( + self.type, {self.info.self_type.id: fill_typevars(current_info)} + ) + return self.type + + def to_var(self, current_info: TypeInfo) -> Var: + return Var(self.alias or self.name, self.expand_type(current_info)) + + def serialize(self) -> JsonDict: + assert self.type + return { + "name": self.name, + "alias": self.alias, + "is_in_init": self.is_in_init, + "is_init_var": self.is_init_var, + "has_default": self.has_default, + "line": self.line, + "column": self.column, + "type": self.type.serialize(), + "kw_only": self.kw_only, + "is_neither_frozen_nor_nonfrozen": self.is_neither_frozen_nor_nonfrozen, + } + + @classmethod + def deserialize( + cls, info: TypeInfo, data: JsonDict, api: SemanticAnalyzerPluginInterface + ) -> DataclassAttribute: + data = data.copy() + typ = deserialize_and_fixup_type(data.pop("type"), api) + return cls(type=typ, info=info, **data, api=api) + + def expand_typevar_from_subtype(self, sub_type: TypeInfo) -> None: + """Expands type vars in the context of a subtype when an attribute is inherited + from a generic super type.""" + if self.type is not None: + with state.strict_optional_set(self._api.options.strict_optional): + self.type = map_type_from_supertype(self.type, sub_type, self.info) + + +class DataclassTransformer: + """Implement the behavior of @dataclass. + + Note that this may be executed multiple times on the same class, so + everything here must be idempotent. + + This runs after the main semantic analysis pass, so you can assume that + there are no placeholders. + """ + + def __init__( + self, + cls: ClassDef, + # Statement must also be accepted since class definition itself may be passed as the reason + # for subclass/metaclass-based uses of `typing.dataclass_transform` + reason: Expression | Statement, + spec: DataclassTransformSpec, + api: SemanticAnalyzerPluginInterface, + ) -> None: + self._cls = cls + self._reason = reason + self._spec = spec + self._api = api + + def transform(self) -> bool: + """Apply all the necessary transformations to the underlying + dataclass so as to ensure it is fully type checked according + to the rules in PEP 557. + """ + info = self._cls.info + attributes = self.collect_attributes() + if attributes is None: + # Some definitions are not ready. We need another pass. + return False + for attr in attributes: + if attr.type is None: + return False + decorator_arguments = { + "init": self._get_bool_arg("init", True), + "eq": self._get_bool_arg("eq", self._spec.eq_default), + "order": self._get_bool_arg("order", self._spec.order_default), + "frozen": self._get_bool_arg("frozen", self._spec.frozen_default), + "slots": self._get_bool_arg("slots", False), + "match_args": self._get_bool_arg("match_args", True), + } + py_version = self._api.options.python_version + + # If there are no attributes, it may be that the semantic analyzer has not + # processed them yet. In order to work around this, we can simply skip generating + # __init__ if there are no attributes, because if the user truly did not define any, + # then the object default __init__ with an empty signature will be present anyway. + if ( + decorator_arguments["init"] + and ("__init__" not in info.names or info.names["__init__"].plugin_generated) + and attributes + ): + args = [ + attr.to_argument(info, of="__init__") + for attr in attributes + if attr.is_in_init and not self._is_kw_only_type(attr.type) + ] + + if info.fallback_to_any: + # Make positional args optional since we don't know their order. + # This will at least allow us to typecheck them if they are called + # as kwargs + for arg in args: + if arg.kind == ARG_POS: + arg.kind = ARG_OPT + + existing_args_names = {arg.variable.name for arg in args} + gen_args_name = "generated_args" + while gen_args_name in existing_args_names: + gen_args_name += "_" + gen_kwargs_name = "generated_kwargs" + while gen_kwargs_name in existing_args_names: + gen_kwargs_name += "_" + args = [ + Argument(Var(gen_args_name), AnyType(TypeOfAny.explicit), None, ARG_STAR), + *args, + Argument(Var(gen_kwargs_name), AnyType(TypeOfAny.explicit), None, ARG_STAR2), + ] + + add_method_to_class( + self._api, self._cls, "__init__", args=args, return_type=NoneType() + ) + + if ( + decorator_arguments["eq"] + and info.get("__eq__") is None + or decorator_arguments["order"] + ): + # Type variable for self types in generated methods. + obj_type = self._api.named_type("builtins.object") + self_tvar_expr = TypeVarExpr( + SELF_TVAR_NAME, + info.fullname + "." + SELF_TVAR_NAME, + [], + obj_type, + AnyType(TypeOfAny.from_omitted_generics), + ) + info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr) + + # Add <, >, <=, >=, but only if the class has an eq method. + if decorator_arguments["order"]: + if not decorator_arguments["eq"]: + self._api.fail('"eq" must be True if "order" is True', self._reason) + + for method_name in ["__lt__", "__gt__", "__le__", "__ge__"]: + # Like for __eq__ and __ne__, we want "other" to match + # the self type. + obj_type = self._api.named_type("builtins.object") + order_tvar_def = TypeVarType( + SELF_TVAR_NAME, + f"{info.fullname}.{SELF_TVAR_NAME}", + id=TypeVarId(-1, namespace=f"{info.fullname}.{method_name}"), + values=[], + upper_bound=obj_type, + default=AnyType(TypeOfAny.from_omitted_generics), + ) + order_return_type = self._api.named_type("builtins.bool") + order_args = [ + Argument(Var("other", order_tvar_def), order_tvar_def, None, ARG_POS) + ] + + existing_method = info.get(method_name) + if existing_method is not None and not existing_method.plugin_generated: + assert existing_method.node + self._api.fail( + f'You may not have a custom "{method_name}" method when "order" is True', + existing_method.node, + ) + + add_method_to_class( + self._api, + self._cls, + method_name, + args=order_args, + return_type=order_return_type, + self_type=order_tvar_def, + tvar_def=order_tvar_def, + ) + + parent_decorator_arguments = [] + for parent in info.mro[1:-1]: + parent_args = parent.metadata.get("dataclass") + + # Ignore parent classes that directly specify a dataclass transform-decorated metaclass + # when searching for usage of the frozen parameter. PEP 681 states that a class that + # directly specifies such a metaclass must be treated as neither frozen nor non-frozen. + if parent_args and not _has_direct_dataclass_transform_metaclass(parent): + parent_decorator_arguments.append(parent_args) + + if decorator_arguments["frozen"]: + if any(not parent["frozen"] for parent in parent_decorator_arguments): + self._api.fail("Cannot inherit frozen dataclass from a non-frozen one", info) + self._propertize_callables(attributes, settable=False) + self._freeze(attributes) + else: + if any(parent["frozen"] for parent in parent_decorator_arguments): + self._api.fail("Cannot inherit non-frozen dataclass from a frozen one", info) + self._propertize_callables(attributes) + + if decorator_arguments["slots"]: + self.add_slots(info, attributes, correct_version=py_version >= (3, 10)) + + self.reset_init_only_vars(info, attributes) + + if ( + decorator_arguments["match_args"] + and ( + "__match_args__" not in info.names or info.names["__match_args__"].plugin_generated + ) + and py_version >= (3, 10) + ): + str_type = self._api.named_type("builtins.str") + literals: list[Type] = [ + LiteralType(attr.name, str_type) for attr in attributes if attr.is_in_init + ] + match_args_type = TupleType(literals, self._api.named_type("builtins.tuple")) + add_attribute_to_class(self._api, self._cls, "__match_args__", match_args_type) + + self._add_dataclass_fields_magic_attribute() + self._add_internal_replace_method(attributes) + if self._api.options.python_version >= (3, 13): + self._add_dunder_replace(attributes) + + if "__post_init__" in info.names: + self._add_internal_post_init_method(attributes) + + info.metadata["dataclass"] = { + "attributes": [attr.serialize() for attr in attributes], + "frozen": decorator_arguments["frozen"], + } + + return True + + def _add_dunder_replace(self, attributes: list[DataclassAttribute]) -> None: + """Add a `__replace__` method to the class, which is used to replace attributes in the `copy` module.""" + args = [ + attr.to_argument(self._cls.info, of="replace") + for attr in attributes + if attr.is_in_init + ] + type_vars = [tv for tv in self._cls.type_vars] + add_method_to_class( + self._api, + self._cls, + "__replace__", + args=args, + return_type=Instance(self._cls.info, type_vars), + ) + + def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> None: + """ + Stashes the signature of 'dataclasses.replace(...)' for this specific dataclass + to be used later whenever 'dataclasses.replace' is called for this dataclass. + """ + add_method_to_class( + self._api, + self._cls, + _INTERNAL_REPLACE_SYM_NAME, + args=[attr.to_argument(self._cls.info, of="replace") for attr in attributes], + return_type=NoneType(), + is_staticmethod=True, + ) + + def _add_internal_post_init_method(self, attributes: list[DataclassAttribute]) -> None: + add_method_to_class( + self._api, + self._cls, + _INTERNAL_POST_INIT_SYM_NAME, + args=[ + attr.to_argument(self._cls.info, of="__post_init__") + for attr in attributes + if attr.is_init_var + ], + return_type=NoneType(), + ) + + def add_slots( + self, info: TypeInfo, attributes: list[DataclassAttribute], *, correct_version: bool + ) -> None: + if not correct_version: + # This means that version is lower than `3.10`, + # it is just a non-existent argument for `dataclass` function. + self._api.fail( + 'Keyword argument "slots" for "dataclass" is only valid in Python 3.10 and higher', + self._reason, + ) + return + + generated_slots = {attr.name for attr in attributes} + if (info.slots is not None and info.slots != generated_slots) or info.names.get( + "__slots__" + ): + # This means we have a slots conflict. + # Class explicitly specifies a different `__slots__` field. + # And `@dataclass(slots=True)` is used. + # In runtime this raises a type error. + self._api.fail( + '"{}" both defines "__slots__" and is used with "slots=True"'.format( + self._cls.name + ), + self._cls, + ) + return + + if any(p.slots is None for p in info.mro[1:-1]): + # At least one type in mro (excluding `self` and `object`) + # does not have concrete `__slots__` defined. Ignoring. + return + + info.slots = generated_slots + + # Now, insert `.__slots__` attribute to class namespace: + slots_type = TupleType( + [self._api.named_type("builtins.str") for _ in generated_slots], + self._api.named_type("builtins.tuple"), + ) + add_attribute_to_class(self._api, self._cls, "__slots__", slots_type) + + def reset_init_only_vars(self, info: TypeInfo, attributes: list[DataclassAttribute]) -> None: + """Remove init-only vars from the class and reset init var declarations.""" + for attr in attributes: + if attr.is_init_var: + if attr.name in info.names: + del info.names[attr.name] + else: + # Nodes of superclass InitVars not used in __init__ cannot be reached. + assert attr.is_init_var + for stmt in info.defn.defs.body: + if isinstance(stmt, AssignmentStmt) and stmt.unanalyzed_type: + lvalue = stmt.lvalues[0] + if isinstance(lvalue, NameExpr) and lvalue.name == attr.name: + # Reset node so that another semantic analysis pass will + # recreate a symbol node for this attribute. + lvalue.node = None + + def _get_assignment_statements_from_if_statement( + self, stmt: IfStmt + ) -> Iterator[AssignmentStmt]: + for body in stmt.body: + if not body.is_unreachable: + yield from self._get_assignment_statements_from_block(body) + if stmt.else_body is not None and not stmt.else_body.is_unreachable: + yield from self._get_assignment_statements_from_block(stmt.else_body) + + def _get_assignment_statements_from_block(self, block: Block) -> Iterator[AssignmentStmt]: + for stmt in block.body: + if isinstance(stmt, AssignmentStmt): + yield stmt + elif isinstance(stmt, IfStmt): + yield from self._get_assignment_statements_from_if_statement(stmt) + + def collect_attributes(self) -> list[DataclassAttribute] | None: + """Collect all attributes declared in the dataclass and its parents. + + All assignments of the form + + a: SomeType + b: SomeOtherType = ... + + are collected. + + Return None if some dataclass base class hasn't been processed + yet and thus we'll need to ask for another pass. + """ + cls = self._cls + + # First, collect attributes belonging to any class in the MRO, ignoring duplicates. + # + # We iterate through the MRO in reverse because attrs defined in the parent must appear + # earlier in the attributes list than attrs defined in the child. See: + # https://docs.python.org/3/library/dataclasses.html#inheritance + # + # However, we also want attributes defined in the subtype to override ones defined + # in the parent. We can implement this via a dict without disrupting the attr order + # because dicts preserve insertion order in Python 3.7+. + found_attrs: dict[str, DataclassAttribute] = {} + found_dataclass_supertype = False + for info in reversed(cls.info.mro[1:-1]): + if "dataclass_tag" in info.metadata and "dataclass" not in info.metadata: + # We haven't processed the base class yet. Need another pass. + return None + if "dataclass" not in info.metadata: + continue + + # Each class depends on the set of attributes in its dataclass ancestors. + self._api.add_plugin_dependency(make_wildcard_trigger(info.fullname)) + found_dataclass_supertype = True + + for data in info.metadata["dataclass"]["attributes"]: + name: str = data["name"] + + attr = DataclassAttribute.deserialize(info, data, self._api) + # TODO: We shouldn't be performing type operations during the main + # semantic analysis pass, since some TypeInfo attributes might + # still be in flux. This should be performed in a later phase. + attr.expand_typevar_from_subtype(cls.info) + found_attrs[name] = attr + + sym_node = cls.info.names.get(name) + if sym_node and sym_node.node and not isinstance(sym_node.node, Var): + self._api.fail( + "Dataclass attribute may only be overridden by another attribute", + sym_node.node, + ) + + # Second, collect attributes belonging to the current class. + current_attr_names: set[str] = set() + kw_only = self._get_bool_arg("kw_only", self._spec.kw_only_default) + for stmt in self._get_assignment_statements_from_block(cls.defs): + # Any assignment that doesn't use the new type declaration + # syntax can be ignored out of hand. + if not stmt.new_syntax: + continue + + # a: int, b: str = 1, 'foo' is not supported syntax so we + # don't have to worry about it. + lhs = stmt.lvalues[0] + if not isinstance(lhs, NameExpr): + continue + + sym = cls.info.names.get(lhs.name) + if sym is None: + # There was probably a semantic analysis error. + continue + + node = sym.node + assert not isinstance(node, PlaceholderNode) + + if isinstance(node, TypeAlias): + self._api.fail( + ("Type aliases inside dataclass definitions are not supported at runtime"), + node, + ) + # Skip processing this node. This doesn't match the runtime behaviour, + # but the only alternative would be to modify the SymbolTable, + # and it's a little hairy to do that in a plugin. + continue + if isinstance(node, Decorator): + # This might be a property / field name clash. + # We will issue an error later. + continue + + assert isinstance(node, Var) + + # x: ClassVar[int] is ignored by dataclasses. + if node.is_classvar: + continue + + # x: InitVar[int] is turned into x: int and is removed from the class. + is_init_var = False + node_type = get_proper_type(node.type) + if ( + isinstance(node_type, Instance) + and node_type.type.fullname == "dataclasses.InitVar" + ): + is_init_var = True + node.type = node_type.args[0] + + if self._is_kw_only_type(node_type): + kw_only = True + + has_field_call, field_args = self._collect_field_args(stmt.rvalue) + + is_in_init_param = field_args.get("init") + if is_in_init_param is None: + is_in_init = self._get_default_init_value_for_field_specifier(stmt.rvalue) + else: + is_in_init = bool(self._api.parse_bool(is_in_init_param)) + + has_default = False + # Ensure that something like x: int = field() is rejected + # after an attribute with a default. + if has_field_call: + has_default = ( + "default" in field_args + or "default_factory" in field_args + # alias for default_factory defined in PEP 681 + or "factory" in field_args + ) + + # All other assignments are already type checked. + elif not isinstance(stmt.rvalue, TempNode): + has_default = True + + if not has_default and self._spec is _TRANSFORM_SPEC_FOR_DATACLASSES: + # Make all non-default dataclass attributes implicit because they are de-facto + # set on self in the generated __init__(), not in the class body. On the other + # hand, we don't know how custom dataclass transforms initialize attributes, + # so we don't treat them as implicit. This is required to support descriptors + # (https://github.com/python/mypy/issues/14868). + sym.implicit = True + + is_kw_only = kw_only + # Use the kw_only field arg if it is provided. Otherwise use the + # kw_only value from the decorator parameter. + field_kw_only_param = field_args.get("kw_only") + if field_kw_only_param is not None: + value = self._api.parse_bool(field_kw_only_param) + if value is not None: + is_kw_only = value + else: + self._api.fail('"kw_only" argument must be a boolean literal', stmt.rvalue) + + if sym.type is None and node.is_final and node.is_inferred: + # This is a special case, assignment like x: Final = 42 is classified + # annotated above, but mypy strips the `Final` turning it into x = 42. + # We do not support inferred types in dataclasses, so we can try inferring + # type for simple literals, and otherwise require an explicit type + # argument for Final[...]. + typ = self._api.analyze_simple_literal_type(stmt.rvalue, is_final=True) + if typ: + node.type = typ + else: + self._api.fail( + "Need type argument for Final[...] with non-literal default in dataclass", + stmt, + ) + node.type = AnyType(TypeOfAny.from_error) + + alias = None + if "alias" in field_args: + alias = self._api.parse_str_literal(field_args["alias"]) + if alias is None: + self._api.fail( + message_registry.DATACLASS_FIELD_ALIAS_MUST_BE_LITERAL, + stmt.rvalue, + code=errorcodes.LITERAL_REQ, + ) + + current_attr_names.add(lhs.name) + with state.strict_optional_set(self._api.options.strict_optional): + init_type = self._infer_dataclass_attr_init_type(sym, lhs.name, stmt) + found_attrs[lhs.name] = DataclassAttribute( + name=lhs.name, + alias=alias, + is_in_init=is_in_init, + is_init_var=is_init_var, + has_default=has_default, + line=stmt.line, + column=stmt.column, + type=init_type, + info=cls.info, + kw_only=is_kw_only, + is_neither_frozen_nor_nonfrozen=_has_direct_dataclass_transform_metaclass( + cls.info + ), + api=self._api, + ) + + all_attrs = list(found_attrs.values()) + if found_dataclass_supertype: + all_attrs.sort(key=lambda a: a.kw_only) + + # Third, ensure that arguments without a default don't follow + # arguments that have a default and that the KW_ONLY sentinel + # is only provided once. + found_default = False + found_kw_sentinel = False + for attr in all_attrs: + # If we find any attribute that is_in_init, not kw_only, and that + # doesn't have a default after one that does have one, + # then that's an error. + if found_default and attr.is_in_init and not attr.has_default and not attr.kw_only: + # If the issue comes from merging different classes, report it + # at the class definition point. + context: Context = cls + if attr.name in current_attr_names: + context = Context(line=attr.line, column=attr.column) + self._api.fail( + "Attributes without a default cannot follow attributes with one", context + ) + + found_default = found_default or (attr.has_default and attr.is_in_init) + if found_kw_sentinel and self._is_kw_only_type(attr.type): + context = cls + if attr.name in current_attr_names: + context = Context(line=attr.line, column=attr.column) + self._api.fail( + "There may not be more than one field with the KW_ONLY type", context + ) + found_kw_sentinel = found_kw_sentinel or self._is_kw_only_type(attr.type) + return all_attrs + + def _freeze(self, attributes: list[DataclassAttribute]) -> None: + """Converts all attributes to @property methods in order to + emulate frozen classes. + """ + info = self._cls.info + for attr in attributes: + # Classes that directly specify a dataclass_transform metaclass must be neither frozen + # non non-frozen per PEP681. Though it is surprising, this means that attributes from + # such a class must be writable even if the rest of the class hierarchy is frozen. This + # matches the behavior of Pyright (the reference implementation). + if attr.is_neither_frozen_nor_nonfrozen: + continue + + sym_node = info.names.get(attr.name) + if sym_node is not None: + var = sym_node.node + if isinstance(var, Var): + var.is_property = True + else: + var = attr.to_var(info) + var.info = info + var.is_property = True + var._fullname = info.fullname + "." + var.name + info.names[var.name] = SymbolTableNode(MDEF, var) + + def _propertize_callables( + self, attributes: list[DataclassAttribute], settable: bool = True + ) -> None: + """Converts all attributes with callable types to @property methods. + + This avoids the typechecker getting confused and thinking that + `my_dataclass_instance.callable_attr(foo)` is going to receive a + `self` argument (it is not). + + """ + info = self._cls.info + for attr in attributes: + if isinstance(get_proper_type(attr.type), CallableType): + var = attr.to_var(info) + var.info = info + var.is_property = True + var.is_settable_property = settable + var._fullname = info.fullname + "." + var.name + info.names[var.name] = SymbolTableNode(MDEF, var) + + def _is_kw_only_type(self, node: Type | None) -> bool: + """Checks if the type of the node is the KW_ONLY sentinel value.""" + if node is None: + return False + node_type = get_proper_type(node) + if not isinstance(node_type, Instance): + return False + return node_type.type.fullname == "dataclasses.KW_ONLY" + + def _add_dataclass_fields_magic_attribute(self) -> None: + attr_name = "__dataclass_fields__" + any_type = AnyType(TypeOfAny.explicit) + # For `dataclasses`, use the type `dict[str, Field[Any]]` for accuracy. For dataclass + # transforms, it's inaccurate to use `Field` since a given transform may use a completely + # different type (or none); fall back to `Any` there. + # + # In either case, we're aiming to match the Typeshed stub for `is_dataclass`, which expects + # the instance to have a `__dataclass_fields__` attribute of type `dict[str, Field[Any]]`. + if self._spec is _TRANSFORM_SPEC_FOR_DATACLASSES: + field_type = self._api.named_type_or_none("dataclasses.Field", [any_type]) or any_type + else: + field_type = any_type + attr_type = self._api.named_type( + "builtins.dict", [self._api.named_type("builtins.str"), field_type] + ) + var = Var(name=attr_name, type=attr_type) + var.info = self._cls.info + var._fullname = self._cls.info.fullname + "." + attr_name + var.is_classvar = True + self._cls.info.names[attr_name] = SymbolTableNode( + kind=MDEF, node=var, plugin_generated=True + ) + + def _collect_field_args(self, expr: Expression) -> tuple[bool, dict[str, Expression]]: + """Returns a tuple where the first value represents whether or not + the expression is a call to dataclass.field and the second is a + dictionary of the keyword arguments that field() was called with. + """ + if ( + isinstance(expr, CallExpr) + and isinstance(expr.callee, RefExpr) + and expr.callee.fullname in self._spec.field_specifiers + ): + # field() only takes keyword arguments. + args = {} + for name, arg, kind in zip(expr.arg_names, expr.args, expr.arg_kinds): + if not kind.is_named(): + if kind.is_named(star=True): + # This means that `field` is used with `**` unpacking, + # the best we can do for now is not to fail. + # TODO: we can infer what's inside `**` and try to collect it. + message = 'Unpacking **kwargs in "field()" is not supported' + elif self._spec is not _TRANSFORM_SPEC_FOR_DATACLASSES: + # dataclasses.field can only be used with keyword args, but this + # restriction is only enforced for the *standardized* arguments to + # dataclass_transform field specifiers. If this is not a + # dataclasses.dataclass class, we can just skip positional args safely. + continue + else: + message = '"field()" does not accept positional arguments' + self._api.fail(message, expr) + return True, {} + assert name is not None + args[name] = arg + return True, args + return False, {} + + def _get_bool_arg(self, name: str, default: bool) -> bool: + # Expressions are always CallExprs (either directly or via a wrapper like Decorator), so + # we can use the helpers from common + if isinstance(self._reason, Expression): + return _get_decorator_bool_argument( + ClassDefContext(self._cls, self._reason, self._api), name, default + ) + + # Subclass/metaclass use of `typing.dataclass_transform` reads the parameters from the + # class's keyword arguments (ie `class Subclass(Parent, kwarg1=..., kwarg2=...)`) + expression = self._cls.keywords.get(name) + if expression is not None: + return require_bool_literal_argument(self._api, expression, name, default) + return default + + def _get_default_init_value_for_field_specifier(self, call: Expression) -> bool: + """ + Find a default value for the `init` parameter of the specifier being called. If the + specifier's type signature includes an `init` parameter with a type of `Literal[True]` or + `Literal[False]`, return the appropriate boolean value from the literal. Otherwise, + fall back to the standard default of `True`. + """ + if not isinstance(call, CallExpr): + return True + + specifier_type = _get_callee_type(call) + if specifier_type is None: + return True + + parameter = specifier_type.argument_by_name("init") + if parameter is None: + return True + + literals = try_getting_literals_from_type(parameter.typ, bool, "builtins.bool") + if literals is None or len(literals) != 1: + return True + + return literals[0] + + def _infer_dataclass_attr_init_type( + self, sym: SymbolTableNode, name: str, context: Context + ) -> Type | None: + """Infer __init__ argument type for an attribute. + + In particular, possibly use the signature of __set__. + """ + default = sym.type + if sym.implicit: + return default + t = get_proper_type(sym.type) + + # Perform a simple-minded inference from the signature of __set__, if present. + # We can't use mypy.checkmember here, since this plugin runs before type checking. + # We only support some basic scanerios here, which is hopefully sufficient for + # the vast majority of use cases. + if not isinstance(t, Instance): + return default + setter = t.type.get("__set__") + if setter: + if isinstance(setter.node, FuncDef): + super_info = t.type.get_containing_type_info("__set__") + assert super_info + if setter.type: + setter_type = get_proper_type( + map_type_from_supertype(setter.type, t.type, super_info) + ) + else: + return AnyType(TypeOfAny.unannotated) + if isinstance(setter_type, CallableType) and setter_type.arg_kinds == [ + ARG_POS, + ARG_POS, + ARG_POS, + ]: + return expand_type_by_instance(setter_type.arg_types[2], t) + else: + self._api.fail( + f'Unsupported signature for "__set__" in "{t.type.name}"', context + ) + else: + self._api.fail(f'Unsupported "__set__" in "{t.type.name}"', context) + + return default + + +def add_dataclass_tag(info: TypeInfo) -> None: + # The value is ignored, only the existence matters. + info.metadata["dataclass_tag"] = {} + + +def dataclass_tag_callback(ctx: ClassDefContext) -> None: + """Record that we have a dataclass in the main semantic analysis pass. + + The later pass implemented by DataclassTransformer will use this + to detect dataclasses in base classes. + """ + add_dataclass_tag(ctx.cls.info) + + +def dataclass_class_maker_callback(ctx: ClassDefContext) -> bool: + """Hooks into the class typechecking process to add support for dataclasses.""" + transformer = DataclassTransformer( + ctx.cls, ctx.reason, _get_transform_spec(ctx.reason), ctx.api + ) + return transformer.transform() + + +def _get_transform_spec(reason: Expression) -> DataclassTransformSpec: + """Find the relevant transform parameters from the decorator/parent class/metaclass that + triggered the dataclasses plugin. + + Although the resulting DataclassTransformSpec is based on the typing.dataclass_transform + function, we also use it for traditional dataclasses.dataclass classes as well for simplicity. + In those cases, we return a default spec rather than one based on a call to + `typing.dataclass_transform`. + """ + if _is_dataclasses_decorator(reason): + return _TRANSFORM_SPEC_FOR_DATACLASSES + + spec = find_dataclass_transform_spec(reason) + assert spec is not None, ( + "trying to find dataclass transform spec, but reason is neither dataclasses.dataclass nor " + "decorated with typing.dataclass_transform" + ) + return spec + + +def _is_dataclasses_decorator(node: Node) -> bool: + if isinstance(node, CallExpr): + node = node.callee + if isinstance(node, RefExpr): + return node.fullname in dataclass_makers + return False + + +def _has_direct_dataclass_transform_metaclass(info: TypeInfo) -> bool: + return ( + info.declared_metaclass is not None + and info.declared_metaclass.type.dataclass_transform_spec is not None + ) + + +def _get_expanded_dataclasses_fields( + ctx: FunctionSigContext, typ: ProperType, display_typ: ProperType, parent_typ: ProperType +) -> list[CallableType] | None: + """ + For a given type, determine what dataclasses it can be: for each class, return the field types. + For generic classes, the field types are expanded. + If the type contains Any or a non-dataclass, returns None; in the latter case, also reports an error. + """ + if isinstance(typ, UnionType): + ret: list[CallableType] | None = [] + for item in typ.relevant_items(): + item = get_proper_type(item) + item_types = _get_expanded_dataclasses_fields(ctx, item, item, parent_typ) + if ret is not None and item_types is not None: + ret += item_types + else: + ret = None # but keep iterating to emit all errors + return ret + elif isinstance(typ, TypeVarType): + return _get_expanded_dataclasses_fields( + ctx, get_proper_type(typ.upper_bound), display_typ, parent_typ + ) + elif isinstance(typ, Instance): + replace_sym = typ.type.get_method(_INTERNAL_REPLACE_SYM_NAME) + if replace_sym is None: + return None + replace_sig = replace_sym.type + assert isinstance(replace_sig, ProperType) + assert isinstance(replace_sig, CallableType) + return [expand_type_by_instance(replace_sig, typ)] + else: + return None + + +# TODO: we can potentially get the function signature hook to allow returning a union +# and leave this to the regular machinery of resolving a union of callables +# (https://github.com/python/mypy/issues/15457) +def _meet_replace_sigs(sigs: list[CallableType]) -> CallableType: + """ + Produces the lowest bound of the 'replace' signatures of multiple dataclasses. + """ + args = { + name: (typ, kind) + for name, typ, kind in zip(sigs[0].arg_names, sigs[0].arg_types, sigs[0].arg_kinds) + } + + for sig in sigs[1:]: + sig_args = { + name: (typ, kind) + for name, typ, kind in zip(sig.arg_names, sig.arg_types, sig.arg_kinds) + } + for name in (*args.keys(), *sig_args.keys()): + sig_typ, sig_kind = args.get(name, (UninhabitedType(), ARG_NAMED_OPT)) + sig2_typ, sig2_kind = sig_args.get(name, (UninhabitedType(), ARG_NAMED_OPT)) + args[name] = ( + meet_types(sig_typ, sig2_typ), + ARG_NAMED_OPT if sig_kind == sig2_kind == ARG_NAMED_OPT else ARG_NAMED, + ) + + return sigs[0].copy_modified( + arg_names=list(args.keys()), + arg_types=[typ for typ, _ in args.values()], + arg_kinds=[kind for _, kind in args.values()], + ) + + +def replace_function_sig_callback(ctx: FunctionSigContext) -> CallableType: + """ + Returns a signature for the 'dataclasses.replace' function that's dependent on the type + of the first positional argument. + """ + if len(ctx.args) != 2: + # Ideally the name and context should be callee's, but we don't have it in FunctionSigContext. + ctx.api.fail(f'"{ctx.default_signature.name}" has unexpected type annotation', ctx.context) + return ctx.default_signature + + if len(ctx.args[0]) != 1: + return ctx.default_signature # leave it to the type checker to complain + + obj_arg = ctx.args[0][0] + obj_type = get_proper_type(ctx.api.get_expression_type(obj_arg)) + inst_type_str = format_type_bare(obj_type, ctx.api.options) + + replace_sigs = _get_expanded_dataclasses_fields(ctx, obj_type, obj_type, obj_type) + if replace_sigs is None: + return ctx.default_signature + replace_sig = _meet_replace_sigs(replace_sigs) + + return replace_sig.copy_modified( + arg_names=[None, *replace_sig.arg_names], + arg_kinds=[ARG_POS, *replace_sig.arg_kinds], + arg_types=[obj_type, *replace_sig.arg_types], + ret_type=obj_type, + fallback=ctx.default_signature.fallback, + name=f"{ctx.default_signature.name} of {inst_type_str}", + ) + + +def is_processed_dataclass(info: TypeInfo) -> bool: + return bool(info) and "dataclass" in info.metadata + + +def check_post_init(api: TypeChecker, defn: FuncItem, info: TypeInfo) -> None: + if defn.type is None: + return + assert isinstance(defn.type, FunctionLike) + + ideal_sig_method = info.get_method(_INTERNAL_POST_INIT_SYM_NAME) + assert ideal_sig_method is not None and ideal_sig_method.type is not None + ideal_sig = ideal_sig_method.type + assert isinstance(ideal_sig, ProperType) # we set it ourselves + assert isinstance(ideal_sig, CallableType) + ideal_sig = ideal_sig.copy_modified(name="__post_init__") + + api.check_override( + override=defn.type, + original=ideal_sig, + name="__post_init__", + name_in_super="__post_init__", + supertype="dataclass", + original_class_or_static=False, + override_class_or_static=False, + node=defn, + ) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py new file mode 100644 index 000000000000..73c5742614ee --- /dev/null +++ b/mypy/plugins/default.py @@ -0,0 +1,558 @@ +from __future__ import annotations + +from functools import partial +from typing import Callable, Final + +import mypy.errorcodes as codes +from mypy import message_registry +from mypy.nodes import DictExpr, IntExpr, StrExpr, UnaryExpr +from mypy.plugin import ( + AttributeContext, + ClassDefContext, + FunctionContext, + FunctionSigContext, + MethodContext, + MethodSigContext, + Plugin, +) +from mypy.plugins.common import try_getting_str_literals +from mypy.subtypes import is_subtype +from mypy.typeops import is_literal_type_like, make_simplified_union +from mypy.types import ( + TPDICT_FB_NAMES, + AnyType, + CallableType, + FunctionLike, + Instance, + LiteralType, + NoneType, + TupleType, + Type, + TypedDictType, + TypeOfAny, + TypeVarType, + UnionType, + get_proper_type, + get_proper_types, +) + + +class DefaultPlugin(Plugin): + """Type checker plugin that is enabled by default.""" + + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + from mypy.plugins import ctypes, enums, singledispatch + + if fullname == "_ctypes.Array": + return ctypes.array_constructor_callback + elif fullname == "functools.singledispatch": + return singledispatch.create_singledispatch_function_callback + elif fullname == "functools.partial": + import mypy.plugins.functools + + return mypy.plugins.functools.partial_new_callback + elif fullname == "enum.member": + return enums.enum_member_callback + + return None + + def get_function_signature_hook( + self, fullname: str + ) -> Callable[[FunctionSigContext], FunctionLike] | None: + from mypy.plugins import attrs, dataclasses + + if fullname in ("attr.evolve", "attrs.evolve", "attr.assoc", "attrs.assoc"): + return attrs.evolve_function_sig_callback + elif fullname in ("attr.fields", "attrs.fields"): + return attrs.fields_function_sig_callback + elif fullname == "dataclasses.replace": + return dataclasses.replace_function_sig_callback + return None + + def get_method_signature_hook( + self, fullname: str + ) -> Callable[[MethodSigContext], FunctionLike] | None: + from mypy.plugins import ctypes, singledispatch + + if fullname == "typing.Mapping.get": + return typed_dict_get_signature_callback + elif fullname in {n + ".setdefault" for n in TPDICT_FB_NAMES}: + return typed_dict_setdefault_signature_callback + elif fullname in {n + ".pop" for n in TPDICT_FB_NAMES}: + return typed_dict_pop_signature_callback + elif fullname == "_ctypes.Array.__setitem__": + return ctypes.array_setitem_callback + elif fullname == singledispatch.SINGLEDISPATCH_CALLABLE_CALL_METHOD: + return singledispatch.call_singledispatch_function_callback + + typed_dict_updates = set() + for n in TPDICT_FB_NAMES: + typed_dict_updates.add(n + ".update") + typed_dict_updates.add(n + ".__or__") + typed_dict_updates.add(n + ".__ror__") + typed_dict_updates.add(n + ".__ior__") + + if fullname in typed_dict_updates: + return typed_dict_update_signature_callback + + return None + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + from mypy.plugins import ctypes, singledispatch + + if fullname == "typing.Mapping.get": + return typed_dict_get_callback + elif fullname == "builtins.int.__pow__": + return int_pow_callback + elif fullname == "builtins.int.__neg__": + return int_neg_callback + elif fullname == "builtins.int.__pos__": + return int_pos_callback + elif fullname in ("builtins.tuple.__mul__", "builtins.tuple.__rmul__"): + return tuple_mul_callback + elif fullname in {n + ".setdefault" for n in TPDICT_FB_NAMES}: + return typed_dict_setdefault_callback + elif fullname in {n + ".pop" for n in TPDICT_FB_NAMES}: + return typed_dict_pop_callback + elif fullname in {n + ".__delitem__" for n in TPDICT_FB_NAMES}: + return typed_dict_delitem_callback + elif fullname == "_ctypes.Array.__getitem__": + return ctypes.array_getitem_callback + elif fullname == "_ctypes.Array.__iter__": + return ctypes.array_iter_callback + elif fullname == singledispatch.SINGLEDISPATCH_REGISTER_METHOD: + return singledispatch.singledispatch_register_callback + elif fullname == singledispatch.REGISTER_CALLABLE_CALL_METHOD: + return singledispatch.call_singledispatch_function_after_register_argument + elif fullname == "functools.partial.__call__": + import mypy.plugins.functools + + return mypy.plugins.functools.partial_call_callback + return None + + def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: + from mypy.plugins import ctypes, enums + + if fullname == "_ctypes.Array.value": + return ctypes.array_value_callback + elif fullname == "_ctypes.Array.raw": + return ctypes.array_raw_callback + elif fullname in enums.ENUM_NAME_ACCESS: + return enums.enum_name_callback + elif fullname in enums.ENUM_VALUE_ACCESS: + return enums.enum_value_callback + return None + + def get_class_decorator_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + from mypy.plugins import attrs, dataclasses + + # These dataclass and attrs hooks run in the main semantic analysis pass + # and only tag known dataclasses/attrs classes, so that the second + # hooks (in get_class_decorator_hook_2) can detect dataclasses/attrs classes + # in the MRO. + if fullname in dataclasses.dataclass_makers: + return dataclasses.dataclass_tag_callback + if ( + fullname in attrs.attr_class_makers + or fullname in attrs.attr_dataclass_makers + or fullname in attrs.attr_frozen_makers + or fullname in attrs.attr_define_makers + ): + return attrs.attr_tag_callback + + return None + + def get_class_decorator_hook_2( + self, fullname: str + ) -> Callable[[ClassDefContext], bool] | None: + import mypy.plugins.functools + from mypy.plugins import attrs, dataclasses + + if fullname in dataclasses.dataclass_makers: + return dataclasses.dataclass_class_maker_callback + elif fullname in mypy.plugins.functools.functools_total_ordering_makers: + return mypy.plugins.functools.functools_total_ordering_maker_callback + elif fullname in attrs.attr_class_makers: + return attrs.attr_class_maker_callback + elif fullname in attrs.attr_dataclass_makers: + return partial(attrs.attr_class_maker_callback, auto_attribs_default=True) + elif fullname in attrs.attr_frozen_makers: + return partial( + attrs.attr_class_maker_callback, auto_attribs_default=None, frozen_default=True + ) + elif fullname in attrs.attr_define_makers: + return partial( + attrs.attr_class_maker_callback, auto_attribs_default=None, slots_default=True + ) + + return None + + +def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: + """Try to infer a better signature type for TypedDict.get. + + This is used to get better type context for the second argument that + depends on a TypedDict value type. + """ + signature = ctx.default_signature + if ( + isinstance(ctx.type, TypedDictType) + and len(ctx.args) == 2 + and len(ctx.args[0]) == 1 + and isinstance(ctx.args[0][0], StrExpr) + and len(signature.arg_types) == 2 + and len(signature.variables) == 1 + and len(ctx.args[1]) == 1 + ): + key = ctx.args[0][0].value + value_type = get_proper_type(ctx.type.items.get(key)) + ret_type = signature.ret_type + if value_type: + default_arg = ctx.args[1][0] + if ( + isinstance(value_type, TypedDictType) + and isinstance(default_arg, DictExpr) + and len(default_arg.items) == 0 + ): + # Caller has empty dict {} as default for typed dict. + value_type = value_type.copy_modified(required_keys=set()) + # Tweak the signature to include the value type as context. It's + # only needed for type inference since there's a union with a type + # variable that accepts everything. + tv = signature.variables[0] + assert isinstance(tv, TypeVarType) + return signature.copy_modified( + arg_types=[signature.arg_types[0], make_simplified_union([value_type, tv])], + ret_type=ret_type, + ) + return signature + + +def typed_dict_get_callback(ctx: MethodContext) -> Type: + """Infer a precise return type for TypedDict.get with literal first argument.""" + if ( + isinstance(ctx.type, TypedDictType) + and len(ctx.arg_types) >= 1 + and len(ctx.arg_types[0]) == 1 + ): + keys = try_getting_str_literals(ctx.args[0][0], ctx.arg_types[0][0]) + if keys is None: + return ctx.default_return_type + + output_types: list[Type] = [] + for key in keys: + value_type = get_proper_type(ctx.type.items.get(key)) + if value_type is None: + return ctx.default_return_type + + if len(ctx.arg_types) == 1: + output_types.append(value_type) + elif len(ctx.arg_types) == 2 and len(ctx.arg_types[1]) == 1 and len(ctx.args[1]) == 1: + default_arg = ctx.args[1][0] + if ( + isinstance(default_arg, DictExpr) + and len(default_arg.items) == 0 + and isinstance(value_type, TypedDictType) + ): + # Special case '{}' as the default for a typed dict type. + output_types.append(value_type.copy_modified(required_keys=set())) + else: + output_types.append(value_type) + output_types.append(ctx.arg_types[1][0]) + + if len(ctx.arg_types) == 1: + output_types.append(NoneType()) + + return make_simplified_union(output_types) + return ctx.default_return_type + + +def typed_dict_pop_signature_callback(ctx: MethodSigContext) -> CallableType: + """Try to infer a better signature type for TypedDict.pop. + + This is used to get better type context for the second argument that + depends on a TypedDict value type. + """ + signature = ctx.default_signature + str_type = ctx.api.named_generic_type("builtins.str", []) + if ( + isinstance(ctx.type, TypedDictType) + and len(ctx.args) == 2 + and len(ctx.args[0]) == 1 + and isinstance(ctx.args[0][0], StrExpr) + and len(signature.arg_types) == 2 + and len(signature.variables) == 1 + and len(ctx.args[1]) == 1 + ): + key = ctx.args[0][0].value + value_type = ctx.type.items.get(key) + if value_type: + # Tweak the signature to include the value type as context. It's + # only needed for type inference since there's a union with a type + # variable that accepts everything. + tv = signature.variables[0] + assert isinstance(tv, TypeVarType) + typ = make_simplified_union([value_type, tv]) + return signature.copy_modified(arg_types=[str_type, typ], ret_type=typ) + return signature.copy_modified(arg_types=[str_type, signature.arg_types[1]]) + + +def typed_dict_pop_callback(ctx: MethodContext) -> Type: + """Type check and infer a precise return type for TypedDict.pop.""" + if ( + isinstance(ctx.type, TypedDictType) + and len(ctx.arg_types) >= 1 + and len(ctx.arg_types[0]) == 1 + ): + keys = try_getting_str_literals(ctx.args[0][0], ctx.arg_types[0][0]) + if keys is None: + ctx.api.fail( + message_registry.TYPEDDICT_KEY_MUST_BE_STRING_LITERAL, + ctx.context, + code=codes.LITERAL_REQ, + ) + return AnyType(TypeOfAny.from_error) + + value_types = [] + for key in keys: + if key in ctx.type.required_keys: + ctx.api.msg.typeddict_key_cannot_be_deleted(ctx.type, key, ctx.context) + + value_type = ctx.type.items.get(key) + if value_type: + value_types.append(value_type) + else: + ctx.api.msg.typeddict_key_not_found(ctx.type, key, ctx.context) + return AnyType(TypeOfAny.from_error) + + if len(ctx.args[1]) == 0: + return make_simplified_union(value_types) + elif len(ctx.arg_types) == 2 and len(ctx.arg_types[1]) == 1 and len(ctx.args[1]) == 1: + return make_simplified_union([*value_types, ctx.arg_types[1][0]]) + return ctx.default_return_type + + +def typed_dict_setdefault_signature_callback(ctx: MethodSigContext) -> CallableType: + """Try to infer a better signature type for TypedDict.setdefault. + + This is used to get better type context for the second argument that + depends on a TypedDict value type. + """ + signature = ctx.default_signature + str_type = ctx.api.named_generic_type("builtins.str", []) + if ( + isinstance(ctx.type, TypedDictType) + and len(ctx.args) == 2 + and len(ctx.args[0]) == 1 + and isinstance(ctx.args[0][0], StrExpr) + and len(signature.arg_types) == 2 + and len(ctx.args[1]) == 1 + ): + key = ctx.args[0][0].value + value_type = ctx.type.items.get(key) + if value_type: + return signature.copy_modified(arg_types=[str_type, value_type]) + return signature.copy_modified(arg_types=[str_type, signature.arg_types[1]]) + + +def typed_dict_setdefault_callback(ctx: MethodContext) -> Type: + """Type check TypedDict.setdefault and infer a precise return type.""" + if ( + isinstance(ctx.type, TypedDictType) + and len(ctx.arg_types) == 2 + and len(ctx.arg_types[0]) == 1 + and len(ctx.arg_types[1]) == 1 + ): + keys = try_getting_str_literals(ctx.args[0][0], ctx.arg_types[0][0]) + if keys is None: + ctx.api.fail( + message_registry.TYPEDDICT_KEY_MUST_BE_STRING_LITERAL, + ctx.context, + code=codes.LITERAL_REQ, + ) + return AnyType(TypeOfAny.from_error) + + assigned_readonly_keys = ctx.type.readonly_keys & set(keys) + if assigned_readonly_keys: + ctx.api.msg.readonly_keys_mutated(assigned_readonly_keys, context=ctx.context) + + default_type = ctx.arg_types[1][0] + + value_types = [] + for key in keys: + value_type = ctx.type.items.get(key) + + if value_type is None: + ctx.api.msg.typeddict_key_not_found(ctx.type, key, ctx.context) + return AnyType(TypeOfAny.from_error) + + # The signature_callback above can't always infer the right signature + # (e.g. when the expression is a variable that happens to be a Literal str) + # so we need to handle the check ourselves here and make sure the provided + # default can be assigned to all key-value pairs we're updating. + if not is_subtype(default_type, value_type): + ctx.api.msg.typeddict_setdefault_arguments_inconsistent( + default_type, value_type, ctx.context + ) + return AnyType(TypeOfAny.from_error) + + value_types.append(value_type) + + return make_simplified_union(value_types) + return ctx.default_return_type + + +def typed_dict_delitem_callback(ctx: MethodContext) -> Type: + """Type check TypedDict.__delitem__.""" + if ( + isinstance(ctx.type, TypedDictType) + and len(ctx.arg_types) == 1 + and len(ctx.arg_types[0]) == 1 + ): + keys = try_getting_str_literals(ctx.args[0][0], ctx.arg_types[0][0]) + if keys is None: + ctx.api.fail( + message_registry.TYPEDDICT_KEY_MUST_BE_STRING_LITERAL, + ctx.context, + code=codes.LITERAL_REQ, + ) + return AnyType(TypeOfAny.from_error) + + for key in keys: + if key in ctx.type.required_keys or key in ctx.type.readonly_keys: + ctx.api.msg.typeddict_key_cannot_be_deleted(ctx.type, key, ctx.context) + elif key not in ctx.type.items: + ctx.api.msg.typeddict_key_not_found(ctx.type, key, ctx.context) + return ctx.default_return_type + + +_TP_DICT_MUTATING_METHODS: Final = frozenset({"update of TypedDict", "__ior__ of TypedDict"}) + + +def typed_dict_update_signature_callback(ctx: MethodSigContext) -> CallableType: + """Try to infer a better signature type for methods that update `TypedDict`. + + This includes: `TypedDict.update`, `TypedDict.__or__`, `TypedDict.__ror__`, + and `TypedDict.__ior__`. + """ + signature = ctx.default_signature + if isinstance(ctx.type, TypedDictType) and len(signature.arg_types) == 1: + arg_type = get_proper_type(signature.arg_types[0]) + if not isinstance(arg_type, TypedDictType): + return signature + arg_type = arg_type.as_anonymous() + arg_type = arg_type.copy_modified(required_keys=set()) + if ctx.args and ctx.args[0]: + if signature.name in _TP_DICT_MUTATING_METHODS: + # If we want to mutate this object in place, we need to set this flag, + # it will trigger an extra check in TypedDict's checker. + arg_type.to_be_mutated = True + with ctx.api.msg.filter_errors( + filter_errors=lambda name, info: info.code != codes.TYPEDDICT_READONLY_MUTATED, + save_filtered_errors=True, + ): + inferred = get_proper_type( + ctx.api.get_expression_type(ctx.args[0][0], type_context=arg_type) + ) + if arg_type.to_be_mutated: + arg_type.to_be_mutated = False # Done! + possible_tds = [] + if isinstance(inferred, TypedDictType): + possible_tds = [inferred] + elif isinstance(inferred, UnionType): + possible_tds = [ + t + for t in get_proper_types(inferred.relevant_items()) + if isinstance(t, TypedDictType) + ] + items = [] + for td in possible_tds: + item = arg_type.copy_modified( + required_keys=(arg_type.required_keys | td.required_keys) + & arg_type.items.keys() + ) + if not ctx.api.options.extra_checks: + item = item.copy_modified(item_names=list(td.items)) + items.append(item) + if items: + arg_type = make_simplified_union(items) + return signature.copy_modified(arg_types=[arg_type]) + return signature + + +def int_pow_callback(ctx: MethodContext) -> Type: + """Infer a more precise return type for int.__pow__.""" + # int.__pow__ has an optional modulo argument, + # so we expect 2 argument positions + if len(ctx.arg_types) == 2 and len(ctx.arg_types[0]) == 1 and len(ctx.arg_types[1]) == 0: + arg = ctx.args[0][0] + if isinstance(arg, IntExpr): + exponent = arg.value + elif isinstance(arg, UnaryExpr) and arg.op == "-" and isinstance(arg.expr, IntExpr): + exponent = -arg.expr.value + else: + # Right operand not an int literal or a negated literal -- give up. + return ctx.default_return_type + if exponent >= 0: + return ctx.api.named_generic_type("builtins.int", []) + else: + return ctx.api.named_generic_type("builtins.float", []) + return ctx.default_return_type + + +def int_neg_callback(ctx: MethodContext, multiplier: int = -1) -> Type: + """Infer a more precise return type for int.__neg__ and int.__pos__. + + This is mainly used to infer the return type as LiteralType + if the original underlying object is a LiteralType object. + """ + if isinstance(ctx.type, Instance) and ctx.type.last_known_value is not None: + value = ctx.type.last_known_value.value + fallback = ctx.type.last_known_value.fallback + if isinstance(value, int): + if is_literal_type_like(ctx.api.type_context[-1]): + return LiteralType(value=multiplier * value, fallback=fallback) + else: + return ctx.type.copy_modified( + last_known_value=LiteralType( + value=multiplier * value, + fallback=fallback, + line=ctx.type.line, + column=ctx.type.column, + ) + ) + elif isinstance(ctx.type, LiteralType): + value = ctx.type.value + fallback = ctx.type.fallback + if isinstance(value, int): + return LiteralType(value=multiplier * value, fallback=fallback) + return ctx.default_return_type + + +def int_pos_callback(ctx: MethodContext) -> Type: + """Infer a more precise return type for int.__pos__. + + This is identical to __neg__, except the value is not inverted. + """ + return int_neg_callback(ctx, +1) + + +def tuple_mul_callback(ctx: MethodContext) -> Type: + """Infer a more precise return type for tuple.__mul__ and tuple.__rmul__. + + This is used to return a specific sized tuple if multiplied by Literal int + """ + if not isinstance(ctx.type, TupleType): + return ctx.default_return_type + + arg_type = get_proper_type(ctx.arg_types[0][0]) + if isinstance(arg_type, Instance) and arg_type.last_known_value is not None: + value = arg_type.last_known_value.value + if isinstance(value, int): + return ctx.type.copy_modified(items=ctx.type.items * value) + elif isinstance(ctx.type, LiteralType): + value = arg_type.value + if isinstance(value, int): + return ctx.type.copy_modified(items=ctx.type.items * value) + + return ctx.default_return_type diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py new file mode 100644 index 000000000000..816241fa6e9a --- /dev/null +++ b/mypy/plugins/enums.py @@ -0,0 +1,286 @@ +""" +This file contains a variety of plugins for refining how mypy infers types of +expressions involving Enums. + +Currently, this file focuses on providing better inference for expressions like +'SomeEnum.FOO.name' and 'SomeEnum.FOO.value'. Note that the type of both expressions +will vary depending on exactly which instance of SomeEnum we're looking at. + +Note that this file does *not* contain all special-cased logic related to enums: +we actually bake some of it directly in to the semantic analysis layer (see +semanal_enum.py). +""" + +from __future__ import annotations + +from typing import Final, Iterable, Sequence, TypeVar, cast + +import mypy.plugin # To avoid circular imports. +from mypy.nodes import TypeInfo +from mypy.semanal_enum import ENUM_BASES +from mypy.subtypes import is_equivalent +from mypy.typeops import fixup_partial_type, make_simplified_union +from mypy.types import ( + CallableType, + Instance, + LiteralType, + ProperType, + Type, + get_proper_type, + is_named_instance, +) + +ENUM_NAME_ACCESS: Final = {f"{prefix}.name" for prefix in ENUM_BASES} | { + f"{prefix}._name_" for prefix in ENUM_BASES +} +ENUM_VALUE_ACCESS: Final = {f"{prefix}.value" for prefix in ENUM_BASES} | { + f"{prefix}._value_" for prefix in ENUM_BASES +} + + +def enum_name_callback(ctx: mypy.plugin.AttributeContext) -> Type: + """This plugin refines the 'name' attribute in enums to act as if + they were declared to be final. + + For example, the expression 'MyEnum.FOO.name' normally is inferred + to be of type 'str'. + + This plugin will instead make the inferred type be a 'str' where the + last known value is 'Literal["FOO"]'. This means it would be legal to + use 'MyEnum.FOO.name' in contexts that expect a Literal type, just like + any other Final variable or attribute. + + This plugin assumes that the provided context is an attribute access + matching one of the strings found in 'ENUM_NAME_ACCESS'. + """ + enum_field_name = _extract_underlying_field_name(ctx.type) + if enum_field_name is None: + return ctx.default_attr_type + else: + str_type = ctx.api.named_generic_type("builtins.str", []) + literal_type = LiteralType(enum_field_name, fallback=str_type) + return str_type.copy_modified(last_known_value=literal_type) + + +_T = TypeVar("_T") + + +def _first(it: Iterable[_T]) -> _T | None: + """Return the first value from any iterable. + + Returns ``None`` if the iterable is empty. + """ + for val in it: + return val + return None + + +def _infer_value_type_with_auto_fallback( + ctx: mypy.plugin.AttributeContext, proper_type: ProperType | None +) -> Type | None: + """Figure out the type of an enum value accounting for `auto()`. + + This method is a no-op for a `None` proper_type and also in the case where + the type is not "enum.auto" + """ + if proper_type is None: + return None + proper_type = get_proper_type(fixup_partial_type(proper_type)) + if not (isinstance(proper_type, Instance) and proper_type.type.fullname == "enum.auto"): + if is_named_instance(proper_type, "enum.member") and proper_type.args: + return proper_type.args[0] + return proper_type + assert isinstance(ctx.type, Instance), "An incorrect ctx.type was passed." + info = ctx.type.type + # Find the first _generate_next_value_ on the mro. We need to know + # if it is `Enum` because `Enum` types say that the return-value of + # `_generate_next_value_` is `Any`. In reality the default `auto()` + # returns an `int` (presumably the `Any` in typeshed is to make it + # easier to subclass and change the returned type). + type_with_gnv = _first(ti for ti in info.mro if ti.names.get("_generate_next_value_")) + if type_with_gnv is None: + return ctx.default_attr_type + + stnode = type_with_gnv.names["_generate_next_value_"] + + # This should be a `CallableType` + node_type = get_proper_type(stnode.type) + if isinstance(node_type, CallableType): + if type_with_gnv.fullname == "enum.Enum": + int_type = ctx.api.named_generic_type("builtins.int", []) + return int_type + return get_proper_type(node_type.ret_type) + return ctx.default_attr_type + + +def _implements_new(info: TypeInfo) -> bool: + """Check whether __new__ comes from enum.Enum or was implemented in a + subclass. In the latter case, we must infer Any as long as mypy can't infer + the type of _value_ from assignments in __new__. + """ + type_with_new = _first( + ti + for ti in info.mro + if ti.names.get("__new__") and not ti.fullname.startswith("builtins.") + ) + if type_with_new is None: + return False + return type_with_new.fullname not in ("enum.Enum", "enum.IntEnum", "enum.StrEnum") + + +def enum_member_callback(ctx: mypy.plugin.FunctionContext) -> Type: + """By default `member(1)` will be infered as `member[int]`, + we want to improve the inference to be `Literal[1]` here.""" + if ctx.arg_types or ctx.arg_types[0]: + arg = get_proper_type(ctx.arg_types[0][0]) + proper_return = get_proper_type(ctx.default_return_type) + if ( + isinstance(arg, Instance) + and arg.last_known_value + and isinstance(proper_return, Instance) + and len(proper_return.args) == 1 + ): + return proper_return.copy_modified(args=[arg]) + return ctx.default_return_type + + +def enum_value_callback(ctx: mypy.plugin.AttributeContext) -> Type: + """This plugin refines the 'value' attribute in enums to refer to + the original underlying value. For example, suppose we have the + following: + + class SomeEnum: + FOO = A() + BAR = B() + + By default, mypy will infer that 'SomeEnum.FOO.value' and + 'SomeEnum.BAR.value' both are of type 'Any'. This plugin refines + this inference so that mypy understands the expressions are + actually of types 'A' and 'B' respectively. This better reflects + the actual runtime behavior. + + This plugin works simply by looking up the original value assigned + to the enum. For example, when this plugin sees 'SomeEnum.BAR.value', + it will look up whatever type 'BAR' had in the SomeEnum TypeInfo and + use that as the inferred type of the overall expression. + + This plugin assumes that the provided context is an attribute access + matching one of the strings found in 'ENUM_VALUE_ACCESS'. + """ + enum_field_name = _extract_underlying_field_name(ctx.type) + if enum_field_name is None: + # We do not know the enum field name (perhaps it was passed to a + # function and we only know that it _is_ a member). All is not lost + # however, if we can prove that the all of the enum members have the + # same value-type, then it doesn't matter which member was passed in. + # The value-type is still known. + if isinstance(ctx.type, Instance): + info = ctx.type.type + + # As long as mypy doesn't understand attribute creation in __new__, + # there is no way to predict the value type if the enum class has a + # custom implementation + if _implements_new(info): + return ctx.default_attr_type + + stnodes = (info.get(name) for name in info.names) + + # Enums _can_ have methods, instance attributes, and `nonmember`s. + # Omit methods and attributes created by assigning to self.* + # for our value inference. + node_types = ( + get_proper_type(n.type) if n else None + for n in stnodes + if n is None or not n.implicit + ) + proper_types = [ + _infer_value_type_with_auto_fallback(ctx, t) + for t in node_types + if t is None + or (not isinstance(t, CallableType) and not is_named_instance(t, "enum.nonmember")) + ] + underlying_type = _first(proper_types) + if underlying_type is None: + return ctx.default_attr_type + + # At first we try to predict future `value` type if all other items + # have the same type. For example, `int`. + # If this is the case, we simply return this type. + # See https://github.com/python/mypy/pull/9443 + all_same_value_type = all( + proper_type is not None and proper_type == underlying_type + for proper_type in proper_types + ) + if all_same_value_type: + if underlying_type is not None: + return underlying_type + + # But, after we started treating all `Enum` values as `Final`, + # we start to infer types in + # `item = 1` as `Literal[1]`, not just `int`. + # So, for example types in this `Enum` will all be different: + # + # class Ordering(IntEnum): + # one = 1 + # two = 2 + # three = 3 + # + # We will infer three `Literal` types here. + # They are not the same, but they are equivalent. + # So, we unify them to make sure `.value` prediction still works. + # Result will be `Literal[1] | Literal[2] | Literal[3]` for this case. + all_equivalent_types = all( + proper_type is not None and is_equivalent(proper_type, underlying_type) + for proper_type in proper_types + ) + if all_equivalent_types: + return make_simplified_union(cast(Sequence[Type], proper_types)) + return ctx.default_attr_type + + assert isinstance(ctx.type, Instance) + info = ctx.type.type + + # As long as mypy doesn't understand attribute creation in __new__, + # there is no way to predict the value type if the enum class has a + # custom implementation + if _implements_new(info): + return ctx.default_attr_type + + stnode = info.get(enum_field_name) + if stnode is None: + return ctx.default_attr_type + + underlying_type = _infer_value_type_with_auto_fallback(ctx, get_proper_type(stnode.type)) + if underlying_type is None: + return ctx.default_attr_type + + return underlying_type + + +def _extract_underlying_field_name(typ: Type) -> str | None: + """If the given type corresponds to some Enum instance, returns the + original name of that enum. For example, if we receive in the type + corresponding to 'SomeEnum.FOO', we return the string "SomeEnum.Foo". + + This helper takes advantage of the fact that Enum instances are valid + to use inside Literal[...] types. An expression like 'SomeEnum.FOO' is + actually represented by an Instance type with a Literal enum fallback. + + We can examine this Literal fallback to retrieve the string. + """ + typ = get_proper_type(typ) + if not isinstance(typ, Instance): + return None + + if not typ.type.is_enum: + return None + + underlying_literal = typ.last_known_value + if underlying_literal is None: + return None + + # The checks above have verified this LiteralType is representing an enum value, + # which means the 'value' field is guaranteed to be the name of the enum field + # as a string. + assert isinstance(underlying_literal.value, str) + return underlying_literal.value diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py new file mode 100644 index 000000000000..6a063174bfcb --- /dev/null +++ b/mypy/plugins/functools.py @@ -0,0 +1,381 @@ +"""Plugin for supporting the functools standard library module.""" + +from __future__ import annotations + +from typing import Final, NamedTuple + +import mypy.checker +import mypy.plugin +import mypy.semanal +from mypy.argmap import map_actuals_to_formals +from mypy.nodes import ARG_POS, ARG_STAR2, ArgKind, Argument, CallExpr, FuncItem, NameExpr, Var +from mypy.plugins.common import add_method_to_class +from mypy.typeops import get_all_type_vars +from mypy.types import ( + AnyType, + CallableType, + Instance, + Overloaded, + ParamSpecFlavor, + ParamSpecType, + Type, + TypeOfAny, + TypeVarType, + UnboundType, + UnionType, + get_proper_type, +) + +functools_total_ordering_makers: Final = {"functools.total_ordering"} + +_ORDERING_METHODS: Final = {"__lt__", "__le__", "__gt__", "__ge__"} + +PARTIAL: Final = "functools.partial" + + +class _MethodInfo(NamedTuple): + is_static: bool + type: CallableType + + +def functools_total_ordering_maker_callback( + ctx: mypy.plugin.ClassDefContext, auto_attribs_default: bool = False +) -> bool: + """Add dunder methods to classes decorated with functools.total_ordering.""" + comparison_methods = _analyze_class(ctx) + if not comparison_methods: + ctx.api.fail( + 'No ordering operation defined when using "functools.total_ordering": < > <= >=', + ctx.reason, + ) + return True + + # prefer __lt__ to __le__ to __gt__ to __ge__ + root = max(comparison_methods, key=lambda k: (comparison_methods[k] is None, k)) + root_method = comparison_methods[root] + if not root_method: + # None of the defined comparison methods can be analysed + return True + + other_type = _find_other_type(root_method) + bool_type = ctx.api.named_type("builtins.bool") + ret_type: Type = bool_type + if root_method.type.ret_type != ctx.api.named_type("builtins.bool"): + proper_ret_type = get_proper_type(root_method.type.ret_type) + if not ( + isinstance(proper_ret_type, UnboundType) + and proper_ret_type.name.split(".")[-1] == "bool" + ): + ret_type = AnyType(TypeOfAny.implementation_artifact) + for additional_op in _ORDERING_METHODS: + # Either the method is not implemented + # or has an unknown signature that we can now extrapolate. + if not comparison_methods.get(additional_op): + args = [Argument(Var("other", other_type), other_type, None, ARG_POS)] + add_method_to_class(ctx.api, ctx.cls, additional_op, args, ret_type) + + return True + + +def _find_other_type(method: _MethodInfo) -> Type: + """Find the type of the ``other`` argument in a comparison method.""" + first_arg_pos = 0 if method.is_static else 1 + cur_pos_arg = 0 + other_arg = None + for arg_kind, arg_type in zip(method.type.arg_kinds, method.type.arg_types): + if arg_kind.is_positional(): + if cur_pos_arg == first_arg_pos: + other_arg = arg_type + break + + cur_pos_arg += 1 + elif arg_kind != ARG_STAR2: + other_arg = arg_type + break + + if other_arg is None: + return AnyType(TypeOfAny.implementation_artifact) + + return other_arg + + +def _analyze_class(ctx: mypy.plugin.ClassDefContext) -> dict[str, _MethodInfo | None]: + """Analyze the class body, its parents, and return the comparison methods found.""" + # Traverse the MRO and collect ordering methods. + comparison_methods: dict[str, _MethodInfo | None] = {} + # Skip object because total_ordering does not use methods from object + for cls in ctx.cls.info.mro[:-1]: + for name in _ORDERING_METHODS: + if name in cls.names and name not in comparison_methods: + node = cls.names[name].node + if isinstance(node, FuncItem) and isinstance(node.type, CallableType): + comparison_methods[name] = _MethodInfo(node.is_static, node.type) + continue + + if isinstance(node, Var): + proper_type = get_proper_type(node.type) + if isinstance(proper_type, CallableType): + comparison_methods[name] = _MethodInfo(node.is_staticmethod, proper_type) + continue + + comparison_methods[name] = None + + return comparison_methods + + +def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: + """Infer a more precise return type for functools.partial""" + if not isinstance(ctx.api, mypy.checker.TypeChecker): # use internals + return ctx.default_return_type + if len(ctx.arg_types) != 3: # fn, *args, **kwargs + return ctx.default_return_type + if len(ctx.arg_types[0]) != 1: + return ctx.default_return_type + + if isinstance(get_proper_type(ctx.arg_types[0][0]), Overloaded): + # TODO: handle overloads, just fall back to whatever the non-plugin code does + return ctx.default_return_type + return handle_partial_with_callee(ctx, callee=ctx.arg_types[0][0]) + + +def handle_partial_with_callee(ctx: mypy.plugin.FunctionContext, callee: Type) -> Type: + if not isinstance(ctx.api, mypy.checker.TypeChecker): # use internals + return ctx.default_return_type + + if isinstance(callee_proper := get_proper_type(callee), UnionType): + return UnionType.make_union( + [handle_partial_with_callee(ctx, item) for item in callee_proper.items] + ) + + fn_type = ctx.api.extract_callable_type(callee, ctx=ctx.default_return_type) + if fn_type is None: + return ctx.default_return_type + + # We must normalize from the start to have coherent view together with TypeChecker. + fn_type = fn_type.with_unpacked_kwargs().with_normalized_var_args() + + last_context = ctx.api.type_context[-1] + if not fn_type.is_type_obj(): + # We wrap the return type to get use of a possible type context provided by caller. + # We cannot do this in case of class objects, since otherwise the plugin may get + # falsely triggered when evaluating the constructed call itself. + ret_type: Type = ctx.api.named_generic_type(PARTIAL, [fn_type.ret_type]) + wrapped_return = True + else: + ret_type = fn_type.ret_type + # Instead, for class objects we ignore any type context to avoid spurious errors, + # since the type context will be partial[X] etc., not X. + ctx.api.type_context[-1] = None + wrapped_return = False + + # Flatten actual to formal mapping, since this is what check_call() expects. + actual_args = [] + actual_arg_kinds = [] + actual_arg_names = [] + actual_types = [] + seen_args = set() + for i, param in enumerate(ctx.args[1:], start=1): + for j, a in enumerate(param): + if a in seen_args: + # Same actual arg can map to multiple formals, but we need to include + # each one only once. + continue + # Here we rely on the fact that expressions are essentially immutable, so + # they can be compared by identity. + seen_args.add(a) + actual_args.append(a) + actual_arg_kinds.append(ctx.arg_kinds[i][j]) + actual_arg_names.append(ctx.arg_names[i][j]) + actual_types.append(ctx.arg_types[i][j]) + + formal_to_actual = map_actuals_to_formals( + actual_kinds=actual_arg_kinds, + actual_names=actual_arg_names, + formal_kinds=fn_type.arg_kinds, + formal_names=fn_type.arg_names, + actual_arg_type=lambda i: actual_types[i], + ) + + # We need to remove any type variables that appear only in formals that have + # no actuals, to avoid eagerly binding them in check_call() below. + can_infer_ids = set() + for i, arg_type in enumerate(fn_type.arg_types): + if not formal_to_actual[i]: + continue + can_infer_ids.update({tv.id for tv in get_all_type_vars(arg_type)}) + + # special_sig="partial" allows omission of args/kwargs typed with ParamSpec + defaulted = fn_type.copy_modified( + arg_kinds=[ + ( + ArgKind.ARG_OPT + if k == ArgKind.ARG_POS + else (ArgKind.ARG_NAMED_OPT if k == ArgKind.ARG_NAMED else k) + ) + for k in fn_type.arg_kinds + ], + ret_type=ret_type, + variables=[ + tv + for tv in fn_type.variables + # Keep TypeVarTuple/ParamSpec to avoid spurious errors on empty args. + if tv.id in can_infer_ids or not isinstance(tv, TypeVarType) + ], + special_sig="partial", + ) + if defaulted.line < 0: + # Make up a line number if we don't have one + defaulted.set_line(ctx.default_return_type) + + # Create a valid context for various ad-hoc inspections in check_call(). + call_expr = CallExpr( + callee=ctx.args[0][0], + args=actual_args, + arg_kinds=actual_arg_kinds, + arg_names=actual_arg_names, + analyzed=ctx.context.analyzed if isinstance(ctx.context, CallExpr) else None, + ) + call_expr.set_line(ctx.context) + + _, bound = ctx.api.expr_checker.check_call( + callee=defaulted, + args=actual_args, + arg_kinds=actual_arg_kinds, + arg_names=actual_arg_names, + context=call_expr, + ) + if not wrapped_return: + # Restore previously ignored context. + ctx.api.type_context[-1] = last_context + + bound = get_proper_type(bound) + if not isinstance(bound, CallableType): + return ctx.default_return_type + + if wrapped_return: + # Reverse the wrapping we did above. + ret_type = get_proper_type(bound.ret_type) + if not isinstance(ret_type, Instance) or ret_type.type.fullname != PARTIAL: + return ctx.default_return_type + bound = bound.copy_modified(ret_type=ret_type.args[0]) + + partial_kinds = [] + partial_types = [] + partial_names = [] + # We need to fully apply any positional arguments (they cannot be respecified) + # However, keyword arguments can be respecified, so just give them a default + for i, actuals in enumerate(formal_to_actual): + if len(bound.arg_types) == len(fn_type.arg_types): + arg_type = bound.arg_types[i] + if not mypy.checker.is_valid_inferred_type(arg_type): + arg_type = fn_type.arg_types[i] # bit of a hack + else: + # TODO: I assume that bound and fn_type have the same arguments. It appears this isn't + # true when PEP 646 things are happening. See testFunctoolsPartialTypeVarTuple + arg_type = fn_type.arg_types[i] + + if not actuals or fn_type.arg_kinds[i] in (ArgKind.ARG_STAR, ArgKind.ARG_STAR2): + partial_kinds.append(fn_type.arg_kinds[i]) + partial_types.append(arg_type) + partial_names.append(fn_type.arg_names[i]) + else: + assert actuals + if any(actual_arg_kinds[j] in (ArgKind.ARG_POS, ArgKind.ARG_STAR) for j in actuals): + # Don't add params for arguments passed positionally + continue + # Add defaulted params for arguments passed via keyword + kind = actual_arg_kinds[actuals[0]] + if kind == ArgKind.ARG_NAMED or kind == ArgKind.ARG_STAR2: + kind = ArgKind.ARG_NAMED_OPT + partial_kinds.append(kind) + partial_types.append(arg_type) + partial_names.append(fn_type.arg_names[i]) + + ret_type = bound.ret_type + if not mypy.checker.is_valid_inferred_type(ret_type): + ret_type = fn_type.ret_type # same kind of hack as above + + partially_applied = fn_type.copy_modified( + arg_types=partial_types, + arg_kinds=partial_kinds, + arg_names=partial_names, + ret_type=ret_type, + special_sig="partial", + ) + + ret = ctx.api.named_generic_type(PARTIAL, [ret_type]) + ret = ret.copy_with_extra_attr("__mypy_partial", partially_applied) + if partially_applied.param_spec(): + assert ret.extra_attrs is not None # copy_with_extra_attr above ensures this + attrs = ret.extra_attrs.copy() + if ArgKind.ARG_STAR in actual_arg_kinds: + attrs.immutable.add("__mypy_partial_paramspec_args_bound") + if ArgKind.ARG_STAR2 in actual_arg_kinds: + attrs.immutable.add("__mypy_partial_paramspec_kwargs_bound") + ret.extra_attrs = attrs + return ret + + +def partial_call_callback(ctx: mypy.plugin.MethodContext) -> Type: + """Infer a more precise return type for functools.partial.__call__.""" + if ( + not isinstance(ctx.api, mypy.checker.TypeChecker) # use internals + or not isinstance(ctx.type, Instance) + or ctx.type.type.fullname != PARTIAL + or not ctx.type.extra_attrs + or "__mypy_partial" not in ctx.type.extra_attrs.attrs + ): + return ctx.default_return_type + + extra_attrs = ctx.type.extra_attrs + partial_type = get_proper_type(extra_attrs.attrs["__mypy_partial"]) + if len(ctx.arg_types) != 2: # *args, **kwargs + return ctx.default_return_type + + # See comments for similar actual to formal code above + actual_args = [] + actual_arg_kinds = [] + actual_arg_names = [] + seen_args = set() + for i, param in enumerate(ctx.args): + for j, a in enumerate(param): + if a in seen_args: + continue + seen_args.add(a) + actual_args.append(a) + actual_arg_kinds.append(ctx.arg_kinds[i][j]) + actual_arg_names.append(ctx.arg_names[i][j]) + + result, _ = ctx.api.expr_checker.check_call( + callee=partial_type, + args=actual_args, + arg_kinds=actual_arg_kinds, + arg_names=actual_arg_names, + context=ctx.context, + ) + if not isinstance(partial_type, CallableType) or partial_type.param_spec() is None: + return result + + args_bound = "__mypy_partial_paramspec_args_bound" in extra_attrs.immutable + kwargs_bound = "__mypy_partial_paramspec_kwargs_bound" in extra_attrs.immutable + + passed_paramspec_parts = [ + arg.node.type + for arg in actual_args + if isinstance(arg, NameExpr) + and isinstance(arg.node, Var) + and isinstance(arg.node.type, ParamSpecType) + ] + # ensure *args: P.args + args_passed = any(part.flavor == ParamSpecFlavor.ARGS for part in passed_paramspec_parts) + if not args_bound and not args_passed: + ctx.api.expr_checker.msg.too_few_arguments(partial_type, ctx.context, actual_arg_names) + elif args_bound and args_passed: + ctx.api.expr_checker.msg.too_many_arguments(partial_type, ctx.context) + + # ensure **kwargs: P.kwargs + kwargs_passed = any(part.flavor == ParamSpecFlavor.KWARGS for part in passed_paramspec_parts) + if not kwargs_bound and not kwargs_passed: + ctx.api.expr_checker.msg.too_few_arguments(partial_type, ctx.context, actual_arg_names) + + return result diff --git a/mypy/plugins/proper_plugin.py b/mypy/plugins/proper_plugin.py new file mode 100644 index 000000000000..f51685c80afa --- /dev/null +++ b/mypy/plugins/proper_plugin.py @@ -0,0 +1,176 @@ +""" +This plugin is helpful for mypy development itself. +By default, it is not enabled for mypy users. + +It also can be used by plugin developers as a part of their CI checks. + +It finds missing ``get_proper_type()`` call, which can lead to multiple errors. +""" + +from __future__ import annotations + +from typing import Callable + +from mypy.checker import TypeChecker +from mypy.nodes import TypeInfo +from mypy.plugin import FunctionContext, Plugin +from mypy.subtypes import is_proper_subtype +from mypy.types import ( + AnyType, + FunctionLike, + Instance, + NoneTyp, + ProperType, + TupleType, + Type, + UnionType, + get_proper_type, + get_proper_types, +) + + +class ProperTypePlugin(Plugin): + """ + A plugin to ensure that every type is expanded before doing any special-casing. + + This solves the problem that we have hundreds of call sites like: + + if isinstance(typ, UnionType): + ... # special-case union + + But after introducing a new type TypeAliasType (and removing immediate expansion) + all these became dangerous because typ may be e.g. an alias to union. + """ + + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname == "builtins.isinstance": + return isinstance_proper_hook + if fullname == "mypy.types.get_proper_type": + return proper_type_hook + if fullname == "mypy.types.get_proper_types": + return proper_types_hook + return None + + +def isinstance_proper_hook(ctx: FunctionContext) -> Type: + if len(ctx.arg_types) != 2 or not ctx.arg_types[1]: + return ctx.default_return_type + + right = get_proper_type(ctx.arg_types[1][0]) + for arg in ctx.arg_types[0]: + if ( + is_improper_type(arg) or isinstance(get_proper_type(arg), AnyType) + ) and is_dangerous_target(right): + if is_special_target(right): + return ctx.default_return_type + ctx.api.fail( + "Never apply isinstance() to unexpanded types;" + " use mypy.types.get_proper_type() first", + ctx.context, + ) + ctx.api.note( # type: ignore[attr-defined] + "If you pass on the original type" + " after the check, always use its unexpanded version", + ctx.context, + ) + return ctx.default_return_type + + +def is_special_target(right: ProperType) -> bool: + """Whitelist some special cases for use in isinstance() with improper types.""" + if isinstance(right, FunctionLike) and right.is_type_obj(): + if right.type_object().fullname == "builtins.tuple": + # Used with Union[Type, Tuple[Type, ...]]. + return True + if right.type_object().fullname in ( + "mypy.types.Type", + "mypy.types.ProperType", + "mypy.types.TypeAliasType", + ): + # Special case: things like assert isinstance(typ, ProperType) are always OK. + return True + if right.type_object().fullname in ( + "mypy.types.UnboundType", + "mypy.types.TypeVarLikeType", + "mypy.types.TypeVarType", + "mypy.types.UnpackType", + "mypy.types.TypeVarTupleType", + "mypy.types.ParamSpecType", + "mypy.types.Parameters", + "mypy.types.RawExpressionType", + "mypy.types.EllipsisType", + "mypy.types.StarType", + "mypy.types.TypeList", + "mypy.types.CallableArgument", + "mypy.types.PartialType", + "mypy.types.ErasedType", + "mypy.types.DeletedType", + "mypy.types.RequiredType", + "mypy.types.ReadOnlyType", + ): + # Special case: these are not valid targets for a type alias and thus safe. + # TODO: introduce a SyntheticType base to simplify this? + return True + elif isinstance(right, TupleType): + return all(is_special_target(t) for t in get_proper_types(right.items)) + return False + + +def is_improper_type(typ: Type) -> bool: + """Is this a type that is not a subtype of ProperType?""" + typ = get_proper_type(typ) + if isinstance(typ, Instance): + info = typ.type + return info.has_base("mypy.types.Type") and not info.has_base("mypy.types.ProperType") + if isinstance(typ, UnionType): + return any(is_improper_type(t) for t in typ.items) + return False + + +def is_dangerous_target(typ: ProperType) -> bool: + """Is this a dangerous target (right argument) for an isinstance() check?""" + if isinstance(typ, TupleType): + return any(is_dangerous_target(get_proper_type(t)) for t in typ.items) + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + return typ.type_object().has_base("mypy.types.Type") + return False + + +def proper_type_hook(ctx: FunctionContext) -> Type: + """Check if this get_proper_type() call is not redundant.""" + arg_types = ctx.arg_types[0] + if arg_types: + arg_type = get_proper_type(arg_types[0]) + proper_type = get_proper_type_instance(ctx) + if is_proper_subtype(arg_type, UnionType.make_union([NoneTyp(), proper_type])): + # Minimize amount of spurious errors from overload machinery. + # TODO: call the hook on the overload as a whole? + if isinstance(arg_type, (UnionType, Instance)): + ctx.api.fail("Redundant call to get_proper_type()", ctx.context) + return ctx.default_return_type + + +def proper_types_hook(ctx: FunctionContext) -> Type: + """Check if this get_proper_types() call is not redundant.""" + arg_types = ctx.arg_types[0] + if arg_types: + arg_type = arg_types[0] + proper_type = get_proper_type_instance(ctx) + item_type = UnionType.make_union([NoneTyp(), proper_type]) + ok_type = ctx.api.named_generic_type("typing.Iterable", [item_type]) + if is_proper_subtype(arg_type, ok_type): + ctx.api.fail("Redundant call to get_proper_types()", ctx.context) + return ctx.default_return_type + + +def get_proper_type_instance(ctx: FunctionContext) -> Instance: + checker = ctx.api + assert isinstance(checker, TypeChecker) + types = checker.modules["mypy.types"] + proper_type_info = types.names["ProperType"] + assert isinstance(proper_type_info.node, TypeInfo) + return Instance(proper_type_info.node, []) + + +def plugin(version: str) -> type[ProperTypePlugin]: + return ProperTypePlugin diff --git a/mypy/plugins/singledispatch.py b/mypy/plugins/singledispatch.py new file mode 100644 index 000000000000..c5ce20233a0a --- /dev/null +++ b/mypy/plugins/singledispatch.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +from typing import Final, NamedTuple, Sequence, TypeVar, Union +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.messages import format_type +from mypy.nodes import ARG_POS, Argument, Block, ClassDef, Context, SymbolTable, TypeInfo, Var +from mypy.options import Options +from mypy.plugin import CheckerPluginInterface, FunctionContext, MethodContext, MethodSigContext +from mypy.plugins.common import add_method_to_class +from mypy.subtypes import is_subtype +from mypy.types import ( + AnyType, + CallableType, + FunctionLike, + Instance, + NoneType, + Overloaded, + Type, + TypeOfAny, + get_proper_type, +) + + +class SingledispatchTypeVars(NamedTuple): + return_type: Type + fallback: CallableType + + +class RegisterCallableInfo(NamedTuple): + register_type: Type + singledispatch_obj: Instance + + +SINGLEDISPATCH_TYPE: Final = "functools._SingleDispatchCallable" + +SINGLEDISPATCH_REGISTER_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.register" + +SINGLEDISPATCH_CALLABLE_CALL_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.__call__" + + +def get_singledispatch_info(typ: Instance) -> SingledispatchTypeVars | None: + if len(typ.args) == 2: + return SingledispatchTypeVars(*typ.args) # type: ignore[arg-type] + return None + + +T = TypeVar("T") + + +def get_first_arg(args: list[list[T]]) -> T | None: + """Get the element that corresponds to the first argument passed to the function""" + if args and args[0]: + return args[0][0] + return None + + +REGISTER_RETURN_CLASS: Final = "_SingleDispatchRegisterCallable" + +REGISTER_CALLABLE_CALL_METHOD: Final = f"functools.{REGISTER_RETURN_CLASS}.__call__" + + +def make_fake_register_class_instance( + api: CheckerPluginInterface, type_args: Sequence[Type] +) -> Instance: + defn = ClassDef(REGISTER_RETURN_CLASS, Block([])) + defn.fullname = f"functools.{REGISTER_RETURN_CLASS}" + info = TypeInfo(SymbolTable(), defn, "functools") + obj_type = api.named_generic_type("builtins.object", []).type + info.bases = [Instance(obj_type, [])] + info.mro = [info, obj_type] + defn.info = info + + func_arg = Argument(Var("name"), AnyType(TypeOfAny.implementation_artifact), None, ARG_POS) + add_method_to_class(api, defn, "__call__", [func_arg], NoneType()) + + return Instance(info, type_args) + + +PluginContext: _TypeAlias = Union[FunctionContext, MethodContext] + + +def fail(ctx: PluginContext, msg: str, context: Context | None) -> None: + """Emit an error message. + + This tries to emit an error message at the location specified by `context`, falling back to the + location specified by `ctx.context`. This is helpful when the only context information about + where you want to put the error message may be None (like it is for `CallableType.definition`) + and falling back to the location of the calling function is fine.""" + # TODO: figure out if there is some more reliable way of getting context information, so this + # function isn't necessary + if context is not None: + err_context = context + else: + err_context = ctx.context + ctx.api.fail(msg, err_context) + + +def create_singledispatch_function_callback(ctx: FunctionContext) -> Type: + """Called for functools.singledispatch""" + func_type = get_proper_type(get_first_arg(ctx.arg_types)) + if isinstance(func_type, CallableType): + if len(func_type.arg_kinds) < 1: + fail( + ctx, "Singledispatch function requires at least one argument", func_type.definition + ) + return ctx.default_return_type + + elif not func_type.arg_kinds[0].is_positional(star=True): + fail( + ctx, + "First argument to singledispatch function must be a positional argument", + func_type.definition, + ) + return ctx.default_return_type + + # singledispatch returns an instance of functools._SingleDispatchCallable according to + # typeshed + singledispatch_obj = get_proper_type(ctx.default_return_type) + assert isinstance(singledispatch_obj, Instance) + singledispatch_obj.args += (func_type,) + + return ctx.default_return_type + + +def singledispatch_register_callback(ctx: MethodContext) -> Type: + """Called for functools._SingleDispatchCallable.register""" + assert isinstance(ctx.type, Instance) + # TODO: check that there's only one argument + first_arg_type = get_proper_type(get_first_arg(ctx.arg_types)) + if isinstance(first_arg_type, (CallableType, Overloaded)) and first_arg_type.is_type_obj(): + # HACK: We received a class as an argument to register. We need to be able + # to access the function that register is being applied to, and the typeshed definition + # of register has it return a generic Callable, so we create a new + # SingleDispatchRegisterCallable class, define a __call__ method, and then add a + # plugin hook for that. + + # is_subtype doesn't work when the right type is Overloaded, so we need the + # actual type + register_type = first_arg_type.items[0].ret_type + type_args = RegisterCallableInfo(register_type, ctx.type) + register_callable = make_fake_register_class_instance(ctx.api, type_args) + return register_callable + elif isinstance(first_arg_type, CallableType): + # TODO: do more checking for registered functions + register_function(ctx, ctx.type, first_arg_type, ctx.api.options) + # The typeshed stubs for register say that the function returned is Callable[..., T], even + # though the function returned is the same as the one passed in. We return the type of the + # function so that mypy can properly type check cases where the registered function is used + # directly (instead of through singledispatch) + return first_arg_type + + # fallback in case we don't recognize the arguments + return ctx.default_return_type + + +def register_function( + ctx: PluginContext, + singledispatch_obj: Instance, + func: Type, + options: Options, + register_arg: Type | None = None, +) -> None: + """Register a function""" + + func = get_proper_type(func) + if not isinstance(func, CallableType): + return + metadata = get_singledispatch_info(singledispatch_obj) + if metadata is None: + # if we never added the fallback to the type variables, we already reported an error, so + # just don't do anything here + return + dispatch_type = get_dispatch_type(func, register_arg) + if dispatch_type is None: + # TODO: report an error here that singledispatch requires at least one argument + # (might want to do the error reporting in get_dispatch_type) + return + fallback = metadata.fallback + + fallback_dispatch_type = fallback.arg_types[0] + if not is_subtype(dispatch_type, fallback_dispatch_type): + fail( + ctx, + "Dispatch type {} must be subtype of fallback function first argument {}".format( + format_type(dispatch_type, options), format_type(fallback_dispatch_type, options) + ), + func.definition, + ) + return + return + + +def get_dispatch_type(func: CallableType, register_arg: Type | None) -> Type | None: + if register_arg is not None: + return register_arg + if func.arg_types: + return func.arg_types[0] + return None + + +def call_singledispatch_function_after_register_argument(ctx: MethodContext) -> Type: + """Called on the function after passing a type to register""" + register_callable = ctx.type + if isinstance(register_callable, Instance): + type_args = RegisterCallableInfo(*register_callable.args) # type: ignore[arg-type] + func = get_first_arg(ctx.arg_types) + if func is not None: + register_function( + ctx, type_args.singledispatch_obj, func, ctx.api.options, type_args.register_type + ) + # see call to register_function in the callback for register + return func + return ctx.default_return_type + + +def call_singledispatch_function_callback(ctx: MethodSigContext) -> FunctionLike: + """Called for functools._SingleDispatchCallable.__call__""" + if not isinstance(ctx.type, Instance): + return ctx.default_signature + metadata = get_singledispatch_info(ctx.type) + if metadata is None: + return ctx.default_signature + return metadata.fallback diff --git a/mypy/py.typed b/mypy/py.typed new file mode 100644 index 000000000000..9724ed562890 --- /dev/null +++ b/mypy/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. The mypy package uses inline types. diff --git a/mypy/pyinfo.py b/mypy/pyinfo.py new file mode 100644 index 000000000000..ee5307cfaebb --- /dev/null +++ b/mypy/pyinfo.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +"""Utilities to find the site and prefix information of a Python executable. + +This file MUST remain compatible with all Python 3.8+ versions. Since we cannot make any +assumptions about the Python being executed, this module should not use *any* dependencies outside +of the standard library found in Python 3.8. This file is run each mypy run, so it should be kept +as fast as possible. +""" +import sys + +if __name__ == "__main__": + # HACK: We don't want to pick up mypy.types as the top-level types + # module. This could happen if this file is run as a script. + # This workaround fixes this for Python versions before 3.11. + if sys.version_info < (3, 11): + old_sys_path = sys.path + sys.path = sys.path[1:] + import types # noqa: F401 + + sys.path = old_sys_path + +import os +import site +import sysconfig + + +def getsitepackages() -> list[str]: + res = [] + if hasattr(site, "getsitepackages"): + res.extend(site.getsitepackages()) + + if hasattr(site, "getusersitepackages") and site.ENABLE_USER_SITE: + res.insert(0, site.getusersitepackages()) + else: + res = [sysconfig.get_paths()["purelib"]] + return res + + +def getsyspath() -> list[str]: + # Do not include things from the standard library + # because those should come from typeshed. + stdlib_zip = os.path.join( + sys.base_exec_prefix, + getattr(sys, "platlibdir", "lib"), + f"python{sys.version_info.major}{sys.version_info.minor}.zip", + ) + stdlib = sysconfig.get_path("stdlib") + stdlib_ext = os.path.join(stdlib, "lib-dynload") + excludes = {stdlib_zip, stdlib, stdlib_ext} + + # Drop the first entry of sys.path + # - If pyinfo.py is executed as a script (in a subprocess), this is the directory + # containing pyinfo.py + # - Otherwise, if mypy launched via console script, this is the directory of the script + # - Otherwise, if mypy launched via python -m mypy, this is the current directory + # In all these cases, it is desirable to drop the first entry + # Note that mypy adds the cwd to SearchPaths.python_path, so we still find things on the + # cwd consistently (the return value here sets SearchPaths.package_path) + + # Python 3.11 adds a "safe_path" flag wherein Python won't automatically prepend + # anything to sys.path. In this case, the first entry of sys.path is no longer special. + offset = 0 if sys.version_info >= (3, 11) and sys.flags.safe_path else 1 + + abs_sys_path = (os.path.abspath(p) for p in sys.path[offset:]) + return [p for p in abs_sys_path if p not in excludes] + + +def getsearchdirs() -> tuple[list[str], list[str]]: + return (getsyspath(), getsitepackages()) + + +if __name__ == "__main__": + sys.stdout.reconfigure(encoding="utf-8") # type: ignore[union-attr] + if sys.argv[-1] == "getsearchdirs": + print(repr(getsearchdirs())) + else: + print("ERROR: incorrect argument to pyinfo.py.", file=sys.stderr) + sys.exit(1) diff --git a/mypy/reachability.py b/mypy/reachability.py new file mode 100644 index 000000000000..a25b9dff4581 --- /dev/null +++ b/mypy/reachability.py @@ -0,0 +1,362 @@ +"""Utilities related to determining the reachability of code (in semantic analysis).""" + +from __future__ import annotations + +from typing import Final, Tuple, TypeVar + +from mypy.literals import literal +from mypy.nodes import ( + LITERAL_YES, + AssertStmt, + Block, + CallExpr, + ComparisonExpr, + Expression, + FuncDef, + IfStmt, + Import, + ImportAll, + ImportFrom, + IndexExpr, + IntExpr, + MatchStmt, + MemberExpr, + NameExpr, + OpExpr, + SliceExpr, + StrExpr, + TupleExpr, + UnaryExpr, +) +from mypy.options import Options +from mypy.patterns import AsPattern, OrPattern, Pattern +from mypy.traverser import TraverserVisitor + +# Inferred truth value of an expression. +ALWAYS_TRUE: Final = 1 +MYPY_TRUE: Final = 2 # True in mypy, False at runtime +ALWAYS_FALSE: Final = 3 +MYPY_FALSE: Final = 4 # False in mypy, True at runtime +TRUTH_VALUE_UNKNOWN: Final = 5 + +inverted_truth_mapping: Final = { + ALWAYS_TRUE: ALWAYS_FALSE, + ALWAYS_FALSE: ALWAYS_TRUE, + TRUTH_VALUE_UNKNOWN: TRUTH_VALUE_UNKNOWN, + MYPY_TRUE: MYPY_FALSE, + MYPY_FALSE: MYPY_TRUE, +} + +reverse_op: Final = {"==": "==", "!=": "!=", "<": ">", ">": "<", "<=": ">=", ">=": "<="} + + +def infer_reachability_of_if_statement(s: IfStmt, options: Options) -> None: + for i in range(len(s.expr)): + result = infer_condition_value(s.expr[i], options) + if result in (ALWAYS_FALSE, MYPY_FALSE): + # The condition is considered always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result in (ALWAYS_TRUE, MYPY_TRUE): + # This condition is considered always true, so all of the remaining + # elif/else bodies should not be checked. + if result == MYPY_TRUE: + # This condition is false at runtime; this will affect + # import priorities. + mark_block_mypy_only(s.body[i]) + for body in s.body[i + 1 :]: + mark_block_unreachable(body) + + # Make sure else body always exists and is marked as + # unreachable so the type checker always knows that + # all control flow paths will flow through the if + # statement body. + if not s.else_body: + s.else_body = Block([]) + mark_block_unreachable(s.else_body) + break + + +def infer_reachability_of_match_statement(s: MatchStmt, options: Options) -> None: + for i, guard in enumerate(s.guards): + pattern_value = infer_pattern_value(s.patterns[i]) + + if guard is not None: + guard_value = infer_condition_value(guard, options) + else: + guard_value = ALWAYS_TRUE + + if pattern_value in (ALWAYS_FALSE, MYPY_FALSE) or guard_value in ( + ALWAYS_FALSE, + MYPY_FALSE, + ): + # The case is considered always false, so we skip the case body. + mark_block_unreachable(s.bodies[i]) + elif pattern_value in (ALWAYS_FALSE, MYPY_TRUE) and guard_value in ( + ALWAYS_TRUE, + MYPY_TRUE, + ): + for body in s.bodies[i + 1 :]: + mark_block_unreachable(body) + + if guard_value == MYPY_TRUE: + # This condition is false at runtime; this will affect + # import priorities. + mark_block_mypy_only(s.bodies[i]) + + +def assert_will_always_fail(s: AssertStmt, options: Options) -> bool: + return infer_condition_value(s.expr, options) in (ALWAYS_FALSE, MYPY_FALSE) + + +def infer_condition_value(expr: Expression, options: Options) -> int: + """Infer whether the given condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + MYPY_TRUE if true under mypy and false at runtime, MYPY_FALSE if + false under mypy and true at runtime, else TRUTH_VALUE_UNKNOWN. + """ + pyversion = options.python_version + name = "" + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == "not": + expr = alias.expr + negated = True + result = TRUTH_VALUE_UNKNOWN + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + elif isinstance(expr, OpExpr) and expr.op in ("and", "or"): + left = infer_condition_value(expr.left, options) + if (left in (ALWAYS_TRUE, MYPY_TRUE) and expr.op == "and") or ( + left in (ALWAYS_FALSE, MYPY_FALSE) and expr.op == "or" + ): + # Either `True and ` or `False or `: the result will + # always be the right-hand-side. + return infer_condition_value(expr.right, options) + else: + # The result will always be the left-hand-side (e.g. ALWAYS_* or + # TRUTH_VALUE_UNKNOWN). + return left + else: + result = consider_sys_version_info(expr, pyversion) + if result == TRUTH_VALUE_UNKNOWN: + result = consider_sys_platform(expr, options.platform) + if result == TRUTH_VALUE_UNKNOWN: + if name == "PY2": + result = ALWAYS_FALSE + elif name == "PY3": + result = ALWAYS_TRUE + elif name == "MYPY" or name == "TYPE_CHECKING": + result = MYPY_TRUE + elif name in options.always_true: + result = ALWAYS_TRUE + elif name in options.always_false: + result = ALWAYS_FALSE + if negated: + result = inverted_truth_mapping[result] + return result + + +def infer_pattern_value(pattern: Pattern) -> int: + if isinstance(pattern, AsPattern) and pattern.pattern is None: + return ALWAYS_TRUE + elif isinstance(pattern, OrPattern) and any( + infer_pattern_value(p) == ALWAYS_TRUE for p in pattern.patterns + ): + return ALWAYS_TRUE + else: + return TRUTH_VALUE_UNKNOWN + + +def consider_sys_version_info(expr: Expression, pyversion: tuple[int, ...]) -> int: + """Consider whether expr is a comparison involving sys.version_info. + + Return ALWAYS_TRUE, ALWAYS_FALSE, or TRUTH_VALUE_UNKNOWN. + """ + # Cases supported: + # - sys.version_info[] + # - sys.version_info[:] + # - sys.version_info + # (in this case must be >, >=, <, <=, but cannot be ==, !=) + if not isinstance(expr, ComparisonExpr): + return TRUTH_VALUE_UNKNOWN + # Let's not yet support chained comparisons. + if len(expr.operators) > 1: + return TRUTH_VALUE_UNKNOWN + op = expr.operators[0] + if op not in ("==", "!=", "<=", ">=", "<", ">"): + return TRUTH_VALUE_UNKNOWN + + index = contains_sys_version_info(expr.operands[0]) + thing = contains_int_or_tuple_of_ints(expr.operands[1]) + if index is None or thing is None: + index = contains_sys_version_info(expr.operands[1]) + thing = contains_int_or_tuple_of_ints(expr.operands[0]) + op = reverse_op[op] + if isinstance(index, int) and isinstance(thing, int): + # sys.version_info[i] k + if 0 <= index <= 1: + return fixed_comparison(pyversion[index], op, thing) + else: + return TRUTH_VALUE_UNKNOWN + elif isinstance(index, tuple) and isinstance(thing, tuple): + lo, hi = index + if lo is None: + lo = 0 + if hi is None: + hi = 2 + if 0 <= lo < hi <= 2: + val = pyversion[lo:hi] + if len(val) == len(thing) or len(val) > len(thing) and op not in ("==", "!="): + return fixed_comparison(val, op, thing) + return TRUTH_VALUE_UNKNOWN + + +def consider_sys_platform(expr: Expression, platform: str) -> int: + """Consider whether expr is a comparison involving sys.platform. + + Return ALWAYS_TRUE, ALWAYS_FALSE, or TRUTH_VALUE_UNKNOWN. + """ + # Cases supported: + # - sys.platform == 'posix' + # - sys.platform != 'win32' + # - sys.platform.startswith('win') + if isinstance(expr, ComparisonExpr): + # Let's not yet support chained comparisons. + if len(expr.operators) > 1: + return TRUTH_VALUE_UNKNOWN + op = expr.operators[0] + if op not in ("==", "!="): + return TRUTH_VALUE_UNKNOWN + if not is_sys_attr(expr.operands[0], "platform"): + return TRUTH_VALUE_UNKNOWN + right = expr.operands[1] + if not isinstance(right, StrExpr): + return TRUTH_VALUE_UNKNOWN + return fixed_comparison(platform, op, right.value) + elif isinstance(expr, CallExpr): + if not isinstance(expr.callee, MemberExpr): + return TRUTH_VALUE_UNKNOWN + if len(expr.args) != 1 or not isinstance(expr.args[0], StrExpr): + return TRUTH_VALUE_UNKNOWN + if not is_sys_attr(expr.callee.expr, "platform"): + return TRUTH_VALUE_UNKNOWN + if expr.callee.name != "startswith": + return TRUTH_VALUE_UNKNOWN + if platform.startswith(expr.args[0].value): + return ALWAYS_TRUE + else: + return ALWAYS_FALSE + else: + return TRUTH_VALUE_UNKNOWN + + +Targ = TypeVar("Targ", int, str, Tuple[int, ...]) + + +def fixed_comparison(left: Targ, op: str, right: Targ) -> int: + rmap = {False: ALWAYS_FALSE, True: ALWAYS_TRUE} + if op == "==": + return rmap[left == right] + if op == "!=": + return rmap[left != right] + if op == "<=": + return rmap[left <= right] + if op == ">=": + return rmap[left >= right] + if op == "<": + return rmap[left < right] + if op == ">": + return rmap[left > right] + return TRUTH_VALUE_UNKNOWN + + +def contains_int_or_tuple_of_ints(expr: Expression) -> None | int | tuple[int, ...]: + if isinstance(expr, IntExpr): + return expr.value + if isinstance(expr, TupleExpr): + if literal(expr) == LITERAL_YES: + thing = [] + for x in expr.items: + if not isinstance(x, IntExpr): + return None + thing.append(x.value) + return tuple(thing) + return None + + +def contains_sys_version_info(expr: Expression) -> None | int | tuple[int | None, int | None]: + if is_sys_attr(expr, "version_info"): + return (None, None) # Same as sys.version_info[:] + if isinstance(expr, IndexExpr) and is_sys_attr(expr.base, "version_info"): + index = expr.index + if isinstance(index, IntExpr): + return index.value + if isinstance(index, SliceExpr): + if index.stride is not None: + if not isinstance(index.stride, IntExpr) or index.stride.value != 1: + return None + begin = end = None + if index.begin_index is not None: + if not isinstance(index.begin_index, IntExpr): + return None + begin = index.begin_index.value + if index.end_index is not None: + if not isinstance(index.end_index, IntExpr): + return None + end = index.end_index.value + return (begin, end) + return None + + +def is_sys_attr(expr: Expression, name: str) -> bool: + # TODO: This currently doesn't work with code like this: + # - import sys as _sys + # - from sys import version_info + if isinstance(expr, MemberExpr) and expr.name == name: + if isinstance(expr.expr, NameExpr) and expr.expr.name == "sys": + # TODO: Guard against a local named sys, etc. + # (Though later passes will still do most checking.) + return True + return False + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def mark_block_mypy_only(block: Block) -> None: + block.accept(MarkImportsMypyOnlyVisitor()) + + +class MarkImportsMypyOnlyVisitor(TraverserVisitor): + """Visitor that sets is_mypy_only (which affects priority).""" + + def visit_import(self, node: Import) -> None: + node.is_mypy_only = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_mypy_only = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_mypy_only = True + + def visit_func_def(self, node: FuncDef) -> None: + node.is_mypy_only = True diff --git a/mypy/refinfo.py b/mypy/refinfo.py new file mode 100644 index 000000000000..a5b92832bb7e --- /dev/null +++ b/mypy/refinfo.py @@ -0,0 +1,92 @@ +"""Find line-level reference information from a mypy AST (undocumented feature)""" + +from __future__ import annotations + +from mypy.nodes import ( + LDEF, + Expression, + FuncDef, + MemberExpr, + MypyFile, + NameExpr, + RefExpr, + SymbolNode, + TypeInfo, +) +from mypy.traverser import TraverserVisitor +from mypy.typeops import tuple_fallback +from mypy.types import ( + FunctionLike, + Instance, + TupleType, + Type, + TypeType, + TypeVarLikeType, + get_proper_type, +) + + +class RefInfoVisitor(TraverserVisitor): + def __init__(self, type_map: dict[Expression, Type]) -> None: + super().__init__() + self.type_map = type_map + self.data: list[dict[str, object]] = [] + + def visit_name_expr(self, expr: NameExpr) -> None: + super().visit_name_expr(expr) + self.record_ref_expr(expr) + + def visit_member_expr(self, expr: MemberExpr) -> None: + super().visit_member_expr(expr) + self.record_ref_expr(expr) + + def visit_func_def(self, func: FuncDef) -> None: + if func.expanded: + for item in func.expanded: + if isinstance(item, FuncDef): + super().visit_func_def(item) + else: + super().visit_func_def(func) + + def record_ref_expr(self, expr: RefExpr) -> None: + fullname = None + if expr.kind != LDEF and "." in expr.fullname: + fullname = expr.fullname + elif isinstance(expr, MemberExpr): + typ = self.type_map.get(expr.expr) + sym = None + if isinstance(expr.expr, RefExpr): + sym = expr.expr.node + if typ: + tfn = type_fullname(typ, sym) + if tfn: + fullname = f"{tfn}.{expr.name}" + if not fullname: + fullname = f"*.{expr.name}" + if fullname is not None: + self.data.append({"line": expr.line, "column": expr.column, "target": fullname}) + + +def type_fullname(typ: Type, node: SymbolNode | None = None) -> str | None: + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return typ.type.fullname + elif isinstance(typ, TypeType): + return type_fullname(typ.item) + elif isinstance(typ, FunctionLike) and typ.is_type_obj(): + if isinstance(node, TypeInfo): + return node.fullname + return type_fullname(typ.fallback) + elif isinstance(typ, TupleType): + return type_fullname(tuple_fallback(typ)) + elif isinstance(typ, TypeVarLikeType): + return type_fullname(typ.upper_bound) + return None + + +def get_undocumented_ref_info_json( + tree: MypyFile, type_map: dict[Expression, Type] +) -> list[dict[str, object]]: + visitor = RefInfoVisitor(type_map) + tree.accept(visitor) + return visitor.data diff --git a/mypy/renaming.py b/mypy/renaming.py new file mode 100644 index 000000000000..8db336205960 --- /dev/null +++ b/mypy/renaming.py @@ -0,0 +1,568 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import Final, Iterator + +from mypy.nodes import ( + AssignmentStmt, + Block, + BreakStmt, + ClassDef, + ContinueStmt, + ForStmt, + FuncDef, + Import, + ImportAll, + ImportFrom, + IndexExpr, + ListExpr, + Lvalue, + MatchStmt, + MemberExpr, + MypyFile, + NameExpr, + StarExpr, + TryStmt, + TupleExpr, + WhileStmt, + WithStmt, +) +from mypy.patterns import AsPattern +from mypy.traverser import TraverserVisitor + +# Scope kinds +FILE: Final = 0 +FUNCTION: Final = 1 +CLASS: Final = 2 + + +class VariableRenameVisitor(TraverserVisitor): + """Rename variables to allow redefinition of variables. + + For example, consider this code: + + x = 0 + f(x) + + x = "a" + g(x) + + It will be transformed like this: + + x' = 0 + f(x') + + x = "a" + g(x) + + There will be two independent variables (x' and x) that will have separate + inferred types. The publicly exposed variant will get the non-suffixed name. + This is the last definition at module top level and the first definition + (argument) within a function. + + Renaming only happens for assignments within the same block. Renaming is + performed before semantic analysis, immediately after parsing. + + The implementation performs a rudimentary static analysis. The analysis is + overly conservative to keep things simple. + """ + + def __init__(self) -> None: + # Counter for labeling new blocks + self.block_id = 0 + # Number of surrounding try statements that disallow variable redefinition + self.disallow_redef_depth = 0 + # Number of surrounding loop statements + self.loop_depth = 0 + # Map block id to loop depth. + self.block_loop_depth: dict[int, int] = {} + # Stack of block ids being processed. + self.blocks: list[int] = [] + # List of scopes; each scope maps short (unqualified) name to block id. + self.var_blocks: list[dict[str, int]] = [] + + # References to variables that we may need to rename. List of + # scopes; each scope is a mapping from name to list of collections + # of names that refer to the same logical variable. + self.refs: list[dict[str, list[list[NameExpr]]]] = [] + # Number of reads of the most recent definition of a variable (per scope) + self.num_reads: list[dict[str, int]] = [] + # Kinds of nested scopes (FILE, FUNCTION or CLASS) + self.scope_kinds: list[int] = [] + + def visit_mypy_file(self, file_node: MypyFile) -> None: + """Rename variables within a file. + + This is the main entry point to this class. + """ + self.clear() + with self.enter_scope(FILE), self.enter_block(): + for d in file_node.defs: + d.accept(self) + + def visit_func_def(self, fdef: FuncDef) -> None: + # Conservatively do not allow variable defined before a function to + # be redefined later, since function could refer to either definition. + self.reject_redefinition_of_vars_in_scope() + + with self.enter_scope(FUNCTION), self.enter_block(): + for arg in fdef.arguments: + name = arg.variable.name + # 'self' can't be redefined since it's special as it allows definition of + # attributes. 'cls' can't be used to define attributes so we can ignore it. + can_be_redefined = name != "self" # TODO: Proper check + self.record_assignment(arg.variable.name, can_be_redefined) + self.handle_arg(name) + + for stmt in fdef.body.body: + stmt.accept(self) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.reject_redefinition_of_vars_in_scope() + with self.enter_scope(CLASS): + super().visit_class_def(cdef) + + def visit_block(self, block: Block) -> None: + with self.enter_block(): + super().visit_block(block) + + def visit_while_stmt(self, stmt: WhileStmt) -> None: + with self.enter_loop(): + super().visit_while_stmt(stmt) + + def visit_for_stmt(self, stmt: ForStmt) -> None: + stmt.expr.accept(self) + self.analyze_lvalue(stmt.index, True) + # Also analyze as non-lvalue so that every for loop index variable is assumed to be read. + stmt.index.accept(self) + with self.enter_loop(): + stmt.body.accept(self) + if stmt.else_body: + stmt.else_body.accept(self) + + def visit_break_stmt(self, stmt: BreakStmt) -> None: + self.reject_redefinition_of_vars_in_loop() + + def visit_continue_stmt(self, stmt: ContinueStmt) -> None: + self.reject_redefinition_of_vars_in_loop() + + def visit_try_stmt(self, stmt: TryStmt) -> None: + # Variables defined by a try statement get special treatment in the + # type checker which allows them to be always redefined, so no need to + # do renaming here. + with self.enter_try(): + super().visit_try_stmt(stmt) + + def visit_with_stmt(self, stmt: WithStmt) -> None: + for expr in stmt.expr: + expr.accept(self) + for target in stmt.target: + if target is not None: + self.analyze_lvalue(target) + # We allow redefinitions in the body of a with statement for + # convenience. This is unsafe since with statements can affect control + # flow by catching exceptions, but this is rare except for + # assertRaises() and other similar functions, where the exception is + # raised by the last statement in the body, which usually isn't a + # problem. + stmt.body.accept(self) + + def visit_import(self, imp: Import) -> None: + for id, as_id in imp.ids: + self.record_assignment(as_id or id, False) + + def visit_import_from(self, imp: ImportFrom) -> None: + for id, as_id in imp.names: + self.record_assignment(as_id or id, False) + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + s.rvalue.accept(self) + for lvalue in s.lvalues: + self.analyze_lvalue(lvalue) + + def visit_match_stmt(self, s: MatchStmt) -> None: + s.subject.accept(self) + for i in range(len(s.patterns)): + with self.enter_block(): + s.patterns[i].accept(self) + guard = s.guards[i] + if guard is not None: + guard.accept(self) + # We already entered a block, so visit this block's statements directly + for stmt in s.bodies[i].body: + stmt.accept(self) + + def visit_capture_pattern(self, p: AsPattern) -> None: + if p.name is not None: + self.analyze_lvalue(p.name) + + def analyze_lvalue(self, lvalue: Lvalue, is_nested: bool = False) -> None: + """Process assignment; in particular, keep track of (re)defined names. + + Args: + is_nested: True for non-outermost Lvalue in a multiple assignment such as + "x, y = ..." + """ + if isinstance(lvalue, NameExpr): + name = lvalue.name + is_new = self.record_assignment(name, True) + if is_new: + self.handle_def(lvalue) + else: + self.handle_refine(lvalue) + if is_nested: + # This allows these to be redefined freely even if never read. Multiple + # assignment like "x, _ _ = y" defines dummy variables that are never read. + self.handle_ref(lvalue) + elif isinstance(lvalue, (ListExpr, TupleExpr)): + for item in lvalue.items: + self.analyze_lvalue(item, is_nested=True) + elif isinstance(lvalue, MemberExpr): + lvalue.expr.accept(self) + elif isinstance(lvalue, IndexExpr): + lvalue.base.accept(self) + lvalue.index.accept(self) + elif isinstance(lvalue, StarExpr): + # Propagate is_nested since in a typical use case like "x, *rest = ..." 'rest' may + # be freely reused. + self.analyze_lvalue(lvalue.expr, is_nested=is_nested) + + def visit_name_expr(self, expr: NameExpr) -> None: + self.handle_ref(expr) + + # Helpers for renaming references + + def handle_arg(self, name: str) -> None: + """Store function argument.""" + self.refs[-1][name] = [[]] + self.num_reads[-1][name] = 0 + + def handle_def(self, expr: NameExpr) -> None: + """Store new name definition.""" + name = expr.name + names = self.refs[-1].setdefault(name, []) + names.append([expr]) + self.num_reads[-1][name] = 0 + + def handle_refine(self, expr: NameExpr) -> None: + """Store assignment to an existing name (that replaces previous value, if any).""" + name = expr.name + if name in self.refs[-1]: + names = self.refs[-1][name] + if not names: + names.append([]) + names[-1].append(expr) + + def handle_ref(self, expr: NameExpr) -> None: + """Store reference to defined name.""" + name = expr.name + if name in self.refs[-1]: + names = self.refs[-1][name] + if not names: + names.append([]) + names[-1].append(expr) + num_reads = self.num_reads[-1] + num_reads[name] = num_reads.get(name, 0) + 1 + + def flush_refs(self) -> None: + """Rename all references within the current scope. + + This will be called at the end of a scope. + """ + is_func = self.scope_kinds[-1] == FUNCTION + for refs in self.refs[-1].values(): + if len(refs) == 1: + # Only one definition -- no renaming needed. + continue + if is_func: + # In a function, don't rename the first definition, as it + # may be an argument that must preserve the name. + to_rename = refs[1:] + else: + # At module top level, don't rename the final definition, + # as it will be publicly visible outside the module. + to_rename = refs[:-1] + for i, item in enumerate(to_rename): + rename_refs(item, i) + self.refs.pop() + + # Helpers for determining which assignments define new variables + + def clear(self) -> None: + self.blocks = [] + self.var_blocks = [] + + @contextmanager + def enter_block(self) -> Iterator[None]: + self.block_id += 1 + self.blocks.append(self.block_id) + self.block_loop_depth[self.block_id] = self.loop_depth + try: + yield + finally: + self.blocks.pop() + + @contextmanager + def enter_try(self) -> Iterator[None]: + self.disallow_redef_depth += 1 + try: + yield + finally: + self.disallow_redef_depth -= 1 + + @contextmanager + def enter_loop(self) -> Iterator[None]: + self.loop_depth += 1 + try: + yield + finally: + self.loop_depth -= 1 + + def current_block(self) -> int: + return self.blocks[-1] + + @contextmanager + def enter_scope(self, kind: int) -> Iterator[None]: + self.var_blocks.append({}) + self.refs.append({}) + self.num_reads.append({}) + self.scope_kinds.append(kind) + try: + yield + finally: + self.flush_refs() + self.var_blocks.pop() + self.num_reads.pop() + self.scope_kinds.pop() + + def is_nested(self) -> int: + return len(self.var_blocks) > 1 + + def reject_redefinition_of_vars_in_scope(self) -> None: + """Make it impossible to redefine defined variables in the current scope. + + This is used if we encounter a function definition that + can make it ambiguous which definition is live. Example: + + x = 0 + + def f() -> int: + return x + + x = '' # Error -- cannot redefine x across function definition + """ + var_blocks = self.var_blocks[-1] + for key in var_blocks: + var_blocks[key] = -1 + + def reject_redefinition_of_vars_in_loop(self) -> None: + """Reject redefinition of variables in the innermost loop. + + If there is an early exit from a loop, there may be ambiguity about which + value may escape the loop. Example where this matters: + + while f(): + x = 0 + if g(): + break + x = '' # Error -- not a redefinition + reveal_type(x) # int + + This method ensures that the second assignment to 'x' doesn't introduce a new + variable. + """ + var_blocks = self.var_blocks[-1] + for key, block in var_blocks.items(): + if self.block_loop_depth.get(block) == self.loop_depth: + var_blocks[key] = -1 + + def record_assignment(self, name: str, can_be_redefined: bool) -> bool: + """Record assignment to given name and return True if it defines a new variable. + + Args: + can_be_redefined: If True, allows assignment in the same block to redefine + this name (if this is a new definition) + """ + if self.num_reads[-1].get(name, -1) == 0: + # Only set, not read, so no reason to redefine + return False + if self.disallow_redef_depth > 0: + # Can't redefine within try/with a block. + can_be_redefined = False + block = self.current_block() + var_blocks = self.var_blocks[-1] + if name not in var_blocks: + # New definition in this scope. + if can_be_redefined: + # Store the block where this was defined to allow redefinition in + # the same block only. + var_blocks[name] = block + else: + # This doesn't support arbitrary redefinition. + var_blocks[name] = -1 + return True + elif var_blocks[name] == block: + # Redefinition -- defines a new variable with the same name. + return True + else: + # Assigns to an existing variable. + return False + + +class LimitedVariableRenameVisitor(TraverserVisitor): + """Perform some limited variable renaming in with statements. + + This allows reusing a variable in multiple with statements with + different types. For example, the two instances of 'x' can have + incompatible types: + + with C() as x: + f(x) + with D() as x: + g(x) + + The above code gets renamed conceptually into this (not valid Python!): + + with C() as x': + f(x') + with D() as x: + g(x) + + If there's a reference to a variable defined in 'with' outside the + statement, or if there's any trickiness around variable visibility + (e.g. function definitions), we give up and won't perform renaming. + + The main use case is to allow binding both readable and writable + binary files into the same variable. These have different types: + + with open(fnam, 'rb') as f: ... + with open(fnam, 'wb') as f: ... + """ + + def __init__(self) -> None: + # Short names of variables bound in with statements using "as" + # in a surrounding scope + self.bound_vars: list[str] = [] + # Stack of names that can't be safely renamed, per scope ('*' means that + # no names can be renamed) + self.skipped: list[set[str]] = [] + # References to variables that we may need to rename. Stack of + # scopes; each scope is a mapping from name to list of collections + # of names that refer to the same logical variable. + self.refs: list[dict[str, list[list[NameExpr]]]] = [] + + def visit_mypy_file(self, file_node: MypyFile) -> None: + """Rename variables within a file. + + This is the main entry point to this class. + """ + with self.enter_scope(): + for d in file_node.defs: + d.accept(self) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.reject_redefinition_of_vars_in_scope() + with self.enter_scope(): + for arg in fdef.arguments: + self.record_skipped(arg.variable.name) + super().visit_func_def(fdef) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.reject_redefinition_of_vars_in_scope() + with self.enter_scope(): + super().visit_class_def(cdef) + + def visit_with_stmt(self, stmt: WithStmt) -> None: + for expr in stmt.expr: + expr.accept(self) + old_len = len(self.bound_vars) + for target in stmt.target: + if target is not None: + self.analyze_lvalue(target) + for target in stmt.target: + if target: + target.accept(self) + stmt.body.accept(self) + + while len(self.bound_vars) > old_len: + self.bound_vars.pop() + + def analyze_lvalue(self, lvalue: Lvalue) -> None: + if isinstance(lvalue, NameExpr): + name = lvalue.name + if name in self.bound_vars: + # Name bound in a surrounding with statement, so it can be renamed + self.visit_name_expr(lvalue) + else: + var_info = self.refs[-1] + if name not in var_info: + var_info[name] = [] + var_info[name].append([]) + self.bound_vars.append(name) + elif isinstance(lvalue, (ListExpr, TupleExpr)): + for item in lvalue.items: + self.analyze_lvalue(item) + elif isinstance(lvalue, MemberExpr): + lvalue.expr.accept(self) + elif isinstance(lvalue, IndexExpr): + lvalue.base.accept(self) + lvalue.index.accept(self) + elif isinstance(lvalue, StarExpr): + self.analyze_lvalue(lvalue.expr) + + def visit_import(self, imp: Import) -> None: + # We don't support renaming imports + for id, as_id in imp.ids: + self.record_skipped(as_id or id) + + def visit_import_from(self, imp: ImportFrom) -> None: + # We don't support renaming imports + for id, as_id in imp.names: + self.record_skipped(as_id or id) + + def visit_import_all(self, imp: ImportAll) -> None: + # Give up, since we don't know all imported names yet + self.reject_redefinition_of_vars_in_scope() + + def visit_name_expr(self, expr: NameExpr) -> None: + name = expr.name + if name in self.bound_vars: + # Record reference so that it can be renamed later + for scope in reversed(self.refs): + if name in scope: + scope[name][-1].append(expr) + else: + self.record_skipped(name) + + @contextmanager + def enter_scope(self) -> Iterator[None]: + self.skipped.append(set()) + self.refs.append({}) + yield None + self.flush_refs() + + def reject_redefinition_of_vars_in_scope(self) -> None: + self.record_skipped("*") + + def record_skipped(self, name: str) -> None: + self.skipped[-1].add(name) + + def flush_refs(self) -> None: + ref_dict = self.refs.pop() + skipped = self.skipped.pop() + if "*" not in skipped: + for name, refs in ref_dict.items(): + if len(refs) <= 1 or name in skipped: + continue + # At module top level we must not rename the final definition, + # as it may be publicly visible + to_rename = refs[:-1] + for i, item in enumerate(to_rename): + rename_refs(item, i) + + +def rename_refs(names: list[NameExpr], index: int) -> None: + name = names[0].name + new_name = name + "'" * (index + 1) + for expr in names: + expr.name = new_name diff --git a/mypy/replacetvars.py b/mypy/replacetvars.py deleted file mode 100644 index fa6fe3a2e9a9..000000000000 --- a/mypy/replacetvars.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Type operations""" - -import typing - -from mypy.lex import Token -from mypy.types import Type, AnyType, NoneTyp, TypeTranslator, TypeVar -from mypy.typerepr import AnyRepr - - -def replace_type_vars(typ: Type, func_tvars: bool = True) -> Type: - """Replace type variable references in a type with the Any type. If - func_tvars is false, only replace instance type variables. - """ - return typ.accept(ReplaceTypeVarsVisitor(func_tvars)) - - -class ReplaceTypeVarsVisitor(TypeTranslator): - # Only override type variable handling; otherwise perform an indentity - # transformation. - - func_tvars = False - - def __init__(self, func_tvars: bool) -> None: - self.func_tvars = func_tvars - - def visit_type_var(self, t: TypeVar) -> Type: - if t.id > 0 or self.func_tvars: - if t.repr is not None: - # Give a representation for the dynamic type. - tok = Token('Any') - tok.pre = t.repr.name.pre - return AnyType(t.line, AnyRepr(tok)) - else: - return AnyType() - else: - return t - - -def replace_func_type_vars(typ: Type, target_type: Type) -> Type: - """Replace function type variables in a type with the target type.""" - return typ.accept(ReplaceFuncTypeVarsVisitor(target_type)) - - -class ReplaceFuncTypeVarsVisitor(TypeTranslator): - def __init__(self, target_type: Type) -> None: - self.target_type = target_type - - def visit_type_var(self, t: TypeVar) -> Type: - if t.id < 0: - return self.target_type - else: - return t diff --git a/mypy/report.py b/mypy/report.py new file mode 100644 index 000000000000..73942b6c5ae3 --- /dev/null +++ b/mypy/report.py @@ -0,0 +1,926 @@ +"""Classes for producing HTML reports about imprecision.""" + +from __future__ import annotations + +import collections +import itertools +import json +import os +import shutil +import sys +import time +import tokenize +from abc import ABCMeta, abstractmethod +from operator import attrgetter +from typing import Any, Callable, Dict, Final, Iterator, Tuple +from typing_extensions import TypeAlias as _TypeAlias +from urllib.request import pathname2url + +from mypy import stats +from mypy.defaults import REPORTER_NAMES +from mypy.nodes import Expression, FuncDef, MypyFile +from mypy.options import Options +from mypy.traverser import TraverserVisitor +from mypy.types import Type, TypeOfAny +from mypy.version import __version__ + +try: + from lxml import etree # type: ignore[import-untyped] + + LXML_INSTALLED = True +except ImportError: + LXML_INSTALLED = False + +type_of_any_name_map: Final[collections.OrderedDict[int, str]] = collections.OrderedDict( + [ + (TypeOfAny.unannotated, "Unannotated"), + (TypeOfAny.explicit, "Explicit"), + (TypeOfAny.from_unimported_type, "Unimported"), + (TypeOfAny.from_omitted_generics, "Omitted Generics"), + (TypeOfAny.from_error, "Error"), + (TypeOfAny.special_form, "Special Form"), + (TypeOfAny.implementation_artifact, "Implementation Artifact"), + ] +) + +ReporterClasses: _TypeAlias = Dict[ + str, Tuple[Callable[["Reports", str], "AbstractReporter"], bool] +] + +reporter_classes: Final[ReporterClasses] = {} + + +class Reports: + def __init__(self, data_dir: str, report_dirs: dict[str, str]) -> None: + self.data_dir = data_dir + self.reporters: list[AbstractReporter] = [] + self.named_reporters: dict[str, AbstractReporter] = {} + + for report_type, report_dir in sorted(report_dirs.items()): + self.add_report(report_type, report_dir) + + def add_report(self, report_type: str, report_dir: str) -> AbstractReporter: + try: + return self.named_reporters[report_type] + except KeyError: + pass + reporter_cls, needs_lxml = reporter_classes[report_type] + if needs_lxml and not LXML_INSTALLED: + print( + ( + "You must install the lxml package before you can run mypy" + " with `--{}-report`.\n" + "You can do this with `python3 -m pip install lxml`." + ).format(report_type), + file=sys.stderr, + ) + raise ImportError + reporter = reporter_cls(self, report_dir) + self.reporters.append(reporter) + self.named_reporters[report_type] = reporter + return reporter + + def file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + for reporter in self.reporters: + reporter.on_file(tree, modules, type_map, options) + + def finish(self) -> None: + for reporter in self.reporters: + reporter.on_finish() + + +class AbstractReporter(metaclass=ABCMeta): + def __init__(self, reports: Reports, output_dir: str) -> None: + self.output_dir = output_dir + if output_dir != "": + os.makedirs(output_dir, exist_ok=True) + + @abstractmethod + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + pass + + @abstractmethod + def on_finish(self) -> None: + pass + + +def register_reporter( + report_name: str, + reporter: Callable[[Reports, str], AbstractReporter], + needs_lxml: bool = False, +) -> None: + reporter_classes[report_name] = (reporter, needs_lxml) + + +def alias_reporter(source_reporter: str, target_reporter: str) -> None: + reporter_classes[target_reporter] = reporter_classes[source_reporter] + + +def should_skip_path(path: str) -> bool: + if stats.is_special_module(path): + return True + if path.startswith(".."): + return True + if "stubs" in path.split("/") or "stubs" in path.split(os.sep): + return True + return False + + +def iterate_python_lines(path: str) -> Iterator[tuple[int, str]]: + """Return an iterator over (line number, line text) from a Python file.""" + try: + with tokenize.open(path) as input_file: + yield from enumerate(input_file, 1) + except IsADirectoryError: + # can happen with namespace packages + pass + + +class FuncCounterVisitor(TraverserVisitor): + def __init__(self) -> None: + super().__init__() + self.counts = [0, 0] + + def visit_func_def(self, defn: FuncDef) -> None: + self.counts[defn.type is not None] += 1 + + +class LineCountReporter(AbstractReporter): + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + self.counts: dict[str, tuple[int, int, int, int]] = {} + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + # Count physical lines. This assumes the file's encoding is a + # superset of ASCII (or at least uses \n in its line endings). + try: + with open(tree.path, "rb") as f: + physical_lines = len(f.readlines()) + except IsADirectoryError: + # can happen with namespace packages + physical_lines = 0 + + func_counter = FuncCounterVisitor() + tree.accept(func_counter) + unannotated_funcs, annotated_funcs = func_counter.counts + total_funcs = annotated_funcs + unannotated_funcs + + # Don't count lines or functions as annotated if they have their errors ignored. + if options.ignore_errors: + annotated_funcs = 0 + + imputed_annotated_lines = ( + physical_lines * annotated_funcs // total_funcs if total_funcs else physical_lines + ) + + self.counts[tree._fullname] = ( + imputed_annotated_lines, + physical_lines, + annotated_funcs, + total_funcs, + ) + + def on_finish(self) -> None: + counts: list[tuple[tuple[int, int, int, int], str]] = sorted( + ((c, p) for p, c in self.counts.items()), reverse=True + ) + total_counts = tuple(sum(c[i] for c, p in counts) for i in range(4)) + with open(os.path.join(self.output_dir, "linecount.txt"), "w") as f: + f.write("{:7} {:7} {:6} {:6} total\n".format(*total_counts)) + for c, p in counts: + f.write(f"{c[0]:7} {c[1]:7} {c[2]:6} {c[3]:6} {p}\n") + + +register_reporter("linecount", LineCountReporter) + + +class AnyExpressionsReporter(AbstractReporter): + """Report frequencies of different kinds of Any types.""" + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + self.counts: dict[str, tuple[int, int]] = {} + self.any_types_counter: dict[str, collections.Counter[int]] = {} + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + visitor = stats.StatisticsVisitor( + inferred=True, + filename=tree.fullname, + modules=modules, + typemap=type_map, + all_nodes=True, + visit_untyped_defs=False, + ) + tree.accept(visitor) + self.any_types_counter[tree.fullname] = visitor.type_of_any_counter + num_unanalyzed_lines = list(visitor.line_map.values()).count(stats.TYPE_UNANALYZED) + # count each line of dead code as one expression of type "Any" + num_any = visitor.num_any_exprs + num_unanalyzed_lines + num_total = visitor.num_imprecise_exprs + visitor.num_precise_exprs + num_any + if num_total > 0: + self.counts[tree.fullname] = (num_any, num_total) + + def on_finish(self) -> None: + self._report_any_exprs() + self._report_types_of_anys() + + def _write_out_report( + self, filename: str, header: list[str], rows: list[list[str]], footer: list[str] + ) -> None: + row_len = len(header) + assert all(len(row) == row_len for row in rows + [header, footer]) + min_column_distance = 3 # minimum distance between numbers in two columns + widths = [-1] * row_len + for row in rows + [header, footer]: + for i, value in enumerate(row): + widths[i] = max(widths[i], len(value)) + for i, w in enumerate(widths): + # Do not add min_column_distance to the first column. + if i > 0: + widths[i] = w + min_column_distance + with open(os.path.join(self.output_dir, filename), "w") as f: + header_str = ("{:>{}}" * len(widths)).format(*itertools.chain(*zip(header, widths))) + separator = "-" * len(header_str) + f.write(header_str + "\n") + f.write(separator + "\n") + for row_values in rows: + r = ("{:>{}}" * len(widths)).format(*itertools.chain(*zip(row_values, widths))) + f.write(r + "\n") + f.write(separator + "\n") + footer_str = ("{:>{}}" * len(widths)).format(*itertools.chain(*zip(footer, widths))) + f.write(footer_str + "\n") + + def _report_any_exprs(self) -> None: + total_any = sum(num_any for num_any, _ in self.counts.values()) + total_expr = sum(total for _, total in self.counts.values()) + total_coverage = 100.0 + if total_expr > 0: + total_coverage = (float(total_expr - total_any) / float(total_expr)) * 100 + + column_names = ["Name", "Anys", "Exprs", "Coverage"] + rows: list[list[str]] = [] + for filename in sorted(self.counts): + (num_any, num_total) = self.counts[filename] + coverage = (float(num_total - num_any) / float(num_total)) * 100 + coverage_str = f"{coverage:.2f}%" + rows.append([filename, str(num_any), str(num_total), coverage_str]) + rows.sort(key=lambda x: x[0]) + total_row = ["Total", str(total_any), str(total_expr), f"{total_coverage:.2f}%"] + self._write_out_report("any-exprs.txt", column_names, rows, total_row) + + def _report_types_of_anys(self) -> None: + total_counter: collections.Counter[int] = collections.Counter() + for counter in self.any_types_counter.values(): + for any_type, value in counter.items(): + total_counter[any_type] += value + file_column_name = "Name" + total_row_name = "Total" + column_names = [file_column_name] + list(type_of_any_name_map.values()) + rows: list[list[str]] = [] + for filename, counter in self.any_types_counter.items(): + rows.append([filename] + [str(counter[typ]) for typ in type_of_any_name_map]) + rows.sort(key=lambda x: x[0]) + total_row = [total_row_name] + [str(total_counter[typ]) for typ in type_of_any_name_map] + self._write_out_report("types-of-anys.txt", column_names, rows, total_row) + + +register_reporter("any-exprs", AnyExpressionsReporter) + + +class LineCoverageVisitor(TraverserVisitor): + def __init__(self, source: list[str]) -> None: + self.source = source + + # For each line of source, we maintain a pair of + # * the indentation level of the surrounding function + # (-1 if not inside a function), and + # * whether the surrounding function is typed. + # Initially, everything is covered at indentation level -1. + self.lines_covered = [(-1, True) for l in source] + + # The Python AST has position information for the starts of + # elements, but not for their ends. Fortunately the + # indentation-based syntax makes it pretty easy to find where a + # block ends without doing any real parsing. + + # TODO: Handle line continuations (explicit and implicit) and + # multi-line string literals. (But at least line continuations + # are normally more indented than their surrounding block anyways, + # by PEP 8.) + + def indentation_level(self, line_number: int) -> int | None: + """Return the indentation of a line of the source (specified by + zero-indexed line number). Returns None for blank lines or comments.""" + line = self.source[line_number] + indent = 0 + for char in list(line): + if char == " ": + indent += 1 + elif char == "\t": + indent = 8 * ((indent + 8) // 8) + elif char == "#": + # Line is a comment; ignore it + return None + elif char == "\n": + # Line is entirely whitespace; ignore it + return None + # TODO line continuation (\) + else: + # Found a non-whitespace character + return indent + # Line is entirely whitespace, and at end of file + # with no trailing newline; ignore it + return None + + def visit_func_def(self, defn: FuncDef) -> None: + start_line = defn.line - 1 + start_indent = None + # When a function is decorated, sometimes the start line will point to + # whitespace or comments between the decorator and the function, so + # we have to look for the start. + while start_line < len(self.source): + start_indent = self.indentation_level(start_line) + if start_indent is not None: + break + start_line += 1 + # If we can't find the function give up and don't annotate anything. + # Our line numbers are not reliable enough to be asserting on. + if start_indent is None: + return + + cur_line = start_line + 1 + end_line = cur_line + # After this loop, function body will be lines [start_line, end_line) + while cur_line < len(self.source): + cur_indent = self.indentation_level(cur_line) + if cur_indent is None: + # Consume the line, but don't mark it as belonging to the function yet. + cur_line += 1 + elif cur_indent > start_indent: + # A non-blank line that belongs to the function. + cur_line += 1 + end_line = cur_line + else: + # We reached a line outside the function definition. + break + + is_typed = defn.type is not None + for line in range(start_line, end_line): + old_indent, _ = self.lines_covered[line] + # If there was an old indent level for this line, and the new + # level isn't increasing the indentation, ignore it. + # This is to be defensive against funniness in our line numbers, + # which are not always reliable. + if old_indent <= start_indent: + self.lines_covered[line] = (start_indent, is_typed) + + # Visit the body, in case there are nested functions + super().visit_func_def(defn) + + +class LineCoverageReporter(AbstractReporter): + """Exact line coverage reporter. + + This reporter writes a JSON dictionary with one field 'lines' to + the file 'coverage.json' in the specified report directory. The + value of that field is a dictionary which associates to each + source file's absolute pathname the list of line numbers that + belong to typed functions in that file. + """ + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + self.lines_covered: dict[str, list[int]] = {} + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + with open(tree.path) as f: + tree_source = f.readlines() + + coverage_visitor = LineCoverageVisitor(tree_source) + tree.accept(coverage_visitor) + + covered_lines = [] + for line_number, (_, typed) in enumerate(coverage_visitor.lines_covered): + if typed: + covered_lines.append(line_number + 1) + + self.lines_covered[os.path.abspath(tree.path)] = covered_lines + + def on_finish(self) -> None: + with open(os.path.join(self.output_dir, "coverage.json"), "w") as f: + json.dump({"lines": self.lines_covered}, f) + + +register_reporter("linecoverage", LineCoverageReporter) + + +class FileInfo: + def __init__(self, name: str, module: str) -> None: + self.name = name + self.module = module + self.counts = [0] * len(stats.precision_names) + + def total(self) -> int: + return sum(self.counts) + + def attrib(self) -> dict[str, str]: + return {name: str(val) for name, val in sorted(zip(stats.precision_names, self.counts))} + + +class MemoryXmlReporter(AbstractReporter): + """Internal reporter that generates XML in memory. + + This is used by all other XML-based reporters to avoid duplication. + """ + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + + self.xslt_html_path = os.path.join(reports.data_dir, "xml", "mypy-html.xslt") + self.xslt_txt_path = os.path.join(reports.data_dir, "xml", "mypy-txt.xslt") + self.css_html_path = os.path.join(reports.data_dir, "xml", "mypy-html.css") + xsd_path = os.path.join(reports.data_dir, "xml", "mypy.xsd") + self.schema = etree.XMLSchema(etree.parse(xsd_path)) + self.last_xml: Any | None = None + self.files: list[FileInfo] = [] + + # XML doesn't like control characters, but they are sometimes + # legal in source code (e.g. comments, string literals). + # Tabs (#x09) are allowed in XML content. + control_fixer: Final = str.maketrans("".join(chr(i) for i in range(32) if i != 9), "?" * 31) + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + self.last_xml = None + + try: + path = os.path.relpath(tree.path) + except ValueError: + return + + if should_skip_path(path) or os.path.isdir(path): + return # `path` can sometimes be a directory, see #11334 + + visitor = stats.StatisticsVisitor( + inferred=True, + filename=tree.fullname, + modules=modules, + typemap=type_map, + all_nodes=True, + ) + tree.accept(visitor) + + root = etree.Element("mypy-report-file", name=path, module=tree._fullname) + doc = etree.ElementTree(root) + file_info = FileInfo(path, tree._fullname) + + for lineno, line_text in iterate_python_lines(path): + status = visitor.line_map.get(lineno, stats.TYPE_EMPTY) + file_info.counts[status] += 1 + etree.SubElement( + root, + "line", + any_info=self._get_any_info_for_line(visitor, lineno), + content=line_text.rstrip("\n").translate(self.control_fixer), + number=str(lineno), + precision=stats.precision_names[status], + ) + # Assumes a layout similar to what XmlReporter uses. + xslt_path = os.path.relpath("mypy-html.xslt", path) + transform_pi = etree.ProcessingInstruction( + "xml-stylesheet", f'type="text/xsl" href="{pathname2url(xslt_path)}"' + ) + root.addprevious(transform_pi) + self.schema.assertValid(doc) + + self.last_xml = doc + self.files.append(file_info) + + @staticmethod + def _get_any_info_for_line(visitor: stats.StatisticsVisitor, lineno: int) -> str: + if lineno in visitor.any_line_map: + result = "Any Types on this line: " + counter: collections.Counter[int] = collections.Counter() + for typ in visitor.any_line_map[lineno]: + counter[typ.type_of_any] += 1 + for any_type, occurrences in counter.items(): + result += f"\n{type_of_any_name_map[any_type]} (x{occurrences})" + return result + else: + return "No Anys on this line!" + + def on_finish(self) -> None: + self.last_xml = None + # index_path = os.path.join(self.output_dir, 'index.xml') + output_files = sorted(self.files, key=lambda x: x.module) + + root = etree.Element("mypy-report-index", name="index") + doc = etree.ElementTree(root) + + for file_info in output_files: + etree.SubElement( + root, + "file", + file_info.attrib(), + module=file_info.module, + name=pathname2url(file_info.name), + total=str(file_info.total()), + ) + xslt_path = os.path.relpath("mypy-html.xslt", ".") + transform_pi = etree.ProcessingInstruction( + "xml-stylesheet", f'type="text/xsl" href="{pathname2url(xslt_path)}"' + ) + root.addprevious(transform_pi) + self.schema.assertValid(doc) + + self.last_xml = doc + + +register_reporter("memory-xml", MemoryXmlReporter, needs_lxml=True) + + +def get_line_rate(covered_lines: int, total_lines: int) -> str: + if total_lines == 0: + return str(1.0) + else: + return f"{covered_lines / total_lines:.4f}" + + +class CoberturaPackage: + """Container for XML and statistics mapping python modules to Cobertura package.""" + + def __init__(self, name: str) -> None: + self.name = name + self.classes: dict[str, Any] = {} + self.packages: dict[str, CoberturaPackage] = {} + self.total_lines = 0 + self.covered_lines = 0 + + def as_xml(self) -> Any: + package_element = etree.Element("package", complexity="1.0", name=self.name) + package_element.attrib["branch-rate"] = "0" + package_element.attrib["line-rate"] = get_line_rate(self.covered_lines, self.total_lines) + classes_element = etree.SubElement(package_element, "classes") + for class_name in sorted(self.classes): + classes_element.append(self.classes[class_name]) + self.add_packages(package_element) + return package_element + + def add_packages(self, parent_element: Any) -> None: + if self.packages: + packages_element = etree.SubElement(parent_element, "packages") + for package in sorted(self.packages.values(), key=attrgetter("name")): + packages_element.append(package.as_xml()) + + +class CoberturaXmlReporter(AbstractReporter): + """Reporter for generating Cobertura compliant XML.""" + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + + self.root = etree.Element("coverage", timestamp=str(int(time.time())), version=__version__) + self.doc = etree.ElementTree(self.root) + self.root_package = CoberturaPackage(".") + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + path = os.path.relpath(tree.path) + visitor = stats.StatisticsVisitor( + inferred=True, + filename=tree.fullname, + modules=modules, + typemap=type_map, + all_nodes=True, + ) + tree.accept(visitor) + + class_name = os.path.basename(path) + file_info = FileInfo(path, tree._fullname) + class_element = etree.Element("class", complexity="1.0", filename=path, name=class_name) + etree.SubElement(class_element, "methods") + lines_element = etree.SubElement(class_element, "lines") + + class_lines_covered = 0 + class_total_lines = 0 + for lineno, _ in iterate_python_lines(path): + status = visitor.line_map.get(lineno, stats.TYPE_EMPTY) + hits = 0 + branch = False + if status == stats.TYPE_EMPTY: + continue + class_total_lines += 1 + if status != stats.TYPE_ANY: + class_lines_covered += 1 + hits = 1 + if status == stats.TYPE_IMPRECISE: + branch = True + file_info.counts[status] += 1 + line_element = etree.SubElement( + lines_element, + "line", + branch=str(branch).lower(), + hits=str(hits), + number=str(lineno), + precision=stats.precision_names[status], + ) + if branch: + line_element.attrib["condition-coverage"] = "50% (1/2)" + class_element.attrib["branch-rate"] = "0" + class_element.attrib["line-rate"] = get_line_rate(class_lines_covered, class_total_lines) + # parent_module is set to whichever module contains this file. For most files, we want + # to simply strip the last element off of the module. But for __init__.py files, + # the module == the parent module. + parent_module = file_info.module.rsplit(".", 1)[0] + if file_info.name.endswith("__init__.py"): + parent_module = file_info.module + + if parent_module not in self.root_package.packages: + self.root_package.packages[parent_module] = CoberturaPackage(parent_module) + current_package = self.root_package.packages[parent_module] + packages_to_update = [self.root_package, current_package] + for package in packages_to_update: + package.total_lines += class_total_lines + package.covered_lines += class_lines_covered + current_package.classes[class_name] = class_element + + def on_finish(self) -> None: + self.root.attrib["line-rate"] = get_line_rate( + self.root_package.covered_lines, self.root_package.total_lines + ) + self.root.attrib["branch-rate"] = "0" + self.root.attrib["lines-covered"] = str(self.root_package.covered_lines) + self.root.attrib["lines-valid"] = str(self.root_package.total_lines) + sources = etree.SubElement(self.root, "sources") + source_element = etree.SubElement(sources, "source") + source_element.text = os.getcwd() + self.root_package.add_packages(self.root) + out_path = os.path.join(self.output_dir, "cobertura.xml") + self.doc.write(out_path, encoding="utf-8", pretty_print=True) + print("Generated Cobertura report:", os.path.abspath(out_path)) + + +register_reporter("cobertura-xml", CoberturaXmlReporter, needs_lxml=True) + + +class AbstractXmlReporter(AbstractReporter): + """Internal abstract class for reporters that work via XML.""" + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + + memory_reporter = reports.add_report("memory-xml", "") + assert isinstance(memory_reporter, MemoryXmlReporter) + # The dependency will be called first. + self.memory_xml = memory_reporter + + +class XmlReporter(AbstractXmlReporter): + """Public reporter that exports XML. + + The produced XML files contain a reference to the absolute path + of the html transform, so they will be locally viewable in a browser. + + However, there is a bug in Chrome and all other WebKit-based browsers + that makes it fail from file:// URLs but work on http:// URLs. + """ + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + last_xml = self.memory_xml.last_xml + if last_xml is None: + return + path = os.path.relpath(tree.path) + if path.startswith(".."): + return + out_path = os.path.join(self.output_dir, "xml", path + ".xml") + os.makedirs(os.path.dirname(out_path), exist_ok=True) + last_xml.write(out_path, encoding="utf-8") + + def on_finish(self) -> None: + last_xml = self.memory_xml.last_xml + assert last_xml is not None + out_path = os.path.join(self.output_dir, "index.xml") + out_xslt = os.path.join(self.output_dir, "mypy-html.xslt") + out_css = os.path.join(self.output_dir, "mypy-html.css") + last_xml.write(out_path, encoding="utf-8") + shutil.copyfile(self.memory_xml.xslt_html_path, out_xslt) + shutil.copyfile(self.memory_xml.css_html_path, out_css) + print("Generated XML report:", os.path.abspath(out_path)) + + +register_reporter("xml", XmlReporter, needs_lxml=True) + + +class XsltHtmlReporter(AbstractXmlReporter): + """Public reporter that exports HTML via XSLT. + + This is slightly different than running `xsltproc` on the .xml files, + because it passes a parameter to rewrite the links. + """ + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + + self.xslt_html = etree.XSLT(etree.parse(self.memory_xml.xslt_html_path)) + self.param_html = etree.XSLT.strparam("html") + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + last_xml = self.memory_xml.last_xml + if last_xml is None: + return + path = os.path.relpath(tree.path) + if path.startswith(".."): + return + out_path = os.path.join(self.output_dir, "html", path + ".html") + os.makedirs(os.path.dirname(out_path), exist_ok=True) + transformed_html = bytes(self.xslt_html(last_xml, ext=self.param_html)) + with open(out_path, "wb") as out_file: + out_file.write(transformed_html) + + def on_finish(self) -> None: + last_xml = self.memory_xml.last_xml + assert last_xml is not None + out_path = os.path.join(self.output_dir, "index.html") + out_css = os.path.join(self.output_dir, "mypy-html.css") + transformed_html = bytes(self.xslt_html(last_xml, ext=self.param_html)) + with open(out_path, "wb") as out_file: + out_file.write(transformed_html) + shutil.copyfile(self.memory_xml.css_html_path, out_css) + print("Generated HTML report (via XSLT):", os.path.abspath(out_path)) + + +register_reporter("xslt-html", XsltHtmlReporter, needs_lxml=True) + + +class XsltTxtReporter(AbstractXmlReporter): + """Public reporter that exports TXT via XSLT. + + Currently this only does the summary, not the individual reports. + """ + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + + self.xslt_txt = etree.XSLT(etree.parse(self.memory_xml.xslt_txt_path)) + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + pass + + def on_finish(self) -> None: + last_xml = self.memory_xml.last_xml + assert last_xml is not None + out_path = os.path.join(self.output_dir, "index.txt") + transformed_txt = bytes(self.xslt_txt(last_xml)) + with open(out_path, "wb") as out_file: + out_file.write(transformed_txt) + print("Generated TXT report (via XSLT):", os.path.abspath(out_path)) + + +register_reporter("xslt-txt", XsltTxtReporter, needs_lxml=True) + +alias_reporter("xslt-html", "html") +alias_reporter("xslt-txt", "txt") + + +class LinePrecisionReporter(AbstractReporter): + """Report per-module line counts for typing precision. + + Each line is classified into one of these categories: + + * precise (fully type checked) + * imprecise (Any types in a type component, such as List[Any]) + * any (something with an Any type, implicit or explicit) + * empty (empty line, comment or docstring) + * unanalyzed (mypy considers line unreachable) + + The meaning of these categories varies slightly depending on + context. + """ + + def __init__(self, reports: Reports, output_dir: str) -> None: + super().__init__(reports, output_dir) + self.files: list[FileInfo] = [] + + def on_file( + self, + tree: MypyFile, + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + options: Options, + ) -> None: + try: + path = os.path.relpath(tree.path) + except ValueError: + return + + if should_skip_path(path): + return + + visitor = stats.StatisticsVisitor( + inferred=True, + filename=tree.fullname, + modules=modules, + typemap=type_map, + all_nodes=True, + ) + tree.accept(visitor) + + file_info = FileInfo(path, tree._fullname) + for lineno, _ in iterate_python_lines(path): + status = visitor.line_map.get(lineno, stats.TYPE_EMPTY) + file_info.counts[status] += 1 + + self.files.append(file_info) + + def on_finish(self) -> None: + if not self.files: + # Nothing to do. + return + output_files = sorted(self.files, key=lambda x: x.module) + report_file = os.path.join(self.output_dir, "lineprecision.txt") + width = max(4, max(len(info.module) for info in output_files)) + titles = ("Lines", "Precise", "Imprecise", "Any", "Empty", "Unanalyzed") + widths = (width,) + tuple(len(t) for t in titles) + fmt = "{:%d} {:%d} {:%d} {:%d} {:%d} {:%d} {:%d}\n" % widths + with open(report_file, "w") as f: + f.write(fmt.format("Name", *titles)) + f.write("-" * (width + 51) + "\n") + for file_info in output_files: + counts = file_info.counts + f.write( + fmt.format( + file_info.module.ljust(width), + file_info.total(), + counts[stats.TYPE_PRECISE], + counts[stats.TYPE_IMPRECISE], + counts[stats.TYPE_ANY], + counts[stats.TYPE_EMPTY], + counts[stats.TYPE_UNANALYZED], + ) + ) + + +register_reporter("lineprecision", LinePrecisionReporter) + + +# Reporter class names are defined twice to speed up mypy startup, as this +# module is slow to import. Ensure that the two definitions match. +assert set(reporter_classes) == set(REPORTER_NAMES) diff --git a/mypy/sametypes.py b/mypy/sametypes.py deleted file mode 100644 index 0438be08247e..000000000000 --- a/mypy/sametypes.py +++ /dev/null @@ -1,92 +0,0 @@ -from typing import List, cast - -from mypy.types import ( - Type, UnboundType, ErrorType, AnyType, NoneTyp, Void, TupleType, UnionType, Callable, - TypeVar, Instance, TypeVisitor, ErasedType, TypeList -) - - -def is_same_type(left: Type, right: Type) -> bool: - """Is 'left' the same type as 'right'?""" - - if isinstance(right, UnboundType): - # Make unbound types same as anything else to reduce the number of - # generated spurious error messages. - return True - else: - return left.accept(SameTypeVisitor(right)) - - -def is_same_types(a1: List[Type], a2: List[Type]) -> bool: - if len(a1) != len(a2): - return False - for i in range(len(a1)): - if not is_same_type(a1[i], a2[i]): - return False - return True - - -class SameTypeVisitor(TypeVisitor[bool]): - """Visitor for checking whether two types are the 'same' type.""" - - def __init__(self, right: Type) -> None: - self.right = right - - # visit_x(left) means: is left (which is an instance of X) the same type as - # right? - - def visit_unbound_type(self, left: UnboundType) -> bool: - return True - - def visit_error_type(self, left: ErrorType) -> bool: - return False - - def visit_type_list(self, t: TypeList) -> bool: - assert False, 'Not supported' - - def visit_any(self, left: AnyType) -> bool: - return isinstance(self.right, AnyType) - - def visit_void(self, left: Void) -> bool: - return isinstance(self.right, Void) - - def visit_none_type(self, left: NoneTyp) -> bool: - return isinstance(self.right, NoneTyp) - - def visit_erased_type(self, left: ErasedType) -> bool: - # Should not get here. - raise RuntimeError() - - def visit_instance(self, left: Instance) -> bool: - return (isinstance(self.right, Instance) and - left.type == (cast(Instance, self.right)).type and - is_same_types(left.args, (cast(Instance, self.right)).args)) - - def visit_type_var(self, left: TypeVar) -> bool: - return (isinstance(self.right, TypeVar) and - left.id == (cast(TypeVar, self.right)).id) - - def visit_callable(self, left: Callable) -> bool: - # FIX generics - if isinstance(self.right, Callable): - cright = cast(Callable, self.right) - return (is_same_type(left.ret_type, cright.ret_type) and - is_same_types(left.arg_types, cright.arg_types) and - left.arg_names == cright.arg_names and - left.arg_kinds == cright.arg_kinds and - left.is_type_obj() == cright.is_type_obj()) - else: - return False - - def visit_tuple_type(self, left: TupleType) -> bool: - if isinstance(self.right, TupleType): - return is_same_types(left.items, cast(TupleType, self.right).items) - else: - return False - - def visit_union_type(self, left: UnionType) -> bool: - # XXX This is a test for syntactic equality, not equivalence - if isinstance(self.right, UnionType): - return is_same_types(left.items, cast(UnionType, self.right).items) - else: - return False diff --git a/mypy/scope.py b/mypy/scope.py new file mode 100644 index 000000000000..021dd9a7d8a5 --- /dev/null +++ b/mypy/scope.py @@ -0,0 +1,125 @@ +"""Track current scope to easily calculate the corresponding fine-grained target. + +TODO: Use everywhere where we track targets, including in mypy.errors. +""" + +from __future__ import annotations + +from contextlib import contextmanager, nullcontext +from typing import Iterator, Optional, Tuple +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.nodes import FuncBase, TypeInfo + +SavedScope: _TypeAlias = Tuple[str, Optional[TypeInfo], Optional[FuncBase]] + + +class Scope: + """Track which target we are processing at any given time.""" + + def __init__(self) -> None: + self.module: str | None = None + self.classes: list[TypeInfo] = [] + self.function: FuncBase | None = None + self.functions: list[FuncBase] = [] + # Number of nested scopes ignored (that don't get their own separate targets) + self.ignored = 0 + + def current_module_id(self) -> str: + assert self.module + return self.module + + def current_target(self) -> str: + """Return the current target (non-class; for a class return enclosing module).""" + assert self.module + if self.function: + fullname = self.function.fullname + return fullname or "" + return self.module + + def current_full_target(self) -> str: + """Return the current target (may be a class).""" + assert self.module + if self.function: + return self.function.fullname + if self.classes: + return self.classes[-1].fullname + return self.module + + def current_type_name(self) -> str | None: + """Return the current type's short name if it exists""" + return self.classes[-1].name if self.classes else None + + def current_function_name(self) -> str | None: + """Return the current function's short name if it exists""" + return self.function.name if self.function else None + + @contextmanager + def module_scope(self, prefix: str) -> Iterator[None]: + self.module = prefix + self.classes = [] + self.function = None + self.ignored = 0 + yield + assert self.module + self.module = None + + @contextmanager + def function_scope(self, fdef: FuncBase) -> Iterator[None]: + self.functions.append(fdef) + if not self.function: + self.function = fdef + else: + # Nested functions are part of the topmost function target. + self.ignored += 1 + yield + self.functions.pop() + if self.ignored: + # Leave a scope that's included in the enclosing target. + self.ignored -= 1 + else: + assert self.function + self.function = None + + def outer_functions(self) -> list[FuncBase]: + return self.functions[:-1] + + def enter_class(self, info: TypeInfo) -> None: + """Enter a class target scope.""" + if not self.function: + self.classes.append(info) + else: + # Classes within functions are part of the enclosing function target. + self.ignored += 1 + + def leave_class(self) -> None: + """Leave a class target scope.""" + if self.ignored: + # Leave a scope that's included in the enclosing target. + self.ignored -= 1 + else: + assert self.classes + # Leave the innermost class. + self.classes.pop() + + @contextmanager + def class_scope(self, info: TypeInfo) -> Iterator[None]: + self.enter_class(info) + yield + self.leave_class() + + def save(self) -> SavedScope: + """Produce a saved scope that can be entered with saved_scope()""" + assert self.module + # We only save the innermost class, which is sufficient since + # the rest are only needed for when classes are left. + cls = self.classes[-1] if self.classes else None + return self.module, cls, self.function + + @contextmanager + def saved_scope(self, saved: SavedScope) -> Iterator[None]: + module, info, function = saved + with self.module_scope(module): + with self.class_scope(info) if info else nullcontext(): + with self.function_scope(function) if function else nullcontext(): + yield diff --git a/mypy/semanal.py b/mypy/semanal.py index 7bebf23a9b79..edcc50e66e30 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,7 +1,13 @@ """The semantic analyzer. Bind names to definitions and do various other simple consistency -checks. For example, consider this program: +checks. Populate symbol tables. The semantic analyzer also detects +special forms which reuse generic syntax such as NamedTuple and +cast(). Multiple analysis iterations may be needed to analyze forward +references and import cycles. Each iteration "fills in" additional +bindings and references until everything has been bound. + +For example, consider this program: x = 1 y = x @@ -10,545 +16,2733 @@ defines a new variable, the type of which is to be inferred (in a later pass; type inference or type checking is not part of semantic analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. +module-level variable (Var) node. The second assignment would also +be analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis of types is implemented in typeanal.py. + +See semanal_main.py for the top-level logic. + +Some important properties: + +* After semantic analysis is complete, no PlaceholderNode and + PlaceholderType instances should remain. During semantic analysis, + if we encounter one of these, the current target should be deferred. + +* A TypeInfo is only created once we know certain basic information about + a type, such as the MRO, existence of a Tuple base class (e.g., for named + tuples), and whether we have a TypedDict. We use a temporary + PlaceholderNode node in the symbol table if some such information is + missing. + +* For assignments, we only add a non-placeholder symbol table entry once + we know the sort of thing being defined (variable, NamedTuple, type alias, + etc.). + +* Every part of the analysis step must support multiple iterations over + the same AST nodes, and each iteration must be able to fill in arbitrary + things that were missing or incomplete in previous iterations. + +* Changes performed by the analysis need to be reversible, since mypy + daemon strips and reuses existing ASTs (to improve performance and/or + reduce memory use). """ -from typing import ( - Undefined, List, Dict, Set, Tuple, cast, Any, overload, typevar, Union +from __future__ import annotations + +from contextlib import contextmanager +from typing import Any, Callable, Collection, Final, Iterable, Iterator, List, TypeVar, cast +from typing_extensions import TypeAlias as _TypeAlias, TypeGuard + +from mypy import errorcodes as codes, message_registry +from mypy.constant_fold import constant_fold_expr +from mypy.errorcodes import PROPERTY_DECORATOR, ErrorCode +from mypy.errors import Errors, report_internal_error +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.message_registry import ErrorMessage +from mypy.messages import ( + SUGGESTED_TEST_FIXTURES, + TYPES_FOR_UNIMPORTED_HINTS, + MessageBuilder, + best_matches, + pretty_seq, ) - +from mypy.mro import MroError, calculate_mro from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, VarDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, ParenExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, YieldStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, UndefinedExpr, TypeVarExpr, - StrExpr, PrintStmt, ConditionalExpr, DucktypeExpr, DisjointclassExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromStmt, YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension + ARG_NAMED, + ARG_POS, + ARG_STAR, + ARG_STAR2, + CONTRAVARIANT, + COVARIANT, + GDEF, + IMPLICITLY_ABSTRACT, + INVARIANT, + IS_ABSTRACT, + LDEF, + MDEF, + NOT_ABSTRACT, + PARAM_SPEC_KIND, + REVEAL_LOCALS, + REVEAL_TYPE, + RUNTIME_PROTOCOL_DECOS, + TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, + VARIANCE_NOT_READY, + ArgKind, + AssertStmt, + AssertTypeExpr, + AssignmentExpr, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + CallExpr, + CastExpr, + ClassDef, + ComparisonExpr, + ConditionalExpr, + Context, + ContinueStmt, + DataclassTransformSpec, + Decorator, + DelStmt, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, + Expression, + ExpressionStmt, + FakeExpression, + ForStmt, + FuncBase, + FuncDef, + FuncItem, + GeneratorExpr, + GlobalDecl, + IfStmt, + Import, + ImportAll, + ImportBase, + ImportFrom, + IndexExpr, + LambdaExpr, + ListComprehension, + ListExpr, + Lvalue, + MatchStmt, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + Node, + NonlocalDecl, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + OverloadPart, + ParamSpecExpr, + PassStmt, + PlaceholderNode, + PromoteExpr, + RaiseStmt, + RefExpr, + ReturnStmt, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + Statement, + StrExpr, + SuperExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TempNode, + TryStmt, + TupleExpr, + TypeAlias, + TypeAliasExpr, + TypeAliasStmt, + TypeApplication, + TypedDictExpr, + TypeInfo, + TypeParam, + TypeVarExpr, + TypeVarLikeExpr, + TypeVarTupleExpr, + UnaryExpr, + Var, + WhileStmt, + WithStmt, + YieldExpr, + YieldFromExpr, + get_member_expr_fullname, + get_nongen_builtins, + implicit_module_attrs, + is_final_node, + type_aliases, + type_aliases_source_versions, + typing_extensions_aliases, ) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors +from mypy.options import Options +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + SequencePattern, + StarredPattern, + ValuePattern, +) +from mypy.plugin import ( + ClassDefContext, + DynamicClassDefContext, + Plugin, + SemanticAnalyzerPluginInterface, +) +from mypy.plugins import dataclasses as dataclasses_plugin +from mypy.reachability import ( + ALWAYS_FALSE, + ALWAYS_TRUE, + MYPY_FALSE, + MYPY_TRUE, + infer_condition_value, + infer_reachability_of_if_statement, + infer_reachability_of_match_statement, +) +from mypy.scope import Scope +from mypy.semanal_enum import EnumCallAnalyzer +from mypy.semanal_namedtuple import NamedTupleAnalyzer +from mypy.semanal_newtype import NewTypeAnalyzer +from mypy.semanal_shared import ( + ALLOW_INCOMPATIBLE_OVERRIDE, + PRIORITY_FALLBACKS, + SemanticAnalyzerInterface, + calculate_tuple_fallback, + find_dataclass_transform_spec, + has_placeholder, + parse_bool, + require_bool_literal_argument, + set_callable_name as set_callable_name, +) +from mypy.semanal_typeddict import TypedDictAnalyzer +from mypy.tvar_scope import TypeVarLikeScope +from mypy.typeanal import ( + SELF_TYPE_NAMES, + FindTypeVarVisitor, + TypeAnalyser, + TypeVarDefaultTranslator, + TypeVarLikeList, + analyze_type_alias, + check_for_explicit_any, + detect_diverging_alias, + find_self_type, + fix_instance, + has_any_from_unimported_type, + no_subscript_builtin_alias, + type_constructors, + validate_instance, +) +from mypy.typeops import function_type, get_type_vars, try_getting_str_literals_from_type from mypy.types import ( - NoneTyp, Callable, Overloaded, Instance, Type, TypeVar, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType + ASSERT_TYPE_NAMES, + DATACLASS_TRANSFORM_NAMES, + DEPRECATED_TYPE_NAMES, + FINAL_DECORATOR_NAMES, + FINAL_TYPE_NAMES, + IMPORTED_REVEAL_TYPE_NAMES, + NEVER_NAMES, + OVERLOAD_NAMES, + OVERRIDE_DECORATOR_NAMES, + PROTOCOL_NAMES, + REVEAL_TYPE_NAMES, + TPDICT_NAMES, + TYPE_ALIAS_NAMES, + TYPE_CHECK_ONLY_NAMES, + TYPE_VAR_LIKE_NAMES, + TYPED_NAMEDTUPLE_NAMES, + AnyType, + CallableType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PlaceholderType, + ProperType, + TrivialSyntheticTypeTranslator, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UnionType, + UnpackType, + get_proper_type, + get_proper_types, + has_type_vars, + is_named_instance, + remove_dups, + type_vars_as_args, ) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyse_node -from mypy.parsetype import parse_str_as_type, TypeParseError +from mypy.types_utils import is_invalid_recursive_alias, store_argument_type +from mypy.typevars import fill_typevars +from mypy.util import correct_relative_import, is_dunder, module_prefix, unmangle, unnamed_function +from mypy.visitor import NodeVisitor +T = TypeVar("T") -T = typevar('T') +FUTURE_IMPORTS: Final = { + "__future__.nested_scopes": "nested_scopes", + "__future__.generators": "generators", + "__future__.division": "division", + "__future__.absolute_import": "absolute_import", + "__future__.with_statement": "with_statement", + "__future__.print_function": "print_function", + "__future__.unicode_literals": "unicode_literals", + "__future__.barry_as_FLUFL": "barry_as_FLUFL", + "__future__.generator_stop": "generator_stop", + "__future__.annotations": "annotations", +} -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 +# Special cased built-in classes that are needed for basic functionality and need to be +# available very early on. +CORE_BUILTIN_CLASSES: Final = ["object", "bool", "function"] -class TypeTranslationError(Exception): - """Exception raised when an expression is not valid as a type.""" +# Python has several different scope/namespace kinds with subtly different semantics. +SCOPE_GLOBAL: Final = 0 # Module top level +SCOPE_CLASS: Final = 1 # Class body +SCOPE_FUNC: Final = 2 # Function or lambda +SCOPE_COMPREHENSION: Final = 3 # Comprehension or generator expression +SCOPE_ANNOTATION: Final = 4 # Annotation scopes for type parameters and aliases (PEP 695) + + +# Used for tracking incomplete references +Tag: _TypeAlias = int -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. +class SemanticAnalyzer( + NodeVisitor[None], SemanticAnalyzerInterface, SemanticAnalyzerPluginInterface +): + """Semantically analyze parsed mypy files. - This is the second phase of semantic analysis. + The analyzer binds names and does various consistency checks for an + AST. Note that type checking is performed as a separate pass. """ - # Library search paths - lib_path = Undefined(List[str]) + __deletable__ = ["patches", "options", "cur_mod_node"] + # Module name space - modules = Undefined(Dict[str, MypyFile]) + modules: dict[str, MypyFile] # Global name space for current module - globals = Undefined(SymbolTable) + globals: SymbolTable # Names declared using "global" (separate set for each scope) - global_decls = Undefined(List[Set[str]]) - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = Undefined(List[Set[str]]) + global_decls: list[set[str]] + # Names declared using "nonlocal" (separate set for each scope) + nonlocal_decls: list[set[str]] # Local names of function scopes; None for non-function scopes. - locals = Undefined(List[SymbolTable]) + locals: list[SymbolTable | None] + # Type of each scope (SCOPE_*, indexes match locals) + scope_stack: list[int] # Nested block depths of scopes - block_depth = Undefined(List[int]) + block_depth: list[int] # TypeInfo of directly enclosing class (or None) - type = Undefined(TypeInfo) + _type: TypeInfo | None = None # Stack of outer classes (the second tuple item contains tvars). - type_stack = Undefined(List[Tuple[TypeInfo, List[SymbolTableNode]]]) - # Stack of functions being analyzed - function_stack = Undefined(List[FuncItem]) - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - imports = Undefined(Set[str]) # Imported modules (during phase 2 analysis) - errors = Undefined(Errors) # Keep track of generated errors + type_stack: list[TypeInfo | None] + # Type variables bound by the current scope, be it class or function + tvar_scope: TypeVarLikeScope + # Per-module options + options: Options - def __init__(self, lib_path: List[str], errors: Errors, - pyversion: int = 3) -> None: + # Stack of functions being analyzed + function_stack: list[FuncItem] + + # Set to True if semantic analysis defines a name, or replaces a + # placeholder definition. If some iteration makes no progress, + # there can be at most one additional final iteration (see below). + progress = False + deferred = False # Set to true if another analysis pass is needed + incomplete = False # Set to true if current module namespace is missing things + # Is this the final iteration of semantic analysis (where we report + # unbound names due to cyclic definitions and should not defer)? + _final_iteration = False + # These names couldn't be added to the symbol table due to incomplete deps. + # Note that missing names are per module, _not_ per namespace. This means that e.g. + # a missing name at global scope will block adding same name at a class scope. + # This should not affect correctness and is purely a performance issue, + # since it can cause unnecessary deferrals. These are represented as + # PlaceholderNodes in the symbol table. We use this to ensure that the first + # definition takes precedence even if it's incomplete. + # + # Note that a star import adds a special name '*' to the set, this blocks + # adding _any_ names in the current file. + missing_names: list[set[str]] + # Callbacks that will be called after semantic analysis to tweak things. + patches: list[tuple[int, Callable[[], None]]] + loop_depth: list[int] # Depth of breakable loops + cur_mod_id = "" # Current module id (or None) (phase 2) + _is_stub_file = False # Are we analyzing a stub file? + _is_typeshed_stub_file = False # Are we analyzing a typeshed stub file? + imports: set[str] # Imported modules (during phase 2 analysis) + # Note: some imports (and therefore dependencies) might + # not be found in phase 1, for example due to * imports. + errors: Errors # Keeps track of generated errors + plugin: Plugin # Mypy plugin for special casing of library features + statement: Statement | None = None # Statement/definition being analyzed + + # Mapping from 'async def' function definitions to their return type wrapped as a + # 'Coroutine[Any, Any, T]'. Used to keep track of whether a function definition's + # return type has already been wrapped, by checking if the function definition's + # type is stored in this mapping and that it still matches. + wrapped_coro_return_types: dict[FuncDef, Type] = {} + + def __init__( + self, + modules: dict[str, MypyFile], + missing_modules: set[str], + incomplete_namespaces: set[str], + errors: Errors, + plugin: Plugin, + ) -> None: """Construct semantic analyzer. - Use lib_path to search for modules, and report analysis errors - using the Errors instance. + We reuse the same semantic analyzer instance across multiple modules. + + Args: + modules: Global modules dictionary + missing_modules: Modules that could not be imported encountered so far + incomplete_namespaces: Namespaces that are being populated during semantic analysis + (can contain modules and classes within the current SCC; mutated by the caller) + errors: Report analysis errors using this instance """ self.locals = [None] + self.scope_stack = [SCOPE_GLOBAL] + # Saved namespaces from previous iteration. Every top-level function/method body is + # analyzed in several iterations until all names are resolved. We need to save + # the local namespaces for the top level function and all nested functions between + # these iterations. See also semanal_main.process_top_level_function(). + self.saved_locals: dict[ + FuncItem | GeneratorExpr | DictionaryComprehension, SymbolTable + ] = {} self.imports = set() - self.type = None + self._type = None self.type_stack = [] + # Are the namespaces of classes being processed complete? + self.incomplete_type_stack: list[bool] = [] + self.tvar_scope = TypeVarLikeScope() self.function_stack = [] self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path + self.loop_depth = [0] self.errors = errors - self.modules = {} - self.pyversion = pyversion - self.stored_vars = Dict[Node, Type]() + self.modules = modules + self.msg = MessageBuilder(errors, modules) + self.missing_modules = missing_modules + self.missing_names = [set()] + # These namespaces are still in process of being populated. If we encounter a + # missing name in these namespaces, we need to defer the current analysis target, + # since it's possible that the name will be there once the namespace is complete. + self.incomplete_namespaces = incomplete_namespaces + self.all_exports: list[str] = [] + # Map from module id to list of explicitly exported names (i.e. names in __all__). + self.export_map: dict[str, list[str]] = {} + self.plugin = plugin + # If True, process function definitions. If False, don't. This is used + # for processing module top levels in fine-grained incremental mode. + self.recurse_into_functions = True + self.scope = Scope() + + # Trace line numbers for every file where deferral happened during analysis of + # current SCC or top-level function. + self.deferral_debug_context: list[tuple[str, int]] = [] + + # This is needed to properly support recursive type aliases. The problem is that + # Foo[Bar] could mean three things depending on context: a target for type alias, + # a normal index expression (including enum index), or a type application. + # The latter is particularly problematic as it can falsely create incomplete + # refs while analysing rvalues of type aliases. To avoid this we first analyse + # rvalues while temporarily setting this to True. + self.basic_type_applications = False + + # Used to temporarily enable unbound type variables in some contexts. Namely, + # in base class expressions, and in right hand sides of type aliases. Do not add + # new uses of this, as this may cause leaking `UnboundType`s to type checking. + self.allow_unbound_tvars = False + + # Used to pass information about current overload index to visit_func_def(). + self.current_overload_item: int | None = None + + # Used to track whether currently inside an except* block. This helps + # to invoke errors when continue/break/return is used inside except* block. + self.inside_except_star_block: bool = False + # Used to track edge case when return is still inside except* if it enters a loop + self.return_stmt_inside_except_star_block: bool = False + + # mypyc doesn't properly handle implementing an abstractproperty + # with a regular attribute so we make them properties + @property + def type(self) -> TypeInfo | None: + return self._type + + @property + def is_stub_file(self) -> bool: + return self._is_stub_file + + @property + def is_typeshed_stub_file(self) -> bool: + return self._is_typeshed_stub_file + + @property + def final_iteration(self) -> bool: + return self._final_iteration + + @contextmanager + def allow_unbound_tvars_set(self) -> Iterator[None]: + old = self.allow_unbound_tvars + self.allow_unbound_tvars = True + try: + yield + finally: + self.allow_unbound_tvars = old + + @contextmanager + def inside_except_star_block_set( + self, value: bool, entering_loop: bool = False + ) -> Iterator[None]: + old = self.inside_except_star_block + self.inside_except_star_block = value + + # Return statement would still be in except* scope if entering loops + if not entering_loop: + old_return_stmt_flag = self.return_stmt_inside_except_star_block + self.return_stmt_inside_except_star_block = value - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.globals = file_node.names + try: + yield + finally: + self.inside_except_star_block = old + if not entering_loop: + self.return_stmt_inside_except_star_block = old_return_stmt_flag + + # + # Preparing module (performed before semantic analysis) + # - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) + def prepare_file(self, file_node: MypyFile) -> None: + """Prepare a freshly parsed file for semantic analysis.""" + if "builtins" in self.modules: + file_node.names["__builtins__"] = SymbolTableNode(GDEF, self.modules["builtins"]) + if file_node.fullname == "builtins": + self.prepare_builtins_namespace(file_node) + if file_node.fullname == "typing": + self.prepare_typing_namespace(file_node, type_aliases) + if file_node.fullname == "typing_extensions": + self.prepare_typing_namespace(file_node, typing_extensions_aliases) + + def prepare_typing_namespace(self, file_node: MypyFile, aliases: dict[str, str]) -> None: + """Remove dummy alias definitions such as List = TypeAlias(object) from typing. + + They will be replaced with real aliases when corresponding targets are ready. + """ - defs = file_node.defs - for d in defs: - d.accept(self) + # This is all pretty unfortunate. typeshed now has a + # sys.version_info check for OrderedDict, and we shouldn't + # take it out, because it is correct and a typechecker should + # use that as a source of truth. But instead we rummage + # through IfStmts to remove the info first. (I tried to + # remove this whole machinery and ran into issues with the + # builtins/typing import cycle.) + def helper(defs: list[Statement]) -> None: + for stmt in defs.copy(): + if isinstance(stmt, IfStmt): + for body in stmt.body: + helper(body.body) + if stmt.else_body: + helper(stmt.else_body.body) + if ( + isinstance(stmt, AssignmentStmt) + and len(stmt.lvalues) == 1 + and isinstance(stmt.lvalues[0], NameExpr) + ): + # Assignment to a simple name, remove it if it is a dummy alias. + if f"{file_node.fullname}.{stmt.lvalues[0].name}" in aliases: + defs.remove(stmt) + + helper(file_node.defs) + + def prepare_builtins_namespace(self, file_node: MypyFile) -> None: + """Add certain special-cased definitions to the builtins module. + + Some definitions are too special or fundamental to be processed + normally from the AST. + """ + names = file_node.names + + # Add empty definition for core built-in classes, since they are required for basic + # operation. These will be completed later on. + for name in CORE_BUILTIN_CLASSES: + cdef = ClassDef(name, Block([])) # Dummy ClassDef, will be replaced later + info = TypeInfo(SymbolTable(), cdef, "builtins") + info._fullname = f"builtins.{name}" + names[name] = SymbolTableNode(GDEF, info) + + bool_info = names["bool"].node + assert isinstance(bool_info, TypeInfo) + bool_type = Instance(bool_info, []) + + special_var_types: list[tuple[str, Type]] = [ + ("None", NoneType()), + # reveal_type is a mypy-only function that gives an error with + # the type of its arg. + ("reveal_type", AnyType(TypeOfAny.special_form)), + # reveal_locals is a mypy-only function that gives an error with the types of + # locals + ("reveal_locals", AnyType(TypeOfAny.special_form)), + ("True", bool_type), + ("False", bool_type), + ("__debug__", bool_type), + ] + + for name, typ in special_var_types: + v = Var(name, typ) + v._fullname = f"builtins.{name}" + file_node.names[name] = SymbolTableNode(GDEF, v) + + # + # Analyzing a target + # + + def refresh_partial( + self, + node: MypyFile | FuncDef | OverloadedFuncDef, + patches: list[tuple[int, Callable[[], None]]], + final_iteration: bool, + file_node: MypyFile, + options: Options, + active_type: TypeInfo | None = None, + ) -> None: + """Refresh a stale target in fine-grained incremental mode.""" + self.patches = patches + self.deferred = False + self.incomplete = False + self._final_iteration = final_iteration + self.missing_names[-1] = set() + + with self.file_context(file_node, options, active_type): + if isinstance(node, MypyFile): + self.refresh_top_level(node) + else: + self.recurse_into_functions = True + self.accept(node) + del self.patches + + def refresh_top_level(self, file_node: MypyFile) -> None: + """Reanalyze a stale module top-level in fine-grained incremental mode.""" + self.recurse_into_functions = False + self.add_implicit_module_attrs(file_node) + for d in file_node.defs: + self.accept(d) + if file_node.fullname == "typing": + self.add_builtin_aliases(file_node) + if file_node.fullname == "typing_extensions": + self.add_typing_extension_aliases(file_node) + self.adjust_public_exports() + self.export_map[self.cur_mod_id] = self.all_exports + self.all_exports = [] + + def add_implicit_module_attrs(self, file_node: MypyFile) -> None: + """Manually add implicit definitions of module '__name__' etc.""" + str_type: Type | None = self.named_type_or_none("builtins.str") + if str_type is None: + str_type = UnboundType("builtins.str") + inst: Type | None + for name, t in implicit_module_attrs.items(): + if name == "__doc__": + typ: Type = str_type + elif name == "__path__": + if not file_node.is_package_init_file(): + continue + # Need to construct the type ourselves, to avoid issues with __builtins__.list + # not being subscriptable or typing.List not getting bound + inst = self.named_type_or_none("builtins.list", [str_type]) + if inst is None: + assert not self.final_iteration, "Cannot find builtins.list to add __path__" + self.defer() + return + typ = inst + elif name == "__annotations__": + inst = self.named_type_or_none( + "builtins.dict", [str_type, AnyType(TypeOfAny.special_form)] + ) + if inst is None: + assert ( + not self.final_iteration + ), "Cannot find builtins.dict to add __annotations__" + self.defer() + return + typ = inst + elif name == "__spec__": + if self.options.use_builtins_fixtures: + inst = self.named_type_or_none("builtins.object") + else: + inst = self.named_type_or_none("importlib.machinery.ModuleSpec") + if inst is None: + if self.final_iteration: + inst = self.named_type_or_none("builtins.object") + assert inst is not None, "Cannot find builtins.object" + else: + self.defer() + return + if file_node.name == "__main__": + # https://docs.python.org/3/reference/import.html#main-spec + inst = UnionType.make_union([inst, NoneType()]) + typ = inst + else: + assert t is not None, f"type should be specified for {name}" + typ = UnboundType(t) + + existing = file_node.names.get(name) + if existing is not None and not isinstance(existing.node, PlaceholderNode): + # Already exists. + continue + + an_type = self.anal_type(typ) + if an_type: + var = Var(name, an_type) + var._fullname = self.qualified_name(name) + var.is_ready = True + self.add_symbol(name, var, dummy_context()) + else: + self.add_symbol( + name, + PlaceholderNode(self.qualified_name(name), file_node, -1), + dummy_context(), + ) + + def add_builtin_aliases(self, tree: MypyFile) -> None: + """Add builtin type aliases to typing module. + + For historical reasons, the aliases like `List = list` are not defined + in typeshed stubs for typing module. Instead we need to manually add the + corresponding nodes on the fly. We explicitly mark these aliases as normalized, + so that a user can write `typing.List[int]`. + """ + assert tree.fullname == "typing" + for alias, target_name in type_aliases.items(): + if ( + alias in type_aliases_source_versions + and type_aliases_source_versions[alias] > self.options.python_version + ): + # This alias is not available on this Python version. + continue + name = alias.split(".")[-1] + if name in tree.names and not isinstance(tree.names[name].node, PlaceholderNode): + continue + self.create_alias(tree, target_name, alias, name) + + def add_typing_extension_aliases(self, tree: MypyFile) -> None: + """Typing extensions module does contain some type aliases. + + We need to analyze them as such, because in typeshed + they are just defined as `_Alias()` call. + Which is not supported natively. + """ + assert tree.fullname == "typing_extensions" + + for alias, target_name in typing_extensions_aliases.items(): + name = alias.split(".")[-1] + if name in tree.names and isinstance(tree.names[name].node, TypeAlias): + continue # Do not reset TypeAliases on the second pass. + + # We need to remove any node that is there at the moment. It is invalid. + tree.names.pop(name, None) + + # Now, create a new alias. + self.create_alias(tree, target_name, alias, name) + + def create_alias(self, tree: MypyFile, target_name: str, alias: str, name: str) -> None: + tag = self.track_incomplete_refs() + n = self.lookup_fully_qualified_or_none(target_name) + if n: + if isinstance(n.node, PlaceholderNode): + self.mark_incomplete(name, tree) + else: + # Found built-in class target. Create alias. + target = self.named_type_or_none(target_name, []) + assert target is not None + # Transform List to List[Any], etc. + fix_instance( + target, self.fail, self.note, disallow_any=False, options=self.options + ) + alias_node = TypeAlias( + target, + alias, + line=-1, + column=-1, # there is no context + no_args=True, + normalized=True, + ) + self.add_symbol(name, alias_node, tree) + elif self.found_incomplete_ref(tag): + # Built-in class target may not ready yet -- defer. + self.mark_incomplete(name, tree) + else: + # Test fixtures may be missing some builtin classes, which is okay. + # Kill the placeholder if there is one. + if name in tree.names: + assert isinstance(tree.names[name].node, PlaceholderNode) + del tree.names[name] + + def adjust_public_exports(self) -> None: + """Adjust the module visibility of globals due to __all__.""" + if "__all__" in self.globals: + for name, g in self.globals.items(): + # Being included in __all__ explicitly exports and makes public. + if name in self.all_exports: + g.module_public = True + g.module_hidden = False + # But when __all__ is defined, and a symbol is not included in it, + # it cannot be public. + else: + g.module_public = False + + @contextmanager + def file_context( + self, file_node: MypyFile, options: Options, active_type: TypeInfo | None = None + ) -> Iterator[None]: + """Configure analyzer for analyzing targets within a file/class. + + Args: + file_node: target file + options: options specific to the file + active_type: must be the surrounding class to analyze method targets + """ + scope = self.scope + self.options = options + self.errors.set_file(file_node.path, file_node.fullname, scope=scope, options=options) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname + with scope.module_scope(self.cur_mod_id): + self._is_stub_file = file_node.path.lower().endswith(".pyi") + self._is_typeshed_stub_file = file_node.is_typeshed_file(options) + self.globals = file_node.names + self.tvar_scope = TypeVarLikeScope() + + self.named_tuple_analyzer = NamedTupleAnalyzer(options, self, self.msg) + self.typed_dict_analyzer = TypedDictAnalyzer(options, self, self.msg) + self.enum_call_analyzer = EnumCallAnalyzer(options, self) + self.newtype_analyzer = NewTypeAnalyzer(options, self, self.msg) + + # Counter that keeps track of references to undefined things potentially caused by + # incomplete namespaces. + self.num_incomplete_refs = 0 + + if active_type: + enclosing_fullname = active_type.fullname.rsplit(".", 1)[0] + if "." in enclosing_fullname: + enclosing_node = self.lookup_fully_qualified_or_none(enclosing_fullname) + if enclosing_node and isinstance(enclosing_node.node, TypeInfo): + self._type = enclosing_node.node + self.push_type_args(active_type.defn.type_args, active_type.defn) + self.incomplete_type_stack.append(False) + scope.enter_class(active_type) + self.enter_class(active_type.defn.info) + for tvar in active_type.defn.type_vars: + self.tvar_scope.bind_existing(tvar) + + yield + + if active_type: + scope.leave_class() + self.leave_class() + self._type = None + self.incomplete_type_stack.pop() + self.pop_type_args(active_type.defn.type_args) + del self.options - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') + # + # Functions + # def visit_func_def(self, defn: FuncDef) -> None: - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() + self.statement = defn + + # Visit default values because they may contain assignment expressions. + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + + defn.is_conditional = self.block_depth[-1] > 0 + + # Set full names even for those definitions that aren't added + # to a symbol table. For example, for overload items. + defn._fullname = self.qualified_name(defn.name) + + # We don't add module top-level functions to symbol tables + # when we analyze their bodies in the second phase on analysis, + # since they were added in the first phase. Nested functions + # get always added, since they aren't separate targets. + if not self.recurse_into_functions or len(self.function_stack) > 0: + if not defn.is_decorated and not defn.is_overload: + self.add_function_to_symbol_table(defn) + + if not self.recurse_into_functions: + return + + with self.scope.function_scope(defn): + with self.inside_except_star_block_set(value=False): + self.analyze_func_def(defn) + + def function_fullname(self, fullname: str) -> str: + if self.current_overload_item is None: + return fullname + return f"{fullname}#{self.current_overload_item}" + + def analyze_func_def(self, defn: FuncDef) -> None: + if self.push_type_args(defn.type_args, defn) is None: + self.defer(defn) + return + + self.function_stack.append(defn) + + if defn.type: + assert isinstance(defn.type, CallableType) + has_self_type = self.update_function_type_variables(defn.type, defn) + else: + has_self_type = False + + self.function_stack.pop() if self.is_class_scope(): # Method definition - defn.is_conditional = self.block_depth[-1] > 0 + assert self.type is not None defn.info = self.type - if not defn.is_decorated: - if not defn.is_overload: - if defn.name() in self.type.names: - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) + if defn.type is not None and defn.name in ("__init__", "__init_subclass__"): + assert isinstance(defn.type, CallableType) + if isinstance(get_proper_type(defn.type.ret_type), AnyType): + defn.type = defn.type.copy_modified(ret_type=NoneType()) + self.prepare_method_signature(defn, self.type, has_self_type) + + # Analyze function signature + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + if defn.type: + self.check_classvar_in_signature(defn.type) + assert isinstance(defn.type, CallableType) + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + analyzer = self.type_analyzer() + tag = self.track_incomplete_refs() + result = analyzer.visit_callable_type(defn.type, nested=False, namespace=fullname) + # Don't store not ready types (including placeholders). + if self.found_incomplete_ref(tag) or has_placeholder(result): + self.defer(defn) + self.pop_type_args(defn.type_args) + return + assert isinstance(result, ProperType) + if isinstance(result, CallableType): + # type guards need to have a positional argument, to spec + skip_self = self.is_class_scope() and not defn.is_static + if result.type_guard and ARG_POS not in result.arg_kinds[skip_self:]: + self.fail( + "TypeGuard functions must have a positional argument", + result, + code=codes.VALID_TYPE, + ) + # in this case, we just kind of just ... remove the type guard. + result = result.copy_modified(type_guard=None) + if result.type_is and ARG_POS not in result.arg_kinds[skip_self:]: + self.fail( + '"TypeIs" functions must have a positional argument', + result, + code=codes.VALID_TYPE, + ) + result = result.copy_modified(type_is=None) + + result = self.remove_unpack_kwargs(defn, result) + if has_self_type and self.type is not None: + info = self.type + if info.self_type is not None: + result.variables = [info.self_type] + list(result.variables) + defn.type = result + self.add_type_alias_deps(analyzer.aliases_used) + self.check_function_signature(defn) + self.check_paramspec_definition(defn) + if isinstance(defn, FuncDef): + assert isinstance(defn.type, CallableType) + defn.type = set_callable_name(defn.type, defn) + + self.analyze_arg_initializers(defn) + self.analyze_function_body(defn) + + if self.is_class_scope(): + assert self.type is not None + # Mark protocol methods with empty bodies as implicitly abstract. + # This makes explicit protocol subclassing type-safe. + if ( + self.type.is_protocol + and not self.is_stub_file # Bodies in stub files are always empty. + and (not isinstance(self.scope.function, OverloadedFuncDef) or defn.is_property) + and defn.abstract_status != IS_ABSTRACT + and is_trivial_body(defn.body) + ): + defn.abstract_status = IMPLICITLY_ABSTRACT + if ( + is_trivial_body(defn.body) + and not self.is_stub_file + and defn.abstract_status != NOT_ABSTRACT + ): + defn.is_trivial_body = True + + if ( + defn.is_coroutine + and isinstance(defn.type, CallableType) + and self.wrapped_coro_return_types.get(defn) != defn.type + ): + if defn.is_async_generator: + # Async generator types are handled elsewhere + pass + else: + # A coroutine defined as `async def foo(...) -> T: ...` + # has external return type `Coroutine[Any, Any, T]`. + any_type = AnyType(TypeOfAny.special_form) + ret_type = self.named_type_or_none( + "typing.Coroutine", [any_type, any_type, defn.type.ret_type] + ) + assert ret_type is not None, "Internal error: typing.Coroutine not found" + defn.type = defn.type.copy_modified(ret_type=ret_type) + self.wrapped_coro_return_types[defn] = defn.type + + self.pop_type_args(defn.type_args) + + def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType: + if not typ.arg_kinds or typ.arg_kinds[-1] is not ArgKind.ARG_STAR2: + return typ + last_type = typ.arg_types[-1] + if not isinstance(last_type, UnpackType): + return typ + last_type = get_proper_type(last_type.type) + if not isinstance(last_type, TypedDictType): + self.fail("Unpack item in ** argument must be a TypedDict", last_type) + new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] + return typ.copy_modified(arg_types=new_arg_types) + overlap = set(typ.arg_names) & set(last_type.items) + # It is OK for TypedDict to have a key named 'kwargs'. + overlap.discard(typ.arg_names[-1]) + if overlap: + overlapped = ", ".join([f'"{name}"' for name in overlap]) + self.fail(f"Overlap between argument names and ** TypedDict items: {overlapped}", defn) + new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] + return typ.copy_modified(arg_types=new_arg_types) + # OK, everything looks right now, mark the callable type as using unpack. + new_arg_types = typ.arg_types[:-1] + [last_type] + return typ.copy_modified(arg_types=new_arg_types, unpack_kwargs=True) + + def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: bool) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special, as well as __new__. + functype = func.type + if func.name == "__new__": + func.is_static = True + if not func.is_static or func.name == "__new__": + if func.name in ["__init_subclass__", "__class_getitem__"]: + func.is_class = True + if not func.arguments: + self.fail( + 'Method must have at least one argument. Did you forget the "self" argument?', + func, + ) + elif isinstance(functype, CallableType): + self_type = get_proper_type(functype.arg_types[0]) + if isinstance(self_type, AnyType): + if has_self_type: + assert self.type is not None and self.type.self_type is not None + leading_type: Type = self.type.self_type + else: + leading_type = fill_typevars(info) + if func.is_class or func.name == "__new__": + leading_type = self.class_type(leading_type) + func.type = replace_implicit_first_type(functype, leading_type) + elif has_self_type and isinstance(func.unanalyzed_type, CallableType): + if not isinstance(get_proper_type(func.unanalyzed_type.arg_types[0]), AnyType): + if self.is_expected_self_type( + self_type, func.is_class or func.name == "__new__" + ): + # This error is off by default, since it is explicitly allowed + # by the PEP 673. + self.fail( + 'Redundant "Self" annotation for the first method argument', + func, + code=codes.REDUNDANT_SELF_TYPE, + ) else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - if not defn.is_static: - if not defn.args: - self.fail('Method must have at least one argument', defn) - elif defn.type: - sig = cast(FunctionLike, defn.type) - # TODO: A classmethod's first argument should be more - # precisely typed than Any. - leading_type = AnyType() if defn.is_class else self_type(self.type) - defn.type = replace_implicit_first_type(sig, leading_type) - - if self.is_func_scope() and (not defn.is_decorated and - not defn.is_overload): - self.add_local_func(defn, defn) - defn._fullname = defn.name() - - self.errors.push_function(defn.name()) - self.analyse_function(defn) - self.errors.pop_function() - - def is_conditional_func(self, n: Node, defn: FuncDef) -> bool: - return (isinstance(n, FuncDef) and cast(FuncDef, n).is_conditional and - defn.is_conditional) - - def update_function_type_variables(self, defn: FuncDef) -> None: + self.fail( + "Method cannot have explicit self annotation and Self type", func + ) + elif has_self_type: + self.fail("Static methods cannot use Self type", func) + + def is_expected_self_type(self, typ: Type, is_classmethod: bool) -> bool: + """Does this (analyzed or not) type represent the expected Self type for a method?""" + assert self.type is not None + typ = get_proper_type(typ) + if is_classmethod: + if isinstance(typ, TypeType): + return self.is_expected_self_type(typ.item, is_classmethod=False) + if isinstance(typ, UnboundType): + sym = self.lookup_qualified(typ.name, typ, suppress_errors=True) + if ( + sym is not None + and ( + sym.fullname == "typing.Type" + or ( + sym.fullname == "builtins.type" + and ( + self.is_stub_file + or self.is_future_flag_set("annotations") + or self.options.python_version >= (3, 9) + ) + ) + ) + and typ.args + ): + return self.is_expected_self_type(typ.args[0], is_classmethod=False) + return False + if isinstance(typ, TypeVarType): + return typ == self.type.self_type + if isinstance(typ, UnboundType): + sym = self.lookup_qualified(typ.name, typ, suppress_errors=True) + return sym is not None and sym.fullname in SELF_TYPE_NAMES + return False + + def set_original_def(self, previous: Node | None, new: FuncDef | Decorator) -> bool: + """If 'new' conditionally redefine 'previous', set 'previous' as original + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + def f(): ... + def f(): ... # Error: 'f' redefined + """ + if isinstance(new, Decorator): + new = new.func + if ( + isinstance(previous, (FuncDef, Decorator)) + and unnamed_function(new.name) + and unnamed_function(previous.name) + ): + return True + if isinstance(previous, (FuncDef, Var, Decorator)) and new.is_conditional: + new.original_def = previous + return True + else: + return False + + def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) -> bool: """Make any type variables in the signature of defn explicit. Update the signature of defn to contain type variable definitions - if defn is generic. + if defn is generic. Return True, if the signature contains typing.Self + type, or False otherwise. """ - if defn.type: - functype = cast(Callable, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(tvar, values) for tvar, values in typevars - if not self.is_defined_type_var(tvar, defn)] - if typevars: - defs = [TypeVarDef(tvar[0], -i - 1, tvar[1], self.object_type()) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: Callable) -> List[Tuple[str, List[Type]]]: - """Return list of unique type variables referred to in a callable.""" - names = List[str]() - values = List[List[Type]]() - for arg in type.arg_types + [type.ret_type]: - for tvar, vals in self.find_type_variables_in_type(arg): - if tvar not in names: - names.append(tvar) - values.append(vals) - return list(zip(names, values)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, List[Type]]]: - """Return a list of all unique type variable references in type.""" - result = List[Tuple[str, List[Type]]]() - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node).values)) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + a = self.type_analyzer() + fun_type.variables, has_self_type = a.bind_function_type_variables(fun_type, defn) + if has_self_type and self.type is not None: + self.setup_self_type() + if defn.type_args: + bound_fullnames = {v.fullname for v in fun_type.variables} + declared_fullnames = {self.qualified_name(p.name) for p in defn.type_args} + extra = sorted(bound_fullnames - declared_fullnames) + if extra: + self.msg.type_parameters_should_be_declared( + [n.split(".")[-1] for n in extra], defn + ) + return has_self_type + + def setup_self_type(self) -> None: + """Setup a (shared) Self type variable for current class. + + We intentionally don't add it to the class symbol table, + so it can be accessed only by mypy and will not cause + clashes with user defined names. + """ + assert self.type is not None + info = self.type + if info.self_type is not None: + if has_placeholder(info.self_type.upper_bound): + # Similar to regular (user defined) type variables. + self.process_placeholder( + None, + "Self upper bound", + info, + force_progress=info.self_type.upper_bound != fill_typevars(info), + ) + else: + return + info.self_type = TypeVarType( + "Self", + f"{info.fullname}.Self", + id=TypeVarId(0), # 0 is a special value for self-types. + values=[], + upper_bound=fill_typevars(info), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + self.statement = defn + self.add_function_to_symbol_table(defn) + + if not self.recurse_into_functions: + return + + # NB: Since _visit_overloaded_func_def will call accept on the + # underlying FuncDefs, the function might get entered twice. + # This is fine, though, because only the outermost function is + # used to compute targets. + with self.scope.function_scope(defn): + self.analyze_overloaded_func_def(defn) + + @contextmanager + def overload_item_set(self, item: int | None) -> Iterator[None]: + self.current_overload_item = item + try: + yield + finally: + self.current_overload_item = None + + def analyze_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + # OverloadedFuncDef refers to any legitimate situation where you have + # more than one declaration for the same function in a row. This occurs + # with a @property with a setter or a deleter, and for a classic + # @overload. + + defn._fullname = self.qualified_name(defn.name) + # TODO: avoid modifying items. + defn.items = defn.unanalyzed_items.copy() + + first_item = defn.items[0] + first_item.is_overload = True + with self.overload_item_set(0): + first_item.accept(self) + + if isinstance(first_item, Decorator) and first_item.func.is_property: + # This is a property. + first_item.func.is_overload = True + self.analyze_property_with_multi_part_definition(defn) + typ = function_type(first_item.func, self.named_type("builtins.function")) + assert isinstance(typ, CallableType) + types = [typ] else: - assert False, 'Unsupported type %s' % type - return result + # This is an a normal overload. Find the item signatures, the + # implementation (if outside a stub), and any missing @overload + # decorators. + types, impl, non_overload_indexes = self.analyze_overload_sigs_and_impl(defn) + defn.impl = impl + if non_overload_indexes: + self.handle_missing_overload_decorators( + defn, non_overload_indexes, some_overload_decorators=len(types) > 0 + ) + # If we found an implementation, remove it from the overload item list, + # as it's special. + if impl is not None: + assert impl is defn.items[-1] + defn.items = defn.items[:-1] + elif not non_overload_indexes: + self.handle_missing_overload_implementation(defn) + + if types and not any( + # If some overload items are decorated with other decorators, then + # the overload type will be determined during type checking. + isinstance(it, Decorator) and len(it.decorators) > 1 + for it in defn.items + ): + # TODO: should we enforce decorated overloads consistency somehow? + # Some existing code uses both styles: + # * Put decorator only on implementation, use "effective" types in overloads + # * Put decorator everywhere, use "bare" types in overloads. + defn.type = Overloaded(types) + defn.type.line = defn.line + + if not defn.items: + # It was not a real overload after all, but function redefinition. We've + # visited the redefinition(s) already. + if not defn.impl: + # For really broken overloads with no items and no implementation we need to keep + # at least one item to hold basic information like function name. + defn.impl = defn.unanalyzed_items[-1] + return - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == TVAR + # We know this is an overload def. Infer properties and perform some checks. + self.process_deprecated_overload(defn) + self.process_final_in_overload(defn) + self.process_static_or_class_method_in_overload(defn) + self.process_overload_impl(defn) + + def process_deprecated_overload(self, defn: OverloadedFuncDef) -> None: + if defn.is_property: + return + + if isinstance(impl := defn.impl, Decorator) and ( + (deprecated := impl.func.deprecated) is not None + ): + defn.deprecated = deprecated + for item in defn.items: + if isinstance(item, Decorator): + item.func.deprecated = deprecated - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = List[Callable]() for item in defn.items: - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(Callable, function_type(item.func, - self.builtin_type('builtins.function')))) - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) + deprecation = False + if isinstance(item, Decorator): + for d in item.decorators: + if deprecation and refers_to_fullname(d, OVERLOAD_NAMES): + self.msg.note("@overload should be placed before @deprecated", d) + elif (deprecated := self.get_deprecated(d)) is not None: + deprecation = True + if isinstance(typ := item.func.type, CallableType): + typestr = f" {typ} " + else: + typestr = " " + item.func.deprecated = ( + f"overload{typestr}of function {defn.fullname} is deprecated: " + f"{deprecated}" + ) + + @staticmethod + def get_deprecated(expression: Expression) -> str | None: + if ( + isinstance(expression, CallExpr) + and refers_to_fullname(expression.callee, DEPRECATED_TYPE_NAMES) + and (len(args := expression.args) >= 1) + and isinstance(deprecated := args[0], StrExpr) + ): + return deprecated.value + return None - defn.type = Overloaded(t) - defn.type.line = defn.line + def process_overload_impl(self, defn: OverloadedFuncDef) -> None: + """Set flags for an overload implementation. - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local_func(defn, defn) + Currently, this checks for a trivial body in protocols classes, + where it makes the method implicitly abstract. + """ + if defn.impl is None: + return + impl = defn.impl if isinstance(defn.impl, FuncDef) else defn.impl.func + if is_trivial_body(impl.body) and self.is_class_scope() and not self.is_stub_file: + assert self.type is not None + if self.type.is_protocol: + impl.abstract_status = IMPLICITLY_ABSTRACT + if impl.abstract_status != NOT_ABSTRACT: + impl.is_trivial_body = True + + def analyze_overload_sigs_and_impl( + self, defn: OverloadedFuncDef + ) -> tuple[list[CallableType], OverloadPart | None, list[int]]: + """Find overload signatures, the implementation, and items with missing @overload. + + Assume that the first was already analyzed. As a side effect: + analyzes remaining items and updates 'is_overload' flags. + """ + types = [] + non_overload_indexes = [] + impl: OverloadPart | None = None + for i, item in enumerate(defn.items): + if i != 0: + # Assume that the first item was already visited + item.is_overload = True + with self.overload_item_set(i if i < len(defn.items) - 1 else None): + item.accept(self) + # TODO: support decorated overloaded functions properly + if isinstance(item, Decorator): + callable = function_type(item.func, self.named_type("builtins.function")) + assert isinstance(callable, CallableType) + if not any(refers_to_fullname(dec, OVERLOAD_NAMES) for dec in item.decorators): + if i == len(defn.items) - 1 and not self.is_stub_file: + # Last item outside a stub is impl + impl = item + else: + # Oops it wasn't an overload after all. A clear error + # will vary based on where in the list it is, record + # that. + non_overload_indexes.append(i) + else: + item.func.is_overload = True + types.append(callable) + if item.var.is_property: + self.fail("An overload can not be a property", item) + # If any item was decorated with `@override`, the whole overload + # becomes an explicit override. + defn.is_explicit_override |= item.func.is_explicit_override + elif isinstance(item, FuncDef): + if i == len(defn.items) - 1 and not self.is_stub_file: + impl = item + else: + non_overload_indexes.append(i) + return types, impl, non_overload_indexes + + def handle_missing_overload_decorators( + self, + defn: OverloadedFuncDef, + non_overload_indexes: list[int], + some_overload_decorators: bool, + ) -> None: + """Generate errors for overload items without @overload. + + Side effect: remote non-overload items. + """ + if some_overload_decorators: + # Some of them were overloads, but not all. + for idx in non_overload_indexes: + if self.is_stub_file: + self.fail( + "An implementation for an overloaded function " + "is not allowed in a stub file", + defn.items[idx], + ) + else: + self.fail( + "The implementation for an overloaded function must come last", + defn.items[idx], + ) + else: + for idx in non_overload_indexes[1:]: + self.name_already_defined(defn.name, defn.items[idx], defn.items[0]) + if defn.impl: + self.name_already_defined(defn.name, defn.impl, defn.items[0]) + # Remove the non-overloads + for idx in reversed(non_overload_indexes): + del defn.items[idx] + + def handle_missing_overload_implementation(self, defn: OverloadedFuncDef) -> None: + """Generate error about missing overload implementation (only if needed).""" + if not self.is_stub_file: + if self.type and self.type.is_protocol and not self.is_func_scope(): + # An overloaded protocol method doesn't need an implementation, + # but if it doesn't have one, then it is considered abstract. + for item in defn.items: + if isinstance(item, Decorator): + item.func.abstract_status = IS_ABSTRACT + else: + item.abstract_status = IS_ABSTRACT + else: + # TODO: also allow omitting an implementation for abstract methods in ABCs? + self.fail( + "An overloaded function outside a stub file must have an implementation", + defn, + code=codes.NO_OVERLOAD_IMPL, + ) + + def process_final_in_overload(self, defn: OverloadedFuncDef) -> None: + """Detect the @final status of an overloaded function (and perform checks).""" + # If the implementation is marked as @final (or the first overload in + # stubs), then the whole overloaded definition if @final. + if any(item.is_final for item in defn.items): + # We anyway mark it as final because it was probably the intention. + defn.is_final = True + # Only show the error once per overload + bad_final = next(ov for ov in defn.items if ov.is_final) + if not self.is_stub_file: + self.fail("@final should be applied only to overload implementation", bad_final) + elif any(item.is_final for item in defn.items[1:]): + bad_final = next(ov for ov in defn.items[1:] if ov.is_final) + self.fail( + "In a stub file @final must be applied only to the first overload", bad_final + ) + if defn.impl is not None and defn.impl.is_final: + defn.is_final = True + + def process_static_or_class_method_in_overload(self, defn: OverloadedFuncDef) -> None: + class_status = [] + static_status = [] + for item in defn.items: + if isinstance(item, Decorator): + inner = item.func + elif isinstance(item, FuncDef): + inner = item + else: + assert False, f"The 'item' variable is an unexpected type: {type(item)}" + class_status.append(inner.is_class) + static_status.append(inner.is_static) + + if defn.impl is not None: + if isinstance(defn.impl, Decorator): + inner = defn.impl.func + elif isinstance(defn.impl, FuncDef): + inner = defn.impl + else: + assert False, f"Unexpected impl type: {type(defn.impl)}" + class_status.append(inner.is_class) + static_status.append(inner.is_static) + + if len(set(class_status)) != 1: + self.msg.overload_inconsistently_applies_decorator("classmethod", defn) + elif len(set(static_status)) != 1: + self.msg.overload_inconsistently_applies_decorator("staticmethod", defn) + else: + defn.is_class = class_status[0] + defn.is_static = static_status[0] + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). - def analyse_function(self, defn: FuncItem) -> None: + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + first_item = defn.items[0] + assert isinstance(first_item, Decorator) + deleted_items = [] + for i, item in enumerate(items[1:]): + if isinstance(item, Decorator): + if len(item.decorators) >= 1: + first_node = item.decorators[0] + if isinstance(first_node, MemberExpr): + if first_node.name == "setter": + # The first item represents the entire property. + first_item.var.is_settable_property = True + # Get abstractness from the original definition. + item.func.abstract_status = first_item.func.abstract_status + if first_node.name == "deleter": + item.func.abstract_status = first_item.func.abstract_status + for other_node in item.decorators[1:]: + other_node.accept(self) + else: + self.fail( + f"Only supported top decorator is @{first_item.func.name}.setter", item + ) + item.func.accept(self) + else: + self.fail(f'Unexpected definition for property "{first_item.func.name}"', item) + deleted_items.append(i + 1) + for i in reversed(deleted_items): + del items[i] + + for item in items[1:]: + if isinstance(item, Decorator): + for d in item.decorators: + if (deprecated := self.get_deprecated(d)) is not None: + item.func.deprecated = ( + f"function {item.fullname} is deprecated: {deprecated}" + ) + + def add_function_to_symbol_table(self, func: FuncDef | OverloadedFuncDef) -> None: + if self.is_class_scope(): + assert self.type is not None + func.info = self.type + func._fullname = self.qualified_name(func.name) + self.add_symbol(func.name, func, func) + + def analyze_arg_initializers(self, defn: FuncItem) -> None: + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + # Analyze default arguments + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + + def analyze_function_body(self, defn: FuncItem) -> None: is_method = self.is_class_scope() - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.info = self.type - defn.type = set_callable_name(defn.type, defn) - for init in defn.init: - if init: - init.rvalue.accept(self) - self.function_stack.append(defn) - self.enter() - for v in defn.args: - self.add_local(v, defn) - for init_ in defn.init: - if init_: - init_.lvalues[0].accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.args: - defn.args[0].is_self = True - - defn.body.accept(self) - disable_typevars(tvarnodes) - self.leave() - self.function_stack.pop() + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + # Bind the type variables again to visit the body. + if defn.type: + a = self.type_analyzer() + typ = defn.type + assert isinstance(typ, CallableType) + a.bind_function_type_variables(typ, defn) + for i in range(len(typ.arg_types)): + store_argument_type(defn, i, typ, self.named_type) + self.function_stack.append(defn) + with self.enter(defn): + for arg in defn.arguments: + self.add_local(arg.variable, defn) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and (not defn.is_static or defn.name == "__new__") and defn.arguments: + if not defn.is_class: + defn.arguments[0].variable.is_self = True + else: + defn.arguments[0].variable.is_cls = True - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = List[SymbolTableNode]() - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(Callable, tt).variables - for i, item in enumerate(items): - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.add_type_var(name, -i - 1, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def add_type_var(self, fullname: str, id: int, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = TVAR - node.tvar_id = id - return node + defn.body.accept(self) + self.function_stack.pop() + + def check_classvar_in_signature(self, typ: ProperType) -> None: + t: ProperType + if isinstance(typ, Overloaded): + for t in typ.items: + self.check_classvar_in_signature(t) + return + if not isinstance(typ, CallableType): + return + for t in get_proper_types(typ.arg_types) + [get_proper_type(typ.ret_type)]: + if self.is_classvar(t): + self.fail_invalid_classvar(t) + # Show only one error per signature + break def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(Callable, fdef.type) - if len(sig.arg_types) < len(fdef.args): - self.fail('Type signature has too few arguments', fdef) - elif len(sig.arg_types) > len(fdef.args): - self.fail('Type signature has too many arguments', fdef) + sig = fdef.type + assert isinstance(sig, CallableType) + if len(sig.arg_types) < len(fdef.arguments): + self.fail("Type signature has too few arguments", fdef) + # Add dummy Any arguments to prevent crashes later. + num_extra_anys = len(fdef.arguments) - len(sig.arg_types) + extra_anys = [AnyType(TypeOfAny.from_error)] * num_extra_anys + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail("Type signature has too many arguments", fdef, blocker=True) + + def check_paramspec_definition(self, defn: FuncDef) -> None: + func = defn.type + assert isinstance(func, CallableType) + + if not any(isinstance(var, ParamSpecType) for var in func.variables): + return # Function does not have param spec variables + + args = func.var_arg() + kwargs = func.kw_arg() + if args is None and kwargs is None: + return # Looks like this function does not have starred args + + args_defn_type = None + kwargs_defn_type = None + for arg_def, arg_kind in zip(defn.arguments, defn.arg_kinds): + if arg_kind == ARG_STAR: + args_defn_type = arg_def.type_annotation + elif arg_kind == ARG_STAR2: + kwargs_defn_type = arg_def.type_annotation + + # This may happen on invalid `ParamSpec` args / kwargs definition, + # type analyzer sets types of arguments to `Any`, but keeps + # definition types as `UnboundType` for now. + if not ( + (isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args")) + or ( + isinstance(kwargs_defn_type, UnboundType) + and kwargs_defn_type.name.endswith(".kwargs") + ) + ): + # Looks like both `*args` and `**kwargs` are not `ParamSpec` + # It might be something else, skipping. + return + + args_type = args.typ if args is not None else None + kwargs_type = kwargs.typ if kwargs is not None else None + + if ( + not isinstance(args_type, ParamSpecType) + or not isinstance(kwargs_type, ParamSpecType) + or args_type.name != kwargs_type.name + ): + if isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"): + param_name = args_defn_type.name.split(".")[0] + elif isinstance(kwargs_defn_type, UnboundType) and kwargs_defn_type.name.endswith( + ".kwargs" + ): + param_name = kwargs_defn_type.name.split(".")[0] + else: + # Fallback for cases that probably should not ever happen: + param_name = "P" + + self.fail( + f'ParamSpec must have "*args" typed as "{param_name}.args" and "**kwargs" typed as "{param_name}.kwargs"', + func, + code=codes.VALID_TYPE, + ) + + def visit_decorator(self, dec: Decorator) -> None: + self.statement = dec + # TODO: better don't modify them at all. + dec.decorators = dec.original_decorators.copy() + dec.func.is_conditional = self.block_depth[-1] > 0 + if not dec.is_overload: + self.add_symbol(dec.name, dec, dec) + dec.func._fullname = self.qualified_name(dec.name) + dec.var._fullname = self.qualified_name(dec.name) + for d in dec.decorators: + d.accept(self) + removed: list[int] = [] + no_type_check = False + could_be_decorated_property = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, "abc.abstractmethod"): + removed.append(i) + dec.func.abstract_status = IS_ABSTRACT + self.check_decorated_function_is_method("abstractmethod", dec) + elif refers_to_fullname(d, ("asyncio.coroutines.coroutine", "types.coroutine")): + removed.append(i) + dec.func.is_awaitable_coroutine = True + elif refers_to_fullname(d, "builtins.staticmethod"): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method("staticmethod", dec) + elif refers_to_fullname(d, "builtins.classmethod"): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method("classmethod", dec) + elif refers_to_fullname(d, OVERRIDE_DECORATOR_NAMES): + removed.append(i) + dec.func.is_explicit_override = True + self.check_decorated_function_is_method("override", dec) + elif refers_to_fullname( + d, + ( + "builtins.property", + "abc.abstractproperty", + "functools.cached_property", + "enum.property", + "types.DynamicClassAttribute", + ), + ): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, "abc.abstractproperty"): + dec.func.abstract_status = IS_ABSTRACT + elif refers_to_fullname(d, "functools.cached_property"): + dec.var.is_settable_property = True + self.check_decorated_function_is_method("property", dec) + elif refers_to_fullname(d, "typing.no_type_check"): + dec.var.type = AnyType(TypeOfAny.special_form) + no_type_check = True + elif refers_to_fullname(d, FINAL_DECORATOR_NAMES): + if self.is_class_scope(): + assert self.type is not None, "No type set at class scope" + if self.type.is_protocol: + self.msg.protocol_members_cant_be_final(d) + else: + dec.func.is_final = True + dec.var.is_final = True + removed.append(i) + else: + self.fail("@final cannot be used with non-method functions", d) + elif refers_to_fullname(d, TYPE_CHECK_ONLY_NAMES): + # TODO: support `@overload` funcs. + dec.func.is_type_check_only = True + elif isinstance(d, CallExpr) and refers_to_fullname( + d.callee, DATACLASS_TRANSFORM_NAMES + ): + dec.func.dataclass_transform_spec = self.parse_dataclass_transform_spec(d) + elif (deprecated := self.get_deprecated(d)) is not None: + dec.func.deprecated = f"function {dec.fullname} is deprecated: {deprecated}" + elif not dec.var.is_property: + # We have seen a "non-trivial" decorator before seeing @property, if + # we will see a @property later, give an error, as we don't support this. + could_be_decorated_property = True + for i in reversed(removed): + del dec.decorators[i] + if (not dec.is_overload or dec.var.is_property) and self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + if not no_type_check and self.recurse_into_functions: + dec.func.accept(self) + if could_be_decorated_property and dec.decorators and dec.var.is_property: + self.fail( + "Decorators on top of @property are not supported", dec, code=PROPERTY_DECORATOR + ) + if (dec.func.is_static or dec.func.is_class) and dec.var.is_property: + self.fail("Only instance methods can be decorated with @property", dec) + if dec.func.abstract_status == IS_ABSTRACT and dec.func.is_final: + self.fail(f"Method {dec.func.name} is both abstract and final", dec) + if dec.func.is_static and dec.func.is_class: + self.fail(message_registry.CLASS_PATTERN_CLASS_OR_STATIC_METHOD, dec) + + def check_decorated_function_is_method(self, decorator: str, context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail(f'"{decorator}" used with a non-method', context) + + # + # Classes + # def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) + self.statement = defn + self.incomplete_type_stack.append(not defn.info) + namespace = self.qualified_name(defn.name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + if self.push_type_args(defn.type_args, defn) is None: + self.mark_incomplete(defn.name, defn) + return - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) + self.analyze_class(defn) + self.pop_type_args(defn.type_args) + self.incomplete_type_stack.pop() + + def push_type_args( + self, type_args: list[TypeParam] | None, context: Context + ) -> list[tuple[str, TypeVarLikeExpr]] | None: + if not type_args: + return [] + self.locals.append(SymbolTable()) + self.scope_stack.append(SCOPE_ANNOTATION) + tvs: list[tuple[str, TypeVarLikeExpr]] = [] + for p in type_args: + tv = self.analyze_type_param(p, context) + if tv is None: + return None + tvs.append((p.name, tv)) + + for name, tv in tvs: + if self.is_defined_type_param(name): + self.fail(f'"{name}" already defined as a type parameter', context) + else: + self.add_symbol(name, tv, context, no_progress=True, type_param=True) + + return tvs + + def is_defined_type_param(self, name: str) -> bool: + for names in self.locals: + if names is None: + continue + if name in names: + node = names[name].node + if isinstance(node, TypeVarLikeExpr): + return True + return False + + def analyze_type_param( + self, type_param: TypeParam, context: Context + ) -> TypeVarLikeExpr | None: + fullname = self.qualified_name(type_param.name) + if type_param.upper_bound: + upper_bound = self.anal_type(type_param.upper_bound, allow_placeholder=True) + # TODO: we should validate the upper bound is valid for a given kind. + if upper_bound is None: + # This and below copies special-casing for old-style type variables, that + # is equally necessary for new-style classes to break a vicious circle. + upper_bound = PlaceholderType(None, [], context.line) + else: + if type_param.kind == TYPE_VAR_TUPLE_KIND: + upper_bound = self.named_type("builtins.tuple", [self.object_type()]) + else: + upper_bound = self.object_type() + if type_param.default: + default = self.anal_type( + type_param.default, + allow_placeholder=True, + allow_unbound_tvars=True, + report_invalid_types=False, + allow_param_spec_literals=type_param.kind == PARAM_SPEC_KIND, + allow_tuple_literal=type_param.kind == PARAM_SPEC_KIND, + allow_unpack=type_param.kind == TYPE_VAR_TUPLE_KIND, + ) + if default is None: + default = PlaceholderType(None, [], context.line) + elif type_param.kind == TYPE_VAR_KIND: + default = self.check_typevar_default(default, type_param.default) + elif type_param.kind == PARAM_SPEC_KIND: + default = self.check_paramspec_default(default, type_param.default) + elif type_param.kind == TYPE_VAR_TUPLE_KIND: + default = self.check_typevartuple_default(default, type_param.default) + else: + default = AnyType(TypeOfAny.from_omitted_generics) + if type_param.kind == TYPE_VAR_KIND: + values: list[Type] = [] + if type_param.values: + for value in type_param.values: + analyzed = self.anal_type(value, allow_placeholder=True) + if analyzed is None: + analyzed = PlaceholderType(None, [], context.line) + if has_type_vars(analyzed): + self.fail(message_registry.TYPE_VAR_GENERIC_CONSTRAINT_TYPE, context) + values.append(AnyType(TypeOfAny.from_error)) + else: + values.append(analyzed) + return TypeVarExpr( + name=type_param.name, + fullname=fullname, + values=values, + upper_bound=upper_bound, + default=default, + variance=VARIANCE_NOT_READY, + is_new_style=True, + line=context.line, + ) + elif type_param.kind == PARAM_SPEC_KIND: + return ParamSpecExpr( + name=type_param.name, + fullname=fullname, + upper_bound=upper_bound, + default=default, + is_new_style=True, + line=context.line, + ) + else: + assert type_param.kind == TYPE_VAR_TUPLE_KIND + tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) + return TypeVarTupleExpr( + name=type_param.name, + fullname=fullname, + upper_bound=upper_bound, + tuple_fallback=tuple_fallback, + default=default, + is_new_style=True, + line=context.line, + ) + + def pop_type_args(self, type_args: list[TypeParam] | None) -> None: + if not type_args: + return + self.locals.pop() + self.scope_stack.pop() + + def analyze_class(self, defn: ClassDef) -> None: + fullname = self.qualified_name(defn.name) + if not defn.info and not self.is_core_builtin_class(defn): + # Add placeholder so that self-references in base classes can be + # resolved. We don't want this to cause a deferral, since if there + # are no incomplete references, we'll replace this with a TypeInfo + # before returning. + placeholder = PlaceholderNode(fullname, defn, defn.line, becomes_typeinfo=True) + self.add_symbol(defn.name, placeholder, defn, can_defer=False) + + tag = self.track_incomplete_refs() + + # Restore base classes after previous iteration (things like Generic[T] might be removed). + defn.base_type_exprs.extend(defn.removed_base_type_exprs) + defn.removed_base_type_exprs.clear() + + self.infer_metaclass_and_bases_from_compat_helpers(defn) + + bases = defn.base_type_exprs + bases, tvar_defs, is_protocol = self.clean_up_bases_and_infer_type_variables( + defn, bases, context=defn + ) + + self.check_type_alias_bases(bases) + + for tvd in tvar_defs: + if isinstance(tvd, TypeVarType) and any( + has_placeholder(t) for t in [tvd.upper_bound] + tvd.values + ): + # Some type variable bounds or values are not ready, we need + # to re-analyze this class. + self.defer() + if has_placeholder(tvd.default): + # Placeholder values in TypeVarLikeTypes may get substituted in. + # Defer current target until they are ready. + self.mark_incomplete(defn.name, defn) + return + + self.analyze_class_keywords(defn) + bases_result = self.analyze_base_classes(bases) + if bases_result is None or self.found_incomplete_ref(tag): + # Something was incomplete. Defer current target. + self.mark_incomplete(defn.name, defn) + return + + base_types, base_error = bases_result + if any(isinstance(base, PlaceholderType) for base, _ in base_types): + # We need to know the TypeInfo of each base to construct the MRO. Placeholder types + # are okay in nested positions, since they can't affect the MRO. + self.mark_incomplete(defn.name, defn) + return + + declared_metaclass, should_defer, any_meta = self.get_declared_metaclass( + defn.name, defn.metaclass + ) + if should_defer or self.found_incomplete_ref(tag): + # Metaclass was not ready. Defer current target. + self.mark_incomplete(defn.name, defn) + return + + if self.analyze_typeddict_classdef(defn): + if defn.info: + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) + return + + if self.analyze_namedtuple_classdef(defn, tvar_defs): + return + + # Create TypeInfo for class now that base classes and the MRO can be calculated. + self.prepare_class_def(defn) + self.setup_type_vars(defn, tvar_defs) + if base_error: + defn.info.fallback_to_any = True + if any_meta: + defn.info.meta_fallback_to_any = True + + with self.scope.class_scope(defn.info): + self.configure_base_classes(defn, base_types) + defn.info.is_protocol = is_protocol + self.recalculate_metaclass(defn, declared_metaclass) + defn.info.runtime_protocol = False + + if defn.type_args: + # PEP 695 type parameters are not in scope in class decorators, so + # temporarily disable type parameter namespace. + type_params_names = self.locals.pop() + self.scope_stack.pop() + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + if defn.type_args: + self.locals.append(type_params_names) + self.scope_stack.append(SCOPE_ANNOTATION) + + self.analyze_class_body_common(defn) + + def check_type_alias_bases(self, bases: list[Expression]) -> None: + for base in bases: + if isinstance(base, IndexExpr): + base = base.base + if ( + isinstance(base, RefExpr) + and isinstance(base.node, TypeAlias) + and base.node.python_3_12_type_alias + ): + self.fail( + 'Type alias defined using "type" statement not valid as base class', base + ) + + def setup_type_vars(self, defn: ClassDef, tvar_defs: list[TypeVarLikeType]) -> None: + defn.type_vars = tvar_defs + defn.info.type_vars = [] + # we want to make sure any additional logic in add_type_vars gets run + defn.info.add_type_vars() + + def setup_alias_type_vars(self, defn: ClassDef) -> None: + assert defn.info.special_alias is not None + defn.info.special_alias.alias_tvars = list(defn.type_vars) + # It is a bit unfortunate that we need to inline some logic from TypeAlias constructor, + # but it is required, since type variables may change during semantic analyzer passes. + for i, t in enumerate(defn.type_vars): + if isinstance(t, TypeVarTupleType): + defn.info.special_alias.tvar_tuple_index = i + target = defn.info.special_alias.target + assert isinstance(target, ProperType) + if isinstance(target, TypedDictType): + target.fallback.args = type_vars_as_args(defn.type_vars) + elif isinstance(target, TupleType): + target.partial_fallback.args = type_vars_as_args(defn.type_vars) + else: + assert False, f"Unexpected special alias type: {type(target)}" + + def is_core_builtin_class(self, defn: ClassDef) -> bool: + return self.cur_mod_id == "builtins" and defn.name in CORE_BUILTIN_CLASSES - # Analyze class body. + def analyze_class_body_common(self, defn: ClassDef) -> None: + """Parts of class body analysis that are common to all kinds of class defs.""" + self.enter_class(defn.info) + if any(b.self_type is not None for b in defn.info.mro): + self.setup_self_type() defn.defs.accept(self) + self.apply_class_plugin_hooks(defn) + self.leave_class() + + def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: + if ( + defn.info + and defn.info.typeddict_type + and not has_placeholder(defn.info.typeddict_type) + ): + # This is a valid TypedDict, and it is fully analyzed. + return True + is_typeddict, info = self.typed_dict_analyzer.analyze_typeddict_classdef(defn) + if is_typeddict: + for decorator in defn.decorators: + decorator.accept(self) + if info is not None: + self.analyze_class_decorator_common(defn, info, decorator) + if info is None: + self.mark_incomplete(defn.name, defn) + else: + self.prepare_class_def(defn, info, custom_names=True) + return True + return False + + def analyze_namedtuple_classdef( + self, defn: ClassDef, tvar_defs: list[TypeVarLikeType] + ) -> bool: + """Check if this class can define a named tuple.""" + if ( + defn.info + and defn.info.is_named_tuple + and defn.info.tuple_type + and not has_placeholder(defn.info.tuple_type) + ): + # Don't reprocess everything. We just need to process methods defined + # in the named tuple class body. + is_named_tuple = True + info: TypeInfo | None = defn.info + else: + is_named_tuple, info = self.named_tuple_analyzer.analyze_namedtuple_classdef( + defn, self.is_stub_file, self.is_func_scope() + ) + if is_named_tuple: + if info is None: + self.mark_incomplete(defn.name, defn) + else: + self.prepare_class_def(defn, info, custom_names=True) + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) + with self.scope.class_scope(defn.info): + for deco in defn.decorators: + deco.accept(self) + self.analyze_class_decorator_common(defn, defn.info, deco) + with self.named_tuple_analyzer.save_namedtuple_body(info): + self.analyze_class_body_common(defn) + return True + return False + + def apply_class_plugin_hooks(self, defn: ClassDef) -> None: + """Apply a plugin hook that may infer a more precise definition for a class.""" + + for decorator in defn.decorators: + decorator_name = self.get_fullname_for_hook(decorator) + if decorator_name: + hook = self.plugin.get_class_decorator_hook(decorator_name) + # Special case: if the decorator is itself decorated with + # typing.dataclass_transform, apply the hook for the dataclasses plugin + # TODO: remove special casing here + if hook is None and find_dataclass_transform_spec(decorator): + hook = dataclasses_plugin.dataclass_tag_callback + if hook: + hook(ClassDefContext(defn, decorator, self)) + + if defn.metaclass: + metaclass_name = self.get_fullname_for_hook(defn.metaclass) + if metaclass_name: + hook = self.plugin.get_metaclass_hook(metaclass_name) + if hook: + hook(ClassDefContext(defn, defn.metaclass, self)) + + for base_expr in defn.base_type_exprs: + base_name = self.get_fullname_for_hook(base_expr) + if base_name: + hook = self.plugin.get_base_class_hook(base_name) + if hook: + hook(ClassDefContext(defn, base_expr, self)) + + # Check if the class definition itself triggers a dataclass transform (via a parent class/ + # metaclass) + spec = find_dataclass_transform_spec(defn) + if spec is not None: + dataclasses_plugin.add_dataclass_tag(defn.info) + + def get_fullname_for_hook(self, expr: Expression) -> str | None: + if isinstance(expr, CallExpr): + return self.get_fullname_for_hook(expr.callee) + elif isinstance(expr, IndexExpr): + return self.get_fullname_for_hook(expr.base) + elif isinstance(expr, RefExpr): + if expr.fullname: + return expr.fullname + # If we don't have a fullname look it up. This happens because base classes are + # analyzed in a different manner (see exprtotype.py) and therefore those AST + # nodes will not have full names. + sym = self.lookup_type_node(expr) + if sym: + return sym.fullname + return None - self.calculate_abstract_status(defn.info) - self.setup_ducktyping(defn) + def analyze_class_keywords(self, defn: ClassDef) -> None: + for value in defn.keywords.values(): + value.accept(self) + + def enter_class(self, info: TypeInfo) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.scope_stack.append(SCOPE_CLASS) + self.block_depth.append(-1) # The class body increments this to 0 + self.loop_depth.append(0) + self._type = info + self.missing_names.append(set()) - # Restore analyzer state. + def leave_class(self) -> None: + """Restore analyzer state.""" self.block_depth.pop() + self.loop_depth.pop() self.locals.pop() - self.type, tvarnodes = self.type_stack.pop() - disable_typevars(tvarnodes) - if self.type_stack: - # Enable type variables of the enclosing class again. - enable_typevars(self.type_stack[-1][1]) + self.scope_stack.pop() + self._type = self.type_stack.pop() + self.missing_names.pop() - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None: decorator.accept(self) - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = Set[str]() - abstract = List[str]() - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node + self.analyze_class_decorator_common(defn, defn.info, decorator) + if isinstance(decorator, RefExpr): + if decorator.fullname in RUNTIME_PROTOCOL_DECOS: + if defn.info.is_protocol: + defn.info.runtime_protocol = True else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_ducktyping(self, defn: ClassDef) -> None: - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, DucktypeExpr): - defn.info.ducktype = analyzed.type - elif isinstance(analyzed, DisjointclassExpr): - node = analyzed.cls.node - if isinstance(node, TypeInfo): - defn.info.disjoint_classes.append(node) - defn.info.disjointclass_decls.append(node) - node.disjoint_classes.append(defn.info) - else: - self.fail('Argument 1 to disjointclass does not refer ' - 'to a class', analyzed) - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + self.fail("@runtime_checkable can only be used with protocol classes", defn) + elif isinstance(decorator, CallExpr) and refers_to_fullname( + decorator.callee, DATACLASS_TRANSFORM_NAMES + ): + defn.info.dataclass_transform_spec = self.parse_dataclass_transform_spec(decorator) + + def analyze_class_decorator_common( + self, defn: ClassDef, info: TypeInfo, decorator: Expression + ) -> None: + """Common method for applying class decorators. + + Called on regular classes, typeddicts, and namedtuples. + """ + if refers_to_fullname(decorator, FINAL_DECORATOR_NAMES): + info.is_final = True + elif refers_to_fullname(decorator, TYPE_CHECK_ONLY_NAMES): + info.is_type_check_only = True + elif (deprecated := self.get_deprecated(decorator)) is not None: + info.deprecated = f"class {defn.fullname} is deprecated: {deprecated}" + + def clean_up_bases_and_infer_type_variables( + self, defn: ClassDef, base_type_exprs: list[Expression], context: Context + ) -> tuple[list[Expression], list[TypeVarLikeType], bool]: """Remove extra base classes such as Generic and infer type vars. For example, consider this class: - class Foo(Bar, Generic[T]): ... + class Foo(Bar, Generic[T]): ... Now we will remove Generic[T] from bases of Foo and infer that the type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + + Returns (remaining base expressions, inferred type variables, is protocol). """ - removed = List[int]() - type_vars = List[TypeVarDef]() - for i, base in enumerate(defn.base_types): - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic or AbstractGeneric in bases', - defn) + removed: list[int] = [] + declared_tvars: TypeVarLikeList = [] + is_protocol = False + if defn.type_args is not None: + for p in defn.type_args: + node = self.lookup(p.name, context) + assert node is not None + assert isinstance(node.node, TypeVarLikeExpr) + declared_tvars.append((p.name, node.node)) + + for i, base_expr in enumerate(base_type_exprs): + if isinstance(base_expr, StarExpr): + base_expr.valid = True + self.analyze_type_expr(base_expr) + + try: + base = self.expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + result = self.analyze_class_typevar_declaration(base) + if result is not None: + tvars = result[0] + is_protocol |= result[1] + if declared_tvars: + if defn.type_args: + if is_protocol: + self.fail('No arguments expected for "Protocol" base class', context) + else: + self.fail("Generic[...] base class is redundant", context) + else: + self.fail( + "Only single Generic[...] or Protocol[...] can be in bases", context + ) removed.append(i) - for j, tvar in enumerate(tvars): - name, values = tvar - type_vars.append(TypeVarDef(name, j + 1, values, - self.object_type())) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] + declared_tvars.extend(tvars) + if isinstance(base, UnboundType): + sym = self.lookup_qualified(base.name, base) + if sym is not None and sym.node is not None: + if sym.node.fullname in PROTOCOL_NAMES and i not in removed: + # also remove bare 'Protocol' bases + removed.append(i) + is_protocol = True + + all_tvars = self.get_all_bases_tvars(base_type_exprs, removed) + if declared_tvars: + if len(remove_dups(declared_tvars)) < len(declared_tvars) and not defn.type_args: + self.fail("Duplicate type variables in Generic[...] or Protocol[...]", context) + declared_tvars = remove_dups(declared_tvars) + if not set(all_tvars).issubset(set(declared_tvars)): + if defn.type_args: + undeclared = sorted(set(all_tvars) - set(declared_tvars)) + self.msg.type_parameters_should_be_declared( + [tv[0] for tv in undeclared], context + ) + else: + self.fail( + "If Generic[...] or Protocol[...] is present" + " it should list all type variables", + context, + ) + # In case of error, Generic tvars will go first + declared_tvars = remove_dups(declared_tvars + all_tvars) + else: + declared_tvars = all_tvars for i in reversed(removed): - del defn.base_types[i] - - def analyze_typevar_declaration(self, t: Type) -> List[Tuple[str, - List[Type]]]: - if not isinstance(t, UnboundType): + # We need to actually remove the base class expressions like Generic[T], + # mostly because otherwise they will create spurious dependencies in fine + # grained incremental mode. + defn.removed_base_type_exprs.append(defn.base_type_exprs[i]) + del base_type_exprs[i] + tvar_defs = self.tvar_defs_from_tvars(declared_tvars, context) + return base_type_exprs, tvar_defs, is_protocol + + def analyze_class_typevar_declaration(self, base: Type) -> tuple[TypeVarLikeList, bool] | None: + """Analyze type variables declared using Generic[...] or Protocol[...]. + + Args: + base: Non-analyzed base class + + Return None if the base class does not declare type variables. Otherwise, + return the type variables. + """ + if not isinstance(base, UnboundType): return None - unbound = cast(UnboundType, t) + unbound = base sym = self.lookup_qualified(unbound.name, unbound) - if sym is None: + if sym is None or sym.node is None: return None - if sym.node.fullname() in ('typing.Generic', - 'typing.AbstractGeneric'): - tvars = List[Tuple[str, List[Type]]]() + if ( + sym.node.fullname == "typing.Generic" + or sym.node.fullname in PROTOCOL_NAMES + and base.args + ): + is_proto = sym.node.fullname != "typing.Generic" + tvars: TypeVarLikeList = [] + have_type_var_tuple = False for arg in unbound.args: + tag = self.track_incomplete_refs() tvar = self.analyze_unbound_tvar(arg) if tvar: + if isinstance(tvar[1], TypeVarTupleExpr): + if have_type_var_tuple: + self.fail("Can only use one type var tuple in a class def", base) + continue + have_type_var_tuple = True tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars + elif not self.found_incomplete_ref(tag): + self.fail("Free type variable expected in %s[...]" % sym.node.name, base) + return tvars, is_proto return None - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, List[Type]]: - if not isinstance(t, UnboundType): - return None - unbound = cast(UnboundType, t) - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node).values[:] + def analyze_unbound_tvar(self, t: Type) -> tuple[str, TypeVarLikeExpr] | None: + if isinstance(t, UnpackType) and isinstance(t.type, UnboundType): + return self.analyze_unbound_tvar_impl(t.type, is_unpacked=True) + if isinstance(t, UnboundType): + sym = self.lookup_qualified(t.name, t) + if sym and sym.fullname in ("typing.Unpack", "typing_extensions.Unpack"): + inner_t = t.args[0] + if isinstance(inner_t, UnboundType): + return self.analyze_unbound_tvar_impl(inner_t, is_unpacked=True) + return None + return self.analyze_unbound_tvar_impl(t) return None - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" + def analyze_unbound_tvar_impl( + self, t: UnboundType, is_unpacked: bool = False, is_typealias_param: bool = False + ) -> tuple[str, TypeVarLikeExpr] | None: + assert not is_unpacked or not is_typealias_param, "Mutually exclusive conditions" + sym = self.lookup_qualified(t.name, t) + if sym and isinstance(sym.node, PlaceholderNode): + self.record_incomplete_ref() + if not is_unpacked and sym and isinstance(sym.node, ParamSpecExpr): + if sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): + # It's bound by our type variable scope + return None + return t.name, sym.node + if (is_unpacked or is_typealias_param) and sym and isinstance(sym.node, TypeVarTupleExpr): + if sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): + # It's bound by our type variable scope + return None + return t.name, sym.node + if sym is None or not isinstance(sym.node, TypeVarExpr) or is_unpacked: + return None + elif sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): + # It's bound by our type variable scope + return None + else: + assert isinstance(sym.node, TypeVarExpr) + return t.name, sym.node + + def find_type_var_likes(self, t: Type) -> TypeVarLikeList: + visitor = FindTypeVarVisitor(self, self.tvar_scope) + t.accept(visitor) + return visitor.type_var_likes + + def get_all_bases_tvars( + self, base_type_exprs: list[Expression], removed: list[int] + ) -> TypeVarLikeList: + """Return all type variable references in bases.""" + tvars: TypeVarLikeList = [] + for i, base_expr in enumerate(base_type_exprs): + if i not in removed: + try: + base = self.expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + base_tvars = self.find_type_var_likes(base) + tvars.extend(base_tvars) + return remove_dups(tvars) + + def tvar_defs_from_tvars( + self, tvars: TypeVarLikeList, context: Context + ) -> list[TypeVarLikeType]: + tvar_defs: list[TypeVarLikeType] = [] + last_tvar_name_with_default: str | None = None + for name, tvar_expr in tvars: + tvar_expr.default = tvar_expr.default.accept( + TypeVarDefaultTranslator(self, tvar_expr.name, context) + ) + tvar_def = self.tvar_scope.bind_new(name, tvar_expr) + if last_tvar_name_with_default is not None and not tvar_def.has_default(): + self.msg.tvar_without_default_type( + tvar_def.name, last_tvar_name_with_default, context + ) + tvar_def.default = AnyType(TypeOfAny.from_error) + elif tvar_def.has_default(): + last_tvar_name_with_default = tvar_def.name + tvar_defs.append(tvar_def) + return tvar_defs + + def get_and_bind_all_tvars(self, type_exprs: list[Expression]) -> list[TypeVarLikeType]: + """Return all type variable references in item type expressions. + + This is a helper for generic TypedDicts and NamedTuples. Essentially it is + a simplified version of the logic we use for ClassDef bases. We duplicate + some amount of code, because it is hard to refactor common pieces. + """ + tvars = [] + for base_expr in type_exprs: + try: + base = self.expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + base_tvars = self.find_type_var_likes(base) + tvars.extend(base_tvars) + tvars = remove_dups(tvars) # Variables are defined in order of textual appearance. + tvar_defs = [] + for name, tvar_expr in tvars: + tvar_def = self.tvar_scope.bind_new(name, tvar_expr) + tvar_defs.append(tvar_def) + return tvar_defs + + def prepare_class_def( + self, defn: ClassDef, info: TypeInfo | None = None, custom_names: bool = False + ) -> None: + """Prepare for the analysis of a class definition. + + Create an empty TypeInfo and store it in a symbol table, or if the 'info' + argument is provided, store it instead (used for magic type definitions). + """ if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - if self.type_stack: - # Disable type variables of the enclosing class. - disable_typevars(self.type_stack[-1][1]) - tvarnodes = self.add_class_type_variables_to_symbol_table(defn.info) - # Remember previous active class and type vars of *this* class. - self.type_stack.append((self.type, tvarnodes)) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.type = defn.info + defn.fullname = self.qualified_name(defn.name) + # TODO: Nested classes + info = info or self.make_empty_type_info(defn) + defn.info = info + info.defn = defn + if not custom_names: + # Some special classes (in particular NamedTuples) use custom fullname logic. + # Don't override it here (also see comment below, this needs cleanup). + if not self.is_func_scope(): + info._fullname = self.qualified_name(defn.name) + else: + info._fullname = info.name + local_name = defn.name + if "@" in local_name: + local_name = local_name.split("@")[0] + self.add_symbol(local_name, defn.info, defn) + if self.is_nested_within_func_scope(): + # We need to preserve local classes, let's store them + # in globals under mangled unique names + # + # TODO: Putting local classes into globals breaks assumptions in fine-grained + # incremental mode and we should avoid it. In general, this logic is too + # ad-hoc and needs to be removed/refactored. + if "@" not in defn.info._fullname: + global_name = defn.info.name + "@" + str(defn.line) + defn.info._fullname = self.cur_mod_id + "." + global_name + else: + # Preserve name from previous fine-grained incremental run. + global_name = defn.info.name + defn.fullname = defn.info._fullname + if defn.info.is_named_tuple or defn.info.typeddict_type: + # Named tuples and Typed dicts nested within a class are stored + # in the class symbol table. + self.add_symbol_skip_local(global_name, defn.info) + else: + self.globals[global_name] = SymbolTableNode(GDEF, defn.info) + + def make_empty_type_info(self, defn: ClassDef) -> TypeInfo: + if ( + self.is_module_scope() + and self.cur_mod_id == "builtins" + and defn.name in CORE_BUILTIN_CLASSES + ): + # Special case core built-in classes. A TypeInfo was already + # created for it before semantic analysis, but with a dummy + # ClassDef. Patch the real ClassDef object. + info = self.globals[defn.name].node + assert isinstance(info, TypeInfo) + else: + info = TypeInfo(SymbolTable(), defn, self.cur_mod_id) + info.set_line(defn) + return info + + def get_name_repr_of_expr(self, expr: Expression) -> str | None: + """Try finding a short simplified textual representation of a base class expression.""" + if isinstance(expr, NameExpr): + return expr.name + if isinstance(expr, MemberExpr): + return get_member_expr_fullname(expr) + if isinstance(expr, IndexExpr): + return self.get_name_repr_of_expr(expr.base) + if isinstance(expr, CallExpr): + return self.get_name_repr_of_expr(expr.callee) + return None + + def analyze_base_classes( + self, base_type_exprs: list[Expression] + ) -> tuple[list[tuple[ProperType, Expression]], bool] | None: + """Analyze base class types. + + Return None if some definition was incomplete. Otherwise, return a tuple + with these items: + + * List of (analyzed type, original expression) tuples + * Boolean indicating whether one of the bases had a semantic analysis error + """ + is_error = False + bases = [] + for base_expr in base_type_exprs: + if ( + isinstance(base_expr, RefExpr) + and base_expr.fullname in TYPED_NAMEDTUPLE_NAMES + TPDICT_NAMES + ) or ( + isinstance(base_expr, CallExpr) + and isinstance(base_expr.callee, RefExpr) + and base_expr.callee.fullname in TPDICT_NAMES + ): + # Ignore magic bases for now. + # For example: + # class Foo(TypedDict): ... # RefExpr + # class Foo(NamedTuple): ... # RefExpr + # class Foo(TypedDict("Foo", {"a": int})): ... # CallExpr + continue + + try: + base = self.expr_to_analyzed_type( + base_expr, allow_placeholder=True, allow_type_any=True + ) + except TypeTranslationError: + name = self.get_name_repr_of_expr(base_expr) + if isinstance(base_expr, CallExpr): + msg = "Unsupported dynamic base class" + else: + msg = "Invalid base class" + if name: + msg += f' "{name}"' + self.fail(msg, base_expr) + is_error = True + continue + if base is None: + return None + base = get_proper_type(base) + bases.append((base, base_expr)) + return bases, is_error + + def configure_base_classes( + self, defn: ClassDef, bases: list[tuple[ProperType, Expression]] + ) -> None: + """Set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + base_types: list[Instance] = [] + info = defn.info - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes.""" - bases = List[Instance]() - for i in range(len(defn.base_types)): - base = self.anal_type(defn.base_types[i]) + for base, base_expr in bases: if isinstance(base, TupleType): - if defn.info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - defn.info.tuple_type = base - base = base.fallback - if isinstance(base, Instance): - defn.base_types[i] = base - bases.append(base) + actual_base = self.configure_tuple_base_class(defn, base) + base_types.append(actual_base) + elif isinstance(base, Instance): + if base.type.is_newtype: + self.fail('Cannot subclass "NewType"', defn) + base_types.append(base) + elif isinstance(base, AnyType): + if self.options.disallow_subclassing_any: + if isinstance(base_expr, (NameExpr, MemberExpr)): + msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' + else: + msg = 'Class cannot subclass value of type "Any"' + self.fail(msg, base_expr) + info.fallback_to_any = True + elif isinstance(base, TypedDictType): + base_types.append(base.fallback) + else: + msg = "Invalid base class" + name = self.get_name_repr_of_expr(base_expr) + if name: + msg += f' "{name}"' + self.fail(msg, base_expr) + info.fallback_to_any = True + if self.options.disallow_any_unimported and has_any_from_unimported_type(base): + if isinstance(base_expr, (NameExpr, MemberExpr)): + prefix = f"Base type {base_expr.name}" + else: + prefix = "Base type" + self.msg.unimported_type_becomes_any(prefix, base, base_expr) + check_for_explicit_any( + base, self.options, self.is_typeshed_stub_file, self.msg, context=base_expr + ) + # Add 'object' as implicit base if there is no other base class. - if (not bases and defn.fullname != 'builtins.object'): - obj = self.object_type() - defn.base_types.insert(0, obj) - bases.append(obj) - defn.info.bases = bases + if not base_types and defn.fullname != "builtins.object": + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. if not self.verify_base_classes(defn): + self.set_dummy_mro(defn.info) return + if not self.verify_duplicate_base_classes(defn): + # We don't want to block the typechecking process, + # so, we just insert `Any` as the base class and show an error. + self.set_any_mro(defn.info) + self.calculate_class_mro(defn, self.object_type) + + def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instance: + info = defn.info + + # There may be an existing valid tuple type from previous semanal iterations. + # Use equality to check if it is the case. + if info.tuple_type and info.tuple_type != base and not has_placeholder(info.tuple_type): + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.has_incompatible_baseclass = True + if info.special_alias and has_placeholder(info.special_alias.target): + self.process_placeholder( + None, "tuple base", defn, force_progress=base != info.tuple_type + ) + info.update_tuple_type(base) + self.setup_alias_type_vars(defn) + + if base.partial_fallback.type.fullname == "builtins.tuple" and not has_placeholder(base): + # Fallback can only be safely calculated after semantic analysis, since base + # classes may be incomplete. Postpone the calculation. + self.schedule_patch(PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(base)) + + return base.partial_fallback + + def set_dummy_mro(self, info: TypeInfo) -> None: + # Give it an MRO consisting of just the class itself and object. + info.mro = [info, self.object_type().type] + info.bad_mro = True + + def set_any_mro(self, info: TypeInfo) -> None: + # Give it an MRO consisting direct `Any` subclass. + info.fallback_to_any = True + info.mro = [info, self.object_type().type] + + def calculate_class_mro( + self, defn: ClassDef, obj_type: Callable[[], Instance] | None = None + ) -> None: + """Calculate method resolution order for a class. + + `obj_type` exists just to fill in empty base class list in case of an error. + """ try: - defn.info.calculate_mro() + calculate_mro(defn.info, obj_type) except MroError: - self.fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - else: - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if defn.info.mro[-1].fullname() != 'builtins.object': - defn.info.mro.append(self.object_type().type) + self.fail( + f'Cannot determine consistent method resolution order (MRO) for "{defn.name}"', + defn, + ) + self.set_dummy_mro(defn.info) + # Allow plugins to alter the MRO to handle the fact that `def mro()` + # on metaclasses permits MRO rewriting. + if defn.fullname: + hook = self.plugin.get_customize_class_mro_hook(defn.fullname) + if hook: + hook(ClassDefContext(defn, FakeExpression(), self)) + + def infer_metaclass_and_bases_from_compat_helpers(self, defn: ClassDef) -> None: + """Lookup for special metaclass declarations, and update defn fields accordingly. + + * six.with_metaclass(M, B1, B2, ...) + * @six.add_metaclass(M) + * future.utils.with_metaclass(M, B1, B2, ...) + * past.utils.with_metaclass(M, B1, B2, ...) + """ + + # Look for six.with_metaclass(M, B1, B2, ...) + with_meta_expr: Expression | None = None + if len(defn.base_type_exprs) == 1: + base_expr = defn.base_type_exprs[0] + if isinstance(base_expr, CallExpr) and isinstance(base_expr.callee, RefExpr): + self.analyze_type_expr(base_expr) + if ( + base_expr.callee.fullname + in { + "six.with_metaclass", + "future.utils.with_metaclass", + "past.utils.with_metaclass", + } + and len(base_expr.args) >= 1 + and all(kind == ARG_POS for kind in base_expr.arg_kinds) + ): + with_meta_expr = base_expr.args[0] + defn.base_type_exprs = base_expr.args[1:] + + # Look for @six.add_metaclass(M) + add_meta_expr: Expression | None = None + for dec_expr in defn.decorators: + if isinstance(dec_expr, CallExpr) and isinstance(dec_expr.callee, RefExpr): + dec_expr.callee.accept(self) + if ( + dec_expr.callee.fullname == "six.add_metaclass" + and len(dec_expr.args) == 1 + and dec_expr.arg_kinds[0] == ARG_POS + ): + add_meta_expr = dec_expr.args[0] + break + + metas = {defn.metaclass, with_meta_expr, add_meta_expr} - {None} + if len(metas) == 0: + return + if len(metas) > 1: + self.fail("Multiple metaclass definitions", defn) + return + defn.metaclass = metas.pop() def verify_base_classes(self, defn: ClassDef) -> bool: - base_classes = List[str]() info = defn.info + cycle = False for base in info.bases: baseinfo = base.type if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) + self.fail("Cycle in inheritance hierarchy", defn) + cycle = True + return not cycle + + def verify_duplicate_base_classes(self, defn: ClassDef) -> bool: + dup = find_duplicate(defn.info.direct_base_classes()) if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True + self.fail(f'Duplicate base class "{dup.name}"', defn) + return not dup def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: """Determine if t is a base class of s (but do not use mro).""" @@ -565,766 +2759,2874 @@ def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: visited.add(base.type) return False - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') + def get_declared_metaclass( + self, name: str, metaclass_expr: Expression | None + ) -> tuple[Instance | None, bool, bool]: + """Get declared metaclass from metaclass expression. - def named_type(self, qualified_name: str) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), []) + Returns a tuple of three values: + * A metaclass instance or None + * A boolean indicating whether we should defer + * A boolean indicating whether we should set metaclass Any fallback + (either for Any metaclass or invalid/dynamic metaclass). - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) + The two boolean flags can only be True if instance is None. + """ + declared_metaclass = None + if metaclass_expr: + metaclass_name = None + if isinstance(metaclass_expr, NameExpr): + metaclass_name = metaclass_expr.name + elif isinstance(metaclass_expr, MemberExpr): + metaclass_name = get_member_expr_fullname(metaclass_expr) + if metaclass_name is None: + self.fail(f'Dynamic metaclass not supported for "{name}"', metaclass_expr) + return None, False, True + sym = self.lookup_qualified(metaclass_name, metaclass_expr) + if sym is None: + # Probably a name error - it is already handled elsewhere + return None, False, True + if isinstance(sym.node, Var) and isinstance(get_proper_type(sym.node.type), AnyType): + if self.options.disallow_subclassing_any: + self.fail( + f'Class cannot use "{sym.node.name}" as a metaclass (has type "Any")', + metaclass_expr, + ) + return None, False, True + if isinstance(sym.node, PlaceholderNode): + return None, True, False # defer later in the caller + + # Support type aliases, like `_Meta: TypeAlias = type` + if ( + isinstance(sym.node, TypeAlias) + and sym.node.no_args + and isinstance(sym.node.target, ProperType) + and isinstance(sym.node.target, Instance) + ): + metaclass_info: Node | None = sym.node.target.type + else: + metaclass_info = sym.node + + if not isinstance(metaclass_info, TypeInfo) or metaclass_info.tuple_type is not None: + self.fail(f'Invalid metaclass "{metaclass_name}"', metaclass_expr) + return None, False, False + if not metaclass_info.is_metaclass(): + self.fail( + 'Metaclasses not inheriting from "type" are not supported', metaclass_expr + ) + return None, False, False + inst = fill_typevars(metaclass_info) + assert isinstance(inst, Instance) + declared_metaclass = inst + return declared_metaclass, False, False + + def recalculate_metaclass(self, defn: ClassDef, declared_metaclass: Instance | None) -> None: + defn.info.declared_metaclass = declared_metaclass + defn.info.metaclass_type = defn.info.calculate_metaclass_type() + if any(info.is_protocol for info in defn.info.mro): + if ( + not defn.info.metaclass_type + or defn.info.metaclass_type.type.fullname == "builtins.type" + ): + # All protocols and their subclasses have ABCMeta metaclass by default. + # TODO: add a metaclass conflict check if there is another metaclass. + abc_meta = self.named_type_or_none("abc.ABCMeta", []) + if abc_meta is not None: # May be None in tests with incomplete lib-stub. + defn.info.metaclass_type = abc_meta + if defn.info.metaclass_type and defn.info.metaclass_type.type.has_base("enum.EnumMeta"): + defn.info.is_enum = True + if defn.type_vars: + self.fail("Enum class cannot be generic", defn) - def add_class_type_variables_to_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - vars = info.type_vars - nodes = List[SymbolTableNode]() - if vars: - for i in range(len(vars)): - node = self.add_type_var(vars[i], i + 1, info) - nodes.append(node) - return nodes + # + # Imports + # def visit_import(self, i: Import) -> None: + self.statement = i for id, as_id in i.ids: - if as_id != id: - self.add_module_symbol(id, as_id, i) + # Modules imported in a stub file without using 'import X as X' won't get exported + # When implicit re-exporting is disabled, we have the same behavior as stubs. + use_implicit_reexport = not self.is_stub_file and self.options.implicit_reexport + if as_id is not None: + base_id = id + imported_id = as_id + module_public = use_implicit_reexport or id == as_id else: - base = id.split('.')[0] - self.add_module_symbol(base, base, i) - - def add_module_symbol(self, id: str, as_id: str, context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, i: ImportFrom) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for id, as_id in i.names: - node = m.names.get(id, None) - if node: - node = self.normalize_type_alias(node, i) - if not node: - return - self.add_symbol(as_id, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) + base_id = id.split(".")[0] + imported_id = base_id + module_public = use_implicit_reexport + + if base_id in self.modules: + node = self.modules[base_id] + if self.is_func_scope(): + kind = LDEF + elif self.type is not None: + kind = MDEF else: - self.fail("Module has no attribute '{}'".format(id), i) - else: - for id, as_id in i.names: - self.add_unknown_symbol(as_id, i) - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id + kind = GDEF + symbol = SymbolTableNode( + kind, node, module_public=module_public, module_hidden=not module_public + ) + self.add_imported_symbol( + imported_id, + symbol, + context=i, + module_public=module_public, + module_hidden=not module_public, + ) + else: + self.add_unknown_imported_symbol( + imported_id, + context=i, + target_name=base_id, + module_public=module_public, + module_hidden=not module_public, + ) + + def visit_import_from(self, imp: ImportFrom) -> None: + self.statement = imp + module_id = self.correct_relative_import(imp) + module = self.modules.get(module_id) + for id, as_id in imp.names: + fullname = module_id + "." + id + self.set_future_import_flags(fullname) + if module is None: + node = None + elif module_id == self.cur_mod_id and fullname in self.modules: + # Submodule takes precedence over definition in surround package, for + # compatibility with runtime semantics in typical use cases. This + # could more precisely model runtime semantics by taking into account + # the line number beyond which the local definition should take + # precedence, but doesn't seem to be important in most use cases. + node = SymbolTableNode(GDEF, self.modules[fullname]) + else: + if id == as_id == "__all__" and module_id in self.export_map: + self.all_exports[:] = self.export_map[module_id] + node = module.names.get(id) + + missing_submodule = False + imported_id = as_id or id + + # Modules imported in a stub file without using 'from Y import X as X' will + # not get exported. + # When implicit re-exporting is disabled, we have the same behavior as stubs. + use_implicit_reexport = not self.is_stub_file and self.options.implicit_reexport + module_public = use_implicit_reexport or (as_id is not None and id == as_id) + + # If the module does not contain a symbol with the name 'id', + # try checking if it's a module instead. + if not node: + mod = self.modules.get(fullname) + if mod is not None: + kind = self.current_symbol_kind() + node = SymbolTableNode(kind, mod) + elif fullname in self.missing_modules: + missing_submodule = True + # If it is still not resolved, check for a module level __getattr__ + if module and not node and "__getattr__" in module.names: + # We store the fullname of the original definition so that we can + # detect whether two imported names refer to the same thing. + fullname = module_id + "." + id + gvar = self.create_getattr_var(module.names["__getattr__"], imported_id, fullname) + if gvar: + self.add_symbol( + imported_id, + gvar, + imp, + module_public=module_public, + module_hidden=not module_public, + ) + continue + + if node: + self.process_imported_symbol( + node, module_id, id, imported_id, fullname, module_public, context=imp + ) + if node.module_hidden: + self.report_missing_module_attribute( + module_id, + id, + imported_id, + module_public=module_public, + module_hidden=not module_public, + context=imp, + add_unknown_imported_symbol=False, + ) + elif module and not missing_submodule: + # Target module exists but the imported name is missing or hidden. + self.report_missing_module_attribute( + module_id, + id, + imported_id, + module_public=module_public, + module_hidden=not module_public, + context=imp, + ) + else: + # Import of a missing (sub)module. + self.add_unknown_imported_symbol( + imported_id, + imp, + target_name=fullname, + module_public=module_public, + module_hidden=not module_public, + ) + + def process_imported_symbol( + self, + node: SymbolTableNode, + module_id: str, + id: str, + imported_id: str, + fullname: str, + module_public: bool, + context: ImportBase, + ) -> None: + module_hidden = not module_public and ( + # `from package import submodule` should work regardless of whether package + # re-exports submodule, so we shouldn't hide it + not isinstance(node.node, MypyFile) + or fullname not in self.modules + # but given `from somewhere import random_unrelated_module` we should hide + # random_unrelated_module + or not fullname.startswith(self.cur_mod_id + ".") + ) + + if isinstance(node.node, PlaceholderNode): + if self.final_iteration: + self.report_missing_module_attribute( + module_id, + id, + imported_id, + module_public=module_public, + module_hidden=module_hidden, + context=context, + ) + return + else: + # This might become a type. + self.mark_incomplete( + imported_id, + node.node, + module_public=module_public, + module_hidden=module_hidden, + becomes_typeinfo=True, + ) + # NOTE: we take the original node even for final `Var`s. This is to support + # a common pattern when constants are re-exported (same applies to import *). + self.add_imported_symbol( + imported_id, node, context, module_public=module_public, module_hidden=module_hidden + ) + + def report_missing_module_attribute( + self, + import_id: str, + source_id: str, + imported_id: str, + module_public: bool, + module_hidden: bool, + context: Node, + add_unknown_imported_symbol: bool = True, + ) -> None: + # Missing attribute. + if self.is_incomplete_namespace(import_id): + # We don't know whether the name will be there, since the namespace + # is incomplete. Defer the current target. + self.mark_incomplete( + imported_id, context, module_public=module_public, module_hidden=module_hidden + ) + return + message = f'Module "{import_id}" has no attribute "{source_id}"' + # Suggest alternatives, if any match is found. + module = self.modules.get(import_id) + if module: + if source_id in module.names.keys() and not module.names[source_id].module_public: + message = ( + f'Module "{import_id}" does not explicitly export attribute "{source_id}"' + ) + else: + alternatives = set(module.names.keys()).difference({source_id}) + matches = best_matches(source_id, alternatives, n=3) + if matches: + suggestion = f"; maybe {pretty_seq(matches, 'or')}?" + message += f"{suggestion}" + self.fail(message, context, code=codes.ATTR_DEFINED) + if add_unknown_imported_symbol: + self.add_unknown_imported_symbol( + imported_id, + context, + target_name=None, + module_public=module_public, + module_hidden=not module_public, + ) + + if import_id == "typing": + # The user probably has a missing definition in a test fixture. Let's verify. + fullname = f"builtins.{source_id.lower()}" + if ( + self.lookup_fully_qualified_or_none(fullname) is None + and fullname in SUGGESTED_TEST_FIXTURES + ): + # Yes. Generate a helpful note. + self.msg.add_fixture_note(fullname, context) + else: + typing_extensions = self.modules.get("typing_extensions") + if typing_extensions and source_id in typing_extensions.names: + self.msg.note( + f"Use `from typing_extensions import {source_id}` instead", + context, + code=codes.ATTR_DEFINED, + ) + self.msg.note( + "See https://mypy.readthedocs.io/en/stable/runtime_troubles.html#using-new-additions-to-the-typing-module", + context, + code=codes.ATTR_DEFINED, + ) + + def process_import_over_existing_name( + self, + imported_id: str, + existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase, + ) -> bool: + if existing_symbol.node is module_symbol.node: + # We added this symbol on previous iteration. + return False + if existing_symbol.kind in (LDEF, GDEF, MDEF) and isinstance( + existing_symbol.node, (Var, FuncDef, TypeInfo, Decorator, TypeAlias) + ): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + if isinstance(rvalue.node, TypeAlias): + # Suppress bogus errors from the dummy assignment if rvalue is an alias. + # Otherwise mypy may complain that alias is invalid in runtime context. + rvalue.is_alias_rvalue = True + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: + def correct_relative_import(self, node: ImportFrom | ImportAll) -> str: + import_id, ok = correct_relative_import( + self.cur_mod_id, node.relative, node.id, self.cur_mod_node.is_package_init_file() + ) + if not ok: self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") + return import_id def visit_import_all(self, i: ImportAll) -> None: i_id = self.correct_relative_import(i) if i_id in self.modules: m = self.modules[i_id] + if self.is_incomplete_namespace(i_id): + # Any names could be missing from the current namespace if the target module + # namespace is incomplete. + self.mark_incomplete("*", i) for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_'): - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) + fullname = i_id + "." + name + self.set_future_import_flags(fullname) + if node is None: + continue + # if '__all__' exists, all nodes not included have had module_public set to + # False, and we can skip checking '_' because it's been explicitly included. + if node.module_public and (not name.startswith("_") or "__all__" in m.names): + if isinstance(node.node, MypyFile): + # Star import of submodule from a package, add it as a dependency. + self.imports.add(node.node.fullname) + # `from x import *` always reexports symbols + self.add_imported_symbol( + name, node, context=i, module_public=True, module_hidden=False + ) + else: # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. pass - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - # - # Statements + # Assignment # - def visit_block(self, b: Block) -> None: - if b.is_unreachable: + def visit_assignment_expr(self, s: AssignmentExpr) -> None: + s.value.accept(self) + if self.is_func_scope(): + if not self.check_valid_comprehension(s): + return + self.analyze_lvalue(s.target, escape_comprehensions=True, has_explicit_value=True) + + def check_valid_comprehension(self, s: AssignmentExpr) -> bool: + """Check that assignment expression is not nested within comprehension at class scope. + + class C: + [(j := i) for i in [1, 2, 3]] + is a syntax error that is not enforced by Python parser, but at later steps. + """ + for i, scope_type in enumerate(reversed(self.scope_stack)): + if scope_type != SCOPE_COMPREHENSION and i < len(self.locals) - 1: + if self.locals[-1 - i] is None: + self.fail( + "Assignment expression within a comprehension" + " cannot be used in a class body", + s, + code=codes.SYNTAX, + serious=True, + blocker=True, + ) + return False + break + return True + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.statement = s + + # Special case assignment like X = X. + if self.analyze_identity_global_assignment(s): return - self.block_depth[-1] += 1 - for s in b.body: - s.accept(self) - self.block_depth[-1] -= 1 - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) + tag = self.track_incomplete_refs() + + # Here we have a chicken and egg problem: at this stage we can't call + # can_be_type_alias(), because we have not enough information about rvalue. + # But we can't use a full visit because it may emit extra incomplete refs (namely + # when analysing any type applications there) thus preventing the further analysis. + # To break the tie, we first analyse rvalue partially, if it can be a type alias. + if self.can_possibly_be_type_form(s): + old_basic_type_applications = self.basic_type_applications + self.basic_type_applications = True + with self.allow_unbound_tvars_set(): + s.rvalue.accept(self) + self.basic_type_applications = old_basic_type_applications + elif self.can_possibly_be_typevarlike_declaration(s): + # Allow unbound tvars inside TypeVarLike defaults to be evaluated later + with self.allow_unbound_tvars_set(): + s.rvalue.accept(self) + else: + s.rvalue.accept(self) + + if self.found_incomplete_ref(tag) or self.should_wait_rhs(s.rvalue): + # Initializer couldn't be fully analyzed. Defer the current node and give up. + # Make sure that if we skip the definition of some local names, they can't be + # added later in this scope, since an earlier definition should take precedence. + for expr in names_modified_by_assignment(s): + self.mark_incomplete(expr.name, expr) + return + if self.can_possibly_be_type_form(s): + # Now re-visit those rvalues that were we skipped type applications above. + # This should be safe as generally semantic analyzer is idempotent. + with self.allow_unbound_tvars_set(): + s.rvalue.accept(self) + + # The r.h.s. is now ready to be classified, first check if it is a special form: + special_form = False + # * type alias + if self.check_and_set_up_type_alias(s): + s.is_alias_def = True + special_form = True + elif isinstance(s.rvalue, CallExpr): + # * type variable definition + if self.process_typevar_declaration(s): + special_form = True + elif self.process_paramspec_declaration(s): + special_form = True + elif self.process_typevartuple_declaration(s): + special_form = True + # * type constructors + elif self.analyze_namedtuple_assign(s): + special_form = True + elif self.analyze_typeddict_assign(s): + special_form = True + elif self.newtype_analyzer.process_newtype_declaration(s): + special_form = True + elif self.analyze_enum_assign(s): + special_form = True + + if special_form: + self.record_special_form_lvalue(s) + return + # Clear the alias flag if assignment turns out not a special form after all. It + # may be set to True while there were still placeholders due to forward refs. + s.is_alias_def = False + + # OK, this is a regular assignment, perform the necessary analysis steps. + s.is_final_def = self.unwrap_final(s) + self.analyze_lvalues(s) + self.check_final_implicit_def(s) + self.store_final_status(s) + self.check_classvar(s) + self.process_type_annotation(s) + self.apply_dynamic_class_hook(s) + if not s.type: + self.process_module_assignment(s.lvalues, s.rvalue, s) + self.process__all__(s) + self.process__deletable__(s) + self.process__slots__(s) + + def analyze_identity_global_assignment(self, s: AssignmentStmt) -> bool: + """Special case 'X = X' in global scope. + + This allows supporting some important use cases. + + Return true if special casing was applied. + """ + if not isinstance(s.rvalue, NameExpr) or len(s.lvalues) != 1: + # Not of form 'X = X' + return False + lvalue = s.lvalues[0] + if not isinstance(lvalue, NameExpr) or s.rvalue.name != lvalue.name: + # Not of form 'X = X' + return False + if self.type is not None or self.is_func_scope(): + # Not in global scope + return False + # It's an assignment like 'X = X' in the global scope. + name = lvalue.name + sym = self.lookup(name, s) + if sym is None: + if self.final_iteration: + # Fall back to normal assignment analysis. + return False + else: + self.defer() + return True + else: + if sym.node is None: + # Something special -- fall back to normal assignment analysis. + return False + if name not in self.globals: + # The name is from builtins. Add an alias to the current module. + self.add_symbol(name, sym.node, s) + if not isinstance(sym.node, PlaceholderNode): + for node in s.rvalue, lvalue: + node.node = sym.node + node.kind = GDEF + node.fullname = sym.node.fullname + return True + + def should_wait_rhs(self, rv: Expression) -> bool: + """Can we already classify this r.h.s. of an assignment or should we wait? + + This returns True if we don't have enough information to decide whether + an assignment is just a normal variable definition or a special form. + Always return False if this is a final iteration. This will typically cause + the lvalue to be classified as a variable plus emit an error. + """ + if self.final_iteration: + # No chance, nothing has changed. + return False + if isinstance(rv, NameExpr): + n = self.lookup(rv.name, rv) + if n and isinstance(n.node, PlaceholderNode) and not n.node.becomes_typeinfo: + return True + elif isinstance(rv, MemberExpr): + fname = get_member_expr_fullname(rv) + if fname: + n = self.lookup_qualified(fname, rv, suppress_errors=True) + if n and isinstance(n.node, PlaceholderNode) and not n.node.becomes_typeinfo: + return True + elif isinstance(rv, IndexExpr) and isinstance(rv.base, RefExpr): + return self.should_wait_rhs(rv.base) + elif isinstance(rv, CallExpr) and isinstance(rv.callee, RefExpr): + # This is only relevant for builtin SCC where things like 'TypeVar' + # may be not ready. + return self.should_wait_rhs(rv.callee) + return False - def visit_var_def(self, defn: VarDef) -> None: - for i in range(len(defn.items)): - defn.items[i].type = self.anal_type(defn.items[i].type) - - for v in defn.items: - if self.is_func_scope(): - defn.kind = LDEF - self.add_local(v, defn) - elif self.type: - v.info = self.type - v.is_initialized_in_class = defn.init is not None - self.type.names[v.name()] = SymbolTableNode(MDEF, v, - typ=v.type) - elif v.name not in self.globals: - defn.kind = GDEF - self.add_var(v, defn) - - if defn.init: - defn.init.accept(self) - - def anal_type(self, t: Type) -> Type: - if t: - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.stored_vars, self.fail) - return t.accept(a) + def can_be_type_alias(self, rv: Expression, allow_none: bool = False) -> bool: + """Is this a valid r.h.s. for an alias definition? + + Note: this function should be only called for expressions where self.should_wait_rhs() + returns False. + """ + if isinstance(rv, RefExpr) and self.is_type_ref(rv, bare=True): + return True + if isinstance(rv, IndexExpr) and self.is_type_ref(rv.base, bare=False): + return True + if self.is_none_alias(rv): + return True + if allow_none and isinstance(rv, NameExpr) and rv.fullname == "builtins.None": + return True + if isinstance(rv, OpExpr) and rv.op == "|": + if self.is_stub_file: + return True + if self.can_be_type_alias(rv.left, allow_none=True) and self.can_be_type_alias( + rv.right, allow_none=True + ): + return True + return False + + def can_possibly_be_type_form(self, s: AssignmentStmt) -> bool: + """Like can_be_type_alias(), but simpler and doesn't require fully analyzed rvalue. + + Instead, use lvalues/annotations structure to figure out whether this can potentially be + a type alias definition, NamedTuple, or TypedDict. Another difference from above function + is that we are only interested IndexExpr, CallExpr and OpExpr rvalues, since only those + can be potentially recursive (things like `A = A` are never valid). + """ + if len(s.lvalues) > 1: + return False + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.callee, RefExpr): + ref = s.rvalue.callee.fullname + return ref in TPDICT_NAMES or ref in TYPED_NAMEDTUPLE_NAMES + if not isinstance(s.lvalues[0], NameExpr): + return False + if s.unanalyzed_type is not None and not self.is_pep_613(s): + return False + if not isinstance(s.rvalue, (IndexExpr, OpExpr)): + return False + # Something that looks like Foo = Bar[Baz, ...] + return True + + def can_possibly_be_typevarlike_declaration(self, s: AssignmentStmt) -> bool: + """Check if r.h.s. can be a TypeVarLike declaration.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return False + if not isinstance(s.rvalue, CallExpr) or not isinstance(s.rvalue.callee, NameExpr): + return False + ref = s.rvalue.callee + ref.accept(self) + return ref.fullname in TYPE_VAR_LIKE_NAMES + + def is_type_ref(self, rv: Expression, bare: bool = False) -> bool: + """Does this expression refer to a type? + + This includes: + * Special forms, like Any or Union + * Classes (except subscripted enums) + * Other type aliases + * PlaceholderNodes with becomes_typeinfo=True (these can be not ready class + definitions, and not ready aliases). + + If bare is True, this is not a base of an index expression, so some special + forms are not valid (like a bare Union). + + Note: This method should be only used in context of a type alias definition. + This method can only return True for RefExprs, to check if C[int] is a valid + target for type alias call this method on expr.base (i.e. on C in C[int]). + See also can_be_type_alias(). + """ + if not isinstance(rv, RefExpr): + return False + if isinstance(rv.node, TypeVarLikeExpr): + self.fail(f'Type variable "{rv.fullname}" is invalid as target for type alias', rv) + return False + + if bare: + # These three are valid even if bare, for example + # A = Tuple is just equivalent to A = Tuple[Any, ...]. + valid_refs = {"typing.Any", "typing.Tuple", "typing.Callable"} else: - return None + valid_refs = type_constructors + + if isinstance(rv.node, TypeAlias) or rv.fullname in valid_refs: + return True + if isinstance(rv.node, TypeInfo): + if bare: + return True + # Assignment color = Color['RED'] defines a variable, not an alias. + return not rv.node.is_enum + if isinstance(rv.node, Var): + return rv.node.fullname in NEVER_NAMES + + if isinstance(rv, NameExpr): + n = self.lookup(rv.name, rv) + if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: + return True + elif isinstance(rv, MemberExpr): + fname = get_member_expr_fullname(rv) + if fname: + # The r.h.s. for variable definitions may not be a type reference but just + # an instance attribute, so suppress the errors. + n = self.lookup_qualified(fname, rv, suppress_errors=True) + if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: + return True + return False + + def is_none_alias(self, node: Expression) -> bool: + """Is this a r.h.s. for a None alias? + + We special case the assignments like Void = type(None), to allow using + Void in type annotations. + """ + if isinstance(node, CallExpr): + if ( + isinstance(node.callee, NameExpr) + and len(node.args) == 1 + and isinstance(node.args[0], NameExpr) + ): + call = self.lookup_qualified(node.callee.name, node.callee) + arg = self.lookup_qualified(node.args[0].name, node.args[0]) + if ( + call is not None + and call.node + and call.node.fullname == "builtins.type" + and arg is not None + and arg.node + and arg.node.fullname == "builtins.None" + ): + return True + return False + + def record_special_form_lvalue(self, s: AssignmentStmt) -> None: + """Record minimal necessary information about l.h.s. of a special form. + + This exists mostly for compatibility with the old semantic analyzer. + """ + lvalue = s.lvalues[0] + assert isinstance(lvalue, NameExpr) + lvalue.is_special_form = True + if self.current_symbol_kind() == GDEF: + lvalue.fullname = self.qualified_name(lvalue.name) + lvalue.kind = self.current_symbol_kind() + + def analyze_enum_assign(self, s: AssignmentStmt) -> bool: + """Check if s defines an Enum.""" + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, EnumCallExpr): + # Already analyzed enum -- nothing to do here. + return True + return self.enum_call_analyzer.process_enum_call(s, self.is_func_scope()) + + def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: + """Check if s defines a namedtuple.""" + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, NamedTupleExpr): + if s.rvalue.analyzed.info.tuple_type and not has_placeholder( + s.rvalue.analyzed.info.tuple_type + ): + return True # This is a valid and analyzed named tuple definition, nothing to do here. + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): + return False + lvalue = s.lvalues[0] + if isinstance(lvalue, MemberExpr): + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.callee, RefExpr): + fullname = s.rvalue.callee.fullname + if fullname == "collections.namedtuple" or fullname in TYPED_NAMEDTUPLE_NAMES: + self.fail("NamedTuple type as an attribute is not supported", lvalue) + return False + name = lvalue.name + namespace = self.qualified_name(name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( + s.rvalue, name, self.is_func_scope() + ) + if internal_name is None: + return False + if internal_name != name: + self.fail( + 'First argument to namedtuple() should be "{}", not "{}"'.format( + name, internal_name + ), + s.rvalue, + code=codes.NAME_MATCH, + ) + return True + # Yes, it's a valid namedtuple, but defer if it is not ready. + if not info: + self.mark_incomplete(name, lvalue, becomes_typeinfo=True) + else: + self.setup_type_vars(info.defn, tvar_defs) + self.setup_alias_type_vars(info.defn) + return True + + def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: + """Check if s defines a typed dict.""" + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, TypedDictExpr): + if s.rvalue.analyzed.info.typeddict_type and not has_placeholder( + s.rvalue.analyzed.info.typeddict_type + ): + # This is a valid and analyzed typed dict definition, nothing to do here. + return True + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): + return False + lvalue = s.lvalues[0] + name = lvalue.name + namespace = self.qualified_name(name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict( + s.rvalue, name, self.is_func_scope() + ) + if not is_typed_dict: + return False + if isinstance(lvalue, MemberExpr): + self.fail("TypedDict type as attribute is not supported", lvalue) + return False + # Yes, it's a valid typed dict, but defer if it is not ready. + if not info: + self.mark_incomplete(name, lvalue, becomes_typeinfo=True) + else: + defn = info.defn + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) + return True + + def analyze_lvalues(self, s: AssignmentStmt) -> None: + # We cannot use s.type, because analyze_simple_literal_type() will set it. + explicit = s.unanalyzed_type is not None + if self.is_final_type(s.unanalyzed_type): + # We need to exclude bare Final. + assert isinstance(s.unanalyzed_type, UnboundType) + if not s.unanalyzed_type.args: + explicit = False + + if s.rvalue: + if isinstance(s.rvalue, TempNode): + has_explicit_value = not s.rvalue.no_rhs + else: + has_explicit_value = True + else: + has_explicit_value = False - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: for lval in s.lvalues: - self.analyse_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - s.type = self.anal_type(s.type) - else: - s.type = self.infer_type_from_undefined(s.rvalue) - # For simple assignments, allow binding type aliases - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - res = analyse_node(self.lookup_qualified, s.rvalue, s) - if res: - # XXX Need to remove this later if reassigned - x = cast(NameExpr, s.lvalues[0]) - self.stored_vars[x.node] = res + self.analyze_lvalue( + lval, + explicit_type=explicit, + is_final=s.is_final_def, + has_explicit_value=has_explicit_value, + ) + + def apply_dynamic_class_hook(self, s: AssignmentStmt) -> None: + if not isinstance(s.rvalue, CallExpr): + return + fname = "" + call = s.rvalue + while True: + if isinstance(call.callee, RefExpr): + fname = call.callee.fullname + # check if method call + if not fname and isinstance(call.callee, MemberExpr): + callee_expr = call.callee.expr + if isinstance(callee_expr, RefExpr) and callee_expr.fullname: + method_name = call.callee.name + fname = callee_expr.fullname + "." + method_name + elif ( + isinstance(callee_expr, IndexExpr) + and isinstance(callee_expr.base, RefExpr) + and isinstance(callee_expr.analyzed, TypeApplication) + ): + method_name = call.callee.name + fname = callee_expr.base.fullname + "." + method_name + elif isinstance(callee_expr, CallExpr): + # check if chain call + call = callee_expr + continue + break + if not fname: + return + hook = self.plugin.get_dynamic_class_hook(fname) + if not hook: + return + for lval in s.lvalues: + if not isinstance(lval, NameExpr): + continue + hook(DynamicClassDefContext(call, lval.name, self)) + + def unwrap_final(self, s: AssignmentStmt) -> bool: + """Strip Final[...] if present in an assignment. + + This is done to invoke type inference during type checking phase for this + assignment. Also, Final[...] doesn't affect type in any way -- it is rather an + access qualifier for given `Var`. + + Also perform various consistency checks. + + Returns True if Final[...] was present. + """ + if not s.unanalyzed_type or not self.is_final_type(s.unanalyzed_type): + return False + assert isinstance(s.unanalyzed_type, UnboundType) + if len(s.unanalyzed_type.args) > 1: + self.fail("Final[...] takes at most one type argument", s.unanalyzed_type) + invalid_bare_final = False + if not s.unanalyzed_type.args: + s.type = None + if isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs: + invalid_bare_final = True + self.fail("Type in Final[...] can only be omitted if there is an initializer", s) + else: + s.type = s.unanalyzed_type.args[0] + + if s.type is not None and self.is_classvar(s.type): + self.fail("Variable should not be annotated with both ClassVar and Final", s) + return False + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], RefExpr): + self.fail("Invalid final declaration", s) + return False + lval = s.lvalues[0] + assert isinstance(lval, RefExpr) + + # Reset inferred status if it was set due to simple literal rvalue on previous iteration. + # TODO: this is a best-effort quick fix, we should avoid the need to manually sync this, + # see https://github.com/python/mypy/issues/6458. + if lval.is_new_def: + lval.is_inferred_def = s.type is None + + if self.loop_depth[-1] > 0: + self.fail("Cannot use Final inside a loop", s) + if self.type and self.type.is_protocol: + if self.is_class_scope(): + self.msg.protocol_members_cant_be_final(s) + if ( + isinstance(s.rvalue, TempNode) + and s.rvalue.no_rhs + and not self.is_stub_file + and not self.is_class_scope() + ): + if not invalid_bare_final: # Skip extra error messages. + self.msg.final_without_value(s) + return True + + def check_final_implicit_def(self, s: AssignmentStmt) -> None: + """Do basic checks for final declaration on self in __init__. + + Additional re-definition checks are performed by `analyze_lvalue`. + """ + if not s.is_final_def: + return + lval = s.lvalues[0] + assert isinstance(lval, RefExpr) + if isinstance(lval, MemberExpr): + if not self.is_self_member_ref(lval): + self.fail("Final can be only applied to a name or an attribute on self", s) + s.is_final_def = False + return + else: + assert self.function_stack + if self.function_stack[-1].name != "__init__": + self.fail("Can only declare a final attribute in class body or __init__", s) + s.is_final_def = False + return + + def store_final_status(self, s: AssignmentStmt) -> None: + """If this is a locally valid final declaration, set the corresponding flag on `Var`.""" + if s.is_final_def: + if len(s.lvalues) == 1 and isinstance(s.lvalues[0], RefExpr): + node = s.lvalues[0].node + if isinstance(node, Var): + node.is_final = True + if s.type: + node.final_value = constant_fold_expr(s.rvalue, self.cur_mod_id) + if self.is_class_scope() and ( + isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs + ): + node.final_unset_in_class = True + else: + for lval in self.flatten_lvalues(s.lvalues): + # Special case: we are working with an `Enum`: + # + # class MyEnum(Enum): + # key = 'some value' + # + # Here `key` is implicitly final. In runtime, code like + # + # MyEnum.key = 'modified' + # + # will fail with `AttributeError: Cannot reassign members.` + # That's why we need to replicate this. + if ( + isinstance(lval, NameExpr) + and isinstance(self.type, TypeInfo) + and self.type.is_enum + ): + cur_node = self.type.names.get(lval.name, None) + if ( + cur_node + and isinstance(cur_node.node, Var) + and not (isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs) + ): + # Double underscored members are writable on an `Enum`. + # (Except read-only `__members__` but that is handled in type checker) + cur_node.node.is_final = s.is_final_def = not is_dunder(cur_node.node.name) + + # Special case: deferred initialization of a final attribute in __init__. + # In this case we just pretend this is a valid final definition to suppress + # errors about assigning to final attribute. + if isinstance(lval, MemberExpr) and self.is_self_member_ref(lval): + assert self.type, "Self member outside a class" + cur_node = self.type.names.get(lval.name, None) + if cur_node and isinstance(cur_node.node, Var) and cur_node.node.is_final: + assert self.function_stack + top_function = self.function_stack[-1] + if ( + top_function.name == "__init__" + and cur_node.node.final_unset_in_class + and not cur_node.node.final_set_in_init + and not (isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs) + ): + cur_node.node.final_set_in_init = True + s.is_final_def = True + + def flatten_lvalues(self, lvalues: list[Expression]) -> list[Expression]: + res: list[Expression] = [] + for lv in lvalues: + if isinstance(lv, (TupleExpr, ListExpr)): + res.extend(self.flatten_lvalues(lv.items)) + else: + res.append(lv) + return res + + def process_type_annotation(self, s: AssignmentStmt) -> None: + """Analyze type annotation or infer simple literal type.""" + if s.type: + lvalue = s.lvalues[-1] + allow_tuple_literal = isinstance(lvalue, TupleExpr) + analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) + # Don't store not ready types (including placeholders). + if analyzed is None or has_placeholder(analyzed): + self.defer(s) + return + s.type = analyzed + if ( + self.type + and self.type.is_protocol + and isinstance(lvalue, NameExpr) + and isinstance(s.rvalue, TempNode) + and s.rvalue.no_rhs + ): + if isinstance(lvalue.node, Var): + lvalue.node.is_abstract_var = True + else: + if ( + self.type + and self.type.is_protocol + and self.is_annotated_protocol_member(s) + and not self.is_func_scope() + ): + self.fail("All protocol members must have explicitly declared types", s) + # Set the type if the rvalue is a simple literal (even if the above error occurred). + if len(s.lvalues) == 1 and isinstance(s.lvalues[0], RefExpr): + ref_expr = s.lvalues[0] + safe_literal_inference = True + if self.type and isinstance(ref_expr, NameExpr) and len(self.type.mro) > 1: + # Check if there is a definition in supertype. If yes, we can't safely + # decide here what to infer: int or Literal[42]. + safe_literal_inference = self.type.mro[1].get(ref_expr.name) is None + if safe_literal_inference and ref_expr.is_inferred_def: + s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) if s.type: # Store type into nodes. for lvalue in s.lvalues: self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyse_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. + def is_annotated_protocol_member(self, s: AssignmentStmt) -> bool: + """Check whether a protocol member is annotated. + + There are some exceptions that can be left unannotated, like ``__slots__``.""" + return any( + (isinstance(lv, NameExpr) and lv.name != "__slots__" and lv.is_inferred_def) + for lv in s.lvalues + ) + + def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Type | None: + """Return builtins.int if rvalue is an int literal, etc. + + If this is a 'Final' context, we return "Literal[...]" instead. """ + if self.function_stack: + # Skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None - if isinstance(lval, NameExpr): - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - v = cast(Var, lval.node) - assert v.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) + value = constant_fold_expr(rvalue, self.cur_mod_id) + if value is None or isinstance(value, complex): + return None + + if isinstance(value, bool): + type_name = "builtins.bool" + elif isinstance(value, int): + type_name = "builtins.int" + elif isinstance(value, str): + type_name = "builtins.str" + elif isinstance(value, float): + type_name = "builtins.float" + + typ = self.named_type_or_none(type_name) + if typ and is_final: + return typ.copy_modified(last_known_value=LiteralType(value=value, fallback=typ)) + return typ + + def analyze_alias( + self, + name: str, + rvalue: Expression, + allow_placeholder: bool = False, + declared_type_vars: TypeVarLikeList | None = None, + all_declared_type_params_names: list[str] | None = None, + python_3_12_type_alias: bool = False, + ) -> tuple[Type | None, list[TypeVarLikeType], set[str], list[str], bool]: + """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable). + + If yes, return the corresponding type, a list of + qualified type variable names for generic aliases, a set of names the alias depends on, + and a list of type variables if the alias is generic. + A schematic example for the dependencies: + A = int + B = str + analyze_alias(Dict[A, B])[2] == {'__main__.A', '__main__.B'} + """ + dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic()) + global_scope = not self.type and not self.function_stack + try: + typ = expr_to_unanalyzed_type( + rvalue, self.options, self.is_stub_file, lookup_qualified=self.lookup_qualified + ) + except TypeTranslationError: + self.fail( + "Invalid type alias: expression is not a valid type", rvalue, code=codes.VALID_TYPE + ) + return None, [], set(), [], False + + found_type_vars = self.find_type_var_likes(typ) + tvar_defs: list[TypeVarLikeType] = [] + namespace = self.qualified_name(name) + alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + tvar_defs = self.tvar_defs_from_tvars(alias_type_vars, typ) + + if python_3_12_type_alias: + with self.allow_unbound_tvars_set(): + rvalue.accept(self) + + analyzed, depends_on = analyze_type_alias( + typ, + self, + self.tvar_scope, + self.plugin, + self.options, + self.cur_mod_node, + self.is_typeshed_stub_file, + allow_placeholder=allow_placeholder, + in_dynamic_func=dynamic, + global_scope=global_scope, + allowed_alias_tvars=tvar_defs, + alias_type_params_names=all_declared_type_params_names, + python_3_12_type_alias=python_3_12_type_alias, + ) + + # There can be only one variadic variable at most, the error is reported elsewhere. + new_tvar_defs = [] + variadic = False + for td in tvar_defs: + if isinstance(td, TypeVarTupleType): + if variadic: + continue + variadic = True + new_tvar_defs.append(td) + + qualified_tvars = [node.fullname for _name, node in alias_type_vars] + empty_tuple_index = typ.empty_tuple_index if isinstance(typ, UnboundType) else False + return analyzed, new_tvar_defs, depends_on, qualified_tvars, empty_tuple_index + + def is_pep_613(self, s: AssignmentStmt) -> bool: + if s.unanalyzed_type is not None and isinstance(s.unanalyzed_type, UnboundType): + lookup = self.lookup_qualified(s.unanalyzed_type.name, s, suppress_errors=True) + if lookup and lookup.fullname in TYPE_ALIAS_NAMES: + return True + return False + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: + """Check if assignment creates a type alias and set it up as needed. + + Return True if it is a type alias (even if the target is not ready), + or False otherwise. + + Note: the resulting types for subscripted (including generic) aliases + are also stored in rvalue.analyzed. + """ + if s.invalid_recursive_alias: + return True + lvalue = s.lvalues[0] + if len(s.lvalues) > 1 or not isinstance(lvalue, NameExpr): + # First rule: Only simple assignments like Alias = ... create aliases. + return False + + pep_613 = self.is_pep_613(s) + if not pep_613 and s.unanalyzed_type is not None: + # Second rule: Explicit type (cls: Type[A] = A) always creates variable, not alias. + # unless using PEP 613 `cls: TypeAlias = A` + return False + + # It can be `A = TypeAliasType('A', ...)` call, in this case, + # we just take the second argument and analyze it: + type_params: TypeVarLikeList | None + all_type_params_names: list[str] | None + if self.check_type_alias_type_call(s.rvalue, name=lvalue.name): + rvalue = s.rvalue.args[1] + pep_695 = True + type_params, all_type_params_names = self.analyze_type_alias_type_params(s.rvalue) + else: + rvalue = s.rvalue + pep_695 = False + type_params = None + all_type_params_names = None + + if isinstance(rvalue, CallExpr) and rvalue.analyzed: + return False + + existing = self.current_symbol_table().get(lvalue.name) + # Third rule: type aliases can't be re-defined. For example: + # A: Type[float] = int + # A = float # OK, but this doesn't define an alias + # B = int + # B = float # Error! + # Don't create an alias in these cases: + if existing and ( + isinstance(existing.node, Var) # existing variable + or (isinstance(existing.node, TypeAlias) and not s.is_alias_def) # existing alias + or (isinstance(existing.node, PlaceholderNode) and existing.node.node.line < s.line) + ): # previous incomplete definition + # TODO: find a more robust way to track the order of definitions. + # Note: if is_alias_def=True, this is just a node from previous iteration. + if isinstance(existing.node, TypeAlias) and not s.is_alias_def: + self.fail( + 'Cannot assign multiple types to name "{}"' + ' without an explicit "Type[...]" annotation'.format(lvalue.name), + lvalue, + ) + return False + + non_global_scope = self.type or self.is_func_scope() + if not pep_613 and not pep_695 and isinstance(rvalue, RefExpr) and non_global_scope: + # Fourth rule (special case): Non-subscripted right hand side creates a variable + # at class and function scopes. For example: + # + # class Model: + # ... + # class C: + # model = Model # this is automatically a variable with type 'Type[Model]' + # + # without this rule, this typical use case will require a lot of explicit + # annotations (see the second rule). + return False + if not pep_613 and not pep_695 and not self.can_be_type_alias(rvalue): + return False + + if existing and not isinstance(existing.node, (PlaceholderNode, TypeAlias)): + # Cannot redefine existing node as type alias. + return False + + res: Type | None = None + if self.is_none_alias(rvalue): + res = NoneType() + alias_tvars: list[TypeVarLikeType] = [] + depends_on: set[str] = set() + qualified_tvars: list[str] = [] + empty_tuple_index = False + else: + tag = self.track_incomplete_refs() + res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( + lvalue.name, + rvalue, + allow_placeholder=True, + declared_type_vars=type_params, + all_declared_type_params_names=all_type_params_names, + ) + if not res: + return False + if not self.is_func_scope(): + # Only marking incomplete for top-level placeholders makes recursive aliases like + # `A = Sequence[str | A]` valid here, similar to how we treat base classes in class + # definitions, allowing `class str(Sequence[str]): ...` + incomplete_target = isinstance(res, ProperType) and isinstance( + res, PlaceholderType + ) else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) + incomplete_target = has_placeholder(res) + if self.found_incomplete_ref(tag) or incomplete_target: + # Since we have got here, we know this must be a type alias (incomplete refs + # may appear in nested positions), therefore use becomes_typeinfo=True. + self.mark_incomplete(lvalue.name, rvalue, becomes_typeinfo=True) + return True + self.add_type_alias_deps(depends_on) + # In addition to the aliases used, we add deps on unbound + # type variables, since they are erased from target type. + self.add_type_alias_deps(qualified_tvars) + # The above are only direct deps on other aliases. + # For subscripted aliases, type deps from expansion are added in deps.py + # (because the type is stored). + check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s) + # When this type alias gets "inlined", the Any is not explicit anymore, + # so we need to replace it with non-explicit Anys. + res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + # Only show error message once, when the type is fully analyzed. + if not has_placeholder(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) + # Note: with the new (lazy) type alias representation we only need to set no_args to True + # if the expected number of arguments is non-zero, so that aliases like `A = List` work + # but not aliases like `A = TypeAliasType("A", List)` as these need explicit type params. + # However, eagerly expanding aliases like Text = str is a nice performance optimization. + no_args = ( + isinstance(res, ProperType) + and isinstance(res, Instance) + and not res.args + and not empty_tuple_index + and not pep_695 + ) + if isinstance(res, ProperType) and isinstance(res, Instance): + if not validate_instance(res, self.fail, empty_tuple_index): + fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) + # Aliases defined within functions can't be accessed outside + # the function, since the symbol table will no longer + # exist. Work around by expanding them eagerly when used. + eager = self.is_func_scope() + alias_node = TypeAlias( + res, + self.qualified_name(lvalue.name), + s.line, + s.column, + alias_tvars=alias_tvars, + no_args=no_args, + eager=eager, + python_3_12_type_alias=pep_695, + ) + if isinstance(s.rvalue, (IndexExpr, CallExpr, OpExpr)) and ( + not isinstance(rvalue, OpExpr) + or (self.options.python_version >= (3, 10) or self.is_stub_file) + ): + # Note: CallExpr is for "void = type(None)" and OpExpr is for "X | Y" union syntax. + if not isinstance(s.rvalue.analyzed, TypeAliasExpr): + # Any existing node will be updated in-place below. + s.rvalue.analyzed = TypeAliasExpr(alias_node) + s.rvalue.analyzed.line = s.line + # we use the column from resulting target, to get better location for errors + s.rvalue.analyzed.column = res.column + elif isinstance(s.rvalue, RefExpr): + s.rvalue.is_alias_rvalue = True + + if existing: + # An alias gets updated. + updated = False + if isinstance(existing.node, TypeAlias): + if existing.node.target != res: + # Copy expansion to the existing alias, this matches how we update base classes + # for a TypeInfo _in place_ if there are nested placeholders. + existing.node.target = res + existing.node.alias_tvars = alias_tvars + existing.node.no_args = no_args + updated = True + # Invalidate recursive status cache in case it was previously set. + existing.node._is_recursive = None + else: + # Otherwise just replace existing placeholder with type alias. + existing.node = alias_node + updated = True + if updated: + if self.final_iteration: + self.cannot_resolve_name(lvalue.name, "name", s) + return True + else: + # We need to defer so that this change can get propagated to base classes. + self.defer(s, force_progress=True) + else: + self.add_symbol(lvalue.name, alias_node, s) + if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias): + alias_node.normalized = rvalue.node.normalized + current_node = existing.node if existing else alias_node + assert isinstance(current_node, TypeAlias) + self.disable_invalid_recursive_aliases(s, current_node, s.rvalue) + if self.is_class_scope(): + assert self.type is not None + if self.type.is_protocol: + self.fail("Type aliases are prohibited in protocol bodies", s) + if not lvalue.name[0].isupper(): + self.note("Use variable annotation syntax to define protocol members", s) + return True + + def check_type_alias_type_call(self, rvalue: Expression, *, name: str) -> TypeGuard[CallExpr]: + if not isinstance(rvalue, CallExpr): + return False + + names = ["typing_extensions.TypeAliasType"] + if self.options.python_version >= (3, 12): + names.append("typing.TypeAliasType") + if not refers_to_fullname(rvalue.callee, tuple(names)): + return False + + return self.check_typevarlike_name(rvalue, name, rvalue) + + def analyze_type_alias_type_params( + self, rvalue: CallExpr + ) -> tuple[TypeVarLikeList, list[str]]: + """Analyze type_params of TypeAliasType. + + Returns declared unbound type variable expressions and a list of all decalred type + variable names for error reporting. + """ + if "type_params" in rvalue.arg_names: + type_params_arg = rvalue.args[rvalue.arg_names.index("type_params")] + if not isinstance(type_params_arg, TupleExpr): + self.fail( + "Tuple literal expected as the type_params argument to TypeAliasType", + type_params_arg, + ) + return [], [] + type_params = type_params_arg.items + else: + return [], [] + + declared_tvars: TypeVarLikeList = [] + all_declared_tvar_names: list[str] = [] # includes bound type variables + have_type_var_tuple = False + for tp_expr in type_params: + if isinstance(tp_expr, StarExpr): + tp_expr.valid = False + self.analyze_type_expr(tp_expr) + try: + base = self.expr_to_unanalyzed_type(tp_expr) + except TypeTranslationError: + continue + if not isinstance(base, UnboundType): + continue + + tag = self.track_incomplete_refs() + tvar = self.analyze_unbound_tvar_impl(base, is_typealias_param=True) + if tvar: + if isinstance(tvar[1], TypeVarTupleExpr): + if have_type_var_tuple: + self.fail( + "Can only use one TypeVarTuple in type_params argument to TypeAliasType", + base, + code=codes.TYPE_VAR, + ) + have_type_var_tuple = True + continue + have_type_var_tuple = True + elif not self.found_incomplete_ref(tag): + sym = self.lookup_qualified(base.name, base) + if sym and isinstance(sym.node, TypeVarLikeExpr): + all_declared_tvar_names.append(sym.node.name) # Error will be reported later + else: + self.fail( + "Free type variable expected in type_params argument to TypeAliasType", + base, + code=codes.TYPE_VAR, + ) + if sym and sym.fullname in ("typing.Unpack", "typing_extensions.Unpack"): + self.note( + "Don't Unpack type variables in type_params", base, code=codes.TYPE_VAR + ) + continue + if tvar in declared_tvars: + self.fail( + f'Duplicate type variable "{tvar[0]}" in type_params argument to TypeAliasType', + base, + code=codes.TYPE_VAR, + ) + continue + if tvar: + all_declared_tvar_names.append(tvar[0]) + declared_tvars.append(tvar) + return declared_tvars, all_declared_tvar_names + + def disable_invalid_recursive_aliases( + self, s: AssignmentStmt | TypeAliasStmt, current_node: TypeAlias, ctx: Context + ) -> None: + """Prohibit and fix recursive type aliases that are invalid/unsupported.""" + messages = [] + if is_invalid_recursive_alias({current_node}, current_node.target): + target = ( + "tuple" if isinstance(get_proper_type(current_node.target), TupleType) else "union" + ) + messages.append(f"Invalid recursive alias: a {target} item of itself") + if detect_diverging_alias( + current_node, current_node.target, self.lookup_qualified, self.tvar_scope + ): + messages.append("Invalid recursive alias: type variable nesting on right hand side") + if messages: + current_node.target = AnyType(TypeOfAny.from_error) + s.invalid_recursive_alias = True + for msg in messages: + self.fail(msg, ctx) + + def analyze_lvalue( + self, + lval: Lvalue, + nested: bool = False, + explicit_type: bool = False, + is_final: bool = False, + escape_comprehensions: bool = False, + has_explicit_value: bool = False, + is_index_var: bool = False, + ) -> None: + """Analyze an lvalue or assignment target. + + Args: + lval: The target lvalue + nested: If true, the lvalue is within a tuple or list lvalue expression + explicit_type: Assignment has type annotation + escape_comprehensions: If we are inside a comprehension, set the variable + in the enclosing scope instead. This implements + https://www.python.org/dev/peps/pep-0572/#scope-of-the-target + is_index_var: If lval is the index variable in a for loop + """ + if escape_comprehensions: + assert isinstance(lval, NameExpr), "assignment expression target must be NameExpr" + if isinstance(lval, NameExpr): + self.analyze_name_lvalue( + lval, + explicit_type, + is_final, + escape_comprehensions, + has_explicit_value=has_explicit_value, + is_index_var=is_index_var, + ) elif isinstance(lval, MemberExpr): - if not add_global: - self.analyse_member_lvalue(lval) + self.analyze_member_lvalue(lval, explicit_type, is_final, has_explicit_value) if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) + self.fail("Type cannot be declared in assignment to non-self attribute", lval) elif isinstance(lval, IndexExpr): if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif isinstance(lval, ParenExpr): - self.analyse_lvalue(lval.expr, nested, add_global, explicit_type) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = (Any(lval)).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyse_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) + self.fail("Unexpected type declaration", lval) + lval.accept(self) + elif isinstance(lval, TupleExpr): + self.analyze_tuple_or_list_lvalue(lval, explicit_type) elif isinstance(lval, StarExpr): if nested: - self.analyse_lvalue(lval.expr, nested, add_global, explicit_type) + self.analyze_lvalue(lval.expr, nested, explicit_type) else: - self.fail('Starred assignment target must be in a list or tuple', lval) + self.fail("Starred assignment target must be in a list or tuple", lval) else: - self.fail('Invalid assignment target', lval) + self.fail("Invalid assignment target", lval) + + def analyze_name_lvalue( + self, + lvalue: NameExpr, + explicit_type: bool, + is_final: bool, + escape_comprehensions: bool, + has_explicit_value: bool, + is_index_var: bool, + ) -> None: + """Analyze an lvalue that targets a name expression. + + Arguments are similar to "analyze_lvalue". + """ + if lvalue.node: + # This has been bound already in a previous iteration. + return - def analyse_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items + name = lvalue.name + if self.is_alias_for_final_name(name): + if is_final: + self.fail("Cannot redefine an existing name as final", lvalue) + else: + self.msg.cant_assign_to_final(name, self.type is not None, lvalue) + + kind = self.current_symbol_kind() + names = self.current_symbol_table(escape_comprehensions=escape_comprehensions) + existing = names.get(name) + + outer = self.is_global_or_nonlocal(name) + if ( + kind == MDEF + and isinstance(self.type, TypeInfo) + and self.type.is_enum + and not name.startswith("__") + ): + # Special case: we need to be sure that `Enum` keys are unique. + if existing is not None and not isinstance(existing.node, PlaceholderNode): + self.fail( + 'Attempted to reuse member name "{}" in Enum definition "{}"'.format( + name, self.type.name + ), + lvalue, + ) + + if explicit_type and has_explicit_value: + self.fail("Enum members must be left unannotated", lvalue) + self.note( + "See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members", + lvalue, + ) + + if (not existing or isinstance(existing.node, PlaceholderNode)) and not outer: + # Define new variable. + var = self.make_name_lvalue_var( + lvalue, kind, not explicit_type, has_explicit_value, is_index_var + ) + added = self.add_symbol(name, var, lvalue, escape_comprehensions=escape_comprehensions) + # Only bind expression if we successfully added name to symbol table. + if added: + lvalue.is_new_def = True + lvalue.is_inferred_def = True + lvalue.kind = kind + lvalue.node = var + if kind == GDEF: + lvalue.fullname = var._fullname + else: + lvalue.fullname = lvalue.name + if self.is_func_scope(): + if unmangle(name) == "_": + # Special case for assignment to local named '_': always infer 'Any'. + typ = AnyType(TypeOfAny.special_form) + self.store_declared_types(lvalue, typ) + if is_final and self.is_final_redefinition(kind, name): + self.fail("Cannot redefine an existing name as final", lvalue) + else: + self.make_name_lvalue_point_to_existing_def(lvalue, explicit_type, is_final) + + def is_final_redefinition(self, kind: int, name: str) -> bool: + if kind == GDEF: + return self.is_mangled_global(name) and not self.is_initial_mangled_global(name) + elif kind == MDEF and self.type: + return unmangle(name) + "'" in self.type.names + return False - def strip_parens(node: Node) -> Node: - if isinstance(node, ParenExpr): - return strip_parens(node.expr) + def is_alias_for_final_name(self, name: str) -> bool: + if self.is_func_scope(): + if not name.endswith("'"): + # Not a mangled name -- can't be an alias + return False + name = unmangle(name) + assert self.locals[-1] is not None, "No locals at function scope" + existing = self.locals[-1].get(name) + return existing is not None and is_final_node(existing.node) + elif self.type is not None: + orig_name = unmangle(name) + "'" + if name == orig_name: + return False + existing = self.type.names.get(orig_name) + return existing is not None and is_final_node(existing.node) + else: + orig_name = unmangle(name) + "'" + if name == orig_name: + return False + existing = self.globals.get(orig_name) + return existing is not None and is_final_node(existing.node) + + def make_name_lvalue_var( + self, + lvalue: NameExpr, + kind: int, + inferred: bool, + has_explicit_value: bool, + is_index_var: bool, + ) -> Var: + """Return a Var node for an lvalue that is a name expression.""" + name = lvalue.name + v = Var(name) + v.set_line(lvalue) + v.is_inferred = inferred + if kind == MDEF: + assert self.type is not None + v.info = self.type + v.is_initialized_in_class = True + v.allow_incompatible_override = name in ALLOW_INCOMPATIBLE_OVERRIDE + if kind != LDEF: + v._fullname = self.qualified_name(name) + else: + # fullanme should never stay None + v._fullname = name + v.is_ready = False # Type not inferred yet + v.has_explicit_value = has_explicit_value + v.is_index_var = is_index_var + return v + + def make_name_lvalue_point_to_existing_def( + self, lval: NameExpr, explicit_type: bool, is_final: bool + ) -> None: + """Update an lvalue to point to existing definition in the same scope. + + Arguments are similar to "analyze_lvalue". + + Assume that an existing name exists. + """ + if is_final: + # Redefining an existing name with final is always an error. + self.fail("Cannot redefine an existing name as final", lval) + original_def = self.lookup(lval.name, lval, suppress_errors=True) + if original_def is None and self.type and not self.is_func_scope(): + # Workaround to allow "x, x = ..." in class body. + original_def = self.type.get(lval.name) + if explicit_type: + # Don't re-bind if there is a type annotation. + self.name_already_defined(lval.name, lval, original_def) + else: + # Bind to an existing name. + if original_def: + self.bind_name_expr(lval, original_def) else: - return node + self.name_not_defined(lval.name, lval) + self.check_lvalue_validity(lval.node, lval) - star_exprs = [cast(StarExpr, strip_parens(item)) for item in items - if isinstance(strip_parens(item), StarExpr)] + def analyze_tuple_or_list_lvalue(self, lval: TupleExpr, explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [item for item in items if isinstance(item, StarExpr)] if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) + self.fail("Two starred expressions in assignment", lval) else: if len(star_exprs) == 1: star_exprs[0].valid = True for i in items: - self.analyse_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyse_member_lvalue(self, lval: MemberExpr) -> None: + self.analyze_lvalue( + lval=i, + nested=True, + explicit_type=explicit_type, + # Lists and tuples always have explicit values defined: + # `a, b, c = value` + has_explicit_value=True, + ) + + def analyze_member_lvalue( + self, lval: MemberExpr, explicit_type: bool, is_final: bool, has_explicit_value: bool + ) -> None: + """Analyze lvalue that is a member expression. + + Arguments: + lval: The target lvalue + explicit_type: Assignment has type annotation + is_final: Is the target final + """ + if lval.node: + # This has been bound already in a previous iteration. + return lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) + if self.is_self_member_ref(lval): + assert self.type, "Self member outside a class" + cur_node = self.type.names.get(lval.name) + node = self.type.get(lval.name) + if cur_node and is_final: + # Overrides will be checked in type checker. + self.fail("Cannot redefine an existing name as final", lval) + # On first encounter with this definition, if this attribute was defined before + # with an inferred type and it's marked with an explicit type now, give an error. + if ( + not lval.node + and cur_node + and isinstance(cur_node.node, Var) + and cur_node.node.is_inferred + and explicit_type + ): + self.attribute_already_defined(lval.name, lval, cur_node) + if self.type.is_protocol and has_explicit_value and cur_node is not None: + # Make this variable non-abstract, it would be safer to do this only if we + # are inside __init__, but we do this always to preserve historical behaviour. + if isinstance(cur_node.node, Var): + cur_node.node.is_abstract_var = False + if ( + # If the attribute of self is not defined, create a new Var, ... + node is None + # ... or if it is defined as abstract in a *superclass*. + or (cur_node is None and isinstance(node.node, Var) and node.node.is_abstract_var) + # ... also an explicit declaration on self also creates a new Var. + # Note that `explicit_type` might have been erased for bare `Final`, + # so we also check if `is_final` is passed. + or (cur_node is None and (explicit_type or is_final)) + ): + if self.type.is_protocol and node is None: + self.fail("Protocol members cannot be defined via assignment to self", lval) + else: + # Implicit attribute definition in __init__. + lval.is_new_def = True + lval.is_inferred_def = True + v = Var(lval.name) + v.set_line(lval) + v._fullname = self.qualified_name(lval.name) + v.info = self.type + v.is_ready = False + v.explicit_self_type = explicit_type or is_final + lval.def_var = v + lval.node = v + # TODO: should we also set lval.kind = MDEF? + self.type.names[lval.name] = SymbolTableNode(MDEF, v, implicit=True) self.check_lvalue_validity(lval.node, lval) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: """Does memberexpr to refer to an attribute of self?""" if not isinstance(memberexpr.expr, NameExpr): return False - node = (cast(NameExpr, memberexpr.expr)).node - return isinstance(node, Var) and (cast(Var, node)).is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (FuncDef, TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def infer_type_from_undefined(self, rvalue: Node) -> Type: - if isinstance(rvalue, CallExpr): - if isinstance(rvalue.analyzed, UndefinedExpr): - undef = cast(UndefinedExpr, rvalue.analyzed) - return undef.type - return None + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) + def check_lvalue_validity(self, node: Expression | SymbolNode | None, ctx: Context) -> None: + if isinstance(node, TypeVarExpr): + self.fail("Invalid assignment target", ctx) + elif isinstance(node, TypeInfo): + self.fail(message_registry.CANNOT_ASSIGN_TO_TYPE, ctx) + + def store_declared_types(self, lvalue: Lvalue, typ: Type) -> None: if isinstance(lvalue, RefExpr): - lvalue.is_def = False + lvalue.is_inferred_def = False if isinstance(lvalue.node, Var): - var = cast(Var, lvalue.node) + var = lvalue.node var.type = typ var.is_ready = True + typ = get_proper_type(typ) + if ( + var.is_final + and isinstance(typ, Instance) + and typ.last_known_value + and (not self.type or not self.type.is_enum) + ): + var.final_value = typ.last_known_value.value # If node is not a variable, we'll catch it elsewhere. elif isinstance(lvalue, TupleExpr): + typ = get_proper_type(typ) if isinstance(typ, TupleType): if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) + self.fail("Incompatible number of tuple items", lvalue) return for item, itemtype in zip(lvalue.items, typ.items): self.store_declared_types(item, itemtype) else: - self.fail('Tuple type expected for multiple variables', - lvalue) + self.fail("Tuple type expected for multiple variables", lvalue) elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - elif isinstance(lvalue, ParenExpr): + # Historical behavior for the old parser self.store_declared_types(lvalue.expr, typ) else: - raise RuntimeError('Internal error (%s)' % type(lvalue)) + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> bool: + """Check if s declares a TypeVar; it yes, store it in symbol table. + + Return True if this looks like a type variable declaration (but maybe + with errors), otherwise return False. + """ + call = self.get_typevarlike_declaration(s, ("typing.TypeVar", "typing_extensions.TypeVar")) + if not call: + return False + + name = self.extract_typevarlike_name(s, call) + if name is None: + return False + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_value_types(call.args[1 : 1 + n_values]) + + res = self.process_typevar_parameters( + call.args[1 + n_values :], + call.arg_names[1 + n_values :], + call.arg_kinds[1 + n_values :], + n_values, + s, + ) + if res is None: + return False + variance, upper_bound, default = res + + existing = self.current_symbol_table().get(name) + if existing and not ( + isinstance(existing.node, PlaceholderNode) + or + # Also give error for another type variable with the same name. + (isinstance(existing.node, TypeVarExpr) and existing.node is call.analyzed) + ): + self.fail(f'Cannot redefine "{name}" as a type variable', s) + return False + + if self.options.disallow_any_unimported: + for idx, constraint in enumerate(values, start=1): + if has_any_from_unimported_type(constraint): + prefix = f"Constraint {idx}" + self.msg.unimported_type_becomes_any(prefix, constraint, s) + + if has_any_from_unimported_type(upper_bound): + prefix = "Upper bound of type variable" + self.msg.unimported_type_becomes_any(prefix, upper_bound, s) + + for t in values + [upper_bound, default]: + check_for_explicit_any( + t, self.options, self.is_typeshed_stub_file, self.msg, context=s + ) + + # mypyc suppresses making copies of a function to check each + # possible type, so set the upper bound to Any to prevent that + # from causing errors. + if values and self.options.mypyc: + upper_bound = AnyType(TypeOfAny.implementation_artifact) + + # Yes, it's a valid type variable definition! Add it to the symbol table. + if not call.analyzed: + type_var = TypeVarExpr( + name, self.qualified_name(name), values, upper_bound, default, variance + ) + type_var.line = call.line + call.analyzed = type_var + updated = True + else: + assert isinstance(call.analyzed, TypeVarExpr) + updated = ( + values != call.analyzed.values + or upper_bound != call.analyzed.upper_bound + or default != call.analyzed.default + ) + call.analyzed.upper_bound = upper_bound + call.analyzed.values = values + call.analyzed.default = default + if any(has_placeholder(v) for v in values): + self.process_placeholder(None, "TypeVar values", s, force_progress=updated) + elif has_placeholder(upper_bound): + self.process_placeholder(None, "TypeVar upper bound", s, force_progress=updated) + elif has_placeholder(default): + self.process_placeholder(None, "TypeVar default", s, force_progress=updated) + + self.add_symbol(name, call.analyzed, s) + return True + + def check_typevar_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if isinstance(typ, AnyType) and typ.is_from_error: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format("TypeVar", "default"), context + ) + return default + + def check_paramspec_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if isinstance(typ, Parameters): + for i, arg_type in enumerate(typ.arg_types): + arg_ptype = get_proper_type(arg_type) + if isinstance(arg_ptype, AnyType) and arg_ptype.is_from_error: + self.fail(f"Argument {i} of ParamSpec default must be a type", context) + elif ( + isinstance(typ, AnyType) + and typ.is_from_error + or not isinstance(typ, (AnyType, UnboundType)) + ): + self.fail( + "The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec", + context, + ) + default = AnyType(TypeOfAny.from_error) + return default + + def check_typevartuple_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if not isinstance(typ, UnpackType): + self.fail("The default argument to TypeVarTuple must be an Unpacked tuple", context) + default = AnyType(TypeOfAny.from_error) + return default + + def check_typevarlike_name(self, call: CallExpr, name: str, context: Context) -> bool: + """Checks that the name of a TypeVar or ParamSpec matches its variable.""" + name = unmangle(name) + assert isinstance(call.callee, RefExpr) + typevarlike_type = ( + call.callee.name if isinstance(call.callee, NameExpr) else call.callee.fullname + ) + if len(call.args) < 1: + self.fail(f"Too few arguments for {typevarlike_type}()", context) + return False + if not isinstance(call.args[0], StrExpr) or call.arg_kinds[0] != ARG_POS: + self.fail(f"{typevarlike_type}() expects a string literal as first argument", context) + return False + elif call.args[0].value != name: + msg = 'String argument 1 "{}" to {}(...) does not match variable name "{}"' + self.fail(msg.format(call.args[0].value, typevarlike_type, name), context) + return False + return True - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a typevar; it yes, store it in symbol table.""" + def get_typevarlike_declaration( + self, s: AssignmentStmt, typevarlike_types: tuple[str, ...] + ) -> CallExpr | None: + """Returns the call expression if `s` is a declaration of `typevarlike_type` + (TypeVar or ParamSpec), or None otherwise. + """ if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return + return None if not isinstance(s.rvalue, CallExpr): - return - call = cast(CallExpr, s.rvalue) - if not isinstance(call.callee, RefExpr): - return - callee = cast(RefExpr, call.callee) - if callee.fullname != 'typing.typevar': - return - # TODO Share code with check_argument_count in checkexpr.py? - if len(call.args) < 1: - self.fail("Too few arguments for typevar()", s) - return - if len(call.args) > 2: - self.fail("Too many arguments for typevar()", s) - return - if call.arg_kinds not in ([ARG_POS], [ARG_POS, ARG_NAMED]): - self.fail("Unexpected arguments to typevar()", s) - return - if not isinstance(call.args[0], StrExpr): - self.fail("typevar() expects a string literal argument", s) - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected typevar() argument value", s) - return - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - if len(call.args) == 2: - # Analyze values=(...) argument. - if call.arg_names[1] != 'values': - self.fail("Unexpected keyword argument '{}' to typevar()". - format(call.arg_names[1]), s) - return - if isinstance(call.args[1], ParenExpr): - expr = cast(ParenExpr, call.args[1]).expr - if isinstance(expr, TupleExpr): - values = self.analyze_types(expr.items) + return None + call = s.rvalue + callee = call.callee + if not isinstance(callee, RefExpr): + return None + if callee.fullname not in typevarlike_types: + return None + return call + + def process_typevar_parameters( + self, + args: list[Expression], + names: list[str | None], + kinds: list[ArgKind], + num_values: int, + context: Context, + ) -> tuple[int, Type, Type] | None: + has_values = num_values > 0 + covariant = False + contravariant = False + upper_bound: Type = self.object_type() + default: Type = AnyType(TypeOfAny.from_omitted_generics) + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind.is_named(): + self.fail(message_registry.TYPEVAR_UNEXPECTED_ARGUMENT, context) + return None + if param_name == "covariant": + if isinstance(param_value, NameExpr) and param_value.name in ("True", "False"): + covariant = param_value.name == "True" else: - self.fail('The values argument must be a tuple literal', s) - return + self.fail(message_registry.TYPEVAR_VARIANCE_DEF.format("covariant"), context) + return None + elif param_name == "contravariant": + if isinstance(param_value, NameExpr) and param_value.name in ("True", "False"): + contravariant = param_value.name == "True" + else: + self.fail( + message_registry.TYPEVAR_VARIANCE_DEF.format("contravariant"), context + ) + return None + elif param_name == "bound": + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + tv_arg = self.get_typevarlike_argument("TypeVar", param_name, param_value, context) + if tv_arg is None: + return None + upper_bound = tv_arg + elif param_name == "default": + tv_arg = self.get_typevarlike_argument( + "TypeVar", param_name, param_value, context, allow_unbound_tvars=True + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + elif param_name == "values": + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail('TypeVar "values" argument not supported', context) + self.fail( + "Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", context + ) + return None else: - self.fail('The values argument must be in parentheses (...)', - s) - return - else: - values = [] - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - typevar = TypeVarExpr(name, node.fullname, values) - typevar.line = call.line - call.analyzed = typevar - node.node = typevar - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - if not isinstance(s.rvalue, CallExpr): - return - call = cast(CallExpr, s.rvalue) - named_tuple = self.check_namedtuple(call) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple + self.fail( + f'{message_registry.TYPEVAR_UNEXPECTED_ARGUMENT}: "{param_name}"', context + ) + return None - def check_namedtuple(self, call: CallExpr) -> TypeInfo: - """Check if a call defines a namedtuple. + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif num_values == 1: + self.fail(message_registry.TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES, context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return variance, upper_bound, default + + def get_typevarlike_argument( + self, + typevarlike_name: str, + param_name: str, + param_value: Expression, + context: Context, + *, + allow_unbound_tvars: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_typevar_arg: bool = True, + ) -> ProperType | None: + try: + # We want to use our custom error message below, so we suppress + # the default error message for invalid types here. + analyzed = self.expr_to_analyzed_type( + param_value, + allow_placeholder=True, + report_invalid_types=False, + allow_unbound_tvars=allow_unbound_tvars, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + ) + if analyzed is None: + # Type variables are special: we need to place them in the symbol table + # soon, even if upper bound is not ready yet. Otherwise avoiding + # a "deadlock" in this common pattern would be tricky: + # T = TypeVar('T', bound=Custom[Any]) + # class Custom(Generic[T]): + # ... + analyzed = PlaceholderType(None, [], context.line) + typ = get_proper_type(analyzed) + if report_invalid_typevar_arg and isinstance(typ, AnyType) and typ.is_from_error: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name), + param_value, + ) + # Note: we do not return 'None' here -- we want to continue + # using the AnyType. + return typ + except TypeTranslationError: + if report_invalid_typevar_arg: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name), + param_value, + ) + return None - If it does, return the corresponding TypeInfo. Return None otherwise. + def extract_typevarlike_name(self, s: AssignmentStmt, call: CallExpr) -> str | None: + if not call: + return None - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(call.callee, RefExpr): + lvalue = s.lvalues[0] + assert isinstance(lvalue, NameExpr) + if s.type: + self.fail("Cannot declare the type of a TypeVar or similar construct", s) return None - callee = cast(RefExpr, call.callee) - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + + if not self.check_typevarlike_name(call, lvalue.name, s): return None - items, types = self.parse_namedtuple_args(call, fullname) - if not items: - # Error. Construct dummy return value. - error_classdef = ClassDef('namedtuple', Block([])) - info = TypeInfo(SymbolTable(), error_classdef) - else: - listexpr = cast(ListExpr, call.args[1]) - name = cast(StrExpr, call.args[0]).value - info = self.build_namedtuple_typeinfo(name, items, types) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info + return lvalue.name - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type]]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], StrExpr): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - if not isinstance(args[1], ListExpr): - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - listexpr = cast(ListExpr, args[1]) - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, StrExpr) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - types = [AnyType() for _ in listexpr.items] # type: List[Type] - else: - # The fields argument contains (name, type) tuples. - items, types = self.parse_namedtuple_fields_with_types(listexpr.items, call) - return items, types - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type]]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - while isinstance(item, ParenExpr): - item = item.expr - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, StrExpr): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) + def process_paramspec_declaration(self, s: AssignmentStmt) -> bool: + """Checks if s declares a ParamSpec; if yes, store it in symbol table. + + Return True if this looks like a ParamSpec (maybe with errors), otherwise return False. + + In the future, ParamSpec may accept bounds and variance arguments, in which + case more aggressive sharing of code with process_typevar_declaration should be pursued. + """ + call = self.get_typevarlike_declaration( + s, ("typing_extensions.ParamSpec", "typing.ParamSpec") + ) + if not call: + return False + + name = self.extract_typevarlike_name(s, call) + if name is None: + return False + + n_values = call.arg_kinds[1:].count(ARG_POS) + if n_values != 0: + self.fail('Too many positional arguments for "ParamSpec"', s) + + default: Type = AnyType(TypeOfAny.from_omitted_generics) + for param_value, param_name in zip( + call.args[1 + n_values :], call.arg_names[1 + n_values :] + ): + if param_name == "default": + tv_arg = self.get_typevarlike_argument( + "ParamSpec", + param_name, + param_value, + s, + allow_unbound_tvars=True, + allow_param_spec_literals=True, + report_invalid_typevar_arg=False, + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + default = self.check_paramspec_default(default, param_value) + else: + # ParamSpec is different from a regular TypeVar: + # arguments are not semantically valid. But, allowed in runtime. + # So, we need to warn users about possible invalid usage. + self.fail( + "The variance and bound arguments to ParamSpec do not have defined semantics yet", + s, + ) + + # PEP 612 reserves the right to define bound, covariant and contravariant arguments to + # ParamSpec in a later PEP. If and when that happens, we should do something + # on the lines of process_typevar_parameters + + if not call.analyzed: + paramspec_var = ParamSpecExpr( + name, self.qualified_name(name), self.object_type(), default, INVARIANT + ) + paramspec_var.line = call.line + call.analyzed = paramspec_var + updated = True + else: + assert isinstance(call.analyzed, ParamSpecExpr) + updated = default != call.analyzed.default + call.analyzed.default = default + if has_placeholder(default): + self.process_placeholder(None, "ParamSpec default", s, force_progress=updated) + + self.add_symbol(name, call.analyzed, s) + return True + + def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: + """Checks if s declares a TypeVarTuple; if yes, store it in symbol table. + + Return True if this looks like a TypeVarTuple (maybe with errors), otherwise return False. + """ + call = self.get_typevarlike_declaration( + s, ("typing_extensions.TypeVarTuple", "typing.TypeVarTuple") + ) + if not call: + return False + + n_values = call.arg_kinds[1:].count(ARG_POS) + if n_values != 0: + self.fail('Too many positional arguments for "TypeVarTuple"', s) + + default: Type = AnyType(TypeOfAny.from_omitted_generics) + for param_value, param_name in zip( + call.args[1 + n_values :], call.arg_names[1 + n_values :] + ): + if param_name == "default": + tv_arg = self.get_typevarlike_argument( + "TypeVarTuple", + param_name, + param_value, + s, + allow_unbound_tvars=True, + report_invalid_typevar_arg=False, + allow_unpack=True, + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + default = self.check_typevartuple_default(default, param_value) else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types + self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s) + + name = self.extract_typevarlike_name(s, call) + if name is None: + return False - def fail_namedtuple_arg(self, message: str, context: Context) -> Tuple[List[str], List[Type]]: - self.fail(message, context) - return [], [] + # PEP 646 does not specify the behavior of variance, constraints, or bounds. + if not call.analyzed: + tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) + typevartuple_var = TypeVarTupleExpr( + name, + self.qualified_name(name), + # Upper bound for *Ts is *tuple[object, ...], it can never be object. + tuple_fallback.copy_modified(), + tuple_fallback, + default, + INVARIANT, + ) + typevartuple_var.line = call.line + call.analyzed = typevartuple_var + updated = True + else: + assert isinstance(call.analyzed, TypeVarTupleExpr) + updated = default != call.analyzed.default + call.analyzed.default = default + if has_placeholder(default): + self.process_placeholder(None, "TypeVarTuple default", s, force_progress=updated) + + self.add_symbol(name, call.analyzed, s) + return True - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() + def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance, line: int) -> TypeInfo: + if self.is_func_scope() and not self.type and "@" not in name: + name += "@" + str(line) class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = TypeInfo(symbols, class_def) - # Add named tuple items as attributes. - # TODO: Make them read-only. - for item, typ in zip(items, types): - var = Var(item) - var.info = info - var.type = typ - symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__ method. - init = self.make_namedtuple_init(info, items, types) - symbols['__init__'] = SymbolTableNode(MDEF, init) - info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple')) - info.mro = [info] + info.tuple_type.fallback.type.mro + if self.is_func_scope() and not self.type: + # Full names of generated classes should always be prefixed with the module names + # even if they are nested in a function, since these classes will be (de-)serialized. + # (Note that the caller should append @line to the name to avoid collisions.) + # TODO: clean this up, see #6422. + class_def.fullname = self.cur_mod_id + "." + self.qualified_name(name) + else: + class_def.fullname = self.qualified_name(name) + + info = TypeInfo(SymbolTable(), class_def, self.cur_mod_id) + class_def.info = info + mro = basetype_or_fallback.type.mro + if not mro: + # Probably an error, we should not crash so generate something meaningful. + mro = [basetype_or_fallback.type, self.object_type().type] + info.mro = [info] + mro + info.bases = [basetype_or_fallback] return info - def make_namedtuple_init(self, info: TypeInfo, items: List[str], - types: List[Type]) -> FuncDef: - args = [Var(item) for item in items] - for arg, type in zip(args, types): - arg.type = type - # TODO: Make sure that the self argument name is not visible? - args = [Var('__self')] + args - arg_kinds = [ARG_POS] * (len(items) + 1) - signature = Callable([cast(Type, None)] + types, - arg_kinds, - ['__self'] + items, - NoneTyp(), - self.named_type('__builtins__.function'), - name=info.name()) - return FuncDef('__init__', - args, arg_kinds, - [None] * (len(items) + 1), - Block([]), - typ=signature) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = List[Type]() + def analyze_value_types(self, items: list[Expression]) -> list[Type]: + """Analyze types from values expressions in type variable definition.""" + result: list[Type] = [] for node in items: try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) + analyzed = self.anal_type( + self.expr_to_unanalyzed_type(node), allow_placeholder=True + ) + if analyzed is None: + # Type variables are special: we need to place them in the symbol table + # soon, even if some value is not ready yet, see process_typevar_parameters() + # for an example. + analyzed = PlaceholderType(None, [], node.line) + if has_type_vars(analyzed): + self.fail(message_registry.TYPE_VAR_GENERIC_CONSTRAINT_TYPE, node) + result.append(AnyType(TypeOfAny.from_error)) + else: + result.append(analyzed) except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) + self.fail("Type expected", node) + result.append(AnyType(TypeOfAny.from_error)) return result - def visit_decorator(self, dec: Decorator) -> None: - if not dec.is_overload: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - for d in dec.decorators: - d.accept(self) - removed = List[int]() - for i, d in enumerate(dec.decorators): - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - dec.func.is_coroutine = True - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif refers_to_fullname(d, 'builtins.property'): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if dec.is_overload: - self.fail('A property cannot be overloaded', dec) - self.check_decorated_function_is_method('property', dec) - if len(dec.func.args) > 1: - self.fail('Too many arguments', dec.func) - for i in reversed(removed): - del dec.decorators[i] - dec.func.accept(self) - if not dec.decorators and not dec.var.is_property: - # No non-special decorators left. We can trivially infer the type - # of the function here. - dec.var.type = dec.func.type - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) + def check_classvar(self, s: AssignmentStmt) -> None: + """Check if assignment defines a class variable.""" + lvalue = s.lvalues[0] + if len(s.lvalues) != 1 or not isinstance(lvalue, RefExpr): + return + if not s.type or not self.is_classvar(s.type): + return + if self.is_class_scope() and isinstance(lvalue, NameExpr): + node = lvalue.node + if isinstance(node, Var): + node.is_classvar = True + analyzed = self.anal_type(s.type) + assert self.type is not None + if analyzed is not None and set(get_type_vars(analyzed)) & set( + self.type.defn.type_vars + ): + # This means that we have a type var defined inside of a ClassVar. + # This is not allowed by PEP526. + # See https://github.com/python/mypy/issues/11538 + + self.fail(message_registry.CLASS_VAR_WITH_TYPEVARS, s) + if ( + analyzed is not None + and self.type.self_type in get_type_vars(analyzed) + and self.type.defn.type_vars + ): + self.fail(message_registry.CLASS_VAR_WITH_GENERIC_SELF, s) + elif not isinstance(lvalue, MemberExpr) or self.is_self_member_ref(lvalue): + # In case of member access, report error only when assigning to self + # Other kinds of member assignments should be already reported + self.fail_invalid_classvar(lvalue) + + def is_classvar(self, typ: Type) -> bool: + if not isinstance(typ, UnboundType): + return False + sym = self.lookup_qualified(typ.name, typ) + if not sym or not sym.node: + return False + return sym.node.fullname == "typing.ClassVar" + + def is_final_type(self, typ: Type | None) -> bool: + if not isinstance(typ, UnboundType): + return False + sym = self.lookup_qualified(typ.name, typ) + if not sym or not sym.node: + return False + return sym.node.fullname in FINAL_TYPE_NAMES + + def fail_invalid_classvar(self, context: Context) -> None: + self.fail(message_registry.CLASS_VAR_OUTSIDE_OF_CLASS, context) + + def process_module_assignment( + self, lvals: list[Lvalue], rval: Expression, ctx: AssignmentStmt + ) -> None: + """Propagate module references across assignments. + + Recursively handles the simple form of iterable unpacking; doesn't + handle advanced unpacking with *rest, dictionary unpacking, etc. + + In an expression like x = y = z, z is the rval and lvals will be [x, + y]. + + """ + if isinstance(rval, (TupleExpr, ListExpr)) and all( + isinstance(v, TupleExpr) for v in lvals + ): + # rval and all lvals are either list or tuple, so we are dealing + # with unpacking assignment like `x, y = a, b`. Mypy didn't + # understand our all(isinstance(...)), so cast them as TupleExpr + # so mypy knows it is safe to access their .items attribute. + seq_lvals = cast(List[TupleExpr], lvals) + # given an assignment like: + # (x, y) = (m, n) = (a, b) + # we now have: + # seq_lvals = [(x, y), (m, n)] + # seq_rval = (a, b) + # We now zip this into: + # elementwise_assignments = [(a, x, m), (b, y, n)] + # where each elementwise assignment includes one element of rval and the + # corresponding element of each lval. Basically we unpack + # (x, y) = (m, n) = (a, b) + # into elementwise assignments + # x = m = a + # y = n = b + # and then we recursively call this method for each of those assignments. + # If the rval and all lvals are not all of the same length, zip will just ignore + # extra elements, so no error will be raised here; mypy will later complain + # about the length mismatch in type-checking. + elementwise_assignments = zip(rval.items, *[v.items for v in seq_lvals]) + for rv, *lvs in elementwise_assignments: + self.process_module_assignment(lvs, rv, ctx) + elif isinstance(rval, RefExpr): + rnode = self.lookup_type_node(rval) + if rnode and isinstance(rnode.node, MypyFile): + for lval in lvals: + if not isinstance(lval, RefExpr): + continue + # respect explicitly annotated type + if isinstance(lval.node, Var) and lval.node.type is not None: + continue + + # We can handle these assignments to locals and to self + if isinstance(lval, NameExpr): + lnode = self.current_symbol_table().get(lval.name) + elif isinstance(lval, MemberExpr) and self.is_self_member_ref(lval): + assert self.type is not None + lnode = self.type.names.get(lval.name) + else: + continue + + if lnode: + if isinstance(lnode.node, MypyFile) and lnode.node is not rnode.node: + assert isinstance(lval, (NameExpr, MemberExpr)) + self.fail( + 'Cannot assign multiple modules to name "{}" ' + 'without explicit "types.ModuleType" annotation'.format(lval.name), + ctx, + ) + # never create module alias except on initial var definition + elif lval.is_inferred_def: + assert rnode.node is not None + lnode.node = rnode.node + + def process__all__(self, s: AssignmentStmt) -> None: + """Export names if argument is a __all__ assignment.""" + if ( + len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and s.lvalues[0].name == "__all__" + and s.lvalues[0].kind == GDEF + and isinstance(s.rvalue, (ListExpr, TupleExpr)) + ): + self.add_exports(s.rvalue.items) + + def process__deletable__(self, s: AssignmentStmt) -> None: + if not self.options.mypyc: + return + if ( + len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and s.lvalues[0].name == "__deletable__" + and s.lvalues[0].kind == MDEF + ): + rvalue = s.rvalue + if not isinstance(rvalue, (ListExpr, TupleExpr)): + self.fail('"__deletable__" must be initialized with a list or tuple expression', s) + return + items = rvalue.items + attrs = [] + for item in items: + if not isinstance(item, StrExpr): + self.fail('Invalid "__deletable__" item; string literal expected', item) + else: + attrs.append(item.value) + assert self.type + self.type.deletable_attributes = attrs + + def process__slots__(self, s: AssignmentStmt) -> None: + """ + Processing ``__slots__`` if defined in type. + + See: https://docs.python.org/3/reference/datamodel.html#slots + """ + # Later we can support `__slots__` defined as `__slots__ = other = ('a', 'b')` + if ( + isinstance(self.type, TypeInfo) + and len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and s.lvalues[0].name == "__slots__" + and s.lvalues[0].kind == MDEF + ): + # We understand `__slots__` defined as string, tuple, list, set, and dict: + if not isinstance(s.rvalue, (StrExpr, ListExpr, TupleExpr, SetExpr, DictExpr)): + # For example, `__slots__` can be defined as a variable, + # we don't support it for now. + return + + if any(p.slots is None for p in self.type.mro[1:-1]): + # At least one type in mro (excluding `self` and `object`) + # does not have concrete `__slots__` defined. Ignoring. + return + + concrete_slots = True + rvalue: list[Expression] = [] + if isinstance(s.rvalue, StrExpr): + rvalue.append(s.rvalue) + elif isinstance(s.rvalue, (ListExpr, TupleExpr, SetExpr)): + rvalue.extend(s.rvalue.items) + else: + # We have a special treatment of `dict` with possible `{**kwargs}` usage. + # In this case we consider all `__slots__` to be non-concrete. + for key, _ in s.rvalue.items: + if concrete_slots and key is not None: + rvalue.append(key) + else: + concrete_slots = False + + slots = [] + for item in rvalue: + # Special case for `'__dict__'` value: + # when specified it will still allow any attribute assignment. + if isinstance(item, StrExpr) and item.value != "__dict__": + slots.append(item.value) + else: + concrete_slots = False + if not concrete_slots: + # Some slot items are dynamic, we don't want any false positives, + # so, we just pretend that this type does not have any slots at all. + return + + # We need to copy all slots from super types: + for super_type in self.type.mro[1:-1]: + assert super_type.slots is not None + slots.extend(super_type.slots) + self.type.slots = set(slots) + + # + # Misc statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block | None) -> None: + if b: + self.visit_block(b) def visit_expression_stmt(self, s: ExpressionStmt) -> None: + self.statement = s s.expr.accept(self) def visit_return_stmt(self, s: ReturnStmt) -> None: + self.statement = s if not self.is_func_scope(): - self.fail("'return' outside function", s) + self.fail('"return" outside function', s) + if self.return_stmt_inside_except_star_block: + self.fail('"return" not allowed in except* block', s, serious=True) if s.expr: s.expr.accept(self) def visit_raise_stmt(self, s: RaiseStmt) -> None: + self.statement = s if s.expr: s.expr.accept(self) if s.from_expr: s.from_expr.accept(self) - def visit_yield_stmt(self, s: YieldStmt) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", s) - else: - self.function_stack[-1].is_generator = True - if s.expr: - s.expr.accept(self) - - def visit_yield_from_stmt(self, s: YieldFromStmt) -> None: - if not self.is_func_scope(): - self.fail("'yield from' outside function", s) - if s.expr: - s.expr.accept(self) - def visit_assert_stmt(self, s: AssertStmt) -> None: + self.statement = s if s.expr: s.expr.accept(self) + if s.msg: + s.msg.accept(self) - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: + def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> None: + self.statement = s s.lvalue.accept(self) s.rvalue.accept(self) + if ( + isinstance(s.lvalue, NameExpr) + and s.lvalue.name == "__all__" + and s.lvalue.kind == GDEF + and isinstance(s.rvalue, (ListExpr, TupleExpr)) + ): + self.add_exports(s.rvalue.items) def visit_while_stmt(self, s: WhileStmt) -> None: + self.statement = s s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 + self.loop_depth[-1] += 1 + with self.inside_except_star_block_set(value=False, entering_loop=True): + s.body.accept(self) + self.loop_depth[-1] -= 1 self.visit_block_maybe(s.else_body) def visit_for_stmt(self, s: ForStmt) -> None: + if s.is_async: + if not self.is_func_scope() or not self.function_stack[-1].is_coroutine: + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, s, code=codes.SYNTAX) + + self.statement = s s.expr.accept(self) # Bind index variables and check if they define new names. - self.analyse_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - + self.analyze_lvalue(s.index, explicit_type=s.index_type is not None, is_index_var=True) + if s.index_type: + if self.is_classvar(s.index_type): + self.fail_invalid_classvar(s.index) + allow_tuple_literal = isinstance(s.index, TupleExpr) + analyzed = self.anal_type(s.index_type, allow_tuple_literal=allow_tuple_literal) + if analyzed is not None: + self.store_declared_types(s.index, analyzed) + s.index_type = analyzed + + self.loop_depth[-1] += 1 + with self.inside_except_star_block_set(value=False, entering_loop=True): + self.visit_block(s.body) + self.loop_depth[-1] -= 1 self.visit_block_maybe(s.else_body) def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s) + self.statement = s + if self.loop_depth[-1] == 0: + self.fail('"break" outside loop', s, serious=True, blocker=True) + if self.inside_except_star_block: + self.fail('"break" not allowed in except* block', s, serious=True) def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s) + self.statement = s + if self.loop_depth[-1] == 0: + self.fail('"continue" outside loop', s, serious=True, blocker=True) + if self.inside_except_star_block: + self.fail('"continue" not allowed in except* block', s, serious=True) def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) + self.statement = s + infer_reachability_of_if_statement(s, self.options) for i in range(len(s.expr)): s.expr[i].accept(self) self.visit_block(s.body[i]) self.visit_block_maybe(s.else_body) def visit_try_stmt(self, s: TryStmt) -> None: + self.statement = s self.analyze_try_stmt(s, self) - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor[None]) -> None: s.body.accept(visitor) for type, var, handler in zip(s.types, s.vars, s.handlers): if type: type.accept(visitor) if var: - self.analyse_lvalue(var, add_global=add_global) - handler.accept(visitor) + self.analyze_lvalue(var) + with self.inside_except_star_block_set(self.inside_except_star_block or s.is_star): + handler.accept(visitor) if s.else_body: s.else_body.accept(visitor) if s.finally_body: s.finally_body.accept(visitor) def visit_with_stmt(self, s: WithStmt) -> None: - for e in s.expr: + self.statement = s + types: list[Type] = [] + + if s.is_async: + if not self.is_func_scope() or not self.function_stack[-1].is_coroutine: + self.fail(message_registry.ASYNC_WITH_OUTSIDE_COROUTINE, s, code=codes.SYNTAX) + + if s.unanalyzed_type: + assert isinstance(s.unanalyzed_type, ProperType) + actual_targets = [t for t in s.target if t is not None] + if len(actual_targets) == 0: + # We have a type for no targets + self.fail('Invalid type comment: "with" statement has no targets', s) + elif len(actual_targets) == 1: + # We have one target and one type + types = [s.unanalyzed_type] + elif isinstance(s.unanalyzed_type, TupleType): + # We have multiple targets and multiple types + if len(actual_targets) == len(s.unanalyzed_type.items): + types = s.unanalyzed_type.items.copy() + else: + # But it's the wrong number of items + self.fail('Incompatible number of types for "with" targets', s) + else: + # We have multiple targets and one type + self.fail('Multiple types expected for multiple "with" targets', s) + + new_types: list[Type] = [] + for e, n in zip(s.expr, s.target): e.accept(self) - for n in s.name: if n: - self.analyse_lvalue(n) + self.analyze_lvalue(n, explicit_type=s.unanalyzed_type is not None) + + # Since we have a target, pop the next type from types + if types: + t = types.pop(0) + if self.is_classvar(t): + self.fail_invalid_classvar(n) + allow_tuple_literal = isinstance(n, TupleExpr) + analyzed = self.anal_type(t, allow_tuple_literal=allow_tuple_literal) + if analyzed is not None: + # TODO: Deal with this better + new_types.append(analyzed) + self.store_declared_types(n, analyzed) + + s.analyzed_types = new_types + self.visit_block(s.body) def visit_del_stmt(self, s: DelStmt) -> None: + self.statement = s s.expr.accept(self) - if not isinstance(s.expr, (IndexExpr, NameExpr, MemberExpr)): - self.fail('Invalid delete target', s) + if not self.is_valid_del_target(s.expr): + self.fail("Invalid delete target", s) + + def is_valid_del_target(self, s: Expression) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, (TupleExpr, ListExpr)): + return all(self.is_valid_del_target(item) for item in s.items) + else: + return False def visit_global_decl(self, g: GlobalDecl) -> None: + self.statement = g for name in g.names: if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) + self.fail(f'Name "{name}" is nonlocal and global', g) self.global_decls[-1].add(name) def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): + self.statement = d + if self.is_module_scope(): self.fail("nonlocal declaration not allowed at module level", d) else: for name in d.names: - for table in reversed(self.locals[:-1]): + for table, scope_type in zip( + reversed(self.locals[:-1]), reversed(self.scope_stack[:-1]) + ): if table is not None and name in table: + if scope_type == SCOPE_ANNOTATION: + self.fail( + f'nonlocal binding not allowed for type parameter "{name}"', d + ) break else: - self.fail("No binding for nonlocal '{}' found".format(name), d) + self.fail(f'No binding for nonlocal "{name}" found', d) if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) + self.fail( + 'Name "{}" is already defined in local ' + "scope before nonlocal declaration".format(name), + d, + ) if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) + self.fail(f'Name "{name}" is nonlocal and global', d) self.nonlocal_decls[-1].add(name) - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) + def visit_match_stmt(self, s: MatchStmt) -> None: + self.statement = s + infer_reachability_of_match_statement(s, self.options) + s.subject.accept(self) + for i in range(len(s.patterns)): + s.patterns[i].accept(self) + guard = s.guards[i] + if guard is not None: + guard.accept(self) + self.visit_block(s.bodies[i]) + + def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: + if s.invalid_recursive_alias: + return + self.statement = s + type_params = self.push_type_args(s.type_args, s) + if type_params is None: + self.defer(s) + return + all_type_params_names = [p.name for p in s.type_args] + + try: + existing = self.current_symbol_table().get(s.name.name) + if existing and not ( + isinstance(existing.node, TypeAlias) + or (isinstance(existing.node, PlaceholderNode) and existing.node.line == s.line) + ): + self.already_defined(s.name.name, s, existing, "Name") + return + + tag = self.track_incomplete_refs() + res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( + s.name.name, + s.value.expr(), + allow_placeholder=True, + declared_type_vars=type_params, + all_declared_type_params_names=all_type_params_names, + python_3_12_type_alias=True, + ) + if not res: + res = AnyType(TypeOfAny.from_error) + + if not self.is_func_scope(): + # Only marking incomplete for top-level placeholders makes recursive aliases like + # `A = Sequence[str | A]` valid here, similar to how we treat base classes in class + # definitions, allowing `class str(Sequence[str]): ...` + incomplete_target = isinstance(res, ProperType) and isinstance( + res, PlaceholderType + ) + else: + incomplete_target = has_placeholder(res) + + if self.found_incomplete_ref(tag) or incomplete_target: + # Since we have got here, we know this must be a type alias (incomplete refs + # may appear in nested positions), therefore use becomes_typeinfo=True. + self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) + return + + self.add_type_alias_deps(depends_on) + # In addition to the aliases used, we add deps on unbound + # type variables, since they are erased from target type. + self.add_type_alias_deps(qualified_tvars) + # The above are only direct deps on other aliases. + # For subscripted aliases, type deps from expansion are added in deps.py + # (because the type is stored). + check_for_explicit_any( + res, self.options, self.is_typeshed_stub_file, self.msg, context=s + ) + # When this type alias gets "inlined", the Any is not explicit anymore, + # so we need to replace it with non-explicit Anys. + res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) + eager = self.is_func_scope() + if isinstance(res, ProperType) and isinstance(res, Instance) and not res.args: + fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) + alias_node = TypeAlias( + res, + self.qualified_name(s.name.name), + s.line, + s.column, + alias_tvars=alias_tvars, + no_args=False, + eager=eager, + python_3_12_type_alias=True, + ) + s.alias_node = alias_node + + if ( + existing + and isinstance(existing.node, (PlaceholderNode, TypeAlias)) + and existing.node.line == s.line + ): + updated = False + if isinstance(existing.node, TypeAlias): + if existing.node.target != res: + # Copy expansion to the existing alias, this matches how we update base classes + # for a TypeInfo _in place_ if there are nested placeholders. + existing.node.target = res + existing.node.alias_tvars = alias_tvars + updated = True + else: + # Otherwise just replace existing placeholder with type alias. + existing.node = alias_node + updated = True + + if updated: + if self.final_iteration: + self.cannot_resolve_name(s.name.name, "name", s) + return + else: + # We need to defer so that this change can get propagated to base classes. + self.defer(s, force_progress=True) + else: + self.add_symbol(s.name.name, alias_node, s) + + current_node = existing.node if existing else alias_node + assert isinstance(current_node, TypeAlias) + self.disable_invalid_recursive_aliases(s, current_node, s.value) + s.name.accept(self) + finally: + self.pop_type_args(s.type_args) # # Expressions @@ -1333,49 +5635,75 @@ def visit_print_stmt(self, s: PrintStmt) -> None: def visit_name_expr(self, expr: NameExpr) -> None: n = self.lookup(expr.name, expr) if n: - if n.kind == TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = (cast(Node, n.node)) - expr.fullname = n.fullname + self.bind_name_expr(expr, n) + + def bind_name_expr(self, expr: NameExpr, sym: SymbolTableNode) -> None: + """Bind name expression to a symbol table node.""" + if ( + isinstance(sym.node, TypeVarExpr) + and self.tvar_scope.get_binding(sym) + and not self.allow_unbound_tvars + ): + self.fail(f'"{expr.name}" is a type variable and only valid in type context', expr) + elif isinstance(sym.node, PlaceholderNode): + self.process_placeholder(expr.name, "name", expr) + else: + expr.kind = sym.kind + expr.node = sym.node + expr.fullname = sym.fullname or "" def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: + if not self.type and not expr.call.args: self.fail('"super" used outside class', expr) return expr.info = self.type + for arg in expr.call.args: + arg.accept(self) def visit_tuple_expr(self, expr: TupleExpr) -> None: for item in expr.items: + if isinstance(item, StarExpr): + item.valid = True item.accept(self) def visit_list_expr(self, expr: ListExpr) -> None: for item in expr.items: + if isinstance(item, StarExpr): + item.valid = True item.accept(self) def visit_set_expr(self, expr: SetExpr) -> None: for item in expr.items: + if isinstance(item, StarExpr): + item.valid = True item.accept(self) def visit_dict_expr(self, expr: DictExpr) -> None: for key, value in expr.items: - key.accept(self) + if key is not None: + key.accept(self) value.accept(self) - def visit_paren_expr(self, expr: ParenExpr) -> None: - expr.expr.accept(self) - def visit_star_expr(self, expr: StarExpr) -> None: if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) + self.fail("can't use starred expression here", expr, blocker=True) else: expr.expr.accept(self) def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e) + if not self.is_func_scope(): + self.fail('"yield from" outside function', e, serious=True, blocker=True) + elif self.scope_stack[-1] == SCOPE_COMPREHENSION: + self.fail( + '"yield from" inside comprehension or generator expression', + e, + serious=True, + blocker=True, + ) + elif self.function_stack[-1].is_coroutine: + self.fail('"yield from" in async function', e, serious=True, blocker=True) + else: + self.function_stack[-1].is_generator = True if e.expr: e.expr.accept(self) @@ -1383,109 +5711,242 @@ def visit_call_expr(self, expr: CallExpr) -> None: """Analyze a call expression. Some call expressions are recognized as special forms, including - cast(...), Undefined(...) and Any(...). + cast(...). """ expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): + if refers_to_fullname(expr.callee, "typing.cast"): # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): + if not self.check_fixed_args(expr, 2, "cast"): return # Translate first argument to an unanalyzed type. try: - target = expr_to_unanalyzed_type(expr.args[0]) + target = self.expr_to_unanalyzed_type(expr.args[0]) except TypeTranslationError: - self.fail('Cast target is not a type', expr) + self.fail("Cast target is not a type", expr) return - # Pigguback CastExpr object to the CallExpr object; it takes + # Piggyback CastExpr object to the CallExpr object; it takes # precedence over the CallExpr semantics. expr.analyzed = CastExpr(expr.args[1], target) expr.analyzed.line = expr.line + expr.analyzed.column = expr.column expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Undefined'): - # Special form Undefined(...). - if not self.check_fixed_args(expr, 1, 'Undefined'): + elif refers_to_fullname(expr.callee, ASSERT_TYPE_NAMES): + if not self.check_fixed_args(expr, 2, "assert_type"): return + # Translate second argument to an unanalyzed type. try: - type = expr_to_unanalyzed_type(expr.args[0]) + target = self.expr_to_unanalyzed_type(expr.args[1]) except TypeTranslationError: - self.fail('Argument to Undefined is not a type', expr) + self.fail("assert_type() type is not a type", expr) + return + expr.analyzed = AssertTypeExpr(expr.args[0], target) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, REVEAL_TYPE_NAMES): + if not self.check_fixed_args(expr, 1, "reveal_type"): return - expr.analyzed = UndefinedExpr(type) + reveal_imported = False + reveal_type_node = self.lookup("reveal_type", expr, suppress_errors=True) + if ( + reveal_type_node + and isinstance(reveal_type_node.node, FuncBase) + and reveal_type_node.fullname in IMPORTED_REVEAL_TYPE_NAMES + ): + reveal_imported = True + expr.analyzed = RevealExpr( + kind=REVEAL_TYPE, expr=expr.args[0], is_imported=reveal_imported + ) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, "builtins.reveal_locals"): + # Store the local variable names into the RevealExpr for use in the + # type checking pass + local_nodes: list[Var] = [] + if self.is_module_scope(): + # try to determine just the variable declarations in module scope + # self.globals.values() contains SymbolTableNode's + # Each SymbolTableNode has an attribute node that is nodes.Var + # look for variable nodes that marked as is_inferred + # Each symboltable node has a Var node as .node + local_nodes = [ + n.node + for name, n in self.globals.items() + if getattr(n.node, "is_inferred", False) and isinstance(n.node, Var) + ] + elif self.is_class_scope(): + # type = None # type: Optional[TypeInfo] + if self.type is not None: + local_nodes = [ + st.node for st in self.type.names.values() if isinstance(st.node, Var) + ] + elif self.is_func_scope(): + # locals = None # type: List[Optional[SymbolTable]] + if self.locals is not None: + symbol_table = self.locals[-1] + if symbol_table is not None: + local_nodes = [ + st.node for st in symbol_table.values() if isinstance(st.node, Var) + ] + expr.analyzed = RevealExpr(kind=REVEAL_LOCALS, local_nodes=local_nodes) expr.analyzed.line = expr.line + expr.analyzed.column = expr.column expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.ducktype'): - # Special form ducktype(...). - if not self.check_fixed_args(expr, 1, 'ducktype'): + elif refers_to_fullname(expr.callee, "typing.Any"): + # Special form Any(...) no longer supported. + self.fail("Any(...) is no longer supported. Use cast(Any, ...) instead", expr) + elif refers_to_fullname(expr.callee, "typing._promote"): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, "_promote"): return # Translate first argument to an unanalyzed type. try: - target = expr_to_unanalyzed_type(expr.args[0]) + target = self.expr_to_unanalyzed_type(expr.args[0]) except TypeTranslationError: - self.fail('Argument 1 to ducktype is not a type', expr) + self.fail("Argument 1 to _promote is not a type", expr) return - expr.analyzed = DucktypeExpr(target) + expr.analyzed = PromoteExpr(target) expr.analyzed.line = expr.line expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.disjointclass'): - # Special form disjointclass(...). - if not self.check_fixed_args(expr, 1, 'disjointclass'): + elif refers_to_fullname(expr.callee, "builtins.dict"): + expr.analyzed = self.translate_dict_call(expr) + elif refers_to_fullname(expr.callee, "builtins.divmod"): + if not self.check_fixed_args(expr, 2, "divmod"): return - arg = expr.args[0] - if isinstance(arg, RefExpr): - expr.analyzed = DisjointclassExpr(arg) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - else: - self.fail('Argument 1 to disjointclass is not a class', expr) + expr.analyzed = OpExpr("divmod", expr.args[0], expr.args[1]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname( + expr.callee, ("typing.TypeAliasType", "typing_extensions.TypeAliasType") + ): + with self.allow_unbound_tvars_set(): + for a in expr.args: + a.accept(self) else: # Normal call expression. for a in expr.args: a.accept(self) - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: + if ( + isinstance(expr.callee, MemberExpr) + and isinstance(expr.callee.expr, NameExpr) + and expr.callee.expr.name == "__all__" + and expr.callee.expr.kind == GDEF + and expr.callee.name in ("append", "extend", "remove") + ): + if expr.callee.name == "append" and expr.args: + self.add_exports(expr.args[0]) + elif ( + expr.callee.name == "extend" + and expr.args + and isinstance(expr.args[0], (ListExpr, TupleExpr)) + ): + self.add_exports(expr.args[0].items) + elif ( + expr.callee.name == "remove" + and expr.args + and isinstance(expr.args[0], StrExpr) + ): + self.all_exports = [n for n in self.all_exports if n != expr.args[0].value] + + def translate_dict_call(self, call: CallExpr) -> DictExpr | None: + """Translate 'dict(x=y, ...)' to {'x': y, ...} and 'dict()' to {}. + + For other variants of dict(...), return None. + """ + if not all(kind in (ARG_NAMED, ARG_STAR2) for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr( + [ + (StrExpr(key) if key is not None else None, value) + for key, value in zip(call.arg_names, call.args) + ] + ) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, name: str) -> bool: """Verify that expr has specified number of positional args. Return True if the arguments are valid. """ - s = 's' + s = "s" if numargs == 1: - s = '' + s = "" if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) + self.fail('"%s" expects %d argument%s' % (name, numargs, s), expr) return False if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) + self.fail(f'"{name}" must be called with {numargs} positional argument{s}', expr) return False return True def visit_member_expr(self, expr: MemberExpr) -> None: base = expr.expr base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and cast(RefExpr, - base).kind == MODULE_REF: - names = (cast(MypyFile, (cast(RefExpr, base)).node)).names - n = names.get(expr.name, None) - if n: - n = self.normalize_type_alias(n, expr) - if not n: + if isinstance(base, RefExpr) and isinstance(base.node, MypyFile): + # Handle module attribute. + sym = self.get_module_symbol(base.node, expr.name) + if sym: + if isinstance(sym.node, PlaceholderNode): + self.process_placeholder(expr.name, "attribute", expr) return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node + expr.kind = sym.kind + expr.fullname = sym.fullname or "" + expr.node = sym.node + elif isinstance(base, RefExpr): + # This branch handles the case C.bar (or cls.bar or self.bar inside + # a classmethod/method), where C is a class and bar is a type + # definition or a module resulting from `import bar` (or a module + # assignment) inside class C. We look up bar in the class' TypeInfo + # namespace. This is done only when bar is a module or a type; + # other things (e.g. methods) are handled by other code in + # checkmember. + type_info = None + if isinstance(base.node, TypeInfo): + # C.bar where C is a class + type_info = base.node + elif isinstance(base.node, Var) and self.type and self.function_stack: + # check for self.bar or cls.bar in method/classmethod + func_def = self.function_stack[-1] + if not func_def.is_static and isinstance(func_def.type, CallableType): + formal_arg = func_def.type.argument_by_name(base.node.name) + if formal_arg and formal_arg.pos == 0: + type_info = self.type + elif isinstance(base.node, TypeAlias) and base.node.no_args: + assert isinstance(base.node.target, ProperType) + if isinstance(base.node.target, Instance): + type_info = base.node.target.type + + if type_info: + n = type_info.names.get(expr.name) + if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)): + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname or "" + expr.node = n.node def visit_op_expr(self, expr: OpExpr) -> None: expr.left.accept(self) + + if expr.op in ("and", "or"): + inferred = infer_condition_value(expr.left, self.options) + if (inferred in (ALWAYS_FALSE, MYPY_FALSE) and expr.op == "and") or ( + inferred in (ALWAYS_TRUE, MYPY_TRUE) and expr.op == "or" + ): + expr.right_unreachable = True + return + elif (inferred in (ALWAYS_TRUE, MYPY_TRUE) and expr.op == "and") or ( + inferred in (ALWAYS_FALSE, MYPY_FALSE) and expr.op == "or" + ): + expr.right_always = True + expr.right.accept(self) def visit_comparison_expr(self, expr: ComparisonExpr) -> None: @@ -1496,28 +5957,130 @@ def visit_unary_expr(self, expr: UnaryExpr) -> None: expr.expr.accept(self) def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = List[Type]() - if isinstance(expr.index, TupleExpr): - items = (cast(TupleExpr, expr.index)).items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line + base = expr.base + base.accept(self) + if ( + isinstance(base, RefExpr) + and isinstance(base.node, TypeInfo) + and not base.node.is_generic() + ): + expr.index.accept(self) + elif ( + isinstance(base, RefExpr) and isinstance(base.node, TypeAlias) + ) or refers_to_class_or_function(base): + # We need to do full processing on every iteration, since some type + # arguments may contain placeholder types. + self.analyze_type_application(expr) else: expr.index.accept(self) + def analyze_type_application(self, expr: IndexExpr) -> None: + """Analyze special form -- type application (either direct or via type aliasing).""" + types = self.analyze_type_application_args(expr) + if types is None: + return + base = expr.base + expr.analyzed = TypeApplication(base, types) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + # Types list, dict, set are not subscriptable, prohibit this if + # subscripted either via type alias... + if isinstance(base, RefExpr) and isinstance(base.node, TypeAlias): + alias = base.node + target = get_proper_type(alias.target) + if isinstance(target, Instance): + name = target.type.fullname + if ( + alias.no_args + and name # this avoids bogus errors for already reported aliases + in get_nongen_builtins(self.options.python_version) + and not self.is_stub_file + and not alias.normalized + ): + self.fail(no_subscript_builtin_alias(name, propose_alt=False), expr) + # ...or directly. + else: + n = self.lookup_type_node(base) + if ( + n + and n.fullname in get_nongen_builtins(self.options.python_version) + and not self.is_stub_file + ): + self.fail(no_subscript_builtin_alias(n.fullname, propose_alt=False), expr) + + def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None: + """Analyze type arguments (index) in a type application. + + Return None if anything was incomplete. + """ + index = expr.index + tag = self.track_incomplete_refs() + self.analyze_type_expr(index) + if self.found_incomplete_ref(tag): + return None + if self.basic_type_applications: + # Postpone the rest until we have more information (for r.h.s. of an assignment) + return None + types: list[Type] = [] + if isinstance(index, TupleExpr): + items = index.items + is_tuple = isinstance(expr.base, RefExpr) and expr.base.fullname == "builtins.tuple" + if is_tuple and len(items) == 2 and isinstance(items[-1], EllipsisExpr): + items = items[:-1] + else: + items = [index] + + # TODO: this needs a clean-up. + # Probably always allow Parameters literals, and validate in semanal_typeargs.py + base = expr.base + if isinstance(base, RefExpr) and isinstance(base.node, TypeAlias): + allow_unpack = base.node.tvar_tuple_index is not None + alias = base.node + if any(isinstance(t, ParamSpecType) for t in alias.alias_tvars): + has_param_spec = True + num_args = len(alias.alias_tvars) + else: + has_param_spec = False + num_args = -1 + elif isinstance(base, RefExpr) and isinstance(base.node, TypeInfo): + allow_unpack = ( + base.node.has_type_var_tuple_type or base.node.fullname == "builtins.tuple" + ) + has_param_spec = base.node.has_param_spec_type + num_args = len(base.node.type_vars) + else: + allow_unpack = False + has_param_spec = False + num_args = -1 + + for item in items: + try: + typearg = self.expr_to_unanalyzed_type(item, allow_unpack=True) + except TypeTranslationError: + self.fail("Type expected within [...]", expr) + return None + analyzed = self.anal_type( + typearg, + # The type application may appear in base class expression, + # where type variables are not bound yet. Or when accepting + # r.h.s. of type alias before we figured out it is a type alias. + allow_unbound_tvars=self.allow_unbound_tvars, + allow_placeholder=True, + allow_param_spec_literals=has_param_spec, + allow_unpack=allow_unpack, + ) + if analyzed is None: + return None + types.append(analyzed) + + if has_param_spec and num_args == 1 and types: + first_arg = get_proper_type(types[0]) + single_any = len(types) == 1 and isinstance(first_arg, AnyType) + if not (single_any or any(isinstance(t, (Parameters, ParamSpecType)) for t in types)): + types = [Parameters(types, [ARG_POS] * len(types), [None] * len(types))] + + return types + def visit_slice_expr(self, expr: SliceExpr) -> None: if expr.begin_index: expr.begin_index.accept(self) @@ -1528,508 +6091,1510 @@ def visit_slice_expr(self, expr: SliceExpr) -> None: def visit_cast_expr(self, expr: CastExpr) -> None: expr.expr.accept(self) - expr.type = self.anal_type(expr.type) + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed - def visit_undefined_expr(self, expr: UndefinedExpr) -> None: - expr.type = self.anal_type(expr.type) + def visit_assert_type_expr(self, expr: AssertTypeExpr) -> None: + expr.expr.accept(self) + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed + + def visit_reveal_expr(self, expr: RevealExpr) -> None: + if expr.kind == REVEAL_TYPE: + if expr.expr is not None: + expr.expr.accept(self) + else: + # Reveal locals doesn't have an inner expression, there's no + # need to traverse inside it + pass def visit_type_application(self, expr: TypeApplication) -> None: expr.expr.accept(self) for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) + analyzed = self.anal_type(expr.types[i]) + if analyzed is not None: + expr.types[i] = analyzed def visit_list_comprehension(self, expr: ListComprehension) -> None: + if any(expr.generator.is_async): + if not self.is_func_scope() or not self.function_stack[-1].is_coroutine: + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, expr, code=codes.SYNTAX) + expr.generator.accept(self) def visit_set_comprehension(self, expr: SetComprehension) -> None: + if any(expr.generator.is_async): + if not self.is_func_scope() or not self.function_stack[-1].is_coroutine: + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, expr, code=codes.SYNTAX) + expr.generator.accept(self) def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyse_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() + if any(expr.is_async): + if not self.is_func_scope() or not self.function_stack[-1].is_coroutine: + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, expr, code=codes.SYNTAX) + + with self.enter(expr): + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.analyze_comp_for_2(expr) def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyse_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - - def analyse_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions. - That is the part after 'for' in (x for x in l if p) - """ - for index, sequence, conditions in zip(expr.indices, expr.sequences, - expr.condlists): - sequence.accept(self) + with self.enter(expr): + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: GeneratorExpr | DictionaryComprehension) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate( + zip(expr.indices, expr.sequences, expr.condlists) + ): + if i > 0: + sequence.accept(self) # Bind index variables. - self.analyse_lvalue(index) + self.analyze_lvalue(index) for cond in conditions: cond.accept(self) - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyse_function(expr) + def analyze_comp_for_2(self, expr: GeneratorExpr | DictionaryComprehension) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_lambda_expr(self, expr: LambdaExpr) -> None: + self.analyze_arg_initializers(expr) + self.analyze_function_body(expr) def visit_conditional_expr(self, expr: ConditionalExpr) -> None: expr.if_expr.accept(self) expr.cond.accept(self) expr.else_expr.accept(self) - def visit_ducktype_expr(self, expr: DucktypeExpr) -> None: - expr.type = self.anal_type(expr.type) + def visit__promote_expr(self, expr: PromoteExpr) -> None: + analyzed = self.anal_type(expr.type) + if analyzed is not None: + assert isinstance(analyzed, ProperType), "Cannot use type aliases for promotions" + expr.type = analyzed + + def visit_yield_expr(self, e: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail('"yield" outside function', e, serious=True, blocker=True) + elif self.scope_stack[-1] == SCOPE_COMPREHENSION: + self.fail( + '"yield" inside comprehension or generator expression', + e, + serious=True, + blocker=True, + ) + elif self.function_stack[-1].is_coroutine: + self.function_stack[-1].is_generator = True + self.function_stack[-1].is_async_generator = True + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_await_expr(self, expr: AwaitExpr) -> None: + if not self.is_func_scope() or not self.function_stack: + # We check both because is_function_scope() returns True inside comprehensions. + # This is not a blocker, because some enviroments (like ipython) + # support top level awaits. + self.fail('"await" outside function', expr, serious=True, code=codes.TOP_LEVEL_AWAIT) + elif not self.function_stack[-1].is_coroutine: + self.fail( + '"await" outside coroutine ("async def")', + expr, + serious=True, + code=codes.AWAIT_NOT_ASYNC, + ) + expr.expr.accept(self) + + # + # Patterns + # + + def visit_as_pattern(self, p: AsPattern) -> None: + if p.pattern is not None: + p.pattern.accept(self) + if p.name is not None: + self.analyze_lvalue(p.name) + + def visit_or_pattern(self, p: OrPattern) -> None: + for pattern in p.patterns: + pattern.accept(self) + + def visit_value_pattern(self, p: ValuePattern) -> None: + p.expr.accept(self) + + def visit_sequence_pattern(self, p: SequencePattern) -> None: + for pattern in p.patterns: + pattern.accept(self) + + def visit_starred_pattern(self, p: StarredPattern) -> None: + if p.capture is not None: + self.analyze_lvalue(p.capture) - def visit_disjointclass_expr(self, expr: DisjointclassExpr) -> None: - expr.cls.accept(self) + def visit_mapping_pattern(self, p: MappingPattern) -> None: + for key in p.keys: + key.accept(self) + for value in p.values: + value.accept(self) + if p.rest is not None: + self.analyze_lvalue(p.rest) + + def visit_class_pattern(self, p: ClassPattern) -> None: + p.class_ref.accept(self) + for pos in p.positionals: + pos.accept(self) + for v in p.keyword_values: + v.accept(self) # - # Helpers + # Lookup functions # - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" + def lookup( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + """Look up an unqualified (no dots) name in all active namespaces. + + Note that the result may contain a PlaceholderNode. The caller may + want to defer in that case. + + Generate an error if the name is not defined unless suppress_errors + is true or the current namespace is incomplete. In the latter case + defer. + """ + implicit_name = False # 1a. Name declared using 'global x' takes precedence if name in self.global_decls[-1]: if name in self.globals: return self.globals[name] - else: + if not suppress_errors: self.name_not_defined(name, ctx) - return None + return None # 1b. Name declared using 'nonlocal x' takes precedence if name in self.nonlocal_decls[-1]: for table in reversed(self.locals[:-1]): if table is not None and name in table: return table[name] - else: + if not suppress_errors: self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] + return None + # 2a. Class attributes (if within class definition) + if self.type and not self.is_func_scope() and name in self.type.names: + node = self.type.names[name] + if not node.implicit: + if self.is_active_symbol_in_class_body(node.node): + return node + else: + # Defined through self.x assignment + implicit_name = True + implicit_node = node + # 2b. Class attributes __qualname__ and __module__ + if self.type and not self.is_func_scope() and name in {"__qualname__", "__module__"}: + return SymbolTableNode(MDEF, Var(name, self.str_type())) # 3. Local (function) scopes for table in reversed(self.locals): if table is not None and name in table: return table[name] + # 4. Current file global scope if name in self.globals: return self.globals[name] # 5. Builtins - b = self.globals.get('__builtins__', None) + b = self.globals.get("__builtins__", None) if b: - table = cast(MypyFile, b.node).names + assert isinstance(b.node, MypyFile) + table = b.node.names if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) + if len(name) > 1 and name[0] == "_" and name[1] != "_": + if not suppress_errors: + self.name_not_defined(name, ctx) return None node = table[name] - # Only succeed if we are not using a type alias such List -- these must be - # be accessed via the typing module. - if node.node.name() == name: - return node + return node # Give up. - self.name_not_defined(name, ctx) + if not implicit_name and not suppress_errors: + self.name_not_defined(name, ctx) + else: + if implicit_name: + return implicit_node return None - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - n = cast(TypeInfo, n.node).get(parts[i]) - elif isinstance(n.node, MypyFile): - n = cast(MypyFile, n.node).names.get(parts[i], None) - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n + def is_active_symbol_in_class_body(self, node: SymbolNode | None) -> bool: + """Can a symbol defined in class body accessed at current statement? - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) + Only allow access to class attributes textually after + the definition, so that it's possible to fall back to the + outer scope. Example: - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. + class X: ... - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. + class C: + X = X # Initializer refers to outer scope + + Nested classes are an exception, since we want to support + arbitrary forward references in type annotations. Also, we + allow forward references to type aliases to support recursive + types. + """ + # TODO: Forward reference to name imported in class body is not + # caught. + if self.statement is None: + # Assume it's fine -- don't have enough context to check + return True + return ( + node is None + or self.is_textually_before_statement(node) + or not self.is_defined_in_current_module(node.fullname) + or isinstance(node, (TypeInfo, TypeAlias)) + or (isinstance(node, PlaceholderNode) and node.becomes_typeinfo) + ) + + def is_textually_before_statement(self, node: SymbolNode) -> bool: + """Check if a node is defined textually before the current statement + + Note that decorated functions' line number are the same as + the top decorator. """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] + assert self.statement + line_diff = self.statement.line - node.line - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n + # The first branch handles reference an overloaded function variant inside itself, + # this is a corner case where mypy technically deviates from runtime name resolution, + # but it is fine because we want an overloaded function to be treated as a single unit. + if self.is_overloaded_item(node, self.statement): + return False + elif isinstance(node, Decorator) and not node.is_overload: + return line_diff > len(node.original_decorators) + else: + return line_diff > 0 + + def is_overloaded_item(self, node: SymbolNode, statement: Statement) -> bool: + """Check whether the function belongs to the overloaded variants""" + if isinstance(node, OverloadedFuncDef) and isinstance(statement, FuncDef): + in_items = statement in { + item.func if isinstance(item, Decorator) else item for item in node.items + } + in_impl = node.impl is not None and ( + (isinstance(node.impl, Decorator) and statement is node.impl.func) + or statement is node.impl + ) + return in_items or in_impl + return False - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) + def is_defined_in_current_module(self, fullname: str | None) -> bool: + if not fullname: + return False + return module_prefix(self.modules, fullname) == self.cur_mod_id - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + """Lookup a qualified name in all activate namespaces. - def is_func_scope(self) -> bool: - return self.locals[-1] is not None + Note that the result may contain a PlaceholderNode. The caller may + want to defer in that case. - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() + Generate an error if the name is not defined unless suppress_errors + is true or the current namespace is incomplete. In the latter case + defer. + """ + if "." not in name: + # Simple case: look up a short name. + return self.lookup(name, ctx, suppress_errors=suppress_errors) + parts = name.split(".") + namespace = self.cur_mod_id + sym = self.lookup(parts[0], ctx, suppress_errors=suppress_errors) + if sym: + for i in range(1, len(parts)): + node = sym.node + part = parts[i] + if isinstance(node, TypeInfo): + nextsym = node.get(part) + elif isinstance(node, MypyFile): + nextsym = self.get_module_symbol(node, part) + namespace = node.fullname + elif isinstance(node, PlaceholderNode): + return sym + elif isinstance(node, TypeAlias) and node.no_args: + assert isinstance(node.target, ProperType) + if isinstance(node.target, Instance): + nextsym = node.target.type.get(part) + else: + nextsym = None + else: + if isinstance(node, Var): + typ = get_proper_type(node.type) + if isinstance(typ, AnyType): + # Allow access through Var with Any type without error. + return self.implicit_symbol(sym, name, parts[i:], typ) + # This might be something like valid `P.args` or invalid `P.__bound__` access. + # Important note that `ParamSpecExpr` is also ignored in other places. + # See https://github.com/python/mypy/pull/13468 + if isinstance(node, ParamSpecExpr) and part in ("args", "kwargs"): + return None + # Lookup through invalid node, such as variable or function + nextsym = None + if not nextsym or nextsym.module_hidden: + if not suppress_errors: + self.name_not_defined(name, ctx, namespace=namespace) + return None + sym = nextsym + return sym - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - if name in self.globals and (not isinstance(node.node, MypyFile) or - self.globals[name].node != node.node): - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - self.name_already_defined(name, context) - self.globals[name] = node + def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None: + try: + t = self.expr_to_unanalyzed_type(expr) + except TypeTranslationError: + return None + if isinstance(t, UnboundType): + n = self.lookup_qualified(t.name, expr, suppress_errors=True) + return n + return None - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, v: Var, ctx: Context) -> None: - if v.name() in self.locals[-1]: - self.name_already_defined(v.name(), ctx) - v._fullname = v.name() - self.locals[-1][v.name()] = SymbolTableNode(LDEF, v) - - def add_local_func(self, defn: FuncBase, ctx: Context) -> None: - # TODO combine with above - if defn.name() in self.locals[-1]: - self.name_already_defined(defn.name(), ctx) - self.locals[-1][defn.name()] = SymbolTableNode(LDEF, defn) - - def check_no_global(self, n: str, ctx: Context, - is_func: bool = False) -> None: - if n in self.globals: - if is_func and isinstance(self.globals[n].node, FuncDef): - self.fail(("Name '{}' already defined (overload variants " - "must be next to each other)").format(n), ctx) - else: - self.name_already_defined(n, ctx) + def get_module_symbol(self, node: MypyFile, name: str) -> SymbolTableNode | None: + """Look up a symbol from a module. - def name_not_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' is not defined".format(name), ctx) + Return None if no matching symbol could be bound. + """ + module = node.fullname + names = node.names + sym = names.get(name) + if not sym: + fullname = module + "." + name + if fullname in self.modules: + sym = SymbolTableNode(GDEF, self.modules[fullname]) + elif self.is_incomplete_namespace(module): + self.record_incomplete_ref() + elif "__getattr__" in names: + gvar = self.create_getattr_var(names["__getattr__"], name, fullname) + if gvar: + sym = SymbolTableNode(GDEF, gvar) + elif self.is_missing_module(fullname): + # We use the fullname of the original definition so that we can + # detect whether two names refer to the same thing. + var_type = AnyType(TypeOfAny.from_unimported_type) + v = Var(name, type=var_type) + v._fullname = fullname + sym = SymbolTableNode(GDEF, v) + elif sym.module_hidden: + sym = None + return sym + + def is_missing_module(self, module: str) -> bool: + return module in self.missing_modules + + def implicit_symbol( + self, sym: SymbolTableNode, name: str, parts: list[str], source_type: AnyType + ) -> SymbolTableNode: + """Create symbol for a qualified name reference through Any type.""" + if sym.node is None: + basename = None + else: + basename = sym.node.fullname + if basename is None: + fullname = name + else: + fullname = basename + "." + ".".join(parts) + var_type = AnyType(TypeOfAny.from_another_any, source_type) + var = Var(parts[-1], var_type) + var._fullname = fullname + return SymbolTableNode(GDEF, var) + + def create_getattr_var( + self, getattr_defn: SymbolTableNode, name: str, fullname: str + ) -> Var | None: + """Create a dummy variable using module-level __getattr__ return type. + + If not possible, return None. + + Note that multiple Var nodes can be created for a single name. We + can use the from_module_getattr and the fullname attributes to + check if two dummy Var nodes refer to the same thing. Reusing Var + nodes would require non-local mutable state, which we prefer to + avoid. + """ + if isinstance(getattr_defn.node, (FuncDef, Var)): + node_type = get_proper_type(getattr_defn.node.type) + if isinstance(node_type, CallableType): + typ = node_type.ret_type + else: + typ = AnyType(TypeOfAny.from_error) + v = Var(name, type=typ) + v._fullname = fullname + v.from_module_getattr = True + return v + return None - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) + def lookup_fully_qualified(self, fullname: str) -> SymbolTableNode: + ret = self.lookup_fully_qualified_or_none(fullname) + assert ret is not None, fullname + return ret - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) + def lookup_fully_qualified_or_none(self, fullname: str) -> SymbolTableNode | None: + """Lookup a fully qualified name that refers to a module-level definition. + Don't assume that the name is defined. This happens in the global namespace -- + the local module namespace is ignored. This does not dereference indirect + refs. -class FirstPass(NodeVisitor): - """First phase of semantic analysis""" + Note that this can't be used for names nested in class namespaces. + """ + # TODO: unify/clean-up/simplify lookup methods, see #4157. + module, name = fullname.rsplit(".", maxsplit=1) + + if module in self.modules: + # If the module exists, look up the name in the module. + # This is the common case. + filenode = self.modules[module] + result = filenode.names.get(name) + if result is None and self.is_incomplete_namespace(module): + # TODO: More explicit handling of incomplete refs? + self.record_incomplete_ref() + return result + else: + # Else, try to find the longest prefix of the module name that is in the modules dictionary. + splitted_modules = fullname.split(".") + names = [] - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.pyversion + while splitted_modules and ".".join(splitted_modules) not in self.modules: + names.append(splitted_modules.pop()) - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. + if not splitted_modules or not names: + # If no module or name is found, return None. + return None - Resolve the full names of definitions not nested within functions and - construct type info structures, but do not resolve inter-definition - references such as base classes. + # Reverse the names list to get the correct order of names. + names.reverse() - Also add implicit definitions such as __name__. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] + module = ".".join(splitted_modules) + filenode = self.modules[module] + result = filenode.names.get(names[0]) - defs = file.defs + if result is None and self.is_incomplete_namespace(module): + # TODO: More explicit handling of incomplete refs? + self.record_incomplete_ref() - # Add implicit definitions of module '__name__' etc. - for n in implicit_module_attrs: - name_def = VarDef([Var(n, AnyType())], True) - defs.insert(0, name_def) + for part in names[1:]: + if result is not None and isinstance(result.node, TypeInfo): + result = result.node.names.get(part) + else: + return None + return result - for d in defs: - d.accept(self) + def object_type(self) -> Instance: + return self.named_type("builtins.object") + + def str_type(self) -> Instance: + return self.named_type("builtins.str") + + def named_type(self, fullname: str, args: list[Type] | None = None) -> Instance: + sym = self.lookup_fully_qualified(fullname) + assert sym, "Internal error: attempted to construct unknown type" + node = sym.node + assert isinstance(node, TypeInfo) + if args: + # TODO: assert len(args) == len(node.defn.type_vars) + return Instance(node, args) + return Instance(node, [AnyType(TypeOfAny.special_form)] * len(node.defn.type_vars)) + + def named_type_or_none(self, fullname: str, args: list[Type] | None = None) -> Instance | None: + sym = self.lookup_fully_qualified_or_none(fullname) + if not sym or isinstance(sym.node, PlaceholderNode): + return None + node = sym.node + if isinstance(node, TypeAlias): + assert isinstance(node.target, Instance) # type: ignore[misc] + node = node.target.type + assert isinstance(node, TypeInfo), node + if args is not None: + # TODO: assert len(args) == len(node.defn.type_vars) + return Instance(node, args) + return Instance(node, [AnyType(TypeOfAny.unannotated)] * len(node.defn.type_vars)) - # Add implicit definition of 'None' to builtins, as we cannot define a - # variable with a None type explicitly. - if mod_id == 'builtins': - none_def = VarDef([Var('None', NoneTyp())], True) - defs.append(none_def) - none_def.accept(self) + def builtin_type(self, fully_qualified_name: str) -> Instance: + """Legacy function -- use named_type() instead.""" + return self.named_type(fully_qualified_name) + + def lookup_current_scope(self, name: str) -> SymbolTableNode | None: + if self.locals[-1] is not None: + return self.locals[-1].get(name) + elif self.type is not None: + return self.type.names.get(name) + else: + return self.globals.get(name) - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 + # + # Adding symbols + # - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.sem.analyse_lvalue(lval, add_global=True, - explicit_type=s.type is not None) - - def visit_func_def(self, d: FuncDef) -> None: - sem = self.sem - d.is_conditional = sem.block_depth[-1] > 0 - if d.name() in sem.globals: - n = sem.globals[d.name()].node - if sem.is_conditional_func(n, d): - # Conditional function definition -- multiple defs are ok. - d.original_def = cast(FuncDef, n) + def add_symbol( + self, + name: str, + node: SymbolNode, + context: Context, + module_public: bool = True, + module_hidden: bool = False, + can_defer: bool = True, + escape_comprehensions: bool = False, + no_progress: bool = False, + type_param: bool = False, + ) -> bool: + """Add symbol to the currently active symbol table. + + Generally additions to symbol table should go through this method or + one of the methods below so that kinds, redefinitions, conditional + definitions, and skipped names are handled consistently. + + Return True if we actually added the symbol, or False if we refused to do so + (because something is not ready). + + If can_defer is True, defer current target if adding a placeholder. + """ + if self.is_func_scope(): + kind = LDEF + elif self.type is not None: + kind = MDEF + else: + kind = GDEF + symbol = SymbolTableNode( + kind, node, module_public=module_public, module_hidden=module_hidden + ) + return self.add_symbol_table_node( + name, symbol, context, can_defer, escape_comprehensions, no_progress, type_param + ) + + def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None: + """Same as above, but skipping the local namespace. + + This doesn't check for previous definition and is only used + for serialization of method-level classes. + + Classes defined within methods can be exposed through an + attribute type, but method-level symbol tables aren't serialized. + This method can be used to add such classes to an enclosing, + serialized symbol table. + """ + # TODO: currently this is only used by named tuples and typed dicts. + # Use this method also by normal classes, see issue #6422. + if self.type is not None: + names = self.type.names + kind = MDEF + else: + names = self.globals + kind = GDEF + symbol = SymbolTableNode(kind, node) + names[name] = symbol + + def add_symbol_table_node( + self, + name: str, + symbol: SymbolTableNode, + context: Context | None = None, + can_defer: bool = True, + escape_comprehensions: bool = False, + no_progress: bool = False, + type_param: bool = False, + ) -> bool: + """Add symbol table node to the currently active symbol table. + + Return True if we actually added the symbol, or False if we refused + to do so (because something is not ready or it was a no-op). + + Generate an error if there is an invalid redefinition. + + If context is None, unconditionally add node, since we can't report + an error. Note that this is used by plugins to forcibly replace nodes! + + TODO: Prevent plugins from replacing nodes, as it could cause problems? + + Args: + name: short name of symbol + symbol: Node to add + can_defer: if True, defer current target if adding a placeholder + context: error context (see above about None value) + """ + names = self.current_symbol_table( + escape_comprehensions=escape_comprehensions, type_param=type_param + ) + existing = names.get(name) + if isinstance(symbol.node, PlaceholderNode) and can_defer: + if context is not None: + self.process_placeholder(name, "name", context) else: - sem.check_no_global(d.name(), d, True) - d._fullname = sem.qualified_name(d.name()) - sem.globals[d.name()] = SymbolTableNode(GDEF, d, sem.cur_mod_id) - - def visit_overloaded_func_def(self, d: OverloadedFuncDef) -> None: - self.sem.check_no_global(d.name(), d) - d._fullname = self.sem.qualified_name(d.name()) - self.sem.globals[d.name()] = SymbolTableNode(GDEF, d, - self.sem.cur_mod_id) - - def visit_class_def(self, d: ClassDef) -> None: - self.sem.check_no_global(d.name, d) - d.fullname = self.sem.qualified_name(d.name) - info = TypeInfo(SymbolTable(), d) - info.set_line(d.line) - d.info = info - self.sem.globals[d.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - - def visit_var_def(self, d: VarDef) -> None: - for v in d.items: - self.sem.check_no_global(v.name(), d) - v._fullname = self.sem.qualified_name(v.name()) - self.sem.globals[v.name()] = SymbolTableNode(GDEF, v, - self.sem.cur_mod_id) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.sem.analyse_lvalue(s.index, add_global=True) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.name: - if n: - self.sem.analyse_lvalue(n, add_global=True) + # see note in docstring describing None contexts + self.defer() + if ( + existing is not None + and context is not None + and not is_valid_replacement(existing, symbol) + ): + # There is an existing node, so this may be a redefinition. + # If the new node points to the same node as the old one, + # or if both old and new nodes are placeholders, we don't + # need to do anything. + old = existing.node + new = symbol.node + if isinstance(new, PlaceholderNode): + # We don't know whether this is okay. Let's wait until the next iteration. + return False + if not is_same_symbol(old, new): + if isinstance(new, (FuncDef, Decorator, OverloadedFuncDef, TypeInfo)): + self.add_redefinition(names, name, symbol) + if not (isinstance(new, (FuncDef, Decorator)) and self.set_original_def(old, new)): + self.name_already_defined(name, context, existing) + elif name not in self.missing_names[-1] and "*" not in self.missing_names[-1]: + names[name] = symbol + if not no_progress: + self.progress = True + return True + return False - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + def add_redefinition(self, names: SymbolTable, name: str, symbol: SymbolTableNode) -> None: + """Add a symbol table node that reflects a redefinition as a function or a class. - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) + Redefinitions need to be added to the symbol table so that they can be found + through AST traversal, but they have dummy names of form 'name-redefinition[N]', + where N ranges over 2, 3, ... (omitted for the first redefinition). - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) + Note: we always store redefinitions independently of whether they are valid or not + (so they will be semantically analyzed), the caller should give an error for invalid + redefinitions (such as e.g. variable redefined as a class). + """ + i = 1 + # Don't serialize redefined nodes. They are likely to have + # busted internal references which can cause problems with + # serialization and they can't have any external references to + # them. + symbol.no_serialize = True + while True: + if i == 1: + new_name = f"{name}-redefinition" + else: + new_name = f"{name}-redefinition{i}" + existing = names.get(new_name) + if existing is None: + names[new_name] = symbol + return + elif existing.node is symbol.node: + # Already there + return + i += 1 + + def add_local(self, node: Var | FuncDef | OverloadedFuncDef, context: Context) -> None: + """Add local variable or function.""" + assert self.is_func_scope() + name = node.name + node._fullname = name + self.add_symbol(name, node, context) + + def _get_node_for_class_scoped_import( + self, name: str, symbol_node: SymbolNode | None, context: Context + ) -> SymbolNode | None: + if symbol_node is None: + return None + # I promise this type checks; I'm just making mypyc issues go away. + # mypyc is absolutely convinced that `symbol_node` narrows to a Var in the following, + # when it can also be a FuncBase. Once fixed, `f` in the following can be removed. + # See also https://github.com/mypyc/mypyc/issues/892 + f: Callable[[object], Any] = lambda x: x + if isinstance(f(symbol_node), (Decorator, FuncBase, Var)): + # For imports in class scope, we construct a new node to represent the symbol and + # set its `info` attribute to `self.type`. + existing = self.current_symbol_table().get(name) + if ( + # The redefinition checks in `add_symbol_table_node` don't work for our + # constructed Var / FuncBase, so check for possible redefinitions here. + existing is not None + and isinstance(f(existing.node), (Decorator, FuncBase, Var)) + and ( + isinstance(f(existing.type), f(AnyType)) + or f(existing.type) == f(symbol_node).type + ) + ): + return existing.node + + # Construct the new node + if isinstance(f(symbol_node), (FuncBase, Decorator)): + # In theory we could construct a new node here as well, but in practice + # it doesn't work well, see #12197 + typ: Type | None = AnyType(TypeOfAny.from_error) + self.fail("Unsupported class scoped import", context) + else: + typ = f(symbol_node).type + symbol_node = Var(name, typ) + symbol_node._fullname = self.qualified_name(name) + assert self.type is not None # guaranteed by is_class_scope + symbol_node.info = self.type + symbol_node.line = context.line + symbol_node.column = context.column + return symbol_node + + def add_imported_symbol( + self, + name: str, + node: SymbolTableNode, + context: ImportBase, + module_public: bool, + module_hidden: bool, + ) -> None: + """Add an alias to an existing symbol through import.""" + assert not module_hidden or not module_public + + existing_symbol = self.lookup_current_scope(name) + if ( + existing_symbol + and not isinstance(existing_symbol.node, PlaceholderNode) + and not isinstance(node.node, PlaceholderNode) + ): + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name(name, existing_symbol, node, context): + return + symbol_node: SymbolNode | None = node.node -class ThirdPass(TraverserVisitor[None]): - """The third and final pass of semantic analysis. + if self.is_class_scope(): + symbol_node = self._get_node_for_class_scoped_import(name, symbol_node, context) + + symbol = SymbolTableNode( + node.kind, symbol_node, module_public=module_public, module_hidden=module_hidden + ) + self.add_symbol_table_node(name, symbol, context) + + def add_unknown_imported_symbol( + self, + name: str, + context: Context, + target_name: str | None, + module_public: bool, + module_hidden: bool, + ) -> None: + """Add symbol that we don't know what it points to because resolving an import failed. + + This can happen if a module is missing, or it is present, but doesn't have + the imported attribute. The `target_name` is the name of symbol in the namespace + it is imported from. For example, for 'from mod import x as y' the target_name is + 'mod.x'. This is currently used only to track logical dependencies. + """ + existing = self.current_symbol_table().get(name) + if existing and isinstance(existing.node, Var) and existing.node.is_suppressed_import: + # This missing import was already added -- nothing to do here. + return + var = Var(name) + if self.options.logical_deps and target_name is not None: + # This makes it possible to add logical fine-grained dependencies + # from a missing module. We can't use this by default, since in a + # few places we assume that the full name points to a real + # definition, but this name may point to nothing. + var._fullname = target_name + elif self.type: + var._fullname = self.type.fullname + "." + name + var.info = self.type + else: + var._fullname = self.qualified_name(name) + var.is_ready = True + any_type = AnyType(TypeOfAny.from_unimported_type, missing_import_name=var._fullname) + var.type = any_type + var.is_suppressed_import = True + self.add_symbol( + name, var, context, module_public=module_public, module_hidden=module_hidden + ) - Check type argument counts and values of generic types. Also update - TypeInfo disjointclass information. - """ + # + # Other helpers + # - def __init__(self, errors: Errors) -> None: - self.errors = errors + @contextmanager + def tvar_scope_frame(self, frame: TypeVarLikeScope) -> Iterator[None]: + old_scope = self.tvar_scope + self.tvar_scope = frame + yield + self.tvar_scope = old_scope + + def defer(self, debug_context: Context | None = None, force_progress: bool = False) -> None: + """Defer current analysis target to be analyzed again. + + This must be called if something in the current target is + incomplete or has a placeholder node. However, this must *not* + be called during the final analysis iteration! Instead, an error + should be generated. Often 'process_placeholder' is a good + way to either defer or generate an error. + + NOTE: Some methods, such as 'anal_type', 'mark_incomplete' and + 'record_incomplete_ref', call this implicitly, or when needed. + They are usually preferable to a direct defer() call. + """ + assert not self.final_iteration, "Must not defer during final iteration" + if force_progress: + # Usually, we report progress if we have replaced a placeholder node + # with an actual valid node. However, sometimes we need to update an + # existing node *in-place*. For example, this is used by type aliases + # in context of forward references and/or recursive aliases, and in + # similar situations (recursive named tuples etc). + self.progress = True + self.deferred = True + # Store debug info for this deferral. + line = ( + debug_context.line if debug_context else self.statement.line if self.statement else -1 + ) + self.deferral_debug_context.append((self.cur_mod_id, line)) + + def track_incomplete_refs(self) -> Tag: + """Return tag that can be used for tracking references to incomplete names.""" + return self.num_incomplete_refs + + def found_incomplete_ref(self, tag: Tag) -> bool: + """Have we encountered an incomplete reference since starting tracking?""" + return self.num_incomplete_refs != tag + + def record_incomplete_ref(self) -> None: + """Record the encounter of an incomplete reference and defer current analysis target.""" + self.defer() + self.num_incomplete_refs += 1 + + def mark_incomplete( + self, + name: str, + node: Node, + becomes_typeinfo: bool = False, + module_public: bool = True, + module_hidden: bool = False, + ) -> None: + """Mark a definition as incomplete (and defer current analysis target). + + Also potentially mark the current namespace as incomplete. + + Args: + name: The name that we weren't able to define (or '*' if the name is unknown) + node: The node that refers to the name (definition or lvalue) + becomes_typeinfo: Pass this to PlaceholderNode (used by special forms like + named tuples that will create TypeInfos). + """ + self.defer(node) + if name == "*": + self.incomplete = True + elif not self.is_global_or_nonlocal(name): + fullname = self.qualified_name(name) + assert self.statement + placeholder = PlaceholderNode( + fullname, node, self.statement.line, becomes_typeinfo=becomes_typeinfo + ) + self.add_symbol( + name, + placeholder, + module_public=module_public, + module_hidden=module_hidden, + context=dummy_context(), + ) + self.missing_names[-1].add(name) + + def is_incomplete_namespace(self, fullname: str) -> bool: + """Is a module or class namespace potentially missing some definitions? + + If a name is missing from an incomplete namespace, we'll need to defer the + current analysis target. + """ + return fullname in self.incomplete_namespaces - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - file_node.accept(self) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - info = tdef.info - # Collect declared disjoint classes from all base classes. - for base in info.mro: - for disjoint in base.disjoint_classes: - if disjoint not in info.disjoint_classes: - info.disjoint_classes.append(disjoint) - for subtype in disjoint.all_subtypes(): - if info not in subtype.disjoint_classes: - subtype.disjoint_classes.append(info) - super().visit_class_def(tdef) + def process_placeholder( + self, name: str | None, kind: str, ctx: Context, force_progress: bool = False + ) -> None: + """Process a reference targeting placeholder node. - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) + If this is not a final iteration, defer current node, + otherwise report an error. - def visit_undefined_expr(self, e: UndefinedExpr) -> None: - self.analyze(e.type) + The 'kind' argument indicates if this a name or attribute expression + (used for better error message). + """ + if self.final_iteration: + self.cannot_resolve_name(name, kind, ctx) + else: + self.defer(ctx, force_progress=force_progress) - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) + def cannot_resolve_name(self, name: str | None, kind: str, ctx: Context) -> None: + name_format = f' "{name}"' if name else "" + self.fail(f"Cannot resolve {kind}{name_format} (possible cyclic definition)", ctx) + if self.is_func_scope(): + self.note("Recursive types are not allowed at function scope", ctx) - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) + def qualified_name(self, name: str) -> str: + if self.type is not None: + return self.type._fullname + "." + name + elif self.is_func_scope(): + return name + else: + return self.cur_mod_id + "." + name + + @contextmanager + def enter( + self, function: FuncItem | GeneratorExpr | DictionaryComprehension + ) -> Iterator[None]: + """Enter a function, generator or comprehension scope.""" + names = self.saved_locals.setdefault(function, SymbolTable()) + self.locals.append(names) + is_comprehension = isinstance(function, (GeneratorExpr, DictionaryComprehension)) + self.scope_stack.append(SCOPE_FUNC if not is_comprehension else SCOPE_COMPREHENSION) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + self.loop_depth.append(0) + self.missing_names.append(set()) + try: + yield + finally: + self.locals.pop() + self.scope_stack.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + self.loop_depth.pop() + self.missing_names.pop() - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) + def is_func_scope(self) -> bool: + scope_type = self.scope_stack[-1] + if scope_type == SCOPE_ANNOTATION: + scope_type = self.scope_stack[-2] + return scope_type in (SCOPE_FUNC, SCOPE_COMPREHENSION) - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) + def is_nested_within_func_scope(self) -> bool: + """Are we underneath a function scope, even if we are in a nested class also?""" + return any(s in (SCOPE_FUNC, SCOPE_COMPREHENSION) for s in self.scope_stack) + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = List[Type]() - for i in range(len(typ.type_vars)): - tv.append(TypeVar(typ.type_vars[i], i + 1, - typ.defn.type_vars[i].values, - typ.defn.type_vars[i].upper_bound)) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - else: - return TupleType(typ.tuple_type.items, inst) + def is_module_scope(self) -> bool: + return not (self.is_class_scope() or self.is_func_scope()) + def current_symbol_kind(self) -> int: + if self.is_class_scope(): + kind = MDEF + elif self.is_func_scope(): + kind = LDEF + else: + kind = GDEF + return kind + + def current_symbol_table( + self, escape_comprehensions: bool = False, type_param: bool = False + ) -> SymbolTable: + if type_param and self.scope_stack[-1] == SCOPE_ANNOTATION: + n = self.locals[-1] + assert n is not None + return n + elif self.is_func_scope(): + if self.scope_stack[-1] == SCOPE_ANNOTATION: + n = self.locals[-2] + else: + n = self.locals[-1] + assert n is not None + if escape_comprehensions: + assert len(self.locals) == len(self.scope_stack) + # Retrieve the symbol table from the enclosing non-comprehension scope. + for i, scope_type in enumerate(reversed(self.scope_stack)): + if scope_type != SCOPE_COMPREHENSION: + if i == len(self.locals) - 1: # The last iteration. + # The caller of the comprehension is in the global space. + names = self.globals + else: + names_candidate = self.locals[-1 - i] + assert ( + names_candidate is not None + ), "Escaping comprehension from invalid scope" + names = names_candidate + break + else: + assert False, "Should have at least one non-comprehension scope" + else: + names = n + assert names is not None + elif self.type is not None: + names = self.type.names + else: + names = self.globals + return names + + def is_global_or_nonlocal(self, name: str) -> bool: + return self.is_func_scope() and ( + name in self.global_decls[-1] or name in self.nonlocal_decls[-1] + ) + + def add_exports(self, exp_or_exps: Iterable[Expression] | Expression) -> None: + exps = [exp_or_exps] if isinstance(exp_or_exps, Expression) else exp_or_exps + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.append(exp.value) + + def name_not_defined(self, name: str, ctx: Context, namespace: str | None = None) -> None: + incomplete = self.is_incomplete_namespace(namespace or self.cur_mod_id) + if ( + namespace is None + and self.type + and not self.is_func_scope() + and self.incomplete_type_stack + and self.incomplete_type_stack[-1] + and not self.final_iteration + ): + # We are processing a class body for the first time, so it is incomplete. + incomplete = True + if incomplete: + # Target namespace is incomplete, so it's possible that the name will be defined + # later on. Defer current target. + self.record_incomplete_ref() + return + message = f'Name "{name}" is not defined' + self.fail(message, ctx, code=codes.NAME_DEFINED) + + if f"builtins.{name}" in SUGGESTED_TEST_FIXTURES: + # The user probably has a missing definition in a test fixture. Let's verify. + fullname = f"builtins.{name}" + if self.lookup_fully_qualified_or_none(fullname) is None: + # Yes. Generate a helpful note. + self.msg.add_fixture_note(fullname, ctx) + + modules_with_unimported_hints = { + name.split(".", 1)[0] for name in TYPES_FOR_UNIMPORTED_HINTS + } + lowercased = {name.lower(): name for name in TYPES_FOR_UNIMPORTED_HINTS} + for module in modules_with_unimported_hints: + fullname = f"{module}.{name}".lower() + if fullname not in lowercased: + continue + # User probably forgot to import these types. + hint = ( + 'Did you forget to import it from "{module}"?' + ' (Suggestion: "from {module} import {name}")' + ).format(module=module, name=lowercased[fullname].rsplit(".", 1)[-1]) + self.note(hint, ctx, code=codes.NAME_DEFINED) + + def already_defined( + self, name: str, ctx: Context, original_ctx: SymbolTableNode | SymbolNode | None, noun: str + ) -> None: + if isinstance(original_ctx, SymbolTableNode): + node: SymbolNode | None = original_ctx.node + elif isinstance(original_ctx, SymbolNode): + node = original_ctx + else: + node = None + + if isinstance(original_ctx, SymbolTableNode) and isinstance(original_ctx.node, MypyFile): + # Since this is an import, original_ctx.node points to the module definition. + # Therefore its line number is always 1, which is not useful for this + # error message. + extra_msg = " (by an import)" + elif node and node.line != -1 and self.is_local_name(node.fullname): + # TODO: Using previous symbol node may give wrong line. We should use + # the line number where the binding was established instead. + extra_msg = f" on line {node.line}" + else: + extra_msg = " (possibly by an import)" + self.fail( + f'{noun} "{unmangle(name)}" already defined{extra_msg}', ctx, code=codes.NO_REDEF + ) + + def name_already_defined( + self, name: str, ctx: Context, original_ctx: SymbolTableNode | SymbolNode | None = None + ) -> None: + self.already_defined(name, ctx, original_ctx, noun="Name") + + def attribute_already_defined( + self, name: str, ctx: Context, original_ctx: SymbolTableNode | SymbolNode | None = None + ) -> None: + self.already_defined(name, ctx, original_ctx, noun="Attribute") + + def is_local_name(self, name: str) -> bool: + """Does name look like reference to a definition in the current module?""" + return self.is_defined_in_current_module(name) or "." not in name + + def in_checked_function(self) -> bool: + """Should we type-check the current function? + + - Yes if --check-untyped-defs is set. + - Yes outside functions. + - Yes in annotated functions. + - No otherwise. + """ + if self.options.check_untyped_defs or not self.function_stack: + return True + + current_index = len(self.function_stack) - 1 + while current_index >= 0: + current_func = self.function_stack[current_index] + if not isinstance(current_func, LambdaExpr): + return not current_func.is_dynamic() + + # Special case, `lambda` inherits the "checked" state from its parent. + # Because `lambda` itself cannot be annotated. + # `lambdas` can be deeply nested, so we try to find at least one other parent. + current_index -= 1 + + # This means that we only have a stack of `lambda` functions, + # no regular functions. + return True -@overload -def replace_implicit_first_type(sig: Callable, new: Type) -> Callable: - # We can detect implicit self type by it having no representation. - if not sig.arg_types[0].repr: - return replace_leading_arg_type(sig, new) - else: - return sig + def fail( + self, + msg: str | ErrorMessage, + ctx: Context, + serious: bool = False, + *, + code: ErrorCode | None = None, + blocker: bool = False, + ) -> None: + if not serious and not self.in_checked_function(): + return + # In case it's a bug and we don't really have context + assert ctx is not None, msg + if isinstance(msg, ErrorMessage): + if code is None: + code = msg.code + msg = msg.value + self.errors.report(ctx.line, ctx.column, msg, blocker=blocker, code=code) + + def note(self, msg: str, ctx: Context, code: ErrorCode | None = None) -> None: + if not self.in_checked_function(): + return + self.errors.report(ctx.line, ctx.column, msg, severity="note", code=code) + + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + if feature not in self.options.enable_incomplete_feature: + self.fail( + f'"{feature}" support is experimental,' + f" use --enable-incomplete-feature={feature} to enable", + ctx, + ) + return False + return True + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line, self.errors, self.options) + + def expr_to_analyzed_type( + self, + expr: Expression, + report_invalid_types: bool = True, + allow_placeholder: bool = False, + allow_type_any: bool = False, + allow_unbound_tvars: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + ) -> Type | None: + if isinstance(expr, CallExpr): + # This is a legacy syntax intended mostly for Python 2, we keep it for + # backwards compatibility, but new features like generic named tuples + # and recursive named tuples will be not supported. + expr.accept(self) + internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( + expr, None, self.is_func_scope() + ) + if tvar_defs: + self.fail("Generic named tuples are not supported for legacy class syntax", expr) + self.note("Use either Python 3 class syntax, or the assignment syntax", expr) + if internal_name is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + elif not info: + self.defer(expr) + return None + assert info.tuple_type, "NamedTuple without tuple type" + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = self.expr_to_unanalyzed_type(expr) + return self.anal_type( + typ, + report_invalid_types=report_invalid_types, + allow_placeholder=allow_placeholder, + allow_type_any=allow_type_any, + allow_unbound_tvars=allow_unbound_tvars, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + ) + + def analyze_type_expr(self, expr: Expression) -> None: + # There are certain expressions that mypy does not need to semantically analyze, + # since they analyzed solely as type. (For example, indexes in type alias definitions + # and base classes in class defs). External consumers of the mypy AST may need + # them semantically analyzed, however, if they need to treat it as an expression + # and not a type. (Which is to say, mypyc needs to do this.) Do the analysis + # in a fresh tvar scope in order to suppress any errors about using type variables. + with self.tvar_scope_frame(TypeVarLikeScope()), self.allow_unbound_tvars_set(): + expr.accept(self) + + def type_analyzer( + self, + *, + tvar_scope: TypeVarLikeScope | None = None, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + allow_placeholder: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_types: bool = True, + prohibit_self_type: str | None = None, + allow_type_any: bool = False, + ) -> TypeAnalyser: + if tvar_scope is None: + tvar_scope = self.tvar_scope + tpan = TypeAnalyser( + self, + tvar_scope, + self.plugin, + self.options, + self.cur_mod_node, + self.is_typeshed_stub_file, + allow_unbound_tvars=allow_unbound_tvars, + allow_tuple_literal=allow_tuple_literal, + report_invalid_types=report_invalid_types, + allow_placeholder=allow_placeholder, + allow_typed_dict_special_forms=allow_typed_dict_special_forms, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + prohibit_self_type=prohibit_self_type, + allow_type_any=allow_type_any, + ) + tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) + tpan.global_scope = not self.type and not self.function_stack + return tpan + + def expr_to_unanalyzed_type(self, node: Expression, allow_unpack: bool = False) -> ProperType: + return expr_to_unanalyzed_type( + node, self.options, self.is_stub_file, allow_unpack=allow_unpack + ) + + def anal_type( + self, + typ: Type, + *, + tvar_scope: TypeVarLikeScope | None = None, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + allow_placeholder: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_types: bool = True, + prohibit_self_type: str | None = None, + allow_type_any: bool = False, + ) -> Type | None: + """Semantically analyze a type. + + Args: + typ: Type to analyze (if already analyzed, this is a no-op) + allow_placeholder: If True, may return PlaceholderType if + encountering an incomplete definition + + Return None only if some part of the type couldn't be bound *and* it + referred to an incomplete namespace or definition. In this case also + defer as needed. During a final iteration this won't return None; + instead report an error if the type can't be analyzed and return + AnyType. + + In case of other errors, report an error message and return AnyType. + + NOTE: The caller shouldn't defer even if this returns None or a + placeholder type. + """ + has_self_type = find_self_type( + typ, lambda name: self.lookup_qualified(name, typ, suppress_errors=True) + ) + if has_self_type and self.type and prohibit_self_type is None: + self.setup_self_type() + a = self.type_analyzer( + tvar_scope=tvar_scope, + allow_unbound_tvars=allow_unbound_tvars, + allow_tuple_literal=allow_tuple_literal, + allow_placeholder=allow_placeholder, + allow_typed_dict_special_forms=allow_typed_dict_special_forms, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + report_invalid_types=report_invalid_types, + prohibit_self_type=prohibit_self_type, + allow_type_any=allow_type_any, + ) + tag = self.track_incomplete_refs() + typ = typ.accept(a) + if self.found_incomplete_ref(tag): + # Something could not be bound yet. + return None + self.add_type_alias_deps(a.aliases_used) + return typ + + def class_type(self, self_type: Type) -> Type: + return TypeType.make_normalized(self_type) + + def schedule_patch(self, priority: int, patch: Callable[[], None]) -> None: + self.patches.append((priority, patch)) + + def report_hang(self) -> None: + print("Deferral trace:") + for mod, line in self.deferral_debug_context: + print(f" {mod}:{line}") + self.errors.report( + -1, + -1, + "INTERNAL ERROR: maximum semantic analysis iteration count reached", + blocker=True, + ) + + def add_plugin_dependency(self, trigger: str, target: str | None = None) -> None: + """Add dependency from trigger to a target. + + If the target is not given explicitly, use the current target. + """ + if target is None: + target = self.scope.current_target() + self.cur_mod_node.plugin_deps.setdefault(trigger, set()).add(target) -@overload -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - osig = cast(Overloaded, sig) - return Overloaded([replace_implicit_first_type(i, new) - for i in osig.items()]) + def add_type_alias_deps( + self, aliases_used: Collection[str], target: str | None = None + ) -> None: + """Add full names of type aliases on which the current node depends. + This is used by fine-grained incremental mode to re-check the corresponding nodes. + If `target` is None, then the target node used will be the current scope. + """ + if not aliases_used: + # A basic optimization to avoid adding targets with no dependencies to + # the `alias_deps` dict. + return + if target is None: + target = self.scope.current_target() + self.cur_mod_node.alias_deps[target].update(aliases_used) + + def is_mangled_global(self, name: str) -> bool: + # A global is mangled if there exists at least one renamed variant. + return unmangle(name) + "'" in self.globals + + def is_initial_mangled_global(self, name: str) -> bool: + # If there are renamed definitions for a global, the first one has exactly one prime. + return name == unmangle(name) + "'" + + def parse_bool(self, expr: Expression) -> bool | None: + # This wrapper is preserved for plugins. + return parse_bool(expr) + + def parse_str_literal(self, expr: Expression) -> str | None: + """Attempt to find the string literal value of the given expression. Returns `None` if no + literal value can be found.""" + if isinstance(expr, StrExpr): + return expr.value + if isinstance(expr, RefExpr) and isinstance(expr.node, Var) and expr.node.type is not None: + values = try_getting_str_literals_from_type(expr.node.type) + if values is not None and len(values) == 1: + return values[0] + return None -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig + def set_future_import_flags(self, module_name: str) -> None: + if module_name in FUTURE_IMPORTS: + self.modules[self.cur_mod_id].future_import_flags.add(FUTURE_IMPORTS[module_name]) + + def is_future_flag_set(self, flag: str) -> bool: + return self.modules[self.cur_mod_id].is_future_flag_set(flag) + + def parse_dataclass_transform_spec(self, call: CallExpr) -> DataclassTransformSpec: + """Build a DataclassTransformSpec from the arguments passed to the given call to + typing.dataclass_transform.""" + parameters = DataclassTransformSpec() + for name, value in zip(call.arg_names, call.args): + # Skip any positional args. Note that any such args are invalid, but we can rely on + # typeshed to enforce this and don't need an additional error here. + if name is None: + continue + + # field_specifiers is currently the only non-boolean argument; check for it first so + # so the rest of the block can fail through to handling booleans + if name == "field_specifiers": + parameters.field_specifiers = self.parse_dataclass_transform_field_specifiers( + value + ) + continue + + boolean = require_bool_literal_argument(self, value, name) + if boolean is None: + continue + + if name == "eq_default": + parameters.eq_default = boolean + elif name == "order_default": + parameters.order_default = boolean + elif name == "kw_only_default": + parameters.kw_only_default = boolean + elif name == "frozen_default": + parameters.frozen_default = boolean + else: + self.fail(f'Unrecognized dataclass_transform parameter "{name}"', call) + return parameters -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, - RefExpr) and cast(RefExpr, node).fullname == fullname + def parse_dataclass_transform_field_specifiers(self, arg: Expression) -> tuple[str, ...]: + if not isinstance(arg, TupleExpr): + self.fail('"field_specifiers" argument must be a tuple literal', arg) + return () + names = [] + for specifier in arg.items: + if not isinstance(specifier, RefExpr): + self.fail('"field_specifiers" must only contain identifiers', specifier) + return () + names.append(specifier.fullname) + return tuple(names) -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(cast(RefExpr, node).node, (TypeInfo, FuncDef, - OverloadedFuncDef))) +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + if len(sig.arg_types) == 0: + return sig + return sig.copy_modified(arg_types=[new] + sig.arg_types[1:]) + elif isinstance(sig, Overloaded): + return Overloaded( + [cast(CallableType, replace_implicit_first_type(i, new)) for i in sig.items] + ) + else: + assert False -def expr_to_unanalyzed_type(expr: Node) -> Type: - """Translate an expression to the corresonding type. - The result is not semantically analyzed. It can be UnboundType or ListType. - Raise TypeTranslationError if the expression cannot represent a type. - """ - if isinstance(expr, NameExpr): - name = expr.name - return UnboundType(name, line=expr.line) - elif isinstance(expr, MemberExpr): - fullname = get_member_expr_fullname(expr) - if fullname: - return UnboundType(fullname, line=expr.line) - else: - raise TypeTranslationError() - elif isinstance(expr, IndexExpr): - base = expr_to_unanalyzed_type(expr.base) - if isinstance(base, UnboundType): - if base.args: - raise TypeTranslationError() - if isinstance(expr.index, TupleExpr): - args = cast(TupleExpr, expr.index).items - else: - args = [expr.index] - base.args = [expr_to_unanalyzed_type(arg) for arg in args] - return base - else: - raise TypeTranslationError() - elif isinstance(expr, ListExpr): - return TypeList([expr_to_unanalyzed_type(t) for t in expr.items], - line=expr.line) - elif isinstance(expr, StrExpr): - # Parse string literal type. - try: - result = parse_str_as_type(expr.value, expr.line) - except TypeParseError: - raise TypeTranslationError() - return result - else: - raise TypeTranslationError() +def refers_to_fullname(node: Expression, fullnames: str | tuple[str, ...]) -> bool: + """Is node a name or member expression with the given full name?""" + if not isinstance(fullnames, tuple): + fullnames = (fullnames,) + if not isinstance(node, RefExpr): + return False + if node.fullname in fullnames: + return True + if isinstance(node.node, TypeAlias): + return is_named_instance(node.node.target, fullnames) + return False -def get_member_expr_fullname(expr: MemberExpr) -> str: - """Return the qualified name represention of a member expression. - Return a string of form foo.bar, foo.bar.baz, or similar, or None if the - argument cannot be represented in this form. - """ - if isinstance(expr.expr, NameExpr): - initial = cast(NameExpr, expr.expr).name - elif isinstance(expr.expr, MemberExpr): - initial = get_member_expr_fullname(cast(MemberExpr, expr.expr)) - else: - return None - return '{}.{}'.format(initial, expr.name) +def refers_to_class_or_function(node: Expression) -> bool: + """Does semantically analyzed node refer to a class?""" + return isinstance(node, RefExpr) and isinstance( + node.node, (TypeInfo, FuncDef, OverloadedFuncDef) + ) -def find_duplicate(list: List[T]) -> T: +def find_duplicate(list: list[T]) -> T | None: """If the list has duplicates, return one of the duplicates. Otherwise, return None. @@ -2040,94 +7605,168 @@ def find_duplicate(list: List[T]) -> T: return None -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (TVAR, UNBOUND_TVAR) - node.kind = TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: +def remove_imported_names_from_symtable(names: SymbolTable, module: str) -> None: """Remove all imported names from the symbol table of a module.""" - removed = List[str]() + removed: list[str] = [] for name, node in names.items(): - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] + if node.node is None: + continue + fullname = node.node.fullname + prefix = fullname[: fullname.rfind(".")] if prefix != module: removed.append(name) for name in removed: del names[name] -def infer_reachability_of_if_statement(s: IfStmt, pyversion: int) -> None: - always_true = False - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - always_true = True - for body in s.body[i + 1:]: - mark_block_unreachable(s.body[i]) - if s.else_body: - mark_block_unreachable(s.else_body) - break +def make_any_non_explicit(t: Type) -> Type: + """Replace all Any types within in with Any that has attribute 'explicit' set to False""" + return t.accept(MakeAnyNonExplicit()) + + +class MakeAnyNonExplicit(TrivialSyntheticTypeTranslator): + def visit_any(self, t: AnyType) -> Type: + if t.type_of_any == TypeOfAny.explicit: + return t.copy_modified(TypeOfAny.special_form) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) -def infer_if_condition_value(expr: Node, pyversion: int) -> int: - """Infer whether if condition is always true/false. +def make_any_non_unimported(t: Type) -> Type: + """Replace all Any types that come from unimported types with special form Any.""" + return t.accept(MakeAnyNonUnimported()) - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. + +class MakeAnyNonUnimported(TrivialSyntheticTypeTranslator): + def visit_any(self, t: AnyType) -> Type: + if t.type_of_any == TypeOfAny.from_unimported_type: + return t.copy_modified(TypeOfAny.special_form, missing_import_name=None) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + +def apply_semantic_analyzer_patches(patches: list[tuple[int, Callable[[], None]]]) -> None: + """Call patch callbacks in the right order. + + This should happen after semantic analyzer pass 3. """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE + patches_by_priority = sorted(patches, key=lambda x: x[0]) + for priority, patch_func in patches_by_priority: + patch_func() + + +def names_modified_by_assignment(s: AssignmentStmt) -> list[NameExpr]: + """Return all unqualified (short) names assigned to in an assignment statement.""" + result: list[NameExpr] = [] + for lvalue in s.lvalues: + result += names_modified_in_lvalue(lvalue) return result -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) +def names_modified_in_lvalue(lvalue: Lvalue) -> list[NameExpr]: + """Return all NameExpr assignment targets in an Lvalue.""" + if isinstance(lvalue, NameExpr): + return [lvalue] + elif isinstance(lvalue, StarExpr): + return names_modified_in_lvalue(lvalue.expr) + elif isinstance(lvalue, (ListExpr, TupleExpr)): + result: list[NameExpr] = [] + for item in lvalue.items: + result += names_modified_in_lvalue(item) + return result + return [] + + +def is_same_var_from_getattr(n1: SymbolNode | None, n2: SymbolNode | None) -> bool: + """Do n1 and n2 refer to the same Var derived from module-level __getattr__?""" + return ( + isinstance(n1, Var) + and n1.from_module_getattr + and isinstance(n2, Var) + and n2.from_module_getattr + and n1.fullname == n2.fullname + ) + + +def dummy_context() -> Context: + return TempNode(AnyType(TypeOfAny.special_form)) + + +def is_valid_replacement(old: SymbolTableNode, new: SymbolTableNode) -> bool: + """Can symbol table node replace an existing one? + + These are the only valid cases: + + 1. Placeholder gets replaced with a non-placeholder + 2. Placeholder that isn't known to become type replaced with a + placeholder that can become a type + """ + if isinstance(old.node, PlaceholderNode): + if isinstance(new.node, PlaceholderNode): + return not old.node.becomes_typeinfo and new.node.becomes_typeinfo + else: + return True + return False + + +def is_same_symbol(a: SymbolNode | None, b: SymbolNode | None) -> bool: + return ( + a == b + or (isinstance(a, PlaceholderNode) and isinstance(b, PlaceholderNode)) + or is_same_var_from_getattr(a, b) + ) + + +def is_trivial_body(block: Block) -> bool: + """Returns 'true' if the given body is "trivial" -- if it contains just a "pass", + "..." (ellipsis), or "raise NotImplementedError()". A trivial body may also + start with a statement containing just a string (e.g. a docstring). + + Note: Functions that raise other kinds of exceptions do not count as + "trivial". We use this function to help us determine when it's ok to + relax certain checks on body, but functions that raise arbitrary exceptions + are more likely to do non-trivial work. For example: + + def halt(self, reason: str = ...) -> NoReturn: + raise MyCustomError("Fatal error: " + reason, self.line, self.context) + A function that raises just NotImplementedError is much less likely to be + this complex. -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" + Note: If you update this, you may also need to update + mypy.fastparse.is_possible_trivial_body! + """ + body = block.body + if not body: + # Functions have empty bodies only if the body is stripped or the function is + # generated or deserialized. In these cases the body is unknown. + return False + + # Skip a docstring + if isinstance(body[0], ExpressionStmt) and isinstance(body[0].expr, StrExpr): + body = block.body[1:] + + if len(body) == 0: + # There's only a docstring (or no body at all). + return True + elif len(body) > 1: + return False - def visit_import(self, node: Import) -> None: - node.is_unreachable = True + stmt = body[0] + + if isinstance(stmt, RaiseStmt): + expr = stmt.expr + if expr is None: + return False + if isinstance(expr, CallExpr): + expr = expr.callee - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True + return isinstance(expr, NameExpr) and expr.fullname == "builtins.NotImplementedError" - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True + return isinstance(stmt, PassStmt) or ( + isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr) + ) diff --git a/mypy/semanal_classprop.py b/mypy/semanal_classprop.py new file mode 100644 index 000000000000..c5ad34122f6c --- /dev/null +++ b/mypy/semanal_classprop.py @@ -0,0 +1,188 @@ +"""Calculate some properties of classes. + +These happen after semantic analysis and before type checking. +""" + +from __future__ import annotations + +from typing import Final + +from mypy.errors import Errors +from mypy.nodes import ( + IMPLICITLY_ABSTRACT, + IS_ABSTRACT, + CallExpr, + Decorator, + FuncDef, + Node, + OverloadedFuncDef, + PromoteExpr, + SymbolTable, + TypeInfo, + Var, +) +from mypy.options import Options +from mypy.types import MYPYC_NATIVE_INT_NAMES, Instance, ProperType + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS: Final = { + "builtins.int": "float", + "builtins.float": "complex", + "builtins.bytearray": "bytes", + "builtins.memoryview": "bytes", +} + + +def calculate_class_abstract_status(typ: TypeInfo, is_stub_file: bool, errors: Errors) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + Report error is required ABCMeta metaclass is missing. + """ + typ.is_abstract = False + typ.abstract_attributes = [] + if typ.typeddict_type: + return # TypedDict can't be abstract + concrete: set[str] = set() + # List of abstract attributes together with their abstract status + abstract: list[tuple[str, int]] = [] + abstract_in_this_class: list[str] = [] + if typ.is_newtype: + # Special case: NewTypes are considered as always non-abstract, so they can be used as: + # Config = NewType('Config', Mapping[str, str]) + # default = Config({'cannot': 'modify'}) # OK + return + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + if node.items: # can be empty for invalid overloads + func: Node | None = node.items[0] + else: + func = None + else: + func = node + if isinstance(func, Decorator): + func = func.func + if isinstance(func, FuncDef): + if ( + func.abstract_status in (IS_ABSTRACT, IMPLICITLY_ABSTRACT) + and name not in concrete + ): + typ.is_abstract = True + abstract.append((name, func.abstract_status)) + if base is typ: + abstract_in_this_class.append(name) + elif isinstance(node, Var): + if node.is_abstract_var and name not in concrete: + typ.is_abstract = True + abstract.append((name, IS_ABSTRACT)) + if base is typ: + abstract_in_this_class.append(name) + concrete.add(name) + # In stubs, abstract classes need to be explicitly marked because it is too + # easy to accidentally leave a concrete class abstract by forgetting to + # implement some methods. + typ.abstract_attributes = sorted(abstract) + if is_stub_file: + if typ.declared_metaclass and typ.declared_metaclass.type.has_base("abc.ABCMeta"): + return + if typ.is_protocol: + return + if abstract and not abstract_in_this_class: + + def report(message: str, severity: str) -> None: + errors.report(typ.line, typ.column, message, severity=severity) + + attrs = ", ".join(f'"{attr}"' for attr, _ in sorted(abstract)) + report(f"Class {typ.fullname} has abstract attributes {attrs}", "error") + report( + "If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass", "note" + ) + if typ.is_final and abstract: + attrs = ", ".join(f'"{attr}"' for attr, _ in sorted(abstract)) + errors.report( + typ.line, typ.column, f"Final class {typ.fullname} has abstract attributes {attrs}" + ) + + +def check_protocol_status(info: TypeInfo, errors: Errors) -> None: + """Check that all classes in MRO of a protocol are protocols""" + if info.is_protocol: + for type in info.bases: + if not type.type.is_protocol and type.type.fullname != "builtins.object": + errors.report( + info.line, + info.column, + "All bases of a protocol must be protocols", + severity="error", + ) + + +def calculate_class_vars(info: TypeInfo) -> None: + """Try to infer additional class variables. + + Subclass attribute assignments with no type annotation are assumed + to be classvar if overriding a declared classvar from the base + class. + + This must happen after the main semantic analysis pass, since + this depends on base class bodies having been fully analyzed. + """ + for name, sym in info.names.items(): + node = sym.node + if isinstance(node, Var) and node.info and node.is_inferred and not node.is_classvar: + for base in info.mro[1:]: + member = base.names.get(name) + if member is not None and isinstance(member.node, Var) and member.node.is_classvar: + node.is_classvar = True + + +def add_type_promotion( + info: TypeInfo, module_names: SymbolTable, options: Options, builtin_names: SymbolTable +) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + defn = info.defn + promote_targets: list[ProperType] = [] + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented feature). + promote_targets.append(analyzed.type) + if not promote_targets: + if defn.fullname in TYPE_PROMOTIONS: + target_sym = module_names.get(TYPE_PROMOTIONS[defn.fullname]) + if defn.fullname == "builtins.bytearray" and options.disable_bytearray_promotion: + target_sym = None + elif defn.fullname == "builtins.memoryview" and options.disable_memoryview_promotion: + target_sym = None + # With test stubs, the target may not exist. + if target_sym: + target_info = target_sym.node + assert isinstance(target_info, TypeInfo) + promote_targets.append(Instance(target_info, [])) + # Special case the promotions between 'int' and native integer types. + # These have promotions going both ways, such as from 'int' to 'i64' + # and 'i64' to 'int', for convenience. + if defn.fullname in MYPYC_NATIVE_INT_NAMES: + int_sym = builtin_names["int"] + assert isinstance(int_sym.node, TypeInfo) + int_sym.node._promote.append(Instance(defn.info, [])) + defn.info.alt_promote = Instance(int_sym.node, []) + if promote_targets: + defn.info._promote.extend(promote_targets) diff --git a/mypy/semanal_enum.py b/mypy/semanal_enum.py new file mode 100644 index 000000000000..b1e267b4c781 --- /dev/null +++ b/mypy/semanal_enum.py @@ -0,0 +1,269 @@ +"""Semantic analysis of call-based Enum definitions. + +This is conceptually part of mypy.semanal (semantic analyzer pass 2). +""" + +from __future__ import annotations + +from typing import Final, cast + +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + EXCLUDED_ENUM_ATTRIBUTES, + MDEF, + AssignmentStmt, + CallExpr, + Context, + DictExpr, + EnumCallExpr, + Expression, + ListExpr, + MemberExpr, + NameExpr, + RefExpr, + StrExpr, + SymbolTableNode, + TupleExpr, + TypeInfo, + Var, + is_StrExpr_list, +) +from mypy.options import Options +from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.types import LiteralType, get_proper_type + +# Note: 'enum.EnumMeta' is deliberately excluded from this list. Classes that directly use +# enum.EnumMeta do not necessarily automatically have the 'name' and 'value' attributes. +ENUM_BASES: Final = frozenset( + ("enum.Enum", "enum.IntEnum", "enum.Flag", "enum.IntFlag", "enum.StrEnum") +) +ENUM_SPECIAL_PROPS: Final = frozenset( + ( + "name", + "value", + "_name_", + "_value_", + *EXCLUDED_ENUM_ATTRIBUTES, + # Also attributes from `object`: + "__module__", + "__annotations__", + "__doc__", + "__slots__", + "__dict__", + ) +) + + +class EnumCallAnalyzer: + def __init__(self, options: Options, api: SemanticAnalyzerInterface) -> None: + self.options = options + self.api = api + + def process_enum_call(self, s: AssignmentStmt, is_func_scope: bool) -> bool: + """Check if s defines an Enum; if yes, store the definition in symbol table. + + Return True if this looks like an Enum definition (but maybe with errors), + otherwise return False. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): + return False + lvalue = s.lvalues[0] + name = lvalue.name + enum_call = self.check_enum_call(s.rvalue, name, is_func_scope) + if enum_call is None: + return False + if isinstance(lvalue, MemberExpr): + self.fail("Enum type as attribute is not supported", lvalue) + return False + # Yes, it's a valid Enum definition. Add it to the symbol table. + self.api.add_symbol(name, enum_call, s) + return True + + def check_enum_call( + self, node: Expression, var_name: str, is_func_scope: bool + ) -> TypeInfo | None: + """Check if a call defines an Enum. + + Example: + + A = enum.Enum('A', 'foo bar') + + is equivalent to: + + class A(enum.Enum): + foo = 1 + bar = 2 + """ + if not isinstance(node, CallExpr): + return None + call = node + callee = call.callee + if not isinstance(callee, RefExpr): + return None + fullname = callee.fullname + if fullname not in ENUM_BASES: + return None + + new_class_name, items, values, ok = self.parse_enum_call_args( + call, fullname.split(".")[-1] + ) + if not ok: + # Error. Construct dummy return value. + name = var_name + if is_func_scope: + name += "@" + str(call.line) + info = self.build_enum_call_typeinfo(name, [], fullname, node.line) + else: + if new_class_name != var_name: + msg = f'String argument 1 "{new_class_name}" to {fullname}(...) does not match variable name "{var_name}"' + self.fail(msg, call) + + name = cast(StrExpr, call.args[0]).value + if name != var_name or is_func_scope: + # Give it a unique name derived from the line number. + name += "@" + str(call.line) + info = self.build_enum_call_typeinfo(name, items, fullname, call.line) + # Store generated TypeInfo under both names, see semanal_namedtuple for more details. + if name != var_name or is_func_scope: + self.api.add_symbol_skip_local(name, info) + call.analyzed = EnumCallExpr(info, items, values) + call.analyzed.set_line(call) + info.line = node.line + return info + + def build_enum_call_typeinfo( + self, name: str, items: list[str], fullname: str, line: int + ) -> TypeInfo: + base = self.api.named_type_or_none(fullname) + assert base is not None + info = self.api.basic_new_typeinfo(name, base, line) + info.metaclass_type = info.calculate_metaclass_type() + info.is_enum = True + for item in items: + var = Var(item) + var.info = info + var.is_property = True + # When an enum is created by its functional form `Enum(name, values)` + # - if it is a string it is first split by commas/whitespace + # - if it is an iterable of single items each item is assigned a value starting at `start` + # - if it is an iterable of (name, value) then the given values will be used + # either way, each item should be treated as if it has an explicit value. + var.has_explicit_value = True + var._fullname = f"{info.fullname}.{item}" + info.names[item] = SymbolTableNode(MDEF, var) + return info + + def parse_enum_call_args( + self, call: CallExpr, class_name: str + ) -> tuple[str, list[str], list[Expression | None], bool]: + """Parse arguments of an Enum call. + + Return a tuple of fields, values, was there an error. + """ + args = call.args + if not all(arg_kind in [ARG_POS, ARG_NAMED] for arg_kind in call.arg_kinds): + return self.fail_enum_call_arg(f"Unexpected arguments to {class_name}()", call) + if len(args) < 2: + return self.fail_enum_call_arg(f"Too few arguments for {class_name}()", call) + if len(args) > 6: + return self.fail_enum_call_arg(f"Too many arguments for {class_name}()", call) + valid_name = [None, "value", "names", "module", "qualname", "type", "start"] + for arg_name in call.arg_names: + if arg_name not in valid_name: + self.fail_enum_call_arg(f'Unexpected keyword argument "{arg_name}"', call) + value, names = None, None + for arg_name, arg in zip(call.arg_names, args): + if arg_name == "value": + value = arg + if arg_name == "names": + names = arg + if value is None: + value = args[0] + if names is None: + names = args[1] + if not isinstance(value, StrExpr): + return self.fail_enum_call_arg( + f"{class_name}() expects a string literal as the first argument", call + ) + new_class_name = value.value + + items = [] + values: list[Expression | None] = [] + if isinstance(names, StrExpr): + fields = names.value + for field in fields.replace(",", " ").split(): + items.append(field) + elif isinstance(names, (TupleExpr, ListExpr)): + seq_items = names.items + if is_StrExpr_list(seq_items): + items = [seq_item.value for seq_item in seq_items] + elif all( + isinstance(seq_item, (TupleExpr, ListExpr)) + and len(seq_item.items) == 2 + and isinstance(seq_item.items[0], StrExpr) + for seq_item in seq_items + ): + for seq_item in seq_items: + assert isinstance(seq_item, (TupleExpr, ListExpr)) + name, value = seq_item.items + assert isinstance(name, StrExpr) + items.append(name.value) + values.append(value) + else: + return self.fail_enum_call_arg( + "%s() with tuple or list expects strings or (name, value) pairs" % class_name, + call, + ) + elif isinstance(names, DictExpr): + for key, value in names.items: + if not isinstance(key, StrExpr): + return self.fail_enum_call_arg( + f"{class_name}() with dict literal requires string literals", call + ) + items.append(key.value) + values.append(value) + elif isinstance(args[1], RefExpr) and isinstance(args[1].node, Var): + proper_type = get_proper_type(args[1].node.type) + if ( + proper_type is not None + and isinstance(proper_type, LiteralType) + and isinstance(proper_type.value, str) + ): + fields = proper_type.value + for field in fields.replace(",", " ").split(): + items.append(field) + elif args[1].node.is_final and isinstance(args[1].node.final_value, str): + fields = args[1].node.final_value + for field in fields.replace(",", " ").split(): + items.append(field) + else: + return self.fail_enum_call_arg( + "Second argument of %s() must be string, tuple, list or dict literal for mypy to determine Enum members" + % class_name, + call, + ) + else: + # TODO: Allow dict(x=1, y=2) as a substitute for {'x': 1, 'y': 2}? + return self.fail_enum_call_arg( + "Second argument of %s() must be string, tuple, list or dict literal for mypy to determine Enum members" + % class_name, + call, + ) + if not items: + return self.fail_enum_call_arg(f"{class_name}() needs at least one item", call) + if not values: + values = [None] * len(items) + assert len(items) == len(values) + return new_class_name, items, values, True + + def fail_enum_call_arg( + self, message: str, context: Context + ) -> tuple[str, list[str], list[Expression | None], bool]: + self.fail(message, context) + return "", [], [], False + + # Helpers + + def fail(self, msg: str, ctx: Context) -> None: + self.api.fail(msg, ctx) diff --git a/mypy/semanal_infer.py b/mypy/semanal_infer.py new file mode 100644 index 000000000000..a146b56dc2d3 --- /dev/null +++ b/mypy/semanal_infer.py @@ -0,0 +1,128 @@ +"""Simple type inference for decorated functions during semantic analysis.""" + +from __future__ import annotations + +from mypy.nodes import ARG_POS, CallExpr, Decorator, Expression, FuncDef, RefExpr, Var +from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.typeops import function_type +from mypy.types import ( + AnyType, + CallableType, + ProperType, + Type, + TypeOfAny, + TypeVarType, + get_proper_type, +) +from mypy.typevars import has_no_typevars + + +def infer_decorator_signature_if_simple( + dec: Decorator, analyzer: SemanticAnalyzerInterface +) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve additional references to decorated functions + during type checking. Otherwise the type might not be available + when we need it, since module top levels can't be deferred. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType(TypeOfAny.special_form)], + [ARG_POS], + [None], + AnyType(TypeOfAny.special_form), + analyzer.named_type("builtins.function"), + name=dec.var.name, + ) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if expr.node.type and is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, analyzer.named_type("builtins.function")) + if dec.decorators: + return_type = calculate_return_type(dec.decorators[0]) + if return_type and isinstance(return_type, AnyType): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType(TypeOfAny.from_another_any, source_any=return_type) + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decorated function. + orig_sig = function_type(dec.func, analyzer.named_type("builtins.function")) + sig.name = orig_sig.items[0].name + dec.var.type = sig + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + sig = get_proper_type(sig) + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def calculate_return_type(expr: Expression) -> ProperType | None: + """Return the return type if we can calculate it. + + This only uses information available during semantic analysis so this + will sometimes return None because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return AnyType(TypeOfAny.unannotated) + # Explicit Any return? + if isinstance(typ, CallableType): + return get_proper_type(typ.ret_type) + return None + elif isinstance(expr.node, Var): + return get_proper_type(expr.node.type) + elif isinstance(expr, CallExpr): + return calculate_return_type(expr.callee) + return None + + +def find_fixed_callable_return(expr: Expression) -> CallableType | None: + """Return the return type, if expression refers to a callable that returns a callable. + + But only do this if the return type has no type variables. Return None otherwise. + This approximates things a lot as this is supposed to be called before type checking + when full type information is not available yet. + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + ret_type = get_proper_type(typ.ret_type) + if isinstance(ret_type, CallableType): + return ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + ret_type = get_proper_type(t.ret_type) + if isinstance(ret_type, CallableType): + return ret_type + return None diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py new file mode 100644 index 000000000000..09a1223be6aa --- /dev/null +++ b/mypy/semanal_main.py @@ -0,0 +1,513 @@ +"""Top-level logic for the semantic analyzer. + +The semantic analyzer binds names, resolves imports, detects various +special constructs that don't have dedicated AST nodes after parse +(such as 'cast' which looks like a call), populates symbol tables, and +performs various simple consistency checks. + +Semantic analysis of each SCC (strongly connected component; import +cycle) is performed in one unit. Each module is analyzed as multiple +separate *targets*; the module top level is one target and each function +is a target. Nested functions are not separate targets, however. This is +mostly identical to targets used by mypy daemon (but classes aren't +targets in semantic analysis). + +We first analyze each module top level in an SCC. If we encounter some +names that we can't bind because the target of the name may not have +been processed yet, we *defer* the current target for further +processing. Deferred targets will be analyzed additional times until +everything can be bound, or we reach a maximum number of iterations. + +We keep track of a set of incomplete namespaces, i.e. namespaces that we +haven't finished populating yet. References to these namespaces cause a +deferral if they can't be satisfied. Initially every module in the SCC +will be incomplete. +""" + +from __future__ import annotations + +from contextlib import nullcontext +from typing import TYPE_CHECKING, Callable, Final, List, Optional, Tuple, Union +from typing_extensions import TypeAlias as _TypeAlias + +import mypy.build +import mypy.state +from mypy.checker import FineGrainedDeferredNode +from mypy.errors import Errors +from mypy.nodes import Decorator, FuncDef, MypyFile, OverloadedFuncDef, TypeInfo, Var +from mypy.options import Options +from mypy.plugin import ClassDefContext +from mypy.plugins import dataclasses as dataclasses_plugin +from mypy.semanal import ( + SemanticAnalyzer, + apply_semantic_analyzer_patches, + remove_imported_names_from_symtable, +) +from mypy.semanal_classprop import ( + add_type_promotion, + calculate_class_abstract_status, + calculate_class_vars, + check_protocol_status, +) +from mypy.semanal_infer import infer_decorator_signature_if_simple +from mypy.semanal_shared import find_dataclass_transform_spec +from mypy.semanal_typeargs import TypeArgumentAnalyzer +from mypy.server.aststrip import SavedAttributes +from mypy.util import is_typeshed_file + +if TYPE_CHECKING: + from mypy.build import Graph, State + + +Patches: _TypeAlias = List[Tuple[int, Callable[[], None]]] + + +# If we perform this many iterations, raise an exception since we are likely stuck. +MAX_ITERATIONS: Final = 20 + + +# Number of passes over core modules before going on to the rest of the builtin SCC. +CORE_WARMUP: Final = 2 +core_modules: Final = [ + "typing", + "_collections_abc", + "builtins", + "abc", + "collections", + "collections.abc", +] + + +def semantic_analysis_for_scc(graph: Graph, scc: list[str], errors: Errors) -> None: + """Perform semantic analysis for all modules in a SCC (import cycle). + + Assume that reachability analysis has already been performed. + + The scc will be processed roughly in the order the modules are included + in the list. + """ + patches: Patches = [] + # Note that functions can't define new module-level attributes + # using 'global x', since module top levels are fully processed + # before functions. This limitation is unlikely to go away soon. + process_top_levels(graph, scc, patches) + process_functions(graph, scc, patches) + # We use patch callbacks to fix up things when we expect relatively few + # callbacks to be required. + apply_semantic_analyzer_patches(patches) + # Run class decorator hooks (they requite complete MROs and no placeholders). + apply_class_plugin_hooks(graph, scc, errors) + # This pass might need fallbacks calculated above and the results of hooks. + check_type_arguments(graph, scc, errors) + calculate_class_properties(graph, scc, errors) + check_blockers(graph, scc) + # Clean-up builtins, so that TypeVar etc. are not accessible without importing. + if "builtins" in scc: + cleanup_builtin_scc(graph["builtins"]) + + +def cleanup_builtin_scc(state: State) -> None: + """Remove imported names from builtins namespace. + + This way names imported from typing in builtins.pyi aren't available + by default (without importing them). We can only do this after processing + the whole SCC is finished, when the imported names aren't needed for + processing builtins.pyi itself. + """ + assert state.tree is not None + remove_imported_names_from_symtable(state.tree.names, "builtins") + + +def semantic_analysis_for_targets( + state: State, nodes: list[FineGrainedDeferredNode], graph: Graph, saved_attrs: SavedAttributes +) -> None: + """Semantically analyze only selected nodes in a given module. + + This essentially mirrors the logic of semantic_analysis_for_scc() + except that we process only some targets. This is used in fine grained + incremental mode, when propagating an update. + + The saved_attrs are implicitly declared instance attributes (attributes + defined on self) removed by AST stripper that may need to be reintroduced + here. They must be added before any methods are analyzed. + """ + patches: Patches = [] + if any(isinstance(n.node, MypyFile) for n in nodes): + # Process module top level first (if needed). + process_top_levels(graph, [state.id], patches) + restore_saved_attrs(saved_attrs) + analyzer = state.manager.semantic_analyzer + for n in nodes: + if isinstance(n.node, MypyFile): + # Already done above. + continue + process_top_level_function( + analyzer, state, state.id, n.node.fullname, n.node, n.active_typeinfo, patches + ) + apply_semantic_analyzer_patches(patches) + apply_class_plugin_hooks(graph, [state.id], state.manager.errors) + check_type_arguments_in_targets(nodes, state, state.manager.errors) + calculate_class_properties(graph, [state.id], state.manager.errors) + + +def restore_saved_attrs(saved_attrs: SavedAttributes) -> None: + """Restore instance variables removed during AST strip that haven't been added yet.""" + for (cdef, name), sym in saved_attrs.items(): + info = cdef.info + existing = info.get(name) + defined_in_this_class = name in info.names + assert isinstance(sym.node, Var) + # This needs to mimic the logic in SemanticAnalyzer.analyze_member_lvalue() + # regarding the existing variable in class body or in a superclass: + # If the attribute of self is not defined in superclasses, create a new Var. + if ( + existing is None + or + # (An abstract Var is considered as not defined.) + (isinstance(existing.node, Var) and existing.node.is_abstract_var) + or + # Also an explicit declaration on self creates a new Var unless + # there is already one defined in the class body. + sym.node.explicit_self_type + and not defined_in_this_class + ): + info.names[name] = sym + + +def process_top_levels(graph: Graph, scc: list[str], patches: Patches) -> None: + # Process top levels until everything has been bound. + + # Reverse order of the scc so the first modules in the original list will be + # be processed first. This helps with performance. + scc = list(reversed(scc)) + + # Initialize ASTs and symbol tables. + for id in scc: + state = graph[id] + assert state.tree is not None + state.manager.semantic_analyzer.prepare_file(state.tree) + + # Initially all namespaces in the SCC are incomplete (well they are empty). + state.manager.incomplete_namespaces.update(scc) + + worklist = scc.copy() + # HACK: process core stuff first. This is mostly needed to support defining + # named tuples in builtin SCC. + if all(m in worklist for m in core_modules): + worklist += list(reversed(core_modules)) * CORE_WARMUP + final_iteration = False + iteration = 0 + analyzer = state.manager.semantic_analyzer + analyzer.deferral_debug_context.clear() + + while worklist: + iteration += 1 + if iteration > MAX_ITERATIONS: + # Just pick some module inside the current SCC for error context. + assert state.tree is not None + with analyzer.file_context(state.tree, state.options): + analyzer.report_hang() + break + if final_iteration: + # Give up. It's impossible to bind all names. + state.manager.incomplete_namespaces.clear() + all_deferred: list[str] = [] + any_progress = False + while worklist: + next_id = worklist.pop() + state = graph[next_id] + assert state.tree is not None + deferred, incomplete, progress = semantic_analyze_target( + next_id, next_id, state, state.tree, None, final_iteration, patches + ) + all_deferred += deferred + any_progress = any_progress or progress + if not incomplete: + state.manager.incomplete_namespaces.discard(next_id) + if final_iteration: + assert not all_deferred, "Must not defer during final iteration" + # Reverse to process the targets in the same order on every iteration. This avoids + # processing the same target twice in a row, which is inefficient. + worklist = list(reversed(all_deferred)) + final_iteration = not any_progress + + +def process_functions(graph: Graph, scc: list[str], patches: Patches) -> None: + # Process functions. + for module in scc: + tree = graph[module].tree + assert tree is not None + analyzer = graph[module].manager.semantic_analyzer + # In principle, functions can be processed in arbitrary order, + # but _methods_ must be processed in the order they are defined, + # because some features (most notably partial types) depend on + # order of definitions on self. + # + # There can be multiple generated methods per line. Use target + # name as the second sort key to get a repeatable sort order on + # Python 3.5, which doesn't preserve dictionary order. + targets = sorted(get_all_leaf_targets(tree), key=lambda x: (x[1].line, x[0])) + for target, node, active_type in targets: + assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator)) + process_top_level_function( + analyzer, graph[module], module, target, node, active_type, patches + ) + + +def process_top_level_function( + analyzer: SemanticAnalyzer, + state: State, + module: str, + target: str, + node: FuncDef | OverloadedFuncDef | Decorator, + active_type: TypeInfo | None, + patches: Patches, +) -> None: + """Analyze single top-level function or method. + + Process the body of the function (including nested functions) again and again, + until all names have been resolved (or iteration limit reached). + """ + # We need one more iteration after incomplete is False (e.g. to report errors, if any). + final_iteration = False + incomplete = True + # Start in the incomplete state (no missing names will be reported on first pass). + # Note that we use module name, since functions don't create qualified names. + deferred = [module] + analyzer.deferral_debug_context.clear() + analyzer.incomplete_namespaces.add(module) + iteration = 0 + while deferred: + iteration += 1 + if iteration == MAX_ITERATIONS: + # Just pick some module inside the current SCC for error context. + assert state.tree is not None + with analyzer.file_context(state.tree, state.options): + analyzer.report_hang() + break + if not (deferred or incomplete) or final_iteration: + # OK, this is one last pass, now missing names will be reported. + analyzer.incomplete_namespaces.discard(module) + deferred, incomplete, progress = semantic_analyze_target( + target, module, state, node, active_type, final_iteration, patches + ) + if not incomplete: + state.manager.incomplete_namespaces.discard(module) + if final_iteration: + assert not deferred, "Must not defer during final iteration" + if not progress: + final_iteration = True + + analyzer.incomplete_namespaces.discard(module) + # After semantic analysis is done, discard local namespaces + # to avoid memory hoarding. + analyzer.saved_locals.clear() + + +TargetInfo: _TypeAlias = Tuple[ + str, Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator], Optional[TypeInfo] +] + + +def get_all_leaf_targets(file: MypyFile) -> list[TargetInfo]: + """Return all leaf targets in a symbol table (module-level and methods).""" + result: list[TargetInfo] = [] + for fullname, node, active_type in file.local_definitions(): + if isinstance(node.node, (FuncDef, OverloadedFuncDef, Decorator)): + result.append((fullname, node.node, active_type)) + return result + + +def semantic_analyze_target( + target: str, + module: str, + state: State, + node: MypyFile | FuncDef | OverloadedFuncDef | Decorator, + active_type: TypeInfo | None, + final_iteration: bool, + patches: Patches, +) -> tuple[list[str], bool, bool]: + """Semantically analyze a single target. + + Return tuple with these items: + - list of deferred targets + - was some definition incomplete (need to run another pass) + - were any new names defined (or placeholders replaced) + """ + state.manager.processed_targets.append((module, target)) + tree = state.tree + assert tree is not None + analyzer = state.manager.semantic_analyzer + # TODO: Move initialization to somewhere else + analyzer.global_decls = [set()] + analyzer.nonlocal_decls = [set()] + analyzer.globals = tree.names + analyzer.progress = False + with state.wrap_context(check_blockers=False): + refresh_node = node + if isinstance(refresh_node, Decorator): + # Decorator expressions will be processed as part of the module top level. + refresh_node = refresh_node.func + analyzer.refresh_partial( + refresh_node, + patches, + final_iteration, + file_node=tree, + options=state.options, + active_type=active_type, + ) + if isinstance(node, Decorator): + infer_decorator_signature_if_simple(node, analyzer) + for dep in analyzer.imports: + state.add_dependency(dep) + priority = mypy.build.PRI_LOW + if priority <= state.priorities.get(dep, priority): + state.priorities[dep] = priority + + # Clear out some stale data to avoid memory leaks and astmerge + # validity check confusion + analyzer.statement = None + del analyzer.cur_mod_node + + if analyzer.deferred: + return [target], analyzer.incomplete, analyzer.progress + else: + return [], analyzer.incomplete, analyzer.progress + + +def check_type_arguments(graph: Graph, scc: list[str], errors: Errors) -> None: + for module in scc: + state = graph[module] + assert state.tree + analyzer = TypeArgumentAnalyzer( + errors, + state.options, + state.tree.is_typeshed_file(state.options), + state.manager.semantic_analyzer.named_type, + ) + with state.wrap_context(): + with mypy.state.state.strict_optional_set(state.options.strict_optional): + state.tree.accept(analyzer) + + +def check_type_arguments_in_targets( + targets: list[FineGrainedDeferredNode], state: State, errors: Errors +) -> None: + """Check type arguments against type variable bounds and restrictions. + + This mirrors the logic in check_type_arguments() except that we process only + some targets. This is used in fine grained incremental mode. + """ + analyzer = TypeArgumentAnalyzer( + errors, + state.options, + is_typeshed_file(state.options.abs_custom_typeshed_dir, state.path or ""), + state.manager.semantic_analyzer.named_type, + ) + with state.wrap_context(): + with mypy.state.state.strict_optional_set(state.options.strict_optional): + for target in targets: + func: FuncDef | OverloadedFuncDef | None = None + if isinstance(target.node, (FuncDef, OverloadedFuncDef)): + func = target.node + saved = (state.id, target.active_typeinfo, func) # module, class, function + with errors.scope.saved_scope(saved) if errors.scope else nullcontext(): + analyzer.recurse_into_functions = func is not None + target.node.accept(analyzer) + + +def apply_class_plugin_hooks(graph: Graph, scc: list[str], errors: Errors) -> None: + """Apply class plugin hooks within a SCC. + + We run these after to the main semantic analysis so that the hooks + don't need to deal with incomplete definitions such as placeholder + types. + + Note that some hooks incorrectly run during the main semantic + analysis pass, for historical reasons. + """ + num_passes = 0 + incomplete = True + # If we encounter a base class that has not been processed, we'll run another + # pass. This should eventually reach a fixed point. + while incomplete: + assert num_passes < 10, "Internal error: too many class plugin hook passes" + num_passes += 1 + incomplete = False + for module in scc: + state = graph[module] + tree = state.tree + assert tree + for _, node, _ in tree.local_definitions(): + if isinstance(node.node, TypeInfo): + if not apply_hooks_to_class( + state.manager.semantic_analyzer, + module, + node.node, + state.options, + tree, + errors, + ): + incomplete = True + + +def apply_hooks_to_class( + self: SemanticAnalyzer, + module: str, + info: TypeInfo, + options: Options, + file_node: MypyFile, + errors: Errors, +) -> bool: + # TODO: Move more class-related hooks here? + defn = info.defn + ok = True + for decorator in defn.decorators: + with self.file_context(file_node, options, info): + hook = None + + decorator_name = self.get_fullname_for_hook(decorator) + if decorator_name: + hook = self.plugin.get_class_decorator_hook_2(decorator_name) + # Special case: if the decorator is itself decorated with + # typing.dataclass_transform, apply the hook for the dataclasses plugin + # TODO: remove special casing here + if hook is None and find_dataclass_transform_spec(decorator): + hook = dataclasses_plugin.dataclass_class_maker_callback + + if hook: + ok = ok and hook(ClassDefContext(defn, decorator, self)) + + # Check if the class definition itself triggers a dataclass transform (via a parent class/ + # metaclass) + spec = find_dataclass_transform_spec(info) + if spec is not None: + with self.file_context(file_node, options, info): + # We can't use the normal hook because reason = defn, and ClassDefContext only accepts + # an Expression for reason + ok = ok and dataclasses_plugin.DataclassTransformer(defn, defn, spec, self).transform() + + return ok + + +def calculate_class_properties(graph: Graph, scc: list[str], errors: Errors) -> None: + builtins = graph["builtins"].tree + assert builtins + for module in scc: + state = graph[module] + tree = state.tree + assert tree + for _, node, _ in tree.local_definitions(): + if isinstance(node.node, TypeInfo): + with state.manager.semantic_analyzer.file_context(tree, state.options, node.node): + calculate_class_abstract_status(node.node, tree.is_stub, errors) + check_protocol_status(node.node, errors) + calculate_class_vars(node.node) + add_type_promotion( + node.node, tree.names, graph[module].options, builtins.names + ) + + +def check_blockers(graph: Graph, scc: list[str]) -> None: + for module in scc: + graph[module].check_blockers() diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py new file mode 100644 index 000000000000..7c6da7721e8f --- /dev/null +++ b/mypy/semanal_namedtuple.py @@ -0,0 +1,718 @@ +"""Semantic analysis of named tuple definitions. + +This is conceptually part of mypy.semanal. +""" + +from __future__ import annotations + +import keyword +from contextlib import contextmanager +from typing import Container, Final, Iterator, List, Mapping, cast + +from mypy.errorcodes import ARG_TYPE, ErrorCode +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.messages import MessageBuilder +from mypy.nodes import ( + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + MDEF, + Argument, + AssignmentStmt, + Block, + CallExpr, + ClassDef, + Context, + Decorator, + EllipsisExpr, + Expression, + ExpressionStmt, + FuncBase, + FuncDef, + ListExpr, + NamedTupleExpr, + NameExpr, + PassStmt, + RefExpr, + Statement, + StrExpr, + SymbolTable, + SymbolTableNode, + TempNode, + TupleExpr, + TypeInfo, + TypeVarExpr, + Var, + is_StrExpr_list, +) +from mypy.options import Options +from mypy.semanal_shared import ( + PRIORITY_FALLBACKS, + SemanticAnalyzerInterface, + calculate_tuple_fallback, + has_placeholder, + set_callable_name, +) +from mypy.types import ( + TYPED_NAMEDTUPLE_NAMES, + AnyType, + CallableType, + LiteralType, + TupleType, + Type, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarType, + UnboundType, + has_type_vars, +) +from mypy.util import get_unique_redefinition_name + +# Matches "_prohibited" in typing.py, but adds __annotations__, which works at runtime but can't +# easily be supported in a static checker. +NAMEDTUPLE_PROHIBITED_NAMES: Final = ( + "__new__", + "__init__", + "__slots__", + "__getnewargs__", + "_fields", + "_field_defaults", + "_field_types", + "_make", + "_replace", + "_asdict", + "_source", + "__annotations__", +) + +NAMEDTUP_CLASS_ERROR: Final = ( + 'Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]"' +) + +SELF_TVAR_NAME: Final = "_NT" + + +class NamedTupleAnalyzer: + def __init__( + self, options: Options, api: SemanticAnalyzerInterface, msg: MessageBuilder + ) -> None: + self.options = options + self.api = api + self.msg = msg + + def analyze_namedtuple_classdef( + self, defn: ClassDef, is_stub_file: bool, is_func_scope: bool + ) -> tuple[bool, TypeInfo | None]: + """Analyze if given class definition can be a named tuple definition. + + Return a tuple where first item indicates whether this can possibly be a named tuple, + and the second item is the corresponding TypeInfo (may be None if not ready and should be + deferred). + """ + for base_expr in defn.base_type_exprs: + if isinstance(base_expr, RefExpr): + self.api.accept(base_expr) + if base_expr.fullname in TYPED_NAMEDTUPLE_NAMES: + result = self.check_namedtuple_classdef(defn, is_stub_file) + if result is None: + # This is a valid named tuple, but some types are incomplete. + return True, None + items, types, default_items, statements = result + if is_func_scope and "@" not in defn.name: + defn.name += "@" + str(defn.line) + existing_info = None + if isinstance(defn.analyzed, NamedTupleExpr): + existing_info = defn.analyzed.info + info = self.build_namedtuple_typeinfo( + defn.name, items, types, default_items, defn.line, existing_info + ) + defn.analyzed = NamedTupleExpr(info, is_typed=True) + defn.analyzed.line = defn.line + defn.analyzed.column = defn.column + defn.defs.body = statements + # All done: this is a valid named tuple with all types known. + return True, info + # This can't be a valid named tuple. + return False, None + + def check_namedtuple_classdef( + self, defn: ClassDef, is_stub_file: bool + ) -> tuple[list[str], list[Type], dict[str, Expression], list[Statement]] | None: + """Parse and validate fields in named tuple class definition. + + Return a four tuple: + * field names + * field types + * field default values + * valid statements + or None, if any of the types are not ready. + """ + if len(defn.base_type_exprs) > 1: + self.fail("NamedTuple should be a single base", defn) + items: list[str] = [] + types: list[Type] = [] + default_items: dict[str, Expression] = {} + statements: list[Statement] = [] + for stmt in defn.defs.body: + statements.append(stmt) + if not isinstance(stmt, AssignmentStmt): + # Still allow pass or ... (for empty namedtuples). + if isinstance(stmt, PassStmt) or ( + isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr) + ): + continue + # Also allow methods, including decorated ones. + if isinstance(stmt, (Decorator, FuncBase)): + continue + # And docstrings. + if isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, StrExpr): + continue + statements.pop() + defn.removed_statements.append(stmt) + self.fail(NAMEDTUP_CLASS_ERROR, stmt) + elif len(stmt.lvalues) > 1 or not isinstance(stmt.lvalues[0], NameExpr): + # An assignment, but an invalid one. + statements.pop() + defn.removed_statements.append(stmt) + self.fail(NAMEDTUP_CLASS_ERROR, stmt) + else: + # Append name and type in this case... + name = stmt.lvalues[0].name + items.append(name) + if stmt.type is None: + types.append(AnyType(TypeOfAny.unannotated)) + else: + # We never allow recursive types at function scope. Although it is + # possible to support this for named tuples, it is still tricky, and + # it would be inconsistent with type aliases. + analyzed = self.api.anal_type( + stmt.type, + allow_placeholder=not self.api.is_func_scope(), + prohibit_self_type="NamedTuple item type", + ) + if analyzed is None: + # Something is incomplete. We need to defer this named tuple. + return None + types.append(analyzed) + # ...despite possible minor failures that allow further analyzis. + if name.startswith("_"): + self.fail( + f"NamedTuple field name cannot start with an underscore: {name}", stmt + ) + if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax: + self.fail(NAMEDTUP_CLASS_ERROR, stmt) + elif isinstance(stmt.rvalue, TempNode): + # x: int assigns rvalue to TempNode(AnyType()) + if default_items: + self.fail( + "Non-default NamedTuple fields cannot follow default fields", stmt + ) + else: + default_items[name] = stmt.rvalue + if defn.keywords: + for_function = ' for "__init_subclass__" of "NamedTuple"' + for key in defn.keywords: + self.msg.unexpected_keyword_argument_for_function(for_function, key, defn) + return items, types, default_items, statements + + def check_namedtuple( + self, node: Expression, var_name: str | None, is_func_scope: bool + ) -> tuple[str | None, TypeInfo | None, list[TypeVarLikeType]]: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + Return a tuple of two items: + * Internal name of the named tuple (e.g. the name passed as an argument to namedtuple) + or None if it is not a valid named tuple + * Corresponding TypeInfo, or None if not ready. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None, None, [] + call = node + callee = call.callee + if not isinstance(callee, RefExpr): + return None, None, [] + fullname = callee.fullname + if fullname == "collections.namedtuple": + is_typed = False + elif fullname in TYPED_NAMEDTUPLE_NAMES: + is_typed = True + else: + return None, None, [] + result = self.parse_namedtuple_args(call, fullname) + if result: + items, types, defaults, typename, tvar_defs, ok = result + else: + # Error. Construct dummy return value. + if var_name: + name = var_name + if is_func_scope: + name += "@" + str(call.line) + else: + name = var_name = "namedtuple@" + str(call.line) + info = self.build_namedtuple_typeinfo(name, [], [], {}, node.line, None) + self.store_namedtuple_info(info, var_name, call, is_typed) + if name != var_name or is_func_scope: + # NOTE: we skip local namespaces since they are not serialized. + self.api.add_symbol_skip_local(name, info) + return var_name, info, [] + if not ok: + # This is a valid named tuple but some types are not ready. + return typename, None, [] + + # We use the variable name as the class name if it exists. If + # it doesn't, we use the name passed as an argument. We prefer + # the variable name because it should be unique inside a + # module, and so we don't need to disambiguate it with a line + # number. + if var_name: + name = var_name + else: + name = typename + + if var_name is None or is_func_scope: + # There are two special cases where need to give it a unique name derived + # from the line number: + # * This is a base class expression, since it often matches the class name: + # class NT(NamedTuple('NT', [...])): + # ... + # * This is a local (function or method level) named tuple, since + # two methods of a class can define a named tuple with the same name, + # and they will be stored in the same namespace (see below). + name += "@" + str(call.line) + if defaults: + default_items = { + arg_name: default for arg_name, default in zip(items[-len(defaults) :], defaults) + } + else: + default_items = {} + + existing_info = None + if isinstance(node.analyzed, NamedTupleExpr): + existing_info = node.analyzed.info + info = self.build_namedtuple_typeinfo( + name, items, types, default_items, node.line, existing_info + ) + + # If var_name is not None (i.e. this is not a base class expression), we always + # store the generated TypeInfo under var_name in the current scope, so that + # other definitions can use it. + if var_name: + self.store_namedtuple_info(info, var_name, call, is_typed) + else: + call.analyzed = NamedTupleExpr(info, is_typed=is_typed) + call.analyzed.set_line(call) + # There are three cases where we need to store the generated TypeInfo + # second time (for the purpose of serialization): + # * If there is a name mismatch like One = NamedTuple('Other', [...]) + # we also store the info under name 'Other@lineno', this is needed + # because classes are (de)serialized using their actual fullname, not + # the name of l.h.s. + # * If this is a method level named tuple. It can leak from the method + # via assignment to self attribute and therefore needs to be serialized + # (local namespaces are not serialized). + # * If it is a base class expression. It was not stored above, since + # there is no var_name (but it still needs to be serialized + # since it is in MRO of some class). + if name != var_name or is_func_scope: + # NOTE: we skip local namespaces since they are not serialized. + self.api.add_symbol_skip_local(name, info) + return typename, info, tvar_defs + + def store_namedtuple_info( + self, info: TypeInfo, name: str, call: CallExpr, is_typed: bool + ) -> None: + self.api.add_symbol(name, info, call) + call.analyzed = NamedTupleExpr(info, is_typed=is_typed) + call.analyzed.set_line(call) + + def parse_namedtuple_args( + self, call: CallExpr, fullname: str + ) -> None | (tuple[list[str], list[Type], list[Expression], str, list[TypeVarLikeType], bool]): + """Parse a namedtuple() call into data needed to construct a type. + + Returns a 6-tuple: + - List of argument names + - List of argument types + - List of default values + - First argument of namedtuple + - All typevars found in the field definition + - Whether all types are ready. + + Return None if the definition didn't typecheck. + """ + type_name = "NamedTuple" if fullname in TYPED_NAMEDTUPLE_NAMES else "namedtuple" + # TODO: Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + self.fail(f'Too few arguments for "{type_name}()"', call) + return None + defaults: list[Expression] = [] + rename = False + if len(args) > 2: + # Typed namedtuple doesn't support additional arguments. + if fullname in TYPED_NAMEDTUPLE_NAMES: + self.fail('Too many arguments for "NamedTuple()"', call) + return None + for i, arg_name in enumerate(call.arg_names[2:], 2): + if arg_name == "defaults": + arg = args[i] + # We don't care what the values are, as long as the argument is an iterable + # and we can count how many defaults there are. + if isinstance(arg, (ListExpr, TupleExpr)): + defaults = list(arg.items) + else: + self.fail( + "List or tuple literal expected as the defaults argument to " + "{}()".format(type_name), + arg, + ) + elif arg_name == "rename": + arg = args[i] + if isinstance(arg, NameExpr) and arg.name in ("True", "False"): + rename = arg.name == "True" + else: + self.fail( + f'Boolean literal expected as the "rename" argument to {type_name}()', + arg, + code=ARG_TYPE, + ) + if call.arg_kinds[:2] != [ARG_POS, ARG_POS]: + self.fail(f'Unexpected arguments to "{type_name}()"', call) + return None + if not isinstance(args[0], StrExpr): + self.fail(f'"{type_name}()" expects a string literal as the first argument', call) + return None + typename = args[0].value + types: list[Type] = [] + tvar_defs = [] + if not isinstance(args[1], (ListExpr, TupleExpr)): + if fullname == "collections.namedtuple" and isinstance(args[1], StrExpr): + str_expr = args[1] + items = str_expr.value.replace(",", " ").split() + else: + self.fail( + 'List or tuple literal expected as the second argument to "{}()"'.format( + type_name + ), + call, + ) + return None + else: + listexpr = args[1] + if fullname == "collections.namedtuple": + # The fields argument contains just names, with implicit Any types. + if not is_StrExpr_list(listexpr.items): + self.fail('String literal expected as "namedtuple()" item', call) + return None + items = [item.value for item in listexpr.items] + else: + type_exprs = [ + t.items[1] + for t in listexpr.items + if isinstance(t, TupleExpr) and len(t.items) == 2 + ] + tvar_defs = self.api.get_and_bind_all_tvars(type_exprs) + # The fields argument contains (name, type) tuples. + result = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if result is None: + # One of the types is not ready, defer. + return None + items, types, _, ok = result + if not ok: + return [], [], [], typename, [], False + if not types: + types = [AnyType(TypeOfAny.unannotated) for _ in items] + processed_items = [] + seen_names: set[str] = set() + for i, item in enumerate(items): + problem = self.check_namedtuple_field_name(item, seen_names) + if problem is None: + processed_items.append(item) + seen_names.add(item) + else: + if not rename: + self.fail(f'"{type_name}()" {problem}', call) + # Even if rename=False, we pretend that it is True. + # At runtime namedtuple creation would throw an error; + # applying the rename logic means we create a more sensible + # namedtuple. + new_name = f"_{i}" + processed_items.append(new_name) + seen_names.add(new_name) + + if len(defaults) > len(items): + self.fail(f'Too many defaults given in call to "{type_name}()"', call) + defaults = defaults[: len(items)] + return processed_items, types, defaults, typename, tvar_defs, True + + def parse_namedtuple_fields_with_types( + self, nodes: list[Expression], context: Context + ) -> tuple[list[str], list[Type], list[Expression], bool] | None: + """Parse typed named tuple fields. + + Return (names, types, defaults, whether types are all ready), or None if error occurred. + """ + items: list[str] = [] + types: list[Type] = [] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + self.fail('Invalid "NamedTuple()" field definition', item) + return None + name, type_node = item.items + if isinstance(name, StrExpr): + items.append(name.value) + else: + self.fail('Invalid "NamedTuple()" field name', item) + return None + try: + type = expr_to_unanalyzed_type(type_node, self.options, self.api.is_stub_file) + except TypeTranslationError: + self.fail("Invalid field type", type_node) + return None + # We never allow recursive types at function scope. + analyzed = self.api.anal_type( + type, + allow_placeholder=not self.api.is_func_scope(), + prohibit_self_type="NamedTuple item type", + ) + # Workaround #4987 and avoid introducing a bogus UnboundType + if isinstance(analyzed, UnboundType): + analyzed = AnyType(TypeOfAny.from_error) + # These should be all known, otherwise we would defer in visit_assignment_stmt(). + if analyzed is None: + return [], [], [], False + types.append(analyzed) + else: + self.fail('Tuple expected as "NamedTuple()" field', item) + return None + return items, types, [], True + + def build_namedtuple_typeinfo( + self, + name: str, + items: list[str], + types: list[Type], + default_items: Mapping[str, Expression], + line: int, + existing_info: TypeInfo | None, + ) -> TypeInfo: + strtype = self.api.named_type("builtins.str") + implicit_any = AnyType(TypeOfAny.special_form) + basetuple_type = self.api.named_type("builtins.tuple", [implicit_any]) + dictype = self.api.named_type("builtins.dict", [strtype, implicit_any]) + # Actual signature should return OrderedDict[str, Union[types]] + ordereddictype = self.api.named_type("builtins.dict", [strtype, implicit_any]) + fallback = self.api.named_type("builtins.tuple", [implicit_any]) + # Note: actual signature should accept an invariant version of Iterable[UnionType[types]]. + # but it can't be expressed. 'new' and 'len' should be callable types. + iterable_type = self.api.named_type_or_none("typing.Iterable", [implicit_any]) + function_type = self.api.named_type("builtins.function") + + literals: list[Type] = [LiteralType(item, strtype) for item in items] + match_args_type = TupleType(literals, basetuple_type) + + info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) + info.is_named_tuple = True + tuple_base = TupleType(types, fallback) + if info.special_alias and has_placeholder(info.special_alias.target): + self.api.process_placeholder( + None, "NamedTuple item", info, force_progress=tuple_base != info.tuple_type + ) + info.update_tuple_type(tuple_base) + info.line = line + # For use by mypyc. + info.metadata["namedtuple"] = {"fields": items.copy()} + + # We can't calculate the complete fallback type until after semantic + # analysis, since otherwise base classes might be incomplete. Postpone a + # callback function that patches the fallback. + if not has_placeholder(tuple_base) and not has_type_vars(tuple_base): + self.api.schedule_patch( + PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(tuple_base) + ) + + def add_field( + var: Var, is_initialized_in_class: bool = False, is_property: bool = False + ) -> None: + var.info = info + var.is_initialized_in_class = is_initialized_in_class + var.is_property = is_property + var._fullname = f"{info.fullname}.{var.name}" + info.names[var.name] = SymbolTableNode(MDEF, var) + + fields = [Var(item, typ) for item, typ in zip(items, types)] + for var in fields: + add_field(var, is_property=True) + # We can't share Vars between fields and method arguments, since they + # have different full names (the latter are normally used as local variables + # in functions, so their full names are set to short names when generated methods + # are analyzed). + vars = [Var(item, typ) for item, typ in zip(items, types)] + + tuple_of_strings = TupleType([strtype for _ in items], basetuple_type) + add_field(Var("_fields", tuple_of_strings), is_initialized_in_class=True) + add_field(Var("_field_types", dictype), is_initialized_in_class=True) + add_field(Var("_field_defaults", dictype), is_initialized_in_class=True) + add_field(Var("_source", strtype), is_initialized_in_class=True) + add_field(Var("__annotations__", ordereddictype), is_initialized_in_class=True) + add_field(Var("__doc__", strtype), is_initialized_in_class=True) + if self.options.python_version >= (3, 10): + add_field(Var("__match_args__", match_args_type), is_initialized_in_class=True) + + assert info.tuple_type is not None # Set by update_tuple_type() above. + shared_self_type = TypeVarType( + name=SELF_TVAR_NAME, + fullname=f"{info.fullname}.{SELF_TVAR_NAME}", + # Namespace is patched per-method below. + id=self.api.tvar_scope.new_unique_func_id(), + values=[], + upper_bound=info.tuple_type, + default=AnyType(TypeOfAny.from_omitted_generics), + ) + + def add_method( + funcname: str, + ret: Type | None, # None means use (patched) self-type + args: list[Argument], + is_classmethod: bool = False, + is_new: bool = False, + ) -> None: + fullname = f"{info.fullname}.{funcname}" + self_type = shared_self_type.copy_modified( + id=TypeVarId(shared_self_type.id.raw_id, namespace=fullname) + ) + if ret is None: + ret = self_type + if is_classmethod or is_new: + first = [Argument(Var("_cls"), TypeType.make_normalized(self_type), None, ARG_POS)] + else: + first = [Argument(Var("_self"), self_type, None, ARG_POS)] + args = first + args + + types = [arg.type_annotation for arg in args] + items = [arg.variable.name for arg in args] + arg_kinds = [arg.kind for arg in args] + assert None not in types + signature = CallableType(cast(List[Type], types), arg_kinds, items, ret, function_type) + signature.variables = [self_type] + func = FuncDef(funcname, args, Block([])) + func.info = info + func.is_class = is_classmethod + func.type = set_callable_name(signature, func) + func._fullname = fullname + func.line = line + if is_classmethod: + v = Var(funcname, func.type) + v.is_classmethod = True + v.info = info + v._fullname = func._fullname + func.is_decorated = True + dec = Decorator(func, [NameExpr("classmethod")], v) + dec.line = line + sym = SymbolTableNode(MDEF, dec) + else: + sym = SymbolTableNode(MDEF, func) + sym.plugin_generated = True + info.names[funcname] = sym + + add_method( + "_replace", + ret=None, + args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], + ) + if self.options.python_version >= (3, 13): + add_method( + "__replace__", + ret=None, + args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], + ) + + def make_init_arg(var: Var) -> Argument: + default = default_items.get(var.name, None) + kind = ARG_POS if default is None else ARG_OPT + return Argument(var, var.type, default, kind) + + add_method("__new__", ret=None, args=[make_init_arg(var) for var in vars], is_new=True) + add_method("_asdict", args=[], ret=ordereddictype) + add_method( + "_make", + ret=None, + is_classmethod=True, + args=[Argument(Var("iterable", iterable_type), iterable_type, None, ARG_POS)], + ) + + self_tvar_expr = TypeVarExpr( + SELF_TVAR_NAME, + info.fullname + "." + SELF_TVAR_NAME, + [], + info.tuple_type, + AnyType(TypeOfAny.from_omitted_generics), + ) + info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr) + return info + + @contextmanager + def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]: + """Preserve the generated body of class-based named tuple and then restore it. + + Temporarily clear the names dict so we don't get errors about duplicate names + that were already set in build_namedtuple_typeinfo (we already added the tuple + field names while generating the TypeInfo, and actual duplicates are + already reported). + """ + nt_names = named_tuple_info.names + named_tuple_info.names = SymbolTable() + + yield + + # Make sure we didn't use illegal names, then reset the names in the typeinfo. + for prohibited in NAMEDTUPLE_PROHIBITED_NAMES: + if prohibited in named_tuple_info.names: + if nt_names.get(prohibited) is named_tuple_info.names[prohibited]: + continue + ctx = named_tuple_info.names[prohibited].node + assert ctx is not None + self.fail(f'Cannot overwrite NamedTuple attribute "{prohibited}"', ctx) + + # Restore the names in the original symbol table. This ensures that the symbol + # table contains the field objects created by build_namedtuple_typeinfo. Exclude + # __doc__, which can legally be overwritten by the class. + for key, value in nt_names.items(): + if key in named_tuple_info.names: + if key == "__doc__": + continue + sym = named_tuple_info.names[key] + if isinstance(sym.node, (FuncBase, Decorator)) and not sym.plugin_generated: + # Keep user-defined methods as is. + continue + # Keep existing (user-provided) definitions under mangled names, so they + # get semantically analyzed. + r_key = get_unique_redefinition_name(key, named_tuple_info.names) + named_tuple_info.names[r_key] = sym + named_tuple_info.names[key] = value + + # Helpers + + def check_namedtuple_field_name(self, field: str, seen_names: Container[str]) -> str | None: + """Return None for valid fields, a string description for invalid ones.""" + if field in seen_names: + return f'has duplicate field name "{field}"' + elif not field.isidentifier(): + return f'field name "{field}" is not a valid identifier' + elif field.startswith("_"): + return f'field name "{field}" starts with an underscore' + elif keyword.iskeyword(field): + return f'field name "{field}" is a keyword' + return None + + def fail(self, msg: str, ctx: Context, code: ErrorCode | None = None) -> None: + self.api.fail(msg, ctx, code=code) diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py new file mode 100644 index 000000000000..c9c0c46f7aee --- /dev/null +++ b/mypy/semanal_newtype.py @@ -0,0 +1,273 @@ +"""Semantic analysis of NewType definitions. + +This is conceptually part of mypy.semanal (semantic analyzer pass 2). +""" + +from __future__ import annotations + +from mypy import errorcodes as codes +from mypy.errorcodes import ErrorCode +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.messages import MessageBuilder, format_type +from mypy.nodes import ( + ARG_POS, + MDEF, + Argument, + AssignmentStmt, + Block, + CallExpr, + Context, + FuncDef, + NameExpr, + NewTypeExpr, + PlaceholderNode, + RefExpr, + StrExpr, + SymbolTableNode, + TypeInfo, + Var, +) +from mypy.options import Options +from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder +from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type +from mypy.types import ( + AnyType, + CallableType, + Instance, + NoneType, + PlaceholderType, + TupleType, + Type, + TypeOfAny, + get_proper_type, +) + + +class NewTypeAnalyzer: + def __init__( + self, options: Options, api: SemanticAnalyzerInterface, msg: MessageBuilder + ) -> None: + self.options = options + self.api = api + self.msg = msg + + def process_newtype_declaration(self, s: AssignmentStmt) -> bool: + """Check if s declares a NewType; if yes, store it in symbol table. + + Return True if it's a NewType declaration. The current target may be + deferred as a side effect if the base type is not ready, even if + the return value is True. + + The logic in this function mostly copies the logic for visit_class_def() + with a single (non-Generic) base. + """ + var_name, call = self.analyze_newtype_declaration(s) + if var_name is None or call is None: + return False + name = var_name + # OK, now we know this is a NewType. But the base type may be not ready yet, + # add placeholder as we do for ClassDef. + + if self.api.is_func_scope(): + name += "@" + str(s.line) + fullname = self.api.qualified_name(name) + + if not call.analyzed or isinstance(call.analyzed, NewTypeExpr) and not call.analyzed.info: + # Start from labeling this as a future class, as we do for normal ClassDefs. + placeholder = PlaceholderNode(fullname, s, s.line, becomes_typeinfo=True) + self.api.add_symbol(var_name, placeholder, s, can_defer=False) + + old_type, should_defer = self.check_newtype_args(var_name, call, s) + old_type = get_proper_type(old_type) + if not isinstance(call.analyzed, NewTypeExpr): + call.analyzed = NewTypeExpr(var_name, old_type, line=call.line, column=call.column) + else: + call.analyzed.old_type = old_type + if old_type is None: + if should_defer: + # Base type is not ready. + self.api.defer() + return True + + # Create the corresponding class definition if the aliased type is subtypeable + assert isinstance(call.analyzed, NewTypeExpr) + if isinstance(old_type, TupleType): + newtype_class_info = self.build_newtype_typeinfo( + name, old_type, old_type.partial_fallback, s.line, call.analyzed.info + ) + newtype_class_info.update_tuple_type(old_type) + elif isinstance(old_type, Instance): + if old_type.type.is_protocol: + self.fail("NewType cannot be used with protocol classes", s) + newtype_class_info = self.build_newtype_typeinfo( + name, old_type, old_type, s.line, call.analyzed.info + ) + else: + if old_type is not None: + message = "Argument 2 to NewType(...) must be subclassable (got {})" + self.fail( + message.format(format_type(old_type, self.options)), + s, + code=codes.VALID_NEWTYPE, + ) + # Otherwise the error was already reported. + old_type = AnyType(TypeOfAny.from_error) + object_type = self.api.named_type("builtins.object") + newtype_class_info = self.build_newtype_typeinfo( + name, old_type, object_type, s.line, call.analyzed.info + ) + newtype_class_info.fallback_to_any = True + + check_for_explicit_any( + old_type, self.options, self.api.is_typeshed_stub_file, self.msg, context=s + ) + + if self.options.disallow_any_unimported and has_any_from_unimported_type(old_type): + self.msg.unimported_type_becomes_any("Argument 2 to NewType(...)", old_type, s) + + # If so, add it to the symbol table. + assert isinstance(call.analyzed, NewTypeExpr) + # As we do for normal classes, create the TypeInfo only once, then just + # update base classes on next iterations (to get rid of placeholders there). + if not call.analyzed.info: + call.analyzed.info = newtype_class_info + else: + call.analyzed.info.bases = newtype_class_info.bases + self.api.add_symbol(var_name, call.analyzed.info, s) + if self.api.is_func_scope(): + self.api.add_symbol_skip_local(name, call.analyzed.info) + newtype_class_info.line = s.line + return True + + def analyze_newtype_declaration(self, s: AssignmentStmt) -> tuple[str | None, CallExpr | None]: + """Return the NewType call expression if `s` is a newtype declaration or None otherwise.""" + name, call = None, None + if ( + len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and isinstance(s.rvalue, CallExpr) + and isinstance(s.rvalue.callee, RefExpr) + and (s.rvalue.callee.fullname in ("typing.NewType", "typing_extensions.NewType")) + ): + name = s.lvalues[0].name + + if s.type: + self.fail("Cannot declare the type of a NewType declaration", s) + + names = self.api.current_symbol_table() + existing = names.get(name) + # Give a better error message than generic "Name already defined". + if ( + existing + and not isinstance(existing.node, PlaceholderNode) + and not s.rvalue.analyzed + ): + self.fail(f'Cannot redefine "{name}" as a NewType', s) + + # This dummy NewTypeExpr marks the call as sufficiently analyzed; it will be + # overwritten later with a fully complete NewTypeExpr if there are no other + # errors with the NewType() call. + call = s.rvalue + + return name, call + + def check_newtype_args( + self, name: str, call: CallExpr, context: Context + ) -> tuple[Type | None, bool]: + """Ananlyze base type in NewType call. + + Return a tuple (type, should defer). + """ + has_failed = False + args, arg_kinds = call.args, call.arg_kinds + if len(args) != 2 or arg_kinds[0] != ARG_POS or arg_kinds[1] != ARG_POS: + self.fail("NewType(...) expects exactly two positional arguments", context) + return None, False + + # Check first argument + if not isinstance(args[0], StrExpr): + self.fail("Argument 1 to NewType(...) must be a string literal", context) + has_failed = True + elif args[0].value != name: + msg = 'String argument 1 "{}" to NewType(...) does not match variable name "{}"' + self.fail(msg.format(args[0].value, name), context) + has_failed = True + + # Check second argument + msg = "Argument 2 to NewType(...) must be a valid type" + try: + unanalyzed_type = expr_to_unanalyzed_type(args[1], self.options, self.api.is_stub_file) + except TypeTranslationError: + self.fail(msg, context) + return None, False + + # We want to use our custom error message (see above), so we suppress + # the default error message for invalid types here. + old_type = get_proper_type( + self.api.anal_type( + unanalyzed_type, + report_invalid_types=False, + allow_placeholder=not self.api.is_func_scope(), + ) + ) + should_defer = False + if isinstance(old_type, PlaceholderType): + old_type = None + if old_type is None: + should_defer = True + + # The caller of this function assumes that if we return a Type, it's always + # a valid one. So, we translate AnyTypes created from errors into None. + if isinstance(old_type, AnyType) and old_type.is_from_error: + self.fail(msg, context) + return None, False + + return None if has_failed else old_type, should_defer + + def build_newtype_typeinfo( + self, + name: str, + old_type: Type, + base_type: Instance, + line: int, + existing_info: TypeInfo | None, + ) -> TypeInfo: + info = existing_info or self.api.basic_new_typeinfo(name, base_type, line) + info.bases = [base_type] # Update in case there were nested placeholders. + info.is_newtype = True + + # Add __init__ method + args = [ + Argument(Var("self"), NoneType(), None, ARG_POS), + self.make_argument("item", old_type), + ] + signature = CallableType( + arg_types=[Instance(info, []), old_type], + arg_kinds=[arg.kind for arg in args], + arg_names=["self", "item"], + ret_type=NoneType(), + fallback=self.api.named_type("builtins.function"), + name=name, + ) + init_func = FuncDef("__init__", args, Block([]), typ=signature) + init_func.info = info + init_func._fullname = info.fullname + ".__init__" + if not existing_info: + updated = True + else: + previous_sym = info.names["__init__"].node + assert isinstance(previous_sym, FuncDef) + updated = old_type != previous_sym.arguments[1].variable.type + info.names["__init__"] = SymbolTableNode(MDEF, init_func) + + if has_placeholder(old_type): + self.api.process_placeholder(None, "NewType base", info, force_progress=updated) + return info + + # Helpers + + def make_argument(self, name: str, type: Type) -> Argument: + return Argument(Var(name), type, None, ARG_POS) + + def fail(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + self.api.fail(msg, ctx, code=code) diff --git a/mypy/semanal_pass1.py b/mypy/semanal_pass1.py new file mode 100644 index 000000000000..aaa01969217a --- /dev/null +++ b/mypy/semanal_pass1.py @@ -0,0 +1,156 @@ +"""Block/import reachability analysis.""" + +from __future__ import annotations + +from mypy.nodes import ( + AssertStmt, + AssignmentStmt, + Block, + ClassDef, + ExpressionStmt, + ForStmt, + FuncDef, + IfStmt, + Import, + ImportAll, + ImportFrom, + MatchStmt, + MypyFile, + ReturnStmt, +) +from mypy.options import Options +from mypy.reachability import ( + assert_will_always_fail, + infer_reachability_of_if_statement, + infer_reachability_of_match_statement, +) +from mypy.traverser import TraverserVisitor + + +class SemanticAnalyzerPreAnalysis(TraverserVisitor): + """Analyze reachability of blocks and imports and other local things. + + This runs before semantic analysis, so names have not been bound. Imports are + also not resolved yet, so we can only access the current module. + + This determines static reachability of blocks and imports due to version and + platform checks, among others. + + The main entry point is 'visit_file'. + + Reachability of imports needs to be determined very early in the build since + this affects which modules will ultimately be processed. + + Consider this example: + + import sys + + def do_stuff() -> None: + if sys.version_info >= (3, 10): + import xyz # Only available in Python 3.10+ + xyz.whatever() + ... + + The block containing 'import xyz' is unreachable in Python 3 mode. The import + shouldn't be processed in Python 3 mode, even if the module happens to exist. + """ + + def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) -> None: + self.platform = options.platform + self.cur_mod_id = mod_id + self.cur_mod_node = file + self.options = options + self.is_global_scope = True + self.skipped_lines: set[int] = set() + + for i, defn in enumerate(file.defs): + defn.accept(self) + if isinstance(defn, AssertStmt) and assert_will_always_fail(defn, options): + # We've encountered an assert that's always false, + # e.g. assert sys.platform == 'lol'. Truncate the + # list of statements. This mutates file.defs too. + if i < len(file.defs) - 1: + next_def, last = file.defs[i + 1], file.defs[-1] + if last.end_line is not None: + # We are on a Python version recent enough to support end lines. + self.skipped_lines |= set(range(next_def.line, last.end_line + 1)) + del file.defs[i + 1 :] + break + file.skipped_lines = self.skipped_lines + + def visit_func_def(self, node: FuncDef) -> None: + old_global_scope = self.is_global_scope + self.is_global_scope = False + super().visit_func_def(node) + self.is_global_scope = old_global_scope + file_node = self.cur_mod_node + if ( + self.is_global_scope + and file_node.is_stub + and node.name == "__getattr__" + and file_node.is_package_init_file() + ): + # __init__.pyi with __getattr__ means that any submodules are assumed + # to exist, even if there is no stub. Note that we can't verify that the + # return type is compatible, since we haven't bound types yet. + file_node.is_partial_stub_package = True + + def visit_class_def(self, node: ClassDef) -> None: + old_global_scope = self.is_global_scope + self.is_global_scope = False + super().visit_class_def(node) + self.is_global_scope = old_global_scope + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_top_level = self.is_global_scope + super().visit_import_from(node) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = self.is_global_scope + super().visit_import_all(node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = self.is_global_scope + super().visit_import(node) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, self.options) + for expr in s.expr: + expr.accept(self) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + if b.end_line is not None: + # We are on a Python version recent enough to support end lines. + self.skipped_lines |= set(range(b.line, b.end_line + 1)) + return + super().visit_block(b) + + def visit_match_stmt(self, s: MatchStmt) -> None: + infer_reachability_of_match_statement(s, self.options) + for guard in s.guards: + if guard is not None: + guard.accept(self) + for body in s.bodies: + body.accept(self) + + # The remaining methods are an optimization: don't visit nested expressions + # of common statements, since they can have no effect. + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + pass + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + pass + + def visit_return_stmt(self, s: ReturnStmt) -> None: + pass + + def visit_for_stmt(self, s: ForStmt) -> None: + s.body.accept(self) + if s.else_body is not None: + s.else_body.accept(self) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py new file mode 100644 index 000000000000..cb0bdebab724 --- /dev/null +++ b/mypy/semanal_shared.py @@ -0,0 +1,490 @@ +"""Shared definitions used by different parts of semantic analysis.""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import Callable, Final, overload +from typing_extensions import Literal, Protocol + +from mypy_extensions import trait + +from mypy.errorcodes import LITERAL_REQ, ErrorCode +from mypy.nodes import ( + CallExpr, + ClassDef, + Context, + DataclassTransformSpec, + Decorator, + Expression, + FuncDef, + NameExpr, + Node, + OverloadedFuncDef, + RefExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TypeInfo, +) +from mypy.plugin import SemanticAnalyzerPluginInterface +from mypy.tvar_scope import TypeVarLikeScope +from mypy.type_visitor import ANY_STRATEGY, BoolTypeQuery +from mypy.typeops import make_simplified_union +from mypy.types import ( + TPDICT_FB_NAMES, + AnyType, + FunctionLike, + Instance, + Parameters, + ParamSpecFlavor, + ParamSpecType, + PlaceholderType, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + UnpackType, + get_proper_type, +) + +# Subclasses can override these Var attributes with incompatible types. This can also be +# set for individual attributes using 'allow_incompatible_override' of Var. +ALLOW_INCOMPATIBLE_OVERRIDE: Final = ("__slots__", "__deletable__", "__match_args__") + + +# Priorities for ordering of patches within the "patch" phase of semantic analysis +# (after the main pass): + +# Fix fallbacks (does subtype checks). +PRIORITY_FALLBACKS: Final = 1 + + +@trait +class SemanticAnalyzerCoreInterface: + """A core abstract interface to generic semantic analyzer functionality. + + This is implemented by both semantic analyzer passes 2 and 3. + """ + + @abstractmethod + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + raise NotImplementedError + + @abstractmethod + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + raise NotImplementedError + + @abstractmethod + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode | None: + raise NotImplementedError + + @abstractmethod + def fail( + self, + msg: str, + ctx: Context, + serious: bool = False, + *, + blocker: bool = False, + code: ErrorCode | None = None, + ) -> None: + raise NotImplementedError + + @abstractmethod + def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + raise NotImplementedError + + @abstractmethod + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + raise NotImplementedError + + @abstractmethod + def record_incomplete_ref(self) -> None: + raise NotImplementedError + + @abstractmethod + def defer(self, debug_context: Context | None = None, force_progress: bool = False) -> None: + raise NotImplementedError + + @abstractmethod + def is_incomplete_namespace(self, fullname: str) -> bool: + """Is a module or class namespace potentially missing some definitions?""" + raise NotImplementedError + + @property + @abstractmethod + def final_iteration(self) -> bool: + """Is this the final iteration of semantic analysis?""" + raise NotImplementedError + + @abstractmethod + def is_future_flag_set(self, flag: str) -> bool: + """Is the specific __future__ feature imported""" + raise NotImplementedError + + @property + @abstractmethod + def is_stub_file(self) -> bool: + raise NotImplementedError + + @abstractmethod + def is_func_scope(self) -> bool: + raise NotImplementedError + + @property + @abstractmethod + def type(self) -> TypeInfo | None: + raise NotImplementedError + + +@trait +class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface): + """A limited abstract interface to some generic semantic analyzer pass 2 functionality. + + We use this interface for various reasons: + + * Looser coupling + * Cleaner import graph + * Less need to pass around callback functions + """ + + tvar_scope: TypeVarLikeScope + + @abstractmethod + def lookup( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + raise NotImplementedError + + @abstractmethod + def named_type(self, fullname: str, args: list[Type] | None = None) -> Instance: + raise NotImplementedError + + @abstractmethod + def named_type_or_none(self, fullname: str, args: list[Type] | None = None) -> Instance | None: + raise NotImplementedError + + @abstractmethod + def accept(self, node: Node) -> None: + raise NotImplementedError + + @abstractmethod + def anal_type( + self, + t: Type, + *, + tvar_scope: TypeVarLikeScope | None = None, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_placeholder: bool = False, + report_invalid_types: bool = True, + prohibit_self_type: str | None = None, + ) -> Type | None: + raise NotImplementedError + + @abstractmethod + def get_and_bind_all_tvars(self, type_exprs: list[Expression]) -> list[TypeVarLikeType]: + raise NotImplementedError + + @abstractmethod + def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance, line: int) -> TypeInfo: + raise NotImplementedError + + @abstractmethod + def schedule_patch(self, priority: int, fn: Callable[[], None]) -> None: + raise NotImplementedError + + @abstractmethod + def add_symbol_table_node(self, name: str, stnode: SymbolTableNode) -> bool: + """Add node to the current symbol table.""" + raise NotImplementedError + + @abstractmethod + def current_symbol_table(self) -> SymbolTable: + """Get currently active symbol table. + + May be module, class, or local namespace. + """ + raise NotImplementedError + + @abstractmethod + def add_symbol( + self, + name: str, + node: SymbolNode, + context: Context, + module_public: bool = True, + module_hidden: bool = False, + can_defer: bool = True, + ) -> bool: + """Add symbol to the current symbol table.""" + raise NotImplementedError + + @abstractmethod + def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None: + """Add symbol to the current symbol table, skipping locals. + + This is used to store symbol nodes in a symbol table that + is going to be serialized (local namespaces are not serialized). + See implementation docstring for more details. + """ + raise NotImplementedError + + @abstractmethod + def parse_bool(self, expr: Expression) -> bool | None: + raise NotImplementedError + + @abstractmethod + def qualified_name(self, n: str) -> str: + raise NotImplementedError + + @property + @abstractmethod + def is_typeshed_stub_file(self) -> bool: + raise NotImplementedError + + @abstractmethod + def process_placeholder( + self, name: str | None, kind: str, ctx: Context, force_progress: bool = False + ) -> None: + raise NotImplementedError + + +def set_callable_name(sig: Type, fdef: FuncDef) -> ProperType: + sig = get_proper_type(sig) + if isinstance(sig, FunctionLike): + if fdef.info: + if fdef.info.fullname in TPDICT_FB_NAMES: + # Avoid exposing the internal _TypedDict name. + class_name = "TypedDict" + else: + class_name = fdef.info.name + return sig.with_name(f"{fdef.name} of {class_name}") + else: + return sig.with_name(fdef.name) + else: + return sig + + +def calculate_tuple_fallback(typ: TupleType) -> None: + """Calculate a precise item type for the fallback of a tuple type. + + This must be called only after the main semantic analysis pass, since joins + aren't available before that. + + Note that there is an apparent chicken and egg problem with respect + to verifying type arguments against bounds. Verifying bounds might + require fallbacks, but we might use the bounds to calculate the + fallbacks. In practice this is not a problem, since the worst that + can happen is that we have invalid type argument values, and these + can happen in later stages as well (they will generate errors, but + we don't prevent their existence). + """ + fallback = typ.partial_fallback + assert fallback.type.fullname == "builtins.tuple" + items = [] + for item in typ.items: + # TODO: this duplicates some logic in typeops.tuple_fallback(). + if isinstance(item, UnpackType): + unpacked_type = get_proper_type(item.type) + if isinstance(unpacked_type, TypeVarTupleType): + unpacked_type = get_proper_type(unpacked_type.upper_bound) + if ( + isinstance(unpacked_type, Instance) + and unpacked_type.type.fullname == "builtins.tuple" + ): + items.append(unpacked_type.args[0]) + else: + raise NotImplementedError + else: + items.append(item) + fallback.args = (make_simplified_union(items),) + + +class _NamedTypeCallback(Protocol): + def __call__(self, fully_qualified_name: str, args: list[Type] | None = None) -> Instance: ... + + +def paramspec_args( + name: str, + fullname: str, + id: TypeVarId, + *, + named_type_func: _NamedTypeCallback, + line: int = -1, + column: int = -1, + prefix: Parameters | None = None, +) -> ParamSpecType: + return ParamSpecType( + name, + fullname, + id, + flavor=ParamSpecFlavor.ARGS, + upper_bound=named_type_func("builtins.tuple", [named_type_func("builtins.object")]), + default=AnyType(TypeOfAny.from_omitted_generics), + line=line, + column=column, + prefix=prefix, + ) + + +def paramspec_kwargs( + name: str, + fullname: str, + id: TypeVarId, + *, + named_type_func: _NamedTypeCallback, + line: int = -1, + column: int = -1, + prefix: Parameters | None = None, +) -> ParamSpecType: + return ParamSpecType( + name, + fullname, + id, + flavor=ParamSpecFlavor.KWARGS, + upper_bound=named_type_func( + "builtins.dict", [named_type_func("builtins.str"), named_type_func("builtins.object")] + ), + default=AnyType(TypeOfAny.from_omitted_generics), + line=line, + column=column, + prefix=prefix, + ) + + +class HasPlaceholders(BoolTypeQuery): + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + + def visit_placeholder_type(self, t: PlaceholderType) -> bool: + return True + + +def has_placeholder(typ: Type) -> bool: + """Check if a type contains any placeholder types (recursively).""" + return typ.accept(HasPlaceholders()) + + +def find_dataclass_transform_spec(node: Node | None) -> DataclassTransformSpec | None: + """ + Find the dataclass transform spec for the given node, if any exists. + + Per PEP 681 (https://peps.python.org/pep-0681/#the-dataclass-transform-decorator), dataclass + transforms can be specified in multiple ways, including decorator functions and + metaclasses/base classes. This function resolves the spec from any of these variants. + """ + + # The spec only lives on the function/class definition itself, so we need to unwrap down to that + # point + if isinstance(node, CallExpr): + # Like dataclasses.dataclass, transform-based decorators can be applied either with or + # without parameters; ie, both of these forms are accepted: + # + # @typing.dataclass_transform + # class Foo: ... + # @typing.dataclass_transform(eq=True, order=True, ...) + # class Bar: ... + # + # We need to unwrap the call for the second variant. + node = node.callee + + if isinstance(node, RefExpr): + node = node.node + + if isinstance(node, Decorator): + # typing.dataclass_transform usage must always result in a Decorator; it always uses the + # `@dataclass_transform(...)` syntax and never `@dataclass_transform` + node = node.func + + if isinstance(node, OverloadedFuncDef): + # The dataclass_transform decorator may be attached to any single overload, so we must + # search them all. + # Note that using more than one decorator is undefined behavior, so we can just take the + # first that we find. + for candidate in node.items: + spec = find_dataclass_transform_spec(candidate) + if spec is not None: + return spec + return find_dataclass_transform_spec(node.impl) + + # For functions, we can directly consult the AST field for the spec + if isinstance(node, FuncDef): + return node.dataclass_transform_spec + + if isinstance(node, ClassDef): + node = node.info + if isinstance(node, TypeInfo): + # Search all parent classes to see if any are decorated with `typing.dataclass_transform` + for base in node.mro[1:]: + if base.dataclass_transform_spec is not None: + return base.dataclass_transform_spec + + # Check if there is a metaclass that is decorated with `typing.dataclass_transform` + # + # Note that PEP 681 only discusses using a metaclass that is directly decorated with + # `typing.dataclass_transform`; subclasses thereof should be treated with dataclass + # semantics rather than as transforms: + # + # > If dataclass_transform is applied to a class, dataclass-like semantics will be assumed + # > for any class that directly or indirectly derives from the decorated class or uses the + # > decorated class as a metaclass. + # + # The wording doesn't make this entirely explicit, but Pyright (the reference + # implementation for this PEP) only handles directly-decorated metaclasses. + metaclass_type = node.metaclass_type + if metaclass_type is not None and metaclass_type.type.dataclass_transform_spec is not None: + return metaclass_type.type.dataclass_transform_spec + + return None + + +# Never returns `None` if a default is given +@overload +def require_bool_literal_argument( + api: SemanticAnalyzerInterface | SemanticAnalyzerPluginInterface, + expression: Expression, + name: str, + default: Literal[True] | Literal[False], +) -> bool: ... + + +@overload +def require_bool_literal_argument( + api: SemanticAnalyzerInterface | SemanticAnalyzerPluginInterface, + expression: Expression, + name: str, + default: None = None, +) -> bool | None: ... + + +def require_bool_literal_argument( + api: SemanticAnalyzerInterface | SemanticAnalyzerPluginInterface, + expression: Expression, + name: str, + default: bool | None = None, +) -> bool | None: + """Attempt to interpret an expression as a boolean literal, and fail analysis if we can't.""" + value = parse_bool(expression) + if value is None: + api.fail( + f'"{name}" argument must be a True or False literal', expression, code=LITERAL_REQ + ) + return default + + return value + + +def parse_bool(expr: Expression) -> bool | None: + if isinstance(expr, NameExpr): + if expr.fullname == "builtins.True": + return True + if expr.fullname == "builtins.False": + return False + return None diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py new file mode 100644 index 000000000000..646bb28a3b6e --- /dev/null +++ b/mypy/semanal_typeargs.py @@ -0,0 +1,285 @@ +"""Verify properties of type arguments, like 'int' in C[int] being valid. + +This must happen after semantic analysis since there can be placeholder +types until the end of semantic analysis, and these break various type +operations, including subtype checks. +""" + +from __future__ import annotations + +from typing import Callable + +from mypy import errorcodes as codes, message_registry +from mypy.errorcodes import ErrorCode +from mypy.errors import Errors +from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE +from mypy.messages import format_type +from mypy.mixedtraverser import MixedTraverserVisitor +from mypy.nodes import Block, ClassDef, Context, FakeInfo, FuncItem, MypyFile +from mypy.options import Options +from mypy.scope import Scope +from mypy.subtypes import is_same_type, is_subtype +from mypy.types import ( + AnyType, + CallableType, + Instance, + Parameters, + ParamSpecType, + TupleType, + Type, + TypeAliasType, + TypeOfAny, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UnpackType, + flatten_nested_tuples, + get_proper_type, + get_proper_types, + split_with_prefix_and_suffix, +) +from mypy.typevartuples import erased_vars + + +class TypeArgumentAnalyzer(MixedTraverserVisitor): + def __init__( + self, + errors: Errors, + options: Options, + is_typeshed_file: bool, + named_type: Callable[[str, list[Type]], Instance], + ) -> None: + super().__init__() + self.errors = errors + self.options = options + self.is_typeshed_file = is_typeshed_file + self.named_type = named_type + self.scope = Scope() + # Should we also analyze function definitions, or only module top-levels? + self.recurse_into_functions = True + # Keep track of the type aliases already visited. This is needed to avoid + # infinite recursion on types like A = Union[int, List[A]]. + self.seen_aliases: set[TypeAliasType] = set() + + def visit_mypy_file(self, o: MypyFile) -> None: + self.errors.set_file(o.path, o.fullname, scope=self.scope, options=self.options) + with self.scope.module_scope(o.fullname): + super().visit_mypy_file(o) + + def visit_func(self, defn: FuncItem) -> None: + if not self.recurse_into_functions: + return + with self.scope.function_scope(defn): + super().visit_func(defn) + + def visit_class_def(self, defn: ClassDef) -> None: + with self.scope.class_scope(defn.info): + super().visit_class_def(defn) + + def visit_block(self, o: Block) -> None: + if not o.is_unreachable: + super().visit_block(o) + + def visit_type_alias_type(self, t: TypeAliasType) -> None: + super().visit_type_alias_type(t) + if t in self.seen_aliases: + # Avoid infinite recursion on recursive type aliases. + # Note: it is fine to skip the aliases we have already seen in non-recursive + # types, since errors there have already been reported. + return + self.seen_aliases.add(t) + assert t.alias is not None, f"Unfixed type alias {t.type_ref}" + is_error, is_invalid = self.validate_args( + t.alias.name, tuple(t.args), t.alias.alias_tvars, t + ) + if is_invalid: + # If there is an arity error (e.g. non-Parameters used for ParamSpec etc.), + # then it is safer to erase the arguments completely, to avoid crashes later. + # TODO: can we move this logic to typeanal.py? + t.args = erased_vars(t.alias.alias_tvars, TypeOfAny.from_error) + if not is_error: + # If there was already an error for the alias itself, there is no point in checking + # the expansion, most likely it will result in the same kind of error. + get_proper_type(t).accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + t.items = flatten_nested_tuples(t.items) + # We could also normalize Tuple[*tuple[X, ...]] -> tuple[X, ...] like in + # expand_type() but we can't do this here since it is not a translator visitor, + # and we need to return an Instance instead of TupleType. + super().visit_tuple_type(t) + + def visit_callable_type(self, t: CallableType) -> None: + super().visit_callable_type(t) + t.normalize_trivial_unpack() + + def visit_instance(self, t: Instance) -> None: + super().visit_instance(t) + # Type argument counts were checked in the main semantic analyzer pass. We assume + # that the counts are correct here. + info = t.type + if isinstance(info, FakeInfo): + return # https://github.com/python/mypy/issues/11079 + _, is_invalid = self.validate_args(info.name, t.args, info.defn.type_vars, t) + if is_invalid: + t.args = tuple(erased_vars(info.defn.type_vars, TypeOfAny.from_error)) + if t.type.fullname == "builtins.tuple" and len(t.args) == 1: + # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] + arg = t.args[0] + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + t.args = unpacked.args + + def validate_args( + self, name: str, args: tuple[Type, ...], type_vars: list[TypeVarLikeType], ctx: Context + ) -> tuple[bool, bool]: + if any(isinstance(v, TypeVarTupleType) for v in type_vars): + prefix = next(i for (i, v) in enumerate(type_vars) if isinstance(v, TypeVarTupleType)) + tvt = type_vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + start, middle, end = split_with_prefix_and_suffix( + tuple(args), prefix, len(type_vars) - prefix - 1 + ) + args = start + (TupleType(list(middle), tvt.tuple_fallback),) + end + + is_error = False + is_invalid = False + for (i, arg), tvar in zip(enumerate(args), type_vars): + if isinstance(tvar, TypeVarType): + if isinstance(arg, ParamSpecType): + is_invalid = True + self.fail( + INVALID_PARAM_SPEC_LOCATION.format(format_type(arg, self.options)), + ctx, + code=codes.VALID_TYPE, + ) + self.note( + INVALID_PARAM_SPEC_LOCATION_NOTE.format(arg.name), + ctx, + code=codes.VALID_TYPE, + ) + continue + if isinstance(arg, Parameters): + is_invalid = True + self.fail( + f"Cannot use {format_type(arg, self.options)} for regular type variable," + " only for ParamSpec", + ctx, + code=codes.VALID_TYPE, + ) + continue + if tvar.values: + if isinstance(arg, TypeVarType): + if self.in_type_alias_expr: + # Type aliases are allowed to use unconstrained type variables + # error will be checked at substitution point. + continue + arg_values = arg.values + if not arg_values: + is_error = True + self.fail( + message_registry.INVALID_TYPEVAR_AS_TYPEARG.format(arg.name, name), + ctx, + code=codes.TYPE_VAR, + ) + continue + else: + arg_values = [arg] + if self.check_type_var_values(name, arg_values, tvar.name, tvar.values, ctx): + is_error = True + # Check against upper bound. Since it's object the vast majority of the time, + # add fast path to avoid a potentially slow subtype check. + upper_bound = tvar.upper_bound + object_upper_bound = ( + type(upper_bound) is Instance + and upper_bound.type.fullname == "builtins.object" + ) + if not object_upper_bound and not is_subtype(arg, upper_bound): + if self.in_type_alias_expr and isinstance(arg, TypeVarType): + # Type aliases are allowed to use unconstrained type variables + # error will be checked at substitution point. + continue + is_error = True + self.fail( + message_registry.INVALID_TYPEVAR_ARG_BOUND.format( + format_type(arg, self.options), + name, + format_type(upper_bound, self.options), + ), + ctx, + code=codes.TYPE_VAR, + ) + elif isinstance(tvar, ParamSpecType): + if not isinstance( + get_proper_type(arg), (ParamSpecType, Parameters, AnyType, UnboundType) + ): + is_invalid = True + self.fail( + "Can only replace ParamSpec with a parameter types list or" + f" another ParamSpec, got {format_type(arg, self.options)}", + ctx, + code=codes.VALID_TYPE, + ) + if is_invalid: + is_error = True + return is_error, is_invalid + + def visit_unpack_type(self, typ: UnpackType) -> None: + super().visit_unpack_type(typ) + proper_type = get_proper_type(typ.type) + if isinstance(proper_type, TupleType): + return + if isinstance(proper_type, TypeVarTupleType): + return + # TODO: this should probably be .has_base("builtins.tuple"), also elsewhere. This is + # tricky however, since this needs map_instance_to_supertype() available in many places. + if isinstance(proper_type, Instance) and proper_type.type.fullname == "builtins.tuple": + return + if not isinstance(proper_type, (UnboundType, AnyType)): + # Avoid extra errors if there were some errors already. Also interpret plain Any + # as tuple[Any, ...] (this is better for the code in type checker). + self.fail( + message_registry.INVALID_UNPACK.format(format_type(proper_type, self.options)), + typ.type, + code=codes.VALID_TYPE, + ) + typ.type = self.named_type("builtins.tuple", [AnyType(TypeOfAny.from_error)]) + + def check_type_var_values( + self, name: str, actuals: list[Type], arg_name: str, valids: list[Type], context: Context + ) -> bool: + is_error = False + for actual in get_proper_types(actuals): + # We skip UnboundType here, since they may appear in defn.bases, + # the error will be caught when visiting info.bases, that have bound type + # variables. + if not isinstance(actual, (AnyType, UnboundType)) and not any( + is_same_type(actual, value) for value in valids + ): + is_error = True + if len(actuals) > 1 or not isinstance(actual, Instance): + self.fail( + message_registry.INVALID_TYPEVAR_ARG_VALUE.format(name), + context, + code=codes.TYPE_VAR, + ) + else: + class_name = f'"{name}"' + actual_type_name = f'"{actual.type.name}"' + self.fail( + message_registry.INCOMPATIBLE_TYPEVAR_VALUE.format( + arg_name, class_name, actual_type_name + ), + context, + code=codes.TYPE_VAR, + ) + return is_error + + def fail(self, msg: str, context: Context, *, code: ErrorCode | None = None) -> None: + self.errors.report(context.line, context.column, msg, code=code) + + def note(self, msg: str, context: Context, *, code: ErrorCode | None = None) -> None: + self.errors.report(context.line, context.column, msg, severity="note", code=code) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py new file mode 100644 index 000000000000..d081898bf010 --- /dev/null +++ b/mypy/semanal_typeddict.py @@ -0,0 +1,617 @@ +"""Semantic analysis of TypedDict definitions.""" + +from __future__ import annotations + +from typing import Final + +from mypy import errorcodes as codes, message_registry +from mypy.errorcodes import ErrorCode +from mypy.expandtype import expand_type +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE +from mypy.messages import MessageBuilder +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + AssignmentStmt, + CallExpr, + ClassDef, + Context, + DictExpr, + EllipsisExpr, + Expression, + ExpressionStmt, + IndexExpr, + NameExpr, + PassStmt, + RefExpr, + Statement, + StrExpr, + TempNode, + TupleExpr, + TypedDictExpr, + TypeInfo, +) +from mypy.options import Options +from mypy.semanal_shared import ( + SemanticAnalyzerInterface, + has_placeholder, + require_bool_literal_argument, +) +from mypy.state import state +from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type +from mypy.types import ( + TPDICT_NAMES, + AnyType, + ReadOnlyType, + RequiredType, + Type, + TypedDictType, + TypeOfAny, + TypeVarLikeType, +) + +TPDICT_CLASS_ERROR: Final = ( + 'Invalid statement in TypedDict definition; expected "field_name: field_type"' +) + + +class TypedDictAnalyzer: + def __init__( + self, options: Options, api: SemanticAnalyzerInterface, msg: MessageBuilder + ) -> None: + self.options = options + self.api = api + self.msg = msg + + def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | None]: + """Analyze a class that may define a TypedDict. + + Assume that base classes have been analyzed already. + + Note: Unlike normal classes, we won't create a TypeInfo until + the whole definition of the TypeDict (including the body and all + key names and types) is complete. This is mostly because we + store the corresponding TypedDictType in the TypeInfo. + + Return (is this a TypedDict, new TypeInfo). Specifics: + * If we couldn't finish due to incomplete reference anywhere in + the definition, return (True, None). + * If this is not a TypedDict, return (False, None). + """ + possible = False + for base_expr in defn.base_type_exprs: + if isinstance(base_expr, CallExpr): + base_expr = base_expr.callee + if isinstance(base_expr, IndexExpr): + base_expr = base_expr.base + if isinstance(base_expr, RefExpr): + self.api.accept(base_expr) + if base_expr.fullname in TPDICT_NAMES or self.is_typeddict(base_expr): + possible = True + if isinstance(base_expr.node, TypeInfo) and base_expr.node.is_final: + err = message_registry.CANNOT_INHERIT_FROM_FINAL + self.fail(err.format(base_expr.node.name).value, defn, code=err.code) + if not possible: + return False, None + existing_info = None + if isinstance(defn.analyzed, TypedDictExpr): + existing_info = defn.analyzed.info + if ( + len(defn.base_type_exprs) == 1 + and isinstance(defn.base_type_exprs[0], RefExpr) + and defn.base_type_exprs[0].fullname in TPDICT_NAMES + ): + # Building a new TypedDict + fields, types, statements, required_keys, readonly_keys = ( + self.analyze_typeddict_classdef_fields(defn) + ) + if fields is None: + return True, None # Defer + if self.api.is_func_scope() and "@" not in defn.name: + defn.name += "@" + str(defn.line) + info = self.build_typeddict_typeinfo( + defn.name, fields, types, required_keys, readonly_keys, defn.line, existing_info + ) + defn.analyzed = TypedDictExpr(info) + defn.analyzed.line = defn.line + defn.analyzed.column = defn.column + defn.defs.body = statements + return True, info + + # Extending/merging existing TypedDicts + typeddict_bases: list[Expression] = [] + typeddict_bases_set = set() + for expr in defn.base_type_exprs: + ok, maybe_type_info, _ = self.check_typeddict(expr, None, False) + if ok and maybe_type_info is not None: + # expr is a CallExpr + info = maybe_type_info + typeddict_bases_set.add(info.fullname) + typeddict_bases.append(expr) + elif isinstance(expr, RefExpr) and expr.fullname in TPDICT_NAMES: + if "TypedDict" not in typeddict_bases_set: + typeddict_bases_set.add("TypedDict") + else: + self.fail('Duplicate base class "TypedDict"', defn) + elif isinstance(expr, RefExpr) and self.is_typeddict(expr): + assert expr.fullname + if expr.fullname not in typeddict_bases_set: + typeddict_bases_set.add(expr.fullname) + typeddict_bases.append(expr) + else: + assert isinstance(expr.node, TypeInfo) + self.fail(f'Duplicate base class "{expr.node.name}"', defn) + elif isinstance(expr, IndexExpr) and self.is_typeddict(expr.base): + assert isinstance(expr.base, RefExpr) + assert expr.base.fullname + if expr.base.fullname not in typeddict_bases_set: + typeddict_bases_set.add(expr.base.fullname) + typeddict_bases.append(expr) + else: + assert isinstance(expr.base.node, TypeInfo) + self.fail(f'Duplicate base class "{expr.base.node.name}"', defn) + else: + self.fail("All bases of a new TypedDict must be TypedDict types", defn) + + keys: list[str] = [] + types = [] + required_keys = set() + readonly_keys = set() + # Iterate over bases in reverse order so that leftmost base class' keys take precedence + for base in reversed(typeddict_bases): + self.add_keys_and_types_from_base( + base, keys, types, required_keys, readonly_keys, defn + ) + (new_keys, new_types, new_statements, new_required_keys, new_readonly_keys) = ( + self.analyze_typeddict_classdef_fields(defn, keys) + ) + if new_keys is None: + return True, None # Defer + keys.extend(new_keys) + types.extend(new_types) + required_keys.update(new_required_keys) + readonly_keys.update(new_readonly_keys) + info = self.build_typeddict_typeinfo( + defn.name, keys, types, required_keys, readonly_keys, defn.line, existing_info + ) + defn.analyzed = TypedDictExpr(info) + defn.analyzed.line = defn.line + defn.analyzed.column = defn.column + defn.defs.body = new_statements + return True, info + + def add_keys_and_types_from_base( + self, + base: Expression, + keys: list[str], + types: list[Type], + required_keys: set[str], + readonly_keys: set[str], + ctx: Context, + ) -> None: + base_args: list[Type] = [] + if isinstance(base, RefExpr): + assert isinstance(base.node, TypeInfo) + info = base.node + elif isinstance(base, IndexExpr): + assert isinstance(base.base, RefExpr) + assert isinstance(base.base.node, TypeInfo) + info = base.base.node + args = self.analyze_base_args(base, ctx) + if args is None: + return + base_args = args + else: + assert isinstance(base, CallExpr) + assert isinstance(base.analyzed, TypedDictExpr) + info = base.analyzed.info + + assert info.typeddict_type is not None + base_typed_dict = info.typeddict_type + base_items = base_typed_dict.items + valid_items = base_items.copy() + + # Always fix invalid bases to avoid crashes. + tvars = info.defn.type_vars + if len(base_args) != len(tvars): + any_kind = TypeOfAny.from_omitted_generics + if base_args: + self.fail(f'Invalid number of type arguments for "{info.name}"', ctx) + any_kind = TypeOfAny.from_error + base_args = [AnyType(any_kind) for _ in tvars] + + with state.strict_optional_set(self.options.strict_optional): + valid_items = self.map_items_to_base(valid_items, tvars, base_args) + for key in base_items: + if key in keys: + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx) + keys.extend(valid_items.keys()) + types.extend(valid_items.values()) + required_keys.update(base_typed_dict.required_keys) + readonly_keys.update(base_typed_dict.readonly_keys) + + def analyze_base_args(self, base: IndexExpr, ctx: Context) -> list[Type] | None: + """Analyze arguments of base type expressions as types. + + We need to do this, because normal base class processing happens after + the TypedDict special-casing (plus we get a custom error message). + """ + base_args = [] + if isinstance(base.index, TupleExpr): + args = base.index.items + else: + args = [base.index] + + for arg_expr in args: + try: + type = expr_to_unanalyzed_type(arg_expr, self.options, self.api.is_stub_file) + except TypeTranslationError: + self.fail("Invalid TypedDict type argument", ctx) + return None + analyzed = self.api.anal_type( + type, + allow_typed_dict_special_forms=True, + allow_placeholder=not self.api.is_func_scope(), + ) + if analyzed is None: + return None + base_args.append(analyzed) + return base_args + + def map_items_to_base( + self, valid_items: dict[str, Type], tvars: list[TypeVarLikeType], base_args: list[Type] + ) -> dict[str, Type]: + """Map item types to how they would look in their base with type arguments applied. + + Note it is safe to use expand_type() during semantic analysis, because it should never + (indirectly) call is_subtype(). + """ + mapped_items = {} + for key in valid_items: + type_in_base = valid_items[key] + if not tvars: + mapped_items[key] = type_in_base + continue + # TODO: simple zip can't be used for variadic types. + mapped_items[key] = expand_type( + type_in_base, {t.id: a for (t, a) in zip(tvars, base_args)} + ) + return mapped_items + + def analyze_typeddict_classdef_fields( + self, defn: ClassDef, oldfields: list[str] | None = None + ) -> tuple[list[str] | None, list[Type], list[Statement], set[str], set[str]]: + """Analyze fields defined in a TypedDict class definition. + + This doesn't consider inherited fields (if any). Also consider totality, + if given. + + Return tuple with these items: + * List of keys (or None if found an incomplete reference --> deferral) + * List of types for each key + * List of statements from defn.defs.body that are legally allowed to be a + part of a TypedDict definition + * Set of required keys + """ + fields: list[str] = [] + types: list[Type] = [] + statements: list[Statement] = [] + for stmt in defn.defs.body: + if not isinstance(stmt, AssignmentStmt): + # Still allow pass or ... (for empty TypedDict's) and docstrings + if isinstance(stmt, PassStmt) or ( + isinstance(stmt, ExpressionStmt) + and isinstance(stmt.expr, (EllipsisExpr, StrExpr)) + ): + statements.append(stmt) + else: + defn.removed_statements.append(stmt) + self.fail(TPDICT_CLASS_ERROR, stmt) + elif len(stmt.lvalues) > 1 or not isinstance(stmt.lvalues[0], NameExpr): + # An assignment, but an invalid one. + defn.removed_statements.append(stmt) + self.fail(TPDICT_CLASS_ERROR, stmt) + else: + name = stmt.lvalues[0].name + if name in (oldfields or []): + self.fail(f'Overwriting TypedDict field "{name}" while extending', stmt) + if name in fields: + self.fail(f'Duplicate TypedDict key "{name}"', stmt) + continue + # Append stmt, name, and type in this case... + fields.append(name) + statements.append(stmt) + if stmt.unanalyzed_type is None: + types.append(AnyType(TypeOfAny.unannotated)) + else: + analyzed = self.api.anal_type( + stmt.unanalyzed_type, + allow_typed_dict_special_forms=True, + allow_placeholder=not self.api.is_func_scope(), + prohibit_self_type="TypedDict item type", + ) + if analyzed is None: + return None, [], [], set(), set() # Need to defer + types.append(analyzed) + if not has_placeholder(analyzed): + stmt.type = self.extract_meta_info(analyzed, stmt)[0] + # ...despite possible minor failures that allow further analysis. + if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax: + self.fail(TPDICT_CLASS_ERROR, stmt) + elif not isinstance(stmt.rvalue, TempNode): + # x: int assigns rvalue to TempNode(AnyType()) + self.fail("Right hand side values are not supported in TypedDict", stmt) + total: bool | None = True + if "total" in defn.keywords: + total = require_bool_literal_argument(self.api, defn.keywords["total"], "total", True) + if defn.keywords and defn.keywords.keys() != {"total"}: + for_function = ' for "__init_subclass__" of "TypedDict"' + for key in defn.keywords: + if key == "total": + continue + self.msg.unexpected_keyword_argument_for_function(for_function, key, defn) + + res_types = [] + readonly_keys = set() + required_keys = set() + for field, t in zip(fields, types): + typ, required, readonly = self.extract_meta_info(t) + res_types.append(typ) + if (total or required is True) and required is not False: + required_keys.add(field) + if readonly: + readonly_keys.add(field) + + return fields, res_types, statements, required_keys, readonly_keys + + def extract_meta_info( + self, typ: Type, context: Context | None = None + ) -> tuple[Type, bool | None, bool]: + """Unwrap all metadata types.""" + is_required = None # default, no modification + readonly = False # by default all is mutable + + seen_required = False + seen_readonly = False + while isinstance(typ, (RequiredType, ReadOnlyType)): + if isinstance(typ, RequiredType): + if context is not None and seen_required: + self.fail( + '"{}" type cannot be nested'.format( + "Required[]" if typ.required else "NotRequired[]" + ), + context, + code=codes.VALID_TYPE, + ) + is_required = typ.required + seen_required = True + typ = typ.item + if isinstance(typ, ReadOnlyType): + if context is not None and seen_readonly: + self.fail('"ReadOnly[]" type cannot be nested', context, code=codes.VALID_TYPE) + readonly = True + seen_readonly = True + typ = typ.item + return typ, is_required, readonly + + def check_typeddict( + self, node: Expression, var_name: str | None, is_func_scope: bool + ) -> tuple[bool, TypeInfo | None, list[TypeVarLikeType]]: + """Check if a call defines a TypedDict. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + Return a pair (is it a typed dict, corresponding TypeInfo). + + If the definition is invalid but looks like a TypedDict, + report errors but return (some) TypeInfo. If some type is not ready, + return (True, None). + """ + if not isinstance(node, CallExpr): + return False, None, [] + call = node + callee = call.callee + if not isinstance(callee, RefExpr): + return False, None, [] + fullname = callee.fullname + if fullname not in TPDICT_NAMES: + return False, None, [] + res = self.parse_typeddict_args(call) + if res is None: + # This is a valid typed dict, but some type is not ready. + # The caller should defer this until next iteration. + return True, None, [] + name, items, types, total, tvar_defs, ok = res + if not ok: + # Error. Construct dummy return value. + if var_name: + name = var_name + if is_func_scope: + name += "@" + str(call.line) + else: + name = var_name = "TypedDict@" + str(call.line) + info = self.build_typeddict_typeinfo(name, [], [], set(), set(), call.line, None) + else: + if var_name is not None and name != var_name: + self.fail( + 'First argument "{}" to TypedDict() does not match variable name "{}"'.format( + name, var_name + ), + node, + code=codes.NAME_MATCH, + ) + if name != var_name or is_func_scope: + # Give it a unique name derived from the line number. + name += "@" + str(call.line) + required_keys = { + field + for (field, t) in zip(items, types) + if (total or (isinstance(t, RequiredType) and t.required)) + and not (isinstance(t, RequiredType) and not t.required) + } + readonly_keys = { + field for (field, t) in zip(items, types) if isinstance(t, ReadOnlyType) + } + types = [ # unwrap Required[T] or ReadOnly[T] to just T + t.item if isinstance(t, (RequiredType, ReadOnlyType)) else t for t in types + ] + + # Perform various validations after unwrapping. + for t in types: + check_for_explicit_any( + t, self.options, self.api.is_typeshed_stub_file, self.msg, context=call + ) + if self.options.disallow_any_unimported: + for t in types: + if has_any_from_unimported_type(t): + self.msg.unimported_type_becomes_any("Type of a TypedDict key", t, call) + + existing_info = None + if isinstance(node.analyzed, TypedDictExpr): + existing_info = node.analyzed.info + info = self.build_typeddict_typeinfo( + name, items, types, required_keys, readonly_keys, call.line, existing_info + ) + info.line = node.line + # Store generated TypeInfo under both names, see semanal_namedtuple for more details. + if name != var_name or is_func_scope: + self.api.add_symbol_skip_local(name, info) + if var_name: + self.api.add_symbol(var_name, info, node) + call.analyzed = TypedDictExpr(info) + call.analyzed.set_line(call) + return True, info, tvar_defs + + def parse_typeddict_args( + self, call: CallExpr + ) -> tuple[str, list[str], list[Type], bool, list[TypeVarLikeType], bool] | None: + """Parse typed dict call expression. + + Return names, types, totality, was there an error during parsing. + If some type is not ready, return None. + """ + # TODO: Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_typeddict_arg("Too few arguments for TypedDict()", call) + if len(args) > 3: + return self.fail_typeddict_arg("Too many arguments for TypedDict()", call) + # TODO: Support keyword arguments + if call.arg_kinds not in ([ARG_POS, ARG_POS], [ARG_POS, ARG_POS, ARG_NAMED]): + return self.fail_typeddict_arg("Unexpected arguments to TypedDict()", call) + if len(args) == 3 and call.arg_names[2] != "total": + return self.fail_typeddict_arg( + f'Unexpected keyword argument "{call.arg_names[2]}" for "TypedDict"', call + ) + if not isinstance(args[0], StrExpr): + return self.fail_typeddict_arg( + "TypedDict() expects a string literal as the first argument", call + ) + if not isinstance(args[1], DictExpr): + return self.fail_typeddict_arg( + "TypedDict() expects a dictionary literal as the second argument", call + ) + total: bool | None = True + if len(args) == 3: + total = require_bool_literal_argument(self.api, call.args[2], "total") + if total is None: + return "", [], [], True, [], False + dictexpr = args[1] + tvar_defs = self.api.get_and_bind_all_tvars([t for k, t in dictexpr.items]) + res = self.parse_typeddict_fields_with_types(dictexpr.items, call) + if res is None: + # One of the types is not ready, defer. + return None + items, types, ok = res + assert total is not None + return args[0].value, items, types, total, tvar_defs, ok + + def parse_typeddict_fields_with_types( + self, dict_items: list[tuple[Expression | None, Expression]], context: Context + ) -> tuple[list[str], list[Type], bool] | None: + """Parse typed dict items passed as pairs (name expression, type expression). + + Return names, types, was there an error. If some type is not ready, return None. + """ + seen_keys = set() + items: list[str] = [] + types: list[Type] = [] + for field_name_expr, field_type_expr in dict_items: + if isinstance(field_name_expr, StrExpr): + key = field_name_expr.value + items.append(key) + if key in seen_keys: + self.fail(f'Duplicate TypedDict key "{key}"', field_name_expr) + seen_keys.add(key) + else: + name_context = field_name_expr or field_type_expr + self.fail_typeddict_arg("Invalid TypedDict() field name", name_context) + return [], [], False + try: + type = expr_to_unanalyzed_type( + field_type_expr, self.options, self.api.is_stub_file + ) + except TypeTranslationError: + self.fail_typeddict_arg("Use dict literal for nested TypedDict", field_type_expr) + return [], [], False + analyzed = self.api.anal_type( + type, + allow_typed_dict_special_forms=True, + allow_placeholder=not self.api.is_func_scope(), + prohibit_self_type="TypedDict item type", + ) + if analyzed is None: + return None + types.append(analyzed) + return items, types, True + + def fail_typeddict_arg( + self, message: str, context: Context + ) -> tuple[str, list[str], list[Type], bool, list[TypeVarLikeType], bool]: + self.fail(message, context) + return "", [], [], True, [], False + + def build_typeddict_typeinfo( + self, + name: str, + items: list[str], + types: list[Type], + required_keys: set[str], + readonly_keys: set[str], + line: int, + existing_info: TypeInfo | None, + ) -> TypeInfo: + # Prefer typing then typing_extensions if available. + fallback = ( + self.api.named_type_or_none("typing._TypedDict", []) + or self.api.named_type_or_none("typing_extensions._TypedDict", []) + or self.api.named_type_or_none("mypy_extensions._TypedDict", []) + ) + assert fallback is not None + info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) + typeddict_type = TypedDictType( + dict(zip(items, types)), required_keys, readonly_keys, fallback + ) + if info.special_alias and has_placeholder(info.special_alias.target): + self.api.process_placeholder( + None, "TypedDict item", info, force_progress=typeddict_type != info.typeddict_type + ) + info.update_typeddict_type(typeddict_type) + return info + + # Helpers + + def is_typeddict(self, expr: Expression) -> bool: + return ( + isinstance(expr, RefExpr) + and isinstance(expr.node, TypeInfo) + and expr.node.typeddict_type is not None + ) + + def fail(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + self.api.fail(msg, ctx, code=code) + + def note(self, msg: str, ctx: Context) -> None: + self.api.note(msg, ctx) diff --git a/mypy/codec/__init__.py b/mypy/server/__init__.py similarity index 100% rename from mypy/codec/__init__.py rename to mypy/server/__init__.py diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py new file mode 100644 index 000000000000..85f77a269e43 --- /dev/null +++ b/mypy/server/astdiff.py @@ -0,0 +1,532 @@ +"""Utilities for comparing two versions of a module symbol table. + +The goal is to find which AST nodes have externally visible changes, so +that we can fire triggers and re-process other parts of the program +that are stale because of the changes. + +Only look at detail at definitions at the current module -- don't +recurse into other modules. + +A summary of the module contents: + +* snapshot_symbol_table(...) creates an opaque snapshot description of a + module/class symbol table (recursing into nested class symbol tables). + +* compare_symbol_table_snapshots(...) compares two snapshots for the same + module id and returns fully qualified names of differences (which act as + triggers). + +To compare two versions of a module symbol table, take snapshots of both +versions and compare the snapshots. The use of snapshots makes it easy to +compare two versions of the *same* symbol table that is being mutated. + +Summary of how this works for certain kinds of differences: + +* If a symbol table node is deleted or added (only present in old/new version + of the symbol table), it is considered different, of course. + +* If a symbol table node refers to a different sort of thing in the new version, + it is considered different (for example, if a class is replaced with a + function). + +* If the signature of a function has changed, it is considered different. + +* If the type of a variable changes, it is considered different. + +* If the MRO of a class changes, or a non-generic class is turned into a + generic class, the class is considered different (there are other such "big" + differences that cause a class to be considered changed). However, just changes + to attributes or methods don't generally constitute a difference at the + class level -- these are handled at attribute level (say, 'mod.Cls.method' + is different rather than 'mod.Cls' being different). + +* If an imported name targets a different name (say, 'from x import y' is + replaced with 'from z import y'), the name in the module is considered + different. If the target of an import continues to have the same name, + but it's specifics change, this doesn't mean that the imported name is + treated as changed. Say, there is 'from x import y' in 'm', and the + type of 'x.y' has changed. This doesn't mean that that 'm.y' is considered + changed. Instead, processing the difference in 'm' will be handled through + fine-grained dependencies. +""" + +from __future__ import annotations + +from typing import Sequence, Tuple, Union +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.expandtype import expand_type +from mypy.nodes import ( + UNBOUND_IMPORTED, + Decorator, + FuncBase, + FuncDef, + FuncItem, + MypyFile, + OverloadedFuncDef, + ParamSpecExpr, + SymbolNode, + SymbolTable, + TypeAlias, + TypeInfo, + TypeVarExpr, + TypeVarTupleExpr, + Var, +) +from mypy.semanal_shared import find_dataclass_transform_spec +from mypy.state import state +from mypy.types import ( + AnyType, + CallableType, + DeletedType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, +) +from mypy.util import get_prefix + +# Snapshot representation of a symbol table node or type. The representation is +# opaque -- the only supported operations are comparing for equality and +# hashing (latter for type snapshots only). Snapshots can contain primitive +# objects, nested tuples, lists and dictionaries and primitive objects (type +# snapshots are immutable). +# +# For example, the snapshot of the 'int' type is ('Instance', 'builtins.int', ()). + +# Type snapshots are strict, they must be hashable and ordered (e.g. for Unions). +Primitive: _TypeAlias = Union[str, float, int, bool] # float is for Literal[3.14] support. +SnapshotItem: _TypeAlias = Tuple[Union[Primitive, "SnapshotItem"], ...] + +# Symbol snapshots can be more lenient. +SymbolSnapshot: _TypeAlias = Tuple[object, ...] + + +def compare_symbol_table_snapshots( + name_prefix: str, snapshot1: dict[str, SymbolSnapshot], snapshot2: dict[str, SymbolSnapshot] +) -> set[str]: + """Return names that are different in two snapshots of a symbol table. + + Only shallow (intra-module) differences are considered. References to things defined + outside the module are compared based on the name of the target only. + + Recurse into class symbol tables (if the class is defined in the target module). + + Return a set of fully-qualified names (e.g., 'mod.func' or 'mod.Class.method'). + """ + # Find names only defined only in one version. + names1 = {f"{name_prefix}.{name}" for name in snapshot1} + names2 = {f"{name_prefix}.{name}" for name in snapshot2} + triggers = names1 ^ names2 + + # Look for names defined in both versions that are different. + for name in set(snapshot1.keys()) & set(snapshot2.keys()): + item1 = snapshot1[name] + item2 = snapshot2[name] + kind1 = item1[0] + kind2 = item2[0] + item_name = f"{name_prefix}.{name}" + if kind1 != kind2: + # Different kind of node in two snapshots -> trivially different. + triggers.add(item_name) + elif kind1 == "TypeInfo": + if item1[:-1] != item2[:-1]: + # Record major difference (outside class symbol tables). + triggers.add(item_name) + # Look for differences in nested class symbol table entries. + assert isinstance(item1[-1], dict) + assert isinstance(item2[-1], dict) + triggers |= compare_symbol_table_snapshots(item_name, item1[-1], item2[-1]) + else: + # Shallow node (no interesting internal structure). Just use equality. + if snapshot1[name] != snapshot2[name]: + triggers.add(item_name) + + return triggers + + +def snapshot_symbol_table(name_prefix: str, table: SymbolTable) -> dict[str, SymbolSnapshot]: + """Create a snapshot description that represents the state of a symbol table. + + The snapshot has a representation based on nested tuples and dicts + that makes it easy and fast to find differences. + + Only "shallow" state is included in the snapshot -- references to + things defined in other modules are represented just by the names of + the targets. + """ + result: dict[str, SymbolSnapshot] = {} + for name, symbol in table.items(): + node = symbol.node + # TODO: cross_ref? + fullname = node.fullname if node else None + common = (fullname, symbol.kind, symbol.module_public) + if isinstance(node, MypyFile): + # This is a cross-reference to another module. + # If the reference is busted because the other module is missing, + # the node will be a "stale_info" TypeInfo produced by fixup, + # but that doesn't really matter to us here. + result[name] = ("Moduleref", common) + elif isinstance(node, TypeVarExpr): + result[name] = ( + "TypeVar", + node.variance, + [snapshot_type(value) for value in node.values], + snapshot_type(node.upper_bound), + snapshot_type(node.default), + ) + elif isinstance(node, TypeAlias): + result[name] = ( + "TypeAlias", + snapshot_types(node.alias_tvars), + node.normalized, + node.no_args, + snapshot_optional_type(node.target), + ) + elif isinstance(node, ParamSpecExpr): + result[name] = ( + "ParamSpec", + node.variance, + snapshot_type(node.upper_bound), + snapshot_type(node.default), + ) + elif isinstance(node, TypeVarTupleExpr): + result[name] = ( + "TypeVarTuple", + node.variance, + snapshot_type(node.upper_bound), + snapshot_type(node.default), + ) + else: + assert symbol.kind != UNBOUND_IMPORTED + if node and get_prefix(node.fullname) != name_prefix: + # This is a cross-reference to a node defined in another module. + # Include the node kind (FuncDef, Decorator, TypeInfo, ...), so that we will + # reprocess when a *new* node is created instead of merging an existing one. + result[name] = ("CrossRef", common, type(node).__name__) + else: + result[name] = snapshot_definition(node, common) + return result + + +def snapshot_definition(node: SymbolNode | None, common: SymbolSnapshot) -> SymbolSnapshot: + """Create a snapshot description of a symbol table node. + + The representation is nested tuples and dicts. Only externally + visible attributes are included. + """ + if isinstance(node, FuncBase): + # TODO: info + if node.type: + signature = snapshot_type(node.type) + else: + signature = snapshot_untyped_signature(node) + impl: FuncDef | None = None + if isinstance(node, FuncDef): + impl = node + elif isinstance(node, OverloadedFuncDef) and node.impl: + impl = node.impl.func if isinstance(node.impl, Decorator) else node.impl + is_trivial_body = impl.is_trivial_body if impl else False + dataclass_transform_spec = find_dataclass_transform_spec(node) + return ( + "Func", + common, + node.is_property, + node.is_final, + node.is_class, + node.is_static, + signature, + is_trivial_body, + dataclass_transform_spec.serialize() if dataclass_transform_spec is not None else None, + node.deprecated if isinstance(node, FuncDef) else None, + ) + elif isinstance(node, Var): + return ("Var", common, snapshot_optional_type(node.type), node.is_final) + elif isinstance(node, Decorator): + # Note that decorated methods are represented by Decorator instances in + # a symbol table since we need to preserve information about the + # decorated function (whether it's a class function, for + # example). Top-level decorated functions, however, are represented by + # the corresponding Var node, since that happens to provide enough + # context. + return ( + "Decorator", + node.is_overload, + snapshot_optional_type(node.var.type), + snapshot_definition(node.func, common), + ) + elif isinstance(node, TypeInfo): + dataclass_transform_spec = node.dataclass_transform_spec + if dataclass_transform_spec is None: + dataclass_transform_spec = find_dataclass_transform_spec(node) + + attrs = ( + node.is_abstract, + node.is_enum, + node.is_protocol, + node.fallback_to_any, + node.meta_fallback_to_any, + node.is_named_tuple, + node.is_newtype, + # We need this to e.g. trigger metaclass calculation in subclasses. + snapshot_optional_type(node.metaclass_type), + snapshot_optional_type(node.tuple_type), + snapshot_optional_type(node.typeddict_type), + [base.fullname for base in node.mro], + # Note that the structure of type variables is a part of the external interface, + # since creating instances might fail, for example: + # T = TypeVar('T', bound=int) + # class C(Generic[T]): + # ... + # x: C[str] <- this is invalid, and needs to be re-checked if `T` changes. + # An alternative would be to create both deps: <...> -> C, and <...> -> , + # but this currently seems a bit ad hoc. + tuple(snapshot_type(tdef) for tdef in node.defn.type_vars), + [snapshot_type(base) for base in node.bases], + [snapshot_type(p) for p in node._promote], + dataclass_transform_spec.serialize() if dataclass_transform_spec is not None else None, + node.deprecated, + ) + prefix = node.fullname + symbol_table = snapshot_symbol_table(prefix, node.names) + # Special dependency for abstract attribute handling. + symbol_table["(abstract)"] = ("Abstract", tuple(sorted(node.abstract_attributes))) + return ("TypeInfo", common, attrs, symbol_table) + else: + # Other node types are handled elsewhere. + assert False, type(node) + + +def snapshot_type(typ: Type) -> SnapshotItem: + """Create a snapshot representation of a type using nested tuples.""" + return typ.accept(SnapshotTypeVisitor()) + + +def snapshot_optional_type(typ: Type | None) -> SnapshotItem: + if typ: + return snapshot_type(typ) + else: + return ("",) + + +def snapshot_types(types: Sequence[Type]) -> SnapshotItem: + return tuple(snapshot_type(item) for item in types) + + +def snapshot_simple_type(typ: Type) -> SnapshotItem: + return (type(typ).__name__,) + + +def encode_optional_str(s: str | None) -> str: + if s is None: + return "" + else: + return s + + +class SnapshotTypeVisitor(TypeVisitor[SnapshotItem]): + """Creates a read-only, self-contained snapshot of a type object. + + Properties of a snapshot: + + - Contains (nested) tuples and other immutable primitive objects only. + - References to AST nodes are replaced with full names of targets. + - Has no references to mutable or non-primitive objects. + - Two snapshots represent the same object if and only if they are + equal. + - Results must be sortable. It's important that tuples have + consistent types and can't arbitrarily mix str and None values, + for example, since they can't be compared. + """ + + def visit_unbound_type(self, typ: UnboundType) -> SnapshotItem: + return ( + "UnboundType", + typ.name, + typ.optional, + typ.empty_tuple_index, + snapshot_types(typ.args), + ) + + def visit_any(self, typ: AnyType) -> SnapshotItem: + return snapshot_simple_type(typ) + + def visit_none_type(self, typ: NoneType) -> SnapshotItem: + return snapshot_simple_type(typ) + + def visit_uninhabited_type(self, typ: UninhabitedType) -> SnapshotItem: + return snapshot_simple_type(typ) + + def visit_erased_type(self, typ: ErasedType) -> SnapshotItem: + return snapshot_simple_type(typ) + + def visit_deleted_type(self, typ: DeletedType) -> SnapshotItem: + return snapshot_simple_type(typ) + + def visit_instance(self, typ: Instance) -> SnapshotItem: + extra_attrs: SnapshotItem + if typ.extra_attrs: + extra_attrs = ( + tuple(sorted((k, v.accept(self)) for k, v in typ.extra_attrs.attrs.items())), + tuple(typ.extra_attrs.immutable), + ) + else: + extra_attrs = () + return ( + "Instance", + encode_optional_str(typ.type.fullname), + snapshot_types(typ.args), + ("None",) if typ.last_known_value is None else snapshot_type(typ.last_known_value), + extra_attrs, + ) + + def visit_type_var(self, typ: TypeVarType) -> SnapshotItem: + return ( + "TypeVar", + typ.name, + typ.fullname, + typ.id.raw_id, + typ.id.meta_level, + snapshot_types(typ.values), + snapshot_type(typ.upper_bound), + snapshot_type(typ.default), + typ.variance, + ) + + def visit_param_spec(self, typ: ParamSpecType) -> SnapshotItem: + return ( + "ParamSpec", + typ.id.raw_id, + typ.id.meta_level, + typ.flavor, + snapshot_type(typ.upper_bound), + snapshot_type(typ.default), + ) + + def visit_type_var_tuple(self, typ: TypeVarTupleType) -> SnapshotItem: + return ( + "TypeVarTupleType", + typ.id.raw_id, + typ.id.meta_level, + snapshot_type(typ.upper_bound), + snapshot_type(typ.default), + ) + + def visit_unpack_type(self, typ: UnpackType) -> SnapshotItem: + return ("UnpackType", snapshot_type(typ.type)) + + def visit_parameters(self, typ: Parameters) -> SnapshotItem: + return ( + "Parameters", + snapshot_types(typ.arg_types), + tuple(encode_optional_str(name) for name in typ.arg_names), + tuple(k.value for k in typ.arg_kinds), + ) + + def visit_callable_type(self, typ: CallableType) -> SnapshotItem: + if typ.is_generic(): + typ = self.normalize_callable_variables(typ) + return ( + "CallableType", + snapshot_types(typ.arg_types), + snapshot_type(typ.ret_type), + tuple(encode_optional_str(name) for name in typ.arg_names), + tuple(k.value for k in typ.arg_kinds), + typ.is_type_obj(), + typ.is_ellipsis_args, + snapshot_types(typ.variables), + ) + + def normalize_callable_variables(self, typ: CallableType) -> CallableType: + """Normalize all type variable ids to run from -1 to -len(variables).""" + tvs = [] + tvmap: dict[TypeVarId, Type] = {} + for i, v in enumerate(typ.variables): + tid = TypeVarId(-1 - i) + if isinstance(v, TypeVarType): + tv: TypeVarLikeType = v.copy_modified(id=tid) + elif isinstance(v, TypeVarTupleType): + tv = v.copy_modified(id=tid) + else: + assert isinstance(v, ParamSpecType) + tv = v.copy_modified(id=tid) + tvs.append(tv) + tvmap[v.id] = tv + with state.strict_optional_set(True): + return expand_type(typ, tvmap).copy_modified(variables=tvs) + + def visit_tuple_type(self, typ: TupleType) -> SnapshotItem: + return ("TupleType", snapshot_types(typ.items)) + + def visit_typeddict_type(self, typ: TypedDictType) -> SnapshotItem: + items = tuple((key, snapshot_type(item_type)) for key, item_type in typ.items.items()) + required = tuple(sorted(typ.required_keys)) + readonly = tuple(sorted(typ.readonly_keys)) + return ("TypedDictType", items, required, readonly) + + def visit_literal_type(self, typ: LiteralType) -> SnapshotItem: + return ("LiteralType", snapshot_type(typ.fallback), typ.value) + + def visit_union_type(self, typ: UnionType) -> SnapshotItem: + # Sort and remove duplicates so that we can use equality to test for + # equivalent union type snapshots. + items = {snapshot_type(item) for item in typ.items} + normalized = tuple(sorted(items)) + return ("UnionType", normalized) + + def visit_overloaded(self, typ: Overloaded) -> SnapshotItem: + return ("Overloaded", snapshot_types(typ.items)) + + def visit_partial_type(self, typ: PartialType) -> SnapshotItem: + # A partial type is not fully defined, so the result is indeterminate. We shouldn't + # get here. + raise RuntimeError + + def visit_type_type(self, typ: TypeType) -> SnapshotItem: + return ("TypeType", snapshot_type(typ.item)) + + def visit_type_alias_type(self, typ: TypeAliasType) -> SnapshotItem: + assert typ.alias is not None + return ("TypeAliasType", typ.alias.fullname, snapshot_types(typ.args)) + + +def snapshot_untyped_signature(func: OverloadedFuncDef | FuncItem) -> SymbolSnapshot: + """Create a snapshot of the signature of a function that has no explicit signature. + + If the arguments to a function without signature change, it must be + considered as different. We have this special casing since we don't store + the implicit signature anywhere, and we'd rather not construct new + Callable objects in this module (the idea is to only read properties of + the AST here). + """ + if isinstance(func, FuncItem): + return (tuple(func.arg_names), tuple(func.arg_kinds)) + else: + result: list[SymbolSnapshot] = [] + for item in func.items: + if isinstance(item, Decorator): + if item.var.type: + result.append(snapshot_type(item.var.type)) + else: + result.append(("DecoratorWithoutType",)) + else: + result.append(snapshot_untyped_signature(item)) + return tuple(result) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py new file mode 100644 index 000000000000..5dc254422328 --- /dev/null +++ b/mypy/server/astmerge.py @@ -0,0 +1,562 @@ +"""Merge a new version of a module AST and symbol table to older versions of those. + +When the source code of a module has a change in fine-grained incremental mode, +we build a new AST from the updated source. However, other parts of the program +may have direct references to parts of the old AST (namely, those nodes exposed +in the module symbol table). The merge operation changes the identities of new +AST nodes that have a correspondence in the old AST to the old ones so that +existing cross-references in other modules will continue to point to the correct +nodes. Also internal cross-references within the new AST are replaced. AST nodes +that aren't externally visible will get new, distinct object identities. This +applies to most expression and statement nodes, for example. + +We perform this merge operation so that we don't have to update all +external references (which would be slow and fragile) or always perform +translation when looking up references (which would be hard to retrofit). + +The AST merge operation is performed after semantic analysis. Semantic +analysis has to deal with potentially multiple aliases to certain AST +nodes (in particular, MypyFile nodes). Type checking assumes that we +don't have multiple variants of a single AST node visible to the type +checker. + +Discussion of some notable special cases: + +* If a node is replaced with a different kind of node (say, a function is + replaced with a class), we don't perform the merge. Fine-grained dependencies + will be used to rebind all references to the node. + +* If a function is replaced with another function with an identical signature, + call sites continue to point to the same object (by identity) and don't need + to be reprocessed. Similarly, if a class is replaced with a class that is + sufficiently similar (MRO preserved, etc.), class references don't need any + processing. A typical incremental update to a file only changes a few + externally visible things in a module, and this means that often only few + external references need any processing, even if the modified module is large. + +* A no-op update of a module should not require any processing outside the + module, since all relevant object identities are preserved. + +* The AST diff operation (mypy.server.astdiff) and the top-level fine-grained + incremental logic (mypy.server.update) handle the cases where the new AST has + differences from the old one that may need to be propagated to elsewhere in the + program. + +See the main entry point merge_asts for more details. +""" + +from __future__ import annotations + +from typing import TypeVar, cast + +from mypy.nodes import ( + MDEF, + AssertTypeExpr, + AssignmentStmt, + Block, + CallExpr, + CastExpr, + ClassDef, + EnumCallExpr, + FuncBase, + FuncDef, + LambdaExpr, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + NewTypeExpr, + OverloadedFuncDef, + RefExpr, + Statement, + SuperExpr, + SymbolNode, + SymbolTable, + TypeAlias, + TypedDictExpr, + TypeInfo, + Var, +) +from mypy.traverser import TraverserVisitor +from mypy.types import ( + AnyType, + CallableArgument, + CallableType, + DeletedType, + EllipsisType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + PlaceholderType, + RawExpressionType, + SyntheticTypeVisitor, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeList, + TypeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, +) +from mypy.typestate import type_state +from mypy.util import get_prefix, replace_object_state + + +def merge_asts( + old: MypyFile, old_symbols: SymbolTable, new: MypyFile, new_symbols: SymbolTable +) -> None: + """Merge a new version of a module AST to a previous version. + + The main idea is to preserve the identities of externally visible + nodes in the old AST (that have a corresponding node in the new AST). + All old node state (outside identity) will come from the new AST. + + When this returns, 'old' will refer to the merged AST, but 'new_symbols' + will be the new symbol table. 'new' and 'old_symbols' will no longer be + valid. + """ + assert new.fullname == old.fullname + # Find the mapping from new to old node identities for all nodes + # whose identities should be preserved. + replacement_map = replacement_map_from_symbol_table( + old_symbols, new_symbols, prefix=old.fullname + ) + # Also replace references to the new MypyFile node. + replacement_map[new] = old + # Perform replacements to everywhere within the new AST (not including symbol + # tables). + node = replace_nodes_in_ast(new, replacement_map) + assert node is old + # Also replace AST node references in the *new* symbol table (we'll + # continue to use the new symbol table since it has all the new definitions + # that have no correspondence in the old AST). + replace_nodes_in_symbol_table(new_symbols, replacement_map) + + +def replacement_map_from_symbol_table( + old: SymbolTable, new: SymbolTable, prefix: str +) -> dict[SymbolNode, SymbolNode]: + """Create a new-to-old object identity map by comparing two symbol table revisions. + + Both symbol tables must refer to revisions of the same module id. The symbol tables + are compared recursively (recursing into nested class symbol tables), but only within + the given module prefix. Don't recurse into other modules accessible through the symbol + table. + """ + replacements: dict[SymbolNode, SymbolNode] = {} + for name, node in old.items(): + if name in new and ( + node.kind == MDEF or node.node and get_prefix(node.node.fullname) == prefix + ): + new_node = new[name] + if ( + type(new_node.node) == type(node.node) + and new_node.node + and node.node + and new_node.node.fullname == node.node.fullname + and new_node.kind == node.kind + ): + replacements[new_node.node] = node.node + if isinstance(node.node, TypeInfo) and isinstance(new_node.node, TypeInfo): + type_repl = replacement_map_from_symbol_table( + node.node.names, new_node.node.names, prefix + ) + replacements.update(type_repl) + if node.node.special_alias and new_node.node.special_alias: + replacements[new_node.node.special_alias] = node.node.special_alias + return replacements + + +def replace_nodes_in_ast( + node: SymbolNode, replacements: dict[SymbolNode, SymbolNode] +) -> SymbolNode: + """Replace all references to replacement map keys within an AST node, recursively. + + Also replace the *identity* of any nodes that have replacements. Return the + *replaced* version of the argument node (which may have a different identity, if + it's included in the replacement map). + """ + visitor = NodeReplaceVisitor(replacements) + node.accept(visitor) + return replacements.get(node, node) + + +SN = TypeVar("SN", bound=SymbolNode) + + +class NodeReplaceVisitor(TraverserVisitor): + """Transform some nodes to new identities in an AST. + + Only nodes that live in the symbol table may be + replaced, which simplifies the implementation some. Also + replace all references to the old identities. + """ + + def __init__(self, replacements: dict[SymbolNode, SymbolNode]) -> None: + self.replacements = replacements + + def visit_mypy_file(self, node: MypyFile) -> None: + node = self.fixup(node) + node.defs = self.replace_statements(node.defs) + super().visit_mypy_file(node) + + def visit_block(self, node: Block) -> None: + node.body = self.replace_statements(node.body) + super().visit_block(node) + + def visit_func_def(self, node: FuncDef) -> None: + node = self.fixup(node) + self.process_base_func(node) + super().visit_func_def(node) + + def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> None: + self.process_base_func(node) + super().visit_overloaded_func_def(node) + + def visit_class_def(self, node: ClassDef) -> None: + # TODO additional things? + node.info = self.fixup_and_reset_typeinfo(node.info) + node.defs.body = self.replace_statements(node.defs.body) + info = node.info + for tv in node.type_vars: + if isinstance(tv, TypeVarType): + self.process_type_var_def(tv) + if info: + if info.is_named_tuple: + self.process_synthetic_type_info(info) + else: + self.process_type_info(info) + super().visit_class_def(node) + + def process_base_func(self, node: FuncBase) -> None: + self.fixup_type(node.type) + node.info = self.fixup(node.info) + if node.unanalyzed_type: + # Unanalyzed types can have AST node references + self.fixup_type(node.unanalyzed_type) + + def process_type_var_def(self, tv: TypeVarType) -> None: + for value in tv.values: + self.fixup_type(value) + self.fixup_type(tv.upper_bound) + self.fixup_type(tv.default) + + def process_param_spec_def(self, tv: ParamSpecType) -> None: + self.fixup_type(tv.upper_bound) + self.fixup_type(tv.default) + + def process_type_var_tuple_def(self, tv: TypeVarTupleType) -> None: + self.fixup_type(tv.upper_bound) + self.fixup_type(tv.default) + + def visit_assignment_stmt(self, node: AssignmentStmt) -> None: + self.fixup_type(node.type) + super().visit_assignment_stmt(node) + + # Expressions + + def visit_name_expr(self, node: NameExpr) -> None: + self.visit_ref_expr(node) + + def visit_member_expr(self, node: MemberExpr) -> None: + if node.def_var: + node.def_var = self.fixup(node.def_var) + self.visit_ref_expr(node) + super().visit_member_expr(node) + + def visit_ref_expr(self, node: RefExpr) -> None: + if node.node is not None: + node.node = self.fixup(node.node) + if isinstance(node.node, Var): + # The Var node may be an orphan and won't otherwise be processed. + node.node.accept(self) + + def visit_namedtuple_expr(self, node: NamedTupleExpr) -> None: + super().visit_namedtuple_expr(node) + node.info = self.fixup_and_reset_typeinfo(node.info) + self.process_synthetic_type_info(node.info) + + def visit_cast_expr(self, node: CastExpr) -> None: + super().visit_cast_expr(node) + self.fixup_type(node.type) + + def visit_assert_type_expr(self, node: AssertTypeExpr) -> None: + super().visit_assert_type_expr(node) + self.fixup_type(node.type) + + def visit_super_expr(self, node: SuperExpr) -> None: + super().visit_super_expr(node) + if node.info is not None: + node.info = self.fixup(node.info) + + def visit_call_expr(self, node: CallExpr) -> None: + super().visit_call_expr(node) + if isinstance(node.analyzed, SymbolNode): + node.analyzed = self.fixup(node.analyzed) + + def visit_newtype_expr(self, node: NewTypeExpr) -> None: + if node.info: + node.info = self.fixup_and_reset_typeinfo(node.info) + self.process_synthetic_type_info(node.info) + self.fixup_type(node.old_type) + super().visit_newtype_expr(node) + + def visit_lambda_expr(self, node: LambdaExpr) -> None: + node.info = self.fixup(node.info) + super().visit_lambda_expr(node) + + def visit_typeddict_expr(self, node: TypedDictExpr) -> None: + super().visit_typeddict_expr(node) + node.info = self.fixup_and_reset_typeinfo(node.info) + self.process_synthetic_type_info(node.info) + + def visit_enum_call_expr(self, node: EnumCallExpr) -> None: + node.info = self.fixup_and_reset_typeinfo(node.info) + self.process_synthetic_type_info(node.info) + super().visit_enum_call_expr(node) + + # Others + + def visit_var(self, node: Var) -> None: + node.info = self.fixup(node.info) + self.fixup_type(node.type) + super().visit_var(node) + + def visit_type_alias(self, node: TypeAlias) -> None: + self.fixup_type(node.target) + for v in node.alias_tvars: + self.fixup_type(v) + super().visit_type_alias(node) + + # Helpers + + def fixup(self, node: SN) -> SN: + if node in self.replacements: + new = self.replacements[node] + skip_slots: tuple[str, ...] = () + if isinstance(node, TypeInfo) and isinstance(new, TypeInfo): + # Special case: special_alias is not exposed in symbol tables, but may appear + # in external types (e.g. named tuples), so we need to update it manually. + skip_slots = ("special_alias",) + replace_object_state(new.special_alias, node.special_alias) + replace_object_state(new, node, skip_slots=skip_slots) + return cast(SN, new) + return node + + def fixup_and_reset_typeinfo(self, node: TypeInfo) -> TypeInfo: + """Fix-up type info and reset subtype caches. + + This needs to be called at least once per each merged TypeInfo, as otherwise we + may leak stale caches. + """ + if node in self.replacements: + # The subclass relationships may change, so reset all caches relevant to the + # old MRO. + new = self.replacements[node] + assert isinstance(new, TypeInfo) + type_state.reset_all_subtype_caches_for(new) + return self.fixup(node) + + def fixup_type(self, typ: Type | None) -> None: + if typ is not None: + typ.accept(TypeReplaceVisitor(self.replacements)) + + def process_type_info(self, info: TypeInfo | None) -> None: + if info is None: + return + self.fixup_type(info.declared_metaclass) + self.fixup_type(info.metaclass_type) + for target in info._promote: + self.fixup_type(target) + self.fixup_type(info.tuple_type) + self.fixup_type(info.typeddict_type) + if info.special_alias: + self.fixup_type(info.special_alias.target) + info.defn.info = self.fixup(info) + replace_nodes_in_symbol_table(info.names, self.replacements) + for i, item in enumerate(info.mro): + info.mro[i] = self.fixup(info.mro[i]) + for i, base in enumerate(info.bases): + self.fixup_type(info.bases[i]) + + def process_synthetic_type_info(self, info: TypeInfo) -> None: + # Synthetic types (types not created using a class statement) don't + # have bodies in the AST so we need to iterate over their symbol + # tables separately, unlike normal classes. + self.process_type_info(info) + for node in info.names.values(): + if node.node: + node.node.accept(self) + + def replace_statements(self, nodes: list[Statement]) -> list[Statement]: + result = [] + for node in nodes: + if isinstance(node, SymbolNode): + node = self.fixup(node) + result.append(node) + return result + + +class TypeReplaceVisitor(SyntheticTypeVisitor[None]): + """Similar to NodeReplaceVisitor, but for type objects. + + Note: this visitor may sometimes visit unanalyzed types + such as 'UnboundType' and 'RawExpressionType' For example, see + NodeReplaceVisitor.process_base_func. + """ + + def __init__(self, replacements: dict[SymbolNode, SymbolNode]) -> None: + self.replacements = replacements + + def visit_instance(self, typ: Instance) -> None: + typ.type = self.fixup(typ.type) + for arg in typ.args: + arg.accept(self) + if typ.last_known_value: + typ.last_known_value.accept(self) + + def visit_type_alias_type(self, typ: TypeAliasType) -> None: + assert typ.alias is not None + typ.alias = self.fixup(typ.alias) + for arg in typ.args: + arg.accept(self) + + def visit_any(self, typ: AnyType) -> None: + pass + + def visit_none_type(self, typ: NoneType) -> None: + pass + + def visit_callable_type(self, typ: CallableType) -> None: + for arg in typ.arg_types: + arg.accept(self) + typ.ret_type.accept(self) + if typ.definition: + # No need to fixup since this is just a cross-reference. + typ.definition = self.replacements.get(typ.definition, typ.definition) + # Fallback can be None for callable types that haven't been semantically analyzed. + if typ.fallback is not None: + typ.fallback.accept(self) + for tv in typ.variables: + if isinstance(tv, TypeVarType): + tv.upper_bound.accept(self) + for value in tv.values: + value.accept(self) + + def visit_overloaded(self, t: Overloaded) -> None: + for item in t.items: + item.accept(self) + # Fallback can be None for overloaded types that haven't been semantically analyzed. + if t.fallback is not None: + t.fallback.accept(self) + + def visit_erased_type(self, t: ErasedType) -> None: + # This type should exist only temporarily during type inference + raise RuntimeError("Cannot handle erased type") + + def visit_deleted_type(self, typ: DeletedType) -> None: + pass + + def visit_partial_type(self, typ: PartialType) -> None: + raise RuntimeError("Cannot handle partial type") + + def visit_tuple_type(self, typ: TupleType) -> None: + for item in typ.items: + item.accept(self) + # Fallback can be None for implicit tuple types that haven't been semantically analyzed. + if typ.partial_fallback is not None: + typ.partial_fallback.accept(self) + + def visit_type_type(self, typ: TypeType) -> None: + typ.item.accept(self) + + def visit_type_var(self, typ: TypeVarType) -> None: + typ.upper_bound.accept(self) + typ.default.accept(self) + for value in typ.values: + value.accept(self) + + def visit_param_spec(self, typ: ParamSpecType) -> None: + typ.upper_bound.accept(self) + typ.default.accept(self) + + def visit_type_var_tuple(self, typ: TypeVarTupleType) -> None: + typ.upper_bound.accept(self) + typ.default.accept(self) + + def visit_unpack_type(self, typ: UnpackType) -> None: + typ.type.accept(self) + + def visit_parameters(self, typ: Parameters) -> None: + for arg in typ.arg_types: + arg.accept(self) + + def visit_typeddict_type(self, typ: TypedDictType) -> None: + for value_type in typ.items.values(): + value_type.accept(self) + typ.fallback.accept(self) + + def visit_raw_expression_type(self, t: RawExpressionType) -> None: + pass + + def visit_literal_type(self, typ: LiteralType) -> None: + typ.fallback.accept(self) + + def visit_unbound_type(self, typ: UnboundType) -> None: + for arg in typ.args: + arg.accept(self) + + def visit_type_list(self, typ: TypeList) -> None: + for item in typ.items: + item.accept(self) + + def visit_callable_argument(self, typ: CallableArgument) -> None: + typ.typ.accept(self) + + def visit_ellipsis_type(self, typ: EllipsisType) -> None: + pass + + def visit_uninhabited_type(self, typ: UninhabitedType) -> None: + pass + + def visit_union_type(self, typ: UnionType) -> None: + for item in typ.items: + item.accept(self) + + def visit_placeholder_type(self, t: PlaceholderType) -> None: + for item in t.args: + item.accept(self) + + # Helpers + + def fixup(self, node: SN) -> SN: + if node in self.replacements: + new = self.replacements[node] + return cast(SN, new) + return node + + +def replace_nodes_in_symbol_table( + symbols: SymbolTable, replacements: dict[SymbolNode, SymbolNode] +) -> None: + for node in symbols.values(): + if node.node: + if node.node in replacements: + new = replacements[node.node] + old = node.node + # Needed for TypeInfo, see comment in fixup() above. + replace_object_state(new, old, skip_slots=("special_alias",)) + node.node = new + if isinstance(node.node, (Var, TypeAlias)): + # Handle them here just in case these aren't exposed through the AST. + node.node.accept(NodeReplaceVisitor(replacements)) diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py new file mode 100644 index 000000000000..05af6a3d53a1 --- /dev/null +++ b/mypy/server/aststrip.py @@ -0,0 +1,281 @@ +"""Strip/reset AST in-place to match state after semantic analyzer pre-analysis. + +Fine-grained incremental mode reruns semantic analysis main pass +and type checking for *existing* AST nodes (targets) when changes are +propagated using fine-grained dependencies. AST nodes attributes are +sometimes changed during semantic analysis main pass, and running +semantic analysis again on those nodes would produce incorrect +results, since this pass isn't idempotent. This pass resets AST +nodes to reflect the state after semantic pre-analysis, so that we +can rerun semantic analysis. +(The above is in contrast to behavior with modules that have source code +changes, for which we re-parse the entire module and reconstruct a fresh +AST. No stripping is required in this case. Both modes of operation should +have the same outcome.) +Notes: +* This is currently pretty fragile, as we must carefully undo whatever + changes can be made in semantic analysis main pass, including changes + to symbol tables. +* We reuse existing AST nodes because it makes it relatively straightforward + to reprocess only a single target within a module efficiently. If there + was a way to parse a single target within a file, in time proportional to + the size of the target, we'd rather create fresh AST nodes than strip them. + (This is possible only in Python 3.8+) +* Currently we don't actually reset all changes, but only those known to affect + non-idempotent semantic analysis behavior. + TODO: It would be more principled and less fragile to reset everything + changed in semantic analysis main pass and later. +* Reprocessing may recreate AST nodes (such as Var nodes, and TypeInfo nodes + created with assignment statements) that will get different identities from + the original AST. Thus running an AST merge is necessary after stripping, + even though some identities are preserved. +""" + +from __future__ import annotations + +from contextlib import contextmanager, nullcontext +from typing import Dict, Iterator, Tuple +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.nodes import ( + CLASSDEF_NO_INFO, + AssignmentStmt, + Block, + CallExpr, + ClassDef, + Decorator, + ForStmt, + FuncDef, + ImportAll, + ImportFrom, + IndexExpr, + ListExpr, + MemberExpr, + MypyFile, + NameExpr, + Node, + OpExpr, + OverloadedFuncDef, + RefExpr, + StarExpr, + SuperExpr, + SymbolTableNode, + TupleExpr, + TypeInfo, + Var, +) +from mypy.traverser import TraverserVisitor +from mypy.types import CallableType +from mypy.typestate import type_state + +SavedAttributes: _TypeAlias = Dict[Tuple[ClassDef, str], SymbolTableNode] + + +def strip_target( + node: MypyFile | FuncDef | OverloadedFuncDef, saved_attrs: SavedAttributes +) -> None: + """Reset a fine-grained incremental target to state before semantic analysis. + + All TypeInfos are killed. Therefore we need to preserve the variables + defined as attributes on self. This is done by patches (callbacks) + returned from this function that re-add these variables when called. + + Args: + node: node to strip + saved_attrs: collect attributes here that may need to be re-added to + classes afterwards if stripping a class body (this dict is mutated) + """ + visitor = NodeStripVisitor(saved_attrs) + if isinstance(node, MypyFile): + visitor.strip_file_top_level(node) + else: + node.accept(visitor) + + +class NodeStripVisitor(TraverserVisitor): + def __init__(self, saved_class_attrs: SavedAttributes) -> None: + # The current active class. + self.type: TypeInfo | None = None + # This is True at class scope, but not in methods. + self.is_class_body = False + # By default, process function definitions. If False, don't -- this is used for + # processing module top levels. + self.recurse_into_functions = True + # These attributes were removed from top-level classes during strip and + # will be added afterwards (if no existing definition is found). These + # must be added back before semantically analyzing any methods. + self.saved_class_attrs = saved_class_attrs + + def strip_file_top_level(self, file_node: MypyFile) -> None: + """Strip a module top-level (don't recursive into functions).""" + self.recurse_into_functions = False + file_node.plugin_deps.clear() + file_node.accept(self) + for name in file_node.names.copy(): + # TODO: this is a hot fix, we should delete all names, + # see https://github.com/python/mypy/issues/6422. + if "@" not in name: + del file_node.names[name] + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_class_def(self, node: ClassDef) -> None: + """Strip class body and type info, but don't strip methods.""" + # We need to save the implicitly defined instance variables, + # i.e. those defined as attributes on self. Otherwise, they would + # be lost if we only reprocess top-levels (this kills TypeInfos) + # but not the methods that defined those variables. + if not self.recurse_into_functions: + self.save_implicit_attributes(node) + # We need to delete any entries that were generated by plugins, + # since they will get regenerated. + to_delete = {v.node for v in node.info.names.values() if v.plugin_generated} + node.type_vars = [] + node.base_type_exprs.extend(node.removed_base_type_exprs) + node.removed_base_type_exprs = [] + node.defs.body = [ + s for s in node.defs.body if s not in to_delete # type: ignore[comparison-overlap] + ] + with self.enter_class(node.info): + super().visit_class_def(node) + node.defs.body.extend(node.removed_statements) + node.removed_statements = [] + type_state.reset_subtype_caches_for(node.info) + # Kill the TypeInfo, since there is none before semantic analysis. + node.info = CLASSDEF_NO_INFO + node.analyzed = None + + def save_implicit_attributes(self, node: ClassDef) -> None: + """Produce callbacks that re-add attributes defined on self.""" + for name, sym in node.info.names.items(): + if isinstance(sym.node, Var) and sym.implicit: + self.saved_class_attrs[node, name] = sym + + def visit_func_def(self, node: FuncDef) -> None: + if not self.recurse_into_functions: + return + node.expanded = [] + node.type = node.unanalyzed_type + if node.type: + # Type variable binder binds type variables before the type is analyzed, + # this causes unanalyzed_type to be modified in place. We needed to revert this + # in order to get the state exactly as it was before semantic analysis. + # See also #4814. + assert isinstance(node.type, CallableType) + node.type.variables = [] + with self.enter_method(node.info) if node.info else nullcontext(): + super().visit_func_def(node) + + def visit_decorator(self, node: Decorator) -> None: + node.var.type = None + for expr in node.decorators: + expr.accept(self) + if self.recurse_into_functions: + node.func.accept(self) + else: + # Only touch the final status if we re-process + # the top level, since decorators are processed there. + node.var.is_final = False + node.func.is_final = False + + def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> None: + if not self.recurse_into_functions: + return + # Revert change made during semantic analysis main pass. + node.items = node.unanalyzed_items.copy() + node.impl = None + node.is_final = False + super().visit_overloaded_func_def(node) + + def visit_assignment_stmt(self, node: AssignmentStmt) -> None: + node.type = node.unanalyzed_type + node.is_final_def = False + node.is_alias_def = False + if self.type and not self.is_class_body: + for lvalue in node.lvalues: + # Revert assignments made via self attributes. + self.process_lvalue_in_method(lvalue) + super().visit_assignment_stmt(node) + + def visit_import_from(self, node: ImportFrom) -> None: + node.assignments = [] + + def visit_import_all(self, node: ImportAll) -> None: + node.assignments = [] + + def visit_for_stmt(self, node: ForStmt) -> None: + node.index_type = node.unanalyzed_index_type + node.inferred_item_type = None + node.inferred_iterator_type = None + super().visit_for_stmt(node) + + def visit_name_expr(self, node: NameExpr) -> None: + self.strip_ref_expr(node) + + def visit_member_expr(self, node: MemberExpr) -> None: + self.strip_ref_expr(node) + super().visit_member_expr(node) + + def visit_index_expr(self, node: IndexExpr) -> None: + node.analyzed = None # May have been an alias or type application. + super().visit_index_expr(node) + + def visit_op_expr(self, node: OpExpr) -> None: + node.analyzed = None # May have been an alias + super().visit_op_expr(node) + + def strip_ref_expr(self, node: RefExpr) -> None: + node.kind = None + node.node = None + node.fullname = "" + node.is_new_def = False + node.is_inferred_def = False + + def visit_call_expr(self, node: CallExpr) -> None: + node.analyzed = None + super().visit_call_expr(node) + + def visit_super_expr(self, node: SuperExpr) -> None: + node.info = None + super().visit_super_expr(node) + + def process_lvalue_in_method(self, lvalue: Node) -> None: + if isinstance(lvalue, MemberExpr): + if lvalue.is_new_def: + # Remove defined attribute from the class symbol table. If is_new_def is + # true for a MemberExpr, we know that it must be an assignment through + # self, since only those can define new attributes. + assert self.type is not None + if lvalue.name in self.type.names: + del self.type.names[lvalue.name] + key = (self.type.defn, lvalue.name) + if key in self.saved_class_attrs: + del self.saved_class_attrs[key] + elif isinstance(lvalue, (TupleExpr, ListExpr)): + for item in lvalue.items: + self.process_lvalue_in_method(item) + elif isinstance(lvalue, StarExpr): + self.process_lvalue_in_method(lvalue.expr) + + @contextmanager + def enter_class(self, info: TypeInfo) -> Iterator[None]: + old_type = self.type + old_is_class_body = self.is_class_body + self.type = info + self.is_class_body = True + yield + self.type = old_type + self.is_class_body = old_is_class_body + + @contextmanager + def enter_method(self, info: TypeInfo) -> Iterator[None]: + old_type = self.type + old_is_class_body = self.is_class_body + self.type = info + self.is_class_body = False + yield + self.type = old_type + self.is_class_body = old_is_class_body diff --git a/mypy/server/deps.py b/mypy/server/deps.py new file mode 100644 index 000000000000..6376600ffc0c --- /dev/null +++ b/mypy/server/deps.py @@ -0,0 +1,1137 @@ +"""Generate fine-grained dependencies for AST nodes, for use in the daemon mode. + +Dependencies are stored in a map from *triggers* to *sets of affected locations*. + +A trigger is a string that represents a program property that has changed, such +as the signature of a specific function. Triggers are written as '<...>' (angle +brackets). When a program property changes, we determine the relevant trigger(s) +and all affected locations. The latter are stale and will have to be reprocessed. + +An affected location is a string than can refer to a *target* (a non-nested +function or method, or a module top level), a class, or a trigger (for +recursively triggering other triggers). + +Here's an example representation of a simple dependency map (in format +" -> locations"): + + -> m.f + -> , m.A, m.f + +Assuming 'A' is a class, this means that + +1) if a property of 'm.A.g', such as the signature, is changed, we need + to process target (function) 'm.f' + +2) if the MRO or other significant property of class 'm.A' changes, we + need to process target 'm.f', the entire class 'm.A', and locations + triggered by trigger '' (this explanation is a bit simplified; + see below for more details). + +The triggers to fire are determined using mypy.server.astdiff. + +Examples of triggers: + +* '' represents a module attribute/function/class. If any externally + visible property of 'x' changes, this gets fired. For changes within + classes, only "big" changes cause the class to be triggered (such as a + change in MRO). Smaller changes, such as changes to some attributes, don't + trigger the entire class. +* '' represents the type and kind of attribute/method 'x' of + class 'mod.Cls'. This can also refer to an attribute inherited from a + base class (relevant if it's accessed through a value of type 'Cls' + instead of the base class type). +* '' represents the existence of module 'package.mod'. This + gets triggered if 'package.mod' is created or deleted, or if it gets + changed into something other than a module. + +Examples of locations: + +* 'mod' is the top level of module 'mod' (doesn't include any function bodies, + but includes class bodies not nested within a function). +* 'mod.f' is function 'f' in module 'mod' (module-level variables aren't separate + locations but are included in the module top level). Functions also include + any nested functions and classes -- such nested definitions aren't separate + locations, for simplicity of implementation. +* 'mod.Cls.f' is method 'f' of 'mod.Cls'. Non-method attributes aren't locations. +* 'mod.Cls' represents each method in class 'mod.Cls' + the top-level of the + module 'mod'. (To simplify the implementation, there is no location that only + includes the body of a class without the entire surrounding module top level.) +* Trigger '<...>' as a location is an indirect way of referring to all + locations triggered by the trigger. These indirect locations keep the + dependency map smaller and easier to manage. + +Triggers can be triggered by program changes such as these: + +* Addition or deletion of an attribute (or module). +* Change of the kind of thing a name represents (such as a change from a function + to a class). +* Change of the static type of a name. + +Changes in the body of a function that aren't reflected in the signature don't +cause the function to be triggered. More generally, we trigger only on changes +that may affect type checking results outside the module that contains the +change. + +We don't generate dependencies from builtins and certain other stdlib modules, +since these change very rarely, and they would just increase the size of the +dependency map significantly without significant benefit. + +Test cases for this module live in 'test-data/unit/deps*.test'. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import List + +from mypy.nodes import ( + GDEF, + LDEF, + MDEF, + AssertTypeExpr, + AssignmentStmt, + AwaitExpr, + Block, + CallExpr, + CastExpr, + ClassDef, + ComparisonExpr, + Decorator, + DelStmt, + DictionaryComprehension, + EnumCallExpr, + Expression, + ForStmt, + FuncBase, + FuncDef, + GeneratorExpr, + Import, + ImportAll, + ImportFrom, + IndexExpr, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + NewTypeExpr, + Node, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + RefExpr, + StarExpr, + SuperExpr, + TupleExpr, + TypeAliasExpr, + TypeApplication, + TypedDictExpr, + TypeInfo, + TypeVarExpr, + UnaryExpr, + Var, + WithStmt, + YieldFromExpr, +) +from mypy.operators import ( + op_methods, + ops_with_inplace_method, + reverse_op_methods, + unary_op_methods, +) +from mypy.options import Options +from mypy.scope import Scope +from mypy.server.trigger import make_trigger, make_wildcard_trigger +from mypy.traverser import TraverserVisitor +from mypy.typeops import bind_self +from mypy.types import ( + AnyType, + CallableType, + DeletedType, + ErasedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, +) +from mypy.typestate import type_state +from mypy.util import correct_relative_import + + +def get_dependencies( + target: MypyFile, + type_map: dict[Expression, Type], + python_version: tuple[int, int], + options: Options, +) -> dict[str, set[str]]: + """Get all dependencies of a node, recursively.""" + visitor = DependencyVisitor(type_map, python_version, target.alias_deps, options) + target.accept(visitor) + return visitor.map + + +def get_dependencies_of_target( + module_id: str, + module_tree: MypyFile, + target: Node, + type_map: dict[Expression, Type], + python_version: tuple[int, int], +) -> dict[str, set[str]]: + """Get dependencies of a target -- don't recursive into nested targets.""" + # TODO: Add tests for this function. + visitor = DependencyVisitor(type_map, python_version, module_tree.alias_deps) + with visitor.scope.module_scope(module_id): + if isinstance(target, MypyFile): + # Only get dependencies of the top-level of the module. Don't recurse into + # functions. + for defn in target.defs: + # TODO: Recurse into top-level statements and class bodies but skip functions. + if not isinstance(defn, (ClassDef, Decorator, FuncDef, OverloadedFuncDef)): + defn.accept(visitor) + elif isinstance(target, FuncBase) and target.info: + # It's a method. + # TODO: Methods in nested classes. + with visitor.scope.class_scope(target.info): + target.accept(visitor) + else: + target.accept(visitor) + return visitor.map + + +class DependencyVisitor(TraverserVisitor): + def __init__( + self, + type_map: dict[Expression, Type], + python_version: tuple[int, int], + alias_deps: defaultdict[str, set[str]], + options: Options | None = None, + ) -> None: + self.scope = Scope() + self.type_map = type_map + # This attribute holds a mapping from target to names of type aliases + # it depends on. These need to be processed specially, since they are + # only present in expanded form in symbol tables. For example, after: + # A = List[int] + # x: A + # The module symbol table will just have a Var `x` with type `List[int]`, + # and the dependency of `x` on `A` is lost. Therefore the alias dependencies + # are preserved at alias expansion points in `semanal.py`, stored as an attribute + # on MypyFile, and then passed here. + self.alias_deps = alias_deps + self.map: dict[str, set[str]] = {} + self.is_class = False + self.is_package_init_file = False + self.options = options + + def visit_mypy_file(self, o: MypyFile) -> None: + with self.scope.module_scope(o.fullname): + self.is_package_init_file = o.is_package_init_file() + self.add_type_alias_deps(self.scope.current_target()) + for trigger, targets in o.plugin_deps.items(): + self.map.setdefault(trigger, set()).update(targets) + super().visit_mypy_file(o) + + def visit_func_def(self, o: FuncDef) -> None: + with self.scope.function_scope(o): + target = self.scope.current_target() + if o.type: + if self.is_class and isinstance(o.type, FunctionLike): + signature: Type = bind_self(o.type) + else: + signature = o.type + for trigger in self.get_type_triggers(signature): + self.add_dependency(trigger) + self.add_dependency(trigger, target=make_trigger(target)) + if o.info: + for base in non_trivial_bases(o.info): + # Base class __init__/__new__ doesn't generate a logical + # dependency since the override can be incompatible. + if not self.use_logical_deps() or o.name not in ("__init__", "__new__"): + self.add_dependency(make_trigger(base.fullname + "." + o.name)) + self.add_type_alias_deps(self.scope.current_target()) + super().visit_func_def(o) + variants = set(o.expanded) - {o} + for ex in variants: + if isinstance(ex, FuncDef): + super().visit_func_def(ex) + + def visit_decorator(self, o: Decorator) -> None: + if not self.use_logical_deps(): + # We don't need to recheck outer scope for an overload, only overload itself. + # Also if any decorator is nested, it is not externally visible, so we don't need to + # generate dependency. + if not o.func.is_overload and self.scope.current_function_name() is None: + self.add_dependency(make_trigger(o.func.fullname)) + else: + # Add logical dependencies from decorators to the function. For example, + # if we have + # @dec + # def func(): ... + # then if `dec` is unannotated, then it will "spoil" `func` and consequently + # all call sites, making them all `Any`. + for d in o.decorators: + tname: str | None = None + if isinstance(d, RefExpr) and d.fullname: + tname = d.fullname + if isinstance(d, CallExpr) and isinstance(d.callee, RefExpr) and d.callee.fullname: + tname = d.callee.fullname + if tname is not None: + self.add_dependency(make_trigger(tname), make_trigger(o.func.fullname)) + super().visit_decorator(o) + + def visit_class_def(self, o: ClassDef) -> None: + with self.scope.class_scope(o.info): + target = self.scope.current_full_target() + self.add_dependency(make_trigger(target), target) + old_is_class = self.is_class + self.is_class = True + # Add dependencies to type variables of a generic class. + for tv in o.type_vars: + self.add_dependency(make_trigger(tv.fullname), target) + self.process_type_info(o.info) + super().visit_class_def(o) + self.is_class = old_is_class + + def visit_newtype_expr(self, o: NewTypeExpr) -> None: + if o.info: + with self.scope.class_scope(o.info): + self.process_type_info(o.info) + + def process_type_info(self, info: TypeInfo) -> None: + target = self.scope.current_full_target() + for base in info.bases: + self.add_type_dependencies(base, target=target) + if info.tuple_type: + self.add_type_dependencies(info.tuple_type, target=make_trigger(target)) + if info.typeddict_type: + self.add_type_dependencies(info.typeddict_type, target=make_trigger(target)) + if info.declared_metaclass: + self.add_type_dependencies(info.declared_metaclass, target=make_trigger(target)) + if info.is_protocol: + for base_info in info.mro[:-1]: + # We add dependencies from whole MRO to cover explicit subprotocols. + # For example: + # + # class Super(Protocol): + # x: int + # class Sub(Super, Protocol): + # y: int + # + # In this example we add -> , to invalidate Sub if + # a new member is added to Super. + self.add_dependency( + make_wildcard_trigger(base_info.fullname), target=make_trigger(target) + ) + # More protocol dependencies are collected in type_state._snapshot_protocol_deps + # after a full run or update is finished. + + self.add_type_alias_deps(self.scope.current_target()) + for name, node in info.names.items(): + if isinstance(node.node, Var): + # Recheck Liskov if needed, self definitions are checked in the defining method + if node.node.is_initialized_in_class and has_user_bases(info): + self.add_dependency(make_trigger(info.fullname + "." + name)) + for base_info in non_trivial_bases(info): + # If the type of an attribute changes in a base class, we make references + # to the attribute in the subclass stale. + self.add_dependency( + make_trigger(base_info.fullname + "." + name), + target=make_trigger(info.fullname + "." + name), + ) + for base_info in non_trivial_bases(info): + for name, node in base_info.names.items(): + if self.use_logical_deps(): + # Skip logical dependency if an attribute is not overridden. For example, + # in case of: + # class Base: + # x = 1 + # y = 2 + # class Sub(Base): + # x = 3 + # we skip -> , because even if `y` is unannotated it + # doesn't affect precision of Liskov checking. + if name not in info.names: + continue + # __init__ and __new__ can be overridden with different signatures, so no + # logical dependency. + if name in ("__init__", "__new__"): + continue + self.add_dependency( + make_trigger(base_info.fullname + "." + name), + target=make_trigger(info.fullname + "." + name), + ) + if not self.use_logical_deps(): + # These dependencies are only useful for propagating changes -- + # they aren't logical dependencies since __init__ and __new__ can be + # overridden with a different signature. + self.add_dependency( + make_trigger(base_info.fullname + ".__init__"), + target=make_trigger(info.fullname + ".__init__"), + ) + self.add_dependency( + make_trigger(base_info.fullname + ".__new__"), + target=make_trigger(info.fullname + ".__new__"), + ) + # If the set of abstract attributes change, this may invalidate class + # instantiation, or change the generated error message, since Python checks + # class abstract status when creating an instance. + self.add_dependency( + make_trigger(base_info.fullname + ".(abstract)"), + target=make_trigger(info.fullname + ".__init__"), + ) + # If the base class abstract attributes change, subclass abstract + # attributes need to be recalculated. + self.add_dependency(make_trigger(base_info.fullname + ".(abstract)")) + + def visit_import(self, o: Import) -> None: + for id, as_id in o.ids: + self.add_dependency(make_trigger(id), self.scope.current_target()) + + def visit_import_from(self, o: ImportFrom) -> None: + if self.use_logical_deps(): + # Just importing a name doesn't create a logical dependency. + return + module_id, _ = correct_relative_import( + self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file + ) + self.add_dependency(make_trigger(module_id)) # needed if module is added/removed + for name, as_name in o.names: + self.add_dependency(make_trigger(module_id + "." + name)) + + def visit_import_all(self, o: ImportAll) -> None: + module_id, _ = correct_relative_import( + self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file + ) + # The current target needs to be rechecked if anything "significant" changes in the + # target module namespace (as the imported definitions will need to be updated). + self.add_dependency(make_wildcard_trigger(module_id)) + + def visit_block(self, o: Block) -> None: + if not o.is_unreachable: + super().visit_block(o) + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + rvalue = o.rvalue + if isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypeVarExpr): + analyzed = rvalue.analyzed + self.add_type_dependencies( + analyzed.upper_bound, target=make_trigger(analyzed.fullname) + ) + for val in analyzed.values: + self.add_type_dependencies(val, target=make_trigger(analyzed.fullname)) + # We need to re-analyze the definition if bound or value is deleted. + super().visit_call_expr(rvalue) + elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, NamedTupleExpr): + # Depend on types of named tuple items. + info = rvalue.analyzed.info + prefix = f"{self.scope.current_full_target()}.{info.name}" + for name, symnode in info.names.items(): + if not name.startswith("_") and isinstance(symnode.node, Var): + typ = symnode.node.type + if typ: + self.add_type_dependencies(typ) + self.add_type_dependencies(typ, target=make_trigger(prefix)) + attr_target = make_trigger(f"{prefix}.{name}") + self.add_type_dependencies(typ, target=attr_target) + elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypedDictExpr): + # Depend on the underlying typeddict type + info = rvalue.analyzed.info + assert info.typeddict_type is not None + prefix = f"{self.scope.current_full_target()}.{info.name}" + self.add_type_dependencies(info.typeddict_type, target=make_trigger(prefix)) + elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, EnumCallExpr): + # Enum values are currently not checked, but for future we add the deps on them + for name, symnode in rvalue.analyzed.info.names.items(): + if isinstance(symnode.node, Var) and symnode.node.type: + self.add_type_dependencies(symnode.node.type) + elif o.is_alias_def: + assert len(o.lvalues) == 1 + lvalue = o.lvalues[0] + assert isinstance(lvalue, NameExpr) + typ = get_proper_type(self.type_map.get(lvalue)) + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + class_name = typ.type_object().fullname + self.add_dependency(make_trigger(class_name + ".__init__")) + self.add_dependency(make_trigger(class_name + ".__new__")) + if isinstance(rvalue, IndexExpr) and isinstance(rvalue.analyzed, TypeAliasExpr): + self.add_type_dependencies(rvalue.analyzed.node.target) + elif typ: + self.add_type_dependencies(typ) + else: + # Normal assignment + super().visit_assignment_stmt(o) + for lvalue in o.lvalues: + self.process_lvalue(lvalue) + items = o.lvalues + [rvalue] + for i in range(len(items) - 1): + lvalue = items[i] + rvalue = items[i + 1] + if isinstance(lvalue, TupleExpr): + self.add_attribute_dependency_for_expr(rvalue, "__iter__") + if o.type: + self.add_type_dependencies(o.type) + if self.use_logical_deps() and o.unanalyzed_type is None: + # Special case: for definitions without an explicit type like this: + # x = func(...) + # we add a logical dependency -> , because if `func` is not annotated, + # then it will make all points of use of `x` unchecked. + if ( + isinstance(rvalue, CallExpr) + and isinstance(rvalue.callee, RefExpr) + and rvalue.callee.fullname + ): + fname: str | None = None + if isinstance(rvalue.callee.node, TypeInfo): + # use actual __init__ as a dependency source + init = rvalue.callee.node.get("__init__") + if init and isinstance(init.node, FuncBase): + fname = init.node.fullname + else: + fname = rvalue.callee.fullname + if not fname: + return + for lv in o.lvalues: + if isinstance(lv, RefExpr) and lv.fullname and lv.is_new_def: + if lv.kind == LDEF: + return # local definitions don't generate logical deps + self.add_dependency(make_trigger(fname), make_trigger(lv.fullname)) + + def process_lvalue(self, lvalue: Expression) -> None: + """Generate additional dependencies for an lvalue.""" + if isinstance(lvalue, IndexExpr): + self.add_operator_method_dependency(lvalue.base, "__setitem__") + elif isinstance(lvalue, NameExpr): + if lvalue.kind in (MDEF, GDEF): + # Assignment to an attribute in the class body, or direct assignment to a + # global variable. + lvalue_type = self.get_non_partial_lvalue_type(lvalue) + type_triggers = self.get_type_triggers(lvalue_type) + attr_trigger = make_trigger(f"{self.scope.current_full_target()}.{lvalue.name}") + for type_trigger in type_triggers: + self.add_dependency(type_trigger, attr_trigger) + elif isinstance(lvalue, MemberExpr): + if self.is_self_member_ref(lvalue) and lvalue.is_new_def: + node = lvalue.node + if isinstance(node, Var): + info = node.info + if info and has_user_bases(info): + # Recheck Liskov for self definitions + self.add_dependency(make_trigger(info.fullname + "." + lvalue.name)) + if lvalue.kind is None: + # Reference to a non-module attribute + if lvalue.expr not in self.type_map: + # Unreachable assignment -> not checked so no dependencies to generate. + return + object_type = self.type_map[lvalue.expr] + lvalue_type = self.get_non_partial_lvalue_type(lvalue) + type_triggers = self.get_type_triggers(lvalue_type) + for attr_trigger in self.attribute_triggers(object_type, lvalue.name): + for type_trigger in type_triggers: + self.add_dependency(type_trigger, attr_trigger) + elif isinstance(lvalue, TupleExpr): + for item in lvalue.items: + self.process_lvalue(item) + elif isinstance(lvalue, StarExpr): + self.process_lvalue(lvalue.expr) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def get_non_partial_lvalue_type(self, lvalue: RefExpr) -> Type: + if lvalue not in self.type_map: + # Likely a block considered unreachable during type checking. + return UninhabitedType() + lvalue_type = get_proper_type(self.type_map[lvalue]) + if isinstance(lvalue_type, PartialType): + if isinstance(lvalue.node, Var): + if lvalue.node.type: + lvalue_type = get_proper_type(lvalue.node.type) + else: + lvalue_type = UninhabitedType() + else: + # Probably a secondary, non-definition assignment that doesn't + # result in a non-partial type. We won't be able to infer any + # dependencies from this so just return something. (The first, + # definition assignment with a partial type is handled + # differently, in the semantic analyzer.) + assert not lvalue.is_new_def + return UninhabitedType() + return lvalue_type + + def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: + super().visit_operator_assignment_stmt(o) + self.process_lvalue(o.lvalue) + method = op_methods[o.op] + self.add_attribute_dependency_for_expr(o.lvalue, method) + if o.op in ops_with_inplace_method: + inplace_method = "__i" + method[2:] + self.add_attribute_dependency_for_expr(o.lvalue, inplace_method) + + def visit_for_stmt(self, o: ForStmt) -> None: + super().visit_for_stmt(o) + if not o.is_async: + # __getitem__ is only used if __iter__ is missing but for simplicity we + # just always depend on both. + self.add_attribute_dependency_for_expr(o.expr, "__iter__") + self.add_attribute_dependency_for_expr(o.expr, "__getitem__") + if o.inferred_iterator_type: + self.add_attribute_dependency(o.inferred_iterator_type, "__next__") + else: + self.add_attribute_dependency_for_expr(o.expr, "__aiter__") + if o.inferred_iterator_type: + self.add_attribute_dependency(o.inferred_iterator_type, "__anext__") + + self.process_lvalue(o.index) + if isinstance(o.index, TupleExpr): + # Process multiple assignment to index variables. + item_type = o.inferred_item_type + if item_type: + # This is similar to above. + self.add_attribute_dependency(item_type, "__iter__") + self.add_attribute_dependency(item_type, "__getitem__") + if o.index_type: + self.add_type_dependencies(o.index_type) + + def visit_with_stmt(self, o: WithStmt) -> None: + super().visit_with_stmt(o) + for e in o.expr: + if not o.is_async: + self.add_attribute_dependency_for_expr(e, "__enter__") + self.add_attribute_dependency_for_expr(e, "__exit__") + else: + self.add_attribute_dependency_for_expr(e, "__aenter__") + self.add_attribute_dependency_for_expr(e, "__aexit__") + for typ in o.analyzed_types: + self.add_type_dependencies(typ) + + def visit_del_stmt(self, o: DelStmt) -> None: + super().visit_del_stmt(o) + if isinstance(o.expr, IndexExpr): + self.add_attribute_dependency_for_expr(o.expr.base, "__delitem__") + + # Expressions + + def process_global_ref_expr(self, o: RefExpr) -> None: + if o.fullname: + self.add_dependency(make_trigger(o.fullname)) + + # If this is a reference to a type, generate a dependency to its + # constructor. + # IDEA: Avoid generating spurious dependencies for except statements, + # class attribute references, etc., if performance is a problem. + typ = get_proper_type(self.type_map.get(o)) + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + class_name = typ.type_object().fullname + self.add_dependency(make_trigger(class_name + ".__init__")) + self.add_dependency(make_trigger(class_name + ".__new__")) + + def visit_name_expr(self, o: NameExpr) -> None: + if o.kind == LDEF: + # We don't track dependencies to local variables, since they + # aren't externally visible. + return + if o.kind == MDEF: + # Direct reference to member is only possible in the scope that + # defined the name, so no dependency is required. + return + self.process_global_ref_expr(o) + + def visit_member_expr(self, e: MemberExpr) -> None: + if isinstance(e.expr, RefExpr) and isinstance(e.expr.node, TypeInfo): + # Special case class attribute so that we don't depend on "__init__". + self.add_dependency(make_trigger(e.expr.node.fullname)) + else: + super().visit_member_expr(e) + if e.kind is not None: + # Reference to a module attribute + self.process_global_ref_expr(e) + else: + # Reference to a non-module (or missing) attribute + if e.expr not in self.type_map: + # No type available -- this happens for unreachable code. Since it's unreachable, + # it wasn't type checked and we don't need to generate dependencies. + return + if isinstance(e.expr, RefExpr) and isinstance(e.expr.node, MypyFile): + # Special case: reference to a missing module attribute. + self.add_dependency(make_trigger(e.expr.node.fullname + "." + e.name)) + return + typ = get_proper_type(self.type_map[e.expr]) + self.add_attribute_dependency(typ, e.name) + if self.use_logical_deps() and isinstance(typ, AnyType): + name = self.get_unimported_fullname(e, typ) + if name is not None: + # Generate a logical dependency from an unimported + # definition (which comes from a missing module). + # Example: + # import missing # "missing" not in build + # + # def g() -> None: + # missing.f() # Generate dependency from "missing.f" + self.add_dependency(make_trigger(name)) + + def get_unimported_fullname(self, e: MemberExpr, typ: AnyType) -> str | None: + """If e refers to an unimported definition, infer the fullname of this. + + Return None if e doesn't refer to an unimported definition or if we can't + determine the name. + """ + suffix = "" + # Unwrap nested member expression to handle cases like "a.b.c.d" where + # "a.b" is a known reference to an unimported module. Find the base + # reference to an unimported module (such as "a.b") and the name suffix + # (such as "c.d") needed to build a full name. + while typ.type_of_any == TypeOfAny.from_another_any and isinstance(e.expr, MemberExpr): + suffix = "." + e.name + suffix + e = e.expr + if e.expr not in self.type_map: + return None + obj_type = get_proper_type(self.type_map[e.expr]) + if not isinstance(obj_type, AnyType): + # Can't find the base reference to the unimported module. + return None + typ = obj_type + if typ.type_of_any == TypeOfAny.from_unimported_type and typ.missing_import_name: + # Infer the full name of the unimported definition. + return typ.missing_import_name + "." + e.name + suffix + return None + + def visit_super_expr(self, e: SuperExpr) -> None: + # Arguments in "super(C, self)" won't generate useful logical deps. + if not self.use_logical_deps(): + super().visit_super_expr(e) + if e.info is not None: + name = e.name + for base in non_trivial_bases(e.info): + self.add_dependency(make_trigger(base.fullname + "." + name)) + if name in base.names: + # No need to depend on further base classes, since we found + # the target. This is safe since if the target gets + # deleted or modified, we'll trigger it. + break + + def visit_call_expr(self, e: CallExpr) -> None: + if isinstance(e.callee, RefExpr) and e.callee.fullname == "builtins.isinstance": + self.process_isinstance_call(e) + else: + super().visit_call_expr(e) + typ = self.type_map.get(e.callee) + if typ is not None: + typ = get_proper_type(typ) + if not isinstance(typ, FunctionLike): + self.add_attribute_dependency(typ, "__call__") + + def process_isinstance_call(self, e: CallExpr) -> None: + """Process "isinstance(...)" in a way to avoid some extra dependencies.""" + if len(e.args) == 2: + arg = e.args[1] + if ( + isinstance(arg, RefExpr) + and arg.kind == GDEF + and isinstance(arg.node, TypeInfo) + and arg.fullname + ): + # Special case to avoid redundant dependencies from "__init__". + self.add_dependency(make_trigger(arg.fullname)) + return + # In uncommon cases generate normal dependencies. These will include + # spurious dependencies, but the performance impact is small. + super().visit_call_expr(e) + + def visit_cast_expr(self, e: CastExpr) -> None: + super().visit_cast_expr(e) + self.add_type_dependencies(e.type) + + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: + super().visit_assert_type_expr(e) + self.add_type_dependencies(e.type) + + def visit_type_application(self, e: TypeApplication) -> None: + super().visit_type_application(e) + for typ in e.types: + self.add_type_dependencies(typ) + + def visit_index_expr(self, e: IndexExpr) -> None: + super().visit_index_expr(e) + self.add_operator_method_dependency(e.base, "__getitem__") + + def visit_unary_expr(self, e: UnaryExpr) -> None: + super().visit_unary_expr(e) + if e.op not in unary_op_methods: + return + method = unary_op_methods[e.op] + self.add_operator_method_dependency(e.expr, method) + + def visit_op_expr(self, e: OpExpr) -> None: + super().visit_op_expr(e) + self.process_binary_op(e.op, e.left, e.right) + + def visit_comparison_expr(self, e: ComparisonExpr) -> None: + super().visit_comparison_expr(e) + for i, op in enumerate(e.operators): + left = e.operands[i] + right = e.operands[i + 1] + self.process_binary_op(op, left, right) + + def process_binary_op(self, op: str, left: Expression, right: Expression) -> None: + method = op_methods.get(op) + if method: + if op == "in": + self.add_operator_method_dependency(right, method) + else: + self.add_operator_method_dependency(left, method) + rev_method = reverse_op_methods.get(method) + if rev_method: + self.add_operator_method_dependency(right, rev_method) + + def add_operator_method_dependency(self, e: Expression, method: str) -> None: + typ = get_proper_type(self.type_map.get(e)) + if typ is not None: + self.add_operator_method_dependency_for_type(typ, method) + + def add_operator_method_dependency_for_type(self, typ: ProperType, method: str) -> None: + # Note that operator methods can't be (non-metaclass) methods of type objects + # (that is, TypeType objects or Callables representing a type). + if isinstance(typ, TypeVarType): + typ = get_proper_type(typ.upper_bound) + if isinstance(typ, TupleType): + typ = typ.partial_fallback + if isinstance(typ, Instance): + trigger = make_trigger(typ.type.fullname + "." + method) + self.add_dependency(trigger) + elif isinstance(typ, UnionType): + for item in typ.items: + self.add_operator_method_dependency_for_type(get_proper_type(item), method) + elif isinstance(typ, FunctionLike) and typ.is_type_obj(): + self.add_operator_method_dependency_for_type(typ.fallback, method) + elif isinstance(typ, TypeType): + if isinstance(typ.item, Instance) and typ.item.type.metaclass_type is not None: + self.add_operator_method_dependency_for_type(typ.item.type.metaclass_type, method) + + def visit_generator_expr(self, e: GeneratorExpr) -> None: + super().visit_generator_expr(e) + for seq in e.sequences: + self.add_iter_dependency(seq) + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> None: + super().visit_dictionary_comprehension(e) + for seq in e.sequences: + self.add_iter_dependency(seq) + + def visit_star_expr(self, e: StarExpr) -> None: + super().visit_star_expr(e) + self.add_iter_dependency(e.expr) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + super().visit_yield_from_expr(e) + self.add_iter_dependency(e.expr) + + def visit_await_expr(self, e: AwaitExpr) -> None: + super().visit_await_expr(e) + self.add_attribute_dependency_for_expr(e.expr, "__await__") + + # Helpers + + def add_type_alias_deps(self, target: str) -> None: + # Type aliases are special, because some of the dependencies are calculated + # in semanal.py, before they are expanded. + if target in self.alias_deps: + for alias in self.alias_deps[target]: + self.add_dependency(make_trigger(alias)) + + def add_dependency(self, trigger: str, target: str | None = None) -> None: + """Add dependency from trigger to a target. + + If the target is not given explicitly, use the current target. + """ + if trigger.startswith( + (" None: + """Add dependencies to all components of a type. + + Args: + target: If not None, override the default (current) target of the + generated dependency. + """ + for trigger in self.get_type_triggers(typ): + self.add_dependency(trigger, target) + + def add_attribute_dependency(self, typ: Type, name: str) -> None: + """Add dependencies for accessing a named attribute of a type.""" + targets = self.attribute_triggers(typ, name) + for target in targets: + self.add_dependency(target) + + def attribute_triggers(self, typ: Type, name: str) -> list[str]: + """Return all triggers associated with the attribute of a type.""" + typ = get_proper_type(typ) + if isinstance(typ, TypeVarType): + typ = get_proper_type(typ.upper_bound) + if isinstance(typ, TupleType): + typ = typ.partial_fallback + if isinstance(typ, Instance): + member = f"{typ.type.fullname}.{name}" + return [make_trigger(member)] + elif isinstance(typ, FunctionLike) and typ.is_type_obj(): + member = f"{typ.type_object().fullname}.{name}" + triggers = [make_trigger(member)] + triggers.extend(self.attribute_triggers(typ.fallback, name)) + return triggers + elif isinstance(typ, UnionType): + targets = [] + for item in typ.items: + targets.extend(self.attribute_triggers(item, name)) + return targets + elif isinstance(typ, TypeType): + triggers = self.attribute_triggers(typ.item, name) + if isinstance(typ.item, Instance) and typ.item.type.metaclass_type is not None: + triggers.append( + make_trigger(f"{typ.item.type.metaclass_type.type.fullname}.{name}") + ) + return triggers + else: + return [] + + def add_attribute_dependency_for_expr(self, e: Expression, name: str) -> None: + typ = self.type_map.get(e) + if typ is not None: + self.add_attribute_dependency(typ, name) + + def add_iter_dependency(self, node: Expression) -> None: + typ = self.type_map.get(node) + if typ: + self.add_attribute_dependency(typ, "__iter__") + + def use_logical_deps(self) -> bool: + return self.options is not None and self.options.logical_deps + + def get_type_triggers(self, typ: Type) -> list[str]: + return get_type_triggers(typ, self.use_logical_deps()) + + +def get_type_triggers( + typ: Type, use_logical_deps: bool, seen_aliases: set[TypeAliasType] | None = None +) -> list[str]: + """Return all triggers that correspond to a type becoming stale.""" + return typ.accept(TypeTriggersVisitor(use_logical_deps, seen_aliases)) + + +class TypeTriggersVisitor(TypeVisitor[List[str]]): + def __init__( + self, use_logical_deps: bool, seen_aliases: set[TypeAliasType] | None = None + ) -> None: + self.deps: list[str] = [] + self.seen_aliases: set[TypeAliasType] = seen_aliases or set() + self.use_logical_deps = use_logical_deps + + def get_type_triggers(self, typ: Type) -> list[str]: + return get_type_triggers(typ, self.use_logical_deps, self.seen_aliases) + + def visit_instance(self, typ: Instance) -> list[str]: + trigger = make_trigger(typ.type.fullname) + triggers = [trigger] + for arg in typ.args: + triggers.extend(self.get_type_triggers(arg)) + if typ.last_known_value: + triggers.extend(self.get_type_triggers(typ.last_known_value)) + if typ.extra_attrs and typ.extra_attrs.mod_name: + # Module as type effectively depends on all module attributes, use wildcard. + triggers.append(make_wildcard_trigger(typ.extra_attrs.mod_name)) + return triggers + + def visit_type_alias_type(self, typ: TypeAliasType) -> list[str]: + if typ in self.seen_aliases: + return [] + self.seen_aliases.add(typ) + assert typ.alias is not None + trigger = make_trigger(typ.alias.fullname) + triggers = [trigger] + for arg in typ.args: + triggers.extend(self.get_type_triggers(arg)) + # TODO: Now that type aliases are its own kind of types we can simplify + # the logic to rely on intermediate dependencies (like for instance types). + triggers.extend(self.get_type_triggers(typ.alias.target)) + return triggers + + def visit_any(self, typ: AnyType) -> list[str]: + if typ.missing_import_name is not None: + return [make_trigger(typ.missing_import_name)] + return [] + + def visit_none_type(self, typ: NoneType) -> list[str]: + return [] + + def visit_callable_type(self, typ: CallableType) -> list[str]: + triggers = [] + for arg in typ.arg_types: + triggers.extend(self.get_type_triggers(arg)) + triggers.extend(self.get_type_triggers(typ.ret_type)) + # fallback is a metaclass type for class objects, and is + # processed separately. + return triggers + + def visit_overloaded(self, typ: Overloaded) -> list[str]: + triggers = [] + for item in typ.items: + triggers.extend(self.get_type_triggers(item)) + return triggers + + def visit_erased_type(self, t: ErasedType) -> list[str]: + # This type should exist only temporarily during type inference + assert False, "Should not see an erased type here" + + def visit_deleted_type(self, typ: DeletedType) -> list[str]: + return [] + + def visit_partial_type(self, typ: PartialType) -> list[str]: + assert False, "Should not see a partial type here" + + def visit_tuple_type(self, typ: TupleType) -> list[str]: + triggers = [] + for item in typ.items: + triggers.extend(self.get_type_triggers(item)) + triggers.extend(self.get_type_triggers(typ.partial_fallback)) + return triggers + + def visit_type_type(self, typ: TypeType) -> list[str]: + triggers = self.get_type_triggers(typ.item) + if not self.use_logical_deps: + old_triggers = triggers.copy() + for trigger in old_triggers: + triggers.append(trigger.rstrip(">") + ".__init__>") + triggers.append(trigger.rstrip(">") + ".__new__>") + return triggers + + def visit_type_var(self, typ: TypeVarType) -> list[str]: + triggers = [] + if typ.fullname: + triggers.append(make_trigger(typ.fullname)) + if typ.upper_bound: + triggers.extend(self.get_type_triggers(typ.upper_bound)) + if typ.default: + triggers.extend(self.get_type_triggers(typ.default)) + for val in typ.values: + triggers.extend(self.get_type_triggers(val)) + return triggers + + def visit_param_spec(self, typ: ParamSpecType) -> list[str]: + triggers = [] + if typ.fullname: + triggers.append(make_trigger(typ.fullname)) + if typ.upper_bound: + triggers.extend(self.get_type_triggers(typ.upper_bound)) + if typ.default: + triggers.extend(self.get_type_triggers(typ.default)) + triggers.extend(self.get_type_triggers(typ.upper_bound)) + return triggers + + def visit_type_var_tuple(self, typ: TypeVarTupleType) -> list[str]: + triggers = [] + if typ.fullname: + triggers.append(make_trigger(typ.fullname)) + if typ.upper_bound: + triggers.extend(self.get_type_triggers(typ.upper_bound)) + if typ.default: + triggers.extend(self.get_type_triggers(typ.default)) + triggers.extend(self.get_type_triggers(typ.upper_bound)) + return triggers + + def visit_unpack_type(self, typ: UnpackType) -> list[str]: + return typ.type.accept(self) + + def visit_parameters(self, typ: Parameters) -> list[str]: + triggers = [] + for arg in typ.arg_types: + triggers.extend(self.get_type_triggers(arg)) + return triggers + + def visit_typeddict_type(self, typ: TypedDictType) -> list[str]: + triggers = [] + for item in typ.items.values(): + triggers.extend(self.get_type_triggers(item)) + triggers.extend(self.get_type_triggers(typ.fallback)) + return triggers + + def visit_literal_type(self, typ: LiteralType) -> list[str]: + return self.get_type_triggers(typ.fallback) + + def visit_unbound_type(self, typ: UnboundType) -> list[str]: + return [] + + def visit_uninhabited_type(self, typ: UninhabitedType) -> list[str]: + return [] + + def visit_union_type(self, typ: UnionType) -> list[str]: + triggers = [] + for item in typ.items: + triggers.extend(self.get_type_triggers(item)) + return triggers + + +def merge_dependencies(new_deps: dict[str, set[str]], deps: dict[str, set[str]]) -> None: + for trigger, targets in new_deps.items(): + deps.setdefault(trigger, set()).update(targets) + + +def non_trivial_bases(info: TypeInfo) -> list[TypeInfo]: + return [base for base in info.mro[1:] if base.fullname != "builtins.object"] + + +def has_user_bases(info: TypeInfo) -> bool: + return any(base.module_name not in ("builtins", "typing", "enum") for base in info.mro[1:]) + + +def dump_all_dependencies( + modules: dict[str, MypyFile], + type_map: dict[Expression, Type], + python_version: tuple[int, int], + options: Options, +) -> None: + """Generate dependencies for all interesting modules and print them to stdout.""" + all_deps: dict[str, set[str]] = {} + for id, node in modules.items(): + # Uncomment for debugging: + # print('processing', id) + if id in ("builtins", "typing") or "/typeshed/" in node.path: + continue + assert id == node.fullname + deps = get_dependencies(node, type_map, python_version, options) + for trigger, targets in deps.items(): + all_deps.setdefault(trigger, set()).update(targets) + type_state.add_all_protocol_deps(all_deps) + + for trigger, targets in sorted(all_deps.items(), key=lambda x: x[0]): + print(trigger) + for target in sorted(targets): + print(f" {target}") diff --git a/mypy/server/mergecheck.py b/mypy/server/mergecheck.py new file mode 100644 index 000000000000..6f044a5ea8b9 --- /dev/null +++ b/mypy/server/mergecheck.py @@ -0,0 +1,83 @@ +"""Check for duplicate AST nodes after merge.""" + +from __future__ import annotations + +from typing import Final + +from mypy.nodes import Decorator, FakeInfo, FuncDef, SymbolNode, Var +from mypy.server.objgraph import get_path, get_reachable_graph + +# If True, print more verbose output on failure. +DUMP_MISMATCH_NODES: Final = False + + +def check_consistency(o: object) -> None: + """Fail if there are two AST nodes with the same fullname reachable from 'o'. + + Raise AssertionError on failure and print some debugging output. + """ + seen, parents = get_reachable_graph(o) + reachable = list(seen.values()) + syms = [x for x in reachable if isinstance(x, SymbolNode)] + + m: dict[str, SymbolNode] = {} + for sym in syms: + if isinstance(sym, FakeInfo): + continue + + fn = sym.fullname + # Skip None names, since they are ambiguous. + # TODO: Everything should have a proper full name? + if fn is None: + continue + # Skip stuff that should be expected to have duplicate names + if isinstance(sym, (Var, Decorator)): + continue + if isinstance(sym, FuncDef) and sym.is_overload: + continue + + if fn not in m: + m[sym.fullname] = sym + continue + + # We have trouble and need to decide what to do about it. + sym1, sym2 = sym, m[fn] + + # If the type changed, then it shouldn't have been merged. + if type(sym1) is not type(sym2): + continue + + path1 = get_path(sym1, seen, parents) + path2 = get_path(sym2, seen, parents) + + if fn in m: + print(f"\nDuplicate {type(sym).__name__!r} nodes with fullname {fn!r} found:") + print("[1] %d: %s" % (id(sym1), path_to_str(path1))) + print("[2] %d: %s" % (id(sym2), path_to_str(path2))) + + if DUMP_MISMATCH_NODES and fn in m: + # Add verbose output with full AST node contents. + print("---") + print(id(sym1), sym1) + print("---") + print(id(sym2), sym2) + + assert sym.fullname not in m + + +def path_to_str(path: list[tuple[object, object]]) -> str: + result = "" + for attr, obj in path: + t = type(obj).__name__ + if t in ("dict", "tuple", "SymbolTable", "list"): + result += f"[{repr(attr)}]" + else: + if isinstance(obj, Var): + result += f".{attr}({t}:{obj.name})" + elif t in ("BuildManager", "FineGrainedBuildManager"): + # Omit class name for some classes that aren't part of a class + # hierarchy since there isn't much ambiguity. + result += f".{attr}" + else: + result += f".{attr}({t})" + return result diff --git a/mypy/server/objgraph.py b/mypy/server/objgraph.py new file mode 100644 index 000000000000..a13fd8412934 --- /dev/null +++ b/mypy/server/objgraph.py @@ -0,0 +1,101 @@ +"""Find all objects reachable from a root object.""" + +from __future__ import annotations + +import types +import weakref +from collections.abc import Iterable +from typing import Final, Iterator, Mapping + +method_descriptor_type: Final = type(object.__dir__) +method_wrapper_type: Final = type(object().__ne__) +wrapper_descriptor_type: Final = type(object.__ne__) + +FUNCTION_TYPES: Final = ( + types.BuiltinFunctionType, + types.FunctionType, + types.MethodType, + method_descriptor_type, + wrapper_descriptor_type, + method_wrapper_type, +) + +ATTR_BLACKLIST: Final = {"__doc__", "__name__", "__class__", "__dict__"} + +# Instances of these types can't have references to other objects +ATOMIC_TYPE_BLACKLIST: Final = {bool, int, float, str, type(None), object} + +# Don't look at most attributes of these types +COLLECTION_TYPE_BLACKLIST: Final = {list, set, dict, tuple} + +# Don't return these objects +TYPE_BLACKLIST: Final = {weakref.ReferenceType} + + +def isproperty(o: object, attr: str) -> bool: + return isinstance(getattr(type(o), attr, None), property) + + +def get_edge_candidates(o: object) -> Iterator[tuple[object, object]]: + # use getattr because mypyc expects dict, not mappingproxy + if "__getattribute__" in getattr(type(o), "__dict__"): # noqa: B009 + return + if type(o) not in COLLECTION_TYPE_BLACKLIST: + for attr in dir(o): + try: + if attr not in ATTR_BLACKLIST and hasattr(o, attr) and not isproperty(o, attr): + e = getattr(o, attr) + if type(e) not in ATOMIC_TYPE_BLACKLIST: + yield attr, e + except AssertionError: + pass + if isinstance(o, Mapping): + yield from o.items() + elif isinstance(o, Iterable) and not isinstance(o, str): + for i, e in enumerate(o): + yield i, e + + +def get_edges(o: object) -> Iterator[tuple[object, object]]: + for s, e in get_edge_candidates(o): + if isinstance(e, FUNCTION_TYPES): + # We don't want to collect methods, but do want to collect values + # in closures and self pointers to other objects + + if hasattr(e, "__closure__"): + yield (s, "__closure__"), e.__closure__ + if hasattr(e, "__self__"): + se = e.__self__ + if se is not o and se is not type(o) and hasattr(s, "__self__"): + yield s.__self__, se + else: + if type(e) not in TYPE_BLACKLIST: + yield s, e + + +def get_reachable_graph(root: object) -> tuple[dict[int, object], dict[int, tuple[int, object]]]: + parents = {} + seen = {id(root): root} + worklist = [root] + while worklist: + o = worklist.pop() + for s, e in get_edges(o): + if id(e) in seen: + continue + parents[id(e)] = (id(o), s) + seen[id(e)] = e + worklist.append(e) + + return seen, parents + + +def get_path( + o: object, seen: dict[int, object], parents: dict[int, tuple[int, object]] +) -> list[tuple[object, object]]: + path = [] + while id(o) in parents: + pid, attr = parents[id(o)] + o = seen[pid] + path.append((attr, o)) + path.reverse() + return path diff --git a/mypy/server/subexpr.py b/mypy/server/subexpr.py new file mode 100644 index 000000000000..c94db44445dc --- /dev/null +++ b/mypy/server/subexpr.py @@ -0,0 +1,198 @@ +"""Find all subexpressions of an AST node.""" + +from __future__ import annotations + +from mypy.nodes import ( + AssertTypeExpr, + AssignmentExpr, + AwaitExpr, + CallExpr, + CastExpr, + ComparisonExpr, + ConditionalExpr, + DictExpr, + DictionaryComprehension, + Expression, + GeneratorExpr, + IndexExpr, + LambdaExpr, + ListComprehension, + ListExpr, + MemberExpr, + Node, + OpExpr, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + TupleExpr, + TypeApplication, + UnaryExpr, + YieldExpr, + YieldFromExpr, +) +from mypy.traverser import TraverserVisitor + + +def get_subexpressions(node: Node) -> list[Expression]: + visitor = SubexpressionFinder() + node.accept(visitor) + return visitor.expressions + + +class SubexpressionFinder(TraverserVisitor): + def __init__(self) -> None: + self.expressions: list[Expression] = [] + + def visit_int_expr(self, o: Expression) -> None: + self.add(o) + + def visit_name_expr(self, o: Expression) -> None: + self.add(o) + + def visit_float_expr(self, o: Expression) -> None: + self.add(o) + + def visit_str_expr(self, o: Expression) -> None: + self.add(o) + + def visit_bytes_expr(self, o: Expression) -> None: + self.add(o) + + def visit_unicode_expr(self, o: Expression) -> None: + self.add(o) + + def visit_complex_expr(self, o: Expression) -> None: + self.add(o) + + def visit_ellipsis(self, o: Expression) -> None: + self.add(o) + + def visit_super_expr(self, o: Expression) -> None: + self.add(o) + + def visit_type_var_expr(self, o: Expression) -> None: + self.add(o) + + def visit_type_alias_expr(self, o: Expression) -> None: + self.add(o) + + def visit_namedtuple_expr(self, o: Expression) -> None: + self.add(o) + + def visit_typeddict_expr(self, o: Expression) -> None: + self.add(o) + + def visit__promote_expr(self, o: Expression) -> None: + self.add(o) + + def visit_newtype_expr(self, o: Expression) -> None: + self.add(o) + + def visit_member_expr(self, e: MemberExpr) -> None: + self.add(e) + super().visit_member_expr(e) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + self.add(e) + super().visit_yield_from_expr(e) + + def visit_yield_expr(self, e: YieldExpr) -> None: + self.add(e) + super().visit_yield_expr(e) + + def visit_call_expr(self, e: CallExpr) -> None: + self.add(e) + super().visit_call_expr(e) + + def visit_op_expr(self, e: OpExpr) -> None: + self.add(e) + super().visit_op_expr(e) + + def visit_comparison_expr(self, e: ComparisonExpr) -> None: + self.add(e) + super().visit_comparison_expr(e) + + def visit_slice_expr(self, e: SliceExpr) -> None: + self.add(e) + super().visit_slice_expr(e) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.add(e) + super().visit_cast_expr(e) + + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: + self.add(e) + super().visit_assert_type_expr(e) + + def visit_reveal_expr(self, e: RevealExpr) -> None: + self.add(e) + super().visit_reveal_expr(e) + + def visit_assignment_expr(self, e: AssignmentExpr) -> None: + self.add(e) + super().visit_assignment_expr(e) + + def visit_unary_expr(self, e: UnaryExpr) -> None: + self.add(e) + super().visit_unary_expr(e) + + def visit_list_expr(self, e: ListExpr) -> None: + self.add(e) + super().visit_list_expr(e) + + def visit_tuple_expr(self, e: TupleExpr) -> None: + self.add(e) + super().visit_tuple_expr(e) + + def visit_dict_expr(self, e: DictExpr) -> None: + self.add(e) + super().visit_dict_expr(e) + + def visit_set_expr(self, e: SetExpr) -> None: + self.add(e) + super().visit_set_expr(e) + + def visit_index_expr(self, e: IndexExpr) -> None: + self.add(e) + super().visit_index_expr(e) + + def visit_generator_expr(self, e: GeneratorExpr) -> None: + self.add(e) + super().visit_generator_expr(e) + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> None: + self.add(e) + super().visit_dictionary_comprehension(e) + + def visit_list_comprehension(self, e: ListComprehension) -> None: + self.add(e) + super().visit_list_comprehension(e) + + def visit_set_comprehension(self, e: SetComprehension) -> None: + self.add(e) + super().visit_set_comprehension(e) + + def visit_conditional_expr(self, e: ConditionalExpr) -> None: + self.add(e) + super().visit_conditional_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + self.add(e) + super().visit_type_application(e) + + def visit_lambda_expr(self, e: LambdaExpr) -> None: + self.add(e) + super().visit_lambda_expr(e) + + def visit_star_expr(self, e: StarExpr) -> None: + self.add(e) + super().visit_star_expr(e) + + def visit_await_expr(self, e: AwaitExpr) -> None: + self.add(e) + super().visit_await_expr(e) + + def add(self, e: Expression) -> None: + self.expressions.append(e) diff --git a/mypy/server/target.py b/mypy/server/target.py new file mode 100644 index 000000000000..c06eeeb923f9 --- /dev/null +++ b/mypy/server/target.py @@ -0,0 +1,11 @@ +from __future__ import annotations + + +def trigger_to_target(s: str) -> str: + assert s[0] == "<" + # Strip off the angle brackets + s = s[1:-1] + # If there is a [wildcard] or similar, strip that off too + if s[-1] == "]": + s = s.split("[")[0] + return s diff --git a/mypy/server/trigger.py b/mypy/server/trigger.py new file mode 100644 index 000000000000..97b5f89cd3ba --- /dev/null +++ b/mypy/server/trigger.py @@ -0,0 +1,26 @@ +"""AST triggers that are used for fine-grained dependency handling.""" + +from __future__ import annotations + +from typing import Final + +# Used as a suffix for triggers to handle "from m import *" dependencies (see also +# make_wildcard_trigger) + +WILDCARD_TAG: Final = "[wildcard]" + + +def make_trigger(name: str) -> str: + return f"<{name}>" + + +def make_wildcard_trigger(module: str) -> str: + """Special trigger fired when any top-level name is changed in a module. + + Note that this is different from a module trigger, as module triggers are only + fired if the module is created, deleted, or replaced with a non-module, whereas + a wildcard trigger is triggered for namespace changes. + + This is used for "from m import *" dependencies. + """ + return f"<{module}{WILDCARD_TAG}>" diff --git a/mypy/server/update.py b/mypy/server/update.py new file mode 100644 index 000000000000..fdc311bbfa6b --- /dev/null +++ b/mypy/server/update.py @@ -0,0 +1,1338 @@ +"""Update build by processing changes using fine-grained dependencies. + +Use fine-grained dependencies to update targets in other modules that +may be affected by externally-visible changes in the changed modules. + +This forms the core of the fine-grained incremental daemon mode. This +module is not used at all by the 'classic' (non-daemon) incremental +mode. + +Here is some motivation for this mode: + +* By keeping program state in memory between incremental runs, we + only have to process changed modules, not their dependencies. The + classic incremental mode has to deserialize the symbol tables of + all dependencies of changed modules, which can be slow for large + programs. + +* Fine-grained dependencies allow processing only the relevant parts + of modules indirectly affected by a change. Say, if only one function + in a large module is affected by a change in another module, only this + function is processed. The classic incremental mode always processes + an entire file as a unit, which is typically much slower. + +* It's possible to independently process individual modules within an + import cycle (SCC). Small incremental changes can be fast independent + of the size of the related SCC. In classic incremental mode, any change + within a SCC requires the entire SCC to be processed, which can slow + things down considerably. + +Some terms: + +* A *target* is a function/method definition or the top level of a module. + We refer to targets using their fully qualified name (e.g. + 'mod.Cls.method'). Targets are the smallest units of processing during + fine-grained incremental checking. + +* A *trigger* represents the properties of a part of a program, and it + gets triggered/fired when these properties change. For example, + '' refers to a module-level function. It gets triggered if + the signature of the function changes, or if the function is removed, + for example. + +Some program state is maintained across multiple build increments in +memory: + +* The full ASTs of all modules are stored in memory all the time (this + includes the type map). + +* A fine-grained dependency map is maintained, which maps triggers to + affected program locations (these can be targets, triggers, or + classes). The latter determine what other parts of a program need to + be processed again due to a fired trigger. + +Here's a summary of how a fine-grained incremental program update happens: + +* Determine which modules have changes in their source code since the + previous update. + +* Process changed modules one at a time. Perform a separate full update + for each changed module, but only report the errors after all modules + have been processed, since the intermediate states can generate bogus + errors due to only seeing a partial set of changes. + +* Each changed module is processed in full. We parse the module, and + run semantic analysis to create a new AST and symbol table for the + module. Reuse the existing ASTs and symbol tables of modules that + have no changes in their source code. At the end of this stage, we have + two ASTs and symbol tables for the changed module (the old and the new + versions). The latter AST has not yet been type checked. + +* Take a snapshot of the old symbol table. This is used later to determine + which properties of the module have changed and which triggers to fire. + +* Merge the old AST with the new AST, preserving the identities of + externally visible AST nodes for which we can find a corresponding node + in the new AST. (Look at mypy.server.astmerge for the details.) This + way all external references to AST nodes in the changed module will + continue to point to the right nodes (assuming they still have a valid + target). + +* Type check the new module. + +* Take another snapshot of the symbol table of the changed module. + Look at the differences between the old and new snapshots to determine + which parts of the changed modules have changed. The result is a set of + fired triggers. + +* Using the dependency map and the fired triggers, decide which other + targets have become stale and need to be reprocessed. + +* Create new fine-grained dependencies for the changed module. We don't + garbage collect old dependencies, since extra dependencies are relatively + harmless (they take some memory and can theoretically slow things down + a bit by causing redundant work). This is implemented in + mypy.server.deps. + +* Strip the stale AST nodes that we found above. This returns them to a + state resembling the end of semantic analysis pass 1. We'll run semantic + analysis again on the existing AST nodes, and since semantic analysis + is not idempotent, we need to revert some changes made during semantic + analysis. This is implemented in mypy.server.aststrip. + +* Run semantic analyzer passes 2 and 3 on the stale AST nodes, and type + check them. We also need to do the symbol table snapshot comparison + dance to find any changes, and we need to merge ASTs to preserve AST node + identities. + +* If some triggers haven been fired, continue processing and repeat the + previous steps until no triggers are fired. + +This is module is tested using end-to-end fine-grained incremental mode +test cases (test-data/unit/fine-grained*.test). +""" + +from __future__ import annotations + +import os +import re +import sys +import time +from typing import Callable, Final, NamedTuple, Sequence, Union +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.build import ( + DEBUG_FINE_GRAINED, + FAKE_ROOT_MODULE, + BuildManager, + BuildResult, + Graph, + State, + load_graph, + process_fresh_modules, +) +from mypy.checker import FineGrainedDeferredNode +from mypy.errors import CompileError +from mypy.fscache import FileSystemCache +from mypy.modulefinder import BuildSource +from mypy.nodes import ( + Decorator, + FuncDef, + ImportFrom, + MypyFile, + OverloadedFuncDef, + SymbolNode, + SymbolTable, + TypeInfo, +) +from mypy.options import Options +from mypy.semanal_main import semantic_analysis_for_scc, semantic_analysis_for_targets +from mypy.server.astdiff import ( + SymbolSnapshot, + compare_symbol_table_snapshots, + snapshot_symbol_table, +) +from mypy.server.astmerge import merge_asts +from mypy.server.aststrip import SavedAttributes, strip_target +from mypy.server.deps import get_dependencies_of_target, merge_dependencies +from mypy.server.target import trigger_to_target +from mypy.server.trigger import WILDCARD_TAG, make_trigger +from mypy.typestate import type_state +from mypy.util import is_stdlib_file, module_prefix, split_target + +MAX_ITER: Final = 1000 + +# These are modules beyond stdlib that have some special meaning for mypy. +SENSITIVE_INTERNAL_MODULES = ("mypy_extensions", "typing_extensions") + + +class FineGrainedBuildManager: + def __init__(self, result: BuildResult) -> None: + """Initialize fine-grained build based on a batch build. + + Args: + result: Result from the initialized build. + The manager and graph will be taken over by this class. + manager: State of the build (mutated by this class) + graph: Additional state of the build (mutated by this class) + """ + manager = result.manager + self.manager = manager + self.graph = result.graph + self.previous_modules = get_module_to_path_map(self.graph) + self.deps = manager.fg_deps + # Merge in any root dependencies that may not have been loaded + merge_dependencies(manager.load_fine_grained_deps(FAKE_ROOT_MODULE), self.deps) + self.previous_targets_with_errors = manager.errors.targets() + self.previous_messages: list[str] = result.errors.copy() + # Module, if any, that had blocking errors in the last run as (id, path) tuple. + self.blocking_error: tuple[str, str] | None = None + # Module that we haven't processed yet but that are known to be stale. + self.stale: list[tuple[str, str]] = [] + # Disable the cache so that load_graph doesn't try going back to disk + # for the cache. + self.manager.cache_enabled = False + + # Some hints to the test suite about what is going on: + # Active triggers during the last update + self.triggered: list[str] = [] + # Modules passed to update during the last update + self.changed_modules: list[tuple[str, str]] = [] + # Modules processed during the last update + self.updated_modules: list[str] = [] + # Targets processed during last update (for testing only). + self.processed_targets: list[str] = [] + + def update( + self, + changed_modules: list[tuple[str, str]], + removed_modules: list[tuple[str, str]], + followed: bool = False, + ) -> list[str]: + """Update previous build result by processing changed modules. + + Also propagate changes to other modules as needed, but only process + those parts of other modules that are affected by the changes. Retain + the existing ASTs and symbol tables of unaffected modules. + + Reuses original BuildManager and Graph. + + Args: + changed_modules: Modules changed since the previous update/build; each is + a (module id, path) tuple. Includes modified and added modules. + Assume this is correct; it's not validated here. + removed_modules: Modules that have been deleted since the previous update + or removed from the build. + followed: If True, the modules were found through following imports + + Returns: + A list of errors. + """ + self.processed_targets.clear() + changed_modules = changed_modules + removed_modules + removed_set = {module for module, _ in removed_modules} + self.changed_modules = changed_modules + + if not changed_modules: + return self.previous_messages + + # Reset find_module's caches for the new build. + self.manager.find_module_cache.clear() + + self.triggered = [] + self.updated_modules = [] + changed_modules = dedupe_modules(changed_modules + self.stale) + initial_set = {id for id, _ in changed_modules} + self.manager.log_fine_grained( + "==== update %s ====" % ", ".join(repr(id) for id, _ in changed_modules) + ) + if self.previous_targets_with_errors and is_verbose(self.manager): + self.manager.log_fine_grained( + "previous targets with errors: %s" % sorted(self.previous_targets_with_errors) + ) + + blocking_error = None + if self.blocking_error: + # Handle blocking errors first. We'll exit as soon as we find a + # module that still has blocking errors. + self.manager.log_fine_grained(f"existing blocker: {self.blocking_error[0]}") + changed_modules = dedupe_modules([self.blocking_error] + changed_modules) + blocking_error = self.blocking_error[0] + self.blocking_error = None + + while True: + result = self.update_one( + changed_modules, initial_set, removed_set, blocking_error, followed + ) + changed_modules, (next_id, next_path), blocker_messages = result + + if blocker_messages is not None: + self.blocking_error = (next_id, next_path) + self.stale = changed_modules + messages = blocker_messages + break + + # It looks like we are done processing everything, so now + # reprocess all targets with errors. We are careful to + # support the possibility that reprocessing an errored module + # might trigger loading of a module, but I am not sure + # if this can really happen. + if not changed_modules: + # N.B: We just checked next_id, so manager.errors contains + # the errors from it. Thus we consider next_id up to date + # when propagating changes from the errored targets, + # which prevents us from reprocessing errors in it. + changed_modules = propagate_changes_using_dependencies( + self.manager, + self.graph, + self.deps, + set(), + {next_id}, + self.previous_targets_with_errors, + self.processed_targets, + ) + changed_modules = dedupe_modules(changed_modules) + if not changed_modules: + # Preserve state needed for the next update. + self.previous_targets_with_errors = self.manager.errors.targets() + messages = self.manager.errors.new_messages() + break + + messages = sort_messages_preserving_file_order(messages, self.previous_messages) + self.previous_messages = messages.copy() + return messages + + def trigger(self, target: str) -> list[str]: + """Trigger a specific target explicitly. + + This is intended for use by the suggestions engine. + """ + self.manager.errors.reset() + changed_modules = propagate_changes_using_dependencies( + self.manager, + self.graph, + self.deps, + set(), + set(), + self.previous_targets_with_errors | {target}, + [], + ) + # Preserve state needed for the next update. + self.previous_targets_with_errors = self.manager.errors.targets() + self.previous_messages = self.manager.errors.new_messages().copy() + return self.update(changed_modules, []) + + def flush_cache(self) -> None: + """Flush AST cache. + + This needs to be called after each increment, or file changes won't + be detected reliably. + """ + self.manager.ast_cache.clear() + + def update_one( + self, + changed_modules: list[tuple[str, str]], + initial_set: set[str], + removed_set: set[str], + blocking_error: str | None, + followed: bool, + ) -> tuple[list[tuple[str, str]], tuple[str, str], list[str] | None]: + """Process a module from the list of changed modules. + + Returns: + Tuple with these items: + + - Updated list of pending changed modules as (module id, path) tuples + - Module which was actually processed as (id, path) tuple + - If there was a blocking error, the error messages from it + """ + t0 = time.time() + next_id, next_path = changed_modules.pop(0) + + # If we have a module with a blocking error that is no longer + # in the import graph, we must skip it as otherwise we'll be + # stuck with the blocking error. + if ( + next_id == blocking_error + and next_id not in self.previous_modules + and next_id not in initial_set + ): + self.manager.log_fine_grained( + f"skip {next_id!r} (module with blocking error not in import graph)" + ) + return changed_modules, (next_id, next_path), None + + result = self.update_module(next_id, next_path, next_id in removed_set, followed) + remaining, (next_id, next_path), blocker_messages = result + changed_modules = [(id, path) for id, path in changed_modules if id != next_id] + changed_modules = dedupe_modules(remaining + changed_modules) + t1 = time.time() + + self.manager.log_fine_grained( + f"update once: {next_id} in {t1 - t0:.3f}s - {len(changed_modules)} left" + ) + + return changed_modules, (next_id, next_path), blocker_messages + + def update_module( + self, module: str, path: str, force_removed: bool, followed: bool + ) -> tuple[list[tuple[str, str]], tuple[str, str], list[str] | None]: + """Update a single modified module. + + If the module contains imports of previously unseen modules, only process one of + the new modules and return the remaining work to be done. + + Args: + module: Id of the module + path: File system path of the module + force_removed: If True, consider module removed from the build even if path + exists (used for removing an existing file from the build) + followed: Was this found via import following? + + Returns: + Tuple with these items: + + - Remaining modules to process as (module id, path) tuples + - Module which was actually processed as (id, path) tuple + - If there was a blocking error, the error messages from it + """ + self.manager.log_fine_grained(f"--- update single {module!r} ---") + self.updated_modules.append(module) + + # builtins and friends could potentially get triggered because + # of protocol stuff, but nothing good could possibly come from + # actually updating them. + if ( + is_stdlib_file(self.manager.options.abs_custom_typeshed_dir, path) + or module in SENSITIVE_INTERNAL_MODULES + ): + return [], (module, path), None + + manager = self.manager + previous_modules = self.previous_modules + graph = self.graph + + ensure_deps_loaded(module, self.deps, graph) + + # If this is an already existing module, make sure that we have + # its tree loaded so that we can snapshot it for comparison. + ensure_trees_loaded(manager, graph, [module]) + + t0 = time.time() + # Record symbol table snapshot of old version the changed module. + old_snapshots: dict[str, dict[str, SymbolSnapshot]] = {} + if module in manager.modules: + snapshot = snapshot_symbol_table(module, manager.modules[module].names) + old_snapshots[module] = snapshot + + manager.errors.reset() + self.processed_targets.append(module) + result = update_module_isolated( + module, path, manager, previous_modules, graph, force_removed, followed + ) + if isinstance(result, BlockedUpdate): + # Blocking error -- just give up + module, path, remaining, errors = result + self.previous_modules = get_module_to_path_map(graph) + return remaining, (module, path), errors + assert isinstance(result, NormalUpdate) # Work around #4124 + module, path, remaining, tree = result + + # TODO: What to do with stale dependencies? + t1 = time.time() + triggered = calculate_active_triggers(manager, old_snapshots, {module: tree}) + if is_verbose(self.manager): + filtered = [trigger for trigger in triggered if not trigger.endswith("__>")] + self.manager.log_fine_grained(f"triggered: {sorted(filtered)!r}") + self.triggered.extend(triggered | self.previous_targets_with_errors) + if module in graph: + graph[module].update_fine_grained_deps(self.deps) + graph[module].free_state() + remaining += propagate_changes_using_dependencies( + manager, + graph, + self.deps, + triggered, + {module}, + targets_with_errors=set(), + processed_targets=self.processed_targets, + ) + t2 = time.time() + manager.add_stats(update_isolated_time=t1 - t0, propagate_time=t2 - t1) + + # Preserve state needed for the next update. + self.previous_targets_with_errors.update(manager.errors.targets()) + self.previous_modules = get_module_to_path_map(graph) + + return remaining, (module, path), None + + +def find_unloaded_deps( + manager: BuildManager, graph: dict[str, State], initial: Sequence[str] +) -> list[str]: + """Find all the deps of the nodes in initial that haven't had their tree loaded. + + The key invariant here is that if a module is loaded, so are all + of their dependencies. This means that when we encounter a loaded + module, we don't need to explore its dependencies. (This + invariant is slightly violated when dependencies are added, which + can be handled by calling find_unloaded_deps directly on the new + dependencies.) + """ + worklist = list(initial) + seen: set[str] = set() + unloaded = [] + while worklist: + node = worklist.pop() + if node in seen or node not in graph: + continue + seen.add(node) + if node not in manager.modules: + ancestors = graph[node].ancestors or [] + worklist.extend(graph[node].dependencies + ancestors) + unloaded.append(node) + + return unloaded + + +def ensure_deps_loaded(module: str, deps: dict[str, set[str]], graph: dict[str, State]) -> None: + """Ensure that the dependencies on a module are loaded. + + Dependencies are loaded into the 'deps' dictionary. + + This also requires loading dependencies from any parent modules, + since dependencies will get stored with parent modules when a module + doesn't exist. + """ + if module in graph and graph[module].fine_grained_deps_loaded: + return + parts = module.split(".") + for i in range(len(parts)): + base = ".".join(parts[: i + 1]) + if base in graph and not graph[base].fine_grained_deps_loaded: + merge_dependencies(graph[base].load_fine_grained_deps(), deps) + graph[base].fine_grained_deps_loaded = True + + +def ensure_trees_loaded( + manager: BuildManager, graph: dict[str, State], initial: Sequence[str] +) -> None: + """Ensure that the modules in initial and their deps have loaded trees.""" + to_process = find_unloaded_deps(manager, graph, initial) + if to_process: + if is_verbose(manager): + manager.log_fine_grained( + "Calling process_fresh_modules on set of size {} ({})".format( + len(to_process), sorted(to_process) + ) + ) + process_fresh_modules(graph, to_process, manager) + + +# The result of update_module_isolated when no blockers, with these items: +# +# - Id of the changed module (can be different from the module argument) +# - Path of the changed module +# - New AST for the changed module (None if module was deleted) +# - Remaining changed modules that are not processed yet as (module id, path) +# tuples (non-empty if the original changed module imported other new +# modules) +class NormalUpdate(NamedTuple): + module: str + path: str + remaining: list[tuple[str, str]] + tree: MypyFile | None + + +# The result of update_module_isolated when there is a blocking error. Items +# are similar to NormalUpdate (but there are fewer). +class BlockedUpdate(NamedTuple): + module: str + path: str + remaining: list[tuple[str, str]] + messages: list[str] + + +UpdateResult: _TypeAlias = Union[NormalUpdate, BlockedUpdate] + + +def update_module_isolated( + module: str, + path: str, + manager: BuildManager, + previous_modules: dict[str, str], + graph: Graph, + force_removed: bool, + followed: bool, +) -> UpdateResult: + """Build a new version of one changed module only. + + Don't propagate changes to elsewhere in the program. Raise CompileError on + encountering a blocking error. + + Args: + module: Changed module (modified, created or deleted) + path: Path of the changed module + manager: Build manager + graph: Build graph + force_removed: If True, consider the module removed from the build even it the + file exists + + Returns a named tuple describing the result (see above for details). + """ + if module not in graph: + manager.log_fine_grained(f"new module {module!r}") + + if not manager.fscache.isfile(path) or force_removed: + delete_module(module, path, graph, manager) + return NormalUpdate(module, path, [], None) + + sources = get_sources(manager.fscache, previous_modules, [(module, path)], followed) + + if module in manager.missing_modules: + manager.missing_modules.remove(module) + + orig_module = module + orig_state = graph.get(module) + orig_tree = manager.modules.get(module) + + def restore(ids: list[str]) -> None: + # For each of the modules in ids, restore that id's old + # manager.modules and graphs entries. (Except for the original + # module, this means deleting them.) + for id in ids: + if id == orig_module and orig_tree: + manager.modules[id] = orig_tree + elif id in manager.modules: + del manager.modules[id] + if id == orig_module and orig_state: + graph[id] = orig_state + elif id in graph: + del graph[id] + + new_modules: list[State] = [] + try: + if module in graph: + del graph[module] + load_graph(sources, manager, graph, new_modules) + except CompileError as err: + # Parse error somewhere in the program -- a blocker + assert err.module_with_blocker + restore([module] + [st.id for st in new_modules]) + return BlockedUpdate(err.module_with_blocker, path, [], err.messages) + + # Reparsing the file may have brought in dependencies that we + # didn't have before. Make sure that they are loaded to restore + # the invariant that a module having a loaded tree implies that + # its dependencies do as well. + ensure_trees_loaded(manager, graph, graph[module].dependencies) + + # Find any other modules brought in by imports. + changed_modules = [(st.id, st.xpath) for st in new_modules] + + # If there are multiple modules to process, only process one of them and return + # the remaining ones to the caller. + if len(changed_modules) > 1: + # As an optimization, look for a module that imports no other changed modules. + module, path = find_relative_leaf_module(changed_modules, graph) + changed_modules.remove((module, path)) + remaining_modules = changed_modules + # The remaining modules haven't been processed yet so drop them. + restore([id for id, _ in remaining_modules]) + manager.log_fine_grained(f"--> {module!r} (newly imported)") + else: + remaining_modules = [] + + state = graph[module] + + # Process the changed file. + state.parse_file() + assert state.tree is not None, "file must be at least parsed" + t0 = time.time() + try: + semantic_analysis_for_scc(graph, [state.id], manager.errors) + except CompileError as err: + # There was a blocking error, so module AST is incomplete. Restore old modules. + restore([module]) + return BlockedUpdate(module, path, remaining_modules, err.messages) + + # Merge old and new ASTs. + new_modules_dict: dict[str, MypyFile | None] = {module: state.tree} + replace_modules_with_new_variants(manager, graph, {orig_module: orig_tree}, new_modules_dict) + + t1 = time.time() + # Perform type checking. + state.type_checker().reset() + state.type_check_first_pass() + state.type_check_second_pass() + state.detect_possibly_undefined_vars() + t2 = time.time() + state.finish_passes() + t3 = time.time() + manager.add_stats(semanal_time=t1 - t0, typecheck_time=t2 - t1, finish_passes_time=t3 - t2) + + graph[module] = state + + return NormalUpdate(module, path, remaining_modules, state.tree) + + +def find_relative_leaf_module(modules: list[tuple[str, str]], graph: Graph) -> tuple[str, str]: + """Find a module in a list that directly imports no other module in the list. + + If no such module exists, return the lexicographically first module from the list. + Always return one of the items in the modules list. + + NOTE: If both 'abc' and 'typing' have changed, an effect of the above rule is that + we prefer 'abc', even if both are in the same SCC. This works around a false + positive in 'typing', at least in tests. + + Args: + modules: List of (module, path) tuples (non-empty) + graph: Program import graph that contains all modules in the module list + """ + assert modules + # Sort for repeatable results. + modules = sorted(modules) + module_set = {module for module, _ in modules} + for module, path in modules: + state = graph[module] + if len(set(state.dependencies) & module_set) == 0: + # Found it! + return module, path + # Could not find any. Just return the first module (by lexicographic order). + return modules[0] + + +def delete_module(module_id: str, path: str, graph: Graph, manager: BuildManager) -> None: + manager.log_fine_grained(f"delete module {module_id!r}") + # TODO: Remove deps for the module (this only affects memory use, not correctness) + if module_id in graph: + del graph[module_id] + if module_id in manager.modules: + del manager.modules[module_id] + components = module_id.split(".") + if len(components) > 1: + # Delete reference to module in parent module. + parent_id = ".".join(components[:-1]) + # If parent module is ignored, it won't be included in the modules dictionary. + if parent_id in manager.modules: + parent = manager.modules[parent_id] + if components[-1] in parent.names: + del parent.names[components[-1]] + # If the module is removed from the build but still exists, then + # we mark it as missing so that it will get picked up by import from still. + if manager.fscache.isfile(path): + manager.missing_modules.add(module_id) + + +def dedupe_modules(modules: list[tuple[str, str]]) -> list[tuple[str, str]]: + seen: set[str] = set() + result = [] + for id, path in modules: + if id not in seen: + seen.add(id) + result.append((id, path)) + return result + + +def get_module_to_path_map(graph: Graph) -> dict[str, str]: + return {module: node.xpath for module, node in graph.items()} + + +def get_sources( + fscache: FileSystemCache, + modules: dict[str, str], + changed_modules: list[tuple[str, str]], + followed: bool, +) -> list[BuildSource]: + sources = [] + for id, path in changed_modules: + if fscache.isfile(path): + sources.append(BuildSource(path, id, None, followed=followed)) + return sources + + +def calculate_active_triggers( + manager: BuildManager, + old_snapshots: dict[str, dict[str, SymbolSnapshot]], + new_modules: dict[str, MypyFile | None], +) -> set[str]: + """Determine activated triggers by comparing old and new symbol tables. + + For example, if only the signature of function m.f is different in the new + symbol table, return {''}. + """ + names: set[str] = set() + for id in new_modules: + snapshot1 = old_snapshots.get(id) + if snapshot1 is None: + names.add(id) + snapshot1 = {} + new = new_modules[id] + if new is None: + snapshot2 = snapshot_symbol_table(id, SymbolTable()) + names.add(id) + else: + snapshot2 = snapshot_symbol_table(id, new.names) + diff = compare_symbol_table_snapshots(id, snapshot1, snapshot2) + package_nesting_level = id.count(".") + for item in diff.copy(): + if item.count(".") <= package_nesting_level + 1 and item.split(".")[-1] not in ( + "__builtins__", + "__file__", + "__name__", + "__package__", + "__doc__", + ): + # Activate catch-all wildcard trigger for top-level module changes (used for + # "from m import *"). This also gets triggered by changes to module-private + # entries, but as these unneeded dependencies only result in extra processing, + # it's a minor problem. + # + # TODO: Some __* names cause mistriggers. Fix the underlying issue instead of + # special casing them here. + diff.add(id + WILDCARD_TAG) + if item.count(".") > package_nesting_level + 1: + # These are for changes within classes, used by protocols. + diff.add(item.rsplit(".", 1)[0] + WILDCARD_TAG) + + names |= diff + return {make_trigger(name) for name in names} + + +def replace_modules_with_new_variants( + manager: BuildManager, + graph: dict[str, State], + old_modules: dict[str, MypyFile | None], + new_modules: dict[str, MypyFile | None], +) -> None: + """Replace modules with newly builds versions. + + Retain the identities of externally visible AST nodes in the + old ASTs so that references to the affected modules from other + modules will still be valid (unless something was deleted or + replaced with an incompatible definition, in which case there + will be dangling references that will be handled by + propagate_changes_using_dependencies). + """ + for id in new_modules: + preserved_module = old_modules.get(id) + new_module = new_modules[id] + if preserved_module and new_module is not None: + merge_asts(preserved_module, preserved_module.names, new_module, new_module.names) + manager.modules[id] = preserved_module + graph[id].tree = preserved_module + + +def propagate_changes_using_dependencies( + manager: BuildManager, + graph: dict[str, State], + deps: dict[str, set[str]], + triggered: set[str], + up_to_date_modules: set[str], + targets_with_errors: set[str], + processed_targets: list[str], +) -> list[tuple[str, str]]: + """Transitively rechecks targets based on triggers and the dependency map. + + Returns a list (module id, path) tuples representing modules that contain + a target that needs to be reprocessed but that has not been parsed yet. + + Processed targets should be appended to processed_targets (used in tests only, + to test the order of processing targets). + """ + + num_iter = 0 + remaining_modules: list[tuple[str, str]] = [] + + # Propagate changes until nothing visible has changed during the last + # iteration. + while triggered or targets_with_errors: + num_iter += 1 + if num_iter > MAX_ITER: + raise RuntimeError("Max number of iterations (%d) reached (endless loop?)" % MAX_ITER) + + todo, unloaded, stale_protos = find_targets_recursive( + manager, graph, triggered, deps, up_to_date_modules + ) + # TODO: we sort to make it deterministic, but this is *incredibly* ad hoc + remaining_modules.extend((id, graph[id].xpath) for id in sorted(unloaded)) + # Also process targets that used to have errors, as otherwise some + # errors might be lost. + for target in targets_with_errors: + id = module_prefix(graph, target) + if id is not None and id not in up_to_date_modules: + if id not in todo: + todo[id] = set() + manager.log_fine_grained(f"process target with error: {target}") + more_nodes, _ = lookup_target(manager, target) + todo[id].update(more_nodes) + triggered = set() + # First invalidate subtype caches in all stale protocols. + # We need to do this to avoid false negatives if the protocol itself is + # unchanged, but was marked stale because its sub- (or super-) type changed. + for info in stale_protos: + type_state.reset_subtype_caches_for(info) + # Then fully reprocess all targets. + # TODO: Preserve order (set is not optimal) + for id, nodes in sorted(todo.items(), key=lambda x: x[0]): + assert id not in up_to_date_modules + triggered |= reprocess_nodes(manager, graph, id, nodes, deps, processed_targets) + # Changes elsewhere may require us to reprocess modules that were + # previously considered up to date. For example, there may be a + # dependency loop that loops back to an originally processed module. + up_to_date_modules = set() + targets_with_errors = set() + if is_verbose(manager): + manager.log_fine_grained(f"triggered: {list(triggered)!r}") + + return remaining_modules + + +def find_targets_recursive( + manager: BuildManager, + graph: Graph, + triggers: set[str], + deps: dict[str, set[str]], + up_to_date_modules: set[str], +) -> tuple[dict[str, set[FineGrainedDeferredNode]], set[str], set[TypeInfo]]: + """Find names of all targets that need to reprocessed, given some triggers. + + Returns: A tuple containing a: + * Dictionary from module id to a set of stale targets. + * A set of module ids for unparsed modules with stale targets. + """ + result: dict[str, set[FineGrainedDeferredNode]] = {} + worklist = triggers + processed: set[str] = set() + stale_protos: set[TypeInfo] = set() + unloaded_files: set[str] = set() + + # Find AST nodes corresponding to each target. + # + # TODO: Don't rely on a set, since the items are in an unpredictable order. + while worklist: + processed |= worklist + current = worklist + worklist = set() + for target in current: + if target.startswith("<"): + module_id = module_prefix(graph, trigger_to_target(target)) + if module_id: + ensure_deps_loaded(module_id, deps, graph) + + worklist |= deps.get(target, set()) - processed + else: + module_id = module_prefix(graph, target) + if module_id is None: + # Deleted module. + continue + if module_id in up_to_date_modules: + # Already processed. + continue + if ( + module_id not in manager.modules + or manager.modules[module_id].is_cache_skeleton + ): + # We haven't actually parsed and checked the module, so we don't have + # access to the actual nodes. + # Add it to the queue of files that need to be processed fully. + unloaded_files.add(module_id) + continue + + if module_id not in result: + result[module_id] = set() + manager.log_fine_grained(f"process: {target}") + deferred, stale_proto = lookup_target(manager, target) + if stale_proto: + stale_protos.add(stale_proto) + result[module_id].update(deferred) + + return result, unloaded_files, stale_protos + + +def reprocess_nodes( + manager: BuildManager, + graph: dict[str, State], + module_id: str, + nodeset: set[FineGrainedDeferredNode], + deps: dict[str, set[str]], + processed_targets: list[str], +) -> set[str]: + """Reprocess a set of nodes within a single module. + + Return fired triggers. + """ + if module_id not in graph: + manager.log_fine_grained("%s not in graph (blocking errors or deleted?)" % module_id) + return set() + + file_node = manager.modules[module_id] + old_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names) + old_symbols = {name: names.copy() for name, names in old_symbols.items()} + old_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names) + + def key(node: FineGrainedDeferredNode) -> int: + # Unlike modules which are sorted by name within SCC, + # nodes within the same module are sorted by line number, because + # this is how they are processed in normal mode. + return node.node.line + + nodes = sorted(nodeset, key=key) + + state = graph[module_id] + options = state.options + manager.errors.set_file_ignored_lines( + file_node.path, file_node.ignored_lines, options.ignore_errors or state.ignore_all + ) + manager.errors.set_skipped_lines(file_node.path, file_node.skipped_lines) + + targets = set() + for node in nodes: + target = target_from_node(module_id, node.node) + if target is not None: + targets.add(target) + manager.errors.clear_errors_in_targets(file_node.path, targets) + + # If one of the nodes is the module itself, emit any errors that + # happened before semantic analysis. + for target in targets: + if target == module_id: + for info in graph[module_id].early_errors: + manager.errors.add_error_info(info) + + # Strip semantic analysis information. + saved_attrs: SavedAttributes = {} + for deferred in nodes: + processed_targets.append(deferred.node.fullname) + strip_target(deferred.node, saved_attrs) + semantic_analysis_for_targets(graph[module_id], nodes, graph, saved_attrs) + # Merge symbol tables to preserve identities of AST nodes. The file node will remain + # the same, but other nodes may have been recreated with different identities, such as + # NamedTuples defined using assignment statements. + new_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names) + for name in old_symbols: + if name in new_symbols: + merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) + + # Type check. + checker = graph[module_id].type_checker() + checker.reset() + # We seem to need additional passes in fine-grained incremental mode. + checker.pass_num = 0 + checker.last_pass = 3 + more = checker.check_second_pass(nodes) + while more: + more = False + if graph[module_id].type_checker().check_second_pass(): + more = True + + if manager.options.export_types: + manager.all_types.update(graph[module_id].type_map()) + + new_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names) + # Check if any attribute types were changed and need to be propagated further. + changed = compare_symbol_table_snapshots( + file_node.fullname, old_symbols_snapshot, new_symbols_snapshot + ) + new_triggered = {make_trigger(name) for name in changed} + + # Dependencies may have changed. + update_deps(module_id, nodes, graph, deps, options) + + # Report missing imports. + graph[module_id].verify_dependencies() + + graph[module_id].free_state() + + return new_triggered + + +def find_symbol_tables_recursive(prefix: str, symbols: SymbolTable) -> dict[str, SymbolTable]: + """Find all nested symbol tables. + + Args: + prefix: Full name prefix (used for return value keys and to filter result so that + cross references to other modules aren't included) + symbols: Root symbol table + + Returns a dictionary from full name to corresponding symbol table. + """ + result = {prefix: symbols} + for name, node in symbols.items(): + if isinstance(node.node, TypeInfo) and node.node.fullname.startswith(prefix + "."): + more = find_symbol_tables_recursive(prefix + "." + name, node.node.names) + result.update(more) + return result + + +def update_deps( + module_id: str, + nodes: list[FineGrainedDeferredNode], + graph: dict[str, State], + deps: dict[str, set[str]], + options: Options, +) -> None: + for deferred in nodes: + node = deferred.node + type_map = graph[module_id].type_map() + tree = graph[module_id].tree + assert tree is not None, "Tree must be processed at this stage" + new_deps = get_dependencies_of_target( + module_id, tree, node, type_map, options.python_version + ) + for trigger, targets in new_deps.items(): + deps.setdefault(trigger, set()).update(targets) + # Merge also the newly added protocol deps (if any). + type_state.update_protocol_deps(deps) + + +def lookup_target( + manager: BuildManager, target: str +) -> tuple[list[FineGrainedDeferredNode], TypeInfo | None]: + """Look up a target by fully-qualified name. + + The first item in the return tuple is a list of deferred nodes that + needs to be reprocessed. If the target represents a TypeInfo corresponding + to a protocol, return it as a second item in the return tuple, otherwise None. + """ + + def not_found() -> None: + manager.log_fine_grained(f"Can't find matching target for {target} (stale dependency?)") + + modules = manager.modules + items = split_target(modules, target) + if items is None: + not_found() # Stale dependency + return [], None + module, rest = items + if rest: + components = rest.split(".") + else: + components = [] + node: SymbolNode | None = modules[module] + file: MypyFile | None = None + active_class = None + for c in components: + if isinstance(node, TypeInfo): + active_class = node + if isinstance(node, MypyFile): + file = node + if not isinstance(node, (MypyFile, TypeInfo)) or c not in node.names: + not_found() # Stale dependency + return [], None + # Don't reprocess plugin generated targets. They should get + # stripped and regenerated when the containing target is + # reprocessed. + if node.names[c].plugin_generated: + return [], None + node = node.names[c].node + if isinstance(node, TypeInfo): + # A ClassDef target covers the body of the class and everything defined + # within it. To get the body we include the entire surrounding target, + # typically a module top-level, since we don't support processing class + # bodies as separate entities for simplicity. + assert file is not None + if node.fullname != target: + # This is a reference to a different TypeInfo, likely due to a stale dependency. + # Processing them would spell trouble -- for example, we could be refreshing + # a deserialized TypeInfo with missing attributes. + not_found() + return [], None + result = [FineGrainedDeferredNode(file, None)] + stale_info: TypeInfo | None = None + if node.is_protocol: + stale_info = node + for name, symnode in node.names.items(): + node = symnode.node + if isinstance(node, FuncDef): + method, _ = lookup_target(manager, target + "." + name) + result.extend(method) + return result, stale_info + if isinstance(node, Decorator): + # Decorator targets actually refer to the function definition only. + node = node.func + if not isinstance(node, (FuncDef, MypyFile, OverloadedFuncDef)): + # The target can't be refreshed. It's possible that the target was + # changed to another type and we have a stale dependency pointing to it. + not_found() + return [], None + if node.fullname != target: + # Stale reference points to something unexpected. We shouldn't process since the + # context will be wrong and it could be a partially initialized deserialized node. + not_found() + return [], None + return [FineGrainedDeferredNode(node, active_class)], None + + +def is_verbose(manager: BuildManager) -> bool: + return manager.options.verbosity >= 1 or DEBUG_FINE_GRAINED + + +def target_from_node(module: str, node: FuncDef | MypyFile | OverloadedFuncDef) -> str | None: + """Return the target name corresponding to a deferred node. + + Args: + module: Must be module id of the module that defines 'node' + + Returns the target name, or None if the node is not a valid target in the given + module (for example, if it's actually defined in another module). + """ + if isinstance(node, MypyFile): + if module != node.fullname: + # Actually a reference to another module -- likely a stale dependency. + return None + return module + else: # OverloadedFuncDef or FuncDef + if node.info: + return f"{node.info.fullname}.{node.name}" + else: + return f"{module}.{node.name}" + + +if sys.platform != "win32": + INIT_SUFFIXES: Final = ("/__init__.py", "/__init__.pyi") +else: + INIT_SUFFIXES: Final = ( + os.sep + "__init__.py", + os.sep + "__init__.pyi", + os.altsep + "__init__.py", + os.altsep + "__init__.pyi", + ) + + +def refresh_suppressed_submodules( + module: str, + path: str | None, + deps: dict[str, set[str]], + graph: Graph, + fscache: FileSystemCache, + refresh_file: Callable[[str, str], list[str]], +) -> list[str] | None: + """Look for submodules that are now suppressed in target package. + + If a submodule a.b gets added, we need to mark it as suppressed + in modules that contain "from a import b". Previously we assumed + that 'a.b' is not a module but a regular name. + + This is only relevant when following imports normally. + + Args: + module: target package in which to look for submodules + path: path of the module + refresh_file: function that reads the AST of a module (returns error messages) + + Return a list of errors from refresh_file() if it was called. If the + return value is None, we didn't call refresh_file(). + """ + messages = None + if path is None or not path.endswith(INIT_SUFFIXES): + # Only packages have submodules. + return None + # Find any submodules present in the directory. + pkgdir = os.path.dirname(path) + try: + entries = fscache.listdir(pkgdir) + except FileNotFoundError: + entries = [] + for fnam in entries: + if ( + not fnam.endswith((".py", ".pyi")) + or fnam.startswith("__init__.") + or fnam.count(".") != 1 + ): + continue + shortname = fnam.split(".")[0] + submodule = module + "." + shortname + trigger = make_trigger(submodule) + + # We may be missing the required fine-grained deps. + ensure_deps_loaded(module, deps, graph) + + if trigger in deps: + for dep in deps[trigger]: + # We can ignore <...> deps since a submodule can't trigger any. + state = graph.get(dep) + if not state: + # Maybe it's a non-top-level target. We only care about the module. + dep_module = module_prefix(graph, dep) + if dep_module is not None: + state = graph.get(dep_module) + if state: + # Is the file may missing an AST in case it's read from cache? + if state.tree is None: + # Create AST for the file. This may produce some new errors + # that we need to propagate. + assert state.path is not None + messages = refresh_file(state.id, state.path) + tree = state.tree + assert tree # Will be fine, due to refresh_file() above + for imp in tree.imports: + if isinstance(imp, ImportFrom): + if ( + imp.id == module + and any(name == shortname for name, _ in imp.names) + and submodule not in state.suppressed_set + ): + state.suppressed.append(submodule) + state.suppressed_set.add(submodule) + return messages + + +def extract_fnam_from_message(message: str) -> str | None: + m = re.match(r"([^:]+):[0-9]+: (error|note): ", message) + if m: + return m.group(1) + return None + + +def extract_possible_fnam_from_message(message: str) -> str: + # This may return non-path things if there is some random colon on the line + return message.split(":", 1)[0] + + +def sort_messages_preserving_file_order( + messages: list[str], prev_messages: list[str] +) -> list[str]: + """Sort messages so that the order of files is preserved. + + An update generates messages so that the files can be in a fairly + arbitrary order. Preserve the order of files to avoid messages + getting reshuffled continuously. If there are messages in + additional files, sort them towards the end. + """ + # Calculate file order from the previous messages + n = 0 + order = {} + for msg in prev_messages: + fnam = extract_fnam_from_message(msg) + if fnam and fnam not in order: + order[fnam] = n + n += 1 + + # Related messages must be sorted as a group of successive lines + groups = [] + i = 0 + while i < len(messages): + msg = messages[i] + maybe_fnam = extract_possible_fnam_from_message(msg) + group = [msg] + if maybe_fnam in order: + # This looks like a file name. Collect all lines related to this message. + while ( + i + 1 < len(messages) + and extract_possible_fnam_from_message(messages[i + 1]) not in order + and extract_fnam_from_message(messages[i + 1]) is None + and not messages[i + 1].startswith("mypy: ") + ): + i += 1 + group.append(messages[i]) + groups.append((order.get(maybe_fnam, n), group)) + i += 1 + + groups = sorted(groups, key=lambda g: g[0]) + result = [] + for key, group in groups: + result.extend(group) + return result diff --git a/mypy/sharedparse.py b/mypy/sharedparse.py new file mode 100644 index 000000000000..ef2e4f720664 --- /dev/null +++ b/mypy/sharedparse.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from typing import Final + +"""Shared logic between our three mypy parser files.""" + + +_NON_BINARY_MAGIC_METHODS: Final = { + "__abs__", + "__call__", + "__complex__", + "__contains__", + "__del__", + "__delattr__", + "__delitem__", + "__enter__", + "__exit__", + "__float__", + "__getattr__", + "__getattribute__", + "__getitem__", + "__hex__", + "__init__", + "__init_subclass__", + "__int__", + "__invert__", + "__iter__", + "__len__", + "__long__", + "__neg__", + "__new__", + "__oct__", + "__pos__", + "__repr__", + "__reversed__", + "__setattr__", + "__setitem__", + "__str__", +} + +MAGIC_METHODS_ALLOWING_KWARGS: Final = { + "__init__", + "__init_subclass__", + "__new__", + "__call__", + "__setattr__", +} + +BINARY_MAGIC_METHODS: Final = { + "__add__", + "__and__", + "__divmod__", + "__eq__", + "__floordiv__", + "__ge__", + "__gt__", + "__iadd__", + "__iand__", + "__idiv__", + "__ifloordiv__", + "__ilshift__", + "__imatmul__", + "__imod__", + "__imul__", + "__ior__", + "__ipow__", + "__irshift__", + "__isub__", + "__itruediv__", + "__ixor__", + "__le__", + "__lshift__", + "__lt__", + "__matmul__", + "__mod__", + "__mul__", + "__ne__", + "__or__", + "__pow__", + "__radd__", + "__rand__", + "__rdiv__", + "__rfloordiv__", + "__rlshift__", + "__rmatmul__", + "__rmod__", + "__rmul__", + "__ror__", + "__rpow__", + "__rrshift__", + "__rshift__", + "__rsub__", + "__rtruediv__", + "__rxor__", + "__sub__", + "__truediv__", + "__xor__", +} + +assert not (_NON_BINARY_MAGIC_METHODS & BINARY_MAGIC_METHODS) + +MAGIC_METHODS: Final = _NON_BINARY_MAGIC_METHODS | BINARY_MAGIC_METHODS + +MAGIC_METHODS_POS_ARGS_ONLY: Final = MAGIC_METHODS - MAGIC_METHODS_ALLOWING_KWARGS + + +def special_function_elide_names(name: str) -> bool: + return name in MAGIC_METHODS_POS_ARGS_ONLY + + +def argument_elide_name(name: str | None) -> bool: + return name is not None and name.startswith("__") and not name.endswith("__") diff --git a/mypy/solve.py b/mypy/solve.py index af635bcaf02b..8a1495a9a246 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -1,68 +1,581 @@ """Type inference constraint solving""" -from typing import List, Dict +from __future__ import annotations -from mypy.types import Type, Void, NoneTyp, AnyType, ErrorType -from mypy.constraints import Constraint, SUPERTYPE_OF +from collections import defaultdict +from typing import Iterable, Sequence +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.constraints import SUBTYPE_OF, SUPERTYPE_OF, Constraint, infer_constraints, neg_op +from mypy.expandtype import expand_type +from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort from mypy.join import join_types -from mypy.meet import meet_types +from mypy.meet import meet_type_list, meet_types from mypy.subtypes import is_subtype +from mypy.typeops import get_all_type_vars +from mypy.types import ( + AnyType, + Instance, + NoneType, + ParamSpecType, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, +) +from mypy.typestate import type_state + +Bounds: _TypeAlias = "dict[TypeVarId, set[Type]]" +Graph: _TypeAlias = "set[tuple[TypeVarId, TypeVarId]]" +Solutions: _TypeAlias = "dict[TypeVarId, Type | None]" -def solve_constraints(vars: List[int], constraints: List[Constraint]) -> List[Type]: +def solve_constraints( + original_vars: Sequence[TypeVarLikeType], + constraints: list[Constraint], + strict: bool = True, + allow_polymorphic: bool = False, + skip_unsatisfied: bool = False, +) -> tuple[list[Type | None], list[TypeVarLikeType]]: """Solve type constraints. - Return the best type(s) for type variables; each type can be None if the value of the variable - could not be solved. If a variable has no constraints, arbitrarily pick NoneTyp as the value of - the type variable. + Return the best type(s) for type variables; each type can be None if the value of + the variable could not be solved. + + If a variable has no constraints, if strict=True then arbitrarily + pick UninhabitedType as the value of the type variable. If strict=False, pick AnyType. + If allow_polymorphic=True, then use the full algorithm that can potentially return + free type variables in solutions (these require special care when applying). Otherwise, + use a simplified algorithm that just solves each type variable individually if possible. + + The skip_unsatisfied flag matches the same one in applytype.apply_generic_arguments(). """ + vars = [tv.id for tv in original_vars] + if not vars: + return [], [] + + originals = {tv.id: tv for tv in original_vars} + extra_vars: list[TypeVarId] = [] + # Get additional type variables from generic actuals. + for c in constraints: + extra_vars.extend([v.id for v in c.extra_tvars if v.id not in vars + extra_vars]) + originals.update({v.id: v for v in c.extra_tvars if v.id not in originals}) + + if allow_polymorphic: + # Constraints inferred from unions require special handling in polymorphic inference. + constraints = skip_reverse_union_constraints(constraints) + # Collect a list of constraints for each type variable. - cmap = Dict[int, List[Constraint]]() + cmap: dict[TypeVarId, list[Constraint]] = {tv: [] for tv in vars + extra_vars} for con in constraints: - a = cmap.get(con.type_var, []) - a.append(con) - cmap[con.type_var] = a - - res = [] # type: List[Type] - - # Solve each type variable separately. - for tvar in vars: - bottom = None # type: Type - top = None # type: Type - - # Process each contraint separely, and calculate the lower and upper - # bounds based on constraints. Note that we assume that the constraint - # targets do not have constraint references. - for c in cmap.get(tvar, []): - if c.op == SUPERTYPE_OF: - if bottom is None: - bottom = c.target - else: - bottom = join_types(bottom, c.target) + if con.type_var in vars + extra_vars: + cmap[con.type_var].append(con) + + if allow_polymorphic: + if constraints: + solutions, free_vars = solve_with_dependent( + vars + extra_vars, constraints, vars, originals + ) + else: + solutions = {} + free_vars = [] + else: + solutions = {} + free_vars = [] + for tv, cs in cmap.items(): + if not cs: + continue + lowers = [c.target for c in cs if c.op == SUPERTYPE_OF] + uppers = [c.target for c in cs if c.op == SUBTYPE_OF] + solution = solve_one(lowers, uppers) + + # Do not leak type variables in non-polymorphic solutions. + if solution is None or not get_vars( + solution, [tv for tv in extra_vars if tv not in vars] + ): + solutions[tv] = solution + + res: list[Type | None] = [] + for v in vars: + if v in solutions: + res.append(solutions[v]) + else: + # No constraints for type variable -- 'UninhabitedType' is the most specific type. + candidate: Type + if strict: + candidate = UninhabitedType() + candidate.ambiguous = True else: - if top is None: - top = c.target - else: - top = meet_types(top, c.target) + candidate = AnyType(TypeOfAny.special_form) + res.append(candidate) + + if not free_vars and not skip_unsatisfied: + # Most of the validation for solutions is done in applytype.py, but here we can + # quickly test solutions w.r.t. to upper bounds, and use the latter (if possible), + # if solutions are actually not valid (due to poor inference context). + res = pre_validate_solutions(res, original_vars, constraints) + + return res, free_vars + + +def solve_with_dependent( + vars: list[TypeVarId], + constraints: list[Constraint], + original_vars: list[TypeVarId], + originals: dict[TypeVarId, TypeVarLikeType], +) -> tuple[Solutions, list[TypeVarLikeType]]: + """Solve set of constraints that may depend on each other, like T <: List[S]. + + The whole algorithm consists of five steps: + * Propagate via linear constraints and use secondary constraints to get transitive closure + * Find dependencies between type variables, group them in SCCs, and sort topologically + * Check that all SCC are intrinsically linear, we can't solve (express) T <: List[T] + * Variables in leaf SCCs that don't have constant bounds are free (choose one per SCC) + * Solve constraints iteratively starting from leafs, updating bounds after each step. + """ + graph, lowers, uppers = transitive_closure(vars, constraints) + + dmap = compute_dependencies(vars, graph, lowers, uppers) + sccs = list(strongly_connected_components(set(vars), dmap)) + if not all(check_linear(scc, lowers, uppers) for scc in sccs): + return {}, [] + raw_batches = list(topsort(prepare_sccs(sccs, dmap))) + + free_vars = [] + free_solutions = {} + for scc in raw_batches[0]: + # If there are no bounds on this SCC, then the only meaningful solution we can + # express, is that each variable is equal to a new free variable. For example, + # if we have T <: S, S <: U, we deduce: T = S = U = . + if all(not lowers[tv] and not uppers[tv] for tv in scc): + best_free = choose_free([originals[tv] for tv in scc], original_vars) + if best_free: + # TODO: failing to choose may cause leaking type variables, + # we need to fail gracefully instead. + free_vars.append(best_free.id) + free_solutions[best_free.id] = best_free + + # Update lowers/uppers with free vars, so these can now be used + # as valid solutions. + for l, u in graph: + if l in free_vars: + lowers[u].add(free_solutions[l]) + if u in free_vars: + uppers[l].add(free_solutions[u]) - if isinstance(top, AnyType) or isinstance(bottom, AnyType): - res.append(AnyType()) + # Flatten the SCCs that are independent, we can solve them together, + # since we don't need to update any targets in between. + batches = [] + for batch in raw_batches: + next_bc = [] + for scc in batch: + next_bc.extend(list(scc)) + batches.append(next_bc) + + solutions: dict[TypeVarId, Type | None] = {} + for flat_batch in batches: + res = solve_iteratively(flat_batch, graph, lowers, uppers) + solutions.update(res) + return solutions, [free_solutions[tv] for tv in free_vars] + + +def solve_iteratively( + batch: list[TypeVarId], graph: Graph, lowers: Bounds, uppers: Bounds +) -> Solutions: + """Solve transitive closure sequentially, updating upper/lower bounds after each step. + + Transitive closure is represented as a linear graph plus lower/upper bounds for each + type variable, see transitive_closure() docstring for details. + + We solve for type variables that appear in `batch`. If a bound is not constant (i.e. it + looks like T :> F[S, ...]), we substitute solutions found so far in the target F[S, ...] + after solving the batch. + + Importantly, after solving each variable in a batch, we move it from linear graph to + upper/lower bounds, this way we can guarantee consistency of solutions (see comment below + for an example when this is important). + """ + solutions = {} + s_batch = set(batch) + while s_batch: + for tv in sorted(s_batch, key=lambda x: x.raw_id): + if lowers[tv] or uppers[tv]: + solvable_tv = tv + break + else: + break + # Solve each solvable type variable separately. + s_batch.remove(solvable_tv) + result = solve_one(lowers[solvable_tv], uppers[solvable_tv]) + solutions[solvable_tv] = result + if result is None: + # TODO: support backtracking lower/upper bound choices and order within SCCs. + # (will require switching this function from iterative to recursive). continue - elif bottom is None: - if top: - candidate = top + + # Update the (transitive) bounds from graph if there is a solution. + # This is needed to guarantee solutions will never contradict the initial + # constraints. For example, consider {T <: S, T <: A, S :> B} with A :> B. + # If we would not update the uppers/lowers from graph, we would infer T = A, S = B + # which is not correct. + for l, u in graph.copy(): + if l == u: + continue + if l == solvable_tv: + lowers[u].add(result) + graph.remove((l, u)) + if u == solvable_tv: + uppers[l].add(result) + graph.remove((l, u)) + + # We can update uppers/lowers only once after solving the whole SCC, + # since uppers/lowers can't depend on type variables in the SCC + # (and we would reject such SCC as non-linear and therefore not solvable). + subs = {tv: s for (tv, s) in solutions.items() if s is not None} + for tv in lowers: + lowers[tv] = {expand_type(lt, subs) for lt in lowers[tv]} + for tv in uppers: + uppers[tv] = {expand_type(ut, subs) for ut in uppers[tv]} + return solutions + + +def solve_one(lowers: Iterable[Type], uppers: Iterable[Type]) -> Type | None: + """Solve constraints by finding by using meets of upper bounds, and joins of lower bounds.""" + bottom: Type | None = None + top: Type | None = None + candidate: Type | None = None + + # Filter out previous results of failed inference, they will only spoil the current pass... + new_uppers = [] + for u in uppers: + pu = get_proper_type(u) + if not isinstance(pu, UninhabitedType) or not pu.ambiguous: + new_uppers.append(u) + uppers = new_uppers + + # ...unless this is the only information we have, then we just pass it on. + if not uppers and not lowers: + candidate = UninhabitedType() + candidate.ambiguous = True + return candidate + + # Process each bound separately, and calculate the lower and upper + # bounds based on constraints. Note that we assume that the constraint + # targets do not have constraint references. + for target in lowers: + if bottom is None: + bottom = target + else: + if type_state.infer_unions: + # This deviates from the general mypy semantics because + # recursive types are union-heavy in 95% of cases. + bottom = UnionType.make_union([bottom, target]) else: - # No constraints for type variable -- type 'None' is the most specific type. - candidate = NoneTyp() - elif top is None: - candidate = bottom - elif is_subtype(bottom, top): - candidate = bottom + bottom = join_types(bottom, target) + + for target in uppers: + if top is None: + top = target else: - candidate = None - if isinstance(candidate, ErrorType): - res.append(None) + top = meet_types(top, target) + + p_top = get_proper_type(top) + p_bottom = get_proper_type(bottom) + if isinstance(p_top, AnyType) or isinstance(p_bottom, AnyType): + source_any = top if isinstance(p_top, AnyType) else bottom + assert isinstance(source_any, ProperType) and isinstance(source_any, AnyType) + return AnyType(TypeOfAny.from_another_any, source_any=source_any) + elif bottom is None: + if top: + candidate = top else: - res.append(candidate) + # No constraints for type variable + return None + elif top is None: + candidate = bottom + elif is_subtype(bottom, top): + candidate = bottom + else: + candidate = None + return candidate + + +def choose_free( + scc: list[TypeVarLikeType], original_vars: list[TypeVarId] +) -> TypeVarLikeType | None: + """Choose the best solution for an SCC containing only type variables. + + This is needed to preserve e.g. the upper bound in a situation like this: + def dec(f: Callable[[T], S]) -> Callable[[T], S]: ... + @dec + def test(x: U) -> U: ... + + where U <: A. + """ + + if len(scc) == 1: + # Fast path, choice is trivial. + return scc[0] + + common_upper_bound = meet_type_list([t.upper_bound for t in scc]) + common_upper_bound_p = get_proper_type(common_upper_bound) + # We include None for when strict-optional is disabled. + if isinstance(common_upper_bound_p, (UninhabitedType, NoneType)): + # This will cause to infer Never, which is better than a free TypeVar + # that has an upper bound Never. + return None + + values: list[Type] = [] + for tv in scc: + if isinstance(tv, TypeVarType) and tv.values: + if values: + # It is too tricky to support multiple TypeVars with values + # within the same SCC. + return None + values = tv.values.copy() + + if values and not is_trivial_bound(common_upper_bound_p): + # If there are both values and upper bound present, we give up, + # since type variables having both are not supported. + return None + + # For convenience with current type application machinery, we use a stable + # choice that prefers the original type variables (not polymorphic ones) in SCC. + best = sorted(scc, key=lambda x: (x.id not in original_vars, x.id.raw_id))[0] + if isinstance(best, TypeVarType): + return best.copy_modified(values=values, upper_bound=common_upper_bound) + if is_trivial_bound(common_upper_bound_p, allow_tuple=True): + # TODO: support more cases for ParamSpecs/TypeVarTuples + return best + return None + + +def is_trivial_bound(tp: ProperType, allow_tuple: bool = False) -> bool: + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + return allow_tuple and is_trivial_bound(get_proper_type(tp.args[0])) + return isinstance(tp, Instance) and tp.type.fullname == "builtins.object" + + +def find_linear(c: Constraint) -> tuple[bool, TypeVarId | None]: + """Find out if this constraint represent a linear relationship, return target id if yes.""" + if isinstance(c.origin_type_var, TypeVarType): + if isinstance(c.target, TypeVarType): + return True, c.target.id + if isinstance(c.origin_type_var, ParamSpecType): + if isinstance(c.target, ParamSpecType) and not c.target.prefix.arg_types: + return True, c.target.id + if isinstance(c.origin_type_var, TypeVarTupleType): + target = get_proper_type(c.target) + if isinstance(target, TupleType) and len(target.items) == 1: + item = target.items[0] + if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): + return True, item.type.id + return False, None + + +def transitive_closure( + tvars: list[TypeVarId], constraints: list[Constraint] +) -> tuple[Graph, Bounds, Bounds]: + """Find transitive closure for given constraints on type variables. + + Transitive closure gives maximal set of lower/upper bounds for each type variable, + such that we cannot deduce any further bounds by chaining other existing bounds. + + The transitive closure is represented by: + * A set of lower and upper bounds for each type variable, where only constant and + non-linear terms are included in the bounds. + * A graph of linear constraints between type variables (represented as a set of pairs) + Such separation simplifies reasoning, and allows an efficient and simple incremental + transitive closure algorithm that we use here. + + For example if we have initial constraints [T <: S, S <: U, U <: int], the transitive + closure is given by: + * {} <: T <: {int} + * {} <: S <: {int} + * {} <: U <: {int} + * {T <: S, S <: U, T <: U} + """ + uppers: Bounds = defaultdict(set) + lowers: Bounds = defaultdict(set) + graph: Graph = {(tv, tv) for tv in tvars} + + remaining = set(constraints) + while remaining: + c = remaining.pop() + # Note that ParamSpec constraint P <: Q may be considered linear only if Q has no prefix, + # for cases like P <: Concatenate[T, Q] we should consider this non-linear and put {P} and + # {T, Q} into separate SCCs. Similarly, Ts <: Tuple[*Us] considered linear, while + # Ts <: Tuple[*Us, U] is non-linear. + is_linear, target_id = find_linear(c) + if is_linear and target_id in tvars: + assert target_id is not None + if c.op == SUBTYPE_OF: + lower, upper = c.type_var, target_id + else: + lower, upper = target_id, c.type_var + if (lower, upper) in graph: + continue + graph |= { + (l, u) for l in tvars for u in tvars if (l, lower) in graph and (upper, u) in graph + } + for u in tvars: + if (upper, u) in graph: + lowers[u] |= lowers[lower] + for l in tvars: + if (l, lower) in graph: + uppers[l] |= uppers[upper] + for lt in lowers[lower]: + for ut in uppers[upper]: + add_secondary_constraints(remaining, lt, ut) + elif c.op == SUBTYPE_OF: + if c.target in uppers[c.type_var]: + continue + for l in tvars: + if (l, c.type_var) in graph: + uppers[l].add(c.target) + for lt in lowers[c.type_var]: + add_secondary_constraints(remaining, lt, c.target) + else: + assert c.op == SUPERTYPE_OF + if c.target in lowers[c.type_var]: + continue + for u in tvars: + if (c.type_var, u) in graph: + lowers[u].add(c.target) + for ut in uppers[c.type_var]: + add_secondary_constraints(remaining, c.target, ut) + return graph, lowers, uppers + + +def add_secondary_constraints(cs: set[Constraint], lower: Type, upper: Type) -> None: + """Add secondary constraints inferred between lower and upper (in place).""" + if isinstance(get_proper_type(upper), UnionType) and isinstance( + get_proper_type(lower), UnionType + ): + # When both types are unions, this can lead to inferring spurious constraints, + # for example Union[T, int] <: S <: Union[T, int] may infer T <: int. + # To avoid this, just skip them for now. + return + # TODO: what if secondary constraints result in inference against polymorphic actual? + cs.update(set(infer_constraints(lower, upper, SUBTYPE_OF))) + cs.update(set(infer_constraints(upper, lower, SUPERTYPE_OF))) + + +def compute_dependencies( + tvars: list[TypeVarId], graph: Graph, lowers: Bounds, uppers: Bounds +) -> dict[TypeVarId, list[TypeVarId]]: + """Compute dependencies between type variables induced by constraints. + + If we have a constraint like T <: List[S], we say that T depends on S, since + we will need to solve for S first before we can solve for T. + """ + res = {} + for tv in tvars: + deps = set() + for lt in lowers[tv]: + deps |= get_vars(lt, tvars) + for ut in uppers[tv]: + deps |= get_vars(ut, tvars) + for other in tvars: + if other == tv: + continue + if (tv, other) in graph or (other, tv) in graph: + deps.add(other) + res[tv] = list(deps) return res + + +def check_linear(scc: set[TypeVarId], lowers: Bounds, uppers: Bounds) -> bool: + """Check there are only linear constraints between type variables in SCC. + + Linear are constraints like T <: S (while T <: F[S] are non-linear). + """ + for tv in scc: + if any(get_vars(lt, list(scc)) for lt in lowers[tv]): + return False + if any(get_vars(ut, list(scc)) for ut in uppers[tv]): + return False + return True + + +def skip_reverse_union_constraints(cs: list[Constraint]) -> list[Constraint]: + """Avoid ambiguities for constraints inferred from unions during polymorphic inference. + + Polymorphic inference implicitly relies on assumption that a reverse of a linear constraint + is a linear constraint. This is however not true in presence of union types, for example + T :> Union[S, int] vs S <: T. Trying to solve such constraints would be detected ambiguous + as (T, S) form a non-linear SCC. However, simply removing the linear part results in a valid + solution T = Union[S, int], S = . A similar scenario is when we get T <: Union[T, int], + such constraints carry no information, and will equally confuse linearity check. + + TODO: a cleaner solution may be to avoid inferring such constraints in first place, but + this would require passing around a flag through all infer_constraints() calls. + """ + reverse_union_cs = set() + for c in cs: + p_target = get_proper_type(c.target) + if isinstance(p_target, UnionType): + for item in p_target.items: + if isinstance(item, TypeVarType): + if item == c.origin_type_var and c.op == SUBTYPE_OF: + reverse_union_cs.add(c) + continue + # These two forms are semantically identical, but are different from + # the point of view of Constraint.__eq__(). + reverse_union_cs.add(Constraint(item, neg_op(c.op), c.origin_type_var)) + reverse_union_cs.add(Constraint(c.origin_type_var, c.op, item)) + return [c for c in cs if c not in reverse_union_cs] + + +def get_vars(target: Type, vars: list[TypeVarId]) -> set[TypeVarId]: + """Find type variables for which we are solving in a target type.""" + return {tv.id for tv in get_all_type_vars(target)} & set(vars) + + +def pre_validate_solutions( + solutions: list[Type | None], + original_vars: Sequence[TypeVarLikeType], + constraints: list[Constraint], +) -> list[Type | None]: + """Check is each solution satisfies the upper bound of the corresponding type variable. + + If it doesn't satisfy the bound, check if bound itself satisfies all constraints, and + if yes, use it instead as a fallback solution. + """ + new_solutions: list[Type | None] = [] + for t, s in zip(original_vars, solutions): + if is_callable_protocol(t.upper_bound): + # This is really ad-hoc, but a proper fix would be much more complex, + # and otherwise this may cause crash in a relatively common scenario. + new_solutions.append(s) + continue + if s is not None and not is_subtype(s, t.upper_bound): + bound_satisfies_all = True + for c in constraints: + if c.op == SUBTYPE_OF and not is_subtype(t.upper_bound, c.target): + bound_satisfies_all = False + break + if c.op == SUPERTYPE_OF and not is_subtype(c.target, t.upper_bound): + bound_satisfies_all = False + break + if bound_satisfies_all: + new_solutions.append(t.upper_bound) + continue + new_solutions.append(s) + return new_solutions + + +def is_callable_protocol(t: Type) -> bool: + proper_t = get_proper_type(t) + if isinstance(proper_t, Instance) and proper_t.type.is_protocol: + return "__call__" in proper_t.type.protocol_members + return False diff --git a/mypy/split_namespace.py b/mypy/split_namespace.py new file mode 100644 index 000000000000..d1720cce82b0 --- /dev/null +++ b/mypy/split_namespace.py @@ -0,0 +1,35 @@ +"""Split namespace for argparse to allow separating options by prefix. + +We use this to direct some options to an Options object and some to a +regular namespace. +""" + +# In its own file largely because mypyc doesn't support its use of +# __getattr__/__setattr__ and has some issues with __dict__ + +from __future__ import annotations + +import argparse +from typing import Any + + +class SplitNamespace(argparse.Namespace): + def __init__(self, standard_namespace: object, alt_namespace: object, alt_prefix: str) -> None: + self.__dict__["_standard_namespace"] = standard_namespace + self.__dict__["_alt_namespace"] = alt_namespace + self.__dict__["_alt_prefix"] = alt_prefix + + def _get(self) -> tuple[Any, Any]: + return (self._standard_namespace, self._alt_namespace) + + def __setattr__(self, name: str, value: Any) -> None: + if name.startswith(self._alt_prefix): + setattr(self._alt_namespace, name[len(self._alt_prefix) :], value) + else: + setattr(self._standard_namespace, name, value) + + def __getattr__(self, name: str) -> Any: + if name.startswith(self._alt_prefix): + return getattr(self._alt_namespace, name[len(self._alt_prefix) :]) + else: + return getattr(self._standard_namespace, name) diff --git a/mypy/state.py b/mypy/state.py new file mode 100644 index 000000000000..533dceeb1f24 --- /dev/null +++ b/mypy/state.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import Final, Iterator + +# These are global mutable state. Don't add anything here unless there's a very +# good reason. + + +class StrictOptionalState: + # Wrap this in a class since it's faster that using a module-level attribute. + + def __init__(self, strict_optional: bool) -> None: + # Value varies by file being processed + self.strict_optional = strict_optional + + @contextmanager + def strict_optional_set(self, value: bool) -> Iterator[None]: + saved = self.strict_optional + self.strict_optional = value + try: + yield + finally: + self.strict_optional = saved + + +state: Final = StrictOptionalState(strict_optional=True) +find_occurrences: tuple[str, str] | None = None diff --git a/mypy/stats.py b/mypy/stats.py index 9fcbd2ae9a21..9c69a245741b 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -1,72 +1,193 @@ """Utilities for calculating and reporting statistics about types.""" -import cgi -import os.path -import re +from __future__ import annotations -from typing import Any, Dict, List, cast, Tuple +import os +from collections import Counter +from contextlib import contextmanager +from typing import Final, Iterator -from mypy.traverser import TraverserVisitor -from mypy.types import ( - Type, AnyType, Instance, FunctionLike, TupleType, Void, TypeVar, - TypeQuery, ANY_TYPE_STRATEGY, Callable -) from mypy import nodes +from mypy.argmap import map_formals_to_actuals from mypy.nodes import ( - Node, FuncDef, TypeApplication, AssignmentStmt, NameExpr, CallExpr, - MemberExpr, OpExpr, ComparisonExpr, IndexExpr, UnaryExpr, YieldFromExpr + AssignmentExpr, + AssignmentStmt, + BreakStmt, + BytesExpr, + CallExpr, + ClassDef, + ComparisonExpr, + ComplexExpr, + ContinueStmt, + EllipsisExpr, + Expression, + ExpressionStmt, + FloatExpr, + FuncDef, + Import, + ImportAll, + ImportFrom, + IndexExpr, + IntExpr, + MemberExpr, + MypyFile, + NameExpr, + Node, + OpExpr, + PassStmt, + RefExpr, + StrExpr, + TypeApplication, + UnaryExpr, + YieldFromExpr, ) +from mypy.traverser import TraverserVisitor +from mypy.typeanal import collect_all_inner_types +from mypy.types import ( + AnyType, + CallableType, + FunctionLike, + Instance, + TupleType, + Type, + TypeOfAny, + TypeQuery, + TypeVarType, + get_proper_type, + get_proper_types, +) +from mypy.util import correct_relative_import +TYPE_EMPTY: Final = 0 +TYPE_UNANALYZED: Final = 1 # type of non-typechecked code +TYPE_PRECISE: Final = 2 +TYPE_IMPRECISE: Final = 3 +TYPE_ANY: Final = 4 -TYPE_PRECISE = 0 -TYPE_IMPRECISE = 1 -TYPE_ANY = 2 +precision_names: Final = ["empty", "unanalyzed", "precise", "imprecise", "any"] class StatisticsVisitor(TraverserVisitor): - def __init__(self, inferred: bool, typemap: Dict[Node, Type] = None, - all_nodes: bool = False) -> None: + def __init__( + self, + inferred: bool, + filename: str, + modules: dict[str, MypyFile], + typemap: dict[Expression, Type] | None = None, + all_nodes: bool = False, + visit_untyped_defs: bool = True, + ) -> None: self.inferred = inferred + self.filename = filename + self.modules = modules self.typemap = typemap self.all_nodes = all_nodes + self.visit_untyped_defs = visit_untyped_defs - self.num_precise = 0 - self.num_imprecise = 0 - self.num_any = 0 + self.num_precise_exprs = 0 + self.num_imprecise_exprs = 0 + self.num_any_exprs = 0 - self.num_simple = 0 - self.num_generic = 0 - self.num_tuple = 0 - self.num_function = 0 - self.num_typevar = 0 - self.num_complex = 0 + self.num_simple_types = 0 + self.num_generic_types = 0 + self.num_tuple_types = 0 + self.num_function_types = 0 + self.num_typevar_types = 0 + self.num_complex_types = 0 + self.num_any_types = 0 self.line = -1 - self.line_map = Dict[int, int]() + self.line_map: dict[int, int] = {} + + self.type_of_any_counter: Counter[int] = Counter() + self.any_line_map: dict[int, list[AnyType]] = {} - self.output = List[str]() + # For each scope (top level/function), whether the scope was type checked + # (annotated function). + # + # TODO: Handle --check-untyped-defs + self.checked_scopes = [True] + + self.output: list[str] = [] TraverserVisitor.__init__(self) - def visit_func_def(self, o: FuncDef) -> None: - self.line = o.line - if len(o.expanded) > 1: - for defn in o.expanded: - self.visit_func_def(cast(FuncDef, defn)) + def visit_mypy_file(self, o: MypyFile) -> None: + self.cur_mod_node = o + self.cur_mod_id = o.fullname + super().visit_mypy_file(o) + + def visit_import_from(self, imp: ImportFrom) -> None: + self.process_import(imp) + + def visit_import_all(self, imp: ImportAll) -> None: + self.process_import(imp) + + def process_import(self, imp: ImportFrom | ImportAll) -> None: + import_id, ok = correct_relative_import( + self.cur_mod_id, imp.relative, imp.id, self.cur_mod_node.is_package_init_file() + ) + if ok and import_id in self.modules: + kind = TYPE_PRECISE + else: + kind = TYPE_ANY + self.record_line(imp.line, kind) + + def visit_import(self, imp: Import) -> None: + if all(id in self.modules for id, _ in imp.ids): + kind = TYPE_PRECISE else: - if o.type: - sig = cast(Callable, o.type) - arg_types = sig.arg_types - if (sig.arg_names and sig.arg_names[0] == 'self' and - not self.inferred): - arg_types = arg_types[1:] - for arg in arg_types: - self.type(arg) - self.type(sig.ret_type) - elif self.all_nodes: - self.record_line(self.line, TYPE_ANY) - super().visit_func_def(o) + kind = TYPE_ANY + self.record_line(imp.line, kind) + + def visit_func_def(self, o: FuncDef) -> None: + with self.enter_scope(o): + self.line = o.line + if len(o.expanded) > 1 and o.expanded != [o] * len(o.expanded): + if o in o.expanded: + print( + "{}:{}: ERROR: cycle in function expansion; skipping".format( + self.filename, o.line + ) + ) + return + for defn in o.expanded: + assert isinstance(defn, FuncDef) + self.visit_func_def(defn) + else: + if o.type: + assert isinstance(o.type, CallableType) + sig = o.type + arg_types = sig.arg_types + if sig.arg_names and sig.arg_names[0] == "self" and not self.inferred: + arg_types = arg_types[1:] + for arg in arg_types: + self.type(arg) + self.type(sig.ret_type) + elif self.all_nodes: + self.record_line(self.line, TYPE_ANY) + if not o.is_dynamic() or self.visit_untyped_defs: + super().visit_func_def(o) + + @contextmanager + def enter_scope(self, o: FuncDef) -> Iterator[None]: + self.checked_scopes.append(o.type is not None and self.checked_scopes[-1]) + yield None + self.checked_scopes.pop() + + def is_checked_scope(self) -> bool: + return self.checked_scopes[-1] + + def visit_class_def(self, o: ClassDef) -> None: + self.record_line(o.line, TYPE_PRECISE) # TODO: Look at base classes + # Override this method because we don't want to analyze base_type_exprs (base_type_exprs + # are base classes in a class declaration). + # While base_type_exprs are technically expressions, type analyzer does not visit them and + # they are not in the typemap. + for d in o.decorators: + d.accept(self) + o.defs.accept(self) def visit_type_application(self, o: TypeApplication) -> None: self.line = o.line @@ -76,38 +197,52 @@ def visit_type_application(self, o: TypeApplication) -> None: def visit_assignment_stmt(self, o: AssignmentStmt) -> None: self.line = o.line - if (isinstance(o.rvalue, nodes.CallExpr) and - isinstance(cast(nodes.CallExpr, o.rvalue).analyzed, - nodes.TypeVarExpr)): + if isinstance(o.rvalue, nodes.CallExpr) and isinstance( + o.rvalue.analyzed, nodes.TypeVarExpr + ): # Type variable definition -- not a real assignment. return if o.type: + # If there is an explicit type, don't visit the l.h.s. as an expression + # to avoid double-counting and mishandling special forms. self.type(o.type) - elif self.inferred: + o.rvalue.accept(self) + return + elif self.inferred and not self.all_nodes: + # if self.all_nodes is set, lvalues will be visited later for lvalue in o.lvalues: - lvalue_ref = lvalue - if isinstance(lvalue_ref, nodes.ParenExpr): - lvalue = lvalue_ref.expr if isinstance(lvalue, nodes.TupleExpr): items = lvalue.items - elif isinstance(lvalue, nodes.ListExpr): - items = lvalue.items else: items = [lvalue] for item in items: - if hasattr(item, 'is_def') and Any(item).is_def: - t = self.typemap.get(item) - if t: - self.type(t) - else: - self.log(' !! No inferred type on line %d' % - self.line) - self.record_line(self.line, TYPE_ANY) + if isinstance(item, RefExpr) and item.is_inferred_def: + if self.typemap is not None: + self.type(self.typemap.get(item)) super().visit_assignment_stmt(o) + def visit_expression_stmt(self, o: ExpressionStmt) -> None: + if isinstance(o.expr, (StrExpr, BytesExpr)): + # Docstring + self.record_line(o.line, TYPE_EMPTY) + else: + super().visit_expression_stmt(o) + + def visit_pass_stmt(self, o: PassStmt) -> None: + self.record_precise_if_checked_scope(o) + + def visit_break_stmt(self, o: BreakStmt) -> None: + self.record_precise_if_checked_scope(o) + + def visit_continue_stmt(self, o: ContinueStmt) -> None: + self.record_precise_if_checked_scope(o) + def visit_name_expr(self, o: NameExpr) -> None: - self.process_node(o) - super().visit_name_expr(o) + if o.fullname in ("builtins.None", "builtins.True", "builtins.False", "builtins.Ellipsis"): + self.record_precise_if_checked_scope(o) + else: + self.process_node(o) + super().visit_name_expr(o) def visit_yield_from_expr(self, o: YieldFromExpr) -> None: if o.expr: @@ -121,6 +256,43 @@ def visit_call_expr(self, o: CallExpr) -> None: o.callee.accept(self) for a in o.args: a.accept(self) + self.record_call_target_precision(o) + + def record_call_target_precision(self, o: CallExpr) -> None: + """Record precision of formal argument types used in a call.""" + if not self.typemap or o.callee not in self.typemap: + # Type not available. + return + callee_type = get_proper_type(self.typemap[o.callee]) + if isinstance(callee_type, CallableType): + self.record_callable_target_precision(o, callee_type) + else: + pass # TODO: Handle overloaded functions, etc. + + def record_callable_target_precision(self, o: CallExpr, callee: CallableType) -> None: + """Record imprecision caused by callee argument types. + + This only considers arguments passed in a call expression. Arguments + with default values that aren't provided in a call arguably don't + contribute to typing imprecision at the *call site* (but they + contribute at the function definition). + """ + assert self.typemap + typemap = self.typemap + actual_to_formal = map_formals_to_actuals( + o.arg_kinds, + o.arg_names, + callee.arg_kinds, + callee.arg_names, + lambda n: typemap[o.args[n]], + ) + for formals in actual_to_formal: + for n in formals: + formal = get_proper_type(callee.arg_types[n]) + if isinstance(formal, AnyType): + self.record_line(o.line, TYPE_ANY) + elif is_imprecise(formal): + self.record_line(o.line, TYPE_IMPRECISE) def visit_member_expr(self, o: MemberExpr) -> None: self.process_node(o) @@ -138,102 +310,154 @@ def visit_index_expr(self, o: IndexExpr) -> None: self.process_node(o) super().visit_index_expr(o) + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + self.process_node(o) + super().visit_assignment_expr(o) + def visit_unary_expr(self, o: UnaryExpr) -> None: self.process_node(o) super().visit_unary_expr(o) - def process_node(self, node: Node) -> None: + def visit_str_expr(self, o: StrExpr) -> None: + self.record_precise_if_checked_scope(o) + + def visit_bytes_expr(self, o: BytesExpr) -> None: + self.record_precise_if_checked_scope(o) + + def visit_int_expr(self, o: IntExpr) -> None: + self.record_precise_if_checked_scope(o) + + def visit_float_expr(self, o: FloatExpr) -> None: + self.record_precise_if_checked_scope(o) + + def visit_complex_expr(self, o: ComplexExpr) -> None: + self.record_precise_if_checked_scope(o) + + def visit_ellipsis(self, o: EllipsisExpr) -> None: + self.record_precise_if_checked_scope(o) + + # Helpers + + def process_node(self, node: Expression) -> None: if self.all_nodes: - typ = self.typemap.get(node) - if typ: + if self.typemap is not None: self.line = node.line - self.type(typ) + self.type(self.typemap.get(node)) + + def record_precise_if_checked_scope(self, node: Node) -> None: + if isinstance(node, Expression) and self.typemap and node not in self.typemap: + kind = TYPE_UNANALYZED + elif self.is_checked_scope(): + kind = TYPE_PRECISE + else: + kind = TYPE_ANY + self.record_line(node.line, kind) + + def type(self, t: Type | None) -> None: + t = get_proper_type(t) + + if not t: + # If an expression does not have a type, it is often due to dead code. + # Don't count these because there can be an unanalyzed value on a line with other + # analyzed expressions, which overwrite the TYPE_UNANALYZED. + self.record_line(self.line, TYPE_UNANALYZED) + return + + if isinstance(t, AnyType) and is_special_form_any(t): + # TODO: What if there is an error in special form definition? + self.record_line(self.line, TYPE_PRECISE) + return - def type(self, t: Type) -> None: if isinstance(t, AnyType): - self.log(' !! Any type around line %d' % self.line) - self.num_any += 1 + self.log(" !! Any type around line %d" % self.line) + self.num_any_exprs += 1 self.record_line(self.line, TYPE_ANY) - elif ((not self.all_nodes and is_imprecise(t)) or - (self.all_nodes and is_imprecise2(t))): - self.log(' !! Imprecise type around line %d' % self.line) - self.num_imprecise += 1 + elif (not self.all_nodes and is_imprecise(t)) or (self.all_nodes and is_imprecise2(t)): + self.log(" !! Imprecise type around line %d" % self.line) + self.num_imprecise_exprs += 1 self.record_line(self.line, TYPE_IMPRECISE) else: - self.num_precise += 1 + self.num_precise_exprs += 1 self.record_line(self.line, TYPE_PRECISE) - if isinstance(t, Instance): - if t.args: - if any(is_complex(arg) for arg in t.args): - self.num_complex += 1 + for typ in get_proper_types(collect_all_inner_types(t)) + [t]: + if isinstance(typ, AnyType): + typ = get_original_any(typ) + if is_special_form_any(typ): + continue + self.type_of_any_counter[typ.type_of_any] += 1 + self.num_any_types += 1 + if self.line in self.any_line_map: + self.any_line_map[self.line].append(typ) else: - self.num_generic += 1 - else: - self.num_simple += 1 - elif isinstance(t, Void): - self.num_simple += 1 - elif isinstance(t, FunctionLike): - self.num_function += 1 - elif isinstance(t, TupleType): - if any(is_complex(item) for item in t.items): - self.num_complex += 1 - else: - self.num_tuple += 1 - elif isinstance(t, TypeVar): - self.num_typevar += 1 + self.any_line_map[self.line] = [typ] + elif isinstance(typ, Instance): + if typ.args: + if any(is_complex(arg) for arg in typ.args): + self.num_complex_types += 1 + else: + self.num_generic_types += 1 + else: + self.num_simple_types += 1 + elif isinstance(typ, FunctionLike): + self.num_function_types += 1 + elif isinstance(typ, TupleType): + if any(is_complex(item) for item in typ.items): + self.num_complex_types += 1 + else: + self.num_tuple_types += 1 + elif isinstance(typ, TypeVarType): + self.num_typevar_types += 1 def log(self, string: str) -> None: self.output.append(string) def record_line(self, line: int, precision: int) -> None: - self.line_map[line] = max(precision, - self.line_map.get(line, TYPE_PRECISE)) + self.line_map[line] = max(precision, self.line_map.get(line, TYPE_EMPTY)) -def dump_type_stats(tree: Node, path: str, inferred: bool = False, - typemap: Dict[Node, Type] = None) -> None: +def dump_type_stats( + tree: MypyFile, + path: str, + modules: dict[str, MypyFile], + inferred: bool = False, + typemap: dict[Expression, Type] | None = None, +) -> None: if is_special_module(path): return print(path) - visitor = StatisticsVisitor(inferred, typemap) + visitor = StatisticsVisitor(inferred, filename=tree.fullname, modules=modules, typemap=typemap) tree.accept(visitor) for line in visitor.output: print(line) - print(' ** precision **') - print(' precise ', visitor.num_precise) - print(' imprecise', visitor.num_imprecise) - print(' any ', visitor.num_any) - print(' ** kinds **') - print(' simple ', visitor.num_simple) - print(' generic ', visitor.num_generic) - print(' function ', visitor.num_function) - print(' tuple ', visitor.num_tuple) - print(' typevar ', visitor.num_typevar) - print(' complex ', visitor.num_complex) - print(' any ', visitor.num_any) + print(" ** precision **") + print(" precise ", visitor.num_precise_exprs) + print(" imprecise", visitor.num_imprecise_exprs) + print(" any ", visitor.num_any_exprs) + print(" ** kinds **") + print(" simple ", visitor.num_simple_types) + print(" generic ", visitor.num_generic_types) + print(" function ", visitor.num_function_types) + print(" tuple ", visitor.num_tuple_types) + print(" TypeVar ", visitor.num_typevar_types) + print(" complex ", visitor.num_complex_types) + print(" any ", visitor.num_any_types) def is_special_module(path: str) -> bool: - return os.path.basename(path) in ('abc.py', 'typing.py', 'builtins.py') + return os.path.basename(path) in ("abc.pyi", "typing.pyi", "builtins.pyi") def is_imprecise(t: Type) -> bool: return t.accept(HasAnyQuery()) -class HasAnyQuery(TypeQuery): +class HasAnyQuery(TypeQuery[bool]): def __init__(self) -> None: - super().__init__(False, ANY_TYPE_STRATEGY) + super().__init__(any) def visit_any(self, t: AnyType) -> bool: - return True - - def visit_instance(self, t: Instance) -> bool: - if t.type.fullname() == 'builtins.tuple': - return True - else: - return super().visit_instance(t) + return not is_special_form_any(t) def is_imprecise2(t: Type) -> bool: @@ -241,114 +465,29 @@ def is_imprecise2(t: Type) -> bool: class HasAnyQuery2(HasAnyQuery): - def visit_callable(self, t: Callable) -> bool: + def visit_callable_type(self, t: CallableType) -> bool: # We don't want to flag references to functions with some Any # argument types (etc.) since they generally don't mean trouble. return False def is_generic(t: Type) -> bool: - return isinstance(t, Instance) and bool(cast(Instance, t).args) + t = get_proper_type(t) + return isinstance(t, Instance) and bool(t.args) def is_complex(t: Type) -> bool: - return is_generic(t) or isinstance(t, (FunctionLike, TupleType, - TypeVar)) + t = get_proper_type(t) + return is_generic(t) or isinstance(t, (FunctionLike, TupleType, TypeVarType)) -html_files = [] # type: List[Tuple[str, str, int, int]] +def is_special_form_any(t: AnyType) -> bool: + return get_original_any(t).type_of_any == TypeOfAny.special_form -def generate_html_report(tree: Node, path: str, type_map: Dict[Node, Type], - output_dir: str) -> None: - if is_special_module(path): - return - visitor = StatisticsVisitor(inferred=True, typemap=type_map, all_nodes=True) - tree.accept(visitor) - target_path = os.path.join(output_dir, 'html', path) - target_path = re.sub(r'\.py$', '.html', target_path) - ensure_dir_exists(os.path.dirname(target_path)) - output = [] # type: List[str] - append = output.append - append('''\ - - - - - -

''')
-    num_imprecise_lines = 0
-    num_lines = 0
-    with open(path) as input_file:
-        for i, line in enumerate(input_file):
-            lineno = i + 1
-            status = visitor.line_map.get(lineno, TYPE_PRECISE)
-            style_map = { TYPE_PRECISE: 'white',
-                          TYPE_IMPRECISE: 'yellow',
-                          TYPE_ANY: 'red' }
-            style = style_map[status]
-            append('%4d   ' % lineno +
-                   '%s' % (style,
-                                                   cgi.escape(line)))
-            if status != TYPE_PRECISE:
-                num_imprecise_lines += 1
-            if line.strip():
-                num_lines += 1
-    append('
') - append('') - with open(target_path, 'w') as output_file: - output_file.writelines(output) - target_path = target_path[len(output_dir) + 1:] - html_files.append((path, target_path, num_lines, num_imprecise_lines)) - - -def generate_html_index(output_dir: str) -> None: - path = os.path.join(output_dir, 'index.html') - output = [] # type: List[str] - append = output.append - append('''\ - - - - -''') - append('

Mypy Type Check Coverage Report

\n') - append('\n') - for source_path, target_path, num_lines, num_imprecise in sorted(html_files): - if num_lines == 0: - continue - source_path = os.path.normpath(source_path) - # TODO: Windows paths. - if (source_path.startswith('stubs/') or - '/stubs/' in source_path): - continue - percent = 100.0 * num_imprecise / num_lines - style = '' - if percent >= 20: - style = 'class="red"' - elif percent >= 5: - style = 'class="yellow"' - append('
%s%.1f%% imprecise%d LOC' % ( - style, target_path, source_path, percent, num_lines)) - append('
') - append('') - with open(path, 'w') as file: - file.writelines(output) - print('Generated HTML report: %s' % os.path.abspath(path)) - - -def ensure_dir_exists(dir: str) -> None: - if not os.path.exists(dir): - os.makedirs(dir) +def get_original_any(t: AnyType) -> AnyType: + if t.type_of_any == TypeOfAny.from_another_any: + assert t.source_any + assert t.source_any.type_of_any != TypeOfAny.from_another_any + t = t.source_any + return t diff --git a/mypy/strconv.py b/mypy/strconv.py index a609c46ef776..2d595d4b67b0 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -1,17 +1,23 @@ """Conversion of parse tree nodes to strings.""" -import re -import os +from __future__ import annotations -import typing +import os +import re +from typing import TYPE_CHECKING, Any, Sequence -from mypy.util import dump_tagged, short_type import mypy.nodes +from mypy.options import Options +from mypy.util import IdMapper, short_type from mypy.visitor import NodeVisitor +if TYPE_CHECKING: + import mypy.patterns + import mypy.types + class StrConv(NodeVisitor[str]): - """Visitor for converting a Node to a human-readable string. + """Visitor for converting a node to a human-readable string. For example, an MypyFile node from program '1' is converted into something like this: @@ -21,167 +27,203 @@ class StrConv(NodeVisitor[str]): ExpressionStmt:1( IntExpr(1))) """ - def dump(self, nodes, obj): + + __slots__ = ["options", "show_ids", "id_mapper"] + + def __init__(self, *, show_ids: bool = False, options: Options) -> None: + self.options = options + self.show_ids = show_ids + self.id_mapper: IdMapper | None = None + if show_ids: + self.id_mapper = IdMapper() + + def stringify_type(self, t: mypy.types.Type) -> str: + import mypy.types + + return t.accept(mypy.types.TypeStrVisitor(id_mapper=self.id_mapper, options=self.options)) + + def get_id(self, o: object) -> int | None: + if self.id_mapper: + return self.id_mapper.id(o) + return None + + def format_id(self, o: object) -> str: + if self.id_mapper: + return f"<{self.get_id(o)}>" + else: + return "" + + def dump(self, nodes: Sequence[object], obj: mypy.nodes.Context) -> str: """Convert a list of items to a multiline pretty-printed string. The tag is produced from the type name of obj and its line number. See mypy.util.dump_tagged for a description of the nodes argument. """ - return dump_tagged(nodes, short_type(obj) + ':' + str(obj.line)) + tag = short_type(obj) + ":" + str(obj.line) + if self.show_ids: + assert self.id_mapper is not None + tag += f"<{self.get_id(obj)}>" + return dump_tagged(nodes, tag, self) - def func_helper(self, o): + def func_helper(self, o: mypy.nodes.FuncItem) -> list[object]: """Return a list in a format suitable for dump() that represents the arguments and the body of a function. The caller can then decorate the array with information specific to methods, global functions or anonymous functions. """ - args = [] - init = [] - extra = [] - for i, kind in enumerate(o.arg_kinds): - if kind == mypy.nodes.ARG_POS: - args.append(o.args[i]) - elif kind in (mypy.nodes.ARG_OPT, mypy.nodes.ARG_NAMED): - args.append(o.args[i]) - init.append(o.init[i]) + args: list[mypy.nodes.Var | tuple[str, list[mypy.nodes.Node]]] = [] + extra: list[tuple[str, list[mypy.nodes.Var]]] = [] + for arg in o.arguments: + kind: mypy.nodes.ArgKind = arg.kind + if kind.is_required(): + args.append(arg.variable) + elif kind.is_optional(): + assert arg.initializer is not None + args.append(("default", [arg.variable, arg.initializer])) elif kind == mypy.nodes.ARG_STAR: - extra.append(('VarArg', [o.args[i]])) + extra.append(("VarArg", [arg.variable])) elif kind == mypy.nodes.ARG_STAR2: - extra.append(('DictVarArg', [o.args[i]])) - a = [] + extra.append(("DictVarArg", [arg.variable])) + a: list[Any] = [] + if o.type_args: + for p in o.type_args: + a.append(self.type_param(p)) if args: - a.append(('Args', args)) + a.append(("Args", args)) if o.type: a.append(o.type) - if init: - a.append(('Init', init)) if o.is_generator: - a.append('Generator') + a.append("Generator") a.extend(extra) a.append(o.body) return a # Top-level structures - def visit_mypy_file(self, o): + def visit_mypy_file(self, o: mypy.nodes.MypyFile) -> str: # Skip implicit definitions. - defs = o.defs - while (defs and isinstance(defs[0], mypy.nodes.VarDef) and - not defs[0].repr): - defs = defs[1:] - a = [defs] + a: list[Any] = [o.defs] if o.is_bom: - a.insert(0, 'BOM') + a.insert(0, "BOM") # Omit path to special file with name "main". This is used to simplify # test case descriptions; the file "main" is used by default in many # test cases. - if o.path is not None and o.path != 'main': + if o.path != "main": # Insert path. Normalize directory separators to / to unify test # case# output in all platforms. - a.insert(0, o.path.replace(os.sep, '/')) + a.insert(0, o.path.replace(os.getcwd() + os.sep, "").replace(os.sep, "/")) + if o.ignored_lines: + a.append("IgnoredLines(%s)" % ", ".join(str(line) for line in sorted(o.ignored_lines))) return self.dump(a, o) - def visit_import(self, o): + def visit_import(self, o: mypy.nodes.Import) -> str: a = [] for id, as_id in o.ids: - a.append('{} : {}'.format(id, as_id)) - return 'Import:{}({})'.format(o.line, ', '.join(a)) + if as_id is not None: + a.append(f"{id} : {as_id}") + else: + a.append(id) + return f"Import:{o.line}({', '.join(a)})" - def visit_import_from(self, o): + def visit_import_from(self, o: mypy.nodes.ImportFrom) -> str: a = [] for name, as_name in o.names: - a.append('{} : {}'.format(name, as_name)) - return 'ImportFrom:{}({}, [{}])'.format(o.line, "." * o.relative + o.id, ', '.join(a)) + if as_name is not None: + a.append(f"{name} : {as_name}") + else: + a.append(name) + return f"ImportFrom:{o.line}({'.' * o.relative + o.id}, [{', '.join(a)}])" - def visit_import_all(self, o): - return 'ImportAll:{}({})'.format(o.line, "." * o.relative + o.id) + def visit_import_all(self, o: mypy.nodes.ImportAll) -> str: + return f"ImportAll:{o.line}({'.' * o.relative + o.id})" # Definitions - def visit_func_def(self, o): + def visit_func_def(self, o: mypy.nodes.FuncDef) -> str: a = self.func_helper(o) - a.insert(0, o.name()) - if mypy.nodes.ARG_NAMED in o.arg_kinds: - a.insert(1, 'MaxPos({})'.format(o.max_pos)) - if o.is_abstract: - a.insert(-1, 'Abstract') + a.insert(0, o.name) + arg_kinds = {arg.kind for arg in o.arguments} + if len(arg_kinds & {mypy.nodes.ARG_NAMED, mypy.nodes.ARG_NAMED_OPT}) > 0: + a.insert(1, f"MaxPos({o.max_pos})") + if o.abstract_status in (mypy.nodes.IS_ABSTRACT, mypy.nodes.IMPLICITLY_ABSTRACT): + a.insert(-1, "Abstract") if o.is_static: - a.insert(-1, 'Static') + a.insert(-1, "Static") if o.is_class: - a.insert(-1, 'Class') + a.insert(-1, "Class") if o.is_property: - a.insert(-1, 'Property') + a.insert(-1, "Property") return self.dump(a, o) - def visit_overloaded_func_def(self, o): - a = o.items[:] + def visit_overloaded_func_def(self, o: mypy.nodes.OverloadedFuncDef) -> str: + a: Any = o.items.copy() if o.type: a.insert(0, o.type) + if o.impl: + a.insert(0, o.impl) + if o.is_static: + a.insert(-1, "Static") + if o.is_class: + a.insert(-1, "Class") return self.dump(a, o) - def visit_class_def(self, o): + def visit_class_def(self, o: mypy.nodes.ClassDef) -> str: a = [o.name, o.defs.body] # Display base types unless they are implicitly just builtins.object - # (in this case there is no representation). - if len(o.base_types) > 1 or (len(o.base_types) == 1 - and o.base_types[0].repr): - a.insert(1, ('BaseType', o.base_types)) + # (in this case base_type_exprs is empty). + if o.base_type_exprs: + if o.info and o.info.bases: + if len(o.info.bases) != 1 or o.info.bases[0].type.fullname != "builtins.object": + a.insert(1, ("BaseType", o.info.bases)) + else: + a.insert(1, ("BaseTypeExpr", o.base_type_exprs)) if o.type_vars: - a.insert(1, ('TypeVars', o.type_vars)) + a.insert(1, ("TypeVars", o.type_vars)) if o.metaclass: - a.insert(1, 'Metaclass({})'.format(o.metaclass)) + a.insert(1, f"Metaclass({o.metaclass.accept(self)})") if o.decorators: - a.insert(1, ('Decorators', o.decorators)) - if o.is_builtinclass: - a.insert(1, 'Builtinclass') - if o.info and o.info.ducktype: - a.insert(1, 'Ducktype({})'.format(o.info.ducktype)) - if o.info and o.info.disjoint_classes: - a.insert(1, ('Disjointclasses', [info.fullname() for - info in o.info.disjoint_classes])) - return self.dump(a, o) - - def visit_var_def(self, o): - a = [] - for n in o.items: - a.append('Var({})'.format(n.name())) - a.append('Type({})'.format(n.type)) - if o.init: - a.append(o.init) + a.insert(1, ("Decorators", o.decorators)) + if o.info and o.info._promote: + a.insert(1, f"Promote([{','.join(self.stringify_type(p) for p in o.info._promote)}])") + if o.info and o.info.tuple_type: + a.insert(1, ("TupleType", [o.info.tuple_type])) + if o.info and o.info.fallback_to_any: + a.insert(1, "FallbackToAny") + if o.type_args: + for p in reversed(o.type_args): + a.insert(1, self.type_param(p)) return self.dump(a, o) - def visit_var(self, o): - l = '' + def visit_var(self, o: mypy.nodes.Var) -> str: + lst = "" # Add :nil line number tag if no line number is specified to remain # compatible with old test case descriptions that assume this. if o.line < 0: - l = ':nil' - return 'Var' + l + '(' + o.name() + ')' + lst = ":nil" + return "Var" + lst + "(" + o.name + ")" - def visit_global_decl(self, o): + def visit_global_decl(self, o: mypy.nodes.GlobalDecl) -> str: return self.dump([o.names], o) - def visit_nonlocal_decl(self, o): + def visit_nonlocal_decl(self, o: mypy.nodes.NonlocalDecl) -> str: return self.dump([o.names], o) - def visit_decorator(self, o): + def visit_decorator(self, o: mypy.nodes.Decorator) -> str: return self.dump([o.var, o.decorators, o.func], o) - def visit_annotation(self, o): - return 'Type:{}({})'.format(o.line, o.type) - # Statements - def visit_block(self, o): + def visit_block(self, o: mypy.nodes.Block) -> str: return self.dump(o.body, o) - def visit_expression_stmt(self, o): + def visit_expression_stmt(self, o: mypy.nodes.ExpressionStmt) -> str: return self.dump([o.expr], o) - def visit_assignment_stmt(self, o): + def visit_assignment_stmt(self, o: mypy.nodes.AssignmentStmt) -> str: + a: list[Any] = [] if len(o.lvalues) > 1: - a = [('Lvalues', o.lvalues)] + a = [("Lvalues", o.lvalues)] else: a = [o.lvalues[0]] a.append(o.rvalue) @@ -189,62 +231,69 @@ def visit_assignment_stmt(self, o): a.append(o.type) return self.dump(a, o) - def visit_operator_assignment_stmt(self, o): + def visit_operator_assignment_stmt(self, o: mypy.nodes.OperatorAssignmentStmt) -> str: return self.dump([o.op, o.lvalue, o.rvalue], o) - def visit_while_stmt(self, o): - a = [o.expr, o.body] + def visit_while_stmt(self, o: mypy.nodes.WhileStmt) -> str: + a: list[Any] = [o.expr, o.body] if o.else_body: - a.append(('Else', o.else_body.body)) + a.append(("Else", o.else_body.body)) return self.dump(a, o) - def visit_for_stmt(self, o): - a = [o.index] + def visit_for_stmt(self, o: mypy.nodes.ForStmt) -> str: + a: list[Any] = [] + if o.is_async: + a.append(("Async", "")) + a.append(o.index) + if o.index_type: + a.append(o.index_type) a.extend([o.expr, o.body]) if o.else_body: - a.append(('Else', o.else_body.body)) + a.append(("Else", o.else_body.body)) return self.dump(a, o) - def visit_return_stmt(self, o): + def visit_return_stmt(self, o: mypy.nodes.ReturnStmt) -> str: return self.dump([o.expr], o) - def visit_if_stmt(self, o): - a = [] + def visit_if_stmt(self, o: mypy.nodes.IfStmt) -> str: + a: list[Any] = [] for i in range(len(o.expr)): - a.append(('If', [o.expr[i]])) - a.append(('Then', o.body[i].body)) + a.append(("If", [o.expr[i]])) + a.append(("Then", o.body[i].body)) if not o.else_body: return self.dump(a, o) else: - return self.dump([a, ('Else', o.else_body.body)], o) + return self.dump([a, ("Else", o.else_body.body)], o) - def visit_break_stmt(self, o): + def visit_break_stmt(self, o: mypy.nodes.BreakStmt) -> str: return self.dump([], o) - def visit_continue_stmt(self, o): + def visit_continue_stmt(self, o: mypy.nodes.ContinueStmt) -> str: return self.dump([], o) - def visit_pass_stmt(self, o): + def visit_pass_stmt(self, o: mypy.nodes.PassStmt) -> str: return self.dump([], o) - def visit_raise_stmt(self, o): + def visit_raise_stmt(self, o: mypy.nodes.RaiseStmt) -> str: return self.dump([o.expr, o.from_expr], o) - def visit_assert_stmt(self, o): - return self.dump([o.expr], o) - - def visit_yield_stmt(self, o): - return self.dump([o.expr], o) + def visit_assert_stmt(self, o: mypy.nodes.AssertStmt) -> str: + if o.msg is not None: + return self.dump([o.expr, o.msg], o) + else: + return self.dump([o.expr], o) - def visit_yield_from_stmt(self, o): + def visit_await_expr(self, o: mypy.nodes.AwaitExpr) -> str: return self.dump([o.expr], o) - def visit_del_stmt(self, o): + def visit_del_stmt(self, o: mypy.nodes.DelStmt) -> str: return self.dump([o.expr], o) - def visit_try_stmt(self, o): - a = [o.body] + def visit_try_stmt(self, o: mypy.nodes.TryStmt) -> str: + a: list[Any] = [o.body] + if o.is_star: + a.append("*") for i in range(len(o.vars)): a.append(o.types[i]) @@ -253,189 +302,370 @@ def visit_try_stmt(self, o): a.append(o.handlers[i]) if o.else_body: - a.append(('Else', o.else_body.body)) + a.append(("Else", o.else_body.body)) if o.finally_body: - a.append(('Finally', o.finally_body.body)) + a.append(("Finally", o.finally_body.body)) return self.dump(a, o) - def visit_with_stmt(self, o): - a = [] + def visit_with_stmt(self, o: mypy.nodes.WithStmt) -> str: + a: list[Any] = [] + if o.is_async: + a.append(("Async", "")) for i in range(len(o.expr)): - a.append(('Expr', [o.expr[i]])) - if o.name[i]: - a.append(('Name', [o.name[i]])) + a.append(("Expr", [o.expr[i]])) + if o.target[i]: + a.append(("Target", [o.target[i]])) + if o.unanalyzed_type: + a.append(o.unanalyzed_type) return self.dump(a + [o.body], o) - def visit_print_stmt(self, o): - a = o.args[:] - if o.newline: - a.append('Newline') + def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> str: + a: list[Any] = [o.subject] + for i in range(len(o.patterns)): + a.append(("Pattern", [o.patterns[i]])) + if o.guards[i] is not None: + a.append(("Guard", [o.guards[i]])) + a.append(("Body", o.bodies[i].body)) return self.dump(a, o) + def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> str: + a: list[Any] = [o.name] + for p in o.type_args: + a.append(self.type_param(p)) + a.append(o.value) + return self.dump(a, o) + + def type_param(self, p: mypy.nodes.TypeParam) -> list[Any]: + a: list[Any] = [] + if p.kind == mypy.nodes.PARAM_SPEC_KIND: + prefix = "**" + elif p.kind == mypy.nodes.TYPE_VAR_TUPLE_KIND: + prefix = "*" + else: + prefix = "" + a.append(prefix + p.name) + if p.upper_bound: + a.append(p.upper_bound) + if p.values: + a.append(("Values", p.values)) + if p.default: + a.append(("Default", [p.default])) + return [("TypeParam", a)] + # Expressions # Simple expressions - def visit_int_expr(self, o): - return 'IntExpr({})'.format(o.value) - - def visit_str_expr(self, o): - return 'StrExpr({})'.format(self.str_repr(o.value)) + def visit_int_expr(self, o: mypy.nodes.IntExpr) -> str: + return f"IntExpr({o.value})" - def visit_bytes_expr(self, o): - return 'BytesExpr({})'.format(self.str_repr(o.value)) + def visit_str_expr(self, o: mypy.nodes.StrExpr) -> str: + return f"StrExpr({self.str_repr(o.value)})" - def visit_unicode_expr(self, o): - return 'UnicodeExpr({})'.format(self.str_repr(o.value)) + def visit_bytes_expr(self, o: mypy.nodes.BytesExpr) -> str: + return f"BytesExpr({self.str_repr(o.value)})" - def str_repr(self, s): - s = re.sub(r'\\u[0-9a-fA-F]{4}', lambda m: '\\' + m.group(0), s) - return re.sub('[^\\x20-\\x7e]', - lambda m: r'\u%.4x' % ord(m.group(0)), s) + def str_repr(self, s: str) -> str: + s = re.sub(r"\\u[0-9a-fA-F]{4}", lambda m: "\\" + m.group(0), s) + return re.sub("[^\\x20-\\x7e]", lambda m: r"\u%.4x" % ord(m.group(0)), s) - def visit_float_expr(self, o): - return 'FloatExpr({})'.format(o.value) + def visit_float_expr(self, o: mypy.nodes.FloatExpr) -> str: + return f"FloatExpr({o.value})" - def visit_complex_expr(self, o): - return 'ComplexExpr({})'.format(o.value) + def visit_complex_expr(self, o: mypy.nodes.ComplexExpr) -> str: + return f"ComplexExpr({o.value})" - def visit_paren_expr(self, o): - return self.dump([o.expr], o) + def visit_ellipsis(self, o: mypy.nodes.EllipsisExpr) -> str: + return "Ellipsis" - def visit_star_expr(self, o): + def visit_star_expr(self, o: mypy.nodes.StarExpr) -> str: return self.dump([o.expr], o) - def visit_name_expr(self, o): - return (short_type(o) + '(' + self.pretty_name(o.name, o.kind, - o.fullname, o.is_def) - + ')') - - def pretty_name(self, name, kind, fullname, is_def): + def visit_name_expr(self, o: mypy.nodes.NameExpr) -> str: + pretty = self.pretty_name( + o.name, o.kind, o.fullname, o.is_inferred_def or o.is_special_form, o.node + ) + if isinstance(o.node, mypy.nodes.Var) and o.node.is_final: + pretty += f" = {o.node.final_value}" + return short_type(o) + "(" + pretty + ")" + + def pretty_name( + self, + name: str, + kind: int | None, + fullname: str | None, + is_inferred_def: bool, + target_node: mypy.nodes.Node | None = None, + ) -> str: n = name - if is_def: - n += '*' - if kind == mypy.nodes.GDEF or (fullname != name and - fullname is not None): + if is_inferred_def: + n += "*" + if target_node: + id = self.format_id(target_node) + else: + id = "" + if isinstance(target_node, mypy.nodes.MypyFile) and name == fullname: + n += id + elif kind == mypy.nodes.GDEF or (fullname != name and fullname): # Append fully qualified name for global references. - n += ' [{}]'.format(fullname) + n += f" [{fullname}{id}]" elif kind == mypy.nodes.LDEF: # Add tag to signify a local reference. - n += ' [l]' + n += f" [l{id}]" elif kind == mypy.nodes.MDEF: # Add tag to signify a member reference. - n += ' [m]' + n += f" [m{id}]" + else: + n += id return n - def visit_member_expr(self, o): - return self.dump([o.expr, self.pretty_name(o.name, o.kind, o.fullname, - o.is_def)], o) + def visit_member_expr(self, o: mypy.nodes.MemberExpr) -> str: + pretty = self.pretty_name(o.name, o.kind, o.fullname, o.is_inferred_def, o.node) + return self.dump([o.expr, pretty], o) + + def visit_yield_expr(self, o: mypy.nodes.YieldExpr) -> str: + return self.dump([o.expr], o) - def visit_yield_from_expr(self, o): + def visit_yield_from_expr(self, o: mypy.nodes.YieldFromExpr) -> str: if o.expr: return self.dump([o.expr.accept(self)], o) else: return self.dump([], o) - def visit_call_expr(self, o): + def visit_call_expr(self, o: mypy.nodes.CallExpr) -> str: if o.analyzed: return o.analyzed.accept(self) - args = [] - extra = [] + args: list[mypy.nodes.Expression] = [] + extra: list[str | tuple[str, list[Any]]] = [] for i, kind in enumerate(o.arg_kinds): if kind in [mypy.nodes.ARG_POS, mypy.nodes.ARG_STAR]: args.append(o.args[i]) if kind == mypy.nodes.ARG_STAR: - extra.append('VarArg') + extra.append("VarArg") elif kind == mypy.nodes.ARG_NAMED: - extra.append(('KwArgs', [o.arg_names[i], o.args[i]])) + extra.append(("KwArgs", [o.arg_names[i], o.args[i]])) elif kind == mypy.nodes.ARG_STAR2: - extra.append(('DictVarArg', [o.args[i]])) + extra.append(("DictVarArg", [o.args[i]])) else: - raise RuntimeError('unknown kind %d' % kind) + raise RuntimeError(f"unknown kind {kind}") + a: list[Any] = [o.callee, ("Args", args)] + return self.dump(a + extra, o) - return self.dump([o.callee, ('Args', args)] + extra, o) - - def visit_op_expr(self, o): + def visit_op_expr(self, o: mypy.nodes.OpExpr) -> str: + if o.analyzed: + return o.analyzed.accept(self) return self.dump([o.op, o.left, o.right], o) - def visit_comparison_expr(self, o): + def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> str: return self.dump([o.operators, o.operands], o) - def visit_cast_expr(self, o): + def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> str: + return self.dump([o.expr, o.type], o) + + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> str: return self.dump([o.expr, o.type], o) - def visit_unary_expr(self, o): + def visit_reveal_expr(self, o: mypy.nodes.RevealExpr) -> str: + if o.kind == mypy.nodes.REVEAL_TYPE: + return self.dump([o.expr], o) + else: + # REVEAL_LOCALS + return self.dump([o.local_nodes], o) + + def visit_assignment_expr(self, o: mypy.nodes.AssignmentExpr) -> str: + return self.dump([o.target, o.value], o) + + def visit_unary_expr(self, o: mypy.nodes.UnaryExpr) -> str: return self.dump([o.op, o.expr], o) - def visit_list_expr(self, o): + def visit_list_expr(self, o: mypy.nodes.ListExpr) -> str: return self.dump(o.items, o) - def visit_dict_expr(self, o): + def visit_dict_expr(self, o: mypy.nodes.DictExpr) -> str: return self.dump([[k, v] for k, v in o.items], o) - def visit_set_expr(self, o): + def visit_set_expr(self, o: mypy.nodes.SetExpr) -> str: return self.dump(o.items, o) - def visit_tuple_expr(self, o): + def visit_tuple_expr(self, o: mypy.nodes.TupleExpr) -> str: return self.dump(o.items, o) - def visit_index_expr(self, o): + def visit_index_expr(self, o: mypy.nodes.IndexExpr) -> str: if o.analyzed: return o.analyzed.accept(self) return self.dump([o.base, o.index], o) - def visit_super_expr(self, o): - return self.dump([o.name], o) + def visit_super_expr(self, o: mypy.nodes.SuperExpr) -> str: + return self.dump([o.name, o.call], o) - def visit_undefined_expr(self, o): - return 'UndefinedExpr:{}({})'.format(o.line, o.type) + def visit_type_application(self, o: mypy.nodes.TypeApplication) -> str: + return self.dump([o.expr, ("Types", o.types)], o) - def visit_type_application(self, o): - return self.dump([o.expr, ('Types', o.types)], o) + def visit_type_var_expr(self, o: mypy.nodes.TypeVarExpr) -> str: + import mypy.types - def visit_type_var_expr(self, o): + a: list[Any] = [] + if o.variance == mypy.nodes.COVARIANT: + a += ["Variance(COVARIANT)"] + if o.variance == mypy.nodes.CONTRAVARIANT: + a += ["Variance(CONTRAVARIANT)"] if o.values: - return self.dump([('Values', o.values)], o) - else: - return 'TypeVarExpr:{}()'.format(o.line) + a += [("Values", o.values)] + if not mypy.types.is_named_instance(o.upper_bound, "builtins.object"): + a += [f"UpperBound({self.stringify_type(o.upper_bound)})"] + return self.dump(a, o) + + def visit_paramspec_expr(self, o: mypy.nodes.ParamSpecExpr) -> str: + import mypy.types + + a: list[Any] = [] + if o.variance == mypy.nodes.COVARIANT: + a += ["Variance(COVARIANT)"] + if o.variance == mypy.nodes.CONTRAVARIANT: + a += ["Variance(CONTRAVARIANT)"] + if not mypy.types.is_named_instance(o.upper_bound, "builtins.object"): + a += [f"UpperBound({self.stringify_type(o.upper_bound)})"] + return self.dump(a, o) + + def visit_type_var_tuple_expr(self, o: mypy.nodes.TypeVarTupleExpr) -> str: + import mypy.types + + a: list[Any] = [] + if o.variance == mypy.nodes.COVARIANT: + a += ["Variance(COVARIANT)"] + if o.variance == mypy.nodes.CONTRAVARIANT: + a += ["Variance(CONTRAVARIANT)"] + if not mypy.types.is_named_instance(o.upper_bound, "builtins.object"): + a += [f"UpperBound({self.stringify_type(o.upper_bound)})"] + return self.dump(a, o) + + def visit_type_alias_expr(self, o: mypy.nodes.TypeAliasExpr) -> str: + return f"TypeAliasExpr({self.stringify_type(o.node.target)})" - def visit_namedtuple_expr(self, o): - return 'NamedTupleExpr:{}({}, {})'.format(o.line, - o.info.name(), - o.info.tuple_type) + def visit_namedtuple_expr(self, o: mypy.nodes.NamedTupleExpr) -> str: + return f"NamedTupleExpr:{o.line}({o.info.name}, {self.stringify_type(o.info.tuple_type) if o.info.tuple_type is not None else None})" - def visit_ducktype_expr(self, o): - return 'DucktypeExpr:{}({})'.format(o.line, o.type) + def visit_enum_call_expr(self, o: mypy.nodes.EnumCallExpr) -> str: + return f"EnumCallExpr:{o.line}({o.info.name}, {o.items})" - def visit_disjointclass_expr(self, o): - return 'DisjointclassExpr:{}({})'.format(o.line, o.cls.fullname) + def visit_typeddict_expr(self, o: mypy.nodes.TypedDictExpr) -> str: + return f"TypedDictExpr:{o.line}({o.info.name})" - def visit_func_expr(self, o): + def visit__promote_expr(self, o: mypy.nodes.PromoteExpr) -> str: + return f"PromoteExpr:{o.line}({self.stringify_type(o.type)})" + + def visit_newtype_expr(self, o: mypy.nodes.NewTypeExpr) -> str: + return f"NewTypeExpr:{o.line}({o.name}, {self.dump([o.old_type], o)})" + + def visit_lambda_expr(self, o: mypy.nodes.LambdaExpr) -> str: a = self.func_helper(o) return self.dump(a, o) - def visit_generator_expr(self, o): + def visit_generator_expr(self, o: mypy.nodes.GeneratorExpr) -> str: condlists = o.condlists if any(o.condlists) else None return self.dump([o.left_expr, o.indices, o.sequences, condlists], o) - def visit_list_comprehension(self, o): + def visit_list_comprehension(self, o: mypy.nodes.ListComprehension) -> str: return self.dump([o.generator], o) - def visit_set_comprehension(self, o): + def visit_set_comprehension(self, o: mypy.nodes.SetComprehension) -> str: return self.dump([o.generator], o) - def visit_dictionary_comprehension(self, o): + def visit_dictionary_comprehension(self, o: mypy.nodes.DictionaryComprehension) -> str: condlists = o.condlists if any(o.condlists) else None return self.dump([o.key, o.value, o.indices, o.sequences, condlists], o) - def visit_conditional_expr(self, o): - return self.dump([('Condition', [o.cond]), o.if_expr, o.else_expr], o) + def visit_conditional_expr(self, o: mypy.nodes.ConditionalExpr) -> str: + return self.dump([("Condition", [o.cond]), o.if_expr, o.else_expr], o) - def visit_slice_expr(self, o): - a = [o.begin_index, o.end_index, o.stride] + def visit_slice_expr(self, o: mypy.nodes.SliceExpr) -> str: + a: list[Any] = [o.begin_index, o.end_index, o.stride] if not a[0]: - a[0] = '' + a[0] = "" if not a[1]: - a[1] = '' + a[1] = "" return self.dump(a, o) + + def visit_temp_node(self, o: mypy.nodes.TempNode) -> str: + return self.dump([o.type], o) + + def visit_as_pattern(self, o: mypy.patterns.AsPattern) -> str: + return self.dump([o.pattern, o.name], o) + + def visit_or_pattern(self, o: mypy.patterns.OrPattern) -> str: + return self.dump(o.patterns, o) + + def visit_value_pattern(self, o: mypy.patterns.ValuePattern) -> str: + return self.dump([o.expr], o) + + def visit_singleton_pattern(self, o: mypy.patterns.SingletonPattern) -> str: + return self.dump([o.value], o) + + def visit_sequence_pattern(self, o: mypy.patterns.SequencePattern) -> str: + return self.dump(o.patterns, o) + + def visit_starred_pattern(self, o: mypy.patterns.StarredPattern) -> str: + return self.dump([o.capture], o) + + def visit_mapping_pattern(self, o: mypy.patterns.MappingPattern) -> str: + a: list[Any] = [] + for i in range(len(o.keys)): + a.append(("Key", [o.keys[i]])) + a.append(("Value", [o.values[i]])) + if o.rest is not None: + a.append(("Rest", [o.rest])) + return self.dump(a, o) + + def visit_class_pattern(self, o: mypy.patterns.ClassPattern) -> str: + a: list[Any] = [o.class_ref] + if len(o.positionals) > 0: + a.append(("Positionals", o.positionals)) + for i in range(len(o.keyword_keys)): + a.append(("Keyword", [o.keyword_keys[i], o.keyword_values[i]])) + + return self.dump(a, o) + + +def dump_tagged(nodes: Sequence[object], tag: str | None, str_conv: StrConv) -> str: + """Convert an array into a pretty-printed multiline string representation. + + The format is + tag( + item1.. + itemN) + Individual items are formatted like this: + - arrays are flattened + - pairs (str, array) are converted recursively, so that str is the tag + - other items are converted to strings and indented + """ + from mypy.types import Type, TypeStrVisitor + + a: list[str] = [] + if tag: + a.append(tag + "(") + for n in nodes: + if isinstance(n, list): + if n: + a.append(dump_tagged(n, None, str_conv)) + elif isinstance(n, tuple): + s = dump_tagged(n[1], n[0], str_conv) + a.append(indent(s, 2)) + elif isinstance(n, mypy.nodes.Node): + a.append(indent(n.accept(str_conv), 2)) + elif isinstance(n, Type): + a.append( + indent(n.accept(TypeStrVisitor(str_conv.id_mapper, options=str_conv.options)), 2) + ) + elif n is not None: + a.append(indent(str(n), 2)) + if tag: + a[-1] += ")" + return "\n".join(a) + + +def indent(s: str, n: int) -> str: + """Indent all the lines in s (separated by newlines) by n spaces.""" + s = " " * n + s + s = s.replace("\n", "\n" + " " * n) + return s diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py new file mode 100644 index 000000000000..434de0ea3bcb --- /dev/null +++ b/mypy/stubdoc.py @@ -0,0 +1,490 @@ +"""Parsing/inferring signatures from documentation. + +This module provides several functions to generate better stubs using +docstrings and Sphinx docs (.rst files). +""" + +from __future__ import annotations + +import contextlib +import io +import keyword +import re +import tokenize +from typing import Any, Final, MutableMapping, MutableSequence, NamedTuple, Sequence, Tuple +from typing_extensions import TypeAlias as _TypeAlias + +import mypy.util + +# Type alias for signatures strings in format ('func_name', '(arg, opt_arg=False)'). +Sig: _TypeAlias = Tuple[str, str] + + +_TYPE_RE: Final = re.compile(r"^[a-zA-Z_][\w\[\], .\"\']*(\.[a-zA-Z_][\w\[\], ]*)*$") +_ARG_NAME_RE: Final = re.compile(r"\**[A-Za-z_][A-Za-z0-9_]*$") + + +def is_valid_type(s: str) -> bool: + """Try to determine whether a string might be a valid type annotation.""" + if s in ("True", "False", "retval"): + return False + if "," in s and "[" not in s: + return False + return _TYPE_RE.match(s) is not None + + +class ArgSig: + """Signature info for a single argument.""" + + def __init__( + self, + name: str, + type: str | None = None, + *, + default: bool = False, + default_value: str = "...", + ) -> None: + self.name = name + self.type = type + # Does this argument have a default value? + self.default = default + self.default_value = default_value + + def is_star_arg(self) -> bool: + return self.name.startswith("*") and not self.name.startswith("**") + + def is_star_kwarg(self) -> bool: + return self.name.startswith("**") + + def __repr__(self) -> str: + return "ArgSig(name={}, type={}, default={})".format( + repr(self.name), repr(self.type), repr(self.default) + ) + + def __eq__(self, other: Any) -> bool: + if isinstance(other, ArgSig): + return ( + self.name == other.name + and self.type == other.type + and self.default == other.default + and self.default_value == other.default_value + ) + return False + + +class FunctionSig(NamedTuple): + name: str + args: list[ArgSig] + ret_type: str | None + type_args: str = "" # TODO implement in stubgenc and remove the default + + def is_special_method(self) -> bool: + return bool( + self.name.startswith("__") + and self.name.endswith("__") + and self.args + and self.args[0].name in ("self", "cls") + ) + + def has_catchall_args(self) -> bool: + """Return if this signature has catchall args: (*args, **kwargs)""" + if self.args and self.args[0].name in ("self", "cls"): + args = self.args[1:] + else: + args = self.args + return ( + len(args) == 2 + and all(a.type in (None, "object", "Any", "typing.Any") for a in args) + and args[0].is_star_arg() + and args[1].is_star_kwarg() + ) + + def is_catchall_signature(self) -> bool: + """Return if this signature is the catchall identity: (*args, **kwargs) -> Any""" + return self.has_catchall_args() and self.ret_type in (None, "Any", "typing.Any") + + def format_sig( + self, + indent: str = "", + is_async: bool = False, + any_val: str | None = None, + docstring: str | None = None, + ) -> str: + args: list[str] = [] + for arg in self.args: + arg_def = arg.name + + if arg_def in keyword.kwlist: + arg_def = "_" + arg_def + + if ( + arg.type is None + and any_val is not None + and arg.name not in ("self", "cls") + and not arg.name.startswith("*") + ): + arg_type: str | None = any_val + else: + arg_type = arg.type + if arg_type: + arg_def += ": " + arg_type + if arg.default: + arg_def += f" = {arg.default_value}" + + elif arg.default: + arg_def += f"={arg.default_value}" + + args.append(arg_def) + + retfield = "" + ret_type = self.ret_type if self.ret_type else any_val + if ret_type is not None: + retfield = " -> " + ret_type + + prefix = "async " if is_async else "" + sig = f"{indent}{prefix}def {self.name}{self.type_args}({', '.join(args)}){retfield}:" + if docstring: + suffix = f"\n{indent} {mypy.util.quote_docstring(docstring)}" + else: + suffix = " ..." + return f"{sig}{suffix}" + + +# States of the docstring parser. +STATE_INIT: Final = 1 +STATE_FUNCTION_NAME: Final = 2 +STATE_ARGUMENT_LIST: Final = 3 +STATE_ARGUMENT_TYPE: Final = 4 +STATE_ARGUMENT_DEFAULT: Final = 5 +STATE_RETURN_VALUE: Final = 6 +STATE_OPEN_BRACKET: Final = 7 # For generic types. + + +class DocStringParser: + """Parse function signatures in documentation.""" + + def __init__(self, function_name: str) -> None: + # Only search for signatures of function with this name. + self.function_name = function_name + self.state = [STATE_INIT] + self.accumulator = "" + self.arg_type: str | None = None + self.arg_name = "" + self.arg_default: str | None = None + self.ret_type = "Any" + self.found = False + self.args: list[ArgSig] = [] + # Valid signatures found so far. + self.signatures: list[FunctionSig] = [] + + def add_token(self, token: tokenize.TokenInfo) -> None: + """Process next token from the token stream.""" + if ( + token.type == tokenize.NAME + and token.string == self.function_name + and self.state[-1] == STATE_INIT + ): + self.state.append(STATE_FUNCTION_NAME) + + elif ( + token.type == tokenize.OP + and token.string == "(" + and self.state[-1] == STATE_FUNCTION_NAME + ): + self.state.pop() + self.accumulator = "" + self.found = True + self.state.append(STATE_ARGUMENT_LIST) + + elif self.state[-1] == STATE_FUNCTION_NAME: + # Reset state, function name not followed by '('. + self.state.pop() + + elif ( + token.type == tokenize.OP + and token.string in ("[", "(", "{") + and self.state[-1] != STATE_INIT + ): + self.accumulator += token.string + self.state.append(STATE_OPEN_BRACKET) + + elif ( + token.type == tokenize.OP + and token.string in ("]", ")", "}") + and self.state[-1] == STATE_OPEN_BRACKET + ): + self.accumulator += token.string + self.state.pop() + + elif ( + token.type == tokenize.OP + and token.string == ":" + and self.state[-1] == STATE_ARGUMENT_LIST + ): + self.arg_name = self.accumulator + self.accumulator = "" + self.state.append(STATE_ARGUMENT_TYPE) + + elif ( + token.type == tokenize.OP + and token.string == "=" + and self.state[-1] in (STATE_ARGUMENT_LIST, STATE_ARGUMENT_TYPE) + ): + if self.state[-1] == STATE_ARGUMENT_TYPE: + self.arg_type = self.accumulator + self.state.pop() + else: + self.arg_name = self.accumulator + self.accumulator = "" + self.state.append(STATE_ARGUMENT_DEFAULT) + + elif ( + token.type == tokenize.OP + and token.string in (",", ")") + and self.state[-1] + in (STATE_ARGUMENT_LIST, STATE_ARGUMENT_DEFAULT, STATE_ARGUMENT_TYPE) + ): + if self.state[-1] == STATE_ARGUMENT_DEFAULT: + self.arg_default = self.accumulator + self.state.pop() + elif self.state[-1] == STATE_ARGUMENT_TYPE: + self.arg_type = self.accumulator + self.state.pop() + elif self.state[-1] == STATE_ARGUMENT_LIST: + self.arg_name = self.accumulator + if not ( + token.string == ")" and self.accumulator.strip() == "" + ) and not _ARG_NAME_RE.match(self.arg_name): + # Invalid argument name. + self.reset() + return + + if token.string == ")": + self.state.pop() + + # arg_name is empty when there are no args. e.g. func() + if self.arg_name: + if self.arg_type and not is_valid_type(self.arg_type): + # wrong type, use Any + self.args.append( + ArgSig(name=self.arg_name, type=None, default=bool(self.arg_default)) + ) + else: + self.args.append( + ArgSig( + name=self.arg_name, type=self.arg_type, default=bool(self.arg_default) + ) + ) + self.arg_name = "" + self.arg_type = None + self.arg_default = None + self.accumulator = "" + + elif token.type == tokenize.OP and token.string == "->" and self.state[-1] == STATE_INIT: + self.accumulator = "" + self.state.append(STATE_RETURN_VALUE) + + # ENDMAKER is necessary for python 3.4 and 3.5. + elif token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and self.state[-1] in ( + STATE_INIT, + STATE_RETURN_VALUE, + ): + if self.state[-1] == STATE_RETURN_VALUE: + if not is_valid_type(self.accumulator): + self.reset() + return + self.ret_type = self.accumulator + self.accumulator = "" + self.state.pop() + + if self.found: + self.signatures.append( + FunctionSig(name=self.function_name, args=self.args, ret_type=self.ret_type) + ) + self.found = False + self.args = [] + self.ret_type = "Any" + # Leave state as INIT. + else: + self.accumulator += token.string + + def reset(self) -> None: + self.state = [STATE_INIT] + self.args = [] + self.found = False + self.accumulator = "" + + def get_signatures(self) -> list[FunctionSig]: + """Return sorted copy of the list of signatures found so far.""" + + def has_arg(name: str, signature: FunctionSig) -> bool: + return any(x.name == name for x in signature.args) + + def args_kwargs(signature: FunctionSig) -> bool: + return has_arg("*args", signature) and has_arg("**kwargs", signature) + + # Move functions with (*args, **kwargs) in their signature to last place. + return sorted(self.signatures, key=lambda x: 1 if args_kwargs(x) else 0) + + +def infer_sig_from_docstring(docstr: str | None, name: str) -> list[FunctionSig] | None: + """Convert function signature to list of FunctionSig + + Look for function signatures of function in docstring. Signature is a string of + the format () -> or perhaps without + the return type. + + Returns empty list, when no signature is found, one signature in typical case, + multiple signatures, if docstring specifies multiple signatures for overload functions. + Return None if the docstring is empty. + + Arguments: + * docstr: docstring + * name: name of function for which signatures are to be found + """ + if not (isinstance(docstr, str) and docstr): + return None + + state = DocStringParser(name) + # Return all found signatures, even if there is a parse error after some are found. + with contextlib.suppress(tokenize.TokenError): + try: + tokens = tokenize.tokenize(io.BytesIO(docstr.encode("utf-8")).readline) + for token in tokens: + state.add_token(token) + except IndentationError: + return None + sigs = state.get_signatures() + + def is_unique_args(sig: FunctionSig) -> bool: + """return true if function argument names are unique""" + return len(sig.args) == len({arg.name for arg in sig.args}) + + # Return only signatures that have unique argument names. Mypy fails on non-unique arg names. + return [sig for sig in sigs if is_unique_args(sig)] + + +def infer_arg_sig_from_anon_docstring(docstr: str) -> list[ArgSig]: + """Convert signature in form of "(self: TestClass, arg0: str='ada')" to List[TypedArgList].""" + ret = infer_sig_from_docstring("stub" + docstr, "stub") + if ret: + return ret[0].args + return [] + + +def infer_ret_type_sig_from_docstring(docstr: str, name: str) -> str | None: + """Convert signature in form of "func(self: TestClass, arg0) -> int" to their return type.""" + ret = infer_sig_from_docstring(docstr, name) + if ret: + return ret[0].ret_type + return None + + +def infer_ret_type_sig_from_anon_docstring(docstr: str) -> str | None: + """Convert signature in form of "(self: TestClass, arg0) -> int" to their return type.""" + lines = ["stub" + line.strip() for line in docstr.splitlines() if line.strip().startswith("(")] + return infer_ret_type_sig_from_docstring("".join(lines), "stub") + + +def parse_signature(sig: str) -> tuple[str, list[str], list[str]] | None: + """Split function signature into its name, positional an optional arguments. + + The expected format is "func_name(arg, opt_arg=False)". Return the name of function + and lists of positional and optional argument names. + """ + m = re.match(r"([.a-zA-Z0-9_]+)\(([^)]*)\)", sig) + if not m: + return None + name = m.group(1) + name = name.split(".")[-1] + arg_string = m.group(2) + if not arg_string.strip(): + # Simple case -- no arguments. + return name, [], [] + + args = [arg.strip() for arg in arg_string.split(",")] + positional = [] + optional = [] + i = 0 + while i < len(args): + # Accept optional arguments as in both formats: x=None and [x]. + if args[i].startswith("[") or "=" in args[i]: + break + positional.append(args[i].rstrip("[")) + i += 1 + if args[i - 1].endswith("["): + break + while i < len(args): + arg = args[i] + arg = arg.strip("[]") + arg = arg.split("=")[0] + optional.append(arg) + i += 1 + return name, positional, optional + + +def build_signature(positional: Sequence[str], optional: Sequence[str]) -> str: + """Build function signature from lists of positional and optional argument names.""" + args: MutableSequence[str] = [] + args.extend(positional) + for arg in optional: + if arg.startswith("*"): + args.append(arg) + else: + args.append(f"{arg}=...") + sig = f"({', '.join(args)})" + # Ad-hoc fixes. + sig = sig.replace("(self)", "") + return sig + + +def parse_all_signatures(lines: Sequence[str]) -> tuple[list[Sig], list[Sig]]: + """Parse all signatures in a given reST document. + + Return lists of found signatures for functions and classes. + """ + sigs = [] + class_sigs = [] + for line in lines: + line = line.strip() + m = re.match(r"\.\. *(function|method|class) *:: *[a-zA-Z_]", line) + if m: + sig = line.split("::")[1].strip() + parsed = parse_signature(sig) + if parsed: + name, fixed, optional = parsed + if m.group(1) != "class": + sigs.append((name, build_signature(fixed, optional))) + else: + class_sigs.append((name, build_signature(fixed, optional))) + + return sorted(sigs), sorted(class_sigs) + + +def find_unique_signatures(sigs: Sequence[Sig]) -> list[Sig]: + """Remove names with duplicate found signatures.""" + sig_map: MutableMapping[str, list[str]] = {} + for name, sig in sigs: + sig_map.setdefault(name, []).append(sig) + + result = [] + for name, name_sigs in sig_map.items(): + if len(set(name_sigs)) == 1: + result.append((name, name_sigs[0])) + return sorted(result) + + +def infer_prop_type_from_docstring(docstr: str | None) -> str | None: + """Check for Google/Numpy style docstring type annotation for a property. + + The docstring has the format ": ". + In the type string, we allow the following characters: + * dot: because sometimes classes are annotated using full path + * brackets: to allow type hints like List[int] + * comma/space: things like Tuple[int, int] + """ + if not docstr: + return None + test_str = r"^([a-zA-Z0-9_, \.\[\]]*): " + m = re.match(test_str, docstr) + return m.group(1) if m else None diff --git a/mypy/stubgen.py b/mypy/stubgen.py new file mode 100755 index 000000000000..fdad5c2ddd89 --- /dev/null +++ b/mypy/stubgen.py @@ -0,0 +1,1952 @@ +#!/usr/bin/env python3 +"""Generator of dynamically typed draft stubs for arbitrary modules. + +The logic of this script can be split in three steps: +* parsing options and finding sources: + - use runtime imports be default (to find also C modules) + - or use mypy's mechanisms, if importing is prohibited +* (optionally) semantically analysing the sources using mypy (as a single set) +* emitting the stubs text: + - for Python modules: from ASTs using ASTStubGenerator + - for C modules using runtime introspection and (optionally) Sphinx docs + +During first and third steps some problematic files can be skipped, but any +blocking error during second step will cause the whole program to stop. + +Basic usage: + + $ stubgen foo.py bar.py some_directory + => Generate out/foo.pyi, out/bar.pyi, and stubs for some_directory (recursively). + + $ stubgen -m urllib.parse + => Generate out/urllib/parse.pyi. + + $ stubgen -p urllib + => Generate stubs for whole urllib package (recursively). + +For C modules, you can get more precise function signatures by parsing .rst (Sphinx) +documentation for extra information. For this, use the --doc-dir option: + + $ stubgen --doc-dir /Python-3.4.2/Doc/library -m curses + +Note: The generated stubs should be verified manually. + +TODO: + - maybe use .rst docs also for Python modules + - maybe export more imported names if there is no __all__ (this affects ssl.SSLError, for example) + - a quick and dirty heuristic would be to turn this on if a module has something like + 'from x import y as _y' + - we don't seem to always detect properties ('closed' in 'io', for example) +""" + +from __future__ import annotations + +import argparse +import keyword +import os +import os.path +import sys +import traceback +from typing import Final, Iterable, Iterator + +import mypy.build +import mypy.mixedtraverser +import mypy.parse +import mypy.traverser +import mypy.util +import mypy.version +from mypy.build import build +from mypy.errors import CompileError, Errors +from mypy.find_sources import InvalidSourceList, create_source_list +from mypy.modulefinder import ( + BuildSource, + FindModuleCache, + ModuleNotFoundReason, + SearchPaths, + default_lib_path, +) +from mypy.moduleinspect import ModuleInspect, is_pyc_only +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + ARG_STAR, + ARG_STAR2, + IS_ABSTRACT, + NOT_ABSTRACT, + AssignmentStmt, + Block, + BytesExpr, + CallExpr, + ClassDef, + ComparisonExpr, + ComplexExpr, + Decorator, + DictExpr, + EllipsisExpr, + Expression, + ExpressionStmt, + FloatExpr, + FuncBase, + FuncDef, + IfStmt, + Import, + ImportAll, + ImportFrom, + IndexExpr, + IntExpr, + ListExpr, + MemberExpr, + MypyFile, + NameExpr, + OpExpr, + OverloadedFuncDef, + SetExpr, + StarExpr, + Statement, + StrExpr, + TempNode, + TupleExpr, + TypeAliasStmt, + TypeInfo, + UnaryExpr, + Var, +) +from mypy.options import Options as MypyOptions +from mypy.sharedparse import MAGIC_METHODS_POS_ARGS_ONLY +from mypy.stubdoc import ArgSig, FunctionSig +from mypy.stubgenc import InspectionStubGenerator, generate_stub_for_c_module +from mypy.stubutil import ( + TYPING_BUILTIN_REPLACEMENTS, + BaseStubGenerator, + CantImport, + ClassInfo, + FunctionContext, + common_dir_prefix, + fail_missing, + find_module_path_and_all_py3, + generate_guarded, + infer_method_arg_types, + infer_method_ret_type, + remove_misplaced_type_comments, + report_missing, + walk_packages, +) +from mypy.traverser import ( + all_yield_expressions, + has_return_statement, + has_yield_expression, + has_yield_from_expression, +) +from mypy.types import ( + OVERLOAD_NAMES, + TPDICT_NAMES, + TYPED_NAMEDTUPLE_NAMES, + AnyType, + CallableType, + Instance, + TupleType, + Type, + UnboundType, + get_proper_type, +) +from mypy.visitor import NodeVisitor + +# Common ways of naming package containing vendored modules. +VENDOR_PACKAGES: Final = ["packages", "vendor", "vendored", "_vendor", "_vendored_packages"] + +# Avoid some file names that are unnecessary or likely to cause trouble (\n for end of path). +BLACKLIST: Final = [ + "/six.py\n", # Likely vendored six; too dynamic for us to handle + "/vendored/", # Vendored packages + "/vendor/", # Vendored packages + "/_vendor/", + "/_vendored_packages/", +] + +# These methods are expected to always return a non-trivial value. +METHODS_WITH_RETURN_VALUE: Final = { + "__ne__", + "__eq__", + "__lt__", + "__le__", + "__gt__", + "__ge__", + "__hash__", + "__iter__", +} + + +class Options: + """Represents stubgen options. + + This class is mutable to simplify testing. + """ + + def __init__( + self, + pyversion: tuple[int, int], + no_import: bool, + inspect: bool, + doc_dir: str, + search_path: list[str], + interpreter: str, + parse_only: bool, + ignore_errors: bool, + include_private: bool, + output_dir: str, + modules: list[str], + packages: list[str], + files: list[str], + verbose: bool, + quiet: bool, + export_less: bool, + include_docstrings: bool, + ) -> None: + # See parse_options for descriptions of the flags. + self.pyversion = pyversion + self.no_import = no_import + self.inspect = inspect + self.doc_dir = doc_dir + self.search_path = search_path + self.interpreter = interpreter + self.decointerpreter = interpreter + self.parse_only = parse_only + self.ignore_errors = ignore_errors + self.include_private = include_private + self.output_dir = output_dir + self.modules = modules + self.packages = packages + self.files = files + self.verbose = verbose + self.quiet = quiet + self.export_less = export_less + self.include_docstrings = include_docstrings + + +class StubSource: + """A single source for stub: can be a Python or C module. + + A simple extension of BuildSource that also carries the AST and + the value of __all__ detected at runtime. + """ + + def __init__( + self, module: str, path: str | None = None, runtime_all: list[str] | None = None + ) -> None: + self.source = BuildSource(path, module, None) + self.runtime_all = runtime_all + self.ast: MypyFile | None = None + + def __repr__(self) -> str: + return f"StubSource({self.source})" + + @property + def module(self) -> str: + return self.source.module + + @property + def path(self) -> str | None: + return self.source.path + + +# What was generated previously in the stub file. We keep track of these to generate +# nicely formatted output (add empty line between non-empty classes, for example). +EMPTY: Final = "EMPTY" +FUNC: Final = "FUNC" +CLASS: Final = "CLASS" +EMPTY_CLASS: Final = "EMPTY_CLASS" +VAR: Final = "VAR" +NOT_IN_ALL: Final = "NOT_IN_ALL" + +# Indicates that we failed to generate a reasonable output +# for a given node. These should be manually replaced by a user. + +ERROR_MARKER: Final = "" + + +class AliasPrinter(NodeVisitor[str]): + """Visitor used to collect type aliases _and_ type variable definitions. + + Visit r.h.s of the definition to get the string representation of type alias. + """ + + def __init__(self, stubgen: ASTStubGenerator) -> None: + self.stubgen = stubgen + super().__init__() + + def visit_call_expr(self, node: CallExpr) -> str: + # Call expressions are not usually types, but we also treat `X = TypeVar(...)` as a + # type alias that has to be preserved (even if TypeVar is not the same as an alias) + callee = node.callee.accept(self) + args = [] + for name, arg, kind in zip(node.arg_names, node.args, node.arg_kinds): + if kind == ARG_POS: + args.append(arg.accept(self)) + elif kind == ARG_STAR: + args.append("*" + arg.accept(self)) + elif kind == ARG_STAR2: + args.append("**" + arg.accept(self)) + elif kind == ARG_NAMED: + args.append(f"{name}={arg.accept(self)}") + else: + raise ValueError(f"Unknown argument kind {kind} in call") + return f"{callee}({', '.join(args)})" + + def _visit_ref_expr(self, node: NameExpr | MemberExpr) -> str: + fullname = self.stubgen.get_fullname(node) + if fullname in TYPING_BUILTIN_REPLACEMENTS: + return self.stubgen.add_name(TYPING_BUILTIN_REPLACEMENTS[fullname], require=False) + qualname = get_qualified_name(node) + self.stubgen.import_tracker.require_name(qualname) + return qualname + + def visit_name_expr(self, node: NameExpr) -> str: + return self._visit_ref_expr(node) + + def visit_member_expr(self, o: MemberExpr) -> str: + return self._visit_ref_expr(o) + + def _visit_literal_node( + self, node: StrExpr | BytesExpr | IntExpr | FloatExpr | ComplexExpr + ) -> str: + return repr(node.value) + + def visit_str_expr(self, node: StrExpr) -> str: + return self._visit_literal_node(node) + + def visit_bytes_expr(self, node: BytesExpr) -> str: + return f"b{self._visit_literal_node(node)}" + + def visit_int_expr(self, node: IntExpr) -> str: + return self._visit_literal_node(node) + + def visit_float_expr(self, node: FloatExpr) -> str: + return self._visit_literal_node(node) + + def visit_complex_expr(self, node: ComplexExpr) -> str: + return self._visit_literal_node(node) + + def visit_index_expr(self, node: IndexExpr) -> str: + base_fullname = self.stubgen.get_fullname(node.base) + if base_fullname == "typing.Union": + if isinstance(node.index, TupleExpr): + return " | ".join([item.accept(self) for item in node.index.items]) + return node.index.accept(self) + if base_fullname == "typing.Optional": + if isinstance(node.index, TupleExpr): + return self.stubgen.add_name("_typeshed.Incomplete") + return f"{node.index.accept(self)} | None" + base = node.base.accept(self) + index = node.index.accept(self) + if len(index) > 2 and index.startswith("(") and index.endswith(")"): + index = index[1:-1] + return f"{base}[{index}]" + + def visit_tuple_expr(self, node: TupleExpr) -> str: + return f"({', '.join(n.accept(self) for n in node.items)})" + + def visit_list_expr(self, node: ListExpr) -> str: + return f"[{', '.join(n.accept(self) for n in node.items)}]" + + def visit_dict_expr(self, o: DictExpr) -> str: + dict_items = [] + for key, value in o.items: + # This is currently only used for TypedDict where all keys are strings. + assert isinstance(key, StrExpr) + dict_items.append(f"{key.accept(self)}: {value.accept(self)}") + return f"{{{', '.join(dict_items)}}}" + + def visit_ellipsis(self, node: EllipsisExpr) -> str: + return "..." + + def visit_op_expr(self, o: OpExpr) -> str: + return f"{o.left.accept(self)} {o.op} {o.right.accept(self)}" + + def visit_star_expr(self, o: StarExpr) -> str: + return f"*{o.expr.accept(self)}" + + +def find_defined_names(file: MypyFile) -> set[str]: + finder = DefinitionFinder() + file.accept(finder) + return finder.names + + +def get_assigned_names(lvalues: Iterable[Expression]) -> Iterator[str]: + for lvalue in lvalues: + if isinstance(lvalue, NameExpr): + yield lvalue.name + elif isinstance(lvalue, TupleExpr): + yield from get_assigned_names(lvalue.items) + + +class DefinitionFinder(mypy.traverser.TraverserVisitor): + """Find names of things defined at the top level of a module.""" + + def __init__(self) -> None: + # Short names of things defined at the top level. + self.names: set[str] = set() + + def visit_class_def(self, o: ClassDef) -> None: + # Don't recurse into classes, as we only keep track of top-level definitions. + self.names.add(o.name) + + def visit_func_def(self, o: FuncDef) -> None: + # Don't recurse, as we only keep track of top-level definitions. + self.names.add(o.name) + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + for name in get_assigned_names(o.lvalues): + self.names.add(name) + + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + self.names.add(o.name.name) + + +def find_referenced_names(file: MypyFile) -> set[str]: + finder = ReferenceFinder() + file.accept(finder) + return finder.refs + + +def is_none_expr(expr: Expression) -> bool: + return isinstance(expr, NameExpr) and expr.name == "None" + + +class ReferenceFinder(mypy.mixedtraverser.MixedTraverserVisitor): + """Find all name references (both local and global).""" + + # TODO: Filter out local variable and class attribute references + + def __init__(self) -> None: + # Short names of things defined at the top level. + self.refs: set[str] = set() + + def visit_block(self, block: Block) -> None: + if not block.is_unreachable: + super().visit_block(block) + + def visit_name_expr(self, e: NameExpr) -> None: + self.refs.add(e.name) + + def visit_instance(self, t: Instance) -> None: + self.add_ref(t.type.name) + super().visit_instance(t) + + def visit_unbound_type(self, t: UnboundType) -> None: + if t.name: + self.add_ref(t.name) + + def visit_tuple_type(self, t: TupleType) -> None: + # Ignore fallback + for item in t.items: + item.accept(self) + + def visit_callable_type(self, t: CallableType) -> None: + # Ignore fallback + for arg in t.arg_types: + arg.accept(self) + t.ret_type.accept(self) + + def add_ref(self, fullname: str) -> None: + self.refs.add(fullname) + while "." in fullname: + fullname = fullname.rsplit(".", 1)[0] + self.refs.add(fullname) + + +class ASTStubGenerator(BaseStubGenerator, mypy.traverser.TraverserVisitor): + """Generate stub text from a mypy AST.""" + + def __init__( + self, + _all_: list[str] | None = None, + include_private: bool = False, + analyzed: bool = False, + export_less: bool = False, + include_docstrings: bool = False, + ) -> None: + super().__init__(_all_, include_private, export_less, include_docstrings) + self._decorators: list[str] = [] + # Stack of defined variables (per scope). + self._vars: list[list[str]] = [[]] + # What was generated previously in the stub file. + self._state = EMPTY + self._class_stack: list[ClassDef] = [] + # Was the tree semantically analysed before? + self.analyzed = analyzed + # Short names of methods defined in the body of the current class + self.method_names: set[str] = set() + self.processing_enum = False + self.processing_dataclass = False + + @property + def _current_class(self) -> ClassDef | None: + return self._class_stack[-1] if self._class_stack else None + + def visit_mypy_file(self, o: MypyFile) -> None: + self.module_name = o.fullname # Current module being processed + self.path = o.path + self.set_defined_names(find_defined_names(o)) + self.referenced_names = find_referenced_names(o) + super().visit_mypy_file(o) + self.check_undefined_names() + + def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: + """@property with setters and getters, @overload chain and some others.""" + overload_chain = False + for item in o.items: + if not isinstance(item, Decorator): + continue + if self.is_private_name(item.func.name, item.func.fullname): + continue + + self.process_decorator(item) + if not overload_chain: + self.visit_func_def(item.func) + if item.func.is_overload: + overload_chain = True + elif item.func.is_overload: + self.visit_func_def(item.func) + else: + # skip the overload implementation and clear the decorator we just processed + self.clear_decorators() + + def get_default_function_sig(self, func_def: FuncDef, ctx: FunctionContext) -> FunctionSig: + args = self._get_func_args(func_def, ctx) + retname = self._get_func_return(func_def, ctx) + type_args = self.format_type_args(func_def) + return FunctionSig(func_def.name, args, retname, type_args) + + def _get_func_args(self, o: FuncDef, ctx: FunctionContext) -> list[ArgSig]: + args: list[ArgSig] = [] + + # Ignore pos-only status of magic methods whose args names are elided by mypy at parse + actually_pos_only_args = o.name not in MAGIC_METHODS_POS_ARGS_ONLY + pos_only_marker_position = 0 # Where to insert "/", if any + for i, arg_ in enumerate(o.arguments): + var = arg_.variable + kind = arg_.kind + name = var.name + annotated_type = ( + o.unanalyzed_type.arg_types[i] + if isinstance(o.unanalyzed_type, CallableType) + else None + ) + # I think the name check is incorrect: there are libraries which + # name their 0th argument other than self/cls + is_self_arg = i == 0 and name == "self" + is_cls_arg = i == 0 and name == "cls" + typename: str | None = None + if annotated_type and not is_self_arg and not is_cls_arg: + # Luckily, an argument explicitly annotated with "Any" has + # type "UnboundType" and will not match. + if not isinstance(get_proper_type(annotated_type), AnyType): + typename = self.print_annotation(annotated_type) + + if actually_pos_only_args and arg_.pos_only: + pos_only_marker_position += 1 + + if kind.is_named() and not any(arg.name.startswith("*") for arg in args): + args.append(ArgSig("*")) + + default = "..." + if arg_.initializer: + if not typename: + typename = self.get_str_type_of_node(arg_.initializer, True, False) + potential_default, valid = self.get_str_default_of_node(arg_.initializer) + if valid and len(potential_default) <= 200: + default = potential_default + elif kind == ARG_STAR: + name = f"*{name}" + elif kind == ARG_STAR2: + name = f"**{name}" + + args.append( + ArgSig(name, typename, default=bool(arg_.initializer), default_value=default) + ) + if pos_only_marker_position: + args.insert(pos_only_marker_position, ArgSig("/")) + + if ctx.class_info is not None and all( + arg.type is None and arg.default is False for arg in args + ): + new_args = infer_method_arg_types( + ctx.name, ctx.class_info.self_var, [arg.name for arg in args] + ) + if new_args is not None: + args = new_args + + return args + + def _get_func_return(self, o: FuncDef, ctx: FunctionContext) -> str | None: + if o.name != "__init__" and isinstance(o.unanalyzed_type, CallableType): + if isinstance(get_proper_type(o.unanalyzed_type.ret_type), AnyType): + # Luckily, a return type explicitly annotated with "Any" has + # type "UnboundType" and will enter the else branch. + return None # implicit Any + else: + return self.print_annotation(o.unanalyzed_type.ret_type) + if o.abstract_status == IS_ABSTRACT or o.name in METHODS_WITH_RETURN_VALUE: + # Always assume abstract methods return Any unless explicitly annotated. Also + # some dunder methods should not have a None return type. + return None # implicit Any + retname = infer_method_ret_type(o.name) + if retname is not None: + return retname + if has_yield_expression(o) or has_yield_from_expression(o): + generator_name = self.add_name("collections.abc.Generator") + yield_name = "None" + send_name: str | None = None + return_name: str | None = None + if has_yield_from_expression(o): + yield_name = send_name = self.add_name("_typeshed.Incomplete") + else: + for expr, in_assignment in all_yield_expressions(o): + if expr.expr is not None and not is_none_expr(expr.expr): + yield_name = self.add_name("_typeshed.Incomplete") + if in_assignment: + send_name = self.add_name("_typeshed.Incomplete") + if has_return_statement(o): + return_name = self.add_name("_typeshed.Incomplete") + if return_name is not None: + if send_name is None: + send_name = "None" + return f"{generator_name}[{yield_name}, {send_name}, {return_name}]" + elif send_name is not None: + return f"{generator_name}[{yield_name}, {send_name}]" + else: + return f"{generator_name}[{yield_name}]" + if not has_return_statement(o) and o.abstract_status == NOT_ABSTRACT: + return "None" + return None + + def _get_func_docstring(self, node: FuncDef) -> str | None: + if not node.body.body: + return None + expr = node.body.body[0] + if isinstance(expr, ExpressionStmt) and isinstance(expr.expr, StrExpr): + return expr.expr.value + return None + + def visit_func_def(self, o: FuncDef) -> None: + is_dataclass_generated = ( + self.analyzed and self.processing_dataclass and o.info.names[o.name].plugin_generated + ) + if is_dataclass_generated and o.name != "__init__": + # Skip methods generated by the @dataclass decorator (except for __init__) + return + if ( + self.is_private_name(o.name, o.fullname) + or self.is_not_in_all(o.name) + or (self.is_recorded_name(o.name) and not o.is_overload) + ): + self.clear_decorators() + return + if self.is_top_level() and self._state not in (EMPTY, FUNC): + self.add("\n") + if not self.is_top_level(): + self_inits = find_self_initializers(o) + for init, value in self_inits: + if init in self.method_names: + # Can't have both an attribute and a method/property with the same name. + continue + init_code = self.get_init(init, value) + if init_code: + self.add(init_code) + + if self._class_stack: + if len(o.arguments): + self_var = o.arguments[0].variable.name + else: + self_var = "self" + class_info: ClassInfo | None = None + for class_def in self._class_stack: + class_info = ClassInfo(class_def.name, self_var, parent=class_info) + else: + class_info = None + + ctx = FunctionContext( + module_name=self.module_name, + name=o.name, + docstring=self._get_func_docstring(o), + is_abstract=o.abstract_status != NOT_ABSTRACT, + class_info=class_info, + ) + + self.record_name(o.name) + + default_sig = self.get_default_function_sig(o, ctx) + sigs = self.get_signatures(default_sig, self.sig_generators, ctx) + + for output in self.format_func_def( + sigs, is_coroutine=o.is_coroutine, decorators=self._decorators, docstring=ctx.docstring + ): + self.add(output + "\n") + + self.clear_decorators() + self._state = FUNC + + def visit_decorator(self, o: Decorator) -> None: + if self.is_private_name(o.func.name, o.func.fullname): + return + self.process_decorator(o) + self.visit_func_def(o.func) + + def process_decorator(self, o: Decorator) -> None: + """Process a series of decorators. + + Only preserve certain special decorators such as @abstractmethod. + """ + o.func.is_overload = False + for decorator in o.original_decorators: + if not isinstance(decorator, (NameExpr, MemberExpr)): + continue + qualname = get_qualified_name(decorator) + fullname = self.get_fullname(decorator) + if fullname in ( + "builtins.property", + "builtins.staticmethod", + "builtins.classmethod", + "functools.cached_property", + ): + self.add_decorator(qualname, require_name=True) + elif fullname in ( + "asyncio.coroutine", + "asyncio.coroutines.coroutine", + "types.coroutine", + ): + o.func.is_awaitable_coroutine = True + self.add_decorator(qualname, require_name=True) + elif fullname == "abc.abstractmethod": + self.add_decorator(qualname, require_name=True) + o.func.abstract_status = IS_ABSTRACT + elif fullname in ( + "abc.abstractproperty", + "abc.abstractstaticmethod", + "abc.abstractclassmethod", + ): + abc_module = qualname.rpartition(".")[0] + if not abc_module: + self.import_tracker.add_import("abc") + builtin_decorator_replacement = fullname[len("abc.abstract") :] + self.add_decorator(builtin_decorator_replacement, require_name=False) + self.add_decorator(f"{abc_module or 'abc'}.abstractmethod", require_name=True) + o.func.abstract_status = IS_ABSTRACT + elif fullname in OVERLOAD_NAMES: + self.add_decorator(qualname, require_name=True) + o.func.is_overload = True + elif qualname.endswith((".setter", ".deleter")): + self.add_decorator(qualname, require_name=False) + + def get_fullname(self, expr: Expression) -> str: + """Return the expression's full name.""" + if ( + self.analyzed + and isinstance(expr, (NameExpr, MemberExpr)) + and expr.fullname + and not (isinstance(expr.node, Var) and expr.node.is_suppressed_import) + ): + return expr.fullname + name = get_qualified_name(expr) + return self.resolve_name(name) + + def visit_class_def(self, o: ClassDef) -> None: + self._class_stack.append(o) + self.method_names = find_method_names(o.defs.body) + sep: int | None = None + if self.is_top_level() and self._state != EMPTY: + sep = len(self._output) + self.add("\n") + decorators = self.get_class_decorators(o) + for d in decorators: + self.add(f"{self._indent}@{d}\n") + self.record_name(o.name) + base_types = self.get_base_types(o) + if base_types: + for base in base_types: + self.import_tracker.require_name(base) + if self.analyzed and o.info.is_enum: + self.processing_enum = True + if isinstance(o.metaclass, (NameExpr, MemberExpr)): + meta = o.metaclass.accept(AliasPrinter(self)) + base_types.append("metaclass=" + meta) + elif self.analyzed and o.info.is_abstract and not o.info.is_protocol: + base_types.append("metaclass=abc.ABCMeta") + self.import_tracker.add_import("abc") + self.import_tracker.require_name("abc") + bases = f"({', '.join(base_types)})" if base_types else "" + type_args = self.format_type_args(o) + self.add(f"{self._indent}class {o.name}{type_args}{bases}:\n") + self.indent() + if self._include_docstrings and o.docstring: + docstring = mypy.util.quote_docstring(o.docstring) + self.add(f"{self._indent}{docstring}\n") + n = len(self._output) + self._vars.append([]) + super().visit_class_def(o) + self.dedent() + self._vars.pop() + self._vars[-1].append(o.name) + if len(self._output) == n: + if self._state == EMPTY_CLASS and sep is not None: + self._output[sep] = "" + if not (self._include_docstrings and o.docstring): + self._output[-1] = self._output[-1][:-1] + " ...\n" + self._state = EMPTY_CLASS + else: + self._state = CLASS + self.method_names = set() + self.processing_dataclass = False + self._class_stack.pop(-1) + self.processing_enum = False + + def get_base_types(self, cdef: ClassDef) -> list[str]: + """Get list of base classes for a class.""" + base_types: list[str] = [] + p = AliasPrinter(self) + for base in cdef.base_type_exprs + cdef.removed_base_type_exprs: + if isinstance(base, (NameExpr, MemberExpr)): + if self.get_fullname(base) != "builtins.object": + base_types.append(get_qualified_name(base)) + elif isinstance(base, IndexExpr): + base_types.append(base.accept(p)) + elif isinstance(base, CallExpr): + # namedtuple(typename, fields), NamedTuple(typename, fields) calls can + # be used as a base class. The first argument is a string literal that + # is usually the same as the class name. + # + # Note: + # A call-based named tuple as a base class cannot be safely converted to + # a class-based NamedTuple definition because class attributes defined + # in the body of the class inheriting from the named tuple call are not + # namedtuple fields at runtime. + if self.is_namedtuple(base): + nt_fields = self._get_namedtuple_fields(base) + assert isinstance(base.args[0], StrExpr) + typename = base.args[0].value + if nt_fields is None: + # Invalid namedtuple() call, cannot determine fields + base_types.append(self.add_name("_typeshed.Incomplete")) + continue + fields_str = ", ".join(f"({f!r}, {t})" for f, t in nt_fields) + namedtuple_name = self.add_name("typing.NamedTuple") + base_types.append(f"{namedtuple_name}({typename!r}, [{fields_str}])") + elif self.is_typed_namedtuple(base): + base_types.append(base.accept(p)) + else: + # At this point, we don't know what the base class is, so we + # just use Incomplete as the base class. + base_types.append(self.add_name("_typeshed.Incomplete")) + for name, value in cdef.keywords.items(): + if name == "metaclass": + continue # handled separately + processed_value = value.accept(p) or "..." # at least, don't crash + base_types.append(f"{name}={processed_value}") + return base_types + + def get_class_decorators(self, cdef: ClassDef) -> list[str]: + decorators: list[str] = [] + p = AliasPrinter(self) + for d in cdef.decorators: + if self.is_dataclass(d): + decorators.append(d.accept(p)) + self.import_tracker.require_name(get_qualified_name(d)) + self.processing_dataclass = True + return decorators + + def is_dataclass(self, expr: Expression) -> bool: + if isinstance(expr, CallExpr): + expr = expr.callee + return self.get_fullname(expr) == "dataclasses.dataclass" + + def visit_block(self, o: Block) -> None: + # Unreachable statements may be partially uninitialized and that may + # cause trouble. + if not o.is_unreachable: + super().visit_block(o) + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + foundl = [] + + for lvalue in o.lvalues: + if isinstance(lvalue, NameExpr) and isinstance(o.rvalue, CallExpr): + if self.is_namedtuple(o.rvalue) or self.is_typed_namedtuple(o.rvalue): + self.process_namedtuple(lvalue, o.rvalue) + foundl.append(False) # state is updated in process_namedtuple + continue + if self.is_typeddict(o.rvalue): + self.process_typeddict(lvalue, o.rvalue) + foundl.append(False) # state is updated in process_typeddict + continue + if ( + isinstance(lvalue, NameExpr) + and not self.is_private_name(lvalue.name) + # it is never an alias with explicit annotation + and not o.unanalyzed_type + and self.is_alias_expression(o.rvalue) + ): + self.process_typealias(lvalue, o.rvalue) + continue + if isinstance(lvalue, (TupleExpr, ListExpr)): + items = lvalue.items + if isinstance(o.unanalyzed_type, TupleType): # type: ignore[misc] + annotations: Iterable[Type | None] = o.unanalyzed_type.items + else: + annotations = [None] * len(items) + else: + items = [lvalue] + annotations = [o.unanalyzed_type] + sep = False + found = False + for item, annotation in zip(items, annotations): + if isinstance(item, NameExpr): + init = self.get_init(item.name, o.rvalue, annotation) + if init: + found = True + if not sep and self.is_top_level() and self._state not in (EMPTY, VAR): + init = "\n" + init + sep = True + self.add(init) + self.record_name(item.name) + foundl.append(found) + + if all(foundl): + self._state = VAR + + def is_namedtuple(self, expr: CallExpr) -> bool: + return self.get_fullname(expr.callee) == "collections.namedtuple" + + def is_typed_namedtuple(self, expr: CallExpr) -> bool: + return self.get_fullname(expr.callee) in TYPED_NAMEDTUPLE_NAMES + + def _get_namedtuple_fields(self, call: CallExpr) -> list[tuple[str, str]] | None: + if self.is_namedtuple(call): + fields_arg = call.args[1] + if isinstance(fields_arg, StrExpr): + field_names = fields_arg.value.replace(",", " ").split() + elif isinstance(fields_arg, (ListExpr, TupleExpr)): + field_names = [] + for field in fields_arg.items: + if not isinstance(field, StrExpr): + return None + field_names.append(field.value) + else: + return None # Invalid namedtuple fields type + if field_names: + incomplete = self.add_name("_typeshed.Incomplete") + return [(field_name, incomplete) for field_name in field_names] + else: + return [] + + elif self.is_typed_namedtuple(call): + fields_arg = call.args[1] + if not isinstance(fields_arg, (ListExpr, TupleExpr)): + return None + fields: list[tuple[str, str]] = [] + p = AliasPrinter(self) + for field in fields_arg.items: + if not (isinstance(field, TupleExpr) and len(field.items) == 2): + return None + field_name, field_type = field.items + if not isinstance(field_name, StrExpr): + return None + fields.append((field_name.value, field_type.accept(p))) + return fields + else: + return None # Not a named tuple call + + def process_namedtuple(self, lvalue: NameExpr, rvalue: CallExpr) -> None: + if self._state == CLASS: + self.add("\n") + + if not isinstance(rvalue.args[0], StrExpr): + self.annotate_as_incomplete(lvalue) + return + + fields = self._get_namedtuple_fields(rvalue) + if fields is None: + self.annotate_as_incomplete(lvalue) + return + bases = self.add_name("typing.NamedTuple") + # TODO: Add support for generic NamedTuples. Requires `Generic` as base class. + class_def = f"{self._indent}class {lvalue.name}({bases}):" + if len(fields) == 0: + self.add(f"{class_def} ...\n") + self._state = EMPTY_CLASS + else: + if self._state not in (EMPTY, CLASS): + self.add("\n") + self.add(f"{class_def}\n") + for f_name, f_type in fields: + self.add(f"{self._indent} {f_name}: {f_type}\n") + self._state = CLASS + + def is_typeddict(self, expr: CallExpr) -> bool: + return self.get_fullname(expr.callee) in TPDICT_NAMES + + def process_typeddict(self, lvalue: NameExpr, rvalue: CallExpr) -> None: + if self._state == CLASS: + self.add("\n") + + if not isinstance(rvalue.args[0], StrExpr): + self.annotate_as_incomplete(lvalue) + return + + items: list[tuple[str, Expression]] = [] + total: Expression | None = None + if len(rvalue.args) > 1 and rvalue.arg_kinds[1] == ARG_POS: + if not isinstance(rvalue.args[1], DictExpr): + self.annotate_as_incomplete(lvalue) + return + for attr_name, attr_type in rvalue.args[1].items: + if not isinstance(attr_name, StrExpr): + self.annotate_as_incomplete(lvalue) + return + items.append((attr_name.value, attr_type)) + if len(rvalue.args) > 2: + if rvalue.arg_kinds[2] != ARG_NAMED or rvalue.arg_names[2] != "total": + self.annotate_as_incomplete(lvalue) + return + total = rvalue.args[2] + else: + for arg_name, arg in zip(rvalue.arg_names[1:], rvalue.args[1:]): + if not isinstance(arg_name, str): + self.annotate_as_incomplete(lvalue) + return + if arg_name == "total": + total = arg + else: + items.append((arg_name, arg)) + p = AliasPrinter(self) + if any(not key.isidentifier() or keyword.iskeyword(key) for key, _ in items): + # Keep the call syntax if there are non-identifier or reserved keyword keys. + self.add(f"{self._indent}{lvalue.name} = {rvalue.accept(p)}\n") + self._state = VAR + else: + bases = self.add_name("typing_extensions.TypedDict") + # TODO: Add support for generic TypedDicts. Requires `Generic` as base class. + if total is not None: + bases += f", total={total.accept(p)}" + class_def = f"{self._indent}class {lvalue.name}({bases}):" + if len(items) == 0: + self.add(f"{class_def} ...\n") + self._state = EMPTY_CLASS + else: + if self._state not in (EMPTY, CLASS): + self.add("\n") + self.add(f"{class_def}\n") + for key, key_type in items: + self.add(f"{self._indent} {key}: {key_type.accept(p)}\n") + self._state = CLASS + + def annotate_as_incomplete(self, lvalue: NameExpr) -> None: + incomplete = self.add_name("_typeshed.Incomplete") + self.add(f"{self._indent}{lvalue.name}: {incomplete}\n") + self._state = VAR + + def is_alias_expression(self, expr: Expression, top_level: bool = True) -> bool: + """Return True for things that look like target for an alias. + + Used to know if assignments look like type aliases, function alias, + or module alias. + """ + # Assignment of TypeVar(...) and other typevar-likes are passed through + if isinstance(expr, CallExpr) and self.get_fullname(expr.callee) in ( + "typing.TypeVar", + "typing_extensions.TypeVar", + "typing.ParamSpec", + "typing_extensions.ParamSpec", + "typing.TypeVarTuple", + "typing_extensions.TypeVarTuple", + ): + return True + elif isinstance(expr, EllipsisExpr): + return not top_level + elif isinstance(expr, NameExpr): + if expr.name in ("True", "False"): + return False + elif expr.name == "None": + return not top_level + else: + return not self.is_private_name(expr.name) + elif isinstance(expr, MemberExpr) and self.analyzed: + # Also add function and module aliases. + return ( + top_level + and isinstance(expr.node, (FuncDef, Decorator, MypyFile)) + or isinstance(expr.node, TypeInfo) + ) and not self.is_private_member(expr.node.fullname) + elif isinstance(expr, IndexExpr) and ( + (isinstance(expr.base, NameExpr) and not self.is_private_name(expr.base.name)) + or ( # Also some known aliases that could be member expression + isinstance(expr.base, MemberExpr) + and not self.is_private_member(get_qualified_name(expr.base)) + and self.get_fullname(expr.base).startswith( + ("builtins.", "typing.", "typing_extensions.", "collections.abc.") + ) + ) + ): + if isinstance(expr.index, TupleExpr): + indices = expr.index.items + else: + indices = [expr.index] + if expr.base.name == "Callable" and len(indices) == 2: + args, ret = indices + if isinstance(args, EllipsisExpr): + indices = [ret] + elif isinstance(args, ListExpr): + indices = args.items + [ret] + else: + return False + return all(self.is_alias_expression(i, top_level=False) for i in indices) + elif isinstance(expr, OpExpr) and expr.op == "|": + return self.is_alias_expression( + expr.left, top_level=False + ) and self.is_alias_expression(expr.right, top_level=False) + else: + return False + + def process_typealias(self, lvalue: NameExpr, rvalue: Expression) -> None: + p = AliasPrinter(self) + self.add(f"{self._indent}{lvalue.name} = {rvalue.accept(p)}\n") + self.record_name(lvalue.name) + self._vars[-1].append(lvalue.name) + + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + """Type aliases defined with the `type` keyword (PEP 695).""" + p = AliasPrinter(self) + name = o.name.name + rvalue = o.value.expr() + type_args = self.format_type_args(o) + self.add(f"{self._indent}type {name}{type_args} = {rvalue.accept(p)}\n") + self.record_name(name) + self._vars[-1].append(name) + + def visit_if_stmt(self, o: IfStmt) -> None: + # Ignore if __name__ == '__main__'. + expr = o.expr[0] + if ( + isinstance(expr, ComparisonExpr) + and isinstance(expr.operands[0], NameExpr) + and isinstance(expr.operands[1], StrExpr) + and expr.operands[0].name == "__name__" + and "__main__" in expr.operands[1].value + ): + return + super().visit_if_stmt(o) + + def visit_import_all(self, o: ImportAll) -> None: + self.add_import_line(f"from {'.' * o.relative}{o.id} import *\n") + + def visit_import_from(self, o: ImportFrom) -> None: + exported_names: set[str] = set() + import_names = [] + module, relative = translate_module_name(o.id, o.relative) + if self.module_name: + full_module, ok = mypy.util.correct_relative_import( + self.module_name, relative, module, self.path.endswith(".__init__.py") + ) + if not ok: + full_module = module + else: + full_module = module + if module == "__future__": + return # Not preserved + for name, as_name in o.names: + if name == "six": + # Vendored six -- translate into plain 'import six'. + self.visit_import(Import([("six", None)])) + continue + if self.should_reexport(name, full_module, as_name is not None): + self.import_tracker.reexport(name) + as_name = name + import_names.append((name, as_name)) + self.import_tracker.add_import_from("." * relative + module, import_names) + self._vars[-1].extend(alias or name for name, alias in import_names) + for name, alias in import_names: + self.record_name(alias or name) + + if self._all_: + # Include "import from"s that import names defined in __all__. + names = [ + name + for name, alias in o.names + if name in self._all_ and alias is None and name not in self.IGNORED_DUNDERS + ] + exported_names.update(names) + + def visit_import(self, o: Import) -> None: + for id, as_id in o.ids: + self.import_tracker.add_import(id, as_id) + if as_id is None: + target_name = id.split(".")[0] + else: + target_name = as_id + self._vars[-1].append(target_name) + self.record_name(target_name) + + def get_init( + self, lvalue: str, rvalue: Expression, annotation: Type | None = None + ) -> str | None: + """Return initializer for a variable. + + Return None if we've generated one already or if the variable is internal. + """ + if lvalue in self._vars[-1]: + # We've generated an initializer already for this variable. + return None + # TODO: Only do this at module top level. + if self.is_private_name(lvalue) or self.is_not_in_all(lvalue): + return None + self._vars[-1].append(lvalue) + if annotation is not None: + typename = self.print_annotation(annotation) + if ( + isinstance(annotation, UnboundType) + and not annotation.args + and annotation.name == "Final" + and self.import_tracker.module_for.get("Final") in self.TYPING_MODULE_NAMES + ): + # Final without type argument is invalid in stubs. + final_arg = self.get_str_type_of_node(rvalue) + typename += f"[{final_arg}]" + elif self.processing_enum: + initializer, _ = self.get_str_default_of_node(rvalue) + return f"{self._indent}{lvalue} = {initializer}\n" + elif self.processing_dataclass: + # attribute without annotation is not a dataclass field, don't add annotation. + return f"{self._indent}{lvalue} = ...\n" + else: + typename = self.get_str_type_of_node(rvalue) + initializer = self.get_assign_initializer(rvalue) + return f"{self._indent}{lvalue}: {typename}{initializer}\n" + + def get_assign_initializer(self, rvalue: Expression) -> str: + """Does this rvalue need some special initializer value?""" + if not self._current_class: + return "" + # Current rules + # 1. Return `...` if we are dealing with `NamedTuple` or `dataclass` field and + # it has an existing default value + if ( + self._current_class.info + and self._current_class.info.is_named_tuple + and not isinstance(rvalue, TempNode) + ): + return " = ..." + if self.processing_dataclass and not (isinstance(rvalue, TempNode) and rvalue.no_rhs): + return " = ..." + # TODO: support other possible cases, where initializer is important + + # By default, no initializer is required: + return "" + + def add_decorator(self, name: str, require_name: bool = False) -> None: + if require_name: + self.import_tracker.require_name(name) + self._decorators.append(f"@{name}") + + def clear_decorators(self) -> None: + self._decorators.clear() + + def is_private_member(self, fullname: str) -> bool: + parts = fullname.split(".") + return any(self.is_private_name(part) for part in parts) + + def get_str_type_of_node( + self, rvalue: Expression, can_infer_optional: bool = False, can_be_any: bool = True + ) -> str: + rvalue = self.maybe_unwrap_unary_expr(rvalue) + + if isinstance(rvalue, IntExpr): + return "int" + if isinstance(rvalue, StrExpr): + return "str" + if isinstance(rvalue, BytesExpr): + return "bytes" + if isinstance(rvalue, FloatExpr): + return "float" + if isinstance(rvalue, ComplexExpr): # 1j + return "complex" + if isinstance(rvalue, OpExpr) and rvalue.op in ("-", "+"): # -1j + 1 + if isinstance(self.maybe_unwrap_unary_expr(rvalue.left), ComplexExpr) or isinstance( + self.maybe_unwrap_unary_expr(rvalue.right), ComplexExpr + ): + return "complex" + if isinstance(rvalue, NameExpr) and rvalue.name in ("True", "False"): + return "bool" + if can_infer_optional and isinstance(rvalue, NameExpr) and rvalue.name == "None": + return f"{self.add_name('_typeshed.Incomplete')} | None" + if can_be_any: + return self.add_name("_typeshed.Incomplete") + else: + return "" + + def maybe_unwrap_unary_expr(self, expr: Expression) -> Expression: + """Unwrap (possibly nested) unary expressions. + + But, some unary expressions can change the type of expression. + While we want to preserve it. For example, `~True` is `int`. + So, we only allow a subset of unary expressions to be unwrapped. + """ + if not isinstance(expr, UnaryExpr): + return expr + + # First, try to unwrap `[+-]+ (int|float|complex)` expr: + math_ops = ("+", "-") + if expr.op in math_ops: + while isinstance(expr, UnaryExpr): + if expr.op not in math_ops or not isinstance( + expr.expr, (IntExpr, FloatExpr, ComplexExpr, UnaryExpr) + ): + break + expr = expr.expr + return expr + + # Next, try `not bool` expr: + if expr.op == "not": + while isinstance(expr, UnaryExpr): + if expr.op != "not" or not isinstance(expr.expr, (NameExpr, UnaryExpr)): + break + if isinstance(expr.expr, NameExpr) and expr.expr.name not in ("True", "False"): + break + expr = expr.expr + return expr + + # This is some other unary expr, we cannot do anything with it (yet?). + return expr + + def get_str_default_of_node(self, rvalue: Expression) -> tuple[str, bool]: + """Get a string representation of the default value of a node. + + Returns a 2-tuple of the default and whether or not it is valid. + """ + if isinstance(rvalue, NameExpr): + if rvalue.name in ("None", "True", "False"): + return rvalue.name, True + elif isinstance(rvalue, (IntExpr, FloatExpr)): + return f"{rvalue.value}", True + elif isinstance(rvalue, UnaryExpr): + if isinstance(rvalue.expr, (IntExpr, FloatExpr)): + return f"{rvalue.op}{rvalue.expr.value}", True + elif isinstance(rvalue, StrExpr): + return repr(rvalue.value), True + elif isinstance(rvalue, BytesExpr): + return "b" + repr(rvalue.value).replace("\\\\", "\\"), True + elif isinstance(rvalue, TupleExpr): + items_defaults = [] + for e in rvalue.items: + e_default, valid = self.get_str_default_of_node(e) + if not valid: + break + items_defaults.append(e_default) + else: + closing = ",)" if len(items_defaults) == 1 else ")" + default = "(" + ", ".join(items_defaults) + closing + return default, True + elif isinstance(rvalue, ListExpr): + items_defaults = [] + for e in rvalue.items: + e_default, valid = self.get_str_default_of_node(e) + if not valid: + break + items_defaults.append(e_default) + else: + default = "[" + ", ".join(items_defaults) + "]" + return default, True + elif isinstance(rvalue, SetExpr): + items_defaults = [] + for e in rvalue.items: + e_default, valid = self.get_str_default_of_node(e) + if not valid: + break + items_defaults.append(e_default) + else: + if items_defaults: + default = "{" + ", ".join(items_defaults) + "}" + return default, True + elif isinstance(rvalue, DictExpr): + items_defaults = [] + for k, v in rvalue.items: + if k is None: + break + k_default, k_valid = self.get_str_default_of_node(k) + v_default, v_valid = self.get_str_default_of_node(v) + if not (k_valid and v_valid): + break + items_defaults.append(f"{k_default}: {v_default}") + else: + default = "{" + ", ".join(items_defaults) + "}" + return default, True + return "...", False + + def should_reexport(self, name: str, full_module: str, name_is_alias: bool) -> bool: + is_private = self.is_private_name(name, full_module + "." + name) + if ( + not name_is_alias + and name not in self.referenced_names + and (not self._all_ or name in self.IGNORED_DUNDERS) + and not is_private + and full_module not in ("abc", "asyncio") + self.TYPING_MODULE_NAMES + ): + # An imported name that is never referenced in the module is assumed to be + # exported, unless there is an explicit __all__. Note that we need to special + # case 'abc' since some references are deleted during semantic analysis. + return True + return super().should_reexport(name, full_module, name_is_alias) + + +def find_method_names(defs: list[Statement]) -> set[str]: + # TODO: Traverse into nested definitions + result = set() + for defn in defs: + if isinstance(defn, FuncDef): + result.add(defn.name) + elif isinstance(defn, Decorator): + result.add(defn.func.name) + elif isinstance(defn, OverloadedFuncDef): + for item in defn.items: + result.update(find_method_names([item])) + return result + + +class SelfTraverser(mypy.traverser.TraverserVisitor): + def __init__(self) -> None: + self.results: list[tuple[str, Expression]] = [] + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + lvalue = o.lvalues[0] + if ( + isinstance(lvalue, MemberExpr) + and isinstance(lvalue.expr, NameExpr) + and lvalue.expr.name == "self" + ): + self.results.append((lvalue.name, o.rvalue)) + + +def find_self_initializers(fdef: FuncBase) -> list[tuple[str, Expression]]: + """Find attribute initializers in a method. + + Return a list of pairs (attribute name, r.h.s. expression). + """ + traverser = SelfTraverser() + fdef.accept(traverser) + return traverser.results + + +def get_qualified_name(o: Expression) -> str: + if isinstance(o, NameExpr): + return o.name + elif isinstance(o, MemberExpr): + return f"{get_qualified_name(o.expr)}.{o.name}" + else: + return ERROR_MARKER + + +def remove_blacklisted_modules(modules: list[StubSource]) -> list[StubSource]: + return [ + module for module in modules if module.path is None or not is_blacklisted_path(module.path) + ] + + +def split_pyc_from_py(modules: list[StubSource]) -> tuple[list[StubSource], list[StubSource]]: + py_modules = [] + pyc_modules = [] + for mod in modules: + if is_pyc_only(mod.path): + pyc_modules.append(mod) + else: + py_modules.append(mod) + return pyc_modules, py_modules + + +def is_blacklisted_path(path: str) -> bool: + return any(substr in (normalize_path_separators(path) + "\n") for substr in BLACKLIST) + + +def normalize_path_separators(path: str) -> str: + if sys.platform == "win32": + return path.replace("\\", "/") + return path + + +def collect_build_targets( + options: Options, mypy_opts: MypyOptions +) -> tuple[list[StubSource], list[StubSource], list[StubSource]]: + """Collect files for which we need to generate stubs. + + Return list of py modules, pyc modules, and C modules. + """ + if options.packages or options.modules: + if options.no_import: + py_modules = find_module_paths_using_search( + options.modules, options.packages, options.search_path, options.pyversion + ) + c_modules: list[StubSource] = [] + else: + # Using imports is the default, since we can also find C modules. + py_modules, c_modules = find_module_paths_using_imports( + options.modules, options.packages, options.verbose, options.quiet + ) + else: + # Use mypy native source collection for files and directories. + try: + source_list = create_source_list(options.files, mypy_opts) + except InvalidSourceList as e: + raise SystemExit(str(e)) from e + py_modules = [StubSource(m.module, m.path) for m in source_list] + c_modules = [] + + py_modules = remove_blacklisted_modules(py_modules) + pyc_mod, py_mod = split_pyc_from_py(py_modules) + return py_mod, pyc_mod, c_modules + + +def find_module_paths_using_imports( + modules: list[str], packages: list[str], verbose: bool, quiet: bool +) -> tuple[list[StubSource], list[StubSource]]: + """Find path and runtime value of __all__ (if possible) for modules and packages. + + This function uses runtime Python imports to get the information. + """ + with ModuleInspect() as inspect: + py_modules: list[StubSource] = [] + c_modules: list[StubSource] = [] + found = list(walk_packages(inspect, packages, verbose)) + modules = modules + found + modules = [ + mod for mod in modules if not is_non_library_module(mod) + ] # We don't want to run any tests or scripts + for mod in modules: + try: + result = find_module_path_and_all_py3(inspect, mod, verbose) + except CantImport as e: + tb = traceback.format_exc() + if verbose: + sys.stdout.write(tb) + if not quiet: + report_missing(mod, e.message, tb) + continue + if not result: + c_modules.append(StubSource(mod)) + else: + path, runtime_all = result + py_modules.append(StubSource(mod, path, runtime_all)) + return py_modules, c_modules + + +def is_non_library_module(module: str) -> bool: + """Does module look like a test module or a script?""" + if module.endswith( + ( + ".tests", + ".test", + ".testing", + "_tests", + "_test_suite", + "test_util", + "test_utils", + "test_base", + ".__main__", + ".conftest", # Used by pytest + ".setup", # Typically an install script + ) + ): + return True + if module.split(".")[-1].startswith("test_"): + return True + if ( + ".tests." in module + or ".test." in module + or ".testing." in module + or ".SelfTest." in module + ): + return True + return False + + +def translate_module_name(module: str, relative: int) -> tuple[str, int]: + for pkg in VENDOR_PACKAGES: + for alt in "six.moves", "six": + substr = f"{pkg}.{alt}" + if module.endswith("." + substr) or (module == substr and relative): + return alt, 0 + if "." + substr + "." in module: + return alt + "." + module.partition("." + substr + ".")[2], 0 + return module, relative + + +def find_module_paths_using_search( + modules: list[str], packages: list[str], search_path: list[str], pyversion: tuple[int, int] +) -> list[StubSource]: + """Find sources for modules and packages requested. + + This function just looks for source files at the file system level. + This is used if user passes --no-import, and will not find C modules. + Exit if some of the modules or packages can't be found. + """ + result: list[StubSource] = [] + typeshed_path = default_lib_path(mypy.build.default_data_dir(), pyversion, None) + search_paths = SearchPaths((".",) + tuple(search_path), (), (), tuple(typeshed_path)) + cache = FindModuleCache(search_paths, fscache=None, options=None) + for module in modules: + m_result = cache.find_module(module) + if isinstance(m_result, ModuleNotFoundReason): + fail_missing(module, m_result) + module_path = None + else: + module_path = m_result + result.append(StubSource(module, module_path)) + for package in packages: + p_result = cache.find_modules_recursive(package) + if p_result: + fail_missing(package, ModuleNotFoundReason.NOT_FOUND) + sources = [StubSource(m.module, m.path) for m in p_result] + result.extend(sources) + + result = [m for m in result if not is_non_library_module(m.module)] + + return result + + +def mypy_options(stubgen_options: Options) -> MypyOptions: + """Generate mypy options using the flag passed by user.""" + options = MypyOptions() + options.follow_imports = "skip" + options.incremental = False + options.ignore_errors = True + options.semantic_analysis_only = True + options.python_version = stubgen_options.pyversion + options.show_traceback = True + options.transform_source = remove_misplaced_type_comments + options.preserve_asts = True + options.include_docstrings = stubgen_options.include_docstrings + + # Override cache_dir if provided in the environment + environ_cache_dir = os.getenv("MYPY_CACHE_DIR", "") + if environ_cache_dir.strip(): + options.cache_dir = environ_cache_dir + options.cache_dir = os.path.expanduser(options.cache_dir) + + return options + + +def parse_source_file(mod: StubSource, mypy_options: MypyOptions) -> None: + """Parse a source file. + + On success, store AST in the corresponding attribute of the stub source. + If there are syntax errors, print them and exit. + """ + assert mod.path is not None, "Not found module was not skipped" + with open(mod.path, "rb") as f: + data = f.read() + source = mypy.util.decode_python_encoding(data) + errors = Errors(mypy_options) + mod.ast = mypy.parse.parse( + source, fnam=mod.path, module=mod.module, errors=errors, options=mypy_options + ) + mod.ast._fullname = mod.module + if errors.is_blockers(): + # Syntax error! + for m in errors.new_messages(): + sys.stderr.write(f"{m}\n") + sys.exit(1) + + +def generate_asts_for_modules( + py_modules: list[StubSource], parse_only: bool, mypy_options: MypyOptions, verbose: bool +) -> None: + """Use mypy to parse (and optionally analyze) source files.""" + if not py_modules: + return # Nothing to do here, but there may be C modules + if verbose: + print(f"Processing {len(py_modules)} files...") + if parse_only: + for mod in py_modules: + parse_source_file(mod, mypy_options) + return + # Perform full semantic analysis of the source set. + try: + res = build([module.source for module in py_modules], mypy_options) + except CompileError as e: + raise SystemExit(f"Critical error during semantic analysis: {e}") from e + + for mod in py_modules: + mod.ast = res.graph[mod.module].tree + # Use statically inferred __all__ if there is no runtime one. + if mod.runtime_all is None: + mod.runtime_all = res.manager.semantic_analyzer.export_map[mod.module] + + +def generate_stub_for_py_module( + mod: StubSource, + target: str, + *, + parse_only: bool = False, + inspect: bool = False, + include_private: bool = False, + export_less: bool = False, + include_docstrings: bool = False, + doc_dir: str = "", + all_modules: list[str], +) -> None: + """Use analysed (or just parsed) AST to generate type stub for single file. + + If directory for target doesn't exist it will created. Existing stub + will be overwritten. + """ + if inspect: + ngen = InspectionStubGenerator( + module_name=mod.module, + known_modules=all_modules, + _all_=mod.runtime_all, + doc_dir=doc_dir, + include_private=include_private, + export_less=export_less, + include_docstrings=include_docstrings, + ) + ngen.generate_module() + output = ngen.output() + + else: + gen = ASTStubGenerator( + mod.runtime_all, + include_private=include_private, + analyzed=not parse_only, + export_less=export_less, + include_docstrings=include_docstrings, + ) + assert mod.ast is not None, "This function must be used only with analyzed modules" + mod.ast.accept(gen) + output = gen.output() + + # Write output to file. + subdir = os.path.dirname(target) + if subdir and not os.path.isdir(subdir): + os.makedirs(subdir) + with open(target, "w", encoding="utf-8") as file: + file.write(output) + + +def generate_stubs(options: Options) -> None: + """Main entry point for the program.""" + mypy_opts = mypy_options(options) + py_modules, pyc_modules, c_modules = collect_build_targets(options, mypy_opts) + all_modules = py_modules + pyc_modules + c_modules + all_module_names = sorted(m.module for m in all_modules) + # Use parsed sources to generate stubs for Python modules. + generate_asts_for_modules(py_modules, options.parse_only, mypy_opts, options.verbose) + files = [] + for mod in py_modules + pyc_modules: + assert mod.path is not None, "Not found module was not skipped" + target = mod.module.replace(".", "/") + if os.path.basename(mod.path) in ["__init__.py", "__init__.pyc"]: + target += "/__init__.pyi" + else: + target += ".pyi" + target = os.path.join(options.output_dir, target) + files.append(target) + with generate_guarded(mod.module, target, options.ignore_errors, options.verbose): + generate_stub_for_py_module( + mod, + target, + parse_only=options.parse_only, + inspect=options.inspect or mod in pyc_modules, + include_private=options.include_private, + export_less=options.export_less, + include_docstrings=options.include_docstrings, + doc_dir=options.doc_dir, + all_modules=all_module_names, + ) + + # Separately analyse C modules using different logic. + for mod in c_modules: + if any(py_mod.module.startswith(mod.module + ".") for py_mod in all_modules): + target = mod.module.replace(".", "/") + "/__init__.pyi" + else: + target = mod.module.replace(".", "/") + ".pyi" + target = os.path.join(options.output_dir, target) + files.append(target) + with generate_guarded(mod.module, target, options.ignore_errors, options.verbose): + generate_stub_for_c_module( + mod.module, + target, + known_modules=all_module_names, + doc_dir=options.doc_dir, + include_private=options.include_private, + export_less=options.export_less, + include_docstrings=options.include_docstrings, + ) + num_modules = len(all_modules) + if not options.quiet and num_modules > 0: + print("Processed %d modules" % num_modules) + if len(files) == 1: + print(f"Generated {files[0]}") + else: + print(f"Generated files under {common_dir_prefix(files)}" + os.sep) + + +HEADER = """%(prog)s [-h] [more options, see -h] + [-m MODULE] [-p PACKAGE] [files ...]""" + +DESCRIPTION = """ +Generate draft stubs for modules. + +Stubs are generated in directory ./out, to avoid overriding files with +manual changes. This directory is assumed to exist. +""" + + +def parse_options(args: list[str]) -> Options: + parser = argparse.ArgumentParser( + prog="stubgen", usage=HEADER, description=DESCRIPTION, fromfile_prefix_chars="@" + ) + + parser.add_argument( + "--ignore-errors", + action="store_true", + help="ignore errors when trying to generate stubs for modules", + ) + parser.add_argument( + "--no-import", + action="store_true", + help="don't import the modules, just parse and analyze them " + "(doesn't work with C extension modules and might not " + "respect __all__)", + ) + parser.add_argument( + "--no-analysis", + "--parse-only", + dest="parse_only", + action="store_true", + help="don't perform semantic analysis of sources, just parse them " + "(only applies to Python modules, might affect quality of stubs. " + "Not compatible with --inspect-mode)", + ) + parser.add_argument( + "--inspect-mode", + dest="inspect", + action="store_true", + help="import and inspect modules instead of parsing source code." + "This is the default behavior for c modules and pyc-only packages, but " + "it is also useful for pure python modules with dynamically generated members.", + ) + parser.add_argument( + "--include-private", + action="store_true", + help="generate stubs for objects and members considered private " + "(single leading underscore and no trailing underscores)", + ) + parser.add_argument( + "--export-less", + action="store_true", + help="don't implicitly export all names imported from other modules in the same package", + ) + parser.add_argument( + "--include-docstrings", + action="store_true", + help="include existing docstrings with the stubs", + ) + parser.add_argument("-v", "--verbose", action="store_true", help="show more verbose messages") + parser.add_argument("-q", "--quiet", action="store_true", help="show fewer messages") + parser.add_argument( + "--doc-dir", + metavar="PATH", + default="", + help="use .rst documentation in PATH (this may result in " + "better stubs in some cases; consider setting this to " + "DIR/Python-X.Y.Z/Doc/library)", + ) + parser.add_argument( + "--search-path", + metavar="PATH", + default="", + help="specify module search directories, separated by ':' " + "(currently only used if --no-import is given)", + ) + parser.add_argument( + "-o", + "--output", + metavar="PATH", + dest="output_dir", + default="out", + help="change the output directory [default: %(default)s]", + ) + parser.add_argument( + "-m", + "--module", + action="append", + metavar="MODULE", + dest="modules", + default=[], + help="generate stub for module; can repeat for more modules", + ) + parser.add_argument( + "-p", + "--package", + action="append", + metavar="PACKAGE", + dest="packages", + default=[], + help="generate stubs for package recursively; can be repeated", + ) + parser.add_argument( + metavar="files", + nargs="*", + dest="files", + help="generate stubs for given files or directories", + ) + parser.add_argument( + "--version", action="version", version="%(prog)s " + mypy.version.__version__ + ) + + ns = parser.parse_args(args) + + pyversion = sys.version_info[:2] + ns.interpreter = sys.executable + + if ns.modules + ns.packages and ns.files: + parser.error("May only specify one of: modules/packages or files.") + if ns.quiet and ns.verbose: + parser.error("Cannot specify both quiet and verbose messages") + if ns.inspect and ns.parse_only: + parser.error("Cannot specify both --parse-only/--no-analysis and --inspect-mode") + + # Create the output folder if it doesn't already exist. + os.makedirs(ns.output_dir, exist_ok=True) + + return Options( + pyversion=pyversion, + no_import=ns.no_import, + inspect=ns.inspect, + doc_dir=ns.doc_dir, + search_path=ns.search_path.split(":"), + interpreter=ns.interpreter, + ignore_errors=ns.ignore_errors, + parse_only=ns.parse_only, + include_private=ns.include_private, + output_dir=ns.output_dir, + modules=ns.modules, + packages=ns.packages, + files=ns.files, + verbose=ns.verbose, + quiet=ns.quiet, + export_less=ns.export_less, + include_docstrings=ns.include_docstrings, + ) + + +def main(args: list[str] | None = None) -> None: + mypy.util.check_python_version("stubgen") + # Make sure that the current directory is in sys.path so that + # stubgen can be run on packages in the current directory. + if not ("" in sys.path or "." in sys.path): + sys.path.insert(0, "") + + options = parse_options(sys.argv[1:] if args is None else args) + generate_stubs(options) + + +if __name__ == "__main__": + main() diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py new file mode 100755 index 000000000000..1cd709b9d603 --- /dev/null +++ b/mypy/stubgenc.py @@ -0,0 +1,1027 @@ +#!/usr/bin/env python3 +"""Stub generator for C modules. + +The public interface is via the mypy.stubgen module. +""" + +from __future__ import annotations + +import glob +import importlib +import inspect +import keyword +import os.path +from types import FunctionType, ModuleType +from typing import Any, Callable, Mapping + +from mypy.fastparse import parse_type_comment +from mypy.moduleinspect import is_c_module +from mypy.stubdoc import ( + ArgSig, + FunctionSig, + Sig, + find_unique_signatures, + infer_arg_sig_from_anon_docstring, + infer_prop_type_from_docstring, + infer_ret_type_sig_from_anon_docstring, + infer_ret_type_sig_from_docstring, + infer_sig_from_docstring, + parse_all_signatures, +) +from mypy.stubutil import ( + BaseStubGenerator, + ClassInfo, + FunctionContext, + SignatureGenerator, + infer_method_arg_types, + infer_method_ret_type, +) + + +class ExternalSignatureGenerator(SignatureGenerator): + def __init__( + self, func_sigs: dict[str, str] | None = None, class_sigs: dict[str, str] | None = None + ) -> None: + """ + Takes a mapping of function/method names to signatures and class name to + class signatures (usually corresponds to __init__). + """ + self.func_sigs = func_sigs or {} + self.class_sigs = class_sigs or {} + + @classmethod + def from_doc_dir(cls, doc_dir: str) -> ExternalSignatureGenerator: + """Instantiate from a directory of .rst files.""" + all_sigs: list[Sig] = [] + all_class_sigs: list[Sig] = [] + for path in glob.glob(f"{doc_dir}/*.rst"): + with open(path) as f: + loc_sigs, loc_class_sigs = parse_all_signatures(f.readlines()) + all_sigs += loc_sigs + all_class_sigs += loc_class_sigs + sigs = dict(find_unique_signatures(all_sigs)) + class_sigs = dict(find_unique_signatures(all_class_sigs)) + return ExternalSignatureGenerator(sigs, class_sigs) + + def get_function_sig( + self, default_sig: FunctionSig, ctx: FunctionContext + ) -> list[FunctionSig] | None: + # method: + if ( + ctx.class_info + and ctx.name in ("__new__", "__init__") + and ctx.name not in self.func_sigs + and ctx.class_info.name in self.class_sigs + ): + return [ + FunctionSig( + name=ctx.name, + args=infer_arg_sig_from_anon_docstring(self.class_sigs[ctx.class_info.name]), + ret_type=infer_method_ret_type(ctx.name), + ) + ] + + # function: + if ctx.name not in self.func_sigs: + return None + + inferred = [ + FunctionSig( + name=ctx.name, + args=infer_arg_sig_from_anon_docstring(self.func_sigs[ctx.name]), + ret_type=None, + ) + ] + if ctx.class_info: + return self.remove_self_type(inferred, ctx.class_info.self_var) + else: + return inferred + + def get_property_type(self, default_type: str | None, ctx: FunctionContext) -> str | None: + return None + + +class DocstringSignatureGenerator(SignatureGenerator): + def get_function_sig( + self, default_sig: FunctionSig, ctx: FunctionContext + ) -> list[FunctionSig] | None: + inferred = infer_sig_from_docstring(ctx.docstring, ctx.name) + if inferred: + assert ctx.docstring is not None + if is_pybind11_overloaded_function_docstring(ctx.docstring, ctx.name): + # Remove pybind11 umbrella (*args, **kwargs) for overloaded functions + del inferred[-1] + + if ctx.class_info: + if not inferred and ctx.name == "__init__": + # look for class-level constructor signatures of the form () + inferred = infer_sig_from_docstring(ctx.class_info.docstring, ctx.class_info.name) + if inferred: + inferred = [sig._replace(name="__init__") for sig in inferred] + return self.remove_self_type(inferred, ctx.class_info.self_var) + else: + return inferred + + def get_property_type(self, default_type: str | None, ctx: FunctionContext) -> str | None: + """Infer property type from docstring or docstring signature.""" + if ctx.docstring is not None: + inferred = infer_ret_type_sig_from_anon_docstring(ctx.docstring) + if inferred: + return inferred + inferred = infer_ret_type_sig_from_docstring(ctx.docstring, ctx.name) + if inferred: + return inferred + inferred = infer_prop_type_from_docstring(ctx.docstring) + return inferred + else: + return None + + +def is_pybind11_overloaded_function_docstring(docstring: str, name: str) -> bool: + return docstring.startswith(f"{name}(*args, **kwargs)\nOverloaded function.\n\n") + + +def generate_stub_for_c_module( + module_name: str, + target: str, + known_modules: list[str], + doc_dir: str = "", + *, + include_private: bool = False, + export_less: bool = False, + include_docstrings: bool = False, +) -> None: + """Generate stub for C module. + + Signature generators are called in order until a list of signatures is returned. The order + is: + - signatures inferred from .rst documentation (if given) + - simple runtime introspection (looking for docstrings and attributes + with simple builtin types) + - fallback based special method names or "(*args, **kwargs)" + + If directory for target doesn't exist it will be created. Existing stub + will be overwritten. + """ + subdir = os.path.dirname(target) + if subdir and not os.path.isdir(subdir): + os.makedirs(subdir) + + gen = InspectionStubGenerator( + module_name, + known_modules, + doc_dir, + include_private=include_private, + export_less=export_less, + include_docstrings=include_docstrings, + ) + gen.generate_module() + output = gen.output() + + with open(target, "w", encoding="utf-8") as file: + file.write(output) + + +class CFunctionStub: + """ + Class that mimics a C function in order to provide parseable docstrings. + """ + + def __init__(self, name: str, doc: str, is_abstract: bool = False) -> None: + self.__name__ = name + self.__doc__ = doc + self.__abstractmethod__ = is_abstract + + @classmethod + def _from_sig(cls, sig: FunctionSig, is_abstract: bool = False) -> CFunctionStub: + return CFunctionStub(sig.name, sig.format_sig()[:-4], is_abstract) + + @classmethod + def _from_sigs(cls, sigs: list[FunctionSig], is_abstract: bool = False) -> CFunctionStub: + return CFunctionStub( + sigs[0].name, "\n".join(sig.format_sig()[:-4] for sig in sigs), is_abstract + ) + + def __get__(self) -> None: + """ + This exists to make this object look like a method descriptor and thus + return true for CStubGenerator.ismethod() + """ + pass + + +class InspectionStubGenerator(BaseStubGenerator): + """Stub generator that does not parse code. + + Generation is performed by inspecting the module's contents, and thus works + for highly dynamic modules, pyc files, and C modules (via the CStubGenerator + subclass). + """ + + def __init__( + self, + module_name: str, + known_modules: list[str], + doc_dir: str = "", + _all_: list[str] | None = None, + include_private: bool = False, + export_less: bool = False, + include_docstrings: bool = False, + module: ModuleType | None = None, + ) -> None: + self.doc_dir = doc_dir + if module is None: + self.module = importlib.import_module(module_name) + else: + self.module = module + self.is_c_module = is_c_module(self.module) + self.known_modules = known_modules + self.resort_members = self.is_c_module + super().__init__(_all_, include_private, export_less, include_docstrings) + self.module_name = module_name + if self.is_c_module: + # Add additional implicit imports. + # C-extensions are given more lattitude since they do not import the typing module. + self.known_imports.update( + { + "typing": [ + "Any", + "Callable", + "ClassVar", + "Dict", + "Iterable", + "Iterator", + "List", + "Literal", + "NamedTuple", + "Optional", + "Tuple", + "Union", + ] + } + ) + + def get_default_function_sig(self, func: object, ctx: FunctionContext) -> FunctionSig: + argspec = None + if not self.is_c_module: + # Get the full argument specification of the function + try: + argspec = inspect.getfullargspec(func) + except TypeError: + # some callables cannot be inspected, e.g. functools.partial + pass + if argspec is None: + if ctx.class_info is not None: + # method: + return FunctionSig( + name=ctx.name, + args=infer_c_method_args(ctx.name, ctx.class_info.self_var), + ret_type=infer_method_ret_type(ctx.name), + ) + else: + # function: + return FunctionSig( + name=ctx.name, + args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], + ret_type=None, + ) + + # Extract the function arguments, defaults, and varargs + args = argspec.args + defaults = argspec.defaults + varargs = argspec.varargs + kwargs = argspec.varkw + annotations = argspec.annotations + kwonlyargs = argspec.kwonlyargs + kwonlydefaults = argspec.kwonlydefaults + + def get_annotation(key: str) -> str | None: + if key not in annotations: + return None + argtype = annotations[key] + if argtype is None: + return "None" + if not isinstance(argtype, str): + return self.get_type_fullname(argtype) + return argtype + + arglist: list[ArgSig] = [] + + # Add the arguments to the signature + def add_args( + args: list[str], get_default_value: Callable[[int, str], object | None] + ) -> None: + for i, arg in enumerate(args): + # Check if the argument has a default value + default_value = get_default_value(i, arg) + if default_value is not None: + if arg in annotations: + argtype = annotations[arg] + else: + argtype = self.get_type_annotation(default_value) + if argtype == "None": + # None is not a useful annotation, but we can infer that the arg + # is optional + incomplete = self.add_name("_typeshed.Incomplete") + argtype = f"{incomplete} | None" + + arglist.append(ArgSig(arg, argtype, default=True)) + else: + arglist.append(ArgSig(arg, get_annotation(arg), default=False)) + + def get_pos_default(i: int, _arg: str) -> Any | None: + if defaults and i >= len(args) - len(defaults): + return defaults[i - (len(args) - len(defaults))] + else: + return None + + add_args(args, get_pos_default) + + # Add *args if present + if varargs: + arglist.append(ArgSig(f"*{varargs}", get_annotation(varargs))) + # if we have keyword only args, then wee need to add "*" + elif kwonlyargs: + arglist.append(ArgSig("*")) + + def get_kw_default(_i: int, arg: str) -> Any | None: + if kwonlydefaults: + return kwonlydefaults.get(arg) + else: + return None + + add_args(kwonlyargs, get_kw_default) + + # Add **kwargs if present + if kwargs: + arglist.append(ArgSig(f"**{kwargs}", get_annotation(kwargs))) + + # add types for known special methods + if ctx.class_info is not None and all( + arg.type is None and arg.default is False for arg in arglist + ): + new_args = infer_method_arg_types( + ctx.name, ctx.class_info.self_var, [arg.name for arg in arglist if arg.name] + ) + if new_args is not None: + arglist = new_args + + ret_type = get_annotation("return") or infer_method_ret_type(ctx.name) + return FunctionSig(ctx.name, arglist, ret_type) + + def get_sig_generators(self) -> list[SignatureGenerator]: + if not self.is_c_module: + return [] + else: + sig_generators: list[SignatureGenerator] = [DocstringSignatureGenerator()] + if self.doc_dir: + # Collect info from docs (if given). Always check these first. + sig_generators.insert(0, ExternalSignatureGenerator.from_doc_dir(self.doc_dir)) + return sig_generators + + def strip_or_import(self, type_name: str) -> str: + """Strips unnecessary module names from typ. + + If typ represents a type that is inside module or is a type coming from builtins, remove + module declaration from it. Return stripped name of the type. + + Arguments: + typ: name of the type + """ + local_modules = ["builtins", self.module_name] + parsed_type = parse_type_comment(type_name, 0, 0, None)[1] + assert parsed_type is not None, type_name + return self.print_annotation(parsed_type, self.known_modules, local_modules) + + def get_obj_module(self, obj: object) -> str | None: + """Return module name of the object.""" + return getattr(obj, "__module__", None) + + def is_defined_in_module(self, obj: object) -> bool: + """Check if object is considered defined in the current module.""" + module = self.get_obj_module(obj) + return module is None or module == self.module_name + + def generate_module(self) -> None: + all_items = self.get_members(self.module) + if self.resort_members: + all_items = sorted(all_items, key=lambda x: x[0]) + items = [] + for name, obj in all_items: + if inspect.ismodule(obj) and obj.__name__ in self.known_modules: + module_name = obj.__name__ + if module_name.startswith(self.module_name + "."): + # from {.rel_name} import {mod_name} as {name} + pkg_name, mod_name = module_name.rsplit(".", 1) + rel_module = pkg_name[len(self.module_name) :] or "." + self.import_tracker.add_import_from(rel_module, [(mod_name, name)]) + self.import_tracker.reexport(name) + else: + # import {module_name} as {name} + self.import_tracker.add_import(module_name, name) + self.import_tracker.reexport(name) + elif self.is_defined_in_module(obj) and not inspect.ismodule(obj): + # process this below + items.append((name, obj)) + else: + # from {obj_module} import {obj_name} + obj_module_name = self.get_obj_module(obj) + if obj_module_name: + self.import_tracker.add_import_from(obj_module_name, [(name, None)]) + if self.should_reexport(name, obj_module_name, name_is_alias=False): + self.import_tracker.reexport(name) + + self.set_defined_names({name for name, obj in all_items if not inspect.ismodule(obj)}) + + if self.resort_members: + functions: list[str] = [] + types: list[str] = [] + variables: list[str] = [] + else: + output: list[str] = [] + functions = types = variables = output + + for name, obj in items: + if self.is_function(obj): + self.generate_function_stub(name, obj, output=functions) + elif inspect.isclass(obj): + self.generate_class_stub(name, obj, output=types) + else: + self.generate_variable_stub(name, obj, output=variables) + + self._output = [] + + if self.resort_members: + for line in variables: + self._output.append(line + "\n") + for line in types: + if line.startswith("class") and self._output and self._output[-1]: + self._output.append("\n") + self._output.append(line + "\n") + if self._output and functions: + self._output.append("\n") + for line in functions: + self._output.append(line + "\n") + else: + for i, line in enumerate(output): + if ( + self._output + and line.startswith("class") + and ( + not self._output[-1].startswith("class") + or (len(output) > i + 1 and output[i + 1].startswith(" ")) + ) + ) or ( + self._output + and self._output[-1].startswith("def") + and not line.startswith("def") + ): + self._output.append("\n") + self._output.append(line + "\n") + self.check_undefined_names() + + def is_skipped_attribute(self, attr: str) -> bool: + return ( + attr + in ( + "__class__", + "__getattribute__", + "__str__", + "__repr__", + "__doc__", + "__dict__", + "__module__", + "__weakref__", + "__annotations__", + "__firstlineno__", + "__static_attributes__", + "__annotate__", + ) + or attr in self.IGNORED_DUNDERS + or is_pybind_skipped_attribute(attr) # For pickling + or keyword.iskeyword(attr) + ) + + def get_members(self, obj: object) -> list[tuple[str, Any]]: + obj_dict: Mapping[str, Any] = getattr(obj, "__dict__") # noqa: B009 + results = [] + for name in obj_dict: + if self.is_skipped_attribute(name): + continue + # Try to get the value via getattr + try: + value = getattr(obj, name) + except AttributeError: + continue + else: + results.append((name, value)) + return results + + def get_type_annotation(self, obj: object) -> str: + """ + Given an instance, return a string representation of its type that is valid + to use as a type annotation. + """ + if obj is None or obj is type(None): + return "None" + elif inspect.isclass(obj): + return f"type[{self.get_type_fullname(obj)}]" + elif isinstance(obj, FunctionType): + return self.add_name("typing.Callable") + elif isinstance(obj, ModuleType): + return self.add_name("types.ModuleType", require=False) + else: + return self.get_type_fullname(type(obj)) + + def is_function(self, obj: object) -> bool: + if self.is_c_module: + return inspect.isbuiltin(obj) + else: + return inspect.isfunction(obj) + + def is_method(self, class_info: ClassInfo, name: str, obj: object) -> bool: + if self.is_c_module: + return inspect.ismethoddescriptor(obj) or type(obj) in ( + type(str.index), + type(str.__add__), + type(str.__new__), + ) + else: + # this is valid because it is only called on members of a class + return inspect.isfunction(obj) + + def is_classmethod(self, class_info: ClassInfo, name: str, obj: object) -> bool: + if self.is_c_module: + return inspect.isbuiltin(obj) or type(obj).__name__ in ( + "classmethod", + "classmethod_descriptor", + ) + else: + return inspect.ismethod(obj) + + def is_staticmethod(self, class_info: ClassInfo | None, name: str, obj: object) -> bool: + if class_info is None: + return False + elif self.is_c_module: + raw_lookup: Mapping[str, Any] = getattr(class_info.cls, "__dict__") # noqa: B009 + raw_value = raw_lookup.get(name, obj) + return isinstance(raw_value, staticmethod) + else: + return isinstance(inspect.getattr_static(class_info.cls, name), staticmethod) + + @staticmethod + def is_abstract_method(obj: object) -> bool: + return getattr(obj, "__abstractmethod__", False) + + @staticmethod + def is_property(class_info: ClassInfo, name: str, obj: object) -> bool: + return inspect.isdatadescriptor(obj) or hasattr(obj, "fget") + + @staticmethod + def is_property_readonly(prop: Any) -> bool: + return hasattr(prop, "fset") and prop.fset is None + + def is_static_property(self, obj: object) -> bool: + """For c-modules, whether the property behaves like an attribute""" + if self.is_c_module: + # StaticProperty is from boost-python + return type(obj).__name__ in ("pybind11_static_property", "StaticProperty") + else: + return False + + def process_inferred_sigs(self, inferred: list[FunctionSig]) -> None: + for i, sig in enumerate(inferred): + for arg in sig.args: + if arg.type is not None: + arg.type = self.strip_or_import(arg.type) + if sig.ret_type is not None: + inferred[i] = sig._replace(ret_type=self.strip_or_import(sig.ret_type)) + + def generate_function_stub( + self, name: str, obj: object, *, output: list[str], class_info: ClassInfo | None = None + ) -> None: + """Generate stub for a single function or method. + + The result (always a single line) will be appended to 'output'. + If necessary, any required names will be added to 'imports'. + The 'class_name' is used to find signature of __init__ or __new__ in + 'class_sigs'. + """ + docstring: Any = getattr(obj, "__doc__", None) + if not isinstance(docstring, str): + docstring = None + + ctx = FunctionContext( + self.module_name, + name, + docstring=docstring, + is_abstract=self.is_abstract_method(obj), + class_info=class_info, + ) + if self.is_private_name(name, ctx.fullname) or self.is_not_in_all(name): + return + + self.record_name(ctx.name) + default_sig = self.get_default_function_sig(obj, ctx) + inferred = self.get_signatures(default_sig, self.sig_generators, ctx) + self.process_inferred_sigs(inferred) + + decorators = [] + if len(inferred) > 1: + decorators.append("@{}".format(self.add_name("typing.overload"))) + + if ctx.is_abstract: + decorators.append("@{}".format(self.add_name("abc.abstractmethod"))) + + if class_info is not None: + if self.is_staticmethod(class_info, name, obj): + decorators.append("@staticmethod") + else: + for sig in inferred: + if not sig.args or sig.args[0].name not in ("self", "cls"): + sig.args.insert(0, ArgSig(name=class_info.self_var)) + # a sig generator indicates @classmethod by specifying the cls arg. + if inferred[0].args and inferred[0].args[0].name == "cls": + decorators.append("@classmethod") + + if docstring: + docstring = self._indent_docstring(docstring) + output.extend(self.format_func_def(inferred, decorators=decorators, docstring=docstring)) + self._fix_iter(ctx, inferred, output) + + def _indent_docstring(self, docstring: str) -> str: + """Fix indentation of docstring extracted from pybind11 or other binding generators.""" + lines = docstring.splitlines(keepends=True) + indent = self._indent + " " + if len(lines) > 1: + if not all(line.startswith(indent) or not line.strip() for line in lines): + # if the docstring is not indented, then indent all but the first line + for i, line in enumerate(lines[1:]): + if line.strip(): + lines[i + 1] = indent + line + # if there's a trailing newline, add a final line to visually indent the quoted docstring + if lines[-1].endswith("\n"): + if len(lines) > 1: + lines.append(indent) + else: + lines[-1] = lines[-1][:-1] + return "".join(lines) + + def _fix_iter( + self, ctx: FunctionContext, inferred: list[FunctionSig], output: list[str] + ) -> None: + """Ensure that objects which implement old-style iteration via __getitem__ + are considered iterable. + """ + if ( + ctx.class_info + and ctx.class_info.cls is not None + and ctx.name == "__getitem__" + and "__iter__" not in ctx.class_info.cls.__dict__ + ): + item_type: str | None = None + for sig in inferred: + if sig.args and sig.args[-1].type == "int": + item_type = sig.ret_type + break + if item_type is None: + return + obj = CFunctionStub( + "__iter__", f"def __iter__(self) -> typing.Iterator[{item_type}]\n" + ) + self.generate_function_stub("__iter__", obj, output=output, class_info=ctx.class_info) + + def generate_property_stub( + self, + name: str, + raw_obj: object, + obj: object, + static_properties: list[str], + rw_properties: list[str], + ro_properties: list[str], + class_info: ClassInfo | None = None, + ) -> None: + """Generate property stub using introspection of 'obj'. + + Try to infer type from docstring, append resulting lines to 'output'. + + raw_obj : object before evaluation of descriptor (if any) + obj : object after evaluation of descriptor + """ + + docstring = getattr(raw_obj, "__doc__", None) + fget = getattr(raw_obj, "fget", None) + if fget: + alt_docstr = getattr(fget, "__doc__", None) + if alt_docstr and docstring: + docstring += "\n" + alt_docstr + elif alt_docstr: + docstring = alt_docstr + + ctx = FunctionContext( + self.module_name, name, docstring=docstring, is_abstract=False, class_info=class_info + ) + + if self.is_private_name(name, ctx.fullname) or self.is_not_in_all(name): + return + + self.record_name(ctx.name) + static = self.is_static_property(raw_obj) + readonly = self.is_property_readonly(raw_obj) + if static: + ret_type: str | None = self.strip_or_import(self.get_type_annotation(obj)) + else: + default_sig = self.get_default_function_sig(raw_obj, ctx) + ret_type = default_sig.ret_type + + inferred_type = self.get_property_type(ret_type, self.sig_generators, ctx) + if inferred_type is not None: + inferred_type = self.strip_or_import(inferred_type) + + if static: + classvar = self.add_name("typing.ClassVar") + trailing_comment = " # read-only" if readonly else "" + if inferred_type is None: + inferred_type = self.add_name("_typeshed.Incomplete") + + static_properties.append( + f"{self._indent}{name}: {classvar}[{inferred_type}] = ...{trailing_comment}" + ) + else: # regular property + if readonly: + ro_properties.append(f"{self._indent}@property") + sig = FunctionSig(name, [ArgSig("self")], inferred_type) + ro_properties.append(sig.format_sig(indent=self._indent)) + else: + if inferred_type is None: + inferred_type = self.add_name("_typeshed.Incomplete") + + rw_properties.append(f"{self._indent}{name}: {inferred_type}") + + def get_type_fullname(self, typ: type) -> str: + """Given a type, return a string representation""" + if typ is Any: # type: ignore[comparison-overlap] + return "Any" + typename = getattr(typ, "__qualname__", typ.__name__) + module_name = self.get_obj_module(typ) + assert module_name is not None, typ + if module_name != "builtins": + typename = f"{module_name}.{typename}" + return typename + + def get_base_types(self, obj: type) -> list[str]: + all_bases = type.mro(obj) + if all_bases[-1] is object: + # TODO: Is this always object? + del all_bases[-1] + # remove pybind11_object. All classes generated by pybind11 have pybind11_object in their MRO, + # which only overrides a few functions in object type + if all_bases and all_bases[-1].__name__ == "pybind11_object": + del all_bases[-1] + # remove the class itself + all_bases = all_bases[1:] + # Remove base classes of other bases as redundant. + bases: list[type] = [] + for base in all_bases: + if not any(issubclass(b, base) for b in bases): + bases.append(base) + return [self.strip_or_import(self.get_type_fullname(base)) for base in bases] + + def generate_class_stub( + self, class_name: str, cls: type, output: list[str], parent_class: ClassInfo | None = None + ) -> None: + """Generate stub for a single class using runtime introspection. + + The result lines will be appended to 'output'. If necessary, any + required names will be added to 'imports'. + """ + raw_lookup: Mapping[str, Any] = getattr(cls, "__dict__") # noqa: B009 + items = self.get_members(cls) + if self.resort_members: + items = sorted(items, key=lambda x: method_name_sort_key(x[0])) + names = {x[0] for x in items} + methods: list[str] = [] + types: list[str] = [] + static_properties: list[str] = [] + rw_properties: list[str] = [] + ro_properties: list[str] = [] + attrs: list[tuple[str, Any]] = [] + + self.record_name(class_name) + self.indent() + + class_info = ClassInfo( + class_name, "", getattr(cls, "__doc__", None), cls, parent=parent_class + ) + + for attr, value in items: + # use unevaluated descriptors when dealing with property inspection + raw_value = raw_lookup.get(attr, value) + if self.is_method(class_info, attr, value) or self.is_classmethod( + class_info, attr, value + ): + if attr == "__new__": + # TODO: We should support __new__. + if "__init__" in names: + # Avoid duplicate functions if both are present. + # But is there any case where .__new__() has a + # better signature than __init__() ? + continue + attr = "__init__" + # FIXME: make this nicer + if self.is_staticmethod(class_info, attr, value): + class_info.self_var = "" + elif self.is_classmethod(class_info, attr, value): + class_info.self_var = "cls" + else: + class_info.self_var = "self" + self.generate_function_stub(attr, value, output=methods, class_info=class_info) + elif self.is_property(class_info, attr, raw_value): + self.generate_property_stub( + attr, + raw_value, + value, + static_properties, + rw_properties, + ro_properties, + class_info, + ) + elif inspect.isclass(value) and self.is_defined_in_module(value): + self.generate_class_stub(attr, value, types, parent_class=class_info) + else: + attrs.append((attr, value)) + + for attr, value in attrs: + if attr == "__hash__" and value is None: + # special case for __hash__ + continue + prop_type_name = self.strip_or_import(self.get_type_annotation(value)) + classvar = self.add_name("typing.ClassVar") + static_properties.append(f"{self._indent}{attr}: {classvar}[{prop_type_name}] = ...") + + self.dedent() + + bases = self.get_base_types(cls) + if bases: + bases_str = "(%s)" % ", ".join(bases) + else: + bases_str = "" + if types or static_properties or rw_properties or methods or ro_properties: + output.append(f"{self._indent}class {class_name}{bases_str}:") + for line in types: + if ( + output + and output[-1] + and not output[-1].strip().startswith("class") + and line.strip().startswith("class") + ): + output.append("") + output.append(line) + for line in static_properties: + output.append(line) + for line in rw_properties: + output.append(line) + for line in methods: + output.append(line) + for line in ro_properties: + output.append(line) + else: + output.append(f"{self._indent}class {class_name}{bases_str}: ...") + + def generate_variable_stub(self, name: str, obj: object, output: list[str]) -> None: + """Generate stub for a single variable using runtime introspection. + + The result lines will be appended to 'output'. If necessary, any + required names will be added to 'imports'. + """ + if self.is_private_name(name, f"{self.module_name}.{name}") or self.is_not_in_all(name): + return + self.record_name(name) + type_str = self.strip_or_import(self.get_type_annotation(obj)) + output.append(f"{name}: {type_str}") + + +def method_name_sort_key(name: str) -> tuple[int, str]: + """Sort methods in classes in a typical order. + + I.e.: constructor, normal methods, special methods. + """ + if name in ("__new__", "__init__"): + return 0, name + if name.startswith("__") and name.endswith("__"): + return 2, name + return 1, name + + +def is_pybind_skipped_attribute(attr: str) -> bool: + return attr.startswith("__pybind11_module_local_") + + +def infer_c_method_args( + name: str, self_var: str = "self", arg_names: list[str] | None = None +) -> list[ArgSig]: + args: list[ArgSig] | None = None + if name.startswith("__") and name.endswith("__"): + name = name[2:-2] + if name in ( + "hash", + "iter", + "next", + "sizeof", + "copy", + "deepcopy", + "reduce", + "getinitargs", + "int", + "float", + "trunc", + "complex", + "bool", + "abs", + "bytes", + "dir", + "len", + "reversed", + "round", + "index", + "enter", + ): + args = [] + elif name == "getitem": + args = [ArgSig(name="index")] + elif name == "setitem": + args = [ArgSig(name="index"), ArgSig(name="object")] + elif name in ("delattr", "getattr"): + args = [ArgSig(name="name")] + elif name == "setattr": + args = [ArgSig(name="name"), ArgSig(name="value")] + elif name == "getstate": + args = [] + elif name == "setstate": + args = [ArgSig(name="state")] + elif name in ("eq", "ne", "lt", "le", "gt", "ge"): + args = [ArgSig(name="other", type="object")] + elif name in ( + "add", + "radd", + "sub", + "rsub", + "mul", + "rmul", + "mod", + "rmod", + "floordiv", + "rfloordiv", + "truediv", + "rtruediv", + "divmod", + "rdivmod", + "pow", + "rpow", + "xor", + "rxor", + "or", + "ror", + "and", + "rand", + "lshift", + "rlshift", + "rshift", + "rrshift", + "contains", + "delitem", + "iadd", + "iand", + "ifloordiv", + "ilshift", + "imod", + "imul", + "ior", + "ipow", + "irshift", + "isub", + "itruediv", + "ixor", + ): + args = [ArgSig(name="other")] + elif name in ("neg", "pos", "invert"): + args = [] + elif name == "get": + args = [ArgSig(name="instance"), ArgSig(name="owner")] + elif name == "set": + args = [ArgSig(name="instance"), ArgSig(name="value")] + elif name == "reduce_ex": + args = [ArgSig(name="protocol")] + elif name == "exit": + args = [ + ArgSig(name="type", type="type[BaseException] | None"), + ArgSig(name="value", type="BaseException | None"), + ArgSig(name="traceback", type="types.TracebackType | None"), + ] + if args is None: + args = infer_method_arg_types(name, self_var, arg_names) + else: + args = [ArgSig(name=self_var)] + args + if args is None: + args = [ArgSig(name="*args"), ArgSig(name="**kwargs")] + return args diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py new file mode 100644 index 000000000000..8d89a2a4bede --- /dev/null +++ b/mypy/stubinfo.py @@ -0,0 +1,189 @@ +from __future__ import annotations + + +def is_module_from_legacy_bundled_package(module: str) -> bool: + top_level = module.split(".", 1)[0] + return top_level in legacy_bundled_packages + + +def stub_distribution_name(module: str) -> str | None: + top_level = module.split(".", 1)[0] + + dist = legacy_bundled_packages.get(top_level) + if dist: + return dist + dist = non_bundled_packages_flat.get(top_level) + if dist: + return dist + + if top_level in non_bundled_packages_namespace: + namespace = non_bundled_packages_namespace[top_level] + components = module.split(".") + for i in range(len(components), 0, -1): + module = ".".join(components[:i]) + dist = namespace.get(module) + if dist: + return dist + + return None + + +# Stubs for these third-party packages used to be shipped with mypy. +# +# Map package name to PyPI stub distribution name. +legacy_bundled_packages: dict[str, str] = { + "aiofiles": "types-aiofiles", + "bleach": "types-bleach", + "boto": "types-boto", + "cachetools": "types-cachetools", + "click_spinner": "types-click-spinner", + "contextvars": "types-contextvars", + "croniter": "types-croniter", + "dataclasses": "types-dataclasses", + "dateparser": "types-dateparser", + "dateutil": "types-python-dateutil", + "decorator": "types-decorator", + "deprecated": "types-Deprecated", + "docutils": "types-docutils", + "first": "types-first", + "gflags": "types-python-gflags", + "markdown": "types-Markdown", + "mock": "types-mock", + "OpenSSL": "types-pyOpenSSL", + "paramiko": "types-paramiko", + "pkg_resources": "types-setuptools", + "polib": "types-polib", + "pycurl": "types-pycurl", + "pymysql": "types-PyMySQL", + "pyrfc3339": "types-pyRFC3339", + "python2": "types-six", + "pytz": "types-pytz", + "pyVmomi": "types-pyvmomi", + "redis": "types-redis", + "requests": "types-requests", + "retry": "types-retry", + "simplejson": "types-simplejson", + "singledispatch": "types-singledispatch", + "six": "types-six", + "slugify": "types-python-slugify", + "tabulate": "types-tabulate", + "toml": "types-toml", + "typed_ast": "types-typed-ast", + "tzlocal": "types-tzlocal", + "ujson": "types-ujson", + "waitress": "types-waitress", + "yaml": "types-PyYAML", +} + +# Map package name to PyPI stub distribution name from typeshed. +# Stubs for these packages were never bundled with mypy. Don't +# include packages that have a release that includes PEP 561 type +# information. +# +# Note that these packages are omitted for now: +# pika: typeshed's stubs are on PyPI as types-pika-ts. +# types-pika already exists on PyPI, and is more complete in many ways, +# but is a non-typeshed stubs package. +non_bundled_packages_flat: dict[str, str] = { + "MySQLdb": "types-mysqlclient", + "PIL": "types-Pillow", + "PyInstaller": "types-pyinstaller", + "Xlib": "types-python-xlib", + "aws_xray_sdk": "types-aws-xray-sdk", + "babel": "types-babel", + "braintree": "types-braintree", + "bs4": "types-beautifulsoup4", + "bugbear": "types-flake8-bugbear", + "caldav": "types-caldav", + "cffi": "types-cffi", + "chevron": "types-chevron", + "colorama": "types-colorama", + "commonmark": "types-commonmark", + "consolemenu": "types-console-menu", + "crontab": "types-python-crontab", + "d3dshot": "types-D3DShot", + "dockerfile_parse": "types-dockerfile-parse", + "docopt": "types-docopt", + "editdistance": "types-editdistance", + "entrypoints": "types-entrypoints", + "farmhash": "types-pyfarmhash", + "flake8_2020": "types-flake8-2020", + "flake8_builtins": "types-flake8-builtins", + "flake8_docstrings": "types-flake8-docstrings", + "flake8_plugin_utils": "types-flake8-plugin-utils", + "flake8_rst_docstrings": "types-flake8-rst-docstrings", + "flake8_simplify": "types-flake8-simplify", + "flake8_typing_imports": "types-flake8-typing-imports", + "flask_cors": "types-Flask-Cors", + "flask_migrate": "types-Flask-Migrate", + "fpdf": "types-fpdf2", + "gdb": "types-gdb", + "hdbcli": "types-hdbcli", + "html5lib": "types-html5lib", + "httplib2": "types-httplib2", + "humanfriendly": "types-humanfriendly", + "invoke": "types-invoke", + "jack": "types-JACK-Client", + "jmespath": "types-jmespath", + "jose": "types-python-jose", + "jsonschema": "types-jsonschema", + "keyboard": "types-keyboard", + "ldap3": "types-ldap3", + "nmap": "types-python-nmap", + "oauthlib": "types-oauthlib", + "openpyxl": "types-openpyxl", + "opentracing": "types-opentracing", + "parsimonious": "types-parsimonious", + "passlib": "types-passlib", + "passpy": "types-passpy", + "peewee": "types-peewee", + "pep8ext_naming": "types-pep8-naming", + "playsound": "types-playsound", + "psutil": "types-psutil", + "psycopg2": "types-psycopg2", + "pyaudio": "types-pyaudio", + "pyautogui": "types-PyAutoGUI", + "pycocotools": "types-pycocotools", + "pyflakes": "types-pyflakes", + "pygments": "types-Pygments", + "pyi_splash": "types-pyinstaller", + "pynput": "types-pynput", + "pythoncom": "types-pywin32", + "pythonwin": "types-pywin32", + "pyscreeze": "types-PyScreeze", + "pysftp": "types-pysftp", + "pytest_lazyfixture": "types-pytest-lazy-fixture", + "pywintypes": "types-pywin32", + "regex": "types-regex", + "send2trash": "types-Send2Trash", + "slumber": "types-slumber", + "stdlib_list": "types-stdlib-list", + "stripe": "types-stripe", + "toposort": "types-toposort", + "tqdm": "types-tqdm", + "tree_sitter": "types-tree-sitter", + "tree_sitter_languages": "types-tree-sitter-languages", + "ttkthemes": "types-ttkthemes", + "vobject": "types-vobject", + "whatthepatch": "types-whatthepatch", + "win32": "types-pywin32", + "win32api": "types-pywin32", + "win32con": "types-pywin32", + "win32com": "types-pywin32", + "win32comext": "types-pywin32", + "win32gui": "types-pywin32", + "xmltodict": "types-xmltodict", + "zxcvbn": "types-zxcvbn", + # Stub packages that are not from typeshed + # Since these can be installed automatically via --install-types, we have a high trust bar + # for additions here + "pandas": "pandas-stubs", # https://github.com/pandas-dev/pandas-stubs + "lxml": "lxml-stubs", # https://github.com/lxml/lxml-stubs +} + + +non_bundled_packages_namespace: dict[str, dict[str, str]] = { + "backports": {"backports.ssl_match_hostname": "types-backports.ssl_match_hostname"}, + "google": {"google.cloud.ndb": "types-google-cloud-ndb", "google.protobuf": "types-protobuf"}, + "paho": {"paho.mqtt": "types-paho-mqtt"}, +} diff --git a/mypy/stubtest.py b/mypy/stubtest.py new file mode 100644 index 000000000000..36cd0a213d4d --- /dev/null +++ b/mypy/stubtest.py @@ -0,0 +1,2112 @@ +"""Tests for stubs. + +Verify that various things in stubs are consistent with how things behave at runtime. + +""" + +from __future__ import annotations + +import argparse +import collections.abc +import copy +import enum +import functools +import importlib +import importlib.machinery +import inspect +import os +import pkgutil +import re +import symtable +import sys +import traceback +import types +import typing +import typing_extensions +import warnings +from collections import defaultdict +from contextlib import redirect_stderr, redirect_stdout +from functools import singledispatch +from pathlib import Path +from typing import AbstractSet, Any, Generic, Iterator, TypeVar, Union +from typing_extensions import get_origin, is_typeddict + +import mypy.build +import mypy.modulefinder +import mypy.nodes +import mypy.state +import mypy.types +import mypy.version +from mypy import nodes +from mypy.config_parser import parse_config_file +from mypy.evalexpr import UNKNOWN, evaluate_expression +from mypy.options import Options +from mypy.util import FancyFormatter, bytes_to_human_readable_repr, is_dunder, plural_s + + +class Missing: + """Marker object for things that are missing (from a stub or the runtime).""" + + def __repr__(self) -> str: + return "MISSING" + + +MISSING: typing_extensions.Final = Missing() + +T = TypeVar("T") +MaybeMissing: typing_extensions.TypeAlias = Union[T, Missing] + + +class Unrepresentable: + """Marker object for unrepresentable parameter defaults.""" + + def __repr__(self) -> str: + return "" + + +UNREPRESENTABLE: typing_extensions.Final = Unrepresentable() + + +_formatter: typing_extensions.Final = FancyFormatter(sys.stdout, sys.stderr, False) + + +def _style(message: str, **kwargs: Any) -> str: + """Wrapper around mypy.util for fancy formatting.""" + kwargs.setdefault("color", "none") + return _formatter.style(message, **kwargs) + + +def _truncate(message: str, length: int) -> str: + if len(message) > length: + return message[: length - 3] + "..." + return message + + +class StubtestFailure(Exception): + pass + + +class Error: + def __init__( + self, + object_path: list[str], + message: str, + stub_object: MaybeMissing[nodes.Node], + runtime_object: MaybeMissing[Any], + *, + stub_desc: str | None = None, + runtime_desc: str | None = None, + ) -> None: + """Represents an error found by stubtest. + + :param object_path: Location of the object with the error, + e.g. ``["module", "Class", "method"]`` + :param message: Error message + :param stub_object: The mypy node representing the stub + :param runtime_object: Actual object obtained from the runtime + :param stub_desc: Specialised description for the stub object, should you wish + :param runtime_desc: Specialised description for the runtime object, should you wish + + """ + self.object_path = object_path + self.object_desc = ".".join(object_path) + self.message = message + self.stub_object = stub_object + self.runtime_object = runtime_object + self.stub_desc = stub_desc or str(getattr(stub_object, "type", stub_object)) + + if runtime_desc is None: + runtime_sig = safe_inspect_signature(runtime_object) + if runtime_sig is None: + self.runtime_desc = _truncate(repr(runtime_object), 100) + else: + runtime_is_async = inspect.iscoroutinefunction(runtime_object) + description = describe_runtime_callable(runtime_sig, is_async=runtime_is_async) + self.runtime_desc = _truncate(description, 100) + else: + self.runtime_desc = runtime_desc + + def is_missing_stub(self) -> bool: + """Whether or not the error is for something missing from the stub.""" + return isinstance(self.stub_object, Missing) + + def is_positional_only_related(self) -> bool: + """Whether or not the error is for something being (or not being) positional-only.""" + # TODO: This is hacky, use error codes or something more resilient + return "should be positional" in self.message + + def get_description(self, concise: bool = False) -> str: + """Returns a description of the error. + + :param concise: Whether to return a concise, one-line description + + """ + if concise: + return _style(self.object_desc, bold=True) + " " + self.message + + stub_line = None + stub_file = None + if not isinstance(self.stub_object, Missing): + stub_line = self.stub_object.line + stub_node = get_stub(self.object_path[0]) + if stub_node is not None: + stub_file = stub_node.path or None + + stub_loc_str = "" + if stub_file: + stub_loc_str += f" in file {Path(stub_file)}" + if stub_line: + stub_loc_str += f"{':' if stub_file else ' at line '}{stub_line}" + + runtime_line = None + runtime_file = None + if not isinstance(self.runtime_object, Missing): + try: + runtime_line = inspect.getsourcelines(self.runtime_object)[1] + except (OSError, TypeError, SyntaxError): + pass + try: + runtime_file = inspect.getsourcefile(self.runtime_object) + except TypeError: + pass + + runtime_loc_str = "" + if runtime_file: + runtime_loc_str += f" in file {Path(runtime_file)}" + if runtime_line: + runtime_loc_str += f"{':' if runtime_file else ' at line '}{runtime_line}" + + output = [ + _style("error: ", color="red", bold=True), + _style(self.object_desc, bold=True), + " ", + self.message, + "\n", + "Stub:", + _style(stub_loc_str, dim=True), + "\n", + _style(self.stub_desc + "\n", color="blue", dim=True), + "Runtime:", + _style(runtime_loc_str, dim=True), + "\n", + _style(self.runtime_desc + "\n", color="blue", dim=True), + ] + return "".join(output) + + +# ==================== +# Core logic +# ==================== + + +def silent_import_module(module_name: str) -> types.ModuleType: + with open(os.devnull, "w") as devnull: + with warnings.catch_warnings(), redirect_stdout(devnull), redirect_stderr(devnull): + warnings.simplefilter("ignore") + runtime = importlib.import_module(module_name) + # Also run the equivalent of `from module import *` + # This could have the additional effect of loading not-yet-loaded submodules + # mentioned in __all__ + __import__(module_name, fromlist=["*"]) + return runtime + + +def test_module(module_name: str) -> Iterator[Error]: + """Tests a given module's stub against introspecting it at runtime. + + Requires the stub to have been built already, accomplished by a call to ``build_stubs``. + + :param module_name: The module to test + + """ + stub = get_stub(module_name) + if stub is None: + if not is_probably_private(module_name.split(".")[-1]): + runtime_desc = repr(sys.modules[module_name]) if module_name in sys.modules else "N/A" + yield Error( + [module_name], "failed to find stubs", MISSING, None, runtime_desc=runtime_desc + ) + return + + try: + runtime = silent_import_module(module_name) + except KeyboardInterrupt: + raise + except BaseException as e: + note = "" + if isinstance(e, ModuleNotFoundError): + note = " Maybe install the runtime package or alter PYTHONPATH?" + yield Error( + [module_name], f"failed to import.{note} {type(e).__name__}: {e}", stub, MISSING + ) + return + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + yield from verify(stub, runtime, [module_name]) + except Exception as e: + bottom_frame = list(traceback.walk_tb(e.__traceback__))[-1][0] + bottom_module = bottom_frame.f_globals.get("__name__", "") + # Pass on any errors originating from stubtest or mypy + # These can occur expectedly, e.g. StubtestFailure + if bottom_module == "__main__" or bottom_module.split(".")[0] == "mypy": + raise + yield Error( + [module_name], + f"encountered unexpected error, {type(e).__name__}: {e}", + stub, + runtime, + stub_desc="N/A", + runtime_desc=( + "This is most likely the fault of something very dynamic in your library. " + "It's also possible this is a bug in stubtest.\nIf in doubt, please " + "open an issue at https://github.com/python/mypy\n\n" + + traceback.format_exc().strip() + ), + ) + + +@singledispatch +def verify( + stub: MaybeMissing[nodes.Node], runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + """Entry point for comparing a stub to a runtime object. + + We use single dispatch based on the type of ``stub``. + + :param stub: The mypy node representing a part of the stub + :param runtime: The runtime object corresponding to ``stub`` + + """ + yield Error(object_path, "is an unknown mypy node", stub, runtime) + + +def _verify_exported_names( + object_path: list[str], stub: nodes.MypyFile, runtime_all_as_set: set[str] +) -> Iterator[Error]: + # note that this includes the case the stub simply defines `__all__: list[str]` + assert "__all__" in stub.names + public_names_in_stub = {m for m, o in stub.names.items() if o.module_public} + names_in_stub_not_runtime = sorted(public_names_in_stub - runtime_all_as_set) + names_in_runtime_not_stub = sorted(runtime_all_as_set - public_names_in_stub) + if not (names_in_runtime_not_stub or names_in_stub_not_runtime): + return + yield Error( + object_path + ["__all__"], + ( + "names exported from the stub do not correspond to the names exported at runtime. " + "This is probably due to things being missing from the stub or an inaccurate `__all__` in the stub" + ), + # Pass in MISSING instead of the stub and runtime objects, as the line numbers aren't very + # relevant here, and it makes for a prettier error message + # This means this error will be ignored when using `--ignore-missing-stub`, which is + # desirable in at least the `names_in_runtime_not_stub` case + stub_object=MISSING, + runtime_object=MISSING, + stub_desc=(f"Names exported in the stub but not at runtime: {names_in_stub_not_runtime}"), + runtime_desc=( + f"Names exported at runtime but not in the stub: {names_in_runtime_not_stub}" + ), + ) + + +@functools.lru_cache +def _module_symbol_table(runtime: types.ModuleType) -> symtable.SymbolTable | None: + """Retrieve the symbol table for the module (or None on failure). + + 1) Use inspect to retrieve the source code of the module + 2) Use symtable to parse the source (and use what symtable knows for its purposes) + """ + try: + source = inspect.getsource(runtime) + except (OSError, TypeError, SyntaxError): + return None + + try: + return symtable.symtable(source, runtime.__name__, "exec") + except SyntaxError: + return None + + +@verify.register(nodes.MypyFile) +def verify_mypyfile( + stub: nodes.MypyFile, runtime: MaybeMissing[types.ModuleType], object_path: list[str] +) -> Iterator[Error]: + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime) + return + if not isinstance(runtime, types.ModuleType): + yield Error(object_path, "is not a module", stub, runtime) + return + + runtime_all_as_set: set[str] | None + + if hasattr(runtime, "__all__"): + runtime_all_as_set = set(runtime.__all__) + if "__all__" in stub.names: + # Only verify the contents of the stub's __all__ + # if the stub actually defines __all__ + yield from _verify_exported_names(object_path, stub, runtime_all_as_set) + else: + yield Error(object_path + ["__all__"], "is not present in stub", MISSING, runtime) + else: + runtime_all_as_set = None + + # Check things in the stub + to_check = { + m + for m, o in stub.names.items() + if not o.module_hidden and (not is_probably_private(m) or hasattr(runtime, m)) + } + + def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: + """Heuristics to determine whether a name originates from another module.""" + obj = getattr(r, attr) + if isinstance(obj, types.ModuleType): + return False + + symbol_table = _module_symbol_table(r) + if symbol_table is not None: + try: + symbol = symbol_table.lookup(attr) + except KeyError: + pass + else: + if symbol.is_imported(): + # symtable says we got this from another module + return False + # But we can't just return True here, because symtable doesn't know about symbols + # that come from `from module import *` + if symbol.is_assigned(): + # symtable knows we assigned this symbol in the module + return True + + # The __module__ attribute is unreliable for anything except functions and classes, + # but it's our best guess at this point + try: + obj_mod = obj.__module__ + except Exception: + pass + else: + if isinstance(obj_mod, str): + return bool(obj_mod == r.__name__) + return True + + runtime_public_contents = ( + runtime_all_as_set + if runtime_all_as_set is not None + else { + m + for m in dir(runtime) + if not is_probably_private(m) + # Filter out objects that originate from other modules (best effort). Note that in the + # absence of __all__, we don't have a way to detect explicit / intentional re-exports + # at runtime + and _belongs_to_runtime(runtime, m) + } + ) + # Check all things declared in module's __all__, falling back to our best guess + to_check.update(runtime_public_contents) + to_check.difference_update(IGNORED_MODULE_DUNDERS) + + for entry in sorted(to_check): + stub_entry = stub.names[entry].node if entry in stub.names else MISSING + if isinstance(stub_entry, nodes.MypyFile): + # Don't recursively check exported modules, since that leads to infinite recursion + continue + assert stub_entry is not None + try: + runtime_entry = getattr(runtime, entry, MISSING) + except Exception: + # Catch all exceptions in case the runtime raises an unexpected exception + # from __getattr__ or similar. + continue + yield from verify(stub_entry, runtime_entry, object_path + [entry]) + + +def _verify_final( + stub: nodes.TypeInfo, runtime: type[Any], object_path: list[str] +) -> Iterator[Error]: + try: + + class SubClass(runtime): # type: ignore[misc] + pass + + except TypeError: + # Enum classes are implicitly @final + if not stub.is_final and not issubclass(runtime, enum.Enum): + yield Error( + object_path, + "cannot be subclassed at runtime, but isn't marked with @final in the stub", + stub, + runtime, + stub_desc=repr(stub), + ) + except Exception: + # The class probably wants its subclasses to do something special. + # Examples: ctypes.Array, ctypes._SimpleCData + pass + + # Runtime class might be annotated with `@final`: + try: + runtime_final = getattr(runtime, "__final__", False) + except Exception: + runtime_final = False + + if runtime_final and not stub.is_final: + yield Error( + object_path, + "has `__final__` attribute, but isn't marked with @final in the stub", + stub, + runtime, + stub_desc=repr(stub), + ) + + +def _verify_metaclass( + stub: nodes.TypeInfo, runtime: type[Any], object_path: list[str], *, is_runtime_typeddict: bool +) -> Iterator[Error]: + # We exclude protocols, because of how complex their implementation is in different versions of + # python. Enums are also hard, as are runtime TypedDicts; ignoring. + # TODO: check that metaclasses are identical? + if not stub.is_protocol and not stub.is_enum and not is_runtime_typeddict: + runtime_metaclass = type(runtime) + if runtime_metaclass is not type and stub.metaclass_type is None: + # This means that runtime has a custom metaclass, but a stub does not. + yield Error( + object_path, + "is inconsistent, metaclass differs", + stub, + runtime, + stub_desc="N/A", + runtime_desc=f"{runtime_metaclass}", + ) + elif ( + runtime_metaclass is type + and stub.metaclass_type is not None + # We ignore extra `ABCMeta` metaclass on stubs, this might be typing hack. + # We also ignore `builtins.type` metaclass as an implementation detail in mypy. + and not mypy.types.is_named_instance( + stub.metaclass_type, ("abc.ABCMeta", "builtins.type") + ) + ): + # This means that our stub has a metaclass that is not present at runtime. + yield Error( + object_path, + "metaclass mismatch", + stub, + runtime, + stub_desc=f"{stub.metaclass_type.type.fullname}", + runtime_desc="N/A", + ) + + +@verify.register(nodes.TypeInfo) +def verify_typeinfo( + stub: nodes.TypeInfo, runtime: MaybeMissing[type[Any]], object_path: list[str] +) -> Iterator[Error]: + if stub.is_type_check_only: + # This type only exists in stubs, we only check that the runtime part + # is missing. Other checks are not required. + if not isinstance(runtime, Missing): + yield Error( + object_path, + 'is marked as "@type_check_only", but also exists at runtime', + stub, + runtime, + stub_desc=repr(stub), + ) + return + + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime, stub_desc=repr(stub)) + return + if not isinstance(runtime, type): + yield Error(object_path, "is not a type", stub, runtime, stub_desc=repr(stub)) + return + + yield from _verify_final(stub, runtime, object_path) + is_runtime_typeddict = stub.typeddict_type is not None and is_typeddict(runtime) + yield from _verify_metaclass( + stub, runtime, object_path, is_runtime_typeddict=is_runtime_typeddict + ) + + # Check everything already defined on the stub class itself (i.e. not inherited) + # + # Filter out non-identifier names, as these are (hopefully always?) whacky/fictional things + # (like __mypy-replace or __mypy-post_init, etc.) that don't exist at runtime, + # and exist purely for internal mypy reasons + to_check = {name for name in stub.names if name.isidentifier()} + # Check all public things on the runtime class + to_check.update( + m for m in vars(runtime) if not is_probably_private(m) and m not in IGNORABLE_CLASS_DUNDERS + ) + # Special-case the __init__ method for Protocols and the __new__ method for TypedDicts + # + # TODO: On Python <3.11, __init__ methods on Protocol classes + # are silently discarded and replaced. + # However, this is not the case on Python 3.11+. + # Ideally, we'd figure out a good way of validating Protocol __init__ methods on 3.11+. + if stub.is_protocol: + to_check.discard("__init__") + if is_runtime_typeddict: + to_check.discard("__new__") + + for entry in sorted(to_check): + mangled_entry = entry + if entry.startswith("__") and not entry.endswith("__"): + mangled_entry = f"_{stub.name.lstrip('_')}{entry}" + stub_to_verify = next((t.names[entry].node for t in stub.mro if entry in t.names), MISSING) + assert stub_to_verify is not None + try: + try: + runtime_attr = getattr(runtime, mangled_entry) + except AttributeError: + runtime_attr = inspect.getattr_static(runtime, mangled_entry, MISSING) + except Exception: + # Catch all exceptions in case the runtime raises an unexpected exception + # from __getattr__ or similar. + continue + # Do not error for an object missing from the stub + # If the runtime object is a types.WrapperDescriptorType object + # and has a non-special dunder name. + # The vast majority of these are false positives. + if not ( + isinstance(stub_to_verify, Missing) + and isinstance(runtime_attr, types.WrapperDescriptorType) + and is_dunder(mangled_entry, exclude_special=True) + ): + yield from verify(stub_to_verify, runtime_attr, object_path + [entry]) + + +def _static_lookup_runtime(object_path: list[str]) -> MaybeMissing[Any]: + static_runtime = importlib.import_module(object_path[0]) + for entry in object_path[1:]: + try: + static_runtime = inspect.getattr_static(static_runtime, entry) + except AttributeError: + # This can happen with mangled names, ignore for now. + # TODO: pass more information about ancestors of nodes/objects to verify, so we don't + # have to do this hacky lookup. Would be useful in several places. + return MISSING + return static_runtime + + +def _verify_static_class_methods( + stub: nodes.FuncBase, runtime: Any, static_runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[str]: + if stub.name in ("__new__", "__init_subclass__", "__class_getitem__"): + # Special cased by Python, so don't bother checking + return + if inspect.isbuiltin(runtime): + # The isinstance checks don't work reliably for builtins, e.g. datetime.datetime.now, so do + # something a little hacky that seems to work well + probably_class_method = isinstance(getattr(runtime, "__self__", None), type) + if probably_class_method and not stub.is_class: + yield "runtime is a classmethod but stub is not" + if not probably_class_method and stub.is_class: + yield "stub is a classmethod but runtime is not" + return + + if static_runtime is MISSING: + return + + if isinstance(static_runtime, classmethod) and not stub.is_class: + yield "runtime is a classmethod but stub is not" + if not isinstance(static_runtime, classmethod) and stub.is_class: + yield "stub is a classmethod but runtime is not" + if isinstance(static_runtime, staticmethod) and not stub.is_static: + yield "runtime is a staticmethod but stub is not" + if not isinstance(static_runtime, staticmethod) and stub.is_static: + yield "stub is a staticmethod but runtime is not" + + +def _verify_arg_name( + stub_arg: nodes.Argument, runtime_arg: inspect.Parameter, function_name: str +) -> Iterator[str]: + """Checks whether argument names match.""" + # Ignore exact names for most dunder methods + if is_dunder(function_name, exclude_special=True): + return + + def strip_prefix(s: str, prefix: str) -> str: + return s[len(prefix) :] if s.startswith(prefix) else s + + if strip_prefix(stub_arg.variable.name, "__") == runtime_arg.name: + return + + nonspecific_names = {"object", "args"} + if runtime_arg.name in nonspecific_names: + return + + def names_approx_match(a: str, b: str) -> bool: + a = a.strip("_") + b = b.strip("_") + return a.startswith(b) or b.startswith(a) or len(a) == 1 or len(b) == 1 + + # Be more permissive about names matching for positional-only arguments + if runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY and names_approx_match( + stub_arg.variable.name, runtime_arg.name + ): + return + # This comes up with namedtuples, so ignore + if stub_arg.variable.name == "_self": + return + yield ( + f'stub argument "{stub_arg.variable.name}" ' + f'differs from runtime argument "{runtime_arg.name}"' + ) + + +def _verify_arg_default_value( + stub_arg: nodes.Argument, runtime_arg: inspect.Parameter +) -> Iterator[str]: + """Checks whether argument default values are compatible.""" + if runtime_arg.default != inspect.Parameter.empty: + if stub_arg.kind.is_required(): + yield ( + f'runtime argument "{runtime_arg.name}" ' + "has a default value but stub argument does not" + ) + else: + runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default) + # Fallback to the type annotation type if var type is missing. The type annotation + # is an UnboundType, but I don't know enough to know what the pros and cons here are. + # UnboundTypes have ugly question marks following them, so default to var type. + # Note we do this same fallback when constructing signatures in from_overloadedfuncdef + stub_type = stub_arg.variable.type or stub_arg.type_annotation + if isinstance(stub_type, mypy.types.TypeVarType): + stub_type = stub_type.upper_bound + if ( + runtime_type is not None + and stub_type is not None + # Avoid false positives for marker objects + and type(runtime_arg.default) is not object + # And ellipsis + and runtime_arg.default is not ... + and not is_subtype_helper(runtime_type, stub_type) + ): + yield ( + f'runtime argument "{runtime_arg.name}" ' + f"has a default value of type {runtime_type}, " + f"which is incompatible with stub argument type {stub_type}" + ) + if stub_arg.initializer is not None: + stub_default = evaluate_expression(stub_arg.initializer) + if ( + stub_default is not UNKNOWN + and stub_default is not ... + and runtime_arg.default is not UNREPRESENTABLE + and ( + stub_default != runtime_arg.default + # We want the types to match exactly, e.g. in case the stub has + # True and the runtime has 1 (or vice versa). + or type(stub_default) is not type(runtime_arg.default) + ) + ): + yield ( + f'runtime argument "{runtime_arg.name}" ' + f"has a default value of {runtime_arg.default!r}, " + f"which is different from stub argument default {stub_default!r}" + ) + else: + if stub_arg.kind.is_optional(): + yield ( + f'stub argument "{stub_arg.variable.name}" has a default value ' + f"but runtime argument does not" + ) + + +def maybe_strip_cls(name: str, args: list[nodes.Argument]) -> list[nodes.Argument]: + if args and name in ("__init_subclass__", "__class_getitem__"): + # These are implicitly classmethods. If the stub chooses not to have @classmethod, we + # should remove the cls argument + if args[0].variable.name == "cls": + return args[1:] + return args + + +class Signature(Generic[T]): + def __init__(self) -> None: + self.pos: list[T] = [] + self.kwonly: dict[str, T] = {} + self.varpos: T | None = None + self.varkw: T | None = None + + def __str__(self) -> str: + def get_name(arg: Any) -> str: + if isinstance(arg, inspect.Parameter): + return arg.name + if isinstance(arg, nodes.Argument): + return arg.variable.name + raise AssertionError + + def get_type(arg: Any) -> str | None: + if isinstance(arg, inspect.Parameter): + return None + if isinstance(arg, nodes.Argument): + return str(arg.variable.type or arg.type_annotation) + raise AssertionError + + def has_default(arg: Any) -> bool: + if isinstance(arg, inspect.Parameter): + return bool(arg.default != inspect.Parameter.empty) + if isinstance(arg, nodes.Argument): + return arg.kind.is_optional() + raise AssertionError + + def get_desc(arg: Any) -> str: + arg_type = get_type(arg) + return ( + get_name(arg) + + (f": {arg_type}" if arg_type else "") + + (" = ..." if has_default(arg) else "") + ) + + kw_only = sorted(self.kwonly.values(), key=lambda a: (has_default(a), get_name(a))) + ret = "def (" + ret += ", ".join( + [get_desc(arg) for arg in self.pos] + + (["*" + get_name(self.varpos)] if self.varpos else (["*"] if self.kwonly else [])) + + [get_desc(arg) for arg in kw_only] + + (["**" + get_name(self.varkw)] if self.varkw else []) + ) + ret += ")" + return ret + + @staticmethod + def from_funcitem(stub: nodes.FuncItem) -> Signature[nodes.Argument]: + stub_sig: Signature[nodes.Argument] = Signature() + stub_args = maybe_strip_cls(stub.name, stub.arguments) + for stub_arg in stub_args: + if stub_arg.kind.is_positional(): + stub_sig.pos.append(stub_arg) + elif stub_arg.kind.is_named(): + stub_sig.kwonly[stub_arg.variable.name] = stub_arg + elif stub_arg.kind == nodes.ARG_STAR: + stub_sig.varpos = stub_arg + elif stub_arg.kind == nodes.ARG_STAR2: + stub_sig.varkw = stub_arg + else: + raise AssertionError + return stub_sig + + @staticmethod + def from_inspect_signature(signature: inspect.Signature) -> Signature[inspect.Parameter]: + runtime_sig: Signature[inspect.Parameter] = Signature() + for runtime_arg in signature.parameters.values(): + if runtime_arg.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ): + runtime_sig.pos.append(runtime_arg) + elif runtime_arg.kind == inspect.Parameter.KEYWORD_ONLY: + runtime_sig.kwonly[runtime_arg.name] = runtime_arg + elif runtime_arg.kind == inspect.Parameter.VAR_POSITIONAL: + runtime_sig.varpos = runtime_arg + elif runtime_arg.kind == inspect.Parameter.VAR_KEYWORD: + runtime_sig.varkw = runtime_arg + else: + raise AssertionError + return runtime_sig + + @staticmethod + def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Argument]: + """Returns a Signature from an OverloadedFuncDef. + + If life were simple, to verify_overloadedfuncdef, we'd just verify_funcitem for each of its + items. Unfortunately, life isn't simple and overloads are pretty deceitful. So instead, we + try and combine the overload's items into a single signature that is compatible with any + lies it might try to tell. + + """ + # For most dunder methods, just assume all args are positional-only + assume_positional_only = is_dunder(stub.name, exclude_special=True) + + all_args: dict[str, list[tuple[nodes.Argument, int]]] = {} + for func in map(_resolve_funcitem_from_decorator, stub.items): + assert func is not None + args = maybe_strip_cls(stub.name, func.arguments) + for index, arg in enumerate(args): + # For positional-only args, we allow overloads to have different names for the same + # argument. To accomplish this, we just make up a fake index-based name. + name = ( + f"__{index}" + if arg.variable.name.startswith("__") + or arg.pos_only + or assume_positional_only + or arg.variable.name.strip("_") == "self" + else arg.variable.name + ) + all_args.setdefault(name, []).append((arg, index)) + + def get_position(arg_name: str) -> int: + # We just need this to return the positional args in the correct order. + return max(index for _, index in all_args[arg_name]) + + def get_type(arg_name: str) -> mypy.types.ProperType: + with mypy.state.state.strict_optional_set(True): + all_types = [ + arg.variable.type or arg.type_annotation for arg, _ in all_args[arg_name] + ] + return mypy.typeops.make_simplified_union([t for t in all_types if t]) + + def get_kind(arg_name: str) -> nodes.ArgKind: + kinds = {arg.kind for arg, _ in all_args[arg_name]} + if nodes.ARG_STAR in kinds: + return nodes.ARG_STAR + if nodes.ARG_STAR2 in kinds: + return nodes.ARG_STAR2 + # The logic here is based on two tenets: + # 1) If an arg is ever optional (or unspecified), it is optional + # 2) If an arg is ever positional, it is positional + is_opt = ( + len(all_args[arg_name]) < len(stub.items) + or nodes.ARG_OPT in kinds + or nodes.ARG_NAMED_OPT in kinds + ) + is_pos = nodes.ARG_OPT in kinds or nodes.ARG_POS in kinds + if is_opt: + return nodes.ARG_OPT if is_pos else nodes.ARG_NAMED_OPT + return nodes.ARG_POS if is_pos else nodes.ARG_NAMED + + sig: Signature[nodes.Argument] = Signature() + for arg_name in sorted(all_args, key=get_position): + # example_arg_name gives us a real name (in case we had a fake index-based name) + example_arg_name = all_args[arg_name][0][0].variable.name + arg = nodes.Argument( + nodes.Var(example_arg_name, get_type(arg_name)), + type_annotation=None, + initializer=None, + kind=get_kind(arg_name), + pos_only=all(arg.pos_only for arg, _ in all_args[arg_name]), + ) + if arg.kind.is_positional(): + sig.pos.append(arg) + elif arg.kind.is_named(): + sig.kwonly[arg.variable.name] = arg + elif arg.kind == nodes.ARG_STAR: + sig.varpos = arg + elif arg.kind == nodes.ARG_STAR2: + sig.varkw = arg + else: + raise AssertionError + return sig + + +def _verify_signature( + stub: Signature[nodes.Argument], runtime: Signature[inspect.Parameter], function_name: str +) -> Iterator[str]: + # Check positional arguments match up + for stub_arg, runtime_arg in zip(stub.pos, runtime.pos): + yield from _verify_arg_name(stub_arg, runtime_arg, function_name) + yield from _verify_arg_default_value(stub_arg, runtime_arg) + if ( + runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY + and not stub_arg.pos_only + and not stub_arg.variable.name.startswith("__") + and stub_arg.variable.name.strip("_") != "self" + and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods + ): + yield ( + f'stub argument "{stub_arg.variable.name}" should be positional-only ' + f'(add "/", e.g. "{runtime_arg.name}, /")' + ) + if ( + runtime_arg.kind != inspect.Parameter.POSITIONAL_ONLY + and (stub_arg.pos_only or stub_arg.variable.name.startswith("__")) + and stub_arg.variable.name.strip("_") != "self" + and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods + ): + yield ( + f'stub argument "{stub_arg.variable.name}" should be positional or keyword ' + '(remove "/")' + ) + + # Check unmatched positional args + if len(stub.pos) > len(runtime.pos): + # There are cases where the stub exhaustively lists out the extra parameters the function + # would take through *args. Hence, a) if runtime accepts *args, we don't check whether the + # runtime has all of the stub's parameters, b) below, we don't enforce that the stub takes + # *args, since runtime logic may prevent arbitrary arguments from actually being accepted. + if runtime.varpos is None: + for stub_arg in stub.pos[len(runtime.pos) :]: + # If the variable is in runtime.kwonly, it's just mislabelled as not a + # keyword-only argument + if stub_arg.variable.name not in runtime.kwonly: + msg = f'runtime does not have argument "{stub_arg.variable.name}"' + if runtime.varkw is not None: + msg += ". Maybe you forgot to make it keyword-only in the stub?" + yield msg + else: + yield f'stub argument "{stub_arg.variable.name}" is not keyword-only' + if stub.varpos is not None: + yield f'runtime does not have *args argument "{stub.varpos.variable.name}"' + elif len(stub.pos) < len(runtime.pos): + for runtime_arg in runtime.pos[len(stub.pos) :]: + if runtime_arg.name not in stub.kwonly: + if not _is_private_parameter(runtime_arg): + yield f'stub does not have argument "{runtime_arg.name}"' + else: + yield f'runtime argument "{runtime_arg.name}" is not keyword-only' + + # Checks involving *args + if len(stub.pos) <= len(runtime.pos) or runtime.varpos is None: + if stub.varpos is None and runtime.varpos is not None: + yield f'stub does not have *args argument "{runtime.varpos.name}"' + if stub.varpos is not None and runtime.varpos is None: + yield f'runtime does not have *args argument "{stub.varpos.variable.name}"' + + # Check keyword-only args + for arg in sorted(set(stub.kwonly) & set(runtime.kwonly)): + stub_arg, runtime_arg = stub.kwonly[arg], runtime.kwonly[arg] + yield from _verify_arg_name(stub_arg, runtime_arg, function_name) + yield from _verify_arg_default_value(stub_arg, runtime_arg) + + # Check unmatched keyword-only args + if runtime.varkw is None or not set(runtime.kwonly).issubset(set(stub.kwonly)): + # There are cases where the stub exhaustively lists out the extra parameters the function + # would take through **kwargs. Hence, a) if runtime accepts **kwargs (and the stub hasn't + # exhaustively listed out params), we don't check whether the runtime has all of the stub's + # parameters, b) below, we don't enforce that the stub takes **kwargs, since runtime logic + # may prevent arbitrary keyword arguments from actually being accepted. + for arg in sorted(set(stub.kwonly) - set(runtime.kwonly)): + if arg in {runtime_arg.name for runtime_arg in runtime.pos}: + # Don't report this if we've reported it before + if arg not in {runtime_arg.name for runtime_arg in runtime.pos[len(stub.pos) :]}: + yield f'runtime argument "{arg}" is not keyword-only' + else: + yield f'runtime does not have argument "{arg}"' + for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)): + if arg in {stub_arg.variable.name for stub_arg in stub.pos}: + # Don't report this if we've reported it before + if not ( + runtime.varpos is None + and arg in {stub_arg.variable.name for stub_arg in stub.pos[len(runtime.pos) :]} + ): + yield f'stub argument "{arg}" is not keyword-only' + else: + if not _is_private_parameter(runtime.kwonly[arg]): + yield f'stub does not have argument "{arg}"' + + # Checks involving **kwargs + if stub.varkw is None and runtime.varkw is not None: + # As mentioned above, don't enforce that the stub takes **kwargs. + # Also check against positional parameters, to avoid a nitpicky message when an argument + # isn't marked as keyword-only + stub_pos_names = {stub_arg.variable.name for stub_arg in stub.pos} + # Ideally we'd do a strict subset check, but in practice the errors from that aren't useful + if not set(runtime.kwonly).issubset(set(stub.kwonly) | stub_pos_names): + yield f'stub does not have **kwargs argument "{runtime.varkw.name}"' + if stub.varkw is not None and runtime.varkw is None: + yield f'runtime does not have **kwargs argument "{stub.varkw.variable.name}"' + + +def _is_private_parameter(arg: inspect.Parameter) -> bool: + return ( + arg.name.startswith("_") + and not arg.name.startswith("__") + and arg.default is not inspect.Parameter.empty + ) + + +@verify.register(nodes.FuncItem) +def verify_funcitem( + stub: nodes.FuncItem, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime) + return + + if not is_probably_a_function(runtime): + yield Error(object_path, "is not a function", stub, runtime) + if not callable(runtime): + return + + # Look the object up statically, to avoid binding by the descriptor protocol + static_runtime = _static_lookup_runtime(object_path) + + if isinstance(stub, nodes.FuncDef): + for error_text in _verify_abstract_status(stub, runtime): + yield Error(object_path, error_text, stub, runtime) + for error_text in _verify_final_method(stub, runtime, static_runtime): + yield Error(object_path, error_text, stub, runtime) + + for message in _verify_static_class_methods(stub, runtime, static_runtime, object_path): + yield Error(object_path, "is inconsistent, " + message, stub, runtime) + + signature = safe_inspect_signature(runtime) + runtime_is_coroutine = inspect.iscoroutinefunction(runtime) + + if signature: + stub_sig = Signature.from_funcitem(stub) + runtime_sig = Signature.from_inspect_signature(signature) + runtime_sig_desc = describe_runtime_callable(signature, is_async=runtime_is_coroutine) + stub_desc = str(stub_sig) + else: + runtime_sig_desc, stub_desc = None, None + + # Don't raise an error if the stub is a coroutine, but the runtime isn't. + # That results in false positives. + # See https://github.com/python/typeshed/issues/7344 + if runtime_is_coroutine and not stub.is_coroutine: + yield Error( + object_path, + 'is an "async def" function at runtime, but not in the stub', + stub, + runtime, + stub_desc=stub_desc, + runtime_desc=runtime_sig_desc, + ) + + if not signature: + return + + for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name): + yield Error( + object_path, + "is inconsistent, " + message, + stub, + runtime, + runtime_desc=runtime_sig_desc, + ) + + +@verify.register(Missing) +def verify_none( + stub: Missing, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + yield Error(object_path, "is not present in stub", stub, runtime) + + +@verify.register(nodes.Var) +def verify_var( + stub: nodes.Var, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + if isinstance(runtime, Missing): + # Don't always yield an error here, because we often can't find instance variables + if len(object_path) <= 2: + yield Error(object_path, "is not present at runtime", stub, runtime) + return + + if ( + stub.is_initialized_in_class + and is_read_only_property(runtime) + and (stub.is_settable_property or not stub.is_property) + ): + yield Error(object_path, "is read-only at runtime but not in the stub", stub, runtime) + + runtime_type = get_mypy_type_of_runtime_value(runtime) + if ( + runtime_type is not None + and stub.type is not None + and not is_subtype_helper(runtime_type, stub.type) + ): + should_error = True + # Avoid errors when defining enums, since runtime_type is the enum itself, but we'd + # annotate it with the type of runtime.value + if isinstance(runtime, enum.Enum): + runtime_type = get_mypy_type_of_runtime_value(runtime.value) + if runtime_type is not None and is_subtype_helper(runtime_type, stub.type): + should_error = False + # We always allow setting the stub value to ... + proper_type = mypy.types.get_proper_type(stub.type) + if ( + isinstance(proper_type, mypy.types.Instance) + and proper_type.type.fullname == "builtins.ellipsis" + ): + should_error = False + + if should_error: + yield Error( + object_path, f"variable differs from runtime type {runtime_type}", stub, runtime + ) + + +@verify.register(nodes.OverloadedFuncDef) +def verify_overloadedfuncdef( + stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + # TODO: support `@type_check_only` decorator + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime) + return + + if stub.is_property: + # Any property with a setter is represented as an OverloadedFuncDef + if is_read_only_property(runtime): + yield Error(object_path, "is read-only at runtime but not in the stub", stub, runtime) + return + + if not is_probably_a_function(runtime): + yield Error(object_path, "is not a function", stub, runtime) + if not callable(runtime): + return + + # mypy doesn't allow overloads where one overload is abstract but another isn't, + # so it should be okay to just check whether the first overload is abstract or not. + # + # TODO: Mypy *does* allow properties where e.g. the getter is abstract but the setter is not; + # and any property with a setter is represented as an OverloadedFuncDef internally; + # not sure exactly what (if anything) we should do about that. + first_part = stub.items[0] + if isinstance(first_part, nodes.Decorator) and first_part.is_overload: + for msg in _verify_abstract_status(first_part.func, runtime): + yield Error(object_path, msg, stub, runtime) + + # Look the object up statically, to avoid binding by the descriptor protocol + static_runtime = _static_lookup_runtime(object_path) + + for message in _verify_static_class_methods(stub, runtime, static_runtime, object_path): + yield Error(object_path, "is inconsistent, " + message, stub, runtime) + + # TODO: Should call _verify_final_method here, + # but overloaded final methods in stubs cause a stubtest crash: see #14950 + + signature = safe_inspect_signature(runtime) + if not signature: + return + + stub_sig = Signature.from_overloadedfuncdef(stub) + runtime_sig = Signature.from_inspect_signature(signature) + + for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name): + # TODO: This is a little hacky, but the addition here is super useful + if "has a default value of type" in message: + message += ( + ". This is often caused by overloads failing to account for explicitly passing " + "in the default value." + ) + yield Error( + object_path, + "is inconsistent, " + message, + stub, + runtime, + stub_desc=(str(stub.type)) + f"\nInferred signature: {stub_sig}", + runtime_desc="def " + str(signature), + ) + + +@verify.register(nodes.TypeVarExpr) +def verify_typevarexpr( + stub: nodes.TypeVarExpr, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + if isinstance(runtime, Missing): + # We seem to insert these typevars into NamedTuple stubs, but they + # don't exist at runtime. Just ignore! + if stub.name == "_NT": + return + yield Error(object_path, "is not present at runtime", stub, runtime) + return + if not isinstance(runtime, TypeVar): + yield Error(object_path, "is not a TypeVar", stub, runtime) + return + + +@verify.register(nodes.ParamSpecExpr) +def verify_paramspecexpr( + stub: nodes.ParamSpecExpr, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime) + return + maybe_paramspec_types = ( + getattr(typing, "ParamSpec", None), + getattr(typing_extensions, "ParamSpec", None), + ) + paramspec_types = tuple(t for t in maybe_paramspec_types if t is not None) + if not paramspec_types or not isinstance(runtime, paramspec_types): + yield Error(object_path, "is not a ParamSpec", stub, runtime) + return + + +def _verify_readonly_property(stub: nodes.Decorator, runtime: Any) -> Iterator[str]: + assert stub.func.is_property + if isinstance(runtime, property): + yield from _verify_final_method(stub.func, runtime.fget, MISSING) + return + if isinstance(runtime, functools.cached_property): + yield from _verify_final_method(stub.func, runtime.func, MISSING) + return + if inspect.isdatadescriptor(runtime): + # It's enough like a property... + return + # Sometimes attributes pretend to be properties, for instance, to express that they + # are read only. So allowlist if runtime_type matches the return type of stub. + runtime_type = get_mypy_type_of_runtime_value(runtime) + func_type = ( + stub.func.type.ret_type if isinstance(stub.func.type, mypy.types.CallableType) else None + ) + if ( + runtime_type is not None + and func_type is not None + and is_subtype_helper(runtime_type, func_type) + ): + return + yield "is inconsistent, cannot reconcile @property on stub with runtime object" + + +def _verify_abstract_status(stub: nodes.FuncDef, runtime: Any) -> Iterator[str]: + stub_abstract = stub.abstract_status == nodes.IS_ABSTRACT + runtime_abstract = getattr(runtime, "__isabstractmethod__", False) + # The opposite can exist: some implementations omit `@abstractmethod` decorators + if runtime_abstract and not stub_abstract: + item_type = "property" if stub.is_property else "method" + yield f"is inconsistent, runtime {item_type} is abstract but stub is not" + + +def _verify_final_method( + stub: nodes.FuncDef, runtime: Any, static_runtime: MaybeMissing[Any] +) -> Iterator[str]: + if stub.is_final: + return + if getattr(runtime, "__final__", False) or ( + static_runtime is not MISSING and getattr(static_runtime, "__final__", False) + ): + yield "is decorated with @final at runtime, but not in the stub" + + +def _resolve_funcitem_from_decorator(dec: nodes.OverloadPart) -> nodes.FuncItem | None: + """Returns a FuncItem that corresponds to the output of the decorator. + + Returns None if we can't figure out what that would be. For convenience, this function also + accepts FuncItems. + """ + if isinstance(dec, nodes.FuncItem): + return dec + if dec.func.is_property: + return None + + def apply_decorator_to_funcitem( + decorator: nodes.Expression, func: nodes.FuncItem + ) -> nodes.FuncItem | None: + if ( + isinstance(decorator, nodes.CallExpr) + and isinstance(decorator.callee, nodes.RefExpr) + and decorator.callee.fullname in mypy.types.DEPRECATED_TYPE_NAMES + ): + return func + if not isinstance(decorator, nodes.RefExpr): + return None + if not decorator.fullname: + # Happens with namedtuple + return None + if ( + decorator.fullname in ("builtins.staticmethod", "abc.abstractmethod") + or decorator.fullname in mypy.types.OVERLOAD_NAMES + or decorator.fullname in mypy.types.FINAL_DECORATOR_NAMES + ): + return func + if decorator.fullname == "builtins.classmethod": + if func.arguments[0].variable.name not in ("cls", "mcs", "metacls"): + raise StubtestFailure( + f"unexpected class argument name {func.arguments[0].variable.name!r} " + f"in {dec.fullname}" + ) + # FuncItem is written so that copy.copy() actually works, even when compiled + ret = copy.copy(func) + # Remove the cls argument, since it's not present in inspect.signature of classmethods + ret.arguments = ret.arguments[1:] + return ret + # Just give up on any other decorators. After excluding properties, we don't run into + # anything else when running on typeshed's stdlib. + return None + + func: nodes.FuncItem = dec.func + for decorator in dec.original_decorators: + resulting_func = apply_decorator_to_funcitem(decorator, func) + if resulting_func is None: + return None + func = resulting_func + return func + + +@verify.register(nodes.Decorator) +def verify_decorator( + stub: nodes.Decorator, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + if stub.func.is_type_check_only: + # This function only exists in stubs, we only check that the runtime part + # is missing. Other checks are not required. + if not isinstance(runtime, Missing): + yield Error( + object_path, + 'is marked as "@type_check_only", but also exists at runtime', + stub, + runtime, + stub_desc=repr(stub), + ) + return + + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime) + return + if stub.func.is_property: + for message in _verify_readonly_property(stub, runtime): + yield Error(object_path, message, stub, runtime) + for message in _verify_abstract_status(stub.func, runtime): + yield Error(object_path, message, stub, runtime) + return + + func = _resolve_funcitem_from_decorator(stub) + if func is not None: + yield from verify(func, runtime, object_path) + + +@verify.register(nodes.TypeAlias) +def verify_typealias( + stub: nodes.TypeAlias, runtime: MaybeMissing[Any], object_path: list[str] +) -> Iterator[Error]: + stub_target = mypy.types.get_proper_type(stub.target) + stub_desc = f"Type alias for {stub_target}" + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime, stub_desc=stub_desc) + return + runtime_origin = get_origin(runtime) or runtime + if isinstance(stub_target, mypy.types.Instance): + if not isinstance(runtime_origin, type): + yield Error( + object_path, + "is inconsistent, runtime is not a type", + stub, + runtime, + stub_desc=stub_desc, + ) + return + + stub_origin = stub_target.type + # Do our best to figure out the fullname of the runtime object... + runtime_name: object + try: + runtime_name = runtime_origin.__qualname__ + except AttributeError: + runtime_name = getattr(runtime_origin, "__name__", MISSING) + if isinstance(runtime_name, str): + runtime_module: object = getattr(runtime_origin, "__module__", MISSING) + if isinstance(runtime_module, str): + if runtime_module == "collections.abc" or ( + runtime_module == "re" and runtime_name in {"Match", "Pattern"} + ): + runtime_module = "typing" + runtime_fullname = f"{runtime_module}.{runtime_name}" + if re.fullmatch(rf"_?{re.escape(stub_origin.fullname)}", runtime_fullname): + # Okay, we're probably fine. + return + + # Okay, either we couldn't construct a fullname + # or the fullname of the stub didn't match the fullname of the runtime. + # Fallback to a full structural check of the runtime vis-a-vis the stub. + yield from verify(stub_origin, runtime_origin, object_path) + return + if isinstance(stub_target, mypy.types.UnionType): + # complain if runtime is not a Union or UnionType + if runtime_origin is not Union and ( + not (sys.version_info >= (3, 10) and isinstance(runtime, types.UnionType)) + ): + yield Error(object_path, "is not a Union", stub, runtime, stub_desc=str(stub_target)) + # could check Union contents here... + return + if isinstance(stub_target, mypy.types.TupleType): + if tuple not in getattr(runtime_origin, "__mro__", ()): + yield Error( + object_path, "is not a subclass of tuple", stub, runtime, stub_desc=stub_desc + ) + # could check Tuple contents here... + return + if isinstance(stub_target, mypy.types.CallableType): + if runtime_origin is not collections.abc.Callable: + yield Error( + object_path, "is not a type alias for Callable", stub, runtime, stub_desc=stub_desc + ) + # could check Callable contents here... + return + if isinstance(stub_target, mypy.types.AnyType): + return + yield Error(object_path, "is not a recognised type alias", stub, runtime, stub_desc=stub_desc) + + +# ==================== +# Helpers +# ==================== + + +IGNORED_MODULE_DUNDERS: typing_extensions.Final = frozenset( + { + "__file__", + "__doc__", + "__name__", + "__builtins__", + "__package__", + "__cached__", + "__loader__", + "__spec__", + "__annotations__", + "__path__", # mypy adds __path__ to packages, but C packages don't have it + "__getattr__", # resulting behaviour might be typed explicitly + # Created by `warnings.warn`, does not make much sense to have in stubs: + "__warningregistry__", + # TODO: remove the following from this list + "__author__", + "__version__", + "__copyright__", + } +) + +IGNORABLE_CLASS_DUNDERS: typing_extensions.Final = frozenset( + { + # Special attributes + "__dict__", + "__annotations__", + "__text_signature__", + "__weakref__", + "__hash__", + "__getattr__", # resulting behaviour might be typed explicitly + "__setattr__", # defining this on a class can cause worse type checking + "__vectorcalloffset__", # undocumented implementation detail of the vectorcall protocol + "__firstlineno__", + "__static_attributes__", + # isinstance/issubclass hooks that type-checkers don't usually care about + "__instancecheck__", + "__subclasshook__", + "__subclasscheck__", + # python2 only magic methods: + "__cmp__", + "__nonzero__", + "__unicode__", + "__div__", + # cython methods + "__pyx_vtable__", + # Pickle methods + "__setstate__", + "__getstate__", + "__getnewargs__", + "__getinitargs__", + "__reduce_ex__", + "__reduce__", + # ctypes weirdness + "__ctype_be__", + "__ctype_le__", + "__ctypes_from_outparam__", + # mypy limitations + "__abstractmethods__", # Classes with metaclass=ABCMeta inherit this attribute + "__new_member__", # If an enum defines __new__, the method is renamed as __new_member__ + "__dataclass_fields__", # Generated by dataclasses + "__dataclass_params__", # Generated by dataclasses + "__doc__", # mypy's semanal for namedtuples assumes this is str, not Optional[str] + # Added to all protocol classes on 3.12+ (or if using typing_extensions.Protocol) + "__protocol_attrs__", + "__callable_proto_members_only__", + "__non_callable_proto_members__", + # typing implementation details, consider removing some of these: + "__parameters__", + "__origin__", + "__args__", + "__orig_bases__", + "__final__", # Has a specialized check + # Consider removing __slots__? + "__slots__", + } +) + + +def is_probably_private(name: str) -> bool: + return name.startswith("_") and not is_dunder(name) + + +def is_probably_a_function(runtime: Any) -> bool: + return ( + isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType)) + or isinstance(runtime, (types.MethodType, types.BuiltinMethodType)) + or (inspect.ismethoddescriptor(runtime) and callable(runtime)) + ) + + +def is_read_only_property(runtime: object) -> bool: + return isinstance(runtime, property) and runtime.fset is None + + +def safe_inspect_signature(runtime: Any) -> inspect.Signature | None: + try: + try: + return inspect.signature(runtime) + except ValueError: + if ( + hasattr(runtime, "__text_signature__") + and "" in runtime.__text_signature__ + ): + # Try to fix up the signature. Workaround for + # https://github.com/python/cpython/issues/87233 + sig = runtime.__text_signature__.replace("", "...") + sig = inspect._signature_fromstr(inspect.Signature, runtime, sig) # type: ignore[attr-defined] + assert isinstance(sig, inspect.Signature) + new_params = [ + ( + parameter.replace(default=UNREPRESENTABLE) + if parameter.default is ... + else parameter + ) + for parameter in sig.parameters.values() + ] + return sig.replace(parameters=new_params) + else: + raise + except Exception: + # inspect.signature throws ValueError all the time + # catch RuntimeError because of https://bugs.python.org/issue39504 + # catch TypeError because of https://github.com/python/typeshed/pull/5762 + # catch AttributeError because of inspect.signature(_curses.window.border) + return None + + +def describe_runtime_callable(signature: inspect.Signature, *, is_async: bool) -> str: + return f'{"async " if is_async else ""}def {signature}' + + +def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool: + """Checks whether ``left`` is a subtype of ``right``.""" + left = mypy.types.get_proper_type(left) + right = mypy.types.get_proper_type(right) + if ( + isinstance(left, mypy.types.LiteralType) + and isinstance(left.value, int) + and left.value in (0, 1) + and mypy.types.is_named_instance(right, "builtins.bool") + ): + # Pretend Literal[0, 1] is a subtype of bool to avoid unhelpful errors. + return True + + if isinstance(right, mypy.types.TypedDictType) and mypy.types.is_named_instance( + left, "builtins.dict" + ): + # Special case checks against TypedDicts + return True + + with mypy.state.state.strict_optional_set(True): + return mypy.subtypes.is_subtype(left, right) + + +def get_mypy_type_of_runtime_value(runtime: Any) -> mypy.types.Type | None: + """Returns a mypy type object representing the type of ``runtime``. + + Returns None if we can't find something that works. + + """ + if runtime is None: + return mypy.types.NoneType() + if isinstance(runtime, property): + # Give up on properties to avoid issues with things that are typed as attributes. + return None + + def anytype() -> mypy.types.AnyType: + return mypy.types.AnyType(mypy.types.TypeOfAny.unannotated) + + if isinstance( + runtime, + (types.FunctionType, types.BuiltinFunctionType, types.MethodType, types.BuiltinMethodType), + ): + builtins = get_stub("builtins") + assert builtins is not None + type_info = builtins.names["function"].node + assert isinstance(type_info, nodes.TypeInfo) + fallback = mypy.types.Instance(type_info, [anytype()]) + signature = safe_inspect_signature(runtime) + if signature: + arg_types = [] + arg_kinds = [] + arg_names = [] + for arg in signature.parameters.values(): + arg_types.append(anytype()) + arg_names.append( + None if arg.kind == inspect.Parameter.POSITIONAL_ONLY else arg.name + ) + has_default = arg.default == inspect.Parameter.empty + if arg.kind == inspect.Parameter.POSITIONAL_ONLY: + arg_kinds.append(nodes.ARG_POS if has_default else nodes.ARG_OPT) + elif arg.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD: + arg_kinds.append(nodes.ARG_POS if has_default else nodes.ARG_OPT) + elif arg.kind == inspect.Parameter.KEYWORD_ONLY: + arg_kinds.append(nodes.ARG_NAMED if has_default else nodes.ARG_NAMED_OPT) + elif arg.kind == inspect.Parameter.VAR_POSITIONAL: + arg_kinds.append(nodes.ARG_STAR) + elif arg.kind == inspect.Parameter.VAR_KEYWORD: + arg_kinds.append(nodes.ARG_STAR2) + else: + raise AssertionError + else: + arg_types = [anytype(), anytype()] + arg_kinds = [nodes.ARG_STAR, nodes.ARG_STAR2] + arg_names = [None, None] + + return mypy.types.CallableType( + arg_types, + arg_kinds, + arg_names, + ret_type=anytype(), + fallback=fallback, + is_ellipsis_args=True, + ) + + # Try and look up a stub for the runtime object + stub = get_stub(type(runtime).__module__) + if stub is None: + return None + type_name = type(runtime).__name__ + if type_name not in stub.names: + return None + type_info = stub.names[type_name].node + if isinstance(type_info, nodes.Var): + return type_info.type + if not isinstance(type_info, nodes.TypeInfo): + return None + + if isinstance(runtime, tuple): + # Special case tuples so we construct a valid mypy.types.TupleType + optional_items = [get_mypy_type_of_runtime_value(v) for v in runtime] + items = [(i if i is not None else anytype()) for i in optional_items] + fallback = mypy.types.Instance(type_info, [anytype()]) + return mypy.types.TupleType(items, fallback) + + fallback = mypy.types.Instance(type_info, [anytype() for _ in type_info.type_vars]) + + value: bool | int | str + if isinstance(runtime, enum.Enum) and isinstance(runtime.name, str): + value = runtime.name + elif isinstance(runtime, bytes): + value = bytes_to_human_readable_repr(runtime) + elif isinstance(runtime, (bool, int, str)): + value = runtime + else: + return fallback + + return mypy.types.LiteralType(value=value, fallback=fallback) + + +# ==================== +# Build and entrypoint +# ==================== + + +_all_stubs: dict[str, nodes.MypyFile] = {} + + +def build_stubs(modules: list[str], options: Options, find_submodules: bool = False) -> list[str]: + """Uses mypy to construct stub objects for the given modules. + + This sets global state that ``get_stub`` can access. + + Returns all modules we might want to check. If ``find_submodules`` is False, this is equal + to ``modules``. + + :param modules: List of modules to build stubs for. + :param options: Mypy options for finding and building stubs. + :param find_submodules: Whether to attempt to find submodules of the given modules as well. + + """ + data_dir = mypy.build.default_data_dir() + search_path = mypy.modulefinder.compute_search_paths([], options, data_dir) + find_module_cache = mypy.modulefinder.FindModuleCache( + search_path, fscache=None, options=options + ) + + all_modules = [] + sources = [] + for module in modules: + all_modules.append(module) + if not find_submodules: + module_path = find_module_cache.find_module(module) + if not isinstance(module_path, str): + # test_module will yield an error later when it can't find stubs + continue + sources.append(mypy.modulefinder.BuildSource(module_path, module, None)) + else: + found_sources = find_module_cache.find_modules_recursive(module) + sources.extend(found_sources) + # find submodules via mypy + all_modules.extend(s.module for s in found_sources if s.module not in all_modules) + # find submodules via pkgutil + try: + runtime = silent_import_module(module) + all_modules.extend( + m.name + for m in pkgutil.walk_packages(runtime.__path__, runtime.__name__ + ".") + if m.name not in all_modules + ) + except KeyboardInterrupt: + raise + except BaseException: + pass + + if sources: + try: + res = mypy.build.build(sources=sources, options=options) + except mypy.errors.CompileError as e: + raise StubtestFailure(f"failed mypy compile:\n{e}") from e + if res.errors: + raise StubtestFailure("mypy build errors:\n" + "\n".join(res.errors)) + + global _all_stubs + _all_stubs = res.files + + return all_modules + + +def get_stub(module: str) -> nodes.MypyFile | None: + """Returns a stub object for the given module, if we've built one.""" + return _all_stubs.get(module) + + +def get_typeshed_stdlib_modules( + custom_typeshed_dir: str | None, version_info: tuple[int, int] | None = None +) -> set[str]: + """Returns a list of stdlib modules in typeshed (for current Python version).""" + stdlib_py_versions = mypy.modulefinder.load_stdlib_py_versions(custom_typeshed_dir) + if version_info is None: + version_info = sys.version_info[0:2] + + def exists_in_version(module: str) -> bool: + assert version_info is not None + parts = module.split(".") + for i in range(len(parts), 0, -1): + current_module = ".".join(parts[:i]) + if current_module in stdlib_py_versions: + minver, maxver = stdlib_py_versions[current_module] + return version_info >= minver and (maxver is None or version_info <= maxver) + return False + + if custom_typeshed_dir: + typeshed_dir = Path(custom_typeshed_dir) + else: + typeshed_dir = Path(mypy.build.default_data_dir()) / "typeshed" + stdlib_dir = typeshed_dir / "stdlib" + + modules: set[str] = set() + for path in stdlib_dir.rglob("*.pyi"): + if path.stem == "__init__": + path = path.parent + module = ".".join(path.relative_to(stdlib_dir).parts[:-1] + (path.stem,)) + if exists_in_version(module): + modules.add(module) + return modules + + +def get_importable_stdlib_modules() -> set[str]: + """Return all importable stdlib modules at runtime.""" + all_stdlib_modules: AbstractSet[str] + if sys.version_info >= (3, 10): + all_stdlib_modules = sys.stdlib_module_names + else: + all_stdlib_modules = set(sys.builtin_module_names) + modules_by_finder: defaultdict[importlib.machinery.FileFinder, set[str]] = defaultdict(set) + for m in pkgutil.iter_modules(): + if isinstance(m.module_finder, importlib.machinery.FileFinder): + modules_by_finder[m.module_finder].add(m.name) + for finder, module_group in modules_by_finder.items(): + if ( + "site-packages" not in Path(finder.path).parts + # if "_queue" is present, it's most likely the module finder + # for stdlib extension modules; + # if "queue" is present, it's most likely the module finder + # for pure-Python stdlib modules. + # In either case, we'll want to add all the modules that the finder has to offer us. + # This is a bit hacky, but seems to work well in a cross-platform way. + and {"_queue", "queue"} & module_group + ): + all_stdlib_modules.update(module_group) + + importable_stdlib_modules: set[str] = set() + for module_name in all_stdlib_modules: + if module_name in ANNOYING_STDLIB_MODULES: + continue + + try: + runtime = silent_import_module(module_name) + except ImportError: + continue + else: + importable_stdlib_modules.add(module_name) + + try: + # some stdlib modules (e.g. `nt`) don't have __path__ set... + runtime_path = runtime.__path__ + runtime_name = runtime.__name__ + except AttributeError: + continue + + for submodule in pkgutil.walk_packages(runtime_path, runtime_name + "."): + submodule_name = submodule.name + + # There are many annoying *.__main__ stdlib modules, + # and including stubs for them isn't really that useful anyway: + # tkinter.__main__ opens a tkinter windows; unittest.__main__ raises SystemExit; etc. + # + # The idlelib.* submodules are similarly annoying in opening random tkinter windows, + # and we're unlikely to ever add stubs for idlelib in typeshed + # (see discussion in https://github.com/python/typeshed/pull/9193) + # + # test.* modules do weird things like raising exceptions in __del__ methods, + # leading to unraisable exceptions being logged to the terminal + # as a warning at the end of the stubtest run + if submodule_name.endswith(".__main__") or submodule_name.startswith( + ("idlelib.", "test.") + ): + continue + + try: + silent_import_module(submodule_name) + except KeyboardInterrupt: + raise + # importing multiprocessing.popen_forkserver on Windows raises AttributeError... + # some submodules also appear to raise SystemExit as well on some Python versions + # (not sure exactly which) + except BaseException: + continue + else: + importable_stdlib_modules.add(submodule_name) + + return importable_stdlib_modules + + +def get_allowlist_entries(allowlist_file: str) -> Iterator[str]: + def strip_comments(s: str) -> str: + try: + return s[: s.index("#")].strip() + except ValueError: + return s.strip() + + with open(allowlist_file) as f: + for line in f: + entry = strip_comments(line) + if entry: + yield entry + + +class _Arguments: + modules: list[str] + concise: bool + ignore_missing_stub: bool + ignore_positional_only: bool + allowlist: list[str] + generate_allowlist: bool + ignore_unused_allowlist: bool + mypy_config_file: str | None + custom_typeshed_dir: str | None + check_typeshed: bool + version: str + show_traceback: bool + pdb: bool + + +# typeshed added a stub for __main__, but that causes stubtest to check itself +ANNOYING_STDLIB_MODULES: typing_extensions.Final = frozenset( + {"antigravity", "this", "__main__", "_ios_support"} +) + + +def test_stubs(args: _Arguments, use_builtins_fixtures: bool = False) -> int: + """This is stubtest! It's time to test the stubs!""" + # Load the allowlist. This is a series of strings corresponding to Error.object_desc + # Values in the dict will store whether we used the allowlist entry or not. + allowlist = { + entry: False + for allowlist_file in args.allowlist + for entry in get_allowlist_entries(allowlist_file) + } + allowlist_regexes = {entry: re.compile(entry) for entry in allowlist} + + # If we need to generate an allowlist, we store Error.object_desc for each error here. + generated_allowlist = set() + + modules = args.modules + if args.check_typeshed: + if args.modules: + print( + _style("error:", color="red", bold=True), + "cannot pass both --check-typeshed and a list of modules", + ) + return 1 + typeshed_modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir) + runtime_modules = get_importable_stdlib_modules() + modules = sorted((typeshed_modules | runtime_modules) - ANNOYING_STDLIB_MODULES) + + if not modules: + print(_style("error:", color="red", bold=True), "no modules to check") + return 1 + + options = Options() + options.incremental = False + options.custom_typeshed_dir = args.custom_typeshed_dir + if options.custom_typeshed_dir: + options.abs_custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir) + options.config_file = args.mypy_config_file + options.use_builtins_fixtures = use_builtins_fixtures + options.show_traceback = args.show_traceback + options.pdb = args.pdb + + if options.config_file: + + def set_strict_flags() -> None: # not needed yet + return + + parse_config_file(options, set_strict_flags, options.config_file, sys.stdout, sys.stderr) + + def error_callback(msg: str) -> typing.NoReturn: + print(_style("error:", color="red", bold=True), msg) + sys.exit(1) + + def warning_callback(msg: str) -> None: + print(_style("warning:", color="yellow", bold=True), msg) + + options.process_error_codes(error_callback=error_callback) + options.process_incomplete_features( + error_callback=error_callback, warning_callback=warning_callback + ) + + try: + modules = build_stubs(modules, options, find_submodules=not args.check_typeshed) + except StubtestFailure as stubtest_failure: + print( + _style("error:", color="red", bold=True), + f"not checking stubs due to {stubtest_failure}", + ) + return 1 + + exit_code = 0 + error_count = 0 + for module in modules: + for error in test_module(module): + # Filter errors + if args.ignore_missing_stub and error.is_missing_stub(): + continue + if args.ignore_positional_only and error.is_positional_only_related(): + continue + if error.object_desc in allowlist: + allowlist[error.object_desc] = True + continue + is_allowlisted = False + for w in allowlist: + if allowlist_regexes[w].fullmatch(error.object_desc): + allowlist[w] = True + is_allowlisted = True + break + if is_allowlisted: + continue + + # We have errors, so change exit code, and output whatever necessary + exit_code = 1 + if args.generate_allowlist: + generated_allowlist.add(error.object_desc) + continue + print(error.get_description(concise=args.concise)) + error_count += 1 + + # Print unused allowlist entries + if not args.ignore_unused_allowlist: + for w in allowlist: + # Don't consider an entry unused if it regex-matches the empty string + # This lets us allowlist errors that don't manifest at all on some systems + if not allowlist[w] and not allowlist_regexes[w].fullmatch(""): + exit_code = 1 + error_count += 1 + print(f"note: unused allowlist entry {w}") + + # Print the generated allowlist + if args.generate_allowlist: + for e in sorted(generated_allowlist): + print(e) + exit_code = 0 + elif not args.concise: + if error_count: + print( + _style( + f"Found {error_count} error{plural_s(error_count)}" + f" (checked {len(modules)} module{plural_s(modules)})", + color="red", + bold=True, + ) + ) + else: + print( + _style( + f"Success: no issues found in {len(modules)} module{plural_s(modules)}", + color="green", + bold=True, + ) + ) + + return exit_code + + +def parse_options(args: list[str]) -> _Arguments: + parser = argparse.ArgumentParser( + description="Compares stubs to objects introspected from the runtime." + ) + parser.add_argument("modules", nargs="*", help="Modules to test") + parser.add_argument( + "--concise", + action="store_true", + help="Makes stubtest's output more concise, one line per error", + ) + parser.add_argument( + "--ignore-missing-stub", + action="store_true", + help="Ignore errors for stub missing things that are present at runtime", + ) + parser.add_argument( + "--ignore-positional-only", + action="store_true", + help="Ignore errors for whether an argument should or shouldn't be positional-only", + ) + parser.add_argument( + "--allowlist", + "--whitelist", + action="append", + metavar="FILE", + default=[], + help=( + "Use file as an allowlist. Can be passed multiple times to combine multiple " + "allowlists. Allowlists can be created with --generate-allowlist. Allowlists " + "support regular expressions." + ), + ) + parser.add_argument( + "--generate-allowlist", + "--generate-whitelist", + action="store_true", + help="Print an allowlist (to stdout) to be used with --allowlist", + ) + parser.add_argument( + "--ignore-unused-allowlist", + "--ignore-unused-whitelist", + action="store_true", + help="Ignore unused allowlist entries", + ) + parser.add_argument( + "--mypy-config-file", + metavar="FILE", + help=("Use specified mypy config file to determine mypy plugins and mypy path"), + ) + parser.add_argument( + "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR" + ) + parser.add_argument( + "--check-typeshed", action="store_true", help="Check all stdlib modules in typeshed" + ) + parser.add_argument( + "--version", action="version", version="%(prog)s " + mypy.version.__version__ + ) + parser.add_argument("--pdb", action="store_true", help="Invoke pdb on fatal error") + parser.add_argument( + "--show-traceback", "--tb", action="store_true", help="Show traceback on fatal error" + ) + + return parser.parse_args(args, namespace=_Arguments()) + + +def main() -> int: + mypy.util.check_python_version("stubtest") + return test_stubs(parse_options(sys.argv[1:])) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/mypy/stubutil.py b/mypy/stubutil.py new file mode 100644 index 000000000000..c11843c57f2a --- /dev/null +++ b/mypy/stubutil.py @@ -0,0 +1,876 @@ +"""Utilities for mypy.stubgen, mypy.stubgenc, and mypy.stubdoc modules.""" + +from __future__ import annotations + +import os.path +import re +import sys +from abc import abstractmethod +from collections import defaultdict +from contextlib import contextmanager +from typing import Final, Iterable, Iterator, Mapping +from typing_extensions import overload + +from mypy_extensions import mypyc_attr + +import mypy.options +from mypy.modulefinder import ModuleNotFoundReason +from mypy.moduleinspect import InspectError, ModuleInspect +from mypy.nodes import PARAM_SPEC_KIND, TYPE_VAR_TUPLE_KIND, ClassDef, FuncDef, TypeAliasStmt +from mypy.stubdoc import ArgSig, FunctionSig +from mypy.types import AnyType, NoneType, Type, TypeList, TypeStrVisitor, UnboundType, UnionType + +# Modules that may fail when imported, or that may have side effects (fully qualified). +NOT_IMPORTABLE_MODULES = () + +# Typing constructs to be replaced by their builtin equivalents. +TYPING_BUILTIN_REPLACEMENTS: Final = { + # From typing + "typing.Text": "builtins.str", + "typing.Tuple": "builtins.tuple", + "typing.List": "builtins.list", + "typing.Dict": "builtins.dict", + "typing.Set": "builtins.set", + "typing.FrozenSet": "builtins.frozenset", + "typing.Type": "builtins.type", + # From typing_extensions + "typing_extensions.Text": "builtins.str", + "typing_extensions.Tuple": "builtins.tuple", + "typing_extensions.List": "builtins.list", + "typing_extensions.Dict": "builtins.dict", + "typing_extensions.Set": "builtins.set", + "typing_extensions.FrozenSet": "builtins.frozenset", + "typing_extensions.Type": "builtins.type", +} + + +class CantImport(Exception): + def __init__(self, module: str, message: str) -> None: + self.module = module + self.message = message + + +def walk_packages( + inspect: ModuleInspect, packages: list[str], verbose: bool = False +) -> Iterator[str]: + """Iterates through all packages and sub-packages in the given list. + + This uses runtime imports (in another process) to find both Python and C modules. + For Python packages we simply pass the __path__ attribute to pkgutil.walk_packages() to + get the content of the package (all subpackages and modules). However, packages in C + extensions do not have this attribute, so we have to roll out our own logic: recursively + find all modules imported in the package that have matching names. + """ + for package_name in packages: + if package_name in NOT_IMPORTABLE_MODULES: + print(f"{package_name}: Skipped (blacklisted)") + continue + if verbose: + print(f"Trying to import {package_name!r} for runtime introspection") + try: + prop = inspect.get_package_properties(package_name) + except InspectError: + report_missing(package_name) + continue + yield prop.name + if prop.is_c_module: + # Recursively iterate through the subpackages + yield from walk_packages(inspect, prop.subpackages, verbose) + else: + yield from prop.subpackages + + +def find_module_path_using_sys_path(module: str, sys_path: list[str]) -> str | None: + relative_candidates = ( + module.replace(".", "/") + ".py", + os.path.join(module.replace(".", "/"), "__init__.py"), + ) + for base in sys_path: + for relative_path in relative_candidates: + path = os.path.join(base, relative_path) + if os.path.isfile(path): + return path + return None + + +def find_module_path_and_all_py3( + inspect: ModuleInspect, module: str, verbose: bool +) -> tuple[str | None, list[str] | None] | None: + """Find module and determine __all__ for a Python 3 module. + + Return None if the module is a C or pyc-only module. + Return (module_path, __all__) if it is a Python module. + Raise CantImport if import failed. + """ + if module in NOT_IMPORTABLE_MODULES: + raise CantImport(module, "") + + # TODO: Support custom interpreters. + if verbose: + print(f"Trying to import {module!r} for runtime introspection") + try: + mod = inspect.get_package_properties(module) + except InspectError as e: + # Fall back to finding the module using sys.path. + path = find_module_path_using_sys_path(module, sys.path) + if path is None: + raise CantImport(module, str(e)) from e + return path, None + if mod.is_c_module: + return None + return mod.file, mod.all + + +@contextmanager +def generate_guarded( + mod: str, target: str, ignore_errors: bool = True, verbose: bool = False +) -> Iterator[None]: + """Ignore or report errors during stub generation. + + Optionally report success. + """ + if verbose: + print(f"Processing {mod}") + try: + yield + except Exception as e: + if not ignore_errors: + raise e + else: + # --ignore-errors was passed + print("Stub generation failed for", mod, file=sys.stderr) + else: + if verbose: + print(f"Created {target}") + + +def report_missing(mod: str, message: str | None = "", traceback: str = "") -> None: + if message: + message = " with error: " + message + print(f"{mod}: Failed to import, skipping{message}") + + +def fail_missing(mod: str, reason: ModuleNotFoundReason) -> None: + if reason is ModuleNotFoundReason.NOT_FOUND: + clarification = "(consider using --search-path)" + elif reason is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: + clarification = "(module likely exists, but is not PEP 561 compatible)" + else: + clarification = f"(unknown reason '{reason}')" + raise SystemExit(f"Can't find module '{mod}' {clarification}") + + +@overload +def remove_misplaced_type_comments(source: bytes) -> bytes: ... + + +@overload +def remove_misplaced_type_comments(source: str) -> str: ... + + +def remove_misplaced_type_comments(source: str | bytes) -> str | bytes: + """Remove comments from source that could be understood as misplaced type comments. + + Normal comments may look like misplaced type comments, and since they cause blocking + parse errors, we want to avoid them. + """ + if isinstance(source, bytes): + # This gives us a 1-1 character code mapping, so it's roundtrippable. + text = source.decode("latin1") + else: + text = source + + # Remove something that looks like a variable type comment but that's by itself + # on a line, as it will often generate a parse error (unless it's # type: ignore). + text = re.sub(r'^[ \t]*# +type: +["\'a-zA-Z_].*$', "", text, flags=re.MULTILINE) + + # Remove something that looks like a function type comment after docstring, + # which will result in a parse error. + text = re.sub(r'""" *\n[ \t\n]*# +type: +\(.*$', '"""\n', text, flags=re.MULTILINE) + text = re.sub(r"''' *\n[ \t\n]*# +type: +\(.*$", "'''\n", text, flags=re.MULTILINE) + + # Remove something that looks like a badly formed function type comment. + text = re.sub(r"^[ \t]*# +type: +\([^()]+(\)[ \t]*)?$", "", text, flags=re.MULTILINE) + + if isinstance(source, bytes): + return text.encode("latin1") + else: + return text + + +def common_dir_prefix(paths: list[str]) -> str: + if not paths: + return "." + cur = os.path.dirname(os.path.normpath(paths[0])) + for path in paths[1:]: + while True: + path = os.path.dirname(os.path.normpath(path)) + if (cur + os.sep).startswith(path + os.sep): + cur = path + break + return cur or "." + + +class AnnotationPrinter(TypeStrVisitor): + """Visitor used to print existing annotations in a file. + + The main difference from TypeStrVisitor is a better treatment of + unbound types. + + Notes: + * This visitor doesn't add imports necessary for annotations, this is done separately + by ImportTracker. + * It can print all kinds of types, but the generated strings may not be valid (notably + callable types) since it prints the same string that reveal_type() does. + * For Instance types it prints the fully qualified names. + """ + + # TODO: Generate valid string representation for callable types. + # TODO: Use short names for Instances. + def __init__( + self, + stubgen: BaseStubGenerator, + known_modules: list[str] | None = None, + local_modules: list[str] | None = None, + ) -> None: + super().__init__(options=mypy.options.Options()) + self.stubgen = stubgen + self.known_modules = known_modules + self.local_modules = local_modules or ["builtins"] + + def visit_any(self, t: AnyType) -> str: + s = super().visit_any(t) + self.stubgen.import_tracker.require_name(s) + return s + + def visit_unbound_type(self, t: UnboundType) -> str: + s = t.name + fullname = self.stubgen.resolve_name(s) + if fullname == "typing.Union": + return " | ".join([item.accept(self) for item in t.args]) + if fullname == "typing.Optional": + if len(t.args) == 1: + return f"{t.args[0].accept(self)} | None" + return self.stubgen.add_name("_typeshed.Incomplete") + if fullname in TYPING_BUILTIN_REPLACEMENTS: + s = self.stubgen.add_name(TYPING_BUILTIN_REPLACEMENTS[fullname], require=True) + if self.known_modules is not None and "." in s: + # see if this object is from any of the modules that we're currently processing. + # reverse sort so that subpackages come before parents: e.g. "foo.bar" before "foo". + for module_name in self.local_modules + sorted(self.known_modules, reverse=True): + if s.startswith(module_name + "."): + if module_name in self.local_modules: + s = s[len(module_name) + 1 :] + arg_module = module_name + break + else: + arg_module = s[: s.rindex(".")] + if arg_module not in self.local_modules: + self.stubgen.import_tracker.add_import(arg_module, require=True) + elif s == "NoneType": + # when called without analysis all types are unbound, so this won't hit + # visit_none_type(). + s = "None" + else: + self.stubgen.import_tracker.require_name(s) + if t.args: + s += f"[{self.args_str(t.args)}]" + elif t.empty_tuple_index: + s += "[()]" + return s + + def visit_none_type(self, t: NoneType) -> str: + return "None" + + def visit_type_list(self, t: TypeList) -> str: + return f"[{self.list_str(t.items)}]" + + def visit_union_type(self, t: UnionType) -> str: + return " | ".join([item.accept(self) for item in t.items]) + + def args_str(self, args: Iterable[Type]) -> str: + """Convert an array of arguments to strings and join the results with commas. + + The main difference from list_str is the preservation of quotes for string + arguments + """ + types = ["builtins.bytes", "builtins.str"] + res = [] + for arg in args: + arg_str = arg.accept(self) + if isinstance(arg, UnboundType) and arg.original_str_fallback in types: + res.append(f"'{arg_str}'") + else: + res.append(arg_str) + return ", ".join(res) + + +class ClassInfo: + def __init__( + self, + name: str, + self_var: str, + docstring: str | None = None, + cls: type | None = None, + parent: ClassInfo | None = None, + ) -> None: + self.name = name + self.self_var = self_var + self.docstring = docstring + self.cls = cls + self.parent = parent + + +class FunctionContext: + def __init__( + self, + module_name: str, + name: str, + docstring: str | None = None, + is_abstract: bool = False, + class_info: ClassInfo | None = None, + ) -> None: + self.module_name = module_name + self.name = name + self.docstring = docstring + self.is_abstract = is_abstract + self.class_info = class_info + self._fullname: str | None = None + + @property + def fullname(self) -> str: + if self._fullname is None: + if self.class_info: + parents = [] + class_info: ClassInfo | None = self.class_info + while class_info is not None: + parents.append(class_info.name) + class_info = class_info.parent + namespace = ".".join(reversed(parents)) + self._fullname = f"{self.module_name}.{namespace}.{self.name}" + else: + self._fullname = f"{self.module_name}.{self.name}" + return self._fullname + + +def infer_method_ret_type(name: str) -> str | None: + """Infer return types for known special methods""" + if name.startswith("__") and name.endswith("__"): + name = name[2:-2] + if name in ("float", "bool", "bytes", "int", "complex", "str"): + return name + # Note: __eq__ and co may return arbitrary types, but bool is good enough for stubgen. + elif name in ("eq", "ne", "lt", "le", "gt", "ge", "contains"): + return "bool" + elif name in ("len", "length_hint", "index", "hash", "sizeof", "trunc", "floor", "ceil"): + return "int" + elif name in ("format", "repr"): + return "str" + elif name in ("init", "setitem", "del", "delitem"): + return "None" + return None + + +def infer_method_arg_types( + name: str, self_var: str = "self", arg_names: list[str] | None = None +) -> list[ArgSig] | None: + """Infer argument types for known special methods""" + args: list[ArgSig] | None = None + if name.startswith("__") and name.endswith("__"): + if arg_names and len(arg_names) >= 1 and arg_names[0] == "self": + arg_names = arg_names[1:] + + name = name[2:-2] + if name == "exit": + if arg_names is None: + arg_names = ["type", "value", "traceback"] + if len(arg_names) == 3: + arg_types = [ + "type[BaseException] | None", + "BaseException | None", + "types.TracebackType | None", + ] + args = [ + ArgSig(name=arg_name, type=arg_type) + for arg_name, arg_type in zip(arg_names, arg_types) + ] + if args is not None: + return [ArgSig(name=self_var)] + args + return None + + +@mypyc_attr(allow_interpreted_subclasses=True) +class SignatureGenerator: + """Abstract base class for extracting a list of FunctionSigs for each function.""" + + def remove_self_type( + self, inferred: list[FunctionSig] | None, self_var: str + ) -> list[FunctionSig] | None: + """Remove type annotation from self/cls argument""" + if inferred: + for signature in inferred: + if signature.args: + if signature.args[0].name == self_var: + signature.args[0].type = None + return inferred + + @abstractmethod + def get_function_sig( + self, default_sig: FunctionSig, ctx: FunctionContext + ) -> list[FunctionSig] | None: + """Return a list of signatures for the given function. + + If no signature can be found, return None. If all of the registered SignatureGenerators + for the stub generator return None, then the default_sig will be used. + """ + pass + + @abstractmethod + def get_property_type(self, default_type: str | None, ctx: FunctionContext) -> str | None: + """Return the type of the given property""" + pass + + +class ImportTracker: + """Record necessary imports during stub generation.""" + + def __init__(self) -> None: + # module_for['foo'] has the module name where 'foo' was imported from, or None if + # 'foo' is a module imported directly; + # direct_imports['foo'] is the module path used when the name 'foo' was added to the + # namespace. + # reverse_alias['foo'] is the name that 'foo' had originally when imported with an + # alias; examples + # 'from pkg import mod' ==> module_for['mod'] == 'pkg' + # 'from pkg import mod as m' ==> module_for['m'] == 'pkg' + # ==> reverse_alias['m'] == 'mod' + # 'import pkg.mod as m' ==> module_for['m'] == None + # ==> reverse_alias['m'] == 'pkg.mod' + # 'import pkg.mod' ==> module_for['pkg'] == None + # ==> module_for['pkg.mod'] == None + # ==> direct_imports['pkg'] == 'pkg.mod' + # ==> direct_imports['pkg.mod'] == 'pkg.mod' + self.module_for: dict[str, str | None] = {} + self.direct_imports: dict[str, str] = {} + self.reverse_alias: dict[str, str] = {} + + # required_names is the set of names that are actually used in a type annotation + self.required_names: set[str] = set() + + # Names that should be reexported if they come from another module + self.reexports: set[str] = set() + + def add_import_from( + self, module: str, names: list[tuple[str, str | None]], require: bool = False + ) -> None: + for name, alias in names: + if alias: + # 'from {module} import {name} as {alias}' + self.module_for[alias] = module + self.reverse_alias[alias] = name + else: + # 'from {module} import {name}' + self.module_for[name] = module + self.reverse_alias.pop(name, None) + if require: + self.require_name(alias or name) + self.direct_imports.pop(alias or name, None) + + def add_import(self, module: str, alias: str | None = None, require: bool = False) -> None: + if alias: + # 'import {module} as {alias}' + assert "." not in alias # invalid syntax + self.module_for[alias] = None + self.reverse_alias[alias] = module + if require: + self.required_names.add(alias) + else: + # 'import {module}' + name = module + if require: + self.required_names.add(name) + # add module and its parent packages + while name: + self.module_for[name] = None + self.direct_imports[name] = module + self.reverse_alias.pop(name, None) + name = name.rpartition(".")[0] + + def require_name(self, name: str) -> None: + while name not in self.direct_imports and "." in name: + name = name.rsplit(".", 1)[0] + self.required_names.add(name) + + def reexport(self, name: str) -> None: + """Mark a given non qualified name as needed in __all__. + + This means that in case it comes from a module, it should be + imported with an alias even if the alias is the same as the name. + """ + self.require_name(name) + self.reexports.add(name) + + def import_lines(self) -> list[str]: + """The list of required import lines (as strings with python code). + + In order for a module be included in this output, an identifier must be both + 'required' via require_name() and 'imported' via add_import_from() + or add_import() + """ + result = [] + + # To summarize multiple names imported from a same module, we collect those + # in the `module_map` dictionary, mapping a module path to the list of names that should + # be imported from it. the names can also be alias in the form 'original as alias' + module_map: Mapping[str, list[str]] = defaultdict(list) + + for name in sorted( + self.required_names, + key=lambda n: (self.reverse_alias[n], n) if n in self.reverse_alias else (n, ""), + ): + # If we haven't seen this name in an import statement, ignore it + if name not in self.module_for: + continue + + m = self.module_for[name] + if m is not None: + # This name was found in a from ... import ... + # Collect the name in the module_map + if name in self.reverse_alias: + name = f"{self.reverse_alias[name]} as {name}" + elif name in self.reexports: + name = f"{name} as {name}" + module_map[m].append(name) + else: + # This name was found in an import ... + # We can already generate the import line + if name in self.reverse_alias: + source = self.reverse_alias[name] + result.append(f"import {source} as {name}\n") + elif name in self.reexports: + assert "." not in name # Because reexports only has nonqualified names + result.append(f"import {name} as {name}\n") + else: + result.append(f"import {name}\n") + + # Now generate all the from ... import ... lines collected in module_map + for module, names in sorted(module_map.items()): + result.append(f"from {module} import {', '.join(sorted(names))}\n") + return result + + +@mypyc_attr(allow_interpreted_subclasses=True) +class BaseStubGenerator: + # These names should be omitted from generated stubs. + IGNORED_DUNDERS: Final = { + "__all__", + "__author__", + "__about__", + "__copyright__", + "__email__", + "__license__", + "__summary__", + "__title__", + "__uri__", + "__str__", + "__repr__", + "__getstate__", + "__setstate__", + "__slots__", + "__builtins__", + "__cached__", + "__file__", + "__name__", + "__package__", + "__path__", + "__spec__", + "__loader__", + } + TYPING_MODULE_NAMES: Final = ("typing", "typing_extensions") + # Special-cased names that are implicitly exported from the stub (from m import y as y). + EXTRA_EXPORTED: Final = { + "pyasn1_modules.rfc2437.univ", + "pyasn1_modules.rfc2459.char", + "pyasn1_modules.rfc2459.univ", + } + + def __init__( + self, + _all_: list[str] | None = None, + include_private: bool = False, + export_less: bool = False, + include_docstrings: bool = False, + ) -> None: + # Best known value of __all__. + self._all_ = _all_ + self._include_private = include_private + self._include_docstrings = include_docstrings + # Disable implicit exports of package-internal imports? + self.export_less = export_less + self._import_lines: list[str] = [] + self._output: list[str] = [] + # Current indent level (indent is hardcoded to 4 spaces). + self._indent = "" + self._toplevel_names: list[str] = [] + self.import_tracker = ImportTracker() + # Top-level members + self.defined_names: set[str] = set() + self.sig_generators = self.get_sig_generators() + # populated by visit_mypy_file + self.module_name: str = "" + # These are "soft" imports for objects which might appear in annotations but not have + # a corresponding import statement. + self.known_imports = { + "_typeshed": ["Incomplete"], + "typing": ["Any", "TypeVar", "NamedTuple", "TypedDict"], + "collections.abc": ["Generator"], + "typing_extensions": ["ParamSpec", "TypeVarTuple"], + } + + def get_sig_generators(self) -> list[SignatureGenerator]: + return [] + + def resolve_name(self, name: str) -> str: + """Return the full name resolving imports and import aliases.""" + if "." not in name: + real_module = self.import_tracker.module_for.get(name) + real_short = self.import_tracker.reverse_alias.get(name, name) + if real_module is None and real_short not in self.defined_names: + real_module = "builtins" # not imported and not defined, must be a builtin + else: + name_module, real_short = name.split(".", 1) + real_module = self.import_tracker.reverse_alias.get(name_module, name_module) + resolved_name = real_short if real_module is None else f"{real_module}.{real_short}" + return resolved_name + + def add_name(self, fullname: str, require: bool = True) -> str: + """Add a name to be imported and return the name reference. + + The import will be internal to the stub (i.e don't reexport). + """ + module, name = fullname.rsplit(".", 1) + alias = "_" + name if name in self.defined_names else None + while alias in self.defined_names: + alias = "_" + alias + if module != "builtins" or alias: # don't import from builtins unless needed + self.import_tracker.add_import_from(module, [(name, alias)], require=require) + return alias or name + + def add_import_line(self, line: str) -> None: + """Add a line of text to the import section, unless it's already there.""" + if line not in self._import_lines: + self._import_lines.append(line) + + def get_imports(self) -> str: + """Return the import statements for the stub.""" + imports = "" + if self._import_lines: + imports += "".join(self._import_lines) + imports += "".join(self.import_tracker.import_lines()) + return imports + + def output(self) -> str: + """Return the text for the stub.""" + pieces: list[str] = [] + if imports := self.get_imports(): + pieces.append(imports) + if dunder_all := self.get_dunder_all(): + pieces.append(dunder_all) + if self._output: + pieces.append("".join(self._output)) + return "\n".join(pieces) + + def get_dunder_all(self) -> str: + """Return the __all__ list for the stub.""" + if self._all_: + # Note we emit all names in the runtime __all__ here, even if they + # don't actually exist. If that happens, the runtime has a bug, and + # it's not obvious what the correct behavior should be. We choose + # to reflect the runtime __all__ as closely as possible. + return f"__all__ = {self._all_!r}\n" + return "" + + def add(self, string: str) -> None: + """Add text to generated stub.""" + self._output.append(string) + + def is_top_level(self) -> bool: + """Are we processing the top level of a file?""" + return self._indent == "" + + def indent(self) -> None: + """Add one level of indentation.""" + self._indent += " " + + def dedent(self) -> None: + """Remove one level of indentation.""" + self._indent = self._indent[:-4] + + def record_name(self, name: str) -> None: + """Mark a name as defined. + + This only does anything if at the top level of a module. + """ + if self.is_top_level(): + self._toplevel_names.append(name) + + def is_recorded_name(self, name: str) -> bool: + """Has this name been recorded previously?""" + return self.is_top_level() and name in self._toplevel_names + + def set_defined_names(self, defined_names: set[str]) -> None: + self.defined_names = defined_names + # Names in __all__ are required + for name in self._all_ or (): + self.import_tracker.reexport(name) + + for pkg, imports in self.known_imports.items(): + for t in imports: + # require=False means that the import won't be added unless require_name() is called + # for the object during generation. + self.add_name(f"{pkg}.{t}", require=False) + + def check_undefined_names(self) -> None: + undefined_names = [name for name in self._all_ or [] if name not in self._toplevel_names] + if undefined_names: + if self._output: + self.add("\n") + self.add("# Names in __all__ with no definition:\n") + for name in sorted(undefined_names): + self.add(f"# {name}\n") + + def get_signatures( + self, + default_signature: FunctionSig, + sig_generators: list[SignatureGenerator], + func_ctx: FunctionContext, + ) -> list[FunctionSig]: + for sig_gen in sig_generators: + inferred = sig_gen.get_function_sig(default_signature, func_ctx) + if inferred: + return inferred + + return [default_signature] + + def get_property_type( + self, + default_type: str | None, + sig_generators: list[SignatureGenerator], + func_ctx: FunctionContext, + ) -> str | None: + for sig_gen in sig_generators: + inferred = sig_gen.get_property_type(default_type, func_ctx) + if inferred: + return inferred + + return default_type + + def format_func_def( + self, + sigs: list[FunctionSig], + is_coroutine: bool = False, + decorators: list[str] | None = None, + docstring: str | None = None, + ) -> list[str]: + lines: list[str] = [] + if decorators is None: + decorators = [] + + for signature in sigs: + # dump decorators, just before "def ..." + for deco in decorators: + lines.append(f"{self._indent}{deco}") + + lines.append( + signature.format_sig( + indent=self._indent, + is_async=is_coroutine, + docstring=docstring if self._include_docstrings else None, + ) + ) + return lines + + def format_type_args(self, o: TypeAliasStmt | FuncDef | ClassDef) -> str: + if not o.type_args: + return "" + p = AnnotationPrinter(self) + type_args_list: list[str] = [] + for type_arg in o.type_args: + if type_arg.kind == PARAM_SPEC_KIND: + prefix = "**" + elif type_arg.kind == TYPE_VAR_TUPLE_KIND: + prefix = "*" + else: + prefix = "" + if type_arg.upper_bound: + bound_or_values = f": {type_arg.upper_bound.accept(p)}" + elif type_arg.values: + bound_or_values = f": ({', '.join(v.accept(p) for v in type_arg.values)})" + else: + bound_or_values = "" + if type_arg.default: + default = f" = {type_arg.default.accept(p)}" + else: + default = "" + type_args_list.append(f"{prefix}{type_arg.name}{bound_or_values}{default}") + return "[" + ", ".join(type_args_list) + "]" + + def print_annotation( + self, + t: Type, + known_modules: list[str] | None = None, + local_modules: list[str] | None = None, + ) -> str: + printer = AnnotationPrinter(self, known_modules, local_modules) + return t.accept(printer) + + def is_not_in_all(self, name: str) -> bool: + if self.is_private_name(name): + return False + if self._all_: + return self.is_top_level() and name not in self._all_ + return False + + def is_private_name(self, name: str, fullname: str | None = None) -> bool: + if "__mypy-" in name: + return True # Never include mypy generated symbols + if self._include_private: + return False + if fullname in self.EXTRA_EXPORTED: + return False + if name == "_": + return False + if not name.startswith("_"): + return False + if self._all_ and name in self._all_: + return False + if name.startswith("__") and name.endswith("__"): + return name in self.IGNORED_DUNDERS + return True + + def should_reexport(self, name: str, full_module: str, name_is_alias: bool) -> bool: + if ( + not name_is_alias + and self.module_name + and (self.module_name + "." + name) in self.EXTRA_EXPORTED + ): + # Special case certain names that should be exported, against our general rules. + return True + if name_is_alias: + return False + if self.export_less: + return False + if not self.module_name: + return False + is_private = self.is_private_name(name, full_module + "." + name) + if is_private: + return False + top_level = full_module.split(".")[0] + self_top_level = self.module_name.split(".", 1)[0] + if top_level not in (self_top_level, "_" + self_top_level): + # Export imports from the same package, since we can't reliably tell whether they + # are part of the public API. + return False + if self._all_: + return name in self._all_ + return True diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 1c9c6f853e19..a26aaf798b58 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,296 +1,2113 @@ -from typing import cast, List, Dict +from __future__ import annotations + +from contextlib import contextmanager +from typing import Any, Callable, Final, Iterator, List, TypeVar, cast +from typing_extensions import TypeAlias as _TypeAlias -from mypy.types import ( - Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVar, Callable, TupleType, UnionType, Overloaded, ErasedType, TypeList, - is_named_instance -) import mypy.applytype import mypy.constraints -from mypy import messages, sametypes -from mypy.nodes import TypeInfo -from mypy.expandtype import expand_type +import mypy.typeops +from mypy.erasetype import erase_type +from mypy.expandtype import ( + expand_self_type, + expand_type, + expand_type_by_instance, + freshen_function_type_vars, +) from mypy.maptype import map_instance_to_supertype +# Circular import; done in the function instead. +# import mypy.solve +from mypy.nodes import ( + ARG_STAR, + ARG_STAR2, + CONTRAVARIANT, + COVARIANT, + INVARIANT, + VARIANCE_NOT_READY, + Decorator, + FuncBase, + OverloadedFuncDef, + TypeInfo, + Var, +) +from mypy.options import Options +from mypy.state import state +from mypy.types import ( + MYPYC_NATIVE_INT_NAMES, + TUPLE_LIKE_INSTANCE_NAMES, + TYPED_NAMEDTUPLE_NAMES, + AnyType, + CallableType, + DeletedType, + ErasedType, + FormalArgument, + FunctionLike, + Instance, + LiteralType, + NoneType, + NormalizedCallableType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeType, + TypeVarTupleType, + TypeVarType, + TypeVisitor, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + find_unpack_in_list, + get_proper_type, + is_named_instance, + split_with_prefix_and_suffix, +) +from mypy.types_utils import flatten_types +from mypy.typestate import SubtypeKind, type_state +from mypy.typevars import fill_typevars, fill_typevars_with_any + +# Flags for detected protocol members +IS_SETTABLE: Final = 1 +IS_CLASSVAR: Final = 2 +IS_CLASS_OR_STATIC: Final = 3 +IS_VAR: Final = 4 + +TypeParameterChecker: _TypeAlias = Callable[[Type, Type, int, bool, "SubtypeContext"], bool] -def is_immutable(t: Instance) -> bool: - # TODO: The name is confusing, since the values need not be immutable. - return t.type.fullname() in ('typing.Iterable', - 'typing.Sequence', - 'typing.Reversible', - ) +class SubtypeContext: + def __init__( + self, + *, + # Non-proper subtype flags + ignore_type_params: bool = False, + ignore_pos_arg_names: bool = False, + ignore_declared_variance: bool = False, + # Supported for both proper and non-proper + always_covariant: bool = False, + ignore_promotions: bool = False, + # Proper subtype flags + erase_instances: bool = False, + keep_erased_types: bool = False, + options: Options | None = None, + ) -> None: + self.ignore_type_params = ignore_type_params + self.ignore_pos_arg_names = ignore_pos_arg_names + self.ignore_declared_variance = ignore_declared_variance + self.always_covariant = always_covariant + self.ignore_promotions = ignore_promotions + self.erase_instances = erase_instances + self.keep_erased_types = keep_erased_types + self.options = options + + def check_context(self, proper_subtype: bool) -> None: + # Historically proper and non-proper subtypes were defined using different helpers + # and different visitors. Check if flag values are such that we definitely support. + if proper_subtype: + assert not self.ignore_pos_arg_names and not self.ignore_declared_variance + else: + assert not self.erase_instances and not self.keep_erased_types -def is_subtype(left: Type, right: Type) -> bool: + +def is_subtype( + left: Type, + right: Type, + *, + subtype_context: SubtypeContext | None = None, + ignore_type_params: bool = False, + ignore_pos_arg_names: bool = False, + ignore_declared_variance: bool = False, + always_covariant: bool = False, + ignore_promotions: bool = False, + options: Options | None = None, +) -> bool: """Is 'left' subtype of 'right'? Also consider Any to be a subtype of any type, and vice versa. This recursively applies to components of composite types (List[int] is subtype of List[Any], for example). + + type_parameter_checker is used to check the type parameters (for example, + A with B in is_subtype(C[A], C[B]). The default checks for subtype relation + between the type arguments (e.g., A and B), taking the variance of the + type var into account. """ - if (isinstance(right, AnyType) or isinstance(right, UnboundType) - or isinstance(right, ErasedType)): + if subtype_context is None: + subtype_context = SubtypeContext( + ignore_type_params=ignore_type_params, + ignore_pos_arg_names=ignore_pos_arg_names, + ignore_declared_variance=ignore_declared_variance, + always_covariant=always_covariant, + ignore_promotions=ignore_promotions, + options=options, + ) + else: + assert not any( + { + ignore_type_params, + ignore_pos_arg_names, + ignore_declared_variance, + always_covariant, + ignore_promotions, + options, + } + ), "Don't pass both context and individual flags" + if type_state.is_assumed_subtype(left, right): return True - elif isinstance(left, UnionType): - return all(is_subtype(item, right) for item in left.items) - elif isinstance(right, UnionType): - return any(is_subtype(left, item) for item in right.items) + if mypy.typeops.is_recursive_pair(left, right): + # This case requires special care because it may cause infinite recursion. + # Our view on recursive types is known under a fancy name of iso-recursive mu-types. + # Roughly this means that a recursive type is defined as an alias where right hand side + # can refer to the type as a whole, for example: + # A = Union[int, Tuple[A, ...]] + # and an alias unrolled once represents the *same type*, in our case all these represent + # the same type: + # A + # Union[int, Tuple[A, ...]] + # Union[int, Tuple[Union[int, Tuple[A, ...]], ...]] + # The algorithm for subtyping is then essentially under the assumption that left <: right, + # check that get_proper_type(left) <: get_proper_type(right). On the example above, + # If we start with: + # A = Union[int, Tuple[A, ...]] + # B = Union[int, Tuple[B, ...]] + # When checking if A <: B we push pair (A, B) onto 'assuming' stack, then when after few + # steps we come back to initial call is_subtype(A, B) and immediately return True. + with pop_on_exit(type_state.get_assumptions(is_proper=False), left, right): + return _is_subtype(left, right, subtype_context, proper_subtype=False) + return _is_subtype(left, right, subtype_context, proper_subtype=False) + + +def is_proper_subtype( + left: Type, + right: Type, + *, + subtype_context: SubtypeContext | None = None, + ignore_promotions: bool = False, + erase_instances: bool = False, + keep_erased_types: bool = False, +) -> bool: + """Is left a proper subtype of right? + + For proper subtypes, there's no need to rely on compatibility due to + Any types. Every usable type is a proper subtype of itself. + + If erase_instances is True, erase left instance *after* mapping it to supertype + (this is useful for runtime isinstance() checks). If keep_erased_types is True, + do not consider ErasedType a subtype of all types (used by type inference against unions). + """ + if subtype_context is None: + subtype_context = SubtypeContext( + ignore_promotions=ignore_promotions, + erase_instances=erase_instances, + keep_erased_types=keep_erased_types, + ) else: - return left.accept(SubtypeVisitor(right)) + assert not any( + {ignore_promotions, erase_instances, keep_erased_types} + ), "Don't pass both context and individual flags" + if type_state.is_assumed_proper_subtype(left, right): + return True + if mypy.typeops.is_recursive_pair(left, right): + # Same as for non-proper subtype, see detailed comment there for explanation. + with pop_on_exit(type_state.get_assumptions(is_proper=True), left, right): + return _is_subtype(left, right, subtype_context, proper_subtype=True) + return _is_subtype(left, right, subtype_context, proper_subtype=True) -def is_equivalent(a: Type, b: Type) -> bool: - return is_subtype(a, b) and is_subtype(b, a) +def is_equivalent( + a: Type, + b: Type, + *, + ignore_type_params: bool = False, + ignore_pos_arg_names: bool = False, + options: Options | None = None, + subtype_context: SubtypeContext | None = None, +) -> bool: + return is_subtype( + a, + b, + ignore_type_params=ignore_type_params, + ignore_pos_arg_names=ignore_pos_arg_names, + options=options, + subtype_context=subtype_context, + ) and is_subtype( + b, + a, + ignore_type_params=ignore_type_params, + ignore_pos_arg_names=ignore_pos_arg_names, + options=options, + subtype_context=subtype_context, + ) -class SubtypeVisitor(TypeVisitor[bool]): - def __init__(self, right: Type) -> None: - self.right = right +def is_same_type( + a: Type, b: Type, ignore_promotions: bool = True, subtype_context: SubtypeContext | None = None +) -> bool: + """Are these types proper subtypes of each other? - # visit_x(left) means: is left (which is an instance of X) a subtype of - # right? + This means types may have different representation (e.g. an alias, or + a non-simplified union) but are semantically exchangeable in all contexts. + """ + # First, use fast path for some common types. This is performance-critical. + if ( + type(a) is Instance + and type(b) is Instance + and a.type == b.type + and len(a.args) == len(b.args) + and a.last_known_value is b.last_known_value + ): + return all(is_same_type(x, y) for x, y in zip(a.args, b.args)) + elif isinstance(a, TypeVarType) and isinstance(b, TypeVarType) and a.id == b.id: + return True - def visit_unbound_type(self, left: UnboundType) -> bool: + # Note that using ignore_promotions=True (default) makes types like int and int64 + # considered not the same type (which is the case at runtime). + # Also Union[bool, int] (if it wasn't simplified before) will be different + # from plain int, etc. + return is_proper_subtype( + a, b, ignore_promotions=ignore_promotions, subtype_context=subtype_context + ) and is_proper_subtype( + b, a, ignore_promotions=ignore_promotions, subtype_context=subtype_context + ) + + +# This is a common entry point for subtyping checks (both proper and non-proper). +# Never call this private function directly, use the public versions. +def _is_subtype( + left: Type, right: Type, subtype_context: SubtypeContext, proper_subtype: bool +) -> bool: + subtype_context.check_context(proper_subtype) + orig_right = right + orig_left = left + left = get_proper_type(left) + right = get_proper_type(right) + + # Note: Unpack type should not be a subtype of Any, since it may represent + # multiple types. This should always go through the visitor, to check arity. + if ( + not proper_subtype + and isinstance(right, (AnyType, UnboundType, ErasedType)) + and not isinstance(left, UnpackType) + ): + # TODO: should we consider all types proper subtypes of UnboundType and/or + # ErasedType as we do for non-proper subtyping. return True - def visit_error_type(self, left: ErrorType) -> bool: - return False + if isinstance(right, UnionType) and not isinstance(left, UnionType): + # Normally, when 'left' is not itself a union, the only way + # 'left' can be a subtype of the union 'right' is if it is a + # subtype of one of the items making up the union. + if proper_subtype: + is_subtype_of_item = any( + is_proper_subtype(orig_left, item, subtype_context=subtype_context) + for item in right.items + ) + else: + is_subtype_of_item = any( + is_subtype(orig_left, item, subtype_context=subtype_context) + for item in right.items + ) + # Recombine rhs literal types, to make an enum type a subtype + # of a union of all enum items as literal types. Only do it if + # the previous check didn't succeed, since recombining can be + # expensive. + # `bool` is a special case, because `bool` is `Literal[True, False]`. + if ( + not is_subtype_of_item + and isinstance(left, Instance) + and (left.type.is_enum or left.type.fullname == "builtins.bool") + ): + right = UnionType(mypy.typeops.try_contracting_literals_in_union(right.items)) + if proper_subtype: + is_subtype_of_item = any( + is_proper_subtype(orig_left, item, subtype_context=subtype_context) + for item in right.items + ) + else: + is_subtype_of_item = any( + is_subtype(orig_left, item, subtype_context=subtype_context) + for item in right.items + ) + # However, if 'left' is a type variable T, T might also have + # an upper bound which is itself a union. This case will be + # handled below by the SubtypeVisitor. We have to check both + # possibilities, to handle both cases like T <: Union[T, U] + # and cases like T <: B where B is the upper bound of T and is + # a union. (See #2314.) + if not isinstance(left, TypeVarType): + return is_subtype_of_item + elif is_subtype_of_item: + return True + # otherwise, fall through + return left.accept(SubtypeVisitor(orig_right, subtype_context, proper_subtype)) - def visit_type_list(self, t: TypeList) -> bool: - assert False, 'Not supported' - def visit_any(self, left: AnyType) -> bool: +def check_type_parameter( + left: Type, right: Type, variance: int, proper_subtype: bool, subtype_context: SubtypeContext +) -> bool: + # It is safe to consider empty collection literals and similar as covariant, since + # such type can't be stored in a variable, see checker.is_valid_inferred_type(). + if variance == INVARIANT: + p_left = get_proper_type(left) + if isinstance(p_left, UninhabitedType) and p_left.ambiguous: + variance = COVARIANT + # If variance hasn't been inferred yet, we are lenient and default to + # covariance. This shouldn't happen often, but it's very difficult to + # avoid these cases altogether. + if variance == COVARIANT or variance == VARIANCE_NOT_READY: + if proper_subtype: + return is_proper_subtype(left, right, subtype_context=subtype_context) + else: + return is_subtype(left, right, subtype_context=subtype_context) + elif variance == CONTRAVARIANT: + if proper_subtype: + return is_proper_subtype(right, left, subtype_context=subtype_context) + else: + return is_subtype(right, left, subtype_context=subtype_context) + else: + if proper_subtype: + # We pass ignore_promotions=False because it is a default for subtype checks. + # The actual value will be taken from the subtype_context, and it is whatever + # the original caller passed. + return is_same_type( + left, right, ignore_promotions=False, subtype_context=subtype_context + ) + else: + return is_equivalent(left, right, subtype_context=subtype_context) + + +class SubtypeVisitor(TypeVisitor[bool]): + def __init__(self, right: Type, subtype_context: SubtypeContext, proper_subtype: bool) -> None: + self.right = get_proper_type(right) + self.orig_right = right + self.proper_subtype = proper_subtype + self.subtype_context = subtype_context + self.options = subtype_context.options + self._subtype_kind = SubtypeVisitor.build_subtype_kind(subtype_context, proper_subtype) + + @staticmethod + def build_subtype_kind(subtype_context: SubtypeContext, proper_subtype: bool) -> SubtypeKind: + return ( + state.strict_optional, + proper_subtype, + subtype_context.ignore_type_params, + subtype_context.ignore_pos_arg_names, + subtype_context.ignore_declared_variance, + subtype_context.always_covariant, + subtype_context.ignore_promotions, + subtype_context.erase_instances, + subtype_context.keep_erased_types, + ) + + def _is_subtype(self, left: Type, right: Type) -> bool: + if self.proper_subtype: + return is_proper_subtype(left, right, subtype_context=self.subtype_context) + return is_subtype(left, right, subtype_context=self.subtype_context) + + # visit_x(left) means: is left (which is an instance of X) a subtype of right? + + def visit_unbound_type(self, left: UnboundType) -> bool: + # This can be called if there is a bad type annotation. The result probably + # doesn't matter much but by returning True we simplify these bad types away + # from unions, which could filter out some bogus messages. return True - def visit_void(self, left: Void) -> bool: - return isinstance(self.right, Void) + def visit_any(self, left: AnyType) -> bool: + return isinstance(self.right, AnyType) if self.proper_subtype else True - def visit_none_type(self, left: NoneTyp) -> bool: - return not isinstance(self.right, Void) + def visit_none_type(self, left: NoneType) -> bool: + if state.strict_optional: + if isinstance(self.right, NoneType) or is_named_instance( + self.right, "builtins.object" + ): + return True + if isinstance(self.right, Instance) and self.right.type.is_protocol: + members = self.right.type.protocol_members + # None is compatible with Hashable (and other similar protocols). This is + # slightly sloppy since we don't check the signature of "__hash__". + # None is also compatible with `SupportsStr` protocol. + return not members or all(member in ("__hash__", "__str__") for member in members) + return False + else: + return True + + def visit_uninhabited_type(self, left: UninhabitedType) -> bool: + return True def visit_erased_type(self, left: ErasedType) -> bool: + # This may be encountered during type inference. The result probably doesn't + # matter much. + # TODO: it actually does matter, figure out more principled logic about this. + return not self.subtype_context.keep_erased_types + + def visit_deleted_type(self, left: DeletedType) -> bool: return True def visit_instance(self, left: Instance) -> bool: + if left.type.fallback_to_any and not self.proper_subtype: + # NOTE: `None` is a *non-subclassable* singleton, therefore no class + # can by a subtype of it, even with an `Any` fallback. + # This special case is needed to treat descriptors in classes with + # dynamic base classes correctly, see #5456. + return not isinstance(self.right, NoneType) right = self.right + if isinstance(right, TupleType) and right.partial_fallback.type.is_enum: + return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) + if isinstance(right, TupleType): + if len(right.items) == 1: + # Non-normalized Tuple type (may be left after semantic analysis + # because semanal_typearg visitor is not a type translator). + item = right.items[0] + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, Instance): + return self._is_subtype(left, unpacked) + if left.type.has_base(right.partial_fallback.type.fullname): + if not self.proper_subtype: + # Special case to consider Foo[*tuple[Any, ...]] (i.e. bare Foo) a + # subtype of Foo[], when Foo is user defined variadic tuple type. + mapped = map_instance_to_supertype(left, right.partial_fallback.type) + for arg in map(get_proper_type, mapped.args): + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if not isinstance(unpacked, Instance): + break + assert unpacked.type.fullname == "builtins.tuple" + if not isinstance(get_proper_type(unpacked.args[0]), AnyType): + break + elif not isinstance(arg, AnyType): + break + else: + return True + return False + if isinstance(right, TypeVarTupleType): + # tuple[Any, ...] is like Any in the world of tuples (see special case above). + if left.type.has_base("builtins.tuple"): + mapped = map_instance_to_supertype(left, right.tuple_fallback.type) + if isinstance(get_proper_type(mapped.args[0]), AnyType): + return not self.proper_subtype if isinstance(right, Instance): - if left.type.ducktype and is_subtype(left.type.ducktype, - self.right): + if type_state.is_cached_subtype_check(self._subtype_kind, left, right): return True - rname = right.type.fullname() - if not left.type.has_base(rname) and rname != 'builtins.object': + if type_state.is_cached_negative_subtype_check(self._subtype_kind, left, right): return False - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(left, right.type) - if not is_immutable(right): - result = all(is_equivalent(ta, ra) for (ta, ra) in - zip(t.args, right.args)) - else: - result = all(is_subtype(ta, ra) for (ta, ra) in - zip(t.args, right.args)) - return result + if not self.subtype_context.ignore_promotions: + for base in left.type.mro: + if base._promote and any( + self._is_subtype(p, self.right) for p in base._promote + ): + type_state.record_subtype_cache_entry(self._subtype_kind, left, right) + return True + # Special case: Low-level integer types are compatible with 'int'. We can't + # use promotions, since 'int' is already promoted to low-level integer types, + # and we can't have circular promotions. + if left.type.alt_promote and left.type.alt_promote.type is right.type: + return True + rname = right.type.fullname + # Always try a nominal check if possible, + # there might be errors that a user wants to silence *once*. + # NamedTuples are a special case, because `NamedTuple` is not listed + # in `TypeInfo.mro`, so when `(a: NamedTuple) -> None` is used, + # we need to check for `is_named_tuple` property + if ( + left.type.has_base(rname) + or rname == "builtins.object" + or ( + rname in TYPED_NAMEDTUPLE_NAMES + and any(l.is_named_tuple for l in left.type.mro) + ) + ) and not self.subtype_context.ignore_declared_variance: + # Map left type to corresponding right instances. + t = map_instance_to_supertype(left, right.type) + if self.subtype_context.erase_instances: + erased = erase_type(t) + assert isinstance(erased, Instance) + t = erased + nominal = True + if right.type.has_type_var_tuple_type: + # For variadic instances we simply find the correct type argument mappings, + # all the heavy lifting is done by the tuple subtyping. + assert right.type.type_var_tuple_prefix is not None + assert right.type.type_var_tuple_suffix is not None + prefix = right.type.type_var_tuple_prefix + suffix = right.type.type_var_tuple_suffix + tvt = right.type.defn.type_vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + left_prefix, left_middle, left_suffix = split_with_prefix_and_suffix( + t.args, prefix, suffix + ) + right_prefix, right_middle, right_suffix = split_with_prefix_and_suffix( + right.args, prefix, suffix + ) + left_args = ( + left_prefix + (TupleType(list(left_middle), fallback),) + left_suffix + ) + right_args = ( + right_prefix + (TupleType(list(right_middle), fallback),) + right_suffix + ) + if not self.proper_subtype and t.args: + for arg in map(get_proper_type, t.args): + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if not isinstance(unpacked, Instance): + break + assert unpacked.type.fullname == "builtins.tuple" + if not isinstance(get_proper_type(unpacked.args[0]), AnyType): + break + elif not isinstance(arg, AnyType): + break + else: + return True + if len(left_args) != len(right_args): + return False + type_params = zip(left_args, right_args, right.type.defn.type_vars) + else: + type_params = zip(t.args, right.args, right.type.defn.type_vars) + if not self.subtype_context.ignore_type_params: + tried_infer = False + for lefta, righta, tvar in type_params: + if isinstance(tvar, TypeVarType): + if tvar.variance == VARIANCE_NOT_READY and not tried_infer: + infer_class_variances(right.type) + tried_infer = True + if ( + self.subtype_context.always_covariant + and tvar.variance == INVARIANT + ): + variance = COVARIANT + else: + variance = tvar.variance + if not check_type_parameter( + lefta, righta, variance, self.proper_subtype, self.subtype_context + ): + nominal = False + else: + # TODO: everywhere else ParamSpecs are handled as invariant. + if not check_type_parameter( + lefta, righta, COVARIANT, self.proper_subtype, self.subtype_context + ): + nominal = False + if nominal: + type_state.record_subtype_cache_entry(self._subtype_kind, left, right) + else: + type_state.record_negative_subtype_cache_entry(self._subtype_kind, left, right) + return nominal + if right.type.is_protocol and is_protocol_implementation( + left, right, proper_subtype=self.proper_subtype, options=self.options + ): + return True + # We record negative cache entry here, and not in the protocol check like we do for + # positive cache, to avoid accidentally adding a type that is not a structural + # subtype, but is a nominal subtype (involving type: ignore override). + type_state.record_negative_subtype_cache_entry(self._subtype_kind, left, right) + return False + if isinstance(right, TypeType): + item = right.item + if isinstance(item, TupleType): + item = mypy.typeops.tuple_fallback(item) + # TODO: this is a bit arbitrary, we should only skip Any-related cases. + if not self.proper_subtype: + if is_named_instance(left, "builtins.type"): + return self._is_subtype(TypeType(AnyType(TypeOfAny.special_form)), right) + if left.type.is_metaclass(): + if isinstance(item, AnyType): + return True + if isinstance(item, Instance): + return is_named_instance(item, "builtins.object") + if isinstance(right, LiteralType) and left.last_known_value is not None: + return self._is_subtype(left.last_known_value, right) + if isinstance(right, FunctionLike): + # Special case: Instance can be a subtype of Callable / Overloaded. + call = find_member("__call__", left, left, is_operator=True) + if call: + return self._is_subtype(call, right) + return False else: return False - def visit_type_var(self, left: TypeVar) -> bool: + def visit_type_var(self, left: TypeVarType) -> bool: right = self.right - if isinstance(right, TypeVar): - return left.name == right.name - else: - return is_named_instance(self.right, 'builtins.object') + if isinstance(right, TypeVarType) and left.id == right.id: + return True + if left.values and self._is_subtype(UnionType.make_union(left.values), right): + return True + return self._is_subtype(left.upper_bound, self.right) - def visit_callable(self, left: Callable) -> bool: + def visit_param_spec(self, left: ParamSpecType) -> bool: right = self.right - if isinstance(right, Callable): - return is_callable_subtype(left, right) - elif isinstance(right, Overloaded): - return all(is_subtype(left, item) for item in right.items()) - elif is_named_instance(right, 'builtins.object'): + if ( + isinstance(right, ParamSpecType) + and right.id == left.id + and right.flavor == left.flavor + ): + return self._is_subtype(left.prefix, right.prefix) + if isinstance(right, Parameters) and are_trivial_parameters(right): return True - elif (is_named_instance(right, 'builtins.type') and - left.is_type_obj()): + return self._is_subtype(left.upper_bound, self.right) + + def visit_type_var_tuple(self, left: TypeVarTupleType) -> bool: + right = self.right + if isinstance(right, TypeVarTupleType) and right.id == left.id: + return left.min_len >= right.min_len + return self._is_subtype(left.upper_bound, self.right) + + def visit_unpack_type(self, left: UnpackType) -> bool: + # TODO: Ideally we should not need this (since it is not a real type). + # Instead callers (upper level types) should handle it when it appears in type list. + if isinstance(self.right, UnpackType): + return self._is_subtype(left.type, self.right.type) + if isinstance(self.right, Instance) and self.right.type.fullname == "builtins.object": return True + return False + + def visit_parameters(self, left: Parameters) -> bool: + if isinstance(self.right, Parameters): + return are_parameters_compatible( + left, + self.right, + is_compat=self._is_subtype, + # TODO: this should pass the current value, but then couple tests fail. + is_proper_subtype=False, + ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, + ) + elif isinstance(self.right, Instance): + return self.right.type.fullname == "builtins.object" + else: + return False + + def visit_callable_type(self, left: CallableType) -> bool: + right = self.right + if isinstance(right, CallableType): + if left.type_guard is not None and right.type_guard is not None: + if not self._is_subtype(left.type_guard, right.type_guard): + return False + elif left.type_is is not None and right.type_is is not None: + # For TypeIs we have to check both ways; it is unsafe to pass + # a TypeIs[Child] when a TypeIs[Parent] is expected, because + # if the narrower returns False, we assume that the narrowed value is + # *not* a Parent. + if not self._is_subtype(left.type_is, right.type_is) or not self._is_subtype( + right.type_is, left.type_is + ): + return False + elif right.type_guard is not None and left.type_guard is None: + # This means that one function has `TypeGuard` and other does not. + # They are not compatible. See https://github.com/python/mypy/issues/11307 + return False + elif right.type_is is not None and left.type_is is None: + # Similarly, if one function has `TypeIs` and the other does not, + # they are not compatible. + return False + return is_callable_compatible( + left, + right, + is_compat=self._is_subtype, + is_proper_subtype=self.proper_subtype, + ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, + strict_concatenate=( + (self.options.extra_checks or self.options.strict_concatenate) + if self.options + else False + ), + ) + elif isinstance(right, Overloaded): + return all(self._is_subtype(left, item) for item in right.items) + elif isinstance(right, Instance): + if right.type.is_protocol and "__call__" in right.type.protocol_members: + # OK, a callable can implement a protocol with a `__call__` member. + # TODO: we should probably explicitly exclude self-types in this case. + call = find_member("__call__", right, left, is_operator=True) + assert call is not None + if self._is_subtype(left, call): + if len(right.type.protocol_members) == 1: + return True + if is_protocol_implementation(left.fallback, right, skip=["__call__"]): + return True + if right.type.is_protocol and left.is_type_obj(): + ret_type = get_proper_type(left.ret_type) + if isinstance(ret_type, TupleType): + ret_type = mypy.typeops.tuple_fallback(ret_type) + if isinstance(ret_type, Instance) and is_protocol_implementation( + ret_type, right, proper_subtype=self.proper_subtype, class_obj=True + ): + return True + return self._is_subtype(left.fallback, right) + elif isinstance(right, TypeType): + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and self._is_subtype(left.ret_type, right.item) else: return False def visit_tuple_type(self, left: TupleType) -> bool: right = self.right if isinstance(right, Instance): - if (is_named_instance(right, 'builtins.object') or - is_named_instance(right, 'builtins.tuple')): + if is_named_instance(right, "typing.Sized"): + return True + elif is_named_instance(right, TUPLE_LIKE_INSTANCE_NAMES): + if right.args: + iter_type = right.args[0] + else: + if self.proper_subtype: + return False + iter_type = AnyType(TypeOfAny.special_form) + if is_named_instance(right, "builtins.tuple") and isinstance( + get_proper_type(iter_type), AnyType + ): + # TODO: We shouldn't need this special case. This is currently needed + # for isinstance(x, tuple), though it's unclear why. + return True + for li in left.items: + if isinstance(li, UnpackType): + unpack = get_proper_type(li.type) + if isinstance(unpack, TypeVarTupleType): + unpack = get_proper_type(unpack.upper_bound) + assert ( + isinstance(unpack, Instance) + and unpack.type.fullname == "builtins.tuple" + ) + li = unpack.args[0] + if not self._is_subtype(li, iter_type): + return False + return True + elif self._is_subtype(left.partial_fallback, right) and self._is_subtype( + mypy.typeops.tuple_fallback(left), right + ): return True - elif (is_named_instance(right, 'typing.Iterable') or - is_named_instance(right, 'typing.Sequence') or - is_named_instance(right, 'typing.Reversible')): - iter_type = right.args[0] - return all(is_subtype(li, iter_type) for li in left.items) return False elif isinstance(right, TupleType): + # If right has a variadic unpack this needs special handling. If there is a TypeVarTuple + # unpack, item count must coincide. If the left has variadic unpack but right + # doesn't have one, we will fall through to False down the line. + if self.variadic_tuple_subtype(left, right): + return True if len(left.items) != len(right.items): return False - for i in range(len(left.items)): - if not is_subtype(left.items[i], right.items[i]): + if any(not self._is_subtype(l, r) for l, r in zip(left.items, right.items)): + return False + if is_named_instance(right.partial_fallback, "builtins.tuple"): + # No need to verify fallback. This is useful since the calculated fallback + # may be inconsistent due to how we calculate joins between unions vs. + # non-unions. For example, join(int, str) == object, whereas + # join(Union[int, C], Union[str, C]) == Union[int, str, C]. + return True + if is_named_instance(left.partial_fallback, "builtins.tuple"): + # Again, no need to verify. At this point we know the right fallback + # is a subclass of tuple, so if left is plain tuple, it cannot be a subtype. + return False + # At this point we know both fallbacks are non-tuple. + return self._is_subtype(left.partial_fallback, right.partial_fallback) + else: + return False + + def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool: + """Check subtyping between two potentially variadic tuples. + + Most non-trivial cases here are due to variadic unpacks like *tuple[X, ...], + we handle such unpacks as infinite unions Tuple[()] | Tuple[X] | Tuple[X, X] | ... + + Note: the cases where right is fixed or has *Ts unpack should be handled + by the caller. + """ + right_unpack_index = find_unpack_in_list(right.items) + if right_unpack_index is None: + # This case should be handled by the caller. + return False + right_unpack = right.items[right_unpack_index] + assert isinstance(right_unpack, UnpackType) + right_unpacked = get_proper_type(right_unpack.type) + if not isinstance(right_unpacked, Instance): + # This case should be handled by the caller. + return False + assert right_unpacked.type.fullname == "builtins.tuple" + right_item = right_unpacked.args[0] + right_prefix = right_unpack_index + right_suffix = len(right.items) - right_prefix - 1 + left_unpack_index = find_unpack_in_list(left.items) + if left_unpack_index is None: + # Simple case: left is fixed, simply find correct mapping to the right + # (effectively selecting item with matching length from an infinite union). + if len(left.items) < right_prefix + right_suffix: + return False + prefix, middle, suffix = split_with_prefix_and_suffix( + tuple(left.items), right_prefix, right_suffix + ) + if not all( + self._is_subtype(li, ri) for li, ri in zip(prefix, right.items[:right_prefix]) + ): + return False + if right_suffix and not all( + self._is_subtype(li, ri) for li, ri in zip(suffix, right.items[-right_suffix:]) + ): + return False + return all(self._is_subtype(li, right_item) for li in middle) + else: + if len(left.items) < len(right.items): + # There are some items on the left that will never have a matching length + # on the right. + return False + left_unpack = left.items[left_unpack_index] + assert isinstance(left_unpack, UnpackType) + left_unpacked = get_proper_type(left_unpack.type) + if not isinstance(left_unpacked, Instance): + # *Ts unpacks can't be split. + return False + assert left_unpacked.type.fullname == "builtins.tuple" + left_item = left_unpacked.args[0] + + # The most tricky case with two variadic unpacks we handle similar to union + # subtyping: *each* item on the left, must be a subtype of *some* item on the right. + # For this we first check the "asymptotic case", i.e. that both unpacks a subtypes, + # and then check subtyping for all finite overlaps. + if not self._is_subtype(left_item, right_item): + return False + left_prefix = left_unpack_index + left_suffix = len(left.items) - left_prefix - 1 + max_overlap = max(0, right_prefix - left_prefix, right_suffix - left_suffix) + for overlap in range(max_overlap + 1): + repr_items = left.items[:left_prefix] + [left_item] * overlap + if left_suffix: + repr_items += left.items[-left_suffix:] + left_repr = left.copy_modified(items=repr_items) + if not self._is_subtype(left_repr, right): return False - if not is_subtype(left.fallback, right.fallback): + return True + + def visit_typeddict_type(self, left: TypedDictType) -> bool: + right = self.right + if isinstance(right, Instance): + return self._is_subtype(left.fallback, right) + elif isinstance(right, TypedDictType): + if left == right: + return True # Fast path + if not left.names_are_wider_than(right): return False + for name, l, r in left.zip(right): + # TODO: should we pass on the full subtype_context here and below? + right_readonly = name in right.readonly_keys + if not right_readonly: + if self.proper_subtype: + check = is_same_type(l, r) + else: + check = is_equivalent( + l, + r, + ignore_type_params=self.subtype_context.ignore_type_params, + options=self.options, + ) + else: + # Read-only items behave covariantly + check = self._is_subtype(l, r) + if not check: + return False + # Non-required key is not compatible with a required key since + # indexing may fail unexpectedly if a required key is missing. + # Required key is not compatible with a non-read-only non-required + # key since the prior doesn't support 'del' but the latter should + # support it. + # Required key is compatible with a read-only non-required key. + required_differ = (name in left.required_keys) != (name in right.required_keys) + if not right_readonly and required_differ: + return False + # Readonly fields check: + # + # A = TypedDict('A', {'x': ReadOnly[int]}) + # B = TypedDict('B', {'x': int}) + # def reset_x(b: B) -> None: + # b['x'] = 0 + # + # So, `A` cannot be a subtype of `B`, while `B` can be a subtype of `A`, + # because you can use `B` everywhere you use `A`, but not the other way around. + if name in left.readonly_keys and name not in right.readonly_keys: + return False + # (NOTE: Fallbacks don't matter.) return True else: return False + def visit_literal_type(self, left: LiteralType) -> bool: + if isinstance(self.right, LiteralType): + return left == self.right + else: + return self._is_subtype(left.fallback, self.right) + def visit_overloaded(self, left: Overloaded) -> bool: right = self.right - if is_named_instance(right, 'builtins.object'): - return True - elif isinstance(right, Callable) or is_named_instance( - right, 'builtins.type'): - for item in left.items(): - if is_subtype(item, right): + if isinstance(right, Instance): + if right.type.is_protocol and "__call__" in right.type.protocol_members: + # same as for CallableType + call = find_member("__call__", right, left, is_operator=True) + assert call is not None + if self._is_subtype(left, call): + if len(right.type.protocol_members) == 1: + return True + if is_protocol_implementation(left.fallback, right, skip=["__call__"]): + return True + return self._is_subtype(left.fallback, right) + elif isinstance(right, CallableType): + for item in left.items: + if self._is_subtype(item, right): return True return False elif isinstance(right, Overloaded): - # TODO: this may be too restrictive - if len(left.items()) != len(right.items()): - return False - for i in range(len(left.items())): - if not is_subtype(left.items()[i], right.items()[i]): + if left == self.right: + # When it is the same overload, then the types are equal. + return True + + # Ensure each overload on the right side (the supertype) is accounted for. + previous_match_left_index = -1 + matched_overloads = set() + + for right_item in right.items: + found_match = False + + for left_index, left_item in enumerate(left.items): + subtype_match = self._is_subtype(left_item, right_item) + + # Order matters: we need to make sure that the index of + # this item is at least the index of the previous one. + if subtype_match and previous_match_left_index <= left_index: + previous_match_left_index = left_index + found_match = True + matched_overloads.add(left_index) + break + else: + # If this one overlaps with the supertype in any way, but it wasn't + # an exact match, then it's a potential error. + strict_concat = ( + (self.options.extra_checks or self.options.strict_concatenate) + if self.options + else False + ) + if left_index not in matched_overloads and ( + is_callable_compatible( + left_item, + right_item, + is_compat=self._is_subtype, + is_proper_subtype=self.proper_subtype, + ignore_return=True, + ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, + strict_concatenate=strict_concat, + ) + or is_callable_compatible( + right_item, + left_item, + is_compat=self._is_subtype, + is_proper_subtype=self.proper_subtype, + ignore_return=True, + ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, + strict_concatenate=strict_concat, + ) + ): + return False + + if not found_match: return False return True elif isinstance(right, UnboundType): return True + elif isinstance(right, TypeType): + # All the items must have the same type object status, so + # it's sufficient to query only (any) one of them. + # This is unsound, we don't check all the __init__ signatures. + return left.is_type_obj() and self._is_subtype(left.items[0], right) else: return False + def visit_union_type(self, left: UnionType) -> bool: + if isinstance(self.right, Instance): + literal_types: set[Instance] = set() + # avoid redundant check for union of literals + for item in left.relevant_items(): + p_item = get_proper_type(item) + lit_type = mypy.typeops.simple_literal_type(p_item) + if lit_type is not None: + if lit_type in literal_types: + continue + literal_types.add(lit_type) + item = lit_type + if not self._is_subtype(item, self.orig_right): + return False + return True -def is_callable_subtype(left: Callable, right: Callable, ignore_return: bool = False) -> bool: - """Is left a subtype of right?""" - # TODO: Support named arguments, **args, etc. - # Non-type cannot be a subtype of type. - if right.is_type_obj() and not left.is_type_obj(): + elif isinstance(self.right, UnionType): + # prune literals early to avoid nasty quadratic behavior which would otherwise arise when checking + # subtype relationships between slightly different narrowings of an Enum + # we achieve O(N+M) instead of O(N*M) + + fast_check: set[ProperType] = set() + + for item in flatten_types(self.right.relevant_items()): + p_item = get_proper_type(item) + fast_check.add(p_item) + if isinstance(p_item, Instance) and p_item.last_known_value is not None: + fast_check.add(p_item.last_known_value) + + for item in left.relevant_items(): + p_item = get_proper_type(item) + if p_item in fast_check: + continue + lit_type = mypy.typeops.simple_literal_type(p_item) + if lit_type in fast_check: + continue + if not self._is_subtype(item, self.orig_right): + return False + return True + + return all(self._is_subtype(item, self.orig_right) for item in left.items) + + def visit_partial_type(self, left: PartialType) -> bool: + # This is indeterminate as we don't really know the complete type yet. + if self.proper_subtype: + # TODO: What's the right thing to do here? + return False + if left.type is None: + # Special case, partial `None`. This might happen when defining + # class-level attributes with explicit `None`. + # We can still recover from this. + # https://github.com/python/mypy/issues/11105 + return self.visit_none_type(NoneType()) + raise RuntimeError(f'Partial type "{left}" cannot be checked with "issubtype()"') + + def visit_type_type(self, left: TypeType) -> bool: + right = self.right + if isinstance(right, TypeType): + return self._is_subtype(left.item, right.item) + if isinstance(right, CallableType): + if self.proper_subtype and not right.is_type_obj(): + # We can't accept `Type[X]` as a *proper* subtype of Callable[P, X] + # since this will break transitivity of subtyping. + return False + # This is unsound, we don't check the __init__ signature. + return self._is_subtype(left.item, right.ret_type) + if isinstance(right, Instance): + if right.type.fullname in ["builtins.object", "builtins.type"]: + # TODO: Strictly speaking, the type builtins.type is considered equivalent to + # Type[Any]. However, this would break the is_proper_subtype check in + # conditional_types for cases like isinstance(x, type) when the type + # of x is Type[int]. It's unclear what's the right way to address this. + return True + item = left.item + if isinstance(item, TypeVarType): + item = get_proper_type(item.upper_bound) + if isinstance(item, Instance): + if right.type.is_protocol and is_protocol_implementation( + item, right, proper_subtype=self.proper_subtype, class_obj=True + ): + return True + metaclass = item.type.metaclass_type + return metaclass is not None and self._is_subtype(metaclass, right) return False - if right.variables: - # Subtyping is not currently supported for generic function as the supertype. + + def visit_type_alias_type(self, left: TypeAliasType) -> bool: + assert False, f"This should be never called, got {left}" + + +T = TypeVar("T", bound=Type) + + +@contextmanager +def pop_on_exit(stack: list[tuple[T, T]], left: T, right: T) -> Iterator[None]: + stack.append((left, right)) + yield + stack.pop() + + +def is_protocol_implementation( + left: Instance, + right: Instance, + proper_subtype: bool = False, + class_obj: bool = False, + skip: list[str] | None = None, + options: Options | None = None, +) -> bool: + """Check whether 'left' implements the protocol 'right'. + + If 'proper_subtype' is True, then check for a proper subtype. + Treat recursive protocols by using the 'assuming' structural subtype matrix + (in sparse representation, i.e. as a list of pairs (subtype, supertype)), + see also comment in nodes.TypeInfo. When we enter a check for classes + (A, P), defined as following:: + + class P(Protocol): + def f(self) -> P: ... + class A: + def f(self) -> A: ... + + this results in A being a subtype of P without infinite recursion. + On every false result, we pop the assumption, thus avoiding an infinite recursion + as well. + """ + assert right.type.is_protocol + if skip is None: + skip = [] + # We need to record this check to generate protocol fine-grained dependencies. + type_state.record_protocol_subtype_check(left.type, right.type) + # nominal subtyping currently ignores '__init__' and '__new__' signatures + members_not_to_check = {"__init__", "__new__"} + members_not_to_check.update(skip) + # Trivial check that circumvents the bug described in issue 9771: + if left.type.is_protocol: + members_right = set(right.type.protocol_members) - members_not_to_check + members_left = set(left.type.protocol_members) - members_not_to_check + if not members_right.issubset(members_left): + return False + assuming = right.type.assuming_proper if proper_subtype else right.type.assuming + for l, r in reversed(assuming): + if l == left and r == right: + return True + with pop_on_exit(assuming, left, right): + for member in right.type.protocol_members: + if member in members_not_to_check: + continue + ignore_names = member != "__call__" # __call__ can be passed kwargs + # The third argument below indicates to what self type is bound. + # We always bind self to the subtype. (Similarly to nominal types). + supertype = get_proper_type(find_member(member, right, left)) + assert supertype is not None + + subtype = mypy.typeops.get_protocol_member(left, member, class_obj) + # Useful for debugging: + # print(member, 'of', left, 'has type', subtype) + # print(member, 'of', right, 'has type', supertype) + if not subtype: + return False + if isinstance(subtype, PartialType): + subtype = ( + NoneType() + if subtype.type is None + else Instance( + subtype.type, + [AnyType(TypeOfAny.unannotated)] * len(subtype.type.type_vars), + ) + ) + if not proper_subtype: + # Nominal check currently ignores arg names + # NOTE: If we ever change this, be sure to also change the call to + # SubtypeVisitor.build_subtype_kind(...) down below. + is_compat = is_subtype( + subtype, supertype, ignore_pos_arg_names=ignore_names, options=options + ) + else: + is_compat = is_proper_subtype(subtype, supertype) + if not is_compat: + return False + if isinstance(subtype, NoneType) and isinstance(supertype, CallableType): + # We want __hash__ = None idiom to work even without --strict-optional + return False + subflags = get_member_flags(member, left, class_obj=class_obj) + superflags = get_member_flags(member, right) + if IS_SETTABLE in superflags: + # Check opposite direction for settable attributes. + if not is_subtype(supertype, subtype, options=options): + return False + if not class_obj: + if IS_SETTABLE not in superflags: + if IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: + return False + elif (IS_CLASSVAR in subflags) != (IS_CLASSVAR in superflags): + return False + else: + if IS_VAR in superflags and IS_CLASSVAR not in subflags: + # Only class variables are allowed for class object access. + return False + if IS_CLASSVAR in superflags: + # This can be never matched by a class object. + return False + if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: + return False + # This rule is copied from nominal check in checker.py + if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: + return False + + if not proper_subtype: + # Nominal check currently ignores arg names, but __call__ is special for protocols + ignore_names = right.type.protocol_members != ["__call__"] + else: + ignore_names = False + subtype_kind = SubtypeVisitor.build_subtype_kind( + subtype_context=SubtypeContext(ignore_pos_arg_names=ignore_names), + proper_subtype=proper_subtype, + ) + type_state.record_subtype_cache_entry(subtype_kind, left, right) + return True + + +def find_member( + name: str, itype: Instance, subtype: Type, is_operator: bool = False, class_obj: bool = False +) -> Type | None: + """Find the type of member by 'name' in 'itype's TypeInfo. + + Find the member type after applying type arguments from 'itype', and binding + 'self' to 'subtype'. Return None if member was not found. + """ + # TODO: this code shares some logic with checkmember.analyze_member_access, + # consider refactoring. + info = itype.type + method = info.get_method(name) + if method: + if isinstance(method, Decorator): + return find_node_type(method.var, itype, subtype, class_obj=class_obj) + if method.is_property: + assert isinstance(method, OverloadedFuncDef) + dec = method.items[0] + assert isinstance(dec, Decorator) + return find_node_type(dec.var, itype, subtype, class_obj=class_obj) + return find_node_type(method, itype, subtype, class_obj=class_obj) + else: + # don't have such method, maybe variable or decorator? + node = info.get(name) + v = node.node if node else None + if isinstance(v, Var): + return find_node_type(v, itype, subtype, class_obj=class_obj) + if ( + not v + and name not in ["__getattr__", "__setattr__", "__getattribute__"] + and not is_operator + and not class_obj + and itype.extra_attrs is None # skip ModuleType.__getattr__ + ): + for method_name in ("__getattribute__", "__getattr__"): + # Normally, mypy assumes that instances that define __getattr__ have all + # attributes with the corresponding return type. If this will produce + # many false negatives, then this could be prohibited for + # structural subtyping. + method = info.get_method(method_name) + if method and method.info.fullname != "builtins.object": + if isinstance(method, Decorator): + getattr_type = get_proper_type(find_node_type(method.var, itype, subtype)) + else: + getattr_type = get_proper_type(find_node_type(method, itype, subtype)) + if isinstance(getattr_type, CallableType): + return getattr_type.ret_type + return getattr_type + if itype.type.fallback_to_any or class_obj and itype.type.meta_fallback_to_any: + return AnyType(TypeOfAny.special_form) + if isinstance(v, TypeInfo): + # PEP 544 doesn't specify anything about such use cases. So we just try + # to do something meaningful (at least we should not crash). + return TypeType(fill_typevars_with_any(v)) + if itype.extra_attrs and name in itype.extra_attrs.attrs: + return itype.extra_attrs.attrs[name] + return None + + +def get_member_flags(name: str, itype: Instance, class_obj: bool = False) -> set[int]: + """Detect whether a member 'name' is settable, whether it is an + instance or class variable, and whether it is class or static method. + + The flags are defined as following: + * IS_SETTABLE: whether this attribute can be set, not set for methods and + non-settable properties; + * IS_CLASSVAR: set if the variable is annotated as 'x: ClassVar[t]'; + * IS_CLASS_OR_STATIC: set for methods decorated with @classmethod or + with @staticmethod. + """ + info = itype.type + method = info.get_method(name) + setattr_meth = info.get_method("__setattr__") + if method: + if isinstance(method, Decorator): + if method.var.is_staticmethod or method.var.is_classmethod: + return {IS_CLASS_OR_STATIC} + elif method.var.is_property: + return {IS_VAR} + elif method.is_property: # this could be settable property + assert isinstance(method, OverloadedFuncDef) + dec = method.items[0] + assert isinstance(dec, Decorator) + if dec.var.is_settable_property or setattr_meth: + return {IS_VAR, IS_SETTABLE} + else: + return {IS_VAR} + return set() # Just a regular method + node = info.get(name) + if not node: + if setattr_meth: + return {IS_SETTABLE} + if itype.extra_attrs and name in itype.extra_attrs.attrs: + flags = set() + if name not in itype.extra_attrs.immutable: + flags.add(IS_SETTABLE) + return flags + return set() + v = node.node + # just a variable + if isinstance(v, Var): + if v.is_property: + return {IS_VAR} + flags = {IS_VAR} + if not v.is_final: + flags.add(IS_SETTABLE) + if v.is_classvar: + flags.add(IS_CLASSVAR) + if class_obj and v.is_inferred: + flags.add(IS_CLASSVAR) + return flags + return set() + + +def find_node_type( + node: Var | FuncBase, itype: Instance, subtype: Type, class_obj: bool = False +) -> Type: + """Find type of a variable or method 'node' (maybe also a decorated method). + Apply type arguments from 'itype', and bind 'self' to 'subtype'. + """ + from mypy.typeops import bind_self + + if isinstance(node, FuncBase): + typ: Type | None = mypy.typeops.function_type( + node, fallback=Instance(itype.type.mro[-1], []) + ) + else: + typ = node.type + if typ is not None: + typ = expand_self_type(node, typ, subtype) + p_typ = get_proper_type(typ) + if typ is None: + return AnyType(TypeOfAny.from_error) + # We don't need to bind 'self' for static methods, since there is no 'self'. + if isinstance(node, FuncBase) or ( + isinstance(p_typ, FunctionLike) + and node.is_initialized_in_class + and not node.is_staticmethod + ): + assert isinstance(p_typ, FunctionLike) + if class_obj and not ( + node.is_class if isinstance(node, FuncBase) else node.is_classmethod + ): + # Don't bind instance methods on class objects. + signature = p_typ + else: + signature = bind_self( + p_typ, subtype, is_classmethod=isinstance(node, Var) and node.is_classmethod + ) + if node.is_property and not class_obj: + assert isinstance(signature, CallableType) + typ = signature.ret_type + else: + typ = signature + itype = map_instance_to_supertype(itype, node.info) + typ = expand_type_by_instance(typ, itype) + return typ + + +def non_method_protocol_members(tp: TypeInfo) -> list[str]: + """Find all non-callable members of a protocol.""" + + assert tp.is_protocol + result: list[str] = [] + anytype = AnyType(TypeOfAny.special_form) + instance = Instance(tp, [anytype] * len(tp.defn.type_vars)) + + for member in tp.protocol_members: + typ = get_proper_type(find_member(member, instance, instance)) + if not isinstance(typ, (Overloaded, CallableType)): + result.append(member) + return result + + +def is_callable_compatible( + left: CallableType, + right: CallableType, + *, + is_compat: Callable[[Type, Type], bool], + is_proper_subtype: bool, + is_compat_return: Callable[[Type, Type], bool] | None = None, + ignore_return: bool = False, + ignore_pos_arg_names: bool = False, + check_args_covariantly: bool = False, + allow_partial_overlap: bool = False, + strict_concatenate: bool = False, +) -> bool: + """Is the left compatible with the right, using the provided compatibility check? + + is_compat: + The check we want to run against the parameters. + + is_compat_return: + The check we want to run against the return type. + If None, use the 'is_compat' check. + + check_args_covariantly: + If true, check if the left's args is compatible with the right's + instead of the other way around (contravariantly). + + This function is mostly used to check if the left is a subtype of the right which + is why the default is to check the args contravariantly. However, it's occasionally + useful to check the args using some other check, so we leave the variance + configurable. + + For example, when checking the validity of overloads, it's useful to see if + the first overload alternative has more precise arguments than the second. + We would want to check the arguments covariantly in that case. + + Note! The following two function calls are NOT equivalent: + + is_callable_compatible(f, g, is_compat=is_subtype, check_args_covariantly=False) + is_callable_compatible(g, f, is_compat=is_subtype, check_args_covariantly=True) + + The two calls are similar in that they both check the function arguments in + the same direction: they both run `is_subtype(argument_from_g, argument_from_f)`. + + However, the two calls differ in which direction they check things like + keyword arguments. For example, suppose f and g are defined like so: + + def f(x: int, *y: int) -> int: ... + def g(x: int) -> int: ... + + In this case, the first call will succeed and the second will fail: f is a + valid stand-in for g but not vice-versa. + + allow_partial_overlap: + By default this function returns True if and only if *all* calls to left are + also calls to right (with respect to the provided 'is_compat' function). + + If this parameter is set to 'True', we return True if *there exists at least one* + call to left that's also a call to right. + + In other words, we perform an existential check instead of a universal one; + we require left to only overlap with right instead of being a subset. + + For example, suppose we set 'is_compat' to some subtype check and compare following: + + f(x: float, y: str = "...", *args: bool) -> str + g(*args: int) -> str + + This function would normally return 'False': f is not a subtype of g. + However, we would return True if this parameter is set to 'True': the two + calls are compatible if the user runs "f_or_g(3)". In the context of that + specific call, the two functions effectively have signatures of: + + f2(float) -> str + g2(int) -> str + + Here, f2 is a valid subtype of g2 so we return True. + + Specifically, if this parameter is set this function will: + + - Ignore optional arguments on either the left or right that have no + corresponding match. + - No longer mandate optional arguments on either side are also optional + on the other. + - No longer mandate that if right has a *arg or **kwarg that left must also + have the same. + + Note: when this argument is set to True, this function becomes "symmetric" -- + the following calls are equivalent: + + is_callable_compatible(f, g, + is_compat=some_check, + check_args_covariantly=False, + allow_partial_overlap=True) + is_callable_compatible(g, f, + is_compat=some_check, + check_args_covariantly=True, + allow_partial_overlap=True) + + If the 'some_check' function is also symmetric, the two calls would be equivalent + whether or not we check the args covariantly. + """ + # Normalize both types before comparing them. + left = left.with_unpacked_kwargs().with_normalized_var_args() + right = right.with_unpacked_kwargs().with_normalized_var_args() + + if is_compat_return is None: + is_compat_return = is_compat + + # If either function is implicitly typed, ignore positional arg names too + if left.implicit or right.implicit: + ignore_pos_arg_names = True + + # Non-type cannot be a subtype of type. + if right.is_type_obj() and not left.is_type_obj() and not allow_partial_overlap: return False + + # A callable L is a subtype of a generic callable R if L is a + # subtype of every type obtained from R by substituting types for + # the variables of R. We can check this by simply leaving the + # generic variables of R as type variables, effectively varying + # over all possible values. + + # It's okay even if these variables share ids with generic + # type variables of L, because generating and solving + # constraints for the variables of L to make L a subtype of R + # (below) treats type variables on the two sides as independent. if left.variables: # Apply generic type variables away in left via type inference. - left = unify_generic_callable(left, right) - if left is None: + unified = unify_generic_callable(left, right, ignore_return=ignore_return) + if unified is None: return False + left = unified # Check return types. - if not ignore_return and not is_subtype(left.ret_type, right.ret_type): + if not ignore_return and not is_compat_return(left.ret_type, right.ret_type): return False - # Check argument types. - if left.min_args > right.min_args: - return False - if left.is_var_arg: - return is_var_arg_callable_subtype_helper(left, right) - if right.is_var_arg: - return False - if len(left.arg_types) < len(right.arg_types): + if check_args_covariantly: + is_compat = flip_compat_check(is_compat) + + if not strict_concatenate and (left.from_concatenate or right.from_concatenate): + strict_concatenate_check = False + else: + strict_concatenate_check = True + + return are_parameters_compatible( + left, + right, + is_compat=is_compat, + is_proper_subtype=is_proper_subtype, + ignore_pos_arg_names=ignore_pos_arg_names, + allow_partial_overlap=allow_partial_overlap, + strict_concatenate_check=strict_concatenate_check, + ) + + +def are_trivial_parameters(param: Parameters | NormalizedCallableType) -> bool: + param_star = param.var_arg() + param_star2 = param.kw_arg() + return ( + param.arg_kinds == [ARG_STAR, ARG_STAR2] + and param_star is not None + and isinstance(get_proper_type(param_star.typ), AnyType) + and param_star2 is not None + and isinstance(get_proper_type(param_star2.typ), AnyType) + ) + + +def is_trivial_suffix(param: Parameters | NormalizedCallableType) -> bool: + param_star = param.var_arg() + param_star2 = param.kw_arg() + return ( + param.arg_kinds[-2:] == [ARG_STAR, ARG_STAR2] + and param_star is not None + and isinstance(get_proper_type(param_star.typ), AnyType) + and param_star2 is not None + and isinstance(get_proper_type(param_star2.typ), AnyType) + ) + + +def are_parameters_compatible( + left: Parameters | NormalizedCallableType, + right: Parameters | NormalizedCallableType, + *, + is_compat: Callable[[Type, Type], bool], + is_proper_subtype: bool, + ignore_pos_arg_names: bool = False, + allow_partial_overlap: bool = False, + strict_concatenate_check: bool = False, +) -> bool: + """Helper function for is_callable_compatible, used for Parameter compatibility""" + if right.is_ellipsis_args and not is_proper_subtype: + return True + + left_star = left.var_arg() + left_star2 = left.kw_arg() + right_star = right.var_arg() + right_star2 = right.kw_arg() + + # Treat "def _(*a: Any, **kw: Any) -> X" similarly to "Callable[..., X]" + if are_trivial_parameters(right) and not is_proper_subtype: + return True + trivial_suffix = is_trivial_suffix(right) and not is_proper_subtype + + if ( + right.arg_kinds == [ARG_STAR] + and isinstance(get_proper_type(right.arg_types[0]), AnyType) + and not is_proper_subtype + ): + # Similar to how (*Any, **Any) is considered a supertype of all callables, we consider + # (*Any) a supertype of all callables with positional arguments. This is needed in + # particular because we often refuse to try type inference if actual type is not + # a subtype of erased template type. + if all(k.is_positional() for k in left.arg_kinds) and ignore_pos_arg_names: + return True + + # Match up corresponding arguments and check them for compatibility. In + # every pair (argL, argR) of corresponding arguments from L and R, argL must + # be "more general" than argR if L is to be a subtype of R. + + # Arguments are corresponding if they either share a name, share a position, + # or both. If L's corresponding argument is ambiguous, L is not a subtype of R. + + # If left has one corresponding argument by name and another by position, + # consider them to be one "merged" argument (and not ambiguous) if they're + # both optional, they're name-only and position-only respectively, and they + # have the same type. This rule allows functions with (*args, **kwargs) to + # properly stand in for the full domain of formal arguments that they're + # used for in practice. + + # Every argument in R must have a corresponding argument in L, and every + # required argument in L must have a corresponding argument in R. + + # Phase 1: Confirm every argument in R has a corresponding argument in L. + + # Phase 1a: If left and right can both accept an infinite number of args, + # their types must be compatible. + # + # Furthermore, if we're checking for compatibility in all cases, + # we confirm that if R accepts an infinite number of arguments, + # L must accept the same. + def _incompatible(left_arg: FormalArgument | None, right_arg: FormalArgument | None) -> bool: + if right_arg is None: + return False + if left_arg is None: + return not allow_partial_overlap and not trivial_suffix + return not is_compat(right_arg.typ, left_arg.typ) + + if _incompatible(left_star, right_star) or _incompatible(left_star2, right_star2): return False - for i in range(len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[i]): + + # Phase 1b: Check non-star args: for every arg right can accept, left must + # also accept. The only exception is if we are allowing partial + # overlaps: in that case, we ignore optional args on the right. + for right_arg in right.formal_arguments(): + left_arg = mypy.typeops.callable_corresponding_argument(left, right_arg) + if left_arg is None: + if allow_partial_overlap and not right_arg.required: + continue + return False + if not are_args_compatible( + left_arg, + right_arg, + is_compat, + ignore_pos_arg_names=ignore_pos_arg_names, + allow_partial_overlap=allow_partial_overlap, + allow_imprecise_kinds=right.imprecise_arg_kinds, + ): + return False + + # Phase 1c: Check var args. Right has an infinite series of optional positional + # arguments. Get all further positional args of left, and make sure + # they're more general than the corresponding member in right. + # TODO: are we handling UnpackType correctly here? + if right_star is not None and not trivial_suffix: + # Synthesize an anonymous formal argument for the right + right_by_position = right.try_synthesizing_arg_from_vararg(None) + assert right_by_position is not None + + i = right_star.pos + assert i is not None + while i < len(left.arg_kinds) and left.arg_kinds[i].is_positional(): + if allow_partial_overlap and left.arg_kinds[i].is_optional(): + break + + left_by_position = left.argument_by_position(i) + assert left_by_position is not None + + if not are_args_compatible( + left_by_position, + right_by_position, + is_compat, + ignore_pos_arg_names=ignore_pos_arg_names, + allow_partial_overlap=allow_partial_overlap, + ): + return False + i += 1 + + # Phase 1d: Check kw args. Right has an infinite series of optional named + # arguments. Get all further named args of left, and make sure + # they're more general than the corresponding member in right. + if right_star2 is not None and not trivial_suffix: + right_names = {name for name in right.arg_names if name is not None} + left_only_names = set() + for name, kind in zip(left.arg_names, left.arg_kinds): + if ( + name is None + or kind.is_star() + or name in right_names + or not strict_concatenate_check + ): + continue + left_only_names.add(name) + + # Synthesize an anonymous formal argument for the right + right_by_name = right.try_synthesizing_arg_from_kwarg(None) + assert right_by_name is not None + + for name in left_only_names: + left_by_name = left.argument_by_name(name) + assert left_by_name is not None + + if allow_partial_overlap and not left_by_name.required: + continue + + if not are_args_compatible( + left_by_name, + right_by_name, + is_compat, + ignore_pos_arg_names=ignore_pos_arg_names, + allow_partial_overlap=allow_partial_overlap, + ): + return False + + # Phase 2: Left must not impose additional restrictions. + # (Every required argument in L must have a corresponding argument in R) + # Note: we already checked the *arg and **kwarg arguments in phase 1a. + for left_arg in left.formal_arguments(): + right_by_name = ( + right.argument_by_name(left_arg.name) if left_arg.name is not None else None + ) + + right_by_pos = ( + right.argument_by_position(left_arg.pos) if left_arg.pos is not None else None + ) + + # If the left hand argument corresponds to two right-hand arguments, + # neither of them can be required. + if ( + right_by_name is not None + and right_by_pos is not None + and right_by_name != right_by_pos + and (right_by_pos.required or right_by_name.required) + and strict_concatenate_check + and not right.imprecise_arg_kinds + ): + return False + + # All *required* left-hand arguments must have a corresponding + # right-hand argument. Optional args do not matter. + if left_arg.required and right_by_pos is None and right_by_name is None: return False + return True -def is_var_arg_callable_subtype_helper(left: Callable, right: Callable) -> bool: - """Is left a subtype of right, assuming left has *args? +def are_args_compatible( + left: FormalArgument, + right: FormalArgument, + is_compat: Callable[[Type, Type], bool], + *, + ignore_pos_arg_names: bool, + allow_partial_overlap: bool, + allow_imprecise_kinds: bool = False, +) -> bool: + if left.required and right.required: + # If both arguments are required allow_partial_overlap has no effect. + allow_partial_overlap = False - See also is_callable_subtype for additional assumptions we can make. - """ - left_fixed = left.max_fixed_args() - right_fixed = right.max_fixed_args() - num_fixed_matching = min(left_fixed, right_fixed) - for i in range(num_fixed_matching): - if not is_subtype(right.arg_types[i], left.arg_types[i]): + def is_different( + left_item: object | None, right_item: object | None, allow_overlap: bool + ) -> bool: + """Checks if the left and right items are different. + + If the right item is unspecified (e.g. if the right callable doesn't care + about what name or position its arg has), we default to returning False. + + If we're allowing partial overlap, we also default to returning False + if the left callable also doesn't care.""" + if right_item is None: return False - if not right.is_var_arg: - for i in range(num_fixed_matching, len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False + if allow_overlap and left_item is None: + return False + return left_item != right_item + + # If right has a specific name it wants this argument to be, left must + # have the same. + if is_different(left.name, right.name, allow_partial_overlap): + # But pay attention to whether we're ignoring positional arg names + if not ignore_pos_arg_names or right.pos is None: + return False + + # If right is at a specific position, left must have the same. + # TODO: partial overlap logic is flawed for positions. + # We disable it to avoid false positives at a cost of few false negatives. + if is_different(left.pos, right.pos, allow_overlap=False) and not allow_imprecise_kinds: + return False + + # If right's argument is optional, left's must also be + # (unless we're relaxing the checks to allow potential + # rather than definite compatibility). + if not allow_partial_overlap and not right.required and left.required: + return False + + # If we're allowing partial overlaps and neither arg is required, + # the types don't actually need to be the same + if allow_partial_overlap and not left.required and not right.required: return True - else: - for i in range(left_fixed, right_fixed): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - for i in range(right_fixed, left_fixed): - if not is_subtype(right.arg_types[-1], left.arg_types[i]): - return False - return is_subtype(right.arg_types[-1], left.arg_types[-1]) + + # Left must have a more general type + return is_compat(right.typ, left.typ) + + +def flip_compat_check(is_compat: Callable[[Type, Type], bool]) -> Callable[[Type, Type], bool]: + def new_is_compat(left: Type, right: Type) -> bool: + return is_compat(right, left) + + return new_is_compat -def unify_generic_callable(type: Callable, target: Callable) -> Callable: +def unify_generic_callable( + type: NormalizedCallableType, + target: NormalizedCallableType, + ignore_return: bool, + return_constraint_direction: int | None = None, +) -> NormalizedCallableType | None: """Try to unify a generic callable type with another callable type. - Return unified Callable if successful; otherwise, return None. + Return unified CallableType if successful; otherwise, return None. """ - constraints = [] # type: List[mypy.constraints.Constraint] - for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): + import mypy.solve + + if set(type.type_var_ids()) & {v.id for v in mypy.typeops.get_all_type_vars(target)}: + # Overload overlap check does nasty things like unifying in opposite direction. + # This can easily create type variable clashes, so we need to refresh. + type = freshen_function_type_vars(type) + + if return_constraint_direction is None: + return_constraint_direction = mypy.constraints.SUBTYPE_OF + + constraints: list[mypy.constraints.Constraint] = [] + # There is some special logic for inference in callables, so better use them + # as wholes instead of picking separate arguments. + cs = mypy.constraints.infer_constraints( + type.copy_modified(ret_type=UninhabitedType()), + target.copy_modified(ret_type=UninhabitedType()), + mypy.constraints.SUBTYPE_OF, + skip_neg_op=True, + ) + constraints.extend(cs) + if not ignore_return: c = mypy.constraints.infer_constraints( - arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) + type.ret_type, target.ret_type, return_constraint_direction + ) constraints.extend(c) - type_var_ids = [tvar.id for tvar in type.variables] - inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) + inferred_vars, _ = mypy.solve.solve_constraints( + type.variables, constraints, allow_polymorphic=True + ) if None in inferred_vars: return None - msg = messages.temp_message_builder() - applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) - if msg.is_errors() or not isinstance(applied, Callable): + non_none_inferred_vars = cast(List[Type], inferred_vars) + had_errors = False + + def report(*args: Any) -> None: + nonlocal had_errors + had_errors = True + + # This function may be called by the solver, so we need to allow erased types here. + # We anyway allow checking subtyping between other types containing + # (probably also because solver needs subtyping). See also comment in + # ExpandTypeVisitor.visit_erased_type(). + applied = mypy.applytype.apply_generic_arguments( + type, non_none_inferred_vars, report, context=target + ) + if had_errors: + return None + return cast(NormalizedCallableType, applied) + + +def try_restrict_literal_union(t: UnionType, s: Type) -> list[Type] | None: + """Return the items of t, excluding any occurrence of s, if and only if + - t only contains simple literals + - s is a simple literal + + Otherwise, returns None + """ + ps = get_proper_type(s) + if not mypy.typeops.is_simple_literal(ps): return None - return cast(Callable, applied) + + new_items: list[Type] = [] + for i in t.relevant_items(): + pi = get_proper_type(i) + if not mypy.typeops.is_simple_literal(pi): + return None + if pi != ps: + new_items.append(i) + return new_items def restrict_subtype_away(t: Type, s: Type) -> Type: - """Return a supertype of (t intersect not s) + """Return t minus s for runtime type assertions. - Currently just remove elements of a union type. + If we can't determine a precise result, return a supertype of the + ideal result (just t is a valid result). + + This is used for type inference of runtime type checks such as + isinstance(). Currently, this just removes elements of a union type. """ - if isinstance(t, UnionType): - new_items = [item for item in t.items if not is_subtype(item, s)] + p_t = get_proper_type(t) + if isinstance(p_t, UnionType): + new_items = try_restrict_literal_union(p_t, s) + if new_items is None: + new_items = [ + restrict_subtype_away(item, s) + for item in p_t.relevant_items() + if (isinstance(get_proper_type(item), AnyType) or not covers_at_runtime(item, s)) + ] return UnionType.make_union(new_items) + elif isinstance(p_t, TypeVarType): + return p_t.copy_modified(upper_bound=restrict_subtype_away(p_t.upper_bound, s)) + elif covers_at_runtime(t, s): + return UninhabitedType() else: return t -def is_proper_subtype(t: Type, s: Type) -> bool: - """Check if t is a proper subtype of s? +def covers_at_runtime(item: Type, supertype: Type) -> bool: + """Will isinstance(item, supertype) always return True at runtime?""" + item = get_proper_type(item) + supertype = get_proper_type(supertype) - For proper subtypes, there's no need to rely on compatibility due to - Any types. Any instance type t is also a proper subtype of t. - """ - # FIX tuple types - if isinstance(t, Instance): - if isinstance(s, Instance): - if not t.type.has_base(s.type.fullname()): - return False - t = map_instance_to_supertype(t, s.type) - if not is_immutable(s): - return all(sametypes.is_same_type(ta, ra) for (ta, ra) in - zip(t.args, s.args)) - else: - return all(is_proper_subtype(ta, ra) for (ta, ra) in - zip(t.args, s.args)) - return False - else: - return sametypes.is_same_type(t, s) + # Since runtime type checks will ignore type arguments, erase the types. + supertype = erase_type(supertype) + if is_proper_subtype( + erase_type(item), supertype, ignore_promotions=True, erase_instances=True + ): + return True + if isinstance(supertype, Instance): + if supertype.type.is_protocol: + # TODO: Implement more robust support for runtime isinstance() checks, see issue #3827. + if is_proper_subtype(item, supertype, ignore_promotions=True): + return True + if isinstance(item, TypedDictType): + # Special case useful for selecting TypedDicts from unions using isinstance(x, dict). + if supertype.type.fullname == "builtins.dict": + return True + elif isinstance(item, TypeVarType): + if is_proper_subtype(item.upper_bound, supertype, ignore_promotions=True): + return True + elif isinstance(item, Instance) and supertype.type.fullname == "builtins.int": + # "int" covers all native int types + if item.type.fullname in MYPYC_NATIVE_INT_NAMES: + return True + # TODO: Add more special cases. + return False -def is_more_precise(t: Type, s: Type) -> bool: - """Check if t is a more precise type than s. +def is_more_precise(left: Type, right: Type, *, ignore_promotions: bool = False) -> bool: + """Check if left is a more precise type than right. - A t is a proper subtype of s, t is also more precise than s. Also, if - s is Any, t is more precise than s for any t. Finally, if t is the same - type as s, t is more precise than s. + A left is a proper subtype of right, left is also more precise than + right. Also, if right is Any, left is more precise than right, for + any left. """ # TODO Should List[int] be more precise than List[Any]? - if isinstance(s, AnyType): + right = get_proper_type(right) + if isinstance(right, AnyType): return True - if isinstance(s, Instance): - if isinstance(t, Callable): - # Fall back to subclass check and ignore other properties of the callable. - return is_proper_subtype(t.fallback, s) - return is_proper_subtype(t, s) - return sametypes.is_same_type(t, s) + return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) + + +def all_non_object_members(info: TypeInfo) -> set[str]: + members = set(info.names) + for base in info.mro[1:-1]: + members.update(base.names) + return members + + +def infer_variance(info: TypeInfo, i: int) -> bool: + """Infer the variance of the ith type variable of a generic class. + + Return True if successful. This can fail if some inferred types aren't ready. + """ + object_type = Instance(info.mro[-1], []) + + for variance in COVARIANT, CONTRAVARIANT, INVARIANT: + tv = info.defn.type_vars[i] + assert isinstance(tv, TypeVarType) + if tv.variance != VARIANCE_NOT_READY: + continue + tv.variance = variance + co = True + contra = True + tvar = info.defn.type_vars[i] + self_type = fill_typevars(info) + for member in all_non_object_members(info): + # __mypy-replace is an implementation detail of the dataclass plugin + if member in ("__init__", "__new__", "__mypy-replace"): + continue + + if isinstance(self_type, TupleType): + self_type = mypy.typeops.tuple_fallback(self_type) + flags = get_member_flags(member, self_type) + settable = IS_SETTABLE in flags + + node = info[member].node + if isinstance(node, Var): + if node.type is None: + tv.variance = VARIANCE_NOT_READY + return False + if has_underscore_prefix(member): + # Special case to avoid false positives (and to pass conformance tests) + settable = False + + typ = find_member(member, self_type, self_type) + if typ: + # It's okay for a method in a generic class with a contravariant type + # variable to return a generic instance of the class, if it doesn't involve + # variance (i.e. values of type variables are propagated). Our normal rules + # would disallow this. Replace such return types with 'Any' to allow this. + # + # This could probably be more lenient (e.g. allow self type be nested, don't + # require all type arguments to be identical to self_type), but this will + # hopefully cover the vast majority of such cases, including Self. + typ = erase_return_self_types(typ, self_type) + + typ2 = expand_type(typ, {tvar.id: object_type}) + if not is_subtype(typ, typ2): + co = False + if not is_subtype(typ2, typ): + contra = False + if settable: + co = False + + # Infer variance from base classes, in case they have explicit variances + for base in info.bases: + base2 = expand_type(base, {tvar.id: object_type}) + if not is_subtype(base, base2): + co = False + if not is_subtype(base2, base): + contra = False + + if co: + v = COVARIANT + elif contra: + v = CONTRAVARIANT + else: + v = INVARIANT + if v == variance: + break + tv.variance = VARIANCE_NOT_READY + return True + + +def has_underscore_prefix(name: str) -> bool: + return name.startswith("_") and not (name.startswith("__") and name.endswith("__")) + + +def infer_class_variances(info: TypeInfo) -> bool: + if not info.defn.type_args: + return True + tvs = info.defn.type_vars + success = True + for i, tv in enumerate(tvs): + if isinstance(tv, TypeVarType) and tv.variance == VARIANCE_NOT_READY: + if not infer_variance(info, i): + success = False + return success + + +def erase_return_self_types(typ: Type, self_type: Instance) -> Type: + """If a typ is function-like and returns self_type, replace return type with Any.""" + proper_type = get_proper_type(typ) + if isinstance(proper_type, CallableType): + ret = get_proper_type(proper_type.ret_type) + if isinstance(ret, Instance) and ret == self_type: + return proper_type.copy_modified(ret_type=AnyType(TypeOfAny.implementation_artifact)) + elif isinstance(proper_type, Overloaded): + return Overloaded( + [ + cast(CallableType, erase_return_self_types(it, self_type)) + for it in proper_type.items + ] + ) + return typ diff --git a/mypy/suggestions.py b/mypy/suggestions.py new file mode 100644 index 000000000000..268f3032fc9b --- /dev/null +++ b/mypy/suggestions.py @@ -0,0 +1,1046 @@ +"""Mechanisms for inferring function types based on callsites. + +Currently works by collecting all argument types at callsites, +synthesizing a list of possible function types from that, trying them +all, and picking the one with the fewest errors that we think is the +"best". + +Can return JSON that pyannotate can use to apply the annotations to code. + +There are a bunch of TODOs here: + * Maybe want a way to surface the choices not selected?? + * We can generate an exponential number of type suggestions, and probably want + a way to not always need to check them all. + * Our heuristics for what types to try are primitive and not yet + supported by real practice. + * More! + +Other things: + * This is super brute force. Could we integrate with the typechecker + more to understand more about what is going on? + * Like something with tracking constraints/unification variables? + * No understanding of type variables at *all* +""" + +from __future__ import annotations + +import itertools +import json +import os +from contextlib import contextmanager +from typing import Callable, Iterator, NamedTuple, TypeVar, cast +from typing_extensions import TypedDict + +from mypy.argmap import map_actuals_to_formals +from mypy.build import Graph, State +from mypy.checkexpr import has_any_type +from mypy.find_sources import InvalidSourceList, SourceFinder +from mypy.join import join_type_list +from mypy.meet import meet_type_list +from mypy.modulefinder import PYTHON_EXTENSIONS +from mypy.nodes import ( + ARG_STAR, + ARG_STAR2, + ArgKind, + CallExpr, + Decorator, + Expression, + FuncDef, + MypyFile, + RefExpr, + ReturnStmt, + SymbolNode, + SymbolTable, + TypeInfo, + reverse_builtin_aliases, +) +from mypy.options import Options +from mypy.plugin import FunctionContext, MethodContext, Plugin +from mypy.server.update import FineGrainedBuildManager +from mypy.state import state +from mypy.traverser import TraverserVisitor +from mypy.typeops import make_simplified_union +from mypy.types import ( + AnyType, + CallableType, + FunctionLike, + Instance, + NoneType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeStrVisitor, + TypeTranslator, + TypeVarType, + UninhabitedType, + UnionType, + get_proper_type, +) +from mypy.types_utils import is_overlapping_none, remove_optional +from mypy.util import split_target + + +class PyAnnotateSignature(TypedDict): + return_type: str + arg_types: list[str] + + +class Callsite(NamedTuple): + path: str + line: int + arg_kinds: list[list[ArgKind]] + callee_arg_names: list[str | None] + arg_names: list[list[str | None]] + arg_types: list[list[Type]] + + +class SuggestionPlugin(Plugin): + """Plugin that records all calls to a given target.""" + + def __init__(self, target: str) -> None: + if target.endswith((".__new__", ".__init__")): + target = target.rsplit(".", 1)[0] + + self.target = target + # List of call sites found by dmypy suggest: + # (path, line, , , ) + self.mystery_hits: list[Callsite] = [] + + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname == self.target: + return self.log + else: + return None + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + if fullname == self.target: + return self.log + else: + return None + + def log(self, ctx: FunctionContext | MethodContext) -> Type: + self.mystery_hits.append( + Callsite( + ctx.api.path, + ctx.context.line, + ctx.arg_kinds, + ctx.callee_arg_names, + ctx.arg_names, + ctx.arg_types, + ) + ) + return ctx.default_return_type + + +# NOTE: We could make this a bunch faster by implementing a StatementVisitor that skips +# traversing into expressions +class ReturnFinder(TraverserVisitor): + """Visitor for finding all types returned from a function.""" + + def __init__(self, typemap: dict[Expression, Type]) -> None: + self.typemap = typemap + self.return_types: list[Type] = [] + + def visit_return_stmt(self, o: ReturnStmt) -> None: + if o.expr is not None and o.expr in self.typemap: + self.return_types.append(self.typemap[o.expr]) + + def visit_func_def(self, o: FuncDef) -> None: + # Skip nested functions + pass + + +def get_return_types(typemap: dict[Expression, Type], func: FuncDef) -> list[Type]: + """Find all the types returned by return statements in func.""" + finder = ReturnFinder(typemap) + func.body.accept(finder) + return finder.return_types + + +class ArgUseFinder(TraverserVisitor): + """Visitor for finding all the types of arguments that each arg is passed to. + + This is extremely simple minded but might be effective anyways. + """ + + def __init__(self, func: FuncDef, typemap: dict[Expression, Type]) -> None: + self.typemap = typemap + self.arg_types: dict[SymbolNode, list[Type]] = {arg.variable: [] for arg in func.arguments} + + def visit_call_expr(self, o: CallExpr) -> None: + if not any(isinstance(e, RefExpr) and e.node in self.arg_types for e in o.args): + return + + typ = get_proper_type(self.typemap.get(o.callee)) + if not isinstance(typ, CallableType): + return + + formal_to_actual = map_actuals_to_formals( + o.arg_kinds, + o.arg_names, + typ.arg_kinds, + typ.arg_names, + lambda n: AnyType(TypeOfAny.special_form), + ) + + for i, args in enumerate(formal_to_actual): + for arg_idx in args: + arg = o.args[arg_idx] + if isinstance(arg, RefExpr) and arg.node in self.arg_types: + self.arg_types[arg.node].append(typ.arg_types[i]) + + +def get_arg_uses(typemap: dict[Expression, Type], func: FuncDef) -> list[list[Type]]: + """Find all the types of arguments that each arg is passed to. + + For example, given + def foo(x: int) -> None: ... + def bar(x: str) -> None: ... + def test(x, y): + foo(x) + bar(y) + + this will return [[int], [str]]. + """ + finder = ArgUseFinder(func, typemap) + func.body.accept(finder) + return [finder.arg_types[arg.variable] for arg in func.arguments] + + +class SuggestionFailure(Exception): + pass + + +def is_explicit_any(typ: AnyType) -> bool: + # Originally I wanted to count as explicit anything derived from an explicit any, but that + # seemed too strict in some testing. + # return (typ.type_of_any == TypeOfAny.explicit + # or (typ.source_any is not None and typ.source_any.type_of_any == TypeOfAny.explicit)) + # Important question: what should we do with source_any stuff? Does that count? + # And actually should explicit anys count at all?? Maybe not! + return typ.type_of_any == TypeOfAny.explicit + + +def is_implicit_any(typ: Type) -> bool: + typ = get_proper_type(typ) + return isinstance(typ, AnyType) and not is_explicit_any(typ) + + +class SuggestionEngine: + """Engine for finding call sites and suggesting signatures.""" + + def __init__( + self, + fgmanager: FineGrainedBuildManager, + *, + json: bool, + no_errors: bool = False, + no_any: bool = False, + flex_any: float | None = None, + use_fixme: str | None = None, + max_guesses: int | None = None, + ) -> None: + self.fgmanager = fgmanager + self.manager = fgmanager.manager + self.plugin = self.manager.plugin + self.graph = fgmanager.graph + self.finder = SourceFinder(self.manager.fscache, self.manager.options) + + self.give_json = json + self.no_errors = no_errors + self.flex_any = flex_any + if no_any: + self.flex_any = 1.0 + + self.max_guesses = max_guesses or 64 + self.use_fixme = use_fixme + + def suggest(self, function: str) -> str: + """Suggest an inferred type for function.""" + mod, func_name, node = self.find_node(function) + + with self.restore_after(mod): + with self.with_export_types(): + suggestion = self.get_suggestion(mod, node) + + if self.give_json: + return self.json_suggestion(mod, func_name, node, suggestion) + else: + return self.format_signature(suggestion) + + def suggest_callsites(self, function: str) -> str: + """Find a list of call sites of function.""" + mod, _, node = self.find_node(function) + with self.restore_after(mod): + callsites, _ = self.get_callsites(node) + + return "\n".join( + dedup( + [ + f"{path}:{line}: {self.format_args(arg_kinds, arg_names, arg_types)}" + for path, line, arg_kinds, _, arg_names, arg_types in callsites + ] + ) + ) + + @contextmanager + def restore_after(self, module: str) -> Iterator[None]: + """Context manager that reloads a module after executing the body. + + This should undo any damage done to the module state while mucking around. + """ + try: + yield + finally: + self.reload(self.graph[module]) + + @contextmanager + def with_export_types(self) -> Iterator[None]: + """Context manager that enables the export_types flag in the body. + + This causes type information to be exported into the manager's all_types variable. + """ + old = self.manager.options.export_types + self.manager.options.export_types = True + try: + yield + finally: + self.manager.options.export_types = old + + def get_trivial_type(self, fdef: FuncDef) -> CallableType: + """Generate a trivial callable type from a func def, with all Anys""" + # The Anys are marked as being from the suggestion engine + # since they need some special treatment (specifically, + # constraint generation ignores them.) + return CallableType( + [AnyType(TypeOfAny.suggestion_engine) for _ in fdef.arg_kinds], + fdef.arg_kinds, + fdef.arg_names, + AnyType(TypeOfAny.suggestion_engine), + self.named_type("builtins.function"), + ) + + def get_starting_type(self, fdef: FuncDef) -> CallableType: + if isinstance(fdef.type, CallableType): + return make_suggestion_anys(fdef.type) + else: + return self.get_trivial_type(fdef) + + def get_args( + self, + is_method: bool, + base: CallableType, + defaults: list[Type | None], + callsites: list[Callsite], + uses: list[list[Type]], + ) -> list[list[Type]]: + """Produce a list of type suggestions for each argument type.""" + types: list[list[Type]] = [] + for i in range(len(base.arg_kinds)): + # Make self args Any but this will get overridden somewhere in the checker + if i == 0 and is_method: + types.append([AnyType(TypeOfAny.suggestion_engine)]) + continue + + all_arg_types = [] + for call in callsites: + for typ in call.arg_types[i - is_method]: + # Collect all the types except for implicit anys + if not is_implicit_any(typ): + all_arg_types.append(typ) + all_use_types = [] + for typ in uses[i]: + # Collect all the types except for implicit anys + if not is_implicit_any(typ): + all_use_types.append(typ) + # Add in any default argument types + default = defaults[i] + if default: + all_arg_types.append(default) + if all_use_types: + all_use_types.append(default) + + arg_types = [] + + if all_arg_types and all( + isinstance(get_proper_type(tp), NoneType) for tp in all_arg_types + ): + arg_types.append( + UnionType.make_union([all_arg_types[0], AnyType(TypeOfAny.explicit)]) + ) + elif all_arg_types: + arg_types.extend(generate_type_combinations(all_arg_types)) + else: + arg_types.append(AnyType(TypeOfAny.explicit)) + + if all_use_types: + # This is a meet because the type needs to be compatible with all the uses + arg_types.append(meet_type_list(all_use_types)) + + types.append(arg_types) + return types + + def get_default_arg_types(self, fdef: FuncDef) -> list[Type | None]: + return [ + self.manager.all_types[arg.initializer] if arg.initializer else None + for arg in fdef.arguments + ] + + def get_guesses( + self, + is_method: bool, + base: CallableType, + defaults: list[Type | None], + callsites: list[Callsite], + uses: list[list[Type]], + ) -> list[CallableType]: + """Compute a list of guesses for a function's type. + + This focuses just on the argument types, and doesn't change the provided return type. + """ + options = self.get_args(is_method, base, defaults, callsites, uses) + + # Take the first `max_guesses` guesses. + product = itertools.islice(itertools.product(*options), 0, self.max_guesses) + return [refine_callable(base, base.copy_modified(arg_types=list(x))) for x in product] + + def get_callsites(self, func: FuncDef) -> tuple[list[Callsite], list[str]]: + """Find all call sites of a function.""" + new_type = self.get_starting_type(func) + + collector_plugin = SuggestionPlugin(func.fullname) + + self.plugin._plugins.insert(0, collector_plugin) + try: + errors = self.try_type(func, new_type) + finally: + self.plugin._plugins.pop(0) + + return collector_plugin.mystery_hits, errors + + def filter_options( + self, guesses: list[CallableType], is_method: bool, ignore_return: bool + ) -> list[CallableType]: + """Apply any configured filters to the possible guesses. + + Currently the only option is filtering based on Any prevalance.""" + return [ + t + for t in guesses + if self.flex_any is None + or any_score_callable(t, is_method, ignore_return) >= self.flex_any + ] + + def find_best(self, func: FuncDef, guesses: list[CallableType]) -> tuple[CallableType, int]: + """From a list of possible function types, find the best one. + + For best, we want the fewest errors, then the best "score" from score_callable. + """ + if not guesses: + raise SuggestionFailure("No guesses that match criteria!") + errors = {guess: self.try_type(func, guess) for guess in guesses} + best = min(guesses, key=lambda s: (count_errors(errors[s]), self.score_callable(s))) + return best, count_errors(errors[best]) + + def get_guesses_from_parent(self, node: FuncDef) -> list[CallableType]: + """Try to get a guess of a method type from a parent class.""" + if not node.info: + return [] + + for parent in node.info.mro[1:]: + pnode = parent.names.get(node.name) + if pnode and isinstance(pnode.node, (FuncDef, Decorator)): + typ = get_proper_type(pnode.node.type) + # FIXME: Doesn't work right with generic tyeps + if isinstance(typ, CallableType) and len(typ.arg_types) == len(node.arguments): + # Return the first thing we find, since it probably doesn't make sense + # to grab things further up in the chain if an earlier parent has it. + return [typ] + + return [] + + def get_suggestion(self, mod: str, node: FuncDef) -> PyAnnotateSignature: + """Compute a suggestion for a function. + + Return the type and whether the first argument should be ignored. + """ + graph = self.graph + callsites, orig_errors = self.get_callsites(node) + uses = get_arg_uses(self.manager.all_types, node) + + if self.no_errors and orig_errors: + raise SuggestionFailure("Function does not typecheck.") + + is_method = bool(node.info) and not node.is_static + + with state.strict_optional_set(graph[mod].options.strict_optional): + guesses = self.get_guesses( + is_method, + self.get_starting_type(node), + self.get_default_arg_types(node), + callsites, + uses, + ) + guesses += self.get_guesses_from_parent(node) + guesses = self.filter_options(guesses, is_method, ignore_return=True) + best, _ = self.find_best(node, guesses) + + # Now try to find the return type! + self.try_type(node, best) + returns = get_return_types(self.manager.all_types, node) + with state.strict_optional_set(graph[mod].options.strict_optional): + if returns: + ret_types = generate_type_combinations(returns) + else: + ret_types = [NoneType()] + + guesses = [best.copy_modified(ret_type=refine_type(best.ret_type, t)) for t in ret_types] + guesses = self.filter_options(guesses, is_method, ignore_return=False) + best, errors = self.find_best(node, guesses) + + if self.no_errors and errors: + raise SuggestionFailure("No annotation without errors") + + return self.pyannotate_signature(mod, is_method, best) + + def format_args( + self, + arg_kinds: list[list[ArgKind]], + arg_names: list[list[str | None]], + arg_types: list[list[Type]], + ) -> str: + args: list[str] = [] + for i in range(len(arg_types)): + for kind, name, typ in zip(arg_kinds[i], arg_names[i], arg_types[i]): + arg = self.format_type(None, typ) + if kind == ARG_STAR: + arg = "*" + arg + elif kind == ARG_STAR2: + arg = "**" + arg + elif kind.is_named(): + if name: + arg = f"{name}={arg}" + args.append(arg) + return f"({', '.join(args)})" + + def find_node(self, key: str) -> tuple[str, str, FuncDef]: + """From a target name, return module/target names and the func def. + + The 'key' argument can be in one of two formats: + * As the function full name, e.g., package.module.Cls.method + * As the function location as file and line separated by column, + e.g., path/to/file.py:42 + """ + # TODO: Also return OverloadedFuncDef -- currently these are ignored. + node: SymbolNode | None = None + if ":" in key: + if key.count(":") > 1: + raise SuggestionFailure( + "Malformed location for function: {}. Must be either" + " package.module.Class.method or path/to/file.py:line".format(key) + ) + file, line = key.split(":") + if not line.isdigit(): + raise SuggestionFailure(f"Line number must be a number. Got {line}") + line_number = int(line) + modname, node = self.find_node_by_file_and_line(file, line_number) + tail = node.fullname[len(modname) + 1 :] # add one to account for '.' + else: + target = split_target(self.fgmanager.graph, key) + if not target: + raise SuggestionFailure(f"Cannot find module for {key}") + modname, tail = target + node = self.find_node_by_module_and_name(modname, tail) + + if isinstance(node, Decorator): + node = self.extract_from_decorator(node) + if not node: + raise SuggestionFailure(f"Object {key} is a decorator we can't handle") + + if not isinstance(node, FuncDef): + raise SuggestionFailure(f"Object {key} is not a function") + + return modname, tail, node + + def find_node_by_module_and_name(self, modname: str, tail: str) -> SymbolNode | None: + """Find symbol node by module id and qualified name. + + Raise SuggestionFailure if can't find one. + """ + tree = self.ensure_loaded(self.fgmanager.graph[modname]) + + # N.B. This is reimplemented from update's lookup_target + # basically just to produce better error messages. + + names: SymbolTable = tree.names + + # Look through any classes + components = tail.split(".") + for i, component in enumerate(components[:-1]): + if component not in names: + raise SuggestionFailure( + "Unknown class {}.{}".format(modname, ".".join(components[: i + 1])) + ) + node: SymbolNode | None = names[component].node + if not isinstance(node, TypeInfo): + raise SuggestionFailure( + "Object {}.{} is not a class".format(modname, ".".join(components[: i + 1])) + ) + names = node.names + + # Look for the actual function/method + funcname = components[-1] + if funcname not in names: + key = modname + "." + tail + raise SuggestionFailure( + "Unknown {} {}".format("method" if len(components) > 1 else "function", key) + ) + return names[funcname].node + + def find_node_by_file_and_line(self, file: str, line: int) -> tuple[str, SymbolNode]: + """Find symbol node by path to file and line number. + + Find the first function declared *before or on* the line number. + + Return module id and the node found. Raise SuggestionFailure if can't find one. + """ + if not any(file.endswith(ext) for ext in PYTHON_EXTENSIONS): + raise SuggestionFailure("Source file is not a Python file") + try: + modname, _ = self.finder.crawl_up(os.path.normpath(file)) + except InvalidSourceList as e: + raise SuggestionFailure("Invalid source file name: " + file) from e + if modname not in self.graph: + raise SuggestionFailure("Unknown module: " + modname) + # We must be sure about any edits in this file as this might affect the line numbers. + tree = self.ensure_loaded(self.fgmanager.graph[modname], force=True) + node: SymbolNode | None = None + closest_line: int | None = None + # TODO: Handle nested functions. + for _, sym, _ in tree.local_definitions(): + if isinstance(sym.node, (FuncDef, Decorator)): + sym_line = sym.node.line + # TODO: add support for OverloadedFuncDef. + else: + continue + + # We want the closest function above the specified line + if sym_line <= line and (closest_line is None or sym_line > closest_line): + closest_line = sym_line + node = sym.node + if not node: + raise SuggestionFailure(f"Cannot find a function at line {line}") + return modname, node + + def extract_from_decorator(self, node: Decorator) -> FuncDef | None: + for dec in node.decorators: + typ = None + if isinstance(dec, RefExpr) and isinstance(dec.node, FuncDef): + typ = dec.node.type + elif ( + isinstance(dec, CallExpr) + and isinstance(dec.callee, RefExpr) + and isinstance(dec.callee.node, FuncDef) + and isinstance(dec.callee.node.type, CallableType) + ): + typ = get_proper_type(dec.callee.node.type.ret_type) + + if not isinstance(typ, FunctionLike): + return None + for ct in typ.items: + if not ( + len(ct.arg_types) == 1 + and isinstance(ct.arg_types[0], TypeVarType) + and ct.arg_types[0] == ct.ret_type + ): + return None + + return node.func + + def try_type(self, func: FuncDef, typ: ProperType) -> list[str]: + """Recheck a function while assuming it has type typ. + + Return all error messages. + """ + old = func.unanalyzed_type + # During reprocessing, unanalyzed_type gets copied to type (by aststrip). + # We set type to None to ensure that the type always changes during + # reprocessing. + func.type = None + func.unanalyzed_type = typ + try: + res = self.fgmanager.trigger(func.fullname) + # if res: + # print('===', typ) + # print('\n'.join(res)) + return res + finally: + func.unanalyzed_type = old + + def reload(self, state: State) -> list[str]: + """Recheck the module given by state.""" + assert state.path is not None + self.fgmanager.flush_cache() + return self.fgmanager.update([(state.id, state.path)], []) + + def ensure_loaded(self, state: State, force: bool = False) -> MypyFile: + """Make sure that the module represented by state is fully loaded.""" + if not state.tree or state.tree.is_cache_skeleton or force: + self.reload(state) + assert state.tree is not None + return state.tree + + def named_type(self, s: str) -> Instance: + return self.manager.semantic_analyzer.named_type(s) + + def json_suggestion( + self, mod: str, func_name: str, node: FuncDef, suggestion: PyAnnotateSignature + ) -> str: + """Produce a json blob for a suggestion suitable for application by pyannotate.""" + # pyannotate irritatingly drops class names for class and static methods + if node.is_class or node.is_static: + func_name = func_name.split(".", 1)[-1] + + # pyannotate works with either paths relative to where the + # module is rooted or with absolute paths. We produce absolute + # paths because it is simpler. + path = os.path.abspath(self.graph[mod].xpath) + + obj = { + "signature": suggestion, + "line": node.line, + "path": path, + "func_name": func_name, + "samples": 0, + } + return json.dumps([obj], sort_keys=True) + + def pyannotate_signature( + self, cur_module: str | None, is_method: bool, typ: CallableType + ) -> PyAnnotateSignature: + """Format a callable type as a pyannotate dict""" + start = int(is_method) + return { + "arg_types": [self.format_type(cur_module, t) for t in typ.arg_types[start:]], + "return_type": self.format_type(cur_module, typ.ret_type), + } + + def format_signature(self, sig: PyAnnotateSignature) -> str: + """Format a callable type in a way suitable as an annotation... kind of""" + return f"({', '.join(sig['arg_types'])}) -> {sig['return_type']}" + + def format_type(self, cur_module: str | None, typ: Type) -> str: + if self.use_fixme and isinstance(get_proper_type(typ), AnyType): + return self.use_fixme + return typ.accept(TypeFormatter(cur_module, self.graph, self.manager.options)) + + def score_type(self, t: Type, arg_pos: bool) -> int: + """Generate a score for a type that we use to pick which type to use. + + Lower is better, prefer non-union/non-any types. Don't penalize optionals. + """ + t = get_proper_type(t) + if isinstance(t, AnyType): + return 20 + if arg_pos and isinstance(t, NoneType): + return 20 + if isinstance(t, UnionType): + if any(isinstance(get_proper_type(x), AnyType) for x in t.items): + return 20 + if any(has_any_type(x) for x in t.items): + return 15 + if not is_overlapping_none(t): + return 10 + if isinstance(t, CallableType) and (has_any_type(t) or is_tricky_callable(t)): + return 10 + return 0 + + def score_callable(self, t: CallableType) -> int: + return sum(self.score_type(x, arg_pos=True) for x in t.arg_types) + self.score_type( + t.ret_type, arg_pos=False + ) + + +def any_score_type(ut: Type, arg_pos: bool) -> float: + """Generate a very made up number representing the Anyness of a type. + + Higher is better, 1.0 is max + """ + t = get_proper_type(ut) + if isinstance(t, AnyType) and t.type_of_any != TypeOfAny.suggestion_engine: + return 0 + if isinstance(t, NoneType) and arg_pos: + return 0.5 + if isinstance(t, UnionType): + if any(isinstance(get_proper_type(x), AnyType) for x in t.items): + return 0.5 + if any(has_any_type(x) for x in t.items): + return 0.25 + if isinstance(t, CallableType) and is_tricky_callable(t): + return 0.5 + if has_any_type(t): + return 0.5 + + return 1.0 + + +def any_score_callable(t: CallableType, is_method: bool, ignore_return: bool) -> float: + # Ignore the first argument of methods + scores = [any_score_type(x, arg_pos=True) for x in t.arg_types[int(is_method) :]] + # Return type counts twice (since it spreads type information), unless it is + # None in which case it does not count at all. (Though it *does* still count + # if there are no arguments.) + if not isinstance(get_proper_type(t.ret_type), NoneType) or not scores: + ret = 1.0 if ignore_return else any_score_type(t.ret_type, arg_pos=False) + scores += [ret, ret] + + return sum(scores) / len(scores) + + +def is_tricky_callable(t: CallableType) -> bool: + """Is t a callable that we need to put a ... in for syntax reasons?""" + return t.is_ellipsis_args or any(k.is_star() or k.is_named() for k in t.arg_kinds) + + +class TypeFormatter(TypeStrVisitor): + """Visitor used to format types""" + + # TODO: Probably a lot + def __init__(self, module: str | None, graph: Graph, options: Options) -> None: + super().__init__(options=options) + self.module = module + self.graph = graph + + def visit_any(self, t: AnyType) -> str: + if t.missing_import_name: + return t.missing_import_name + else: + return "Any" + + def visit_instance(self, t: Instance) -> str: + s = t.type.fullname or t.type.name or None + if s is None: + return "" + if s in reverse_builtin_aliases: + s = reverse_builtin_aliases[s] + + mod_obj = split_target(self.graph, s) + assert mod_obj + mod, obj = mod_obj + + # If a class is imported into the current module, rewrite the reference + # to point to the current module. This helps the annotation tool avoid + # inserting redundant imports when a type has been reexported. + if self.module: + parts = obj.split(".") # need to split the object part if it is a nested class + tree = self.graph[self.module].tree + if tree and parts[0] in tree.names: + mod = self.module + + if (mod, obj) == ("builtins", "tuple"): + mod, obj = "typing", "Tuple[" + t.args[0].accept(self) + ", ...]" + elif t.args: + obj += f"[{self.list_str(t.args)}]" + + if mod_obj == ("builtins", "unicode"): + return "Text" + elif mod == "builtins": + return obj + else: + delim = "." if "." not in obj else ":" + return mod + delim + obj + + def visit_tuple_type(self, t: TupleType) -> str: + if t.partial_fallback and t.partial_fallback.type: + fallback_name = t.partial_fallback.type.fullname + if fallback_name != "builtins.tuple": + return t.partial_fallback.accept(self) + s = self.list_str(t.items) + return f"Tuple[{s}]" + + def visit_uninhabited_type(self, t: UninhabitedType) -> str: + return "Any" + + def visit_typeddict_type(self, t: TypedDictType) -> str: + return t.fallback.accept(self) + + def visit_union_type(self, t: UnionType) -> str: + if len(t.items) == 2 and is_overlapping_none(t): + return f"Optional[{remove_optional(t).accept(self)}]" + else: + return super().visit_union_type(t) + + def visit_callable_type(self, t: CallableType) -> str: + # TODO: use extended callables? + if is_tricky_callable(t): + arg_str = "..." + else: + # Note: for default arguments, we just assume that they + # are required. This isn't right, but neither is the + # other thing, and I suspect this will produce more better + # results than falling back to `...` + args = [typ.accept(self) for typ in t.arg_types] + arg_str = f"[{', '.join(args)}]" + + return f"Callable[{arg_str}, {t.ret_type.accept(self)}]" + + +TType = TypeVar("TType", bound=Type) + + +def make_suggestion_anys(t: TType) -> TType: + """Make all anys in the type as coming from the suggestion engine. + + This keeps those Anys from influencing constraint generation, + which allows us to do better when refining types. + """ + return cast(TType, t.accept(MakeSuggestionAny())) + + +class MakeSuggestionAny(TypeTranslator): + def visit_any(self, t: AnyType) -> Type: + if not t.missing_import_name: + return t.copy_modified(type_of_any=TypeOfAny.suggestion_engine) + else: + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + +def generate_type_combinations(types: list[Type]) -> list[Type]: + """Generate possible combinations of a list of types. + + mypy essentially supports two different ways to do this: joining the types + and unioning the types. We try both. + """ + joined_type = join_type_list(types) + union_type = make_simplified_union(types) + if joined_type == union_type: + return [joined_type] + else: + return [joined_type, union_type] + + +def count_errors(msgs: list[str]) -> int: + return len([x for x in msgs if " error: " in x]) + + +def refine_type(ti: Type, si: Type) -> Type: + """Refine `ti` by replacing Anys in it with information taken from `si` + + This basically works by, when the types have the same structure, + traversing both of them in parallel and replacing Any on the left + with whatever the type on the right is. If the types don't have the + same structure (or aren't supported), the left type is chosen. + + For example: + refine(Any, T) = T, for all T + refine(float, int) = float + refine(List[Any], List[int]) = List[int] + refine(Dict[int, Any], Dict[Any, int]) = Dict[int, int] + refine(Tuple[int, Any], Tuple[Any, int]) = Tuple[int, int] + + refine(Callable[[Any], Any], Callable[[int], int]) = Callable[[int], int] + refine(Callable[..., int], Callable[[int, float], Any]) = Callable[[int, float], int] + + refine(Optional[Any], int) = Optional[int] + refine(Optional[Any], Optional[int]) = Optional[int] + refine(Optional[Any], Union[int, str]) = Optional[Union[int, str]] + refine(Optional[List[Any]], List[int]) = List[int] + + """ + t = get_proper_type(ti) + s = get_proper_type(si) + + if isinstance(t, AnyType): + # If s is also an Any, we return if it is a missing_import Any + return t if isinstance(s, AnyType) and t.missing_import_name else s + + if isinstance(t, Instance) and isinstance(s, Instance) and t.type == s.type: + return t.copy_modified(args=[refine_type(ta, sa) for ta, sa in zip(t.args, s.args)]) + + if ( + isinstance(t, TupleType) + and isinstance(s, TupleType) + and t.partial_fallback == s.partial_fallback + and len(t.items) == len(s.items) + ): + return t.copy_modified(items=[refine_type(ta, sa) for ta, sa in zip(t.items, s.items)]) + + if isinstance(t, CallableType) and isinstance(s, CallableType): + return refine_callable(t, s) + + if isinstance(t, UnionType): + return refine_union(t, s) + + # TODO: Refining of builtins.tuple, Type? + + return t + + +def refine_union(t: UnionType, s: ProperType) -> Type: + """Refine a union type based on another type. + + This is done by refining every component of the union against the + right hand side type (or every component of its union if it is + one). If an element of the union is successfully refined, we drop it + from the union in favor of the refined versions. + """ + # Don't try to do any union refining if the types are already the + # same. This prevents things like refining Optional[Any] against + # itself and producing None. + if t == s: + return t + + rhs_items = s.items if isinstance(s, UnionType) else [s] + + new_items = [] + for lhs in t.items: + refined = False + for rhs in rhs_items: + new = refine_type(lhs, rhs) + if new != lhs: + new_items.append(new) + refined = True + if not refined: + new_items.append(lhs) + + # Turn strict optional on when simplifying the union since we + # don't want to drop Nones. + with state.strict_optional_set(True): + return make_simplified_union(new_items) + + +def refine_callable(t: CallableType, s: CallableType) -> CallableType: + """Refine a callable based on another. + + See comments for refine_type. + """ + if t.fallback != s.fallback: + return t + + if t.is_ellipsis_args and not is_tricky_callable(s): + return s.copy_modified(ret_type=refine_type(t.ret_type, s.ret_type)) + + if is_tricky_callable(t) or t.arg_kinds != s.arg_kinds: + return t + + return t.copy_modified( + arg_types=[refine_type(ta, sa) for ta, sa in zip(t.arg_types, s.arg_types)], + ret_type=refine_type(t.ret_type, s.ret_type), + ) + + +T = TypeVar("T") + + +def dedup(old: list[T]) -> list[T]: + new: list[T] = [] + for x in old: + if x not in new: + new.append(x) + return new diff --git a/mypy/test/config.py b/mypy/test/config.py index 5a7b5767a3d7..3806cf3dfa13 100644 --- a/mypy/test/config.py +++ b/mypy/test/config.py @@ -1,22 +1,28 @@ -import os -import os.path - -import typing +from __future__ import annotations +import os.path -PREFIX = '' +provided_prefix = os.getenv("MYPY_TEST_PREFIX", None) +if provided_prefix: + PREFIX = provided_prefix +else: + this_file_dir = os.path.dirname(os.path.realpath(__file__)) + PREFIX = os.path.dirname(os.path.dirname(this_file_dir)) # Location of test data files such as test case descriptions. -test_data_prefix = os.path.join(PREFIX, 'mypy', 'test', 'data') - -assert os.path.isdir(test_data_prefix), \ - 'Test data prefix ({}) not set correctly'.format(test_data_prefix) +test_data_prefix = os.path.join(PREFIX, "test-data", "unit") +package_path = os.path.join(PREFIX, "test-data", "packages") # Temp directory used for the temp files created when running test cases. -test_temp_dir = os.path.join(PREFIX, 'tmp') - -if not os.path.isdir(test_temp_dir): - os.mkdir(test_temp_dir) - -assert os.path.isdir(test_temp_dir), \ - 'Test temp dir ({}) not set correctly'.format(test_temp_dir) +# This is *within* the tempfile.TemporaryDirectory that is chroot'ed per testcase. +# It is also hard-coded in numerous places, so don't change it. +test_temp_dir = "tmp" + +# The PEP 561 tests do a bunch of pip installs which, even though they operate +# on distinct temporary virtual environments, run into race conditions on shared +# file-system state. To make this work reliably in parallel mode, we'll use a +# FileLock courtesy of the tox-dev/py-filelock package. +# Ref. https://github.com/python/mypy/issues/12615 +# Ref. mypy/test/testpep561.py +pip_lock = os.path.join(package_path, ".pip_lock") +pip_timeout = 60 diff --git a/mypy/test/data.py b/mypy/test/data.py index 364cbc9d9d86..bc17178d20e0 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -1,173 +1,424 @@ """Utilities for processing .test files containing test case descriptions.""" -import os.path +from __future__ import annotations + import os +import os.path +import posixpath import re -from os import remove, rmdir import shutil +import sys +import tempfile +from abc import abstractmethod +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final, Iterator, NamedTuple, NoReturn, Pattern, Union +from typing_extensions import TypeAlias as _TypeAlias -from typing import Function, List, Tuple, Undefined +import pytest -from mypy.myunit import TestCase, SkipTestCaseException +from mypy import defaults +from mypy.test.config import PREFIX, test_data_prefix, test_temp_dir +root_dir = os.path.normpath(PREFIX) -def parse_test_cases( - path: str, - perform: Function[['DataDrivenTestCase'], None], - base_path: str = '.', - optional_out: bool = False, - include_path: str = None) -> List['DataDrivenTestCase']: - """Parse a file with test case descriptions. +# Debuggers that we support for debugging mypyc run tests +# implementation of using each of these debuggers is in test_run.py +# TODO: support more debuggers +SUPPORTED_DEBUGGERS: Final = ["gdb", "lldb"] - Return an array of test cases. - """ - if not include_path: - include_path = os.path.dirname(path) - l = open(path).readlines() - for i in range(len(l)): - l[i] = l[i].rstrip('\n') - p = parse_test_data(l, path) - out = [] # type: List[DataDrivenTestCase] +# File modify/create operation: copy module contents from source_path. +class UpdateFile(NamedTuple): + module: str + content: str + target_path: str + + +# File delete operation: delete module file. +class DeleteFile(NamedTuple): + module: str + path: str + + +FileOperation: _TypeAlias = Union[UpdateFile, DeleteFile] + + +def _file_arg_to_module(filename: str) -> str: + filename, _ = os.path.splitext(filename) + parts = filename.split("/") # not os.sep since it comes from test data + if parts[-1] == "__init__": + parts.pop() + return ".".join(parts) + + +def parse_test_case(case: DataDrivenTestCase) -> None: + """Parse and prepare a single case from suite with test case descriptions. + + This method is part of the setup phase, just before the test case is run. + """ + test_items = parse_test_data(case.data, case.name) + base_path = case.suite.base_path + if case.suite.native_sep: + join = os.path.join + else: + join = posixpath.join + + out_section_missing = case.suite.required_out_section + + files: list[tuple[str, str]] = [] # path and contents + output_files: list[tuple[str, str | Pattern[str]]] = [] # output path and contents + output: list[str] = [] # Regular output errors + output2: dict[int, list[str]] = {} # Output errors for incremental, runs 2+ + deleted_paths: dict[int, set[str]] = {} # from run number of paths + stale_modules: dict[int, set[str]] = {} # from run number to module names + rechecked_modules: dict[int, set[str]] = {} # from run number module names + triggered: list[str] = [] # Active triggers (one line per incremental step) + targets: dict[int, list[str]] = {} # Fine-grained targets (per fine-grained update) + test_modules: list[str] = [] # Modules which are deemed "test" (vs "fixture") + + def _case_fail(msg: str) -> NoReturn: + pytest.fail(f"{case.file}:{case.line}: {msg}", pytrace=False) # Process the parsed items. Each item has a header of form [id args], # optionally followed by lines of text. - i = 0 - while i < len(p): - ok = False - i0 = i - if p[i].id == 'case': - i += 1 - - files = [] # type: List[Tuple[str, str]] # path and contents - while i < len(p) and p[i].id not in ['out', 'case']: - if p[i].id == 'file': - # Record an extra file needed for the test case. - files.append((os.path.join(base_path, p[i].arg), - '\n'.join(p[i].data))) - elif p[i].id == 'builtins': - # Use a custom source file for the std module. - mpath = os.path.join(os.path.dirname(path), p[i].arg) - f = open(mpath) - files.append((os.path.join(base_path, 'builtins.py'), - f.read())) - f.close() + item = first_item = test_items[0] + test_modules.append("__main__") + for item in test_items[1:]: + + def _item_fail(msg: str) -> NoReturn: + item_abs_line = case.line + item.line - 2 + pytest.fail(f"{case.file}:{item_abs_line}: {msg}", pytrace=False) + + if item.id in {"file", "fixture", "outfile", "outfile-re"}: + # Record an extra file needed for the test case. + assert item.arg is not None + contents = expand_variables("\n".join(item.data)) + path = join(base_path, item.arg) + if item.id != "fixture": + test_modules.append(_file_arg_to_module(item.arg)) + if item.id in {"file", "fixture"}: + files.append((path, contents)) + elif item.id == "outfile-re": + output_files.append((path, re.compile(contents.rstrip(), re.S))) + elif item.id == "outfile": + output_files.append((path, contents)) + elif item.id == "builtins": + # Use an alternative stub file for the builtins module. + assert item.arg is not None + mpath = join(os.path.dirname(case.file), item.arg) + with open(mpath, encoding="utf8") as f: + files.append((join(base_path, "builtins.pyi"), f.read())) + elif item.id == "typing": + # Use an alternative stub file for the typing module. + assert item.arg is not None + src_path = join(os.path.dirname(case.file), item.arg) + with open(src_path, encoding="utf8") as f: + files.append((join(base_path, "typing.pyi"), f.read())) + elif item.id == "_typeshed": + # Use an alternative stub file for the _typeshed module. + assert item.arg is not None + src_path = join(os.path.dirname(case.file), item.arg) + with open(src_path, encoding="utf8") as f: + files.append((join(base_path, "_typeshed.pyi"), f.read())) + elif re.match(r"stale[0-9]*$", item.id): + passnum = 1 if item.id == "stale" else int(item.id[len("stale") :]) + assert passnum > 0 + modules = set() if item.arg is None else {t.strip() for t in item.arg.split(",")} + stale_modules[passnum] = modules + elif re.match(r"rechecked[0-9]*$", item.id): + passnum = 1 if item.id == "rechecked" else int(item.id[len("rechecked") :]) + assert passnum > 0 + modules = set() if item.arg is None else {t.strip() for t in item.arg.split(",")} + rechecked_modules[passnum] = modules + elif re.match(r"targets[0-9]*$", item.id): + passnum = 1 if item.id == "targets" else int(item.id[len("targets") :]) + assert passnum > 0 + reprocessed = [] if item.arg is None else [t.strip() for t in item.arg.split(",")] + targets[passnum] = reprocessed + elif item.id == "delete": + # File/directory to delete during a multi-step test case + assert item.arg is not None + m = re.match(r"(.*)\.([0-9]+)$", item.arg) + if m is None: + _item_fail(f"Invalid delete section {item.arg!r}") + num = int(m.group(2)) + if num < 2: + _item_fail(f"Can't delete during step {num}") + full = join(base_path, m.group(1)) + deleted_paths.setdefault(num, set()).add(full) + elif re.match(r"out[0-9]*$", item.id): + if item.arg is None: + args = [] + else: + args = item.arg.split(",") + + version_check = True + for arg in args: + if arg.startswith("version"): + compare_op = arg[7:9] + if compare_op not in {">=", "=="}: + _item_fail("Only >= and == version checks are currently supported") + version_str = arg[9:] + try: + version = tuple(int(x) for x in version_str.split(".")) + except ValueError: + _item_fail(f"{version_str!r} is not a valid python version") + if compare_op == ">=": + if version <= defaults.PYTHON3_VERSION: + _item_fail( + f"{arg} always true since minimum runtime version is {defaults.PYTHON3_VERSION}" + ) + version_check = sys.version_info >= version + elif compare_op == "==": + if version < defaults.PYTHON3_VERSION: + _item_fail( + f"{arg} always false since minimum runtime version is {defaults.PYTHON3_VERSION}" + ) + if not 1 < len(version) < 4: + _item_fail( + f'Only minor or patch version checks are currently supported with "==": {version_str!r}' + ) + version_check = sys.version_info[: len(version)] == version + if version_check: + tmp_output = [expand_variables(line) for line in item.data] + if os.path.sep == "\\" and case.normalize_output: + tmp_output = [fix_win_path(line) for line in tmp_output] + if item.id == "out" or item.id == "out1": + output = tmp_output else: - raise ValueError( - 'Invalid section header {} in {} at line {}'.format( - p[i].id, path, p[i].line)) - i += 1 - - tcout = [] # type: List[str] - if i < len(p) and p[i].id == 'out': - tcout = p[i].data - ok = True - i += 1 - elif optional_out: - ok = True - - if ok: - input = expand_includes(p[i0].data, include_path) - expand_errors(input, tcout, 'main') - lastline = p[i].line if i < len(p) else p[i - 1].line + 9999 - tc = DataDrivenTestCase(p[i0].arg, input, tcout, path, - p[i0].line, lastline, perform, files) - out.append(tc) - if not ok: - raise ValueError( - '{}, line {}: Error in test case description'.format( - path, p[i0].line)) - - return out - - -class DataDrivenTestCase(TestCase): - input = Undefined(List[str]) - output = Undefined(List[str]) - - file = '' + passnum = int(item.id[len("out") :]) + assert passnum > 1 + output2[passnum] = tmp_output + out_section_missing = False + elif item.id == "triggered" and item.arg is None: + triggered = item.data + else: + section_str = item.id + (f" {item.arg}" if item.arg else "") + _item_fail(f"Invalid section header [{section_str}] in case {case.name!r}") + + if out_section_missing: + _case_fail(f"Required output section not found in case {case.name!r}") + + for passnum in stale_modules.keys(): + if passnum not in rechecked_modules: + # If the set of rechecked modules isn't specified, make it the same as the set + # of modules with a stale public interface. + rechecked_modules[passnum] = stale_modules[passnum] + if ( + passnum in stale_modules + and passnum in rechecked_modules + and not stale_modules[passnum].issubset(rechecked_modules[passnum]) + ): + _case_fail(f"Stale modules after pass {passnum} must be a subset of rechecked modules") + + output_inline_start = len(output) + input = first_item.data + expand_errors(input, output, "main") + for file_path, contents in files: + expand_errors(contents.split("\n"), output, file_path) + + seen_files = set() + for file, _ in files: + if file in seen_files: + _case_fail(f"Duplicated filename {file}. Did you include it multiple times?") + + seen_files.add(file) + + case.input = input + case.output = output + case.output_inline_start = output_inline_start + case.output2 = output2 + case.last_line = case.line + item.line + len(item.data) - 2 + case.files = files + case.output_files = output_files + case.expected_stale_modules = stale_modules + case.expected_rechecked_modules = rechecked_modules + case.deleted_paths = deleted_paths + case.triggered = triggered or [] + case.expected_fine_grained_targets = targets + case.test_modules = test_modules + + +class DataDrivenTestCase(pytest.Item): + """Holds parsed data-driven test cases, and handles directory setup and teardown.""" + + # Override parent member type + parent: DataSuiteCollector + + input: list[str] + output: list[str] # Output for the first pass + output_inline_start: int + output2: dict[int, list[str]] # Output for runs 2+, indexed by run number + + # full path of test suite + file = "" line = 0 - perform = Undefined(Function[['DataDrivenTestCase'], None]) - # (file path, file content) tuples - files = Undefined(List[Tuple[str, str]]) - - clean_up = Undefined(List[Tuple[bool, str]]) - - def __init__(self, name, input, output, file, line, lastline, - perform, files): - super().__init__(name) - self.input = input - self.output = output - self.lastline = lastline + files: list[tuple[str, str]] + # Modules which is to be considered "test" rather than "fixture" + test_modules: list[str] + expected_stale_modules: dict[int, set[str]] + expected_rechecked_modules: dict[int, set[str]] + expected_fine_grained_targets: dict[int, list[str]] + + # Whether or not we should normalize the output to standardize things like + # forward vs backward slashes in file paths for Windows vs Linux. + normalize_output: bool + + # Extra attributes used by some tests. + last_line: int + output_files: list[tuple[str, str | Pattern[str]]] # Path and contents for output files + deleted_paths: dict[int, set[str]] # Mapping run number -> paths + triggered: list[str] # Active triggers (one line per incremental step) + + def __init__( + self, + parent: DataSuiteCollector, + suite: DataSuite, + *, + file: str, + name: str, + writescache: bool, + only_when: str, + normalize_output: bool, + platform: str | None, + skip: bool, + xfail: bool, + data: str, + line: int, + ) -> None: + super().__init__(name, parent) + self.suite = suite self.file = file + self.writescache = writescache + self.only_when = only_when + self.normalize_output = normalize_output + if (platform == "windows" and sys.platform != "win32") or ( + platform == "posix" and sys.platform == "win32" + ): + skip = True + self.skip = skip + self.xfail = xfail + self.data = data self.line = line - self.perform = perform - self.files = files + self.old_cwd: str | None = None + self.tmpdir: str | None = None + + def runtest(self) -> None: + if self.skip: + pytest.skip() + # TODO: add a better error message for when someone uses skip and xfail at the same time + elif self.xfail: + self.add_marker(pytest.mark.xfail) + parent = self.getparent(DataSuiteCollector) + assert parent is not None, "Should not happen" + suite = parent.obj() + suite.setup() + try: + suite.run_case(self) + except Exception: + # As a debugging aid, support copying the contents of the tmp directory somewhere + save_dir: str | None = self.config.getoption("--save-failures-to", None) + if save_dir: + assert self.tmpdir is not None + target_dir = os.path.join(save_dir, os.path.basename(self.tmpdir)) + print(f"Copying data from test {self.name} to {target_dir}") + if not os.path.isabs(target_dir): + assert self.old_cwd + target_dir = os.path.join(self.old_cwd, target_dir) + shutil.copytree(self.tmpdir, target_dir) + raise + + def setup(self) -> None: + parse_test_case(case=self) + self.old_cwd = os.getcwd() + self.tmpdir = tempfile.mkdtemp(prefix="mypy-test-") + os.chdir(self.tmpdir) + os.mkdir(test_temp_dir) + + # Precalculate steps for find_steps() + steps: dict[int, list[FileOperation]] = {} - def set_up(self) -> None: - super().set_up() - self.clean_up = [] for path, content in self.files: - dir = os.path.dirname(path) - for d in self.add_dirs(dir): - self.clean_up.append((True, d)) - f = open(path, 'w') - f.write(content) - f.close() - self.clean_up.append((False, path)) - - def add_dirs(self, dir: str) -> List[str]: - """Add all subdirectories required to create dir. - - Return an array of the created directories in the order of creation. - """ - if dir == '' or os.path.isdir(dir): - return [] + m = re.match(r".*\.([0-9]+)$", path) + if m: + # Skip writing subsequent incremental steps - rather + # store them as operations. + num = int(m.group(1)) + assert num >= 2 + target_path = re.sub(r"\.[0-9]+$", "", path) + module = module_from_path(target_path) + operation = UpdateFile(module, content, target_path) + steps.setdefault(num, []).append(operation) + else: + # Write the first incremental steps + dir = os.path.dirname(path) + os.makedirs(dir, exist_ok=True) + with open(path, "w", encoding="utf8") as f: + f.write(content) + + for num, paths in self.deleted_paths.items(): + assert num >= 2 + for path in paths: + module = module_from_path(path) + steps.setdefault(num, []).append(DeleteFile(module, path)) + max_step = max(steps) if steps else 2 + self.steps = [steps.get(num, []) for num in range(2, max_step + 1)] + + def teardown(self) -> None: + if self.old_cwd is not None: + os.chdir(self.old_cwd) + if self.tmpdir is not None: + shutil.rmtree(self.tmpdir, ignore_errors=True) + self.old_cwd = None + self.tmpdir = None + + def reportinfo(self) -> tuple[str, int, str]: + return self.file, self.line, self.name + + def repr_failure( + self, excinfo: pytest.ExceptionInfo[BaseException], style: Any | None = None + ) -> str: + excrepr: object + if isinstance(excinfo.value, SystemExit): + # We assume that before doing exit() (which raises SystemExit) we've printed + # enough context about what happened so that a stack trace is not useful. + # In particular, uncaught exceptions during semantic analysis or type checking + # call exit() and they already print out a stack trace. + excrepr = excinfo.exconly() + elif isinstance(excinfo.value, pytest.fail.Exception) and not excinfo.value.pytrace: + excrepr = excinfo.exconly() else: - dirs = self.add_dirs(os.path.dirname(dir)) + [dir] - os.mkdir(dir) - return dirs + excinfo.traceback = self.parent._traceback_filter(excinfo) + excrepr = excinfo.getrepr(style="short") + + return f"data: {self.file}:{self.line}:\n{excrepr}" + + def find_steps(self) -> list[list[FileOperation]]: + """Return a list of descriptions of file operations for each incremental step. + + The first list item corresponds to the first incremental step, the second for the + second step, etc. Each operation can either be a file modification/creation (UpdateFile) + or deletion (DeleteFile). + + Defaults to having two steps if there aern't any operations. + """ + return self.steps - def run(self): - if self.name.endswith('-skip'): - raise SkipTestCaseException() - else: - self.perform(self) - - def tear_down(self) -> None: - # First remove files. - for is_dir, path in reversed(self.clean_up): - if not is_dir: - remove(path) - # Then remove directories. - for is_dir, path in reversed(self.clean_up): - if is_dir: - pycache = os.path.join(path, '__pycache__') - if os.path.isdir(pycache): - shutil.rmtree(pycache) - try: - rmdir(path) - except OSError as error: - print(' ** Error removing directory %s -- contents:' % path) - for item in os.listdir(path): - print(' ', item) - # Most likely, there are some files in the - # directory. Use rmtree to nuke the directory, but - # fail the test case anyway, since this seems like - # a bug in a test case -- we shouldn't leave - # garbage lying around. By nuking the directory, - # the next test run hopefully passes. - path = error.filename - # Be defensive -- only call rmtree if we're sure we aren't removing anything - # valuable. - if path.startswith('tmp/') and os.path.isdir(path): - shutil.rmtree(path) - raise - super().tear_down() +def module_from_path(path: str) -> str: + path = re.sub(r"\.pyi?$", "", path) + # We can have a mix of Unix-style and Windows-style separators. + parts = re.split(r"[/\\]", path) + del parts[0] + module = ".".join(parts) + module = re.sub(r"\.__init__$", "", module) + return module + +@dataclass class TestItem: """Parsed test caseitem. @@ -176,124 +427,394 @@ class TestItem: .. data .. """ - id = '' - arg = '' - - # Text data, array of 8-bit strings - data = Undefined(List[str]) + id: str + arg: str | None + # Processed, collapsed text data + data: list[str] + # Start line: 1-based, inclusive, relative to testcase + line: int + # End line: 1-based, exclusive, relative to testcase; not same as `line + len(test_item.data)` due to collapsing + end_line: int - file = '' - line = 0 # Line number in file - - def __init__(self, id: str, arg: str, data: List[str], file: str, - line: int) -> None: - self.id = id - self.arg = arg - self.data = data - self.file = file - self.line = line + @property + def trimmed_newlines(self) -> int: # compensates for strip_list + return self.end_line - self.line - len(self.data) -def parse_test_data(l: List[str], fnam: str) -> List[TestItem]: +def parse_test_data(raw_data: str, name: str) -> list[TestItem]: """Parse a list of lines that represent a sequence of test items.""" - ret = [] # type: List[TestItem] - data = [] # type: List[str] + lines = ["", "[case " + name + "]"] + raw_data.split("\n") + ret: list[TestItem] = [] + data: list[str] = [] - id = None # type: str - arg = None # type: str + id: str | None = None + arg: str | None = None i = 0 i0 = 0 - while i < len(l): - s = l[i].strip() + while i < len(lines): + s = lines[i].strip() - if l[i].startswith('[') and s.endswith(']') and not s.startswith('[['): + if lines[i].startswith("[") and s.endswith("]"): if id: data = collapse_line_continuation(data) data = strip_list(data) - ret.append(TestItem(id, arg, strip_list(data), fnam, i0 + 1)) + ret.append(TestItem(id, arg, data, i0 + 1, i)) + i0 = i id = s[1:-1] arg = None - if ' ' in id: - arg = id[id.index(' ') + 1:] - id = id[:id.index(' ')] + if " " in id: + arg = id[id.index(" ") + 1 :] + id = id[: id.index(" ")] data = [] - elif l[i].startswith('[['): - data.append(l[i][1:]) - elif not l[i].startswith('--'): - data.append(l[i]) - elif l[i].startswith('----'): - data.append(l[i][2:]) + elif lines[i].startswith("\\["): + data.append(lines[i][1:]) + elif not lines[i].startswith("--"): + data.append(lines[i]) + elif lines[i].startswith("----"): + data.append(lines[i][2:]) i += 1 # Process the last item. if id: data = collapse_line_continuation(data) data = strip_list(data) - ret.append(TestItem(id, arg, data, fnam, i0 + 1)) + ret.append(TestItem(id, arg, data, i0 + 1, i - 1)) return ret -def strip_list(l: List[str]) -> List[str]: +def strip_list(l: list[str]) -> list[str]: """Return a stripped copy of l. Strip whitespace at the end of all lines, and strip all empty lines from the end of the array. """ - r = [] # type: List[str] + r: list[str] = [] for s in l: # Strip spaces at end of line - r.append(re.sub(r'\s+$', '', s)) + r.append(re.sub(r"\s+$", "", s)) - while len(r) > 0 and r[-1] == '': + while r and r[-1] == "": r.pop() return r -def collapse_line_continuation(l: List[str]) -> List[str]: - r = [] # type: List[str] +def collapse_line_continuation(l: list[str]) -> list[str]: + r: list[str] = [] cont = False for s in l: - ss = re.sub(r'\\$', '', s) + ss = re.sub(r"\\$", "", s) if cont: - r[-1] += re.sub('^ +', '', ss) + r[-1] += re.sub("^ +", "", ss) else: r.append(ss) - cont = s.endswith('\\') + cont = s.endswith("\\") return r -def expand_includes(a: List[str], base_path: str) -> List[str]: - """Expand @includes within a list of lines. +def expand_variables(s: str) -> str: + return s.replace("", root_dir) + - Replace all lies starting with @include with the contents of the - file name following the prefix. Look for the files in base_path. +def expand_errors(input: list[str], output: list[str], fnam: str) -> None: + """Transform comments such as '# E: message' or + '# E:3: message' in input. + + The result is lines like 'fnam:line: error: message'. """ - res = [] # type: List[str] - for s in a: - if s.startswith('@include '): - fn = s.split(' ', 1)[1].strip() - f = open(os.path.join(base_path, fn)) - res.extend(f.readlines()) - f.close() - else: - res.append(s) - return res + for i in range(len(input)): + # The first in the split things isn't a comment + for possible_err_comment in input[i].split(" # ")[1:]: + m = re.search( + r"^([ENW]):((?P\d+):)? (?P.*)$", possible_err_comment.strip() + ) + if m: + if m.group(1) == "E": + severity = "error" + elif m.group(1) == "N": + severity = "note" + elif m.group(1) == "W": + severity = "warning" + col = m.group("col") + message = m.group("message") + message = message.replace("\\#", "#") # adds back escaped # character + if col is None: + output.append(f"{fnam}:{i + 1}: {severity}: {message}") + else: + output.append(f"{fnam}:{i + 1}:{col}: {severity}: {message}") -def expand_errors(input, output, fnam): - """Transform comments such as '# E: message' in input. +def fix_win_path(line: str) -> str: + r"""Changes Windows paths to Linux paths in error messages. - The result is lines like 'fnam, line N: message'. + E.g. foo\bar.py -> foo/bar.py. """ + line = line.replace(root_dir, root_dir.replace("\\", "/")) + m = re.match(r"^([\S/]+):(\d+:)?(\s+.*)", line) + if not m: + return line + else: + filename, lineno, message = m.groups() + return "{}:{}{}".format(filename.replace("\\", "/"), lineno or "", message) - for i in range(len(input)): - m = re.search('# E: (.*)$', input[i]) - if m: - output.append('{}, line {}: {}'.format(fnam, i + 1, m.group(1))) + +def fix_cobertura_filename(line: str) -> str: + r"""Changes filename paths to Linux paths in Cobertura output files. + + E.g. filename="pkg\subpkg\a.py" -> filename="pkg/subpkg/a.py". + """ + m = re.search(r' None: + group = parser.getgroup("mypy") + group.addoption( + "--update-data", + action="store_true", + default=False, + help="Update test data to reflect actual output (supported only for certain tests)", + ) + group.addoption( + "--save-failures-to", + default=None, + help="Copy the temp directories from failing tests to a target directory", + ) + group.addoption( + "--mypy-verbose", action="count", help="Set the verbose flag when creating mypy Options" + ) + group.addoption( + "--mypyc-showc", + action="store_true", + default=False, + help="Display C code on mypyc test failures", + ) + group.addoption( + "--mypyc-debug", + default=None, + dest="debugger", + choices=SUPPORTED_DEBUGGERS, + help="Run the first mypyc run test with the specified debugger", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_cmdline_main(config: pytest.Config) -> None: + if config.getoption("--collectonly"): + return + # --update-data is not compatible with parallelized tests, disable parallelization + if config.getoption("--update-data"): + config.option.numprocesses = 0 + + +# This function name is special to pytest. See +# https://doc.pytest.org/en/latest/how-to/writing_plugins.html#collection-hooks +def pytest_pycollect_makeitem(collector: Any, name: str, obj: object) -> Any | None: + """Called by pytest on each object in modules configured in conftest.py files. + + collector is pytest.Collector, returns Optional[pytest.Class] + """ + if isinstance(obj, type): + # Only classes derived from DataSuite contain test cases, not the DataSuite class itself + if issubclass(obj, DataSuite) and obj is not DataSuite: + # Non-None result means this obj is a test case. + # The collect method of the returned DataSuiteCollector instance will be called later, + # with self.obj being obj. + return DataSuiteCollector.from_parent(parent=collector, name=name) + return None + + +_case_name_pattern = re.compile( + r"(?P[a-zA-Z_0-9]+)" + r"(?P-writescache)?" + r"(?P-only_when_cache|-only_when_nocache)?" + r"(?P-skip_path_normalization)?" + r"(-(?Pposix|windows))?" + r"(?P-skip)?" + r"(?P-xfail)?" +) + + +def split_test_cases( + parent: DataFileCollector, suite: DataSuite, file: str +) -> Iterator[DataDrivenTestCase]: + """Iterate over raw test cases in file, at collection time, ignoring sub items. + + The collection phase is slow, so any heavy processing should be deferred to after + uninteresting tests are filtered (when using -k PATTERN switch). + """ + with open(file, encoding="utf-8") as f: + data = f.read() + cases = re.split(r"^\[case ([^]+)]+)\][ \t]*$\n", data, flags=re.DOTALL | re.MULTILINE) + cases_iter = iter(cases) + line_no = next(cases_iter).count("\n") + 1 + test_names = set() + for case_id in cases_iter: + data = next(cases_iter) + + m = _case_name_pattern.fullmatch(case_id) + if not m: + raise RuntimeError(f"Invalid testcase id {case_id!r}") + name = m.group("name") + if name in test_names: + raise RuntimeError( + 'Found a duplicate test name "{}" in {} on line {}'.format( + name, parent.name, line_no + ) + ) + yield DataDrivenTestCase.from_parent( + parent=parent, + suite=suite, + file=file, + name=add_test_name_suffix(name, suite.test_name_suffix), + writescache=bool(m.group("writescache")), + only_when=m.group("only_when"), + platform=m.group("platform"), + skip=bool(m.group("skip")), + xfail=bool(m.group("xfail")), + normalize_output=not m.group("skip_path_normalization"), + data=data, + line=line_no, + ) + line_no += data.count("\n") + 1 + + # Record existing tests to prevent duplicates: + test_names.update({name}) + + +class DataSuiteCollector(pytest.Class): + def collect(self) -> Iterator[DataFileCollector]: + """Called by pytest on each of the object returned from pytest_pycollect_makeitem""" + + # obj is the object for which pytest_pycollect_makeitem returned self. + suite: DataSuite = self.obj + + assert os.path.isdir( + suite.data_prefix + ), f"Test data prefix ({suite.data_prefix}) not set correctly" + + for data_file in suite.files: + yield DataFileCollector.from_parent(parent=self, name=data_file) + + +class DataFileFix(NamedTuple): + lineno: int # 1-offset, inclusive + end_lineno: int # 1-offset, exclusive + lines: list[str] + + +class DataFileCollector(pytest.Collector): + """Represents a single `.test` data driven test file. + + More context: https://github.com/python/mypy/issues/11662 + """ + + parent: DataSuiteCollector + + _fixes: list[DataFileFix] + + @classmethod # We have to fight with pytest here: + def from_parent( + cls, parent: DataSuiteCollector, *, name: str # type: ignore[override] + ) -> DataFileCollector: + collector = super().from_parent(parent, name=name) + assert isinstance(collector, DataFileCollector) + return collector + + def collect(self) -> Iterator[DataDrivenTestCase]: + yield from split_test_cases( + parent=self, + suite=self.parent.obj, + file=os.path.join(self.parent.obj.data_prefix, self.name), + ) + + def setup(self) -> None: + super().setup() + self._fixes = [] + + def teardown(self) -> None: + super().teardown() + self._apply_fixes() + + def enqueue_fix(self, fix: DataFileFix) -> None: + self._fixes.append(fix) + + def _apply_fixes(self) -> None: + if not self._fixes: + return + data_path = Path(self.parent.obj.data_prefix) / self.name + lines = data_path.read_text().split("\n") + # start from end to prevent line offsets from shifting as we update + for fix in sorted(self._fixes, reverse=True): + lines[fix.lineno - 1 : fix.end_lineno - 1] = fix.lines + data_path.write_text("\n".join(lines)) + + +def add_test_name_suffix(name: str, suffix: str) -> str: + # Find magic suffix of form "-foobar" (used for things like "-skip"). + m = re.search(r"-[-A-Za-z0-9]+$", name) + if m: + # Insert suite-specific test name suffix before the magic suffix + # which must be the last thing in the test case name since we + # are using endswith() checks. + magic_suffix = m.group(0) + return name[: -len(magic_suffix)] + suffix + magic_suffix + else: + return name + suffix + + +def is_incremental(testcase: DataDrivenTestCase) -> bool: + return "incremental" in testcase.name.lower() or "incremental" in testcase.file + + +def has_stable_flags(testcase: DataDrivenTestCase) -> bool: + if any(re.match(r"# flags[2-9]:", line) for line in testcase.input): + return False + for filename, contents in testcase.files: + if os.path.basename(filename).startswith("mypy.ini."): + return False + return True + + +class DataSuite: + # option fields - class variables + files: list[str] + + base_path = test_temp_dir + + # Allow external users of the test code to override the data prefix + data_prefix = test_data_prefix + + required_out_section = False + + native_sep = False + + # Name suffix automatically added to each test case in the suite (can be + # used to distinguish test cases in suites that share data files) + test_name_suffix = "" + + def setup(self) -> None: + """Setup fixtures (ad-hoc)""" + + @abstractmethod + def run_case(self, testcase: DataDrivenTestCase) -> None: + raise NotImplementedError diff --git a/mypy/test/data/check-abstract.test b/mypy/test/data/check-abstract.test deleted file mode 100644 index a1dcae6ff84d..000000000000 --- a/mypy/test/data/check-abstract.test +++ /dev/null @@ -1,514 +0,0 @@ --- Type checker test cases for abstract classes. - - --- Subtyping with abstract classes --- ------------------------------- - - -[case testAbstractClassSubclasses] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -i = Undefined # type: I -j = Undefined # type: J -a = Undefined # type: A -b = Undefined # type: B -c = Undefined # type: C - -j = c # E: Incompatible types in assignment (expression has type "C", variable has type "J") -a = i # E: Incompatible types in assignment (expression has type "I", variable has type "A") -a = j # E: Incompatible types in assignment (expression has type "J", variable has type "A") -b = i # E: Incompatible types in assignment (expression has type "I", variable has type "B") - -i = a -i = b -i = c -j = a -j = b -a = b - -class I(metaclass=ABCMeta): - @abstractmethod - def f(self): pass -class J(metaclass=ABCMeta): - @abstractmethod - def g(self): pass -class A(I, J): pass -class B(A): pass -class C(I): pass - -[case testAbstractClassSubtypingViaExtension] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -i = Undefined # type: I -j = Undefined # type: J -a = Undefined # type: A -o = Undefined # type: object - -j = i # E: Incompatible types in assignment (expression has type "I", variable has type "J") -a = i # E: Incompatible types in assignment (expression has type "I", variable has type "A") -a = j # E: Incompatible types in assignment (expression has type "J", variable has type "A") -i = o # E: Incompatible types in assignment (expression has type "object", variable has type "I") -j = o # E: Incompatible types in assignment (expression has type "object", variable has type "J") - -i = a -j = a -i = j -o = i -o = j - -class I(metaclass=ABCMeta): - @abstractmethod - def f(self): pass -class J(I): pass -class A(J): pass - -[case testInheritingAbstractClassInSubclass] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -i = Undefined # type: I -a = Undefined # type: A -b = Undefined # type: B - -i = a # E: Incompatible types in assignment (expression has type "A", variable has type "I") -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = b -i = b - -class I(metaclass=ABCMeta): - @abstractmethod - def f(self): pass -class A: pass -class B(A, I): pass - - --- Abstract class objects --- ---------------------- - - -[case testAbstractClassAsTypeObject] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -o = Undefined # type: object -t = Undefined # type: type - -o = I -t = I - -class I(metaclass=ABCMeta): - @abstractmethod - def f(self): pass - -[case testAbstractClassInCasts] -from typing import Undefined, cast -from abc import abstractmethod, ABCMeta - -class I(metaclass=ABCMeta): - @abstractmethod - def f(self): pass -class A(I): pass -class B: pass - -i, a, b = Undefined, Undefined, Undefined # type: (I, A, B) -o = Undefined # type: object - -a = cast(I, o) # E: Incompatible types in assignment (expression has type "I", variable has type "A") -b = cast(B, i) # Ok; a subclass of B might inherit I -i = cast(I, b) # Ok; a subclass of B might inherit I - -i = cast(I, o) -i = cast(I, a) - -[case testInstantiatingClassThatImplementsAbstractMethod] -from abc import abstractmethod, ABCMeta -import typing -class A(metaclass=ABCMeta): - @abstractmethod - def f(self): pass -class B(A): - def f(self): pass -B() -[out] - -[case testInstantiatingAbstractClass] -from abc import abstractmethod, ABCMeta -import typing -class A(metaclass=ABCMeta): pass -class B(metaclass=ABCMeta): - @abstractmethod - def f(self): pass -A() # OK -B() # E: Cannot instantiate abstract class 'B' with abstract method 'f' -[out] - -[case testInstantiatingClassWithInheritedAbstractMethod] -from abc import abstractmethod, ABCMeta -import typing -class A(metaclass=ABCMeta): - @abstractmethod - def f(self): pass - @abstractmethod - def g(self): pass -class B(A): pass -B()# E: Cannot instantiate abstract class 'B' with abstract methods 'f' and 'g' -[out] - - --- Implementing abstract methods --- ----------------------------- - - -[case testImplementingAbstractMethod] -from abc import abstractmethod, ABCMeta -import typing -class A(metaclass=ABCMeta): - @abstractmethod - def f(self, x: int) -> int: pass - @abstractmethod - def g(self, x: int) -> int: pass -class B(A): - def f(self, x: str) -> int: \ - # E: Argument 1 of "f" incompatible with supertype "A" - pass - def g(self, x: int) -> int: pass -[out] -main: In class "B": - -[case testImplementingAbstractMethodWithMultipleBaseClasses] -from abc import abstractmethod, ABCMeta -import typing -class I(metaclass=ABCMeta): - @abstractmethod - def f(self, x: int) -> int: pass -class J(metaclass=ABCMeta): - @abstractmethod - def g(self, x: str) -> str: pass -class A(I, J): - def f(self, x: str) -> int: pass \ - # E: Argument 1 of "f" incompatible with supertype "I" - def g(self, x: str) -> int: pass \ - # E: Return type of "g" incompatible with supertype "J" - def h(self) -> int: pass # Not related to any base class -[out] -main: In class "A": - -[case testImplementingAbstractMethodWithExtension] -from abc import abstractmethod, ABCMeta -import typing -class J(metaclass=ABCMeta): - @abstractmethod - def f(self, x: int) -> int: pass -class I(J): pass -class A(I): - def f(self, x: str) -> int: pass \ - # E: Argument 1 of "f" incompatible with supertype "J" -[out] -main: In class "A": - -[case testInvalidOverridingAbstractMethod] -from abc import abstractmethod, ABCMeta -import typing -class J(metaclass=ABCMeta): - @abstractmethod - def f(self, x: 'J') -> None: pass -class I(J): - @abstractmethod - def f(self, x: 'I') -> None: pass # E: Argument 1 of "f" incompatible with supertype "J" -[out] -main: In class "I": - -[case testAbstractClassCoAndContraVariance] -from abc import abstractmethod, ABCMeta -import typing -class I(metaclass=ABCMeta): - @abstractmethod - def f(self, a: A) -> 'I': pass - @abstractmethod - def g(self, a: A) -> 'I': pass - @abstractmethod - def h(self, a: 'I') -> A: pass -class A(I): - def h(self, a: 'A') -> 'I': # Fail - pass - def f(self, a: 'I') -> 'I': - pass - def g(self, a: 'A') -> 'A': - pass -[out] -main: In class "A": -main, line 11: Argument 1 of "h" incompatible with supertype "I" -main, line 11: Return type of "h" incompatible with supertype "I" - - --- Accessing abstract members --- -------------------------- - - -[case testAccessingAbstractMethod] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -class I(metaclass=ABCMeta): - @abstractmethod - def f(self, a: int) -> str: pass - -i, a, b = Undefined, Undefined, Undefined # type: (I, int, str) - -a = i.f(a) # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = i.f(b) # E: Argument 1 to "f" of "I" has incompatible type "str"; expected "int" -i.g() # E: "I" has no attribute "g" - -b = i.f(a) - -[case testAccessingInheritedAbstractMethod] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -class J(metaclass=ABCMeta): - @abstractmethod - def f(self, a: int) -> str: pass -class I(J): pass - -i, a, b = Undefined, Undefined, Undefined # type: (I, int, str) - -a = i.f(1) # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = i.f(1) - - --- Any (dynamic) types --- ------------------- - - -[case testAbstractClassWithAllDynamicTypes] -from abc import abstractmethod, ABCMeta -import typing -class I(metaclass=ABCMeta): - @abstractmethod - def f(self, x): pass - @abstractmethod - def g(self, x): pass -class A(I): - def f(self, x): pass - def g(self, x, y): pass \ - # E: Signature of "g" incompatible with supertype "I" -[out] -main: In class "A": - -[case testAbstractClassWithImplementationUsingDynamicTypes] -from abc import abstractmethod, ABCMeta -import typing -class I(metaclass=ABCMeta): - @abstractmethod - def f(self, x: int) -> None: pass - @abstractmethod - def g(self, x: int) -> None: pass -class A(I): - def f(self, x): pass - def g(self, x, y): pass \ - # E: Signature of "g" incompatible with supertype "I" -[out] -main: In class "A": - - --- Special cases --- ------------- - - -[case testMultipleAbstractBases] -from abc import abstractmethod, ABCMeta -import typing -class A(metaclass=ABCMeta): - @abstractmethod - def f(self) -> None: pass -class B(metaclass=ABCMeta): - @abstractmethod - def g(self) -> None: pass -class C(A, B): - @abstractmethod - def h(self) -> None: pass - -[case testMemberAccessWithMultipleAbstractBaseClasses] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -class A(metaclass=ABCMeta): - @abstractmethod - def f(self) -> None: pass -class B(metaclass=ABCMeta): - @abstractmethod - def g(self) -> None: pass -class C(A, B): pass -x = Undefined # type: C -x.f() -x.g() -x.f(x) # E: Too many arguments for "f" of "A" -x.g(x) # E: Too many arguments for "g" of "B" - -[case testInstantiatingAbstractClassWithMultipleBaseClasses] -from typing import Undefined -from abc import abstractmethod, ABCMeta - -class A(metaclass=ABCMeta): - @abstractmethod - def f(self) -> None: pass -class B(metaclass=ABCMeta): - @abstractmethod - def g(self) -> None: pass -class C(A, B): - def f(self) -> None: pass -class D(A, B): - def g(self) -> None: pass -class E(A, B): - def f(self) -> None: pass - def g(self) -> None: pass -C() # E: Cannot instantiate abstract class 'C' with abstract method 'g' -D() # E: Cannot instantiate abstract class 'D' with abstract method 'f' -E() - -[case testInconsistentMro] -from abc import abstractmethod, ABCMeta -import typing - -class A(metaclass=ABCMeta): pass -class B(object, A): pass \ - # E: Cannot determine consistent method resolution order (MRO) for "B" - -[case testOverloadedAbstractMethod] -from abc import abstractmethod, ABCMeta -from typing import overload - -class A(metaclass=ABCMeta): - @abstractmethod - @overload - def f(self, x: int) -> int: pass - @abstractmethod - @overload - def f(self, x: str) -> str: pass - -class B(A): - @overload - def f(self, x: int) -> int: pass - @overload - def f(self, x: str) -> str: pass -A() # E: Cannot instantiate abstract class 'A' with abstract method 'f' -B() -B().f(1) -a = B() # type: A -a.f(1) -a.f('') -a.f(B()) # E: No overload variant of "f" of "A" matches argument types - -[case testOverloadedAbstractMethodWithAlternativeDecoratorOrder] -from abc import abstractmethod, ABCMeta -from typing import overload - -class A(metaclass=ABCMeta): - @overload - @abstractmethod - def f(self, x: int) -> int: pass - @overload - @abstractmethod - def f(self, x: str) -> str: pass - -class B(A): - @overload - def f(self, x: int) -> int: pass - @overload - def f(self, x: str) -> str: pass -A() # E: Cannot instantiate abstract class 'A' with abstract method 'f' -B() -B().f(1) -a = B() # type: A -a.f(1) -a.f('') -a.f(B()) # E: No overload variant of "f" of "A" matches argument types - -[case testOverloadedAbstractMethodVariantMissingDecorator1] -from abc import abstractmethod, ABCMeta -from typing import overload - -class A(metaclass=ABCMeta): - @abstractmethod \ - # E: Overloaded method has both abstract and non-abstract variants - @overload - def f(self, x: int) -> int: pass - @overload - def f(self, x: str) -> str: pass -[out] -main: In class "A": - -[case testOverloadedAbstractMethodVariantMissingDecorator1] -from abc import abstractmethod, ABCMeta -from typing import overload - -class A(metaclass=ABCMeta): - @overload \ - # E: Overloaded method has both abstract and non-abstract variants - def f(self, x: int) -> int: pass - @abstractmethod - @overload - def f(self, x: str) -> str: pass -[out] -main: In class "A": - -[case testMultipleInheritanceAndAbstractMethod] -import typing -from abc import abstractmethod, ABCMeta -class A: - def f(self, x: str) -> None: pass -class B(metaclass=ABCMeta): - @abstractmethod - def f(self, x: str) -> None: pass -class C(A, B): pass - -[case testMultipleInheritanceAndAbstractMethod2] -import typing -from abc import abstractmethod, ABCMeta -class A: - def f(self, x: str) -> None: pass -class B(metaclass=ABCMeta): - @abstractmethod - def f(self, x: int) -> None: pass -class C(A, B): pass -[out] -main: In class "C": -main, line 8: Definition of "f" in base class "A" is incompatible with definition in base class "B" - -[case testCallAbstractMethodBeforeDefinition] -import typing -from abc import abstractmethod, ABCMeta -class A(metaclass=ABCMeta): - def f(self) -> None: - self.g(1) # E: Argument 1 to "g" of "A" has incompatible type "int"; expected "str" - @abstractmethod - def g(self, x: str) -> None: pass -[out] -main: In member "f" of class "A": - -[case testAbstractOperatorMethods1] -import typing -from abc import abstractmethod, ABCMeta -class A(metaclass=ABCMeta): - @abstractmethod - def __lt__(self, other: 'A') -> int: pass - @abstractmethod - def __gt__(self, other: 'A') -> int: pass - -[case testAbstractOperatorMethods2] -import typing -from abc import abstractmethod, ABCMeta -class A(metaclass=ABCMeta): - @abstractmethod - def __radd__(self, other: 'C') -> str: pass # Error -class B: - @abstractmethod - def __add__(self, other: 'A') -> int: pass -class C: - def __add__(self, other: int) -> B: pass -[out] -main: In member "__radd__" of class "A": -main, line 5: Signatures of "__radd__" of "A" and "__add__" of "C" are unsafely overlapping diff --git a/mypy/test/data/check-basic.test b/mypy/test/data/check-basic.test deleted file mode 100644 index 3e2df0220caf..000000000000 --- a/mypy/test/data/check-basic.test +++ /dev/null @@ -1,332 +0,0 @@ -[case testEmptyFile] -[out] - -[case testAssignmentAndVarDef] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: B -a = a -a = b # Fail -class A: pass -class B: pass -[out] -main, line 5: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testConstructionAndAssignment] -from typing import Undefined -x = Undefined # type: A -x = A() -x = B() -class A: - def __init__(self): pass -class B: - def __init__(self): pass -[out] -main, line 4: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testInheritInitFromObject] -from typing import Undefined -x = Undefined # type: A -x = A() -x = B() -class A(object): pass -class B(object): pass -[out] -main, line 4: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testImplicitInheritInitFromObject] -from typing import Undefined -x = Undefined # type: A -o = Undefined # type: object -x = o # E: Incompatible types in assignment (expression has type "object", variable has type "A") -x = A() -o = x -class A: pass -class B: pass -[out] - -[case testTooManyConstructorArgs] -import typing -object(object()) -[out] -main, line 2: Too many arguments for "object" - -[case testVarDefWithInit] -import typing -a = A() # type: A -b = object() # type: A -class A: pass -[out] -main, line 3: Incompatible types in assignment (expression has type "object", variable has type "A") - -[case testInheritanceBasedSubtyping] -import typing -x = B() # type: A -y = A() # type: B # Fail -class A: pass -class B(A): pass -[out] -main, line 3: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testDeclaredVariableInParentheses] -from typing import Undefined -(x) = Undefined # type: int -x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -x = 1 - - --- Simple functions and calling --- ---------------------------- - - -[case testFunction] -import typing -def f(x: 'A') -> None: pass -f(A()) -f(B()) # Fail -class A: pass -class B: pass -[out] -main, line 4: Argument 1 to "f" has incompatible type "B"; expected "A" - -[case testNotCallable] -import typing -A()() -class A: pass -[out] -main, line 2: "A" not callable - -[case testSubtypeArgument] -import typing -def f(x: 'A', y: 'B') -> None: pass -f(B(), A()) # Fail -f(B(), B()) - -class A: pass -class B(A): pass -[out] -main, line 3: Argument 2 to "f" has incompatible type "A"; expected "B" - -[case testInvalidArgumentCount] -import typing -def f(x, y) -> None: pass -f(object()) -f(object(), object(), object()) -[out] -main, line 3: Too few arguments for "f" -main, line 4: Too many arguments for "f" - - --- Locals --- ------ - - -[case testLocalVariables] -from typing import Undefined -def f() -> None: - x = Undefined # type: A - y = Undefined # type: B - x = x - x = y # Fail -class A: pass -class B: pass -[out] -main: In function "f": -main, line 6: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testLocalVariableScope] -from typing import Undefined -def f() -> None: - x = Undefined # type: A - x = A() -def g() -> None: - x = Undefined # type: B - x = A() # Fail -class A: pass -class B: pass -[out] -main: In function "g": -main, line 7: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testFunctionArguments] -import typing -def f(x: 'A', y: 'B') -> None: - x = y # Fail - x = x - y = B() -class A: pass -class B: pass -[out] -main: In function "f": -main, line 3: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testLocalVariableInitialization] -import typing -def f() -> None: - a = A() # type: A - b = B() # type: A # Fail -class A: pass -class B: pass -[out] -main: In function "f": -main, line 4: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testVariableInitializationWithSubtype] -import typing -x = B() # type: A -y = A() # type: B # Fail -class A: pass -class B(A): pass -[out] -main, line 3: Incompatible types in assignment (expression has type "A", variable has type "B") - - --- Misc --- ---- - - -[case testInvalidReturn] -import typing -def f() -> 'A': - return B() -class A: pass -class B: pass -[out] -main: In function "f": -main, line 3: Incompatible return value type: expected __main__.A, got __main__.B - -[case testTopLevelContextAndInvalidReturn] -import typing -def f() -> 'A': - return B() -a = B() # type: A -class A: pass -class B: pass -[out] -main: In function "f": -main, line 3: Incompatible return value type: expected __main__.A, got __main__.B -main: At top level: -main, line 4: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testModule__name__] -import typing -x = __name__ # type: str -a = __name__ # type: A # TODO should be an error -class A: pass -[builtins fixtures/primitives.py] - -[case testModule__doc__] -import typing -x = __doc__ # type: str -a = __doc__ # type: A # TODO should be an error -class A: pass -[builtins fixtures/primitives.py] - -[case testModule__file__] -import typing -x = __file__ # type: str -a = __file__ # type: A # TODO should be an error -class A: pass -[builtins fixtures/primitives.py] - - --- Scoping and shadowing --- --------------------- - - -[case testLocalVariableShadowing] -from typing import Undefined -a = Undefined # type: A -a = B() # Fail -a = A() -def f() -> None: - a = Undefined # type: B - a = A() # Fail - a = B() -a = B() # Fail -a = A() - -class A: pass -class B: pass -[out] -main, line 3: Incompatible types in assignment (expression has type "B", variable has type "A") -main: In function "f": -main, line 7: Incompatible types in assignment (expression has type "A", variable has type "B") -main: At top level: -main, line 9: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testGlobalDefinedInBlockWithType] -from typing import Undefined -class A: pass -while A: - a = Undefined # type: A - a = A() - a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") - - --- Using Undefined with a type argument --- ------------------------------------ - - -[case testDeclareGlobalUsingUndefindWithArgument] -from typing import Undefined -a = Undefined(int) -a = 1 -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testDeclareGlobalUsingUndefindWithAnyArgument] -from typing import Undefined, Any -a = Undefined(Any) -a = 1 -a = '' -a = None - -[case testDeclareLocalUsingUndefinedWithArgument] -from typing import Undefined -def f() -> None: - a = Undefined(int) - a = 1 - a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[out] -main: In function "f": - -[case testDeclareTypeUsingUndefinedAndComment] -from typing import Undefined -x = Undefined(int) # type: str # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testDeclareMultipleVarsUsingUndefinedWithArgument] -from typing import Undefined -x = y = Undefined(int) -x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -x = 1 -y = 1 - - --- # type: signatures --- ------------------ - - -[case testFunctionSignatureAsComment] -def f(x): # type: (int) -> str - return 1 -f('') -[out] -main: In function "f": -main, line 2: Incompatible return value type: expected builtins.str, got builtins.int -main: At top level: -main, line 3: Argument 1 to "f" has incompatible type "str"; expected "int" - -[case testMethodSignatureAsComment] -class A: - def f(self, x): - # type: (int) -> str - self.f('') # Fail - return 1 -A().f('') # Fail -[out] -main: In member "f" of class "A": -main, line 4: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -main, line 5: Incompatible return value type: expected builtins.str, got builtins.int -main: At top level: -main, line 6: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" diff --git a/mypy/test/data/check-classes.test b/mypy/test/data/check-classes.test deleted file mode 100644 index ae341051ed4f..000000000000 --- a/mypy/test/data/check-classes.test +++ /dev/null @@ -1,1152 +0,0 @@ --- Methods --- ------- - - -[case testMethodCall] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: B - -a.foo(B()) # Fail -a.bar(B(), A()) # Fail - -a.foo(A()) -b.bar(B(), A()) - -class A: - def foo(self, x: 'A') -> None: pass -class B: - def bar(self, x: 'B', y: A) -> None: pass -[out] -main, line 5: Argument 1 to "foo" of "A" has incompatible type "B"; expected "A" -main, line 6: "A" has no attribute "bar" - -[case testMethodCallWithSubtype] -from typing import Undefined -a = Undefined # type: A -a.foo(A()) -a.foo(B()) -a.bar(A()) # Fail -a.bar(B()) - -class A: - def foo(self, x: 'A') -> None: pass - def bar(self, x: 'B') -> None: pass -class B(A): pass -[out] -main, line 5: Argument 1 to "bar" of "A" has incompatible type "A"; expected "B" - -[case testInheritingMethod] -from typing import Undefined -a = Undefined # type: B -a.foo(A()) # Fail -a.foo(B()) - -class A: - def foo(self, x: 'B') -> None: pass -class B(A): pass -[out] -main, line 3: Argument 1 to "foo" of "A" has incompatible type "A"; expected "B" - -[case testMethodCallWithInvalidNumberOfArguments] -from typing import Undefined -a = Undefined # type: A -a.foo() # Fail -a.foo(object(), A()) # Fail - -class A: - def foo(self, x: 'A') -> None: pass -[out] -main, line 3: Too few arguments for "foo" of "A" -main, line 4: Too many arguments for "foo" of "A" -main, line 4: Argument 1 to "foo" of "A" has incompatible type "object"; expected "A" - -[case testMethodBody] -import typing -class A: - def f(self) -> None: - a = object() # type: A # Fail -[out] -main: In member "f" of class "A": -main, line 4: Incompatible types in assignment (expression has type "object", variable has type "A") - -[case testMethodArguments] -import typing -class A: - def f(self, a: 'A', b: 'B') -> None: - a = B() # Fail - b = A() # Fail - a = A() - b = B() - a = a - a = b # Fail -class B: pass -[out] -main: In member "f" of class "A": -main, line 4: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 5: Incompatible types in assignment (expression has type "A", variable has type "B") -main, line 9: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testReturnFromMethod] -import typing -class A: - def f(self) -> 'A': - return B() # Fail - return A() -class B: pass -[out] -main: In member "f" of class "A": -main, line 4: Incompatible return value type: expected __main__.A, got __main__.B - -[case testSelfArgument] -import typing -class A: - def f(self) -> None: - o = self # type: B # Fail - self.g() # Fail - a = self # type: A - self.f() -class B: pass -[out] -main: In member "f" of class "A": -main, line 4: Incompatible types in assignment (expression has type "A", variable has type "B") -main, line 5: "A" has no attribute "g" - -[case testAssignToMethodViaInstance] -import typing -class A: - def f(self): pass -A().f = None # E: Cannot assign to a method - - --- Attributes --- ---------- - - -[case testReferToInvalidAttribute] -from typing import Undefined -class A: - def __init__(self): - self.x = object() -a = Undefined # type: A -a.y -a.y = object() -a.x -a.x = object() -[out] -main, line 6: "A" has no attribute "y" -main, line 7: "A" has no attribute "y" - -[case testArgumentTypeInference] -from typing import Undefined -class A: - def __init__(self, aa: 'A', bb: 'B') -> None: - self.a = aa - self.b = bb -class B: pass -a = Undefined # type: A -b = Undefined # type: B -a.a = b # Fail -a.b = a # Fail -b.a # Fail -a.a = a -a.b = b -[out] -main, line 9: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 10: Incompatible types in assignment (expression has type "A", variable has type "B") -main, line 11: "B" has no attribute "a" - -[case testExplicitAttributeInBody] -from typing import Undefined -a = Undefined # type: A -a.x = object() # Fail -a.x = A() -class A: - x = Undefined # type: A -[out] -main, line 3: Incompatible types in assignment (expression has type "object", variable has type "A") - -[case testAttributeDefinedInNonInitMethod] -import typing -class A: - def f(self) -> None: - self.x = 1 - self.y = '' - self.x = 1 -a = A() -a.x = 1 -a.y = '' -a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -a.z = 0 # E: "A" has no attribute "z" - -[case testInheritanceAndAttributeAssignment] -import typing -class A: - def f(self) -> None: - self.x = 0 -class B(A): - def f(self) -> None: - self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[out] -main: In member "f" of class "B": - -[case testAssignmentToAttributeInMultipleMethods] -import typing -class A: - def f(self) -> None: - self.x = 0 - def g(self) -> None: - self.x = '' # Fail - def __init__(self) -> None: - self.x = '' # Fail -[out] -main: In member "g" of class "A": -main, line 6: Incompatible types in assignment (expression has type "str", variable has type "int") -main: In member "__init__" of class "A": -main, line 8: Incompatible types in assignment (expression has type "str", variable has type "int") - - --- Method overriding --- ----------------- - - -[case testMethodOverridingWithIdenticalSignature] -import typing -class A: - def f(self, x: 'A') -> None: pass - def g(self, x: 'B' , y: object) -> 'A': pass - def h(self) -> None: pass -class B(A): - def f(self, x: A) -> None: pass - def g(self, x: 'B' , y: object) -> A: pass - def h(self) -> None: pass -[out] - -[case testMethodOverridingWithReturnTypeCoVariance] -import typing -class A: - def f(self, x: 'A', y: 'B') -> 'A': pass -class B(A): - def f(self, x: A, y: 'B') -> 'B': pass -[out] - -[case testMethodOverridingWithIncompatibleTypes] -import typing -class A: - def f(self, x: 'A', y: 'B') -> 'A': pass - def g(self, x: 'A', y: 'B') -> 'A': pass - def h(self, x: 'A', y: 'B') -> 'A': pass -class B(A): - def f(self, x: 'B', y: 'B') -> A: pass # Fail - def g(self, x: A, y: A) -> A: pass - def h(self, x: A, y: 'B') -> object: pass # Fail -[out] -main: In class "B": -main, line 7: Argument 1 of "f" incompatible with supertype "A" -main, line 9: Return type of "h" incompatible with supertype "A" - -[case testMethodOverridingWithIncompatibleArgumentCount] -import typing -class A: - def f(self, x: 'A') -> None: pass - def g(self, x: 'A', y: 'B') -> 'A': pass -class B(A): - def f(self, x: A, y: A) -> None: pass # Fail - def g(self, x: A) -> A: pass # Fail -[out] -main: In class "B": -main, line 6: Signature of "f" incompatible with supertype "A" -main, line 7: Signature of "g" incompatible with supertype "A" - -[case testMethodOverridingAcrossDeepInheritanceHierarchy1] -import typing -class A: - def f(self, x: 'B') -> None: pass -class B(A): pass -class C(B): # with gap in implementations - def f(self, x: 'C') -> None: # Fail - pass -[out] -main: In class "C": -main, line 6: Argument 1 of "f" incompatible with supertype "A" - -[case testMethodOverridingAcrossDeepInheritanceHierarchy2] -import typing -class A: - def f(self) -> 'B': pass -class B(A): - def f(self) -> 'C': pass -class C(B): # with multiple implementations - def f(self) -> B: # Fail - pass -[out] -main: In class "C": -main, line 7: Return type of "f" incompatible with supertype "B" - -[case testMethodOverridingWithVoidReturnValue] -import typing -class A: - def f(self) -> None: pass - def g(self) -> 'A': pass -class B(A): - def f(self) -> A: pass # Fail - def g(self) -> None: pass # Fail -[out] -main: In class "B": -main, line 6: Return type of "f" incompatible with supertype "A" -main, line 7: Return type of "g" incompatible with supertype "A" - - --- Constructors --- ------------ - - -[case testTrivialConstructor] -import typing -a = A() # type: A -b = A() # type: B # Fail -class A: - def __init__(self) -> None: pass -class B: pass -[out] -main, line 3: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testConstructor] -import typing -a = A(B()) # type: A -aa = A(object()) # type: A # Fail -b = A(B()) # type: B # Fail -class A: - def __init__(self, x: 'B') -> None: pass -class B: pass -[out] -main, line 3: Argument 1 to "A" has incompatible type "object"; expected "B" -main, line 4: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testConstructorWithTwoArguments] -import typing -a = A(C(), B()) # type: A # Fail - -class A: - def __init__(self, x: 'B', y: 'C') -> None: pass -class B: pass -class C(B): pass -[out] -main, line 2: Argument 2 to "A" has incompatible type "B"; expected "C" - -[case testInheritedConstructor] -import typing -b = B(C()) # type: B -a = B(D()) # type: A # Fail -class A: - def __init__(self, x: 'C') -> None: pass -class B(A): pass -class C: pass -class D: pass -[out] -main, line 3: Argument 1 to "B" has incompatible type "D"; expected "C" - -[case testOverridingWithIncompatibleConstructor] -import typing -A() # Fail -B(C()) # Fail -A(C()) -B() -class A: - def __init__(self, x: 'C') -> None: pass -class B(A): - def __init__(self) -> None: pass -class C: pass -[out] -main, line 2: Too few arguments for "A" -main, line 3: Too many arguments for "B" - -[case testConstructorWithReturnValueType] -import typing -class A: - def __init__(self) -> 'A': pass -[out] -main: In member "__init__" of class "A": -main, line 3: Cannot define return type for "__init__" - -[case testGlobalFunctionInitWithReturnType] -import typing -a = __init__() # type: A -b = __init__() # type: B # Fail -def __init__() -> 'A': pass -class A: pass -class B: pass -[out] -main, line 3: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testAccessingInit] -from typing import Undefined, Any -class A: - def __init__(self, a: 'A') -> None: pass -a = Undefined # type: A -a.__init__(a) # E: Cannot access "__init__" directly -(Any(a)).__init__(a) - -[case testDeepInheritanceHierarchy] -import typing -d = C() # type: D # Fail -d = B() # Fail -d = A() # Fail -d = D2() # Fail -a = D() # type: A -a = D2() -b = D() # type: B -b = D2() - -class A: pass -class B(A): pass -class C(B): pass -class D(C): pass -class D2(C): pass -[out] -main, line 2: Incompatible types in assignment (expression has type "C", variable has type "D") -main, line 3: Incompatible types in assignment (expression has type "B", variable has type "D") -main, line 4: Incompatible types in assignment (expression has type "A", variable has type "D") -main, line 5: Incompatible types in assignment (expression has type "D2", variable has type "D") - - --- Attribute access in class body --- ------------------------------ - - -[case testDataAttributeRefInClassBody] -import typing -class B: pass -class A: - x = B() - y = x - b = x # type: B - b = x - c = x # type: A # E: Incompatible types in assignment (expression has type "B", variable has type "A") - c = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[out] -main: In class "A": - -[case testMethodRefInClassBody] -from typing import Function -class B: pass -class A: - def f(self) -> None: pass - g = f - h = f # type: Function[[A], None] - h = f - g = h - ff = f # type: Function[[B], None] # E: Incompatible types in assignment (expression has type Function[[A], None], variable has type Function[[B], None]) - g = ff # E: Incompatible types in assignment (expression has type Function[[B], None], variable has type Function[[A], None]) -[out] -main: In class "A": - - --- Arbitrary statements in class body --- ---------------------------------- - - -[case testStatementsInClassBody] -import typing -class B: pass -class A: - for x in [A()]: - y = x - y = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - x = A() - y = A() - x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[builtins fixtures/for.py] -[out] -main: In class "A": - - --- Class attributes --- ---------------- - - -[case testAccessMethodViaClass] -import typing -class A: - def f(self) -> None: pass -A.f(A()) -A.f(object()) # E: Argument 1 to "f" of "A" has incompatible type "object"; expected "A" -A.f() # E: Too few arguments for "f" of "A" -A.f(None, None) # E: Too many arguments for "f" of "A" - -[case testAccessAttributeViaClass] -import typing -class B: pass -class A: - x = None # type: A -a = A.x # type: A -b = A.x # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testAccessingUndefinedAttributeViaClass] -import typing -class A: pass -A.x # E: "A" has no attribute "x" - -[case testAccessingUndefinedAttributeViaClassWithOverloadedInit] -from typing import overload -class A: - @overload - def __init__(self): pass - @overload - def __init__(self, x): pass -A.x # E: "A" has no attribute "x" - -[case testAccessMethodOfClassWithOverloadedInit] -from typing import overload, Any -class A: - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, x: Any) -> None: pass - def f(self) -> None: pass -A.f(A()) -A.f() # E: Too few arguments for "f" of "A" - -[case testAssignmentToClassDataAttribute] -import typing -class B: pass -class A: - x = None # type: B -A.x = B() -A.x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "B") - -[case testAssignmentToInferredClassDataAttribute] -import typing -class B: pass -class A: - x = B() -A.x = B() -A.x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testInitMethodUnbound] -from typing import Undefined -class B: pass -class A: - def __init__(self, b: B) -> None: pass -a = Undefined # type: A -b = Undefined # type: B -A.__init__(a, b) -A.__init__(b, b) # E: Argument 1 to "__init__" of "A" has incompatible type "B"; expected "A" -A.__init__(a, a) # E: Argument 2 to "__init__" of "A" has incompatible type "A"; expected "B" - -[case testAssignToMethodViaClass] -import typing -class A: - def f(self): pass -A.f = None # E: Cannot assign to a method - -[case testAssignToNestedClassViaClass] -import typing -class A: - class B: pass -A.B = None # E: Cannot assign to a type - - --- Nested classes --- -------------- - - -[case testClassWithinFunction] -from typing import Undefined -def f() -> None: - class A: - def g(self) -> None: pass - a = Undefined # type: A - a.g() - a.g(a) # E: Too many arguments for "g" of "A" -[out] -main: In function "f": - -[case testConstructNestedClass] -import typing -class A: - class B: pass - b = B() - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - b = B(b) # E: Too many arguments for "B" -[out] -main: In class "A": - -[case testConstructNestedClassWithCustomInit] -import typing -class A: - def f(self) -> None: - class B: - def __init__(self, a: 'A') -> None: pass - b = B(A()) - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - b = B() # E: Too few arguments for "B" -[out] -main: In member "f" of class "A": - -[case testDeclareVariableWithNestedClassType] -from typing import Undefined -def f() -> None: - class A: pass - a = Undefined # type: A - a = A() - a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -[out] -main: In function "f": - -[case testExternalReferenceToClassWithinClass] -from typing import Undefined -class A: - class B: pass -b = Undefined # type: A.B -b = A.B() -b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b = A.B(b) # E: Too many arguments for "B" - - --- Declaring attribute type in method --- ---------------------------------- - - -[case testDeclareAttributeTypeInInit] -from typing import Undefined -class A: - def __init__(self): - self.x = None # type: int -a = Undefined # type: A -a.x = 1 -a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testAccessAttributeDeclaredInInitBeforeDeclaration] -from typing import Undefined -a = Undefined # type: A -a.x = 1 -a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -class A: - def __init__(self): - self.x = None # type: int - - --- Special cases --- ------------- - - -[case testMultipleClassDefinition] -import typing -A() -class A: pass -class A: pass -[out] -main, line 4: Name 'A' already defined - -[case testDocstringInClass] -import typing -class A: - """Foo""" -class B: - 'x' - y = B() -[builtins fixtures/primitives.py] - -[case testErrorMessageInFunctionNestedWithinMethod] -import typing -class A: - def f(self) -> None: - def g() -> None: - a = None - b = None -[out] -main: In function "g": -main, line 5: Need type annotation for variable -main: In member "f" of class "A": -main, line 6: Need type annotation for variable - - --- Using Undefined(type) to declare attributes --- ------------------------------------------- - - -[case testDeclareClassVariableUsingUndefinedWithArgument] -from typing import Undefined -class A: - x = Undefined(int) -a = Undefined(A) -a.x = 1 -a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testDeclareInstanceVariableUsingUndefinedWithArgument] -from typing import Undefined -class A: - def __init__(self): - self.x = Undefined(int) -a = Undefined(A) -a.x = 1 -a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testAccessInstanceVariableDeclaredUsingUndefinedWithArgumentBeforeDecl] -from typing import Undefined -class A: - def f(self) -> None: - self.x = 1 - self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - def __init__(self): - self.x = Undefined(int) -[out] -main: In member "f" of class "A": - - --- Static methods --- -------------- - - -[case testSimpleStaticMethod] -import typing -class A: - @staticmethod - def f(x: int) -> None: pass -A.f(1) -A().f(1) -A.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -[builtins fixtures/staticmethod.py] - -[case testBuiltinStaticMethod] -import typing -int.from_bytes(b'', '') -int.from_bytes('', '') # E: Argument 1 to "from_bytes" of "int" has incompatible type "str"; expected "bytes" -[builtins fixtures/staticmethod.py] - -[case testAssignStaticMethodOnInstance] -import typing -class A: - @staticmethod - def f(x: int) -> None: pass -A().f = A.f # E: Cannot assign to a method -[builtins fixtures/staticmethod.py] - - --- Class methods --- ------------- - - -[case testSimpleClassMethod] -import typing -class A: - @classmethod - def f(cls, x: int) -> None: pass -A.f(1) -A().f(1) -A.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -[builtins fixtures/classmethod.py] - -[case testBuiltinClassMethod] -import typing -int.from_bytes(b'', '') -int.from_bytes('', '') # E: Argument 1 to "from_bytes" of "int" has incompatible type "str"; expected "bytes" -[builtins fixtures/classmethod.py] - -[case testAssignClassMethodOnClass] -import typing -class A: - @classmethod - def f(cls, x: int) -> None: pass -A.f = A.f # E: Cannot assign to a method -[builtins fixtures/classmethod.py] - -[case testAssignClassMethodOnInstance] -import typing -class A: - @classmethod - def f(cls, x: int) -> None: pass -A().f = A.f # E: Cannot assign to a method -[builtins fixtures/classmethod.py] - - --- Properties --- ---------- - - -[case testAccessingReadOnlyProperty] -import typing -class A: - @property - def f(self) -> str: pass -a = A() # type. A -s = '' # type: str -s = a.f -a = a.f # E: Incompatible types in assignment (expression has type "str", variable has type "A") -[builtins fixtures/property.py] - -[case testAssigningToReadOnlyProperty] -import typing -class A: - @property - def f(self) -> str: pass -A().f = '' # E: Property "f" defined in "A" is read-only -[builtins fixtures/property.py] - -[case testPropertyGetterBody] -import typing -class A: - @property - def f(self) -> str: - self.x = 1 - self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/property.py] -[out] -main: In member "f" of class "A": - -[case testDynamicallyTypedProperty] -import typing -class A: - @property - def f(self): pass -a = A() # type. A -a.f.xx -a.f = '' # E: Property "f" defined in "A" is read-only -[builtins fixtures/property.py] - - --- Multiple inheritance, non-object built-in class as base --- ------------------------------------------------------- - - -[case testInvalidMultipleInheritanceFromBuiltins] -import typing -class A(int, str): pass # E: Instance layout conflict in multiple inheritance -[out] -main: In class "A": - -[case testInvalidMultipleInheritanceFromBuiltins] -import typing -class S(str): pass -class A(S, str): pass -class B(S, int): pass # E: Instance layout conflict in multiple inheritance -[out] -main: In class "B": - - --- Ducktype decorators --- ------------------- - - -[case testSimpleDucktypeDecorator] -from typing import ducktype, Undefined -class A: pass -@ducktype(A) -class B: pass -a = Undefined(A) -b = Undefined(B) -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = b - -[case testDucktypeTransitivityDecorator] -from typing import ducktype, Undefined -class A: pass -@ducktype(A) -class B: pass -@ducktype(B) -class C: pass -a = Undefined(A) -c = Undefined(C) -c = a # E: Incompatible types in assignment (expression has type "A", variable has type "C") -a = c - - --- Disjointclass decorators --- ------------------------ - - -[case testDisjointclassDecorator] -from typing import disjointclass -class A: pass -@disjointclass(A) -class B: pass -class C(A, B): pass # Error -class D(B, A): pass # Error -b = B() -[out] -main: In class "C": -main, line 5: disjointclass constraint of class B disallows A as a base class -main: In class "D": -main, line 6: disjointclass constraint of class B disallows A as a base class - - --- Operator methods --- ---------------- - - -[case testOperatorMethodOverrideIntroducingOverloading] -from typing import overload -class A: - def __add__(self, x: int) -> int: pass -class B(A): - @overload # E: Signature of "__add__" incompatible with supertype "A" - def __add__(self, x: int) -> int: pass - @overload - def __add__(self, x: str) -> str: pass -[out] -main: In class "B": - -[case testOperatorMethodOverrideWideningArgumentType] -import typing -class A: - def __add__(self, x: int) -> int: pass -class B(A): - def __add__(self, x: object) -> int: pass -[out] - -[case testOperatorMethodOverrideNarrowingReturnType] -import typing -class A: - def __add__(self, x: int) -> 'A': pass -class B(A): - def __add__(self, x: int) -> 'B': pass - -[case testOperatorMethodOverrideWithDynamicallyTyped] -import typing -class A: - def __add__(self, x: int) -> 'A': pass -class B(A): - def __add__(self, x): pass - -[case testOperatorMethodOverrideWithIdenticalOverloadedType] -from typing import overload -class A: - @overload - def __add__(self, x: int) -> 'A': pass - @overload - def __add__(self, x: str) -> 'A': pass -class B(A): - @overload - def __add__(self, x: int) -> 'A': pass - @overload - def __add__(self, x: str) -> 'A': pass - -[case testOverloadedOperatorMethodOverrideWithDynamicallyTypedMethod] -from typing import overload, Any -class A: - @overload - def __add__(self, x: int) -> 'A': pass - @overload - def __add__(self, x: str) -> 'A': pass -class B(A): - def __add__(self, x): pass -class C(A): - def __add__(self, x: Any) -> A: pass - -[case testOverloadedOperatorMethodOverrideWithNewItem] -from typing import overload, Any -class A: - @overload - def __add__(self, x: int) -> 'A': pass - @overload - def __add__(self, x: str) -> 'A': pass -class B(A): - @overload - def __add__(self, x: int) -> A: pass - @overload - def __add__(self, x: str) -> A: pass - @overload - def __add__(self, x: type) -> A: pass -[out] -main: In class "B": -main, line 8: Signature of "__add__" incompatible with supertype "A" - -[case testOverloadedOperatorMethodOverrideWithSwitchedItemOrder] -from typing import overload, Any -class A: - @overload - def __add__(self, x: 'B') -> 'B': pass - @overload - def __add__(self, x: 'A') -> 'A': pass -class B(A): - @overload - def __add__(self, x: 'A') -> 'A': pass - @overload - def __add__(self, x: 'B') -> 'B': pass -[out] -main: In class "B": -main, line 8: Signature of "__add__" incompatible with supertype "A" - -[case testReverseOperatorMethoddArgumentType] -from typing import Any -class A: pass -class B: - def __radd__(self, x: A) -> int: pass # Error -class C: - def __radd__(self, x: A) -> Any: pass -class D: - def __radd__(self, x: A) -> object: pass -[out] -main: In member "__radd__" of class "B": -main, line 4: "Any" return type expected since argument to __radd__ does not support __add__ - -[case testReverseOperatorMethoddArgumentType2] -from typing import Any, Tuple, Function -class A: - def __radd__(self, x: Tuple[int, str]) -> int: pass -class B: - def __radd__(self, x: Function[[], int]) -> int: pass -class C: - def __radd__(self, x: Any) -> int: pass -[out] -main: In member "__radd__" of class "A": -main, line 3: "Any" return type expected since argument to __radd__ does not support __add__ -main: In member "__radd__" of class "B": -main, line 5: "Any" return type expected since argument to __radd__ does not support __add__ -main: In member "__radd__" of class "C": -main, line 7: "Any" return type expected since argument to __radd__ has type "Any" - -[case testOverloadedReverseOperatorMethodArgumentType] -from typing import overload, Any -class A: - @overload - def __radd__(self, x: 'A') -> str: pass # Error - @overload - def __radd__(self, x: 'A') -> Any: pass -[out] -main: In member "__radd__" of class "A": -main, line 4: "Any" return type expected since argument to __radd__ does not support __add__ - -[case testReverseOperatorMethodArgumentTypeAndOverloadedMethod] -from typing import overload, disjointclass -@disjointclass(int) -@disjointclass(str) -class A: - @overload - def __add__(self, x: int) -> int: pass - @overload - def __add__(self, x: str) -> int: pass - def __radd__(self, x: 'A') -> str: pass - -[case testAbstractReverseOperatorMethod] -import typing -from abc import abstractmethod -class A: - @abstractmethod - def __lt__(self, x: 'A') -> int: pass -class B: - @abstractmethod - def __lt__(self, x: 'B') -> int: pass - @abstractmethod - def __gt__(self, x: 'B') -> int: pass -[out] -main: In member "__lt__" of class "A": -main, line 5: "Any" return type expected since argument to __lt__ does not support __gt__ - -[case testUnsafeOverlappingForwardAndReverseMethods] -import typing -class A: - def __add__(self, x: int) -> str: pass -class B: - def __radd__(self, x: A) -> int: pass # Error -[out] -main: In member "__radd__" of class "B": -main, line 5: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping - -[case testUnsafeOverlappingWithOperatorMethodsAndOverloading] -from typing import overload -class A: - @overload - def __add__(self, x: 'A') -> int: pass - @overload - def __add__(self, x: str) -> int: pass -class B: - def __radd__(self, x: 'A') -> str: pass # Error -[out] -main: In member "__radd__" of class "B": -main, line 8: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping - -[case testUnsafeOverlappingWithOperatorMethodsAndOverloading2] -from typing import overload -class A: - def __add__(self, x: 'A') -> int: pass -class B: - @overload - def __radd__(self, x: 'X') -> str: pass # Error - @overload - def __radd__(self, x: A) -> str: pass # Error -class X: - def __add__(self, x): pass -[out] -main: In member "__radd__" of class "B": -main, line 6: Signatures of "__radd__" of "B" and "__add__" of "X" are unsafely overlapping -main, line 8: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping - -[case testOverlappingNormalAndInplaceOperatorMethod] -import typing -class A: - # Incompatible (potential trouble with __radd__) - def __add__(self, x: 'A') -> int: pass - def __iadd__(self, x: 'B') -> int: pass -class B: - # Safe - def __add__(self, x: 'C') -> int: pass - def __iadd__(self, x: A) -> int: pass -class C(A): pass -[out] -main: In class "A": -main, line 5: Signatures of "__iadd__" and "__add__" are incompatible - -[case testOverloadedNormalAndInplaceOperatorMethod] -from typing import overload -class A: - @overload - def __add__(self, x: int) -> int: pass - @overload - def __add__(self, x: str) -> int: pass - @overload # Error - def __iadd__(self, x: int) -> int: pass - @overload - def __iadd__(self, x: object) -> int: pass -class B: - @overload - def __add__(self, x: int) -> int: pass - @overload - def __add__(self, x: str) -> str: pass - @overload - def __iadd__(self, x: int) -> int: pass - @overload - def __iadd__(self, x: str) -> str: pass -[out] -main: In class "A": -main, line 7: Signatures of "__iadd__" and "__add__" are incompatible - -[case testIntroducingInplaceOperatorInSubclass] -import typing -class A: - def __add__(self, x: 'A') -> 'B': pass -class B(A): - # __iadd__ effectively partially overrides __add__ - def __iadd__(self, x: 'A') -> 'A': pass # Error -class C(A): - def __iadd__(self, x: int) -> 'B': pass # Error -class D(A): - def __iadd__(self, x: 'A') -> 'B': pass -[out] -main: In class "B": -main, line 6: Return type of "__iadd__" incompatible with "__add__" of supertype "A" -main: In class "C": -main, line 8: Argument 1 of "__iadd__" incompatible with "__add__" of supertype "A" -main, line 8: Signatures of "__iadd__" and "__add__" are incompatible - - --- Special cases --- ------------- - - -[case testSubclassInt] -import typing -class A(int): pass -n = 0 -n = A() -a = A() -a = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "A") - - --- TODO --- attribute inherited from superclass; assign in __init__ --- refer to attribute before type has been inferred (the initialization in --- __init__ has not been analysed) diff --git a/mypy/test/data/check-dynamic-typing.test b/mypy/test/data/check-dynamic-typing.test deleted file mode 100644 index e6531adfec4b..000000000000 --- a/mypy/test/data/check-dynamic-typing.test +++ /dev/null @@ -1,647 +0,0 @@ --- Assignment --- ---------- - - -[case testAssignmentWithDynamic] -from typing import Undefined, Any -d = Undefined # type: Any -a = Undefined # type: A - -a = d # Everything ok -d = a -d = d -d.x = a -d.x = d - -class A: pass - -[case testMultipleAssignmentWithDynamic] -from typing import Undefined, Any -d = Undefined # type: Any -a, b = Undefined, Undefined # type: (A, B) - -d, a = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") -d, d = d, d, d # E: Too many values to unpack (2 expected, 3 provided) - -a, b = d, d -d, d = a, b -a, b = d -s, t = d - -class A: pass -class B: pass - - --- Expressions --- ----------- - - -[case testCallingFunctionWithDynamicArgumentTypes] -from typing import Undefined, Any -a, b = Undefined, Undefined # type: (A, B) - -b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = f(a) -a = f(b) -a = f(None) -a = f(f) - -def f(x: Any) -> 'A': - pass - -class A: pass -class B: pass - -[case testCallingWithDynamicReturnType] -from typing import Undefined, Any -a, b = Undefined, Undefined # type: (A, B) - -a = f(b) # E: Argument 1 to "f" has incompatible type "B"; expected "A" - -a = f(a) -b = f(a) - -def f(x: 'A') -> Any: - pass - -class A: pass -class B: pass - -[case testBinaryOperationsWithDynamicLeftOperand] -from typing import Undefined, Any -d = Undefined # type: Any -a = Undefined # type: A -c = Undefined # type: C -b = Undefined # type: bool -n = 0 - -d in a # E: Unsupported right operand type for in ("A") -d and a -d or a -c = d and b # Unintuitive type inference? -c = d or b # Unintuitive type inference? - -c = d + a -c = d - a -c = d * a -c = d / a -c = d // a -c = d % a -c = d ** a -b = d == a -b = d != a -b = d < a -b = d <= a -b = d > a -b = d >= a -b = d in c -b = d and b -b = d or b - -class A: pass -class C: - def __contains__(self, a: A) -> bool: - pass -[file builtins.py] -class object: - def __init__(self): pass -class bool: pass -class int: pass -class type: pass -class function: pass - -[case testBinaryOperationsWithDynamicAsRightOperand] -from typing import Undefined, Any -d = Undefined # type: Any -a = Undefined # type: A -c = Undefined # type: C -b = Undefined # type: bool -n = 0 - -a and d -a or d -c = a in d -c = b and d # Unintuitive type inference? -c = b or d # Unintuitive type inference? -b = a + d -b = a / d - -c = a + d -c = a - d -c = a * d -c = a / d -c = a // d -c = a % d -c = a ** d -b = a in d -b = b and d -b = b or d - -class A: - def __add__(self, a: 'A') -> 'C': - pass - def __sub__(self, a: 'A') -> 'C': - pass - def __mul__(self, a: 'A') -> 'C': - pass - def __truediv__(self, a: 'A') -> 'C': - pass - def __floordiv__(self, a: 'A') -> 'C': - pass - def __mod__(self, a: 'A') -> 'C': - pass - def __pow__(self, a: 'A') -> 'C': - pass - def _lt(self, a: 'A') -> bool: - pass - def _gt(self, a: 'A') -> bool: - pass - -class C: pass -[file builtins.py] -class object: - def __init__(self): pass -class bool: pass -class int: pass -class type: pass -class function: pass - -[case testDynamicWithUnaryExpressions] -from typing import Undefined, Any -d = Undefined # type: Any -a = Undefined # type: A -b = Undefined # type: bool -a = not d # E: Incompatible types in assignment (expression has type "bool", variable has type "A") -b = not d -a = -d -class A: pass -[builtins fixtures/bool.py] -[out] - -[case testDynamicWithMemberAccess] -from typing import Undefined, Any -d = Undefined # type: Any -a = Undefined # type: A - -a = d.foo(a()) # E: "A" not callable - -a = d.x -a = d.foo(a, a) -d.x = a -d.x.y.z - -class A: pass -[out] - -[case testIndexingWithDynamic] -from typing import Undefined, Any -d = Undefined # type: Any -a = Undefined # type: A - -a = d[a()] # E: "A" not callable -d[a()] = a # E: "A" not callable - -a = d[a] -d[a] = a -d[a], d[a] = a, a - -class A: pass - -[case testTupleExpressionsWithDynamci] -from typing import Undefined, Tuple, Any -t2 = Undefined # type: Tuple[A, A] -d = Undefined # type: Any - -t2 = (d, d, d) # E: Incompatible types in assignment (expression has type "Tuple[Any, Any, Any]", variable has type "Tuple[A, A]") -t2 = (d, d) - -class A: pass -[builtins fixtures/tuple.py] - -[case testCastsWithDynamicType] -from typing import Undefined, Any, cast -class A: pass -class B: pass -d = Undefined # type: Any -a = Undefined # type: A -b = Undefined # type: B -b = cast(A, d) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = cast(A, d) -b = Any(d) -a = Any(f()) -def f() -> None: pass - -[case testCompatibilityOfDynamicWithOtherTypes] -from typing import Undefined, Any, Tuple -d = Undefined # type: Any -t = Undefined # type: Tuple[A, A] -# TODO: callable types, overloaded functions - -d = None # All ok -d = t -d = g -d = A -t = d -f = d - -def g(a: 'A') -> None: - pass - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - - --- Statements --- ---------- - - -[case testDynamicCondition] -from typing import Undefined, Any -d = Undefined # type: Any -while d: - pass -if d: - pass -elif d: - pass -[builtins fixtures/bool.py] - -[case testRaiseWithDynamic] -from typing import Undefined, Any -d = Undefined # type: Any -raise d -[builtins fixtures/exception.py] - -[case testReturnWithDynamic] -from typing import Undefined, Any -d = Undefined # type: Any - -def f() -> None: - return d # Ok - -def g() -> 'A': - return d # Ok - -class A: pass - - --- Implicit dynamic types for functions --- ------------------------------------ - - -[case testImplicitGlobalFunctionSignature] -from typing import Undefined, Any, Function -x = Undefined # type: Any -a = Undefined # type: A -g = Undefined # type: Function[[], None] -h = Undefined # type: Function[[A], None] - -f() # E: Too few arguments for "f" -f(x, x) # E: Too many arguments for "f" -g = f # E: Incompatible types in assignment (expression has type Function[[Any], Any], variable has type Function[[], None]) -f(a) -f(x) -a = f(a) -h = f - -def f(x): pass - -class A: pass - -[case testImplicitGlobalFunctionSignatureWithDifferentArgCounts] -from typing import Undefined, Function -g0 = Undefined # type: Function[[], None] -g1 = Undefined # type: Function[[A], None] -g2 = Undefined # type: Function[[A, A], None] -a = Undefined # type: A - -g1 = f0 # E: Incompatible types in assignment (expression has type Function[[], Any], variable has type Function[[A], None]) -g2 = f0 # E: Incompatible types in assignment (expression has type Function[[], Any], variable has type Function[[A, A], None]) -g0 = f2 # E: Incompatible types in assignment (expression has type Function[[Any, Any], Any], variable has type Function[[], None]) -g1 = f2 # E: Incompatible types in assignment (expression has type Function[[Any, Any], Any], variable has type Function[[A], None]) - -g0 = g0 -g2 = f2 -f0() -f2(a, a) - -def f0(): pass - -def f2(x, y): pass - -class A: pass - -[case testImplicitGlobalFunctionSignatureWithDefaultArgs] -from typing import Undefined, Function -a, b = Undefined, Undefined # type: (A, B) - -g0 = Undefined # type: Function[[], None] -g1 = Undefined # type: Function[[A], None] -g2 = Undefined # type: Function[[A, A], None] -g3 = Undefined # type: Function[[A, A, A], None] -g4 = Undefined # type: Function[[A, A, A, A], None] - -f01(a, a) # Fail -f13() # Fail -f13(a, a, a, a) # Fail -g2 = f01 # Fail -g0 = f13 # Fail -g4 = f13 # Fail - -f01() -f01(a) -f13(a) -f13(a, a) -f13(a, a, a) - -g0 = f01 -g1 = f01 -g1 = f13 -g2 = f13 -g3 = f13 - -def f01(x = b): pass -def f13(x, y = b, z = b): pass - -class A: pass -class B: pass -[out] -main, line 10: Too many arguments for "f01" -main, line 11: Too few arguments for "f13" -main, line 12: Too many arguments for "f13" -main, line 13: Incompatible types in assignment (expression has type Function[[Any], Any], variable has type Function[[A, A], None]) -main, line 14: Incompatible types in assignment (expression has type Function[[Any, Any, Any], Any], variable has type Function[[], None]) -main, line 15: Incompatible types in assignment (expression has type Function[[Any, Any, Any], Any], variable has type Function[[A, A, A, A], None]) - -[case testSkipTypeCheckingWithImplicitSignature] -from typing import Undefined -a = Undefined # type: A -def f(): - a() -def g(x): - a() - a.x - a + a - if a(): - a() -class A: pass -[builtins fixtures/bool.py] - -[case testSkipTypeCheckingWithImplicitSignatureAndDefaultArgs] -from typing import Undefined -a = Undefined # type: A -def f(x=a()): - a() -def g(x, y=a, z=a()): - a() -class A: pass - -[case testImplicitMethodSignature] -from typing import Undefined, Function -g0 = Undefined # type: Function[[], None] -g1 = Undefined # type: Function[[A], None] -g2 = Undefined # type: Function[[A, A], None] -a = Undefined # type: A - -g0 = a.f # E: Incompatible types in assignment (expression has type Function[[Any], Any], variable has type Function[[], None]) -g2 = a.f # E: Incompatible types in assignment (expression has type Function[[Any], Any], variable has type Function[[A, A], None]) -a = a.f # E: Incompatible types in assignment (expression has type Function[[Any], Any], variable has type "A") - -class A: - def g(self) -> None: - a = self.f(a) - def f(self, x): pass - -g1 = a.f -a = a.f(a) - -[case testSkipTypeCheckingImplicitMethod] -from typing import Undefined -a = Undefined # type: A -class A: - def f(self): - a() - def g(self, x, y=a()): - a() - -[case testImplicitInheritedMethod] -from typing import Undefined, Function -g0 = Undefined # type: Function[[], None] -g1 = Undefined # type: Function[[A], None] -a = Undefined # type: A - -g0 = a.f # E: Incompatible types in assignment (expression has type Function[[Any], Any], variable has type Function[[], None]) - -g1 = a.f -a = a.f(a) - -class B: - def f(self, x): - pass -class A(B): - def g(self) -> None: - a = self.f(a) - -[case testEmptyReturnWithImplicitSignature] -import typing -def f(): - return -class A: - def g(self): - return - -[case testVarArgsWithImplicitSignature] -from typing import Undefined, Any -o = Undefined # type: Any -def f(x, *a): pass -f() # E: Too few arguments for "f" -f(o) -f(o, o) -f(o, o, o) -[builtins fixtures/list.py] - - --- Implicit types for constructors --- ------------------------------- - - -[case testInitMethodWithImplicitSignature] -from typing import Undefined, Function -f1 = Undefined # type: Function[[A], A] -f2 = Undefined # type: Function[[A, A], A] -a = Undefined # type: A - -A(a) # Fail -f1 = A # Fail - -A(a, a) -f2 = A - -class A: - def __init__(self, a, b): pass -[out] -main, line 6: Too few arguments for "A" -main, line 7: Incompatible types in assignment (expression has type "A" (type object), variable has type Function[[A], A]) - -[case testUsingImplicitTypeObjectWithIs] -from typing import Undefined -t = Undefined # type: type -t = A -t = B - -class A: pass -class B: - def __init__(self): pass - - --- Type compatibility --- ------------------ - - -[case testTupleTypeCompatibility] -from typing import Undefined, Any, Tuple -t1 = Undefined # type: Tuple[Any, A] -t2 = Undefined # type: Tuple[A, Any] -t3 = Undefined # type: Tuple[Any, Any] -t4 = Undefined # type: Tuple[A, A] -t5 = Undefined # type: Tuple[Any, Any, Any] - -t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[Any, Any, Any]", variable has type "Tuple[Any, Any]") -t5 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[Any, Any, Any]") - -t1 = t1 -t1 = t2 -t1 = t3 -t1 = t4 -t2 = t1 -t2 = t3 -t2 = t4 -t3 = t1 -t3 = t2 -t3 = t4 -t4 = t1 -t4 = t2 -t4 = t3 - -class A: pass -[builtins fixtures/tuple.py] - -[case testFunctionTypeCompatibilityAndReturnTypes] -from typing import Undefined, Any, Function -f1 = Undefined # type: Function[[], Any] -f11 = Undefined # type: Function[[], Any] -f2 = Undefined # type: Function[[], A] -f3 = Undefined # type: Function[[], None] - -f2 = f3 # E: Incompatible types in assignment (expression has type Function[[], None], variable has type Function[[], A]) - -f1 = f2 -f1 = f3 -f2 = f11 -f3 = f11 - -class A: pass - -[case testFunctionTypeCompatibilityAndArgumentTypes] -from typing import Undefined, Any, Function -f1 = Undefined # type: Function[[A, Any], None] -f2 = Undefined # type: Function[[Any, A], None] -f3 = Undefined # type: Function[[A, A], None] - -f1 = f1 -f1 = f2 -f1 = f3 - -f2 = f1 -f2 = f2 -f2 = f3 - -f3 = f1 -f3 = f2 -f3 = f3 - -class A: pass - -[case testFunctionTypeCompatibilityAndArgumentCounts] -from typing import Undefined, Any, Function -f1 = Undefined # type: Function[[Any], None] -f2 = Undefined # type: Function[[Any, Any], None] - -f1 = f2 # E: Incompatible types in assignment (expression has type Function[[Any, Any], None], variable has type Function[[Any], None]) - - --- Overriding --- ---------- - - -[case testOverridingMethodWithDynamicTypes] -from typing import Undefined, Any -a, b = Undefined, Undefined # type: (A, B) - -b.f(b) # E: Argument 1 to "f" of "B" has incompatible type "B"; expected "A" -a = a.f(b) - -class B: - def f(self, x: 'A') -> 'B': - pass - def g(self, x: 'B') -> None: - pass -class A(B): - def f(self, x: Any) -> Any: - pass - def g(self, x: Any) -> None: - pass - -[case testOverridingMethodWithImplicitDynamicTypes] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) - -b.f(b) # E: Argument 1 to "f" of "B" has incompatible type "B"; expected "A" -a = a.f(b) - -class B: - def f(self, x: 'A') -> 'B': - pass - def g(self, x: 'B') -> None: - pass -class A(B): - def f(self, x): - pass - def g(self, x): - pass - -[case testOverridingMethodAcrossHierarchy] -import typing -class C: - def f(self, a: 'A') -> None: pass -class B(C): - def f(self, a): pass -class A(B): - def f(self, a: 'D') -> None: # E: Argument 1 of "f" incompatible with supertype "C" - pass -class D: pass -[out] -main: In class "A": - -[case testInvalidOverrideArgumentCountWithImplicitSignature1] -import typing -class B: - def f(self, x: A) -> None: pass -class A(B): - def f(self, x, y): # Fail - x() -[out] -main: In class "A": -main, line 5: Signature of "f" incompatible with supertype "B" - -[case testInvalidOverrideArgumentCountWithImplicitSignature2] -import typing -class B: - def f(self, x, y): pass -class A(B): - def f(self, x: 'A') -> None: # E: Signature of "f" incompatible with supertype "B" - pass -[out] -main: In class "A": diff --git a/mypy/test/data/check-expressions.test b/mypy/test/data/check-expressions.test deleted file mode 100644 index f37bdc9e7a65..000000000000 --- a/mypy/test/data/check-expressions.test +++ /dev/null @@ -1,1278 +0,0 @@ --- Test cases for simple expressions. --- --- See also: --- * check-functions.test contains test cases for calls. --- * check-varargs.test contains test cases for *args. --- * check-dynamic.test contains test cases related to 'any' type. --- * check-generics.test contains test cases for generic values. - - --- None --- ---- - - -[case testNoneAsRvalue] -import typing -a = None # type: A -class A: pass -[out] - -[case testNoneAsArgument] -import typing -def f(x: 'A', y: 'B') -> None: pass -f(None, None) -class A: pass -class B(A): pass -[out] - - --- Simple expressions --- ------------------ - - -[case testIntLiteral] -from typing import Undefined -a = 0 -b = Undefined # type: A -b = 1 -a = 1 -class A: - pass -[out] -main, line 4: Incompatible types in assignment (expression has type "int", variable has type "A") - -[case testStrLiteral] -from typing import Undefined -a = '' -b = Undefined # type: A -b = 'x' -a = 'x' -a = r"x" -a = """foo""" -class A: - pass -[out] -main, line 4: Incompatible types in assignment (expression has type "str", variable has type "A") - -[case testFloatLiteral] -from typing import Undefined -a = 0.0 -b = Undefined # type: A -b = 1.1 -a = 1.1 -class A: - pass -[file builtins.py] -class object: - def __init__(self): pass -class type: pass -class function: pass -class float: pass -[out] -main, line 4: Incompatible types in assignment (expression has type "float", variable has type "A") - -[case testComplexLiteral] -from typing import Undefined -a = 0.0j -b = Undefined # type: A -b = 1.1j -a = 1.1j -class A: - pass -[file builtins.py] -class object: - def __init__(self): pass -class type: pass -class function: pass -class complex: pass -[out] -main, line 4: Incompatible types in assignment (expression has type "complex", variable has type "A") - -[case testBytesLiteral] -from typing import Undefined -b, a = Undefined, Undefined # type: (bytes, A) -b = b'foo' -b = br"foo" -b = b'''foo''' -a = b'foo' # E: Incompatible types in assignment (expression has type "bytes", variable has type "A") -class A: pass -[file builtins.py] -class object: - def __init__(self): pass -class type: pass -class tuple: pass -class function: pass -class bytes: pass - -[case testUnicodeLiteralInPython3] -from typing import Undefined -s = Undefined(str) -s = u'foo' -b = Undefined(bytes) -b = u'foo' # E: Incompatible types in assignment (expression has type "str", variable has type "bytes") -[builtins fixtures/primitives.py] - - --- Binary operators --- ---------------- - - -[case testAdd] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a + c # Fail -a = a + b # Fail -c = b + a # Fail -c = a + b - -class A: - def __add__(self, x: 'B') -> 'C': pass -class B: pass -class C: pass -[out] -main, line 3: Unsupported operand types for + ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for + ("B") -[case testAdd] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a + c # Fail -a = a + b # Fail -c = b + a # Fail -c = a + b - -class A: - def __add__(self, x: 'B') -> 'C': - pass -class B: - pass -class C: - pass -[out] -main, line 3: Unsupported operand types for + ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for + ("B") - -[case testSub] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a - c # Fail -a = a - b # Fail -c = b - a # Fail -c = a - b - -class A: - def __sub__(self, x: 'B') -> 'C': - pass -class B: - pass -class C: - pass -[out] -main, line 3: Unsupported operand types for - ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for - ("B") - -[case testMul] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a * c # Fail -a = a * b # Fail -c = b * a # Fail -c = a * b - -class A: - def __mul__(self, x: 'B') -> 'C': - pass -class B: - pass -class C: - pass -[out] -main, line 3: Unsupported operand types for * ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for * ("B") - -[case testDiv] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a / c # Fail -a = a / b # Fail -c = b / a # Fail -c = a / b - -class A: - def __truediv__(self, x: 'B') -> 'C': - pass -class B: - pass -class C: - pass -[out] -main, line 3: Unsupported operand types for / ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for / ("B") - -[case testIntDiv] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a // c # Fail -a = a // b # Fail -c = b // a # Fail -c = a // b - -class A: - def __floordiv__(self, x: 'B') -> 'C': - pass -class B: - pass -class C: - pass -[out] -main, line 3: Unsupported operand types for // ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for // ("B") - -[case testMod] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a % c # Fail -a = a % b # Fail -c = b % a # Fail -c = a % b - -class A: - def __mod__(self, x: 'B') -> 'C': - pass -class B: - pass -class C: - pass -[out] -main, line 3: Unsupported operand types for % ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for % ("B") - -[case testPow] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a ** c # Fail -a = a ** b # Fail -c = b ** a # Fail -c = a ** b - -class A: - def __pow__(self, x: 'B') -> 'C': - pass -class B: - pass -class C: - pass -[out] -main, line 3: Unsupported operand types for ** ("A" and "C") -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Unsupported left operand type for ** ("B") - -[case testMiscBinaryOperators] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -b = a & a # Fail -b = a | b # Fail -b = a ^ a # Fail -b = a << b # Fail -b = a >> a # Fail - -b = a & b -b = a | a -b = a ^ b -b = a << a -b = a >> b -class A: - def __and__(self, x: 'B') -> 'B': pass - def __or__(self, x: 'A') -> 'B': pass - def __xor__(self, x: 'B') -> 'B': pass - def __lshift__(self, x: 'A') -> 'B': pass - def __rshift__(self, x: 'B') -> 'B': pass -class B: pass -[out] -main, line 3: Unsupported operand types for & ("A" and "A") -main, line 4: Unsupported operand types for | ("A" and "B") -main, line 5: Unsupported operand types for ^ ("A" and "A") -main, line 6: Unsupported operand types for << ("A" and "B") -main, line 7: Unsupported operand types for >> ("A" and "A") - -[case testBooleanAndOr] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, bool) -b = b and b -b = b or b -b = b and a # E: Incompatible types in assignment (expression has type "object", variable has type "bool") -b = a and b # E: Incompatible types in assignment (expression has type "object", variable has type "bool") -b = b or a # E: Incompatible types in assignment (expression has type "object", variable has type "bool") -b = a or b # E: Incompatible types in assignment (expression has type "object", variable has type "bool") -class A: pass -[builtins fixtures/bool.py] - -[case testNonBooleanOr] -from typing import Undefined -c, d, b = Undefined, Undefined, Undefined # type: (C, D, bool) -c = c or c -c = c or d -c = d or c -b = c or c # E: Incompatible types in assignment (expression has type "C", variable has type "bool") -d = c or d # E: Incompatible types in assignment (expression has type "C", variable has type "D") -d = d or c # E: Incompatible types in assignment (expression has type "C", variable has type "D") -class C: pass -class D(C): pass -[builtins fixtures/bool.py] - -[case testInOperator] -from typing import Undefined, Iterator, Iterable, Any -a, b, c, d, e = Undefined, Undefined, Undefined, Undefined, Undefined # type: (A, B, bool, D, Any) -c = c in a # Fail -a = b in a # Fail -c = a in b # Fail -c = b in d # Fail -c = b in a -c = a in d -c = e in d -c = a in e - -class A: - def __contains__(self, x: 'B') -> bool: pass -class B: pass -class D(Iterable[A]): - def __iter__(self) -> Iterator[A]: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Unsupported operand types for in ("bool" and "A") -main, line 4: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 5: Unsupported right operand type for in ("B") -main, line 6: Unsupported operand types for in ("B" and "D") - -[case testNotInOperator] -from typing import Undefined, Iterator, Iterable, Any -a, b, c, d, e = Undefined, Undefined, Undefined, Undefined, Undefined # type: (A, B, bool, D, Any) -c = c not in a # Fail -a = b not in a # Fail -c = a not in b # Fail -c = b not in d # Fail -c = b not in a -c = a not in d -c = e in d -c = a in e - -class A: - def __contains__(self, x: 'B') -> bool: pass -class B: pass -class D(Iterable[A]): - def __iter__(self) -> Iterator[A]: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Unsupported operand types for in ("bool" and "A") -main, line 4: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 5: Unsupported right operand type for in ("B") -main, line 6: Unsupported operand types for in ("B" and "D") - -[case testNonBooleanContainsReturnValue] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, bool) -b = a not in a -b = a in a - -class A: - def __contains__(self, x: 'A') -> object: pass -[builtins fixtures/bool.py] -[out] -main, line 4: Incompatible types in assignment (expression has type "object", variable has type "bool") - -[case testEq] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, bool) -a = a == b # Fail -a = a != b # Fail -b = a == b -b = a != b - -class A: - def __eq__(self, o: object) -> bool: pass - def __ne__(self, o: object) -> bool: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 4: Incompatible types in assignment (expression has type "bool", variable has type "A") - -[case testLtAndGt] -from typing import Undefined -a, b, bo = Undefined, Undefined, Undefined # type: (A, B, bool) -a = a < b # Fail -a = a > b # Fail -bo = a < b -bo = a > b - -class A: - def __lt__(self, o: 'B') -> bool: pass - def __gt__(self, o: 'B') -> bool: pass -class B: - def __lt__(self, o: 'B') -> bool: pass - def __gt__(self, o: 'B') -> bool: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 4: Incompatible types in assignment (expression has type "bool", variable has type "A") - -[case testLeAndGe] -from typing import Undefined -a, b, bo = Undefined, Undefined, Undefined # type: (A, B, bool) -a = a <= b # Fail -a = a >= b # Fail -bo = a <= b -bo = a >= b - -class A: - def __le__(self, o: 'B') -> bool: pass - def __ge__(self, o: 'B') -> bool: pass -class B: - def __le__(self, o: 'B') -> bool: pass - def __ge__(self, o: 'B') -> bool: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 4: Incompatible types in assignment (expression has type "bool", variable has type "A") - -[case testChainedComp] -from typing import Undefined -a, b, bo = Undefined, Undefined, Undefined # type: (A, B, bool) -a < a < b < b # Fail -a < b < b < b -a < a > a < b # Fail - -class A: - def __lt__(self, o: 'B') -> bool: pass - def __gt__(self, o: 'B') -> bool: pass -class B: - def __lt__(self, o: 'B') -> bool: pass - def __gt__(self, o: 'B') -> bool: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Unsupported operand types for > ("A" and "A") -main, line 5: Unsupported operand types for > ("A" and "A") -main, line 5: Unsupported operand types for < ("A" and "A") - - -[case testChainedCompBoolRes] -from typing import Undefined -a, b, bo = Undefined, Undefined, Undefined # type: (A, B, bool) -bo = a < b < b -a = a < b < b # Fail - -class A: - def __lt__(self, o: 'B') -> bool: pass - def __gt__(self, o: 'B') -> bool: pass -class B: - def __lt__(self, o: 'B') -> bool: pass - def __gt__(self, o: 'B') -> bool: pass -[builtins fixtures/bool.py] -[out] -main, line 4: Incompatible types in assignment (expression has type "bool", variable has type "A") - - -[case testChainedCompResTyp] -from typing import Undefined -x, y = Undefined, Undefined # type: (X, Y) -a, b, p, bo = Undefined, Undefined, Undefined, Undefined # type: (A, B, P, bool) -b = y == y == y -bo = y == y == y # Fail -a = x < y -a = x < y == y # Fail -p = x < y == y - -class P: - pass -class A(P): - pass -class B(P): - pass - -class X: - def __lt__(self, o: 'Y') -> A: pass - def __gt__(self, o: 'Y') -> A: pass -class Y: - def __lt__(self, o: 'Y') -> A: pass - def __gt__(self, o: 'Y') -> A: pass - def __eq__(self, o: 'Y') -> B: pass -[builtins fixtures/bool.py] -[out] -main, line 5: Incompatible types in assignment (expression has type "B", variable has type "bool") -main, line 7: Incompatible types in assignment (expression has type "P", variable has type "A") - - -[case testIs] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, bool) -a = a is b # Fail -b = a is b -b = b is a -b = a is None -class A: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "bool", variable has type "A") - -[case testIsNot] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, bool) -a = a is not b # Fail -b = a is not b -b = b is not a -b = a is not None -class A: pass -[builtins fixtures/bool.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "bool", variable has type "A") - -[case testReverseBinaryOperator] -from typing import Undefined, disjointclass -class A: - def __add__(self, x: int) -> int: pass -@disjointclass(int) -class B: - def __radd__(self, x: A) -> str: pass -s = Undefined(str) -n = Undefined(int) -n = A() + 1 -s = A() + B() -n = A() + B() # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testReverseBinaryOperator2] -from typing import Undefined, disjointclass -class A: - def __add__(self, x: 'A') -> object: pass -@disjointclass(A) -class B: - def __radd__(self, x: A) -> str: pass -s = Undefined(str) -n = Undefined(int) -s = A() + B() -n = A() + B() # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testReverseBinaryOperator3] -from typing import Undefined, disjointclass -class N: - def __add__(self, x: 'N') -> object: pass -class A: - def __add__(self, x: N) -> int: pass -@disjointclass(N) -class B: - def __radd__(self, x: N) -> str: pass -s = Undefined(str) -s = A() + B() # E: Unsupported operand types for + ("A" and "B") - -[case testBinaryOperatorWithAnyRightOperand] -from typing import Any -class A: pass -A() + Any(1) - -[case testReverseComparisonOperator] -from typing import Undefined, disjointclass -class C: - def __gt__(self, x: 'A') -> object: pass -class A: - def __lt__(self, x: C) -> int: pass -@disjointclass(C) -class B: - def __gt__(self, x: A) -> str: pass -s = Undefined(str) -n = Undefined(int) -n = A() < C() -s = A() < B() -n = A() < B() # E: Incompatible types in assignment (expression has type "str", variable has type "int") -s = object() < B() # E: Unsupported operand types for > ("B" and "object") - -[case testErrorContextAndBinaryOperators] -import typing -class A: - def __getitem__(self, i: str) -> int: pass -def f() -> None: - A()[1] # Error -class B: - A()[1] # Error -A()[1] # Error -[out] -main: In function "f": -main, line 5: Invalid index type "int" for "A" -main: In class "B": -main, line 7: Invalid index type "int" for "A" -main: At top level: -main, line 8: Invalid index type "int" for "A" - -[case testErrorContextAndBinaryOperators2] -import m -[file m.py] -import typing -class A: - def __getitem__(self, i: str) -> int: pass -def f() -> None: - A()[1] # Error -class B: - A()[1] # Error -A()[1] # Error -[out] -In module imported in main, line 1: -tmp/m.py: In function "f": -tmp/m.py, line 5: Invalid index type "int" for "A" -tmp/m.py: In class "B": -tmp/m.py, line 7: Invalid index type "int" for "A" -tmp/m.py: At top level: -tmp/m.py, line 8: Invalid index type "int" for "A" - - --- Unary operators --- --------------- - - -[case testUnaryMinus] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -a = -a # Fail -b = -b # Fail -b = -a - -class A: - def __neg__(self) -> 'B': - pass -class B: - pass -[out] -main, line 3: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 4: Unsupported operand type for unary - ("B") - -[case testUnaryPlus] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -a = +a # Fail -b = +b # Fail -b = +a - -class A: - def __pos__(self) -> 'B': - pass -class B: - pass -[out] -main, line 3: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 4: Unsupported operand type for unary + ("B") - -[case testUnaryNot] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, bool) -a = not b # Fail -b = not a -b = not b -class A: - pass -[builtins fixtures/bool.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "bool", variable has type "A") - -[case testUnaryBitwiseNeg] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -a = ~a # Fail -b = ~b # Fail -b = ~a - -class A: - def __invert__(self) -> 'B': - pass -class B: - pass -[out] -main, line 3: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 4: Unsupported operand type for ~ ("B") - - --- Indexing --- -------- - - -[case testIndexing] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -c = a[c] # Fail -a = a[b] # Fail -c = b[a] # Fail -c = a[b] - -class A: - def __getitem__(self, x: 'B') -> 'C': - pass -class B: pass -class C: pass -[out] -main, line 3: Invalid index type "C" for "A" -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 5: Value of type "B" is not indexable - -[case testIndexingAsLvalue] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -a[c] = c # Fail -a[b] = a # Fail -b[a] = c # Fail -a[b] = c - -class A: - def __setitem__(self, x: 'B', y: 'C') -> None: - pass -class B: - pass -class C: - pass -[out] -main, line 3: Invalid index type "C" for "A" -main, line 4: Incompatible types in assignment -main, line 5: Unsupported target for indexed assignment - - --- Cast expression --- --------------- - - -[case testCastExpressions] -from typing import Undefined, cast, Any -class A: pass -class B: pass -class C(A): pass -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) - -a = cast(A, a()) # E: "A" not callable -a = cast(Any, a()) # E: "A" not callable -b = cast(A, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = cast(A, b) -a = cast(A, a) -c = cast(C, a) -a = cast(A, c) -a = cast(Any, b) -b = cast(Any, a) -[out] - -[case testAnyCast] -from typing import Undefined, cast, Any -a, b = Undefined, Undefined # type: (A, B) -a = Any(a()) # Fail -a = Any(b) -b = Any(a) -class A: pass -class B: pass -[out] -main, line 3: "A" not callable - - --- Void type --- --------- - - -[case testVoidTypeBasics] -from typing import Undefined -a, o = Undefined, Undefined # type: (A, object) -a = f() # Fail -o = A().g(a) # Fail -A().g(f()) # Fail -x = f() # type: A # Fail -f() -A().g(a) - -def f() -> None: - pass - -class A: - def g(self, x: object) -> None: - pass -[out] -main, line 3: "f" does not return a value -main, line 4: "g" of "A" does not return a value -main, line 5: "f" does not return a value -main, line 6: "f" does not return a value - -[case testVoidWithStatements] -import typing -raise f() # Fail -if f(): # Fail - pass -elif f(): # Fail - pass -while f(): # Fail - pass -def g() -> object: - return f() # Fail - -def f() -> None: pass -[builtins fixtures/exception.py] -[out] -main, line 2: "f" does not return a value -main, line 3: "f" does not return a value -main, line 5: "f" does not return a value -main, line 7: "f" does not return a value -main: In function "g": -main, line 10: "f" does not return a value - -[case testVoidWithExpressions] -from typing import Undefined, cast -a = Undefined # type: A -[f()] # E: "f" does not return a value -f() + a # E: "f" does not return a value -a + f() # E: "f" does not return a value -f() == a # E: "f" does not return a value -a != f() # E: Unsupported left operand type for != ("A") -cast(A, f()) # E: "f" does not return a value -f().foo # E: "f" does not return a value - -def f() -> None: pass -class A: - def __add__(self, x: 'A') -> 'A': pass -[builtins fixtures/list.py] - -[case testVoidWithExpressions2] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, bool) -a < f() # E: Unsupported left operand type for < ("A") -f() <= a # E: "f" does not return a value -f() in a # E: Unsupported right operand type for in ("A") -a in f() # E: "f" does not return a value --f() # E: "f" does not return a value -not f() # E: "f" does not return a value -f() and b # E: "f" does not return a value -b or f() # E: "f" does not return a value - -def f() -> None: pass -class A: - def __add__(self, x: 'A') -> 'A': - pass -[builtins fixtures/bool.py] - - --- Slicing --- ------- - - -[case testGetSlice] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -a = a[1:2] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = a[1:] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = a[:2] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = a[:] # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -b = a[1:2] -b = a[1:] -b = a[:2] -b = a[:] - -class A: - def __getitem__(self, s: slice) -> 'B': pass -class B: pass -[builtins fixtures/slice.py] - -[case testSlicingWithInvalidBase] -from typing import Undefined -a = Undefined # type: A -a[1:2] # E: Invalid index type "slice" for "A" -a[:] # E: Invalid index type "slice" for "A" -class A: - def __getitem__(self, n: int) -> 'A': pass -[builtins fixtures/slice.py] - -[case testSlicingWithNonindexable] -from typing import Undefined -o = Undefined # type: object -o[1:2] # E: Value of type "object" is not indexable -o[:] # E: Value of type "object" is not indexable -[builtins fixtures/slice.py] - -[case testNonIntSliceBounds] -from typing import Undefined, Any -a, o = Undefined, Undefined # type: (Any, object) -a[o:1] # E: Slice index must be an integer or None -a[1:o] # E: Slice index must be an integer or None -a[o:] # E: Slice index must be an integer or None -a[:o] # E: Slice index must be an integer or None -[builtins fixtures/slice.py] - -[case testNoneSliceBounds] -from typing import Undefined, Any -a = Undefined # type: Any -a[None:1] -a[1:None] -a[None:] -a[:None] -[builtins fixtures/slice.py] - - --- String interpolation --- -------------------- - - -[case testStringInterpolationType] -from typing import Undefined, Any -i, f, s = Undefined, Undefined, Undefined # type: (int, float, str) -'%d' % i -'%f' % f -'%s' % s -'%d' % s # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float]") -'%f' % s # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float]") -[builtins fixtures/primitives.py] - -[case testStringInterpolationSAcceptsAnyType] -from typing import Undefined, Any -i, o, s = Undefined, Undefined, Undefined # type: (int, object, str) -'%s %s %s' % (i, o, s) -[builtins fixtures/primitives.py] - -[case testStringInterpolationCount] -'%d %d' % 1 # E: Not enough arguments for format string -'%d %d' % (1, 2) -'%d %d' % (1, 2, 3) # E: Not all arguments converted during string formatting -t = 1, 's' -'%d %s' % t -'%s %d' % t # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float]") -'%d' % t # E: Not all arguments converted during string formatting -[builtins fixtures/primitives.py] - -[case testStringInterpolationWithAnyType] -from typing import Undefined, Any -a = Undefined # type: Any -'%d %d' % a -[builtins fixtures/primitives.py] - -[case testStringInterpolationInvalidPlaceholder] -'%W' % 1 # E: Unsupported format character 'W' - -[case testStringInterpolationWidth] -'%2f' % 3.14 -'%*f' % 3.14 # E: Not enough arguments for format string -'%*f' % (4, 3.14) -'%*f' % (1.1, 3.14) # E: * wants int -[builtins fixtures/primitives.py] - -[case testStringInterpolationPrecision] -'%.2f' % 3.14 -'%.*f' % 3.14 # E: Not enough arguments for format string -'%.*f' % (4, 3.14) -'%.*f' % (1.1, 3.14) # E: * wants int -[builtins fixtures/primitives.py] - -[case testStringInterpolationWidthAndPrecision] -'%4.2f' % 3.14 -'%4.*f' % 3.14 # E: Not enough arguments for format string -'%*.2f' % 3.14 # E: Not enough arguments for format string -'%*.*f' % 3.14 # E: Not enough arguments for format string -'%*.*f' % (4, 2, 3.14) -[builtins fixtures/primitives.py] - -[case testStringInterpolationFlagsAndLengthModifers] -'%04hd' % 1 -'%-.4ld' % 1 -'%+*Ld' % (1, 1) -'% .*ld' % (1, 1) -[builtins fixtures/primitives.py] - -[case testStringInterpolationDoublePercentage] -'%% %d' % 1 -'%3% %d' % 1 -'%*%' % 1 -'%*% %d' % 1 # E: Not enough arguments for format string -[builtins fixtures/primitives.py] - -[case testStringInterpolationC] -'%c' % 1 -'%c' % 's' -'%c' % '' # E: %c requires int or char -'%c' % 'ab' # E: %c requires int or char -[builtins fixtures/primitives.py] - -[case testStringInterpolationMappingTypes] -'%(a)d %(b)s' % {'a': 1, 'b': 's'} -'%(a)d %(b)s' % {'a': 's', 'b': 1} # E: Incompatible types in string interpolation (expression has type "str", placeholder with key 'a' has type "Union[int, float]") -[builtins fixtures/primitives.py] - -[case testStringInterpolationMappingKeys] -'%()d' % {'': 2} -'%(a)d' % {'a': 1, 'b': 2, 'c': 3} -'%(q)d' % {'a': 1, 'b': 2, 'c': 3} # E: Key 'q' not found in mapping -[builtins fixtures/dict.py] - -[case testStringInterpolationMappingDictTypes] -from typing import Undefined, Any, Dict -a = Undefined # type: Any -ds, do, di = Undefined, Undefined, Undefined # type: Dict[str, int], Dict[object, int], Dict[int, int] -'%(a)' % 1 # E: Format requires a mapping (expression has type "int", expected type for mapping is Dict[Any, Any]) -'%()d' % a -'%()d' % ds -'%()d' % do -[builtins fixtures/dict.py] - -[case testStringInterpolationMappingInvalidDictTypes-skip] -from typing import Undefined, Any, Dict -di = Undefined # type: Dict[int, int] -'%()d' % di # E: Format requires a mapping (expression has type Dict[int, int], expected type for mapping is Dict[str, Any]) -[builtins fixtures/dict.py] - -[case testStringInterpolationMappingInvalidSpecifiers] -'%(a)d %d' % 1 # E: String interpolation mixes specifier with and without mapping keys -'%(b)*d' % 1 # E: String interpolation contains both stars and mapping keys -'%(b).*d' % 1 # E: String interpolation contains both stars and mapping keys - -[case testStringInterpolationMappingFlagsAndLengthModifiers] -'%(a)1d' % {'a': 1} -'%(a).1d' % {'a': 1} -'%(a)#1.1ld' % {'a': 1} -[builtins fixtures/dict.py] - - --- Lambdas --- ------- - - -[case testTrivialLambda] -from typing import Function -f = lambda: 1 # type: Function[[], int] -f = lambda: ''.x -f = lambda: '' -[out] -main, line 3: "str" has no attribute "x" -main, line 4: Incompatible return value type: expected builtins.int, got builtins.str -main, line 4: Incompatible types in assignment (expression has type Function[[], str], variable has type Function[[], int]) - -[case testVoidLambda] -import typing -def void() -> None: - pass -x = lambda: void() # type: typing.Function[[], None] - - --- List comprehensions --- ------------------- - - -[case testSimpleListComprehension] -from typing import Undefined, List -a = Undefined # type: List[A] -a = [x for x in a] -b = [x for x in a] # type: List[B] # E: List comprehension has incompatible type List[A] -class A: pass -class B: pass -[builtins fixtures/for.py] - -[case testSimpleListComprehensionNestedTuples] -from typing import Undefined, List -l = Undefined # type: List[(A, (A, B))] -a = [a2 for a1, (a2, b1) in l] # type: List[A] -b = [a2 for a1, (a2, b1) in l] # type: List[B] # E: List comprehension has incompatible type List[A] -class A: pass -class B: pass -[builtins fixtures/for.py] - -[case testSimpleListComprehensionNestedTuples2] -from typing import Undefined, List -l = Undefined # type: List[(int, (int, str))] -a = [f(d) for d, (i, s) in l] -b = [f(s) for d, (i, s) in l] # E: Argument 1 to "f" has incompatible type "str"; expected "int" - -def f(x: int): pass -[builtins fixtures/for.py] - -[case testListComprehensionWithNonDirectMapping] -from typing import Undefined, List -a = Undefined # type: List[A] -b = Undefined # type: List[B] -b = [f(x) for x in a] -a = [f(x) for x in a] # E: List comprehension has incompatible type List[B] -([f(x) for x in b]) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -class A: pass -class B: pass -def f(a: A) -> B: pass -[builtins fixtures/for.py] - -[case testErrorInListComprehensionCondition] -from typing import Undefined, List -a = Undefined # type: List[A] -a = [x for x in a if x()] # E: "A" not callable -class A: pass -[builtins fixtures/for.py] - -[case testTypeInferenceOfListComprehension] -from typing import Undefined, List -a = Undefined # type: List[A] -o = [x for x in a] # type: List[object] -class A: pass -[builtins fixtures/for.py] - - --- Set comprehension --- ----------------- - - -[case testSimpleSetComprehension] -from typing import Undefined, Set -a = Undefined # type: Set[A] -a = {x for x in a} -b = {x for x in a} # type: Set[B] # E: Set comprehension has incompatible type Set[A] -class A: pass -class B: pass -[builtins fixtures/set.py] - - --- Dictionary comprehension --- ----------------- - - -[case testSimpleDictionaryComprehension] -from typing import Undefined, Dict, List, Tuple -abd = Undefined # type: Dict[A, B] -abl = Undefined # type: List[Tuple[A, B]] -abd = {a: b for a, b in abl} -x = {a: b for a, b in abl} # type: Dict[B, A] -y = {a: b for a, b in abl} # type: A -class A: pass -class B: pass -[builtins fixtures/dict.py] -[out] -main, line 5: Key expression in dictionary comprehension has incompatible type "A"; expected type "B" -main, line 5: Value expression in dictionary comprehension has incompatible type "B"; expected type "A" -main, line 6: Incompatible types in assignment (expression has type Dict[A, B], variable has type "A") - - -[case testDictionaryComprehensionWithNonDirectMapping] -from typing import Undefined, Dict, List, Tuple -abd = Undefined # type: Dict[A, B] -abl = Undefined # type: List[Tuple[A, B]] -abd = {a: f(b) for a, b in abl} -class A: pass -class B: pass -class C: pass -def f(b: A) -> C: pass -[builtins fixtures/dict.py] -[out] -main, line 4: Argument 1 to "f" has incompatible type "B"; expected "A" -main, line 4: Value expression in dictionary comprehension has incompatible type "C"; expected type "B" - - --- Generator expressions --- --------------------- - - -[case testSimpleGeneratorExpression] -from typing import Undefined, Iterator -# The implementation is mostly identical to list comprehensions, so a single -# test case is ok. -a = Undefined # type: Iterator[int] -a = x for x in a -b = Undefined # type: Iterator[str] -b = x for x in a # E: Generator has incompatible item type "int" -[builtins fixtures/for.py] - - --- Conditional expressions --- ----------------------- - - -[case testSimpleConditionalExpression] -import typing -y = '' -x = 1 if y else 2 -x = 3 -x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testConditionalExpressionWithEmptyCondition] -import typing -def f() -> None: pass -x = 1 if f() else 2 # E: "f" does not return a value - -[case testConditionalExpressionWithSubtyping] -import typing -class A: pass -class B(A): pass -x = B() if None else A() -x = A() -x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") -y = A() if None else B() -y = A() -y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") - -[case testConditionalExpressionAndTypeContext] -import typing -x = [1] if None else [] -x = [1] -x = ['x'] # E: List item 1 has incompatible type "str" -[builtins fixtures/list.py] - - --- Special cases --- ------------- - - -[case testOperationsWithNonInstanceTypes] -from typing import Undefined, cast -class A: - def __add__(self, a: 'A') -> 'A': pass -a = Undefined # type: A -None + a # Fail -f + a # Fail -a + f # Fail -cast(A, f) - -def f() -> None: - pass -[out] -main, line 5: Unsupported left operand type for + (None) -main, line 6: Unsupported left operand type for + (Function[[], None]) -main, line 7: Unsupported operand types for + ("A" and Function[[], None]) - -[case testOperatorMethodWithInvalidArgCount] -from typing import Undefined -a = Undefined # type: A -a + a # Fail - -class A: - def __add__(self) -> 'A': - pass -[out] -main, line 3: Too many arguments for "__add__" of "A" - -[case testOperatorMethodAsVar] -from typing import Any, Undefined -class A: - def __init__(self, _add: Any) -> None: - self.__add__ = _add -a = Undefined # type: A -a + a -[out] - -[case testOperatorMethodAsVar2] -from typing import Undefined -class A: - def f(self, x: int) -> str: pass - __add__ = f -s = Undefined(str) -s = A() + 1 -A() + (A() + 1) -[out] -main, line 7: Argument 1 has incompatible type "str"; expected "int" - -[case testIndexedLvalueWithSubtypes] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -a[c] = c -a[b] = c -a[c] = b - -class A: - def __setitem__(self, x: 'B', y: 'B') -> None: - pass -class B: - pass -class C(B): - pass -[out] diff --git a/mypy/test/data/check-functions.test b/mypy/test/data/check-functions.test deleted file mode 100644 index 88efac58fe4d..000000000000 --- a/mypy/test/data/check-functions.test +++ /dev/null @@ -1,731 +0,0 @@ --- Test cases for the type checker related to functions, function types and --- calls. - --- See also check-varargs.test. - - --- Function type basics --- -------------------- - - -[case testCallingVariableWithFunctionType] -from typing import Undefined, Function -f = Undefined # type: Function[[A], B] -a, b = Undefined, Undefined # type: (A, B) -a = f(a) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = f(b) # E: Argument 1 has incompatible type "B"; expected "A" -b = f() # E: Too few arguments -b = f(a, a) # E: Too many arguments -b = f(a) - -class A: pass -class B: pass - -[case testSubtypingFunctionTypes] -from typing import Undefined, Function - -class A: pass -class B(A): pass - -f = Undefined(Function[[B], A]) -g = Undefined(Function[[A], A]) # subtype of f -h = Undefined(Function[[B], B]) # subtype of f -g = h # E: Incompatible types in assignment (expression has type Function[[B], B], variable has type Function[[A], A]) -h = f # E: Incompatible types in assignment (expression has type Function[[B], A], variable has type Function[[B], B]) -h = g # E: Incompatible types in assignment (expression has type Function[[A], A], variable has type Function[[B], B]) -g = f # E: Incompatible types in assignment (expression has type Function[[B], A], variable has type Function[[A], A]) -f = g -f = h -f = f -g = g -h = h - -[case testFunctionTypeCompatibilityWithOtherTypes] -from typing import Undefined, Function -f = Undefined # type: Function[[], None] -a, o = Undefined, Undefined # type: (A, object) -a = f # E: Incompatible types in assignment (expression has type Function[[], None], variable has type "A") -f = a # E: Incompatible types in assignment (expression has type "A", variable has type Function[[], None]) -f = o # E: Incompatible types in assignment (expression has type "object", variable has type Function[[], None]) -f = f() # E: Function does not return a value - -f = f -f = None -o = f - -class A: pass - -[case testFunctionSubtypingWithVoid] -from typing import Undefined, Function -f = Undefined # type: Function[[], None] -g = Undefined # type: Function[[], object] -f = g # E: Incompatible types in assignment (expression has type Function[[], object], variable has type Function[[], None]) -g = f # E: Incompatible types in assignment (expression has type Function[[], None], variable has type Function[[], object]) - -f = f -g = g - -[case testFunctionSubtypingWithMultipleArgs] -from typing import Undefined, Function -f = Undefined # type: Function[[A, A], None] -g = Undefined # type: Function[[A, B], None] -h = Undefined # type: Function[[B, B], None] -f = g # E: Incompatible types in assignment (expression has type Function[[A, B], None], variable has type Function[[A, A], None]) -f = h # E: Incompatible types in assignment (expression has type Function[[B, B], None], variable has type Function[[A, A], None]) -g = h # E: Incompatible types in assignment (expression has type Function[[B, B], None], variable has type Function[[A, B], None]) -g = f -h = f -h = g -f = f -g = g -h = h - -class A: pass -class B(A): pass - -[case testFunctionTypesWithDifferentArgumentCounts] -from typing import Undefined, Function -f = Undefined # type: Function[[], None] -g = Undefined # type: Function[[A], None] -h = Undefined # type: Function[[A, A], None] - -f = g # E: Incompatible types in assignment (expression has type Function[[A], None], variable has type Function[[], None]) -f = h # E: Incompatible types in assignment (expression has type Function[[A, A], None], variable has type Function[[], None]) -h = f # E: Incompatible types in assignment (expression has type Function[[], None], variable has type Function[[A, A], None]) -h = g # E: Incompatible types in assignment (expression has type Function[[A], None], variable has type Function[[A, A], None]) - -f = f -g = g -h = h - -class A: pass -[out] - -[case testCompatibilityOfSimpleTypeObjectWithStdType] -from typing import Undefined -t = Undefined # type: type -a = Undefined # type: A - -a = A # E: Incompatible types in assignment (expression has type "A" (type object), variable has type "A") -t = f # E: Incompatible types in assignment (expression has type Function[[], None], variable has type "type") -t = A - -class A: - def __init__(self, a: 'A') -> None: pass - -def f() -> None: pass - -[case testFunctionTypesWithOverloads] -from typing import Undefined, Function, overload, disjointclass -f = Undefined # type: Function[[AA], A] -g = Undefined # type: Function[[B], B] -h = Undefined # type: Function[[A], AA] - -h = i # E: Incompatible types in assignment (expression has type overloaded function, variable has type Function[[A], AA]) -f = j - -f = i -g = i -g = j - -class A: pass -class AA(A): pass - -@disjointclass(A) -class B: pass - -@overload -def i(x: AA) -> A: - pass -@overload -def i(x: B) -> B: - pass - -@overload -def j(x: B) -> B: - pass -@overload -def j(x: A) -> AA: - pass - -[case testOverloadWithThreeItems] -from typing import Undefined, Function, overload, disjointclass -g1 = Undefined # type: Function[[A], A] -g2 = Undefined # type: Function[[B], B] -g3 = Undefined # type: Function[[C], C] -g4 = Undefined # type: Function[[A], B] -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) - -b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = f(c) # E: Incompatible types in assignment (expression has type "C", variable has type "B") -g4 = f # E: Incompatible types in assignment (expression has type overloaded function, variable has type Function[[A], B]) - -g1 = f -g2 = f -g3 = f -a = f(a) -b = f(b) -c = f(c) - -class A: pass - -@disjointclass(A) -class B: pass - -@disjointclass(A) -@disjointclass(B) -class C: pass - -@overload -def f(x: A) -> A: pass -@overload -def f(x: B) -> B: pass -@overload -def f(x: C) -> C: pass - - --- Default argument values --- ----------------------- - - -[case testCallingFunctionsWithDefaultArgumentValues] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -a = f() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = f(b) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -b = f(a, a) # E: Too many arguments for "f" - -b = f() -b = f(a) -b = f(AA()) - -def f(x: 'A' = None) -> 'B': pass - -class A: pass -class AA(A): pass -class B: pass - -[case testDefaultArgumentExpressions] -import typing -def f(x: 'A' = A()) -> None: - b = x # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") - a = x # type: A - -class B: pass -class A: pass -[out] -main: In function "f": - -[case testDefaultArgumentExpressions2] -import typing -def f(x: 'A' = B()) -> None: # E: Incompatible types in assignment (expression has type "B", variable has type "A") - b = x # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") - a = x # type: A - -class B: pass -class A: pass -[out] -main: In function "f": - -[case testDefaultArgumentsWithSubtypes] -import typing -def f(x: 'B' = A()) -> None: # E: Incompatible types in assignment (expression has type "A", variable has type "B") - pass -def g(x: 'A' = B()) -> None: - pass - -class A: pass -class B(A): pass -[out] -main: In function "f": - -[case testMultipleDefaultArgumentExpressions] -import typing -def f(x: 'A' = B(), y: 'B' = B()) -> None: # E: Incompatible types in assignment (expression has type "B", variable has type "A") - pass -def h(x: 'A' = A(), y: 'B' = B()) -> None: - pass - -class A: pass -class B: pass -[out] -main: In function "f": - -[case testMultipleDefaultArgumentExpressions2] -import typing -def g(x: 'A' = A(), y: 'B' = A()) -> None: # E: Incompatible types in assignment (expression has type "A", variable has type "B") - pass - -class A: pass -class B: pass -[out] -main: In function "g": - -[case testDefaultArgumentsAndSignatureAsComment] -import typing -def f(x = 1): # type: (int) -> str - pass -f() -f(1) -f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" - -[case testMethodDefaultArgumentsAndSignatureAsComment] -import typing -class A: - def f(self, x = 1): # type: (int) -> str - pass -A().f() -A().f(1) -A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - - --- Access to method defined as a data attribute --- -------------------------------------------- - - -[case testMethodAsDataAttribute] -from typing import Undefined, Any, Function -class B: pass -x = Undefined # type: Any -class A: - f = x # type: Function[[A], None] - g = x # type: Function[[A, B], None] -a = Undefined # type: A -a.f() -a.g(B()) -a.f(a) # E: Too many arguments -a.g() # E: Too few arguments - -[case testMethodWithInvalidMethodAsDataAttribute] -from typing import Undefined, Any, Function -class B: pass -x = Undefined # type: Any -class A: - f = x # type: Function[[], None] - g = x # type: Function[[B], None] -a = Undefined # type: A -a.f() # E: Invalid method type -a.g() # E: Invalid method type - -[case testMethodWithDynamicallyTypedMethodAsDataAttribute] -from typing import Undefined, Any, Function -class B: pass -x = Undefined # type: Any -class A: - f = x # type: Function[[Any], Any] -a = Undefined # type: A -a.f() -a.f(a) # E: Too many arguments - -[case testOverloadedMethodAsDataAttribute] -from typing import overload, Undefined -class B: pass -class A: - @overload - def f(self) -> None: pass - @overload - def f(self, b: B) -> None: pass - g = f -a = Undefined # type: A -a.g() -a.g(B()) -a.g(a) # E: No overload variant matches argument types - -[case testMethodAsDataAttributeInferredFromDynamicallyTypedMethod] -from typing import Undefined -class A: - def f(self, x): pass - g = f -a = Undefined # type: A -a.g(object()) -a.g(a, a) # E: Too many arguments -a.g() # E: Too few arguments - -[case testMethodAsDataAttributeInGenericClass] -from typing import typevar, Generic, Undefined -t = typevar('t') -class B: pass -class A(Generic[t]): - def f(self, x: t) -> None: pass - g = f -a = Undefined # type: A[B] -a.g(B()) -a.g(a) # E: Argument 1 has incompatible type A[B]; expected "B" - -[case testInvalidMethodAsDataAttributeInGenericClass] -from typing import Undefined, Any, typevar, Generic, Function -t = typevar('t') -class B: pass -class C: pass -x = Undefined # type: Any -class A(Generic[t]): - f = x # type: Function[[A[B]], None] -ab = Undefined # type: A[B] -ac = Undefined # type: A[C] -ab.f() -ac.f() # E: Invalid method type - -[case testPartiallyTypedSelfInMethodDataAttribute] -from typing import Undefined, Any, typevar, Generic, Function -t = typevar('t') -class B: pass -class C: pass -x = Undefined # type: Any -class A(Generic[t]): - f = x # type: Function[[A], None] -ab = Undefined # type: A[B] -ac = Undefined # type: A[C] -ab.f() -ac.f() - -[case testCallableDataAttribute] -from typing import Undefined, Function -class A: - g = Undefined # type: Function[[A], None] - def __init__(self, f: Function[[], None]) -> None: - self.f = f -a = A(None) -a.f() -a.g() -a.f(a) # E: Too many arguments -a.g(a) # E: Too many arguments - - --- Nested functions --- ---------------- - - -[case testSimpleNestedFunction] -import typing -def f(a: 'A') -> None: - def g(b: 'B') -> None: - b = a # fail - aa = a # type: A # ok - b = B() - g(a) # fail - g(B()) -class A: pass -class B: pass -[out] -main: In function "g": -main, line 4: Incompatible types in assignment (expression has type "A", variable has type "B") -main: In function "f": -main, line 7: Argument 1 to "g" has incompatible type "A"; expected "B" - -[case testReturnAndNestedFunction] -import typing -def f() -> 'A': - def g() -> 'B': - return A() # fail - return B() - return B() # fail - return A() -class A: pass -class B: pass -[out] -main: In function "g": -main, line 4: Incompatible return value type: expected __main__.B, got __main__.A -main: In function "f": -main, line 6: Incompatible return value type: expected __main__.A, got __main__.B - -[case testDynamicallyTypedNestedFunction] -import typing -def f(x: object) -> None: - def g(y): - pass - g() # E: Too few arguments for "g" - g(x) -[out] -main: In function "f": - -[case testNestedFunctionInMethod] -import typing -class A: - def f(self) -> None: - def g(x: int) -> None: - y = x # type: int - a = x # type: A # fail - g(2) - g(A()) # fail -[out] -main: In function "g": -main, line 6: Incompatible types in assignment (expression has type "int", variable has type "A") -main: In member "f" of class "A": -main, line 8: Argument 1 to "g" of "A" has incompatible type "A"; expected "int" - - --- Casts --- ----- - - -[case testCastsToAndFromFunctionTypes] -from typing import typevar, Function, Any, cast -t = typevar('t') -def f(x: t, - f1: Function[[], None], - f2: Function[[Any], None], o: object) -> None: - x = cast(t, f1) - f1 = cast(Function[[], None], x) - f1 = cast(Function[[], None], f2) - f1 = cast(Function[[], None], o) - - --- Function decorators --- ------------------- - - -[case testTrivialStaticallyTypedFunctionDecorator] -from typing import typevar -t = typevar('t') -def dec(f: t) -> t: - return f -@dec -def f(x: int) -> None: pass -f(1) -f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" - -[case testTrivialStaticallyTypedMethodDecorator] -from typing import typevar -t = typevar('t') -def dec(f: t) -> t: - return f -class A: - @dec - def f(self, x: int) -> None: pass -A().f(1) -A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -class B: pass - -[case testTrivialDecoratedNestedFunction] -from typing import typevar -t = typevar('t') -def dec(f: t) -> t: - return f -def g() -> None: - @dec - def f(x: int) -> None: pass - f(1) - f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -[out] -main: In function "g": - -[case testCheckingDecoratedFunction] -import typing -def dec(f): pass -@dec -def f(x: 'A') -> None: - a = x # type: A - x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -class A: pass -[out] -main: In function "f": - -[case testDecoratorThatSwitchesType] -from typing import Function -def dec(x) -> Function[[], None]: pass -@dec -def f(y): pass -f() -f(None) # E: Too many arguments for "f" - -[case testDecoratorThatSwitchesTypeWithMethod] -from typing import Any, Function, Undefined -def dec(x) -> Function[[Any], None]: pass -class A: - @dec - def f(self, a, b, c): pass -a = Undefined # type: A -a.f() -a.f(None) # E: Too many arguments for "f" of "A" - -[case testNestedDecorators] -from typing import Any, Function -def dec1(f: Function[[Any], None]) -> Function[[], None]: pass -def dec2(f: Function[[Any, Any], None]) -> Function[[Any], None]: pass -@dec1 -@dec2 -def f(x, y): pass -f() -f(None) # E: Too many arguments for "f" - -[case testInvalidDecorator1] -from typing import Any, Function -def dec1(f: Function[[Any], None]) -> Function[[], None]: pass -def dec2(f: Function[[Any, Any], None]) -> Function[[Any], None]: pass -@dec1 # E: Argument 1 to "dec2" has incompatible type Function[[Any], Any]; expected Function[[Any, Any], None] -@dec2 -def f(x): pass - -[case testInvalidDecorator2] -from typing import Any, Function -def dec1(f: Function[[Any, Any], None]) -> Function[[], None]: pass -def dec2(f: Function[[Any, Any], None]) -> Function[[Any], None]: pass -@dec1 # E: Argument 1 to "dec1" has incompatible type Function[[Any], None]; expected Function[[Any, Any], None] -@dec2 -def f(x, y): pass - - --- Conditional function definition --- ------------------------------- - - -[case testTypeCheckBodyOfConditionalFunction] -from typing import Undefined, Any -x = Undefined # type: Any -if x: - def f(x: int) -> None: - x = 1 - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[out] -main: In function "f": - -[case testCallConditionalFunction] -from typing import Undefined, Any -x = Undefined # type: Any -if x: - def f(x: int) -> None: pass - f(1) - f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -f(1) -f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" - -[case testConditionalFunctionDefinitionWithIfElse] -from typing import Undefined, Any -x = Undefined # type: Any -if x: - def f(x: int) -> None: - x = 'x' # fail - x = 1 -else: - def f(x: int) -> None: - x + 'x' # fail - x = 1 -f(1) -f('x') # fail -[out] -main: In function "f": -main, line 5: Incompatible types in assignment (expression has type "str", variable has type "int") -main, line 9: Unsupported left operand type for + ("int") -main: At top level: -main, line 12: Argument 1 to "f" has incompatible type "str"; expected "int" - -[case testUnconditionalRedefinitionOfConditionalFunction] -from typing import Undefined, Any -x = Undefined # type: Any -if x: - def f(): pass -def f(): pass # E: Name 'f' already defined (overload variants must be next to each other) - -[case testIncompatibleConditionalFunctionDefinition] -from typing import Undefined, Any -x = Undefined # type: Any -if x: - def f(x: int) -> None: pass -else: - def f(x): pass # E: All conditional function variants must have identical signatures - -[case testIncompatibleConditionalFunctionDefinition2] -from typing import Undefined, Any -x = Undefined # type: Any -if x: - def f(x: int) -> None: pass -else: - def f(y: int) -> None: pass # E: All conditional function variants must have identical signatures - -[case testIncompatibleConditionalFunctionDefinition3] -from typing import Undefined, Any -x = Undefined # type: Any -if x: - def f(x: int) -> None: pass -else: - def f(x: int = 0) -> None: pass # E: All conditional function variants must have identical signatures - - --- Conditional method definition --- ----------------------------- - - -[case testTypeCheckBodyOfConditionalMethod] -from typing import Undefined, Any -x = Undefined # type: Any -class A: - if x: - def f(self, x: int) -> None: - x = 1 - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[out] -main: In member "f" of class "A": - -[case testCallConditionalMethodInClassBody] -from typing import Undefined, Any -x = Undefined # type: Any -class A: - if x: - def f(self, x: int) -> None: pass - f(x, 1) - f(x, 'x') # E: Argument 2 to "f" of "A" has incompatible type "str"; expected "int" - f(x, 1) - f(x, 'x') # E: Argument 2 to "f" of "A" has incompatible type "str"; expected "int" -[out] -main: In class "A": - -[case testCallConditionalMethodViaInstance] -from typing import Undefined, Any -x = Undefined # type: Any -class A: - if x: - def f(self, x: int) -> None: pass -A().f(1) -A().f('x') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - -[case testConditionalMethodDefinitionWithIfElse] -from typing import Undefined, Any -x = Undefined # type: Any -class A: - if x: - def f(self, x: int) -> None: - x = 'x' # fail - x = 1 - else: - def f(self, x: int) -> None: - x + 'x' # fail - x = 1 -A().f(1) -A().f('x') # fail -[out] -main: In member "f" of class "A": -main, line 6: Incompatible types in assignment (expression has type "str", variable has type "int") -main, line 10: Unsupported left operand type for + ("int") -main: At top level: -main, line 13: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - -[case testUnconditionalRedefinitionOfConditionalMethod] -from typing import Undefined, Any -x = Undefined # type: Any -class A: - if x: - def f(self): pass - def f(self): pass # E: Name 'f' already defined - -[case testIncompatibleConditionalMethodDefinition] -from typing import Undefined, Any -x = Undefined # type: Any -class A: - if x: - def f(self, x: int) -> None: pass - else: - def f(self, x): pass # E: All conditional function variants must have identical signatures -[out] -main: In class "A": - -[case testConditionalFunctionDefinitionInTry] -import typing -try: - def f(x: int) -> None: pass -except: - def g(x: str) -> None: pass -f(1) -f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -g('x') -g(1) # E: Argument 1 to "g" has incompatible type "int"; expected "str" diff --git a/mypy/test/data/check-generic-subtyping.test b/mypy/test/data/check-generic-subtyping.test deleted file mode 100644 index 19fa7838dd72..000000000000 --- a/mypy/test/data/check-generic-subtyping.test +++ /dev/null @@ -1,615 +0,0 @@ --- Test cases for the type checker related to subtyping and inheritance with --- generics. - - --- Subtyping + inheritance --- ----------------------- - - -[case testSubtypingAndInheritingNonGenericTypeFromGenericType] -from typing import Undefined, typevar, Generic -T = typevar('T') -ac = Undefined # type: A[C] -ad = Undefined # type: A[D] -b = Undefined # type: B - -b = ad # E: Incompatible types in assignment (expression has type A[D], variable has type "B") -ad = b # E: Incompatible types in assignment (expression has type "B", variable has type A[D]) -b = ac # E: Incompatible types in assignment (expression has type A[C], variable has type "B") - -b = b -ac = b - -class C: pass -class A(Generic[T]): pass -class B(A[C]): pass -class D: pass - -[case testSubtypingAndInheritingGenericTypeFromNonGenericType] -from typing import Undefined, typevar, Generic -T = typevar('T') -a = Undefined # type: A -bc = Undefined # type: B[C] -bd = Undefined # type: B[D] - -bc = bd # E: Incompatible types in assignment (expression has type B[D], variable has type B[C]) -bd = bc # E: Incompatible types in assignment (expression has type B[C], variable has type B[D]) -bc = a # E: Incompatible types in assignment (expression has type "A", variable has type B[C]) -bd = a # E: Incompatible types in assignment (expression has type "A", variable has type B[D]) - -a = bc -a = bd - -class A: pass -class B(A, Generic[T]): pass -class C: pass -class D: pass - -[case testSubtypingAndInheritingGenericTypeFromGenericType] -from typing import Undefined, typevar, Generic -T = typevar('T') -S = typevar('S') -ac = Undefined # type: A[C] -ad = Undefined # type: A[D] -bcc = Undefined # type: B[C, C] -bdc = Undefined # type: B[D, C] - -ad = bcc # E: Incompatible types in assignment (expression has type B[C, C], variable has type A[D]) -ad = bdc # E: Incompatible types in assignment (expression has type B[D, C], variable has type A[D]) -bcc = ac # E: Incompatible types in assignment (expression has type A[C], variable has type B[C, C]) -bdc = ac # E: Incompatible types in assignment (expression has type A[C], variable has type B[D, C]) - -bcc = bcc -bdc = bdc -ac = bcc -ac = bdc - -class A(Generic[T]): pass -class B(A[S], Generic[T, S]): pass -class C: pass -class D: pass - -[case testSubtypingAndInheritingGenericTypeFromGenericTypeAcrossHierarchy] -from typing import Undefined, typevar, Generic -T = typevar('T') -S = typevar('S') -X = typevar('X') -Y = typevar('Y') -ae = Undefined # type: A[A[E]] -af = Undefined # type: A[A[F]] - -cef = Undefined # type: C[E, F] -cff = Undefined # type: C[F, F] -cfe = Undefined # type: C[F, E] - -ae = cef # E: Incompatible types in assignment (expression has type C[E, F], variable has type A[A[E]]) -af = cfe # E: Incompatible types in assignment (expression has type C[F, E], variable has type A[A[F]]) - -ae = cfe -af = cef -af = cff - -class A(Generic[T]): pass -class B(A[S], Generic[T, S]): pass -class C(B[A[X], A[Y]], Generic[X, Y]): pass -class E: pass -class F: pass - -[case testIncludingBaseClassTwice] -from typing import typevar, Generic -t = typevar('t') -class I(Generic[t]): pass -class A(I[C], I[object]): pass # E: Duplicate base class "I" -class C: pass - - --- Accessing inherited generic members --- ----------------------------------- - - -[case testAccessingMethodInheritedFromGenericType] -from typing import Undefined, typevar, Generic -T = typevar('T') -S = typevar('S') -b = Undefined # type: B[C, D] -c, d = Undefined, Undefined # type: (C, D) - -b.f(c) # E: Argument 1 to "f" of "A" has incompatible type "C"; expected "D" -b.f(d) - -class A(Generic[T]): - def f(self, a: T) -> None: - pass -class B(A[S], Generic[T, S]): pass -class C: pass -class D: pass - -[case testAccessingMethodInheritedFromGenericTypeInNonGenericType] -from typing import Undefined, typevar, Generic -T = typevar('T') -b, c, d = Undefined, Undefined, Undefined # type: (B, C, D) - -b.f(c) # E: Argument 1 to "f" of "A" has incompatible type "C"; expected "D" -b.f(d) - -class C: pass -class D: pass -class A(Generic[T]): - def f(self, a: T) -> None: - pass -class B(A[D]): pass - -[case testAccessingMemberVarInheritedFromGenericType] -from typing import typevar, Generic, Undefined -T = typevar('T') -S = typevar('S') -class A(Generic[T]): - def __init__(self, a: T) -> None: - self.a = a - -b = Undefined # type: B[C, D] -c, d = Undefined, Undefined # type: (C, D) - -b.a = c # E: Incompatible types in assignment (expression has type "C", variable has type "D") -b.a = d - -class B(A[S], Generic[T, S]): pass -class C: pass -class D: pass - - --- Overriding with generic types --- ----------------------------- - - -[case testOverridingMethodInSimpleTypeInheritingGenericType] -from typing import typevar, Generic -T = typevar('T') -class B(Generic[T]): - def f(self, a: T) -> None: pass - def g(self, a: T) -> None: pass -class C: pass -class D: pass -class A(B[C]): - def f(self, a: D) -> None: pass \ - # E: Argument 1 of "f" incompatible with supertype "B" - def g(self, a: C) -> None: pass -[out] -main: In class "A": - -[case testOverridingMethodInGenericTypeInheritingSimpleType] -from typing import typevar, Generic -T = typevar('T') -class C: pass -class B: - def f(self, a: C) -> None: pass - def g(self, a: C) -> None: pass -class A(B, Generic[T]): - def f(self, a: T) -> None: pass \ - # E: Argument 1 of "f" incompatible with supertype "B" - def g(self, a: 'C') -> None: pass -[out] -main: In class "A": - -[case testOverridingMethodInGenericTypeInheritingGenericType] -from typing import typevar, Generic -T = typevar('T') -S = typevar('S') -class B(Generic[T]): - def f(self, a: T) -> None: pass - def g(self, a: T) -> None: pass -class A(B[S], Generic[T, S]): - def f(self, a: T) -> None: pass \ - # E: Argument 1 of "f" incompatible with supertype "B" - def g(self, a: S) -> None: pass -[out] -main: In class "A": - -[case testOverridingMethodInMultilevelHierarchyOfGenericTypes] -from typing import typevar, Generic -T = typevar('T') -S = typevar('S') -U = typevar('U') -V = typevar('V') - -class D: pass -class C(Generic[T, U, V]): - def f(self, a: V) -> None: pass - def g(self, a: V) -> None: pass -class B(C[D, D, T], Generic[T]): pass -class A(B[S], Generic[T, S]): - def f(self, a: T) -> None: pass \ - # E: Argument 1 of "f" incompatible with supertype "C" - def g(self, a: S) -> None: pass -[out] -main: In class "A": - - --- Inheritance from generic types with implicit dynamic supertype --- -------------------------------------------------------------- - - -[case testInheritanceFromGenericWithImplicitDynamicAndSubtyping] -from typing import Undefined, typevar, Generic -T = typevar('T') -a = Undefined # type: A -bc = Undefined # type: B[C] -bd = Undefined # type: B[D] - -a = bc # E: Incompatible types in assignment (expression has type B[C], variable has type "A") -bc = a -bd = a - -class B(Generic[T]): pass -class A(B): pass -class C: pass -class D: pass -[out] - -[case testInheritanceFromGenericWithImplicitDynamicAndExternalAccess] -from typing import Undefined, typevar, Generic -T = typevar('T') - -class B(Generic[T]): - def f(self, a: 'B[T]') -> None: pass - def __init__(self, x: 'B[T]') -> None: - self.x = x -class A(B): pass -class C: pass - -a = Undefined # type: A -c = Undefined # type: C -bc = Undefined # type: B[C] - -a.x = c # E: Incompatible types in assignment (expression has type "C", variable has type B[Any]) -a.f(c) # E: Argument 1 to "f" of "B" has incompatible type "C"; expected B[Any] -a.x = bc -a.f(bc) -[out] - -[case testInheritanceFromGenericWithImplicitDynamic] -from typing import Undefined, typevar, Generic -T = typevar('T') -a = Undefined # type: A -c = Undefined # type: C -bc = Undefined # type: B[C] - -class B(Generic[T]): - def f(self, a: 'B[T]') -> None: pass - def __init__(self, x: 'B[T]') -> None: - self.x = x - -class A(B): - def g(self) -> None: - self.x = c # E: Incompatible types in assignment (expression has type "C", variable has type B[Any]) - self.f(c) # E: Argument 1 to "f" of "B" has incompatible type "C"; expected B[Any] - self.x = bc - self.f(bc) - -class C: pass -[out] -main: In member "g" of class "A": - -[case testInheritanceFromGenericWithImplicitDynamicAndOverriding] -from typing import typevar, Generic, Tuple -T = typevar('T') -class B(Generic[T]): - def f(self, a: T, b: 'Tuple[T, B[T]]') -> None: - pass -class A(B): - def f(self, a, b): pass -[builtins fixtures/tuple.py] -[out] - - --- Inheritance from generic types and super expressions --- ---------------------------------------------------- - - -[case testSuperExpressionsWhenInheritingFromGenericType] -from typing import typevar, Generic -T = typevar('T') -S = typevar('S') -class B(Generic[T]): - def f(self, a: T) -> None: pass -class A(B[S], Generic[T, S]): - def g(self, t: T, s: S) -> None: - super().f(t) # E: Argument 1 to "f" of "B" has incompatible type "T"; expected "S" - super().f(s) -[out] -main: In member "g" of class "A": - -[case testSuperExpressionsWhenInheritingFromGenericTypeAndDeepHierarchy] -from typing import typevar, Generic -T = typevar('T') -S = typevar('S') -U = typevar('U') -V = typevar('V') -class C(Generic[T, U, V]): - def f(self, a: V) -> None: pass -class D: pass -class B(C[D, D, T], Generic[T]): pass -class A(B[S], Generic[T, S]): - def g(self, t: T, s: S) -> None: - super().f(t) # E: Argument 1 to "f" of "C" has incompatible type "T"; expected "S" - super().f(s) -[out] -main: In member "g" of class "A": - - --- Subtyping with a generic abstract base class --- -------------------------------------------- - - -[case testSubtypingWithGenericTypeSubclassingGenericAbstractClass] -from typing import Undefined, typevar, Generic, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -S = typevar('S') -acd = Undefined # type: A[C, D] -adc = Undefined # type: A[D, C] -ic = Undefined # type: I[C] -id = Undefined # type: I[D] - -ic = acd # E: Incompatible types in assignment (expression has type A[C, D], variable has type I[C]) -id = adc # E: Incompatible types in assignment (expression has type A[D, C], variable has type I[D]) -adc = ic # E: Incompatible types in assignment (expression has type I[C], variable has type A[D, C]) - -ic = adc -id = acd - -class I(AbstractGeneric[T]): - @abstractmethod - def f(self): pass -class A(I[S], AbstractGeneric[T, S]): pass -class C: pass -class D: pass - -[case testSubtypingWithTypeImplementingGenericABCViaInheritance] -from typing import Undefined, typevar, AbstractGeneric -S = typevar('S') -a, b = Undefined, Undefined # type: (A, B) -ic, id, ie = Undefined, Undefined, Undefined # type: (I[C], I[D], I[E]) - -class I(AbstractGeneric[S]): pass -class B(I[C]): pass -class A(B): pass - -ie = a # E: Incompatible types in assignment (expression has type "A", variable has type I[E]) -a = ic # E: Incompatible types in assignment (expression has type I[C], variable has type "A") -a = id # E: Incompatible types in assignment (expression has type I[D], variable has type "A") -a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") -id = a # E: Incompatible types in assignment (expression has type "A", variable has type I[D]) - -ic = a -b = a - -class C: pass -class D: pass -class E: pass -[out] - -[case testSubtypingWithTypeImplementingGenericABCViaInheritance2-skip] -from typing import typevar, AbstractGeneric -T = typevar('T') -class I(AbstractGeneric[T]): pass -class A(I[C]): pass -class B(A, I[D]): pass # Fail - -class C: pass -class D: pass -[out] -main: In class "B": -main, line 5: Class "B" has base "I" duplicated inconsistently - -[case testSubtypingAndABCExtension] -from typing import Undefined, typevar, AbstractGeneric, Generic -from abc import abstractmethod, ABCMeta -t = typevar('t') -a, i, j = Undefined, Undefined, Undefined # type: (A[object], I[object], J[object]) -(ii, jj) = (i, j) -ii = a -jj = a -jj = i -a = i # E: Incompatible types in assignment (expression has type I[object], variable has type A[object]) -a = j # E: Incompatible types in assignment (expression has type J[object], variable has type A[object]) - -class J(AbstractGeneric[t]): pass -class X(metaclass=ABCMeta): pass -class I(X, J[t], AbstractGeneric[t]): pass -class A(I[t], AbstractGeneric[t]): pass - - --- Subclassing a generic ABC --- ------------------------- - - -[case testSubclassingGenericABC1] -from typing import typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -class I(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T) -> None: pass - @abstractmethod - def g(self, a: T) -> None: pass -class A(I[C]): - def f(self, a: 'D') -> None: pass \ - # E: Argument 1 of "f" incompatible with supertype "I" - def g(self, a: 'C') -> None: pass -class C: pass -class D: pass -[out] -main: In class "A": - - --- Extending a generic ABC with deep type hierarchy --- ------------------------------------------------ - - -[case testSubclassingGenenericABCWithDeepHierarchy] -from typing import Undefined, Any, typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -a = Undefined # type: A -ic, id = Undefined, Undefined # type: (I[C], I[D]) - -id = a # Fail -ic = a - -class I(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T, b: T) -> None: pass - @abstractmethod - def g(self, a: T, b: 'D') -> None: pass -class B(I[C]): - def f(self, a: 'C', b: 'C') -> None: pass - def g(self, a: 'C', b: Any) -> None: pass -class A(B): - def g(self, a: 'C', b: 'C') -> None: pass \ - # E: Argument 2 of "g" incompatible with supertype "I" - def f(self, a: 'C', b: 'C') -> None: pass -class C: pass -class D: pass -[out] -main, line 7: Incompatible types in assignment (expression has type "A", variable has type I[D]) -main: In class "A": - -[case testSubclassingGenenericABCWithDeepHierarchy2] -from typing import Any, typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -class I(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T, b: T) -> None: pass -class B(I[C]): - def f(self, a: 'C', b: Any) -> None: pass -class A(B): - def f(self, a: 'C', b: 'D') -> None: pass \ - # E: Argument 2 of "f" incompatible with supertype "I" -class C: pass -class D: pass -[out] -main: In class "A": - - --- Implicit Any types and subclassing generic ABC --- ---------------------------------------------- - - -[case testSubclassingGenericABCWithImplicitAny] -from typing import Undefined, Any, typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -a = Undefined # type: Any -ic = Undefined # type: I[C] -id = Undefined # type: I[D] - -ic = a -id = a - -class I(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T) -> None: pass -class A(I): - def f(self, a): pass - -class C: pass -class D: pass - -[case testSubclassingGenericABCWithImplicitAnyAndDeepHierarchy] -from typing import Undefined, Any, typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -a = Undefined # type: Any -ic = Undefined # type: I[C] -id = Undefined # type: I[D] - -ic = a -id = a - -class I(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T, b: T) -> None: pass -class B(I): - def f(self, a, b): pass -class A(B): - def f(self, a: 'C', b: 'D') -> None: pass -class C: pass -class D: pass - -[case testImplementingGenericABCWithImplicitAnyAndDeepHierarchy2] -from typing import Undefined, Any, typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -a = Undefined # type: Any -jc = Undefined # type: J[C] -jd = Undefined # type: J[D] - -jc = a -jd = a - -class J(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T, b: T) -> None: pass -class I(J): - @abstractmethod - def f(self, a, b): pass -class A(I): - def f(self, a: 'C', b: 'D') -> None: pass - -class C: pass -class D: pass - - --- Accessing generic ABC members --- ----------------------------- - - -[case testAccessingGenericABCMembers] -from typing import Undefined, typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -class I(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T) -> None: pass -class A: pass -class B: pass - -a, b = Undefined, Undefined # type: (A, B) -ia = Undefined # type: I[A] - -ia.f(b) # E: Argument 1 to "f" of "I" has incompatible type "B"; expected "A" -ia.f(a) - -[case testAccessingInheritedGenericABCMembers] -from typing import Undefined, typevar, AbstractGeneric -from abc import abstractmethod -T = typevar('T') -class J(AbstractGeneric[T]): - @abstractmethod - def f(self, a: T) -> None: pass -class I(J[T], AbstractGeneric[T]): pass -class A: pass -class B: pass -a, b = Undefined, Undefined # type: (A, B) -ia = Undefined # type: I[A] - -ia.f(b) # E: Argument 1 to "f" of "J" has incompatible type "B"; expected "A" -ia.f(a) - - --- Misc --- ---- - - -[case testMultipleAssignmentAndGenericSubtyping] -from typing import Iterable, Undefined -n, s = Undefined(int), Undefined(str) -class Nums(Iterable[int]): - def __iter__(self): pass - def __next__(self): pass -n, n = Nums() -s, s = Nums() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[builtins fixtures/for.py] -[out] diff --git a/mypy/test/data/check-generics.test b/mypy/test/data/check-generics.test deleted file mode 100644 index da5cd2a842b8..000000000000 --- a/mypy/test/data/check-generics.test +++ /dev/null @@ -1,830 +0,0 @@ --- Simple generic types --- -------------------- - - -[case testGenericMethodReturnType] -from typing import Undefined, typevar, Generic -T = typevar('T') -a, b, c = Undefined, Undefined, Undefined # type: (A[B], B, C) -c = a.f() # Fail -b = a.f() - -class A(Generic[T]): - def f(self) -> T: pass - -class B: pass -class C: pass -[out] -main, line 4: Incompatible types in assignment (expression has type "B", variable has type "C") - -[case testGenericMethodArgument] -from typing import Undefined, typevar, Generic -T = typevar('T') -a.f(c) # Fail -a.f(b) - -a = Undefined # type: A[B] -b = Undefined # type: B -c = Undefined # type: C - -class A(Generic[T]): - def f(self, a: T) -> None: pass - -class B: pass -class C: pass -[out] -main, line 3: Argument 1 to "f" of "A" has incompatible type "C"; expected "B" - -[case testGenericMemberVariable] -from typing import typevar, Generic, Undefined -T = typevar('T') -class A(Generic[T]): - def __init__(self, v: T) -> None: - self.v = v - -a, b, c = Undefined, Undefined, Undefined # type: (A[B], B, C) -a.v = c # Fail -a.v = b - -class B: pass -class C: pass -[out] -main, line 8: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testGenericMemberVariable] -from typing import Undefined, typevar, Generic -T = typevar('T') -a, b, c = Undefined, Undefined, Undefined # type: (A[B], B, C) -a.v = c # Fail -a.v = b - -class A(Generic[T]): - v = Undefined # type: T -class B: pass -class C: pass -[out] -main, line 4: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testSimpleGenericSubtyping] -from typing import Undefined, typevar, Generic -T = typevar('T') -b, bb, c = Undefined, Undefined, Undefined # type: (A[B], A[B], A[C]) -c = b # Fail -b = c # Fail - -b = b -b = bb - -class A(Generic[T]): pass -class B: pass -class C(B): pass -[out] -main, line 4: Incompatible types in assignment (expression has type A[B], variable has type A[C]) -main, line 5: Incompatible types in assignment (expression has type A[C], variable has type A[B]) - -[case testGenericTypeCompatibilityWithAny] -from typing import Undefined, Any, typevar, Generic -T = typevar('T') -b, c, d = Undefined, Undefined, Undefined # type: (A[B], A[C], A[Any]) - -b = d -c = d -d = b -d = c - -class A(Generic[T]): pass -class B: pass -class C(B): pass -[out] - -[case testTypeVariableAsTypeArgument] -from typing import Undefined, typevar, Generic -T = typevar('T') -a = Undefined # type: A[B] -b = Undefined # type: A[B] -c = Undefined # type: A[C] - -a.v = c # Fail -c = a.v # Fail -a.v = b -b = a.v - -class A(Generic[T]): - v = None # type: A[T] - -class B: pass -class C: pass -[out] -main, line 7: Incompatible types in assignment (expression has type A[C], variable has type A[B]) -main, line 8: Incompatible types in assignment (expression has type A[B], variable has type A[C]) - -[case testMultipleGenericTypeParametersWithMemberVars] -from typing import Undefined, typevar, Generic -S = typevar('S') -T = typevar('T') -a = Undefined # type: A[B, C] -s = Undefined # type: B -t = Undefined # type: C - -t = a.s # Fail -s = a.t # Fail - -s = a.s -t = a.t - -class A(Generic[S, T]): - s = Undefined # type: S - t = Undefined # type: T -class B: pass -class C: pass -[out] -main, line 8: Incompatible types in assignment (expression has type "B", variable has type "C") -main, line 9: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testMultipleGenericTypeParametersWithMethods] -from typing import Undefined, typevar, Generic -S = typevar('S') -T = typevar('T') -a = Undefined # type: A[B, C] -s = Undefined # type: B -t = Undefined # type: C - -a.f(s, s) # Fail -a.f(t, t) # Fail -a.f(s, t) - -class A(Generic[S, T]): - def f(self, s: S, t: T) -> None: pass -class B: pass -class C: pass -[out] -main, line 8: Argument 2 to "f" of "A" has incompatible type "B"; expected "C" -main, line 9: Argument 1 to "f" of "A" has incompatible type "C"; expected "B" - -[case testMultipleGenericTypeParametersAndSubtyping] -from typing import Undefined, typevar, Generic -S = typevar('S') -T = typevar('T') -bc = Undefined # type: A[B, C] -bb = Undefined # type: A[B, B] -cb = Undefined # type: A[C, B] - -bb = bc # Fail -bb = cb # Fail -bc = bb # Fail - -bb = bb -bc = bc - -class A(Generic[S, T]): - s = Undefined # type: S - t = Undefined # type: T - -class B: pass -class C(B):pass -[out] -main, line 8: Incompatible types in assignment (expression has type A[B, C], variable has type A[B, B]) -main, line 9: Incompatible types in assignment (expression has type A[C, B], variable has type A[B, B]) -main, line 10: Incompatible types in assignment (expression has type A[B, B], variable has type A[B, C]) - - --- Simple generic type bodies --- -------------------------- - - -[case testGenericTypeBody1] -from typing import typevar, Generic, Undefined -T = typevar('T') -class A(Generic[T]): - a = Undefined # type: T - - def f(self, b: T) -> T: - self.f(x) # Fail - d = self # type: A[B] # Fail - self.a = self.f(self.a) - return self.a - c = self # type: A[T] -x = Undefined # type: B -class B: pass -[out] -main: In member "f" of class "A": -main, line 7: Argument 1 to "f" of "A" has incompatible type "B"; expected "T" -main, line 8: Incompatible types in assignment (expression has type A[T], variable has type A[B]) - -[case testGenericTypeBodyWithMultipleVariables] -from typing import typevar, Generic, Undefined -S = typevar('S') -T = typevar('T') -class A(Generic[S, T]): - def f(self) -> None: - s = Undefined # type: S - t = Undefined # type: T - s = t # Fail - t = s # Fail - a = self # type: A[S, B] # Fail - b = self # type: A[T, T] # Fail - c = self # type: A[S, T] - t = t - -class B: pass -[out] -main: In member "f" of class "A": -main, line 8: Incompatible types in assignment (expression has type "T", variable has type "S") -main, line 9: Incompatible types in assignment (expression has type "S", variable has type "T") -main, line 10: Incompatible types in assignment (expression has type A[S, T], variable has type A[S, B]) -main, line 11: Incompatible types in assignment (expression has type A[S, T], variable has type A[T, T]) - -[case testCompatibilityOfNoneWithTypeVar] -from typing import typevar, Generic -T = typevar('T') -class A(Generic[T]): - def f(self) -> None: - a = None # type: T - a = None -[out] - -[case testCompatibilityOfTypeVarWithObject] -from typing import typevar, Generic -T = typevar('T') -class A(Generic[T]): - def f(self) -> T: - a = object() # type: T # Fail - a = object() # Fail - b = self.f() # type: object - b = self.f() -[out] -main: In member "f" of class "A": -main, line 5: Incompatible types in assignment (expression has type "object", variable has type "T") -main, line 6: Incompatible types in assignment (expression has type "object", variable has type "T") - - --- Operations with generic types --- ----------------------------- - - -[case testGenericOperations] -from typing import Undefined, typevar, Generic -S = typevar('S') -T = typevar('T') -a = Undefined # type: A[B, C] -b = Undefined # type: B -c = Undefined # type: C - -b = a + b # Fail -c = a + c # Fail -c = a[c] # Fail -b = a[b] # Fail - -c = a + b -b = a[c] - -class A(Generic[S, T]): - def __add__(self, a: S) -> T: pass - def __getitem__(self, i: T) -> S: pass - -class B: pass -class C: pass -[out] -main, line 8: Incompatible types in assignment (expression has type "C", variable has type "B") -main, line 9: Unsupported operand types for + ("A" and "C") -main, line 10: Incompatible types in assignment (expression has type "B", variable has type "C") -main, line 11: Invalid index type "B" for "A" - -[case testOperatorAssignmentWithIndexLvalue1] -from typing import Undefined, typevar, Generic -T = typevar('T') -b = Undefined # type: B -c = Undefined # type: C -ac = Undefined # type: A[C] - -ac[b] += b # Fail -ac[c] += c # Fail -ac[b] += c -ac[b] = ac[b] + c - -class A(Generic[T]): - def __getitem__(self, i: 'B') -> T: pass - def __setitem__(self, i: 'B', v: T) -> None: pass - -class B: pass -class C: - def __add__(self, o: 'C') -> 'C': pass -[out] -main, line 7: Unsupported operand types for + ("C" and "B") -main, line 7: Incompatible types in assignment -main, line 8: Invalid index type "C" for "A" - -[case testOperatorAssignmentWithIndexLvalue2] -from typing import Undefined, typevar, Generic -T = typevar('T') -b = Undefined # type: B -c = Undefined # type: C -ac = Undefined # type: A[C] - -ac[b] += c # Fail -ac[c] += c # Fail -ac[b] = ac[b] + c # Fail - -class A(Generic[T]): - def __getitem__(self, i: 'B') -> T: pass - def __setitem__(self, i: 'C', v: T) -> None: pass - -class B: pass -class C: - def __add__(self, o: 'C') -> 'C': pass -[out] -main, line 7: Invalid index type "B" for "A" -main, line 8: Invalid index type "C" for "A" -main, line 9: Invalid index type "B" for "A" - - --- Nested generic types --- -------------------- - - -[case testNestedGenericTypes] -from typing import Undefined, typevar, Generic -T = typevar('T') -aab = Undefined # type: A[A[B]] -aac = Undefined # type: A[A[C]] -ab = Undefined # type: A[B] -ac = Undefined # type: A[C] - -ac = aab.x # Fail -ac.y = aab # Fail - -ab = aab.x -ac = aac.x -ab.y = aab -ac.y = aac - -class A(Generic[T]): - x = None # type: T - y = None # type: A[A[T]] - -class B: - pass -class C: - pass -[out] -main, line 8: Incompatible types in assignment (expression has type A[B], variable has type A[C]) -main, line 9: Incompatible types in assignment (expression has type A[A[B]], variable has type A[A[C]]) - - --- Generic functions --- ----------------- - - -[case testTrivialTypeApplication] -from typing import typevar -T = typevar('T') -def f(x: T) -> None: pass -f[object](None) -[out] - -[case testSimpleGenericFunction] -from typing import Undefined, typevar -T = typevar('T') -class A: pass -class B: pass -a = Undefined # type: A -b = Undefined # type: B - -def id(a: T) -> T: - return a - -b = id[B](a) # E: Argument 1 to "id" has incompatible type "A"; expected "B" -a = id[B](b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = id[A](a) - -[case testGenericFunctionWithTwoTypeArguments] -from typing import Undefined, typevar, Generic -S = typevar('S') -T = typevar('T') - -class t(Generic[S, T]): pass -class B: pass -class C: pass - -tbc = Undefined # type: t[B, C] -tbb = Undefined # type: t[B, B] -b = Undefined # type: B -c = Undefined # type: C - -def f(a: S, b: t[S, T]) -> t[S, T]: pass - -tbc = f[B, C](c, tbc) # E: Argument 1 to "f" has incompatible type "C"; expected "B" -tbc = f[B, C](b, tbb) # E: Argument 2 to "f" has incompatible type t[B, B]; expected t[B, C] -tbb = f[B, C](b, tbc) # E: Incompatible types in assignment (expression has type t[B, C], variable has type t[B, B]) -tbc = f[B, C](b, tbc) - -[case testInvalidNumberOfTypeArgumentsForGenericFunction] -from typing import Undefined, typevar, Generic -s = typevar('s') -t = typevar('t') -class p(Generic[s, t]): pass -def f(a: p[s, t]) -> None: pass -def g() -> None: pass -a = Undefined # type: p[object, object] - -a = f[object](a) # E: Type application has too few types (2 expected) -a = f[object, object, object](a) # E: Type application has too many types (2 expected) -g[object]() # E: Type application targets a non-generic function -f[object, object](a) - --- NOTE: Commented out test cases that do not make sense currently. - ---[case testGenericMethodInNonGenericType] ---from typing import Undefined, typevar ---T = typevar('T') ---a, b = Undefined, Undefined # type: (A, B) ---class A: --- def f(self, a: T) -> T: pass ---class B: pass --- ---a = a.f[B](b) # E: Incompatible types in assignment ---a = a.f[A](b) # E: Argument 1 to "f" of "A" has incompatible type "B" ---a = a.f[A](a) - ---[case testGenericMethodInGenericType] ---from typing import Undefined, typevar, Generic ---T = typevar('T') ---S = typevar('S') ---class A(Generic[T]): --- def f(self, a: S) -> 'A[t[S, T]]': --- pass ---class t(Generic[T, S]): pass ---class B: pass ---class C: pass ---ab = Undefined # type: A[B] ---acb = Undefined # type: A[t[C, B]] ---abc = Undefined # type: A[t[B, C]] ---b = Undefined # type: B ---c = Undefined # type: C --- ---abc = ab.f[C](c) # E: Incompatible types in assignment ---acb = ab.f[C](b) # E: Argument 1 to "f" of "A" has incompatible type "B" ---acb = ab.f[C](c) - -[case testTypeCheckingGenericFunctionBody] -from typing import typevar, Generic -S = typevar('S') -T = typevar('T') -class A: pass -class p(Generic[T, S]): - def __init__(self, t: T, a: S) -> None: pass -def f(s: S, t: T) -> p[T, A]: - a = t # type: S # E: Incompatible types in assignment (expression has type "T", variable has type "S") - s = t # E: Incompatible types in assignment (expression has type "T", variable has type "S") - return p[S, A](s, A()) # E: Incompatible return value type: expected __main__.p[T`-2, __main__.A], got __main__.p[S`-1, __main__.A*] - b = t # type: T - c = s # type: S - return p[T, A](t, A()) -[out] -main: In function "f": - -[case testTypeCheckingGenericMethodBody] -from typing import typevar, Generic -T = typevar('T') -S = typevar('S') -class p(Generic[T, S]): - def __init__(self, t: T, a: S) -> None: pass -class A(Generic[T]): - def f(self, s: S, t: T) -> p[S, T]: - s = t # E: Incompatible types in assignment (expression has type "T", variable has type "S") - return p[S, S](s, s) # E: Incompatible return value type: expected __main__.p[S`-1, T`1], got __main__.p[S`-1, S`-1] - return p[T, T](t, t) # E: Incompatible return value type: expected __main__.p[S`-1, T`1], got __main__.p[T`1, T`1] - t = t - s = s - return p[S, T](s, t) -[out] -main: In member "f" of class "A": - - --- Constructing instances + generics --- --------------------------------- - - -[case testConstructingGenericInstanceWithEmptyConstructor] -from typing import typevar, Generic -T = typevar('T') -class A(Generic[T]): pass -class B: pass -class C: pass -a = A[B]() # type: A[C] # Fail -a = A[B]() # Fail -e = A[B]() # type: A[B] -e = A[B]() -[out] -main, line 6: Incompatible types in assignment (expression has type A[B], variable has type A[C]) -main, line 7: Incompatible types in assignment (expression has type A[B], variable has type A[C]) - -[case testNonEmptyGenericTypeConstructor] -from typing import Undefined, typevar, Generic -T = typevar('T') -class B: pass -class C: pass -class A(Generic[T]): - def __init__(self, a: T, b: B) -> None: pass -ac = Undefined # type: A[C] -ab = Undefined # type: A[B] -b = Undefined # type: B -c = Undefined # type: C - -ac = A[C](b, b) # E: Argument 1 to "A" has incompatible type "B"; expected "C" -ab = A[C](c, b) # E: Incompatible types in assignment (expression has type A[C], variable has type A[B]) -ac = A[C](c, b) - -[case testConstructorForGenericTypeWithMultipleArguments] -from typing import Undefined, typevar, Generic -S = typevar('S') -T = typevar('T') -class A(Generic[S, T]): - def __init__(self, t: T, s: S) -> None: pass -class B: pass -class C: pass - -abc = Undefined # type: A[B, C] -acb = Undefined # type: A[C, B] -b, c = Undefined, Undefined # type: (B, C) - -abc = A[B, C](b, b) # E: Argument 1 to "A" has incompatible type "B"; expected "C" -abc = A[B, C](c, c) # E: Argument 2 to "A" has incompatible type "C"; expected "B" -acb = A[B, C](c, b) # E: Incompatible types in assignment (expression has type A[B, C], variable has type A[C, B]) -abc = A[B, C](c, b) -acb = A[C, B](b, c) - -[case testGenericConstructorOfNonGenericType] -from typing import Undefined, typevar -T = typevar('T') -class A: - def __init__(self, a: T, b: T) -> None: pass -class B: pass -class C: pass -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) - -a = A[B](b, c) # E: Argument 2 to "A" has incompatible type "C"; expected "B" -a = A[B](c, b) # E: Argument 1 to "A" has incompatible type "C"; expected "B" -b = A[B](b, b) # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = A[B](b, b) -a = A[C](c, c) - -[case testInvalidTypeApplicationTarget] -from typing import Undefined -a = Undefined # type: A -class A: pass -a[A]() # Fail -A[A]() # Fail -[out] -main, line 4: Value of type "A" is not indexable -main, line 5: Type application targets a non-generic function - -[case testInvalidNumberOfGenericArgsInTypeApplication] -from typing import typevar, Generic -T = typevar('T') -S = typevar('S') -class A(Generic[T, S]): pass -x = A[int, str, int]() # E: Type application has too many types (2 expected) -x = A[int]() # E: Type application has too few types (2 expected) -[out] - - --- Multiple assignment with lists --- ------------------------------ - - -[case testMultipleAsssignmentWithLists] -from typing import Undefined, List -class A: pass -class B: pass -class B2(B): pass -a = Undefined # type: A -b = Undefined # type: B -b2 = Undefined # type: B2 - -a, b = List[A]() # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b, a = List[A]() # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b2, b2 = List[B]() # E: Incompatible types in assignment (expression has type "B", variable has type "B2") - -a, a = List[A]() -b, b2, b = List[B2]() -[builtins fixtures/for.py] - -[case testMultipleAssignmentWithListsInInitialization] -from typing import List -class A: pass -a, b = List[object]() # type: (A, object) # E: Incompatible types in assignment (expression has type "object", variable has type "A") -c, d = List[object]() # type: (object, A) # E: Incompatible types in assignment (expression has type "object", variable has type "A") -e, f = List[A]() # type: (A, object) -[builtins fixtures/for.py] - -[case testMultipleAssignmentWithListAndIndexing] -from typing import Undefined, List -a = Undefined # type: List[A] -b = Undefined # type: List[int] - -a[1], b[1] = a # E: Incompatible types in assignment -a[1], a[2] = a - -class A: pass -[file builtins.py] -from typing import typevar, Generic, Iterable -T = typevar('T') -class object: pass -class list(Iterable[T], Generic[T]): - def __setitem__(self, x: int, v: T) -> None: pass -class int: pass -class type: pass -class tuple: pass -class function: pass - -[case testMultipleAsssignmentWithIterable] -from typing import Iterable, Undefined, typevar -a = Undefined(int) -b = Undefined(str) -T = typevar('T') - -def f(x: T) -> Iterable[T]: pass - -a, b = f(a) # E: Incompatible types in assignment (expression has type "int", variable has type "str") -b, b = f(a) # E: Incompatible types in assignment (expression has type "int", variable has type "str") -a, a = f(a) -b, b = f(b) -[builtins fixtures/for.py] - - --- Error messages --- -------------- - - -[case testErrorWithLongGenericTypeName] -from typing import Undefined, typevar, Generic -B = typevar('B') -C = typevar('C') -D = typevar('D') -E = typevar('E') -F = typevar('F') -G = typevar('G') -class O: pass -a = Undefined # type: A[object, object, object, object, object, object] - -f(a) # E: Argument 1 to "f" has incompatible type A[...]; expected "O" - -def f(a: O) -> None: - pass -class A(Generic[B, C, D, E, F, G]): pass - -[case testErrorWithShorterGenericTypeName] -from typing import Undefined, typevar, Generic -S = typevar('S') -T = typevar('T') -a = Undefined # type: A[object, B] -f(a) # E: Argument 1 to "f" has incompatible type A[object, B]; expected "B" - -def f(a: 'B') -> None: pass -class A(Generic[S, T]): pass -class B: pass - -[case testErrorWithShorterGenericTypeName2] -from typing import Undefined, Function, typevar, Generic -S = typevar('S') -T = typevar('T') -a = Undefined # type: A[object, Function[[], None]] -f(a) # E: Argument 1 to "f" has incompatible type A[...]; expected "B" - -def f(a: 'B') -> None: pass -class A(Generic[S, T]): pass -class B: pass - - --- Overloads + generics --- -------------------- - - -[case testGenericArgumentInOverload] -from typing import Undefined, overload, List, disjointclass -class A: pass -@disjointclass(list) -class B: pass -a, b = Undefined, Undefined # type: (A, B) - -@overload -def f(a: List[A]) -> A: pass -@overload -def f(a: B) -> B: pass - -b = f([a]) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f([b]) # E: List item 1 has incompatible type "B" -a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -a = f([a]) -b = f(b) -[builtins fixtures/list.py] - -[case testGenericFunctionAsOverloadItem] -from typing import Undefined, overload, typevar, List, disjointclass -T = typevar('T') -class A: pass -@disjointclass(list) -class B: pass - -@overload -def f(a: B) -> B: pass -@overload -def f(a: List[T]) -> T: pass - -a, b = Undefined, Undefined # type: (A, B) - -b = f([a]) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f([b]) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -a = f([a]) -b = f([b]) -b = f(b) -[builtins fixtures/list.py] - -[case testOverloadedGenericFunction] -from typing import overload, typevar, Undefined -t = typevar('t') -class A: pass -class B: pass - -@overload -def f(x: int, a: A) -> A: pass -@overload -def f(x: str, a: t) -> t: pass - -a, b = Undefined, Undefined # type: (A, B) -b = f[B]('', b) -a = f[B]('', b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -f[B](1, a) # E: No overload variant of "f" matches argument types - - --- Type variable scoping --- --------------------- - - -[case testLocalTypeVariable] -from typing import typevar -def f() -> None: - T = typevar('T') - def g(x: T) -> T: pass - a = g(1) - a = 1 - a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[out] -main: In function "f": - -[case testClassLevelTypeVariable] -from typing import typevar -class A: - T = typevar('T') - def g(self, x: T) -> T: pass -a = A().g(1) -a = 1 -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - - --- Function subtyping with generic functions --- ----------------------------------------- - - -[case testSubtypingWithGenericFunctionUsingTypevarWithValues] -from typing import typevar, Function -T = typevar('T', values=(int, str)) -def f(x: T) -> T: pass -def g1(f: Function[[str], str]) -> None: pass -g1(f) -def g2(f: Function[[int], int]) -> None: pass -g2(f) -def g3(f: Function[[object], object]) -> None: pass -g3(f) # E: Argument 1 to "g3" has incompatible type Function[[T], T]; \ - expected Function[[object], object] - - --- Special cases --- ------------- - - -[case testIdentityHigherOrderFunction] -from typing import Function, typevar -A = typevar('A') -B = typevar('B') -def square(n: int) -> int: - return n -def id(f: Function[[A], B]) -> Function[[A], B]: - return f -g = id(square) -g(1) -g('x') # E: Argument 1 has incompatible type "str"; expected "int" - - -[case testIdentityHigherOrderFunction2] -from typing import Function, typevar -A = typevar('A') -def voidify(n: int) -> None: pass -def identity(f: Function[[A], None]) -> Function[[A], None]: - return f -identity(voidify)(3) diff --git a/mypy/test/data/check-inference-context.test b/mypy/test/data/check-inference-context.test deleted file mode 100644 index f9c94735cb1c..000000000000 --- a/mypy/test/data/check-inference-context.test +++ /dev/null @@ -1,730 +0,0 @@ - - --- Basic test cases --- ---------------- - - -[case testBasicContextInference] -from typing import Undefined, typevar, Generic -T = typevar('T') -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] -b = Undefined # type: B - -ao = f() -ab = f() -b = f() # E: Incompatible types in assignment (expression has type A[None], variable has type "B") - -def f() -> 'A[T]': pass - -class A(Generic[T]): pass -class B: pass - -[case testBasicContextInferenceForConstructor] -from typing import Undefined, typevar, Generic -T = typevar('T') -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] -b = Undefined # type: B - -ao = A() -ab = A() -b = A() # E: Incompatible types in assignment (expression has type A[None], variable has type "B") - -class A(Generic[T]): pass -class B: pass - -[case testIncompatibleContextInference] -from typing import Undefined, typevar, Generic -T = typevar('T') -b = Undefined # type: B -c = Undefined # type: C -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] -ac = Undefined # type: A[C] - -ac = f(b) # E: Argument 1 to "f" has incompatible type "B"; expected "C" -ab = f(c) # E: Argument 1 to "f" has incompatible type "C"; expected "B" - -ao = f(b) -ab = f(b) -ao = f(c) -ac = f(c) - -def f(a: T) -> 'A[T]': - pass - -class A(Generic[T]): pass - -class B: pass -class C: pass - - --- Local variables --- --------------- - - -[case testInferGenericLocalVariableTypeWithEmptyContext] -from typing import Undefined, typevar, Generic -T = typevar('T') -def g() -> None: - ao = Undefined # type: A[object] - ab = Undefined # type: A[B] - o = Undefined # type: object - b = Undefined # type: B - - x = f(o) - ab = x # E: Incompatible types in assignment (expression has type A[object], variable has type A[B]) - ao = x - y = f(b) - ao = y # E: Incompatible types in assignment (expression has type A[B], variable has type A[object]) - ab = y - -def f(a: T) -> 'A[T]': pass - -class A(Generic[T]): pass -class B: pass -[out] -main: In function "g": - -[case testInferLocalVariableTypeWithUnderspecifiedGenericType] -from typing import typevar, Generic -T = typevar('T') -def g() -> None: - x = f() # E: Need type annotation for variable - -def f() -> 'A[T]': pass -class A(Generic[T]): pass -[out] -main: In function "g": - -[case testInferMultipleLocalVariableTypesWithTupleRvalue] -from typing import Undefined, typevar, Generic -T = typevar('T') -def g() -> None: - ao = Undefined # type: A[object] - ab = Undefined # type: A[B] - b = Undefined # type: B - x, y = f(b), f(b) - ao = x # E: Incompatible types in assignment (expression has type A[B], variable has type A[object]) - ao = y # E: Incompatible types in assignment (expression has type A[B], variable has type A[object]) - ab = x - ab = y - -def f(a: T) -> 'A[T]': pass -class A(Generic[T]): pass -class B: pass -[out] -main: In function "g": - -[case testInferMultipleLocalVariableTypesWithArrayRvalueAndNesting] -from typing import Undefined, typevar, List, Generic -T = typevar('T') -def h() -> None: - ao = Undefined # type: A[object] - ab = Undefined # type: A[B] - b = Undefined # type: B - x, y = g(f(b)) - ao = x # E: Incompatible types in assignment (expression has type A[B], variable has type A[object]) - ao = y # E: Incompatible types in assignment (expression has type A[B], variable has type A[object]) - ab = x - ab = y - -def f(a: T) -> 'A[T]': pass -def g(a: T) -> List[T]: pass - -class A(Generic[T]): pass -class B: pass -[builtins fixtures/for.py] -[out] -main: In function "h": - - --- Return types with multiple tvar instances --- ----------------------------------------- - - -[case testInferenceWithTypeVariableTwiceInReturnType] -from typing import Undefined, typevar, Tuple, Generic -T = typevar('T') -b = Undefined # type: B -o = Undefined # type: object -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] - -ab, ao = f(b) # Fail -ao, ab = f(b) # Fail - -ao, ao = f(b) -ab, ab = f(b) -ao, ao = f(o) - -def f(a: T) -> 'Tuple[A[T], A[T]]': pass - -class A(Generic[T]): pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main, line 8: Incompatible types in assignment (expression has type A[B], variable has type A[object]) -main, line 9: Incompatible types in assignment (expression has type A[B], variable has type A[object]) - -[case testInferenceWithTypeVariableTwiceInReturnTypeAndMultipleVariables] -from typing import Undefined, typevar, Tuple, Generic -S = typevar('S') -T = typevar('T') -b = Undefined # type: B -o = Undefined # type: object -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] - -ao, ao, ab = f(b, b) # Fail -ao, ab, ao = g(b, b) # Fail -ao, ab, ab, ab = h(b, b) # Fail -ab, ab, ao, ab = h(b, b) # Fail - -ao, ab, ab = f(b, b) -ab, ab, ao = g(b, b) -ab, ab, ab, ab = h(b, b) - -def f(a: S, b: T) -> 'Tuple[A[S], A[T], A[T]]': pass -def g(a: S, b: T) -> 'Tuple[A[S], A[S], A[T]]': pass -def h(a: S, b: T) -> 'Tuple[A[S], A[S], A[T], A[T]]': pass - -class A(Generic[T]): pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main, line 9: Incompatible types in assignment (expression has type A[B], variable has type A[object]) -main, line 10: Incompatible types in assignment (expression has type A[B], variable has type A[object]) -main, line 11: Incompatible types in assignment (expression has type A[B], variable has type A[object]) -main, line 12: Incompatible types in assignment (expression has type A[B], variable has type A[object]) - - --- Multiple tvar instances in arguments --- ------------------------------------ - - -[case testMultipleTvatInstancesInArgs] -from typing import Undefined, typevar, Generic -T = typevar('T') -ac = Undefined # type: A[C] -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] -b = Undefined # type: B -c = Undefined # type: C -o = Undefined # type: object - -ab = f(b, o) # E: Argument 2 to "f" has incompatible type "object"; expected "B" -ab = f(o, b) # E: Argument 1 to "f" has incompatible type "object"; expected "B" -ac = f(b, c) # E: Argument 1 to "f" has incompatible type "B"; expected "C" -ac = f(c, b) # E: Argument 2 to "f" has incompatible type "B"; expected "C" - -ao = f(b, c) -ao = f(c, b) -ab = f(c, b) - -def f(a: T, b: T) -> 'A[T]': pass - -class A(Generic[T]): pass -class B: pass -class C(B): pass - - --- Nested generic function calls --- ----------------------------- - - -[case testNestedGenericFunctionCall1] -from typing import Undefined, typevar, Generic -T = typevar('T') -aab = Undefined # type: A[A[B]] -aao = Undefined # type: A[A[object]] -ao = Undefined # type: A[object] -b = Undefined # type: B -o = Undefined # type: object - -aab = f(f(o)) # E: Argument 1 to "f" has incompatible type "object"; expected "B" - -aab = f(f(b)) -aao = f(f(b)) -ao = f(f(b)) - -def f(a: T) -> 'A[T]': pass - -class A(Generic[T]): pass -class B: pass - -[case testNestedGenericFunctionCall2] -from typing import Undefined, typevar, Generic -T = typevar('T') -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] -b = Undefined # type: B -o = Undefined # type: object - -ab = f(g(o)) # E: Argument 1 to "g" has incompatible type "object"; expected "B" - -ab = f(g(b)) -ao = f(g(b)) - -def f(a: T) -> T: pass - -def g(a: T) -> 'A[T]': pass - -class A(Generic[T]): pass -class B: pass - -[case testNestedGenericFunctionCall3] -from typing import Undefined, typevar, Generic -T = typevar('T') -ab = Undefined # type: A[B] -ao = Undefined # type: A[object] -b = Undefined # type: B -o = Undefined # type: object - -ab = f(g(o), g(b)) # E: Argument 1 to "g" has incompatible type "object"; expected "B" -ab = f(g(b), g(o)) # E: Argument 1 to "g" has incompatible type "object"; expected "B" - -ab = f(g(b), g(b)) -ao = f(g(b), g(o)) -ao = f(g(o), g(b)) - -def f(a: T, b: T) -> T: - pass - -def g(a: T) -> 'A[T]': pass - -class A(Generic[T]): pass -class B: pass - - --- Method calls --- ------------ - - -[case testMethodCallWithContextInference] -from typing import Undefined, typevar, Generic -T = typevar('T') -o = Undefined # type: object -b = Undefined # type: B -c = Undefined # type: C -ao = Undefined # type: A[object] -ab = Undefined # type: A[B] -ac = Undefined # type: A[C] - -ab.g(f(o)) # E: Argument 1 to "f" has incompatible type "object"; expected "B" -ac = f(b).g(f(c)) # E: Incompatible types in assignment (expression has type A[B], variable has type A[C]) -ac = f(c).g(f(b)) # E: Argument 1 to "f" has incompatible type "B"; expected "C" - -ab = f(b).g(f(c)) -ab.g(f(c)) - -def f(a: T) -> 'A[T]': pass - -class A(Generic[T]): - def g(self, a: 'A[T]') -> 'A[T]': pass - -class B: pass -class C(B): pass - - --- List expressions --- ---------------- - - -[case testEmptyListExpression] -from typing import Undefined, List -aa = Undefined # type: List[A] -ao = Undefined # type: List[object] -a = Undefined # type: A - -a = [] # E: Incompatible types in assignment (expression has type List[None], variable has type "A") - -aa = [] -ao = [] - -class A: pass -[builtins fixtures/list.py] - -[case testSingleItemListExpressions] -from typing import Undefined, List -aa = Undefined # type: List[A] -ab = Undefined # type: List[B] -ao = Undefined # type: List[object] -a = Undefined # type: A -b = Undefined # type: B - -aa = [b] # E: List item 1 has incompatible type "B" -ab = [a] # E: List item 1 has incompatible type "A" - -aa = [a] -ab = [b] -ao = [a] -aa = [None] -ao = [None] - -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testMultiItemListExpressions] -from typing import Undefined, List -aa = Undefined # type: List[A] -ab = Undefined # type: List[B] -ao = Undefined # type: List[object] -a = Undefined # type: A -b = Undefined # type: B - -ab = [b, a] # E: List item 2 has incompatible type "A" -ab = [a, b] # E: List item 1 has incompatible type "A" - -aa = [a, b, a] -ao = [a, b] - -class A: pass -class B(A): pass -[builtins fixtures/list.py] - -[case testLocalVariableInferenceFromEmptyList] -import typing -def f() -> None: - a = [] # E: Need type annotation for variable - b = [None] # E: Need type annotation for variable - c = [B()] - c = [object()] # E: List item 1 has incompatible type "object" - c = [B()] -class B: pass -[builtins fixtures/list.py] -[out] -main: In function "f": - -[case testNestedListExpressions] -from typing import Undefined, List -aao = Undefined # type: List[List[object]] -aab = Undefined # type: List[List[B]] -ab = Undefined # type: List[B] -b = Undefined # type: B -o = Undefined # type: object - -aao = [[o], ab] # E: List item 2 has incompatible type List[B] -aab = [[], [o]] # E: List item 1 has incompatible type "object" - -aao = [[None], [b], [], [o]] -aab = [[None], [b], []] -aab = [ab, []] - -class B: pass -[builtins fixtures/list.py] - - --- Complex context --- --------------- - - -[case testParenthesesAndContext] -from typing import List -l = ([A()]) # type: List[object] -class A: pass -[builtins fixtures/list.py] - -[case testComplexTypeInferenceWithTuple] -from typing import typevar, Tuple, Generic -k = typevar('k') -t = typevar('t') -v = typevar('v') -def f(x: Tuple[k]) -> 'A[k]': pass - -d = f((A(),)) # type: A[A[B]] - -class A(Generic[t]): pass -class B: pass -class C: pass -class D(Generic[k, v]): pass -[builtins fixtures/list.py] - - --- Dictionary literals --- ------------------- - - -[case testDictionaryLiteralInContext] -from typing import Dict, typevar, Generic -t = typevar('t') -class A(Generic[t]): pass -class B: pass -class C: pass -d = {A() : A[C](), - A[B]() : A()} # type: Dict[A[B], A[C]] -[builtins fixtures/dict.py] - - --- Special cases (regression tests etc.) --- ------------------------------------- - - -[case testInitializationWithInferredGenericType] -from typing import typevar, Generic -T = typevar('T') -c = f(A()) # type: C[A] # E: Argument 1 to "f" has incompatible type "A"; expected C[A] - -def f(x: T) -> T: pass -class C(Generic[T]): pass -class A: pass - -[case testInferredGenericTypeAsReturnValue] -from typing import typevar, Generic -T = typevar('T') -def t() -> 'A[B]': - return f(D()) # E: Argument 1 to "f" has incompatible type "D"; expected "B" - return A() - return f(C()) - -def f(a: T) -> 'A[T]': pass -class A(Generic[T]): pass -class B: pass -class C(B): pass -class D: pass -[out] -main: In function "t": - -[case testIntersectionWithInferredGenericArgument] -from typing import overload, typevar, Generic -T = typevar('T') -f(A()) - -@overload -def f(x: 'A[B]') -> None: pass -@overload -def f(x: 'B') -> None: pass -class A(Generic[T]): pass -class B: pass - -[case testInferenceWithAbstractClassContext] -from typing import typevar, AbstractGeneric -from abc import abstractmethod, ABCMeta -t = typevar('t') -x = A() # type: I[int] -y = A[object]() # type: I[int] # E: Incompatible types in assignment (expression has type A[object], variable has type I[int]) - -class I(AbstractGeneric[t]): - @abstractmethod - def f(self): pass -class A(I[t], AbstractGeneric[t]): - def f(self): pass - -[case testInferenceWithAbstractClassContext2] -from typing import typevar, AbstractGeneric -from abc import abstractmethod, ABCMeta -t = typevar('t') -a = f(A()) # type: A[int] -aa = f(A[int]()) -class I(AbstractGeneric[t]): pass -class A(I[t], AbstractGeneric[t]): pass -def f(i: I[t]) -> A[t]: pass - -[case testInferenceWithAbstractClassContext3] -from typing import typevar, Generic, Iterable -t = typevar('t') -class set(Generic[t]): - def __init__(self, iterable: Iterable[t]) -> None: pass -b = bool() -l = set([b]) -l = set[bool]([]) -l = set[object]([]) # E: Incompatible types in assignment (expression has type set[object], variable has type set[bool]) -[builtins fixtures/for.py] - -[case testInferenceWithAbstractClassContext4] -from typing import typevar, Generic, Iterable -t = typevar('t') -class set(Generic[t]): pass -def f(iterable: Iterable[t]) -> set[t]: pass -b = bool() -l = f([b]) -l = f[bool]([]) -l = f[object]([]) # E: Incompatible types in assignment (expression has type set[object], variable has type set[bool]) -[builtins fixtures/for.py] - - --- Infer generic type in 'any' context --- ----------------------------------- - - -[case testInferGenericTypeInAnyContext] -from typing import Any, typevar, Generic -s = typevar('s') -t = typevar('t') -x = [] # type: Any -y = C() # type: Any -class C(Generic[s, t]): pass -[builtins fixtures/list.py] - - --- Lambdas --- ------- - - -[case testInferLambdaArgumentTypeUsingContext] -from typing import Undefined, Function -f = Undefined # type: Function[[B], A] -f = lambda x: x.o -f = lambda x: x.x # E: "B" has no attribute "x" -class A: pass -class B: - o = Undefined # type: A - -[case testInferLambdaReturnTypeUsingContext] -from typing import Undefined, List, Function -f = Undefined # type: Function[[], List[A]] -f = lambda: [] -f = lambda: [B()] # E: List item 1 has incompatible type "B" -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testInferLambdaAsGenericFunctionArgument] -from typing import typevar, List, Any, Function, Undefined -t = typevar('t') -class A: - x = Undefined # type: A -def f(a: List[t], fn: Function[[t], Any]) -> None: pass -f(List[A](), lambda a: a.x) -[builtins fixtures/list.py] - -[case testInvalidContextForLambda] -from typing import Function -f = lambda x: A() # type: Function[[], A] -f2 = lambda: A() # type: Function[[A], A] -class A: pass -[out] -main, line 2: Cannot infer type of lambda -main, line 3: Cannot infer type of lambda -main, line 3: Incompatible types in assignment (expression has type Function[[], A], variable has type Function[[A], A]) - - --- Overloads + generic functions --- ----------------------------- - - -[case testMapWithOverloadedFunc] -from typing import typevar, Function, List, overload, Any -t = typevar('t') -s = typevar('s') -def map(f: Function[[t], s], seq: List[t]) -> List[s]: pass - -@overload -def g(o: object) -> 'B': pass -@overload -def g(o: 'A', x: Any = None) -> 'B': pass - -class A: pass -class B: pass - -m = map(g, [A()]) -b = m # type: List[B] -a = m # type: List[A] # E: Incompatible types in assignment (expression has type List[B], variable has type List[A]) -[builtins fixtures/list.py] - - --- Boolean operators --- ----------------- - - -[case testOrOperationInferredFromContext] -from typing import Undefined, List -a, b, c = Undefined, Undefined, Undefined # type: (List[A], List[B], List[C]) -a = a or [] -a = [] or a -b = b or [C()] -a = a or b # E: Incompatible types in assignment (expression has type "object", variable has type List[A]) -b = b or c # E: Incompatible types in assignment (expression has type "object", variable has type List[B]) - -class A: pass -class B: pass -class C(B): pass -[builtins fixtures/list.py] - - --- Special cases --- ------------- - - -[case testSomeTypeVarsInferredFromContext] -from typing import Undefined, List, typevar -t = typevar('t') -s = typevar('s') -# Some type variables can be inferred using context, but not all of them. -a = Undefined # type: List[A] -a = f(A(), B()) -a = f(B(), B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -def f(a: s, b: t) -> List[s]: pass -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testSomeTypeVarsInferredFromContext2] -from typing import Undefined, List, typevar -s = typevar('s') -t = typevar('t') -# Like testSomeTypeVarsInferredFromContext, but tvars in different order. -a = Undefined # type: List[A] -a = f(A(), B()) -a = f(B(), B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -def f(a: s, b: t) -> List[s]: pass -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testLambdaInListAndHigherOrderFunction] -from typing import typevar, Function, List -t = typevar('t') -s = typevar('s') -map( - [lambda x: x], []) -def map(f: List[Function[[t], s]], a: List[t]) -> List[s]: pass -class A: pass -[builtins fixtures/list.py] -[out] - -[case testChainedAssignmentInferenceContexts] -from typing import Undefined, List -i = Undefined # type: List[int] -s = Undefined # type: List[str] -i = i = [] -i = s = [] # E: Incompatible types in assignment (expression has type List[str], variable has type List[int]) -[builtins fixtures/list.py] - -[case testContextForAttributeDeclaredInInit] -from typing import List -class A: - def __init__(self): - self.x = [] # type: List[int] -a = A() -a.x = [] -a.x = [1] -a.x = [''] # E: List item 1 has incompatible type "str" -[builtins fixtures/list.py] - -[case testListMultiplyInContext] -from typing import Undefined, List -a = Undefined(List[int]) -a = [None] * 3 -a = [''] * 3 # E: List item 1 has incompatible type "str" -[builtins fixtures/list.py] - -[case testUnionTypeContext] -from typing import Undefined, Union, List, typevar -T = typevar('T') -def f(x: Union[List[T], str]) -> None: pass -f([1]) -f('') -f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "Union[List[None], str]" -[builtins fixtures/isinstancelist.py] - -[case testIgnoringInferenceContext] -from typing import typevar, List -T = typevar('T') -def f(x: List[T]) -> T: pass -def g(y: object) -> None: pass -a = [1] -g(f(a)) -[builtins fixtures/list.py] diff --git a/mypy/test/data/check-inference.test b/mypy/test/data/check-inference.test deleted file mode 100644 index 1bc320f155fa..000000000000 --- a/mypy/test/data/check-inference.test +++ /dev/null @@ -1,1005 +0,0 @@ --- Inferring locals/globals with simple types --- ------------------------------------------ - - -[case testInferSimpleGvarType] -import typing -x = A() -y = B() -x = B() # Fail -x = A() -x = y # Fail -x = x -class A: pass -class B: pass -[out] -main, line 4: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 6: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testInferSimpleLvarType] -import typing -def f() -> None: - x = A() - y = B() - x = B() # Fail - x = A() - x = y # Fail - x = x -class A: pass -class B: pass -[out] -main: In function "f": -main, line 5: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 7: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testLvarInitializedToNoneWithoutType] -import typing -def f() -> None: - a = None # E: Need type annotation for variable - a.x() -[out] -main: In function "f": - -[case testLvarInitializedToVoid] -import typing -def f() -> None: - a = g() # E: "g" does not return a value - #b, c = g() # "g" does not return a value TODO - -def g() -> None: pass -[out] -main: In function "f": - -[case testInferringLvarTypeFromArgument] -import typing -def f(a: 'A') -> None: - b = a - b = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - b = a - a = b - -class A: pass -class B: pass -[out] -main: In function "f": - -[case testInferringLvarTypeFromGvar] -from typing import Undefined -g = Undefined # type: B - -def f() -> None: - a = g - a = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - a = B() - -class A: pass -class B: pass -[out] -main: In function "f": - -[case testInferringImplicitDynamicTypeForLvar] -import typing -def f() -> None: - a = g() - None(a) # E: None not callable - a.x() - -def g(): pass -[out] -main: In function "f": - -[case testInferringExplicitDynamicTypeForLvar] -from typing import Undefined, Any -g = Undefined # type: Any - -def f(a: Any) -> None: - b = g - None(b) # E: None not callable - a.x() -[out] -main: In function "f": - - --- Inferring types of local variables with complex types --- ----------------------------------------------------- - - -[case testInferringTupleTypeForLvar] -from typing import Undefined -def f() -> None: - a = A(), B() - aa = Undefined # type: A - bb = Undefined # type: B - bb = a[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - aa = a[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") - aa = a[0] - bb = a[1] - -class A: pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main: In function "f": - -[case testInferringTupleTypeForLvarWithNones] -import typing -def f() -> None: - a = A(), None # E: Need type annotation for variable - b = None, A() # E: Need type annotation for variable - -class A: pass -[builtins fixtures/tuple.py] -[out] -main: In function "f": - -[case testInferringGenericTypeForLvar] -from typing import Undefined, typevar, Generic -T = typevar('T') -class A(Generic[T]): pass -a_i = Undefined # type: A[int] -a_s = Undefined # type: A[str] - -def f() -> None: - a = A[int]() - a = a_s # E: Incompatible types in assignment (expression has type A[str], variable has type A[int]) - a = a_i -[builtins fixtures/tuple.py] -[out] -main: In function "f": - -[case testInferringFunctionTypeForLvar] -import typing -def f() -> None: - a = g - a(B()) # E: Argument 1 has incompatible type "B"; expected "A" - a(A()) - -def g(a: 'A') -> None: pass - -class A: pass -class B: pass -[out] -main: In function "f": - -[case testInferringFunctionTypeForLvarFromTypeObject] -import typing -def f() -> None: - a = A - a(A()) # E: Too many arguments - a() - t = a # type: type - -class A: pass -[out] -main: In function "f": - - --- Inferring variable types in multiple definition --- ----------------------------------------------- - - -[case testInferringLvarTypesInMultiDef] -import typing -def f() -> None: - a, b = A(), B() - a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - - a = A() - b = B() - -class A: pass -class B: pass -[out] -main: In function "f": - -[case testInferringLvarTypesInTupleAssignment] -from typing import Undefined, Tuple -def f() -> None: - t = Undefined # type: Tuple[A, B] - a, b = t - a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - - a = A() - b = B() - -class A: pass -class B: pass -[out] -main: In function "f": - -[case testInferringLvarTypesInNestedTupleAssignment1] -from typing import Undefined, Tuple -def f() -> None: - t = Undefined # type: Tuple[A, B] - a1, (a, b) = A(), t - a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - - a = A() - b = B() - -class A: pass -class B: pass -[out] -main: In function "f": - -[case testInferringLvarTypesInNestedTupleAssignment2] -import typing -def f() -> None: - a, (b, c) = A(), (B(), C()) - a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - c = A() # E: Incompatible types in assignment (expression has type "A", variable has type "C") - - a = A() - b = B() - c = C() - -class A: pass -class B: pass -class C: pass -[out] -main: In function "f": - -[case testInferringLvarTypesInNestedListAssignment] -import typing -def f() -> None: - a, (b, c) = A(), [B(), C()] - a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - c = A() # E: Incompatible types in assignment (expression has type "A", variable has type "C") - - a = A() - b = B() - c = C() - -class A: pass -class B: pass -class C: pass -[out] -main: In function "f": - -[case testInferringLvarTypesInMultiDefWithNoneTypes] -import typing -def f() -> None: - a, b = A(), None # E: Need type annotation for variable - c, d = None, A() # E: Need type annotation for variable - -class A: pass -[out] -main: In function "f": - -[case testInferringLvarTypesInNestedTupleAssignmentWithNoneTypes] -import typing -def f() -> None: - a1, (a2, b) = A(), (A(), None) # E: Need type annotation for variable - -class A: pass -[out] -main: In function "f": - -[case testInferringLvarTypesInMultiDefWithInvalidTuple] -from typing import Undefined, Tuple -t = Undefined # type: Tuple[object, object, object] - -def f() -> None: - a, b = t # Fail - c, d, e, f = t # Fail - g, h, i = t -[builtins fixtures/tuple.py] -[out] -main: In function "f": -main, line 5: Too many values to unpack (2 expected, 3 provided) -main, line 6: Need more than 3 values to unpack (4 expected) - -[case testInvalidRvalueTypeInInferredMultipleLvarDefinition] -import typing -def f() -> None: - a, b = f # E: 'def ()' object is not iterable - c, d = A() # E: '__main__.A' object is not iterable -class A: pass -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testInvalidRvalueTypeInInferredNestedTupleAssignment] -import typing -def f() -> None: - a1, (a2, b) = A(), f # E: 'def ()' object is not iterable - a3, (c, d) = A(), A() # E: '__main__.A' object is not iterable -class A: pass -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testInferringMultipleLvarDefinitionWithListRvalue] -from typing import List - -class C: pass -class D: pass - -def f() -> None: - a, b = List[C]() - c, d, e = List[D]() - a = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") - b = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") - c = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") - b = c # E: Incompatible types in assignment (expression has type "D", variable has type "C") - - a = C() - b = C() - c = D() - d = D() - e = D() - - a = b - c = d - d = e -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testInferringNestedTupleAssignmentWithListRvalue] -from typing import List - -class C: pass -class D: pass - -def f() -> None: - c1, (a, b) = C(), List[C]() - c2, (c, d, e) = C(), List[D]() - a = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") - b = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") - c = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") - b = c # E: Incompatible types in assignment (expression has type "D", variable has type "C") - - a = C() - b = C() - c = D() - d = D() - e = D() - - a = b - c = d - d = e -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testInferringMultipleLvarDefinitionWithImplicitDynamicRvalue] -import typing -def f() -> None: - a, b = g() - a.x - b.x -def g(): pass - -[case testInferringMultipleLvarDefinitionWithExplicitDynamicRvalue] -from typing import Any -def f(d: Any) -> None: - a, b = d - a.x - b.x - -[case testInferringTypesFromIterable] -from typing import Iterable -class Nums(Iterable[int]): - def __iter__(self): pass - def __next__(self): pass -a, b = Nums() -a = b = 1 -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/for.py] - - --- Type variable inference for generic functions --- --------------------------------------------- - - -[case testInferSimpleGenericFunction] -from typing import Undefined, Tuple, typevar -T = typevar('T') -a = Undefined # type: A -b = Undefined # type: B -c = Undefined # type: Tuple[A, object] - -b = id(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = id(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = id(c) # E: Incompatible types in assignment (expression has type "Tuple[A, object]", variable has type "A") - -a = id(a) -b = id(b) -c = id(c) - -def id(a: T) -> T: pass - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testInferringGenericFunctionTypeForLvar] -from typing import Undefined, typevar -T = typevar('T') -def f() -> None: - a = id - b = Undefined # type: int - c = Undefined # type: str - b = a(c) # E: Incompatible types in assignment (expression has type "str", variable has type "int") - b = a(b) - c = a(c) -def id(x: T) -> T: - return x -[out] -main: In function "f": - -[case testUnderspecifiedInferenceResult] -from typing import Undefined, typevar -T = typevar('T') -class A: pass -a = Undefined # type: A - -def ff() -> None: - x = f() # E: Need type annotation for variable - -g(None) # Ok -f() # Ok because not used to infer local variable type -f[A]() -g(a) - -def f() -> T: pass -def g(a: T) -> None: pass -[out] -main: In function "ff": - -[case testUnsolvableInferenceResult] -from typing import typevar -T = typevar('T') -f(A(), g()) # Fail -f(A(), A()) - -def f(a: T, b: T) -> None: pass -def g() -> None: pass -class A: pass -[out] -main, line 3: Cannot infer type argument 1 of "f" -main, line 3: "g" does not return a value - -[case testInferenceWithMultipleConstraints] -from typing import Undefined, typevar -T = typevar('T') -a = Undefined # type: A -b = Undefined # type: B - -b = f(a, b) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b = f(b, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f(a, b) -a = f(b, a) - -def f(a: T, b: T) -> T: pass - -class A: pass -class B(A): pass - -[case testInferenceWithMultipleVariables] -from typing import Undefined, Tuple, typevar -T = typevar('T') -S = typevar('S') -a, b = Undefined, Undefined # type: (A, B) -taa = Undefined # type: Tuple[A, A] -tab = Undefined # type: Tuple[A, B] -tba = Undefined # type: Tuple[B, A] - -taa = f(a, b) # Fail -taa = f(b, a) # Fail -tba = f(a, b) # Fail - -tab = f(a, b) -tba = f(b, a) - -def f(a: T, b: S) -> Tuple[T, S]: pass - -class A: pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main, line 9: Argument 2 to "f" has incompatible type "B"; expected "A" -main, line 10: Argument 1 to "f" has incompatible type "B"; expected "A" -main, line 11: Argument 1 to "f" has incompatible type "A"; expected "B" -main, line 11: Argument 2 to "f" has incompatible type "B"; expected "A" - -[case testConstraintSolvingWithSimpleGenerics] -from typing import Undefined, typevar, Generic -T = typevar('T') -ao = Undefined # type: A[object] -ab = Undefined # type: A[B] -ac = Undefined # type: A[C] - -ab = f(ao) # E: Argument 1 to "f" has incompatible type A[object]; expected A[B] -ao = f(ab) # E: Argument 1 to "f" has incompatible type A[B]; expected A[object] -ab = f(ac) # E: Argument 1 to "f" has incompatible type A[C]; expected A[B] -ab = g(ao) # E: Argument 1 to "g" has incompatible type A[object]; expected A[B] -ao = g(ab) # E: Argument 1 to "g" has incompatible type A[B]; expected A[object] - -ab = f(ab) -ac = f(ac) -ao = f(ao) - -ab = g(ab) -ao = g(ao) - -def f(a: 'A[T]') -> 'A[T]': pass - -def g(a: T) -> T: pass - -class A(Generic[T]): pass -class B: pass -class C: pass - -[case testConstraintSolvingFailureWithSimpleGenerics] -from typing import Undefined, typevar, Generic -T = typevar('T') -ao = Undefined # type: A[object] -ab = Undefined # type: A[B] - -f(ao, ab) # E: Cannot infer type argument 1 of "f" -f(ab, ao) # E: Cannot infer type argument 1 of "f" -f(ao, ao) -f(ab, ab) - -def f(a: 'A[T]', b: 'A[T]') -> None: pass - -class A(Generic[T]): pass -class B: pass - -[case testTypeInferenceWithCalleeDefaultArgs] -from typing import Undefined, typevar -T = typevar('T') -a = Undefined # type: A -o = Undefined # type: object - -a = f(o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") -a = g(a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") - -o = f() -o = f(o) -a = f(a) -a = g(a) - -def f(a: T = None) -> T: pass -def g(a: T, b: T = None) -> T: pass - -class A: pass - - --- Generic function inference with multiple inheritance --- ---------------------------------------------------- - - -[case testGenericFunctionInferenceWithMultipleInheritance] -from typing import typevar - -class I: pass -class J: pass - -class A(I, J): pass -class B(I, J): pass -class C(I): pass -class D(J): pass - -T = typevar('T') -def f(a: T, b: T) -> T: pass -def g(x: I) -> None: pass - -a = f(A(), C()) -g(a) -b = f(A(), B()) -g(b) -c = f(A(), D()) -g(c) # E: Argument 1 to "g" has incompatible type "object"; expected "I" -d = f(D(), A()) -g(d) # E: Argument 1 to "g" has incompatible type "object"; expected "I" -e = f(D(), C()) -g(e) # E: Argument 1 to "g" has incompatible type "object"; expected "I" - -[case testGenericFunctionInferenceWithMultipleInheritance2] -from typing import typevar - -class I: pass -class J: pass - -class A(I): pass -class B(A, J): pass -class C(I, J): pass - -T = typevar('T') -def f(a: T, b: T) -> T: pass -def g(x: I) -> None: pass -def h(x: J) -> None: pass - -a = f(B(), C()) -g(a) -h(a) # E: Argument 1 to "h" has incompatible type "I"; expected "J" -b = f(C(), B()) -g(b) -h(b) # E: Argument 1 to "h" has incompatible type "I"; expected "J" -c = f(A(), B()) -g(a) -h(b) # E: Argument 1 to "h" has incompatible type "I"; expected "J" - -[case testGenericFunctionInferenceWithMultipleInheritance3] -from typing import typevar - -class I: pass -class J: pass -class K(J): pass - -class A(K): pass -class B(A, I): pass -class C(I, J): pass - -T = typevar('T') -def f(a: T, b: T) -> T: pass -def g(x: K) -> None: pass - -a = f(B(), C()) -g(a) # E: Argument 1 to "g" has incompatible type "object"; expected "K" -b = f(A(), C()) -g(b) # E: Argument 1 to "g" has incompatible type "object"; expected "K" -c = f(A(), B()) -g(c) - -[case testPrecedenceOfFirstBaseAsInferenceResult] -from typing import Undefined, typevar -from abc import abstractmethod, ABCMeta -T = typevar('T') -a, i, j = Undefined, Undefined, Undefined # type: (A, I, J) - -a = f(B(), C()) - -class I(metaclass=ABCMeta): pass -class J(metaclass=ABCMeta): pass - -def f(a: T, b: T) -> T: pass - -class A: pass -class B(A, I, J): pass -class C(A, I, J): pass - - --- Generic function inference with function arguments --- -------------------------------------------------- - - -[case testNonOverloadedMapInference] -from typing import typevar, Function, List -t = typevar('t') -s = typevar('s') -class A: pass -b = bool() -def f(x: bool) -> A: pass -def mymap(f: Function[[t], s], a: List[t]) -> List[s]: pass - -l = mymap(f, [b]) -l = List[A]() -l = List[bool]() # E: Incompatible types in assignment (expression has type List[bool], variable has type List[A]) -[builtins fixtures/for.py] - - --- Literal expressions --- ------------------- - - -[case testDictLiteral] -from typing import Undefined, Dict -class A: pass -class B: pass -a, b = Undefined, Undefined # type: (A, B) -d = {a:b} -d = Dict[A, B]() -d = Dict[A, A]() # E: Incompatible types in assignment (expression has type Dict[A, A], variable has type Dict[A, B]) -d = Dict[B, B]() # E: Incompatible types in assignment (expression has type Dict[B, B], variable has type Dict[A, B]) -[builtins fixtures/dict.py] - -[case testSetLiteral] -from typing import Undefined, Any, Set -a, x = Undefined, Undefined # type: (int, Any) -s = {a} -s = {x} -s = Set[int]() -s = Set[str]() # E: Incompatible types in assignment (expression has type Set[str], variable has type Set[int]) -[builtins fixtures/set.py] - - --- For statements --- -------------- - - -[case testInferenceOfFor1] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) - -for x in [A()]: - b = x # E: Incompatible types in assignment (expression has type "A", variable has type "B") - a = x - -for y in []: # E: Need type annotation for variable - a = y # E: Cannot determine type of 'y' - -class A: pass -class B: pass -[builtins fixtures/for.py] - -[case testInferenceOfFor2] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -for x, (y, z) in [(A(), (B(), C()))]: - b = x # Fail - c = y # Fail - a = z # Fail - a = x - b = y - c = z -for xx, yy, zz in [(A(), B())]: # Fail - pass -for xx, (yy, zz) in [(A(), B())]: # Fail - pass -for xxx, yyy in [(None, None)]: # Fail - pass - -class A: pass -class B: pass -class C: pass -[builtins fixtures/for.py] -[out] -main, line 4: Incompatible types in assignment (expression has type "A", variable has type "B") -main, line 5: Incompatible types in assignment (expression has type "B", variable has type "C") -main, line 6: Incompatible types in assignment (expression has type "C", variable has type "A") -main, line 10: Need more than 2 values to unpack (3 expected) -main, line 12: '__main__.B' object is not iterable -main, line 14: Need type annotation for variable - -[case testInferenceOfFor3] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) - -for x, y in [[A()]]: - b = x # E: Incompatible types in assignment (expression has type "A", variable has type "B") - b = y # E: Incompatible types in assignment (expression has type "A", variable has type "B") - a = x - a = y - -for e, f in [[]]: # E: Need type annotation for variable - pass - -class A: pass -class B: pass -[builtins fixtures/for.py] - -[case testForStatementInferenceWithVoid] -import typing -for x in f(): # E: "f" does not return a value - pass -def f() -> None: pass -[builtins fixtures/for.py] - -[case testReusingInferredForIndex] -import typing -for a in [A()]: pass -a = A() -a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -for a in []: pass -a = A() -a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -class A: pass -class B: pass -[builtins fixtures/for.py] - -[case testReusingInferredForIndex2] -import typing -def f() -> None: - for a in [A()]: pass - a = A() - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - for a in []: pass - a = A() - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -class A: pass -class B: pass -[builtins fixtures/for.py] -[out] -main: In function "f": - - --- Regression tests --- ---------------- - - -[case testMultipleAssignmentWithPartialDefinition] -from typing import Undefined -a = Undefined # type: A -x, a = a, a -x = a -a = x -x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -class A: pass - -[case testMultipleAssignmentWithPartialDefinition2] -from typing import Undefined -a = Undefined # type: A -a, x = [a, a] -x = a -a = x -x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -class A: pass -[builtins fixtures/for.py] - -[case testMultipleAssignmentWithPartialDefinition3] -from typing import Undefined, Any -a = Undefined # type: A -x, a = Any(a) -x = a -a = x -x = object() -a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -class A: pass - -[case testInferGlobalDefinedInBlock] -import typing -if A: - a = A() - a = A() - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -class A: pass -class B: pass - - --- Inferring attribute types --- ------------------------- - - -[case testInferAttributeType] -import typing -class A: - a = B() -class B: pass - -A().a = B() -A().a = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testInferAttributeTypeAndAssignInInit] -import typing -class A: - a = B() - def __init__(self) -> None: - self.a = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - self.a = B() -class B: pass -[out] -main: In member "__init__" of class "A": - -[case testInferAttributeInInit] -import typing -class B: pass -class A: - def __init__(self) -> None: - self.a = A() - self.b = B() -a = A() -a.a = A() -a.b = B() -a.a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a.b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testInferAttributeInInitUsingChainedAssignment] -import typing -class B: pass -class A: - def __init__(self) -> None: - self.a = self.b = A() -a = A() -a.a = A() -a.b = A() -a.a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a.b = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - - --- Lambdas --- ------- - - -[case testInferLambdaType] -from typing import List, Function, Undefined -l = lambda: List[int]() -f1 = l # type: Function[[], List[int]] -f2 = l # type: Function[[], List[str]] # E: Incompatible types in assignment (expression has type Function[[], List[int]], variable has type Function[[], List[str]]) -[builtins fixtures/list.py] - -[case testInferLambdaType2] -from typing import List, Function -l = lambda: [B()] -f1 = l # type: Function[[], List[B]] -f2 = l # type: Function[[], List[A]] # E: Incompatible types in assignment (expression has type Function[[], List[B]], variable has type Function[[], List[A]]) - -class A: pass -class B: pass -[builtins fixtures/list.py] - - --- Boolean operators --- ----------------- - - -[case testOrOperationWithGenericOperands] -from typing import Undefined, List -a = Undefined # type: List[A] -o = Undefined # type: List[object] -a2 = a or [] -a = a2 -a2 = o # E: Incompatible types in assignment (expression has type List[object], variable has type List[A]) -class A: pass -[builtins fixtures/list.py] - - --- Accessing variable before its type has been inferred --- ---------------------------------------------------- - - -[case testAccessGlobalVarBeforeItsTypeIsAvailable] -import typing -x.y # E: Cannot determine type of 'x' -x = object() -x.y # E: "object" has no attribute "y" - -[case testAccessDataAttributeBeforeItsTypeIsAvailable] -from typing import Undefined -a = Undefined # type: A -a.x.y # E: Cannot determine type of 'x' -class A: - def __init__(self) -> None: - self.x = object() -a.x.y # E: "object" has no attribute "y" - - --- Ducktype declarations --- --------------------- - - -[case testListWithDucktypeCompatibility] -from typing import List, Undefined, ducktype -class A: pass -@ducktype(A) -class B: pass -a = Undefined(List[A]) -x1 = [A(), B()] -x2 = [B(), A()] -x3 = [B(), B()] -a = x1 -a = x2 -a = x3 # E: Incompatible types in assignment (expression has type List[B], variable has type List[A]) -[builtins fixtures/list.py] - -[case testListWithDucktypeCompatibilityAndTransitivity] -from typing import List, Undefined, ducktype -class A: pass -@ducktype(A) -class B: pass -@ducktype(B) -class C: pass -a = Undefined(List[A]) -x1 = [A(), C()] -x2 = [C(), A()] -x3 = [B(), C()] -a = x1 -a = x2 -a = x3 # E: Incompatible types in assignment (expression has type List[B], variable has type List[A]) -[builtins fixtures/list.py] diff --git a/mypy/test/data/check-isinstance.test b/mypy/test/data/check-isinstance.test deleted file mode 100644 index 17fe205aeaa9..000000000000 --- a/mypy/test/data/check-isinstance.test +++ /dev/null @@ -1,685 +0,0 @@ -[case testForcedAssignment] -x = 1 # type: object -y = 1 -y = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") -x = 2 -y = x -[case testJoinAny] -from typing import List, Any - -x = None # type: List[Any] - -def foo() -> List[int]: pass -def bar() -> List[str]: pass - -if 1: - x = foo() -else: - x = bar() - -x * 2 -[builtins fixtures/list.py] - -[case testGeneratorExpressionTypes] - -class A: y = 1 -x = [A()] -y = [x] - -z = [1,2] -z = [a.y for b in y for a in b] - -[builtins fixtures/list.py] - -[case testClassAttributeInitialization-skip] -class A: - x = None # type: int - def __init__(self) -> None: - self.y = None # type: int - z = self.x - w = self.y -[case testAssignmentSubtypes-skip] -from typing import Union - -def foo(x: Union[str, int]): - if isinstance(x, int): - x = 'a' - x + 'a' # Works in the current code - z = x # We probably want this to be of type str. - y = [x] # But what type should this be? - y[0] + 'a' # (1) Should this work? - y + [1] # (2) Or this? - z = 1 # Also, is this valid? - -x = None # type: int -y = [x] -[builtins fixtures/isinstancelist.py] -[out] -main: In function "foo": - -[case testFunctionDefaultArgs] - -class A: pass -class B(A): y = 1 - -x = A() - -def foo(x: A = B()): - x.y # E: "A" has no attribute "y" -[builtins fixtures/isinstance.py] -[out] -main: In function "foo": - -[case testIsinstanceFancyConditionals] - -class A: pass -class B(A): y = 1 - -x = A() - -if isinstance(x, B): - x.y - -while isinstance(x, B): - x.y - -while isinstance(x, B): - x.y - x = B() - -[builtins fixtures/isinstance.py] - - -[case testSubtypingWithAny] - -class A: y = 1 -class B(A): z = 1 - -def foo(): pass - -x = A() -x = B() -x.z -x = foo() -x.z # E: "A" has no attribute "z" -x.y - -[case testSingleMultiAssignment-skip] - -x = 'a' -(x, ) = ('a',) - -[case testUnionMultiAssignment] -from typing import Union, Undefined - -x = Undefined # type: Union[int, str] -x = 1 -x = 'a' -x + 1 # E: Unsupported operand types for + ("str" and "int") -x = 1 -(x, y) = ('a', 1) -x + 1 # E: Unsupported operand types for + ("str" and "int") - -[builtins fixtures/isinstancelist.py] - -[case testUnionIfZigzag] -from typing import Union - -def f(x: Union[int, str]) -> None: - x = 1 - if x: - x = 'a' - x = 1 - x + 1 -[builtins fixtures/isinstancelist.py] - - -[case testTwoLoopsUnion] -from typing import Union - -def foo() -> Union[int, str]: pass - -def bar() -> None: - x = foo() - if isinstance(x, int): - return - while 1: - x + 'a' - while 1: - x = foo() - if 1: - return - x = 'a' - x + 'a' - -[builtins fixtures/isinstancelist.py] - -[case testComplicatedBlocks] -from typing import Union - -def foo() -> Union[int, str]: pass - -def bar() -> None: - x = foo() - if isinstance(x, int): - return - while 1: - x + 'a' - while 1: - x = foo() - if 1: - return - x = 'a' - x + 'a' - - x = foo() - if isinstance(x, int): - return - while 1: - x + 'a' - while 1: - x + 'a' # E: Unsupported operand types for + (likely involving Union) - x = foo() - if 1: - continue - x = 'a' - x = 'a' - x + 'a' -[builtins fixtures/isinstancelist.py] -[out] -main: In function "bar": - -[case testUnionTryExcept] - -class A: y = A() -class B(A): z = 1 - -x = A() -x = B() -x.z -try: - x.z - x = A() - x = B() - x.z -except: - pass -x.z # E: "A" has no attribute "z" - - -[case testUnionTryExcept2] - -class A: y = A() -class B(A): z = 1 - -x = A() -try: - x.z # E: "A" has no attribute "z" - x = A() - x = B() - x.z -except: - x.z # E: "A" has no attribute "z" - x = B() - x.z -else: - x = B() -x.z - - -[case testUnionListIsinstance] - -from typing import Union, List - -class A: y = A() -class B(A): z = C() - -class C: pass -class D(C): d = 1 - - -def f(x: Union[List[int], List[str], int]) -> None: - if isinstance(x, list): - a = x[0] - if isinstance(a, int): - a + 1 - a + 'x' # E: Unsupported operand types for + ("int" and "str") - - # type of a? - x + 1 # E: Unsupported operand types for + (likely involving Union) - else: - x[0] # E: Value of type "int" is not indexable - x + 1 - x[0] # E: Value of type "int" is not indexable - x + 1 # E: Unsupported operand types for + (likely involving Union) -[builtins fixtures/isinstancelist.py] -[out] -main: In function "f": - -[case testUnionStrictDefnBasic] -from typing import Union, Undefined - -def foo() -> Union[int, str]: pass - -x = foo() -x = 1 -x = x + 1 -x = foo() -x = x + 1 # E: Unsupported operand types for + (likely involving Union) -if isinstance(x, str): - x = x + 1 # E: Unsupported operand types for + ("str" and "int") - x = 1 - x = x + 1 - -[builtins fixtures/isinstancelist.py] - -[case testSubtypeRedefinitionBasic] -from typing import Union, Undefined - -class A: pass -class B(A): y = 1 - -x = A() -x.y # E: "A" has no attribute "y" -x = B() -x.y # OK: x is known to be a B - -[builtins fixtures/isinstancelist.py] - -[case testIsInstanceBasic] -from typing import Union, Undefined - -x = Undefined # type: Union[int, str] - -if isinstance(x, str): - x = x + 1 # E: Unsupported operand types for + ("str" and "int") - x = x + 'a' -else: - x = x + 'a' # E: Unsupported operand types for + ("int" and "str") - x = x + 1 -[builtins fixtures/isinstancelist.py] - -[case testIsInstanceIndexing] -from typing import Union, Undefined - -x = Undefined # type: Union[int, str] -j = [x] - -if isinstance(j[0], str): - j[0] = j[0] + 'a' - j[0] = j[0] + 1 # E: Unsupported operand types for + ("str" and "int") -else: - j[0] = j[0] + 'a' # E: Unsupported operand types for + ("int" and "str") - j[0] = j[0] + 1 -[builtins fixtures/isinstancelist.py] - -[case testIsInstanceSubClassMember] -from typing import Union, Undefined - -class Animal: - pass - -class Dog(Animal): - paws = 4 # type: Union[int, str] - - def bark(self): pass - -class House: - pet = Undefined(Animal) - -h = House() -h.pet = Dog() - -while 1: - if isinstance(h.pet, Dog): - if isinstance(h.pet.paws, str): - x = h.pet.paws + 'a' - y = h.pet.paws + 1 # E: Unsupported operand types for + (likely involving Union) - z = h.pet.paws + 'a' # E: Unsupported operand types for + (likely involving Union) - if isinstance(h.pet.paws, str): - x = h.pet.paws + 'a' - break - y = h.pet.paws + 1 - z = h.pet.paws + 'a' # E: Unsupported operand types for + ("int" and "str") -[builtins fixtures/isinstancelist.py] - -[case testMemberAssignmentChanges-skip] -from typing import Union, Undefined - -class Dog: - paws = 1 # type: Union[int, str] - -pet = Dog() - -pet.paws + 'a' # E: moo -pet.paws = 'a' -pet.paws + 'a' -pet.paws = 1 -pet.paws + 1 - -[builtins fixtures/isinstancelist.py] - -[case testIsInstanceSubClassMemberHard-skip] -from typing import Union, Undefined - -class Animal: - pass - -class Dog(Animal): - paws = 4 # type: Union[int, str] - - def bark(self): pass - -class House: - pet = Undefined(Animal) - -h = House() -h.pet = Dog() - -if isinstance(h.pet, Dog): - if isinstance(h.pet.paws, str): - for i in [1]: - h.pet.paws + 'a' - if 1: - break - h.pet.paws = 1 - h.pet.paws + 1 - - if isinstance(h.pet.paws, str): - h.pet.paws + 'a' - else: - h.pet.paws + 1 - -[builtins fixtures/isinstancelist.py] - -[case testIsInstanceReturn] -from typing import Union - -def foo() -> None: - x = 1 # type: Union[int, str] - if isinstance(x, int): - return - else: - pass - y = x + 'asdad' - -foo() -[builtins fixtures/isinstancelist.py] -[case testIsInstanceBadBreak] -from typing import Union, Undefined - -def foo() -> None: - x = Undefined # type: Union[int, str] - if isinstance(x, int): - for z in [1,2]: - break - else: - pass - y = x + 'asdad' # E: Unsupported operand types for + (likely involving Union) - -foo() -[builtins fixtures/isinstancelist.py] -[out] -main: In function "foo": -[case testIsInstanceThreeUnion] -from typing import Union, List, Undefined - -x = Undefined # type: Union[int, str, List[int]] - -while 1: - if isinstance(x, int): - x + 1 - elif isinstance(x, str): - x + 'a' - else: - x + [1] - x + 'a' # E: Unsupported operand types for + (likely involving Union) -x + [1] # E: Unsupported operand types for + (likely involving Union) - -[builtins fixtures/isinstancelist.py] -[case testIsInstanceThreeUnion2] -from typing import Union, List, Undefined - -x = Undefined # type: Union[int, str, List[int]] - -while 1: - if isinstance(x, int): - x + 1 - break - elif isinstance(x, str): - x + 'a' - break - x + [1] - x + 'a' # E: Unsupported operand types for + ("list" and "str") -x + [1] # E: Unsupported operand types for + (likely involving Union) - -[builtins fixtures/isinstancelist.py] - -[case testIsInstanceThreeUnion3] -from typing import Union, List - -while 1: - x = None # type: Union[int, str, List[int]] - x = 1 - if isinstance(x, int): - x + 1 - break - elif isinstance(x, str): - x + 'a' - break - x + [1] # These lines aren't reached because x was an int - x + 'a' -x + [1] # E: Unsupported operand types for + (likely involving Union) - -[builtins fixtures/isinstancelist.py] - -[case testRemovingTypeRepeatedly] -from typing import Union - -def foo() -> Union[int, str]: pass - -for i in [1, 2]: - x = foo() - x + 'a' # E: Unsupported operand types for + (likely involving Union) - if isinstance(x, int): - break - x + 'a' - - x = foo() - x + 'a' # E: Unsupported operand types for + (likely involving Union) - if isinstance(x, int): - break - x + 'a' - - x = foo() - x + 'a' # E: Unsupported operand types for + (likely involving Union) - if isinstance(x, int): - break - x + 'a' - -x + 'a' # E: Unsupported operand types for + (likely involving Union) - -[builtins fixtures/isinstancelist.py] - - - -[case testModifyRepeatedly] -from typing import Union - -def foo() -> Union[int, str]: pass - -x = foo() -x + 1 # E: Unsupported operand types for + (likely involving Union) -x + 'a' # E: Unsupported operand types for + (likely involving Union) - -x = 1 -x + 1 -x + 'a' # E: Unsupported operand types for + ("int" and "str") - -x = 'a' -x + 1 # E: Unsupported operand types for + ("str" and "int") -x + 'a' - -x = foo() -x + 1 # E: Unsupported operand types for + (likely involving Union) -x + 'a' # E: Unsupported operand types for + (likely involving Union) - -[builtins fixtures/isinstancelist.py] - -[case testModifyLoop] -from typing import Union - -def foo() -> Union[int, str]: pass - -x = foo() -x + 1 # E: Unsupported operand types for + (likely involving Union) -x = 'a' -x + 1 # E: Unsupported operand types for + ("str" and "int") -x = 1 -x + 1 - -while 1: - x + 1 # E: Unsupported operand types for + (likely involving Union) - x = 'a' -[builtins fixtures/isinstancelist.py] - -[case testModifyLoop2] -from typing import Union - -def foo() -> Union[int, str]: pass - -x = foo() -x + 1 # E: Unsupported operand types for + (likely involving Union) -x = 'a' -x + 1 # E: Unsupported operand types for + ("str" and "int") -x = 1 -x + 1 - -for i in [1]: - x = 'a' - -x + 1 # E: Unsupported operand types for + (likely involving Union) - -[builtins fixtures/isinstancelist.py] - - -[case testModifyLoop3] -from typing import Union - -def foo() -> Union[int, str]: pass - -x = foo() -x = 1 - -while 1: - x + 1 - x = 'a' - break -else: - x + 1 -x + 1 # E: Unsupported operand types for + (likely involving Union) -[builtins fixtures/isinstancelist.py] - -[case testModifyLoop4] -from typing import Union - -def foo() -> Union[int, str]: pass - -x = foo() -x = 1 - -while 1: - x + 1 - if 1: - x = 'a' - break -else: - x + 1 - x = 'a' -x + 'a' -x = 1 -while 1: - x + 1 # E: Unsupported operand types for + (likely involving Union) - if 1: - x = 'a' - continue -else: - x + 1 # E: Unsupported operand types for + (likely involving Union) - x = 'a' -x + 'a' -[builtins fixtures/isinstancelist.py] - -[case testModifyLoopLong] -from typing import Union - -class A: a = 1 - -def foo() -> Union[int, str, A]: pass - -def bar() -> None: - x = foo() - x + 1 # E: Unsupported left operand type for + (some union) - if isinstance(x, A): - x.a - else: - if isinstance(x, int): - x + 1 - x + 'a' # E: Unsupported operand types for + ("int" and "str") - else: - x + 'a' - x.a # E: "str" has no attribute "a" - x = A() - - if isinstance(x, str): - x + 'a' - else: - while 1: - if isinstance(x, int): - x + 1 - else: - x.a - break - while 1: - if isinstance(x, int): - x + 1 - else: - x.a - continue - - #for i in [1]: - while 1: - if isinstance(x, int): - x + 1 - else: - x.a # E: Some element of union has no attribute "a" - x = 'a' - -[builtins fixtures/isinstancelist.py] -[out] -main: In function "bar": - -[case testCastIsinstance] -from typing import Union - -def foo() -> Union[int, str]: pass - -x = foo() -y = 1 # type: int - -if isinstance(x, str): - x = y -x + 1 -x + 'a' # E: Unsupported operand types for + ("int" and "str") - -[builtins fixtures/isinstancelist.py] - - -[case testUnreachableCode] -x = 1 # type: int - -while 1: - x = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - break - x = 'a' # Note: no error because unreachable code -[builtins fixtures/isinstancelist.py] - diff --git a/mypy/test/data/check-kwargs.test b/mypy/test/data/check-kwargs.test deleted file mode 100644 index decea530bba0..000000000000 --- a/mypy/test/data/check-kwargs.test +++ /dev/null @@ -1,262 +0,0 @@ --- Test cases for keyword arguments. - - -[case testTypeErrorInKeywordArgument] -import typing -def f(o: object) -> None: pass -f(o=None()) # E: None not callable - -[case testSimpleKeywordArgument] -import typing -def f(a: 'A') -> None: pass -f(a=A()) -f(a=object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A" -class A: pass - -[case testTwoKeywordArgumentsNotInOrder] -import typing -def f(a: 'A', b: 'B') -> None: pass -f(b=A(), a=A()) # E: Argument 1 to "f" has incompatible type "A"; expected "A" -f(b=B(), a=B()) # E: Argument 2 to "f" has incompatible type "B"; expected "B" -f(a=A(), b=B()) -f(b=B(), a=A()) -class A: pass -class B: pass - -[case testOneOfSeveralOptionalKeywordArguments] -import typing -def f(a: 'A' = None, b: 'B' = None, c: 'C' = None) -> None: pass -f(a=A()) -f(b=B()) -f(c=C()) -f(b=B(), c=C()) -f(a=B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -f(b=A()) # E: Argument 1 to "f" has incompatible type "A"; expected "A" -f(c=B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -f(b=B(), c=A()) # E: Argument 2 to "f" has incompatible type "A"; expected "B" -class A: pass -class B: pass -class C: pass - -[case testBothPositionalAndKeywordArguments] -import typing -def f(a: 'A', b: 'B') -> None: pass -f(A(), b=A()) # E: Argument 2 to "f" has incompatible type "A"; expected "B" -f(A(), b=B()) -class A: pass -class B: pass - -[case testContextSensitiveTypeInferenceForKeywordArg] -from typing import List -def f(a: 'A', b: 'List[A]') -> None: pass -f(b=[], a=A()) -class A: pass -[builtins fixtures/list.py] - -[case testGivingSameKeywordArgumentTwice] -import typing -def f(a: 'A', b: 'B') -> None: pass -f(a=A(), b=B(), a=A()) # E: "f" gets multiple values for keyword argument "a" -f(b=B(), a=A(), b=B()) # E: "f" gets multiple values for keyword argument "b" -class A: pass -class B: pass - -[case testGivingArgumentAsPositionalAndKeywordArg] -import typing -def f(a: 'A', b: 'B' = None) -> None: pass -f(A(), a=A()) # E: "f" gets multiple values for keyword argument "a" -class A: pass -class B: pass - -[case testGivingArgumentAsPositionalAndKeywordArg2] -import typing -def f(a: 'A' = None, b: 'B' = None) -> None: pass -f(A(), a=A()) # E: "f" gets multiple values for keyword argument "a" -class A: pass -class B: pass - -[case testInvalidKeywordArgument] -import typing -def f(a: 'A') -> None: pass -f(b=object()) # E: Unexpected keyword argument "b" for "f" -class A: pass - -[case testKeywordArgumentsWithDynamicallyTypedCallable] -from typing import Undefined, Any -f = Undefined # type: Any -f(x=f(), z=None()) # E: None not callable -f(f, zz=None()) # E: None not callable -f(x=None) - -[case testDuplicateKeywordArgumentWithDynamicallyTypedCallable] -from typing import Undefined, Any -f = Undefined # type: Any -# Postpone checking to runtime -f(x=None, x=None) - -[case testKeywordArgumentWithFunctionObject] -from typing import Undefined, Function -f = Undefined # type: Function[[A, B], None] -f(a=A(), b=B()) -f(A(), b=B()) -class A: pass -class B: pass -[out] -main, line 3: Unexpected keyword argument "a" -main, line 3: Unexpected keyword argument "b" -main, line 4: Unexpected keyword argument "b" - -[case testKeywordOnlyArguments] -import typing -def f(a: 'A', *, b: 'B' = None) -> None: pass -f(A(), b=B()) -f(b=B(), a=A()) -f(A()) -f(A(), B()) # E: Too many positional arguments for "f" -class A: pass -class B: pass - -[case testKeywordArgAfterVarArgs] -import typing -def f(*a: 'A', b: 'B' = None,) -> None: pass -f() -f(A()) -f(A(), A()) -f(b=B()) -f(A(), b=B()) -f(A(), A(), b=B()) -f(B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -f(A(), B()) # E: Argument 2 to "f" has incompatible type "B"; expected "B" -f(b=A()) # E: Argument 1 to "f" has incompatible type "A"; expected "A" -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testKeywordArgAfterVarArgsWithBothCallerAndCalleeVarArgs] -from typing import Undefined, List -def f(*a: 'A', b: 'B' = None) -> None: pass -a = Undefined # type: List[A] -f(*a) -f(A(), *a) -f(b=B()) -f(*a, b=B()) -f(A(), *a, b=B()) -f(A(), B()) # E: Argument 2 to "f" has incompatible type "B"; expected "B" -f(A(), b=A()) # E: Argument 2 to "f" has incompatible type "A"; expected "B" -f(*a, b=A()) # E: Argument 2 to "f" has incompatible type "A"; expected "B" -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testCallingDynamicallyTypedFunctionWithKeywordArgs] -import typing -def f(x, y=A()): pass -f(x=A(), y=A()) -f(y=A(), x=A()) -f(y=A()) # E: Too few arguments for "f" -f(A(), z=A()) # E: Unexpected keyword argument "z" for "f" -class A: pass - -[case testKwargsArgumentInFunctionBody] -from typing import Dict, Any -def f( **kwargs: 'A') -> None: - d1 = kwargs # type: Dict[str, A] - d2 = kwargs # type: Dict[A, Any] # E: Incompatible types in assignment (expression has type Dict[str, A], variable has type Dict[A, Any]) - d3 = kwargs # type: Dict[Any, str] # E: Incompatible types in assignment (expression has type Dict[str, A], variable has type Dict[Any, str]) -class A: pass -[builtins fixtures/dict.py] -[out] -main: In function "f": - -[case testKwargsArgumentInFunctionBodyWithImplicitAny] -from typing import Dict, Any -def f(**kwargs) -> None: - d1 = kwargs # type: Dict[str, A] - d2 = kwargs # type: Dict[str, str] - d3 = kwargs # type: Dict[A, Any] # E: Incompatible types in assignment (expression has type Dict[str, Any], variable has type Dict[A, Any]) -class A: pass -[builtins fixtures/dict.py] -[out] -main: In function "f": - -[case testCallingFunctionThatAcceptsVarKwargs] -import typing -def f( **kwargs: 'A') -> None: pass -f() -f(x=A()) -f(y=A(), z=A()) -f(x=B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -f(A()) # E: Too many arguments for "f" -# Perhaps a better message would be "Too many *positional* arguments..." -class A: pass -class B: pass -[builtins fixtures/dict.py] - -[case testCallingFunctionWithKeywordVarArgs] -from typing import Undefined, Dict -def f( **kwargs: 'A') -> None: pass -d = Undefined # type: Dict[str, A] -f(**d) -f(x=A(), **d) -d2 = Undefined # type: Dict[str, B] -f(**d2) # E: Argument 1 to "f" has incompatible type Dict[str, B]; expected "A" -f(x=A(), **d2) # E: Argument 2 to "f" has incompatible type Dict[str, B]; expected "A" -class A: pass -class B: pass -[builtins fixtures/dict.py] - -[case testInvalidTypeForKeywordVarArg] -from typing import Undefined, Dict -def f( **kwargs: 'A') -> None: pass -d = Undefined # type: Dict[A, A] -f(**d) # E: Keywords must be strings -f(**A()) # E: Argument after ** must be a dictionary -class A: pass -[builtins fixtures/dict.py] - -[case testPassingKeywordVarArgsToNonVarArgsFunction] -from typing import Undefined, Any, Dict -def f(a: 'A', b: 'B') -> None: pass -d = Undefined # type: Dict[str, Any] -f(**d) -d2 = Undefined # type: Dict[str, A] -f(**d2) # E: Argument 1 to "f" has incompatible type Dict[str, A]; expected "A" -class A: pass -class B: pass -[builtins fixtures/dict.py] - -[case testBothKindsOfVarArgs] -from typing import Undefined, Any, List, Dict -def f(a: 'A', b: 'A') -> None: pass -l = Undefined # type: List[Any] -d = Undefined # type: Dict[Any, Any] -f(*l, **d) -class A: pass -[builtins fixtures/dict.py] - -[case testKeywordArgumentAndCommentSignature] -import typing -def f(x): # type: (int) -> str - pass -f(x='') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -f(x=0) -f(y=0) # E: Unexpected keyword argument "y" for "f" - -[case testKeywordArgumentAndCommentSignature2] -import typing -class A: - def f(self, x): # type: (int) -> str - pass -A().f(x='') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -A().f(x=0) -A().f(y=0) # E: Unexpected keyword argument "y" for "f" of "A" - -[case testKeywordVarArgsAndCommentSignature] -import typing -def f(**kwargs): # type: (**int) -> None - pass -f(z=1) -f(x=1, y=1) -f(x='', y=1) # E: Argument 1 to "f" has incompatible type "str"; expected "int" -f(x=1, y='') # E: Argument 2 to "f" has incompatible type "str"; expected "int" -[builtins fixtures/dict.py] diff --git a/mypy/test/data/check-lists.test b/mypy/test/data/check-lists.test deleted file mode 100644 index 61fc351a82fa..000000000000 --- a/mypy/test/data/check-lists.test +++ /dev/null @@ -1,64 +0,0 @@ --- Nested list assignment --- ----------------------------- - - -[case testNestedListAssignment] -from typing import Undefined, List -a1, b1, c1 = Undefined, Undefined, Undefined # type: (A, B, C) -a2, b2, c2 = Undefined, Undefined, Undefined # type: (A, B, C) - -a1, [b1, c1] = a2, [b2, c2] -a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] -a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testNestedListAssignmenToTuple] -from typing import Undefined, List -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) - -a, b = [a, b] -a, b = [a] # E: Need more than 1 value to unpack (2 expected) -a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentFromTuple] -from typing import Undefined, List -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -t = a, b - -[a, b], c = t, c -[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") -[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) -[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentUnequalAmountToUnpack] -from typing import Undefined, List -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) - -def f() -> None: # needed because test parser tries to parse [a, b] as section header - [a, b] = [a, b] - [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) - [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] -main: In function "f": \ No newline at end of file diff --git a/mypy/test/data/check-modules.test b/mypy/test/data/check-modules.test deleted file mode 100644 index b145814c5673..000000000000 --- a/mypy/test/data/check-modules.test +++ /dev/null @@ -1,378 +0,0 @@ - --- Test cases for the type checker. - -[case testAccessImportedDefinitions] -import m -import typing -m.f() # E: Too few arguments for "f" -m.f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A" -m.x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") -m.f(m.A()) -m.x = m.A() -[file m.py] -class A: pass -def f(a: A) -> None: pass -x = A() - -[case testAccessImportedDefinitions] -import m -import typing -m.f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A" -m.f(m.A()) -[file m.py] -class A: pass -def f(a: A) -> None: pass - -[case testAccessImportedDefinitions2] -from m import f, A -import typing -f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A" -f(A()) -[file m.py] -class A: pass -def f(a: A) -> None: pass - -[case testImportedExceptionType] -import m -import typing -try: - pass -except m.Err: - pass -except m.Bad: # E: Exception type must be derived from BaseException - pass -[file m.py] -class Err(BaseException): pass -class Bad: pass -[builtins fixtures/exception.py] - -[case testImportedExceptionType2] -from m import Err, Bad -import typing -try: - pass -except Err: - pass -except Bad: # E: Exception type must be derived from BaseException - pass -[file m.py] -class Err(BaseException): pass -class Bad: pass -[builtins fixtures/exception.py] - -[case testImportWithinBlock] -import typing -if None: - import m - m.a = m.b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - m.a = m.a - m.f() - m.f(m.a) # E: Too many arguments for "f" - m.a = m.A() - m.a = m.B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[file m.py] -class A: pass -class B: pass -a = A() -b = B() -def f() -> None: pass - -[case testImportWithinFunction] -import typing -def f() -> None: - from m import a, b, f, A, B - a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = a - f() - f(a) # E: Too many arguments for "f" - a = A() - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[file m.py] -class A: pass -class B: pass -a = A() -b = B() -def f() -> None: pass -[out] -main: In function "f": - -[case testImportWithinMethod] -import typing -class C: - def f(self) -> None: - from m import * - a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = a - f() - f(a) # E: Too many arguments for "f" - a = A() - a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[file m.py] -class A: pass -class B: pass -a = A() -b = B() -def f() -> None: pass -[out] -main: In member "f" of class "C": - -[case testImportWithinClassBody] -import typing -class C: - import m - m.f() - m.f(C) # E: Too many arguments for "f" -[file m.py] -def f() -> None: pass -[out] -main: In class "C": - -[case testImportWithinClassBody2] -import typing -class C: - from m import f - f() - f(C) # E: Too many arguments for "f" -[file m.py] -def f() -> None: pass -[out] -main: In class "C": - -[case testImportWithStub] -import _m -_m.f("hola") -[file _m.pyi] -def f(c:str) -> None: pass -[out] - - -[case testImportWithStubIncompatibleType] -import _m -_m.f("hola") -_m.f(12) # E: Argument 1 to "f" has incompatible type "int"; expected "str" -[file _m.py] -def f(c): - print(c) -[file _m.pyi] -def f(c:str) -> None: pass - -[case testInvalidOperationsOnModules] -import m -import typing -class A: pass -m() # E: "module" not callable -a = m # type: A # E: Incompatible types in assignment (expression has type "module", variable has type "A") -m + None # E: Unsupported left operand type for + ("module") -[file m.py] -[builtins fixtures/module.py] - -[case testNameDefinedInDifferentModule] -import m, n -import typing -m.x # E: "module" has no attribute "x" -[file m.py] -y = object() -[file n.py] -x = object() -[builtins fixtures/module.py] - -[case testChainedAssignmentAndImports] -import m -from typing import Undefined -i, s = Undefined, Undefined # type: (int, str) -i = m.x -i = m.y -s = m.x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -s = m.y # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[file m.py] -x = y = 1 -[builtins fixtures/primitives.py] - -[case testConditionalFunctionDefinitionAndImports] -import m -import typing -m.f(1) -m.f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -[file m.py] -x = object() -if x: - def f(x: int) -> None: pass -else: - def f(x: int) -> None: pass - -[case testTypeCheckWithUnknownModule] -import nonexistent -None + '' -[out] -main, line 1: No module named 'nonexistent' -main, line 2: Unsupported left operand type for + (None) - -[case testTypeCheckWithUnknownModule2] -import m, nonexistent -None + '' -m.x = 1 -m.x = '' -[file m.py] -x = 1 -[out] -main, line 1: No module named 'nonexistent' -main, line 2: Unsupported left operand type for + (None) -main, line 4: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testTypeCheckWithUnknownModule3] -import nonexistent, m -None + '' -m.x = 1 -m.x = '' -[file m.py] -x = 1 -[out] -main, line 1: No module named 'nonexistent' -main, line 2: Unsupported left operand type for + (None) -main, line 4: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testTypeCheckWithUnknownModule4] -import nonexistent, another -None + '' -[out] -main, line 1: No module named 'nonexistent' -main, line 1: No module named 'another' -main, line 2: Unsupported left operand type for + (None) - -[case testTypeCheckWithUnknownModule5] -import nonexistent as x -None + '' -[out] -main, line 1: No module named 'nonexistent' -main, line 2: Unsupported left operand type for + (None) - -[case testTypeCheckWithUnknownModuleUsingFromImport] -from nonexistent import x -None + '' -[out] -main, line 1: No module named 'nonexistent' -main, line 2: Unsupported left operand type for + (None) - -[case testTypeCheckWithUnknownModuleUsingImportStar] -from nonexistent import * -None + '' -[out] -main, line 1: No module named 'nonexistent' -main, line 2: Unsupported left operand type for + (None) - -[case testAccessingUnknownModule] -import xyz -xyz.foo() -xyz() -[out] -main, line 1: No module named 'xyz' - -[case testAccessingUnknownModule2] -import xyz, bar -xyz.foo() -bar() -[out] -main, line 1: No module named 'xyz' -main, line 1: No module named 'bar' - -[case testAccessingUnknownModule3] -import xyz as z -xyz.foo() -z() -[out] -main, line 1: No module named 'xyz' -main, line 2: Name 'xyz' is not defined - -[case testAccessingNameImportedFromUnknownModule] -from xyz import y, z -y.foo() -z() -[out] -main, line 1: No module named 'xyz' - -[case testAccessingNameImportedFromUnknownModule2] -from xyz import * -y -[out] -main, line 1: No module named 'xyz' -main, line 2: Name 'y' is not defined - -[case testAccessingNameImportedFromUnknownModule3] -from xyz import y as z -y -z -[out] -main, line 1: No module named 'xyz' -main, line 2: Name 'y' is not defined - -[case testUnknownModuleRedefinition] -import xab -def xab(): pass -[out] -main, line 1: No module named 'xab' -main, line 1: Name 'xab' already defined - -[case testAccessingUnknownModuleFromOtherModule] -import x -x.nonexistent.foo -x.z -[file x.py] -import nonexistent -[builtins fixtures/module.py] -[out] -In module imported in main, line 1: -tmp/x.py, line 1: No module named 'nonexistent' -main, line 3: "module" has no attribute "z" - -[case testUnknownModuleImportedWithinFunction] -def f(): - import foobar -def foobar(): pass -foobar('') -[out] -main, line 2: No module named 'foobar' -main, line 4: Too many arguments for "foobar" - -[case testUnknownModuleImportedWithinFunction2] -def f(): - from foobar import x -def x(): pass -x('') -[out] -main, line 2: No module named 'foobar' -main, line 4: Too many arguments for "x" - -[case testRelativeImports] -import typing -import m.a -m.a.x = m.a.y # Error -[file m/__init__.py] -[file m/a.py] -import typing -from .b import A, B, x, y -z = x -z = y # Error -[file m/b.py] -import typing -class A: pass -class B: pass -x = A() -y = B() -[out] -In module imported in main, line 2: -tmp/m/a.py, line 4: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testRelativeImports2] -import typing -import m.a -m.a.x = m.a.y # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[file m/__init__.py] -[file m/a.py] -import typing -from .b import A, B, x, y -[file m/b.py] -import typing -class A: pass -class B: pass -x = A() -y = B() diff --git a/mypy/test/data/check-multiple-inheritance.test b/mypy/test/data/check-multiple-inheritance.test deleted file mode 100644 index a25cd4fb07da..000000000000 --- a/mypy/test/data/check-multiple-inheritance.test +++ /dev/null @@ -1,207 +0,0 @@ --- Test cases for multiple inheritance. --- --- Related: check-abstract.test - - --- No name collisions --- ------------------ - - -[case testSimpleMultipleInheritanceAndMethods] -import typing -class A: - def f(self, x: int) -> None: pass -class B: - def g(self, x: str) -> None: pass -class C(A, B): pass -c = C() -c.f(1) -c.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -c.g('') -c.g(1) # E: Argument 1 to "g" of "B" has incompatible type "int"; expected "str" - -[case testSimpleMultipleInheritanceAndMethods2] -import typing -class A: - def f(self, x: int) -> None: pass -class B: - def g(self, x): pass -class C(A, B): pass -c = C() -c.f(1) -c.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -c.g('') -c.g(1) - -[case testSimpleMultipleInheritanceAndInstanceVariables] -import typing -class A: - def f(self) -> None: - self.x = 1 -class B: - def g(self) -> None: - self.y = '' -class C(A, B): pass -c = C() -c.x = 1 -c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -c.y = '' -c.y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testSimpleMultipleInheritanceAndInstanceVariableInClassBody] -import typing -class A: - x = 1 -class B: - y = '' -class C(A, B): pass -c = C() -c.x = 1 -c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -c.y = '' -c.y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testSimpleMultipleInheritanceAndClassVariable] -import typing -class A: - x = 1 -class B: - y = '' -class C(A, B): pass -C.x = 1 -C.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -C.y = '' -C.y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - - --- Name collisions --- --------------- - - -[case testMethodNameCollisionInMultipleInheritanceWithValidSigs] -import typing -class A: - def f(self, x: int) -> None: pass -class B: - def f(self, x: int) -> None: pass -class C(A, B): pass -c = C() -c.f(1) -c.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - -[case testInstanceVarNameOverlapInMultipleInheritanceWithCompatibleTypes] -import typing -class A: - def f(self) -> None: - self.x = 1 -class B: - def g(self) -> None: - self.x = 1 -class C(A, B): pass -c = C() -c.x = 1 -c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testClassVarNameOverlapInMultipleInheritanceWithCompatibleTypes] -import typing -class A: - x = 1 -class B: - x = 1 -class C(A, B): pass -c = C() -c.x = 1 -c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -C.x = 1 -C.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testMethodNameCollisionInMultipleInheritanceWithIncompatibleSigs] -import typing -class A: - def f(self, x: int) -> None: pass -class B: - def f(self, x: str) -> None: pass -class C(A, B): pass -[out] -main: In class "C": -main, line 6: Definition of "f" in base class "A" is incompatible with definition in base class "B" - -[case testMethodNameCollisionInMultipleInheritanceWithIncompatibleSigs2] -import typing -class A: - def f(self, x: int) -> None: pass -class B: - def f(self, x, y): pass -class C(A, B): pass -[out] -main: In class "C": -main, line 6: Definition of "f" in base class "A" is incompatible with definition in base class "B" - -[case testInstanceVarNameOverlapInMultipleInheritanceWithInvalidTypes] -import typing -class A: - def f(self) -> None: - self.x = 1 -class B: - def g(self) -> None: - self.x = '' -class C(A, B): pass -[out] -main: In class "C": -main, line 8: Definition of "x" in base class "A" is incompatible with definition in base class "B" - -[case testClassVarNameOverlapInMultipleInheritanceWithInvalidTypes] -import typing -class A: - x = 1 -class B: - x = '' -class C(A, B): pass -[out] -main: In class "C": -main, line 6: Definition of "x" in base class "A" is incompatible with definition in base class "B" - -[case testMethodOverlapsWithClassVariableInMultipleInheritance] -from typing import Function -class A: - def f(self) -> None: pass -class B: - f = '' -class C(A, B): pass -[out] -main: In class "C": -main, line 6: Definition of "f" in base class "A" is incompatible with definition in base class "B" - -[case testMethodOverlapsWithInstanceVariableInMultipleInheritance] -from typing import Function -class A: - def f(self) -> None: pass -class B: - def g(self) -> None: - self.f = '' -class C(A, B): pass -[out] -main: In class "C": -main, line 7: Definition of "f" in base class "A" is incompatible with definition in base class "B" - -[case testMultipleInheritanceAndInit] -import typing -class A: - def __init__(self, x: int) -> None: pass -class B: - def __init__(self) -> None: pass -class C(A, B): pass - - --- Special cases --- ------------- - - -[case testGenericInheritanceAndOverridingWithMultipleInheritance] -from typing import Generic, typevar -T = typevar('T') -class G(Generic[T]): - def f(self, s: int) -> 'G[T]': pass -class A(G[int]): - def f(self, s: int) -> 'A': pass -class B(A, int): pass diff --git a/mypy/test/data/check-namedtuple.test b/mypy/test/data/check-namedtuple.test deleted file mode 100644 index 9c7ddf2e1b2f..000000000000 --- a/mypy/test/data/check-namedtuple.test +++ /dev/null @@ -1,147 +0,0 @@ -[case testNamedTupleUsedAsTuple] -from collections import namedtuple -from typing import Undefined -X = namedtuple('X', ['x', 'y']) -x = Undefined(X) -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple -from typing import Undefined -X = namedtuple('X', ['x', 'y']) -x = Undefined(X) -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple -from typing import Undefined -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple -from typing import Undefined -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Too few arguments for "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple -from typing import Undefined -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Undefined, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = Undefined(Tuple[int, str]) -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Undefined, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] diff --git a/mypy/test/data/check-overloading.test b/mypy/test/data/check-overloading.test deleted file mode 100644 index fb3dcefe6f72..000000000000 --- a/mypy/test/data/check-overloading.test +++ /dev/null @@ -1,498 +0,0 @@ --- Test cases for function overloading - - -[case testTypeCheckOverloadedFunctionBody] -from typing import overload -@overload -def f(x: 'A'): - x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - x = A() -@overload -def f(x: 'B'): - x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - x = B() -class A: pass -class B: pass -[out] -main: In function "f": - -[case testTypeCheckOverloadedMethodBody] -from typing import overload -class A: - @overload - def f(self, x: 'A'): - x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - x = A() - @overload - def f(self, x: 'B'): - x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - x = B() -class B: pass -[out] -main: In member "f" of class "A": - -[case testCallToOverloadedFunction] -from typing import overload -f(C()) # E: No overload variant of "f" matches argument types -f(A()) -f(B()) - -@overload -def f(x: 'A') -> None: pass -@overload -def f(x: 'B') -> None: pass - -class A: pass -class B: pass -class C: pass - -[case testOverloadedFunctionReturnValue] -from typing import Undefined, overload, disjointclass -a, b = Undefined, Undefined # type: (A, B) -b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = f(a) -b = f(b) - -@overload -def f(x: 'A') -> 'A': pass -@overload -def f(x: 'B') -> 'B': pass -class A: pass -@disjointclass(A) -class B: pass - -[case testCallToOverloadedMethod] -from typing import overload -A().f(C()) # E: No overload variant of "f" of "A" matches argument types -A().f(A()) -A().f(B()) - -class A: - @overload - def f(self, x: 'A') -> None: pass - @overload - def f(self, x: 'B') -> None: pass - -class B: pass -class C: pass - -[case testOverloadedMethodReturnValue] -from typing import Undefined, overload, disjointclass -a, b = Undefined, Undefined # type: (A, B) -b = a.f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = a.f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = a.f(a) -b = a.f(b) - -class A: - @overload - def f(self, x: 'A') -> 'A': pass - @overload - def f(self, x: 'B') -> 'B': pass -@disjointclass(A) -class B: pass - -[case testNonConsecutiveOverloads] -from typing import Any -def f() -> None: pass -def g() -> None: pass -def f(x: Any) -> None: pass # E: Name 'f' already defined (overload variants must be next to each other) - -[case testOverloadsWithDifferentArgumentCounts] -from typing import Undefined, overload -a, b = Undefined, Undefined # type: (A, B) -a = f(a) -b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -f(b) # E: No overload variant of "f" matches argument types -b = f(b, a) -a = f(b, a) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -f(a, a) # E: No overload variant of "f" matches argument types -f(b, b) # E: No overload variant of "f" matches argument types - -@overload -def f(x: 'A') -> 'A': pass -@overload -def f(x: 'B', y: 'A') -> 'B': pass -class A: pass -class B: pass - -[case testGenericOverloadVariant] -from typing import Undefined, overload, typevar, Generic, disjointclass -t = typevar('t') -ab, ac, b, c = Undefined, Undefined, Undefined, Undefined # type: (A[B], A[C], B, C) -b = f(ab) -c = f(ac) -b = f(ac) # E: Incompatible types in assignment (expression has type "C", variable has type "B") -b = f(b) -c = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "C") -@overload -def f(x: 'A[t]') -> t: pass -@overload -def f(x: 'B') -> 'B': pass -class A(Generic[t]): pass -@disjointclass(A) -class B: pass -class C: pass - -[case testOverloadedInit] -from typing import Undefined, overload -a, b = Undefined, Undefined # type: (A, B) -a = A(a) -a = A(b) -a = A(object()) # E: No overload variant of "A" matches argument types - -class A: - @overload - def __init__(self, a: 'A') -> None: pass - @overload - def __init__(self, b: 'B') -> None: pass -class B: pass - -[case testIntersectionTypeCompatibility] -from typing import Undefined, overload, Function -o = Undefined # type: object -a = Undefined # type: A - -a = f # E: Incompatible types in assignment (expression has type overloaded function, variable has type "A") -o = f - -@overload -def f(a: 'A') -> None: pass -@overload -def f(a: Function[[], None]) -> None: pass -class A: pass - -[case testCompatibilityOfIntersectionTypeObjectWithStdType] -from typing import Undefined, overload -t, a = Undefined, Undefined # type: (type, A) - -a = A # E: Incompatible types in assignment (expression has type "A" (type object), variable has type "A") -t = A - -class A: - @overload - def __init__(self, a: 'A') -> None: pass - @overload - def __init__(self, a: 'B') -> None: pass -class B: pass - -[case testOverloadedGetitem] -from typing import Undefined, overload -a, b = Undefined(int), Undefined(str) -a = A()[a] -b = A()[a] # E: Incompatible types in assignment (expression has type "int", variable has type "str") -b = A()[b] -a = A()[b] # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -class A: - @overload - def __getitem__(self, a: int) -> int: pass - @overload - def __getitem__(self, b: str) -> str: pass - -[case testOverloadedGetitemWithGenerics] -from typing import Undefined, typevar, Generic, overload -t = typevar('t') -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C[A]) -a = c[a] -b = c[a] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = c[b] -b = c[b] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -class C(Generic[t]): - @overload - def __getitem__(self, a: 'A') -> t: pass - @overload - def __getitem__(self, b: 'B') -> t: pass -class A: pass -class B: pass - -[case testImplementingOverloadedMethod] -from typing import overload -from abc import abstractmethod, ABCMeta - -class I(metaclass=ABCMeta): - @overload - @abstractmethod - def f(self) -> None: pass - @overload - @abstractmethod - def f(self, a: 'A') -> None: pass -class A(I): - @overload - def f(self) -> None: pass - @overload - def f(self, a: 'A') -> None: pass - -[case testOverloadWithFunctionType] -from typing import overload, Function -class A: pass -@overload -def f(x: A) -> None: pass -@overload -def f(x: Function[[], None]) -> None: pass - -f(A()) -[builtins fixtures/function.py] - -[case testVarArgsOverload] -from typing import overload, Any -@overload -def f(x: 'A', *more: Any) -> 'A': pass -@overload -def f(x: 'B', *more: Any) -> 'A': pass -f(A()) -f(A(), A, A) -f(B()) -f(B(), B) -f(B(), B, B) -f(object()) # E: No overload variant of "f" matches argument types -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testVarArgsOverload2] -from typing import overload -@overload -def f(x: 'A', *more: 'B') -> 'A': pass -@overload -def f(x: 'B', *more: 'A') -> 'A': pass -f(A(), B()) -f(A(), B(), B()) -f(A(), A(), B()) # E: No overload variant of "f" matches argument types -f(A(), B(), A()) # E: No overload variant of "f" matches argument types -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testOverloadWithTypeObject] -from typing import overload -@overload -def f(a: 'A', t: type) -> None: pass -@overload -def f(a: 'B', t: type) -> None: pass -f(A(), B) -f(B(), A) -class A: pass -class B: pass -[builtins fixtures/function.py] - -[case testOverloadedInitAndTypeObjectInOverload] -from typing import overload, Undefined, disjointclass -@overload -def f(t: type) -> 'A': pass -@overload -def f(t: 'A') -> 'B': pass -a, b = Undefined, Undefined # type: (A, B) -a = f(A) -b = f(a) -b = f(A) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f(a) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -@disjointclass(type) -class A: - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, a: 'A') -> None: pass -class B: - pass - -[case testOverlappingErasedSignatures] -from typing import overload, List -@overload -def f(a: List[int]) -> int: pass -@overload -def f(a: List[str]) -> int: pass -n = f(List[int]()) -m = f(List[str]()) -n = 1 -m = 1 -n = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -m = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -f(List[object]()) # E: Argument 1 to "f" has incompatible type List[object]; expected List[int] -[builtins fixtures/list.py] - -[case testOverlappingOverloadSignatures] -from typing import overload -class A: pass -class B: pass -@overload -def f(x: A) -> A: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types -@overload -def f(x: B) -> B: pass - -[case testDisjointclassesAndOverload] -from typing import overload, disjointclass -class A: pass -@disjointclass(A) -class B: pass -@overload -def f(x: A) -> A: pass -@overload -def f(x: B) -> B: pass - -[case testContravariantOverlappingOverloadSignatures] -from typing import overload -class A: pass -class B(A): pass -@overload -def f(x: A) -> A: pass -@overload -def f(x: B) -> B: pass # This is more specific than the first item, and thus - # will never be called. - -[case testPartiallyCovariantOverlappingOverloadSignatures] -from typing import overload -class A: pass -class B(A): pass -@overload -def f(x: B) -> A: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types -@overload -def f(x: A) -> B: pass - -[case testPartiallyContravariantOverloadSignatures] -from typing import overload -class A: pass -class B(A): pass -@overload -def g(x: A) -> int: pass # Fine, since A us supertype of B. -@overload -def g(x: B) -> str: pass - -[case testCovariantOverlappingOverloadSignatures] -from typing import overload -class A: pass -class B(A): pass -@overload -def g(x: B) -> B: pass -@overload -def g(x: A) -> A: pass - -[case testCovariantOverlappingOverloadSignaturesWithSomeSameArgTypes] -from typing import overload -class A: pass -class B(A): pass -@overload -def g(x: int, y: B) -> B: pass -@overload -def g(x: int, y: A) -> A: pass - -[case testCovariantOverlappingOverloadSignaturesWithAnyType] -from typing import Any, overload -@overload -def g(x: int) -> int: pass -@overload -def g(x: Any) -> Any: pass - -[case testContravariantOverlappingOverloadSignaturesWithAnyType] -from typing import Any, overload -@overload -def g(x: Any) -> Any: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types -@overload -def g(x: int) -> int: pass - -[case testOverloadedLtAndGtMethods] -from typing import overload -class A: - def __lt__(self, x: A) -> int: pass - def __gt__(self, x: A) -> int: pass -class B: - @overload - def __lt__(self, x: B) -> int: pass - @overload - def __lt__(self, x: A) -> int: pass - @overload - def __gt__(self, x: B) -> int: pass - @overload - def __gt__(self, x: A) -> int: pass -A() < A() -A() < B() -B() < A() -B() < B() -A() < object() # E: Unsupported operand types for < ("A" and "object") -B() < object() # E: No overload variant of "__lt__" of "B" matches argument types - -[case testOverloadedForwardMethodAndCallingReverseMethod] -from typing import overload -class A: - @overload - def __add__(self, x: 'A') -> int: pass - @overload - def __add__(self, x: int) -> int: pass -class B: - def __radd__(self, x: A) -> int: pass -A() + A() -A() + 1 -A() + B() -A() + '' # E: No overload variant of "__add__" of "A" matches argument types - -[case testOverrideOverloadedMethodWithMoreGeneralArgumentTypes] -from typing import overload, builtinclass -@builtinclass -class IntSub(int): pass -@builtinclass -class StrSub(str): pass -class A: - @overload - def f(self, x: IntSub) -> int: return 0 - @overload - def f(self, x: StrSub) -> str: return '' -class B(A): - @overload - def f(self, x: int) -> int: return 0 - @overload - def f(self, x: str) -> str: return '' -[out] - -[case testOverrideOverloadedMethodWithMoreSpecificArgumentTypes] -from typing import overload, builtinclass -@builtinclass -class IntSub(int): pass -@builtinclass -class StrSub(str): pass -class A: - @overload - def f(self, x: int) -> int: return 0 - @overload - def f(self, x: str) -> str: return '' -class B(A): - @overload - def f(self, x: IntSub) -> int: return 0 - @overload - def f(self, x: str) -> str: return '' -class C(A): - @overload - def f(self, x: int) -> int: return 0 - @overload - def f(self, x: StrSub) -> str: return '' -class D(A): - @overload - def f(self, x: int) -> int: return 0 - @overload - def f(self, x: str) -> str: return '' -[out] -main: In class "B": -main, line 12: Signature of "f" incompatible with supertype "A" -main: In class "C": -main, line 17: Signature of "f" incompatible with supertype "A" - -[case testOverloadingAndDucktypeCompatibility] -from typing import overload, ducktype, builtinclass - -@builtinclass -class A: pass - -@builtinclass -@ducktype(A) -class B: pass - -@overload -def f(n: B) -> B: - return n -@overload -def f(n: A) -> A: - return n - -f(B()) + 'x' # E: Unsupported left operand type for + ("B") -f(A()) + 'x' # E: Unsupported left operand type for + ("A") diff --git a/mypy/test/data/check-python2.test b/mypy/test/data/check-python2.test deleted file mode 100644 index 0dcbed20c129..000000000000 --- a/mypy/test/data/check-python2.test +++ /dev/null @@ -1,24 +0,0 @@ --- Type checker test cases for Python 2.x mode. - - -[case testUnicode] -import typing -u = u'foo' -u = unicode() -s = '' -s = u'foo' # E: Incompatible types in assignment (expression has type "unicode", variable has type "str") -s = b'foo' -[builtins fixtures/python2.py] - -[case testPrintStatement] -import typing -print ''() # E: "str" not callable -print 1, 1() # E: "int" not callable - -[case testDivision] -import typing -class A: - def __div__(self, x: int) -> str: pass -s = A() / 1 -s = '' -s = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") diff --git a/mypy/test/data/check-statements.test b/mypy/test/data/check-statements.test deleted file mode 100644 index a3d339e011fd..000000000000 --- a/mypy/test/data/check-statements.test +++ /dev/null @@ -1,937 +0,0 @@ --- Return statement --- ---------------- - - -[case testReturnValue] -import typing -def f() -> 'A': - return A() - return B() -def g() -> 'B': - return A() -class A: - pass -class B: - pass -[out] -main: In function "f": -main, line 4: Incompatible return value type: expected __main__.A, got __main__.B -main: In function "g": -main, line 6: Incompatible return value type: expected __main__.B, got __main__.A - -[case testReturnSubtype] -import typing -def f() -> 'B': - return A() -def g() -> 'A': - return B() -class A: - pass -class B(A): - pass -[out] -main: In function "f": -main, line 3: Incompatible return value type: expected __main__.B, got __main__.A - -[case testReturnWithoutAValue] -import typing -def f() -> 'A': - return -def g() -> None: - return -class A: - pass -[out] -main: In function "f": -main, line 3: Return value expected - -[case testReturnNoneInFunctionReturningNone] -import typing -def f() -> None: - return None -def g() -> None: - return f() # E: No return value expected -[out] -main: In function "g": - - --- If statement --- ------------ - - -[case testIfStatement] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: bool -if a: - a = b # Fail -elif a: - a = b # Fail -elif a: - a = b # Fail -else: - a = b # Fail -if b: - pass -elif b: - pass -if b: - pass - -class A: pass -[builtins fixtures/bool.py] -[out] -main, line 5: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 7: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 9: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 11: Incompatible types in assignment (expression has type "bool", variable has type "A") - - --- Loops --- ----- - - -[case testWhileStatement] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: bool -while a: - a = b # Fail -else: - a = b # Fail -while b: - b = b - -class A: pass -[builtins fixtures/bool.py] -[out] -main, line 5: Incompatible types in assignment (expression has type "bool", variable has type "A") -main, line 7: Incompatible types in assignment (expression has type "bool", variable has type "A") - -[case testForStatement] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: object -for a in [A()]: - a = b # Fail -else: - a = b # Fail - -class A: pass -[builtins fixtures/list.py] -[out] -main, line 5: Incompatible types in assignment (expression has type "object", variable has type "A") -main, line 7: Incompatible types in assignment (expression has type "object", variable has type "A") - -[case testBreakStatement] -import typing -while None: - break -[builtins fixtures/bool.py] -[out] - -[case testContinueStatement] -import typing -while None: - continue -[builtins fixtures/bool.py] -[out] - - --- Operator assignment --- ------------------- - - -[case testPlusAssign] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -a += b # Fail -b += a # Fail -c += a # Fail -a += c - -class A: - def __add__(self, x: 'C') -> 'A': pass - -class B: - def __add__(self, x: A) -> 'C': pass - -class C: pass -[out] -main, line 3: Unsupported operand types for + ("A" and "B") -main, line 4: Result type of + incompatible in assignment -main, line 5: Unsupported left operand type for + ("C") - -[case testMinusAssign] -from typing import Undefined -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) -a -= b # Fail -b -= a # Fail -c -= a # Fail -a -= c - -class A: - def __sub__(self, x: 'C') -> 'A': pass - -class B: - def __sub__(self, x: A) -> 'C': pass - -class C: pass -[out] -main, line 3: Unsupported operand types for - ("A" and "B") -main, line 4: Result type of - incompatible in assignment -main, line 5: Unsupported left operand type for - ("C") - -[case testMulAssign] -from typing import Undefined -a, c = Undefined, Undefined # type: (A, C) -a *= a # Fail -c *= a # Fail -a *= c - -class A: - def __mul__(self, x: 'C') -> 'A': pass - -class C: pass -[out] -main, line 3: Unsupported operand types for * ("A" and "A") -main, line 4: Unsupported left operand type for * ("C") - -[case testDivAssign] -from typing import Undefined -a, c = Undefined, Undefined # type: (A, C) -a /= a # Fail -c /= a # Fail -a /= c - -class A: - def __truediv__(self, x: 'C') -> 'A': pass - -class C: pass -[out] -main, line 3: Unsupported operand types for / ("A" and "A") -main, line 4: Unsupported left operand type for / ("C") - -[case testPowAssign] -from typing import Undefined -a, c = Undefined, Undefined # type: (A, C) -a **= a # Fail -c **= a # Fail -a **= c - -class A: - def __pow__(self, x: 'C') -> 'A': pass - -class C: pass -[out] -main, line 3: Unsupported operand types for ** ("A" and "A") -main, line 4: Unsupported left operand type for ** ("C") - -[case testSubtypesInOperatorAssignment] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -b += b -b += a -a += b - -class A: - def __add__(self, x: 'A') -> 'B': pass - -class B(A): pass -[out] - -[case testAdditionalOperatorsInOpAssign] -from typing import Undefined -a, c = Undefined, Undefined # type: (A, C) -a &= a # Fail -a >>= a # Fail -a //= a # Fail -a &= c -a >>= c -a //= c -class A: - def __and__(self, x: 'C') -> 'A': pass - def __rshift__(self, x: 'C') -> 'A': pass - def __floordiv__(self, x: 'C') -> 'A': pass -class C: pass -[out] -main, line 3: Unsupported operand types for & ("A" and "A") -main, line 4: Unsupported operand types for >> ("A" and "A") -main, line 5: Unsupported operand types for // ("A" and "A") - -[case testInplaceOperatorMethods] -import typing -class A: - def __iadd__(self, x: int) -> 'A': pass - def __imul__(self, x: str) -> 'A': pass -a = A() -a += 1 -a *= '' -a += '' # E: Argument 1 to "__iadd__" of "A" has incompatible type "str"; expected "int" -a *= 1 # E: Argument 1 to "__imul__" of "A" has incompatible type "int"; expected "str" - - --- Assert statement --- ---------------- - - -[case testAssert] -import typing -assert None + None # Fail -assert None -[out] -main, line 2: Unsupported left operand type for + (None) - - --- Exception handling --- ------------------ - - -[case testRaiseStatement] -from typing import Undefined -e = Undefined # type: BaseException -f = Undefined # type: MyError -a = Undefined # type: A -raise a # Fail -raise e -raise f -class A: pass -class MyError(BaseException): pass -[builtins fixtures/exception.py] -[out] -main, line 5: Exception must be derived from BaseException - -[case testRaiseClassobject] -import typing -class A: pass -class MyError(BaseException): pass -def f(): pass -raise BaseException -raise MyError -raise A # E: Exception must be derived from BaseException -raise object # E: Exception must be derived from BaseException -raise f # E: Exception must be derived from BaseException -[builtins fixtures/exception.py] - -[case testRaiseFromStatement] -from typing import Undefined -e = Undefined # type: BaseException -f = Undefined # type: MyError -a = Undefined # type: A -raise e from a # E: Exception must be derived from BaseException -raise e from e -raise e from f -class A: pass -class MyError(BaseException): pass -[builtins fixtures/exception.py] - -[case testRaiseFromClassobject] -import typing -class A: pass -class MyError(BaseException): pass -def f(): pass -raise BaseException from BaseException -raise BaseException from MyError -raise BaseException from A # E: Exception must be derived from BaseException -raise BaseException from object # E: Exception must be derived from BaseException -raise BaseException from f # E: Exception must be derived from BaseException -[builtins fixtures/exception.py] - -[case testTryFinallyStatement] -import typing -try: - b = object() # type: A # Fail -finally: - c = object() # type: A # Fail -class A: pass -[out] -main, line 3: Incompatible types in assignment (expression has type "object", variable has type "A") -main, line 5: Incompatible types in assignment (expression has type "object", variable has type "A") - -[case testSimpleTryExcept] -from typing import Undefined -try: - pass -except BaseException as e: - a, o = Undefined, Undefined # type: (BaseException, object) - e = a - e = o # Fail -class A: pass -class B: pass -[builtins fixtures/exception.py] -[out] -main, line 7: Incompatible types in assignment (expression has type "object", variable has type "BaseException") - -[case testTypeErrorInBlock] -from typing import Undefined -while object: - x = Undefined # type: A - x = object() - x = B() -class A: pass -class B: pass -[out] -main, line 4: Incompatible types in assignment (expression has type "object", variable has type "A") -main, line 5: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testTypeErrorInvolvingBaseException] -from typing import Undefined -x, a = Undefined, Undefined # type: (BaseException, A) -a = BaseException() # Fail -a = object() # Fail -x = object() # Fail -x = A() # Fail -x = BaseException() -class A: pass -[builtins fixtures/exception.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "BaseException", variable has type "A") -main, line 4: Incompatible types in assignment (expression has type "object", variable has type "A") -main, line 5: Incompatible types in assignment (expression has type "object", variable has type "BaseException") -main, line 6: Incompatible types in assignment (expression has type "A", variable has type "BaseException") - -[case testSimpleTryExcept2] -import typing -try: - pass -except BaseException as e: - e = object() # Fail - e = BaseException() -[builtins fixtures/exception.py] -[out] -main, line 5: Incompatible types in assignment (expression has type "object", variable has type "BaseException") - -[case testBaseClassAsExceptionTypeInExcept] -import typing -try: - pass -except Err as e: - e = BaseException() # Fail - e = Err() -class Err(BaseException): pass -[builtins fixtures/exception.py] -[out] -main, line 5: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") - -[case testMultipleExceptHandlers] -import typing -try: - pass -except BaseException as e: - pass -except Err as f: - f = BaseException() # Fail - f = Err() -class Err(BaseException): pass -[builtins fixtures/exception.py] -[out] -main, line 7: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") - -[case testTryExceptStatement] -import typing -try: - a = B() # type: A # Fail -except BaseException as e: - e = A() # Fail - e = Err() -except Err as f: - f = BaseException() # Fail - f = Err() -class A: pass -class B: pass -class Err(BaseException): pass -[builtins fixtures/exception.py] -[out] -main, line 3: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 5: Incompatible types in assignment (expression has type "A", variable has type "BaseException") -main, line 8: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") - -[case testTryExceptWithinFunction] -import typing -def f() -> None: - try: pass - except BaseException as e: - e = object() # Fail - e = BaseException() - except Err as f: - f = BaseException() # Fail - f = Err() -class Err(BaseException): pass -[builtins fixtures/exception.py] -[out] -main: In function "f": -main, line 5: Incompatible types in assignment (expression has type "object", variable has type "BaseException") -main, line 8: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") - -[case testTryWithElse] -import typing -try: pass -except BaseException: pass -else: - object(None) # E: Too many arguments for "object" -[builtins fixtures/exception.py] - -[case testExceptWithoutType] -import typing -try: - -None # E: Unsupported operand type for unary - (None) -except: - ~None # E: Unsupported operand type for ~ (None) -[builtins fixtures/exception.py] - -[case testRaiseWithoutArgument] -import typing -try: - None -except: - raise -[builtins fixtures/exception.py] - -[case testExceptWithMultipleTypes] -import typing -class E1(BaseException): pass -class E2(E1): pass -try: - pass -except (E1, E2): pass -except (E1, object): pass # E: Exception type must be derived from BaseException -except (object, E2): pass # E: Exception type must be derived from BaseException -[builtins fixtures/exception.py] - -[case testExceptWithMultipleTypes2] -import typing -class E1(BaseException): pass -class E2(E1): pass -try: - pass -except (E1, E2) as e1: - x = e1 # type: E1 - y = e1 # type: E2 # E: Incompatible types in assignment (expression has type "E1", variable has type "E2") -except (E2, E1) as e2: - a = e2 # type: E1 - b = e2 # type: E2 # E: Incompatible types in assignment (expression has type "E1", variable has type "E2") -[builtins fixtures/exception.py] - -[case testReuseTryExceptionVariable] -import typing -class E1(BaseException): pass -class E2(BaseException): pass -try: pass -except E1 as e: pass -try: pass -except E1 as e: pass -try: pass -except E2 as e: pass # E: Incompatible types in assignment (expression has type "E2", variable has type "E1") -[builtins fixtures/exception.py] - -[case testArbitraryExpressionAsExceptionType] -import typing -a = BaseException -try: pass -except a as b: - b = BaseException() - b = object() # E: Incompatible types in assignment (expression has type "object", variable has type "BaseException") -[builtins fixtures/exception.py] - -[case testInvalidExceptionCallable] -import typing -def exc() -> BaseException: pass -try: pass -except exc as e: pass # E: Exception type must be derived from BaseException -except BaseException() as b: pass # E: Exception type must be derived from BaseException -[builtins fixtures/exception.py] - -[case testOverloadedExceptionType] -from typing import overload -class E(BaseException): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, x) -> None: pass -try: - pass -except E as e: - e = E() - e = BaseException() # E: Incompatible types in assignment (expression has type "BaseException", variable has type "E") -[builtins fixtures/exception.py] - - --- Del statement --- ------------- - - -[case testDelStmtWithIndex] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -del b[a] -del b[b] # E: Argument 1 to "__delitem__" of "B" has incompatible type "B"; expected "A" -del a[a] # E: "A" has no attribute "__delitem__" -del a[b] # E: "A" has no attribute "__delitem__" -class B: - def __delitem__(self, index: 'A'): pass -class A: pass - -[case testDelStmtWithAttribute] -import typing -class A: - def f(self): pass - x = 0 -a = A() -del a.f -del a.x -del a.z # E: "A" has no attribute "z" - - --- Yield statement --- --------------- - - -[case testSimpleYield] -from typing import Iterator -def f() -> Iterator[int]: - yield 1 - yield '' # E: Incompatible types in yield (actual type "str", expected type "int") -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testYieldInFunctionReturningAny] -from typing import Any -def f() -> Any: - yield object() -[out] - -[case testYieldInFunctionReturningFunction] -from typing import Function -def f() -> Function[[], None]: - yield object() # E: Iterator function return type expected for "yield" -[out] -main: In function "f": - -[case testYieldInDynamicallyTypedFunction] -import typing -def f(): - yield f - -[case testWithInvalidInstanceReturnType] -import typing -def f() -> int: - yield 1 # E: Iterator function return type expected for "yield" -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testTypeInferenceContextAndYield] -from typing import List, Iterator -def f() -> 'Iterator[List[int]]': - yield [] - yield [object()] # E: List item 1 has incompatible type "object" -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testYieldAndReturnWithoutValue] -from typing import Iterator -def f() -> Iterator[int]: - yield 1 - return -[builtins fixtures/for.py] - -[case testYieldWithNoValue] -from typing import Iterator -def f() -> Iterator[None]: - yield -[builtins fixtures/for.py] - - --- Yield from statement --- -------------------- - --- Iterables --- ---------- - -[case testSimpleYieldFromWithIterator] -from typing import Iterator -def g() -> Iterator[str]: - yield '42' -def h() -> Iterator[int]: - yield 42 -def f() -> Iterator[str]: - yield from g() - yield from h() # E: Incompatible types in "yield from" (actual type "int", expected type "str") -[out] -main: In function "f": - -[case testYieldFromAppliedToAny] -from typing import Any -def g() -> Any: - yield object() -def f() -> Any: - yield from g() -[out] - -[case testYieldFromInFunctionReturningFunction] -from typing import Iterator, Function -def g() -> Iterator[int]: - yield 42 -def f() -> Function[[], None]: - yield from g() # E: Iterable function return type expected for "yield from" -[out] -main: In function "f": - -[case testYieldFromNotIterableReturnType] -from typing import Iterator -def g() -> Iterator[int]: - yield 42 -def f() -> int: - yield from g() # E: Iterable function return type expected for "yield from" -[out] -main: In function "f": - -[case testYieldFromNotAppliedIterator] -from typing import Iterator -def g() -> int: - return 42 -def f() -> Iterator[int]: - yield from g() # E: "yield from" can't be applied to "int" -[out] -main: In function "f": - -[case testYieldFromCheckIncompatibleTypesTwoIterables] -from typing import List, Iterator -def g() -> Iterator[List[int]]: - yield [2, 3, 4] -def f() -> Iterator[List[int]]: - yield from g() - yield from [1, 2, 3] # E: Incompatible types in "yield from" (actual type "int", expected type List[int]) -[builtins fixtures/for.py] -[out] -main: In function "f": - -[case testYieldFromNotAppliedToNothing] -def h(): - yield from # E: Parse error before end of line -[out] -main: In function "h": - -[case testYieldFromAndYieldTogether] -from typing import Iterator -def f() -> Iterator[str]: - yield "g1 ham" - yield from g() - yield "g1 eggs" -def g() -> Iterator[str]: - yield "g2 spam" - yield "g2 more spam" -[out] - - --- With statement --- -------------- - - -[case testSimpleWith] -import typing -class A: - def __enter__(self) -> None: pass - def __exit__(self, x, y, z) -> None: pass -with A(): - object(A) # E: Too many arguments for "object" - -[case testWithStmtAndInvalidExit] -import typing -class A: - def __enter__(self) -> None: pass - def __exit__(self, x, y) -> None: pass -with A(): # E: Too many arguments for "__exit__" of "A" - pass - -[case testWithStmtAndMissingExit] -import typing -class A: - def __enter__(self) -> None: pass -with A(): # E: "A" has no attribute "__exit__" - pass - -[case testWithStmtAndInvalidEnter] -import typing -class A: - def __enter__(self, x) -> None: pass - def __exit__(self, x, y, z) -> None: pass -with A(): # E: Too few arguments for "__enter__" of "A" - pass - -[case testWithStmtAndMissingEnter] -import typing -class A: - def __exit__(self, x, y, z) -> None: pass -with A(): # E: "A" has no attribute "__enter__" - pass - -[case testWithStmtAndMultipleExprs] -import typing -class A: - def __enter__(self) -> None: pass - def __exit__(self, x, y, z) -> None: pass -class B: - def __enter__(self) -> None: pass -with A(), B(): # E: "B" has no attribute "__exit__" - pass -with B(), A(): # E: "B" has no attribute "__exit__" - pass - -[case testWithStmtAndResult] -import typing -class B: pass -class A: - def __enter__(self) -> B: pass - def __exit__(self, x, y, z): pass -with A() as b: - b = B() - b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testWithStmtAndMultipleResults] -from typing import typevar, Generic -t = typevar('t') -class B: pass -class C: pass -class A(Generic[t]): - def __enter__(self) -> t: pass - def __exit__(self, x, y, z): pass -with A[B]() as b, A[C]() as c: - b = B() - c = C() - b = c # E: Incompatible types in assignment (expression has type "C", variable has type "B") - c = b # E: Incompatible types in assignment (expression has type "B", variable has type "C") - - --- Chained assignment --- ------------------ - - -[case testChainedAssignment] -import typing -class A: pass -class B: pass -x = y = A() -x = A() -y = A() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") -y = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testChainedAssignment2] -import typing -def f() -> None: - x = 1 - y = 'x' - x = y = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - x = y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[builtins fixtures/primitives.py] -[out] -main: In function "f": - -[case testChainedAssignmentWithType] -from typing import Undefined -x = y = None # type: int -x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -x = 1 -y = 1 - - --- Star assignment --- --------------- - - -[case testAssignListToStarExpr] -from typing import Undefined, List -bs, cs = Undefined, Undefined # type: List[A], List[B] -*bs, b = bs -*bs, c = cs # E: Incompatible types in assignment (expression has type List[B], variable has type List[A]) -*ns, c = cs -nc = cs - -class A: pass -class B: pass -[builtins fixtures/list.py] - - --- Type aliases --- ------------ - - -[case testSimpleTypeAlias] -import typing -foo = int -def f(x: foo) -> None: pass -f(1) -f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" - -[case testTypeAliasDefinedInAModule] -import typing -import m -def f(x: m.foo) -> None: pass -f(1) -f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -[file m.py] -import typing -foo = int - -[case testTypeAliasDefinedInAModule2] -import typing -from m import foo -def f(x: foo) -> None: pass -f(1) -f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -[file m.py] -import typing -foo = int - - --- nonlocal and global --- ------------------- - - -[case testTypeOfGlobalUsed] -import typing -g = A() -def f() -> None: - global g - g = B() - -class A(): pass -class B(): pass -[out] -main: In function "f": -main, line 5: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testTypeOfNonlocalUsed] -import typing -def f() -> None: - a = A() - def g() -> None: - nonlocal a - a = B() - -class A(): pass -class B(): pass -[out] -main: In function "g": -main, line 6: Incompatible types in assignment (expression has type "B", variable has type "A") - -[case testTypeOfOuterMostNonlocalUsed] -import typing -def f() -> None: - a = A() - def g() -> None: - a = B() - def h() -> None: - nonlocal a - a = A() - a = B() - -class A(): pass -class B(): pass -[out] -main: In function "h": -main, line 8: Incompatible types in assignment (expression has type "A", variable has type "B") diff --git a/mypy/test/data/check-super.test b/mypy/test/data/check-super.test deleted file mode 100644 index 950dc91bee08..000000000000 --- a/mypy/test/data/check-super.test +++ /dev/null @@ -1,68 +0,0 @@ --- Test cases for type checker related to super(). - - --- Supertype member reference --- -------------------------- - - -[case testAccessingSupertypeMethod] -from typing import Undefined -class B: - def f(self) -> 'B': pass -class A(B): - def f(self) -> 'A': - a, b = Undefined, Undefined # type: (A, B) - a = super().f() # E: Incompatible types in assignment (expression has type "B", variable has type "A") - a = super().g() # E: "g" undefined in superclass - b = super().f() -[out] -main: In member "f" of class "A": - -[case testAccessingSuperTypeMethodWithArgs] -from typing import Any, Undefined -class B: - def f(self, y: 'A') -> None: pass -class A(B): - def f(self, x: Any) -> None: - a, b = Undefined, Undefined # type: (A, B) - super().f(b) # E: Argument 1 to "f" of "B" has incompatible type "B"; expected "A" - super().f(a) - self.f(b) - self.f(a) -[out] -main: In member "f" of class "A": - -[case testAccessingSuperInit] -import typing -class B: - def __init__(self, x: A) -> None: pass -class A(B): - def __init__(self) -> None: - super().__init__(B(None)) # E: Argument 1 to "__init__" of "B" has incompatible type "B"; expected "A" - super().__init__() # E: Too few arguments for "__init__" of "B" - super().__init__(A()) -[out] -main: In member "__init__" of class "A": - -[case testAccessingSuperMemberWithDeepHierarchy] -import typing -class C: - def f(self) -> None: pass -class B(C): pass -class A(B): - def f(self) -> None: - super().g() # E: "g" undefined in superclass - super().f() -[out] -main: In member "f" of class "A": - -[case testAssignToBaseClassMethod] -import typing -class A: - def f(self) -> None: pass -class B(A): - def g(self) -> None: - super().f = None -[out] -main: In function "g": -main, line 6: Invalid assignment target diff --git a/mypy/test/data/check-tuples.test b/mypy/test/data/check-tuples.test deleted file mode 100644 index b197da2b7736..000000000000 --- a/mypy/test/data/check-tuples.test +++ /dev/null @@ -1,656 +0,0 @@ --- Normal assignment and subtyping --- ------------------------------- - - -[case testAssignmentWithTupleTypes] -from typing import Undefined, Tuple -t1 = Undefined # type: Tuple[A] -t2 = Undefined # type: Tuple[B] -t3 = Undefined # type: Tuple[A, A] -t4 = Undefined # type: Tuple[A, B] -t5 = Undefined # type: Tuple[B, A] - -t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") -t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") -t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") -t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") - -# Ok -t1 = t1 -t2 = t2 -t3 = t3 -t4 = t4 -t5 = t5 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testTupleSubtyping] -from typing import Undefined, Tuple -t1 = Undefined # type: Tuple[A, A] -t2 = Undefined # type: Tuple[A, B] -t3 = Undefined # type: Tuple[B, A] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") -t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") - -t1 = t2 -t1 = t3 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testTupleCompatibilityWithOtherTypes] -from typing import Undefined, Tuple -a, o = Undefined, Undefined # type: (A, object) -t = Undefined # type: Tuple[A, A] - -a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") -t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") -t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") -# TODO: callable types + tuples - -# Ok -o = t -t = None - -class A: pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes] -from typing import Undefined, Tuple -t1 = Undefined # type: Tuple[A, Tuple[A, A]] -t2 = Undefined # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes2] -from typing import Undefined, Tuple -t1 = Undefined # type: (A, (A, A)) -t2 = Undefined # type: B, (B, B) - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testSubtypingWithNamedTupleType] -from typing import Undefined, Tuple -t1 = Undefined # type: Tuple[A, A] -t2 = Undefined # type: tuple - -t1 = t2 # E: Incompatible types in assignment (expression has type "tuple", variable has type "Tuple[A, A]") -t2 = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testTupleInitializationWithNone] -from typing import Tuple -t = None # type: Tuple[A, A] -t = None -class A: pass -[builtins fixtures/tuple.py] - - --- Tuple expressions --- ----------------- - - -[case testTupleExpressions] -from typing import Undefined, Tuple -t1 = Undefined # type: tuple -t2 = Undefined # type: Tuple[A] -t3 = Undefined # type: Tuple[A, B] - -a, b, c = Undefined, Undefined, Undefined # type: (A, B, C) - -t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") -t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") -t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") - -t1 = () -t1 = (a,) -t2 = (a,) -t3 = (a, b) -t3 = (a, c) -t3 = (None, None) - -class A: pass -class B: pass -class C(B): pass -[builtins fixtures/tuple.py] - -[case testVoidValueInTuple] -import typing -(None, f()) # E: "f" does not return a value -(f(), None) # E: "f" does not return a value - -def f() -> None: pass -[builtins fixtures/tuple.py] - - --- Indexing --- -------- - - -[case testIndexingTuples] -from typing import Undefined, Tuple -t1 = Undefined # type: Tuple[A, B] -t2 = Undefined # type: Tuple[A] -a, b = Undefined, Undefined # type: (A, B) -n = 0 - -a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[2] # E: Tuple index out of range -t1[3] # E: Tuple index out of range -t2[1] # E: Tuple index out of range -t1[n] # E: Tuple index must an integer literal -b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[0] -b = t1[1] -a = t1[(0)] - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testIndexingTuplesWithNegativeIntegers] -from typing import Undefined, Tuple -t1 = Undefined(Tuple[A, B]) -t2 = Undefined(Tuple[A]) -a, b = Undefined(A), Undefined(B) - -a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[-3] # E: Tuple index out of range -t1[-4] # E: Tuple index out of range -b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[-2] -b = t1[-1] -a = t2[(-1)] - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testAssigningToTupleItems] -from typing import Undefined, Tuple -t = Undefined # type: Tuple[A, B] -n = 0 - -t[0] = A() # E: Unsupported target for indexed assignment -t[2] = A() # E: Unsupported target for indexed assignment -t[n] = A() # E: Unsupported target for indexed assignment - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - - --- Multiple assignment --- ------------------- - - -[case testMultipleAssignmentWithTuples] -from typing import Undefined, Tuple -t1 = Undefined # type: Tuple[A, B] -t2 = Undefined # type: Tuple[A, B, A] -a, b = Undefined, Undefined # type: (A, B) - -a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = t1 -a, b, a = t2 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithInvalidNumberOfValues] -from typing import Undefined, Tuple -t1 = Undefined # type: Tuple[A, A, A] -a = Undefined # type: A - -a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) -a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) - -a, a, a = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithTupleExpressionRvalue] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) - -a, b = a, a # Fail -a, b = b, a # Fail - -a, b = a, b -a, a = a, a - -class A: pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main, line 4: Incompatible types in assignment (expression has type "A", variable has type "B") -main, line 5: Incompatible types in assignment (expression has type "B", variable has type "A") -main, line 5: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testSubtypingInMultipleAssignment] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) - -b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = b, b -b, a = b, b - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testInitializationWithMultipleValues] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) - -a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable -a4, b4 = None # type: (A, B) # E: ''None'' object is not iterable -a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) - -ax, bx = a, b # type: (A, B) - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithNonTupleRvalue] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -def f(): pass - -a, b = None # E: ''None'' object is not iterable -a, b = a # E: '__main__.A' object is not iterable -a, b = f # E: 'def () -> Any' object is not iterable - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithIndexedLvalues] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -aa, bb = Undefined, Undefined # type: (AA, BB) - -a[a], b[b] = a, bb # E: Incompatible types in assignment -a[a], b[b] = aa, b # E: Incompatible types in assignment -a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" -a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" -a[a], b[b] = aa, bb - -class A: - def __setitem__(self, x: 'A', y: 'AA') -> None: pass -class B: - def __setitem__(self, x: 'B', y: 'BB') -> None: pass - -class AA: pass -class BB: pass -[builtins fixtures/tuple.py] - -[case testMultipleDeclarationWithParentheses] -from typing import Undefined -(a, b) = (None, None) # type: int, str -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -a = 1 -b = '' - -[case testMultipleAssignmentWithExtraParentheses] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) - -(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -(a, b) = (a, b) -((a), (b)) = ((a), (b)) -[a, b] = a, b - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentUsingUndefinedWithArgument] -from typing import Undefined, Tuple -a, b = Undefined(Tuple[int, str]) -a = 1 -b = '' -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testMultipleAssignmentUsingRepeatedUndefinedWithArgument] -from typing import Undefined -a, b = Undefined(int), Undefined(str) -a = 1 -b = '' -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testMultipleAssignmentUsingUndefinedWithArgumentAndNormalExpression] -from typing import Undefined -a, b = Undefined(int), '' -a = 1 -b = '' -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - - --- Assignment to starred expressions --- --------------------------------- - - -[case testAssignmentToStarMissingAnnotation] -from typing import Undefined, List -t = 1, 2 -a, b, *c = 1, 2 # E: Need type annotation for variable -aa, bb, *cc = t # E: Need type annotation for variable -[builtins fixtures/list.py] - -[case testAssignmentToStarAnnotation] -from typing import Undefined, List -li, lo = Undefined, Undefined # type: List[int], List[object] -a, b, *c = 1, 2 # type: int, int, *List[int] -c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) -c = li -[builtins fixtures/list.py] - -[case testAssignmentToStarCount1] -from typing import Undefined, List -ca = Undefined # type: List[int] -c = [1] -a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) -a, b, *c = 1, 2 -a, b, *c = 1, 2, 3 -a, b, *c = 1, 2, 3, 4 -[builtins fixtures/list.py] - -[case testAssignmentToStarCount2] -from typing import Undefined, List -ca = Undefined # type: List[int] -t1 = 1, -t2 = 1, 2 -t3 = 1, 2, 3 -t4 = 1, 2, 3, 4 -c = [1] -a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) -a, b, *c = t2 -a, b, *c = t3 -a, b, *c = t4 -[builtins fixtures/list.py] - -[case testAssignmentToStarFromAny] -from typing import Undefined, Any -a, c = Any(1), C() -p, *q = a -c = a -c = q - -class C: pass - -[case testAssignmentToComplexStar] -from typing import Undefined, List -li = Undefined # type: List[int] -a, *(li) = 1, -a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) -a, *(b, c) = 1, 2, 3 -a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) -[builtins fixtures/list.py] - -[case testAssignmentToStarFromTupleType] -from typing import Undefined, List, Tuple -li = Undefined # type: List[int] -la = Undefined # type: List[A] -ta = Undefined # type: Tuple[A, A, A] -a, *la = ta -a, *li = ta # E -a, *na = ta -na = la -na = a # E - -class A: pass -[builtins fixtures/list.py] -[out] -main, line 6: List item 1 has incompatible type "A" -main, line 6: List item 2 has incompatible type "A" -main, line 9: Incompatible types in assignment (expression has type "A", variable has type List[A]) - -[case testAssignmentToStarFromTupleInference] -from typing import Undefined, List -li = Undefined # type: List[int] -la = Undefined # type: List[A] -a, *l = A(), A() -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListInference] -from typing import Undefined, List -li = Undefined # type: List[int] -la = Undefined # type: List[A] -a, *l = [A(), A()] -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromTupleTypeInference] -from typing import Undefined, List, Tuple -li = Undefined # type: List[int] -la = Undefined # type: List[A] -ta = Undefined # type: Tuple[A, A, A] -a, *l = ta -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListTypeInference] -from typing import Undefined, List -li = Undefined # type: List[int] -la = Undefined # type: List[A] -a, *l = la -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - - --- Nested tuple assignment --- ---------------------------- - - -[case testNestedTupleAssignment1] -from typing import Undefined -a1, b1, c1 = Undefined, Undefined, Undefined # type: (A, B, C) -a2, b2, c2 = Undefined, Undefined, Undefined # type: (A, B, C) - -a1, (b1, c1) = a2, (b2, c2) -a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) -a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail - -class A: pass -class B: pass -class C: pass -[out] -main, line 7: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testNestedTupleAssignment2] -from typing import Undefined -a1, b1, c1 = Undefined, Undefined, Undefined # type: (A, B, C) -a2, b2, c2 = Undefined, Undefined, Undefined # type: (A, B, C) -t = a1, b1 - -a2, b2 = t -(a2, b2), c2 = t, c1 -(a2, c2), c2 = t, c1 # Fail -t, c2 = (a2, b2), c2 -t, c2 = (a2, a2), c2 # Fail -t = a1, a1, a1 # Fail -t = a1 # Fail -a2, a2, a2 = t # Fail -a2, = t # Fail -a2 = t # Fail - -class A: pass -class B: pass -class C: pass -[out] -main, line 8: Incompatible types in assignment (expression has type "B", variable has type "C") -main, line 10: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -main, line 11: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") -main, line 12: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") -main, line 13: Need more than 2 values to unpack (3 expected) -main, line 14: Too many values to unpack (1 expected, 2 provided) -main, line 15: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") - - --- Error messages --- -------------- - - -[case testTupleErrorMessages] -from typing import Undefined -a = Undefined # type: A - -(a, a) + a # E: Unsupported left operand type for + ("tuple") -a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") -f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" -(a, a).foo # E: "tuple" has no attribute "foo" - -def f(x: 'A') -> None: pass - -class A: - def __add__(self, x: 'A') -> 'A': pass -[builtins fixtures/tuple.py] - -[case testLargeTuplesInErrorMessages] -from typing import Undefined -a = Undefined # type: LongTypeName -a + (a, a, a, a, a, a, a) # Fail - -class LongTypeName: - def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass -[builtins fixtures/tuple.py] -[out] -main, line 3: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) - - --- Tuple methods --- ------------- - - -[case testTupleMethods] -from typing import Undefined, Tuple -t = Undefined # type: Tuple[int, str] -i = 0 -s = '' -b = bool() - -s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") -i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") -t.foo # E: "tuple" has no attribute "foo" - -i = t.__len__() -s = t.__str__() -b = s in t - -[file builtins.py] -class object: - def __init__(self) -> None: pass -class tuple: - def __len__(self) -> int: pass - def __str__(self) -> str: pass - def __contains__(self, o: object) -> bool: pass -class int: pass -class str: pass -class bool: pass -class type: pass -class function: pass - - --- For loop over tuple --- ------------------- - - -[case testForLoopOverTuple] -import typing -t = 1, 2 -for x in t: - x = 1 - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/for.py] - -[case testForLoopOverEmptyTuple] -import typing -t = () -for x in t: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverNoneValuedTuple] -import typing -t = () -for x in None, None: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverTupleAndSubtyping] -import typing -class A: pass -class B(A): pass -for x in B(), A(): - x = A() - x = B() - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") -[builtins fixtures/for.py] - -[case testTupleIterable] -y = 'a' -x = sum((1,2)) -y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[builtins fixtures/tuple.py] diff --git a/mypy/test/data/check-type-checks.test b/mypy/test/data/check-type-checks.test deleted file mode 100644 index 6f6ca329fa9b..000000000000 --- a/mypy/test/data/check-type-checks.test +++ /dev/null @@ -1,120 +0,0 @@ --- Conditional type checks. - - -[case testSimpleIsinstance] -from typing import Undefined -x = Undefined(object) -n = Undefined(int) -s = Undefined(str) -n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") -if isinstance(x, int): - n = x - s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") -[builtins fixtures/isinstance.py] - -[case testSimpleIsinstance2] -import typing -def f(x: object, n: int, s: str) -> None: - n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") - if isinstance(x, int): - n = x - s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testSimpleIsinstance3] -from typing import Undefined -class A: - x = Undefined(object) - n = Undefined(int) - s = Undefined(str) - n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") - if isinstance(x, int): - n = x - s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - else: - n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") -[builtins fixtures/isinstance.py] -[out] -main: In class "A": - -[case testMultipleIsinstanceTests] -import typing -class A: pass -class B(A): pass -def f(x: object, a: A, b: B, c: int) -> None: - if isinstance(x, A): - if isinstance(x, B): - b = x - x = a - a = x - c = x # E: Incompatible types in assignment (expression has type "A", variable has type "int") -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testMultipleIsinstanceTests2] -import typing -class A: pass -class B(A): pass -def f(x: object, y: object, n: int, s: str) -> None: - if isinstance(x, int): - if isinstance(y, str): - n = x - s = y - s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - n = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - s = y # E: Incompatible types in assignment (expression has type "object", variable has type "str") - n = y # E: Incompatible types in assignment (expression has type "object", variable has type "int") - n = x -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testIsinstanceAndElif] -import typing -def f(x: object, n: int, s: str) -> None: - n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") - if isinstance(x, int): - n = x - s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - elif isinstance(x, str): - s = x - n = x # E: Incompatible types in assignment (expression has type "str", variable has type "int") - else: - n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") - s = x # E: Incompatible types in assignment (expression has type "object", variable has type "str") - n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testIsinstanceAndAnyType] -from typing import Any -def f(x: Any, n: int, s: str) -> None: - s = x - if isinstance(x, int): - n = x - s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - s = x -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testIsinstanceAndGenericType] -from typing import typevar, Generic -T = typevar('T') -class C(Generic[T]): - def f(self, x: T) -> None: pass -def f(x: object) -> None: - if isinstance(x, C): - x.f(1) - x.f('') - x.g() # E: C[Any] has no attribute "g" - x.g() # E: "object" has no attribute "g" -[builtins fixtures/isinstance.py] -[out] -main: In function "f": diff --git a/mypy/test/data/check-typevar-values.test b/mypy/test/data/check-typevar-values.test deleted file mode 100644 index 9f7c4f29f13f..000000000000 --- a/mypy/test/data/check-typevar-values.test +++ /dev/null @@ -1,487 +0,0 @@ --- Test cases for type variables with values restriction. - - -[case testCallGenericFunctionWithTypeVarValueRestriction] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> None: pass -f(1) -f('x') -f(object()) # E: Type argument 1 of "f" has incompatible value "object" - -[case testCallGenericFunctionWithTypeVarValueRestrictionUsingContext] -from typing import typevar, List -T = typevar('T', values=(int, str)) -def f(x: T) -> List[T]: pass -i = [1] -s = ['x'] -o = [object()] -i = f(1) -s = f('') -o = f(1) # E: Type argument 1 of "f" has incompatible value "object" -[builtins fixtures/list.py] - -[case testCallGenericFunctionWithTypeVarValueRestrictionAndAnyArgs] -from typing import typevar, Any -T = typevar('T', values=(int, str)) -def f(x: T) -> None: pass -f(Any(object())) -[out] - -[case testCallGenericFunctionWithTypeVarValueRestrictionInDynamicFunc] -from typing import typevar, Any -T = typevar('T', values=(int, str)) -def f(x: T) -> None: pass -def g(): - f(object()) -[out] - -[case testCallGenericFunctionWithTypeVarValueRestrictionUsingSubtype] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> None: pass -class S(str): pass -f(S()) -[out] - -[case testCheckGenericFunctionBodyWithTypeVarValues] -from typing import typevar -class A: - def f(self, x: int) -> A: return self -class B: - def f(self, x: int) -> B: return self -AB = typevar('AB', values=(A, B)) -def f(x: AB) -> AB: - x = x.f(1) - return x.f(1) - -[case testCheckGenericFunctionBodyWithTypeVarValues2] -from typing import typevar -class A: - def f(self) -> A: return A() - def g(self) -> B: return B() -class B: - def f(self) -> A: return A() - def g(self) -> B: return B() -AB = typevar('AB', values=(A, B)) -def f(x: AB) -> AB: - return x.f() # Error -def g(x: AB) -> AB: - return x.g() # Error -[out] -main: In function "f": -main, line 10: Incompatible return value type: expected __main__.B*, got __main__.A -main: In function "g": -main, line 12: Incompatible return value type: expected __main__.A*, got __main__.B - -[case testTypeInferenceAndTypeVarValues] -from typing import typevar -class A: - def f(self) -> A: return self - def g(self) -> B: return B() -class B: - def f(self) -> B: return self - def g(self) -> B: return B() -AB = typevar('AB', values=(A, B)) -def f(x: AB) -> AB: - y = x - if y: - return y.f() - else: - return y.g() # E: Incompatible return value type: expected __main__.A*, got __main__.B -[out] -main: In function "f": - -[case testTypeApplicationAndTypeVarValues] -from typing import typevar, List -T = typevar('T', values=(int, str)) -def f(x: T) -> List[T]: - return List[T]() -def g(x: T) -> List[T]: - return List[int]() # E: Incompatible return value type: expected builtins.list[builtins.str*], got builtins.list[builtins.int*] -[builtins fixtures/list.py] -[out] -main: In function "g": - -[case testTypeDeclaredBasedOnTypeVarWithValues] -from typing import typevar, Undefined -T = typevar('T', values=(int, str)) -def f(x: T) -> T: - a = Undefined(T) - b = None # type: T - a = x - b = x - a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[out] -main: In function "f": - -[case testIsinstanceAndTypeVarValues] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> T: - if isinstance(x, int): - return 2 -def g(x: T) -> T: - if isinstance(x, str): - return '' -def h(x: T) -> T: - if isinstance(x, int): - return '' # E: Incompatible return value type: expected builtins.int*, got builtins.str - return x -[builtins fixtures/isinstance.py] -[out] -main: In function "h": - -[case testIsinstanceAndTypeVarValues2] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> T: - if isinstance(x, int): - return 2 - else: - return '' -def g(x: T) -> T: - if isinstance(x, int): - return '' # E: Incompatible return value type: expected builtins.int*, got builtins.str - else: - return 2 # E: Incompatible return value type: expected builtins.str*, got builtins.int - return x -[builtins fixtures/isinstance.py] -[out] -main: In function "g": - -[case testIsinstanceAndTypeVarValues3] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> T: - if isinstance(x, int): - y = 1 - else: - y = '' - return y -[builtins fixtures/isinstance.py] - -[case testIsinstanceAndTypeVarValues4] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> T: - if isinstance(x, int): - y = 1 - else: - y = object() - return y # E: Incompatible return value type: expected builtins.str*, got builtins.object -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testIsinstanceAndTypeVarValues5] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> T: - if isinstance(x, int): - y = object() - else: - y = '' - return y # E: Incompatible return value type: expected builtins.int*, got builtins.object -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testIsinstanceWithUserDefinedTypeAndTypeVarValues] -from typing import typevar, Undefined -class A: pass -class B: pass -T = typevar('T', values=(A, B)) -def f(x: T) -> None: - y = x - if isinstance(x, A): - # This is checked both when x is A or B, since A and B are overlapping - # types. - x = y - x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") - else: - x = B() - x = y - x.foo() # E: "B" has no attribute "foo" -S = typevar('S', values=(int, str)) -def g(x: S) -> None: - y = x - if isinstance(x, int): - x = y -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testIsinstanceWithUserDefinedTypeAndTypeVarValues2] -from typing import typevar, Undefined -class S(str): pass -T = typevar('T', values=(S, int)) -def f(x: T) -> None: - y = x - if isinstance(x, S): - # This is checked only when type of x is str. - x = y - x = S() - x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "S") - else: - x = y - x = 1 - x = S() # E: Incompatible types in assignment (expression has type "S", variable has type "int") -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - -[case testTypeVarValuesAndNestedCalls] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(m: T) -> int: pass -def h(x: int) -> int: pass -def g(a: T) -> None: - h(f(a)) -[out] - -[case testGenericTypeWithTypevarValues] -from typing import typevar, Generic, Undefined, Any -X = typevar('X', values=(int, str)) -class A(Generic[X]): pass -a = Undefined(A[int]) -b = Undefined(A[str]) -d = Undefined(A[object]) # E: Invalid type argument value for "A" -c = Undefined(A[Any]) - -[case testGenericTypeWithTypevarValuesAndTypeApplication] -from typing import typevar, Generic, Undefined, Any -X = typevar('X', values=(int, str)) -class A(Generic[X]): pass -A[int]() -A[str]() -A[Any]() -A[object]() # E: Type argument 1 of "A" has incompatible value "object" - -[case testConstructGenericTypeWithTypevarValuesAndTypeInference] -from typing import typevar, Generic, Any -X = typevar('X', values=(int, str)) -class A(Generic[X]): - def __init__(self, x: X) -> None: pass -A(1) -A('x') -A(Any(object())) -A(object()) # E: Type argument 1 of "A" has incompatible value "object" - -[case testGenericTypeWithTypevarValuesAndTypevarArgument] -from typing import typevar, Generic, Undefined -class C: pass -X = typevar('X', values=(int, str)) -Y = typevar('Y', values=(int, C)) -Z = typevar('Z') -class D(Generic[X]): - def __init__(self, x: X) -> None: pass -def f(x: X) -> None: - a = Undefined(D[X]) -def g(x: Y) -> None: - a = Undefined(D[Y]) -def h(x: Z) -> None: - a = Undefined(D[Z]) -[out] -main: In function "g": -main, line 11: Invalid type argument value for "D" -main: In function "h": -main, line 13: Type variable "Z" not valid as type argument value for "D" - -[case testGenericTypeWithTypevarValuesAndSubtypePromotion] -from typing import typevar, Generic, Undefined -X = typevar('X', values=(int, str)) -class S(str): pass -class C(Generic[X]): - def __init__(self, x: X) -> None: pass -x = Undefined(C[str]) -y = C(S()) -x = y -y = x -y = C[int](1) # E: Incompatible types in assignment (expression has type C[int], variable has type C[str]) - -[case testGenericTypeBodyWithTypevarValues] -from typing import typevar, Generic -class A: - def f(self, x: int) -> None: pass - def g(self, x: int) -> None: pass - def h(self, x: str) -> None: pass -class B: - def f(self, x: int) -> None: pass - def g(self, x: str) -> None: pass - def h(self, x: int) -> None: pass -X = typevar('X', values=(A, B)) -class C(Generic[X]): - def f(self, x: X) -> None: - x.f(1) - x.g(1) # E: Argument 1 to "g" of "B" has incompatible type "int"; expected "str" - x.h(1) # E: Argument 1 to "h" of "A" has incompatible type "int"; expected "str" -[out] -main: In member "f" of class "C": - -[case testAttributeInGenericTypeWithTypevarValues1] -from typing import typevar, Generic, Undefined -X = typevar('X', values=(int, str)) -class C(Generic[X]): - x = Undefined(X) - def f(self, x: X) -> None: - self.x = x - self.x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[out] -main: In member "f" of class "C": - -[case testAttributeInGenericTypeWithTypevarValues2] -from typing import typevar, Generic, Undefined -X = typevar('X', values=(int, str)) -class C(Generic[X]): - x = Undefined(X) -cn = C[int]() -cn.x = 1 -cn.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -cs = C[str]() -cs.x = '' -cs.x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testInferredAttributeInGenericClassBodyWithTypevarValues] -from typing import typevar, Generic, Undefined -X = typevar('X', values=(int, str)) -class C(Generic[X]): - x = 1 -C.x = 1 -C.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testMultipleClassTypevarsWithValues1] -from typing import typevar, Generic -class A: - def f(self, x: int) -> None: pass -class B: - def f(self, x: str) -> None: pass -X = typevar('X', values=(A, B)) -Y = typevar('Y', values=(int, str)) -class C(Generic[X, Y]): - def f(self, x: X, y: Y) -> None: - x.f(y) -[out] -main: In member "f" of class "C": -main, line 10: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -main, line 10: Argument 1 to "f" of "B" has incompatible type "int"; expected "str" - -[case testMultipleClassTypevarsWithValues2] -from typing import typevar, Generic, Undefined -class A: pass -class B: pass -X = typevar('X', values=(A, B)) -Y = typevar('Y', values=(int, str)) -class C(Generic[X, Y]): pass -a = Undefined(C[A, int]) -b = Undefined(C[B, str]) -c = Undefined(C[int, int]) # E: Invalid type argument value for "C" -c = Undefined(C[A, A]) # E: Invalid type argument value for "C" - -[case testCallGenericFunctionUsingMultipleTypevarsWithValues] -from typing import typevar -class A: pass -class B: pass -X = typevar('X', values=(A, B)) -Y = typevar('Y', values=(int, str)) -def f(x: X, y: Y) -> None: pass -f(A(), '') -f(B(), 1) -f(A(), A()) # E: Type argument 2 of "f" has incompatible value "A" -f(1, 1) # E: Type argument 1 of "f" has incompatible value "int" - -[case testGenericFunctionWithNormalAndRestrictedTypevar] -from typing import typevar, Generic, Undefined -X = typevar('X') -Y = typevar('Y', values=(int, str)) -class C(Generic[Y]): - def __init__(self, y: Y) -> None: pass -def f(x: X, y: Y, z: int) -> None: - C(y) - C(x) # Error - z = x # Error - z = y # Error - y.foo # Error -[out] -main: In function "f": -main, line 8: Type argument 1 of "C" has incompatible value "X" -main, line 9: Incompatible types in assignment (expression has type "X", variable has type "int") -main, line 10: Incompatible types in assignment (expression has type "str", variable has type "int") -main, line 11: "int" has no attribute "foo" -main, line 11: "str" has no attribute "foo" - -[case testTypeVarWithValueInferredFromObjectReturnTypeContext] -from typing import typevar -T = typevar('T', values=(int, str)) -def c1(x: object) -> None: pass -def c2(x: int) -> None: pass -def c3(x: str) -> None: pass -def g(x: T) -> T: pass -c1(g('')) -c2(g(1)) -c3(g('')) -c2(g('')) # E: Argument 1 to "c2" has incompatible type "str"; expected "int" -c3(g(1)) # E: Argument 1 to "c3" has incompatible type "int"; expected "str" - -[case testTypeVarWithValueInferredFromObjectReturnTypeContext2] -from typing import typevar -T = typevar('T', values=(int, str)) -class ss(str): pass -def c(x: ss) -> None: pass -def g(x: T) -> T: pass -c(g('')) -c(g(1)) -[out] -main, line 6: Argument 1 to "c" has incompatible type "str"; expected "ss" -main, line 7: Argument 1 to "c" has incompatible type "int"; expected "ss" - - --- Special cases --- ------------- - - -[case testTypevarValuesSpecialCase1] -from typing import typevar, Generic -from abc import abstractmethod -T = typevar('T', values=(int, str)) -class A(Generic[T]): - @abstractmethod - def f(self) -> 'A[T]': pass -class B(A[str]): - @abstractmethod - def f(self) -> 'B': pass -class C(A[str]): - @abstractmethod - def f(self) -> int: # E: Return type of "f" incompatible with supertype "A" - pass -[out] -main: In class "C": - -[case testDefaultArgumentValueInGenericClassWithTypevarValues] -from typing import typevar, Generic -T = typevar('T', values=(int, str)) -class C(Generic[T]): - def f(self, x: int = None) -> None: pass - -[case testTypevarValuesWithOverloadedFunctionSpecialCase] -from typing import typevar, overload, Function - -T = typevar('T', values=(int, str)) -def f(x: T) -> None: - y = m(g, x) - x = y - y = object() - -A = typevar('A') -R = typevar('R') -def m(f: Function[[A], R], it: A) -> A: pass - -@overload -def g(x: int) -> int: return x -@overload -def g(x: str) -> str: return x -[out] -main: In function "f": -main, line 7: Incompatible types in assignment (expression has type "object", variable has type "int") -main, line 7: Incompatible types in assignment (expression has type "object", variable has type "str") diff --git a/mypy/test/data/check-unions.test b/mypy/test/data/check-unions.test deleted file mode 100644 index 7b69d65d057b..000000000000 --- a/mypy/test/data/check-unions.test +++ /dev/null @@ -1,99 +0,0 @@ --- Type checking of union types - -[case testUnion1] -from typing import Union -def f(x: Union[int, str]) -> None: - if isinstance(x, int): - y = 1 - y = x - elif isinstance(x, str): - z = 'a' - z = x -[builtins fixtures/isinstance.py] - -[case testUnion2] -from typing import Union -def f(x: Union[int, str]) -> None: - if isinstance(x, int): - y = 1 - y = x - else: - z = 'a' - z = x -[builtins fixtures/isinstance.py] - -[case testUnion3] -from typing import Union -def f(x: Union[int, str]) -> None: - if isinstance(x, int): - y = 1 - y = x - else: - z = 2 - z = x # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/isinstance.py] -[out] -main: In function "f": - - -[case testUnionAttributeAccess] -from typing import Union, Undefined - -class A: y = 1 -class B: y = 2 -class C: pass - -w = Undefined # type: Union[A, B] -x = Undefined # type: Union[A, C] -y = Undefined # type: int -z = Undefined # type: str - -y = w.y -z = w.y # E: Incompatible types in assignment (expression has type "int", variable has type "str") -w.y = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -y = x.y # E: Some element of union has no attribute "y" -z = x.y # E: Some element of union has no attribute "y" - -[builtins fixtures/isinstance.py] - -[case testUnionMethodCalls] -from typing import Union, Undefined - -class A: - def foo(self) -> int: pass -class B: - def foo(self) -> int: pass -class C: - def foo(self) -> str: pass - -x = Undefined # type: Union[A, B] -y = Undefined # type: Union[A, C] -i = Undefined # type: int - -x.foo() -y.foo() -i = x.foo() -i = y.foo() # E: Incompatible types in assignment (expression has type "Union[int, str]", variable has type "int") - -[builtins fixtures/isinstance.py] - -[case testUnionIndexing] -from typing import Union, List, Undefined -x = Undefined # type: Union[List[int], str] -x[2] -x[2] + 1 # E: Unsupported operand types for + (likely involving Union) -[builtins fixtures/isinstancelist.py] - -[case testUnionAsOverloadArg] -from typing import Union, overload -@overload -def f(x: Union[int, str]) -> int: pass -@overload -def f(x: type) -> str: pass -x = 0 -x = f(1) -x = f('') -s = '' -s = f(int) -s = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") -x = f(int) # E: Incompatible types in assignment (expression has type "str", variable has type "int") diff --git a/mypy/test/data/check-unreachable-code.test b/mypy/test/data/check-unreachable-code.test deleted file mode 100644 index c6f266123a32..000000000000 --- a/mypy/test/data/check-unreachable-code.test +++ /dev/null @@ -1,88 +0,0 @@ --- Type checker test cases for conditional checks that result in some --- blocks classified as unreachable (they are not type checked or semantically --- analyzed). --- --- For example, we skip blocks that won't be executed on the active --- Python version. - -[case testConditionalTypeAliasPY3] -import typing -def f(): pass -PY3 = f() -if PY3: - t = int - x = object() + 'x' # E: Unsupported left operand type for + ("object") -else: - t = str - y = 'x' / 1 -x -z = 1 # type: t - -[case testConditionalTypeAliasPY3_python2] -import typing -def f(): pass -PY3 = f() -if PY3: - t = int - x = object() + 'x' -else: - t = str - y = 'x' / 1 # E: "str" has no attribute "__div__" -y -z = '' # type: t - -[case testConditionalAssignmentPY2] -import typing -def f(): pass -PY2 = f() -if PY2: - x = object() + 'x' -else: - y = 'x' / 1 # E: Unsupported left operand type for / ("str") -y - -[case testConditionalAssignmentPY2_python2] -import typing -def f(): pass -PY2 = f() -if PY2: - x = object() + 'x' # E: Unsupported left operand type for + ("object") -else: - y = 'x' / 1 -x - -[case testConditionalImport] -import typing -def f(): pass -PY2 = f() -if PY2: - import fuzzybar - from barbar import * - from pawwaw import a, bc -else: - import m -[file m.py] -import typing -x = 1 -x = 'a' -[out] -In module imported in main, line 9: -tmp/m.py, line 3: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNegatedMypyConditional] -import typing -MYPY = 0 -if not MYPY: - import xyz753 -else: - import pow123 # E: No module named 'pow123' -[builtins fixtures/bool.py] - -[case testMypyConditional] -import typing -MYPY = 0 -if MYPY: - None + 1 # E: Unsupported left operand type for + (None) -else: - None + '' -[builtins fixtures/bool.py] diff --git a/mypy/test/data/check-unsupported.test b/mypy/test/data/check-unsupported.test deleted file mode 100644 index 050b0dabdd31..000000000000 --- a/mypy/test/data/check-unsupported.test +++ /dev/null @@ -1,15 +0,0 @@ --- Tests for unsupported features - - -[case testDecorateOverloadedFunction] -# The error messages are not the most informative ever. -def d(x): pass -@d -def f(): pass -def f(x): pass -def g(): pass -@d -def g(x): pass -[out] -main, line 5: Name 'f' already defined -main, line 7: Name 'g' already defined diff --git a/mypy/test/data/check-varargs.test b/mypy/test/data/check-varargs.test deleted file mode 100644 index 1d0e103fdfe1..000000000000 --- a/mypy/test/data/check-varargs.test +++ /dev/null @@ -1,518 +0,0 @@ --- Test cases for the type checker related to varargs. - - --- Varargs within body --- ------------------- - - -[case testVarArgsWithinFunction] -from typing import Undefined, List -def f( *b: 'B') -> None: - ab = Undefined # type: List[B] - ac = Undefined # type: List[C] - b = ac # E: Incompatible types in assignment (expression has type List[C], variable has type List[B]) - ac = b # E: Incompatible types in assignment (expression has type List[B], variable has type List[C]) - b = ab - ab = b - -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] -main: In function "f": - - --- Calling varargs function --- ------------------------ - - -[case testCallingVarArgsFunction] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: B -c = Undefined # type: C - -f(c) # E: Argument 1 to "f" has incompatible type "C"; expected "A" -f(a, b, c) # E: Argument 3 to "f" has incompatible type "C"; expected "A" -f(g()) # E: "g" does not return a value -f(a, g()) # E: "g" does not return a value -f() -f(a) -f(b) -f(a, b, a, b) - -def f( *a: 'A') -> None: pass - -def g() -> None: pass - -class A: pass -class B(A): pass -class C: pass -[builtins fixtures/list.py] - -[case testCallingVarArgsFunctionWithAlsoNormalArgs] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: B -c = Undefined # type: C - -f(a) # E: Argument 1 to "f" has incompatible type "A"; expected "C" -f(c, c) # E: Argument 2 to "f" has incompatible type "C"; expected "A" -f(c, a, b, c) # E: Argument 4 to "f" has incompatible type "C"; expected "A" -f(c) -f(c, a) -f(c, b, b, a, b) - -def f(a: 'C', *b: 'A') -> None: pass - -class A: pass -class B(A): pass -class C: pass -[builtins fixtures/list.py] - -[case testCallingVarArgsFunctionWithDefaultArgs] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: B -c = Undefined # type: C - -f(a) # E: Argument 1 to "f" has incompatible type "A"; expected "C" -f(c, c) # E: Argument 2 to "f" has incompatible type "C"; expected "A" -f(c, a, b, c) # E: Argument 4 to "f" has incompatible type "C"; expected "A" -f() -f(c) -f(c, a) -f(c, b, b, a, b) - -def f(a: 'C' = None, *b: 'A') -> None: - pass - -class A: pass -class B(A): pass -class C: pass -[builtins fixtures/list.py] - -[case testCallVarargsFunctionWithIterable] -from typing import Iterable, Undefined -it1 = Undefined(Iterable[int]) -it2 = Undefined(Iterable[str]) -def f(*x: int) -> None: pass -f(*it1) -f(*it2) # E: Argument 1 to "f" has incompatible type Iterable[str]; expected "int" -[builtins fixtures/for.py] - - --- Calling varargs function + type inference --- ----------------------------------------- - - -[case testTypeInferenceWithCalleeVarArgs] -from typing import Undefined, typevar -T = typevar('T') -a = Undefined # type: A -b = Undefined # type: B -c = Undefined # type: C -o = Undefined # type: object - -a = f(o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") -b = f(b, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b = f(a, b) # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -o = f() -a = f(a) -a = f(b) -a = f(a, b, a) -o = f(a, b, o) -c = f(c) - -def f( *a: T) -> T: - pass - -class A: pass -class B(A): pass -class C: pass -[builtins fixtures/list.py] - -[case testTypeInferenceWithCalleeVarArgsAndDefaultArgs] -from typing import Undefined, typevar -T = typevar('T') -a = Undefined # type: A -o = Undefined # type: object - -a = f(o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") -a = f(a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") -a = f(a, a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") -a = f(a, a, a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") - -a = f(a) -a = f(a, a) -a = f(a, a, a) - -def f(a: T, b: T = None, *c: T) -> T: - pass - -class A: pass -[builtins fixtures/list.py] - - --- Calling normal function with varargs --- ------------------------------------ - - -[case testCallingWithListVarArgs] -from typing import Undefined, List, Any -aa = Undefined # type: List[A] -ab = Undefined # type: List[B] -a = Undefined # type: A -b = Undefined # type: B - -f(*aa) # Fail -f(a, *ab) # Ok -f(a, b) -(Any(f))(*aa) # IDEA: Move to check-dynamic? -(Any(f))(a, *ab) # IDEA: Move to check-dynamic? - -def f(a: 'A', b: 'B') -> None: - pass - -class A: pass -class B: pass -[builtins fixtures/list.py] -[out] -main, line 7: Argument 1 to "f" has incompatible type List[A]; expected "A" - -[case testCallingWithTupleVarArgs] -from typing import Undefined -a = Undefined # type: A -b = Undefined # type: B -c = Undefined # type: C -cc = Undefined # type: CC - -f(*(a, b, b)) # E: Argument 1 to "f" has incompatible type "Tuple[A, B, B]"; expected "A" -f(*(b, b, c)) # E: Argument 1 to "f" has incompatible type "Tuple[B, B, C]"; expected "A" -f(a, *(b, b)) # E: Argument 2 to "f" has incompatible type "Tuple[B, B]"; expected "B" -f(b, *(b, c)) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -f(*(a, b)) # E: Too few arguments for "f" -f(*(a, b, c, c)) # E: Too many arguments for "f" -f(a, *(b, c, c)) # E: Too many arguments for "f" -f(*(a, b, c)) -f(a, *(b, c)) -f(a, b, *(c,)) -f(a, *(b, cc)) - -def f(a: 'A', b: 'B', c: 'C') -> None: pass - -class A: pass -class B: pass -class C: pass -class CC(C): pass -[builtins fixtures/tuple.py] - -[case testInvalidVarArg] -from typing import Undefined -a = Undefined # type: A - -f(*None) -f(*a) # E: List or tuple expected as variable arguments -f(*(a,)) - -def f(a: 'A') -> None: - pass - -class A: pass -[builtins fixtures/tuple.py] - - --- Calling varargs function with varargs --- ------------------------------------- - - -[case testCallingVarArgsFunctionWithListVarArgs] -from typing import Undefined, List -aa, ab, a, b = Undefined, Undefined, Undefined, Undefined # type: (List[A], List[B], A, B) -f(*aa) # Fail -f(a, *aa) # Fail -f(b, *ab) # Fail -f(a, a, *ab) # Fail -f(a, b, *aa) # Fail -f(b, b, *ab) # Fail -g(*ab) # Fail -f(a, *ab) -f(a, b, *ab) -f(a, b, b, *ab) -g(*aa) - -def f(a: 'A', *b: 'B') -> None: pass -def g(a: 'A', *b: 'A') -> None: pass -class A: pass -class B: pass -[builtins fixtures/list.py] -[out] -main, line 3: Argument 1 to "f" has incompatible type List[A]; expected "A" -main, line 4: Argument 2 to "f" has incompatible type List[A]; expected "B" -main, line 5: Argument 1 to "f" has incompatible type "B"; expected "A" -main, line 6: Argument 2 to "f" has incompatible type "A"; expected "B" -main, line 7: Argument 3 to "f" has incompatible type List[A]; expected "B" -main, line 8: Argument 1 to "f" has incompatible type "B"; expected "A" -main, line 9: Argument 1 to "g" has incompatible type List[B]; expected "A" - -[case testCallingVarArgsFunctionWithTupleVarArgs] -from typing import Undefined -a, b, c, cc = Undefined, Undefined, Undefined, Undefined # type: (A, B, C, CC) - -f(*(b, b, b)) # E: Argument 1 to "f" has incompatible type "Tuple[B, B, B]"; expected "A" -f(*(a, a, b)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A, B]"; expected "A" -f(*(a, b, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, B, A]"; expected "A" -f(a, *(a, b)) # E: Argument 2 to "f" has incompatible type "Tuple[A, B]"; expected "B" -f(b, *(b, b)) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -f(b, b, *(b,)) # E: Argument 1 to "f" has incompatible type "B"; expected "A" -f(a, a, *(b,)) # E: Argument 2 to "f" has incompatible type "A"; expected "B" -f(a, b, *(a,)) # E: Argument 3 to "f" has incompatible type "Tuple[A]"; expected "B" -f(*()) # E: Too few arguments for "f" -f(*(a, b, b)) -f(a, *(b, b)) -f(a, b, *(b,)) - -def f(a: 'A', *b: 'B') -> None: - pass - -class A: pass -class B: pass -class C: pass -class CC(C): pass -[builtins fixtures/list.py] - - --- Varargs special cases --- --------------------- - - -[case testDynamicVarArg] -from typing import Undefined, Any -d, a = Undefined, Undefined # type: (Any, A) -f(a, a, *d) # Fail -f(a, *d) # Fail -f(*d) # Ok - -g(*d) -g(a, *d) -g(a, a, *d) - -def f(a: 'A') -> None: pass -def g(a: 'A', *b: 'A') -> None: pass -class A: pass -[builtins fixtures/list.py] -[out] -main, line 3: Too many arguments for "f" -main, line 4: Too many arguments for "f" - -[case testListVarArgsAndSubtyping] -from typing import Undefined, List -aa = Undefined # type: List[A] -ab = Undefined # type: List[B] - -g(*aa) # E: Argument 1 to "g" has incompatible type List[A]; expected "B" -f(*aa) -f(*ab) -g(*ab) - -def f( *a: 'A') -> None: - pass - -def g( *a: 'B') -> None: - pass - -class A: pass -class B(A): pass -[builtins fixtures/list.py] - -[case testCallerVarArgsAndDefaultArgs] -from typing import Undefined -a, b = Undefined, Undefined # type: (A, B) -f(*()) # Fail -f(a, *[a]) # Fail -f(a, b, *[a]) # Fail -f(*(a, a, b)) # Fail -f(*(a,)) -f(*(a, b)) -f(*(a, b, b, b)) -f(a, *[]) -f(a, *[b]) -f(a, *[b, b]) - -def f(a: 'A', b: 'B' = None, *c: 'B') -> None: - pass - -class A: pass -class B: pass -[builtins fixtures/list.py] -[out] -main, line 3: Too few arguments for "f" -main, line 4: Argument 2 to "f" has incompatible type List[A]; expected "B" -main, line 5: Argument 3 to "f" has incompatible type List[A]; expected "B" -main, line 6: Argument 1 to "f" has incompatible type "Tuple[A, A, B]"; expected "A" - - --- Overloads + varargs --- ------------------- - - -[case testIntersectionTypesAndVarArgs] -from typing import Undefined, overload, disjointclass -a, b = Undefined, Undefined # type: (A, B) - -b = f() # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b = f(a, b) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = f(b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = f(a, *[b]) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b = f(*()) -b = f(*(a,)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b = f(*(a, b)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = f(*(b,)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = f(*(b, b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a = f(*[b]) # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -a = f() -a = f(a) -a = f(a, b) -b = f(b) -b = f(b, b) -a = f(a, *[b]) -a = f(*()) -a = f(*(a,)) -a = f(*(a, b)) -b = f(*(b,)) -b = f(*(b, b)) -b = f(*[b]) - -class A: pass -@disjointclass(A) -class B: pass - -@overload -def f(a: A = None, *b: B) -> A: pass - -@overload -def f(a: B, *b: B) -> B: pass -[builtins fixtures/list.py] - - --- Caller varargs + type inference --- ------------------------------- - - -[case testCallerVarArgsListWithTypeInference] -from typing import Undefined, List, typevar, Tuple -S = typevar('S') -T = typevar('T') -a, b, aa = Undefined, Undefined, Undefined # type: (A, B, List[A]) - -a, b = f(*aa) # Fail -b, b = f(*aa) # Fail -a, a = f(b, *aa) # Fail -b, b = f(b, *aa) # Fail -b, b = f(b, b, *aa) # Fail -a, b = f(a, *a) # Fail -a, b = f(*a) # Fail - -a, a = f(*aa) -b, a = f(b, *aa) -b, a = f(b, a, *aa) - -def f(a: S, *b: T) -> Tuple[S, T]: - pass - -class A: pass -class B: pass -[builtins fixtures/list.py] -[out] -main, line 6: Argument 1 to "f" has incompatible type List[A]; expected "A" -main, line 7: Argument 1 to "f" has incompatible type List[A]; expected "B" -main, line 8: Argument 1 to "f" has incompatible type "B"; expected "A" -main, line 9: Argument 2 to "f" has incompatible type List[A]; expected "B" -main, line 10: Argument 3 to "f" has incompatible type List[A]; expected "B" -main, line 11: List or tuple expected as variable arguments -main, line 12: List or tuple expected as variable arguments - -[case testCallerVarArgsTupleWithTypeInference] -from typing import Undefined, typevar, Tuple -S = typevar('S') -T = typevar('T') -a, b = Undefined, Undefined # type: (A, B) - -a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "Tuple[A, B]"; expected "A" -b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" -a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "Tuple[A, B]"; expected "A" -b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" -a, b = f(*(a, b, b)) # E: Too many arguments for "f" - -a, b = f(*(a, b)) -a, b = f(a, *(b,)) - -def f(a: S, b: T) -> Tuple[S, T]: pass - -class A: pass -class B: pass -[builtins fixtures/list.py] - -[case testCallerVarargsAndComplexTypeInference] -from typing import Undefined, List, typevar, Generic, Tuple -T = typevar('T') -S = typevar('S') -a, b = Undefined, Undefined # type: (A, B) -ao = Undefined # type: List[object] -aa = Undefined # type: List[A] -ab = Undefined # type: List[B] - -a, aa = G().f(*[a]) # Fail -aa, a = G().f(*[a]) # Fail -ab, aa = G().f(*[a]) # Fail - -ao, ao = G().f(*[a]) # E: Incompatible types in assignment (expression has type List[None], variable has type List[object]) -aa, aa = G().f(*[a]) # E: Incompatible types in assignment (expression has type List[None], variable has type List[A]) - -class G(Generic[T]): - def f(self, *a: S) -> Tuple[List[S], List[T]]: - pass - -class A: pass -class B: pass -[builtins fixtures/list.py] -[out] -main, line 9: Incompatible types in assignment (expression has type List[A], variable has type "A") -main, line 9: Incompatible types in assignment (expression has type List[None], variable has type List[A]) -main, line 10: Incompatible types in assignment (expression has type List[None], variable has type "A") -main, line 11: Argument 1 to "f" of "G" has incompatible type List[A]; expected "B" -main, line 11: Incompatible types in assignment (expression has type List[None], variable has type List[A]) - - --- Comment signatures --- ------------------ - - -[case testVarArgsAndCommentSignature] -import typing -def f(*x): # type: (*int) -> None - pass -f(1) -f(1, 2) -f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" -f(1, '') # E: Argument 2 to "f" has incompatible type "str"; expected "int" -[builtins fixtures/list.py] - - --- Subtyping --- --------- - - -[case testVarArgsFunctionSubtyping] -from typing import Function, Undefined -x = Undefined # type: Function[[int], None] -def f(*x: int) -> None: pass -def g(*x: str) -> None: pass -x = f -x = g # E: Incompatible types in assignment (expression has type Function[[str], None], variable has type Function[[int], None]) -[builtins fixtures/list.py] -[out] diff --git a/mypy/test/data/fixtures/bool.py b/mypy/test/data/fixtures/bool.py deleted file mode 100644 index eac62168b813..000000000000 --- a/mypy/test/data/fixtures/bool.py +++ /dev/null @@ -1,13 +0,0 @@ -# builtins stub used in boolean-related test cases. - -from typing import builtinclass - -@builtinclass -class object: - def __init__(self) -> None: pass - -class type: pass -class tuple: pass -class function: pass -class bool: pass -class int: pass diff --git a/mypy/test/data/fixtures/classmethod.py b/mypy/test/data/fixtures/classmethod.py deleted file mode 100644 index 634e1e0c30e8..000000000000 --- a/mypy/test/data/fixtures/classmethod.py +++ /dev/null @@ -1,18 +0,0 @@ -import typing - -class object: - def __init__(self) -> None: pass - -class type: - def __init__(self, x) -> None: pass - -class function: pass - -classmethod = object() # Dummy definition. - -class int: - @classmethod - def from_bytes(cls, bytes: bytes, byteorder: str) -> int: pass - -class str: pass -class bytes: pass diff --git a/mypy/test/data/fixtures/dict.py b/mypy/test/data/fixtures/dict.py deleted file mode 100644 index 3b72368c7f7d..000000000000 --- a/mypy/test/data/fixtures/dict.py +++ /dev/null @@ -1,22 +0,0 @@ -# Builtins stub used in dictionary-related test cases. - -from typing import typevar, Generic, Iterable, Iterator - -T = typevar('T') -S = typevar('S') - -class object: - def __init__(self) -> None: pass - -class type: pass - -class dict(Generic[T, S]): pass -class int: pass # for convenience -class str: pass # for keyword argument key type -class list(Iterable[T], Generic[T]): # needed by some test cases - def __iter__(self) -> Iterator[T]: pass - def __mul__(self, x: int) -> list[T]: pass - -class tuple: pass -class function: pass -class float: pass diff --git a/mypy/test/data/fixtures/exception.py b/mypy/test/data/fixtures/exception.py deleted file mode 100644 index 8177156ccc4d..000000000000 --- a/mypy/test/data/fixtures/exception.py +++ /dev/null @@ -1,9 +0,0 @@ - -class object: - def __init__(self): pass - -class type: pass -class tuple: pass -class function: pass - -class BaseException: pass diff --git a/mypy/test/data/fixtures/for.py b/mypy/test/data/fixtures/for.py deleted file mode 100644 index c7de135b6023..000000000000 --- a/mypy/test/data/fixtures/for.py +++ /dev/null @@ -1,19 +0,0 @@ -# builtins stub used in for statement test cases - -from typing import typevar, Generic, Iterable, Iterator -from abc import abstractmethod, ABCMeta - -t = typevar('t') - -class object: - def __init__(self) -> None: pass - -class type: pass -class tuple: pass -class function: pass -class bool: pass -class int: pass # for convenience -class str: pass # for convenience - -class list(Iterable[t], Generic[t]): - def __iter__(self) -> Iterator[t]: pass diff --git a/mypy/test/data/fixtures/function.py b/mypy/test/data/fixtures/function.py deleted file mode 100644 index b54ae00b3342..000000000000 --- a/mypy/test/data/fixtures/function.py +++ /dev/null @@ -1,8 +0,0 @@ -from typing import builtinclass - -@builtinclass -class object: - def __init__(self): pass - -class type: pass -class function: pass diff --git a/mypy/test/data/fixtures/isinstance.py b/mypy/test/data/fixtures/isinstance.py deleted file mode 100644 index 293e8de2df6f..000000000000 --- a/mypy/test/data/fixtures/isinstance.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import builtinclass - -@builtinclass -class object: - def __init__(self) -> None: pass - -@builtinclass -class type: - def __init__(self, x) -> None: pass - -class tuple: pass -class function: pass - -def isinstance(x: object, t: type) -> bool: pass - -@builtinclass -class int: pass -@builtinclass -class bool(int): pass -@builtinclass -class str: pass diff --git a/mypy/test/data/fixtures/isinstancelist.py b/mypy/test/data/fixtures/isinstancelist.py deleted file mode 100644 index 209fa195f2cf..000000000000 --- a/mypy/test/data/fixtures/isinstancelist.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import builtinclass, Iterable, Iterator, Generic, typevar, List - -@builtinclass -class object: - def __init__(self) -> None: pass - -@builtinclass -class type: - def __init__(self, x) -> None: pass - -class tuple: pass -class function: pass - -def isinstance(x: object, t: type) -> bool: pass - -@builtinclass -class int: - def __add__(self, x: int) -> int: pass -@builtinclass -class bool(int): pass -@builtinclass -class str: - def __add__(self, x: str) -> str: pass - def __getitem__(self, x: int) -> str: pass - -T = typevar('T') - -class list(Iterable[T], Generic[T]): - def __iter__(self) -> Iterator[T]: pass - def __mul__(self, x: int) -> list[T]: pass - def __setitem__(self, x: int, v: T) -> None: pass - def __getitem__(self, x: int) -> T: pass - def __add__(self, x: List[T]) -> T: pass diff --git a/mypy/test/data/fixtures/list.py b/mypy/test/data/fixtures/list.py deleted file mode 100644 index 7cefdc31a9b4..000000000000 --- a/mypy/test/data/fixtures/list.py +++ /dev/null @@ -1,21 +0,0 @@ -# Builtins stub used in list-related test cases. - -from typing import typevar, Generic, builtinclass, Iterable, Iterator - -T = typevar('T') - -@builtinclass -class object: - def __init__(self): pass - -class type: pass - -class list(Iterable[T], Generic[T]): - def __iter__(self) -> Iterator[T]: pass - def __mul__(self, x: int) -> list[T]: pass - def __getitem__(self, x: int) -> T: pass - -class tuple: pass -class function: pass -class int: pass -class str: pass diff --git a/mypy/test/data/fixtures/module.py b/mypy/test/data/fixtures/module.py deleted file mode 100644 index 5b7614f226f2..000000000000 --- a/mypy/test/data/fixtures/module.py +++ /dev/null @@ -1,5 +0,0 @@ -class object: - def __init__(self) -> None: pass -class module: pass -class type: pass -class function: pass diff --git a/mypy/test/data/fixtures/ops.py b/mypy/test/data/fixtures/ops.py deleted file mode 100644 index b527090f865e..000000000000 --- a/mypy/test/data/fixtures/ops.py +++ /dev/null @@ -1,43 +0,0 @@ -from typing import Undefined, builtinclass - -# This is an extension of transform builtins with additional operations. - -@builtinclass -class object: - def __init__(self) -> None: pass - def __eq__(self, o: 'object') -> 'bool': pass - def __ne__(self, o: 'object') -> 'bool': pass - -class type: pass -class tuple: pass -class function: pass - -class bool: pass - -class str: - def __init__(self, x: 'int') -> None: pass - def __add__(self, x: 'str') -> 'str': pass - -class int: - def __add__(self, x: 'int') -> 'int': pass - def __sub__(self, x: 'int') -> 'int': pass - def __mul__(self, x: 'int') -> 'int': pass - def __mod__(self, x: 'int') -> 'int': pass - def __floordiv__(self, x: 'int') -> 'int': pass - def __pos__(self) -> 'int': pass - def __neg__(self) -> 'int': pass - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: 'int') -> bool: pass - def __le__(self, x: 'int') -> bool: pass - def __gt__(self, x: 'int') -> bool: pass - def __ge__(self, x: 'int') -> bool: pass - -class float: pass - -class BaseException: pass - -True = Undefined(bool) -False = Undefined(bool) - -def __print(a1=None, a2=None, a3=None, a4=None): pass diff --git a/mypy/test/data/fixtures/primitives.py b/mypy/test/data/fixtures/primitives.py deleted file mode 100644 index e3600e55bd77..000000000000 --- a/mypy/test/data/fixtures/primitives.py +++ /dev/null @@ -1,15 +0,0 @@ -# builtins stub with non-generic primitive types - -class object: - def __init__(self) -> None: pass - -class type: - def __init__(self, x) -> None: pass - -class int: pass -class str: pass -class float: pass -class bool: pass -class bytes: pass -class tuple: pass -class function: pass diff --git a/mypy/test/data/fixtures/property.py b/mypy/test/data/fixtures/property.py deleted file mode 100644 index 98c0e176974f..000000000000 --- a/mypy/test/data/fixtures/property.py +++ /dev/null @@ -1,15 +0,0 @@ -import typing - -class object: - def __init__(self) -> None: pass - -class type: - def __init__(self, x) -> None: pass - -class function: pass - -property = object() # Dummy definition. - -class int: pass -class str: pass -class bytes: pass diff --git a/mypy/test/data/fixtures/python2.py b/mypy/test/data/fixtures/python2.py deleted file mode 100644 index 76245b717e55..000000000000 --- a/mypy/test/data/fixtures/python2.py +++ /dev/null @@ -1,13 +0,0 @@ -class object: - def __init__(self) -> None: pass - -class type: - def __init__(self, x) -> None: pass - -class function: pass - -class int: pass -class str: pass -class unicode: pass - -# Definition of None is implicit diff --git a/mypy/test/data/fixtures/set.py b/mypy/test/data/fixtures/set.py deleted file mode 100644 index fde16a211ccb..000000000000 --- a/mypy/test/data/fixtures/set.py +++ /dev/null @@ -1,18 +0,0 @@ -# Builtins stub used in set-related test cases. - -from typing import typevar, Generic, Iterator, Iterable - -T = typevar('T') - -class object: - def __init__(self) -> None: pass - -class type: pass -class tuple: pass -class function: pass - -class int: pass -class str: pass - -class set(Iterable[T], Generic[T]): - def __iter__(self) -> Iterator[T]: pass diff --git a/mypy/test/data/fixtures/slice.py b/mypy/test/data/fixtures/slice.py deleted file mode 100644 index bd1076849714..000000000000 --- a/mypy/test/data/fixtures/slice.py +++ /dev/null @@ -1,12 +0,0 @@ -# Builtins stub used in slicing test cases. - -class object: - def __init__(self): pass - -class type: pass -class tuple: pass -class function: pass - -class int: pass - -class slice: pass diff --git a/mypy/test/data/fixtures/staticmethod.py b/mypy/test/data/fixtures/staticmethod.py deleted file mode 100644 index 139aceedbacc..000000000000 --- a/mypy/test/data/fixtures/staticmethod.py +++ /dev/null @@ -1,18 +0,0 @@ -import typing - -class object: - def __init__(self) -> None: pass - -class type: - def __init__(self, x) -> None: pass - -class function: pass - -staticmethod = object() # Dummy definition. - -class int: - @staticmethod - def from_bytes(bytes: bytes, byteorder: str) -> int: pass - -class str: pass -class bytes: pass diff --git a/mypy/test/data/fixtures/transform.py b/mypy/test/data/fixtures/transform.py deleted file mode 100644 index 09bd12ed027b..000000000000 --- a/mypy/test/data/fixtures/transform.py +++ /dev/null @@ -1,30 +0,0 @@ -# Builtins stubs used implicitly in program transformation test cases. - -class object: - def __init__(self) -> None: pass - -class type: pass - -# str is handy for debugging; allows outputting messages. -class str: pass - -# Primitive types int/float have special coercion behaviour (they may have -# a different representation from ordinary values). - -class int: pass - -class float: pass - - -# The functions below are special functions used in test cases; their -# implementations are actually in the __dynchk module, but they are defined -# here so that the semantic analyser and the type checker are happy without -# having to analyse the entire __dynchk module all the time. -# -# The transformation implementation has special case handling for these -# functions; it's a bit ugly but it works for now. - -def __print(a1=None, a2=None, a3=None, a4=None): - # Do not use *args since this would require list and break many test - # cases. - pass diff --git a/mypy/test/data/fixtures/tuple.py b/mypy/test/data/fixtures/tuple.py deleted file mode 100644 index 3051291381f7..000000000000 --- a/mypy/test/data/fixtures/tuple.py +++ /dev/null @@ -1,18 +0,0 @@ -# Builtins stub used in tuple-related test cases. - -from typing import Iterable, typevar - -class object: - def __init__(self): pass - -class type: pass -class tuple: pass -class function: pass - -# We need int for indexing tuples. -class int: pass -class str: pass # For convenience - -T = typevar('T') - -def sum(iterable: Iterable[T], start: T = None) -> T: pass diff --git a/mypy/test/data/fixtures/union.py b/mypy/test/data/fixtures/union.py deleted file mode 100644 index 5502082138e2..000000000000 --- a/mypy/test/data/fixtures/union.py +++ /dev/null @@ -1,18 +0,0 @@ -# Builtins stub used in tuple-related test cases. - -from isinstance import isinstance -from typing import Iterable, typevar - -class object: - def __init__(self): pass - -class type: pass -class function: pass - -# Current tuple types get special treatment in the type checker, thus there -# is no need for type arguments here. -class tuple: pass - -# We need int for indexing tuples. -class int: pass -class str: pass # For convenience diff --git a/mypy/test/data/lib-stub/abc.py b/mypy/test/data/lib-stub/abc.py deleted file mode 100644 index b69aab771bb5..000000000000 --- a/mypy/test/data/lib-stub/abc.py +++ /dev/null @@ -1,2 +0,0 @@ -class ABCMeta: pass -abstractmethod = object() diff --git a/mypy/test/data/lib-stub/builtins.py b/mypy/test/data/lib-stub/builtins.py deleted file mode 100644 index 49b9743b9a5e..000000000000 --- a/mypy/test/data/lib-stub/builtins.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import builtinclass - -@builtinclass -class object: - def __init__(self) -> None: pass - -@builtinclass -class type: - def __init__(self, x) -> None: pass - -# These are provided here for convenience. -@builtinclass -class int: pass -@builtinclass -class str: pass - -class tuple: pass -class function: pass - -# Definition of None is implicit diff --git a/mypy/test/data/lib-stub/collections.py b/mypy/test/data/lib-stub/collections.py deleted file mode 100644 index fa6f4ee58bcc..000000000000 --- a/mypy/test/data/lib-stub/collections.py +++ /dev/null @@ -1 +0,0 @@ -namedtuple = object() diff --git a/mypy/test/data/lib-stub/typing.py b/mypy/test/data/lib-stub/typing.py deleted file mode 100644 index fed6889476ca..000000000000 --- a/mypy/test/data/lib-stub/typing.py +++ /dev/null @@ -1,34 +0,0 @@ -# Stub for typing module. Many of the definitions have special handling in -# the type checker, so they can just be initialized to anything. - -from abc import abstractmethod - -cast = object() -overload = object() -Undefined = object() -Any = object() -Union = object() -typevar = object() -Generic = object() -AbstractGeneric = object() -Tuple = object() -Function = object() -builtinclass = object() -ducktype = object() -disjointclass = object() -NamedTuple = object() - -# Type aliases. -List = object() -Dict = object() -Set = object() - -T = typevar('T') - -class Iterable(AbstractGeneric[T]): - @abstractmethod - def __iter__(self) -> 'Iterator[T]': pass - -class Iterator(Iterable[T], AbstractGeneric[T]): - @abstractmethod - def __next__(self) -> T: pass diff --git a/mypy/test/data/output.test b/mypy/test/data/output.test deleted file mode 100644 index e49e36f90e58..000000000000 --- a/mypy/test/data/output.test +++ /dev/null @@ -1,524 +0,0 @@ --- Special cases --- ------------- - - -[case testEmpty] -[out] - -[case testCommentOnlyFile] -# comment - - --- Expressions --- ----------- - - -[case testExpressionStatement] -1 - -[case testBinaryOperation] -1 + 2 # x - -[case testNameExpr] -x + __yDSA2 - -[case testParentheses] -( ( x ) ) - -[case testBinaryOps] -x + 1 - 2 * 3 / 4 % 5 // 6 ** 7 -x == 1 != 2 < 3 > 4 <= 5 >= 6 -x & 1 ^ 2 | 3 << 4 >> 5 -x in 1 -x not in 2 -x is 2 -x is not 2 -x and y or z - -[case testUnaryOps] -x = [ + 1, - 2, ~ 3, not 4 ] - -[case testLiterals] -1 + 12359 + 0xAf09 + 0o027 -1.0 + .1 + 1. + 1e123 -'' 'foo' '\n' '\xab' '\u123f' -"" "foo" -r'foo' r"foo" -'''foo''' """bar""" -b'foo' b"foo" -3j + 2J + 1.1j + 1e3J + -1j + .1j - -[case testTupleLiterals] -( ) -( 1, ) -( 1, 2 ) -( 1 , 2, 3 ) -( 1 , ) - -[case testMemberExpr] -x . y - -[case testIndexExpr] -x [ y ] -x [ 1 , 2 ] - -[case testSlices] -x [ 1 : 2 : 3 ] -x [ : ] -x [ : : 2 ] - -[case testSimpleCalls] -f ( ) -f ( x ) -f ( x , y ) -f ( x , y , z ) - -[case testVarArgsCalls] -f ( * x ) -f ( x , * x ) - -[case testCallsWithKeywordArgs] -f ( x = y ) -f ( x, y = z , a=b+1) -f(x *y, z=1) - -[case testDictLiterals] -{ } -{ x : y } -{ x : y, 1 : 2 } -{ x : y, 1 : 2, 3 : 4 , } - -[case testSetLiterals] -{ x } -{ x, y } -{ x , y , } - -[case testCasts] -from typing import cast, Any -cast(x, y) -Any(z) -cast(c[a, x[d]], foo.bar) - -[case testLambdaExpression] -lambda: 1 -lambda x: x -lambda x, y: (x, y) -lambda x, y=1, z = 2: 2 -lambda x, *y: y - -[case testGeneratorExpression] -x + 1 for x in z -x for x in y if foo - -[case testListComprehension] -a = [ x + 1 for x in [1, 2, 3]] -a = [ x for x in y if z] - - --- Assignment --- ---------- - - -[case testSingleAssignment] -x = y - -[case testChainedAssignment] -x = y = z - -[case testListLiteral] -x = [ ] -x = [ 1 ] -x = [ 1 , 2 ] -x = [ 1, ] - -[case testMultipleAssignment] -x, y = 1, 2 -(x, y,) = z -[x, y] = a - - --- Misc --- ---- - - -[case testLineContinuation] -x = \ - 1 -'''fooo - bar''' -x = ( #foo - x + 2) - - --- Imports --- ------- - - -[case testImport] -import x , y . z -import re - -[case testFromImport] -from x import y -from x . y import z, blah -from x import (y, z) - -[case testFromImportAsterisk] -from x . y import * - -[case testImportAs] -import x as y -import x, y as z, a as b - -[case testFromImportAs] -from x import y as z -from x import x, y as z, a as b - - --- Statements --- ---------- - -[case testOperatorAssignment] -x *= 2 -x += 1 - -[case testBlock] -while 1: - x = 1 -while 1: x = 1 -while 1 : # foo - while 2: - x - y -# foo - -[case testWhileElse] -while 1: - x -else: - y - -[case testIf] -if x: - y -if x: - y -else: - z -if a: - 1 -elif 2: - 22 -elif 3: - 4 - -[case testBreak] -while x: - break - -[case testContinue] -while x: - continue - -[case testPass] -if x: - pass - -[case testRaise] -raise x -raise y from z - -[case testAssert] -assert x != 1 - -[case testYield] -yield x -yield - -[case testDel] -del x -del y.x, z[0] - -[case testGlobal] -global x, y - -[case testTryExcept] -try: - x -except E: - y -try: - x -except e as f: - y -except x: - z -except: - xx -else: - yy - -[case testTryFinally] -try: - x -finally: - z - -[case testComplexTry] -try: - x - y -except (x, y) as e: - foo -finally: - bar - -[case testVarDefWithType] -from typing import Undefined, Any, List, Dict -x = 0 -z = 1 + 2 # type: int -a = Undefined # type: Any -a = Undefined # type: x[y] # Comment -a = Undefined(Any) -z = Undefined(x.y) -l = Undefined(List[int]) -d = Undefined(Dict[x, c[ Any]]) -x = Undefined(c[ d[ e]]) - -[case testMultipleVarDef] -from typing import Undefined, List -x , y = Undefined, Undefined # type: (int, List[c]) -x , y = Undefined(int), Undefined(List[c]) -x, y , z = 123 # type: int, x, foobar - -[case testListAndTupleTypes] -from typing import Undefined, List, Tuple -foo = Undefined # type: List[int] -foo = Undefined(List[int]) -t = Undefined # type: List[List[Tuple[ x , y]]] -t = Undefined(List[List[Tuple[ x , y]]]) - -[case testForStatement] -for x in y: pass -for x, y in z: pass - -[case testForElse] -for x in y: - x -else: - y - -[case testForWithParensAroundIndexVars-skip] -for ( x ) in y: pass -for ( x, y ) in y: pass - -[case testWith] -with foo(): - pass -with bar() as x: - x = 2 -with f() as x, y() as z: - pass - - --- Function definitions --- -------------------- - - -[case testSimpleFunctions] -def f(): pass -def g ( x ) : pass -def h(x, y): pass -def i(x,y , z , zz ) : - z = y - return 5 - -[case testFunctionDefWithTypes-skip] -import typing -def f() -> int: pass -def g(x: int) -> int: pass -def h(x: int, y: str) -> int: pass -def i() -> None: pass -def j(x: p[g]) -> c[l]: pass - -[case testGenericFunctionDef-skip] -from typing import typevar -t = typevar('t') -s = typevar('s') -x = typevar('x') -def f() -> None: pass -def g ( x: x[s] ) -> None : pass - -[case testInitializersInFunctionDefs-skip] -import typing -def f(x = 1 , y = 2): pass -def g(x, y=1): pass -def h (x: int = 4, z: float = 2.2) -> None : pass - -[case testFunctionWithPartialTypes-skip] -import typing -def f(x, y) -> None: pass -def g(x: int, y, z: str): pass - -[case testFunctionWithVarArgs-skip] -import typing -def f(*x): pass -def g(x, y=1, *z): pass -def h( *x: int) -> None: pass -def i(x, *x: int) -> None: pass -def j(x: int, y: int = 2, *z: str) -> None: pass - -[case testOverloadedFunction-skip] -from typing import overload, Any -@overload -def f() -> None: pass - -@overload -def f(x: Any) -> Any: return x - -[case testOverloadedFunctionInBlock-skip] -from typing import overload, Any -if 1: - @overload - def f() -> None: - pass - - @overload - def f(x: Any) -> Any: - return x - - --- Classes --- ------- - - -[case testSimpleClass] -class A: pass - -[case testBaseClass] -class A(B): pass - -[case testMultipleBaseClasses] -class A(B, C): pass -class A(B, C, D): pass - -[case testGenericBaseClasses-skip] -import typing -class A(B[x], C[y, D[z]]): pass - -[case testGenericClass-skip] -from typing import typevar, Generic -t = typevar('t') -x = typevar('x') -class C(Generic[t]): - pass -class C(foo[t, x], Generic[t, x]): pass - -[case testClassBody] -class C: - x = y - def f(self, x): - return x - -[case testSuper] -class C: - def f(self): - super().__init__() - - --- Abstract base classes --- --------------------- - - -[case testABC-skip] -from abc import abstractmethod, ABCMeta -import typing -class x(metaclass=ABCMeta): - @abstractmethod - def f(self, x): pass - @abstractmethod - def g(self) -> None: return self - - --- Types --- ----- - - -[case testTupleType] -from typing import Undefined, Any, Tuple -class A: - t = Undefined # type: Tuple[object, Any] - t = Undefined(Tuple[object, Any]) - -[case testFuncType] -from typing import Undefined, Any, Function, List -class A: - t = Undefined # type: Function[[], Any] - t = Undefined # type: Function[[object], None] - t = Undefined # type: Function[[ Any, Any], None] - t = Undefined # type: List[Function[[ a, b , List[c]], List[a]]] - t = Undefined(Function[[], Any]) - t = Undefined(Function[[object], None]) - t = Undefined([[ Any, Any], None]) - t = Undefined(List[Function[[ a, b , List[c]], List[a]]]) - - --- Decorators --- ---------- - - -[case testFunctionDecorator] -import typing -@foo -def f(): pass -@foo(bar) -@zar.x -def g(x): - return x - - --- Semantic analysis --- ----------------- - - -[case testFunctionType_SemanticAnalyzer-skip] -from typing import Any, Function -def f( x: Function[[ object], Any] ) -> None: pass - -[case testTupleType_SemanticAnalyzer-skip] -from typing import Any, Tuple -def f ( x: Tuple[ object , Any] ) -> None: pass -[builtins fixtures/tuple.py] - -[case testClasses_SemanticAnalyzer-skip] -from abc import abstractmethod, ABCMeta -import typing -class A: pass -class B(A): pass -class I(metaclass=ABCMeta): pass -class C(I): pass - -[case testMethodWithSignature_SemanticAnalyzer-skip] -import typing -class A: - def __init__(self, x: 'A') -> None: - self.x = x - - --- ** arguments --- ------------ - - -[case testKeywordVarArgs-skip] -from typing import Undefined, Dict -def f( ** x): pass -def g(x: A, y: B = None, **z: C) -> None: pass -d = Undefined(dict) -g(**d) -g( None , ** d) -g( None , ** d) -class A: pass -[builtins fixtures/dict.py] - -[case testFunctionDefinitionWithBothVarArgs] -def f ( * x , ** y) : pass diff --git a/mypy/test/data/parse-errors.test b/mypy/test/data/parse-errors.test deleted file mode 100644 index 6352599ba646..000000000000 --- a/mypy/test/data/parse-errors.test +++ /dev/null @@ -1,376 +0,0 @@ --- Test cases for parser errors. Each test case consists of two sections. --- The first section contains [case NAME] followed by the input code, while --- the second section contains [out] followed by the output from the parser. --- --- The input file name in errors is "file". --- --- Comments starting with "--" in this file will be ignored, except for lines --- starting with "----" that are not ignored. The first two dashes of these --- lines are interpreted as escapes and removed. - -[case testInvalidFunction] -def f() - pass -[out] -file: In function "f": -file, line 1: Parse error before end of line -file: At top level: -file, line 2: Inconsistent indentation - -[case testMissingIndent] -if x: -1 -[out] -file, line 2: Expected an indented block - -[case testUnexpectedIndent] -1 - 2 -[out] -file, line 2: Inconsistent indentation - -[case testInconsistentIndent] -if x: - 1 - 1 -[out] -file, line 3: Inconsistent indentation - -[case testInconsistentIndent] -if x: - 1 - 1 -[out] -file, line 3: Inconsistent indentation - -[case testInvalidBinaryOp] -1> -a* -a+1* -[out] -file, line 1: Parse error before end of line -file, line 2: Parse error before end of line -file, line 3: Parse error before end of line - -[case testDoubleStar] -**a -[out] -file, line 1: Parse error before ** - -[case testInvalidSuperClass] -class A(C[): - pass -[out] -file: In class "A": -file, line 1: Parse error before ) -file, line 2: Parse error before end of file - -[case testMissingSuperClass] -class A(: - pass -[out] -file: In class "A": -file, line 1: Parse error before : -file, line 2: Parse error before end of file - -[case testUnexpectedEof] -if 1: -[out] -file, line 1: Expected an indented block - -[case testInvalidKeywordArguments1] -f(x=y, z) -[out] -file, line 1: Parse error before "z" - -[case testInvalidKeywordArguments2] -f(**x, y=z) -[out] -file, line 1: Parse error before "y" - -[case testInvalidKeywordArguments3] -f(**x, y) -[out] -file, line 1: Parse error before "y" - -[case testInvalidVarArgs] -f(*x, y) -[out] -file, line 1: Parse error before "y" - -[case testInvalidBareAsteriskAndVarArgs1] -def f(*x: A, *, y: B = 1) -> None: pass -[out] -file: In function "f": -file, line 1: Parse error before * -file, line 1: Parse error before end of line - -[case testInvalidBareAsteriskAndVarArgs2] -def f(*x: A, *) -> None: pass -[out] -file: In function "f": -file, line 1: Parse error before ) -file, line 1: Parse error before end of line - -[case testInvalidBareAsteriskAndVarArgs3] -def f(*, *x: A) -> None: pass -[out] -file: In function "f": -file, line 1: Parse error before * -file, line 1: Parse error before end of line - -[case testInvalidBareAsteriskAndVarArgs4] -def f(*, **x: A) -> None: pass -[out] -file: In function "f": -file, line 1: Parse error before ** -file, line 1: Parse error before end of line - -[case testInvalidBareAsterisk1] -def f(*) -> None: pass -[out] -file: In function "f": -file, line 1: Parse error before ) -file, line 1: Parse error before end of line - -[case testInvalidBareAsterisk2] -def f(x, *) -> None: pass -[out] -file: In function "f": -file, line 1: Parse error before ) -file, line 1: Parse error before end of line - -[case testInvalidFuncDefArgs1] -def f(x = y, x): pass -[out] -file: In function "f": -file, line 1: Invalid argument list - -[case testInvalidFuncDefArgs3] -def f(**x, y): - pass -[out] -file: In function "f": -file, line 1: Invalid argument list - -[case testInvalidFuncDefArgs4] -def f(**x, y=x): - pass -[out] -file: In function "f": -file, line 1: Invalid argument list - -[case testInvalidStringLiteralType] -def f(x: - 'A[' - ) -> None: pass -[out] -file: In function "f": -file, line 2: Parse error before end of line -file, line 3: Parse error before end of line - -[case testInvalidStringLiteralType2] -def f(x: - 'A B' - ) -> None: pass -[out] -file: In function "f": -file, line 2: Parse error before "B" -file, line 3: Parse error before end of line - -[case testInvalidTypeComment] -0 -x = 0 # type: A A -[out] -file, line 2: Parse error before "A" - -[case testInvalidTypeComment2] -0 -x = 0 # type: A[ -[out] -file, line 2: Parse error before end of line - -[case testInvalidTypeComment3] -0 -x = 0 # type: -[out] -file, line 2: Empty type annotation - -[case testInvalidTypeComment4] -0 -x = 0 # type: * -[out] -file, line 2: Parse error before end of line - -[case testInvalidMultilineLiteralType] -def f() -> "A\nB": pass -[out] -file: In function "f": -file, line 1: Parse error before end of line - -[case testInvalidMetaclass] -class A(metaclass=1): pass -[out] -file: In class "A": -file, line 1: Parse error before numeric literal -file, line 1: Parse error before end of file - -[case testInvalidSignatureInComment1] -def f(): # type: x - pass -[out] -file: In function "f": -file, line 1: Parse error before "x" - -[case testInvalidSignatureInComment2] -def f(): # type: - pass -[out] -file: In function "f": -file, line 1: Empty type annotation - -[case testInvalidSignatureInComment3] -def f(): # type: ( - pass -[out] -file: In function "f": -file, line 1: Parse error before end of line - -[case testInvalidSignatureInComment4] -def f(): # type: (. - pass -[out] -file: In function "f": -file, line 1: Parse error before . - -[case testInvalidSignatureInComment5] -def f(): # type: (x - pass -[out] -file: In function "f": -file, line 1: Parse error before end of line - -[case testInvalidSignatureInComment6] -def f(): # type: (x) - pass -[out] -file: In function "f": -file, line 1: Parse error before end of line - -[case testInvalidSignatureInComment7] -def f(): # type: (x) - - pass -[out] -file: In function "f": -file, line 1: Parse error before - - -[case testInvalidSignatureInComment8] -def f(): # type: (x) -> - pass -[out] -file: In function "f": -file, line 1: Parse error before end of line - -[case testInvalidSignatureInComment9] -def f(): # type: (x) -> . - pass -[out] -file: In function "f": -file, line 1: Parse error before . - -[case testInvalidSignatureInComment10] -def f(): # type: (x) -> x x - pass -[out] -file: In function "f": -file, line 1: Parse error before "x" - -[case testDuplicateSignatures1] -def f() -> None: # type: () -> None - pass -def f(): # type: () -> None - pass -[out] -file: In function "f": -file, line 1: Function has duplicate type signatures - -[case testDuplicateSignatures2] -def f(x, y: Z): # type: (x, y) -> z - pass -[out] -file: In function "f": -file, line 1: Function has duplicate type signatures - -[case testCommentFunctionAnnotationVarArgMispatch] -def f(x): # type: (*X) -> Y - pass -def g(*x): # type: (X) -> Y - pass -[out] -file: In function "f": -file, line 1: Inconsistent use of '*' in function signature -file: In function "g": -file, line 3: Inconsistent use of '*' in function signature - -[case testCommentFunctionAnnotationVarArgMispatch2] -def f(*x, **y): # type: (**X, *Y) -> Z - pass -def g(*x, **y): # type: (*X, *Y) -> Z - pass -[out] -file: In function "f": -file, line 1: Inconsistent use of '*' in function signature -file, line 1: Inconsistent use of '**' in function signature -file: In function "g": -file, line 3: Inconsistent use of '*' in function signature -file, line 3: Inconsistent use of '**' in function signature - -[case testPrintStatementInPython3] -print 1 -[out] -file, line 1: Parse error before numeric literal - -[case testInvalidConditionInConditionalExpression] -1 if 2, 3 else 4 -[out] -file, line 1: Parse error before , - -[case testInvalidConditionInConditionalExpression2] -1 if x for y in z else 4 -[out] -file, line 1: Parse error before "for" - -[case testInvalidConditionInConditionalExpression2] -1 if x else for y in z -[out] -file, line 1: Parse error before "for" - -[case testYieldFromNotRightParameter] -def f(): - yield from -[out] -file: In function "f": -file, line 2: Parse error before end of line - -[case testYieldFromAfterReturn] -def f(): - return yield from h() -[out] -file: In function "f": -file, line 2: Parse error before end of line - -[case testImportDotModule] -import .x -[out] -file, line 1: Parse error before . - -[case testImportDot] -import . -[out] -file, line 1: Parse error before . - -[case testInvalidFunctionName] -def while(): pass -[out] -file, line 1: Parse error before "while" -file, line 1: Parse error before end of line diff --git a/mypy/test/data/parse-python2.test b/mypy/test/data/parse-python2.test deleted file mode 100644 index ae518041e028..000000000000 --- a/mypy/test/data/parse-python2.test +++ /dev/null @@ -1,63 +0,0 @@ --- Test cases for parser -- Python 2 syntax. --- --- See parse.test for a description of this file format. - -[case testEmptyFile] -[out] -MypyFile:1() - -[case testStringLiterals] -'bar' -u'foo' -ur'foo' -u'''bar''' -b'foo' -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr(bar)) - ExpressionStmt:2( - UnicodeExpr(foo)) - ExpressionStmt:3( - UnicodeExpr(foo)) - ExpressionStmt:4( - UnicodeExpr(bar)) - ExpressionStmt:5( - StrExpr(foo))) - -[case testSimplePrint] -print 1 -print 2, 3 -print (4, 5) -[out] -MypyFile:1( - PrintStmt:1( - IntExpr(1) - Newline) - PrintStmt:2( - IntExpr(2) - IntExpr(3) - Newline) - PrintStmt:3( - ParenExpr:3( - TupleExpr:3( - IntExpr(4) - IntExpr(5))) - Newline)) - -[case testSimpleWithTrailingComma] -print 1, -print 2, 3, -print (4, 5), -[out] -MypyFile:1( - PrintStmt:1( - IntExpr(1)) - PrintStmt:2( - IntExpr(2) - IntExpr(3)) - PrintStmt:3( - ParenExpr:3( - TupleExpr:3( - IntExpr(4) - IntExpr(5))))) diff --git a/mypy/test/data/parse.test b/mypy/test/data/parse.test deleted file mode 100644 index a374d26e2144..000000000000 --- a/mypy/test/data/parse.test +++ /dev/null @@ -1,2948 +0,0 @@ --- Test cases for parser. Each test case consists of two sections. --- The first section contains [case NAME] followed by the input code, while --- the second section contains [out] followed by the output from the parser. --- --- Lines starting with "--" in this file will be ignored, except for lines --- starting with "----" that are not ignored. The first two dashes of these --- lines are interpreted as escapes and removed. - -[case testEmptyFile] -[out] -MypyFile:1() - -[case testExpressionStatement] -1 -[out] -MypyFile:1( - ExpressionStmt:1( - IntExpr(1))) - -[case testAssignment] -x = 1 -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(1))) - -[case testExpressionBasics] -x = f(1, None) -123 * (2 + x) -"hello".lower() --1.23 -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - CallExpr:1( - NameExpr(f) - Args( - IntExpr(1) - NameExpr(None)))) - ExpressionStmt:2( - OpExpr:2( - * - IntExpr(123) - ParenExpr:2( - OpExpr:2( - + - IntExpr(2) - NameExpr(x))))) - ExpressionStmt:3( - CallExpr:3( - MemberExpr:3( - StrExpr(hello) - lower) - Args())) - ExpressionStmt:4( - UnaryExpr:4( - - - FloatExpr(1.23)))) - -[case testSingleQuotedStr] -'' -'foo' -'foo\ -bar' -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr()) - ExpressionStmt:2( - StrExpr(foo)) - ExpressionStmt:3( - StrExpr(foobar))) - -[case testDoubleQuotedStr] -"" -"foo" -"foo\ -bar" -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr()) - ExpressionStmt:2( - StrExpr(foo)) - ExpressionStmt:3( - StrExpr(foobar))) - -[case testTripleQuotedStr] -'''''' -'''foo''' -'''foo\ -bar''' -'''\nfoo -bar''' -'''fo''bar''' -"""""" -"""foo""" -"""foo\ -bar""" -"""\nfoo -bar""" -"""fo""bar""" -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr()) - ExpressionStmt:2( - StrExpr(foo)) - ExpressionStmt:3( - StrExpr(foobar)) - ExpressionStmt:4( - StrExpr(\nfoo\u000abar)) - ExpressionStmt:6( - StrExpr(fo''bar)) - ExpressionStmt:7( - StrExpr()) - ExpressionStmt:8( - StrExpr(foo)) - ExpressionStmt:9( - StrExpr(foobar)) - ExpressionStmt:10( - StrExpr(\nfoo\u000abar)) - ExpressionStmt:12( - StrExpr(fo""bar))) - -[case testRawStr] -r'x\n\'' -r"x\n\"" -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr(x\n')) - ExpressionStmt:2( - StrExpr(x\n"))) ---" fix syntax highlight - -[case testBytes] -b'foo' -b"foo\ -bar" -br'x\n\'' -[out] -MypyFile:1( - ExpressionStmt:1( - BytesExpr(foo)) - ExpressionStmt:2( - BytesExpr(foobar)) - ExpressionStmt:3( - BytesExpr(x\n'))) - -[case testEscapesInStrings] -'\r\n\t\x2f\u123f' -b'\r\n\t\x2f\u123f' -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr(\u000d\u000a\u0009/\u123f)) - ExpressionStmt:2( - BytesExpr(\u000d\u000a\u0009/\\u123f))) --- Note \\u in the b'...' case (\u sequence not translated) - -[case testEscapedQuote] -'\'' -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr('))) ---' - -[case testOctalEscapes] -'\0\1\177\1234' -b'\1\476' -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr(\u0000\u0001\u007fS4)) - ExpressionStmt:2( - BytesExpr(\u0001>))) - -[case testUnicodeLiteralInPython3] -u'foo' -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr(foo))) - -[case testArrays] -a = [] -a = [1, 2] -a[[1]] = a[2] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(a) - ListExpr:1()) - AssignmentStmt:2( - NameExpr(a) - ListExpr:2( - IntExpr(1) - IntExpr(2))) - AssignmentStmt:3( - IndexExpr:3( - NameExpr(a) - ListExpr:3( - IntExpr(1))) - IndexExpr:3( - NameExpr(a) - IntExpr(2)))) - -[case testTuples] -() -(1,) -(1, foo) -a, b = 1, (2, 3) -[out] -MypyFile:1( - ExpressionStmt:1( - TupleExpr:1()) - ExpressionStmt:2( - ParenExpr:2( - TupleExpr:2( - IntExpr(1)))) - ExpressionStmt:3( - ParenExpr:3( - TupleExpr:3( - IntExpr(1) - NameExpr(foo)))) - AssignmentStmt:4( - TupleExpr:4( - NameExpr(a) - NameExpr(b)) - TupleExpr:4( - IntExpr(1) - ParenExpr:4( - TupleExpr:4( - IntExpr(2) - IntExpr(3)))))) - -[case testSimpleFunction] -def main(): - 1 -[out] -MypyFile:1( - FuncDef:1( - main - Block:1( - ExpressionStmt:2( - IntExpr(1))))) - -[case testPass] -def f(): - pass -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - PassStmt:2()))) - -[case testIf] -if 1: - 2 -[out] -MypyFile:1( - IfStmt:1( - If( - IntExpr(1)) - Then( - ExpressionStmt:2( - IntExpr(2))))) - -[case testIfElse] -if 1: - 2 -else: - 3 -[out] -MypyFile:1( - IfStmt:1( - If( - IntExpr(1)) - Then( - ExpressionStmt:2( - IntExpr(2))) - Else( - ExpressionStmt:4( - IntExpr(3))))) - -[case testIfElif] -if 1: - 2 -elif 3: - 4 -elif 5: - 6 -else: - 7 -[out] -MypyFile:1( - IfStmt:1( - If( - IntExpr(1)) - Then( - ExpressionStmt:2( - IntExpr(2))) - If( - IntExpr(3)) - Then( - ExpressionStmt:4( - IntExpr(4))) - If( - IntExpr(5)) - Then( - ExpressionStmt:6( - IntExpr(6))) - Else( - ExpressionStmt:8( - IntExpr(7))))) - -[case testWhile] -while 1: - pass -[out] -MypyFile:1( - WhileStmt:1( - IntExpr(1) - Block:1( - PassStmt:2()))) - -[case testReturn] -def f(): - return 1 -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - ReturnStmt:2( - IntExpr(1))))) - - -[case testReturnWithoutValue] -def f(): - return -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - ReturnStmt:2()))) - -[case testBreak] -while 1: - break -[out] -MypyFile:1( - WhileStmt:1( - IntExpr(1) - Block:1( - BreakStmt:2()))) - -[case testLargeBlock] -if 1: - x = 1 - while 2: - pass - y = 2 -[out] -MypyFile:1( - IfStmt:1( - If( - IntExpr(1)) - Then( - AssignmentStmt:2( - NameExpr(x) - IntExpr(1)) - WhileStmt:3( - IntExpr(2) - Block:3( - PassStmt:4())) - AssignmentStmt:5( - NameExpr(y) - IntExpr(2))))) - -[case testSimpleClass] -class A: - def f(self): - pass -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - f - Args( - Var(self)) - Block:2( - PassStmt:3())))) - -[case testGlobalVarWithType] -x = 0 # type: int -y = False # type: bool -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(0) - int?) - AssignmentStmt:2( - NameExpr(y) - NameExpr(False) - bool?)) - -[case testLocalVarWithType] -def f(): - x = 0 # type: int - y = False # type: bool - a = None # type: Any -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - AssignmentStmt:2( - NameExpr(x) - IntExpr(0) - int?) - AssignmentStmt:3( - NameExpr(y) - NameExpr(False) - bool?) - AssignmentStmt:4( - NameExpr(a) - NameExpr(None) - Any?)))) - -[case testLocalVarWithTypeOnNextLine] -x = 0 - # type: int -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(0) - int?)) - -[case testFunctionDefWithType] -def f(y: str) -> int: - return -class A: - def f(self, a: int, b: Any) -> x: - pass - def g(self) -> Any: - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(y)) - def (y: str?) -> int? - Block:1( - ReturnStmt:2())) - ClassDef:3( - A - FuncDef:4( - f - Args( - Var(self) - Var(a) - Var(b)) - def (self: Any, a: int?, b: Any?) -> x? - Block:4( - PassStmt:5())) - FuncDef:6( - g - Args( - Var(self)) - def (self: Any) -> Any? - Block:6( - PassStmt:7())))) - -[case testFuncWithNoneReturn] -def f() -> None: - pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> None? - Block:1( - PassStmt:2()))) - -[case testVarDefWithGenericType] -x = None # type: List[str] -y = None # type: Dict[int, Any] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - NameExpr(None) - List?[str?]) - AssignmentStmt:2( - NameExpr(y) - NameExpr(None) - Dict?[int?, Any?])) - -[case testSignatureWithGenericTypes] -def f(y: t[Any, x]) -> a[b[c], d]: - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(y)) - def (y: t?[Any?, x?]) -> a?[b?[c?], d?] - Block:1( - PassStmt:2()))) - -[case testParsingExpressionsWithLessAndGreaterThan] -# The operators < > can sometimes be confused with generic types. -x = a < b > c -f(x < b, y > c) -a < b > 1 -x < b, y > 2 -(a < b > c) -[out] -MypyFile:1( - AssignmentStmt:2( - NameExpr(x) - ComparisonExpr:2( - < - > - NameExpr(a) - NameExpr(b) - NameExpr(c))) - ExpressionStmt:3( - CallExpr:3( - NameExpr(f) - Args( - ComparisonExpr:3( - < - NameExpr(x) - NameExpr(b)) - ComparisonExpr:3( - > - NameExpr(y) - NameExpr(c))))) - ExpressionStmt:4( - ComparisonExpr:4( - < - > - NameExpr(a) - NameExpr(b) - IntExpr(1))) - ExpressionStmt:5( - TupleExpr:5( - ComparisonExpr:5( - < - NameExpr(x) - NameExpr(b)) - ComparisonExpr:5( - > - NameExpr(y) - IntExpr(2)))) - ExpressionStmt:6( - ParenExpr:6( - ComparisonExpr:6( - < - > - NameExpr(a) - NameExpr(b) - NameExpr(c))))) - -[case testLineContinuation] -if (1 + - 2): - pass -[out] -MypyFile:1( - IfStmt:1( - If( - ParenExpr:1( - OpExpr:1( - + - IntExpr(1) - IntExpr(2)))) - Then( - PassStmt:3()))) - -[case testMultipleVarDef] -x, y = z # type: int, a[c] -[out] -MypyFile:1( - AssignmentStmt:1( - TupleExpr:1( - NameExpr(x) - NameExpr(y)) - NameExpr(z) - Tuple[int?, a?[c?]])) - -[case testMultipleVarDef2] -(xx, z, i) = 1 # type: (a[c], Any, int) -[out] -MypyFile:1( - AssignmentStmt:1( - ParenExpr:1( - TupleExpr:1( - NameExpr(xx) - NameExpr(z) - NameExpr(i))) - IntExpr(1) - Tuple[a?[c?], Any?, int?])) - -[case testMultipleVarDef3] -(xx, (z, i)) = 1 # type: (a[c], (Any, int)) -[out] -MypyFile:1( - AssignmentStmt:1( - ParenExpr:1( - TupleExpr:1( - NameExpr(xx) - ParenExpr:1( - TupleExpr:1( - NameExpr(z) - NameExpr(i))))) - IntExpr(1) - Tuple[a?[c?], Tuple[Any?, int?]])) - -[case testAnnotateAssignmentViaSelf] -class A: - def __init__(self): - self.x = 1 # type: int -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - __init__ - Args( - Var(self)) - Block:2( - AssignmentStmt:3( - MemberExpr:3( - NameExpr(self) - x) - IntExpr(1) - int?))))) - -[case testCommentAfterTypeComment] -x = 0 # type: int # bar! -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(0) - int?)) - -[case testMultilineAssignmentAndAnnotations] -(x, - y) = (1, - 2) # type: foo, bar -[out] -MypyFile:1( - AssignmentStmt:1( - ParenExpr:1( - TupleExpr:1( - NameExpr(x) - NameExpr(y))) - ParenExpr:2( - TupleExpr:2( - IntExpr(1) - IntExpr(2))) - Tuple[foo?, bar?])) - -[case testWhitespaceAndCommentAnnotation] -x = 1#type:int -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(1) - int?)) - -[case testWhitespaceAndCommentAnnotation2] -x = 1# type: int -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(1) - int?)) - -[case testWhitespaceAndCommentAnnotation3] -x = 1# type : int # not recognized! -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(1))) - -[case testInvalidAnnotation] -x=1 ##type: int -y=1 #.type: int -z=1 # Type: int -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - IntExpr(1)) - AssignmentStmt:2( - NameExpr(y) - IntExpr(1)) - AssignmentStmt:3( - NameExpr(z) - IntExpr(1))) - -[case testEmptyClass] -class C: - pass -[out] -MypyFile:1( - ClassDef:1( - C - PassStmt:2())) - -[case testOperatorPrecedence] -a | b ^ c -a & b << c -[out] -MypyFile:1( - ExpressionStmt:1( - OpExpr:1( - | - NameExpr(a) - OpExpr:1( - ^ - NameExpr(b) - NameExpr(c)))) - ExpressionStmt:2( - OpExpr:2( - & - NameExpr(a) - OpExpr:2( - << - NameExpr(b) - NameExpr(c))))) - -[case testOperatorAssociativity] -1 - 2 + 3 -1 << 2 << 3 -[out] -MypyFile:1( - ExpressionStmt:1( - OpExpr:1( - + - OpExpr:1( - - - IntExpr(1) - IntExpr(2)) - IntExpr(3))) - ExpressionStmt:2( - OpExpr:2( - << - OpExpr:2( - << - IntExpr(1) - IntExpr(2)) - IntExpr(3)))) - -[case testUnaryOperators] --2 * +3 * ~3 * 2 -~3**2 -[out] -MypyFile:1( - ExpressionStmt:1( - OpExpr:1( - * - OpExpr:1( - * - OpExpr:1( - * - UnaryExpr:1( - - - IntExpr(2)) - UnaryExpr:1( - + - IntExpr(3))) - UnaryExpr:1( - ~ - IntExpr(3))) - IntExpr(2))) - ExpressionStmt:2( - UnaryExpr:2( - ~ - OpExpr:2( - ** - IntExpr(3) - IntExpr(2))))) - -[case testSingleLineBodies] -if 1: pass -while 1: pass -def f(): pass -def g() -> int: return 1 -[out] -MypyFile:1( - IfStmt:1( - If( - IntExpr(1)) - Then( - PassStmt:1())) - WhileStmt:2( - IntExpr(1) - Block:2( - PassStmt:2())) - FuncDef:3( - f - Block:3( - PassStmt:3())) - FuncDef:4( - g - def () -> int? - Block:4( - ReturnStmt:4( - IntExpr(1))))) - -[case testForStatement] -for x in y: - pass -for x, (y, w) in z: - 1 -for [x, (y, w)] in z: - 1 -[out] -MypyFile:1( - ForStmt:1( - NameExpr(x) - NameExpr(y) - Block:1( - PassStmt:2())) - ForStmt:3( - TupleExpr:3( - NameExpr(x) - ParenExpr:3( - TupleExpr:3( - NameExpr(y) - NameExpr(w)))) - NameExpr(z) - Block:3( - ExpressionStmt:4( - IntExpr(1)))) - ForStmt:5( - ListExpr:5( - NameExpr(x) - ParenExpr:5( - TupleExpr:5( - NameExpr(y) - NameExpr(w)))) - NameExpr(z) - Block:5( - ExpressionStmt:6( - IntExpr(1))))) - -[case testGlobalDecl] -global x -def f(): - global x, y -[out] -MypyFile:1( - GlobalDecl:1( - x) - FuncDef:2( - f - Block:2( - GlobalDecl:3( - x - y)))) - -[case testNonlocalDecl] -def f(): - def g(): - nonlocal x, y -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - FuncDef:2( - g - Block:2( - NonlocalDecl:3( - x - y)))))) - -[case testRaiseStatement] -raise foo -[out] -MypyFile:1( - RaiseStmt:1( - NameExpr(foo))) - -[case testRaiseWithoutArg] -try: - pass -except: - raise -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - Block:3( - RaiseStmt:4()))) - -[case testRaiseFrom] -raise e from x -[out] -MypyFile:1( - RaiseStmt:1( - NameExpr(e) - NameExpr(x))) - -[case testBaseclasses] -class A(B): - pass -class A(B[T], C[Any, d[x]]): - pass -[out] -MypyFile:1( - ClassDef:1( - A - BaseType( - B?) - PassStmt:2()) - ClassDef:3( - A - BaseType( - B?[T?] - C?[Any?, d?[x?]]) - PassStmt:4())) - -[case testIsNot] -x is not y -[out] -MypyFile:1( - ExpressionStmt:1( - ComparisonExpr:1( - is not - NameExpr(x) - NameExpr(y)))) - -[case testNotAsRightOperand] -x in not y -[out] -MypyFile:1( - ExpressionStmt:1( - ComparisonExpr:1( - in - NameExpr(x) - UnaryExpr:1( - not - NameExpr(y))))) - -[case testNotIn] -x not in y -not x not in y -x not in y | z -[out] -MypyFile:1( - ExpressionStmt:1( - ComparisonExpr:1( - not in - NameExpr(x) - NameExpr(y))) - ExpressionStmt:2( - UnaryExpr:2( - not - ComparisonExpr:2( - not in - NameExpr(x) - NameExpr(y)))) - ExpressionStmt:3( - ComparisonExpr:3( - not in - NameExpr(x) - OpExpr:3( - | - NameExpr(y) - NameExpr(z))))) - -[case testNotAsBinaryOp] -x not y -x not is y -[out] -, line 1: Parse error before "y" -, line 2: Parse error before is - -[case testBinaryNegAsBinaryOp] -1 ~ 2 -[out] -, line 1: Parse error before ~ - -[case testDictionaryExpression] -{} -{1:x} -{1:x, 2 or 1:2 and 3} -[out] -MypyFile:1( - ExpressionStmt:1( - DictExpr:1()) - ExpressionStmt:2( - DictExpr:2( - IntExpr(1) - NameExpr(x))) - ExpressionStmt:3( - DictExpr:3( - IntExpr(1) - NameExpr(x) - OpExpr:3( - or - IntExpr(2) - IntExpr(1)) - OpExpr:3( - and - IntExpr(2) - IntExpr(3))))) - -[case testImport] -import x -import y.z.foo, __foo__.bar -[out] -MypyFile:1( - Import:1(x : x) - Import:2(y.z.foo : y.z.foo, __foo__.bar : __foo__.bar)) - -[case testVariableTypeWithQualifiedName] -x = None # type: x.y -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - NameExpr(None) - x.y?)) - -[case testTypeInSignatureWithQualifiedName] -def f() -> x.y[a.b.c]: pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> x.y?[a.b.c?] - Block:1( - PassStmt:1()))) - -[case testImportFrom] -from m import x -from m.n import x, y, z -[out] -MypyFile:1( - ImportFrom:1(m, [x : x]) - ImportFrom:2(m.n, [x : x, y : y, z : z])) - -[case testImportFromAs] -from m import x as y -from x import y, z as a, c -[out] -MypyFile:1( - ImportFrom:1(m, [x : y]) - ImportFrom:2(x, [y : y, z : a, c : c])) - -[case testImportStar] -from x import * -[out] -MypyFile:1( - ImportAll:1(x)) - -[case testImportsInDifferentPlaces] -1 -import x -def f(): - from x import y - from z import * -[out] -MypyFile:1( - ExpressionStmt:1( - IntExpr(1)) - Import:2(x : x) - FuncDef:3( - f - Block:3( - ImportFrom:4(x, [y : y]) - ImportAll:5(z)))) - -[case testImportWithExtraComma] -from x import (y, z,) -[out] -MypyFile:1( - ImportFrom:1(x, [y : y, z : z])) - -[case testDefaultArgs] -def f(x=1): - pass -def g(x, y=1+2, z=(1, 2)): - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - Init( - AssignmentStmt:1( - NameExpr(x) - IntExpr(1))) - Block:1( - PassStmt:2())) - FuncDef:3( - g - Args( - Var(x) - Var(y) - Var(z)) - Init( - AssignmentStmt:3( - NameExpr(y) - OpExpr:3( - + - IntExpr(1) - IntExpr(2))) - AssignmentStmt:3( - NameExpr(z) - ParenExpr:3( - TupleExpr:3( - IntExpr(1) - IntExpr(2))))) - Block:3( - PassStmt:4()))) - -[case testTryFinally] -try: - 1 -finally: - 2 -[out] -MypyFile:1( - TryStmt:1( - Block:1( - ExpressionStmt:2( - IntExpr(1))) - Finally( - ExpressionStmt:4( - IntExpr(2))))) - -[case testTry] -try: - 1 -except x: - 2 -[out] -MypyFile:1( - TryStmt:1( - Block:1( - ExpressionStmt:2( - IntExpr(1))) - NameExpr(x) - Block:3( - ExpressionStmt:4( - IntExpr(2))))) - -[case testComplexTry] -try: - 1 -except x as y: - 2 -except x.y: - 3 -[out] -MypyFile:1( - TryStmt:1( - Block:1( - ExpressionStmt:2( - IntExpr(1))) - NameExpr(x) - NameExpr(y) - Block:3( - ExpressionStmt:4( - IntExpr(2))) - MemberExpr:5( - NameExpr(x) - y) - Block:5( - ExpressionStmt:6( - IntExpr(3))))) - -[case testGeneratorExpression] -x for y in z -[out] -MypyFile:1( - ExpressionStmt:1( - GeneratorExpr:1( - NameExpr(x) - NameExpr(y) - NameExpr(z)))) - -[case testGeneratorExpressionNested] -x for y, (p, q) in z -[out] -MypyFile:1( - ExpressionStmt:1( - GeneratorExpr:1( - NameExpr(x) - TupleExpr:1( - NameExpr(y) - ParenExpr:1( - TupleExpr:1( - NameExpr(p) - NameExpr(q)))) - NameExpr(z)))) - -[case testListComprehension] -x=[x for y in z] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - ListComprehension:1( - GeneratorExpr:1( - NameExpr(x) - NameExpr(y) - NameExpr(z))))) - -[case testComplexListComprehension] -x=[(x, y) for y, z in 1, 2] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - ListComprehension:1( - GeneratorExpr:1( - ParenExpr:1( - TupleExpr:1( - NameExpr(x) - NameExpr(y))) - TupleExpr:1( - NameExpr(y) - NameExpr(z)) - TupleExpr:1( - IntExpr(1) - IntExpr(2)))))) - -[case testListComprehension2] -([x + 1 for x in a]) -[out] -MypyFile:1( - ExpressionStmt:1( - ParenExpr:1( - ListComprehension:1( - GeneratorExpr:1( - OpExpr:1( - + - NameExpr(x) - IntExpr(1)) - NameExpr(x) - NameExpr(a)))))) - -[case testSlices] -x[1:2] -x[:1] -x[1:] -x[:] -[out] -MypyFile:1( - ExpressionStmt:1( - IndexExpr:1( - NameExpr(x) - SliceExpr:1( - IntExpr(1) - IntExpr(2)))) - ExpressionStmt:2( - IndexExpr:2( - NameExpr(x) - SliceExpr:2( - - IntExpr(1)))) - ExpressionStmt:3( - IndexExpr:3( - NameExpr(x) - SliceExpr:3( - IntExpr(1) - ))) - ExpressionStmt:4( - IndexExpr:4( - NameExpr(x) - SliceExpr:4( - - )))) - -[case testSliceWithStride] -x[1:2:3] -x[1::2] -x[:1:2] -x[::2] -x[1:2:] -[out] -MypyFile:1( - ExpressionStmt:1( - IndexExpr:1( - NameExpr(x) - SliceExpr:1( - IntExpr(1) - IntExpr(2) - IntExpr(3)))) - ExpressionStmt:2( - IndexExpr:2( - NameExpr(x) - SliceExpr:2( - IntExpr(1) - - IntExpr(2)))) - ExpressionStmt:3( - IndexExpr:3( - NameExpr(x) - SliceExpr:3( - - IntExpr(1) - IntExpr(2)))) - ExpressionStmt:4( - IndexExpr:4( - NameExpr(x) - SliceExpr:4( - - - IntExpr(2)))) - ExpressionStmt:5( - IndexExpr:5( - NameExpr(x) - SliceExpr:5( - IntExpr(1) - IntExpr(2))))) - -[case testYield] -def f(): - yield x + 1 -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - YieldStmt:2( - OpExpr:2( - + - NameExpr(x) - IntExpr(1)))))) - -[case testYieldFrom] -def f(): - yield from h() -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - YieldFromStmt:2( - CallExpr:2( - NameExpr(h) - Args()))))) - -[case testYieldFromAssignment] -def f(): - a = yield from h() -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - AssignmentStmt:2( - NameExpr(a) - YieldFromExpr:2( - CallExpr:2( - NameExpr(h) - Args())))))) - -[case testDel] -del x -del x[0], y[1] -[out] -MypyFile:1( - DelStmt:1( - NameExpr(x)) - DelStmt:2( - TupleExpr:2( - IndexExpr:2( - NameExpr(x) - IntExpr(0)) - IndexExpr:2( - NameExpr(y) - IntExpr(1))))) - -[case testExtraCommas] -1, 2, -+[1, 2,] -f(1,) -{1:2,} -[out] -MypyFile:1( - ExpressionStmt:1( - TupleExpr:1( - IntExpr(1) - IntExpr(2))) - ExpressionStmt:2( - UnaryExpr:2( - + - ListExpr:2( - IntExpr(1) - IntExpr(2)))) - ExpressionStmt:3( - CallExpr:3( - NameExpr(f) - Args( - IntExpr(1)))) - ExpressionStmt:4( - DictExpr:4( - IntExpr(1) - IntExpr(2)))) - -[case testExtraCommaInFunc] -def f(x,): - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - Block:1( - PassStmt:2()))) - -[case testLambda] -lambda: 1 -lambda x: y + 1 -lambda x, y: 1 -[out] -MypyFile:1( - ExpressionStmt:1( - FuncExpr:1( - Block:1( - ReturnStmt:1( - IntExpr(1))))) - ExpressionStmt:2( - FuncExpr:2( - Args( - Var(x)) - Block:2( - ReturnStmt:2( - OpExpr:2( - + - NameExpr(y) - IntExpr(1)))))) - ExpressionStmt:3( - FuncExpr:3( - Args( - Var(x) - Var(y)) - Block:3( - ReturnStmt:3( - IntExpr(1)))))) - -[case testComplexLambda] -lambda x=2: x -[out] -MypyFile:1( - ExpressionStmt:1( - FuncExpr:1( - Args( - Var(x)) - Init( - AssignmentStmt:1( - NameExpr(x) - IntExpr(2))) - Block:1( - ReturnStmt:1( - NameExpr(x)))))) - -[case testLambdaPrecedence] -lambda x: 1, 2 -[out] -MypyFile:1( - ExpressionStmt:1( - TupleExpr:1( - FuncExpr:1( - Args( - Var(x)) - Block:1( - ReturnStmt:1( - IntExpr(1)))) - IntExpr(2)))) - -[case testForIndicesInParens] -for (i, j) in x: - pass -[out] -MypyFile:1( - ForStmt:1( - ParenExpr:1( - TupleExpr:1( - NameExpr(i) - NameExpr(j))) - NameExpr(x) - Block:1( - PassStmt:2()))) - -[case testGeneratorWithCondition] -x for y in z if 0 -[out] -MypyFile:1( - ExpressionStmt:1( - GeneratorExpr:1( - NameExpr(x) - NameExpr(y) - NameExpr(z) - IntExpr(0)))) - -[case testListComprehensionWithCondition] -raise [x for y in z if 0] -[out] -MypyFile:1( - RaiseStmt:1( - ListComprehension:1( - GeneratorExpr:1( - NameExpr(x) - NameExpr(y) - NameExpr(z) - IntExpr(0))))) - -[case testListComprehensionWithConditions] -raise [x for y in z if 0 if 1] -[out] -MypyFile:1( - RaiseStmt:1( - ListComprehension:1( - GeneratorExpr:1( - NameExpr(x) - NameExpr(y) - NameExpr(z) - IntExpr(0) - IntExpr(1))))) - -[case testListComprehensionWithCrazyConditions] -raise [x for y in z if (1 if 2 else 3) if 1] -[out] -MypyFile:1( - RaiseStmt:1( - ListComprehension:1( - GeneratorExpr:1( - NameExpr(x) - NameExpr(y) - NameExpr(z) - ParenExpr:1( - ConditionalExpr:1( - Condition( - IntExpr(2)) - IntExpr(1) - IntExpr(3))) - IntExpr(1))))) - -[case testDictionaryComprehension] -a = {x: y for x, y in xys} -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(a) - DictionaryComprehension:1( - NameExpr(x) - NameExpr(y) - TupleExpr:1( - NameExpr(x) - NameExpr(y)) - NameExpr(xys)))) - -[case testDictionaryComprehensionComplex] -a = {x: y for x, y in xys for p, q in pqs if c} -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(a) - DictionaryComprehension:1( - NameExpr(x) - NameExpr(y) - TupleExpr:1( - NameExpr(x) - NameExpr(y)) - TupleExpr:1( - NameExpr(p) - NameExpr(q)) - NameExpr(xys) - NameExpr(pqs) - NameExpr(c)))) - -[case testSetComprehension] -a = {i for i in l} -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(a) - SetComprehension:1( - GeneratorExpr:1( - NameExpr(i) - NameExpr(i) - NameExpr(l))))) - -[case testSetComprehensionComplex] -a = {x + p for x in xys for p in pqs if c} -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(a) - SetComprehension:1( - GeneratorExpr:1( - OpExpr:1( - + - NameExpr(x) - NameExpr(p)) - NameExpr(x) - NameExpr(p) - NameExpr(xys) - NameExpr(pqs) - NameExpr(c))))) - -[case testWithStatement] -with open('foo') as f: - pass -[out] -MypyFile:1( - WithStmt:1( - Expr( - CallExpr:1( - NameExpr(open) - Args( - StrExpr(foo)))) - Name( - NameExpr(f)) - Block:1( - PassStmt:2()))) - -[case testWithStatementWithoutTarget] -with foo: - pass -[out] -MypyFile:1( - WithStmt:1( - Expr( - NameExpr(foo)) - Block:1( - PassStmt:2()))) - -[case testHexAndOctLiterals] -0xa, 0Xaf, 0o7, 0O12 -[out] -MypyFile:1( - ExpressionStmt:1( - TupleExpr:1( - IntExpr(10) - IntExpr(175) - IntExpr(7) - IntExpr(10)))) - -[case testImportFromWithParens] -from x import (y) -from x import (y, - z) -[out] -MypyFile:1( - ImportFrom:1(x, [y : y]) - ImportFrom:2(x, [y : y, z : z])) - -[case testContinueStmt] -while 1: - continue -[out] -MypyFile:1( - WhileStmt:1( - IntExpr(1) - Block:1( - ContinueStmt:2()))) - -[case testStrLiteralConcatenate] -'f' 'bar' -('x' - 'y' - 'z') -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr(fbar)) - ExpressionStmt:2( - ParenExpr:2( - StrExpr(xyz)))) - -[case testCatchAllExcept] -try: - 1 -except: - pass -try: - 1 -except x: - pass -except: - 2 -[out] -MypyFile:1( - TryStmt:1( - Block:1( - ExpressionStmt:2( - IntExpr(1))) - Block:3( - PassStmt:4())) - TryStmt:5( - Block:5( - ExpressionStmt:6( - IntExpr(1))) - NameExpr(x) - Block:7( - PassStmt:8()) - Block:9( - ExpressionStmt:10( - IntExpr(2))))) - -[case testTryElse] -try: - pass -except x: - 1 -else: - 2 -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - NameExpr(x) - Block:3( - ExpressionStmt:4( - IntExpr(1))) - Else( - ExpressionStmt:6( - IntExpr(2))))) - -[case testExceptWithMultipleTypes] -try: - pass -except (x, y): - pass -except (a, b, c) as e: - pass -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - ParenExpr:3( - TupleExpr:3( - NameExpr(x) - NameExpr(y))) - Block:3( - PassStmt:4()) - ParenExpr:5( - TupleExpr:5( - NameExpr(a) - NameExpr(b) - NameExpr(c))) - NameExpr(e) - Block:5( - PassStmt:6()))) - -[case testNestedFunctions] -def f(): - def g(): - pass -def h() -> int: - def g() -> int: - pass -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - FuncDef:2( - g - Block:2( - PassStmt:3())))) - FuncDef:4( - h - def () -> int? - Block:4( - FuncDef:5( - g - def () -> int? - Block:5( - PassStmt:6()))))) - -[case testStatementsAndDocStringsInClassBody] -class A: - "doc string" - x = y - def f(self): - pass -[out] -MypyFile:1( - ClassDef:1( - A - ExpressionStmt:2( - StrExpr(doc string)) - AssignmentStmt:3( - NameExpr(x) - NameExpr(y)) - FuncDef:4( - f - Args( - Var(self)) - Block:4( - PassStmt:5())))) - -[case testSingleLineClass] -class a: pass -[out] -MypyFile:1( - ClassDef:1( - a - PassStmt:1())) - -[case testDecorator] -@property -def f(): - pass -[out] -MypyFile:1( - Decorator:1( - Var(f) - NameExpr(property) - FuncDef:2( - f - Block:2( - PassStmt:3())))) - -[case testComplexDecorator] -@foo(bar, 1) -@zar -def f() -> int: - pass -[out] -MypyFile:1( - Decorator:1( - Var(f) - CallExpr:1( - NameExpr(foo) - Args( - NameExpr(bar) - IntExpr(1))) - NameExpr(zar) - FuncDef:3( - f - def () -> int? - Block:3( - PassStmt:4())))) - -[case testKeywordArgInCall] -f(x=1) -[out] -MypyFile:1( - ExpressionStmt:1( - CallExpr:1( - NameExpr(f) - Args() - KwArgs( - x - IntExpr(1))))) - -[case testComplexKeywordArgs] -f(x, y=1 or 2, z=y) -[out] -MypyFile:1( - ExpressionStmt:1( - CallExpr:1( - NameExpr(f) - Args( - NameExpr(x)) - KwArgs( - y - OpExpr:1( - or - IntExpr(1) - IntExpr(2))) - KwArgs( - z - NameExpr(y))))) - -[case testChainedAssignment] -x = z = 1 -[out] -MypyFile:1( - AssignmentStmt:1( - Lvalues( - NameExpr(x) - NameExpr(z)) - IntExpr(1))) - -[case testVarArgs] -def f(x, *a): pass -f(1, *2) -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - VarArg( - Var(a)) - Block:1( - PassStmt:1())) - ExpressionStmt:2( - CallExpr:2( - NameExpr(f) - Args( - IntExpr(1) - IntExpr(2)) - VarArg))) - -[case testVarArgWithType] -def f(x: str, *a: int): pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - def (x: str?, *a: int?) -> Any - VarArg( - Var(a)) - Block:1( - PassStmt:1()))) - -[case testDictVarArgs] -def f(x, **a): pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - DictVarArg( - Var(a)) - Block:1( - PassStmt:1()))) - -[case testBothVarArgs] -def f(x, *a, **b): pass -def g(*a, **b): pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - VarArg( - Var(a)) - DictVarArg( - Var(b)) - Block:1( - PassStmt:1())) - FuncDef:2( - g - VarArg( - Var(a)) - DictVarArg( - Var(b)) - Block:2( - PassStmt:2()))) - -[case testDictVarArgsWithType] -def f(x: X, **a: A) -> None: pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - def (x: X?, **a: A?) -> None? - DictVarArg( - Var(a)) - Block:1( - PassStmt:1()))) - -[case testCallDictVarArgs] -f(**x) -f(x, **y) -f(*x, **y) -f(x, *y, **z) -[out] -MypyFile:1( - ExpressionStmt:1( - CallExpr:1( - NameExpr(f) - Args() - DictVarArg( - NameExpr(x)))) - ExpressionStmt:2( - CallExpr:2( - NameExpr(f) - Args( - NameExpr(x)) - DictVarArg( - NameExpr(y)))) - ExpressionStmt:3( - CallExpr:3( - NameExpr(f) - Args( - NameExpr(x)) - VarArg - DictVarArg( - NameExpr(y)))) - ExpressionStmt:4( - CallExpr:4( - NameExpr(f) - Args( - NameExpr(x) - NameExpr(y)) - VarArg - DictVarArg( - NameExpr(z))))) - -[case testAssert] -assert x == y -[out] -MypyFile:1( - AssertStmt:1( - ComparisonExpr:1( - == - NameExpr(x) - NameExpr(y)))) - -[case testYieldWithoutExpressions] -def f(): - yield -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - YieldStmt:2()))) - -[case testConditionalExpression] -x if y else z -[out] -MypyFile:1( - ExpressionStmt:1( - ConditionalExpr:1( - Condition( - NameExpr(y)) - NameExpr(x) - NameExpr(z)))) - -[case testConditionalExpressionInListComprehension] -a = [x if y else z for a in b] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(a) - ListComprehension:1( - GeneratorExpr:1( - ConditionalExpr:1( - Condition( - NameExpr(y)) - NameExpr(x) - NameExpr(z)) - NameExpr(a) - NameExpr(b))))) - -[case testConditionalExpressionInTuple] -1 if 2 else 3, 4 -[out] -MypyFile:1( - ExpressionStmt:1( - TupleExpr:1( - ConditionalExpr:1( - Condition( - IntExpr(2)) - IntExpr(1) - IntExpr(3)) - IntExpr(4)))) - -[case testSetLiteral] -{x or y} -{1, 2} -[out] -MypyFile:1( - ExpressionStmt:1( - SetExpr:1( - OpExpr:1( - or - NameExpr(x) - NameExpr(y)))) - ExpressionStmt:2( - SetExpr:2( - IntExpr(1) - IntExpr(2)))) - -[case testSetLiteralWithExtraComma] -{x,} -[out] -MypyFile:1( - ExpressionStmt:1( - SetExpr:1( - NameExpr(x)))) - -[case testImportAs] -import x as y -import x, z as y, a.b as c -[out] -MypyFile:1( - Import:1(x : y) - Import:2(x : x, z : y, a.b : c)) - -[case testForAndElse] -for x in y: - pass -else: - x -[out] -MypyFile:1( - ForStmt:1( - NameExpr(x) - NameExpr(y) - Block:1( - PassStmt:2()) - Else( - ExpressionStmt:4( - NameExpr(x))))) - -[case testWhileAndElse] -while x: - pass -else: - y -[out] -MypyFile:1( - WhileStmt:1( - NameExpr(x) - Block:1( - PassStmt:2()) - Else( - ExpressionStmt:4( - NameExpr(y))))) - -[case testWithAndMultipleOperands] -with x as y, a as b: - pass -with x(), y(): - pass -[out] -MypyFile:1( - WithStmt:1( - Expr( - NameExpr(x)) - Name( - NameExpr(y)) - Expr( - NameExpr(a)) - Name( - NameExpr(b)) - Block:1( - PassStmt:2())) - WithStmt:3( - Expr( - CallExpr:3( - NameExpr(x) - Args())) - Expr( - CallExpr:3( - NameExpr(y) - Args())) - Block:3( - PassStmt:4()))) - -[case testOperatorAssignment] -x += 1 -x -= 1 -x *= 1 -x /= 1 -x //= 1 -x %= 1 -x **= 1 -x |= 1 -x &= 1 -x ^= 1 -x >>= 1 -x <<= 1 -[out] -MypyFile:1( - OperatorAssignmentStmt:1( - + - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:2( - - - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:3( - * - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:4( - / - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:5( - // - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:6( - % - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:7( - ** - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:8( - | - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:9( - & - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:10( - ^ - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:11( - >> - NameExpr(x) - IntExpr(1)) - OperatorAssignmentStmt:12( - << - NameExpr(x) - IntExpr(1))) - -[case testNestedClasses] -class A: - class B: - pass - class C: - pass -[out] -MypyFile:1( - ClassDef:1( - A - ClassDef:2( - B - PassStmt:3()) - ClassDef:4( - C - PassStmt:5()))) - -[case testTryWithExceptAndFinally] -try: - pass -except x: - x -finally: - y -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - NameExpr(x) - Block:3( - ExpressionStmt:4( - NameExpr(x))) - Finally( - ExpressionStmt:6( - NameExpr(y))))) - -[case testBareAsteriskInFuncDef] -def f(x, *, y=1): pass -[out] -MypyFile:1( - FuncDef:1( - f - MaxPos(1) - Args( - Var(x) - Var(y)) - Init( - AssignmentStmt:1( - NameExpr(y) - IntExpr(1))) - Block:1( - PassStmt:1()))) - -[case testBareAsteriskInFuncDefWithSignature] -def f(x: A, *, y: B = 1) -> None: pass -[out] -MypyFile:1( - FuncDef:1( - f - MaxPos(1) - Args( - Var(x) - Var(y)) - def (x: A?, *, y: B?) -> None? - Init( - AssignmentStmt:1( - NameExpr(y) - IntExpr(1))) - Block:1( - PassStmt:1()))) - -[case testBareAsteriskNamedDefault] -def f(*, y: B = 1) -> None: pass -[out] -MypyFile:1( - FuncDef:1( - f - MaxPos(0) - Args( - Var(y)) - def (*, y: B?) -> None? - Init( - AssignmentStmt:1( - NameExpr(y) - IntExpr(1))) - Block:1( - PassStmt:1()))) - -[case testBareAsteriskNamedNoDefault] -def f(*, y: B) -> None: pass -[out] -MypyFile:1( - FuncDef:1( - f - MaxPos(0) - Args( - Var(y)) - def (*, y: B?) -> None? - Init() - Block:1( - PassStmt:1()))) - -[case testSuperExpr] -super().x -[out] -MypyFile:1( - ExpressionStmt:1( - SuperExpr:1( - x))) - -[case testKeywordAndDictArgs] -f(x = y, **kwargs) -[out] -MypyFile:1( - ExpressionStmt:1( - CallExpr:1( - NameExpr(f) - Args() - KwArgs( - x - NameExpr(y)) - DictVarArg( - NameExpr(kwargs))))) - -[case testSimpleFunctionType] -f = None # type: Function[[], None] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(f) - NameExpr(None) - Function?[, None?])) - -[case testFunctionTypeWithArgument] -f = None # type: Function[[str], int] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(f) - NameExpr(None) - Function?[, int?])) - -[case testFunctionTypeWithTwoArguments] -f = None # type: Function[[a[b], x.y], List[int]] -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(f) - NameExpr(None) - Function?[, List?[int?]])) - -[case testFunctionTypeWithExtraComma] -def f(x: Function[[str,], int]): pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - def (x: Function?[, int?]) -> Any - Block:1( - PassStmt:1()))) - -[case testSimpleStringLiteralType] -def f() -> 'A': pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> A? - Block:1( - PassStmt:1()))) - -[case testGenericStringLiteralType] -def f() -> 'A[B, C]': pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> A?[B?, C?] - Block:1( - PassStmt:1()))) - -[case testPartialStringLiteralType] -def f() -> A['B', C]: pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> A?[B?, C?] - Block:1( - PassStmt:1()))) - -[case testWhitespaceInStringLiteralType] -def f() -> ' A [ X ] ': pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> A?[X?] - Block:1( - PassStmt:1()))) - -[case testEscapeInStringLiteralType] -def f() -> '\x41': pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> A? - Block:1( - PassStmt:1()))) - -[case testMetaclass] -class Foo(metaclass=Bar): pass -[out] -MypyFile:1( - ClassDef:1( - Foo - Metaclass(Bar) - PassStmt:1())) - -[case testQualifiedMetaclass] -class Foo(metaclass=foo.Bar): pass -[out] -MypyFile:1( - ClassDef:1( - Foo - Metaclass(foo.Bar) - PassStmt:1())) - -[case testBaseAndMetaclass] -class Foo(foo.bar[x], metaclass=Bar): pass -[out] -MypyFile:1( - ClassDef:1( - Foo - Metaclass(Bar) - BaseType( - foo.bar?[x?]) - PassStmt:1())) - -[case testNamesThatAreNoLongerKeywords] -any = interface -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(any) - NameExpr(interface))) - -[case testFunctionOverload] -@foo -def f() -> x: pass -@foo -def f() -> y: pass -[out] -MypyFile:1( - OverloadedFuncDef:1( - Decorator:1( - Var(f) - NameExpr(foo) - FuncDef:2( - f - def () -> x? - Block:2( - PassStmt:2()))) - Decorator:3( - Var(f) - NameExpr(foo) - FuncDef:4( - f - def () -> y? - Block:4( - PassStmt:4()))))) - -[case testFunctionOverloadAndOtherStatements] -x -@foo -def f() -> x: pass -@foo -def f() -> y: pass -x -[out] -MypyFile:1( - ExpressionStmt:1( - NameExpr(x)) - OverloadedFuncDef:2( - Decorator:2( - Var(f) - NameExpr(foo) - FuncDef:3( - f - def () -> x? - Block:3( - PassStmt:3()))) - Decorator:4( - Var(f) - NameExpr(foo) - FuncDef:5( - f - def () -> y? - Block:5( - PassStmt:5())))) - ExpressionStmt:6( - NameExpr(x))) - -[case testFunctionOverloadWithThreeVariants] -@foo -def f() -> x: pass -@foo -def f() -> y: pass -@foo -def f(y): pass -[out] -MypyFile:1( - OverloadedFuncDef:1( - Decorator:1( - Var(f) - NameExpr(foo) - FuncDef:2( - f - def () -> x? - Block:2( - PassStmt:2()))) - Decorator:3( - Var(f) - NameExpr(foo) - FuncDef:4( - f - def () -> y? - Block:4( - PassStmt:4()))) - Decorator:5( - Var(f) - NameExpr(foo) - FuncDef:6( - f - Args( - Var(y)) - Block:6( - PassStmt:6()))))) - -[case testDecoratorsThatAreNotOverloads] -@foo -def f() -> x: pass -@foo -def g() -> y: pass -[out] -MypyFile:1( - Decorator:1( - Var(f) - NameExpr(foo) - FuncDef:2( - f - def () -> x? - Block:2( - PassStmt:2()))) - Decorator:3( - Var(g) - NameExpr(foo) - FuncDef:4( - g - def () -> y? - Block:4( - PassStmt:4())))) - -[case testFunctionOverloadWithinFunction] -def f(): - @foo - def g(): pass - @foo - def g() -> x: pass -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - OverloadedFuncDef:2( - Decorator:2( - Var(g) - NameExpr(foo) - FuncDef:3( - g - Block:3( - PassStmt:3()))) - Decorator:4( - Var(g) - NameExpr(foo) - FuncDef:5( - g - def () -> x? - Block:5( - PassStmt:5()))))))) - -[case testCommentFunctionAnnotation] -def f(): # type: () -> A - pass -def g(x): # type: (A) -> B - pass -[out] -MypyFile:1( - FuncDef:1( - f - def () -> A? - Block:1( - PassStmt:2())) - FuncDef:3( - g - Args( - Var(x)) - def (x: A?) -> B? - Block:3( - PassStmt:4()))) - -[case testCommentMethodAnnotation] -class A: - def f(self): # type: () -> A - pass - def g(xself, x): # type: (A) -> B - pass -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - f - Args( - Var(self)) - def (self: Any) -> A? - Block:2( - PassStmt:3())) - FuncDef:4( - g - Args( - Var(xself) - Var(x)) - def (xself: Any, x: A?) -> B? - Block:4( - PassStmt:5())))) - -[case testCommentMethodAnnotationAndNestedFunction] -class A: - def f(self): # type: () -> A - def g(x): # type: (A) -> B - pass -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - f - Args( - Var(self)) - def (self: Any) -> A? - Block:2( - FuncDef:3( - g - Args( - Var(x)) - def (x: A?) -> B? - Block:3( - PassStmt:4())))))) - -[case testCommentFunctionAnnotationOnSeparateLine] -def f(x): - # type: (X) -> Y - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - def (x: X?) -> Y? - Block:1( - PassStmt:3()))) - -[case testCommentFunctionAnnotationOnSeparateLine2] -def f(x): - - # type: (X) -> Y # bar - - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - def (x: X?) -> Y? - Block:1( - PassStmt:5()))) - -[case testCommentFunctionAnnotationAndVarArg] -def f(x, *y): # type: (X, *Y) -> Z - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - def (x: X?, *y: Y?) -> Z? - VarArg( - Var(y)) - Block:1( - PassStmt:2()))) - -[case testCommentFunctionAnnotationAndAllVarArgs] -def f(x, *y, **z): # type: (X, *Y, **Z) -> A - pass -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - def (x: X?, *y: Y?, **z: Z?) -> A? - VarArg( - Var(y)) - DictVarArg( - Var(z)) - Block:1( - PassStmt:2()))) - -[case testClassDecorator] -@foo -class X: pass -@foo(bar) -@x.y -class Z: pass -[out] -MypyFile:1( - ClassDef:1( - X - Decorators( - NameExpr(foo)) - PassStmt:2()) - ClassDef:3( - Z - Decorators( - CallExpr:3( - NameExpr(foo) - Args( - NameExpr(bar))) - MemberExpr:4( - NameExpr(x) - y)) - PassStmt:5())) - -[case testTrailingSemicolon] -def x(): - pass; - -def y(): - pass -[out] -MypyFile:1( - FuncDef:1( - x - Block:1( - PassStmt:2())) - FuncDef:4( - y - Block:4( - PassStmt:5()))) - -[case testEmptySuperClass] -class A(): - pass -[out] -MypyFile:1( - ClassDef:1( - A - PassStmt:2())) - -[case testStarExpression] -*a -*a, b -a, *b -a, (*x, y) -a, (x, *y) -[out] -MypyFile:1( - ExpressionStmt:1( - StarExpr:1( - NameExpr(a))) - ExpressionStmt:2( - TupleExpr:2( - StarExpr:2( - NameExpr(a)) - NameExpr(b))) - ExpressionStmt:3( - TupleExpr:3( - NameExpr(a) - StarExpr:3( - NameExpr(b)))) - ExpressionStmt:4( - TupleExpr:4( - NameExpr(a) - ParenExpr:4( - TupleExpr:4( - StarExpr:4( - NameExpr(x)) - NameExpr(y))))) - ExpressionStmt:5( - TupleExpr:5( - NameExpr(a) - ParenExpr:5( - TupleExpr:5( - NameExpr(x) - StarExpr:5( - NameExpr(y))))))) - -[case testStarExpressionParenthesis] -*(a) -*(a,b) -[out] -MypyFile:1( - ExpressionStmt:1( - StarExpr:1( - ParenExpr:1( - NameExpr(a)))) - ExpressionStmt:2( - StarExpr:2( - ParenExpr:2( - TupleExpr:2( - NameExpr(a) - NameExpr(b)))))) - -[case testStarExpressionInFor] -for *a in b: - pass - -for a, *b in c: - pass - -for *a, b in c: - pass -[out] -MypyFile:1( - ForStmt:1( - StarExpr:1( - NameExpr(a)) - NameExpr(b) - Block:1( - PassStmt:2())) - ForStmt:4( - TupleExpr:4( - NameExpr(a) - StarExpr:4( - NameExpr(b))) - NameExpr(c) - Block:4( - PassStmt:5())) - ForStmt:7( - TupleExpr:7( - StarExpr:7( - NameExpr(a)) - NameExpr(b)) - NameExpr(c) - Block:7( - PassStmt:8()))) - -[case testStarExprInGeneratorExpr] -x for y, *p in z -x for *p, y in z -x for y, *p, q in z -[out] -MypyFile:1( - ExpressionStmt:1( - GeneratorExpr:1( - NameExpr(x) - TupleExpr:1( - NameExpr(y) - StarExpr:1( - NameExpr(p))) - NameExpr(z))) - ExpressionStmt:2( - GeneratorExpr:2( - NameExpr(x) - TupleExpr:2( - StarExpr:2( - NameExpr(p)) - NameExpr(y)) - NameExpr(z))) - ExpressionStmt:3( - GeneratorExpr:3( - NameExpr(x) - TupleExpr:3( - NameExpr(y) - StarExpr:3( - NameExpr(p)) - NameExpr(q)) - NameExpr(z)))) diff --git a/mypy/test/data/python2eval.test b/mypy/test/data/python2eval.test deleted file mode 100644 index 326d84faeeab..000000000000 --- a/mypy/test/data/python2eval.test +++ /dev/null @@ -1,309 +0,0 @@ --- Test cases for type checking mypy programs using full stubs and running --- using CPython. -- Python 2 syntax. --- --- These are mostly regression tests -- no attempt is made to make these --- complete. - - -[case testAbs2_python2] -from typing import Undefined -n = Undefined(int) -f = Undefined(float) -n = abs(1) -abs(1) + 'x' # Error -f = abs(1.1) -abs(1.1) + 'x' # Error -[out] -_program.py, line 5: Unsupported operand types for + ("int" and "str") -_program.py, line 7: Unsupported operand types for + ("float" and "str") - -[case testUnicode_python2] -import typing -x = unicode('xyz', 'latin1') -print x -x = u'foo' -print repr(x) -[out] -xyz -u'foo' - -[case testXrangeAndRange_python2] -import typing -for i in xrange(2): - print i -for i in range(3): - print i -[out] -0 -1 -0 -1 -2 - -[case testIterator_python2] -import typing, sys -x = iter('bar') -print x.next(), x.next() -[out] -b a - -[case testEncodeAndDecode_python2] -import typing -print 'a'.encode('latin1') -print 'b'.decode('latin1') -print u'c'.encode('latin1') -print u'd'.decode('latin1') -[out] -a -b -c -d - -[case testHasKey_python2] -import typing -d = {1: 'x'} -print d.has_key(1) -print d.has_key(2) -[out] -True -False - -[case testIntegerDivision_python2] -import typing -x = 1 / 2 -x = 2 -[out] - -[case testFloatDivision_python2] -import typing -x = 1.0 / 2.0 -x = 1.0 / 2 -x = 1 / 2.0 -x = 1.5 -[out] - -[case testAnyStr_python2] -from typing import AnyStr -def f(x): # type: (AnyStr) -> AnyStr - if isinstance(x, str): - return 'foo' - else: - return u'zar' -print f('') -print f(u'') -[out] -foo -zar - -[case testGenericPatterns_python2] -from typing import Pattern, Undefined -import re -p = Undefined(Pattern[unicode]) -p = re.compile(u'foo*') -b = Undefined(Pattern[str]) -b = re.compile('foo*') -print(p.match(u'fooo').group(0)) -[out] -fooo - -[case testGenericMatch_python2] -from typing import Match -import re -def f(m): # type: (Match[str]) -> None - print(m.group(0)) -f(re.match('x*', 'xxy')) -[out] -xx - -[case testFromFuturePrintFunction_python2] -from __future__ import print_function -print('a', 'b') -[out] -a b - -[case testFromFutureImportUnicodeLiterals_python2] -from __future__ import unicode_literals -print '>', ['a', b'b', u'c'] -[out] -> [u'a', 'b', u'c'] - -[case testFromFutureImportUnicodeLiterals2_python2] -from __future__ import unicode_literals -def f(x: str) -> None: pass -f(b'') -f(u'') -f('') -[out] -_program.py, line 4: Argument 1 to "f" has incompatible type "unicode"; expected "str" -_program.py, line 5: Argument 1 to "f" has incompatible type "unicode"; expected "str" - -[case testStrUnicodeCompatibility_python2] -import typing -def f(s): # type: (unicode) -> None - pass -f(u'') -f('') -[out] - -[case testStrUnicodeCompatibilityInBuiltins_python2] -import typing -'x'.count('x') -'x'.count(u'x') -[out] - -[case testTupleAsSubtypeOfSequence_python2] -from typing import typevar, Sequence -T = typevar('T') -def f(a): # type: (Sequence[T]) -> None - print a -f(tuple()) -[out] -() - -[case testReadOnlyProperty_python2] -import typing -class A: - @property - def foo(self): # type: () -> int - return 1 -print(A().foo + 2) -[out] -3 - -[case testIOTypes_python2] -from typing import IO, TextIO, BinaryIO, Any -class X(IO[str]): pass -class Y(TextIO): pass -class Z(BinaryIO): pass -[out] - -[case testOpenReturnType_python2] -import typing -f = open('/tmp/xyz', 'w') -f.write(u'foo') -f.write('bar') -f.close() -[out] -_program.py, line 3: No overload variant of "write" of "BinaryIO" matches argument types - -[case testPrintFunctionWithFileArg_python2] -from __future__ import print_function -import typing -if 1 == 2: # Don't want to run the code below, since it would create a file. - f = open('/tmp/xyz', 'w') - print('foo', file=f) - f.close() -print('ok') -[out] -ok - -[case testStringIO_python2] -import typing -import io -c = io.StringIO() -c.write(u'\x89') -print(repr(c.getvalue())) -[out] -u'\x89' - -[case testBytesIO_python2] -import typing -import io -c = io.BytesIO() -c.write('\x89') -print(repr(c.getvalue())) -[out] -'\x89' - -[case testTextIOWrapper_python2] -import typing -import io -b = io.BytesIO(u'\xab'.encode('utf8')) -w = io.TextIOWrapper(b, encoding='utf8') -print(repr(w.read())) -[out] -u'\xab' - -[case testIoOpen_python2] -import typing -import io -if 1 == 2: # Only type check, do not execute - f = io.open('/tmp/xyz', 'w', encoding='utf8') - f.write(u'\xab') - f.close() -print 'ok' -[out] -ok - -[case testUnionType_python2] -from typing import Undefined, Union -y = Undefined(Union[int, str]) -def f(x): # type: (Union[int, str]) -> str - if isinstance(x, int): - x = str(x) - return x -print f(12) -print f('ab') -[out] -12 -ab - -[case testStrAdd_python2] -import typing -s = '' -u = u'' -n = 0 -n = s + '' # E -s = s + u'' # E -[out] -_program.py, line 5: Incompatible types in assignment (expression has type "str", variable has type "int") -_program.py, line 6: Incompatible types in assignment (expression has type "unicode", variable has type "str") - -[case testStrJoin_python2] -import typing -s = '' -u = u'' -n = 0 -n = ''.join(['']) # Error -s = ''.join([u'']) # Error -[out] -_program.py, line 5: Incompatible types in assignment (expression has type "str", variable has type "int") -_program.py, line 6: Incompatible types in assignment (expression has type "unicode", variable has type "str") - -[case testNamedTuple_python2] -import typing -from collections import namedtuple -X = namedtuple('X', ['a', 'b']) -x = X(a=1, b='s') -print x.a, x.b -[out] -1 s - -[case testNamedTupleError_python2] -import typing -from collections import namedtuple -X = namedtuple('X', ['a', 'b']) -x = X(a=1, b='s') -x.c -[out] -_program.py, line 5: "X" has no attribute "c" - -[case testAssignToComplexReal_python2] -import typing -x = 4j -y = x.real -y = x # Error -x.imag = 2.0 # Error -[out] -_program.py, line 4: Incompatible types in assignment (expression has type "complex", variable has type "float") -_program.py, line 5: Property "imag" defined in "complex" is read-only - -[case testComplexArithmetic] -import typing -print(5 + 8j) -print(3j * 2.0) -print(4j / 2.0) -[out] -(5+8j) -6j -2j diff --git a/mypy/test/data/pythoneval-asyncio.test b/mypy/test/data/pythoneval-asyncio.test deleted file mode 100644 index ac435f68f7d5..000000000000 --- a/mypy/test/data/pythoneval-asyncio.test +++ /dev/null @@ -1,479 +0,0 @@ --- Test cases for type checking mypy programs using full stubs and running --- using CPython. --- --- These are mostly regression tests -- no attempt is made to make these --- complete. --- --- This test file check Asyncio and yield from interaction - -[case testImportAsyncio] -import asyncio -print('Imported') -[out] -Imported - -[case testSimpleCoroutineSleep] -from typing import Any -import asyncio -from asyncio import Future - -@asyncio.coroutine -def greet_every_two_seconds() -> 'Future[None]': - n = 0 - while n < 5: - print('Prev', n) - yield from asyncio.sleep(0.1) - print('After', n) - n += 1 - -loop = asyncio.get_event_loop() -try: - loop.run_until_complete(greet_every_two_seconds()) -finally: - loop.close() -[out] -Prev 0 -After 0 -Prev 1 -After 1 -Prev 2 -After 2 -Prev 3 -After 3 -Prev 4 -After 4 - -[case testCoroutineCallingOtherCoroutine] -import asyncio -from asyncio import Future - -@asyncio.coroutine -def compute(x: int, y: int) -> 'Future[int]': - print("Compute %s + %s ..." % (x, y)) - yield from asyncio.sleep(0.1) - return x + y # Here the int is wrapped in Future[int] - -@asyncio.coroutine -def print_sum(x: int, y: int) -> 'Future[None]': - result = yield from compute(x, y) # The type of result will be int (is extracted from Future[int] - print("%s + %s = %s" % (x, y, result)) - -loop = asyncio.get_event_loop() -loop.run_until_complete(print_sum(1, 2)) -loop.close() -[out] -Compute 1 + 2 ... -1 + 2 = 3 - -[case testCoroutineChangingFuture] -import asyncio -from asyncio import Future - -@asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': - yield from asyncio.sleep(0.1) - future.set_result('Future is done!') - -loop = asyncio.get_event_loop() -future = asyncio.Future() # type: Future[str] -asyncio.Task(slow_operation(future)) -loop.run_until_complete(future) -print(future.result()) -loop.close() -[out] -Future is done! - -[case testFunctionAssignedAsCallback] -import typing -import asyncio -from asyncio import Future, AbstractEventLoop - -@asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': - yield from asyncio.sleep(1) - future.set_result('Callback works!') - -def got_result(future: 'Future[str]') -> None: - print(future.result()) - loop.stop() - -loop = asyncio.get_event_loop() # type: AbstractEventLoop -future = asyncio.Future() # type: Future[str] -asyncio.Task(slow_operation(future)) # Here create a task with the function. (The Task need a Future[T] as first argument) -future.add_done_callback(got_result) # and assignt the callback to the future -try: - loop.run_forever() -finally: - loop.close() -[out] -Callback works! - -[case testMultipleTasks] -import typing -import asyncio -from asyncio import Task, Future -@asyncio.coroutine -def factorial(name, number) -> 'Future[None]': - f = 1 - for i in range(2, number+1): - print("Task %s: Compute factorial(%s)..." % (name, i)) - yield from asyncio.sleep(0.1) - f *= i - print("Task %s: factorial(%s) = %s" % (name, number, f)) - -loop = asyncio.get_event_loop() -tasks = [ - asyncio.Task(factorial("A", 2)), - asyncio.Task(factorial("B", 3)), - asyncio.Task(factorial("C", 4))] -loop.run_until_complete(asyncio.wait(tasks)) -loop.close() -[out] -Task A: Compute factorial(2)... -Task B: Compute factorial(2)... -Task C: Compute factorial(2)... -Task A: factorial(2) = 2 -Task B: Compute factorial(3)... -Task C: Compute factorial(3)... -Task B: factorial(3) = 6 -Task C: Compute factorial(4)... -Task C: factorial(4) = 24 - - -[case testConcatenatedCoroutines] -import typing -import asyncio -from asyncio import Future - -@asyncio.coroutine -def h4() -> 'Future[int]': - x = yield from future - return x - -@asyncio.coroutine -def h3() -> 'Future[int]': - x = yield from h4() - print("h3: %s" % x) - return x - -@asyncio.coroutine -def h2() -> 'Future[int]': - x = yield from h3() - print("h2: %s" % x) - return x - -@asyncio.coroutine -def h() -> 'Future[None]': - x = yield from h2() - print("h: %s" % x) - -loop = asyncio.get_event_loop() -future = asyncio.Future() # type: Future[int] -future.set_result(42) -loop.run_until_complete(h()) -print("Outside %s" % future.result()) -loop.close() -[out] -h3: 42 -h2: 42 -h: 42 -Outside 42 - -[case testConcantenatedCoroutinesReturningFutures] -import typing -import asyncio -from asyncio import Future - -@asyncio.coroutine -def h4() -> 'Future[Future[int]]': - yield from asyncio.sleep(0.1) - f = asyncio.Future() #type: Future[int] - return f - -@asyncio.coroutine -def h3() -> 'Future[Future[Future[int]]]': - x = yield from h4() - x.set_result(42) - f = asyncio.Future() #type: Future[Future[int]] - f.set_result(x) - return f - -@asyncio.coroutine -def h() -> 'Future[None]': - print("Before") - x = yield from h3() - y = yield from x - z = yield from y - print(z) - def normalize(future): - # The str conversion seems inconsistent; not sure exactly why. Normalize - # the result. - return str(future).replace(' -Future> - - -[case testCoroutineWithOwnClass] -import typing -import asyncio -from asyncio import Future - -class A: - def __init__(self, x: int) -> None: - self.x = x - -@asyncio.coroutine -def h() -> 'Future[None]': - x = yield from future - print("h: %s" % x.x) - -loop = asyncio.get_event_loop() -future = asyncio.Future() # type: Future[A] -future.set_result(A(42)) -loop.run_until_complete(h()) -print("Outside %s" % future.result().x) -loop.close() -[out] -h: 42 -Outside 42 - - --- Errors - -[case testErrorAssigningCoroutineThatDontReturn] -from typing import Any -import asyncio -from asyncio import Future - -@asyncio.coroutine -def greet() -> 'Future[None]': - yield from asyncio.sleep(0.2) - print('Hello World') - -@asyncio.coroutine -def test() -> 'Future[None]': - yield from greet() - x = yield from greet() # Error - -loop = asyncio.get_event_loop() -try: - loop.run_until_complete(test()) -finally: - loop.close() -[out] -_program.py: In function "test": -_program.py, line 13: Function does not return a value - -[case testErrorReturnIsNotTheSameType] -import asyncio -from asyncio import Future - -@asyncio.coroutine -def compute(x: int, y: int) -> 'Future[int]': - print("Compute %s + %s ..." % (x, y)) - yield from asyncio.sleep(0.1) - return str(x + y) # Error - -@asyncio.coroutine -def print_sum(x: int, y: int) -> 'Future[None]': - result = yield from compute(x, y) - print("%s + %s = %s" % (x, y, result)) - -loop = asyncio.get_event_loop() -loop.run_until_complete(print_sum(1, 2)) -loop.close() - -[out] -_program.py: In function "compute": -_program.py, line 8: Incompatible return value type: expected asyncio.futures.Future[builtins.int], got asyncio.futures.Future[builtins.str] - -[case testErrorSetFutureDifferentInternalType] -import asyncio -from asyncio import Future - -@asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': - yield from asyncio.sleep(1) - future.set_result(42) # Error - -loop = asyncio.get_event_loop() -future = asyncio.Future() # type: Future[str] -asyncio.Task(slow_operation(future)) -loop.run_until_complete(future) -print(future.result()) -loop.close() -[out] -_program.py: In function "slow_operation": -_program.py, line 7: Argument 1 to "set_result" of "Future" has incompatible type "int"; expected "str" - - -[case testErrorUsingDifferentFutureType] -import asyncio -from asyncio import Future - -@asyncio.coroutine -def slow_operation(future: 'Future[int]') -> 'Future[None]': - yield from asyncio.sleep(1) - future.set_result(42) - -loop = asyncio.get_event_loop() -future = asyncio.Future() # type: Future[str] -asyncio.Task(slow_operation(future)) # Error -loop.run_until_complete(future) -print(future.result()) -loop.close() -[out] -_program.py, line 11: Argument 1 to "slow_operation" has incompatible type Future[str]; expected Future[int] - -[case testErrorUsingDifferentFutureTypeAndSetFutureDifferentInternalType] -import asyncio -from asyncio import Future - -asyncio.coroutine -def slow_operation(future: 'Future[int]') -> 'Future[None]': - yield from asyncio.sleep(1) - future.set_result('42') #Try to set an str as result to a Future[int] - -loop = asyncio.get_event_loop() -future = asyncio.Future() # type: Future[str] -asyncio.Task(slow_operation(future)) # Error -loop.run_until_complete(future) -print(future.result()) -loop.close() -[out] -_program.py: In function "slow_operation": -_program.py, line 7: Argument 1 to "set_result" of "Future" has incompatible type "str"; expected "int" -_program.py: At top level: -_program.py, line 11: Argument 1 to "slow_operation" has incompatible type Future[str]; expected Future[int] - -[case testErrorSettingCallbackWithDifferentFutureType] -import typing -import asyncio -from asyncio import Future, AbstractEventLoop - -@asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': - yield from asyncio.sleep(1) - future.set_result('Future is done!') - -def got_result(future: 'Future[int]') -> None: - print(future.result()) - loop.stop() - -loop = asyncio.get_event_loop() # type: AbstractEventLoop -future = asyncio.Future() # type: Future[str] -asyncio.Task(slow_operation(future)) -future.add_done_callback(got_result) # Error - -try: - loop.run_forever() -finally: - loop.close() -[out] -_program.py, line 17: Argument 1 to "add_done_callback" of "Future" has incompatible type Function[[Future[int]], None]; expected Function[[Future[str]], Any] - -[case testErrorOneMoreFutureInReturnType] -import typing -import asyncio -from asyncio import Future - -@asyncio.coroutine -def h4() -> 'Future[Future[int]]': - yield from asyncio.sleep(1) - f = asyncio.Future() #type: Future[int] - return f - -@asyncio.coroutine -def h3() -> 'Future[Future[Future[Future[int]]]]': - x = yield from h4() - x.set_result(42) - f = asyncio.Future() #type: Future[Future[int]] - f.set_result(x) - return f - -@asyncio.coroutine -def h() -> 'Future[None]': - print("Before") - x = yield from h3() - y = yield from x - z = yield from y - print(z) - print(y) - print(x) - -loop = asyncio.get_event_loop() -loop.run_until_complete(h()) -loop.close() -[out] -_program.py: In function "h3": -_program.py, line 17: Incompatible return value type: expected asyncio.futures.Future[asyncio.futures.Future[asyncio.futures.Future[asyncio.futures.Future[builtins.int]]]], got asyncio.futures.Future[asyncio.futures.Future[builtins.int]] - -[case testErrorOneLessFutureInReturnType] -import typing -import asyncio -from asyncio import Future - -@asyncio.coroutine -def h4() -> 'Future[Future[int]]': - yield from asyncio.sleep(1) - f = asyncio.Future() #type: Future[int] - return f - -@asyncio.coroutine -def h3() -> 'Future[Future[int]]': - x = yield from h4() - x.set_result(42) - f = asyncio.Future() #type: Future[Future[int]] - f.set_result(x) - return f - -@asyncio.coroutine -def h() -> 'Future[None]': - print("Before") - x = yield from h3() - y = yield from x - print(y) - print(x) - -loop = asyncio.get_event_loop() -loop.run_until_complete(h()) -loop.close() -[out] -_program.py: In function "h3": -_program.py, line 17: Incompatible return value type: expected asyncio.futures.Future[asyncio.futures.Future[builtins.int]], got asyncio.futures.Future[asyncio.futures.Future[builtins.int]] - -[case testErrorAssignmentDifferentType] -import typing -import asyncio -from asyncio import Future - -class A: - def __init__(self, x: int) -> None: - self.x = x - -class B: - def __init__(self, x: int) -> None: - self.x = x - -@asyncio.coroutine -def h() -> 'Future[None]': - x = yield from future # type: B # Error - print("h: %s" % x.x) - -loop = asyncio.get_event_loop() -future = asyncio.Future() # type: Future[A] -future.set_result(A(42)) -loop.run_until_complete(h()) -loop.close() -[out] -_program.py: In function "h": -_program.py, line 15: Incompatible types in assignment (expression has type "A", variable has type "B") diff --git a/mypy/test/data/pythoneval.test b/mypy/test/data/pythoneval.test deleted file mode 100644 index c8c25c7bd6ec..000000000000 --- a/mypy/test/data/pythoneval.test +++ /dev/null @@ -1,1040 +0,0 @@ --- Test cases for type checking mypy programs using full stubs and running --- using CPython. --- --- These are mostly regression tests -- no attempt is made to make these --- complete. - - -[case testHello] -import typing -print('hello, world') -[out] -hello, world - -[case testAbstractBaseClasses] -import re -from typing import Sized, Sequence, Iterator, Iterable, Mapping, AbstractSet - -def check(o, t): - rep = re.sub('0x[0-9a-f]+', '0x...', repr(o)) - rep = rep.replace('sequenceiterator', 'str_iterator') - print(rep, t, isinstance(o, t)) - -def f(): - check('x', Sized) - check([1], Sequence) - check({1:3}, Sequence) - check(iter('x'), Iterator) - check('x', Iterable) - check({}, Mapping) - check(set([1]), AbstractSet) - -f() -[out] -'x' True -[1] True -{1: 3} False - True -'x' True -{} True -{1} True - -[case testSized] -from typing import Sized -class A(Sized): - def __len__(self): return 5 -print(len(A())) -[out] -5 - -[case testAbstractClassInOverload] -from typing import overload, Sized -class A(Sized): - def __len__(self): return 5 -class B: - def __len__(self): return 6 -@overload -def f(s: Sized) -> None: print('sized') -@overload -def f(s) -> None: print('other') -f('x') -f(A()) -f(1) -f(B()) # Sized is actually a Protocol -[out] -sized -sized -other -sized - -[case testOverloadUsingMatchAndPattern-skip] -import re -from typing import overload, Any, Match, Pattern -@overload -def f(x: Match) -> None: print('match') -@overload -def f(x: Any) -> None: print('something else') - -f(1) -f(re.match('x', 'x')) - -@overload -def g(x: Pattern) -> None: print('pattern') -@overload -def g(x: Any) -> None: print('something else') - -g(1) -g(re.compile('')) -[out] -something else -match -something else -pattern - -[case testReversed] -from typing import Reversible -class A(Reversible): - def __reversed__(self): return iter('foo') -print(list(reversed(range(5)))) -print(list(reversed([1,2,3]))) -print(list(reversed('abc'))) -print(list(reversed(A()))) -[out] --- Duplicate [ at line beginning. -[[4, 3, 2, 1, 0] -[[3, 2, 1] -[['c', 'b', 'a'] -[['f', 'o', 'o'] - -[case testIntAndFloatConversion] -from typing import SupportsInt, SupportsFloat -class A(SupportsInt): - def __int__(self): return 5 -class B(SupportsFloat): - def __float__(self): return 1.2 -print(int(1)) -print(int(6.2)) -print(int('3')) -print(int(b'4')) -print(int(A())) -print(float(-9)) -print(float(B())) -[out] -1 -6 -3 -4 -5 --9.0 -1.2 - -[case testAbs] -from typing import SupportsAbs -class A(SupportsAbs[float]): - def __abs__(self) -> float: return 5.5 - -print(abs(-1)) -print(abs(-1.2)) -print(abs(A())) -[out] -1 -1.2 -5.5 - -[case testAbs2] -from typing import Undefined -n = Undefined(int) -f = Undefined(float) -n = abs(1) -abs(1) + 'x' # Error -f = abs(1.1) -abs(1.1) + 'x' # Error -[out] -_program.py, line 5: Unsupported operand types for + ("int" and "str") -_program.py, line 7: Unsupported operand types for + ("float" and "str") - -[case testRound] -from typing import SupportsRound -class A(SupportsRound): - def __round__(self, ndigits=0): return 'x%d' % ndigits -print(round(1.6)) -print(round(A())) -print(round(A(), 2)) -[out] -2 -x0 -x2 - -[case testOverloadUsingSupportsInt] -from typing import overload, SupportsInt -@overload -def f(i: SupportsInt) -> None: print('int-like') -@overload -def f(i) -> None: print('other') -f(1.2) -f(3) -f('x') -f([2]) -[out] -int-like -int-like -other -other - -[case testOverloadWithOptionalArg] -from typing import overload -@overload -def f(x: int, y: str = 'x') -> None: print(x, y) -@overload -def f(x: str) -> None: print(x) -f(1) -f('x') -f(1, 'y') -[out] -1 x -x -1 y - -[case testOverloadWithOptionalArg2-skip] -from typing import overload -@overload -def f(x: int, y: str = 'x') -> None: print(x, y) -@overload -def f(x: str) -> None: print(x) -f(x=1) -f(x='x') -f(x=1, y='y') -[out] -1 x -x -1 y - -[case testCallMethodViaTypeObject] -import typing -print(list.__add__([1, 2], [3, 4])) -[out] -[[1, 2, 3, 4] - -[case testClassDataAttribute] -import typing -class A: - x = 0 -print(A.x) -A.x += 1 -print(A.x) -[out] -0 -1 - -[case testInheritedClassAttribute] -import typing -class A: - x = 1 - def f(self) -> None: print('f') -class B(A): - pass -B.f(None) -print(B.x) -[out] -f -1 - -[case testFunctionDecorator] -from typing import typevar, cast -ftype = typevar('ftype') -def logged(f: ftype) -> ftype: - def g(*args, **kwargs): - print('enter', f.__name__) - r = f(*args, **kwargs) - print('exit', f.__name__) - return r - return cast(ftype, g) - -@logged -def foo(s: str) -> str: - print('foo', s) - return s + '!' - -print(foo('y')) -print(foo('x')) -[out] -enter foo -foo y -exit foo -y! -enter foo -foo x -exit foo -x! - -[case testModuleAttributes] -import math -import typing -print(math.__name__) -print(type(math.__dict__)) -print(type(typing.__doc__)) -print(math.__class__) -[out] -math - - - - -[case testSpecialAttributes] -import typing -class A: pass -print(object().__doc__) -print(A().__class__) -[out] -The most base type - - -[case testFunctionAttributes] -import typing -ord.__class__ -print(type(ord.__doc__ + '')) -print(ord.__name__) -print(ord.__module__) -[out] - -ord -builtins - -[case testTypeAttributes] -import typing -print(str.__class__) -print(type(str.__doc__)) -print(str.__name__) -print(str.__module__) -print(str.__dict__ is not None) -[out] - - -str -builtins -True - -[case testBoolCompatibilityWithInt] -import typing -x = 0 -x = True -print(bool('x')) -print(bool('')) -[out] -True -False - -[case testCallBuiltinTypeObjectsWithoutArguments] -import typing -print(int()) -print(repr(str())) -print(repr(bytes())) -print(float()) -print(bool()) -[out] -0 -'' -b'' -0.0 -False - -[case testIntegerDivision] -import typing -x = 1 / 2 -x = 1.5 -[out] - -[case testStaticmethod] -import typing -class A: - @staticmethod - def f(x: str) -> int: return int(x) -print(A.f('12')) -print(A().f('34')) -[out] -12 -34 - -[case testClassmethod] -import typing -class A: - @classmethod - def f(cls, x: str) -> int: return int(x) -print(A.f('12')) -print(A().f('34')) -[out] -12 -34 - -[case testIntMethods] -import typing -print(int.from_bytes(b'ab', 'big')) -n = 0 -print(n.from_bytes(b'ac', 'big')) -print(n.from_bytes([2, 3], 'big')) -print(n.to_bytes(2, 'big')) -[out] -24930 -24931 -515 -b'\x00\x00' - -[case testFloatMethods] -import typing -print(1.5.as_integer_ratio()) -print(1.5.hex()) -print(2.0.is_integer()) -print(float.fromhex('0x1.8')) -[out] -(3, 2) -0x1.8000000000000p+0 -True -1.5 - -[case testArray] -import typing -import array -array.array('b', [1, 2]) -[out] - -[case testDictFromkeys] -import typing -d = dict.fromkeys('foo') -d['x'] = 2 -d2 = dict.fromkeys([1, 2], b'') -d2[2] = b'foo' -[out] - -[case testReadOnlyProperty] -class A: - x = 2 - @property - def f(self) -> int: - return self.x + 1 -print(A().f) -[out] -3 - -[case testIsinstanceWithTuple] -from typing import Any -x = Any((1, 'x')) -if isinstance(x, tuple): - print(x[0], x[1]) -[out] -1 x - -[case testTypevarValues] -from typing import typevar -T = typevar('T', values=(str, bytes)) -def f(x: T) -> T: - if isinstance(x, str): - return 'foo' - else: - return b'bar' -print(f('')) -print(f(b'')) -[out] -foo -b'bar' - -[case testAnyStr] -from typing import AnyStr -def f(x: AnyStr) -> AnyStr: - if isinstance(x, str): - return 'foo' - else: - return b'zar' -print(f('')) -print(f(b'')) -[out] -foo -b'zar' - -[case testNameNotImportedFromTyping] -import typing -Any(1) -[out] -_program.py, line 2: Name 'Any' is not defined - -[case testBinaryIOType] -from typing import BinaryIO -def f(f: BinaryIO) -> None: - f.write(b'foo') - f.write(bytearray(b'foo')) -[out] - -[case testIOTypes] -from typing import IO -import sys -def txt(f: IO[str]) -> None: - f.write('foo') - f.write(b'foo') -def bin(f: IO[bytes]) -> None: - f.write(b'foo') - f.write(bytearray(b'foo')) -txt(sys.stdout) -bin(sys.stdout) -[out] -_program.py: In function "txt": -_program.py, line 5: Argument 1 to "write" of "IO" has incompatible type "bytes"; expected "str" -_program.py: In function "bin": -_program.py, line 8: Argument 1 to "write" of "IO" has incompatible type "bytearray"; expected "bytes" -_program.py: At top level: -_program.py, line 10: Argument 1 to "bin" has incompatible type "TextIO"; expected IO[bytes] - -[case testBuiltinOpen] -f = open('x') -f.write('x') -f.write(b'x') -f.foobar() -[out] -_program.py, line 4: IO[Any] has no attribute "foobar" - -[case testGenericPatterns] -from typing import Pattern, Undefined -import re -p = Undefined(Pattern[str]) -p = re.compile('foo*') -b = Undefined(Pattern[bytes]) -b = re.compile(b'foo*') -print(p.match('fooo').group(0)) -[out] -fooo - -[case testGenericMatch] -from typing import Match -import re -def f(m: Match[bytes]) -> None: - print(m.group(0)) -f(re.match(b'x*', b'xxy')) -[out] -b'xx' - -[case testMultipleTypevarsWithValues] -from typing import typevar - -T = typevar('T', values=(int, str)) -S = typevar('S', values=(int, str)) - -def f(t: T, s: S) -> None: - t + s -[out] -_program.py: In function "f": -_program.py, line 7: Unsupported operand types for + ("int" and "str") -_program.py, line 7: Unsupported operand types for + ("str" and "int") - -[case testBuiltinclass] -from typing import builtinclass -@builtinclass -class A: - def __init__(self, x): - self.x = x + 1 -print(A(2).x) -[out] -3 - -[case testSystemExitCode] -import typing -print(SystemExit(5).code) -[out] -5 - -[case testIntFloatDucktyping] -from typing import Undefined -x = Undefined(float) -x = 2.2 -x = 2 -def f(x: float) -> None: pass -f(1.1) -f(1) -[out] - -[case testDucktypeDeclaration] -from typing import Undefined, ducktype -@ducktype(str) -class S: pass -def f() -> None: - 'foo' + S() -print(type(S())) -[out] - - -[case testsFloatOperations] -import typing -print(1.5 + 1.5) -print(1.5 + 1) -[out] -3.0 -2.5 - -[case testMathFunctionWithIntArgument] -import typing -import math -math.sin(2) -math.sin(2.2) - -[case testAbsReturnType] -from typing import Undefined -f = Undefined(float) -n = Undefined(int) -n = abs(2) -f = abs(2.2) -abs(2.2) + 'x' -[out] -_program.py, line 6: Unsupported operand types for + ("float" and "str") - -[case testROperatorMethods] -from typing import Undefined -b = Undefined(bytes) -s = Undefined(str) -s = b'foo' * 5 # Error -b = 5 * b'foo' -b = b'foo' * 5 -s = 5 * 'foo' -s = 'foo' * 5 -[out] -_program.py, line 4: Incompatible types in assignment (expression has type "bytes", variable has type "str") - -[case testROperatorMethods2] -import typing -print(2 / 0.5) -print(' ', 2 * [3, 4]) -[out] -4.0 - [3, 4, 3, 4] - -[case testNotImplemented] -from typing import disjointclass -class A: - def __add__(self, x: int) -> int: - if isinstance(x, int): - return x + 1 - return NotImplemented -@disjointclass(int) -class B: - def __radd__(self, x: A) -> str: - return 'x' -print(A() + 1) -print(A() + B()) -[out] -2 -x - -[case testDisjointclass] -from typing import disjointclass -@disjointclass(int) -class A: pass -print(A) -[out] - - -[case testMappingMethods] -# Regression test -from typing import Mapping -x = {'x': 'y'} # type: Mapping[str, str] -print('x' in x) -print('y' in x) -[out] -True -False - -[case testOverlappingOperatorMethods] -from typing import Undefined -class X: pass -class A: - def __add__(self, x: X) -> int: - if isinstance(x, X): - return 1 - return NotImplemented -class B: - def __radd__(self, x: A) -> str: return 'x' -class C(X, B): pass -b = Undefined(B) -b = C() -print(A() + b) -[out] -_program.py: In member "__radd__" of class "B": -_program.py, line 9: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping - -[case testBytesAndBytearrayComparisons] -import typing -print(b'ab' < bytearray(b'b')) -print(bytearray(b'ab') < b'a') -[out] -True -False - -[case testBytesAndBytearrayComparisons2] -import typing -'' < b'' -b'' < '' -'' < bytearray() -bytearray() < '' -[out] -_program.py, line 2: Unsupported operand types for > ("bytes" and "str") -_program.py, line 3: Unsupported operand types for > ("str" and "bytes") -_program.py, line 4: Unsupported operand types for > ("bytearray" and "str") -_program.py, line 5: Unsupported operand types for > ("str" and "bytearray") - -[case testOverloadedOperatorMethodAndReverseMethod] -from typing import overload -class A: - @overload - def __add__(self, x: 'A') -> int: return 1 - @overload - def __add__(self, x: int) -> int: - if isinstance(x, int): - return 2 - else: - return NotImplemented -class B: - def __radd__(self, x: A) -> int: return 3 -print(A() + A()) -print(A() + 1) -print(A() + B()) -[out] -1 -2 -3 - -[case testInplaceOperatorMethod] -import typing -a = [1] -print('', a.__iadd__([2])) -print('', a) -[out] - [1, 2] - [1, 2] - -[case testListInplaceAdd] -import typing -a = [1] -a += iter([2, 3]) -print(tuple(a)) -[out] -(1, 2, 3) - -[case testListConcatenateWithIterable] -import typing -[1] + iter([2, 3]) -[out] -_program.py, line 2: Unsupported operand types for + ("list" and Iterator[int]) - -[case testInferHeterogeneousListOfIterables] -from typing import Sequence -s = ['x', 'y'] # type: Sequence[str] -a = [['x', 'x'], 'fo', s, iter('foo'), {'aa'}] -for i, x in enumerate(a): - print(i, next(iter(x))) -[out] -0 x -1 f -2 x -3 f -4 aa - -[case testTextIOProperties] -import typing -import sys -print(type(sys.stdin.encoding)) -print(type(sys.stdin.errors)) -sys.stdin.line_buffering -sys.stdin.buffer -sys.stdin.newlines -[out] - - - -[case testIOProperties] -import typing -import sys -print(sys.stdin.name) -print(sys.stdin.buffer.mode) -[out] - -rb - -[case testSetUnion] -import typing -s = {'x', 'y'} -print('>', sorted(s.union('foo'))) -[out] -> ['f', 'o', 'x', 'y'] - -[case testOverloadedErasedType] -from typing import overload, typevar, Function, List - -T = typevar("T") -V = typevar("V") - -@overload -def fun(x: int) -> int: return 1 -@overload -def fun(x: str) -> int: return 2 - -def m(fun: Function[[T], V], iter: List[T]) -> int: return 17 - -nums = [1] # type: List[int] -print('>', m(lambda x: fun(x), - nums)) -[out] -> 17 - -[case testFromFuturePrintFunction] -from __future__ import print_function -print('a', 'b') -[out] -a b - -[case testLenOfTuple] -import typing -print(len((1, 'x'))) -[out] -2 - -[case testListMethods] -import typing -import sys -l = [0, 1, 2, 3, 4] -if sys.version >= '3.3': - l.clear() -else: - l = [] -l.append(0) -print('>', l) -if sys.version >= '3.3': - m = l.copy() -else: - m = l[:] -m.extend([1, 2, 3, 4]) -print('>', m) -print(l.index(0)) -print(l.index(0, 0)) -print(l.index(0, 0, 1)) -try: - print(l.index(1)) - print('expected ValueError') -except ValueError: - pass -l.insert(0, 1) -print('>', l) -print(l.pop(0)) -print(l.pop()) -m.remove(0) -try: - m.remove(0) - print('expected ValueError') -except ValueError: - pass -m.reverse() -m.sort() -m.sort(key=lambda x: -x) -m.sort(reverse=False) -m.sort(key=lambda x: -x, reverse=True) -print('>', m) -[out] -> [0] -> [0, 1, 2, 3, 4] -0 -0 -0 -> [1, 0] -1 -0 -> [1, 2, 3, 4] - -[case testListOperators] -import typing -l = [0, 1] -print('+', l + [2]) -print('*', l * 2) -print('*', 2 * l) -print('in', 1 in l) -print('==', l == [1, 2]) -print('!=', l != [1, 2]) -print('>', l > [1, 2, 3]) -print('>=', l >= [1, 2, 3]) -print('<', l < [1, 2, 3]) -print('<=', l <= [1, 2, 3]) -print('>[0]', l[0]) -l += [2] -print('+=', l) -l *= 2 -print('*=', l) -print('iter', list(iter(l))) -print('len', len(l)) -print('repr', repr(l)) -l[:3] = [] -print('setslice', l) -print('reversed', list(reversed(l))) -[out] -+ [0, 1, 2] -* [0, 1, 0, 1] -* [0, 1, 0, 1] -in True -== False -!= True -> False ->= False -< True -<= True ->[0] 0 -+= [0, 1, 2] -*= [0, 1, 2, 0, 1, 2] -iter [0, 1, 2, 0, 1, 2] -len 6 -repr [0, 1, 2, 0, 1, 2] -setslice [0, 1, 2] -reversed [2, 1, 0] - -[case testTupleAsSubtypeOfSequence] -from typing import typevar, Sequence -T = typevar('T') -def f(a: Sequence[T]) -> None: print(a) -f(tuple()) -[out] -() - -[case testMapWithLambdaSpecialCase] -from typing import List, Iterator -a = [[1], [3]] -b = map(lambda y: y[0], a) -print('>', list(b)) -[out] -> [1, 3] - -[case testInternalBuiltinDefinition] -import typing -def f(x: _T) -> None: pass -[out] -_program.py: In function "f": -_program.py, line 2: Name '_T' is not defined - -[case testVarArgsFunctionSubtyping] -import typing -def f(*args: str) -> str: return args[0] -map(f, ['x']) -map(f, [1]) -[out] -_program.py, line 4: Argument 1 to "map" has incompatible type Function[[str], str]; expected Function[[int], str] - -[case testMapStr] -import typing -x = range(3) -a = list(map(str, x)) -a + 1 -[out] -_program.py, line 4: Unsupported operand types for + (List[str] and "int") - -[case testNamedTuple] -import typing -from collections import namedtuple -X = namedtuple('X', ['a', 'b']) -x = X(a=1, b='s') -print(x.a, x.b) -[out] -1 s - -[case testNamedTupleError] -import typing -from collections import namedtuple -X = namedtuple('X', ['a', 'b']) -x = X(a=1, b='s') -x.c -[out] -_program.py, line 5: "X" has no attribute "c" - -[case testNamedTupleTupleOperations] -from typing import Iterable -from collections import namedtuple -X = namedtuple('X', ['a', 'b']) -def f(x: Iterable[int]) -> None: pass -x = X(a=1, b='s') -f(x) -print(len(x)) -print(x.index(1)) -print(x.count(1)) -print(x + x) -[out] -2 -0 -1 -(1, 's', 1, 's') - -[case testNamedTupleWithTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), ('b', str)]) -n = N(1, 'x') -print(n) -a, b = n -print(a, b) -print(n[0]) -[out] -N(a=1, b='x') -1 x -1 - -[case testRelativeImport] -import typing -from m import x -print(x) -[file m/__init__.py] -from .n import x -[file m/n.py] -x = 1 -[out] -1 - -[case testRelativeImport2] -import typing -from m.n import x -print(x) -[file m/__init__.py] -[file m/n.py] -from .nn import x -[file m/nn.py] -x = 2 -[out] -2 - -[case testPyiTakesPrecedenceOverPy] -import m -m.f(1) -[file m.py] -def f(x): - print(x) -[file m.pyi] -import typing -def f(x: str) -> None: pass -[out] -_program.py, line 2: Argument 1 to "f" has incompatible type "int"; expected "str" - -[case testAssignToComplexReal] -import typing -x = 4j -y = x.real -y = x # Error -x.real = 2.0 # Error -[out] -_program.py, line 4: Incompatible types in assignment (expression has type "complex", variable has type "float") -_program.py, line 5: Property "real" defined in "complex" is read-only - -[case testComplexArithmetic] -import typing -print(5 + 8j) -print(3j * 2.0) -print(4J / 2.0) -[out] -(5+8j) -6j -2j - -[case testComplexArithmetic2] -import typing -x = 5 + 8j -x = '' -y = 3j * 2.0 -y = '' -[out] -_program.py, line 3: Incompatible types in assignment (expression has type "str", variable has type "complex") -_program.py, line 5: Incompatible types in assignment (expression has type "str", variable has type "complex") diff --git a/mypy/test/data/semanal-basic.test b/mypy/test/data/semanal-basic.test deleted file mode 100644 index 192f85cd57bd..000000000000 --- a/mypy/test/data/semanal-basic.test +++ /dev/null @@ -1,461 +0,0 @@ -[case testEmptyFile] -[out] -MypyFile:1() - -[case testGlobalVariable] -x = 1 -x -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - ExpressionStmt:2( - NameExpr(x [__main__.x]))) - -[case testMultipleGlobals] -x = y = 2 -z = 3 -(x, y, z) -[out] -MypyFile:1( - AssignmentStmt:1( - Lvalues( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - IntExpr(2)) - AssignmentStmt:2( - NameExpr(z* [__main__.z]) - IntExpr(3)) - ExpressionStmt:3( - ParenExpr:3( - TupleExpr:3( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y]) - NameExpr(z [__main__.z]))))) - -[case testEmptyFunction] -def f(): pass -f() -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - PassStmt:1())) - ExpressionStmt:2( - CallExpr:2( - NameExpr(f [__main__.f]) - Args()))) - -[case testAccessingGlobalNameBeforeDefinition] -x -f() -x = 1 -def f(): pass -[out] -MypyFile:1( - ExpressionStmt:1( - NameExpr(x [__main__.x])) - ExpressionStmt:2( - CallExpr:2( - NameExpr(f [__main__.f]) - Args())) - AssignmentStmt:3( - NameExpr(x* [__main__.x]) - IntExpr(1)) - FuncDef:4( - f - Block:4( - PassStmt:4()))) - -[case testFunctionArgs] -def f(x, y): - (x, y) -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x) - Var(y)) - Block:1( - ExpressionStmt:2( - ParenExpr:2( - TupleExpr:2( - NameExpr(x [l]) - NameExpr(y [l]))))))) - -[case testLocalVar] -def f(): - x = 1 - x -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - AssignmentStmt:2( - NameExpr(x* [l]) - IntExpr(1)) - ExpressionStmt:3( - NameExpr(x [l]))))) - -[case testAccessGlobalInFn] -def f(): - x - g() -x = 1 -def g(): pass -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - ExpressionStmt:2( - NameExpr(x [__main__.x])) - ExpressionStmt:3( - CallExpr:3( - NameExpr(g [__main__.g]) - Args())))) - AssignmentStmt:4( - NameExpr(x* [__main__.x]) - IntExpr(1)) - FuncDef:5( - g - Block:5( - PassStmt:5()))) - -[case testAssignmentAfterInit] -x = 1 -x = 2 -def f(y): - y = 1 - z = 1 - z = 2 -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - AssignmentStmt:2( - NameExpr(x [__main__.x]) - IntExpr(2)) - FuncDef:3( - f - Args( - Var(y)) - Block:3( - AssignmentStmt:4( - NameExpr(y [l]) - IntExpr(1)) - AssignmentStmt:5( - NameExpr(z* [l]) - IntExpr(1)) - AssignmentStmt:6( - NameExpr(z [l]) - IntExpr(2))))) - -[case testLocalAndGlobalAliasing] -x = 1 -def f(): - x = 2 - x -x -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - FuncDef:2( - f - Block:2( - AssignmentStmt:3( - NameExpr(x* [l]) - IntExpr(2)) - ExpressionStmt:4( - NameExpr(x [l])))) - ExpressionStmt:5( - NameExpr(x [__main__.x]))) - -[case testArgumentInitializers] -def f(x = f, y = object): - x, y -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x) - Var(y)) - Init( - AssignmentStmt:1( - NameExpr(x [l]) - NameExpr(f [__main__.f])) - AssignmentStmt:1( - NameExpr(y [l]) - NameExpr(object [builtins.object]))) - Block:1( - ExpressionStmt:2( - TupleExpr:2( - NameExpr(x [l]) - NameExpr(y [l])))))) - -[case testVarArgs] -def f(x, *y): - x, y -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - VarArg( - Var(y)) - Block:1( - ExpressionStmt:2( - TupleExpr:2( - NameExpr(x [l]) - NameExpr(y [l])))))) - -[case testGlobalDecl] -x = None -def f(): - global x - x = None - x -class A: pass -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - NameExpr(None [builtins.None])) - FuncDef:2( - f - Block:2( - GlobalDecl:3( - x) - AssignmentStmt:4( - NameExpr(x [__main__.x]) - NameExpr(None [builtins.None])) - ExpressionStmt:5( - NameExpr(x [__main__.x])))) - ClassDef:6( - A - PassStmt:6())) - -[case testMultipleNamesInGlobalDecl] -x, y = None, None -def f(): - global x, y - x = y -[out] -MypyFile:1( - AssignmentStmt:1( - TupleExpr:1( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - TupleExpr:1( - NameExpr(None [builtins.None]) - NameExpr(None [builtins.None]))) - FuncDef:2( - f - Block:2( - GlobalDecl:3( - x - y) - AssignmentStmt:4( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y]))))) - -[case testGlobalDeclScope] -x = None -def f(): - global x -def g(): - x = None -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - NameExpr(None [builtins.None])) - FuncDef:2( - f - Block:2( - GlobalDecl:3( - x))) - FuncDef:4( - g - Block:4( - AssignmentStmt:5( - NameExpr(x* [l]) - NameExpr(None [builtins.None]))))) - -[case testGlobalDeclScope] -x = None -def f(): - global x -def g(): - x = None -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - NameExpr(None [builtins.None])) - FuncDef:2( - f - Block:2( - GlobalDecl:3( - x))) - FuncDef:4( - g - Block:4( - AssignmentStmt:5( - NameExpr(x* [l]) - NameExpr(None [builtins.None]))))) - -[case testGlobaWithinMethod] -x = None -class A: - def f(self): - global x - x = self -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - NameExpr(None [builtins.None])) - ClassDef:2( - A - FuncDef:3( - f - Args( - Var(self)) - Block:3( - GlobalDecl:4( - x) - AssignmentStmt:5( - NameExpr(x [__main__.x]) - NameExpr(self [l])))))) - -[case testGlobalDefinedInBlock] -if object: - x = object() - x = x -x -[out] -MypyFile:1( - IfStmt:1( - If( - NameExpr(object [builtins.object])) - Then( - AssignmentStmt:2( - NameExpr(x* [__main__.x]) - CallExpr:2( - NameExpr(object [builtins.object]) - Args())) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(x [__main__.x])))) - ExpressionStmt:4( - NameExpr(x [__main__.x]))) - -[case testNonlocalDecl] -def g(): - x = None - def f(): - nonlocal x - x = None - x -[out] -MypyFile:1( - FuncDef:1( - g - Block:1( - AssignmentStmt:2( - NameExpr(x* [l]) - NameExpr(None [builtins.None])) - FuncDef:3( - f - Block:3( - NonlocalDecl:4( - x) - AssignmentStmt:5( - NameExpr(x [l]) - NameExpr(None [builtins.None])) - ExpressionStmt:6( - NameExpr(x [l]))))))) - -[case testMultipleNamesInNonlocalDecl] -def g(): - x, y = None, None - def f(z): - nonlocal x, y - x = y -[out] -MypyFile:1( - FuncDef:1( - g - Block:1( - AssignmentStmt:2( - TupleExpr:2( - NameExpr(x* [l]) - NameExpr(y* [l])) - TupleExpr:2( - NameExpr(None [builtins.None]) - NameExpr(None [builtins.None]))) - FuncDef:3( - f - Args( - Var(z)) - Block:3( - NonlocalDecl:4( - x - y) - AssignmentStmt:5( - NameExpr(x [l]) - NameExpr(y [l]))))))) - -[case testNestedFunctions] -def f(x): - def g(y): - z = y + x - return g -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - Block:1( - FuncDef:2( - g - Args( - Var(y)) - Block:2( - AssignmentStmt:3( - NameExpr(z* [l]) - OpExpr:3( - + - NameExpr(y [l]) - NameExpr(x [l]))))) - ReturnStmt:4( - NameExpr(g [l]))))) - -[case testNestedFunctionWithOverlappingName] -def f(x): - def g(): - x = 1 -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - Block:1( - FuncDef:2( - g - Block:2( - AssignmentStmt:3( - NameExpr(x* [l]) - IntExpr(1))))))) diff --git a/mypy/test/data/semanal-classes.test b/mypy/test/data/semanal-classes.test deleted file mode 100644 index 2c85e119fe49..000000000000 --- a/mypy/test/data/semanal-classes.test +++ /dev/null @@ -1,542 +0,0 @@ --- Test cases related to classes for the semantic analyser. - -[case testSimpleClass] -class A: pass -x = A -[out] -MypyFile:1( - ClassDef:1( - A - PassStmt:1()) - AssignmentStmt:2( - NameExpr(x* [__main__.x]) - NameExpr(A [__main__.A]))) - -[case testMethods] -class A: - def __init__(self, x): - y = x - def f(self): - y = self -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - __init__ - Args( - Var(self) - Var(x)) - Block:2( - AssignmentStmt:3( - NameExpr(y* [l]) - NameExpr(x [l])))) - FuncDef:4( - f - Args( - Var(self)) - Block:4( - AssignmentStmt:5( - NameExpr(y* [l]) - NameExpr(self [l])))))) - -[case testMemberDefinitionInInit] -class A: - def __init__(self): - self.x = 1 - self.y = 2 -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - __init__ - Args( - Var(self)) - Block:2( - AssignmentStmt:3( - MemberExpr:3( - NameExpr(self [l]) - x*) - IntExpr(1)) - AssignmentStmt:4( - MemberExpr:4( - NameExpr(self [l]) - y*) - IntExpr(2)))))) - -[case testMemberAssignmentViaSelfOutsideInit] -class A: - def f(self): - self.x = 1 -def __init__(self): - self.y = 1 -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - f - Args( - Var(self)) - Block:2( - AssignmentStmt:3( - MemberExpr:3( - NameExpr(self [l]) - x*) - IntExpr(1))))) - FuncDef:4( - __init__ - Args( - Var(self)) - Block:4( - AssignmentStmt:5( - MemberExpr:5( - NameExpr(self [l]) - y) - IntExpr(1))))) - -[case testMemberAssignmentNotViaSelf] -class A: - def __init__(x, self): - self.y = 1 # not really self -class B: - def __init__(x): - self = x - self.z = 1 -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - __init__ - Args( - Var(x) - Var(self)) - Block:2( - AssignmentStmt:3( - MemberExpr:3( - NameExpr(self [l]) - y) - IntExpr(1))))) - ClassDef:4( - B - FuncDef:5( - __init__ - Args( - Var(x)) - Block:5( - AssignmentStmt:6( - NameExpr(self* [l]) - NameExpr(x [l])) - AssignmentStmt:7( - MemberExpr:7( - NameExpr(self [l]) - z) - IntExpr(1)))))) - -[case testNonStandardNameForSelfAndInit] -class A: - def __init__(x): - x.y = 1 -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - __init__ - Args( - Var(x)) - Block:2( - AssignmentStmt:3( - MemberExpr:3( - NameExpr(x [l]) - y*) - IntExpr(1)))))) - -[case testAssignmentAfterAttributeInit] -class A: - def __init__(self): - self.x = 1 - self.x = 2 -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - __init__ - Args( - Var(self)) - Block:2( - AssignmentStmt:3( - MemberExpr:3( - NameExpr(self [l]) - x*) - IntExpr(1)) - AssignmentStmt:4( - MemberExpr:4( - NameExpr(self [l]) - x) - IntExpr(2)))))) - -[case testOverloadedMethod] -from typing import overload -class A: - @overload - def f(self) -> None: self - @overload - def f(self, x: 'A') -> None: self -[out] -MypyFile:1( - ImportFrom:1(typing, [overload : overload]) - ClassDef:2( - A - OverloadedFuncDef:3( - Overload(def (self: __main__.A), \ - def (self: __main__.A, x: __main__.A)) - Decorator:3( - Var(f) - NameExpr(overload [typing.overload]) - FuncDef:4( - f - Args( - Var(self)) - def (self: __main__.A) - Block:4( - ExpressionStmt:4( - NameExpr(self [l]))))) - Decorator:5( - Var(f) - NameExpr(overload [typing.overload]) - FuncDef:6( - f - Args( - Var(self) - Var(x)) - def (self: __main__.A, x: __main__.A) - Block:6( - ExpressionStmt:6( - NameExpr(self [l])))))))) - -[case testAttributeWithoutType] -class A: - a = object -[out] -MypyFile:1( - ClassDef:1( - A - AssignmentStmt:2( - NameExpr(a* [m]) - NameExpr(object [builtins.object])))) - -[case testDataAttributeRefInClassBody] -class A: - x = 1 - y = x -[out] -MypyFile:1( - ClassDef:1( - A - AssignmentStmt:2( - NameExpr(x* [m]) - IntExpr(1)) - AssignmentStmt:3( - NameExpr(y* [m]) - NameExpr(x [m])))) - -[case testMethodRefInClassBody] -class A: - def f(self): pass - g = f -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - f - Args( - Var(self)) - Block:2( - PassStmt:2())) - AssignmentStmt:3( - NameExpr(g* [m]) - NameExpr(f [m])))) - -[case testIfStatementInClassBody] -class A: - if A: - x = 1 - else: - x = 2 -[out] -MypyFile:1( - ClassDef:1( - A - IfStmt:2( - If( - NameExpr(A [__main__.A])) - Then( - AssignmentStmt:3( - NameExpr(x* [m]) - IntExpr(1))) - Else( - AssignmentStmt:5( - NameExpr(x [m]) - IntExpr(2)))))) - -[case testForStatementInClassBody] -class A: - for x in [1, 2]: - y = x -[out] -MypyFile:1( - ClassDef:1( - A - ForStmt:2( - NameExpr(x* [m]) - ListExpr:2( - IntExpr(1) - IntExpr(2)) - Block:2( - AssignmentStmt:3( - NameExpr(y* [m]) - NameExpr(x [m])))))) - -[case testReferenceToClassWithinFunction] -def f(): - class A: pass - A -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - ClassDef:2( - A - PassStmt:2()) - ExpressionStmt:3( - NameExpr(A [l]))))) - -[case testReferenceToClassWithinClass] -class A: - class B: pass - B -[out] -MypyFile:1( - ClassDef:1( - A - ClassDef:2( - B - PassStmt:2()) - ExpressionStmt:3( - NameExpr(B [m])))) - -[case testDeclarationReferenceToNestedClass] -def f() -> None: - class A: pass - x = None # type: A -[out] -MypyFile:1( - FuncDef:1( - f - def () - Block:1( - ClassDef:2( - A - PassStmt:2()) - AssignmentStmt:3( - NameExpr(x [l]) - NameExpr(None [builtins.None]) - A)))) - -[case testAccessToLocalInOuterScopeWithinNestedClass] -def f(x): - class A: - y = x - def g(self): - z = x -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - Block:1( - ClassDef:2( - A - AssignmentStmt:3( - NameExpr(y* [m]) - NameExpr(x [l])) - FuncDef:4( - g - Args( - Var(self)) - Block:4( - AssignmentStmt:5( - NameExpr(z* [l]) - NameExpr(x [l])))))))) - -[case testQualifiedMetaclass] -import abc -class A(metaclass=abc.ABCMeta): pass -[out] -MypyFile:1( - Import:1(abc : abc) - ClassDef:2( - A - Metaclass(abc.ABCMeta) - PassStmt:2())) - -[case testStaticMethod] -class A: - @staticmethod - def f(z: int) -> str: pass -[builtins fixtures/staticmethod.py] -[out] -MypyFile:1( - ClassDef:1( - A - Decorator:2( - Var(f) - FuncDef:3( - f - Args( - Var(z)) - def (z: builtins.int) -> builtins.str - Static - Block:3( - PassStmt:3()))))) - -[case testStaticMethodWithNoArgs] -class A: - @staticmethod - def f() -> str: pass -[builtins fixtures/staticmethod.py] -[out] -MypyFile:1( - ClassDef:1( - A - Decorator:2( - Var(f) - FuncDef:3( - f - def () -> builtins.str - Static - Block:3( - PassStmt:3()))))) - -[case testClassMethod] -class A: - @classmethod - def f(cls, z: int) -> str: pass -[builtins fixtures/classmethod.py] -[out] -MypyFile:1( - ClassDef:1( - A - Decorator:2( - Var(f) - FuncDef:3( - f - Args( - Var(cls) - Var(z)) - def (cls: Any, z: builtins.int) -> builtins.str - Class - Block:3( - PassStmt:3()))))) - -[case testClassMethodWithNoArgs] -class A: - @classmethod - def f(cls) -> str: pass -[builtins fixtures/classmethod.py] -[out] -MypyFile:1( - ClassDef:1( - A - Decorator:2( - Var(f) - FuncDef:3( - f - Args( - Var(cls)) - def (cls: Any) -> builtins.str - Class - Block:3( - PassStmt:3()))))) - -[case testProperty] -import typing -class A: - @property - def f(self) -> str: pass -[builtins fixtures/property.py] -[out] -MypyFile:1( - Import:1(typing : typing) - ClassDef:2( - A - Decorator:3( - Var(f) - FuncDef:4( - f - Args( - Var(self)) - def (self: __main__.A) -> builtins.str - Property - Block:4( - PassStmt:4()))))) - -[case testClassDecorator] -import typing -@object -class A: pass -[out] -MypyFile:1( - Import:1(typing : typing) - ClassDef:2( - A - Decorators( - NameExpr(object [builtins.object])) - PassStmt:3())) - -[case testBuiltinclassDecorator] -from typing import builtinclass -@builtinclass -class A: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [builtinclass : builtinclass]) - ClassDef:2( - A - Builtinclass - Decorators( - NameExpr(builtinclass [typing.builtinclass])) - PassStmt:3())) - -[case testClassAttributeAsMethodDefaultArgumentValue] -import typing -class A: - X = 1 - def f(self, x : int = X) -> None: pass -[out] -MypyFile:1( - Import:1(typing : typing) - ClassDef:2( - A - AssignmentStmt:3( - NameExpr(X* [m]) - IntExpr(1)) - FuncDef:4( - f - Args( - Var(self) - Var(x)) - def (self: __main__.A, x: builtins.int =) - Init( - AssignmentStmt:4( - NameExpr(x [l]) - NameExpr(X [m]))) - Block:4( - PassStmt:4())))) diff --git a/mypy/test/data/semanal-errors.test b/mypy/test/data/semanal-errors.test deleted file mode 100644 index 33da3bc3bcf4..000000000000 --- a/mypy/test/data/semanal-errors.test +++ /dev/null @@ -1,1316 +0,0 @@ -[case testPropagatingParseErrors] -in 1 -def f(): - 1 1 -[out] -main, line 1: Parse error before in -main: In function "f": -main, line 3: Parse error before numeric literal - -[case testUndefinedVariableInGlobalStatement] -import typing -x -y -[out] -main, line 2: Name 'x' is not defined -main, line 3: Name 'y' is not defined - -[case testUndefinedVariableWithinFunctionContext] -import typing -def f(): - x -y -[out] -main: In function "f": -main, line 3: Name 'x' is not defined -main: At top level: -main, line 4: Name 'y' is not defined - -[case testMethodScope] -import typing -class A: - def f(self): pass -f -[out] -main, line 4: Name 'f' is not defined - -[case testMethodScope2] -import typing -class A: - def f(self): pass -class B: - def g(self): - f # error - g # error -[out] -main: In function "g": -main, line 6: Name 'f' is not defined -main, line 7: Name 'g' is not defined - -[case testInvalidType] -import typing -x = None # type: X -[out] -main, line 2: Name 'X' is not defined - -[case testInvalidGenericArg] -from typing import typevar, Generic -t = typevar('t') -class A(Generic[t]): pass -x = 0 # type: A[y] -[out] -main, line 4: Name 'y' is not defined - -[case testInvalidNumberOfGenericArgsInTypeDecl] -from typing import typevar, Generic -t = typevar('t') -class A: pass -class B(Generic[t]): pass -x = 0 # type: B[A, A] -y = 0 # type: A[A] -[out] -main, line 5: "B" expects 1 type argument, but 2 given -main, line 6: "A" expects no type arguments, but 1 given - -[case testInvalidNumberOfGenericArgsInUndefinedArg] -from typing import Undefined -class A: pass -x = Undefined(A[int]) # E: "A" expects no type arguments, but 1 given -[out] - -[case testInvalidNumberOfGenericArgsInNestedBlock] -from typing import Undefined -class A: pass -class B: - def f(self) -> None: - while 1: - x = Undefined(A[int]) \ - # E: "A" expects no type arguments, but 1 given -[out] -main: In function "f": - -[case testInvalidNumberOfGenericArgsInSignature] -import typing -class A: pass -def f() -> A[int]: pass # E: "A" expects no type arguments, but 1 given -[out] -main: In function "f": - -[case testInvalidNumberOfGenericArgsInOverloadedSignature] -from typing import overload -class A: pass -@overload -def f(): pass -@overload -def f(x: A[int]) -> None: pass # E: "A" expects no type arguments, but 1 given -[out] -main: In function "f": - -[case testInvalidNumberOfGenericArgsInBaseType] -import typing -class A: pass -class B(A[int]): pass # E: "A" expects no type arguments, but 1 given -[out] - -[case testInvalidNumberOfGenericArgsInCast] -from typing import cast -class A: pass -x = cast(A[int], 1) # E: "A" expects no type arguments, but 1 given -[out] - -[case testInvalidNumberOfGenericArgsInTypeApplication] -import typing -class A: pass -class B: pass -x = A[B[int]]() # E: "B" expects no type arguments, but 1 given -[out] - -[case testInvalidNumberOfGenericArgsInNestedGenericType] -from typing import typevar, Generic -T = typevar('T') -class A(Generic[T]): pass -class B: pass -def f() -> A[B[int]]: pass # E: "B" expects no type arguments, but 1 given -[out] -main: In function "f": - -[case testInvalidNumberOfGenericArgsInTupleType] -from typing import Tuple, Undefined -class A: pass -x = Undefined(Tuple[A[int]]) # E: "A" expects no type arguments, but 1 given -[out] - -[case testInvalidNumberOfGenericArgsInFunctionType] -from typing import Function, Undefined -class A: pass -x = Undefined(Function[[A[int]], int]) \ - # E: "A" expects no type arguments, but 1 given -x = Undefined(Function[[], A[int]]) \ - # E: "A" expects no type arguments, but 1 given -[out] - -[case testVarOrFuncAsType] -import typing -def f(): pass -x = 1 -y = 0 # type: f -z = 0 # type: x -[out] -main, line 4: Invalid type "__main__.f" -main, line 5: Invalid type "__main__.x" - -[case testTwoStarsInType] -import typing -x = 1 # type: *object, *object -y = 1 # type: object, (*object, *object) -z = 1 # type: *object, (object, *object) -[out] -main, line 2: At most one star type allowed in a tuple -main, line 3: At most one star type allowed in a tuple - -[case testGlobalVarRedefinition] -import typing -class A: pass -x = 0 # type: A -x = 0 # type: A -[out] -main, line 4: Name 'x' already defined - -[case testLocalVarRedefinition] -import typing -class A: pass -def f(): - x = 0 # type: A - x = 0 # type: A -[out] -main: In function "f": -main, line 5: Name 'x' already defined - -[case testClassVarRedefinition] -import typing -class A: - x = 0 # type: object - x = 0 # type: object -[out] -main, line 4: Name 'x' already defined - -[case testMultipleClassDefinitions] -import typing -class A: pass -class A: pass -[out] -main, line 3: Name 'A' already defined - -[case testMultipleMixedDefinitions] -import typing -x = 1 -def x(): pass -class x: pass -[out] -main, line 3: Name 'x' already defined -main, line 4: Name 'x' already defined - -[case testNameNotImported] -import typing -from m import y -x -[file m.py] -x = y = 1 -[out] -main, line 3: Name 'x' is not defined - -[case testMissingNameInImportFrom] -import typing -from m import y -[file m.py] -x = 1 -[out] -main, line 2: Module has no attribute 'y' - -[case testMissingModule] -import typing -import m -[out] -main, line 2: No module named 'm' - -[case testMissingModule2] -import typing -from m import x -[out] -main, line 2: No module named 'm' - -[case testMissingModule3] -import typing -from m import * -[out] -main, line 2: No module named 'm' - -[case testMissingModuleRelativeImport] -import typing -import m -[file m/__init__.py] -from .x import y -[out] -In module imported in main, line 2: -tmp/m/__init__.py, line 1: No module named 'm.x' - -[case testMissingModuleRelativeImport2] -import typing -import m.a -[file m/__init__.py] -[file m/a.py] -from .x import y -[out] -In module imported in main, line 2: -tmp/m/a.py, line 1: No module named 'm.x' - -[case testModuleNotImported] -import typing -import _m -_n.x -[file _m.py] -import _n -[file _n.py] -x = 1 -[out] -main, line 3: Name '_n' is not defined - -[case testImportAsteriskPlusUnderscore] -import typing -from _m import * -_x -__x__ -[file _m.py] -_x = __x__ = 1 -[out] -main, line 3: Name '_x' is not defined -main, line 4: Name '__x__' is not defined - -[case testRelativeImportAtTopLevelModule] -from . import m -[out] -main, line 1: No parent module -- cannot perform relative import - -[case testRelativeImportAtTopLevelModule] -from .. import m -[out] -main, line 1: No parent module -- cannot perform relative import - -[case testUndefinedTypeWithQualifiedName] -import typing -import m -def f() -> m.c: pass -def g() -> n.c: pass -[file m.py] -[out] -main: In function "f": -main, line 3: Name 'm.c' is not defined -main: In function "g": -main, line 4: Name 'n' is not defined - -[case testMissingPackage] -import typing -import m.n -[out] -main, line 2: No module named 'm.n' - -[case testMissingPackage] -import typing -from m.n import x -from a.b import * -[out] -main, line 2: No module named 'm.n' -main, line 3: No module named 'a.b' - -[case testErrorInImportedModule] -import m -[file m.py] -import typing -x = y -[out] -In module imported in main, line 1: -tmp/m.py, line 2: Name 'y' is not defined - -[case testErrorInImportedModule2] -import m.n -[file m/__init__.py] -[file m/n.py] -import k -[file k.py] -import typing -x = y -[out] -In module imported in tmp/m/n.py, line 1, - in main, line 1: -tmp/k.py, line 2: Name 'y' is not defined - -[case testPackageWithoutInitFile] -import typing -import m.n -m.n.x -[file m/n.py] -x = 1 -[out] -main, line 2: No module named 'm.n' - -[case testBreakOutsideLoop] -break -def f(): - break -[out] -main, line 1: 'break' outside loop -main: In function "f": -main, line 3: 'break' outside loop - -[case testContinueOutsideLoop] -continue -def f(): - continue -[out] -main, line 1: 'continue' outside loop -main: In function "f": -main, line 3: 'continue' outside loop - -[case testReturnOutsideFunction] -def f(): pass -return -return 1 -[out] -main, line 2: 'return' outside function -main, line 3: 'return' outside function - -[case testYieldOutsideFunction] -yield 1 -yield -[out] -main, line 1: 'yield' outside function -main, line 2: 'yield' outside function - -[case testInvalidLvalues] -1 = 1 -(1) = 1 -(1, 1) = 1 -[1, 1] = 1 -() = 1 -[out] -main, line 1: Invalid assignment target -main, line 2: Invalid assignment target -main, line 3: Invalid assignment target -main, line 4: Invalid assignment target -main, line 5: Can't assign to () ---' (hack to fix syntax highlighting) - -[case testInvalidLvalues2] -x = y = z = 1 -x, (y, 1) = 1 -x, [y, 1] = 1 -x, [y, [z, 1]] = 1 -x, (y, (z, 1)) = 1 -x, (y) = 1 # ok -x, (y, (z, z)) = 1 # ok -[out] -main, line 2: Invalid assignment target -main, line 3: Invalid assignment target -main, line 4: Invalid assignment target -main, line 5: Invalid assignment target - -[case testInvalidLvalues3] -x = 1 -x + x = 1 --x = 1 -1.1 = 1 -'x' = 1 -x() = 1 -[out] -main, line 2: Invalid assignment target -main, line 3: Invalid assignment target -main, line 4: Invalid assignment target -main, line 5: Invalid assignment target -main, line 6: Invalid assignment target - -[case testInvalidStarType] -a = 1 # type: *int -[out] -main, line 1: Star type only allowed for starred expressions - -[case testInvalidStarType] -*a, b = 1 # type: int, int -[out] -main, line 1: Star type expected for starred expression - -[case testTwoStarExpressions] -a, *b, *c = 1 -*a, (*b, c) = 1 -a, (*b, *c) = 1 -[*a, *b] = 1 -[out] -main, line 1: Two starred expressions in assignment -main, line 3: Two starred expressions in assignment -main, line 4: Two starred expressions in assignment - -[case testTwoStarExpressionsInForStmt] -for a, *b, *c in z: - pass -for *a, (*b, c) in z: - pass -for a, (*b, *c) in z: - pass -for [*a, *b] in z: - pass -[out] -main, line 1: Two starred expressions in assignment -main, line 5: Two starred expressions in assignment -main, line 7: Two starred expressions in assignment - -[case testTwoStarExpressionsInGeneratorExpr] -(a for a, *b, *c in []) -(a for *a, (*b, c) in []) -(a for a, (*b, *c) in []) -[out] -main, line 1: Two starred expressions in assignment -main, line 1: Name 'a' is not defined -main, line 3: Two starred expressions in assignment - -[case testStarExpressionRhs] -b = 1 -c = 1 -d = 1 -a = *b -a = b, (c, *d) -[out] -main, line 4: Can use starred expression only as assignment target -main, line 5: Can use starred expression only as assignment target - -[case testStarExpressionInExp] -a = 1 -*a + 1 -[out] -main, line 2: Can use starred expression only as assignment target - -[case testInvalidDel] -import typing -x = 1 -del x(1) # E: Invalid delete target -del x + 1 # E: Invalid delete target -del z # E: Name 'z' is not defined -[out] - -[case testFunctionTvarScope] -from typing import typevar -t = typevar('t') -def f(x: t) -> t: pass -x = 0 # type: t -[out] -main, line 4: Invalid type "__main__.t" - -[case testClassTvarScope] -from typing import Generic, typevar -t = typevar('t') -class c(Generic[t]): pass -x = 0 # type: t -[out] -main, line 4: Invalid type "__main__.t" - -[case testExpressionRefersToTypeVariable] -from typing import typevar, Generic -t = typevar('t') -class c(Generic[t]): - def f(self): x = t -def f(y: t): x = t -[out] -main: In function "f": -main, line 4: 't' is a type variable and only valid in type context -main, line 5: 't' is a type variable and only valid in type context - -[case testMissingSelf] -import typing -class A: - def f(): pass -[out] -main, line 3: Method must have at least one argument - -[case testInvalidBaseClass] -import typing -class A(B): pass -[out] -main, line 2: Name 'B' is not defined - -[case testSuperOutsideClass] -class A: pass -super().x -def f(): super().y -[out] -main, line 2: "super" used outside class -main: In function "f": -main, line 3: "super" used outside class - -[case testMissingSelfInMethod] -import typing -class A: - def f() -> None: pass - def g(): pass -[out] -main, line 3: Method must have at least one argument -main, line 4: Method must have at least one argument - -[case testMultipleMethodDefinition] -import typing -class A: - def f(self) -> None: pass - def g(self) -> None: pass - def f(self, x: object) -> None: pass -[out] -main, line 5: Name 'f' already defined - -[case testInvalidGlobalDecl] -import typing -def f(): - global x - x = None -[out] -main: In function "f": -main, line 4: Name 'x' is not defined - -[case testInvalidNonlocalDecl] -import typing -def f(): - def g(): - nonlocal x - x = None -[out] -main: In function "g": -main, line 4: No binding for nonlocal 'x' found -main, line 5: Name 'x' is not defined - -[case testNonlocalDeclNotMatchingGlobal] -import typing -x = None -def f(): - nonlocal x - x = None -[out] -main: In function "f": -main, line 4: No binding for nonlocal 'x' found -main, line 5: Name 'x' is not defined - -[case testNonlocalDeclConflictingWithParameter] -import typing -def g(): - x = None - def f(x): - nonlocal x - x = None -[out] -main: In function "f": -main, line 5: Name 'x' is already defined in local scope before nonlocal declaration - -[case testNonlocalDeclOutsideFunction] -x = 2 -nonlocal x -[out] -main, line 2: nonlocal declaration not allowed at module level - -[case testGlobalAndNonlocalDecl] -import typing -x = 1 -def f(): - x = 1 - def g(): - global x - nonlocal x - x = None -[out] -main: In function "g": -main, line 7: Name 'x' is nonlocal and global - -[case testNonlocalAndGlobalDecl] -import typing -x = 1 -def f(): - x = 1 - def g(): - nonlocal x - global x - x = None -[out] -main: In function "g": -main, line 7: Name 'x' is nonlocal and global - -[case testNestedFunctionAndScoping] -import typing -def f(x): - def g(y): - z = x - z - y - x -[out] -main: In function "f": -main, line 5: Name 'z' is not defined -main, line 6: Name 'y' is not defined - -[case testMultipleNestedFunctionDef] -import typing -def f(x): - def g(): pass - x = 1 - def g(): pass -[out] -main: In function "f": -main, line 5: Name 'g' already defined - -[case testRedefinedOverloadedFunction] -from typing import overload, Any -def f(): - @overload - def p(o: object) -> None: pass # no error - @overload - def p(o: Any) -> None: pass # no error - x = 1 - def p(): pass # fail -[out] -main: In function "f": -main, line 8: Name 'p' already defined - -[case testNestedFunctionInMethod] -import typing -class A: - def f(self): - def g(): - x - y -[out] -main: In function "g": -main, line 5: Name 'x' is not defined -main: In function "f": -main, line 6: Name 'y' is not defined - -[case testImportScope] -import typing -def f() -> None: - import x -x.y # E: Name 'x' is not defined -[file x.py] -y = 1 -[out] - -[case testImportScope2] -import typing -def f() -> None: - from x import y - y -y # E: Name 'y' is not defined -[file x.py] -y = 1 -[out] - -[case testImportScope3] -import typing -def f() -> None: - from x import * - y -y # E: Name 'y' is not defined -[file x.py] -y = 1 -[out] - -[case testImportScope4] -import typing -class A: - from x import * - y -y # E: Name 'y' is not defined -[file x.py] -y = 1 -[out] - -[case testImportTwice] -import typing -from x import a, a -def f() -> None: - from x import a - from x import a -import x -import x # ok, since we may import multiple submodules of a package -[file x.py] -a = 1 -[out] -main, line 2: Name 'a' already defined -main: In function "f": -main, line 5: Name 'a' already defined - -[case testScopeOfNestedClass] -import typing -def f(): - class A: pass - A -A # E: Name 'A' is not defined -[out] - -[case testScopeOfNestedClass2] -import typing -class A: - class B: pass -B # E: Name 'B' is not defined -[out] - -[case testScopeOfNestedClass3] -import typing -class A: - def f(self): - class B: pass - B # E: Name 'B' is not defined -B # E: Name 'B' is not defined -[out] - -[case testInvalidNestedClassReferenceInDecl] -import typing -class A: pass -foo = 0 # type: A.x # E: Name 'A.x' is not defined -[out] - -[case testTvarScopingWithNestedClass] -from typing import typevar, Generic -t = typevar('t') -s = typevar('s') -class A(Generic[t]): - class B(Generic[s]): - x = 0 # type: A[s] - y = 0 # type: A[t] # E: Invalid type "__main__.t" - z = 0 # type: A[s] # E: Invalid type "__main__.s" - a = 0 # type: A[t] -[out] - -[case testTestExtendPrimitives] -class C(bool): pass # E: 'bool' is not a valid base class -class A(int): pass # ok -class B(float): pass # ok -class D(str): pass # ok -[builtins fixtures/primitives.py] -[out] - -[case testCyclicInheritance] -class A(A): pass # E: Cycle in inheritance hierarchy -[out] - -[case testAssignToFuncDef] -def f(): pass -f = None # E: Invalid assignment target -[out] - -[case testAssignToFuncDefViaModule] -import m -m.f = None # E: Invalid assignment target -[file m.py] -def f(): pass -[out] - -[case testAssignToFuncDefViaNestedModules] -import m.n -m.n.f = None # E: Invalid assignment target -[file m/__init__.py] -[file m/n.py] -def f(): pass -[out] - -[case testAssignToFuncDefViaImport] -from m import f # E: Name 'f' already defined -f = None -[file m.py] -def f(): pass -[out] - -[case testAssignToFuncDefViaImport2] -from m import * # E: Name 'f' already defined -f = None -[file m.py] -def f(): pass -[out] - -[case testAssignToFuncDefViaGlobalDecl2] -import typing -from m import f -def g() -> None: - global f - f = None # E: Invalid assignment target -[file m.py] -def f(): pass -[out] -main: In function "g": - -[case testAssignToTypeDef] -import typing -class A: pass -A = None # E: Invalid assignment target -[out] - -[case testInvalidQualifiedName] -from typing import Any -x = 0 # type: Any -z = 0 # type: x.y # E: Invalid type "__main__.x" -[out] - -[case testInvalidCastTargetSyntax] -from typing import cast, typevar, Generic -t = typevar('t') -class C(Generic[t]): pass -cast(str + str, None) # E: Cast target is not a type -cast(C[str][str], None) # E: Cast target is not a type -cast((str).str, None) # E: Cast target is not a type -cast(C[str + str], None) # E: Cast target is not a type -cast([int, str], None) # E: Invalid type -[out] - -[case testInvalidCastTargetType] -from typing import cast -x = 0 -cast(x, None) # E: Invalid type "__main__.x" -cast(t, None) # E: Name 't' is not defined -cast(__builtins__.x, None) # E: Name '__builtins__.x' is not defined -[out] - -[case testInvalidCastTargetType2] -from typing import cast -x = 0 -cast(str[str], None) # E: "str" expects no type arguments, but 1 given -[out] - -[case testInvalidNumberOfArgsToCast] -from typing import cast -cast(str) # E: 'cast' expects 2 arguments -cast(str, None, None) # E: 'cast' expects 2 arguments -[out] - -[case testInvalidKindsOfArgsToCast] -from typing import cast -cast(str, *None) # E: 'cast' must be called with 2 positional arguments -cast(str, target=None) # E: 'cast' must be called with 2 positional arguments -[out] - -[case testInvalidArgsToAny] -from typing import Any -Any(str, None) # E: 'Any' expects 1 argument -Any(arg=str) # E: 'Any' must be called with 1 positional argument -[out] - -[case testTypeListAsType] -def f(x:[int, str]) -> None: # E: Invalid type - pass -[out] -main: In function "f": - -[case testInvalidFunctionType] -from typing import Function -x = None # type: Function[int, str] -y = None # type: Function[int] -z = None # type: Function[int, int, int] -[out] -main, line 2: Invalid function type -main, line 3: Invalid function type -main, line 4: Invalid function type - -[case testAbstractGlobalFunction] -import typing -from abc import abstractmethod -@abstractmethod -def foo(): pass -[out] -main, line 3: 'abstractmethod' used with a non-method - -[case testAbstractNestedFunction] -import typing -from abc import abstractmethod -def g(): - @abstractmethod - def foo(): pass -[out] -main: In function "g": -main, line 4: 'abstractmethod' used with a non-method - -[case testInvalidTypeDeclaration] -import typing -def f(): pass -f() = 1 # type: int -[out] -main, line 3: Invalid assignment target - -[case testIndexedAssignmentWithTypeDeclaration] -import typing -None[1] = 1 # type: int -[out] -main, line 2: Unexpected type declaration - -[case testNonSelfMemberAssignmentWithTypeDeclaration] -import typing -None.x = 1 # type: int -[out] -main, line 2: Type cannot be declared in assignment to non-self attribute - -[case testNonSelfMemberAssignmentWithTypeDeclarationInMethod] -import typing -class A: - def f(self, x) -> None: - x.y = 1 # type: int -[out] -main: In function "f": -main, line 4: Type cannot be declared in assignment to non-self attribute - -[case testInvalidUndefined] -from typing import Undefined -y = 1 -x = Undefined(Undefined) # E: Invalid type "typing.Undefined" -[out] - -[case testInvalidTypeInTypeApplication] -from typing import typevar, Generic -t = typevar('t') -class A(Generic[t]): pass -A[typevar] # E: Invalid type "typing.typevar" -[out] - -[case testInvalidTypeInTypeApplication2] -from typing import typevar, Generic -t = typevar('t') -class A(Generic[t]): pass -A[1] # E: Type expected within [...] -[out] - -[case testVariableDeclWithInvalidNumberOfTypes] -from typing import Undefined -x, y = 1, 2 # type: int, str, int # E: Incompatible number of tuple items -[out] - -[case testVariableDeclWithInvalidNumberOfTypesNested] -from typing import Undefined -x, (y, z) = 1, (2, 3) # type: int, (str, int, int) # E: Incompatible number of tuple items -[out] - -[case testVariableDeclWithInvalidNumberOfTypesNested2] -from typing import Undefined -x, (y, z) = 1, (2, 3) # type: int, (str, ) # E: Incompatible number of tuple items -[out] - -[case testVariableDeclWithInvalidNumberOfTypesNested3] -from typing import Undefined -x, (y, z) = 1, (2, 3) # type: int, str # E: Tuple type expected for multiple variables -[out] - -[case testVariableDeclWithInvalidNumberOfTypesNested4] -from typing import Undefined -x, (y, z) = 1, (2, 3) # type: int, str, int # E: Incompatible number of tuple items -[out] - -[case testVariableDeclWithInvalidNumberOfTypesNested5] -from typing import Undefined -x, (y, ) = 1, (2, ) # type: int, str # E: Tuple type expected for multiple variables -[out] - -[case testVariableDeclWithInvalidType] -from typing import Undefined -x, y = 1, 2 # type: int # E: Tuple type expected for multiple variables -[out] - -[case testInvalidLvalueWithExplicitType] -from typing import Undefined -a = 1 -a[1] = Undefined # type: int # E: Unexpected type declaration -a.x = Undefined # type: int \ - # E: Type cannot be declared in assignment to non-self attribute -a() = Undefined # type: int # E: Invalid assignment target -[out] - -[case testInvalidLvalueWithExplicitType2] -from typing import Undefined -a = 1 -a[1] = Undefined(int) # TODO: Disallow this? -a() = Undefined(int) # E: Invalid assignment target -[out] - -[case testInvalidLvalueWithExplicitType3] -from typing import Undefined -a = 1 -a.y, a.x = Undefined, Undefined # type: int, int \ - # E: Type cannot be declared in assignment to non-self attribute -a[1], a[2] = Undefined, Undefined # type: int, int \ - # E: Unexpected type declaration -[out] - -[case testMissingGenericImport] -from typing import typevar -T = typevar('T') -class A(Generic[T]): pass -[out] -main, line 3: Name 'Generic' is not defined - -[case testMissingAbstractGenericImport] -from typing import typevar, Generic -T = typevar('T') -class A(AbstractGeneric[T]): pass -[out] -main, line 3: Name 'AbstractGeneric' is not defined - -[case testInvalidTypeWithinGeneric] -from typing import Generic -class A(Generic[int]): pass # E: Free type variable expected in Generic[...] -[out] - -[case testInvalidTypeWithinAbstractGeneric] -from typing import AbstractGeneric, typevar -T = typevar('T') -class A(AbstractGeneric[T]): - class B(AbstractGeneric[T]): pass \ - # E: Free type variable expected in AbstractGeneric[...] -[out] - -[case testIncludingGenericTwiceInBaseClassList] -from typing import Generic, AbstractGeneric, typevar -T = typevar('T') -S = typevar('S') -class A(Generic[T], Generic[S]): pass \ - # E: Duplicate Generic or AbstractGeneric in bases -class B(Generic[T], AbstractGeneric[T]): pass \ - # E: Duplicate Generic or AbstractGeneric in bases -[out] - -[case testInvalidMetaclass] -class A(metaclass=x): pass # E: Name 'x' is not defined -[out] - -[case testInvalidQualifiedMetaclass] -import abc -class A(metaclass=abc.Foo): pass # E: Name 'abc.Foo' is not defined -[out] - -[case testNonClassMetaclass] -def f(): pass -class A(metaclass=f): pass # E: Invalid metaclass 'f' -[out] - -[case testInvalidTypevarArguments] -from typing import typevar -a = typevar() # E: Too few arguments for typevar() -b = typevar(x='b') # E: Unexpected arguments to typevar() -c = typevar(1) # E: typevar() expects a string literal argument -d = typevar('D') # E: Unexpected typevar() argument value -e = typevar('e', values=(int, str), x=1) # E: Too many arguments for typevar() -f = typevar('f', (int, str)) # E: Unexpected arguments to typevar() -g = typevar('g', x=(int, str)) # E: Unexpected keyword argument 'x' to typevar() -[out] - -[case testInvalidTypevarValues] -from typing import typevar -a = typevar('a', values=int) # E: The values argument must be in parentheses (...) -b = typevar('b', values=(int)) # E: The values argument must be a tuple literal -c = typevar('c', values=(int, 2)) # E: Type expected -[out] - -[case testLocalTypevarScope] -from typing import typevar -def f() -> None: - T = typevar('T') -def g(x: T) -> None: pass # E: Name 'T' is not defined -[out] -main: In function "g": - -[case testClassTypevarScope] -from typing import typevar -class A: - T = typevar('T') -def g(x: T) -> None: pass # E: Name 'T' is not defined -[out] -main: In function "g": - -[case testRedefineVariableAsTypevar] -from typing import typevar -x = 0 -x = typevar('x') # E: Cannot redefine 'x' as a type variable -[out] - -[case testTypevarWithType] -from typing import typevar -x = typevar('x') # type: int # E: Cannot declare the type of a type variable -[out] - -[case testRedefineTypevar] -from typing import typevar -t = typevar('t') -t = 1 # E: Invalid assignment target -[out] - -[case testRedefineTypevar2] -from typing import typevar -t = typevar('t') -def t(): pass # E: Name 't' already defined -[out] - -[case testRedefineTypevar3] -from typing import typevar -t = typevar('t') -class t: pass # E: Name 't' already defined -[out] - -[case testRedefineTypevar4] -from typing import typevar -t = typevar('t') -from typing import Generic as t # E: Name 't' already defined -[out] - -[case testInvalidStrLiteralType] -from typing import Undefined -x = Undefined('foo') # E: Name 'foo' is not defined -[out] - -[case testInvalidStrLiteralType2] -from typing import Undefined -x = Undefined('int[') # E: Argument to Undefined is not a type -[out] - -[case testInconsistentOverload] -from typing import overload -def dec(x): pass -@overload -def f(): pass -@dec # E: 'overload' decorator expected -def f(): pass -[out] - -[case testInconsistentOverload2] -from typing import overload -def dec(x): pass -@dec # E: 'overload' decorator expected -def f(): pass -@overload -def f(): pass -[out] - -[case testMissingOverloadDecorator] -from typing import overload -def dec(x): pass -@dec # E: 'overload' decorator expected -def f(): pass -@dec # E: 'overload' decorator expected -def f(): pass -[out] - -[case testIncompatibleSignatureInComment] -import typing -def f(): # type: (int) -> int - pass -def g(x): # type: () -> int - pass -[out] -main: In function "f": -main, line 2: Type signature has too many arguments -main: In function "g": -main, line 4: Type signature has too few arguments - -[case testStaticmethodAndNonMethod] -import typing -@staticmethod -def f(): pass -class A: - def g(self): - @staticmethod - def h(): pass -[builtins fixtures/staticmethod.py] -[out] -main, line 2: 'staticmethod' used with a non-method -main: In function "g": -main, line 6: 'staticmethod' used with a non-method - -[case testClassmethodAndNonMethod] -import typing -@classmethod -def f(): pass -class A: - def g(self): - @classmethod - def h(): pass -[builtins fixtures/classmethod.py] -[out] -main, line 2: 'classmethod' used with a non-method -main: In function "g": -main, line 6: 'classmethod' used with a non-method - -[case testNonMethodProperty] -import typing -@property # E: 'property' used with a non-method -def f() -> int: pass -[builtins fixtures/property.py] -[out] - -[case testInvalidArgCountForProperty] -import typing -class A: - @property - def f(self, x) -> int: pass # E: Too many arguments - @property - def g() -> int: pass # E: Method must have at least one argument -[builtins fixtures/property.py] -[out] - -[case testOverloadedProperty] -from typing import overload -class A: - @overload # E: A property cannot be overloaded - @property - def f(self) -> int: pass - @property # E: A property cannot be overloaded - @overload - def f(self) -> int: pass -[builtins fixtures/property.py] -[out] - -[case testImportTwoModulesWithSameNameInFunction] -import typing -def f() -> None: - import x - import y as x # E: Name 'x' already defined - x.y -[file x.py] -y = 1 -[file y.py] -[out] -main: In function "f": - -[case testImportTwoModulesWithSameNameInGlobalContext] -import typing -import x -import y as x # E: Name 'x' already defined -x.y -[file x.py] -y = 1 -[file y.py] -[out] - -[case testListTypeAliasWithoutImport] -import typing -def f() -> List[int]: pass -[builtins fixtures/list.py] -[out] -main: In function "f": -main, line 2: Name 'List' is not defined - -[case testNamedTupleWithTooFewArguments] -from collections import namedtuple -N = namedtuple('N') # E: Too few arguments for namedtuple() - -[case testNamedTupleWithTooManyArguments] -from collections import namedtuple -N = namedtuple('N', ['x'], 'y') # E: Too many arguments for namedtuple() - -[case testNamedTupleWithInvalidName] -from collections import namedtuple -N = namedtuple(1, ['x']) # E: namedtuple() expects a string literal as the first argument - -[case testNamedTupleWithInvalidItems] -from collections import namedtuple -N = namedtuple('N', 1) # E: List literal expected as the second argument to namedtuple() - -[case testNamedTupleWithInvalidItems2] -from collections import namedtuple -N = namedtuple('N', ['x', 1]) # E: String literal expected as namedtuple() item - -[case testNamedTupleWithInvalidArgs] -from collections import namedtuple -N = namedtuple('N', x=['x']) # E: Unexpected arguments to namedtuple() diff --git a/mypy/test/data/semanal-modules.test b/mypy/test/data/semanal-modules.test deleted file mode 100644 index 9c6767389297..000000000000 --- a/mypy/test/data/semanal-modules.test +++ /dev/null @@ -1,805 +0,0 @@ --- NOTE: If a module has a name starting or ending with _, it is skipped in --- output. - -[case testImport] -import x -x.y -[file x.py] -y = 1 -[out] -MypyFile:1( - Import:1(x : x) - ExpressionStmt:2( - MemberExpr:2( - NameExpr(x) - y [x.y]))) -MypyFile:1( - tmp/x.py - AssignmentStmt:1( - NameExpr(y* [x.y]) - IntExpr(1))) - -[case testImportedNameInType] -from typing import Undefined -import m -x = Undefined # type: m.c -[file m.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - Import:2(m : m) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - m.c)) -MypyFile:1( - tmp/m.py - ClassDef:1( - c - PassStmt:1())) - -[case testImportFrom] -from m import y -x = y -[file m.py] -y = 1 -[out] -MypyFile:1( - ImportFrom:1(m, [y : y]) - AssignmentStmt:2( - NameExpr(x* [__main__.x]) - NameExpr(y [m.y]))) -MypyFile:1( - tmp/m.py - AssignmentStmt:1( - NameExpr(y* [m.y]) - IntExpr(1))) - -[case testImportFromType] -from typing import Undefined -from m import c -x = Undefined # type: c -[file m.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ImportFrom:2(m, [c : c]) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - m.c)) -MypyFile:1( - tmp/m.py - ClassDef:1( - c - PassStmt:1())) - -[case testImportMultiple] -import _m, _n -_m.x, _n.y -[file _m.py] -x = 1 -[file _n.py] -y = 2 -[out] -MypyFile:1( - Import:1(_m : _m, _n : _n) - ExpressionStmt:2( - TupleExpr:2( - MemberExpr:2( - NameExpr(_m) - x [_m.x]) - MemberExpr:2( - NameExpr(_n) - y [_n.y])))) - -[case testImportAs] -import _m as n -n.x -[file _m.py] -x = 1 -[out] -MypyFile:1( - Import:1(_m : n) - ExpressionStmt:2( - MemberExpr:2( - NameExpr(n [_m]) - x [_m.x]))) - -[case testImportFromMultiple] -from _m import x, y -x, y -[file _m.py] -x = y = 1 -[out] -MypyFile:1( - ImportFrom:1(_m, [x : x, y : y]) - ExpressionStmt:2( - TupleExpr:2( - NameExpr(x [_m.x]) - NameExpr(y [_m.y])))) - -[case testImportFromAs] -from _m import y as z -z -[file _m.py] -y = 1 -[out] -MypyFile:1( - ImportFrom:1(_m, [y : z]) - ExpressionStmt:2( - NameExpr(z [_m.y]))) - -[case testAccessImportedName] -from m import x -y = x -[file m.py] -from _n import x -[file _n.py] -x = 1 -[out] -MypyFile:1( - ImportFrom:1(m, [x : x]) - AssignmentStmt:2( - NameExpr(y* [__main__.y]) - NameExpr(x [_n.x]))) -MypyFile:1( - tmp/m.py - ImportFrom:1(_n, [x : x])) - -[case testAccessImportedName2] -import _m -y = _m.x -[file _m.py] -from _n import x -[file _n.py] -x = 1 -[out] -MypyFile:1( - Import:1(_m : _m) - AssignmentStmt:2( - NameExpr(y* [__main__.y]) - MemberExpr:2( - NameExpr(_m) - x [_n.x]))) - -[case testAccessingImportedNameInType] -from typing import Undefined -from _m import c -x = Undefined # type: c -[file _m.py] -from _n import c -[file _n.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ImportFrom:2(_m, [c : c]) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - _n.c)) - -[case testAccessingImportedNameInType2] -from typing import Undefined -import _m -x = Undefined # type: _m.c -[file _m.py] -from _n import c -[file _n.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - Import:2(_m : _m) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - _n.c)) - -[case testAccessingImportedModule] -from _m import _n -_n.x -[file _m.py] -import _n -[file _n.py] -x = 1 -[out] -MypyFile:1( - ImportFrom:1(_m, [_n : _n]) - ExpressionStmt:2( - MemberExpr:2( - NameExpr(_n) - x [_n.x]))) - -[case testAccessingImportedModule] -import _m -_m._n.x -[file _m.py] -import _n -[file _n.py] -x = 1 -[out] -MypyFile:1( - Import:1(_m : _m) - ExpressionStmt:2( - MemberExpr:2( - MemberExpr:2( - NameExpr(_m) - _n) - x [_n.x]))) - -[case testAccessTypeViaDoubleIndirection] -from typing import Undefined -from _m import c -a = Undefined # type: c -[file _m.py] -from _n import c -[file _n.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ImportFrom:2(_m, [c : c]) - AssignmentStmt:3( - NameExpr(a [__main__.a]) - NameExpr(Undefined [typing.Undefined]) - _n.c)) - -[case testAccessTypeViaDoubleIndirection2] -from typing import Undefined -import _m -a = Undefined # type: _m.c -[file _m.py] -from _n import c -[file _n.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - Import:2(_m : _m) - AssignmentStmt:3( - NameExpr(a [__main__.a]) - NameExpr(Undefined [typing.Undefined]) - _n.c)) - -[case testImportAsterisk] -from _m import * -x, y -[file _m.py] -x = y = 1 -[out] -MypyFile:1( - ImportAll:1(_m) - ExpressionStmt:2( - TupleExpr:2( - NameExpr(x [_m.x]) - NameExpr(y [_m.y])))) - -[case testImportAsteriskAndImportedNames] -from _m import * -n_.x, y -[file _m.py] -import n_ -from n_ import y -[file n_.py] -x = y = 1 -[out] -MypyFile:1( - ImportAll:1(_m) - ExpressionStmt:2( - TupleExpr:2( - MemberExpr:2( - NameExpr(n_) - x [n_.x]) - NameExpr(y [n_.y])))) - -[case testImportAsteriskAndImportedNamesInTypes] -from typing import Undefined -from _m import * -x = Undefined # type: n_.c -y = Undefined # type: d -[file _m.py] -import n_ -from n_ import d -[file n_.py] -class c: pass -class d: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ImportAll:2(_m) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - n_.c) - AssignmentStmt:4( - NameExpr(y [__main__.y]) - NameExpr(Undefined [typing.Undefined]) - n_.d)) - -[case testModuleInSubdir] -import _m -_m.x -[file _m/__init__.py] -x = 1 -[out] -MypyFile:1( - Import:1(_m : _m) - ExpressionStmt:2( - MemberExpr:2( - NameExpr(_m) - x [_m.x]))) - -[case testNestedModules] -import m.n -m.n.x, m.y -[file m/__init__.py] -y = 1 -[file m/n.py] -x = 1 -[out] -MypyFile:1( - Import:1(m.n : m.n) - ExpressionStmt:2( - TupleExpr:2( - MemberExpr:2( - MemberExpr:2( - NameExpr(m) - n [m.n]) - x [m.n.x]) - MemberExpr:2( - NameExpr(m) - y [m.y])))) -MypyFile:1( - tmp/m/n.py - AssignmentStmt:1( - NameExpr(x* [m.n.x]) - IntExpr(1))) - -[case testImportFromSubmodule] -from m._n import x -x -[file m/__init__.py] -[file m/_n.py] -x = 1 -[out] -MypyFile:1( - ImportFrom:1(m._n, [x : x]) - ExpressionStmt:2( - NameExpr(x [m._n.x]))) - -[case testImportAllFromSubmodule] -from m._n import * -x, y -[file m/__init__.py] -[file m/_n.py] -x = y = 1 -[out] -MypyFile:1( - ImportAll:1(m._n) - ExpressionStmt:2( - TupleExpr:2( - NameExpr(x [m._n.x]) - NameExpr(y [m._n.y])))) - -[case testSubmodulesAndTypes] -from typing import Undefined -import m._n -x = Undefined # type: m._n.c -[file m/__init__.py] -[file m/_n.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - Import:2(m._n : m._n) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - m._n.c)) - -[case testSubmodulesAndTypes] -from typing import Undefined -from m._n import c -x = Undefined # type: c -[file m/__init__.py] -[file m/_n.py] -class c: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ImportFrom:2(m._n, [c : c]) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - m._n.c)) - -[case testFromPackageImportModule] -from m import _n -_n.x -[file m/__init__.py] -[file m/_n.py] -x = 1 -[out] -MypyFile:1( - ImportFrom:1(m, [_n : _n]) - ExpressionStmt:2( - MemberExpr:2( - NameExpr(_n [m._n]) - x [m._n.x]))) - -[case testDeeplyNestedModule] -import m.n.k -m.n.k.x -m.n.b -m.a -[file m/__init__.py] -a = 1 -[file m/n/__init__.py] -b = 1 -[file m/n/k.py] -x = 1 -[out] -MypyFile:1( - Import:1(m.n.k : m.n.k) - ExpressionStmt:2( - MemberExpr:2( - MemberExpr:2( - MemberExpr:2( - NameExpr(m) - n [m.n]) - k [m.n.k]) - x [m.n.k.x])) - ExpressionStmt:3( - MemberExpr:3( - MemberExpr:3( - NameExpr(m) - n [m.n]) - b [m.n.b])) - ExpressionStmt:4( - MemberExpr:4( - NameExpr(m) - a [m.a]))) -MypyFile:1( - tmp/m/n/k.py - AssignmentStmt:1( - NameExpr(x* [m.n.k.x]) - IntExpr(1))) - -[case testImportInSubmodule] -import m._n -y = m._n.x -[file m/__init__.py] -[file m/_n.py] -from m._k import x -[file m/_k.py] -x = 1 -[out] -MypyFile:1( - Import:1(m._n : m._n) - AssignmentStmt:2( - NameExpr(y* [__main__.y]) - MemberExpr:2( - MemberExpr:2( - NameExpr(m) - _n [m._n]) - x [m._k.x]))) - -[case testBuiltinsUsingModule] -from typing import Undefined -o = Undefined # type: __builtins__.object -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - AssignmentStmt:2( - NameExpr(o [__main__.o]) - NameExpr(Undefined [typing.Undefined]) - builtins.object)) - -[case testImplicitAccessToBuiltins] -object -[out] -MypyFile:1( - ExpressionStmt:1( - NameExpr(object [builtins.object]))) - -[case testAssignmentToModuleAttribute] -import _m -_m.x = ( - _m.x) -[file _m.py] -x = None -[out] -MypyFile:1( - Import:1(_m : _m) - AssignmentStmt:2( - MemberExpr:2( - NameExpr(_m) - x [_m.x]) - ParenExpr:2( - MemberExpr:3( - NameExpr(_m) - x [_m.x])))) - -[case testAssignmentThatRefersToModule] -import _m -_m.x[None] = None -[file _m.py] -x = None -[out] -MypyFile:1( - Import:1(_m : _m) - AssignmentStmt:2( - IndexExpr:2( - MemberExpr:2( - NameExpr(_m) - x [_m.x]) - NameExpr(None [builtins.None])) - NameExpr(None [builtins.None]))) - -[case testImportInBlock] -if 1: - import _x - _x.y -[file _x.py] -y = 1 -[out] -MypyFile:1( - IfStmt:1( - If( - IntExpr(1)) - Then( - Import:2(_x : _x) - ExpressionStmt:3( - MemberExpr:3( - NameExpr(_x) - y [_x.y]))))) - -[case testImportInFunction] -def f() -> None: - import _x - _x.y -[file _x.py] -y = 1 -[out] -MypyFile:1( - FuncDef:1( - f - def () - Block:1( - Import:2(_x : _x) - ExpressionStmt:3( - MemberExpr:3( - NameExpr(_x) - y [_x.y]))))) - -[case testImportInClassBody] -class A: - from _x import y - z = y -[file _x.py] -y = 1 -[out] -MypyFile:1( - ClassDef:1( - A - ImportFrom:2(_x, [y : y]) - AssignmentStmt:3( - NameExpr(z* [m]) - NameExpr(y [_x.y])))) - -[case testImportInClassBody2] -class A: - import _x - z = _x.y -[file _x.py] -y = 1 -[out] -MypyFile:1( - ClassDef:1( - A - Import:2(_x : _x) - AssignmentStmt:3( - NameExpr(z* [m]) - MemberExpr:3( - NameExpr(_x) - y [_x.y])))) - -[case testImportModuleTwice] -def f() -> None: - import x - import x - x.y -[file x.py] -y = 1 -[out] -MypyFile:1( - FuncDef:1( - f - def () - Block:1( - Import:2(x : x) - Import:3(x : x) - ExpressionStmt:4( - MemberExpr:4( - NameExpr(x) - y [x.y]))))) -MypyFile:1( - tmp/x.py - AssignmentStmt:1( - NameExpr(y* [x.y]) - IntExpr(1))) - -[case testRelativeImport0] -import m.x -m.x.z.y -[file m/__init__.py] -[file m/x.py] -from . import z -[file m/z.py] -y = 1 -[out] -MypyFile:1( - Import:1(m.x : m.x) - ExpressionStmt:2( - MemberExpr:2( - MemberExpr:2( - MemberExpr:2( - NameExpr(m) - x [m.x]) - z [m.z]) - y [m.z.y]))) -MypyFile:1( - tmp/m/x.py - ImportFrom:1(., [z : z])) -MypyFile:1( - tmp/m/z.py - AssignmentStmt:1( - NameExpr(y* [m.z.y]) - IntExpr(1))) - -[case testRelativeImport1] -import m.t.b as b -b.x.y -b.z.y -[file m/__init__.py] -[file m/x.py] -y = 1 -[file m/z.py] -y = 3 -[file m/t/__init__.py] -[file m/t/b.py] -from .. import x, z -[out] -MypyFile:1( - Import:1(m.t.b : b) - ExpressionStmt:2( - MemberExpr:2( - MemberExpr:2( - NameExpr(b [m.t.b]) - x [m.x]) - y [m.x.y])) - ExpressionStmt:3( - MemberExpr:3( - MemberExpr:3( - NameExpr(b [m.t.b]) - z [m.z]) - y [m.z.y]))) -MypyFile:1( - tmp/m/t/b.py - ImportFrom:1(.., [x : x, z : z])) -MypyFile:1( - tmp/m/x.py - AssignmentStmt:1( - NameExpr(y* [m.x.y]) - IntExpr(1))) -MypyFile:1( - tmp/m/z.py - AssignmentStmt:1( - NameExpr(y* [m.z.y]) - IntExpr(3))) - -[case testRelativeImport2] -import m.t.b as b -b.xy -b.zy -[file m/__init__.py] -[file m/x.py] -y = 1 -[file m/z.py] -y = 3 -[file m/t/__init__.py] -[file m/t/b.py] -from ..x import y as xy -from ..z import y as zy -[out] -MypyFile:1( - Import:1(m.t.b : b) - ExpressionStmt:2( - MemberExpr:2( - NameExpr(b [m.t.b]) - xy [m.x.y])) - ExpressionStmt:3( - MemberExpr:3( - NameExpr(b [m.t.b]) - zy [m.z.y]))) -MypyFile:1( - tmp/m/t/b.py - ImportFrom:1(..x, [y : xy]) - ImportFrom:2(..z, [y : zy])) -MypyFile:1( - tmp/m/x.py - AssignmentStmt:1( - NameExpr(y* [m.x.y]) - IntExpr(1))) -MypyFile:1( - tmp/m/z.py - AssignmentStmt:1( - NameExpr(y* [m.z.y]) - IntExpr(3))) - -[case testRelativeImport3] -import m.t -m.zy -m.xy -m.t.y -[file m/__init__.py] -from .x import * -from .z import * -[file m/x.py] -from .z import zy as xy -[file m/z.py] -zy = 3 -[file m/t/__init__.py] -from .b import * -[file m/t/b.py] -from .. import xy as y -[out] -MypyFile:1( - Import:1(m.t : m.t) - ExpressionStmt:2( - MemberExpr:2( - NameExpr(m) - zy [m.z.zy])) - ExpressionStmt:3( - MemberExpr:3( - NameExpr(m) - xy [m.z.zy])) - ExpressionStmt:4( - MemberExpr:4( - MemberExpr:4( - NameExpr(m) - t [m.t]) - y [m.z.zy]))) -MypyFile:1( - tmp/m/t/b.py - ImportFrom:1(.., [xy : y])) -MypyFile:1( - tmp/m/x.py - ImportFrom:1(.z, [zy : xy])) -MypyFile:1( - tmp/m/z.py - AssignmentStmt:1( - NameExpr(zy* [m.z.zy]) - IntExpr(3))) - -[case testRelativeImportFromSameModule] -import m.x -[file m/__init__.py] -[file m/x.py] -from .x import nonexistent -[out] -In module imported in main, line 1: -tmp/m/x.py, line 1: Module has no attribute 'nonexistent' - -[case testImportFromSameModule] -import m.x -[file m/__init__.py] -[file m/x.py] -from m.x import nonexistent -[out] -In module imported in main, line 1: -tmp/m/x.py, line 1: Module has no attribute 'nonexistent' diff --git a/mypy/test/data/semanal-namedtuple.test b/mypy/test/data/semanal-namedtuple.test deleted file mode 100644 index 3cc39e006f28..000000000000 --- a/mypy/test/data/semanal-namedtuple.test +++ /dev/null @@ -1,42 +0,0 @@ -[case testSimpleNamedtuple] -from collections import namedtuple -N = namedtuple('N', ['a']) -def f() -> N: pass -[out] -MypyFile:1( - ImportFrom:1(collections, [namedtuple : namedtuple]) - AssignmentStmt:2( - NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any])) - FuncDef:3( - f - def () -> Tuple[Any, fallback=__main__.N] - Block:3( - PassStmt:3()))) - -[case testTwoItemNamedtuple] -from collections import namedtuple -N = namedtuple('N', ['a', 'xyz']) -def f() -> N: pass -[out] -MypyFile:1( - ImportFrom:1(collections, [namedtuple : namedtuple]) - AssignmentStmt:2( - NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any, Any])) - FuncDef:3( - f - def () -> Tuple[Any, Any, fallback=__main__.N] - Block:3( - PassStmt:3()))) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -[out] -MypyFile:1( - ImportFrom:1(typing, [NamedTuple : NamedTuple]) - AssignmentStmt:2( - NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[builtins.int, builtins.str]))) diff --git a/mypy/test/data/semanal-python2.test b/mypy/test/data/semanal-python2.test deleted file mode 100644 index 2780c4087887..000000000000 --- a/mypy/test/data/semanal-python2.test +++ /dev/null @@ -1,10 +0,0 @@ --- Python 2 semantic analysis test cases. - -[case testPrintStatement] -print int, None -[out] -MypyFile:1( - PrintStmt:1( - NameExpr(int [builtins.int]) - NameExpr(None [builtins.None]) - Newline)) diff --git a/mypy/test/data/semanal-statements.test b/mypy/test/data/semanal-statements.test deleted file mode 100644 index 26555adf3596..000000000000 --- a/mypy/test/data/semanal-statements.test +++ /dev/null @@ -1,869 +0,0 @@ -[case testReturn] -def f(x): return x -def g(): return -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - Block:1( - ReturnStmt:1( - NameExpr(x [l])))) - FuncDef:2( - g - Block:2( - ReturnStmt:2()))) - -[case testRaise] -raise object() -[out] -MypyFile:1( - RaiseStmt:1( - CallExpr:1( - NameExpr(object [builtins.object]) - Args()))) - -[case testYield] -def f(): yield f -[out] -MypyFile:1( - FuncDef:1( - f - Generator - Block:1( - YieldStmt:1( - NameExpr(f [__main__.f]))))) - -[case testAssert] -assert object -[out] -MypyFile:1( - AssertStmt:1( - NameExpr(object [builtins.object]))) - -[case testOperatorAssignment] -x = y = 1 -x += y -y |= x -[out] -MypyFile:1( - AssignmentStmt:1( - Lvalues( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - IntExpr(1)) - OperatorAssignmentStmt:2( - + - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])) - OperatorAssignmentStmt:3( - | - NameExpr(y [__main__.y]) - NameExpr(x [__main__.x]))) - -[case testWhile] -x = y = 1 -while x: - y -[out] -MypyFile:1( - AssignmentStmt:1( - Lvalues( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - IntExpr(1)) - WhileStmt:2( - NameExpr(x [__main__.x]) - Block:2( - ExpressionStmt:3( - NameExpr(y [__main__.y]))))) - -[case testFor] -for x in object: - x -[out] -MypyFile:1( - ForStmt:1( - NameExpr(x* [__main__.x]) - NameExpr(object [builtins.object]) - Block:1( - ExpressionStmt:2( - NameExpr(x [__main__.x]))))) - -[case testForInFunction] -def f(): - for x in f: - x -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - ForStmt:2( - NameExpr(x* [l]) - NameExpr(f [__main__.f]) - Block:2( - ExpressionStmt:3( - NameExpr(x [l]))))))) - -[case testMultipleForIndexVars] -for x, y in []: - x, y -[out] -MypyFile:1( - ForStmt:1( - TupleExpr:1( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - ListExpr:1() - Block:1( - ExpressionStmt:2( - TupleExpr:2( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])))))) - -[case testForIndexVarScope] -for x in []: - pass -x -[out] -MypyFile:1( - ForStmt:1( - NameExpr(x* [__main__.x]) - ListExpr:1() - Block:1( - PassStmt:2())) - ExpressionStmt:3( - NameExpr(x [__main__.x]))) - -[case testForIndexVarScope2] -def f(): - for x in []: - pass - x -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - ForStmt:2( - NameExpr(x* [l]) - ListExpr:2() - Block:2( - PassStmt:3())) - ExpressionStmt:4( - NameExpr(x [l]))))) - -[case testReusingForLoopIndexVariable] -for x in None: - pass -for x in None: - pass -[out] -MypyFile:1( - ForStmt:1( - NameExpr(x* [__main__.x]) - NameExpr(None [builtins.None]) - Block:1( - PassStmt:2())) - ForStmt:3( - NameExpr(x [__main__.x]) - NameExpr(None [builtins.None]) - Block:3( - PassStmt:4()))) - -[case testReusingForLoopIndexVariable2] -def f(): - for x in None: - pass - for x in None: - pass -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - ForStmt:2( - NameExpr(x* [l]) - NameExpr(None [builtins.None]) - Block:2( - PassStmt:3())) - ForStmt:4( - NameExpr(x [l]) - NameExpr(None [builtins.None]) - Block:4( - PassStmt:5()))))) - -[case testLoopWithElse] -for x in []: - pass -else: - x -while 1: - pass -else: - x -[out] -MypyFile:1( - ForStmt:1( - NameExpr(x* [__main__.x]) - ListExpr:1() - Block:1( - PassStmt:2()) - Else( - ExpressionStmt:4( - NameExpr(x [__main__.x])))) - WhileStmt:5( - IntExpr(1) - Block:5( - PassStmt:6()) - Else( - ExpressionStmt:8( - NameExpr(x [__main__.x]))))) - -[case testBreak] -while 1: - break -for x in []: - break -[out] -MypyFile:1( - WhileStmt:1( - IntExpr(1) - Block:1( - BreakStmt:2())) - ForStmt:3( - NameExpr(x* [__main__.x]) - ListExpr:3() - Block:3( - BreakStmt:4()))) - -[case testContinue] -while 1: - continue -for x in []: - continue -[out] -MypyFile:1( - WhileStmt:1( - IntExpr(1) - Block:1( - ContinueStmt:2())) - ForStmt:3( - NameExpr(x* [__main__.x]) - ListExpr:3() - Block:3( - ContinueStmt:4()))) - -[case testIf] -x = 1 -if x: - x -elif x: - x -elif x: - x -else: - x -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - IfStmt:2( - If( - NameExpr(x [__main__.x])) - Then( - ExpressionStmt:3( - NameExpr(x [__main__.x]))) - If( - NameExpr(x [__main__.x])) - Then( - ExpressionStmt:5( - NameExpr(x [__main__.x]))) - If( - NameExpr(x [__main__.x])) - Then( - ExpressionStmt:7( - NameExpr(x [__main__.x]))) - Else( - ExpressionStmt:9( - NameExpr(x [__main__.x]))))) - -[case testSimpleIf] -if object: - object -[out] -MypyFile:1( - IfStmt:1( - If( - NameExpr(object [builtins.object])) - Then( - ExpressionStmt:2( - NameExpr(object [builtins.object]))))) - -[case testLvalues] -x = y = 1 -x = 1 -x.m = 1 -x[y] = 1 -x, y = 1 -[x, y] = 1 -(x, y) = 1 -[out] -MypyFile:1( - AssignmentStmt:1( - Lvalues( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - IntExpr(1)) - AssignmentStmt:2( - NameExpr(x [__main__.x]) - IntExpr(1)) - AssignmentStmt:3( - MemberExpr:3( - NameExpr(x [__main__.x]) - m) - IntExpr(1)) - AssignmentStmt:4( - IndexExpr:4( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])) - IntExpr(1)) - AssignmentStmt:5( - TupleExpr:5( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])) - IntExpr(1)) - AssignmentStmt:6( - ListExpr:6( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])) - IntExpr(1)) - AssignmentStmt:7( - ParenExpr:7( - TupleExpr:7( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y]))) - IntExpr(1))) - -[case testStarLvalues] -*x, y = 1 -*x, (y, *z) = 1 -*(x, q), r = 1 -[out] -MypyFile:1( - AssignmentStmt:1( - TupleExpr:1( - StarExpr:1( - NameExpr(x* [__main__.x])) - NameExpr(y* [__main__.y])) - IntExpr(1)) - AssignmentStmt:2( - TupleExpr:2( - StarExpr:2( - NameExpr(x [__main__.x])) - ParenExpr:2( - TupleExpr:2( - NameExpr(y [__main__.y]) - StarExpr:2( - NameExpr(z* [__main__.z]))))) - IntExpr(1)) - AssignmentStmt:3( - TupleExpr:3( - StarExpr:3( - ParenExpr:3( - TupleExpr:3( - NameExpr(x [__main__.x]) - NameExpr(q* [__main__.q])))) - NameExpr(r* [__main__.r])) - IntExpr(1))) - -[case testMultipleDefinition] -x, y = 1 -x, y = 2 -[out] -MypyFile:1( - AssignmentStmt:1( - TupleExpr:1( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - IntExpr(1)) - AssignmentStmt:2( - TupleExpr:2( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])) - IntExpr(2))) - -[case testComplexDefinitions] -(x) = 1 -([y]) = 2 -[out] -MypyFile:1( - AssignmentStmt:1( - ParenExpr:1( - NameExpr(x* [__main__.x])) - IntExpr(1)) - AssignmentStmt:2( - ParenExpr:2( - ListExpr:2( - NameExpr(y* [__main__.y]))) - IntExpr(2))) - -[case testLocalComplexDefinition] -def f(): - (x) = 1 - x -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - AssignmentStmt:2( - ParenExpr:2( - NameExpr(x* [l])) - IntExpr(1)) - ExpressionStmt:3( - NameExpr(x [l]))))) - -[case testMultipleDefOnlySomeNew] -x = 1 -y, x = 1 -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - AssignmentStmt:2( - TupleExpr:2( - NameExpr(y* [__main__.y]) - NameExpr(x [__main__.x])) - IntExpr(1))) - -[case testMultipleDefOnlySomeNewNestedTuples] -x = 1 -y, (x, z) = 1 -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - AssignmentStmt:2( - TupleExpr:2( - NameExpr(y* [__main__.y]) - ParenExpr:2( - TupleExpr:2( - NameExpr(x [__main__.x]) - NameExpr(z* [__main__.z])))) - IntExpr(1))) - -[case testMultipleDefOnlySomeNewNestedLists] -x = 1 -y, [x, z] = 1 -[p, [x, r]] = 1 -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - AssignmentStmt:2( - TupleExpr:2( - NameExpr(y* [__main__.y]) - ListExpr:2( - NameExpr(x [__main__.x]) - NameExpr(z* [__main__.z]))) - IntExpr(1)) - AssignmentStmt:3( - ListExpr:3( - NameExpr(p* [__main__.p]) - ListExpr:3( - NameExpr(x [__main__.x]) - NameExpr(r* [__main__.r]))) - IntExpr(1))) - -[case testIndexedDel] -x = y = 1 -del x[y] -[out] -MypyFile:1( - AssignmentStmt:1( - Lvalues( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y])) - IntExpr(1)) - DelStmt:2( - IndexExpr:2( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])))) - -[case testDelGlobalName] -x = 1 -del x -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) - DelStmt:2( - NameExpr(x [__main__.x]))) - -[case testLocalName] -def f(x): - del x -[out] -MypyFile:1( - FuncDef:1( - f - Args( - Var(x)) - Block:1( - DelStmt:2( - NameExpr(x [l]))))) - -[case testTry] -class c: pass -try: - c -except object: - c -except c as e: - e -except: - c -finally: - c -[out] -MypyFile:1( - ClassDef:1( - c - PassStmt:1()) - TryStmt:2( - Block:2( - ExpressionStmt:3( - NameExpr(c [__main__.c]))) - NameExpr(object [builtins.object]) - Block:4( - ExpressionStmt:5( - NameExpr(c [__main__.c]))) - NameExpr(c [__main__.c]) - NameExpr(e* [__main__.e]) - Block:6( - ExpressionStmt:7( - NameExpr(e [__main__.e]))) - Block:8( - ExpressionStmt:9( - NameExpr(c [__main__.c]))) - Finally( - ExpressionStmt:11( - NameExpr(c [__main__.c]))))) - -[case testTryElse] -try: - pass -except: - pass -else: - object -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - Block:3( - PassStmt:4()) - Else( - ExpressionStmt:6( - NameExpr(object [builtins.object]))))) - -[case testTryWithOnlyFinally] -try: - pass -finally: - pass -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - Finally( - PassStmt:4()))) - -[case testExceptWithMultipleTypes] -class c: pass -try: - pass -except (c, object) as e: - e -[out] -MypyFile:1( - ClassDef:1( - c - PassStmt:1()) - TryStmt:2( - Block:2( - PassStmt:3()) - ParenExpr:4( - TupleExpr:4( - NameExpr(c [__main__.c]) - NameExpr(object [builtins.object]))) - NameExpr(e* [__main__.e]) - Block:4( - ExpressionStmt:5( - NameExpr(e [__main__.e]))))) - -[case testRaiseWithoutExpr] -raise -[out] -MypyFile:1( - RaiseStmt:1()) - -[case testWith] -with object: - object -[out] -MypyFile:1( - WithStmt:1( - Expr( - NameExpr(object [builtins.object])) - Block:1( - ExpressionStmt:2( - NameExpr(object [builtins.object]))))) - -[case testWithAndVariable] -with object as x: - x -[out] -MypyFile:1( - WithStmt:1( - Expr( - NameExpr(object [builtins.object])) - Name( - NameExpr(x* [__main__.x])) - Block:1( - ExpressionStmt:2( - NameExpr(x [__main__.x]))))) - -[case testWithInFunction] -def f(): - with f as x: - x -[out] -MypyFile:1( - FuncDef:1( - f - Block:1( - WithStmt:2( - Expr( - NameExpr(f [__main__.f])) - Name( - NameExpr(x* [l])) - Block:2( - ExpressionStmt:3( - NameExpr(x [l]))))))) - -[case testComplexWith] -with object, object: - pass -with object as a, object as b: - pass -[out] -MypyFile:1( - WithStmt:1( - Expr( - NameExpr(object [builtins.object])) - Expr( - NameExpr(object [builtins.object])) - Block:1( - PassStmt:2())) - WithStmt:3( - Expr( - NameExpr(object [builtins.object])) - Name( - NameExpr(a* [__main__.a])) - Expr( - NameExpr(object [builtins.object])) - Name( - NameExpr(b* [__main__.b])) - Block:3( - PassStmt:4()))) - -[case testVariableInBlock] -while object: - x = None - x = x -[out] -MypyFile:1( - WhileStmt:1( - NameExpr(object [builtins.object]) - Block:1( - AssignmentStmt:2( - NameExpr(x* [__main__.x]) - NameExpr(None [builtins.None])) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - NameExpr(x [__main__.x]))))) - -[case testVariableInExceptHandler] -try: - pass -except object as o: - x = None - o = x -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - NameExpr(object [builtins.object]) - NameExpr(o* [__main__.o]) - Block:3( - AssignmentStmt:4( - NameExpr(x* [__main__.x]) - NameExpr(None [builtins.None])) - AssignmentStmt:5( - NameExpr(o [__main__.o]) - NameExpr(x [__main__.x]))))) - -[case testCallInExceptHandler] -try: - pass -except object as o: - o = object() -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - NameExpr(object [builtins.object]) - NameExpr(o* [__main__.o]) - Block:3( - AssignmentStmt:4( - NameExpr(o [__main__.o]) - CallExpr:4( - NameExpr(object [builtins.object]) - Args()))))) - -[case testTryExceptWithMultipleHandlers] -try: - pass -except BaseException as e: - pass -except Err as f: - f = BaseException() # Fail - f = Err() -class Err(BaseException): pass -[builtins fixtures/exception.py] -[out] -MypyFile:1( - TryStmt:1( - Block:1( - PassStmt:2()) - NameExpr(BaseException [builtins.BaseException]) - NameExpr(e* [__main__.e]) - Block:3( - PassStmt:4()) - NameExpr(Err [__main__.Err]) - NameExpr(f* [__main__.f]) - Block:5( - AssignmentStmt:6( - NameExpr(f [__main__.f]) - CallExpr:6( - NameExpr(BaseException [builtins.BaseException]) - Args())) - AssignmentStmt:7( - NameExpr(f [__main__.f]) - CallExpr:7( - NameExpr(Err [__main__.Err]) - Args())))) - ClassDef:8( - Err - BaseType( - builtins.BaseException) - PassStmt:8())) - -[case testMultipleAssignmentWithPartialNewDef] -o = None -x, o = o, o -[out] -MypyFile:1( - AssignmentStmt:1( - NameExpr(o* [__main__.o]) - NameExpr(None [builtins.None])) - AssignmentStmt:2( - TupleExpr:2( - NameExpr(x* [__main__.x]) - NameExpr(o [__main__.o])) - TupleExpr:2( - NameExpr(o [__main__.o]) - NameExpr(o [__main__.o])))) - -[case testFunctionDecorator] -def decorate(f): pass -@decorate -def g(): - g() -[out] -MypyFile:1( - FuncDef:1( - decorate - Args( - Var(f)) - Block:1( - PassStmt:1())) - Decorator:2( - Var(g) - NameExpr(decorate [__main__.decorate]) - FuncDef:3( - g - Block:3( - ExpressionStmt:4( - CallExpr:4( - NameExpr(g [__main__.g]) - Args())))))) - -[case testTryWithinFunction] -def f() -> None: - try: - pass - except object as o: - pass -[out] -MypyFile:1( - FuncDef:1( - f - def () - Block:1( - TryStmt:2( - Block:2( - PassStmt:3()) - NameExpr(object [builtins.object]) - NameExpr(o* [l]) - Block:4( - PassStmt:5()))))) - -[case testReuseExceptionVariable] -def f() -> None: - try: - pass - except object as o: - pass - except object as o: - pass -[out] -MypyFile:1( - FuncDef:1( - f - def () - Block:1( - TryStmt:2( - Block:2( - PassStmt:3()) - NameExpr(object [builtins.object]) - NameExpr(o* [l]) - Block:4( - PassStmt:5()) - NameExpr(object [builtins.object]) - NameExpr(o [l]) - Block:6( - PassStmt:7()))))) diff --git a/mypy/test/data/semanal-symtable.test b/mypy/test/data/semanal-symtable.test deleted file mode 100644 index 4821635aba14..000000000000 --- a/mypy/test/data/semanal-symtable.test +++ /dev/null @@ -1,52 +0,0 @@ -[case testEmptyFile] -[out] --- Note that builtins are ignored to simplify output. -__main__: - SymbolTable() - -[case testVarDef] -x = 1 -[out] -__main__: - SymbolTable( - x : Gdef/Var (__main__)) - -[case testFuncDef] -def f(): pass -[out] -__main__: - SymbolTable( - f : Gdef/FuncDef (__main__)) - -[case testEmptyClassDef] -class c: pass -[out] -__main__: - SymbolTable( - c : Gdef/TypeInfo (__main__)) - -[case testImport] -import m -[file m.py] -x = 1 -[out] -__main__: - SymbolTable( - m : ModuleRef/MypyFile (__main__)) -m: - SymbolTable( - x : Gdef/Var (m)) - -[case testImportFromModule] -from m import x -[file m.py] -class x: pass -y = 1 -[out] -__main__: - SymbolTable( - x : Gdef/TypeInfo (__main__)) -m: - SymbolTable( - x : Gdef/TypeInfo (m) - y : Gdef/Var (m)) diff --git a/mypy/test/data/semanal-typeinfo.test b/mypy/test/data/semanal-typeinfo.test deleted file mode 100644 index 6bb62e1c57ce..000000000000 --- a/mypy/test/data/semanal-typeinfo.test +++ /dev/null @@ -1,80 +0,0 @@ -[case testEmptyFile] -[out] -TypeInfoMap() - -[case testEmptyClass] -class c: pass -[out] -TypeInfoMap( - __main__.c : TypeInfo( - Name(__main__.c) - Bases(builtins.object) - Names())) - -[case testClassWithMethod] -class c: - def f(self): pass -[out] -TypeInfoMap( - __main__.c : TypeInfo( - Name(__main__.c) - Bases(builtins.object) - Names( - f))) - -[case testClassWithAttributes] -class c: - def __init__(self, x): - self.y = x - self.z = 1 -[out] -TypeInfoMap( - __main__.c : TypeInfo( - Name(__main__.c) - Bases(builtins.object) - Names( - __init__ - y - z))) - -[case testBaseClass] -class base: pass -class c(base): pass -[out] -TypeInfoMap( - __main__.base : TypeInfo( - Name(__main__.base) - Bases(builtins.object) - Names()) - __main__.c : TypeInfo( - Name(__main__.c) - Bases(__main__.base) - Names())) - -[case testClassAndAbstractClass] -from abc import abstractmethod, ABCMeta -import typing - -class i(metaclass=ABCMeta): pass -class c(i): pass -[out] -TypeInfoMap( - __main__.c : TypeInfo( - Name(__main__.c) - Bases(__main__.i) - Names()) - __main__.i : TypeInfo( - Name(__main__.i) - Bases(builtins.object) - Names())) - -[case testAttributeWithoutType] -class A: - a = A -[out] -TypeInfoMap( - __main__.A : TypeInfo( - Name(__main__.A) - Bases(builtins.object) - Names( - a))) diff --git a/mypy/test/data/semanal-types.test b/mypy/test/data/semanal-types.test deleted file mode 100644 index 413a0f448a65..000000000000 --- a/mypy/test/data/semanal-types.test +++ /dev/null @@ -1,1553 +0,0 @@ -[case testVarWithType] -import typing -class A: pass -x = A() # type: A -y = x -[out] -MypyFile:1( - Import:1(typing : typing) - ClassDef:2( - A - PassStmt:2()) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - CallExpr:3( - NameExpr(A [__main__.A]) - Args()) - __main__.A) - AssignmentStmt:4( - NameExpr(y* [__main__.y]) - NameExpr(x [__main__.x]))) - -[case testInitializeToUndefined] -from typing import Undefined - -class A: pass -x = Undefined # type: A -y = x -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ClassDef:3( - A - PassStmt:3()) - AssignmentStmt:4( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - __main__.A) - AssignmentStmt:5( - NameExpr(y* [__main__.y]) - NameExpr(x [__main__.x]))) - -[case testLocalVarWithType] -from typing import Undefined -class A: pass -def f(): - x = Undefined # type: A - y = x -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ClassDef:2( - A - PassStmt:2()) - FuncDef:3( - f - Block:3( - AssignmentStmt:4( - NameExpr(x [l]) - NameExpr(Undefined [typing.Undefined]) - __main__.A) - AssignmentStmt:5( - NameExpr(y* [l]) - NameExpr(x [l]))))) - -[case testAnyType] -from typing import Undefined, Any -x = Undefined # type: Any -y = x -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined, Any : Any]) - AssignmentStmt:2( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - Any) - AssignmentStmt:3( - NameExpr(y* [__main__.y]) - NameExpr(x [__main__.x]))) - -[case testMemberVarWithType] -import typing -class A: - def __init__(self): - self.x = None # type: int -[out] -MypyFile:1( - Import:1(typing : typing) - ClassDef:2( - A - FuncDef:3( - __init__ - Args( - Var(self)) - Block:3( - AssignmentStmt:4( - MemberExpr:4( - NameExpr(self [l]) - x) - NameExpr(None [builtins.None]) - builtins.int))))) - -[case testClassVarWithType] -import typing -class A: - x = None # type: int - x = 1 -[out] -MypyFile:1( - Import:1(typing : typing) - ClassDef:2( - A - AssignmentStmt:3( - NameExpr(x [m]) - NameExpr(None [builtins.None]) - builtins.int) - AssignmentStmt:4( - NameExpr(x [m]) - IntExpr(1)))) - -[case testFunctionSig] -from typing import Any -class A: pass -def f(x: A) -> A: pass -def g(x: Any, y: A) -> None: - z = x, y -[out] -MypyFile:1( - ImportFrom:1(typing, [Any : Any]) - ClassDef:2( - A - PassStmt:2()) - FuncDef:3( - f - Args( - Var(x)) - def (x: __main__.A) -> __main__.A - Block:3( - PassStmt:3())) - FuncDef:4( - g - Args( - Var(x) - Var(y)) - def (x: Any, y: __main__.A) - Block:4( - AssignmentStmt:5( - NameExpr(z* [l]) - TupleExpr:5( - NameExpr(x [l]) - NameExpr(y [l])))))) - -[case testBaseclass] -class A: pass -class B(A): pass -[out] -MypyFile:1( - ClassDef:1( - A - PassStmt:1()) - ClassDef:2( - B - BaseType( - __main__.A) - PassStmt:2())) - -[case testMultipleVarDef] -from typing import Undefined -class A: pass -class B: pass -a, b = Undefined, Undefined # type: (A, B) -x = a, b -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ClassDef:2( - A - PassStmt:2()) - ClassDef:3( - B - PassStmt:3()) - AssignmentStmt:4( - TupleExpr:4( - NameExpr(a [__main__.a]) - NameExpr(b [__main__.b])) - TupleExpr:4( - NameExpr(Undefined [typing.Undefined]) - NameExpr(Undefined [typing.Undefined])) - Tuple[__main__.A, __main__.B]) - AssignmentStmt:5( - NameExpr(x* [__main__.x]) - TupleExpr:5( - NameExpr(a [__main__.a]) - NameExpr(b [__main__.b])))) - -[case testGenericType] -from typing import typevar, Generic, Any - -t = typevar('t') - -class A(Generic[t]): pass -class B: pass -x = None # type: A[B] -y = None # type: A[Any] -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic, Any : Any]) - AssignmentStmt:3( - NameExpr(t* [__main__.t]) - TypeVarExpr:3()) - ClassDef:5( - A - TypeVars( - t) - PassStmt:5()) - ClassDef:6( - B - PassStmt:6()) - AssignmentStmt:7( - NameExpr(x [__main__.x]) - NameExpr(None [builtins.None]) - __main__.A[__main__.B]) - AssignmentStmt:8( - NameExpr(y [__main__.y]) - NameExpr(None [builtins.None]) - __main__.A[Any])) - -[case testGenericType2] -from typing import typevar, Generic, Any -t = typevar('t') -s = typevar('s') -class A(Generic[t, s]): pass -class B: pass -x = None # type: A[B, Any] -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic, Any : Any]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - AssignmentStmt:3( - NameExpr(s* [__main__.s]) - TypeVarExpr:3()) - ClassDef:4( - A - TypeVars( - t - s) - PassStmt:4()) - ClassDef:5( - B - PassStmt:5()) - AssignmentStmt:6( - NameExpr(x [__main__.x]) - NameExpr(None [builtins.None]) - __main__.A[__main__.B, Any])) - -[case testAssignmentAfterDef] -from typing import Undefined - -class A: pass -a = Undefined # type: A -a = 1 -def f(): - b = Undefined # type: A - b = 1 -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - ClassDef:3( - A - PassStmt:3()) - AssignmentStmt:4( - NameExpr(a [__main__.a]) - NameExpr(Undefined [typing.Undefined]) - __main__.A) - AssignmentStmt:5( - NameExpr(a [__main__.a]) - IntExpr(1)) - FuncDef:6( - f - Block:6( - AssignmentStmt:7( - NameExpr(b [l]) - NameExpr(Undefined [typing.Undefined]) - __main__.A) - AssignmentStmt:8( - NameExpr(b [l]) - IntExpr(1))))) - -[case testUndefinedWithArgument] -from typing import Undefined -x = Undefined(int) -x = 1 -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - AssignmentStmt:2( - NameExpr(x [__main__.x]) - UndefinedExpr:2(builtins.int) - builtins.int) - AssignmentStmt:3( - NameExpr(x [__main__.x]) - IntExpr(1))) - -[case testUndefinedWithComplexType] -from typing import Undefined, Tuple -x = Undefined(Tuple[int, str]) -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined, Tuple : Tuple]) - AssignmentStmt:2( - NameExpr(x [__main__.x]) - UndefinedExpr:2(Tuple[builtins.int, builtins.str]) - Tuple[builtins.int, builtins.str])) - -[case testUndefinedInMultipleAssignment] -from typing import Undefined, Any -x, y, z = Undefined(int), 1, Undefined(Any) -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined, Any : Any]) - AssignmentStmt:2( - TupleExpr:2( - NameExpr(x* [__main__.x]) - NameExpr(y* [__main__.y]) - NameExpr(z* [__main__.z])) - TupleExpr:2( - UndefinedExpr:2(builtins.int) - IntExpr(1) - UndefinedExpr:2(Any)))) - -[case testUndefinedWithStrLiteral] -from typing import Undefined -x = Undefined('A') -class A: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [Undefined : Undefined]) - AssignmentStmt:2( - NameExpr(x [__main__.x]) - UndefinedExpr:2(__main__.A) - __main__.A) - ClassDef:3( - A - PassStmt:3())) - -[case testCast] -from typing import typevar, Generic, Any, cast -t = typevar('t') -class c: pass -class d(Generic[t]): pass -cast(Any, 1) -cast(c, 1) -cast(d[c], c) -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic, Any : Any, cast : cast]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - c - PassStmt:3()) - ClassDef:4( - d - TypeVars( - t) - PassStmt:4()) - ExpressionStmt:5( - CastExpr:5( - IntExpr(1) - Any)) - ExpressionStmt:6( - CastExpr:6( - IntExpr(1) - __main__.c)) - ExpressionStmt:7( - CastExpr:7( - NameExpr(c [__main__.c]) - __main__.d[__main__.c]))) - -[case testCastToQualifiedTypeAndCast] -import typing -import _m -typing.cast(_m.C, object) -[file _m.py] -class C: pass -[out] -MypyFile:1( - Import:1(typing : typing) - Import:2(_m : _m) - ExpressionStmt:3( - CastExpr:3( - NameExpr(object [builtins.object]) - _m.C))) - -[case testLongQualifiedCast] -import typing -import _m._n -typing.cast(_m._n.C, object) -[file _m/__init__.py] -[file _m/_n.py] -class C: pass -[out] -MypyFile:1( - Import:1(typing : typing) - Import:2(_m._n : _m._n) - ExpressionStmt:3( - CastExpr:3( - NameExpr(object [builtins.object]) - _m._n.C))) - -[case testCastTargetWithTwoTypeArgs] -from typing import typevar, Generic, cast -t = typevar('t') -s = typevar('s') -class C(Generic[t, s]): pass -cast(C[str, int], C) -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic, cast : cast]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - AssignmentStmt:3( - NameExpr(s* [__main__.s]) - TypeVarExpr:3()) - ClassDef:4( - C - TypeVars( - t - s) - PassStmt:4()) - ExpressionStmt:5( - CastExpr:5( - NameExpr(C [__main__.C]) - __main__.C[builtins.str, builtins.int]))) - -[case testCastToAny] -from typing import Any -Any(None) -[out] -MypyFile:1( - ImportFrom:1(typing, [Any : Any]) - ExpressionStmt:2( - CastExpr:2( - NameExpr(None [builtins.None]) - Any))) - -[case testCastToTupleType] -from typing import Tuple, cast -cast(Tuple[int, str], None) -[out] -MypyFile:1( - ImportFrom:1(typing, [Tuple : Tuple, cast : cast]) - ExpressionStmt:2( - CastExpr:2( - NameExpr(None [builtins.None]) - Tuple[builtins.int, builtins.str]))) - -[case testCastToFunctionType] -from typing import Function, cast -cast(Function[[int], str], None) -[out] -MypyFile:1( - ImportFrom:1(typing, [Function : Function, cast : cast]) - ExpressionStmt:2( - CastExpr:2( - NameExpr(None [builtins.None]) - def (builtins.int) -> builtins.str))) - -[case testCastToStringLiteralType] -from typing import cast -cast('int', 1) -[out] -MypyFile:1( - ImportFrom:1(typing, [cast : cast]) - ExpressionStmt:2( - CastExpr:2( - IntExpr(1) - builtins.int))) - -[case testFunctionTypeVariable] -from typing import typevar, Undefined -t = typevar('t') -def f(x: t) -> None: - y = Undefined # type: t -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Undefined : Undefined]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - FuncDef:3( - f - Args( - Var(x)) - def [t] (x: t`-1) - Block:3( - AssignmentStmt:4( - NameExpr(y [l]) - NameExpr(Undefined [typing.Undefined]) - t`-1)))) - -[case testTwoFunctionTypeVariables] -from typing import typevar -t = typevar('t') -u = typevar('u') -def f(x: t, y: u, z: t) -> None: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - AssignmentStmt:3( - NameExpr(u* [__main__.u]) - TypeVarExpr:3()) - FuncDef:4( - f - Args( - Var(x) - Var(y) - Var(z)) - def [t, u] (x: t`-1, y: u`-2, z: t`-1) - Block:4( - PassStmt:4()))) - -[case testNestedGenericFunctionTypeVariable] -from typing import typevar, Generic -t = typevar('t') -class A(Generic[t]): pass -def f(x: A[t], y) -> None: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - A - TypeVars( - t) - PassStmt:3()) - FuncDef:4( - f - Args( - Var(x) - Var(y)) - def [t] (x: __main__.A[t`-1], y: Any) - Block:4( - PassStmt:4()))) - -[case testNestedGenericFunctionTypeVariable2] -from typing import typevar, Tuple, Generic -t = typevar('t') -class A(Generic[t]): pass -def f(x: Tuple[int, t]) -> None: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Tuple : Tuple, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - A - TypeVars( - t) - PassStmt:3()) - FuncDef:4( - f - Args( - Var(x)) - def [t] (x: Tuple[builtins.int, t`-1]) - Block:4( - PassStmt:4()))) - -[case testNestedGenericFunctionTypeVariable3] -from typing import typevar, Function, Generic -t = typevar('t') -class A(Generic[t]): pass -def f(x: Function[[int, t], int]) -> None: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Function : Function, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - A - TypeVars( - t) - PassStmt:3()) - FuncDef:4( - f - Args( - Var(x)) - def [t] (x: def (builtins.int, t`-1) -> builtins.int) - Block:4( - PassStmt:4()))) - -[case testNestedGenericFunctionTypeVariable4] -from typing import typevar, Function, Generic -t = typevar('t') -class A(Generic[t]): pass -def f(x: Function[[], t]) -> None: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Function : Function, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - A - TypeVars( - t) - PassStmt:3()) - FuncDef:4( - f - Args( - Var(x)) - def [t] (x: def () -> t`-1) - Block:4( - PassStmt:4()))) - -[case testGenericFunctionTypeVariableInReturnType] -from typing import typevar -t = typevar('t') -def f() -> t: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - FuncDef:3( - f - def [t] () -> t`-1 - Block:3( - PassStmt:3()))) - -[case testSelfType] -class A: - def f(self, o: object) -> None: pass -[out] -MypyFile:1( - ClassDef:1( - A - FuncDef:2( - f - Args( - Var(self) - Var(o)) - def (self: __main__.A, o: builtins.object) - Block:2( - PassStmt:2())))) - -[case testNestedGenericFunction] -from typing import typevar -t = typevar('t') -def f() -> None: - def g() -> t: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - FuncDef:3( - f - def () - Block:3( - FuncDef:4( - g - def [t] () -> t`-1 - Block:4( - PassStmt:4()))))) - -[case testClassTvar] -from typing import typevar, Generic - -t = typevar('t') - -class c(Generic[t]): - def f(self) -> t: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:3( - NameExpr(t* [__main__.t]) - TypeVarExpr:3()) - ClassDef:5( - c - TypeVars( - t) - FuncDef:6( - f - Args( - Var(self)) - def (self: __main__.c[t`1]) -> t`1 - Block:6( - PassStmt:6())))) - -[case testClassTvar2] -from typing import typevar, Generic - -t = typevar('t') -s = typevar('s') - -class c(Generic[t, s]): - def f(self, x: s) -> t: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:3( - NameExpr(t* [__main__.t]) - TypeVarExpr:3()) - AssignmentStmt:4( - NameExpr(s* [__main__.s]) - TypeVarExpr:4()) - ClassDef:6( - c - TypeVars( - t - s) - FuncDef:7( - f - Args( - Var(self) - Var(x)) - def (self: __main__.c[t`1, s`2], x: s`2) -> t`1 - Block:7( - PassStmt:7())))) - -[case testGenericBaseClass] -from typing import typevar, Generic -t = typevar('t') -class d(Generic[t]): pass -class c(d[t], Generic[t]): pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - d - TypeVars( - t) - PassStmt:3()) - ClassDef:4( - c - TypeVars( - t) - BaseType( - __main__.d[t`1]) - PassStmt:4())) - -[case testTupleType] -from typing import Tuple -t = None # type: tuple -t1 = None # type: Tuple[object] -t2 = None # type: Tuple[int, object] -[builtins fixtures/tuple.py] -[out] -MypyFile:1( - ImportFrom:1(typing, [Tuple : Tuple]) - AssignmentStmt:2( - NameExpr(t [__main__.t]) - NameExpr(None [builtins.None]) - builtins.tuple) - AssignmentStmt:3( - NameExpr(t1 [__main__.t1]) - NameExpr(None [builtins.None]) - Tuple[builtins.object]) - AssignmentStmt:4( - NameExpr(t2 [__main__.t2]) - NameExpr(None [builtins.None]) - Tuple[builtins.int, builtins.object])) - -[case testFunctionTypes] -from typing import Function -f = None # type: Function[[object, int], str] -g = None # type: Function[[], None] -[out] -MypyFile:1( - ImportFrom:1(typing, [Function : Function]) - AssignmentStmt:2( - NameExpr(f [__main__.f]) - NameExpr(None [builtins.None]) - def (builtins.object, builtins.int) -> builtins.str) - AssignmentStmt:3( - NameExpr(g [__main__.g]) - NameExpr(None [builtins.None]) - def ())) - -[case testOverloadedFunction] -from typing import overload -@overload -def f(o: object) -> int: o -@overload -def f(a: str) -> object: a -[out] -MypyFile:1( - ImportFrom:1(typing, [overload : overload]) - OverloadedFuncDef:2( - Overload(def (o: builtins.object) -> builtins.int, \ - def (a: builtins.str) -> builtins.object) - Decorator:2( - Var(f) - NameExpr(overload [typing.overload]) - FuncDef:3( - f - Args( - Var(o)) - def (o: builtins.object) -> builtins.int - Block:3( - ExpressionStmt:3( - NameExpr(o [l]))))) - Decorator:4( - Var(f) - NameExpr(overload [typing.overload]) - FuncDef:5( - f - Args( - Var(a)) - def (a: builtins.str) -> builtins.object - Block:5( - ExpressionStmt:5( - NameExpr(a [l]))))))) - -[case testReferenceToOverloadedFunction] -from typing import overload -@overload -def f() -> None: pass -@overload -def f(x: int) -> None: pass -x = f -[out] -MypyFile:1( - ImportFrom:1(typing, [overload : overload]) - OverloadedFuncDef:2( - Overload(def (), def (x: builtins.int)) - Decorator:2( - Var(f) - NameExpr(overload [typing.overload]) - FuncDef:3( - f - def () - Block:3( - PassStmt:3()))) - Decorator:4( - Var(f) - NameExpr(overload [typing.overload]) - FuncDef:5( - f - Args( - Var(x)) - def (x: builtins.int) - Block:5( - PassStmt:5())))) - AssignmentStmt:6( - NameExpr(x* [__main__.x]) - NameExpr(f [__main__.f]))) - -[case testNestedOverloadedFunction] -from typing import overload -def f(): - @overload - def g(): pass - @overload - def g(x): pass - y = g -[out] -MypyFile:1( - ImportFrom:1(typing, [overload : overload]) - FuncDef:2( - f - Block:2( - OverloadedFuncDef:3( - Overload(def () -> Any, def (x: Any) -> Any) - Decorator:3( - Var(g) - NameExpr(overload [typing.overload]) - FuncDef:4( - g - Block:4( - PassStmt:4()))) - Decorator:5( - Var(g) - NameExpr(overload [typing.overload]) - FuncDef:6( - g - Args( - Var(x)) - Block:6( - PassStmt:6())))) - AssignmentStmt:7( - NameExpr(y* [l]) - NameExpr(g [l]))))) - -[case testImplicitGenericTypeArgs] -from typing import typevar, Generic, Undefined -t = typevar('t') -s = typevar('s') -class A(Generic[t, s]): pass -x = Undefined # type: A -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic, Undefined : Undefined]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - AssignmentStmt:3( - NameExpr(s* [__main__.s]) - TypeVarExpr:3()) - ClassDef:4( - A - TypeVars( - t - s) - PassStmt:4()) - AssignmentStmt:5( - NameExpr(x [__main__.x]) - NameExpr(Undefined [typing.Undefined]) - __main__.A[Any, Any])) - -[case testImplicitTypeArgsAndGenericBaseClass] -from typing import typevar, Generic -t = typevar('t') -s = typevar('s') -class B(Generic[s]): pass -class A(B, Generic[t]): pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - AssignmentStmt:3( - NameExpr(s* [__main__.s]) - TypeVarExpr:3()) - ClassDef:4( - B - TypeVars( - s) - PassStmt:4()) - ClassDef:5( - A - TypeVars( - t) - BaseType( - __main__.B[Any]) - PassStmt:5())) - -[case testTypeApplication] -from typing import typevar, Generic -t = typevar('t') -class A(Generic[t]): pass -x = A[int]() -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - A - TypeVars( - t) - PassStmt:3()) - AssignmentStmt:4( - NameExpr(x* [__main__.x]) - CallExpr:4( - TypeApplication:4( - NameExpr(A [__main__.A]) - Types( - builtins.int)) - Args()))) - -[case testTypeApplicationWithTwoTypeArgs] -from typing import typevar, Generic, Any -t = typevar('t') -s = typevar('s') -class A(Generic[t, s]): pass -x = A[int, Any]() -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic, Any : Any]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - AssignmentStmt:3( - NameExpr(s* [__main__.s]) - TypeVarExpr:3()) - ClassDef:4( - A - TypeVars( - t - s) - PassStmt:4()) - AssignmentStmt:5( - NameExpr(x* [__main__.x]) - CallExpr:5( - TypeApplication:5( - NameExpr(A [__main__.A]) - Types( - builtins.int - Any)) - Args()))) - -[case testFunctionTypeApplication] -from typing import typevar -t = typevar('t') -def f(x: t) -> None: pass -f[int](1) -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - FuncDef:3( - f - Args( - Var(x)) - def [t] (x: t`-1) - Block:3( - PassStmt:3())) - ExpressionStmt:4( - CallExpr:4( - TypeApplication:4( - NameExpr(f [__main__.f]) - Types( - builtins.int)) - Args( - IntExpr(1))))) - -[case testTypeApplicationWithStringLiteralType] -from typing import typevar, Generic -t = typevar('t') -class A(Generic[t]): pass -A['int']() -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:2( - NameExpr(t* [__main__.t]) - TypeVarExpr:2()) - ClassDef:3( - A - TypeVars( - t) - PassStmt:3()) - ExpressionStmt:4( - CallExpr:4( - TypeApplication:4( - NameExpr(A [__main__.A]) - Types( - builtins.int)) - Args()))) - -[case testListTypeAlias] -from typing import List -def f() -> List[int]: pass -[builtins fixtures/list.py] -[out] -MypyFile:1( - ImportFrom:1(typing, [List : List]) - FuncDef:2( - f - def () -> builtins.list[builtins.int] - Block:2( - PassStmt:2()))) - -[case testDictTypeAlias] -from typing import Dict -def f() -> Dict[int, str]: pass -[builtins fixtures/dict.py] -[out] -MypyFile:1( - ImportFrom:1(typing, [Dict : Dict]) - FuncDef:2( - f - def () -> builtins.dict[builtins.int, builtins.str] - Block:2( - PassStmt:2()))) - -[case testQualifiedTypeAlias] -import typing -def f() -> typing.List[int]: pass -[builtins fixtures/list.py] -[out] -MypyFile:1( - Import:1(typing : typing) - FuncDef:2( - f - def () -> builtins.list[builtins.int] - Block:2( - PassStmt:2()))) - -[case testTypeApplicationWithTypeAlias] -from typing import List -List[List[int]] -[builtins fixtures/list.py] -[out] -MypyFile:1( - ImportFrom:1(typing, [List : List]) - ExpressionStmt:2( - TypeApplication:2( - NameExpr(List [builtins.list]) - Types( - builtins.list[builtins.int])))) - -[case testTypeApplicationWithQualifiedTypeAlias] -import typing -typing.List[typing.List[int]] -[builtins fixtures/list.py] -[out] -MypyFile:1( - Import:1(typing : typing) - ExpressionStmt:2( - TypeApplication:2( - MemberExpr:2( - NameExpr(typing) - List [builtins.list]) - Types( - builtins.list[builtins.int])))) - -[case testTypeApplicationWithinLambda] -from typing import List -lambda: List[int]() -[builtins fixtures/list.py] -[out] -MypyFile:1( - ImportFrom:1(typing, [List : List]) - ExpressionStmt:2( - FuncExpr:2( - Block:2( - ReturnStmt:2( - CallExpr:2( - TypeApplication:2( - NameExpr(List [builtins.list]) - Types( - builtins.int)) - Args())))))) - -[case testVarArgsAndKeywordArgs] -def g(*x: int, y: str = ''): pass -[out] -MypyFile:1( - FuncDef:1( - g - MaxPos(0) - Args( - Var(y)) - def (*x: builtins.int, *, y: builtins.str) -> Any - Init( - AssignmentStmt:1( - NameExpr(y [l]) - StrExpr())) - VarArg( - Var(x)) - Block:1( - PassStmt:1()))) - -[case testQualifiedGeneric] -from typing import typevar -import typing -T = typevar('T') -class A(typing.Generic[T]): pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - Import:2(typing : typing) - AssignmentStmt:3( - NameExpr(T* [__main__.T]) - TypeVarExpr:3()) - ClassDef:4( - A - TypeVars( - T) - PassStmt:4())) - -[case testQualifiedTypevar] -import typing -T = typing.typevar('T') -def f(x: T) -> T: pass -[out] -MypyFile:1( - Import:1(typing : typing) - AssignmentStmt:2( - NameExpr(T* [__main__.T]) - TypeVarExpr:2()) - FuncDef:3( - f - Args( - Var(x)) - def [T] (x: T`-1) -> T`-1 - Block:3( - PassStmt:3()))) - -[case testAliasedTypevar] -from typing import typevar as tv -T = tv('T') -def f(x: T) -> T: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : tv]) - AssignmentStmt:2( - NameExpr(T* [__main__.T]) - TypeVarExpr:2()) - FuncDef:3( - f - Args( - Var(x)) - def [T] (x: T`-1) -> T`-1 - Block:3( - PassStmt:3()))) - -[case testLocalTypevar] -from typing import typevar -def f(): - T = typevar('T') - def g(x: T) -> T: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - FuncDef:2( - f - Block:2( - AssignmentStmt:3( - NameExpr(T* [l]) - TypeVarExpr:3()) - FuncDef:4( - g - Args( - Var(x)) - def [T] (x: T`-1) -> T`-1 - Block:4( - PassStmt:4()))))) - -[case testClassLevelTypevar] -from typing import typevar -class A: - T = typevar('T') - def g(self, x: T) -> T: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - ClassDef:2( - A - AssignmentStmt:3( - NameExpr(T* [m]) - TypeVarExpr:3()) - FuncDef:4( - g - Args( - Var(self) - Var(x)) - def [T] (self: __main__.A, x: T`-1) -> T`-1 - Block:4( - PassStmt:4())))) - -[case testImportTypevar] -from typing import Generic -from _m import T -class A(Generic[T]): - y = None # type: T -[file _m.py] -from typing import typevar -T = typevar('T') -[out] -MypyFile:1( - ImportFrom:1(typing, [Generic : Generic]) - ImportFrom:2(_m, [T : T]) - ClassDef:3( - A - TypeVars( - T) - AssignmentStmt:4( - NameExpr(y [m]) - NameExpr(None [builtins.None]) - T`1))) - -[case testQualifiedReferenceToTypevarInClass] -from typing import Generic -import _m -class A(Generic[_m.T]): - a = None # type: _m.T - def f(self, x: _m.T): - b = None # type: _m.T -[file _m.py] -from typing import typevar -T = typevar('T') -[out] -MypyFile:1( - ImportFrom:1(typing, [Generic : Generic]) - Import:2(_m : _m) - ClassDef:3( - A - TypeVars( - _m.T) - AssignmentStmt:4( - NameExpr(a [m]) - NameExpr(None [builtins.None]) - _m.T`1) - FuncDef:5( - f - Args( - Var(self) - Var(x)) - def (self: __main__.A[_m.T`1], x: _m.T`1) -> Any - Block:5( - AssignmentStmt:6( - NameExpr(b [l]) - NameExpr(None [builtins.None]) - _m.T`1))))) - -[case testQualifiedReferenceToTypevarInFunctionSignature] -import _m -def f(x: _m.T) -> None: - a = None # type: _m.T -[file _m.py] -from typing import typevar -T = typevar('T') -[out] -MypyFile:1( - Import:1(_m : _m) - FuncDef:2( - f - Args( - Var(x)) - def [_m.T] (x: _m.T`-1) - Block:2( - AssignmentStmt:3( - NameExpr(a [l]) - NameExpr(None [builtins.None]) - _m.T`-1)))) - -[case testFunctionCommentAnnotation] -from typing import Any -def f(x): # type: (int) -> Any - x = 1 -[out] -MypyFile:1( - ImportFrom:1(typing, [Any : Any]) - FuncDef:2( - f - Args( - Var(x)) - def (x: builtins.int) -> Any - Block:2( - AssignmentStmt:3( - NameExpr(x [l]) - IntExpr(1))))) - -[case testMethodCommentAnnotation] -import typing -class A: - def f(self, x): # type: (int) -> str - x = 1 -[out] -MypyFile:1( - Import:1(typing : typing) - ClassDef:2( - A - FuncDef:3( - f - Args( - Var(self) - Var(x)) - def (self: __main__.A, x: builtins.int) -> builtins.str - Block:3( - AssignmentStmt:4( - NameExpr(x [l]) - IntExpr(1)))))) - -[case testTypevarWithValues] -from typing import typevar, Any -T = typevar('T', values=(int, str)) -S = typevar('S', values=(Any, int, str)) -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Any : Any]) - AssignmentStmt:2( - NameExpr(T* [__main__.T]) - TypeVarExpr:2( - Values( - builtins.int - builtins.str))) - AssignmentStmt:3( - NameExpr(S* [__main__.S]) - TypeVarExpr:3( - Values( - Any - builtins.int - builtins.str)))) - -[case testGenericFunctionWithValueSet] -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> T: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar]) - AssignmentStmt:2( - NameExpr(T* [__main__.T]) - TypeVarExpr:2( - Values( - builtins.int - builtins.str))) - FuncDef:3( - f - Args( - Var(x)) - def [T in (builtins.int, builtins.str)] (x: T`-1) -> T`-1 - Block:3( - PassStmt:3()))) - -[case testGenericClassWithValueSet] -from typing import typevar, Generic -T = typevar('T', values=(int, str)) -class C(Generic[T]): pass -[out] -MypyFile:1( - ImportFrom:1(typing, [typevar : typevar, Generic : Generic]) - AssignmentStmt:2( - NameExpr(T* [__main__.T]) - TypeVarExpr:2( - Values( - builtins.int - builtins.str))) - ClassDef:3( - C - TypeVars( - T in (builtins.int, builtins.str)) - PassStmt:3())) - -[case testSimpleDucktypeDecorator] -from typing import ducktype -@ducktype(str) -class S: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [ducktype : ducktype]) - ClassDef:2( - S - Ducktype(builtins.str) - Decorators( - DucktypeExpr:2(builtins.str)) - PassStmt:3())) - -[case testDisjointclassDecorator] -from typing import disjointclass -@disjointclass(str) -class A: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [disjointclass : disjointclass]) - ClassDef:2( - A - Disjointclasses( - builtins.str) - Decorators( - DisjointclassExpr:2(builtins.str)) - PassStmt:3())) - -[case testDisjointclassDecorators] -from typing import disjointclass -@disjointclass(str) -@disjointclass(int) -class A: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [disjointclass : disjointclass]) - ClassDef:2( - A - Disjointclasses( - builtins.str - builtins.int) - Decorators( - DisjointclassExpr:2(builtins.str) - DisjointclassExpr:3(builtins.int)) - PassStmt:4())) - -[case testDisjointClassSymmetry] -from typing import disjointclass -class B: pass -@disjointclass(B) -class A: pass -[out] -MypyFile:1( - ImportFrom:1(typing, [disjointclass : disjointclass]) - ClassDef:2( - B - Disjointclasses( - __main__.A) - PassStmt:2()) - ClassDef:3( - A - Disjointclasses( - __main__.B) - Decorators( - DisjointclassExpr:3(__main__.B)) - PassStmt:4())) - -[case testDisjointnessAndInheritance] -from typing import disjointclass -class A: pass -class B(A): pass -@disjointclass(B) -class C: pass -class D(C): pass -class E(B): pass -[out] -MypyFile:1( - ImportFrom:1(typing, [disjointclass : disjointclass]) - ClassDef:2( - A - PassStmt:2()) - ClassDef:3( - B - Disjointclasses( - __main__.C - __main__.D) - BaseType( - __main__.A) - PassStmt:3()) - ClassDef:4( - C - Disjointclasses( - __main__.B - __main__.E) - Decorators( - DisjointclassExpr:4(__main__.B)) - PassStmt:5()) - ClassDef:6( - D - Disjointclasses( - __main__.B - __main__.E) - BaseType( - __main__.C) - PassStmt:6()) - ClassDef:7( - E - Disjointclasses( - __main__.C - __main__.D) - BaseType( - __main__.B) - PassStmt:7())) diff --git a/mypy/test/data/typexport-basic.test b/mypy/test/data/typexport-basic.test deleted file mode 100644 index aadf4d3312e9..000000000000 --- a/mypy/test/data/typexport-basic.test +++ /dev/null @@ -1,1188 +0,0 @@ --- Test cases for exporting node types from the type checker. --- --- Each test case consists of at least two sections. --- The first section contains [case NAME-skip] followed by the input code, --- while the second section contains [out] followed by the output from the type --- checker. --- --- The first line of input code should be a regexp in comment that describes --- the information to dump (prefix with ##). The regexp is matched against --- the following items: --- --- * each name of an expression node --- * each type string of a node (e.g. OpExpr) --- --- Lines starting with "--" in this file will be ignored. - - --- Expressions --- ----------- - - -[case testConstructorCall] -import typing -A() -B() -class A: pass -class B: pass -[out] -CallExpr(2) : A -NameExpr(2) : def () -> A -CallExpr(3) : B -NameExpr(3) : def () -> B - -[case testLiterals] -import typing -5 -2.3 -'foo' -None -[builtins fixtures/primitives.py] -[out] -IntExpr(2) : builtins.int -FloatExpr(3) : builtins.float -StrExpr(4) : builtins.str -NameExpr(5) : 'None' - -[case testNameExpression] -from typing import Undefined -a = Undefined # type: A -a # node -def f(aa: 'A') -> None: - b = Undefined # type: B - aa # node - b # node -class A: - def g(self) -> None: - self # node -class B: pass -[out] -NameExpr(3) : A -NameExpr(6) : A -NameExpr(7) : B -NameExpr(10) : A - -[case testMemberAccess] -## MemberExpr|CallExpr -from typing import Undefined -a = Undefined # type: A -a.m -a.f -a.f() -class A: - m = Undefined # type: A - def f(self) -> 'B': pass -class B: pass -[out] -MemberExpr(4) : A -MemberExpr(5) : def () -> B -CallExpr(6) : B -MemberExpr(6) : def () -> B - -[case testCastExpression] -## CastExpr|[a-z] -from typing import Undefined, Any, cast -d = Undefined # type: Any -b = Undefined # type: B -class A: pass -class B(A): pass -cast(A, d) -cast(A, b) -cast(B, b) -[out] -CastExpr(7) : A -NameExpr(7) : Any -CastExpr(8) : A -NameExpr(8) : B -CastExpr(9) : B -NameExpr(9) : B - -[case testArithmeticOps] -## OpExpr -import typing -a = 1 + 2 -1.2 * 3 -2.2 - 3 -1 / 2 -[file builtins.py] -class object: - def __init__(self) -> None: pass -class function: pass -class int: - def __add__(self, x: int) -> int: pass - def __truediv__(self, x: int) -> float: pass -class float: - def __mul__(self, x: int) -> float: pass - def __sub__(self, x: int) -> float: pass -class type: pass -[out] -OpExpr(3) : builtins.int -OpExpr(4) : builtins.float -OpExpr(5) : builtins.float -OpExpr(6) : builtins.float - -[case testComparisonOps] -## ComparisonExpr -import typing -1 == object() -1 == 2 -2 < 3 -1 < 2 < 3 -8 > 3 -4 < 6 > 2 -[file builtins.py] -from typing import builtinclass -@builtinclass -class object: - def __init__(self) -> None: pass -class int: - def __eq__(self, x: object) -> bool: pass - def __lt__(self, x: int) -> bool: pass - def __gt__(self, x: int) -> int: pass -class bool: pass -class type: pass -class function: pass -[out] -ComparisonExpr(3) : builtins.bool -ComparisonExpr(4) : builtins.bool -ComparisonExpr(5) : builtins.bool -ComparisonExpr(6) : builtins.bool -ComparisonExpr(7) : builtins.int -ComparisonExpr(8) : builtins.object - -[case testBooleanOps] -## OpExpr|UnaryExpr -import typing -a = 1 -a and a -a or a -not a -[builtins fixtures/bool.py] -[out] -OpExpr(4) : builtins.int -OpExpr(5) : builtins.int -UnaryExpr(6) : builtins.bool - -[case testBooleanOpsOnBools] -## OpExpr|UnaryExpr -import typing -a = bool() -a and a -a or a -not a -[builtins fixtures/bool.py] -[out] -OpExpr(4) : builtins.bool -OpExpr(5) : builtins.bool -UnaryExpr(6) : builtins.bool - -[case testFunctionCall] -## CallExpr -from typing import Tuple -f( - A(), - B()) -class A: pass -class B: pass -def f(a: A, b: B) -> Tuple[A, B]: pass -[builtins fixtures/tuple.py] -[out] -CallExpr(3) : Tuple[A, B] -CallExpr(4) : A -CallExpr(5) : B - - --- Statements --- ---------- - - -[case testSimpleAssignment] -from typing import Undefined, Any -a = Undefined # type: A -b = a # type: Any -b = a -a = b - -class A: pass -[out] -NameExpr(3) : A -NameExpr(4) : A -NameExpr(4) : Any -NameExpr(5) : A -NameExpr(5) : Any - -[case testMemberAssignment] -from typing import Undefined, Any -class A: - a = Undefined # type: A - b = Undefined # type: Any - def f(self) -> None: - self.b = self.a - self.a.a = self.b -[out] -MemberExpr(6) : A -MemberExpr(6) : Any -NameExpr(6) : A -NameExpr(6) : A -MemberExpr(7) : A -MemberExpr(7) : A -MemberExpr(7) : Any -NameExpr(7) : A -NameExpr(7) : A - -[case testIf] -from typing import Undefined -a = Undefined # type: bool -if a: - None -elif not a: - None -[builtins fixtures/bool.py] -[out] -NameExpr(3) : builtins.bool -NameExpr(4) : 'None' -NameExpr(5) : builtins.bool -UnaryExpr(5) : builtins.bool -NameExpr(6) : 'None' - -[case testWhile] -from typing import Undefined -a = Undefined # type: bool -while a: - None -[builtins fixtures/bool.py] -[out] -NameExpr(3) : builtins.bool -NameExpr(4) : 'None' - - --- Simple type inference --- --------------------- - - -[case testInferSingleType] -import typing -x = 1 -[builtins fixtures/primitives.py] -[out] -IntExpr(2) : builtins.int -NameExpr(2) : builtins.int - -[case testInferTwoTypes] -## NameExpr -import typing -(s, -i) = 'x', 1 -[builtins fixtures/primitives.py] -[out] -NameExpr(3) : builtins.str -NameExpr(4) : builtins.int - -[case testInferSingleLocalVarType] -import typing -def f() -> None: - x = 1 -[builtins fixtures/primitives.py] -[out] -IntExpr(3) : builtins.int -NameExpr(3) : builtins.int - - --- Basic generics --- -------------- - - -[case testImplicitBoundTypeVarsForMethod] -## MemberExpr -from typing import typevar, Generic, Undefined -T = typevar('T') -class A(Generic[T]): - def f(self) -> T: pass -class B: pass -def g() -> None: - a = Undefined # type: A[B] - f = a.f -[out] -MemberExpr(9) : def [1:B] () -> B - -[case testImplicitBoundTypeVarsForConstructor] -## CallExpr|TypeApplication|NameExpr -from typing import Any, typevar, Generic -T = typevar('T') -class A(Generic[T]): - def __init__(self) -> None: pass -class B: pass -A[B]() -A[Any]() -[out] -CallExpr(7) : A[B] -NameExpr(7) : def [-1:B] () -> A[B] -TypeApplication(7) : def [-1:B] () -> A[B] -CallExpr(8) : A[Any] -NameExpr(8) : def [-1:Any] () -> A[Any] -TypeApplication(8) : def [-1:Any] () -> A[Any] - -[case testImplicitBoundTypeVarsForSelfMethodReference] -from typing import typevar, Generic -T = typevar('T') -class A(Generic[T]): - def f(self) -> T: - self.f() -[out] -CallExpr(5) : T`1 -MemberExpr(5) : def [1:T`1] () -> T`1 -NameExpr(5) : A[T`1] - -[case testGenericFunctionCallWithTypeApp-skip] -## CallExpr|TypeApplication|NameExpr -from typing import Any, typevar, Tuple -T = typevar('T') -class A: pass -f[A](A()) -f[Any](A()) -def f(a: T) -> Tuple[T, T]: pass -[builtins fixtures/tuple.py] -[out] -CallExpr(5) : A -CallExpr(5) : Tuple[A, A] -NameExpr(5) : def () -> A -NameExpr(5) : def [-1:A] (a: A) -> Tuple[A, A] -TypeApplication(5) : def [-1:A] (a: A) -> Tuple[A, A] -CallExpr(6) : A -CallExpr(6) : Tuple[Any, Any] -NameExpr(6) : def () -> A -NameExpr(6) : def [-1:Any] (a: Any) -> Tuple[Any, Any] -TypeApplication(6) : def [-1:Any] (a: Any) -> Tuple[Any, Any] - --- NOTE: Type applications are not supported for generic methods, so the --- following test cases are commented out. - ---[case testGenericMethodCallWithTypeApp] ---## CallExpr|MemberExpr|TypeApplication ---from typing import Any, typevar, Tuple, Undefined ---T = typevar('T') ---class A: --- def f(self, a: T) -> Tuple[T, T]: pass ---a.f[A](a) ---a.f[Any](a) ---a = Undefined # type: A ---[builtins fixtures/tuple.py] ---[out] ---CallExpr(2) : Tuple[A, A] ---MemberExpr(2) : def [-1:A] (A a) -> Tuple[A, A] ---TypeApplication(2) : def [-1:A] (A a) -> Tuple[A, A] ---CallExpr(3) : Tuple[Any, Any] ---MemberExpr(3) : def [-1:Any] (any a) -> Tuple[Any, Any] ---TypeApplication(3) : def [-1:Any] (any a) -> Tuple[Any, Any] - ---[case testGenericMethodCallInGenericTypeWithTypeApp] ---## CallExpr|MemberExpr|TypeApplication ---from typing import Any, typevar, Generic, Tuple, Undefined ---T = typevar('T') ---S = typevar('S') ---class B: pass ---class C: pass ---a.f[B](b) ---a.f[Any](b) ---class A(Generic[T]): --- def f(self, a: S) -> Tuple[T, S]: pass ---a = Undefined # type: A[C] ---b = Undefined # type: B ---[builtins fixtures/tuple.py] ---[out] ---CallExpr(6) : Tuple[C, B] ---MemberExpr(6) : def [1:C, -1:B] (B a) -> Tuple[C, B] ---TypeApplication(6) : def [1:C, -1:B] (B a) -> Tuple[C, B] ---CallExpr(7) : Tuple[C, Any] ---MemberExpr(7) : def [1:C, -1:Any] (any a) -> Tuple[C, Any] ---TypeApplication(7) : def [1:C, -1:Any] (any a) -> Tuple[C, Any] - -[case testGenericFunctionWithTwoArgsAndTypeApp] -## CallExpr|TypeApplication -from typing import Any, typevar, Tuple -S = typevar('S') -T = typevar('T') -class A: pass -f[A, Any]() -def f() -> Tuple[S, T]: pass -[builtins fixtures/tuple.py] -[out] -CallExpr(6) : Tuple[A, Any] -TypeApplication(6) : def [-1:A, -2:Any] () -> Tuple[A, Any] - -[case testGenericTypeVariableInference] -from typing import typevar, Generic -T = typevar('T') -class A(Generic[T]): - def __init__(self, a: T) -> None: pass -class B: pass -A(A(B())) -[out] -CallExpr(6) : A[A[B]] -CallExpr(6) : A[B] -CallExpr(6) : B -NameExpr(6) : def [-1:A[B]] (a: A[B]) -> A[A[B]] -NameExpr(6) : def [-1:B] (a: B) -> A[B] -NameExpr(6) : def () -> B - - --- Generic inheritance --- ------------------- - - -[case testInheritedMethodReferenceWithGenericInheritance] -from typing import typevar, Generic -T = typevar('T') -class C: pass -class A(Generic[T]): - def f(self, a: T) -> None: pass -class B(A[C]): - def g(self, c: C) -> None: - self.f(c) -[out] -CallExpr(8) : void -MemberExpr(8) : def [1:C] (a: C) -NameExpr(8) : C -NameExpr(8) : B - -[case testInheritedMethodReferenceWithGenericSubclass] -from typing import typevar, Generic -S = typevar('S') -T = typevar('T') -class C: pass -class A(Generic[S, T]): - def f(self, a: C) -> None: pass -class B(A[C, T], Generic[T]): - def g(self, c: C) -> None: - self.f(c) -[out] -CallExpr(9) : void -MemberExpr(9) : def [1:C, 2:T`1] (a: C) -NameExpr(9) : C -NameExpr(9) : B[T`1] - -[case testExternalReferenceWithGenericInheritance] -from typing import typevar, Generic, Undefined -T = typevar('T') -class C: pass -class A(Generic[T]): - def f(self, a: T) -> None: pass -class B(A[C]): pass -b = Undefined # type: B -c = Undefined # type: C -b.f(c) -[out] -CallExpr(9) : void -MemberExpr(9) : def [1:C] (a: C) -NameExpr(9) : B -NameExpr(9) : C - - --- Implicit Any types --- ------------------ - - -[case testDynamicallyTypedFunction] -from typing import Undefined -def f(x): - y = x + o - z = o - z -o = Undefined # type: object -[out] -NameExpr(3) : builtins.object -NameExpr(3) : Any -NameExpr(3) : Any -OpExpr(3) : Any -NameExpr(4) : builtins.object -NameExpr(4) : Any -NameExpr(5) : Any - -[case testDynamicallyTypedMethod] -from typing import Undefined -class A: - def f(self, x): - y = ( - o) # Place y and o on separate lines - x - y -o = Undefined # type: object -[out] -NameExpr(4) : Any -ParenExpr(4) : Any -NameExpr(5) : builtins.object -NameExpr(6) : Any -NameExpr(7) : Any - -[case testDynamicallyTypedConstructor] -from typing import Undefined -class A: - def __init__(self, x): - y = o - x - y -o = Undefined # type: object -[out] -NameExpr(4) : builtins.object -NameExpr(4) : Any -NameExpr(5) : Any -NameExpr(6) : Any - -[case testCallInDynamicallyTypedFunction] -from typing import Undefined -def f(): - g(o) -def g(a: object) -> object: pass -o = Undefined # type: object -[out] -CallExpr(3) : builtins.object -NameExpr(3) : def (a: builtins.object) -> builtins.object -NameExpr(3) : builtins.object - -[case testExpressionInDynamicallyTypedFn] -import typing -def f(): - x = None - x.f() -[out] -NameExpr(3) : 'None' -NameExpr(3) : Any -CallExpr(4) : Any -MemberExpr(4) : Any -NameExpr(4) : Any - -[case testGenericCall] -from typing import typevar, Generic, Undefined -T = typevar('T') -def f() -> None: - a1 = A(b) # type: A[B] - a2 = A(b) # type: A[object] -class A(Generic[T]): - def __init__(self, a: T) -> None: pass -class B: pass -b = Undefined # type: B -[out] -CallExpr(4) : A[B] -NameExpr(4) : def [-1:B] (a: B) -> A[B] -NameExpr(4) : B -CallExpr(5) : A[builtins.object] -NameExpr(5) : def [-1:builtins.object] (a: builtins.object) -> A[builtins.object] -NameExpr(5) : B - -[case testGenericCallInDynamicallyTypedFunction] -from typing import typevar, Generic -T = typevar('T') -def f(): - A() -class A(Generic[T]): pass -[out] -CallExpr(4) : A[Any] -NameExpr(4) : def [-1:Any] () -> A[Any] - -[case testGenericCallInDynamicallyTypedFunction2] -from typing import typevar, Generic -T = typevar('T') -def f(): - A(f) -class A(Generic[T]): - def __init__(self, x: T) -> None: pass -[out] -CallExpr(4) : A[Any] -NameExpr(4) : def [-1:Any] (x: Any) -> A[Any] -NameExpr(4) : def () -> Any - -[case testGenericCallInDynamicallyTypedFunction3] -from typing import typevar -t = typevar('t') -def f(): - g(None) -def g(x: t) -> t: pass -[out] -CallExpr(4) : Any -NameExpr(4) : 'None' -NameExpr(4) : def [-1:Any] (x: Any) -> Any - - --- Generic types and type inference --- -------------------------------- - - -[case testInferenceInArgumentContext] -## CallExpr -from typing import Undefined, typevar, Generic -T = typevar('T') -f(g()) -f(h(b)) -f(h(c)) - -b = Undefined # type: B -c = Undefined # type: C - -def f(a: 'A[B]') -> None: pass - -def g() -> 'A[T]': pass -def h(a: T) -> 'A[T]': pass - -class A(Generic[T]): pass -class B: pass -class C(B): pass -[out] -CallExpr(4) : void -CallExpr(4) : A[B] -CallExpr(5) : void -CallExpr(5) : A[B] -CallExpr(6) : void -CallExpr(6) : A[B] - -[case testInferGenericTypeForLocalVariable] -from typing import Undefined, typevar, Generic -T = typevar('T') -def f() -> None: - a = A(b) - a - a2, a3 = A(b), A(c) - a2 - a3 -b = Undefined # type: B -c = Undefined # type: C -class A(Generic[T]): - def __init__(self, x: T) -> None: pass -class B: pass -class C: pass -[out] -CallExpr(4) : A[B] -NameExpr(4) : def [-1:B] (x: B) -> A[B] -NameExpr(4) : A[B] -NameExpr(4) : B -NameExpr(5) : A[B] -CallExpr(6) : A[B] -CallExpr(6) : A[C] -NameExpr(6) : def [-1:B] (x: B) -> A[B] -NameExpr(6) : def [-1:C] (x: C) -> A[C] -NameExpr(6) : A[B] -NameExpr(6) : A[C] -NameExpr(6) : B -NameExpr(6) : C -NameExpr(7) : A[B] -NameExpr(8) : A[C] - -[case testNestedGenericCalls] -from typing import Undefined, typevar, Generic -T = typevar('T') -S = typevar('S') -def h() -> None: - g(f(c)) - -c = Undefined # type: C - -class A(Generic[T]): pass -class B(Generic[T]): pass -class C: pass -def f(a: T) -> A[T]: pass -def g(a: S) -> B[S]: pass -[out] -CallExpr(5) : A[C] -CallExpr(5) : B[A[C]] -NameExpr(5) : C -NameExpr(5) : def [-1:C] (a: C) -> A[C] -NameExpr(5) : def [-1:A[C]] (a: A[C]) -> B[A[C]] - -[case testInferListLiterals] -from typing import List -a = [] # type: List[A] -class A: pass -[builtins fixtures/list.py] -[out] -ListExpr(2) : builtins.list[A] - -[case testInferGenericTypeInTypeAnyContext] -from typing import Any -a = [] # type: Any -[builtins fixtures/list.py] -[out] -ListExpr(2) : builtins.list[Any] - -[case testHigherOrderFunction] -from typing import typevar, Function, List -t = typevar('t') -s = typevar('s') -map( - f, - [A()]) -def map(f: Function[[t], s], a: List[t]) -> List[s]: pass -class A: pass -class B: pass -def f(a: A) -> B: pass -[builtins fixtures/list.py] -[out] -CallExpr(4) : builtins.list[B] -NameExpr(4) : def [-1:A, -2:B] (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] -NameExpr(5) : def (a: A) -> B -CallExpr(6) : A -ListExpr(6) : builtins.list[A] -NameExpr(6) : def () -> A - - --- Lambdas --- ------- - - -[case testLambdaWithTypeInferredFromContext] -from typing import Function, Undefined -f = lambda x: x.a # type: Function[[B], A] -class A: pass -class B: - a = Undefined # type: A -[out] -FuncExpr(2) : def (B) -> A -MemberExpr(2) : A -NameExpr(2) : B - -[case testLambdaWithInferredType] -## FuncExpr|NameExpr -import typing -f = lambda: 1 -[out] -FuncExpr(3) : def () -> builtins.int -NameExpr(3) : def () -> builtins.int - -[case testLambdaWithInferredType2] -## FuncExpr|NameExpr -import typing -f = lambda: [1] -[builtins fixtures/list.py] -[out] -FuncExpr(3) : def () -> builtins.list[builtins.int] -NameExpr(3) : def () -> builtins.list[builtins.int] - -[case testLambdaWithInferredType2] -from typing import List, Function, Undefined -f = lambda x: [] # type: Function[[B], List[A]] -class A: pass -class B: - a = Undefined # type: A -[builtins fixtures/list.py] -[out] -FuncExpr(2) : def (B) -> builtins.list[A] -ListExpr(2) : builtins.list[A] - -[case testLambdaAndHigherOrderFunction] -from typing import typevar, Function, Undefined, List -t = typevar('t') -s = typevar('s') -l = Undefined # type: List[A] -map( - lambda x: f(x), l) -def map(f: Function[[t], s], a: List[t]) -> List[s]: pass -class A: pass -class B: pass -def f(a: A) -> B: pass -[builtins fixtures/list.py] -[out] -CallExpr(5) : builtins.list[B] -NameExpr(5) : def [-1:A, -2:B] (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] -CallExpr(6) : B -FuncExpr(6) : def (A) -> B -NameExpr(6) : def (a: A) -> B -NameExpr(6) : builtins.list[A] -NameExpr(6) : A - -[case testLambdaAndHigherOrderFunction2] -## FuncExpr|NameExpr|ListExpr -from typing import typevar, Undefined, List, Function -t = typevar('t') -s = typevar('s') -l = Undefined # type: List[A] -map( - lambda x: [f(x)], l) -def map(f: Function[[t], List[s]], a: List[t]) -> List[s]: pass -class A: pass -class B: pass -def f(a: A) -> B: pass -[builtins fixtures/list.py] -[out] -NameExpr(6) : def [-1:A, -2:B] (f: def (A) -> builtins.list[B], a: builtins.list[A]) -> builtins.list[B] -FuncExpr(7) : def (A) -> builtins.list[B] -ListExpr(7) : builtins.list[B] -NameExpr(7) : def (a: A) -> B -NameExpr(7) : builtins.list[A] -NameExpr(7) : A - -[case testLambdaInListAndHigherOrderFunction] -from typing import typevar, Undefined, Function, List -t = typevar('t') -s = typevar('s') -l = Undefined # type: List[A] -map( - [lambda x: x], - l) -def map(f: List[Function[[t], s]], a: List[t]) -> List[s]: pass -class A: pass -[builtins fixtures/list.py] -[out] --- TODO We probably should not silently infer 'Any' types in statically typed --- context. Perhaps just fail instead? -CallExpr(5) : builtins.list[Any] -NameExpr(5) : def [-1:A, -2:Any] (f: builtins.list[def (A) -> Any], a: builtins.list[A]) -> builtins.list[Any] -FuncExpr(6) : def (A) -> A -ListExpr(6) : builtins.list[def (A) -> Any] -NameExpr(6) : A -NameExpr(7) : builtins.list[A] - -[case testLambdaAndHigherOrderFunction3] -from typing import typevar, Undefined, Function, List -t = typevar('t') -s = typevar('s') -l = Undefined # type: List[A] -map( - lambda x: x.b, - l) -def map(f: Function[[t], s], a: List[t]) -> List[s]: pass -class A: - b = Undefined # type: B -class B: pass -[builtins fixtures/list.py] -[out] -CallExpr(5) : builtins.list[B] -NameExpr(5) : def [-1:A, -2:B] (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] -FuncExpr(6) : def (A) -> B -MemberExpr(6) : B -NameExpr(6) : A -NameExpr(7) : builtins.list[A] - -[case testLambdaAndHigherOrderFunctionAndKeywordArgs] -from typing import typevar, Function, List, Undefined -t = typevar('t') -s = typevar('s') -l = Undefined # type: List[A] -map( - a=l, - f=lambda x: x.b) -def map(f: Function[[t], s], a: List[t]) -> List[s]: pass -class A: - b = Undefined # type: B -class B: pass -[builtins fixtures/list.py] -[out] -CallExpr(5) : builtins.list[B] -NameExpr(5) : def [-1:A, -2:B] (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] -NameExpr(6) : builtins.list[A] -FuncExpr(7) : def (A) -> B -MemberExpr(7) : B -NameExpr(7) : A - - --- Boolean operations --- ------------------ - - -[case testBooleanOr] -from typing import Undefined, List -a = Undefined # type: List[A] -a or [] -a = a or [] -a = [] or a -class A: pass -[builtins fixtures/list.py] -[out] -ListExpr(3) : builtins.list[A] -NameExpr(3) : builtins.list[A] -OpExpr(3) : builtins.list[A] -ListExpr(4) : builtins.list[A] -NameExpr(4) : builtins.list[A] -NameExpr(4) : builtins.list[A] -OpExpr(4) : builtins.list[A] -ListExpr(5) : builtins.list[A] -NameExpr(5) : builtins.list[A] -NameExpr(5) : builtins.list[A] -OpExpr(5) : builtins.list[A] - - --- Class attributes --- ---------------- - - -[case testUnboundMethod] -## MemberExpr -import typing -class A: - def f(self) -> None: pass -A.f -[out] -MemberExpr(5) : def (self: A) - -[case testUnboundMethodWithImplicitSig] -## MemberExpr -import typing -class A: - def f(self): pass -A.f -[out] -MemberExpr(5) : def (self: Any) -> Any - -[case testOverloadedUnboundMethod] -## MemberExpr -from typing import overload -class A: - @overload - def f(self) -> None: pass - @overload - def f(self, x: object) -> None: pass -A.f -[out] -MemberExpr(8) : Overload(def (self: A), def (self: A, x: builtins.object)) - -[case testOverloadedUnboundMethodWithImplicitSig] -## MemberExpr -from typing import overload -class A: - @overload - def f(self): pass - @overload - def f(self, x): pass -A.f -[out] -MemberExpr(8) : Overload(def (self: Any) -> Any, def (self: Any, x: Any) -> Any) - -[case testUnboundMethodWithInheritance] -## MemberExpr -import typing -class A: - def __init__(self) -> None: pass - def f(self) -> None: pass -class B(A): - pass -B.f -[out] -MemberExpr(8) : def (self: A) - -[case testUnboundGenericMethod] -## MemberExpr -from typing import typevar -t = typevar('t') -class B: pass -class A: - def f(self, x: t) -> None: pass -A.f(A(), B()) -[out] -MemberExpr(7) : def [-1:B] (self: A, x: B) - -[case testUnboundMethodOfGenericClass] -## MemberExpr -from typing import typevar, Generic -t = typevar('t') -class B: pass -class A(Generic[t]): - def f(self, x: t) -> None: pass -A.f -A.f(A[B](), B()) -[out] -MemberExpr(7) : def [t] (self: A[t`1], x: t`1) -MemberExpr(8) : def [1:B] (self: A[B], x: B) - -[case testUnboundOverloadedMethodOfGenericClass] -## CallExpr -from typing import typevar, Generic, overload, Undefined -t = typevar('t') -class B: pass -class A(Generic[t]): - @overload - def f(self, x: t) -> t: pass - @overload - def f(self) -> object: pass -ab, b = Undefined, Undefined # type: (A[B], B) -A.f(ab, b) -[out] -CallExpr(11) : B - -[case testUnboundMethodOfGenericClassWithImplicitSig] -## MemberExpr -from typing import typevar, Generic -t = typevar('t') -class B: pass -class A(Generic[t]): - def f(self, x): pass -A.f(None, None) -[out] -MemberExpr(7) : def (self: Any, x: Any) -> Any - -[case testGenericMethodOfGenericClass] -## MemberExpr -from typing import typevar, Generic, Undefined -t = typevar('t') -s = typevar('s') -class B: pass -class A(Generic[t]): - def f(self, y: s) -> None: pass -ab = Undefined # type: A[B] -o = Undefined # type: object -A.f(ab, o) -[out] -MemberExpr(10) : def [1:B, -1:builtins.object] (self: A[B], y: builtins.object) - - --- Type variables with value restriction --- ------------------------------------- - - -[case testTypeVariableWithValueRestriction] -## NameExpr -from typing import typevar -T = typevar('T', values=(int, str)) -def f(x: T) -> None: pass -f(1) -f('x') -[out] -NameExpr(5) : def [-1:builtins.int] (x: builtins.int) -NameExpr(6) : def [-1:builtins.str] (x: builtins.str) - -[case testTypeVariableWithValueRestrictionAndSubtype] -## NameExpr|CallExpr -from typing import typevar, Undefined -T = typevar('T', values=(int, str)) -def f(x: T) -> T: pass -class S(str): pass -s = Undefined # type: S -f(s) -[out] -CallExpr(7) : builtins.str -NameExpr(7) : def [-1:builtins.str] (x: builtins.str) -> builtins.str -NameExpr(7) : S - - --- Binary operations --- ----------------- - - -[case testBinaryOperatorWithAnyLeftOperand] -## OpExpr -from typing import Any, disjointclass -class B: - def __add__(self, x: int) -> str: pass -@disjointclass(int) -class A: - def __radd__(self, x: B) -> int: pass -Any(1) + A() -B() + A() -[out] -OpExpr(8) : Any -OpExpr(9) : builtins.int - -[case testBinaryOperatorWithAnyRightOperand] -## OpExpr -from typing import Any -class A: - def __add__(self, x: str) -> int: pass -A() + Any(1) -[out] -OpExpr(5) : Any - - --- Function overloading --- -------------------- - - -[case testOverloadedFunctionType] -## CallExpr -from typing import overload -@overload -def f(x: int) -> str: pass -@overload -def f(x: str) -> int: pass -f(1) -f('') -[out] -CallExpr(7) : builtins.str -CallExpr(8) : builtins.int - -[case testOverlappingOverloadedFunctionType] -## CallExpr -from typing import overload, Undefined -class A: pass -class B(A): pass -@overload -def f(x: B) -> B: pass -@overload -def f(x: A) -> A: pass -a = Undefined # type: A -b = Undefined # type: B -f(a) -f(b) -[out] -CallExpr(11) : A -CallExpr(12) : B - - - -[case testOverloadedErasedType] -from typing import Function -from typing import List -from typing import overload -from typing import typevar - -T = typevar("T") -V = typevar("V") - -def fun(s: int) -> int: pass - -def m(fun: Function[[T], V], iter: List[T]) -> None: pass - -nums = [1] # type: List[int] -m(fun, - nums) -[builtins fixtures/list.py] -[out] -IntExpr(13) : builtins.int -ListExpr(13) : builtins.list[builtins.int] -CallExpr(14) : void -NameExpr(14) : def (s: builtins.int) -> builtins.int -NameExpr(14) : def [-1:builtins.int, -2:builtins.int] (fun: def (builtins.int) -> builtins.int, iter: builtins.list[builtins.int]) -NameExpr(15) : builtins.list[builtins.int] - - --- Special cases --- ------------- - - -[case testImplicitDataAttributeInit] -## NameExpr -import typing -class A: - def __init__(self) -> None: - self.x = ( - A()) -[out] -NameExpr(5) : A -NameExpr(6) : def () -> A - -[case testListMultiplicationInContext] -## ListExpr|OpExpr|IntExpr -from typing import List -a = [None] * 3 # type: List[str] -[builtins fixtures/list.py] -[out] -IntExpr(3) : builtins.int -ListExpr(3) : builtins.list[builtins.str] -OpExpr(3) : builtins.list[builtins.str] - - --- TODO --- --- test expressions --- list literal --- tuple literal --- unary minus --- indexing --- super expression --- more complex lambda (multiple arguments etc.) --- list comprehension --- generator expression --- overloads --- other things --- type inference --- default argument value --- for loop variable --- exception variable --- varargs --- generics --- explicit types diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 1b470bc7d5d2..4a80207d3ec7 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -1,115 +1,167 @@ -import sys -import re +from __future__ import annotations + +import contextlib +import difflib import os +import pathlib +import re +import shutil +import sys +import time +from typing import IO, Any, Callable, Iterable, Iterator, Pattern + +# Exporting Suite as alias to TestCase for backwards compatibility +# TODO: avoid aliasing - import and subclass TestCase directly +from unittest import TestCase -from typing import List, Dict +Suite = TestCase # re-exporting -from mypy.myunit import AssertionFailure -from mypy.test import config +import pytest +import mypy.api as api +import mypy.version +from mypy import defaults +from mypy.main import process_options +from mypy.options import Options +from mypy.test.config import test_data_prefix, test_temp_dir +from mypy.test.data import DataDrivenTestCase, DeleteFile, UpdateFile, fix_cobertura_filename + +skip = pytest.mark.skip # AssertStringArraysEqual displays special line alignment helper messages if # the first different line has at least this many characters, MIN_LINE_LENGTH_FOR_ALIGNMENT = 5 -def assert_string_arrays_equal(expected: List[str], actual: List[str], - msg: str) -> None: +def run_mypy(args: list[str]) -> None: + __tracebackhide__ = True + # We must enable site packages even though they could cause problems, + # since stubs for typing_extensions live there. + outval, errval, status = api.run(args + ["--show-traceback", "--no-silence-site-packages"]) + if status != 0: + sys.stdout.write(outval) + sys.stderr.write(errval) + pytest.fail(reason="Sample check failed", pytrace=False) + + +def diff_ranges( + left: list[str], right: list[str] +) -> tuple[list[tuple[int, int]], list[tuple[int, int]]]: + seq = difflib.SequenceMatcher(None, left, right) + # note last triple is a dummy, so don't need to worry + blocks = seq.get_matching_blocks() + + i = 0 + j = 0 + left_ranges = [] + right_ranges = [] + for block in blocks: + # mismatched range + left_ranges.append((i, block.a)) + right_ranges.append((j, block.b)) + + i = block.a + block.size + j = block.b + block.size + + # matched range + left_ranges.append((block.a, i)) + right_ranges.append((block.b, j)) + return left_ranges, right_ranges + + +def render_diff_range( + ranges: list[tuple[int, int]], + content: list[str], + *, + colour: str | None = None, + output: IO[str] = sys.stderr, + indent: int = 2, +) -> None: + for i, line_range in enumerate(ranges): + is_matching = i % 2 == 1 + lines = content[line_range[0] : line_range[1]] + for j, line in enumerate(lines): + if ( + is_matching + # elide the middle of matching blocks + and j >= 3 + and j < len(lines) - 3 + ): + if j == 3: + output.write(" " * indent + "...\n") + continue + + if not is_matching and colour: + output.write(colour) + + output.write(" " * indent + line) + + if not is_matching: + if colour: + output.write("\033[0m") + output.write(" (diff)") + + output.write("\n") + + +def assert_string_arrays_equal( + expected: list[str], actual: list[str], msg: str, *, traceback: bool = False +) -> None: """Assert that two string arrays are equal. Display any differences in a human-readable form. """ - actual = clean_up(actual) - - if actual != expected: - num_skip_start = num_skipped_prefix_lines(expected, actual) - num_skip_end = num_skipped_suffix_lines(expected, actual) - - sys.stderr.write('Expected:\n') - - # If omit some lines at the beginning, indicate it by displaying a line - # with '...'. - if num_skip_start > 0: - sys.stderr.write(' ...\n') - - # Keep track of the first different line. - first_diff = -1 - - # Display only this many first characers of identical lines. - width = 75 - - for i in range(num_skip_start, len(expected) - num_skip_end): - if i >= len(actual) or expected[i] != actual[i]: - if first_diff < 0: - first_diff = i - sys.stderr.write(' {:<45} (diff)'.format(expected[i])) - else: - e = expected[i] - sys.stderr.write(' ' + e[:width]) - if len(e) > width: - sys.stderr.write('...') - sys.stderr.write('\n') - if num_skip_end > 0: - sys.stderr.write(' ...\n') - - sys.stderr.write('Actual:\n') - - if num_skip_start > 0: - sys.stderr.write(' ...\n') - - for j in range(num_skip_start, len(actual) - num_skip_end): - if j >= len(expected) or expected[j] != actual[j]: - sys.stderr.write(' {:<45} (diff)'.format(actual[j])) - else: - a = actual[j] - sys.stderr.write(' ' + a[:width]) - if len(a) > width: - sys.stderr.write('...') - sys.stderr.write('\n') - if actual == []: - sys.stderr.write(' (empty)\n') - if num_skip_end > 0: - sys.stderr.write(' ...\n') - - sys.stderr.write('\n') - - if first_diff >= 0 and first_diff < len(actual) and ( - len(expected[first_diff]) >= MIN_LINE_LENGTH_FOR_ALIGNMENT - or len(actual[first_diff]) >= MIN_LINE_LENGTH_FOR_ALIGNMENT): + if expected != actual: + expected_ranges, actual_ranges = diff_ranges(expected, actual) + sys.stderr.write("Expected:\n") + red = "\033[31m" if sys.platform != "win32" else None + render_diff_range(expected_ranges, expected, colour=red) + sys.stderr.write("Actual:\n") + green = "\033[32m" if sys.platform != "win32" else None + render_diff_range(actual_ranges, actual, colour=green) + + sys.stderr.write("\n") + first_diff = next( + (i for i, (a, b) in enumerate(zip(expected, actual)) if a != b), + max(len(expected), len(actual)), + ) + if 0 <= first_diff < len(actual) and ( + len(expected[first_diff]) >= MIN_LINE_LENGTH_FOR_ALIGNMENT + or len(actual[first_diff]) >= MIN_LINE_LENGTH_FOR_ALIGNMENT + ): # Display message that helps visualize the differences between two # long lines. show_align_message(expected[first_diff], actual[first_diff]) - raise AssertionFailure(msg) + sys.stderr.write( + "Update the test output using --update-data " + "(implies -n0; you can additionally use the -k selector to update only specific tests)\n" + ) + pytest.fail(msg, pytrace=traceback) -def update_testcase_output(testcase, output, append): - newfile = testcase.file + append - data_lines = open(testcase.file).read().splitlines() - test = '\n'.join(data_lines[testcase.line:testcase.lastline]) +def assert_module_equivalence(name: str, expected: Iterable[str], actual: Iterable[str]) -> None: + expected_normalized = sorted(expected) + actual_normalized = sorted(set(actual).difference({"__main__"})) + assert_string_arrays_equal( + expected_normalized, + actual_normalized, + ('Actual modules ({}) do not match expected modules ({}) for "[{} ...]"').format( + ", ".join(actual_normalized), ", ".join(expected_normalized), name + ), + ) - mapping = {} # type: Dict[str, List[str]] - for old, new in zip(testcase.output, output): - ind = old.index(':') - if ind != -1 and old[:ind] == new[:ind]: - old, new = old[ind:], new[ind:] - mapping.setdefault(old, []).append(new) - for old in mapping: - if test.count(old) == len(mapping[old]): - betweens = test.split(old) - - #interleave betweens and mapping[old] - from itertools import chain - interleaved = [betweens[0]] + \ - list(chain.from_iterable(zip(mapping[old], betweens[1:]))) - test = ''.join(interleaved) - - data_lines[testcase.line:testcase.lastline] = [test] - data = '\n'.join(data_lines) - with open(newfile, 'w') as f: - print(data, file=f) +def assert_target_equivalence(name: str, expected: list[str], actual: list[str]) -> None: + """Compare actual and expected targets (order sensitive).""" + assert_string_arrays_equal( + expected, + actual, + ('Actual targets ({}) do not match expected targets ({}) for "[{} ...]"').format( + ", ".join(actual), ", ".join(expected), name + ), + ) def show_align_message(s1: str, s2: str) -> None: @@ -133,7 +185,7 @@ def show_align_message(s1: str, s2: str) -> None: maxw = 72 # Maximum number of characters shown - sys.stderr.write('Alignment of first line difference:\n') + sys.stderr.write("Alignment of first line difference:\n") trunc = False while s1[:30] == s2[:30]: @@ -142,132 +194,278 @@ def show_align_message(s1: str, s2: str) -> None: trunc = True if trunc: - s1 = '...' + s1 - s2 = '...' + s2 + s1 = "..." + s1 + s2 = "..." + s2 max_len = max(len(s1), len(s2)) - extra = '' + extra = "" if max_len > maxw: - extra = '...' + extra = "..." # Write a chunk of both lines, aligned. - sys.stderr.write(' E: {}{}\n'.format(s1[:maxw], extra)) - sys.stderr.write(' A: {}{}\n'.format(s2[:maxw], extra)) + sys.stderr.write(f" E: {s1[:maxw]}{extra}\n") + sys.stderr.write(f" A: {s2[:maxw]}{extra}\n") # Write an indicator character under the different columns. - sys.stderr.write(' ') + sys.stderr.write(" ") for j in range(min(maxw, max(len(s1), len(s2)))): - if s1[j:j + 1] != s2[j:j + 1]: - sys.stderr.write('^') # Difference + if s1[j : j + 1] != s2[j : j + 1]: + sys.stderr.write("^") # Difference break else: - sys.stderr.write(' ') # Equal - sys.stderr.write('\n') - - -def assert_string_arrays_equal_wildcards(expected: List[str], - actual: List[str], - msg: str) -> None: - # Like above, but let a line with only '...' in expected match any number - # of lines in actual. - actual = clean_up(actual) + sys.stderr.write(" ") # Equal + sys.stderr.write("\n") - while actual != [] and actual[-1] == '': - actual = actual[:-1] - # Expand "..." wildcards away. - expected = match_array(expected, actual) - assert_string_arrays_equal(expected, actual, msg) - - -def clean_up(a): +def clean_up(a: list[str]) -> list[str]: """Remove common directory prefix from all strings in a. This uses a naive string replace; it seems to work well enough. Also remove trailing carriage returns. """ res = [] + pwd = os.getcwd() + driver = pwd + "/driver.py" for s in a: - prefix = config.PREFIX + os.sep + prefix = os.sep ss = s - for p in prefix, prefix.replace(os.sep, '/'): - if p != '/' and p != '//' and p != '\\' and p != '\\\\': - ss = ss.replace(p, '') + for p in prefix, prefix.replace(os.sep, "/"): + if p != "/" and p != "//" and p != "\\" and p != "\\\\": + ss = ss.replace(p, "") # Ignore spaces at end of line. - ss = re.sub(' +$', '', ss) - res.append(re.sub('\\r$', '', ss)) + ss = re.sub(" +$", "", ss) + # Remove pwd from driver.py's path + ss = ss.replace(driver, "driver.py") + res.append(re.sub("\\r$", "", ss)) return res -def match_array(pattern: List[str], target: List[str]) -> List[str]: - """Expand '...' wildcards in pattern by matching against target.""" +@contextlib.contextmanager +def local_sys_path_set() -> Iterator[None]: + """Temporary insert current directory into sys.path. - res = [] # type: List[str] - i = 0 - j = 0 + This can be used by test cases that do runtime imports, for example + by the stubgen tests. + """ + old_sys_path = sys.path.copy() + if not ("" in sys.path or "." in sys.path): + sys.path.insert(0, "") + try: + yield + finally: + sys.path = old_sys_path + + +def testfile_pyversion(path: str) -> tuple[int, int]: + m = re.search(r"python3([0-9]+)\.test$", path) + if m: + return 3, int(m.group(1)) + else: + return defaults.PYTHON3_VERSION - while i < len(pattern): - if pattern[i] == '...': - # Wildcard in pattern. - if i + 1 == len(pattern): - # Wildcard at end of pattern; match the rest of target. - res.extend(target[j:]) - # Finished. - break - else: - # Must find the instance of the next pattern line in target. - jj = j - while jj < len(target): - if target[jj] == pattern[i + 1]: - break - jj += 1 - if jj == len(target): - # No match. Get out. - res.extend(pattern[i:]) - break - res.extend(target[j:jj]) - i += 1 - j = jj - elif (j < len(target) and (pattern[i] == target[j] - or (i + 1 < len(pattern) - and j + 1 < len(target) - and pattern[i + 1] == target[j + 1]))): - # In sync; advance one line. The above condition keeps sync also if - # only a single line is different, but loses it if two consecutive - # lines fail to match. - res.append(pattern[i]) - i += 1 - j += 1 - else: - # Out of sync. Get out. - res.extend(pattern[i:]) - break - return res +def normalize_error_messages(messages: list[str]) -> list[str]: + """Translate an array of error messages to use / as path separator.""" -def num_skipped_prefix_lines(a1: List[str], a2: List[str]) -> int: - num_eq = 0 - while num_eq < min(len(a1), len(a2)) and a1[num_eq] == a2[num_eq]: - num_eq += 1 - return max(0, num_eq - 4) + a = [] + for m in messages: + a.append(m.replace(os.sep, "/")) + return a -def num_skipped_suffix_lines(a1: List[str], a2: List[str]) -> int: - num_eq = 0 - while (num_eq < min(len(a1), len(a2)) - and a1[-num_eq - 1] == a2[-num_eq - 1]): - num_eq += 1 - return max(0, num_eq - 4) +def retry_on_error(func: Callable[[], Any], max_wait: float = 1.0) -> None: + """Retry callback with exponential backoff when it raises OSError. + If the function still generates an error after max_wait seconds, propagate + the exception. -def testfile_pyversion(path: str) -> int: - if path.endswith('python2.test'): - return 2 + This can be effective against random file system operation failures on + Windows. + """ + t0 = time.time() + wait_time = 0.01 + while True: + try: + func() + return + except OSError: + wait_time = min(wait_time * 2, t0 + max_wait - time.time()) + if wait_time <= 0.01: + # Done enough waiting, the error seems persistent. + raise + time.sleep(wait_time) + + +def good_repr(obj: object) -> str: + if isinstance(obj, str): + if obj.count("\n") > 1: + bits = ["'''\\"] + for line in obj.split("\n"): + # force repr to use ' not ", then cut it off + bits.append(repr('"' + line)[2:-1]) + bits[-1] += "'''" + return "\n".join(bits) + return repr(obj) + + +def assert_equal(a: object, b: object, fmt: str = "{} != {}") -> None: + __tracebackhide__ = True + if a != b: + raise AssertionError(fmt.format(good_repr(a), good_repr(b))) + + +def typename(t: type) -> str: + if "." in str(t): + return str(t).split(".")[-1].rstrip("'>") else: - return 3 - - -def testcase_pyversion(path: str, testcase_name: str) -> int: - if testcase_name.endswith('python2'): - return 2 + return str(t)[8:-2] + + +def assert_type(typ: type, value: object) -> None: + __tracebackhide__ = True + if type(value) != typ: + raise AssertionError(f"Invalid type {typename(type(value))}, expected {typename(typ)}") + + +def parse_options( + program_text: str, testcase: DataDrivenTestCase, incremental_step: int +) -> Options: + """Parse comments like '# flags: --foo' in a test case.""" + options = Options() + flags = re.search("# flags: (.*)$", program_text, flags=re.MULTILINE) + if incremental_step > 1: + flags2 = re.search(f"# flags{incremental_step}: (.*)$", program_text, flags=re.MULTILINE) + if flags2: + flags = flags2 + + if flags: + flag_list = flags.group(1).split() + flag_list.append("--no-site-packages") # the tests shouldn't need an installed Python + targets, options = process_options(flag_list, require_targets=False) + if targets: + # TODO: support specifying targets via the flags pragma + raise RuntimeError("Specifying targets via the flags pragma is not supported.") + if "--show-error-codes" not in flag_list: + options.hide_error_codes = True else: - return testfile_pyversion(path) + flag_list = [] + options = Options() + options.error_summary = False + options.hide_error_codes = True + options.force_uppercase_builtins = True + options.force_union_syntax = True + + # Allow custom python version to override testfile_pyversion. + if all(flag.split("=")[0] != "--python-version" for flag in flag_list): + options.python_version = testfile_pyversion(testcase.file) + + if testcase.config.getoption("--mypy-verbose"): + options.verbosity = testcase.config.getoption("--mypy-verbose") + + return options + + +def split_lines(*streams: bytes) -> list[str]: + """Returns a single list of string lines from the byte streams in args.""" + return [s for stream in streams for s in stream.decode("utf8").splitlines()] + + +def write_and_fudge_mtime(content: str, target_path: str) -> None: + # In some systems, mtime has a resolution of 1 second which can + # cause annoying-to-debug issues when a file has the same size + # after a change. We manually set the mtime to circumvent this. + # Note that we increment the old file's mtime, which guarantees a + # different value, rather than incrementing the mtime after the + # copy, which could leave the mtime unchanged if the old file had + # a similarly fudged mtime. + new_time = None + if os.path.isfile(target_path): + new_time = os.stat(target_path).st_mtime + 1 + + dir = os.path.dirname(target_path) + os.makedirs(dir, exist_ok=True) + with open(target_path, "w", encoding="utf-8") as target: + target.write(content) + + if new_time: + os.utime(target_path, times=(new_time, new_time)) + + +def perform_file_operations(operations: list[UpdateFile | DeleteFile]) -> None: + for op in operations: + if isinstance(op, UpdateFile): + # Modify/create file + write_and_fudge_mtime(op.content, op.target_path) + else: + # Delete file/directory + if os.path.isdir(op.path): + # Sanity check to avoid unexpected deletions + assert op.path.startswith("tmp") + shutil.rmtree(op.path) + else: + # Use retries to work around potential flakiness on Windows (AppVeyor). + path = op.path + retry_on_error(lambda: os.remove(path)) + + +def check_test_output_files( + testcase: DataDrivenTestCase, step: int, strip_prefix: str = "" +) -> None: + for path, expected_content in testcase.output_files: + if path.startswith(strip_prefix): + path = path[len(strip_prefix) :] + if not os.path.exists(path): + raise AssertionError( + "Expected file {} was not produced by test case{}".format( + path, " on step %d" % step if testcase.output2 else "" + ) + ) + with open(path, encoding="utf8") as output_file: + actual_output_content = output_file.read() + + if isinstance(expected_content, Pattern): + if expected_content.fullmatch(actual_output_content) is not None: + continue + raise AssertionError( + "Output file {} did not match its expected output pattern\n---\n{}\n---".format( + path, actual_output_content + ) + ) + + normalized_output = normalize_file_output( + actual_output_content.splitlines(), os.path.abspath(test_temp_dir) + ) + # We always normalize things like timestamp, but only handle operating-system + # specific things if requested. + if testcase.normalize_output: + if testcase.suite.native_sep and os.path.sep == "\\": + normalized_output = [fix_cobertura_filename(line) for line in normalized_output] + normalized_output = normalize_error_messages(normalized_output) + assert_string_arrays_equal( + expected_content.splitlines(), + normalized_output, + "Output file {} did not match its expected output{}".format( + path, " on step %d" % step if testcase.output2 else "" + ), + ) + + +def normalize_file_output(content: list[str], current_abs_path: str) -> list[str]: + """Normalize file output for comparison.""" + timestamp_regex = re.compile(r"\d{10}") + result = [x.replace(current_abs_path, "$PWD") for x in content] + version = mypy.version.__version__ + result = [re.sub(r"\b" + re.escape(version) + r"\b", "$VERSION", x) for x in result] + # We generate a new mypy.version when building mypy wheels that + # lacks base_version, so handle that case. + base_version = getattr(mypy.version, "base_version", version) + result = [re.sub(r"\b" + re.escape(base_version) + r"\b", "$VERSION", x) for x in result] + result = [timestamp_regex.sub("$TIMESTAMP", x) for x in result] + return result + + +def find_test_files(pattern: str, exclude: list[str] | None = None) -> list[str]: + return [ + path.name + for path in (pathlib.Path(test_data_prefix).rglob(pattern)) + if path.name not in (exclude or []) + ] diff --git a/mypy/codec/test/__init__.py b/mypy/test/meta/__init__.py similarity index 100% rename from mypy/codec/test/__init__.py rename to mypy/test/meta/__init__.py diff --git a/mypy/test/meta/_pytest.py b/mypy/test/meta/_pytest.py new file mode 100644 index 000000000000..b8648f033143 --- /dev/null +++ b/mypy/test/meta/_pytest.py @@ -0,0 +1,72 @@ +import shlex +import subprocess +import sys +import textwrap +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from mypy.test.config import test_data_prefix + + +@dataclass +class PytestResult: + input: str + input_updated: str # any updates made by --update-data + stdout: str + stderr: str + + +def dedent_docstring(s: str) -> str: + return textwrap.dedent(s).lstrip() + + +def run_pytest_data_suite( + data_suite: str, + *, + data_file_prefix: str = "check", + pytest_node_prefix: str = "mypy/test/testcheck.py::TypeCheckSuite", + extra_args: Iterable[str], + max_attempts: int, +) -> PytestResult: + """ + Runs a suite of data test cases through pytest until either tests pass + or until a maximum number of attempts (needed for incremental tests). + + :param data_suite: the actual "suite" i.e. the contents of a .test file + """ + p_test_data = Path(test_data_prefix) + p_root = p_test_data.parent.parent + p = p_test_data / f"{data_file_prefix}-meta-{uuid.uuid4()}.test" + assert not p.exists() + data_suite = dedent_docstring(data_suite) + try: + p.write_text(data_suite) + + test_nodeid = f"{pytest_node_prefix}::{p.name}" + extra_args = [sys.executable, "-m", "pytest", "-n", "0", "-s", *extra_args, test_nodeid] + cmd = shlex.join(extra_args) + for i in range(max_attempts - 1, -1, -1): + print(f">> {cmd}") + proc = subprocess.run(extra_args, capture_output=True, check=False, cwd=p_root) + if proc.returncode == 0: + break + prefix = "NESTED PYTEST STDOUT" + for line in proc.stdout.decode().splitlines(): + print(f"{prefix}: {line}") + prefix = " " * len(prefix) + prefix = "NESTED PYTEST STDERR" + for line in proc.stderr.decode().splitlines(): + print(f"{prefix}: {line}") + prefix = " " * len(prefix) + print(f"Exit code {proc.returncode} ({i} attempts remaining)") + + return PytestResult( + input=data_suite, + input_updated=p.read_text(), + stdout=proc.stdout.decode(), + stderr=proc.stderr.decode(), + ) + finally: + p.unlink() diff --git a/mypy/test/meta/test_diff_helper.py b/mypy/test/meta/test_diff_helper.py new file mode 100644 index 000000000000..047751fee1d2 --- /dev/null +++ b/mypy/test/meta/test_diff_helper.py @@ -0,0 +1,47 @@ +import io + +from mypy.test.helpers import Suite, diff_ranges, render_diff_range + + +class DiffHelperSuite(Suite): + def test_render_diff_range(self) -> None: + expected = ["hello", "world"] + actual = ["goodbye", "world"] + + expected_ranges, actual_ranges = diff_ranges(expected, actual) + + output = io.StringIO() + render_diff_range(expected_ranges, expected, output=output) + assert output.getvalue() == " hello (diff)\n world\n" + output = io.StringIO() + render_diff_range(actual_ranges, actual, output=output) + assert output.getvalue() == " goodbye (diff)\n world\n" + + expected = ["a", "b", "c", "d", "e", "f", "g", "h", "circle", "i", "j"] + actual = ["a", "b", "c", "d", "e", "f", "g", "h", "square", "i", "j"] + + expected_ranges, actual_ranges = diff_ranges(expected, actual) + + output = io.StringIO() + render_diff_range(expected_ranges, expected, output=output, indent=0) + assert output.getvalue() == "a\nb\nc\n...\nf\ng\nh\ncircle (diff)\ni\nj\n" + output = io.StringIO() + render_diff_range(actual_ranges, actual, output=output, indent=0) + assert output.getvalue() == "a\nb\nc\n...\nf\ng\nh\nsquare (diff)\ni\nj\n" + + def test_diff_ranges(self) -> None: + a = ["hello", "world"] + b = ["hello", "world"] + + assert diff_ranges(a, b) == ( + [(0, 0), (0, 2), (2, 2), (2, 2)], + [(0, 0), (0, 2), (2, 2), (2, 2)], + ) + + a = ["hello", "world"] + b = ["goodbye", "world"] + + assert diff_ranges(a, b) == ( + [(0, 1), (1, 2), (2, 2), (2, 2)], + [(0, 1), (1, 2), (2, 2), (2, 2)], + ) diff --git a/mypy/test/meta/test_parse_data.py b/mypy/test/meta/test_parse_data.py new file mode 100644 index 000000000000..bff2d6977612 --- /dev/null +++ b/mypy/test/meta/test_parse_data.py @@ -0,0 +1,73 @@ +""" +A "meta test" which tests the parsing of .test files. This is not meant to become exhaustive +but to ensure we maintain a basic level of ergonomics for mypy contributors. +""" + +from mypy.test.helpers import Suite +from mypy.test.meta._pytest import PytestResult, run_pytest_data_suite + + +def _run_pytest(data_suite: str) -> PytestResult: + return run_pytest_data_suite(data_suite, extra_args=[], max_attempts=1) + + +class ParseTestDataSuite(Suite): + def test_parse_invalid_case(self) -> None: + # Act + result = _run_pytest( + """ + [case abc] + s: str + [case foo-XFAIL] + s: str + """ + ) + + # Assert + assert "Invalid testcase id 'foo-XFAIL'" in result.stdout + + def test_parse_invalid_section(self) -> None: + # Act + result = _run_pytest( + """ + [case abc] + s: str + [unknownsection] + abc + """ + ) + + # Assert + expected_lineno = result.input.splitlines().index("[unknownsection]") + 1 + expected = ( + f".test:{expected_lineno}: Invalid section header [unknownsection] in case 'abc'" + ) + assert expected in result.stdout + + def test_bad_ge_version_check(self) -> None: + # Act + actual = _run_pytest( + """ + [case abc] + s: str + [out version>=3.8] + abc + """ + ) + + # Assert + assert "version>=3.8 always true since minimum runtime version is (3, 8)" in actual.stdout + + def test_bad_eq_version_check(self) -> None: + # Act + actual = _run_pytest( + """ + [case abc] + s: str + [out version==3.7] + abc + """ + ) + + # Assert + assert "version==3.7 always false since minimum runtime version is (3, 8)" in actual.stdout diff --git a/mypy/test/meta/test_update_data.py b/mypy/test/meta/test_update_data.py new file mode 100644 index 000000000000..820fd359893e --- /dev/null +++ b/mypy/test/meta/test_update_data.py @@ -0,0 +1,135 @@ +""" +A "meta test" which tests the `--update-data` feature for updating .test files. +Updating the expected output, especially when it's in the form of inline (comment) assertions, +can be brittle, which is why we're "meta-testing" here. +""" + +from mypy.test.helpers import Suite +from mypy.test.meta._pytest import PytestResult, dedent_docstring, run_pytest_data_suite + + +def _run_pytest_update_data(data_suite: str) -> PytestResult: + """ + Runs a suite of data test cases through 'pytest --update-data' until either tests pass + or until a maximum number of attempts (needed for incremental tests). + """ + return run_pytest_data_suite(data_suite, extra_args=["--update-data"], max_attempts=3) + + +class UpdateDataSuite(Suite): + def test_update_data(self) -> None: + # Note: We test multiple testcases rather than 'test case per test case' + # so we could also exercise rewriting multiple testcases at once. + result = _run_pytest_update_data( + """ + [case testCorrect] + s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testWrong] + s: str = 42 # E: wrong error + + [case testXfail-xfail] + s: str = 42 # E: wrong error + + [case testWrongMultiline] + s: str = 42 # E: foo \ + # N: bar + + [case testMissingMultiline] + s: str = 42; i: int = 'foo' + + [case testExtraneous] + s: str = 'foo' # E: wrong error + + [case testExtraneousMultiline] + s: str = 'foo' # E: foo \ + # E: bar + + [case testExtraneousMultilineNonError] + s: str = 'foo' # W: foo \ + # N: bar + + [case testOutCorrect] + s: str = 42 + [out] + main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testOutWrong] + s: str = 42 + [out] + main:1: error: foobar + + [case testOutWrongIncremental] + s: str = 42 + [out] + main:1: error: foobar + [out2] + main:1: error: foobar + + [case testWrongMultipleFiles] + import a, b + s: str = 42 # E: foo + [file a.py] + s1: str = 42 # E: bar + [file b.py] + s2: str = 43 # E: baz + [builtins fixtures/list.pyi] + """ + ) + + # Assert + expected = dedent_docstring( + """ + [case testCorrect] + s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testWrong] + s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testXfail-xfail] + s: str = 42 # E: wrong error + + [case testWrongMultiline] + s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testMissingMultiline] + s: str = 42; i: int = 'foo' # E: Incompatible types in assignment (expression has type "int", variable has type "str") \\ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + + [case testExtraneous] + s: str = 'foo' + + [case testExtraneousMultiline] + s: str = 'foo' + + [case testExtraneousMultilineNonError] + s: str = 'foo' + + [case testOutCorrect] + s: str = 42 + [out] + main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testOutWrong] + s: str = 42 + [out] + main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testOutWrongIncremental] + s: str = 42 + [out] + main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + [out2] + main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + + [case testWrongMultipleFiles] + import a, b + s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + [file a.py] + s1: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + [file b.py] + s2: str = 43 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + [builtins fixtures/list.pyi] + """ + ) + assert result.input_updated == expected diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py new file mode 100644 index 000000000000..21ba0903a824 --- /dev/null +++ b/mypy/test/test_find_sources.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest + +import pytest + +from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list +from mypy.fscache import FileSystemCache +from mypy.modulefinder import BuildSource +from mypy.options import Options + + +class FakeFSCache(FileSystemCache): + def __init__(self, files: set[str]) -> None: + self.files = {os.path.abspath(f) for f in files} + + def isfile(self, file: str) -> bool: + return file in self.files + + def isdir(self, dir: str) -> bool: + if not dir.endswith(os.sep): + dir += os.sep + return any(f.startswith(dir) for f in self.files) + + def listdir(self, dir: str) -> list[str]: + if not dir.endswith(os.sep): + dir += os.sep + return list({f[len(dir) :].split(os.sep)[0] for f in self.files if f.startswith(dir)}) + + def init_under_package_root(self, file: str) -> bool: + return False + + +def normalise_path(path: str) -> str: + path = os.path.splitdrive(path)[1] + path = path.replace(os.sep, "/") + return path + + +def normalise_build_source_list(sources: list[BuildSource]) -> list[tuple[str, str | None]]: + return sorted( + (s.module, (normalise_path(s.base_dir) if s.base_dir is not None else None)) + for s in sources + ) + + +def crawl(finder: SourceFinder, f: str) -> tuple[str, str]: + module, base_dir = finder.crawl_up(f) + return module, normalise_path(base_dir) + + +def find_sources_in_dir(finder: SourceFinder, f: str) -> list[tuple[str, str | None]]: + return normalise_build_source_list(finder.find_sources_in_dir(os.path.abspath(f))) + + +def find_sources( + paths: list[str], options: Options, fscache: FileSystemCache +) -> list[tuple[str, str | None]]: + paths = [os.path.abspath(p) for p in paths] + return normalise_build_source_list(create_source_list(paths, options, fscache)) + + +class SourceFinderSuite(unittest.TestCase): + def setUp(self) -> None: + self.tempdir = tempfile.mkdtemp() + self.oldcwd = os.getcwd() + os.chdir(self.tempdir) + + def tearDown(self) -> None: + os.chdir(self.oldcwd) + shutil.rmtree(self.tempdir) + + def test_crawl_no_namespace(self) -> None: + options = Options() + options.namespace_packages = False + + finder = SourceFinder(FakeFSCache({"/setup.py"}), options) + assert crawl(finder, "/setup.py") == ("setup", "/") + + finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options) + assert crawl(finder, "/a/setup.py") == ("setup", "/a") + + finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options) + assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b") + + finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/setup.py") == ("a.setup", "/") + + finder = SourceFinder(FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name") + + finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b") + + finder = SourceFinder( + FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options + ) + assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b") + + def test_crawl_namespace(self) -> None: + options = Options() + options.namespace_packages = True + + finder = SourceFinder(FakeFSCache({"/setup.py"}), options) + assert crawl(finder, "/setup.py") == ("setup", "/") + + finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options) + assert crawl(finder, "/a/setup.py") == ("setup", "/a") + + finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options) + assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b") + + finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/setup.py") == ("a.setup", "/") + + finder = SourceFinder(FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name") + + finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/b/setup.py") == ("a.b.setup", "/") + + finder = SourceFinder( + FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options + ) + assert crawl(finder, "/a/b/c/setup.py") == ("a.b.c.setup", "/") + + def test_crawl_namespace_explicit_base(self) -> None: + options = Options() + options.namespace_packages = True + options.explicit_package_bases = True + + finder = SourceFinder(FakeFSCache({"/setup.py"}), options) + assert crawl(finder, "/setup.py") == ("setup", "/") + + finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options) + assert crawl(finder, "/a/setup.py") == ("setup", "/a") + + finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options) + assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b") + + finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/setup.py") == ("a.setup", "/") + + finder = SourceFinder(FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name") + + finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options) + assert crawl(finder, "/a/b/setup.py") == ("a.b.setup", "/") + + finder = SourceFinder( + FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options + ) + assert crawl(finder, "/a/b/c/setup.py") == ("a.b.c.setup", "/") + + # set mypy path, so we actually have some explicit base dirs + options.mypy_path = ["/a/b"] + + finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options) + assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b") + + finder = SourceFinder( + FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}), options + ) + assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b") + + options.mypy_path = ["/a/b", "/a/b/c"] + finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options) + assert crawl(finder, "/a/b/c/setup.py") == ("setup", "/a/b/c") + + def test_crawl_namespace_multi_dir(self) -> None: + options = Options() + options.namespace_packages = True + options.explicit_package_bases = True + options.mypy_path = ["/a", "/b"] + + finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options) + assert crawl(finder, "/a/pkg/a.py") == ("pkg.a", "/a") + assert crawl(finder, "/b/pkg/b.py") == ("pkg.b", "/b") + + def test_find_sources_in_dir_no_namespace(self) -> None: + options = Options() + options.namespace_packages = False + + files = { + "/pkg/a1/b/c/d/e.py", + "/pkg/a1/b/f.py", + "/pkg/a2/__init__.py", + "/pkg/a2/b/c/d/e.py", + "/pkg/a2/b/f.py", + } + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources_in_dir(finder, "/") == [ + ("a2", "/pkg"), + ("e", "/pkg/a1/b/c/d"), + ("e", "/pkg/a2/b/c/d"), + ("f", "/pkg/a1/b"), + ("f", "/pkg/a2/b"), + ] + + def test_find_sources_in_dir_namespace(self) -> None: + options = Options() + options.namespace_packages = True + + files = { + "/pkg/a1/b/c/d/e.py", + "/pkg/a1/b/f.py", + "/pkg/a2/__init__.py", + "/pkg/a2/b/c/d/e.py", + "/pkg/a2/b/f.py", + } + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources_in_dir(finder, "/") == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ("e", "/pkg/a1/b/c/d"), + ("f", "/pkg/a1/b"), + ] + + def test_find_sources_in_dir_namespace_explicit_base(self) -> None: + options = Options() + options.namespace_packages = True + options.explicit_package_bases = True + options.mypy_path = ["/"] + + files = { + "/pkg/a1/b/c/d/e.py", + "/pkg/a1/b/f.py", + "/pkg/a2/__init__.py", + "/pkg/a2/b/c/d/e.py", + "/pkg/a2/b/f.py", + } + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources_in_dir(finder, "/") == [ + ("pkg.a1.b.c.d.e", "/"), + ("pkg.a1.b.f", "/"), + ("pkg.a2", "/"), + ("pkg.a2.b.c.d.e", "/"), + ("pkg.a2.b.f", "/"), + ] + + options.mypy_path = ["/pkg"] + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources_in_dir(finder, "/") == [ + ("a1.b.c.d.e", "/pkg"), + ("a1.b.f", "/pkg"), + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + + def test_find_sources_in_dir_namespace_multi_dir(self) -> None: + options = Options() + options.namespace_packages = True + options.explicit_package_bases = True + options.mypy_path = ["/a", "/b"] + + finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options) + assert find_sources_in_dir(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] + + def test_find_sources_exclude(self) -> None: + options = Options() + options.namespace_packages = True + + # default + for excluded_dir in ["site-packages", ".whatever", "node_modules", ".x/.z"]: + fscache = FakeFSCache({"/dir/a.py", f"/dir/venv/{excluded_dir}/b.py"}) + assert find_sources(["/"], options, fscache) == [("a", "/dir")] + with pytest.raises(InvalidSourceList): + find_sources(["/dir/venv/"], options, fscache) + assert find_sources([f"/dir/venv/{excluded_dir}"], options, fscache) == [ + ("b", f"/dir/venv/{excluded_dir}") + ] + assert find_sources([f"/dir/venv/{excluded_dir}/b.py"], options, fscache) == [ + ("b", f"/dir/venv/{excluded_dir}") + ] + + files = { + "/pkg/a1/b/c/d/e.py", + "/pkg/a1/b/f.py", + "/pkg/a2/__init__.py", + "/pkg/a2/b/c/d/e.py", + "/pkg/a2/b/f.py", + } + + # file name + options.exclude = [r"/f\.py$"] + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("e", "/pkg/a1/b/c/d"), + ] + assert find_sources(["/pkg/a1/b/f.py"], options, fscache) == [("f", "/pkg/a1/b")] + assert find_sources(["/pkg/a2/b/f.py"], options, fscache) == [("a2.b.f", "/pkg")] + + # directory name + options.exclude = ["/a1/"] + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1"], options, fscache) + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1/"], options, fscache) + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1/b"], options, fscache) + + options.exclude = ["/a1/$"] + assert find_sources(["/pkg/a1"], options, fscache) == [ + ("e", "/pkg/a1/b/c/d"), + ("f", "/pkg/a1/b"), + ] + + # paths + options.exclude = ["/pkg/a1/"] + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1"], options, fscache) + + # OR two patterns together + for orred in [["/(a1|a3)/"], ["a1", "a3"], ["a3", "a1"]]: + options.exclude = orred + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + + options.exclude = ["b/c/"] + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.f", "/pkg"), + ("f", "/pkg/a1/b"), + ] + + # nothing should be ignored as a result of this + big_exclude1 = [ + "/pkg/a/", + "/2", + "/1", + "/pk/", + "/kg", + "/g.py", + "/bc", + "/xxx/pkg/a2/b/f.py", + "xxx/pkg/a2/b/f.py", + ] + big_exclude2 = ["|".join(big_exclude1)] + for big_exclude in [big_exclude1, big_exclude2]: + options.exclude = big_exclude + fscache = FakeFSCache(files) + assert len(find_sources(["/"], options, fscache)) == len(files) + + files = { + "pkg/a1/b/c/d/e.py", + "pkg/a1/b/f.py", + "pkg/a2/__init__.py", + "pkg/a2/b/c/d/e.py", + "pkg/a2/b/f.py", + } + fscache = FakeFSCache(files) + assert len(find_sources(["."], options, fscache)) == len(files) diff --git a/mypy/test/test_ref_info.py b/mypy/test/test_ref_info.py new file mode 100644 index 000000000000..05052e491657 --- /dev/null +++ b/mypy/test/test_ref_info.py @@ -0,0 +1,45 @@ +"""Test exporting line-level reference information (undocumented feature)""" + +from __future__ import annotations + +import json +import os +import sys + +from mypy import build +from mypy.modulefinder import BuildSource +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal + + +class RefInfoSuite(DataSuite): + required_out_section = True + files = ["ref-info.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + options = Options() + options.use_builtins_fixtures = True + options.show_traceback = True + options.export_ref_info = True # This is the flag we are testing + + src = "\n".join(testcase.input) + result = build.build( + sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir + ) + assert not result.errors + + major, minor = sys.version_info[:2] + ref_path = os.path.join(options.cache_dir, f"{major}.{minor}", "__main__.refs.json") + + with open(ref_path) as refs_file: + data = json.load(refs_file) + + a = [] + for item in data: + a.append(f"{item['line']}:{item['column']}:{item['target']}") + + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) diff --git a/mypy/test/testapi.py b/mypy/test/testapi.py new file mode 100644 index 000000000000..95bd95ece785 --- /dev/null +++ b/mypy/test/testapi.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import sys +from io import StringIO + +import mypy.api +from mypy.test.helpers import Suite + + +class APISuite(Suite): + def setUp(self) -> None: + self.sys_stdout = sys.stdout + self.sys_stderr = sys.stderr + sys.stdout = self.stdout = StringIO() + sys.stderr = self.stderr = StringIO() + + def tearDown(self) -> None: + sys.stdout = self.sys_stdout + sys.stderr = self.sys_stderr + assert self.stdout.getvalue() == "" + assert self.stderr.getvalue() == "" + + def test_capture_bad_opt(self) -> None: + """stderr should be captured when a bad option is passed.""" + _, stderr, _ = mypy.api.run(["--some-bad-option"]) + assert isinstance(stderr, str) + assert stderr != "" + + def test_capture_empty(self) -> None: + """stderr should be captured when a bad option is passed.""" + _, stderr, _ = mypy.api.run([]) + assert isinstance(stderr, str) + assert stderr != "" + + def test_capture_help(self) -> None: + """stdout should be captured when --help is passed.""" + stdout, _, _ = mypy.api.run(["--help"]) + assert isinstance(stdout, str) + assert stdout != "" + + def test_capture_version(self) -> None: + """stdout should be captured when --version is passed.""" + stdout, _, _ = mypy.api.run(["--version"]) + assert isinstance(stdout, str) + assert stdout != "" diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py new file mode 100644 index 000000000000..7c139902fe90 --- /dev/null +++ b/mypy/test/testargs.py @@ -0,0 +1,77 @@ +"""Ensure the argparse parser and Options class are in sync. + +In particular, verify that the argparse defaults are the same as the Options +defaults, and that argparse doesn't assign any new members to the Options +object it creates. +""" + +from __future__ import annotations + +import argparse +import sys + +from mypy.main import infer_python_executable, process_options +from mypy.options import Options +from mypy.test.helpers import Suite, assert_equal + + +class ArgSuite(Suite): + def test_coherence(self) -> None: + options = Options() + _, parsed_options = process_options([], require_targets=False) + # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' + options.config_file = parsed_options.config_file + assert_equal(options.snapshot(), parsed_options.snapshot()) + + def test_executable_inference(self) -> None: + """Test the --python-executable flag with --python-version""" + sys_ver_str = "{ver.major}.{ver.minor}".format(ver=sys.version_info) + + base = ["file.py"] # dummy file + + # test inference given one (infer the other) + matching_version = base + [f"--python-version={sys_ver_str}"] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + matching_version = base + [f"--python-executable={sys.executable}"] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test inference given both + matching_version = base + [ + f"--python-version={sys_ver_str}", + f"--python-executable={sys.executable}", + ] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test that --no-site-packages will disable executable inference + matching_version = base + [f"--python-version={sys_ver_str}", "--no-site-packages"] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable is None + + # Test setting python_version/executable from config file + special_opts = argparse.Namespace() + special_opts.python_executable = None + special_opts.python_version = None + special_opts.no_executable = None + + # first test inferring executable from version + options = Options() + options.python_executable = None + options.python_version = sys.version_info[:2] + infer_python_executable(options, special_opts) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # then test inferring version from executable + options = Options() + options.python_executable = sys.executable + infer_python_executable(options, special_opts) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 84ba2202cf69..330e191af252 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -1,84 +1,324 @@ """Type checker test cases""" -import os.path +from __future__ import annotations -import typing +import os +import re +import sys from mypy import build -from mypy.myunit import Suite, run_test -from mypy.test.config import test_temp_dir, test_data_prefix -from mypy.test.data import parse_test_cases -from mypy.test.helpers import assert_string_arrays_equal, testcase_pyversion, update_testcase_output -from mypy.test.testsemanal import normalize_error_messages +from mypy.build import Graph from mypy.errors import CompileError +from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths +from mypy.test.config import test_data_prefix, test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path +from mypy.test.helpers import ( + assert_module_equivalence, + assert_string_arrays_equal, + assert_target_equivalence, + check_test_output_files, + find_test_files, + normalize_error_messages, + parse_options, + perform_file_operations, +) +from mypy.test.update_data import update_testcase_output + +try: + import lxml # type: ignore[import-untyped] +except ImportError: + lxml = None -import sys -APPEND_TESTCASES = '.new' if '-u' in sys.argv else '' -UPDATE_TESTCASES = '-i' in sys.argv or '-u' in sys.argv + +import pytest # List of files that contain test case descriptions. -files = [ - 'check-basic.test', - 'check-classes.test', - 'check-expressions.test', - 'check-statements.test', - 'check-generics.test', - 'check-tuples.test', - 'check-dynamic-typing.test', - 'check-functions.test', - 'check-inference.test', - 'check-inference-context.test', - 'check-varargs.test', - 'check-kwargs.test', - 'check-overloading.test', - 'check-type-checks.test', - 'check-abstract.test', - 'check-multiple-inheritance.test', - 'check-super.test', - 'check-modules.test', - 'check-generic-subtyping.test', - 'check-typevar-values.test', - 'check-python2.test', - 'check-unsupported.test', - 'check-unreachable-code.test', - 'check-unions.test', - 'check-isinstance.test', - 'check-lists.test', - 'check-namedtuple.test', -] - - -class TypeCheckSuite(Suite): - def cases(self): - c = [] - for f in files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - self.run_test, test_temp_dir, True) - return c - - def run_test(self, testcase): - a = [] - pyversion = testcase_pyversion(testcase.file, testcase.name) +# Includes all check-* files with the .test extension in the test-data/unit directory +typecheck_files = find_test_files(pattern="check-*.test") + +# Tests that use Python version specific features: +if sys.version_info < (3, 9): + typecheck_files.remove("check-python39.test") +if sys.version_info < (3, 10): + typecheck_files.remove("check-python310.test") +if sys.version_info < (3, 11): + typecheck_files.remove("check-python311.test") +if sys.version_info < (3, 12): + typecheck_files.remove("check-python312.test") +if sys.version_info < (3, 13): + typecheck_files.remove("check-python313.test") + +# Special tests for platforms with case-insensitive filesystems. +if sys.platform not in ("darwin", "win32"): + typecheck_files.remove("check-modules-case.test") + + +class TypeCheckSuite(DataSuite): + files = typecheck_files + + def run_case(self, testcase: DataDrivenTestCase) -> None: + if lxml is None and os.path.basename(testcase.file) == "check-reports.test": + pytest.skip("Cannot import lxml. Is it installed?") + incremental = ( + "incremental" in testcase.name.lower() + or "incremental" in testcase.file + or "serialize" in testcase.file + ) + if incremental: + # Incremental tests are run once with a cold cache, once with a warm cache. + # Expect success on first run, errors from testcase.output (if any) on second run. + num_steps = max([2] + list(testcase.output2.keys())) + # Check that there are no file changes beyond the last run (they would be ignored). + for dn, dirs, files in os.walk(os.curdir): + for file in files: + m = re.search(r"\.([2-9])$", file) + if m and int(m.group(1)) > num_steps: + raise ValueError( + "Output file {} exists though test case only has {} runs".format( + file, num_steps + ) + ) + steps = testcase.find_steps() + for step in range(1, num_steps + 1): + idx = step - 2 + ops = steps[idx] if idx < len(steps) and idx >= 0 else [] + self.run_case_once(testcase, ops, step) + else: + self.run_case_once(testcase) + + def _sort_output_if_needed(self, testcase: DataDrivenTestCase, a: list[str]) -> None: + idx = testcase.output_inline_start + if not testcase.files or idx == len(testcase.output): + return + + def _filename(_msg: str) -> str: + return _msg.partition(":")[0] + + file_weights = {file: idx for idx, file in enumerate(_filename(msg) for msg in a)} + testcase.output[idx:] = sorted( + testcase.output[idx:], key=lambda msg: file_weights.get(_filename(msg), -1) + ) + + def run_case_once( + self, + testcase: DataDrivenTestCase, + operations: list[FileOperation] | None = None, + incremental_step: int = 0, + ) -> None: + if operations is None: + operations = [] + original_program_text = "\n".join(testcase.input) + module_data = self.parse_module(original_program_text, incremental_step) + + # Unload already loaded plugins, they may be updated. + for file, _ in testcase.files: + module = module_from_path(file) + if module.endswith("_plugin") and module in sys.modules: + del sys.modules[module] + if incremental_step == 0 or incremental_step == 1: + # In run 1, copy program text to program file. + for module_name, program_path, program_text in module_data: + if module_name == "__main__": + with open(program_path, "w", encoding="utf8") as f: + f.write(program_text) + break + elif incremental_step > 1: + # In runs 2+, copy *.[num] files to * files. + perform_file_operations(operations) + + # Parse options after moving files (in case mypy.ini is being moved). + options = parse_options(original_program_text, testcase, incremental_step) + options.use_builtins_fixtures = True + options.show_traceback = True + + # Enable some options automatically based on test file name. + if "columns" in testcase.file: + options.show_column_numbers = True + if "errorcodes" in testcase.file: + options.hide_error_codes = False + if "abstract" not in testcase.file: + options.allow_empty_bodies = not testcase.name.endswith("_no_empty") + if "lowercase" not in testcase.file: + options.force_uppercase_builtins = True + if "union-error" not in testcase.file: + options.force_union_syntax = True + + if incremental_step and options.incremental: + # Don't overwrite # flags: --no-incremental in incremental test cases + options.incremental = True + else: + options.incremental = False + # Don't waste time writing cache unless we are specifically looking for it + if not testcase.writescache: + options.cache_dir = os.devnull + + sources = [] + for module_name, program_path, program_text in module_data: + # Always set to none so we're forced to reread the module in incremental mode + sources.append( + BuildSource(program_path, module_name, None if incremental_step else program_text) + ) + + plugin_dir = os.path.join(test_data_prefix, "plugins") + sys.path.insert(0, plugin_dir) + + res = None try: - src = '\n'.join(testcase.input) - build.build('main', - target=build.TYPE_CHECK, - program_text=src, - pyversion=pyversion, - flags=[build.TEST_BUILTINS], - alt_lib_path=test_temp_dir) + res = build.build(sources=sources, options=options, alt_lib_path=test_temp_dir) + a = res.errors except CompileError as e: - a = normalize_error_messages(e.messages) + a = e.messages + finally: + assert sys.path[0] == plugin_dir + del sys.path[0] + + if testcase.normalize_output: + a = normalize_error_messages(a) + + # Make sure error messages match + if incremental_step < 2: + if incremental_step == 1: + msg = "Unexpected type checker output in incremental, run 1 ({}, line {})" + else: + assert incremental_step == 0 + msg = "Unexpected type checker output ({}, line {})" + self._sort_output_if_needed(testcase, a) + output = testcase.output + else: + msg = ( + f"Unexpected type checker output in incremental, run {incremental_step}" + + " ({}, line {})" + ) + output = testcase.output2.get(incremental_step, []) + + if output != a and testcase.config.getoption("--update-data", False): + update_testcase_output(testcase, a, incremental_step=incremental_step) + + assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line)) + + if res: + if options.cache_dir != os.devnull: + self.verify_cache(module_data, res.errors, res.manager, res.graph) + + name = "targets" + if incremental_step: + name += str(incremental_step + 1) + expected = testcase.expected_fine_grained_targets.get(incremental_step + 1) + actual = [ + target + for module, target in res.manager.processed_targets + if module in testcase.test_modules + ] + if expected is not None: + assert_target_equivalence(name, expected, actual) + if incremental_step > 1: + suffix = "" if incremental_step == 2 else str(incremental_step - 1) + expected_rechecked = testcase.expected_rechecked_modules.get(incremental_step - 1) + if expected_rechecked is not None: + assert_module_equivalence( + "rechecked" + suffix, expected_rechecked, res.manager.rechecked_modules + ) + expected_stale = testcase.expected_stale_modules.get(incremental_step - 1) + if expected_stale is not None: + assert_module_equivalence( + "stale" + suffix, expected_stale, res.manager.stale_modules + ) + + if testcase.output_files: + check_test_output_files(testcase, incremental_step, strip_prefix="tmp/") + + def verify_cache( + self, + module_data: list[tuple[str, str, str]], + a: list[str], + manager: build.BuildManager, + graph: Graph, + ) -> None: + # There should be valid cache metadata for each module except + # for those that had an error in themselves or one of their + # dependencies. + error_paths = self.find_error_message_paths(a) + busted_paths = {m.path for id, m in manager.modules.items() if graph[id].transitive_error} + modules = self.find_module_files(manager) + modules.update({module_name: path for module_name, path, text in module_data}) + missing_paths = self.find_missing_cache_files(modules, manager) + # We would like to assert error_paths.issubset(busted_paths) + # but this runs into trouble because while some 'notes' are + # really errors that cause an error to be marked, many are + # just notes attached to other errors. + assert error_paths or not busted_paths, "Some modules reported error despite no errors" + if not missing_paths == busted_paths: + raise AssertionError(f"cache data discrepancy {missing_paths} != {busted_paths}") + assert os.path.isfile(os.path.join(manager.options.cache_dir, ".gitignore")) + cachedir_tag = os.path.join(manager.options.cache_dir, "CACHEDIR.TAG") + assert os.path.isfile(cachedir_tag) + with open(cachedir_tag) as f: + assert f.read().startswith("Signature: 8a477f597d28d172789f06886806bc55") + + def find_error_message_paths(self, a: list[str]) -> set[str]: + hits = set() + for line in a: + m = re.match(r"([^\s:]+):(\d+:)?(\d+:)? (error|warning|note):", line) + if m: + p = m.group(1) + hits.add(p) + return hits + + def find_module_files(self, manager: build.BuildManager) -> dict[str, str]: + return {id: module.path for id, module in manager.modules.items()} + + def find_missing_cache_files( + self, modules: dict[str, str], manager: build.BuildManager + ) -> set[str]: + ignore_errors = True + missing = {} + for id, path in modules.items(): + meta = build.find_cache_meta(id, path, manager) + if not build.validate_meta(meta, id, path, ignore_errors, manager): + missing[id] = path + return set(missing.values()) + + def parse_module( + self, program_text: str, incremental_step: int = 0 + ) -> list[tuple[str, str, str]]: + """Return the module and program names for a test case. + + Normally, the unit tests will parse the default ('__main__') + module and follow all the imports listed there. You can override + this behavior and instruct the tests to check multiple modules + by using a comment like this in the test case input: - if testcase.output != a and UPDATE_TESTCASES: - update_testcase_output(testcase, a, APPEND_TESTCASES) + # cmd: mypy -m foo.bar foo.baz - assert_string_arrays_equal( - testcase.output, a, - 'Invalid type checker output ({}, line {})'.format( - testcase.file, testcase.line)) + You can also use `# cmdN:` to have a different cmd for incremental + step N (2, 3, ...). + Return a list of tuples (module name, file name, program text). + """ + m = re.search("# cmd: mypy -m ([a-zA-Z0-9_. ]+)$", program_text, flags=re.MULTILINE) + if incremental_step > 1: + alt_regex = f"# cmd{incremental_step}: mypy -m ([a-zA-Z0-9_. ]+)$" + alt_m = re.search(alt_regex, program_text, flags=re.MULTILINE) + if alt_m is not None: + # Optionally return a different command if in a later step + # of incremental mode, otherwise default to reusing the + # original cmd. + m = alt_m -if __name__ == '__main__': - import sys - run_test(TypeCheckSuite(), sys.argv[1:]) + if m: + # The test case wants to use a non-default main + # module. Look up the module and give it as the thing to + # analyze. + module_names = m.group(1) + out = [] + search_paths = SearchPaths((test_temp_dir,), (), (), ()) + cache = FindModuleCache(search_paths, fscache=None, options=None) + for module_name in module_names.split(" "): + path = cache.find_module(module_name) + assert isinstance(path, str), f"Can't find ad hoc case file: {module_name}" + with open(path, encoding="utf8") as f: + program_text = f.read() + out.append((module_name, path, program_text)) + return out + else: + return [("__main__", "main", program_text)] diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py new file mode 100644 index 000000000000..9bc02d319964 --- /dev/null +++ b/mypy/test/testcmdline.py @@ -0,0 +1,152 @@ +"""Test cases for the command line. + +To begin we test that "mypy [/]" always recurses down the +whole tree. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys + +from mypy.test.config import PREFIX, test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import ( + assert_string_arrays_equal, + check_test_output_files, + normalize_error_messages, +) + +try: + import lxml # type: ignore[import-untyped] +except ImportError: + lxml = None + +import pytest + +# Path to Python 3 interpreter +python3_path = sys.executable + +# Files containing test case descriptions. +cmdline_files = ["cmdline.test", "cmdline.pyproject.test", "reports.test", "envvars.test"] + + +class PythonCmdlineSuite(DataSuite): + files = cmdline_files + native_sep = True + + def run_case(self, testcase: DataDrivenTestCase) -> None: + if lxml is None and os.path.basename(testcase.file) == "reports.test": + pytest.skip("Cannot import lxml. Is it installed?") + for step in [1] + sorted(testcase.output2): + test_python_cmdline(testcase, step) + + +def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None: + assert testcase.old_cwd is not None, "test was not properly set up" + # Write the program to a file. + program = "_program.py" + program_path = os.path.join(test_temp_dir, program) + with open(program_path, "w", encoding="utf8") as file: + for s in testcase.input: + file.write(f"{s}\n") + args = parse_args(testcase.input[0]) + custom_cwd = parse_cwd(testcase.input[1]) if len(testcase.input) > 1 else None + args.append("--show-traceback") + if "--error-summary" not in args: + args.append("--no-error-summary") + if "--show-error-codes" not in args: + args.append("--hide-error-codes") + if "--disallow-empty-bodies" not in args: + args.append("--allow-empty-bodies") + if "--no-force-uppercase-builtins" not in args: + args.append("--force-uppercase-builtins") + if "--no-force-union-syntax" not in args: + args.append("--force-union-syntax") + # Type check the program. + fixed = [python3_path, "-m", "mypy"] + env = os.environ.copy() + env.pop("COLUMNS", None) + extra_path = os.path.join(os.path.abspath(test_temp_dir), "pypath") + env["PYTHONPATH"] = PREFIX + if os.path.isdir(extra_path): + env["PYTHONPATH"] += os.pathsep + extra_path + cwd = os.path.join(test_temp_dir, custom_cwd or "") + args = [arg.replace("$CWD", os.path.abspath(cwd)) for arg in args] + process = subprocess.Popen( + fixed + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env + ) + outb, errb = process.communicate() + result = process.returncode + # Split output into lines. + out = [s.rstrip("\n\r") for s in str(outb, "utf8").splitlines()] + err = [s.rstrip("\n\r") for s in str(errb, "utf8").splitlines()] + + if "PYCHARM_HOSTED" in os.environ: + for pos, line in enumerate(err): + if line.startswith("pydev debugger: "): + # Delete the attaching debugger message itself, plus the extra newline added. + del err[pos : pos + 2] + break + + # Remove temp file. + os.remove(program_path) + # Compare actual output to expected. + if testcase.output_files: + # Ignore stdout, but we insist on empty stderr and zero status. + if err or result: + raise AssertionError( + "Expected zero status and empty stderr%s, got %d and\n%s" + % (" on step %d" % step if testcase.output2 else "", result, "\n".join(err + out)) + ) + check_test_output_files(testcase, step) + else: + if testcase.normalize_output: + out = normalize_error_messages(err + out) + obvious_result = 1 if out else 0 + if obvious_result != result: + out.append(f"== Return code: {result}") + expected_out = testcase.output if step == 1 else testcase.output2[step] + # Strip "tmp/" out of the test so that # E: works... + expected_out = [s.replace("tmp" + os.sep, "") for s in expected_out] + assert_string_arrays_equal( + expected_out, + out, + "Invalid output ({}, line {}){}".format( + testcase.file, testcase.line, " on step %d" % step if testcase.output2 else "" + ), + ) + + +def parse_args(line: str) -> list[str]: + """Parse the first line of the program for the command line. + + This should have the form + + # cmd: mypy + + For example: + + # cmd: mypy pkg/ + """ + m = re.match("# cmd: mypy (.*)$", line) + if not m: + return [] # No args; mypy will spit out an error. + return m.group(1).split() + + +def parse_cwd(line: str) -> str | None: + """Parse the second line of the program for the command line. + + This should have the form + + # cwd: + + For example: + + # cwd: main/subdir + """ + m = re.match("# cwd: (.*)$", line) + return m.group(1) if m else None diff --git a/mypy/test/testconstraints.py b/mypy/test/testconstraints.py new file mode 100644 index 000000000000..a701a173cbaa --- /dev/null +++ b/mypy/test/testconstraints.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from mypy.constraints import SUBTYPE_OF, SUPERTYPE_OF, Constraint, infer_constraints +from mypy.test.helpers import Suite +from mypy.test.typefixture import TypeFixture +from mypy.types import Instance, TupleType, UnpackType + + +class ConstraintsSuite(Suite): + def setUp(self) -> None: + self.fx = TypeFixture() + + def test_no_type_variables(self) -> None: + assert not infer_constraints(self.fx.o, self.fx.o, SUBTYPE_OF) + + def test_basic_type_variable(self) -> None: + fx = self.fx + for direction in [SUBTYPE_OF, SUPERTYPE_OF]: + assert infer_constraints(fx.gt, fx.ga, direction) == [ + Constraint(type_var=fx.t, op=direction, target=fx.a) + ] + + def test_basic_type_var_tuple_subtype(self) -> None: + fx = self.fx + assert infer_constraints( + Instance(fx.gvi, [UnpackType(fx.ts)]), Instance(fx.gvi, [fx.a, fx.b]), SUBTYPE_OF + ) == [ + Constraint(type_var=fx.ts, op=SUBTYPE_OF, target=TupleType([fx.a, fx.b], fx.std_tuple)) + ] + + def test_basic_type_var_tuple(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gvi, [UnpackType(fx.ts)]), Instance(fx.gvi, [fx.a, fx.b]), SUPERTYPE_OF + ) + ) == { + Constraint( + type_var=fx.ts, op=SUPERTYPE_OF, target=TupleType([fx.a, fx.b], fx.std_tuple) + ), + Constraint( + type_var=fx.ts, op=SUBTYPE_OF, target=TupleType([fx.a, fx.b], fx.std_tuple) + ), + } + + def test_type_var_tuple_with_prefix_and_suffix(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gv2i, [fx.t, UnpackType(fx.ts), fx.s]), + Instance(fx.gv2i, [fx.a, fx.b, fx.c, fx.d]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint( + type_var=fx.ts, op=SUPERTYPE_OF, target=TupleType([fx.b, fx.c], fx.std_tuple) + ), + Constraint( + type_var=fx.ts, op=SUBTYPE_OF, target=TupleType([fx.b, fx.c], fx.std_tuple) + ), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.d), + } + + def test_unpack_homogenous_tuple(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gvi, [UnpackType(Instance(fx.std_tuplei, [fx.t]))]), + Instance(fx.gvi, [fx.a, fx.b]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.t, op=SUBTYPE_OF, target=fx.a), + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.t, op=SUBTYPE_OF, target=fx.b), + } + + def test_unpack_homogenous_tuple_with_prefix_and_suffix(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gv2i, [fx.t, UnpackType(Instance(fx.std_tuplei, [fx.s])), fx.u]), + Instance(fx.gv2i, [fx.a, fx.b, fx.c, fx.d]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.s, op=SUBTYPE_OF, target=fx.b), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.c), + Constraint(type_var=fx.s, op=SUBTYPE_OF, target=fx.c), + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), + } + + def test_unpack_with_prefix_and_suffix(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gv2i, [fx.u, fx.t, fx.s, fx.u]), + Instance(fx.gv2i, [fx.a, fx.b, fx.c, fx.d]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.t, op=SUBTYPE_OF, target=fx.b), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.c), + Constraint(type_var=fx.s, op=SUBTYPE_OF, target=fx.c), + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), + } + + def test_unpack_tuple_length_non_match(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gv2i, [fx.u, fx.t, fx.s, fx.u]), + Instance(fx.gv2i, [fx.a, fx.b, fx.d]), + SUPERTYPE_OF, + ) + # We still get constraints on the prefix/suffix in this case. + ) == { + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), + } + + def test_var_length_tuple_with_fixed_length_tuple(self) -> None: + fx = self.fx + assert not infer_constraints( + TupleType([fx.t, fx.s], fallback=Instance(fx.std_tuplei, [fx.o])), + Instance(fx.std_tuplei, [fx.a]), + SUPERTYPE_OF, + ) diff --git a/mypy/test/testdaemon.py b/mypy/test/testdaemon.py new file mode 100644 index 000000000000..7115e682e60d --- /dev/null +++ b/mypy/test/testdaemon.py @@ -0,0 +1,132 @@ +"""End-to-end test cases for the daemon (dmypy). + +These are special because they run multiple shell commands. + +This also includes some unit tests. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest + +from mypy.dmypy_server import filter_out_missing_top_level_packages +from mypy.fscache import FileSystemCache +from mypy.modulefinder import SearchPaths +from mypy.test.config import PREFIX, test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages + +# Files containing test cases descriptions. +daemon_files = ["daemon.test"] + + +class DaemonSuite(DataSuite): + files = daemon_files + + def run_case(self, testcase: DataDrivenTestCase) -> None: + try: + test_daemon(testcase) + finally: + # Kill the daemon if it's still running. + run_cmd("dmypy kill") + + +def test_daemon(testcase: DataDrivenTestCase) -> None: + assert testcase.old_cwd is not None, "test was not properly set up" + for i, step in enumerate(parse_script(testcase.input)): + cmd = step[0] + expected_lines = step[1:] + assert cmd.startswith("$") + cmd = cmd[1:].strip() + cmd = cmd.replace("{python}", sys.executable) + sts, output = run_cmd(cmd) + output_lines = output.splitlines() + output_lines = normalize_error_messages(output_lines) + if sts: + output_lines.append("== Return code: %d" % sts) + assert_string_arrays_equal( + expected_lines, + output_lines, + "Command %d (%s) did not give expected output" % (i + 1, cmd), + ) + + +def parse_script(input: list[str]) -> list[list[str]]: + """Parse testcase.input into steps. + + Each command starts with a line starting with '$'. + The first line (less '$') is sent to the shell. + The remaining lines are expected output. + """ + steps = [] + step: list[str] = [] + for line in input: + if line.startswith("$"): + if step: + assert step[0].startswith("$") + steps.append(step) + step = [] + step.append(line) + if step: + steps.append(step) + return steps + + +def run_cmd(input: str) -> tuple[int, str]: + if input[1:].startswith("mypy run --") and "--show-error-codes" not in input: + input += " --hide-error-codes" + if input.startswith("dmypy "): + input = sys.executable + " -m mypy." + input + if input.startswith("mypy "): + input = sys.executable + " -m" + input + env = os.environ.copy() + env["PYTHONPATH"] = PREFIX + try: + output = subprocess.check_output( + input, shell=True, stderr=subprocess.STDOUT, text=True, cwd=test_temp_dir, env=env + ) + return 0, output + except subprocess.CalledProcessError as err: + return err.returncode, err.output + + +class DaemonUtilitySuite(unittest.TestCase): + """Unit tests for helpers""" + + def test_filter_out_missing_top_level_packages(self) -> None: + with tempfile.TemporaryDirectory() as td: + self.make_file(td, "base/a/") + self.make_file(td, "base/b.py") + self.make_file(td, "base/c.pyi") + self.make_file(td, "base/missing.txt") + self.make_file(td, "typeshed/d.pyi") + self.make_file(td, "typeshed/@python2/e") # outdated + self.make_file(td, "pkg1/f-stubs") + self.make_file(td, "pkg2/g-python2-stubs") # outdated + self.make_file(td, "mpath/sub/long_name/") + + def makepath(p: str) -> str: + return os.path.join(td, p) + + search = SearchPaths( + python_path=(makepath("base"),), + mypy_path=(makepath("mpath/sub"),), + package_path=(makepath("pkg1"), makepath("pkg2")), + typeshed_path=(makepath("typeshed"),), + ) + fscache = FileSystemCache() + res = filter_out_missing_top_level_packages( + {"a", "b", "c", "d", "e", "f", "g", "long_name", "ff", "missing"}, search, fscache + ) + assert res == {"a", "b", "c", "d", "f", "long_name"} + + def make_file(self, base: str, path: str) -> None: + fullpath = os.path.join(base, path) + os.makedirs(os.path.dirname(fullpath), exist_ok=True) + if not path.endswith("/"): + with open(fullpath, "w") as f: + f.write("# test file") diff --git a/mypy/test/testdeps.py b/mypy/test/testdeps.py new file mode 100644 index 000000000000..7c845eab8b57 --- /dev/null +++ b/mypy/test/testdeps.py @@ -0,0 +1,82 @@ +"""Test cases for generating node-level dependencies (for fine-grained incremental checking)""" + +from __future__ import annotations + +import os +import sys +from collections import defaultdict + +import pytest + +from mypy import build +from mypy.errors import CompileError +from mypy.modulefinder import BuildSource +from mypy.nodes import Expression, MypyFile +from mypy.options import Options +from mypy.server.deps import get_dependencies +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, find_test_files, parse_options +from mypy.types import Type +from mypy.typestate import type_state + +# Only dependencies in these modules are dumped +dumped_modules = ["__main__", "pkg", "pkg.mod"] + + +class GetDependenciesSuite(DataSuite): + files = find_test_files(pattern="deps*.test") + + def run_case(self, testcase: DataDrivenTestCase) -> None: + src = "\n".join(testcase.input) + dump_all = "# __dump_all__" in src + options = parse_options(src, testcase, incremental_step=1) + if options.python_version > sys.version_info: + pytest.skip("Test case requires a newer Python version") + options.use_builtins_fixtures = True + options.show_traceback = True + options.cache_dir = os.devnull + options.export_types = True + options.preserve_asts = True + options.allow_empty_bodies = True + messages, files, type_map = self.build(src, options) + a = messages + if files is None or type_map is None: + if not a: + a = ["Unknown compile error (likely syntax error in test case or fixture)"] + else: + deps: defaultdict[str, set[str]] = defaultdict(set) + for module, file in files.items(): + if (module in dumped_modules or dump_all) and (module in testcase.test_modules): + new_deps = get_dependencies(file, type_map, options.python_version, options) + for source in new_deps: + deps[source].update(new_deps[source]) + + type_state.add_all_protocol_deps(deps) + + for source, targets in sorted(deps.items()): + if source.startswith((" {', '.join(sorted(targets))}" + # Clean up output a bit + line = line.replace("__main__", "m") + a.append(line) + + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) + + def build( + self, source: str, options: Options + ) -> tuple[list[str], dict[str, MypyFile] | None, dict[Expression, Type] | None]: + try: + result = build.build( + sources=[BuildSource("main", None, source)], + options=options, + alt_lib_path=test_temp_dir, + ) + except CompileError as e: + # TODO: Should perhaps not return None here. + return e.messages, None, None + return result.errors, result.files, result.types diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py new file mode 100644 index 000000000000..0559b33c33e2 --- /dev/null +++ b/mypy/test/testdiff.py @@ -0,0 +1,70 @@ +"""Test cases for AST diff (used for fine-grained incremental checking)""" + +from __future__ import annotations + +import os +import sys + +import pytest + +from mypy import build +from mypy.errors import CompileError +from mypy.modulefinder import BuildSource +from mypy.nodes import MypyFile +from mypy.options import Options +from mypy.server.astdiff import compare_symbol_table_snapshots, snapshot_symbol_table +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, parse_options + + +class ASTDiffSuite(DataSuite): + files = ["diff.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + first_src = "\n".join(testcase.input) + files_dict = dict(testcase.files) + second_src = files_dict["tmp/next.py"] + options = parse_options(first_src, testcase, 1) + if options.python_version > sys.version_info: + pytest.skip("Test case requires a newer Python version") + + messages1, files1 = self.build(first_src, options) + messages2, files2 = self.build(second_src, options) + + a = [] + if messages1: + a.extend(messages1) + if messages2: + a.append("== next ==") + a.extend(messages2) + + assert ( + files1 is not None and files2 is not None + ), "cases where CompileError occurred should not be run" + prefix = "__main__" + snapshot1 = snapshot_symbol_table(prefix, files1["__main__"].names) + snapshot2 = snapshot_symbol_table(prefix, files2["__main__"].names) + diff = compare_symbol_table_snapshots(prefix, snapshot1, snapshot2) + for trigger in sorted(diff): + a.append(trigger) + + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) + + def build(self, source: str, options: Options) -> tuple[list[str], dict[str, MypyFile] | None]: + options.use_builtins_fixtures = True + options.show_traceback = True + options.cache_dir = os.devnull + options.allow_empty_bodies = True + try: + result = build.build( + sources=[BuildSource("main", None, source)], + options=options, + alt_lib_path=test_temp_dir, + ) + except CompileError as e: + # TODO: Is it okay to return None? + return e.messages, None + return result.errors, result.files diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py new file mode 100644 index 000000000000..a54a3495ddb2 --- /dev/null +++ b/mypy/test/testerrorstream.py @@ -0,0 +1,46 @@ +"""Tests for mypy incremental error output.""" + +from __future__ import annotations + +from mypy import build +from mypy.errors import CompileError +from mypy.modulefinder import BuildSource +from mypy.options import Options +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal + + +class ErrorStreamSuite(DataSuite): + required_out_section = True + base_path = "." + files = ["errorstream.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_error_stream(testcase) + + +def test_error_stream(testcase: DataDrivenTestCase) -> None: + """Perform a single error streaming test case. + + The argument contains the description of the test case. + """ + options = Options() + options.show_traceback = True + options.hide_error_codes = True + + logged_messages: list[str] = [] + + def flush_errors(filename: str | None, msgs: list[str], serious: bool) -> None: + if msgs: + logged_messages.append("==== Errors flushed ====") + logged_messages.extend(msgs) + + sources = [BuildSource("main", "__main__", "\n".join(testcase.input))] + try: + build.build(sources=sources, options=options, flush_errors=flush_errors) + except CompileError as e: + assert e.messages == [] + + assert_string_arrays_equal( + testcase.output, logged_messages, f"Invalid output ({testcase.file}, line {testcase.line})" + ) diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py new file mode 100644 index 000000000000..cb8672dfaf29 --- /dev/null +++ b/mypy/test/testfinegrained.py @@ -0,0 +1,442 @@ +"""Test cases for fine-grained incremental checking. + +Each test cases runs a batch build followed by one or more fine-grained +incremental steps. We verify that each step produces the expected output. + +See the comment at the top of test-data/unit/fine-grained.test for more +information. + +N.B.: Unlike most of the other test suites, testfinegrained does not +rely on an alt_lib_path for finding source files. This means that they +can test interactions with the lib_path that is built implicitly based +on specified sources. +""" + +from __future__ import annotations + +import os +import re +import sys +import unittest +from typing import Any + +import pytest + +from mypy import build +from mypy.config_parser import parse_config_file +from mypy.dmypy_server import Server +from mypy.dmypy_util import DEFAULT_STATUS_FILE +from mypy.errors import CompileError +from mypy.find_sources import create_source_list +from mypy.modulefinder import BuildSource +from mypy.options import Options +from mypy.server.mergecheck import check_consistency +from mypy.server.update import sort_messages_preserving_file_order +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite, DeleteFile, UpdateFile +from mypy.test.helpers import ( + assert_module_equivalence, + assert_string_arrays_equal, + assert_target_equivalence, + find_test_files, + parse_options, + perform_file_operations, +) + +# Set to True to perform (somewhat expensive) checks for duplicate AST nodes after merge +CHECK_CONSISTENCY = False + + +class FineGrainedSuite(DataSuite): + files = find_test_files( + pattern="fine-grained*.test", exclude=["fine-grained-cache-incremental.test"] + ) + + # Whether to use the fine-grained cache in the testing. This is overridden + # by a trivial subclass to produce a suite that uses the cache. + use_cache = False + + def should_skip(self, testcase: DataDrivenTestCase) -> bool: + # Decide whether to skip the test. This could have been structured + # as a filter() classmethod also, but we want the tests reported + # as skipped, not just elided. + if self.use_cache: + if testcase.only_when == "-only_when_nocache": + return True + # TODO: In caching mode we currently don't well support + # starting from cached states with errors in them. + if testcase.output and testcase.output[0] != "==": + return True + else: + if testcase.only_when == "-only_when_cache": + return True + return False + + def run_case(self, testcase: DataDrivenTestCase) -> None: + if self.should_skip(testcase): + pytest.skip() + return + + main_src = "\n".join(testcase.input) + main_path = os.path.join(test_temp_dir, "main") + with open(main_path, "w", encoding="utf8") as f: + f.write(main_src) + + options = self.get_options(main_src, testcase, build_cache=False) + if options.python_version > sys.version_info: + pytest.skip("Test case requires a newer Python version") + + build_options = self.get_options(main_src, testcase, build_cache=True) + server = Server(options, DEFAULT_STATUS_FILE) + + num_regular_incremental_steps = self.get_build_steps(main_src) + step = 1 + sources = self.parse_sources(main_src, step, options) + if step <= num_regular_incremental_steps: + messages = self.build(build_options, sources) + else: + messages = self.run_check(server, sources) + + a = [] + if messages: + a.extend(normalize_messages(messages)) + + assert testcase.tmpdir is not None + a.extend(self.maybe_suggest(step, server, main_src, testcase.tmpdir)) + a.extend(self.maybe_inspect(step, server, main_src)) + + if server.fine_grained_manager: + if CHECK_CONSISTENCY: + check_consistency(server.fine_grained_manager) + + steps = testcase.find_steps() + all_triggered = [] + + for operations in steps: + step += 1 + output, triggered = self.perform_step( + operations, + server, + options, + build_options, + testcase, + main_src, + step, + num_regular_incremental_steps, + ) + a.append("==") + a.extend(output) + all_triggered.extend(triggered) + + # Normalize paths in test output (for Windows). + a = [line.replace("\\", "/") for line in a] + + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) + + if testcase.triggered: + assert_string_arrays_equal( + testcase.triggered, + self.format_triggered(all_triggered), + f"Invalid active triggers ({testcase.file}, line {testcase.line})", + ) + + def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bool) -> Options: + # This handles things like '# flags: --foo'. + options = parse_options(source, testcase, incremental_step=1) + options.incremental = True + options.use_builtins_fixtures = True + options.show_traceback = True + options.error_summary = False + options.fine_grained_incremental = not build_cache + options.use_fine_grained_cache = self.use_cache and not build_cache + options.cache_fine_grained = self.use_cache + options.local_partial_types = True + options.export_types = "inspect" in testcase.file + # Treat empty bodies safely for these test cases. + options.allow_empty_bodies = not testcase.name.endswith("_no_empty") + if re.search("flags:.*--follow-imports", source) is None: + # Override the default for follow_imports + options.follow_imports = "error" + + for name, _ in testcase.files: + if "mypy.ini" in name or "pyproject.toml" in name: + parse_config_file(options, lambda: None, name) + break + + return options + + def run_check(self, server: Server, sources: list[BuildSource]) -> list[str]: + response = server.check(sources, export_types=False, is_tty=False, terminal_width=-1) + out = response["out"] or response["err"] + assert isinstance(out, str) + return out.splitlines() + + def build(self, options: Options, sources: list[BuildSource]) -> list[str]: + try: + result = build.build(sources=sources, options=options) + except CompileError as e: + return e.messages + return result.errors + + def format_triggered(self, triggered: list[list[str]]) -> list[str]: + result = [] + for n, triggers in enumerate(triggered): + filtered = [trigger for trigger in triggers if not trigger.endswith("__>")] + filtered = sorted(filtered) + result.append(("%d: %s" % (n + 2, ", ".join(filtered))).strip()) + return result + + def get_build_steps(self, program_text: str) -> int: + """Get the number of regular incremental steps to run, from the test source""" + if not self.use_cache: + return 0 + m = re.search("# num_build_steps: ([0-9]+)$", program_text, flags=re.MULTILINE) + if m is not None: + return int(m.group(1)) + return 1 + + def perform_step( + self, + operations: list[UpdateFile | DeleteFile], + server: Server, + options: Options, + build_options: Options, + testcase: DataDrivenTestCase, + main_src: str, + step: int, + num_regular_incremental_steps: int, + ) -> tuple[list[str], list[list[str]]]: + """Perform one fine-grained incremental build step (after some file updates/deletions). + + Return (mypy output, triggered targets). + """ + perform_file_operations(operations) + sources = self.parse_sources(main_src, step, options) + + if step <= num_regular_incremental_steps: + new_messages = self.build(build_options, sources) + else: + new_messages = self.run_check(server, sources) + + updated: list[str] = [] + changed: list[str] = [] + targets: list[str] = [] + triggered = [] + if server.fine_grained_manager: + if CHECK_CONSISTENCY: + check_consistency(server.fine_grained_manager) + triggered.append(server.fine_grained_manager.triggered) + + updated = server.fine_grained_manager.updated_modules + changed = [mod for mod, file in server.fine_grained_manager.changed_modules] + targets = server.fine_grained_manager.processed_targets + + expected_stale = testcase.expected_stale_modules.get(step - 1) + if expected_stale is not None: + assert_module_equivalence("stale" + str(step - 1), expected_stale, changed) + + expected_rechecked = testcase.expected_rechecked_modules.get(step - 1) + if expected_rechecked is not None: + assert_module_equivalence("rechecked" + str(step - 1), expected_rechecked, updated) + + expected = testcase.expected_fine_grained_targets.get(step) + if expected: + assert_target_equivalence("targets" + str(step), expected, targets) + + new_messages = normalize_messages(new_messages) + + a = new_messages + assert testcase.tmpdir is not None + a.extend(self.maybe_suggest(step, server, main_src, testcase.tmpdir)) + a.extend(self.maybe_inspect(step, server, main_src)) + + return a, triggered + + def parse_sources( + self, program_text: str, incremental_step: int, options: Options + ) -> list[BuildSource]: + """Return target BuildSources for a test case. + + Normally, the unit tests will check all files included in the test + case. This differs from how testcheck works by default, as dmypy + doesn't currently support following imports. + + You can override this behavior and instruct the tests to check + multiple modules by using a comment like this in the test case + input: + + # cmd: main a.py + + You can also use `# cmdN:` to have a different cmd for incremental + step N (2, 3, ...). + + """ + m = re.search("# cmd: mypy ([a-zA-Z0-9_./ ]+)$", program_text, flags=re.MULTILINE) + regex = f"# cmd{incremental_step}: mypy ([a-zA-Z0-9_./ ]+)$" + alt_m = re.search(regex, program_text, flags=re.MULTILINE) + if alt_m is not None: + # Optionally return a different command if in a later step + # of incremental mode, otherwise default to reusing the + # original cmd. + m = alt_m + + if m: + # The test case wants to use a non-default set of files. + paths = [os.path.join(test_temp_dir, path) for path in m.group(1).strip().split()] + return create_source_list(paths, options) + else: + base = BuildSource(os.path.join(test_temp_dir, "main"), "__main__", None) + # Use expand_dir instead of create_source_list to avoid complaints + # when there aren't any .py files in an increment + return [base] + create_source_list([test_temp_dir], options, allow_empty_dir=True) + + def maybe_suggest(self, step: int, server: Server, src: str, tmp_dir: str) -> list[str]: + output: list[str] = [] + targets = self.get_suggest(src, step) + for flags, target in targets: + json = "--json" in flags + callsites = "--callsites" in flags + no_any = "--no-any" in flags + no_errors = "--no-errors" in flags + m = re.match("--flex-any=([0-9.]+)", flags) + flex_any = float(m.group(1)) if m else None + m = re.match(r"--use-fixme=(\w+)", flags) + use_fixme = m.group(1) if m else None + m = re.match("--max-guesses=([0-9]+)", flags) + max_guesses = int(m.group(1)) if m else None + res: dict[str, Any] = server.cmd_suggest( + target.strip(), + json=json, + no_any=no_any, + no_errors=no_errors, + flex_any=flex_any, + use_fixme=use_fixme, + callsites=callsites, + max_guesses=max_guesses, + ) + val = res["error"] if "error" in res else res["out"] + res["err"] + if json: + # JSON contains already escaped \ on Windows, so requires a bit of care. + val = val.replace("\\\\", "\\") + val = val.replace(os.path.realpath(tmp_dir) + os.path.sep, "") + val = val.replace(os.path.abspath(tmp_dir) + os.path.sep, "") + output.extend(val.strip().split("\n")) + return normalize_messages(output) + + def maybe_inspect(self, step: int, server: Server, src: str) -> list[str]: + output: list[str] = [] + targets = self.get_inspect(src, step) + for flags, location in targets: + m = re.match(r"--show=(\w+)", flags) + show = m.group(1) if m else "type" + verbosity = 0 + if "-v" in flags: + verbosity = 1 + if "-vv" in flags: + verbosity = 2 + m = re.match(r"--limit=([0-9]+)", flags) + limit = int(m.group(1)) if m else 0 + include_span = "--include-span" in flags + include_kind = "--include-kind" in flags + include_object_attrs = "--include-object-attrs" in flags + union_attrs = "--union-attrs" in flags + force_reload = "--force-reload" in flags + res: dict[str, Any] = server.cmd_inspect( + show, + location, + verbosity=verbosity, + limit=limit, + include_span=include_span, + include_kind=include_kind, + include_object_attrs=include_object_attrs, + union_attrs=union_attrs, + force_reload=force_reload, + ) + val = res["error"] if "error" in res else res["out"] + res["err"] + output.extend(val.strip().split("\n")) + return output + + def get_suggest(self, program_text: str, incremental_step: int) -> list[tuple[str, str]]: + step_bit = "1?" if incremental_step == 1 else str(incremental_step) + regex = f"# suggest{step_bit}: (--[a-zA-Z0-9_\\-./=?^ ]+ )*([a-zA-Z0-9_.:/?^ ]+)$" + m = re.findall(regex, program_text, flags=re.MULTILINE) + return m + + def get_inspect(self, program_text: str, incremental_step: int) -> list[tuple[str, str]]: + step_bit = "1?" if incremental_step == 1 else str(incremental_step) + regex = f"# inspect{step_bit}: (--[a-zA-Z0-9_\\-=?^ ]+ )*([a-zA-Z0-9_.:/?^ ]+)$" + m = re.findall(regex, program_text, flags=re.MULTILINE) + return m + + +def normalize_messages(messages: list[str]) -> list[str]: + return [re.sub("^tmp" + re.escape(os.sep), "", message) for message in messages] + + +class TestMessageSorting(unittest.TestCase): + def test_simple_sorting(self) -> None: + msgs = ['x.py:1: error: "int" not callable', 'foo/y.py:123: note: "X" not defined'] + old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable'] + assert sort_messages_preserving_file_order(msgs, old_msgs) == list(reversed(msgs)) + assert sort_messages_preserving_file_order(list(reversed(msgs)), old_msgs) == list( + reversed(msgs) + ) + + def test_long_form_sorting(self) -> None: + # Multi-line errors should be sorted together and not split. + msg1 = [ + 'x.py:1: error: "int" not callable', + "and message continues (x: y)", + " 1()", + " ^~~", + ] + msg2 = [ + 'foo/y.py: In function "f":', + 'foo/y.py:123: note: "X" not defined', + "and again message continues", + ] + old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable'] + assert sort_messages_preserving_file_order(msg1 + msg2, old_msgs) == msg2 + msg1 + assert sort_messages_preserving_file_order(msg2 + msg1, old_msgs) == msg2 + msg1 + + def test_mypy_error_prefix(self) -> None: + # Some errors don't have a file and start with "mypy: ". These + # shouldn't be sorted together with file-specific errors. + msg1 = 'x.py:1: error: "int" not callable' + msg2 = 'foo/y:123: note: "X" not defined' + msg3 = "mypy: Error not associated with a file" + old_msgs = [ + "mypy: Something wrong", + 'foo/y:12: note: "Y" not defined', + 'x.py:8: error: "str" not callable', + ] + assert sort_messages_preserving_file_order([msg1, msg2, msg3], old_msgs) == [ + msg2, + msg1, + msg3, + ] + assert sort_messages_preserving_file_order([msg3, msg2, msg1], old_msgs) == [ + msg2, + msg1, + msg3, + ] + + def test_new_file_at_the_end(self) -> None: + msg1 = 'x.py:1: error: "int" not callable' + msg2 = 'foo/y.py:123: note: "X" not defined' + new1 = "ab.py:3: error: Problem: error" + new2 = "aaa:3: error: Bad" + old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable'] + assert sort_messages_preserving_file_order([msg1, msg2, new1], old_msgs) == [ + msg2, + msg1, + new1, + ] + assert sort_messages_preserving_file_order([new1, msg1, msg2, new2], old_msgs) == [ + msg2, + msg1, + new1, + new2, + ] diff --git a/mypy/test/testfinegrainedcache.py b/mypy/test/testfinegrainedcache.py new file mode 100644 index 000000000000..45523a1f9139 --- /dev/null +++ b/mypy/test/testfinegrainedcache.py @@ -0,0 +1,18 @@ +"""Tests for fine-grained incremental checking using the cache. + +All of the real code for this lives in testfinegrained.py. +""" + +# We can't "import FineGrainedSuite from ..." because that will cause pytest +# to collect the non-caching tests when running this file. +from __future__ import annotations + +import mypy.test.testfinegrained + + +class FineGrainedCacheSuite(mypy.test.testfinegrained.FineGrainedSuite): + use_cache = True + test_name_suffix = "_cached" + files = mypy.test.testfinegrained.FineGrainedSuite.files + [ + "fine-grained-cache-incremental.test" + ] diff --git a/mypy/test/testformatter.py b/mypy/test/testformatter.py new file mode 100644 index 000000000000..9f8bb5d82408 --- /dev/null +++ b/mypy/test/testformatter.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from unittest import TestCase, main + +from mypy.util import split_words, trim_source_line + + +class FancyErrorFormattingTestCases(TestCase): + def test_trim_source(self) -> None: + assert trim_source_line("0123456789abcdef", max_len=16, col=5, min_width=2) == ( + "0123456789abcdef", + 0, + ) + + # Locations near start. + assert trim_source_line("0123456789abcdef", max_len=7, col=0, min_width=2) == ( + "0123456...", + 0, + ) + assert trim_source_line("0123456789abcdef", max_len=7, col=4, min_width=2) == ( + "0123456...", + 0, + ) + + # Middle locations. + assert trim_source_line("0123456789abcdef", max_len=7, col=5, min_width=2) == ( + "...1234567...", + -2, + ) + assert trim_source_line("0123456789abcdef", max_len=7, col=6, min_width=2) == ( + "...2345678...", + -1, + ) + assert trim_source_line("0123456789abcdef", max_len=7, col=8, min_width=2) == ( + "...456789a...", + 1, + ) + + # Locations near the end. + assert trim_source_line("0123456789abcdef", max_len=7, col=11, min_width=2) == ( + "...789abcd...", + 4, + ) + assert trim_source_line("0123456789abcdef", max_len=7, col=13, min_width=2) == ( + "...9abcdef", + 6, + ) + assert trim_source_line("0123456789abcdef", max_len=7, col=15, min_width=2) == ( + "...9abcdef", + 6, + ) + + def test_split_words(self) -> None: + assert split_words("Simple message") == ["Simple", "message"] + assert split_words('Message with "Some[Long, Types]" in it') == [ + "Message", + "with", + '"Some[Long, Types]"', + "in", + "it", + ] + assert split_words('Message with "Some[Long, Types]" and [error-code]') == [ + "Message", + "with", + '"Some[Long, Types]"', + "and", + "[error-code]", + ] + assert split_words('"Type[Stands, First]" then words') == [ + '"Type[Stands, First]"', + "then", + "words", + ] + assert split_words('First words "Then[Stands, Type]"') == [ + "First", + "words", + '"Then[Stands, Type]"', + ] + assert split_words('"Type[Only, Here]"') == ['"Type[Only, Here]"'] + assert split_words("OneWord") == ["OneWord"] + assert split_words(" ") == ["", ""] + + +if __name__ == "__main__": + main() diff --git a/mypy/test/testfscache.py b/mypy/test/testfscache.py new file mode 100644 index 000000000000..44b0d32f5797 --- /dev/null +++ b/mypy/test/testfscache.py @@ -0,0 +1,101 @@ +"""Unit tests for file system cache.""" + +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +import unittest + +from mypy.fscache import FileSystemCache + + +class TestFileSystemCache(unittest.TestCase): + def setUp(self) -> None: + self.tempdir = tempfile.mkdtemp() + self.oldcwd = os.getcwd() + os.chdir(self.tempdir) + self.fscache = FileSystemCache() + + def tearDown(self) -> None: + os.chdir(self.oldcwd) + shutil.rmtree(self.tempdir) + + def test_isfile_case_1(self) -> None: + self.make_file("bar.py") + self.make_file("pkg/sub_package/__init__.py") + self.make_file("pkg/sub_package/foo.py") + # Run twice to test both cached and non-cached code paths. + for i in range(2): + assert self.isfile_case("bar.py") + assert self.isfile_case("pkg/sub_package/__init__.py") + assert self.isfile_case("pkg/sub_package/foo.py") + assert not self.isfile_case("non_existent.py") + assert not self.isfile_case("pkg/non_existent.py") + assert not self.isfile_case("pkg/") + assert not self.isfile_case("bar.py/") + for i in range(2): + assert not self.isfile_case("Bar.py") + assert not self.isfile_case("pkg/sub_package/__init__.PY") + assert not self.isfile_case("pkg/Sub_Package/foo.py") + assert not self.isfile_case("Pkg/sub_package/foo.py") + + def test_isfile_case_2(self) -> None: + self.make_file("bar.py") + self.make_file("pkg/sub_package/__init__.py") + self.make_file("pkg/sub_package/foo.py") + # Run twice to test both cached and non-cached code paths. + # This reverses the order of checks from test_isfile_case_1. + for i in range(2): + assert not self.isfile_case("Bar.py") + assert not self.isfile_case("pkg/sub_package/__init__.PY") + assert not self.isfile_case("pkg/Sub_Package/foo.py") + assert not self.isfile_case("Pkg/sub_package/foo.py") + for i in range(2): + assert self.isfile_case("bar.py") + assert self.isfile_case("pkg/sub_package/__init__.py") + assert self.isfile_case("pkg/sub_package/foo.py") + assert not self.isfile_case("non_existent.py") + assert not self.isfile_case("pkg/non_existent.py") + + def test_isfile_case_3(self) -> None: + self.make_file("bar.py") + self.make_file("pkg/sub_package/__init__.py") + self.make_file("pkg/sub_package/foo.py") + # Run twice to test both cached and non-cached code paths. + for i in range(2): + assert self.isfile_case("bar.py") + assert not self.isfile_case("non_existent.py") + assert not self.isfile_case("pkg/non_existent.py") + assert not self.isfile_case("Bar.py") + assert not self.isfile_case("pkg/sub_package/__init__.PY") + assert not self.isfile_case("pkg/Sub_Package/foo.py") + assert not self.isfile_case("Pkg/sub_package/foo.py") + assert self.isfile_case("pkg/sub_package/__init__.py") + assert self.isfile_case("pkg/sub_package/foo.py") + + def test_isfile_case_other_directory(self) -> None: + self.make_file("bar.py") + with tempfile.TemporaryDirectory() as other: + self.make_file("other_dir.py", base=other) + self.make_file("pkg/other_dir.py", base=other) + assert self.isfile_case(os.path.join(other, "other_dir.py")) + assert not self.isfile_case(os.path.join(other, "Other_Dir.py")) + assert not self.isfile_case(os.path.join(other, "bar.py")) + if sys.platform in ("win32", "darwin"): + # We only check case for directories under our prefix, and since + # this path is not under the prefix, case difference is fine. + assert self.isfile_case(os.path.join(other, "PKG/other_dir.py")) + + def make_file(self, path: str, base: str | None = None) -> None: + if base is None: + base = self.tempdir + fullpath = os.path.join(base, path) + os.makedirs(os.path.dirname(fullpath), exist_ok=True) + if not path.endswith("/"): + with open(fullpath, "w") as f: + f.write("# test file") + + def isfile_case(self, path: str) -> bool: + return self.fscache.isfile_case(os.path.join(self.tempdir, path), self.tempdir) diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py new file mode 100644 index 000000000000..0355e75e8c34 --- /dev/null +++ b/mypy/test/testgraph.py @@ -0,0 +1,83 @@ +"""Test cases for graph processing code in build.py.""" + +from __future__ import annotations + +import sys +from typing import AbstractSet + +from mypy.build import BuildManager, BuildSourceSet, State, order_ascc, sorted_components +from mypy.errors import Errors +from mypy.fscache import FileSystemCache +from mypy.graph_utils import strongly_connected_components, topsort +from mypy.modulefinder import SearchPaths +from mypy.options import Options +from mypy.plugin import Plugin +from mypy.report import Reports +from mypy.test.helpers import Suite, assert_equal +from mypy.version import __version__ + + +class GraphSuite(Suite): + def test_topsort(self) -> None: + a = frozenset({"A"}) + b = frozenset({"B"}) + c = frozenset({"C"}) + d = frozenset({"D"}) + data: dict[AbstractSet[str], set[AbstractSet[str]]] = {a: {b, c}, b: {d}, c: {d}} + res = list(topsort(data)) + assert_equal(res, [{d}, {b, c}, {a}]) + + def test_scc(self) -> None: + vertices = {"A", "B", "C", "D"} + edges: dict[str, list[str]] = {"A": ["B", "C"], "B": ["C"], "C": ["B", "D"], "D": []} + sccs = {frozenset(x) for x in strongly_connected_components(vertices, edges)} + assert_equal(sccs, {frozenset({"A"}), frozenset({"B", "C"}), frozenset({"D"})}) + + def _make_manager(self) -> BuildManager: + options = Options() + options.use_builtins_fixtures = True + errors = Errors(options) + fscache = FileSystemCache() + search_paths = SearchPaths((), (), (), ()) + manager = BuildManager( + data_dir="", + search_paths=search_paths, + ignore_prefix="", + source_set=BuildSourceSet([]), + reports=Reports("", {}), + options=options, + version_id=__version__, + plugin=Plugin(options), + plugins_snapshot={}, + errors=errors, + flush_errors=lambda filename, msgs, serious: None, + fscache=fscache, + stdout=sys.stdout, + stderr=sys.stderr, + ) + return manager + + def test_sorted_components(self) -> None: + manager = self._make_manager() + graph = { + "a": State("a", None, "import b, c", manager), + "d": State("d", None, "pass", manager), + "b": State("b", None, "import c", manager), + "c": State("c", None, "import b, d", manager), + } + res = sorted_components(graph) + assert_equal(res, [frozenset({"d"}), frozenset({"c", "b"}), frozenset({"a"})]) + + def test_order_ascc(self) -> None: + manager = self._make_manager() + graph = { + "a": State("a", None, "import b, c", manager), + "d": State("d", None, "def f(): import a", manager), + "b": State("b", None, "import c", manager), + "c": State("c", None, "import b, d", manager), + } + res = sorted_components(graph) + assert_equal(res, [frozenset({"a", "d", "c", "b"})]) + ascc = res[0] + scc = order_ascc(graph, ascc) + assert_equal(scc, ["d", "c", "b", "a"]) diff --git a/mypy/test/testinfer.py b/mypy/test/testinfer.py index 2a8760671940..08926c179623 100644 --- a/mypy/test/testinfer.py +++ b/mypy/test/testinfer.py @@ -1,189 +1,128 @@ """Test cases for type inference helper functions.""" -import typing +from __future__ import annotations -from mypy.myunit import Suite, assert_equal, assert_true, run_test -from mypy.checkexpr import map_actuals_to_formals -from mypy.nodes import ARG_POS, ARG_OPT, ARG_STAR, ARG_STAR2, ARG_NAMED -from mypy.types import AnyType, TupleType +from mypy.argmap import map_actuals_to_formals +from mypy.checker import DisjointDict, group_comparison_operands +from mypy.literals import Key +from mypy.nodes import ARG_NAMED, ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, ArgKind, NameExpr +from mypy.test.helpers import Suite, assert_equal +from mypy.test.typefixture import TypeFixture +from mypy.types import AnyType, TupleType, Type, TypeOfAny class MapActualsToFormalsSuite(Suite): - """Test cases for checkexpr.map_actuals_to_formals.""" + """Test cases for argmap.map_actuals_to_formals.""" - def test_basic(self): + def test_basic(self) -> None: self.assert_map([], [], []) - def test_positional_only(self): - self.assert_map([ARG_POS], - [ARG_POS], - [[0]]) - self.assert_map([ARG_POS, ARG_POS], - [ARG_POS, ARG_POS], - [[0], [1]]) - - def test_optional(self): - self.assert_map([], - [ARG_OPT], - [[]]) - self.assert_map([ARG_POS], - [ARG_OPT], - [[0]]) - self.assert_map([ARG_POS], - [ARG_OPT, ARG_OPT], - [[0], []]) - - def test_callee_star(self): - self.assert_map([], - [ARG_STAR], - [[]]) - self.assert_map([ARG_POS], - [ARG_STAR], - [[0]]) - self.assert_map([ARG_POS, ARG_POS], - [ARG_STAR], - [[0, 1]]) - - def test_caller_star(self): - self.assert_map([ARG_STAR], - [ARG_STAR], - [[0]]) - self.assert_map([ARG_POS, ARG_STAR], - [ARG_STAR], - [[0, 1]]) - self.assert_map([ARG_STAR], - [ARG_POS, ARG_STAR], - [[0], [0]]) - self.assert_map([ARG_STAR], - [ARG_OPT, ARG_STAR], - [[0], [0]]) - - def test_too_many_caller_args(self): - self.assert_map([ARG_POS], - [], - []) - self.assert_map([ARG_STAR], - [], - []) - self.assert_map([ARG_STAR], - [ARG_POS], - [[0]]) - - def test_tuple_star(self): + def test_positional_only(self) -> None: + self.assert_map([ARG_POS], [ARG_POS], [[0]]) + self.assert_map([ARG_POS, ARG_POS], [ARG_POS, ARG_POS], [[0], [1]]) + + def test_optional(self) -> None: + self.assert_map([], [ARG_OPT], [[]]) + self.assert_map([ARG_POS], [ARG_OPT], [[0]]) + self.assert_map([ARG_POS], [ARG_OPT, ARG_OPT], [[0], []]) + + def test_callee_star(self) -> None: + self.assert_map([], [ARG_STAR], [[]]) + self.assert_map([ARG_POS], [ARG_STAR], [[0]]) + self.assert_map([ARG_POS, ARG_POS], [ARG_STAR], [[0, 1]]) + + def test_caller_star(self) -> None: + self.assert_map([ARG_STAR], [ARG_STAR], [[0]]) + self.assert_map([ARG_POS, ARG_STAR], [ARG_STAR], [[0, 1]]) + self.assert_map([ARG_STAR], [ARG_POS, ARG_STAR], [[0], [0]]) + self.assert_map([ARG_STAR], [ARG_OPT, ARG_STAR], [[0], [0]]) + + def test_too_many_caller_args(self) -> None: + self.assert_map([ARG_POS], [], []) + self.assert_map([ARG_STAR], [], []) + self.assert_map([ARG_STAR], [ARG_POS], [[0]]) + + def test_tuple_star(self) -> None: + any_type = AnyType(TypeOfAny.special_form) + self.assert_vararg_map([ARG_STAR], [ARG_POS], [[0]], self.make_tuple(any_type)) self.assert_vararg_map( - [ARG_STAR], - [ARG_POS], - [[0]], - self.tuple(AnyType())) - self.assert_vararg_map( - [ARG_STAR], - [ARG_POS, ARG_POS], - [[0], [0]], - self.tuple(AnyType(), AnyType())) + [ARG_STAR], [ARG_POS, ARG_POS], [[0], [0]], self.make_tuple(any_type, any_type) + ) self.assert_vararg_map( [ARG_STAR], [ARG_POS, ARG_OPT, ARG_OPT], [[0], [0], []], - self.tuple(AnyType(), AnyType())) - - def tuple(self, *args): - return TupleType(args, None) - - def test_named_args(self): - self.assert_map( - ['x'], - [(ARG_POS, 'x')], - [[0]]) - self.assert_map( - ['y', 'x'], - [(ARG_POS, 'x'), (ARG_POS, 'y')], - [[1], [0]]) - - def test_some_named_args(self): - self.assert_map( - ['y'], - [(ARG_OPT, 'x'), (ARG_OPT, 'y'), (ARG_OPT, 'z')], - [[], [0], []]) - - def test_missing_named_arg(self): - self.assert_map( - ['y'], - [(ARG_OPT, 'x')], - [[]]) - - def test_duplicate_named_arg(self): - self.assert_map( - ['x', 'x'], - [(ARG_OPT, 'x')], - [[0, 1]]) - - def test_varargs_and_bare_asterisk(self): - self.assert_map( - [ARG_STAR], - [ARG_STAR, (ARG_NAMED, 'x')], - [[0], []]) - self.assert_map( - [ARG_STAR, 'x'], - [ARG_STAR, (ARG_NAMED, 'x')], - [[0], [1]]) - - def test_keyword_varargs(self): - self.assert_map( - ['x'], - [ARG_STAR2], - [[0]]) - self.assert_map( - ['x', ARG_STAR2], - [ARG_STAR2], - [[0, 1]]) - self.assert_map( - ['x', ARG_STAR2], - [(ARG_POS, 'x'), ARG_STAR2], - [[0], [1]]) - self.assert_map( - [ARG_POS, ARG_STAR2], - [(ARG_POS, 'x'), ARG_STAR2], - [[0], [1]]) - - def test_both_kinds_of_varargs(self): - self.assert_map( - [ARG_STAR, ARG_STAR2], - [(ARG_POS, 'x'), (ARG_POS, 'y')], - [[0, 1], [0, 1]]) - - def test_special_cases(self): - self.assert_map([ARG_STAR], - [ARG_STAR, ARG_STAR2], - [[0], []]) - self.assert_map([ARG_STAR, ARG_STAR2], - [ARG_STAR, ARG_STAR2], - [[0], [1]]) - - def assert_map(self, caller_kinds, callee_kinds, expected): - caller_kinds, caller_names = expand_caller_kinds(caller_kinds) - callee_kinds, callee_names = expand_callee_kinds(callee_kinds) + self.make_tuple(any_type, any_type), + ) + + def make_tuple(self, *args: Type) -> TupleType: + return TupleType(list(args), TypeFixture().std_tuple) + + def test_named_args(self) -> None: + self.assert_map(["x"], [(ARG_POS, "x")], [[0]]) + self.assert_map(["y", "x"], [(ARG_POS, "x"), (ARG_POS, "y")], [[1], [0]]) + + def test_some_named_args(self) -> None: + self.assert_map(["y"], [(ARG_OPT, "x"), (ARG_OPT, "y"), (ARG_OPT, "z")], [[], [0], []]) + + def test_missing_named_arg(self) -> None: + self.assert_map(["y"], [(ARG_OPT, "x")], [[]]) + + def test_duplicate_named_arg(self) -> None: + self.assert_map(["x", "x"], [(ARG_OPT, "x")], [[0, 1]]) + + def test_varargs_and_bare_asterisk(self) -> None: + self.assert_map([ARG_STAR], [ARG_STAR, (ARG_NAMED, "x")], [[0], []]) + self.assert_map([ARG_STAR, "x"], [ARG_STAR, (ARG_NAMED, "x")], [[0], [1]]) + + def test_keyword_varargs(self) -> None: + self.assert_map(["x"], [ARG_STAR2], [[0]]) + self.assert_map(["x", ARG_STAR2], [ARG_STAR2], [[0, 1]]) + self.assert_map(["x", ARG_STAR2], [(ARG_POS, "x"), ARG_STAR2], [[0], [1]]) + self.assert_map([ARG_POS, ARG_STAR2], [(ARG_POS, "x"), ARG_STAR2], [[0], [1]]) + + def test_both_kinds_of_varargs(self) -> None: + self.assert_map([ARG_STAR, ARG_STAR2], [(ARG_POS, "x"), (ARG_POS, "y")], [[0, 1], [0, 1]]) + + def test_special_cases(self) -> None: + self.assert_map([ARG_STAR], [ARG_STAR, ARG_STAR2], [[0], []]) + self.assert_map([ARG_STAR, ARG_STAR2], [ARG_STAR, ARG_STAR2], [[0], [1]]) + self.assert_map([ARG_STAR2], [(ARG_POS, "x"), ARG_STAR2], [[0], [0]]) + self.assert_map([ARG_STAR2], [ARG_STAR2], [[0]]) + + def assert_map( + self, + caller_kinds_: list[ArgKind | str], + callee_kinds_: list[ArgKind | tuple[ArgKind, str]], + expected: list[list[int]], + ) -> None: + caller_kinds, caller_names = expand_caller_kinds(caller_kinds_) + callee_kinds, callee_names = expand_callee_kinds(callee_kinds_) result = map_actuals_to_formals( caller_kinds, caller_names, callee_kinds, callee_names, - lambda i: AnyType()) + lambda i: AnyType(TypeOfAny.special_form), + ) assert_equal(result, expected) - def assert_vararg_map(self, caller_kinds, callee_kinds, expected, - vararg_type): - result = map_actuals_to_formals( - caller_kinds, - [], - callee_kinds, - [], - lambda i: vararg_type) + def assert_vararg_map( + self, + caller_kinds: list[ArgKind], + callee_kinds: list[ArgKind], + expected: list[list[int]], + vararg_type: Type, + ) -> None: + result = map_actuals_to_formals(caller_kinds, [], callee_kinds, [], lambda i: vararg_type) assert_equal(result, expected) -def expand_caller_kinds(kinds_or_names): +def expand_caller_kinds( + kinds_or_names: list[ArgKind | str], +) -> tuple[list[ArgKind], list[str | None]]: kinds = [] - names = [] + names: list[str | None] = [] for k in kinds_or_names: if isinstance(k, str): kinds.append(ARG_NAMED) @@ -194,9 +133,11 @@ def expand_caller_kinds(kinds_or_names): return kinds, names -def expand_callee_kinds(kinds_and_names): +def expand_callee_kinds( + kinds_and_names: list[ArgKind | tuple[ArgKind, str]] +) -> tuple[list[ArgKind], list[str | None]]: kinds = [] - names = [] + names: list[str | None] = [] for v in kinds_and_names: if isinstance(v, tuple): kinds.append(v[0]) @@ -207,6 +148,226 @@ def expand_callee_kinds(kinds_and_names): return kinds, names -if __name__ == '__main__': - import sys - run_test(MapActualsToFormalsSuite(), sys.argv[1:]) +class OperandDisjointDictSuite(Suite): + """Test cases for checker.DisjointDict, which is used for type inference with operands.""" + + def new(self) -> DisjointDict[int, str]: + return DisjointDict() + + def test_independent_maps(self) -> None: + d = self.new() + d.add_mapping({0, 1}, {"group1"}) + d.add_mapping({2, 3, 4}, {"group2"}) + d.add_mapping({5, 6, 7}, {"group3"}) + + self.assertEqual( + d.items(), [({0, 1}, {"group1"}), ({2, 3, 4}, {"group2"}), ({5, 6, 7}, {"group3"})] + ) + + def test_partial_merging(self) -> None: + d = self.new() + d.add_mapping({0, 1}, {"group1"}) + d.add_mapping({1, 2}, {"group2"}) + d.add_mapping({3, 4}, {"group3"}) + d.add_mapping({5, 0}, {"group4"}) + d.add_mapping({5, 6}, {"group5"}) + d.add_mapping({4, 7}, {"group6"}) + + self.assertEqual( + d.items(), + [ + ({0, 1, 2, 5, 6}, {"group1", "group2", "group4", "group5"}), + ({3, 4, 7}, {"group3", "group6"}), + ], + ) + + def test_full_merging(self) -> None: + d = self.new() + d.add_mapping({0, 1, 2}, {"a"}) + d.add_mapping({3, 4, 2}, {"b"}) + d.add_mapping({10, 11, 12}, {"c"}) + d.add_mapping({13, 14, 15}, {"d"}) + d.add_mapping({14, 10, 16}, {"e"}) + d.add_mapping({0, 10}, {"f"}) + + self.assertEqual( + d.items(), + [({0, 1, 2, 3, 4, 10, 11, 12, 13, 14, 15, 16}, {"a", "b", "c", "d", "e", "f"})], + ) + + def test_merge_with_multiple_overlaps(self) -> None: + d = self.new() + d.add_mapping({0, 1, 2}, {"a"}) + d.add_mapping({3, 4, 5}, {"b"}) + d.add_mapping({1, 2, 4, 5}, {"c"}) + d.add_mapping({6, 1, 2, 4, 5}, {"d"}) + d.add_mapping({6, 1, 2, 4, 5}, {"e"}) + + self.assertEqual(d.items(), [({0, 1, 2, 3, 4, 5, 6}, {"a", "b", "c", "d", "e"})]) + + +class OperandComparisonGroupingSuite(Suite): + """Test cases for checker.group_comparison_operands.""" + + def literal_keymap(self, assignable_operands: dict[int, NameExpr]) -> dict[int, Key]: + output: dict[int, Key] = {} + for index, expr in assignable_operands.items(): + output[index] = ("FakeExpr", expr.name) + return output + + def test_basic_cases(self) -> None: + # Note: the grouping function doesn't actually inspect the input exprs, so we + # just default to using NameExprs for simplicity. + x0 = NameExpr("x0") + x1 = NameExpr("x1") + x2 = NameExpr("x2") + x3 = NameExpr("x3") + x4 = NameExpr("x4") + + basic_input = [("==", x0, x1), ("==", x1, x2), ("<", x2, x3), ("==", x3, x4)] + + none_assignable = self.literal_keymap({}) + all_assignable = self.literal_keymap({0: x0, 1: x1, 2: x2, 3: x3, 4: x4}) + + for assignable in [none_assignable, all_assignable]: + self.assertEqual( + group_comparison_operands(basic_input, assignable, set()), + [("==", [0, 1]), ("==", [1, 2]), ("<", [2, 3]), ("==", [3, 4])], + ) + self.assertEqual( + group_comparison_operands(basic_input, assignable, {"=="}), + [("==", [0, 1, 2]), ("<", [2, 3]), ("==", [3, 4])], + ) + self.assertEqual( + group_comparison_operands(basic_input, assignable, {"<"}), + [("==", [0, 1]), ("==", [1, 2]), ("<", [2, 3]), ("==", [3, 4])], + ) + self.assertEqual( + group_comparison_operands(basic_input, assignable, {"==", "<"}), + [("==", [0, 1, 2]), ("<", [2, 3]), ("==", [3, 4])], + ) + + def test_multiple_groups(self) -> None: + x0 = NameExpr("x0") + x1 = NameExpr("x1") + x2 = NameExpr("x2") + x3 = NameExpr("x3") + x4 = NameExpr("x4") + x5 = NameExpr("x5") + + self.assertEqual( + group_comparison_operands( + [("==", x0, x1), ("==", x1, x2), ("is", x2, x3), ("is", x3, x4)], + self.literal_keymap({}), + {"==", "is"}, + ), + [("==", [0, 1, 2]), ("is", [2, 3, 4])], + ) + self.assertEqual( + group_comparison_operands( + [("==", x0, x1), ("==", x1, x2), ("==", x2, x3), ("==", x3, x4)], + self.literal_keymap({}), + {"==", "is"}, + ), + [("==", [0, 1, 2, 3, 4])], + ) + self.assertEqual( + group_comparison_operands( + [("is", x0, x1), ("==", x1, x2), ("==", x2, x3), ("==", x3, x4)], + self.literal_keymap({}), + {"==", "is"}, + ), + [("is", [0, 1]), ("==", [1, 2, 3, 4])], + ) + self.assertEqual( + group_comparison_operands( + [("is", x0, x1), ("is", x1, x2), ("<", x2, x3), ("==", x3, x4), ("==", x4, x5)], + self.literal_keymap({}), + {"==", "is"}, + ), + [("is", [0, 1, 2]), ("<", [2, 3]), ("==", [3, 4, 5])], + ) + + def test_multiple_groups_coalescing(self) -> None: + x0 = NameExpr("x0") + x1 = NameExpr("x1") + x2 = NameExpr("x2") + x3 = NameExpr("x3") + x4 = NameExpr("x4") + + nothing_combined = [("==", [0, 1, 2]), ("<", [2, 3]), ("==", [3, 4, 5])] + everything_combined = [("==", [0, 1, 2, 3, 4, 5]), ("<", [2, 3])] + + # Note: We do 'x4 == x0' at the very end! + two_groups = [ + ("==", x0, x1), + ("==", x1, x2), + ("<", x2, x3), + ("==", x3, x4), + ("==", x4, x0), + ] + self.assertEqual( + group_comparison_operands( + two_groups, self.literal_keymap({0: x0, 1: x1, 2: x2, 3: x3, 4: x4, 5: x0}), {"=="} + ), + everything_combined, + "All vars are assignable, everything is combined", + ) + self.assertEqual( + group_comparison_operands( + two_groups, self.literal_keymap({1: x1, 2: x2, 3: x3, 4: x4}), {"=="} + ), + nothing_combined, + "x0 is unassignable, so no combining", + ) + self.assertEqual( + group_comparison_operands( + two_groups, self.literal_keymap({0: x0, 1: x1, 3: x3, 5: x0}), {"=="} + ), + everything_combined, + "Some vars are unassignable but x0 is, so we combine", + ) + self.assertEqual( + group_comparison_operands(two_groups, self.literal_keymap({0: x0, 5: x0}), {"=="}), + everything_combined, + "All vars are unassignable but x0 is, so we combine", + ) + + def test_multiple_groups_different_operators(self) -> None: + x0 = NameExpr("x0") + x1 = NameExpr("x1") + x2 = NameExpr("x2") + x3 = NameExpr("x3") + + groups = [("==", x0, x1), ("==", x1, x2), ("is", x2, x3), ("is", x3, x0)] + keymap = self.literal_keymap({0: x0, 1: x1, 2: x2, 3: x3, 4: x0}) + self.assertEqual( + group_comparison_operands(groups, keymap, {"==", "is"}), + [("==", [0, 1, 2]), ("is", [2, 3, 4])], + "Different operators can never be combined", + ) + + def test_single_pair(self) -> None: + x0 = NameExpr("x0") + x1 = NameExpr("x1") + + single_comparison = [("==", x0, x1)] + expected_output = [("==", [0, 1])] + + assignable_combinations: list[dict[int, NameExpr]] = [{}, {0: x0}, {1: x1}, {0: x0, 1: x1}] + to_group_by: list[set[str]] = [set(), {"=="}, {"is"}] + + for combo in assignable_combinations: + for operators in to_group_by: + keymap = self.literal_keymap(combo) + self.assertEqual( + group_comparison_operands(single_comparison, keymap, operators), + expected_output, + ) + + def test_empty_pair_list(self) -> None: + # This case should never occur in practice -- ComparisionExprs + # always contain at least one comparison. But in case it does... + + self.assertEqual(group_comparison_operands([], {}, set()), []) + self.assertEqual(group_comparison_operands([], {}, {"=="}), []) diff --git a/mypy/test/testipc.py b/mypy/test/testipc.py new file mode 100644 index 000000000000..0224035a7b61 --- /dev/null +++ b/mypy/test/testipc.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import sys +import time +from multiprocessing import Queue, get_context +from unittest import TestCase, main + +import pytest + +from mypy.ipc import IPCClient, IPCServer + +CONNECTION_NAME = "dmypy-test-ipc" + + +def server(msg: str, q: Queue[str]) -> None: + server = IPCServer(CONNECTION_NAME) + q.put(server.connection_name) + data = "" + while not data: + with server: + server.write(msg) + data = server.read() + server.cleanup() + + +def server_multi_message_echo(q: Queue[str]) -> None: + server = IPCServer(CONNECTION_NAME) + q.put(server.connection_name) + data = "" + with server: + while data != "quit": + data = server.read() + server.write(data) + server.cleanup() + + +class IPCTests(TestCase): + def setUp(self) -> None: + if sys.platform == "linux": + # The default "fork" start method is potentially unsafe + self.ctx = get_context("forkserver") + else: + self.ctx = get_context("spawn") + + def test_transaction_large(self) -> None: + queue: Queue[str] = self.ctx.Queue() + msg = "t" * 200000 # longer than the max read size of 100_000 + p = self.ctx.Process(target=server, args=(msg, queue), daemon=True) + p.start() + connection_name = queue.get() + with IPCClient(connection_name, timeout=1) as client: + assert client.read() == msg + client.write("test") + queue.close() + queue.join_thread() + p.join() + + def test_connect_twice(self) -> None: + queue: Queue[str] = self.ctx.Queue() + msg = "this is a test message" + p = self.ctx.Process(target=server, args=(msg, queue), daemon=True) + p.start() + connection_name = queue.get() + with IPCClient(connection_name, timeout=1) as client: + assert client.read() == msg + client.write("") # don't let the server hang up yet, we want to connect again. + + with IPCClient(connection_name, timeout=1) as client: + assert client.read() == msg + client.write("test") + queue.close() + queue.join_thread() + p.join() + assert p.exitcode == 0 + + def test_multiple_messages(self) -> None: + queue: Queue[str] = self.ctx.Queue() + p = self.ctx.Process(target=server_multi_message_echo, args=(queue,), daemon=True) + p.start() + connection_name = queue.get() + with IPCClient(connection_name, timeout=1) as client: + # "foo bar" with extra accents on letters. + # In UTF-8 encoding so we don't confuse editors opening this file. + fancy_text = b"f\xcc\xb6o\xcc\xb2\xf0\x9d\x91\x9c \xd0\xb2\xe2\xb7\xa1a\xcc\xb6r\xcc\x93\xcd\x98\xcd\x8c" + client.write(fancy_text.decode("utf-8")) + assert client.read() == fancy_text.decode("utf-8") + + client.write("Test with spaces") + client.write("Test write before reading previous") + time.sleep(0) # yield to the server to force reading of all messages by server. + assert client.read() == "Test with spaces" + assert client.read() == "Test write before reading previous" + + client.write("quit") + assert client.read() == "quit" + queue.close() + queue.join_thread() + p.join() + assert p.exitcode == 0 + + # Run test_connect_twice a lot, in the hopes of finding issues. + # This is really slow, so it is skipped, but can be enabled if + # needed to debug IPC issues. + @pytest.mark.skip + def test_connect_alot(self) -> None: + t0 = time.time() + for i in range(1000): + try: + print(i, "start") + self.test_connect_twice() + finally: + t1 = time.time() + print(i, t1 - t0) + sys.stdout.flush() + t0 = t1 + + +if __name__ == "__main__": + main() diff --git a/mypy/test/testlex.py b/mypy/test/testlex.py deleted file mode 100644 index eb0a85b47555..000000000000 --- a/mypy/test/testlex.py +++ /dev/null @@ -1,411 +0,0 @@ -"""Lexical analyzer test cases""" - -import typing - -from mypy.myunit import Suite, assert_equal -from mypy.lex import lex - - -class LexerSuite(Suite): - def test_empty(self): - self.assert_lex('', 'Eof()') - - def test_keywords(self): - self.assert_lex( - 'if else elif def return pass', - 'Keyword(if) Keyword( else) Keyword( elif) Keyword( def) ' - 'Keyword( return) Keyword( pass) Break() Eof()') - - self.assert_lex( - 'from import as class global', - 'Keyword(from) Keyword( import) Keyword( as) Keyword( class) ' - 'Keyword( global) ...') - - def test_identifiers(self): - self.assert_lex( - 'i x FooBar FOO_BAR __x var', - 'Name(i) Name( x) Name( FooBar) Name( FOO_BAR) Name( __x) ' - 'Name( var) Break() Eof()') - - self.assert_lex( - 'any interface void', - 'Name(any) Name( interface) Name( void) Break() Eof()') - - def test_int_literals(self): - self.assert_lex( - '0 1 0987654321 10002000300040005000600070008000', - 'IntLit(0) IntLit( 1) IntLit( 0987654321) ' - 'IntLit( 10002000300040005000600070008000) Break() Eof()') - - def test_hex_int_literals(self): - self.assert_lex('0x0 0xabcedf0189 0xAFe 0X2', - 'IntLit(0x0) IntLit( 0xabcedf0189) IntLit( 0xAFe) ' - 'IntLit( 0X2) ...') - - def test_oct_int_literals(self): - self.assert_lex('0o0 0o127 0O1', - 'IntLit(0o0) IntLit( 0o127) IntLit( 0O1) ...') - - def test_float_literals(self): - self.assert_lex('1.2 .1 1.', - 'FloatLit(1.2) FloatLit( .1) FloatLit( 1.) ...') - - self.assert_lex( - '1e2 1.2e+3 1.3e-12', - 'FloatLit(1e2) FloatLit( 1.2e+3) FloatLit( 1.3e-12) ...') - - self.assert_lex('1.e2', 'FloatLit(1.e2) ...') - - def test_comments(self): - self.assert_lex('# foo "" bar' + '\n' + 'x #x', - 'Name(# foo "" bar\\nx) Break( #x) Eof()') - - def test_empty_lines(self): - self.assert_lex(r'\n1', r'IntLit(\n1) ...') - self.assert_lex(r'\n\n1', r'IntLit(\n\n1) ...') - self.assert_lex(r'1\n\n2', r'IntLit(1) Break(\n\n) IntLit(2) ...') - - def test_line_breaks(self): - self.assert_lex('1\\r2', 'IntLit(1) Break(\\r) IntLit(2) ...') - self.assert_lex('1\\r\\n2', 'IntLit(1) Break(\\r\\n) IntLit(2) ...') - - def test_operators(self): - self.assert_lex('- + < > == != <= >= .', - 'Op(-) Op( +) Op( <) Op( >) Op( ==) Op( !=) Op( <=) ' - 'Op( >=) Op( .) ...') - - self.assert_lex('* / % // **', - 'Op(*) Op( /) Op( %) Op( //) Op( **) ...') - - self.assert_lex('& | ^ ~ << >>', - 'Op(&) Op( |) Op( ^) Op( ~) Op( <<) Op( >>) ...') - - self.assert_lex('in is and or not', - 'Op(in) Op( is) Op( and) Op( or) Op( not) ...') - - def test_punctuators(self): - self.assert_lex(': = ,', 'Colon(:) Punct( =) Punct( ,) ...') - - self.assert_lex( - '+= -= *= %= //=', - 'Punct(+=) Punct( -=) Punct( *=) Punct( %=) Punct( //=) ...') - self.assert_lex('**=', 'Punct(**=) ...') - self.assert_lex( - '&= |= ^= <<= >>=', - 'Punct(&=) Punct( |=) Punct( ^=) Punct( <<=) Punct( >>=) ...') - - def test_basic_indentation(self): - self.assert_lex( - 'y' + '\n' + ' x', - 'Name(y) Break(\\n) Indent( ) Name(x) Break() Dedent() Eof()') - - self.assert_lex( - 'y' + '\n' + ' x' + '\n' + 'z', - 'Name(y) Break(\\n) Indent( ) Name(x) Break(\\n) Dedent() ' - 'Name(z) Break() Eof()') - - def test_multiple_indent_levels(self): - self.assert_lex('y' + '\n' + - ' x' + '\n' + - ' y' + '\n' + - ' z', - 'Name(y) Break(\\n) ' + - 'Indent( ) Name(x) Break(\\n) ' + - 'Name( y) Break(\\n) ' + - 'Indent( ) Name(z) Break() ' + - 'Dedent() Dedent() Eof()') - - self.assert_lex('y' + '\n' + - ' x' + '\n' + - ' z' + '\n' + - ' y', - 'Name(y) Break(\\n) ' + - 'Indent( ) Name(x) Break(\\n) ' + - 'Indent( ) Name(z) Break(\\n) ' + - 'Dedent() Name( y) Break() ' + - 'Dedent() Eof()') - - def test_tab_indent(self): - self.assert_lex('y' + '\n' + - '\t' + 'x' + '\n' + - ' y' + '\n' + - ' ' + '\t' + 'z', - 'Name(y) Break(\\n) ' + - 'Indent(\\t) Name(x) Break(\\n) ' + - 'Name( y) Break(\\n) ' + - 'Name( \\tz) Break() ' + - 'Dedent() Eof()') - - def test_comment_after_dedent(self): - self.assert_lex('y\n' - ' x\n' - '# Foo\n' - 'z', - r'Name(y) Break(\n) Indent( ) Name(x) ' - r'Break(\n# Foo\n) ' - r'Dedent() Name(z) Break() Eof()') - - def test_parens(self): - self.assert_lex('( x )', 'Punct(() Name( x) Punct( )) Break() Eof()') - self.assert_lex( - '( x' + '\n' + ' y )', - 'Punct(() Name( x) Name(\\n y) Punct( )) Break() Eof()') - - self.assert_lex('()' + '\n' + ' y', - 'Punct(() Punct()) Break(\\n) Indent( ) Name(y) ' - 'Break() Dedent() Eof()') - - # [ ... ] and { ... }. - self.assert_lex( - '[ x' + '\n' + ' y ]', - 'Punct([) Name( x) Name(\\n y) Punct( ]) Break() Eof()') - self.assert_lex( - '{ x' + '\n' + ' y }', - 'Punct({) Name( x) Name(\\n y) Punct( }) Break() Eof()') - - # Nested brackets. - self.assert_lex( - '({}' + '\n' + ' y)', - 'Punct(() Punct({) Punct(}) Name(\\n y) Punct()) Break() Eof()') - - def test_brackets_and_line_breaks(self): - # This used to fail. - self.assert_lex('{}' + '\n' + '1', - 'Punct({) Punct(}) Break(\\n) IntLit(1) Break() Eof()') - - def test_str_literals(self): - self.assert_lex("'' 'foo_bar'", - "StrLit('') StrLit( 'foo_bar') Break() Eof()") - self.assert_lex('"" "foo_bar"', - 'StrLit("") StrLit( "foo_bar") Break() Eof()') - - self.assert_lex('"\\"" 1', 'StrLit("\\"") IntLit( 1) Break() Eof()') - self.assert_lex("'\\'' 1", "StrLit('\\'') IntLit( 1) Break() Eof()") - - self.assert_lex('"\\\\" 1', 'StrLit("\\\\") IntLit( 1) Break() Eof()') - self.assert_lex("'\\\\' 1", "StrLit('\\\\') IntLit( 1) Break() Eof()") - - def test_triple_quoted_string_literals(self): - # Single-line - - self.assert_lex("''''''", "StrLit('''''') ...") - self.assert_lex("1 '''x''y'''1", - "IntLit(1) StrLit( '''x''y''') IntLit(1) ...") - - self.assert_lex('""""""', 'StrLit("""""") ...') - self.assert_lex('"""x""y"""', 'StrLit("""x""y""") ...') - - # Multiple-line - - self.assert_lex("'''" + '\n' + "'''", "StrLit('''\\n''') ...") - self.assert_lex("'''x''" + '\n' + "''x'''", - "StrLit('''x''\\n''x''') ...") - self.assert_lex("'''''" + '\n' + "'''''", - "StrLit('''''\\n''') StrLit('') ...") - self.assert_lex("'''x" + '\n' + 'xyz' + '\n' + "''x'''", - "StrLit('''x\\nxyz\\n''x''') ...") - - self.assert_lex('"""x' + '\n' + 'y"""', 'StrLit("""x\\ny""") ...') - - def test_unicode_literals(self): - self.assert_lex("u'' u'foo'", - "UnicodeLit(u'') UnicodeLit( u'foo') ...") - self.assert_lex('u"" u"foo"', - 'UnicodeLit(u"") UnicodeLit( u"foo") ...') - self.assert_lex('ur"" ur"foo"', - 'UnicodeLit(ur"") UnicodeLit( ur"foo") ...') - self.assert_lex('u"""foo\n"""', - r'UnicodeLit(u"""foo\n""") ...') - - def test_semicolons(self): - self.assert_lex('a;b', 'Name(a) Break(;) Name(b) ...') - self.assert_lex('a;', 'Name(a) Break(;) Eof()') - - self.assert_lex(';a', 'Break(;) Name(a) ...') - self.assert_lex('a;;b', 'Name(a) Break(;) Break(;) Name(b) ...') - - def test_raw_string(self): - self.assert_lex("r'' r'foo bar'", - "StrLit(r'') StrLit( r'foo bar') ...") - self.assert_lex('r"" r"foo bar"', - 'StrLit(r"") StrLit( r"foo bar") ...') - - self.assert_lex("r'\\x\\''", "StrLit(r'\\x\\'') ...") - self.assert_lex('r"\\x\\""', 'StrLit(r"\\x\\"") ...') - - self.assert_lex("r'\\\\' ''", "StrLit(r'\\\\') StrLit( '') ...") - self.assert_lex('r"\\\\" ""', 'StrLit(r"\\\\") StrLit( "") ...') - - self.assert_lex("r'''" + '\n' + "x'''", "StrLit(r'''\\nx''') ...") - - def test_bytes(self): - self.assert_lex("b'\\'' b'foo bar'", - "BytesLit(b'\\'') BytesLit( b'foo bar') ...") - self.assert_lex('b"\\"" b"foo bar"', - 'BytesLit(b"\\"") BytesLit( b"foo bar") ...') - - self.assert_lex("b'''" + '\n' + " x'''", "BytesLit(b'''\\n x''') ...") - - def test_raw_bytes(self): - self.assert_lex("br'x\\x\\''", "BytesLit(br'x\\x\\'') ...") - self.assert_lex('br"x\\y\\""', 'BytesLit(br"x\\y\\"") ...') - - self.assert_lex('br"""' + '\n' + 'x"""', 'BytesLit(br"""\\nx""") ...') - - def test_backslash(self): - self.assert_lex('a\\' + '\n' + ' b', 'Name(a) Name(\\\\n b) ...') - self.assert_lex( - 'a = \\' + '\n' + ' 1' + '\n' + '=', - 'Name(a) Punct( =) IntLit( \\\\n 1) Break(\\n) Punct(=) ...') - - def test_backslash_in_string(self): - self.assert_lex("'foo\\" + '\n' + "bar'", "StrLit('foo\\\\nbar') ...") - self.assert_lex("'foo\\" + '\n' + ' zar\\' + '\n' + " bar'", - "StrLit('foo\\\\n zar\\\\n bar') ...") - - self.assert_lex('"foo\\' + '\n' + 'bar"', 'StrLit("foo\\\\nbar") ...') - - def test_backslash_in_raw_string(self): - self.assert_lex("r'a\\" + '\n' + "b\\'1", - "StrLit(r'a\\\\nb\\') IntLit(1) ...") - self.assert_lex("r'a\\" + '\n' + '-\\' + '\n' + "b\\'1", - "StrLit(r'a\\\\n-\\\\nb\\') IntLit(1) ...") - self.assert_lex('r"a\\' + '\n' + 'b\\"1', - 'StrLit(r"a\\\\nb\\") IntLit(1) ...') - self.assert_lex('r"a\\' + '\n' + '-\\' + '\n' + 'b\\"1', - 'StrLit(r"a\\\\n-\\\\nb\\") IntLit(1) ...') - - def test_final_dedent(self): - self.assert_lex( - '1' + '\n' + ' 1' + '\n', - 'IntLit(1) Break(\\n) Indent( ) IntLit(1) Break(\\n) Dedent() Eof()') - - def test_empty_line(self): - self.assert_lex('1' + '\n' + ' 1' + '\n' + '\n', - r'IntLit(1) Break(\n) Indent( ) IntLit(1) ' - r'Break(\n\n) Dedent() Eof()') - - def test_comments_and_indents(self): - self.assert_lex('1' + '\n' + ' #x' + '\n' + ' y', - r'IntLit(1) Break(\n #x\n) Indent( ) Name(y) ' - r'Break() Dedent() Eof()') - self.assert_lex('1' + '\n' + '#x' + '\n' + ' y', - r'IntLit(1) Break(\n#x\n) Indent( ) Name(y) ' - r'Break() Dedent() Eof()') - - def test_form_feed(self): - self.assert_lex('\x0c' + '\n' + 'x', 'Name(\x0c\\nx) ...') - - def test_comment_after_linebreak(self): - self.assert_lex('1\n# foo\n2', - 'IntLit(1) Break(\\n# foo\\n) IntLit(2) ...') - self.assert_lex('1\n# foo', - 'IntLit(1) Break(\\n# foo) Eof()') - - def test_line_numbers(self): - self.assert_line('a\\nb', [1, 1, 2, 2, 2]) - - self.assert_line('(\\nb)', [1, 2, 2]) # Note: omit break and eof tokens - - self.assert_line('a\\n b', [1, 1, # a, break - 2, 2, 2, # indent, b, break - 2, 2]) # dedent, break - self.assert_line('a\\n b\\nc', [1, 1, # a, break - 2, 2, 2, # indent, b, break - 3, 3]) # dedent, c - - self.assert_line('a\\rb', [1, 1, 2]) - self.assert_line('a\\r\\nb', [1, 1, 2]) - - self.assert_line('"""x""" 1', [1, 1]) - self.assert_line('"""x\\ny""" 1', [1, 2]) - self.assert_line('"""x\\r\\ny""" 1', [1, 2]) - self.assert_line('"""x\\ry""" 1', [1, 2]) - self.assert_line('"""x\\n\\ny""" 1', [1, 3]) - self.assert_line('\\n"""x\\ny""" 1', [2, 3]) - - self.assert_line('"x" 1', [1, 1]) - self.assert_line('"\\\\n" 1', [1, 2]) - self.assert_line('"\\\\nx\\\\n" 1', [1, 3]) - - self.assert_line('r"x" 1', [1, 1]) - self.assert_line('r"\\\\n" 1', [1, 2]) - self.assert_line('r"\\\\nx\\\\n" 1', [1, 3]) - - def test_backslash_line(self): - self.assert_line('a\\\\n 1\\n=', [1, 2, 2, 3]) - - def test_invalid_parens(self): - self.assert_lex('([\\n )\\n1', - 'Punct(() Punct([) Punct(\\n )) IntLit(\\n1) ...') - self.assert_lex('])', 'Punct(]) Punct()) ...') - self.assert_lex('(]\\n )', 'Punct(() Punct(]) Punct(\\n )) ...') - self.assert_lex('(\\n ])', 'Punct(() Punct(\\n ]) Punct()) ...') - - def test_invalid_indent(self): - self.assert_lex('x\\n y\\n z', - 'Name(x) Break(\\n) Indent( ) Name(y) ' + - 'Break(\\n) Dedent() LexError( ) Name(z) ...') - - def test_invalid_backslash(self): - self.assert_lex('\\ \\nx', 'LexError(\\) Break( \\n) Name(x) ...') - self.assert_lex('\\ \\nx', 'LexError(\\) Break( \\n) Name(x) ...') - - def test_non_terminated_string_literal(self): - self.assert_lex("'", 'LexError(\') ...') - self.assert_lex("'\\na", 'LexError(\') Break(\\n) Name(a) ...') - - self.assert_lex('"', 'LexError(") ...') - self.assert_lex('"\\na', 'LexError(") Break(\\n) Name(a) ...') - - self.assert_lex("r'", 'LexError(r\') ...') - self.assert_lex('r"', 'LexError(r") ...') - - self.assert_lex('"""', 'LexError(""") ...') - self.assert_lex('"""\\n', 'LexError("""\\n) ...') - - self.assert_lex("'''", "LexError(''') ...") - self.assert_lex("'''\\n", "LexError('''\\n) ...") - - self.assert_lex("'\\", 'LexError(\'\\) ...') - self.assert_lex("'\\\\n", 'LexError(\'\\\\n) ...') - self.assert_lex("r'\\", 'LexError(r\'\\) ...') - self.assert_lex("r'\\\\n", 'LexError(r\'\\\\n) ...') - - def test_invalid_hex_int_literals(self): - self.assert_lex('0x', 'LexError( ) ...') - self.assert_lex('0xax', 'LexError( ) ...') - - # TODO - # invalid escape sequences in string literals etc. - - def assert_lex(self, src, lexed): - src = src.replace('\\n', '\n') - src = src.replace('\\r', '\r') - - if lexed.endswith(' ...'): - lexed = lexed[:-3] + 'Break() Eof()' - - l = lex(src) - r = [] - for t in l: - r.append(str(t)) - act = ' '.join(r) - if act != lexed: - print('Actual: ', act) - print('Expected:', lexed) - assert_equal(act, lexed) - - def assert_line(self, s, a): - s = s.replace('\\n', '\n') - s = s.replace('\\r', '\r') - - tt = lex(s) - r = [] - for t in tt: - r.append(t.line) - if len(r) == len(a) + 2: - a = a[:] - a.append(a[-1]) - a.append(a[-1]) - assert_equal(r, a) diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py new file mode 100644 index 000000000000..0582c9ed5882 --- /dev/null +++ b/mypy/test/testmerge.py @@ -0,0 +1,238 @@ +"""Test cases for AST merge (used for fine-grained incremental checking)""" + +from __future__ import annotations + +import os +import shutil + +from mypy import build +from mypy.build import BuildResult +from mypy.errors import CompileError +from mypy.modulefinder import BuildSource +from mypy.nodes import ( + UNBOUND_IMPORTED, + Expression, + MypyFile, + Node, + SymbolTable, + SymbolTableNode, + TypeInfo, + TypeVarExpr, + Var, +) +from mypy.options import Options +from mypy.server.subexpr import get_subexpressions +from mypy.server.update import FineGrainedBuildManager +from mypy.strconv import StrConv +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options +from mypy.types import Type, TypeStrVisitor +from mypy.util import IdMapper, short_type + +# Which data structures to dump in a test case? +SYMTABLE = "SYMTABLE" +TYPEINFO = " TYPEINFO" +TYPES = "TYPES" +AST = "AST" + + +class ASTMergeSuite(DataSuite): + files = ["merge.test"] + + def setup(self) -> None: + super().setup() + self.str_conv = StrConv(show_ids=True, options=Options()) + assert self.str_conv.id_mapper is not None + self.id_mapper: IdMapper = self.str_conv.id_mapper + self.type_str_conv = TypeStrVisitor(self.id_mapper, options=Options()) + + def run_case(self, testcase: DataDrivenTestCase) -> None: + name = testcase.name + # We use the test case name to decide which data structures to dump. + # Dumping everything would result in very verbose test cases. + if name.endswith("_symtable"): + kind = SYMTABLE + elif name.endswith("_typeinfo"): + kind = TYPEINFO + elif name.endswith("_types"): + kind = TYPES + else: + kind = AST + + main_src = "\n".join(testcase.input) + result = self.build(main_src, testcase) + assert result is not None, "cases where CompileError occurred should not be run" + result.manager.fscache.flush() + fine_grained_manager = FineGrainedBuildManager(result) + + a = [] + if result.errors: + a.extend(result.errors) + + target_path = os.path.join(test_temp_dir, "target.py") + shutil.copy(os.path.join(test_temp_dir, "target.py.next"), target_path) + + a.extend(self.dump(fine_grained_manager, kind, testcase.test_modules)) + old_subexpr = get_subexpressions(result.manager.modules["target"]) + + a.append("==>") + + new_file, new_types = self.build_increment(fine_grained_manager, "target", target_path) + a.extend(self.dump(fine_grained_manager, kind, testcase.test_modules)) + + for expr in old_subexpr: + if isinstance(expr, TypeVarExpr): + # These are merged so we can't perform the check. + continue + # Verify that old AST nodes are removed from the expression type map. + assert expr not in new_types + + if testcase.normalize_output: + a = normalize_error_messages(a) + + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) + + def build(self, source: str, testcase: DataDrivenTestCase) -> BuildResult | None: + options = parse_options(source, testcase, incremental_step=1) + options.incremental = True + options.fine_grained_incremental = True + options.use_builtins_fixtures = True + options.export_types = True + options.show_traceback = True + options.allow_empty_bodies = True + options.force_uppercase_builtins = True + main_path = os.path.join(test_temp_dir, "main") + + self.str_conv.options = options + self.type_str_conv.options = options + with open(main_path, "w", encoding="utf8") as f: + f.write(source) + try: + result = build.build( + sources=[BuildSource(main_path, None, None)], + options=options, + alt_lib_path=test_temp_dir, + ) + except CompileError: + # TODO: Is it okay to return None? + return None + return result + + def build_increment( + self, manager: FineGrainedBuildManager, module_id: str, path: str + ) -> tuple[MypyFile, dict[Expression, Type]]: + manager.flush_cache() + manager.update([(module_id, path)], []) + module = manager.manager.modules[module_id] + type_map = manager.graph[module_id].type_map() + return module, type_map + + def dump( + self, manager: FineGrainedBuildManager, kind: str, test_modules: list[str] + ) -> list[str]: + modules = { + name: file for name, file in manager.manager.modules.items() if name in test_modules + } + if kind == AST: + return self.dump_asts(modules) + elif kind == TYPEINFO: + return self.dump_typeinfos(modules) + elif kind == SYMTABLE: + return self.dump_symbol_tables(modules) + elif kind == TYPES: + return self.dump_types(modules, manager) + assert False, f"Invalid kind {kind}" + + def dump_asts(self, modules: dict[str, MypyFile]) -> list[str]: + a = [] + for m in sorted(modules): + s = modules[m].accept(self.str_conv) + a.extend(s.splitlines()) + return a + + def dump_symbol_tables(self, modules: dict[str, MypyFile]) -> list[str]: + a = [] + for id in sorted(modules): + a.extend(self.dump_symbol_table(id, modules[id].names)) + return a + + def dump_symbol_table(self, module_id: str, symtable: SymbolTable) -> list[str]: + a = [f"{module_id}:"] + for name in sorted(symtable): + if name.startswith("__"): + continue + a.append(f" {name}: {self.format_symbol_table_node(symtable[name])}") + return a + + def format_symbol_table_node(self, node: SymbolTableNode) -> str: + if node.node is None: + if node.kind == UNBOUND_IMPORTED: + return "UNBOUND_IMPORTED" + return "None" + if isinstance(node.node, Node): + s = f"{str(type(node.node).__name__)}<{self.id_mapper.id(node.node)}>" + else: + s = f"? ({type(node.node)})" + if ( + isinstance(node.node, Var) + and node.node.type + and not node.node.fullname.startswith("typing.") + ): + typestr = self.format_type(node.node.type) + s += f"({typestr})" + return s + + def dump_typeinfos(self, modules: dict[str, MypyFile]) -> list[str]: + a = [] + for id in sorted(modules): + a.extend(self.dump_typeinfos_recursive(modules[id].names)) + return a + + def dump_typeinfos_recursive(self, names: SymbolTable) -> list[str]: + a = [] + for name, node in sorted(names.items(), key=lambda x: x[0]): + if isinstance(node.node, TypeInfo): + a.extend(self.dump_typeinfo(node.node)) + a.extend(self.dump_typeinfos_recursive(node.node.names)) + return a + + def dump_typeinfo(self, info: TypeInfo) -> list[str]: + if info.fullname == "enum.Enum": + # Avoid noise + return [] + s = info.dump(str_conv=self.str_conv, type_str_conv=self.type_str_conv) + return s.splitlines() + + def dump_types( + self, modules: dict[str, MypyFile], manager: FineGrainedBuildManager + ) -> list[str]: + a = [] + # To make the results repeatable, we try to generate unique and + # deterministic sort keys. + for module_id in sorted(modules): + all_types = manager.manager.all_types + # Compute a module type map from the global type map + tree = manager.graph[module_id].tree + assert tree is not None + type_map = { + node: all_types[node] for node in get_subexpressions(tree) if node in all_types + } + if type_map: + a.append(f"## {module_id}") + for expr in sorted( + type_map, + key=lambda n: ( + n.line, + short_type(n), + n.str_with_options(self.str_conv.options) + str(type_map[n]), + ), + ): + typ = type_map[expr] + a.append(f"{short_type(expr)}:{expr.line}: {self.format_type(typ)}") + return a + + def format_type(self, typ: Type) -> str: + return typ.accept(self.type_str_conv) diff --git a/mypy/test/testmodulefinder.py b/mypy/test/testmodulefinder.py new file mode 100644 index 000000000000..65d9a66c5fa0 --- /dev/null +++ b/mypy/test/testmodulefinder.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import os + +from mypy.modulefinder import FindModuleCache, ModuleNotFoundReason, SearchPaths +from mypy.options import Options +from mypy.test.config import package_path +from mypy.test.helpers import Suite, assert_equal + +data_path = os.path.relpath(os.path.join(package_path, "modulefinder")) + + +class ModuleFinderSuite(Suite): + def setUp(self) -> None: + self.search_paths = SearchPaths( + python_path=(), + mypy_path=( + os.path.join(data_path, "nsx-pkg1"), + os.path.join(data_path, "nsx-pkg2"), + os.path.join(data_path, "nsx-pkg3"), + os.path.join(data_path, "nsy-pkg1"), + os.path.join(data_path, "nsy-pkg2"), + os.path.join(data_path, "pkg1"), + os.path.join(data_path, "pkg2"), + ), + package_path=(), + typeshed_path=(), + ) + options = Options() + options.namespace_packages = True + self.fmc_ns = FindModuleCache(self.search_paths, fscache=None, options=options) + + options = Options() + options.namespace_packages = False + self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) + + def test__no_namespace_packages__nsx(self) -> None: + """ + If namespace_packages is False, we shouldn't find nsx + """ + found_module = self.fmc_nons.find_module("nsx") + assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module) + + def test__no_namespace_packages__nsx_a(self) -> None: + """ + If namespace_packages is False, we shouldn't find nsx.a. + """ + found_module = self.fmc_nons.find_module("nsx.a") + assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module) + + def test__no_namespace_packages__find_a_in_pkg1(self) -> None: + """ + Find find pkg1/a.py for "a" with namespace_packages False. + """ + found_module = self.fmc_nons.find_module("a") + expected = os.path.abspath(os.path.join(data_path, "pkg1", "a.py")) + assert_equal(expected, found_module) + + def test__no_namespace_packages__find_b_in_pkg2(self) -> None: + found_module = self.fmc_ns.find_module("b") + expected = os.path.abspath(os.path.join(data_path, "pkg2", "b", "__init__.py")) + assert_equal(expected, found_module) + + def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None: + """ + There's no __init__.py in any of the nsx dirs, return + the path to the first one found in mypypath. + """ + found_module = self.fmc_ns.find_module("nsx") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg1", "nsx")) + assert_equal(expected, found_module) + + def test__find_nsx_a_init_in_pkg1(self) -> None: + """ + Find nsx-pkg1/nsx/a/__init__.py for "nsx.a" in namespace mode. + """ + found_module = self.fmc_ns.find_module("nsx.a") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py")) + assert_equal(expected, found_module) + + def test__find_nsx_b_init_in_pkg2(self) -> None: + """ + Find nsx-pkg2/nsx/b/__init__.py for "nsx.b" in namespace mode. + """ + found_module = self.fmc_ns.find_module("nsx.b") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py")) + assert_equal(expected, found_module) + + def test__find_nsx_c_c_in_pkg3(self) -> None: + """ + Find nsx-pkg3/nsx/c/c.py for "nsx.c.c" in namespace mode. + """ + found_module = self.fmc_ns.find_module("nsx.c.c") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py")) + assert_equal(expected, found_module) + + def test__find_nsy_a__init_pyi(self) -> None: + """ + Prefer nsy-pkg1/a/__init__.pyi file over __init__.py. + """ + found_module = self.fmc_ns.find_module("nsy.a") + expected = os.path.abspath(os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi")) + assert_equal(expected, found_module) + + def test__find_nsy_b__init_py(self) -> None: + """ + There is a nsy-pkg2/nsy/b.pyi, but also a nsy-pkg2/nsy/b/__init__.py. + We expect to find the latter when looking up "nsy.b" as + a package is preferred over a module. + """ + found_module = self.fmc_ns.find_module("nsy.b") + expected = os.path.abspath(os.path.join(data_path, "nsy-pkg2", "nsy", "b", "__init__.py")) + assert_equal(expected, found_module) + + def test__find_nsy_c_pyi(self) -> None: + """ + There is a nsy-pkg2/nsy/c.pyi and nsy-pkg2/nsy/c.py + We expect to find the former when looking up "nsy.b" as + .pyi is preferred over .py. + """ + found_module = self.fmc_ns.find_module("nsy.c") + expected = os.path.abspath(os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi")) + assert_equal(expected, found_module) + + def test__find_a_in_pkg1(self) -> None: + found_module = self.fmc_ns.find_module("a") + expected = os.path.abspath(os.path.join(data_path, "pkg1", "a.py")) + assert_equal(expected, found_module) + + def test__find_b_init_in_pkg2(self) -> None: + found_module = self.fmc_ns.find_module("b") + expected = os.path.abspath(os.path.join(data_path, "pkg2", "b", "__init__.py")) + assert_equal(expected, found_module) + + def test__find_d_nowhere(self) -> None: + found_module = self.fmc_ns.find_module("d") + assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module) + + +class ModuleFinderSitePackagesSuite(Suite): + def setUp(self) -> None: + self.package_dir = os.path.relpath( + os.path.join(package_path, "modulefinder-site-packages") + ) + + package_paths = ( + os.path.join(self.package_dir, "baz"), + os.path.join(self.package_dir, "..", "not-a-directory"), + os.path.join(self.package_dir, "..", "modulefinder-src"), + self.package_dir, + ) + + self.search_paths = SearchPaths( + python_path=(), + mypy_path=(os.path.join(data_path, "pkg1"),), + package_path=tuple(package_paths), + typeshed_path=(), + ) + options = Options() + options.namespace_packages = True + self.fmc_ns = FindModuleCache(self.search_paths, fscache=None, options=options) + + options = Options() + options.namespace_packages = False + self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) + + def path(self, *parts: str) -> str: + return os.path.abspath(os.path.join(self.package_dir, *parts)) + + def test__packages_with_ns(self) -> None: + cases = [ + # Namespace package with py.typed + ("ns_pkg_typed", self.path("ns_pkg_typed")), + ("ns_pkg_typed.a", self.path("ns_pkg_typed", "a.py")), + ("ns_pkg_typed.b", self.path("ns_pkg_typed", "b")), + ("ns_pkg_typed.b.c", self.path("ns_pkg_typed", "b", "c.py")), + ("ns_pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), + # Namespace package without py.typed + ("ns_pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Namespace package without stub package + ("ns_pkg_w_stubs", self.path("ns_pkg_w_stubs")), + ("ns_pkg_w_stubs.typed", self.path("ns_pkg_w_stubs-stubs", "typed", "__init__.pyi")), + ( + "ns_pkg_w_stubs.typed_inline", + self.path("ns_pkg_w_stubs", "typed_inline", "__init__.py"), + ), + ("ns_pkg_w_stubs.untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Regular package with py.typed + ("pkg_typed", self.path("pkg_typed", "__init__.py")), + ("pkg_typed.a", self.path("pkg_typed", "a.py")), + ("pkg_typed.b", self.path("pkg_typed", "b", "__init__.py")), + ("pkg_typed.b.c", self.path("pkg_typed", "b", "c.py")), + ("pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), + # Regular package without py.typed + ("pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Top-level Python file in site-packages + ("standalone", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("standalone.standalone_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Packages found by following .pth files + ("baz_pkg", self.path("baz", "baz_pkg", "__init__.py")), + ("ns_baz_pkg.a", self.path("baz", "ns_baz_pkg", "a.py")), + ("neighbor_pkg", self.path("..", "modulefinder-src", "neighbor_pkg", "__init__.py")), + ("ns_neighbor_pkg.a", self.path("..", "modulefinder-src", "ns_neighbor_pkg", "a.py")), + # Something that doesn't exist + ("does_not_exist", ModuleNotFoundReason.NOT_FOUND), + # A regular package with an installed set of stubs + ("foo.bar", self.path("foo-stubs", "bar.pyi")), + # A regular, non-site-packages module + ("a", os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))), + ] + for module, expected in cases: + template = "Find(" + module + ") got {}; expected {}" + + actual = self.fmc_ns.find_module(module) + assert_equal(actual, expected, template) + + def test__packages_without_ns(self) -> None: + cases = [ + # Namespace package with py.typed + ("ns_pkg_typed", ModuleNotFoundReason.NOT_FOUND), + ("ns_pkg_typed.a", ModuleNotFoundReason.NOT_FOUND), + ("ns_pkg_typed.b", ModuleNotFoundReason.NOT_FOUND), + ("ns_pkg_typed.b.c", ModuleNotFoundReason.NOT_FOUND), + ("ns_pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), + # Namespace package without py.typed + ("ns_pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Namespace package without stub package + ("ns_pkg_w_stubs", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("ns_pkg_w_stubs.typed", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ( + "ns_pkg_w_stubs.typed_inline", + self.path("ns_pkg_w_stubs", "typed_inline", "__init__.py"), + ), + ("ns_pkg_w_stubs.untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Regular package with py.typed + ("pkg_typed", self.path("pkg_typed", "__init__.py")), + ("pkg_typed.a", self.path("pkg_typed", "a.py")), + ("pkg_typed.b", self.path("pkg_typed", "b", "__init__.py")), + ("pkg_typed.b.c", self.path("pkg_typed", "b", "c.py")), + ("pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), + # Regular package without py.typed + ("pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Top-level Python file in site-packages + ("standalone", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + ("standalone.standalone_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), + # Packages found by following .pth files + ("baz_pkg", self.path("baz", "baz_pkg", "__init__.py")), + ("ns_baz_pkg.a", ModuleNotFoundReason.NOT_FOUND), + ("neighbor_pkg", self.path("..", "modulefinder-src", "neighbor_pkg", "__init__.py")), + ("ns_neighbor_pkg.a", ModuleNotFoundReason.NOT_FOUND), + # Something that doesn't exist + ("does_not_exist", ModuleNotFoundReason.NOT_FOUND), + # A regular package with an installed set of stubs + ("foo.bar", self.path("foo-stubs", "bar.pyi")), + # A regular, non-site-packages module + ("a", os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))), + ] + for module, expected in cases: + template = "Find(" + module + ") got {}; expected {}" + + actual = self.fmc_nons.find_module(module) + assert_equal(actual, expected, template) diff --git a/mypy/test/testmypyc.py b/mypy/test/testmypyc.py new file mode 100644 index 000000000000..e8436f407694 --- /dev/null +++ b/mypy/test/testmypyc.py @@ -0,0 +1,14 @@ +"""A basic check to make sure that we are using a mypyc-compiled version when expected.""" + +from __future__ import annotations + +import os +from unittest import TestCase + +import mypy + + +class MypycTest(TestCase): + def test_using_mypyc(self) -> None: + if os.getenv("TEST_MYPYC", None) == "1": + assert not mypy.__file__.endswith(".py"), "Expected to find a mypyc-compiled version" diff --git a/mypy/test/testopgen.py b/mypy/test/testopgen.py deleted file mode 100644 index 3387cacda924..000000000000 --- a/mypy/test/testopgen.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Test Alore type operation generation (OBSOLETE).""" - -import os.path - -import typing - -from build import build -from myunit import Suite, run_test -from config import test_data_prefix, test_temp_dir -from helpers import assert_string_arrays_equal_wildcards -from data import parse_test_cases -from testoutput import remove_prefix -from testtransform import ( - remove_comment_lines, builtins_wrapper, TRANSFORM_BUILTINS -) -from transform import DyncheckTransformVisitor -from opgen import generate_runtime_support -from errors import CompileError - - -class DyncheckOpGenSuite(Suite): - test_case_files = ['dyncheck-opgen.test'] - - def cases(self): - c = [] - for f in self.test_case_files: - c += parse_test_cases( - os.path.join(test_data_prefix, f), - builtins_wrapper(test_op_gen, - os.path.join(test_data_prefix, - TRANSFORM_BUILTINS)), - test_temp_dir, True) - return c - - -def test_op_gen(testcase): - """Perform a type operation support data and code genereation test case.""" - expected = remove_comment_lines(testcase.output) - try: - src = '\n'.join(testcase.input) - # Parse and type check the input program. - trees, symtable, infos, types = build(src, 'main', False, - test_temp_dir, True) - a = [] - first = True - # Transform each file separately. - for t in trees: - # Skip the builtins module and files with '_skip.' in the path. - if not t.path.endswith('/builtins.py') and '_skip.' not in t.path: - if not first: - # Display path for files other than the first. - a.append('{}:'.format( - remove_prefix(t.path, test_temp_dir))) - - # Transform parse tree and produce the code for operations. - # Note that currently we generate this for each file - # separately; this needs to be fixed eventually. - v = DyncheckTransformVisitor(types, symtable, True) - t.accept(v) - s = generate_runtime_support(t) - if s != '': - a += s.split('\n') - first = False - except CompileError as e: - a = e.messages - assert_string_arrays_equal_wildcards( - expected, a, - 'Invalid source code output ({}, line {})'.format(testcase.file, - testcase.line)) - - -if __name__ == '__main__': - import sys - run_test(DyncheckOpGenSuite(), sys.argv[1:]) diff --git a/mypy/test/testoutput.py b/mypy/test/testoutput.py index b19ecf61bad2..41f6881658c8 100644 --- a/mypy/test/testoutput.py +++ b/mypy/test/testoutput.py @@ -1,97 +1,58 @@ -"""Tests for parse tree pretty printing that preserves formatting +"""Test cases for `--output=json`. -Test case descriptions are in file test/data/output.test. +These cannot be run by the usual unit test runner because of the backslashes in +the output, which get normalized to forward slashes by the test suite on Windows. """ -import os.path -import re +from __future__ import annotations -from typing import Undefined, Any +import os +import os.path -from mypy import build -from mypy.myunit import Suite, run_test -from mypy.test.helpers import assert_string_arrays_equal -from mypy.test.data import parse_test_cases -from mypy.test.config import test_data_prefix, test_temp_dir -from mypy.parse import parse -from mypy.output import OutputVisitor -from mypy.errors import CompileError +from mypy import api +from mypy.defaults import PYTHON3_VERSION +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite -# Files which contain test case descriptions. -output_files = ['output.test'] +class OutputJSONsuite(DataSuite): + files = ["outputjson.test"] + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_output_json(testcase) -class OutputSuite(Suite): - def cases(self): - c = [] - for f in output_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - test_output, test_temp_dir, True) - return c +def test_output_json(testcase: DataDrivenTestCase) -> None: + """Runs Mypy in a subprocess, and ensures that `--output=json` works as intended.""" + mypy_cmdline = ["--output=json"] + mypy_cmdline.append(f"--python-version={'.'.join(map(str, PYTHON3_VERSION))}") -def test_output(testcase): - """Perform an identity source code transformation test case.""" - expected = testcase.output - if expected == []: - expected = testcase.input - try: - src = '\n'.join(testcase.input) - # Parse and semantically analyze the source program. + # Write the program to a file. + program_path = os.path.join(test_temp_dir, "main") + mypy_cmdline.append(program_path) + with open(program_path, "w", encoding="utf8") as file: + for s in testcase.input: + file.write(f"{s}\n") - # Test case names with a special suffix get semantically analyzed. This - # lets us test that semantic analysis does not break source code pretty - # printing. - if testcase.name.endswith('_SemanticAnalyzer'): - result = build.build('main', - target=build.SEMANTIC_ANALYSIS, - program_text=src, - flags=[build.TEST_BUILTINS], - alt_lib_path=test_temp_dir) - files = result.files + output = [] + # Type check the program. + out, err, returncode = api.run(mypy_cmdline) + # split lines, remove newlines, and remove directory of test case + for line in (out + err).rstrip("\n").splitlines(): + if line.startswith(test_temp_dir + os.sep): + output.append(line[len(test_temp_dir + os.sep) :].rstrip("\r\n")) else: - files = {'main': parse(src, 'main')} - a = [] - first = True - - # Produce an output containing the pretty-printed forms (with original - # formatting) of all the relevant source files. - for fnam in sorted(files.keys()): - f = files[fnam] - # Omit the builtins and files marked for omission. - if (not f.path.endswith(os.sep + 'builtins.py') and - '-skip.' not in f.path): - # Add file name + colon for files other than the first. - if not first: - a.append('{}:'.format(fix_path(remove_prefix( - f.path, test_temp_dir)))) - - v = OutputVisitor() - f.accept(v) - s = v.output() - if s != '': - a += s.split('\n') - first = False - except CompileError as e: - a = e.messages - assert_string_arrays_equal( - expected, a, 'Invalid source code output ({}, line {})'.format( - testcase.file, testcase.line)) - - -def remove_prefix(path, prefix): - regexp = '^' + prefix.replace('\\', '\\\\') - np = re.sub(regexp, '', path) - if np.startswith(os.sep): - np = np[1:] - return np + output.append(line.rstrip("\r\n")) + if returncode > 1: + output.append("!!! Mypy crashed !!!") -def fix_path(path): - return path.replace('\\', '/') + # Remove temp file. + os.remove(program_path) + # JSON encodes every `\` character into `\\`, so we need to remove `\\` from windows paths + # and `/` from POSIX paths + json_os_separator = os.sep.replace("\\", "\\\\") + normalized_output = [line.replace(test_temp_dir + json_os_separator, "") for line in output] -if __name__ == '__main__': - import sys - run_test(OutputSuite(), sys.argv[1:]) + assert normalized_output == testcase.output diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index 2a49de3c865c..074ccfb379d0 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -1,86 +1,115 @@ -"""Tests for the mypy parser +"""Tests for the mypy parser.""" -Test case descriptions are in files test/data/parse[-errors].test.""" +from __future__ import annotations -import os.path +import sys -import typing +from pytest import skip -from mypy.myunit import Suite, AssertionFailure, run_test -from mypy.test.helpers import assert_string_arrays_equal -from mypy.test.data import parse_test_cases -from mypy.test import config +from mypy import defaults +from mypy.config_parser import parse_mypy_comments +from mypy.errors import CompileError, Errors +from mypy.options import Options from mypy.parse import parse -from mypy.errors import CompileError +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, find_test_files, parse_options +from mypy.util import get_mypy_comments -class ParserSuite(Suite): - parse_files = ['parse.test', - 'parse-python2.test'] +class ParserSuite(DataSuite): + required_out_section = True + base_path = "." + files = find_test_files(pattern="parse*.test", exclude=["parse-errors.test"]) - def cases(self): - # The test case descriptions are stored in data files. - c = [] - for f in self.parse_files: - c += parse_test_cases( - os.path.join(config.test_data_prefix, f), test_parser) - return c + if sys.version_info < (3, 10): + files.remove("parse-python310.test") + if sys.version_info < (3, 12): + files.remove("parse-python312.test") + if sys.version_info < (3, 13): + files.remove("parse-python313.test") + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_parser(testcase) -def test_parser(testcase): + +def test_parser(testcase: DataDrivenTestCase) -> None: """Perform a single parser test case. The argument contains the description of the test case. """ - - pyversion = 3 - if testcase.file.endswith('python2.test'): - pyversion = 2 + options = Options() + options.force_uppercase_builtins = True + options.hide_error_codes = True + + if testcase.file.endswith("python310.test"): + options.python_version = (3, 10) + elif testcase.file.endswith("python312.test"): + options.python_version = (3, 12) + elif testcase.file.endswith("python313.test"): + options.python_version = (3, 13) + else: + options.python_version = defaults.PYTHON3_VERSION + + source = "\n".join(testcase.input) + + # Apply mypy: comments to options. + comments = get_mypy_comments(source) + changes, _ = parse_mypy_comments(comments, options) + options = options.apply_changes(changes) try: - n = parse('\n'.join(testcase.input), pyversion=pyversion) - a = str(n).split('\n') + n = parse( + bytes(source, "ascii"), + fnam="main", + module="__main__", + errors=Errors(options), + options=options, + raise_on_error=True, + ) + a = n.str_with_options(options).split("\n") except CompileError as e: a = e.messages - assert_string_arrays_equal(testcase.output, a, - 'Invalid parser output ({}, line {})'.format( - testcase.file, testcase.line)) + assert_string_arrays_equal( + testcase.output, a, f"Invalid parser output ({testcase.file}, line {testcase.line})" + ) # The file name shown in test case output. This is displayed in error # messages, and must match the file name in the test case descriptions. -INPUT_FILE_NAME = 'file' +INPUT_FILE_NAME = "file" -class ParseErrorSuite(Suite): - def cases(self): - # Test case descriptions are in an external file. - return parse_test_cases(os.path.join(config.test_data_prefix, - 'parse-errors.test'), - test_parse_error) +class ParseErrorSuite(DataSuite): + required_out_section = True + base_path = "." + files = ["parse-errors.test"] + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_parse_error(testcase) -def test_parse_error(testcase): + +def test_parse_error(testcase: DataDrivenTestCase) -> None: try: - # Compile temporary file. - parse('\n'.join(testcase.input), INPUT_FILE_NAME) - raise AssertionFailure('No errors reported') + options = parse_options("\n".join(testcase.input), testcase, 0) + if options.python_version != sys.version_info[:2]: + skip() + # Compile temporary file. The test file contains non-ASCII characters. + parse( + bytes("\n".join(testcase.input), "utf-8"), + INPUT_FILE_NAME, + "__main__", + errors=Errors(options), + options=options, + raise_on_error=True, + ) + raise AssertionError("No errors reported") except CompileError as e: + if e.module_with_blocker is not None: + assert e.module_with_blocker == "__main__" # Verify that there was a compile error and that the error messages # are equivalent. assert_string_arrays_equal( - testcase.output, e.messages, - 'Invalid compiler output ({}, line {})'.format(testcase.file, - testcase.line)) - - -class CombinedParserSuite(Suite): - def __init__(self): - self.test_parse = ParserSuite() - self.test_parse_errors = ParseErrorSuite() - super().__init__() - - -if __name__ == '__main__': - import sys - run_test(CombinedParserSuite(), sys.argv[1:]) + testcase.output, + e.messages, + f"Invalid compiler output ({testcase.file}, line {testcase.line})", + ) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py new file mode 100644 index 000000000000..9d2628c1fa5f --- /dev/null +++ b/mypy/test/testpep561.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from typing import Iterator + +import filelock + +import mypy.api +from mypy.test.config import package_path, pip_lock, pip_timeout, test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, perform_file_operations + +# NOTE: options.use_builtins_fixtures should not be set in these +# tests, otherwise mypy will ignore installed third-party packages. + + +class PEP561Suite(DataSuite): + files = ["pep561.test"] + base_path = "." + + def run_case(self, test_case: DataDrivenTestCase) -> None: + test_pep561(test_case) + + +@contextmanager +def virtualenv(python_executable: str = sys.executable) -> Iterator[tuple[str, str]]: + """Context manager that creates a virtualenv in a temporary directory + + Returns the path to the created Python executable + """ + with tempfile.TemporaryDirectory() as venv_dir: + proc = subprocess.run( + [python_executable, "-m", "venv", venv_dir], cwd=os.getcwd(), capture_output=True + ) + if proc.returncode != 0: + err = proc.stdout.decode("utf-8") + proc.stderr.decode("utf-8") + raise Exception("Failed to create venv.\n" + err) + if sys.platform == "win32": + yield venv_dir, os.path.abspath(os.path.join(venv_dir, "Scripts", "python")) + else: + yield venv_dir, os.path.abspath(os.path.join(venv_dir, "bin", "python")) + + +def upgrade_pip(python_executable: str) -> None: + """Install pip>=21.3.1. Required for editable installs with PEP 660.""" + if ( + sys.version_info >= (3, 11) + or (3, 10, 3) <= sys.version_info < (3, 11) + or (3, 9, 11) <= sys.version_info < (3, 10) + or (3, 8, 13) <= sys.version_info < (3, 9) + ): + # Skip for more recent Python releases which come with pip>=21.3.1 + # out of the box - for performance reasons. + return + + install_cmd = [python_executable, "-m", "pip", "install", "pip>=21.3.1"] + try: + with filelock.FileLock(pip_lock, timeout=pip_timeout): + proc = subprocess.run(install_cmd, capture_output=True, env=os.environ) + except filelock.Timeout as err: + raise Exception(f"Failed to acquire {pip_lock}") from err + if proc.returncode != 0: + raise Exception(proc.stdout.decode("utf-8") + proc.stderr.decode("utf-8")) + + +def install_package( + pkg: str, python_executable: str = sys.executable, editable: bool = False +) -> None: + """Install a package from test-data/packages/pkg/""" + working_dir = os.path.join(package_path, pkg) + with tempfile.TemporaryDirectory() as dir: + install_cmd = [python_executable, "-m", "pip", "install"] + if editable: + install_cmd.append("-e") + install_cmd.append(".") + + # Note that newer versions of pip (21.3+) don't + # follow this env variable, but this is for compatibility + env = {"PIP_BUILD": dir} + # Inherit environment for Windows + env.update(os.environ) + try: + with filelock.FileLock(pip_lock, timeout=pip_timeout): + proc = subprocess.run(install_cmd, cwd=working_dir, capture_output=True, env=env) + except filelock.Timeout as err: + raise Exception(f"Failed to acquire {pip_lock}") from err + if proc.returncode != 0: + raise Exception(proc.stdout.decode("utf-8") + proc.stderr.decode("utf-8")) + + +def test_pep561(testcase: DataDrivenTestCase) -> None: + """Test running mypy on files that depend on PEP 561 packages.""" + assert testcase.old_cwd is not None, "test was not properly set up" + python = sys.executable + + assert python is not None, "Should be impossible" + pkgs, pip_args = parse_pkgs(testcase.input[0]) + mypy_args = parse_mypy_args(testcase.input[1]) + editable = False + for arg in pip_args: + if arg == "editable": + editable = True + else: + raise ValueError(f"Unknown pip argument: {arg}") + assert pkgs, "No packages to install for PEP 561 test?" + with virtualenv(python) as venv: + venv_dir, python_executable = venv + if editable: + # Editable installs with PEP 660 require pip>=21.3 + upgrade_pip(python_executable) + for pkg in pkgs: + install_package(pkg, python_executable, editable) + + cmd_line = list(mypy_args) + has_program = not ("-p" in cmd_line or "--package" in cmd_line) + if has_program: + program = testcase.name + ".py" + with open(program, "w", encoding="utf-8") as f: + for s in testcase.input: + f.write(f"{s}\n") + cmd_line.append(program) + + cmd_line.extend(["--no-error-summary", "--hide-error-codes"]) + if python_executable != sys.executable: + cmd_line.append(f"--python-executable={python_executable}") + + steps = testcase.find_steps() + if steps != [[]]: + steps = [[]] + steps + + for i, operations in enumerate(steps): + perform_file_operations(operations) + + output = [] + # Type check the module + out, err, returncode = mypy.api.run(cmd_line) + + # split lines, remove newlines, and remove directory of test case + for line in (out + err).splitlines(): + if line.startswith(test_temp_dir + os.sep): + output.append(line[len(test_temp_dir + os.sep) :].rstrip("\r\n")) + else: + # Normalize paths so that the output is the same on Windows and Linux/macOS. + line = line.replace(test_temp_dir + os.sep, test_temp_dir + "/") + output.append(line.rstrip("\r\n")) + iter_count = "" if i == 0 else f" on iteration {i + 1}" + expected = testcase.output if i == 0 else testcase.output2.get(i + 1, []) + + assert_string_arrays_equal( + expected, + output, + f"Invalid output ({testcase.file}, line {testcase.line}){iter_count}", + ) + + if has_program: + os.remove(program) + + +def parse_pkgs(comment: str) -> tuple[list[str], list[str]]: + if not comment.startswith("# pkgs:"): + return ([], []) + else: + pkgs_str, *args = comment[7:].split(";") + return ([pkg.strip() for pkg in pkgs_str.split(",")], [arg.strip() for arg in args]) + + +def parse_mypy_args(line: str) -> list[str]: + m = re.match("# flags: (.*)$", line) + if not m: + return [] # No args; mypy will spit out an error. + return m.group(1).split() + + +def test_mypy_path_is_respected() -> None: + assert False + packages = "packages" + pkg_name = "a" + with tempfile.TemporaryDirectory() as temp_dir: + old_dir = os.getcwd() + os.chdir(temp_dir) + try: + # Create the pkg for files to go into + full_pkg_name = os.path.join(temp_dir, packages, pkg_name) + os.makedirs(full_pkg_name) + + # Create the empty __init__ file to declare a package + pkg_init_name = os.path.join(temp_dir, packages, pkg_name, "__init__.py") + open(pkg_init_name, "w", encoding="utf8").close() + + mypy_config_path = os.path.join(temp_dir, "mypy.ini") + with open(mypy_config_path, "w") as mypy_file: + mypy_file.write("[mypy]\n") + mypy_file.write(f"mypy_path = ./{packages}\n") + + with virtualenv() as venv: + venv_dir, python_executable = venv + + cmd_line_args = [] + if python_executable != sys.executable: + cmd_line_args.append(f"--python-executable={python_executable}") + cmd_line_args.extend(["--config-file", mypy_config_path, "--package", pkg_name]) + + out, err, returncode = mypy.api.run(cmd_line_args) + assert returncode == 0 + finally: + os.chdir(old_dir) diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 4772838b935f..baeea1853ded 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -10,109 +10,108 @@ this suite would slow down the main suite too much. """ +from __future__ import annotations + import os import os.path +import re import subprocess import sys +from tempfile import TemporaryDirectory -import typing - -from mypy.myunit import Suite, run_test, SkipTestCaseException -from mypy.test.config import test_data_prefix, test_temp_dir -from mypy.test.data import parse_test_cases -from mypy.test.helpers import assert_string_arrays_equal - - -# Files which contain test case descriptions. -python_eval_files = ['pythoneval.test', - 'python2eval.test'] - -python_34_eval_files = ['pythoneval-asyncio.test'] +from mypy import api +from mypy.defaults import PYTHON3_VERSION +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, split_lines # Path to Python 3 interpreter -python3_path = 'python3' - -default_python2_interpreter = 'python' - - -class PythonEvaluationSuite(Suite): - def cases(self): - c = [] - for f in python_eval_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - test_python_evaluation, test_temp_dir, True) - if sys.version_info.major == 3 and sys.version_info.minor >= 4: - for f in python_34_eval_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - test_python_evaluation, test_temp_dir, True) - return c - - -def test_python_evaluation(testcase): - python2_interpreter = try_find_python2_interpreter() - # Use Python 2 interpreter if running a Python 2 test case. - if testcase.name.lower().endswith('python2'): - if not python2_interpreter: - # Skip, can't find a Python 2 interpreter. - raise SkipTestCaseException() - interpreter = python2_interpreter - args = ['--py2'] - py2 = True - else: - interpreter = python3_path - args = [] - py2 = False +python3_path = sys.executable +program_re = re.compile(r"\b_program.py\b") + + +class PythonEvaluationSuite(DataSuite): + files = ["pythoneval.test", "pythoneval-asyncio.test"] + cache_dir = TemporaryDirectory() + + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_python_evaluation(testcase, os.path.join(self.cache_dir.name, ".mypy_cache")) + + +def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None: + """Runs Mypy in a subprocess. + + If this passes without errors, executes the script again with a given Python + version. + """ + assert testcase.old_cwd is not None, "test was not properly set up" + # We must enable site packages to get access to installed stubs. + mypy_cmdline = [ + "--show-traceback", + "--no-silence-site-packages", + "--no-error-summary", + "--hide-error-codes", + "--allow-empty-bodies", + "--force-uppercase-builtins", + "--test-env", # Speeds up some checks + ] + interpreter = python3_path + mypy_cmdline.append(f"--python-version={'.'.join(map(str, PYTHON3_VERSION))}") + + m = re.search("# flags: (.*)$", "\n".join(testcase.input), re.MULTILINE) + if m: + additional_flags = m.group(1).split() + for flag in additional_flags: + if flag.startswith("--python-version="): + targetted_python_version = flag.split("=")[1] + targetted_major, targetted_minor = targetted_python_version.split(".") + if (int(targetted_major), int(targetted_minor)) > ( + sys.version_info.major, + sys.version_info.minor, + ): + return + mypy_cmdline.extend(additional_flags) + # Write the program to a file. - program = '_program.py' + program = "_" + testcase.name + ".py" program_path = os.path.join(test_temp_dir, program) - with open(program_path, 'w') as file: + mypy_cmdline.append(program_path) + with open(program_path, "w", encoding="utf8") as file: for s in testcase.input: - file.write('{}\n'.format(s)) - # Set up module path. - typing_path = os.path.join(os.getcwd(), 'lib-typing', '3.2') - assert os.path.isdir(typing_path) - env = os.environ.copy() - env['PYTHONPATH'] = os.pathsep.join([typing_path, os.getcwd()]) + file.write(f"{s}\n") + mypy_cmdline.append(f"--cache-dir={cache_dir}") + output = [] # Type check the program. - process = subprocess.Popen([python3_path, - os.path.join(os.getcwd(), 'scripts', 'mypy')] + args + [program], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=test_temp_dir, - env=env) - outb = process.stdout.read() - # Split output into lines. - out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] - if not process.wait(): - if py2: - typing_path = os.path.join(os.getcwd(), 'lib-typing', '2.7') - env['PYTHONPATH'] = typing_path - process = subprocess.Popen([interpreter, program], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=test_temp_dir, - env=env) - outb = process.stdout.read() - # Split output into lines. - out += [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] + out, err, returncode = api.run(mypy_cmdline) + # split lines, remove newlines, and remove directory of test case + for line in (out + err).splitlines(): + if line.startswith(test_temp_dir + os.sep): + output.append(line[len(test_temp_dir + os.sep) :].rstrip("\r\n")) + else: + # Normalize paths so that the output is the same on Windows and Linux/macOS. + line = line.replace(test_temp_dir + os.sep, test_temp_dir + "/") + output.append(line.rstrip("\r\n")) + if returncode > 1 and not testcase.output: + # Either api.run() doesn't work well in case of a crash, or pytest interferes with it. + # Tweak output to prevent tests with empty expected output to pass in case of a crash. + output.append("!!! Mypy crashed !!!") + if returncode == 0 and not output: + # Execute the program. + proc = subprocess.run( + [interpreter, "-Wignore", program], cwd=test_temp_dir, capture_output=True + ) + output.extend(split_lines(proc.stdout, proc.stderr)) # Remove temp file. os.remove(program_path) - assert_string_arrays_equal(testcase.output, out, - 'Invalid output ({}, line {})'.format( - testcase.file, testcase.line)) - - -def try_find_python2_interpreter(): - try: - process = subprocess.Popen([default_python2_interpreter, '-V'], stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if b'Python 2.7' in stderr: - return default_python2_interpreter - else: - return None - except OSError: - return False - - -if __name__ == '__main__': - run_test(PythonEvaluationSuite(), sys.argv[1:]) + for i, line in enumerate(output): + if os.path.sep + "typeshed" + os.path.sep in line: + output[i] = line.split(os.path.sep)[-1] + assert_string_arrays_equal( + adapt_output(testcase), output, f"Invalid output ({testcase.file}, line {testcase.line})" + ) + + +def adapt_output(testcase: DataDrivenTestCase) -> list[str]: + """Translates the generic _program.py into the actual filename.""" + program = "_" + testcase.name + ".py" + return [program_re.sub(program, line) for line in testcase.output] diff --git a/mypy/test/testreports.py b/mypy/test/testreports.py new file mode 100644 index 000000000000..f638756ad819 --- /dev/null +++ b/mypy/test/testreports.py @@ -0,0 +1,55 @@ +"""Test cases for reports generated by mypy.""" + +from __future__ import annotations + +import textwrap + +from mypy.report import CoberturaPackage, get_line_rate +from mypy.test.helpers import Suite, assert_equal + +try: + import lxml # type: ignore[import-untyped] +except ImportError: + lxml = None + +import pytest + + +class CoberturaReportSuite(Suite): + @pytest.mark.skipif(lxml is None, reason="Cannot import lxml. Is it installed?") + def test_get_line_rate(self) -> None: + assert_equal("1.0", get_line_rate(0, 0)) + assert_equal("0.3333", get_line_rate(1, 3)) + + @pytest.mark.skipif(lxml is None, reason="Cannot import lxml. Is it installed?") + def test_as_xml(self) -> None: + import lxml.etree as etree # type: ignore[import-untyped] + + cobertura_package = CoberturaPackage("foobar") + cobertura_package.covered_lines = 21 + cobertura_package.total_lines = 42 + + child_package = CoberturaPackage("raz") + child_package.covered_lines = 10 + child_package.total_lines = 10 + child_package.classes["class"] = etree.Element("class") + + cobertura_package.packages["raz"] = child_package + + expected_output = textwrap.dedent( + """\ + + + + + + + + + + + """ + ).encode("ascii") + assert_equal( + expected_output, etree.tostring(cobertura_package.as_xml(), pretty_print=True) + ) diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index fe350ae2e928..cdecc4739168 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -1,42 +1,63 @@ """Semantic analyzer test cases""" -import os.path +from __future__ import annotations -from typing import Dict, List +import sys +from typing import Dict from mypy import build -from mypy.myunit import Suite, run_test -from mypy.test.helpers import assert_string_arrays_equal, testfile_pyversion -from mypy.test.data import parse_test_cases -from mypy.test.config import test_data_prefix, test_temp_dir +from mypy.defaults import PYTHON3_VERSION from mypy.errors import CompileError +from mypy.modulefinder import BuildSource from mypy.nodes import TypeInfo +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import ( + assert_string_arrays_equal, + find_test_files, + normalize_error_messages, + parse_options, + testfile_pyversion, +) + +# Semantic analyzer test cases: dump parse tree +# Semantic analysis test case description files. +semanal_files = find_test_files( + pattern="semanal-*.test", + exclude=[ + "semanal-errors-python310.test", + "semanal-errors.test", + "semanal-typeinfo.test", + "semanal-symtable.test", + ], +) -# Semantic analyser test cases: dump parse tree -# Semantic analysis test case description files. -semanal_files = ['semanal-basic.test', - 'semanal-expressions.test', - 'semanal-classes.test', - 'semanal-types.test', - 'semanal-modules.test', - 'semanal-statements.test', - 'semanal-abstractclasses.test', - 'semanal-namedtuple.test', - 'semanal-python2.test'] - - -class SemAnalSuite(Suite): - def cases(self): - c = [] - for f in semanal_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - test_semanal, test_temp_dir) - return c - - -def test_semanal(testcase): +if sys.version_info < (3, 10): + semanal_files.remove("semanal-python310.test") + + +def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Options: + options = parse_options(program_text, testcase, 1) + options.use_builtins_fixtures = True + options.semantic_analysis_only = True + options.show_traceback = True + options.python_version = PYTHON3_VERSION + options.force_uppercase_builtins = True + return options + + +class SemAnalSuite(DataSuite): + files = semanal_files + native_sep = True + + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_semanal(testcase) + + +def test_semanal(testcase: DataDrivenTestCase) -> None: """Perform a semantic analysis test case. The testcase argument contains a description of the test case @@ -44,185 +65,145 @@ def test_semanal(testcase): """ try: - src = '\n'.join(testcase.input) - result = build.build('main', - target=build.SEMANTIC_ANALYSIS, - program_text=src, - pyversion=testfile_pyversion(testcase.file), - flags=[build.TEST_BUILTINS], - alt_lib_path=test_temp_dir) - a = [] + src = "\n".join(testcase.input) + options = get_semanal_options(src, testcase) + options.python_version = testfile_pyversion(testcase.file) + result = build.build( + sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir + ) + a = result.errors + if a: + raise CompileError(a) # Include string representations of the source files in the actual # output. - for fnam in sorted(result.files.keys()): - f = result.files[fnam] - # Omit the builtins module and files with a special marker in the - # path. - # TODO the test is not reliable - if (not f.path.endswith((os.sep + 'builtins.py', - 'typing.py', - 'abc.py', - 'collections.py')) - and not os.path.basename(f.path).startswith('_') - and not os.path.splitext( - os.path.basename(f.path))[0].endswith('_')): - a += str(f).split('\n') + for module in sorted(result.files.keys()): + if module in testcase.test_modules: + a += result.files[module].str_with_options(options).split("\n") except CompileError as e: a = e.messages + if testcase.normalize_output: + a = normalize_error_messages(a) assert_string_arrays_equal( - testcase.output, a, - 'Invalid semantic analyzer output ({}, line {})'.format(testcase.file, - testcase.line)) + testcase.output, + a, + f"Invalid semantic analyzer output ({testcase.file}, line {testcase.line})", + ) + -# Semantic analyser error test cases +# Semantic analyzer error test cases -# Paths to files containing test case descriptions. -semanal_error_files = ['semanal-errors.test'] +class SemAnalErrorSuite(DataSuite): + files = ["semanal-errors.test"] + if sys.version_info >= (3, 10): + semanal_files.append("semanal-errors-python310.test") -class SemAnalErrorSuite(Suite): - def cases(self): - # Read test cases from test case description files. - c = [] - for f in semanal_error_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - test_semanal_error, test_temp_dir, True) - return c + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_semanal_error(testcase) -def test_semanal_error(testcase): +def test_semanal_error(testcase: DataDrivenTestCase) -> None: """Perform a test case.""" try: - src = '\n'.join(testcase.input) - build.build('main', - target=build.SEMANTIC_ANALYSIS, - program_text=src, - flags=[build.TEST_BUILTINS], - alt_lib_path=test_temp_dir) - raise AssertionError('No errors reported in {}, line {}'.format( - testcase.file, testcase.line)) + src = "\n".join(testcase.input) + res = build.build( + sources=[BuildSource("main", None, src)], + options=get_semanal_options(src, testcase), + alt_lib_path=test_temp_dir, + ) + a = res.errors except CompileError as e: # Verify that there was a compile error and that the error messages # are equivalent. - assert_string_arrays_equal( - testcase.output, normalize_error_messages(e.messages), - 'Invalid compiler output ({}, line {})'.format(testcase.file, - testcase.line)) - - -def normalize_error_messages(messages): - """Translate an array of error messages to use / as path separator.""" - - a = [] - for m in messages: - a.append(m.replace(os.sep, '/')) - return a + a = e.messages + if testcase.normalize_output: + a = normalize_error_messages(a) + assert_string_arrays_equal( + testcase.output, a, f"Invalid compiler output ({testcase.file}, line {testcase.line})" + ) # SymbolNode table export test cases -# Test case descriptions -semanal_symtable_files = ['semanal-symtable.test'] - -class SemAnalSymtableSuite(Suite): - def cases(self): - c = [] - for f in semanal_symtable_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - self.run_test, test_temp_dir) - return c +class SemAnalSymtableSuite(DataSuite): + required_out_section = True + files = ["semanal-symtable.test"] - def run_test(self, testcase): + def run_case(self, testcase: DataDrivenTestCase) -> None: """Perform a test case.""" try: # Build test case input. - src = '\n'.join(testcase.input) - result = build.build('main', - target=build.SEMANTIC_ANALYSIS, - program_text=src, - flags=[build.TEST_BUILTINS], - alt_lib_path=test_temp_dir) + src = "\n".join(testcase.input) + result = build.build( + sources=[BuildSource("main", None, src)], + options=get_semanal_options(src, testcase), + alt_lib_path=test_temp_dir, + ) # The output is the symbol table converted into a string. - a = [] - for f in sorted(result.files.keys()): - if f not in ('builtins', 'typing', 'abc'): - a.append('{}:'.format(f)) - for s in str(result.files[f].names).split('\n'): - a.append(' ' + s) + a = result.errors + if a: + raise CompileError(a) + for module in sorted(result.files.keys()): + if module in testcase.test_modules: + a.append(f"{module}:") + for s in str(result.files[module].names).split("\n"): + a.append(" " + s) except CompileError as e: a = e.messages assert_string_arrays_equal( - testcase.output, a, - 'Invalid semantic analyzer output ({}, line {})'.format( - testcase.file, testcase.line)) + testcase.output, + a, + f"Invalid semantic analyzer output ({testcase.file}, line {testcase.line})", + ) # Type info export test cases +class SemAnalTypeInfoSuite(DataSuite): + required_out_section = True + files = ["semanal-typeinfo.test"] -semanal_typeinfo_files = ['semanal-typeinfo.test'] - - -class SemAnalTypeInfoSuite(Suite): - def cases(self): - """Test case descriptions""" - c = [] - for f in semanal_typeinfo_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - self.run_test, test_temp_dir) - return c - - def run_test(self, testcase): + def run_case(self, testcase: DataDrivenTestCase) -> None: """Perform a test case.""" try: # Build test case input. - src = '\n'.join(testcase.input) - result = build.build('main', - target=build.SEMANTIC_ANALYSIS, - program_text=src, - flags=[build.TEST_BUILTINS], - alt_lib_path=test_temp_dir) + src = "\n".join(testcase.input) + result = build.build( + sources=[BuildSource("main", None, src)], + options=get_semanal_options(src, testcase), + alt_lib_path=test_temp_dir, + ) + a = result.errors + if a: + raise CompileError(a) # Collect all TypeInfos in top-level modules. typeinfos = TypeInfoMap() - for f in result.files.values(): - for n in f.names.values(): - if isinstance(n.node, TypeInfo): - typeinfos[n.fullname] = n.node + for module, file in result.files.items(): + if module in testcase.test_modules: + for n in file.names.values(): + if isinstance(n.node, TypeInfo): + assert n.fullname + if any(n.fullname.startswith(m + ".") for m in testcase.test_modules): + typeinfos[n.fullname] = n.node # The output is the symbol table converted into a string. - a = str(typeinfos).split('\n') + a = str(typeinfos).split("\n") except CompileError as e: a = e.messages assert_string_arrays_equal( - testcase.output, a, - 'Invalid semantic analyzer output ({}, line {})'.format( - testcase.file, testcase.line)) + testcase.output, + a, + f"Invalid semantic analyzer output ({testcase.file}, line {testcase.line})", + ) class TypeInfoMap(Dict[str, TypeInfo]): def __str__(self) -> str: - a = ['TypeInfoMap('] # type: List[str] + a: list[str] = ["TypeInfoMap("] for x, y in sorted(self.items()): - if isinstance(x, str) and (not x.startswith('builtins.') and - not x.startswith('typing.') and - not x.startswith('abc.')): - ti = ('\n' + ' ').join(str(y).split('\n')) - a.append(' {} : {}'.format(x, ti)) - a[-1] += ')' - return '\n'.join(a) - - -class CombinedSemAnalSuite(Suite): - def __init__(self): - self.test_semanal = SemAnalSuite() - self.test_semanal_errors = SemAnalErrorSuite() - self.test_semanal_symtable = SemAnalSymtableSuite() - self.test_semanal_typeinfos = SemAnalTypeInfoSuite() - super().__init__() - - -if __name__ == '__main__': - import sys - run_test(CombinedSemAnalSuite(), sys.argv[1:]) + ti = ("\n" + " ").join(str(y).split("\n")) + a.append(f" {x} : {ti}") + a[-1] += ")" + return "\n".join(a) diff --git a/mypy/test/testsolve.py b/mypy/test/testsolve.py index 0b470421280b..6566b03ef5e9 100644 --- a/mypy/test/testsolve.py +++ b/mypy/test/testsolve.py @@ -1,156 +1,285 @@ """Test cases for the constraint solver used in type inference.""" -import typing +from __future__ import annotations -from mypy.myunit import Suite, assert_equal, run_test -from mypy.constraints import SUPERTYPE_OF, SUBTYPE_OF, Constraint -from mypy.solve import solve_constraints -from mypy.typefixture import TypeFixture +from mypy.constraints import SUBTYPE_OF, SUPERTYPE_OF, Constraint +from mypy.solve import Bounds, Graph, solve_constraints, transitive_closure +from mypy.test.helpers import Suite, assert_equal +from mypy.test.typefixture import TypeFixture +from mypy.types import Type, TypeVarId, TypeVarLikeType, TypeVarType class SolveSuite(Suite): - def __init__(self): - super().__init__() + def setUp(self) -> None: self.fx = TypeFixture() - def test_empty_input(self): + def test_empty_input(self) -> None: self.assert_solve([], [], []) - def test_simple_supertype_constraints(self): - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.a)], - [(self.fx.a, self.fx.o)]) - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.a), - self.supc(self.fx.t, self.fx.b)], - [(self.fx.a, self.fx.o)]) - - def test_simple_subtype_constraints(self): - self.assert_solve(['T'], - [self.subc(self.fx.t, self.fx.a)], - [self.fx.a]) - self.assert_solve(['T'], - [self.subc(self.fx.t, self.fx.a), - self.subc(self.fx.t, self.fx.b)], - [self.fx.b]) - - def test_both_kinds_of_constraints(self): - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.b), - self.subc(self.fx.t, self.fx.a)], - [(self.fx.b, self.fx.a)]) - - def test_unsatisfiable_constraints(self): + def test_simple_supertype_constraints(self) -> None: + self.assert_solve([self.fx.t], [self.supc(self.fx.t, self.fx.a)], [self.fx.a]) + self.assert_solve( + [self.fx.t], + [self.supc(self.fx.t, self.fx.a), self.supc(self.fx.t, self.fx.b)], + [self.fx.a], + ) + + def test_simple_subtype_constraints(self) -> None: + self.assert_solve([self.fx.t], [self.subc(self.fx.t, self.fx.a)], [self.fx.a]) + self.assert_solve( + [self.fx.t], + [self.subc(self.fx.t, self.fx.a), self.subc(self.fx.t, self.fx.b)], + [self.fx.b], + ) + + def test_both_kinds_of_constraints(self) -> None: + self.assert_solve( + [self.fx.t], + [self.supc(self.fx.t, self.fx.b), self.subc(self.fx.t, self.fx.a)], + [self.fx.b], + ) + + def test_unsatisfiable_constraints(self) -> None: # The constraints are impossible to satisfy. - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.a), - self.subc(self.fx.t, self.fx.b)], - [None]) - - def test_exactly_specified_result(self): - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.b), - self.subc(self.fx.t, self.fx.b)], - [(self.fx.b, self.fx.b)]) - - def test_multiple_variables(self): - self.assert_solve(['T', 'S'], - [self.supc(self.fx.t, self.fx.b), - self.supc(self.fx.s, self.fx.c), - self.subc(self.fx.t, self.fx.a)], - [(self.fx.b, self.fx.a), (self.fx.c, self.fx.o)]) - - def test_no_constraints_for_var(self): - self.assert_solve(['T'], - [], - [self.fx.nonet]) - self.assert_solve(['T', 'S'], - [], - [self.fx.nonet, self.fx.nonet]) - self.assert_solve(['T', 'S'], - [self.supc(self.fx.s, self.fx.a)], - [self.fx.nonet, (self.fx.a, self.fx.o)]) - - def test_void_constraints(self): - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.void)], - [(self.fx.void, self.fx.void)]) - self.assert_solve(['T'], - [self.subc(self.fx.t, self.fx.void)], - [(self.fx.void, self.fx.void)]) - - # Both bounds void. - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.void), - self.subc(self.fx.t, self.fx.void)], - [(self.fx.void, self.fx.void)]) - - # Cannot infer any type. - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.a), - self.supc(self.fx.t, self.fx.void)], - [None]) - self.assert_solve(['T'], - [self.subc(self.fx.t, self.fx.a), - self.subc(self.fx.t, self.fx.void)], - [None]) - - def test_simple_constraints_with_dynamic_type(self): - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.anyt)], - [(self.fx.anyt, self.fx.anyt)]) - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.anyt), - self.supc(self.fx.t, self.fx.anyt)], - [(self.fx.anyt, self.fx.anyt)]) - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.anyt), - self.supc(self.fx.t, self.fx.a)], - [(self.fx.anyt, self.fx.anyt)]) - - self.assert_solve(['T'], - [self.subc(self.fx.t, self.fx.anyt)], - [(self.fx.anyt, self.fx.anyt)]) - self.assert_solve(['T'], - [self.subc(self.fx.t, self.fx.anyt), - self.subc(self.fx.t, self.fx.anyt)], - [(self.fx.anyt, self.fx.anyt)]) - #self.assert_solve(['T'], - # [self.subc(self.fx.t, self.fx.anyt), - # self.subc(self.fx.t, self.fx.a)], - # [(self.fx.anyt, self.fx.anyt)]) - # TODO: figure out what this should be after changes to meet(any, X) - - def test_both_normal_and_any_types_in_results(self): + self.assert_solve( + [self.fx.t], [self.supc(self.fx.t, self.fx.a), self.subc(self.fx.t, self.fx.b)], [None] + ) + + def test_exactly_specified_result(self) -> None: + self.assert_solve( + [self.fx.t], + [self.supc(self.fx.t, self.fx.b), self.subc(self.fx.t, self.fx.b)], + [self.fx.b], + ) + + def test_multiple_variables(self) -> None: + self.assert_solve( + [self.fx.t, self.fx.s], + [ + self.supc(self.fx.t, self.fx.b), + self.supc(self.fx.s, self.fx.c), + self.subc(self.fx.t, self.fx.a), + ], + [self.fx.b, self.fx.c], + ) + + def test_no_constraints_for_var(self) -> None: + self.assert_solve([self.fx.t], [], [self.fx.uninhabited]) + self.assert_solve([self.fx.t, self.fx.s], [], [self.fx.uninhabited, self.fx.uninhabited]) + self.assert_solve( + [self.fx.t, self.fx.s], + [self.supc(self.fx.s, self.fx.a)], + [self.fx.uninhabited, self.fx.a], + ) + + def test_simple_constraints_with_dynamic_type(self) -> None: + self.assert_solve([self.fx.t], [self.supc(self.fx.t, self.fx.anyt)], [self.fx.anyt]) + self.assert_solve( + [self.fx.t], + [self.supc(self.fx.t, self.fx.anyt), self.supc(self.fx.t, self.fx.anyt)], + [self.fx.anyt], + ) + self.assert_solve( + [self.fx.t], + [self.supc(self.fx.t, self.fx.anyt), self.supc(self.fx.t, self.fx.a)], + [self.fx.anyt], + ) + + self.assert_solve([self.fx.t], [self.subc(self.fx.t, self.fx.anyt)], [self.fx.anyt]) + self.assert_solve( + [self.fx.t], + [self.subc(self.fx.t, self.fx.anyt), self.subc(self.fx.t, self.fx.anyt)], + [self.fx.anyt], + ) + # self.assert_solve([self.fx.t], + # [self.subc(self.fx.t, self.fx.anyt), + # self.subc(self.fx.t, self.fx.a)], + # [self.fx.anyt]) + # TODO: figure out what this should be after changes to meet(any, X) + + def test_both_normal_and_any_types_in_results(self) -> None: # If one of the bounds is any, we promote the other bound to # any as well, since otherwise the type range does not make sense. - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.a), - self.subc(self.fx.t, self.fx.anyt)], - [(self.fx.anyt, self.fx.anyt)]) - - self.assert_solve(['T'], - [self.supc(self.fx.t, self.fx.anyt), - self.subc(self.fx.t, self.fx.a)], - [(self.fx.anyt, self.fx.anyt)]) - - def assert_solve(self, vars, constraints, results): - res = [] - for r in results: - if isinstance(r, tuple): - res.append(r[0]) - else: - res.append(r) - actual = solve_constraints(vars, constraints) - assert_equal(str(actual), str(res)) - - def supc(self, type_var, bound): - return Constraint(type_var.name, SUPERTYPE_OF, bound) - - def subc(self, type_var, bound): - return Constraint(type_var.name, SUBTYPE_OF, bound) - - -if __name__ == '__main__': - import sys - run_test(SolveSuite(), sys.argv[1:]) + self.assert_solve( + [self.fx.t], + [self.supc(self.fx.t, self.fx.a), self.subc(self.fx.t, self.fx.anyt)], + [self.fx.anyt], + ) + + self.assert_solve( + [self.fx.t], + [self.supc(self.fx.t, self.fx.anyt), self.subc(self.fx.t, self.fx.a)], + [self.fx.anyt], + ) + + def test_poly_no_constraints(self) -> None: + self.assert_solve( + [self.fx.t, self.fx.u], + [], + [self.fx.uninhabited, self.fx.uninhabited], + allow_polymorphic=True, + ) + + def test_poly_trivial_free(self) -> None: + self.assert_solve( + [self.fx.t, self.fx.u], + [self.subc(self.fx.t, self.fx.a)], + [self.fx.a, self.fx.u], + [self.fx.u], + allow_polymorphic=True, + ) + + def test_poly_free_pair(self) -> None: + self.assert_solve( + [self.fx.t, self.fx.u], + [self.subc(self.fx.t, self.fx.u)], + [self.fx.t, self.fx.t], + [self.fx.t], + allow_polymorphic=True, + ) + + def test_poly_free_pair_with_bounds(self) -> None: + t_prime = self.fx.t.copy_modified(upper_bound=self.fx.b) + self.assert_solve( + [self.fx.t, self.fx.ub], + [self.subc(self.fx.t, self.fx.ub)], + [t_prime, t_prime], + [t_prime], + allow_polymorphic=True, + ) + + def test_poly_free_pair_with_bounds_uninhabited(self) -> None: + self.assert_solve( + [self.fx.ub, self.fx.uc], + [self.subc(self.fx.ub, self.fx.uc)], + [self.fx.uninhabited, self.fx.uninhabited], + [], + allow_polymorphic=True, + ) + + def test_poly_bounded_chain(self) -> None: + # B <: T <: U <: S <: A + self.assert_solve( + [self.fx.t, self.fx.u, self.fx.s], + [ + self.supc(self.fx.t, self.fx.b), + self.subc(self.fx.t, self.fx.u), + self.subc(self.fx.u, self.fx.s), + self.subc(self.fx.s, self.fx.a), + ], + [self.fx.b, self.fx.b, self.fx.b], + allow_polymorphic=True, + ) + + def test_poly_reverse_overlapping_chain(self) -> None: + # A :> T <: S :> B + self.assert_solve( + [self.fx.t, self.fx.s], + [ + self.subc(self.fx.t, self.fx.s), + self.subc(self.fx.t, self.fx.a), + self.supc(self.fx.s, self.fx.b), + ], + [self.fx.a, self.fx.a], + allow_polymorphic=True, + ) + + def test_poly_reverse_split_chain(self) -> None: + # B :> T <: S :> A + self.assert_solve( + [self.fx.t, self.fx.s], + [ + self.subc(self.fx.t, self.fx.s), + self.subc(self.fx.t, self.fx.b), + self.supc(self.fx.s, self.fx.a), + ], + [self.fx.b, self.fx.a], + allow_polymorphic=True, + ) + + def test_poly_unsolvable_chain(self) -> None: + # A <: T <: U <: S <: B + self.assert_solve( + [self.fx.t, self.fx.u, self.fx.s], + [ + self.supc(self.fx.t, self.fx.a), + self.subc(self.fx.t, self.fx.u), + self.subc(self.fx.u, self.fx.s), + self.subc(self.fx.s, self.fx.b), + ], + [None, None, None], + allow_polymorphic=True, + ) + + def test_simple_chain_closure(self) -> None: + self.assert_transitive_closure( + [self.fx.t.id, self.fx.s.id], + [ + self.supc(self.fx.t, self.fx.b), + self.subc(self.fx.t, self.fx.s), + self.subc(self.fx.s, self.fx.a), + ], + {(self.fx.t.id, self.fx.s.id)}, + {self.fx.t.id: {self.fx.b}, self.fx.s.id: {self.fx.b}}, + {self.fx.t.id: {self.fx.a}, self.fx.s.id: {self.fx.a}}, + ) + + def test_reverse_chain_closure(self) -> None: + self.assert_transitive_closure( + [self.fx.t.id, self.fx.s.id], + [ + self.subc(self.fx.t, self.fx.s), + self.subc(self.fx.t, self.fx.a), + self.supc(self.fx.s, self.fx.b), + ], + {(self.fx.t.id, self.fx.s.id)}, + {self.fx.t.id: set(), self.fx.s.id: {self.fx.b}}, + {self.fx.t.id: {self.fx.a}, self.fx.s.id: set()}, + ) + + def test_secondary_constraint_closure(self) -> None: + self.assert_transitive_closure( + [self.fx.t.id, self.fx.s.id], + [self.supc(self.fx.s, self.fx.gt), self.subc(self.fx.s, self.fx.ga)], + set(), + {self.fx.t.id: set(), self.fx.s.id: {self.fx.gt}}, + {self.fx.t.id: {self.fx.a}, self.fx.s.id: {self.fx.ga}}, + ) + + def assert_solve( + self, + vars: list[TypeVarLikeType], + constraints: list[Constraint], + results: list[None | Type], + free_vars: list[TypeVarLikeType] | None = None, + allow_polymorphic: bool = False, + ) -> None: + if free_vars is None: + free_vars = [] + actual, actual_free = solve_constraints( + vars, constraints, allow_polymorphic=allow_polymorphic + ) + assert_equal(actual, results) + assert_equal(actual_free, free_vars) + + def assert_transitive_closure( + self, + vars: list[TypeVarId], + constraints: list[Constraint], + graph: Graph, + lowers: Bounds, + uppers: Bounds, + ) -> None: + actual_graph, actual_lowers, actual_uppers = transitive_closure(vars, constraints) + # Add trivial elements. + for v in vars: + graph.add((v, v)) + assert_equal(actual_graph, graph) + assert_equal(dict(actual_lowers), lowers) + assert_equal(dict(actual_uppers), uppers) + + def supc(self, type_var: TypeVarType, bound: Type) -> Constraint: + return Constraint(type_var, SUPERTYPE_OF, bound) + + def subc(self, type_var: TypeVarType, bound: Type) -> Constraint: + return Constraint(type_var, SUBTYPE_OF, bound) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py new file mode 100644 index 000000000000..dffa1aa80c5d --- /dev/null +++ b/mypy/test/teststubgen.py @@ -0,0 +1,1463 @@ +from __future__ import annotations + +import io +import os.path +import re +import shutil +import sys +import tempfile +import unittest +from types import ModuleType +from typing import Any + +import pytest + +from mypy.errors import CompileError +from mypy.moduleinspect import InspectError, ModuleInspect +from mypy.stubdoc import ( + ArgSig, + FunctionSig, + build_signature, + find_unique_signatures, + infer_arg_sig_from_anon_docstring, + infer_prop_type_from_docstring, + infer_sig_from_docstring, + is_valid_type, + parse_all_signatures, + parse_signature, +) +from mypy.stubgen import ( + Options, + collect_build_targets, + generate_stubs, + is_blacklisted_path, + is_non_library_module, + mypy_options, + parse_options, +) +from mypy.stubgenc import InspectionStubGenerator, infer_c_method_args +from mypy.stubutil import ( + ClassInfo, + FunctionContext, + common_dir_prefix, + infer_method_ret_type, + remove_misplaced_type_comments, + walk_packages, +) +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_equal, assert_string_arrays_equal, local_sys_path_set + + +class StubgenCmdLineSuite(unittest.TestCase): + """Test cases for processing command-line options and finding files.""" + + @unittest.skipIf(sys.platform == "win32", "clean up fails on Windows") + def test_files_found(self) -> None: + current = os.getcwd() + with tempfile.TemporaryDirectory() as tmp: + try: + os.chdir(tmp) + os.mkdir("subdir") + self.make_file("subdir", "a.py") + self.make_file("subdir", "b.py") + os.mkdir(os.path.join("subdir", "pack")) + self.make_file("subdir", "pack", "__init__.py") + opts = parse_options(["subdir"]) + py_mods, pyi_mods, c_mods = collect_build_targets(opts, mypy_options(opts)) + assert_equal(pyi_mods, []) + assert_equal(c_mods, []) + files = {mod.path for mod in py_mods} + assert_equal( + files, + { + os.path.join("subdir", "pack", "__init__.py"), + os.path.join("subdir", "a.py"), + os.path.join("subdir", "b.py"), + }, + ) + finally: + os.chdir(current) + + @unittest.skipIf(sys.platform == "win32", "clean up fails on Windows") + def test_packages_found(self) -> None: + current = os.getcwd() + with tempfile.TemporaryDirectory() as tmp: + try: + os.chdir(tmp) + os.mkdir("pack") + self.make_file("pack", "__init__.py", content="from . import a, b") + self.make_file("pack", "a.py") + self.make_file("pack", "b.py") + opts = parse_options(["-p", "pack"]) + py_mods, pyi_mods, c_mods = collect_build_targets(opts, mypy_options(opts)) + assert_equal(pyi_mods, []) + assert_equal(c_mods, []) + files = {os.path.relpath(mod.path or "FAIL") for mod in py_mods} + assert_equal( + files, + { + os.path.join("pack", "__init__.py"), + os.path.join("pack", "a.py"), + os.path.join("pack", "b.py"), + }, + ) + finally: + os.chdir(current) + + @unittest.skipIf(sys.platform == "win32", "clean up fails on Windows") + def test_module_not_found(self) -> None: + current = os.getcwd() + captured_output = io.StringIO() + sys.stdout = captured_output + with tempfile.TemporaryDirectory() as tmp: + try: + os.chdir(tmp) + self.make_file(tmp, "mymodule.py", content="import a") + opts = parse_options(["-m", "mymodule"]) + collect_build_targets(opts, mypy_options(opts)) + assert captured_output.getvalue() == "" + finally: + sys.stdout = sys.__stdout__ + os.chdir(current) + + def make_file(self, *path: str, content: str = "") -> None: + file = os.path.join(*path) + with open(file, "w") as f: + f.write(content) + + def run(self, result: Any | None = None) -> Any | None: + with local_sys_path_set(): + return super().run(result) + + +class StubgenCliParseSuite(unittest.TestCase): + def test_walk_packages(self) -> None: + with ModuleInspect() as m: + assert_equal(set(walk_packages(m, ["mypy.errors"])), {"mypy.errors"}) + + assert_equal( + set(walk_packages(m, ["mypy.errors", "mypy.stubgen"])), + {"mypy.errors", "mypy.stubgen"}, + ) + + all_mypy_packages = set(walk_packages(m, ["mypy"])) + self.assertTrue( + all_mypy_packages.issuperset( + {"mypy", "mypy.errors", "mypy.stubgen", "mypy.test", "mypy.test.helpers"} + ) + ) + + +class StubgenUtilSuite(unittest.TestCase): + """Unit tests for stubgen utility functions.""" + + def test_parse_signature(self) -> None: + self.assert_parse_signature("func()", ("func", [], [])) + + def test_parse_signature_with_args(self) -> None: + self.assert_parse_signature("func(arg)", ("func", ["arg"], [])) + self.assert_parse_signature("do(arg, arg2)", ("do", ["arg", "arg2"], [])) + + def test_parse_signature_with_optional_args(self) -> None: + self.assert_parse_signature("func([arg])", ("func", [], ["arg"])) + self.assert_parse_signature("func(arg[, arg2])", ("func", ["arg"], ["arg2"])) + self.assert_parse_signature("func([arg[, arg2]])", ("func", [], ["arg", "arg2"])) + + def test_parse_signature_with_default_arg(self) -> None: + self.assert_parse_signature("func(arg=None)", ("func", [], ["arg"])) + self.assert_parse_signature("func(arg, arg2=None)", ("func", ["arg"], ["arg2"])) + self.assert_parse_signature('func(arg=1, arg2="")', ("func", [], ["arg", "arg2"])) + + def test_parse_signature_with_qualified_function(self) -> None: + self.assert_parse_signature("ClassName.func(arg)", ("func", ["arg"], [])) + + def test_parse_signature_with_kw_only_arg(self) -> None: + self.assert_parse_signature( + "ClassName.func(arg, *, arg2=1)", ("func", ["arg", "*"], ["arg2"]) + ) + + def test_parse_signature_with_star_arg(self) -> None: + self.assert_parse_signature("ClassName.func(arg, *args)", ("func", ["arg", "*args"], [])) + + def test_parse_signature_with_star_star_arg(self) -> None: + self.assert_parse_signature("ClassName.func(arg, **args)", ("func", ["arg", "**args"], [])) + + def assert_parse_signature(self, sig: str, result: tuple[str, list[str], list[str]]) -> None: + assert_equal(parse_signature(sig), result) + + def test_build_signature(self) -> None: + assert_equal(build_signature([], []), "()") + assert_equal(build_signature(["arg"], []), "(arg)") + assert_equal(build_signature(["arg", "arg2"], []), "(arg, arg2)") + assert_equal(build_signature(["arg"], ["arg2"]), "(arg, arg2=...)") + assert_equal(build_signature(["arg"], ["arg2", "**x"]), "(arg, arg2=..., **x)") + + def test_parse_all_signatures(self) -> None: + assert_equal( + parse_all_signatures( + [ + "random text", + ".. function:: fn(arg", + ".. function:: fn()", + " .. method:: fn2(arg)", + ] + ), + ([("fn", "()"), ("fn2", "(arg)")], []), + ) + + def test_find_unique_signatures(self) -> None: + assert_equal( + find_unique_signatures( + [ + ("func", "()"), + ("func", "()"), + ("func2", "()"), + ("func2", "(arg)"), + ("func3", "(arg, arg2)"), + ] + ), + [("func", "()"), ("func3", "(arg, arg2)")], + ) + + def test_infer_sig_from_docstring(self) -> None: + assert_equal( + infer_sig_from_docstring("\nfunc(x) - y", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x")], ret_type="Any")], + ) + assert_equal( + infer_sig_from_docstring("\nfunc(x)", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x")], ret_type="Any")], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x, Y_a=None)", "func"), + [ + FunctionSig( + name="func", + args=[ArgSig(name="x"), ArgSig(name="Y_a", default=True)], + ret_type="Any", + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x, Y_a=3)", "func"), + [ + FunctionSig( + name="func", + args=[ArgSig(name="x"), ArgSig(name="Y_a", default=True)], + ret_type="Any", + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x, Y_a=[1, 2, 3])", "func"), + [ + FunctionSig( + name="func", + args=[ArgSig(name="x"), ArgSig(name="Y_a", default=True)], + ret_type="Any", + ) + ], + ) + + assert_equal(infer_sig_from_docstring("\nafunc(x) - y", "func"), []) + assert_equal(infer_sig_from_docstring("\nfunc(x, y", "func"), []) + assert_equal( + infer_sig_from_docstring("\nfunc(x=z(y))", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x", default=True)], ret_type="Any")], + ) + + assert_equal(infer_sig_from_docstring("\nfunc x", "func"), []) + # Try to infer signature from type annotation. + assert_equal( + infer_sig_from_docstring("\nfunc(x: int)", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x", type="int")], ret_type="Any")], + ) + assert_equal( + infer_sig_from_docstring("\nfunc(x: int=3)", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type="int", default=True)], ret_type="Any" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x=3)", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type=None, default=True)], ret_type="Any" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc() -> int", "func"), + [FunctionSig(name="func", args=[], ret_type="int")], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x: int=3) -> int", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type="int", default=True)], ret_type="int" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x: int=3) -> int \n", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type="int", default=True)], ret_type="int" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x: Tuple[int, str]) -> str", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type="Tuple[int,str]")], ret_type="str" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring( + "\nfunc(x: Tuple[int, Tuple[str, int], str], y: int) -> str", "func" + ), + [ + FunctionSig( + name="func", + args=[ + ArgSig(name="x", type="Tuple[int,Tuple[str,int],str]"), + ArgSig(name="y", type="int"), + ], + ret_type="str", + ) + ], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x: foo.bar)", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x", type="foo.bar")], ret_type="Any")], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(x: list=[1,2,[3,4]])", "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type="list", default=True)], ret_type="Any" + ) + ], + ) + + assert_equal( + infer_sig_from_docstring('\nfunc(x: str="nasty[")', "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type="str", default=True)], ret_type="Any" + ) + ], + ) + + assert_equal(infer_sig_from_docstring("\nfunc[(x: foo.bar, invalid]", "func"), []) + + assert_equal( + infer_sig_from_docstring("\nfunc(x: invalid::type)", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x", type=None)], ret_type="Any")], + ) + + assert_equal( + infer_sig_from_docstring('\nfunc(x: str="")', "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="x", type="str", default=True)], ret_type="Any" + ) + ], + ) + + def test_infer_sig_from_docstring_duplicate_args(self) -> None: + assert_equal( + infer_sig_from_docstring("\nfunc(x, x) -> str\nfunc(x, y) -> int", "func"), + [FunctionSig(name="func", args=[ArgSig(name="x"), ArgSig(name="y")], ret_type="int")], + ) + + def test_infer_sig_from_docstring_bad_indentation(self) -> None: + assert_equal( + infer_sig_from_docstring( + """ + x + x + x + """, + "func", + ), + None, + ) + + def test_infer_arg_sig_from_anon_docstring(self) -> None: + assert_equal( + infer_arg_sig_from_anon_docstring("(*args, **kwargs)"), + [ArgSig(name="*args"), ArgSig(name="**kwargs")], + ) + + assert_equal( + infer_arg_sig_from_anon_docstring( + "(x: Tuple[int, Tuple[str, int], str]=(1, ('a', 2), 'y'), y: int=4)" + ), + [ + ArgSig(name="x", type="Tuple[int,Tuple[str,int],str]", default=True), + ArgSig(name="y", type="int", default=True), + ], + ) + + def test_infer_prop_type_from_docstring(self) -> None: + assert_equal(infer_prop_type_from_docstring("str: A string."), "str") + assert_equal(infer_prop_type_from_docstring("Optional[int]: An int."), "Optional[int]") + assert_equal( + infer_prop_type_from_docstring("Tuple[int, int]: A tuple."), "Tuple[int, int]" + ) + assert_equal(infer_prop_type_from_docstring("\nstr: A string."), None) + + def test_infer_sig_from_docstring_square_brackets(self) -> None: + assert ( + infer_sig_from_docstring("fetch_row([maxrows, how]) -- Fetches stuff", "fetch_row") + == [] + ) + + def test_remove_misplaced_type_comments_1(self) -> None: + good = """ + \u1234 + def f(x): # type: (int) -> int + + def g(x): + # type: (int) -> int + + def h(): + + # type: () int + + x = 1 # type: int + """ + + assert_equal(remove_misplaced_type_comments(good), good) + + def test_remove_misplaced_type_comments_2(self) -> None: + bad = """ + def f(x): + # type: Callable[[int], int] + pass + + # type: "foo" + # type: 'bar' + x = 1 + # type: int + """ + bad_fixed = """ + def f(x): + + pass + + + + x = 1 + + """ + assert_equal(remove_misplaced_type_comments(bad), bad_fixed) + + def test_remove_misplaced_type_comments_3(self) -> None: + bad = ''' + def f(x): + """docstring""" + # type: (int) -> int + pass + + def g(x): + """docstring + """ + # type: (int) -> int + pass + ''' + bad_fixed = ''' + def f(x): + """docstring""" + + pass + + def g(x): + """docstring + """ + + pass + ''' + assert_equal(remove_misplaced_type_comments(bad), bad_fixed) + + def test_remove_misplaced_type_comments_4(self) -> None: + bad = """ + def f(x): + '''docstring''' + # type: (int) -> int + pass + + def g(x): + '''docstring + ''' + # type: (int) -> int + pass + """ + bad_fixed = """ + def f(x): + '''docstring''' + + pass + + def g(x): + '''docstring + ''' + + pass + """ + assert_equal(remove_misplaced_type_comments(bad), bad_fixed) + + def test_remove_misplaced_type_comments_5(self) -> None: + bad = """ + def f(x): + # type: (int, List[Any], + # float, bool) -> int + pass + + def g(x): + # type: (int, List[Any]) + pass + """ + bad_fixed = """ + def f(x): + + # float, bool) -> int + pass + + def g(x): + + pass + """ + assert_equal(remove_misplaced_type_comments(bad), bad_fixed) + + def test_remove_misplaced_type_comments_bytes(self) -> None: + original = b""" + \xbf + def f(x): # type: (int) -> int + + def g(x): + # type: (int) -> int + pass + + def h(): + # type: int + pass + + x = 1 # type: int + """ + + dest = b""" + \xbf + def f(x): # type: (int) -> int + + def g(x): + # type: (int) -> int + pass + + def h(): + + pass + + x = 1 # type: int + """ + + assert_equal(remove_misplaced_type_comments(original), dest) + + @unittest.skipIf(sys.platform == "win32", "Tests building the paths common ancestor on *nix") + def test_common_dir_prefix_unix(self) -> None: + assert common_dir_prefix([]) == "." + assert common_dir_prefix(["x.pyi"]) == "." + assert common_dir_prefix(["./x.pyi"]) == "." + assert common_dir_prefix(["foo/bar/x.pyi"]) == "foo/bar" + assert common_dir_prefix(["foo/bar/x.pyi", "foo/bar/y.pyi"]) == "foo/bar" + assert common_dir_prefix(["foo/bar/x.pyi", "foo/y.pyi"]) == "foo" + assert common_dir_prefix(["foo/x.pyi", "foo/bar/y.pyi"]) == "foo" + assert common_dir_prefix(["foo/bar/zar/x.pyi", "foo/y.pyi"]) == "foo" + assert common_dir_prefix(["foo/x.pyi", "foo/bar/zar/y.pyi"]) == "foo" + assert common_dir_prefix(["foo/bar/zar/x.pyi", "foo/bar/y.pyi"]) == "foo/bar" + assert common_dir_prefix(["foo/bar/x.pyi", "foo/bar/zar/y.pyi"]) == "foo/bar" + assert common_dir_prefix([r"foo/bar\x.pyi"]) == "foo" + assert common_dir_prefix([r"foo\bar/x.pyi"]) == r"foo\bar" + + @unittest.skipIf( + sys.platform != "win32", "Tests building the paths common ancestor on Windows" + ) + def test_common_dir_prefix_win(self) -> None: + assert common_dir_prefix(["x.pyi"]) == "." + assert common_dir_prefix([r".\x.pyi"]) == "." + assert common_dir_prefix([r"foo\bar\x.pyi"]) == r"foo\bar" + assert common_dir_prefix([r"foo\bar\x.pyi", r"foo\bar\y.pyi"]) == r"foo\bar" + assert common_dir_prefix([r"foo\bar\x.pyi", r"foo\y.pyi"]) == "foo" + assert common_dir_prefix([r"foo\x.pyi", r"foo\bar\y.pyi"]) == "foo" + assert common_dir_prefix([r"foo\bar\zar\x.pyi", r"foo\y.pyi"]) == "foo" + assert common_dir_prefix([r"foo\x.pyi", r"foo\bar\zar\y.pyi"]) == "foo" + assert common_dir_prefix([r"foo\bar\zar\x.pyi", r"foo\bar\y.pyi"]) == r"foo\bar" + assert common_dir_prefix([r"foo\bar\x.pyi", r"foo\bar\zar\y.pyi"]) == r"foo\bar" + assert common_dir_prefix([r"foo/bar\x.pyi"]) == r"foo\bar" + assert common_dir_prefix([r"foo\bar/x.pyi"]) == r"foo\bar" + assert common_dir_prefix([r"foo/bar/x.pyi"]) == r"foo\bar" + + def test_function_context_nested_classes(self) -> None: + ctx = FunctionContext( + module_name="spangle", + name="foo", + class_info=ClassInfo( + name="Nested", self_var="self", parent=ClassInfo(name="Parent", self_var="self") + ), + ) + assert ctx.fullname == "spangle.Parent.Nested.foo" + + +class StubgenHelpersSuite(unittest.TestCase): + def test_is_blacklisted_path(self) -> None: + assert not is_blacklisted_path("foo/bar.py") + assert not is_blacklisted_path("foo.py") + assert not is_blacklisted_path("foo/xvendor/bar.py") + assert not is_blacklisted_path("foo/vendorx/bar.py") + assert is_blacklisted_path("foo/vendor/bar.py") + assert is_blacklisted_path("foo/vendored/bar.py") + assert is_blacklisted_path("foo/vendored/bar/thing.py") + assert is_blacklisted_path("foo/six.py") + + def test_is_non_library_module(self) -> None: + assert not is_non_library_module("foo") + assert not is_non_library_module("foo.bar") + + # The following could be test modules, but we are very conservative and + # don't treat them as such since they could plausibly be real modules. + assert not is_non_library_module("foo.bartest") + assert not is_non_library_module("foo.bartests") + assert not is_non_library_module("foo.testbar") + + assert is_non_library_module("foo.test") + assert is_non_library_module("foo.test.foo") + assert is_non_library_module("foo.tests") + assert is_non_library_module("foo.tests.foo") + assert is_non_library_module("foo.testing.foo") + assert is_non_library_module("foo.SelfTest.foo") + + assert is_non_library_module("foo.test_bar") + assert is_non_library_module("foo.bar_tests") + assert is_non_library_module("foo.testing") + assert is_non_library_module("foo.conftest") + assert is_non_library_module("foo.bar_test_util") + assert is_non_library_module("foo.bar_test_utils") + assert is_non_library_module("foo.bar_test_base") + + assert is_non_library_module("foo.setup") + + assert is_non_library_module("foo.__main__") + + +class StubgenPythonSuite(DataSuite): + """Data-driven end-to-end test cases that generate stub files. + + You can use these magic test case name suffixes: + + *_semanal + Run semantic analysis (slow as this uses real stubs -- only use + when necessary) + *_import + Import module and perform runtime introspection (in the current + process!) + + You can use these magic comments: + + # flags: --some-stubgen-option ... + Specify custom stubgen options + + # modules: module1 module2 ... + Specify which modules to output (by default only 'main') + """ + + required_out_section = True + base_path = "." + files = ["stubgen.test"] + + @unittest.skipIf(sys.platform == "win32", "clean up fails on Windows") + def run_case(self, testcase: DataDrivenTestCase) -> None: + with local_sys_path_set(): + self.run_case_inner(testcase) + + def run_case_inner(self, testcase: DataDrivenTestCase) -> None: + extra = [] # Extra command-line args + mods = [] # Module names to process + source = "\n".join(testcase.input) + for file, content in testcase.files + [("./main.py", source)]: + # Strip ./ prefix and .py suffix. + mod = file[2:-3].replace("/", ".") + if mod.endswith(".__init__"): + mod, _, _ = mod.rpartition(".") + mods.append(mod) + if "-p " not in source: + extra.extend(["-m", mod]) + with open(file, "w") as f: + f.write(content) + + options = self.parse_flags(source, extra) + if sys.version_info < options.pyversion: + pytest.skip() + modules = self.parse_modules(source) + out_dir = "out" + try: + try: + if testcase.name.endswith("_inspect"): + options.inspect = True + else: + if not testcase.name.endswith("_import"): + options.no_import = True + if not testcase.name.endswith("_semanal"): + options.parse_only = True + + generate_stubs(options) + a: list[str] = [] + for module in modules: + fnam = module_to_path(out_dir, module) + self.add_file(fnam, a, header=len(modules) > 1) + except CompileError as e: + a = e.messages + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) + finally: + for mod in mods: + if mod in sys.modules: + del sys.modules[mod] + shutil.rmtree(out_dir) + + def parse_flags(self, program_text: str, extra: list[str]) -> Options: + flags = re.search("# flags: (.*)$", program_text, flags=re.MULTILINE) + pyversion = None + if flags: + flag_list = flags.group(1).split() + for i, flag in enumerate(flag_list): + if flag.startswith("--python-version="): + pyversion = flag.split("=", 1)[1] + del flag_list[i] + break + else: + flag_list = [] + options = parse_options(flag_list + extra) + if pyversion: + # A hack to allow testing old python versions with new language constructs + # This should be rarely used in general as stubgen output should not be version-specific + major, minor = pyversion.split(".", 1) + options.pyversion = (int(major), int(minor)) + if "--verbose" not in flag_list: + options.quiet = True + else: + options.verbose = True + return options + + def parse_modules(self, program_text: str) -> list[str]: + modules = re.search("# modules: (.*)$", program_text, flags=re.MULTILINE) + if modules: + return modules.group(1).split() + else: + return ["main"] + + def add_file(self, path: str, result: list[str], header: bool) -> None: + if not os.path.exists(path): + result.append("<%s was not generated>" % path.replace("\\", "/")) + return + if header: + result.append(f"# {path[4:]}") + with open(path, encoding="utf8") as file: + result.extend(file.read().splitlines()) + + +self_arg = ArgSig(name="self") + + +class TestBaseClass: + pass + + +class TestClass(TestBaseClass): + pass + + +class StubgencSuite(unittest.TestCase): + """Unit tests for stub generation from C modules using introspection. + + Note that these don't cover a lot! + """ + + def test_infer_hash_sig(self) -> None: + assert_equal(infer_c_method_args("__hash__"), [self_arg]) + assert_equal(infer_method_ret_type("__hash__"), "int") + + def test_infer_getitem_sig(self) -> None: + assert_equal(infer_c_method_args("__getitem__"), [self_arg, ArgSig(name="index")]) + + def test_infer_setitem_sig(self) -> None: + assert_equal( + infer_c_method_args("__setitem__"), + [self_arg, ArgSig(name="index"), ArgSig(name="object")], + ) + assert_equal(infer_method_ret_type("__setitem__"), "None") + + def test_infer_eq_op_sig(self) -> None: + for op in ("eq", "ne", "lt", "le", "gt", "ge"): + assert_equal( + infer_c_method_args(f"__{op}__"), [self_arg, ArgSig(name="other", type="object")] + ) + + def test_infer_binary_op_sig(self) -> None: + for op in ("add", "radd", "sub", "rsub", "mul", "rmul"): + assert_equal(infer_c_method_args(f"__{op}__"), [self_arg, ArgSig(name="other")]) + + def test_infer_equality_op_sig(self) -> None: + for op in ("eq", "ne", "lt", "le", "gt", "ge", "contains"): + assert_equal(infer_method_ret_type(f"__{op}__"), "bool") + + def test_infer_unary_op_sig(self) -> None: + for op in ("neg", "pos"): + assert_equal(infer_c_method_args(f"__{op}__"), [self_arg]) + + def test_infer_cast_sig(self) -> None: + for op in ("float", "bool", "bytes", "int"): + assert_equal(infer_method_ret_type(f"__{op}__"), op) + + def test_generate_class_stub_no_crash_for_object(self) -> None: + output: list[str] = [] + mod = ModuleType("module", "") # any module is fine + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + + gen.generate_class_stub("alias", object, output) + assert_equal(gen.get_imports().splitlines(), []) + assert_equal(output[0], "class alias:") + + def test_generate_class_stub_variable_type_annotation(self) -> None: + # This class mimics the stubgen unit test 'testClassVariable' + class TestClassVariableCls: + x = 1 + + output: list[str] = [] + mod = ModuleType("module", "") # any module is fine + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_class_stub("C", TestClassVariableCls, output) + assert_equal(gen.get_imports().splitlines(), ["from typing import ClassVar"]) + assert_equal(output, ["class C:", " x: ClassVar[int] = ..."]) + + def test_non_c_generate_signature_with_kw_only_args(self) -> None: + class TestClass: + def test( + self, arg0: str, *, keyword_only: str, keyword_only_with_default: int = 7 + ) -> None: + pass + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.is_c_module = False + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo( + self_var="self", + cls=TestClass, + name="TestClass", + docstring=getattr(TestClass, "__doc__", None), + ), + ) + assert_equal( + output, + [ + "def test(self, arg0: str, *, keyword_only: str, keyword_only_with_default: int = ...) -> None: ..." + ], + ) + + def test_generate_c_type_inheritance(self) -> None: + class TestClass(KeyError): + pass + + output: list[str] = [] + mod = ModuleType("module, ") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_class_stub("C", TestClass, output) + assert_equal(output, ["class C(KeyError): ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_inheritance_same_module(self) -> None: + output: list[str] = [] + mod = ModuleType(TestBaseClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_class_stub("C", TestClass, output) + assert_equal(output, ["class C(TestBaseClass): ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_inheritance_other_module(self) -> None: + import argparse + + class TestClass(argparse.Action): + pass + + output: list[str] = [] + mod = ModuleType("module", "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_class_stub("C", TestClass, output) + assert_equal(output, ["class C(argparse.Action): ..."]) + assert_equal(gen.get_imports().splitlines(), ["import argparse"]) + + def test_generate_c_type_inheritance_builtin_type(self) -> None: + class TestClass(type): + pass + + output: list[str] = [] + mod = ModuleType("module", "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_class_stub("C", TestClass, output) + assert_equal(output, ["class C(type): ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_with_docstring(self) -> None: + class TestClass: + def test(self, arg0: str) -> None: + """ + test(self: TestClass, arg0: int) + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: int) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_with_docstring_no_self_arg(self) -> None: + class TestClass: + def test(self, arg0: str) -> None: + """ + test(arg0: int) + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: int) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_classmethod(self) -> None: + class TestClass: + @classmethod + def test(cls, arg0: str) -> None: + pass + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="cls", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["@classmethod", "def test(cls, *args, **kwargs): ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_classmethod_with_overloads(self) -> None: + class TestClass: + @classmethod + def test(cls, arg0: str) -> None: + """ + test(cls, arg0: str) + test(cls, arg0: int) + """ + pass + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="cls", cls=TestClass, name="TestClass"), + ) + assert_equal( + output, + [ + "@overload", + "@classmethod", + "def test(cls, arg0: str) -> Any: ...", + "@overload", + "@classmethod", + "def test(cls, arg0: int) -> Any: ...", + ], + ) + assert_equal(gen.get_imports().splitlines(), ["from typing import overload"]) + + def test_generate_c_type_with_docstring_empty_default(self) -> None: + class TestClass: + def test(self, arg0: str = "") -> None: + """ + test(self: TestClass, arg0: str = "") + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: str = ...) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_function_other_module_arg(self) -> None: + """Test that if argument references type from other module, module will be imported.""" + + # Provide different type in python spec than in docstring to make sure, that docstring + # information is used. + def test(arg0: str) -> None: + """ + test(arg0: argparse.Action) + """ + + output: list[str] = [] + mod = ModuleType(self.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub("test", test, output=output) + assert_equal(output, ["def test(arg0: argparse.Action) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), ["import argparse"]) + + def test_generate_c_function_same_module(self) -> None: + """Test that if annotation references type from same module but using full path, no module + will be imported, and type specification will be striped to local reference. + """ + + # Provide different type in python spec than in docstring to make sure, that docstring + # information is used. + def test(arg0: str) -> None: + """ + test(arg0: argparse.Action) -> argparse.Action + """ + + output: list[str] = [] + mod = ModuleType("argparse", "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub("test", test, output=output) + assert_equal(output, ["def test(arg0: Action) -> Action: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_function_other_module(self) -> None: + """Test that if annotation references type from other module, module will be imported.""" + + def test(arg0: str) -> None: + """ + test(arg0: argparse.Action) -> argparse.Action + """ + + output: list[str] = [] + mod = ModuleType(self.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub("test", test, output=output) + assert_equal(output, ["def test(arg0: argparse.Action) -> argparse.Action: ..."]) + assert_equal(gen.get_imports().splitlines(), ["import argparse"]) + + def test_generate_c_function_same_module_nested(self) -> None: + """Test that if annotation references type from same module but using full path, no module + will be imported, and type specification will be stripped to local reference. + """ + + # Provide different type in python spec than in docstring to make sure, that docstring + # information is used. + def test(arg0: str) -> None: + """ + test(arg0: list[argparse.Action]) -> list[argparse.Action] + """ + + output: list[str] = [] + mod = ModuleType("argparse", "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub("test", test, output=output) + assert_equal(output, ["def test(arg0: list[Action]) -> list[Action]: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_function_same_module_compound(self) -> None: + """Test that if annotation references type from same module but using full path, no module + will be imported, and type specification will be stripped to local reference. + """ + + # Provide different type in python spec than in docstring to make sure, that docstring + # information is used. + def test(arg0: str) -> None: + """ + test(arg0: Union[argparse.Action, NoneType]) -> Tuple[argparse.Action, NoneType] + """ + + output: list[str] = [] + mod = ModuleType("argparse", "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub("test", test, output=output) + assert_equal(output, ["def test(arg0: Union[Action, None]) -> Tuple[Action, None]: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_function_other_module_nested(self) -> None: + """Test that if annotation references type from other module, module will be imported, + and the import will be restricted to one of the known modules.""" + + def test(arg0: str) -> None: + """ + test(arg0: foo.bar.Action) -> other.Thing + """ + + output: list[str] = [] + mod = ModuleType(self.__module__, "") + gen = InspectionStubGenerator( + mod.__name__, known_modules=["foo", "foo.spangle", "bar"], module=mod + ) + gen.generate_function_stub("test", test, output=output) + assert_equal(output, ["def test(arg0: foo.bar.Action) -> other.Thing: ..."]) + assert_equal(gen.get_imports().splitlines(), ["import foo", "import other"]) + + def test_generate_c_function_no_crash_for_non_str_docstring(self) -> None: + def test(arg0: str) -> None: ... + + test.__doc__ = property(lambda self: "test(arg0: str) -> None") # type: ignore[assignment] + + output: list[str] = [] + mod = ModuleType(self.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub("test", test, output=output) + assert_equal(output, ["def test(*args, **kwargs): ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_property_with_pybind11(self) -> None: + """Signatures included by PyBind11 inside property.fget are read.""" + + class TestClass: + def get_attribute(self) -> None: + """ + (self: TestClass) -> str + """ + + attribute = property(get_attribute, doc="") + + readwrite_properties: list[str] = [] + readonly_properties: list[str] = [] + mod = ModuleType("module", "") # any module is fine + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_property_stub( + "attribute", + TestClass.__dict__["attribute"], + TestClass.attribute, + [], + readwrite_properties, + readonly_properties, + ) + assert_equal(readwrite_properties, []) + assert_equal(readonly_properties, ["@property", "def attribute(self) -> str: ..."]) + + def test_generate_c_property_with_rw_property(self) -> None: + class TestClass: + def __init__(self) -> None: + self._attribute = 0 + + @property + def attribute(self) -> int: + return self._attribute + + @attribute.setter + def attribute(self, value: int) -> None: + self._attribute = value + + readwrite_properties: list[str] = [] + readonly_properties: list[str] = [] + mod = ModuleType("module", "") # any module is fine + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_property_stub( + "attribute", + TestClass.__dict__["attribute"], + TestClass.attribute, + [], + readwrite_properties, + readonly_properties, + ) + assert_equal(readwrite_properties, ["attribute: Incomplete"]) + assert_equal(readonly_properties, []) + + def test_generate_c_type_with_single_arg_generic(self) -> None: + class TestClass: + def test(self, arg0: str) -> None: + """ + test(self: TestClass, arg0: List[int]) + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: List[int]) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_with_double_arg_generic(self) -> None: + class TestClass: + def test(self, arg0: str) -> None: + """ + test(self: TestClass, arg0: Dict[str, int]) + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: Dict[str, int]) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_with_nested_generic(self) -> None: + class TestClass: + def test(self, arg0: str) -> None: + """ + test(self: TestClass, arg0: Dict[str, List[int]]) + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: Dict[str, List[int]]) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), []) + + def test_generate_c_type_with_generic_using_other_module_first(self) -> None: + class TestClass: + def test(self, arg0: str) -> None: + """ + test(self: TestClass, arg0: Dict[argparse.Action, int]) + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: Dict[argparse.Action, int]) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), ["import argparse"]) + + def test_generate_c_type_with_generic_using_other_module_last(self) -> None: + class TestClass: + def test(self, arg0: str) -> None: + """ + test(self: TestClass, arg0: Dict[str, argparse.Action]) + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal(output, ["def test(self, arg0: Dict[str, argparse.Action]) -> Any: ..."]) + assert_equal(gen.get_imports().splitlines(), ["import argparse"]) + + def test_generate_c_type_with_overload_pybind11(self) -> None: + class TestClass: + def __init__(self, arg0: str) -> None: + """ + __init__(*args, **kwargs) + Overloaded function. + + 1. __init__(self: TestClass, arg0: str) -> None + + 2. __init__(self: TestClass, arg0: str, arg1: str) -> None + """ + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "__init__", + TestClass.__init__, + output=output, + class_info=ClassInfo(self_var="self", cls=TestClass, name="TestClass"), + ) + assert_equal( + output, + [ + "@overload", + "def __init__(self, arg0: str) -> None: ...", + "@overload", + "def __init__(self, arg0: str, arg1: str) -> None: ...", + "@overload", + "def __init__(self, *args, **kwargs) -> Any: ...", + ], + ) + assert_equal(gen.get_imports().splitlines(), ["from typing import overload"]) + + def test_generate_c_type_with_overload_shiboken(self) -> None: + class TestClass: + """ + TestClass(self: TestClass, arg0: str) -> None + TestClass(self: TestClass, arg0: str, arg1: str) -> None + """ + + def __init__(self, arg0: str) -> None: + pass + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.generate_function_stub( + "__init__", + TestClass.__init__, + output=output, + class_info=ClassInfo( + self_var="self", + cls=TestClass, + name="TestClass", + docstring=getattr(TestClass, "__doc__", None), + ), + ) + assert_equal( + output, + [ + "@overload", + "def __init__(self, arg0: str) -> None: ...", + "@overload", + "def __init__(self, arg0: str, arg1: str) -> None: ...", + ], + ) + assert_equal(gen.get_imports().splitlines(), ["from typing import overload"]) + + +class ArgSigSuite(unittest.TestCase): + def test_repr(self) -> None: + assert_equal( + repr(ArgSig(name='asd"dsa')), "ArgSig(name='asd\"dsa', type=None, default=False)" + ) + assert_equal( + repr(ArgSig(name="asd'dsa")), 'ArgSig(name="asd\'dsa", type=None, default=False)' + ) + assert_equal(repr(ArgSig("func", "str")), "ArgSig(name='func', type='str', default=False)") + assert_equal( + repr(ArgSig("func", "str", default=True)), + "ArgSig(name='func', type='str', default=True)", + ) + + +class IsValidTypeSuite(unittest.TestCase): + def test_is_valid_type(self) -> None: + assert is_valid_type("int") + assert is_valid_type("str") + assert is_valid_type("Foo_Bar234") + assert is_valid_type("foo.bar") + assert is_valid_type("List[int]") + assert is_valid_type("Dict[str, int]") + assert is_valid_type("None") + assert is_valid_type("Literal[26]") + assert is_valid_type("Literal[0x1A]") + assert is_valid_type('Literal["hello world"]') + assert is_valid_type('Literal[b"hello world"]') + assert is_valid_type('Literal[u"hello world"]') + assert is_valid_type("Literal[True]") + assert is_valid_type("Literal[Color.RED]") + assert is_valid_type("Literal[None]") + assert is_valid_type( + 'Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]' + ) + assert not is_valid_type("foo-bar") + assert not is_valid_type("x->y") + assert not is_valid_type("True") + assert not is_valid_type("False") + assert not is_valid_type("x,y") + assert not is_valid_type("x, y") + + +class ModuleInspectSuite(unittest.TestCase): + def test_python_module(self) -> None: + with ModuleInspect() as m: + p = m.get_package_properties("inspect") + assert p is not None + assert p.name == "inspect" + assert p.file + assert p.path is None + assert p.is_c_module is False + assert p.subpackages == [] + + def test_python_package(self) -> None: + with ModuleInspect() as m: + p = m.get_package_properties("unittest") + assert p is not None + assert p.name == "unittest" + assert p.file + assert p.path + assert p.is_c_module is False + assert p.subpackages + assert all(sub.startswith("unittest.") for sub in p.subpackages) + + def test_c_module(self) -> None: + with ModuleInspect() as m: + p = m.get_package_properties("_socket") + assert p is not None + assert p.name == "_socket" + assert p.path is None + assert p.is_c_module is True + assert p.subpackages == [] + + def test_non_existent(self) -> None: + with ModuleInspect() as m: + with self.assertRaises(InspectError) as e: + m.get_package_properties("foobar-non-existent") + assert str(e.exception) == "No module named 'foobar-non-existent'" + + +def module_to_path(out_dir: str, module: str) -> str: + fnam = os.path.join(out_dir, f"{module.replace('.', '/')}.pyi") + if not os.path.exists(fnam): + alt_fnam = fnam.replace(".pyi", "/__init__.pyi") + if os.path.exists(alt_fnam): + return alt_fnam + return fnam diff --git a/mypy/test/teststubinfo.py b/mypy/test/teststubinfo.py new file mode 100644 index 000000000000..518194d35e1d --- /dev/null +++ b/mypy/test/teststubinfo.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import unittest + +from mypy.stubinfo import ( + is_module_from_legacy_bundled_package, + legacy_bundled_packages, + non_bundled_packages_flat, + stub_distribution_name, +) + + +class TestStubInfo(unittest.TestCase): + def test_is_legacy_bundled_packages(self) -> None: + assert not is_module_from_legacy_bundled_package("foobar_asdf") + assert not is_module_from_legacy_bundled_package("PIL") + assert is_module_from_legacy_bundled_package("pycurl") + assert is_module_from_legacy_bundled_package("dataclasses") + + def test_stub_distribution_name(self) -> None: + assert stub_distribution_name("foobar_asdf") is None + assert stub_distribution_name("pycurl") == "types-pycurl" + assert stub_distribution_name("babel") == "types-babel" + assert stub_distribution_name("google.cloud.ndb") == "types-google-cloud-ndb" + assert stub_distribution_name("google.cloud.ndb.submodule") == "types-google-cloud-ndb" + assert stub_distribution_name("google.cloud.unknown") is None + assert stub_distribution_name("google.protobuf") == "types-protobuf" + assert stub_distribution_name("google.protobuf.submodule") == "types-protobuf" + assert stub_distribution_name("google") is None + + def test_period_in_top_level(self) -> None: + for packages in (non_bundled_packages_flat, legacy_bundled_packages): + for top_level_module in packages: + assert "." not in top_level_module diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py new file mode 100644 index 000000000000..fcbf07b4d371 --- /dev/null +++ b/mypy/test/teststubtest.py @@ -0,0 +1,2645 @@ +from __future__ import annotations + +import contextlib +import inspect +import io +import os +import re +import sys +import tempfile +import textwrap +import unittest +from typing import Any, Callable, Iterator + +import mypy.stubtest +from mypy.stubtest import parse_options, test_stubs +from mypy.test.data import root_dir + + +@contextlib.contextmanager +def use_tmp_dir(mod_name: str) -> Iterator[str]: + current = os.getcwd() + current_syspath = sys.path.copy() + with tempfile.TemporaryDirectory() as tmp: + try: + os.chdir(tmp) + if sys.path[0] != tmp: + sys.path.insert(0, tmp) + yield tmp + finally: + sys.path = current_syspath.copy() + if mod_name in sys.modules: + del sys.modules[mod_name] + + os.chdir(current) + + +TEST_MODULE_NAME = "test_module" + + +stubtest_typing_stub = """ +Any = object() + +class _SpecialForm: + def __getitem__(self, typeargs: Any) -> object: ... + +Callable: _SpecialForm = ... +Generic: _SpecialForm = ... +Protocol: _SpecialForm = ... +Union: _SpecialForm = ... + +class TypeVar: + def __init__(self, name, covariant: bool = ..., contravariant: bool = ...) -> None: ... + +class ParamSpec: + def __init__(self, name: str) -> None: ... + +AnyStr = TypeVar("AnyStr", str, bytes) +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_K = TypeVar("_K") +_V = TypeVar("_V") +_S = TypeVar("_S", contravariant=True) +_R = TypeVar("_R", covariant=True) + +class Coroutine(Generic[_T_co, _S, _R]): ... +class Iterable(Generic[_T_co]): ... +class Iterator(Iterable[_T_co]): ... +class Mapping(Generic[_K, _V]): ... +class Match(Generic[AnyStr]): ... +class Sequence(Iterable[_T_co]): ... +class Tuple(Sequence[_T_co]): ... +class NamedTuple(tuple[Any, ...]): ... +def overload(func: _T) -> _T: ... +def type_check_only(func: _T) -> _T: ... +def final(func: _T) -> _T: ... +""" + +stubtest_builtins_stub = """ +from typing import Generic, Mapping, Sequence, TypeVar, overload + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +KT = TypeVar('KT') +VT = TypeVar('VT') + +class object: + __module__: str + def __init__(self) -> None: pass + def __repr__(self) -> str: pass +class type: ... + +class tuple(Sequence[T_co], Generic[T_co]): + def __ge__(self, __other: tuple[T_co, ...]) -> bool: pass + +class dict(Mapping[KT, VT]): ... + +class function: pass +class ellipsis: pass + +class int: ... +class float: ... +class bool(int): ... +class str: ... +class bytes: ... + +class list(Sequence[T]): ... + +def property(f: T) -> T: ... +def classmethod(f: T) -> T: ... +def staticmethod(f: T) -> T: ... +""" + +stubtest_enum_stub = """ +import sys +from typing import Any, TypeVar, Iterator + +_T = TypeVar('_T') + +class EnumMeta(type): + def __len__(self) -> int: pass + def __iter__(self: type[_T]) -> Iterator[_T]: pass + def __reversed__(self: type[_T]) -> Iterator[_T]: pass + def __getitem__(self: type[_T], name: str) -> _T: pass + +class Enum(metaclass=EnumMeta): + def __new__(cls: type[_T], value: object) -> _T: pass + def __repr__(self) -> str: pass + def __str__(self) -> str: pass + def __format__(self, format_spec: str) -> str: pass + def __hash__(self) -> Any: pass + def __reduce_ex__(self, proto: Any) -> Any: pass + name: str + value: Any + +class Flag(Enum): + def __or__(self: _T, other: _T) -> _T: pass + def __and__(self: _T, other: _T) -> _T: pass + def __xor__(self: _T, other: _T) -> _T: pass + def __invert__(self: _T) -> _T: pass + if sys.version_info >= (3, 11): + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ +""" + + +def run_stubtest_with_stderr( + stub: str, runtime: str, options: list[str], config_file: str | None = None +) -> tuple[str, str]: + with use_tmp_dir(TEST_MODULE_NAME) as tmp_dir: + with open("builtins.pyi", "w") as f: + f.write(stubtest_builtins_stub) + with open("typing.pyi", "w") as f: + f.write(stubtest_typing_stub) + with open("enum.pyi", "w") as f: + f.write(stubtest_enum_stub) + with open(f"{TEST_MODULE_NAME}.pyi", "w") as f: + f.write(stub) + with open(f"{TEST_MODULE_NAME}.py", "w") as f: + f.write(runtime) + if config_file: + with open(f"{TEST_MODULE_NAME}_config.ini", "w") as f: + f.write(config_file) + options = options + ["--mypy-config-file", f"{TEST_MODULE_NAME}_config.ini"] + output = io.StringIO() + outerr = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(outerr): + test_stubs(parse_options([TEST_MODULE_NAME] + options), use_builtins_fixtures=True) + filtered_output = remove_color_code( + output.getvalue() + # remove cwd as it's not available from outside + .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "") + ) + filtered_outerr = remove_color_code( + outerr.getvalue() + # remove cwd as it's not available from outside + .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "") + ) + return filtered_output, filtered_outerr + + +def run_stubtest( + stub: str, runtime: str, options: list[str], config_file: str | None = None +) -> str: + return run_stubtest_with_stderr(stub, runtime, options, config_file)[0] + + +class Case: + def __init__(self, stub: str, runtime: str, error: str | None) -> None: + self.stub = stub + self.runtime = runtime + self.error = error + + +def collect_cases(fn: Callable[..., Iterator[Case]]) -> Callable[..., None]: + """run_stubtest used to be slow, so we used this decorator to combine cases. + + If you're reading this and bored, feel free to refactor this and make it more like + other mypy tests. + + """ + + def test(*args: Any, **kwargs: Any) -> None: + cases = list(fn(*args, **kwargs)) + expected_errors = set() + for c in cases: + if c.error is None: + continue + expected_error = c.error + if expected_error == "": + expected_error = TEST_MODULE_NAME + elif not expected_error.startswith(f"{TEST_MODULE_NAME}."): + expected_error = f"{TEST_MODULE_NAME}.{expected_error}" + assert expected_error not in expected_errors, ( + "collect_cases merges cases into a single stubtest invocation; we already " + "expect an error for {}".format(expected_error) + ) + expected_errors.add(expected_error) + output = run_stubtest( + stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases), + runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases), + options=["--generate-allowlist"], + ) + + actual_errors = set(output.splitlines()) + if actual_errors != expected_errors: + output = run_stubtest( + stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases), + runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases), + options=[], + ) + assert actual_errors == expected_errors, output + + return test + + +class StubtestUnit(unittest.TestCase): + @collect_cases + def test_basic_good(self) -> Iterator[Case]: + yield Case( + stub="def f(number: int, text: str) -> None: ...", + runtime="def f(number, text): pass", + error=None, + ) + yield Case( + stub=""" + class X: + def f(self, number: int, text: str) -> None: ... + """, + runtime=""" + class X: + def f(self, number, text): pass + """, + error=None, + ) + + @collect_cases + def test_types(self) -> Iterator[Case]: + yield Case( + stub="def mistyped_class() -> None: ...", + runtime="class mistyped_class: pass", + error="mistyped_class", + ) + yield Case( + stub="class mistyped_fn: ...", runtime="def mistyped_fn(): pass", error="mistyped_fn" + ) + yield Case( + stub=""" + class X: + def mistyped_var(self) -> int: ... + """, + runtime=""" + class X: + mistyped_var = 1 + """, + error="X.mistyped_var", + ) + + @collect_cases + def test_coroutines(self) -> Iterator[Case]: + yield Case(stub="def bar() -> int: ...", runtime="async def bar(): return 5", error="bar") + # Don't error for this one -- we get false positives otherwise + yield Case(stub="async def foo() -> int: ...", runtime="def foo(): return 5", error=None) + yield Case(stub="def baz() -> int: ...", runtime="def baz(): return 5", error=None) + yield Case( + stub="async def bingo() -> int: ...", runtime="async def bingo(): return 5", error=None + ) + + @collect_cases + def test_arg_name(self) -> Iterator[Case]: + yield Case( + stub="def bad(number: int, text: str) -> None: ...", + runtime="def bad(num, text) -> None: pass", + error="bad", + ) + yield Case( + stub="def good_posonly(__number: int, text: str) -> None: ...", + runtime="def good_posonly(num, /, text): pass", + error=None, + ) + yield Case( + stub="def bad_posonly(__number: int, text: str) -> None: ...", + runtime="def bad_posonly(flag, /, text): pass", + error="bad_posonly", + ) + yield Case( + stub=""" + class BadMethod: + def f(self, number: int, text: str) -> None: ... + """, + runtime=""" + class BadMethod: + def f(self, n, text): pass + """, + error="BadMethod.f", + ) + yield Case( + stub=""" + class GoodDunder: + def __exit__(self, t, v, tb) -> None: ... + """, + runtime=""" + class GoodDunder: + def __exit__(self, exc_type, exc_val, exc_tb): pass + """, + error=None, + ) + + @collect_cases + def test_arg_kind(self) -> Iterator[Case]: + yield Case( + stub="def runtime_kwonly(number: int, text: str) -> None: ...", + runtime="def runtime_kwonly(number, *, text): pass", + error="runtime_kwonly", + ) + yield Case( + stub="def stub_kwonly(number: int, *, text: str) -> None: ...", + runtime="def stub_kwonly(number, text): pass", + error="stub_kwonly", + ) + yield Case( + stub="def stub_posonly(__number: int, text: str) -> None: ...", + runtime="def stub_posonly(number, text): pass", + error="stub_posonly", + ) + yield Case( + stub="def good_posonly(__number: int, text: str) -> None: ...", + runtime="def good_posonly(number, /, text): pass", + error=None, + ) + yield Case( + stub="def runtime_posonly(number: int, text: str) -> None: ...", + runtime="def runtime_posonly(number, /, text): pass", + error="runtime_posonly", + ) + yield Case( + stub="def stub_posonly_570(number: int, /, text: str) -> None: ...", + runtime="def stub_posonly_570(number, text): pass", + error="stub_posonly_570", + ) + + @collect_cases + def test_private_parameters(self) -> Iterator[Case]: + # Private parameters can optionally be omitted. + yield Case( + stub="def priv_pos_arg_missing() -> None: ...", + runtime="def priv_pos_arg_missing(_p1=None): pass", + error=None, + ) + yield Case( + stub="def multi_priv_args() -> None: ...", + runtime="def multi_priv_args(_p='', _q=''): pass", + error=None, + ) + yield Case( + stub="def priv_kwarg_missing() -> None: ...", + runtime="def priv_kwarg_missing(*, _p2=''): pass", + error=None, + ) + # But if they are included, they must be correct. + yield Case( + stub="def priv_pos_arg_wrong(_p: int = ...) -> None: ...", + runtime="def priv_pos_arg_wrong(_p=None): pass", + error="priv_pos_arg_wrong", + ) + yield Case( + stub="def priv_kwarg_wrong(*, _p: int = ...) -> None: ...", + runtime="def priv_kwarg_wrong(*, _p=None): pass", + error="priv_kwarg_wrong", + ) + # Private parameters must have a default and start with exactly one + # underscore. + yield Case( + stub="def pos_arg_no_default() -> None: ...", + runtime="def pos_arg_no_default(_np): pass", + error="pos_arg_no_default", + ) + yield Case( + stub="def kwarg_no_default() -> None: ...", + runtime="def kwarg_no_default(*, _np): pass", + error="kwarg_no_default", + ) + yield Case( + stub="def double_underscore_pos_arg() -> None: ...", + runtime="def double_underscore_pos_arg(__np = None): pass", + error="double_underscore_pos_arg", + ) + yield Case( + stub="def double_underscore_kwarg() -> None: ...", + runtime="def double_underscore_kwarg(*, __np = None): pass", + error="double_underscore_kwarg", + ) + # But spot parameters that are accidentally not marked kw-only and + # vice-versa. + yield Case( + stub="def priv_arg_is_kwonly(_p=...) -> None: ...", + runtime="def priv_arg_is_kwonly(*, _p=''): pass", + error="priv_arg_is_kwonly", + ) + yield Case( + stub="def priv_arg_is_positional(*, _p=...) -> None: ...", + runtime="def priv_arg_is_positional(_p=''): pass", + error="priv_arg_is_positional", + ) + # Private parameters not at the end of the parameter list must be + # included so that users can pass the following arguments using + # positional syntax. + yield Case( + stub="def priv_args_not_at_end(*, q='') -> None: ...", + runtime="def priv_args_not_at_end(_p='', q=''): pass", + error="priv_args_not_at_end", + ) + + @collect_cases + def test_default_presence(self) -> Iterator[Case]: + yield Case( + stub="def f1(text: str = ...) -> None: ...", + runtime="def f1(text = 'asdf'): pass", + error=None, + ) + yield Case( + stub="def f2(text: str = ...) -> None: ...", runtime="def f2(text): pass", error="f2" + ) + yield Case( + stub="def f3(text: str) -> None: ...", + runtime="def f3(text = 'asdf'): pass", + error="f3", + ) + yield Case( + stub="def f4(text: str = ...) -> None: ...", + runtime="def f4(text = None): pass", + error="f4", + ) + yield Case( + stub="def f5(data: bytes = ...) -> None: ...", + runtime="def f5(data = 'asdf'): pass", + error="f5", + ) + yield Case( + stub=""" + from typing import TypeVar + _T = TypeVar("_T", bound=str) + def f6(text: _T = ...) -> None: ... + """, + runtime="def f6(text = None): pass", + error="f6", + ) + + @collect_cases + def test_default_value(self) -> Iterator[Case]: + yield Case( + stub="def f1(text: str = 'x') -> None: ...", + runtime="def f1(text = 'y'): pass", + error="f1", + ) + yield Case( + stub='def f2(text: bytes = b"x\'") -> None: ...', + runtime='def f2(text = b"x\'"): pass', + error=None, + ) + yield Case( + stub='def f3(text: bytes = b"y\'") -> None: ...', + runtime='def f3(text = b"x\'"): pass', + error="f3", + ) + yield Case( + stub="def f4(text: object = 1) -> None: ...", + runtime="def f4(text = 1.0): pass", + error="f4", + ) + yield Case( + stub="def f5(text: object = True) -> None: ...", + runtime="def f5(text = 1): pass", + error="f5", + ) + yield Case( + stub="def f6(text: object = True) -> None: ...", + runtime="def f6(text = True): pass", + error=None, + ) + yield Case( + stub="def f7(text: object = not True) -> None: ...", + runtime="def f7(text = False): pass", + error=None, + ) + yield Case( + stub="def f8(text: object = not True) -> None: ...", + runtime="def f8(text = True): pass", + error="f8", + ) + yield Case( + stub="def f9(text: object = {1: 2}) -> None: ...", + runtime="def f9(text = {1: 3}): pass", + error="f9", + ) + yield Case( + stub="def f10(text: object = [1, 2]) -> None: ...", + runtime="def f10(text = [1, 2]): pass", + error=None, + ) + + # Simulate "" + yield Case( + stub="def f11() -> None: ...", + runtime=""" + def f11(text=None) -> None: pass + f11.__text_signature__ = "(text=)" + """, + error="f11", + ) + + @collect_cases + def test_static_class_method(self) -> Iterator[Case]: + yield Case( + stub=""" + class Good: + @classmethod + def f(cls, number: int, text: str) -> None: ... + """, + runtime=""" + class Good: + @classmethod + def f(cls, number, text): pass + """, + error=None, + ) + yield Case( + stub=""" + class Bad1: + def f(cls, number: int, text: str) -> None: ... + """, + runtime=""" + class Bad1: + @classmethod + def f(cls, number, text): pass + """, + error="Bad1.f", + ) + yield Case( + stub=""" + class Bad2: + @classmethod + def f(cls, number: int, text: str) -> None: ... + """, + runtime=""" + class Bad2: + @staticmethod + def f(self, number, text): pass + """, + error="Bad2.f", + ) + yield Case( + stub=""" + class Bad3: + @staticmethod + def f(cls, number: int, text: str) -> None: ... + """, + runtime=""" + class Bad3: + @classmethod + def f(self, number, text): pass + """, + error="Bad3.f", + ) + yield Case( + stub=""" + class GoodNew: + def __new__(cls, *args, **kwargs): ... + """, + runtime=""" + class GoodNew: + def __new__(cls, *args, **kwargs): pass + """, + error=None, + ) + + @collect_cases + def test_arg_mismatch(self) -> Iterator[Case]: + yield Case( + stub="def f1(a, *, b, c) -> None: ...", runtime="def f1(a, *, b, c): pass", error=None + ) + yield Case( + stub="def f2(a, *, b) -> None: ...", runtime="def f2(a, *, b, c): pass", error="f2" + ) + yield Case( + stub="def f3(a, *, b, c) -> None: ...", runtime="def f3(a, *, b): pass", error="f3" + ) + yield Case( + stub="def f4(a, *, b, c) -> None: ...", runtime="def f4(a, b, *, c): pass", error="f4" + ) + yield Case( + stub="def f5(a, b, *, c) -> None: ...", runtime="def f5(a, *, b, c): pass", error="f5" + ) + + @collect_cases + def test_varargs_varkwargs(self) -> Iterator[Case]: + yield Case( + stub="def f1(*args, **kwargs) -> None: ...", + runtime="def f1(*args, **kwargs): pass", + error=None, + ) + yield Case( + stub="def f2(*args, **kwargs) -> None: ...", + runtime="def f2(**kwargs): pass", + error="f2", + ) + yield Case( + stub="def g1(a, b, c, d) -> None: ...", runtime="def g1(a, *args): pass", error=None + ) + yield Case( + stub="def g2(a, b, c, d, *args) -> None: ...", runtime="def g2(a): pass", error="g2" + ) + yield Case( + stub="def g3(a, b, c, d, *args) -> None: ...", + runtime="def g3(a, *args): pass", + error=None, + ) + yield Case( + stub="def h1(a) -> None: ...", runtime="def h1(a, b, c, d, *args): pass", error="h1" + ) + yield Case( + stub="def h2(a, *args) -> None: ...", runtime="def h2(a, b, c, d): pass", error="h2" + ) + yield Case( + stub="def h3(a, *args) -> None: ...", + runtime="def h3(a, b, c, d, *args): pass", + error="h3", + ) + yield Case( + stub="def j1(a: int, *args) -> None: ...", runtime="def j1(a): pass", error="j1" + ) + yield Case( + stub="def j2(a: int) -> None: ...", runtime="def j2(a, *args): pass", error="j2" + ) + yield Case( + stub="def j3(a, b, c) -> None: ...", runtime="def j3(a, *args, c): pass", error="j3" + ) + yield Case(stub="def k1(a, **kwargs) -> None: ...", runtime="def k1(a): pass", error="k1") + yield Case( + # In theory an error, but led to worse results in practice + stub="def k2(a) -> None: ...", + runtime="def k2(a, **kwargs): pass", + error=None, + ) + yield Case( + stub="def k3(a, b) -> None: ...", runtime="def k3(a, **kwargs): pass", error="k3" + ) + yield Case( + stub="def k4(a, *, b) -> None: ...", runtime="def k4(a, **kwargs): pass", error=None + ) + yield Case( + stub="def k5(a, *, b) -> None: ...", + runtime="def k5(a, *, b, c, **kwargs): pass", + error="k5", + ) + yield Case( + stub="def k6(a, *, b, **kwargs) -> None: ...", + runtime="def k6(a, *, b, c, **kwargs): pass", + error="k6", + ) + + @collect_cases + def test_overload(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing import overload + + @overload + def f1(a: int, *, c: int = ...) -> int: ... + @overload + def f1(a: int, b: int, c: int = ...) -> str: ... + """, + runtime="def f1(a, b = 0, c = 0): pass", + error=None, + ) + yield Case( + stub=""" + @overload + def f2(a: int, *, c: int = ...) -> int: ... + @overload + def f2(a: int, b: int, c: int = ...) -> str: ... + """, + runtime="def f2(a, b, c = 0): pass", + error="f2", + ) + yield Case( + stub=""" + @overload + def f3(a: int) -> int: ... + @overload + def f3(a: int, b: str) -> str: ... + """, + runtime="def f3(a, b = None): pass", + error="f3", + ) + yield Case( + stub=""" + @overload + def f4(a: int, *args, b: int, **kwargs) -> int: ... + @overload + def f4(a: str, *args, b: int, **kwargs) -> str: ... + """, + runtime="def f4(a, *args, b, **kwargs): pass", + error=None, + ) + yield Case( + stub=""" + @overload + def f5(__a: int) -> int: ... + @overload + def f5(__b: str) -> str: ... + """, + runtime="def f5(x, /): pass", + error=None, + ) + yield Case( + stub=""" + from typing import final + from typing_extensions import deprecated + class Foo: + @overload + @final + def f6(self, __a: int) -> int: ... + @overload + @deprecated("evil") + def f6(self, __b: str) -> str: ... + """, + runtime=""" + class Foo: + def f6(self, x, /): pass + """, + error=None, + ) + yield Case( + stub=""" + @overload + def f7(a: int, /) -> int: ... + @overload + def f7(b: str, /) -> str: ... + """, + runtime="def f7(x, /): pass", + error=None, + ) + yield Case( + stub=""" + @overload + def f8(a: int, c: int = 0, /) -> int: ... + @overload + def f8(b: str, d: int, /) -> str: ... + """, + runtime="def f8(x, y, /): pass", + error="f8", + ) + yield Case( + stub=""" + @overload + def f9(a: int, c: int = 0, /) -> int: ... + @overload + def f9(b: str, d: int, /) -> str: ... + """, + runtime="def f9(x, y=0, /): pass", + error=None, + ) + yield Case( + stub=""" + class Bar: + @overload + def f1(self) -> int: ... + @overload + def f1(self, a: int, /) -> int: ... + + @overload + def f2(self, a: int, /) -> int: ... + @overload + def f2(self, a: str, /) -> int: ... + """, + runtime=""" + class Bar: + def f1(self, *a) -> int: ... + def f2(self, *a) -> int: ... + """, + error=None, + ) + + @collect_cases + def test_property(self) -> Iterator[Case]: + yield Case( + stub=""" + class Good: + @property + def read_only_attr(self) -> int: ... + """, + runtime=""" + class Good: + @property + def read_only_attr(self): return 1 + """, + error=None, + ) + yield Case( + stub=""" + class Bad: + @property + def f(self) -> int: ... + """, + runtime=""" + class Bad: + def f(self) -> int: return 1 + """, + error="Bad.f", + ) + yield Case( + stub=""" + class GoodReadOnly: + @property + def f(self) -> int: ... + """, + runtime=""" + class GoodReadOnly: + f = 1 + """, + error=None, + ) + yield Case( + stub=""" + class BadReadOnly: + @property + def f(self) -> str: ... + """, + runtime=""" + class BadReadOnly: + f = 1 + """, + error="BadReadOnly.f", + ) + yield Case( + stub=""" + class Y: + @property + def read_only_attr(self) -> int: ... + @read_only_attr.setter + def read_only_attr(self, val: int) -> None: ... + """, + runtime=""" + class Y: + @property + def read_only_attr(self): return 5 + """, + error="Y.read_only_attr", + ) + yield Case( + stub=""" + class Z: + @property + def read_write_attr(self) -> int: ... + @read_write_attr.setter + def read_write_attr(self, val: int) -> None: ... + """, + runtime=""" + class Z: + @property + def read_write_attr(self): return self._val + @read_write_attr.setter + def read_write_attr(self, val): self._val = val + """, + error=None, + ) + yield Case( + stub=""" + class FineAndDandy: + @property + def attr(self) -> int: ... + """, + runtime=""" + class _EvilDescriptor: + def __get__(self, instance, ownerclass=None): + if instance is None: + raise AttributeError('no') + return 42 + def __set__(self, instance, value): + raise AttributeError('no') + + class FineAndDandy: + attr = _EvilDescriptor() + """, + error=None, + ) + + @collect_cases + def test_cached_property(self) -> Iterator[Case]: + yield Case( + stub=""" + from functools import cached_property + class Good: + @cached_property + def read_only_attr(self) -> int: ... + @cached_property + def read_only_attr2(self) -> int: ... + """, + runtime=""" + import functools as ft + from functools import cached_property + class Good: + @cached_property + def read_only_attr(self): return 1 + @ft.cached_property + def read_only_attr2(self): return 1 + """, + error=None, + ) + yield Case( + stub=""" + from functools import cached_property + class Bad: + @cached_property + def f(self) -> int: ... + """, + runtime=""" + class Bad: + def f(self) -> int: return 1 + """, + error="Bad.f", + ) + yield Case( + stub=""" + from functools import cached_property + class GoodCachedAttr: + @cached_property + def f(self) -> int: ... + """, + runtime=""" + class GoodCachedAttr: + f = 1 + """, + error=None, + ) + yield Case( + stub=""" + from functools import cached_property + class BadCachedAttr: + @cached_property + def f(self) -> str: ... + """, + runtime=""" + class BadCachedAttr: + f = 1 + """, + error="BadCachedAttr.f", + ) + yield Case( + stub=""" + from functools import cached_property + from typing import final + class FinalGood: + @cached_property + @final + def attr(self) -> int: ... + """, + runtime=""" + from functools import cached_property + from typing import final + class FinalGood: + @cached_property + @final + def attr(self): + return 1 + """, + error=None, + ) + yield Case( + stub=""" + from functools import cached_property + class FinalBad: + @cached_property + def attr(self) -> int: ... + """, + runtime=""" + from functools import cached_property + from typing_extensions import final + class FinalBad: + @cached_property + @final + def attr(self): + return 1 + """, + error="FinalBad.attr", + ) + + @collect_cases + def test_var(self) -> Iterator[Case]: + yield Case(stub="x1: int", runtime="x1 = 5", error=None) + yield Case(stub="x2: str", runtime="x2 = 5", error="x2") + yield Case("from typing import Tuple", "", None) # dummy case + yield Case( + stub=""" + x3: Tuple[int, int] + """, + runtime="x3 = (1, 3)", + error=None, + ) + yield Case( + stub=""" + x4: Tuple[int, int] + """, + runtime="x4 = (1, 3, 5)", + error="x4", + ) + yield Case(stub="x5: int", runtime="def x5(a, b): pass", error="x5") + yield Case( + stub="def foo(a: int, b: int) -> None: ...\nx6 = foo", + runtime="def foo(a, b): pass\ndef x6(c, d): pass", + error="x6", + ) + yield Case( + stub=""" + class X: + f: int + """, + runtime=""" + class X: + def __init__(self): + self.f = "asdf" + """, + error=None, + ) + yield Case( + stub=""" + class Y: + read_only_attr: int + """, + runtime=""" + class Y: + @property + def read_only_attr(self): return 5 + """, + error="Y.read_only_attr", + ) + yield Case( + stub=""" + class Z: + read_write_attr: int + """, + runtime=""" + class Z: + @property + def read_write_attr(self): return self._val + @read_write_attr.setter + def read_write_attr(self, val): self._val = val + """, + error=None, + ) + + @collect_cases + def test_type_alias(self) -> Iterator[Case]: + yield Case( + stub=""" + import collections.abc + import re + import typing + from typing import Callable, Dict, Generic, Iterable, List, Match, Tuple, TypeVar, Union + """, + runtime=""" + import collections.abc + import re + from typing import Callable, Dict, Generic, Iterable, List, Match, Tuple, TypeVar, Union + """, + error=None, + ) + yield Case( + stub=""" + class X: + def f(self) -> None: ... + Y = X + """, + runtime=""" + class X: + def f(self) -> None: ... + class Y: ... + """, + error="Y.f", + ) + yield Case(stub="A = Tuple[int, str]", runtime="A = (int, str)", error="A") + # Error if an alias isn't present at runtime... + yield Case(stub="B = str", runtime="", error="B") + # ... but only if the alias isn't private + yield Case(stub="_C = int", runtime="", error=None) + yield Case( + stub=""" + D = tuple[str, str] + E = Tuple[int, int, int] + F = Tuple[str, int] + """, + runtime=""" + D = Tuple[str, str] + E = Tuple[int, int, int] + F = List[str] + """, + error="F", + ) + yield Case( + stub=""" + G = str | int + H = Union[str, bool] + I = str | int + """, + runtime=""" + G = Union[str, int] + H = Union[str, bool] + I = str + """, + error="I", + ) + yield Case( + stub=""" + K = dict[str, str] + L = Dict[int, int] + KK = collections.abc.Iterable[str] + LL = typing.Iterable[str] + """, + runtime=""" + K = Dict[str, str] + L = Dict[int, int] + KK = Iterable[str] + LL = Iterable[str] + """, + error=None, + ) + yield Case( + stub=""" + _T = TypeVar("_T") + class _Spam(Generic[_T]): + def foo(self) -> None: ... + IntFood = _Spam[int] + """, + runtime=""" + _T = TypeVar("_T") + class _Bacon(Generic[_T]): + def foo(self, arg): pass + IntFood = _Bacon[int] + """, + error="IntFood.foo", + ) + yield Case(stub="StrList = list[str]", runtime="StrList = ['foo', 'bar']", error="StrList") + yield Case( + stub=""" + N = typing.Callable[[str], bool] + O = collections.abc.Callable[[int], str] + P = typing.Callable[[str], bool] + """, + runtime=""" + N = Callable[[str], bool] + O = Callable[[int], str] + P = int + """, + error="P", + ) + yield Case( + stub=""" + class Foo: + class Bar: ... + BarAlias = Foo.Bar + """, + runtime=""" + class Foo: + class Bar: pass + BarAlias = Foo.Bar + """, + error=None, + ) + yield Case( + stub=""" + from io import StringIO + StringIOAlias = StringIO + """, + runtime=""" + from _io import StringIO + StringIOAlias = StringIO + """, + error=None, + ) + yield Case(stub="M = Match[str]", runtime="M = Match[str]", error=None) + yield Case( + stub=""" + class Baz: + def fizz(self) -> None: ... + BazAlias = Baz + """, + runtime=""" + class Baz: + def fizz(self): pass + BazAlias = Baz + Baz.__name__ = Baz.__qualname__ = Baz.__module__ = "New" + """, + error=None, + ) + yield Case( + stub=""" + class FooBar: + __module__: None # type: ignore + def fizz(self) -> None: ... + FooBarAlias = FooBar + """, + runtime=""" + class FooBar: + def fizz(self): pass + FooBarAlias = FooBar + FooBar.__module__ = None + """, + error=None, + ) + if sys.version_info >= (3, 10): + yield Case( + stub=""" + Q = Dict[str, str] + R = dict[int, int] + S = Tuple[int, int] + T = tuple[str, str] + U = int | str + V = Union[int, str] + W = typing.Callable[[str], bool] + Z = collections.abc.Callable[[str], bool] + QQ = typing.Iterable[str] + RR = collections.abc.Iterable[str] + MM = typing.Match[str] + MMM = re.Match[str] + """, + runtime=""" + Q = dict[str, str] + R = dict[int, int] + S = tuple[int, int] + T = tuple[str, str] + U = int | str + V = int | str + W = collections.abc.Callable[[str], bool] + Z = collections.abc.Callable[[str], bool] + QQ = collections.abc.Iterable[str] + RR = collections.abc.Iterable[str] + MM = re.Match[str] + MMM = re.Match[str] + """, + error=None, + ) + + @collect_cases + def test_enum(self) -> Iterator[Case]: + yield Case(stub="import enum", runtime="import enum", error=None) + yield Case( + stub=""" + class X(enum.Enum): + a = ... + b = "asdf" + c = "oops" + """, + runtime=""" + class X(enum.Enum): + a = 1 + b = "asdf" + c = 2 + """, + error="X.c", + ) + yield Case( + stub=""" + class Flags1(enum.Flag): + a = ... + b = 2 + def foo(x: Flags1 = ...) -> None: ... + """, + runtime=""" + class Flags1(enum.Flag): + a = 1 + b = 2 + def foo(x=Flags1.a|Flags1.b): pass + """, + error=None, + ) + yield Case( + stub=""" + class Flags2(enum.Flag): + a = ... + b = 2 + def bar(x: Flags2 | None = None) -> None: ... + """, + runtime=""" + class Flags2(enum.Flag): + a = 1 + b = 2 + def bar(x=Flags2.a|Flags2.b): pass + """, + error="bar", + ) + yield Case( + stub=""" + class Flags3(enum.Flag): + a = ... + b = 2 + def baz(x: Flags3 | None = ...) -> None: ... + """, + runtime=""" + class Flags3(enum.Flag): + a = 1 + b = 2 + def baz(x=Flags3(0)): pass + """, + error=None, + ) + yield Case( + runtime=""" + import enum + class SomeObject: ... + + class WeirdEnum(enum.Enum): + a = SomeObject() + b = SomeObject() + """, + stub=""" + import enum + class SomeObject: ... + class WeirdEnum(enum.Enum): + _value_: SomeObject + a = ... + b = ... + """, + error=None, + ) + yield Case( + stub=""" + class Flags4(enum.Flag): + a = 1 + b = 2 + def spam(x: Flags4 | None = None) -> None: ... + """, + runtime=""" + class Flags4(enum.Flag): + a = 1 + b = 2 + def spam(x=Flags4(0)): pass + """, + error="spam", + ) + yield Case( + stub=""" + from typing_extensions import Final, Literal + class BytesEnum(bytes, enum.Enum): + a = b'foo' + FOO: Literal[BytesEnum.a] + BAR: Final = BytesEnum.a + BAZ: BytesEnum + EGGS: bytes + """, + runtime=""" + class BytesEnum(bytes, enum.Enum): + a = b'foo' + FOO = BytesEnum.a + BAR = BytesEnum.a + BAZ = BytesEnum.a + EGGS = BytesEnum.a + """, + error=None, + ) + + @collect_cases + def test_decorator(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing import Any, Callable + def decorator(f: Callable[[], int]) -> Callable[..., Any]: ... + @decorator + def f() -> Any: ... + """, + runtime=""" + def decorator(f): return f + @decorator + def f(): return 3 + """, + error=None, + ) + + @collect_cases + def test_all_at_runtime_not_stub(self) -> Iterator[Case]: + yield Case( + stub="Z: int", + runtime=""" + __all__ = [] + Z = 5""", + error="__all__", + ) + + @collect_cases + def test_all_in_stub_not_at_runtime(self) -> Iterator[Case]: + yield Case(stub="__all__ = ()", runtime="", error="__all__") + + @collect_cases + def test_all_in_stub_different_to_all_at_runtime(self) -> Iterator[Case]: + # We *should* emit an error with the module name itself + __all__, + # if the stub *does* define __all__, + # but the stub's __all__ is inconsistent with the runtime's __all__ + yield Case( + stub=""" + __all__ = ['foo'] + foo: str + """, + runtime=""" + __all__ = [] + foo = 'foo' + """, + error="__all__", + ) + + @collect_cases + def test_missing(self) -> Iterator[Case]: + yield Case(stub="x = 5", runtime="", error="x") + yield Case(stub="def f(): ...", runtime="", error="f") + yield Case(stub="class X: ...", runtime="", error="X") + yield Case( + stub=""" + from typing import overload + @overload + def h(x: int): ... + @overload + def h(x: str): ... + """, + runtime="", + error="h", + ) + yield Case(stub="", runtime="__all__ = []", error="__all__") # dummy case + yield Case(stub="", runtime="__all__ += ['y']\ny = 5", error="y") + yield Case(stub="", runtime="__all__ += ['g']\ndef g(): pass", error="g") + # Here we should only check that runtime has B, since the stub explicitly re-exports it + yield Case( + stub="from mystery import A, B as B, C as D # type: ignore", runtime="", error="B" + ) + yield Case( + stub="class Y: ...", + runtime="__all__ += ['Y']\nclass Y:\n def __or__(self, other): return self|other", + error="Y.__or__", + ) + yield Case( + stub="class Z: ...", + runtime="__all__ += ['Z']\nclass Z:\n def __reduce__(self): return (Z,)", + error=None, + ) + + @collect_cases + def test_missing_no_runtime_all(self) -> Iterator[Case]: + yield Case(stub="", runtime="import sys", error=None) + yield Case(stub="", runtime="def g(): ...", error="g") + yield Case(stub="", runtime="CONSTANT = 0", error="CONSTANT") + yield Case(stub="", runtime="import re; constant = re.compile('foo')", error="constant") + yield Case(stub="", runtime="from json.scanner import NUMBER_RE", error=None) + yield Case(stub="", runtime="from string import ascii_letters", error=None) + + @collect_cases + def test_missing_no_runtime_all_terrible(self) -> Iterator[Case]: + yield Case( + stub="", + runtime=""" +import sys +import types +import __future__ +_m = types.SimpleNamespace() +_m.annotations = __future__.annotations +sys.modules["_terrible_stubtest_test_module"] = _m + +from _terrible_stubtest_test_module import * +assert annotations +""", + error=None, + ) + + @collect_cases + def test_non_public_1(self) -> Iterator[Case]: + yield Case( + stub="__all__: list[str]", runtime="", error=f"{TEST_MODULE_NAME}.__all__" + ) # dummy case + yield Case(stub="_f: int", runtime="def _f(): ...", error="_f") + + @collect_cases + def test_non_public_2(self) -> Iterator[Case]: + yield Case(stub="__all__: list[str] = ['f']", runtime="__all__ = ['f']", error=None) + yield Case(stub="f: int", runtime="def f(): ...", error="f") + yield Case(stub="g: int", runtime="def g(): ...", error="g") + + @collect_cases + def test_dunders(self) -> Iterator[Case]: + yield Case( + stub="class A:\n def __init__(self, a: int, b: int) -> None: ...", + runtime="class A:\n def __init__(self, a, bx): pass", + error="A.__init__", + ) + yield Case( + stub="class B:\n def __call__(self, c: int, d: int) -> None: ...", + runtime="class B:\n def __call__(self, c, dx): pass", + error="B.__call__", + ) + yield Case( + stub=( + "class C:\n" + " def __init_subclass__(\n" + " cls, e: int = ..., **kwargs: int\n" + " ) -> None: ...\n" + ), + runtime="class C:\n def __init_subclass__(cls, e=1, **kwargs): pass", + error=None, + ) + if sys.version_info >= (3, 9): + yield Case( + stub="class D:\n def __class_getitem__(cls, type: type) -> type: ...", + runtime="class D:\n def __class_getitem__(cls, type): ...", + error=None, + ) + + @collect_cases + def test_not_subclassable(self) -> Iterator[Case]: + yield Case( + stub="class CanBeSubclassed: ...", runtime="class CanBeSubclassed: ...", error=None + ) + yield Case( + stub="class CannotBeSubclassed:\n def __init_subclass__(cls) -> None: ...", + runtime="class CannotBeSubclassed:\n def __init_subclass__(cls): raise TypeError", + error="CannotBeSubclassed", + ) + + @collect_cases + def test_has_runtime_final_decorator(self) -> Iterator[Case]: + yield Case( + stub="from typing_extensions import final", + runtime=""" + import functools + from typing_extensions import final + """, + error=None, + ) + yield Case( + stub=""" + @final + class A: ... + """, + runtime=""" + @final + class A: ... + """, + error=None, + ) + yield Case( # Runtime can miss `@final` decorator + stub=""" + @final + class B: ... + """, + runtime=""" + class B: ... + """, + error=None, + ) + yield Case( # Stub cannot miss `@final` decorator + stub=""" + class C: ... + """, + runtime=""" + @final + class C: ... + """, + error="C", + ) + yield Case( + stub=""" + class D: + @final + def foo(self) -> None: ... + @final + @staticmethod + def bar() -> None: ... + @staticmethod + @final + def bar2() -> None: ... + @final + @classmethod + def baz(cls) -> None: ... + @classmethod + @final + def baz2(cls) -> None: ... + @property + @final + def eggs(self) -> int: ... + @final + @property + def eggs2(self) -> int: ... + @final + def ham(self, obj: int) -> int: ... + """, + runtime=""" + class D: + @final + def foo(self): pass + @final + @staticmethod + def bar(): pass + @staticmethod + @final + def bar2(): pass + @final + @classmethod + def baz(cls): pass + @classmethod + @final + def baz2(cls): pass + @property + @final + def eggs(self): return 42 + @final + @property + def eggs2(self): pass + @final + @functools.lru_cache() + def ham(self, obj): return obj * 2 + """, + error=None, + ) + # Stub methods are allowed to have @final even if the runtime doesn't... + yield Case( + stub=""" + class E: + @final + def foo(self) -> None: ... + @final + @staticmethod + def bar() -> None: ... + @staticmethod + @final + def bar2() -> None: ... + @final + @classmethod + def baz(cls) -> None: ... + @classmethod + @final + def baz2(cls) -> None: ... + @property + @final + def eggs(self) -> int: ... + @final + @property + def eggs2(self) -> int: ... + @final + def ham(self, obj: int) -> int: ... + """, + runtime=""" + class E: + def foo(self): pass + @staticmethod + def bar(): pass + @staticmethod + def bar2(): pass + @classmethod + def baz(cls): pass + @classmethod + def baz2(cls): pass + @property + def eggs(self): return 42 + @property + def eggs2(self): return 42 + @functools.lru_cache() + def ham(self, obj): return obj * 2 + """, + error=None, + ) + # ...But if the runtime has @final, the stub must have it as well + yield Case( + stub=""" + class F: + def foo(self) -> None: ... + """, + runtime=""" + class F: + @final + def foo(self): pass + """, + error="F.foo", + ) + yield Case( + stub=""" + class G: + @staticmethod + def foo() -> None: ... + """, + runtime=""" + class G: + @final + @staticmethod + def foo(): pass + """, + error="G.foo", + ) + yield Case( + stub=""" + class H: + @staticmethod + def foo() -> None: ... + """, + runtime=""" + class H: + @staticmethod + @final + def foo(): pass + """, + error="H.foo", + ) + yield Case( + stub=""" + class I: + @classmethod + def foo(cls) -> None: ... + """, + runtime=""" + class I: + @final + @classmethod + def foo(cls): pass + """, + error="I.foo", + ) + yield Case( + stub=""" + class J: + @classmethod + def foo(cls) -> None: ... + """, + runtime=""" + class J: + @classmethod + @final + def foo(cls): pass + """, + error="J.foo", + ) + yield Case( + stub=""" + class K: + @property + def foo(self) -> int: ... + """, + runtime=""" + class K: + @property + @final + def foo(self): return 42 + """, + error="K.foo", + ) + # This test wouldn't pass, + # because the runtime can't set __final__ on instances of builtins.property, + # so stubtest has non way of knowing that the runtime was decorated with @final: + # + # yield Case( + # stub=""" + # class K2: + # @property + # def foo(self) -> int: ... + # """, + # runtime=""" + # class K2: + # @final + # @property + # def foo(self): return 42 + # """, + # error="K2.foo", + # ) + yield Case( + stub=""" + class L: + def foo(self, obj: int) -> int: ... + """, + runtime=""" + class L: + @final + @functools.lru_cache() + def foo(self, obj): return obj * 2 + """, + error="L.foo", + ) + + @collect_cases + def test_name_mangling(self) -> Iterator[Case]: + yield Case( + stub=""" + class X: + def __mangle_good(self, text: str) -> None: ... + def __mangle_bad(self, number: int) -> None: ... + """, + runtime=""" + class X: + def __mangle_good(self, text): pass + def __mangle_bad(self, text): pass + """, + error="X.__mangle_bad", + ) + yield Case( + stub=""" + class Klass: + class __Mangled1: + class __Mangled2: + def __mangle_good(self, text: str) -> None: ... + def __mangle_bad(self, number: int) -> None: ... + """, + runtime=""" + class Klass: + class __Mangled1: + class __Mangled2: + def __mangle_good(self, text): pass + def __mangle_bad(self, text): pass + """, + error="Klass.__Mangled1.__Mangled2.__mangle_bad", + ) + yield Case( + stub=""" + class __Dunder__: + def __mangle_good(self, text: str) -> None: ... + def __mangle_bad(self, number: int) -> None: ... + """, + runtime=""" + class __Dunder__: + def __mangle_good(self, text): pass + def __mangle_bad(self, text): pass + """, + error="__Dunder__.__mangle_bad", + ) + yield Case( + stub=""" + class _Private: + def __mangle_good(self, text: str) -> None: ... + def __mangle_bad(self, number: int) -> None: ... + """, + runtime=""" + class _Private: + def __mangle_good(self, text): pass + def __mangle_bad(self, text): pass + """, + error="_Private.__mangle_bad", + ) + + @collect_cases + def test_mro(self) -> Iterator[Case]: + yield Case( + stub=""" + class A: + def foo(self, x: int) -> None: ... + class B(A): + pass + class C(A): + pass + """, + runtime=""" + class A: + def foo(self, x: int) -> None: ... + class B(A): + def foo(self, x: int) -> None: ... + class C(A): + def foo(self, y: int) -> None: ... + """, + error="C.foo", + ) + yield Case( + stub=""" + class X: ... + """, + runtime=""" + class X: + def __init__(self, x): pass + """, + error="X.__init__", + ) + + @collect_cases + def test_good_literal(self) -> Iterator[Case]: + yield Case( + stub=r""" + from typing_extensions import Literal + + import enum + class Color(enum.Enum): + RED = ... + + NUM: Literal[1] + CHAR: Literal['a'] + FLAG: Literal[True] + NON: Literal[None] + BYT1: Literal[b'abc'] + BYT2: Literal[b'\x90'] + ENUM: Literal[Color.RED] + """, + runtime=r""" + import enum + class Color(enum.Enum): + RED = 3 + + NUM = 1 + CHAR = 'a' + NON = None + FLAG = True + BYT1 = b"abc" + BYT2 = b'\x90' + ENUM = Color.RED + """, + error=None, + ) + + @collect_cases + def test_bad_literal(self) -> Iterator[Case]: + yield Case("from typing_extensions import Literal", "", None) # dummy case + yield Case( + stub="INT_FLOAT_MISMATCH: Literal[1]", + runtime="INT_FLOAT_MISMATCH = 1.0", + error="INT_FLOAT_MISMATCH", + ) + yield Case(stub="WRONG_INT: Literal[1]", runtime="WRONG_INT = 2", error="WRONG_INT") + yield Case(stub="WRONG_STR: Literal['a']", runtime="WRONG_STR = 'b'", error="WRONG_STR") + yield Case( + stub="BYTES_STR_MISMATCH: Literal[b'value']", + runtime="BYTES_STR_MISMATCH = 'value'", + error="BYTES_STR_MISMATCH", + ) + yield Case( + stub="STR_BYTES_MISMATCH: Literal['value']", + runtime="STR_BYTES_MISMATCH = b'value'", + error="STR_BYTES_MISMATCH", + ) + yield Case( + stub="WRONG_BYTES: Literal[b'abc']", + runtime="WRONG_BYTES = b'xyz'", + error="WRONG_BYTES", + ) + yield Case( + stub="WRONG_BOOL_1: Literal[True]", + runtime="WRONG_BOOL_1 = False", + error="WRONG_BOOL_1", + ) + yield Case( + stub="WRONG_BOOL_2: Literal[False]", + runtime="WRONG_BOOL_2 = True", + error="WRONG_BOOL_2", + ) + + @collect_cases + def test_special_subtype(self) -> Iterator[Case]: + yield Case( + stub=""" + b1: bool + b2: bool + b3: bool + """, + runtime=""" + b1 = 0 + b2 = 1 + b3 = 2 + """, + error="b3", + ) + yield Case( + stub=""" + from typing_extensions import TypedDict + + class _Options(TypedDict): + a: str + b: int + + opt1: _Options + opt2: _Options + opt3: _Options + """, + runtime=""" + opt1 = {"a": "3.", "b": 14} + opt2 = {"some": "stuff"} # false negative + opt3 = 0 + """, + error="opt3", + ) + + @collect_cases + def test_runtime_typing_objects(self) -> Iterator[Case]: + yield Case( + stub="from typing_extensions import Protocol, TypedDict", + runtime="from typing_extensions import Protocol, TypedDict", + error=None, + ) + yield Case( + stub=""" + class X(Protocol): + bar: int + def foo(self, x: int, y: bytes = ...) -> str: ... + """, + runtime=""" + class X(Protocol): + bar: int + def foo(self, x: int, y: bytes = ...) -> str: ... + """, + error=None, + ) + yield Case( + stub=""" + class Y(TypedDict): + a: int + """, + runtime=""" + class Y(TypedDict): + a: int + """, + error=None, + ) + + @collect_cases + def test_named_tuple(self) -> Iterator[Case]: + yield Case( + stub="from typing import NamedTuple", + runtime="from typing import NamedTuple", + error=None, + ) + yield Case( + stub=""" + class X1(NamedTuple): + bar: int + foo: str = ... + """, + runtime=""" + class X1(NamedTuple): + bar: int + foo: str = 'a' + """, + error=None, + ) + yield Case( + stub=""" + class X2(NamedTuple): + bar: int + foo: str + """, + runtime=""" + class X2(NamedTuple): + bar: int + foo: str = 'a' + """, + # `__new__` will miss a default value for a `foo` parameter, + # but we don't generate special errors for `foo` missing `...` part. + error="X2.__new__", + ) + + @collect_cases + def test_named_tuple_typing_and_collections(self) -> Iterator[Case]: + yield Case( + stub="from typing import NamedTuple", + runtime="from collections import namedtuple", + error=None, + ) + yield Case( + stub=""" + class X1(NamedTuple): + bar: int + foo: str = ... + """, + runtime=""" + X1 = namedtuple('X1', ['bar', 'foo'], defaults=['a']) + """, + error=None, + ) + yield Case( + stub=""" + class X2(NamedTuple): + bar: int + foo: str + """, + runtime=""" + X2 = namedtuple('X1', ['bar', 'foo'], defaults=['a']) + """, + error="X2.__new__", + ) + + @collect_cases + def test_type_var(self) -> Iterator[Case]: + yield Case( + stub="from typing import TypeVar", runtime="from typing import TypeVar", error=None + ) + yield Case(stub="A = TypeVar('A')", runtime="A = TypeVar('A')", error=None) + yield Case(stub="B = TypeVar('B')", runtime="B = 5", error="B") + if sys.version_info >= (3, 10): + yield Case( + stub="from typing import ParamSpec", + runtime="from typing import ParamSpec", + error=None, + ) + yield Case(stub="C = ParamSpec('C')", runtime="C = ParamSpec('C')", error=None) + + @collect_cases + def test_metaclass_match(self) -> Iterator[Case]: + yield Case(stub="class Meta(type): ...", runtime="class Meta(type): ...", error=None) + yield Case(stub="class A0: ...", runtime="class A0: ...", error=None) + yield Case( + stub="class A1(metaclass=Meta): ...", + runtime="class A1(metaclass=Meta): ...", + error=None, + ) + yield Case(stub="class A2: ...", runtime="class A2(metaclass=Meta): ...", error="A2") + yield Case(stub="class A3(metaclass=Meta): ...", runtime="class A3: ...", error="A3") + + # Explicit `type` metaclass can always be added in any part: + yield Case( + stub="class T1(metaclass=type): ...", + runtime="class T1(metaclass=type): ...", + error=None, + ) + yield Case(stub="class T2: ...", runtime="class T2(metaclass=type): ...", error=None) + yield Case(stub="class T3(metaclass=type): ...", runtime="class T3: ...", error=None) + + # Explicit check that `_protected` names are also supported: + yield Case(stub="class _P1(type): ...", runtime="class _P1(type): ...", error=None) + yield Case(stub="class P2: ...", runtime="class P2(metaclass=_P1): ...", error="P2") + + # With inheritance: + yield Case( + stub=""" + class I1(metaclass=Meta): ... + class S1(I1): ... + """, + runtime=""" + class I1(metaclass=Meta): ... + class S1(I1): ... + """, + error=None, + ) + yield Case( + stub=""" + class I2(metaclass=Meta): ... + class S2: ... # missing inheritance + """, + runtime=""" + class I2(metaclass=Meta): ... + class S2(I2): ... + """, + error="S2", + ) + + @collect_cases + def test_metaclass_abcmeta(self) -> Iterator[Case]: + # Handling abstract metaclasses is special: + yield Case(stub="from abc import ABCMeta", runtime="from abc import ABCMeta", error=None) + yield Case( + stub="class A1(metaclass=ABCMeta): ...", + runtime="class A1(metaclass=ABCMeta): ...", + error=None, + ) + # Stubs cannot miss abstract metaclass: + yield Case(stub="class A2: ...", runtime="class A2(metaclass=ABCMeta): ...", error="A2") + # But, stubs can add extra abstract metaclass, this might be a typing hack: + yield Case(stub="class A3(metaclass=ABCMeta): ...", runtime="class A3: ...", error=None) + + @collect_cases + def test_abstract_methods(self) -> Iterator[Case]: + yield Case( + stub=""" + from abc import abstractmethod + from typing import overload + """, + runtime="from abc import abstractmethod", + error=None, + ) + yield Case( + stub=""" + class A1: + def some(self) -> None: ... + """, + runtime=""" + class A1: + @abstractmethod + def some(self) -> None: ... + """, + error="A1.some", + ) + yield Case( + stub=""" + class A2: + @abstractmethod + def some(self) -> None: ... + """, + runtime=""" + class A2: + @abstractmethod + def some(self) -> None: ... + """, + error=None, + ) + yield Case( + stub=""" + class A3: + @overload + def some(self, other: int) -> str: ... + @overload + def some(self, other: str) -> int: ... + """, + runtime=""" + class A3: + @abstractmethod + def some(self, other) -> None: ... + """, + error="A3.some", + ) + yield Case( + stub=""" + class A4: + @overload + @abstractmethod + def some(self, other: int) -> str: ... + @overload + @abstractmethod + def some(self, other: str) -> int: ... + """, + runtime=""" + class A4: + @abstractmethod + def some(self, other) -> None: ... + """, + error=None, + ) + yield Case( + stub=""" + class A5: + @abstractmethod + @overload + def some(self, other: int) -> str: ... + @abstractmethod + @overload + def some(self, other: str) -> int: ... + """, + runtime=""" + class A5: + @abstractmethod + def some(self, other) -> None: ... + """, + error=None, + ) + # Runtime can miss `@abstractmethod`: + yield Case( + stub=""" + class A6: + @abstractmethod + def some(self) -> None: ... + """, + runtime=""" + class A6: + def some(self) -> None: ... + """, + error=None, + ) + + @collect_cases + def test_abstract_properties(self) -> Iterator[Case]: + # TODO: test abstract properties with setters + yield Case( + stub="from abc import abstractmethod", + runtime="from abc import abstractmethod", + error=None, + ) + # Ensure that `@property` also can be abstract: + yield Case( + stub=""" + class AP1: + @property + def some(self) -> int: ... + """, + runtime=""" + class AP1: + @property + @abstractmethod + def some(self) -> int: ... + """, + error="AP1.some", + ) + yield Case( + stub=""" + class AP1_2: + def some(self) -> int: ... # missing `@property` decorator + """, + runtime=""" + class AP1_2: + @property + @abstractmethod + def some(self) -> int: ... + """, + error="AP1_2.some", + ) + yield Case( + stub=""" + class AP2: + @property + @abstractmethod + def some(self) -> int: ... + """, + runtime=""" + class AP2: + @property + @abstractmethod + def some(self) -> int: ... + """, + error=None, + ) + # Runtime can miss `@abstractmethod`: + yield Case( + stub=""" + class AP3: + @property + @abstractmethod + def some(self) -> int: ... + """, + runtime=""" + class AP3: + @property + def some(self) -> int: ... + """, + error=None, + ) + + @collect_cases + def test_type_check_only(self) -> Iterator[Case]: + yield Case( + stub="from typing import type_check_only, overload", + runtime="from typing import overload", + error=None, + ) + # You can have public types that are only defined in stubs + # with `@type_check_only`: + yield Case( + stub=""" + @type_check_only + class A1: ... + """, + runtime="", + error=None, + ) + # Having `@type_check_only` on a type that exists at runtime is an error + yield Case( + stub=""" + @type_check_only + class A2: ... + """, + runtime="class A2: ...", + error="A2", + ) + # The same is true for NamedTuples and TypedDicts: + yield Case( + stub="from typing_extensions import NamedTuple, TypedDict", + runtime="from typing_extensions import NamedTuple, TypedDict", + error=None, + ) + yield Case( + stub=""" + @type_check_only + class NT1(NamedTuple): ... + """, + runtime="class NT1(NamedTuple): ...", + error="NT1", + ) + yield Case( + stub=""" + @type_check_only + class TD1(TypedDict): ... + """, + runtime="class TD1(TypedDict): ...", + error="TD1", + ) + # The same is true for functions: + yield Case( + stub=""" + @type_check_only + def func1() -> None: ... + """, + runtime="", + error=None, + ) + yield Case( + stub=""" + @type_check_only + def func2() -> None: ... + """, + runtime="def func2() -> None: ...", + error="func2", + ) + + +def remove_color_code(s: str) -> str: + return re.sub("\\x1b.*?m", "", s) # this works! + + +class StubtestMiscUnit(unittest.TestCase): + def test_output(self) -> None: + output = run_stubtest( + stub="def bad(number: int, text: str) -> None: ...", + runtime="def bad(num, text): pass", + options=[], + ) + expected = ( + f'error: {TEST_MODULE_NAME}.bad is inconsistent, stub argument "number" differs ' + 'from runtime argument "num"\n' + f"Stub: in file {TEST_MODULE_NAME}.pyi:1\n" + "def (number: builtins.int, text: builtins.str)\n" + f"Runtime: in file {TEST_MODULE_NAME}.py:1\ndef (num, text)\n\n" + "Found 1 error (checked 1 module)\n" + ) + assert output == expected + + output = run_stubtest( + stub="def bad(number: int, text: str) -> None: ...", + runtime="def bad(num, text): pass", + options=["--concise"], + ) + expected = ( + "{}.bad is inconsistent, " + 'stub argument "number" differs from runtime argument "num"\n'.format(TEST_MODULE_NAME) + ) + assert output == expected + + def test_ignore_flags(self) -> None: + output = run_stubtest( + stub="", runtime="__all__ = ['f']\ndef f(): pass", options=["--ignore-missing-stub"] + ) + assert output == "Success: no issues found in 1 module\n" + + output = run_stubtest(stub="", runtime="def f(): pass", options=["--ignore-missing-stub"]) + assert output == "Success: no issues found in 1 module\n" + + output = run_stubtest( + stub="def f(__a): ...", runtime="def f(a): pass", options=["--ignore-positional-only"] + ) + assert output == "Success: no issues found in 1 module\n" + + def test_allowlist(self) -> None: + # Can't use this as a context because Windows + allowlist = tempfile.NamedTemporaryFile(mode="w+", delete=False) + try: + with allowlist: + allowlist.write(f"{TEST_MODULE_NAME}.bad # comment\n# comment") + + output = run_stubtest( + stub="def bad(number: int, text: str) -> None: ...", + runtime="def bad(asdf, text): pass", + options=["--allowlist", allowlist.name], + ) + assert output == "Success: no issues found in 1 module\n" + + # test unused entry detection + output = run_stubtest(stub="", runtime="", options=["--allowlist", allowlist.name]) + assert output == ( + f"note: unused allowlist entry {TEST_MODULE_NAME}.bad\n" + "Found 1 error (checked 1 module)\n" + ) + + output = run_stubtest( + stub="", + runtime="", + options=["--allowlist", allowlist.name, "--ignore-unused-allowlist"], + ) + assert output == "Success: no issues found in 1 module\n" + + # test regex matching + with open(allowlist.name, mode="w+") as f: + f.write(f"{TEST_MODULE_NAME}.b.*\n") + f.write("(unused_missing)?\n") + f.write("unused.*\n") + + output = run_stubtest( + stub=textwrap.dedent( + """ + def good() -> None: ... + def bad(number: int) -> None: ... + def also_bad(number: int) -> None: ... + """.lstrip( + "\n" + ) + ), + runtime=textwrap.dedent( + """ + def good(): pass + def bad(asdf): pass + def also_bad(asdf): pass + """.lstrip( + "\n" + ) + ), + options=["--allowlist", allowlist.name, "--generate-allowlist"], + ) + assert output == ( + f"note: unused allowlist entry unused.*\n{TEST_MODULE_NAME}.also_bad\n" + ) + finally: + os.unlink(allowlist.name) + + def test_mypy_build(self) -> None: + output = run_stubtest(stub="+", runtime="", options=[]) + assert output == ( + "error: not checking stubs due to failed mypy compile:\n{}.pyi:1: " + "error: invalid syntax [syntax]\n".format(TEST_MODULE_NAME) + ) + + output = run_stubtest(stub="def f(): ...\ndef f(): ...", runtime="", options=[]) + assert output == ( + "error: not checking stubs due to mypy build errors:\n{}.pyi:2: " + 'error: Name "f" already defined on line 1 [no-redef]\n'.format(TEST_MODULE_NAME) + ) + + def test_missing_stubs(self) -> None: + output = io.StringIO() + with contextlib.redirect_stdout(output): + test_stubs(parse_options(["not_a_module"])) + assert remove_color_code(output.getvalue()) == ( + "error: not_a_module failed to find stubs\n" + "Stub:\nMISSING\nRuntime:\nN/A\n\n" + "Found 1 error (checked 1 module)\n" + ) + + def test_only_py(self) -> None: + # in this case, stubtest will check the py against itself + # this is useful to support packages with a mix of stubs and inline types + with use_tmp_dir(TEST_MODULE_NAME): + with open(f"{TEST_MODULE_NAME}.py", "w") as f: + f.write("a = 1") + output = io.StringIO() + with contextlib.redirect_stdout(output): + test_stubs(parse_options([TEST_MODULE_NAME])) + output_str = remove_color_code(output.getvalue()) + assert output_str == "Success: no issues found in 1 module\n" + + def test_get_typeshed_stdlib_modules(self) -> None: + stdlib = mypy.stubtest.get_typeshed_stdlib_modules(None, (3, 7)) + assert "builtins" in stdlib + assert "os" in stdlib + assert "os.path" in stdlib + assert "asyncio" in stdlib + assert "graphlib" not in stdlib + assert "formatter" in stdlib + assert "contextvars" in stdlib # 3.7+ + assert "importlib.metadata" not in stdlib + + stdlib = mypy.stubtest.get_typeshed_stdlib_modules(None, (3, 10)) + assert "graphlib" in stdlib + assert "formatter" not in stdlib + assert "importlib.metadata" in stdlib + + def test_signature(self) -> None: + def f(a: int, b: int, *, c: int, d: int = 0, **kwargs: Any) -> None: + pass + + assert ( + str(mypy.stubtest.Signature.from_inspect_signature(inspect.signature(f))) + == "def (a, b, *, c, d = ..., **kwargs)" + ) + + def test_builtin_signature_with_unrepresentable_default(self) -> None: + sig = mypy.stubtest.safe_inspect_signature(bytes.hex) + assert sig is not None + assert ( + str(mypy.stubtest.Signature.from_inspect_signature(sig)) + == "def (self, sep = ..., bytes_per_sep = ...)" + ) + + def test_config_file(self) -> None: + runtime = "temp = 5\n" + stub = "from decimal import Decimal\ntemp: Decimal\n" + config_file = f"[mypy]\nplugins={root_dir}/test-data/unit/plugins/decimal_to_int.py\n" + output = run_stubtest(stub=stub, runtime=runtime, options=[]) + assert output == ( + f"error: {TEST_MODULE_NAME}.temp variable differs from runtime type Literal[5]\n" + f"Stub: in file {TEST_MODULE_NAME}.pyi:2\n_decimal.Decimal\nRuntime:\n5\n\n" + "Found 1 error (checked 1 module)\n" + ) + output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) + assert output == "Success: no issues found in 1 module\n" + + def test_config_file_error_codes(self) -> None: + runtime = "temp = 5\n" + stub = "temp = SOME_GLOBAL_CONST" + output = run_stubtest(stub=stub, runtime=runtime, options=[]) + assert output == ( + "error: not checking stubs due to mypy build errors:\n" + 'test_module.pyi:1: error: Name "SOME_GLOBAL_CONST" is not defined [name-defined]\n' + ) + + config_file = "[mypy]\ndisable_error_code = name-defined\n" + output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) + assert output == "Success: no issues found in 1 module\n" + + def test_config_file_error_codes_invalid(self) -> None: + runtime = "temp = 5\n" + stub = "temp: int\n" + config_file = "[mypy]\ndisable_error_code = not-a-valid-name\n" + output, outerr = run_stubtest_with_stderr( + stub=stub, runtime=runtime, options=[], config_file=config_file + ) + assert output == "Success: no issues found in 1 module\n" + assert outerr == ( + "test_module_config.ini: [mypy]: disable_error_code: " + "Invalid error code(s): not-a-valid-name\n" + ) + + def test_config_file_wrong_incomplete_feature(self) -> None: + runtime = "x = 1\n" + stub = "x: int\n" + config_file = "[mypy]\nenable_incomplete_feature = Unpack\n" + output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) + assert output == ( + "warning: Warning: Unpack is already enabled by default\n" + "Success: no issues found in 1 module\n" + ) + + config_file = "[mypy]\nenable_incomplete_feature = not-a-valid-name\n" + with self.assertRaises(SystemExit): + run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) + + def test_no_modules(self) -> None: + output = io.StringIO() + with contextlib.redirect_stdout(output): + test_stubs(parse_options([])) + assert remove_color_code(output.getvalue()) == "error: no modules to check\n" + + def test_module_and_typeshed(self) -> None: + output = io.StringIO() + with contextlib.redirect_stdout(output): + test_stubs(parse_options(["--check-typeshed", "some_module"])) + assert remove_color_code(output.getvalue()) == ( + "error: cannot pass both --check-typeshed and a list of modules\n" + ) diff --git a/mypy/test/testsubtypes.py b/mypy/test/testsubtypes.py index cffedfb77931..175074a2b140 100644 --- a/mypy/test/testsubtypes.py +++ b/mypy/test/testsubtypes.py @@ -1,27 +1,31 @@ -import typing +from __future__ import annotations -from mypy.myunit import Suite, assert_true, run_test +from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT from mypy.subtypes import is_subtype -from mypy.typefixture import TypeFixture, InterfaceTypeFixture +from mypy.test.helpers import Suite +from mypy.test.typefixture import InterfaceTypeFixture, TypeFixture +from mypy.types import Instance, Type, UninhabitedType, UnpackType class SubtypingSuite(Suite): - def set_up(self): - self.fx = TypeFixture() + def setUp(self) -> None: + self.fx = TypeFixture(INVARIANT) + self.fx_contra = TypeFixture(CONTRAVARIANT) + self.fx_co = TypeFixture(COVARIANT) - def test_trivial_cases(self): - for simple in self.fx.void, self.fx.a, self.fx.o, self.fx.b: + def test_trivial_cases(self) -> None: + for simple in self.fx_co.a, self.fx_co.o, self.fx_co.b: self.assert_subtype(simple, simple) - def test_instance_subtyping(self): - self.assert_proper_subtype(self.fx.a, self.fx.o) - self.assert_proper_subtype(self.fx.b, self.fx.o) - self.assert_proper_subtype(self.fx.b, self.fx.a) + def test_instance_subtyping(self) -> None: + self.assert_strict_subtype(self.fx.a, self.fx.o) + self.assert_strict_subtype(self.fx.b, self.fx.o) + self.assert_strict_subtype(self.fx.b, self.fx.a) self.assert_not_subtype(self.fx.a, self.fx.d) self.assert_not_subtype(self.fx.b, self.fx.c) - def test_simple_generic_instance_subtyping(self): + def test_simple_generic_instance_subtyping_invariant(self) -> None: self.assert_subtype(self.fx.ga, self.fx.ga) self.assert_subtype(self.fx.hab, self.fx.hab) @@ -29,19 +33,44 @@ def test_simple_generic_instance_subtyping(self): self.assert_not_subtype(self.fx.ga, self.fx.gb) self.assert_not_subtype(self.fx.gb, self.fx.ga) - def test_generic_subtyping_with_inheritance(self): + def test_simple_generic_instance_subtyping_covariant(self) -> None: + self.assert_subtype(self.fx_co.ga, self.fx_co.ga) + self.assert_subtype(self.fx_co.hab, self.fx_co.hab) + + self.assert_not_subtype(self.fx_co.ga, self.fx_co.g2a) + self.assert_not_subtype(self.fx_co.ga, self.fx_co.gb) + self.assert_subtype(self.fx_co.gb, self.fx_co.ga) + + def test_simple_generic_instance_subtyping_contravariant(self) -> None: + self.assert_subtype(self.fx_contra.ga, self.fx_contra.ga) + self.assert_subtype(self.fx_contra.hab, self.fx_contra.hab) + + self.assert_not_subtype(self.fx_contra.ga, self.fx_contra.g2a) + self.assert_subtype(self.fx_contra.ga, self.fx_contra.gb) + self.assert_not_subtype(self.fx_contra.gb, self.fx_contra.ga) + + def test_generic_subtyping_with_inheritance_invariant(self) -> None: self.assert_subtype(self.fx.gsab, self.fx.gb) self.assert_not_subtype(self.fx.gsab, self.fx.ga) + self.assert_not_subtype(self.fx.gsaa, self.fx.gb) + + def test_generic_subtyping_with_inheritance_covariant(self) -> None: + self.assert_subtype(self.fx_co.gsab, self.fx_co.gb) + self.assert_subtype(self.fx_co.gsab, self.fx_co.ga) + self.assert_not_subtype(self.fx_co.gsaa, self.fx_co.gb) - def test_interface_subtyping(self): + def test_generic_subtyping_with_inheritance_contravariant(self) -> None: + self.assert_subtype(self.fx_contra.gsab, self.fx_contra.gb) + self.assert_not_subtype(self.fx_contra.gsab, self.fx_contra.ga) + self.assert_subtype(self.fx_contra.gsaa, self.fx_contra.gb) + + def test_interface_subtyping(self) -> None: self.assert_subtype(self.fx.e, self.fx.f) self.assert_equivalent(self.fx.f, self.fx.f) self.assert_not_subtype(self.fx.a, self.fx.f) - def test_generic_interface_subtyping(self): + def test_generic_interface_subtyping(self) -> None: # TODO make this work - self.skip() - fx2 = InterfaceTypeFixture() self.assert_subtype(fx2.m1, fx2.gfa) @@ -49,105 +78,201 @@ def test_generic_interface_subtyping(self): self.assert_equivalent(fx2.gfa, fx2.gfa) - def test_basic_callable_subtyping(self): - self.assert_proper_subtype(self.fx.callable(self.fx.o, self.fx.d), - self.fx.callable(self.fx.a, self.fx.d)) - self.assert_proper_subtype(self.fx.callable(self.fx.d, self.fx.b), - self.fx.callable(self.fx.d, self.fx.a)) + def test_basic_callable_subtyping(self) -> None: + self.assert_strict_subtype( + self.fx.callable(self.fx.o, self.fx.d), self.fx.callable(self.fx.a, self.fx.d) + ) + self.assert_strict_subtype( + self.fx.callable(self.fx.d, self.fx.b), self.fx.callable(self.fx.d, self.fx.a) + ) - self.assert_unrelated(self.fx.callable(self.fx.a, self.fx.a), - self.fx.callable(self.fx.a, self.fx.void)) + self.assert_strict_subtype( + self.fx.callable(self.fx.a, UninhabitedType()), self.fx.callable(self.fx.a, self.fx.a) + ) self.assert_unrelated( self.fx.callable(self.fx.a, self.fx.a, self.fx.a), - self.fx.callable(self.fx.a, self.fx.a)) + self.fx.callable(self.fx.a, self.fx.a), + ) - def test_default_arg_callable_subtyping(self): - self.assert_proper_subtype( + def test_default_arg_callable_subtyping(self) -> None: + self.assert_strict_subtype( self.fx.callable_default(1, self.fx.a, self.fx.d, self.fx.a), - self.fx.callable(self.fx.a, self.fx.d, self.fx.a)) + self.fx.callable(self.fx.a, self.fx.d, self.fx.a), + ) - self.assert_proper_subtype( + self.assert_strict_subtype( self.fx.callable_default(1, self.fx.a, self.fx.d, self.fx.a), - self.fx.callable(self.fx.a, self.fx.a)) + self.fx.callable(self.fx.a, self.fx.a), + ) - self.assert_proper_subtype( + self.assert_strict_subtype( self.fx.callable_default(0, self.fx.a, self.fx.d, self.fx.a), - self.fx.callable_default(1, self.fx.a, self.fx.d, self.fx.a)) + self.fx.callable_default(1, self.fx.a, self.fx.d, self.fx.a), + ) self.assert_unrelated( self.fx.callable_default(1, self.fx.a, self.fx.d, self.fx.a), - self.fx.callable(self.fx.d, self.fx.d, self.fx.a)) + self.fx.callable(self.fx.d, self.fx.d, self.fx.a), + ) self.assert_unrelated( self.fx.callable_default(0, self.fx.a, self.fx.d, self.fx.a), - self.fx.callable_default(1, self.fx.a, self.fx.a, self.fx.a)) + self.fx.callable_default(1, self.fx.a, self.fx.a, self.fx.a), + ) self.assert_unrelated( self.fx.callable_default(1, self.fx.a, self.fx.a), - self.fx.callable(self.fx.a, self.fx.a, self.fx.a)) + self.fx.callable(self.fx.a, self.fx.a, self.fx.a), + ) - def test_var_arg_callable_subtyping_1(self): - self.assert_proper_subtype( + def test_var_arg_callable_subtyping_1(self) -> None: + self.assert_strict_subtype( self.fx.callable_var_arg(0, self.fx.a, self.fx.a), - self.fx.callable_var_arg(0, self.fx.b, self.fx.a)) + self.fx.callable_var_arg(0, self.fx.b, self.fx.a), + ) - def test_var_arg_callable_subtyping_2(self): - self.assert_proper_subtype( + def test_var_arg_callable_subtyping_2(self) -> None: + self.assert_strict_subtype( self.fx.callable_var_arg(0, self.fx.a, self.fx.a), - self.fx.callable(self.fx.b, self.fx.a)) + self.fx.callable(self.fx.b, self.fx.a), + ) - def test_var_arg_callable_subtyping_3(self): - self.assert_proper_subtype( - self.fx.callable_var_arg(0, self.fx.a, self.fx.a), - self.fx.callable(self.fx.a)) + def test_var_arg_callable_subtyping_3(self) -> None: + self.assert_strict_subtype( + self.fx.callable_var_arg(0, self.fx.a, self.fx.a), self.fx.callable(self.fx.a) + ) - def test_var_arg_callable_subtyping_4(self): - self.assert_proper_subtype( + def test_var_arg_callable_subtyping_4(self) -> None: + self.assert_strict_subtype( self.fx.callable_var_arg(1, self.fx.a, self.fx.d, self.fx.a), - self.fx.callable(self.fx.b, self.fx.a)) + self.fx.callable(self.fx.b, self.fx.a), + ) - def test_var_arg_callable_subtyping_5(self): - self.assert_proper_subtype( + def test_var_arg_callable_subtyping_5(self) -> None: + self.assert_strict_subtype( self.fx.callable_var_arg(0, self.fx.a, self.fx.d, self.fx.a), - self.fx.callable(self.fx.b, self.fx.a)) + self.fx.callable(self.fx.b, self.fx.a), + ) - def test_var_arg_callable_subtyping_6(self): - self.assert_proper_subtype( + def test_var_arg_callable_subtyping_6(self) -> None: + self.assert_strict_subtype( self.fx.callable_var_arg(0, self.fx.a, self.fx.f, self.fx.d), - self.fx.callable_var_arg(0, self.fx.b, self.fx.e, self.fx.d)) + self.fx.callable_var_arg(0, self.fx.b, self.fx.e, self.fx.d), + ) - def test_var_arg_callable_subtyping_7(self): + def test_var_arg_callable_subtyping_7(self) -> None: self.assert_not_subtype( self.fx.callable_var_arg(0, self.fx.b, self.fx.d), - self.fx.callable(self.fx.a, self.fx.d)) + self.fx.callable(self.fx.a, self.fx.d), + ) - def test_var_arg_callable_subtyping_8(self): + def test_var_arg_callable_subtyping_8(self) -> None: self.assert_not_subtype( self.fx.callable_var_arg(0, self.fx.b, self.fx.d), - self.fx.callable_var_arg(0, self.fx.a, self.fx.a, self.fx.d)) + self.fx.callable_var_arg(0, self.fx.a, self.fx.a, self.fx.d), + ) self.assert_subtype( self.fx.callable_var_arg(0, self.fx.a, self.fx.d), - self.fx.callable_var_arg(0, self.fx.b, self.fx.b, self.fx.d)) + self.fx.callable_var_arg(0, self.fx.b, self.fx.b, self.fx.d), + ) - def test_var_arg_callable_subtyping_9(self): + def test_var_arg_callable_subtyping_9(self) -> None: self.assert_not_subtype( self.fx.callable_var_arg(0, self.fx.b, self.fx.b, self.fx.d), - self.fx.callable_var_arg(0, self.fx.a, self.fx.d)) + self.fx.callable_var_arg(0, self.fx.a, self.fx.d), + ) self.assert_subtype( self.fx.callable_var_arg(0, self.fx.a, self.fx.a, self.fx.d), - self.fx.callable_var_arg(0, self.fx.b, self.fx.d)) + self.fx.callable_var_arg(0, self.fx.b, self.fx.d), + ) + + def test_type_callable_subtyping(self) -> None: + self.assert_subtype(self.fx.callable_type(self.fx.d, self.fx.a), self.fx.type_type) + + self.assert_strict_subtype( + self.fx.callable_type(self.fx.d, self.fx.b), self.fx.callable(self.fx.d, self.fx.a) + ) + + self.assert_strict_subtype( + self.fx.callable_type(self.fx.a, self.fx.b), self.fx.callable(self.fx.a, self.fx.b) + ) + + def test_type_var_tuple(self) -> None: + self.assert_subtype(Instance(self.fx.gvi, []), Instance(self.fx.gvi, [])) + self.assert_subtype( + Instance(self.fx.gvi, [self.fx.a, self.fx.b]), + Instance(self.fx.gvi, [self.fx.a, self.fx.b]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [self.fx.a, self.fx.b]), + Instance(self.fx.gvi, [self.fx.b, self.fx.a]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [self.fx.a, self.fx.b]), Instance(self.fx.gvi, [self.fx.a]) + ) + + self.assert_subtype( + Instance(self.fx.gvi, [UnpackType(self.fx.ss)]), + Instance(self.fx.gvi, [UnpackType(self.fx.ss)]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [UnpackType(self.fx.ss)]), + Instance(self.fx.gvi, [UnpackType(self.fx.us)]), + ) + + self.assert_not_subtype( + Instance(self.fx.gvi, [UnpackType(self.fx.ss)]), Instance(self.fx.gvi, []) + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [UnpackType(self.fx.ss)]), Instance(self.fx.gvi, [self.fx.anyt]) + ) + + def test_type_var_tuple_with_prefix_suffix(self) -> None: + self.assert_subtype( + Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss)]), + Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss)]), + ) + self.assert_subtype( + Instance(self.fx.gvi, [self.fx.a, self.fx.b, UnpackType(self.fx.ss)]), + Instance(self.fx.gvi, [self.fx.a, self.fx.b, UnpackType(self.fx.ss)]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss)]), + Instance(self.fx.gvi, [self.fx.b, UnpackType(self.fx.ss)]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss)]), + Instance(self.fx.gvi, [self.fx.a, self.fx.b, UnpackType(self.fx.ss)]), + ) - def test_type_callable_subtyping(self): - self.assert_proper_subtype( - self.fx.callable_type(self.fx.d, self.fx.a), self.fx.type_type) + self.assert_subtype( + Instance(self.fx.gvi, [UnpackType(self.fx.ss), self.fx.a]), + Instance(self.fx.gvi, [UnpackType(self.fx.ss), self.fx.a]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [UnpackType(self.fx.ss), self.fx.a]), + Instance(self.fx.gvi, [UnpackType(self.fx.ss), self.fx.b]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [UnpackType(self.fx.ss), self.fx.a]), + Instance(self.fx.gvi, [UnpackType(self.fx.ss), self.fx.a, self.fx.b]), + ) - self.assert_proper_subtype( - self.fx.callable_type(self.fx.d, self.fx.b), - self.fx.callable(self.fx.d, self.fx.a)) + self.assert_subtype( + Instance(self.fx.gvi, [self.fx.a, self.fx.b, UnpackType(self.fx.ss), self.fx.c]), + Instance(self.fx.gvi, [self.fx.a, self.fx.b, UnpackType(self.fx.ss), self.fx.c]), + ) + self.assert_not_subtype( + Instance(self.fx.gvi, [self.fx.a, self.fx.b, UnpackType(self.fx.ss), self.fx.c]), + Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss), self.fx.b, self.fx.c]), + ) - self.assert_proper_subtype(self.fx.callable_type(self.fx.a, self.fx.b), - self.fx.callable(self.fx.a, self.fx.b)) + def test_type_var_tuple_unpacked_variable_length_tuple(self) -> None: + self.assert_subtype( + Instance(self.fx.gvi, [self.fx.a, self.fx.a]), + Instance(self.fx.gvi, [UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))]), + ) # IDEA: Maybe add these test cases (they are tested pretty well in type # checker tests already): @@ -155,30 +280,24 @@ def test_type_callable_subtyping(self): # * more generic interface subtyping test cases # * type variables # * tuple types - # * void type # * None type # * any type # * generic function types - def assert_subtype(self, s, t): - assert_true(is_subtype(s, t), '{} not subtype of {}'.format(s, t)) + def assert_subtype(self, s: Type, t: Type) -> None: + assert is_subtype(s, t), f"{s} not subtype of {t}" - def assert_not_subtype(self, s, t): - assert_true(not is_subtype(s, t), '{} subtype of {}'.format(s, t)) + def assert_not_subtype(self, s: Type, t: Type) -> None: + assert not is_subtype(s, t), f"{s} subtype of {t}" - def assert_proper_subtype(self, s, t): + def assert_strict_subtype(self, s: Type, t: Type) -> None: self.assert_subtype(s, t) self.assert_not_subtype(t, s) - def assert_equivalent(self, s, t): + def assert_equivalent(self, s: Type, t: Type) -> None: self.assert_subtype(s, t) self.assert_subtype(t, s) - def assert_unrelated(self, s, t): + def assert_unrelated(self, s: Type, t: Type) -> None: self.assert_not_subtype(s, t) self.assert_not_subtype(t, s) - - -if __name__ == '__main__': - import sys - run_test(SubtypingSuite(), sys.argv[1:]) diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py index d44ab21b8c49..9388dca02c7a 100644 --- a/mypy/test/testtransform.py +++ b/mypy/test/testtransform.py @@ -1,81 +1,64 @@ """Identity AST transform test cases""" -import os.path - -from typing import Dict, List +from __future__ import annotations from mypy import build -from mypy.myunit import Suite, run_test -from mypy.test.helpers import assert_string_arrays_equal, testfile_pyversion -from mypy.test.data import parse_test_cases -from mypy.test.config import test_data_prefix, test_temp_dir from mypy.errors import CompileError -from mypy.nodes import TypeInfo -from mypy.treetransform import TransformVisitor +from mypy.modulefinder import BuildSource +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options +from mypy.test.visitors import TypeAssertTransformVisitor -class TransformSuite(Suite): +class TransformSuite(DataSuite): + required_out_section = True # Reuse semantic analysis test cases. - transform_files = ['semanal-basic.test', - 'semanal-expressions.test', - 'semanal-classes.test', - 'semanal-types.test', - 'semanal-modules.test', - 'semanal-statements.test', - 'semanal-abstractclasses.test', - 'semanal-python2.test'] + files = [ + "semanal-basic.test", + "semanal-expressions.test", + "semanal-classes.test", + "semanal-types.test", + "semanal-modules.test", + "semanal-statements.test", + "semanal-abstractclasses.test", + ] + native_sep = True - def cases(self): - c = [] - for f in self.transform_files: - c += parse_test_cases(os.path.join(test_data_prefix, f), - test_transform, test_temp_dir) - return c + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_transform(testcase) -def test_transform(testcase): +def test_transform(testcase: DataDrivenTestCase) -> None: """Perform an identity transform test case.""" try: - src = '\n'.join(testcase.input) - result = build.build('main', - target=build.SEMANTIC_ANALYSIS, - program_text=src, - pyversion=testfile_pyversion(testcase.file), - flags=[build.TEST_BUILTINS], - alt_lib_path=test_temp_dir) - a = [] + src = "\n".join(testcase.input) + options = parse_options(src, testcase, 1) + options.use_builtins_fixtures = True + options.semantic_analysis_only = True + options.show_traceback = True + options.force_uppercase_builtins = True + result = build.build( + sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir + ) + a = result.errors + if a: + raise CompileError(a) # Include string representations of the source files in the actual # output. - for fnam in sorted(result.files.keys()): - f = result.files[fnam] - - # Omit the builtins module and files with a special marker in the - # path. - # TODO the test is not reliable - if (not f.path.endswith((os.sep + 'builtins.py', - 'typing.py', - 'abc.py')) - and not os.path.basename(f.path).startswith('_') - and not os.path.splitext( - os.path.basename(f.path))[0].endswith('_')): - t = TestTransformVisitor() - f = t.node(f) - a += str(f).split('\n') + for module in sorted(result.files.keys()): + if module in testcase.test_modules: + t = TypeAssertTransformVisitor() + t.test_only = True + file = t.mypyfile(result.files[module]) + a += file.str_with_options(options).split("\n") except CompileError as e: a = e.messages + if testcase.normalize_output: + a = normalize_error_messages(a) assert_string_arrays_equal( - testcase.output, a, - 'Invalid semantic analyzer output ({}, line {})'.format(testcase.file, - testcase.line)) - - -class TestTransformVisitor(TransformVisitor): - def type(self, type): - assert type is not None - return type - - -if __name__ == '__main__': - import sys - run_test(TransformSuite(), sys.argv[1:]) + testcase.output, + a, + f"Invalid semantic analyzer output ({testcase.file}, line {testcase.line})", + ) diff --git a/mypy/test/testtypegen.py b/mypy/test/testtypegen.py index eeae93226114..4933bd3522a0 100644 --- a/mypy/test/testtypegen.py +++ b/mypy/test/testtypegen.py @@ -1,113 +1,83 @@ """Test cases for the type checker: exporting inferred types""" -import os.path -import re +from __future__ import annotations -import typing +import re from mypy import build -from mypy.myunit import Suite, run_test -from mypy.test import config -from mypy.test.data import parse_test_cases +from mypy.errors import CompileError +from mypy.modulefinder import BuildSource +from mypy.nodes import NameExpr, TempNode +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal +from mypy.test.visitors import SkippedNodeSearcher, ignore_node from mypy.util import short_type -from mypy.nodes import NameExpr, TypeVarExpr, CallExpr -from mypy.traverser import TraverserVisitor -from mypy.errors import CompileError -class TypeExportSuite(Suite): - # List of files that contain test case descriptions. - files = ['typexport-basic.test'] +class TypeExportSuite(DataSuite): + required_out_section = True + files = ["typexport-basic.test"] - def cases(self): - c = [] - for f in self.files: - c += parse_test_cases(os.path.join(config.test_data_prefix, f), - self.run_test, config.test_temp_dir) - return c - - def run_test(self, testcase): - a = [] + def run_case(self, testcase: DataDrivenTestCase) -> None: try: line = testcase.input[0] - mask = '' - if line.startswith('##'): - mask = '(' + line[2:].strip() + ')$' - - src = '\n'.join(testcase.input) - result = build.build(program_path='main', - target=build.TYPE_CHECK, - program_text=src, - flags=[build.TEST_BUILTINS], - alt_lib_path=config.test_temp_dir) + mask = "" + if line.startswith("##"): + mask = "(" + line[2:].strip() + ")$" + + src = "\n".join(testcase.input) + options = Options() + options.strict_optional = False # TODO: Enable strict optional checking + options.use_builtins_fixtures = True + options.show_traceback = True + options.export_types = True + options.preserve_asts = True + options.allow_empty_bodies = True + options.force_uppercase_builtins = True + result = build.build( + sources=[BuildSource("main", None, src)], + options=options, + alt_lib_path=test_temp_dir, + ) + a = result.errors map = result.types nodes = map.keys() # Ignore NameExpr nodes of variables with explicit (trivial) types - # to simplify output. Also ignore 'Undefined' nodes. - searcher = VariableDefinitionNodeSearcher() + # to simplify output. + searcher = SkippedNodeSearcher() for file in result.files.values(): + searcher.ignore_file = file.fullname not in testcase.test_modules file.accept(searcher) ignored = searcher.nodes # Filter nodes that should be included in the output. keys = [] for node in nodes: - if node.line is not None and node.line != -1 and map[node]: + if isinstance(node, TempNode): + continue + if node.line != -1 and map[node]: if ignore_node(node) or node in ignored: continue - if (re.match(mask, short_type(node)) - or (isinstance(node, NameExpr) - and re.match(mask, node.name))): + if re.match(mask, short_type(node)) or ( + isinstance(node, NameExpr) and re.match(mask, node.name) + ): # Include node in output. keys.append(node) - for key in sorted(keys, - key=lambda n: (n.line, short_type(n), - str(n) + str(map[n]))): - ts = str(map[key]).replace('*', '') # Remove erased tags - ts = ts.replace('__main__.', '') - a.append('{}({}) : {}'.format(short_type(key), key.line, ts)) + for key in sorted( + keys, + key=lambda n: (n.line, short_type(n), str(n) + map[n].str_with_options(options)), + ): + ts = map[key].str_with_options(options).replace("*", "") # Remove erased tags + ts = ts.replace("__main__.", "") + a.append(f"{short_type(key)}({key.line}) : {ts}") except CompileError as e: a = e.messages assert_string_arrays_equal( - testcase.output, a, - 'Invalid type checker output ({}, line {})'.format(testcase.file, - testcase.line)) - - -class VariableDefinitionNodeSearcher(TraverserVisitor): - def __init__(self): - self.nodes = set() - - def visit_assignment_stmt(self, s): - if s.type or ignore_node(s.rvalue): - for lvalue in s.lvalues: - if isinstance(lvalue, NameExpr): - self.nodes.add(lvalue) - if (isinstance(s.rvalue, NameExpr) - and s.rvalue.fullname == 'typing.Undefined'): - self.nodes.add(s.rvalue) - - -def ignore_node(node): - """Return True if node is to be omitted from test case output.""" - - # We want to get rid of object() expressions in the typing module stub - # and also typevar(...) expressions. Since detecting whether a node comes - # from the typing module is not easy, we just to strip them all away. - if isinstance(node, TypeVarExpr): - return True - if isinstance(node, NameExpr) and node.fullname == 'builtins.object': - return True - if isinstance(node, CallExpr) and (ignore_node(node.callee) or - node.analyzed): - return True - - return False - - -if __name__ == '__main__': - import sys - run_test(TypeExportSuite(), sys.argv[1:]) + testcase.output, + a, + f"Invalid type checker output ({testcase.file}, line {testcase.line})", + ) diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 1687cfb341e3..0380d1aa82d1 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -1,414 +1,954 @@ """Test cases for mypy types and type operations.""" -from typing import List - -from mypy.myunit import ( - Suite, assert_equal, assert_true, assert_false, run_test +from __future__ import annotations + +import re +from unittest import TestCase, skipUnless + +from mypy.erasetype import erase_type, remove_instance_last_known_values +from mypy.indirection import TypeIndirectionVisitor +from mypy.join import join_simple, join_types +from mypy.meet import meet_types, narrow_declared_type +from mypy.nodes import ( + ARG_NAMED, + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + CONTRAVARIANT, + COVARIANT, + INVARIANT, + ArgKind, + CallExpr, + Expression, + NameExpr, ) -from mypy.erasetype import erase_type -from mypy.expandtype import expand_type -from mypy.join import join_types -from mypy.meet import meet_types +from mypy.options import Options +from mypy.plugins.common import find_shallow_matching_overload_item +from mypy.state import state +from mypy.subtypes import is_more_precise, is_proper_subtype, is_same_type, is_subtype +from mypy.test.helpers import Suite, assert_equal, assert_type, skip +from mypy.test.typefixture import InterfaceTypeFixture, TypeFixture +from mypy.typeops import false_only, make_simplified_union, true_only from mypy.types import ( - UnboundType, AnyType, Void, Callable, TupleType, TypeVarDef, Type, - Instance, NoneTyp, ErrorType + AnyType, + CallableType, + Instance, + LiteralType, + NoneType, + Overloaded, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, + has_recursive_types, ) -from mypy.nodes import ARG_POS, ARG_OPT, ARG_STAR -from mypy.replacetvars import replace_type_vars -from mypy.subtypes import is_subtype, is_more_precise, is_proper_subtype -from mypy.typefixture import TypeFixture, InterfaceTypeFixture + +# Solving the import cycle: +import mypy.expandtype # ruff: isort: skip class TypesSuite(Suite): - def __init__(self): - super().__init__() - self.x = UnboundType('X') # Helpers - self.y = UnboundType('Y') + def setUp(self) -> None: + self.x = UnboundType("X") # Helpers + self.y = UnboundType("Y") self.fx = TypeFixture() - self.function = self.fx.std_function - - def test_any(self): - assert_equal(str(AnyType()), 'Any') - - def test_simple_unbound_type(self): - u = UnboundType('Foo') - assert_equal(str(u), 'Foo?') - - def test_generic_unbound_type(self): - u = UnboundType('Foo', [UnboundType('T'), AnyType()]) - assert_equal(str(u), 'Foo?[T?, Any]') - - def test_void_type(self): - assert_equal(str(Void(None)), 'void') - - def test_callable_type(self): - c = Callable([self.x, self.y], - [ARG_POS, ARG_POS], - [None, None], - AnyType(), self.function) - assert_equal(str(c), 'def (X?, Y?) -> Any') - - c2 = Callable([], [], [], Void(None), False) - assert_equal(str(c2), 'def ()') - - def test_callable_type_with_default_args(self): - c = Callable([self.x, self.y], [ARG_POS, ARG_OPT], [None, None], - AnyType(), self.function) - assert_equal(str(c), 'def (X?, Y? =) -> Any') - - c2 = Callable([self.x, self.y], [ARG_OPT, ARG_OPT], [None, None], - AnyType(), self.function) - assert_equal(str(c2), 'def (X? =, Y? =) -> Any') - - def test_callable_type_with_var_args(self): - c = Callable([self.x], [ARG_STAR], [None], AnyType(), self.function) - assert_equal(str(c), 'def (*X?) -> Any') - - c2 = Callable([self.x, self.y], [ARG_POS, ARG_STAR], - [None, None], AnyType(), self.function) - assert_equal(str(c2), 'def (X?, *Y?) -> Any') - - c3 = Callable([self.x, self.y], [ARG_OPT, ARG_STAR], [None, None], - AnyType(), self.function) - assert_equal(str(c3), 'def (X? =, *Y?) -> Any') - - def test_tuple_type(self): - assert_equal(str(TupleType([], None)), 'Tuple[]') - assert_equal(str(TupleType([self.x], None)), 'Tuple[X?]') - assert_equal(str(TupleType([self.x, AnyType()], None)), 'Tuple[X?, Any]') - - def test_type_variable_binding(self): - assert_equal(str(TypeVarDef('X', 1, None, self.fx.o)), 'X') - assert_equal(str(TypeVarDef('X', 1, [self.x, self.y], self.fx.o)), - 'X in (X?, Y?)') - - def test_generic_function_type(self): - c = Callable([self.x, self.y], [ARG_POS, ARG_POS], [None, None], - self.y, self.function, None, - [TypeVarDef('X', -1, None, self.fx.o)]) - assert_equal(str(c), 'def [X] (X?, Y?) -> Y?') - - v = [TypeVarDef('Y', -1, None, self.fx.o), - TypeVarDef('X', -2, None, self.fx.o)] - c2 = Callable([], [], [], Void(None), self.function, None, v) - assert_equal(str(c2), 'def [Y, X] ()') + self.function = self.fx.function + + def test_any(self) -> None: + assert_equal(str(AnyType(TypeOfAny.special_form)), "Any") + + def test_simple_unbound_type(self) -> None: + u = UnboundType("Foo") + assert_equal(str(u), "Foo?") + + def test_generic_unbound_type(self) -> None: + u = UnboundType("Foo", [UnboundType("T"), AnyType(TypeOfAny.special_form)]) + assert_equal(str(u), "Foo?[T?, Any]") + + def test_callable_type(self) -> None: + c = CallableType( + [self.x, self.y], + [ARG_POS, ARG_POS], + [None, None], + AnyType(TypeOfAny.special_form), + self.function, + ) + assert_equal(str(c), "def (X?, Y?) -> Any") + + c2 = CallableType([], [], [], NoneType(), self.fx.function) + assert_equal(str(c2), "def ()") + + def test_callable_type_with_default_args(self) -> None: + c = CallableType( + [self.x, self.y], + [ARG_POS, ARG_OPT], + [None, None], + AnyType(TypeOfAny.special_form), + self.function, + ) + assert_equal(str(c), "def (X?, Y? =) -> Any") + + c2 = CallableType( + [self.x, self.y], + [ARG_OPT, ARG_OPT], + [None, None], + AnyType(TypeOfAny.special_form), + self.function, + ) + assert_equal(str(c2), "def (X? =, Y? =) -> Any") + + def test_callable_type_with_var_args(self) -> None: + c = CallableType( + [self.x], [ARG_STAR], [None], AnyType(TypeOfAny.special_form), self.function + ) + assert_equal(str(c), "def (*X?) -> Any") + + c2 = CallableType( + [self.x, self.y], + [ARG_POS, ARG_STAR], + [None, None], + AnyType(TypeOfAny.special_form), + self.function, + ) + assert_equal(str(c2), "def (X?, *Y?) -> Any") + + c3 = CallableType( + [self.x, self.y], + [ARG_OPT, ARG_STAR], + [None, None], + AnyType(TypeOfAny.special_form), + self.function, + ) + assert_equal(str(c3), "def (X? =, *Y?) -> Any") + + def test_tuple_type_upper(self) -> None: + options = Options() + options.force_uppercase_builtins = True + assert_equal(TupleType([], self.fx.std_tuple).str_with_options(options), "Tuple[()]") + assert_equal(TupleType([self.x], self.fx.std_tuple).str_with_options(options), "Tuple[X?]") + assert_equal( + TupleType( + [self.x, AnyType(TypeOfAny.special_form)], self.fx.std_tuple + ).str_with_options(options), + "Tuple[X?, Any]", + ) + + def test_type_variable_binding(self) -> None: + assert_equal( + str( + TypeVarType( + "X", "X", TypeVarId(1), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ) + ), + "X`1", + ) + assert_equal( + str( + TypeVarType( + "X", + "X", + TypeVarId(1), + [self.x, self.y], + self.fx.o, + AnyType(TypeOfAny.from_omitted_generics), + ) + ), + "X`1", + ) + + def test_generic_function_type(self) -> None: + c = CallableType( + [self.x, self.y], + [ARG_POS, ARG_POS], + [None, None], + self.y, + self.function, + name=None, + variables=[ + TypeVarType( + "X", + "X", + TypeVarId(-1), + [], + self.fx.o, + AnyType(TypeOfAny.from_omitted_generics), + ) + ], + ) + assert_equal(str(c), "def [X] (X?, Y?) -> Y?") + + v = [ + TypeVarType( + "Y", "Y", TypeVarId(-1), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ), + TypeVarType( + "X", "X", TypeVarId(-2), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ), + ] + c2 = CallableType([], [], [], NoneType(), self.function, name=None, variables=v) + assert_equal(str(c2), "def [Y, X] ()") + + def test_type_alias_expand_once(self) -> None: + A, target = self.fx.def_alias_1(self.fx.a) + assert get_proper_type(A) == target + assert get_proper_type(target) == target + + A, target = self.fx.def_alias_2(self.fx.a) + assert get_proper_type(A) == target + assert get_proper_type(target) == target + + def test_type_alias_expand_all(self) -> None: + A, _ = self.fx.def_alias_1(self.fx.a) + assert A.expand_all_if_possible() is None + A, _ = self.fx.def_alias_2(self.fx.a) + assert A.expand_all_if_possible() is None + + B = self.fx.non_rec_alias(self.fx.a) + C = self.fx.non_rec_alias(TupleType([B, B], Instance(self.fx.std_tuplei, [B]))) + assert C.expand_all_if_possible() == TupleType( + [self.fx.a, self.fx.a], Instance(self.fx.std_tuplei, [self.fx.a]) + ) + + def test_recursive_nested_in_non_recursive(self) -> None: + A, _ = self.fx.def_alias_1(self.fx.a) + T = TypeVarType( + "T", "T", TypeVarId(-1), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ) + NA = self.fx.non_rec_alias(Instance(self.fx.gi, [T]), [T], [A]) + assert not NA.is_recursive + assert has_recursive_types(NA) + + def test_indirection_no_infinite_recursion(self) -> None: + A, _ = self.fx.def_alias_1(self.fx.a) + visitor = TypeIndirectionVisitor() + modules = A.accept(visitor) + assert modules == {"__main__", "builtins"} + + A, _ = self.fx.def_alias_2(self.fx.a) + visitor = TypeIndirectionVisitor() + modules = A.accept(visitor) + assert modules == {"__main__", "builtins"} class TypeOpsSuite(Suite): - def set_up(self): - self.fx = TypeFixture() + def setUp(self) -> None: + self.fx = TypeFixture(INVARIANT) + self.fx_co = TypeFixture(COVARIANT) + self.fx_contra = TypeFixture(CONTRAVARIANT) # expand_type - def test_trivial_expand(self): - for t in (self.fx.a, self.fx.o, self.fx.t, self.fx.void, self.fx.nonet, - self.tuple(self.fx.a), - self.callable([], self.fx.a, self.fx.a), self.fx.anyt): + def test_trivial_expand(self) -> None: + for t in ( + self.fx.a, + self.fx.o, + self.fx.t, + self.fx.nonet, + self.tuple(self.fx.a), + self.callable([], self.fx.a, self.fx.a), + self.fx.anyt, + ): self.assert_expand(t, [], t) self.assert_expand(t, [], t) self.assert_expand(t, [], t) - def test_expand_naked_type_var(self): - self.assert_expand(self.fx.t, [(1, self.fx.a)], self.fx.a) - self.assert_expand(self.fx.t, [(2, self.fx.a)], self.fx.t) + def test_trivial_expand_recursive(self) -> None: + A, _ = self.fx.def_alias_1(self.fx.a) + self.assert_expand(A, [], A) + A, _ = self.fx.def_alias_2(self.fx.a) + self.assert_expand(A, [], A) + + def test_expand_naked_type_var(self) -> None: + self.assert_expand(self.fx.t, [(self.fx.t.id, self.fx.a)], self.fx.a) + self.assert_expand(self.fx.t, [(self.fx.s.id, self.fx.a)], self.fx.t) - def test_expand_basic_generic_types(self): - self.assert_expand(self.fx.gt, [(1, self.fx.a)], self.fx.ga) + def test_expand_basic_generic_types(self) -> None: + self.assert_expand(self.fx.gt, [(self.fx.t.id, self.fx.a)], self.fx.ga) # IDEA: Add test cases for # tuple types # callable types # multiple arguments - def assert_expand(self, orig, map_items, result): + def assert_expand( + self, orig: Type, map_items: list[tuple[TypeVarId, Type]], result: Type + ) -> None: lower_bounds = {} for id, t in map_items: lower_bounds[id] = t - exp = expand_type(orig, lower_bounds) + exp = mypy.expandtype.expand_type(orig, lower_bounds) # Remove erased tags (asterisks). - assert_equal(str(exp).replace('*', ''), str(result)) - - # replace_type_vars - - def test_trivial_replace(self): - for t in (self.fx.a, self.fx.o, self.fx.void, self.fx.nonet, - self.tuple(self.fx.a), - self.callable([], self.fx.a, self.fx.a), self.fx.anyt, - self.fx.err): - self.assert_replace(t, t) - - def test_replace_type_var(self): - self.assert_replace(self.fx.t, self.fx.anyt) - - def test_replace_generic_instance(self): - self.assert_replace(self.fx.ga, self.fx.ga) - self.assert_replace(self.fx.gt, self.fx.gdyn) - - def assert_replace(self, orig, result): - assert_equal(str(replace_type_vars(orig)), str(result)) + assert_equal(str(exp).replace("*", ""), str(result)) # erase_type - def test_trivial_erase(self): - for t in (self.fx.a, self.fx.o, self.fx.void, self.fx.nonet, - self.fx.anyt, self.fx.err): + def test_trivial_erase(self) -> None: + for t in (self.fx.a, self.fx.o, self.fx.nonet, self.fx.anyt): self.assert_erase(t, t) - def test_erase_with_type_variable(self): + def test_erase_with_type_variable(self) -> None: self.assert_erase(self.fx.t, self.fx.anyt) - def test_erase_with_generic_type(self): + def test_erase_with_generic_type(self) -> None: self.assert_erase(self.fx.ga, self.fx.gdyn) - self.assert_erase(self.fx.hab, - Instance(self.fx.hi, [self.fx.anyt, self.fx.anyt])) + self.assert_erase(self.fx.hab, Instance(self.fx.hi, [self.fx.anyt, self.fx.anyt])) - def test_erase_with_tuple_type(self): - self.assert_erase(self.tuple(self.fx.a), self.fx.std_tuple) + def test_erase_with_generic_type_recursive(self) -> None: + tuple_any = Instance(self.fx.std_tuplei, [AnyType(TypeOfAny.explicit)]) + A, _ = self.fx.def_alias_1(self.fx.a) + self.assert_erase(A, tuple_any) + A, _ = self.fx.def_alias_2(self.fx.a) + self.assert_erase(A, UnionType([self.fx.a, tuple_any])) - def test_erase_with_function_type(self): - self.assert_erase(self.fx.callable(self.fx.a, self.fx.b), - self.fx.callable_type(self.fx.void)) - - def test_erase_with_type_object(self): - self.assert_erase(self.fx.callable_type(self.fx.a, self.fx.b), - self.fx.callable_type(self.fx.void)) + def test_erase_with_tuple_type(self) -> None: + self.assert_erase(self.tuple(self.fx.a), self.fx.std_tuple) - def assert_erase(self, orig, result): + def test_erase_with_function_type(self) -> None: + self.assert_erase( + self.fx.callable(self.fx.a, self.fx.b), + CallableType( + arg_types=[self.fx.anyt, self.fx.anyt], + arg_kinds=[ARG_STAR, ARG_STAR2], + arg_names=[None, None], + ret_type=self.fx.anyt, + fallback=self.fx.function, + ), + ) + + def test_erase_with_type_object(self) -> None: + self.assert_erase( + self.fx.callable_type(self.fx.a, self.fx.b), + CallableType( + arg_types=[self.fx.anyt, self.fx.anyt], + arg_kinds=[ARG_STAR, ARG_STAR2], + arg_names=[None, None], + ret_type=self.fx.anyt, + fallback=self.fx.type_type, + ), + ) + + def test_erase_with_type_type(self) -> None: + self.assert_erase(self.fx.type_a, self.fx.type_a) + self.assert_erase(self.fx.type_t, self.fx.type_any) + + def assert_erase(self, orig: Type, result: Type) -> None: assert_equal(str(erase_type(orig)), str(result)) # is_more_precise - def test_is_more_precise(self): + def test_is_more_precise(self) -> None: fx = self.fx - assert_true(is_more_precise(fx.b, fx.a)) - assert_true(is_more_precise(fx.b, fx.b)) - assert_true(is_more_precise(fx.b, fx.b)) - assert_true(is_more_precise(fx.b, fx.anyt)) - assert_true(is_more_precise(self.tuple(fx.b, fx.a), - self.tuple(fx.b, fx.a))) - - assert_false(is_more_precise(fx.a, fx.b)) - assert_false(is_more_precise(fx.anyt, fx.b)) - assert_false(is_more_precise(self.tuple(fx.b, fx.b), - self.tuple(fx.b, fx.a))) + assert is_more_precise(fx.b, fx.a) + assert is_more_precise(fx.b, fx.b) + assert is_more_precise(fx.b, fx.b) + assert is_more_precise(fx.b, fx.anyt) + assert is_more_precise(self.tuple(fx.b, fx.a), self.tuple(fx.b, fx.a)) + assert is_more_precise(self.tuple(fx.b, fx.b), self.tuple(fx.b, fx.a)) + + assert not is_more_precise(fx.a, fx.b) + assert not is_more_precise(fx.anyt, fx.b) # is_proper_subtype - def test_is_proper_subtype(self): + def test_is_proper_subtype(self) -> None: + fx = self.fx + + assert is_proper_subtype(fx.a, fx.a) + assert is_proper_subtype(fx.b, fx.a) + assert is_proper_subtype(fx.b, fx.o) + assert is_proper_subtype(fx.b, fx.o) + + assert not is_proper_subtype(fx.a, fx.b) + assert not is_proper_subtype(fx.o, fx.b) + + assert is_proper_subtype(fx.anyt, fx.anyt) + assert not is_proper_subtype(fx.a, fx.anyt) + assert not is_proper_subtype(fx.anyt, fx.a) + + assert is_proper_subtype(fx.ga, fx.ga) + assert is_proper_subtype(fx.gdyn, fx.gdyn) + assert not is_proper_subtype(fx.ga, fx.gdyn) + assert not is_proper_subtype(fx.gdyn, fx.ga) + + assert is_proper_subtype(fx.t, fx.t) + assert not is_proper_subtype(fx.t, fx.s) + + assert is_proper_subtype(fx.a, UnionType([fx.a, fx.b])) + assert is_proper_subtype(UnionType([fx.a, fx.b]), UnionType([fx.a, fx.b, fx.c])) + assert not is_proper_subtype(UnionType([fx.a, fx.b]), UnionType([fx.b, fx.c])) + + def test_is_proper_subtype_covariance(self) -> None: + fx_co = self.fx_co + + assert is_proper_subtype(fx_co.gsab, fx_co.gb) + assert is_proper_subtype(fx_co.gsab, fx_co.ga) + assert not is_proper_subtype(fx_co.gsaa, fx_co.gb) + assert is_proper_subtype(fx_co.gb, fx_co.ga) + assert not is_proper_subtype(fx_co.ga, fx_co.gb) + + def test_is_proper_subtype_contravariance(self) -> None: + fx_contra = self.fx_contra + + assert is_proper_subtype(fx_contra.gsab, fx_contra.gb) + assert not is_proper_subtype(fx_contra.gsab, fx_contra.ga) + assert is_proper_subtype(fx_contra.gsaa, fx_contra.gb) + assert not is_proper_subtype(fx_contra.gb, fx_contra.ga) + assert is_proper_subtype(fx_contra.ga, fx_contra.gb) + + def test_is_proper_subtype_invariance(self) -> None: + fx = self.fx + + assert is_proper_subtype(fx.gsab, fx.gb) + assert not is_proper_subtype(fx.gsab, fx.ga) + assert not is_proper_subtype(fx.gsaa, fx.gb) + assert not is_proper_subtype(fx.gb, fx.ga) + assert not is_proper_subtype(fx.ga, fx.gb) + + def test_is_proper_subtype_and_subtype_literal_types(self) -> None: + fx = self.fx + + lit1 = fx.lit1 + lit2 = fx.lit2 + lit3 = fx.lit3 + + assert is_proper_subtype(lit1, fx.a) + assert not is_proper_subtype(lit1, fx.d) + assert not is_proper_subtype(fx.a, lit1) + assert is_proper_subtype(fx.uninhabited, lit1) + assert not is_proper_subtype(lit1, fx.uninhabited) + assert is_proper_subtype(lit1, lit1) + assert not is_proper_subtype(lit1, lit2) + assert not is_proper_subtype(lit2, lit3) + + assert is_subtype(lit1, fx.a) + assert not is_subtype(lit1, fx.d) + assert not is_subtype(fx.a, lit1) + assert is_subtype(fx.uninhabited, lit1) + assert not is_subtype(lit1, fx.uninhabited) + assert is_subtype(lit1, lit1) + assert not is_subtype(lit1, lit2) + assert not is_subtype(lit2, lit3) + + assert not is_proper_subtype(lit1, fx.anyt) + assert not is_proper_subtype(fx.anyt, lit1) + + assert is_subtype(lit1, fx.anyt) + assert is_subtype(fx.anyt, lit1) + + def test_subtype_aliases(self) -> None: + A1, _ = self.fx.def_alias_1(self.fx.a) + AA1, _ = self.fx.def_alias_1(self.fx.a) + assert is_subtype(A1, AA1) + assert is_subtype(AA1, A1) + + A2, _ = self.fx.def_alias_2(self.fx.a) + AA2, _ = self.fx.def_alias_2(self.fx.a) + assert is_subtype(A2, AA2) + assert is_subtype(AA2, A2) + + B1, _ = self.fx.def_alias_1(self.fx.b) + B2, _ = self.fx.def_alias_2(self.fx.b) + assert is_subtype(B1, A1) + assert is_subtype(B2, A2) + assert not is_subtype(A1, B1) + assert not is_subtype(A2, B2) + + assert not is_subtype(A2, A1) + assert is_subtype(A1, A2) + + # can_be_true / can_be_false + + def test_empty_tuple_always_false(self) -> None: + tuple_type = self.tuple() + assert tuple_type.can_be_false + assert not tuple_type.can_be_true + + def test_nonempty_tuple_always_true(self) -> None: + tuple_type = self.tuple(AnyType(TypeOfAny.special_form), AnyType(TypeOfAny.special_form)) + assert tuple_type.can_be_true + assert not tuple_type.can_be_false + + def test_union_can_be_true_if_any_true(self) -> None: + union_type = UnionType([self.fx.a, self.tuple()]) + assert union_type.can_be_true + + def test_union_can_not_be_true_if_none_true(self) -> None: + union_type = UnionType([self.tuple(), self.tuple()]) + assert not union_type.can_be_true + + def test_union_can_be_false_if_any_false(self) -> None: + union_type = UnionType([self.fx.a, self.tuple()]) + assert union_type.can_be_false + + def test_union_can_not_be_false_if_none_false(self) -> None: + union_type = UnionType([self.tuple(self.fx.a), self.tuple(self.fx.d)]) + assert not union_type.can_be_false + + # true_only / false_only + + def test_true_only_of_false_type_is_uninhabited(self) -> None: + to = true_only(NoneType()) + assert_type(UninhabitedType, to) + + def test_true_only_of_true_type_is_idempotent(self) -> None: + always_true = self.tuple(AnyType(TypeOfAny.special_form)) + to = true_only(always_true) + assert always_true is to + + def test_true_only_of_instance(self) -> None: + to = true_only(self.fx.a) + assert_equal(str(to), "A") + assert to.can_be_true + assert not to.can_be_false + assert_type(Instance, to) + # The original class still can be false + assert self.fx.a.can_be_false + + def test_true_only_of_union(self) -> None: + tup_type = self.tuple(AnyType(TypeOfAny.special_form)) + # Union of something that is unknown, something that is always true, something + # that is always false + union_type = UnionType([self.fx.a, tup_type, self.tuple()]) + to = true_only(union_type) + assert isinstance(to, UnionType) + assert_equal(len(to.items), 2) + assert to.items[0].can_be_true + assert not to.items[0].can_be_false + assert to.items[1] is tup_type + + def test_false_only_of_true_type_is_uninhabited(self) -> None: + with state.strict_optional_set(True): + fo = false_only(self.tuple(AnyType(TypeOfAny.special_form))) + assert_type(UninhabitedType, fo) + + def test_false_only_tuple(self) -> None: + with state.strict_optional_set(False): + fo = false_only(self.tuple(self.fx.a)) + assert_equal(fo, NoneType()) + with state.strict_optional_set(True): + fo = false_only(self.tuple(self.fx.a)) + assert_equal(fo, UninhabitedType()) + + def test_false_only_of_false_type_is_idempotent(self) -> None: + always_false = NoneType() + fo = false_only(always_false) + assert always_false is fo + + def test_false_only_of_instance(self) -> None: + fo = false_only(self.fx.a) + assert_equal(str(fo), "A") + assert not fo.can_be_true + assert fo.can_be_false + assert_type(Instance, fo) + # The original class still can be true + assert self.fx.a.can_be_true + + def test_false_only_of_union(self) -> None: + with state.strict_optional_set(True): + tup_type = self.tuple() + # Union of something that is unknown, something that is always true, something + # that is always false + union_type = UnionType( + [self.fx.a, self.tuple(AnyType(TypeOfAny.special_form)), tup_type] + ) + assert_equal(len(union_type.items), 3) + fo = false_only(union_type) + assert isinstance(fo, UnionType) + assert_equal(len(fo.items), 2) + assert not fo.items[0].can_be_true + assert fo.items[0].can_be_false + assert fo.items[1] is tup_type + + def test_simplified_union(self) -> None: + fx = self.fx + + self.assert_simplified_union([fx.a, fx.a], fx.a) + self.assert_simplified_union([fx.a, fx.b], fx.a) + self.assert_simplified_union([fx.a, fx.d], UnionType([fx.a, fx.d])) + self.assert_simplified_union([fx.a, fx.uninhabited], fx.a) + self.assert_simplified_union([fx.ga, fx.gs2a], fx.ga) + self.assert_simplified_union([fx.ga, fx.gsab], UnionType([fx.ga, fx.gsab])) + self.assert_simplified_union([fx.ga, fx.gsba], fx.ga) + self.assert_simplified_union([fx.a, UnionType([fx.d])], UnionType([fx.a, fx.d])) + self.assert_simplified_union([fx.a, UnionType([fx.a])], fx.a) + self.assert_simplified_union( + [fx.b, UnionType([fx.c, UnionType([fx.d])])], UnionType([fx.b, fx.c, fx.d]) + ) + + def test_simplified_union_with_literals(self) -> None: + fx = self.fx + + self.assert_simplified_union([fx.lit1, fx.a], fx.a) + self.assert_simplified_union([fx.lit1, fx.lit2, fx.a], fx.a) + self.assert_simplified_union([fx.lit1, fx.lit1], fx.lit1) + self.assert_simplified_union([fx.lit1, fx.lit2], UnionType([fx.lit1, fx.lit2])) + self.assert_simplified_union([fx.lit1, fx.lit3], UnionType([fx.lit1, fx.lit3])) + self.assert_simplified_union([fx.lit1, fx.uninhabited], fx.lit1) + self.assert_simplified_union([fx.lit1_inst, fx.a], fx.a) + self.assert_simplified_union([fx.lit1_inst, fx.lit1_inst], fx.lit1_inst) + self.assert_simplified_union( + [fx.lit1_inst, fx.lit2_inst], UnionType([fx.lit1_inst, fx.lit2_inst]) + ) + self.assert_simplified_union( + [fx.lit1_inst, fx.lit3_inst], UnionType([fx.lit1_inst, fx.lit3_inst]) + ) + self.assert_simplified_union([fx.lit1_inst, fx.uninhabited], fx.lit1_inst) + self.assert_simplified_union([fx.lit1, fx.lit1_inst], fx.lit1) + self.assert_simplified_union([fx.lit1, fx.lit2_inst], UnionType([fx.lit1, fx.lit2_inst])) + self.assert_simplified_union([fx.lit1, fx.lit3_inst], UnionType([fx.lit1, fx.lit3_inst])) + + def test_simplified_union_with_str_literals(self) -> None: fx = self.fx - assert_true(is_proper_subtype(fx.a, fx.a)) - assert_true(is_proper_subtype(fx.b, fx.a)) - assert_true(is_proper_subtype(fx.b, fx.o)) - assert_true(is_proper_subtype(fx.b, fx.o)) + self.assert_simplified_union([fx.lit_str1, fx.lit_str2, fx.str_type], fx.str_type) + self.assert_simplified_union([fx.lit_str1, fx.lit_str1, fx.lit_str1], fx.lit_str1) + self.assert_simplified_union( + [fx.lit_str1, fx.lit_str2, fx.lit_str3], + UnionType([fx.lit_str1, fx.lit_str2, fx.lit_str3]), + ) + self.assert_simplified_union( + [fx.lit_str1, fx.lit_str2, fx.uninhabited], UnionType([fx.lit_str1, fx.lit_str2]) + ) + + def test_simplify_very_large_union(self) -> None: + fx = self.fx + literals = [] + for i in range(5000): + literals.append(LiteralType("v%d" % i, fx.str_type)) + # This shouldn't be very slow, even if the union is big. + self.assert_simplified_union([*literals, fx.str_type], fx.str_type) - assert_false(is_proper_subtype(fx.a, fx.b)) - assert_false(is_proper_subtype(fx.o, fx.b)) + def test_simplified_union_with_str_instance_literals(self) -> None: + fx = self.fx - assert_true(is_proper_subtype(fx.anyt, fx.anyt)) - assert_false(is_proper_subtype(fx.a, fx.anyt)) - assert_false(is_proper_subtype(fx.anyt, fx.a)) + self.assert_simplified_union( + [fx.lit_str1_inst, fx.lit_str2_inst, fx.str_type], fx.str_type + ) + self.assert_simplified_union( + [fx.lit_str1_inst, fx.lit_str1_inst, fx.lit_str1_inst], fx.lit_str1_inst + ) + self.assert_simplified_union( + [fx.lit_str1_inst, fx.lit_str2_inst, fx.lit_str3_inst], + UnionType([fx.lit_str1_inst, fx.lit_str2_inst, fx.lit_str3_inst]), + ) + self.assert_simplified_union( + [fx.lit_str1_inst, fx.lit_str2_inst, fx.uninhabited], + UnionType([fx.lit_str1_inst, fx.lit_str2_inst]), + ) + + def test_simplified_union_with_mixed_str_literals(self) -> None: + fx = self.fx - assert_true(is_proper_subtype(fx.ga, fx.ga)) - assert_true(is_proper_subtype(fx.gdyn, fx.gdyn)) - assert_true(is_proper_subtype(fx.gsab, fx.gb)) - assert_false(is_proper_subtype(fx.gsab, fx.ga)) - assert_false(is_proper_subtype(fx.gb, fx.ga)) - assert_false(is_proper_subtype(fx.ga, fx.gdyn)) - assert_false(is_proper_subtype(fx.gdyn, fx.ga)) + self.assert_simplified_union( + [fx.lit_str1, fx.lit_str2, fx.lit_str3_inst], + UnionType([fx.lit_str1, fx.lit_str2, fx.lit_str3_inst]), + ) + self.assert_simplified_union([fx.lit_str1, fx.lit_str1, fx.lit_str1_inst], fx.lit_str1) - assert_true(is_proper_subtype(fx.t, fx.t)) - assert_false(is_proper_subtype(fx.t, fx.s)) + def assert_simplified_union(self, original: list[Type], union: Type) -> None: + assert_equal(make_simplified_union(original), union) + assert_equal(make_simplified_union(list(reversed(original))), union) # Helpers - def tuple(self, *a): - return TupleType(a, self.fx.std_tuple) + def tuple(self, *a: Type) -> TupleType: + return TupleType(list(a), self.fx.std_tuple) - def callable(self, vars, *a) -> Callable: + def callable(self, vars: list[str], *a: Type) -> CallableType: """callable(args, a1, ..., an, r) constructs a callable with argument types a1, ... an and return type r and type arguments vars. """ - tv = [] # type: List[TypeVarDef] + tv: list[TypeVarType] = [] n = -1 for v in vars: - tv.append(TypeVarDef(v, n, None, self.fx.o)) + tv.append( + TypeVarType( + v, v, TypeVarId(n), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ) + ) n -= 1 - return Callable(a[:-1], - [ARG_POS] * (len(a) - 1), - [None] * (len(a) - 1), - a[-1], - self.fx.std_function, - None, - tv) + return CallableType( + list(a[:-1]), + [ARG_POS] * (len(a) - 1), + [None] * (len(a) - 1), + a[-1], + self.fx.function, + name=None, + variables=tv, + ) class JoinSuite(Suite): - def set_up(self): - self.fx = TypeFixture() + def setUp(self) -> None: + self.fx = TypeFixture(INVARIANT) + self.fx_co = TypeFixture(COVARIANT) + self.fx_contra = TypeFixture(CONTRAVARIANT) - def test_trivial_cases(self): - for simple in self.fx.void, self.fx.a, self.fx.o, self.fx.b: + def test_trivial_cases(self) -> None: + for simple in self.fx.a, self.fx.o, self.fx.b: self.assert_join(simple, simple, simple) - def test_class_subtyping(self): + def test_class_subtyping(self) -> None: self.assert_join(self.fx.a, self.fx.o, self.fx.o) self.assert_join(self.fx.b, self.fx.o, self.fx.o) self.assert_join(self.fx.a, self.fx.d, self.fx.o) self.assert_join(self.fx.b, self.fx.c, self.fx.a) self.assert_join(self.fx.b, self.fx.d, self.fx.o) - def test_tuples(self): + def test_tuples(self) -> None: self.assert_join(self.tuple(), self.tuple(), self.tuple()) - self.assert_join(self.tuple(self.fx.a), - self.tuple(self.fx.a), - self.tuple(self.fx.a)) - self.assert_join(self.tuple(self.fx.b, self.fx.c), - self.tuple(self.fx.a, self.fx.d), - self.tuple(self.fx.a, self.fx.o)) - - self.assert_join(self.tuple(self.fx.a, self.fx.a), - self.fx.std_tuple, - self.fx.o) - self.assert_join(self.tuple(self.fx.a), - self.tuple(self.fx.a, self.fx.a), - self.fx.o) - - def test_function_types(self): - self.assert_join(self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.a, self.fx.b)) - - self.assert_join(self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.b, self.fx.b), - self.fx.o) - self.assert_join(self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.a, self.fx.a), - self.fx.o) - - def test_type_vars(self): + self.assert_join(self.tuple(self.fx.a), self.tuple(self.fx.a), self.tuple(self.fx.a)) + self.assert_join( + self.tuple(self.fx.b, self.fx.c), + self.tuple(self.fx.a, self.fx.d), + self.tuple(self.fx.a, self.fx.o), + ) + + self.assert_join( + self.tuple(self.fx.a, self.fx.a), self.fx.std_tuple, self.var_tuple(self.fx.anyt) + ) + self.assert_join( + self.tuple(self.fx.a), self.tuple(self.fx.a, self.fx.a), self.var_tuple(self.fx.a) + ) + self.assert_join( + self.tuple(self.fx.b), self.tuple(self.fx.a, self.fx.c), self.var_tuple(self.fx.a) + ) + self.assert_join(self.tuple(), self.tuple(self.fx.a), self.var_tuple(self.fx.a)) + + def test_var_tuples(self) -> None: + self.assert_join( + self.tuple(self.fx.a), self.var_tuple(self.fx.a), self.var_tuple(self.fx.a) + ) + self.assert_join( + self.var_tuple(self.fx.a), self.tuple(self.fx.a), self.var_tuple(self.fx.a) + ) + self.assert_join(self.var_tuple(self.fx.a), self.tuple(), self.var_tuple(self.fx.a)) + + def test_function_types(self) -> None: + self.assert_join( + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.a, self.fx.b), + ) + + self.assert_join( + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.b, self.fx.b), + self.callable(self.fx.b, self.fx.b), + ) + self.assert_join( + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.a, self.fx.a), + self.callable(self.fx.a, self.fx.a), + ) + self.assert_join(self.callable(self.fx.a, self.fx.b), self.fx.function, self.fx.function) + self.assert_join( + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.d, self.fx.b), + self.fx.function, + ) + + def test_type_vars(self) -> None: self.assert_join(self.fx.t, self.fx.t, self.fx.t) self.assert_join(self.fx.s, self.fx.s, self.fx.s) self.assert_join(self.fx.t, self.fx.s, self.fx.o) - def test_void(self): - self.assert_join(self.fx.void, self.fx.void, self.fx.void) - self.assert_join(self.fx.void, self.fx.anyt, self.fx.anyt) - - # Join of any other type against void results in ErrorType, since there - # is no other meaningful result. - for t in [self.fx.a, self.fx.o, NoneTyp(), UnboundType('x'), - self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_join(t, self.fx.void, self.fx.err) - - def test_none(self): - # Any type t joined with None results in t. - for t in [NoneTyp(), self.fx.a, self.fx.o, UnboundType('x'), - self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b), self.fx.anyt]: - self.assert_join(t, NoneTyp(), t) - - def test_unbound_type(self): - self.assert_join(UnboundType('x'), UnboundType('x'), self.fx.anyt) - self.assert_join(UnboundType('x'), UnboundType('y'), self.fx.anyt) + def test_none(self) -> None: + with state.strict_optional_set(False): + # Any type t joined with None results in t. + for t in [ + NoneType(), + self.fx.a, + self.fx.o, + UnboundType("x"), + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + self.fx.anyt, + ]: + self.assert_join(t, NoneType(), t) + + def test_unbound_type(self) -> None: + self.assert_join(UnboundType("x"), UnboundType("x"), self.fx.anyt) + self.assert_join(UnboundType("x"), UnboundType("y"), self.fx.anyt) # Any type t joined with an unbound type results in dynamic. Unbound # type means that there is an error somewhere in the program, so this # does not affect type safety (whatever the result). - for t in [self.fx.a, self.fx.o, self.fx.ga, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_join(t, UnboundType('X'), self.fx.anyt) - - def test_any_type(self): + for t in [ + self.fx.a, + self.fx.o, + self.fx.ga, + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + ]: + self.assert_join(t, UnboundType("X"), self.fx.anyt) + + def test_any_type(self) -> None: # Join against 'Any' type always results in 'Any'. - for t in [self.fx.anyt, self.fx.a, self.fx.o, NoneTyp(), - UnboundType('x'), self.fx.void, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: + with state.strict_optional_set(False): + self.assert_join(NoneType(), self.fx.anyt, self.fx.anyt) + + for t in [ + self.fx.anyt, + self.fx.a, + self.fx.o, + NoneType(), + UnboundType("x"), + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + ]: self.assert_join(t, self.fx.anyt, self.fx.anyt) - def test_other_mixed_types(self): + def test_mixed_truth_restricted_type_simple(self) -> None: + # join_simple against differently restricted truthiness types drops restrictions. + true_a = true_only(self.fx.a) + false_o = false_only(self.fx.o) + j = join_simple(self.fx.o, true_a, false_o) + assert j.can_be_true + assert j.can_be_false + + def test_mixed_truth_restricted_type(self) -> None: + # join_types against differently restricted truthiness types drops restrictions. + true_any = true_only(AnyType(TypeOfAny.special_form)) + false_o = false_only(self.fx.o) + j = join_types(true_any, false_o) + assert j.can_be_true + assert j.can_be_false + + def test_other_mixed_types(self) -> None: # In general, joining unrelated types produces object. - for t1 in [self.fx.a, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - for t2 in [self.fx.a, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: + for t1 in [self.fx.a, self.fx.t, self.tuple(), self.callable(self.fx.a, self.fx.b)]: + for t2 in [self.fx.a, self.fx.t, self.tuple(), self.callable(self.fx.a, self.fx.b)]: if str(t1) != str(t2): self.assert_join(t1, t2, self.fx.o) - def test_error_type(self): - self.assert_join(self.fx.err, self.fx.anyt, self.fx.anyt) + def test_simple_generics(self) -> None: + with state.strict_optional_set(False): + self.assert_join(self.fx.ga, self.fx.nonet, self.fx.ga) + with state.strict_optional_set(True): + self.assert_join(self.fx.ga, self.fx.nonet, UnionType([self.fx.ga, NoneType()])) + + self.assert_join(self.fx.ga, self.fx.anyt, self.fx.anyt) - # Meet against any type except dynamic results in ErrorType. - for t in [self.fx.a, self.fx.o, NoneTyp(), UnboundType('x'), - self.fx.void, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_join(t, self.fx.err, self.fx.err) + for t in [ + self.fx.a, + self.fx.o, + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + ]: + self.assert_join(t, self.fx.ga, self.fx.o) - def test_simple_generics(self): + def test_generics_invariant(self) -> None: self.assert_join(self.fx.ga, self.fx.ga, self.fx.ga) self.assert_join(self.fx.ga, self.fx.gb, self.fx.o) + self.assert_join(self.fx.ga, self.fx.gd, self.fx.o) self.assert_join(self.fx.ga, self.fx.g2a, self.fx.o) - self.assert_join(self.fx.ga, self.fx.nonet, self.fx.ga) - self.assert_join(self.fx.ga, self.fx.anyt, self.fx.anyt) - - for t in [self.fx.a, self.fx.o, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_join(t, self.fx.ga, self.fx.o) + def test_generics_covariant(self) -> None: + self.assert_join(self.fx_co.ga, self.fx_co.ga, self.fx_co.ga) + self.assert_join(self.fx_co.ga, self.fx_co.gb, self.fx_co.ga) + self.assert_join(self.fx_co.ga, self.fx_co.gd, self.fx_co.go) + self.assert_join(self.fx_co.ga, self.fx_co.g2a, self.fx_co.o) + + def test_generics_contravariant(self) -> None: + self.assert_join(self.fx_contra.ga, self.fx_contra.ga, self.fx_contra.ga) + # TODO: this can be more precise than "object", see a comment in mypy/join.py + self.assert_join(self.fx_contra.ga, self.fx_contra.gb, self.fx_contra.o) + self.assert_join(self.fx_contra.ga, self.fx_contra.g2a, self.fx_contra.o) + + def test_generics_with_multiple_args(self) -> None: + self.assert_join(self.fx_co.hab, self.fx_co.hab, self.fx_co.hab) + self.assert_join(self.fx_co.hab, self.fx_co.hbb, self.fx_co.hab) + self.assert_join(self.fx_co.had, self.fx_co.haa, self.fx_co.hao) + + def test_generics_with_inheritance(self) -> None: + self.assert_join(self.fx_co.gsab, self.fx_co.gb, self.fx_co.gb) + self.assert_join(self.fx_co.gsba, self.fx_co.gb, self.fx_co.ga) + self.assert_join(self.fx_co.gsab, self.fx_co.gd, self.fx_co.go) + + def test_generics_with_inheritance_and_shared_supertype(self) -> None: + self.assert_join(self.fx_co.gsba, self.fx_co.gs2a, self.fx_co.ga) + self.assert_join(self.fx_co.gsab, self.fx_co.gs2a, self.fx_co.ga) + self.assert_join(self.fx_co.gsab, self.fx_co.gs2d, self.fx_co.go) + + def test_generic_types_and_any(self) -> None: + self.assert_join(self.fx.gdyn, self.fx.ga, self.fx.gdyn) + self.assert_join(self.fx_co.gdyn, self.fx_co.ga, self.fx_co.gdyn) + self.assert_join(self.fx_contra.gdyn, self.fx_contra.ga, self.fx_contra.gdyn) - def test_generics_with_multiple_args(self): - self.assert_join(self.fx.hab, self.fx.hab, self.fx.hab) - self.assert_join(self.fx.hab, self.fx.haa, self.fx.o) - self.assert_join(self.fx.hab, self.fx.hbb, self.fx.o) + def test_callables_with_any(self) -> None: + self.assert_join( + self.callable(self.fx.a, self.fx.a, self.fx.anyt, self.fx.a), + self.callable(self.fx.a, self.fx.anyt, self.fx.a, self.fx.anyt), + self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, self.fx.anyt), + ) - def test_generics_with_inheritance(self): - self.assert_join(self.fx.gsab, self.fx.gb, self.fx.gb) - self.assert_join(self.fx.gsba, self.fx.gb, self.fx.o) + def test_overloaded(self) -> None: + c = self.callable - def test_generics_with_inheritance_and_shared_supertype(self): - self.assert_join(self.fx.gsba, self.fx.gs2a, self.fx.ga) - self.assert_join(self.fx.gsab, self.fx.gs2a, self.fx.o) + def ov(*items: CallableType) -> Overloaded: + return Overloaded(list(items)) - def test_generic_types_and_any(self): - self.assert_join(self.fx.gdyn, self.fx.ga, self.fx.gdyn) + fx = self.fx + func = fx.function + c1 = c(fx.a, fx.a) + c2 = c(fx.b, fx.b) + c3 = c(fx.c, fx.c) + self.assert_join(ov(c1, c2), c1, c1) + self.assert_join(ov(c1, c2), c2, c2) + self.assert_join(ov(c1, c2), ov(c1, c2), ov(c1, c2)) + self.assert_join(ov(c1, c2), ov(c1, c3), c1) + self.assert_join(ov(c2, c1), ov(c3, c1), c1) + self.assert_join(ov(c1, c2), c3, func) + + def test_overloaded_with_any(self) -> None: + c = self.callable + + def ov(*items: CallableType) -> Overloaded: + return Overloaded(list(items)) - def test_callables_with_any(self): - self.assert_join(self.callable(self.fx.a, self.fx.a, self.fx.anyt, - self.fx.a), - self.callable(self.fx.a, self.fx.anyt, self.fx.a, - self.fx.anyt), - self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, - self.fx.anyt)) + fx = self.fx + any = fx.anyt + self.assert_join(ov(c(fx.a, fx.a), c(fx.b, fx.b)), c(any, fx.b), c(any, fx.b)) + self.assert_join(ov(c(fx.a, fx.a), c(any, fx.b)), c(fx.b, fx.b), c(any, fx.b)) - def test_join_interface_types(self): - self.skip() # FIX + def test_join_interface_types(self) -> None: self.assert_join(self.fx.f, self.fx.f, self.fx.f) self.assert_join(self.fx.f, self.fx.f2, self.fx.o) self.assert_join(self.fx.f, self.fx.f3, self.fx.f) - def test_join_interface_and_class_types(self): - self.skip() # FIX - + def test_join_interface_and_class_types(self) -> None: self.assert_join(self.fx.o, self.fx.f, self.fx.o) self.assert_join(self.fx.a, self.fx.f, self.fx.o) self.assert_join(self.fx.e, self.fx.f, self.fx.f) - def test_join_class_types_with_interface_result(self): - self.skip() # FIX + @skip + def test_join_class_types_with_interface_result(self) -> None: # Unique result self.assert_join(self.fx.e, self.fx.e2, self.fx.f) # Ambiguous result - self.assert_join(self.fx.e2, self.fx.e3, self.fx.err) - - def test_generic_interfaces(self): - self.skip() # FIX + self.assert_join(self.fx.e2, self.fx.e3, self.fx.anyt) + @skip + def test_generic_interfaces(self) -> None: fx = InterfaceTypeFixture() self.assert_join(fx.gfa, fx.gfa, fx.gfa) @@ -418,270 +958,635 @@ def test_generic_interfaces(self): self.assert_join(fx.m1, fx.gfb, fx.o) - def test_simple_type_objects(self): + def test_simple_type_objects(self) -> None: t1 = self.type_callable(self.fx.a, self.fx.a) t2 = self.type_callable(self.fx.b, self.fx.b) + tr = self.type_callable(self.fx.b, self.fx.a) self.assert_join(t1, t1, t1) - assert_true(join_types(t1, t1).is_type_obj()) + j = join_types(t1, t1) + assert isinstance(j, CallableType) + assert j.is_type_obj() - self.assert_join(t1, t2, self.fx.type_type) + self.assert_join(t1, t2, tr) self.assert_join(t1, self.fx.type_type, self.fx.type_type) - self.assert_join(self.fx.type_type, self.fx.type_type, - self.fx.type_type) + self.assert_join(self.fx.type_type, self.fx.type_type, self.fx.type_type) + + def test_type_type(self) -> None: + self.assert_join(self.fx.type_a, self.fx.type_b, self.fx.type_a) + self.assert_join(self.fx.type_b, self.fx.type_any, self.fx.type_any) + self.assert_join(self.fx.type_b, self.fx.type_type, self.fx.type_type) + self.assert_join(self.fx.type_b, self.fx.type_c, self.fx.type_a) + self.assert_join(self.fx.type_c, self.fx.type_d, TypeType.make_normalized(self.fx.o)) + self.assert_join(self.fx.type_type, self.fx.type_any, self.fx.type_type) + self.assert_join(self.fx.type_b, self.fx.anyt, self.fx.anyt) + + def test_literal_type(self) -> None: + a = self.fx.a + d = self.fx.d + lit1 = self.fx.lit1 + lit2 = self.fx.lit2 + lit3 = self.fx.lit3 + + self.assert_join(lit1, lit1, lit1) + self.assert_join(lit1, a, a) + self.assert_join(lit1, d, self.fx.o) + self.assert_join(lit1, lit2, a) + self.assert_join(lit1, lit3, self.fx.o) + self.assert_join(lit1, self.fx.anyt, self.fx.anyt) + self.assert_join(UnionType([lit1, lit2]), lit2, UnionType([lit1, lit2])) + self.assert_join(UnionType([lit1, lit2]), a, a) + self.assert_join(UnionType([lit1, lit3]), a, UnionType([a, lit3])) + self.assert_join(UnionType([d, lit3]), lit3, d) + self.assert_join(UnionType([d, lit3]), d, UnionType([d, lit3])) + self.assert_join(UnionType([a, lit1]), lit1, a) + self.assert_join(UnionType([a, lit1]), lit2, a) + self.assert_join(UnionType([lit1, lit2]), UnionType([lit1, lit2]), UnionType([lit1, lit2])) + + # The order in which we try joining two unions influences the + # ordering of the items in the final produced unions. So, we + # manually call 'assert_simple_join' and tune the output + # after swapping the arguments here. + self.assert_simple_join( + UnionType([lit1, lit2]), UnionType([lit2, lit3]), UnionType([lit1, lit2, lit3]) + ) + self.assert_simple_join( + UnionType([lit2, lit3]), UnionType([lit1, lit2]), UnionType([lit2, lit3, lit1]) + ) + + def test_variadic_tuple_joins(self) -> None: + # These tests really test just the "arity", to be sure it is handled correctly. + self.assert_join( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + ) + self.assert_join( + self.tuple(self.fx.a, self.fx.a), + self.tuple(self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + ) + self.assert_join( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple( + self.fx.b, UnpackType(Instance(self.fx.std_tuplei, [self.fx.b])), self.fx.b + ), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + ) # There are additional test cases in check-inference.test. - # FIX interfaces with different paths - - # FIX generic interfaces + inheritance - # FIX generic interfaces + ranges - - # FIX function types + varargs and default args + # TODO: Function types + varargs and default args. - def assert_join(self, s, t, join): + def assert_join(self, s: Type, t: Type, join: Type) -> None: self.assert_simple_join(s, t, join) self.assert_simple_join(t, s, join) - def assert_simple_join(self, s, t, join): + def assert_simple_join(self, s: Type, t: Type, join: Type) -> None: result = join_types(s, t) actual = str(result) expected = str(join) - assert_equal(actual, expected, - 'join({}, {}) == {{}} ({{}} expected)'.format(s, t)) - if not isinstance(s, ErrorType) and not isinstance(result, ErrorType): - assert_true(is_subtype(s, result), - '{} not subtype of {}'.format(s, result)) - if not isinstance(t, ErrorType) and not isinstance(result, ErrorType): - assert_true(is_subtype(t, result), - '{} not subtype of {}'.format(t, result)) - - def tuple(self, *a): - return TupleType(a, self.fx.std_tuple) - - def callable(self, *a): + assert_equal(actual, expected, f"join({s}, {t}) == {{}} ({{}} expected)") + assert is_subtype(s, result), f"{s} not subtype of {result}" + assert is_subtype(t, result), f"{t} not subtype of {result}" + + def tuple(self, *a: Type) -> TupleType: + return TupleType(list(a), self.fx.std_tuple) + + def var_tuple(self, t: Type) -> Instance: + """Construct a variable-length tuple type""" + return Instance(self.fx.std_tuplei, [t]) + + def callable(self, *a: Type) -> CallableType: """callable(a1, ..., an, r) constructs a callable with argument types a1, ... an and return type r. """ n = len(a) - 1 - return Callable(a[:-1], [ARG_POS] * n, [None] * n, - a[-1], self.fx.std_function) + return CallableType(list(a[:-1]), [ARG_POS] * n, [None] * n, a[-1], self.fx.function) - def type_callable(self, *a): + def type_callable(self, *a: Type) -> CallableType: """type_callable(a1, ..., an, r) constructs a callable with argument types a1, ... an and return type r, and which represents a type. """ n = len(a) - 1 - return Callable(a[:-1], [ARG_POS] * n, [None] * n, - a[-1], self.fx.type_type) + return CallableType(list(a[:-1]), [ARG_POS] * n, [None] * n, a[-1], self.fx.type_type) class MeetSuite(Suite): - def set_up(self): + def setUp(self) -> None: self.fx = TypeFixture() - def test_trivial_cases(self): - for simple in self.fx.void, self.fx.a, self.fx.o, self.fx.b: + def test_trivial_cases(self) -> None: + for simple in self.fx.a, self.fx.o, self.fx.b: self.assert_meet(simple, simple, simple) - def test_class_subtyping(self): + def test_class_subtyping(self) -> None: self.assert_meet(self.fx.a, self.fx.o, self.fx.a) self.assert_meet(self.fx.a, self.fx.b, self.fx.b) self.assert_meet(self.fx.b, self.fx.o, self.fx.b) - self.assert_meet(self.fx.a, self.fx.d, NoneTyp()) - self.assert_meet(self.fx.b, self.fx.c, NoneTyp()) + self.assert_meet(self.fx.a, self.fx.d, UninhabitedType()) + self.assert_meet(self.fx.b, self.fx.c, UninhabitedType()) - def test_tuples(self): + def test_tuples(self) -> None: self.assert_meet(self.tuple(), self.tuple(), self.tuple()) - self.assert_meet(self.tuple(self.fx.a), - self.tuple(self.fx.a), - self.tuple(self.fx.a)) - self.assert_meet(self.tuple(self.fx.b, self.fx.c), - self.tuple(self.fx.a, self.fx.d), - self.tuple(self.fx.b, NoneTyp())) - - self.assert_meet(self.tuple(self.fx.a, self.fx.a), - self.fx.std_tuple, - NoneTyp()) - self.assert_meet(self.tuple(self.fx.a), - self.tuple(self.fx.a, self.fx.a), - NoneTyp()) - - def test_function_types(self): - self.assert_meet(self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.a, self.fx.b)) - - self.assert_meet(self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.b, self.fx.b), - NoneTyp()) - self.assert_meet(self.callable(self.fx.a, self.fx.b), - self.callable(self.fx.a, self.fx.a), - NoneTyp()) - - def test_type_vars(self): + self.assert_meet(self.tuple(self.fx.a), self.tuple(self.fx.a), self.tuple(self.fx.a)) + self.assert_meet( + self.tuple(self.fx.b, self.fx.c), + self.tuple(self.fx.a, self.fx.d), + self.tuple(self.fx.b, UninhabitedType()), + ) + + self.assert_meet( + self.tuple(self.fx.a, self.fx.a), self.fx.std_tuple, self.tuple(self.fx.a, self.fx.a) + ) + self.assert_meet( + self.tuple(self.fx.a), self.tuple(self.fx.a, self.fx.a), UninhabitedType() + ) + + def test_function_types(self) -> None: + self.assert_meet( + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.a, self.fx.b), + ) + + self.assert_meet( + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.b, self.fx.b), + self.callable(self.fx.a, self.fx.b), + ) + self.assert_meet( + self.callable(self.fx.a, self.fx.b), + self.callable(self.fx.a, self.fx.a), + self.callable(self.fx.a, self.fx.b), + ) + + def test_type_vars(self) -> None: self.assert_meet(self.fx.t, self.fx.t, self.fx.t) self.assert_meet(self.fx.s, self.fx.s, self.fx.s) - self.assert_meet(self.fx.t, self.fx.s, NoneTyp()) - - def test_void(self): - self.assert_meet(self.fx.void, self.fx.void, self.fx.void) - self.assert_meet(self.fx.void, self.fx.anyt, self.fx.void) - - # Meet of any other type against void results in ErrorType, since there - # is no meaningful valid result. - for t in [self.fx.a, self.fx.o, UnboundType('x'), NoneTyp(), - self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_meet(t, self.fx.void, self.fx.err) - - def test_none(self): - self.assert_meet(NoneTyp(), NoneTyp(), NoneTyp()) - - self.assert_meet(NoneTyp(), self.fx.anyt, NoneTyp()) - self.assert_meet(NoneTyp(), self.fx.void, self.fx.err) - - # Any type t joined with None results in None, unless t is any or - # void. - for t in [self.fx.a, self.fx.o, UnboundType('x'), self.fx.t, - self.tuple(), self.callable(self.fx.a, self.fx.b)]: - self.assert_meet(t, NoneTyp(), NoneTyp()) - - def test_unbound_type(self): - self.assert_meet(UnboundType('x'), UnboundType('x'), self.fx.anyt) - self.assert_meet(UnboundType('x'), UnboundType('y'), self.fx.anyt) - - self.assert_meet(UnboundType('x'), self.fx.void, self.fx.err) - self.assert_meet(UnboundType('x'), self.fx.anyt, UnboundType('x')) - - # The meet of any type t with an unbound type results in dynamic - # (except for void). Unbound type means that there is an error - # somewhere in the program, so this does not affect type safety. - for t in [self.fx.a, self.fx.o, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_meet(t, UnboundType('X'), self.fx.anyt) - - def test_dynamic_type(self): + self.assert_meet(self.fx.t, self.fx.s, UninhabitedType()) + + def test_none(self) -> None: + self.assert_meet(NoneType(), NoneType(), NoneType()) + + self.assert_meet(NoneType(), self.fx.anyt, NoneType()) + + # Any type t joined with None results in None, unless t is Any. + with state.strict_optional_set(False): + for t in [ + self.fx.a, + self.fx.o, + UnboundType("x"), + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + ]: + self.assert_meet(t, NoneType(), NoneType()) + + with state.strict_optional_set(True): + self.assert_meet(self.fx.o, NoneType(), NoneType()) + for t in [ + self.fx.a, + UnboundType("x"), + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + ]: + self.assert_meet(t, NoneType(), UninhabitedType()) + + def test_unbound_type(self) -> None: + self.assert_meet(UnboundType("x"), UnboundType("x"), self.fx.anyt) + self.assert_meet(UnboundType("x"), UnboundType("y"), self.fx.anyt) + + self.assert_meet(UnboundType("x"), self.fx.anyt, UnboundType("x")) + + # The meet of any type t with an unbound type results in dynamic. + # Unbound type means that there is an error somewhere in the program, + # so this does not affect type safety. + for t in [ + self.fx.a, + self.fx.o, + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + ]: + self.assert_meet(t, UnboundType("X"), self.fx.anyt) + + def test_dynamic_type(self) -> None: # Meet against dynamic type always results in dynamic. - for t in [self.fx.anyt, self.fx.a, self.fx.o, NoneTyp(), - UnboundType('x'), self.fx.void, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: + for t in [ + self.fx.anyt, + self.fx.a, + self.fx.o, + NoneType(), + UnboundType("x"), + self.fx.t, + self.tuple(), + self.callable(self.fx.a, self.fx.b), + ]: self.assert_meet(t, self.fx.anyt, t) - def test_error_type(self): - self.assert_meet(self.fx.err, self.fx.anyt, self.fx.err) - - # Meet against any type except dynamic results in ErrorType. - for t in [self.fx.a, self.fx.o, NoneTyp(), UnboundType('x'), - self.fx.void, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_meet(t, self.fx.err, self.fx.err) - - def test_simple_generics(self): + def test_simple_generics(self) -> None: self.assert_meet(self.fx.ga, self.fx.ga, self.fx.ga) self.assert_meet(self.fx.ga, self.fx.o, self.fx.ga) - self.assert_meet(self.fx.ga, self.fx.gb, self.fx.nonet) - self.assert_meet(self.fx.ga, self.fx.g2a, self.fx.nonet) + self.assert_meet(self.fx.ga, self.fx.gb, self.fx.gb) + self.assert_meet(self.fx.ga, self.fx.gd, UninhabitedType()) + self.assert_meet(self.fx.ga, self.fx.g2a, UninhabitedType()) - self.assert_meet(self.fx.ga, self.fx.nonet, self.fx.nonet) + self.assert_meet(self.fx.ga, self.fx.nonet, UninhabitedType()) self.assert_meet(self.fx.ga, self.fx.anyt, self.fx.ga) - for t in [self.fx.a, self.fx.t, self.tuple(), - self.callable(self.fx.a, self.fx.b)]: - self.assert_meet(t, self.fx.ga, self.fx.nonet) + for t in [self.fx.a, self.fx.t, self.tuple(), self.callable(self.fx.a, self.fx.b)]: + self.assert_meet(t, self.fx.ga, UninhabitedType()) - def test_generics_with_multiple_args(self): + def test_generics_with_multiple_args(self) -> None: self.assert_meet(self.fx.hab, self.fx.hab, self.fx.hab) - self.assert_meet(self.fx.hab, self.fx.haa, self.fx.nonet) - self.assert_meet(self.fx.hab, self.fx.hbb, self.fx.nonet) + self.assert_meet(self.fx.hab, self.fx.haa, self.fx.hab) + self.assert_meet(self.fx.hab, self.fx.had, UninhabitedType()) + self.assert_meet(self.fx.hab, self.fx.hbb, self.fx.hbb) - def test_generics_with_inheritance(self): + def test_generics_with_inheritance(self) -> None: self.assert_meet(self.fx.gsab, self.fx.gb, self.fx.gsab) - self.assert_meet(self.fx.gsba, self.fx.gb, self.fx.nonet) + self.assert_meet(self.fx.gsba, self.fx.gb, UninhabitedType()) - def test_generics_with_inheritance_and_shared_supertype(self): - self.assert_meet(self.fx.gsba, self.fx.gs2a, self.fx.nonet) - self.assert_meet(self.fx.gsab, self.fx.gs2a, self.fx.nonet) + def test_generics_with_inheritance_and_shared_supertype(self) -> None: + self.assert_meet(self.fx.gsba, self.fx.gs2a, UninhabitedType()) + self.assert_meet(self.fx.gsab, self.fx.gs2a, UninhabitedType()) - def test_generic_types_and_dynamic(self): + def test_generic_types_and_dynamic(self) -> None: self.assert_meet(self.fx.gdyn, self.fx.ga, self.fx.ga) - def test_callables_with_dynamic(self): - self.assert_meet(self.callable(self.fx.a, self.fx.a, self.fx.anyt, - self.fx.a), - self.callable(self.fx.a, self.fx.anyt, self.fx.a, - self.fx.anyt), - self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, - self.fx.anyt)) + def test_callables_with_dynamic(self) -> None: + self.assert_meet( + self.callable(self.fx.a, self.fx.a, self.fx.anyt, self.fx.a), + self.callable(self.fx.a, self.fx.anyt, self.fx.a, self.fx.anyt), + self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, self.fx.anyt), + ) - def test_meet_interface_types(self): + def test_meet_interface_types(self) -> None: self.assert_meet(self.fx.f, self.fx.f, self.fx.f) - self.assert_meet(self.fx.f, self.fx.f2, self.fx.nonet) + self.assert_meet(self.fx.f, self.fx.f2, UninhabitedType()) self.assert_meet(self.fx.f, self.fx.f3, self.fx.f3) - def test_join_interface_and_class_types(self): + def test_meet_interface_and_class_types(self) -> None: self.assert_meet(self.fx.o, self.fx.f, self.fx.f) - self.assert_meet(self.fx.a, self.fx.f, self.fx.nonet) + self.assert_meet(self.fx.a, self.fx.f, UninhabitedType()) self.assert_meet(self.fx.e, self.fx.f, self.fx.e) - def test_join_class_types_with_shared_interfaces(self): + def test_meet_class_types_with_shared_interfaces(self) -> None: # These have nothing special with respect to meets, unlike joins. These # are for completeness only. - self.assert_meet(self.fx.e, self.fx.e2, self.fx.nonet) - self.assert_meet(self.fx.e2, self.fx.e3, self.fx.nonet) - - def test_meet_with_generic_interfaces(self): - # TODO fix - self.skip() + self.assert_meet(self.fx.e, self.fx.e2, UninhabitedType()) + self.assert_meet(self.fx.e2, self.fx.e3, UninhabitedType()) + def test_meet_with_generic_interfaces(self) -> None: fx = InterfaceTypeFixture() self.assert_meet(fx.gfa, fx.m1, fx.m1) self.assert_meet(fx.gfa, fx.gfa, fx.gfa) - self.assert_meet(fx.gfb, fx.m1, fx.nonet) + self.assert_meet(fx.gfb, fx.m1, UninhabitedType()) + + def test_type_type(self) -> None: + self.assert_meet(self.fx.type_a, self.fx.type_b, self.fx.type_b) + self.assert_meet(self.fx.type_b, self.fx.type_any, self.fx.type_b) + self.assert_meet(self.fx.type_b, self.fx.type_type, self.fx.type_b) + self.assert_meet(self.fx.type_b, self.fx.type_c, self.fx.type_never) + self.assert_meet(self.fx.type_c, self.fx.type_d, self.fx.type_never) + self.assert_meet(self.fx.type_type, self.fx.type_any, self.fx.type_any) + self.assert_meet(self.fx.type_b, self.fx.anyt, self.fx.type_b) + + def test_literal_type(self) -> None: + a = self.fx.a + lit1 = self.fx.lit1 + lit2 = self.fx.lit2 + lit3 = self.fx.lit3 + + self.assert_meet(lit1, lit1, lit1) + self.assert_meet(lit1, a, lit1) + self.assert_meet_uninhabited(lit1, lit3) + self.assert_meet_uninhabited(lit1, lit2) + self.assert_meet(UnionType([lit1, lit2]), lit1, lit1) + self.assert_meet(UnionType([lit1, lit2]), UnionType([lit2, lit3]), lit2) + self.assert_meet(UnionType([lit1, lit2]), UnionType([lit1, lit2]), UnionType([lit1, lit2])) + self.assert_meet(lit1, self.fx.anyt, lit1) + self.assert_meet(lit1, self.fx.o, lit1) + + assert is_same_type(lit1, narrow_declared_type(lit1, a)) + assert is_same_type(lit2, narrow_declared_type(lit2, a)) # FIX generic interfaces + ranges - def assert_meet(self, s, t, meet): + def assert_meet_uninhabited(self, s: Type, t: Type) -> None: + with state.strict_optional_set(False): + self.assert_meet(s, t, self.fx.nonet) + with state.strict_optional_set(True): + self.assert_meet(s, t, self.fx.uninhabited) + + def test_variadic_tuple_meets(self) -> None: + # These tests really test just the "arity", to be sure it is handled correctly. + self.assert_meet( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(self.fx.a, self.fx.a), + ) + self.assert_meet( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple(self.fx.a, self.fx.a), + ) + self.assert_meet( + self.tuple(self.fx.a, self.fx.a), + self.tuple(self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(self.fx.a, self.fx.a), + ) + self.assert_meet( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_meet( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple(self.fx.b, UnpackType(Instance(self.fx.std_tuplei, [self.fx.b]))), + self.tuple(self.fx.b, UnpackType(Instance(self.fx.std_tuplei, [self.fx.b]))), + ) + + def assert_meet(self, s: Type, t: Type, meet: Type) -> None: self.assert_simple_meet(s, t, meet) self.assert_simple_meet(t, s, meet) - def assert_simple_meet(self, s, t, meet): + def assert_simple_meet(self, s: Type, t: Type, meet: Type) -> None: result = meet_types(s, t) actual = str(result) expected = str(meet) - assert_equal(actual, expected, - 'meet({}, {}) == {{}} ({{}} expected)'.format(s, t)) - if not isinstance(s, ErrorType) and not isinstance(result, ErrorType): - assert_true(is_subtype(result, s), - '{} not subtype of {}'.format(result, s)) - if not isinstance(t, ErrorType) and not isinstance(result, ErrorType): - assert_true(is_subtype(result, t), - '{} not subtype of {}'.format(result, t)) - - def tuple(self, *a): - return TupleType(a, self.fx.std_tuple) - - def callable(self, *a): + assert_equal(actual, expected, f"meet({s}, {t}) == {{}} ({{}} expected)") + assert is_subtype(result, s), f"{result} not subtype of {s}" + assert is_subtype(result, t), f"{result} not subtype of {t}" + + def tuple(self, *a: Type) -> TupleType: + return TupleType(list(a), self.fx.std_tuple) + + def callable(self, *a: Type) -> CallableType: """callable(a1, ..., an, r) constructs a callable with argument types a1, ... an and return type r. """ n = len(a) - 1 - return Callable(a[:-1], - [ARG_POS] * n, [None] * n, - a[-1], self.fx.std_function) + return CallableType(list(a[:-1]), [ARG_POS] * n, [None] * n, a[-1], self.fx.function) + + +class SameTypeSuite(Suite): + def setUp(self) -> None: + self.fx = TypeFixture() + + def test_literal_type(self) -> None: + a = self.fx.a + b = self.fx.b # Reminder: b is a subclass of a + + lit1 = self.fx.lit1 + lit2 = self.fx.lit2 + lit3 = self.fx.lit3 + + self.assert_same(lit1, lit1) + self.assert_same(UnionType([lit1, lit2]), UnionType([lit1, lit2])) + self.assert_same(UnionType([lit1, lit2]), UnionType([lit2, lit1])) + self.assert_same(UnionType([a, b]), UnionType([b, a])) + self.assert_not_same(lit1, b) + self.assert_not_same(lit1, lit2) + self.assert_not_same(lit1, lit3) + self.assert_not_same(lit1, self.fx.anyt) + self.assert_not_same(lit1, self.fx.nonet) -class CombinedTypesSuite(Suite): - def __init__(self): - self.test_types = TypesSuite() - self.test_type_ops = TypeOpsSuite() - self.test_join = JoinSuite() - self.test_meet = MeetSuite() - super().__init__() + def assert_same(self, s: Type, t: Type, strict: bool = True) -> None: + self.assert_simple_is_same(s, t, expected=True, strict=strict) + self.assert_simple_is_same(t, s, expected=True, strict=strict) + def assert_not_same(self, s: Type, t: Type, strict: bool = True) -> None: + self.assert_simple_is_same(s, t, False, strict=strict) + self.assert_simple_is_same(t, s, False, strict=strict) -if __name__ == '__main__': - import sys - run_test(CombinedTypesSuite(), sys.argv[1:]) + def assert_simple_is_same(self, s: Type, t: Type, expected: bool, strict: bool) -> None: + actual = is_same_type(s, t) + assert_equal(actual, expected, f"is_same_type({s}, {t}) is {{}} ({{}} expected)") + + if strict: + actual2 = s == t + assert_equal(actual2, expected, f"({s} == {t}) is {{}} ({{}} expected)") + assert_equal( + hash(s) == hash(t), expected, f"(hash({s}) == hash({t}) is {{}} ({{}} expected)" + ) + + +class RemoveLastKnownValueSuite(Suite): + def setUp(self) -> None: + self.fx = TypeFixture() + + def test_optional(self) -> None: + t = UnionType.make_union([self.fx.a, self.fx.nonet]) + self.assert_union_result(t, [self.fx.a, self.fx.nonet]) + + def test_two_instances(self) -> None: + t = UnionType.make_union([self.fx.a, self.fx.b]) + self.assert_union_result(t, [self.fx.a, self.fx.b]) + + def test_multiple_same_instances(self) -> None: + t = UnionType.make_union([self.fx.a, self.fx.a]) + assert remove_instance_last_known_values(t) == self.fx.a + t = UnionType.make_union([self.fx.a, self.fx.a, self.fx.b]) + self.assert_union_result(t, [self.fx.a, self.fx.b]) + t = UnionType.make_union([self.fx.a, self.fx.nonet, self.fx.a, self.fx.b]) + self.assert_union_result(t, [self.fx.a, self.fx.nonet, self.fx.b]) + + def test_single_last_known_value(self) -> None: + t = UnionType.make_union([self.fx.lit1_inst, self.fx.nonet]) + self.assert_union_result(t, [self.fx.a, self.fx.nonet]) + + def test_last_known_values_with_merge(self) -> None: + t = UnionType.make_union([self.fx.lit1_inst, self.fx.lit2_inst, self.fx.lit4_inst]) + assert remove_instance_last_known_values(t) == self.fx.a + t = UnionType.make_union( + [self.fx.lit1_inst, self.fx.b, self.fx.lit2_inst, self.fx.lit4_inst] + ) + self.assert_union_result(t, [self.fx.a, self.fx.b]) + + def test_generics(self) -> None: + t = UnionType.make_union([self.fx.ga, self.fx.gb]) + self.assert_union_result(t, [self.fx.ga, self.fx.gb]) + + def assert_union_result(self, t: ProperType, expected: list[Type]) -> None: + t2 = remove_instance_last_known_values(t) + assert type(t2) is UnionType + assert t2.items == expected + + +class ShallowOverloadMatchingSuite(Suite): + def setUp(self) -> None: + self.fx = TypeFixture() + + def test_simple(self) -> None: + fx = self.fx + ov = self.make_overload([[("x", fx.anyt, ARG_NAMED)], [("y", fx.anyt, ARG_NAMED)]]) + # Match first only + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", "x")), 0) + # Match second only + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", "y")), 1) + # No match -- invalid keyword arg name + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", "z")), 1) + # No match -- missing arg + self.assert_find_shallow_matching_overload_item(ov, make_call(), 1) + # No match -- extra arg + self.assert_find_shallow_matching_overload_item( + ov, make_call(("foo", "x"), ("foo", "z")), 1 + ) + + def test_match_using_types(self) -> None: + fx = self.fx + ov = self.make_overload( + [ + [("x", fx.nonet, ARG_POS)], + [("x", fx.lit_false, ARG_POS)], + [("x", fx.lit_true, ARG_POS)], + [("x", fx.anyt, ARG_POS)], + ] + ) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", None)), 0) + self.assert_find_shallow_matching_overload_item(ov, make_call(("builtins.False", None)), 1) + self.assert_find_shallow_matching_overload_item(ov, make_call(("builtins.True", None)), 2) + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", None)), 3) + + def test_none_special_cases(self) -> None: + fx = self.fx + ov = self.make_overload( + [[("x", fx.callable(fx.nonet), ARG_POS)], [("x", fx.nonet, ARG_POS)]] + ) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", None)), 1) + self.assert_find_shallow_matching_overload_item(ov, make_call(("func", None)), 0) + ov = self.make_overload([[("x", fx.str_type, ARG_POS)], [("x", fx.nonet, ARG_POS)]]) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", None)), 1) + self.assert_find_shallow_matching_overload_item(ov, make_call(("func", None)), 0) + ov = self.make_overload( + [[("x", UnionType([fx.str_type, fx.a]), ARG_POS)], [("x", fx.nonet, ARG_POS)]] + ) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", None)), 1) + self.assert_find_shallow_matching_overload_item(ov, make_call(("func", None)), 0) + ov = self.make_overload([[("x", fx.o, ARG_POS)], [("x", fx.nonet, ARG_POS)]]) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", None)), 0) + self.assert_find_shallow_matching_overload_item(ov, make_call(("func", None)), 0) + ov = self.make_overload( + [[("x", UnionType([fx.str_type, fx.nonet]), ARG_POS)], [("x", fx.nonet, ARG_POS)]] + ) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", None)), 0) + self.assert_find_shallow_matching_overload_item(ov, make_call(("func", None)), 0) + ov = self.make_overload([[("x", fx.anyt, ARG_POS)], [("x", fx.nonet, ARG_POS)]]) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", None)), 0) + self.assert_find_shallow_matching_overload_item(ov, make_call(("func", None)), 0) + + def test_optional_arg(self) -> None: + fx = self.fx + ov = self.make_overload( + [[("x", fx.anyt, ARG_NAMED)], [("y", fx.anyt, ARG_OPT)], [("z", fx.anyt, ARG_NAMED)]] + ) + self.assert_find_shallow_matching_overload_item(ov, make_call(), 1) + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", "x")), 0) + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", "y")), 1) + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", "z")), 2) + + def test_two_args(self) -> None: + fx = self.fx + ov = self.make_overload( + [ + [("x", fx.nonet, ARG_OPT), ("y", fx.anyt, ARG_OPT)], + [("x", fx.anyt, ARG_OPT), ("y", fx.anyt, ARG_OPT)], + ] + ) + self.assert_find_shallow_matching_overload_item(ov, make_call(), 0) + self.assert_find_shallow_matching_overload_item(ov, make_call(("None", "x")), 0) + self.assert_find_shallow_matching_overload_item(ov, make_call(("foo", "x")), 1) + self.assert_find_shallow_matching_overload_item( + ov, make_call(("foo", "y"), ("None", "x")), 0 + ) + self.assert_find_shallow_matching_overload_item( + ov, make_call(("foo", "y"), ("bar", "x")), 1 + ) + + def assert_find_shallow_matching_overload_item( + self, ov: Overloaded, call: CallExpr, expected_index: int + ) -> None: + c = find_shallow_matching_overload_item(ov, call) + assert c in ov.items + assert ov.items.index(c) == expected_index + + def make_overload(self, items: list[list[tuple[str, Type, ArgKind]]]) -> Overloaded: + result = [] + for item in items: + arg_types = [] + arg_names = [] + arg_kinds = [] + for name, typ, kind in item: + arg_names.append(name) + arg_types.append(typ) + arg_kinds.append(kind) + result.append( + CallableType( + arg_types, arg_kinds, arg_names, ret_type=NoneType(), fallback=self.fx.o + ) + ) + return Overloaded(result) + + +def make_call(*items: tuple[str, str | None]) -> CallExpr: + args: list[Expression] = [] + arg_names = [] + arg_kinds = [] + for arg, name in items: + shortname = arg.split(".")[-1] + n = NameExpr(shortname) + n.fullname = arg + args.append(n) + arg_names.append(name) + if name: + arg_kinds.append(ARG_NAMED) + else: + arg_kinds.append(ARG_POS) + return CallExpr(NameExpr("f"), args, arg_kinds, arg_names) + + +class TestExpandTypeLimitGetProperType(TestCase): + # WARNING: do not increase this number unless absolutely necessary, + # and you understand what you are doing. + ALLOWED_GET_PROPER_TYPES = 9 + + @skipUnless(mypy.expandtype.__file__.endswith(".py"), "Skip for compiled mypy") + def test_count_get_proper_type(self) -> None: + with open(mypy.expandtype.__file__) as f: + code = f.read() + get_proper_type_count = len(re.findall("get_proper_type", code)) + assert get_proper_type_count == self.ALLOWED_GET_PROPER_TYPES diff --git a/mypy/test/testutil.py b/mypy/test/testutil.py new file mode 100644 index 000000000000..a7c3f1c00fee --- /dev/null +++ b/mypy/test/testutil.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import os +from unittest import TestCase, mock + +from mypy.inspections import parse_location +from mypy.util import _generate_junit_contents, get_terminal_width + + +class TestGetTerminalSize(TestCase): + def test_get_terminal_size_in_pty_defaults_to_80(self) -> None: + # when run using a pty, `os.get_terminal_size()` returns `0, 0` + ret = os.terminal_size((0, 0)) + mock_environ = os.environ.copy() + mock_environ.pop("COLUMNS", None) + with mock.patch.object(os, "get_terminal_size", return_value=ret): + with mock.patch.dict(os.environ, values=mock_environ, clear=True): + assert get_terminal_width() == 80 + + def test_parse_location_windows(self) -> None: + assert parse_location(r"C:\test.py:1:1") == (r"C:\test.py", [1, 1]) + assert parse_location(r"C:\test.py:1:1:1:1") == (r"C:\test.py", [1, 1, 1, 1]) + + +class TestWriteJunitXml(TestCase): + def test_junit_pass(self) -> None: + serious = False + messages_by_file: dict[str | None, list[str]] = {} + expected = """ + + + + +""" + result = _generate_junit_contents( + dt=1.23, + serious=serious, + messages_by_file=messages_by_file, + version="3.14", + platform="test-plat", + ) + assert result == expected + + def test_junit_fail_escape_xml_chars(self) -> None: + serious = False + messages_by_file: dict[str | None, list[str]] = { + "file1.py": ["Test failed", "another line < > &"] + } + expected = """ + + + Test failed +another line < > & + + +""" + result = _generate_junit_contents( + dt=1.23, + serious=serious, + messages_by_file=messages_by_file, + version="3.14", + platform="test-plat", + ) + assert result == expected + + def test_junit_fail_two_files(self) -> None: + serious = False + messages_by_file: dict[str | None, list[str]] = { + "file1.py": ["Test failed", "another line"], + "file2.py": ["Another failure", "line 2"], + } + expected = """ + + + Test failed +another line + + + Another failure +line 2 + + +""" + result = _generate_junit_contents( + dt=1.23, + serious=serious, + messages_by_file=messages_by_file, + version="3.14", + platform="test-plat", + ) + assert result == expected + + def test_serious_error(self) -> None: + serious = True + messages_by_file: dict[str | None, list[str]] = {None: ["Error line 1", "Error line 2"]} + expected = """ + + + Error line 1 +Error line 2 + + +""" + result = _generate_junit_contents( + dt=1.23, + serious=serious, + messages_by_file=messages_by_file, + version="3.14", + platform="test-plat", + ) + assert result == expected diff --git a/mypy/test/typefixture.py b/mypy/test/typefixture.py new file mode 100644 index 000000000000..d6c904732b17 --- /dev/null +++ b/mypy/test/typefixture.py @@ -0,0 +1,417 @@ +"""Fixture used in type-related test cases. + +It contains class TypeInfos and Type objects. +""" + +from __future__ import annotations + +from mypy.nodes import ( + ARG_OPT, + ARG_POS, + ARG_STAR, + COVARIANT, + MDEF, + Block, + ClassDef, + FuncDef, + SymbolTable, + SymbolTableNode, + TypeAlias, + TypeInfo, +) +from mypy.semanal_shared import set_callable_name +from mypy.types import ( + AnyType, + CallableType, + Instance, + LiteralType, + NoneType, + Type, + TypeAliasType, + TypeOfAny, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UninhabitedType, + UnionType, +) + + +class TypeFixture: + """Helper class that is used as a fixture in type-related unit tests. + + The members are initialized to contain various type-related values. + """ + + def __init__(self, variance: int = COVARIANT) -> None: + # The 'object' class + self.oi = self.make_type_info("builtins.object") # class object + self.o = Instance(self.oi, []) # object + + # Type variables (these are effectively global) + + def make_type_var( + name: str, id: int, values: list[Type], upper_bound: Type, variance: int + ) -> TypeVarType: + return TypeVarType( + name, + name, + TypeVarId(id), + values, + upper_bound, + AnyType(TypeOfAny.from_omitted_generics), + variance, + ) + + self.t = make_type_var("T", 1, [], self.o, variance) # T`1 (type variable) + self.tf = make_type_var("T", -1, [], self.o, variance) # T`-1 (type variable) + self.tf2 = make_type_var("T", -2, [], self.o, variance) # T`-2 (type variable) + self.s = make_type_var("S", 2, [], self.o, variance) # S`2 (type variable) + self.s1 = make_type_var("S", 1, [], self.o, variance) # S`1 (type variable) + self.sf = make_type_var("S", -2, [], self.o, variance) # S`-2 (type variable) + self.sf1 = make_type_var("S", -1, [], self.o, variance) # S`-1 (type variable) + self.u = make_type_var("U", 3, [], self.o, variance) # U`3 (type variable) + + # Simple types + self.anyt = AnyType(TypeOfAny.special_form) + self.nonet = NoneType() + self.uninhabited = UninhabitedType() + + # Abstract class TypeInfos + + # class F + self.fi = self.make_type_info("F", is_abstract=True) + + # class F2 + self.f2i = self.make_type_info("F2", is_abstract=True) + + # class F3(F) + self.f3i = self.make_type_info("F3", is_abstract=True, mro=[self.fi]) + + # Class TypeInfos + self.std_tuplei = self.make_type_info( + "builtins.tuple", mro=[self.oi], typevars=["T"], variances=[COVARIANT] + ) # class tuple + self.type_typei = self.make_type_info("builtins.type") # class type + self.bool_type_info = self.make_type_info("builtins.bool") + self.str_type_info = self.make_type_info("builtins.str") + self.functioni = self.make_type_info("builtins.function") # function TODO + self.ai = self.make_type_info("A", mro=[self.oi]) # class A + self.bi = self.make_type_info("B", mro=[self.ai, self.oi]) # class B(A) + self.ci = self.make_type_info("C", mro=[self.ai, self.oi]) # class C(A) + self.di = self.make_type_info("D", mro=[self.oi]) # class D + # class E(F) + self.ei = self.make_type_info("E", mro=[self.fi, self.oi]) + # class E2(F2, F) + self.e2i = self.make_type_info("E2", mro=[self.f2i, self.fi, self.oi]) + # class E3(F, F2) + self.e3i = self.make_type_info("E3", mro=[self.fi, self.f2i, self.oi]) + + # Generic class TypeInfos + # G[T] + self.gi = self.make_type_info("G", mro=[self.oi], typevars=["T"], variances=[variance]) + # G2[T] + self.g2i = self.make_type_info("G2", mro=[self.oi], typevars=["T"], variances=[variance]) + # H[S, T] + self.hi = self.make_type_info( + "H", mro=[self.oi], typevars=["S", "T"], variances=[variance, variance] + ) + # GS[T, S] <: G[S] + self.gsi = self.make_type_info( + "GS", + mro=[self.gi, self.oi], + typevars=["T", "S"], + variances=[variance, variance], + bases=[Instance(self.gi, [self.s])], + ) + # GS2[S] <: G[S] + self.gs2i = self.make_type_info( + "GS2", + mro=[self.gi, self.oi], + typevars=["S"], + variances=[variance], + bases=[Instance(self.gi, [self.s1])], + ) + + # list[T] + self.std_listi = self.make_type_info( + "builtins.list", mro=[self.oi], typevars=["T"], variances=[variance] + ) + + # Instance types + self.std_tuple = Instance(self.std_tuplei, [self.anyt]) # tuple + self.type_type = Instance(self.type_typei, []) # type + self.function = Instance(self.functioni, []) # function TODO + self.str_type = Instance(self.str_type_info, []) + self.bool_type = Instance(self.bool_type_info, []) + self.a = Instance(self.ai, []) # A + self.b = Instance(self.bi, []) # B + self.c = Instance(self.ci, []) # C + self.d = Instance(self.di, []) # D + + self.e = Instance(self.ei, []) # E + self.e2 = Instance(self.e2i, []) # E2 + self.e3 = Instance(self.e3i, []) # E3 + + self.f = Instance(self.fi, []) # F + self.f2 = Instance(self.f2i, []) # F2 + self.f3 = Instance(self.f3i, []) # F3 + + # Generic instance types + self.ga = Instance(self.gi, [self.a]) # G[A] + self.gb = Instance(self.gi, [self.b]) # G[B] + self.gd = Instance(self.gi, [self.d]) # G[D] + self.go = Instance(self.gi, [self.o]) # G[object] + self.gt = Instance(self.gi, [self.t]) # G[T`1] + self.gtf = Instance(self.gi, [self.tf]) # G[T`-1] + self.gtf2 = Instance(self.gi, [self.tf2]) # G[T`-2] + self.gs = Instance(self.gi, [self.s]) # G[S] + self.gdyn = Instance(self.gi, [self.anyt]) # G[Any] + self.gn = Instance(self.gi, [NoneType()]) # G[None] + + self.g2a = Instance(self.g2i, [self.a]) # G2[A] + + self.gsaa = Instance(self.gsi, [self.a, self.a]) # GS[A, A] + self.gsab = Instance(self.gsi, [self.a, self.b]) # GS[A, B] + self.gsba = Instance(self.gsi, [self.b, self.a]) # GS[B, A] + + self.gs2a = Instance(self.gs2i, [self.a]) # GS2[A] + self.gs2b = Instance(self.gs2i, [self.b]) # GS2[B] + self.gs2d = Instance(self.gs2i, [self.d]) # GS2[D] + + self.hab = Instance(self.hi, [self.a, self.b]) # H[A, B] + self.haa = Instance(self.hi, [self.a, self.a]) # H[A, A] + self.hbb = Instance(self.hi, [self.b, self.b]) # H[B, B] + self.hts = Instance(self.hi, [self.t, self.s]) # H[T, S] + self.had = Instance(self.hi, [self.a, self.d]) # H[A, D] + self.hao = Instance(self.hi, [self.a, self.o]) # H[A, object] + + self.lsta = Instance(self.std_listi, [self.a]) # List[A] + self.lstb = Instance(self.std_listi, [self.b]) # List[B] + + self.lit1 = LiteralType(1, self.a) + self.lit2 = LiteralType(2, self.a) + self.lit3 = LiteralType("foo", self.d) + self.lit4 = LiteralType(4, self.a) + self.lit1_inst = Instance(self.ai, [], last_known_value=self.lit1) + self.lit2_inst = Instance(self.ai, [], last_known_value=self.lit2) + self.lit3_inst = Instance(self.di, [], last_known_value=self.lit3) + self.lit4_inst = Instance(self.ai, [], last_known_value=self.lit4) + + self.lit_str1 = LiteralType("x", self.str_type) + self.lit_str2 = LiteralType("y", self.str_type) + self.lit_str3 = LiteralType("z", self.str_type) + self.lit_str1_inst = Instance(self.str_type_info, [], last_known_value=self.lit_str1) + self.lit_str2_inst = Instance(self.str_type_info, [], last_known_value=self.lit_str2) + self.lit_str3_inst = Instance(self.str_type_info, [], last_known_value=self.lit_str3) + + self.lit_false = LiteralType(False, self.bool_type) + self.lit_true = LiteralType(True, self.bool_type) + + self.type_a = TypeType.make_normalized(self.a) + self.type_b = TypeType.make_normalized(self.b) + self.type_c = TypeType.make_normalized(self.c) + self.type_d = TypeType.make_normalized(self.d) + self.type_t = TypeType.make_normalized(self.t) + self.type_any = TypeType.make_normalized(self.anyt) + self.type_never = TypeType.make_normalized(UninhabitedType()) + + self._add_bool_dunder(self.bool_type_info) + self._add_bool_dunder(self.ai) + + # TypeVars with non-trivial bounds + self.ub = make_type_var("UB", 5, [], self.b, variance) # UB`5 (type variable) + self.uc = make_type_var("UC", 6, [], self.c, variance) # UC`6 (type variable) + + def make_type_var_tuple(name: str, id: int, upper_bound: Type) -> TypeVarTupleType: + return TypeVarTupleType( + name, + name, + TypeVarId(id), + upper_bound, + self.std_tuple, + AnyType(TypeOfAny.from_omitted_generics), + ) + + obj_tuple = self.std_tuple.copy_modified(args=[self.o]) + self.ts = make_type_var_tuple("Ts", 1, obj_tuple) # Ts`1 (type var tuple) + self.ss = make_type_var_tuple("Ss", 2, obj_tuple) # Ss`2 (type var tuple) + self.us = make_type_var_tuple("Us", 3, obj_tuple) # Us`3 (type var tuple) + + self.gvi = self.make_type_info("GV", mro=[self.oi], typevars=["Ts"], typevar_tuple_index=0) + self.gv2i = self.make_type_info( + "GV2", mro=[self.oi], typevars=["T", "Ts", "S"], typevar_tuple_index=1 + ) + + def _add_bool_dunder(self, type_info: TypeInfo) -> None: + signature = CallableType([], [], [], Instance(self.bool_type_info, []), self.function) + bool_func = FuncDef("__bool__", [], Block([])) + bool_func.type = set_callable_name(signature, bool_func) + type_info.names[bool_func.name] = SymbolTableNode(MDEF, bool_func) + + # Helper methods + + def callable(self, *a: Type) -> CallableType: + """callable(a1, ..., an, r) constructs a callable with argument types + a1, ... an and return type r. + """ + return CallableType( + list(a[:-1]), [ARG_POS] * (len(a) - 1), [None] * (len(a) - 1), a[-1], self.function + ) + + def callable_type(self, *a: Type) -> CallableType: + """callable_type(a1, ..., an, r) constructs a callable with + argument types a1, ... an and return type r, and which + represents a type. + """ + return CallableType( + list(a[:-1]), [ARG_POS] * (len(a) - 1), [None] * (len(a) - 1), a[-1], self.type_type + ) + + def callable_default(self, min_args: int, *a: Type) -> CallableType: + """callable_default(min_args, a1, ..., an, r) constructs a + callable with argument types a1, ... an and return type r, + with min_args mandatory fixed arguments. + """ + n = len(a) - 1 + return CallableType( + list(a[:-1]), + [ARG_POS] * min_args + [ARG_OPT] * (n - min_args), + [None] * n, + a[-1], + self.function, + ) + + def callable_var_arg(self, min_args: int, *a: Type) -> CallableType: + """callable_var_arg(min_args, a1, ..., an, r) constructs a callable + with argument types a1, ... *an and return type r. + """ + n = len(a) - 1 + return CallableType( + list(a[:-1]), + [ARG_POS] * min_args + [ARG_OPT] * (n - 1 - min_args) + [ARG_STAR], + [None] * n, + a[-1], + self.function, + ) + + def make_type_info( + self, + name: str, + module_name: str | None = None, + is_abstract: bool = False, + mro: list[TypeInfo] | None = None, + bases: list[Instance] | None = None, + typevars: list[str] | None = None, + typevar_tuple_index: int | None = None, + variances: list[int] | None = None, + ) -> TypeInfo: + """Make a TypeInfo suitable for use in unit tests.""" + + class_def = ClassDef(name, Block([]), None, []) + class_def.fullname = name + + if module_name is None: + if "." in name: + module_name = name.rsplit(".", 1)[0] + else: + module_name = "__main__" + + if typevars: + v: list[TypeVarLikeType] = [] + for id, n in enumerate(typevars, 1): + if typevar_tuple_index is not None and id - 1 == typevar_tuple_index: + v.append( + TypeVarTupleType( + n, + n, + TypeVarId(id), + self.std_tuple.copy_modified(args=[self.o]), + self.std_tuple.copy_modified(args=[self.o]), + AnyType(TypeOfAny.from_omitted_generics), + ) + ) + else: + if variances: + variance = variances[id - 1] + else: + variance = COVARIANT + v.append( + TypeVarType( + n, + n, + TypeVarId(id), + [], + self.o, + AnyType(TypeOfAny.from_omitted_generics), + variance=variance, + ) + ) + class_def.type_vars = v + + info = TypeInfo(SymbolTable(), class_def, module_name) + if mro is None: + mro = [] + if name != "builtins.object": + mro.append(self.oi) + info.mro = [info] + mro + if bases is None: + if mro: + # By default, assume that there is a single non-generic base. + bases = [Instance(mro[0], [])] + else: + bases = [] + info.bases = bases + + return info + + def def_alias_1(self, base: Instance) -> tuple[TypeAliasType, Type]: + A = TypeAliasType(None, []) + target = Instance( + self.std_tuplei, [UnionType([base, A])] + ) # A = Tuple[Union[base, A], ...] + AN = TypeAlias(target, "__main__.A", -1, -1) + A.alias = AN + return A, target + + def def_alias_2(self, base: Instance) -> tuple[TypeAliasType, Type]: + A = TypeAliasType(None, []) + target = UnionType( + [base, Instance(self.std_tuplei, [A])] + ) # A = Union[base, Tuple[A, ...]] + AN = TypeAlias(target, "__main__.A", -1, -1) + A.alias = AN + return A, target + + def non_rec_alias( + self, + target: Type, + alias_tvars: list[TypeVarLikeType] | None = None, + args: list[Type] | None = None, + ) -> TypeAliasType: + AN = TypeAlias(target, "__main__.A", -1, -1, alias_tvars=alias_tvars) + if args is None: + args = [] + return TypeAliasType(AN, args) + + +class InterfaceTypeFixture(TypeFixture): + """Extension of TypeFixture that contains additional generic + interface types.""" + + def __init__(self) -> None: + super().__init__() + # GF[T] + self.gfi = self.make_type_info("GF", typevars=["T"], is_abstract=True) + + # M1 <: GF[A] + self.m1i = self.make_type_info( + "M1", is_abstract=True, mro=[self.gfi, self.oi], bases=[Instance(self.gfi, [self.a])] + ) + + self.gfa = Instance(self.gfi, [self.a]) # GF[A] + self.gfb = Instance(self.gfi, [self.b]) # GF[B] + + self.m1 = Instance(self.m1i, []) # M1 diff --git a/mypy/test/update_data.py b/mypy/test/update_data.py new file mode 100644 index 000000000000..2d66752f61bd --- /dev/null +++ b/mypy/test/update_data.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from typing import Iterator + +from mypy.test.data import DataDrivenTestCase, DataFileCollector, DataFileFix, parse_test_data + + +def update_testcase_output( + testcase: DataDrivenTestCase, actual: list[str], *, incremental_step: int +) -> None: + if testcase.xfail: + return + collector = testcase.parent + assert isinstance(collector, DataFileCollector) + for fix in _iter_fixes(testcase, actual, incremental_step=incremental_step): + collector.enqueue_fix(fix) + + +def _iter_fixes( + testcase: DataDrivenTestCase, actual: list[str], *, incremental_step: int +) -> Iterator[DataFileFix]: + reports_by_line: dict[tuple[str, int], list[tuple[str, str]]] = defaultdict(list) + for error_line in actual: + comment_match = re.match( + r"^(?P[^:]+):(?P\d+): (?Perror|note|warning): (?P.+)$", + error_line, + ) + if comment_match: + filename = comment_match.group("filename") + lineno = int(comment_match.group("lineno")) + severity = comment_match.group("severity") + msg = comment_match.group("msg") + reports_by_line[filename, lineno].append((severity, msg)) + + test_items = parse_test_data(testcase.data, testcase.name) + + # If we have [out] and/or [outN], we update just those sections. + if any(re.match(r"^out\d*$", test_item.id) for test_item in test_items): + for test_item in test_items: + if (incremental_step < 2 and test_item.id == "out") or ( + incremental_step >= 2 and test_item.id == f"out{incremental_step}" + ): + yield DataFileFix( + lineno=testcase.line + test_item.line - 1, + end_lineno=testcase.line + test_item.end_line - 1, + lines=actual + [""] * test_item.trimmed_newlines, + ) + + return + + # Update assertion comments within the sections + for test_item in test_items: + if test_item.id == "case": + source_lines = test_item.data + file_path = "main" + elif test_item.id == "file": + source_lines = test_item.data + file_path = f"tmp/{test_item.arg}" + else: + continue # other sections we don't touch + + fix_lines = [] + for lineno, source_line in enumerate(source_lines, start=1): + reports = reports_by_line.get((file_path, lineno)) + comment_match = re.search(r"(?P\s+)(?P# [EWN]: .+)$", source_line) + if comment_match: + source_line = source_line[: comment_match.start("indent")] # strip old comment + if reports: + indent = comment_match.group("indent") if comment_match else " " + # multiline comments are on the first line and then on subsequent lines emtpy lines + # with a continuation backslash + for j, (severity, msg) in enumerate(reports): + out_l = source_line if j == 0 else " " * len(source_line) + is_last = j == len(reports) - 1 + severity_char = severity[0].upper() + continuation = "" if is_last else " \\" + fix_lines.append(f"{out_l}{indent}# {severity_char}: {msg}{continuation}") + else: + fix_lines.append(source_line) + + yield DataFileFix( + lineno=testcase.line + test_item.line - 1, + end_lineno=testcase.line + test_item.end_line - 1, + lines=fix_lines + [""] * test_item.trimmed_newlines, + ) diff --git a/mypy/test/visitors.py b/mypy/test/visitors.py new file mode 100644 index 000000000000..2b748ec1bdc4 --- /dev/null +++ b/mypy/test/visitors.py @@ -0,0 +1,63 @@ +"""Visitor classes pulled out from different tests + +These are here because we don't currently support having interpreted +classes subtype compiled ones but pytest grabs the python file +even if the test was compiled. + +""" + +from __future__ import annotations + +from mypy.nodes import AssignmentStmt, CallExpr, Expression, IntExpr, NameExpr, Node, TypeVarExpr +from mypy.traverser import TraverserVisitor +from mypy.treetransform import TransformVisitor +from mypy.types import Type + + +# from testtypegen +class SkippedNodeSearcher(TraverserVisitor): + def __init__(self) -> None: + self.nodes: set[Node] = set() + self.ignore_file = False + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + if s.type or ignore_node(s.rvalue): + for lvalue in s.lvalues: + if isinstance(lvalue, NameExpr): + self.nodes.add(lvalue) + super().visit_assignment_stmt(s) + + def visit_name_expr(self, n: NameExpr) -> None: + if self.ignore_file: + self.nodes.add(n) + super().visit_name_expr(n) + + def visit_int_expr(self, n: IntExpr) -> None: + if self.ignore_file: + self.nodes.add(n) + super().visit_int_expr(n) + + +def ignore_node(node: Expression) -> bool: + """Return True if node is to be omitted from test case output.""" + + # We want to get rid of object() expressions in the typing module stub + # and also TypeVar(...) expressions. Since detecting whether a node comes + # from the typing module is not easy, we just to strip them all away. + if isinstance(node, TypeVarExpr): + return True + if isinstance(node, NameExpr) and node.fullname == "builtins.object": + return True + if isinstance(node, NameExpr) and node.fullname == "builtins.None": + return True + if isinstance(node, CallExpr) and (ignore_node(node.callee) or node.analyzed): + return True + + return False + + +# from testtransform +class TypeAssertTransformVisitor(TransformVisitor): + def type(self, type: Type) -> Type: + assert type is not None + return type diff --git a/mypy/traverser.py b/mypy/traverser.py index 62baba7cc39e..9c333c587f7c 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -1,117 +1,204 @@ """Generic node traverser visitor""" -from typing import typevar, Generic +from __future__ import annotations + +from mypy_extensions import mypyc_attr, trait -from mypy.visitor import NodeVisitor from mypy.nodes import ( - Block, MypyFile, VarDef, FuncItem, CallExpr, ClassDef, Decorator, FuncDef, - ExpressionStmt, AssignmentStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, ReturnStmt, AssertStmt, YieldStmt, DelStmt, IfStmt, RaiseStmt, - TryStmt, WithStmt, ParenExpr, MemberExpr, OpExpr, SliceExpr, CastExpr, - UnaryExpr, ListExpr, TupleExpr, DictExpr, SetExpr, IndexExpr, - GeneratorExpr, ListComprehension, ConditionalExpr, TypeApplication, - FuncExpr, ComparisonExpr, OverloadedFuncDef, YieldFromStmt, YieldFromExpr + REVEAL_TYPE, + AssertStmt, + AssertTypeExpr, + AssignmentExpr, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + BytesExpr, + CallExpr, + CastExpr, + ClassDef, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + ContinueStmt, + Decorator, + DelStmt, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, + Expression, + ExpressionStmt, + FloatExpr, + ForStmt, + FuncBase, + FuncDef, + FuncItem, + GeneratorExpr, + GlobalDecl, + IfStmt, + Import, + ImportAll, + ImportFrom, + IndexExpr, + IntExpr, + LambdaExpr, + ListComprehension, + ListExpr, + MatchStmt, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + NewTypeExpr, + Node, + NonlocalDecl, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + ParamSpecExpr, + PassStmt, + RaiseStmt, + ReturnStmt, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + StrExpr, + SuperExpr, + TryStmt, + TupleExpr, + TypeAlias, + TypeAliasExpr, + TypeAliasStmt, + TypeApplication, + TypedDictExpr, + TypeVarExpr, + TypeVarTupleExpr, + UnaryExpr, + WhileStmt, + WithStmt, + YieldExpr, + YieldFromExpr, ) +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, +) +from mypy.visitor import NodeVisitor -T = typevar('T') - - -class TraverserVisitor(NodeVisitor[T], Generic[T]): +@trait +@mypyc_attr(allow_interpreted_subclasses=True) +class TraverserVisitor(NodeVisitor[None]): """A parse tree visitor that traverses the parse tree during visiting. - It does not peform any actions outside the travelsal. Subclasses + It does not perform any actions outside the traversal. Subclasses should override visit methods to perform actions during - travelsal. Calling the superclass method allows reusing the - travelsal implementation. + traversal. Calling the superclass method allows reusing the + traversal implementation. """ + def __init__(self) -> None: + pass + # Visit methods - def visit_mypy_file(self, o: MypyFile) -> T: + def visit_mypy_file(self, o: MypyFile) -> None: for d in o.defs: d.accept(self) - def visit_block(self, block: Block) -> T: + def visit_block(self, block: Block) -> None: for s in block.body: s.accept(self) - def visit_func(self, o: FuncItem) -> T: - for i in o.init: - if i is not None: - i.accept(self) - for v in o.args: - self.visit_var(v) + def visit_func(self, o: FuncItem) -> None: + if o.arguments is not None: + for arg in o.arguments: + init = arg.initializer + if init is not None: + init.accept(self) + + for arg in o.arguments: + self.visit_var(arg.variable) + o.body.accept(self) - def visit_func_def(self, o: FuncDef) -> T: + def visit_func_def(self, o: FuncDef) -> None: self.visit_func(o) - def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> T: + def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: for item in o.items: item.accept(self) + if o.impl: + o.impl.accept(self) - def visit_class_def(self, o: ClassDef) -> T: + def visit_class_def(self, o: ClassDef) -> None: + for d in o.decorators: + d.accept(self) + for base in o.base_type_exprs: + base.accept(self) + if o.metaclass: + o.metaclass.accept(self) + for v in o.keywords.values(): + v.accept(self) o.defs.accept(self) + if o.analyzed: + o.analyzed.accept(self) - def visit_decorator(self, o: Decorator) -> T: + def visit_decorator(self, o: Decorator) -> None: o.func.accept(self) o.var.accept(self) for decorator in o.decorators: decorator.accept(self) - def visit_var_def(self, o: VarDef) -> T: - if o.init is not None: - o.init.accept(self) - for v in o.items: - self.visit_var(v) - - def visit_expression_stmt(self, o: ExpressionStmt) -> T: + def visit_expression_stmt(self, o: ExpressionStmt) -> None: o.expr.accept(self) - def visit_assignment_stmt(self, o: AssignmentStmt) -> T: + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: o.rvalue.accept(self) for l in o.lvalues: l.accept(self) - def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> T: + def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: o.rvalue.accept(self) o.lvalue.accept(self) - def visit_while_stmt(self, o: WhileStmt) -> T: + def visit_while_stmt(self, o: WhileStmt) -> None: o.expr.accept(self) o.body.accept(self) if o.else_body: o.else_body.accept(self) - def visit_for_stmt(self, o: ForStmt) -> T: + def visit_for_stmt(self, o: ForStmt) -> None: o.index.accept(self) o.expr.accept(self) o.body.accept(self) if o.else_body: o.else_body.accept(self) - def visit_return_stmt(self, o: ReturnStmt) -> T: - if o.expr is not None: - o.expr.accept(self) - - def visit_assert_stmt(self, o: AssertStmt) -> T: - if o.expr is not None: - o.expr.accept(self) - - def visit_yield_stmt(self, o: YieldStmt) -> T: + def visit_return_stmt(self, o: ReturnStmt) -> None: if o.expr is not None: o.expr.accept(self) - def visit_yield_from_stmt(self, o: YieldFromStmt) -> T: + def visit_assert_stmt(self, o: AssertStmt) -> None: if o.expr is not None: o.expr.accept(self) + if o.msg is not None: + o.msg.accept(self) - def visit_del_stmt(self, o: DelStmt) -> T: + def visit_del_stmt(self, o: DelStmt) -> None: if o.expr is not None: o.expr.accept(self) - def visit_if_stmt(self, o: IfStmt) -> T: + def visit_if_stmt(self, o: IfStmt) -> None: for e in o.expr: e.accept(self) for b in o.body: @@ -119,55 +206,76 @@ def visit_if_stmt(self, o: IfStmt) -> T: if o.else_body: o.else_body.accept(self) - def visit_raise_stmt(self, o: RaiseStmt) -> T: + def visit_raise_stmt(self, o: RaiseStmt) -> None: if o.expr is not None: o.expr.accept(self) if o.from_expr is not None: o.from_expr.accept(self) - def visit_try_stmt(self, o: TryStmt) -> T: + def visit_try_stmt(self, o: TryStmt) -> None: o.body.accept(self) for i in range(len(o.types)): - if o.types[i]: - o.types[i].accept(self) + tp = o.types[i] + if tp is not None: + tp.accept(self) o.handlers[i].accept(self) + for v in o.vars: + if v is not None: + v.accept(self) if o.else_body is not None: o.else_body.accept(self) if o.finally_body is not None: o.finally_body.accept(self) - def visit_with_stmt(self, o: WithStmt) -> T: + def visit_with_stmt(self, o: WithStmt) -> None: for i in range(len(o.expr)): o.expr[i].accept(self) - if o.name[i] is not None: - o.name[i].accept(self) + targ = o.target[i] + if targ is not None: + targ.accept(self) o.body.accept(self) - def visit_paren_expr(self, o: ParenExpr) -> T: - o.expr.accept(self) + def visit_match_stmt(self, o: MatchStmt) -> None: + o.subject.accept(self) + for i in range(len(o.patterns)): + o.patterns[i].accept(self) + guard = o.guards[i] + if guard is not None: + guard.accept(self) + o.bodies[i].accept(self) - def visit_member_expr(self, o: MemberExpr) -> T: + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + o.name.accept(self) + o.value.accept(self) + + def visit_member_expr(self, o: MemberExpr) -> None: o.expr.accept(self) - def visit_yield_from_expr(self, o: YieldFromExpr) -> T: + def visit_yield_from_expr(self, o: YieldFromExpr) -> None: o.expr.accept(self) - def visit_call_expr(self, o: CallExpr) -> T: + def visit_yield_expr(self, o: YieldExpr) -> None: + if o.expr: + o.expr.accept(self) + + def visit_call_expr(self, o: CallExpr) -> None: + o.callee.accept(self) for a in o.args: a.accept(self) - o.callee.accept(self) if o.analyzed: o.analyzed.accept(self) - def visit_op_expr(self, o: OpExpr) -> T: + def visit_op_expr(self, o: OpExpr) -> None: o.left.accept(self) o.right.accept(self) + if o.analyzed is not None: + o.analyzed.accept(self) - def visit_comparison_expr(self, o: ComparisonExpr) -> T: + def visit_comparison_expr(self, o: ComparisonExpr) -> None: for operand in o.operands: operand.accept(self) - def visit_slice_expr(self, o: SliceExpr) -> T: + def visit_slice_expr(self, o: SliceExpr) -> None: if o.begin_index is not None: o.begin_index.accept(self) if o.end_index is not None: @@ -175,54 +283,678 @@ def visit_slice_expr(self, o: SliceExpr) -> T: if o.stride is not None: o.stride.accept(self) - def visit_cast_expr(self, o: CastExpr) -> T: + def visit_cast_expr(self, o: CastExpr) -> None: o.expr.accept(self) - def visit_unary_expr(self, o: UnaryExpr) -> T: + def visit_assert_type_expr(self, o: AssertTypeExpr) -> None: + o.expr.accept(self) + + def visit_reveal_expr(self, o: RevealExpr) -> None: + if o.kind == REVEAL_TYPE: + assert o.expr is not None + o.expr.accept(self) + else: + # RevealLocalsExpr doesn't have an inner expression + pass + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + o.target.accept(self) + o.value.accept(self) + + def visit_unary_expr(self, o: UnaryExpr) -> None: o.expr.accept(self) - def visit_list_expr(self, o: ListExpr) -> T: + def visit_list_expr(self, o: ListExpr) -> None: for item in o.items: item.accept(self) - def visit_tuple_expr(self, o: TupleExpr) -> T: + def visit_tuple_expr(self, o: TupleExpr) -> None: for item in o.items: item.accept(self) - def visit_dict_expr(self, o: DictExpr) -> T: + def visit_dict_expr(self, o: DictExpr) -> None: for k, v in o.items: - k.accept(self) + if k is not None: + k.accept(self) v.accept(self) - def visit_set_expr(self, o: SetExpr) -> T: + def visit_set_expr(self, o: SetExpr) -> None: for item in o.items: item.accept(self) - def visit_index_expr(self, o: IndexExpr) -> T: + def visit_index_expr(self, o: IndexExpr) -> None: o.base.accept(self) o.index.accept(self) if o.analyzed: o.analyzed.accept(self) - def visit_generator_expr(self, o: GeneratorExpr) -> T: - for index, sequence, conditions in zip(o.indices, o.sequences, - o.condlists): + def visit_generator_expr(self, o: GeneratorExpr) -> None: + for index, sequence, conditions in zip(o.indices, o.sequences, o.condlists): sequence.accept(self) index.accept(self) for cond in conditions: cond.accept(self) o.left_expr.accept(self) - def visit_list_comprehension(self, o: ListComprehension) -> T: + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + for index, sequence, conditions in zip(o.indices, o.sequences, o.condlists): + sequence.accept(self) + index.accept(self) + for cond in conditions: + cond.accept(self) + o.key.accept(self) + o.value.accept(self) + + def visit_list_comprehension(self, o: ListComprehension) -> None: + o.generator.accept(self) + + def visit_set_comprehension(self, o: SetComprehension) -> None: o.generator.accept(self) - def visit_conditional_expr(self, o: ConditionalExpr) -> T: + def visit_conditional_expr(self, o: ConditionalExpr) -> None: o.cond.accept(self) o.if_expr.accept(self) o.else_expr.accept(self) - def visit_type_application(self, o: TypeApplication) -> T: + def visit_type_application(self, o: TypeApplication) -> None: o.expr.accept(self) - def visit_func_expr(self, o: FuncExpr) -> T: + def visit_lambda_expr(self, o: LambdaExpr) -> None: self.visit_func(o) + + def visit_star_expr(self, o: StarExpr) -> None: + o.expr.accept(self) + + def visit_await_expr(self, o: AwaitExpr) -> None: + o.expr.accept(self) + + def visit_super_expr(self, o: SuperExpr) -> None: + o.call.accept(self) + + def visit_as_pattern(self, o: AsPattern) -> None: + if o.pattern is not None: + o.pattern.accept(self) + if o.name is not None: + o.name.accept(self) + + def visit_or_pattern(self, o: OrPattern) -> None: + for p in o.patterns: + p.accept(self) + + def visit_value_pattern(self, o: ValuePattern) -> None: + o.expr.accept(self) + + def visit_sequence_pattern(self, o: SequencePattern) -> None: + for p in o.patterns: + p.accept(self) + + def visit_starred_pattern(self, o: StarredPattern) -> None: + if o.capture is not None: + o.capture.accept(self) + + def visit_mapping_pattern(self, o: MappingPattern) -> None: + for key in o.keys: + key.accept(self) + for value in o.values: + value.accept(self) + if o.rest is not None: + o.rest.accept(self) + + def visit_class_pattern(self, o: ClassPattern) -> None: + o.class_ref.accept(self) + for p in o.positionals: + p.accept(self) + for v in o.keyword_values: + v.accept(self) + + def visit_import(self, o: Import) -> None: + for a in o.assignments: + a.accept(self) + + def visit_import_from(self, o: ImportFrom) -> None: + for a in o.assignments: + a.accept(self) + + +class ExtendedTraverserVisitor(TraverserVisitor): + """This is a more flexible traverser. + + In addition to the base traverser it: + * has visit_ methods for leaf nodes + * has common method that is called for all nodes + * allows to skip recursing into a node + + Note that this traverser still doesn't visit some internal + mypy constructs like _promote expression and Var. + """ + + def visit(self, o: Node) -> bool: + # If returns True, will continue to nested nodes. + return True + + def visit_mypy_file(self, o: MypyFile) -> None: + if not self.visit(o): + return + super().visit_mypy_file(o) + + # Module structure + + def visit_import(self, o: Import) -> None: + if not self.visit(o): + return + super().visit_import(o) + + def visit_import_from(self, o: ImportFrom) -> None: + if not self.visit(o): + return + super().visit_import_from(o) + + def visit_import_all(self, o: ImportAll) -> None: + if not self.visit(o): + return + super().visit_import_all(o) + + # Definitions + + def visit_func_def(self, o: FuncDef) -> None: + if not self.visit(o): + return + super().visit_func_def(o) + + def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: + if not self.visit(o): + return + super().visit_overloaded_func_def(o) + + def visit_class_def(self, o: ClassDef) -> None: + if not self.visit(o): + return + super().visit_class_def(o) + + def visit_global_decl(self, o: GlobalDecl) -> None: + if not self.visit(o): + return + super().visit_global_decl(o) + + def visit_nonlocal_decl(self, o: NonlocalDecl) -> None: + if not self.visit(o): + return + super().visit_nonlocal_decl(o) + + def visit_decorator(self, o: Decorator) -> None: + if not self.visit(o): + return + super().visit_decorator(o) + + def visit_type_alias(self, o: TypeAlias) -> None: + if not self.visit(o): + return + super().visit_type_alias(o) + + # Statements + + def visit_block(self, block: Block) -> None: + if not self.visit(block): + return + super().visit_block(block) + + def visit_expression_stmt(self, o: ExpressionStmt) -> None: + if not self.visit(o): + return + super().visit_expression_stmt(o) + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + if not self.visit(o): + return + super().visit_assignment_stmt(o) + + def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: + if not self.visit(o): + return + super().visit_operator_assignment_stmt(o) + + def visit_while_stmt(self, o: WhileStmt) -> None: + if not self.visit(o): + return + super().visit_while_stmt(o) + + def visit_for_stmt(self, o: ForStmt) -> None: + if not self.visit(o): + return + super().visit_for_stmt(o) + + def visit_return_stmt(self, o: ReturnStmt) -> None: + if not self.visit(o): + return + super().visit_return_stmt(o) + + def visit_assert_stmt(self, o: AssertStmt) -> None: + if not self.visit(o): + return + super().visit_assert_stmt(o) + + def visit_del_stmt(self, o: DelStmt) -> None: + if not self.visit(o): + return + super().visit_del_stmt(o) + + def visit_if_stmt(self, o: IfStmt) -> None: + if not self.visit(o): + return + super().visit_if_stmt(o) + + def visit_break_stmt(self, o: BreakStmt) -> None: + if not self.visit(o): + return + super().visit_break_stmt(o) + + def visit_continue_stmt(self, o: ContinueStmt) -> None: + if not self.visit(o): + return + super().visit_continue_stmt(o) + + def visit_pass_stmt(self, o: PassStmt) -> None: + if not self.visit(o): + return + super().visit_pass_stmt(o) + + def visit_raise_stmt(self, o: RaiseStmt) -> None: + if not self.visit(o): + return + super().visit_raise_stmt(o) + + def visit_try_stmt(self, o: TryStmt) -> None: + if not self.visit(o): + return + super().visit_try_stmt(o) + + def visit_with_stmt(self, o: WithStmt) -> None: + if not self.visit(o): + return + super().visit_with_stmt(o) + + def visit_match_stmt(self, o: MatchStmt) -> None: + if not self.visit(o): + return + super().visit_match_stmt(o) + + # Expressions (default no-op implementation) + + def visit_int_expr(self, o: IntExpr) -> None: + if not self.visit(o): + return + super().visit_int_expr(o) + + def visit_str_expr(self, o: StrExpr) -> None: + if not self.visit(o): + return + super().visit_str_expr(o) + + def visit_bytes_expr(self, o: BytesExpr) -> None: + if not self.visit(o): + return + super().visit_bytes_expr(o) + + def visit_float_expr(self, o: FloatExpr) -> None: + if not self.visit(o): + return + super().visit_float_expr(o) + + def visit_complex_expr(self, o: ComplexExpr) -> None: + if not self.visit(o): + return + super().visit_complex_expr(o) + + def visit_ellipsis(self, o: EllipsisExpr) -> None: + if not self.visit(o): + return + super().visit_ellipsis(o) + + def visit_star_expr(self, o: StarExpr) -> None: + if not self.visit(o): + return + super().visit_star_expr(o) + + def visit_name_expr(self, o: NameExpr) -> None: + if not self.visit(o): + return + super().visit_name_expr(o) + + def visit_member_expr(self, o: MemberExpr) -> None: + if not self.visit(o): + return + super().visit_member_expr(o) + + def visit_yield_from_expr(self, o: YieldFromExpr) -> None: + if not self.visit(o): + return + super().visit_yield_from_expr(o) + + def visit_yield_expr(self, o: YieldExpr) -> None: + if not self.visit(o): + return + super().visit_yield_expr(o) + + def visit_call_expr(self, o: CallExpr) -> None: + if not self.visit(o): + return + super().visit_call_expr(o) + + def visit_op_expr(self, o: OpExpr) -> None: + if not self.visit(o): + return + super().visit_op_expr(o) + + def visit_comparison_expr(self, o: ComparisonExpr) -> None: + if not self.visit(o): + return + super().visit_comparison_expr(o) + + def visit_cast_expr(self, o: CastExpr) -> None: + if not self.visit(o): + return + super().visit_cast_expr(o) + + def visit_assert_type_expr(self, o: AssertTypeExpr) -> None: + if not self.visit(o): + return + super().visit_assert_type_expr(o) + + def visit_reveal_expr(self, o: RevealExpr) -> None: + if not self.visit(o): + return + super().visit_reveal_expr(o) + + def visit_super_expr(self, o: SuperExpr) -> None: + if not self.visit(o): + return + super().visit_super_expr(o) + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + if not self.visit(o): + return + super().visit_assignment_expr(o) + + def visit_unary_expr(self, o: UnaryExpr) -> None: + if not self.visit(o): + return + super().visit_unary_expr(o) + + def visit_list_expr(self, o: ListExpr) -> None: + if not self.visit(o): + return + super().visit_list_expr(o) + + def visit_dict_expr(self, o: DictExpr) -> None: + if not self.visit(o): + return + super().visit_dict_expr(o) + + def visit_tuple_expr(self, o: TupleExpr) -> None: + if not self.visit(o): + return + super().visit_tuple_expr(o) + + def visit_set_expr(self, o: SetExpr) -> None: + if not self.visit(o): + return + super().visit_set_expr(o) + + def visit_index_expr(self, o: IndexExpr) -> None: + if not self.visit(o): + return + super().visit_index_expr(o) + + def visit_type_application(self, o: TypeApplication) -> None: + if not self.visit(o): + return + super().visit_type_application(o) + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + if not self.visit(o): + return + super().visit_lambda_expr(o) + + def visit_list_comprehension(self, o: ListComprehension) -> None: + if not self.visit(o): + return + super().visit_list_comprehension(o) + + def visit_set_comprehension(self, o: SetComprehension) -> None: + if not self.visit(o): + return + super().visit_set_comprehension(o) + + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + if not self.visit(o): + return + super().visit_dictionary_comprehension(o) + + def visit_generator_expr(self, o: GeneratorExpr) -> None: + if not self.visit(o): + return + super().visit_generator_expr(o) + + def visit_slice_expr(self, o: SliceExpr) -> None: + if not self.visit(o): + return + super().visit_slice_expr(o) + + def visit_conditional_expr(self, o: ConditionalExpr) -> None: + if not self.visit(o): + return + super().visit_conditional_expr(o) + + def visit_type_var_expr(self, o: TypeVarExpr) -> None: + if not self.visit(o): + return + super().visit_type_var_expr(o) + + def visit_paramspec_expr(self, o: ParamSpecExpr) -> None: + if not self.visit(o): + return + super().visit_paramspec_expr(o) + + def visit_type_var_tuple_expr(self, o: TypeVarTupleExpr) -> None: + if not self.visit(o): + return + super().visit_type_var_tuple_expr(o) + + def visit_type_alias_expr(self, o: TypeAliasExpr) -> None: + if not self.visit(o): + return + super().visit_type_alias_expr(o) + + def visit_namedtuple_expr(self, o: NamedTupleExpr) -> None: + if not self.visit(o): + return + super().visit_namedtuple_expr(o) + + def visit_enum_call_expr(self, o: EnumCallExpr) -> None: + if not self.visit(o): + return + super().visit_enum_call_expr(o) + + def visit_typeddict_expr(self, o: TypedDictExpr) -> None: + if not self.visit(o): + return + super().visit_typeddict_expr(o) + + def visit_newtype_expr(self, o: NewTypeExpr) -> None: + if not self.visit(o): + return + super().visit_newtype_expr(o) + + def visit_await_expr(self, o: AwaitExpr) -> None: + if not self.visit(o): + return + super().visit_await_expr(o) + + # Patterns + + def visit_as_pattern(self, o: AsPattern) -> None: + if not self.visit(o): + return + super().visit_as_pattern(o) + + def visit_or_pattern(self, o: OrPattern) -> None: + if not self.visit(o): + return + super().visit_or_pattern(o) + + def visit_value_pattern(self, o: ValuePattern) -> None: + if not self.visit(o): + return + super().visit_value_pattern(o) + + def visit_singleton_pattern(self, o: SingletonPattern) -> None: + if not self.visit(o): + return + super().visit_singleton_pattern(o) + + def visit_sequence_pattern(self, o: SequencePattern) -> None: + if not self.visit(o): + return + super().visit_sequence_pattern(o) + + def visit_starred_pattern(self, o: StarredPattern) -> None: + if not self.visit(o): + return + super().visit_starred_pattern(o) + + def visit_mapping_pattern(self, o: MappingPattern) -> None: + if not self.visit(o): + return + super().visit_mapping_pattern(o) + + def visit_class_pattern(self, o: ClassPattern) -> None: + if not self.visit(o): + return + super().visit_class_pattern(o) + + +class ReturnSeeker(TraverserVisitor): + def __init__(self) -> None: + self.found = False + + def visit_return_stmt(self, o: ReturnStmt) -> None: + if o.expr is None or isinstance(o.expr, NameExpr) and o.expr.name == "None": + return + self.found = True + + +def has_return_statement(fdef: FuncBase) -> bool: + """Find if a function has a non-trivial return statement. + + Plain 'return' and 'return None' don't count. + """ + seeker = ReturnSeeker() + fdef.accept(seeker) + return seeker.found + + +class FuncCollectorBase(TraverserVisitor): + def __init__(self) -> None: + self.inside_func = False + + def visit_func_def(self, defn: FuncDef) -> None: + if not self.inside_func: + self.inside_func = True + super().visit_func_def(defn) + self.inside_func = False + + +class YieldSeeker(FuncCollectorBase): + def __init__(self) -> None: + super().__init__() + self.found = False + + def visit_yield_expr(self, o: YieldExpr) -> None: + self.found = True + + +def has_yield_expression(fdef: FuncBase) -> bool: + seeker = YieldSeeker() + fdef.accept(seeker) + return seeker.found + + +class YieldFromSeeker(FuncCollectorBase): + def __init__(self) -> None: + super().__init__() + self.found = False + + def visit_yield_from_expr(self, o: YieldFromExpr) -> None: + self.found = True + + +def has_yield_from_expression(fdef: FuncBase) -> bool: + seeker = YieldFromSeeker() + fdef.accept(seeker) + return seeker.found + + +class AwaitSeeker(TraverserVisitor): + def __init__(self) -> None: + super().__init__() + self.found = False + + def visit_await_expr(self, o: AwaitExpr) -> None: + self.found = True + + +def has_await_expression(expr: Expression) -> bool: + seeker = AwaitSeeker() + expr.accept(seeker) + return seeker.found + + +class ReturnCollector(FuncCollectorBase): + def __init__(self) -> None: + super().__init__() + self.return_statements: list[ReturnStmt] = [] + + def visit_return_stmt(self, stmt: ReturnStmt) -> None: + self.return_statements.append(stmt) + + +def all_return_statements(node: Node) -> list[ReturnStmt]: + v = ReturnCollector() + node.accept(v) + return v.return_statements + + +class YieldCollector(FuncCollectorBase): + def __init__(self) -> None: + super().__init__() + self.in_assignment = False + self.yield_expressions: list[tuple[YieldExpr, bool]] = [] + + def visit_assignment_stmt(self, stmt: AssignmentStmt) -> None: + self.in_assignment = True + super().visit_assignment_stmt(stmt) + self.in_assignment = False + + def visit_yield_expr(self, expr: YieldExpr) -> None: + self.yield_expressions.append((expr, self.in_assignment)) + + +def all_yield_expressions(node: Node) -> list[tuple[YieldExpr, bool]]: + v = YieldCollector() + node.accept(v) + return v.yield_expressions + + +class YieldFromCollector(FuncCollectorBase): + def __init__(self) -> None: + super().__init__() + self.in_assignment = False + self.yield_from_expressions: list[tuple[YieldFromExpr, bool]] = [] + + def visit_assignment_stmt(self, stmt: AssignmentStmt) -> None: + self.in_assignment = True + super().visit_assignment_stmt(stmt) + self.in_assignment = False + + def visit_yield_from_expr(self, expr: YieldFromExpr) -> None: + self.yield_from_expressions.append((expr, self.in_assignment)) diff --git a/mypy/treetransform.py b/mypy/treetransform.py index ba981bf7a087..aafa4e95d530 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -3,24 +3,108 @@ Subclass TransformVisitor to perform non-trivial transformations. """ -from typing import List, Dict, cast +from __future__ import annotations + +from typing import Iterable, Optional, cast from mypy.nodes import ( - MypyFile, Import, Node, ImportAll, ImportFrom, FuncItem, FuncDef, - OverloadedFuncDef, ClassDef, Decorator, Block, Var, VarDef, - OperatorAssignmentStmt, ExpressionStmt, AssignmentStmt, ReturnStmt, - RaiseStmt, AssertStmt, YieldStmt, DelStmt, BreakStmt, ContinueStmt, - PassStmt, GlobalDecl, WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, - CastExpr, ParenExpr, TupleExpr, GeneratorExpr, ListComprehension, ListExpr, - ConditionalExpr, DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, - UnicodeExpr, FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, - SliceExpr, OpExpr, UnaryExpr, FuncExpr, TypeApplication, PrintStmt, - SymbolTable, RefExpr, UndefinedExpr, TypeVarExpr, DucktypeExpr, - DisjointclassExpr, ComparisonExpr, TempNode, StarExpr, YieldFromStmt, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, - DictionaryComprehension, ComplexExpr + GDEF, + REVEAL_TYPE, + Argument, + AssertStmt, + AssertTypeExpr, + AssignmentExpr, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + BytesExpr, + CallExpr, + CastExpr, + ClassDef, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + ContinueStmt, + Decorator, + DelStmt, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, + Expression, + ExpressionStmt, + FloatExpr, + ForStmt, + FuncDef, + FuncItem, + GeneratorExpr, + GlobalDecl, + IfStmt, + Import, + ImportAll, + ImportFrom, + IndexExpr, + IntExpr, + LambdaExpr, + ListComprehension, + ListExpr, + MatchStmt, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + NewTypeExpr, + Node, + NonlocalDecl, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + OverloadPart, + ParamSpecExpr, + PassStmt, + PromoteExpr, + RaiseStmt, + RefExpr, + ReturnStmt, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + Statement, + StrExpr, + SuperExpr, + SymbolTable, + TempNode, + TryStmt, + TupleExpr, + TypeAliasExpr, + TypeApplication, + TypedDictExpr, + TypeVarExpr, + TypeVarTupleExpr, + UnaryExpr, + Var, + WhileStmt, + WithStmt, + YieldExpr, + YieldFromExpr, +) +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + Pattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, ) -from mypy.types import Type, FunctionLike +from mypy.traverser import TraverserVisitor +from mypy.types import FunctionLike, ProperType, Type +from mypy.util import replace_object_state from mypy.visitor import NodeVisitor @@ -33,10 +117,12 @@ class TransformVisitor(NodeVisitor[Node]): Notes: + * This can only be used to transform functions or classes, not top-level + statements, and/or modules as a whole. * Do not duplicate TypeInfo nodes. This would generally not be desirable. * Only update some name binding cross-references, but only those that - refer to Var nodes, not those targeting ClassDef, TypeInfo or FuncDef - nodes. + refer to Var, Decorator or FuncDef nodes, not those targeting ClassDef or + TypeInfo nodes. * Types are not transformed, but you can override type() to also perform type transformation. @@ -44,124 +130,159 @@ class TransformVisitor(NodeVisitor[Node]): """ def __init__(self) -> None: + # To simplify testing, set this flag to True if you want to transform + # all statements in a file (this is prohibited in normal mode). + self.test_only = False # There may be multiple references to a Var node. Keep track of # Var translations using a dictionary. - self.var_map = Dict[Var, Var]() - - def visit_mypy_file(self, node: MypyFile) -> Node: + self.var_map: dict[Var, Var] = {} + # These are uninitialized placeholder nodes used temporarily for nested + # functions while we are transforming a top-level function. This maps an + # untransformed node to a placeholder (which will later become the + # transformed node). + self.func_placeholder_map: dict[FuncDef, FuncDef] = {} + + def visit_mypy_file(self, node: MypyFile) -> MypyFile: + assert self.test_only, "This visitor should not be used for whole files." # NOTE: The 'names' and 'imports' instance variables will be empty! - new = MypyFile(self.nodes(node.defs), [], node.is_bom) - new._name = node._name + ignored_lines = {line: codes.copy() for line, codes in node.ignored_lines.items()} + new = MypyFile(self.statements(node.defs), [], node.is_bom, ignored_lines=ignored_lines) new._fullname = node._fullname new.path = node.path new.names = SymbolTable() return new - def visit_import(self, node: Import) -> Node: - return Import(node.ids[:]) + def visit_import(self, node: Import) -> Import: + return Import(node.ids.copy()) - def visit_import_from(self, node: ImportFrom) -> Node: - return ImportFrom(node.id, node.relative, node.names[:]) + def visit_import_from(self, node: ImportFrom) -> ImportFrom: + return ImportFrom(node.id, node.relative, node.names.copy()) - def visit_import_all(self, node: ImportAll) -> Node: + def visit_import_all(self, node: ImportAll) -> ImportAll: return ImportAll(node.id, node.relative) + def copy_argument(self, argument: Argument) -> Argument: + arg = Argument( + self.visit_var(argument.variable), + argument.type_annotation, + argument.initializer, + argument.kind, + ) + + # Refresh lines of the inner things + arg.set_line(argument) + + return arg + def visit_func_def(self, node: FuncDef) -> FuncDef: # Note that a FuncDef must be transformed to a FuncDef. - new = FuncDef(node.name(), - [self.visit_var(var) for var in node.args], - node.arg_kinds[:], - [None] * len(node.init), - self.block(node.body), - cast(FunctionLike, self.optional_type(node.type))) + + # These contortions are needed to handle the case of recursive + # references inside the function being transformed. + # Set up placeholder nodes for references within this function + # to other functions defined inside it. + # Don't create an entry for this function itself though, + # since we want self-references to point to the original + # function if this is the top-level node we are transforming. + init = FuncMapInitializer(self) + for stmt in node.body.body: + stmt.accept(init) + + new = FuncDef( + node.name, + [self.copy_argument(arg) for arg in node.arguments], + self.block(node.body), + cast(Optional[FunctionLike], self.optional_type(node.type)), + ) self.copy_function_attributes(new, node) new._fullname = node._fullname new.is_decorated = node.is_decorated new.is_conditional = node.is_conditional - new.is_abstract = node.is_abstract + new.abstract_status = node.abstract_status new.is_static = node.is_static new.is_class = node.is_class new.is_property = node.is_property + new.is_final = node.is_final new.original_def = node.original_def - return new - def visit_func_expr(self, node: FuncExpr) -> Node: - new = FuncExpr([self.visit_var(var) for var in node.args], - node.arg_kinds[:], - [None] * len(node.init), - self.block(node.body), - cast(FunctionLike, self.optional_type(node.type))) + if node in self.func_placeholder_map: + # There is a placeholder definition for this function. Replace + # the attributes of the placeholder with those form the transformed + # function. We know that the classes will be identical (otherwise + # this wouldn't work). + result = self.func_placeholder_map[node] + replace_object_state(result, new) + return result + else: + return new + + def visit_lambda_expr(self, node: LambdaExpr) -> LambdaExpr: + new = LambdaExpr( + [self.copy_argument(arg) for arg in node.arguments], + self.block(node.body), + cast(Optional[FunctionLike], self.optional_type(node.type)), + ) self.copy_function_attributes(new, node) return new - def copy_function_attributes(self, new: FuncItem, - original: FuncItem) -> None: + def copy_function_attributes(self, new: FuncItem, original: FuncItem) -> None: new.info = original.info new.min_args = original.min_args new.max_pos = original.max_pos - new.is_implicit = original.is_implicit new.is_overload = original.is_overload new.is_generator = original.is_generator - new.init = self.duplicate_inits(original.init) - - def duplicate_inits(self, - inits: List[AssignmentStmt]) -> List[AssignmentStmt]: - result = List[AssignmentStmt]() - for init in inits: - if init: - result.append(self.duplicate_assignment(init)) - else: - result.append(None) - return result + new.is_coroutine = original.is_coroutine + new.is_async_generator = original.is_async_generator + new.is_awaitable_coroutine = original.is_awaitable_coroutine + new.line = original.line - def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> Node: - items = [self.visit_decorator(decorator) - for decorator in node.items] + def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> OverloadedFuncDef: + items = [cast(OverloadPart, item.accept(self)) for item in node.items] for newitem, olditem in zip(items, node.items): newitem.line = olditem.line new = OverloadedFuncDef(items) new._fullname = node._fullname - new.type = self.type(node.type) + new_type = self.optional_type(node.type) + assert isinstance(new_type, ProperType) + new.type = new_type new.info = node.info + new.is_static = node.is_static + new.is_class = node.is_class + new.is_property = node.is_property + new.is_final = node.is_final + if node.impl: + new.impl = cast(OverloadPart, node.impl.accept(self)) return new - def visit_class_def(self, node: ClassDef) -> Node: - new = ClassDef(node.name, - self.block(node.defs), - node.type_vars, - self.types(node.base_types), - node.metaclass) + def visit_class_def(self, node: ClassDef) -> ClassDef: + new = ClassDef( + node.name, + self.block(node.defs), + node.type_vars, + self.expressions(node.base_type_exprs), + self.optional_expr(node.metaclass), + ) new.fullname = node.fullname new.info = node.info - new.decorators = [decorator.accept(self) - for decorator in node.decorators] - new.is_builtinclass = node.is_builtinclass - return new - - def visit_var_def(self, node: VarDef) -> Node: - new = VarDef([self.visit_var(var) for var in node.items], - node.is_top_level, - self.optional_node(node.init)) - new.kind = node.kind + new.decorators = [self.expr(decorator) for decorator in node.decorators] return new - def visit_global_decl(self, node: GlobalDecl) -> Node: - return GlobalDecl(node.names[:]) + def visit_global_decl(self, node: GlobalDecl) -> GlobalDecl: + return GlobalDecl(node.names.copy()) - def visit_nonlocal_decl(self, node: NonlocalDecl) -> Node: - return NonlocalDecl(node.names[:]) + def visit_nonlocal_decl(self, node: NonlocalDecl) -> NonlocalDecl: + return NonlocalDecl(node.names.copy()) def visit_block(self, node: Block) -> Block: - return Block(self.nodes(node.body)) + return Block(self.statements(node.body), is_unreachable=node.is_unreachable) def visit_decorator(self, node: Decorator) -> Decorator: # Note that a Decorator must be transformed to a Decorator. func = self.visit_func_def(node.func) func.line = node.func.line - new = Decorator(func, self.nodes(node.decorators), - self.visit_var(node.var)) + new = Decorator(func, self.expressions(node.decorators), self.visit_var(node.var)) new.is_overload = node.is_overload return new @@ -169,7 +290,7 @@ def visit_var(self, node: Var) -> Var: # Note that a Var must be transformed to a Var. if node in self.var_map: return self.var_map[node] - new = Var(node.name(), self.optional_type(node.type)) + new = Var(node.name, self.optional_type(node.type)) new.line = node.line new._fullname = node._fullname new.info = node.info @@ -179,130 +300,188 @@ def visit_var(self, node: Var) -> Var: new.is_staticmethod = node.is_staticmethod new.is_classmethod = node.is_classmethod new.is_property = node.is_property - new.set_line(node.line) + new.is_final = node.is_final + new.final_value = node.final_value + new.final_unset_in_class = node.final_unset_in_class + new.final_set_in_init = node.final_set_in_init + new.set_line(node) self.var_map[node] = new return new - def visit_expression_stmt(self, node: ExpressionStmt) -> Node: - return ExpressionStmt(self.node(node.expr)) + def visit_expression_stmt(self, node: ExpressionStmt) -> ExpressionStmt: + return ExpressionStmt(self.expr(node.expr)) - def visit_assignment_stmt(self, node: AssignmentStmt) -> Node: + def visit_assignment_stmt(self, node: AssignmentStmt) -> AssignmentStmt: return self.duplicate_assignment(node) def duplicate_assignment(self, node: AssignmentStmt) -> AssignmentStmt: - new = AssignmentStmt(self.nodes(node.lvalues), - self.node(node.rvalue), - self.optional_type(node.type)) + new = AssignmentStmt( + self.expressions(node.lvalues), + self.expr(node.rvalue), + self.optional_type(node.unanalyzed_type), + ) new.line = node.line + new.is_final_def = node.is_final_def + new.type = self.optional_type(node.type) return new - def visit_operator_assignment_stmt(self, - node: OperatorAssignmentStmt) -> Node: - return OperatorAssignmentStmt(node.op, - self.node(node.lvalue), - self.node(node.rvalue)) - - def visit_while_stmt(self, node: WhileStmt) -> Node: - return WhileStmt(self.node(node.expr), - self.block(node.body), - self.optional_block(node.else_body)) - - def visit_for_stmt(self, node: ForStmt) -> Node: - return ForStmt(self.node(node.index), - self.node(node.expr), - self.block(node.body), - self.optional_block(node.else_body)) - - def visit_return_stmt(self, node: ReturnStmt) -> Node: - return ReturnStmt(self.optional_node(node.expr)) - - def visit_assert_stmt(self, node: AssertStmt) -> Node: - return AssertStmt(self.node(node.expr)) + def visit_operator_assignment_stmt( + self, node: OperatorAssignmentStmt + ) -> OperatorAssignmentStmt: + return OperatorAssignmentStmt(node.op, self.expr(node.lvalue), self.expr(node.rvalue)) + + def visit_while_stmt(self, node: WhileStmt) -> WhileStmt: + return WhileStmt( + self.expr(node.expr), self.block(node.body), self.optional_block(node.else_body) + ) + + def visit_for_stmt(self, node: ForStmt) -> ForStmt: + new = ForStmt( + self.expr(node.index), + self.expr(node.expr), + self.block(node.body), + self.optional_block(node.else_body), + self.optional_type(node.unanalyzed_index_type), + ) + new.is_async = node.is_async + new.index_type = self.optional_type(node.index_type) + return new - def visit_yield_stmt(self, node: YieldStmt) -> Node: - return YieldStmt(self.node(node.expr)) + def visit_return_stmt(self, node: ReturnStmt) -> ReturnStmt: + return ReturnStmt(self.optional_expr(node.expr)) - def visit_yield_from_stmt(self, node: YieldFromStmt) -> Node: - return YieldFromStmt(self.node(node.expr)) + def visit_assert_stmt(self, node: AssertStmt) -> AssertStmt: + return AssertStmt(self.expr(node.expr), self.optional_expr(node.msg)) - def visit_del_stmt(self, node: DelStmt) -> Node: - return DelStmt(self.node(node.expr)) + def visit_del_stmt(self, node: DelStmt) -> DelStmt: + return DelStmt(self.expr(node.expr)) - def visit_if_stmt(self, node: IfStmt) -> Node: - return IfStmt(self.nodes(node.expr), - self.blocks(node.body), - self.optional_block(node.else_body)) + def visit_if_stmt(self, node: IfStmt) -> IfStmt: + return IfStmt( + self.expressions(node.expr), + self.blocks(node.body), + self.optional_block(node.else_body), + ) - def visit_break_stmt(self, node: BreakStmt) -> Node: + def visit_break_stmt(self, node: BreakStmt) -> BreakStmt: return BreakStmt() - def visit_continue_stmt(self, node: ContinueStmt) -> Node: + def visit_continue_stmt(self, node: ContinueStmt) -> ContinueStmt: return ContinueStmt() - def visit_pass_stmt(self, node: PassStmt) -> Node: + def visit_pass_stmt(self, node: PassStmt) -> PassStmt: return PassStmt() - def visit_raise_stmt(self, node: RaiseStmt) -> Node: - return RaiseStmt(self.optional_node(node.expr), - self.optional_node(node.from_expr)) - - def visit_try_stmt(self, node: TryStmt) -> Node: - return TryStmt(self.block(node.body), - self.optional_names(node.vars), - self.optional_nodes(node.types), - self.blocks(node.handlers), - self.optional_block(node.else_body), - self.optional_block(node.finally_body)) - - def visit_with_stmt(self, node: WithStmt) -> Node: - return WithStmt(self.nodes(node.expr), - self.optional_names(node.name), - self.block(node.body)) + def visit_raise_stmt(self, node: RaiseStmt) -> RaiseStmt: + return RaiseStmt(self.optional_expr(node.expr), self.optional_expr(node.from_expr)) + + def visit_try_stmt(self, node: TryStmt) -> TryStmt: + new = TryStmt( + self.block(node.body), + self.optional_names(node.vars), + self.optional_expressions(node.types), + self.blocks(node.handlers), + self.optional_block(node.else_body), + self.optional_block(node.finally_body), + ) + new.is_star = node.is_star + return new - def visit_print_stmt(self, node: PrintStmt) -> Node: - return PrintStmt(self.nodes(node.args), - node.newline) + def visit_with_stmt(self, node: WithStmt) -> WithStmt: + new = WithStmt( + self.expressions(node.expr), + self.optional_expressions(node.target), + self.block(node.body), + self.optional_type(node.unanalyzed_type), + ) + new.is_async = node.is_async + new.analyzed_types = [self.type(typ) for typ in node.analyzed_types] + return new - def visit_star_expr(self, node: StarExpr) -> Node: + def visit_as_pattern(self, p: AsPattern) -> AsPattern: + return AsPattern( + pattern=self.pattern(p.pattern) if p.pattern is not None else None, + name=self.duplicate_name(p.name) if p.name is not None else None, + ) + + def visit_or_pattern(self, p: OrPattern) -> OrPattern: + return OrPattern([self.pattern(pat) for pat in p.patterns]) + + def visit_value_pattern(self, p: ValuePattern) -> ValuePattern: + return ValuePattern(self.expr(p.expr)) + + def visit_singleton_pattern(self, p: SingletonPattern) -> SingletonPattern: + return SingletonPattern(p.value) + + def visit_sequence_pattern(self, p: SequencePattern) -> SequencePattern: + return SequencePattern([self.pattern(pat) for pat in p.patterns]) + + def visit_starred_pattern(self, p: StarredPattern) -> StarredPattern: + return StarredPattern(self.duplicate_name(p.capture) if p.capture is not None else None) + + def visit_mapping_pattern(self, p: MappingPattern) -> MappingPattern: + return MappingPattern( + keys=[self.expr(expr) for expr in p.keys], + values=[self.pattern(pat) for pat in p.values], + rest=self.duplicate_name(p.rest) if p.rest is not None else None, + ) + + def visit_class_pattern(self, p: ClassPattern) -> ClassPattern: + class_ref = p.class_ref.accept(self) + assert isinstance(class_ref, RefExpr) + return ClassPattern( + class_ref=class_ref, + positionals=[self.pattern(pat) for pat in p.positionals], + keyword_keys=list(p.keyword_keys), + keyword_values=[self.pattern(pat) for pat in p.keyword_values], + ) + + def visit_match_stmt(self, o: MatchStmt) -> MatchStmt: + return MatchStmt( + subject=self.expr(o.subject), + patterns=[self.pattern(p) for p in o.patterns], + guards=self.optional_expressions(o.guards), + bodies=self.blocks(o.bodies), + ) + + def visit_star_expr(self, node: StarExpr) -> StarExpr: return StarExpr(node.expr) - def visit_int_expr(self, node: IntExpr) -> Node: + def visit_int_expr(self, node: IntExpr) -> IntExpr: return IntExpr(node.value) - def visit_str_expr(self, node: StrExpr) -> Node: + def visit_str_expr(self, node: StrExpr) -> StrExpr: return StrExpr(node.value) - def visit_bytes_expr(self, node: BytesExpr) -> Node: + def visit_bytes_expr(self, node: BytesExpr) -> BytesExpr: return BytesExpr(node.value) - def visit_unicode_expr(self, node: UnicodeExpr) -> Node: - return UnicodeExpr(node.value) - - def visit_float_expr(self, node: FloatExpr) -> Node: + def visit_float_expr(self, node: FloatExpr) -> FloatExpr: return FloatExpr(node.value) - def visit_complex_expr(self, node: ComplexExpr) -> Node: + def visit_complex_expr(self, node: ComplexExpr) -> ComplexExpr: return ComplexExpr(node.value) - def visit_paren_expr(self, node: ParenExpr) -> Node: - return ParenExpr(self.node(node.expr)) + def visit_ellipsis(self, node: EllipsisExpr) -> EllipsisExpr: + return EllipsisExpr() - def visit_name_expr(self, node: NameExpr) -> Node: + def visit_name_expr(self, node: NameExpr) -> NameExpr: return self.duplicate_name(node) def duplicate_name(self, node: NameExpr) -> NameExpr: # This method is used when the transform result must be a NameExpr. # visit_name_expr() is used when there is no such restriction. new = NameExpr(node.name) - new.info = node.info self.copy_ref(new, node) + new.is_special_form = node.is_special_form return new - def visit_member_expr(self, node: MemberExpr) -> Node: - member = MemberExpr(self.node(node.expr), - node.name) + def visit_member_expr(self, node: MemberExpr) -> MemberExpr: + member = MemberExpr(self.expr(node.expr), node.name) if node.def_var: - member.def_var = self.visit_var(node.def_var) + # This refers to an attribute and we don't transform attributes by default, + # just normal variables. + member.def_var = node.def_var self.copy_ref(member, node) return member @@ -311,138 +490,244 @@ def copy_ref(self, new: RefExpr, original: RefExpr) -> None: new.fullname = original.fullname target = original.node if isinstance(target, Var): - target = self.visit_var(target) + # Do not transform references to global variables. See + # testGenericFunctionAliasExpand for an example where this is important. + if original.kind != GDEF: + target = self.visit_var(target) + elif isinstance(target, Decorator): + target = self.visit_var(target.var) + elif isinstance(target, FuncDef): + # Use a placeholder node for the function if it exists. + target = self.func_placeholder_map.get(target, target) new.node = target - new.is_def = original.is_def - - def visit_yield_from_expr(self, node: YieldFromExpr) -> Node: - return YieldFromExpr(self.node(node.expr)) - - def visit_call_expr(self, node: CallExpr) -> Node: - return CallExpr(self.node(node.callee), - self.nodes(node.args), - node.arg_kinds[:], - node.arg_names[:], - self.optional_node(node.analyzed)) - - def visit_op_expr(self, node: OpExpr) -> Node: - new = OpExpr(node.op, self.node(node.left), self.node(node.right)) + new.is_new_def = original.is_new_def + new.is_inferred_def = original.is_inferred_def + + def visit_yield_from_expr(self, node: YieldFromExpr) -> YieldFromExpr: + return YieldFromExpr(self.expr(node.expr)) + + def visit_yield_expr(self, node: YieldExpr) -> YieldExpr: + return YieldExpr(self.optional_expr(node.expr)) + + def visit_await_expr(self, node: AwaitExpr) -> AwaitExpr: + return AwaitExpr(self.expr(node.expr)) + + def visit_call_expr(self, node: CallExpr) -> CallExpr: + return CallExpr( + self.expr(node.callee), + self.expressions(node.args), + node.arg_kinds.copy(), + node.arg_names.copy(), + self.optional_expr(node.analyzed), + ) + + def visit_op_expr(self, node: OpExpr) -> OpExpr: + new = OpExpr( + node.op, + self.expr(node.left), + self.expr(node.right), + cast(Optional[TypeAliasExpr], self.optional_expr(node.analyzed)), + ) new.method_type = self.optional_type(node.method_type) return new - def visit_comparison_expr(self, node: ComparisonExpr) -> Node: - new = ComparisonExpr(node.operators, self.nodes(node.operands)) + def visit_comparison_expr(self, node: ComparisonExpr) -> ComparisonExpr: + new = ComparisonExpr(node.operators, self.expressions(node.operands)) new.method_types = [self.optional_type(t) for t in node.method_types] return new - def visit_cast_expr(self, node: CastExpr) -> Node: - return CastExpr(self.node(node.expr), - self.type(node.type)) + def visit_cast_expr(self, node: CastExpr) -> CastExpr: + return CastExpr(self.expr(node.expr), self.type(node.type)) + + def visit_assert_type_expr(self, node: AssertTypeExpr) -> AssertTypeExpr: + return AssertTypeExpr(self.expr(node.expr), self.type(node.type)) + + def visit_reveal_expr(self, node: RevealExpr) -> RevealExpr: + if node.kind == REVEAL_TYPE: + assert node.expr is not None + return RevealExpr(kind=REVEAL_TYPE, expr=self.expr(node.expr)) + else: + # Reveal locals expressions don't have any sub expressions + return node - def visit_super_expr(self, node: SuperExpr) -> Node: - new = SuperExpr(node.name) + def visit_super_expr(self, node: SuperExpr) -> SuperExpr: + call = self.expr(node.call) + assert isinstance(call, CallExpr) + new = SuperExpr(node.name, call) new.info = node.info return new - def visit_unary_expr(self, node: UnaryExpr) -> Node: - new = UnaryExpr(node.op, self.node(node.expr)) + def visit_assignment_expr(self, node: AssignmentExpr) -> AssignmentExpr: + return AssignmentExpr(self.expr(node.target), self.expr(node.value)) + + def visit_unary_expr(self, node: UnaryExpr) -> UnaryExpr: + new = UnaryExpr(node.op, self.expr(node.expr)) new.method_type = self.optional_type(node.method_type) return new - def visit_list_expr(self, node: ListExpr) -> Node: - return ListExpr(self.nodes(node.items)) + def visit_list_expr(self, node: ListExpr) -> ListExpr: + return ListExpr(self.expressions(node.items)) - def visit_dict_expr(self, node: DictExpr) -> Node: - return DictExpr([(self.node(key), self.node(value)) - for key, value in node.items]) + def visit_dict_expr(self, node: DictExpr) -> DictExpr: + return DictExpr( + [(self.expr(key) if key else None, self.expr(value)) for key, value in node.items] + ) - def visit_tuple_expr(self, node: TupleExpr) -> Node: - return TupleExpr(self.nodes(node.items)) + def visit_tuple_expr(self, node: TupleExpr) -> TupleExpr: + return TupleExpr(self.expressions(node.items)) - def visit_set_expr(self, node: SetExpr) -> Node: - return SetExpr(self.nodes(node.items)) + def visit_set_expr(self, node: SetExpr) -> SetExpr: + return SetExpr(self.expressions(node.items)) - def visit_index_expr(self, node: IndexExpr) -> Node: - new = IndexExpr(self.node(node.base), self.node(node.index)) + def visit_index_expr(self, node: IndexExpr) -> IndexExpr: + new = IndexExpr(self.expr(node.base), self.expr(node.index)) if node.method_type: new.method_type = self.type(node.method_type) if node.analyzed: - new.analyzed = self.visit_type_application(node.analyzed) - new.analyzed.set_line(node.analyzed.line) + if isinstance(node.analyzed, TypeApplication): + new.analyzed = self.visit_type_application(node.analyzed) + else: + new.analyzed = self.visit_type_alias_expr(node.analyzed) + new.analyzed.set_line(node.analyzed) return new - def visit_undefined_expr(self, node: UndefinedExpr) -> Node: - return UndefinedExpr(self.type(node.type)) - def visit_type_application(self, node: TypeApplication) -> TypeApplication: - return TypeApplication(self.node(node.expr), - self.types(node.types)) + return TypeApplication(self.expr(node.expr), self.types(node.types)) - def visit_list_comprehension(self, node: ListComprehension) -> Node: + def visit_list_comprehension(self, node: ListComprehension) -> ListComprehension: generator = self.duplicate_generator(node.generator) - generator.set_line(node.generator.line) + generator.set_line(node.generator) return ListComprehension(generator) - def visit_set_comprehension(self, node: SetComprehension) -> Node: + def visit_set_comprehension(self, node: SetComprehension) -> SetComprehension: generator = self.duplicate_generator(node.generator) - generator.set_line(node.generator.line) + generator.set_line(node.generator) return SetComprehension(generator) - def visit_dictionary_comprehension(self, node: DictionaryComprehension) -> Node: - return DictionaryComprehension(self.node(node.key), self.node(node.value), - [self.node(index) for index in node.indices], - [self.node(s) for s in node.sequences], - [[self.node(cond) for cond in conditions] - for conditions in node.condlists]) - - def visit_generator_expr(self, node: GeneratorExpr) -> Node: + def visit_dictionary_comprehension( + self, node: DictionaryComprehension + ) -> DictionaryComprehension: + return DictionaryComprehension( + self.expr(node.key), + self.expr(node.value), + [self.expr(index) for index in node.indices], + [self.expr(s) for s in node.sequences], + [[self.expr(cond) for cond in conditions] for conditions in node.condlists], + node.is_async, + ) + + def visit_generator_expr(self, node: GeneratorExpr) -> GeneratorExpr: return self.duplicate_generator(node) def duplicate_generator(self, node: GeneratorExpr) -> GeneratorExpr: - return GeneratorExpr(self.node(node.left_expr), - [self.node(index) for index in node.indices], - [self.node(s) for s in node.sequences], - [[self.node(cond) for cond in conditions] - for conditions in node.condlists]) - - def visit_slice_expr(self, node: SliceExpr) -> Node: - return SliceExpr(self.optional_node(node.begin_index), - self.optional_node(node.end_index), - self.optional_node(node.stride)) - - def visit_conditional_expr(self, node: ConditionalExpr) -> Node: - return ConditionalExpr(self.node(node.cond), - self.node(node.if_expr), - self.node(node.else_expr)) - - def visit_type_var_expr(self, node: TypeVarExpr) -> Node: - return TypeVarExpr(node.name(), node.fullname(), - self.types(node.values)) - - def visit_namedtuple_expr(self, node: NamedTupleExpr) -> Node: + return GeneratorExpr( + self.expr(node.left_expr), + [self.expr(index) for index in node.indices], + [self.expr(s) for s in node.sequences], + [[self.expr(cond) for cond in conditions] for conditions in node.condlists], + node.is_async, + ) + + def visit_slice_expr(self, node: SliceExpr) -> SliceExpr: + return SliceExpr( + self.optional_expr(node.begin_index), + self.optional_expr(node.end_index), + self.optional_expr(node.stride), + ) + + def visit_conditional_expr(self, node: ConditionalExpr) -> ConditionalExpr: + return ConditionalExpr( + self.expr(node.cond), self.expr(node.if_expr), self.expr(node.else_expr) + ) + + def visit_type_var_expr(self, node: TypeVarExpr) -> TypeVarExpr: + return TypeVarExpr( + node.name, + node.fullname, + self.types(node.values), + self.type(node.upper_bound), + self.type(node.default), + variance=node.variance, + ) + + def visit_paramspec_expr(self, node: ParamSpecExpr) -> ParamSpecExpr: + return ParamSpecExpr( + node.name, + node.fullname, + self.type(node.upper_bound), + self.type(node.default), + variance=node.variance, + ) + + def visit_type_var_tuple_expr(self, node: TypeVarTupleExpr) -> TypeVarTupleExpr: + return TypeVarTupleExpr( + node.name, + node.fullname, + self.type(node.upper_bound), + node.tuple_fallback, + self.type(node.default), + variance=node.variance, + ) + + def visit_type_alias_expr(self, node: TypeAliasExpr) -> TypeAliasExpr: + return TypeAliasExpr(node.node) + + def visit_newtype_expr(self, node: NewTypeExpr) -> NewTypeExpr: + res = NewTypeExpr(node.name, node.old_type, line=node.line, column=node.column) + res.info = node.info + return res + + def visit_namedtuple_expr(self, node: NamedTupleExpr) -> NamedTupleExpr: return NamedTupleExpr(node.info) - def visit_ducktype_expr(self, node: DucktypeExpr) -> Node: - return DucktypeExpr(node.type) + def visit_enum_call_expr(self, node: EnumCallExpr) -> EnumCallExpr: + return EnumCallExpr(node.info, node.items, node.values) - def visit_disjointclass_expr(self, node: DisjointclassExpr) -> Node: - return DisjointclassExpr(node.cls) + def visit_typeddict_expr(self, node: TypedDictExpr) -> Node: + return TypedDictExpr(node.info) - def visit_temp_node(self, node: TempNode) -> Node: + def visit__promote_expr(self, node: PromoteExpr) -> PromoteExpr: + return PromoteExpr(node.type) + + def visit_temp_node(self, node: TempNode) -> TempNode: return TempNode(self.type(node.type)) def node(self, node: Node) -> Node: new = node.accept(self) - new.set_line(node.line) + new.set_line(node) + return new + + def mypyfile(self, node: MypyFile) -> MypyFile: + new = node.accept(self) + assert isinstance(new, MypyFile) + new.set_line(node) + return new + + def expr(self, expr: Expression) -> Expression: + new = expr.accept(self) + assert isinstance(new, Expression) + new.set_line(expr) + return new + + def stmt(self, stmt: Statement) -> Statement: + new = stmt.accept(self) + assert isinstance(new, Statement) + new.set_line(stmt) + return new + + def pattern(self, pattern: Pattern) -> Pattern: + new = pattern.accept(self) + assert isinstance(new, Pattern) + new.set_line(pattern) return new # Helpers # # All the node helpers also propagate line numbers. - def optional_node(self, node: Node) -> Node: - if node: - return self.node(node) + def optional_expr(self, expr: Expression | None) -> Expression | None: + if expr: + return self.expr(expr) else: return None @@ -451,26 +736,31 @@ def block(self, block: Block) -> Block: new.line = block.line return new - def optional_block(self, block: Block) -> Block: + def optional_block(self, block: Block | None) -> Block | None: if block: return self.block(block) else: return None - def nodes(self, nodes: List[Node]) -> List[Node]: - return [self.node(node) for node in nodes] + def statements(self, statements: list[Statement]) -> list[Statement]: + return [self.stmt(stmt) for stmt in statements] - def optional_nodes(self, nodes: List[Node]) -> List[Node]: - return [self.optional_node(node) for node in nodes] + def expressions(self, expressions: list[Expression]) -> list[Expression]: + return [self.expr(expr) for expr in expressions] - def blocks(self, blocks: List[Block]) -> List[Block]: + def optional_expressions( + self, expressions: Iterable[Expression | None] + ) -> list[Expression | None]: + return [self.optional_expr(expr) for expr in expressions] + + def blocks(self, blocks: list[Block]) -> list[Block]: return [self.block(block) for block in blocks] - def names(self, names: List[NameExpr]) -> List[NameExpr]: + def names(self, names: list[NameExpr]) -> list[NameExpr]: return [self.duplicate_name(name) for name in names] - def optional_names(self, names: List[NameExpr]) -> List[NameExpr]: - result = List[NameExpr]() + def optional_names(self, names: Iterable[NameExpr | None]) -> list[NameExpr | None]: + result: list[NameExpr | None] = [] for name in names: if name: result.append(self.duplicate_name(name)) @@ -482,14 +772,29 @@ def type(self, type: Type) -> Type: # Override this method to transform types. return type - def optional_type(self, type: Type) -> Type: + def optional_type(self, type: Type | None) -> Type | None: if type: return self.type(type) else: return None - def types(self, types: List[Type]) -> List[Type]: + def types(self, types: list[Type]) -> list[Type]: return [self.type(type) for type in types] - def optional_types(self, types: List[Type]) -> List[Type]: - return [self.optional_type(type) for type in types] + +class FuncMapInitializer(TraverserVisitor): + """This traverser creates mappings from nested FuncDefs to placeholder FuncDefs. + + The placeholders will later be replaced with transformed nodes. + """ + + def __init__(self, transformer: TransformVisitor) -> None: + self.transformer = transformer + + def visit_func_def(self, node: FuncDef) -> None: + if node not in self.transformer.func_placeholder_map: + # Haven't seen this FuncDef before, so create a placeholder node. + self.transformer.func_placeholder_map[node] = FuncDef( + node.name, node.arguments, node.body, None + ) + super().visit_func_def(node) diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py new file mode 100644 index 000000000000..fe97a8359287 --- /dev/null +++ b/mypy/tvar_scope.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +from mypy.nodes import ( + ParamSpecExpr, + SymbolTableNode, + TypeVarExpr, + TypeVarLikeExpr, + TypeVarTupleExpr, +) +from mypy.types import ( + ParamSpecFlavor, + ParamSpecType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, +) +from mypy.typetraverser import TypeTraverserVisitor + + +class TypeVarLikeNamespaceSetter(TypeTraverserVisitor): + """Set namespace for all TypeVarLikeTypes types.""" + + def __init__(self, namespace: str) -> None: + self.namespace = namespace + + def visit_type_var(self, t: TypeVarType) -> None: + t.id.namespace = self.namespace + super().visit_type_var(t) + + def visit_param_spec(self, t: ParamSpecType) -> None: + t.id.namespace = self.namespace + return super().visit_param_spec(t) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + t.id.namespace = self.namespace + super().visit_type_var_tuple(t) + + +class TypeVarLikeScope: + """Scope that holds bindings for type variables and parameter specifications. + + Node fullname -> TypeVarLikeType. + """ + + def __init__( + self, + parent: TypeVarLikeScope | None = None, + is_class_scope: bool = False, + prohibited: TypeVarLikeScope | None = None, + namespace: str = "", + ) -> None: + """Initializer for TypeVarLikeScope + + Parameters: + parent: the outer scope for this scope + is_class_scope: True if this represents a generic class + prohibited: Type variables that aren't strictly in scope exactly, + but can't be bound because they're part of an outer class's scope. + """ + self.scope: dict[str, TypeVarLikeType] = {} + self.parent = parent + self.func_id = 0 + self.class_id = 0 + self.is_class_scope = is_class_scope + self.prohibited = prohibited + self.namespace = namespace + if parent is not None: + self.func_id = parent.func_id + self.class_id = parent.class_id + + def get_function_scope(self) -> TypeVarLikeScope | None: + """Get the nearest parent that's a function scope, not a class scope""" + it: TypeVarLikeScope | None = self + while it is not None and it.is_class_scope: + it = it.parent + return it + + def allow_binding(self, fullname: str) -> bool: + if fullname in self.scope: + return False + elif self.parent and not self.parent.allow_binding(fullname): + return False + elif self.prohibited and not self.prohibited.allow_binding(fullname): + return False + return True + + def method_frame(self, namespace: str) -> TypeVarLikeScope: + """A new scope frame for binding a method""" + return TypeVarLikeScope(self, False, None, namespace=namespace) + + def class_frame(self, namespace: str) -> TypeVarLikeScope: + """A new scope frame for binding a class. Prohibits *this* class's tvars""" + return TypeVarLikeScope(self.get_function_scope(), True, self, namespace=namespace) + + def new_unique_func_id(self) -> TypeVarId: + """Used by plugin-like code that needs to make synthetic generic functions.""" + self.func_id -= 1 + return TypeVarId(self.func_id) + + def bind_new(self, name: str, tvar_expr: TypeVarLikeExpr) -> TypeVarLikeType: + if self.is_class_scope: + self.class_id += 1 + i = self.class_id + else: + self.func_id -= 1 + i = self.func_id + namespace = self.namespace + tvar_expr.default.accept(TypeVarLikeNamespaceSetter(namespace)) + + if isinstance(tvar_expr, TypeVarExpr): + tvar_def: TypeVarLikeType = TypeVarType( + name=name, + fullname=tvar_expr.fullname, + id=TypeVarId(i, namespace=namespace), + values=tvar_expr.values, + upper_bound=tvar_expr.upper_bound, + default=tvar_expr.default, + variance=tvar_expr.variance, + line=tvar_expr.line, + column=tvar_expr.column, + ) + elif isinstance(tvar_expr, ParamSpecExpr): + tvar_def = ParamSpecType( + name=name, + fullname=tvar_expr.fullname, + id=TypeVarId(i, namespace=namespace), + flavor=ParamSpecFlavor.BARE, + upper_bound=tvar_expr.upper_bound, + default=tvar_expr.default, + line=tvar_expr.line, + column=tvar_expr.column, + ) + elif isinstance(tvar_expr, TypeVarTupleExpr): + tvar_def = TypeVarTupleType( + name=name, + fullname=tvar_expr.fullname, + id=TypeVarId(i, namespace=namespace), + upper_bound=tvar_expr.upper_bound, + tuple_fallback=tvar_expr.tuple_fallback, + default=tvar_expr.default, + line=tvar_expr.line, + column=tvar_expr.column, + ) + else: + assert False + self.scope[tvar_expr.fullname] = tvar_def + return tvar_def + + def bind_existing(self, tvar_def: TypeVarLikeType) -> None: + self.scope[tvar_def.fullname] = tvar_def + + def get_binding(self, item: str | SymbolTableNode) -> TypeVarLikeType | None: + fullname = item.fullname if isinstance(item, SymbolTableNode) else item + assert fullname + if fullname in self.scope: + return self.scope[fullname] + elif self.parent is not None: + return self.parent.get_binding(fullname) + else: + return None + + def __str__(self) -> str: + me = ", ".join(f"{k}: {v.name}`{v.id}" for k, v in self.scope.items()) + if self.parent is None: + return me + return f"{self.parent} <- {me}" diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py new file mode 100644 index 000000000000..8aac7e5c2bbd --- /dev/null +++ b/mypy/type_visitor.py @@ -0,0 +1,603 @@ +"""Type visitor classes. + +This module defines the type visitors that are intended to be +subclassed by other code. They have been separated out into their own +module to ease converting mypy to run under mypyc, since currently +mypyc-extension classes can extend interpreted classes but not the +other way around. Separating them out, then, allows us to compile +types before we can compile everything that uses a TypeVisitor. + +The visitors are all re-exported from mypy.types and that is how +other modules refer to them. +""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import Any, Callable, Final, Generic, Iterable, Sequence, TypeVar, cast + +from mypy_extensions import mypyc_attr, trait + +from mypy.types import ( + AnyType, + CallableArgument, + CallableType, + DeletedType, + EllipsisType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + PlaceholderType, + RawExpressionType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeList, + TypeType, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, +) + +T = TypeVar("T") + + +@trait +@mypyc_attr(allow_interpreted_subclasses=True) +class TypeVisitor(Generic[T]): + """Visitor class for types (Type subclasses). + + The parameter T is the return type of the visit methods. + """ + + @abstractmethod + def visit_unbound_type(self, t: UnboundType) -> T: + pass + + @abstractmethod + def visit_any(self, t: AnyType) -> T: + pass + + @abstractmethod + def visit_none_type(self, t: NoneType) -> T: + pass + + @abstractmethod + def visit_uninhabited_type(self, t: UninhabitedType) -> T: + pass + + @abstractmethod + def visit_erased_type(self, t: ErasedType) -> T: + pass + + @abstractmethod + def visit_deleted_type(self, t: DeletedType) -> T: + pass + + @abstractmethod + def visit_type_var(self, t: TypeVarType) -> T: + pass + + @abstractmethod + def visit_param_spec(self, t: ParamSpecType) -> T: + pass + + @abstractmethod + def visit_parameters(self, t: Parameters) -> T: + pass + + @abstractmethod + def visit_type_var_tuple(self, t: TypeVarTupleType) -> T: + pass + + @abstractmethod + def visit_instance(self, t: Instance) -> T: + pass + + @abstractmethod + def visit_callable_type(self, t: CallableType) -> T: + pass + + @abstractmethod + def visit_overloaded(self, t: Overloaded) -> T: + pass + + @abstractmethod + def visit_tuple_type(self, t: TupleType) -> T: + pass + + @abstractmethod + def visit_typeddict_type(self, t: TypedDictType) -> T: + pass + + @abstractmethod + def visit_literal_type(self, t: LiteralType) -> T: + pass + + @abstractmethod + def visit_union_type(self, t: UnionType) -> T: + pass + + @abstractmethod + def visit_partial_type(self, t: PartialType) -> T: + pass + + @abstractmethod + def visit_type_type(self, t: TypeType) -> T: + pass + + @abstractmethod + def visit_type_alias_type(self, t: TypeAliasType) -> T: + pass + + @abstractmethod + def visit_unpack_type(self, t: UnpackType) -> T: + pass + + +@trait +@mypyc_attr(allow_interpreted_subclasses=True) +class SyntheticTypeVisitor(TypeVisitor[T]): + """A TypeVisitor that also knows how to visit synthetic AST constructs. + + Not just real types. + """ + + @abstractmethod + def visit_type_list(self, t: TypeList) -> T: + pass + + @abstractmethod + def visit_callable_argument(self, t: CallableArgument) -> T: + pass + + @abstractmethod + def visit_ellipsis_type(self, t: EllipsisType) -> T: + pass + + @abstractmethod + def visit_raw_expression_type(self, t: RawExpressionType) -> T: + pass + + @abstractmethod + def visit_placeholder_type(self, t: PlaceholderType) -> T: + pass + + +@mypyc_attr(allow_interpreted_subclasses=True) +class TypeTranslator(TypeVisitor[Type]): + """Identity type transformation. + + Subclass this and override some methods to implement a non-trivial + transformation. + + We cache the results of certain translations to avoid + massively expanding the sizes of types. + """ + + def __init__(self, cache: dict[Type, Type] | None = None) -> None: + # For deduplication of results + self.cache = cache + + def get_cached(self, t: Type) -> Type | None: + if self.cache is None: + return None + return self.cache.get(t) + + def set_cached(self, orig: Type, new: Type) -> None: + if self.cache is None: + # Minor optimization: construct lazily + self.cache = {} + self.cache[orig] = new + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_none_type(self, t: NoneType) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_instance(self, t: Instance) -> Type: + last_known_value: LiteralType | None = None + if t.last_known_value is not None: + raw_last_known_value = t.last_known_value.accept(self) + assert isinstance(raw_last_known_value, LiteralType) # type: ignore[misc] + last_known_value = raw_last_known_value + return Instance( + typ=t.type, + args=self.translate_types(t.args), + line=t.line, + column=t.column, + last_known_value=last_known_value, + extra_attrs=t.extra_attrs, + ) + + def visit_type_var(self, t: TypeVarType) -> Type: + return t + + def visit_param_spec(self, t: ParamSpecType) -> Type: + return t + + def visit_parameters(self, t: Parameters) -> Type: + return t.copy_modified(arg_types=self.translate_types(t.arg_types)) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: + return t + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_unpack_type(self, t: UnpackType) -> Type: + return UnpackType(t.type.accept(self)) + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified( + arg_types=self.translate_types(t.arg_types), + ret_type=t.ret_type.accept(self), + variables=self.translate_variables(t.variables), + ) + + def visit_tuple_type(self, t: TupleType) -> Type: + return TupleType( + self.translate_types(t.items), + # TODO: This appears to be unsafe. + cast(Any, t.partial_fallback.accept(self)), + t.line, + t.column, + ) + + def visit_typeddict_type(self, t: TypedDictType) -> Type: + # Use cache to avoid O(n**2) or worse expansion of types during translation + if cached := self.get_cached(t): + return cached + items = {item_name: item_type.accept(self) for (item_name, item_type) in t.items.items()} + result = TypedDictType( + items, + t.required_keys, + t.readonly_keys, + # TODO: This appears to be unsafe. + cast(Any, t.fallback.accept(self)), + t.line, + t.column, + ) + self.set_cached(t, result) + return result + + def visit_literal_type(self, t: LiteralType) -> Type: + fallback = t.fallback.accept(self) + assert isinstance(fallback, Instance) # type: ignore[misc] + return LiteralType(value=t.value, fallback=fallback, line=t.line, column=t.column) + + def visit_union_type(self, t: UnionType) -> Type: + # Use cache to avoid O(n**2) or worse expansion of types during translation + # (only for large unions, since caching adds overhead) + use_cache = len(t.items) > 3 + if use_cache and (cached := self.get_cached(t)): + return cached + + result = UnionType( + self.translate_types(t.items), + t.line, + t.column, + uses_pep604_syntax=t.uses_pep604_syntax, + ) + if use_cache: + self.set_cached(t, result) + return result + + def translate_types(self, types: Iterable[Type]) -> list[Type]: + return [t.accept(self) for t in types] + + def translate_variables( + self, variables: Sequence[TypeVarLikeType] + ) -> Sequence[TypeVarLikeType]: + return variables + + def visit_overloaded(self, t: Overloaded) -> Type: + items: list[CallableType] = [] + for item in t.items: + new = item.accept(self) + assert isinstance(new, CallableType) # type: ignore[misc] + items.append(new) + return Overloaded(items=items) + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType.make_normalized(t.item.accept(self), line=t.line, column=t.column) + + @abstractmethod + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # This method doesn't have a default implementation for type translators, + # because type aliases are special: some information is contained in the + # TypeAlias node, and we normally don't generate new nodes. Every subclass + # must implement this depending on its semantics. + pass + + +@mypyc_attr(allow_interpreted_subclasses=True) +class TypeQuery(SyntheticTypeVisitor[T]): + """Visitor for performing queries of types. + + strategy is used to combine results for a series of types, + common use cases involve a boolean query using `any` or `all`. + + Note: this visitor keeps an internal state (tracks type aliases to avoid + recursion), so it should *never* be re-used for querying different types, + create a new visitor instance instead. + + # TODO: check that we don't have existing violations of this rule. + """ + + def __init__(self, strategy: Callable[[list[T]], T]) -> None: + self.strategy = strategy + # Keep track of the type aliases already visited. This is needed to avoid + # infinite recursion on types like A = Union[int, List[A]]. + self.seen_aliases: set[TypeAliasType] = set() + # By default, we eagerly expand type aliases, and query also types in the + # alias target. In most cases this is a desired behavior, but we may want + # to skip targets in some cases (e.g. when collecting type variables). + self.skip_alias_target = False + + def visit_unbound_type(self, t: UnboundType) -> T: + return self.query_types(t.args) + + def visit_type_list(self, t: TypeList) -> T: + return self.query_types(t.items) + + def visit_callable_argument(self, t: CallableArgument) -> T: + return t.typ.accept(self) + + def visit_any(self, t: AnyType) -> T: + return self.strategy([]) + + def visit_uninhabited_type(self, t: UninhabitedType) -> T: + return self.strategy([]) + + def visit_none_type(self, t: NoneType) -> T: + return self.strategy([]) + + def visit_erased_type(self, t: ErasedType) -> T: + return self.strategy([]) + + def visit_deleted_type(self, t: DeletedType) -> T: + return self.strategy([]) + + def visit_type_var(self, t: TypeVarType) -> T: + return self.query_types([t.upper_bound, t.default] + t.values) + + def visit_param_spec(self, t: ParamSpecType) -> T: + return self.query_types([t.upper_bound, t.default, t.prefix]) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> T: + return self.query_types([t.upper_bound, t.default]) + + def visit_unpack_type(self, t: UnpackType) -> T: + return self.query_types([t.type]) + + def visit_parameters(self, t: Parameters) -> T: + return self.query_types(t.arg_types) + + def visit_partial_type(self, t: PartialType) -> T: + return self.strategy([]) + + def visit_instance(self, t: Instance) -> T: + return self.query_types(t.args) + + def visit_callable_type(self, t: CallableType) -> T: + # FIX generics + return self.query_types(t.arg_types + [t.ret_type]) + + def visit_tuple_type(self, t: TupleType) -> T: + return self.query_types(t.items) + + def visit_typeddict_type(self, t: TypedDictType) -> T: + return self.query_types(t.items.values()) + + def visit_raw_expression_type(self, t: RawExpressionType) -> T: + return self.strategy([]) + + def visit_literal_type(self, t: LiteralType) -> T: + return self.strategy([]) + + def visit_union_type(self, t: UnionType) -> T: + return self.query_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> T: + return self.query_types(t.items) + + def visit_type_type(self, t: TypeType) -> T: + return t.item.accept(self) + + def visit_ellipsis_type(self, t: EllipsisType) -> T: + return self.strategy([]) + + def visit_placeholder_type(self, t: PlaceholderType) -> T: + return self.query_types(t.args) + + def visit_type_alias_type(self, t: TypeAliasType) -> T: + # Skip type aliases already visited types to avoid infinite recursion. + # TODO: Ideally we should fire subvisitors here (or use caching) if we care + # about duplicates. + if t in self.seen_aliases: + return self.strategy([]) + self.seen_aliases.add(t) + if self.skip_alias_target: + return self.query_types(t.args) + return get_proper_type(t).accept(self) + + def query_types(self, types: Iterable[Type]) -> T: + """Perform a query for a list of types using the strategy to combine the results.""" + return self.strategy([t.accept(self) for t in types]) + + +# Return True if at least one type component returns True +ANY_STRATEGY: Final = 0 +# Return True if no type component returns False +ALL_STRATEGY: Final = 1 + + +class BoolTypeQuery(SyntheticTypeVisitor[bool]): + """Visitor for performing recursive queries of types with a bool result. + + Use TypeQuery if you need non-bool results. + + 'strategy' is used to combine results for a series of types. It must + be ANY_STRATEGY or ALL_STRATEGY. + + Note: This visitor keeps an internal state (tracks type aliases to avoid + recursion), so it should *never* be re-used for querying different types + unless you call reset() first. + """ + + def __init__(self, strategy: int) -> None: + self.strategy = strategy + if strategy == ANY_STRATEGY: + self.default = False + else: + assert strategy == ALL_STRATEGY + self.default = True + # Keep track of the type aliases already visited. This is needed to avoid + # infinite recursion on types like A = Union[int, List[A]]. An empty set is + # represented as None as a micro-optimization. + self.seen_aliases: set[TypeAliasType] | None = None + # By default, we eagerly expand type aliases, and query also types in the + # alias target. In most cases this is a desired behavior, but we may want + # to skip targets in some cases (e.g. when collecting type variables). + self.skip_alias_target = False + + def reset(self) -> None: + """Clear mutable state (but preserve strategy). + + This *must* be called if you want to reuse the visitor. + """ + self.seen_aliases = None + + def visit_unbound_type(self, t: UnboundType) -> bool: + return self.query_types(t.args) + + def visit_type_list(self, t: TypeList) -> bool: + return self.query_types(t.items) + + def visit_callable_argument(self, t: CallableArgument) -> bool: + return t.typ.accept(self) + + def visit_any(self, t: AnyType) -> bool: + return self.default + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return self.default + + def visit_none_type(self, t: NoneType) -> bool: + return self.default + + def visit_erased_type(self, t: ErasedType) -> bool: + return self.default + + def visit_deleted_type(self, t: DeletedType) -> bool: + return self.default + + def visit_type_var(self, t: TypeVarType) -> bool: + return self.query_types([t.upper_bound, t.default] + t.values) + + def visit_param_spec(self, t: ParamSpecType) -> bool: + return self.query_types([t.upper_bound, t.default]) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool: + return self.query_types([t.upper_bound, t.default]) + + def visit_unpack_type(self, t: UnpackType) -> bool: + return self.query_types([t.type]) + + def visit_parameters(self, t: Parameters) -> bool: + return self.query_types(t.arg_types) + + def visit_partial_type(self, t: PartialType) -> bool: + return self.default + + def visit_instance(self, t: Instance) -> bool: + return self.query_types(t.args) + + def visit_callable_type(self, t: CallableType) -> bool: + # FIX generics + # Avoid allocating any objects here as an optimization. + args = self.query_types(t.arg_types) + ret = t.ret_type.accept(self) + if self.strategy == ANY_STRATEGY: + return args or ret + else: + return args and ret + + def visit_tuple_type(self, t: TupleType) -> bool: + return self.query_types(t.items) + + def visit_typeddict_type(self, t: TypedDictType) -> bool: + return self.query_types(list(t.items.values())) + + def visit_raw_expression_type(self, t: RawExpressionType) -> bool: + return self.default + + def visit_literal_type(self, t: LiteralType) -> bool: + return self.default + + def visit_union_type(self, t: UnionType) -> bool: + return self.query_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> bool: + return self.query_types(t.items) # type: ignore[arg-type] + + def visit_type_type(self, t: TypeType) -> bool: + return t.item.accept(self) + + def visit_ellipsis_type(self, t: EllipsisType) -> bool: + return self.default + + def visit_placeholder_type(self, t: PlaceholderType) -> bool: + return self.query_types(t.args) + + def visit_type_alias_type(self, t: TypeAliasType) -> bool: + # Skip type aliases already visited types to avoid infinite recursion. + # TODO: Ideally we should fire subvisitors here (or use caching) if we care + # about duplicates. + if self.seen_aliases is None: + self.seen_aliases = set() + elif t in self.seen_aliases: + return self.default + self.seen_aliases.add(t) + if self.skip_alias_target: + return self.query_types(t.args) + return get_proper_type(t).accept(self) + + def query_types(self, types: list[Type] | tuple[Type, ...]) -> bool: + """Perform a query for a sequence of types using the strategy to combine the results.""" + # Special-case for lists and tuples to allow mypyc to produce better code. + if isinstance(types, list): + if self.strategy == ANY_STRATEGY: + return any(t.accept(self) for t in types) + else: + return all(t.accept(self) for t in types) + else: + if self.strategy == ANY_STRATEGY: + return any(t.accept(self) for t in types) + else: + return all(t.accept(self) for t in types) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 93888fd6ce61..2f85e83bb3c3 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,319 +1,2725 @@ """Semantic analysis of types""" -from typing import Undefined, Function, cast, List, Tuple, Dict, Any, Union +from __future__ import annotations -from mypy.types import ( - Type, UnboundType, TypeVar, TupleType, UnionType, Instance, AnyType, Callable, - Void, NoneTyp, TypeList, TypeVarDef, TypeVisitor, StarType +import itertools +from contextlib import contextmanager +from typing import Callable, Final, Iterable, Iterator, List, Sequence, Tuple, TypeVar +from typing_extensions import Protocol + +from mypy import errorcodes as codes, message_registry, nodes +from mypy.errorcodes import ErrorCode +from mypy.expandtype import expand_type +from mypy.message_registry import ( + INVALID_PARAM_SPEC_LOCATION, + INVALID_PARAM_SPEC_LOCATION_NOTE, + TYPEDDICT_OVERRIDE_MERGE, +) +from mypy.messages import ( + MessageBuilder, + format_type, + format_type_bare, + quote_type_string, + wrong_type_arg_count, ) -from mypy.typerepr import TypeVarRepr from mypy.nodes import ( - GDEF, TypeInfo, Context, SymbolTableNode, TVAR, TypeVarExpr, Var, Node, - IndexExpr, NameExpr, TupleExpr + ARG_NAMED, + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + MISSING_FALLBACK, + SYMBOL_FUNCBASE_TYPES, + ArgKind, + Context, + Decorator, + ImportFrom, + MypyFile, + ParamSpecExpr, + PlaceholderNode, + SymbolTableNode, + TypeAlias, + TypeInfo, + TypeVarExpr, + TypeVarLikeExpr, + TypeVarTupleExpr, + Var, + check_arg_kinds, + check_arg_names, + get_nongen_builtins, +) +from mypy.options import INLINE_TYPEDDICT, Options +from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface +from mypy.semanal_shared import ( + SemanticAnalyzerCoreInterface, + SemanticAnalyzerInterface, + paramspec_args, + paramspec_kwargs, +) +from mypy.state import state +from mypy.tvar_scope import TypeVarLikeScope +from mypy.types import ( + ANNOTATED_TYPE_NAMES, + ANY_STRATEGY, + FINAL_TYPE_NAMES, + LITERAL_TYPE_NAMES, + NEVER_NAMES, + TYPE_ALIAS_NAMES, + AnyType, + BoolTypeQuery, + CallableArgument, + CallableType, + DeletedType, + EllipsisType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecFlavor, + ParamSpecType, + PartialType, + PlaceholderType, + ProperType, + RawExpressionType, + ReadOnlyType, + RequiredType, + SyntheticTypeVisitor, + TrivialSyntheticTypeTranslator, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeList, + TypeOfAny, + TypeQuery, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + callable_with_ellipsis, + find_unpack_in_list, + flatten_nested_tuples, + get_proper_type, + has_type_vars, ) -from mypy.sametypes import is_same_type -from mypy import nodes +from mypy.types_utils import is_bad_type_type_item +from mypy.typevars import fill_typevars +T = TypeVar("T") -def analyse_name(lookup: Function[[str, Context], SymbolTableNode], - name: str, ctx: Context) -> Any: - """Convert a name into the constructor for the associated Type""" - if name is None: - return None - if name.startswith('builtins.'): - # TODO: Not sure if this works... - kind = name.split('.')[1] - sym = lookup(kind, ctx) - if isinstance(sym.node, TypeInfo): - res = Instance(cast(TypeInfo, sym.node), []) - return lambda: res - elif name == 'typing.Any': - return AnyType - elif name == 'typing.Tuple': - return TupleType - elif name == 'typing.List': - raise NotImplementedError - elif name == 'typing.Dict': - raise NotImplementedError - elif name == 'typing.Set': - raise NotImplementedError - elif name == 'typing.Union': - return UnionType - - -def analyse_node(lookup: Function[[str, Context], SymbolTableNode], - node: Node, ctx: Context) -> Type: - if isinstance(node, IndexExpr): - if isinstance(node.base, NameExpr): - type_type = analyse_name(lookup, node.base.fullname, ctx) - if type_type is None: - return None - type_ind = Undefined(Any) - if isinstance(node.index, TupleExpr): - type_ind = [analyse_node(lookup, x, ctx) for x in node.index.items] - else: - type_ind = analyse_node(lookup, node.index, ctx) - result = type_type(type_ind) - return result - elif isinstance(node, NameExpr): - type_type = analyse_name(lookup, node.fullname, ctx) - if type_type: - return type_type() +type_constructors: Final = { + "typing.Callable", + "typing.Optional", + "typing.Tuple", + "typing.Type", + "typing.Union", + *LITERAL_TYPE_NAMES, + *ANNOTATED_TYPE_NAMES, +} +ARG_KINDS_BY_CONSTRUCTOR: Final = { + "mypy_extensions.Arg": ARG_POS, + "mypy_extensions.DefaultArg": ARG_OPT, + "mypy_extensions.NamedArg": ARG_NAMED, + "mypy_extensions.DefaultNamedArg": ARG_NAMED_OPT, + "mypy_extensions.VarArg": ARG_STAR, + "mypy_extensions.KwArg": ARG_STAR2, +} -class TypeAnalyser(TypeVisitor[Type]): - """Semantic analyzer for types (semantic analysis pass 2).""" +GENERIC_STUB_NOT_AT_RUNTIME_TYPES: Final = { + "queue.Queue", + "builtins._PathLike", + "asyncio.futures.Future", +} - def __init__(self, - lookup_func: Function[[str, Context], SymbolTableNode], - lookup_fqn_func: Function[[str], SymbolTableNode], - stored_vars: Dict[Node, Type], - fail_func: Function[[str, Context], None]) -> None: - self.lookup = lookup_func - self.lookup_fqn_func = lookup_fqn_func - self.fail = fail_func - self.stored_vars = stored_vars +SELF_TYPE_NAMES: Final = {"typing.Self", "typing_extensions.Self"} - def visit_unbound_type(self, t: UnboundType) -> Type: - sym = self.lookup(t.name, t) + +def analyze_type_alias( + type: Type, + api: SemanticAnalyzerCoreInterface, + tvar_scope: TypeVarLikeScope, + plugin: Plugin, + options: Options, + cur_mod_node: MypyFile, + is_typeshed_stub: bool, + allow_placeholder: bool = False, + in_dynamic_func: bool = False, + global_scope: bool = True, + allowed_alias_tvars: list[TypeVarLikeType] | None = None, + alias_type_params_names: list[str] | None = None, + python_3_12_type_alias: bool = False, +) -> tuple[Type, set[str]]: + """Analyze r.h.s. of a (potential) type alias definition. + + If `node` is valid as a type alias rvalue, return the resulting type and a set of + full names of type aliases it depends on (directly or indirectly). + 'node' must have been semantically analyzed. + """ + analyzer = TypeAnalyser( + api, + tvar_scope, + plugin, + options, + cur_mod_node, + is_typeshed_stub, + defining_alias=True, + allow_placeholder=allow_placeholder, + prohibit_self_type="type alias target", + allowed_alias_tvars=allowed_alias_tvars, + alias_type_params_names=alias_type_params_names, + python_3_12_type_alias=python_3_12_type_alias, + ) + analyzer.in_dynamic_func = in_dynamic_func + analyzer.global_scope = global_scope + res = analyzer.anal_type(type, nested=False) + return res, analyzer.aliases_used + + +def no_subscript_builtin_alias(name: str, propose_alt: bool = True) -> str: + class_name = name.split(".")[-1] + msg = f'"{class_name}" is not subscriptable' + # This should never be called if the python_version is 3.9 or newer + nongen_builtins = get_nongen_builtins((3, 8)) + replacement = nongen_builtins[name] + if replacement and propose_alt: + msg += f', use "{replacement}" instead' + return msg + + +class TypeAnalyser(SyntheticTypeVisitor[Type], TypeAnalyzerPluginInterface): + """Semantic analyzer for types. + + Converts unbound types into bound types. This is a no-op for already + bound types. + + If an incomplete reference is encountered, this does a defer. The + caller never needs to defer. + """ + + # Is this called from an untyped function definition? + in_dynamic_func: bool = False + # Is this called from global scope? + global_scope: bool = True + + def __init__( + self, + api: SemanticAnalyzerCoreInterface, + tvar_scope: TypeVarLikeScope, + plugin: Plugin, + options: Options, + cur_mod_node: MypyFile, + is_typeshed_stub: bool, + *, + defining_alias: bool = False, + python_3_12_type_alias: bool = False, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + allow_placeholder: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_types: bool = True, + prohibit_self_type: str | None = None, + allowed_alias_tvars: list[TypeVarLikeType] | None = None, + allow_type_any: bool = False, + alias_type_params_names: list[str] | None = None, + ) -> None: + self.api = api + self.fail_func = api.fail + self.note_func = api.note + self.tvar_scope = tvar_scope + # Are we analysing a type alias definition rvalue? + self.defining_alias = defining_alias + self.python_3_12_type_alias = python_3_12_type_alias + self.allow_tuple_literal = allow_tuple_literal + # Positive if we are analyzing arguments of another (outer) type + self.nesting_level = 0 + # Should we allow new type syntax when targeting older Python versions + # like 'list[int]' or 'X | Y' (allowed in stubs and with `__future__` import)? + self.always_allow_new_syntax = self.api.is_stub_file or self.api.is_future_flag_set( + "annotations" + ) + # Should we accept unbound type variables? This is currently used for class bases, + # and alias right hand sides (before they are analyzed as type aliases). + self.allow_unbound_tvars = allow_unbound_tvars + if allowed_alias_tvars is None: + allowed_alias_tvars = [] + self.allowed_alias_tvars = allowed_alias_tvars + self.alias_type_params_names = alias_type_params_names + # If false, record incomplete ref if we generate PlaceholderType. + self.allow_placeholder = allow_placeholder + # Are we in a context where Required[] is allowed? + self.allow_typed_dict_special_forms = allow_typed_dict_special_forms + # Are we in a context where ParamSpec literals are allowed? + self.allow_param_spec_literals = allow_param_spec_literals + # Are we in context where literal "..." specifically is allowed? + self.allow_ellipsis = False + # Should we report an error whenever we encounter a RawExpressionType outside + # of a Literal context: e.g. whenever we encounter an invalid type? Normally, + # we want to report an error, but the caller may want to do more specialized + # error handling. + self.report_invalid_types = report_invalid_types + self.plugin = plugin + self.options = options + self.cur_mod_node = cur_mod_node + self.is_typeshed_stub = is_typeshed_stub + # Names of type aliases encountered while analysing a type will be collected here. + self.aliases_used: set[str] = set() + self.prohibit_self_type = prohibit_self_type + # Allow variables typed as Type[Any] and type (useful for base classes). + self.allow_type_any = allow_type_any + self.allow_type_var_tuple = False + self.allow_unpack = allow_unpack + + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + return self.api.lookup_qualified(name, ctx, suppress_errors) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + return self.api.lookup_fully_qualified(name) + + def visit_unbound_type(self, t: UnboundType, defining_literal: bool = False) -> Type: + typ = self.visit_unbound_type_nonoptional(t, defining_literal) + if t.optional: + # We don't need to worry about double-wrapping Optionals or + # wrapping Anys: Union simplification will take care of that. + return make_optional_type(typ) + return typ + + def not_declared_in_type_params(self, tvar_name: str) -> bool: + return ( + self.alias_type_params_names is not None + and tvar_name not in self.alias_type_params_names + ) + + def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) -> Type: + sym = self.lookup_qualified(t.name, t) if sym is not None: - if sym.kind == TVAR: + node = sym.node + if isinstance(node, PlaceholderNode): + if node.becomes_typeinfo: + # Reference to placeholder type. + if self.api.final_iteration: + self.cannot_resolve_type(t) + return AnyType(TypeOfAny.from_error) + elif self.allow_placeholder: + self.api.defer() + else: + self.api.record_incomplete_ref() + # Always allow ParamSpec for placeholders, if they are actually not valid, + # they will be reported later, after we resolve placeholders. + return PlaceholderType( + node.fullname, + self.anal_array( + t.args, + allow_param_spec=True, + allow_param_spec_literals=True, + allow_unpack=True, + ), + t.line, + ) + else: + if self.api.final_iteration: + self.cannot_resolve_type(t) + return AnyType(TypeOfAny.from_error) + else: + # Reference to an unknown placeholder node. + self.api.record_incomplete_ref() + return AnyType(TypeOfAny.special_form) + if node is None: + self.fail(f"Internal error (node is None, kind={sym.kind})", t) + return AnyType(TypeOfAny.special_form) + fullname = node.fullname + hook = self.plugin.get_type_analyze_hook(fullname) + if hook is not None: + return hook(AnalyzeTypeContext(t, t, self)) + if ( + fullname in get_nongen_builtins(self.options.python_version) + and t.args + and not self.always_allow_new_syntax + ): + self.fail( + no_subscript_builtin_alias(fullname, propose_alt=not self.defining_alias), t + ) + tvar_def = self.tvar_scope.get_binding(sym) + if isinstance(sym.node, ParamSpecExpr): + if tvar_def is None: + if self.allow_unbound_tvars: + return t + if self.defining_alias and self.not_declared_in_type_params(t.name): + msg = f'ParamSpec "{t.name}" is not included in type_params' + else: + msg = f'ParamSpec "{t.name}" is unbound' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + assert isinstance(tvar_def, ParamSpecType) + if len(t.args) > 0: + self.fail( + f'ParamSpec "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) + # Change the line number + return ParamSpecType( + tvar_def.name, + tvar_def.fullname, + tvar_def.id, + tvar_def.flavor, + tvar_def.upper_bound, + tvar_def.default, + line=t.line, + column=t.column, + ) + if ( + isinstance(sym.node, TypeVarExpr) + and self.defining_alias + and not defining_literal + and (tvar_def is None or tvar_def not in self.allowed_alias_tvars) + ): + if self.not_declared_in_type_params(t.name): + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{t.name}"' + ) + else: + msg = f'Type variable "{t.name}" is not included in type_params' + else: + msg = f'Can\'t use bound type variable "{t.name}" to define generic alias' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + if isinstance(sym.node, TypeVarExpr) and tvar_def is not None: + assert isinstance(tvar_def, TypeVarType) if len(t.args) > 0: - self.fail('Type variable "{}" used with arguments'.format( - t.name), t) - if t.repr: - rep = TypeVarRepr(t.repr.components[0]) + self.fail( + f'Type variable "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) + # Change the line number + return tvar_def.copy_modified(line=t.line, column=t.column) + if isinstance(sym.node, TypeVarTupleExpr) and ( + tvar_def is not None + and self.defining_alias + and tvar_def not in self.allowed_alias_tvars + ): + if self.not_declared_in_type_params(t.name): + msg = f'Type variable "{t.name}" is not included in type_params' else: - rep = None - values = cast(TypeVarExpr, sym.node).values - return TypeVar(t.name, sym.tvar_id, values, self.builtin_type('builtins.object'), - t.line, rep) - elif sym.node.fullname() == 'builtins.None': - return Void() - elif sym.node.fullname() == 'typing.Any': - return AnyType() - elif sym.node.fullname() == 'typing.Tuple': - return TupleType(self.anal_array(t.args), - self.builtin_type('builtins.tuple')) - elif sym.node.fullname() == 'typing.Union': - return UnionType(self.anal_array(t.args)) - elif sym.node.fullname() == 'typing.Function': - return self.analyze_function_type(t) - elif not isinstance(sym.node, TypeInfo): - name = sym.fullname - if name is None: - name = sym.node.name() - if sym.node in self.stored_vars: - return self.stored_vars[sym.node] - self.fail('Invalid type "{}"'.format(name), t) - return t - info = cast(TypeInfo, sym.node) - if len(t.args) > 0 and info.fullname() == 'builtins.tuple': - return TupleType(self.anal_array(t.args), - Instance(info, [], t.line), - t.line, t.repr) + msg = f'Can\'t use bound type variable "{t.name}" to define generic alias' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + if isinstance(sym.node, TypeVarTupleExpr): + if tvar_def is None: + if self.allow_unbound_tvars: + return t + if self.defining_alias and self.not_declared_in_type_params(t.name): + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{t.name}"' + ) + else: + msg = f'TypeVarTuple "{t.name}" is not included in type_params' + else: + msg = f'TypeVarTuple "{t.name}" is unbound' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + assert isinstance(tvar_def, TypeVarTupleType) + if not self.allow_type_var_tuple: + self.fail( + f'TypeVarTuple "{t.name}" is only valid with an unpack', + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) > 0: + self.fail( + f'Type variable "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) + + # Change the line number + return TypeVarTupleType( + tvar_def.name, + tvar_def.fullname, + tvar_def.id, + tvar_def.upper_bound, + sym.node.tuple_fallback, + tvar_def.default, + line=t.line, + column=t.column, + ) + special = self.try_analyze_special_unbound_type(t, fullname) + if special is not None: + return special + if isinstance(node, TypeAlias): + self.aliases_used.add(fullname) + an_args = self.anal_array( + t.args, + allow_param_spec=True, + allow_param_spec_literals=node.has_param_spec_type, + allow_unpack=True, # Fixed length unpacks can be used for non-variadic aliases. + ) + if node.has_param_spec_type and len(node.alias_tvars) == 1: + an_args = self.pack_paramspec_args(an_args) + + disallow_any = self.options.disallow_any_generics and not self.is_typeshed_stub + res = instantiate_type_alias( + node, + an_args, + self.fail, + node.no_args, + t, + self.options, + unexpanded_type=t, + disallow_any=disallow_any, + empty_tuple_index=t.empty_tuple_index, + ) + # The only case where instantiate_type_alias() can return an incorrect instance is + # when it is top-level instance, so no need to recurse. + if ( + isinstance(res, ProperType) + and isinstance(res, Instance) + and not (self.defining_alias and self.nesting_level == 0) + and not validate_instance(res, self.fail, t.empty_tuple_index) + ): + fix_instance( + res, + self.fail, + self.note, + disallow_any=disallow_any, + options=self.options, + use_generic_error=True, + unexpanded_type=t, + ) + if node.eager: + res = get_proper_type(res) + return res + elif isinstance(node, TypeInfo): + return self.analyze_type_with_type_info(node, t.args, t, t.empty_tuple_index) + elif node.fullname in TYPE_ALIAS_NAMES: + return AnyType(TypeOfAny.special_form) + # Concatenate is an operator, no need for a proper type + elif node.fullname in ("typing_extensions.Concatenate", "typing.Concatenate"): + # We check the return type further up the stack for valid use locations + return self.apply_concatenate_operator(t) else: + return self.analyze_unbound_type_without_type_info(t, sym, defining_literal) + else: # sym is None + return AnyType(TypeOfAny.special_form) - # Analyze arguments and construct Instance type. The - # number of type arguments and their values are - # checked only later, since we do not always know the - # valid count at this point. Thus we may construct an - # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(t.args), t.line, t.repr) - if info.tuple_type is None: - return instance + def pack_paramspec_args(self, an_args: Sequence[Type]) -> list[Type]: + # "Aesthetic" ParamSpec literals for single ParamSpec: C[int, str] -> C[[int, str]]. + # These do not support mypy_extensions VarArgs, etc. as they were already analyzed + # TODO: should these be re-analyzed to get rid of this inconsistency? + count = len(an_args) + if count == 0: + return [] + if count == 1 and isinstance(get_proper_type(an_args[0]), AnyType): + # Single Any is interpreted as ..., rather that a single argument with Any type. + # I didn't find this in the PEP, but it sounds reasonable. + return list(an_args) + if any(isinstance(a, (Parameters, ParamSpecType)) for a in an_args): + if len(an_args) > 1: + first_wrong = next( + arg for arg in an_args if isinstance(arg, (Parameters, ParamSpecType)) + ) + self.fail( + "Nested parameter specifications are not allowed", + first_wrong, + code=codes.VALID_TYPE, + ) + return [AnyType(TypeOfAny.from_error)] + return list(an_args) + first = an_args[0] + return [ + Parameters( + an_args, [ARG_POS] * count, [None] * count, line=first.line, column=first.column + ) + ] + + def cannot_resolve_type(self, t: UnboundType) -> None: + # TODO: Move error message generation to messages.py. We'd first + # need access to MessageBuilder here. Also move the similar + # message generation logic in semanal.py. + self.api.fail(f'Cannot resolve name "{t.name}" (possible cyclic definition)', t) + if self.api.is_func_scope(): + self.note("Recursive types are not allowed at function scope", t) + + def apply_concatenate_operator(self, t: UnboundType) -> Type: + if len(t.args) == 0: + self.api.fail("Concatenate needs type arguments", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + + # Last argument has to be ParamSpec or Ellipsis. + ps = self.anal_type(t.args[-1], allow_param_spec=True, allow_ellipsis=True) + if not isinstance(ps, (ParamSpecType, Parameters)): + if isinstance(ps, UnboundType) and self.allow_unbound_tvars: + sym = self.lookup_qualified(ps.name, t) + if sym is not None and isinstance(sym.node, ParamSpecExpr): + return ps + self.api.fail( + "The last parameter to Concatenate needs to be a ParamSpec", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + elif isinstance(ps, ParamSpecType) and ps.prefix.arg_types: + self.api.fail("Nested Concatenates are invalid", t, code=codes.VALID_TYPE) + + args = self.anal_array(t.args[:-1]) + pre = ps.prefix if isinstance(ps, ParamSpecType) else ps + + # mypy can't infer this :( + names: list[str | None] = [None] * len(args) + + pre = Parameters( + args + pre.arg_types, + [ARG_POS] * len(args) + pre.arg_kinds, + names + pre.arg_names, + line=t.line, + column=t.column, + ) + return ps.copy_modified(prefix=pre) if isinstance(ps, ParamSpecType) else pre + + def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Type | None: + """Bind special type that is recognized through magic name such as 'typing.Any'. + + Return the bound type if successful, and return None if the type is a normal type. + """ + if fullname == "builtins.None": + return NoneType() + elif fullname == "typing.Any" or fullname == "builtins.Any": + return AnyType(TypeOfAny.explicit, line=t.line, column=t.column) + elif fullname in FINAL_TYPE_NAMES: + self.fail( + "Final can be only used as an outermost qualifier in a variable annotation", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + elif fullname == "typing.Tuple" or ( + fullname == "builtins.tuple" + and (self.always_allow_new_syntax or self.options.python_version >= (3, 9)) + ): + # Tuple is special because it is involved in builtin import cycle + # and may be not ready when used. + sym = self.api.lookup_fully_qualified_or_none("builtins.tuple") + if not sym or isinstance(sym.node, PlaceholderNode): + if self.api.is_incomplete_namespace("builtins"): + self.api.record_incomplete_ref() else: - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - return TupleType(self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - else: + self.fail('Name "tuple" is not defined', t) + return AnyType(TypeOfAny.special_form) + if len(t.args) == 0 and not t.empty_tuple_index: + # Bare 'Tuple' is same as 'tuple' + any_type = self.get_omitted_any(t) + return self.named_type("builtins.tuple", [any_type], line=t.line, column=t.column) + if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): + # Tuple[T, ...] (uniform, variable-length tuple) + instance = self.named_type("builtins.tuple", [self.anal_type(t.args[0])]) + instance.line = t.line + return instance + return self.tuple_type( + self.anal_array(t.args, allow_unpack=True), line=t.line, column=t.column + ) + elif fullname == "typing.Union": + items = self.anal_array(t.args) + return UnionType.make_union(items) + elif fullname == "typing.Optional": + if len(t.args) != 1: + self.fail( + "Optional[...] must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + item = self.anal_type(t.args[0]) + return make_optional_type(item) + elif fullname == "typing.Callable": + return self.analyze_callable_type(t) + elif fullname == "typing.Type" or ( + fullname == "builtins.type" + and (self.always_allow_new_syntax or self.options.python_version >= (3, 9)) + ): + if len(t.args) == 0: + if fullname == "typing.Type": + any_type = self.get_omitted_any(t) + return TypeType(any_type, line=t.line, column=t.column) + else: + # To prevent assignment of 'builtins.type' inferred as 'builtins.object' + # See https://github.com/python/mypy/issues/9476 for more information + return None + if len(t.args) != 1: + type_str = "Type[...]" if fullname == "typing.Type" else "type[...]" + self.fail( + type_str + " must have exactly one type argument", t, code=codes.VALID_TYPE + ) + item = self.anal_type(t.args[0]) + if is_bad_type_type_item(item): + self.fail("Type[...] can't contain another Type[...]", t, code=codes.VALID_TYPE) + item = AnyType(TypeOfAny.from_error) + return TypeType.make_normalized(item, line=t.line, column=t.column) + elif fullname == "typing.ClassVar": + if self.nesting_level > 0: + self.fail( + "Invalid type: ClassVar nested inside other type", t, code=codes.VALID_TYPE + ) + if len(t.args) == 0: + return AnyType(TypeOfAny.from_omitted_generics, line=t.line, column=t.column) + if len(t.args) != 1: + self.fail( + "ClassVar[...] must have at most one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return self.anal_type(t.args[0]) + elif fullname in NEVER_NAMES: + return UninhabitedType() + elif fullname in LITERAL_TYPE_NAMES: + return self.analyze_literal_type(t) + elif fullname in ANNOTATED_TYPE_NAMES: + if len(t.args) < 2: + self.fail( + "Annotated[...] must have exactly one type argument" + " and at least one annotation", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + return self.anal_type( + t.args[0], allow_typed_dict_special_forms=self.allow_typed_dict_special_forms + ) + elif fullname in ("typing_extensions.Required", "typing.Required"): + if not self.allow_typed_dict_special_forms: + self.fail( + "Required[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail( + "Required[] must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return RequiredType( + self.anal_type(t.args[0], allow_typed_dict_special_forms=True), required=True + ) + elif fullname in ("typing_extensions.NotRequired", "typing.NotRequired"): + if not self.allow_typed_dict_special_forms: + self.fail( + "NotRequired[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail( + "NotRequired[] must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return RequiredType( + self.anal_type(t.args[0], allow_typed_dict_special_forms=True), required=False + ) + elif fullname in ("typing_extensions.ReadOnly", "typing.ReadOnly"): + if not self.allow_typed_dict_special_forms: + self.fail( + "ReadOnly[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail( + '"ReadOnly[]" must have exactly one type argument', t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return ReadOnlyType(self.anal_type(t.args[0], allow_typed_dict_special_forms=True)) + elif ( + self.anal_type_guard_arg(t, fullname) is not None + or self.anal_type_is_arg(t, fullname) is not None + ): + # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args) + return self.named_type("builtins.bool") + elif fullname in ("typing.Unpack", "typing_extensions.Unpack"): + if len(t.args) != 1: + self.fail("Unpack[...] requires exactly one type argument", t) + return AnyType(TypeOfAny.from_error) + if not self.allow_unpack: + self.fail(message_registry.INVALID_UNPACK_POSITION, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + self.allow_type_var_tuple = True + result = UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) + self.allow_type_var_tuple = False + return result + elif fullname in SELF_TYPE_NAMES: + if t.args: + self.fail("Self type cannot have type arguments", t) + if self.prohibit_self_type is not None: + self.fail(f"Self type cannot be used in {self.prohibit_self_type}", t) + return AnyType(TypeOfAny.from_error) + if self.api.type is None: + self.fail("Self type is only allowed in annotations within class definition", t) + return AnyType(TypeOfAny.from_error) + if self.api.type.has_base("builtins.type"): + self.fail("Self type cannot be used in a metaclass", t) + if self.api.type.self_type is not None: + if self.api.type.is_final: + return fill_typevars(self.api.type) + return self.api.type.self_type.copy_modified(line=t.line, column=t.column) + # TODO: verify this is unreachable and replace with an assert? + self.fail("Unexpected Self type", t) + return AnyType(TypeOfAny.from_error) + return None + + def get_omitted_any(self, typ: Type, fullname: str | None = None) -> AnyType: + disallow_any = not self.is_typeshed_stub and self.options.disallow_any_generics + return get_omitted_any(disallow_any, self.fail, self.note, typ, self.options, fullname) + + def check_and_warn_deprecated(self, info: TypeInfo, ctx: Context) -> None: + """Similar logic to `TypeChecker.check_deprecated` and `TypeChecker.warn_deprecated.""" + + if ( + (deprecated := info.deprecated) + and not self.is_typeshed_stub + and not (self.api.type and (self.api.type.fullname == info.fullname)) + ): + for imp in self.cur_mod_node.imports: + if isinstance(imp, ImportFrom) and any(info.name == n[0] for n in imp.names): + break + else: + warn = self.note if self.options.report_deprecated_as_note else self.fail + warn(deprecated, ctx, code=codes.DEPRECATED) + + def analyze_type_with_type_info( + self, info: TypeInfo, args: Sequence[Type], ctx: Context, empty_tuple_index: bool + ) -> Type: + """Bind unbound type when were able to find target TypeInfo. + + This handles simple cases like 'int', 'modname.UserClass[str]', etc. + """ + + self.check_and_warn_deprecated(info, ctx) + + if len(args) > 0 and info.fullname == "builtins.tuple": + fallback = Instance(info, [AnyType(TypeOfAny.special_form)], ctx.line) + return TupleType(self.anal_array(args, allow_unpack=True), fallback, ctx.line) + + # Analyze arguments and (usually) construct Instance type. The + # number of type arguments and their values are + # checked only later, since we do not always know the + # valid count at this point. Thus we may construct an + # Instance with an invalid number of type arguments. + # + # We allow ParamSpec literals based on a heuristic: it will be + # checked later anyways but the error message may be worse. + instance = Instance( + info, + self.anal_array( + args, + allow_param_spec=True, + allow_param_spec_literals=info.has_param_spec_type, + allow_unpack=True, # Fixed length tuples can be used for non-variadic types. + ), + ctx.line, + ctx.column, + ) + if len(info.type_vars) == 1 and info.has_param_spec_type: + instance.args = tuple(self.pack_paramspec_args(instance.args)) + + # Check type argument count. + instance.args = tuple(flatten_nested_tuples(instance.args)) + if not (self.defining_alias and self.nesting_level == 0) and not validate_instance( + instance, self.fail, empty_tuple_index + ): + fix_instance( + instance, + self.fail, + self.note, + disallow_any=self.options.disallow_any_generics and not self.is_typeshed_stub, + options=self.options, + ) + + tup = info.tuple_type + if tup is not None: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if info.special_alias: + return instantiate_type_alias( + info.special_alias, + # TODO: should we allow NamedTuples generic in ParamSpec? + self.anal_array(args, allow_unpack=True), + self.fail, + False, + ctx, + self.options, + use_standard_error=True, + ) + return tup.copy_modified( + items=self.anal_array(tup.items, allow_unpack=True), fallback=instance + ) + td = info.typeddict_type + if td is not None: + # The class has a TypedDict[...] base class so it will be + # represented as a typeddict type. + if info.special_alias: + return instantiate_type_alias( + info.special_alias, + # TODO: should we allow TypedDicts generic in ParamSpec? + self.anal_array(args, allow_unpack=True), + self.fail, + False, + ctx, + self.options, + use_standard_error=True, + ) + # Create a named TypedDictType + return td.copy_modified( + item_types=self.anal_array(list(td.items.values())), fallback=instance + ) + + if info.fullname == "types.NoneType": + self.fail( + "NoneType should not be used as a type, please use None instead", + ctx, + code=codes.VALID_TYPE, + ) + return NoneType(ctx.line, ctx.column) + + return instance + + def analyze_unbound_type_without_type_info( + self, t: UnboundType, sym: SymbolTableNode, defining_literal: bool + ) -> Type: + """Figure out what an unbound type that doesn't refer to a TypeInfo node means. + + This is something unusual. We try our best to find out what it is. + """ + name = sym.fullname + if name is None: + assert sym.node is not None + name = sym.node.name + # Option 1: + # Something with an Any type -- make it an alias for Any in a type + # context. This is slightly problematic as it allows using the type 'Any' + # as a base class -- however, this will fail soon at runtime so the problem + # is pretty minor. + if isinstance(sym.node, Var): + typ = get_proper_type(sym.node.type) + if isinstance(typ, AnyType): + return AnyType( + TypeOfAny.from_unimported_type, missing_import_name=typ.missing_import_name + ) + elif self.allow_type_any: + if isinstance(typ, Instance) and typ.type.fullname == "builtins.type": + return AnyType(TypeOfAny.special_form) + if isinstance(typ, TypeType) and isinstance(typ.item, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=typ.item) + # Option 2: + # Unbound type variable. Currently these may be still valid, + # for example when defining a generic type alias. + unbound_tvar = ( + isinstance(sym.node, (TypeVarExpr, TypeVarTupleExpr)) + and self.tvar_scope.get_binding(sym) is None + ) + if self.allow_unbound_tvars and unbound_tvar: return t + # Option 3: + # Enum value. Note: we only want to return a LiteralType when + # we're using this enum value specifically within context of + # a "Literal[...]" type. So, if `defining_literal` is not set, + # we bail out early with an error. + # + # If, in the distant future, we decide to permit things like + # `def foo(x: Color.RED) -> None: ...`, we can remove that + # check entirely. + if ( + isinstance(sym.node, Var) + and sym.node.info + and sym.node.info.is_enum + and not sym.node.name.startswith("__") + ): + value = sym.node.name + base_enum_short_name = sym.node.info.name + if not defining_literal: + msg = message_registry.INVALID_TYPE_RAW_ENUM_VALUE.format( + base_enum_short_name, value + ) + self.fail(msg.value, t, code=msg.code) + return AnyType(TypeOfAny.from_error) + return LiteralType( + value=value, + fallback=Instance(sym.node.info, [], line=t.line, column=t.column), + line=t.line, + column=t.column, + ) + + # None of the above options worked. We parse the args (if there are any) + # to make sure there are no remaining semanal-only types, then give up. + t = t.copy_modified(args=self.anal_array(t.args)) + # TODO: Move this message building logic to messages.py. + notes: list[str] = [] + error_code = codes.VALID_TYPE + if isinstance(sym.node, Var): + notes.append( + "See https://mypy.readthedocs.io/en/" + "stable/common_issues.html#variables-vs-type-aliases" + ) + message = 'Variable "{}" is not valid as a type' + elif isinstance(sym.node, (SYMBOL_FUNCBASE_TYPES, Decorator)): + message = 'Function "{}" is not valid as a type' + if name == "builtins.any": + notes.append('Perhaps you meant "typing.Any" instead of "any"?') + elif name == "builtins.callable": + notes.append('Perhaps you meant "typing.Callable" instead of "callable"?') + else: + notes.append('Perhaps you need "Callable[...]" or a callback protocol?') + elif isinstance(sym.node, MypyFile): + message = 'Module "{}" is not valid as a type' + notes.append("Perhaps you meant to use a protocol matching the module structure?") + elif unbound_tvar: + assert isinstance(sym.node, TypeVarLikeExpr) + if sym.node.is_new_style: + # PEP 695 type paramaters are never considered unbound -- they are undefined + # in contexts where they aren't valid, such as in argument default values. + message = 'Name "{}" is not defined' + name = name.split(".")[-1] + error_code = codes.NAME_DEFINED + else: + message = 'Type variable "{}" is unbound' + short = name.split(".")[-1] + notes.append( + f'(Hint: Use "Generic[{short}]" or "Protocol[{short}]" base class' + f' to bind "{short}" inside a class)' + ) + notes.append( + f'(Hint: Use "{short}" in function signature ' + f'to bind "{short}" inside a function)' + ) + else: + message = 'Cannot interpret reference "{}" as a type' + if not defining_literal: + # Literal check already gives a custom error. Avoid duplicating errors. + self.fail(message.format(name), t, code=error_code) + for note in notes: + self.note(note, t, code=error_code) + + # TODO: Would it be better to always return Any instead of UnboundType + # in case of an error? On one hand, UnboundType has a name so error messages + # are more detailed, on the other hand, some of them may be bogus, + # see https://github.com/python/mypy/issues/4987. + return t + def visit_any(self, t: AnyType) -> Type: return t - def visit_void(self, t: Void) -> Type: + def visit_none_type(self, t: NoneType) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: return t - def visit_none_type(self, t: NoneTyp) -> Type: + def visit_erased_type(self, t: ErasedType) -> Type: + # This type should exist only temporarily during type inference + assert False, "Internal error: Unexpected erased type" + + def visit_deleted_type(self, t: DeletedType) -> Type: return t def visit_type_list(self, t: TypeList) -> Type: - self.fail('Invalid type', t) + # Parameters literal (Z[[int, str, Whatever]]) + if self.allow_param_spec_literals: + params = self.analyze_callable_args(t) + if params: + ts, kinds, names = params + # bind these types + return Parameters(self.anal_array(ts), kinds, names, line=t.line, column=t.column) + else: + return AnyType(TypeOfAny.from_error) + else: + self.fail( + 'Bracketed expression "[...]" is not valid as a type', t, code=codes.VALID_TYPE + ) + if len(t.items) == 1: + self.note('Did you mean "List[...]"?', t) + return AnyType(TypeOfAny.from_error) + + def visit_callable_argument(self, t: CallableArgument) -> Type: + self.fail("Invalid type", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) def visit_instance(self, t: Instance) -> Type: return t - def visit_type_var(self, t: TypeVar) -> Type: - raise RuntimeError('TypeVar is already analysed') + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # TODO: should we do something here? + return t + + def visit_type_var(self, t: TypeVarType) -> Type: + return t + + def visit_param_spec(self, t: ParamSpecType) -> Type: + return t + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: + return t + + def visit_unpack_type(self, t: UnpackType) -> Type: + if not self.allow_unpack: + self.fail(message_registry.INVALID_UNPACK_POSITION, t.type, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + self.allow_type_var_tuple = True + result = UnpackType(self.anal_type(t.type), from_star_syntax=t.from_star_syntax) + self.allow_type_var_tuple = False + return result + + def visit_parameters(self, t: Parameters) -> Type: + raise NotImplementedError("ParamSpec literals cannot have unbound TypeVars") + + def visit_callable_type( + self, t: CallableType, nested: bool = True, namespace: str = "" + ) -> Type: + # Every Callable can bind its own type variables, if they're not in the outer scope + # TODO: attach namespace for nested free type variables (these appear in return type only). + with self.tvar_scope_frame(namespace=namespace): + unpacked_kwargs = t.unpack_kwargs + if self.defining_alias: + variables = t.variables + else: + variables, _ = self.bind_function_type_variables(t, t) + type_guard = self.anal_type_guard(t.ret_type) + type_is = self.anal_type_is(t.ret_type) + arg_kinds = t.arg_kinds + if len(arg_kinds) >= 2 and arg_kinds[-2] == ARG_STAR and arg_kinds[-1] == ARG_STAR2: + arg_types = self.anal_array(t.arg_types[:-2], nested=nested) + [ + self.anal_star_arg_type(t.arg_types[-2], ARG_STAR, nested=nested), + self.anal_star_arg_type(t.arg_types[-1], ARG_STAR2, nested=nested), + ] + # If nested is True, it means we are analyzing a Callable[...] type, rather + # than a function definition type. We need to "unpack" ** TypedDict annotation + # here (for function definitions it is done in semanal). + if nested and isinstance(arg_types[-1], UnpackType): + # TODO: it would be better to avoid this get_proper_type() call. + unpacked = get_proper_type(arg_types[-1].type) + if isinstance(unpacked, TypedDictType): + arg_types[-1] = unpacked + unpacked_kwargs = True + arg_types = self.check_unpacks_in_list(arg_types) + else: + star_index = None + if ARG_STAR in arg_kinds: + star_index = arg_kinds.index(ARG_STAR) + star2_index = None + if ARG_STAR2 in arg_kinds: + star2_index = arg_kinds.index(ARG_STAR2) + arg_types = [] + for i, ut in enumerate(t.arg_types): + at = self.anal_type( + ut, nested=nested, allow_unpack=i in (star_index, star2_index) + ) + if nested and isinstance(at, UnpackType) and i == star_index: + # TODO: it would be better to avoid this get_proper_type() call. + p_at = get_proper_type(at.type) + if isinstance(p_at, TypedDictType) and not at.from_star_syntax: + # Automatically detect Unpack[Foo] in Callable as backwards + # compatible syntax for **Foo, if Foo is a TypedDict. + at = p_at + arg_kinds[i] = ARG_STAR2 + unpacked_kwargs = True + arg_types.append(at) + if nested: + arg_types = self.check_unpacks_in_list(arg_types) + # If there were multiple (invalid) unpacks, the arg types list will become shorter, + # we need to trim the kinds/names as well to avoid crashes. + arg_kinds = t.arg_kinds[: len(arg_types)] + arg_names = t.arg_names[: len(arg_types)] - def visit_callable(self, t: Callable) -> Type: - res = Callable(self.anal_array(t.arg_types), - t.arg_kinds, - t.arg_names, - t.ret_type.accept(self), - self.builtin_type('builtins.function'), - t.name, - self.anal_var_defs(t.variables), - self.anal_bound_vars(t.bound_vars), t.line, t.repr) + ret = t.copy_modified( + arg_types=arg_types, + arg_kinds=arg_kinds, + arg_names=arg_names, + ret_type=self.anal_type(t.ret_type, nested=nested), + # If the fallback isn't filled in yet, + # its type will be the falsey FakeInfo + fallback=(t.fallback if t.fallback.type else self.named_type("builtins.function")), + variables=self.anal_var_defs(variables), + type_guard=type_guard, + type_is=type_is, + unpack_kwargs=unpacked_kwargs, + ) + return ret - return res + def anal_type_guard(self, t: Type) -> Type | None: + if isinstance(t, UnboundType): + sym = self.lookup_qualified(t.name, t) + if sym is not None and sym.node is not None: + return self.anal_type_guard_arg(t, sym.node.fullname) + # TODO: What if it's an Instance? Then use t.type.fullname? + return None + + def anal_type_guard_arg(self, t: UnboundType, fullname: str) -> Type | None: + if fullname in ("typing_extensions.TypeGuard", "typing.TypeGuard"): + if len(t.args) != 1: + self.fail( + "TypeGuard must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return self.anal_type(t.args[0]) + return None + + def anal_type_is(self, t: Type) -> Type | None: + if isinstance(t, UnboundType): + sym = self.lookup_qualified(t.name, t) + if sym is not None and sym.node is not None: + return self.anal_type_is_arg(t, sym.node.fullname) + # TODO: What if it's an Instance? Then use t.type.fullname? + return None + + def anal_type_is_arg(self, t: UnboundType, fullname: str) -> Type | None: + if fullname in ("typing_extensions.TypeIs", "typing.TypeIs"): + if len(t.args) != 1: + self.fail("TypeIs must have exactly one type argument", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + return self.anal_type(t.args[0]) + return None + + def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type: + """Analyze signature argument type for *args and **kwargs argument.""" + if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args: + components = t.name.split(".") + tvar_name = ".".join(components[:-1]) + sym = self.lookup_qualified(tvar_name, t) + if sym is not None and isinstance(sym.node, ParamSpecExpr): + tvar_def = self.tvar_scope.get_binding(sym) + if isinstance(tvar_def, ParamSpecType): + if kind == ARG_STAR: + make_paramspec = paramspec_args + if components[-1] != "args": + self.fail( + f'Use "{tvar_name}.args" for variadic "*" parameter', + t, + code=codes.VALID_TYPE, + ) + elif kind == ARG_STAR2: + make_paramspec = paramspec_kwargs + if components[-1] != "kwargs": + self.fail( + f'Use "{tvar_name}.kwargs" for variadic "**" parameter', + t, + code=codes.VALID_TYPE, + ) + else: + assert False, kind + return make_paramspec( + tvar_def.name, + tvar_def.fullname, + tvar_def.id, + named_type_func=self.named_type, + line=t.line, + column=t.column, + ) + return self.anal_type(t, nested=nested, allow_unpack=True) + + def visit_overloaded(self, t: Overloaded) -> Type: + # Overloaded types are manually constructed in semanal.py by analyzing the + # AST and combining together the Callable types this visitor converts. + # + # So if we're ever asked to reanalyze an Overloaded type, we know it's + # fine to just return it as-is. + return t def visit_tuple_type(self, t: TupleType) -> Type: - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return AnyType() - fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple') - return TupleType(self.anal_array(t.items), - fallback, - t.line, t.repr) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line, t.repr) + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if t.implicit and not self.allow_tuple_literal: + self.fail("Syntax error in type annotation", t, code=codes.SYNTAX) + if len(t.items) == 0: + self.note( + "Suggestion: Use Tuple[()] instead of () for an empty tuple, or " + "None for a function without a return value", + t, + code=codes.SYNTAX, + ) + elif len(t.items) == 1: + self.note("Suggestion: Is there a spurious trailing comma?", t, code=codes.SYNTAX) + else: + self.note( + "Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn)", + t, + code=codes.SYNTAX, + ) + return AnyType(TypeOfAny.from_error) + + any_type = AnyType(TypeOfAny.special_form) + # If the fallback isn't filled in yet, its type will be the falsey FakeInfo + fallback = ( + t.partial_fallback + if t.partial_fallback.type + else self.named_type("builtins.tuple", [any_type]) + ) + return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line) + + def visit_typeddict_type(self, t: TypedDictType) -> Type: + req_keys = set() + readonly_keys = set() + items = {} + for item_name, item_type in t.items.items(): + # TODO: rework + analyzed = self.anal_type(item_type, allow_typed_dict_special_forms=True) + if isinstance(analyzed, RequiredType): + if analyzed.required: + req_keys.add(item_name) + analyzed = analyzed.item + else: + # Keys are required by default. + req_keys.add(item_name) + if isinstance(analyzed, ReadOnlyType): + readonly_keys.add(item_name) + analyzed = analyzed.item + items[item_name] = analyzed + if t.fallback.type is MISSING_FALLBACK: # anonymous/inline TypedDict + if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature: + self.fail( + "Inline TypedDict is experimental," + " must be enabled with --enable-incomplete-feature=InlineTypedDict", + t, + ) + required_keys = req_keys + fallback = self.named_type("typing._TypedDict") + for typ in t.extra_items_from: + analyzed = self.analyze_type(typ) + p_analyzed = get_proper_type(analyzed) + if not isinstance(p_analyzed, TypedDictType): + if not isinstance(p_analyzed, (AnyType, PlaceholderType)): + self.fail("Can only merge-in other TypedDict", t, code=codes.VALID_TYPE) + continue + for sub_item_name, sub_item_type in p_analyzed.items.items(): + if sub_item_name in items: + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(sub_item_name), t) + continue + items[sub_item_name] = sub_item_type + if sub_item_name in p_analyzed.required_keys: + req_keys.add(sub_item_name) + if sub_item_name in p_analyzed.readonly_keys: + readonly_keys.add(sub_item_name) + else: + required_keys = t.required_keys + fallback = t.fallback + return TypedDictType(items, required_keys, readonly_keys, fallback, t.line, t.column) + + def visit_raw_expression_type(self, t: RawExpressionType) -> Type: + # We should never see a bare Literal. We synthesize these raw literals + # in the earlier stages of semantic analysis, but those + # "fake literals" should always be wrapped in an UnboundType + # corresponding to 'Literal'. + # + # Note: if at some point in the distant future, we decide to + # make signatures like "foo(x: 20) -> None" legal, we can change + # this method so it generates and returns an actual LiteralType + # instead. + + if self.report_invalid_types: + if t.base_type_name in ("builtins.int", "builtins.bool"): + # The only time it makes sense to use an int or bool is inside of + # a literal type. + msg = f"Invalid type: try using Literal[{repr(t.literal_value)}] instead?" + elif t.base_type_name in ("builtins.float", "builtins.complex"): + # We special-case warnings for floats and complex numbers. + msg = f"Invalid type: {t.simple_name()} literals cannot be used as a type" + else: + # And in all other cases, we default to a generic error message. + # Note: the reason why we use a generic error message for strings + # but not ints or bools is because whenever we see an out-of-place + # string, it's unclear if the user meant to construct a literal type + # or just misspelled a regular type. So we avoid guessing. + msg = "Invalid type comment or annotation" + + self.fail(msg, t, code=codes.VALID_TYPE) + if t.note is not None: + self.note(t.note, t, code=codes.VALID_TYPE) + + return AnyType(TypeOfAny.from_error, line=t.line, column=t.column) + + def visit_literal_type(self, t: LiteralType) -> Type: + return t def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.anal_array(t.items), t.line, t.repr) - - def analyze_function_type(self, t: UnboundType) -> Type: - if len(t.args) != 2: - self.fail('Invalid function type', t) - if not isinstance(t.args[0], TypeList): - self.fail('Invalid function type', t) - return AnyType() - args = (cast(TypeList, t.args[0])).items - return Callable(self.anal_array(args), - [nodes.ARG_POS] * len(args), [None] * len(args), - ret_type=t.args[1].accept(self), - fallback=self.builtin_type('builtins.function')) - - def anal_array(self, a: List[Type]) -> List[Type]: - res = List[Type]() + if ( + t.uses_pep604_syntax is True + and t.is_evaluated is True + and not self.always_allow_new_syntax + and not self.options.python_version >= (3, 10) + ): + self.fail("X | Y syntax for unions requires Python 3.10", t, code=codes.SYNTAX) + return UnionType(self.anal_array(t.items), t.line, uses_pep604_syntax=t.uses_pep604_syntax) + + def visit_partial_type(self, t: PartialType) -> Type: + assert False, "Internal error: Unexpected partial type" + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + if self.allow_ellipsis or self.allow_param_spec_literals: + any_type = AnyType(TypeOfAny.explicit) + return Parameters( + [any_type, any_type], [ARG_STAR, ARG_STAR2], [None, None], is_ellipsis_args=True + ) + else: + self.fail('Unexpected "..."', t) + return AnyType(TypeOfAny.from_error) + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType.make_normalized(self.anal_type(t.item), line=t.line) + + def visit_placeholder_type(self, t: PlaceholderType) -> Type: + n = ( + None + # No dot in fullname indicates we are at function scope, and recursive + # types are not supported there anyway, so we just give up. + if not t.fullname or "." not in t.fullname + else self.api.lookup_fully_qualified(t.fullname) + ) + if not n or isinstance(n.node, PlaceholderNode): + self.api.defer() # Still incomplete + return t + else: + # TODO: Handle non-TypeInfo + assert isinstance(n.node, TypeInfo) + return self.analyze_type_with_type_info(n.node, t.args, t, False) + + def analyze_callable_args_for_paramspec( + self, callable_args: Type, ret_type: Type, fallback: Instance + ) -> CallableType | None: + """Construct a 'Callable[P, RET]', where P is ParamSpec, return None if we cannot.""" + if not isinstance(callable_args, UnboundType): + return None + sym = self.lookup_qualified(callable_args.name, callable_args) + if sym is None: + return None + tvar_def = self.tvar_scope.get_binding(sym) + if not isinstance(tvar_def, ParamSpecType): + if ( + tvar_def is None + and self.allow_unbound_tvars + and isinstance(sym.node, ParamSpecExpr) + ): + # We are analyzing this type in runtime context (e.g. as type application). + # If it is not valid as a type in this position an error will be given later. + return callable_with_ellipsis( + AnyType(TypeOfAny.explicit), ret_type=ret_type, fallback=fallback + ) + return None + elif ( + self.defining_alias + and self.not_declared_in_type_params(tvar_def.name) + and tvar_def not in self.allowed_alias_tvars + ): + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{tvar_def.name}"' + ) + else: + msg = f'ParamSpec "{tvar_def.name}" is not included in type_params' + self.fail(msg, callable_args, code=codes.VALID_TYPE) + return callable_with_ellipsis( + AnyType(TypeOfAny.special_form), ret_type=ret_type, fallback=fallback + ) + + return CallableType( + [ + paramspec_args( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + paramspec_kwargs( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + ], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + ) + + def analyze_callable_args_for_concatenate( + self, callable_args: Type, ret_type: Type, fallback: Instance + ) -> CallableType | AnyType | None: + """Construct a 'Callable[C, RET]', where C is Concatenate[..., P], returning None if we + cannot. + """ + if not isinstance(callable_args, UnboundType): + return None + sym = self.lookup_qualified(callable_args.name, callable_args) + if sym is None: + return None + if sym.node is None: + return None + if sym.node.fullname not in ("typing_extensions.Concatenate", "typing.Concatenate"): + return None + + tvar_def = self.anal_type(callable_args, allow_param_spec=True) + if not isinstance(tvar_def, (ParamSpecType, Parameters)): + if self.allow_unbound_tvars and isinstance(tvar_def, UnboundType): + sym = self.lookup_qualified(tvar_def.name, callable_args) + if sym is not None and isinstance(sym.node, ParamSpecExpr): + # We are analyzing this type in runtime context (e.g. as type application). + # If it is not valid as a type in this position an error will be given later. + return callable_with_ellipsis( + AnyType(TypeOfAny.explicit), ret_type=ret_type, fallback=fallback + ) + # Error was already given, so prevent further errors. + return AnyType(TypeOfAny.from_error) + if isinstance(tvar_def, Parameters): + # This comes from Concatenate[int, ...] + return CallableType( + arg_types=tvar_def.arg_types, + arg_names=tvar_def.arg_names, + arg_kinds=tvar_def.arg_kinds, + ret_type=ret_type, + fallback=fallback, + from_concatenate=True, + ) + + # ick, CallableType should take ParamSpecType + prefix = tvar_def.prefix + # we don't set the prefix here as generic arguments will get updated at some point + # in the future. CallableType.param_spec() accounts for this. + return CallableType( + [ + *prefix.arg_types, + paramspec_args( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + paramspec_kwargs( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + ], + [*prefix.arg_kinds, nodes.ARG_STAR, nodes.ARG_STAR2], + [*prefix.arg_names, None, None], + ret_type=ret_type, + fallback=fallback, + from_concatenate=True, + ) + + def analyze_callable_type(self, t: UnboundType) -> Type: + fallback = self.named_type("builtins.function") + if len(t.args) == 0: + # Callable (bare). Treat as Callable[..., Any]. + any_type = self.get_omitted_any(t) + ret = callable_with_ellipsis(any_type, any_type, fallback) + elif len(t.args) == 2: + callable_args = t.args[0] + ret_type = t.args[1] + if isinstance(callable_args, TypeList): + # Callable[[ARG, ...], RET] (ordinary callable type) + analyzed_args = self.analyze_callable_args(callable_args) + if analyzed_args is None: + return AnyType(TypeOfAny.from_error) + args, kinds, names = analyzed_args + ret = CallableType(args, kinds, names, ret_type=ret_type, fallback=fallback) + elif isinstance(callable_args, EllipsisType): + # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) + ret = callable_with_ellipsis( + AnyType(TypeOfAny.explicit), ret_type=ret_type, fallback=fallback + ) + else: + # Callable[P, RET] (where P is ParamSpec) + with self.tvar_scope_frame(namespace=""): + # Temporarily bind ParamSpecs to allow code like this: + # my_fun: Callable[Q, Foo[Q]] + # We usually do this later in visit_callable_type(), but the analysis + # below happens at very early stage. + variables = [] + for name, tvar_expr in self.find_type_var_likes(callable_args): + variables.append(self.tvar_scope.bind_new(name, tvar_expr)) + maybe_ret = self.analyze_callable_args_for_paramspec( + callable_args, ret_type, fallback + ) or self.analyze_callable_args_for_concatenate( + callable_args, ret_type, fallback + ) + if isinstance(maybe_ret, CallableType): + maybe_ret = maybe_ret.copy_modified(variables=variables) + if maybe_ret is None: + # Callable[?, RET] (where ? is something invalid) + self.fail( + "The first argument to Callable must be a " + 'list of types, parameter specification, or "..."', + t, + code=codes.VALID_TYPE, + ) + self.note( + "See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas", + t, + ) + return AnyType(TypeOfAny.from_error) + elif isinstance(maybe_ret, AnyType): + return maybe_ret + ret = maybe_ret + else: + if self.options.disallow_any_generics: + self.fail('Please use "Callable[[], ]"', t) + else: + self.fail('Please use "Callable[[], ]" or "Callable"', t) + return AnyType(TypeOfAny.from_error) + assert isinstance(ret, CallableType) + return ret.accept(self) + + def refers_to_full_names(self, arg: UnboundType, names: Sequence[str]) -> bool: + sym = self.lookup_qualified(arg.name, arg) + if sym is not None: + if sym.fullname in names: + return True + return False + + def analyze_callable_args( + self, arglist: TypeList + ) -> tuple[list[Type], list[ArgKind], list[str | None]] | None: + args: list[Type] = [] + kinds: list[ArgKind] = [] + names: list[str | None] = [] + seen_unpack = False + unpack_types: list[Type] = [] + invalid_unpacks: list[Type] = [] + second_unpack_last = False + for i, arg in enumerate(arglist.items): + if isinstance(arg, CallableArgument): + args.append(arg.typ) + names.append(arg.name) + if arg.constructor is None: + return None + found = self.lookup_qualified(arg.constructor, arg) + if found is None: + # Looking it up already put an error message in + return None + elif found.fullname not in ARG_KINDS_BY_CONSTRUCTOR: + self.fail(f'Invalid argument constructor "{found.fullname}"', arg) + return None + else: + assert found.fullname is not None + kind = ARG_KINDS_BY_CONSTRUCTOR[found.fullname] + kinds.append(kind) + if arg.name is not None and kind.is_star(): + self.fail(f"{arg.constructor} arguments should not have names", arg) + return None + elif ( + isinstance(arg, UnboundType) + and self.refers_to_full_names(arg, ("typing_extensions.Unpack", "typing.Unpack")) + or isinstance(arg, UnpackType) + ): + if seen_unpack: + # Multiple unpacks, preserve them, so we can give an error later. + if i == len(arglist.items) - 1 and not invalid_unpacks: + # Special case: if there are just two unpacks, and the second one appears + # as last type argument, it can be still valid, if the second unpacked type + # is a TypedDict. This should be checked by the caller. + second_unpack_last = True + invalid_unpacks.append(arg) + continue + seen_unpack = True + unpack_types.append(arg) + else: + if seen_unpack: + unpack_types.append(arg) + else: + args.append(arg) + kinds.append(ARG_POS) + names.append(None) + if seen_unpack: + if len(unpack_types) == 1: + args.append(unpack_types[0]) + else: + first = unpack_types[0] + if isinstance(first, UnpackType): + # UnpackType doesn't have its own line/column numbers, + # so use the unpacked type for error messages. + first = first.type + args.append( + UnpackType(self.tuple_type(unpack_types, line=first.line, column=first.column)) + ) + kinds.append(ARG_STAR) + names.append(None) + for arg in invalid_unpacks: + args.append(arg) + kinds.append(ARG_STAR2 if second_unpack_last else ARG_STAR) + names.append(None) + # Note that arglist below is only used for error context. + check_arg_names(names, [arglist] * len(args), self.fail, "Callable") + check_arg_kinds(kinds, [arglist] * len(args), self.fail) + return args, kinds, names + + def analyze_literal_type(self, t: UnboundType) -> Type: + if len(t.args) == 0: + self.fail("Literal[...] must have at least one parameter", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + + output: list[Type] = [] + for i, arg in enumerate(t.args): + analyzed_types = self.analyze_literal_param(i + 1, arg, t) + if analyzed_types is None: + return AnyType(TypeOfAny.from_error) + else: + output.extend(analyzed_types) + return UnionType.make_union(output, line=t.line) + + def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] | None: + # This UnboundType was originally defined as a string. + if ( + isinstance(arg, ProperType) + and isinstance(arg, (UnboundType, UnionType)) + and arg.original_str_expr is not None + ): + assert arg.original_str_fallback is not None + return [ + LiteralType( + value=arg.original_str_expr, + fallback=self.named_type(arg.original_str_fallback), + line=arg.line, + column=arg.column, + ) + ] + + # If arg is an UnboundType that was *not* originally defined as + # a string, try expanding it in case it's a type alias or something. + if isinstance(arg, UnboundType): + self.nesting_level += 1 + try: + arg = self.visit_unbound_type(arg, defining_literal=True) + finally: + self.nesting_level -= 1 + + # Literal[...] cannot contain Any. Give up and add an error message + # (if we haven't already). + arg = get_proper_type(arg) + if isinstance(arg, AnyType): + # Note: We can encounter Literals containing 'Any' under three circumstances: + # + # 1. If the user attempts use an explicit Any as a parameter + # 2. If the user is trying to use an enum value imported from a module with + # no type hints, giving it an implicit type of 'Any' + # 3. If there's some other underlying problem with the parameter. + # + # We report an error in only the first two cases. In the third case, we assume + # some other region of the code has already reported a more relevant error. + # + # TODO: Once we start adding support for enums, make sure we report a custom + # error for case 2 as well. + if arg.type_of_any not in (TypeOfAny.from_error, TypeOfAny.special_form): + self.fail( + f'Parameter {idx} of Literal[...] cannot be of type "Any"', + ctx, + code=codes.VALID_TYPE, + ) + return None + elif isinstance(arg, RawExpressionType): + # A raw literal. Convert it directly into a literal if we can. + if arg.literal_value is None: + name = arg.simple_name() + if name in ("float", "complex"): + msg = f'Parameter {idx} of Literal[...] cannot be of type "{name}"' + else: + msg = "Invalid type: Literal[...] cannot contain arbitrary expressions" + self.fail(msg, ctx, code=codes.VALID_TYPE) + # Note: we deliberately ignore arg.note here: the extra info might normally be + # helpful, but it generally won't make sense in the context of a Literal[...]. + return None + + # Remap bytes and unicode into the appropriate type for the correct Python version + fallback = self.named_type(arg.base_type_name) + assert isinstance(fallback, Instance) + return [LiteralType(arg.literal_value, fallback, line=arg.line, column=arg.column)] + elif isinstance(arg, (NoneType, LiteralType)): + # Types that we can just add directly to the literal/potential union of literals. + return [arg] + elif isinstance(arg, Instance) and arg.last_known_value is not None: + # Types generated from declarations like "var: Final = 4". + return [arg.last_known_value] + elif isinstance(arg, UnionType): + out = [] + for union_arg in arg.items: + union_result = self.analyze_literal_param(idx, union_arg, ctx) + if union_result is None: + return None + out.extend(union_result) + return out + else: + self.fail(f"Parameter {idx} of Literal[...] is invalid", ctx, code=codes.VALID_TYPE) + return None + + def analyze_type(self, t: Type) -> Type: + return t.accept(self) + + def fail(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + self.fail_func(msg, ctx, code=code) + + def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + self.note_func(msg, ctx, code=code) + + @contextmanager + def tvar_scope_frame(self, namespace: str) -> Iterator[None]: + old_scope = self.tvar_scope + self.tvar_scope = self.tvar_scope.method_frame(namespace) + yield + self.tvar_scope = old_scope + + def find_type_var_likes(self, t: Type) -> TypeVarLikeList: + visitor = FindTypeVarVisitor(self.api, self.tvar_scope) + t.accept(visitor) + return visitor.type_var_likes + + def infer_type_variables( + self, type: CallableType + ) -> tuple[list[tuple[str, TypeVarLikeExpr]], bool]: + """Infer type variables from a callable. + + Return tuple with these items: + - list of unique type variables referred to in a callable + - whether there is a reference to the Self type + """ + visitor = FindTypeVarVisitor(self.api, self.tvar_scope) + for arg in type.arg_types: + arg.accept(visitor) + + # When finding type variables in the return type of a function, don't + # look inside Callable types. Type variables only appearing in + # functions in the return type belong to those functions, not the + # function we're currently analyzing. + visitor.include_callables = False + type.ret_type.accept(visitor) + + return visitor.type_var_likes, visitor.has_self_type + + def bind_function_type_variables( + self, fun_type: CallableType, defn: Context + ) -> tuple[Sequence[TypeVarLikeType], bool]: + """Find the type variables of the function type and bind them in our tvar_scope""" + has_self_type = False + if fun_type.variables: + defs = [] + for var in fun_type.variables: + if self.api.type and self.api.type.self_type and var == self.api.type.self_type: + has_self_type = True + continue + var_node = self.lookup_qualified(var.name, defn) + assert var_node, "Binding for function type variable not found within function" + var_expr = var_node.node + assert isinstance(var_expr, TypeVarLikeExpr) + binding = self.tvar_scope.bind_new(var.name, var_expr) + defs.append(binding) + return defs, has_self_type + typevars, has_self_type = self.infer_type_variables(fun_type) + # Do not define a new type variable if already defined in scope. + typevars = [ + (name, tvar) for name, tvar in typevars if not self.is_defined_type_var(name, defn) + ] + defs = [] + for name, tvar in typevars: + if not self.tvar_scope.allow_binding(tvar.fullname): + self.fail( + f'Type variable "{name}" is bound by an outer class', + defn, + code=codes.VALID_TYPE, + ) + binding = self.tvar_scope.bind_new(name, tvar) + defs.append(binding) + + return defs, has_self_type + + def is_defined_type_var(self, tvar: str, context: Context) -> bool: + tvar_node = self.lookup_qualified(tvar, context) + if not tvar_node: + return False + return self.tvar_scope.get_binding(tvar_node) is not None + + def anal_array( + self, + a: Iterable[Type], + nested: bool = True, + *, + allow_param_spec: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + ) -> list[Type]: + old_allow_param_spec_literals = self.allow_param_spec_literals + self.allow_param_spec_literals = allow_param_spec_literals + res: list[Type] = [] for t in a: - res.append(t.accept(self)) - return res - - def anal_bound_vars(self, - a: List[Tuple[int, Type]]) -> List[Tuple[int, Type]]: - res = List[Tuple[int, Type]]() - for id, t in a: - res.append((id, t.accept(self))) - return res - - def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: - a = List[TypeVarDef]() - for vd in var_defs: - a.append(TypeVarDef(vd.name, vd.id, self.anal_array(vd.values), - vd.upper_bound.accept(self), - vd.line, vd.repr)) - return a - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fqn_func(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - -class TypeAnalyserPass3(TypeVisitor[None]): - """Analyze type argument counts and values of generic types. - - This is semantic analysis pass 3 for types. - - Perform these operations: - - * Report error for invalid type argument counts, such as List[x, y]. - * Make implicit Any type argumenents explicit my modifying types - in-place. For example, modify Foo into Foo[Any] if Foo expects a single - type argument. - * If a type variable has a value restriction, ensure that the value is - valid. For example, reject IO[int] if the type argument must be str - or bytes. - - We can't do this earlier than the third pass, since type argument counts - are only determined in pass 2, and we have to support forward references - to types. + res.append( + self.anal_type( + t, nested, allow_param_spec=allow_param_spec, allow_unpack=allow_unpack + ) + ) + self.allow_param_spec_literals = old_allow_param_spec_literals + return self.check_unpacks_in_list(res) + + def anal_type( + self, + t: Type, + nested: bool = True, + *, + allow_param_spec: bool = False, + allow_unpack: bool = False, + allow_ellipsis: bool = False, + allow_typed_dict_special_forms: bool = False, + ) -> Type: + if nested: + self.nesting_level += 1 + old_allow_typed_dict_special_forms = self.allow_typed_dict_special_forms + self.allow_typed_dict_special_forms = allow_typed_dict_special_forms + old_allow_ellipsis = self.allow_ellipsis + self.allow_ellipsis = allow_ellipsis + old_allow_unpack = self.allow_unpack + self.allow_unpack = allow_unpack + try: + analyzed = t.accept(self) + finally: + if nested: + self.nesting_level -= 1 + self.allow_typed_dict_special_forms = old_allow_typed_dict_special_forms + self.allow_ellipsis = old_allow_ellipsis + self.allow_unpack = old_allow_unpack + if ( + not allow_param_spec + and isinstance(analyzed, ParamSpecType) + and analyzed.flavor == ParamSpecFlavor.BARE + ): + if analyzed.prefix.arg_types: + self.fail("Invalid location for Concatenate", t, code=codes.VALID_TYPE) + self.note("You can use Concatenate as the first argument to Callable", t) + analyzed = AnyType(TypeOfAny.from_error) + else: + self.fail( + INVALID_PARAM_SPEC_LOCATION.format(format_type(analyzed, self.options)), + t, + code=codes.VALID_TYPE, + ) + self.note( + INVALID_PARAM_SPEC_LOCATION_NOTE.format(analyzed.name), + t, + code=codes.VALID_TYPE, + ) + analyzed = AnyType(TypeOfAny.from_error) + return analyzed + + def anal_var_def(self, var_def: TypeVarLikeType) -> TypeVarLikeType: + if isinstance(var_def, TypeVarType): + return TypeVarType( + name=var_def.name, + fullname=var_def.fullname, + id=var_def.id, + values=self.anal_array(var_def.values), + upper_bound=var_def.upper_bound.accept(self), + default=var_def.default.accept(self), + variance=var_def.variance, + line=var_def.line, + column=var_def.column, + ) + else: + return var_def + + def anal_var_defs(self, var_defs: Sequence[TypeVarLikeType]) -> list[TypeVarLikeType]: + return [self.anal_var_def(vd) for vd in var_defs] + + def named_type( + self, + fully_qualified_name: str, + args: list[Type] | None = None, + line: int = -1, + column: int = -1, + ) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + assert isinstance(node.node, TypeInfo) + any_type = AnyType(TypeOfAny.special_form) + if args is not None: + args = self.check_unpacks_in_list(args) + return Instance( + node.node, args or [any_type] * len(node.node.defn.type_vars), line=line, column=column + ) + + def check_unpacks_in_list(self, items: list[Type]) -> list[Type]: + new_items: list[Type] = [] + num_unpacks = 0 + final_unpack = None + for item in items: + # TODO: handle forward references here, they appear as Unpack[Any]. + if isinstance(item, UnpackType) and not isinstance( + get_proper_type(item.type), TupleType + ): + if not num_unpacks: + new_items.append(item) + num_unpacks += 1 + final_unpack = item + else: + new_items.append(item) + + if num_unpacks > 1: + assert final_unpack is not None + self.fail("More than one Unpack in a type is not allowed", final_unpack) + return new_items + + def tuple_type(self, items: list[Type], line: int, column: int) -> TupleType: + any_type = AnyType(TypeOfAny.special_form) + return TupleType( + items, fallback=self.named_type("builtins.tuple", [any_type]), line=line, column=column + ) + + +TypeVarLikeList = List[Tuple[str, TypeVarLikeExpr]] + + +class MsgCallback(Protocol): + def __call__(self, __msg: str, __ctx: Context, *, code: ErrorCode | None = None) -> None: ... + + +def get_omitted_any( + disallow_any: bool, + fail: MsgCallback, + note: MsgCallback, + orig_type: Type, + options: Options, + fullname: str | None = None, + unexpanded_type: Type | None = None, +) -> AnyType: + if disallow_any: + nongen_builtins = get_nongen_builtins(options.python_version) + if fullname in nongen_builtins: + typ = orig_type + # We use a dedicated error message for builtin generics (as the most common case). + alternative = nongen_builtins[fullname] + fail( + message_registry.IMPLICIT_GENERIC_ANY_BUILTIN.format(alternative), + typ, + code=codes.TYPE_ARG, + ) + else: + typ = unexpanded_type or orig_type + type_str = typ.name if isinstance(typ, UnboundType) else format_type_bare(typ, options) + + fail( + message_registry.BARE_GENERIC.format(quote_type_string(type_str)), + typ, + code=codes.TYPE_ARG, + ) + base_type = get_proper_type(orig_type) + base_fullname = ( + base_type.type.fullname if isinstance(base_type, Instance) else fullname + ) + # Ideally, we'd check whether the type is quoted or `from __future__ annotations` + # is set before issuing this note + if ( + options.python_version < (3, 9) + and base_fullname in GENERIC_STUB_NOT_AT_RUNTIME_TYPES + ): + # Recommend `from __future__ import annotations` or to put type in quotes + # (string literal escaping) for classes not generic at runtime + note( + "Subscripting classes that are not generic at runtime may require " + "escaping, see https://mypy.readthedocs.io/en/stable/runtime_troubles.html" + "#not-generic-runtime", + typ, + code=codes.TYPE_ARG, + ) + + any_type = AnyType(TypeOfAny.from_error, line=typ.line, column=typ.column) + else: + any_type = AnyType( + TypeOfAny.from_omitted_generics, line=orig_type.line, column=orig_type.column + ) + return any_type + + +def fix_type_var_tuple_argument(t: Instance) -> None: + if t.type.has_type_var_tuple_type: + args = list(t.args) + assert t.type.type_var_tuple_prefix is not None + tvt = t.type.defn.type_vars[t.type.type_var_tuple_prefix] + assert isinstance(tvt, TypeVarTupleType) + args[t.type.type_var_tuple_prefix] = UnpackType( + Instance(tvt.tuple_fallback.type, [args[t.type.type_var_tuple_prefix]]) + ) + t.args = tuple(args) + + +def fix_instance( + t: Instance, + fail: MsgCallback, + note: MsgCallback, + disallow_any: bool, + options: Options, + use_generic_error: bool = False, + unexpanded_type: Type | None = None, +) -> None: + """Fix a malformed instance by replacing all type arguments with TypeVar default or Any. + + Also emit a suitable error if this is not due to implicit Any's. """ + arg_count = len(t.args) + min_tv_count = sum(not tv.has_default() for tv in t.type.defn.type_vars) + max_tv_count = len(t.type.type_vars) + if arg_count < min_tv_count or arg_count > max_tv_count: + # Don't use existing args if arg_count doesn't match + if arg_count > max_tv_count: + # Already wrong arg count error, don't emit missing type parameters error as well. + disallow_any = False + t.args = () + arg_count = 0 - def __init__(self, fail_func: Function[[str, Context], None]) -> None: - self.fail = fail_func + args: list[Type] = [*(t.args[:max_tv_count])] + any_type: AnyType | None = None + env: dict[TypeVarId, Type] = {} - def visit_instance(self, t: Instance) -> None: - info = t.type - # Check type argument count. - if len(t.args) != len(info.type_vars): - if len(t.args) == 0: - # Insert implicit 'Any' type arguments. - t.args = [AnyType()] * len(info.type_vars) - return - # Invalid number of type parameters. - n = len(info.type_vars) - s = '{} type arguments'.format(n) - if n == 0: - s = 'no type arguments' - elif n == 1: - s = '1 type argument' - act = str(len(t.args)) - if act == '0': - act = 'none' - self.fail('"{}" expects {}, but {} given'.format( - info.name(), s, act), t) - elif info.defn.type_vars: - # Check type argument values. - for arg, typevar in zip(t.args, info.defn.type_vars): - if typevar.values: - if isinstance(arg, TypeVar): - arg_values = arg.values - if not arg_values: - self.fail('Type variable "{}" not valid as type ' - 'argument value for "{}"'.format( - arg.name, info.name()), t) - continue - else: - arg_values = [arg] - self.check_type_var_values(info, arg_values, - typevar.values, t) - for arg in t.args: - arg.accept(self) - - def check_type_var_values(self, type: TypeInfo, actuals: List[Type], - valids: List[Type], context: Context) -> None: - for actual in actuals: - if (not isinstance(actual, AnyType) and - not any(is_same_type(actual, value) for value in valids)): - self.fail('Invalid type argument value for "{}"'.format( - type.name()), context) - - def visit_callable(self, t: Callable) -> None: - t.ret_type.accept(self) - for arg_type in t.arg_types: - arg_type.accept(self) + for tv, arg in itertools.zip_longest(t.type.defn.type_vars, t.args, fillvalue=None): + if tv is None: + continue + if arg is None: + if tv.has_default(): + arg = tv.default + else: + if any_type is None: + fullname = None if use_generic_error else t.type.fullname + any_type = get_omitted_any( + disallow_any, fail, note, t, options, fullname, unexpanded_type + ) + arg = any_type + args.append(arg) + env[tv.id] = arg + t.args = tuple(args) + fix_type_var_tuple_argument(t) + if not t.type.has_type_var_tuple_type: + with state.strict_optional_set(options.strict_optional): + fixed = expand_type(t, env) + assert isinstance(fixed, Instance) + t.args = fixed.args - def visit_tuple_type(self, t: TupleType) -> None: - for item in t.items: - item.accept(self) - def visit_union_type(self, t: UnionType) -> None: - for item in t.items: - item.accept(self) +def instantiate_type_alias( + node: TypeAlias, + args: list[Type], + fail: MsgCallback, + no_args: bool, + ctx: Context, + options: Options, + *, + unexpanded_type: Type | None = None, + disallow_any: bool = False, + use_standard_error: bool = False, + empty_tuple_index: bool = False, +) -> Type: + """Create an instance of a (generic) type alias from alias node and type arguments. + + We are following the rules outlined in TypeAlias docstring. + Here: + node: type alias node (definition) + args: type arguments (types to be substituted in place of type variables + when expanding the alias) + fail: error reporter callback + no_args: whether original definition used a bare generic `A = List` + ctx: context where expansion happens + unexpanded_type, disallow_any, use_standard_error: used to customize error messages + """ + # Type aliases are special, since they can be expanded during semantic analysis, + # so we need to normalize them as soon as possible. + # TODO: can this cause an infinite recursion? + args = flatten_nested_tuples(args) + if any(unknown_unpack(a) for a in args): + # This type is not ready to be validated, because of unknown total count. + # Note that we keep the kind of Any for consistency. + return set_any_tvars(node, [], ctx.line, ctx.column, options, special_form=True) + + max_tv_count = len(node.alias_tvars) + act_len = len(args) + if ( + max_tv_count > 0 + and act_len == 0 + and not (empty_tuple_index and node.tvar_tuple_index is not None) + ): + # Interpret bare Alias same as normal generic, i.e., Alias[Any, Any, ...] + return set_any_tvars( + node, + args, + ctx.line, + ctx.column, + options, + disallow_any=disallow_any, + fail=fail, + unexpanded_type=unexpanded_type, + ) + if max_tv_count == 0 and act_len == 0: + if no_args: + assert isinstance(node.target, Instance) # type: ignore[misc] + # Note: this is the only case where we use an eager expansion. See more info about + # no_args aliases like L = List in the docstring for TypeAlias class. + return Instance(node.target.type, [], line=ctx.line, column=ctx.column) + return TypeAliasType(node, [], line=ctx.line, column=ctx.column) + if ( + max_tv_count == 0 + and act_len > 0 + and isinstance(node.target, Instance) # type: ignore[misc] + and no_args + ): + tp = Instance(node.target.type, args) + tp.line = ctx.line + tp.column = ctx.column + return tp + if node.tvar_tuple_index is None: + if any(isinstance(a, UnpackType) for a in args): + # A variadic unpack in fixed size alias (fixed unpacks must be flattened by the caller) + fail(message_registry.INVALID_UNPACK_POSITION, ctx, code=codes.VALID_TYPE) + return set_any_tvars(node, [], ctx.line, ctx.column, options, from_error=True) + min_tv_count = sum(not tv.has_default() for tv in node.alias_tvars) + fill_typevars = act_len != max_tv_count + correct = min_tv_count <= act_len <= max_tv_count + else: + min_tv_count = sum( + not tv.has_default() and not isinstance(tv, TypeVarTupleType) + for tv in node.alias_tvars + ) + correct = act_len >= min_tv_count + for a in args: + if isinstance(a, UnpackType): + unpacked = get_proper_type(a.type) + if isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple": + # Variadic tuple is always correct. + correct = True + fill_typevars = not correct + if fill_typevars: + if not correct: + if use_standard_error: + # This is used if type alias is an internal representation of another type, + # for example a generic TypedDict or NamedTuple. + msg = wrong_type_arg_count(max_tv_count, max_tv_count, str(act_len), node.name) + else: + if node.tvar_tuple_index is not None: + msg = ( + "Bad number of arguments for type alias," + f" expected at least {min_tv_count}, given {act_len}" + ) + elif min_tv_count != max_tv_count: + msg = ( + "Bad number of arguments for type alias," + f" expected between {min_tv_count} and {max_tv_count}, given {act_len}" + ) + else: + msg = ( + "Bad number of arguments for type alias," + f" expected {min_tv_count}, given {act_len}" + ) + fail(msg, ctx, code=codes.TYPE_ARG) + args = [] + return set_any_tvars(node, args, ctx.line, ctx.column, options, from_error=True) + elif node.tvar_tuple_index is not None: + # We also need to check if we are not performing a type variable tuple split. + unpack = find_unpack_in_list(args) + if unpack is not None: + unpack_arg = args[unpack] + assert isinstance(unpack_arg, UnpackType) + if isinstance(unpack_arg.type, TypeVarTupleType): + exp_prefix = node.tvar_tuple_index + act_prefix = unpack + exp_suffix = len(node.alias_tvars) - node.tvar_tuple_index - 1 + act_suffix = len(args) - unpack - 1 + if act_prefix < exp_prefix or act_suffix < exp_suffix: + fail("TypeVarTuple cannot be split", ctx, code=codes.TYPE_ARG) + return set_any_tvars(node, [], ctx.line, ctx.column, options, from_error=True) + # TODO: we need to check args validity w.r.t alias.alias_tvars. + # Otherwise invalid instantiations will be allowed in runtime context. + # Note: in type context, these will be still caught by semanal_typeargs. + typ = TypeAliasType(node, args, ctx.line, ctx.column) + assert typ.alias is not None + # HACK: Implement FlexibleAlias[T, typ] by expanding it to typ here. + if ( + isinstance(typ.alias.target, Instance) # type: ignore[misc] + and typ.alias.target.type.fullname == "mypy_extensions.FlexibleAlias" + ): + exp = get_proper_type(typ) + assert isinstance(exp, Instance) + return exp.args[-1] + return typ + + +def set_any_tvars( + node: TypeAlias, + args: list[Type], + newline: int, + newcolumn: int, + options: Options, + *, + from_error: bool = False, + disallow_any: bool = False, + special_form: bool = False, + fail: MsgCallback | None = None, + unexpanded_type: Type | None = None, +) -> TypeAliasType: + if from_error or disallow_any: + type_of_any = TypeOfAny.from_error + elif special_form: + type_of_any = TypeOfAny.special_form + else: + type_of_any = TypeOfAny.from_omitted_generics + any_type = AnyType(type_of_any, line=newline, column=newcolumn) + + env: dict[TypeVarId, Type] = {} + used_any_type = False + has_type_var_tuple_type = False + for tv, arg in itertools.zip_longest(node.alias_tvars, args, fillvalue=None): + if tv is None: + continue + if arg is None: + if tv.has_default(): + arg = tv.default + else: + arg = any_type + used_any_type = True + if isinstance(tv, TypeVarTupleType): + # TODO Handle TypeVarTuple defaults + has_type_var_tuple_type = True + arg = UnpackType(Instance(tv.tuple_fallback.type, [any_type])) + args.append(arg) + env[tv.id] = arg + t = TypeAliasType(node, args, newline, newcolumn) + if not has_type_var_tuple_type: + with state.strict_optional_set(options.strict_optional): + fixed = expand_type(t, env) + assert isinstance(fixed, TypeAliasType) + t.args = fixed.args + + if used_any_type and disallow_any and node.alias_tvars: + assert fail is not None + if unexpanded_type: + type_str = ( + unexpanded_type.name + if isinstance(unexpanded_type, UnboundType) + else format_type_bare(unexpanded_type, options) + ) + else: + type_str = node.name + + fail( + message_registry.BARE_GENERIC.format(quote_type_string(type_str)), + Context(newline, newcolumn), + code=codes.TYPE_ARG, + ) + return t + + +class DivergingAliasDetector(TrivialSyntheticTypeTranslator): + """See docstring of detect_diverging_alias() for details.""" + + # TODO: this doesn't really need to be a translator, but we don't have a trivial visitor. + def __init__( + self, + seen_nodes: set[TypeAlias], + lookup: Callable[[str, Context], SymbolTableNode | None], + scope: TypeVarLikeScope, + ) -> None: + super().__init__() + self.seen_nodes = seen_nodes + self.lookup = lookup + self.scope = scope + self.diverging = False + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + assert t.alias is not None, f"Unfixed type alias {t.type_ref}" + if t.alias in self.seen_nodes: + for arg in t.args: + if not ( + isinstance(arg, TypeVarLikeType) + or isinstance(arg, UnpackType) + and isinstance(arg.type, TypeVarLikeType) + ) and has_type_vars(arg): + self.diverging = True + return t + # All clear for this expansion chain. + return t + new_nodes = self.seen_nodes | {t.alias} + visitor = DivergingAliasDetector(new_nodes, self.lookup, self.scope) + _ = get_proper_type(t).accept(visitor) + if visitor.diverging: + self.diverging = True + return t + + +def detect_diverging_alias( + node: TypeAlias, + target: Type, + lookup: Callable[[str, Context], SymbolTableNode | None], + scope: TypeVarLikeScope, +) -> bool: + """This detects type aliases that will diverge during type checking. + + For example F = Something[..., F[List[T]]]. At each expansion step this will produce + *new* type aliases: e.g. F[List[int]], F[List[List[int]]], etc. So we can't detect + recursion. It is a known problem in the literature, recursive aliases and generic types + don't always go well together. It looks like there is no known systematic solution yet. + + # TODO: should we handle such aliases using type_recursion counter and some large limit? + They may be handy in rare cases, e.g. to express a union of non-mixed nested lists: + Nested = Union[T, Nested[List[T]]] ~> Union[T, List[T], List[List[T]], ...] + """ + visitor = DivergingAliasDetector({node}, lookup, scope) + _ = target.accept(visitor) + return visitor.diverging + + +def check_for_explicit_any( + typ: Type | None, + options: Options, + is_typeshed_stub: bool, + msg: MessageBuilder, + context: Context, +) -> None: + if options.disallow_any_explicit and not is_typeshed_stub and typ and has_explicit_any(typ): + msg.explicit_any(context) + + +def has_explicit_any(t: Type) -> bool: + """ + Whether this type is or type it contains is an Any coming from explicit type annotation + """ + return t.accept(HasExplicitAny()) + + +class HasExplicitAny(TypeQuery[bool]): + def __init__(self) -> None: + super().__init__(any) + + def visit_any(self, t: AnyType) -> bool: + return t.type_of_any == TypeOfAny.explicit + + def visit_typeddict_type(self, t: TypedDictType) -> bool: + # typeddict is checked during TypedDict declaration, so don't typecheck it here. + return False + + +def has_any_from_unimported_type(t: Type) -> bool: + """Return true if this type is Any because an import was not followed. + + If type t is such Any type or has type arguments that contain such Any type + this function will return true. + """ + return t.accept(HasAnyFromUnimportedType()) + + +class HasAnyFromUnimportedType(BoolTypeQuery): + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + + def visit_any(self, t: AnyType) -> bool: + return t.type_of_any == TypeOfAny.from_unimported_type + + def visit_typeddict_type(self, t: TypedDictType) -> bool: + # typeddict is checked during TypedDict declaration, so don't typecheck it here + return False + + +def collect_all_inner_types(t: Type) -> list[Type]: + """ + Return all types that `t` contains + """ + return t.accept(CollectAllInnerTypesQuery()) - # Other kinds of type are trivial, since they are atomic (or invalid). + +class CollectAllInnerTypesQuery(TypeQuery[List[Type]]): + def __init__(self) -> None: + super().__init__(self.combine_lists_strategy) + + def query_types(self, types: Iterable[Type]) -> list[Type]: + return self.strategy([t.accept(self) for t in types]) + list(types) + + @classmethod + def combine_lists_strategy(cls, it: Iterable[list[Type]]) -> list[Type]: + return list(itertools.chain.from_iterable(it)) + + +def make_optional_type(t: Type) -> Type: + """Return the type corresponding to Optional[t]. + + Note that we can't use normal union simplification, since this function + is called during semantic analysis and simplification only works during + type checking. + """ + if isinstance(t, ProperType) and isinstance(t, NoneType): + return t + elif isinstance(t, ProperType) and isinstance(t, UnionType): + # Eagerly expanding aliases is not safe during semantic analysis. + items = [item for item in t.items if not isinstance(get_proper_type(item), NoneType)] + return UnionType(items + [NoneType()], t.line, t.column) + else: + return UnionType([t, NoneType()], t.line, t.column) + + +def validate_instance(t: Instance, fail: MsgCallback, empty_tuple_index: bool) -> bool: + """Check if this is a well-formed instance with respect to argument count/positions.""" + # TODO: combine logic with instantiate_type_alias(). + if any(unknown_unpack(a) for a in t.args): + # This type is not ready to be validated, because of unknown total count. + # TODO: is it OK to fill with TypeOfAny.from_error instead of special form? + return False + if t.type.has_type_var_tuple_type: + min_tv_count = sum( + not tv.has_default() and not isinstance(tv, TypeVarTupleType) + for tv in t.type.defn.type_vars + ) + correct = len(t.args) >= min_tv_count + if any( + isinstance(a, UnpackType) and isinstance(get_proper_type(a.type), Instance) + for a in t.args + ): + correct = True + if not t.args: + if not (empty_tuple_index and len(t.type.type_vars) == 1): + # The Any arguments should be set by the caller. + if empty_tuple_index and min_tv_count: + fail( + f"At least {min_tv_count} type argument(s) expected, none given", + t, + code=codes.TYPE_ARG, + ) + return False + elif not correct: + fail( + f"Bad number of arguments, expected: at least {min_tv_count}, given: {len(t.args)}", + t, + code=codes.TYPE_ARG, + ) + return False + else: + # We also need to check if we are not performing a type variable tuple split. + unpack = find_unpack_in_list(t.args) + if unpack is not None: + unpack_arg = t.args[unpack] + assert isinstance(unpack_arg, UnpackType) + if isinstance(unpack_arg.type, TypeVarTupleType): + assert t.type.type_var_tuple_prefix is not None + assert t.type.type_var_tuple_suffix is not None + exp_prefix = t.type.type_var_tuple_prefix + act_prefix = unpack + exp_suffix = t.type.type_var_tuple_suffix + act_suffix = len(t.args) - unpack - 1 + if act_prefix < exp_prefix or act_suffix < exp_suffix: + fail("TypeVarTuple cannot be split", t, code=codes.TYPE_ARG) + return False + elif any(isinstance(a, UnpackType) for a in t.args): + # A variadic unpack in fixed size instance (fixed unpacks must be flattened by the caller) + fail(message_registry.INVALID_UNPACK_POSITION, t, code=codes.VALID_TYPE) + t.args = () + return False + elif len(t.args) != len(t.type.type_vars): + # Invalid number of type parameters. + arg_count = len(t.args) + min_tv_count = sum(not tv.has_default() for tv in t.type.defn.type_vars) + max_tv_count = len(t.type.type_vars) + if arg_count and (arg_count < min_tv_count or arg_count > max_tv_count): + fail( + wrong_type_arg_count(min_tv_count, max_tv_count, str(arg_count), t.type.name), + t, + code=codes.TYPE_ARG, + ) + t.invalid = True + return False + return True + + +def find_self_type(typ: Type, lookup: Callable[[str], SymbolTableNode | None]) -> bool: + return typ.accept(HasSelfType(lookup)) + + +class HasSelfType(BoolTypeQuery): + def __init__(self, lookup: Callable[[str], SymbolTableNode | None]) -> None: + self.lookup = lookup + super().__init__(ANY_STRATEGY) + + def visit_unbound_type(self, t: UnboundType) -> bool: + sym = self.lookup(t.name) + if sym and sym.fullname in SELF_TYPE_NAMES: + return True + return super().visit_unbound_type(t) + + +def unknown_unpack(t: Type) -> bool: + """Check if a given type is an unpack of an unknown type. + + Unfortunately, there is no robust way to distinguish forward references from + genuine undefined names here. But this worked well so far, although it looks + quite fragile. + """ + if isinstance(t, UnpackType): + unpacked = get_proper_type(t.type) + if isinstance(unpacked, AnyType) and unpacked.type_of_any == TypeOfAny.special_form: + return True + return False + + +class FindTypeVarVisitor(SyntheticTypeVisitor[None]): + """Type visitor that looks for type variable types and self types.""" + + def __init__(self, api: SemanticAnalyzerCoreInterface, scope: TypeVarLikeScope) -> None: + self.api = api + self.scope = scope + self.type_var_likes: list[tuple[str, TypeVarLikeExpr]] = [] + self.has_self_type = False + self.seen_aliases: set[TypeAliasType] | None = None + self.include_callables = True + + def _seems_like_callable(self, type: UnboundType) -> bool: + if not type.args: + return False + return isinstance(type.args[0], (EllipsisType, TypeList, ParamSpecType)) def visit_unbound_type(self, t: UnboundType) -> None: - pass + name = t.name + node = None + + # Special case P.args and P.kwargs for ParamSpecs only. + if name.endswith("args"): + if name.endswith((".args", ".kwargs")): + base = ".".join(name.split(".")[:-1]) + n = self.api.lookup_qualified(base, t) + if n is not None and isinstance(n.node, ParamSpecExpr): + node = n + name = base + if node is None: + node = self.api.lookup_qualified(name, t) + if node and node.fullname in SELF_TYPE_NAMES: + self.has_self_type = True + if ( + node + and isinstance(node.node, TypeVarLikeExpr) + and self.scope.get_binding(node) is None + ): + if (name, node.node) not in self.type_var_likes: + self.type_var_likes.append((name, node.node)) + elif not self.include_callables and self._seems_like_callable(t): + if find_self_type( + t, lambda name: self.api.lookup_qualified(name, t, suppress_errors=True) + ): + self.has_self_type = True + return + elif node and node.fullname in LITERAL_TYPE_NAMES: + return + elif node and node.fullname in ANNOTATED_TYPE_NAMES and t.args: + # Don't query the second argument to Annotated for TypeVars + self.process_types([t.args[0]]) + elif t.args: + self.process_types(t.args) + + def visit_type_list(self, t: TypeList) -> None: + self.process_types(t.items) + + def visit_callable_argument(self, t: CallableArgument) -> None: + t.typ.accept(self) def visit_any(self, t: AnyType) -> None: pass - def visit_void(self, t: Void) -> None: + def visit_uninhabited_type(self, t: UninhabitedType) -> None: pass - def visit_none_type(self, t: NoneTyp) -> None: + def visit_none_type(self, t: NoneType) -> None: pass - def visit_type_list(self, t: TypeList) -> None: - self.fail('Invalid type', t) + def visit_erased_type(self, t: ErasedType) -> None: + pass + + def visit_deleted_type(self, t: DeletedType) -> None: + pass + + def visit_type_var(self, t: TypeVarType) -> None: + self.process_types([t.upper_bound, t.default] + t.values) + + def visit_param_spec(self, t: ParamSpecType) -> None: + self.process_types([t.upper_bound, t.default]) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + self.process_types([t.upper_bound, t.default]) + + def visit_unpack_type(self, t: UnpackType) -> None: + self.process_types([t.type]) + + def visit_parameters(self, t: Parameters) -> None: + self.process_types(t.arg_types) + + def visit_partial_type(self, t: PartialType) -> None: + pass + + def visit_instance(self, t: Instance) -> None: + self.process_types(t.args) + + def visit_callable_type(self, t: CallableType) -> None: + # FIX generics + self.process_types(t.arg_types) + t.ret_type.accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + self.process_types(t.items) + + def visit_typeddict_type(self, t: TypedDictType) -> None: + self.process_types(list(t.items.values())) + + def visit_raw_expression_type(self, t: RawExpressionType) -> None: + pass + + def visit_literal_type(self, t: LiteralType) -> None: + pass + + def visit_union_type(self, t: UnionType) -> None: + self.process_types(t.items) - def visit_type_var(self, t: TypeVar) -> None: + def visit_overloaded(self, t: Overloaded) -> None: + self.process_types(t.items) # type: ignore[arg-type] + + def visit_type_type(self, t: TypeType) -> None: + t.item.accept(self) + + def visit_ellipsis_type(self, t: EllipsisType) -> None: pass + + def visit_placeholder_type(self, t: PlaceholderType) -> None: + return self.process_types(t.args) + + def visit_type_alias_type(self, t: TypeAliasType) -> None: + # Skip type aliases in already visited types to avoid infinite recursion. + if self.seen_aliases is None: + self.seen_aliases = set() + elif t in self.seen_aliases: + return + self.seen_aliases.add(t) + self.process_types(t.args) + + def process_types(self, types: list[Type] | tuple[Type, ...]) -> None: + # Redundant type check helps mypyc. + if isinstance(types, list): + for t in types: + t.accept(self) + else: + for t in types: + t.accept(self) + + +class TypeVarDefaultTranslator(TrivialSyntheticTypeTranslator): + """Type translate visitor that replaces UnboundTypes with in-scope TypeVars.""" + + def __init__( + self, api: SemanticAnalyzerInterface, tvar_expr_name: str, context: Context + ) -> None: + super().__init__() + self.api = api + self.tvar_expr_name = tvar_expr_name + self.context = context + + def visit_unbound_type(self, t: UnboundType) -> Type: + sym = self.api.lookup_qualified(t.name, t, suppress_errors=True) + if sym is not None: + if type_var := self.api.tvar_scope.get_binding(sym): + return type_var + if isinstance(sym.node, TypeVarLikeExpr): + self.api.fail( + f'Type parameter "{self.tvar_expr_name}" has a default type ' + "that refers to one or more type variables that are out of scope", + self.context, + ) + return AnyType(TypeOfAny.from_error) + return super().visit_unbound_type(t) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # TypeAliasTypes are analyzed separately already, just return it + return t diff --git a/mypy/typefixture.py b/mypy/typefixture.py deleted file mode 100644 index baf3f785c3be..000000000000 --- a/mypy/typefixture.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Fixture used in type-related test cases. - -It contains class TypeInfos and Type objects. -""" - -from typing import List - -from mypy.types import ( - TypeVar, AnyType, Void, ErrorType, NoneTyp, Instance, Callable, TypeVarDef -) -from mypy.nodes import ( - TypeInfo, ClassDef, Block, ARG_POS, ARG_OPT, ARG_STAR, SymbolTable -) - - -class TypeFixture: - """Helper class that is used as a fixture in type-related unit tests. - - The members are initialized to contain various type-related values. - """ - - def __init__(self): - # The 'object' class - self.oi = self.make_type_info('builtins.object') # class object - self.o = Instance(self.oi, []) # object - - # Type variables - self.t = TypeVar('T', 1, [], self.o) # T`1 (type variable) - self.tf = TypeVar('T', -1, [], self.o) # T`-1 (type variable) - self.tf2 = TypeVar('T', -2, [], self.o) # T`-2 (type variable) - self.s = TypeVar('S', 2, [], self.o) # S`2 (type variable) - self.s1 = TypeVar('S', 1, [], self.o) # S`1 (type variable) - self.sf = TypeVar('S', -2, [], self.o) # S`-2 (type variable) - self.sf1 = TypeVar('S', -1, [], self.o) # S`-1 (type variable) - - # Simple types - self.anyt = AnyType() - self.void = Void() - self.err = ErrorType() - self.nonet = NoneTyp() - - # Abstract class TypeInfos - - # class F - self.fi = self.make_type_info('F', is_abstract=True) - - # class F2 - self.f2i = self.make_type_info('F2', is_abstract=True) - - # class F3(F) - self.f3i = self.make_type_info('F3', is_abstract=True, mro=[self.fi]) - - # Class TypeInfos - self.std_tuplei = self.make_type_info('builtins.tuple') # class tuple - self.type_typei = self.make_type_info('builtins.type') # class type - self.std_functioni = self.make_type_info('builtins.function') # function TODO - self.ai = self.make_type_info('A', mro=[self.oi]) # class A - self.bi = self.make_type_info('B', mro=[self.ai, self.oi]) # class B(A) - self.ci = self.make_type_info('C', mro=[self.ai, self.oi]) # class C(A) - self.di = self.make_type_info('D', mro=[self.oi]) # class D - # class E(F) - self.ei = self.make_type_info('E', mro=[self.fi, self.oi]) - # class E2(F2, F) - self.e2i = self.make_type_info('E2', mro=[self.f2i, self.fi, self.oi]) - # class E3(F, F2) - self.e3i = self.make_type_info('E3', mro=[self.fi, self.f2i, self.oi]) - - # Generic class TypeInfos - # G[T] - self.gi = self.make_type_info('G', mro=[self.oi], typevars=['T']) - # G2[T] - self.g2i = self.make_type_info('G2', mro=[self.oi], typevars=['T']) - # H[S, T] - self.hi = self.make_type_info('H', mro=[self.oi], typevars=['S', 'T']) - # GS[T, S] <: G[S] - self.gsi = self.make_type_info('GS', mro=[self.gi, self.oi], - typevars=['T', 'S'], - bases=[Instance(self.gi, [self.s])]) - # GS2[S] <: G[S] - self.gs2i = self.make_type_info('GS2', mro=[self.gi, self.oi], - typevars=['S'], - bases=[Instance(self.gi, [self.s1])]) - # list[T] - self.std_listi = self.make_type_info('builtins.list', mro=[self.oi], - typevars=['T']) - - # Instance types - self.std_tuple = Instance(self.std_tuplei, []) # tuple - self.type_type = Instance(self.type_typei, []) # type - self.std_function = Instance(self.std_functioni, []) # function TODO - self.a = Instance(self.ai, []) # A - self.b = Instance(self.bi, []) # B - self.c = Instance(self.ci, []) # C - self.d = Instance(self.di, []) # D - - self.e = Instance(self.ei, []) # E - self.e2 = Instance(self.e2i, []) # E2 - self.e3 = Instance(self.e3i, []) # E3 - - self.f = Instance(self.fi, []) # F - self.f2 = Instance(self.f2i, []) # F2 - self.f3 = Instance(self.f3i, []) # F3 - - # Generic instance types - self.ga = Instance(self.gi, [self.a]) # G[A] - self.gb = Instance(self.gi, [self.b]) # G[B] - self.go = Instance(self.gi, [self.o]) # G[object] - self.gt = Instance(self.gi, [self.t]) # G[T`1] - self.gtf = Instance(self.gi, [self.tf]) # G[T`-1] - self.gtf2 = Instance(self.gi, [self.tf2]) # G[T`-2] - self.gs = Instance(self.gi, [self.s]) # G[S] - self.gdyn = Instance(self.gi, [self.anyt]) # G[Any] - - self.g2a = Instance(self.g2i, [self.a]) # G2[A] - - self.gsab = Instance(self.gsi, [self.a, self.b]) # GS[A, B] - self.gsba = Instance(self.gsi, [self.b, self.a]) # GS[B, A] - - self.gs2a = Instance(self.gs2i, [self.a]) # GS2[A] - - self.hab = Instance(self.hi, [self.a, self.b]) # H[A, B] - self.haa = Instance(self.hi, [self.a, self.a]) # H[A, A] - self.hbb = Instance(self.hi, [self.b, self.b]) # H[B, B] - self.hts = Instance(self.hi, [self.t, self.s]) # H[T, S] - - self.lsta = Instance(self.std_listi, [self.a]) # List[A] - self.lstb = Instance(self.std_listi, [self.b]) # List[B] - - # Helper methods - - def callable(self, *a): - """callable(a1, ..., an, r) constructs a callable with argument types - a1, ... an and return type r. - """ - return Callable(a[:-1], [ARG_POS] * (len(a) - 1), - [None] * (len(a) - 1), a[-1], self.std_function) - - def callable_type(self, *a): - """callable_type(a1, ..., an, r) constructs a callable with - argument types a1, ... an and return type r, and which - represents a type. - """ - return Callable(a[:-1], [ARG_POS] * (len(a) - 1), - [None] * (len(a) - 1), a[-1], self.type_type) - - def callable_default(self, min_args, *a): - """callable_default(min_args, a1, ..., an, r) constructs a - callable with argument types a1, ... an and return type r, - with min_args mandatory fixed arguments. - """ - n = len(a) - 1 - return Callable(a[:-1], - [ARG_POS] * min_args + [ARG_OPT] * (n - min_args), - [None] * n, - a[-1], self.std_function) - - def callable_var_arg(self, min_args, *a): - """callable_var_arg(min_args, a1, ..., an, r) constructs a callable - with argument types a1, ... *an and return type r. - """ - n = len(a) - 1 - return Callable(a[:-1], - [ARG_POS] * min_args + - [ARG_OPT] * (n - 1 - min_args) + - [ARG_STAR], [None] * n, - a[-1], self.std_function) - - def make_type_info(self, name: str, - is_abstract: bool = False, - mro: List[TypeInfo] = None, - bases: List[Instance] = None, - typevars: List[str] = None) -> TypeInfo: - """Make a TypeInfo suitable for use in unit tests.""" - - class_def = ClassDef(name, Block([]), None, []) - class_def.fullname = name - - if typevars: - v = [] # type: List[TypeVarDef] - id = 1 - for n in typevars: - v.append(TypeVarDef(n, id, None, self.oi)) - id += 1 - class_def.type_vars = v - - info = TypeInfo(SymbolTable(), class_def) - if mro is None: - mro = [] - if name != 'builtins.object': - mro.append(self.oi) - info.mro = [info] + mro - if bases is None: - if mro: - # By default, assume that there is a single non-generic base. - bases = [Instance(mro[0], [])] - else: - bases = [] - info.bases = bases - - return info - - -class InterfaceTypeFixture(TypeFixture): - """Extension of TypeFixture that contains additional generic - interface types.""" - - def __init__(self): - super().__init__() - # GF[T] - self.gfi = self.make_type_info('GF', typevars=['T'], is_abstract=True) - - # M1 <: GF[A] - self.m1i = self.make_type_info('M1', - is_abstract=True, - mro=[self.gfi, self.oi], - bases=[Instance(self.gfi, [self.a])]) - - self.gfa = Instance(self.gfi, [self.a]) # GF[A] - self.gfb = Instance(self.gfi, [self.b]) # GF[B] - - self.m1 = Instance(self.m1i, []) # M1 diff --git a/mypy/typeops.py b/mypy/typeops.py new file mode 100644 index 000000000000..f190168a18d7 --- /dev/null +++ b/mypy/typeops.py @@ -0,0 +1,1168 @@ +"""Miscellaneous type operations and helpers for use during type checking. + +NOTE: These must not be accessed from mypy.nodes or mypy.types to avoid import + cycles. These must not be called from the semantic analysis main pass + since these may assume that MROs are ready. +""" + +from __future__ import annotations + +import itertools +from typing import Any, Iterable, List, Sequence, TypeVar, cast + +from mypy.copytype import copy_type +from mypy.expandtype import expand_type, expand_type_by_instance +from mypy.maptype import map_instance_to_supertype +from mypy.nodes import ( + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + SYMBOL_FUNCBASE_TYPES, + Decorator, + Expression, + FuncBase, + FuncDef, + FuncItem, + OverloadedFuncDef, + StrExpr, + TypeInfo, + Var, +) +from mypy.state import state +from mypy.types import ( + AnyType, + CallableType, + ExtraAttrs, + FormalArgument, + FunctionLike, + Instance, + LiteralType, + NoneType, + NormalizedCallableType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeQuery, + TypeType, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UninhabitedType, + UnionType, + UnpackType, + flatten_nested_unions, + get_proper_type, + get_proper_types, +) +from mypy.typevars import fill_typevars + + +def is_recursive_pair(s: Type, t: Type) -> bool: + """Is this a pair of recursive types? + + There may be more cases, and we may be forced to use e.g. has_recursive_types() + here, but this function is called in very hot code, so we try to keep it simple + and return True only in cases we know may have problems. + """ + if isinstance(s, TypeAliasType) and s.is_recursive: + return ( + isinstance(get_proper_type(t), (Instance, UnionType)) + or isinstance(t, TypeAliasType) + and t.is_recursive + # Tuple types are special, they can cause an infinite recursion even if + # the other type is not recursive, because of the tuple fallback that is + # calculated "on the fly". + or isinstance(get_proper_type(s), TupleType) + ) + if isinstance(t, TypeAliasType) and t.is_recursive: + return ( + isinstance(get_proper_type(s), (Instance, UnionType)) + or isinstance(s, TypeAliasType) + and s.is_recursive + # Same as above. + or isinstance(get_proper_type(t), TupleType) + ) + return False + + +def tuple_fallback(typ: TupleType) -> Instance: + """Return fallback type for a tuple.""" + info = typ.partial_fallback.type + if info.fullname != "builtins.tuple": + return typ.partial_fallback + items = [] + for item in typ.items: + if isinstance(item, UnpackType): + unpacked_type = get_proper_type(item.type) + if isinstance(unpacked_type, TypeVarTupleType): + unpacked_type = get_proper_type(unpacked_type.upper_bound) + if ( + isinstance(unpacked_type, Instance) + and unpacked_type.type.fullname == "builtins.tuple" + ): + items.append(unpacked_type.args[0]) + else: + raise NotImplementedError + else: + items.append(item) + return Instance( + info, + # Note: flattening recursive unions is dangerous, since it can fool recursive + # types optimization in subtypes.py and go into infinite recursion. + [make_simplified_union(items, handle_recursive=False)], + extra_attrs=typ.partial_fallback.extra_attrs, + ) + + +def get_self_type(func: CallableType, default_self: Instance | TupleType) -> Type | None: + if isinstance(get_proper_type(func.ret_type), UninhabitedType): + return func.ret_type + elif func.arg_types and func.arg_types[0] != default_self and func.arg_kinds[0] == ARG_POS: + return func.arg_types[0] + else: + return None + + +def type_object_type_from_function( + signature: FunctionLike, info: TypeInfo, def_info: TypeInfo, fallback: Instance, is_new: bool +) -> FunctionLike: + # We first need to record all non-trivial (explicit) self types in __init__, + # since they will not be available after we bind them. Note, we use explicit + # self-types only in the defining class, similar to __new__ (but not exactly the same, + # see comment in class_callable below). This is mostly useful for annotating library + # classes such as subprocess.Popen. + default_self = fill_typevars(info) + if not is_new and not info.is_newtype: + orig_self_types = [get_self_type(it, default_self) for it in signature.items] + else: + orig_self_types = [None] * len(signature.items) + + # The __init__ method might come from a generic superclass 'def_info' + # with type variables that do not map identically to the type variables of + # the class 'info' being constructed. For example: + # + # class A(Generic[T]): + # def __init__(self, x: T) -> None: ... + # class B(A[List[T]]): + # ... + # + # We need to map B's __init__ to the type (List[T]) -> None. + signature = bind_self( + signature, + original_type=default_self, + is_classmethod=is_new, + # Explicit instance self annotations have special handling in class_callable(), + # we don't need to bind any type variables in them if they are generic. + ignore_instances=True, + ) + signature = cast(FunctionLike, map_type_from_supertype(signature, info, def_info)) + + special_sig: str | None = None + if def_info.fullname == "builtins.dict": + # Special signature! + special_sig = "dict" + + if isinstance(signature, CallableType): + return class_callable(signature, info, fallback, special_sig, is_new, orig_self_types[0]) + else: + # Overloaded __init__/__new__. + assert isinstance(signature, Overloaded) + items: list[CallableType] = [] + for item, orig_self in zip(signature.items, orig_self_types): + items.append(class_callable(item, info, fallback, special_sig, is_new, orig_self)) + return Overloaded(items) + + +def class_callable( + init_type: CallableType, + info: TypeInfo, + type_type: Instance, + special_sig: str | None, + is_new: bool, + orig_self_type: Type | None = None, +) -> CallableType: + """Create a type object type based on the signature of __init__.""" + variables: list[TypeVarLikeType] = [] + variables.extend(info.defn.type_vars) + variables.extend(init_type.variables) + + from mypy.subtypes import is_subtype + + init_ret_type = get_proper_type(init_type.ret_type) + orig_self_type = get_proper_type(orig_self_type) + default_ret_type = fill_typevars(info) + explicit_type = init_ret_type if is_new else orig_self_type + if ( + isinstance(explicit_type, (Instance, TupleType, UninhabitedType)) + # We have to skip protocols, because it can be a subtype of a return type + # by accident. Like `Hashable` is a subtype of `object`. See #11799 + and isinstance(default_ret_type, Instance) + and not default_ret_type.type.is_protocol + # Only use the declared return type from __new__ or declared self in __init__ + # if it is actually returning a subtype of what we would return otherwise. + and is_subtype(explicit_type, default_ret_type, ignore_type_params=True) + ): + ret_type: Type = explicit_type + else: + ret_type = default_ret_type + + callable_type = init_type.copy_modified( + ret_type=ret_type, + fallback=type_type, + name=None, + variables=variables, + special_sig=special_sig, + ) + c = callable_type.with_name(info.name) + return c + + +def map_type_from_supertype(typ: Type, sub_info: TypeInfo, super_info: TypeInfo) -> Type: + """Map type variables in a type defined in a supertype context to be valid + in the subtype context. Assume that the result is unique; if more than + one type is possible, return one of the alternatives. + + For example, assume + + class D(Generic[S]): ... + class C(D[E[T]], Generic[T]): ... + + Now S in the context of D would be mapped to E[T] in the context of C. + """ + # Create the type of self in subtype, of form t[a1, ...]. + inst_type = fill_typevars(sub_info) + if isinstance(inst_type, TupleType): + inst_type = tuple_fallback(inst_type) + # Map the type of self to supertype. This gets us a description of the + # supertype type variables in terms of subtype variables, i.e. t[t1, ...] + # so that any type variables in tN are to be interpreted in subtype + # context. + inst_type = map_instance_to_supertype(inst_type, super_info) + # Finally expand the type variables in type with those in the previously + # constructed type. Note that both type and inst_type may have type + # variables, but in type they are interpreted in supertype context while + # in inst_type they are interpreted in subtype context. This works even if + # the names of type variables in supertype and subtype overlap. + return expand_type_by_instance(typ, inst_type) + + +def supported_self_type( + typ: ProperType, allow_callable: bool = True, allow_instances: bool = True +) -> bool: + """Is this a supported kind of explicit self-types? + + Currently, this means an X or Type[X], where X is an instance or + a type variable with an instance upper bound. + """ + if isinstance(typ, TypeType): + return supported_self_type(typ.item) + if allow_callable and isinstance(typ, CallableType): + # Special case: allow class callable instead of Type[...] as cls annotation, + # as well as callable self for callback protocols. + return True + return isinstance(typ, TypeVarType) or ( + allow_instances and isinstance(typ, Instance) and typ != fill_typevars(typ.type) + ) + + +F = TypeVar("F", bound=FunctionLike) + + +def bind_self( + method: F, + original_type: Type | None = None, + is_classmethod: bool = False, + ignore_instances: bool = False, +) -> F: + """Return a copy of `method`, with the type of its first parameter (usually + self or cls) bound to original_type. + + If the type of `self` is a generic type (T, or Type[T] for classmethods), + instantiate every occurrence of type with original_type in the rest of the + signature and in the return type. + + original_type is the type of E in the expression E.copy(). It is None in + compatibility checks. In this case we treat it as the erasure of the + declared type of self. + + This way we can express "the type of self". For example: + + T = TypeVar('T', bound='A') + class A: + def copy(self: T) -> T: ... + + class B(A): pass + + b = B().copy() # type: B + + """ + if isinstance(method, Overloaded): + items = [] + original_type = get_proper_type(original_type) + for c in method.items: + if isinstance(original_type, Instance): + # Filter based on whether declared self type can match actual object type. + # For example, if self has type C[int] and method is accessed on a C[str] value, + # omit this item. This is best effort since bind_self can be called in many + # contexts, and doing complete validation might trigger infinite recursion. + # + # Note that overload item filtering normally happens elsewhere. This is needed + # at least during constraint inference. + keep = is_valid_self_type_best_effort(c, original_type) + else: + keep = True + if keep: + items.append(bind_self(c, original_type, is_classmethod, ignore_instances)) + if len(items) == 0: + # If no item matches, returning all items helps avoid some spurious errors + items = [ + bind_self(c, original_type, is_classmethod, ignore_instances) for c in method.items + ] + return cast(F, Overloaded(items)) + assert isinstance(method, CallableType) + func = method + if not func.arg_types: + # Invalid method, return something. + return cast(F, func) + if func.arg_kinds[0] in (ARG_STAR, ARG_STAR2): + # The signature is of the form 'def foo(*args, ...)'. + # In this case we shouldn't drop the first arg, + # since func will be absorbed by the *args. + # TODO: infer bounds on the type of *args? + + # In the case of **kwargs we should probably emit an error, but + # for now we simply skip it, to avoid crashes down the line. + return cast(F, func) + self_param_type = get_proper_type(func.arg_types[0]) + + variables: Sequence[TypeVarLikeType] + # Having a def __call__(self: Callable[...], ...) can cause infinite recursion. Although + # this special-casing looks not very principled, there is nothing meaningful we can infer + # from such definition, since it is inherently indefinitely recursive. + allow_callable = func.name is None or not func.name.startswith("__call__ of") + if func.variables and supported_self_type( + self_param_type, allow_callable=allow_callable, allow_instances=not ignore_instances + ): + from mypy.infer import infer_type_arguments + + if original_type is None: + # TODO: type check method override (see #7861). + original_type = erase_to_bound(self_param_type) + original_type = get_proper_type(original_type) + + # Find which of method type variables appear in the type of "self". + self_ids = {tv.id for tv in get_all_type_vars(self_param_type)} + self_vars = [tv for tv in func.variables if tv.id in self_ids] + + # Solve for these type arguments using the actual class or instance type. + typeargs = infer_type_arguments( + self_vars, self_param_type, original_type, is_supertype=True + ) + if ( + is_classmethod + and any(isinstance(get_proper_type(t), UninhabitedType) for t in typeargs) + and isinstance(original_type, (Instance, TypeVarType, TupleType)) + ): + # In case we call a classmethod through an instance x, fallback to type(x). + typeargs = infer_type_arguments( + self_vars, self_param_type, TypeType(original_type), is_supertype=True + ) + + # Update the method signature with the solutions found. + # Technically, some constraints might be unsolvable, make them Never. + to_apply = [t if t is not None else UninhabitedType() for t in typeargs] + func = expand_type(func, {tv.id: arg for tv, arg in zip(self_vars, to_apply)}) + variables = [v for v in func.variables if v not in self_vars] + else: + variables = func.variables + + original_type = get_proper_type(original_type) + if isinstance(original_type, CallableType) and original_type.is_type_obj(): + original_type = TypeType.make_normalized(original_type.ret_type) + res = func.copy_modified( + arg_types=func.arg_types[1:], + arg_kinds=func.arg_kinds[1:], + arg_names=func.arg_names[1:], + variables=variables, + bound_args=[original_type], + ) + return cast(F, res) + + +def is_valid_self_type_best_effort(c: CallableType, self_type: Instance) -> bool: + """Quickly check if self_type might match the self in a callable. + + Avoid performing any complex type operations. This is performance-critical. + + Default to returning True if we don't know (or it would be too expensive). + """ + if ( + self_type.args + and c.arg_types + and isinstance((arg_type := get_proper_type(c.arg_types[0])), Instance) + and c.arg_kinds[0] in (ARG_POS, ARG_OPT) + and arg_type.args + and self_type.type.fullname != "functools._SingleDispatchCallable" + ): + if self_type.type is not arg_type.type: + # We can't map to supertype, since it could trigger expensive checks for + # protocol types, so we consevatively assume this is fine. + return True + + # Fast path: no explicit annotation on self + if all( + ( + type(arg) is TypeVarType + and type(arg.upper_bound) is Instance + and arg.upper_bound.type.fullname == "builtins.object" + ) + for arg in arg_type.args + ): + return True + + from mypy.meet import is_overlapping_types + + return is_overlapping_types(self_type, c.arg_types[0]) + return True + + +def erase_to_bound(t: Type) -> Type: + # TODO: use value restrictions to produce a union? + t = get_proper_type(t) + if isinstance(t, TypeVarType): + return t.upper_bound + if isinstance(t, TypeType): + if isinstance(t.item, TypeVarType): + return TypeType.make_normalized(t.item.upper_bound) + return t + + +def callable_corresponding_argument( + typ: NormalizedCallableType | Parameters, model: FormalArgument +) -> FormalArgument | None: + """Return the argument a function that corresponds to `model`""" + + by_name = typ.argument_by_name(model.name) + by_pos = typ.argument_by_position(model.pos) + if by_name is None and by_pos is None: + return None + if by_name is not None and by_pos is not None: + if by_name == by_pos: + return by_name + # If we're dealing with an optional pos-only and an optional + # name-only arg, merge them. This is the case for all functions + # taking both *args and **args, or a pair of functions like so: + + # def right(a: int = ...) -> None: ... + # def left(__a: int = ..., *, a: int = ...) -> None: ... + from mypy.subtypes import is_equivalent + + if ( + not (by_name.required or by_pos.required) + and by_pos.name is None + and by_name.pos is None + and is_equivalent(by_name.typ, by_pos.typ) + ): + return FormalArgument(by_name.name, by_pos.pos, by_name.typ, False) + return by_name if by_name is not None else by_pos + + +def simple_literal_type(t: ProperType | None) -> Instance | None: + """Extract the underlying fallback Instance type for a simple Literal""" + if isinstance(t, Instance) and t.last_known_value is not None: + t = t.last_known_value + if isinstance(t, LiteralType): + return t.fallback + return None + + +def is_simple_literal(t: ProperType) -> bool: + if isinstance(t, LiteralType): + return t.fallback.type.is_enum or t.fallback.type.fullname == "builtins.str" + if isinstance(t, Instance): + return t.last_known_value is not None and isinstance(t.last_known_value.value, str) + return False + + +def make_simplified_union( + items: Sequence[Type], + line: int = -1, + column: int = -1, + *, + keep_erased: bool = False, + contract_literals: bool = True, + handle_recursive: bool = True, +) -> ProperType: + """Build union type with redundant union items removed. + + If only a single item remains, this may return a non-union type. + + Examples: + + * [int, str] -> Union[int, str] + * [int, object] -> object + * [int, int] -> int + * [int, Any] -> Union[int, Any] (Any types are not simplified away!) + * [Any, Any] -> Any + * [int, Union[bytes, str]] -> Union[int, bytes, str] + + Note: This must NOT be used during semantic analysis, since TypeInfos may not + be fully initialized. + + The keep_erased flag is used for type inference against union types + containing type variables. If set to True, keep all ErasedType items. + + The contract_literals flag indicates whether we need to contract literal types + back into a sum type. Set it to False when called by try_expanding_sum_type_ + to_union(). + """ + # Step 1: expand all nested unions + items = flatten_nested_unions(items, handle_recursive=handle_recursive) + + # Step 2: fast path for single item + if len(items) == 1: + return get_proper_type(items[0]) + + # Step 3: remove redundant unions + simplified_set: Sequence[Type] = _remove_redundant_union_items(items, keep_erased) + + # Step 4: If more than one literal exists in the union, try to simplify + if ( + contract_literals + and sum(isinstance(get_proper_type(item), LiteralType) for item in simplified_set) > 1 + ): + simplified_set = try_contracting_literals_in_union(simplified_set) + + result = get_proper_type(UnionType.make_union(simplified_set, line, column)) + + nitems = len(items) + if nitems > 1 and ( + nitems > 2 or not (type(items[0]) is NoneType or type(items[1]) is NoneType) + ): + # Step 5: At last, we erase any (inconsistent) extra attributes on instances. + + # Initialize with None instead of an empty set as a micro-optimization. The set + # is needed very rarely, so we try to avoid constructing it. + extra_attrs_set: set[ExtraAttrs] | None = None + for item in items: + instance = try_getting_instance_fallback(item) + if instance and instance.extra_attrs: + if extra_attrs_set is None: + extra_attrs_set = {instance.extra_attrs} + else: + extra_attrs_set.add(instance.extra_attrs) + + if extra_attrs_set is not None and len(extra_attrs_set) > 1: + fallback = try_getting_instance_fallback(result) + if fallback: + fallback.extra_attrs = None + + return result + + +def _remove_redundant_union_items(items: list[Type], keep_erased: bool) -> list[Type]: + from mypy.subtypes import is_proper_subtype + + # The first pass through this loop, we check if later items are subtypes of earlier items. + # The second pass through this loop, we check if earlier items are subtypes of later items + # (by reversing the remaining items) + for _direction in range(2): + new_items: list[Type] = [] + # seen is a map from a type to its index in new_items + seen: dict[ProperType, int] = {} + unduplicated_literal_fallbacks: set[Instance] | None = None + for ti in items: + proper_ti = get_proper_type(ti) + + # UninhabitedType is always redundant + if isinstance(proper_ti, UninhabitedType): + continue + + duplicate_index = -1 + # Quickly check if we've seen this type + if proper_ti in seen: + duplicate_index = seen[proper_ti] + elif ( + isinstance(proper_ti, LiteralType) + and unduplicated_literal_fallbacks is not None + and proper_ti.fallback in unduplicated_literal_fallbacks + ): + # This is an optimisation for unions with many LiteralType + # We've already checked for exact duplicates. This means that any super type of + # the LiteralType must be a super type of its fallback. If we've gone through + # the expensive loop below and found no super type for a previous LiteralType + # with the same fallback, we can skip doing that work again and just add the type + # to new_items + pass + else: + # If not, check if we've seen a supertype of this type + for j, tj in enumerate(new_items): + tj = get_proper_type(tj) + # If tj is an Instance with a last_known_value, do not remove proper_ti + # (unless it's an instance with the same last_known_value) + if ( + isinstance(tj, Instance) + and tj.last_known_value is not None + and not ( + isinstance(proper_ti, Instance) + and tj.last_known_value == proper_ti.last_known_value + ) + ): + continue + + if is_proper_subtype( + ti, tj, keep_erased_types=keep_erased, ignore_promotions=True + ): + duplicate_index = j + break + if duplicate_index != -1: + # If deleted subtypes had more general truthiness, use that + orig_item = new_items[duplicate_index] + if not orig_item.can_be_true and ti.can_be_true: + new_items[duplicate_index] = true_or_false(orig_item) + elif not orig_item.can_be_false and ti.can_be_false: + new_items[duplicate_index] = true_or_false(orig_item) + else: + # We have a non-duplicate item, add it to new_items + seen[proper_ti] = len(new_items) + new_items.append(ti) + if isinstance(proper_ti, LiteralType): + if unduplicated_literal_fallbacks is None: + unduplicated_literal_fallbacks = set() + unduplicated_literal_fallbacks.add(proper_ti.fallback) + + items = new_items + if len(items) <= 1: + break + items.reverse() + + return items + + +def _get_type_method_ret_type(t: Type, *, name: str) -> Type | None: + t = get_proper_type(t) + + # For Enum literals the ret_type can change based on the Enum + # we need to check the type of the enum rather than the literal + if isinstance(t, LiteralType) and t.is_enum_literal(): + t = t.fallback + + if isinstance(t, Instance): + sym = t.type.get(name) + # Fallback to the metaclass for the lookup when necessary + if not sym and (m := t.type.metaclass_type): + sym = m.type.get(name) + if sym: + sym_type = get_proper_type(sym.type) + if isinstance(sym_type, CallableType): + return sym_type.ret_type + + return None + + +def true_only(t: Type) -> ProperType: + """ + Restricted version of t with only True-ish values + """ + t = get_proper_type(t) + + if not t.can_be_true: + # All values of t are False-ish, so there are no true values in it + return UninhabitedType(line=t.line, column=t.column) + elif not t.can_be_false: + # All values of t are already True-ish, so true_only is idempotent in this case + return t + elif isinstance(t, UnionType): + # The true version of a union type is the union of the true versions of its components + new_items = [true_only(item) for item in t.items] + can_be_true_items = [item for item in new_items if item.can_be_true] + return make_simplified_union(can_be_true_items, line=t.line, column=t.column) + else: + ret_type = _get_type_method_ret_type(t, name="__bool__") or _get_type_method_ret_type( + t, name="__len__" + ) + + if ret_type and not ret_type.can_be_true: + return UninhabitedType(line=t.line, column=t.column) + + new_t = copy_type(t) + new_t.can_be_false = False + return new_t + + +def false_only(t: Type) -> ProperType: + """ + Restricted version of t with only False-ish values + """ + t = get_proper_type(t) + + if not t.can_be_false: + if state.strict_optional: + # All values of t are True-ish, so there are no false values in it + return UninhabitedType(line=t.line) + else: + # When strict optional checking is disabled, everything can be + # False-ish since anything can be None + return NoneType(line=t.line) + elif not t.can_be_true: + # All values of t are already False-ish, so false_only is idempotent in this case + return t + elif isinstance(t, UnionType): + # The false version of a union type is the union of the false versions of its components + new_items = [false_only(item) for item in t.items] + can_be_false_items = [item for item in new_items if item.can_be_false] + return make_simplified_union(can_be_false_items, line=t.line, column=t.column) + elif isinstance(t, Instance) and t.type.fullname in ("builtins.str", "builtins.bytes"): + return LiteralType("", fallback=t) + elif isinstance(t, Instance) and t.type.fullname == "builtins.int": + return LiteralType(0, fallback=t) + else: + ret_type = _get_type_method_ret_type(t, name="__bool__") or _get_type_method_ret_type( + t, name="__len__" + ) + + if ret_type: + if not ret_type.can_be_false: + return UninhabitedType(line=t.line) + elif isinstance(t, Instance) and t.type.is_final: + return UninhabitedType(line=t.line) + + new_t = copy_type(t) + new_t.can_be_true = False + return new_t + + +def true_or_false(t: Type) -> ProperType: + """ + Unrestricted version of t with both True-ish and False-ish values + """ + t = get_proper_type(t) + + if isinstance(t, UnionType): + new_items = [true_or_false(item) for item in t.items] + return make_simplified_union(new_items, line=t.line, column=t.column) + + new_t = copy_type(t) + new_t.can_be_true = new_t.can_be_true_default() + new_t.can_be_false = new_t.can_be_false_default() + return new_t + + +def erase_def_to_union_or_bound(tdef: TypeVarLikeType) -> Type: + # TODO(PEP612): fix for ParamSpecType + if isinstance(tdef, ParamSpecType): + return AnyType(TypeOfAny.from_error) + if isinstance(tdef, TypeVarType) and tdef.values: + return make_simplified_union(tdef.values) + else: + return tdef.upper_bound + + +def erase_to_union_or_bound(typ: TypeVarType) -> ProperType: + if typ.values: + return make_simplified_union(typ.values) + else: + return get_proper_type(typ.upper_bound) + + +def function_type(func: FuncBase, fallback: Instance) -> FunctionLike: + if func.type: + assert isinstance(func.type, FunctionLike) + return func.type + else: + # Implicit type signature with dynamic types. + if isinstance(func, FuncItem): + return callable_type(func, fallback) + else: + # Broken overloads can have self.type set to None. + # TODO: should we instead always set the type in semantic analyzer? + assert isinstance(func, OverloadedFuncDef) + any_type = AnyType(TypeOfAny.from_error) + dummy = CallableType( + [any_type, any_type], + [ARG_STAR, ARG_STAR2], + [None, None], + any_type, + fallback, + line=func.line, + is_ellipsis_args=True, + ) + # Return an Overloaded, because some callers may expect that + # an OverloadedFuncDef has an Overloaded type. + return Overloaded([dummy]) + + +def callable_type( + fdef: FuncItem, fallback: Instance, ret_type: Type | None = None +) -> CallableType: + # TODO: somewhat unfortunate duplication with prepare_method_signature in semanal + if fdef.info and (not fdef.is_static or fdef.name == "__new__") and fdef.arg_names: + self_type: Type = fill_typevars(fdef.info) + if fdef.is_class or fdef.name == "__new__": + self_type = TypeType.make_normalized(self_type) + args = [self_type] + [AnyType(TypeOfAny.unannotated)] * (len(fdef.arg_names) - 1) + else: + args = [AnyType(TypeOfAny.unannotated)] * len(fdef.arg_names) + + return CallableType( + args, + fdef.arg_kinds, + fdef.arg_names, + ret_type or AnyType(TypeOfAny.unannotated), + fallback, + name=fdef.name, + line=fdef.line, + column=fdef.column, + implicit=True, + # We need this for better error messages, like missing `self` note: + definition=fdef if isinstance(fdef, FuncDef) else None, + ) + + +def try_getting_str_literals(expr: Expression, typ: Type) -> list[str] | None: + """If the given expression or type corresponds to a string literal + or a union of string literals, returns a list of the underlying strings. + Otherwise, returns None. + + Specifically, this function is guaranteed to return a list with + one or more strings if one of the following is true: + + 1. 'expr' is a StrExpr + 2. 'typ' is a LiteralType containing a string + 3. 'typ' is a UnionType containing only LiteralType of strings + """ + if isinstance(expr, StrExpr): + return [expr.value] + + # TODO: See if we can eliminate this function and call the below one directly + return try_getting_str_literals_from_type(typ) + + +def try_getting_str_literals_from_type(typ: Type) -> list[str] | None: + """If the given expression or type corresponds to a string Literal + or a union of string Literals, returns a list of the underlying strings. + Otherwise, returns None. + + For example, if we had the type 'Literal["foo", "bar"]' as input, this function + would return a list of strings ["foo", "bar"]. + """ + return try_getting_literals_from_type(typ, str, "builtins.str") + + +def try_getting_int_literals_from_type(typ: Type) -> list[int] | None: + """If the given expression or type corresponds to an int Literal + or a union of int Literals, returns a list of the underlying ints. + Otherwise, returns None. + + For example, if we had the type 'Literal[1, 2, 3]' as input, this function + would return a list of ints [1, 2, 3]. + """ + return try_getting_literals_from_type(typ, int, "builtins.int") + + +T = TypeVar("T") + + +def try_getting_literals_from_type( + typ: Type, target_literal_type: type[T], target_fullname: str +) -> list[T] | None: + """If the given expression or type corresponds to a Literal or + union of Literals where the underlying values correspond to the given + target type, returns a list of those underlying values. Otherwise, + returns None. + """ + typ = get_proper_type(typ) + + if isinstance(typ, Instance) and typ.last_known_value is not None: + possible_literals: list[Type] = [typ.last_known_value] + elif isinstance(typ, UnionType): + possible_literals = list(typ.items) + else: + possible_literals = [typ] + + literals: list[T] = [] + for lit in get_proper_types(possible_literals): + if isinstance(lit, LiteralType) and lit.fallback.type.fullname == target_fullname: + val = lit.value + if isinstance(val, target_literal_type): + literals.append(val) + else: + return None + else: + return None + return literals + + +def is_literal_type_like(t: Type | None) -> bool: + """Returns 'true' if the given type context is potentially either a LiteralType, + a Union of LiteralType, or something similar. + """ + t = get_proper_type(t) + if t is None: + return False + elif isinstance(t, LiteralType): + return True + elif isinstance(t, UnionType): + return any(is_literal_type_like(item) for item in t.items) + elif isinstance(t, TypeVarType): + return is_literal_type_like(t.upper_bound) or any( + is_literal_type_like(item) for item in t.values + ) + else: + return False + + +def is_singleton_type(typ: Type) -> bool: + """Returns 'true' if this type is a "singleton type" -- if there exists + exactly only one runtime value associated with this type. + + That is, given two values 'a' and 'b' that have the same type 't', + 'is_singleton_type(t)' returns True if and only if the expression 'a is b' is + always true. + + Currently, this returns True when given NoneTypes, enum LiteralTypes, + enum types with a single value and ... (Ellipses). + + Note that other kinds of LiteralTypes cannot count as singleton types. For + example, suppose we do 'a = 100000 + 1' and 'b = 100001'. It is not guaranteed + that 'a is b' will always be true -- some implementations of Python will end up + constructing two distinct instances of 100001. + """ + typ = get_proper_type(typ) + return typ.is_singleton_type() + + +def try_expanding_sum_type_to_union(typ: Type, target_fullname: str) -> ProperType: + """Attempts to recursively expand any enum Instances with the given target_fullname + into a Union of all of its component LiteralTypes. + + For example, if we have: + + class Color(Enum): + RED = 1 + BLUE = 2 + YELLOW = 3 + + class Status(Enum): + SUCCESS = 1 + FAILURE = 2 + UNKNOWN = 3 + + ...and if we call `try_expanding_enum_to_union(Union[Color, Status], 'module.Color')`, + this function will return Literal[Color.RED, Color.BLUE, Color.YELLOW, Status]. + """ + typ = get_proper_type(typ) + + if isinstance(typ, UnionType): + items = [ + try_expanding_sum_type_to_union(item, target_fullname) for item in typ.relevant_items() + ] + return make_simplified_union(items, contract_literals=False) + + if isinstance(typ, Instance) and typ.type.fullname == target_fullname: + if typ.type.fullname == "builtins.bool": + items = [LiteralType(True, typ), LiteralType(False, typ)] + return make_simplified_union(items, contract_literals=False) + + if typ.type.is_enum: + items = [LiteralType(name, typ) for name in typ.type.enum_members] + if not items: + return typ + return make_simplified_union(items, contract_literals=False) + + return typ + + +def try_contracting_literals_in_union(types: Sequence[Type]) -> list[ProperType]: + """Contracts any literal types back into a sum type if possible. + + Will replace the first instance of the literal with the sum type and + remove all others. + + If we call `try_contracting_union(Literal[Color.RED, Color.BLUE, Color.YELLOW])`, + this function will return Color. + + We also treat `Literal[True, False]` as `bool`. + """ + proper_types = [get_proper_type(typ) for typ in types] + sum_types: dict[str, tuple[set[Any], list[int]]] = {} + marked_for_deletion = set() + for idx, typ in enumerate(proper_types): + if isinstance(typ, LiteralType): + fullname = typ.fallback.type.fullname + if typ.fallback.type.is_enum or isinstance(typ.value, bool): + if fullname not in sum_types: + sum_types[fullname] = ( + ( + set(typ.fallback.type.enum_members) + if typ.fallback.type.is_enum + else {True, False} + ), + [], + ) + literals, indexes = sum_types[fullname] + literals.discard(typ.value) + indexes.append(idx) + if not literals: + first, *rest = indexes + proper_types[first] = typ.fallback + marked_for_deletion |= set(rest) + return list( + itertools.compress( + proper_types, [(i not in marked_for_deletion) for i in range(len(proper_types))] + ) + ) + + +def coerce_to_literal(typ: Type) -> Type: + """Recursively converts any Instances that have a last_known_value or are + instances of enum types with a single value into the corresponding LiteralType. + """ + original_type = typ + typ = get_proper_type(typ) + if isinstance(typ, UnionType): + new_items = [coerce_to_literal(item) for item in typ.items] + return UnionType.make_union(new_items) + elif isinstance(typ, Instance): + if typ.last_known_value: + return typ.last_known_value + elif typ.type.is_enum: + enum_values = typ.type.enum_members + if len(enum_values) == 1: + return LiteralType(value=enum_values[0], fallback=typ) + return original_type + + +def get_type_vars(tp: Type) -> list[TypeVarType]: + return cast("list[TypeVarType]", tp.accept(TypeVarExtractor())) + + +def get_all_type_vars(tp: Type) -> list[TypeVarLikeType]: + # TODO: should we always use this function instead of get_type_vars() above? + return tp.accept(TypeVarExtractor(include_all=True)) + + +class TypeVarExtractor(TypeQuery[List[TypeVarLikeType]]): + def __init__(self, include_all: bool = False) -> None: + super().__init__(self._merge) + self.include_all = include_all + + def _merge(self, iter: Iterable[list[TypeVarLikeType]]) -> list[TypeVarLikeType]: + out = [] + for item in iter: + out.extend(item) + return out + + def visit_type_var(self, t: TypeVarType) -> list[TypeVarLikeType]: + return [t] + + def visit_param_spec(self, t: ParamSpecType) -> list[TypeVarLikeType]: + return [t] if self.include_all else [] + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> list[TypeVarLikeType]: + return [t] if self.include_all else [] + + +def custom_special_method(typ: Type, name: str, check_all: bool = False) -> bool: + """Does this type have a custom special method such as __format__() or __eq__()? + + If check_all is True ensure all items of a union have a custom method, not just some. + """ + typ = get_proper_type(typ) + if isinstance(typ, Instance): + method = typ.type.get(name) + if method and isinstance(method.node, (SYMBOL_FUNCBASE_TYPES, Decorator, Var)): + if method.node.info: + return not method.node.info.fullname.startswith(("builtins.", "typing.")) + return False + if isinstance(typ, UnionType): + if check_all: + return all(custom_special_method(t, name, check_all) for t in typ.items) + return any(custom_special_method(t, name) for t in typ.items) + if isinstance(typ, TupleType): + return custom_special_method(tuple_fallback(typ), name, check_all) + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + # Look up __method__ on the metaclass for class objects. + return custom_special_method(typ.fallback, name, check_all) + if isinstance(typ, AnyType): + # Avoid false positives in uncertain cases. + return True + # TODO: support other types (see ExpressionChecker.has_member())? + return False + + +def separate_union_literals(t: UnionType) -> tuple[Sequence[LiteralType], Sequence[Type]]: + """Separate literals from other members in a union type.""" + literal_items = [] + union_items = [] + + for item in t.items: + proper = get_proper_type(item) + if isinstance(proper, LiteralType): + literal_items.append(proper) + else: + union_items.append(item) + + return literal_items, union_items + + +def try_getting_instance_fallback(typ: Type) -> Instance | None: + """Returns the Instance fallback for this type if one exists or None.""" + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return typ + elif isinstance(typ, LiteralType): + return typ.fallback + elif isinstance(typ, NoneType): + return None # Fast path for None, which is common + elif isinstance(typ, FunctionLike): + return typ.fallback + elif isinstance(typ, TupleType): + return typ.partial_fallback + elif isinstance(typ, TypedDictType): + return typ.fallback + elif isinstance(typ, TypeVarType): + return try_getting_instance_fallback(typ.upper_bound) + return None + + +def fixup_partial_type(typ: Type) -> Type: + """Convert a partial type that we couldn't resolve into something concrete. + + This means, for None we make it Optional[Any], and for anything else we + fill in all of the type arguments with Any. + """ + if not isinstance(typ, PartialType): + return typ + if typ.type is None: + return UnionType.make_union([AnyType(TypeOfAny.unannotated), NoneType()]) + else: + return Instance(typ.type, [AnyType(TypeOfAny.unannotated)] * len(typ.type.type_vars)) + + +def get_protocol_member(left: Instance, member: str, class_obj: bool) -> ProperType | None: + if member == "__call__" and class_obj: + # Special case: class objects always have __call__ that is just the constructor. + from mypy.checkmember import type_object_type + + def named_type(fullname: str) -> Instance: + return Instance(left.type.mro[-1], []) + + return type_object_type(left.type, named_type) + + if member == "__call__" and left.type.is_metaclass(): + # Special case: we want to avoid falling back to metaclass __call__ + # if constructor signature didn't match, this can cause many false negatives. + return None + + from mypy.subtypes import find_member + + return get_proper_type(find_member(member, left, left, class_obj=class_obj)) diff --git a/mypy/typerepr.py b/mypy/typerepr.py deleted file mode 100644 index e6c8d2523724..000000000000 --- a/mypy/typerepr.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Representation classes for Type subclasses and TypeVars. - -These are used for source-source transformation that preserves original -formatting and comments. -""" - -from typing import List, Any - -from mypy.lex import Token - - -class CommonTypeRepr: - """Representation of UnboundType, Instance and Callable.""" - def __init__(self, components: List[Token], langle, commas: List[Token], - rangle: Any) -> None: - # Note: langle and rangle may be empty. - self.components = components - self.langle = langle - self.commas = commas - self.rangle = rangle - - -class ListTypeRepr: - """Representation of list type t[].""" - def __init__(self, lbracket, rbracket) -> None: - self.lbracket = lbracket - self.rbracket = rbracket - - -class AnyRepr: - """Representation of Any.""" - def __init__(self, any_tok: Any) -> None: - self.any_tok = any_tok - - -class VoidRepr: - """Representation of the 'None' type.""" - def __init__(self, void: Any) -> None: - self.void = void - - -class CallableRepr: - """Representation of Callable.""" - def __init__(self, func: Any, langle: Any, lparen: Any, commas: Any, - rparen: Any, rangle: Any) -> None: - self.func = func - self.langle = langle - self.lparen = lparen - self.commas = commas - self.rparen = rparen - self.rangle = rangle - - -class TypeVarRepr: - """Representation of TypeVar.""" - def __init__(self, name: Any) -> None: - self.name = name - - -class TypeVarsRepr: - """Representation of TypeVars.""" - def __init__(self, langle: Any, commas: List[Token], rangle: Any) -> None: - self.langle = langle - self.commas = commas - self.rangle = rangle - - -class TypeVarDefRepr: - """Representation of TypeVarDef.""" - def __init__(self, name: Any, is_tok: Any) -> None: - # TODO remove is_tok - self.name = name - self.is_tok = is_tok diff --git a/mypy/types.py b/mypy/types.py index ebe78d3fe768..e92ab0889991 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1,351 +1,2342 @@ """Classes for representing mypy types.""" +from __future__ import annotations + +import sys from abc import abstractmethod -from typing import Undefined, Any, typevar, List, Tuple, cast, Generic, Set +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Dict, + Final, + Iterable, + NamedTuple, + NewType, + Sequence, + TypeVar, + Union, + cast, +) +from typing_extensions import Self, TypeAlias as _TypeAlias, TypeGuard, overload import mypy.nodes +from mypy.bogus_type import Bogus +from mypy.nodes import ( + ARG_POS, + ARG_STAR, + ARG_STAR2, + INVARIANT, + ArgKind, + FakeInfo, + FuncDef, + SymbolNode, +) +from mypy.options import Options +from mypy.state import state +from mypy.util import IdMapper + +T = TypeVar("T") + +JsonDict: _TypeAlias = Dict[str, Any] + +# The set of all valid expressions that can currently be contained +# inside of a Literal[...]. +# +# Literals can contain bytes and enum-values: we special-case both of these +# and store the value as a string. We rely on the fallback type that's also +# stored with the Literal to determine how a string is being used. +# +# TODO: confirm that we're happy with representing enums (and the +# other types) in the manner described above. +# +# Note: if we change the set of types included below, we must also +# make sure to audit the following methods: +# +# 1. types.LiteralType's serialize and deserialize methods: this method +# needs to make sure it can convert the below types into JSON and back. +# +# 2. types.LiteralType's 'value_repr` method: this method is ultimately used +# by TypeStrVisitor's visit_literal_type to generate a reasonable +# repr-able output. +# +# 3. server.astdiff.SnapshotTypeVisitor's visit_literal_type_method: this +# method assumes that the following types supports equality checks and +# hashability. +# +# Note: Although "Literal[None]" is a valid type, we internally always convert +# such a type directly into "None". So, "None" is not a valid parameter of +# LiteralType and is omitted from this list. +# +# Note: Float values are only used internally. They are not accepted within +# Literal[...]. +LiteralValue: _TypeAlias = Union[int, str, bool, float] + + +# If we only import type_visitor in the middle of the file, mypy +# breaks, and if we do it at the top, it breaks at runtime because of +# import cycle issues, so we do it at the top while typechecking and +# then again in the middle at runtime. +# We should be able to remove this once we are switched to the new +# semantic analyzer! +if TYPE_CHECKING: + from mypy.type_visitor import ( + SyntheticTypeVisitor as SyntheticTypeVisitor, + TypeVisitor as TypeVisitor, + ) + +TYPE_VAR_LIKE_NAMES: Final = ( + "typing.TypeVar", + "typing_extensions.TypeVar", + "typing.ParamSpec", + "typing_extensions.ParamSpec", + "typing.TypeVarTuple", + "typing_extensions.TypeVarTuple", +) + +TYPED_NAMEDTUPLE_NAMES: Final = ("typing.NamedTuple", "typing_extensions.NamedTuple") + +# Supported names of TypedDict type constructors. +TPDICT_NAMES: Final = ( + "typing.TypedDict", + "typing_extensions.TypedDict", + "mypy_extensions.TypedDict", +) + +# Supported fallback instance type names for TypedDict types. +TPDICT_FB_NAMES: Final = ( + "typing._TypedDict", + "typing_extensions._TypedDict", + "mypy_extensions._TypedDict", +) + +# Supported names of Protocol base class. +PROTOCOL_NAMES: Final = ("typing.Protocol", "typing_extensions.Protocol") + +# Supported TypeAlias names. +TYPE_ALIAS_NAMES: Final = ("typing.TypeAlias", "typing_extensions.TypeAlias") + +# Supported Final type names. +FINAL_TYPE_NAMES: Final = ("typing.Final", "typing_extensions.Final") + +# Supported @final decorator names. +FINAL_DECORATOR_NAMES: Final = ("typing.final", "typing_extensions.final") + +# Supported @type_check_only names. +TYPE_CHECK_ONLY_NAMES: Final = ("typing.type_check_only", "typing_extensions.type_check_only") + +# Supported Literal type names. +LITERAL_TYPE_NAMES: Final = ("typing.Literal", "typing_extensions.Literal") + +# Supported Annotated type names. +ANNOTATED_TYPE_NAMES: Final = ("typing.Annotated", "typing_extensions.Annotated") + +# Supported @deprecated type names +DEPRECATED_TYPE_NAMES: Final = ("warnings.deprecated", "typing_extensions.deprecated") + +# We use this constant in various places when checking `tuple` subtyping: +TUPLE_LIKE_INSTANCE_NAMES: Final = ( + "builtins.tuple", + "typing.Iterable", + "typing.Container", + "typing.Sequence", + "typing.Reversible", +) + +IMPORTED_REVEAL_TYPE_NAMES: Final = ("typing.reveal_type", "typing_extensions.reveal_type") +REVEAL_TYPE_NAMES: Final = ("builtins.reveal_type", *IMPORTED_REVEAL_TYPE_NAMES) + +ASSERT_TYPE_NAMES: Final = ("typing.assert_type", "typing_extensions.assert_type") + +OVERLOAD_NAMES: Final = ("typing.overload", "typing_extensions.overload") + +NEVER_NAMES: Final = ( + "typing.NoReturn", + "typing_extensions.NoReturn", + "mypy_extensions.NoReturn", + "typing.Never", + "typing_extensions.Never", +) + +# Mypyc fixed-width native int types (compatible with builtins.int) +MYPYC_NATIVE_INT_NAMES: Final = ( + "mypy_extensions.i64", + "mypy_extensions.i32", + "mypy_extensions.i16", + "mypy_extensions.u8", +) + +DATACLASS_TRANSFORM_NAMES: Final = ( + "typing.dataclass_transform", + "typing_extensions.dataclass_transform", +) +# Supported @override decorator names. +OVERRIDE_DECORATOR_NAMES: Final = ("typing.override", "typing_extensions.override") + +# A placeholder used for Bogus[...] parameters +_dummy: Final[Any] = object() + +# A placeholder for int parameters +_dummy_int: Final = -999999 + + +class TypeOfAny: + """ + This class describes different types of Any. Each 'Any' can be of only one type at a time. + """ - -T = typevar('T') + __slots__ = () + + # Was this Any type inferred without a type annotation? + unannotated: Final = 1 + # Does this Any come from an explicit type annotation? + explicit: Final = 2 + # Does this come from an unfollowed import? See --disallow-any-unimported option + from_unimported_type: Final = 3 + # Does this Any type come from omitted generics? + from_omitted_generics: Final = 4 + # Does this Any come from an error? + from_error: Final = 5 + # Is this a type that can't be represented in mypy's type system? For instance, type of + # call to NewType(...). Even though these types aren't real Anys, we treat them as such. + # Also used for variables named '_'. + special_form: Final = 6 + # Does this Any come from interaction with another Any? + from_another_any: Final = 7 + # Does this Any come from an implementation limitation/bug? + implementation_artifact: Final = 8 + # Does this Any come from use in the suggestion engine? This is + # used to ignore Anys inserted by the suggestion engine when + # generating constraints. + suggestion_engine: Final = 9 + + +def deserialize_type(data: JsonDict | str) -> Type: + if isinstance(data, str): + return Instance.deserialize(data) + classname = data[".class"] + method = deserialize_map.get(classname) + if method is not None: + return method(data) + raise NotImplementedError(f"unexpected .class {classname}") class Type(mypy.nodes.Context): """Abstract base class for all types.""" - line = 0 - repr = Undefined(Any) + __slots__ = ("_can_be_true", "_can_be_false") + # 'can_be_true' and 'can_be_false' mean whether the value of the + # expression can be true or false in a boolean context. They are useful + # when inferring the type of logic expressions like `x and y`. + # + # For example: + # * the literal `False` can't be true while `True` can. + # * a value with type `bool` can be true or false. + # * `None` can't be true + # * ... + + def __init__(self, line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + # Value of these can be -1 (use the default, lazy init), 0 (false) or 1 (true) + self._can_be_true = -1 + self._can_be_false = -1 + + @property + def can_be_true(self) -> bool: + if self._can_be_true == -1: # Lazy init helps mypyc + self._can_be_true = self.can_be_true_default() + return bool(self._can_be_true) + + @can_be_true.setter + def can_be_true(self, v: bool) -> None: + self._can_be_true = v + + @property + def can_be_false(self) -> bool: + if self._can_be_false == -1: # Lazy init helps mypyc + self._can_be_false = self.can_be_false_default() + return bool(self._can_be_false) + + @can_be_false.setter + def can_be_false(self, v: bool) -> None: + self._can_be_false = v + + def can_be_true_default(self) -> bool: + return True + + def can_be_false_default(self) -> bool: + return True + + def accept(self, visitor: TypeVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) - def __init__(self, line: int = -1, repr=None) -> None: - self.line = line - self.repr = repr + def __repr__(self) -> str: + return self.accept(TypeStrVisitor(options=Options())) + + def str_with_options(self, options: Options) -> str: + return self.accept(TypeStrVisitor(options=options)) + + def serialize(self) -> JsonDict | str: + raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") + + @classmethod + def deserialize(cls, data: JsonDict) -> Type: + raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") + + def is_singleton_type(self) -> bool: + return False + + +class TypeAliasType(Type): + """A type alias to another type. + + To support recursive type aliases we don't immediately expand a type alias + during semantic analysis, but create an instance of this type that records the target alias + definition node (mypy.nodes.TypeAlias) and type arguments (for generic aliases). + + This is very similar to how TypeInfo vs Instance interact, where a recursive class-based + structure like + class Node: + value: int + children: List[Node] + can be represented in a tree-like manner. + """ + + __slots__ = ("alias", "args", "type_ref") + + def __init__( + self, + alias: mypy.nodes.TypeAlias | None, + args: list[Type], + line: int = -1, + column: int = -1, + ) -> None: + super().__init__(line, column) + self.alias = alias + self.args = args + self.type_ref: str | None = None + + def _expand_once(self) -> Type: + """Expand to the target type exactly once. + + This doesn't do full expansion, i.e. the result can contain another + (or even this same) type alias. Use this internal helper only when really needed, + its public wrapper mypy.types.get_proper_type() is preferred. + """ + assert self.alias is not None + if self.alias.no_args: + # We know that no_args=True aliases like L = List must have an instance + # as their target. + assert isinstance(self.alias.target, Instance) # type: ignore[misc] + return self.alias.target.copy_modified(args=self.args) + + # TODO: this logic duplicates the one in expand_type_by_instance(). + if self.alias.tvar_tuple_index is None: + mapping = {v.id: s for (v, s) in zip(self.alias.alias_tvars, self.args)} + else: + prefix = self.alias.tvar_tuple_index + suffix = len(self.alias.alias_tvars) - self.alias.tvar_tuple_index - 1 + start, middle, end = split_with_prefix_and_suffix(tuple(self.args), prefix, suffix) + tvar = self.alias.alias_tvars[prefix] + assert isinstance(tvar, TypeVarTupleType) + mapping = {tvar.id: TupleType(list(middle), tvar.tuple_fallback)} + for tvar, sub in zip( + self.alias.alias_tvars[:prefix] + self.alias.alias_tvars[prefix + 1 :], start + end + ): + mapping[tvar.id] = sub + + new_tp = self.alias.target.accept(InstantiateAliasVisitor(mapping)) + new_tp.accept(LocationSetter(self.line, self.column)) + new_tp.line = self.line + new_tp.column = self.column + return new_tp + + def _partial_expansion(self, nothing_args: bool = False) -> tuple[ProperType, bool]: + # Private method mostly for debugging and testing. + unroller = UnrollAliasVisitor(set(), {}) + if nothing_args: + alias = self.copy_modified(args=[UninhabitedType()] * len(self.args)) + else: + alias = self + unrolled = alias.accept(unroller) + assert isinstance(unrolled, ProperType) + return unrolled, unroller.recursed + + def expand_all_if_possible(self, nothing_args: bool = False) -> ProperType | None: + """Attempt a full expansion of the type alias (including nested aliases). + + If the expansion is not possible, i.e. the alias is (mutually-)recursive, + return None. If nothing_args is True, replace all type arguments with an + UninhabitedType() (used to detect recursively defined aliases). + """ + unrolled, recursed = self._partial_expansion(nothing_args=nothing_args) + if recursed: + return None + return unrolled + + @property + def is_recursive(self) -> bool: + """Whether this type alias is recursive. + + Note this doesn't check generic alias arguments, but only if this alias + *definition* is recursive. The property value thus can be cached on the + underlying TypeAlias node. If you want to include all nested types, use + has_recursive_types() function. + """ + assert self.alias is not None, "Unfixed type alias" + is_recursive = self.alias._is_recursive + if is_recursive is None: + is_recursive = self.expand_all_if_possible(nothing_args=True) is None + # We cache the value on the underlying TypeAlias node as an optimization, + # since the value is the same for all instances of the same alias. + self.alias._is_recursive = is_recursive + return is_recursive + + def can_be_true_default(self) -> bool: + if self.alias is not None: + return self.alias.target.can_be_true + return super().can_be_true_default() + + def can_be_false_default(self) -> bool: + if self.alias is not None: + return self.alias.target.can_be_false + return super().can_be_false_default() + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_type_alias_type(self) + + def __hash__(self) -> int: + return hash((self.alias, tuple(self.args))) + + def __eq__(self, other: object) -> bool: + # Note: never use this to determine subtype relationships, use is_subtype(). + if not isinstance(other, TypeAliasType): + return NotImplemented + return self.alias == other.alias and self.args == other.args + + def serialize(self) -> JsonDict: + assert self.alias is not None + data: JsonDict = { + ".class": "TypeAliasType", + "type_ref": self.alias.fullname, + "args": [arg.serialize() for arg in self.args], + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeAliasType: + assert data[".class"] == "TypeAliasType" + args: list[Type] = [] + if "args" in data: + args_list = data["args"] + assert isinstance(args_list, list) + args = [deserialize_type(arg) for arg in args_list] + alias = TypeAliasType(None, args) + alias.type_ref = data["type_ref"] + return alias + + def copy_modified(self, *, args: list[Type] | None = None) -> TypeAliasType: + return TypeAliasType( + self.alias, args if args is not None else self.args.copy(), self.line, self.column + ) + + +class TypeGuardedType(Type): + """Only used by find_isinstance_check() etc.""" + + __slots__ = ("type_guard",) + + def __init__(self, type_guard: Type) -> None: + super().__init__(line=type_guard.line, column=type_guard.column) + self.type_guard = type_guard + + def __repr__(self) -> str: + return f"TypeGuard({self.type_guard})" + + +class RequiredType(Type): + """Required[T] or NotRequired[T]. Only usable at top-level of a TypedDict definition.""" + + def __init__(self, item: Type, *, required: bool) -> None: + super().__init__(line=item.line, column=item.column) + self.item = item + self.required = required + + def __repr__(self) -> str: + if self.required: + return f"Required[{self.item}]" + else: + return f"NotRequired[{self.item}]" + + def accept(self, visitor: TypeVisitor[T]) -> T: + return self.item.accept(visitor) - def get_line(self) -> int: - return self.line - def accept(self, visitor: 'TypeVisitor[T]') -> T: - raise RuntimeError('Not implemented') +class ReadOnlyType(Type): + """ReadOnly[T] Only usable at top-level of a TypedDict definition.""" + + def __init__(self, item: Type) -> None: + super().__init__(line=item.line, column=item.column) + self.item = item def __repr__(self) -> str: - return self.accept(TypeStrVisitor()) + return f"ReadOnly[{self.item}]" + + def accept(self, visitor: TypeVisitor[T]) -> T: + return self.item.accept(visitor) + + +class ProperType(Type): + """Not a type alias. + + Every type except TypeAliasType must inherit from this type. + """ + + __slots__ = () + + +class TypeVarId: + # A type variable is uniquely identified by its raw id and meta level. + + # For plain variables (type parameters of generic classes and + # functions) raw ids are allocated by semantic analysis, using + # positive ids 1, 2, ... for generic class parameters and negative + # ids -1, ... for generic function type arguments. A special value 0 + # is reserved for Self type variable (autogenerated). This convention + # is only used to keep type variable ids distinct when allocating + # them; the type checker makes no distinction between class and + # function type variables. + + # Metavariables are allocated unique ids starting from 1. + raw_id: int + # Level of the variable in type inference. Currently either 0 for + # declared types, or 1 for type inference metavariables. + meta_level: int = 0 -class TypeVarDef(mypy.nodes.Context): - """Definition of a single type variable.""" + # Class variable used for allocating fresh ids for metavariables. + next_raw_id: ClassVar[int] = 1 - name = '' - id = 0 - values = Undefined(List[Type]) - upper_bound = Undefined(Type) - line = 0 - repr = Undefined(Any) + # Fullname of class or function/method which declares this type + # variable (not the fullname of the TypeVar definition!), or '' + namespace: str - def __init__(self, name: str, id: int, values: List[Type], upper_bound: Type, line: int = -1, - repr: Any = None) -> None: + def __init__(self, raw_id: int, meta_level: int = 0, *, namespace: str = "") -> None: + self.raw_id = raw_id + self.meta_level = meta_level + self.namespace = namespace + + @staticmethod + def new(meta_level: int) -> TypeVarId: + raw_id = TypeVarId.next_raw_id + TypeVarId.next_raw_id += 1 + return TypeVarId(raw_id, meta_level) + + def __repr__(self) -> str: + return self.raw_id.__repr__() + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, TypeVarId) + and self.raw_id == other.raw_id + and self.meta_level == other.meta_level + and self.namespace == other.namespace + ) + + def __ne__(self, other: object) -> bool: + return not (self == other) + + def __hash__(self) -> int: + return hash((self.raw_id, self.meta_level, self.namespace)) + + def is_meta_var(self) -> bool: + return self.meta_level > 0 + + def is_self(self) -> bool: + # This is a special value indicating typing.Self variable. + return self.raw_id == 0 + + +class TypeVarLikeType(ProperType): + __slots__ = ("name", "fullname", "id", "upper_bound", "default") + + name: str # Name (may be qualified) + fullname: str # Fully qualified name + id: TypeVarId + upper_bound: Type + default: Type + + def __init__( + self, + name: str, + fullname: str, + id: TypeVarId, + upper_bound: Type, + default: Type, + line: int = -1, + column: int = -1, + ) -> None: + super().__init__(line, column) self.name = name + self.fullname = fullname self.id = id - self.values = values self.upper_bound = upper_bound - self.line = line - self.repr = repr + self.default = default - def get_line(self) -> int: - return self.line + def serialize(self) -> JsonDict: + raise NotImplementedError - def __repr__(self) -> str: - if self.values: - return '{} in {}'.format(self.name, tuple(self.values)) - else: - return self.name + @classmethod + def deserialize(cls, data: JsonDict) -> TypeVarLikeType: + raise NotImplementedError + def copy_modified(self, *, id: TypeVarId, **kwargs: Any) -> Self: + raise NotImplementedError -class UnboundType(Type): - """Instance type that has not been bound during semantic analysis.""" + @classmethod + def new_unification_variable(cls, old: Self) -> Self: + new_id = TypeVarId.new(meta_level=1) + return old.copy_modified(id=new_id) + + def has_default(self) -> bool: + t = get_proper_type(self.default) + return not (isinstance(t, AnyType) and t.type_of_any == TypeOfAny.from_omitted_generics) - name = '' - args = Undefined(List[Type]) - def __init__(self, name: str, args: List[Type] = None, line: int = -1, - repr: Any = None) -> None: +class TypeVarType(TypeVarLikeType): + """Type that refers to a type variable.""" + + __slots__ = ("values", "variance") + + values: list[Type] # Value restriction, empty list if no restriction + variance: int + + def __init__( + self, + name: str, + fullname: str, + id: TypeVarId, + values: list[Type], + upper_bound: Type, + default: Type, + variance: int = INVARIANT, + line: int = -1, + column: int = -1, + ) -> None: + super().__init__(name, fullname, id, upper_bound, default, line, column) + assert values is not None, "No restrictions must be represented by empty list" + self.values = values + self.variance = variance + + def copy_modified( + self, + *, + values: Bogus[list[Type]] = _dummy, + upper_bound: Bogus[Type] = _dummy, + default: Bogus[Type] = _dummy, + id: Bogus[TypeVarId] = _dummy, + line: int = _dummy_int, + column: int = _dummy_int, + **kwargs: Any, + ) -> TypeVarType: + return TypeVarType( + name=self.name, + fullname=self.fullname, + id=self.id if id is _dummy else id, + values=self.values if values is _dummy else values, + upper_bound=self.upper_bound if upper_bound is _dummy else upper_bound, + default=self.default if default is _dummy else default, + variance=self.variance, + line=self.line if line == _dummy_int else line, + column=self.column if column == _dummy_int else column, + ) + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_type_var(self) + + def __hash__(self) -> int: + return hash((self.id, self.upper_bound, tuple(self.values))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypeVarType): + return NotImplemented + return ( + self.id == other.id + and self.upper_bound == other.upper_bound + and self.values == other.values + ) + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return { + ".class": "TypeVarType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "values": [v.serialize() for v in self.values], + "upper_bound": self.upper_bound.serialize(), + "default": self.default.serialize(), + "variance": self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeVarType: + assert data[".class"] == "TypeVarType" + return TypeVarType( + name=data["name"], + fullname=data["fullname"], + id=TypeVarId(data["id"], namespace=data["namespace"]), + values=[deserialize_type(v) for v in data["values"]], + upper_bound=deserialize_type(data["upper_bound"]), + default=deserialize_type(data["default"]), + variance=data["variance"], + ) + + +class ParamSpecFlavor: + # Simple ParamSpec reference such as "P" + BARE: Final = 0 + # P.args + ARGS: Final = 1 + # P.kwargs + KWARGS: Final = 2 + + +class ParamSpecType(TypeVarLikeType): + """Type that refers to a ParamSpec. + + A ParamSpec is a type variable that represents the parameter + types, names and kinds of a callable (i.e., the signature without + the return type). + + This can be one of these forms + * P (ParamSpecFlavor.BARE) + * P.args (ParamSpecFlavor.ARGS) + * P.kwargs (ParamSpecFLavor.KWARGS) + + The upper_bound is really used as a fallback type -- it's shared + with TypeVarType for simplicity. It can't be specified by the user + and the value is directly derived from the flavor (currently + always just 'object'). + """ + + __slots__ = ("flavor", "prefix") + + flavor: int + prefix: Parameters + + def __init__( + self, + name: str, + fullname: str, + id: TypeVarId, + flavor: int, + upper_bound: Type, + default: Type, + *, + line: int = -1, + column: int = -1, + prefix: Parameters | None = None, + ) -> None: + super().__init__(name, fullname, id, upper_bound, default, line=line, column=column) + self.flavor = flavor + self.prefix = prefix or Parameters([], [], []) + + def with_flavor(self, flavor: int) -> ParamSpecType: + return ParamSpecType( + self.name, + self.fullname, + self.id, + flavor, + upper_bound=self.upper_bound, + default=self.default, + prefix=self.prefix, + ) + + def copy_modified( + self, + *, + id: Bogus[TypeVarId] = _dummy, + flavor: int = _dummy_int, + prefix: Bogus[Parameters] = _dummy, + default: Bogus[Type] = _dummy, + **kwargs: Any, + ) -> ParamSpecType: + return ParamSpecType( + self.name, + self.fullname, + id if id is not _dummy else self.id, + flavor if flavor != _dummy_int else self.flavor, + self.upper_bound, + default=default if default is not _dummy else self.default, + line=self.line, + column=self.column, + prefix=prefix if prefix is not _dummy else self.prefix, + ) + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_param_spec(self) + + def name_with_suffix(self) -> str: + n = self.name + if self.flavor == ParamSpecFlavor.ARGS: + return f"{n}.args" + elif self.flavor == ParamSpecFlavor.KWARGS: + return f"{n}.kwargs" + return n + + def __hash__(self) -> int: + return hash((self.id, self.flavor, self.prefix)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ParamSpecType): + return NotImplemented + # Upper bound can be ignored, since it's determined by flavor. + return self.id == other.id and self.flavor == other.flavor and self.prefix == other.prefix + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return { + ".class": "ParamSpecType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "flavor": self.flavor, + "upper_bound": self.upper_bound.serialize(), + "default": self.default.serialize(), + "prefix": self.prefix.serialize(), + } + + @classmethod + def deserialize(cls, data: JsonDict) -> ParamSpecType: + assert data[".class"] == "ParamSpecType" + return ParamSpecType( + data["name"], + data["fullname"], + TypeVarId(data["id"], namespace=data["namespace"]), + data["flavor"], + deserialize_type(data["upper_bound"]), + deserialize_type(data["default"]), + prefix=Parameters.deserialize(data["prefix"]), + ) + + +class TypeVarTupleType(TypeVarLikeType): + """Type that refers to a TypeVarTuple. + + See PEP646 for more information. + """ + + __slots__ = ("tuple_fallback", "min_len") + + def __init__( + self, + name: str, + fullname: str, + id: TypeVarId, + upper_bound: Type, + tuple_fallback: Instance, + default: Type, + *, + line: int = -1, + column: int = -1, + min_len: int = 0, + ) -> None: + super().__init__(name, fullname, id, upper_bound, default, line=line, column=column) + self.tuple_fallback = tuple_fallback + # This value is not settable by a user. It is an internal-only thing to support + # len()-narrowing of variadic tuples. + self.min_len = min_len + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return { + ".class": "TypeVarTupleType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "upper_bound": self.upper_bound.serialize(), + "tuple_fallback": self.tuple_fallback.serialize(), + "default": self.default.serialize(), + "min_len": self.min_len, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeVarTupleType: + assert data[".class"] == "TypeVarTupleType" + return TypeVarTupleType( + data["name"], + data["fullname"], + TypeVarId(data["id"], namespace=data["namespace"]), + deserialize_type(data["upper_bound"]), + Instance.deserialize(data["tuple_fallback"]), + deserialize_type(data["default"]), + min_len=data["min_len"], + ) + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_type_var_tuple(self) + + def __hash__(self) -> int: + return hash((self.id, self.min_len)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypeVarTupleType): + return NotImplemented + return self.id == other.id and self.min_len == other.min_len + + def copy_modified( + self, + *, + id: Bogus[TypeVarId] = _dummy, + upper_bound: Bogus[Type] = _dummy, + default: Bogus[Type] = _dummy, + min_len: Bogus[int] = _dummy, + **kwargs: Any, + ) -> TypeVarTupleType: + return TypeVarTupleType( + self.name, + self.fullname, + self.id if id is _dummy else id, + self.upper_bound if upper_bound is _dummy else upper_bound, + self.tuple_fallback, + self.default if default is _dummy else default, + line=self.line, + column=self.column, + min_len=self.min_len if min_len is _dummy else min_len, + ) + + +class UnboundType(ProperType): + """Instance type that has not been bound during semantic analysis.""" + + __slots__ = ( + "name", + "args", + "optional", + "empty_tuple_index", + "original_str_expr", + "original_str_fallback", + ) + + def __init__( + self, + name: str, + args: Sequence[Type] | None = None, + line: int = -1, + column: int = -1, + optional: bool = False, + empty_tuple_index: bool = False, + original_str_expr: str | None = None, + original_str_fallback: str | None = None, + ) -> None: + super().__init__(line, column) if not args: args = [] self.name = name - self.args = args - super().__init__(line, repr) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: + self.args = tuple(args) + # Should this type be wrapped in an Optional? + self.optional = optional + # Special case for X[()] + self.empty_tuple_index = empty_tuple_index + # If this UnboundType was originally defined as a str or bytes, keep track of + # the original contents of that string-like thing. This way, if this UnboundExpr + # ever shows up inside of a LiteralType, we can determine whether that + # Literal[...] is valid or not. E.g. Literal[foo] is most likely invalid + # (unless 'foo' is an alias for another literal or something) and + # Literal["foo"] most likely is. + # + # We keep track of the entire string instead of just using a boolean flag + # so we can distinguish between things like Literal["foo"] vs + # Literal[" foo "]. + # + # We also keep track of what the original base fallback type was supposed to be + # so we don't have to try and recompute it later + self.original_str_expr = original_str_expr + self.original_str_fallback = original_str_fallback + + def copy_modified(self, args: Bogus[Sequence[Type] | None] = _dummy) -> UnboundType: + if args is _dummy: + args = self.args + return UnboundType( + name=self.name, + args=args, + line=self.line, + column=self.column, + optional=self.optional, + empty_tuple_index=self.empty_tuple_index, + original_str_expr=self.original_str_expr, + original_str_fallback=self.original_str_fallback, + ) + + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_unbound_type(self) + def __hash__(self) -> int: + return hash((self.name, self.optional, tuple(self.args), self.original_str_expr)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UnboundType): + return NotImplemented + return ( + self.name == other.name + and self.optional == other.optional + and self.args == other.args + and self.original_str_expr == other.original_str_expr + and self.original_str_fallback == other.original_str_fallback + ) + + def serialize(self) -> JsonDict: + return { + ".class": "UnboundType", + "name": self.name, + "args": [a.serialize() for a in self.args], + "expr": self.original_str_expr, + "expr_fallback": self.original_str_fallback, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> UnboundType: + assert data[".class"] == "UnboundType" + return UnboundType( + data["name"], + [deserialize_type(a) for a in data["args"]], + original_str_expr=data["expr"], + original_str_fallback=data["expr_fallback"], + ) + + +class CallableArgument(ProperType): + """Represents a Arg(type, 'name') inside a Callable's type list. + + Note that this is a synthetic type for helping parse ASTs, not a real type. + """ -class ErrorType(Type): - """The error type is used as the result of failed type operations.""" + __slots__ = ("typ", "name", "constructor") + + typ: Type + name: str | None + constructor: str | None + + def __init__( + self, + typ: Type, + name: str | None, + constructor: str | None, + line: int = -1, + column: int = -1, + ) -> None: + super().__init__(line, column) + self.typ = typ + self.name = name + self.constructor = constructor + + def accept(self, visitor: TypeVisitor[T]) -> T: + assert isinstance(visitor, SyntheticTypeVisitor) + ret: T = visitor.visit_callable_argument(self) + return ret - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_error_type(self) + def serialize(self) -> JsonDict: + assert False, "Synthetic types don't serialize" -class TypeList(Type): - """A list of types [...]. +class TypeList(ProperType): + """Information about argument types and names [...]. - This is only used for the arguments of a Function type, i.e. for - [arg, ...] in Function[[arg, ...], ret]. + This is used for the arguments of a Callable type, i.e. for + [arg, ...] in Callable[[arg, ...], ret]. This is not a real type + but a syntactic AST construct. UnboundTypes can also have TypeList + types before they are processed into Callable types. """ - items = Undefined(List[Type]) + __slots__ = ("items",) + + items: list[Type] - def __init__(self, items: List[Type], line: int = -1, - repr: Any = None) -> None: - super().__init__(line, repr) + def __init__(self, items: list[Type], line: int = -1, column: int = -1) -> None: + super().__init__(line, column) self.items = items - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_list(self) + def accept(self, visitor: TypeVisitor[T]) -> T: + assert isinstance(visitor, SyntheticTypeVisitor) + ret: T = visitor.visit_type_list(self) + return ret + + def serialize(self) -> JsonDict: + assert False, "Synthetic types don't serialize" + + def __hash__(self) -> int: + return hash(tuple(self.items)) + def __eq__(self, other: object) -> bool: + return isinstance(other, TypeList) and self.items == other.items -class AnyType(Type): + +class UnpackType(ProperType): + """Type operator Unpack from PEP646. Can be either with Unpack[] + or unpacking * syntax. + + The inner type should be either a TypeVarTuple, or a variable length tuple. + In an exceptional case of callable star argument it can be a fixed length tuple. + + Note: the above restrictions are only guaranteed by normalizations after semantic + analysis, if your code needs to handle UnpackType *during* semantic analysis, it is + wild west, technically anything can be present in the wrapped type. + """ + + __slots__ = ["type", "from_star_syntax"] + + def __init__( + self, typ: Type, line: int = -1, column: int = -1, from_star_syntax: bool = False + ) -> None: + super().__init__(line, column) + self.type = typ + self.from_star_syntax = from_star_syntax + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_unpack_type(self) + + def serialize(self) -> JsonDict: + return {".class": "UnpackType", "type": self.type.serialize()} + + @classmethod + def deserialize(cls, data: JsonDict) -> UnpackType: + assert data[".class"] == "UnpackType" + typ = data["type"] + return UnpackType(deserialize_type(typ)) + + def __hash__(self) -> int: + return hash(self.type) + + def __eq__(self, other: object) -> bool: + return isinstance(other, UnpackType) and self.type == other.type + + +class AnyType(ProperType): """The type 'Any'.""" - def accept(self, visitor: 'TypeVisitor[T]') -> T: + __slots__ = ("type_of_any", "source_any", "missing_import_name") + + def __init__( + self, + type_of_any: int, + source_any: AnyType | None = None, + missing_import_name: str | None = None, + line: int = -1, + column: int = -1, + ) -> None: + super().__init__(line, column) + self.type_of_any = type_of_any + # If this Any was created as a result of interacting with another 'Any', record the source + # and use it in reports. + self.source_any = source_any + if source_any and source_any.source_any: + self.source_any = source_any.source_any + + if source_any is None: + self.missing_import_name = missing_import_name + else: + self.missing_import_name = source_any.missing_import_name + + # Only unimported type anys and anys from other anys should have an import name + assert missing_import_name is None or type_of_any in ( + TypeOfAny.from_unimported_type, + TypeOfAny.from_another_any, + ) + # Only Anys that come from another Any can have source_any. + assert type_of_any != TypeOfAny.from_another_any or source_any is not None + # We should not have chains of Anys. + assert not self.source_any or self.source_any.type_of_any != TypeOfAny.from_another_any + + @property + def is_from_error(self) -> bool: + return self.type_of_any == TypeOfAny.from_error + + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_any(self) + def copy_modified( + self, + # Mark with Bogus because _dummy is just an object (with type Any) + type_of_any: int = _dummy_int, + original_any: Bogus[AnyType | None] = _dummy, + missing_import_name: Bogus[str | None] = _dummy, + ) -> AnyType: + if type_of_any == _dummy_int: + type_of_any = self.type_of_any + if original_any is _dummy: + original_any = self.source_any + if missing_import_name is _dummy: + missing_import_name = self.missing_import_name + return AnyType( + type_of_any=type_of_any, + source_any=original_any, + missing_import_name=missing_import_name, + line=self.line, + column=self.column, + ) + + def __hash__(self) -> int: + return hash(AnyType) + + def __eq__(self, other: object) -> bool: + return isinstance(other, AnyType) + + def serialize(self) -> JsonDict: + return { + ".class": "AnyType", + "type_of_any": self.type_of_any, + "source_any": self.source_any.serialize() if self.source_any is not None else None, + "missing_import_name": self.missing_import_name, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> AnyType: + assert data[".class"] == "AnyType" + source = data["source_any"] + return AnyType( + data["type_of_any"], + AnyType.deserialize(source) if source is not None else None, + data["missing_import_name"], + ) + + +class UninhabitedType(ProperType): + """This type has no members. + + This type is the bottom type. + With strict Optional checking, it is the only common subtype between all + other types, which allows `meet` to be well defined. Without strict + Optional checking, NoneType fills this role. + + In general, for any type T: + join(UninhabitedType, T) = T + meet(UninhabitedType, T) = UninhabitedType + is_subtype(UninhabitedType, T) = True + """ -class Void(Type): - """The return type 'None'. + __slots__ = ("ambiguous",) - This can only be used as the return type in a callable type and as - the result type of calling such callable. - """ + ambiguous: bool # Is this a result of inference for a variable without constraints? - source = '' # May be None; function that generated this value + def __init__(self, line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + self.ambiguous = False - def __init__(self, source: str = None, line: int = -1, - repr: Any = None) -> None: - self.source = source - super().__init__(line, repr) + def can_be_true_default(self) -> bool: + return False + + def can_be_false_default(self) -> bool: + return False + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_uninhabited_type(self) + + def __hash__(self) -> int: + return hash(UninhabitedType) + + def __eq__(self, other: object) -> bool: + return isinstance(other, UninhabitedType) - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_void(self) + def serialize(self) -> JsonDict: + return {".class": "UninhabitedType"} - def with_source(self, source: str) -> 'Void': - return Void(source, self.line, self.repr) + @classmethod + def deserialize(cls, data: JsonDict) -> UninhabitedType: + assert data[".class"] == "UninhabitedType" + return UninhabitedType() -class NoneTyp(Type): +class NoneType(ProperType): """The type of 'None'. - This is only used internally during type inference. Programs - cannot declare a variable of this type, and the type checker - refuses to infer this type for a variable. However, subexpressions - often have this type. Note that this is not used as the result - type when calling a function with a void type, even though - semantically such a function returns a None value; the void type - is used instead so that we can report an error if the caller tries - to do anything with the return value. + This type can be written by users as 'None'. """ - def __init__(self, line: int = -1, repr=None) -> None: - super().__init__(line, repr) + __slots__ = () - def accept(self, visitor: 'TypeVisitor[T]') -> T: + def __init__(self, line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + + def can_be_true_default(self) -> bool: + return False + + def __hash__(self) -> int: + return hash(NoneType) + + def __eq__(self, other: object) -> bool: + return isinstance(other, NoneType) + + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_none_type(self) + def serialize(self) -> JsonDict: + return {".class": "NoneType"} + + @classmethod + def deserialize(cls, data: JsonDict) -> NoneType: + assert data[".class"] == "NoneType" + return NoneType() + + def is_singleton_type(self) -> bool: + return True -class ErasedType(Type): + +# NoneType used to be called NoneTyp so to avoid needlessly breaking +# external plugins we keep that alias here. +NoneTyp = NoneType + + +class ErasedType(ProperType): """Placeholder for an erased type. This is used during type inference. This has the special property that it is ignored during type inference. """ - def accept(self, visitor: 'TypeVisitor[T]') -> T: + __slots__ = () + + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_erased_type(self) -class Instance(Type): - """An instance type of form C[T1, ..., Tn]. +class DeletedType(ProperType): + """Type of deleted variables. - The list of type variables may be empty. + These can be used as lvalues but not rvalues. """ - type = Undefined(mypy.nodes.TypeInfo) - args = Undefined(List[Type]) - erased = False # True if result of type variable substitution + __slots__ = ("source",) - def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], - line: int = -1, repr: Any = None, - erased: Any = False) -> None: - self.type = typ - self.args = args - self.erased = erased - super().__init__(line, repr) + source: str | None # May be None; name that generated this value - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_instance(self) + def __init__(self, source: str | None = None, line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + self.source = source + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_deleted_type(self) + def serialize(self) -> JsonDict: + return {".class": "DeletedType", "source": self.source} -class TypeVar(Type): - """A type variable type. + @classmethod + def deserialize(cls, data: JsonDict) -> DeletedType: + assert data[".class"] == "DeletedType" + return DeletedType(data["source"]) - This refers to either a class type variable (id > 0) or a function - type variable (id < 0). + +# Fake TypeInfo to be used as a placeholder during Instance de-serialization. +NOT_READY: Final = mypy.nodes.FakeInfo("De-serialization failure: TypeInfo not fixed") + + +class ExtraAttrs: + """Summary of module attributes and types. + + This is used for instances of types.ModuleType, because they can have different + attributes per instance, and for type narrowing with hasattr() checks. """ - name = '' # Name of the type variable (for messages and debugging) - id = 0 # 1, 2, ... for type-related, -1, ... for function-related - values = Undefined(List[Type]) # Value restriction, empty list if no restriction - upper_bound = Undefined(Type) # Upper bound for values (currently always 'object') + def __init__( + self, + attrs: dict[str, Type], + immutable: set[str] | None = None, + mod_name: str | None = None, + ) -> None: + self.attrs = attrs + if immutable is None: + immutable = set() + self.immutable = immutable + self.mod_name = mod_name + + def __hash__(self) -> int: + return hash((tuple(self.attrs.items()), tuple(sorted(self.immutable)))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ExtraAttrs): + return NotImplemented + return self.attrs == other.attrs and self.immutable == other.immutable + + def copy(self) -> ExtraAttrs: + return ExtraAttrs(self.attrs.copy(), self.immutable.copy(), self.mod_name) - def __init__(self, name: str, id: int, values: List[Type], upper_bound: Type, - line: int = -1, repr: Any = None) -> None: - self.name = name - self.id = id - self.values = values - self.upper_bound = upper_bound - super().__init__(line, repr) + def __repr__(self) -> str: + return f"ExtraAttrs({self.attrs!r}, {self.immutable!r}, {self.mod_name!r})" + + def serialize(self) -> JsonDict: + return { + ".class": "ExtraAttrs", + "attrs": {k: v.serialize() for k, v in self.attrs.items()}, + "immutable": list(self.immutable), + "mod_name": self.mod_name, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> ExtraAttrs: + assert data[".class"] == "ExtraAttrs" + return ExtraAttrs( + {k: deserialize_type(v) for k, v in data["attrs"].items()}, + set(data["immutable"]), + data["mod_name"], + ) + + +class Instance(ProperType): + """An instance type of form C[T1, ..., Tn]. - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_var(self) + The list of type variables may be empty. + + Several types have fallbacks to `Instance`, because in Python everything is an object + and this concept is impossible to express without intersection types. We therefore use + fallbacks for all "non-special" (like UninhabitedType, ErasedType etc) types. + """ + __slots__ = ("type", "args", "invalid", "type_ref", "last_known_value", "_hash", "extra_attrs") + + def __init__( + self, + typ: mypy.nodes.TypeInfo, + args: Sequence[Type], + line: int = -1, + column: int = -1, + *, + last_known_value: LiteralType | None = None, + extra_attrs: ExtraAttrs | None = None, + ) -> None: + super().__init__(line, column) + self.type = typ + self.args = tuple(args) + self.type_ref: str | None = None + + # True if recovered after incorrect number of type arguments error + self.invalid = False + + # This field keeps track of the underlying Literal[...] value associated with + # this instance, if one is known. + # + # This field is set whenever possible within expressions, but is erased upon + # variable assignment (see erasetype.remove_instance_last_known_values) unless + # the variable is declared to be final. + # + # For example, consider the following program: + # + # a = 1 + # b: Final[int] = 2 + # c: Final = 3 + # print(a + b + c + 4) + # + # The 'Instance' objects associated with the expressions '1', '2', '3', and '4' will + # have last_known_values of type Literal[1], Literal[2], Literal[3], and Literal[4] + # respectively. However, the Instance object assigned to 'a' and 'b' will have their + # last_known_value erased: variable 'a' is mutable; variable 'b' was declared to be + # specifically an int. + # + # Or more broadly, this field lets this Instance "remember" its original declaration + # when applicable. We want this behavior because we want implicit Final declarations + # to act pretty much identically with constants: we should be able to replace any + # places where we use some Final variable with the original value and get the same + # type-checking behavior. For example, we want this program: + # + # def expects_literal(x: Literal[3]) -> None: pass + # var: Final = 3 + # expects_literal(var) + # + # ...to type-check in the exact same way as if we had written the program like this: + # + # def expects_literal(x: Literal[3]) -> None: pass + # expects_literal(3) + # + # In order to make this work (especially with literal types), we need var's type + # (an Instance) to remember the "original" value. + # + # Preserving this value within expressions is useful for similar reasons. + # + # Currently most of mypy will ignore this field and will continue to treat this type like + # a regular Instance. We end up using this field only when we are explicitly within a + # Literal context. + self.last_known_value = last_known_value + + # Cached hash value + self._hash = -1 + + # Additional attributes defined per instance of this type. For example modules + # have different attributes per instance of types.ModuleType. + self.extra_attrs = extra_attrs + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_instance(self) -class FunctionLike(Type): + def __hash__(self) -> int: + if self._hash == -1: + self._hash = hash((self.type, self.args, self.last_known_value, self.extra_attrs)) + return self._hash + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Instance): + return NotImplemented + return ( + self.type == other.type + and self.args == other.args + and self.last_known_value == other.last_known_value + and self.extra_attrs == other.extra_attrs + ) + + def serialize(self) -> JsonDict | str: + assert self.type is not None + type_ref = self.type.fullname + if not self.args and not self.last_known_value: + return type_ref + data: JsonDict = { + ".class": "Instance", + "type_ref": type_ref, + "args": [arg.serialize() for arg in self.args], + } + if self.last_known_value is not None: + data["last_known_value"] = self.last_known_value.serialize() + data["extra_attrs"] = self.extra_attrs.serialize() if self.extra_attrs else None + return data + + @classmethod + def deserialize(cls, data: JsonDict | str) -> Instance: + if isinstance(data, str): + inst = Instance(NOT_READY, []) + inst.type_ref = data + return inst + assert data[".class"] == "Instance" + args: list[Type] = [] + if "args" in data: + args_list = data["args"] + assert isinstance(args_list, list) + args = [deserialize_type(arg) for arg in args_list] + inst = Instance(NOT_READY, args) + inst.type_ref = data["type_ref"] # Will be fixed up by fixup.py later. + if "last_known_value" in data: + inst.last_known_value = LiteralType.deserialize(data["last_known_value"]) + if data.get("extra_attrs") is not None: + inst.extra_attrs = ExtraAttrs.deserialize(data["extra_attrs"]) + return inst + + def copy_modified( + self, + *, + args: Bogus[list[Type]] = _dummy, + last_known_value: Bogus[LiteralType | None] = _dummy, + ) -> Instance: + new = Instance( + typ=self.type, + args=args if args is not _dummy else self.args, + line=self.line, + column=self.column, + last_known_value=( + last_known_value if last_known_value is not _dummy else self.last_known_value + ), + extra_attrs=self.extra_attrs, + ) + # We intentionally don't copy the extra_attrs here, so they will be erased. + new.can_be_true = self.can_be_true + new.can_be_false = self.can_be_false + return new + + def copy_with_extra_attr(self, name: str, typ: Type) -> Instance: + if self.extra_attrs: + existing_attrs = self.extra_attrs.copy() + else: + existing_attrs = ExtraAttrs({}, set(), None) + existing_attrs.attrs[name] = typ + new = self.copy_modified() + new.extra_attrs = existing_attrs + return new + + def is_singleton_type(self) -> bool: + # TODO: + # Also make this return True if the type corresponds to NotImplemented? + return ( + self.type.is_enum + and len(self.type.enum_members) == 1 + or self.type.fullname in {"builtins.ellipsis", "types.EllipsisType"} + ) + + +class FunctionLike(ProperType): """Abstract base class for function types.""" + __slots__ = ("fallback",) + + fallback: Instance + + def __init__(self, line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + self._can_be_false = False + @abstractmethod - def is_type_obj(self) -> bool: pass + def is_type_obj(self) -> bool: + pass @abstractmethod - def type_object(self) -> mypy.nodes.TypeInfo: pass + def type_object(self) -> mypy.nodes.TypeInfo: + pass + @property @abstractmethod - def items(self) -> List['Callable']: pass + def items(self) -> list[CallableType]: + pass @abstractmethod - def with_name(self, name: str) -> 'FunctionLike': pass - - # Corresponding instance type (e.g. builtins.type) - fallback = Undefined(Instance) - - -class Callable(FunctionLike): - """Type of a non-overloaded callable object (function).""" - - arg_types = Undefined(List[Type]) # Types of function arguments - arg_kinds = Undefined(List[int]) # mypy.nodes.ARG_ constants - arg_names = Undefined(List[str]) # None if not a keyword argument - min_args = 0 # Minimum number of arguments - is_var_arg = False # Is it a varargs function? - ret_type = Undefined(Type) # Return value type - name = '' # Name (may be None; for error messages) - # Type variables for a generic function - variables = Undefined(List[TypeVarDef]) - - # Implicit bound values of type variables. These can be either for - # class type variables or for generic function type variables. - # For example, the method 'append' of List[int] has implicit value - # 'int' for the list type variable; the explicit method type is - # just 'def append(int) -> None', without any type variable. Implicit - # values are needed for runtime type checking, but they do not - # affect static type checking. - # - # All class type arguments must be stored first, ordered by id, - # and function type arguments must be stored next, again ordered by id - # (absolute value this time). - # - # Stored as tuples (id, type). - bound_vars = Undefined(List[Tuple[int, Type]]) - - _is_type_obj = False # Does this represent a type object? - - def __init__(self, arg_types: List[Type], - arg_kinds: List[int], - arg_names: List[str], - ret_type: Type, - fallback: Instance, - name: str = None, variables: List[TypeVarDef] = None, - bound_vars: List[Tuple[int, Type]] = None, - line: int = -1, repr: Any = None) -> None: + def with_name(self, name: str) -> FunctionLike: + pass + + @abstractmethod + def get_name(self) -> str | None: + pass + + +class FormalArgument(NamedTuple): + name: str | None + pos: int | None + typ: Type + required: bool + + +class Parameters(ProperType): + """Type that represents the parameters to a function. + + Used for ParamSpec analysis. Note that by convention we handle this + type as a Callable without return type, not as a "tuple with names", + so that it behaves contravariantly, in particular [x: int] <: [int]. + """ + + __slots__ = ( + "arg_types", + "arg_kinds", + "arg_names", + "min_args", + "is_ellipsis_args", + # TODO: variables don't really belong here, but they are used to allow hacky support + # for forall . Foo[[x: T], T] by capturing generic callable with ParamSpec, see #15909 + "variables", + "imprecise_arg_kinds", + ) + + def __init__( + self, + arg_types: Sequence[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None], + *, + variables: Sequence[TypeVarLikeType] | None = None, + is_ellipsis_args: bool = False, + imprecise_arg_kinds: bool = False, + line: int = -1, + column: int = -1, + ) -> None: + super().__init__(line, column) + self.arg_types = list(arg_types) + self.arg_kinds = arg_kinds + self.arg_names = list(arg_names) + assert len(arg_types) == len(arg_kinds) == len(arg_names) + assert not any(isinstance(t, Parameters) for t in arg_types) + self.min_args = arg_kinds.count(ARG_POS) + self.is_ellipsis_args = is_ellipsis_args + self.variables = variables or [] + self.imprecise_arg_kinds = imprecise_arg_kinds + + def copy_modified( + self, + arg_types: Bogus[Sequence[Type]] = _dummy, + arg_kinds: Bogus[list[ArgKind]] = _dummy, + arg_names: Bogus[Sequence[str | None]] = _dummy, + *, + variables: Bogus[Sequence[TypeVarLikeType]] = _dummy, + is_ellipsis_args: Bogus[bool] = _dummy, + imprecise_arg_kinds: Bogus[bool] = _dummy, + ) -> Parameters: + return Parameters( + arg_types=arg_types if arg_types is not _dummy else self.arg_types, + arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, + arg_names=arg_names if arg_names is not _dummy else self.arg_names, + is_ellipsis_args=( + is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args + ), + variables=variables if variables is not _dummy else self.variables, + imprecise_arg_kinds=( + imprecise_arg_kinds + if imprecise_arg_kinds is not _dummy + else self.imprecise_arg_kinds + ), + ) + + # TODO: here is a lot of code duplication with Callable type, fix this. + def var_arg(self) -> FormalArgument | None: + """The formal argument for *args.""" + for position, (type, kind) in enumerate(zip(self.arg_types, self.arg_kinds)): + if kind == ARG_STAR: + return FormalArgument(None, position, type, False) + return None + + def kw_arg(self) -> FormalArgument | None: + """The formal argument for **kwargs.""" + for position, (type, kind) in enumerate(zip(self.arg_types, self.arg_kinds)): + if kind == ARG_STAR2: + return FormalArgument(None, position, type, False) + return None + + def formal_arguments(self, include_star_args: bool = False) -> list[FormalArgument]: + """Yields the formal arguments corresponding to this callable, ignoring *arg and **kwargs. + + To handle *args and **kwargs, use the 'callable.var_args' and 'callable.kw_args' fields, + if they are not None. + + If you really want to include star args in the yielded output, set the + 'include_star_args' parameter to 'True'.""" + args = [] + done_with_positional = False + for i in range(len(self.arg_types)): + kind = self.arg_kinds[i] + if kind.is_named() or kind.is_star(): + done_with_positional = True + if not include_star_args and kind.is_star(): + continue + + required = kind.is_required() + pos = None if done_with_positional else i + arg = FormalArgument(self.arg_names[i], pos, self.arg_types[i], required) + args.append(arg) + return args + + def argument_by_name(self, name: str | None) -> FormalArgument | None: + if name is None: + return None + seen_star = False + for i, (arg_name, kind, typ) in enumerate( + zip(self.arg_names, self.arg_kinds, self.arg_types) + ): + # No more positional arguments after these. + if kind.is_named() or kind.is_star(): + seen_star = True + if kind.is_star(): + continue + if arg_name == name: + position = None if seen_star else i + return FormalArgument(name, position, typ, kind.is_required()) + return self.try_synthesizing_arg_from_kwarg(name) + + def argument_by_position(self, position: int | None) -> FormalArgument | None: + if position is None: + return None + if position >= len(self.arg_names): + return self.try_synthesizing_arg_from_vararg(position) + name, kind, typ = ( + self.arg_names[position], + self.arg_kinds[position], + self.arg_types[position], + ) + if kind.is_positional(): + return FormalArgument(name, position, typ, kind == ARG_POS) + else: + return self.try_synthesizing_arg_from_vararg(position) + + def try_synthesizing_arg_from_kwarg(self, name: str | None) -> FormalArgument | None: + kw_arg = self.kw_arg() + if kw_arg is not None: + return FormalArgument(name, None, kw_arg.typ, False) + else: + return None + + def try_synthesizing_arg_from_vararg(self, position: int | None) -> FormalArgument | None: + var_arg = self.var_arg() + if var_arg is not None: + return FormalArgument(None, position, var_arg.typ, False) + else: + return None + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_parameters(self) + + def serialize(self) -> JsonDict: + return { + ".class": "Parameters", + "arg_types": [t.serialize() for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "variables": [tv.serialize() for tv in self.variables], + "imprecise_arg_kinds": self.imprecise_arg_kinds, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> Parameters: + assert data[".class"] == "Parameters" + return Parameters( + [deserialize_type(t) for t in data["arg_types"]], + [ArgKind(x) for x in data["arg_kinds"]], + data["arg_names"], + variables=[cast(TypeVarLikeType, deserialize_type(v)) for v in data["variables"]], + imprecise_arg_kinds=data["imprecise_arg_kinds"], + ) + + def __hash__(self) -> int: + return hash( + ( + self.is_ellipsis_args, + tuple(self.arg_types), + tuple(self.arg_names), + tuple(self.arg_kinds), + ) + ) + + def __eq__(self, other: object) -> bool: + if isinstance(other, (Parameters, CallableType)): + return ( + self.arg_types == other.arg_types + and self.arg_names == other.arg_names + and self.arg_kinds == other.arg_kinds + and self.is_ellipsis_args == other.is_ellipsis_args + ) + else: + return NotImplemented + + +CT = TypeVar("CT", bound="CallableType") + + +class CallableType(FunctionLike): + """Type of a non-overloaded callable object (such as function).""" + + __slots__ = ( + "arg_types", # Types of function arguments + "arg_kinds", # ARG_ constants + "arg_names", # Argument names; None if not a keyword argument + "min_args", # Minimum number of arguments; derived from arg_kinds + "ret_type", # Return value type + "name", # Name (may be None; for error messages and plugins) + "definition", # For error messages. May be None. + "variables", # Type variables for a generic function + "is_ellipsis_args", # Is this Callable[..., t] (with literal '...')? + "implicit", # Was this type implicitly generated instead of explicitly + # specified by the user? + "special_sig", # Non-None for signatures that require special handling + # (currently only values are 'dict' for a signature similar to + # 'dict' and 'partial' for a `functools.partial` evaluation) + "from_type_type", # Was this callable generated by analyzing Type[...] + # instantiation? + "bound_args", # Bound type args, mostly unused but may be useful for + # tools that consume mypy ASTs + "def_extras", # Information about original definition we want to serialize. + # This is used for more detailed error messages. + "type_guard", # T, if -> TypeGuard[T] (ret_type is bool in this case). + "type_is", # T, if -> TypeIs[T] (ret_type is bool in this case). + "from_concatenate", # whether this callable is from a concatenate object + # (this is used for error messages) + "imprecise_arg_kinds", + "unpack_kwargs", # Was an Unpack[...] with **kwargs used to define this callable? + ) + + def __init__( + self, + # maybe this should be refactored to take a Parameters object + arg_types: Sequence[Type], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None], + ret_type: Type, + fallback: Instance, + name: str | None = None, + definition: SymbolNode | None = None, + variables: Sequence[TypeVarLikeType] | None = None, + line: int = -1, + column: int = -1, + is_ellipsis_args: bool = False, + implicit: bool = False, + special_sig: str | None = None, + from_type_type: bool = False, + bound_args: Sequence[Type | None] = (), + def_extras: dict[str, Any] | None = None, + type_guard: Type | None = None, + type_is: Type | None = None, + from_concatenate: bool = False, + imprecise_arg_kinds: bool = False, + unpack_kwargs: bool = False, + ) -> None: + super().__init__(line, column) + assert len(arg_types) == len(arg_kinds) == len(arg_names) + for t, k in zip(arg_types, arg_kinds): + if isinstance(t, ParamSpecType): + assert not t.prefix.arg_types + # TODO: should we assert that only ARG_STAR contain ParamSpecType? + # See testParamSpecJoin, that relies on passing e.g `P.args` as plain argument. if variables is None: variables = [] - if not bound_vars: - bound_vars = [] - self.arg_types = arg_types + self.arg_types = list(arg_types) self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) - self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds + self.arg_names = list(arg_names) + self.min_args = arg_kinds.count(ARG_POS) self.ret_type = ret_type self.fallback = fallback - assert not name or ' CT: + modified = CallableType( + arg_types=arg_types if arg_types is not _dummy else self.arg_types, + arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, + arg_names=arg_names if arg_names is not _dummy else self.arg_names, + ret_type=ret_type if ret_type is not _dummy else self.ret_type, + fallback=fallback if fallback is not _dummy else self.fallback, + name=name if name is not _dummy else self.name, + definition=definition if definition is not _dummy else self.definition, + variables=variables if variables is not _dummy else self.variables, + line=line if line != _dummy_int else self.line, + column=column if column != _dummy_int else self.column, + is_ellipsis_args=( + is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args + ), + implicit=implicit if implicit is not _dummy else self.implicit, + special_sig=special_sig if special_sig is not _dummy else self.special_sig, + from_type_type=from_type_type if from_type_type is not _dummy else self.from_type_type, + bound_args=bound_args if bound_args is not _dummy else self.bound_args, + def_extras=def_extras if def_extras is not _dummy else dict(self.def_extras), + type_guard=type_guard if type_guard is not _dummy else self.type_guard, + type_is=type_is if type_is is not _dummy else self.type_is, + from_concatenate=( + from_concatenate if from_concatenate is not _dummy else self.from_concatenate + ), + imprecise_arg_kinds=( + imprecise_arg_kinds + if imprecise_arg_kinds is not _dummy + else self.imprecise_arg_kinds + ), + unpack_kwargs=unpack_kwargs if unpack_kwargs is not _dummy else self.unpack_kwargs, + ) + # Optimization: Only NewTypes are supported as subtypes since + # the class is effectively final, so we can use a cast safely. + return cast(CT, modified) + + def var_arg(self) -> FormalArgument | None: + """The formal argument for *args.""" + for position, (type, kind) in enumerate(zip(self.arg_types, self.arg_kinds)): + if kind == ARG_STAR: + return FormalArgument(None, position, type, False) + return None + + def kw_arg(self) -> FormalArgument | None: + """The formal argument for **kwargs.""" + for position, (type, kind) in enumerate(zip(self.arg_types, self.arg_kinds)): + if kind == ARG_STAR2: + return FormalArgument(None, position, type, False) + return None + + @property + def is_var_arg(self) -> bool: + """Does this callable have a *args argument?""" + return ARG_STAR in self.arg_kinds + + @property + def is_kw_arg(self) -> bool: + """Does this callable have a **kwargs argument?""" + return ARG_STAR2 in self.arg_kinds def is_type_obj(self) -> bool: - return self.fallback.type.fullname() == 'builtins.type' + return self.fallback.type.is_metaclass() and not isinstance( + get_proper_type(self.ret_type), UninhabitedType + ) def type_object(self) -> mypy.nodes.TypeInfo: assert self.is_type_obj() - ret = self.ret_type + ret = get_proper_type(self.ret_type) + if isinstance(ret, TypeVarType): + ret = get_proper_type(ret.upper_bound) if isinstance(ret, TupleType): + ret = ret.partial_fallback + if isinstance(ret, TypedDictType): ret = ret.fallback - return cast(Instance, ret).type + assert isinstance(ret, Instance) + return ret.type - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_callable(self) + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_callable_type(self) - def with_name(self, name: str) -> 'Callable': + def with_name(self, name: str) -> CallableType: """Return a copy of this type with the specified name.""" - ret = self.ret_type - if isinstance(ret, Void): - ret = ret.with_source(name) - return Callable(self.arg_types, - self.arg_kinds, - self.arg_names, - ret, - self.fallback, - name, - self.variables, - self.bound_vars, - self.line, self.repr) - - def max_fixed_args(self) -> int: - n = len(self.arg_types) - if self.is_var_arg: - n -= 1 - return n + return self.copy_modified(ret_type=self.ret_type, name=name) + + def get_name(self) -> str | None: + return self.name + + def max_possible_positional_args(self) -> int: + """Returns maximum number of positional arguments this method could possibly accept. + + This takes into account *arg and **kwargs but excludes keyword-only args.""" + if self.is_var_arg or self.is_kw_arg: + return sys.maxsize + return sum(kind.is_positional() for kind in self.arg_kinds) + + def formal_arguments(self, include_star_args: bool = False) -> list[FormalArgument]: + """Return a list of the formal arguments of this callable, ignoring *arg and **kwargs. + + To handle *args and **kwargs, use the 'callable.var_args' and 'callable.kw_args' fields, + if they are not None. + + If you really want to include star args in the yielded output, set the + 'include_star_args' parameter to 'True'.""" + args = [] + done_with_positional = False + for i in range(len(self.arg_types)): + kind = self.arg_kinds[i] + if kind.is_named() or kind.is_star(): + done_with_positional = True + if not include_star_args and kind.is_star(): + continue + + required = kind.is_required() + pos = None if done_with_positional else i + arg = FormalArgument(self.arg_names[i], pos, self.arg_types[i], required) + args.append(arg) + return args + + def argument_by_name(self, name: str | None) -> FormalArgument | None: + if name is None: + return None + seen_star = False + for i, (arg_name, kind, typ) in enumerate( + zip(self.arg_names, self.arg_kinds, self.arg_types) + ): + # No more positional arguments after these. + if kind.is_named() or kind.is_star(): + seen_star = True + if kind.is_star(): + continue + if arg_name == name: + position = None if seen_star else i + return FormalArgument(name, position, typ, kind.is_required()) + return self.try_synthesizing_arg_from_kwarg(name) + + def argument_by_position(self, position: int | None) -> FormalArgument | None: + if position is None: + return None + if position >= len(self.arg_names): + return self.try_synthesizing_arg_from_vararg(position) + name, kind, typ = ( + self.arg_names[position], + self.arg_kinds[position], + self.arg_types[position], + ) + if kind.is_positional(): + return FormalArgument(name, position, typ, kind == ARG_POS) + else: + return self.try_synthesizing_arg_from_vararg(position) + + def try_synthesizing_arg_from_kwarg(self, name: str | None) -> FormalArgument | None: + kw_arg = self.kw_arg() + if kw_arg is not None: + return FormalArgument(name, None, kw_arg.typ, False) + else: + return None + + def try_synthesizing_arg_from_vararg(self, position: int | None) -> FormalArgument | None: + var_arg = self.var_arg() + if var_arg is not None: + return FormalArgument(None, position, var_arg.typ, False) + else: + return None - def items(self) -> List['Callable']: + @property + def items(self) -> list[CallableType]: return [self] def is_generic(self) -> bool: return bool(self.variables) - def type_var_ids(self) -> List[int]: - a = List[int]() + def type_var_ids(self) -> list[TypeVarId]: + a: list[TypeVarId] = [] for tv in self.variables: a.append(tv.id) return a + def param_spec(self) -> ParamSpecType | None: + """Return ParamSpec if callable can be called with one. + + A Callable accepting ParamSpec P args (*args, **kwargs) must have the + two final parameters like this: *args: P.args, **kwargs: P.kwargs. + """ + if len(self.arg_types) < 2: + return None + if self.arg_kinds[-2] != ARG_STAR or self.arg_kinds[-1] != ARG_STAR2: + return None + arg_type = self.arg_types[-2] + if not isinstance(arg_type, ParamSpecType): + return None + + # Prepend prefix for def f(prefix..., *args: P.args, **kwargs: P.kwargs) -> ... + # TODO: confirm that all arg kinds are positional + prefix = Parameters(self.arg_types[:-2], self.arg_kinds[:-2], self.arg_names[:-2]) + return arg_type.copy_modified(flavor=ParamSpecFlavor.BARE, prefix=prefix) + + def normalize_trivial_unpack(self) -> None: + # Normalize trivial unpack in var args as *args: *tuple[X, ...] -> *args: X in place. + if self.is_var_arg: + star_index = self.arg_kinds.index(ARG_STAR) + star_type = self.arg_types[star_index] + if isinstance(star_type, UnpackType): + p_type = get_proper_type(star_type.type) + if isinstance(p_type, Instance): + assert p_type.type.fullname == "builtins.tuple" + self.arg_types[star_index] = p_type.args[0] + + def with_unpacked_kwargs(self) -> NormalizedCallableType: + if not self.unpack_kwargs: + return cast(NormalizedCallableType, self) + last_type = get_proper_type(self.arg_types[-1]) + assert isinstance(last_type, TypedDictType) + extra_kinds = [ + ArgKind.ARG_NAMED if name in last_type.required_keys else ArgKind.ARG_NAMED_OPT + for name in last_type.items + ] + new_arg_kinds = self.arg_kinds[:-1] + extra_kinds + new_arg_names = self.arg_names[:-1] + list(last_type.items) + new_arg_types = self.arg_types[:-1] + list(last_type.items.values()) + return NormalizedCallableType( + self.copy_modified( + arg_kinds=new_arg_kinds, + arg_names=new_arg_names, + arg_types=new_arg_types, + unpack_kwargs=False, + ) + ) + + def with_normalized_var_args(self) -> Self: + var_arg = self.var_arg() + if not var_arg or not isinstance(var_arg.typ, UnpackType): + return self + unpacked = get_proper_type(var_arg.typ.type) + if not isinstance(unpacked, TupleType): + # Note that we don't normalize *args: *tuple[X, ...] -> *args: X, + # this should be done once in semanal_typeargs.py for user-defined types, + # and we ourselves rarely construct such type. + return self + unpack_index = find_unpack_in_list(unpacked.items) + if unpack_index == 0 and len(unpacked.items) > 1: + # Already normalized. + return self + + # Boilerplate: + var_arg_index = self.arg_kinds.index(ARG_STAR) + types_prefix = self.arg_types[:var_arg_index] + kinds_prefix = self.arg_kinds[:var_arg_index] + names_prefix = self.arg_names[:var_arg_index] + types_suffix = self.arg_types[var_arg_index + 1 :] + kinds_suffix = self.arg_kinds[var_arg_index + 1 :] + names_suffix = self.arg_names[var_arg_index + 1 :] + no_name: str | None = None # to silence mypy + + # Now we have something non-trivial to do. + if unpack_index is None: + # Plain *Tuple[X, Y, Z] -> replace with ARG_POS completely + types_middle = unpacked.items + kinds_middle = [ARG_POS] * len(unpacked.items) + names_middle = [no_name] * len(unpacked.items) + else: + # *Tuple[X, *Ts, Y, Z] or *Tuple[X, *tuple[T, ...], X, Z], here + # we replace the prefix by ARG_POS (this is how some places expect + # Callables to be represented) + nested_unpack = unpacked.items[unpack_index] + assert isinstance(nested_unpack, UnpackType) + nested_unpacked = get_proper_type(nested_unpack.type) + if unpack_index == len(unpacked.items) - 1: + # Normalize also single item tuples like + # *args: *Tuple[*tuple[X, ...]] -> *args: X + # *args: *Tuple[*Ts] -> *args: *Ts + # This may be not strictly necessary, but these are very verbose. + if isinstance(nested_unpacked, Instance): + assert nested_unpacked.type.fullname == "builtins.tuple" + new_unpack = nested_unpacked.args[0] + else: + if not isinstance(nested_unpacked, TypeVarTupleType): + # We found a non-nomralized tuple type, this means this method + # is called during semantic analysis (e.g. from get_proper_type()) + # there is no point in normalizing callables at this stage. + return self + new_unpack = nested_unpack + else: + new_unpack = UnpackType( + unpacked.copy_modified(items=unpacked.items[unpack_index:]) + ) + types_middle = unpacked.items[:unpack_index] + [new_unpack] + kinds_middle = [ARG_POS] * unpack_index + [ARG_STAR] + names_middle = [no_name] * unpack_index + [self.arg_names[var_arg_index]] + return self.copy_modified( + arg_types=types_prefix + types_middle + types_suffix, + arg_kinds=kinds_prefix + kinds_middle + kinds_suffix, + arg_names=names_prefix + names_middle + names_suffix, + ) + + def __hash__(self) -> int: + # self.is_type_obj() will fail if self.fallback.type is a FakeInfo + if isinstance(self.fallback.type, FakeInfo): + is_type_obj = 2 + else: + is_type_obj = self.is_type_obj() + return hash( + ( + self.ret_type, + is_type_obj, + self.is_ellipsis_args, + self.name, + tuple(self.arg_types), + tuple(self.arg_names), + tuple(self.arg_kinds), + self.fallback, + ) + ) + + def __eq__(self, other: object) -> bool: + if isinstance(other, CallableType): + return ( + self.ret_type == other.ret_type + and self.arg_types == other.arg_types + and self.arg_names == other.arg_names + and self.arg_kinds == other.arg_kinds + and self.name == other.name + and self.is_type_obj() == other.is_type_obj() + and self.is_ellipsis_args == other.is_ellipsis_args + and self.fallback == other.fallback + ) + else: + return NotImplemented + + def serialize(self) -> JsonDict: + # TODO: As an optimization, leave out everything related to + # generic functions for non-generic functions. + return { + ".class": "CallableType", + "arg_types": [t.serialize() for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "ret_type": self.ret_type.serialize(), + "fallback": self.fallback.serialize(), + "name": self.name, + # We don't serialize the definition (only used for error messages). + "variables": [v.serialize() for v in self.variables], + "is_ellipsis_args": self.is_ellipsis_args, + "implicit": self.implicit, + "bound_args": [(None if t is None else t.serialize()) for t in self.bound_args], + "def_extras": dict(self.def_extras), + "type_guard": self.type_guard.serialize() if self.type_guard is not None else None, + "type_is": (self.type_is.serialize() if self.type_is is not None else None), + "from_concatenate": self.from_concatenate, + "imprecise_arg_kinds": self.imprecise_arg_kinds, + "unpack_kwargs": self.unpack_kwargs, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> CallableType: + assert data[".class"] == "CallableType" + # TODO: Set definition to the containing SymbolNode? + return CallableType( + [deserialize_type(t) for t in data["arg_types"]], + [ArgKind(x) for x in data["arg_kinds"]], + data["arg_names"], + deserialize_type(data["ret_type"]), + Instance.deserialize(data["fallback"]), + name=data["name"], + variables=[cast(TypeVarLikeType, deserialize_type(v)) for v in data["variables"]], + is_ellipsis_args=data["is_ellipsis_args"], + implicit=data["implicit"], + bound_args=[(None if t is None else deserialize_type(t)) for t in data["bound_args"]], + def_extras=data["def_extras"], + type_guard=( + deserialize_type(data["type_guard"]) if data["type_guard"] is not None else None + ), + type_is=(deserialize_type(data["type_is"]) if data["type_is"] is not None else None), + from_concatenate=data["from_concatenate"], + imprecise_arg_kinds=data["imprecise_arg_kinds"], + unpack_kwargs=data["unpack_kwargs"], + ) + + +# This is a little safety net to prevent reckless special-casing of callables +# that can potentially break Unpack[...] with **kwargs. +# TODO: use this in more places in checkexpr.py etc? +NormalizedCallableType = NewType("NormalizedCallableType", CallableType) + class Overloaded(FunctionLike): - """Overloaded function type T1, ... Tn, where each Ti is Callable. + """Overloaded function type T1, ... Tn, where each Ti is CallableType. - The variant to call is chosen based on runtime argument types; the first - matching signature is the target. + The variant to call is chosen based on static argument + types. Overloaded function types can only be defined in stub + files, and thus there is no explicit runtime dispatch + implementation. """ - _items = Undefined(List[Callable]) # Must not be empty + __slots__ = ("_items",) + + _items: list[CallableType] # Must not be empty - def __init__(self, items: List[Callable]) -> None: + def __init__(self, items: list[CallableType]) -> None: + super().__init__(items[0].line, items[0].column) self._items = items self.fallback = items[0].fallback - super().__init__(items[0].line, None) - def items(self) -> List[Callable]: + @property + def items(self) -> list[CallableType]: return self._items - def name(self) -> str: - return self._items[0].name + def name(self) -> str | None: + return self.get_name() def is_type_obj(self) -> bool: # All the items must have the same type object status, so it's @@ -357,256 +2348,900 @@ def type_object(self) -> mypy.nodes.TypeInfo: # query only (any) one of them. return self._items[0].type_object() - def with_name(self, name: str) -> 'Overloaded': - ni = List[Callable]() + def with_name(self, name: str) -> Overloaded: + ni: list[CallableType] = [] for it in self._items: ni.append(it.with_name(name)) return Overloaded(ni) - def accept(self, visitor: 'TypeVisitor[T]') -> T: + def get_name(self) -> str | None: + return self._items[0].name + + def with_unpacked_kwargs(self) -> Overloaded: + if any(i.unpack_kwargs for i in self.items): + return Overloaded([i.with_unpacked_kwargs() for i in self.items]) + return self + + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_overloaded(self) + def __hash__(self) -> int: + return hash(tuple(self.items)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Overloaded): + return NotImplemented + return self.items == other.items + + def serialize(self) -> JsonDict: + return {".class": "Overloaded", "items": [t.serialize() for t in self.items]} -class TupleType(Type): + @classmethod + def deserialize(cls, data: JsonDict) -> Overloaded: + assert data[".class"] == "Overloaded" + return Overloaded([CallableType.deserialize(t) for t in data["items"]]) + + +class TupleType(ProperType): """The tuple type Tuple[T1, ..., Tn] (at least one type argument). Instance variables: - items -- tuple item types - fallback -- the underlying instance type that is used for non-tuple methods - (this is currently always builtins.tuple, but it could be different for named - tuples, for example) + items: Tuple item types + partial_fallback: The (imprecise) underlying instance type that is used + for non-tuple methods. This is generally builtins.tuple[Any, ...] for + regular tuples, but it's different for named tuples and classes with + a tuple base class. Use mypy.typeops.tuple_fallback to calculate the + precise fallback type derived from item types. + implicit: If True, derived from a tuple expression (t,....) instead of Tuple[t, ...] """ - items = Undefined(List[Type]) - fallback = Undefined(Instance) - - def __init__(self, items: List[Type], fallback: Instance, line: int = -1, - repr: Any = None) -> None: + __slots__ = ("items", "partial_fallback", "implicit") + + items: list[Type] + partial_fallback: Instance + implicit: bool + + def __init__( + self, + items: list[Type], + fallback: Instance, + line: int = -1, + column: int = -1, + implicit: bool = False, + ) -> None: + super().__init__(line, column) + self.partial_fallback = fallback self.items = items - self.fallback = fallback - super().__init__(line, repr) + self.implicit = implicit + + def can_be_true_default(self) -> bool: + if self.can_be_any_bool(): + # Corner case: it is a `NamedTuple` with `__bool__` method defined. + # It can be anything: both `True` and `False`. + return True + return self.length() > 0 + + def can_be_false_default(self) -> bool: + if self.can_be_any_bool(): + # Corner case: it is a `NamedTuple` with `__bool__` method defined. + # It can be anything: both `True` and `False`. + return True + if self.length() == 0: + return True + if self.length() > 1: + return False + # Special case tuple[*Ts] may or may not be false. + item = self.items[0] + if not isinstance(item, UnpackType): + return False + if not isinstance(item.type, TypeVarTupleType): + # Non-normalized tuple[int, ...] can be false. + return True + return item.type.min_len == 0 + + def can_be_any_bool(self) -> bool: + return bool( + self.partial_fallback.type + and self.partial_fallback.type.fullname != "builtins.tuple" + and self.partial_fallback.type.names.get("__bool__") + ) def length(self) -> int: return len(self.items) - def accept(self, visitor: 'TypeVisitor[T]') -> T: + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_tuple_type(self) + def __hash__(self) -> int: + return hash((tuple(self.items), self.partial_fallback)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TupleType): + return NotImplemented + return self.items == other.items and self.partial_fallback == other.partial_fallback + + def serialize(self) -> JsonDict: + return { + ".class": "TupleType", + "items": [t.serialize() for t in self.items], + "partial_fallback": self.partial_fallback.serialize(), + "implicit": self.implicit, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TupleType: + assert data[".class"] == "TupleType" + return TupleType( + [deserialize_type(t) for t in data["items"]], + Instance.deserialize(data["partial_fallback"]), + implicit=data["implicit"], + ) + + def copy_modified( + self, *, fallback: Instance | None = None, items: list[Type] | None = None + ) -> TupleType: + if fallback is None: + fallback = self.partial_fallback + if items is None: + items = self.items + return TupleType(items, fallback, self.line, self.column) + + def slice( + self, begin: int | None, end: int | None, stride: int | None, *, fallback: Instance | None + ) -> TupleType | None: + if fallback is None: + fallback = self.partial_fallback + + if stride == 0: + return None + + if any(isinstance(t, UnpackType) for t in self.items): + total = len(self.items) + unpack_index = find_unpack_in_list(self.items) + assert unpack_index is not None + if begin is None and end is None: + # We special-case this to support reversing variadic tuples. + # General support for slicing is tricky, so we handle only simple cases. + if stride == -1: + slice_items = self.items[::-1] + elif stride is None or stride == 1: + slice_items = self.items + else: + return None + elif (begin is None or unpack_index >= begin >= 0) and ( + end is not None and unpack_index >= end >= 0 + ): + # Start and end are in the prefix, everything works in this case. + slice_items = self.items[begin:end:stride] + elif (begin is not None and unpack_index - total < begin < 0) and ( + end is None or unpack_index - total < end < 0 + ): + # Start and end are in the suffix, everything works in this case. + slice_items = self.items[begin:end:stride] + elif (begin is None or unpack_index >= begin >= 0) and ( + end is None or unpack_index - total < end < 0 + ): + # Start in the prefix, end in the suffix, we can support only trivial strides. + if stride is None or stride == 1: + slice_items = self.items[begin:end:stride] + else: + return None + elif (begin is not None and unpack_index - total < begin < 0) and ( + end is not None and unpack_index >= end >= 0 + ): + # Start in the suffix, end in the prefix, we can support only trivial strides. + if stride is None or stride == -1: + slice_items = self.items[begin:end:stride] + else: + return None + else: + # TODO: there some additional cases we can support for homogeneous variadic + # items, we can "eat away" finite number of items. + return None + else: + slice_items = self.items[begin:end:stride] + return TupleType(slice_items, fallback, self.line, self.column, self.implicit) -class StarType(Type): - """The star type *type_parameter""" - - type = Undefined(Type) - def __init__(self, type: Type, line: int = -1, repr: Any = None) -> None: - self.type = type - super().__init__(line, repr) +class TypedDictType(ProperType): + """Type of TypedDict object {'k1': v1, ..., 'kn': vn}. - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_star_type(self) + A TypedDict object is a dictionary with specific string (literal) keys. Each + key has a value with a distinct type that depends on the key. TypedDict objects + are normal dict objects at runtime. + A TypedDictType can be either named or anonymous. If it's anonymous, its + fallback will be typing_extensions._TypedDict (Instance). _TypedDict is a subclass + of Mapping[str, object] and defines all non-mapping dict methods that TypedDict + supports. Some dict methods are unsafe and not supported. _TypedDict isn't defined + at runtime. -class UnionType(Type): - """The union type Union[T1, ..., Tn] (at least one type argument).""" + If a TypedDict is named, its fallback will be an Instance of the named type + (ex: "Point") whose TypeInfo has a typeddict_type that is anonymous. This + is similar to how named tuples work. - items = Undefined(List[Type]) + TODO: The fallback structure is perhaps overly complicated. + """ - def __init__(self, items: List[Type], line: int = -1, - repr: Any = None) -> None: + __slots__ = ( + "items", + "required_keys", + "readonly_keys", + "fallback", + "extra_items_from", + "to_be_mutated", + ) + + items: dict[str, Type] # item_name -> item_type + required_keys: set[str] + readonly_keys: set[str] + fallback: Instance + + extra_items_from: list[ProperType] # only used during semantic analysis + to_be_mutated: bool # only used in a plugin for `.update`, `|=`, etc + + def __init__( + self, + items: dict[str, Type], + required_keys: set[str], + readonly_keys: set[str], + fallback: Instance, + line: int = -1, + column: int = -1, + ) -> None: + super().__init__(line, column) self.items = items - super().__init__(line, repr) + self.required_keys = required_keys + self.readonly_keys = readonly_keys + self.fallback = fallback + self.can_be_true = len(self.items) > 0 + self.can_be_false = len(self.required_keys) == 0 + self.extra_items_from = [] + self.to_be_mutated = False + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_typeddict_type(self) + + def __hash__(self) -> int: + return hash( + ( + frozenset(self.items.items()), + self.fallback, + frozenset(self.required_keys), + frozenset(self.readonly_keys), + ) + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypedDictType): + return NotImplemented + if self is other: + return True + return ( + frozenset(self.items.keys()) == frozenset(other.items.keys()) + and all( + left_item_type == right_item_type + for (_, left_item_type, right_item_type) in self.zip(other) + ) + and self.fallback == other.fallback + and self.required_keys == other.required_keys + and self.readonly_keys == other.readonly_keys + ) + + def serialize(self) -> JsonDict: + return { + ".class": "TypedDictType", + "items": [[n, t.serialize()] for (n, t) in self.items.items()], + "required_keys": sorted(self.required_keys), + "readonly_keys": sorted(self.readonly_keys), + "fallback": self.fallback.serialize(), + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TypedDictType: + assert data[".class"] == "TypedDictType" + return TypedDictType( + {n: deserialize_type(t) for (n, t) in data["items"]}, + set(data["required_keys"]), + set(data["readonly_keys"]), + Instance.deserialize(data["fallback"]), + ) + + @property + def is_final(self) -> bool: + return self.fallback.type.is_final + + def is_anonymous(self) -> bool: + return self.fallback.type.fullname in TPDICT_FB_NAMES + + def as_anonymous(self) -> TypedDictType: + if self.is_anonymous(): + return self + assert self.fallback.type.typeddict_type is not None + return self.fallback.type.typeddict_type.as_anonymous() + + def copy_modified( + self, + *, + fallback: Instance | None = None, + item_types: list[Type] | None = None, + item_names: list[str] | None = None, + required_keys: set[str] | None = None, + readonly_keys: set[str] | None = None, + ) -> TypedDictType: + if fallback is None: + fallback = self.fallback + if item_types is None: + items = self.items + else: + items = dict(zip(self.items, item_types)) + if required_keys is None: + required_keys = self.required_keys + if readonly_keys is None: + readonly_keys = self.readonly_keys + if item_names is not None: + items = {k: v for (k, v) in items.items() if k in item_names} + required_keys &= set(item_names) + return TypedDictType(items, required_keys, readonly_keys, fallback, self.line, self.column) + + def create_anonymous_fallback(self) -> Instance: + anonymous = self.as_anonymous() + return anonymous.fallback + + def names_are_wider_than(self, other: TypedDictType) -> bool: + return len(other.items.keys() - self.items.keys()) == 0 + + def zip(self, right: TypedDictType) -> Iterable[tuple[str, Type, Type]]: + left = self + for item_name, left_item_type in left.items.items(): + right_item_type = right.items.get(item_name) + if right_item_type is not None: + yield (item_name, left_item_type, right_item_type) + + def zipall(self, right: TypedDictType) -> Iterable[tuple[str, Type | None, Type | None]]: + left = self + for item_name, left_item_type in left.items.items(): + right_item_type = right.items.get(item_name) + yield (item_name, left_item_type, right_item_type) + for item_name, right_item_type in right.items.items(): + if item_name in left.items: + continue + yield (item_name, None, right_item_type) + + +class RawExpressionType(ProperType): + """A synthetic type representing some arbitrary expression that does not cleanly + translate into a type. + + This synthetic type is only used at the beginning stages of semantic analysis + and should be completely removing during the process for mapping UnboundTypes to + actual types: we either turn it into a LiteralType or an AnyType. + + For example, suppose `Foo[1]` is initially represented as the following: + + UnboundType( + name='Foo', + args=[ + RawExpressionType(value=1, base_type_name='builtins.int'), + ], + ) + + As we perform semantic analysis, this type will transform into one of two + possible forms. + + If 'Foo' was an alias for 'Literal' all along, this type is transformed into: + + LiteralType(value=1, fallback=int_instance_here) + + Alternatively, if 'Foo' is an unrelated class, we report an error and instead + produce something like this: + + Instance(type=typeinfo_for_foo, args=[AnyType(TypeOfAny.from_error)) + + If the "note" field is not None, the provided note will be reported alongside the + error at this point. + + Note: if "literal_value" is None, that means this object is representing some + expression that cannot possibly be a parameter of Literal[...]. For example, + "Foo[3j]" would be represented as: + + UnboundType( + name='Foo', + args=[ + RawExpressionType(value=None, base_type_name='builtins.complex'), + ], + ) + """ - @staticmethod - def make_union(items: List[Type], line: int = -1, repr: Any = None) -> Type: - if len(items) > 1: - return UnionType(items, line, repr) - elif len(items) == 1: - return items[0] + __slots__ = ("literal_value", "base_type_name", "note") + + def __init__( + self, + literal_value: LiteralValue | None, + base_type_name: str, + line: int = -1, + column: int = -1, + note: str | None = None, + ) -> None: + super().__init__(line, column) + self.literal_value = literal_value + self.base_type_name = base_type_name + self.note = note + + def simple_name(self) -> str: + return self.base_type_name.replace("builtins.", "") + + def accept(self, visitor: TypeVisitor[T]) -> T: + assert isinstance(visitor, SyntheticTypeVisitor) + ret: T = visitor.visit_raw_expression_type(self) + return ret + + def serialize(self) -> JsonDict: + assert False, "Synthetic types don't serialize" + + def __hash__(self) -> int: + return hash((self.literal_value, self.base_type_name)) + + def __eq__(self, other: object) -> bool: + if isinstance(other, RawExpressionType): + return ( + self.base_type_name == other.base_type_name + and self.literal_value == other.literal_value + ) else: - return Void() + return NotImplemented - @staticmethod - def make_simplified_union(items: List[Type], line: int = -1, repr: Any = None) -> Type: - while any(isinstance(typ, UnionType) for typ in items): - all_items = [] # type: List[Type] - for typ in items: - if isinstance(typ, UnionType): - all_items.extend(typ.items) - else: - all_items.append(typ) - items = all_items - from mypy.subtypes import is_subtype - removed = Set[int]() - for i in range(len(items)): - if any(is_subtype(items[i], items[j]) for j in range(len(items)) - if j not in removed and j != i): - removed.add(i) +class LiteralType(ProperType): + """The type of a Literal instance. Literal[Value] - simplified_set = [items[i] for i in range(len(items)) if i not in removed] - return UnionType.make_union(simplified_set) + A Literal always consists of: - def length(self) -> int: - return len(self.items) + 1. A native Python object corresponding to the contained inner value + 2. A fallback for this Literal. The fallback also corresponds to the + parent type this Literal subtypes. - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_union_type(self) + For example, 'Literal[42]' is represented as + 'LiteralType(value=42, fallback=instance_of_int)' - def has_readable_member(self, name): - """For a tree of unions of instances, check whether all - instances have a given member. + As another example, `Literal[Color.RED]` (where Color is an enum) is + represented as `LiteralType(value="RED", fallback=instance_of_color)'. + """ - This should probably be refactored to go elsewhere.""" - return all(isinstance(x, UnionType) and x.has_readable_member(name) or - isinstance(x, Instance) and - x.type.has_readable_member(name) - for x in self.items) + __slots__ = ("value", "fallback", "_hash") + def __init__( + self, value: LiteralValue, fallback: Instance, line: int = -1, column: int = -1 + ) -> None: + super().__init__(line, column) + self.value = value + self.fallback = fallback + self._hash = -1 # Cached hash value + + # NOTE: Enum types are always truthy by default, but this can be changed + # in subclasses, so we need to get the truthyness from the Enum + # type rather than base it on the value (which is a non-empty + # string for enums, so always truthy) + # TODO: We should consider moving this branch to the `can_be_true` + # `can_be_false` properties instead, so the truthyness only + # needs to be determined once per set of Enum literals. + # However, the same can be said for `TypeAliasType` in some + # cases and we only set the default based on the type it is + # aliasing. So if we decide to change this, we may want to + # change that as well. perf_compare output was inconclusive + # but slightly favored this version, probably because we have + # almost no test cases where we would redundantly compute + # `can_be_false`/`can_be_true`. + def can_be_false_default(self) -> bool: + if self.fallback.type.is_enum: + return self.fallback.can_be_false + return not self.value + + def can_be_true_default(self) -> bool: + if self.fallback.type.is_enum: + return self.fallback.can_be_true + return bool(self.value) + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_literal_type(self) + + def __hash__(self) -> int: + if self._hash == -1: + self._hash = hash((self.value, self.fallback)) + return self._hash + + def __eq__(self, other: object) -> bool: + if isinstance(other, LiteralType): + return self.fallback == other.fallback and self.value == other.value + else: + return NotImplemented -class RuntimeTypeVar(Type): - """Reference to a runtime variable with the value of a type variable. + def is_enum_literal(self) -> bool: + return self.fallback.type.is_enum - The reference can must be a expression node, but only some node - types are properly supported (NameExpr, MemberExpr and IndexExpr - mainly). - """ + def value_repr(self) -> str: + """Returns the string representation of the underlying type. - node = Undefined(mypy.nodes.Node) + This function is almost equivalent to running `repr(self.value)`, + except it includes some additional logic to correctly handle cases + where the value is a string, byte string, a unicode string, or an enum. + """ + raw = repr(self.value) + fallback_name = self.fallback.type.fullname - def __init__(self, node: mypy.nodes.Node) -> None: - self.node = node - super().__init__(-1, None) + # If this is backed by an enum, + if self.is_enum_literal(): + return f"{fallback_name}.{self.value}" - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_runtime_type_var(self) + if fallback_name == "builtins.bytes": + # Note: 'builtins.bytes' only appears in Python 3, so we want to + # explicitly prefix with a "b" + return "b" + raw + else: + # 'builtins.str' could mean either depending on context, but either way + # we don't prefix: it's the "native" string. And of course, if value is + # some other type, we just return that string repr directly. + return raw + def serialize(self) -> JsonDict | str: + return { + ".class": "LiteralType", + "value": self.value, + "fallback": self.fallback.serialize(), + } -# -# Visitor-related classes -# + @classmethod + def deserialize(cls, data: JsonDict) -> LiteralType: + assert data[".class"] == "LiteralType" + return LiteralType(value=data["value"], fallback=Instance.deserialize(data["fallback"])) + def is_singleton_type(self) -> bool: + return self.is_enum_literal() or isinstance(self.value, bool) -class TypeVisitor(Generic[T]): - """Visitor class for types (Type subclasses). - The parameter T is the return type of the visit methods. - """ +class UnionType(ProperType): + """The union type Union[T1, ..., Tn] (at least one type argument).""" - def visit_unbound_type(self, t: UnboundType) -> T: - pass + __slots__ = ( + "items", + "is_evaluated", + "uses_pep604_syntax", + "original_str_expr", + "original_str_fallback", + ) + + def __init__( + self, + items: Sequence[Type], + line: int = -1, + column: int = -1, + *, + is_evaluated: bool = True, + uses_pep604_syntax: bool = False, + ) -> None: + super().__init__(line, column) + # We must keep this false to avoid crashes during semantic analysis. + # TODO: maybe switch this to True during type-checking pass? + self.items = flatten_nested_unions(items, handle_type_alias_type=False) + # is_evaluated should be set to false for type comments and string literals + self.is_evaluated = is_evaluated + # uses_pep604_syntax is True if Union uses OR syntax (X | Y) + self.uses_pep604_syntax = uses_pep604_syntax + # The meaning of these two is the same as for UnboundType. A UnionType can be + # return by type parser from a string "A|B", and we need to be able to fall back + # to plain string, when such a string appears inside a Literal[...]. + self.original_str_expr: str | None = None + self.original_str_fallback: str | None = None + + def can_be_true_default(self) -> bool: + return any(item.can_be_true for item in self.items) + + def can_be_false_default(self) -> bool: + return any(item.can_be_false for item in self.items) + + def __hash__(self) -> int: + return hash(frozenset(self.items)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UnionType): + return NotImplemented + return frozenset(self.items) == frozenset(other.items) + + @overload + @staticmethod + def make_union( + items: Sequence[ProperType], line: int = -1, column: int = -1 + ) -> ProperType: ... - def visit_type_list(self, t: TypeList) -> T: - pass + @overload + @staticmethod + def make_union(items: Sequence[Type], line: int = -1, column: int = -1) -> Type: ... - def visit_error_type(self, t: ErrorType) -> T: - pass + @staticmethod + def make_union(items: Sequence[Type], line: int = -1, column: int = -1) -> Type: + if len(items) > 1: + return UnionType(items, line, column) + elif len(items) == 1: + return items[0] + else: + return UninhabitedType() - def visit_any(self, t: AnyType) -> T: - pass + def length(self) -> int: + return len(self.items) - def visit_void(self, t: Void) -> T: - pass + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_union_type(self) - def visit_none_type(self, t: NoneTyp) -> T: - pass + def relevant_items(self) -> list[Type]: + """Removes NoneTypes from Unions when strict Optional checking is off.""" + if state.strict_optional: + return self.items + else: + return [i for i in self.items if not isinstance(get_proper_type(i), NoneType)] - def visit_erased_type(self, t: ErasedType) -> T: - pass + def serialize(self) -> JsonDict: + return { + ".class": "UnionType", + "items": [t.serialize() for t in self.items], + "uses_pep604_syntax": self.uses_pep604_syntax, + } - def visit_type_var(self, t: TypeVar) -> T: - pass + @classmethod + def deserialize(cls, data: JsonDict) -> UnionType: + assert data[".class"] == "UnionType" + return UnionType( + [deserialize_type(t) for t in data["items"]], + uses_pep604_syntax=data["uses_pep604_syntax"], + ) - def visit_instance(self, t: Instance) -> T: - pass - def visit_callable(self, t: Callable) -> T: - pass +class PartialType(ProperType): + """Type such as List[?] where type arguments are unknown, or partial None type. - def visit_overloaded(self, t: Overloaded) -> T: - pass + These are used for inferring types in multiphase initialization such as this: - def visit_tuple_type(self, t: TupleType) -> T: - pass + x = [] # x gets a partial type List[?], as item type is unknown + x.append(1) # partial type gets replaced with normal type List[int] - def visit_star_type(self, t: StarType) -> T: - pass + Or with None: - def visit_union_type(self, t: UnionType) -> T: - assert(0) # XXX catch visitors that don't have Union cases yet + x = None # x gets a partial type None + if c: + x = 1 # Infer actual type int for x + """ + + __slots__ = ("type", "var", "value_type") + + # None for the 'None' partial type; otherwise a generic class + type: mypy.nodes.TypeInfo | None + var: mypy.nodes.Var + # For partial defaultdict[K, V], the type V (K is unknown). If V is generic, + # the type argument is Any and will be replaced later. + value_type: Instance | None + + def __init__( + self, + type: mypy.nodes.TypeInfo | None, + var: mypy.nodes.Var, + value_type: Instance | None = None, + ) -> None: + super().__init__() + self.type = type + self.var = var + self.value_type = value_type + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_partial_type(self) - def visit_runtime_type_var(self, t: RuntimeTypeVar) -> T: - pass +class EllipsisType(ProperType): + """The type ... (ellipsis). -class TypeTranslator(TypeVisitor[Type]): - """Identity type transformation. + This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. - Subclass this and override some methods to implement a non-trivial - transformation. + A semantically analyzed type will never have ellipsis types. """ - def visit_unbound_type(self, t: UnboundType) -> Type: - return t + __slots__ = () - def visit_type_list(self, t: TypeList) -> Type: - return t + def accept(self, visitor: TypeVisitor[T]) -> T: + assert isinstance(visitor, SyntheticTypeVisitor) + ret: T = visitor.visit_ellipsis_type(self) + return ret - def visit_error_type(self, t: ErrorType) -> Type: - return t + def serialize(self) -> JsonDict: + assert False, "Synthetic types don't serialize" - def visit_any(self, t: AnyType) -> Type: - return t - def visit_void(self, t: Void) -> Type: - return t +class TypeType(ProperType): + """For types like Type[User]. - def visit_none_type(self, t: NoneTyp) -> Type: - return t + This annotates variables that are class objects, constrained by + the type argument. See PEP 484 for more details. - def visit_erased_type(self, t: ErasedType) -> Type: - return t + We may encounter expressions whose values are specific classes; + those are represented as callables (possibly overloaded) + corresponding to the class's constructor's signature and returning + an instance of that class. The difference with Type[C] is that + those callables always represent the exact class given as the + return type; Type[C] represents any class that's a subclass of C, + and C may also be a type variable or a union (or Any). - def visit_instance(self, t: Instance) -> Type: - return Instance(t.type, self.translate_types(t.args), t.line, t.repr) + Many questions around subtype relationships between Type[C1] and + def(...) -> C2 are answered by looking at the subtype + relationships between C1 and C2, since Type[] is considered + covariant. - def visit_type_var(self, t: TypeVar) -> Type: - return t + There's an unsolved problem with constructor signatures (also + unsolved in PEP 484): calling a variable whose type is Type[C] + assumes the constructor signature for C, even though a subclass of + C might completely change the constructor signature. For now we + just assume that users of Type[C] are careful not to do that (in + the future we might detect when they are violating that + assumption). + """ - def visit_callable(self, t: Callable) -> Type: - return Callable(self.translate_types(t.arg_types), - t.arg_kinds, - t.arg_names, - t.ret_type.accept(self), - t.fallback, - t.name, - self.translate_variables(t.variables), - self.translate_bound_vars(t.bound_vars), - t.line, t.repr) - - def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.translate_types(t.items), - Any(t.fallback.accept(self)), - t.line, t.repr) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line, t.repr) + __slots__ = ("item",) + + # This can't be everything, but it can be a class reference, + # a generic class instance, a union, Any, a type variable... + item: ProperType + + def __init__( + self, + item: Bogus[Instance | AnyType | TypeVarType | TupleType | NoneType | CallableType], + *, + line: int = -1, + column: int = -1, + ) -> None: + """To ensure Type[Union[A, B]] is always represented as Union[Type[A], Type[B]], item of + type UnionType must be handled through make_normalized static method. + """ + super().__init__(line, column) + self.item = item - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.translate_types(t.items), t.line, t.repr) + @staticmethod + def make_normalized(item: Type, *, line: int = -1, column: int = -1) -> ProperType: + item = get_proper_type(item) + if isinstance(item, UnionType): + return UnionType.make_union( + [TypeType.make_normalized(union_item) for union_item in item.items], + line=line, + column=column, + ) + return TypeType(item, line=line, column=column) # type: ignore[arg-type] + + def accept(self, visitor: TypeVisitor[T]) -> T: + return visitor.visit_type_type(self) + + def __hash__(self) -> int: + return hash(self.item) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypeType): + return NotImplemented + return self.item == other.item + + def serialize(self) -> JsonDict: + return {".class": "TypeType", "item": self.item.serialize()} + + @classmethod + def deserialize(cls, data: JsonDict) -> Type: + assert data[".class"] == "TypeType" + return TypeType.make_normalized(deserialize_type(data["item"])) + + +class PlaceholderType(ProperType): + """Temporary, yet-unknown type during semantic analysis. + + This is needed when there's a reference to a type before the real symbol + table entry of the target type is available (specifically, we use a + temporary PlaceholderNode symbol node). Consider this example: + + class str(Sequence[str]): ... + + We use a PlaceholderType for the 'str' in 'Sequence[str]' since we can't create + a TypeInfo for 'str' until all base classes have been resolved. We'll soon + perform another analysis iteration which replaces the base class with a complete + type without any placeholders. After semantic analysis, no placeholder types must + exist. + """ + + __slots__ = ("fullname", "args") + + def __init__(self, fullname: str | None, args: list[Type], line: int) -> None: + super().__init__(line) + self.fullname = fullname # Must be a valid full name of an actual node (or None). + self.args = args + + def accept(self, visitor: TypeVisitor[T]) -> T: + assert isinstance(visitor, SyntheticTypeVisitor) + ret: T = visitor.visit_placeholder_type(self) + return ret + + def __hash__(self) -> int: + return hash((self.fullname, tuple(self.args))) - def translate_types(self, types: List[Type]) -> List[Type]: - return [t.accept(self) for t in types] + def __eq__(self, other: object) -> bool: + if not isinstance(other, PlaceholderType): + return NotImplemented + return self.fullname == other.fullname and self.args == other.args - def translate_bound_vars( - self, types: List[Tuple[int, Type]]) -> List[Tuple[int, Type]]: - return [(id, t.accept(self)) for id, t in types] + def serialize(self) -> str: + # We should never get here since all placeholders should be replaced + # during semantic analysis. + assert False, f"Internal error: unresolved placeholder type {self.fullname}" - def translate_variables(self, - variables: List[TypeVarDef]) -> List[TypeVarDef]: - return variables +@overload +def get_proper_type(typ: None) -> None: ... -class TypeStrVisitor(TypeVisitor[str]): + +@overload +def get_proper_type(typ: Type) -> ProperType: ... + + +def get_proper_type(typ: Type | None) -> ProperType | None: + """Get the expansion of a type alias type. + + If the type is already a proper type, this is a no-op. Use this function + wherever a decision is made on a call like e.g. 'if isinstance(typ, UnionType): ...', + because 'typ' in this case may be an alias to union. Note: if after making the decision + on the isinstance() call you pass on the original type (and not one of its components) + it is recommended to *always* pass on the unexpanded alias. + """ + if typ is None: + return None + if isinstance(typ, TypeGuardedType): # type: ignore[misc] + typ = typ.type_guard + while isinstance(typ, TypeAliasType): + typ = typ._expand_once() + # TODO: store the name of original type alias on this type, so we can show it in errors. + return cast(ProperType, typ) + + +@overload +def get_proper_types(types: list[Type] | tuple[Type, ...]) -> list[ProperType]: ... + + +@overload +def get_proper_types( + types: list[Type | None] | tuple[Type | None, ...] +) -> list[ProperType | None]: ... + + +def get_proper_types( + types: list[Type] | list[Type | None] | tuple[Type | None, ...] +) -> list[ProperType] | list[ProperType | None]: + if isinstance(types, list): + typelist = types + # Optimize for the common case so that we don't need to allocate anything + if not any( + isinstance(t, (TypeAliasType, TypeGuardedType)) for t in typelist # type: ignore[misc] + ): + return cast("list[ProperType]", typelist) + return [get_proper_type(t) for t in typelist] + else: + return [get_proper_type(t) for t in types] + + +# We split off the type visitor base classes to another module +# to make it easier to gradually get modules working with mypyc. +# Import them here, after the types are defined. +# This is intended as a re-export also. +from mypy.type_visitor import ( + ALL_STRATEGY as ALL_STRATEGY, + ANY_STRATEGY as ANY_STRATEGY, + BoolTypeQuery as BoolTypeQuery, + SyntheticTypeVisitor as SyntheticTypeVisitor, + TypeQuery as TypeQuery, + TypeTranslator as TypeTranslator, + TypeVisitor as TypeVisitor, +) +from mypy.typetraverser import TypeTraverserVisitor + + +class TypeStrVisitor(SyntheticTypeVisitor[str]): """Visitor for pretty-printing types into strings. This is mostly for debugging/testing. @@ -614,256 +3249,584 @@ class TypeStrVisitor(TypeVisitor[str]): Do not preserve original formatting. Notes: - - Include implicit bound type variables of callables. - Represent unbound types as Foo? or Foo?[...]. - - Represent the NoneTyp type as None. + - Represent the NoneType type as None. """ - def visit_unbound_type(self, t): - s = t.name + '?' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) + def __init__(self, id_mapper: IdMapper | None = None, *, options: Options) -> None: + self.id_mapper = id_mapper + self.any_as_dots = False + self.options = options + + def visit_unbound_type(self, t: UnboundType) -> str: + s = t.name + "?" + if t.args: + s += f"[{self.list_str(t.args)}]" return s - def visit_type_list(self, t): - return ''.format(self.list_str(t.items)) + def visit_type_list(self, t: TypeList) -> str: + return f"" - def visit_error_type(self, t): - return '' + def visit_callable_argument(self, t: CallableArgument) -> str: + typ = t.typ.accept(self) + if t.name is None: + return f"{t.constructor}({typ})" + else: + return f"{t.constructor}({typ}, {t.name})" - def visit_any(self, t): - return 'Any' + def visit_any(self, t: AnyType) -> str: + if self.any_as_dots and t.type_of_any == TypeOfAny.special_form: + return "..." + return "Any" - def visit_void(self, t): - return 'void' + def visit_none_type(self, t: NoneType) -> str: + return "None" - def visit_none_type(self, t): - # Include quotes to make this distinct from the None value. - return "'None'" + def visit_uninhabited_type(self, t: UninhabitedType) -> str: + return "Never" - def visit_erased_type(self, t): + def visit_erased_type(self, t: ErasedType) -> str: return "" - def visit_instance(self, t): - s = t.type.fullname() - if t.erased: - s += '*' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) + def visit_deleted_type(self, t: DeletedType) -> str: + if t.source is None: + return "" + else: + return f"" + + def visit_instance(self, t: Instance) -> str: + if t.last_known_value and not t.args: + # Instances with a literal fallback should never be generic. If they are, + # something went wrong so we fall back to showing the full Instance repr. + s = f"{t.last_known_value.accept(self)}?" + else: + s = t.type.fullname or t.type.name or "" + + if t.args: + if t.type.fullname == "builtins.tuple": + assert len(t.args) == 1 + s += f"[{self.list_str(t.args)}, ...]" + else: + s += f"[{self.list_str(t.args)}]" + elif t.type.has_type_var_tuple_type and len(t.type.type_vars) == 1: + s += "[()]" + if self.id_mapper: + s += f"<{self.id_mapper.id(t.type)}>" + return s + + def visit_type_var(self, t: TypeVarType) -> str: + if t.name is None: + # Anonymous type variable type (only numeric id). + s = f"`{t.id}" + else: + # Named type variable type. + s = f"{t.name}`{t.id}" + if self.id_mapper and t.upper_bound: + s += f"(upper_bound={t.upper_bound.accept(self)})" + if t.has_default(): + s += f" = {t.default.accept(self)}" return s - def visit_type_var(self, t): + def visit_param_spec(self, t: ParamSpecType) -> str: + # prefixes are displayed as Concatenate + s = "" + if t.prefix.arg_types: + s += f"[{self.list_str(t.prefix.arg_types)}, **" if t.name is None: # Anonymous type variable type (only numeric id). - return '`{}'.format(t.id) + s += f"`{t.id}" else: # Named type variable type. - return '{}`{}'.format(t.name, t.id) + s += f"{t.name_with_suffix()}`{t.id}" + if t.prefix.arg_types: + s += "]" + if t.has_default(): + s += f" = {t.default.accept(self)}" + return s + + def visit_parameters(self, t: Parameters) -> str: + # This is copied from visit_callable -- is there a way to decrease duplication? + if t.is_ellipsis_args: + return "..." - def visit_callable(self, t): - s = '' + s = "" bare_asterisk = False for i in range(len(t.arg_types)): - if s != '': - s += ', ' - if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: - s += '*, ' + if s != "": + s += ", " + if t.arg_kinds[i].is_named() and not bare_asterisk: + s += "*, " bare_asterisk = True - if t.arg_kinds[i] == mypy.nodes.ARG_STAR: - s += '*' - if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: - s += '**' - if t.arg_names[i]: - s += t.arg_names[i] + ': ' - s += str(t.arg_types[i]) - if t.arg_kinds[i] == mypy.nodes.ARG_OPT: - s += ' =' + if t.arg_kinds[i] == ARG_STAR: + s += "*" + if t.arg_kinds[i] == ARG_STAR2: + s += "**" + name = t.arg_names[i] + if name: + s += f"{name}: " + r = t.arg_types[i].accept(self) - s = '({})'.format(s) + s += r - if not isinstance(t.ret_type, Void): - s += ' -> {}'.format(t.ret_type) + if t.arg_kinds[i].is_optional(): + s += " =" - if t.variables: - s = '{} {}'.format(t.variables, s) + return f"[{s}]" - if t.bound_vars != []: - # Include implicit bound type variables. - a = [] - for i, bt in t.bound_vars: - a.append('{}:{}'.format(i, bt)) - s = '[{}] {}'.format(', '.join(a), s) + def visit_type_var_tuple(self, t: TypeVarTupleType) -> str: + if t.name is None: + # Anonymous type variable type (only numeric id). + s = f"`{t.id}" + else: + # Named type variable type. + s = f"{t.name}`{t.id}" + if t.has_default(): + s += f" = {t.default.accept(self)}" + return s + + def visit_callable_type(self, t: CallableType) -> str: + param_spec = t.param_spec() + if param_spec is not None: + num_skip = 2 + else: + num_skip = 0 + + s = "" + asterisk = False + for i in range(len(t.arg_types) - num_skip): + if s != "": + s += ", " + if t.arg_kinds[i].is_named() and not asterisk: + s += "*, " + asterisk = True + if t.arg_kinds[i] == ARG_STAR: + s += "*" + asterisk = True + if t.arg_kinds[i] == ARG_STAR2: + s += "**" + name = t.arg_names[i] + if name: + s += name + ": " + type_str = t.arg_types[i].accept(self) + if t.arg_kinds[i] == ARG_STAR2 and t.unpack_kwargs: + type_str = f"Unpack[{type_str}]" + s += type_str + if t.arg_kinds[i].is_optional(): + s += " =" + + if param_spec is not None: + n = param_spec.name + if s: + s += ", " + s += f"*{n}.args, **{n}.kwargs" + if param_spec.has_default(): + s += f" = {param_spec.default.accept(self)}" + + s = f"({s})" + + if not isinstance(get_proper_type(t.ret_type), NoneType): + if t.type_guard is not None: + s += f" -> TypeGuard[{t.type_guard.accept(self)}]" + elif t.type_is is not None: + s += f" -> TypeIs[{t.type_is.accept(self)}]" + else: + s += f" -> {t.ret_type.accept(self)}" - return 'def {}'.format(s) + if t.variables: + vs = [] + for var in t.variables: + if isinstance(var, TypeVarType): + # We reimplement TypeVarType.__repr__ here in order to support id_mapper. + if var.values: + vals = f"({', '.join(val.accept(self) for val in var.values)})" + vs.append(f"{var.name} in {vals}") + elif not is_named_instance(var.upper_bound, "builtins.object"): + vs.append( + f"{var.name} <: {var.upper_bound.accept(self)}{f' = {var.default.accept(self)}' if var.has_default() else ''}" + ) + else: + vs.append( + f"{var.name}{f' = {var.default.accept(self)}' if var.has_default() else ''}" + ) + else: + # For other TypeVarLikeTypes, use the name and default + vs.append( + f"{var.name}{f' = {var.default.accept(self)}' if var.has_default() else ''}" + ) + s = f"[{', '.join(vs)}] {s}" - def visit_overloaded(self, t): + return f"def {s}" + + def visit_overloaded(self, t: Overloaded) -> str: a = [] - for i in t.items(): + for i in t.items: a.append(i.accept(self)) - return 'Overload({})'.format(', '.join(a)) - - def visit_tuple_type(self, t): + return f"Overload({', '.join(a)})" + + def visit_tuple_type(self, t: TupleType) -> str: + s = self.list_str(t.items) or "()" + tuple_name = "tuple" if self.options.use_lowercase_names() else "Tuple" + if t.partial_fallback and t.partial_fallback.type: + fallback_name = t.partial_fallback.type.fullname + if fallback_name != "builtins.tuple": + return f"{tuple_name}[{s}, fallback={t.partial_fallback.accept(self)}]" + return f"{tuple_name}[{s}]" + + def visit_typeddict_type(self, t: TypedDictType) -> str: + def item_str(name: str, typ: str) -> str: + modifier = "" + if name not in t.required_keys: + modifier += "?" + if name in t.readonly_keys: + modifier += "=" + return f"{name!r}{modifier}: {typ}" + + s = ( + "{" + + ", ".join(item_str(name, typ.accept(self)) for name, typ in t.items.items()) + + "}" + ) + prefix = "" + if t.fallback and t.fallback.type: + if t.fallback.type.fullname not in TPDICT_FB_NAMES: + prefix = repr(t.fallback.type.fullname) + ", " + return f"TypedDict({prefix}{s})" + + def visit_raw_expression_type(self, t: RawExpressionType) -> str: + return repr(t.literal_value) + + def visit_literal_type(self, t: LiteralType) -> str: + return f"Literal[{t.value_repr()}]" + + def visit_union_type(self, t: UnionType) -> str: s = self.list_str(t.items) - if t.fallback: - fallback_name = t.fallback.type.fullname() - if fallback_name != 'builtins.tuple': - return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) - return 'Tuple[{}]'.format(s) + return f"Union[{s}]" - def visit_star_type(self, t): - s = t.type.accept(self) - return '*{}'.format(s) + def visit_partial_type(self, t: PartialType) -> str: + if t.type is None: + return "" + else: + return "".format(t.type.name, ", ".join(["?"] * len(t.type.type_vars))) - def visit_union_type(self, t): - s = self.list_str(t.items) - return 'Union[{}]'.format(s) + def visit_ellipsis_type(self, t: EllipsisType) -> str: + return "..." + + def visit_type_type(self, t: TypeType) -> str: + if self.options.use_lowercase_names(): + type_name = "type" + else: + type_name = "Type" + return f"{type_name}[{t.item.accept(self)}]" + + def visit_placeholder_type(self, t: PlaceholderType) -> str: + return f"" + + def visit_type_alias_type(self, t: TypeAliasType) -> str: + if t.alias is not None: + unrolled, recursed = t._partial_expansion() + self.any_as_dots = recursed + type_str = unrolled.accept(self) + self.any_as_dots = False + return type_str + return "" - def visit_runtime_type_var(self, t): - return '' + def visit_unpack_type(self, t: UnpackType) -> str: + return f"Unpack[{t.type.accept(self)}]" - def list_str(self, a): + def list_str(self, a: Iterable[Type]) -> str: """Convert items of an array to strings (pretty-print types) and join the results with commas. """ res = [] for t in a: - if isinstance(t, Type): - res.append(t.accept(self)) - else: - res.append(str(t)) - return ', '.join(res) + res.append(t.accept(self)) + return ", ".join(res) -# These constants define the method used by TypeQuery to combine multiple -# query results, e.g. for tuple types. The strategy is not used for empty -# result lists; in that case the default value takes precedence. -ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. -ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. +class TrivialSyntheticTypeTranslator(TypeTranslator, SyntheticTypeVisitor[Type]): + """A base class for type translators that need to be run during semantic analysis.""" + def visit_placeholder_type(self, t: PlaceholderType) -> Type: + return t -class TypeQuery(TypeVisitor[bool]): - """Visitor for performing simple boolean queries of types. + def visit_callable_argument(self, t: CallableArgument) -> Type: + return t - This class allows defining the default value for leafs to simplify the - implementation of many queries. - """ + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + return t - default = False # Default result - strategy = 0 # Strategy for combining multiple values + def visit_raw_expression_type(self, t: RawExpressionType) -> Type: + return t - def __init__(self, default: bool, strategy: int) -> None: - """Construct a query visitor. + def visit_type_list(self, t: TypeList) -> Type: + return t - Use the given default result and strategy for combining - multiple results. The strategy must be either - ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. - """ - self.default = default - self.strategy = strategy - def visit_unbound_type(self, t: UnboundType) -> bool: - return self.default +class UnrollAliasVisitor(TrivialSyntheticTypeTranslator): + def __init__( + self, initial_aliases: set[TypeAliasType], cache: dict[Type, Type] | None + ) -> None: + assert cache is not None + super().__init__(cache) + self.recursed = False + self.initial_aliases = initial_aliases + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + if t in self.initial_aliases: + self.recursed = True + return AnyType(TypeOfAny.special_form) + # Create a new visitor on encountering a new type alias, so that an alias like + # A = Tuple[B, B] + # B = int + # will not be detected as recursive on the second encounter of B. + subvisitor = UnrollAliasVisitor(self.initial_aliases | {t}, self.cache) + result = get_proper_type(t).accept(subvisitor) + if subvisitor.recursed: + self.recursed = True + return result + + +def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> TypeGuard[Instance]: + if not isinstance(fullnames, tuple): + fullnames = (fullnames,) + + t = get_proper_type(t) + return isinstance(t, Instance) and t.type.fullname in fullnames + + +class LocationSetter(TypeTraverserVisitor): + # TODO: Should we update locations of other Type subclasses? + def __init__(self, line: int, column: int) -> None: + self.line = line + self.column = column + + def visit_instance(self, typ: Instance) -> None: + typ.line = self.line + typ.column = self.column + super().visit_instance(typ) - def visit_type_list(self, t: TypeList) -> bool: - return self.default + def visit_type_alias_type(self, typ: TypeAliasType) -> None: + typ.line = self.line + typ.column = self.column + super().visit_type_alias_type(typ) - def visit_error_type(self, t: ErrorType) -> bool: - return self.default - def visit_any(self, t: AnyType) -> bool: - return self.default +class HasTypeVars(BoolTypeQuery): + """Visitor for querying whether a type has a type variable component.""" - def visit_void(self, t: Void) -> bool: - return self.default + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + self.skip_alias_target = True - def visit_none_type(self, t: NoneTyp) -> bool: - return self.default + def visit_type_var(self, t: TypeVarType) -> bool: + return True - def visit_erased_type(self, t: ErasedType) -> bool: - return self.default + def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool: + return True - def visit_type_var(self, t: TypeVar) -> bool: - return self.default + def visit_param_spec(self, t: ParamSpecType) -> bool: + return True - def visit_instance(self, t: Instance) -> bool: - return self.query_types(t.args) - def visit_callable(self, t: Callable) -> bool: - # FIX generics - return self.query_types(t.arg_types + [t.ret_type]) +def has_type_vars(typ: Type) -> bool: + """Check if a type contains any type variables (recursively).""" + return typ.accept(HasTypeVars()) - def visit_tuple_type(self, t: TupleType) -> bool: - return self.query_types(t.items) - def visit_star_type(self, t: StarType) -> bool: - return t.type.accept(self) +class HasRecursiveType(BoolTypeQuery): + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) - def visit_union_type(self, t: UnionType) -> bool: - return self.query_types(t.items) + def visit_type_alias_type(self, t: TypeAliasType) -> bool: + return t.is_recursive or self.query_types(t.args) - def visit_runtime_type_var(self, t: RuntimeTypeVar) -> bool: - return self.default - def query_types(self, types: List[Type]) -> bool: - """Perform a query for a list of types. +# Use singleton since this is hot (note: call reset() before using) +_has_recursive_type: Final = HasRecursiveType() - Use the strategy constant to combine the results. - """ - if not types: - # Use default result for empty list. - return self.default - if self.strategy == ANY_TYPE_STRATEGY: - # Return True if at least one component is true. - res = False - for t in types: - res = res or t.accept(self) - if res: - break - return res - else: - # Return True if all components are true. - res = True - for t in types: - res = res and t.accept(self) - if not res: - break - return res - - -def strip_type(typ: Type) -> Type: - """Make a copy of type without 'debugging info' (function name).""" - - if isinstance(typ, Callable): - return Callable(typ.arg_types, - typ.arg_kinds, - typ.arg_names, - typ.ret_type, - typ.fallback, - None, - typ.variables) - elif isinstance(typ, Overloaded): - return Overloaded([cast(Callable, strip_type(item)) - for item in typ.items()]) + +def has_recursive_types(typ: Type) -> bool: + """Check if a type contains any recursive aliases (recursively).""" + _has_recursive_type.reset() + return typ.accept(_has_recursive_type) + + +def split_with_prefix_and_suffix( + types: tuple[Type, ...], prefix: int, suffix: int +) -> tuple[tuple[Type, ...], tuple[Type, ...], tuple[Type, ...]]: + if len(types) <= prefix + suffix: + types = extend_args_for_prefix_and_suffix(types, prefix, suffix) + if suffix: + return types[:prefix], types[prefix:-suffix], types[-suffix:] + else: + return types[:prefix], types[prefix:], () + + +def extend_args_for_prefix_and_suffix( + types: tuple[Type, ...], prefix: int, suffix: int +) -> tuple[Type, ...]: + """Extend list of types by eating out from variadic tuple to satisfy prefix and suffix.""" + idx = None + item = None + for i, t in enumerate(types): + if isinstance(t, UnpackType): + p_type = get_proper_type(t.type) + if isinstance(p_type, Instance) and p_type.type.fullname == "builtins.tuple": + item = p_type.args[0] + idx = i + break + + if idx is None: + return types + assert item is not None + if idx < prefix: + start = (item,) * (prefix - idx) else: - return typ + start = () + if len(types) - idx - 1 < suffix: + end = (item,) * (suffix - len(types) + idx + 1) + else: + end = () + return types[:idx] + start + (types[idx],) + end + types[idx + 1 :] -def replace_leading_arg_type(t: Callable, self_type: Type) -> Callable: - """Return a copy of a callable type with a different self argument type. +def flatten_nested_unions( + types: Sequence[Type], *, handle_type_alias_type: bool = True, handle_recursive: bool = True +) -> list[Type]: + """Flatten nested unions in a type list.""" + if not isinstance(types, list): + typelist = list(types) + else: + typelist = cast("list[Type]", types) - Assume that the callable is the signature of a method. + # Fast path: most of the time there is nothing to flatten + if not any(isinstance(t, (TypeAliasType, UnionType)) for t in typelist): # type: ignore[misc] + return typelist + + flat_items: list[Type] = [] + for t in typelist: + if handle_type_alias_type: + if not handle_recursive and isinstance(t, TypeAliasType) and t.is_recursive: + tp: Type = t + else: + tp = get_proper_type(t) + else: + tp = t + if isinstance(tp, ProperType) and isinstance(tp, UnionType): + flat_items.extend( + flatten_nested_unions(tp.items, handle_type_alias_type=handle_type_alias_type) + ) + else: + # Must preserve original aliases when possible. + flat_items.append(t) + return flat_items + + +def find_unpack_in_list(items: Sequence[Type]) -> int | None: + unpack_index: int | None = None + for i, item in enumerate(items): + if isinstance(item, UnpackType): + # We cannot fail here, so we must check this in an earlier + # semanal phase. + # Funky code here avoids mypyc narrowing the type of unpack_index. + old_index = unpack_index + assert old_index is None + # Don't return so that we can also sanity check there is only one. + unpack_index = i + return unpack_index + + +def flatten_nested_tuples(types: Sequence[Type]) -> list[Type]: + """Recursively flatten TupleTypes nested with Unpack. + + For example this will transform + Tuple[A, Unpack[Tuple[B, Unpack[Tuple[C, D]]]]] + into + Tuple[A, B, C, D] """ - return Callable([self_type] + t.arg_types[1:], - t.arg_kinds, - t.arg_names, - t.ret_type, - t.fallback, - t.name, - t.variables, - t.bound_vars, - t.line, None) - - -def is_named_instance(t: Type, fullname: str) -> bool: - return (isinstance(t, Instance) and - cast(Instance, t).type.fullname() == fullname) + res = [] + for typ in types: + if not isinstance(typ, UnpackType): + res.append(typ) + continue + p_type = get_proper_type(typ.type) + if not isinstance(p_type, TupleType): + res.append(typ) + continue + res.extend(flatten_nested_tuples(p_type.items)) + return res + + +def is_literal_type(typ: ProperType, fallback_fullname: str, value: LiteralValue) -> bool: + """Check if this type is a LiteralType with the given fallback type and value.""" + if isinstance(typ, Instance) and typ.last_known_value: + typ = typ.last_known_value + return ( + isinstance(typ, LiteralType) + and typ.fallback.type.fullname == fallback_fullname + and typ.value == value + ) + + +names: Final = globals().copy() +names.pop("NOT_READY", None) +deserialize_map: Final = { + key: obj.deserialize + for key, obj in names.items() + if isinstance(obj, type) and issubclass(obj, Type) and obj is not Type +} + + +def callable_with_ellipsis(any_type: AnyType, ret_type: Type, fallback: Instance) -> CallableType: + """Construct type Callable[..., ret_type].""" + return CallableType( + [any_type, any_type], + [ARG_STAR, ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + is_ellipsis_args=True, + ) + + +def remove_dups(types: list[T]) -> list[T]: + if len(types) <= 1: + return types + # Get unique elements in order of appearance + all_types: set[T] = set() + new_types: list[T] = [] + for t in types: + if t not in all_types: + new_types.append(t) + all_types.add(t) + return new_types + + +def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: + """Represent type variables as they would appear in a type argument list.""" + args: list[Type] = [] + for tv in type_vars: + if isinstance(tv, TypeVarTupleType): + args.append(UnpackType(tv)) + else: + args.append(tv) + return tuple(args) + + +# This cyclic import is unfortunate, but to avoid it we would need to move away all uses +# of get_proper_type() from types.py. Majority of them have been removed, but few remaining +# are quite tricky to get rid of, but ultimately we want to do it at some point. +from mypy.expandtype import ExpandTypeVisitor + + +class InstantiateAliasVisitor(ExpandTypeVisitor): + def visit_union_type(self, t: UnionType) -> Type: + # Unlike regular expand_type(), we don't do any simplification for unions, + # not even removing strict duplicates. There are three reasons for this: + # * get_proper_type() is a very hot function, even slightest slow down will + # cause a perf regression + # * We want to preserve this historical behaviour, to avoid possible + # regressions + # * Simplifying unions may (indirectly) call get_proper_type(), causing + # infinite recursion. + return TypeTranslator.visit_union_type(self, t) diff --git a/mypy/types_utils.py b/mypy/types_utils.py new file mode 100644 index 000000000000..1cd56eae5835 --- /dev/null +++ b/mypy/types_utils.py @@ -0,0 +1,166 @@ +""" +This module is for (more basic) type operations that should not depend on is_subtype(), +meet_types(), join_types() etc. We don't want to keep them in mypy/types.py for two reasons: +* Reduce the size of that module. +* Reduce use of get_proper_type() in types.py to avoid cyclic imports + expand_type <-> types, if we move get_proper_type() to the former. +""" + +from __future__ import annotations + +from typing import Callable, Iterable, cast + +from mypy.nodes import ARG_STAR, ARG_STAR2, FuncItem, TypeAlias +from mypy.types import ( + AnyType, + CallableType, + Instance, + NoneType, + Overloaded, + ParamSpecType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypeType, + TypeVarType, + UnionType, + UnpackType, + flatten_nested_unions, + get_proper_type, + get_proper_types, +) + + +def flatten_types(types: Iterable[Type]) -> Iterable[Type]: + for t in types: + tp = get_proper_type(t) + if isinstance(tp, UnionType): + yield from flatten_types(tp.items) + else: + yield t + + +def strip_type(typ: Type) -> Type: + """Make a copy of type without 'debugging info' (function name).""" + orig_typ = typ + typ = get_proper_type(typ) + if isinstance(typ, CallableType): + return typ.copy_modified(name=None) + elif isinstance(typ, Overloaded): + return Overloaded([cast(CallableType, strip_type(item)) for item in typ.items]) + else: + return orig_typ + + +def is_invalid_recursive_alias(seen_nodes: set[TypeAlias], target: Type) -> bool: + """Flag aliases like A = Union[int, A], T = tuple[int, *T] (and similar mutual aliases). + + Such aliases don't make much sense, and cause problems in later phases. + """ + if isinstance(target, TypeAliasType): + if target.alias in seen_nodes: + return True + assert target.alias, f"Unfixed type alias {target.type_ref}" + return is_invalid_recursive_alias(seen_nodes | {target.alias}, get_proper_type(target)) + assert isinstance(target, ProperType) + if not isinstance(target, (UnionType, TupleType)): + return False + if isinstance(target, UnionType): + return any(is_invalid_recursive_alias(seen_nodes, item) for item in target.items) + for item in target.items: + if isinstance(item, UnpackType): + if is_invalid_recursive_alias(seen_nodes, item.type): + return True + return False + + +def is_bad_type_type_item(item: Type) -> bool: + """Prohibit types like Type[Type[...]]. + + Such types are explicitly prohibited by PEP 484. Also, they cause problems + with recursive types like T = Type[T], because internal representation of + TypeType item is normalized (i.e. always a proper type). + """ + item = get_proper_type(item) + if isinstance(item, TypeType): + return True + if isinstance(item, UnionType): + return any( + isinstance(get_proper_type(i), TypeType) for i in flatten_nested_unions(item.items) + ) + return False + + +def is_union_with_any(tp: Type) -> bool: + """Is this a union with Any or a plain Any type?""" + tp = get_proper_type(tp) + if isinstance(tp, AnyType): + return True + if not isinstance(tp, UnionType): + return False + return any(is_union_with_any(t) for t in get_proper_types(tp.items)) + + +def is_generic_instance(tp: Type) -> bool: + tp = get_proper_type(tp) + return isinstance(tp, Instance) and bool(tp.args) + + +def is_overlapping_none(t: Type) -> bool: + t = get_proper_type(t) + return isinstance(t, NoneType) or ( + isinstance(t, UnionType) and any(isinstance(get_proper_type(e), NoneType) for e in t.items) + ) + + +def remove_optional(typ: Type) -> Type: + typ = get_proper_type(typ) + if isinstance(typ, UnionType): + return UnionType.make_union( + [t for t in typ.items if not isinstance(get_proper_type(t), NoneType)] + ) + else: + return typ + + +def is_self_type_like(typ: Type, *, is_classmethod: bool) -> bool: + """Does this look like a self-type annotation?""" + typ = get_proper_type(typ) + if not is_classmethod: + return isinstance(typ, TypeVarType) + if not isinstance(typ, TypeType): + return False + return isinstance(typ.item, TypeVarType) + + +def store_argument_type( + defn: FuncItem, i: int, typ: CallableType, named_type: Callable[[str, list[Type]], Instance] +) -> None: + arg_type = typ.arg_types[i] + if typ.arg_kinds[i] == ARG_STAR: + if isinstance(arg_type, ParamSpecType): + pass + elif isinstance(arg_type, UnpackType): + unpacked_type = get_proper_type(arg_type.type) + if isinstance(unpacked_type, TupleType): + # Instead of using Tuple[Unpack[Tuple[...]]], just use Tuple[...] + arg_type = unpacked_type + elif ( + isinstance(unpacked_type, Instance) + and unpacked_type.type.fullname == "builtins.tuple" + ): + arg_type = unpacked_type + else: + # TODO: verify that we can only have a TypeVarTuple here. + arg_type = TupleType( + [arg_type], + fallback=named_type("builtins.tuple", [named_type("builtins.object", [])]), + ) + else: + # builtins.tuple[T] is typing.Tuple[T, ...] + arg_type = named_type("builtins.tuple", [arg_type]) + elif typ.arg_kinds[i] == ARG_STAR2: + if not isinstance(arg_type, ParamSpecType) and not typ.unpack_kwargs: + arg_type = named_type("builtins.dict", [named_type("builtins.str", []), arg_type]) + defn.arguments[i].variable.type = arg_type diff --git a/mypy/typeshed/LICENSE b/mypy/typeshed/LICENSE new file mode 100644 index 000000000000..13264487581f --- /dev/null +++ b/mypy/typeshed/LICENSE @@ -0,0 +1,237 @@ +The "typeshed" project is licensed under the terms of the Apache license, as +reproduced below. + += = = = = + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += = = = = + +Parts of typeshed are licensed under different licenses (like the MIT +license), reproduced below. + += = = = = + +The MIT License + +Copyright (c) 2015 Jukka Lehtosalo and contributors + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + += = = = = diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS new file mode 100644 index 000000000000..7ff14c55d3a8 --- /dev/null +++ b/mypy/typeshed/stdlib/VERSIONS @@ -0,0 +1,342 @@ +# The structure of this file is as follows: +# - Blank lines and comments starting with `#` are ignored. +# - Lines contain the name of a module, followed by a colon, +# a space, and a version range (for example: `symbol: 3.0-3.9`). +# +# Version ranges may be of the form "X.Y-A.B" or "X.Y-". The +# first form means that a module was introduced in version X.Y and last +# available in version A.B. The second form means that the module was +# introduced in version X.Y and is still available in the latest +# version of Python. +# +# If a submodule is not listed separately, it has the same lifetime as +# its parent module. +# +# Python versions before 3.0 are ignored, so any module that was already +# present in 3.0 will have "3.0" as its minimum version. Version ranges +# for unsupported versions of Python 3 are generally accurate but we do +# not guarantee their correctness. + +__future__: 3.0- +__main__: 3.0- +_ast: 3.0- +_asyncio: 3.0- +_bisect: 3.0- +_blake2: 3.6- +_bootlocale: 3.4-3.9 +_bz2: 3.3- +_codecs: 3.0- +_collections_abc: 3.3- +_compat_pickle: 3.1- +_compression: 3.5- +_contextvars: 3.7- +_csv: 3.0- +_ctypes: 3.0- +_curses: 3.0- +_curses_panel: 3.0- +_dbm: 3.0- +_decimal: 3.3- +_dummy_thread: 3.0-3.8 +_dummy_threading: 3.0-3.8 +_frozen_importlib: 3.0- +_frozen_importlib_external: 3.5- +_gdbm: 3.0- +_hashlib: 3.0- +_heapq: 3.0- +_imp: 3.0- +_interpchannels: 3.13- +_interpqueues: 3.13- +_interpreters: 3.13- +_io: 3.0- +_json: 3.0- +_locale: 3.0- +_lsprof: 3.0- +_lzma: 3.3- +_markupbase: 3.0- +_msi: 3.0-3.12 +_multibytecodec: 3.0- +_operator: 3.4- +_osx_support: 3.0- +_posixsubprocess: 3.2- +_py_abc: 3.7- +_pydecimal: 3.5- +_queue: 3.7- +_random: 3.0- +_sitebuiltins: 3.4- +_socket: 3.0- # present in 3.0 at runtime, but not in typeshed +_sqlite3: 3.0- +_ssl: 3.0- +_stat: 3.4- +_struct: 3.0- +_thread: 3.0- +_threading_local: 3.0- +_tkinter: 3.0- +_tracemalloc: 3.4- +_typeshed: 3.0- # not present at runtime, only for type checking +_warnings: 3.0- +_weakref: 3.0- +_weakrefset: 3.0- +_winapi: 3.3- +abc: 3.0- +aifc: 3.0-3.12 +antigravity: 3.0- +argparse: 3.0- +array: 3.0- +ast: 3.0- +asynchat: 3.0-3.11 +asyncio: 3.4- +asyncio.exceptions: 3.8- +asyncio.format_helpers: 3.7- +asyncio.mixins: 3.10- +asyncio.runners: 3.7- +asyncio.staggered: 3.8- +asyncio.taskgroups: 3.11- +asyncio.threads: 3.9- +asyncio.timeouts: 3.11- +asyncio.trsock: 3.8- +asyncore: 3.0-3.11 +atexit: 3.0- +audioop: 3.0-3.12 +base64: 3.0- +bdb: 3.0- +binascii: 3.0- +binhex: 3.0-3.10 +bisect: 3.0- +builtins: 3.0- +bz2: 3.0- +cProfile: 3.0- +calendar: 3.0- +cgi: 3.0-3.12 +cgitb: 3.0-3.12 +chunk: 3.0-3.12 +cmath: 3.0- +cmd: 3.0- +code: 3.0- +codecs: 3.0- +codeop: 3.0- +collections: 3.0- +collections.abc: 3.3- +colorsys: 3.0- +compileall: 3.0- +concurrent: 3.2- +configparser: 3.0- +contextlib: 3.0- +contextvars: 3.7- +copy: 3.0- +copyreg: 3.0- +crypt: 3.0-3.12 +csv: 3.0- +ctypes: 3.0- +curses: 3.0- +dataclasses: 3.7- +datetime: 3.0- +dbm: 3.0- +dbm.sqlite3: 3.13- +decimal: 3.0- +difflib: 3.0- +dis: 3.0- +distutils: 3.0-3.11 +distutils.command.bdist_msi: 3.0-3.10 +distutils.command.bdist_wininst: 3.0-3.9 +doctest: 3.0- +dummy_threading: 3.0-3.8 +email: 3.0- +encodings: 3.0- +encodings.cp1125: 3.4- +encodings.cp273: 3.4- +encodings.cp858: 3.2- +encodings.koi8_t: 3.5- +encodings.kz1048: 3.5- +encodings.mac_centeuro: 3.0-3.8 +ensurepip: 3.0- +enum: 3.4- +errno: 3.0- +faulthandler: 3.3- +fcntl: 3.0- +filecmp: 3.0- +fileinput: 3.0- +fnmatch: 3.0- +formatter: 3.0-3.9 +fractions: 3.0- +ftplib: 3.0- +functools: 3.0- +gc: 3.0- +genericpath: 3.0- +getopt: 3.0- +getpass: 3.0- +gettext: 3.0- +glob: 3.0- +graphlib: 3.9- +grp: 3.0- +gzip: 3.0- +hashlib: 3.0- +heapq: 3.0- +hmac: 3.0- +html: 3.0- +http: 3.0- +imaplib: 3.0- +imghdr: 3.0-3.12 +imp: 3.0-3.11 +importlib: 3.0- +importlib._abc: 3.10- +importlib._bootstrap: 3.0- +importlib._bootstrap_external: 3.5- +importlib.metadata: 3.8- +importlib.metadata._meta: 3.10- +importlib.metadata.diagnose: 3.13- +importlib.readers: 3.10- +importlib.resources: 3.7- +importlib.resources._common: 3.11- +importlib.resources._functional: 3.13- +importlib.resources.abc: 3.11- +importlib.resources.readers: 3.11- +importlib.resources.simple: 3.11- +importlib.simple: 3.11- +inspect: 3.0- +io: 3.0- +ipaddress: 3.3- +itertools: 3.0- +json: 3.0- +keyword: 3.0- +lib2to3: 3.0-3.12 +linecache: 3.0- +locale: 3.0- +logging: 3.0- +lzma: 3.3- +mailbox: 3.0- +mailcap: 3.0-3.12 +marshal: 3.0- +math: 3.0- +mimetypes: 3.0- +mmap: 3.0- +modulefinder: 3.0- +msilib: 3.0-3.12 +msvcrt: 3.0- +multiprocessing: 3.0- +multiprocessing.resource_tracker: 3.8- +multiprocessing.shared_memory: 3.8- +netrc: 3.0- +nis: 3.0-3.12 +nntplib: 3.0-3.12 +nt: 3.0- +ntpath: 3.0- +nturl2path: 3.0- +numbers: 3.0- +opcode: 3.0- +operator: 3.0- +optparse: 3.0- +os: 3.0- +ossaudiodev: 3.0-3.12 +parser: 3.0-3.9 +pathlib: 3.4- +pdb: 3.0- +pickle: 3.0- +pickletools: 3.0- +pipes: 3.0-3.12 +pkgutil: 3.0- +platform: 3.0- +plistlib: 3.0- +poplib: 3.0- +posix: 3.0- +posixpath: 3.0- +pprint: 3.0- +profile: 3.0- +pstats: 3.0- +pty: 3.0- +pwd: 3.0- +py_compile: 3.0- +pyclbr: 3.0- +pydoc: 3.0- +pydoc_data: 3.0- +pyexpat: 3.0- +queue: 3.0- +quopri: 3.0- +random: 3.0- +re: 3.0- +readline: 3.0- +reprlib: 3.0- +resource: 3.0- +rlcompleter: 3.0- +runpy: 3.0- +sched: 3.0- +secrets: 3.6- +select: 3.0- +selectors: 3.4- +shelve: 3.0- +shlex: 3.0- +shutil: 3.0- +signal: 3.0- +site: 3.0- +smtpd: 3.0-3.11 +smtplib: 3.0- +sndhdr: 3.0-3.12 +socket: 3.0- +socketserver: 3.0- +spwd: 3.0-3.12 +sqlite3: 3.0- +sre_compile: 3.0- +sre_constants: 3.0- +sre_parse: 3.0- +ssl: 3.0- +stat: 3.0- +statistics: 3.4- +string: 3.0- +stringprep: 3.0- +struct: 3.0- +subprocess: 3.0- +sunau: 3.0-3.12 +symbol: 3.0-3.9 +symtable: 3.0- +sys: 3.0- +sys._monitoring: 3.12- # Doesn't actually exist. See comments in the stub. +sysconfig: 3.0- +syslog: 3.0- +tabnanny: 3.0- +tarfile: 3.0- +telnetlib: 3.0-3.12 +tempfile: 3.0- +termios: 3.0- +textwrap: 3.0- +this: 3.0- +threading: 3.0- +time: 3.0- +timeit: 3.0- +tkinter: 3.0- +tkinter.tix: 3.0-3.12 +token: 3.0- +tokenize: 3.0- +tomllib: 3.11- +trace: 3.0- +traceback: 3.0- +tracemalloc: 3.4- +tty: 3.0- +turtle: 3.0- +types: 3.0- +typing: 3.5- +typing_extensions: 3.0- +unicodedata: 3.0- +unittest: 3.0- +unittest._log: 3.9- +unittest.async_case: 3.8- +urllib: 3.0- +uu: 3.0-3.12 +uuid: 3.0- +venv: 3.3- +warnings: 3.0- +wave: 3.0- +weakref: 3.0- +webbrowser: 3.0- +winreg: 3.0- +winsound: 3.0- +wsgiref: 3.0- +wsgiref.types: 3.11- +xdrlib: 3.0-3.12 +xml: 3.0- +xmlrpc: 3.0- +xxlimited: 3.2- +zipapp: 3.5- +zipfile: 3.0- +zipfile._path: 3.12- +zipimport: 3.0- +zlib: 3.0- +zoneinfo: 3.9- diff --git a/mypy/typeshed/stdlib/__future__.pyi b/mypy/typeshed/stdlib/__future__.pyi new file mode 100644 index 000000000000..a90cf1eddab7 --- /dev/null +++ b/mypy/typeshed/stdlib/__future__.pyi @@ -0,0 +1,36 @@ +from typing_extensions import TypeAlias + +_VersionInfo: TypeAlias = tuple[int, int, int, str, int] + +class _Feature: + def __init__(self, optionalRelease: _VersionInfo, mandatoryRelease: _VersionInfo | None, compiler_flag: int) -> None: ... + def getOptionalRelease(self) -> _VersionInfo: ... + def getMandatoryRelease(self) -> _VersionInfo | None: ... + compiler_flag: int + +absolute_import: _Feature +division: _Feature +generators: _Feature +nested_scopes: _Feature +print_function: _Feature +unicode_literals: _Feature +with_statement: _Feature +barry_as_FLUFL: _Feature +generator_stop: _Feature +annotations: _Feature + +all_feature_names: list[str] # undocumented + +__all__ = [ + "all_feature_names", + "absolute_import", + "division", + "generators", + "nested_scopes", + "print_function", + "unicode_literals", + "with_statement", + "barry_as_FLUFL", + "generator_stop", + "annotations", +] diff --git a/mypy/typeshed/stdlib/__main__.pyi b/mypy/typeshed/stdlib/__main__.pyi new file mode 100644 index 000000000000..e27843e53382 --- /dev/null +++ b/mypy/typeshed/stdlib/__main__.pyi @@ -0,0 +1,3 @@ +from typing import Any + +def __getattr__(name: str) -> Any: ... diff --git a/mypy/typeshed/stdlib/_ast.pyi b/mypy/typeshed/stdlib/_ast.pyi new file mode 100644 index 000000000000..8dc1bcbea32c --- /dev/null +++ b/mypy/typeshed/stdlib/_ast.pyi @@ -0,0 +1,149 @@ +import sys +from ast import ( + AST as AST, + Add as Add, + And as And, + AnnAssign as AnnAssign, + Assert as Assert, + Assign as Assign, + AsyncFor as AsyncFor, + AsyncFunctionDef as AsyncFunctionDef, + AsyncWith as AsyncWith, + Attribute as Attribute, + AugAssign as AugAssign, + Await as Await, + BinOp as BinOp, + BitAnd as BitAnd, + BitOr as BitOr, + BitXor as BitXor, + BoolOp as BoolOp, + Break as Break, + Call as Call, + ClassDef as ClassDef, + Compare as Compare, + Constant as Constant, + Continue as Continue, + Del as Del, + Delete as Delete, + Dict as Dict, + DictComp as DictComp, + Div as Div, + Eq as Eq, + ExceptHandler as ExceptHandler, + Expr as Expr, + Expression as Expression, + FloorDiv as FloorDiv, + For as For, + FormattedValue as FormattedValue, + FunctionDef as FunctionDef, + FunctionType as FunctionType, + GeneratorExp as GeneratorExp, + Global as Global, + Gt as Gt, + GtE as GtE, + If as If, + IfExp as IfExp, + Import as Import, + ImportFrom as ImportFrom, + In as In, + Interactive as Interactive, + Invert as Invert, + Is as Is, + IsNot as IsNot, + JoinedStr as JoinedStr, + Lambda as Lambda, + List as List, + ListComp as ListComp, + Load as Load, + LShift as LShift, + Lt as Lt, + LtE as LtE, + MatMult as MatMult, + Mod as Mod, + Module as Module, + Mult as Mult, + Name as Name, + NamedExpr as NamedExpr, + Nonlocal as Nonlocal, + Not as Not, + NotEq as NotEq, + NotIn as NotIn, + Or as Or, + Pass as Pass, + Pow as Pow, + Raise as Raise, + Return as Return, + RShift as RShift, + Set as Set, + SetComp as SetComp, + Slice as Slice, + Starred as Starred, + Store as Store, + Sub as Sub, + Subscript as Subscript, + Try as Try, + Tuple as Tuple, + TypeIgnore as TypeIgnore, + UAdd as UAdd, + UnaryOp as UnaryOp, + USub as USub, + While as While, + With as With, + Yield as Yield, + YieldFrom as YieldFrom, + alias as alias, + arg as arg, + arguments as arguments, + boolop as boolop, + cmpop as cmpop, + comprehension as comprehension, + excepthandler as excepthandler, + expr as expr, + expr_context as expr_context, + keyword as keyword, + mod as mod, + operator as operator, + stmt as stmt, + type_ignore as type_ignore, + unaryop as unaryop, + withitem as withitem, +) +from typing import Literal + +if sys.version_info >= (3, 12): + from ast import ParamSpec as ParamSpec, TypeVar as TypeVar, TypeVarTuple as TypeVarTuple, type_param as type_param + +if sys.version_info >= (3, 11): + from ast import TryStar as TryStar + +if sys.version_info >= (3, 10): + from ast import ( + MatchAs as MatchAs, + MatchClass as MatchClass, + MatchMapping as MatchMapping, + MatchOr as MatchOr, + MatchSequence as MatchSequence, + MatchSingleton as MatchSingleton, + MatchStar as MatchStar, + MatchValue as MatchValue, + match_case as match_case, + pattern as pattern, + ) + +if sys.version_info < (3, 9): + from ast import ( + AugLoad as AugLoad, + AugStore as AugStore, + ExtSlice as ExtSlice, + Index as Index, + Param as Param, + Suite as Suite, + slice as slice, + ) + +PyCF_ALLOW_TOP_LEVEL_AWAIT: Literal[8192] +PyCF_ONLY_AST: Literal[1024] +PyCF_TYPE_COMMENTS: Literal[4096] + +if sys.version_info >= (3, 13): + PyCF_OPTIMIZED_AST: Literal[33792] diff --git a/mypy/typeshed/stdlib/_asyncio.pyi b/mypy/typeshed/stdlib/_asyncio.pyi new file mode 100644 index 000000000000..a259026615aa --- /dev/null +++ b/mypy/typeshed/stdlib/_asyncio.pyi @@ -0,0 +1,120 @@ +import sys +from asyncio.events import AbstractEventLoop +from collections.abc import Awaitable, Callable, Coroutine, Generator +from contextvars import Context +from types import FrameType +from typing import Any, Literal, TextIO, TypeVar +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_TaskYieldType: TypeAlias = Future[object] | None + +class Future(Awaitable[_T]): + _state: str + @property + def _exception(self) -> BaseException | None: ... + _blocking: bool + @property + def _log_traceback(self) -> bool: ... + @_log_traceback.setter + def _log_traceback(self, val: Literal[False]) -> None: ... + _asyncio_future_blocking: bool # is a part of duck-typing contract for `Future` + def __init__(self, *, loop: AbstractEventLoop | None = ...) -> None: ... + def __del__(self) -> None: ... + def get_loop(self) -> AbstractEventLoop: ... + @property + def _callbacks(self) -> list[tuple[Callable[[Self], Any], Context]]: ... + def add_done_callback(self, fn: Callable[[Self], object], /, *, context: Context | None = None) -> None: ... + if sys.version_info >= (3, 9): + def cancel(self, msg: Any | None = None) -> bool: ... + else: + def cancel(self) -> bool: ... + + def cancelled(self) -> bool: ... + def done(self) -> bool: ... + def result(self) -> _T: ... + def exception(self) -> BaseException | None: ... + def remove_done_callback(self, fn: Callable[[Self], object], /) -> int: ... + def set_result(self, result: _T, /) -> None: ... + def set_exception(self, exception: type | BaseException, /) -> None: ... + def __iter__(self) -> Generator[Any, None, _T]: ... + def __await__(self) -> Generator[Any, None, _T]: ... + @property + def _loop(self) -> AbstractEventLoop: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +if sys.version_info >= (3, 12): + _TaskCompatibleCoro: TypeAlias = Coroutine[Any, Any, _T_co] +elif sys.version_info >= (3, 9): + _TaskCompatibleCoro: TypeAlias = Generator[_TaskYieldType, None, _T_co] | Coroutine[Any, Any, _T_co] +else: + _TaskCompatibleCoro: TypeAlias = Generator[_TaskYieldType, None, _T_co] | Awaitable[_T_co] + +# mypy and pyright complain that a subclass of an invariant class shouldn't be covariant. +# While this is true in general, here it's sort-of okay to have a covariant subclass, +# since the only reason why `asyncio.Future` is invariant is the `set_result()` method, +# and `asyncio.Task.set_result()` always raises. +class Task(Future[_T_co]): # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments] + if sys.version_info >= (3, 12): + def __init__( + self, + coro: _TaskCompatibleCoro[_T_co], + *, + loop: AbstractEventLoop = ..., + name: str | None = ..., + context: Context | None = None, + eager_start: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __init__( + self, + coro: _TaskCompatibleCoro[_T_co], + *, + loop: AbstractEventLoop = ..., + name: str | None = ..., + context: Context | None = None, + ) -> None: ... + else: + def __init__( + self, coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop = ..., name: str | None = ... + ) -> None: ... + + if sys.version_info >= (3, 12): + def get_coro(self) -> _TaskCompatibleCoro[_T_co] | None: ... + else: + def get_coro(self) -> _TaskCompatibleCoro[_T_co]: ... + + def get_name(self) -> str: ... + def set_name(self, value: object, /) -> None: ... + if sys.version_info >= (3, 12): + def get_context(self) -> Context: ... + + def get_stack(self, *, limit: int | None = None) -> list[FrameType]: ... + def print_stack(self, *, limit: int | None = None, file: TextIO | None = None) -> None: ... + if sys.version_info >= (3, 11): + def cancelling(self) -> int: ... + def uncancel(self) -> int: ... + if sys.version_info < (3, 9): + @classmethod + def current_task(cls, loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... + @classmethod + def all_tasks(cls, loop: AbstractEventLoop | None = None) -> set[Task[Any]]: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +def get_event_loop() -> AbstractEventLoop: ... +def get_running_loop() -> AbstractEventLoop: ... +def _set_running_loop(loop: AbstractEventLoop | None, /) -> None: ... +def _get_running_loop() -> AbstractEventLoop: ... +def _register_task(task: Task[Any]) -> None: ... +def _unregister_task(task: Task[Any]) -> None: ... +def _enter_task(loop: AbstractEventLoop, task: Task[Any]) -> None: ... +def _leave_task(loop: AbstractEventLoop, task: Task[Any]) -> None: ... + +if sys.version_info >= (3, 12): + def current_task(loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... diff --git a/mypy/typeshed/stdlib/_bisect.pyi b/mypy/typeshed/stdlib/_bisect.pyi new file mode 100644 index 000000000000..58488e3d15af --- /dev/null +++ b/mypy/typeshed/stdlib/_bisect.pyi @@ -0,0 +1,84 @@ +import sys +from _typeshed import SupportsLenAndGetItem, SupportsRichComparisonT +from collections.abc import Callable, MutableSequence +from typing import TypeVar, overload + +_T = TypeVar("_T") + +if sys.version_info >= (3, 10): + @overload + def bisect_left( + a: SupportsLenAndGetItem[SupportsRichComparisonT], + x: SupportsRichComparisonT, + lo: int = 0, + hi: int | None = None, + *, + key: None = None, + ) -> int: ... + @overload + def bisect_left( + a: SupportsLenAndGetItem[_T], + x: SupportsRichComparisonT, + lo: int = 0, + hi: int | None = None, + *, + key: Callable[[_T], SupportsRichComparisonT], + ) -> int: ... + @overload + def bisect_right( + a: SupportsLenAndGetItem[SupportsRichComparisonT], + x: SupportsRichComparisonT, + lo: int = 0, + hi: int | None = None, + *, + key: None = None, + ) -> int: ... + @overload + def bisect_right( + a: SupportsLenAndGetItem[_T], + x: SupportsRichComparisonT, + lo: int = 0, + hi: int | None = None, + *, + key: Callable[[_T], SupportsRichComparisonT], + ) -> int: ... + @overload + def insort_left( + a: MutableSequence[SupportsRichComparisonT], + x: SupportsRichComparisonT, + lo: int = 0, + hi: int | None = None, + *, + key: None = None, + ) -> None: ... + @overload + def insort_left( + a: MutableSequence[_T], x: _T, lo: int = 0, hi: int | None = None, *, key: Callable[[_T], SupportsRichComparisonT] + ) -> None: ... + @overload + def insort_right( + a: MutableSequence[SupportsRichComparisonT], + x: SupportsRichComparisonT, + lo: int = 0, + hi: int | None = None, + *, + key: None = None, + ) -> None: ... + @overload + def insort_right( + a: MutableSequence[_T], x: _T, lo: int = 0, hi: int | None = None, *, key: Callable[[_T], SupportsRichComparisonT] + ) -> None: ... + +else: + def bisect_left( + a: SupportsLenAndGetItem[SupportsRichComparisonT], x: SupportsRichComparisonT, lo: int = 0, hi: int | None = None + ) -> int: ... + def bisect_right( + a: SupportsLenAndGetItem[SupportsRichComparisonT], x: SupportsRichComparisonT, lo: int = 0, hi: int | None = None + ) -> int: ... + def insort_left( + a: MutableSequence[SupportsRichComparisonT], x: SupportsRichComparisonT, lo: int = 0, hi: int | None = None + ) -> None: ... + def insort_right( + a: MutableSequence[SupportsRichComparisonT], x: SupportsRichComparisonT, lo: int = 0, hi: int | None = None + ) -> None: ... diff --git a/mypy/typeshed/stdlib/_blake2.pyi b/mypy/typeshed/stdlib/_blake2.pyi new file mode 100644 index 000000000000..10d7019a222f --- /dev/null +++ b/mypy/typeshed/stdlib/_blake2.pyi @@ -0,0 +1,117 @@ +import sys +from _typeshed import ReadableBuffer +from typing import ClassVar, final +from typing_extensions import Self + +BLAKE2B_MAX_DIGEST_SIZE: int = 64 +BLAKE2B_MAX_KEY_SIZE: int = 64 +BLAKE2B_PERSON_SIZE: int = 16 +BLAKE2B_SALT_SIZE: int = 16 +BLAKE2S_MAX_DIGEST_SIZE: int = 32 +BLAKE2S_MAX_KEY_SIZE: int = 32 +BLAKE2S_PERSON_SIZE: int = 8 +BLAKE2S_SALT_SIZE: int = 8 + +@final +class blake2b: + MAX_DIGEST_SIZE: ClassVar[int] = 64 + MAX_KEY_SIZE: ClassVar[int] = 64 + PERSON_SIZE: ClassVar[int] = 16 + SALT_SIZE: ClassVar[int] = 16 + block_size: int + digest_size: int + name: str + if sys.version_info >= (3, 9): + def __init__( + self, + data: ReadableBuffer = b"", + /, + *, + digest_size: int = 64, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + usedforsecurity: bool = True, + ) -> None: ... + else: + def __init__( + self, + data: ReadableBuffer = b"", + /, + *, + digest_size: int = 64, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + ) -> None: ... + + def copy(self) -> Self: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + def update(self, data: ReadableBuffer, /) -> None: ... + +@final +class blake2s: + MAX_DIGEST_SIZE: ClassVar[int] = 32 + MAX_KEY_SIZE: ClassVar[int] = 32 + PERSON_SIZE: ClassVar[int] = 8 + SALT_SIZE: ClassVar[int] = 8 + block_size: int + digest_size: int + name: str + if sys.version_info >= (3, 9): + def __init__( + self, + data: ReadableBuffer = b"", + /, + *, + digest_size: int = 32, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + usedforsecurity: bool = True, + ) -> None: ... + else: + def __init__( + self, + data: ReadableBuffer = b"", + /, + *, + digest_size: int = 32, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + ) -> None: ... + + def copy(self) -> Self: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + def update(self, data: ReadableBuffer, /) -> None: ... diff --git a/mypy/typeshed/stdlib/_bootlocale.pyi b/mypy/typeshed/stdlib/_bootlocale.pyi new file mode 100644 index 000000000000..233d4934f3c6 --- /dev/null +++ b/mypy/typeshed/stdlib/_bootlocale.pyi @@ -0,0 +1 @@ +def getpreferredencoding(do_setlocale: bool = True) -> str: ... diff --git a/mypy/typeshed/stdlib/_bz2.pyi b/mypy/typeshed/stdlib/_bz2.pyi new file mode 100644 index 000000000000..4ba26fe96be0 --- /dev/null +++ b/mypy/typeshed/stdlib/_bz2.pyi @@ -0,0 +1,18 @@ +from _typeshed import ReadableBuffer +from typing import final + +@final +class BZ2Compressor: + def __init__(self, compresslevel: int = 9) -> None: ... + def compress(self, data: ReadableBuffer, /) -> bytes: ... + def flush(self) -> bytes: ... + +@final +class BZ2Decompressor: + def decompress(self, data: ReadableBuffer, max_length: int = -1) -> bytes: ... + @property + def eof(self) -> bool: ... + @property + def needs_input(self) -> bool: ... + @property + def unused_data(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/_codecs.pyi b/mypy/typeshed/stdlib/_codecs.pyi new file mode 100644 index 000000000000..11c5d58a855b --- /dev/null +++ b/mypy/typeshed/stdlib/_codecs.pyi @@ -0,0 +1,136 @@ +import codecs +import sys +from _typeshed import ReadableBuffer +from collections.abc import Callable +from typing import Literal, final, overload, type_check_only +from typing_extensions import TypeAlias + +# This type is not exposed; it is defined in unicodeobject.c +# At runtime it calls itself builtins.EncodingMap +@final +@type_check_only +class _EncodingMap: + def size(self) -> int: ... + +_CharMap: TypeAlias = dict[int, int] | _EncodingMap +_Handler: TypeAlias = Callable[[UnicodeError], tuple[str | bytes, int]] +_SearchFunction: TypeAlias = Callable[[str], codecs.CodecInfo | None] + +def register(search_function: _SearchFunction, /) -> None: ... + +if sys.version_info >= (3, 10): + def unregister(search_function: _SearchFunction, /) -> None: ... + +def register_error(errors: str, handler: _Handler, /) -> None: ... +def lookup_error(name: str, /) -> _Handler: ... + +# The type ignore on `encode` and `decode` is to avoid issues with overlapping overloads, for more details, see #300 +# https://docs.python.org/3/library/codecs.html#binary-transforms +_BytesToBytesEncoding: TypeAlias = Literal[ + "base64", + "base_64", + "base64_codec", + "bz2", + "bz2_codec", + "hex", + "hex_codec", + "quopri", + "quotedprintable", + "quoted_printable", + "quopri_codec", + "uu", + "uu_codec", + "zip", + "zlib", + "zlib_codec", +] +# https://docs.python.org/3/library/codecs.html#text-transforms +_StrToStrEncoding: TypeAlias = Literal["rot13", "rot_13"] + +@overload +def encode(obj: ReadableBuffer, encoding: _BytesToBytesEncoding, errors: str = "strict") -> bytes: ... +@overload +def encode(obj: str, encoding: _StrToStrEncoding, errors: str = "strict") -> str: ... # type: ignore[overload-overlap] +@overload +def encode(obj: str, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... +@overload +def decode(obj: ReadableBuffer, encoding: _BytesToBytesEncoding, errors: str = "strict") -> bytes: ... # type: ignore[overload-overlap] +@overload +def decode(obj: str, encoding: _StrToStrEncoding, errors: str = "strict") -> str: ... + +# these are documented as text encodings but in practice they also accept str as input +@overload +def decode( + obj: str, + encoding: Literal["unicode_escape", "unicode-escape", "raw_unicode_escape", "raw-unicode-escape"], + errors: str = "strict", +) -> str: ... + +# hex is officially documented as a bytes to bytes encoding, but it appears to also work with str +@overload +def decode(obj: str, encoding: Literal["hex", "hex_codec"], errors: str = "strict") -> bytes: ... +@overload +def decode(obj: ReadableBuffer, encoding: str = "utf-8", errors: str = "strict") -> str: ... +def lookup(encoding: str, /) -> codecs.CodecInfo: ... +def charmap_build(map: str, /) -> _CharMap: ... +def ascii_decode(data: ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... +def ascii_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def charmap_decode(data: ReadableBuffer, errors: str | None = None, mapping: _CharMap | None = None, /) -> tuple[str, int]: ... +def charmap_encode(str: str, errors: str | None = None, mapping: _CharMap | None = None, /) -> tuple[bytes, int]: ... +def escape_decode(data: str | ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... +def escape_encode(data: bytes, errors: str | None = None, /) -> tuple[bytes, int]: ... +def latin_1_decode(data: ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... +def latin_1_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +if sys.version_info >= (3, 9): + def raw_unicode_escape_decode( + data: str | ReadableBuffer, errors: str | None = None, final: bool = True, / + ) -> tuple[str, int]: ... + +else: + def raw_unicode_escape_decode(data: str | ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... + +def raw_unicode_escape_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def readbuffer_encode(data: str | ReadableBuffer, errors: str | None = None, /) -> tuple[bytes, int]: ... + +if sys.version_info >= (3, 9): + def unicode_escape_decode( + data: str | ReadableBuffer, errors: str | None = None, final: bool = True, / + ) -> tuple[str, int]: ... + +else: + def unicode_escape_decode(data: str | ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... + +def unicode_escape_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def utf_16_be_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_16_be_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def utf_16_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_16_encode(str: str, errors: str | None = None, byteorder: int = 0, /) -> tuple[bytes, int]: ... +def utf_16_ex_decode( + data: ReadableBuffer, errors: str | None = None, byteorder: int = 0, final: bool = False, / +) -> tuple[str, int, int]: ... +def utf_16_le_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_16_le_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def utf_32_be_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_32_be_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def utf_32_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_32_encode(str: str, errors: str | None = None, byteorder: int = 0, /) -> tuple[bytes, int]: ... +def utf_32_ex_decode( + data: ReadableBuffer, errors: str | None = None, byteorder: int = 0, final: bool = False, / +) -> tuple[str, int, int]: ... +def utf_32_le_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_32_le_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def utf_7_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_7_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... +def utf_8_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... +def utf_8_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +if sys.platform == "win32": + def mbcs_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + def mbcs_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + def code_page_decode( + codepage: int, data: ReadableBuffer, errors: str | None = None, final: bool = False, / + ) -> tuple[str, int]: ... + def code_page_encode(code_page: int, str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + def oem_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + def oem_encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... diff --git a/mypy/typeshed/stdlib/_collections_abc.pyi b/mypy/typeshed/stdlib/_collections_abc.pyi new file mode 100644 index 000000000000..bf7f2991f9a4 --- /dev/null +++ b/mypy/typeshed/stdlib/_collections_abc.pyi @@ -0,0 +1,104 @@ +import sys +from abc import abstractmethod +from types import MappingProxyType +from typing import ( # noqa: Y022,Y038 + AbstractSet as Set, + AsyncGenerator as AsyncGenerator, + AsyncIterable as AsyncIterable, + AsyncIterator as AsyncIterator, + Awaitable as Awaitable, + Callable as Callable, + Collection as Collection, + Container as Container, + Coroutine as Coroutine, + Generator as Generator, + Generic, + Hashable as Hashable, + ItemsView as ItemsView, + Iterable as Iterable, + Iterator as Iterator, + KeysView as KeysView, + Mapping as Mapping, + MappingView as MappingView, + MutableMapping as MutableMapping, + MutableSequence as MutableSequence, + MutableSet as MutableSet, + Protocol, + Reversible as Reversible, + Sequence as Sequence, + Sized as Sized, + TypeVar, + ValuesView as ValuesView, + final, + runtime_checkable, +) + +__all__ = [ + "Awaitable", + "Coroutine", + "AsyncIterable", + "AsyncIterator", + "AsyncGenerator", + "Hashable", + "Iterable", + "Iterator", + "Generator", + "Reversible", + "Sized", + "Container", + "Callable", + "Collection", + "Set", + "MutableSet", + "Mapping", + "MutableMapping", + "MappingView", + "KeysView", + "ItemsView", + "ValuesView", + "Sequence", + "MutableSequence", +] +if sys.version_info < (3, 14): + from typing import ByteString as ByteString # noqa: Y057 + + __all__ += ["ByteString"] + +if sys.version_info >= (3, 12): + __all__ += ["Buffer"] + +_KT_co = TypeVar("_KT_co", covariant=True) # Key type covariant containers. +_VT_co = TypeVar("_VT_co", covariant=True) # Value type covariant containers. + +@final +class dict_keys(KeysView[_KT_co], Generic[_KT_co, _VT_co]): # undocumented + def __eq__(self, value: object, /) -> bool: ... + def __reversed__(self) -> Iterator[_KT_co]: ... + if sys.version_info >= (3, 13): + def isdisjoint(self, other: Iterable[_KT_co], /) -> bool: ... + if sys.version_info >= (3, 10): + @property + def mapping(self) -> MappingProxyType[_KT_co, _VT_co]: ... + +@final +class dict_values(ValuesView[_VT_co], Generic[_KT_co, _VT_co]): # undocumented + def __reversed__(self) -> Iterator[_VT_co]: ... + if sys.version_info >= (3, 10): + @property + def mapping(self) -> MappingProxyType[_KT_co, _VT_co]: ... + +@final +class dict_items(ItemsView[_KT_co, _VT_co]): # undocumented + def __eq__(self, value: object, /) -> bool: ... + def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... + if sys.version_info >= (3, 13): + def isdisjoint(self, other: Iterable[tuple[_KT_co, _VT_co]], /) -> bool: ... + if sys.version_info >= (3, 10): + @property + def mapping(self) -> MappingProxyType[_KT_co, _VT_co]: ... + +if sys.version_info >= (3, 12): + @runtime_checkable + class Buffer(Protocol): + @abstractmethod + def __buffer__(self, flags: int, /) -> memoryview: ... diff --git a/mypy/typeshed/stdlib/_compat_pickle.pyi b/mypy/typeshed/stdlib/_compat_pickle.pyi new file mode 100644 index 000000000000..50fb22442cc9 --- /dev/null +++ b/mypy/typeshed/stdlib/_compat_pickle.pyi @@ -0,0 +1,8 @@ +IMPORT_MAPPING: dict[str, str] +NAME_MAPPING: dict[tuple[str, str], tuple[str, str]] +PYTHON2_EXCEPTIONS: tuple[str, ...] +MULTIPROCESSING_EXCEPTIONS: tuple[str, ...] +REVERSE_IMPORT_MAPPING: dict[str, str] +REVERSE_NAME_MAPPING: dict[tuple[str, str], tuple[str, str]] +PYTHON3_OSERROR_EXCEPTIONS: tuple[str, ...] +PYTHON3_IMPORTERROR_EXCEPTIONS: tuple[str, ...] diff --git a/mypy/typeshed/stdlib/_compression.pyi b/mypy/typeshed/stdlib/_compression.pyi new file mode 100644 index 000000000000..a41a8142cc3a --- /dev/null +++ b/mypy/typeshed/stdlib/_compression.pyi @@ -0,0 +1,25 @@ +from _typeshed import WriteableBuffer +from collections.abc import Callable +from io import DEFAULT_BUFFER_SIZE, BufferedIOBase, RawIOBase +from typing import Any, Protocol + +BUFFER_SIZE = DEFAULT_BUFFER_SIZE + +class _Reader(Protocol): + def read(self, n: int, /) -> bytes: ... + def seekable(self) -> bool: ... + def seek(self, n: int, /) -> Any: ... + +class BaseStream(BufferedIOBase): ... + +class DecompressReader(RawIOBase): + def __init__( + self, + fp: _Reader, + decomp_factory: Callable[..., object], + trailing_error: type[Exception] | tuple[type[Exception], ...] = (), + **decomp_args: Any, + ) -> None: ... + def readinto(self, b: WriteableBuffer) -> int: ... + def read(self, size: int = -1) -> bytes: ... + def seek(self, offset: int, whence: int = 0) -> int: ... diff --git a/mypy/typeshed/stdlib/_contextvars.pyi b/mypy/typeshed/stdlib/_contextvars.pyi new file mode 100644 index 000000000000..2e21a8c5d017 --- /dev/null +++ b/mypy/typeshed/stdlib/_contextvars.pyi @@ -0,0 +1,61 @@ +import sys +from collections.abc import Callable, Iterator, Mapping +from typing import Any, ClassVar, Generic, TypeVar, final, overload +from typing_extensions import ParamSpec + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_D = TypeVar("_D") +_P = ParamSpec("_P") + +@final +class ContextVar(Generic[_T]): + @overload + def __init__(self, name: str) -> None: ... + @overload + def __init__(self, name: str, *, default: _T) -> None: ... + def __hash__(self) -> int: ... + @property + def name(self) -> str: ... + @overload + def get(self) -> _T: ... + @overload + def get(self, default: _T, /) -> _T: ... + @overload + def get(self, default: _D, /) -> _D | _T: ... + def set(self, value: _T, /) -> Token[_T]: ... + def reset(self, token: Token[_T], /) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +@final +class Token(Generic[_T]): + @property + def var(self) -> ContextVar[_T]: ... + @property + def old_value(self) -> Any: ... # returns either _T or MISSING, but that's hard to express + MISSING: ClassVar[object] + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +def copy_context() -> Context: ... + +# It doesn't make sense to make this generic, because for most Contexts each ContextVar will have +# a different value. +@final +class Context(Mapping[ContextVar[Any], Any]): + def __init__(self) -> None: ... + @overload + def get(self, key: ContextVar[_T], default: None = None, /) -> _T | None: ... + @overload + def get(self, key: ContextVar[_T], default: _T, /) -> _T: ... + @overload + def get(self, key: ContextVar[_T], default: _D, /) -> _T | _D: ... + def run(self, callable: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> _T: ... + def copy(self) -> Context: ... + def __getitem__(self, key: ContextVar[_T], /) -> _T: ... + def __iter__(self) -> Iterator[ContextVar[Any]]: ... + def __len__(self) -> int: ... + def __eq__(self, value: object, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi new file mode 100644 index 000000000000..afa2870be158 --- /dev/null +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -0,0 +1,132 @@ +import csv +import sys +from _typeshed import SupportsWrite +from collections.abc import Iterable +from typing import Any, Final, type_check_only +from typing_extensions import Self, TypeAlias + +__version__: Final[str] + +QUOTE_ALL: Final = 1 +QUOTE_MINIMAL: Final = 0 +QUOTE_NONE: Final = 3 +QUOTE_NONNUMERIC: Final = 2 +if sys.version_info >= (3, 12): + QUOTE_STRINGS: Final = 4 + QUOTE_NOTNULL: Final = 5 + +# Ideally this would be `QUOTE_ALL | QUOTE_MINIMAL | QUOTE_NONE | QUOTE_NONNUMERIC` +# However, using literals in situations like these can cause false-positives (see #7258) +_QuotingType: TypeAlias = int + +class Error(Exception): ... + +_DialectLike: TypeAlias = str | Dialect | csv.Dialect | type[Dialect | csv.Dialect] + +class Dialect: + delimiter: str + quotechar: str | None + escapechar: str | None + doublequote: bool + skipinitialspace: bool + lineterminator: str + quoting: _QuotingType + strict: bool + def __init__( + self, + dialect: _DialectLike | None = ..., + delimiter: str = ",", + doublequote: bool = True, + escapechar: str | None = None, + lineterminator: str = "\r\n", + quotechar: str | None = '"', + quoting: _QuotingType = 0, + skipinitialspace: bool = False, + strict: bool = False, + ) -> None: ... + +if sys.version_info >= (3, 10): + # This class calls itself _csv.reader. + class Reader: + @property + def dialect(self) -> Dialect: ... + line_num: int + def __iter__(self) -> Self: ... + def __next__(self) -> list[str]: ... + + # This class calls itself _csv.writer. + class Writer: + @property + def dialect(self) -> Dialect: ... + if sys.version_info >= (3, 13): + def writerow(self, row: Iterable[Any], /) -> Any: ... + def writerows(self, rows: Iterable[Iterable[Any]], /) -> None: ... + else: + def writerow(self, row: Iterable[Any]) -> Any: ... + def writerows(self, rows: Iterable[Iterable[Any]]) -> None: ... + + # For the return types below. + # These aliases can be removed when typeshed drops support for 3.9. + _reader = Reader + _writer = Writer +else: + # This class is not exposed. It calls itself _csv.reader. + @type_check_only + class _reader: + @property + def dialect(self) -> Dialect: ... + line_num: int + def __iter__(self) -> Self: ... + def __next__(self) -> list[str]: ... + + # This class is not exposed. It calls itself _csv.writer. + @type_check_only + class _writer: + @property + def dialect(self) -> Dialect: ... + def writerow(self, row: Iterable[Any]) -> Any: ... + def writerows(self, rows: Iterable[Iterable[Any]]) -> None: ... + +def writer( + csvfile: SupportsWrite[str], + dialect: _DialectLike = "excel", + *, + delimiter: str = ",", + quotechar: str | None = '"', + escapechar: str | None = None, + doublequote: bool = True, + skipinitialspace: bool = False, + lineterminator: str = "\r\n", + quoting: _QuotingType = 0, + strict: bool = False, +) -> _writer: ... +def reader( + csvfile: Iterable[str], + dialect: _DialectLike = "excel", + *, + delimiter: str = ",", + quotechar: str | None = '"', + escapechar: str | None = None, + doublequote: bool = True, + skipinitialspace: bool = False, + lineterminator: str = "\r\n", + quoting: _QuotingType = 0, + strict: bool = False, +) -> _reader: ... +def register_dialect( + name: str, + dialect: type[Dialect] = ..., + *, + delimiter: str = ",", + quotechar: str | None = '"', + escapechar: str | None = None, + doublequote: bool = True, + skipinitialspace: bool = False, + lineterminator: str = "\r\n", + quoting: _QuotingType = 0, + strict: bool = False, +) -> None: ... +def unregister_dialect(name: str) -> None: ... +def get_dialect(name: str) -> Dialect: ... +def list_dialects() -> list[str]: ... +def field_size_limit(new_limit: int = ...) -> int: ... diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi new file mode 100644 index 000000000000..ecb07a29bb75 --- /dev/null +++ b/mypy/typeshed/stdlib/_ctypes.pyi @@ -0,0 +1,330 @@ +import _typeshed +import sys +from _typeshed import ReadableBuffer, StrOrBytesPath, WriteableBuffer +from abc import abstractmethod +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from ctypes import CDLL, ArgumentError as ArgumentError, c_void_p +from typing import Any, ClassVar, Generic, TypeVar, final, overload, type_check_only +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_CT = TypeVar("_CT", bound=_CData) + +FUNCFLAG_CDECL: int +FUNCFLAG_PYTHONAPI: int +FUNCFLAG_USE_ERRNO: int +FUNCFLAG_USE_LASTERROR: int +RTLD_GLOBAL: int +RTLD_LOCAL: int + +if sys.version_info >= (3, 11): + CTYPES_MAX_ARGCOUNT: int + +if sys.version_info >= (3, 12): + SIZEOF_TIME_T: int + +if sys.platform == "win32": + # Description, Source, HelpFile, HelpContext, scode + _COMError_Details: TypeAlias = tuple[str | None, str | None, str | None, int | None, int | None] + + class COMError(Exception): + hresult: int + text: str | None + details: _COMError_Details + + def __init__(self, hresult: int, text: str | None, details: _COMError_Details) -> None: ... + + def CopyComPointer(src: _PointerLike, dst: _PointerLike | _CArgObject) -> int: ... + + FUNCFLAG_HRESULT: int + FUNCFLAG_STDCALL: int + + def FormatError(code: int = ...) -> str: ... + def get_last_error() -> int: ... + def set_last_error(value: int) -> int: ... + def LoadLibrary(name: str, load_flags: int = 0, /) -> int: ... + def FreeLibrary(handle: int, /) -> None: ... + +else: + def dlclose(handle: int, /) -> None: ... + # The default for flag is RTLD_GLOBAL|RTLD_LOCAL, which is platform dependent. + def dlopen(name: StrOrBytesPath, flag: int = ..., /) -> int: ... + def dlsym(handle: int, name: str, /) -> int: ... + +if sys.version_info >= (3, 13): + # This class is not exposed. It calls itself _ctypes.CType_Type. + @type_check_only + class _CType_Type(type): + # By default mypy complains about the following two methods, because strictly speaking cls + # might not be a Type[_CT]. However this doesn't happen because this is only a + # metaclass for subclasses of _CData. + def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + + _CTypeBaseType = _CType_Type + +else: + _CTypeBaseType = type + +# This class is not exposed. +@type_check_only +class _CData: + _b_base_: int + _b_needsfree_: bool + _objects: Mapping[Any, int] | None + def __buffer__(self, flags: int, /) -> memoryview: ... + def __ctypes_from_outparam__(self, /) -> Self: ... + +# this is a union of all the subclasses of _CData, which is useful because of +# the methods that are present on each of those subclasses which are not present +# on _CData itself. +_CDataType: TypeAlias = _SimpleCData[Any] | _Pointer[Any] | CFuncPtr | Union | Structure | Array[Any] + +# This class is not exposed. It calls itself _ctypes.PyCSimpleType. +@type_check_only +class _PyCSimpleType(_CTypeBaseType): + def from_address(self: type[_typeshed.Self], value: int, /) -> _typeshed.Self: ... + def from_buffer(self: type[_typeshed.Self], obj: WriteableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_buffer_copy(self: type[_typeshed.Self], buffer: ReadableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_param(self: type[_typeshed.Self], value: Any, /) -> _typeshed.Self | _CArgObject: ... + def in_dll(self: type[_typeshed.Self], dll: CDLL, name: str, /) -> _typeshed.Self: ... + if sys.version_info < (3, 13): + # Inherited from CType_Type starting on 3.13 + def __mul__(self: type[_CT], value: int, /) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(self: type[_CT], value: int, /) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + +class _SimpleCData(_CData, Generic[_T], metaclass=_PyCSimpleType): + value: _T + # The TypeVar can be unsolved here, + # but we can't use overloads without creating many, many mypy false-positive errors + def __init__(self, value: _T = ...) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] + def __ctypes_from_outparam__(self, /) -> _T: ... # type: ignore[override] + +class _CanCastTo(_CData): ... +class _PointerLike(_CanCastTo): ... + +# This type is not exposed. It calls itself _ctypes.PyCPointerType. +@type_check_only +class _PyCPointerType(_CTypeBaseType): + def from_address(self: type[_typeshed.Self], value: int, /) -> _typeshed.Self: ... + def from_buffer(self: type[_typeshed.Self], obj: WriteableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_buffer_copy(self: type[_typeshed.Self], buffer: ReadableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_param(self: type[_typeshed.Self], value: Any, /) -> _typeshed.Self | _CArgObject: ... + def in_dll(self: type[_typeshed.Self], dll: CDLL, name: str, /) -> _typeshed.Self: ... + def set_type(self, type: Any, /) -> None: ... + if sys.version_info < (3, 13): + # Inherited from CType_Type starting on 3.13 + def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + +class _Pointer(_PointerLike, _CData, Generic[_CT], metaclass=_PyCPointerType): + _type_: type[_CT] + contents: _CT + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, arg: _CT) -> None: ... + @overload + def __getitem__(self, key: int, /) -> Any: ... + @overload + def __getitem__(self, key: slice, /) -> list[Any]: ... + def __setitem__(self, key: int, value: Any, /) -> None: ... + +@overload +def POINTER(type: None, /) -> type[c_void_p]: ... +@overload +def POINTER(type: type[_CT], /) -> type[_Pointer[_CT]]: ... +def pointer(obj: _CT, /) -> _Pointer[_CT]: ... + +# This class is not exposed. It calls itself _ctypes.CArgObject. +@final +@type_check_only +class _CArgObject: ... + +def byref(obj: _CData | _CDataType, offset: int = ...) -> _CArgObject: ... + +_ECT: TypeAlias = Callable[[_CData | _CDataType | None, CFuncPtr, tuple[_CData | _CDataType, ...]], _CDataType] +_PF: TypeAlias = tuple[int] | tuple[int, str | None] | tuple[int, str | None, Any] + +# This class is not exposed. It calls itself _ctypes.PyCFuncPtrType. +@type_check_only +class _PyCFuncPtrType(_CTypeBaseType): + def from_address(self: type[_typeshed.Self], value: int, /) -> _typeshed.Self: ... + def from_buffer(self: type[_typeshed.Self], obj: WriteableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_buffer_copy(self: type[_typeshed.Self], buffer: ReadableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_param(self: type[_typeshed.Self], value: Any, /) -> _typeshed.Self | _CArgObject: ... + def in_dll(self: type[_typeshed.Self], dll: CDLL, name: str, /) -> _typeshed.Self: ... + if sys.version_info < (3, 13): + # Inherited from CType_Type starting on 3.13 + def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + +class CFuncPtr(_PointerLike, _CData, metaclass=_PyCFuncPtrType): + restype: type[_CDataType] | Callable[[int], Any] | None + argtypes: Sequence[type[_CDataType]] + errcheck: _ECT + # Abstract attribute that must be defined on subclasses + _flags_: ClassVar[int] + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, address: int, /) -> None: ... + @overload + def __init__(self, callable: Callable[..., Any], /) -> None: ... + @overload + def __init__(self, func_spec: tuple[str | int, CDLL], paramflags: tuple[_PF, ...] | None = ..., /) -> None: ... + if sys.platform == "win32": + @overload + def __init__( + self, vtbl_index: int, name: str, paramflags: tuple[_PF, ...] | None = ..., iid: _CData | _CDataType | None = ..., / + ) -> None: ... + + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + +_GetT = TypeVar("_GetT") +_SetT = TypeVar("_SetT") + +# This class is not exposed. It calls itself _ctypes.CField. +@final +@type_check_only +class _CField(Generic[_CT, _GetT, _SetT]): + offset: int + size: int + if sys.version_info >= (3, 10): + @overload + def __get__(self, instance: None, owner: type[Any] | None = None, /) -> Self: ... + @overload + def __get__(self, instance: Any, owner: type[Any] | None = None, /) -> _GetT: ... + else: + @overload + def __get__(self, instance: None, owner: type[Any] | None, /) -> Self: ... + @overload + def __get__(self, instance: Any, owner: type[Any] | None, /) -> _GetT: ... + + def __set__(self, instance: Any, value: _SetT, /) -> None: ... + +# This class is not exposed. It calls itself _ctypes.UnionType. +@type_check_only +class _UnionType(_CTypeBaseType): + def from_address(self: type[_typeshed.Self], value: int, /) -> _typeshed.Self: ... + def from_buffer(self: type[_typeshed.Self], obj: WriteableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_buffer_copy(self: type[_typeshed.Self], buffer: ReadableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_param(self: type[_typeshed.Self], value: Any, /) -> _typeshed.Self | _CArgObject: ... + def in_dll(self: type[_typeshed.Self], dll: CDLL, name: str, /) -> _typeshed.Self: ... + # At runtime, various attributes are created on a Union subclass based + # on its _fields_. This method doesn't exist, but represents those + # dynamically created attributes. + def __getattr__(self, name: str) -> _CField[Any, Any, Any]: ... + if sys.version_info < (3, 13): + # Inherited from CType_Type starting on 3.13 + def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + +class Union(_CData, metaclass=_UnionType): + _fields_: ClassVar[Sequence[tuple[str, type[_CDataType]] | tuple[str, type[_CDataType], int]]] + _pack_: ClassVar[int] + _anonymous_: ClassVar[Sequence[str]] + if sys.version_info >= (3, 13): + _align_: ClassVar[int] + + def __init__(self, *args: Any, **kw: Any) -> None: ... + def __getattr__(self, name: str) -> Any: ... + def __setattr__(self, name: str, value: Any) -> None: ... + +# This class is not exposed. It calls itself _ctypes.PyCStructType. +@type_check_only +class _PyCStructType(_CTypeBaseType): + def from_address(self: type[_typeshed.Self], value: int, /) -> _typeshed.Self: ... + def from_buffer(self: type[_typeshed.Self], obj: WriteableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_buffer_copy(self: type[_typeshed.Self], buffer: ReadableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_param(self: type[_typeshed.Self], value: Any, /) -> _typeshed.Self | _CArgObject: ... + def in_dll(self: type[_typeshed.Self], dll: CDLL, name: str, /) -> _typeshed.Self: ... + # At runtime, various attributes are created on a Structure subclass based + # on its _fields_. This method doesn't exist, but represents those + # dynamically created attributes. + def __getattr__(self, name: str) -> _CField[Any, Any, Any]: ... + if sys.version_info < (3, 13): + # Inherited from CType_Type starting on 3.13 + def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + +class Structure(_CData, metaclass=_PyCStructType): + _fields_: ClassVar[Sequence[tuple[str, type[_CDataType]] | tuple[str, type[_CDataType], int]]] + _pack_: ClassVar[int] + _anonymous_: ClassVar[Sequence[str]] + if sys.version_info >= (3, 13): + _align_: ClassVar[int] + + def __init__(self, *args: Any, **kw: Any) -> None: ... + def __getattr__(self, name: str) -> Any: ... + def __setattr__(self, name: str, value: Any) -> None: ... + +# This class is not exposed. It calls itself _ctypes.PyCArrayType. +@type_check_only +class _PyCArrayType(_CTypeBaseType): + def from_address(self: type[_typeshed.Self], value: int, /) -> _typeshed.Self: ... + def from_buffer(self: type[_typeshed.Self], obj: WriteableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_buffer_copy(self: type[_typeshed.Self], buffer: ReadableBuffer, offset: int = 0, /) -> _typeshed.Self: ... + def from_param(self: type[_typeshed.Self], value: Any, /) -> _typeshed.Self | _CArgObject: ... + def in_dll(self: type[_typeshed.Self], dll: CDLL, name: str, /) -> _typeshed.Self: ... + if sys.version_info < (3, 13): + # Inherited from CType_Type starting on 3.13 + def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + +class Array(_CData, Generic[_CT], metaclass=_PyCArrayType): + @property + @abstractmethod + def _length_(self) -> int: ... + @_length_.setter + def _length_(self, value: int) -> None: ... + @property + @abstractmethod + def _type_(self) -> type[_CT]: ... + @_type_.setter + def _type_(self, value: type[_CT]) -> None: ... + raw: bytes # Note: only available if _CT == c_char + value: Any # Note: bytes if _CT == c_char, str if _CT == c_wchar, unavailable otherwise + # TODO These methods cannot be annotated correctly at the moment. + # All of these "Any"s stand for the array's element type, but it's not possible to use _CT + # here, because of a special feature of ctypes. + # By default, when accessing an element of an Array[_CT], the returned object has type _CT. + # However, when _CT is a "simple type" like c_int, ctypes automatically "unboxes" the object + # and converts it to the corresponding Python primitive. For example, when accessing an element + # of an Array[c_int], a Python int object is returned, not a c_int. + # This behavior does *not* apply to subclasses of "simple types". + # If MyInt is a subclass of c_int, then accessing an element of an Array[MyInt] returns + # a MyInt, not an int. + # This special behavior is not easy to model in a stub, so for now all places where + # the array element type would belong are annotated with Any instead. + def __init__(self, *args: Any) -> None: ... + @overload + def __getitem__(self, key: int, /) -> Any: ... + @overload + def __getitem__(self, key: slice, /) -> list[Any]: ... + @overload + def __setitem__(self, key: int, value: Any, /) -> None: ... + @overload + def __setitem__(self, key: slice, value: Iterable[Any], /) -> None: ... + def __iter__(self) -> Iterator[Any]: ... + # Can't inherit from Sized because the metaclass conflict between + # Sized and _CData prevents using _CDataMeta. + def __len__(self) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +def addressof(obj: _CData | _CDataType, /) -> int: ... +def alignment(obj_or_type: _CData | _CDataType | type[_CData | _CDataType], /) -> int: ... +def get_errno() -> int: ... +def resize(obj: _CData | _CDataType, size: int, /) -> None: ... +def set_errno(value: int, /) -> int: ... +def sizeof(obj_or_type: _CData | _CDataType | type[_CData | _CDataType], /) -> int: ... +def PyObj_FromPtr(address: int, /) -> Any: ... +def Py_DECREF(o: _T, /) -> _T: ... +def Py_INCREF(o: _T, /) -> _T: ... +def buffer_info(o: _CData | _CDataType | type[_CData | _CDataType], /) -> tuple[str, int, tuple[int, ...]]: ... +def call_cdeclfunction(address: int, arguments: tuple[Any, ...], /) -> Any: ... +def call_function(address: int, arguments: tuple[Any, ...], /) -> Any: ... diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi new file mode 100644 index 000000000000..9e06a1414da5 --- /dev/null +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -0,0 +1,553 @@ +import sys +from _typeshed import ReadOnlyBuffer, SupportsRead +from curses import _ncurses_version +from typing import IO, Any, final, overload +from typing_extensions import TypeAlias + +# NOTE: This module is ordinarily only available on Unix, but the windows-curses +# package makes it available on Windows as well with the same contents. + +# Handled by PyCurses_ConvertToChtype in _cursesmodule.c. +_ChType: TypeAlias = str | bytes | int + +# ACS codes are only initialized after initscr is called +ACS_BBSS: int +ACS_BLOCK: int +ACS_BOARD: int +ACS_BSBS: int +ACS_BSSB: int +ACS_BSSS: int +ACS_BTEE: int +ACS_BULLET: int +ACS_CKBOARD: int +ACS_DARROW: int +ACS_DEGREE: int +ACS_DIAMOND: int +ACS_GEQUAL: int +ACS_HLINE: int +ACS_LANTERN: int +ACS_LARROW: int +ACS_LEQUAL: int +ACS_LLCORNER: int +ACS_LRCORNER: int +ACS_LTEE: int +ACS_NEQUAL: int +ACS_PI: int +ACS_PLMINUS: int +ACS_PLUS: int +ACS_RARROW: int +ACS_RTEE: int +ACS_S1: int +ACS_S3: int +ACS_S7: int +ACS_S9: int +ACS_SBBS: int +ACS_SBSB: int +ACS_SBSS: int +ACS_SSBB: int +ACS_SSBS: int +ACS_SSSB: int +ACS_SSSS: int +ACS_STERLING: int +ACS_TTEE: int +ACS_UARROW: int +ACS_ULCORNER: int +ACS_URCORNER: int +ACS_VLINE: int +ALL_MOUSE_EVENTS: int +A_ALTCHARSET: int +A_ATTRIBUTES: int +A_BLINK: int +A_BOLD: int +A_CHARTEXT: int +A_COLOR: int +A_DIM: int +A_HORIZONTAL: int +A_INVIS: int +A_ITALIC: int +A_LEFT: int +A_LOW: int +A_NORMAL: int +A_PROTECT: int +A_REVERSE: int +A_RIGHT: int +A_STANDOUT: int +A_TOP: int +A_UNDERLINE: int +A_VERTICAL: int +BUTTON1_CLICKED: int +BUTTON1_DOUBLE_CLICKED: int +BUTTON1_PRESSED: int +BUTTON1_RELEASED: int +BUTTON1_TRIPLE_CLICKED: int +BUTTON2_CLICKED: int +BUTTON2_DOUBLE_CLICKED: int +BUTTON2_PRESSED: int +BUTTON2_RELEASED: int +BUTTON2_TRIPLE_CLICKED: int +BUTTON3_CLICKED: int +BUTTON3_DOUBLE_CLICKED: int +BUTTON3_PRESSED: int +BUTTON3_RELEASED: int +BUTTON3_TRIPLE_CLICKED: int +BUTTON4_CLICKED: int +BUTTON4_DOUBLE_CLICKED: int +BUTTON4_PRESSED: int +BUTTON4_RELEASED: int +BUTTON4_TRIPLE_CLICKED: int +# Darwin ncurses doesn't provide BUTTON5_* constants +if sys.version_info >= (3, 10) and sys.platform != "darwin": + BUTTON5_PRESSED: int + BUTTON5_RELEASED: int + BUTTON5_CLICKED: int + BUTTON5_DOUBLE_CLICKED: int + BUTTON5_TRIPLE_CLICKED: int +BUTTON_ALT: int +BUTTON_CTRL: int +BUTTON_SHIFT: int +COLOR_BLACK: int +COLOR_BLUE: int +COLOR_CYAN: int +COLOR_GREEN: int +COLOR_MAGENTA: int +COLOR_RED: int +COLOR_WHITE: int +COLOR_YELLOW: int +ERR: int +KEY_A1: int +KEY_A3: int +KEY_B2: int +KEY_BACKSPACE: int +KEY_BEG: int +KEY_BREAK: int +KEY_BTAB: int +KEY_C1: int +KEY_C3: int +KEY_CANCEL: int +KEY_CATAB: int +KEY_CLEAR: int +KEY_CLOSE: int +KEY_COMMAND: int +KEY_COPY: int +KEY_CREATE: int +KEY_CTAB: int +KEY_DC: int +KEY_DL: int +KEY_DOWN: int +KEY_EIC: int +KEY_END: int +KEY_ENTER: int +KEY_EOL: int +KEY_EOS: int +KEY_EXIT: int +KEY_F0: int +KEY_F1: int +KEY_F10: int +KEY_F11: int +KEY_F12: int +KEY_F13: int +KEY_F14: int +KEY_F15: int +KEY_F16: int +KEY_F17: int +KEY_F18: int +KEY_F19: int +KEY_F2: int +KEY_F20: int +KEY_F21: int +KEY_F22: int +KEY_F23: int +KEY_F24: int +KEY_F25: int +KEY_F26: int +KEY_F27: int +KEY_F28: int +KEY_F29: int +KEY_F3: int +KEY_F30: int +KEY_F31: int +KEY_F32: int +KEY_F33: int +KEY_F34: int +KEY_F35: int +KEY_F36: int +KEY_F37: int +KEY_F38: int +KEY_F39: int +KEY_F4: int +KEY_F40: int +KEY_F41: int +KEY_F42: int +KEY_F43: int +KEY_F44: int +KEY_F45: int +KEY_F46: int +KEY_F47: int +KEY_F48: int +KEY_F49: int +KEY_F5: int +KEY_F50: int +KEY_F51: int +KEY_F52: int +KEY_F53: int +KEY_F54: int +KEY_F55: int +KEY_F56: int +KEY_F57: int +KEY_F58: int +KEY_F59: int +KEY_F6: int +KEY_F60: int +KEY_F61: int +KEY_F62: int +KEY_F63: int +KEY_F7: int +KEY_F8: int +KEY_F9: int +KEY_FIND: int +KEY_HELP: int +KEY_HOME: int +KEY_IC: int +KEY_IL: int +KEY_LEFT: int +KEY_LL: int +KEY_MARK: int +KEY_MAX: int +KEY_MESSAGE: int +KEY_MIN: int +KEY_MOUSE: int +KEY_MOVE: int +KEY_NEXT: int +KEY_NPAGE: int +KEY_OPEN: int +KEY_OPTIONS: int +KEY_PPAGE: int +KEY_PREVIOUS: int +KEY_PRINT: int +KEY_REDO: int +KEY_REFERENCE: int +KEY_REFRESH: int +KEY_REPLACE: int +KEY_RESET: int +KEY_RESIZE: int +KEY_RESTART: int +KEY_RESUME: int +KEY_RIGHT: int +KEY_SAVE: int +KEY_SBEG: int +KEY_SCANCEL: int +KEY_SCOMMAND: int +KEY_SCOPY: int +KEY_SCREATE: int +KEY_SDC: int +KEY_SDL: int +KEY_SELECT: int +KEY_SEND: int +KEY_SEOL: int +KEY_SEXIT: int +KEY_SF: int +KEY_SFIND: int +KEY_SHELP: int +KEY_SHOME: int +KEY_SIC: int +KEY_SLEFT: int +KEY_SMESSAGE: int +KEY_SMOVE: int +KEY_SNEXT: int +KEY_SOPTIONS: int +KEY_SPREVIOUS: int +KEY_SPRINT: int +KEY_SR: int +KEY_SREDO: int +KEY_SREPLACE: int +KEY_SRESET: int +KEY_SRIGHT: int +KEY_SRSUME: int +KEY_SSAVE: int +KEY_SSUSPEND: int +KEY_STAB: int +KEY_SUNDO: int +KEY_SUSPEND: int +KEY_UNDO: int +KEY_UP: int +OK: int +REPORT_MOUSE_POSITION: int +_C_API: Any +version: bytes + +def baudrate() -> int: ... +def beep() -> None: ... +def can_change_color() -> bool: ... +def cbreak(flag: bool = True, /) -> None: ... +def color_content(color_number: int, /) -> tuple[int, int, int]: ... +def color_pair(pair_number: int, /) -> int: ... +def curs_set(visibility: int, /) -> int: ... +def def_prog_mode() -> None: ... +def def_shell_mode() -> None: ... +def delay_output(ms: int, /) -> None: ... +def doupdate() -> None: ... +def echo(flag: bool = True, /) -> None: ... +def endwin() -> None: ... +def erasechar() -> bytes: ... +def filter() -> None: ... +def flash() -> None: ... +def flushinp() -> None: ... + +if sys.version_info >= (3, 9): + def get_escdelay() -> int: ... + def get_tabsize() -> int: ... + +def getmouse() -> tuple[int, int, int, int, int]: ... +def getsyx() -> tuple[int, int]: ... +def getwin(file: SupportsRead[bytes], /) -> window: ... +def halfdelay(tenths: int, /) -> None: ... +def has_colors() -> bool: ... + +if sys.version_info >= (3, 10): + def has_extended_color_support() -> bool: ... + +def has_ic() -> bool: ... +def has_il() -> bool: ... +def has_key(key: int, /) -> bool: ... +def init_color(color_number: int, r: int, g: int, b: int, /) -> None: ... +def init_pair(pair_number: int, fg: int, bg: int, /) -> None: ... +def initscr() -> window: ... +def intrflush(flag: bool, /) -> None: ... +def is_term_resized(nlines: int, ncols: int, /) -> bool: ... +def isendwin() -> bool: ... +def keyname(key: int, /) -> bytes: ... +def killchar() -> bytes: ... +def longname() -> bytes: ... +def meta(yes: bool, /) -> None: ... +def mouseinterval(interval: int, /) -> None: ... +def mousemask(newmask: int, /) -> tuple[int, int]: ... +def napms(ms: int, /) -> int: ... +def newpad(nlines: int, ncols: int, /) -> window: ... +def newwin(nlines: int, ncols: int, begin_y: int = ..., begin_x: int = ..., /) -> window: ... +def nl(flag: bool = True, /) -> None: ... +def nocbreak() -> None: ... +def noecho() -> None: ... +def nonl() -> None: ... +def noqiflush() -> None: ... +def noraw() -> None: ... +def pair_content(pair_number: int, /) -> tuple[int, int]: ... +def pair_number(attr: int, /) -> int: ... +def putp(string: ReadOnlyBuffer, /) -> None: ... +def qiflush(flag: bool = True, /) -> None: ... +def raw(flag: bool = True, /) -> None: ... +def reset_prog_mode() -> None: ... +def reset_shell_mode() -> None: ... +def resetty() -> None: ... +def resize_term(nlines: int, ncols: int, /) -> None: ... +def resizeterm(nlines: int, ncols: int, /) -> None: ... +def savetty() -> None: ... + +if sys.version_info >= (3, 9): + def set_escdelay(ms: int, /) -> None: ... + def set_tabsize(size: int, /) -> None: ... + +def setsyx(y: int, x: int, /) -> None: ... +def setupterm(term: str | None = None, fd: int = -1) -> None: ... +def start_color() -> None: ... +def termattrs() -> int: ... +def termname() -> bytes: ... +def tigetflag(capname: str, /) -> int: ... +def tigetnum(capname: str, /) -> int: ... +def tigetstr(capname: str, /) -> bytes | None: ... +def tparm( + str: ReadOnlyBuffer, + i1: int = 0, + i2: int = 0, + i3: int = 0, + i4: int = 0, + i5: int = 0, + i6: int = 0, + i7: int = 0, + i8: int = 0, + i9: int = 0, + /, +) -> bytes: ... +def typeahead(fd: int, /) -> None: ... +def unctrl(ch: _ChType, /) -> bytes: ... +def unget_wch(ch: int | str, /) -> None: ... +def ungetch(ch: _ChType, /) -> None: ... +def ungetmouse(id: int, x: int, y: int, z: int, bstate: int, /) -> None: ... +def update_lines_cols() -> None: ... +def use_default_colors() -> None: ... +def use_env(flag: bool, /) -> None: ... + +class error(Exception): ... + +@final +class window: # undocumented + encoding: str + @overload + def addch(self, ch: _ChType, attr: int = ...) -> None: ... + @overload + def addch(self, y: int, x: int, ch: _ChType, attr: int = ...) -> None: ... + @overload + def addnstr(self, str: str, n: int, attr: int = ...) -> None: ... + @overload + def addnstr(self, y: int, x: int, str: str, n: int, attr: int = ...) -> None: ... + @overload + def addstr(self, str: str, attr: int = ...) -> None: ... + @overload + def addstr(self, y: int, x: int, str: str, attr: int = ...) -> None: ... + def attroff(self, attr: int, /) -> None: ... + def attron(self, attr: int, /) -> None: ... + def attrset(self, attr: int, /) -> None: ... + def bkgd(self, ch: _ChType, attr: int = ..., /) -> None: ... + def bkgdset(self, ch: _ChType, attr: int = ..., /) -> None: ... + def border( + self, + ls: _ChType = ..., + rs: _ChType = ..., + ts: _ChType = ..., + bs: _ChType = ..., + tl: _ChType = ..., + tr: _ChType = ..., + bl: _ChType = ..., + br: _ChType = ..., + ) -> None: ... + @overload + def box(self) -> None: ... + @overload + def box(self, vertch: _ChType = ..., horch: _ChType = ...) -> None: ... + @overload + def chgat(self, attr: int) -> None: ... + @overload + def chgat(self, num: int, attr: int) -> None: ... + @overload + def chgat(self, y: int, x: int, attr: int) -> None: ... + @overload + def chgat(self, y: int, x: int, num: int, attr: int) -> None: ... + def clear(self) -> None: ... + def clearok(self, yes: int) -> None: ... + def clrtobot(self) -> None: ... + def clrtoeol(self) -> None: ... + def cursyncup(self) -> None: ... + @overload + def delch(self) -> None: ... + @overload + def delch(self, y: int, x: int) -> None: ... + def deleteln(self) -> None: ... + @overload + def derwin(self, begin_y: int, begin_x: int) -> window: ... + @overload + def derwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... + def echochar(self, ch: _ChType, attr: int = ..., /) -> None: ... + def enclose(self, y: int, x: int, /) -> bool: ... + def erase(self) -> None: ... + def getbegyx(self) -> tuple[int, int]: ... + def getbkgd(self) -> tuple[int, int]: ... + @overload + def getch(self) -> int: ... + @overload + def getch(self, y: int, x: int) -> int: ... + @overload + def get_wch(self) -> int | str: ... + @overload + def get_wch(self, y: int, x: int) -> int | str: ... + @overload + def getkey(self) -> str: ... + @overload + def getkey(self, y: int, x: int) -> str: ... + def getmaxyx(self) -> tuple[int, int]: ... + def getparyx(self) -> tuple[int, int]: ... + @overload + def getstr(self) -> bytes: ... + @overload + def getstr(self, n: int) -> bytes: ... + @overload + def getstr(self, y: int, x: int) -> bytes: ... + @overload + def getstr(self, y: int, x: int, n: int) -> bytes: ... + def getyx(self) -> tuple[int, int]: ... + @overload + def hline(self, ch: _ChType, n: int) -> None: ... + @overload + def hline(self, y: int, x: int, ch: _ChType, n: int) -> None: ... + def idcok(self, flag: bool) -> None: ... + def idlok(self, yes: bool) -> None: ... + def immedok(self, flag: bool) -> None: ... + @overload + def inch(self) -> int: ... + @overload + def inch(self, y: int, x: int) -> int: ... + @overload + def insch(self, ch: _ChType, attr: int = ...) -> None: ... + @overload + def insch(self, y: int, x: int, ch: _ChType, attr: int = ...) -> None: ... + def insdelln(self, nlines: int) -> None: ... + def insertln(self) -> None: ... + @overload + def insnstr(self, str: str, n: int, attr: int = ...) -> None: ... + @overload + def insnstr(self, y: int, x: int, str: str, n: int, attr: int = ...) -> None: ... + @overload + def insstr(self, str: str, attr: int = ...) -> None: ... + @overload + def insstr(self, y: int, x: int, str: str, attr: int = ...) -> None: ... + @overload + def instr(self, n: int = ...) -> bytes: ... + @overload + def instr(self, y: int, x: int, n: int = ...) -> bytes: ... + def is_linetouched(self, line: int, /) -> bool: ... + def is_wintouched(self) -> bool: ... + def keypad(self, yes: bool, /) -> None: ... + def leaveok(self, yes: bool) -> None: ... + def move(self, new_y: int, new_x: int) -> None: ... + def mvderwin(self, y: int, x: int) -> None: ... + def mvwin(self, new_y: int, new_x: int) -> None: ... + def nodelay(self, yes: bool) -> None: ... + def notimeout(self, yes: bool) -> None: ... + @overload + def noutrefresh(self) -> None: ... + @overload + def noutrefresh(self, pminrow: int, pmincol: int, sminrow: int, smincol: int, smaxrow: int, smaxcol: int) -> None: ... + @overload + def overlay(self, destwin: window) -> None: ... + @overload + def overlay( + self, destwin: window, sminrow: int, smincol: int, dminrow: int, dmincol: int, dmaxrow: int, dmaxcol: int + ) -> None: ... + @overload + def overwrite(self, destwin: window) -> None: ... + @overload + def overwrite( + self, destwin: window, sminrow: int, smincol: int, dminrow: int, dmincol: int, dmaxrow: int, dmaxcol: int + ) -> None: ... + def putwin(self, file: IO[Any], /) -> None: ... + def redrawln(self, beg: int, num: int, /) -> None: ... + def redrawwin(self) -> None: ... + @overload + def refresh(self) -> None: ... + @overload + def refresh(self, pminrow: int, pmincol: int, sminrow: int, smincol: int, smaxrow: int, smaxcol: int) -> None: ... + def resize(self, nlines: int, ncols: int) -> None: ... + def scroll(self, lines: int = ...) -> None: ... + def scrollok(self, flag: bool) -> None: ... + def setscrreg(self, top: int, bottom: int, /) -> None: ... + def standend(self) -> None: ... + def standout(self) -> None: ... + @overload + def subpad(self, begin_y: int, begin_x: int) -> window: ... + @overload + def subpad(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... + @overload + def subwin(self, begin_y: int, begin_x: int) -> window: ... + @overload + def subwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... + def syncdown(self) -> None: ... + def syncok(self, flag: bool) -> None: ... + def syncup(self) -> None: ... + def timeout(self, delay: int) -> None: ... + def touchline(self, start: int, count: int, changed: bool = ...) -> None: ... + def touchwin(self) -> None: ... + def untouchwin(self) -> None: ... + @overload + def vline(self, ch: _ChType, n: int) -> None: ... + @overload + def vline(self, y: int, x: int, ch: _ChType, n: int) -> None: ... + +ncurses_version: _ncurses_version diff --git a/mypy/typeshed/stdlib/_curses_panel.pyi b/mypy/typeshed/stdlib/_curses_panel.pyi new file mode 100644 index 000000000000..ddec22236b96 --- /dev/null +++ b/mypy/typeshed/stdlib/_curses_panel.pyi @@ -0,0 +1,27 @@ +from _curses import window +from typing import final + +__version__: str +version: str + +class error(Exception): ... + +@final +class panel: + def above(self) -> panel: ... + def below(self) -> panel: ... + def bottom(self) -> None: ... + def hidden(self) -> bool: ... + def hide(self) -> None: ... + def move(self, y: int, x: int, /) -> None: ... + def replace(self, win: window, /) -> None: ... + def set_userptr(self, obj: object, /) -> None: ... + def show(self) -> None: ... + def top(self) -> None: ... + def userptr(self) -> object: ... + def window(self) -> window: ... + +def bottom_panel() -> panel: ... +def new_panel(win: window, /) -> panel: ... +def top_panel() -> panel: ... +def update_panels() -> panel: ... diff --git a/mypy/typeshed/stdlib/_dbm.pyi b/mypy/typeshed/stdlib/_dbm.pyi new file mode 100644 index 000000000000..7e53cca3c704 --- /dev/null +++ b/mypy/typeshed/stdlib/_dbm.pyi @@ -0,0 +1,44 @@ +import sys +from _typeshed import ReadOnlyBuffer, StrOrBytesPath +from types import TracebackType +from typing import TypeVar, final, overload, type_check_only +from typing_extensions import Self, TypeAlias + +if sys.platform != "win32": + _T = TypeVar("_T") + _KeyType: TypeAlias = str | ReadOnlyBuffer + _ValueType: TypeAlias = str | ReadOnlyBuffer + + class error(OSError): ... + library: str + + # Actual typename dbm, not exposed by the implementation + @final + @type_check_only + class _dbm: + def close(self) -> None: ... + if sys.version_info >= (3, 13): + def clear(self) -> None: ... + + def __getitem__(self, item: _KeyType) -> bytes: ... + def __setitem__(self, key: _KeyType, value: _ValueType) -> None: ... + def __delitem__(self, key: _KeyType) -> None: ... + def __len__(self) -> int: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + @overload + def get(self, k: _KeyType, /) -> bytes | None: ... + @overload + def get(self, k: _KeyType, default: _T, /) -> bytes | _T: ... + def keys(self) -> list[bytes]: ... + def setdefault(self, k: _KeyType, default: _ValueType = ..., /) -> bytes: ... + # This isn't true, but the class can't be instantiated. See #13024 + __new__: None # type: ignore[assignment] + __init__: None # type: ignore[assignment] + + if sys.version_info >= (3, 11): + def open(filename: StrOrBytesPath, flags: str = "r", mode: int = 0o666, /) -> _dbm: ... + else: + def open(filename: str, flags: str = "r", mode: int = 0o666, /) -> _dbm: ... diff --git a/mypy/typeshed/stdlib/_decimal.pyi b/mypy/typeshed/stdlib/_decimal.pyi new file mode 100644 index 000000000000..cdd0268a1bdf --- /dev/null +++ b/mypy/typeshed/stdlib/_decimal.pyi @@ -0,0 +1,67 @@ +import sys +from decimal import ( + Clamped as Clamped, + Context as Context, + ConversionSyntax as ConversionSyntax, + Decimal as Decimal, + DecimalException as DecimalException, + DecimalTuple as DecimalTuple, + DivisionByZero as DivisionByZero, + DivisionImpossible as DivisionImpossible, + DivisionUndefined as DivisionUndefined, + FloatOperation as FloatOperation, + Inexact as Inexact, + InvalidContext as InvalidContext, + InvalidOperation as InvalidOperation, + Overflow as Overflow, + Rounded as Rounded, + Subnormal as Subnormal, + Underflow as Underflow, + _ContextManager, +) +from typing import Final +from typing_extensions import TypeAlias + +_TrapType: TypeAlias = type[DecimalException] + +__version__: Final[str] +__libmpdec_version__: Final[str] + +ROUND_DOWN: Final[str] +ROUND_HALF_UP: Final[str] +ROUND_HALF_EVEN: Final[str] +ROUND_CEILING: Final[str] +ROUND_FLOOR: Final[str] +ROUND_UP: Final[str] +ROUND_HALF_DOWN: Final[str] +ROUND_05UP: Final[str] +HAVE_CONTEXTVAR: Final[bool] +HAVE_THREADS: Final[bool] +MAX_EMAX: Final[int] +MAX_PREC: Final[int] +MIN_EMIN: Final[int] +MIN_ETINY: Final[int] + +def setcontext(context: Context, /) -> None: ... +def getcontext() -> Context: ... + +if sys.version_info >= (3, 11): + def localcontext( + ctx: Context | None = None, + *, + prec: int | None = ..., + rounding: str | None = ..., + Emin: int | None = ..., + Emax: int | None = ..., + capitals: int | None = ..., + clamp: int | None = ..., + traps: dict[_TrapType, bool] | None = ..., + flags: dict[_TrapType, bool] | None = ..., + ) -> _ContextManager: ... + +else: + def localcontext(ctx: Context | None = None) -> _ContextManager: ... + +DefaultContext: Context +BasicContext: Context +ExtendedContext: Context diff --git a/mypy/typeshed/stdlib/_dummy_thread.pyi b/mypy/typeshed/stdlib/_dummy_thread.pyi new file mode 100644 index 000000000000..1182e53c66c3 --- /dev/null +++ b/mypy/typeshed/stdlib/_dummy_thread.pyi @@ -0,0 +1,33 @@ +from collections.abc import Callable +from types import TracebackType +from typing import Any, NoReturn, overload +from typing_extensions import TypeVarTuple, Unpack + +__all__ = ["error", "start_new_thread", "exit", "get_ident", "allocate_lock", "interrupt_main", "LockType", "RLock"] + +_Ts = TypeVarTuple("_Ts") + +TIMEOUT_MAX: int +error = RuntimeError + +@overload +def start_new_thread(function: Callable[[Unpack[_Ts]], object], args: tuple[Unpack[_Ts]]) -> None: ... +@overload +def start_new_thread(function: Callable[..., object], args: tuple[Any, ...], kwargs: dict[str, Any]) -> None: ... +def exit() -> NoReturn: ... +def get_ident() -> int: ... +def allocate_lock() -> LockType: ... +def stack_size(size: int | None = None) -> int: ... + +class LockType: + locked_status: bool + def acquire(self, waitflag: bool | None = None, timeout: int = -1) -> bool: ... + def __enter__(self, waitflag: bool | None = None, timeout: int = -1) -> bool: ... + def __exit__(self, typ: type[BaseException] | None, val: BaseException | None, tb: TracebackType | None) -> None: ... + def release(self) -> bool: ... + def locked(self) -> bool: ... + +class RLock(LockType): + def release(self) -> None: ... # type: ignore[override] + +def interrupt_main() -> None: ... diff --git a/mypy/typeshed/stdlib/_dummy_threading.pyi b/mypy/typeshed/stdlib/_dummy_threading.pyi new file mode 100644 index 000000000000..21d1d1921c0e --- /dev/null +++ b/mypy/typeshed/stdlib/_dummy_threading.pyi @@ -0,0 +1,164 @@ +import sys +from _thread import _excepthook, _ExceptHookArgs +from _typeshed import ProfileFunction, TraceFunction +from collections.abc import Callable, Iterable, Mapping +from types import TracebackType +from typing import Any, TypeVar + +_T = TypeVar("_T") + +__all__ = [ + "get_ident", + "active_count", + "Condition", + "current_thread", + "enumerate", + "main_thread", + "TIMEOUT_MAX", + "Event", + "Lock", + "RLock", + "Semaphore", + "BoundedSemaphore", + "Thread", + "Barrier", + "BrokenBarrierError", + "Timer", + "ThreadError", + "setprofile", + "settrace", + "local", + "stack_size", + "ExceptHookArgs", + "excepthook", +] + +def active_count() -> int: ... +def current_thread() -> Thread: ... +def currentThread() -> Thread: ... +def get_ident() -> int: ... +def enumerate() -> list[Thread]: ... +def main_thread() -> Thread: ... +def settrace(func: TraceFunction) -> None: ... +def setprofile(func: ProfileFunction | None) -> None: ... +def stack_size(size: int | None = None) -> int: ... + +TIMEOUT_MAX: float + +class ThreadError(Exception): ... + +class local: + def __getattribute__(self, name: str) -> Any: ... + def __setattr__(self, name: str, value: Any) -> None: ... + def __delattr__(self, name: str) -> None: ... + +class Thread: + name: str + daemon: bool + @property + def ident(self) -> int | None: ... + def __init__( + self, + group: None = None, + target: Callable[..., object] | None = None, + name: str | None = None, + args: Iterable[Any] = (), + kwargs: Mapping[str, Any] | None = None, + *, + daemon: bool | None = None, + ) -> None: ... + def start(self) -> None: ... + def run(self) -> None: ... + def join(self, timeout: float | None = None) -> None: ... + def getName(self) -> str: ... + def setName(self, name: str) -> None: ... + @property + def native_id(self) -> int | None: ... # only available on some platforms + def is_alive(self) -> bool: ... + if sys.version_info < (3, 9): + def isAlive(self) -> bool: ... + + def isDaemon(self) -> bool: ... + def setDaemon(self, daemonic: bool) -> None: ... + +class _DummyThread(Thread): ... + +class Lock: + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> bool | None: ... + def acquire(self, blocking: bool = ..., timeout: float = ...) -> bool: ... + def release(self) -> None: ... + def locked(self) -> bool: ... + +class _RLock: + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> bool | None: ... + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: ... + def release(self) -> None: ... + +RLock = _RLock + +class Condition: + def __init__(self, lock: Lock | _RLock | None = None) -> None: ... + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> bool | None: ... + def acquire(self, blocking: bool = ..., timeout: float = ...) -> bool: ... + def release(self) -> None: ... + def wait(self, timeout: float | None = None) -> bool: ... + def wait_for(self, predicate: Callable[[], _T], timeout: float | None = None) -> _T: ... + def notify(self, n: int = 1) -> None: ... + def notify_all(self) -> None: ... + def notifyAll(self) -> None: ... + +class Semaphore: + def __init__(self, value: int = 1) -> None: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> bool | None: ... + def acquire(self, blocking: bool = True, timeout: float | None = None) -> bool: ... + def __enter__(self, blocking: bool = True, timeout: float | None = None) -> bool: ... + if sys.version_info >= (3, 9): + def release(self, n: int = ...) -> None: ... + else: + def release(self) -> None: ... + +class BoundedSemaphore(Semaphore): ... + +class Event: + def is_set(self) -> bool: ... + def set(self) -> None: ... + def clear(self) -> None: ... + def wait(self, timeout: float | None = None) -> bool: ... + +excepthook = _excepthook +ExceptHookArgs = _ExceptHookArgs + +class Timer(Thread): + def __init__( + self, + interval: float, + function: Callable[..., object], + args: Iterable[Any] | None = None, + kwargs: Mapping[str, Any] | None = None, + ) -> None: ... + def cancel(self) -> None: ... + +class Barrier: + @property + def parties(self) -> int: ... + @property + def n_waiting(self) -> int: ... + @property + def broken(self) -> bool: ... + def __init__(self, parties: int, action: Callable[[], None] | None = None, timeout: float | None = None) -> None: ... + def wait(self, timeout: float | None = None) -> int: ... + def reset(self) -> None: ... + def abort(self) -> None: ... + +class BrokenBarrierError(RuntimeError): ... diff --git a/mypy/typeshed/stdlib/_frozen_importlib.pyi b/mypy/typeshed/stdlib/_frozen_importlib.pyi new file mode 100644 index 000000000000..b6d7a1842048 --- /dev/null +++ b/mypy/typeshed/stdlib/_frozen_importlib.pyi @@ -0,0 +1,112 @@ +import importlib.abc +import importlib.machinery +import sys +import types +from _typeshed.importlib import LoaderProtocol +from collections.abc import Mapping, Sequence +from types import ModuleType +from typing import Any + +# Signature of `builtins.__import__` should be kept identical to `importlib.__import__` +def __import__( + name: str, + globals: Mapping[str, object] | None = None, + locals: Mapping[str, object] | None = None, + fromlist: Sequence[str] = (), + level: int = 0, +) -> ModuleType: ... +def spec_from_loader( + name: str, loader: LoaderProtocol | None, *, origin: str | None = None, is_package: bool | None = None +) -> importlib.machinery.ModuleSpec | None: ... +def module_from_spec(spec: importlib.machinery.ModuleSpec) -> types.ModuleType: ... +def _init_module_attrs( + spec: importlib.machinery.ModuleSpec, module: types.ModuleType, *, override: bool = False +) -> types.ModuleType: ... + +class ModuleSpec: + def __init__( + self, + name: str, + loader: importlib.abc.Loader | None, + *, + origin: str | None = None, + loader_state: Any = None, + is_package: bool | None = None, + ) -> None: ... + name: str + loader: importlib.abc.Loader | None + origin: str | None + submodule_search_locations: list[str] | None + loader_state: Any + cached: str | None + @property + def parent(self) -> str | None: ... + has_location: bool + def __eq__(self, other: object) -> bool: ... + +class BuiltinImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader): + # MetaPathFinder + if sys.version_info < (3, 12): + @classmethod + def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... + + @classmethod + def find_spec( + cls, fullname: str, path: Sequence[str] | None = None, target: types.ModuleType | None = None + ) -> ModuleSpec | None: ... + # InspectLoader + @classmethod + def is_package(cls, fullname: str) -> bool: ... + @classmethod + def load_module(cls, fullname: str) -> types.ModuleType: ... + @classmethod + def get_code(cls, fullname: str) -> None: ... + @classmethod + def get_source(cls, fullname: str) -> None: ... + # Loader + if sys.version_info < (3, 12): + @staticmethod + def module_repr(module: types.ModuleType) -> str: ... + if sys.version_info >= (3, 10): + @staticmethod + def create_module(spec: ModuleSpec) -> types.ModuleType | None: ... + @staticmethod + def exec_module(module: types.ModuleType) -> None: ... + else: + @classmethod + def create_module(cls, spec: ModuleSpec) -> types.ModuleType | None: ... + @classmethod + def exec_module(cls, module: types.ModuleType) -> None: ... + +class FrozenImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader): + # MetaPathFinder + if sys.version_info < (3, 12): + @classmethod + def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... + + @classmethod + def find_spec( + cls, fullname: str, path: Sequence[str] | None = None, target: types.ModuleType | None = None + ) -> ModuleSpec | None: ... + # InspectLoader + @classmethod + def is_package(cls, fullname: str) -> bool: ... + @classmethod + def load_module(cls, fullname: str) -> types.ModuleType: ... + @classmethod + def get_code(cls, fullname: str) -> None: ... + @classmethod + def get_source(cls, fullname: str) -> None: ... + # Loader + if sys.version_info < (3, 12): + @staticmethod + def module_repr(m: types.ModuleType) -> str: ... + if sys.version_info >= (3, 10): + @staticmethod + def create_module(spec: ModuleSpec) -> types.ModuleType | None: ... + else: + @classmethod + def create_module(cls, spec: ModuleSpec) -> types.ModuleType | None: ... + + @staticmethod + def exec_module(module: types.ModuleType) -> None: ... diff --git a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi new file mode 100644 index 000000000000..d3127666da30 --- /dev/null +++ b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi @@ -0,0 +1,178 @@ +import _ast +import _io +import importlib.abc +import importlib.machinery +import sys +import types +from _typeshed import ReadableBuffer, StrOrBytesPath, StrPath +from _typeshed.importlib import LoaderProtocol +from collections.abc import Callable, Iterable, Iterator, Mapping, MutableSequence, Sequence +from importlib.machinery import ModuleSpec +from importlib.metadata import DistributionFinder, PathDistribution +from typing import Any, Literal +from typing_extensions import Self, deprecated + +if sys.version_info >= (3, 10): + import importlib.readers + +if sys.platform == "win32": + path_separators: Literal["\\/"] + path_sep: Literal["\\"] + path_sep_tuple: tuple[Literal["\\"], Literal["/"]] +else: + path_separators: Literal["/"] + path_sep: Literal["/"] + path_sep_tuple: tuple[Literal["/"]] + +MAGIC_NUMBER: bytes + +def cache_from_source(path: str, debug_override: bool | None = None, *, optimization: Any | None = None) -> str: ... +def source_from_cache(path: str) -> str: ... +def decode_source(source_bytes: ReadableBuffer) -> str: ... +def spec_from_file_location( + name: str, + location: StrOrBytesPath | None = None, + *, + loader: LoaderProtocol | None = None, + submodule_search_locations: list[str] | None = ..., +) -> importlib.machinery.ModuleSpec | None: ... + +class WindowsRegistryFinder(importlib.abc.MetaPathFinder): + if sys.version_info < (3, 12): + @classmethod + def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... + + @classmethod + def find_spec( + cls, fullname: str, path: Sequence[str] | None = None, target: types.ModuleType | None = None + ) -> ModuleSpec | None: ... + +class PathFinder(importlib.abc.MetaPathFinder): + if sys.version_info >= (3, 10): + @staticmethod + def invalidate_caches() -> None: ... + else: + @classmethod + def invalidate_caches(cls) -> None: ... + if sys.version_info >= (3, 10): + @staticmethod + def find_distributions(context: DistributionFinder.Context = ...) -> Iterable[PathDistribution]: ... + else: + @classmethod + def find_distributions(cls, context: DistributionFinder.Context = ...) -> Iterable[PathDistribution]: ... + + @classmethod + def find_spec( + cls, fullname: str, path: Sequence[str] | None = None, target: types.ModuleType | None = None + ) -> ModuleSpec | None: ... + if sys.version_info < (3, 12): + @classmethod + def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... + +SOURCE_SUFFIXES: list[str] +DEBUG_BYTECODE_SUFFIXES: list[str] +OPTIMIZED_BYTECODE_SUFFIXES: list[str] +BYTECODE_SUFFIXES: list[str] +EXTENSION_SUFFIXES: list[str] + +class FileFinder(importlib.abc.PathEntryFinder): + path: str + def __init__(self, path: str, *loader_details: tuple[type[importlib.abc.Loader], list[str]]) -> None: ... + @classmethod + def path_hook( + cls, *loader_details: tuple[type[importlib.abc.Loader], list[str]] + ) -> Callable[[str], importlib.abc.PathEntryFinder]: ... + +class _LoaderBasics: + def is_package(self, fullname: str) -> bool: ... + def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: ... + def exec_module(self, module: types.ModuleType) -> None: ... + def load_module(self, fullname: str) -> types.ModuleType: ... + +class SourceLoader(_LoaderBasics): + def path_mtime(self, path: str) -> float: ... + def set_data(self, path: str, data: bytes) -> None: ... + def get_source(self, fullname: str) -> str | None: ... + def path_stats(self, path: str) -> Mapping[str, Any]: ... + def source_to_code( + self, data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: ReadableBuffer | StrPath + ) -> types.CodeType: ... + def get_code(self, fullname: str) -> types.CodeType | None: ... + +class FileLoader: + name: str + path: str + def __init__(self, fullname: str, path: str) -> None: ... + def get_data(self, path: str) -> bytes: ... + def get_filename(self, name: str | None = None) -> str: ... + def load_module(self, name: str | None = None) -> types.ModuleType: ... + if sys.version_info >= (3, 10): + def get_resource_reader(self, name: str | None = None) -> importlib.readers.FileReader: ... + else: + def get_resource_reader(self, name: str | None = None) -> Self | None: ... + def open_resource(self, resource: str) -> _io.FileIO: ... + def resource_path(self, resource: str) -> str: ... + def is_resource(self, name: str) -> bool: ... + def contents(self) -> Iterator[str]: ... + +class SourceFileLoader(importlib.abc.FileLoader, FileLoader, importlib.abc.SourceLoader, SourceLoader): # type: ignore[misc] # incompatible method arguments in base classes + def set_data(self, path: str, data: ReadableBuffer, *, _mode: int = 0o666) -> None: ... + def path_stats(self, path: str) -> Mapping[str, Any]: ... + +class SourcelessFileLoader(importlib.abc.FileLoader, FileLoader, _LoaderBasics): + def get_code(self, fullname: str) -> types.CodeType | None: ... + def get_source(self, fullname: str) -> None: ... + +class ExtensionFileLoader(FileLoader, _LoaderBasics, importlib.abc.ExecutionLoader): + def __init__(self, name: str, path: str) -> None: ... + def get_filename(self, name: str | None = None) -> str: ... + def get_source(self, fullname: str) -> None: ... + def create_module(self, spec: ModuleSpec) -> types.ModuleType: ... + def exec_module(self, module: types.ModuleType) -> None: ... + def get_code(self, fullname: str) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +if sys.version_info >= (3, 11): + class NamespaceLoader(importlib.abc.InspectLoader): + def __init__( + self, name: str, path: MutableSequence[str], path_finder: Callable[[str, tuple[str, ...]], ModuleSpec] + ) -> None: ... + def is_package(self, fullname: str) -> Literal[True]: ... + def get_source(self, fullname: str) -> Literal[""]: ... + def get_code(self, fullname: str) -> types.CodeType: ... + def create_module(self, spec: ModuleSpec) -> None: ... + def exec_module(self, module: types.ModuleType) -> None: ... + @deprecated("load_module() is deprecated; use exec_module() instead") + def load_module(self, fullname: str) -> types.ModuleType: ... + def get_resource_reader(self, module: types.ModuleType) -> importlib.readers.NamespaceReader: ... + if sys.version_info < (3, 12): + @staticmethod + @deprecated("module_repr() is deprecated, and has been removed in Python 3.12") + def module_repr(module: types.ModuleType) -> str: ... + + _NamespaceLoader = NamespaceLoader +else: + class _NamespaceLoader: + def __init__( + self, name: str, path: MutableSequence[str], path_finder: Callable[[str, tuple[str, ...]], ModuleSpec] + ) -> None: ... + def is_package(self, fullname: str) -> Literal[True]: ... + def get_source(self, fullname: str) -> Literal[""]: ... + def get_code(self, fullname: str) -> types.CodeType: ... + def create_module(self, spec: ModuleSpec) -> None: ... + def exec_module(self, module: types.ModuleType) -> None: ... + @deprecated("load_module() is deprecated; use exec_module() instead") + def load_module(self, fullname: str) -> types.ModuleType: ... + if sys.version_info >= (3, 10): + @staticmethod + @deprecated("module_repr() is deprecated, and has been removed in Python 3.12") + def module_repr(module: types.ModuleType) -> str: ... + def get_resource_reader(self, module: types.ModuleType) -> importlib.readers.NamespaceReader: ... + else: + @classmethod + @deprecated("module_repr() is deprecated, and has been removed in Python 3.12") + def module_repr(cls, module: types.ModuleType) -> str: ... + +if sys.version_info >= (3, 13): + class AppleFrameworkLoader(ExtensionFileLoader, importlib.abc.ExecutionLoader): ... diff --git a/mypy/typeshed/stdlib/_gdbm.pyi b/mypy/typeshed/stdlib/_gdbm.pyi new file mode 100644 index 000000000000..1d1d541f5477 --- /dev/null +++ b/mypy/typeshed/stdlib/_gdbm.pyi @@ -0,0 +1,47 @@ +import sys +from _typeshed import ReadOnlyBuffer, StrOrBytesPath +from types import TracebackType +from typing import TypeVar, overload +from typing_extensions import Self, TypeAlias + +if sys.platform != "win32": + _T = TypeVar("_T") + _KeyType: TypeAlias = str | ReadOnlyBuffer + _ValueType: TypeAlias = str | ReadOnlyBuffer + + open_flags: str + + class error(OSError): ... + # Actual typename gdbm, not exposed by the implementation + class _gdbm: + def firstkey(self) -> bytes | None: ... + def nextkey(self, key: _KeyType) -> bytes | None: ... + def reorganize(self) -> None: ... + def sync(self) -> None: ... + def close(self) -> None: ... + if sys.version_info >= (3, 13): + def clear(self) -> None: ... + + def __getitem__(self, item: _KeyType) -> bytes: ... + def __setitem__(self, key: _KeyType, value: _ValueType) -> None: ... + def __delitem__(self, key: _KeyType) -> None: ... + def __contains__(self, key: _KeyType) -> bool: ... + def __len__(self) -> int: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + @overload + def get(self, k: _KeyType) -> bytes | None: ... + @overload + def get(self, k: _KeyType, default: _T) -> bytes | _T: ... + def keys(self) -> list[bytes]: ... + def setdefault(self, k: _KeyType, default: _ValueType = ...) -> bytes: ... + # Don't exist at runtime + __new__: None # type: ignore[assignment] + __init__: None # type: ignore[assignment] + + if sys.version_info >= (3, 11): + def open(filename: StrOrBytesPath, flags: str = "r", mode: int = 0o666, /) -> _gdbm: ... + else: + def open(filename: str, flags: str = "r", mode: int = 0o666, /) -> _gdbm: ... diff --git a/mypy/typeshed/stdlib/_hashlib.pyi b/mypy/typeshed/stdlib/_hashlib.pyi new file mode 100644 index 000000000000..5cf85e4cacaa --- /dev/null +++ b/mypy/typeshed/stdlib/_hashlib.pyi @@ -0,0 +1,80 @@ +import sys +from _typeshed import ReadableBuffer +from collections.abc import Callable +from types import ModuleType +from typing import AnyStr, final, overload +from typing_extensions import Self, TypeAlias + +_DigestMod: TypeAlias = str | Callable[[], HASH] | ModuleType | None + +openssl_md_meth_names: frozenset[str] + +class HASH: + @property + def digest_size(self) -> int: ... + @property + def block_size(self) -> int: ... + @property + def name(self) -> str: ... + def copy(self) -> Self: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + def update(self, obj: ReadableBuffer, /) -> None: ... + +if sys.version_info >= (3, 10): + class UnsupportedDigestmodError(ValueError): ... + +if sys.version_info >= (3, 9): + class HASHXOF(HASH): + def digest(self, length: int) -> bytes: ... # type: ignore[override] + def hexdigest(self, length: int) -> str: ... # type: ignore[override] + + @final + class HMAC: + @property + def digest_size(self) -> int: ... + @property + def block_size(self) -> int: ... + @property + def name(self) -> str: ... + def copy(self) -> Self: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + def update(self, msg: ReadableBuffer) -> None: ... + + @overload + def compare_digest(a: ReadableBuffer, b: ReadableBuffer, /) -> bool: ... + @overload + def compare_digest(a: AnyStr, b: AnyStr, /) -> bool: ... + def get_fips_mode() -> int: ... + def hmac_new(key: bytes | bytearray, msg: ReadableBuffer = b"", digestmod: _DigestMod = None) -> HMAC: ... + def new(name: str, string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_md5(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha1(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha224(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha384(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha512(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_224(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_384(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_512(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_shake_128(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASHXOF: ... + def openssl_shake_256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASHXOF: ... + +else: + def new(name: str, string: ReadableBuffer = b"") -> HASH: ... + def openssl_md5(string: ReadableBuffer = b"") -> HASH: ... + def openssl_sha1(string: ReadableBuffer = b"") -> HASH: ... + def openssl_sha224(string: ReadableBuffer = b"") -> HASH: ... + def openssl_sha256(string: ReadableBuffer = b"") -> HASH: ... + def openssl_sha384(string: ReadableBuffer = b"") -> HASH: ... + def openssl_sha512(string: ReadableBuffer = b"") -> HASH: ... + +def hmac_digest(key: bytes | bytearray, msg: ReadableBuffer, digest: str) -> bytes: ... +def pbkdf2_hmac( + hash_name: str, password: ReadableBuffer, salt: ReadableBuffer, iterations: int, dklen: int | None = None +) -> bytes: ... +def scrypt( + password: ReadableBuffer, *, salt: ReadableBuffer, n: int, r: int, p: int, maxmem: int = 0, dklen: int = 64 +) -> bytes: ... diff --git a/mypy/typeshed/stdlib/_heapq.pyi b/mypy/typeshed/stdlib/_heapq.pyi new file mode 100644 index 000000000000..9f731bf91eef --- /dev/null +++ b/mypy/typeshed/stdlib/_heapq.pyi @@ -0,0 +1,11 @@ +from typing import Any, Final, TypeVar + +_T = TypeVar("_T") + +__about__: Final[str] + +def heapify(heap: list[Any], /) -> None: ... +def heappop(heap: list[_T], /) -> _T: ... +def heappush(heap: list[_T], item: _T, /) -> None: ... +def heappushpop(heap: list[_T], item: _T, /) -> _T: ... +def heapreplace(heap: list[_T], item: _T, /) -> _T: ... diff --git a/mypy/typeshed/stdlib/_imp.pyi b/mypy/typeshed/stdlib/_imp.pyi new file mode 100644 index 000000000000..de3549a91da5 --- /dev/null +++ b/mypy/typeshed/stdlib/_imp.pyi @@ -0,0 +1,28 @@ +import sys +import types +from _typeshed import ReadableBuffer +from importlib.machinery import ModuleSpec +from typing import Any + +check_hash_based_pycs: str + +def source_hash(key: int, source: ReadableBuffer) -> bytes: ... +def create_builtin(spec: ModuleSpec, /) -> types.ModuleType: ... +def create_dynamic(spec: ModuleSpec, file: Any = None, /) -> types.ModuleType: ... +def acquire_lock() -> None: ... +def exec_builtin(mod: types.ModuleType, /) -> int: ... +def exec_dynamic(mod: types.ModuleType, /) -> int: ... +def extension_suffixes() -> list[str]: ... +def init_frozen(name: str, /) -> types.ModuleType: ... +def is_builtin(name: str, /) -> int: ... +def is_frozen(name: str, /) -> bool: ... +def is_frozen_package(name: str, /) -> bool: ... +def lock_held() -> bool: ... +def release_lock() -> None: ... + +if sys.version_info >= (3, 11): + def find_frozen(name: str, /, *, withdata: bool = False) -> tuple[memoryview | None, bool, str | None] | None: ... + def get_frozen_object(name: str, data: ReadableBuffer | None = None, /) -> types.CodeType: ... + +else: + def get_frozen_object(name: str, /) -> types.CodeType: ... diff --git a/mypy/typeshed/stdlib/_interpchannels.pyi b/mypy/typeshed/stdlib/_interpchannels.pyi new file mode 100644 index 000000000000..c03496044df0 --- /dev/null +++ b/mypy/typeshed/stdlib/_interpchannels.pyi @@ -0,0 +1,86 @@ +from _typeshed import structseq +from typing import Any, Final, Literal, SupportsIndex, final +from typing_extensions import Buffer, Self + +class ChannelError(RuntimeError): ... +class ChannelClosedError(ChannelError): ... +class ChannelEmptyError(ChannelError): ... +class ChannelNotEmptyError(ChannelError): ... +class ChannelNotFoundError(ChannelError): ... + +# Mark as final, since instantiating ChannelID is not supported. +@final +class ChannelID: + @property + def end(self) -> Literal["send", "recv", "both"]: ... + @property + def send(self) -> Self: ... + @property + def recv(self) -> Self: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: ChannelID) -> bool: ... + def __gt__(self, other: ChannelID) -> bool: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __int__(self) -> int: ... + def __le__(self, other: ChannelID) -> bool: ... + def __lt__(self, other: ChannelID) -> bool: ... + def __ne__(self, other: object) -> bool: ... + +@final +class ChannelInfo(structseq[int], tuple[bool, bool, bool, int, int, int, int, int]): + __match_args__: Final = ( + "open", + "closing", + "closed", + "count", + "num_interp_send", + "num_interp_send_released", + "num_interp_recv", + "num_interp_recv_released", + ) + @property + def open(self) -> bool: ... + @property + def closing(self) -> bool: ... + @property + def closed(self) -> bool: ... + @property + def count(self) -> int: ... # type: ignore[override] + @property + def num_interp_send(self) -> int: ... + @property + def num_interp_send_released(self) -> int: ... + @property + def num_interp_recv(self) -> int: ... + @property + def num_interp_recv_released(self) -> int: ... + @property + def num_interp_both(self) -> int: ... + @property + def num_interp_both_recv_released(self) -> int: ... + @property + def num_interp_both_send_released(self) -> int: ... + @property + def num_interp_both_released(self) -> int: ... + @property + def recv_associated(self) -> bool: ... + @property + def recv_released(self) -> bool: ... + @property + def send_associated(self) -> bool: ... + @property + def send_released(self) -> bool: ... + +def create(unboundop: Literal[1, 2, 3]) -> ChannelID: ... +def destroy(cid: SupportsIndex) -> None: ... +def list_all() -> list[ChannelID]: ... +def list_interpreters(cid: SupportsIndex, *, send: bool) -> list[int]: ... +def send(cid: SupportsIndex, obj: object, *, blocking: bool = True, timeout: float | None = None) -> None: ... +def send_buffer(cid: SupportsIndex, obj: Buffer, *, blocking: bool = True, timeout: float | None = None) -> None: ... +def recv(cid: SupportsIndex, default: object = ...) -> tuple[Any, Literal[1, 2, 3]]: ... +def close(cid: SupportsIndex, *, send: bool = False, recv: bool = False) -> None: ... +def get_count(cid: SupportsIndex) -> int: ... +def get_info(cid: SupportsIndex) -> ChannelInfo: ... +def get_channel_defaults(cid: SupportsIndex) -> Literal[1, 2, 3]: ... +def release(cid: SupportsIndex, *, send: bool = False, recv: bool = False, force: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/_interpqueues.pyi b/mypy/typeshed/stdlib/_interpqueues.pyi new file mode 100644 index 000000000000..db5e4cff5068 --- /dev/null +++ b/mypy/typeshed/stdlib/_interpqueues.pyi @@ -0,0 +1,16 @@ +from typing import Any, SupportsIndex + +class QueueError(RuntimeError): ... +class QueueNotFoundError(QueueError): ... + +def bind(qid: SupportsIndex) -> None: ... +def create(maxsize: SupportsIndex, fmt: SupportsIndex) -> int: ... +def destroy(qid: SupportsIndex) -> None: ... +def get(qid: SupportsIndex) -> tuple[Any, int]: ... +def get_count(qid: SupportsIndex) -> int: ... +def get_maxsize(qid: SupportsIndex) -> int: ... +def get_queue_defaults(qid: SupportsIndex) -> tuple[int]: ... +def is_full(qid: SupportsIndex) -> bool: ... +def list_all() -> list[tuple[int, int]]: ... +def put(qid: SupportsIndex, obj: Any, fmt: SupportsIndex) -> None: ... +def release(qid: SupportsIndex) -> None: ... diff --git a/mypy/typeshed/stdlib/_interpreters.pyi b/mypy/typeshed/stdlib/_interpreters.pyi new file mode 100644 index 000000000000..a57ef13c6d0f --- /dev/null +++ b/mypy/typeshed/stdlib/_interpreters.pyi @@ -0,0 +1,50 @@ +import types +from collections.abc import Callable, Mapping +from typing import Final, Literal, SupportsIndex +from typing_extensions import TypeAlias + +_Configs: TypeAlias = Literal["default", "isolated", "legacy", "empty", ""] + +class InterpreterError(Exception): ... +class InterpreterNotFoundError(InterpreterError): ... +class NotShareableError(ValueError): ... + +class CrossInterpreterBufferView: + def __buffer__(self, flags: int, /) -> memoryview: ... + +def new_config(name: _Configs = "isolated", /, **overides: object) -> types.SimpleNamespace: ... +def create(config: types.SimpleNamespace | _Configs | None = "isolated", *, reqrefs: bool = False) -> int: ... +def destroy(id: SupportsIndex, *, restrict: bool = False) -> None: ... +def list_all(*, require_ready: bool) -> list[tuple[int, int]]: ... +def get_current() -> tuple[int, int]: ... +def get_main() -> tuple[int, int]: ... +def is_running(id: SupportsIndex, *, restrict: bool = False) -> bool: ... +def get_config(id: SupportsIndex, *, restrict: bool = False) -> types.SimpleNamespace: ... +def whence(id: SupportsIndex) -> int: ... +def exec(id: SupportsIndex, code: str, shared: bool | None = None, *, restrict: bool = False) -> None: ... +def call( + id: SupportsIndex, + callable: Callable[..., object], + args: tuple[object, ...] | None = None, + kwargs: dict[str, object] | None = None, + *, + restrict: bool = False, +) -> object: ... +def run_string( + id: SupportsIndex, script: str | types.CodeType | Callable[[], object], shared: bool | None = None, *, restrict: bool = False +) -> None: ... +def run_func( + id: SupportsIndex, func: types.CodeType | Callable[[], object], shared: bool | None = None, *, restrict: bool = False +) -> None: ... +def set___main___attrs(id: SupportsIndex, updates: Mapping[str, object], *, restrict: bool = False) -> None: ... +def incref(id: SupportsIndex, *, implieslink: bool = False, restrict: bool = False) -> None: ... +def decref(id: SupportsIndex, *, restrict: bool = False) -> None: ... +def is_shareable(obj: object) -> bool: ... +def capture_exception(exc: BaseException | None = None) -> types.SimpleNamespace: ... + +WHENCE_UNKNOWN: Final = 0 +WHENCE_RUNTIME: Final = 1 +WHENCE_LEGACY_CAPI: Final = 2 +WHENCE_CAPI: Final = 3 +WHENCE_XI: Final = 4 +WHENCE_STDLIB: Final = 5 diff --git a/mypy/typeshed/stdlib/_io.pyi b/mypy/typeshed/stdlib/_io.pyi new file mode 100644 index 000000000000..284d99f92b60 --- /dev/null +++ b/mypy/typeshed/stdlib/_io.pyi @@ -0,0 +1,216 @@ +import builtins +import codecs +import sys +from _typeshed import FileDescriptorOrPath, MaybeNone, ReadableBuffer, WriteableBuffer +from collections.abc import Callable, Iterable, Iterator +from io import BufferedIOBase, RawIOBase, TextIOBase, UnsupportedOperation as UnsupportedOperation +from os import _Opener +from types import TracebackType +from typing import IO, Any, BinaryIO, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload, type_check_only +from typing_extensions import Self + +_T = TypeVar("_T") + +DEFAULT_BUFFER_SIZE: Final = 8192 + +open = builtins.open + +def open_code(path: str) -> IO[bytes]: ... + +BlockingIOError = builtins.BlockingIOError + +class _IOBase: + def __iter__(self) -> Iterator[bytes]: ... + def __next__(self) -> bytes: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def close(self) -> None: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + def readable(self) -> bool: ... + read: Callable[..., Any] + def readlines(self, hint: int = -1, /) -> list[bytes]: ... + def seek(self, offset: int, whence: int = 0, /) -> int: ... + def seekable(self) -> bool: ... + def tell(self) -> int: ... + def truncate(self, size: int | None = None, /) -> int: ... + def writable(self) -> bool: ... + write: Callable[..., Any] + def writelines(self, lines: Iterable[ReadableBuffer], /) -> None: ... + def readline(self, size: int | None = -1, /) -> bytes: ... + def __del__(self) -> None: ... + @property + def closed(self) -> bool: ... + def _checkClosed(self) -> None: ... # undocumented + +class _RawIOBase(_IOBase): + def readall(self) -> bytes: ... + # The following methods can return None if the file is in non-blocking mode + # and no data is available. + def readinto(self, buffer: WriteableBuffer, /) -> int | MaybeNone: ... + def write(self, b: ReadableBuffer, /) -> int | MaybeNone: ... + def read(self, size: int = -1, /) -> bytes | MaybeNone: ... + +class _BufferedIOBase(_IOBase): + def detach(self) -> RawIOBase: ... + def readinto(self, buffer: WriteableBuffer, /) -> int: ... + def write(self, buffer: ReadableBuffer, /) -> int: ... + def readinto1(self, buffer: WriteableBuffer, /) -> int: ... + def read(self, size: int | None = -1, /) -> bytes: ... + def read1(self, size: int = -1, /) -> bytes: ... + +class FileIO(RawIOBase, _RawIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes + mode: str + # The type of "name" equals the argument passed in to the constructor, + # but that can make FileIO incompatible with other I/O types that assume + # "name" is a str. In the future, making FileIO generic might help. + name: Any + def __init__( + self, file: FileDescriptorOrPath, mode: str = "r", closefd: bool = True, opener: _Opener | None = None + ) -> None: ... + @property + def closefd(self) -> bool: ... + def seek(self, pos: int, whence: int = 0, /) -> int: ... + def read(self, size: int | None = -1, /) -> bytes | MaybeNone: ... + +class BytesIO(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes + def __init__(self, initial_bytes: ReadableBuffer = b"") -> None: ... + # BytesIO does not contain a "name" field. This workaround is necessary + # to allow BytesIO sub-classes to add this field, as it is defined + # as a read-only property on IO[]. + name: Any + def getvalue(self) -> bytes: ... + def getbuffer(self) -> memoryview: ... + def read1(self, size: int | None = -1, /) -> bytes: ... + def readlines(self, size: int | None = None, /) -> list[bytes]: ... + def seek(self, pos: int, whence: int = 0, /) -> int: ... + +class BufferedReader(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes + raw: RawIOBase + def __init__(self, raw: RawIOBase, buffer_size: int = 8192) -> None: ... + def peek(self, size: int = 0, /) -> bytes: ... + def seek(self, target: int, whence: int = 0, /) -> int: ... + def truncate(self, pos: int | None = None, /) -> int: ... + +class BufferedWriter(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes + raw: RawIOBase + def __init__(self, raw: RawIOBase, buffer_size: int = 8192) -> None: ... + def write(self, buffer: ReadableBuffer, /) -> int: ... + def seek(self, target: int, whence: int = 0, /) -> int: ... + def truncate(self, pos: int | None = None, /) -> int: ... + +class BufferedRandom(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes + mode: str + name: Any + raw: RawIOBase + def __init__(self, raw: RawIOBase, buffer_size: int = 8192) -> None: ... + def seek(self, target: int, whence: int = 0, /) -> int: ... # stubtest needs this + def peek(self, size: int = 0, /) -> bytes: ... + def truncate(self, pos: int | None = None, /) -> int: ... + +class BufferedRWPair(BufferedIOBase, _BufferedIOBase): + def __init__(self, reader: RawIOBase, writer: RawIOBase, buffer_size: int = 8192) -> None: ... + def peek(self, size: int = 0, /) -> bytes: ... + +class _TextIOBase(_IOBase): + encoding: str + errors: str | None + newlines: str | tuple[str, ...] | None + def __iter__(self) -> Iterator[str]: ... # type: ignore[override] + def __next__(self) -> str: ... # type: ignore[override] + def detach(self) -> BinaryIO: ... + def write(self, s: str, /) -> int: ... + def writelines(self, lines: Iterable[str], /) -> None: ... # type: ignore[override] + def readline(self, size: int = -1, /) -> str: ... # type: ignore[override] + def readlines(self, hint: int = -1, /) -> list[str]: ... # type: ignore[override] + def read(self, size: int | None = -1, /) -> str: ... + +@type_check_only +class _WrappedBuffer(Protocol): + # "name" is wrapped by TextIOWrapper. Its type is inconsistent between + # the various I/O types, see the comments on TextIOWrapper.name and + # TextIO.name. + @property + def name(self) -> Any: ... + @property + def closed(self) -> bool: ... + def read(self, size: int = ..., /) -> ReadableBuffer: ... + # Optional: def read1(self, size: int, /) -> ReadableBuffer: ... + def write(self, b: bytes, /) -> object: ... + def flush(self) -> object: ... + def close(self) -> object: ... + def seekable(self) -> bool: ... + def readable(self) -> bool: ... + def writable(self) -> bool: ... + def truncate(self, size: int, /) -> int: ... + def fileno(self) -> int: ... + def isatty(self) -> bool: ... + # Optional: Only needs to be present if seekable() returns True. + # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... + # def tell(self) -> int: ... + +_BufferT_co = TypeVar("_BufferT_co", bound=_WrappedBuffer, default=_WrappedBuffer, covariant=True) + +class TextIOWrapper(TextIOBase, _TextIOBase, TextIO, Generic[_BufferT_co]): # type: ignore[misc] # incompatible definitions of write in the base classes + def __init__( + self, + buffer: _BufferT_co, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool = False, + write_through: bool = False, + ) -> None: ... + # Equals the "buffer" argument passed in to the constructor. + @property + def buffer(self) -> _BufferT_co: ... # type: ignore[override] + @property + def line_buffering(self) -> bool: ... + @property + def write_through(self) -> bool: ... + def reconfigure( + self, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool | None = None, + write_through: bool | None = None, + ) -> None: ... + def readline(self, size: int = -1, /) -> str: ... # type: ignore[override] + # Equals the "buffer" argument passed in to the constructor. + def detach(self) -> _BufferT_co: ... # type: ignore[override] + # TextIOWrapper's version of seek only supports a limited subset of + # operations. + def seek(self, cookie: int, whence: int = 0, /) -> int: ... + def truncate(self, pos: int | None = None, /) -> int: ... + +class StringIO(TextIOBase, _TextIOBase, TextIO): # type: ignore[misc] # incompatible definitions of write in the base classes + def __init__(self, initial_value: str | None = "", newline: str | None = "\n") -> None: ... + # StringIO does not contain a "name" field. This workaround is necessary + # to allow StringIO sub-classes to add this field, as it is defined + # as a read-only property on IO[]. + name: Any + def getvalue(self) -> str: ... + @property + def line_buffering(self) -> bool: ... + def seek(self, pos: int, whence: int = 0, /) -> int: ... + def truncate(self, pos: int | None = None, /) -> int: ... + +class IncrementalNewlineDecoder: + def __init__(self, decoder: codecs.IncrementalDecoder | None, translate: bool, errors: str = "strict") -> None: ... + def decode(self, input: ReadableBuffer | str, final: bool = False) -> str: ... + @property + def newlines(self) -> str | tuple[str, ...] | None: ... + def getstate(self) -> tuple[bytes, int]: ... + def reset(self) -> None: ... + def setstate(self, state: tuple[bytes, int], /) -> None: ... + +if sys.version_info >= (3, 10): + @overload + def text_encoding(encoding: None, stacklevel: int = 2, /) -> Literal["locale", "utf-8"]: ... + @overload + def text_encoding(encoding: _T, stacklevel: int = 2, /) -> _T: ... diff --git a/mypy/typeshed/stdlib/_json.pyi b/mypy/typeshed/stdlib/_json.pyi new file mode 100644 index 000000000000..e1c7c52ca3b1 --- /dev/null +++ b/mypy/typeshed/stdlib/_json.pyi @@ -0,0 +1,50 @@ +from collections.abc import Callable +from typing import Any, final + +@final +class make_encoder: + @property + def sort_keys(self) -> bool: ... + @property + def skipkeys(self) -> bool: ... + @property + def key_separator(self) -> str: ... + @property + def indent(self) -> int | None: ... + @property + def markers(self) -> dict[int, Any] | None: ... + @property + def default(self) -> Callable[[Any], Any]: ... + @property + def encoder(self) -> Callable[[str], str]: ... + @property + def item_separator(self) -> str: ... + def __init__( + self, + markers: dict[int, Any] | None, + default: Callable[[Any], Any], + encoder: Callable[[str], str], + indent: int | None, + key_separator: str, + item_separator: str, + sort_keys: bool, + skipkeys: bool, + allow_nan: bool, + ) -> None: ... + def __call__(self, obj: object, _current_indent_level: int) -> Any: ... + +@final +class make_scanner: + object_hook: Any + object_pairs_hook: Any + parse_int: Any + parse_constant: Any + parse_float: Any + strict: bool + # TODO: 'context' needs the attrs above (ducktype), but not __call__. + def __init__(self, context: make_scanner) -> None: ... + def __call__(self, string: str, index: int) -> tuple[Any, int]: ... + +def encode_basestring(s: str, /) -> str: ... +def encode_basestring_ascii(s: str, /) -> str: ... +def scanstring(string: str, end: int, strict: bool = ...) -> tuple[str, int]: ... diff --git a/mypy/typeshed/stdlib/_locale.pyi b/mypy/typeshed/stdlib/_locale.pyi new file mode 100644 index 000000000000..ccce7a0d9d70 --- /dev/null +++ b/mypy/typeshed/stdlib/_locale.pyi @@ -0,0 +1,121 @@ +import sys +from _typeshed import StrPath +from typing import Final, Literal, TypedDict, type_check_only + +@type_check_only +class _LocaleConv(TypedDict): + decimal_point: str + grouping: list[int] + thousands_sep: str + int_curr_symbol: str + currency_symbol: str + p_cs_precedes: Literal[0, 1, 127] + n_cs_precedes: Literal[0, 1, 127] + p_sep_by_space: Literal[0, 1, 127] + n_sep_by_space: Literal[0, 1, 127] + mon_decimal_point: str + frac_digits: int + int_frac_digits: int + mon_thousands_sep: str + mon_grouping: list[int] + positive_sign: str + negative_sign: str + p_sign_posn: Literal[0, 1, 2, 3, 4, 127] + n_sign_posn: Literal[0, 1, 2, 3, 4, 127] + +LC_CTYPE: Final[int] +LC_COLLATE: Final[int] +LC_TIME: Final[int] +LC_MONETARY: Final[int] +LC_NUMERIC: Final[int] +LC_ALL: Final[int] +CHAR_MAX: Final = 127 + +def setlocale(category: int, locale: str | None = None, /) -> str: ... +def localeconv() -> _LocaleConv: ... + +if sys.version_info >= (3, 11): + def getencoding() -> str: ... + +def strcoll(os1: str, os2: str, /) -> int: ... +def strxfrm(string: str, /) -> str: ... + +# native gettext functions +# https://docs.python.org/3/library/locale.html#access-to-message-catalogs +# https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Modules/_localemodule.c#L626 +if sys.platform != "win32": + LC_MESSAGES: int + + ABDAY_1: Final[int] + ABDAY_2: Final[int] + ABDAY_3: Final[int] + ABDAY_4: Final[int] + ABDAY_5: Final[int] + ABDAY_6: Final[int] + ABDAY_7: Final[int] + + ABMON_1: Final[int] + ABMON_2: Final[int] + ABMON_3: Final[int] + ABMON_4: Final[int] + ABMON_5: Final[int] + ABMON_6: Final[int] + ABMON_7: Final[int] + ABMON_8: Final[int] + ABMON_9: Final[int] + ABMON_10: Final[int] + ABMON_11: Final[int] + ABMON_12: Final[int] + + DAY_1: Final[int] + DAY_2: Final[int] + DAY_3: Final[int] + DAY_4: Final[int] + DAY_5: Final[int] + DAY_6: Final[int] + DAY_7: Final[int] + + ERA: Final[int] + ERA_D_T_FMT: Final[int] + ERA_D_FMT: Final[int] + ERA_T_FMT: Final[int] + + MON_1: Final[int] + MON_2: Final[int] + MON_3: Final[int] + MON_4: Final[int] + MON_5: Final[int] + MON_6: Final[int] + MON_7: Final[int] + MON_8: Final[int] + MON_9: Final[int] + MON_10: Final[int] + MON_11: Final[int] + MON_12: Final[int] + + CODESET: Final[int] + D_T_FMT: Final[int] + D_FMT: Final[int] + T_FMT: Final[int] + T_FMT_AMPM: Final[int] + AM_STR: Final[int] + PM_STR: Final[int] + + RADIXCHAR: Final[int] + THOUSEP: Final[int] + YESEXPR: Final[int] + NOEXPR: Final[int] + CRNCYSTR: Final[int] + ALT_DIGITS: Final[int] + + def nl_langinfo(key: int, /) -> str: ... + + # This is dependent on `libintl.h` which is a part of `gettext` + # system dependency. These functions might be missing. + # But, we always say that they are present. + def gettext(msg: str, /) -> str: ... + def dgettext(domain: str | None, msg: str, /) -> str: ... + def dcgettext(domain: str | None, msg: str, category: int, /) -> str: ... + def textdomain(domain: str | None, /) -> str: ... + def bindtextdomain(domain: str, dir: StrPath | None, /) -> str: ... + def bind_textdomain_codeset(domain: str, codeset: str | None, /) -> str | None: ... diff --git a/mypy/typeshed/stdlib/_lsprof.pyi b/mypy/typeshed/stdlib/_lsprof.pyi new file mode 100644 index 000000000000..8a6934162c92 --- /dev/null +++ b/mypy/typeshed/stdlib/_lsprof.pyi @@ -0,0 +1,35 @@ +import sys +from _typeshed import structseq +from collections.abc import Callable +from types import CodeType +from typing import Any, Final, final + +class Profiler: + def __init__( + self, timer: Callable[[], float] | None = None, timeunit: float = 0.0, subcalls: bool = True, builtins: bool = True + ) -> None: ... + def getstats(self) -> list[profiler_entry]: ... + def enable(self, subcalls: bool = True, builtins: bool = True) -> None: ... + def disable(self) -> None: ... + def clear(self) -> None: ... + +@final +class profiler_entry(structseq[Any], tuple[CodeType | str, int, int, float, float, list[profiler_subentry]]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("code", "callcount", "reccallcount", "totaltime", "inlinetime", "calls") + code: CodeType | str + callcount: int + reccallcount: int + totaltime: float + inlinetime: float + calls: list[profiler_subentry] + +@final +class profiler_subentry(structseq[Any], tuple[CodeType | str, int, int, float, float]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("code", "callcount", "reccallcount", "totaltime", "inlinetime") + code: CodeType | str + callcount: int + reccallcount: int + totaltime: float + inlinetime: float diff --git a/mypy/typeshed/stdlib/_lzma.pyi b/mypy/typeshed/stdlib/_lzma.pyi new file mode 100644 index 000000000000..1f5be78876c6 --- /dev/null +++ b/mypy/typeshed/stdlib/_lzma.pyi @@ -0,0 +1,60 @@ +from _typeshed import ReadableBuffer +from collections.abc import Mapping, Sequence +from typing import Any, Final, final +from typing_extensions import TypeAlias + +_FilterChain: TypeAlias = Sequence[Mapping[str, Any]] + +FORMAT_AUTO: Final = 0 +FORMAT_XZ: Final = 1 +FORMAT_ALONE: Final = 2 +FORMAT_RAW: Final = 3 +CHECK_NONE: Final = 0 +CHECK_CRC32: Final = 1 +CHECK_CRC64: Final = 4 +CHECK_SHA256: Final = 10 +CHECK_ID_MAX: Final = 15 +CHECK_UNKNOWN: Final = 16 +FILTER_LZMA1: int # v big number +FILTER_LZMA2: Final = 33 +FILTER_DELTA: Final = 3 +FILTER_X86: Final = 4 +FILTER_IA64: Final = 6 +FILTER_ARM: Final = 7 +FILTER_ARMTHUMB: Final = 8 +FILTER_SPARC: Final = 9 +FILTER_POWERPC: Final = 5 +MF_HC3: Final = 3 +MF_HC4: Final = 4 +MF_BT2: Final = 18 +MF_BT3: Final = 19 +MF_BT4: Final = 20 +MODE_FAST: Final = 1 +MODE_NORMAL: Final = 2 +PRESET_DEFAULT: Final = 6 +PRESET_EXTREME: int # v big number + +@final +class LZMADecompressor: + def __init__(self, format: int | None = ..., memlimit: int | None = ..., filters: _FilterChain | None = ...) -> None: ... + def decompress(self, data: ReadableBuffer, max_length: int = -1) -> bytes: ... + @property + def check(self) -> int: ... + @property + def eof(self) -> bool: ... + @property + def unused_data(self) -> bytes: ... + @property + def needs_input(self) -> bool: ... + +@final +class LZMACompressor: + def __init__( + self, format: int | None = ..., check: int = ..., preset: int | None = ..., filters: _FilterChain | None = ... + ) -> None: ... + def compress(self, data: ReadableBuffer, /) -> bytes: ... + def flush(self) -> bytes: ... + +class LZMAError(Exception): ... + +def is_check_supported(check_id: int, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/_markupbase.pyi b/mypy/typeshed/stdlib/_markupbase.pyi new file mode 100644 index 000000000000..62bad25e5ccc --- /dev/null +++ b/mypy/typeshed/stdlib/_markupbase.pyi @@ -0,0 +1,16 @@ +import sys +from typing import Any + +class ParserBase: + def reset(self) -> None: ... + def getpos(self) -> tuple[int, int]: ... + def unknown_decl(self, data: str) -> None: ... + def parse_comment(self, i: int, report: int = 1) -> int: ... # undocumented + def parse_declaration(self, i: int) -> int: ... # undocumented + def parse_marked_section(self, i: int, report: int = 1) -> int: ... # undocumented + def updatepos(self, i: int, j: int) -> int: ... # undocumented + if sys.version_info < (3, 10): + # Removed from ParserBase: https://bugs.python.org/issue31844 + def error(self, message: str) -> Any: ... # undocumented + lineno: int # undocumented + offset: int # undocumented diff --git a/mypy/typeshed/stdlib/_msi.pyi b/mypy/typeshed/stdlib/_msi.pyi new file mode 100644 index 000000000000..779fda3b67fe --- /dev/null +++ b/mypy/typeshed/stdlib/_msi.pyi @@ -0,0 +1,92 @@ +import sys + +if sys.platform == "win32": + class MSIError(Exception): ... + # Actual typename View, not exposed by the implementation + class _View: + def Execute(self, params: _Record | None = ...) -> None: ... + def GetColumnInfo(self, kind: int) -> _Record: ... + def Fetch(self) -> _Record: ... + def Modify(self, mode: int, record: _Record) -> None: ... + def Close(self) -> None: ... + # Don't exist at runtime + __new__: None # type: ignore[assignment] + __init__: None # type: ignore[assignment] + + # Actual typename SummaryInformation, not exposed by the implementation + class _SummaryInformation: + def GetProperty(self, field: int) -> int | bytes | None: ... + def GetPropertyCount(self) -> int: ... + def SetProperty(self, field: int, value: int | str) -> None: ... + def Persist(self) -> None: ... + # Don't exist at runtime + __new__: None # type: ignore[assignment] + __init__: None # type: ignore[assignment] + + # Actual typename Database, not exposed by the implementation + class _Database: + def OpenView(self, sql: str) -> _View: ... + def Commit(self) -> None: ... + def GetSummaryInformation(self, updateCount: int) -> _SummaryInformation: ... + def Close(self) -> None: ... + # Don't exist at runtime + __new__: None # type: ignore[assignment] + __init__: None # type: ignore[assignment] + + # Actual typename Record, not exposed by the implementation + class _Record: + def GetFieldCount(self) -> int: ... + def GetInteger(self, field: int) -> int: ... + def GetString(self, field: int) -> str: ... + def SetString(self, field: int, str: str) -> None: ... + def SetStream(self, field: int, stream: str) -> None: ... + def SetInteger(self, field: int, int: int) -> None: ... + def ClearData(self) -> None: ... + # Don't exist at runtime + __new__: None # type: ignore[assignment] + __init__: None # type: ignore[assignment] + + def UuidCreate() -> str: ... + def FCICreate(cabname: str, files: list[str], /) -> None: ... + def OpenDatabase(path: str, persist: int, /) -> _Database: ... + def CreateRecord(count: int, /) -> _Record: ... + + MSICOLINFO_NAMES: int + MSICOLINFO_TYPES: int + MSIDBOPEN_CREATE: int + MSIDBOPEN_CREATEDIRECT: int + MSIDBOPEN_DIRECT: int + MSIDBOPEN_PATCHFILE: int + MSIDBOPEN_READONLY: int + MSIDBOPEN_TRANSACT: int + MSIMODIFY_ASSIGN: int + MSIMODIFY_DELETE: int + MSIMODIFY_INSERT: int + MSIMODIFY_INSERT_TEMPORARY: int + MSIMODIFY_MERGE: int + MSIMODIFY_REFRESH: int + MSIMODIFY_REPLACE: int + MSIMODIFY_SEEK: int + MSIMODIFY_UPDATE: int + MSIMODIFY_VALIDATE: int + MSIMODIFY_VALIDATE_DELETE: int + MSIMODIFY_VALIDATE_FIELD: int + MSIMODIFY_VALIDATE_NEW: int + + PID_APPNAME: int + PID_AUTHOR: int + PID_CHARCOUNT: int + PID_CODEPAGE: int + PID_COMMENTS: int + PID_CREATE_DTM: int + PID_KEYWORDS: int + PID_LASTAUTHOR: int + PID_LASTPRINTED: int + PID_LASTSAVE_DTM: int + PID_PAGECOUNT: int + PID_REVNUMBER: int + PID_SECURITY: int + PID_SUBJECT: int + PID_TEMPLATE: int + PID_TITLE: int + PID_WORDCOUNT: int diff --git a/mypy/typeshed/stdlib/_multibytecodec.pyi b/mypy/typeshed/stdlib/_multibytecodec.pyi new file mode 100644 index 000000000000..7e408f2aa30e --- /dev/null +++ b/mypy/typeshed/stdlib/_multibytecodec.pyi @@ -0,0 +1,44 @@ +from _typeshed import ReadableBuffer +from codecs import _ReadableStream, _WritableStream +from collections.abc import Iterable +from typing import final, type_check_only + +# This class is not exposed. It calls itself _multibytecodec.MultibyteCodec. +@final +@type_check_only +class _MultibyteCodec: + def decode(self, input: ReadableBuffer, errors: str | None = None) -> str: ... + def encode(self, input: str, errors: str | None = None) -> bytes: ... + +class MultibyteIncrementalDecoder: + errors: str + def __init__(self, errors: str = "strict") -> None: ... + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + def getstate(self) -> tuple[bytes, int]: ... + def reset(self) -> None: ... + def setstate(self, state: tuple[bytes, int], /) -> None: ... + +class MultibyteIncrementalEncoder: + errors: str + def __init__(self, errors: str = "strict") -> None: ... + def encode(self, input: str, final: bool = False) -> bytes: ... + def getstate(self) -> int: ... + def reset(self) -> None: ... + def setstate(self, state: int, /) -> None: ... + +class MultibyteStreamReader: + errors: str + stream: _ReadableStream + def __init__(self, stream: _ReadableStream, errors: str = "strict") -> None: ... + def read(self, sizeobj: int | None = None, /) -> str: ... + def readline(self, sizeobj: int | None = None, /) -> str: ... + def readlines(self, sizehintobj: int | None = None, /) -> list[str]: ... + def reset(self) -> None: ... + +class MultibyteStreamWriter: + errors: str + stream: _WritableStream + def __init__(self, stream: _WritableStream, errors: str = "strict") -> None: ... + def reset(self) -> None: ... + def write(self, strobj: str, /) -> None: ... + def writelines(self, lines: Iterable[str], /) -> None: ... diff --git a/mypy/typeshed/stdlib/_operator.pyi b/mypy/typeshed/stdlib/_operator.pyi new file mode 100644 index 000000000000..967215d8fa21 --- /dev/null +++ b/mypy/typeshed/stdlib/_operator.pyi @@ -0,0 +1,115 @@ +import sys +from _typeshed import SupportsGetItem +from collections.abc import Callable, Container, Iterable, MutableMapping, MutableSequence, Sequence +from operator import attrgetter as attrgetter, itemgetter as itemgetter, methodcaller as methodcaller +from typing import Any, AnyStr, Protocol, SupportsAbs, SupportsIndex, TypeVar, overload +from typing_extensions import ParamSpec, TypeAlias, TypeIs + +_R = TypeVar("_R") +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_K = TypeVar("_K") +_V = TypeVar("_V") +_P = ParamSpec("_P") + +# The following protocols return "Any" instead of bool, since the comparison +# operators can be overloaded to return an arbitrary object. For example, +# the numpy.array comparison dunders return another numpy.array. + +class _SupportsDunderLT(Protocol): + def __lt__(self, other: Any, /) -> Any: ... + +class _SupportsDunderGT(Protocol): + def __gt__(self, other: Any, /) -> Any: ... + +class _SupportsDunderLE(Protocol): + def __le__(self, other: Any, /) -> Any: ... + +class _SupportsDunderGE(Protocol): + def __ge__(self, other: Any, /) -> Any: ... + +_SupportsComparison: TypeAlias = _SupportsDunderLE | _SupportsDunderGE | _SupportsDunderGT | _SupportsDunderLT + +class _SupportsInversion(Protocol[_T_co]): + def __invert__(self) -> _T_co: ... + +class _SupportsNeg(Protocol[_T_co]): + def __neg__(self) -> _T_co: ... + +class _SupportsPos(Protocol[_T_co]): + def __pos__(self) -> _T_co: ... + +# All four comparison functions must have the same signature, or we get false-positive errors +def lt(a: _SupportsComparison, b: _SupportsComparison, /) -> Any: ... +def le(a: _SupportsComparison, b: _SupportsComparison, /) -> Any: ... +def eq(a: object, b: object, /) -> Any: ... +def ne(a: object, b: object, /) -> Any: ... +def ge(a: _SupportsComparison, b: _SupportsComparison, /) -> Any: ... +def gt(a: _SupportsComparison, b: _SupportsComparison, /) -> Any: ... +def not_(a: object, /) -> bool: ... +def truth(a: object, /) -> bool: ... +def is_(a: object, b: object, /) -> bool: ... +def is_not(a: object, b: object, /) -> bool: ... +def abs(a: SupportsAbs[_T], /) -> _T: ... +def add(a: Any, b: Any, /) -> Any: ... +def and_(a: Any, b: Any, /) -> Any: ... +def floordiv(a: Any, b: Any, /) -> Any: ... +def index(a: SupportsIndex, /) -> int: ... +def inv(a: _SupportsInversion[_T_co], /) -> _T_co: ... +def invert(a: _SupportsInversion[_T_co], /) -> _T_co: ... +def lshift(a: Any, b: Any, /) -> Any: ... +def mod(a: Any, b: Any, /) -> Any: ... +def mul(a: Any, b: Any, /) -> Any: ... +def matmul(a: Any, b: Any, /) -> Any: ... +def neg(a: _SupportsNeg[_T_co], /) -> _T_co: ... +def or_(a: Any, b: Any, /) -> Any: ... +def pos(a: _SupportsPos[_T_co], /) -> _T_co: ... +def pow(a: Any, b: Any, /) -> Any: ... +def rshift(a: Any, b: Any, /) -> Any: ... +def sub(a: Any, b: Any, /) -> Any: ... +def truediv(a: Any, b: Any, /) -> Any: ... +def xor(a: Any, b: Any, /) -> Any: ... +def concat(a: Sequence[_T], b: Sequence[_T], /) -> Sequence[_T]: ... +def contains(a: Container[object], b: object, /) -> bool: ... +def countOf(a: Iterable[object], b: object, /) -> int: ... +@overload +def delitem(a: MutableSequence[Any], b: SupportsIndex, /) -> None: ... +@overload +def delitem(a: MutableSequence[Any], b: slice, /) -> None: ... +@overload +def delitem(a: MutableMapping[_K, Any], b: _K, /) -> None: ... +@overload +def getitem(a: Sequence[_T], b: slice, /) -> Sequence[_T]: ... +@overload +def getitem(a: SupportsGetItem[_K, _V], b: _K, /) -> _V: ... +def indexOf(a: Iterable[_T], b: _T, /) -> int: ... +@overload +def setitem(a: MutableSequence[_T], b: SupportsIndex, c: _T, /) -> None: ... +@overload +def setitem(a: MutableSequence[_T], b: slice, c: Sequence[_T], /) -> None: ... +@overload +def setitem(a: MutableMapping[_K, _V], b: _K, c: _V, /) -> None: ... +def length_hint(obj: object, default: int = 0, /) -> int: ... +def iadd(a: Any, b: Any, /) -> Any: ... +def iand(a: Any, b: Any, /) -> Any: ... +def iconcat(a: Any, b: Any, /) -> Any: ... +def ifloordiv(a: Any, b: Any, /) -> Any: ... +def ilshift(a: Any, b: Any, /) -> Any: ... +def imod(a: Any, b: Any, /) -> Any: ... +def imul(a: Any, b: Any, /) -> Any: ... +def imatmul(a: Any, b: Any, /) -> Any: ... +def ior(a: Any, b: Any, /) -> Any: ... +def ipow(a: Any, b: Any, /) -> Any: ... +def irshift(a: Any, b: Any, /) -> Any: ... +def isub(a: Any, b: Any, /) -> Any: ... +def itruediv(a: Any, b: Any, /) -> Any: ... +def ixor(a: Any, b: Any, /) -> Any: ... + +if sys.version_info >= (3, 11): + def call(obj: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs) -> _R: ... + +def _compare_digest(a: AnyStr, b: AnyStr, /) -> bool: ... + +if sys.version_info >= (3, 14): + def is_none(a: object, /) -> TypeIs[None]: ... + def is_not_none(a: _T | None, /) -> TypeIs[_T]: ... diff --git a/mypy/typeshed/stdlib/_osx_support.pyi b/mypy/typeshed/stdlib/_osx_support.pyi new file mode 100644 index 000000000000..fb00e6986dd0 --- /dev/null +++ b/mypy/typeshed/stdlib/_osx_support.pyi @@ -0,0 +1,34 @@ +from collections.abc import Iterable, Sequence +from typing import Final, TypeVar + +_T = TypeVar("_T") +_K = TypeVar("_K") +_V = TypeVar("_V") + +__all__ = ["compiler_fixup", "customize_config_vars", "customize_compiler", "get_platform_osx"] + +_UNIVERSAL_CONFIG_VARS: Final[tuple[str, ...]] # undocumented +_COMPILER_CONFIG_VARS: Final[tuple[str, ...]] # undocumented +_INITPRE: Final[str] # undocumented + +def _find_executable(executable: str, path: str | None = None) -> str | None: ... # undocumented +def _read_output(commandstring: str, capture_stderr: bool = False) -> str | None: ... # undocumented +def _find_build_tool(toolname: str) -> str: ... # undocumented + +_SYSTEM_VERSION: Final[str | None] # undocumented + +def _get_system_version() -> str: ... # undocumented +def _remove_original_values(_config_vars: dict[str, str]) -> None: ... # undocumented +def _save_modified_value(_config_vars: dict[str, str], cv: str, newvalue: str) -> None: ... # undocumented +def _supports_universal_builds() -> bool: ... # undocumented +def _find_appropriate_compiler(_config_vars: dict[str, str]) -> dict[str, str]: ... # undocumented +def _remove_universal_flags(_config_vars: dict[str, str]) -> dict[str, str]: ... # undocumented +def _remove_unsupported_archs(_config_vars: dict[str, str]) -> dict[str, str]: ... # undocumented +def _override_all_archs(_config_vars: dict[str, str]) -> dict[str, str]: ... # undocumented +def _check_for_unavailable_sdk(_config_vars: dict[str, str]) -> dict[str, str]: ... # undocumented +def compiler_fixup(compiler_so: Iterable[str], cc_args: Sequence[str]) -> list[str]: ... +def customize_config_vars(_config_vars: dict[str, str]) -> dict[str, str]: ... +def customize_compiler(_config_vars: dict[str, str]) -> dict[str, str]: ... +def get_platform_osx( + _config_vars: dict[str, str], osname: _T, release: _K, machine: _V +) -> tuple[str | _T, str | _K, str | _V]: ... diff --git a/mypy/typeshed/stdlib/_posixsubprocess.pyi b/mypy/typeshed/stdlib/_posixsubprocess.pyi new file mode 100644 index 000000000000..df05dcd80be8 --- /dev/null +++ b/mypy/typeshed/stdlib/_posixsubprocess.pyi @@ -0,0 +1,32 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Callable, Sequence +from typing import SupportsIndex + +if sys.platform != "win32": + def fork_exec( + args: Sequence[StrOrBytesPath] | None, + executable_list: Sequence[bytes], + close_fds: bool, + pass_fds: tuple[int, ...], + cwd: str, + env: Sequence[bytes] | None, + p2cread: int, + p2cwrite: int, + c2pread: int, + c2pwrite: int, + errread: int, + errwrite: int, + errpipe_read: int, + errpipe_write: int, + restore_signals: int, + call_setsid: int, + pgid_to_set: int, + gid: SupportsIndex | None, + extra_groups: list[int] | None, + uid: SupportsIndex | None, + child_umask: int, + preexec_fn: Callable[[], None], + allow_vfork: bool, + /, + ) -> int: ... diff --git a/mypy/typeshed/stdlib/_py_abc.pyi b/mypy/typeshed/stdlib/_py_abc.pyi new file mode 100644 index 000000000000..1260717489e4 --- /dev/null +++ b/mypy/typeshed/stdlib/_py_abc.pyi @@ -0,0 +1,14 @@ +import _typeshed +from typing import Any, NewType, TypeVar + +_T = TypeVar("_T") + +_CacheToken = NewType("_CacheToken", int) + +def get_cache_token() -> _CacheToken: ... + +class ABCMeta(type): + def __new__( + mcls: type[_typeshed.Self], name: str, bases: tuple[type[Any], ...], namespace: dict[str, Any], / + ) -> _typeshed.Self: ... + def register(cls, subclass: type[_T]) -> type[_T]: ... diff --git a/mypy/typeshed/stdlib/_pydecimal.pyi b/mypy/typeshed/stdlib/_pydecimal.pyi new file mode 100644 index 000000000000..faff626ac0ba --- /dev/null +++ b/mypy/typeshed/stdlib/_pydecimal.pyi @@ -0,0 +1,43 @@ +# This is a slight lie, the implementations aren't exactly identical +# However, in all likelihood, the differences are inconsequential +from _decimal import * + +__all__ = [ + "Decimal", + "Context", + "DecimalTuple", + "DefaultContext", + "BasicContext", + "ExtendedContext", + "DecimalException", + "Clamped", + "InvalidOperation", + "DivisionByZero", + "Inexact", + "Rounded", + "Subnormal", + "Overflow", + "Underflow", + "FloatOperation", + "DivisionImpossible", + "InvalidContext", + "ConversionSyntax", + "DivisionUndefined", + "ROUND_DOWN", + "ROUND_HALF_UP", + "ROUND_HALF_EVEN", + "ROUND_CEILING", + "ROUND_FLOOR", + "ROUND_UP", + "ROUND_HALF_DOWN", + "ROUND_05UP", + "setcontext", + "getcontext", + "localcontext", + "MAX_PREC", + "MAX_EMAX", + "MIN_EMIN", + "MIN_ETINY", + "HAVE_THREADS", + "HAVE_CONTEXTVAR", +] diff --git a/mypy/typeshed/stdlib/_queue.pyi b/mypy/typeshed/stdlib/_queue.pyi new file mode 100644 index 000000000000..0d4caea7442e --- /dev/null +++ b/mypy/typeshed/stdlib/_queue.pyi @@ -0,0 +1,20 @@ +import sys +from typing import Any, Generic, TypeVar + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") + +class Empty(Exception): ... + +class SimpleQueue(Generic[_T]): + def __init__(self) -> None: ... + def empty(self) -> bool: ... + def get(self, block: bool = True, timeout: float | None = None) -> _T: ... + def get_nowait(self) -> _T: ... + def put(self, item: _T, block: bool = True, timeout: float | None = None) -> None: ... + def put_nowait(self, item: _T) -> None: ... + def qsize(self) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... diff --git a/mypy/typeshed/stdlib/_random.pyi b/mypy/typeshed/stdlib/_random.pyi new file mode 100644 index 000000000000..4082344ade8e --- /dev/null +++ b/mypy/typeshed/stdlib/_random.pyi @@ -0,0 +1,12 @@ +from typing_extensions import TypeAlias + +# Actually Tuple[(int,) * 625] +_State: TypeAlias = tuple[int, ...] + +class Random: + def __init__(self, seed: object = ...) -> None: ... + def seed(self, n: object = None, /) -> None: ... + def getstate(self) -> _State: ... + def setstate(self, state: _State, /) -> None: ... + def random(self) -> float: ... + def getrandbits(self, k: int, /) -> int: ... diff --git a/mypy/typeshed/stdlib/_sitebuiltins.pyi b/mypy/typeshed/stdlib/_sitebuiltins.pyi new file mode 100644 index 000000000000..49e88a196825 --- /dev/null +++ b/mypy/typeshed/stdlib/_sitebuiltins.pyi @@ -0,0 +1,16 @@ +from collections.abc import Iterable +from typing import ClassVar, Literal, NoReturn + +class Quitter: + name: str + eof: str + def __init__(self, name: str, eof: str) -> None: ... + def __call__(self, code: int | None = None) -> NoReturn: ... + +class _Printer: + MAXLINES: ClassVar[Literal[23]] + def __init__(self, name: str, data: str, files: Iterable[str] = (), dirs: Iterable[str] = ()) -> None: ... + def __call__(self) -> None: ... + +class _Helper: + def __call__(self, request: object) -> None: ... diff --git a/mypy/typeshed/stdlib/_socket.pyi b/mypy/typeshed/stdlib/_socket.pyi new file mode 100644 index 000000000000..36bc5c31c646 --- /dev/null +++ b/mypy/typeshed/stdlib/_socket.pyi @@ -0,0 +1,845 @@ +import sys +from _typeshed import ReadableBuffer, WriteableBuffer +from collections.abc import Iterable +from socket import error as error, gaierror as gaierror, herror as herror, timeout as timeout +from typing import Any, SupportsIndex, overload +from typing_extensions import CapsuleType, TypeAlias + +_CMSG: TypeAlias = tuple[int, int, bytes] +_CMSGArg: TypeAlias = tuple[int, int, ReadableBuffer] + +# Addresses can be either tuples of varying lengths (AF_INET, AF_INET6, +# AF_NETLINK, AF_TIPC) or strings/buffers (AF_UNIX). +# See getsockaddrarg() in socketmodule.c. +_Address: TypeAlias = tuple[Any, ...] | str | ReadableBuffer +_RetAddress: TypeAlias = Any + +# ===== Constants ===== +# This matches the order in the CPython documentation +# https://docs.python.org/3/library/socket.html#constants + +if sys.platform != "win32": + AF_UNIX: int + +AF_INET: int +AF_INET6: int + +AF_UNSPEC: int + +SOCK_STREAM: int +SOCK_DGRAM: int +SOCK_RAW: int +SOCK_RDM: int +SOCK_SEQPACKET: int + +if sys.platform == "linux": + # Availability: Linux >= 2.6.27 + SOCK_CLOEXEC: int + SOCK_NONBLOCK: int + +# -------------------- +# Many constants of these forms, documented in the Unix documentation on +# sockets and/or the IP protocol, are also defined in the socket module. +# SO_* +# socket.SOMAXCONN +# MSG_* +# SOL_* +# SCM_* +# IPPROTO_* +# IPPORT_* +# INADDR_* +# IP_* +# IPV6_* +# EAI_* +# AI_* +# NI_* +# TCP_* +# -------------------- + +SO_ACCEPTCONN: int +SO_BROADCAST: int +SO_DEBUG: int +SO_DONTROUTE: int +SO_ERROR: int +SO_KEEPALIVE: int +SO_LINGER: int +SO_OOBINLINE: int +SO_RCVBUF: int +SO_RCVLOWAT: int +SO_RCVTIMEO: int +SO_REUSEADDR: int +SO_SNDBUF: int +SO_SNDLOWAT: int +SO_SNDTIMEO: int +SO_TYPE: int +if sys.platform != "linux": + SO_USELOOPBACK: int +if sys.platform == "win32": + SO_EXCLUSIVEADDRUSE: int +if sys.platform != "win32": + SO_REUSEPORT: int +if sys.platform != "win32" and sys.platform != "darwin": + SO_BINDTODEVICE: int + SO_DOMAIN: int + SO_MARK: int + SO_PASSCRED: int + SO_PASSSEC: int + SO_PEERCRED: int + SO_PEERSEC: int + SO_PRIORITY: int + SO_PROTOCOL: int +if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + SO_SETFIB: int +if sys.platform == "linux" and sys.version_info >= (3, 13): + SO_BINDTOIFINDEX: int + +SOMAXCONN: int + +MSG_CTRUNC: int +MSG_DONTROUTE: int +MSG_OOB: int +MSG_PEEK: int +MSG_TRUNC: int +MSG_WAITALL: int +if sys.platform != "win32": + MSG_DONTWAIT: int + MSG_EOR: int + MSG_NOSIGNAL: int # Sometimes this exists on darwin, sometimes not +if sys.platform != "darwin": + MSG_ERRQUEUE: int +if sys.platform == "win32": + MSG_BCAST: int + MSG_MCAST: int +if sys.platform != "win32" and sys.platform != "darwin": + MSG_CMSG_CLOEXEC: int + MSG_CONFIRM: int + MSG_FASTOPEN: int + MSG_MORE: int +if sys.platform != "win32" and sys.platform != "linux": + MSG_EOF: int +if sys.platform != "win32" and sys.platform != "linux" and sys.platform != "darwin": + MSG_NOTIFICATION: int + MSG_BTAG: int # Not FreeBSD either + MSG_ETAG: int # Not FreeBSD either + +SOL_IP: int +SOL_SOCKET: int +SOL_TCP: int +SOL_UDP: int +if sys.platform != "win32" and sys.platform != "darwin": + # Defined in socket.h for Linux, but these aren't always present for + # some reason. + SOL_ATALK: int + SOL_AX25: int + SOL_HCI: int + SOL_IPX: int + SOL_NETROM: int + SOL_ROSE: int + +if sys.platform != "win32": + SCM_RIGHTS: int +if sys.platform != "win32" and sys.platform != "darwin": + SCM_CREDENTIALS: int +if sys.platform != "win32" and sys.platform != "linux": + SCM_CREDS: int + +IPPROTO_ICMP: int +IPPROTO_IP: int +IPPROTO_RAW: int +IPPROTO_TCP: int +IPPROTO_UDP: int +IPPROTO_AH: int +IPPROTO_DSTOPTS: int +IPPROTO_EGP: int +IPPROTO_ESP: int +IPPROTO_FRAGMENT: int +IPPROTO_HOPOPTS: int +IPPROTO_ICMPV6: int +IPPROTO_IDP: int +IPPROTO_IGMP: int +IPPROTO_IPV6: int +IPPROTO_NONE: int +IPPROTO_PIM: int +IPPROTO_PUP: int +IPPROTO_ROUTING: int +IPPROTO_SCTP: int +if sys.platform != "linux": + IPPROTO_GGP: int + IPPROTO_IPV4: int + IPPROTO_MAX: int + IPPROTO_ND: int +if sys.platform == "win32": + IPPROTO_CBT: int + IPPROTO_ICLFXBM: int + IPPROTO_IGP: int + IPPROTO_L2TP: int + IPPROTO_PGM: int + IPPROTO_RDP: int + IPPROTO_ST: int +if sys.platform != "win32": + IPPROTO_GRE: int + IPPROTO_IPIP: int + IPPROTO_RSVP: int + IPPROTO_TP: int +if sys.platform != "win32" and sys.platform != "linux": + IPPROTO_EON: int + IPPROTO_HELLO: int + IPPROTO_IPCOMP: int + IPPROTO_XTP: int +if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + IPPROTO_BIP: int # Not FreeBSD either + IPPROTO_MOBILE: int # Not FreeBSD either + IPPROTO_VRRP: int # Not FreeBSD either +if sys.version_info >= (3, 9) and sys.platform == "linux": + # Availability: Linux >= 2.6.20, FreeBSD >= 10.1 + IPPROTO_UDPLITE: int +if sys.version_info >= (3, 10) and sys.platform == "linux": + IPPROTO_MPTCP: int + +IPPORT_RESERVED: int +IPPORT_USERRESERVED: int + +INADDR_ALLHOSTS_GROUP: int +INADDR_ANY: int +INADDR_BROADCAST: int +INADDR_LOOPBACK: int +INADDR_MAX_LOCAL_GROUP: int +INADDR_NONE: int +INADDR_UNSPEC_GROUP: int + +IP_ADD_MEMBERSHIP: int +IP_DROP_MEMBERSHIP: int +IP_HDRINCL: int +IP_MULTICAST_IF: int +IP_MULTICAST_LOOP: int +IP_MULTICAST_TTL: int +IP_OPTIONS: int +if sys.platform != "linux": + IP_RECVDSTADDR: int +if sys.version_info >= (3, 10): + IP_RECVTOS: int +IP_TOS: int +IP_TTL: int +if sys.platform != "win32": + IP_DEFAULT_MULTICAST_LOOP: int + IP_DEFAULT_MULTICAST_TTL: int + IP_MAX_MEMBERSHIPS: int + IP_RECVOPTS: int + IP_RECVRETOPTS: int + IP_RETOPTS: int +if sys.platform != "win32" and sys.platform != "darwin": + IP_TRANSPARENT: int +if sys.platform != "win32" and sys.platform != "darwin" and sys.version_info >= (3, 11): + IP_BIND_ADDRESS_NO_PORT: int +if sys.version_info >= (3, 12): + IP_ADD_SOURCE_MEMBERSHIP: int + IP_BLOCK_SOURCE: int + IP_DROP_SOURCE_MEMBERSHIP: int + IP_PKTINFO: int + IP_UNBLOCK_SOURCE: int + +IPV6_CHECKSUM: int +IPV6_JOIN_GROUP: int +IPV6_LEAVE_GROUP: int +IPV6_MULTICAST_HOPS: int +IPV6_MULTICAST_IF: int +IPV6_MULTICAST_LOOP: int +IPV6_RECVTCLASS: int +IPV6_TCLASS: int +IPV6_UNICAST_HOPS: int +IPV6_V6ONLY: int +if sys.version_info >= (3, 9) or sys.platform != "darwin": + IPV6_DONTFRAG: int + IPV6_HOPLIMIT: int + IPV6_HOPOPTS: int + IPV6_PKTINFO: int + IPV6_RECVRTHDR: int + IPV6_RTHDR: int +if sys.platform != "win32": + IPV6_RTHDR_TYPE_0: int + if sys.version_info >= (3, 9) or sys.platform != "darwin": + IPV6_DSTOPTS: int + IPV6_NEXTHOP: int + IPV6_PATHMTU: int + IPV6_RECVDSTOPTS: int + IPV6_RECVHOPLIMIT: int + IPV6_RECVHOPOPTS: int + IPV6_RECVPATHMTU: int + IPV6_RECVPKTINFO: int + IPV6_RTHDRDSTOPTS: int + +if sys.platform != "win32" and sys.platform != "linux": + if sys.version_info >= (3, 9) or sys.platform != "darwin": + IPV6_USE_MIN_MTU: int + +EAI_AGAIN: int +EAI_BADFLAGS: int +EAI_FAIL: int +EAI_FAMILY: int +EAI_MEMORY: int +EAI_NODATA: int +EAI_NONAME: int +EAI_SERVICE: int +EAI_SOCKTYPE: int +if sys.platform != "win32": + EAI_ADDRFAMILY: int + EAI_OVERFLOW: int + EAI_SYSTEM: int +if sys.platform != "win32" and sys.platform != "linux": + EAI_BADHINTS: int + EAI_MAX: int + EAI_PROTOCOL: int + +AI_ADDRCONFIG: int +AI_ALL: int +AI_CANONNAME: int +AI_NUMERICHOST: int +AI_NUMERICSERV: int +AI_PASSIVE: int +AI_V4MAPPED: int +if sys.platform != "win32" and sys.platform != "linux": + AI_DEFAULT: int + AI_MASK: int + AI_V4MAPPED_CFG: int + +NI_DGRAM: int +NI_MAXHOST: int +NI_MAXSERV: int +NI_NAMEREQD: int +NI_NOFQDN: int +NI_NUMERICHOST: int +NI_NUMERICSERV: int +if sys.platform == "linux" and sys.version_info >= (3, 13): + NI_IDN: int + +TCP_FASTOPEN: int +TCP_KEEPCNT: int +TCP_KEEPINTVL: int +TCP_MAXSEG: int +TCP_NODELAY: int +if sys.platform != "win32": + TCP_NOTSENT_LOWAT: int +if sys.platform != "darwin": + TCP_KEEPIDLE: int +if sys.version_info >= (3, 10) and sys.platform == "darwin": + TCP_KEEPALIVE: int +if sys.version_info >= (3, 11) and sys.platform == "darwin": + TCP_CONNECTION_INFO: int + +if sys.platform != "win32" and sys.platform != "darwin": + TCP_CONGESTION: int + TCP_CORK: int + TCP_DEFER_ACCEPT: int + TCP_INFO: int + TCP_LINGER2: int + TCP_QUICKACK: int + TCP_SYNCNT: int + TCP_USER_TIMEOUT: int + TCP_WINDOW_CLAMP: int +if sys.platform == "linux" and sys.version_info >= (3, 12): + TCP_CC_INFO: int + TCP_FASTOPEN_CONNECT: int + TCP_FASTOPEN_KEY: int + TCP_FASTOPEN_NO_COOKIE: int + TCP_INQ: int + TCP_MD5SIG: int + TCP_MD5SIG_EXT: int + TCP_QUEUE_SEQ: int + TCP_REPAIR: int + TCP_REPAIR_OPTIONS: int + TCP_REPAIR_QUEUE: int + TCP_REPAIR_WINDOW: int + TCP_SAVED_SYN: int + TCP_SAVE_SYN: int + TCP_THIN_DUPACK: int + TCP_THIN_LINEAR_TIMEOUTS: int + TCP_TIMESTAMP: int + TCP_TX_DELAY: int + TCP_ULP: int + TCP_ZEROCOPY_RECEIVE: int + +# -------------------- +# Specifically documented constants +# -------------------- + +if sys.platform == "linux": + # Availability: Linux >= 2.6.25, NetBSD >= 8 + AF_CAN: int + PF_CAN: int + SOL_CAN_BASE: int + SOL_CAN_RAW: int + CAN_EFF_FLAG: int + CAN_EFF_MASK: int + CAN_ERR_FLAG: int + CAN_ERR_MASK: int + CAN_RAW: int + CAN_RAW_FILTER: int + CAN_RAW_LOOPBACK: int + CAN_RAW_RECV_OWN_MSGS: int + CAN_RTR_FLAG: int + CAN_SFF_MASK: int + if sys.version_info < (3, 11): + CAN_RAW_ERR_FILTER: int + +if sys.platform == "linux": + # Availability: Linux >= 2.6.25 + CAN_BCM: int + CAN_BCM_TX_SETUP: int + CAN_BCM_TX_DELETE: int + CAN_BCM_TX_READ: int + CAN_BCM_TX_SEND: int + CAN_BCM_RX_SETUP: int + CAN_BCM_RX_DELETE: int + CAN_BCM_RX_READ: int + CAN_BCM_TX_STATUS: int + CAN_BCM_TX_EXPIRED: int + CAN_BCM_RX_STATUS: int + CAN_BCM_RX_TIMEOUT: int + CAN_BCM_RX_CHANGED: int + CAN_BCM_SETTIMER: int + CAN_BCM_STARTTIMER: int + CAN_BCM_TX_COUNTEVT: int + CAN_BCM_TX_ANNOUNCE: int + CAN_BCM_TX_CP_CAN_ID: int + CAN_BCM_RX_FILTER_ID: int + CAN_BCM_RX_CHECK_DLC: int + CAN_BCM_RX_NO_AUTOTIMER: int + CAN_BCM_RX_ANNOUNCE_RESUME: int + CAN_BCM_TX_RESET_MULTI_IDX: int + CAN_BCM_RX_RTR_FRAME: int + CAN_BCM_CAN_FD_FRAME: int + +if sys.platform == "linux": + # Availability: Linux >= 3.6 + CAN_RAW_FD_FRAMES: int + +if sys.platform == "linux" and sys.version_info >= (3, 9): + # Availability: Linux >= 4.1 + CAN_RAW_JOIN_FILTERS: int + +if sys.platform == "linux": + # Availability: Linux >= 2.6.25 + CAN_ISOTP: int + +if sys.platform == "linux" and sys.version_info >= (3, 9): + # Availability: Linux >= 5.4 + CAN_J1939: int + + J1939_MAX_UNICAST_ADDR: int + J1939_IDLE_ADDR: int + J1939_NO_ADDR: int + J1939_NO_NAME: int + J1939_PGN_REQUEST: int + J1939_PGN_ADDRESS_CLAIMED: int + J1939_PGN_ADDRESS_COMMANDED: int + J1939_PGN_PDU1_MAX: int + J1939_PGN_MAX: int + J1939_NO_PGN: int + + SO_J1939_FILTER: int + SO_J1939_PROMISC: int + SO_J1939_SEND_PRIO: int + SO_J1939_ERRQUEUE: int + + SCM_J1939_DEST_ADDR: int + SCM_J1939_DEST_NAME: int + SCM_J1939_PRIO: int + SCM_J1939_ERRQUEUE: int + + J1939_NLA_PAD: int + J1939_NLA_BYTES_ACKED: int + J1939_EE_INFO_NONE: int + J1939_EE_INFO_TX_ABORT: int + J1939_FILTER_MAX: int + +if sys.version_info >= (3, 12) and sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin": + # Availability: FreeBSD >= 14.0 + AF_DIVERT: int + PF_DIVERT: int + +if sys.platform == "linux": + # Availability: Linux >= 2.2 + AF_PACKET: int + PF_PACKET: int + PACKET_BROADCAST: int + PACKET_FASTROUTE: int + PACKET_HOST: int + PACKET_LOOPBACK: int + PACKET_MULTICAST: int + PACKET_OTHERHOST: int + PACKET_OUTGOING: int + +if sys.version_info >= (3, 12) and sys.platform == "linux": + ETH_P_ALL: int + +if sys.platform == "linux": + # Availability: Linux >= 2.6.30 + AF_RDS: int + PF_RDS: int + SOL_RDS: int + # These are present in include/linux/rds.h but don't always show up + # here. + RDS_CANCEL_SENT_TO: int + RDS_CMSG_RDMA_ARGS: int + RDS_CMSG_RDMA_DEST: int + RDS_CMSG_RDMA_MAP: int + RDS_CMSG_RDMA_STATUS: int + RDS_CONG_MONITOR: int + RDS_FREE_MR: int + RDS_GET_MR: int + RDS_GET_MR_FOR_DEST: int + RDS_RDMA_DONTWAIT: int + RDS_RDMA_FENCE: int + RDS_RDMA_INVALIDATE: int + RDS_RDMA_NOTIFY_ME: int + RDS_RDMA_READWRITE: int + RDS_RDMA_SILENT: int + RDS_RDMA_USE_ONCE: int + RDS_RECVERR: int + + # This is supported by CPython but doesn't seem to be a real thing. + # The closest existing constant in rds.h is RDS_CMSG_CONG_UPDATE + # RDS_CMSG_RDMA_UPDATE: int + +if sys.platform == "win32": + SIO_RCVALL: int + SIO_KEEPALIVE_VALS: int + SIO_LOOPBACK_FAST_PATH: int + RCVALL_MAX: int + RCVALL_OFF: int + RCVALL_ON: int + RCVALL_SOCKETLEVELONLY: int + +if sys.platform == "linux": + AF_TIPC: int + SOL_TIPC: int + TIPC_ADDR_ID: int + TIPC_ADDR_NAME: int + TIPC_ADDR_NAMESEQ: int + TIPC_CFG_SRV: int + TIPC_CLUSTER_SCOPE: int + TIPC_CONN_TIMEOUT: int + TIPC_CRITICAL_IMPORTANCE: int + TIPC_DEST_DROPPABLE: int + TIPC_HIGH_IMPORTANCE: int + TIPC_IMPORTANCE: int + TIPC_LOW_IMPORTANCE: int + TIPC_MEDIUM_IMPORTANCE: int + TIPC_NODE_SCOPE: int + TIPC_PUBLISHED: int + TIPC_SRC_DROPPABLE: int + TIPC_SUBSCR_TIMEOUT: int + TIPC_SUB_CANCEL: int + TIPC_SUB_PORTS: int + TIPC_SUB_SERVICE: int + TIPC_TOP_SRV: int + TIPC_WAIT_FOREVER: int + TIPC_WITHDRAWN: int + TIPC_ZONE_SCOPE: int + +if sys.platform == "linux": + # Availability: Linux >= 2.6.38 + AF_ALG: int + SOL_ALG: int + ALG_OP_DECRYPT: int + ALG_OP_ENCRYPT: int + ALG_OP_SIGN: int + ALG_OP_VERIFY: int + ALG_SET_AEAD_ASSOCLEN: int + ALG_SET_AEAD_AUTHSIZE: int + ALG_SET_IV: int + ALG_SET_KEY: int + ALG_SET_OP: int + ALG_SET_PUBKEY: int + +if sys.platform == "linux": + # Availability: Linux >= 4.8 (or maybe 3.9, CPython docs are confusing) + AF_VSOCK: int + IOCTL_VM_SOCKETS_GET_LOCAL_CID: int + VMADDR_CID_ANY: int + VMADDR_CID_HOST: int + VMADDR_PORT_ANY: int + SO_VM_SOCKETS_BUFFER_MAX_SIZE: int + SO_VM_SOCKETS_BUFFER_SIZE: int + SO_VM_SOCKETS_BUFFER_MIN_SIZE: int + VM_SOCKETS_INVALID_VERSION: int # undocumented + +if sys.platform != "win32" or sys.version_info >= (3, 9): + # Documented as only available on BSD, macOS, but empirically sometimes + # available on Windows + if sys.platform != "linux": + AF_LINK: int + +has_ipv6: bool + +if sys.platform != "darwin" and sys.platform != "linux": + if sys.platform != "win32" or sys.version_info >= (3, 9): + BDADDR_ANY: str + BDADDR_LOCAL: str + +if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + HCI_FILTER: int # not in NetBSD or DragonFlyBSD + HCI_TIME_STAMP: int # not in FreeBSD, NetBSD, or DragonFlyBSD + HCI_DATA_DIR: int # not in FreeBSD, NetBSD, or DragonFlyBSD + +if sys.platform == "linux": + AF_QIPCRTR: int # Availability: Linux >= 4.7 + +if sys.version_info >= (3, 11) and sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin": + # FreeBSD + SCM_CREDS2: int + LOCAL_CREDS: int + LOCAL_CREDS_PERSISTENT: int + +if sys.version_info >= (3, 11) and sys.platform == "linux": + SO_INCOMING_CPU: int # Availability: Linux >= 3.9 + +if sys.version_info >= (3, 12) and sys.platform == "win32": + # Availability: Windows + AF_HYPERV: int + HV_PROTOCOL_RAW: int + HVSOCKET_CONNECT_TIMEOUT: int + HVSOCKET_CONNECT_TIMEOUT_MAX: int + HVSOCKET_CONNECTED_SUSPEND: int + HVSOCKET_ADDRESS_FLAG_PASSTHRU: int + HV_GUID_ZERO: str + HV_GUID_WILDCARD: str + HV_GUID_BROADCAST: str + HV_GUID_CHILDREN: str + HV_GUID_LOOPBACK: str + HV_GUID_PARENT: str + +if sys.version_info >= (3, 12): + if sys.platform != "win32": + # Availability: Linux, FreeBSD, macOS + ETHERTYPE_ARP: int + ETHERTYPE_IP: int + ETHERTYPE_IPV6: int + ETHERTYPE_VLAN: int + +# -------------------- +# Semi-documented constants +# These are alluded to under the "Socket families" section in the docs +# https://docs.python.org/3/library/socket.html#socket-families +# -------------------- + +if sys.platform == "linux": + # Netlink is defined by Linux + AF_NETLINK: int + NETLINK_CRYPTO: int + NETLINK_DNRTMSG: int + NETLINK_FIREWALL: int + NETLINK_IP6_FW: int + NETLINK_NFLOG: int + NETLINK_ROUTE: int + NETLINK_USERSOCK: int + NETLINK_XFRM: int + # Technically still supported by CPython + # NETLINK_ARPD: int # linux 2.0 to 2.6.12 (EOL August 2005) + # NETLINK_ROUTE6: int # linux 2.2 to 2.6.12 (EOL August 2005) + # NETLINK_SKIP: int # linux 2.0 to 2.6.12 (EOL August 2005) + # NETLINK_TAPBASE: int # linux 2.2 to 2.6.12 (EOL August 2005) + # NETLINK_TCPDIAG: int # linux 2.6.0 to 2.6.13 (EOL December 2005) + # NETLINK_W1: int # linux 2.6.13 to 2.6.17 (EOL October 2006) + +if sys.platform == "darwin": + PF_SYSTEM: int + SYSPROTO_CONTROL: int + +if sys.platform != "darwin" and sys.platform != "linux": + if sys.version_info >= (3, 9) or sys.platform != "win32": + AF_BLUETOOTH: int + +if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + # Linux and some BSD support is explicit in the docs + # Windows and macOS do not support in practice + BTPROTO_HCI: int + BTPROTO_L2CAP: int + BTPROTO_SCO: int # not in FreeBSD +if sys.platform != "darwin" and sys.platform != "linux": + if sys.version_info >= (3, 9) or sys.platform != "win32": + BTPROTO_RFCOMM: int + +if sys.version_info >= (3, 9) and sys.platform == "linux": + UDPLITE_RECV_CSCOV: int + UDPLITE_SEND_CSCOV: int + +# -------------------- +# Documented under socket.shutdown +# -------------------- +SHUT_RD: int +SHUT_RDWR: int +SHUT_WR: int + +# -------------------- +# Undocumented constants +# -------------------- + +# Undocumented address families +AF_APPLETALK: int +AF_DECnet: int +AF_IPX: int +AF_SNA: int + +if sys.platform != "win32": + AF_ROUTE: int + +if sys.platform == "darwin": + AF_SYSTEM: int + +if sys.platform != "darwin": + AF_IRDA: int + +if sys.platform != "win32" and sys.platform != "darwin": + AF_ASH: int + AF_ATMPVC: int + AF_ATMSVC: int + AF_AX25: int + AF_BRIDGE: int + AF_ECONET: int + AF_KEY: int + AF_LLC: int + AF_NETBEUI: int + AF_NETROM: int + AF_PPPOX: int + AF_ROSE: int + AF_SECURITY: int + AF_WANPIPE: int + AF_X25: int + +# Miscellaneous undocumented + +if sys.platform != "win32" and sys.platform != "linux": + LOCAL_PEERCRED: int + +if sys.platform != "win32" and sys.platform != "darwin": + # Defined in linux socket.h, but this isn't always present for + # some reason. + IPX_TYPE: int + +# ===== Classes ===== + +class socket: + @property + def family(self) -> int: ... + @property + def type(self) -> int: ... + @property + def proto(self) -> int: ... + # F811: "Redefinition of unused `timeout`" + @property + def timeout(self) -> float | None: ... # noqa: F811 + if sys.platform == "win32": + def __init__( + self, family: int = ..., type: int = ..., proto: int = ..., fileno: SupportsIndex | bytes | None = ... + ) -> None: ... + else: + def __init__(self, family: int = ..., type: int = ..., proto: int = ..., fileno: SupportsIndex | None = ...) -> None: ... + + def bind(self, address: _Address, /) -> None: ... + def close(self) -> None: ... + def connect(self, address: _Address, /) -> None: ... + def connect_ex(self, address: _Address, /) -> int: ... + def detach(self) -> int: ... + def fileno(self) -> int: ... + def getpeername(self) -> _RetAddress: ... + def getsockname(self) -> _RetAddress: ... + @overload + def getsockopt(self, level: int, optname: int, /) -> int: ... + @overload + def getsockopt(self, level: int, optname: int, buflen: int, /) -> bytes: ... + def getblocking(self) -> bool: ... + def gettimeout(self) -> float | None: ... + if sys.platform == "win32": + def ioctl(self, control: int, option: int | tuple[int, int, int] | bool, /) -> None: ... + + def listen(self, backlog: int = ..., /) -> None: ... + def recv(self, bufsize: int, flags: int = ..., /) -> bytes: ... + def recvfrom(self, bufsize: int, flags: int = ..., /) -> tuple[bytes, _RetAddress]: ... + if sys.platform != "win32": + def recvmsg(self, bufsize: int, ancbufsize: int = ..., flags: int = ..., /) -> tuple[bytes, list[_CMSG], int, Any]: ... + def recvmsg_into( + self, buffers: Iterable[WriteableBuffer], ancbufsize: int = ..., flags: int = ..., / + ) -> tuple[int, list[_CMSG], int, Any]: ... + + def recvfrom_into(self, buffer: WriteableBuffer, nbytes: int = ..., flags: int = ...) -> tuple[int, _RetAddress]: ... + def recv_into(self, buffer: WriteableBuffer, nbytes: int = ..., flags: int = ...) -> int: ... + def send(self, data: ReadableBuffer, flags: int = ..., /) -> int: ... + def sendall(self, data: ReadableBuffer, flags: int = ..., /) -> None: ... + @overload + def sendto(self, data: ReadableBuffer, address: _Address, /) -> int: ... + @overload + def sendto(self, data: ReadableBuffer, flags: int, address: _Address, /) -> int: ... + if sys.platform != "win32": + def sendmsg( + self, + buffers: Iterable[ReadableBuffer], + ancdata: Iterable[_CMSGArg] = ..., + flags: int = ..., + address: _Address | None = ..., + /, + ) -> int: ... + if sys.platform == "linux": + def sendmsg_afalg( + self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... + ) -> int: ... + + def setblocking(self, flag: bool, /) -> None: ... + def settimeout(self, value: float | None, /) -> None: ... + @overload + def setsockopt(self, level: int, optname: int, value: int | ReadableBuffer, /) -> None: ... + @overload + def setsockopt(self, level: int, optname: int, value: None, optlen: int, /) -> None: ... + if sys.platform == "win32": + def share(self, process_id: int, /) -> bytes: ... + + def shutdown(self, how: int, /) -> None: ... + +SocketType = socket + +# ===== Functions ===== + +def close(fd: SupportsIndex, /) -> None: ... +def dup(fd: SupportsIndex, /) -> int: ... + +# the 5th tuple item is an address +def getaddrinfo( + host: bytes | str | None, + port: bytes | str | int | None, + family: int = ..., + type: int = ..., + proto: int = ..., + flags: int = ..., +) -> list[tuple[int, int, int, str, tuple[str, int] | tuple[str, int, int, int]]]: ... +def gethostbyname(hostname: str, /) -> str: ... +def gethostbyname_ex(hostname: str, /) -> tuple[str, list[str], list[str]]: ... +def gethostname() -> str: ... +def gethostbyaddr(ip_address: str, /) -> tuple[str, list[str], list[str]]: ... +def getnameinfo(sockaddr: tuple[str, int] | tuple[str, int, int, int], flags: int, /) -> tuple[str, str]: ... +def getprotobyname(protocolname: str, /) -> int: ... +def getservbyname(servicename: str, protocolname: str = ..., /) -> int: ... +def getservbyport(port: int, protocolname: str = ..., /) -> str: ... +def ntohl(x: int, /) -> int: ... # param & ret val are 32-bit ints +def ntohs(x: int, /) -> int: ... # param & ret val are 16-bit ints +def htonl(x: int, /) -> int: ... # param & ret val are 32-bit ints +def htons(x: int, /) -> int: ... # param & ret val are 16-bit ints +def inet_aton(ip_addr: str, /) -> bytes: ... # ret val 4 bytes in length +def inet_ntoa(packed_ip: ReadableBuffer, /) -> str: ... +def inet_pton(address_family: int, ip_string: str, /) -> bytes: ... +def inet_ntop(address_family: int, packed_ip: ReadableBuffer, /) -> str: ... +def getdefaulttimeout() -> float | None: ... + +# F811: "Redefinition of unused `timeout`" +def setdefaulttimeout(timeout: float | None, /) -> None: ... # noqa: F811 + +if sys.platform != "win32": + def sethostname(name: str, /) -> None: ... + def CMSG_LEN(length: int, /) -> int: ... + def CMSG_SPACE(length: int, /) -> int: ... + def socketpair(family: int = ..., type: int = ..., proto: int = ..., /) -> tuple[socket, socket]: ... + +def if_nameindex() -> list[tuple[int, str]]: ... +def if_nametoindex(oname: str, /) -> int: ... +def if_indextoname(index: int, /) -> str: ... + +CAPI: CapsuleType diff --git a/mypy/typeshed/stdlib/_sqlite3.pyi b/mypy/typeshed/stdlib/_sqlite3.pyi new file mode 100644 index 000000000000..6f06542c1ba7 --- /dev/null +++ b/mypy/typeshed/stdlib/_sqlite3.pyi @@ -0,0 +1,312 @@ +import sys +from _typeshed import ReadableBuffer, StrOrBytesPath +from collections.abc import Callable +from sqlite3 import ( + Connection as Connection, + Cursor as Cursor, + DatabaseError as DatabaseError, + DataError as DataError, + Error as Error, + IntegrityError as IntegrityError, + InterfaceError as InterfaceError, + InternalError as InternalError, + NotSupportedError as NotSupportedError, + OperationalError as OperationalError, + PrepareProtocol as PrepareProtocol, + ProgrammingError as ProgrammingError, + Row as Row, + Warning as Warning, +) +from typing import Any, Final, Literal, TypeVar, overload +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 11): + from sqlite3 import Blob as Blob + +_T = TypeVar("_T") +_ConnectionT = TypeVar("_ConnectionT", bound=Connection) +_SqliteData: TypeAlias = str | ReadableBuffer | int | float | None +_Adapter: TypeAlias = Callable[[_T], _SqliteData] +_Converter: TypeAlias = Callable[[bytes], Any] + +PARSE_COLNAMES: Final[int] +PARSE_DECLTYPES: Final[int] +SQLITE_ALTER_TABLE: Final[int] +SQLITE_ANALYZE: Final[int] +SQLITE_ATTACH: Final[int] +SQLITE_CREATE_INDEX: Final[int] +SQLITE_CREATE_TABLE: Final[int] +SQLITE_CREATE_TEMP_INDEX: Final[int] +SQLITE_CREATE_TEMP_TABLE: Final[int] +SQLITE_CREATE_TEMP_TRIGGER: Final[int] +SQLITE_CREATE_TEMP_VIEW: Final[int] +SQLITE_CREATE_TRIGGER: Final[int] +SQLITE_CREATE_VIEW: Final[int] +SQLITE_CREATE_VTABLE: Final[int] +SQLITE_DELETE: Final[int] +SQLITE_DENY: Final[int] +SQLITE_DETACH: Final[int] +SQLITE_DONE: Final[int] +SQLITE_DROP_INDEX: Final[int] +SQLITE_DROP_TABLE: Final[int] +SQLITE_DROP_TEMP_INDEX: Final[int] +SQLITE_DROP_TEMP_TABLE: Final[int] +SQLITE_DROP_TEMP_TRIGGER: Final[int] +SQLITE_DROP_TEMP_VIEW: Final[int] +SQLITE_DROP_TRIGGER: Final[int] +SQLITE_DROP_VIEW: Final[int] +SQLITE_DROP_VTABLE: Final[int] +SQLITE_FUNCTION: Final[int] +SQLITE_IGNORE: Final[int] +SQLITE_INSERT: Final[int] +SQLITE_OK: Final[int] +SQLITE_PRAGMA: Final[int] +SQLITE_READ: Final[int] +SQLITE_RECURSIVE: Final[int] +SQLITE_REINDEX: Final[int] +SQLITE_SAVEPOINT: Final[int] +SQLITE_SELECT: Final[int] +SQLITE_TRANSACTION: Final[int] +SQLITE_UPDATE: Final[int] +adapters: dict[tuple[type[Any], type[Any]], _Adapter[Any]] +converters: dict[str, _Converter] +sqlite_version: str + +if sys.version_info < (3, 12): + version: str + +if sys.version_info >= (3, 12): + LEGACY_TRANSACTION_CONTROL: Final[int] + SQLITE_DBCONFIG_DEFENSIVE: Final[int] + SQLITE_DBCONFIG_DQS_DDL: Final[int] + SQLITE_DBCONFIG_DQS_DML: Final[int] + SQLITE_DBCONFIG_ENABLE_FKEY: Final[int] + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: Final[int] + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: Final[int] + SQLITE_DBCONFIG_ENABLE_QPSG: Final[int] + SQLITE_DBCONFIG_ENABLE_TRIGGER: Final[int] + SQLITE_DBCONFIG_ENABLE_VIEW: Final[int] + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: Final[int] + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: Final[int] + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: Final[int] + SQLITE_DBCONFIG_RESET_DATABASE: Final[int] + SQLITE_DBCONFIG_TRIGGER_EQP: Final[int] + SQLITE_DBCONFIG_TRUSTED_SCHEMA: Final[int] + SQLITE_DBCONFIG_WRITABLE_SCHEMA: Final[int] + +if sys.version_info >= (3, 11): + SQLITE_ABORT: Final[int] + SQLITE_ABORT_ROLLBACK: Final[int] + SQLITE_AUTH: Final[int] + SQLITE_AUTH_USER: Final[int] + SQLITE_BUSY: Final[int] + SQLITE_BUSY_RECOVERY: Final[int] + SQLITE_BUSY_SNAPSHOT: Final[int] + SQLITE_BUSY_TIMEOUT: Final[int] + SQLITE_CANTOPEN: Final[int] + SQLITE_CANTOPEN_CONVPATH: Final[int] + SQLITE_CANTOPEN_DIRTYWAL: Final[int] + SQLITE_CANTOPEN_FULLPATH: Final[int] + SQLITE_CANTOPEN_ISDIR: Final[int] + SQLITE_CANTOPEN_NOTEMPDIR: Final[int] + SQLITE_CANTOPEN_SYMLINK: Final[int] + SQLITE_CONSTRAINT: Final[int] + SQLITE_CONSTRAINT_CHECK: Final[int] + SQLITE_CONSTRAINT_COMMITHOOK: Final[int] + SQLITE_CONSTRAINT_FOREIGNKEY: Final[int] + SQLITE_CONSTRAINT_FUNCTION: Final[int] + SQLITE_CONSTRAINT_NOTNULL: Final[int] + SQLITE_CONSTRAINT_PINNED: Final[int] + SQLITE_CONSTRAINT_PRIMARYKEY: Final[int] + SQLITE_CONSTRAINT_ROWID: Final[int] + SQLITE_CONSTRAINT_TRIGGER: Final[int] + SQLITE_CONSTRAINT_UNIQUE: Final[int] + SQLITE_CONSTRAINT_VTAB: Final[int] + SQLITE_CORRUPT: Final[int] + SQLITE_CORRUPT_INDEX: Final[int] + SQLITE_CORRUPT_SEQUENCE: Final[int] + SQLITE_CORRUPT_VTAB: Final[int] + SQLITE_EMPTY: Final[int] + SQLITE_ERROR: Final[int] + SQLITE_ERROR_MISSING_COLLSEQ: Final[int] + SQLITE_ERROR_RETRY: Final[int] + SQLITE_ERROR_SNAPSHOT: Final[int] + SQLITE_FORMAT: Final[int] + SQLITE_FULL: Final[int] + SQLITE_INTERNAL: Final[int] + SQLITE_INTERRUPT: Final[int] + SQLITE_IOERR: Final[int] + SQLITE_IOERR_ACCESS: Final[int] + SQLITE_IOERR_AUTH: Final[int] + SQLITE_IOERR_BEGIN_ATOMIC: Final[int] + SQLITE_IOERR_BLOCKED: Final[int] + SQLITE_IOERR_CHECKRESERVEDLOCK: Final[int] + SQLITE_IOERR_CLOSE: Final[int] + SQLITE_IOERR_COMMIT_ATOMIC: Final[int] + SQLITE_IOERR_CONVPATH: Final[int] + SQLITE_IOERR_CORRUPTFS: Final[int] + SQLITE_IOERR_DATA: Final[int] + SQLITE_IOERR_DELETE: Final[int] + SQLITE_IOERR_DELETE_NOENT: Final[int] + SQLITE_IOERR_DIR_CLOSE: Final[int] + SQLITE_IOERR_DIR_FSYNC: Final[int] + SQLITE_IOERR_FSTAT: Final[int] + SQLITE_IOERR_FSYNC: Final[int] + SQLITE_IOERR_GETTEMPPATH: Final[int] + SQLITE_IOERR_LOCK: Final[int] + SQLITE_IOERR_MMAP: Final[int] + SQLITE_IOERR_NOMEM: Final[int] + SQLITE_IOERR_RDLOCK: Final[int] + SQLITE_IOERR_READ: Final[int] + SQLITE_IOERR_ROLLBACK_ATOMIC: Final[int] + SQLITE_IOERR_SEEK: Final[int] + SQLITE_IOERR_SHMLOCK: Final[int] + SQLITE_IOERR_SHMMAP: Final[int] + SQLITE_IOERR_SHMOPEN: Final[int] + SQLITE_IOERR_SHMSIZE: Final[int] + SQLITE_IOERR_SHORT_READ: Final[int] + SQLITE_IOERR_TRUNCATE: Final[int] + SQLITE_IOERR_UNLOCK: Final[int] + SQLITE_IOERR_VNODE: Final[int] + SQLITE_IOERR_WRITE: Final[int] + SQLITE_LIMIT_ATTACHED: Final[int] + SQLITE_LIMIT_COLUMN: Final[int] + SQLITE_LIMIT_COMPOUND_SELECT: Final[int] + SQLITE_LIMIT_EXPR_DEPTH: Final[int] + SQLITE_LIMIT_FUNCTION_ARG: Final[int] + SQLITE_LIMIT_LENGTH: Final[int] + SQLITE_LIMIT_LIKE_PATTERN_LENGTH: Final[int] + SQLITE_LIMIT_SQL_LENGTH: Final[int] + SQLITE_LIMIT_TRIGGER_DEPTH: Final[int] + SQLITE_LIMIT_VARIABLE_NUMBER: Final[int] + SQLITE_LIMIT_VDBE_OP: Final[int] + SQLITE_LIMIT_WORKER_THREADS: Final[int] + SQLITE_LOCKED: Final[int] + SQLITE_LOCKED_SHAREDCACHE: Final[int] + SQLITE_LOCKED_VTAB: Final[int] + SQLITE_MISMATCH: Final[int] + SQLITE_MISUSE: Final[int] + SQLITE_NOLFS: Final[int] + SQLITE_NOMEM: Final[int] + SQLITE_NOTADB: Final[int] + SQLITE_NOTFOUND: Final[int] + SQLITE_NOTICE: Final[int] + SQLITE_NOTICE_RECOVER_ROLLBACK: Final[int] + SQLITE_NOTICE_RECOVER_WAL: Final[int] + SQLITE_OK_LOAD_PERMANENTLY: Final[int] + SQLITE_OK_SYMLINK: Final[int] + SQLITE_PERM: Final[int] + SQLITE_PROTOCOL: Final[int] + SQLITE_RANGE: Final[int] + SQLITE_READONLY: Final[int] + SQLITE_READONLY_CANTINIT: Final[int] + SQLITE_READONLY_CANTLOCK: Final[int] + SQLITE_READONLY_DBMOVED: Final[int] + SQLITE_READONLY_DIRECTORY: Final[int] + SQLITE_READONLY_RECOVERY: Final[int] + SQLITE_READONLY_ROLLBACK: Final[int] + SQLITE_ROW: Final[int] + SQLITE_SCHEMA: Final[int] + SQLITE_TOOBIG: Final[int] + SQLITE_WARNING: Final[int] + SQLITE_WARNING_AUTOINDEX: Final[int] + threadsafety: Final[int] + +# Can take or return anything depending on what's in the registry. +@overload +def adapt(obj: Any, proto: Any, /) -> Any: ... +@overload +def adapt(obj: Any, proto: Any, alt: _T, /) -> Any | _T: ... +def complete_statement(statement: str) -> bool: ... + +if sys.version_info >= (3, 12): + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + cached_statements: int = 128, + uri: bool = False, + *, + autocommit: bool = ..., + ) -> Connection: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float, + detect_types: int, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, + check_same_thread: bool, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + *, + autocommit: bool = ..., + ) -> _ConnectionT: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + *, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + autocommit: bool = ..., + ) -> _ConnectionT: ... + +else: + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + cached_statements: int = 128, + uri: bool = False, + ) -> Connection: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float, + detect_types: int, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, + check_same_thread: bool, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + ) -> _ConnectionT: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + *, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + ) -> _ConnectionT: ... + +def enable_callback_tracebacks(enable: bool, /) -> None: ... + +if sys.version_info < (3, 12): + # takes a pos-or-keyword argument because there is a C wrapper + def enable_shared_cache(do_enable: int) -> None: ... + +if sys.version_info >= (3, 10): + def register_adapter(type: type[_T], adapter: _Adapter[_T], /) -> None: ... + def register_converter(typename: str, converter: _Converter, /) -> None: ... + +else: + def register_adapter(type: type[_T], caster: _Adapter[_T], /) -> None: ... + def register_converter(name: str, converter: _Converter, /) -> None: ... + +if sys.version_info < (3, 10): + OptimizedUnicode = str diff --git a/mypy/typeshed/stdlib/_ssl.pyi b/mypy/typeshed/stdlib/_ssl.pyi new file mode 100644 index 000000000000..938135eb1192 --- /dev/null +++ b/mypy/typeshed/stdlib/_ssl.pyi @@ -0,0 +1,292 @@ +import sys +from _typeshed import ReadableBuffer, StrOrBytesPath +from collections.abc import Callable +from ssl import ( + SSLCertVerificationError as SSLCertVerificationError, + SSLContext, + SSLEOFError as SSLEOFError, + SSLError as SSLError, + SSLObject, + SSLSyscallError as SSLSyscallError, + SSLWantReadError as SSLWantReadError, + SSLWantWriteError as SSLWantWriteError, + SSLZeroReturnError as SSLZeroReturnError, +) +from typing import Any, Literal, TypedDict, final, overload +from typing_extensions import NotRequired, Self, TypeAlias + +_PasswordType: TypeAlias = Callable[[], str | bytes | bytearray] | str | bytes | bytearray +_PCTRTT: TypeAlias = tuple[tuple[str, str], ...] +_PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] +_PeerCertRetDictType: TypeAlias = dict[str, str | _PCTRTTT | _PCTRTT] + +class _Cipher(TypedDict): + aead: bool + alg_bits: int + auth: str + description: str + digest: str | None + id: int + kea: str + name: str + protocol: str + strength_bits: int + symmetric: str + +class _CertInfo(TypedDict): + subject: tuple[tuple[tuple[str, str], ...], ...] + issuer: tuple[tuple[tuple[str, str], ...], ...] + version: int + serialNumber: str + notBefore: str + notAfter: str + subjectAltName: NotRequired[tuple[tuple[str, str], ...] | None] + OCSP: NotRequired[tuple[str, ...] | None] + caIssuers: NotRequired[tuple[str, ...] | None] + crlDistributionPoints: NotRequired[tuple[str, ...] | None] + +def RAND_add(string: str | ReadableBuffer, entropy: float, /) -> None: ... +def RAND_bytes(n: int, /) -> bytes: ... + +if sys.version_info < (3, 12): + def RAND_pseudo_bytes(n: int, /) -> tuple[bytes, bool]: ... + +if sys.version_info < (3, 10): + def RAND_egd(path: str) -> None: ... + +def RAND_status() -> bool: ... +def get_default_verify_paths() -> tuple[str, str, str, str]: ... + +if sys.platform == "win32": + _EnumRetType: TypeAlias = list[tuple[bytes, str, set[str] | bool]] + def enum_certificates(store_name: str) -> _EnumRetType: ... + def enum_crls(store_name: str) -> _EnumRetType: ... + +def txt2obj(txt: str, name: bool = False) -> tuple[int, str, str, str]: ... +def nid2obj(nid: int, /) -> tuple[int, str, str, str]: ... + +class _SSLContext: + check_hostname: bool + keylog_filename: str | None + maximum_version: int + minimum_version: int + num_tickets: int + options: int + post_handshake_auth: bool + protocol: int + if sys.version_info >= (3, 10): + security_level: int + sni_callback: Callable[[SSLObject, str, SSLContext], None | int] | None + verify_flags: int + verify_mode: int + def __new__(cls, protocol: int, /) -> Self: ... + def cert_store_stats(self) -> dict[str, int]: ... + @overload + def get_ca_certs(self, binary_form: Literal[False] = False) -> list[_PeerCertRetDictType]: ... + @overload + def get_ca_certs(self, binary_form: Literal[True]) -> list[bytes]: ... + @overload + def get_ca_certs(self, binary_form: bool = False) -> Any: ... + def get_ciphers(self) -> list[_Cipher]: ... + def load_cert_chain( + self, certfile: StrOrBytesPath, keyfile: StrOrBytesPath | None = None, password: _PasswordType | None = None + ) -> None: ... + def load_dh_params(self, path: str, /) -> None: ... + def load_verify_locations( + self, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, + ) -> None: ... + def session_stats(self) -> dict[str, int]: ... + def set_ciphers(self, cipherlist: str, /) -> None: ... + def set_default_verify_paths(self) -> None: ... + def set_ecdh_curve(self, name: str, /) -> None: ... + if sys.version_info >= (3, 13): + def set_psk_client_callback(self, callback: Callable[[str | None], tuple[str | None, bytes]] | None) -> None: ... + def set_psk_server_callback( + self, callback: Callable[[str | None], bytes] | None, identity_hint: str | None = None + ) -> None: ... + +@final +class MemoryBIO: + eof: bool + pending: int + def __new__(self) -> Self: ... + def read(self, size: int = -1, /) -> bytes: ... + def write(self, b: ReadableBuffer, /) -> int: ... + def write_eof(self) -> None: ... + +@final +class SSLSession: + @property + def has_ticket(self) -> bool: ... + @property + def id(self) -> bytes: ... + @property + def ticket_lifetime_hint(self) -> int: ... + @property + def time(self) -> int: ... + @property + def timeout(self) -> int: ... + +# _ssl.Certificate is weird: it can't be instantiated or subclassed. +# Instances can only be created via methods of the private _ssl._SSLSocket class, +# for which the relevant method signatures are: +# +# class _SSLSocket: +# def get_unverified_chain(self) -> list[Certificate] | None: ... +# def get_verified_chain(self) -> list[Certificate] | None: ... +# +# You can find a _ssl._SSLSocket object as the _sslobj attribute of a ssl.SSLSocket object + +if sys.version_info >= (3, 10): + @final + class Certificate: + def get_info(self) -> _CertInfo: ... + @overload + def public_bytes(self) -> str: ... + @overload + def public_bytes(self, format: Literal[1] = 1, /) -> str: ... # ENCODING_PEM + @overload + def public_bytes(self, format: Literal[2], /) -> bytes: ... # ENCODING_DER + @overload + def public_bytes(self, format: int, /) -> str | bytes: ... + +if sys.version_info < (3, 12): + err_codes_to_names: dict[tuple[int, int], str] + err_names_to_codes: dict[str, tuple[int, int]] + lib_codes_to_names: dict[int, str] + +_DEFAULT_CIPHERS: str + +# SSL error numbers +SSL_ERROR_ZERO_RETURN: int +SSL_ERROR_WANT_READ: int +SSL_ERROR_WANT_WRITE: int +SSL_ERROR_WANT_X509_LOOKUP: int +SSL_ERROR_SYSCALL: int +SSL_ERROR_SSL: int +SSL_ERROR_WANT_CONNECT: int +SSL_ERROR_EOF: int +SSL_ERROR_INVALID_ERROR_CODE: int + +# verify modes +CERT_NONE: int +CERT_OPTIONAL: int +CERT_REQUIRED: int + +# verify flags +VERIFY_DEFAULT: int +VERIFY_CRL_CHECK_LEAF: int +VERIFY_CRL_CHECK_CHAIN: int +VERIFY_X509_STRICT: int +VERIFY_X509_TRUSTED_FIRST: int +if sys.version_info >= (3, 10): + VERIFY_ALLOW_PROXY_CERTS: int + VERIFY_X509_PARTIAL_CHAIN: int + +# alert descriptions +ALERT_DESCRIPTION_CLOSE_NOTIFY: int +ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: int +ALERT_DESCRIPTION_BAD_RECORD_MAC: int +ALERT_DESCRIPTION_RECORD_OVERFLOW: int +ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: int +ALERT_DESCRIPTION_HANDSHAKE_FAILURE: int +ALERT_DESCRIPTION_BAD_CERTIFICATE: int +ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: int +ALERT_DESCRIPTION_CERTIFICATE_REVOKED: int +ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: int +ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: int +ALERT_DESCRIPTION_ILLEGAL_PARAMETER: int +ALERT_DESCRIPTION_UNKNOWN_CA: int +ALERT_DESCRIPTION_ACCESS_DENIED: int +ALERT_DESCRIPTION_DECODE_ERROR: int +ALERT_DESCRIPTION_DECRYPT_ERROR: int +ALERT_DESCRIPTION_PROTOCOL_VERSION: int +ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: int +ALERT_DESCRIPTION_INTERNAL_ERROR: int +ALERT_DESCRIPTION_USER_CANCELLED: int +ALERT_DESCRIPTION_NO_RENEGOTIATION: int +ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: int +ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: int +ALERT_DESCRIPTION_UNRECOGNIZED_NAME: int +ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: int +ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: int +ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: int + +# protocol versions +PROTOCOL_SSLv23: int +PROTOCOL_TLS: int +PROTOCOL_TLS_CLIENT: int +PROTOCOL_TLS_SERVER: int +PROTOCOL_TLSv1: int +PROTOCOL_TLSv1_1: int +PROTOCOL_TLSv1_2: int + +# protocol options +OP_ALL: int +OP_NO_SSLv2: int +OP_NO_SSLv3: int +OP_NO_TLSv1: int +OP_NO_TLSv1_1: int +OP_NO_TLSv1_2: int +OP_NO_TLSv1_3: int +OP_CIPHER_SERVER_PREFERENCE: int +OP_SINGLE_DH_USE: int +OP_NO_TICKET: int +OP_SINGLE_ECDH_USE: int +OP_NO_COMPRESSION: int +OP_ENABLE_MIDDLEBOX_COMPAT: int +OP_NO_RENEGOTIATION: int +if sys.version_info >= (3, 11): + OP_IGNORE_UNEXPECTED_EOF: int +elif sys.version_info >= (3, 8) and sys.platform == "linux": + OP_IGNORE_UNEXPECTED_EOF: int +if sys.version_info >= (3, 12): + OP_LEGACY_SERVER_CONNECT: int + OP_ENABLE_KTLS: int + +# host flags +HOSTFLAG_ALWAYS_CHECK_SUBJECT: int +HOSTFLAG_NEVER_CHECK_SUBJECT: int +HOSTFLAG_NO_WILDCARDS: int +HOSTFLAG_NO_PARTIAL_WILDCARDS: int +HOSTFLAG_MULTI_LABEL_WILDCARDS: int +HOSTFLAG_SINGLE_LABEL_SUBDOMAINS: int + +if sys.version_info >= (3, 10): + # certificate file types + # Typed as Literal so the overload on Certificate.public_bytes can work properly. + ENCODING_PEM: Literal[1] + ENCODING_DER: Literal[2] + +# protocol versions +PROTO_MINIMUM_SUPPORTED: int +PROTO_MAXIMUM_SUPPORTED: int +PROTO_SSLv3: int +PROTO_TLSv1: int +PROTO_TLSv1_1: int +PROTO_TLSv1_2: int +PROTO_TLSv1_3: int + +# feature support +HAS_SNI: bool +HAS_TLS_UNIQUE: bool +HAS_ECDH: bool +HAS_NPN: bool +if sys.version_info >= (3, 13): + HAS_PSK: bool +HAS_ALPN: bool +HAS_SSLv2: bool +HAS_SSLv3: bool +HAS_TLSv1: bool +HAS_TLSv1_1: bool +HAS_TLSv1_2: bool +HAS_TLSv1_3: bool + +# version info +OPENSSL_VERSION_NUMBER: int +OPENSSL_VERSION_INFO: tuple[int, int, int, int, int] +OPENSSL_VERSION: str +_OPENSSL_API_VERSION: tuple[int, int, int, int, int] diff --git a/mypy/typeshed/stdlib/_stat.pyi b/mypy/typeshed/stdlib/_stat.pyi new file mode 100644 index 000000000000..7129a282b574 --- /dev/null +++ b/mypy/typeshed/stdlib/_stat.pyi @@ -0,0 +1,119 @@ +import sys +from typing import Final + +SF_APPEND: Final = 0x00040000 +SF_ARCHIVED: Final = 0x00010000 +SF_IMMUTABLE: Final = 0x00020000 +SF_NOUNLINK: Final = 0x00100000 +SF_SNAPSHOT: Final = 0x00200000 + +ST_MODE: Final = 0 +ST_INO: Final = 1 +ST_DEV: Final = 2 +ST_NLINK: Final = 3 +ST_UID: Final = 4 +ST_GID: Final = 5 +ST_SIZE: Final = 6 +ST_ATIME: Final = 7 +ST_MTIME: Final = 8 +ST_CTIME: Final = 9 + +S_IFIFO: Final = 0o010000 +S_IFLNK: Final = 0o120000 +S_IFREG: Final = 0o100000 +S_IFSOCK: Final = 0o140000 +S_IFBLK: Final = 0o060000 +S_IFCHR: Final = 0o020000 +S_IFDIR: Final = 0o040000 + +# These are 0 on systems that don't support the specific kind of file. +# Example: Linux doesn't support door files, so S_IFDOOR is 0 on linux. +S_IFDOOR: Final[int] +S_IFPORT: Final[int] +S_IFWHT: Final[int] + +S_ISUID: Final = 0o4000 +S_ISGID: Final = 0o2000 +S_ISVTX: Final = 0o1000 + +S_IRWXU: Final = 0o0700 +S_IRUSR: Final = 0o0400 +S_IWUSR: Final = 0o0200 +S_IXUSR: Final = 0o0100 + +S_IRWXG: Final = 0o0070 +S_IRGRP: Final = 0o0040 +S_IWGRP: Final = 0o0020 +S_IXGRP: Final = 0o0010 + +S_IRWXO: Final = 0o0007 +S_IROTH: Final = 0o0004 +S_IWOTH: Final = 0o0002 +S_IXOTH: Final = 0o0001 + +S_ENFMT: Final = 0o2000 +S_IREAD: Final = 0o0400 +S_IWRITE: Final = 0o0200 +S_IEXEC: Final = 0o0100 + +UF_APPEND: Final = 0x00000004 +UF_COMPRESSED: Final = 0x00000020 # OS X 10.6+ only +UF_HIDDEN: Final = 0x00008000 # OX X 10.5+ only +UF_IMMUTABLE: Final = 0x00000002 +UF_NODUMP: Final = 0x00000001 +UF_NOUNLINK: Final = 0x00000010 +UF_OPAQUE: Final = 0x00000008 + +def S_IMODE(mode: int, /) -> int: ... +def S_IFMT(mode: int, /) -> int: ... +def S_ISBLK(mode: int, /) -> bool: ... +def S_ISCHR(mode: int, /) -> bool: ... +def S_ISDIR(mode: int, /) -> bool: ... +def S_ISDOOR(mode: int, /) -> bool: ... +def S_ISFIFO(mode: int, /) -> bool: ... +def S_ISLNK(mode: int, /) -> bool: ... +def S_ISPORT(mode: int, /) -> bool: ... +def S_ISREG(mode: int, /) -> bool: ... +def S_ISSOCK(mode: int, /) -> bool: ... +def S_ISWHT(mode: int, /) -> bool: ... +def filemode(mode: int, /) -> str: ... + +if sys.platform == "win32": + IO_REPARSE_TAG_SYMLINK: Final = 0xA000000C + IO_REPARSE_TAG_MOUNT_POINT: Final = 0xA0000003 + IO_REPARSE_TAG_APPEXECLINK: Final = 0x8000001B + +if sys.platform == "win32": + FILE_ATTRIBUTE_ARCHIVE: Final = 32 + FILE_ATTRIBUTE_COMPRESSED: Final = 2048 + FILE_ATTRIBUTE_DEVICE: Final = 64 + FILE_ATTRIBUTE_DIRECTORY: Final = 16 + FILE_ATTRIBUTE_ENCRYPTED: Final = 16384 + FILE_ATTRIBUTE_HIDDEN: Final = 2 + FILE_ATTRIBUTE_INTEGRITY_STREAM: Final = 32768 + FILE_ATTRIBUTE_NORMAL: Final = 128 + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED: Final = 8192 + FILE_ATTRIBUTE_NO_SCRUB_DATA: Final = 131072 + FILE_ATTRIBUTE_OFFLINE: Final = 4096 + FILE_ATTRIBUTE_READONLY: Final = 1 + FILE_ATTRIBUTE_REPARSE_POINT: Final = 1024 + FILE_ATTRIBUTE_SPARSE_FILE: Final = 512 + FILE_ATTRIBUTE_SYSTEM: Final = 4 + FILE_ATTRIBUTE_TEMPORARY: Final = 256 + FILE_ATTRIBUTE_VIRTUAL: Final = 65536 + +if sys.version_info >= (3, 13): + # Varies by platform. + SF_SETTABLE: Final[int] + # https://github.com/python/cpython/issues/114081#issuecomment-2119017790 + # SF_RESTRICTED: Literal[0x00080000] + SF_FIRMLINK: Final = 0x00800000 + SF_DATALESS: Final = 0x40000000 + + if sys.platform == "darwin": + SF_SUPPORTED: Final = 0x9F0000 + SF_SYNTHETIC: Final = 0xC0000000 + + UF_TRACKED: Final = 0x00000040 + UF_DATAVAULT: Final = 0x00000080 + UF_SETTABLE: Final = 0x0000FFFF diff --git a/mypy/typeshed/stdlib/_struct.pyi b/mypy/typeshed/stdlib/_struct.pyi new file mode 100644 index 000000000000..662170e869f3 --- /dev/null +++ b/mypy/typeshed/stdlib/_struct.pyi @@ -0,0 +1,22 @@ +from _typeshed import ReadableBuffer, WriteableBuffer +from collections.abc import Iterator +from typing import Any + +def pack(fmt: str | bytes, /, *v: Any) -> bytes: ... +def pack_into(fmt: str | bytes, buffer: WriteableBuffer, offset: int, /, *v: Any) -> None: ... +def unpack(format: str | bytes, buffer: ReadableBuffer, /) -> tuple[Any, ...]: ... +def unpack_from(format: str | bytes, /, buffer: ReadableBuffer, offset: int = 0) -> tuple[Any, ...]: ... +def iter_unpack(format: str | bytes, buffer: ReadableBuffer, /) -> Iterator[tuple[Any, ...]]: ... +def calcsize(format: str | bytes, /) -> int: ... + +class Struct: + @property + def format(self) -> str: ... + @property + def size(self) -> int: ... + def __init__(self, format: str | bytes) -> None: ... + def pack(self, *v: Any) -> bytes: ... + def pack_into(self, buffer: WriteableBuffer, offset: int, *v: Any) -> None: ... + def unpack(self, buffer: ReadableBuffer, /) -> tuple[Any, ...]: ... + def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> tuple[Any, ...]: ... + def iter_unpack(self, buffer: ReadableBuffer, /) -> Iterator[tuple[Any, ...]]: ... diff --git a/mypy/typeshed/stdlib/_thread.pyi b/mypy/typeshed/stdlib/_thread.pyi new file mode 100644 index 000000000000..f0b70ed2a0b0 --- /dev/null +++ b/mypy/typeshed/stdlib/_thread.pyi @@ -0,0 +1,91 @@ +import signal +import sys +from _typeshed import structseq +from collections.abc import Callable +from threading import Thread +from types import TracebackType +from typing import Any, Final, NoReturn, final, overload +from typing_extensions import TypeVarTuple, Unpack + +_Ts = TypeVarTuple("_Ts") + +error = RuntimeError + +def _count() -> int: ... +@final +class LockType: + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: ... + def release(self) -> None: ... + def locked(self) -> bool: ... + def acquire_lock(self, blocking: bool = True, timeout: float = -1) -> bool: ... + def release_lock(self) -> None: ... + def locked_lock(self) -> bool: ... + def __enter__(self) -> bool: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + +if sys.version_info >= (3, 13): + @final + class _ThreadHandle: + ident: int + + def join(self, timeout: float | None = None, /) -> None: ... + def is_done(self) -> bool: ... + def _set_done(self) -> None: ... + + def start_joinable_thread( + function: Callable[[], object], handle: _ThreadHandle | None = None, daemon: bool = True + ) -> _ThreadHandle: ... + lock = LockType + +@overload +def start_new_thread(function: Callable[[Unpack[_Ts]], object], args: tuple[Unpack[_Ts]], /) -> int: ... +@overload +def start_new_thread(function: Callable[..., object], args: tuple[Any, ...], kwargs: dict[str, Any], /) -> int: ... + +# Obsolete synonym for start_new_thread() +@overload +def start_new(function: Callable[[Unpack[_Ts]], object], args: tuple[Unpack[_Ts]], /) -> int: ... +@overload +def start_new(function: Callable[..., object], args: tuple[Any, ...], kwargs: dict[str, Any], /) -> int: ... + +if sys.version_info >= (3, 10): + def interrupt_main(signum: signal.Signals = ..., /) -> None: ... + +else: + def interrupt_main() -> None: ... + +def exit() -> NoReturn: ... +def exit_thread() -> NoReturn: ... # Obsolete synonym for exit() +def allocate_lock() -> LockType: ... +def allocate() -> LockType: ... # Obsolete synonym for allocate_lock() +def get_ident() -> int: ... +def stack_size(size: int = 0, /) -> int: ... + +TIMEOUT_MAX: float + +def get_native_id() -> int: ... # only available on some platforms +@final +class _ExceptHookArgs(structseq[Any], tuple[type[BaseException], BaseException | None, TracebackType | None, Thread | None]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("exc_type", "exc_value", "exc_traceback", "thread") + + @property + def exc_type(self) -> type[BaseException]: ... + @property + def exc_value(self) -> BaseException | None: ... + @property + def exc_traceback(self) -> TracebackType | None: ... + @property + def thread(self) -> Thread | None: ... + +_excepthook: Callable[[_ExceptHookArgs], Any] + +if sys.version_info >= (3, 12): + def daemon_threads_allowed() -> bool: ... + +class _local: + def __getattribute__(self, name: str, /) -> Any: ... + def __setattr__(self, name: str, value: Any, /) -> None: ... + def __delattr__(self, name: str, /) -> None: ... diff --git a/mypy/typeshed/stdlib/_threading_local.pyi b/mypy/typeshed/stdlib/_threading_local.pyi new file mode 100644 index 000000000000..07a825f0d816 --- /dev/null +++ b/mypy/typeshed/stdlib/_threading_local.pyi @@ -0,0 +1,22 @@ +from threading import RLock +from typing import Any +from typing_extensions import Self, TypeAlias +from weakref import ReferenceType + +__all__ = ["local"] +_LocalDict: TypeAlias = dict[Any, Any] + +class _localimpl: + key: str + dicts: dict[int, tuple[ReferenceType[Any], _LocalDict]] + # Keep localargs in sync with the *args, **kwargs annotation on local.__new__ + localargs: tuple[list[Any], dict[str, Any]] + locallock: RLock + def get_dict(self) -> _LocalDict: ... + def create_dict(self) -> _LocalDict: ... + +class local: + def __new__(cls, /, *args: Any, **kw: Any) -> Self: ... + def __getattribute__(self, name: str) -> Any: ... + def __setattr__(self, name: str, value: Any) -> None: ... + def __delattr__(self, name: str) -> None: ... diff --git a/mypy/typeshed/stdlib/_tkinter.pyi b/mypy/typeshed/stdlib/_tkinter.pyi new file mode 100644 index 000000000000..63b1e7ca7cb4 --- /dev/null +++ b/mypy/typeshed/stdlib/_tkinter.pyi @@ -0,0 +1,128 @@ +import sys +from collections.abc import Callable +from typing import Any, ClassVar, Final, final +from typing_extensions import TypeAlias + +# _tkinter is meant to be only used internally by tkinter, but some tkinter +# functions e.g. return _tkinter.Tcl_Obj objects. Tcl_Obj represents a Tcl +# object that hasn't been converted to a string. +# +# There are not many ways to get Tcl_Objs from tkinter, and I'm not sure if the +# only existing ways are supposed to return Tcl_Objs as opposed to returning +# strings. Here's one of these things that return Tcl_Objs: +# +# >>> import tkinter +# >>> text = tkinter.Text() +# >>> text.tag_add('foo', '1.0', 'end') +# >>> text.tag_ranges('foo') +# (, ) +@final +class Tcl_Obj: + @property + def string(self) -> str: ... + @property + def typename(self) -> str: ... + __hash__: ClassVar[None] # type: ignore[assignment] + def __eq__(self, value, /): ... + def __ge__(self, value, /): ... + def __gt__(self, value, /): ... + def __le__(self, value, /): ... + def __lt__(self, value, /): ... + def __ne__(self, value, /): ... + +class TclError(Exception): ... + +_TkinterTraceFunc: TypeAlias = Callable[[tuple[str, ...]], object] + +# This class allows running Tcl code. Tkinter uses it internally a lot, and +# it's often handy to drop a piece of Tcl code into a tkinter program. Example: +# +# >>> import tkinter, _tkinter +# >>> tkapp = tkinter.Tk().tk +# >>> isinstance(tkapp, _tkinter.TkappType) +# True +# >>> tkapp.call('set', 'foo', (1,2,3)) +# (1, 2, 3) +# >>> tkapp.eval('return $foo') +# '1 2 3' +# >>> +# +# call args can be pretty much anything. Also, call(some_tuple) is same as call(*some_tuple). +# +# eval always returns str because _tkinter_tkapp_eval_impl in _tkinter.c calls +# Tkapp_UnicodeResult, and it returns a string when it succeeds. +@final +class TkappType: + # Please keep in sync with tkinter.Tk + def adderrorinfo(self, msg, /): ... + def call(self, command: Any, /, *args: Any) -> Any: ... + def createcommand(self, name, func, /): ... + if sys.platform != "win32": + def createfilehandler(self, file, mask, func, /): ... + def deletefilehandler(self, file, /): ... + + def createtimerhandler(self, milliseconds, func, /): ... + def deletecommand(self, name, /): ... + def dooneevent(self, flags: int = 0, /): ... + def eval(self, script: str, /) -> str: ... + def evalfile(self, fileName, /): ... + def exprboolean(self, s, /): ... + def exprdouble(self, s, /): ... + def exprlong(self, s, /): ... + def exprstring(self, s, /): ... + def getboolean(self, arg, /): ... + def getdouble(self, arg, /): ... + def getint(self, arg, /): ... + def getvar(self, *args, **kwargs): ... + def globalgetvar(self, *args, **kwargs): ... + def globalsetvar(self, *args, **kwargs): ... + def globalunsetvar(self, *args, **kwargs): ... + def interpaddr(self): ... + def loadtk(self) -> None: ... + def mainloop(self, threshold: int = 0, /): ... + def quit(self): ... + def record(self, script, /): ... + def setvar(self, *ags, **kwargs): ... + if sys.version_info < (3, 11): + def split(self, arg, /): ... + + def splitlist(self, arg, /): ... + def unsetvar(self, *args, **kwargs): ... + def wantobjects(self, *args, **kwargs): ... + def willdispatch(self): ... + if sys.version_info >= (3, 12): + def gettrace(self, /) -> _TkinterTraceFunc | None: ... + def settrace(self, func: _TkinterTraceFunc | None, /) -> None: ... + +# These should be kept in sync with tkinter.tix constants, except ALL_EVENTS which doesn't match TCL_ALL_EVENTS +ALL_EVENTS: Final = -3 +FILE_EVENTS: Final = 8 +IDLE_EVENTS: Final = 32 +TIMER_EVENTS: Final = 16 +WINDOW_EVENTS: Final = 4 + +DONT_WAIT: Final = 2 +EXCEPTION: Final = 8 +READABLE: Final = 2 +WRITABLE: Final = 4 + +TCL_VERSION: Final[str] +TK_VERSION: Final[str] + +@final +class TkttType: + def deletetimerhandler(self): ... + +def create( + screenName: str | None = None, + baseName: str = "", + className: str = "Tk", + interactive: bool = False, + wantobjects: bool = False, + wantTk: bool = True, + sync: bool = False, + use: str | None = None, + /, +): ... +def getbusywaitinterval(): ... +def setbusywaitinterval(new_val, /): ... diff --git a/mypy/typeshed/stdlib/_tracemalloc.pyi b/mypy/typeshed/stdlib/_tracemalloc.pyi new file mode 100644 index 000000000000..b1aeb710233e --- /dev/null +++ b/mypy/typeshed/stdlib/_tracemalloc.pyi @@ -0,0 +1,17 @@ +import sys +from collections.abc import Sequence +from tracemalloc import _FrameTuple, _TraceTuple + +def _get_object_traceback(obj: object, /) -> Sequence[_FrameTuple] | None: ... +def _get_traces() -> Sequence[_TraceTuple]: ... +def clear_traces() -> None: ... +def get_traceback_limit() -> int: ... +def get_traced_memory() -> tuple[int, int]: ... +def get_tracemalloc_memory() -> int: ... +def is_tracing() -> bool: ... + +if sys.version_info >= (3, 9): + def reset_peak() -> None: ... + +def start(nframe: int = 1, /) -> None: ... +def stop() -> None: ... diff --git a/mypy/typeshed/stdlib/_typeshed/README.md b/mypy/typeshed/stdlib/_typeshed/README.md new file mode 100644 index 000000000000..f4808944fa7b --- /dev/null +++ b/mypy/typeshed/stdlib/_typeshed/README.md @@ -0,0 +1,34 @@ +# Utility types for typeshed + +This package and its submodules contains various common types used by +typeshed. It can also be used by packages outside typeshed, but beware +the API stability guarantees below. + +## Usage + +The `_typeshed` package and its types do not exist at runtime, but can be +used freely in stubs (`.pyi`) files. To import the types from this package in +implementation (`.py`) files, use the following construct: + +```python +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from _typeshed import ... +``` + +Types can then be used in annotations by either quoting them or +using: + +```python +from __future__ import annotations +``` + +## API Stability + +You can use this package and its submodules outside of typeshed, but we +guarantee only limited API stability. Items marked as "stable" will not be +removed or changed in an incompatible way for at least one year. +Before making such a change, the "stable" moniker will be removed +and we will mark the type in question as deprecated. No guarantees +are made about unmarked types. diff --git a/mypy/typeshed/stdlib/_typeshed/__init__.pyi b/mypy/typeshed/stdlib/_typeshed/__init__.pyi new file mode 100644 index 000000000000..7201819b25ed --- /dev/null +++ b/mypy/typeshed/stdlib/_typeshed/__init__.pyi @@ -0,0 +1,363 @@ +# Utility types for typeshed +# +# See the README.md file in this directory for more information. + +import sys +from collections.abc import Awaitable, Callable, Iterable, Sequence, Set as AbstractSet, Sized +from dataclasses import Field +from os import PathLike +from types import FrameType, TracebackType +from typing import ( + Any, + AnyStr, + ClassVar, + Final, + Generic, + Literal, + Protocol, + SupportsFloat, + SupportsIndex, + SupportsInt, + TypeVar, + final, + overload, +) +from typing_extensions import Buffer, LiteralString, TypeAlias + +_KT = TypeVar("_KT") +_KT_co = TypeVar("_KT_co", covariant=True) +_KT_contra = TypeVar("_KT_contra", contravariant=True) +_VT = TypeVar("_VT") +_VT_co = TypeVar("_VT_co", covariant=True) +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_T_contra = TypeVar("_T_contra", contravariant=True) + +# Alternative to `typing_extensions.Self`, exclusively for use with `__new__` +# in metaclasses: +# def __new__(cls: type[Self], ...) -> Self: ... +# In other cases, use `typing_extensions.Self`. +Self = TypeVar("Self") # noqa: Y001 + +# covariant version of typing.AnyStr, useful for protocols +AnyStr_co = TypeVar("AnyStr_co", str, bytes, covariant=True) # noqa: Y001 + +# For partially known annotations. Usually, fields where type annotations +# haven't been added are left unannotated, but in some situations this +# isn't possible or a type is already partially known. In cases like these, +# use Incomplete instead of Any as a marker. For example, use +# "Incomplete | None" instead of "Any | None". +Incomplete: TypeAlias = Any # stable + +# To describe a function parameter that is unused and will work with anything. +Unused: TypeAlias = object # stable + +# Marker for return types that include None, but where forcing the user to +# check for None can be detrimental. Sometimes called "the Any trick". See +# CONTRIBUTING.md for more information. +MaybeNone: TypeAlias = Any # stable + +# Used to mark arguments that default to a sentinel value. This prevents +# stubtest from complaining about the default value not matching. +# +# def foo(x: int | None = sentinel) -> None: ... +# +# In cases where the sentinel object is exported and can be used by user code, +# a construct like this is better: +# +# _SentinelType = NewType("_SentinelType", object) +# sentinel: _SentinelType +# def foo(x: int | None | _SentinelType = ...) -> None: ... +sentinel: Any + +# stable +class IdentityFunction(Protocol): + def __call__(self, x: _T, /) -> _T: ... + +# stable +class SupportsNext(Protocol[_T_co]): + def __next__(self) -> _T_co: ... + +# stable +class SupportsAnext(Protocol[_T_co]): + def __anext__(self) -> Awaitable[_T_co]: ... + +# Comparison protocols + +class SupportsDunderLT(Protocol[_T_contra]): + def __lt__(self, other: _T_contra, /) -> bool: ... + +class SupportsDunderGT(Protocol[_T_contra]): + def __gt__(self, other: _T_contra, /) -> bool: ... + +class SupportsDunderLE(Protocol[_T_contra]): + def __le__(self, other: _T_contra, /) -> bool: ... + +class SupportsDunderGE(Protocol[_T_contra]): + def __ge__(self, other: _T_contra, /) -> bool: ... + +class SupportsAllComparisons( + SupportsDunderLT[Any], SupportsDunderGT[Any], SupportsDunderLE[Any], SupportsDunderGE[Any], Protocol +): ... + +SupportsRichComparison: TypeAlias = SupportsDunderLT[Any] | SupportsDunderGT[Any] +SupportsRichComparisonT = TypeVar("SupportsRichComparisonT", bound=SupportsRichComparison) # noqa: Y001 + +# Dunder protocols + +class SupportsAdd(Protocol[_T_contra, _T_co]): + def __add__(self, x: _T_contra, /) -> _T_co: ... + +class SupportsRAdd(Protocol[_T_contra, _T_co]): + def __radd__(self, x: _T_contra, /) -> _T_co: ... + +class SupportsSub(Protocol[_T_contra, _T_co]): + def __sub__(self, x: _T_contra, /) -> _T_co: ... + +class SupportsRSub(Protocol[_T_contra, _T_co]): + def __rsub__(self, x: _T_contra, /) -> _T_co: ... + +class SupportsDivMod(Protocol[_T_contra, _T_co]): + def __divmod__(self, other: _T_contra, /) -> _T_co: ... + +class SupportsRDivMod(Protocol[_T_contra, _T_co]): + def __rdivmod__(self, other: _T_contra, /) -> _T_co: ... + +# This protocol is generic over the iterator type, while Iterable is +# generic over the type that is iterated over. +class SupportsIter(Protocol[_T_co]): + def __iter__(self) -> _T_co: ... + +# This protocol is generic over the iterator type, while AsyncIterable is +# generic over the type that is iterated over. +class SupportsAiter(Protocol[_T_co]): + def __aiter__(self) -> _T_co: ... + +class SupportsLenAndGetItem(Protocol[_T_co]): + def __len__(self) -> int: ... + def __getitem__(self, k: int, /) -> _T_co: ... + +class SupportsTrunc(Protocol): + def __trunc__(self) -> int: ... + +# Mapping-like protocols + +# stable +class SupportsItems(Protocol[_KT_co, _VT_co]): + def items(self) -> AbstractSet[tuple[_KT_co, _VT_co]]: ... + +# stable +class SupportsKeysAndGetItem(Protocol[_KT, _VT_co]): + def keys(self) -> Iterable[_KT]: ... + def __getitem__(self, key: _KT, /) -> _VT_co: ... + +# This protocol is currently under discussion. Use SupportsContainsAndGetItem +# instead, if you require the __contains__ method. +# See https://github.com/python/typeshed/issues/11822. +class SupportsGetItem(Protocol[_KT_contra, _VT_co]): + def __contains__(self, x: Any, /) -> bool: ... + def __getitem__(self, key: _KT_contra, /) -> _VT_co: ... + +# stable +class SupportsContainsAndGetItem(Protocol[_KT_contra, _VT_co]): + def __contains__(self, x: Any, /) -> bool: ... + def __getitem__(self, key: _KT_contra, /) -> _VT_co: ... + +# stable +class SupportsItemAccess(Protocol[_KT_contra, _VT]): + def __contains__(self, x: Any, /) -> bool: ... + def __getitem__(self, key: _KT_contra, /) -> _VT: ... + def __setitem__(self, key: _KT_contra, value: _VT, /) -> None: ... + def __delitem__(self, key: _KT_contra, /) -> None: ... + +StrPath: TypeAlias = str | PathLike[str] # stable +BytesPath: TypeAlias = bytes | PathLike[bytes] # stable +GenericPath: TypeAlias = AnyStr | PathLike[AnyStr] +StrOrBytesPath: TypeAlias = str | bytes | PathLike[str] | PathLike[bytes] # stable + +OpenTextModeUpdating: TypeAlias = Literal[ + "r+", + "+r", + "rt+", + "r+t", + "+rt", + "tr+", + "t+r", + "+tr", + "w+", + "+w", + "wt+", + "w+t", + "+wt", + "tw+", + "t+w", + "+tw", + "a+", + "+a", + "at+", + "a+t", + "+at", + "ta+", + "t+a", + "+ta", + "x+", + "+x", + "xt+", + "x+t", + "+xt", + "tx+", + "t+x", + "+tx", +] +OpenTextModeWriting: TypeAlias = Literal["w", "wt", "tw", "a", "at", "ta", "x", "xt", "tx"] +OpenTextModeReading: TypeAlias = Literal["r", "rt", "tr", "U", "rU", "Ur", "rtU", "rUt", "Urt", "trU", "tUr", "Utr"] +OpenTextMode: TypeAlias = OpenTextModeUpdating | OpenTextModeWriting | OpenTextModeReading +OpenBinaryModeUpdating: TypeAlias = Literal[ + "rb+", + "r+b", + "+rb", + "br+", + "b+r", + "+br", + "wb+", + "w+b", + "+wb", + "bw+", + "b+w", + "+bw", + "ab+", + "a+b", + "+ab", + "ba+", + "b+a", + "+ba", + "xb+", + "x+b", + "+xb", + "bx+", + "b+x", + "+bx", +] +OpenBinaryModeWriting: TypeAlias = Literal["wb", "bw", "ab", "ba", "xb", "bx"] +OpenBinaryModeReading: TypeAlias = Literal["rb", "br", "rbU", "rUb", "Urb", "brU", "bUr", "Ubr"] +OpenBinaryMode: TypeAlias = OpenBinaryModeUpdating | OpenBinaryModeReading | OpenBinaryModeWriting + +# stable +class HasFileno(Protocol): + def fileno(self) -> int: ... + +FileDescriptor: TypeAlias = int # stable +FileDescriptorLike: TypeAlias = int | HasFileno # stable +FileDescriptorOrPath: TypeAlias = int | StrOrBytesPath + +# stable +class SupportsRead(Protocol[_T_co]): + def read(self, length: int = ..., /) -> _T_co: ... + +# stable +class SupportsReadline(Protocol[_T_co]): + def readline(self, length: int = ..., /) -> _T_co: ... + +# stable +class SupportsNoArgReadline(Protocol[_T_co]): + def readline(self) -> _T_co: ... + +# stable +class SupportsWrite(Protocol[_T_contra]): + def write(self, s: _T_contra, /) -> object: ... + +# stable +class SupportsFlush(Protocol): + def flush(self) -> object: ... + +# Unfortunately PEP 688 does not allow us to distinguish read-only +# from writable buffers. We use these aliases for readability for now. +# Perhaps a future extension of the buffer protocol will allow us to +# distinguish these cases in the type system. +ReadOnlyBuffer: TypeAlias = Buffer # stable +# Anything that implements the read-write buffer interface. +WriteableBuffer: TypeAlias = Buffer +# Same as WriteableBuffer, but also includes read-only buffer types (like bytes). +ReadableBuffer: TypeAlias = Buffer # stable + +class SliceableBuffer(Buffer, Protocol): + def __getitem__(self, slice: slice, /) -> Sequence[int]: ... + +class IndexableBuffer(Buffer, Protocol): + def __getitem__(self, i: int, /) -> int: ... + +class SupportsGetItemBuffer(SliceableBuffer, IndexableBuffer, Protocol): + def __contains__(self, x: Any, /) -> bool: ... + @overload + def __getitem__(self, slice: slice, /) -> Sequence[int]: ... + @overload + def __getitem__(self, i: int, /) -> int: ... + +class SizedBuffer(Sized, Buffer, Protocol): ... + +# for compatibility with third-party stubs that may use this +_BufferWithLen: TypeAlias = SizedBuffer # not stable # noqa: Y047 + +ExcInfo: TypeAlias = tuple[type[BaseException], BaseException, TracebackType] +OptExcInfo: TypeAlias = ExcInfo | tuple[None, None, None] + +# stable +if sys.version_info >= (3, 10): + from types import NoneType as NoneType +else: + # Used by type checkers for checks involving None (does not exist at runtime) + @final + class NoneType: + def __bool__(self) -> Literal[False]: ... + +# This is an internal CPython type that is like, but subtly different from, a NamedTuple +# Subclasses of this type are found in multiple modules. +# In typeshed, `structseq` is only ever used as a mixin in combination with a fixed-length `Tuple` +# See discussion at #6546 & #6560 +# `structseq` classes are unsubclassable, so are all decorated with `@final`. +class structseq(Generic[_T_co]): + n_fields: Final[int] + n_unnamed_fields: Final[int] + n_sequence_fields: Final[int] + # The first parameter will generally only take an iterable of a specific length. + # E.g. `os.uname_result` takes any iterable of length exactly 5. + # + # The second parameter will accept a dict of any kind without raising an exception, + # but only has any meaning if you supply it a dict where the keys are strings. + # https://github.com/python/typeshed/pull/6560#discussion_r767149830 + def __new__(cls: type[Self], sequence: Iterable[_T_co], dict: dict[str, Any] = ...) -> Self: ... + if sys.version_info >= (3, 13): + def __replace__(self: Self, **kwargs: Any) -> Self: ... + +# Superset of typing.AnyStr that also includes LiteralString +AnyOrLiteralStr = TypeVar("AnyOrLiteralStr", str, bytes, LiteralString) # noqa: Y001 + +# Represents when str or LiteralStr is acceptable. Useful for string processing +# APIs where literalness of return value depends on literalness of inputs +StrOrLiteralStr = TypeVar("StrOrLiteralStr", LiteralString, str) # noqa: Y001 + +# Objects suitable to be passed to sys.setprofile, threading.setprofile, and similar +ProfileFunction: TypeAlias = Callable[[FrameType, str, Any], object] + +# Objects suitable to be passed to sys.settrace, threading.settrace, and similar +TraceFunction: TypeAlias = Callable[[FrameType, str, Any], TraceFunction | None] + +# experimental +# Might not work as expected for pyright, see +# https://github.com/python/typeshed/pull/9362 +# https://github.com/microsoft/pyright/issues/4339 +class DataclassInstance(Protocol): + __dataclass_fields__: ClassVar[dict[str, Field[Any]]] + +# Anything that can be passed to the int/float constructors +ConvertibleToInt: TypeAlias = str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc +ConvertibleToFloat: TypeAlias = str | ReadableBuffer | SupportsFloat | SupportsIndex + +# A few classes updated from Foo(str, Enum) to Foo(StrEnum). This is a convenience so these +# can be accurate on all python versions without getting too wordy +if sys.version_info >= (3, 11): + from enum import StrEnum as StrEnum +else: + from enum import Enum + + class StrEnum(str, Enum): ... diff --git a/mypy/typeshed/stdlib/_typeshed/dbapi.pyi b/mypy/typeshed/stdlib/_typeshed/dbapi.pyi new file mode 100644 index 000000000000..d54fbee57042 --- /dev/null +++ b/mypy/typeshed/stdlib/_typeshed/dbapi.pyi @@ -0,0 +1,37 @@ +# PEP 249 Database API 2.0 Types +# https://www.python.org/dev/peps/pep-0249/ + +from collections.abc import Mapping, Sequence +from typing import Any, Protocol +from typing_extensions import TypeAlias + +DBAPITypeCode: TypeAlias = Any | None +# Strictly speaking, this should be a Sequence, but the type system does +# not support fixed-length sequences. +DBAPIColumnDescription: TypeAlias = tuple[str, DBAPITypeCode, int | None, int | None, int | None, int | None, bool | None] + +class DBAPIConnection(Protocol): + def close(self) -> object: ... + def commit(self) -> object: ... + # optional: + # def rollback(self) -> Any: ... + def cursor(self) -> DBAPICursor: ... + +class DBAPICursor(Protocol): + @property + def description(self) -> Sequence[DBAPIColumnDescription] | None: ... + @property + def rowcount(self) -> int: ... + # optional: + # def callproc(self, procname: str, parameters: Sequence[Any] = ..., /) -> Sequence[Any]: ... + def close(self) -> object: ... + def execute(self, operation: str, parameters: Sequence[Any] | Mapping[str, Any] = ..., /) -> object: ... + def executemany(self, operation: str, seq_of_parameters: Sequence[Sequence[Any]], /) -> object: ... + def fetchone(self) -> Sequence[Any] | None: ... + def fetchmany(self, size: int = ..., /) -> Sequence[Sequence[Any]]: ... + def fetchall(self) -> Sequence[Sequence[Any]]: ... + # optional: + # def nextset(self) -> None | Literal[True]: ... + arraysize: int + def setinputsizes(self, sizes: Sequence[DBAPITypeCode | int | None], /) -> object: ... + def setoutputsize(self, size: int, column: int = ..., /) -> object: ... diff --git a/mypy/typeshed/stdlib/_typeshed/importlib.pyi b/mypy/typeshed/stdlib/_typeshed/importlib.pyi new file mode 100644 index 000000000000..a4e56cdaff62 --- /dev/null +++ b/mypy/typeshed/stdlib/_typeshed/importlib.pyi @@ -0,0 +1,18 @@ +# Implicit protocols used in importlib. +# We intentionally omit deprecated and optional methods. + +from collections.abc import Sequence +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import Protocol + +__all__ = ["LoaderProtocol", "MetaPathFinderProtocol", "PathEntryFinderProtocol"] + +class LoaderProtocol(Protocol): + def load_module(self, fullname: str, /) -> ModuleType: ... + +class MetaPathFinderProtocol(Protocol): + def find_spec(self, fullname: str, path: Sequence[str] | None, target: ModuleType | None = ..., /) -> ModuleSpec | None: ... + +class PathEntryFinderProtocol(Protocol): + def find_spec(self, fullname: str, target: ModuleType | None = ..., /) -> ModuleSpec | None: ... diff --git a/mypy/typeshed/stdlib/_typeshed/wsgi.pyi b/mypy/typeshed/stdlib/_typeshed/wsgi.pyi new file mode 100644 index 000000000000..63f204eb889b --- /dev/null +++ b/mypy/typeshed/stdlib/_typeshed/wsgi.pyi @@ -0,0 +1,44 @@ +# Types to support PEP 3333 (WSGI) +# +# Obsolete since Python 3.11: Use wsgiref.types instead. +# +# See the README.md file in this directory for more information. + +import sys +from _typeshed import OptExcInfo +from collections.abc import Callable, Iterable, Iterator +from typing import Any, Protocol +from typing_extensions import TypeAlias + +class _Readable(Protocol): + def read(self, size: int = ..., /) -> bytes: ... + # Optional: def close(self) -> object: ... + +if sys.version_info >= (3, 11): + from wsgiref.types import * +else: + # stable + class StartResponse(Protocol): + def __call__( + self, status: str, headers: list[tuple[str, str]], exc_info: OptExcInfo | None = ..., / + ) -> Callable[[bytes], object]: ... + + WSGIEnvironment: TypeAlias = dict[str, Any] # stable + WSGIApplication: TypeAlias = Callable[[WSGIEnvironment, StartResponse], Iterable[bytes]] # stable + + # WSGI input streams per PEP 3333, stable + class InputStream(Protocol): + def read(self, size: int = ..., /) -> bytes: ... + def readline(self, size: int = ..., /) -> bytes: ... + def readlines(self, hint: int = ..., /) -> list[bytes]: ... + def __iter__(self) -> Iterator[bytes]: ... + + # WSGI error streams per PEP 3333, stable + class ErrorStream(Protocol): + def flush(self) -> object: ... + def write(self, s: str, /) -> object: ... + def writelines(self, seq: list[str], /) -> object: ... + + # Optional file wrapper in wsgi.file_wrapper + class FileWrapper(Protocol): + def __call__(self, file: _Readable, block_size: int = ..., /) -> Iterable[bytes]: ... diff --git a/mypy/typeshed/stdlib/_typeshed/xml.pyi b/mypy/typeshed/stdlib/_typeshed/xml.pyi new file mode 100644 index 000000000000..6cd1b39af628 --- /dev/null +++ b/mypy/typeshed/stdlib/_typeshed/xml.pyi @@ -0,0 +1,9 @@ +# See the README.md file in this directory for more information. + +from typing import Any, Protocol + +# As defined https://docs.python.org/3/library/xml.dom.html#domimplementation-objects +class DOMImplementation(Protocol): + def hasFeature(self, feature: str, version: str | None, /) -> bool: ... + def createDocument(self, namespaceUri: str, qualifiedName: str, doctype: Any | None, /) -> Any: ... + def createDocumentType(self, qualifiedName: str, publicId: str, systemId: str, /) -> Any: ... diff --git a/mypy/typeshed/stdlib/_warnings.pyi b/mypy/typeshed/stdlib/_warnings.pyi new file mode 100644 index 000000000000..2e571e676c97 --- /dev/null +++ b/mypy/typeshed/stdlib/_warnings.pyi @@ -0,0 +1,55 @@ +import sys +from typing import Any, overload + +_defaultaction: str +_onceregistry: dict[Any, Any] +filters: list[tuple[str, str | None, type[Warning], str | None, int]] + +if sys.version_info >= (3, 12): + @overload + def warn( + message: str, + category: type[Warning] | None = None, + stacklevel: int = 1, + source: Any | None = None, + *, + skip_file_prefixes: tuple[str, ...] = (), + ) -> None: ... + @overload + def warn( + message: Warning, + category: Any = None, + stacklevel: int = 1, + source: Any | None = None, + *, + skip_file_prefixes: tuple[str, ...] = (), + ) -> None: ... + +else: + @overload + def warn(message: str, category: type[Warning] | None = None, stacklevel: int = 1, source: Any | None = None) -> None: ... + @overload + def warn(message: Warning, category: Any = None, stacklevel: int = 1, source: Any | None = None) -> None: ... + +@overload +def warn_explicit( + message: str, + category: type[Warning], + filename: str, + lineno: int, + module: str | None = ..., + registry: dict[str | tuple[str, type[Warning], int], int] | None = ..., + module_globals: dict[str, Any] | None = ..., + source: Any | None = ..., +) -> None: ... +@overload +def warn_explicit( + message: Warning, + category: Any, + filename: str, + lineno: int, + module: str | None = ..., + registry: dict[str | tuple[str, type[Warning], int], int] | None = ..., + module_globals: dict[str, Any] | None = ..., + source: Any | None = ..., +) -> None: ... diff --git a/mypy/typeshed/stdlib/_weakref.pyi b/mypy/typeshed/stdlib/_weakref.pyi new file mode 100644 index 000000000000..a744340afaab --- /dev/null +++ b/mypy/typeshed/stdlib/_weakref.pyi @@ -0,0 +1,15 @@ +from collections.abc import Callable +from typing import Any, TypeVar, overload +from weakref import CallableProxyType as CallableProxyType, ProxyType as ProxyType, ReferenceType as ReferenceType, ref as ref + +_C = TypeVar("_C", bound=Callable[..., Any]) +_T = TypeVar("_T") + +def getweakrefcount(object: Any, /) -> int: ... +def getweakrefs(object: Any, /) -> list[Any]: ... + +# Return CallableProxyType if object is callable, ProxyType otherwise +@overload +def proxy(object: _C, callback: Callable[[_C], Any] | None = None, /) -> CallableProxyType[_C]: ... +@overload +def proxy(object: _T, callback: Callable[[_T], Any] | None = None, /) -> Any: ... diff --git a/mypy/typeshed/stdlib/_weakrefset.pyi b/mypy/typeshed/stdlib/_weakrefset.pyi new file mode 100644 index 000000000000..2a4e682f64ed --- /dev/null +++ b/mypy/typeshed/stdlib/_weakrefset.pyi @@ -0,0 +1,51 @@ +import sys +from collections.abc import Iterable, Iterator, MutableSet +from typing import Any, TypeVar, overload +from typing_extensions import Self + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["WeakSet"] + +_S = TypeVar("_S") +_T = TypeVar("_T") + +class WeakSet(MutableSet[_T]): + @overload + def __init__(self, data: None = None) -> None: ... + @overload + def __init__(self, data: Iterable[_T]) -> None: ... + def add(self, item: _T) -> None: ... + def discard(self, item: _T) -> None: ... + def copy(self) -> Self: ... + def remove(self, item: _T) -> None: ... + def update(self, other: Iterable[_T]) -> None: ... + def __contains__(self, item: object) -> bool: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[_T]: ... + def __ior__(self, other: Iterable[_T]) -> Self: ... # type: ignore[override,misc] + def difference(self, other: Iterable[_T]) -> Self: ... + def __sub__(self, other: Iterable[Any]) -> Self: ... + def difference_update(self, other: Iterable[Any]) -> None: ... + def __isub__(self, other: Iterable[Any]) -> Self: ... + def intersection(self, other: Iterable[_T]) -> Self: ... + def __and__(self, other: Iterable[Any]) -> Self: ... + def intersection_update(self, other: Iterable[Any]) -> None: ... + def __iand__(self, other: Iterable[Any]) -> Self: ... + def issubset(self, other: Iterable[_T]) -> bool: ... + def __le__(self, other: Iterable[_T]) -> bool: ... + def __lt__(self, other: Iterable[_T]) -> bool: ... + def issuperset(self, other: Iterable[_T]) -> bool: ... + def __ge__(self, other: Iterable[_T]) -> bool: ... + def __gt__(self, other: Iterable[_T]) -> bool: ... + def __eq__(self, other: object) -> bool: ... + def symmetric_difference(self, other: Iterable[_S]) -> WeakSet[_S | _T]: ... + def __xor__(self, other: Iterable[_S]) -> WeakSet[_S | _T]: ... + def symmetric_difference_update(self, other: Iterable[_T]) -> None: ... + def __ixor__(self, other: Iterable[_T]) -> Self: ... # type: ignore[override,misc] + def union(self, other: Iterable[_S]) -> WeakSet[_S | _T]: ... + def __or__(self, other: Iterable[_S]) -> WeakSet[_S | _T]: ... + def isdisjoint(self, other: Iterable[_T]) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... diff --git a/mypy/typeshed/stdlib/_winapi.pyi b/mypy/typeshed/stdlib/_winapi.pyi new file mode 100644 index 000000000000..0f71a0687748 --- /dev/null +++ b/mypy/typeshed/stdlib/_winapi.pyi @@ -0,0 +1,283 @@ +import sys +from _typeshed import ReadableBuffer +from collections.abc import Sequence +from typing import Any, Final, Literal, NoReturn, final, overload + +if sys.platform == "win32": + ABOVE_NORMAL_PRIORITY_CLASS: Final = 0x8000 + BELOW_NORMAL_PRIORITY_CLASS: Final = 0x4000 + + CREATE_BREAKAWAY_FROM_JOB: Final = 0x1000000 + CREATE_DEFAULT_ERROR_MODE: Final = 0x4000000 + CREATE_NO_WINDOW: Final = 0x8000000 + CREATE_NEW_CONSOLE: Final = 0x10 + CREATE_NEW_PROCESS_GROUP: Final = 0x200 + + DETACHED_PROCESS: Final = 8 + DUPLICATE_CLOSE_SOURCE: Final = 1 + DUPLICATE_SAME_ACCESS: Final = 2 + + ERROR_ALREADY_EXISTS: Final = 183 + ERROR_BROKEN_PIPE: Final = 109 + ERROR_IO_PENDING: Final = 997 + ERROR_MORE_DATA: Final = 234 + ERROR_NETNAME_DELETED: Final = 64 + ERROR_NO_DATA: Final = 232 + ERROR_NO_SYSTEM_RESOURCES: Final = 1450 + ERROR_OPERATION_ABORTED: Final = 995 + ERROR_PIPE_BUSY: Final = 231 + ERROR_PIPE_CONNECTED: Final = 535 + ERROR_SEM_TIMEOUT: Final = 121 + + FILE_FLAG_FIRST_PIPE_INSTANCE: Final = 0x80000 + FILE_FLAG_OVERLAPPED: Final = 0x40000000 + + FILE_GENERIC_READ: Final = 1179785 + FILE_GENERIC_WRITE: Final = 1179926 + + FILE_MAP_ALL_ACCESS: Final = 983071 + FILE_MAP_COPY: Final = 1 + FILE_MAP_EXECUTE: Final = 32 + FILE_MAP_READ: Final = 4 + FILE_MAP_WRITE: Final = 2 + + FILE_TYPE_CHAR: Final = 2 + FILE_TYPE_DISK: Final = 1 + FILE_TYPE_PIPE: Final = 3 + FILE_TYPE_REMOTE: Final = 32768 + FILE_TYPE_UNKNOWN: Final = 0 + + GENERIC_READ: Final = 0x80000000 + GENERIC_WRITE: Final = 0x40000000 + HIGH_PRIORITY_CLASS: Final = 0x80 + INFINITE: Final = 0xFFFFFFFF + # Ignore the Flake8 error -- flake8-pyi assumes + # most numbers this long will be implementation details, + # but here we can see that it's a power of 2 + INVALID_HANDLE_VALUE: Final = 0xFFFFFFFFFFFFFFFF # noqa: Y054 + IDLE_PRIORITY_CLASS: Final = 0x40 + NORMAL_PRIORITY_CLASS: Final = 0x20 + REALTIME_PRIORITY_CLASS: Final = 0x100 + NMPWAIT_WAIT_FOREVER: Final = 0xFFFFFFFF + + MEM_COMMIT: Final = 0x1000 + MEM_FREE: Final = 0x10000 + MEM_IMAGE: Final = 0x1000000 + MEM_MAPPED: Final = 0x40000 + MEM_PRIVATE: Final = 0x20000 + MEM_RESERVE: Final = 0x2000 + + NULL: Final = 0 + OPEN_EXISTING: Final = 3 + + PIPE_ACCESS_DUPLEX: Final = 3 + PIPE_ACCESS_INBOUND: Final = 1 + PIPE_READMODE_MESSAGE: Final = 2 + PIPE_TYPE_MESSAGE: Final = 4 + PIPE_UNLIMITED_INSTANCES: Final = 255 + PIPE_WAIT: Final = 0 + + PAGE_EXECUTE: Final = 0x10 + PAGE_EXECUTE_READ: Final = 0x20 + PAGE_EXECUTE_READWRITE: Final = 0x40 + PAGE_EXECUTE_WRITECOPY: Final = 0x80 + PAGE_GUARD: Final = 0x100 + PAGE_NOACCESS: Final = 0x1 + PAGE_NOCACHE: Final = 0x200 + PAGE_READONLY: Final = 0x2 + PAGE_READWRITE: Final = 0x4 + PAGE_WRITECOMBINE: Final = 0x400 + PAGE_WRITECOPY: Final = 0x8 + + PROCESS_ALL_ACCESS: Final = 0x1FFFFF + PROCESS_DUP_HANDLE: Final = 0x40 + + SEC_COMMIT: Final = 0x8000000 + SEC_IMAGE: Final = 0x1000000 + SEC_LARGE_PAGES: Final = 0x80000000 + SEC_NOCACHE: Final = 0x10000000 + SEC_RESERVE: Final = 0x4000000 + SEC_WRITECOMBINE: Final = 0x40000000 + + if sys.version_info >= (3, 13): + STARTF_FORCEOFFFEEDBACK: Final = 0x80 + STARTF_FORCEONFEEDBACK: Final = 0x40 + STARTF_PREVENTPINNING: Final = 0x2000 + STARTF_RUNFULLSCREEN: Final = 0x20 + STARTF_TITLEISAPPID: Final = 0x1000 + STARTF_TITLEISLINKNAME: Final = 0x800 + STARTF_UNTRUSTEDSOURCE: Final = 0x8000 + STARTF_USECOUNTCHARS: Final = 0x8 + STARTF_USEFILLATTRIBUTE: Final = 0x10 + STARTF_USEHOTKEY: Final = 0x200 + STARTF_USEPOSITION: Final = 0x4 + STARTF_USESIZE: Final = 0x2 + + STARTF_USESHOWWINDOW: Final = 0x1 + STARTF_USESTDHANDLES: Final = 0x100 + + STD_ERROR_HANDLE: Final = 0xFFFFFFF4 + STD_OUTPUT_HANDLE: Final = 0xFFFFFFF5 + STD_INPUT_HANDLE: Final = 0xFFFFFFF6 + + STILL_ACTIVE: Final = 259 + SW_HIDE: Final = 0 + SYNCHRONIZE: Final = 0x100000 + WAIT_ABANDONED_0: Final = 128 + WAIT_OBJECT_0: Final = 0 + WAIT_TIMEOUT: Final = 258 + + if sys.version_info >= (3, 10): + LOCALE_NAME_INVARIANT: str + LOCALE_NAME_MAX_LENGTH: int + LOCALE_NAME_SYSTEM_DEFAULT: str + LOCALE_NAME_USER_DEFAULT: str | None + + LCMAP_FULLWIDTH: int + LCMAP_HALFWIDTH: int + LCMAP_HIRAGANA: int + LCMAP_KATAKANA: int + LCMAP_LINGUISTIC_CASING: int + LCMAP_LOWERCASE: int + LCMAP_SIMPLIFIED_CHINESE: int + LCMAP_TITLECASE: int + LCMAP_TRADITIONAL_CHINESE: int + LCMAP_UPPERCASE: int + + if sys.version_info >= (3, 12): + COPYFILE2_CALLBACK_CHUNK_STARTED: Final = 1 + COPYFILE2_CALLBACK_CHUNK_FINISHED: Final = 2 + COPYFILE2_CALLBACK_STREAM_STARTED: Final = 3 + COPYFILE2_CALLBACK_STREAM_FINISHED: Final = 4 + COPYFILE2_CALLBACK_POLL_CONTINUE: Final = 5 + COPYFILE2_CALLBACK_ERROR: Final = 6 + + COPYFILE2_PROGRESS_CONTINUE: Final = 0 + COPYFILE2_PROGRESS_CANCEL: Final = 1 + COPYFILE2_PROGRESS_STOP: Final = 2 + COPYFILE2_PROGRESS_QUIET: Final = 3 + COPYFILE2_PROGRESS_PAUSE: Final = 4 + + COPY_FILE_FAIL_IF_EXISTS: Final = 0x1 + COPY_FILE_RESTARTABLE: Final = 0x2 + COPY_FILE_OPEN_SOURCE_FOR_WRITE: Final = 0x4 + COPY_FILE_ALLOW_DECRYPTED_DESTINATION: Final = 0x8 + COPY_FILE_COPY_SYMLINK: Final = 0x800 + COPY_FILE_NO_BUFFERING: Final = 0x1000 + COPY_FILE_REQUEST_SECURITY_PRIVILEGES: Final = 0x2000 + COPY_FILE_RESUME_FROM_PAUSE: Final = 0x4000 + COPY_FILE_NO_OFFLOAD: Final = 0x40000 + COPY_FILE_REQUEST_COMPRESSED_TRAFFIC: Final = 0x10000000 + + ERROR_ACCESS_DENIED: Final = 5 + ERROR_PRIVILEGE_NOT_HELD: Final = 1314 + + def CloseHandle(handle: int, /) -> None: ... + @overload + def ConnectNamedPipe(handle: int, overlapped: Literal[True]) -> Overlapped: ... + @overload + def ConnectNamedPipe(handle: int, overlapped: Literal[False] = False) -> None: ... + @overload + def ConnectNamedPipe(handle: int, overlapped: bool) -> Overlapped | None: ... + def CreateFile( + file_name: str, + desired_access: int, + share_mode: int, + security_attributes: int, + creation_disposition: int, + flags_and_attributes: int, + template_file: int, + /, + ) -> int: ... + def CreateJunction(src_path: str, dst_path: str, /) -> None: ... + def CreateNamedPipe( + name: str, + open_mode: int, + pipe_mode: int, + max_instances: int, + out_buffer_size: int, + in_buffer_size: int, + default_timeout: int, + security_attributes: int, + /, + ) -> int: ... + def CreatePipe(pipe_attrs: Any, size: int, /) -> tuple[int, int]: ... + def CreateProcess( + application_name: str | None, + command_line: str | None, + proc_attrs: Any, + thread_attrs: Any, + inherit_handles: bool, + creation_flags: int, + env_mapping: dict[str, str], + current_directory: str | None, + startup_info: Any, + /, + ) -> tuple[int, int, int, int]: ... + def DuplicateHandle( + source_process_handle: int, + source_handle: int, + target_process_handle: int, + desired_access: int, + inherit_handle: bool, + options: int = 0, + /, + ) -> int: ... + def ExitProcess(ExitCode: int, /) -> NoReturn: ... + def GetACP() -> int: ... + def GetFileType(handle: int) -> int: ... + def GetCurrentProcess() -> int: ... + def GetExitCodeProcess(process: int, /) -> int: ... + def GetLastError() -> int: ... + def GetModuleFileName(module_handle: int, /) -> str: ... + def GetStdHandle(std_handle: int, /) -> int: ... + def GetVersion() -> int: ... + def OpenProcess(desired_access: int, inherit_handle: bool, process_id: int, /) -> int: ... + def PeekNamedPipe(handle: int, size: int = 0, /) -> tuple[int, int] | tuple[bytes, int, int]: ... + if sys.version_info >= (3, 10): + def LCMapStringEx(locale: str, flags: int, src: str) -> str: ... + def UnmapViewOfFile(address: int, /) -> None: ... + + @overload + def ReadFile(handle: int, size: int, overlapped: Literal[True]) -> tuple[Overlapped, int]: ... + @overload + def ReadFile(handle: int, size: int, overlapped: Literal[False] = False) -> tuple[bytes, int]: ... + @overload + def ReadFile(handle: int, size: int, overlapped: int | bool) -> tuple[Any, int]: ... + def SetNamedPipeHandleState( + named_pipe: int, mode: int | None, max_collection_count: int | None, collect_data_timeout: int | None, / + ) -> None: ... + def TerminateProcess(handle: int, exit_code: int, /) -> None: ... + def WaitForMultipleObjects(handle_seq: Sequence[int], wait_flag: bool, milliseconds: int = 0xFFFFFFFF, /) -> int: ... + def WaitForSingleObject(handle: int, milliseconds: int, /) -> int: ... + def WaitNamedPipe(name: str, timeout: int, /) -> None: ... + @overload + def WriteFile(handle: int, buffer: ReadableBuffer, overlapped: Literal[True]) -> tuple[Overlapped, int]: ... + @overload + def WriteFile(handle: int, buffer: ReadableBuffer, overlapped: Literal[False] = False) -> tuple[int, int]: ... + @overload + def WriteFile(handle: int, buffer: ReadableBuffer, overlapped: int | bool) -> tuple[Any, int]: ... + @final + class Overlapped: + event: int + def GetOverlappedResult(self, wait: bool, /) -> tuple[int, int]: ... + def cancel(self) -> None: ... + def getbuffer(self) -> bytes | None: ... + + if sys.version_info >= (3, 13): + def BatchedWaitForMultipleObjects( + handle_seq: Sequence[int], wait_all: bool, milliseconds: int = 0xFFFFFFFF + ) -> list[int]: ... + def CreateEventW(security_attributes: int, manual_reset: bool, initial_state: bool, name: str | None) -> int: ... + def CreateMutexW(security_attributes: int, initial_owner: bool, name: str) -> int: ... + def GetLongPathName(path: str) -> str: ... + def GetShortPathName(path: str) -> str: ... + def OpenEventW(desired_access: int, inherit_handle: bool, name: str) -> int: ... + def OpenMutexW(desired_access: int, inherit_handle: bool, name: str) -> int: ... + def ReleaseMutex(mutex: int) -> None: ... + def ResetEvent(event: int) -> None: ... + def SetEvent(event: int) -> None: ... + + if sys.version_info >= (3, 12): + def CopyFile2(existing_file_name: str, new_file_name: str, flags: int, progress_routine: int | None = None) -> int: ... + def NeedCurrentDirectoryForExePath(exe_name: str, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/abc.pyi b/mypy/typeshed/stdlib/abc.pyi new file mode 100644 index 000000000000..fdca48ac7aaf --- /dev/null +++ b/mypy/typeshed/stdlib/abc.pyi @@ -0,0 +1,51 @@ +import _typeshed +import sys +from _typeshed import SupportsWrite +from collections.abc import Callable +from typing import Any, Literal, TypeVar +from typing_extensions import Concatenate, ParamSpec, deprecated + +_T = TypeVar("_T") +_R_co = TypeVar("_R_co", covariant=True) +_FuncT = TypeVar("_FuncT", bound=Callable[..., Any]) +_P = ParamSpec("_P") + +# These definitions have special processing in mypy +class ABCMeta(type): + __abstractmethods__: frozenset[str] + if sys.version_info >= (3, 11): + def __new__( + mcls: type[_typeshed.Self], name: str, bases: tuple[type, ...], namespace: dict[str, Any], /, **kwargs: Any + ) -> _typeshed.Self: ... + else: + def __new__( + mcls: type[_typeshed.Self], name: str, bases: tuple[type, ...], namespace: dict[str, Any], **kwargs: Any + ) -> _typeshed.Self: ... + + def __instancecheck__(cls: ABCMeta, instance: Any) -> bool: ... + def __subclasscheck__(cls: ABCMeta, subclass: type) -> bool: ... + def _dump_registry(cls: ABCMeta, file: SupportsWrite[str] | None = None) -> None: ... + def register(cls: ABCMeta, subclass: type[_T]) -> type[_T]: ... + +def abstractmethod(funcobj: _FuncT) -> _FuncT: ... +@deprecated("Use 'classmethod' with 'abstractmethod' instead") +class abstractclassmethod(classmethod[_T, _P, _R_co]): + __isabstractmethod__: Literal[True] + def __init__(self, callable: Callable[Concatenate[type[_T], _P], _R_co]) -> None: ... + +@deprecated("Use 'staticmethod' with 'abstractmethod' instead") +class abstractstaticmethod(staticmethod[_P, _R_co]): + __isabstractmethod__: Literal[True] + def __init__(self, callable: Callable[_P, _R_co]) -> None: ... + +@deprecated("Use 'property' with 'abstractmethod' instead") +class abstractproperty(property): + __isabstractmethod__: Literal[True] + +class ABC(metaclass=ABCMeta): + __slots__ = () + +def get_cache_token() -> object: ... + +if sys.version_info >= (3, 10): + def update_abstractmethods(cls: type[_T]) -> type[_T]: ... diff --git a/mypy/typeshed/stdlib/aifc.pyi b/mypy/typeshed/stdlib/aifc.pyi new file mode 100644 index 000000000000..05bf53986b29 --- /dev/null +++ b/mypy/typeshed/stdlib/aifc.pyi @@ -0,0 +1,91 @@ +import sys +from types import TracebackType +from typing import IO, Any, Literal, NamedTuple, overload +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + __all__ = ["Error", "open"] +else: + __all__ = ["Error", "open", "openfp"] + +class Error(Exception): ... + +class _aifc_params(NamedTuple): + nchannels: int + sampwidth: int + framerate: int + nframes: int + comptype: bytes + compname: bytes + +_File: TypeAlias = str | IO[bytes] +_Marker: TypeAlias = tuple[int, int, bytes] + +class Aifc_read: + def __init__(self, f: _File) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def initfp(self, file: IO[bytes]) -> None: ... + def getfp(self) -> IO[bytes]: ... + def rewind(self) -> None: ... + def close(self) -> None: ... + def tell(self) -> int: ... + def getnchannels(self) -> int: ... + def getnframes(self) -> int: ... + def getsampwidth(self) -> int: ... + def getframerate(self) -> int: ... + def getcomptype(self) -> bytes: ... + def getcompname(self) -> bytes: ... + def getparams(self) -> _aifc_params: ... + def getmarkers(self) -> list[_Marker] | None: ... + def getmark(self, id: int) -> _Marker: ... + def setpos(self, pos: int) -> None: ... + def readframes(self, nframes: int) -> bytes: ... + +class Aifc_write: + def __init__(self, f: _File) -> None: ... + def __del__(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def initfp(self, file: IO[bytes]) -> None: ... + def aiff(self) -> None: ... + def aifc(self) -> None: ... + def setnchannels(self, nchannels: int) -> None: ... + def getnchannels(self) -> int: ... + def setsampwidth(self, sampwidth: int) -> None: ... + def getsampwidth(self) -> int: ... + def setframerate(self, framerate: int) -> None: ... + def getframerate(self) -> int: ... + def setnframes(self, nframes: int) -> None: ... + def getnframes(self) -> int: ... + def setcomptype(self, comptype: bytes, compname: bytes) -> None: ... + def getcomptype(self) -> bytes: ... + def getcompname(self) -> bytes: ... + def setparams(self, params: tuple[int, int, int, int, bytes, bytes]) -> None: ... + def getparams(self) -> _aifc_params: ... + def setmark(self, id: int, pos: int, name: bytes) -> None: ... + def getmark(self, id: int) -> _Marker: ... + def getmarkers(self) -> list[_Marker] | None: ... + def tell(self) -> int: ... + def writeframesraw(self, data: Any) -> None: ... # Actual type for data is Buffer Protocol + def writeframes(self, data: Any) -> None: ... + def close(self) -> None: ... + +@overload +def open(f: _File, mode: Literal["r", "rb"]) -> Aifc_read: ... +@overload +def open(f: _File, mode: Literal["w", "wb"]) -> Aifc_write: ... +@overload +def open(f: _File, mode: str | None = None) -> Any: ... + +if sys.version_info < (3, 9): + @overload + def openfp(f: _File, mode: Literal["r", "rb"]) -> Aifc_read: ... + @overload + def openfp(f: _File, mode: Literal["w", "wb"]) -> Aifc_write: ... + @overload + def openfp(f: _File, mode: str | None = None) -> Any: ... diff --git a/mypy/typeshed/stdlib/antigravity.pyi b/mypy/typeshed/stdlib/antigravity.pyi new file mode 100644 index 000000000000..3986e7d1c9f2 --- /dev/null +++ b/mypy/typeshed/stdlib/antigravity.pyi @@ -0,0 +1,3 @@ +from _typeshed import ReadableBuffer + +def geohash(latitude: float, longitude: float, datedow: ReadableBuffer) -> None: ... diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi new file mode 100644 index 000000000000..2526322ac8f6 --- /dev/null +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -0,0 +1,767 @@ +import sys +from _typeshed import sentinel +from collections.abc import Callable, Generator, Iterable, Sequence +from re import Pattern +from typing import IO, Any, Final, Generic, NewType, NoReturn, Protocol, TypeVar, overload +from typing_extensions import Self, TypeAlias, deprecated + +__all__ = [ + "ArgumentParser", + "ArgumentError", + "ArgumentTypeError", + "FileType", + "HelpFormatter", + "ArgumentDefaultsHelpFormatter", + "RawDescriptionHelpFormatter", + "RawTextHelpFormatter", + "MetavarTypeHelpFormatter", + "Namespace", + "Action", + "ONE_OR_MORE", + "OPTIONAL", + "PARSER", + "REMAINDER", + "SUPPRESS", + "ZERO_OR_MORE", +] + +if sys.version_info >= (3, 9): + __all__ += ["BooleanOptionalAction"] + +_T = TypeVar("_T") +_ActionT = TypeVar("_ActionT", bound=Action) +_ArgumentParserT = TypeVar("_ArgumentParserT", bound=ArgumentParser) +_N = TypeVar("_N") +_ActionType: TypeAlias = Callable[[str], Any] | FileType | str +# more precisely, Literal["store", "store_const", "store_true", +# "store_false", "append", "append_const", "count", "help", "version", +# "extend"], but using this would make it hard to annotate callers +# that don't use a literal argument +_ActionStr: TypeAlias = str +# more precisely, Literal["?", "*", "+", "...", "A...", +# "==SUPPRESS=="], but using this would make it hard to annotate +# callers that don't use a literal argument +_NArgsStr: TypeAlias = str + +ONE_OR_MORE: Final = "+" +OPTIONAL: Final = "?" +PARSER: Final = "A..." +REMAINDER: Final = "..." +_SUPPRESS_T = NewType("_SUPPRESS_T", str) +SUPPRESS: _SUPPRESS_T | str # not using Literal because argparse sometimes compares SUPPRESS with is +# the | str is there so that foo = argparse.SUPPRESS; foo = "test" checks out in mypy +ZERO_OR_MORE: Final = "*" +_UNRECOGNIZED_ARGS_ATTR: Final[str] # undocumented + +class ArgumentError(Exception): + argument_name: str | None + message: str + def __init__(self, argument: Action | None, message: str) -> None: ... + +# undocumented +class _AttributeHolder: + def _get_kwargs(self) -> list[tuple[str, Any]]: ... + def _get_args(self) -> list[Any]: ... + +# undocumented +class _ActionsContainer: + description: str | None + prefix_chars: str + argument_default: Any + conflict_handler: str + + _registries: dict[str, dict[Any, Any]] + _actions: list[Action] + _option_string_actions: dict[str, Action] + _action_groups: list[_ArgumentGroup] + _mutually_exclusive_groups: list[_MutuallyExclusiveGroup] + _defaults: dict[str, Any] + _negative_number_matcher: Pattern[str] + _has_negative_number_optionals: list[bool] + def __init__(self, description: str | None, prefix_chars: str, argument_default: Any, conflict_handler: str) -> None: ... + def register(self, registry_name: str, value: Any, object: Any) -> None: ... + def _registry_get(self, registry_name: str, value: Any, default: Any = None) -> Any: ... + def set_defaults(self, **kwargs: Any) -> None: ... + def get_default(self, dest: str) -> Any: ... + def add_argument( + self, + *name_or_flags: str, + action: _ActionStr | type[Action] = ..., + nargs: int | _NArgsStr | _SUPPRESS_T | None = None, + const: Any = ..., + default: Any = ..., + type: _ActionType = ..., + choices: Iterable[_T] | None = ..., + required: bool = ..., + help: str | None = ..., + metavar: str | tuple[str, ...] | None = ..., + dest: str | None = ..., + version: str = ..., + **kwargs: Any, + ) -> Action: ... + def add_argument_group( + self, + title: str | None = None, + description: str | None = None, + *, + prefix_chars: str = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + ) -> _ArgumentGroup: ... + def add_mutually_exclusive_group(self, *, required: bool = False) -> _MutuallyExclusiveGroup: ... + def _add_action(self, action: _ActionT) -> _ActionT: ... + def _remove_action(self, action: Action) -> None: ... + def _add_container_actions(self, container: _ActionsContainer) -> None: ... + def _get_positional_kwargs(self, dest: str, **kwargs: Any) -> dict[str, Any]: ... + def _get_optional_kwargs(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ... + def _pop_action_class(self, kwargs: Any, default: type[Action] | None = None) -> type[Action]: ... + def _get_handler(self) -> Callable[[Action, Iterable[tuple[str, Action]]], Any]: ... + def _check_conflict(self, action: Action) -> None: ... + def _handle_conflict_error(self, action: Action, conflicting_actions: Iterable[tuple[str, Action]]) -> NoReturn: ... + def _handle_conflict_resolve(self, action: Action, conflicting_actions: Iterable[tuple[str, Action]]) -> None: ... + +class _FormatterClass(Protocol): + def __call__(self, *, prog: str) -> HelpFormatter: ... + +class ArgumentParser(_AttributeHolder, _ActionsContainer): + prog: str + usage: str | None + epilog: str | None + formatter_class: _FormatterClass + fromfile_prefix_chars: str | None + add_help: bool + allow_abbrev: bool + + # undocumented + _positionals: _ArgumentGroup + _optionals: _ArgumentGroup + _subparsers: _ArgumentGroup | None + + # Note: the constructor arguments are also used in _SubParsersAction.add_parser. + if sys.version_info >= (3, 9): + def __init__( + self, + prog: str | None = None, + usage: str | None = None, + description: str | None = None, + epilog: str | None = None, + parents: Sequence[ArgumentParser] = [], + formatter_class: _FormatterClass = ..., + prefix_chars: str = "-", + fromfile_prefix_chars: str | None = None, + argument_default: Any = None, + conflict_handler: str = "error", + add_help: bool = True, + allow_abbrev: bool = True, + exit_on_error: bool = True, + ) -> None: ... + else: + def __init__( + self, + prog: str | None = None, + usage: str | None = None, + description: str | None = None, + epilog: str | None = None, + parents: Sequence[ArgumentParser] = [], + formatter_class: _FormatterClass = ..., + prefix_chars: str = "-", + fromfile_prefix_chars: str | None = None, + argument_default: Any = None, + conflict_handler: str = "error", + add_help: bool = True, + allow_abbrev: bool = True, + ) -> None: ... + + @overload + def parse_args(self, args: Sequence[str] | None = None, namespace: None = None) -> Namespace: ... + @overload + def parse_args(self, args: Sequence[str] | None, namespace: _N) -> _N: ... + @overload + def parse_args(self, *, namespace: _N) -> _N: ... + @overload + def add_subparsers( + self: _ArgumentParserT, + *, + title: str = ..., + description: str | None = ..., + prog: str = ..., + action: type[Action] = ..., + option_string: str = ..., + dest: str | None = ..., + required: bool = ..., + help: str | None = ..., + metavar: str | None = ..., + ) -> _SubParsersAction[_ArgumentParserT]: ... + @overload + def add_subparsers( + self, + *, + title: str = ..., + description: str | None = ..., + prog: str = ..., + parser_class: type[_ArgumentParserT], + action: type[Action] = ..., + option_string: str = ..., + dest: str | None = ..., + required: bool = ..., + help: str | None = ..., + metavar: str | None = ..., + ) -> _SubParsersAction[_ArgumentParserT]: ... + def print_usage(self, file: IO[str] | None = None) -> None: ... + def print_help(self, file: IO[str] | None = None) -> None: ... + def format_usage(self) -> str: ... + def format_help(self) -> str: ... + @overload + def parse_known_args(self, args: Sequence[str] | None = None, namespace: None = None) -> tuple[Namespace, list[str]]: ... + @overload + def parse_known_args(self, args: Sequence[str] | None, namespace: _N) -> tuple[_N, list[str]]: ... + @overload + def parse_known_args(self, *, namespace: _N) -> tuple[_N, list[str]]: ... + def convert_arg_line_to_args(self, arg_line: str) -> list[str]: ... + def exit(self, status: int = 0, message: str | None = None) -> NoReturn: ... + def error(self, message: str) -> NoReturn: ... + @overload + def parse_intermixed_args(self, args: Sequence[str] | None = None, namespace: None = None) -> Namespace: ... + @overload + def parse_intermixed_args(self, args: Sequence[str] | None, namespace: _N) -> _N: ... + @overload + def parse_intermixed_args(self, *, namespace: _N) -> _N: ... + @overload + def parse_known_intermixed_args( + self, args: Sequence[str] | None = None, namespace: None = None + ) -> tuple[Namespace, list[str]]: ... + @overload + def parse_known_intermixed_args(self, args: Sequence[str] | None, namespace: _N) -> tuple[_N, list[str]]: ... + @overload + def parse_known_intermixed_args(self, *, namespace: _N) -> tuple[_N, list[str]]: ... + # undocumented + def _get_optional_actions(self) -> list[Action]: ... + def _get_positional_actions(self) -> list[Action]: ... + def _parse_known_args(self, arg_strings: list[str], namespace: Namespace) -> tuple[Namespace, list[str]]: ... + def _read_args_from_files(self, arg_strings: list[str]) -> list[str]: ... + def _match_argument(self, action: Action, arg_strings_pattern: str) -> int: ... + def _match_arguments_partial(self, actions: Sequence[Action], arg_strings_pattern: str) -> list[int]: ... + def _parse_optional(self, arg_string: str) -> tuple[Action | None, str, str | None] | None: ... + def _get_option_tuples(self, option_string: str) -> list[tuple[Action, str, str | None]]: ... + def _get_nargs_pattern(self, action: Action) -> str: ... + def _get_values(self, action: Action, arg_strings: list[str]) -> Any: ... + def _get_value(self, action: Action, arg_string: str) -> Any: ... + def _check_value(self, action: Action, value: Any) -> None: ... + def _get_formatter(self) -> HelpFormatter: ... + def _print_message(self, message: str, file: IO[str] | None = None) -> None: ... + +class HelpFormatter: + # undocumented + _prog: str + _indent_increment: int + _max_help_position: int + _width: int + _current_indent: int + _level: int + _action_max_length: int + _root_section: _Section + _current_section: _Section + _whitespace_matcher: Pattern[str] + _long_break_matcher: Pattern[str] + + class _Section: + formatter: HelpFormatter + heading: str | None + parent: Self | None + items: list[tuple[Callable[..., str], Iterable[Any]]] + def __init__(self, formatter: HelpFormatter, parent: Self | None, heading: str | None = None) -> None: ... + def format_help(self) -> str: ... + + def __init__(self, prog: str, indent_increment: int = 2, max_help_position: int = 24, width: int | None = None) -> None: ... + def _indent(self) -> None: ... + def _dedent(self) -> None: ... + def _add_item(self, func: Callable[..., str], args: Iterable[Any]) -> None: ... + def start_section(self, heading: str | None) -> None: ... + def end_section(self) -> None: ... + def add_text(self, text: str | None) -> None: ... + def add_usage( + self, usage: str | None, actions: Iterable[Action], groups: Iterable[_MutuallyExclusiveGroup], prefix: str | None = None + ) -> None: ... + def add_argument(self, action: Action) -> None: ... + def add_arguments(self, actions: Iterable[Action]) -> None: ... + def format_help(self) -> str: ... + def _join_parts(self, part_strings: Iterable[str]) -> str: ... + def _format_usage( + self, usage: str | None, actions: Iterable[Action], groups: Iterable[_MutuallyExclusiveGroup], prefix: str | None + ) -> str: ... + def _format_actions_usage(self, actions: Iterable[Action], groups: Iterable[_MutuallyExclusiveGroup]) -> str: ... + def _format_text(self, text: str) -> str: ... + def _format_action(self, action: Action) -> str: ... + def _format_action_invocation(self, action: Action) -> str: ... + def _metavar_formatter(self, action: Action, default_metavar: str) -> Callable[[int], tuple[str, ...]]: ... + def _format_args(self, action: Action, default_metavar: str) -> str: ... + def _expand_help(self, action: Action) -> str: ... + def _iter_indented_subactions(self, action: Action) -> Generator[Action, None, None]: ... + def _split_lines(self, text: str, width: int) -> list[str]: ... + def _fill_text(self, text: str, width: int, indent: str) -> str: ... + def _get_help_string(self, action: Action) -> str | None: ... + def _get_default_metavar_for_optional(self, action: Action) -> str: ... + def _get_default_metavar_for_positional(self, action: Action) -> str: ... + +class RawDescriptionHelpFormatter(HelpFormatter): ... +class RawTextHelpFormatter(RawDescriptionHelpFormatter): ... +class ArgumentDefaultsHelpFormatter(HelpFormatter): ... +class MetavarTypeHelpFormatter(HelpFormatter): ... + +class Action(_AttributeHolder): + option_strings: Sequence[str] + dest: str + nargs: int | str | None + const: Any + default: Any + type: _ActionType | None + choices: Iterable[Any] | None + required: bool + help: str | None + metavar: str | tuple[str, ...] | None + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + nargs: int | str | None = None, + const: _T | None = None, + default: _T | str | None = None, + type: Callable[[str], _T] | FileType | None = None, + choices: Iterable[_T] | None = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + deprecated: bool = False, + ) -> None: ... + else: + def __init__( + self, + option_strings: Sequence[str], + dest: str, + nargs: int | str | None = None, + const: _T | None = None, + default: _T | str | None = None, + type: Callable[[str], _T] | FileType | None = None, + choices: Iterable[_T] | None = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: ... + + def __call__( + self, parser: ArgumentParser, namespace: Namespace, values: str | Sequence[Any] | None, option_string: str | None = None + ) -> None: ... + if sys.version_info >= (3, 9): + def format_usage(self) -> str: ... + +if sys.version_info >= (3, 12): + class BooleanOptionalAction(Action): + if sys.version_info >= (3, 14): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: bool | None = None, + required: bool = False, + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 13): + @overload + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: bool | None = None, + *, + required: bool = False, + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + @overload + @deprecated("The `type`, `choices`, and `metavar` parameters are ignored and will be removed in Python 3.14.") + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: _T | bool | None = None, + type: Callable[[str], _T] | FileType | None = sentinel, + choices: Iterable[_T] | None = sentinel, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = sentinel, + deprecated: bool = False, + ) -> None: ... + else: + @overload + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: bool | None = None, + *, + required: bool = False, + help: str | None = None, + ) -> None: ... + @overload + @deprecated("The `type`, `choices`, and `metavar` parameters are ignored and will be removed in Python 3.14.") + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: _T | bool | None = None, + type: Callable[[str], _T] | FileType | None = sentinel, + choices: Iterable[_T] | None = sentinel, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = sentinel, + ) -> None: ... + +elif sys.version_info >= (3, 9): + class BooleanOptionalAction(Action): + @overload + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: bool | None = None, + *, + required: bool = False, + help: str | None = None, + ) -> None: ... + @overload + @deprecated("The `type`, `choices`, and `metavar` parameters are ignored and will be removed in Python 3.14.") + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: _T | bool | None = None, + type: Callable[[str], _T] | FileType | None = None, + choices: Iterable[_T] | None = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: ... + +class Namespace(_AttributeHolder): + def __init__(self, **kwargs: Any) -> None: ... + def __getattr__(self, name: str) -> Any: ... + def __setattr__(self, name: str, value: Any, /) -> None: ... + def __contains__(self, key: str) -> bool: ... + def __eq__(self, other: object) -> bool: ... + +class FileType: + # undocumented + _mode: str + _bufsize: int + _encoding: str | None + _errors: str | None + def __init__(self, mode: str = "r", bufsize: int = -1, encoding: str | None = None, errors: str | None = None) -> None: ... + def __call__(self, string: str) -> IO[Any]: ... + +# undocumented +class _ArgumentGroup(_ActionsContainer): + title: str | None + _group_actions: list[Action] + def __init__( + self, + container: _ActionsContainer, + title: str | None = None, + description: str | None = None, + *, + prefix_chars: str = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + ) -> None: ... + +# undocumented +class _MutuallyExclusiveGroup(_ArgumentGroup): + required: bool + _container: _ActionsContainer + def __init__(self, container: _ActionsContainer, required: bool = False) -> None: ... + +# undocumented +class _StoreAction(Action): ... + +# undocumented +class _StoreConstAction(Action): + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + const: Any | None = None, + default: Any = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + deprecated: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + const: Any | None = None, + default: Any = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: ... + else: + def __init__( + self, + option_strings: Sequence[str], + dest: str, + const: Any, + default: Any = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: ... + +# undocumented +class _StoreTrueAction(_StoreConstAction): + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: bool = False, + required: bool = False, + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + else: + def __init__( + self, option_strings: Sequence[str], dest: str, default: bool = False, required: bool = False, help: str | None = None + ) -> None: ... + +# undocumented +class _StoreFalseAction(_StoreConstAction): + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: bool = True, + required: bool = False, + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + else: + def __init__( + self, option_strings: Sequence[str], dest: str, default: bool = True, required: bool = False, help: str | None = None + ) -> None: ... + +# undocumented +class _AppendAction(Action): ... + +# undocumented +class _ExtendAction(_AppendAction): ... + +# undocumented +class _AppendConstAction(Action): + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + const: Any | None = None, + default: Any = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + deprecated: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + const: Any | None = None, + default: Any = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: ... + else: + def __init__( + self, + option_strings: Sequence[str], + dest: str, + const: Any, + default: Any = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: ... + +# undocumented +class _CountAction(Action): + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: Any = None, + required: bool = False, + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + else: + def __init__( + self, option_strings: Sequence[str], dest: str, default: Any = None, required: bool = False, help: str | None = None + ) -> None: ... + +# undocumented +class _HelpAction(Action): + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + dest: str = "==SUPPRESS==", + default: str = "==SUPPRESS==", + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + else: + def __init__( + self, + option_strings: Sequence[str], + dest: str = "==SUPPRESS==", + default: str = "==SUPPRESS==", + help: str | None = None, + ) -> None: ... + +# undocumented +class _VersionAction(Action): + version: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + option_strings: Sequence[str], + version: str | None = None, + dest: str = "==SUPPRESS==", + default: str = "==SUPPRESS==", + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __init__( + self, + option_strings: Sequence[str], + version: str | None = None, + dest: str = "==SUPPRESS==", + default: str = "==SUPPRESS==", + help: str | None = None, + ) -> None: ... + else: + def __init__( + self, + option_strings: Sequence[str], + version: str | None = None, + dest: str = "==SUPPRESS==", + default: str = "==SUPPRESS==", + help: str = "show program's version number and exit", + ) -> None: ... + +# undocumented +class _SubParsersAction(Action, Generic[_ArgumentParserT]): + _ChoicesPseudoAction: type[Any] # nested class + _prog_prefix: str + _parser_class: type[_ArgumentParserT] + _name_parser_map: dict[str, _ArgumentParserT] + choices: dict[str, _ArgumentParserT] + _choices_actions: list[Action] + def __init__( + self, + option_strings: Sequence[str], + prog: str, + parser_class: type[_ArgumentParserT], + dest: str = "==SUPPRESS==", + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: ... + + # Note: `add_parser` accepts all kwargs of `ArgumentParser.__init__`. It also + # accepts its own `help` and `aliases` kwargs. + if sys.version_info >= (3, 13): + def add_parser( + self, + name: str, + *, + deprecated: bool = False, + help: str | None = ..., + aliases: Sequence[str] = ..., + # Kwargs from ArgumentParser constructor + prog: str | None = ..., + usage: str | None = ..., + description: str | None = ..., + epilog: str | None = ..., + parents: Sequence[_ArgumentParserT] = ..., + formatter_class: _FormatterClass = ..., + prefix_chars: str = ..., + fromfile_prefix_chars: str | None = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + add_help: bool = ..., + allow_abbrev: bool = ..., + exit_on_error: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes + ) -> _ArgumentParserT: ... + elif sys.version_info >= (3, 9): + def add_parser( + self, + name: str, + *, + help: str | None = ..., + aliases: Sequence[str] = ..., + # Kwargs from ArgumentParser constructor + prog: str | None = ..., + usage: str | None = ..., + description: str | None = ..., + epilog: str | None = ..., + parents: Sequence[_ArgumentParserT] = ..., + formatter_class: _FormatterClass = ..., + prefix_chars: str = ..., + fromfile_prefix_chars: str | None = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + add_help: bool = ..., + allow_abbrev: bool = ..., + exit_on_error: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes + ) -> _ArgumentParserT: ... + else: + def add_parser( + self, + name: str, + *, + help: str | None = ..., + aliases: Sequence[str] = ..., + # Kwargs from ArgumentParser constructor + prog: str | None = ..., + usage: str | None = ..., + description: str | None = ..., + epilog: str | None = ..., + parents: Sequence[_ArgumentParserT] = ..., + formatter_class: _FormatterClass = ..., + prefix_chars: str = ..., + fromfile_prefix_chars: str | None = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + add_help: bool = ..., + allow_abbrev: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes + ) -> _ArgumentParserT: ... + + def _get_subactions(self) -> list[Action]: ... + +# undocumented +class ArgumentTypeError(Exception): ... + +# undocumented +def _get_action_name(argument: Action | None) -> str | None: ... diff --git a/mypy/typeshed/stdlib/array.pyi b/mypy/typeshed/stdlib/array.pyi new file mode 100644 index 000000000000..878d8d8cb808 --- /dev/null +++ b/mypy/typeshed/stdlib/array.pyi @@ -0,0 +1,92 @@ +import sys +from _typeshed import ReadableBuffer, SupportsRead, SupportsWrite +from collections.abc import Iterable + +# pytype crashes if array inherits from collections.abc.MutableSequence instead of typing.MutableSequence +from typing import Any, Literal, MutableSequence, SupportsIndex, TypeVar, overload # noqa: Y022 +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 12): + from types import GenericAlias + +_IntTypeCode: TypeAlias = Literal["b", "B", "h", "H", "i", "I", "l", "L", "q", "Q"] +_FloatTypeCode: TypeAlias = Literal["f", "d"] +_UnicodeTypeCode: TypeAlias = Literal["u"] +_TypeCode: TypeAlias = _IntTypeCode | _FloatTypeCode | _UnicodeTypeCode + +_T = TypeVar("_T", int, float, str) + +typecodes: str + +class array(MutableSequence[_T]): + @property + def typecode(self) -> _TypeCode: ... + @property + def itemsize(self) -> int: ... + @overload + def __init__(self: array[int], typecode: _IntTypeCode, initializer: bytes | bytearray | Iterable[int] = ..., /) -> None: ... + @overload + def __init__( + self: array[float], typecode: _FloatTypeCode, initializer: bytes | bytearray | Iterable[float] = ..., / + ) -> None: ... + @overload + def __init__( + self: array[str], typecode: _UnicodeTypeCode, initializer: bytes | bytearray | Iterable[str] = ..., / + ) -> None: ... + @overload + def __init__(self, typecode: str, initializer: Iterable[_T], /) -> None: ... + @overload + def __init__(self, typecode: str, initializer: bytes | bytearray = ..., /) -> None: ... + def append(self, v: _T, /) -> None: ... + def buffer_info(self) -> tuple[int, int]: ... + def byteswap(self) -> None: ... + def count(self, v: _T, /) -> int: ... + def extend(self, bb: Iterable[_T], /) -> None: ... + def frombytes(self, buffer: ReadableBuffer, /) -> None: ... + def fromfile(self, f: SupportsRead[bytes], n: int, /) -> None: ... + def fromlist(self, list: list[_T], /) -> None: ... + def fromunicode(self, ustr: str, /) -> None: ... + if sys.version_info >= (3, 10): + def index(self, v: _T, start: int = 0, stop: int = sys.maxsize, /) -> int: ... + else: + def index(self, v: _T, /) -> int: ... # type: ignore[override] + + def insert(self, i: int, v: _T, /) -> None: ... + def pop(self, i: int = -1, /) -> _T: ... + def remove(self, v: _T, /) -> None: ... + def tobytes(self) -> bytes: ... + def tofile(self, f: SupportsWrite[bytes], /) -> None: ... + def tolist(self) -> list[_T]: ... + def tounicode(self) -> str: ... + if sys.version_info < (3, 9): + def fromstring(self, buffer: str | ReadableBuffer, /) -> None: ... + def tostring(self) -> bytes: ... + + def __len__(self) -> int: ... + @overload + def __getitem__(self, key: SupportsIndex, /) -> _T: ... + @overload + def __getitem__(self, key: slice, /) -> array[_T]: ... + @overload # type: ignore[override] + def __setitem__(self, key: SupportsIndex, value: _T, /) -> None: ... + @overload + def __setitem__(self, key: slice, value: array[_T], /) -> None: ... + def __delitem__(self, key: SupportsIndex | slice, /) -> None: ... + def __add__(self, value: array[_T], /) -> array[_T]: ... + def __eq__(self, value: object, /) -> bool: ... + def __ge__(self, value: array[_T], /) -> bool: ... + def __gt__(self, value: array[_T], /) -> bool: ... + def __iadd__(self, value: array[_T], /) -> Self: ... # type: ignore[override] + def __imul__(self, value: int, /) -> Self: ... + def __le__(self, value: array[_T], /) -> bool: ... + def __lt__(self, value: array[_T], /) -> bool: ... + def __mul__(self, value: int, /) -> array[_T]: ... + def __rmul__(self, value: int, /) -> array[_T]: ... + def __copy__(self) -> array[_T]: ... + def __deepcopy__(self, unused: Any, /) -> array[_T]: ... + def __buffer__(self, flags: int, /) -> memoryview: ... + def __release_buffer__(self, buffer: memoryview, /) -> None: ... + if sys.version_info >= (3, 12): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +ArrayType = array diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi new file mode 100644 index 000000000000..351a4af2fb75 --- /dev/null +++ b/mypy/typeshed/stdlib/ast.pyi @@ -0,0 +1,2049 @@ +import os +import sys +import typing_extensions +from _ast import ( + PyCF_ALLOW_TOP_LEVEL_AWAIT as PyCF_ALLOW_TOP_LEVEL_AWAIT, + PyCF_ONLY_AST as PyCF_ONLY_AST, + PyCF_TYPE_COMMENTS as PyCF_TYPE_COMMENTS, +) +from _typeshed import ReadableBuffer, Unused +from collections.abc import Iterator +from typing import Any, ClassVar, Generic, Literal, TypedDict, TypeVar as _TypeVar, overload +from typing_extensions import Self, Unpack, deprecated + +if sys.version_info >= (3, 13): + from _ast import PyCF_OPTIMIZED_AST as PyCF_OPTIMIZED_AST + +# Alias used for fields that must always be valid identifiers +# A string `x` counts as a valid identifier if both the following are True +# (1) `x.isidentifier()` evaluates to `True` +# (2) `keyword.iskeyword(x)` evaluates to `False` +_Identifier: typing_extensions.TypeAlias = str + +# Used for node end positions in constructor keyword arguments +_EndPositionT = typing_extensions.TypeVar("_EndPositionT", int, int | None, default=int | None) + +# Corresponds to the names in the `_attributes` class variable which is non-empty in certain AST nodes +class _Attributes(TypedDict, Generic[_EndPositionT], total=False): + lineno: int + col_offset: int + end_lineno: _EndPositionT + end_col_offset: _EndPositionT + +# The various AST classes are implemented in C, and imported from _ast at runtime, +# but they consider themselves to live in the ast module, +# so we'll define the stubs in this file. +class AST: + if sys.version_info >= (3, 10): + __match_args__ = () + _attributes: ClassVar[tuple[str, ...]] + _fields: ClassVar[tuple[str, ...]] + if sys.version_info >= (3, 13): + _field_types: ClassVar[dict[str, Any]] + + if sys.version_info >= (3, 14): + def __replace__(self) -> Self: ... + +class mod(AST): ... + +class Module(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("body", "type_ignores") + body: list[stmt] + type_ignores: list[TypeIgnore] + if sys.version_info >= (3, 13): + def __init__(self, body: list[stmt] = ..., type_ignores: list[TypeIgnore] = ...) -> None: ... + else: + def __init__(self, body: list[stmt], type_ignores: list[TypeIgnore]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, body: list[stmt] = ..., type_ignores: list[TypeIgnore] = ...) -> Self: ... + +class Interactive(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("body",) + body: list[stmt] + if sys.version_info >= (3, 13): + def __init__(self, body: list[stmt] = ...) -> None: ... + else: + def __init__(self, body: list[stmt]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, body: list[stmt] = ...) -> Self: ... + +class Expression(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("body",) + body: expr + def __init__(self, body: expr) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, body: expr = ...) -> Self: ... + +class FunctionType(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("argtypes", "returns") + argtypes: list[expr] + returns: expr + if sys.version_info >= (3, 13): + @overload + def __init__(self, argtypes: list[expr], returns: expr) -> None: ... + @overload + def __init__(self, argtypes: list[expr] = ..., *, returns: expr) -> None: ... + else: + def __init__(self, argtypes: list[expr], returns: expr) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, argtypes: list[expr] = ..., returns: expr = ...) -> Self: ... + +class stmt(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... + +class FunctionDef(stmt): + if sys.version_info >= (3, 12): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") + elif sys.version_info >= (3, 10): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment") + name: _Identifier + args: arguments + body: list[stmt] + decorator_list: list[expr] + returns: expr | None + type_comment: str | None + if sys.version_info >= (3, 12): + type_params: list[type_param] + if sys.version_info >= (3, 13): + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + returns: expr | None = None, + type_comment: str | None = None, + type_params: list[type_param] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + elif sys.version_info >= (3, 12): + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None, + type_comment: str | None, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + *, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + args: arguments = ..., + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + returns: expr | None = ..., + type_comment: str | None = ..., + type_params: list[type_param] = ..., + ) -> Self: ... + +class AsyncFunctionDef(stmt): + if sys.version_info >= (3, 12): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") + elif sys.version_info >= (3, 10): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment") + name: _Identifier + args: arguments + body: list[stmt] + decorator_list: list[expr] + returns: expr | None + type_comment: str | None + if sys.version_info >= (3, 12): + type_params: list[type_param] + if sys.version_info >= (3, 13): + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + returns: expr | None = None, + type_comment: str | None = None, + type_params: list[type_param] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + elif sys.version_info >= (3, 12): + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None, + type_comment: str | None, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + *, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + args: arguments = ..., + body: list[stmt], + decorator_list: list[expr], + returns: expr | None, + type_comment: str | None, + type_params: list[type_param], + ) -> Self: ... + +class ClassDef(stmt): + if sys.version_info >= (3, 12): + __match_args__ = ("name", "bases", "keywords", "body", "decorator_list", "type_params") + elif sys.version_info >= (3, 10): + __match_args__ = ("name", "bases", "keywords", "body", "decorator_list") + name: _Identifier + bases: list[expr] + keywords: list[keyword] + body: list[stmt] + decorator_list: list[expr] + if sys.version_info >= (3, 12): + type_params: list[type_param] + if sys.version_info >= (3, 13): + def __init__( + self, + name: _Identifier, + bases: list[expr] = ..., + keywords: list[keyword] = ..., + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + type_params: list[type_param] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + elif sys.version_info >= (3, 12): + def __init__( + self, + name: _Identifier, + bases: list[expr], + keywords: list[keyword], + body: list[stmt], + decorator_list: list[expr], + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + name: _Identifier, + bases: list[expr], + keywords: list[keyword], + body: list[stmt], + decorator_list: list[expr], + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier, + bases: list[expr], + keywords: list[keyword], + body: list[stmt], + decorator_list: list[expr], + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class Return(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr | None + def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Delete(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("targets",) + targets: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, targets: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, targets: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, targets: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Assign(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("targets", "value", "type_comment") + targets: list[expr] + value: expr + type_comment: str | None + if sys.version_info >= (3, 13): + @overload + def __init__( + self, targets: list[expr], value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + @overload + def __init__( + self, targets: list[expr] = ..., *, value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__( + self, targets: list[expr], value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, targets: list[expr] = ..., value: expr = ..., type_comment: str | None = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +if sys.version_info >= (3, 12): + class TypeAlias(stmt): + __match_args__ = ("name", "type_params", "value") + name: Name + type_params: list[type_param] + value: expr + if sys.version_info >= (3, 13): + @overload + def __init__( + self, name: Name, type_params: list[type_param], value: expr, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + @overload + def __init__( + self, name: Name, type_params: list[type_param] = ..., *, value: expr, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + else: + def __init__( + self, name: Name, type_params: list[type_param], value: expr, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: Name = ..., + type_params: list[type_param] = ..., + value: expr = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + +class AugAssign(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "op", "value") + target: Name | Attribute | Subscript + op: operator + value: expr + def __init__( + self, target: Name | Attribute | Subscript, op: operator, value: expr, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: Name | Attribute | Subscript = ..., + op: operator = ..., + value: expr = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class AnnAssign(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "annotation", "value", "simple") + target: Name | Attribute | Subscript + annotation: expr + value: expr | None + simple: int + @overload + def __init__( + self, + target: Name | Attribute | Subscript, + annotation: expr, + value: expr | None, + simple: int, + **kwargs: Unpack[_Attributes], + ) -> None: ... + @overload + def __init__( + self, + target: Name | Attribute | Subscript, + annotation: expr, + value: expr | None = None, + *, + simple: int, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: Name | Attribute | Subscript = ..., + annotation: expr = ..., + value: expr | None = ..., + simple: int = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class For(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "iter", "body", "orelse", "type_comment") + target: expr + iter: expr + body: list[stmt] + orelse: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt], + orelse: list[stmt], + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: expr = ..., + iter: expr = ..., + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class AsyncFor(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "iter", "body", "orelse", "type_comment") + target: expr + iter: expr + body: list[stmt] + orelse: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt], + orelse: list[stmt], + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: expr = ..., + iter: expr = ..., + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class While(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "body", "orelse") + test: expr + body: list[stmt] + orelse: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, test: expr, body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> Self: ... + +class If(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "body", "orelse") + test: expr + body: list[stmt] + orelse: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, test: expr, body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, test: expr = ..., body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class With(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("items", "body", "type_comment") + items: list[withitem] + body: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class AsyncWith(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("items", "body", "type_comment") + items: list[withitem] + body: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +if sys.version_info >= (3, 10): + class Match(stmt): + __match_args__ = ("subject", "cases") + subject: expr + cases: list[match_case] + if sys.version_info >= (3, 13): + def __init__(self, subject: expr, cases: list[match_case] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, subject: expr, cases: list[match_case], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, subject: expr = ..., cases: list[match_case] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Raise(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("exc", "cause") + exc: expr | None + cause: expr | None + def __init__(self, exc: expr | None = None, cause: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, exc: expr | None = ..., cause: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Try(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("body", "handlers", "orelse", "finalbody") + body: list[stmt] + handlers: list[ExceptHandler] + orelse: list[stmt] + finalbody: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + body: list[stmt], + handlers: list[ExceptHandler], + orelse: list[stmt], + finalbody: list[stmt], + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +if sys.version_info >= (3, 11): + class TryStar(stmt): + __match_args__ = ("body", "handlers", "orelse", "finalbody") + body: list[stmt] + handlers: list[ExceptHandler] + orelse: list[stmt] + finalbody: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + body: list[stmt], + handlers: list[ExceptHandler], + orelse: list[stmt], + finalbody: list[stmt], + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class Assert(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "msg") + test: expr + msg: expr | None + def __init__(self, test: expr, msg: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, test: expr, msg: expr | None, **kwargs: Unpack[_Attributes]) -> Self: ... + +class Import(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("names",) + names: list[alias] + if sys.version_info >= (3, 13): + def __init__(self, names: list[alias] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, names: list[alias], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[alias] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class ImportFrom(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("module", "names", "level") + module: str | None + names: list[alias] + level: int + if sys.version_info >= (3, 13): + @overload + def __init__(self, module: str | None, names: list[alias], level: int, **kwargs: Unpack[_Attributes]) -> None: ... + @overload + def __init__( + self, module: str | None = None, names: list[alias] = ..., *, level: int, **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + @overload + def __init__(self, module: str | None, names: list[alias], level: int, **kwargs: Unpack[_Attributes]) -> None: ... + @overload + def __init__( + self, module: str | None = None, *, names: list[alias], level: int, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, module: str | None = ..., names: list[alias] = ..., level: int = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Global(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("names",) + names: list[_Identifier] + if sys.version_info >= (3, 13): + def __init__(self, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> Self: ... + +class Nonlocal(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("names",) + names: list[_Identifier] + if sys.version_info >= (3, 13): + def __init__(self, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Expr(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Pass(stmt): ... +class Break(stmt): ... +class Continue(stmt): ... + +class expr(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... + +class BoolOp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("op", "values") + op: boolop + values: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, op: boolop, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, op: boolop, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, op: boolop = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class NamedExpr(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "value") + target: Name + value: expr + def __init__(self, target: Name, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, target: Name = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class BinOp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("left", "op", "right") + left: expr + op: operator + right: expr + def __init__(self, left: expr, op: operator, right: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, left: expr = ..., op: operator = ..., right: expr = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class UnaryOp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("op", "operand") + op: unaryop + operand: expr + def __init__(self, op: unaryop, operand: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, op: unaryop = ..., operand: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Lambda(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("args", "body") + args: arguments + body: expr + def __init__(self, args: arguments, body: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, args: arguments = ..., body: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class IfExp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "body", "orelse") + test: expr + body: expr + orelse: expr + def __init__(self, test: expr, body: expr, orelse: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, test: expr = ..., body: expr = ..., orelse: expr = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Dict(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("keys", "values") + keys: list[expr | None] + values: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, keys: list[expr | None] = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, keys: list[expr | None], values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, keys: list[expr | None] = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Set(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elts",) + elts: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, elts: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elts: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class ListComp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elt", "generators") + elt: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class SetComp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elt", "generators") + elt: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class DictComp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("key", "value", "generators") + key: expr + value: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__( + self, key: expr, value: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, key: expr, value: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, key: expr = ..., value: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class GeneratorExp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elt", "generators") + elt: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Await(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Yield(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr | None + def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class YieldFrom(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Compare(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("left", "ops", "comparators") + left: expr + ops: list[cmpop] + comparators: list[expr] + if sys.version_info >= (3, 13): + def __init__( + self, left: expr, ops: list[cmpop] = ..., comparators: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, left: expr, ops: list[cmpop], comparators: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, left: expr = ..., ops: list[cmpop] = ..., comparators: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Call(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("func", "args", "keywords") + func: expr + args: list[expr] + keywords: list[keyword] + if sys.version_info >= (3, 13): + def __init__( + self, func: expr, args: list[expr] = ..., keywords: list[keyword] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, func: expr, args: list[expr], keywords: list[keyword], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, func: expr = ..., args: list[expr] = ..., keywords: list[keyword] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class FormattedValue(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "conversion", "format_spec") + value: expr + conversion: int + format_spec: expr | None + def __init__(self, value: expr, conversion: int, format_spec: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., conversion: int = ..., format_spec: expr | None = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class JoinedStr(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("values",) + values: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Constant(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "kind") + value: Any # None, str, bytes, bool, int, float, complex, Ellipsis + kind: str | None + if sys.version_info < (3, 14): + # Aliases for value, for backwards compatibility + s: Any + n: int | float | complex + + def __init__(self, value: Any, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: Any = ..., kind: str | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Attribute(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "attr", "ctx") + value: expr + attr: _Identifier + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, value: expr, attr: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., attr: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Subscript(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "slice", "ctx") + value: expr + slice: _Slice + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, value: expr, slice: _Slice, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., slice: _Slice = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Starred(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "ctx") + value: expr + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, value: expr, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Name(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("id", "ctx") + id: _Identifier + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, id: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, id: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class List(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elts", "ctx") + elts: list[expr] + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + if sys.version_info >= (3, 13): + def __init__(self, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Tuple(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elts", "ctx") + elts: list[expr] + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + if sys.version_info >= (3, 9): + dims: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class slice(AST): ... # deprecated and moved to ast.py for >= (3, 9) + +if sys.version_info >= (3, 9): + _Slice: typing_extensions.TypeAlias = expr + _SliceAttributes: typing_extensions.TypeAlias = _Attributes +else: + # alias for use with variables named slice + _Slice: typing_extensions.TypeAlias = slice + + class _SliceAttributes(TypedDict): ... + +class Slice(_Slice): + if sys.version_info >= (3, 10): + __match_args__ = ("lower", "upper", "step") + lower: expr | None + upper: expr | None + step: expr | None + def __init__( + self, lower: expr | None = None, upper: expr | None = None, step: expr | None = None, **kwargs: Unpack[_SliceAttributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + lower: expr | None = ..., + upper: expr | None = ..., + step: expr | None = ..., + **kwargs: Unpack[_SliceAttributes], + ) -> Self: ... + +class ExtSlice(slice): # deprecated and moved to ast.py if sys.version_info >= (3, 9) + dims: list[slice] + def __init__(self, dims: list[slice], **kwargs: Unpack[_SliceAttributes]) -> None: ... + +class Index(slice): # deprecated and moved to ast.py if sys.version_info >= (3, 9) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_SliceAttributes]) -> None: ... + +class expr_context(AST): ... +class AugLoad(expr_context): ... # deprecated and moved to ast.py if sys.version_info >= (3, 9) +class AugStore(expr_context): ... # deprecated and moved to ast.py if sys.version_info >= (3, 9) +class Param(expr_context): ... # deprecated and moved to ast.py if sys.version_info >= (3, 9) + +class Suite(mod): # deprecated and moved to ast.py if sys.version_info >= (3, 9) + body: list[stmt] + def __init__(self, body: list[stmt]) -> None: ... + +class Load(expr_context): ... +class Store(expr_context): ... +class Del(expr_context): ... +class boolop(AST): ... +class And(boolop): ... +class Or(boolop): ... +class operator(AST): ... +class Add(operator): ... +class Sub(operator): ... +class Mult(operator): ... +class MatMult(operator): ... +class Div(operator): ... +class Mod(operator): ... +class Pow(operator): ... +class LShift(operator): ... +class RShift(operator): ... +class BitOr(operator): ... +class BitXor(operator): ... +class BitAnd(operator): ... +class FloorDiv(operator): ... +class unaryop(AST): ... +class Invert(unaryop): ... +class Not(unaryop): ... +class UAdd(unaryop): ... +class USub(unaryop): ... +class cmpop(AST): ... +class Eq(cmpop): ... +class NotEq(cmpop): ... +class Lt(cmpop): ... +class LtE(cmpop): ... +class Gt(cmpop): ... +class GtE(cmpop): ... +class Is(cmpop): ... +class IsNot(cmpop): ... +class In(cmpop): ... +class NotIn(cmpop): ... + +class comprehension(AST): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "iter", "ifs", "is_async") + target: expr + iter: expr + ifs: list[expr] + is_async: int + if sys.version_info >= (3, 13): + @overload + def __init__(self, target: expr, iter: expr, ifs: list[expr], is_async: int) -> None: ... + @overload + def __init__(self, target: expr, iter: expr, ifs: list[expr] = ..., *, is_async: int) -> None: ... + else: + def __init__(self, target: expr, iter: expr, ifs: list[expr], is_async: int) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, target: expr = ..., iter: expr = ..., ifs: list[expr] = ..., is_async: int = ...) -> Self: ... + +class excepthandler(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int | None = ..., end_col_offset: int | None = ... + ) -> Self: ... + +class ExceptHandler(excepthandler): + if sys.version_info >= (3, 10): + __match_args__ = ("type", "name", "body") + type: expr | None + name: _Identifier | None + body: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, type: expr | None = None, name: _Identifier | None = None, body: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + @overload + def __init__( + self, type: expr | None, name: _Identifier | None, body: list[stmt], **kwargs: Unpack[_Attributes] + ) -> None: ... + @overload + def __init__( + self, type: expr | None = None, name: _Identifier | None = None, *, body: list[stmt], **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + type: expr | None = ..., + name: _Identifier | None = ..., + body: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class arguments(AST): + if sys.version_info >= (3, 10): + __match_args__ = ("posonlyargs", "args", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults") + posonlyargs: list[arg] + args: list[arg] + vararg: arg | None + kwonlyargs: list[arg] + kw_defaults: list[expr | None] + kwarg: arg | None + defaults: list[expr] + if sys.version_info >= (3, 13): + def __init__( + self, + posonlyargs: list[arg] = ..., + args: list[arg] = ..., + vararg: arg | None = None, + kwonlyargs: list[arg] = ..., + kw_defaults: list[expr | None] = ..., + kwarg: arg | None = None, + defaults: list[expr] = ..., + ) -> None: ... + else: + @overload + def __init__( + self, + posonlyargs: list[arg], + args: list[arg], + vararg: arg | None, + kwonlyargs: list[arg], + kw_defaults: list[expr | None], + kwarg: arg | None, + defaults: list[expr], + ) -> None: ... + @overload + def __init__( + self, + posonlyargs: list[arg], + args: list[arg], + vararg: arg | None, + kwonlyargs: list[arg], + kw_defaults: list[expr | None], + kwarg: arg | None = None, + *, + defaults: list[expr], + ) -> None: ... + @overload + def __init__( + self, + posonlyargs: list[arg], + args: list[arg], + vararg: arg | None = None, + *, + kwonlyargs: list[arg], + kw_defaults: list[expr | None], + kwarg: arg | None = None, + defaults: list[expr], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + posonlyargs: list[arg] = ..., + args: list[arg] = ..., + vararg: arg | None = ..., + kwonlyargs: list[arg] = ..., + kw_defaults: list[expr | None] = ..., + kwarg: arg | None = ..., + defaults: list[expr] = ..., + ) -> Self: ... + +class arg(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + if sys.version_info >= (3, 10): + __match_args__ = ("arg", "annotation", "type_comment") + arg: _Identifier + annotation: expr | None + type_comment: str | None + def __init__( + self, arg: _Identifier, annotation: expr | None = None, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + arg: _Identifier = ..., + annotation: expr | None = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class keyword(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + if sys.version_info >= (3, 10): + __match_args__ = ("arg", "value") + arg: _Identifier | None + value: expr + @overload + def __init__(self, arg: _Identifier | None, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + @overload + def __init__(self, arg: _Identifier | None = None, *, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, arg: _Identifier | None = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class alias(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + if sys.version_info >= (3, 10): + __match_args__ = ("name", "asname") + name: str + asname: _Identifier | None + def __init__(self, name: str, asname: _Identifier | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, name: str = ..., asname: _Identifier | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class withitem(AST): + if sys.version_info >= (3, 10): + __match_args__ = ("context_expr", "optional_vars") + context_expr: expr + optional_vars: expr | None + def __init__(self, context_expr: expr, optional_vars: expr | None = None) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, context_expr: expr = ..., optional_vars: expr | None = ...) -> Self: ... + +if sys.version_info >= (3, 10): + class match_case(AST): + __match_args__ = ("pattern", "guard", "body") + pattern: _Pattern + guard: expr | None + body: list[stmt] + if sys.version_info >= (3, 13): + def __init__(self, pattern: _Pattern, guard: expr | None = None, body: list[stmt] = ...) -> None: ... + else: + @overload + def __init__(self, pattern: _Pattern, guard: expr | None, body: list[stmt]) -> None: ... + @overload + def __init__(self, pattern: _Pattern, guard: expr | None = None, *, body: list[stmt]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, pattern: _Pattern = ..., guard: expr | None = ..., body: list[stmt] = ...) -> Self: ... + + class pattern(AST): + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int = ..., end_col_offset: int = ... + ) -> Self: ... + + # Without the alias, Pyright complains variables named pattern are recursively defined + _Pattern: typing_extensions.TypeAlias = pattern + + class MatchValue(pattern): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchSingleton(pattern): + __match_args__ = ("value",) + value: Literal[True, False] | None + def __init__(self, value: Literal[True, False] | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: Literal[True, False] | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchSequence(pattern): + __match_args__ = ("patterns",) + patterns: list[pattern] + if sys.version_info >= (3, 13): + def __init__(self, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> None: ... + else: + def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchMapping(pattern): + __match_args__ = ("keys", "patterns", "rest") + keys: list[expr] + patterns: list[pattern] + rest: _Identifier | None + if sys.version_info >= (3, 13): + def __init__( + self, + keys: list[expr] = ..., + patterns: list[pattern] = ..., + rest: _Identifier | None = None, + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + else: + def __init__( + self, + keys: list[expr], + patterns: list[pattern], + rest: _Identifier | None = None, + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + keys: list[expr] = ..., + patterns: list[pattern] = ..., + rest: _Identifier | None = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + + class MatchClass(pattern): + __match_args__ = ("cls", "patterns", "kwd_attrs", "kwd_patterns") + cls: expr + patterns: list[pattern] + kwd_attrs: list[_Identifier] + kwd_patterns: list[pattern] + if sys.version_info >= (3, 13): + def __init__( + self, + cls: expr, + patterns: list[pattern] = ..., + kwd_attrs: list[_Identifier] = ..., + kwd_patterns: list[pattern] = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + else: + def __init__( + self, + cls: expr, + patterns: list[pattern], + kwd_attrs: list[_Identifier], + kwd_patterns: list[pattern], + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + cls: expr = ..., + patterns: list[pattern] = ..., + kwd_attrs: list[_Identifier] = ..., + kwd_patterns: list[pattern] = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + + class MatchStar(pattern): + __match_args__ = ("name",) + name: _Identifier | None + def __init__(self, name: _Identifier | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchAs(pattern): + __match_args__ = ("pattern", "name") + pattern: _Pattern | None + name: _Identifier | None + def __init__( + self, pattern: _Pattern | None = None, name: _Identifier | None = None, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, pattern: _Pattern | None = ..., name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + + class MatchOr(pattern): + __match_args__ = ("patterns",) + patterns: list[pattern] + if sys.version_info >= (3, 13): + def __init__(self, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> None: ... + else: + def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + +class type_ignore(AST): ... + +class TypeIgnore(type_ignore): + if sys.version_info >= (3, 10): + __match_args__ = ("lineno", "tag") + lineno: int + tag: str + def __init__(self, lineno: int, tag: str) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, lineno: int = ..., tag: str = ...) -> Self: ... + +if sys.version_info >= (3, 12): + class type_param(AST): + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class TypeVar(type_param): + if sys.version_info >= (3, 13): + __match_args__ = ("name", "bound", "default_value") + else: + __match_args__ = ("name", "bound") + name: _Identifier + bound: expr | None + if sys.version_info >= (3, 13): + default_value: expr | None + def __init__( + self, + name: _Identifier, + bound: expr | None = None, + default_value: expr | None = None, + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + else: + def __init__(self, name: _Identifier, bound: expr | None = None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + bound: expr | None = ..., + default_value: expr | None = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + + class ParamSpec(type_param): + if sys.version_info >= (3, 13): + __match_args__ = ("name", "default_value") + else: + __match_args__ = ("name",) + name: _Identifier + if sys.version_info >= (3, 13): + default_value: expr | None + def __init__( + self, name: _Identifier, default_value: expr | None = None, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + else: + def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + + class TypeVarTuple(type_param): + if sys.version_info >= (3, 13): + __match_args__ = ("name", "default_value") + else: + __match_args__ = ("name",) + name: _Identifier + if sys.version_info >= (3, 13): + default_value: expr | None + def __init__( + self, name: _Identifier, default_value: expr | None = None, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + else: + def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + +class _ABC(type): + if sys.version_info >= (3, 9): + def __init__(cls, *args: Unused) -> None: ... + +if sys.version_info < (3, 14): + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Num(Constant, metaclass=_ABC): + value: int | float | complex + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Str(Constant, metaclass=_ABC): + value: str + # Aliases for value, for backwards compatibility + s: str + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Bytes(Constant, metaclass=_ABC): + value: bytes + # Aliases for value, for backwards compatibility + s: bytes + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class NameConstant(Constant, metaclass=_ABC): ... + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Ellipsis(Constant, metaclass=_ABC): ... + +# everything below here is defined in ast.py + +_T = _TypeVar("_T", bound=AST) + +if sys.version_info >= (3, 13): + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any] = "", + mode: Literal["exec"] = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> Module: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any], + mode: Literal["eval"], + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> Expression: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any], + mode: Literal["func_type"], + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> FunctionType: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any], + mode: Literal["single"], + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> Interactive: ... + @overload + def parse( + source: str | ReadableBuffer, + *, + mode: Literal["eval"], + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> Expression: ... + @overload + def parse( + source: str | ReadableBuffer, + *, + mode: Literal["func_type"], + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> FunctionType: ... + @overload + def parse( + source: str | ReadableBuffer, + *, + mode: Literal["single"], + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> Interactive: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any] = "", + mode: str = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> AST: ... + +else: + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any] = "", + mode: Literal["exec"] = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> Module: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any], + mode: Literal["eval"], + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> Expression: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any], + mode: Literal["func_type"], + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> FunctionType: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any], + mode: Literal["single"], + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> Interactive: ... + @overload + def parse( + source: str | ReadableBuffer, + *, + mode: Literal["eval"], + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> Expression: ... + @overload + def parse( + source: str | ReadableBuffer, + *, + mode: Literal["func_type"], + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> FunctionType: ... + @overload + def parse( + source: str | ReadableBuffer, + *, + mode: Literal["single"], + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> Interactive: ... + @overload + def parse( + source: str | ReadableBuffer, + filename: str | ReadableBuffer | os.PathLike[Any] = "", + mode: str = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> AST: ... + +def literal_eval(node_or_string: str | AST) -> Any: ... + +if sys.version_info >= (3, 13): + def dump( + node: AST, + annotate_fields: bool = True, + include_attributes: bool = False, + *, + indent: int | str | None = None, + show_empty: bool = False, + ) -> str: ... + +elif sys.version_info >= (3, 9): + def dump( + node: AST, annotate_fields: bool = True, include_attributes: bool = False, *, indent: int | str | None = None + ) -> str: ... + +else: + def dump(node: AST, annotate_fields: bool = True, include_attributes: bool = False) -> str: ... + +def copy_location(new_node: _T, old_node: AST) -> _T: ... +def fix_missing_locations(node: _T) -> _T: ... +def increment_lineno(node: _T, n: int = 1) -> _T: ... +def iter_fields(node: AST) -> Iterator[tuple[str, Any]]: ... +def iter_child_nodes(node: AST) -> Iterator[AST]: ... +def get_docstring(node: AsyncFunctionDef | FunctionDef | ClassDef | Module, clean: bool = True) -> str | None: ... +def get_source_segment(source: str, node: AST, *, padded: bool = False) -> str | None: ... +def walk(node: AST) -> Iterator[AST]: ... + +if sys.version_info >= (3, 14): + def compare(left: AST, right: AST, /, *, compare_attributes: bool = False) -> bool: ... + +class NodeVisitor: + def visit(self, node: AST) -> Any: ... + def generic_visit(self, node: AST) -> Any: ... + def visit_Module(self, node: Module) -> Any: ... + def visit_Interactive(self, node: Interactive) -> Any: ... + def visit_Expression(self, node: Expression) -> Any: ... + def visit_FunctionDef(self, node: FunctionDef) -> Any: ... + def visit_AsyncFunctionDef(self, node: AsyncFunctionDef) -> Any: ... + def visit_ClassDef(self, node: ClassDef) -> Any: ... + def visit_Return(self, node: Return) -> Any: ... + def visit_Delete(self, node: Delete) -> Any: ... + def visit_Assign(self, node: Assign) -> Any: ... + def visit_AugAssign(self, node: AugAssign) -> Any: ... + def visit_AnnAssign(self, node: AnnAssign) -> Any: ... + def visit_For(self, node: For) -> Any: ... + def visit_AsyncFor(self, node: AsyncFor) -> Any: ... + def visit_While(self, node: While) -> Any: ... + def visit_If(self, node: If) -> Any: ... + def visit_With(self, node: With) -> Any: ... + def visit_AsyncWith(self, node: AsyncWith) -> Any: ... + def visit_Raise(self, node: Raise) -> Any: ... + def visit_Try(self, node: Try) -> Any: ... + def visit_Assert(self, node: Assert) -> Any: ... + def visit_Import(self, node: Import) -> Any: ... + def visit_ImportFrom(self, node: ImportFrom) -> Any: ... + def visit_Global(self, node: Global) -> Any: ... + def visit_Nonlocal(self, node: Nonlocal) -> Any: ... + def visit_Expr(self, node: Expr) -> Any: ... + def visit_Pass(self, node: Pass) -> Any: ... + def visit_Break(self, node: Break) -> Any: ... + def visit_Continue(self, node: Continue) -> Any: ... + def visit_Slice(self, node: Slice) -> Any: ... + def visit_BoolOp(self, node: BoolOp) -> Any: ... + def visit_BinOp(self, node: BinOp) -> Any: ... + def visit_UnaryOp(self, node: UnaryOp) -> Any: ... + def visit_Lambda(self, node: Lambda) -> Any: ... + def visit_IfExp(self, node: IfExp) -> Any: ... + def visit_Dict(self, node: Dict) -> Any: ... + def visit_Set(self, node: Set) -> Any: ... + def visit_ListComp(self, node: ListComp) -> Any: ... + def visit_SetComp(self, node: SetComp) -> Any: ... + def visit_DictComp(self, node: DictComp) -> Any: ... + def visit_GeneratorExp(self, node: GeneratorExp) -> Any: ... + def visit_Await(self, node: Await) -> Any: ... + def visit_Yield(self, node: Yield) -> Any: ... + def visit_YieldFrom(self, node: YieldFrom) -> Any: ... + def visit_Compare(self, node: Compare) -> Any: ... + def visit_Call(self, node: Call) -> Any: ... + def visit_FormattedValue(self, node: FormattedValue) -> Any: ... + def visit_JoinedStr(self, node: JoinedStr) -> Any: ... + def visit_Constant(self, node: Constant) -> Any: ... + def visit_NamedExpr(self, node: NamedExpr) -> Any: ... + def visit_TypeIgnore(self, node: TypeIgnore) -> Any: ... + def visit_Attribute(self, node: Attribute) -> Any: ... + def visit_Subscript(self, node: Subscript) -> Any: ... + def visit_Starred(self, node: Starred) -> Any: ... + def visit_Name(self, node: Name) -> Any: ... + def visit_List(self, node: List) -> Any: ... + def visit_Tuple(self, node: Tuple) -> Any: ... + def visit_Del(self, node: Del) -> Any: ... + def visit_Load(self, node: Load) -> Any: ... + def visit_Store(self, node: Store) -> Any: ... + def visit_And(self, node: And) -> Any: ... + def visit_Or(self, node: Or) -> Any: ... + def visit_Add(self, node: Add) -> Any: ... + def visit_BitAnd(self, node: BitAnd) -> Any: ... + def visit_BitOr(self, node: BitOr) -> Any: ... + def visit_BitXor(self, node: BitXor) -> Any: ... + def visit_Div(self, node: Div) -> Any: ... + def visit_FloorDiv(self, node: FloorDiv) -> Any: ... + def visit_LShift(self, node: LShift) -> Any: ... + def visit_Mod(self, node: Mod) -> Any: ... + def visit_Mult(self, node: Mult) -> Any: ... + def visit_MatMult(self, node: MatMult) -> Any: ... + def visit_Pow(self, node: Pow) -> Any: ... + def visit_RShift(self, node: RShift) -> Any: ... + def visit_Sub(self, node: Sub) -> Any: ... + def visit_Invert(self, node: Invert) -> Any: ... + def visit_Not(self, node: Not) -> Any: ... + def visit_UAdd(self, node: UAdd) -> Any: ... + def visit_USub(self, node: USub) -> Any: ... + def visit_Eq(self, node: Eq) -> Any: ... + def visit_Gt(self, node: Gt) -> Any: ... + def visit_GtE(self, node: GtE) -> Any: ... + def visit_In(self, node: In) -> Any: ... + def visit_Is(self, node: Is) -> Any: ... + def visit_IsNot(self, node: IsNot) -> Any: ... + def visit_Lt(self, node: Lt) -> Any: ... + def visit_LtE(self, node: LtE) -> Any: ... + def visit_NotEq(self, node: NotEq) -> Any: ... + def visit_NotIn(self, node: NotIn) -> Any: ... + def visit_comprehension(self, node: comprehension) -> Any: ... + def visit_ExceptHandler(self, node: ExceptHandler) -> Any: ... + def visit_arguments(self, node: arguments) -> Any: ... + def visit_arg(self, node: arg) -> Any: ... + def visit_keyword(self, node: keyword) -> Any: ... + def visit_alias(self, node: alias) -> Any: ... + def visit_withitem(self, node: withitem) -> Any: ... + if sys.version_info >= (3, 10): + def visit_Match(self, node: Match) -> Any: ... + def visit_match_case(self, node: match_case) -> Any: ... + def visit_MatchValue(self, node: MatchValue) -> Any: ... + def visit_MatchSequence(self, node: MatchSequence) -> Any: ... + def visit_MatchSingleton(self, node: MatchSingleton) -> Any: ... + def visit_MatchStar(self, node: MatchStar) -> Any: ... + def visit_MatchMapping(self, node: MatchMapping) -> Any: ... + def visit_MatchClass(self, node: MatchClass) -> Any: ... + def visit_MatchAs(self, node: MatchAs) -> Any: ... + def visit_MatchOr(self, node: MatchOr) -> Any: ... + + if sys.version_info >= (3, 11): + def visit_TryStar(self, node: TryStar) -> Any: ... + + if sys.version_info >= (3, 12): + def visit_TypeVar(self, node: TypeVar) -> Any: ... + def visit_ParamSpec(self, node: ParamSpec) -> Any: ... + def visit_TypeVarTuple(self, node: TypeVarTuple) -> Any: ... + def visit_TypeAlias(self, node: TypeAlias) -> Any: ... + + # visit methods for deprecated nodes + def visit_ExtSlice(self, node: ExtSlice) -> Any: ... + def visit_Index(self, node: Index) -> Any: ... + def visit_Suite(self, node: Suite) -> Any: ... + def visit_AugLoad(self, node: AugLoad) -> Any: ... + def visit_AugStore(self, node: AugStore) -> Any: ... + def visit_Param(self, node: Param) -> Any: ... + + if sys.version_info < (3, 14): + @deprecated("Replaced by visit_Constant; removed in Python 3.14") + def visit_Num(self, node: Num) -> Any: ... # type: ignore[deprecated] + @deprecated("Replaced by visit_Constant; removed in Python 3.14") + def visit_Str(self, node: Str) -> Any: ... # type: ignore[deprecated] + @deprecated("Replaced by visit_Constant; removed in Python 3.14") + def visit_Bytes(self, node: Bytes) -> Any: ... # type: ignore[deprecated] + @deprecated("Replaced by visit_Constant; removed in Python 3.14") + def visit_NameConstant(self, node: NameConstant) -> Any: ... # type: ignore[deprecated] + @deprecated("Replaced by visit_Constant; removed in Python 3.14") + def visit_Ellipsis(self, node: Ellipsis) -> Any: ... # type: ignore[deprecated] + +class NodeTransformer(NodeVisitor): + def generic_visit(self, node: AST) -> AST: ... + # TODO: Override the visit_* methods with better return types. + # The usual return type is AST | None, but Iterable[AST] + # is also allowed in some cases -- this needs to be mapped. + +if sys.version_info >= (3, 9): + def unparse(ast_obj: AST) -> str: ... + +if sys.version_info >= (3, 9): + def main() -> None: ... diff --git a/mypy/typeshed/stdlib/asynchat.pyi b/mypy/typeshed/stdlib/asynchat.pyi new file mode 100644 index 000000000000..79a70d1c1ec8 --- /dev/null +++ b/mypy/typeshed/stdlib/asynchat.pyi @@ -0,0 +1,21 @@ +import asyncore +from abc import abstractmethod + +class simple_producer: + def __init__(self, data: bytes, buffer_size: int = 512) -> None: ... + def more(self) -> bytes: ... + +class async_chat(asyncore.dispatcher): + ac_in_buffer_size: int + ac_out_buffer_size: int + @abstractmethod + def collect_incoming_data(self, data: bytes) -> None: ... + @abstractmethod + def found_terminator(self) -> None: ... + def set_terminator(self, term: bytes | int | None) -> None: ... + def get_terminator(self) -> bytes | int | None: ... + def push(self, data: bytes) -> None: ... + def push_with_producer(self, producer: simple_producer) -> None: ... + def close_when_done(self) -> None: ... + def initiate_send(self) -> None: ... + def discard_buffers(self) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/__init__.pyi b/mypy/typeshed/stdlib/asyncio/__init__.pyi new file mode 100644 index 000000000000..daf28862aa6a --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/__init__.pyi @@ -0,0 +1,41 @@ +import sys +from collections.abc import Awaitable, Coroutine, Generator +from typing import Any, TypeVar +from typing_extensions import TypeAlias + +# As at runtime, this depends on all submodules defining __all__ accurately. +from .base_events import * +from .coroutines import * +from .events import * +from .exceptions import * +from .futures import * +from .locks import * +from .protocols import * +from .queues import * +from .runners import * +from .streams import * +from .subprocess import * +from .tasks import * +from .transports import * + +if sys.version_info >= (3, 9): + from .threads import * + +if sys.version_info >= (3, 11): + from .taskgroups import * + from .timeouts import * + +if sys.platform == "win32": + from .windows_events import * +else: + from .unix_events import * + +_T_co = TypeVar("_T_co", covariant=True) + +# Aliases imported by multiple submodules in typeshed +if sys.version_info >= (3, 12): + _AwaitableLike: TypeAlias = Awaitable[_T_co] # noqa: Y047 + _CoroutineLike: TypeAlias = Coroutine[Any, Any, _T_co] # noqa: Y047 +else: + _AwaitableLike: TypeAlias = Generator[Any, None, _T_co] | Awaitable[_T_co] + _CoroutineLike: TypeAlias = Generator[Any, None, _T_co] | Coroutine[Any, Any, _T_co] diff --git a/mypy/typeshed/stdlib/asyncio/base_events.pyi b/mypy/typeshed/stdlib/asyncio/base_events.pyi new file mode 100644 index 000000000000..cba2c7799528 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/base_events.pyi @@ -0,0 +1,488 @@ +import ssl +import sys +from _typeshed import FileDescriptorLike, ReadableBuffer, WriteableBuffer +from asyncio import _AwaitableLike, _CoroutineLike +from asyncio.events import AbstractEventLoop, AbstractServer, Handle, TimerHandle, _TaskFactory +from asyncio.futures import Future +from asyncio.protocols import BaseProtocol +from asyncio.tasks import Task +from asyncio.transports import BaseTransport, DatagramTransport, ReadTransport, SubprocessTransport, Transport, WriteTransport +from collections.abc import Callable, Iterable, Sequence +from contextvars import Context +from socket import AddressFamily, SocketKind, _Address, _RetAddress, socket +from typing import IO, Any, Literal, TypeVar, overload +from typing_extensions import TypeAlias, TypeVarTuple, Unpack + +if sys.version_info >= (3, 9): + __all__ = ("BaseEventLoop", "Server") +else: + __all__ = ("BaseEventLoop",) + +_T = TypeVar("_T") +_Ts = TypeVarTuple("_Ts") +_ProtocolT = TypeVar("_ProtocolT", bound=BaseProtocol) +_Context: TypeAlias = dict[str, Any] +_ExceptionHandler: TypeAlias = Callable[[AbstractEventLoop, _Context], object] +_ProtocolFactory: TypeAlias = Callable[[], BaseProtocol] +_SSLContext: TypeAlias = bool | None | ssl.SSLContext + +class Server(AbstractServer): + if sys.version_info >= (3, 11): + def __init__( + self, + loop: AbstractEventLoop, + sockets: Iterable[socket], + protocol_factory: _ProtocolFactory, + ssl_context: _SSLContext, + backlog: int, + ssl_handshake_timeout: float | None, + ssl_shutdown_timeout: float | None = None, + ) -> None: ... + else: + def __init__( + self, + loop: AbstractEventLoop, + sockets: Iterable[socket], + protocol_factory: _ProtocolFactory, + ssl_context: _SSLContext, + backlog: int, + ssl_handshake_timeout: float | None, + ) -> None: ... + + if sys.version_info >= (3, 13): + def close_clients(self) -> None: ... + def abort_clients(self) -> None: ... + + def get_loop(self) -> AbstractEventLoop: ... + def is_serving(self) -> bool: ... + async def start_serving(self) -> None: ... + async def serve_forever(self) -> None: ... + @property + def sockets(self) -> tuple[socket, ...]: ... + def close(self) -> None: ... + async def wait_closed(self) -> None: ... + +class BaseEventLoop(AbstractEventLoop): + def run_forever(self) -> None: ... + def run_until_complete(self, future: _AwaitableLike[_T]) -> _T: ... + def stop(self) -> None: ... + def is_running(self) -> bool: ... + def is_closed(self) -> bool: ... + def close(self) -> None: ... + async def shutdown_asyncgens(self) -> None: ... + # Methods scheduling callbacks. All these return Handles. + def call_soon( + self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> Handle: ... + def call_later( + self, delay: float, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> TimerHandle: ... + def call_at( + self, when: float, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> TimerHandle: ... + def time(self) -> float: ... + # Future methods + def create_future(self) -> Future[Any]: ... + # Tasks methods + if sys.version_info >= (3, 11): + def create_task(self, coro: _CoroutineLike[_T], *, name: object = None, context: Context | None = None) -> Task[_T]: ... + else: + def create_task(self, coro: _CoroutineLike[_T], *, name: object = None) -> Task[_T]: ... + + def set_task_factory(self, factory: _TaskFactory | None) -> None: ... + def get_task_factory(self) -> _TaskFactory | None: ... + # Methods for interacting with threads + def call_soon_threadsafe( + self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> Handle: ... + def run_in_executor(self, executor: Any, func: Callable[[Unpack[_Ts]], _T], *args: Unpack[_Ts]) -> Future[_T]: ... + def set_default_executor(self, executor: Any) -> None: ... + # Network I/O methods returning Futures. + async def getaddrinfo( + self, + host: bytes | str | None, + port: bytes | str | int | None, + *, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int] | tuple[str, int, int, int]]]: ... + async def getnameinfo(self, sockaddr: tuple[str, int] | tuple[str, int, int, int], flags: int = 0) -> tuple[str, str]: ... + if sys.version_info >= (3, 12): + @overload + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: str = ..., + port: int = ..., + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: None = None, + local_addr: tuple[str, int] | None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + all_errors: bool = False, + ) -> tuple[Transport, _ProtocolT]: ... + @overload + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: None = None, + port: None = None, + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: socket, + local_addr: None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + all_errors: bool = False, + ) -> tuple[Transport, _ProtocolT]: ... + elif sys.version_info >= (3, 11): + @overload + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: str = ..., + port: int = ..., + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: None = None, + local_addr: tuple[str, int] | None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + @overload + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: None = None, + port: None = None, + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: socket, + local_addr: None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + else: + @overload + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: str = ..., + port: int = ..., + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: None = None, + local_addr: tuple[str, int] | None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + @overload + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: None = None, + port: None = None, + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: socket, + local_addr: None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + + if sys.version_info >= (3, 13): + # 3.13 added `keep_alive`. + @overload + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., + *, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + keep_alive: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + @overload + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: None = None, + port: None = None, + *, + family: int = ..., + flags: int = ..., + sock: socket = ..., + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + keep_alive: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + elif sys.version_info >= (3, 11): + @overload + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., + *, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + @overload + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: None = None, + port: None = None, + *, + family: int = ..., + flags: int = ..., + sock: socket = ..., + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + else: + @overload + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., + *, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + @overload + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: None = None, + port: None = None, + *, + family: int = ..., + flags: int = ..., + sock: socket = ..., + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + + if sys.version_info >= (3, 11): + async def start_tls( + self, + transport: BaseTransport, + protocol: BaseProtocol, + sslcontext: ssl.SSLContext, + *, + server_side: bool = False, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> Transport | None: ... + async def connect_accepted_socket( + self, + protocol_factory: Callable[[], _ProtocolT], + sock: socket, + *, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + else: + async def start_tls( + self, + transport: BaseTransport, + protocol: BaseProtocol, + sslcontext: ssl.SSLContext, + *, + server_side: bool = False, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ) -> Transport | None: ... + async def connect_accepted_socket( + self, + protocol_factory: Callable[[], _ProtocolT], + sock: socket, + *, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + + async def sock_sendfile( + self, sock: socket, file: IO[bytes], offset: int = 0, count: int | None = None, *, fallback: bool | None = True + ) -> int: ... + async def sendfile( + self, transport: WriteTransport, file: IO[bytes], offset: int = 0, count: int | None = None, *, fallback: bool = True + ) -> int: ... + if sys.version_info >= (3, 11): + async def create_datagram_endpoint( # type: ignore[override] + self, + protocol_factory: Callable[[], _ProtocolT], + local_addr: tuple[str, int] | str | None = None, + remote_addr: tuple[str, int] | str | None = None, + *, + family: int = 0, + proto: int = 0, + flags: int = 0, + reuse_port: bool | None = None, + allow_broadcast: bool | None = None, + sock: socket | None = None, + ) -> tuple[DatagramTransport, _ProtocolT]: ... + else: + async def create_datagram_endpoint( + self, + protocol_factory: Callable[[], _ProtocolT], + local_addr: tuple[str, int] | str | None = None, + remote_addr: tuple[str, int] | str | None = None, + *, + family: int = 0, + proto: int = 0, + flags: int = 0, + reuse_address: bool | None = ..., + reuse_port: bool | None = None, + allow_broadcast: bool | None = None, + sock: socket | None = None, + ) -> tuple[DatagramTransport, _ProtocolT]: ... + # Pipes and subprocesses. + async def connect_read_pipe( + self, protocol_factory: Callable[[], _ProtocolT], pipe: Any + ) -> tuple[ReadTransport, _ProtocolT]: ... + async def connect_write_pipe( + self, protocol_factory: Callable[[], _ProtocolT], pipe: Any + ) -> tuple[WriteTransport, _ProtocolT]: ... + async def subprocess_shell( + self, + protocol_factory: Callable[[], _ProtocolT], + cmd: bytes | str, + *, + stdin: int | IO[Any] | None = -1, + stdout: int | IO[Any] | None = -1, + stderr: int | IO[Any] | None = -1, + universal_newlines: Literal[False] = False, + shell: Literal[True] = True, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + **kwargs: Any, + ) -> tuple[SubprocessTransport, _ProtocolT]: ... + async def subprocess_exec( + self, + protocol_factory: Callable[[], _ProtocolT], + program: Any, + *args: Any, + stdin: int | IO[Any] | None = -1, + stdout: int | IO[Any] | None = -1, + stderr: int | IO[Any] | None = -1, + universal_newlines: Literal[False] = False, + shell: Literal[False] = False, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + **kwargs: Any, + ) -> tuple[SubprocessTransport, _ProtocolT]: ... + def add_reader(self, fd: FileDescriptorLike, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... + def remove_reader(self, fd: FileDescriptorLike) -> bool: ... + def add_writer(self, fd: FileDescriptorLike, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... + def remove_writer(self, fd: FileDescriptorLike) -> bool: ... + # The sock_* methods (and probably some others) are not actually implemented on + # BaseEventLoop, only on subclasses. We list them here for now for convenience. + async def sock_recv(self, sock: socket, nbytes: int) -> bytes: ... + async def sock_recv_into(self, sock: socket, buf: WriteableBuffer) -> int: ... + async def sock_sendall(self, sock: socket, data: ReadableBuffer) -> None: ... + async def sock_connect(self, sock: socket, address: _Address) -> None: ... + async def sock_accept(self, sock: socket) -> tuple[socket, _RetAddress]: ... + if sys.version_info >= (3, 11): + async def sock_recvfrom(self, sock: socket, bufsize: int) -> tuple[bytes, _RetAddress]: ... + async def sock_recvfrom_into(self, sock: socket, buf: WriteableBuffer, nbytes: int = 0) -> tuple[int, _RetAddress]: ... + async def sock_sendto(self, sock: socket, data: ReadableBuffer, address: _Address) -> int: ... + # Signal handling. + def add_signal_handler(self, sig: int, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... + def remove_signal_handler(self, sig: int) -> bool: ... + # Error handlers. + def set_exception_handler(self, handler: _ExceptionHandler | None) -> None: ... + def get_exception_handler(self) -> _ExceptionHandler | None: ... + def default_exception_handler(self, context: _Context) -> None: ... + def call_exception_handler(self, context: _Context) -> None: ... + # Debug flag management. + def get_debug(self) -> bool: ... + def set_debug(self, enabled: bool) -> None: ... + if sys.version_info >= (3, 12): + async def shutdown_default_executor(self, timeout: float | None = None) -> None: ... + elif sys.version_info >= (3, 9): + async def shutdown_default_executor(self) -> None: ... + + def __del__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/base_futures.pyi b/mypy/typeshed/stdlib/asyncio/base_futures.pyi new file mode 100644 index 000000000000..55d2fbdbdb62 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/base_futures.pyi @@ -0,0 +1,19 @@ +from collections.abc import Callable, Sequence +from contextvars import Context +from typing import Any, Final + +from . import futures + +__all__ = () + +# asyncio defines 'isfuture()' in base_futures.py and re-imports it in futures.py +# but it leads to circular import error in pytype tool. +# That's why the import order is reversed. +from .futures import isfuture as isfuture + +_PENDING: Final = "PENDING" # undocumented +_CANCELLED: Final = "CANCELLED" # undocumented +_FINISHED: Final = "FINISHED" # undocumented + +def _format_callbacks(cb: Sequence[tuple[Callable[[futures.Future[Any]], None], Context]]) -> str: ... # undocumented +def _future_repr_info(future: futures.Future[Any]) -> list[str]: ... # undocumented diff --git a/mypy/typeshed/stdlib/asyncio/base_subprocess.pyi b/mypy/typeshed/stdlib/asyncio/base_subprocess.pyi new file mode 100644 index 000000000000..a5fe24e8768b --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/base_subprocess.pyi @@ -0,0 +1,63 @@ +import subprocess +from collections import deque +from collections.abc import Callable, Sequence +from typing import IO, Any +from typing_extensions import TypeAlias + +from . import events, futures, protocols, transports + +_File: TypeAlias = int | IO[Any] | None + +class BaseSubprocessTransport(transports.SubprocessTransport): + _closed: bool # undocumented + _protocol: protocols.SubprocessProtocol # undocumented + _loop: events.AbstractEventLoop # undocumented + _proc: subprocess.Popen[Any] | None # undocumented + _pid: int | None # undocumented + _returncode: int | None # undocumented + _exit_waiters: list[futures.Future[Any]] # undocumented + _pending_calls: deque[tuple[Callable[..., Any], tuple[Any, ...]]] # undocumented + _pipes: dict[int, _File] # undocumented + _finished: bool # undocumented + def __init__( + self, + loop: events.AbstractEventLoop, + protocol: protocols.SubprocessProtocol, + args: str | bytes | Sequence[str | bytes], + shell: bool, + stdin: _File, + stdout: _File, + stderr: _File, + bufsize: int, + waiter: futures.Future[Any] | None = None, + extra: Any | None = None, + **kwargs: Any, + ) -> None: ... + def _start( + self, + args: str | bytes | Sequence[str | bytes], + shell: bool, + stdin: _File, + stdout: _File, + stderr: _File, + bufsize: int, + **kwargs: Any, + ) -> None: ... # undocumented + def get_pid(self) -> int | None: ... # type: ignore[override] + def get_pipe_transport(self, fd: int) -> _File: ... # type: ignore[override] + def _check_proc(self) -> None: ... # undocumented + def send_signal(self, signal: int) -> None: ... + async def _connect_pipes(self, waiter: futures.Future[Any] | None) -> None: ... # undocumented + def _call(self, cb: Callable[..., object], *data: Any) -> None: ... # undocumented + def _pipe_connection_lost(self, fd: int, exc: BaseException | None) -> None: ... # undocumented + def _pipe_data_received(self, fd: int, data: bytes) -> None: ... # undocumented + def _process_exited(self, returncode: int) -> None: ... # undocumented + async def _wait(self) -> int: ... # undocumented + def _try_finish(self) -> None: ... # undocumented + def _call_connection_lost(self, exc: BaseException | None) -> None: ... # undocumented + def __del__(self) -> None: ... + +class WriteSubprocessPipeProto(protocols.BaseProtocol): # undocumented + def __init__(self, proc: BaseSubprocessTransport, fd: int) -> None: ... + +class ReadSubprocessPipeProto(WriteSubprocessPipeProto, protocols.Protocol): ... # undocumented diff --git a/mypy/typeshed/stdlib/asyncio/base_tasks.pyi b/mypy/typeshed/stdlib/asyncio/base_tasks.pyi new file mode 100644 index 000000000000..42e952ffacaf --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/base_tasks.pyi @@ -0,0 +1,9 @@ +from _typeshed import StrOrBytesPath +from types import FrameType +from typing import Any + +from . import tasks + +def _task_repr_info(task: tasks.Task[Any]) -> list[str]: ... # undocumented +def _task_get_stack(task: tasks.Task[Any], limit: int | None) -> list[FrameType]: ... # undocumented +def _task_print_stack(task: tasks.Task[Any], limit: int | None, file: StrOrBytesPath) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/asyncio/constants.pyi b/mypy/typeshed/stdlib/asyncio/constants.pyi new file mode 100644 index 000000000000..5c6456b0e9c0 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/constants.pyi @@ -0,0 +1,20 @@ +import enum +import sys +from typing import Final + +LOG_THRESHOLD_FOR_CONNLOST_WRITES: Final = 5 +ACCEPT_RETRY_DELAY: Final = 1 +DEBUG_STACK_DEPTH: Final = 10 +SSL_HANDSHAKE_TIMEOUT: float +SENDFILE_FALLBACK_READBUFFER_SIZE: Final = 262144 +if sys.version_info >= (3, 11): + SSL_SHUTDOWN_TIMEOUT: float + FLOW_CONTROL_HIGH_WATER_SSL_READ: Final = 256 + FLOW_CONTROL_HIGH_WATER_SSL_WRITE: Final = 512 +if sys.version_info >= (3, 12): + THREAD_JOIN_TIMEOUT: Final = 300 + +class _SendfileMode(enum.Enum): + UNSUPPORTED = 1 + TRY_NATIVE = 2 + FALLBACK = 3 diff --git a/mypy/typeshed/stdlib/asyncio/coroutines.pyi b/mypy/typeshed/stdlib/asyncio/coroutines.pyi new file mode 100644 index 000000000000..bc797de7fd51 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/coroutines.pyi @@ -0,0 +1,26 @@ +import sys +from collections.abc import Awaitable, Callable, Coroutine +from typing import Any, TypeVar, overload +from typing_extensions import ParamSpec, TypeGuard, TypeIs + +if sys.version_info >= (3, 11): + __all__ = ("iscoroutinefunction", "iscoroutine") +else: + __all__ = ("coroutine", "iscoroutinefunction", "iscoroutine") + +_T = TypeVar("_T") +_FunctionT = TypeVar("_FunctionT", bound=Callable[..., Any]) +_P = ParamSpec("_P") + +if sys.version_info < (3, 11): + def coroutine(func: _FunctionT) -> _FunctionT: ... + +@overload +def iscoroutinefunction(func: Callable[..., Coroutine[Any, Any, Any]]) -> bool: ... +@overload +def iscoroutinefunction(func: Callable[_P, Awaitable[_T]]) -> TypeGuard[Callable[_P, Coroutine[Any, Any, _T]]]: ... +@overload +def iscoroutinefunction(func: Callable[_P, object]) -> TypeGuard[Callable[_P, Coroutine[Any, Any, Any]]]: ... +@overload +def iscoroutinefunction(func: object) -> TypeGuard[Callable[..., Coroutine[Any, Any, Any]]]: ... +def iscoroutine(obj: object) -> TypeIs[Coroutine[Any, Any, Any]]: ... diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi new file mode 100644 index 000000000000..ead64070671f --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -0,0 +1,653 @@ +import ssl +import sys +from _asyncio import ( + _get_running_loop as _get_running_loop, + _set_running_loop as _set_running_loop, + get_event_loop as get_event_loop, + get_running_loop as get_running_loop, +) +from _typeshed import FileDescriptorLike, ReadableBuffer, StrPath, Unused, WriteableBuffer +from abc import ABCMeta, abstractmethod +from collections.abc import Callable, Sequence +from contextvars import Context +from socket import AddressFamily, SocketKind, _Address, _RetAddress, socket +from typing import IO, Any, Literal, Protocol, TypeVar, overload +from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack, deprecated + +from . import _AwaitableLike, _CoroutineLike +from .base_events import Server +from .futures import Future +from .protocols import BaseProtocol +from .tasks import Task +from .transports import BaseTransport, DatagramTransport, ReadTransport, SubprocessTransport, Transport, WriteTransport +from .unix_events import AbstractChildWatcher + +if sys.version_info >= (3, 14): + __all__ = ( + "AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", + ) +else: + __all__ = ( + "AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "get_child_watcher", + "set_child_watcher", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", + ) + +_T = TypeVar("_T") +_Ts = TypeVarTuple("_Ts") +_ProtocolT = TypeVar("_ProtocolT", bound=BaseProtocol) +_Context: TypeAlias = dict[str, Any] +_ExceptionHandler: TypeAlias = Callable[[AbstractEventLoop, _Context], object] +_ProtocolFactory: TypeAlias = Callable[[], BaseProtocol] +_SSLContext: TypeAlias = bool | None | ssl.SSLContext + +class _TaskFactory(Protocol): + def __call__(self, loop: AbstractEventLoop, factory: _CoroutineLike[_T], /) -> Future[_T]: ... + +class Handle: + _cancelled: bool + _args: Sequence[Any] + def __init__( + self, callback: Callable[..., object], args: Sequence[Any], loop: AbstractEventLoop, context: Context | None = None + ) -> None: ... + def cancel(self) -> None: ... + def _run(self) -> None: ... + def cancelled(self) -> bool: ... + if sys.version_info >= (3, 12): + def get_context(self) -> Context: ... + +class TimerHandle(Handle): + def __init__( + self, + when: float, + callback: Callable[..., object], + args: Sequence[Any], + loop: AbstractEventLoop, + context: Context | None = None, + ) -> None: ... + def __hash__(self) -> int: ... + def when(self) -> float: ... + def __lt__(self, other: TimerHandle) -> bool: ... + def __le__(self, other: TimerHandle) -> bool: ... + def __gt__(self, other: TimerHandle) -> bool: ... + def __ge__(self, other: TimerHandle) -> bool: ... + def __eq__(self, other: object) -> bool: ... + +class AbstractServer: + @abstractmethod + def close(self) -> None: ... + if sys.version_info >= (3, 13): + @abstractmethod + def close_clients(self) -> None: ... + @abstractmethod + def abort_clients(self) -> None: ... + + async def __aenter__(self) -> Self: ... + async def __aexit__(self, *exc: Unused) -> None: ... + @abstractmethod + def get_loop(self) -> AbstractEventLoop: ... + @abstractmethod + def is_serving(self) -> bool: ... + @abstractmethod + async def start_serving(self) -> None: ... + @abstractmethod + async def serve_forever(self) -> None: ... + @abstractmethod + async def wait_closed(self) -> None: ... + +class AbstractEventLoop: + slow_callback_duration: float + @abstractmethod + def run_forever(self) -> None: ... + @abstractmethod + def run_until_complete(self, future: _AwaitableLike[_T]) -> _T: ... + @abstractmethod + def stop(self) -> None: ... + @abstractmethod + def is_running(self) -> bool: ... + @abstractmethod + def is_closed(self) -> bool: ... + @abstractmethod + def close(self) -> None: ... + @abstractmethod + async def shutdown_asyncgens(self) -> None: ... + # Methods scheduling callbacks. All these return Handles. + if sys.version_info >= (3, 9): # "context" added in 3.9.10/3.10.2 + @abstractmethod + def call_soon( + self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> Handle: ... + @abstractmethod + def call_later( + self, delay: float, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> TimerHandle: ... + @abstractmethod + def call_at( + self, when: float, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> TimerHandle: ... + else: + @abstractmethod + def call_soon(self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> Handle: ... + @abstractmethod + def call_later(self, delay: float, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> TimerHandle: ... + @abstractmethod + def call_at(self, when: float, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> TimerHandle: ... + + @abstractmethod + def time(self) -> float: ... + # Future methods + @abstractmethod + def create_future(self) -> Future[Any]: ... + # Tasks methods + if sys.version_info >= (3, 11): + @abstractmethod + def create_task( + self, coro: _CoroutineLike[_T], *, name: str | None = None, context: Context | None = None + ) -> Task[_T]: ... + else: + @abstractmethod + def create_task(self, coro: _CoroutineLike[_T], *, name: str | None = None) -> Task[_T]: ... + + @abstractmethod + def set_task_factory(self, factory: _TaskFactory | None) -> None: ... + @abstractmethod + def get_task_factory(self) -> _TaskFactory | None: ... + # Methods for interacting with threads + if sys.version_info >= (3, 9): # "context" added in 3.9.10/3.10.2 + @abstractmethod + def call_soon_threadsafe( + self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], context: Context | None = None + ) -> Handle: ... + else: + @abstractmethod + def call_soon_threadsafe(self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> Handle: ... + + @abstractmethod + def run_in_executor(self, executor: Any, func: Callable[[Unpack[_Ts]], _T], *args: Unpack[_Ts]) -> Future[_T]: ... + @abstractmethod + def set_default_executor(self, executor: Any) -> None: ... + # Network I/O methods returning Futures. + @abstractmethod + async def getaddrinfo( + self, + host: bytes | str | None, + port: bytes | str | int | None, + *, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int] | tuple[str, int, int, int]]]: ... + @abstractmethod + async def getnameinfo(self, sockaddr: tuple[str, int] | tuple[str, int, int, int], flags: int = 0) -> tuple[str, str]: ... + if sys.version_info >= (3, 11): + @overload + @abstractmethod + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: str = ..., + port: int = ..., + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: None = None, + local_addr: tuple[str, int] | None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + @overload + @abstractmethod + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: None = None, + port: None = None, + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: socket, + local_addr: None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + else: + @overload + @abstractmethod + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: str = ..., + port: int = ..., + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: None = None, + local_addr: tuple[str, int] | None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + @overload + @abstractmethod + async def create_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + host: None = None, + port: None = None, + *, + ssl: _SSLContext = None, + family: int = 0, + proto: int = 0, + flags: int = 0, + sock: socket, + local_addr: None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + + if sys.version_info >= (3, 13): + # 3.13 added `keep_alive`. + @overload + @abstractmethod + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., + *, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + keep_alive: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + @overload + @abstractmethod + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: None = None, + port: None = None, + *, + family: int = ..., + flags: int = ..., + sock: socket = ..., + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + keep_alive: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + elif sys.version_info >= (3, 11): + @overload + @abstractmethod + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., + *, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + @overload + @abstractmethod + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: None = None, + port: None = None, + *, + family: int = ..., + flags: int = ..., + sock: socket = ..., + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + else: + @overload + @abstractmethod + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., + *, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + @overload + @abstractmethod + async def create_server( + self, + protocol_factory: _ProtocolFactory, + host: None = None, + port: None = None, + *, + family: int = ..., + flags: int = ..., + sock: socket = ..., + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ssl_handshake_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + + if sys.version_info >= (3, 11): + @abstractmethod + async def start_tls( + self, + transport: WriteTransport, + protocol: BaseProtocol, + sslcontext: ssl.SSLContext, + *, + server_side: bool = False, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> Transport | None: ... + async def create_unix_server( + self, + protocol_factory: _ProtocolFactory, + path: StrPath | None = None, + *, + sock: socket | None = None, + backlog: int = 100, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + else: + @abstractmethod + async def start_tls( + self, + transport: BaseTransport, + protocol: BaseProtocol, + sslcontext: ssl.SSLContext, + *, + server_side: bool = False, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ) -> Transport | None: ... + async def create_unix_server( + self, + protocol_factory: _ProtocolFactory, + path: StrPath | None = None, + *, + sock: socket | None = None, + backlog: int = 100, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + + if sys.version_info >= (3, 11): + async def connect_accepted_socket( + self, + protocol_factory: Callable[[], _ProtocolT], + sock: socket, + *, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + elif sys.version_info >= (3, 10): + async def connect_accepted_socket( + self, + protocol_factory: Callable[[], _ProtocolT], + sock: socket, + *, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + if sys.version_info >= (3, 11): + async def create_unix_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + path: str | None = None, + *, + ssl: _SSLContext = None, + sock: socket | None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + else: + async def create_unix_connection( + self, + protocol_factory: Callable[[], _ProtocolT], + path: str | None = None, + *, + ssl: _SSLContext = None, + sock: socket | None = None, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + + @abstractmethod + async def sock_sendfile( + self, sock: socket, file: IO[bytes], offset: int = 0, count: int | None = None, *, fallback: bool | None = None + ) -> int: ... + @abstractmethod + async def sendfile( + self, transport: WriteTransport, file: IO[bytes], offset: int = 0, count: int | None = None, *, fallback: bool = True + ) -> int: ... + @abstractmethod + async def create_datagram_endpoint( + self, + protocol_factory: Callable[[], _ProtocolT], + local_addr: tuple[str, int] | str | None = None, + remote_addr: tuple[str, int] | str | None = None, + *, + family: int = 0, + proto: int = 0, + flags: int = 0, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + allow_broadcast: bool | None = None, + sock: socket | None = None, + ) -> tuple[DatagramTransport, _ProtocolT]: ... + # Pipes and subprocesses. + @abstractmethod + async def connect_read_pipe( + self, protocol_factory: Callable[[], _ProtocolT], pipe: Any + ) -> tuple[ReadTransport, _ProtocolT]: ... + @abstractmethod + async def connect_write_pipe( + self, protocol_factory: Callable[[], _ProtocolT], pipe: Any + ) -> tuple[WriteTransport, _ProtocolT]: ... + @abstractmethod + async def subprocess_shell( + self, + protocol_factory: Callable[[], _ProtocolT], + cmd: bytes | str, + *, + stdin: int | IO[Any] | None = -1, + stdout: int | IO[Any] | None = -1, + stderr: int | IO[Any] | None = -1, + universal_newlines: Literal[False] = False, + shell: Literal[True] = True, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = ..., + **kwargs: Any, + ) -> tuple[SubprocessTransport, _ProtocolT]: ... + @abstractmethod + async def subprocess_exec( + self, + protocol_factory: Callable[[], _ProtocolT], + program: Any, + *args: Any, + stdin: int | IO[Any] | None = -1, + stdout: int | IO[Any] | None = -1, + stderr: int | IO[Any] | None = -1, + universal_newlines: Literal[False] = False, + shell: Literal[False] = False, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + **kwargs: Any, + ) -> tuple[SubprocessTransport, _ProtocolT]: ... + @abstractmethod + def add_reader(self, fd: FileDescriptorLike, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... + @abstractmethod + def remove_reader(self, fd: FileDescriptorLike) -> bool: ... + @abstractmethod + def add_writer(self, fd: FileDescriptorLike, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... + @abstractmethod + def remove_writer(self, fd: FileDescriptorLike) -> bool: ... + @abstractmethod + async def sock_recv(self, sock: socket, nbytes: int) -> bytes: ... + @abstractmethod + async def sock_recv_into(self, sock: socket, buf: WriteableBuffer) -> int: ... + @abstractmethod + async def sock_sendall(self, sock: socket, data: ReadableBuffer) -> None: ... + @abstractmethod + async def sock_connect(self, sock: socket, address: _Address) -> None: ... + @abstractmethod + async def sock_accept(self, sock: socket) -> tuple[socket, _RetAddress]: ... + if sys.version_info >= (3, 11): + @abstractmethod + async def sock_recvfrom(self, sock: socket, bufsize: int) -> tuple[bytes, _RetAddress]: ... + @abstractmethod + async def sock_recvfrom_into(self, sock: socket, buf: WriteableBuffer, nbytes: int = 0) -> tuple[int, _RetAddress]: ... + @abstractmethod + async def sock_sendto(self, sock: socket, data: ReadableBuffer, address: _Address) -> int: ... + # Signal handling. + @abstractmethod + def add_signal_handler(self, sig: int, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> None: ... + @abstractmethod + def remove_signal_handler(self, sig: int) -> bool: ... + # Error handlers. + @abstractmethod + def set_exception_handler(self, handler: _ExceptionHandler | None) -> None: ... + @abstractmethod + def get_exception_handler(self) -> _ExceptionHandler | None: ... + @abstractmethod + def default_exception_handler(self, context: _Context) -> None: ... + @abstractmethod + def call_exception_handler(self, context: _Context) -> None: ... + # Debug flag management. + @abstractmethod + def get_debug(self) -> bool: ... + @abstractmethod + def set_debug(self, enabled: bool) -> None: ... + if sys.version_info >= (3, 9): + @abstractmethod + async def shutdown_default_executor(self) -> None: ... + +class AbstractEventLoopPolicy: + @abstractmethod + def get_event_loop(self) -> AbstractEventLoop: ... + @abstractmethod + def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def new_event_loop(self) -> AbstractEventLoop: ... + # Child processes handling (Unix only). + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @abstractmethod + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher(self) -> AbstractChildWatcher: ... + @abstractmethod + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... + else: + @abstractmethod + def get_child_watcher(self) -> AbstractChildWatcher: ... + @abstractmethod + def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... + +class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy, metaclass=ABCMeta): + def get_event_loop(self) -> AbstractEventLoop: ... + def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... + def new_event_loop(self) -> AbstractEventLoop: ... + +def get_event_loop_policy() -> AbstractEventLoopPolicy: ... +def set_event_loop_policy(policy: AbstractEventLoopPolicy | None) -> None: ... +def set_event_loop(loop: AbstractEventLoop | None) -> None: ... +def new_event_loop() -> AbstractEventLoop: ... + +if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher() -> AbstractChildWatcher: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... + + else: + def get_child_watcher() -> AbstractChildWatcher: ... + def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/exceptions.pyi b/mypy/typeshed/stdlib/asyncio/exceptions.pyi new file mode 100644 index 000000000000..0746394d582f --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/exceptions.pyi @@ -0,0 +1,43 @@ +import sys + +if sys.version_info >= (3, 11): + __all__ = ( + "BrokenBarrierError", + "CancelledError", + "InvalidStateError", + "TimeoutError", + "IncompleteReadError", + "LimitOverrunError", + "SendfileNotAvailableError", + ) +else: + __all__ = ( + "CancelledError", + "InvalidStateError", + "TimeoutError", + "IncompleteReadError", + "LimitOverrunError", + "SendfileNotAvailableError", + ) + +class CancelledError(BaseException): ... + +if sys.version_info >= (3, 11): + from builtins import TimeoutError as TimeoutError +else: + class TimeoutError(Exception): ... + +class InvalidStateError(Exception): ... +class SendfileNotAvailableError(RuntimeError): ... + +class IncompleteReadError(EOFError): + expected: int | None + partial: bytes + def __init__(self, partial: bytes, expected: int | None) -> None: ... + +class LimitOverrunError(Exception): + consumed: int + def __init__(self, message: str, consumed: int) -> None: ... + +if sys.version_info >= (3, 11): + class BrokenBarrierError(RuntimeError): ... diff --git a/mypy/typeshed/stdlib/asyncio/format_helpers.pyi b/mypy/typeshed/stdlib/asyncio/format_helpers.pyi new file mode 100644 index 000000000000..41505b14cd08 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/format_helpers.pyi @@ -0,0 +1,31 @@ +import functools +import sys +import traceback +from collections.abc import Iterable +from types import FrameType, FunctionType +from typing import Any, overload +from typing_extensions import TypeAlias + +class _HasWrapper: + __wrapper__: _HasWrapper | FunctionType + +_FuncType: TypeAlias = FunctionType | _HasWrapper | functools.partial[Any] | functools.partialmethod[Any] + +@overload +def _get_function_source(func: _FuncType) -> tuple[str, int]: ... +@overload +def _get_function_source(func: object) -> tuple[str, int] | None: ... + +if sys.version_info >= (3, 13): + def _format_callback_source(func: object, args: Iterable[Any], *, debug: bool = False) -> str: ... + def _format_args_and_kwargs(args: Iterable[Any], kwargs: dict[str, Any], *, debug: bool = False) -> str: ... + def _format_callback( + func: object, args: Iterable[Any], kwargs: dict[str, Any], *, debug: bool = False, suffix: str = "" + ) -> str: ... + +else: + def _format_callback_source(func: object, args: Iterable[Any]) -> str: ... + def _format_args_and_kwargs(args: Iterable[Any], kwargs: dict[str, Any]) -> str: ... + def _format_callback(func: object, args: Iterable[Any], kwargs: dict[str, Any], suffix: str = "") -> str: ... + +def extract_stack(f: FrameType | None = None, limit: int | None = None) -> traceback.StackSummary: ... diff --git a/mypy/typeshed/stdlib/asyncio/futures.pyi b/mypy/typeshed/stdlib/asyncio/futures.pyi new file mode 100644 index 000000000000..28e6ca8c86a3 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/futures.pyi @@ -0,0 +1,16 @@ +from _asyncio import Future as Future +from concurrent.futures._base import Future as _ConcurrentFuture +from typing import Any, TypeVar +from typing_extensions import TypeIs + +from .events import AbstractEventLoop + +__all__ = ("Future", "wrap_future", "isfuture") + +_T = TypeVar("_T") + +# asyncio defines 'isfuture()' in base_futures.py and re-imports it in futures.py +# but it leads to circular import error in pytype tool. +# That's why the import order is reversed. +def isfuture(obj: object) -> TypeIs[Future[Any]]: ... +def wrap_future(future: _ConcurrentFuture[_T] | Future[_T], *, loop: AbstractEventLoop | None = None) -> Future[_T]: ... diff --git a/mypy/typeshed/stdlib/asyncio/locks.pyi b/mypy/typeshed/stdlib/asyncio/locks.pyi new file mode 100644 index 000000000000..0114aeb23329 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/locks.pyi @@ -0,0 +1,121 @@ +import enum +import sys +from _typeshed import Unused +from collections import deque +from collections.abc import Callable, Generator +from types import TracebackType +from typing import Any, Literal, TypeVar +from typing_extensions import Self + +from .events import AbstractEventLoop +from .futures import Future + +if sys.version_info >= (3, 10): + from .mixins import _LoopBoundMixin +else: + _LoopBoundMixin = object + +if sys.version_info >= (3, 11): + __all__ = ("Lock", "Event", "Condition", "Semaphore", "BoundedSemaphore", "Barrier") +else: + __all__ = ("Lock", "Event", "Condition", "Semaphore", "BoundedSemaphore") + +_T = TypeVar("_T") + +if sys.version_info >= (3, 9): + class _ContextManagerMixin: + async def __aenter__(self) -> None: ... + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None + ) -> None: ... + +else: + class _ContextManager: + def __init__(self, lock: Lock | Semaphore) -> None: ... + def __enter__(self) -> None: ... + def __exit__(self, *args: Unused) -> None: ... + + class _ContextManagerMixin: + # Apparently this exists to *prohibit* use as a context manager. + # def __enter__(self) -> NoReturn: ... see: https://github.com/python/typing/issues/1043 + # def __exit__(self, *args: Any) -> None: ... + def __iter__(self) -> Generator[Any, None, _ContextManager]: ... + def __await__(self) -> Generator[Any, None, _ContextManager]: ... + async def __aenter__(self) -> None: ... + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None + ) -> None: ... + +class Lock(_ContextManagerMixin, _LoopBoundMixin): + _waiters: deque[Future[Any]] | None + if sys.version_info >= (3, 10): + def __init__(self) -> None: ... + else: + def __init__(self, *, loop: AbstractEventLoop | None = None) -> None: ... + + def locked(self) -> bool: ... + async def acquire(self) -> Literal[True]: ... + def release(self) -> None: ... + +class Event(_LoopBoundMixin): + _waiters: deque[Future[Any]] + if sys.version_info >= (3, 10): + def __init__(self) -> None: ... + else: + def __init__(self, *, loop: AbstractEventLoop | None = None) -> None: ... + + def is_set(self) -> bool: ... + def set(self) -> None: ... + def clear(self) -> None: ... + async def wait(self) -> Literal[True]: ... + +class Condition(_ContextManagerMixin, _LoopBoundMixin): + _waiters: deque[Future[Any]] + if sys.version_info >= (3, 10): + def __init__(self, lock: Lock | None = None) -> None: ... + else: + def __init__(self, lock: Lock | None = None, *, loop: AbstractEventLoop | None = None) -> None: ... + + def locked(self) -> bool: ... + async def acquire(self) -> Literal[True]: ... + def release(self) -> None: ... + async def wait(self) -> Literal[True]: ... + async def wait_for(self, predicate: Callable[[], _T]) -> _T: ... + def notify(self, n: int = 1) -> None: ... + def notify_all(self) -> None: ... + +class Semaphore(_ContextManagerMixin, _LoopBoundMixin): + _value: int + _waiters: deque[Future[Any]] | None + if sys.version_info >= (3, 10): + def __init__(self, value: int = 1) -> None: ... + else: + def __init__(self, value: int = 1, *, loop: AbstractEventLoop | None = None) -> None: ... + + def locked(self) -> bool: ... + async def acquire(self) -> Literal[True]: ... + def release(self) -> None: ... + def _wake_up_next(self) -> None: ... + +class BoundedSemaphore(Semaphore): ... + +if sys.version_info >= (3, 11): + class _BarrierState(enum.Enum): # undocumented + FILLING = "filling" + DRAINING = "draining" + RESETTING = "resetting" + BROKEN = "broken" + + class Barrier(_LoopBoundMixin): + def __init__(self, parties: int) -> None: ... + async def __aenter__(self) -> Self: ... + async def __aexit__(self, *args: Unused) -> None: ... + async def wait(self) -> int: ... + async def abort(self) -> None: ... + async def reset(self) -> None: ... + @property + def parties(self) -> int: ... + @property + def n_waiting(self) -> int: ... + @property + def broken(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/asyncio/log.pyi b/mypy/typeshed/stdlib/asyncio/log.pyi new file mode 100644 index 000000000000..e1de0b3bb845 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/log.pyi @@ -0,0 +1,3 @@ +import logging + +logger: logging.Logger diff --git a/mypy/typeshed/stdlib/asyncio/mixins.pyi b/mypy/typeshed/stdlib/asyncio/mixins.pyi new file mode 100644 index 000000000000..6ebcf543e6b9 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/mixins.pyi @@ -0,0 +1,9 @@ +import sys +import threading +from typing_extensions import Never + +_global_lock: threading.Lock + +class _LoopBoundMixin: + if sys.version_info < (3, 11): + def __init__(self, *, loop: Never = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/proactor_events.pyi b/mypy/typeshed/stdlib/asyncio/proactor_events.pyi new file mode 100644 index 000000000000..957fdd6ce255 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/proactor_events.pyi @@ -0,0 +1,64 @@ +import sys +from collections.abc import Mapping +from socket import socket +from typing import Any, ClassVar, Literal + +from . import base_events, constants, events, futures, streams, transports + +__all__ = ("BaseProactorEventLoop",) + +class _ProactorBasePipeTransport(transports._FlowControlMixin, transports.BaseTransport): + def __init__( + self, + loop: events.AbstractEventLoop, + sock: socket, + protocol: streams.StreamReaderProtocol, + waiter: futures.Future[Any] | None = None, + extra: Mapping[Any, Any] | None = None, + server: events.AbstractServer | None = None, + ) -> None: ... + def __del__(self) -> None: ... + +class _ProactorReadPipeTransport(_ProactorBasePipeTransport, transports.ReadTransport): + if sys.version_info >= (3, 10): + def __init__( + self, + loop: events.AbstractEventLoop, + sock: socket, + protocol: streams.StreamReaderProtocol, + waiter: futures.Future[Any] | None = None, + extra: Mapping[Any, Any] | None = None, + server: events.AbstractServer | None = None, + buffer_size: int = 65536, + ) -> None: ... + else: + def __init__( + self, + loop: events.AbstractEventLoop, + sock: socket, + protocol: streams.StreamReaderProtocol, + waiter: futures.Future[Any] | None = None, + extra: Mapping[Any, Any] | None = None, + server: events.AbstractServer | None = None, + ) -> None: ... + +class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, transports.WriteTransport): ... +class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport): ... +class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport, _ProactorBaseWritePipeTransport, transports.Transport): ... + +class _ProactorSocketTransport(_ProactorReadPipeTransport, _ProactorBaseWritePipeTransport, transports.Transport): + _sendfile_compatible: ClassVar[constants._SendfileMode] + def __init__( + self, + loop: events.AbstractEventLoop, + sock: socket, + protocol: streams.StreamReaderProtocol, + waiter: futures.Future[Any] | None = None, + extra: Mapping[Any, Any] | None = None, + server: events.AbstractServer | None = None, + ) -> None: ... + def _set_extra(self, sock: socket) -> None: ... + def can_write_eof(self) -> Literal[True]: ... + +class BaseProactorEventLoop(base_events.BaseEventLoop): + def __init__(self, proactor: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/protocols.pyi b/mypy/typeshed/stdlib/asyncio/protocols.pyi new file mode 100644 index 000000000000..5173b74ed5a0 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/protocols.pyi @@ -0,0 +1,34 @@ +from _typeshed import ReadableBuffer +from asyncio import transports +from typing import Any + +__all__ = ("BaseProtocol", "Protocol", "DatagramProtocol", "SubprocessProtocol", "BufferedProtocol") + +class BaseProtocol: + def connection_made(self, transport: transports.BaseTransport) -> None: ... + def connection_lost(self, exc: Exception | None) -> None: ... + def pause_writing(self) -> None: ... + def resume_writing(self) -> None: ... + +class Protocol(BaseProtocol): + def data_received(self, data: bytes) -> None: ... + def eof_received(self) -> bool | None: ... + +class BufferedProtocol(BaseProtocol): + def get_buffer(self, sizehint: int) -> ReadableBuffer: ... + def buffer_updated(self, nbytes: int) -> None: ... + def eof_received(self) -> bool | None: ... + +class DatagramProtocol(BaseProtocol): + def connection_made(self, transport: transports.DatagramTransport) -> None: ... # type: ignore[override] + # addr can be a tuple[int, int] for some unusual protocols like socket.AF_NETLINK. + # Use tuple[str | Any, int] to not cause typechecking issues on most usual cases. + # This could be improved by using tuple[AnyOf[str, int], int] if the AnyOf feature is accepted. + # See https://github.com/python/typing/issues/566 + def datagram_received(self, data: bytes, addr: tuple[str | Any, int]) -> None: ... + def error_received(self, exc: Exception) -> None: ... + +class SubprocessProtocol(BaseProtocol): + def pipe_data_received(self, fd: int, data: bytes) -> None: ... + def pipe_connection_lost(self, fd: int, exc: Exception | None) -> None: ... + def process_exited(self) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/queues.pyi b/mypy/typeshed/stdlib/asyncio/queues.pyi new file mode 100644 index 000000000000..895205aa9519 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/queues.pyi @@ -0,0 +1,56 @@ +import sys +from asyncio.events import AbstractEventLoop +from typing import Any, Generic, TypeVar + +if sys.version_info >= (3, 9): + from types import GenericAlias + +if sys.version_info >= (3, 10): + from .mixins import _LoopBoundMixin +else: + _LoopBoundMixin = object + +class QueueEmpty(Exception): ... +class QueueFull(Exception): ... + +if sys.version_info >= (3, 13): + __all__ = ("Queue", "PriorityQueue", "LifoQueue", "QueueFull", "QueueEmpty", "QueueShutDown") + +else: + __all__ = ("Queue", "PriorityQueue", "LifoQueue", "QueueFull", "QueueEmpty") + +_T = TypeVar("_T") + +if sys.version_info >= (3, 13): + class QueueShutDown(Exception): ... + +# If Generic[_T] is last and _LoopBoundMixin is object, pyright is unhappy. +# We can remove the noqa pragma when dropping 3.9 support. +class Queue(Generic[_T], _LoopBoundMixin): # noqa: Y059 + if sys.version_info >= (3, 10): + def __init__(self, maxsize: int = 0) -> None: ... + else: + def __init__(self, maxsize: int = 0, *, loop: AbstractEventLoop | None = None) -> None: ... + + def _init(self, maxsize: int) -> None: ... + def _get(self) -> _T: ... + def _put(self, item: _T) -> None: ... + def _format(self) -> str: ... + def qsize(self) -> int: ... + @property + def maxsize(self) -> int: ... + def empty(self) -> bool: ... + def full(self) -> bool: ... + async def put(self, item: _T) -> None: ... + def put_nowait(self, item: _T) -> None: ... + async def get(self) -> _T: ... + def get_nowait(self) -> _T: ... + async def join(self) -> None: ... + def task_done(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, type: Any, /) -> GenericAlias: ... + if sys.version_info >= (3, 13): + def shutdown(self, immediate: bool = False) -> None: ... + +class PriorityQueue(Queue[_T]): ... +class LifoQueue(Queue[_T]): ... diff --git a/mypy/typeshed/stdlib/asyncio/runners.pyi b/mypy/typeshed/stdlib/asyncio/runners.pyi new file mode 100644 index 000000000000..37a85b709cdc --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/runners.pyi @@ -0,0 +1,32 @@ +import sys +from _typeshed import Unused +from collections.abc import Callable, Coroutine +from contextvars import Context +from typing import Any, TypeVar, final +from typing_extensions import Self + +from .events import AbstractEventLoop + +if sys.version_info >= (3, 11): + __all__ = ("Runner", "run") +else: + __all__ = ("run",) +_T = TypeVar("_T") + +if sys.version_info >= (3, 11): + @final + class Runner: + def __init__(self, *, debug: bool | None = None, loop_factory: Callable[[], AbstractEventLoop] | None = None) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type: Unused, exc_val: Unused, exc_tb: Unused) -> None: ... + def close(self) -> None: ... + def get_loop(self) -> AbstractEventLoop: ... + def run(self, coro: Coroutine[Any, Any, _T], *, context: Context | None = None) -> _T: ... + +if sys.version_info >= (3, 12): + def run( + main: Coroutine[Any, Any, _T], *, debug: bool | None = ..., loop_factory: Callable[[], AbstractEventLoop] | None = ... + ) -> _T: ... + +else: + def run(main: Coroutine[Any, Any, _T], *, debug: bool | None = None) -> _T: ... diff --git a/mypy/typeshed/stdlib/asyncio/selector_events.pyi b/mypy/typeshed/stdlib/asyncio/selector_events.pyi new file mode 100644 index 000000000000..430f2dd405cd --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/selector_events.pyi @@ -0,0 +1,8 @@ +import selectors + +from . import base_events + +__all__ = ("BaseSelectorEventLoop",) + +class BaseSelectorEventLoop(base_events.BaseEventLoop): + def __init__(self, selector: selectors.BaseSelector | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/sslproto.pyi b/mypy/typeshed/stdlib/asyncio/sslproto.pyi new file mode 100644 index 000000000000..ded1933dd659 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/sslproto.pyi @@ -0,0 +1,165 @@ +import ssl +import sys +from collections import deque +from collections.abc import Callable +from enum import Enum +from typing import Any, ClassVar, Final, Literal +from typing_extensions import TypeAlias + +from . import constants, events, futures, protocols, transports + +def _create_transport_context(server_side: bool, server_hostname: str | None) -> ssl.SSLContext: ... + +if sys.version_info >= (3, 11): + SSLAgainErrors: tuple[type[ssl.SSLWantReadError], type[ssl.SSLSyscallError]] + + class SSLProtocolState(Enum): + UNWRAPPED = "UNWRAPPED" + DO_HANDSHAKE = "DO_HANDSHAKE" + WRAPPED = "WRAPPED" + FLUSHING = "FLUSHING" + SHUTDOWN = "SHUTDOWN" + + class AppProtocolState(Enum): + STATE_INIT = "STATE_INIT" + STATE_CON_MADE = "STATE_CON_MADE" + STATE_EOF = "STATE_EOF" + STATE_CON_LOST = "STATE_CON_LOST" + + def add_flowcontrol_defaults(high: int | None, low: int | None, kb: int) -> tuple[int, int]: ... + +else: + _UNWRAPPED: Final = "UNWRAPPED" + _DO_HANDSHAKE: Final = "DO_HANDSHAKE" + _WRAPPED: Final = "WRAPPED" + _SHUTDOWN: Final = "SHUTDOWN" + +if sys.version_info < (3, 11): + class _SSLPipe: + max_size: ClassVar[int] + + _context: ssl.SSLContext + _server_side: bool + _server_hostname: str | None + _state: str + _incoming: ssl.MemoryBIO + _outgoing: ssl.MemoryBIO + _sslobj: ssl.SSLObject | None + _need_ssldata: bool + _handshake_cb: Callable[[BaseException | None], None] | None + _shutdown_cb: Callable[[], None] | None + def __init__(self, context: ssl.SSLContext, server_side: bool, server_hostname: str | None = None) -> None: ... + @property + def context(self) -> ssl.SSLContext: ... + @property + def ssl_object(self) -> ssl.SSLObject | None: ... + @property + def need_ssldata(self) -> bool: ... + @property + def wrapped(self) -> bool: ... + def do_handshake(self, callback: Callable[[BaseException | None], object] | None = None) -> list[bytes]: ... + def shutdown(self, callback: Callable[[], object] | None = None) -> list[bytes]: ... + def feed_eof(self) -> None: ... + def feed_ssldata(self, data: bytes, only_handshake: bool = False) -> tuple[list[bytes], list[bytes]]: ... + def feed_appdata(self, data: bytes, offset: int = 0) -> tuple[list[bytes], int]: ... + +class _SSLProtocolTransport(transports._FlowControlMixin, transports.Transport): + _sendfile_compatible: ClassVar[constants._SendfileMode] + + _loop: events.AbstractEventLoop + if sys.version_info >= (3, 11): + _ssl_protocol: SSLProtocol | None + else: + _ssl_protocol: SSLProtocol + _closed: bool + def __init__(self, loop: events.AbstractEventLoop, ssl_protocol: SSLProtocol) -> None: ... + def get_extra_info(self, name: str, default: Any | None = None) -> dict[str, Any]: ... + @property + def _protocol_paused(self) -> bool: ... + def write(self, data: bytes | bytearray | memoryview) -> None: ... + def can_write_eof(self) -> Literal[False]: ... + if sys.version_info >= (3, 11): + def get_write_buffer_limits(self) -> tuple[int, int]: ... + def get_read_buffer_limits(self) -> tuple[int, int]: ... + def set_read_buffer_limits(self, high: int | None = None, low: int | None = None) -> None: ... + def get_read_buffer_size(self) -> int: ... + + def __del__(self) -> None: ... + +if sys.version_info >= (3, 11): + _SSLProtocolBase: TypeAlias = protocols.BufferedProtocol +else: + _SSLProtocolBase: TypeAlias = protocols.Protocol + +class SSLProtocol(_SSLProtocolBase): + _server_side: bool + _server_hostname: str | None + _sslcontext: ssl.SSLContext + _extra: dict[str, Any] + _write_backlog: deque[tuple[bytes, int]] + _write_buffer_size: int + _waiter: futures.Future[Any] + _loop: events.AbstractEventLoop + _app_transport: _SSLProtocolTransport + _transport: transports.BaseTransport | None + _ssl_handshake_timeout: int | None + _app_protocol: protocols.BaseProtocol + _app_protocol_is_buffer: bool + + if sys.version_info >= (3, 11): + max_size: ClassVar[int] + else: + _sslpipe: _SSLPipe | None + _session_established: bool + _call_connection_made: bool + _in_handshake: bool + _in_shutdown: bool + + if sys.version_info >= (3, 11): + def __init__( + self, + loop: events.AbstractEventLoop, + app_protocol: protocols.BaseProtocol, + sslcontext: ssl.SSLContext, + waiter: futures.Future[Any], + server_side: bool = False, + server_hostname: str | None = None, + call_connection_made: bool = True, + ssl_handshake_timeout: int | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> None: ... + else: + def __init__( + self, + loop: events.AbstractEventLoop, + app_protocol: protocols.BaseProtocol, + sslcontext: ssl.SSLContext, + waiter: futures.Future[Any], + server_side: bool = False, + server_hostname: str | None = None, + call_connection_made: bool = True, + ssl_handshake_timeout: int | None = None, + ) -> None: ... + + def _set_app_protocol(self, app_protocol: protocols.BaseProtocol) -> None: ... + def _wakeup_waiter(self, exc: BaseException | None = None) -> None: ... + def connection_lost(self, exc: BaseException | None) -> None: ... + def eof_received(self) -> None: ... + def _get_extra_info(self, name: str, default: Any | None = None) -> Any: ... + def _start_shutdown(self) -> None: ... + if sys.version_info >= (3, 11): + def _write_appdata(self, list_of_data: list[bytes]) -> None: ... + else: + def _write_appdata(self, data: bytes) -> None: ... + + def _start_handshake(self) -> None: ... + def _check_handshake_timeout(self) -> None: ... + def _on_handshake_complete(self, handshake_exc: BaseException | None) -> None: ... + def _fatal_error(self, exc: BaseException, message: str = "Fatal error on transport") -> None: ... + if sys.version_info >= (3, 11): + def _abort(self, exc: BaseException | None) -> None: ... + def get_buffer(self, n: int) -> memoryview: ... + else: + def _abort(self) -> None: ... + def _finalize(self) -> None: ... + def _process_write_backlog(self) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/staggered.pyi b/mypy/typeshed/stdlib/asyncio/staggered.pyi new file mode 100644 index 000000000000..3324777f4168 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/staggered.pyi @@ -0,0 +1,10 @@ +from collections.abc import Awaitable, Callable, Iterable +from typing import Any + +from . import events + +__all__ = ("staggered_race",) + +async def staggered_race( + coro_fns: Iterable[Callable[[], Awaitable[Any]]], delay: float | None, *, loop: events.AbstractEventLoop | None = None +) -> tuple[Any, int | None, list[Exception | None]]: ... diff --git a/mypy/typeshed/stdlib/asyncio/streams.pyi b/mypy/typeshed/stdlib/asyncio/streams.pyi new file mode 100644 index 000000000000..ed95583c1847 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/streams.pyi @@ -0,0 +1,157 @@ +import ssl +import sys +from _typeshed import ReadableBuffer, StrPath +from collections.abc import Awaitable, Callable, Iterable, Sequence, Sized +from types import ModuleType +from typing import Any, Protocol, SupportsIndex +from typing_extensions import Self, TypeAlias + +from . import events, protocols, transports +from .base_events import Server + +if sys.platform == "win32": + __all__ = ("StreamReader", "StreamWriter", "StreamReaderProtocol", "open_connection", "start_server") +else: + __all__ = ( + "StreamReader", + "StreamWriter", + "StreamReaderProtocol", + "open_connection", + "start_server", + "open_unix_connection", + "start_unix_server", + ) + +_ClientConnectedCallback: TypeAlias = Callable[[StreamReader, StreamWriter], Awaitable[None] | None] + +class _ReaduntilBuffer(ReadableBuffer, Sized, Protocol): ... + +if sys.version_info >= (3, 10): + async def open_connection( + host: str | None = None, + port: int | str | None = None, + *, + limit: int = 65536, + ssl_handshake_timeout: float | None = ..., + **kwds: Any, + ) -> tuple[StreamReader, StreamWriter]: ... + async def start_server( + client_connected_cb: _ClientConnectedCallback, + host: str | Sequence[str] | None = None, + port: int | str | None = None, + *, + limit: int = 65536, + ssl_handshake_timeout: float | None = ..., + **kwds: Any, + ) -> Server: ... + +else: + async def open_connection( + host: str | None = None, + port: int | str | None = None, + *, + loop: events.AbstractEventLoop | None = None, + limit: int = 65536, + ssl_handshake_timeout: float | None = ..., + **kwds: Any, + ) -> tuple[StreamReader, StreamWriter]: ... + async def start_server( + client_connected_cb: _ClientConnectedCallback, + host: str | None = None, + port: int | str | None = None, + *, + loop: events.AbstractEventLoop | None = None, + limit: int = 65536, + ssl_handshake_timeout: float | None = ..., + **kwds: Any, + ) -> Server: ... + +if sys.platform != "win32": + if sys.version_info >= (3, 10): + async def open_unix_connection( + path: StrPath | None = None, *, limit: int = 65536, **kwds: Any + ) -> tuple[StreamReader, StreamWriter]: ... + async def start_unix_server( + client_connected_cb: _ClientConnectedCallback, path: StrPath | None = None, *, limit: int = 65536, **kwds: Any + ) -> Server: ... + else: + async def open_unix_connection( + path: StrPath | None = None, *, loop: events.AbstractEventLoop | None = None, limit: int = 65536, **kwds: Any + ) -> tuple[StreamReader, StreamWriter]: ... + async def start_unix_server( + client_connected_cb: _ClientConnectedCallback, + path: StrPath | None = None, + *, + loop: events.AbstractEventLoop | None = None, + limit: int = 65536, + **kwds: Any, + ) -> Server: ... + +class FlowControlMixin(protocols.Protocol): + def __init__(self, loop: events.AbstractEventLoop | None = None) -> None: ... + +class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): + def __init__( + self, + stream_reader: StreamReader, + client_connected_cb: _ClientConnectedCallback | None = None, + loop: events.AbstractEventLoop | None = None, + ) -> None: ... + def __del__(self) -> None: ... + +class StreamWriter: + def __init__( + self, + transport: transports.WriteTransport, + protocol: protocols.BaseProtocol, + reader: StreamReader | None, + loop: events.AbstractEventLoop, + ) -> None: ... + @property + def transport(self) -> transports.WriteTransport: ... + def write(self, data: bytes | bytearray | memoryview) -> None: ... + def writelines(self, data: Iterable[bytes | bytearray | memoryview]) -> None: ... + def write_eof(self) -> None: ... + def can_write_eof(self) -> bool: ... + def close(self) -> None: ... + def is_closing(self) -> bool: ... + async def wait_closed(self) -> None: ... + def get_extra_info(self, name: str, default: Any = None) -> Any: ... + async def drain(self) -> None: ... + if sys.version_info >= (3, 12): + async def start_tls( + self, + sslcontext: ssl.SSLContext, + *, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> None: ... + elif sys.version_info >= (3, 11): + async def start_tls( + self, sslcontext: ssl.SSLContext, *, server_hostname: str | None = None, ssl_handshake_timeout: float | None = None + ) -> None: ... + + if sys.version_info >= (3, 13): + def __del__(self, warnings: ModuleType = ...) -> None: ... + elif sys.version_info >= (3, 11): + def __del__(self) -> None: ... + +class StreamReader: + def __init__(self, limit: int = 65536, loop: events.AbstractEventLoop | None = None) -> None: ... + def exception(self) -> Exception: ... + def set_exception(self, exc: Exception) -> None: ... + def set_transport(self, transport: transports.BaseTransport) -> None: ... + def feed_eof(self) -> None: ... + def at_eof(self) -> bool: ... + def feed_data(self, data: Iterable[SupportsIndex]) -> None: ... + async def readline(self) -> bytes: ... + if sys.version_info >= (3, 13): + async def readuntil(self, separator: _ReaduntilBuffer | tuple[_ReaduntilBuffer, ...] = b"\n") -> bytes: ... + else: + async def readuntil(self, separator: _ReaduntilBuffer = b"\n") -> bytes: ... + + async def read(self, n: int = -1) -> bytes: ... + async def readexactly(self, n: int) -> bytes: ... + def __aiter__(self) -> Self: ... + async def __anext__(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/asyncio/subprocess.pyi b/mypy/typeshed/stdlib/asyncio/subprocess.pyi new file mode 100644 index 000000000000..19452d4eb469 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/subprocess.pyi @@ -0,0 +1,229 @@ +import subprocess +import sys +from _typeshed import StrOrBytesPath +from asyncio import events, protocols, streams, transports +from collections.abc import Callable, Collection +from typing import IO, Any, Literal + +__all__ = ("create_subprocess_exec", "create_subprocess_shell") + +PIPE: int +STDOUT: int +DEVNULL: int + +class SubprocessStreamProtocol(streams.FlowControlMixin, protocols.SubprocessProtocol): + stdin: streams.StreamWriter | None + stdout: streams.StreamReader | None + stderr: streams.StreamReader | None + def __init__(self, limit: int, loop: events.AbstractEventLoop) -> None: ... + def pipe_data_received(self, fd: int, data: bytes | str) -> None: ... + +class Process: + stdin: streams.StreamWriter | None + stdout: streams.StreamReader | None + stderr: streams.StreamReader | None + pid: int + def __init__( + self, transport: transports.BaseTransport, protocol: protocols.BaseProtocol, loop: events.AbstractEventLoop + ) -> None: ... + @property + def returncode(self) -> int | None: ... + async def wait(self) -> int: ... + def send_signal(self, signal: int) -> None: ... + def terminate(self) -> None: ... + def kill(self) -> None: ... + async def communicate(self, input: bytes | bytearray | memoryview | None = None) -> tuple[bytes, bytes]: ... + +if sys.version_info >= (3, 11): + async def create_subprocess_shell( + cmd: str | bytes, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + limit: int = 65536, + *, + # These parameters are forced to these values by BaseEventLoop.subprocess_shell + universal_newlines: Literal[False] = False, + shell: Literal[True] = True, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + # These parameters are taken by subprocess.Popen, which this ultimately delegates to + executable: StrOrBytesPath | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + cwd: StrOrBytesPath | None = None, + env: subprocess._ENV | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + group: None | str | int = None, + extra_groups: None | Collection[str | int] = None, + user: None | str | int = None, + umask: int = -1, + process_group: int | None = None, + pipesize: int = -1, + ) -> Process: ... + async def create_subprocess_exec( + program: StrOrBytesPath, + *args: StrOrBytesPath, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + limit: int = 65536, + # These parameters are forced to these values by BaseEventLoop.subprocess_exec + universal_newlines: Literal[False] = False, + shell: Literal[False] = False, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + # These parameters are taken by subprocess.Popen, which this ultimately delegates to + executable: StrOrBytesPath | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + cwd: StrOrBytesPath | None = None, + env: subprocess._ENV | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + group: None | str | int = None, + extra_groups: None | Collection[str | int] = None, + user: None | str | int = None, + umask: int = -1, + process_group: int | None = None, + pipesize: int = -1, + ) -> Process: ... + +elif sys.version_info >= (3, 10): + async def create_subprocess_shell( + cmd: str | bytes, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + limit: int = 65536, + *, + # These parameters are forced to these values by BaseEventLoop.subprocess_shell + universal_newlines: Literal[False] = False, + shell: Literal[True] = True, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + # These parameters are taken by subprocess.Popen, which this ultimately delegates to + executable: StrOrBytesPath | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + cwd: StrOrBytesPath | None = None, + env: subprocess._ENV | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + group: None | str | int = None, + extra_groups: None | Collection[str | int] = None, + user: None | str | int = None, + umask: int = -1, + pipesize: int = -1, + ) -> Process: ... + async def create_subprocess_exec( + program: StrOrBytesPath, + *args: StrOrBytesPath, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + limit: int = 65536, + # These parameters are forced to these values by BaseEventLoop.subprocess_exec + universal_newlines: Literal[False] = False, + shell: Literal[False] = False, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + # These parameters are taken by subprocess.Popen, which this ultimately delegates to + executable: StrOrBytesPath | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + cwd: StrOrBytesPath | None = None, + env: subprocess._ENV | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + group: None | str | int = None, + extra_groups: None | Collection[str | int] = None, + user: None | str | int = None, + umask: int = -1, + pipesize: int = -1, + ) -> Process: ... + +else: # >= 3.9 + async def create_subprocess_shell( + cmd: str | bytes, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + loop: events.AbstractEventLoop | None = None, + limit: int = 65536, + *, + # These parameters are forced to these values by BaseEventLoop.subprocess_shell + universal_newlines: Literal[False] = False, + shell: Literal[True] = True, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + # These parameters are taken by subprocess.Popen, which this ultimately delegates to + executable: StrOrBytesPath | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + cwd: StrOrBytesPath | None = None, + env: subprocess._ENV | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + group: None | str | int = None, + extra_groups: None | Collection[str | int] = None, + user: None | str | int = None, + umask: int = -1, + ) -> Process: ... + async def create_subprocess_exec( + program: StrOrBytesPath, + *args: StrOrBytesPath, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + loop: events.AbstractEventLoop | None = None, + limit: int = 65536, + # These parameters are forced to these values by BaseEventLoop.subprocess_exec + universal_newlines: Literal[False] = False, + shell: Literal[False] = False, + bufsize: Literal[0] = 0, + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + # These parameters are taken by subprocess.Popen, which this ultimately delegates to + executable: StrOrBytesPath | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + cwd: StrOrBytesPath | None = None, + env: subprocess._ENV | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + group: None | str | int = None, + extra_groups: None | Collection[str | int] = None, + user: None | str | int = None, + umask: int = -1, + ) -> Process: ... diff --git a/mypy/typeshed/stdlib/asyncio/taskgroups.pyi b/mypy/typeshed/stdlib/asyncio/taskgroups.pyi new file mode 100644 index 000000000000..aec3f1127f15 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/taskgroups.pyi @@ -0,0 +1,25 @@ +import sys +from contextvars import Context +from types import TracebackType +from typing import Any, TypeVar +from typing_extensions import Self + +from . import _CoroutineLike +from .events import AbstractEventLoop +from .tasks import Task + +if sys.version_info >= (3, 12): + __all__ = ("TaskGroup",) +else: + __all__ = ["TaskGroup"] + +_T = TypeVar("_T") + +class TaskGroup: + _loop: AbstractEventLoop | None + _tasks: set[Task[Any]] + + async def __aenter__(self) -> Self: ... + async def __aexit__(self, et: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None) -> None: ... + def create_task(self, coro: _CoroutineLike[_T], *, name: str | None = None, context: Context | None = None) -> Task[_T]: ... + def _on_task_done(self, task: Task[object]) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi new file mode 100644 index 000000000000..d1ff7d425ba4 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -0,0 +1,459 @@ +import concurrent.futures +import sys +from _asyncio import ( + Task as Task, + _enter_task as _enter_task, + _leave_task as _leave_task, + _register_task as _register_task, + _unregister_task as _unregister_task, +) +from collections.abc import AsyncIterator, Awaitable, Coroutine, Generator, Iterable, Iterator +from typing import Any, Literal, Protocol, TypeVar, overload +from typing_extensions import TypeAlias + +from . import _CoroutineLike +from .events import AbstractEventLoop +from .futures import Future + +if sys.version_info >= (3, 11): + from contextvars import Context + +if sys.version_info >= (3, 12): + __all__ = ( + "Task", + "create_task", + "FIRST_COMPLETED", + "FIRST_EXCEPTION", + "ALL_COMPLETED", + "wait", + "wait_for", + "as_completed", + "sleep", + "gather", + "shield", + "ensure_future", + "run_coroutine_threadsafe", + "current_task", + "all_tasks", + "create_eager_task_factory", + "eager_task_factory", + "_register_task", + "_unregister_task", + "_enter_task", + "_leave_task", + ) +else: + __all__ = ( + "Task", + "create_task", + "FIRST_COMPLETED", + "FIRST_EXCEPTION", + "ALL_COMPLETED", + "wait", + "wait_for", + "as_completed", + "sleep", + "gather", + "shield", + "ensure_future", + "run_coroutine_threadsafe", + "current_task", + "all_tasks", + "_register_task", + "_unregister_task", + "_enter_task", + "_leave_task", + ) + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_T3 = TypeVar("_T3") +_T4 = TypeVar("_T4") +_T5 = TypeVar("_T5") +_T6 = TypeVar("_T6") +_FT = TypeVar("_FT", bound=Future[Any]) +if sys.version_info >= (3, 12): + _FutureLike: TypeAlias = Future[_T] | Awaitable[_T] +else: + _FutureLike: TypeAlias = Future[_T] | Generator[Any, None, _T] | Awaitable[_T] +_TaskYieldType: TypeAlias = Future[object] | None + +FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED +FIRST_EXCEPTION = concurrent.futures.FIRST_EXCEPTION +ALL_COMPLETED = concurrent.futures.ALL_COMPLETED + +if sys.version_info >= (3, 13): + class _SyncAndAsyncIterator(Iterator[_T_co], AsyncIterator[_T_co], Protocol[_T_co]): ... + + def as_completed(fs: Iterable[_FutureLike[_T]], *, timeout: float | None = None) -> _SyncAndAsyncIterator[Future[_T]]: ... + +elif sys.version_info >= (3, 10): + def as_completed(fs: Iterable[_FutureLike[_T]], *, timeout: float | None = None) -> Iterator[Future[_T]]: ... + +else: + def as_completed( + fs: Iterable[_FutureLike[_T]], *, loop: AbstractEventLoop | None = None, timeout: float | None = None + ) -> Iterator[Future[_T]]: ... + +@overload +def ensure_future(coro_or_future: _FT, *, loop: AbstractEventLoop | None = None) -> _FT: ... # type: ignore[overload-overlap] +@overload +def ensure_future(coro_or_future: Awaitable[_T], *, loop: AbstractEventLoop | None = None) -> Task[_T]: ... + +# `gather()` actually returns a list with length equal to the number +# of tasks passed; however, Tuple is used similar to the annotation for +# zip() because typing does not support variadic type variables. See +# typing PR #1550 for discussion. +# +# N.B. Having overlapping overloads is the only way to get acceptable type inference in all edge cases. +if sys.version_info >= (3, 10): + @overload + def gather(coro_or_future1: _FutureLike[_T1], /, *, return_exceptions: Literal[False] = False) -> Future[tuple[_T1]]: ... # type: ignore[overload-overlap] + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], /, *, return_exceptions: Literal[False] = False + ) -> Future[tuple[_T1, _T2]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + /, + *, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + /, + *, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3, _T4]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + coro_or_future5: _FutureLike[_T5], + /, + *, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3, _T4, _T5]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + coro_or_future5: _FutureLike[_T5], + coro_or_future6: _FutureLike[_T6], + /, + *, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3, _T4, _T5, _T6]]: ... + @overload + def gather(*coros_or_futures: _FutureLike[_T], return_exceptions: Literal[False] = False) -> Future[list[_T]]: ... # type: ignore[overload-overlap] + @overload + def gather(coro_or_future1: _FutureLike[_T1], /, *, return_exceptions: bool) -> Future[tuple[_T1 | BaseException]]: ... + @overload + def gather( + coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], /, *, return_exceptions: bool + ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException]]: ... + @overload + def gather( + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + /, + *, + return_exceptions: bool, + ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException]]: ... + @overload + def gather( + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + /, + *, + return_exceptions: bool, + ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException, _T4 | BaseException]]: ... + @overload + def gather( + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + coro_or_future5: _FutureLike[_T5], + /, + *, + return_exceptions: bool, + ) -> Future[ + tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException, _T4 | BaseException, _T5 | BaseException] + ]: ... + @overload + def gather( + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + coro_or_future5: _FutureLike[_T5], + coro_or_future6: _FutureLike[_T6], + /, + *, + return_exceptions: bool, + ) -> Future[ + tuple[ + _T1 | BaseException, + _T2 | BaseException, + _T3 | BaseException, + _T4 | BaseException, + _T5 | BaseException, + _T6 | BaseException, + ] + ]: ... + @overload + def gather(*coros_or_futures: _FutureLike[_T], return_exceptions: bool) -> Future[list[_T | BaseException]]: ... + +else: + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], /, *, loop: AbstractEventLoop | None = None, return_exceptions: Literal[False] = False + ) -> Future[tuple[_T1]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3, _T4]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + coro_or_future5: _FutureLike[_T5], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3, _T4, _T5]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + coro_or_future5: _FutureLike[_T5], + coro_or_future6: _FutureLike[_T6], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: Literal[False] = False, + ) -> Future[tuple[_T1, _T2, _T3, _T4, _T5, _T6]]: ... + @overload + def gather( # type: ignore[overload-overlap] + *coros_or_futures: _FutureLike[_T], loop: AbstractEventLoop | None = None, return_exceptions: Literal[False] = False + ) -> Future[list[_T]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], /, *, loop: AbstractEventLoop | None = None, return_exceptions: bool + ) -> Future[tuple[_T1 | BaseException]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: bool, + ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: bool, + ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: bool, + ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException, _T4 | BaseException]]: ... + @overload + def gather( # type: ignore[overload-overlap] + coro_or_future1: _FutureLike[_T1], + coro_or_future2: _FutureLike[_T2], + coro_or_future3: _FutureLike[_T3], + coro_or_future4: _FutureLike[_T4], + coro_or_future5: _FutureLike[_T5], + coro_or_future6: _FutureLike[_T6], + /, + *, + loop: AbstractEventLoop | None = None, + return_exceptions: bool, + ) -> Future[ + tuple[ + _T1 | BaseException, + _T2 | BaseException, + _T3 | BaseException, + _T4 | BaseException, + _T5 | BaseException, + _T6 | BaseException, + ] + ]: ... + @overload + def gather( + *coros_or_futures: _FutureLike[_T], loop: AbstractEventLoop | None = None, return_exceptions: bool + ) -> Future[list[_T | BaseException]]: ... + +def run_coroutine_threadsafe(coro: _FutureLike[_T], loop: AbstractEventLoop) -> concurrent.futures.Future[_T]: ... + +if sys.version_info >= (3, 10): + def shield(arg: _FutureLike[_T]) -> Future[_T]: ... + @overload + async def sleep(delay: float) -> None: ... + @overload + async def sleep(delay: float, result: _T) -> _T: ... + async def wait_for(fut: _FutureLike[_T], timeout: float | None) -> _T: ... + +else: + def shield(arg: _FutureLike[_T], *, loop: AbstractEventLoop | None = None) -> Future[_T]: ... + @overload + async def sleep(delay: float, *, loop: AbstractEventLoop | None = None) -> None: ... + @overload + async def sleep(delay: float, result: _T, *, loop: AbstractEventLoop | None = None) -> _T: ... + async def wait_for(fut: _FutureLike[_T], timeout: float | None, *, loop: AbstractEventLoop | None = None) -> _T: ... + +if sys.version_info >= (3, 11): + @overload + async def wait( + fs: Iterable[_FT], *, timeout: float | None = None, return_when: str = "ALL_COMPLETED" + ) -> tuple[set[_FT], set[_FT]]: ... + @overload + async def wait( + fs: Iterable[Task[_T]], *, timeout: float | None = None, return_when: str = "ALL_COMPLETED" + ) -> tuple[set[Task[_T]], set[Task[_T]]]: ... + +elif sys.version_info >= (3, 10): + @overload + async def wait( # type: ignore[overload-overlap] + fs: Iterable[_FT], *, timeout: float | None = None, return_when: str = "ALL_COMPLETED" + ) -> tuple[set[_FT], set[_FT]]: ... + @overload + async def wait( + fs: Iterable[Awaitable[_T]], *, timeout: float | None = None, return_when: str = "ALL_COMPLETED" + ) -> tuple[set[Task[_T]], set[Task[_T]]]: ... + +else: + @overload + async def wait( # type: ignore[overload-overlap] + fs: Iterable[_FT], + *, + loop: AbstractEventLoop | None = None, + timeout: float | None = None, + return_when: str = "ALL_COMPLETED", + ) -> tuple[set[_FT], set[_FT]]: ... + @overload + async def wait( + fs: Iterable[Awaitable[_T]], + *, + loop: AbstractEventLoop | None = None, + timeout: float | None = None, + return_when: str = "ALL_COMPLETED", + ) -> tuple[set[Task[_T]], set[Task[_T]]]: ... + +if sys.version_info >= (3, 12): + _TaskCompatibleCoro: TypeAlias = Coroutine[Any, Any, _T_co] +elif sys.version_info >= (3, 9): + _TaskCompatibleCoro: TypeAlias = Generator[_TaskYieldType, None, _T_co] | Coroutine[Any, Any, _T_co] +else: + _TaskCompatibleCoro: TypeAlias = Generator[_TaskYieldType, None, _T_co] | Awaitable[_T_co] + +def all_tasks(loop: AbstractEventLoop | None = None) -> set[Task[Any]]: ... + +if sys.version_info >= (3, 11): + def create_task(coro: _CoroutineLike[_T], *, name: str | None = None, context: Context | None = None) -> Task[_T]: ... + +else: + def create_task(coro: _CoroutineLike[_T], *, name: str | None = None) -> Task[_T]: ... + +if sys.version_info >= (3, 12): + from _asyncio import current_task as current_task +else: + def current_task(loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... + +if sys.version_info >= (3, 12): + _TaskT_co = TypeVar("_TaskT_co", bound=Task[Any], covariant=True) + + class _CustomTaskConstructor(Protocol[_TaskT_co]): + def __call__( + self, + coro: _TaskCompatibleCoro[Any], + /, + *, + loop: AbstractEventLoop, + name: str | None, + context: Context | None, + eager_start: bool, + ) -> _TaskT_co: ... + + class _EagerTaskFactoryType(Protocol[_TaskT_co]): + def __call__( + self, + loop: AbstractEventLoop, + coro: _TaskCompatibleCoro[Any], + *, + name: str | None = None, + context: Context | None = None, + ) -> _TaskT_co: ... + + def create_eager_task_factory( + custom_task_constructor: _CustomTaskConstructor[_TaskT_co], + ) -> _EagerTaskFactoryType[_TaskT_co]: ... + def eager_task_factory( + loop: AbstractEventLoop | None, + coro: _TaskCompatibleCoro[_T_co], + *, + name: str | None = None, + context: Context | None = None, + ) -> Task[_T_co]: ... diff --git a/mypy/typeshed/stdlib/asyncio/threads.pyi b/mypy/typeshed/stdlib/asyncio/threads.pyi new file mode 100644 index 000000000000..799efd25fea4 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/threads.pyi @@ -0,0 +1,9 @@ +from collections.abc import Callable +from typing import TypeVar +from typing_extensions import ParamSpec + +__all__ = ("to_thread",) +_P = ParamSpec("_P") +_R = TypeVar("_R") + +async def to_thread(func: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs) -> _R: ... diff --git a/mypy/typeshed/stdlib/asyncio/timeouts.pyi b/mypy/typeshed/stdlib/asyncio/timeouts.pyi new file mode 100644 index 000000000000..2f0e40e25680 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/timeouts.pyi @@ -0,0 +1,19 @@ +from types import TracebackType +from typing import final +from typing_extensions import Self + +__all__ = ("Timeout", "timeout", "timeout_at") + +@final +class Timeout: + def __init__(self, when: float | None) -> None: ... + def when(self) -> float | None: ... + def reschedule(self, when: float | None) -> None: ... + def expired(self) -> bool: ... + async def __aenter__(self) -> Self: ... + async def __aexit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + +def timeout(delay: float | None) -> Timeout: ... +def timeout_at(when: float | None) -> Timeout: ... diff --git a/mypy/typeshed/stdlib/asyncio/transports.pyi b/mypy/typeshed/stdlib/asyncio/transports.pyi new file mode 100644 index 000000000000..531f77672438 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/transports.pyi @@ -0,0 +1,47 @@ +from asyncio.events import AbstractEventLoop +from asyncio.protocols import BaseProtocol +from collections.abc import Iterable, Mapping +from socket import _Address +from typing import Any + +__all__ = ("BaseTransport", "ReadTransport", "WriteTransport", "Transport", "DatagramTransport", "SubprocessTransport") + +class BaseTransport: + def __init__(self, extra: Mapping[str, Any] | None = None) -> None: ... + def get_extra_info(self, name: str, default: Any = None) -> Any: ... + def is_closing(self) -> bool: ... + def close(self) -> None: ... + def set_protocol(self, protocol: BaseProtocol) -> None: ... + def get_protocol(self) -> BaseProtocol: ... + +class ReadTransport(BaseTransport): + def is_reading(self) -> bool: ... + def pause_reading(self) -> None: ... + def resume_reading(self) -> None: ... + +class WriteTransport(BaseTransport): + def set_write_buffer_limits(self, high: int | None = None, low: int | None = None) -> None: ... + def get_write_buffer_size(self) -> int: ... + def get_write_buffer_limits(self) -> tuple[int, int]: ... + def write(self, data: bytes | bytearray | memoryview) -> None: ... + def writelines(self, list_of_data: Iterable[bytes | bytearray | memoryview]) -> None: ... + def write_eof(self) -> None: ... + def can_write_eof(self) -> bool: ... + def abort(self) -> None: ... + +class Transport(ReadTransport, WriteTransport): ... + +class DatagramTransport(BaseTransport): + def sendto(self, data: bytes | bytearray | memoryview, addr: _Address | None = None) -> None: ... + def abort(self) -> None: ... + +class SubprocessTransport(BaseTransport): + def get_pid(self) -> int: ... + def get_returncode(self) -> int | None: ... + def get_pipe_transport(self, fd: int) -> BaseTransport | None: ... + def send_signal(self, signal: int) -> None: ... + def terminate(self) -> None: ... + def kill(self) -> None: ... + +class _FlowControlMixin(Transport): + def __init__(self, extra: Mapping[str, Any] | None = None, loop: AbstractEventLoop | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/trsock.pyi b/mypy/typeshed/stdlib/asyncio/trsock.pyi new file mode 100644 index 000000000000..e74cf6fd4e05 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/trsock.pyi @@ -0,0 +1,94 @@ +import socket +import sys +from _typeshed import ReadableBuffer +from builtins import type as Type # alias to avoid name clashes with property named "type" +from collections.abc import Iterable +from types import TracebackType +from typing import Any, BinaryIO, NoReturn, overload +from typing_extensions import TypeAlias + +# These are based in socket, maybe move them out into _typeshed.pyi or such +_Address: TypeAlias = socket._Address +_RetAddress: TypeAlias = Any +_WriteBuffer: TypeAlias = bytearray | memoryview +_CMSG: TypeAlias = tuple[int, int, bytes] + +class TransportSocket: + def __init__(self, sock: socket.socket) -> None: ... + @property + def family(self) -> int: ... + @property + def type(self) -> int: ... + @property + def proto(self) -> int: ... + def __getstate__(self) -> NoReturn: ... + def fileno(self) -> int: ... + def dup(self) -> socket.socket: ... + def get_inheritable(self) -> bool: ... + def shutdown(self, how: int) -> None: ... + @overload + def getsockopt(self, level: int, optname: int) -> int: ... + @overload + def getsockopt(self, level: int, optname: int, buflen: int) -> bytes: ... + @overload + def setsockopt(self, level: int, optname: int, value: int | ReadableBuffer) -> None: ... + @overload + def setsockopt(self, level: int, optname: int, value: None, optlen: int) -> None: ... + def getpeername(self) -> _RetAddress: ... + def getsockname(self) -> _RetAddress: ... + def getsockbyname(self) -> NoReturn: ... # This method doesn't exist on socket, yet is passed through? + def settimeout(self, value: float | None) -> None: ... + def gettimeout(self) -> float | None: ... + def setblocking(self, flag: bool) -> None: ... + if sys.version_info < (3, 11): + def _na(self, what: str) -> None: ... + def accept(self) -> tuple[socket.socket, _RetAddress]: ... + def connect(self, address: _Address) -> None: ... + def connect_ex(self, address: _Address) -> int: ... + def bind(self, address: _Address) -> None: ... + if sys.platform == "win32": + def ioctl(self, control: int, option: int | tuple[int, int, int] | bool) -> None: ... + else: + def ioctl(self, control: int, option: int | tuple[int, int, int] | bool) -> NoReturn: ... + + def listen(self, backlog: int = ..., /) -> None: ... + def makefile(self) -> BinaryIO: ... + def sendfile(self, file: BinaryIO, offset: int = ..., count: int | None = ...) -> int: ... + def close(self) -> None: ... + def detach(self) -> int: ... + if sys.platform == "linux": + def sendmsg_afalg( + self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... + ) -> int: ... + else: + def sendmsg_afalg( + self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... + ) -> NoReturn: ... + + def sendmsg( + self, buffers: Iterable[ReadableBuffer], ancdata: Iterable[_CMSG] = ..., flags: int = ..., address: _Address = ..., / + ) -> int: ... + @overload + def sendto(self, data: ReadableBuffer, address: _Address) -> int: ... + @overload + def sendto(self, data: ReadableBuffer, flags: int, address: _Address) -> int: ... + def send(self, data: ReadableBuffer, flags: int = ...) -> int: ... + def sendall(self, data: ReadableBuffer, flags: int = ...) -> None: ... + def set_inheritable(self, inheritable: bool) -> None: ... + if sys.platform == "win32": + def share(self, process_id: int) -> bytes: ... + else: + def share(self, process_id: int) -> NoReturn: ... + + def recv_into(self, buffer: _WriteBuffer, nbytes: int = ..., flags: int = ...) -> int: ... + def recvfrom_into(self, buffer: _WriteBuffer, nbytes: int = ..., flags: int = ...) -> tuple[int, _RetAddress]: ... + def recvmsg_into( + self, buffers: Iterable[_WriteBuffer], ancbufsize: int = ..., flags: int = ..., / + ) -> tuple[int, list[_CMSG], int, Any]: ... + def recvmsg(self, bufsize: int, ancbufsize: int = ..., flags: int = ..., /) -> tuple[bytes, list[_CMSG], int, Any]: ... + def recvfrom(self, bufsize: int, flags: int = ...) -> tuple[bytes, _RetAddress]: ... + def recv(self, bufsize: int, flags: int = ...) -> bytes: ... + def __enter__(self) -> socket.socket: ... + def __exit__( + self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi new file mode 100644 index 000000000000..fb21c5b5fa05 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -0,0 +1,251 @@ +import sys +import types +from _typeshed import StrPath +from abc import ABCMeta, abstractmethod +from collections.abc import Callable +from socket import socket +from typing import Literal +from typing_extensions import Self, TypeVarTuple, Unpack, deprecated + +from .base_events import Server, _ProtocolFactory, _SSLContext +from .events import AbstractEventLoop, BaseDefaultEventLoopPolicy +from .selector_events import BaseSelectorEventLoop + +_Ts = TypeVarTuple("_Ts") + +if sys.platform != "win32": + if sys.version_info >= (3, 14): + __all__ = ("SelectorEventLoop", "DefaultEventLoopPolicy", "EventLoop") + elif sys.version_info >= (3, 13): + __all__ = ( + "SelectorEventLoop", + "AbstractChildWatcher", + "SafeChildWatcher", + "FastChildWatcher", + "PidfdChildWatcher", + "MultiLoopChildWatcher", + "ThreadedChildWatcher", + "DefaultEventLoopPolicy", + "EventLoop", + ) + elif sys.version_info >= (3, 9): + __all__ = ( + "SelectorEventLoop", + "AbstractChildWatcher", + "SafeChildWatcher", + "FastChildWatcher", + "PidfdChildWatcher", + "MultiLoopChildWatcher", + "ThreadedChildWatcher", + "DefaultEventLoopPolicy", + ) + else: + __all__ = ( + "SelectorEventLoop", + "AbstractChildWatcher", + "SafeChildWatcher", + "FastChildWatcher", + "MultiLoopChildWatcher", + "ThreadedChildWatcher", + "DefaultEventLoopPolicy", + ) + +# This is also technically not available on Win, +# but other parts of typeshed need this definition. +# So, it is special cased. +if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class AbstractChildWatcher: + @abstractmethod + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + @abstractmethod + def remove_child_handler(self, pid: int) -> bool: ... + @abstractmethod + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def close(self) -> None: ... + @abstractmethod + def __enter__(self) -> Self: ... + @abstractmethod + def __exit__( + self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None + ) -> None: ... + @abstractmethod + def is_active(self) -> bool: ... + + else: + class AbstractChildWatcher: + @abstractmethod + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + @abstractmethod + def remove_child_handler(self, pid: int) -> bool: ... + @abstractmethod + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def close(self) -> None: ... + @abstractmethod + def __enter__(self) -> Self: ... + @abstractmethod + def __exit__( + self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None + ) -> None: ... + @abstractmethod + def is_active(self) -> bool: ... + +if sys.platform != "win32": + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. + # See discussion in #7412 + class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): + def close(self) -> None: ... + def is_active(self) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class SafeChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class FastChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + + else: + # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. + # See discussion in #7412 + class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): + def close(self) -> None: ... + def is_active(self) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + class SafeChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + + class FastChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + + class _UnixSelectorEventLoop(BaseSelectorEventLoop): + if sys.version_info >= (3, 13): + async def create_unix_server( + self, + protocol_factory: _ProtocolFactory, + path: StrPath | None = None, + *, + sock: socket | None = None, + backlog: int = 100, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + cleanup_socket: bool = True, + ) -> Server: ... + + class _UnixDefaultEventLoopPolicy(BaseDefaultEventLoopPolicy): + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher(self) -> AbstractChildWatcher: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... + else: + def get_child_watcher(self) -> AbstractChildWatcher: ... + def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... + + SelectorEventLoop = _UnixSelectorEventLoop + + DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy + + if sys.version_info >= (3, 13): + EventLoop = SelectorEventLoop + + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class MultiLoopChildWatcher(AbstractChildWatcher): + def is_active(self) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + else: + class MultiLoopChildWatcher(AbstractChildWatcher): + def is_active(self) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + if sys.version_info < (3, 14): + class ThreadedChildWatcher(AbstractChildWatcher): + def is_active(self) -> Literal[True]: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def __del__(self) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + if sys.version_info >= (3, 9): + class PidfdChildWatcher(AbstractChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def is_active(self) -> bool: ... + def close(self) -> None: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/asyncio/windows_events.pyi b/mypy/typeshed/stdlib/asyncio/windows_events.pyi new file mode 100644 index 000000000000..e5205ba4dcb0 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/windows_events.pyi @@ -0,0 +1,100 @@ +import socket +import sys +from _typeshed import Incomplete, ReadableBuffer, WriteableBuffer +from collections.abc import Callable +from typing import IO, Any, ClassVar, Final, NoReturn + +from . import events, futures, proactor_events, selector_events, streams, windows_utils + +if sys.platform == "win32": + if sys.version_info >= (3, 13): + # 3.13 added `EventLoop`. + __all__ = ( + "SelectorEventLoop", + "ProactorEventLoop", + "IocpProactor", + "DefaultEventLoopPolicy", + "WindowsSelectorEventLoopPolicy", + "WindowsProactorEventLoopPolicy", + "EventLoop", + ) + else: + __all__ = ( + "SelectorEventLoop", + "ProactorEventLoop", + "IocpProactor", + "DefaultEventLoopPolicy", + "WindowsSelectorEventLoopPolicy", + "WindowsProactorEventLoopPolicy", + ) + + NULL: Final = 0 + INFINITE: Final = 0xFFFFFFFF + ERROR_CONNECTION_REFUSED: Final = 1225 + ERROR_CONNECTION_ABORTED: Final = 1236 + CONNECT_PIPE_INIT_DELAY: float + CONNECT_PIPE_MAX_DELAY: float + + class PipeServer: + def __init__(self, address: str) -> None: ... + def __del__(self) -> None: ... + def closed(self) -> bool: ... + def close(self) -> None: ... + + class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop): ... + + class ProactorEventLoop(proactor_events.BaseProactorEventLoop): + def __init__(self, proactor: IocpProactor | None = None) -> None: ... + async def create_pipe_connection( + self, protocol_factory: Callable[[], streams.StreamReaderProtocol], address: str + ) -> tuple[proactor_events._ProactorDuplexPipeTransport, streams.StreamReaderProtocol]: ... + async def start_serving_pipe( + self, protocol_factory: Callable[[], streams.StreamReaderProtocol], address: str + ) -> list[PipeServer]: ... + + class IocpProactor: + def __init__(self, concurrency: int = 0xFFFFFFFF) -> None: ... + def __del__(self) -> None: ... + def set_loop(self, loop: events.AbstractEventLoop) -> None: ... + def select(self, timeout: int | None = None) -> list[futures.Future[Any]]: ... + def recv(self, conn: socket.socket, nbytes: int, flags: int = 0) -> futures.Future[bytes]: ... + def recv_into(self, conn: socket.socket, buf: WriteableBuffer, flags: int = 0) -> futures.Future[Any]: ... + def recvfrom( + self, conn: socket.socket, nbytes: int, flags: int = 0 + ) -> futures.Future[tuple[bytes, socket._RetAddress]]: ... + def sendto( + self, conn: socket.socket, buf: ReadableBuffer, flags: int = 0, addr: socket._Address | None = None + ) -> futures.Future[int]: ... + def send(self, conn: socket.socket, buf: WriteableBuffer, flags: int = 0) -> futures.Future[Any]: ... + def accept(self, listener: socket.socket) -> futures.Future[Any]: ... + def connect( + self, + conn: socket.socket, + address: tuple[Incomplete, Incomplete] | tuple[Incomplete, Incomplete, Incomplete, Incomplete], + ) -> futures.Future[Any]: ... + def sendfile(self, sock: socket.socket, file: IO[bytes], offset: int, count: int) -> futures.Future[Any]: ... + def accept_pipe(self, pipe: socket.socket) -> futures.Future[Any]: ... + async def connect_pipe(self, address: str) -> windows_utils.PipeHandle: ... + def wait_for_handle(self, handle: windows_utils.PipeHandle, timeout: int | None = None) -> bool: ... + def close(self) -> None: ... + if sys.version_info >= (3, 11): + def recvfrom_into( + self, conn: socket.socket, buf: WriteableBuffer, flags: int = 0 + ) -> futures.Future[tuple[int, socket._RetAddress]]: ... + + SelectorEventLoop = _WindowsSelectorEventLoop + + class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): + _loop_factory: ClassVar[type[SelectorEventLoop]] + if sys.version_info < (3, 14): + def get_child_watcher(self) -> NoReturn: ... + def set_child_watcher(self, watcher: Any) -> NoReturn: ... + + class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): + _loop_factory: ClassVar[type[ProactorEventLoop]] + def get_child_watcher(self) -> NoReturn: ... + def set_child_watcher(self, watcher: Any) -> NoReturn: ... + + DefaultEventLoopPolicy = WindowsSelectorEventLoopPolicy + if sys.version_info >= (3, 13): + EventLoop = ProactorEventLoop diff --git a/mypy/typeshed/stdlib/asyncio/windows_utils.pyi b/mypy/typeshed/stdlib/asyncio/windows_utils.pyi new file mode 100644 index 000000000000..4fa014532376 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/windows_utils.pyi @@ -0,0 +1,49 @@ +import subprocess +import sys +from collections.abc import Callable +from types import TracebackType +from typing import Any, AnyStr, Final +from typing_extensions import Self + +if sys.platform == "win32": + __all__ = ("pipe", "Popen", "PIPE", "PipeHandle") + + BUFSIZE: Final = 8192 + PIPE = subprocess.PIPE + STDOUT = subprocess.STDOUT + def pipe(*, duplex: bool = False, overlapped: tuple[bool, bool] = (True, True), bufsize: int = 8192) -> tuple[int, int]: ... + + class PipeHandle: + def __init__(self, handle: int) -> None: ... + def __del__(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... + @property + def handle(self) -> int: ... + def fileno(self) -> int: ... + def close(self, *, CloseHandle: Callable[[int], object] = ...) -> None: ... + + class Popen(subprocess.Popen[AnyStr]): + stdin: PipeHandle | None # type: ignore[assignment] + stdout: PipeHandle | None # type: ignore[assignment] + stderr: PipeHandle | None # type: ignore[assignment] + # For simplicity we omit the full overloaded __new__ signature of + # subprocess.Popen. The arguments are mostly the same, but + # subprocess.Popen takes other positional-or-keyword arguments before + # stdin. + def __new__( + cls, + args: subprocess._CMD, + stdin: subprocess._FILE | None = ..., + stdout: subprocess._FILE | None = ..., + stderr: subprocess._FILE | None = ..., + **kwds: Any, + ) -> Self: ... + def __init__( + self, + args: subprocess._CMD, + stdin: subprocess._FILE | None = None, + stdout: subprocess._FILE | None = None, + stderr: subprocess._FILE | None = None, + **kwds: Any, + ) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncore.pyi b/mypy/typeshed/stdlib/asyncore.pyi new file mode 100644 index 000000000000..36d1862fdda7 --- /dev/null +++ b/mypy/typeshed/stdlib/asyncore.pyi @@ -0,0 +1,90 @@ +import sys +from _typeshed import FileDescriptorLike, ReadableBuffer +from socket import socket +from typing import Any, overload +from typing_extensions import TypeAlias + +# cyclic dependence with asynchat +_MapType: TypeAlias = dict[int, Any] +_Socket: TypeAlias = socket + +socket_map: _MapType # undocumented + +class ExitNow(Exception): ... + +def read(obj: Any) -> None: ... +def write(obj: Any) -> None: ... +def readwrite(obj: Any, flags: int) -> None: ... +def poll(timeout: float = 0.0, map: _MapType | None = None) -> None: ... +def poll2(timeout: float = 0.0, map: _MapType | None = None) -> None: ... + +poll3 = poll2 + +def loop(timeout: float = 30.0, use_poll: bool = False, map: _MapType | None = None, count: int | None = None) -> None: ... + +# Not really subclass of socket.socket; it's only delegation. +# It is not covariant to it. +class dispatcher: + debug: bool + connected: bool + accepting: bool + connecting: bool + closing: bool + ignore_log_types: frozenset[str] + socket: _Socket | None + def __init__(self, sock: _Socket | None = None, map: _MapType | None = None) -> None: ... + def add_channel(self, map: _MapType | None = None) -> None: ... + def del_channel(self, map: _MapType | None = None) -> None: ... + def create_socket(self, family: int = ..., type: int = ...) -> None: ... + def set_socket(self, sock: _Socket, map: _MapType | None = None) -> None: ... + def set_reuse_addr(self) -> None: ... + def readable(self) -> bool: ... + def writable(self) -> bool: ... + def listen(self, num: int) -> None: ... + def bind(self, addr: tuple[Any, ...] | str) -> None: ... + def connect(self, address: tuple[Any, ...] | str) -> None: ... + def accept(self) -> tuple[_Socket, Any] | None: ... + def send(self, data: ReadableBuffer) -> int: ... + def recv(self, buffer_size: int) -> bytes: ... + def close(self) -> None: ... + def log(self, message: Any) -> None: ... + def log_info(self, message: Any, type: str = "info") -> None: ... + def handle_read_event(self) -> None: ... + def handle_connect_event(self) -> None: ... + def handle_write_event(self) -> None: ... + def handle_expt_event(self) -> None: ... + def handle_error(self) -> None: ... + def handle_expt(self) -> None: ... + def handle_read(self) -> None: ... + def handle_write(self) -> None: ... + def handle_connect(self) -> None: ... + def handle_accept(self) -> None: ... + def handle_close(self) -> None: ... + +class dispatcher_with_send(dispatcher): + def initiate_send(self) -> None: ... + # incompatible signature: + # def send(self, data: bytes) -> int | None: ... + +def compact_traceback() -> tuple[tuple[str, str, str], type, type, str]: ... +def close_all(map: _MapType | None = None, ignore_all: bool = False) -> None: ... + +if sys.platform != "win32": + class file_wrapper: + fd: int + def __init__(self, fd: int) -> None: ... + def recv(self, bufsize: int, flags: int = ...) -> bytes: ... + def send(self, data: bytes, flags: int = ...) -> int: ... + @overload + def getsockopt(self, level: int, optname: int, buflen: None = None) -> int: ... + @overload + def getsockopt(self, level: int, optname: int, buflen: int) -> bytes: ... + def read(self, bufsize: int, flags: int = ...) -> bytes: ... + def write(self, data: bytes, flags: int = ...) -> int: ... + def close(self) -> None: ... + def fileno(self) -> int: ... + def __del__(self) -> None: ... + + class file_dispatcher(dispatcher): + def __init__(self, fd: FileDescriptorLike, map: _MapType | None = None) -> None: ... + def set_file(self, fd: int) -> None: ... diff --git a/mypy/typeshed/stdlib/atexit.pyi b/mypy/typeshed/stdlib/atexit.pyi new file mode 100644 index 000000000000..7f7b05ccc0a3 --- /dev/null +++ b/mypy/typeshed/stdlib/atexit.pyi @@ -0,0 +1,12 @@ +from collections.abc import Callable +from typing import TypeVar +from typing_extensions import ParamSpec + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +def _clear() -> None: ... +def _ncallbacks() -> int: ... +def _run_exitfuncs() -> None: ... +def register(func: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> Callable[_P, _T]: ... +def unregister(func: Callable[..., object], /) -> None: ... diff --git a/mypy/typeshed/stdlib/audioop.pyi b/mypy/typeshed/stdlib/audioop.pyi new file mode 100644 index 000000000000..f3ce78ccb7fa --- /dev/null +++ b/mypy/typeshed/stdlib/audioop.pyi @@ -0,0 +1,43 @@ +from typing_extensions import Buffer, TypeAlias + +_AdpcmState: TypeAlias = tuple[int, int] +_RatecvState: TypeAlias = tuple[int, tuple[tuple[int, int], ...]] + +class error(Exception): ... + +def add(fragment1: Buffer, fragment2: Buffer, width: int, /) -> bytes: ... +def adpcm2lin(fragment: Buffer, width: int, state: _AdpcmState | None, /) -> tuple[bytes, _AdpcmState]: ... +def alaw2lin(fragment: Buffer, width: int, /) -> bytes: ... +def avg(fragment: Buffer, width: int, /) -> int: ... +def avgpp(fragment: Buffer, width: int, /) -> int: ... +def bias(fragment: Buffer, width: int, bias: int, /) -> bytes: ... +def byteswap(fragment: Buffer, width: int, /) -> bytes: ... +def cross(fragment: Buffer, width: int, /) -> int: ... +def findfactor(fragment: Buffer, reference: Buffer, /) -> float: ... +def findfit(fragment: Buffer, reference: Buffer, /) -> tuple[int, float]: ... +def findmax(fragment: Buffer, length: int, /) -> int: ... +def getsample(fragment: Buffer, width: int, index: int, /) -> int: ... +def lin2adpcm(fragment: Buffer, width: int, state: _AdpcmState | None, /) -> tuple[bytes, _AdpcmState]: ... +def lin2alaw(fragment: Buffer, width: int, /) -> bytes: ... +def lin2lin(fragment: Buffer, width: int, newwidth: int, /) -> bytes: ... +def lin2ulaw(fragment: Buffer, width: int, /) -> bytes: ... +def max(fragment: Buffer, width: int, /) -> int: ... +def maxpp(fragment: Buffer, width: int, /) -> int: ... +def minmax(fragment: Buffer, width: int, /) -> tuple[int, int]: ... +def mul(fragment: Buffer, width: int, factor: float, /) -> bytes: ... +def ratecv( + fragment: Buffer, + width: int, + nchannels: int, + inrate: int, + outrate: int, + state: _RatecvState | None, + weightA: int = 1, + weightB: int = 0, + /, +) -> tuple[bytes, _RatecvState]: ... +def reverse(fragment: Buffer, width: int, /) -> bytes: ... +def rms(fragment: Buffer, width: int, /) -> int: ... +def tomono(fragment: Buffer, width: int, lfactor: float, rfactor: float, /) -> bytes: ... +def tostereo(fragment: Buffer, width: int, lfactor: float, rfactor: float, /) -> bytes: ... +def ulaw2lin(fragment: Buffer, width: int, /) -> bytes: ... diff --git a/mypy/typeshed/stdlib/base64.pyi b/mypy/typeshed/stdlib/base64.pyi new file mode 100644 index 000000000000..8be4cfe69de0 --- /dev/null +++ b/mypy/typeshed/stdlib/base64.pyi @@ -0,0 +1,65 @@ +import sys +from _typeshed import ReadableBuffer +from typing import IO + +__all__ = [ + "encode", + "decode", + "encodebytes", + "decodebytes", + "b64encode", + "b64decode", + "b32encode", + "b32decode", + "b16encode", + "b16decode", + "b85encode", + "b85decode", + "a85encode", + "a85decode", + "standard_b64encode", + "standard_b64decode", + "urlsafe_b64encode", + "urlsafe_b64decode", +] + +if sys.version_info >= (3, 10): + __all__ += ["b32hexencode", "b32hexdecode"] +if sys.version_info >= (3, 13): + __all__ += ["z85decode", "z85encode"] + +def b64encode(s: ReadableBuffer, altchars: ReadableBuffer | None = None) -> bytes: ... +def b64decode(s: str | ReadableBuffer, altchars: str | ReadableBuffer | None = None, validate: bool = False) -> bytes: ... +def standard_b64encode(s: ReadableBuffer) -> bytes: ... +def standard_b64decode(s: str | ReadableBuffer) -> bytes: ... +def urlsafe_b64encode(s: ReadableBuffer) -> bytes: ... +def urlsafe_b64decode(s: str | ReadableBuffer) -> bytes: ... +def b32encode(s: ReadableBuffer) -> bytes: ... +def b32decode(s: str | ReadableBuffer, casefold: bool = False, map01: str | ReadableBuffer | None = None) -> bytes: ... +def b16encode(s: ReadableBuffer) -> bytes: ... +def b16decode(s: str | ReadableBuffer, casefold: bool = False) -> bytes: ... + +if sys.version_info >= (3, 10): + def b32hexencode(s: ReadableBuffer) -> bytes: ... + def b32hexdecode(s: str | ReadableBuffer, casefold: bool = False) -> bytes: ... + +def a85encode( + b: ReadableBuffer, *, foldspaces: bool = False, wrapcol: int = 0, pad: bool = False, adobe: bool = False +) -> bytes: ... +def a85decode( + b: str | ReadableBuffer, *, foldspaces: bool = False, adobe: bool = False, ignorechars: bytearray | bytes = b" \t\n\r\x0b" +) -> bytes: ... +def b85encode(b: ReadableBuffer, pad: bool = False) -> bytes: ... +def b85decode(b: str | ReadableBuffer) -> bytes: ... +def decode(input: IO[bytes], output: IO[bytes]) -> None: ... +def encode(input: IO[bytes], output: IO[bytes]) -> None: ... +def encodebytes(s: ReadableBuffer) -> bytes: ... +def decodebytes(s: ReadableBuffer) -> bytes: ... + +if sys.version_info < (3, 9): + def encodestring(s: ReadableBuffer) -> bytes: ... + def decodestring(s: ReadableBuffer) -> bytes: ... + +if sys.version_info >= (3, 13): + def z85encode(s: ReadableBuffer) -> bytes: ... + def z85decode(s: str | ReadableBuffer) -> bytes: ... diff --git a/mypy/typeshed/stdlib/bdb.pyi b/mypy/typeshed/stdlib/bdb.pyi new file mode 100644 index 000000000000..75bfa91cc379 --- /dev/null +++ b/mypy/typeshed/stdlib/bdb.pyi @@ -0,0 +1,114 @@ +import sys +from _typeshed import ExcInfo, TraceFunction, Unused +from collections.abc import Callable, Iterable, Mapping +from types import CodeType, FrameType, TracebackType +from typing import IO, Any, Final, SupportsInt, TypeVar +from typing_extensions import ParamSpec + +__all__ = ["BdbQuit", "Bdb", "Breakpoint"] + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +# A union of code-object flags at runtime. +# The exact values of code-object flags are implementation details, +# so we don't include the value of this constant in the stubs. +GENERATOR_AND_COROUTINE_FLAGS: Final[int] + +class BdbQuit(Exception): ... + +class Bdb: + skip: set[str] | None + breaks: dict[str, list[int]] + fncache: dict[str, str] + frame_returning: FrameType | None + botframe: FrameType | None + quitting: bool + stopframe: FrameType | None + returnframe: FrameType | None + stoplineno: int + def __init__(self, skip: Iterable[str] | None = None) -> None: ... + def canonic(self, filename: str) -> str: ... + def reset(self) -> None: ... + def trace_dispatch(self, frame: FrameType, event: str, arg: Any) -> TraceFunction: ... + def dispatch_line(self, frame: FrameType) -> TraceFunction: ... + def dispatch_call(self, frame: FrameType, arg: None) -> TraceFunction: ... + def dispatch_return(self, frame: FrameType, arg: Any) -> TraceFunction: ... + def dispatch_exception(self, frame: FrameType, arg: ExcInfo) -> TraceFunction: ... + if sys.version_info >= (3, 13): + def dispatch_opcode(self, frame: FrameType, arg: Unused) -> Callable[[FrameType, str, Any], TraceFunction]: ... + + def is_skipped_module(self, module_name: str) -> bool: ... + def stop_here(self, frame: FrameType) -> bool: ... + def break_here(self, frame: FrameType) -> bool: ... + def do_clear(self, arg: Any) -> bool | None: ... + def break_anywhere(self, frame: FrameType) -> bool: ... + def user_call(self, frame: FrameType, argument_list: None) -> None: ... + def user_line(self, frame: FrameType) -> None: ... + def user_return(self, frame: FrameType, return_value: Any) -> None: ... + def user_exception(self, frame: FrameType, exc_info: ExcInfo) -> None: ... + def set_until(self, frame: FrameType, lineno: int | None = None) -> None: ... + if sys.version_info >= (3, 13): + def user_opcode(self, frame: FrameType) -> None: ... # undocumented + + def set_step(self) -> None: ... + if sys.version_info >= (3, 13): + def set_stepinstr(self) -> None: ... # undocumented + + def set_next(self, frame: FrameType) -> None: ... + def set_return(self, frame: FrameType) -> None: ... + def set_trace(self, frame: FrameType | None = None) -> None: ... + def set_continue(self) -> None: ... + def set_quit(self) -> None: ... + def set_break( + self, filename: str, lineno: int, temporary: bool = False, cond: str | None = None, funcname: str | None = None + ) -> str | None: ... + def clear_break(self, filename: str, lineno: int) -> str | None: ... + def clear_bpbynumber(self, arg: SupportsInt) -> str | None: ... + def clear_all_file_breaks(self, filename: str) -> str | None: ... + def clear_all_breaks(self) -> str | None: ... + def get_bpbynumber(self, arg: SupportsInt) -> Breakpoint: ... + def get_break(self, filename: str, lineno: int) -> bool: ... + def get_breaks(self, filename: str, lineno: int) -> list[Breakpoint]: ... + def get_file_breaks(self, filename: str) -> list[Breakpoint]: ... + def get_all_breaks(self) -> list[Breakpoint]: ... + def get_stack(self, f: FrameType | None, t: TracebackType | None) -> tuple[list[tuple[FrameType, int]], int]: ... + def format_stack_entry(self, frame_lineno: int, lprefix: str = ": ") -> str: ... + def run( + self, cmd: str | CodeType, globals: dict[str, Any] | None = None, locals: Mapping[str, Any] | None = None + ) -> None: ... + def runeval(self, expr: str, globals: dict[str, Any] | None = None, locals: Mapping[str, Any] | None = None) -> None: ... + def runctx(self, cmd: str | CodeType, globals: dict[str, Any] | None, locals: Mapping[str, Any] | None) -> None: ... + def runcall(self, func: Callable[_P, _T], /, *args: _P.args, **kwds: _P.kwargs) -> _T | None: ... + +class Breakpoint: + next: int + bplist: dict[tuple[str, int], list[Breakpoint]] + bpbynumber: list[Breakpoint | None] + + funcname: str | None + func_first_executable_line: int | None + file: str + line: int + temporary: bool + cond: str | None + enabled: bool + ignore: int + hits: int + number: int + def __init__( + self, file: str, line: int, temporary: bool = False, cond: str | None = None, funcname: str | None = None + ) -> None: ... + if sys.version_info >= (3, 11): + @staticmethod + def clearBreakpoints() -> None: ... + + def deleteMe(self) -> None: ... + def enable(self) -> None: ... + def disable(self) -> None: ... + def bpprint(self, out: IO[str] | None = None) -> None: ... + def bpformat(self) -> str: ... + +def checkfuncname(b: Breakpoint, frame: FrameType) -> bool: ... +def effective(file: str, line: int, frame: FrameType) -> tuple[Breakpoint, bool] | tuple[None, None]: ... +def set_trace() -> None: ... diff --git a/mypy/typeshed/stdlib/binascii.pyi b/mypy/typeshed/stdlib/binascii.pyi new file mode 100644 index 000000000000..32e018c653cb --- /dev/null +++ b/mypy/typeshed/stdlib/binascii.pyi @@ -0,0 +1,36 @@ +import sys +from _typeshed import ReadableBuffer +from typing_extensions import TypeAlias + +# Many functions in binascii accept buffer objects +# or ASCII-only strings. +_AsciiBuffer: TypeAlias = str | ReadableBuffer + +def a2b_uu(data: _AsciiBuffer, /) -> bytes: ... +def b2a_uu(data: ReadableBuffer, /, *, backtick: bool = False) -> bytes: ... + +if sys.version_info >= (3, 11): + def a2b_base64(data: _AsciiBuffer, /, *, strict_mode: bool = False) -> bytes: ... + +else: + def a2b_base64(data: _AsciiBuffer, /) -> bytes: ... + +def b2a_base64(data: ReadableBuffer, /, *, newline: bool = True) -> bytes: ... +def a2b_qp(data: _AsciiBuffer, header: bool = False) -> bytes: ... +def b2a_qp(data: ReadableBuffer, quotetabs: bool = False, istext: bool = True, header: bool = False) -> bytes: ... + +if sys.version_info < (3, 11): + def a2b_hqx(data: _AsciiBuffer, /) -> bytes: ... + def rledecode_hqx(data: ReadableBuffer, /) -> bytes: ... + def rlecode_hqx(data: ReadableBuffer, /) -> bytes: ... + def b2a_hqx(data: ReadableBuffer, /) -> bytes: ... + +def crc_hqx(data: ReadableBuffer, crc: int, /) -> int: ... +def crc32(data: ReadableBuffer, crc: int = 0, /) -> int: ... +def b2a_hex(data: ReadableBuffer, sep: str | bytes = ..., bytes_per_sep: int = ...) -> bytes: ... +def hexlify(data: ReadableBuffer, sep: str | bytes = ..., bytes_per_sep: int = ...) -> bytes: ... +def a2b_hex(hexstr: _AsciiBuffer, /) -> bytes: ... +def unhexlify(hexstr: _AsciiBuffer, /) -> bytes: ... + +class Error(ValueError): ... +class Incomplete(Exception): ... diff --git a/mypy/typeshed/stdlib/binhex.pyi b/mypy/typeshed/stdlib/binhex.pyi new file mode 100644 index 000000000000..bdead928468f --- /dev/null +++ b/mypy/typeshed/stdlib/binhex.pyi @@ -0,0 +1,45 @@ +from _typeshed import SizedBuffer +from typing import IO, Any, Final +from typing_extensions import TypeAlias + +__all__ = ["binhex", "hexbin", "Error"] + +class Error(Exception): ... + +REASONABLY_LARGE: Final = 32768 +LINELEN: Final = 64 +RUNCHAR: Final = b"\x90" + +class FInfo: + Type: str + Creator: str + Flags: int + +_FileInfoTuple: TypeAlias = tuple[str, FInfo, int, int] +_FileHandleUnion: TypeAlias = str | IO[bytes] + +def getfileinfo(name: str) -> _FileInfoTuple: ... + +class openrsrc: + def __init__(self, *args: Any) -> None: ... + def read(self, *args: Any) -> bytes: ... + def write(self, *args: Any) -> None: ... + def close(self) -> None: ... + +class BinHex: + def __init__(self, name_finfo_dlen_rlen: _FileInfoTuple, ofp: _FileHandleUnion) -> None: ... + def write(self, data: SizedBuffer) -> None: ... + def close_data(self) -> None: ... + def write_rsrc(self, data: SizedBuffer) -> None: ... + def close(self) -> None: ... + +def binhex(inp: str, out: str) -> None: ... + +class HexBin: + def __init__(self, ifp: _FileHandleUnion) -> None: ... + def read(self, *n: int) -> bytes: ... + def close_data(self) -> None: ... + def read_rsrc(self, *n: int) -> bytes: ... + def close(self) -> None: ... + +def hexbin(inp: str, out: str) -> None: ... diff --git a/mypy/typeshed/stdlib/bisect.pyi b/mypy/typeshed/stdlib/bisect.pyi new file mode 100644 index 000000000000..60dfc48d69bd --- /dev/null +++ b/mypy/typeshed/stdlib/bisect.pyi @@ -0,0 +1,4 @@ +from _bisect import * + +bisect = bisect_right +insort = insort_right diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi new file mode 100644 index 000000000000..1a4ca925168a --- /dev/null +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -0,0 +1,2035 @@ +# ruff: noqa: PYI036 # This is the module declaring BaseException +import _ast +import _typeshed +import sys +import types +from _collections_abc import dict_items, dict_keys, dict_values +from _typeshed import ( + AnyStr_co, + ConvertibleToFloat, + ConvertibleToInt, + FileDescriptorOrPath, + MaybeNone, + OpenBinaryMode, + OpenBinaryModeReading, + OpenBinaryModeUpdating, + OpenBinaryModeWriting, + OpenTextMode, + ReadableBuffer, + SupportsAdd, + SupportsAiter, + SupportsAnext, + SupportsDivMod, + SupportsFlush, + SupportsIter, + SupportsKeysAndGetItem, + SupportsLenAndGetItem, + SupportsNext, + SupportsRAdd, + SupportsRDivMod, + SupportsRichComparison, + SupportsRichComparisonT, + SupportsWrite, +) +from collections.abc import Awaitable, Callable, Iterable, Iterator, MutableSet, Reversible, Set as AbstractSet, Sized +from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper +from types import CellType, CodeType, TracebackType + +# mypy crashes if any of {ByteString, Sequence, MutableSequence, Mapping, MutableMapping} +# are imported from collections.abc in builtins.pyi +from typing import ( # noqa: Y022 + IO, + Any, + BinaryIO, + ClassVar, + Generic, + Mapping, + MutableMapping, + MutableSequence, + NoReturn, + Protocol, + Sequence, + SupportsAbs, + SupportsBytes, + SupportsComplex, + SupportsFloat, + SupportsIndex, + TypeVar, + final, + overload, + type_check_only, +) + +# we can't import `Literal` from typing or mypy crashes: see #11247 +from typing_extensions import ( # noqa: Y023 + Concatenate, + Literal, + ParamSpec, + Self, + TypeAlias, + TypeGuard, + TypeIs, + TypeVarTuple, + deprecated, +) + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_I = TypeVar("_I", default=int) +_T_co = TypeVar("_T_co", covariant=True) +_T_contra = TypeVar("_T_contra", contravariant=True) +_R_co = TypeVar("_R_co", covariant=True) +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_S = TypeVar("_S") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_T3 = TypeVar("_T3") +_T4 = TypeVar("_T4") +_T5 = TypeVar("_T5") +_SupportsNextT = TypeVar("_SupportsNextT", bound=SupportsNext[Any], covariant=True) +_SupportsAnextT = TypeVar("_SupportsAnextT", bound=SupportsAnext[Any], covariant=True) +_AwaitableT = TypeVar("_AwaitableT", bound=Awaitable[Any]) +_AwaitableT_co = TypeVar("_AwaitableT_co", bound=Awaitable[Any], covariant=True) +_P = ParamSpec("_P") +_StartT = TypeVar("_StartT", covariant=True, default=Any) +_StopT = TypeVar("_StopT", covariant=True, default=Any) +_StepT = TypeVar("_StepT", covariant=True, default=Any) + +class object: + __doc__: str | None + __dict__: dict[str, Any] + __module__: str + __annotations__: dict[str, Any] + @property + def __class__(self) -> type[Self]: ... + @__class__.setter + def __class__(self, type: type[Self], /) -> None: ... + def __init__(self) -> None: ... + def __new__(cls) -> Self: ... + # N.B. `object.__setattr__` and `object.__delattr__` are heavily special-cased by type checkers. + # Overriding them in subclasses has different semantics, even if the override has an identical signature. + def __setattr__(self, name: str, value: Any, /) -> None: ... + def __delattr__(self, name: str, /) -> None: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __str__(self) -> str: ... # noqa: Y029 + def __repr__(self) -> str: ... # noqa: Y029 + def __hash__(self) -> int: ... + def __format__(self, format_spec: str, /) -> str: ... + def __getattribute__(self, name: str, /) -> Any: ... + def __sizeof__(self) -> int: ... + # return type of pickle methods is rather hard to express in the current type system + # see #6661 and https://docs.python.org/3/library/pickle.html#object.__reduce__ + def __reduce__(self) -> str | tuple[Any, ...]: ... + def __reduce_ex__(self, protocol: SupportsIndex, /) -> str | tuple[Any, ...]: ... + if sys.version_info >= (3, 11): + def __getstate__(self) -> object: ... + + def __dir__(self) -> Iterable[str]: ... + def __init_subclass__(cls) -> None: ... + @classmethod + def __subclasshook__(cls, subclass: type, /) -> bool: ... + +class staticmethod(Generic[_P, _R_co]): + @property + def __func__(self) -> Callable[_P, _R_co]: ... + @property + def __isabstractmethod__(self) -> bool: ... + def __init__(self, f: Callable[_P, _R_co], /) -> None: ... + @overload + def __get__(self, instance: None, owner: type, /) -> Callable[_P, _R_co]: ... + @overload + def __get__(self, instance: _T, owner: type[_T] | None = None, /) -> Callable[_P, _R_co]: ... + if sys.version_info >= (3, 10): + __name__: str + __qualname__: str + @property + def __wrapped__(self) -> Callable[_P, _R_co]: ... + def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> _R_co: ... + +class classmethod(Generic[_T, _P, _R_co]): + @property + def __func__(self) -> Callable[Concatenate[type[_T], _P], _R_co]: ... + @property + def __isabstractmethod__(self) -> bool: ... + def __init__(self, f: Callable[Concatenate[type[_T], _P], _R_co], /) -> None: ... + @overload + def __get__(self, instance: _T, owner: type[_T] | None = None, /) -> Callable[_P, _R_co]: ... + @overload + def __get__(self, instance: None, owner: type[_T], /) -> Callable[_P, _R_co]: ... + if sys.version_info >= (3, 10): + __name__: str + __qualname__: str + @property + def __wrapped__(self) -> Callable[Concatenate[type[_T], _P], _R_co]: ... + +class type: + # object.__base__ is None. Otherwise, it would be a type. + @property + def __base__(self) -> type | None: ... + __bases__: tuple[type, ...] + @property + def __basicsize__(self) -> int: ... + @property + def __dict__(self) -> types.MappingProxyType[str, Any]: ... # type: ignore[override] + @property + def __dictoffset__(self) -> int: ... + @property + def __flags__(self) -> int: ... + @property + def __itemsize__(self) -> int: ... + __module__: str + @property + def __mro__(self) -> tuple[type, ...]: ... + __name__: str + __qualname__: str + @property + def __text_signature__(self) -> str | None: ... + @property + def __weakrefoffset__(self) -> int: ... + @overload + def __init__(self, o: object, /) -> None: ... + @overload + def __init__(self, name: str, bases: tuple[type, ...], dict: dict[str, Any], /, **kwds: Any) -> None: ... + @overload + def __new__(cls, o: object, /) -> type: ... + @overload + def __new__( + cls: type[_typeshed.Self], name: str, bases: tuple[type, ...], namespace: dict[str, Any], /, **kwds: Any + ) -> _typeshed.Self: ... + def __call__(self, *args: Any, **kwds: Any) -> Any: ... + def __subclasses__(self: _typeshed.Self) -> list[_typeshed.Self]: ... + # Note: the documentation doesn't specify what the return type is, the standard + # implementation seems to be returning a list. + def mro(self) -> list[type]: ... + def __instancecheck__(self, instance: Any, /) -> bool: ... + def __subclasscheck__(self, subclass: type, /) -> bool: ... + @classmethod + def __prepare__(metacls, name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object]: ... + if sys.version_info >= (3, 10): + def __or__(self, value: Any, /) -> types.UnionType: ... + def __ror__(self, value: Any, /) -> types.UnionType: ... + if sys.version_info >= (3, 12): + __type_params__: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] + +class super: + @overload + def __init__(self, t: Any, obj: Any, /) -> None: ... + @overload + def __init__(self, t: Any, /) -> None: ... + @overload + def __init__(self) -> None: ... + +_PositiveInteger: TypeAlias = Literal[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25] +_NegativeInteger: TypeAlias = Literal[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -17, -18, -19, -20] +_LiteralInteger = _PositiveInteger | _NegativeInteger | Literal[0] # noqa: Y026 # TODO: Use TypeAlias once mypy bugs are fixed + +class int: + @overload + def __new__(cls, x: ConvertibleToInt = ..., /) -> Self: ... + @overload + def __new__(cls, x: str | bytes | bytearray, /, base: SupportsIndex) -> Self: ... + def as_integer_ratio(self) -> tuple[int, Literal[1]]: ... + @property + def real(self) -> int: ... + @property + def imag(self) -> Literal[0]: ... + @property + def numerator(self) -> int: ... + @property + def denominator(self) -> Literal[1]: ... + def conjugate(self) -> int: ... + def bit_length(self) -> int: ... + if sys.version_info >= (3, 10): + def bit_count(self) -> int: ... + + if sys.version_info >= (3, 11): + def to_bytes( + self, length: SupportsIndex = 1, byteorder: Literal["little", "big"] = "big", *, signed: bool = False + ) -> bytes: ... + @classmethod + def from_bytes( + cls, + bytes: Iterable[SupportsIndex] | SupportsBytes | ReadableBuffer, + byteorder: Literal["little", "big"] = "big", + *, + signed: bool = False, + ) -> Self: ... + else: + def to_bytes(self, length: SupportsIndex, byteorder: Literal["little", "big"], *, signed: bool = False) -> bytes: ... + @classmethod + def from_bytes( + cls, + bytes: Iterable[SupportsIndex] | SupportsBytes | ReadableBuffer, + byteorder: Literal["little", "big"], + *, + signed: bool = False, + ) -> Self: ... + + if sys.version_info >= (3, 12): + def is_integer(self) -> Literal[True]: ... + + def __add__(self, value: int, /) -> int: ... + def __sub__(self, value: int, /) -> int: ... + def __mul__(self, value: int, /) -> int: ... + def __floordiv__(self, value: int, /) -> int: ... + def __truediv__(self, value: int, /) -> float: ... + def __mod__(self, value: int, /) -> int: ... + def __divmod__(self, value: int, /) -> tuple[int, int]: ... + def __radd__(self, value: int, /) -> int: ... + def __rsub__(self, value: int, /) -> int: ... + def __rmul__(self, value: int, /) -> int: ... + def __rfloordiv__(self, value: int, /) -> int: ... + def __rtruediv__(self, value: int, /) -> float: ... + def __rmod__(self, value: int, /) -> int: ... + def __rdivmod__(self, value: int, /) -> tuple[int, int]: ... + @overload + def __pow__(self, x: Literal[0], /) -> Literal[1]: ... + @overload + def __pow__(self, value: Literal[0], mod: None, /) -> Literal[1]: ... + @overload + def __pow__(self, value: _PositiveInteger, mod: None = None, /) -> int: ... + @overload + def __pow__(self, value: _NegativeInteger, mod: None = None, /) -> float: ... + # positive __value -> int; negative __value -> float + # return type must be Any as `int | float` causes too many false-positive errors + @overload + def __pow__(self, value: int, mod: None = None, /) -> Any: ... + @overload + def __pow__(self, value: int, mod: int, /) -> int: ... + def __rpow__(self, value: int, mod: int | None = None, /) -> Any: ... + def __and__(self, value: int, /) -> int: ... + def __or__(self, value: int, /) -> int: ... + def __xor__(self, value: int, /) -> int: ... + def __lshift__(self, value: int, /) -> int: ... + def __rshift__(self, value: int, /) -> int: ... + def __rand__(self, value: int, /) -> int: ... + def __ror__(self, value: int, /) -> int: ... + def __rxor__(self, value: int, /) -> int: ... + def __rlshift__(self, value: int, /) -> int: ... + def __rrshift__(self, value: int, /) -> int: ... + def __neg__(self) -> int: ... + def __pos__(self) -> int: ... + def __invert__(self) -> int: ... + def __trunc__(self) -> int: ... + def __ceil__(self) -> int: ... + def __floor__(self) -> int: ... + def __round__(self, ndigits: SupportsIndex = ..., /) -> int: ... + def __getnewargs__(self) -> tuple[int]: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __lt__(self, value: int, /) -> bool: ... + def __le__(self, value: int, /) -> bool: ... + def __gt__(self, value: int, /) -> bool: ... + def __ge__(self, value: int, /) -> bool: ... + def __float__(self) -> float: ... + def __int__(self) -> int: ... + def __abs__(self) -> int: ... + def __hash__(self) -> int: ... + def __bool__(self) -> bool: ... + def __index__(self) -> int: ... + +class float: + def __new__(cls, x: ConvertibleToFloat = ..., /) -> Self: ... + def as_integer_ratio(self) -> tuple[int, int]: ... + def hex(self) -> str: ... + def is_integer(self) -> bool: ... + @classmethod + def fromhex(cls, string: str, /) -> Self: ... + @property + def real(self) -> float: ... + @property + def imag(self) -> float: ... + def conjugate(self) -> float: ... + def __add__(self, value: float, /) -> float: ... + def __sub__(self, value: float, /) -> float: ... + def __mul__(self, value: float, /) -> float: ... + def __floordiv__(self, value: float, /) -> float: ... + def __truediv__(self, value: float, /) -> float: ... + def __mod__(self, value: float, /) -> float: ... + def __divmod__(self, value: float, /) -> tuple[float, float]: ... + @overload + def __pow__(self, value: int, mod: None = None, /) -> float: ... + # positive __value -> float; negative __value -> complex + # return type must be Any as `float | complex` causes too many false-positive errors + @overload + def __pow__(self, value: float, mod: None = None, /) -> Any: ... + def __radd__(self, value: float, /) -> float: ... + def __rsub__(self, value: float, /) -> float: ... + def __rmul__(self, value: float, /) -> float: ... + def __rfloordiv__(self, value: float, /) -> float: ... + def __rtruediv__(self, value: float, /) -> float: ... + def __rmod__(self, value: float, /) -> float: ... + def __rdivmod__(self, value: float, /) -> tuple[float, float]: ... + @overload + def __rpow__(self, value: _PositiveInteger, mod: None = None, /) -> float: ... + @overload + def __rpow__(self, value: _NegativeInteger, mod: None = None, /) -> complex: ... + # Returning `complex` for the general case gives too many false-positive errors. + @overload + def __rpow__(self, value: float, mod: None = None, /) -> Any: ... + def __getnewargs__(self) -> tuple[float]: ... + def __trunc__(self) -> int: ... + if sys.version_info >= (3, 9): + def __ceil__(self) -> int: ... + def __floor__(self) -> int: ... + + @overload + def __round__(self, ndigits: None = None, /) -> int: ... + @overload + def __round__(self, ndigits: SupportsIndex, /) -> float: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __lt__(self, value: float, /) -> bool: ... + def __le__(self, value: float, /) -> bool: ... + def __gt__(self, value: float, /) -> bool: ... + def __ge__(self, value: float, /) -> bool: ... + def __neg__(self) -> float: ... + def __pos__(self) -> float: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __abs__(self) -> float: ... + def __hash__(self) -> int: ... + def __bool__(self) -> bool: ... + +class complex: + # Python doesn't currently accept SupportsComplex for the second argument + @overload + def __new__( + cls, + real: complex | SupportsComplex | SupportsFloat | SupportsIndex = ..., + imag: complex | SupportsFloat | SupportsIndex = ..., + ) -> Self: ... + @overload + def __new__(cls, real: str | SupportsComplex | SupportsFloat | SupportsIndex | complex) -> Self: ... + @property + def real(self) -> float: ... + @property + def imag(self) -> float: ... + def conjugate(self) -> complex: ... + def __add__(self, value: complex, /) -> complex: ... + def __sub__(self, value: complex, /) -> complex: ... + def __mul__(self, value: complex, /) -> complex: ... + def __pow__(self, value: complex, mod: None = None, /) -> complex: ... + def __truediv__(self, value: complex, /) -> complex: ... + def __radd__(self, value: complex, /) -> complex: ... + def __rsub__(self, value: complex, /) -> complex: ... + def __rmul__(self, value: complex, /) -> complex: ... + def __rpow__(self, value: complex, mod: None = None, /) -> complex: ... + def __rtruediv__(self, value: complex, /) -> complex: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __neg__(self) -> complex: ... + def __pos__(self) -> complex: ... + def __abs__(self) -> float: ... + def __hash__(self) -> int: ... + def __bool__(self) -> bool: ... + if sys.version_info >= (3, 11): + def __complex__(self) -> complex: ... + +class _FormatMapMapping(Protocol): + def __getitem__(self, key: str, /) -> Any: ... + +class _TranslateTable(Protocol): + def __getitem__(self, key: int, /) -> str | int | None: ... + +class str(Sequence[str]): + @overload + def __new__(cls, object: object = ...) -> Self: ... + @overload + def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... + def capitalize(self) -> str: ... # type: ignore[misc] + def casefold(self) -> str: ... # type: ignore[misc] + def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] + def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... + def endswith( + self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> bool: ... + def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] + def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def format(self, *args: object, **kwargs: object) -> str: ... + def format_map(self, mapping: _FormatMapMapping, /) -> str: ... + def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def isalnum(self) -> bool: ... + def isalpha(self) -> bool: ... + def isascii(self) -> bool: ... + def isdecimal(self) -> bool: ... + def isdigit(self) -> bool: ... + def isidentifier(self) -> bool: ... + def islower(self) -> bool: ... + def isnumeric(self) -> bool: ... + def isprintable(self) -> bool: ... + def isspace(self) -> bool: ... + def istitle(self) -> bool: ... + def isupper(self) -> bool: ... + def join(self, iterable: Iterable[str], /) -> str: ... # type: ignore[misc] + def ljust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] + def lower(self) -> str: ... # type: ignore[misc] + def lstrip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] + def partition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] + if sys.version_info >= (3, 13): + def replace(self, old: str, new: str, /, count: SupportsIndex = -1) -> str: ... # type: ignore[misc] + else: + def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ... # type: ignore[misc] + if sys.version_info >= (3, 9): + def removeprefix(self, prefix: str, /) -> str: ... # type: ignore[misc] + def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] + + def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rjust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] + def rpartition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] + def rsplit(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] + def rstrip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] + def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] + def splitlines(self, keepends: bool = False) -> list[str]: ... # type: ignore[misc] + def startswith( + self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> bool: ... + def strip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] + def swapcase(self) -> str: ... # type: ignore[misc] + def title(self) -> str: ... # type: ignore[misc] + def translate(self, table: _TranslateTable, /) -> str: ... + def upper(self) -> str: ... # type: ignore[misc] + def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] + @staticmethod + @overload + def maketrans(x: dict[int, _T] | dict[str, _T] | dict[str | int, _T], /) -> dict[int, _T]: ... + @staticmethod + @overload + def maketrans(x: str, y: str, /) -> dict[int, int]: ... + @staticmethod + @overload + def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... + def __add__(self, value: str, /) -> str: ... # type: ignore[misc] + # Incompatible with Sequence.__contains__ + def __contains__(self, key: str, /) -> bool: ... # type: ignore[override] + def __eq__(self, value: object, /) -> bool: ... + def __ge__(self, value: str, /) -> bool: ... + def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... + def __gt__(self, value: str, /) -> bool: ... + def __hash__(self) -> int: ... + def __iter__(self) -> Iterator[str]: ... # type: ignore[misc] + def __le__(self, value: str, /) -> bool: ... + def __len__(self) -> int: ... + def __lt__(self, value: str, /) -> bool: ... + def __mod__(self, value: Any, /) -> str: ... + def __mul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] + def __ne__(self, value: object, /) -> bool: ... + def __rmul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] + def __getnewargs__(self) -> tuple[str]: ... + +class bytes(Sequence[int]): + @overload + def __new__(cls, o: Iterable[SupportsIndex] | SupportsIndex | SupportsBytes | ReadableBuffer, /) -> Self: ... + @overload + def __new__(cls, string: str, /, encoding: str, errors: str = ...) -> Self: ... + @overload + def __new__(cls) -> Self: ... + def capitalize(self) -> bytes: ... + def center(self, width: SupportsIndex, fillchar: bytes = b" ", /) -> bytes: ... + def count( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: ... + def endswith( + self, + suffix: ReadableBuffer | tuple[ReadableBuffer, ...], + start: SupportsIndex | None = ..., + end: SupportsIndex | None = ..., + /, + ) -> bool: ... + def expandtabs(self, tabsize: SupportsIndex = 8) -> bytes: ... + def find( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def index( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def isalnum(self) -> bool: ... + def isalpha(self) -> bool: ... + def isascii(self) -> bool: ... + def isdigit(self) -> bool: ... + def islower(self) -> bool: ... + def isspace(self) -> bool: ... + def istitle(self) -> bool: ... + def isupper(self) -> bool: ... + def join(self, iterable_of_bytes: Iterable[ReadableBuffer], /) -> bytes: ... + def ljust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytes: ... + def lower(self) -> bytes: ... + def lstrip(self, bytes: ReadableBuffer | None = None, /) -> bytes: ... + def partition(self, sep: ReadableBuffer, /) -> tuple[bytes, bytes, bytes]: ... + def replace(self, old: ReadableBuffer, new: ReadableBuffer, count: SupportsIndex = -1, /) -> bytes: ... + if sys.version_info >= (3, 9): + def removeprefix(self, prefix: ReadableBuffer, /) -> bytes: ... + def removesuffix(self, suffix: ReadableBuffer, /) -> bytes: ... + + def rfind( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def rindex( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def rjust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytes: ... + def rpartition(self, sep: ReadableBuffer, /) -> tuple[bytes, bytes, bytes]: ... + def rsplit(self, sep: ReadableBuffer | None = None, maxsplit: SupportsIndex = -1) -> list[bytes]: ... + def rstrip(self, bytes: ReadableBuffer | None = None, /) -> bytes: ... + def split(self, sep: ReadableBuffer | None = None, maxsplit: SupportsIndex = -1) -> list[bytes]: ... + def splitlines(self, keepends: bool = False) -> list[bytes]: ... + def startswith( + self, + prefix: ReadableBuffer | tuple[ReadableBuffer, ...], + start: SupportsIndex | None = ..., + end: SupportsIndex | None = ..., + /, + ) -> bool: ... + def strip(self, bytes: ReadableBuffer | None = None, /) -> bytes: ... + def swapcase(self) -> bytes: ... + def title(self) -> bytes: ... + def translate(self, table: ReadableBuffer | None, /, delete: bytes = b"") -> bytes: ... + def upper(self) -> bytes: ... + def zfill(self, width: SupportsIndex, /) -> bytes: ... + @classmethod + def fromhex(cls, string: str, /) -> Self: ... + @staticmethod + def maketrans(frm: ReadableBuffer, to: ReadableBuffer, /) -> bytes: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + def __hash__(self) -> int: ... + @overload + def __getitem__(self, key: SupportsIndex, /) -> int: ... + @overload + def __getitem__(self, key: slice, /) -> bytes: ... + def __add__(self, value: ReadableBuffer, /) -> bytes: ... + def __mul__(self, value: SupportsIndex, /) -> bytes: ... + def __rmul__(self, value: SupportsIndex, /) -> bytes: ... + def __mod__(self, value: Any, /) -> bytes: ... + # Incompatible with Sequence.__contains__ + def __contains__(self, key: SupportsIndex | ReadableBuffer, /) -> bool: ... # type: ignore[override] + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __lt__(self, value: bytes, /) -> bool: ... + def __le__(self, value: bytes, /) -> bool: ... + def __gt__(self, value: bytes, /) -> bool: ... + def __ge__(self, value: bytes, /) -> bool: ... + def __getnewargs__(self) -> tuple[bytes]: ... + if sys.version_info >= (3, 11): + def __bytes__(self) -> bytes: ... + + def __buffer__(self, flags: int, /) -> memoryview: ... + +class bytearray(MutableSequence[int]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, ints: Iterable[SupportsIndex] | SupportsIndex | ReadableBuffer, /) -> None: ... + @overload + def __init__(self, string: str, /, encoding: str, errors: str = ...) -> None: ... + def append(self, item: SupportsIndex, /) -> None: ... + def capitalize(self) -> bytearray: ... + def center(self, width: SupportsIndex, fillchar: bytes = b" ", /) -> bytearray: ... + def count( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def copy(self) -> bytearray: ... + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: ... + def endswith( + self, + suffix: ReadableBuffer | tuple[ReadableBuffer, ...], + start: SupportsIndex | None = ..., + end: SupportsIndex | None = ..., + /, + ) -> bool: ... + def expandtabs(self, tabsize: SupportsIndex = 8) -> bytearray: ... + def extend(self, iterable_of_ints: Iterable[SupportsIndex], /) -> None: ... + def find( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def index( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def insert(self, index: SupportsIndex, item: SupportsIndex, /) -> None: ... + def isalnum(self) -> bool: ... + def isalpha(self) -> bool: ... + def isascii(self) -> bool: ... + def isdigit(self) -> bool: ... + def islower(self) -> bool: ... + def isspace(self) -> bool: ... + def istitle(self) -> bool: ... + def isupper(self) -> bool: ... + def join(self, iterable_of_bytes: Iterable[ReadableBuffer], /) -> bytearray: ... + def ljust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytearray: ... + def lower(self) -> bytearray: ... + def lstrip(self, bytes: ReadableBuffer | None = None, /) -> bytearray: ... + def partition(self, sep: ReadableBuffer, /) -> tuple[bytearray, bytearray, bytearray]: ... + def pop(self, index: int = -1, /) -> int: ... + def remove(self, value: int, /) -> None: ... + if sys.version_info >= (3, 9): + def removeprefix(self, prefix: ReadableBuffer, /) -> bytearray: ... + def removesuffix(self, suffix: ReadableBuffer, /) -> bytearray: ... + + def replace(self, old: ReadableBuffer, new: ReadableBuffer, count: SupportsIndex = -1, /) -> bytearray: ... + def rfind( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def rindex( + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> int: ... + def rjust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytearray: ... + def rpartition(self, sep: ReadableBuffer, /) -> tuple[bytearray, bytearray, bytearray]: ... + def rsplit(self, sep: ReadableBuffer | None = None, maxsplit: SupportsIndex = -1) -> list[bytearray]: ... + def rstrip(self, bytes: ReadableBuffer | None = None, /) -> bytearray: ... + def split(self, sep: ReadableBuffer | None = None, maxsplit: SupportsIndex = -1) -> list[bytearray]: ... + def splitlines(self, keepends: bool = False) -> list[bytearray]: ... + def startswith( + self, + prefix: ReadableBuffer | tuple[ReadableBuffer, ...], + start: SupportsIndex | None = ..., + end: SupportsIndex | None = ..., + /, + ) -> bool: ... + def strip(self, bytes: ReadableBuffer | None = None, /) -> bytearray: ... + def swapcase(self) -> bytearray: ... + def title(self) -> bytearray: ... + def translate(self, table: ReadableBuffer | None, /, delete: bytes = b"") -> bytearray: ... + def upper(self) -> bytearray: ... + def zfill(self, width: SupportsIndex, /) -> bytearray: ... + @classmethod + def fromhex(cls, string: str, /) -> Self: ... + @staticmethod + def maketrans(frm: ReadableBuffer, to: ReadableBuffer, /) -> bytes: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + __hash__: ClassVar[None] # type: ignore[assignment] + @overload + def __getitem__(self, key: SupportsIndex, /) -> int: ... + @overload + def __getitem__(self, key: slice, /) -> bytearray: ... + @overload + def __setitem__(self, key: SupportsIndex, value: SupportsIndex, /) -> None: ... + @overload + def __setitem__(self, key: slice, value: Iterable[SupportsIndex] | bytes, /) -> None: ... + def __delitem__(self, key: SupportsIndex | slice, /) -> None: ... + def __add__(self, value: ReadableBuffer, /) -> bytearray: ... + # The superclass wants us to accept Iterable[int], but that fails at runtime. + def __iadd__(self, value: ReadableBuffer, /) -> Self: ... # type: ignore[override] + def __mul__(self, value: SupportsIndex, /) -> bytearray: ... + def __rmul__(self, value: SupportsIndex, /) -> bytearray: ... + def __imul__(self, value: SupportsIndex, /) -> Self: ... + def __mod__(self, value: Any, /) -> bytes: ... + # Incompatible with Sequence.__contains__ + def __contains__(self, key: SupportsIndex | ReadableBuffer, /) -> bool: ... # type: ignore[override] + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __lt__(self, value: ReadableBuffer, /) -> bool: ... + def __le__(self, value: ReadableBuffer, /) -> bool: ... + def __gt__(self, value: ReadableBuffer, /) -> bool: ... + def __ge__(self, value: ReadableBuffer, /) -> bool: ... + def __alloc__(self) -> int: ... + def __buffer__(self, flags: int, /) -> memoryview: ... + def __release_buffer__(self, buffer: memoryview, /) -> None: ... + +_IntegerFormats: TypeAlias = Literal[ + "b", "B", "@b", "@B", "h", "H", "@h", "@H", "i", "I", "@i", "@I", "l", "L", "@l", "@L", "q", "Q", "@q", "@Q", "P", "@P" +] + +@final +class memoryview(Sequence[_I]): + @property + def format(self) -> str: ... + @property + def itemsize(self) -> int: ... + @property + def shape(self) -> tuple[int, ...] | None: ... + @property + def strides(self) -> tuple[int, ...] | None: ... + @property + def suboffsets(self) -> tuple[int, ...] | None: ... + @property + def readonly(self) -> bool: ... + @property + def ndim(self) -> int: ... + @property + def obj(self) -> ReadableBuffer: ... + @property + def c_contiguous(self) -> bool: ... + @property + def f_contiguous(self) -> bool: ... + @property + def contiguous(self) -> bool: ... + @property + def nbytes(self) -> int: ... + def __new__(cls, obj: ReadableBuffer) -> Self: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, / + ) -> None: ... + @overload + def cast(self, format: Literal["c", "@c"], shape: list[int] | tuple[int, ...] = ...) -> memoryview[bytes]: ... + @overload + def cast(self, format: Literal["f", "@f", "d", "@d"], shape: list[int] | tuple[int, ...] = ...) -> memoryview[float]: ... + @overload + def cast(self, format: Literal["?"], shape: list[int] | tuple[int, ...] = ...) -> memoryview[bool]: ... + @overload + def cast(self, format: _IntegerFormats, shape: list[int] | tuple[int, ...] = ...) -> memoryview: ... + @overload + def __getitem__(self, key: SupportsIndex | tuple[SupportsIndex, ...], /) -> _I: ... + @overload + def __getitem__(self, key: slice, /) -> memoryview[_I]: ... + def __contains__(self, x: object, /) -> bool: ... + def __iter__(self) -> Iterator[_I]: ... + def __len__(self) -> int: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + @overload + def __setitem__(self, key: slice, value: ReadableBuffer, /) -> None: ... + @overload + def __setitem__(self, key: SupportsIndex | tuple[SupportsIndex, ...], value: _I, /) -> None: ... + if sys.version_info >= (3, 10): + def tobytes(self, order: Literal["C", "F", "A"] | None = "C") -> bytes: ... + else: + def tobytes(self, order: Literal["C", "F", "A"] | None = None) -> bytes: ... + + def tolist(self) -> list[int]: ... + def toreadonly(self) -> memoryview: ... + def release(self) -> None: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def __buffer__(self, flags: int, /) -> memoryview: ... + def __release_buffer__(self, buffer: memoryview, /) -> None: ... + + # These are inherited from the Sequence ABC, but don't actually exist on memoryview. + # See https://github.com/python/cpython/issues/125420 + index: ClassVar[None] # type: ignore[assignment] + count: ClassVar[None] # type: ignore[assignment] + +@final +class bool(int): + def __new__(cls, o: object = ..., /) -> Self: ... + # The following overloads could be represented more elegantly with a TypeVar("_B", bool, int), + # however mypy has a bug regarding TypeVar constraints (https://github.com/python/mypy/issues/11880). + @overload + def __and__(self, value: bool, /) -> bool: ... + @overload + def __and__(self, value: int, /) -> int: ... + @overload + def __or__(self, value: bool, /) -> bool: ... + @overload + def __or__(self, value: int, /) -> int: ... + @overload + def __xor__(self, value: bool, /) -> bool: ... + @overload + def __xor__(self, value: int, /) -> int: ... + @overload + def __rand__(self, value: bool, /) -> bool: ... + @overload + def __rand__(self, value: int, /) -> int: ... + @overload + def __ror__(self, value: bool, /) -> bool: ... + @overload + def __ror__(self, value: int, /) -> int: ... + @overload + def __rxor__(self, value: bool, /) -> bool: ... + @overload + def __rxor__(self, value: int, /) -> int: ... + def __getnewargs__(self) -> tuple[int]: ... + @deprecated("Will throw an error in Python 3.14. Use `not` for logical negation of bools instead.") + def __invert__(self) -> int: ... + +@final +class slice(Generic[_StartT, _StopT, _StepT]): + @property + def start(self) -> _StartT: ... + @property + def step(self) -> _StepT: ... + @property + def stop(self) -> _StopT: ... + @overload + def __new__(cls, stop: int | None, /) -> slice[int | MaybeNone, int | MaybeNone, int | MaybeNone]: ... + @overload + def __new__( + cls, start: int | None, stop: int | None, step: int | None = None, / + ) -> slice[int | MaybeNone, int | MaybeNone, int | MaybeNone]: ... + @overload + def __new__(cls, stop: _T2, /) -> slice[Any, _T2, Any]: ... + @overload + def __new__(cls, start: _T1, stop: _T2, /) -> slice[_T1, _T2, Any]: ... + @overload + def __new__(cls, start: _T1, stop: _T2, step: _T3, /) -> slice[_T1, _T2, _T3]: ... + def __eq__(self, value: object, /) -> bool: ... + if sys.version_info >= (3, 12): + def __hash__(self) -> int: ... + else: + __hash__: ClassVar[None] # type: ignore[assignment] + + def indices(self, len: SupportsIndex, /) -> tuple[int, int, int]: ... + +class tuple(Sequence[_T_co]): + def __new__(cls, iterable: Iterable[_T_co] = ..., /) -> Self: ... + def __len__(self) -> int: ... + def __contains__(self, key: object, /) -> bool: ... + @overload + def __getitem__(self, key: SupportsIndex, /) -> _T_co: ... + @overload + def __getitem__(self, key: slice, /) -> tuple[_T_co, ...]: ... + def __iter__(self) -> Iterator[_T_co]: ... + def __lt__(self, value: tuple[_T_co, ...], /) -> bool: ... + def __le__(self, value: tuple[_T_co, ...], /) -> bool: ... + def __gt__(self, value: tuple[_T_co, ...], /) -> bool: ... + def __ge__(self, value: tuple[_T_co, ...], /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + @overload + def __add__(self, value: tuple[_T_co, ...], /) -> tuple[_T_co, ...]: ... + @overload + def __add__(self, value: tuple[_T, ...], /) -> tuple[_T_co | _T, ...]: ... + def __mul__(self, value: SupportsIndex, /) -> tuple[_T_co, ...]: ... + def __rmul__(self, value: SupportsIndex, /) -> tuple[_T_co, ...]: ... + def count(self, value: Any, /) -> int: ... + def index(self, value: Any, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +# Doesn't exist at runtime, but deleting this breaks mypy and pyright. See: +# https://github.com/python/typeshed/issues/7580 +# https://github.com/python/mypy/issues/8240 +@final +@type_check_only +class function: + # Make sure this class definition stays roughly in line with `types.FunctionType` + @property + def __closure__(self) -> tuple[CellType, ...] | None: ... + __code__: CodeType + __defaults__: tuple[Any, ...] | None + __dict__: dict[str, Any] + @property + def __globals__(self) -> dict[str, Any]: ... + __name__: str + __qualname__: str + __annotations__: dict[str, Any] + __kwdefaults__: dict[str, Any] + if sys.version_info >= (3, 10): + @property + def __builtins__(self) -> dict[str, Any]: ... + if sys.version_info >= (3, 12): + __type_params__: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] + + __module__: str + # mypy uses `builtins.function.__get__` to represent methods, properties, and getset_descriptors so we type the return as Any. + def __get__(self, instance: object, owner: type | None = None, /) -> Any: ... + +class list(MutableSequence[_T]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, iterable: Iterable[_T], /) -> None: ... + def copy(self) -> list[_T]: ... + def append(self, object: _T, /) -> None: ... + def extend(self, iterable: Iterable[_T], /) -> None: ... + def pop(self, index: SupportsIndex = -1, /) -> _T: ... + # Signature of `list.index` should be kept in line with `collections.UserList.index()` + # and multiprocessing.managers.ListProxy.index() + def index(self, value: _T, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int: ... + def count(self, value: _T, /) -> int: ... + def insert(self, index: SupportsIndex, object: _T, /) -> None: ... + def remove(self, value: _T, /) -> None: ... + # Signature of `list.sort` should be kept inline with `collections.UserList.sort()` + # and multiprocessing.managers.ListProxy.sort() + # + # Use list[SupportsRichComparisonT] for the first overload rather than [SupportsRichComparison] + # to work around invariance + @overload + def sort(self: list[SupportsRichComparisonT], *, key: None = None, reverse: bool = False) -> None: ... + @overload + def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = False) -> None: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[_T]: ... + __hash__: ClassVar[None] # type: ignore[assignment] + @overload + def __getitem__(self, i: SupportsIndex, /) -> _T: ... + @overload + def __getitem__(self, s: slice, /) -> list[_T]: ... + @overload + def __setitem__(self, key: SupportsIndex, value: _T, /) -> None: ... + @overload + def __setitem__(self, key: slice, value: Iterable[_T], /) -> None: ... + def __delitem__(self, key: SupportsIndex | slice, /) -> None: ... + # Overloading looks unnecessary, but is needed to work around complex mypy problems + @overload + def __add__(self, value: list[_T], /) -> list[_T]: ... + @overload + def __add__(self, value: list[_S], /) -> list[_S | _T]: ... + def __iadd__(self, value: Iterable[_T], /) -> Self: ... # type: ignore[misc] + def __mul__(self, value: SupportsIndex, /) -> list[_T]: ... + def __rmul__(self, value: SupportsIndex, /) -> list[_T]: ... + def __imul__(self, value: SupportsIndex, /) -> Self: ... + def __contains__(self, key: object, /) -> bool: ... + def __reversed__(self) -> Iterator[_T]: ... + def __gt__(self, value: list[_T], /) -> bool: ... + def __ge__(self, value: list[_T], /) -> bool: ... + def __lt__(self, value: list[_T], /) -> bool: ... + def __le__(self, value: list[_T], /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class dict(MutableMapping[_KT, _VT]): + # __init__ should be kept roughly in line with `collections.UserDict.__init__`, which has similar semantics + # Also multiprocessing.managers.SyncManager.dict() + @overload + def __init__(self) -> None: ... + @overload + def __init__(self: dict[str, _VT], **kwargs: _VT) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] #11780 + @overload + def __init__(self, map: SupportsKeysAndGetItem[_KT, _VT], /) -> None: ... + @overload + def __init__( + self: dict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + map: SupportsKeysAndGetItem[str, _VT], + /, + **kwargs: _VT, + ) -> None: ... + @overload + def __init__(self, iterable: Iterable[tuple[_KT, _VT]], /) -> None: ... + @overload + def __init__( + self: dict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + iterable: Iterable[tuple[str, _VT]], + /, + **kwargs: _VT, + ) -> None: ... + # Next two overloads are for dict(string.split(sep) for string in iterable) + # Cannot be Iterable[Sequence[_T]] or otherwise dict(["foo", "bar", "baz"]) is not an error + @overload + def __init__(self: dict[str, str], iterable: Iterable[list[str]], /) -> None: ... + @overload + def __init__(self: dict[bytes, bytes], iterable: Iterable[list[bytes]], /) -> None: ... + def __new__(cls, *args: Any, **kwargs: Any) -> Self: ... + def copy(self) -> dict[_KT, _VT]: ... + def keys(self) -> dict_keys[_KT, _VT]: ... + def values(self) -> dict_values[_KT, _VT]: ... + def items(self) -> dict_items[_KT, _VT]: ... + # Signature of `dict.fromkeys` should be kept identical to + # `fromkeys` methods of `OrderedDict`/`ChainMap`/`UserDict` in `collections` + # TODO: the true signature of `dict.fromkeys` is not expressible in the current type system. + # See #3800 & https://github.com/python/typing/issues/548#issuecomment-683336963. + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], value: None = None, /) -> dict[_T, Any | None]: ... + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], value: _S, /) -> dict[_T, _S]: ... + # Positional-only in dict, but not in MutableMapping + @overload # type: ignore[override] + def get(self, key: _KT, /) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _VT, /) -> _VT: ... + @overload + def get(self, key: _KT, default: _T, /) -> _VT | _T: ... + @overload + def pop(self, key: _KT, /) -> _VT: ... + @overload + def pop(self, key: _KT, default: _VT, /) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T, /) -> _VT | _T: ... + def __len__(self) -> int: ... + def __getitem__(self, key: _KT, /) -> _VT: ... + def __setitem__(self, key: _KT, value: _VT, /) -> None: ... + def __delitem__(self, key: _KT, /) -> None: ... + def __iter__(self) -> Iterator[_KT]: ... + def __eq__(self, value: object, /) -> bool: ... + def __reversed__(self) -> Iterator[_KT]: ... + __hash__: ClassVar[None] # type: ignore[assignment] + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + @overload + def __or__(self, value: dict[_KT, _VT], /) -> dict[_KT, _VT]: ... + @overload + def __or__(self, value: dict[_T1, _T2], /) -> dict[_KT | _T1, _VT | _T2]: ... + @overload + def __ror__(self, value: dict[_KT, _VT], /) -> dict[_KT, _VT]: ... + @overload + def __ror__(self, value: dict[_T1, _T2], /) -> dict[_KT | _T1, _VT | _T2]: ... + # dict.__ior__ should be kept roughly in line with MutableMapping.update() + @overload # type: ignore[misc] + def __ior__(self, value: SupportsKeysAndGetItem[_KT, _VT], /) -> Self: ... + @overload + def __ior__(self, value: Iterable[tuple[_KT, _VT]], /) -> Self: ... + +class set(MutableSet[_T]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, iterable: Iterable[_T], /) -> None: ... + def add(self, element: _T, /) -> None: ... + def copy(self) -> set[_T]: ... + def difference(self, *s: Iterable[Any]) -> set[_T]: ... + def difference_update(self, *s: Iterable[Any]) -> None: ... + def discard(self, element: _T, /) -> None: ... + def intersection(self, *s: Iterable[Any]) -> set[_T]: ... + def intersection_update(self, *s: Iterable[Any]) -> None: ... + def isdisjoint(self, s: Iterable[Any], /) -> bool: ... + def issubset(self, s: Iterable[Any], /) -> bool: ... + def issuperset(self, s: Iterable[Any], /) -> bool: ... + def remove(self, element: _T, /) -> None: ... + def symmetric_difference(self, s: Iterable[_T], /) -> set[_T]: ... + def symmetric_difference_update(self, s: Iterable[_T], /) -> None: ... + def union(self, *s: Iterable[_S]) -> set[_T | _S]: ... + def update(self, *s: Iterable[_T]) -> None: ... + def __len__(self) -> int: ... + def __contains__(self, o: object, /) -> bool: ... + def __iter__(self) -> Iterator[_T]: ... + def __and__(self, value: AbstractSet[object], /) -> set[_T]: ... + def __iand__(self, value: AbstractSet[object], /) -> Self: ... + def __or__(self, value: AbstractSet[_S], /) -> set[_T | _S]: ... + def __ior__(self, value: AbstractSet[_T], /) -> Self: ... # type: ignore[override,misc] + def __sub__(self, value: AbstractSet[_T | None], /) -> set[_T]: ... + def __isub__(self, value: AbstractSet[object], /) -> Self: ... + def __xor__(self, value: AbstractSet[_S], /) -> set[_T | _S]: ... + def __ixor__(self, value: AbstractSet[_T], /) -> Self: ... # type: ignore[override,misc] + def __le__(self, value: AbstractSet[object], /) -> bool: ... + def __lt__(self, value: AbstractSet[object], /) -> bool: ... + def __ge__(self, value: AbstractSet[object], /) -> bool: ... + def __gt__(self, value: AbstractSet[object], /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + __hash__: ClassVar[None] # type: ignore[assignment] + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class frozenset(AbstractSet[_T_co]): + @overload + def __new__(cls) -> Self: ... + @overload + def __new__(cls, iterable: Iterable[_T_co], /) -> Self: ... + def copy(self) -> frozenset[_T_co]: ... + def difference(self, *s: Iterable[object]) -> frozenset[_T_co]: ... + def intersection(self, *s: Iterable[object]) -> frozenset[_T_co]: ... + def isdisjoint(self, s: Iterable[_T_co], /) -> bool: ... + def issubset(self, s: Iterable[object], /) -> bool: ... + def issuperset(self, s: Iterable[object], /) -> bool: ... + def symmetric_difference(self, s: Iterable[_T_co], /) -> frozenset[_T_co]: ... + def union(self, *s: Iterable[_S]) -> frozenset[_T_co | _S]: ... + def __len__(self) -> int: ... + def __contains__(self, o: object, /) -> bool: ... + def __iter__(self) -> Iterator[_T_co]: ... + def __and__(self, value: AbstractSet[_T_co], /) -> frozenset[_T_co]: ... + def __or__(self, value: AbstractSet[_S], /) -> frozenset[_T_co | _S]: ... + def __sub__(self, value: AbstractSet[_T_co], /) -> frozenset[_T_co]: ... + def __xor__(self, value: AbstractSet[_S], /) -> frozenset[_T_co | _S]: ... + def __le__(self, value: AbstractSet[object], /) -> bool: ... + def __lt__(self, value: AbstractSet[object], /) -> bool: ... + def __ge__(self, value: AbstractSet[object], /) -> bool: ... + def __gt__(self, value: AbstractSet[object], /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class enumerate(Generic[_T]): + def __new__(cls, iterable: Iterable[_T], start: int = 0) -> Self: ... + def __iter__(self) -> Self: ... + def __next__(self) -> tuple[int, _T]: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +@final +class range(Sequence[int]): + @property + def start(self) -> int: ... + @property + def stop(self) -> int: ... + @property + def step(self) -> int: ... + @overload + def __new__(cls, stop: SupportsIndex, /) -> Self: ... + @overload + def __new__(cls, start: SupportsIndex, stop: SupportsIndex, step: SupportsIndex = ..., /) -> Self: ... + def count(self, value: int, /) -> int: ... + def index(self, value: int, /) -> int: ... # type: ignore[override] + def __len__(self) -> int: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + def __contains__(self, key: object, /) -> bool: ... + def __iter__(self) -> Iterator[int]: ... + @overload + def __getitem__(self, key: SupportsIndex, /) -> int: ... + @overload + def __getitem__(self, key: slice, /) -> range: ... + def __reversed__(self) -> Iterator[int]: ... + +class property: + fget: Callable[[Any], Any] | None + fset: Callable[[Any, Any], None] | None + fdel: Callable[[Any], None] | None + __isabstractmethod__: bool + if sys.version_info >= (3, 13): + __name__: str + + def __init__( + self, + fget: Callable[[Any], Any] | None = ..., + fset: Callable[[Any, Any], None] | None = ..., + fdel: Callable[[Any], None] | None = ..., + doc: str | None = ..., + ) -> None: ... + def getter(self, fget: Callable[[Any], Any], /) -> property: ... + def setter(self, fset: Callable[[Any, Any], None], /) -> property: ... + def deleter(self, fdel: Callable[[Any], None], /) -> property: ... + def __get__(self, instance: Any, owner: type | None = None, /) -> Any: ... + def __set__(self, instance: Any, value: Any, /) -> None: ... + def __delete__(self, instance: Any, /) -> None: ... + +@final +class _NotImplementedType(Any): + __call__: None + +NotImplemented: _NotImplementedType + +def abs(x: SupportsAbs[_T], /) -> _T: ... +def all(iterable: Iterable[object], /) -> bool: ... +def any(iterable: Iterable[object], /) -> bool: ... +def ascii(obj: object, /) -> str: ... +def bin(number: int | SupportsIndex, /) -> str: ... +def breakpoint(*args: Any, **kws: Any) -> None: ... +def callable(obj: object, /) -> TypeIs[Callable[..., object]]: ... +def chr(i: int, /) -> str: ... + +# We define this here instead of using os.PathLike to avoid import cycle issues. +# See https://github.com/python/typeshed/pull/991#issuecomment-288160993 +class _PathLike(Protocol[AnyStr_co]): + def __fspath__(self) -> AnyStr_co: ... + +if sys.version_info >= (3, 10): + def aiter(async_iterable: SupportsAiter[_SupportsAnextT], /) -> _SupportsAnextT: ... + + class _SupportsSynchronousAnext(Protocol[_AwaitableT_co]): + def __anext__(self) -> _AwaitableT_co: ... + + @overload + # `anext` is not, in fact, an async function. When default is not provided + # `anext` is just a passthrough for `obj.__anext__` + # See discussion in #7491 and pure-Python implementation of `anext` at https://github.com/python/cpython/blob/ea786a882b9ed4261eafabad6011bc7ef3b5bf94/Lib/test/test_asyncgen.py#L52-L80 + def anext(i: _SupportsSynchronousAnext[_AwaitableT], /) -> _AwaitableT: ... + @overload + async def anext(i: SupportsAnext[_T], default: _VT, /) -> _T | _VT: ... + +# compile() returns a CodeType, unless the flags argument includes PyCF_ONLY_AST (=1024), +# in which case it returns ast.AST. We have overloads for flag 0 (the default) and for +# explicitly passing PyCF_ONLY_AST. We fall back to Any for other values of flags. +@overload +def compile( + source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, + filename: str | ReadableBuffer | _PathLike[Any], + mode: str, + flags: Literal[0], + dont_inherit: bool = False, + optimize: int = -1, + *, + _feature_version: int = -1, +) -> CodeType: ... +@overload +def compile( + source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, + filename: str | ReadableBuffer | _PathLike[Any], + mode: str, + *, + dont_inherit: bool = False, + optimize: int = -1, + _feature_version: int = -1, +) -> CodeType: ... +@overload +def compile( + source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, + filename: str | ReadableBuffer | _PathLike[Any], + mode: str, + flags: Literal[1024], + dont_inherit: bool = False, + optimize: int = -1, + *, + _feature_version: int = -1, +) -> _ast.AST: ... +@overload +def compile( + source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, + filename: str | ReadableBuffer | _PathLike[Any], + mode: str, + flags: int, + dont_inherit: bool = False, + optimize: int = -1, + *, + _feature_version: int = -1, +) -> Any: ... +def copyright() -> None: ... +def credits() -> None: ... +def delattr(obj: object, name: str, /) -> None: ... +def dir(o: object = ..., /) -> list[str]: ... +@overload +def divmod(x: SupportsDivMod[_T_contra, _T_co], y: _T_contra, /) -> _T_co: ... +@overload +def divmod(x: _T_contra, y: SupportsRDivMod[_T_contra, _T_co], /) -> _T_co: ... + +# The `globals` argument to `eval` has to be `dict[str, Any]` rather than `dict[str, object]` due to invariance. +# (The `globals` argument has to be a "real dict", rather than any old mapping, unlike the `locals` argument.) +if sys.version_info >= (3, 13): + def eval( + source: str | ReadableBuffer | CodeType, + /, + globals: dict[str, Any] | None = None, + locals: Mapping[str, object] | None = None, + ) -> Any: ... + +else: + def eval( + source: str | ReadableBuffer | CodeType, + globals: dict[str, Any] | None = None, + locals: Mapping[str, object] | None = None, + /, + ) -> Any: ... + +# Comment above regarding `eval` applies to `exec` as well +if sys.version_info >= (3, 13): + def exec( + source: str | ReadableBuffer | CodeType, + /, + globals: dict[str, Any] | None = None, + locals: Mapping[str, object] | None = None, + *, + closure: tuple[CellType, ...] | None = None, + ) -> None: ... + +elif sys.version_info >= (3, 11): + def exec( + source: str | ReadableBuffer | CodeType, + globals: dict[str, Any] | None = None, + locals: Mapping[str, object] | None = None, + /, + *, + closure: tuple[CellType, ...] | None = None, + ) -> None: ... + +else: + def exec( + source: str | ReadableBuffer | CodeType, + globals: dict[str, Any] | None = None, + locals: Mapping[str, object] | None = None, + /, + ) -> None: ... + +def exit(code: sys._ExitCode = None) -> NoReturn: ... + +class filter(Generic[_T]): + @overload + def __new__(cls, function: None, iterable: Iterable[_T | None], /) -> Self: ... + @overload + def __new__(cls, function: Callable[[_S], TypeGuard[_T]], iterable: Iterable[_S], /) -> Self: ... + @overload + def __new__(cls, function: Callable[[_S], TypeIs[_T]], iterable: Iterable[_S], /) -> Self: ... + @overload + def __new__(cls, function: Callable[[_T], Any], iterable: Iterable[_T], /) -> Self: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + +def format(value: object, format_spec: str = "", /) -> str: ... +@overload +def getattr(o: object, name: str, /) -> Any: ... + +# While technically covered by the last overload, spelling out the types for None, bool +# and basic containers help mypy out in some tricky situations involving type context +# (aka bidirectional inference) +@overload +def getattr(o: object, name: str, default: None, /) -> Any | None: ... +@overload +def getattr(o: object, name: str, default: bool, /) -> Any | bool: ... +@overload +def getattr(o: object, name: str, default: list[Any], /) -> Any | list[Any]: ... +@overload +def getattr(o: object, name: str, default: dict[Any, Any], /) -> Any | dict[Any, Any]: ... +@overload +def getattr(o: object, name: str, default: _T, /) -> Any | _T: ... +def globals() -> dict[str, Any]: ... +def hasattr(obj: object, name: str, /) -> bool: ... +def hash(obj: object, /) -> int: ... +def help(request: object = ...) -> None: ... +def hex(number: int | SupportsIndex, /) -> str: ... +def id(obj: object, /) -> int: ... +def input(prompt: object = "", /) -> str: ... + +class _GetItemIterable(Protocol[_T_co]): + def __getitem__(self, i: int, /) -> _T_co: ... + +@overload +def iter(object: SupportsIter[_SupportsNextT], /) -> _SupportsNextT: ... +@overload +def iter(object: _GetItemIterable[_T], /) -> Iterator[_T]: ... +@overload +def iter(object: Callable[[], _T | None], sentinel: None, /) -> Iterator[_T]: ... +@overload +def iter(object: Callable[[], _T], sentinel: object, /) -> Iterator[_T]: ... + +# Keep this alias in sync with unittest.case._ClassInfo +if sys.version_info >= (3, 10): + _ClassInfo: TypeAlias = type | types.UnionType | tuple[_ClassInfo, ...] +else: + _ClassInfo: TypeAlias = type | tuple[_ClassInfo, ...] + +def isinstance(obj: object, class_or_tuple: _ClassInfo, /) -> bool: ... +def issubclass(cls: type, class_or_tuple: _ClassInfo, /) -> bool: ... +def len(obj: Sized, /) -> int: ... +def license() -> None: ... +def locals() -> dict[str, Any]: ... + +class map(Generic[_S]): + @overload + def __new__(cls, func: Callable[[_T1], _S], iter1: Iterable[_T1], /) -> Self: ... + @overload + def __new__(cls, func: Callable[[_T1, _T2], _S], iter1: Iterable[_T1], iter2: Iterable[_T2], /) -> Self: ... + @overload + def __new__( + cls, func: Callable[[_T1, _T2, _T3], _S], iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], / + ) -> Self: ... + @overload + def __new__( + cls, + func: Callable[[_T1, _T2, _T3, _T4], _S], + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + /, + ) -> Self: ... + @overload + def __new__( + cls, + func: Callable[[_T1, _T2, _T3, _T4, _T5], _S], + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + /, + ) -> Self: ... + @overload + def __new__( + cls, + func: Callable[..., _S], + iter1: Iterable[Any], + iter2: Iterable[Any], + iter3: Iterable[Any], + iter4: Iterable[Any], + iter5: Iterable[Any], + iter6: Iterable[Any], + /, + *iterables: Iterable[Any], + ) -> Self: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _S: ... + +@overload +def max( + arg1: SupportsRichComparisonT, arg2: SupportsRichComparisonT, /, *_args: SupportsRichComparisonT, key: None = None +) -> SupportsRichComparisonT: ... +@overload +def max(arg1: _T, arg2: _T, /, *_args: _T, key: Callable[[_T], SupportsRichComparison]) -> _T: ... +@overload +def max(iterable: Iterable[SupportsRichComparisonT], /, *, key: None = None) -> SupportsRichComparisonT: ... +@overload +def max(iterable: Iterable[_T], /, *, key: Callable[[_T], SupportsRichComparison]) -> _T: ... +@overload +def max(iterable: Iterable[SupportsRichComparisonT], /, *, key: None = None, default: _T) -> SupportsRichComparisonT | _T: ... +@overload +def max(iterable: Iterable[_T1], /, *, key: Callable[[_T1], SupportsRichComparison], default: _T2) -> _T1 | _T2: ... +@overload +def min( + arg1: SupportsRichComparisonT, arg2: SupportsRichComparisonT, /, *_args: SupportsRichComparisonT, key: None = None +) -> SupportsRichComparisonT: ... +@overload +def min(arg1: _T, arg2: _T, /, *_args: _T, key: Callable[[_T], SupportsRichComparison]) -> _T: ... +@overload +def min(iterable: Iterable[SupportsRichComparisonT], /, *, key: None = None) -> SupportsRichComparisonT: ... +@overload +def min(iterable: Iterable[_T], /, *, key: Callable[[_T], SupportsRichComparison]) -> _T: ... +@overload +def min(iterable: Iterable[SupportsRichComparisonT], /, *, key: None = None, default: _T) -> SupportsRichComparisonT | _T: ... +@overload +def min(iterable: Iterable[_T1], /, *, key: Callable[[_T1], SupportsRichComparison], default: _T2) -> _T1 | _T2: ... +@overload +def next(i: SupportsNext[_T], /) -> _T: ... +@overload +def next(i: SupportsNext[_T], default: _VT, /) -> _T | _VT: ... +def oct(number: int | SupportsIndex, /) -> str: ... + +_Opener: TypeAlias = Callable[[str, int], int] + +# Text mode: always returns a TextIOWrapper +@overload +def open( + file: FileDescriptorOrPath, + mode: OpenTextMode = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> TextIOWrapper: ... + +# Unbuffered binary mode: returns a FileIO +@overload +def open( + file: FileDescriptorOrPath, + mode: OpenBinaryMode, + buffering: Literal[0], + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> FileIO: ... + +# Buffering is on: return BufferedRandom, BufferedReader, or BufferedWriter +@overload +def open( + file: FileDescriptorOrPath, + mode: OpenBinaryModeUpdating, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> BufferedRandom: ... +@overload +def open( + file: FileDescriptorOrPath, + mode: OpenBinaryModeWriting, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> BufferedWriter: ... +@overload +def open( + file: FileDescriptorOrPath, + mode: OpenBinaryModeReading, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> BufferedReader: ... + +# Buffering cannot be determined: fall back to BinaryIO +@overload +def open( + file: FileDescriptorOrPath, + mode: OpenBinaryMode, + buffering: int = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> BinaryIO: ... + +# Fallback if mode is not specified +@overload +def open( + file: FileDescriptorOrPath, + mode: str, + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> IO[Any]: ... +def ord(c: str | bytes | bytearray, /) -> int: ... + +class _SupportsWriteAndFlush(SupportsWrite[_T_contra], SupportsFlush, Protocol[_T_contra]): ... + +@overload +def print( + *values: object, + sep: str | None = " ", + end: str | None = "\n", + file: SupportsWrite[str] | None = None, + flush: Literal[False] = False, +) -> None: ... +@overload +def print( + *values: object, sep: str | None = " ", end: str | None = "\n", file: _SupportsWriteAndFlush[str] | None = None, flush: bool +) -> None: ... + +_E = TypeVar("_E", contravariant=True) +_M = TypeVar("_M", contravariant=True) + +class _SupportsPow2(Protocol[_E, _T_co]): + def __pow__(self, other: _E, /) -> _T_co: ... + +class _SupportsPow3NoneOnly(Protocol[_E, _T_co]): + def __pow__(self, other: _E, modulo: None = None, /) -> _T_co: ... + +class _SupportsPow3(Protocol[_E, _M, _T_co]): + def __pow__(self, other: _E, modulo: _M, /) -> _T_co: ... + +_SupportsSomeKindOfPow = ( # noqa: Y026 # TODO: Use TypeAlias once mypy bugs are fixed + _SupportsPow2[Any, Any] | _SupportsPow3NoneOnly[Any, Any] | _SupportsPow3[Any, Any, Any] +) + +# TODO: `pow(int, int, Literal[0])` fails at runtime, +# but adding a `NoReturn` overload isn't a good solution for expressing that (see #8566). +@overload +def pow(base: int, exp: int, mod: int) -> int: ... +@overload +def pow(base: int, exp: Literal[0], mod: None = None) -> Literal[1]: ... +@overload +def pow(base: int, exp: _PositiveInteger, mod: None = None) -> int: ... +@overload +def pow(base: int, exp: _NegativeInteger, mod: None = None) -> float: ... + +# int base & positive-int exp -> int; int base & negative-int exp -> float +# return type must be Any as `int | float` causes too many false-positive errors +@overload +def pow(base: int, exp: int, mod: None = None) -> Any: ... +@overload +def pow(base: _PositiveInteger, exp: float, mod: None = None) -> float: ... +@overload +def pow(base: _NegativeInteger, exp: float, mod: None = None) -> complex: ... +@overload +def pow(base: float, exp: int, mod: None = None) -> float: ... + +# float base & float exp could return float or complex +# return type must be Any (same as complex base, complex exp), +# as `float | complex` causes too many false-positive errors +@overload +def pow(base: float, exp: complex | _SupportsSomeKindOfPow, mod: None = None) -> Any: ... +@overload +def pow(base: complex, exp: complex | _SupportsSomeKindOfPow, mod: None = None) -> complex: ... +@overload +def pow(base: _SupportsPow2[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... # type: ignore[overload-overlap] +@overload +def pow(base: _SupportsPow3NoneOnly[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... # type: ignore[overload-overlap] +@overload +def pow(base: _SupportsPow3[_E, _M, _T_co], exp: _E, mod: _M) -> _T_co: ... +@overload +def pow(base: _SupportsSomeKindOfPow, exp: float, mod: None = None) -> Any: ... +@overload +def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex: ... +def quit(code: sys._ExitCode = None) -> NoReturn: ... + +class reversed(Generic[_T]): + @overload + def __new__(cls, sequence: Reversible[_T], /) -> Iterator[_T]: ... # type: ignore[misc] + @overload + def __new__(cls, sequence: SupportsLenAndGetItem[_T], /) -> Iterator[_T]: ... # type: ignore[misc] + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + def __length_hint__(self) -> int: ... + +def repr(obj: object, /) -> str: ... + +# See https://github.com/python/typeshed/pull/9141 +# and https://github.com/python/typeshed/pull/9151 +# on why we don't use `SupportsRound` from `typing.pyi` + +class _SupportsRound1(Protocol[_T_co]): + def __round__(self) -> _T_co: ... + +class _SupportsRound2(Protocol[_T_co]): + def __round__(self, ndigits: int, /) -> _T_co: ... + +@overload +def round(number: _SupportsRound1[_T], ndigits: None = None) -> _T: ... +@overload +def round(number: _SupportsRound2[_T], ndigits: SupportsIndex) -> _T: ... + +# See https://github.com/python/typeshed/pull/6292#discussion_r748875189 +# for why arg 3 of `setattr` should be annotated with `Any` and not `object` +def setattr(obj: object, name: str, value: Any, /) -> None: ... +@overload +def sorted( + iterable: Iterable[SupportsRichComparisonT], /, *, key: None = None, reverse: bool = False +) -> list[SupportsRichComparisonT]: ... +@overload +def sorted(iterable: Iterable[_T], /, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = False) -> list[_T]: ... + +_AddableT1 = TypeVar("_AddableT1", bound=SupportsAdd[Any, Any]) +_AddableT2 = TypeVar("_AddableT2", bound=SupportsAdd[Any, Any]) + +class _SupportsSumWithNoDefaultGiven(SupportsAdd[Any, Any], SupportsRAdd[int, Any], Protocol): ... + +_SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWithNoDefaultGiven) + +# In general, the return type of `x + x` is *not* guaranteed to be the same type as x. +# However, we can't express that in the stub for `sum()` +# without creating many false-positive errors (see #7578). +# Instead, we special-case the most common examples of this: bool and literal integers. +@overload +def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ... +@overload +def sum(iterable: Iterable[_SupportsSumNoDefaultT], /) -> _SupportsSumNoDefaultT | Literal[0]: ... +@overload +def sum(iterable: Iterable[_AddableT1], /, start: _AddableT2) -> _AddableT1 | _AddableT2: ... + +# The argument to `vars()` has to have a `__dict__` attribute, so the second overload can't be annotated with `object` +# (A "SupportsDunderDict" protocol doesn't work) +@overload +def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... +@overload +def vars(object: Any = ..., /) -> dict[str, Any]: ... + +class zip(Generic[_T_co]): + if sys.version_info >= (3, 10): + @overload + def __new__(cls, *, strict: bool = ...) -> zip[Any]: ... + @overload + def __new__(cls, iter1: Iterable[_T1], /, *, strict: bool = ...) -> zip[tuple[_T1]]: ... + @overload + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /, *, strict: bool = ...) -> zip[tuple[_T1, _T2]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /, *, strict: bool = ... + ) -> zip[tuple[_T1, _T2, _T3]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], /, *, strict: bool = ... + ) -> zip[tuple[_T1, _T2, _T3, _T4]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + /, + *, + strict: bool = ..., + ) -> zip[tuple[_T1, _T2, _T3, _T4, _T5]]: ... + @overload + def __new__( + cls, + iter1: Iterable[Any], + iter2: Iterable[Any], + iter3: Iterable[Any], + iter4: Iterable[Any], + iter5: Iterable[Any], + iter6: Iterable[Any], + /, + *iterables: Iterable[Any], + strict: bool = ..., + ) -> zip[tuple[Any, ...]]: ... + else: + @overload + def __new__(cls) -> zip[Any]: ... + @overload + def __new__(cls, iter1: Iterable[_T1], /) -> zip[tuple[_T1]]: ... + @overload + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /) -> zip[tuple[_T1, _T2]]: ... + @overload + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /) -> zip[tuple[_T1, _T2, _T3]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], / + ) -> zip[tuple[_T1, _T2, _T3, _T4]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], iter5: Iterable[_T5], / + ) -> zip[tuple[_T1, _T2, _T3, _T4, _T5]]: ... + @overload + def __new__( + cls, + iter1: Iterable[Any], + iter2: Iterable[Any], + iter3: Iterable[Any], + iter4: Iterable[Any], + iter5: Iterable[Any], + iter6: Iterable[Any], + /, + *iterables: Iterable[Any], + ) -> zip[tuple[Any, ...]]: ... + + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +# Signature of `builtins.__import__` should be kept identical to `importlib.__import__` +# Return type of `__import__` should be kept the same as return type of `importlib.import_module` +def __import__( + name: str, + globals: Mapping[str, object] | None = None, + locals: Mapping[str, object] | None = None, + fromlist: Sequence[str] = (), + level: int = 0, +) -> types.ModuleType: ... +def __build_class__(func: Callable[[], CellType | Any], name: str, /, *bases: Any, metaclass: Any = ..., **kwds: Any) -> Any: ... + +if sys.version_info >= (3, 10): + from types import EllipsisType + + # Backwards compatibility hack for folks who relied on the ellipsis type + # existing in typeshed in Python 3.9 and earlier. + ellipsis = EllipsisType + + Ellipsis: EllipsisType + +else: + # Actually the type of Ellipsis is , but since it's + # not exposed anywhere under that name, we make it private here. + @final + @type_check_only + class ellipsis: ... + + Ellipsis: ellipsis + +class BaseException: + args: tuple[Any, ...] + __cause__: BaseException | None + __context__: BaseException | None + __suppress_context__: bool + __traceback__: TracebackType | None + def __init__(self, *args: object) -> None: ... + def __new__(cls, *args: Any, **kwds: Any) -> Self: ... + def __setstate__(self, state: dict[str, Any] | None, /) -> None: ... + def with_traceback(self, tb: TracebackType | None, /) -> Self: ... + if sys.version_info >= (3, 11): + # only present after add_note() is called + __notes__: list[str] + def add_note(self, note: str, /) -> None: ... + +class GeneratorExit(BaseException): ... +class KeyboardInterrupt(BaseException): ... + +class SystemExit(BaseException): + code: sys._ExitCode + +class Exception(BaseException): ... + +class StopIteration(Exception): + value: Any + +class OSError(Exception): + errno: int | None + strerror: str | None + # filename, filename2 are actually str | bytes | None + filename: Any + filename2: Any + if sys.platform == "win32": + winerror: int + +EnvironmentError = OSError +IOError = OSError +if sys.platform == "win32": + WindowsError = OSError + +class ArithmeticError(Exception): ... +class AssertionError(Exception): ... + +class AttributeError(Exception): + if sys.version_info >= (3, 10): + def __init__(self, *args: object, name: str | None = ..., obj: object = ...) -> None: ... + name: str + obj: object + +class BufferError(Exception): ... +class EOFError(Exception): ... + +class ImportError(Exception): + def __init__(self, *args: object, name: str | None = ..., path: str | None = ...) -> None: ... + name: str | None + path: str | None + msg: str # undocumented + if sys.version_info >= (3, 12): + name_from: str | None # undocumented + +class LookupError(Exception): ... +class MemoryError(Exception): ... + +class NameError(Exception): + if sys.version_info >= (3, 10): + def __init__(self, *args: object, name: str | None = ...) -> None: ... + name: str + +class ReferenceError(Exception): ... +class RuntimeError(Exception): ... + +class StopAsyncIteration(Exception): + value: Any + +class SyntaxError(Exception): + msg: str + filename: str | None + lineno: int | None + offset: int | None + text: str | None + # Errors are displayed differently if this attribute exists on the exception. + # The value is always None. + print_file_and_line: None + if sys.version_info >= (3, 10): + end_lineno: int | None + end_offset: int | None + + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, msg: object, /) -> None: ... + # Second argument is the tuple (filename, lineno, offset, text) + @overload + def __init__(self, msg: str, info: tuple[str | None, int | None, int | None, str | None], /) -> None: ... + if sys.version_info >= (3, 10): + # end_lineno and end_offset must both be provided if one is. + @overload + def __init__( + self, msg: str, info: tuple[str | None, int | None, int | None, str | None, int | None, int | None], / + ) -> None: ... + # If you provide more than two arguments, it still creates the SyntaxError, but + # the arguments from the info tuple are not parsed. This form is omitted. + +class SystemError(Exception): ... +class TypeError(Exception): ... +class ValueError(Exception): ... +class FloatingPointError(ArithmeticError): ... +class OverflowError(ArithmeticError): ... +class ZeroDivisionError(ArithmeticError): ... +class ModuleNotFoundError(ImportError): ... +class IndexError(LookupError): ... +class KeyError(LookupError): ... +class UnboundLocalError(NameError): ... + +class BlockingIOError(OSError): + characters_written: int + +class ChildProcessError(OSError): ... +class ConnectionError(OSError): ... +class BrokenPipeError(ConnectionError): ... +class ConnectionAbortedError(ConnectionError): ... +class ConnectionRefusedError(ConnectionError): ... +class ConnectionResetError(ConnectionError): ... +class FileExistsError(OSError): ... +class FileNotFoundError(OSError): ... +class InterruptedError(OSError): ... +class IsADirectoryError(OSError): ... +class NotADirectoryError(OSError): ... +class PermissionError(OSError): ... +class ProcessLookupError(OSError): ... +class TimeoutError(OSError): ... +class NotImplementedError(RuntimeError): ... +class RecursionError(RuntimeError): ... +class IndentationError(SyntaxError): ... +class TabError(IndentationError): ... +class UnicodeError(ValueError): ... + +class UnicodeDecodeError(UnicodeError): + encoding: str + object: bytes + start: int + end: int + reason: str + def __init__(self, encoding: str, object: ReadableBuffer, start: int, end: int, reason: str, /) -> None: ... + +class UnicodeEncodeError(UnicodeError): + encoding: str + object: str + start: int + end: int + reason: str + def __init__(self, encoding: str, object: str, start: int, end: int, reason: str, /) -> None: ... + +class UnicodeTranslateError(UnicodeError): + encoding: None + object: str + start: int + end: int + reason: str + def __init__(self, object: str, start: int, end: int, reason: str, /) -> None: ... + +class Warning(Exception): ... +class UserWarning(Warning): ... +class DeprecationWarning(Warning): ... +class SyntaxWarning(Warning): ... +class RuntimeWarning(Warning): ... +class FutureWarning(Warning): ... +class PendingDeprecationWarning(Warning): ... +class ImportWarning(Warning): ... +class UnicodeWarning(Warning): ... +class BytesWarning(Warning): ... +class ResourceWarning(Warning): ... + +if sys.version_info >= (3, 10): + class EncodingWarning(Warning): ... + +if sys.version_info >= (3, 11): + _BaseExceptionT_co = TypeVar("_BaseExceptionT_co", bound=BaseException, covariant=True, default=BaseException) + _BaseExceptionT = TypeVar("_BaseExceptionT", bound=BaseException) + _ExceptionT_co = TypeVar("_ExceptionT_co", bound=Exception, covariant=True, default=Exception) + _ExceptionT = TypeVar("_ExceptionT", bound=Exception) + + # See `check_exception_group.py` for use-cases and comments. + class BaseExceptionGroup(BaseException, Generic[_BaseExceptionT_co]): + def __new__(cls, message: str, exceptions: Sequence[_BaseExceptionT_co], /) -> Self: ... + def __init__(self, message: str, exceptions: Sequence[_BaseExceptionT_co], /) -> None: ... + @property + def message(self) -> str: ... + @property + def exceptions(self) -> tuple[_BaseExceptionT_co | BaseExceptionGroup[_BaseExceptionT_co], ...]: ... + @overload + def subgroup( + self, condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...], / + ) -> ExceptionGroup[_ExceptionT] | None: ... + @overload + def subgroup( + self, condition: type[_BaseExceptionT] | tuple[type[_BaseExceptionT], ...], / + ) -> BaseExceptionGroup[_BaseExceptionT] | None: ... + @overload + def subgroup( + self, condition: Callable[[_BaseExceptionT_co | Self], bool], / + ) -> BaseExceptionGroup[_BaseExceptionT_co] | None: ... + @overload + def split( + self, condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...], / + ) -> tuple[ExceptionGroup[_ExceptionT] | None, BaseExceptionGroup[_BaseExceptionT_co] | None]: ... + @overload + def split( + self, condition: type[_BaseExceptionT] | tuple[type[_BaseExceptionT], ...], / + ) -> tuple[BaseExceptionGroup[_BaseExceptionT] | None, BaseExceptionGroup[_BaseExceptionT_co] | None]: ... + @overload + def split( + self, condition: Callable[[_BaseExceptionT_co | Self], bool], / + ) -> tuple[BaseExceptionGroup[_BaseExceptionT_co] | None, BaseExceptionGroup[_BaseExceptionT_co] | None]: ... + # In reality it is `NonEmptySequence`: + @overload + def derive(self, excs: Sequence[_ExceptionT], /) -> ExceptionGroup[_ExceptionT]: ... + @overload + def derive(self, excs: Sequence[_BaseExceptionT], /) -> BaseExceptionGroup[_BaseExceptionT]: ... + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + + class ExceptionGroup(BaseExceptionGroup[_ExceptionT_co], Exception): + def __new__(cls, message: str, exceptions: Sequence[_ExceptionT_co], /) -> Self: ... + def __init__(self, message: str, exceptions: Sequence[_ExceptionT_co], /) -> None: ... + @property + def exceptions(self) -> tuple[_ExceptionT_co | ExceptionGroup[_ExceptionT_co], ...]: ... + # We accept a narrower type, but that's OK. + @overload # type: ignore[override] + def subgroup( + self, condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...], / + ) -> ExceptionGroup[_ExceptionT] | None: ... + @overload + def subgroup(self, condition: Callable[[_ExceptionT_co | Self], bool], /) -> ExceptionGroup[_ExceptionT_co] | None: ... + @overload # type: ignore[override] + def split( + self, condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...], / + ) -> tuple[ExceptionGroup[_ExceptionT] | None, ExceptionGroup[_ExceptionT_co] | None]: ... + @overload + def split( + self, condition: Callable[[_ExceptionT_co | Self], bool], / + ) -> tuple[ExceptionGroup[_ExceptionT_co] | None, ExceptionGroup[_ExceptionT_co] | None]: ... + +if sys.version_info >= (3, 13): + class PythonFinalizationError(RuntimeError): ... diff --git a/mypy/typeshed/stdlib/bz2.pyi b/mypy/typeshed/stdlib/bz2.pyi new file mode 100644 index 000000000000..2f869f9697f4 --- /dev/null +++ b/mypy/typeshed/stdlib/bz2.pyi @@ -0,0 +1,132 @@ +import _compression +import sys +from _bz2 import BZ2Compressor as BZ2Compressor, BZ2Decompressor as BZ2Decompressor +from _compression import BaseStream +from _typeshed import ReadableBuffer, StrOrBytesPath, WriteableBuffer +from collections.abc import Iterable +from typing import IO, Any, Literal, Protocol, SupportsIndex, TextIO, overload +from typing_extensions import Self, TypeAlias + +__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", "open", "compress", "decompress"] + +# The following attributes and methods are optional: +# def fileno(self) -> int: ... +# def close(self) -> object: ... +class _ReadableFileobj(_compression._Reader, Protocol): ... + +class _WritableFileobj(Protocol): + def write(self, b: bytes, /) -> object: ... + # The following attributes and methods are optional: + # def fileno(self) -> int: ... + # def close(self) -> object: ... + +def compress(data: ReadableBuffer, compresslevel: int = 9) -> bytes: ... +def decompress(data: ReadableBuffer) -> bytes: ... + +_ReadBinaryMode: TypeAlias = Literal["", "r", "rb"] +_WriteBinaryMode: TypeAlias = Literal["w", "wb", "x", "xb", "a", "ab"] +_ReadTextMode: TypeAlias = Literal["rt"] +_WriteTextMode: TypeAlias = Literal["wt", "xt", "at"] + +@overload +def open( + filename: _ReadableFileobj, + mode: _ReadBinaryMode = "rb", + compresslevel: int = 9, + encoding: None = None, + errors: None = None, + newline: None = None, +) -> BZ2File: ... +@overload +def open( + filename: _ReadableFileobj, + mode: _ReadTextMode, + compresslevel: int = 9, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIO: ... +@overload +def open( + filename: _WritableFileobj, + mode: _WriteBinaryMode, + compresslevel: int = 9, + encoding: None = None, + errors: None = None, + newline: None = None, +) -> BZ2File: ... +@overload +def open( + filename: _WritableFileobj, + mode: _WriteTextMode, + compresslevel: int = 9, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIO: ... +@overload +def open( + filename: StrOrBytesPath, + mode: _ReadBinaryMode | _WriteBinaryMode = "rb", + compresslevel: int = 9, + encoding: None = None, + errors: None = None, + newline: None = None, +) -> BZ2File: ... +@overload +def open( + filename: StrOrBytesPath, + mode: _ReadTextMode | _WriteTextMode, + compresslevel: int = 9, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIO: ... +@overload +def open( + filename: StrOrBytesPath | _ReadableFileobj | _WritableFileobj, + mode: str, + compresslevel: int = 9, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> BZ2File | TextIO: ... + +class BZ2File(BaseStream, IO[bytes]): + def __enter__(self) -> Self: ... + if sys.version_info >= (3, 9): + @overload + def __init__(self, filename: _WritableFileobj, mode: _WriteBinaryMode, *, compresslevel: int = 9) -> None: ... + @overload + def __init__(self, filename: _ReadableFileobj, mode: _ReadBinaryMode = "r", *, compresslevel: int = 9) -> None: ... + @overload + def __init__( + self, filename: StrOrBytesPath, mode: _ReadBinaryMode | _WriteBinaryMode = "r", *, compresslevel: int = 9 + ) -> None: ... + else: + @overload + def __init__( + self, filename: _WritableFileobj, mode: _WriteBinaryMode, buffering: Any | None = None, compresslevel: int = 9 + ) -> None: ... + @overload + def __init__( + self, filename: _ReadableFileobj, mode: _ReadBinaryMode = "r", buffering: Any | None = None, compresslevel: int = 9 + ) -> None: ... + @overload + def __init__( + self, + filename: StrOrBytesPath, + mode: _ReadBinaryMode | _WriteBinaryMode = "r", + buffering: Any | None = None, + compresslevel: int = 9, + ) -> None: ... + + def read(self, size: int | None = -1) -> bytes: ... + def read1(self, size: int = -1) -> bytes: ... + def readline(self, size: SupportsIndex = -1) -> bytes: ... # type: ignore[override] + def readinto(self, b: WriteableBuffer) -> int: ... + def readlines(self, size: SupportsIndex = -1) -> list[bytes]: ... + def peek(self, n: int = 0) -> bytes: ... + def seek(self, offset: int, whence: int = 0) -> int: ... + def write(self, data: ReadableBuffer) -> int: ... + def writelines(self, seq: Iterable[ReadableBuffer]) -> None: ... diff --git a/mypy/typeshed/stdlib/cProfile.pyi b/mypy/typeshed/stdlib/cProfile.pyi new file mode 100644 index 000000000000..e921584d4390 --- /dev/null +++ b/mypy/typeshed/stdlib/cProfile.pyi @@ -0,0 +1,31 @@ +import _lsprof +from _typeshed import StrOrBytesPath, Unused +from collections.abc import Callable, Mapping +from types import CodeType +from typing import Any, TypeVar +from typing_extensions import ParamSpec, Self, TypeAlias + +__all__ = ["run", "runctx", "Profile"] + +def run(statement: str, filename: str | None = None, sort: str | int = -1) -> None: ... +def runctx( + statement: str, globals: dict[str, Any], locals: Mapping[str, Any], filename: str | None = None, sort: str | int = -1 +) -> None: ... + +_T = TypeVar("_T") +_P = ParamSpec("_P") +_Label: TypeAlias = tuple[str, int, str] + +class Profile(_lsprof.Profiler): + stats: dict[_Label, tuple[int, int, int, int, dict[_Label, tuple[int, int, int, int]]]] # undocumented + def print_stats(self, sort: str | int = -1) -> None: ... + def dump_stats(self, file: StrOrBytesPath) -> None: ... + def create_stats(self) -> None: ... + def snapshot_stats(self) -> None: ... + def run(self, cmd: str) -> Self: ... + def runctx(self, cmd: str, globals: dict[str, Any], locals: Mapping[str, Any]) -> Self: ... + def runcall(self, func: Callable[_P, _T], /, *args: _P.args, **kw: _P.kwargs) -> _T: ... + def __enter__(self) -> Self: ... + def __exit__(self, *exc_info: Unused) -> None: ... + +def label(code: str | CodeType) -> _Label: ... # undocumented diff --git a/mypy/typeshed/stdlib/calendar.pyi b/mypy/typeshed/stdlib/calendar.pyi new file mode 100644 index 000000000000..cabf3b881c30 --- /dev/null +++ b/mypy/typeshed/stdlib/calendar.pyi @@ -0,0 +1,208 @@ +import datetime +import enum +import sys +from _typeshed import Unused +from collections.abc import Iterable, Sequence +from time import struct_time +from typing import ClassVar, Final +from typing_extensions import TypeAlias + +__all__ = [ + "IllegalMonthError", + "IllegalWeekdayError", + "setfirstweekday", + "firstweekday", + "isleap", + "leapdays", + "weekday", + "monthrange", + "monthcalendar", + "prmonth", + "month", + "prcal", + "calendar", + "timegm", + "month_name", + "month_abbr", + "day_name", + "day_abbr", + "Calendar", + "TextCalendar", + "HTMLCalendar", + "LocaleTextCalendar", + "LocaleHTMLCalendar", + "weekheader", +] + +if sys.version_info >= (3, 10): + __all__ += ["FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"] +if sys.version_info >= (3, 12): + __all__ += [ + "Day", + "Month", + "JANUARY", + "FEBRUARY", + "MARCH", + "APRIL", + "MAY", + "JUNE", + "JULY", + "AUGUST", + "SEPTEMBER", + "OCTOBER", + "NOVEMBER", + "DECEMBER", + ] + +_LocaleType: TypeAlias = tuple[str | None, str | None] + +class IllegalMonthError(ValueError): + def __init__(self, month: int) -> None: ... + +class IllegalWeekdayError(ValueError): + def __init__(self, weekday: int) -> None: ... + +def isleap(year: int) -> bool: ... +def leapdays(y1: int, y2: int) -> int: ... +def weekday(year: int, month: int, day: int) -> int: ... +def monthrange(year: int, month: int) -> tuple[int, int]: ... + +class Calendar: + firstweekday: int + def __init__(self, firstweekday: int = 0) -> None: ... + def getfirstweekday(self) -> int: ... + def setfirstweekday(self, firstweekday: int) -> None: ... + def iterweekdays(self) -> Iterable[int]: ... + def itermonthdates(self, year: int, month: int) -> Iterable[datetime.date]: ... + def itermonthdays2(self, year: int, month: int) -> Iterable[tuple[int, int]]: ... + def itermonthdays(self, year: int, month: int) -> Iterable[int]: ... + def monthdatescalendar(self, year: int, month: int) -> list[list[datetime.date]]: ... + def monthdays2calendar(self, year: int, month: int) -> list[list[tuple[int, int]]]: ... + def monthdayscalendar(self, year: int, month: int) -> list[list[int]]: ... + def yeardatescalendar(self, year: int, width: int = 3) -> list[list[list[list[datetime.date]]]]: ... + def yeardays2calendar(self, year: int, width: int = 3) -> list[list[list[list[tuple[int, int]]]]]: ... + def yeardayscalendar(self, year: int, width: int = 3) -> list[list[list[list[int]]]]: ... + def itermonthdays3(self, year: int, month: int) -> Iterable[tuple[int, int, int]]: ... + def itermonthdays4(self, year: int, month: int) -> Iterable[tuple[int, int, int, int]]: ... + +class TextCalendar(Calendar): + def prweek(self, theweek: int, width: int) -> None: ... + def formatday(self, day: int, weekday: int, width: int) -> str: ... + def formatweek(self, theweek: int, width: int) -> str: ... + def formatweekday(self, day: int, width: int) -> str: ... + def formatweekheader(self, width: int) -> str: ... + def formatmonthname(self, theyear: int, themonth: int, width: int, withyear: bool = True) -> str: ... + def prmonth(self, theyear: int, themonth: int, w: int = 0, l: int = 0) -> None: ... + def formatmonth(self, theyear: int, themonth: int, w: int = 0, l: int = 0) -> str: ... + def formatyear(self, theyear: int, w: int = 2, l: int = 1, c: int = 6, m: int = 3) -> str: ... + def pryear(self, theyear: int, w: int = 0, l: int = 0, c: int = 6, m: int = 3) -> None: ... + +def firstweekday() -> int: ... +def monthcalendar(year: int, month: int) -> list[list[int]]: ... +def prweek(theweek: int, width: int) -> None: ... +def week(theweek: int, width: int) -> str: ... +def weekheader(width: int) -> str: ... +def prmonth(theyear: int, themonth: int, w: int = 0, l: int = 0) -> None: ... +def month(theyear: int, themonth: int, w: int = 0, l: int = 0) -> str: ... +def calendar(theyear: int, w: int = 2, l: int = 1, c: int = 6, m: int = 3) -> str: ... +def prcal(theyear: int, w: int = 0, l: int = 0, c: int = 6, m: int = 3) -> None: ... + +class HTMLCalendar(Calendar): + cssclasses: ClassVar[list[str]] + cssclass_noday: ClassVar[str] + cssclasses_weekday_head: ClassVar[list[str]] + cssclass_month_head: ClassVar[str] + cssclass_month: ClassVar[str] + cssclass_year: ClassVar[str] + cssclass_year_head: ClassVar[str] + def formatday(self, day: int, weekday: int) -> str: ... + def formatweek(self, theweek: int) -> str: ... + def formatweekday(self, day: int) -> str: ... + def formatweekheader(self) -> str: ... + def formatmonthname(self, theyear: int, themonth: int, withyear: bool = True) -> str: ... + def formatmonth(self, theyear: int, themonth: int, withyear: bool = True) -> str: ... + def formatyear(self, theyear: int, width: int = 3) -> str: ... + def formatyearpage( + self, theyear: int, width: int = 3, css: str | None = "calendar.css", encoding: str | None = None + ) -> bytes: ... + +class different_locale: + def __init__(self, locale: _LocaleType) -> None: ... + def __enter__(self) -> None: ... + def __exit__(self, *args: Unused) -> None: ... + +class LocaleTextCalendar(TextCalendar): + def __init__(self, firstweekday: int = 0, locale: _LocaleType | None = None) -> None: ... + +class LocaleHTMLCalendar(HTMLCalendar): + def __init__(self, firstweekday: int = 0, locale: _LocaleType | None = None) -> None: ... + def formatweekday(self, day: int) -> str: ... + def formatmonthname(self, theyear: int, themonth: int, withyear: bool = True) -> str: ... + +c: TextCalendar + +def setfirstweekday(firstweekday: int) -> None: ... +def format(cols: int, colwidth: int = 20, spacing: int = 6) -> str: ... +def formatstring(cols: int, colwidth: int = 20, spacing: int = 6) -> str: ... +def timegm(tuple: tuple[int, ...] | struct_time) -> int: ... + +# Data attributes +day_name: Sequence[str] +day_abbr: Sequence[str] +month_name: Sequence[str] +month_abbr: Sequence[str] + +if sys.version_info >= (3, 12): + class Month(enum.IntEnum): + JANUARY = 1 + FEBRUARY = 2 + MARCH = 3 + APRIL = 4 + MAY = 5 + JUNE = 6 + JULY = 7 + AUGUST = 8 + SEPTEMBER = 9 + OCTOBER = 10 + NOVEMBER = 11 + DECEMBER = 12 + + JANUARY = Month.JANUARY + FEBRUARY = Month.FEBRUARY + MARCH = Month.MARCH + APRIL = Month.APRIL + MAY = Month.MAY + JUNE = Month.JUNE + JULY = Month.JULY + AUGUST = Month.AUGUST + SEPTEMBER = Month.SEPTEMBER + OCTOBER = Month.OCTOBER + NOVEMBER = Month.NOVEMBER + DECEMBER = Month.DECEMBER + + class Day(enum.IntEnum): + MONDAY = 0 + TUESDAY = 1 + WEDNESDAY = 2 + THURSDAY = 3 + FRIDAY = 4 + SATURDAY = 5 + SUNDAY = 6 + + MONDAY = Day.MONDAY + TUESDAY = Day.TUESDAY + WEDNESDAY = Day.WEDNESDAY + THURSDAY = Day.THURSDAY + FRIDAY = Day.FRIDAY + SATURDAY = Day.SATURDAY + SUNDAY = Day.SUNDAY +else: + MONDAY: Final = 0 + TUESDAY: Final = 1 + WEDNESDAY: Final = 2 + THURSDAY: Final = 3 + FRIDAY: Final = 4 + SATURDAY: Final = 5 + SUNDAY: Final = 6 + +EPOCH: Final = 1970 diff --git a/mypy/typeshed/stdlib/cgi.pyi b/mypy/typeshed/stdlib/cgi.pyi new file mode 100644 index 000000000000..3a2e2a91b241 --- /dev/null +++ b/mypy/typeshed/stdlib/cgi.pyi @@ -0,0 +1,118 @@ +from _typeshed import SupportsContainsAndGetItem, SupportsGetItem, SupportsItemAccess, Unused +from builtins import list as _list, type as _type +from collections.abc import Iterable, Iterator, Mapping +from email.message import Message +from types import TracebackType +from typing import IO, Any, Protocol +from typing_extensions import Self + +__all__ = [ + "MiniFieldStorage", + "FieldStorage", + "parse", + "parse_multipart", + "parse_header", + "test", + "print_exception", + "print_environ", + "print_form", + "print_directory", + "print_arguments", + "print_environ_usage", +] + +def parse( + fp: IO[Any] | None = None, + environ: SupportsItemAccess[str, str] = ..., + keep_blank_values: bool = ..., + strict_parsing: bool = ..., + separator: str = "&", +) -> dict[str, list[str]]: ... +def parse_multipart( + fp: IO[Any], pdict: SupportsGetItem[str, bytes], encoding: str = "utf-8", errors: str = "replace", separator: str = "&" +) -> dict[str, list[Any]]: ... + +class _Environ(Protocol): + def __getitem__(self, k: str, /) -> str: ... + def keys(self) -> Iterable[str]: ... + +def parse_header(line: str) -> tuple[str, dict[str, str]]: ... +def test(environ: _Environ = ...) -> None: ... +def print_environ(environ: _Environ = ...) -> None: ... +def print_form(form: dict[str, Any]) -> None: ... +def print_directory() -> None: ... +def print_environ_usage() -> None: ... + +class MiniFieldStorage: + # The first five "Any" attributes here are always None, but mypy doesn't support that + filename: Any + list: Any + type: Any + file: IO[bytes] | None + type_options: dict[Any, Any] + disposition: Any + disposition_options: dict[Any, Any] + headers: dict[Any, Any] + name: Any + value: Any + def __init__(self, name: Any, value: Any) -> None: ... + +class FieldStorage: + FieldStorageClass: _type | None + keep_blank_values: int + strict_parsing: int + qs_on_post: str | None + headers: Mapping[str, str] | Message + fp: IO[bytes] + encoding: str + errors: str + outerboundary: bytes + bytes_read: int + limit: int | None + disposition: str + disposition_options: dict[str, str] + filename: str | None + file: IO[bytes] | None + type: str + type_options: dict[str, str] + innerboundary: bytes + length: int + done: int + list: _list[Any] | None + value: None | bytes | _list[Any] + def __init__( + self, + fp: IO[Any] | None = None, + headers: Mapping[str, str] | Message | None = None, + outerboundary: bytes = b"", + environ: SupportsContainsAndGetItem[str, str] = ..., + keep_blank_values: int = 0, + strict_parsing: int = 0, + limit: int | None = None, + encoding: str = "utf-8", + errors: str = "replace", + max_num_fields: int | None = None, + separator: str = "&", + ) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def __iter__(self) -> Iterator[str]: ... + def __getitem__(self, key: str) -> Any: ... + def getvalue(self, key: str, default: Any = None) -> Any: ... + def getfirst(self, key: str, default: Any = None) -> Any: ... + def getlist(self, key: str) -> _list[Any]: ... + def keys(self) -> _list[str]: ... + def __contains__(self, key: str) -> bool: ... + def __len__(self) -> int: ... + def __bool__(self) -> bool: ... + def __del__(self) -> None: ... + # Returns bytes or str IO depending on an internal flag + def make_file(self) -> IO[Any]: ... + +def print_exception( + type: type[BaseException] | None = None, + value: BaseException | None = None, + tb: TracebackType | None = None, + limit: int | None = None, +) -> None: ... +def print_arguments() -> None: ... diff --git a/mypy/typeshed/stdlib/cgitb.pyi b/mypy/typeshed/stdlib/cgitb.pyi new file mode 100644 index 000000000000..565725801159 --- /dev/null +++ b/mypy/typeshed/stdlib/cgitb.pyi @@ -0,0 +1,32 @@ +from _typeshed import OptExcInfo, StrOrBytesPath +from collections.abc import Callable +from types import FrameType, TracebackType +from typing import IO, Any, Final + +__UNDEF__: Final[object] # undocumented sentinel + +def reset() -> str: ... # undocumented +def small(text: str) -> str: ... # undocumented +def strong(text: str) -> str: ... # undocumented +def grey(text: str) -> str: ... # undocumented +def lookup(name: str, frame: FrameType, locals: dict[str, Any]) -> tuple[str | None, Any]: ... # undocumented +def scanvars( + reader: Callable[[], bytes], frame: FrameType, locals: dict[str, Any] +) -> list[tuple[str, str | None, Any]]: ... # undocumented +def html(einfo: OptExcInfo, context: int = 5) -> str: ... +def text(einfo: OptExcInfo, context: int = 5) -> str: ... + +class Hook: # undocumented + def __init__( + self, + display: int = 1, + logdir: StrOrBytesPath | None = None, + context: int = 5, + file: IO[str] | None = None, + format: str = "html", + ) -> None: ... + def __call__(self, etype: type[BaseException] | None, evalue: BaseException | None, etb: TracebackType | None) -> None: ... + def handle(self, info: OptExcInfo | None = None) -> None: ... + +def handler(info: OptExcInfo | None = None) -> None: ... +def enable(display: int = 1, logdir: StrOrBytesPath | None = None, context: int = 5, format: str = "html") -> None: ... diff --git a/mypy/typeshed/stdlib/chunk.pyi b/mypy/typeshed/stdlib/chunk.pyi new file mode 100644 index 000000000000..9788d35f680c --- /dev/null +++ b/mypy/typeshed/stdlib/chunk.pyi @@ -0,0 +1,20 @@ +from typing import IO + +class Chunk: + closed: bool + align: bool + file: IO[bytes] + chunkname: bytes + chunksize: int + size_read: int + offset: int + seekable: bool + def __init__(self, file: IO[bytes], align: bool = True, bigendian: bool = True, inclheader: bool = False) -> None: ... + def getname(self) -> bytes: ... + def getsize(self) -> int: ... + def close(self) -> None: ... + def isatty(self) -> bool: ... + def seek(self, pos: int, whence: int = 0) -> None: ... + def tell(self) -> int: ... + def read(self, size: int = -1) -> bytes: ... + def skip(self) -> None: ... diff --git a/mypy/typeshed/stdlib/cmath.pyi b/mypy/typeshed/stdlib/cmath.pyi new file mode 100644 index 000000000000..fab9d10230f8 --- /dev/null +++ b/mypy/typeshed/stdlib/cmath.pyi @@ -0,0 +1,36 @@ +from typing import SupportsComplex, SupportsFloat, SupportsIndex +from typing_extensions import TypeAlias + +e: float +pi: float +inf: float +infj: complex +nan: float +nanj: complex +tau: float + +_C: TypeAlias = SupportsFloat | SupportsComplex | SupportsIndex | complex + +def acos(z: _C, /) -> complex: ... +def acosh(z: _C, /) -> complex: ... +def asin(z: _C, /) -> complex: ... +def asinh(z: _C, /) -> complex: ... +def atan(z: _C, /) -> complex: ... +def atanh(z: _C, /) -> complex: ... +def cos(z: _C, /) -> complex: ... +def cosh(z: _C, /) -> complex: ... +def exp(z: _C, /) -> complex: ... +def isclose(a: _C, b: _C, *, rel_tol: SupportsFloat = 1e-09, abs_tol: SupportsFloat = 0.0) -> bool: ... +def isinf(z: _C, /) -> bool: ... +def isnan(z: _C, /) -> bool: ... +def log(x: _C, base: _C = ..., /) -> complex: ... +def log10(z: _C, /) -> complex: ... +def phase(z: _C, /) -> float: ... +def polar(z: _C, /) -> tuple[float, float]: ... +def rect(r: float, phi: float, /) -> complex: ... +def sin(z: _C, /) -> complex: ... +def sinh(z: _C, /) -> complex: ... +def sqrt(z: _C, /) -> complex: ... +def tan(z: _C, /) -> complex: ... +def tanh(z: _C, /) -> complex: ... +def isfinite(z: _C, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/cmd.pyi b/mypy/typeshed/stdlib/cmd.pyi new file mode 100644 index 000000000000..6e84133572bf --- /dev/null +++ b/mypy/typeshed/stdlib/cmd.pyi @@ -0,0 +1,46 @@ +from collections.abc import Callable +from typing import IO, Any, Final +from typing_extensions import LiteralString + +__all__ = ["Cmd"] + +PROMPT: Final = "(Cmd) " +IDENTCHARS: Final[LiteralString] # Too big to be `Literal` + +class Cmd: + prompt: str + identchars: str + ruler: str + lastcmd: str + intro: Any | None + doc_leader: str + doc_header: str + misc_header: str + undoc_header: str + nohelp: str + use_rawinput: bool + stdin: IO[str] + stdout: IO[str] + cmdqueue: list[str] + completekey: str + def __init__(self, completekey: str = "tab", stdin: IO[str] | None = None, stdout: IO[str] | None = None) -> None: ... + old_completer: Callable[[str, int], str | None] | None + def cmdloop(self, intro: Any | None = None) -> None: ... + def precmd(self, line: str) -> str: ... + def postcmd(self, stop: bool, line: str) -> bool: ... + def preloop(self) -> None: ... + def postloop(self) -> None: ... + def parseline(self, line: str) -> tuple[str | None, str | None, str]: ... + def onecmd(self, line: str) -> bool: ... + def emptyline(self) -> bool: ... + def default(self, line: str) -> None: ... + def completedefault(self, *ignored: Any) -> list[str]: ... + def completenames(self, text: str, *ignored: Any) -> list[str]: ... + completion_matches: list[str] | None + def complete(self, text: str, state: int) -> list[str] | None: ... + def get_names(self) -> list[str]: ... + # Only the first element of args matters. + def complete_help(self, *args: Any) -> list[str]: ... + def do_help(self, arg: str) -> bool | None: ... + def print_topics(self, header: str, cmds: list[str] | None, cmdlen: Any, maxcol: int) -> None: ... + def columnize(self, list: list[str] | None, displaywidth: int = 80) -> None: ... diff --git a/mypy/typeshed/stdlib/code.pyi b/mypy/typeshed/stdlib/code.pyi new file mode 100644 index 000000000000..54971f3ae93c --- /dev/null +++ b/mypy/typeshed/stdlib/code.pyi @@ -0,0 +1,56 @@ +import sys +from codeop import CommandCompiler +from collections.abc import Callable, Mapping +from types import CodeType +from typing import Any + +__all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact", "compile_command"] + +class InteractiveInterpreter: + locals: Mapping[str, Any] # undocumented + compile: CommandCompiler # undocumented + def __init__(self, locals: Mapping[str, Any] | None = None) -> None: ... + def runsource(self, source: str, filename: str = "", symbol: str = "single") -> bool: ... + def runcode(self, code: CodeType) -> None: ... + if sys.version_info >= (3, 13): + def showsyntaxerror(self, filename: str | None = None, *, source: str = "") -> None: ... + else: + def showsyntaxerror(self, filename: str | None = None) -> None: ... + + def showtraceback(self) -> None: ... + def write(self, data: str) -> None: ... + +class InteractiveConsole(InteractiveInterpreter): + buffer: list[str] # undocumented + filename: str # undocumented + if sys.version_info >= (3, 13): + def __init__( + self, locals: Mapping[str, Any] | None = None, filename: str = "", *, local_exit: bool = False + ) -> None: ... + def push(self, line: str, filename: str | None = None) -> bool: ... + else: + def __init__(self, locals: Mapping[str, Any] | None = None, filename: str = "") -> None: ... + def push(self, line: str) -> bool: ... + + def interact(self, banner: str | None = None, exitmsg: str | None = None) -> None: ... + def resetbuffer(self) -> None: ... + def raw_input(self, prompt: str = "") -> str: ... + +if sys.version_info >= (3, 13): + def interact( + banner: str | None = None, + readfunc: Callable[[str], str] | None = None, + local: Mapping[str, Any] | None = None, + exitmsg: str | None = None, + local_exit: bool = False, + ) -> None: ... + +else: + def interact( + banner: str | None = None, + readfunc: Callable[[str], str] | None = None, + local: Mapping[str, Any] | None = None, + exitmsg: str | None = None, + ) -> None: ... + +def compile_command(source: str, filename: str = "", symbol: str = "single") -> CodeType | None: ... diff --git a/mypy/typeshed/stdlib/codecs.pyi b/mypy/typeshed/stdlib/codecs.pyi new file mode 100644 index 000000000000..b3c721f1e283 --- /dev/null +++ b/mypy/typeshed/stdlib/codecs.pyi @@ -0,0 +1,289 @@ +import types +from _codecs import * +from _typeshed import ReadableBuffer +from abc import abstractmethod +from collections.abc import Callable, Generator, Iterable +from typing import Any, BinaryIO, ClassVar, Final, Literal, Protocol, TextIO +from typing_extensions import Self + +__all__ = [ + "register", + "lookup", + "open", + "EncodedFile", + "BOM", + "BOM_BE", + "BOM_LE", + "BOM32_BE", + "BOM32_LE", + "BOM64_BE", + "BOM64_LE", + "BOM_UTF8", + "BOM_UTF16", + "BOM_UTF16_LE", + "BOM_UTF16_BE", + "BOM_UTF32", + "BOM_UTF32_LE", + "BOM_UTF32_BE", + "CodecInfo", + "Codec", + "IncrementalEncoder", + "IncrementalDecoder", + "StreamReader", + "StreamWriter", + "StreamReaderWriter", + "StreamRecoder", + "getencoder", + "getdecoder", + "getincrementalencoder", + "getincrementaldecoder", + "getreader", + "getwriter", + "encode", + "decode", + "iterencode", + "iterdecode", + "strict_errors", + "ignore_errors", + "replace_errors", + "xmlcharrefreplace_errors", + "backslashreplace_errors", + "namereplace_errors", + "register_error", + "lookup_error", +] + +BOM32_BE: Final = b"\xfe\xff" +BOM32_LE: Final = b"\xff\xfe" +BOM64_BE: Final = b"\x00\x00\xfe\xff" +BOM64_LE: Final = b"\xff\xfe\x00\x00" + +class _WritableStream(Protocol): + def write(self, data: bytes, /) -> object: ... + def seek(self, offset: int, whence: int, /) -> object: ... + def close(self) -> object: ... + +class _ReadableStream(Protocol): + def read(self, size: int = ..., /) -> bytes: ... + def seek(self, offset: int, whence: int, /) -> object: ... + def close(self) -> object: ... + +class _Stream(_WritableStream, _ReadableStream, Protocol): ... + +# TODO: this only satisfies the most common interface, where +# bytes is the raw form and str is the cooked form. +# In the long run, both should become template parameters maybe? +# There *are* bytes->bytes and str->str encodings in the standard library. +# They were much more common in Python 2 than in Python 3. + +class _Encoder(Protocol): + def __call__(self, input: str, errors: str = ..., /) -> tuple[bytes, int]: ... # signature of Codec().encode + +class _Decoder(Protocol): + def __call__(self, input: ReadableBuffer, errors: str = ..., /) -> tuple[str, int]: ... # signature of Codec().decode + +class _StreamReader(Protocol): + def __call__(self, stream: _ReadableStream, errors: str = ..., /) -> StreamReader: ... + +class _StreamWriter(Protocol): + def __call__(self, stream: _WritableStream, errors: str = ..., /) -> StreamWriter: ... + +class _IncrementalEncoder(Protocol): + def __call__(self, errors: str = ...) -> IncrementalEncoder: ... + +class _IncrementalDecoder(Protocol): + def __call__(self, errors: str = ...) -> IncrementalDecoder: ... + +class CodecInfo(tuple[_Encoder, _Decoder, _StreamReader, _StreamWriter]): + _is_text_encoding: bool + @property + def encode(self) -> _Encoder: ... + @property + def decode(self) -> _Decoder: ... + @property + def streamreader(self) -> _StreamReader: ... + @property + def streamwriter(self) -> _StreamWriter: ... + @property + def incrementalencoder(self) -> _IncrementalEncoder: ... + @property + def incrementaldecoder(self) -> _IncrementalDecoder: ... + name: str + def __new__( + cls, + encode: _Encoder, + decode: _Decoder, + streamreader: _StreamReader | None = None, + streamwriter: _StreamWriter | None = None, + incrementalencoder: _IncrementalEncoder | None = None, + incrementaldecoder: _IncrementalDecoder | None = None, + name: str | None = None, + *, + _is_text_encoding: bool | None = None, + ) -> Self: ... + +def getencoder(encoding: str) -> _Encoder: ... +def getdecoder(encoding: str) -> _Decoder: ... +def getincrementalencoder(encoding: str) -> _IncrementalEncoder: ... +def getincrementaldecoder(encoding: str) -> _IncrementalDecoder: ... +def getreader(encoding: str) -> _StreamReader: ... +def getwriter(encoding: str) -> _StreamWriter: ... +def open( + filename: str, mode: str = "r", encoding: str | None = None, errors: str = "strict", buffering: int = -1 +) -> StreamReaderWriter: ... +def EncodedFile(file: _Stream, data_encoding: str, file_encoding: str | None = None, errors: str = "strict") -> StreamRecoder: ... +def iterencode(iterator: Iterable[str], encoding: str, errors: str = "strict") -> Generator[bytes, None, None]: ... +def iterdecode(iterator: Iterable[bytes], encoding: str, errors: str = "strict") -> Generator[str, None, None]: ... + +BOM: Final[Literal[b"\xff\xfe", b"\xfe\xff"]] # depends on `sys.byteorder` +BOM_BE: Final = b"\xfe\xff" +BOM_LE: Final = b"\xff\xfe" +BOM_UTF8: Final = b"\xef\xbb\xbf" +BOM_UTF16: Final[Literal[b"\xff\xfe", b"\xfe\xff"]] # depends on `sys.byteorder` +BOM_UTF16_BE: Final = b"\xfe\xff" +BOM_UTF16_LE: Final = b"\xff\xfe" +BOM_UTF32: Final[Literal[b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff"]] # depends on `sys.byteorder` +BOM_UTF32_BE: Final = b"\x00\x00\xfe\xff" +BOM_UTF32_LE: Final = b"\xff\xfe\x00\x00" + +def strict_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def replace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def ignore_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def xmlcharrefreplace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def backslashreplace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def namereplace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... + +class Codec: + # These are sort of @abstractmethod but sort of not. + # The StreamReader and StreamWriter subclasses only implement one. + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder: + errors: str + def __init__(self, errors: str = "strict") -> None: ... + @abstractmethod + def encode(self, input: str, final: bool = False) -> bytes: ... + def reset(self) -> None: ... + # documentation says int but str is needed for the subclass. + def getstate(self) -> int | str: ... + def setstate(self, state: int | str) -> None: ... + +class IncrementalDecoder: + errors: str + def __init__(self, errors: str = "strict") -> None: ... + @abstractmethod + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + def reset(self) -> None: ... + def getstate(self) -> tuple[bytes, int]: ... + def setstate(self, state: tuple[bytes, int]) -> None: ... + +# These are not documented but used in encodings/*.py implementations. +class BufferedIncrementalEncoder(IncrementalEncoder): + buffer: str + def __init__(self, errors: str = "strict") -> None: ... + @abstractmethod + def _buffer_encode(self, input: str, errors: str, final: bool) -> tuple[bytes, int]: ... + def encode(self, input: str, final: bool = False) -> bytes: ... + +class BufferedIncrementalDecoder(IncrementalDecoder): + buffer: bytes + def __init__(self, errors: str = "strict") -> None: ... + @abstractmethod + def _buffer_decode(self, input: ReadableBuffer, errors: str, final: bool) -> tuple[str, int]: ... + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +# TODO: it is not possible to specify the requirement that all other +# attributes and methods are passed-through from the stream. +class StreamWriter(Codec): + stream: _WritableStream + errors: str + def __init__(self, stream: _WritableStream, errors: str = "strict") -> None: ... + def write(self, object: str) -> None: ... + def writelines(self, list: Iterable[str]) -> None: ... + def reset(self) -> None: ... + def seek(self, offset: int, whence: int = 0) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, type: type[BaseException] | None, value: BaseException | None, tb: types.TracebackType | None) -> None: ... + def __getattr__(self, name: str, getattr: Callable[[Any, str], Any] = ...) -> Any: ... + +class StreamReader(Codec): + stream: _ReadableStream + errors: str + # This is set to str, but some subclasses set to bytes instead. + charbuffertype: ClassVar[type] = ... + def __init__(self, stream: _ReadableStream, errors: str = "strict") -> None: ... + def read(self, size: int = -1, chars: int = -1, firstline: bool = False) -> str: ... + def readline(self, size: int | None = None, keepends: bool = True) -> str: ... + def readlines(self, sizehint: int | None = None, keepends: bool = True) -> list[str]: ... + def reset(self) -> None: ... + def seek(self, offset: int, whence: int = 0) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, type: type[BaseException] | None, value: BaseException | None, tb: types.TracebackType | None) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> str: ... + def __getattr__(self, name: str, getattr: Callable[[Any, str], Any] = ...) -> Any: ... + +# Doesn't actually inherit from TextIO, but wraps a BinaryIO to provide text reading and writing +# and delegates attributes to the underlying binary stream with __getattr__. +class StreamReaderWriter(TextIO): + stream: _Stream + def __init__(self, stream: _Stream, Reader: _StreamReader, Writer: _StreamWriter, errors: str = "strict") -> None: ... + def read(self, size: int = -1) -> str: ... + def readline(self, size: int | None = None) -> str: ... + def readlines(self, sizehint: int | None = None) -> list[str]: ... + def __next__(self) -> str: ... + def __iter__(self) -> Self: ... + def write(self, data: str) -> None: ... # type: ignore[override] + def writelines(self, list: Iterable[str]) -> None: ... + def reset(self) -> None: ... + def seek(self, offset: int, whence: int = 0) -> None: ... # type: ignore[override] + def __enter__(self) -> Self: ... + def __exit__(self, type: type[BaseException] | None, value: BaseException | None, tb: types.TracebackType | None) -> None: ... + def __getattr__(self, name: str) -> Any: ... + # These methods don't actually exist directly, but they are needed to satisfy the TextIO + # interface. At runtime, they are delegated through __getattr__. + def close(self) -> None: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + def readable(self) -> bool: ... + def truncate(self, size: int | None = ...) -> int: ... + def seekable(self) -> bool: ... + def tell(self) -> int: ... + def writable(self) -> bool: ... + +class StreamRecoder(BinaryIO): + def __init__( + self, + stream: _Stream, + encode: _Encoder, + decode: _Decoder, + Reader: _StreamReader, + Writer: _StreamWriter, + errors: str = "strict", + ) -> None: ... + def read(self, size: int = -1) -> bytes: ... + def readline(self, size: int | None = None) -> bytes: ... + def readlines(self, sizehint: int | None = None) -> list[bytes]: ... + def __next__(self) -> bytes: ... + def __iter__(self) -> Self: ... + # Base class accepts more types than just bytes + def write(self, data: bytes) -> None: ... # type: ignore[override] + def writelines(self, list: Iterable[bytes]) -> None: ... # type: ignore[override] + def reset(self) -> None: ... + def __getattr__(self, name: str) -> Any: ... + def __enter__(self) -> Self: ... + def __exit__(self, type: type[BaseException] | None, value: BaseException | None, tb: types.TracebackType | None) -> None: ... + def seek(self, offset: int, whence: int = 0) -> None: ... # type: ignore[override] + # These methods don't actually exist directly, but they are needed to satisfy the BinaryIO + # interface. At runtime, they are delegated through __getattr__. + def close(self) -> None: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + def readable(self) -> bool: ... + def truncate(self, size: int | None = ...) -> int: ... + def seekable(self) -> bool: ... + def tell(self) -> int: ... + def writable(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/codeop.pyi b/mypy/typeshed/stdlib/codeop.pyi new file mode 100644 index 000000000000..6a51b7786384 --- /dev/null +++ b/mypy/typeshed/stdlib/codeop.pyi @@ -0,0 +1,13 @@ +from types import CodeType + +__all__ = ["compile_command", "Compile", "CommandCompiler"] + +def compile_command(source: str, filename: str = "", symbol: str = "single") -> CodeType | None: ... + +class Compile: + flags: int + def __call__(self, source: str, filename: str, symbol: str) -> CodeType: ... + +class CommandCompiler: + compiler: Compile + def __call__(self, source: str, filename: str = "", symbol: str = "single") -> CodeType | None: ... diff --git a/mypy/typeshed/stdlib/collections/__init__.pyi b/mypy/typeshed/stdlib/collections/__init__.pyi new file mode 100644 index 000000000000..2d136318813c --- /dev/null +++ b/mypy/typeshed/stdlib/collections/__init__.pyi @@ -0,0 +1,502 @@ +import sys +from _collections_abc import dict_items, dict_keys, dict_values +from _typeshed import SupportsItems, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT +from typing import Any, Generic, NoReturn, SupportsIndex, TypeVar, final, overload +from typing_extensions import Self + +if sys.version_info >= (3, 9): + from types import GenericAlias + +if sys.version_info >= (3, 10): + from collections.abc import ( + Callable, + ItemsView, + Iterable, + Iterator, + KeysView, + Mapping, + MutableMapping, + MutableSequence, + Sequence, + ValuesView, + ) +else: + from _collections_abc import * + +__all__ = ["ChainMap", "Counter", "OrderedDict", "UserDict", "UserList", "UserString", "defaultdict", "deque", "namedtuple"] + +_S = TypeVar("_S") +_T = TypeVar("_T") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_KT_co = TypeVar("_KT_co", covariant=True) +_VT_co = TypeVar("_VT_co", covariant=True) + +# namedtuple is special-cased in the type checker; the initializer is ignored. +def namedtuple( + typename: str, + field_names: str | Iterable[str], + *, + rename: bool = False, + module: str | None = None, + defaults: Iterable[Any] | None = None, +) -> type[tuple[Any, ...]]: ... + +class UserDict(MutableMapping[_KT, _VT]): + data: dict[_KT, _VT] + # __init__ should be kept roughly in line with `dict.__init__`, which has the same semantics + @overload + def __init__(self, dict: None = None, /) -> None: ... + @overload + def __init__( + self: UserDict[str, _VT], dict: None = None, /, **kwargs: _VT # pyright: ignore[reportInvalidTypeVarUse] #11780 + ) -> None: ... + @overload + def __init__(self, dict: SupportsKeysAndGetItem[_KT, _VT], /) -> None: ... + @overload + def __init__( + self: UserDict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + dict: SupportsKeysAndGetItem[str, _VT], + /, + **kwargs: _VT, + ) -> None: ... + @overload + def __init__(self, iterable: Iterable[tuple[_KT, _VT]], /) -> None: ... + @overload + def __init__( + self: UserDict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + iterable: Iterable[tuple[str, _VT]], + /, + **kwargs: _VT, + ) -> None: ... + @overload + def __init__(self: UserDict[str, str], iterable: Iterable[list[str]], /) -> None: ... + @overload + def __init__(self: UserDict[bytes, bytes], iterable: Iterable[list[bytes]], /) -> None: ... + def __len__(self) -> int: ... + def __getitem__(self, key: _KT) -> _VT: ... + def __setitem__(self, key: _KT, item: _VT) -> None: ... + def __delitem__(self, key: _KT) -> None: ... + def __iter__(self) -> Iterator[_KT]: ... + def __contains__(self, key: object) -> bool: ... + def copy(self) -> Self: ... + def __copy__(self) -> Self: ... + + # `UserDict.fromkeys` has the same semantics as `dict.fromkeys`, so should be kept in line with `dict.fromkeys`. + # TODO: Much like `dict.fromkeys`, the true signature of `UserDict.fromkeys` is inexpressible in the current type system. + # See #3800 & https://github.com/python/typing/issues/548#issuecomment-683336963. + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], value: None = None) -> UserDict[_T, Any | None]: ... + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], value: _S) -> UserDict[_T, _S]: ... + if sys.version_info >= (3, 9): + @overload + def __or__(self, other: UserDict[_KT, _VT] | dict[_KT, _VT]) -> Self: ... + @overload + def __or__(self, other: UserDict[_T1, _T2] | dict[_T1, _T2]) -> UserDict[_KT | _T1, _VT | _T2]: ... + @overload + def __ror__(self, other: UserDict[_KT, _VT] | dict[_KT, _VT]) -> Self: ... + @overload + def __ror__(self, other: UserDict[_T1, _T2] | dict[_T1, _T2]) -> UserDict[_KT | _T1, _VT | _T2]: ... + # UserDict.__ior__ should be kept roughly in line with MutableMapping.update() + @overload # type: ignore[misc] + def __ior__(self, other: SupportsKeysAndGetItem[_KT, _VT]) -> Self: ... + @overload + def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... + if sys.version_info >= (3, 12): + @overload + def get(self, key: _KT, default: None = None) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _T) -> _VT | _T: ... + +class UserList(MutableSequence[_T]): + data: list[_T] + @overload + def __init__(self, initlist: None = None) -> None: ... + @overload + def __init__(self, initlist: Iterable[_T]) -> None: ... + def __lt__(self, other: list[_T] | UserList[_T]) -> bool: ... + def __le__(self, other: list[_T] | UserList[_T]) -> bool: ... + def __gt__(self, other: list[_T] | UserList[_T]) -> bool: ... + def __ge__(self, other: list[_T] | UserList[_T]) -> bool: ... + def __eq__(self, other: object) -> bool: ... + def __contains__(self, item: object) -> bool: ... + def __len__(self) -> int: ... + @overload + def __getitem__(self, i: SupportsIndex) -> _T: ... + @overload + def __getitem__(self, i: slice) -> Self: ... + @overload + def __setitem__(self, i: SupportsIndex, item: _T) -> None: ... + @overload + def __setitem__(self, i: slice, item: Iterable[_T]) -> None: ... + def __delitem__(self, i: SupportsIndex | slice) -> None: ... + def __add__(self, other: Iterable[_T]) -> Self: ... + def __radd__(self, other: Iterable[_T]) -> Self: ... + def __iadd__(self, other: Iterable[_T]) -> Self: ... + def __mul__(self, n: int) -> Self: ... + def __rmul__(self, n: int) -> Self: ... + def __imul__(self, n: int) -> Self: ... + def append(self, item: _T) -> None: ... + def insert(self, i: int, item: _T) -> None: ... + def pop(self, i: int = -1) -> _T: ... + def remove(self, item: _T) -> None: ... + def copy(self) -> Self: ... + def __copy__(self) -> Self: ... + def count(self, item: _T) -> int: ... + # The runtime signature is "item, *args", and the arguments are then passed + # to `list.index`. In order to give more precise types, we pretend that the + # `item` argument is positional-only. + def index(self, item: _T, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int: ... + # All arguments are passed to `list.sort` at runtime, so the signature should be kept in line with `list.sort`. + @overload + def sort(self: UserList[SupportsRichComparisonT], *, key: None = None, reverse: bool = False) -> None: ... + @overload + def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = False) -> None: ... + def extend(self, other: Iterable[_T]) -> None: ... + +class UserString(Sequence[UserString]): + data: str + def __init__(self, seq: object) -> None: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __complex__(self) -> complex: ... + def __getnewargs__(self) -> tuple[str]: ... + def __lt__(self, string: str | UserString) -> bool: ... + def __le__(self, string: str | UserString) -> bool: ... + def __gt__(self, string: str | UserString) -> bool: ... + def __ge__(self, string: str | UserString) -> bool: ... + def __eq__(self, string: object) -> bool: ... + def __hash__(self) -> int: ... + def __contains__(self, char: object) -> bool: ... + def __len__(self) -> int: ... + def __getitem__(self, index: SupportsIndex | slice) -> Self: ... + def __iter__(self) -> Iterator[Self]: ... + def __reversed__(self) -> Iterator[Self]: ... + def __add__(self, other: object) -> Self: ... + def __radd__(self, other: object) -> Self: ... + def __mul__(self, n: int) -> Self: ... + def __rmul__(self, n: int) -> Self: ... + def __mod__(self, args: Any) -> Self: ... + def __rmod__(self, template: object) -> Self: ... + def capitalize(self) -> Self: ... + def casefold(self) -> Self: ... + def center(self, width: int, *args: Any) -> Self: ... + def count(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... + def encode(self: UserString, encoding: str | None = "utf-8", errors: str | None = "strict") -> bytes: ... + def endswith(self, suffix: str | tuple[str, ...], start: int | None = 0, end: int | None = sys.maxsize) -> bool: ... + def expandtabs(self, tabsize: int = 8) -> Self: ... + def find(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... + def format(self, *args: Any, **kwds: Any) -> str: ... + def format_map(self, mapping: Mapping[str, Any]) -> str: ... + def index(self, sub: str, start: int = 0, end: int = sys.maxsize) -> int: ... + def isalpha(self) -> bool: ... + def isalnum(self) -> bool: ... + def isdecimal(self) -> bool: ... + def isdigit(self) -> bool: ... + def isidentifier(self) -> bool: ... + def islower(self) -> bool: ... + def isnumeric(self) -> bool: ... + def isprintable(self) -> bool: ... + def isspace(self) -> bool: ... + def istitle(self) -> bool: ... + def isupper(self) -> bool: ... + def isascii(self) -> bool: ... + def join(self, seq: Iterable[str]) -> str: ... + def ljust(self, width: int, *args: Any) -> Self: ... + def lower(self) -> Self: ... + def lstrip(self, chars: str | None = None) -> Self: ... + maketrans = str.maketrans + def partition(self, sep: str) -> tuple[str, str, str]: ... + if sys.version_info >= (3, 9): + def removeprefix(self, prefix: str | UserString, /) -> Self: ... + def removesuffix(self, suffix: str | UserString, /) -> Self: ... + + def replace(self, old: str | UserString, new: str | UserString, maxsplit: int = -1) -> Self: ... + def rfind(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... + def rindex(self, sub: str | UserString, start: int = 0, end: int = sys.maxsize) -> int: ... + def rjust(self, width: int, *args: Any) -> Self: ... + def rpartition(self, sep: str) -> tuple[str, str, str]: ... + def rstrip(self, chars: str | None = None) -> Self: ... + def split(self, sep: str | None = None, maxsplit: int = -1) -> list[str]: ... + def rsplit(self, sep: str | None = None, maxsplit: int = -1) -> list[str]: ... + def splitlines(self, keepends: bool = False) -> list[str]: ... + def startswith(self, prefix: str | tuple[str, ...], start: int | None = 0, end: int | None = sys.maxsize) -> bool: ... + def strip(self, chars: str | None = None) -> Self: ... + def swapcase(self) -> Self: ... + def title(self) -> Self: ... + def translate(self, *args: Any) -> Self: ... + def upper(self) -> Self: ... + def zfill(self, width: int) -> Self: ... + +class deque(MutableSequence[_T]): + @property + def maxlen(self) -> int | None: ... + @overload + def __init__(self, *, maxlen: int | None = None) -> None: ... + @overload + def __init__(self, iterable: Iterable[_T], maxlen: int | None = None) -> None: ... + def append(self, x: _T, /) -> None: ... + def appendleft(self, x: _T, /) -> None: ... + def copy(self) -> Self: ... + def count(self, x: _T, /) -> int: ... + def extend(self, iterable: Iterable[_T], /) -> None: ... + def extendleft(self, iterable: Iterable[_T], /) -> None: ... + def insert(self, i: int, x: _T, /) -> None: ... + def index(self, x: _T, start: int = 0, stop: int = ..., /) -> int: ... + def pop(self) -> _T: ... # type: ignore[override] + def popleft(self) -> _T: ... + def remove(self, value: _T, /) -> None: ... + def rotate(self, n: int = 1, /) -> None: ... + def __copy__(self) -> Self: ... + def __len__(self) -> int: ... + # These methods of deque don't take slices, unlike MutableSequence, hence the type: ignores + def __getitem__(self, key: SupportsIndex, /) -> _T: ... # type: ignore[override] + def __setitem__(self, key: SupportsIndex, value: _T, /) -> None: ... # type: ignore[override] + def __delitem__(self, key: SupportsIndex, /) -> None: ... # type: ignore[override] + def __contains__(self, key: object, /) -> bool: ... + def __reduce__(self) -> tuple[type[Self], tuple[()], None, Iterator[_T]]: ... + def __iadd__(self, value: Iterable[_T], /) -> Self: ... + def __add__(self, value: Self, /) -> Self: ... + def __mul__(self, value: int, /) -> Self: ... + def __imul__(self, value: int, /) -> Self: ... + def __lt__(self, value: deque[_T], /) -> bool: ... + def __le__(self, value: deque[_T], /) -> bool: ... + def __gt__(self, value: deque[_T], /) -> bool: ... + def __ge__(self, value: deque[_T], /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class Counter(dict[_T, int], Generic[_T]): + @overload + def __init__(self, iterable: None = None, /) -> None: ... + @overload + def __init__(self: Counter[str], iterable: None = None, /, **kwargs: int) -> None: ... + @overload + def __init__(self, mapping: SupportsKeysAndGetItem[_T, int], /) -> None: ... + @overload + def __init__(self, iterable: Iterable[_T], /) -> None: ... + def copy(self) -> Self: ... + def elements(self) -> Iterator[_T]: ... + def most_common(self, n: int | None = None) -> list[tuple[_T, int]]: ... + @classmethod + def fromkeys(cls, iterable: Any, v: int | None = None) -> NoReturn: ... # type: ignore[override] + @overload + def subtract(self, iterable: None = None, /) -> None: ... + @overload + def subtract(self, mapping: Mapping[_T, int], /) -> None: ... + @overload + def subtract(self, iterable: Iterable[_T], /) -> None: ... + # Unlike dict.update(), use Mapping instead of SupportsKeysAndGetItem for the first overload + # (source code does an `isinstance(other, Mapping)` check) + # + # The second overload is also deliberately different to dict.update() + # (if it were `Iterable[_T] | Iterable[tuple[_T, int]]`, + # the tuples would be added as keys, breaking type safety) + @overload # type: ignore[override] + def update(self, m: Mapping[_T, int], /, **kwargs: int) -> None: ... + @overload + def update(self, iterable: Iterable[_T], /, **kwargs: int) -> None: ... + @overload + def update(self, iterable: None = None, /, **kwargs: int) -> None: ... + def __missing__(self, key: _T) -> int: ... + def __delitem__(self, elem: object) -> None: ... + if sys.version_info >= (3, 10): + def __eq__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... + + def __add__(self, other: Counter[_S]) -> Counter[_T | _S]: ... + def __sub__(self, other: Counter[_T]) -> Counter[_T]: ... + def __and__(self, other: Counter[_T]) -> Counter[_T]: ... + def __or__(self, other: Counter[_S]) -> Counter[_T | _S]: ... # type: ignore[override] + def __pos__(self) -> Counter[_T]: ... + def __neg__(self) -> Counter[_T]: ... + # several type: ignores because __iadd__ is supposedly incompatible with __add__, etc. + def __iadd__(self, other: SupportsItems[_T, int]) -> Self: ... # type: ignore[misc] + def __isub__(self, other: SupportsItems[_T, int]) -> Self: ... + def __iand__(self, other: SupportsItems[_T, int]) -> Self: ... + def __ior__(self, other: SupportsItems[_T, int]) -> Self: ... # type: ignore[override,misc] + if sys.version_info >= (3, 10): + def total(self) -> int: ... + def __le__(self, other: Counter[Any]) -> bool: ... + def __lt__(self, other: Counter[Any]) -> bool: ... + def __ge__(self, other: Counter[Any]) -> bool: ... + def __gt__(self, other: Counter[Any]) -> bool: ... + +# The pure-Python implementations of the "views" classes +# These are exposed at runtime in `collections/__init__.py` +class _OrderedDictKeysView(KeysView[_KT_co]): + def __reversed__(self) -> Iterator[_KT_co]: ... + +class _OrderedDictItemsView(ItemsView[_KT_co, _VT_co]): + def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... + +class _OrderedDictValuesView(ValuesView[_VT_co]): + def __reversed__(self) -> Iterator[_VT_co]: ... + +# The C implementations of the "views" classes +# (At runtime, these are called `odict_keys`, `odict_items` and `odict_values`, +# but they are not exposed anywhere) +# pyright doesn't have a specific error code for subclassing error! +@final +class _odict_keys(dict_keys[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __reversed__(self) -> Iterator[_KT_co]: ... + +@final +class _odict_items(dict_items[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... + +@final +class _odict_values(dict_values[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __reversed__(self) -> Iterator[_VT_co]: ... + +class OrderedDict(dict[_KT, _VT]): + def popitem(self, last: bool = True) -> tuple[_KT, _VT]: ... + def move_to_end(self, key: _KT, last: bool = True) -> None: ... + def copy(self) -> Self: ... + def __reversed__(self) -> Iterator[_KT]: ... + def keys(self) -> _odict_keys[_KT, _VT]: ... + def items(self) -> _odict_items[_KT, _VT]: ... + def values(self) -> _odict_values[_KT, _VT]: ... + # The signature of OrderedDict.fromkeys should be kept in line with `dict.fromkeys`, modulo positional-only differences. + # Like dict.fromkeys, its true signature is not expressible in the current type system. + # See #3800 & https://github.com/python/typing/issues/548#issuecomment-683336963. + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], value: None = None) -> OrderedDict[_T, Any | None]: ... + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], value: _S) -> OrderedDict[_T, _S]: ... + # Keep OrderedDict.setdefault in line with MutableMapping.setdefault, modulo positional-only differences. + @overload + def setdefault(self: OrderedDict[_KT, _T | None], key: _KT, default: None = None) -> _T | None: ... + @overload + def setdefault(self, key: _KT, default: _VT) -> _VT: ... + # Same as dict.pop, but accepts keyword arguments + @overload + def pop(self, key: _KT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T) -> _VT | _T: ... + def __eq__(self, value: object, /) -> bool: ... + if sys.version_info >= (3, 9): + @overload + def __or__(self, value: dict[_KT, _VT], /) -> Self: ... + @overload + def __or__(self, value: dict[_T1, _T2], /) -> OrderedDict[_KT | _T1, _VT | _T2]: ... + @overload + def __ror__(self, value: dict[_KT, _VT], /) -> Self: ... + @overload + def __ror__(self, value: dict[_T1, _T2], /) -> OrderedDict[_KT | _T1, _VT | _T2]: ... # type: ignore[misc] + +class defaultdict(dict[_KT, _VT]): + default_factory: Callable[[], _VT] | None + @overload + def __init__(self) -> None: ... + @overload + def __init__(self: defaultdict[str, _VT], **kwargs: _VT) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] #11780 + @overload + def __init__(self, default_factory: Callable[[], _VT] | None, /) -> None: ... + @overload + def __init__( + self: defaultdict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + default_factory: Callable[[], _VT] | None, + /, + **kwargs: _VT, + ) -> None: ... + @overload + def __init__(self, default_factory: Callable[[], _VT] | None, map: SupportsKeysAndGetItem[_KT, _VT], /) -> None: ... + @overload + def __init__( + self: defaultdict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + default_factory: Callable[[], _VT] | None, + map: SupportsKeysAndGetItem[str, _VT], + /, + **kwargs: _VT, + ) -> None: ... + @overload + def __init__(self, default_factory: Callable[[], _VT] | None, iterable: Iterable[tuple[_KT, _VT]], /) -> None: ... + @overload + def __init__( + self: defaultdict[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + default_factory: Callable[[], _VT] | None, + iterable: Iterable[tuple[str, _VT]], + /, + **kwargs: _VT, + ) -> None: ... + def __missing__(self, key: _KT, /) -> _VT: ... + def __copy__(self) -> Self: ... + def copy(self) -> Self: ... + if sys.version_info >= (3, 9): + @overload + def __or__(self, value: dict[_KT, _VT], /) -> Self: ... + @overload + def __or__(self, value: dict[_T1, _T2], /) -> defaultdict[_KT | _T1, _VT | _T2]: ... + @overload + def __ror__(self, value: dict[_KT, _VT], /) -> Self: ... + @overload + def __ror__(self, value: dict[_T1, _T2], /) -> defaultdict[_KT | _T1, _VT | _T2]: ... # type: ignore[misc] + +class ChainMap(MutableMapping[_KT, _VT]): + maps: list[MutableMapping[_KT, _VT]] + def __init__(self, *maps: MutableMapping[_KT, _VT]) -> None: ... + def new_child(self, m: MutableMapping[_KT, _VT] | None = None) -> Self: ... + @property + def parents(self) -> Self: ... + def __setitem__(self, key: _KT, value: _VT) -> None: ... + def __delitem__(self, key: _KT) -> None: ... + def __getitem__(self, key: _KT) -> _VT: ... + def __iter__(self) -> Iterator[_KT]: ... + def __len__(self) -> int: ... + def __contains__(self, key: object) -> bool: ... + @overload + def get(self, key: _KT, default: None = None) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _T) -> _VT | _T: ... + def __missing__(self, key: _KT) -> _VT: ... # undocumented + def __bool__(self) -> bool: ... + # Keep ChainMap.setdefault in line with MutableMapping.setdefault, modulo positional-only differences. + @overload + def setdefault(self: ChainMap[_KT, _T | None], key: _KT, default: None = None) -> _T | None: ... + @overload + def setdefault(self, key: _KT, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T) -> _VT | _T: ... + def copy(self) -> Self: ... + __copy__ = copy + # All arguments to `fromkeys` are passed to `dict.fromkeys` at runtime, + # so the signature should be kept in line with `dict.fromkeys`. + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T]) -> ChainMap[_T, Any | None]: ... + @classmethod + @overload + # Special-case None: the user probably wants to add non-None values later. + def fromkeys(cls, iterable: Iterable[_T], value: None, /) -> ChainMap[_T, Any | None]: ... + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], value: _S, /) -> ChainMap[_T, _S]: ... + if sys.version_info >= (3, 9): + @overload + def __or__(self, other: Mapping[_KT, _VT]) -> Self: ... + @overload + def __or__(self, other: Mapping[_T1, _T2]) -> ChainMap[_KT | _T1, _VT | _T2]: ... + @overload + def __ror__(self, other: Mapping[_KT, _VT]) -> Self: ... + @overload + def __ror__(self, other: Mapping[_T1, _T2]) -> ChainMap[_KT | _T1, _VT | _T2]: ... + # ChainMap.__ior__ should be kept roughly in line with MutableMapping.update() + @overload # type: ignore[misc] + def __ior__(self, other: SupportsKeysAndGetItem[_KT, _VT]) -> Self: ... + @overload + def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... diff --git a/mypy/typeshed/stdlib/collections/abc.pyi b/mypy/typeshed/stdlib/collections/abc.pyi new file mode 100644 index 000000000000..3df2a1d9eb9b --- /dev/null +++ b/mypy/typeshed/stdlib/collections/abc.pyi @@ -0,0 +1,2 @@ +from _collections_abc import * +from _collections_abc import __all__ as __all__ diff --git a/mypy/typeshed/stdlib/colorsys.pyi b/mypy/typeshed/stdlib/colorsys.pyi new file mode 100644 index 000000000000..443ee828ebfe --- /dev/null +++ b/mypy/typeshed/stdlib/colorsys.pyi @@ -0,0 +1,13 @@ +__all__ = ["rgb_to_yiq", "yiq_to_rgb", "rgb_to_hls", "hls_to_rgb", "rgb_to_hsv", "hsv_to_rgb"] + +def rgb_to_yiq(r: float, g: float, b: float) -> tuple[float, float, float]: ... +def yiq_to_rgb(y: float, i: float, q: float) -> tuple[float, float, float]: ... +def rgb_to_hls(r: float, g: float, b: float) -> tuple[float, float, float]: ... +def hls_to_rgb(h: float, l: float, s: float) -> tuple[float, float, float]: ... +def rgb_to_hsv(r: float, g: float, b: float) -> tuple[float, float, float]: ... +def hsv_to_rgb(h: float, s: float, v: float) -> tuple[float, float, float]: ... + +# TODO undocumented +ONE_SIXTH: float +ONE_THIRD: float +TWO_THIRD: float diff --git a/mypy/typeshed/stdlib/compileall.pyi b/mypy/typeshed/stdlib/compileall.pyi new file mode 100644 index 000000000000..9fb3608f2979 --- /dev/null +++ b/mypy/typeshed/stdlib/compileall.pyi @@ -0,0 +1,111 @@ +import sys +from _typeshed import StrPath +from py_compile import PycInvalidationMode +from typing import Any, Protocol + +__all__ = ["compile_dir", "compile_file", "compile_path"] + +class _SupportsSearch(Protocol): + def search(self, string: str, /) -> Any: ... + +if sys.version_info >= (3, 10): + def compile_dir( + dir: StrPath, + maxlevels: int | None = None, + ddir: StrPath | None = None, + force: bool = False, + rx: _SupportsSearch | None = None, + quiet: int = 0, + legacy: bool = False, + optimize: int = -1, + workers: int = 1, + invalidation_mode: PycInvalidationMode | None = None, + *, + stripdir: StrPath | None = None, + prependdir: StrPath | None = None, + limit_sl_dest: StrPath | None = None, + hardlink_dupes: bool = False, + ) -> int: ... + def compile_file( + fullname: StrPath, + ddir: StrPath | None = None, + force: bool = False, + rx: _SupportsSearch | None = None, + quiet: int = 0, + legacy: bool = False, + optimize: int = -1, + invalidation_mode: PycInvalidationMode | None = None, + *, + stripdir: StrPath | None = None, + prependdir: StrPath | None = None, + limit_sl_dest: StrPath | None = None, + hardlink_dupes: bool = False, + ) -> int: ... + +elif sys.version_info >= (3, 9): + def compile_dir( + dir: StrPath, + maxlevels: int | None = None, + ddir: StrPath | None = None, + force: bool = False, + rx: _SupportsSearch | None = None, + quiet: int = 0, + legacy: bool = False, + optimize: int = -1, + workers: int = 1, + invalidation_mode: PycInvalidationMode | None = None, + *, + stripdir: str | None = None, # https://bugs.python.org/issue40447 + prependdir: StrPath | None = None, + limit_sl_dest: StrPath | None = None, + hardlink_dupes: bool = False, + ) -> int: ... + def compile_file( + fullname: StrPath, + ddir: StrPath | None = None, + force: bool = False, + rx: _SupportsSearch | None = None, + quiet: int = 0, + legacy: bool = False, + optimize: int = -1, + invalidation_mode: PycInvalidationMode | None = None, + *, + stripdir: str | None = None, # https://bugs.python.org/issue40447 + prependdir: StrPath | None = None, + limit_sl_dest: StrPath | None = None, + hardlink_dupes: bool = False, + ) -> int: ... + +else: + def compile_dir( + dir: StrPath, + maxlevels: int = 10, + ddir: StrPath | None = None, + force: bool = False, + rx: _SupportsSearch | None = None, + quiet: int = 0, + legacy: bool = False, + optimize: int = -1, + workers: int = 1, + invalidation_mode: PycInvalidationMode | None = None, + ) -> int: ... + def compile_file( + fullname: StrPath, + ddir: StrPath | None = None, + force: bool = False, + rx: _SupportsSearch | None = None, + quiet: int = 0, + legacy: bool = False, + optimize: int = -1, + invalidation_mode: PycInvalidationMode | None = None, + ) -> int: ... + +def compile_path( + skip_curdir: bool = ..., + maxlevels: int = 0, + force: bool = False, + quiet: int = 0, + legacy: bool = False, + optimize: int = -1, + invalidation_mode: PycInvalidationMode | None = None, +) -> int: ... diff --git a/pinfer/__init__.py b/mypy/typeshed/stdlib/concurrent/__init__.pyi similarity index 100% rename from pinfer/__init__.py rename to mypy/typeshed/stdlib/concurrent/__init__.pyi diff --git a/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi b/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi new file mode 100644 index 000000000000..68fd0bc5acb4 --- /dev/null +++ b/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi @@ -0,0 +1,51 @@ +import sys + +from ._base import ( + ALL_COMPLETED as ALL_COMPLETED, + FIRST_COMPLETED as FIRST_COMPLETED, + FIRST_EXCEPTION as FIRST_EXCEPTION, + BrokenExecutor as BrokenExecutor, + CancelledError as CancelledError, + Executor as Executor, + Future as Future, + InvalidStateError as InvalidStateError, + TimeoutError as TimeoutError, + as_completed as as_completed, + wait as wait, +) +from .process import ProcessPoolExecutor as ProcessPoolExecutor +from .thread import ThreadPoolExecutor as ThreadPoolExecutor + +if sys.version_info >= (3, 13): + __all__ = ( + "FIRST_COMPLETED", + "FIRST_EXCEPTION", + "ALL_COMPLETED", + "CancelledError", + "TimeoutError", + "InvalidStateError", + "BrokenExecutor", + "Future", + "Executor", + "wait", + "as_completed", + "ProcessPoolExecutor", + "ThreadPoolExecutor", + ) +else: + __all__ = ( + "FIRST_COMPLETED", + "FIRST_EXCEPTION", + "ALL_COMPLETED", + "CancelledError", + "TimeoutError", + "BrokenExecutor", + "Future", + "Executor", + "wait", + "as_completed", + "ProcessPoolExecutor", + "ThreadPoolExecutor", + ) + +def __dir__() -> tuple[str, ...]: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi new file mode 100644 index 000000000000..0c019457902b --- /dev/null +++ b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi @@ -0,0 +1,126 @@ +import sys +import threading +from _typeshed import Unused +from collections.abc import Callable, Collection, Iterable, Iterator +from logging import Logger +from types import TracebackType +from typing import Any, Final, Generic, NamedTuple, Protocol, TypeVar +from typing_extensions import ParamSpec, Self + +if sys.version_info >= (3, 9): + from types import GenericAlias + +FIRST_COMPLETED: Final = "FIRST_COMPLETED" +FIRST_EXCEPTION: Final = "FIRST_EXCEPTION" +ALL_COMPLETED: Final = "ALL_COMPLETED" +PENDING: Final = "PENDING" +RUNNING: Final = "RUNNING" +CANCELLED: Final = "CANCELLED" +CANCELLED_AND_NOTIFIED: Final = "CANCELLED_AND_NOTIFIED" +FINISHED: Final = "FINISHED" +_FUTURE_STATES: list[str] +_STATE_TO_DESCRIPTION_MAP: dict[str, str] +LOGGER: Logger + +class Error(Exception): ... +class CancelledError(Error): ... + +if sys.version_info >= (3, 11): + from builtins import TimeoutError as TimeoutError +else: + class TimeoutError(Error): ... + +class InvalidStateError(Error): ... +class BrokenExecutor(RuntimeError): ... + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_P = ParamSpec("_P") + +class Future(Generic[_T]): + _condition: threading.Condition + _state: str + _result: _T | None + _exception: BaseException | None + _waiters: list[_Waiter] + def cancel(self) -> bool: ... + def cancelled(self) -> bool: ... + def running(self) -> bool: ... + def done(self) -> bool: ... + def add_done_callback(self, fn: Callable[[Future[_T]], object]) -> None: ... + def result(self, timeout: float | None = None) -> _T: ... + def set_running_or_notify_cancel(self) -> bool: ... + def set_result(self, result: _T) -> None: ... + def exception(self, timeout: float | None = None) -> BaseException | None: ... + def set_exception(self, exception: BaseException | None) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class Executor: + if sys.version_info >= (3, 9): + def submit(self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> Future[_T]: ... + else: + def submit(self, fn: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> Future[_T]: ... + + def map( + self, fn: Callable[..., _T], *iterables: Iterable[Any], timeout: float | None = None, chunksize: int = 1 + ) -> Iterator[_T]: ... + if sys.version_info >= (3, 9): + def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None: ... + else: + def shutdown(self, wait: bool = True) -> None: ... + + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> bool | None: ... + +class _AsCompletedFuture(Protocol[_T_co]): + # as_completed only mutates non-generic aspects of passed Futures and does not do any nominal + # checks. Therefore, we can use a Protocol here to allow as_completed to act covariantly. + # See the tests for concurrent.futures + _condition: threading.Condition + _state: str + _waiters: list[_Waiter] + # Not used by as_completed, but needed to propagate the generic type + def result(self, timeout: float | None = None) -> _T_co: ... + +def as_completed(fs: Iterable[_AsCompletedFuture[_T]], timeout: float | None = None) -> Iterator[Future[_T]]: ... + +class DoneAndNotDoneFutures(NamedTuple, Generic[_T]): + done: set[Future[_T]] + not_done: set[Future[_T]] + +if sys.version_info >= (3, 9): + def wait( + fs: Iterable[Future[_T]], timeout: float | None = None, return_when: str = "ALL_COMPLETED" + ) -> DoneAndNotDoneFutures[_T]: ... + +else: + def wait( + fs: Collection[Future[_T]], timeout: float | None = None, return_when: str = "ALL_COMPLETED" + ) -> DoneAndNotDoneFutures[_T]: ... + +class _Waiter: + event: threading.Event + finished_futures: list[Future[Any]] + def add_result(self, future: Future[Any]) -> None: ... + def add_exception(self, future: Future[Any]) -> None: ... + def add_cancelled(self, future: Future[Any]) -> None: ... + +class _AsCompletedWaiter(_Waiter): + lock: threading.Lock + +class _FirstCompletedWaiter(_Waiter): ... + +class _AllCompletedWaiter(_Waiter): + num_pending_calls: int + stop_on_exception: bool + lock: threading.Lock + def __init__(self, num_pending_calls: int, stop_on_exception: bool) -> None: ... + +class _AcquireFutures: + futures: Iterable[Future[Any]] + def __init__(self, futures: Iterable[Future[Any]]) -> None: ... + def __enter__(self) -> None: ... + def __exit__(self, *args: Unused) -> None: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/process.pyi b/mypy/typeshed/stdlib/concurrent/futures/process.pyi new file mode 100644 index 000000000000..a1de3d679b23 --- /dev/null +++ b/mypy/typeshed/stdlib/concurrent/futures/process.pyi @@ -0,0 +1,234 @@ +import sys +from collections.abc import Callable, Generator, Iterable, Mapping, MutableMapping, MutableSequence +from multiprocessing.connection import Connection +from multiprocessing.context import BaseContext, Process +from multiprocessing.queues import Queue, SimpleQueue +from threading import Lock, Semaphore, Thread +from types import TracebackType +from typing import Any, Generic, TypeVar, overload +from typing_extensions import TypeVarTuple, Unpack +from weakref import ref + +from ._base import BrokenExecutor, Executor, Future + +_T = TypeVar("_T") +_Ts = TypeVarTuple("_Ts") + +_threads_wakeups: MutableMapping[Any, Any] +_global_shutdown: bool + +class _ThreadWakeup: + _closed: bool + # Any: Unused send and recv methods + _reader: Connection[Any, Any] + _writer: Connection[Any, Any] + def close(self) -> None: ... + def wakeup(self) -> None: ... + def clear(self) -> None: ... + +def _python_exit() -> None: ... + +EXTRA_QUEUED_CALLS: int + +_MAX_WINDOWS_WORKERS: int + +class _RemoteTraceback(Exception): + tb: str + def __init__(self, tb: TracebackType) -> None: ... + +class _ExceptionWithTraceback: + exc: BaseException + tb: TracebackType + def __init__(self, exc: BaseException, tb: TracebackType) -> None: ... + def __reduce__(self) -> str | tuple[Any, ...]: ... + +def _rebuild_exc(exc: Exception, tb: str) -> Exception: ... + +class _WorkItem(Generic[_T]): + future: Future[_T] + fn: Callable[..., _T] + args: Iterable[Any] + kwargs: Mapping[str, Any] + def __init__(self, future: Future[_T], fn: Callable[..., _T], args: Iterable[Any], kwargs: Mapping[str, Any]) -> None: ... + +class _ResultItem: + work_id: int + exception: Exception + result: Any + if sys.version_info >= (3, 11): + exit_pid: int | None + def __init__( + self, work_id: int, exception: Exception | None = None, result: Any | None = None, exit_pid: int | None = None + ) -> None: ... + else: + def __init__(self, work_id: int, exception: Exception | None = None, result: Any | None = None) -> None: ... + +class _CallItem: + work_id: int + fn: Callable[..., Any] + args: Iterable[Any] + kwargs: Mapping[str, Any] + def __init__(self, work_id: int, fn: Callable[..., Any], args: Iterable[Any], kwargs: Mapping[str, Any]) -> None: ... + +class _SafeQueue(Queue[Future[Any]]): + pending_work_items: dict[int, _WorkItem[Any]] + shutdown_lock: Lock + thread_wakeup: _ThreadWakeup + if sys.version_info >= (3, 9): + def __init__( + self, + max_size: int | None = 0, + *, + ctx: BaseContext, + pending_work_items: dict[int, _WorkItem[Any]], + shutdown_lock: Lock, + thread_wakeup: _ThreadWakeup, + ) -> None: ... + else: + def __init__( + self, max_size: int | None = 0, *, ctx: BaseContext, pending_work_items: dict[int, _WorkItem[Any]] + ) -> None: ... + + def _on_queue_feeder_error(self, e: Exception, obj: _CallItem) -> None: ... + +def _get_chunks(*iterables: Any, chunksize: int) -> Generator[tuple[Any, ...], None, None]: ... +def _process_chunk(fn: Callable[..., _T], chunk: Iterable[tuple[Any, ...]]) -> list[_T]: ... + +if sys.version_info >= (3, 11): + def _sendback_result( + result_queue: SimpleQueue[_WorkItem[Any]], + work_id: int, + result: Any | None = None, + exception: Exception | None = None, + exit_pid: int | None = None, + ) -> None: ... + +else: + def _sendback_result( + result_queue: SimpleQueue[_WorkItem[Any]], work_id: int, result: Any | None = None, exception: Exception | None = None + ) -> None: ... + +if sys.version_info >= (3, 11): + def _process_worker( + call_queue: Queue[_CallItem], + result_queue: SimpleQueue[_ResultItem], + initializer: Callable[[Unpack[_Ts]], object] | None, + initargs: tuple[Unpack[_Ts]], + max_tasks: int | None = None, + ) -> None: ... + +else: + def _process_worker( + call_queue: Queue[_CallItem], + result_queue: SimpleQueue[_ResultItem], + initializer: Callable[[Unpack[_Ts]], object] | None, + initargs: tuple[Unpack[_Ts]], + ) -> None: ... + +if sys.version_info >= (3, 9): + class _ExecutorManagerThread(Thread): + thread_wakeup: _ThreadWakeup + shutdown_lock: Lock + executor_reference: ref[Any] + processes: MutableMapping[int, Process] + call_queue: Queue[_CallItem] + result_queue: SimpleQueue[_ResultItem] + work_ids_queue: Queue[int] + pending_work_items: dict[int, _WorkItem[Any]] + def __init__(self, executor: ProcessPoolExecutor) -> None: ... + def run(self) -> None: ... + def add_call_item_to_queue(self) -> None: ... + def wait_result_broken_or_wakeup(self) -> tuple[Any, bool, str]: ... + def process_result_item(self, result_item: int | _ResultItem) -> None: ... + def is_shutting_down(self) -> bool: ... + def terminate_broken(self, cause: str) -> None: ... + def flag_executor_shutting_down(self) -> None: ... + def shutdown_workers(self) -> None: ... + def join_executor_internals(self) -> None: ... + def get_n_children_alive(self) -> int: ... + +_system_limits_checked: bool +_system_limited: bool | None + +def _check_system_limits() -> None: ... +def _chain_from_iterable_of_lists(iterable: Iterable[MutableSequence[Any]]) -> Any: ... + +class BrokenProcessPool(BrokenExecutor): ... + +class ProcessPoolExecutor(Executor): + _mp_context: BaseContext | None + _initializer: Callable[..., None] | None + _initargs: tuple[Any, ...] + _executor_manager_thread: _ThreadWakeup + _processes: MutableMapping[int, Process] + _shutdown_thread: bool + _shutdown_lock: Lock + _idle_worker_semaphore: Semaphore + _broken: bool + _queue_count: int + _pending_work_items: dict[int, _WorkItem[Any]] + _cancel_pending_futures: bool + _executor_manager_thread_wakeup: _ThreadWakeup + _result_queue: SimpleQueue[Any] + _work_ids: Queue[Any] + if sys.version_info >= (3, 11): + @overload + def __init__( + self, + max_workers: int | None = None, + mp_context: BaseContext | None = None, + initializer: Callable[[], object] | None = None, + initargs: tuple[()] = (), + *, + max_tasks_per_child: int | None = None, + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None = None, + mp_context: BaseContext | None = None, + *, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + max_tasks_per_child: int | None = None, + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None, + mp_context: BaseContext | None, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + *, + max_tasks_per_child: int | None = None, + ) -> None: ... + else: + @overload + def __init__( + self, + max_workers: int | None = None, + mp_context: BaseContext | None = None, + initializer: Callable[[], object] | None = None, + initargs: tuple[()] = (), + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None = None, + mp_context: BaseContext | None = None, + *, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None, + mp_context: BaseContext | None, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + ) -> None: ... + if sys.version_info >= (3, 9): + def _start_executor_manager_thread(self) -> None: ... + + def _adjust_process_count(self) -> None: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/thread.pyi b/mypy/typeshed/stdlib/concurrent/futures/thread.pyi new file mode 100644 index 000000000000..d1b7858eae02 --- /dev/null +++ b/mypy/typeshed/stdlib/concurrent/futures/thread.pyi @@ -0,0 +1,80 @@ +import queue +import sys +from collections.abc import Callable, Iterable, Mapping, Set as AbstractSet +from threading import Lock, Semaphore, Thread +from typing import Any, Generic, TypeVar, overload +from typing_extensions import TypeVarTuple, Unpack +from weakref import ref + +from ._base import BrokenExecutor, Executor, Future + +_Ts = TypeVarTuple("_Ts") + +_threads_queues: Mapping[Any, Any] +_shutdown: bool +_global_shutdown_lock: Lock + +def _python_exit() -> None: ... + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_S = TypeVar("_S") + +class _WorkItem(Generic[_S]): + future: Future[_S] + fn: Callable[..., _S] + args: Iterable[Any] + kwargs: Mapping[str, Any] + def __init__(self, future: Future[_S], fn: Callable[..., _S], args: Iterable[Any], kwargs: Mapping[str, Any]) -> None: ... + def run(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +def _worker( + executor_reference: ref[Any], + work_queue: queue.SimpleQueue[Any], + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], +) -> None: ... + +class BrokenThreadPool(BrokenExecutor): ... + +class ThreadPoolExecutor(Executor): + _max_workers: int + _idle_semaphore: Semaphore + _threads: AbstractSet[Thread] + _broken: bool + _shutdown: bool + _shutdown_lock: Lock + _thread_name_prefix: str | None + _initializer: Callable[..., None] | None + _initargs: tuple[Any, ...] + _work_queue: queue.SimpleQueue[_WorkItem[Any]] + @overload + def __init__( + self, + max_workers: int | None = None, + thread_name_prefix: str = "", + initializer: Callable[[], object] | None = None, + initargs: tuple[()] = (), + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None = None, + thread_name_prefix: str = "", + *, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None, + thread_name_prefix: str, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + ) -> None: ... + def _adjust_thread_count(self) -> None: ... + def _initializer_failed(self) -> None: ... diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi new file mode 100644 index 000000000000..a44dc2e1c035 --- /dev/null +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -0,0 +1,412 @@ +import sys +from _typeshed import MaybeNone, StrOrBytesPath, SupportsWrite +from collections.abc import Callable, ItemsView, Iterable, Iterator, Mapping, MutableMapping, Sequence +from re import Pattern +from typing import Any, ClassVar, Final, Literal, TypeVar, overload +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 13): + __all__ = ( + "NoSectionError", + "DuplicateOptionError", + "DuplicateSectionError", + "NoOptionError", + "InterpolationError", + "InterpolationDepthError", + "InterpolationMissingOptionError", + "InterpolationSyntaxError", + "ParsingError", + "MissingSectionHeaderError", + "ConfigParser", + "RawConfigParser", + "Interpolation", + "BasicInterpolation", + "ExtendedInterpolation", + "SectionProxy", + "ConverterMapping", + "DEFAULTSECT", + "MAX_INTERPOLATION_DEPTH", + "UNNAMED_SECTION", + "MultilineContinuationError", + ) +elif sys.version_info >= (3, 12): + __all__ = ( + "NoSectionError", + "DuplicateOptionError", + "DuplicateSectionError", + "NoOptionError", + "InterpolationError", + "InterpolationDepthError", + "InterpolationMissingOptionError", + "InterpolationSyntaxError", + "ParsingError", + "MissingSectionHeaderError", + "ConfigParser", + "RawConfigParser", + "Interpolation", + "BasicInterpolation", + "ExtendedInterpolation", + "LegacyInterpolation", + "SectionProxy", + "ConverterMapping", + "DEFAULTSECT", + "MAX_INTERPOLATION_DEPTH", + ) +else: + __all__ = [ + "NoSectionError", + "DuplicateOptionError", + "DuplicateSectionError", + "NoOptionError", + "InterpolationError", + "InterpolationDepthError", + "InterpolationMissingOptionError", + "InterpolationSyntaxError", + "ParsingError", + "MissingSectionHeaderError", + "ConfigParser", + "SafeConfigParser", + "RawConfigParser", + "Interpolation", + "BasicInterpolation", + "ExtendedInterpolation", + "LegacyInterpolation", + "SectionProxy", + "ConverterMapping", + "DEFAULTSECT", + "MAX_INTERPOLATION_DEPTH", + ] + +_Section: TypeAlias = Mapping[str, str] +_Parser: TypeAlias = MutableMapping[str, _Section] +_ConverterCallback: TypeAlias = Callable[[str], Any] +_ConvertersMap: TypeAlias = dict[str, _ConverterCallback] +_T = TypeVar("_T") + +DEFAULTSECT: Final = "DEFAULT" +MAX_INTERPOLATION_DEPTH: Final = 10 + +class Interpolation: + def before_get(self, parser: _Parser, section: str, option: str, value: str, defaults: _Section) -> str: ... + def before_set(self, parser: _Parser, section: str, option: str, value: str) -> str: ... + def before_read(self, parser: _Parser, section: str, option: str, value: str) -> str: ... + def before_write(self, parser: _Parser, section: str, option: str, value: str) -> str: ... + +class BasicInterpolation(Interpolation): ... +class ExtendedInterpolation(Interpolation): ... + +if sys.version_info < (3, 13): + class LegacyInterpolation(Interpolation): + def before_get(self, parser: _Parser, section: str, option: str, value: str, vars: _Section) -> str: ... + +class RawConfigParser(_Parser): + _SECT_TMPL: ClassVar[str] # undocumented + _OPT_TMPL: ClassVar[str] # undocumented + _OPT_NV_TMPL: ClassVar[str] # undocumented + + SECTCRE: Pattern[str] + OPTCRE: ClassVar[Pattern[str]] + OPTCRE_NV: ClassVar[Pattern[str]] # undocumented + NONSPACECRE: ClassVar[Pattern[str]] # undocumented + + BOOLEAN_STATES: ClassVar[Mapping[str, bool]] # undocumented + default_section: str + if sys.version_info >= (3, 13): + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None = None, + dict_type: type[Mapping[str, str]] = ..., + *, + allow_no_value: Literal[True], + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + allow_unnamed_section: bool = False, + ) -> None: ... + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None, + dict_type: type[Mapping[str, str]], + allow_no_value: Literal[True], + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + allow_unnamed_section: bool = False, + ) -> None: ... + @overload + def __init__( + self, + defaults: _Section | None = None, + dict_type: type[Mapping[str, str]] = ..., + allow_no_value: bool = False, + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + allow_unnamed_section: bool = False, + ) -> None: ... + else: + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None = None, + dict_type: type[Mapping[str, str]] = ..., + *, + allow_no_value: Literal[True], + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + ) -> None: ... + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None, + dict_type: type[Mapping[str, str]], + allow_no_value: Literal[True], + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + ) -> None: ... + @overload + def __init__( + self, + defaults: _Section | None = None, + dict_type: type[Mapping[str, str]] = ..., + allow_no_value: bool = False, + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + ) -> None: ... + + def __len__(self) -> int: ... + def __getitem__(self, key: str) -> SectionProxy: ... + def __setitem__(self, key: str, value: _Section) -> None: ... + def __delitem__(self, key: str) -> None: ... + def __iter__(self) -> Iterator[str]: ... + def __contains__(self, key: object) -> bool: ... + def defaults(self) -> _Section: ... + def sections(self) -> list[str]: ... + def add_section(self, section: str) -> None: ... + def has_section(self, section: str) -> bool: ... + def options(self, section: str) -> list[str]: ... + def has_option(self, section: str, option: str) -> bool: ... + def read(self, filenames: StrOrBytesPath | Iterable[StrOrBytesPath], encoding: str | None = None) -> list[str]: ... + def read_file(self, f: Iterable[str], source: str | None = None) -> None: ... + def read_string(self, string: str, source: str = "") -> None: ... + def read_dict(self, dictionary: Mapping[str, Mapping[str, Any]], source: str = "") -> None: ... + if sys.version_info < (3, 12): + def readfp(self, fp: Iterable[str], filename: str | None = None) -> None: ... + # These get* methods are partially applied (with the same names) in + # SectionProxy; the stubs should be kept updated together + @overload + def getint(self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None) -> int: ... + @overload + def getint( + self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None, fallback: _T = ... + ) -> int | _T: ... + @overload + def getfloat(self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None) -> float: ... + @overload + def getfloat( + self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None, fallback: _T = ... + ) -> float | _T: ... + @overload + def getboolean(self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None) -> bool: ... + @overload + def getboolean( + self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None, fallback: _T = ... + ) -> bool | _T: ... + def _get_conv( + self, + section: str, + option: str, + conv: Callable[[str], _T], + *, + raw: bool = False, + vars: _Section | None = None, + fallback: _T = ..., + ) -> _T: ... + # This is incompatible with MutableMapping so we ignore the type + @overload # type: ignore[override] + def get(self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None) -> str | MaybeNone: ... + @overload + def get( + self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None, fallback: _T + ) -> str | _T | MaybeNone: ... + @overload + def items(self, *, raw: bool = False, vars: _Section | None = None) -> ItemsView[str, SectionProxy]: ... + @overload + def items(self, section: str, raw: bool = False, vars: _Section | None = None) -> list[tuple[str, str]]: ... + def set(self, section: str, option: str, value: str | None = None) -> None: ... + def write(self, fp: SupportsWrite[str], space_around_delimiters: bool = True) -> None: ... + def remove_option(self, section: str, option: str) -> bool: ... + def remove_section(self, section: str) -> bool: ... + def optionxform(self, optionstr: str) -> str: ... + @property + def converters(self) -> ConverterMapping: ... + +class ConfigParser(RawConfigParser): + # This is incompatible with MutableMapping so we ignore the type + @overload # type: ignore[override] + def get(self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None) -> str: ... + @overload + def get(self, section: str, option: str, *, raw: bool = False, vars: _Section | None = None, fallback: _T) -> str | _T: ... + +if sys.version_info < (3, 12): + class SafeConfigParser(ConfigParser): ... # deprecated alias + +class SectionProxy(MutableMapping[str, str]): + def __init__(self, parser: RawConfigParser, name: str) -> None: ... + def __getitem__(self, key: str) -> str: ... + def __setitem__(self, key: str, value: str) -> None: ... + def __delitem__(self, key: str) -> None: ... + def __contains__(self, key: object) -> bool: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[str]: ... + @property + def parser(self) -> RawConfigParser: ... + @property + def name(self) -> str: ... + # This is incompatible with MutableMapping so we ignore the type + @overload # type: ignore[override] + def get( + self, option: str, *, raw: bool = False, vars: _Section | None = None, _impl: Any | None = None, **kwargs: Any + ) -> str | None: ... + @overload + def get( + self, + option: str, + fallback: _T, + *, + raw: bool = False, + vars: _Section | None = None, + _impl: Any | None = None, + **kwargs: Any, + ) -> str | _T: ... + # These are partially-applied version of the methods with the same names in + # RawConfigParser; the stubs should be kept updated together + @overload + def getint(self, option: str, *, raw: bool = ..., vars: _Section | None = ...) -> int | None: ... + @overload + def getint(self, option: str, fallback: _T = ..., *, raw: bool = ..., vars: _Section | None = ...) -> int | _T: ... + @overload + def getfloat(self, option: str, *, raw: bool = ..., vars: _Section | None = ...) -> float | None: ... + @overload + def getfloat(self, option: str, fallback: _T = ..., *, raw: bool = ..., vars: _Section | None = ...) -> float | _T: ... + @overload + def getboolean(self, option: str, *, raw: bool = ..., vars: _Section | None = ...) -> bool | None: ... + @overload + def getboolean(self, option: str, fallback: _T = ..., *, raw: bool = ..., vars: _Section | None = ...) -> bool | _T: ... + # SectionProxy can have arbitrary attributes when custom converters are used + def __getattr__(self, key: str) -> Callable[..., Any]: ... + +class ConverterMapping(MutableMapping[str, _ConverterCallback | None]): + GETTERCRE: ClassVar[Pattern[Any]] + def __init__(self, parser: RawConfigParser) -> None: ... + def __getitem__(self, key: str) -> _ConverterCallback: ... + def __setitem__(self, key: str, value: _ConverterCallback | None) -> None: ... + def __delitem__(self, key: str) -> None: ... + def __iter__(self) -> Iterator[str]: ... + def __len__(self) -> int: ... + +class Error(Exception): + message: str + def __init__(self, msg: str = "") -> None: ... + +class NoSectionError(Error): + section: str + def __init__(self, section: str) -> None: ... + +class DuplicateSectionError(Error): + section: str + source: str | None + lineno: int | None + def __init__(self, section: str, source: str | None = None, lineno: int | None = None) -> None: ... + +class DuplicateOptionError(Error): + section: str + option: str + source: str | None + lineno: int | None + def __init__(self, section: str, option: str, source: str | None = None, lineno: int | None = None) -> None: ... + +class NoOptionError(Error): + section: str + option: str + def __init__(self, option: str, section: str) -> None: ... + +class InterpolationError(Error): + section: str + option: str + def __init__(self, option: str, section: str, msg: str) -> None: ... + +class InterpolationDepthError(InterpolationError): + def __init__(self, option: str, section: str, rawval: object) -> None: ... + +class InterpolationMissingOptionError(InterpolationError): + reference: str + def __init__(self, option: str, section: str, rawval: object, reference: str) -> None: ... + +class InterpolationSyntaxError(InterpolationError): ... + +class ParsingError(Error): + source: str + errors: list[tuple[int, str]] + if sys.version_info >= (3, 13): + def __init__(self, source: str, *args: object) -> None: ... + def combine(self, others: Iterable[ParsingError]) -> ParsingError: ... + elif sys.version_info >= (3, 12): + def __init__(self, source: str) -> None: ... + else: + def __init__(self, source: str | None = None, filename: str | None = None) -> None: ... + + def append(self, lineno: int, line: str) -> None: ... + +class MissingSectionHeaderError(ParsingError): + lineno: int + line: str + def __init__(self, filename: str, lineno: int, line: str) -> None: ... + +if sys.version_info >= (3, 13): + class _UNNAMED_SECTION: ... + UNNAMED_SECTION: _UNNAMED_SECTION + + class MultilineContinuationError(ParsingError): + lineno: int + line: str + def __init__(self, filename: str, lineno: int, line: str) -> None: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi new file mode 100644 index 000000000000..dc5d926775f3 --- /dev/null +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -0,0 +1,219 @@ +import abc +import sys +from _typeshed import FileDescriptorOrPath, Unused +from abc import ABC, abstractmethod +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator +from types import TracebackType +from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable +from typing_extensions import ParamSpec, Self, TypeAlias + +__all__ = [ + "contextmanager", + "closing", + "AbstractContextManager", + "ContextDecorator", + "ExitStack", + "redirect_stdout", + "redirect_stderr", + "suppress", + "AbstractAsyncContextManager", + "AsyncExitStack", + "asynccontextmanager", + "nullcontext", +] + +if sys.version_info >= (3, 10): + __all__ += ["aclosing"] + +if sys.version_info >= (3, 11): + __all__ += ["chdir"] + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_T_io = TypeVar("_T_io", bound=IO[str] | None) +_ExitT_co = TypeVar("_ExitT_co", covariant=True, bound=bool | None, default=bool | None) +_F = TypeVar("_F", bound=Callable[..., Any]) +_P = ParamSpec("_P") + +_ExitFunc: TypeAlias = Callable[[type[BaseException] | None, BaseException | None, TracebackType | None], bool | None] +_CM_EF = TypeVar("_CM_EF", bound=AbstractContextManager[Any, Any] | _ExitFunc) + +# mypy and pyright object to this being both ABC and Protocol. +# At runtime it inherits from ABC and is not a Protocol, but it is on the +# allowlist for use as a Protocol. +@runtime_checkable +class AbstractContextManager(ABC, Protocol[_T_co, _ExitT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __enter__(self) -> _T_co: ... + @abstractmethod + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, / + ) -> _ExitT_co: ... + +# mypy and pyright object to this being both ABC and Protocol. +# At runtime it inherits from ABC and is not a Protocol, but it is on the +# allowlist for use as a Protocol. +@runtime_checkable +class AbstractAsyncContextManager(ABC, Protocol[_T_co, _ExitT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + async def __aenter__(self) -> _T_co: ... + @abstractmethod + async def __aexit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, / + ) -> _ExitT_co: ... + +class ContextDecorator: + def _recreate_cm(self) -> Self: ... + def __call__(self, func: _F) -> _F: ... + +class _GeneratorContextManagerBase: ... + +class _GeneratorContextManager(_GeneratorContextManagerBase, AbstractContextManager[_T_co, bool | None], ContextDecorator): + # __init__ and all instance attributes are actually inherited from _GeneratorContextManagerBase + # adding them there is more trouble than it's worth to include in the stub; see #6676 + def __init__(self, func: Callable[..., Iterator[_T_co]], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... + gen: Generator[_T_co, Any, Any] + func: Callable[..., Generator[_T_co, Any, Any]] + args: tuple[Any, ...] + kwds: dict[str, Any] + if sys.version_info >= (3, 9): + def __exit__( + self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> bool | None: ... + else: + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> bool | None: ... + +def contextmanager(func: Callable[_P, Iterator[_T_co]]) -> Callable[_P, _GeneratorContextManager[_T_co]]: ... + +if sys.version_info >= (3, 10): + _AF = TypeVar("_AF", bound=Callable[..., Awaitable[Any]]) + + class AsyncContextDecorator: + def _recreate_cm(self) -> Self: ... + def __call__(self, func: _AF) -> _AF: ... + + class _AsyncGeneratorContextManager( + _GeneratorContextManagerBase, AbstractAsyncContextManager[_T_co, bool | None], AsyncContextDecorator + ): + # __init__ and these attributes are actually defined in the base class _GeneratorContextManagerBase, + # adding them there is more trouble than it's worth to include in the stub (see #6676) + def __init__(self, func: Callable[..., AsyncIterator[_T_co]], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... + gen: AsyncGenerator[_T_co, Any] + func: Callable[..., AsyncGenerator[_T_co, Any]] + args: tuple[Any, ...] + kwds: dict[str, Any] + async def __aexit__( + self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> bool | None: ... + +else: + class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, AbstractAsyncContextManager[_T_co, bool | None]): + def __init__(self, func: Callable[..., AsyncIterator[_T_co]], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... + gen: AsyncGenerator[_T_co, Any] + func: Callable[..., AsyncGenerator[_T_co, Any]] + args: tuple[Any, ...] + kwds: dict[str, Any] + async def __aexit__( + self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> bool | None: ... + +def asynccontextmanager(func: Callable[_P, AsyncIterator[_T_co]]) -> Callable[_P, _AsyncGeneratorContextManager[_T_co]]: ... + +class _SupportsClose(Protocol): + def close(self) -> object: ... + +_SupportsCloseT = TypeVar("_SupportsCloseT", bound=_SupportsClose) + +class closing(AbstractContextManager[_SupportsCloseT, None]): + def __init__(self, thing: _SupportsCloseT) -> None: ... + def __exit__(self, *exc_info: Unused) -> None: ... + +if sys.version_info >= (3, 10): + class _SupportsAclose(Protocol): + def aclose(self) -> Awaitable[object]: ... + + _SupportsAcloseT = TypeVar("_SupportsAcloseT", bound=_SupportsAclose) + + class aclosing(AbstractAsyncContextManager[_SupportsAcloseT, None]): + def __init__(self, thing: _SupportsAcloseT) -> None: ... + async def __aexit__(self, *exc_info: Unused) -> None: ... + +class suppress(AbstractContextManager[None, bool]): + def __init__(self, *exceptions: type[BaseException]) -> None: ... + def __exit__( + self, exctype: type[BaseException] | None, excinst: BaseException | None, exctb: TracebackType | None + ) -> bool: ... + +class _RedirectStream(AbstractContextManager[_T_io, None]): + def __init__(self, new_target: _T_io) -> None: ... + def __exit__( + self, exctype: type[BaseException] | None, excinst: BaseException | None, exctb: TracebackType | None + ) -> None: ... + +class redirect_stdout(_RedirectStream[_T_io]): ... +class redirect_stderr(_RedirectStream[_T_io]): ... + +class _BaseExitStack(Generic[_ExitT_co]): + def enter_context(self, cm: AbstractContextManager[_T, _ExitT_co]) -> _T: ... + def push(self, exit: _CM_EF) -> _CM_EF: ... + def callback(self, callback: Callable[_P, _T], /, *args: _P.args, **kwds: _P.kwargs) -> Callable[_P, _T]: ... + def pop_all(self) -> Self: ... + +# In reality this is a subclass of `AbstractContextManager`; +# see #7961 for why we don't do that in the stub +class ExitStack(_BaseExitStack[_ExitT_co], metaclass=abc.ABCMeta): + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, / + ) -> _ExitT_co: ... + +_ExitCoroFunc: TypeAlias = Callable[ + [type[BaseException] | None, BaseException | None, TracebackType | None], Awaitable[bool | None] +] +_ACM_EF = TypeVar("_ACM_EF", bound=AbstractAsyncContextManager[Any, Any] | _ExitCoroFunc) + +# In reality this is a subclass of `AbstractAsyncContextManager`; +# see #7961 for why we don't do that in the stub +class AsyncExitStack(_BaseExitStack[_ExitT_co], metaclass=abc.ABCMeta): + async def enter_async_context(self, cm: AbstractAsyncContextManager[_T, _ExitT_co]) -> _T: ... + def push_async_exit(self, exit: _ACM_EF) -> _ACM_EF: ... + def push_async_callback( + self, callback: Callable[_P, Awaitable[_T]], /, *args: _P.args, **kwds: _P.kwargs + ) -> Callable[_P, Awaitable[_T]]: ... + async def aclose(self) -> None: ... + async def __aenter__(self) -> Self: ... + async def __aexit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, / + ) -> bool: ... + +if sys.version_info >= (3, 10): + class nullcontext(AbstractContextManager[_T, None], AbstractAsyncContextManager[_T, None]): + enter_result: _T + @overload + def __init__(self: nullcontext[None], enter_result: None = None) -> None: ... + @overload + def __init__(self: nullcontext[_T], enter_result: _T) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] #11780 + def __enter__(self) -> _T: ... + def __exit__(self, *exctype: Unused) -> None: ... + async def __aenter__(self) -> _T: ... + async def __aexit__(self, *exctype: Unused) -> None: ... + +else: + class nullcontext(AbstractContextManager[_T, None]): + enter_result: _T + @overload + def __init__(self: nullcontext[None], enter_result: None = None) -> None: ... + @overload + def __init__(self: nullcontext[_T], enter_result: _T) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] #11780 + def __enter__(self) -> _T: ... + def __exit__(self, *exctype: Unused) -> None: ... + +if sys.version_info >= (3, 11): + _T_fd_or_any_path = TypeVar("_T_fd_or_any_path", bound=FileDescriptorOrPath) + + class chdir(AbstractContextManager[None, None], Generic[_T_fd_or_any_path]): + path: _T_fd_or_any_path + def __init__(self, path: _T_fd_or_any_path) -> None: ... + def __enter__(self) -> None: ... + def __exit__(self, *excinfo: Unused) -> None: ... diff --git a/mypy/typeshed/stdlib/contextvars.pyi b/mypy/typeshed/stdlib/contextvars.pyi new file mode 100644 index 000000000000..22dc33006e9d --- /dev/null +++ b/mypy/typeshed/stdlib/contextvars.pyi @@ -0,0 +1,3 @@ +from _contextvars import Context as Context, ContextVar as ContextVar, Token as Token, copy_context as copy_context + +__all__ = ("Context", "ContextVar", "Token", "copy_context") diff --git a/mypy/typeshed/stdlib/copy.pyi b/mypy/typeshed/stdlib/copy.pyi new file mode 100644 index 000000000000..2cceec6a2250 --- /dev/null +++ b/mypy/typeshed/stdlib/copy.pyi @@ -0,0 +1,27 @@ +import sys +from typing import Any, Protocol, TypeVar +from typing_extensions import Self + +__all__ = ["Error", "copy", "deepcopy"] + +_T = TypeVar("_T") +_SR = TypeVar("_SR", bound=_SupportsReplace) + +class _SupportsReplace(Protocol): + # In reality doesn't support args, but there's no other great way to express this. + def __replace__(self, *args: Any, **kwargs: Any) -> Self: ... + +# None in CPython but non-None in Jython +PyStringMap: Any + +# Note: memo and _nil are internal kwargs. +def deepcopy(x: _T, memo: dict[int, Any] | None = None, _nil: Any = []) -> _T: ... +def copy(x: _T) -> _T: ... + +if sys.version_info >= (3, 13): + __all__ += ["replace"] + def replace(obj: _SR, /, **changes: Any) -> _SR: ... + +class Error(Exception): ... + +error = Error diff --git a/mypy/typeshed/stdlib/copyreg.pyi b/mypy/typeshed/stdlib/copyreg.pyi new file mode 100644 index 000000000000..8f7fd957fc52 --- /dev/null +++ b/mypy/typeshed/stdlib/copyreg.pyi @@ -0,0 +1,21 @@ +from collections.abc import Callable, Hashable +from typing import Any, SupportsInt, TypeVar +from typing_extensions import TypeAlias + +_T = TypeVar("_T") +_Reduce: TypeAlias = tuple[Callable[..., _T], tuple[Any, ...]] | tuple[Callable[..., _T], tuple[Any, ...], Any | None] + +__all__ = ["pickle", "constructor", "add_extension", "remove_extension", "clear_extension_cache"] + +def pickle( + ob_type: type[_T], + pickle_function: Callable[[_T], str | _Reduce[_T]], + constructor_ob: Callable[[_Reduce[_T]], _T] | None = None, +) -> None: ... +def constructor(object: Callable[[_Reduce[_T]], _T]) -> None: ... +def add_extension(module: Hashable, name: Hashable, code: SupportsInt) -> None: ... +def remove_extension(module: Hashable, name: Hashable, code: int) -> None: ... +def clear_extension_cache() -> None: ... + +_DispatchTableType: TypeAlias = dict[type, Callable[[Any], str | _Reduce[Any]]] # imported by multiprocessing.reduction +dispatch_table: _DispatchTableType # undocumented diff --git a/mypy/typeshed/stdlib/crypt.pyi b/mypy/typeshed/stdlib/crypt.pyi new file mode 100644 index 000000000000..bd22b5f8daba --- /dev/null +++ b/mypy/typeshed/stdlib/crypt.pyi @@ -0,0 +1,20 @@ +import sys +from typing import Final, NamedTuple, type_check_only + +if sys.platform != "win32": + @type_check_only + class _MethodBase(NamedTuple): + name: str + ident: str | None + salt_chars: int + total_size: int + + class _Method(_MethodBase): ... + METHOD_CRYPT: Final[_Method] + METHOD_MD5: Final[_Method] + METHOD_SHA256: Final[_Method] + METHOD_SHA512: Final[_Method] + METHOD_BLOWFISH: Final[_Method] + methods: list[_Method] + def mksalt(method: _Method | None = None, *, rounds: int | None = None) -> str: ... + def crypt(word: str, salt: str | _Method | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/csv.pyi b/mypy/typeshed/stdlib/csv.pyi new file mode 100644 index 000000000000..4a82de638136 --- /dev/null +++ b/mypy/typeshed/stdlib/csv.pyi @@ -0,0 +1,157 @@ +import sys +from _csv import ( + QUOTE_ALL as QUOTE_ALL, + QUOTE_MINIMAL as QUOTE_MINIMAL, + QUOTE_NONE as QUOTE_NONE, + QUOTE_NONNUMERIC as QUOTE_NONNUMERIC, + Error as Error, + __version__ as __version__, + _DialectLike, + _QuotingType, + field_size_limit as field_size_limit, + get_dialect as get_dialect, + list_dialects as list_dialects, + reader as reader, + register_dialect as register_dialect, + unregister_dialect as unregister_dialect, + writer as writer, +) + +if sys.version_info >= (3, 12): + from _csv import QUOTE_NOTNULL as QUOTE_NOTNULL, QUOTE_STRINGS as QUOTE_STRINGS +if sys.version_info >= (3, 10): + from _csv import Reader, Writer +else: + from _csv import _reader as Reader, _writer as Writer + +from _typeshed import SupportsWrite +from collections.abc import Collection, Iterable, Mapping, Sequence +from typing import Any, Generic, Literal, TypeVar, overload +from typing_extensions import Self + +if sys.version_info >= (3, 12): + from types import GenericAlias + +__all__ = [ + "QUOTE_MINIMAL", + "QUOTE_ALL", + "QUOTE_NONNUMERIC", + "QUOTE_NONE", + "Error", + "Dialect", + "excel", + "excel_tab", + "field_size_limit", + "reader", + "writer", + "register_dialect", + "get_dialect", + "list_dialects", + "Sniffer", + "unregister_dialect", + "DictReader", + "DictWriter", + "unix_dialect", +] +if sys.version_info >= (3, 12): + __all__ += ["QUOTE_STRINGS", "QUOTE_NOTNULL"] +if sys.version_info < (3, 13): + __all__ += ["__doc__", "__version__"] + +_T = TypeVar("_T") + +class Dialect: + delimiter: str + quotechar: str | None + escapechar: str | None + doublequote: bool + skipinitialspace: bool + lineterminator: str + quoting: _QuotingType + strict: bool + def __init__(self) -> None: ... + +class excel(Dialect): ... +class excel_tab(excel): ... +class unix_dialect(Dialect): ... + +class DictReader(Generic[_T]): + fieldnames: Sequence[_T] | None + restkey: _T | None + restval: str | Any | None + reader: Reader + dialect: _DialectLike + line_num: int + @overload + def __init__( + self, + f: Iterable[str], + fieldnames: Sequence[_T], + restkey: _T | None = None, + restval: str | Any | None = None, + dialect: _DialectLike = "excel", + *, + delimiter: str = ",", + quotechar: str | None = '"', + escapechar: str | None = None, + doublequote: bool = True, + skipinitialspace: bool = False, + lineterminator: str = "\r\n", + quoting: _QuotingType = 0, + strict: bool = False, + ) -> None: ... + @overload + def __init__( + self: DictReader[str], + f: Iterable[str], + fieldnames: Sequence[str] | None = None, + restkey: str | None = None, + restval: str | None = None, + dialect: _DialectLike = "excel", + *, + delimiter: str = ",", + quotechar: str | None = '"', + escapechar: str | None = None, + doublequote: bool = True, + skipinitialspace: bool = False, + lineterminator: str = "\r\n", + quoting: _QuotingType = 0, + strict: bool = False, + ) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> dict[_T | Any, str | Any]: ... + if sys.version_info >= (3, 12): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class DictWriter(Generic[_T]): + fieldnames: Collection[_T] + restval: Any | None + extrasaction: Literal["raise", "ignore"] + writer: Writer + def __init__( + self, + f: SupportsWrite[str], + fieldnames: Collection[_T], + restval: Any | None = "", + extrasaction: Literal["raise", "ignore"] = "raise", + dialect: _DialectLike = "excel", + *, + delimiter: str = ",", + quotechar: str | None = '"', + escapechar: str | None = None, + doublequote: bool = True, + skipinitialspace: bool = False, + lineterminator: str = "\r\n", + quoting: _QuotingType = 0, + strict: bool = False, + ) -> None: ... + def writeheader(self) -> Any: ... + def writerow(self, rowdict: Mapping[_T, Any]) -> Any: ... + def writerows(self, rowdicts: Iterable[Mapping[_T, Any]]) -> None: ... + if sys.version_info >= (3, 12): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class Sniffer: + preferred: list[str] + def sniff(self, sample: str, delimiters: str | None = None) -> type[Dialect]: ... + def has_header(self, sample: str) -> bool: ... diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi new file mode 100644 index 000000000000..3e0e7c45bf15 --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -0,0 +1,212 @@ +import sys +from _ctypes import ( + POINTER as POINTER, + RTLD_GLOBAL as RTLD_GLOBAL, + RTLD_LOCAL as RTLD_LOCAL, + Array as Array, + CFuncPtr as _CFuncPtr, + Structure as Structure, + Union as Union, + _CanCastTo as _CanCastTo, + _CArgObject as _CArgObject, + _CData as _CData, + _CDataType as _CDataType, + _CField as _CField, + _Pointer as _Pointer, + _PointerLike as _PointerLike, + _SimpleCData as _SimpleCData, + addressof as addressof, + alignment as alignment, + byref as byref, + get_errno as get_errno, + pointer as pointer, + resize as resize, + set_errno as set_errno, + sizeof as sizeof, +) +from ctypes._endian import BigEndianStructure as BigEndianStructure, LittleEndianStructure as LittleEndianStructure +from typing import Any, ClassVar, Generic, TypeVar +from typing_extensions import Self, TypeAlias, deprecated + +if sys.platform == "win32": + from _ctypes import FormatError as FormatError, get_last_error as get_last_error, set_last_error as set_last_error + +if sys.version_info >= (3, 11): + from ctypes._endian import BigEndianUnion as BigEndianUnion, LittleEndianUnion as LittleEndianUnion + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_DLLT = TypeVar("_DLLT", bound=CDLL) +_CT = TypeVar("_CT", bound=_CData) + +DEFAULT_MODE: int + +class ArgumentError(Exception): ... + +class CDLL: + _func_flags_: ClassVar[int] + _func_restype_: ClassVar[_CDataType] + _name: str + _handle: int + _FuncPtr: type[_FuncPointer] + def __init__( + self, + name: str | None, + mode: int = ..., + handle: int | None = None, + use_errno: bool = False, + use_last_error: bool = False, + winmode: int | None = None, + ) -> None: ... + def __getattr__(self, name: str) -> _NamedFuncPointer: ... + def __getitem__(self, name_or_ordinal: str) -> _NamedFuncPointer: ... + +if sys.platform == "win32": + class OleDLL(CDLL): ... + class WinDLL(CDLL): ... + +class PyDLL(CDLL): ... + +class LibraryLoader(Generic[_DLLT]): + def __init__(self, dlltype: type[_DLLT]) -> None: ... + def __getattr__(self, name: str) -> _DLLT: ... + def __getitem__(self, name: str) -> _DLLT: ... + def LoadLibrary(self, name: str) -> _DLLT: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +cdll: LibraryLoader[CDLL] +if sys.platform == "win32": + windll: LibraryLoader[WinDLL] + oledll: LibraryLoader[OleDLL] +pydll: LibraryLoader[PyDLL] +pythonapi: PyDLL + +class _FuncPointer(_CFuncPtr): ... + +class _NamedFuncPointer(_FuncPointer): + __name__: str + +def CFUNCTYPE( + restype: type[_CData | _CDataType] | None, + *argtypes: type[_CData | _CDataType], + use_errno: bool = ..., + use_last_error: bool = ..., +) -> type[_FuncPointer]: ... + +if sys.platform == "win32": + def WINFUNCTYPE( + restype: type[_CData | _CDataType] | None, + *argtypes: type[_CData | _CDataType], + use_errno: bool = ..., + use_last_error: bool = ..., + ) -> type[_FuncPointer]: ... + +def PYFUNCTYPE(restype: type[_CData | _CDataType] | None, *argtypes: type[_CData | _CDataType]) -> type[_FuncPointer]: ... + +# Any type that can be implicitly converted to c_void_p when passed as a C function argument. +# (bytes is not included here, see below.) +_CVoidPLike: TypeAlias = _PointerLike | Array[Any] | _CArgObject | int +# Same as above, but including types known to be read-only (i. e. bytes). +# This distinction is not strictly necessary (ctypes doesn't differentiate between const +# and non-const pointers), but it catches errors like memmove(b'foo', buf, 4) +# when memmove(buf, b'foo', 4) was intended. +_CVoidConstPLike: TypeAlias = _CVoidPLike | bytes + +_CastT = TypeVar("_CastT", bound=_CanCastTo) + +def cast(obj: _CData | _CDataType | _CArgObject | int, typ: type[_CastT]) -> _CastT: ... +def create_string_buffer(init: int | bytes, size: int | None = None) -> Array[c_char]: ... + +c_buffer = create_string_buffer + +def create_unicode_buffer(init: int | str, size: int | None = None) -> Array[c_wchar]: ... +@deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") +def SetPointerType( + pointer: type[_Pointer[Any]], cls: Any # noqa: F811 # Redefinition of unused `pointer` from line 22 +) -> None: ... +def ARRAY(typ: _CT, len: int) -> Array[_CT]: ... # Soft Deprecated, no plans to remove + +if sys.platform == "win32": + def DllCanUnloadNow() -> int: ... + def DllGetClassObject(rclsid: Any, riid: Any, ppv: Any) -> int: ... # TODO not documented + def GetLastError() -> int: ... + +def memmove(dst: _CVoidPLike, src: _CVoidConstPLike, count: int) -> int: ... +def memset(dst: _CVoidPLike, c: int, count: int) -> int: ... +def string_at(ptr: _CVoidConstPLike, size: int = -1) -> bytes: ... + +if sys.platform == "win32": + def WinError(code: int | None = None, descr: str | None = None) -> OSError: ... + +def wstring_at(ptr: _CVoidConstPLike, size: int = -1) -> str: ... + +class c_byte(_SimpleCData[int]): ... + +class c_char(_SimpleCData[bytes]): + def __init__(self, value: int | bytes | bytearray = ...) -> None: ... + +class c_char_p(_PointerLike, _SimpleCData[bytes | None]): + def __init__(self, value: int | bytes | None = ...) -> None: ... + @classmethod + def from_param(cls, value: Any, /) -> Self | _CArgObject: ... + +class c_double(_SimpleCData[float]): ... +class c_longdouble(_SimpleCData[float]): ... # can be an alias for c_double +class c_float(_SimpleCData[float]): ... +class c_int(_SimpleCData[int]): ... # can be an alias for c_long +class c_long(_SimpleCData[int]): ... +class c_longlong(_SimpleCData[int]): ... # can be an alias for c_long +class c_short(_SimpleCData[int]): ... +class c_size_t(_SimpleCData[int]): ... # alias for c_uint, c_ulong, or c_ulonglong +class c_ssize_t(_SimpleCData[int]): ... # alias for c_int, c_long, or c_longlong +class c_ubyte(_SimpleCData[int]): ... +class c_uint(_SimpleCData[int]): ... # can be an alias for c_ulong +class c_ulong(_SimpleCData[int]): ... +class c_ulonglong(_SimpleCData[int]): ... # can be an alias for c_ulong +class c_ushort(_SimpleCData[int]): ... + +class c_void_p(_PointerLike, _SimpleCData[int | None]): + @classmethod + def from_param(cls, value: Any, /) -> Self | _CArgObject: ... + +c_voidp = c_void_p # backwards compatibility (to a bug) + +class c_wchar(_SimpleCData[str]): ... + +c_int8 = c_byte + +# these are actually dynamic aliases for c_short, c_int, c_long, or c_longlong +class c_int16(_SimpleCData[int]): ... +class c_int32(_SimpleCData[int]): ... +class c_int64(_SimpleCData[int]): ... + +c_uint8 = c_ubyte + +# these are actually dynamic aliases for c_ushort, c_uint, c_ulong, or c_ulonglong +class c_uint16(_SimpleCData[int]): ... +class c_uint32(_SimpleCData[int]): ... +class c_uint64(_SimpleCData[int]): ... + +class c_wchar_p(_PointerLike, _SimpleCData[str | None]): + def __init__(self, value: int | str | None = ...) -> None: ... + @classmethod + def from_param(cls, value: Any, /) -> Self | _CArgObject: ... + +class c_bool(_SimpleCData[bool]): + def __init__(self, value: bool = ...) -> None: ... + +if sys.platform == "win32": + class HRESULT(_SimpleCData[int]): ... # TODO undocumented + +if sys.version_info >= (3, 12): + c_time_t: type[c_int32 | c_int64] # alias for one or the other at runtime + +class py_object(_CanCastTo, _SimpleCData[_T]): ... + +if sys.version_info >= (3, 14): + class c_float_complex(_SimpleCData[complex]): ... + class c_double_complex(_SimpleCData[complex]): ... + class c_longdouble_complex(_SimpleCData[complex]): ... diff --git a/mypy/typeshed/stdlib/ctypes/_endian.pyi b/mypy/typeshed/stdlib/ctypes/_endian.pyi new file mode 100644 index 000000000000..144f5ba5dd40 --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/_endian.pyi @@ -0,0 +1,12 @@ +import sys +from ctypes import Structure, Union + +# At runtime, the native endianness is an alias for Structure, +# while the other is a subclass with a metaclass added in. +class BigEndianStructure(Structure): ... +class LittleEndianStructure(Structure): ... + +# Same thing for these: one is an alias of Union at runtime +if sys.version_info >= (3, 11): + class BigEndianUnion(Union): ... + class LittleEndianUnion(Union): ... diff --git a/mypy/typeshed/stdlib/ctypes/macholib/__init__.pyi b/mypy/typeshed/stdlib/ctypes/macholib/__init__.pyi new file mode 100644 index 000000000000..bda5b5a7f4cc --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/macholib/__init__.pyi @@ -0,0 +1 @@ +__version__: str diff --git a/mypy/typeshed/stdlib/ctypes/macholib/dyld.pyi b/mypy/typeshed/stdlib/ctypes/macholib/dyld.pyi new file mode 100644 index 000000000000..c7e94daa2149 --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/macholib/dyld.pyi @@ -0,0 +1,8 @@ +from collections.abc import Mapping +from ctypes.macholib.dylib import dylib_info as dylib_info +from ctypes.macholib.framework import framework_info as framework_info + +__all__ = ["dyld_find", "framework_find", "framework_info", "dylib_info"] + +def dyld_find(name: str, executable_path: str | None = None, env: Mapping[str, str] | None = None) -> str: ... +def framework_find(fn: str, executable_path: str | None = None, env: Mapping[str, str] | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/ctypes/macholib/dylib.pyi b/mypy/typeshed/stdlib/ctypes/macholib/dylib.pyi new file mode 100644 index 000000000000..95945edfd155 --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/macholib/dylib.pyi @@ -0,0 +1,14 @@ +from typing import TypedDict, type_check_only + +__all__ = ["dylib_info"] + +# Actual result is produced by re.match.groupdict() +@type_check_only +class _DylibInfo(TypedDict): + location: str + name: str + shortname: str + version: str | None + suffix: str | None + +def dylib_info(filename: str) -> _DylibInfo | None: ... diff --git a/mypy/typeshed/stdlib/ctypes/macholib/framework.pyi b/mypy/typeshed/stdlib/ctypes/macholib/framework.pyi new file mode 100644 index 000000000000..e92bf3700e84 --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/macholib/framework.pyi @@ -0,0 +1,14 @@ +from typing import TypedDict, type_check_only + +__all__ = ["framework_info"] + +# Actual result is produced by re.match.groupdict() +@type_check_only +class _FrameworkInfo(TypedDict): + location: str + name: str + shortname: str + version: str | None + suffix: str | None + +def framework_info(filename: str) -> _FrameworkInfo | None: ... diff --git a/mypy/typeshed/stdlib/ctypes/util.pyi b/mypy/typeshed/stdlib/ctypes/util.pyi new file mode 100644 index 000000000000..316f7a2b3e2f --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/util.pyi @@ -0,0 +1,8 @@ +import sys + +def find_library(name: str) -> str | None: ... + +if sys.platform == "win32": + def find_msvcrt() -> str | None: ... + +def test() -> None: ... diff --git a/mypy/typeshed/stdlib/ctypes/wintypes.pyi b/mypy/typeshed/stdlib/ctypes/wintypes.pyi new file mode 100644 index 000000000000..e938d8f22957 --- /dev/null +++ b/mypy/typeshed/stdlib/ctypes/wintypes.pyi @@ -0,0 +1,304 @@ +from _ctypes import _CArgObject, _CField +from ctypes import ( + Array, + Structure, + _Pointer, + _SimpleCData, + c_byte, + c_char, + c_char_p, + c_double, + c_float, + c_int, + c_long, + c_longlong, + c_short, + c_uint, + c_ulong, + c_ulonglong, + c_ushort, + c_void_p, + c_wchar, + c_wchar_p, +) +from typing import Any, TypeVar +from typing_extensions import Self, TypeAlias + +BYTE = c_byte +WORD = c_ushort +DWORD = c_ulong +CHAR = c_char +WCHAR = c_wchar +UINT = c_uint +INT = c_int +DOUBLE = c_double +FLOAT = c_float +BOOLEAN = BYTE +BOOL = c_long + +class VARIANT_BOOL(_SimpleCData[bool]): ... + +ULONG = c_ulong +LONG = c_long +USHORT = c_ushort +SHORT = c_short +LARGE_INTEGER = c_longlong +_LARGE_INTEGER = c_longlong +ULARGE_INTEGER = c_ulonglong +_ULARGE_INTEGER = c_ulonglong + +OLESTR = c_wchar_p +LPOLESTR = c_wchar_p +LPCOLESTR = c_wchar_p +LPWSTR = c_wchar_p +LPCWSTR = c_wchar_p +LPSTR = c_char_p +LPCSTR = c_char_p +LPVOID = c_void_p +LPCVOID = c_void_p + +# These two types are pointer-sized unsigned and signed ints, respectively. +# At runtime, they are either c_[u]long or c_[u]longlong, depending on the host's pointer size +# (they are not really separate classes). +class WPARAM(_SimpleCData[int]): ... +class LPARAM(_SimpleCData[int]): ... + +ATOM = WORD +LANGID = WORD +COLORREF = DWORD +LGRPID = DWORD +LCTYPE = DWORD +LCID = DWORD + +HANDLE = c_void_p +HACCEL = HANDLE +HBITMAP = HANDLE +HBRUSH = HANDLE +HCOLORSPACE = HANDLE +HDC = HANDLE +HDESK = HANDLE +HDWP = HANDLE +HENHMETAFILE = HANDLE +HFONT = HANDLE +HGDIOBJ = HANDLE +HGLOBAL = HANDLE +HHOOK = HANDLE +HICON = HANDLE +HINSTANCE = HANDLE +HKEY = HANDLE +HKL = HANDLE +HLOCAL = HANDLE +HMENU = HANDLE +HMETAFILE = HANDLE +HMODULE = HANDLE +HMONITOR = HANDLE +HPALETTE = HANDLE +HPEN = HANDLE +HRGN = HANDLE +HRSRC = HANDLE +HSTR = HANDLE +HTASK = HANDLE +HWINSTA = HANDLE +HWND = HANDLE +SC_HANDLE = HANDLE +SERVICE_STATUS_HANDLE = HANDLE + +_CIntLikeT = TypeVar("_CIntLikeT", bound=_SimpleCData[int]) +_CIntLikeField: TypeAlias = _CField[_CIntLikeT, int, _CIntLikeT | int] + +class RECT(Structure): + left: _CIntLikeField[LONG] + top: _CIntLikeField[LONG] + right: _CIntLikeField[LONG] + bottom: _CIntLikeField[LONG] + +RECTL = RECT +_RECTL = RECT +tagRECT = RECT + +class _SMALL_RECT(Structure): + Left: _CIntLikeField[SHORT] + Top: _CIntLikeField[SHORT] + Right: _CIntLikeField[SHORT] + Bottom: _CIntLikeField[SHORT] + +SMALL_RECT = _SMALL_RECT + +class _COORD(Structure): + X: _CIntLikeField[SHORT] + Y: _CIntLikeField[SHORT] + +class POINT(Structure): + x: _CIntLikeField[LONG] + y: _CIntLikeField[LONG] + +POINTL = POINT +_POINTL = POINT +tagPOINT = POINT + +class SIZE(Structure): + cx: _CIntLikeField[LONG] + cy: _CIntLikeField[LONG] + +SIZEL = SIZE +tagSIZE = SIZE + +def RGB(red: int, green: int, blue: int) -> int: ... + +class FILETIME(Structure): + dwLowDateTime: _CIntLikeField[DWORD] + dwHighDateTime: _CIntLikeField[DWORD] + +_FILETIME = FILETIME + +class MSG(Structure): + hWnd: _CField[HWND, int | None, HWND | int | None] + message: _CIntLikeField[UINT] + wParam: _CIntLikeField[WPARAM] + lParam: _CIntLikeField[LPARAM] + time: _CIntLikeField[DWORD] + pt: _CField[POINT, POINT, POINT] + +tagMSG = MSG +MAX_PATH: int + +class WIN32_FIND_DATAA(Structure): + dwFileAttributes: _CIntLikeField[DWORD] + ftCreationTime: _CField[FILETIME, FILETIME, FILETIME] + ftLastAccessTime: _CField[FILETIME, FILETIME, FILETIME] + ftLastWriteTime: _CField[FILETIME, FILETIME, FILETIME] + nFileSizeHigh: _CIntLikeField[DWORD] + nFileSizeLow: _CIntLikeField[DWORD] + dwReserved0: _CIntLikeField[DWORD] + dwReserved1: _CIntLikeField[DWORD] + cFileName: _CField[Array[CHAR], bytes, bytes] + cAlternateFileName: _CField[Array[CHAR], bytes, bytes] + +class WIN32_FIND_DATAW(Structure): + dwFileAttributes: _CIntLikeField[DWORD] + ftCreationTime: _CField[FILETIME, FILETIME, FILETIME] + ftLastAccessTime: _CField[FILETIME, FILETIME, FILETIME] + ftLastWriteTime: _CField[FILETIME, FILETIME, FILETIME] + nFileSizeHigh: _CIntLikeField[DWORD] + nFileSizeLow: _CIntLikeField[DWORD] + dwReserved0: _CIntLikeField[DWORD] + dwReserved1: _CIntLikeField[DWORD] + cFileName: _CField[Array[WCHAR], str, str] + cAlternateFileName: _CField[Array[WCHAR], str, str] + +# These are all defined with the POINTER() function, which keeps a cache and will +# return a previously created class if it can. The self-reported __name__ +# of these classes is f"LP_{typ.__name__}", where typ is the original class +# passed in to the POINTER() function. + +# LP_c_short +class PSHORT(_Pointer[SHORT]): ... + +# LP_c_ushort +class PUSHORT(_Pointer[USHORT]): ... + +PWORD = PUSHORT +LPWORD = PUSHORT + +# LP_c_long +class PLONG(_Pointer[LONG]): ... + +LPLONG = PLONG +PBOOL = PLONG +LPBOOL = PLONG + +# LP_c_ulong +class PULONG(_Pointer[ULONG]): ... + +PDWORD = PULONG +LPDWORD = PDWORD +LPCOLORREF = PDWORD +PLCID = PDWORD + +# LP_c_int (or LP_c_long if int and long have the same size) +class PINT(_Pointer[INT]): ... + +LPINT = PINT + +# LP_c_uint (or LP_c_ulong if int and long have the same size) +class PUINT(_Pointer[UINT]): ... + +LPUINT = PUINT + +# LP_c_float +class PFLOAT(_Pointer[FLOAT]): ... + +# LP_c_longlong (or LP_c_long if long and long long have the same size) +class PLARGE_INTEGER(_Pointer[LARGE_INTEGER]): ... + +# LP_c_ulonglong (or LP_c_ulong if long and long long have the same size) +class PULARGE_INTEGER(_Pointer[ULARGE_INTEGER]): ... + +# LP_c_byte types +class PBYTE(_Pointer[BYTE]): ... + +LPBYTE = PBYTE +PBOOLEAN = PBYTE + +# LP_c_char +class PCHAR(_Pointer[CHAR]): + # this is inherited from ctypes.c_char_p, kind of. + @classmethod + def from_param(cls, value: Any, /) -> Self | _CArgObject: ... + +# LP_c_wchar +class PWCHAR(_Pointer[WCHAR]): + # inherited from ctypes.c_wchar_p, kind of + @classmethod + def from_param(cls, value: Any, /) -> Self | _CArgObject: ... + +# LP_c_void_p +class PHANDLE(_Pointer[HANDLE]): ... + +LPHANDLE = PHANDLE +PHKEY = PHANDLE +LPHKL = PHANDLE +LPSC_HANDLE = PHANDLE + +# LP_FILETIME +class PFILETIME(_Pointer[FILETIME]): ... + +LPFILETIME = PFILETIME + +# LP_MSG +class PMSG(_Pointer[MSG]): ... + +LPMSG = PMSG + +# LP_POINT +class PPOINT(_Pointer[POINT]): ... + +LPPOINT = PPOINT +PPOINTL = PPOINT + +# LP_RECT +class PRECT(_Pointer[RECT]): ... + +LPRECT = PRECT +PRECTL = PRECT +LPRECTL = PRECT + +# LP_SIZE +class PSIZE(_Pointer[SIZE]): ... + +LPSIZE = PSIZE +PSIZEL = PSIZE +LPSIZEL = PSIZE + +# LP__SMALL_RECT +class PSMALL_RECT(_Pointer[SMALL_RECT]): ... + +# LP_WIN32_FIND_DATAA +class PWIN32_FIND_DATAA(_Pointer[WIN32_FIND_DATAA]): ... + +LPWIN32_FIND_DATAA = PWIN32_FIND_DATAA + +# LP_WIN32_FIND_DATAW +class PWIN32_FIND_DATAW(_Pointer[WIN32_FIND_DATAW]): ... + +LPWIN32_FIND_DATAW = PWIN32_FIND_DATAW diff --git a/mypy/typeshed/stdlib/curses/__init__.pyi b/mypy/typeshed/stdlib/curses/__init__.pyi new file mode 100644 index 000000000000..edc64a00cd39 --- /dev/null +++ b/mypy/typeshed/stdlib/curses/__init__.pyi @@ -0,0 +1,45 @@ +import sys +from _curses import * +from _curses import window as window +from _typeshed import structseq +from collections.abc import Callable +from typing import Final, TypeVar, final, type_check_only +from typing_extensions import Concatenate, ParamSpec + +# NOTE: The _curses module is ordinarily only available on Unix, but the +# windows-curses package makes it available on Windows as well with the same +# contents. + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +# available after calling `curses.initscr()` +LINES: int +COLS: int + +# available after calling `curses.start_color()` +COLORS: int +COLOR_PAIRS: int + +def wrapper(func: Callable[Concatenate[window, _P], _T], /, *arg: _P.args, **kwds: _P.kwargs) -> _T: ... + +# typeshed used the name _CursesWindow for the underlying C class before +# it was mapped to the name 'window' in 3.8. +# Kept here as a legacy alias in case any third-party code is relying on it. +_CursesWindow = window + +# At runtime this class is unexposed and calls itself curses.ncurses_version. +# That name would conflict with the actual curses.ncurses_version, which is +# an instance of this class. +@final +@type_check_only +class _ncurses_version(structseq[int], tuple[int, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("major", "minor", "patch") + + @property + def major(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def patch(self) -> int: ... diff --git a/mypy/typeshed/stdlib/curses/ascii.pyi b/mypy/typeshed/stdlib/curses/ascii.pyi new file mode 100644 index 000000000000..66efbe36a7df --- /dev/null +++ b/mypy/typeshed/stdlib/curses/ascii.pyi @@ -0,0 +1,62 @@ +from typing import TypeVar + +_CharT = TypeVar("_CharT", str, int) + +NUL: int +SOH: int +STX: int +ETX: int +EOT: int +ENQ: int +ACK: int +BEL: int +BS: int +TAB: int +HT: int +LF: int +NL: int +VT: int +FF: int +CR: int +SO: int +SI: int +DLE: int +DC1: int +DC2: int +DC3: int +DC4: int +NAK: int +SYN: int +ETB: int +CAN: int +EM: int +SUB: int +ESC: int +FS: int +GS: int +RS: int +US: int +SP: int +DEL: int + +controlnames: list[int] + +def isalnum(c: str | int) -> bool: ... +def isalpha(c: str | int) -> bool: ... +def isascii(c: str | int) -> bool: ... +def isblank(c: str | int) -> bool: ... +def iscntrl(c: str | int) -> bool: ... +def isdigit(c: str | int) -> bool: ... +def isgraph(c: str | int) -> bool: ... +def islower(c: str | int) -> bool: ... +def isprint(c: str | int) -> bool: ... +def ispunct(c: str | int) -> bool: ... +def isspace(c: str | int) -> bool: ... +def isupper(c: str | int) -> bool: ... +def isxdigit(c: str | int) -> bool: ... +def isctrl(c: str | int) -> bool: ... +def ismeta(c: str | int) -> bool: ... +def ascii(c: _CharT) -> _CharT: ... +def ctrl(c: _CharT) -> _CharT: ... +def alt(c: _CharT) -> _CharT: ... +def unctrl(c: str | int) -> str: ... diff --git a/mypy/typeshed/stdlib/curses/has_key.pyi b/mypy/typeshed/stdlib/curses/has_key.pyi new file mode 100644 index 000000000000..3811060b916a --- /dev/null +++ b/mypy/typeshed/stdlib/curses/has_key.pyi @@ -0,0 +1 @@ +def has_key(ch: int | str) -> bool: ... diff --git a/mypy/typeshed/stdlib/curses/panel.pyi b/mypy/typeshed/stdlib/curses/panel.pyi new file mode 100644 index 000000000000..861559d38bc5 --- /dev/null +++ b/mypy/typeshed/stdlib/curses/panel.pyi @@ -0,0 +1 @@ +from _curses_panel import * diff --git a/mypy/typeshed/stdlib/curses/textpad.pyi b/mypy/typeshed/stdlib/curses/textpad.pyi new file mode 100644 index 000000000000..48ef67c9d85f --- /dev/null +++ b/mypy/typeshed/stdlib/curses/textpad.pyi @@ -0,0 +1,11 @@ +from _curses import window +from collections.abc import Callable + +def rectangle(win: window, uly: int, ulx: int, lry: int, lrx: int) -> None: ... + +class Textbox: + stripspaces: bool + def __init__(self, win: window, insert_mode: bool = False) -> None: ... + def edit(self, validate: Callable[[int], int] | None = None) -> str: ... + def do_command(self, ch: str | int) -> None: ... + def gather(self) -> str: ... diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi new file mode 100644 index 000000000000..3295b1c1f835 --- /dev/null +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -0,0 +1,316 @@ +import enum +import sys +import types +from _typeshed import DataclassInstance +from builtins import type as Type # alias to avoid name clashes with fields named "type" +from collections.abc import Callable, Iterable, Mapping +from typing import Any, Generic, Literal, Protocol, TypeVar, overload +from typing_extensions import Never, TypeAlias, TypeIs + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) + +__all__ = [ + "dataclass", + "field", + "Field", + "FrozenInstanceError", + "InitVar", + "MISSING", + "fields", + "asdict", + "astuple", + "make_dataclass", + "replace", + "is_dataclass", +] + +if sys.version_info >= (3, 10): + __all__ += ["KW_ONLY"] + +_DataclassT = TypeVar("_DataclassT", bound=DataclassInstance) + +# define _MISSING_TYPE as an enum within the type stubs, +# even though that is not really its type at runtime +# this allows us to use Literal[_MISSING_TYPE.MISSING] +# for background, see: +# https://github.com/python/typeshed/pull/5900#issuecomment-895513797 +class _MISSING_TYPE(enum.Enum): + MISSING = enum.auto() + +MISSING = _MISSING_TYPE.MISSING + +if sys.version_info >= (3, 10): + class KW_ONLY: ... + +@overload +def asdict(obj: DataclassInstance) -> dict[str, Any]: ... +@overload +def asdict(obj: DataclassInstance, *, dict_factory: Callable[[list[tuple[str, Any]]], _T]) -> _T: ... +@overload +def astuple(obj: DataclassInstance) -> tuple[Any, ...]: ... +@overload +def astuple(obj: DataclassInstance, *, tuple_factory: Callable[[list[Any]], _T]) -> _T: ... +@overload +def dataclass(cls: None, /) -> Callable[[type[_T]], type[_T]]: ... +@overload +def dataclass(cls: type[_T], /) -> type[_T]: ... + +if sys.version_info >= (3, 11): + @overload + def dataclass( + *, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + match_args: bool = True, + kw_only: bool = False, + slots: bool = False, + weakref_slot: bool = False, + ) -> Callable[[type[_T]], type[_T]]: ... + +elif sys.version_info >= (3, 10): + @overload + def dataclass( + *, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + match_args: bool = True, + kw_only: bool = False, + slots: bool = False, + ) -> Callable[[type[_T]], type[_T]]: ... + +else: + @overload + def dataclass( + *, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + ) -> Callable[[type[_T]], type[_T]]: ... + +# See https://github.com/python/mypy/issues/10750 +class _DefaultFactory(Protocol[_T_co]): + def __call__(self) -> _T_co: ... + +class Field(Generic[_T]): + name: str + type: Type[_T] | str | Any + default: _T | Literal[_MISSING_TYPE.MISSING] + default_factory: _DefaultFactory[_T] | Literal[_MISSING_TYPE.MISSING] + repr: bool + hash: bool | None + init: bool + compare: bool + metadata: types.MappingProxyType[Any, Any] + if sys.version_info >= (3, 10): + kw_only: bool | Literal[_MISSING_TYPE.MISSING] + def __init__( + self, + default: _T, + default_factory: Callable[[], _T], + init: bool, + repr: bool, + hash: bool | None, + compare: bool, + metadata: Mapping[Any, Any], + kw_only: bool, + ) -> None: ... + else: + def __init__( + self, + default: _T, + default_factory: Callable[[], _T], + init: bool, + repr: bool, + hash: bool | None, + compare: bool, + metadata: Mapping[Any, Any], + ) -> None: ... + + def __set_name__(self, owner: Type[Any], name: str) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +# NOTE: Actual return type is 'Field[_T]', but we want to help type checkers +# to understand the magic that happens at runtime. +if sys.version_info >= (3, 10): + @overload # `default` and `default_factory` are optional and mutually exclusive. + def field( + *, + default: _T, + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: Mapping[Any, Any] | None = None, + kw_only: bool = ..., + ) -> _T: ... + @overload + def field( + *, + default_factory: Callable[[], _T], + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: Mapping[Any, Any] | None = None, + kw_only: bool = ..., + ) -> _T: ... + @overload + def field( + *, + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: Mapping[Any, Any] | None = None, + kw_only: bool = ..., + ) -> Any: ... + +else: + @overload # `default` and `default_factory` are optional and mutually exclusive. + def field( + *, + default: _T, + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: Mapping[Any, Any] | None = None, + ) -> _T: ... + @overload + def field( + *, + default_factory: Callable[[], _T], + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: Mapping[Any, Any] | None = None, + ) -> _T: ... + @overload + def field( + *, + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: Mapping[Any, Any] | None = None, + ) -> Any: ... + +def fields(class_or_instance: DataclassInstance | type[DataclassInstance]) -> tuple[Field[Any], ...]: ... + +# HACK: `obj: Never` typing matches if object argument is using `Any` type. +@overload +def is_dataclass(obj: Never) -> TypeIs[DataclassInstance | type[DataclassInstance]]: ... # type: ignore[narrowed-type-not-subtype] # pyright: ignore[reportGeneralTypeIssues] +@overload +def is_dataclass(obj: type) -> TypeIs[type[DataclassInstance]]: ... +@overload +def is_dataclass(obj: object) -> TypeIs[DataclassInstance | type[DataclassInstance]]: ... + +class FrozenInstanceError(AttributeError): ... + +if sys.version_info >= (3, 9): + _InitVarMeta: TypeAlias = type +else: + class _InitVarMeta(type): + # Not used, instead `InitVar.__class_getitem__` is called. + # pyright (not unreasonably) thinks this is an invalid use of InitVar. + def __getitem__(self, params: Any) -> InitVar[Any]: ... # pyright: ignore[reportInvalidTypeForm] + +class InitVar(Generic[_T], metaclass=_InitVarMeta): + type: Type[_T] + def __init__(self, type: Type[_T]) -> None: ... + if sys.version_info >= (3, 9): + @overload + def __class_getitem__(cls, type: Type[_T]) -> InitVar[_T]: ... # pyright: ignore[reportInvalidTypeForm] + @overload + def __class_getitem__(cls, type: Any) -> InitVar[Any]: ... # pyright: ignore[reportInvalidTypeForm] + +if sys.version_info >= (3, 12): + def make_dataclass( + cls_name: str, + fields: Iterable[str | tuple[str, Any] | tuple[str, Any, Any]], + *, + bases: tuple[type, ...] = (), + namespace: dict[str, Any] | None = None, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + match_args: bool = True, + kw_only: bool = False, + slots: bool = False, + weakref_slot: bool = False, + module: str | None = None, + ) -> type: ... + +elif sys.version_info >= (3, 11): + def make_dataclass( + cls_name: str, + fields: Iterable[str | tuple[str, Any] | tuple[str, Any, Any]], + *, + bases: tuple[type, ...] = (), + namespace: dict[str, Any] | None = None, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + match_args: bool = True, + kw_only: bool = False, + slots: bool = False, + weakref_slot: bool = False, + ) -> type: ... + +elif sys.version_info >= (3, 10): + def make_dataclass( + cls_name: str, + fields: Iterable[str | tuple[str, Any] | tuple[str, Any, Any]], + *, + bases: tuple[type, ...] = (), + namespace: dict[str, Any] | None = None, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + match_args: bool = True, + kw_only: bool = False, + slots: bool = False, + ) -> type: ... + +else: + def make_dataclass( + cls_name: str, + fields: Iterable[str | tuple[str, Any] | tuple[str, Any, Any]], + *, + bases: tuple[type, ...] = (), + namespace: dict[str, Any] | None = None, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + ) -> type: ... + +def replace(obj: _DataclassT, /, **changes: Any) -> _DataclassT: ... diff --git a/mypy/typeshed/stdlib/datetime.pyi b/mypy/typeshed/stdlib/datetime.pyi new file mode 100644 index 000000000000..87037ef39be7 --- /dev/null +++ b/mypy/typeshed/stdlib/datetime.pyi @@ -0,0 +1,336 @@ +import sys +from abc import abstractmethod +from time import struct_time +from typing import ClassVar, Final, NoReturn, SupportsIndex, final, overload, type_check_only +from typing_extensions import CapsuleType, Self, TypeAlias, deprecated + +if sys.version_info >= (3, 11): + __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") +elif sys.version_info >= (3, 9): + __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR") + +MINYEAR: Final = 1 +MAXYEAR: Final = 9999 + +class tzinfo: + @abstractmethod + def tzname(self, dt: datetime | None, /) -> str | None: ... + @abstractmethod + def utcoffset(self, dt: datetime | None, /) -> timedelta | None: ... + @abstractmethod + def dst(self, dt: datetime | None, /) -> timedelta | None: ... + def fromutc(self, dt: datetime, /) -> datetime: ... + +# Alias required to avoid name conflicts with date(time).tzinfo. +_TzInfo: TypeAlias = tzinfo + +@final +class timezone(tzinfo): + utc: ClassVar[timezone] + min: ClassVar[timezone] + max: ClassVar[timezone] + def __init__(self, offset: timedelta, name: str = ...) -> None: ... + def tzname(self, dt: datetime | None, /) -> str: ... + def utcoffset(self, dt: datetime | None, /) -> timedelta: ... + def dst(self, dt: datetime | None, /) -> None: ... + def __hash__(self) -> int: ... + def __eq__(self, value: object, /) -> bool: ... + +if sys.version_info >= (3, 11): + UTC: timezone + +if sys.version_info >= (3, 9): + # This class calls itself datetime.IsoCalendarDate. It's neither + # NamedTuple nor structseq. + @final + @type_check_only + class _IsoCalendarDate(tuple[int, int, int]): + @property + def year(self) -> int: ... + @property + def week(self) -> int: ... + @property + def weekday(self) -> int: ... + +class date: + min: ClassVar[date] + max: ClassVar[date] + resolution: ClassVar[timedelta] + def __new__(cls, year: SupportsIndex, month: SupportsIndex, day: SupportsIndex) -> Self: ... + @classmethod + def fromtimestamp(cls, timestamp: float, /) -> Self: ... + @classmethod + def today(cls) -> Self: ... + @classmethod + def fromordinal(cls, n: int, /) -> Self: ... + @classmethod + def fromisoformat(cls, date_string: str, /) -> Self: ... + @classmethod + def fromisocalendar(cls, year: int, week: int, day: int) -> Self: ... + @property + def year(self) -> int: ... + @property + def month(self) -> int: ... + @property + def day(self) -> int: ... + def ctime(self) -> str: ... + # On <3.12, the name of the parameter in the pure-Python implementation + # didn't match the name in the C implementation, + # meaning it is only *safe* to pass it as a keyword argument on 3.12+ + if sys.version_info >= (3, 12): + def strftime(self, format: str) -> str: ... + else: + def strftime(self, format: str, /) -> str: ... + + def __format__(self, fmt: str, /) -> str: ... + def isoformat(self) -> str: ... + def timetuple(self) -> struct_time: ... + def toordinal(self) -> int: ... + if sys.version_info >= (3, 13): + def __replace__(self, /, *, year: SupportsIndex = ..., month: SupportsIndex = ..., day: SupportsIndex = ...) -> Self: ... + + def replace(self, year: SupportsIndex = ..., month: SupportsIndex = ..., day: SupportsIndex = ...) -> Self: ... + def __le__(self, value: date, /) -> bool: ... + def __lt__(self, value: date, /) -> bool: ... + def __ge__(self, value: date, /) -> bool: ... + def __gt__(self, value: date, /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __add__(self, value: timedelta, /) -> Self: ... + def __radd__(self, value: timedelta, /) -> Self: ... + @overload + def __sub__(self, value: datetime, /) -> NoReturn: ... + @overload + def __sub__(self, value: Self, /) -> timedelta: ... + @overload + def __sub__(self, value: timedelta, /) -> Self: ... + def __hash__(self) -> int: ... + def weekday(self) -> int: ... + def isoweekday(self) -> int: ... + if sys.version_info >= (3, 9): + def isocalendar(self) -> _IsoCalendarDate: ... + else: + def isocalendar(self) -> tuple[int, int, int]: ... + +class time: + min: ClassVar[time] + max: ClassVar[time] + resolution: ClassVar[timedelta] + def __new__( + cls, + hour: SupportsIndex = ..., + minute: SupportsIndex = ..., + second: SupportsIndex = ..., + microsecond: SupportsIndex = ..., + tzinfo: _TzInfo | None = ..., + *, + fold: int = ..., + ) -> Self: ... + @property + def hour(self) -> int: ... + @property + def minute(self) -> int: ... + @property + def second(self) -> int: ... + @property + def microsecond(self) -> int: ... + @property + def tzinfo(self) -> _TzInfo | None: ... + @property + def fold(self) -> int: ... + def __le__(self, value: time, /) -> bool: ... + def __lt__(self, value: time, /) -> bool: ... + def __ge__(self, value: time, /) -> bool: ... + def __gt__(self, value: time, /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + def isoformat(self, timespec: str = ...) -> str: ... + @classmethod + def fromisoformat(cls, time_string: str, /) -> Self: ... + # On <3.12, the name of the parameter in the pure-Python implementation + # didn't match the name in the C implementation, + # meaning it is only *safe* to pass it as a keyword argument on 3.12+ + if sys.version_info >= (3, 12): + def strftime(self, format: str) -> str: ... + else: + def strftime(self, format: str, /) -> str: ... + + def __format__(self, fmt: str, /) -> str: ... + def utcoffset(self) -> timedelta | None: ... + def tzname(self) -> str | None: ... + def dst(self) -> timedelta | None: ... + if sys.version_info >= (3, 13): + def __replace__( + self, + /, + *, + hour: SupportsIndex = ..., + minute: SupportsIndex = ..., + second: SupportsIndex = ..., + microsecond: SupportsIndex = ..., + tzinfo: _TzInfo | None = ..., + fold: int = ..., + ) -> Self: ... + + def replace( + self, + hour: SupportsIndex = ..., + minute: SupportsIndex = ..., + second: SupportsIndex = ..., + microsecond: SupportsIndex = ..., + tzinfo: _TzInfo | None = ..., + *, + fold: int = ..., + ) -> Self: ... + +_Date: TypeAlias = date +_Time: TypeAlias = time + +class timedelta: + min: ClassVar[timedelta] + max: ClassVar[timedelta] + resolution: ClassVar[timedelta] + def __new__( + cls, + days: float = ..., + seconds: float = ..., + microseconds: float = ..., + milliseconds: float = ..., + minutes: float = ..., + hours: float = ..., + weeks: float = ..., + ) -> Self: ... + @property + def days(self) -> int: ... + @property + def seconds(self) -> int: ... + @property + def microseconds(self) -> int: ... + def total_seconds(self) -> float: ... + def __add__(self, value: timedelta, /) -> timedelta: ... + def __radd__(self, value: timedelta, /) -> timedelta: ... + def __sub__(self, value: timedelta, /) -> timedelta: ... + def __rsub__(self, value: timedelta, /) -> timedelta: ... + def __neg__(self) -> timedelta: ... + def __pos__(self) -> timedelta: ... + def __abs__(self) -> timedelta: ... + def __mul__(self, value: float, /) -> timedelta: ... + def __rmul__(self, value: float, /) -> timedelta: ... + @overload + def __floordiv__(self, value: timedelta, /) -> int: ... + @overload + def __floordiv__(self, value: int, /) -> timedelta: ... + @overload + def __truediv__(self, value: timedelta, /) -> float: ... + @overload + def __truediv__(self, value: float, /) -> timedelta: ... + def __mod__(self, value: timedelta, /) -> timedelta: ... + def __divmod__(self, value: timedelta, /) -> tuple[int, timedelta]: ... + def __le__(self, value: timedelta, /) -> bool: ... + def __lt__(self, value: timedelta, /) -> bool: ... + def __ge__(self, value: timedelta, /) -> bool: ... + def __gt__(self, value: timedelta, /) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __bool__(self) -> bool: ... + def __hash__(self) -> int: ... + +class datetime(date): + min: ClassVar[datetime] + max: ClassVar[datetime] + def __new__( + cls, + year: SupportsIndex, + month: SupportsIndex, + day: SupportsIndex, + hour: SupportsIndex = ..., + minute: SupportsIndex = ..., + second: SupportsIndex = ..., + microsecond: SupportsIndex = ..., + tzinfo: _TzInfo | None = ..., + *, + fold: int = ..., + ) -> Self: ... + @property + def hour(self) -> int: ... + @property + def minute(self) -> int: ... + @property + def second(self) -> int: ... + @property + def microsecond(self) -> int: ... + @property + def tzinfo(self) -> _TzInfo | None: ... + @property + def fold(self) -> int: ... + # On <3.12, the name of the first parameter in the pure-Python implementation + # didn't match the name in the C implementation, + # meaning it is only *safe* to pass it as a keyword argument on 3.12+ + if sys.version_info >= (3, 12): + @classmethod + def fromtimestamp(cls, timestamp: float, tz: _TzInfo | None = ...) -> Self: ... + else: + @classmethod + def fromtimestamp(cls, timestamp: float, /, tz: _TzInfo | None = ...) -> Self: ... + + @classmethod + @deprecated("Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .fromtimestamp(datetime.timezone.utc)") + def utcfromtimestamp(cls, t: float, /) -> Self: ... + @classmethod + def now(cls, tz: _TzInfo | None = None) -> Self: ... + @classmethod + @deprecated("Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .now(datetime.timezone.utc)") + def utcnow(cls) -> Self: ... + @classmethod + def combine(cls, date: _Date, time: _Time, tzinfo: _TzInfo | None = ...) -> Self: ... + def timestamp(self) -> float: ... + def utctimetuple(self) -> struct_time: ... + def date(self) -> _Date: ... + def time(self) -> _Time: ... + def timetz(self) -> _Time: ... + if sys.version_info >= (3, 13): + def __replace__( + self, + /, + *, + year: SupportsIndex = ..., + month: SupportsIndex = ..., + day: SupportsIndex = ..., + hour: SupportsIndex = ..., + minute: SupportsIndex = ..., + second: SupportsIndex = ..., + microsecond: SupportsIndex = ..., + tzinfo: _TzInfo | None = ..., + fold: int = ..., + ) -> Self: ... + + def replace( + self, + year: SupportsIndex = ..., + month: SupportsIndex = ..., + day: SupportsIndex = ..., + hour: SupportsIndex = ..., + minute: SupportsIndex = ..., + second: SupportsIndex = ..., + microsecond: SupportsIndex = ..., + tzinfo: _TzInfo | None = ..., + *, + fold: int = ..., + ) -> Self: ... + def astimezone(self, tz: _TzInfo | None = ...) -> Self: ... + def isoformat(self, sep: str = ..., timespec: str = ...) -> str: ... + @classmethod + def strptime(cls, date_string: str, format: str, /) -> Self: ... + def utcoffset(self) -> timedelta | None: ... + def tzname(self) -> str | None: ... + def dst(self) -> timedelta | None: ... + def __le__(self, value: datetime, /) -> bool: ... # type: ignore[override] + def __lt__(self, value: datetime, /) -> bool: ... # type: ignore[override] + def __ge__(self, value: datetime, /) -> bool: ... # type: ignore[override] + def __gt__(self, value: datetime, /) -> bool: ... # type: ignore[override] + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + @overload # type: ignore[override] + def __sub__(self, value: Self, /) -> timedelta: ... + @overload + def __sub__(self, value: timedelta, /) -> Self: ... + +datetime_CAPI: CapsuleType diff --git a/mypy/typeshed/stdlib/dbm/__init__.pyi b/mypy/typeshed/stdlib/dbm/__init__.pyi new file mode 100644 index 000000000000..7f344060f9ab --- /dev/null +++ b/mypy/typeshed/stdlib/dbm/__init__.pyi @@ -0,0 +1,104 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Iterator, MutableMapping +from types import TracebackType +from typing import Literal, type_check_only +from typing_extensions import Self, TypeAlias + +__all__ = ["open", "whichdb", "error"] + +_KeyType: TypeAlias = str | bytes +_ValueType: TypeAlias = str | bytes | bytearray +_TFlags: TypeAlias = Literal[ + "r", + "w", + "c", + "n", + "rf", + "wf", + "cf", + "nf", + "rs", + "ws", + "cs", + "ns", + "ru", + "wu", + "cu", + "nu", + "rfs", + "wfs", + "cfs", + "nfs", + "rfu", + "wfu", + "cfu", + "nfu", + "rsf", + "wsf", + "csf", + "nsf", + "rsu", + "wsu", + "csu", + "nsu", + "ruf", + "wuf", + "cuf", + "nuf", + "rus", + "wus", + "cus", + "nus", + "rfsu", + "wfsu", + "cfsu", + "nfsu", + "rfus", + "wfus", + "cfus", + "nfus", + "rsfu", + "wsfu", + "csfu", + "nsfu", + "rsuf", + "wsuf", + "csuf", + "nsuf", + "rufs", + "wufs", + "cufs", + "nufs", + "rusf", + "wusf", + "cusf", + "nusf", +] + +class _Database(MutableMapping[_KeyType, bytes]): + def close(self) -> None: ... + def __getitem__(self, key: _KeyType) -> bytes: ... + def __setitem__(self, key: _KeyType, value: _ValueType) -> None: ... + def __delitem__(self, key: _KeyType) -> None: ... + def __iter__(self) -> Iterator[bytes]: ... + def __len__(self) -> int: ... + def __del__(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + +# This class is not exposed. It calls itself dbm.error. +@type_check_only +class _error(Exception): ... + +error: tuple[type[_error], type[OSError]] + +if sys.version_info >= (3, 11): + def whichdb(filename: StrOrBytesPath) -> str | None: ... + def open(file: StrOrBytesPath, flag: _TFlags = "r", mode: int = 0o666) -> _Database: ... + +else: + def whichdb(filename: str) -> str | None: ... + def open(file: str, flag: _TFlags = "r", mode: int = 0o666) -> _Database: ... diff --git a/mypy/typeshed/stdlib/dbm/dumb.pyi b/mypy/typeshed/stdlib/dbm/dumb.pyi new file mode 100644 index 000000000000..1c0b7756f292 --- /dev/null +++ b/mypy/typeshed/stdlib/dbm/dumb.pyi @@ -0,0 +1,37 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Iterator, MutableMapping +from types import TracebackType +from typing_extensions import Self, TypeAlias + +__all__ = ["error", "open"] + +_KeyType: TypeAlias = str | bytes +_ValueType: TypeAlias = str | bytes + +error = OSError + +# This class doesn't exist at runtime. open() can return an instance of +# any of the three implementations of dbm (dumb, gnu, ndbm), and this +# class is intended to represent the common interface supported by all three. +class _Database(MutableMapping[_KeyType, bytes]): + def __init__(self, filebasename: str, mode: str, flag: str = "c") -> None: ... + def sync(self) -> None: ... + def iterkeys(self) -> Iterator[bytes]: ... # undocumented + def close(self) -> None: ... + def __getitem__(self, key: _KeyType) -> bytes: ... + def __setitem__(self, key: _KeyType, val: _ValueType) -> None: ... + def __delitem__(self, key: _KeyType) -> None: ... + def __iter__(self) -> Iterator[bytes]: ... + def __len__(self) -> int: ... + def __del__(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + +if sys.version_info >= (3, 11): + def open(file: StrOrBytesPath, flag: str = "c", mode: int = 0o666) -> _Database: ... + +else: + def open(file: str, flag: str = "c", mode: int = 0o666) -> _Database: ... diff --git a/mypy/typeshed/stdlib/dbm/gnu.pyi b/mypy/typeshed/stdlib/dbm/gnu.pyi new file mode 100644 index 000000000000..2dac3d12b0ca --- /dev/null +++ b/mypy/typeshed/stdlib/dbm/gnu.pyi @@ -0,0 +1 @@ +from _gdbm import * diff --git a/mypy/typeshed/stdlib/dbm/ndbm.pyi b/mypy/typeshed/stdlib/dbm/ndbm.pyi new file mode 100644 index 000000000000..66c943ab640b --- /dev/null +++ b/mypy/typeshed/stdlib/dbm/ndbm.pyi @@ -0,0 +1 @@ +from _dbm import * diff --git a/mypy/typeshed/stdlib/dbm/sqlite3.pyi b/mypy/typeshed/stdlib/dbm/sqlite3.pyi new file mode 100644 index 000000000000..446a0cf155fa --- /dev/null +++ b/mypy/typeshed/stdlib/dbm/sqlite3.pyi @@ -0,0 +1,29 @@ +from _typeshed import ReadableBuffer, StrOrBytesPath, Unused +from collections.abc import Generator, MutableMapping +from typing import Final, Literal +from typing_extensions import LiteralString, Self, TypeAlias + +BUILD_TABLE: Final[LiteralString] +GET_SIZE: Final[LiteralString] +LOOKUP_KEY: Final[LiteralString] +STORE_KV: Final[LiteralString] +DELETE_KEY: Final[LiteralString] +ITER_KEYS: Final[LiteralString] + +_SqliteData: TypeAlias = str | ReadableBuffer | int | float + +class error(OSError): ... + +class _Database(MutableMapping[bytes, bytes]): + def __init__(self, path: StrOrBytesPath, /, *, flag: Literal["r", "w", "c", "n"], mode: int) -> None: ... + def __len__(self) -> int: ... + def __getitem__(self, key: _SqliteData) -> bytes: ... + def __setitem__(self, key: _SqliteData, value: _SqliteData) -> None: ... + def __delitem__(self, key: _SqliteData) -> None: ... + def __iter__(self) -> Generator[bytes]: ... + def close(self) -> None: ... + def keys(self) -> list[bytes]: ... # type: ignore[override] + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + +def open(filename: StrOrBytesPath, /, flag: Literal["r", "w,", "c", "n"] = "r", mode: int = 0o666) -> _Database: ... diff --git a/mypy/typeshed/stdlib/decimal.pyi b/mypy/typeshed/stdlib/decimal.pyi new file mode 100644 index 000000000000..7f8708a020fd --- /dev/null +++ b/mypy/typeshed/stdlib/decimal.pyi @@ -0,0 +1,265 @@ +import numbers +from _decimal import ( + HAVE_CONTEXTVAR as HAVE_CONTEXTVAR, + HAVE_THREADS as HAVE_THREADS, + MAX_EMAX as MAX_EMAX, + MAX_PREC as MAX_PREC, + MIN_EMIN as MIN_EMIN, + MIN_ETINY as MIN_ETINY, + ROUND_05UP as ROUND_05UP, + ROUND_CEILING as ROUND_CEILING, + ROUND_DOWN as ROUND_DOWN, + ROUND_FLOOR as ROUND_FLOOR, + ROUND_HALF_DOWN as ROUND_HALF_DOWN, + ROUND_HALF_EVEN as ROUND_HALF_EVEN, + ROUND_HALF_UP as ROUND_HALF_UP, + ROUND_UP as ROUND_UP, + BasicContext as BasicContext, + DefaultContext as DefaultContext, + ExtendedContext as ExtendedContext, + __libmpdec_version__ as __libmpdec_version__, + __version__ as __version__, + getcontext as getcontext, + localcontext as localcontext, + setcontext as setcontext, +) +from collections.abc import Container, Sequence +from types import TracebackType +from typing import Any, ClassVar, Literal, NamedTuple, final, overload, type_check_only +from typing_extensions import Self, TypeAlias + +_Decimal: TypeAlias = Decimal | int +_DecimalNew: TypeAlias = Decimal | float | str | tuple[int, Sequence[int], int] +_ComparableNum: TypeAlias = Decimal | float | numbers.Rational +_TrapType: TypeAlias = type[DecimalException] + +# At runtime, these classes are implemented in C as part of "_decimal". +# However, they consider themselves to live in "decimal", so we'll put them here. + +# This type isn't exposed at runtime. It calls itself decimal.ContextManager +@final +@type_check_only +class _ContextManager: + def __init__(self, new_context: Context) -> None: ... + def __enter__(self) -> Context: ... + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... + +class DecimalTuple(NamedTuple): + sign: int + digits: tuple[int, ...] + exponent: int | Literal["n", "N", "F"] + +class DecimalException(ArithmeticError): ... +class Clamped(DecimalException): ... +class InvalidOperation(DecimalException): ... +class ConversionSyntax(InvalidOperation): ... +class DivisionByZero(DecimalException, ZeroDivisionError): ... +class DivisionImpossible(InvalidOperation): ... +class DivisionUndefined(InvalidOperation, ZeroDivisionError): ... +class Inexact(DecimalException): ... +class InvalidContext(InvalidOperation): ... +class Rounded(DecimalException): ... +class Subnormal(DecimalException): ... +class Overflow(Inexact, Rounded): ... +class Underflow(Inexact, Rounded, Subnormal): ... +class FloatOperation(DecimalException, TypeError): ... + +class Decimal: + def __new__(cls, value: _DecimalNew = ..., context: Context | None = ...) -> Self: ... + @classmethod + def from_float(cls, f: float, /) -> Self: ... + def __bool__(self) -> bool: ... + def compare(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def __hash__(self) -> int: ... + def as_tuple(self) -> DecimalTuple: ... + def as_integer_ratio(self) -> tuple[int, int]: ... + def to_eng_string(self, context: Context | None = None) -> str: ... + def __abs__(self) -> Decimal: ... + def __add__(self, value: _Decimal, /) -> Decimal: ... + def __divmod__(self, value: _Decimal, /) -> tuple[Decimal, Decimal]: ... + def __eq__(self, value: object, /) -> bool: ... + def __floordiv__(self, value: _Decimal, /) -> Decimal: ... + def __ge__(self, value: _ComparableNum, /) -> bool: ... + def __gt__(self, value: _ComparableNum, /) -> bool: ... + def __le__(self, value: _ComparableNum, /) -> bool: ... + def __lt__(self, value: _ComparableNum, /) -> bool: ... + def __mod__(self, value: _Decimal, /) -> Decimal: ... + def __mul__(self, value: _Decimal, /) -> Decimal: ... + def __neg__(self) -> Decimal: ... + def __pos__(self) -> Decimal: ... + def __pow__(self, value: _Decimal, mod: _Decimal | None = None, /) -> Decimal: ... + def __radd__(self, value: _Decimal, /) -> Decimal: ... + def __rdivmod__(self, value: _Decimal, /) -> tuple[Decimal, Decimal]: ... + def __rfloordiv__(self, value: _Decimal, /) -> Decimal: ... + def __rmod__(self, value: _Decimal, /) -> Decimal: ... + def __rmul__(self, value: _Decimal, /) -> Decimal: ... + def __rsub__(self, value: _Decimal, /) -> Decimal: ... + def __rtruediv__(self, value: _Decimal, /) -> Decimal: ... + def __sub__(self, value: _Decimal, /) -> Decimal: ... + def __truediv__(self, value: _Decimal, /) -> Decimal: ... + def remainder_near(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def __float__(self) -> float: ... + def __int__(self) -> int: ... + def __trunc__(self) -> int: ... + @property + def real(self) -> Decimal: ... + @property + def imag(self) -> Decimal: ... + def conjugate(self) -> Decimal: ... + def __complex__(self) -> complex: ... + @overload + def __round__(self) -> int: ... + @overload + def __round__(self, ndigits: int, /) -> Decimal: ... + def __floor__(self) -> int: ... + def __ceil__(self) -> int: ... + def fma(self, other: _Decimal, third: _Decimal, context: Context | None = None) -> Decimal: ... + def __rpow__(self, value: _Decimal, mod: Context | None = None, /) -> Decimal: ... + def normalize(self, context: Context | None = None) -> Decimal: ... + def quantize(self, exp: _Decimal, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def same_quantum(self, other: _Decimal, context: Context | None = None) -> bool: ... + def to_integral_exact(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def to_integral_value(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def to_integral(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def sqrt(self, context: Context | None = None) -> Decimal: ... + def max(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def min(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def adjusted(self) -> int: ... + def canonical(self) -> Decimal: ... + def compare_signal(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def compare_total(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def compare_total_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def copy_abs(self) -> Decimal: ... + def copy_negate(self) -> Decimal: ... + def copy_sign(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def exp(self, context: Context | None = None) -> Decimal: ... + def is_canonical(self) -> bool: ... + def is_finite(self) -> bool: ... + def is_infinite(self) -> bool: ... + def is_nan(self) -> bool: ... + def is_normal(self, context: Context | None = None) -> bool: ... + def is_qnan(self) -> bool: ... + def is_signed(self) -> bool: ... + def is_snan(self) -> bool: ... + def is_subnormal(self, context: Context | None = None) -> bool: ... + def is_zero(self) -> bool: ... + def ln(self, context: Context | None = None) -> Decimal: ... + def log10(self, context: Context | None = None) -> Decimal: ... + def logb(self, context: Context | None = None) -> Decimal: ... + def logical_and(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def logical_invert(self, context: Context | None = None) -> Decimal: ... + def logical_or(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def logical_xor(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def max_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def min_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def next_minus(self, context: Context | None = None) -> Decimal: ... + def next_plus(self, context: Context | None = None) -> Decimal: ... + def next_toward(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def number_class(self, context: Context | None = None) -> str: ... + def radix(self) -> Decimal: ... + def rotate(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def scaleb(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def shift(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def __reduce__(self) -> tuple[type[Self], tuple[str]]: ... + def __copy__(self) -> Self: ... + def __deepcopy__(self, memo: Any, /) -> Self: ... + def __format__(self, specifier: str, context: Context | None = ..., /) -> str: ... + +class Context: + # TODO: Context doesn't allow you to delete *any* attributes from instances of the class at runtime, + # even settable attributes like `prec` and `rounding`, + # but that's inexpressable in the stub. + # Type checkers either ignore it or misinterpret it + # if you add a `def __delattr__(self, name: str, /) -> NoReturn` method to the stub + prec: int + rounding: str + Emin: int + Emax: int + capitals: int + clamp: int + traps: dict[_TrapType, bool] + flags: dict[_TrapType, bool] + def __init__( + self, + prec: int | None = ..., + rounding: str | None = ..., + Emin: int | None = ..., + Emax: int | None = ..., + capitals: int | None = ..., + clamp: int | None = ..., + flags: None | dict[_TrapType, bool] | Container[_TrapType] = ..., + traps: None | dict[_TrapType, bool] | Container[_TrapType] = ..., + _ignored_flags: list[_TrapType] | None = ..., + ) -> None: ... + def __reduce__(self) -> tuple[type[Self], tuple[Any, ...]]: ... + def clear_flags(self) -> None: ... + def clear_traps(self) -> None: ... + def copy(self) -> Context: ... + def __copy__(self) -> Context: ... + # see https://github.com/python/cpython/issues/94107 + __hash__: ClassVar[None] # type: ignore[assignment] + def Etiny(self) -> int: ... + def Etop(self) -> int: ... + def create_decimal(self, num: _DecimalNew = "0", /) -> Decimal: ... + def create_decimal_from_float(self, f: float, /) -> Decimal: ... + def abs(self, x: _Decimal, /) -> Decimal: ... + def add(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def canonical(self, x: Decimal, /) -> Decimal: ... + def compare(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def compare_signal(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def compare_total(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def compare_total_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def copy_abs(self, x: _Decimal, /) -> Decimal: ... + def copy_decimal(self, x: _Decimal, /) -> Decimal: ... + def copy_negate(self, x: _Decimal, /) -> Decimal: ... + def copy_sign(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def divide(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def divide_int(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def divmod(self, x: _Decimal, y: _Decimal, /) -> tuple[Decimal, Decimal]: ... + def exp(self, x: _Decimal, /) -> Decimal: ... + def fma(self, x: _Decimal, y: _Decimal, z: _Decimal, /) -> Decimal: ... + def is_canonical(self, x: _Decimal, /) -> bool: ... + def is_finite(self, x: _Decimal, /) -> bool: ... + def is_infinite(self, x: _Decimal, /) -> bool: ... + def is_nan(self, x: _Decimal, /) -> bool: ... + def is_normal(self, x: _Decimal, /) -> bool: ... + def is_qnan(self, x: _Decimal, /) -> bool: ... + def is_signed(self, x: _Decimal, /) -> bool: ... + def is_snan(self, x: _Decimal, /) -> bool: ... + def is_subnormal(self, x: _Decimal, /) -> bool: ... + def is_zero(self, x: _Decimal, /) -> bool: ... + def ln(self, x: _Decimal, /) -> Decimal: ... + def log10(self, x: _Decimal, /) -> Decimal: ... + def logb(self, x: _Decimal, /) -> Decimal: ... + def logical_and(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def logical_invert(self, x: _Decimal, /) -> Decimal: ... + def logical_or(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def logical_xor(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def max(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def max_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def min(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def min_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def minus(self, x: _Decimal, /) -> Decimal: ... + def multiply(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def next_minus(self, x: _Decimal, /) -> Decimal: ... + def next_plus(self, x: _Decimal, /) -> Decimal: ... + def next_toward(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def normalize(self, x: _Decimal, /) -> Decimal: ... + def number_class(self, x: _Decimal, /) -> str: ... + def plus(self, x: _Decimal, /) -> Decimal: ... + def power(self, a: _Decimal, b: _Decimal, modulo: _Decimal | None = None) -> Decimal: ... + def quantize(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def radix(self) -> Decimal: ... + def remainder(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def remainder_near(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def rotate(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def same_quantum(self, x: _Decimal, y: _Decimal, /) -> bool: ... + def scaleb(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def shift(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def sqrt(self, x: _Decimal, /) -> Decimal: ... + def subtract(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def to_eng_string(self, x: _Decimal, /) -> str: ... + def to_sci_string(self, x: _Decimal, /) -> str: ... + def to_integral_exact(self, x: _Decimal, /) -> Decimal: ... + def to_integral_value(self, x: _Decimal, /) -> Decimal: ... + def to_integral(self, x: _Decimal, /) -> Decimal: ... diff --git a/mypy/typeshed/stdlib/difflib.pyi b/mypy/typeshed/stdlib/difflib.pyi new file mode 100644 index 000000000000..50154d785c2f --- /dev/null +++ b/mypy/typeshed/stdlib/difflib.pyi @@ -0,0 +1,140 @@ +import sys +from collections.abc import Callable, Iterable, Iterator, Sequence +from typing import Any, AnyStr, Generic, Literal, NamedTuple, TypeVar, overload + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "get_close_matches", + "ndiff", + "restore", + "SequenceMatcher", + "Differ", + "IS_CHARACTER_JUNK", + "IS_LINE_JUNK", + "context_diff", + "unified_diff", + "diff_bytes", + "HtmlDiff", + "Match", +] + +_T = TypeVar("_T") + +class Match(NamedTuple): + a: int + b: int + size: int + +class SequenceMatcher(Generic[_T]): + @overload + def __init__(self, isjunk: Callable[[_T], bool] | None, a: Sequence[_T], b: Sequence[_T], autojunk: bool = True) -> None: ... + @overload + def __init__(self, *, a: Sequence[_T], b: Sequence[_T], autojunk: bool = True) -> None: ... + @overload + def __init__( + self: SequenceMatcher[str], + isjunk: Callable[[str], bool] | None = None, + a: Sequence[str] = "", + b: Sequence[str] = "", + autojunk: bool = True, + ) -> None: ... + def set_seqs(self, a: Sequence[_T], b: Sequence[_T]) -> None: ... + def set_seq1(self, a: Sequence[_T]) -> None: ... + def set_seq2(self, b: Sequence[_T]) -> None: ... + if sys.version_info >= (3, 9): + def find_longest_match(self, alo: int = 0, ahi: int | None = None, blo: int = 0, bhi: int | None = None) -> Match: ... + else: + def find_longest_match(self, alo: int, ahi: int, blo: int, bhi: int) -> Match: ... + + def get_matching_blocks(self) -> list[Match]: ... + def get_opcodes(self) -> list[tuple[Literal["replace", "delete", "insert", "equal"], int, int, int, int]]: ... + def get_grouped_opcodes(self, n: int = 3) -> Iterable[list[tuple[str, int, int, int, int]]]: ... + def ratio(self) -> float: ... + def quick_ratio(self) -> float: ... + def real_quick_ratio(self) -> float: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +@overload +def get_close_matches(word: AnyStr, possibilities: Iterable[AnyStr], n: int = 3, cutoff: float = 0.6) -> list[AnyStr]: ... +@overload +def get_close_matches( + word: Sequence[_T], possibilities: Iterable[Sequence[_T]], n: int = 3, cutoff: float = 0.6 +) -> list[Sequence[_T]]: ... + +class Differ: + def __init__(self, linejunk: Callable[[str], bool] | None = None, charjunk: Callable[[str], bool] | None = None) -> None: ... + def compare(self, a: Sequence[str], b: Sequence[str]) -> Iterator[str]: ... + +def IS_LINE_JUNK(line: str, pat: Any = ...) -> bool: ... # pat is undocumented +def IS_CHARACTER_JUNK(ch: str, ws: str = " \t") -> bool: ... # ws is undocumented +def unified_diff( + a: Sequence[str], + b: Sequence[str], + fromfile: str = "", + tofile: str = "", + fromfiledate: str = "", + tofiledate: str = "", + n: int = 3, + lineterm: str = "\n", +) -> Iterator[str]: ... +def context_diff( + a: Sequence[str], + b: Sequence[str], + fromfile: str = "", + tofile: str = "", + fromfiledate: str = "", + tofiledate: str = "", + n: int = 3, + lineterm: str = "\n", +) -> Iterator[str]: ... +def ndiff( + a: Sequence[str], + b: Sequence[str], + linejunk: Callable[[str], bool] | None = None, + charjunk: Callable[[str], bool] | None = ..., +) -> Iterator[str]: ... + +class HtmlDiff: + def __init__( + self, + tabsize: int = 8, + wrapcolumn: int | None = None, + linejunk: Callable[[str], bool] | None = None, + charjunk: Callable[[str], bool] | None = ..., + ) -> None: ... + def make_file( + self, + fromlines: Sequence[str], + tolines: Sequence[str], + fromdesc: str = "", + todesc: str = "", + context: bool = False, + numlines: int = 5, + *, + charset: str = "utf-8", + ) -> str: ... + def make_table( + self, + fromlines: Sequence[str], + tolines: Sequence[str], + fromdesc: str = "", + todesc: str = "", + context: bool = False, + numlines: int = 5, + ) -> str: ... + +def restore(delta: Iterable[str], which: int) -> Iterator[str]: ... +def diff_bytes( + dfunc: Callable[[Sequence[str], Sequence[str], str, str, str, str, int, str], Iterator[str]], + a: Iterable[bytes | bytearray], + b: Iterable[bytes | bytearray], + fromfile: bytes | bytearray = b"", + tofile: bytes | bytearray = b"", + fromfiledate: bytes | bytearray = b"", + tofiledate: bytes | bytearray = b"", + n: int = 3, + lineterm: bytes | bytearray = b"\n", +) -> Iterator[bytes]: ... diff --git a/mypy/typeshed/stdlib/dis.pyi b/mypy/typeshed/stdlib/dis.pyi new file mode 100644 index 000000000000..cb69eac89c92 --- /dev/null +++ b/mypy/typeshed/stdlib/dis.pyi @@ -0,0 +1,222 @@ +import sys +import types +from collections.abc import Callable, Iterator +from opcode import * # `dis` re-exports it as a part of public API +from typing import IO, Any, NamedTuple +from typing_extensions import Self, TypeAlias + +__all__ = [ + "code_info", + "dis", + "disassemble", + "distb", + "disco", + "findlinestarts", + "findlabels", + "show_code", + "get_instructions", + "Instruction", + "Bytecode", + "cmp_op", + "hasconst", + "hasname", + "hasjrel", + "hasjabs", + "haslocal", + "hascompare", + "hasfree", + "opname", + "opmap", + "HAVE_ARGUMENT", + "EXTENDED_ARG", + "stack_effect", +] +if sys.version_info >= (3, 13): + __all__ += ["hasjump"] + +if sys.version_info >= (3, 12): + __all__ += ["hasarg", "hasexc"] +else: + __all__ += ["hasnargs"] + +# Strictly this should not have to include Callable, but mypy doesn't use FunctionType +# for functions (python/mypy#3171) +_HaveCodeType: TypeAlias = types.MethodType | types.FunctionType | types.CodeType | type | Callable[..., Any] + +if sys.version_info >= (3, 11): + class Positions(NamedTuple): + lineno: int | None = None + end_lineno: int | None = None + col_offset: int | None = None + end_col_offset: int | None = None + +if sys.version_info >= (3, 13): + class _Instruction(NamedTuple): + opname: str + opcode: int + arg: int | None + argval: Any + argrepr: str + offset: int + start_offset: int + starts_line: bool + line_number: int | None + label: int | None = None + positions: Positions | None = None + cache_info: list[tuple[str, int, Any]] | None = None + +elif sys.version_info >= (3, 11): + class _Instruction(NamedTuple): + opname: str + opcode: int + arg: int | None + argval: Any + argrepr: str + offset: int + starts_line: int | None + is_jump_target: bool + positions: Positions | None = None + +else: + class _Instruction(NamedTuple): + opname: str + opcode: int + arg: int | None + argval: Any + argrepr: str + offset: int + starts_line: int | None + is_jump_target: bool + +class Instruction(_Instruction): + if sys.version_info < (3, 13): + def _disassemble(self, lineno_width: int = 3, mark_as_current: bool = False, offset_width: int = 4) -> str: ... + if sys.version_info >= (3, 13): + @property + def oparg(self) -> int: ... + @property + def baseopcode(self) -> int: ... + @property + def baseopname(self) -> str: ... + @property + def cache_offset(self) -> int: ... + @property + def end_offset(self) -> int: ... + @property + def jump_target(self) -> int: ... + @property + def is_jump_target(self) -> bool: ... + +class Bytecode: + codeobj: types.CodeType + first_line: int + if sys.version_info >= (3, 13): + show_offsets: bool + # 3.13 added `show_offsets` + def __init__( + self, + x: _HaveCodeType | str, + *, + first_line: int | None = None, + current_offset: int | None = None, + show_caches: bool = False, + adaptive: bool = False, + show_offsets: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __init__( + self, + x: _HaveCodeType | str, + *, + first_line: int | None = None, + current_offset: int | None = None, + show_caches: bool = False, + adaptive: bool = False, + ) -> None: ... + else: + def __init__( + self, x: _HaveCodeType | str, *, first_line: int | None = None, current_offset: int | None = None + ) -> None: ... + + if sys.version_info >= (3, 11): + @classmethod + def from_traceback(cls, tb: types.TracebackType, *, show_caches: bool = False, adaptive: bool = False) -> Self: ... + else: + @classmethod + def from_traceback(cls, tb: types.TracebackType) -> Self: ... + + def __iter__(self) -> Iterator[Instruction]: ... + def info(self) -> str: ... + def dis(self) -> str: ... + +COMPILER_FLAG_NAMES: dict[int, str] + +def findlabels(code: _HaveCodeType) -> list[int]: ... +def findlinestarts(code: _HaveCodeType) -> Iterator[tuple[int, int]]: ... +def pretty_flags(flags: int) -> str: ... +def code_info(x: _HaveCodeType | str) -> str: ... + +if sys.version_info >= (3, 13): + # 3.13 added `show_offsets` + def dis( + x: _HaveCodeType | str | bytes | bytearray | None = None, + *, + file: IO[str] | None = None, + depth: int | None = None, + show_caches: bool = False, + adaptive: bool = False, + show_offsets: bool = False, + ) -> None: ... + def disassemble( + co: _HaveCodeType, + lasti: int = -1, + *, + file: IO[str] | None = None, + show_caches: bool = False, + adaptive: bool = False, + show_offsets: bool = False, + ) -> None: ... + def distb( + tb: types.TracebackType | None = None, + *, + file: IO[str] | None = None, + show_caches: bool = False, + adaptive: bool = False, + show_offsets: bool = False, + ) -> None: ... + # 3.13 made `show_cache` `None` by default + def get_instructions( + x: _HaveCodeType, *, first_line: int | None = None, show_caches: bool | None = None, adaptive: bool = False + ) -> Iterator[Instruction]: ... + +elif sys.version_info >= (3, 11): + # 3.11 added `show_caches` and `adaptive` + def dis( + x: _HaveCodeType | str | bytes | bytearray | None = None, + *, + file: IO[str] | None = None, + depth: int | None = None, + show_caches: bool = False, + adaptive: bool = False, + ) -> None: ... + def disassemble( + co: _HaveCodeType, lasti: int = -1, *, file: IO[str] | None = None, show_caches: bool = False, adaptive: bool = False + ) -> None: ... + def distb( + tb: types.TracebackType | None = None, *, file: IO[str] | None = None, show_caches: bool = False, adaptive: bool = False + ) -> None: ... + def get_instructions( + x: _HaveCodeType, *, first_line: int | None = None, show_caches: bool = False, adaptive: bool = False + ) -> Iterator[Instruction]: ... + +else: + def dis( + x: _HaveCodeType | str | bytes | bytearray | None = None, *, file: IO[str] | None = None, depth: int | None = None + ) -> None: ... + def disassemble(co: _HaveCodeType, lasti: int = -1, *, file: IO[str] | None = None) -> None: ... + def distb(tb: types.TracebackType | None = None, *, file: IO[str] | None = None) -> None: ... + def get_instructions(x: _HaveCodeType, *, first_line: int | None = None) -> Iterator[Instruction]: ... + +def show_code(co: _HaveCodeType, *, file: IO[str] | None = None) -> None: ... + +disco = disassemble diff --git a/mypy/typeshed/stdlib/distutils/__init__.pyi b/mypy/typeshed/stdlib/distutils/__init__.pyi new file mode 100644 index 000000000000..328a5b783441 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/__init__.pyi @@ -0,0 +1,5 @@ +# Attempts to improve these stubs are probably not the best use of time: +# - distutils is deleted in Python 3.12 and newer +# - Most users already do not use stdlib distutils, due to setuptools monkeypatching +# - We have very little quality assurance on these stubs, since due to the two above issues +# we allowlist all distutils errors in stubtest. diff --git a/mypy/typeshed/stdlib/distutils/_msvccompiler.pyi b/mypy/typeshed/stdlib/distutils/_msvccompiler.pyi new file mode 100644 index 000000000000..bba9373b72db --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/_msvccompiler.pyi @@ -0,0 +1,13 @@ +from _typeshed import Incomplete +from distutils.ccompiler import CCompiler +from typing import ClassVar, Final + +PLAT_SPEC_TO_RUNTIME: Final[dict[str, str]] +PLAT_TO_VCVARS: Final[dict[str, str]] + +class MSVCCompiler(CCompiler): + compiler_type: ClassVar[str] + executables: ClassVar[dict[Incomplete, Incomplete]] + res_extension: ClassVar[str] + initialized: bool + def initialize(self, plat_name: str | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/archive_util.pyi b/mypy/typeshed/stdlib/distutils/archive_util.pyi new file mode 100644 index 000000000000..16684ff06956 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/archive_util.pyi @@ -0,0 +1,35 @@ +from _typeshed import StrOrBytesPath, StrPath +from typing import Literal, overload + +@overload +def make_archive( + base_name: str, + format: str, + root_dir: StrOrBytesPath | None = None, + base_dir: str | None = None, + verbose: bool | Literal[0, 1] = 0, + dry_run: bool | Literal[0, 1] = 0, + owner: str | None = None, + group: str | None = None, +) -> str: ... +@overload +def make_archive( + base_name: StrPath, + format: str, + root_dir: StrOrBytesPath, + base_dir: str | None = None, + verbose: bool | Literal[0, 1] = 0, + dry_run: bool | Literal[0, 1] = 0, + owner: str | None = None, + group: str | None = None, +) -> str: ... +def make_tarball( + base_name: str, + base_dir: StrPath, + compress: str | None = "gzip", + verbose: bool | Literal[0, 1] = 0, + dry_run: bool | Literal[0, 1] = 0, + owner: str | None = None, + group: str | None = None, +) -> str: ... +def make_zipfile(base_name: str, base_dir: str, verbose: bool | Literal[0, 1] = 0, dry_run: bool | Literal[0, 1] = 0) -> str: ... diff --git a/mypy/typeshed/stdlib/distutils/bcppcompiler.pyi b/mypy/typeshed/stdlib/distutils/bcppcompiler.pyi new file mode 100644 index 000000000000..3e432f94b525 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/bcppcompiler.pyi @@ -0,0 +1,3 @@ +from distutils.ccompiler import CCompiler + +class BCPPCompiler(CCompiler): ... diff --git a/mypy/typeshed/stdlib/distutils/ccompiler.pyi b/mypy/typeshed/stdlib/distutils/ccompiler.pyi new file mode 100644 index 000000000000..5bff209807ee --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/ccompiler.pyi @@ -0,0 +1,176 @@ +from _typeshed import BytesPath, StrPath, Unused +from collections.abc import Callable, Iterable, Sequence +from distutils.file_util import _BytesPathT, _StrPathT +from typing import Literal, overload +from typing_extensions import TypeAlias, TypeVarTuple, Unpack + +_Macro: TypeAlias = tuple[str] | tuple[str, str | None] +_Ts = TypeVarTuple("_Ts") + +def gen_lib_options( + compiler: CCompiler, library_dirs: list[str], runtime_library_dirs: list[str], libraries: list[str] +) -> list[str]: ... +def gen_preprocess_options(macros: list[_Macro], include_dirs: list[str]) -> list[str]: ... +def get_default_compiler(osname: str | None = None, platform: str | None = None) -> str: ... +def new_compiler( + plat: str | None = None, + compiler: str | None = None, + verbose: bool | Literal[0, 1] = 0, + dry_run: bool | Literal[0, 1] = 0, + force: bool | Literal[0, 1] = 0, +) -> CCompiler: ... +def show_compilers() -> None: ... + +class CCompiler: + dry_run: bool + force: bool + verbose: bool + output_dir: str | None + macros: list[_Macro] + include_dirs: list[str] + libraries: list[str] + library_dirs: list[str] + runtime_library_dirs: list[str] + objects: list[str] + def __init__( + self, verbose: bool | Literal[0, 1] = 0, dry_run: bool | Literal[0, 1] = 0, force: bool | Literal[0, 1] = 0 + ) -> None: ... + def add_include_dir(self, dir: str) -> None: ... + def set_include_dirs(self, dirs: list[str]) -> None: ... + def add_library(self, libname: str) -> None: ... + def set_libraries(self, libnames: list[str]) -> None: ... + def add_library_dir(self, dir: str) -> None: ... + def set_library_dirs(self, dirs: list[str]) -> None: ... + def add_runtime_library_dir(self, dir: str) -> None: ... + def set_runtime_library_dirs(self, dirs: list[str]) -> None: ... + def define_macro(self, name: str, value: str | None = None) -> None: ... + def undefine_macro(self, name: str) -> None: ... + def add_link_object(self, object: str) -> None: ... + def set_link_objects(self, objects: list[str]) -> None: ... + def detect_language(self, sources: str | list[str]) -> str | None: ... + def find_library_file(self, dirs: list[str], lib: str, debug: bool | Literal[0, 1] = 0) -> str | None: ... + def has_function( + self, + funcname: str, + includes: list[str] | None = None, + include_dirs: list[str] | None = None, + libraries: list[str] | None = None, + library_dirs: list[str] | None = None, + ) -> bool: ... + def library_dir_option(self, dir: str) -> str: ... + def library_option(self, lib: str) -> str: ... + def runtime_library_dir_option(self, dir: str) -> str: ... + def set_executables(self, **args: str) -> None: ... + def compile( + self, + sources: Sequence[StrPath], + output_dir: str | None = None, + macros: list[_Macro] | None = None, + include_dirs: list[str] | None = None, + debug: bool | Literal[0, 1] = 0, + extra_preargs: list[str] | None = None, + extra_postargs: list[str] | None = None, + depends: list[str] | None = None, + ) -> list[str]: ... + def create_static_lib( + self, + objects: list[str], + output_libname: str, + output_dir: str | None = None, + debug: bool | Literal[0, 1] = 0, + target_lang: str | None = None, + ) -> None: ... + def link( + self, + target_desc: str, + objects: list[str], + output_filename: str, + output_dir: str | None = None, + libraries: list[str] | None = None, + library_dirs: list[str] | None = None, + runtime_library_dirs: list[str] | None = None, + export_symbols: list[str] | None = None, + debug: bool | Literal[0, 1] = 0, + extra_preargs: list[str] | None = None, + extra_postargs: list[str] | None = None, + build_temp: str | None = None, + target_lang: str | None = None, + ) -> None: ... + def link_executable( + self, + objects: list[str], + output_progname: str, + output_dir: str | None = None, + libraries: list[str] | None = None, + library_dirs: list[str] | None = None, + runtime_library_dirs: list[str] | None = None, + debug: bool | Literal[0, 1] = 0, + extra_preargs: list[str] | None = None, + extra_postargs: list[str] | None = None, + target_lang: str | None = None, + ) -> None: ... + def link_shared_lib( + self, + objects: list[str], + output_libname: str, + output_dir: str | None = None, + libraries: list[str] | None = None, + library_dirs: list[str] | None = None, + runtime_library_dirs: list[str] | None = None, + export_symbols: list[str] | None = None, + debug: bool | Literal[0, 1] = 0, + extra_preargs: list[str] | None = None, + extra_postargs: list[str] | None = None, + build_temp: str | None = None, + target_lang: str | None = None, + ) -> None: ... + def link_shared_object( + self, + objects: list[str], + output_filename: str, + output_dir: str | None = None, + libraries: list[str] | None = None, + library_dirs: list[str] | None = None, + runtime_library_dirs: list[str] | None = None, + export_symbols: list[str] | None = None, + debug: bool | Literal[0, 1] = 0, + extra_preargs: list[str] | None = None, + extra_postargs: list[str] | None = None, + build_temp: str | None = None, + target_lang: str | None = None, + ) -> None: ... + def preprocess( + self, + source: str, + output_file: str | None = None, + macros: list[_Macro] | None = None, + include_dirs: list[str] | None = None, + extra_preargs: list[str] | None = None, + extra_postargs: list[str] | None = None, + ) -> None: ... + @overload + def executable_filename(self, basename: str, strip_dir: Literal[0, False] = 0, output_dir: StrPath = "") -> str: ... + @overload + def executable_filename(self, basename: StrPath, strip_dir: Literal[1, True], output_dir: StrPath = "") -> str: ... + def library_filename( + self, libname: str, lib_type: str = "static", strip_dir: bool | Literal[0, 1] = 0, output_dir: StrPath = "" + ) -> str: ... + def object_filenames( + self, source_filenames: Iterable[StrPath], strip_dir: bool | Literal[0, 1] = 0, output_dir: StrPath | None = "" + ) -> list[str]: ... + @overload + def shared_object_filename(self, basename: str, strip_dir: Literal[0, False] = 0, output_dir: StrPath = "") -> str: ... + @overload + def shared_object_filename(self, basename: StrPath, strip_dir: Literal[1, True], output_dir: StrPath = "") -> str: ... + def execute( + self, func: Callable[[Unpack[_Ts]], Unused], args: tuple[Unpack[_Ts]], msg: str | None = None, level: int = 1 + ) -> None: ... + def spawn(self, cmd: Iterable[str]) -> None: ... + def mkpath(self, name: str, mode: int = 0o777) -> None: ... + @overload + def move_file(self, src: StrPath, dst: _StrPathT) -> _StrPathT | str: ... + @overload + def move_file(self, src: BytesPath, dst: _BytesPathT) -> _BytesPathT | bytes: ... + def announce(self, msg: str, level: int = 1) -> None: ... + def warn(self, msg: str) -> None: ... + def debug_print(self, msg: str) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/cmd.pyi b/mypy/typeshed/stdlib/distutils/cmd.pyi new file mode 100644 index 000000000000..dcb423a49b09 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/cmd.pyi @@ -0,0 +1,229 @@ +from _typeshed import BytesPath, Incomplete, StrOrBytesPath, StrPath, Unused +from abc import abstractmethod +from collections.abc import Callable, Iterable +from distutils.command.bdist import bdist +from distutils.command.bdist_dumb import bdist_dumb +from distutils.command.bdist_rpm import bdist_rpm +from distutils.command.build import build +from distutils.command.build_clib import build_clib +from distutils.command.build_ext import build_ext +from distutils.command.build_py import build_py +from distutils.command.build_scripts import build_scripts +from distutils.command.check import check +from distutils.command.clean import clean +from distutils.command.config import config +from distutils.command.install import install +from distutils.command.install_data import install_data +from distutils.command.install_egg_info import install_egg_info +from distutils.command.install_headers import install_headers +from distutils.command.install_lib import install_lib +from distutils.command.install_scripts import install_scripts +from distutils.command.register import register +from distutils.command.sdist import sdist +from distutils.command.upload import upload +from distutils.dist import Distribution +from distutils.file_util import _BytesPathT, _StrPathT +from typing import Any, ClassVar, Literal, TypeVar, overload +from typing_extensions import TypeVarTuple, Unpack + +_CommandT = TypeVar("_CommandT", bound=Command) +_Ts = TypeVarTuple("_Ts") + +class Command: + dry_run: Literal[0, 1] # Exposed from __getattr_. Same as Distribution.dry_run + distribution: Distribution + # Any to work around variance issues + sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] + def __init__(self, dist: Distribution) -> None: ... + @abstractmethod + def initialize_options(self) -> None: ... + @abstractmethod + def finalize_options(self) -> None: ... + @abstractmethod + def run(self) -> None: ... + def announce(self, msg: str, level: int = 1) -> None: ... + def debug_print(self, msg: str) -> None: ... + def ensure_string(self, option: str, default: str | None = None) -> None: ... + def ensure_string_list(self, option: str) -> None: ... + def ensure_filename(self, option: str) -> None: ... + def ensure_dirname(self, option: str) -> None: ... + def get_command_name(self) -> str: ... + def set_undefined_options(self, src_cmd: str, *option_pairs: tuple[str, str]) -> None: ... + # NOTE: This list comes directly from the distutils/command folder. Minus bdist_msi and bdist_wininst. + @overload + def get_finalized_command(self, command: Literal["bdist"], create: bool | Literal[0, 1] = 1) -> bdist: ... + @overload + def get_finalized_command(self, command: Literal["bdist_dumb"], create: bool | Literal[0, 1] = 1) -> bdist_dumb: ... + @overload + def get_finalized_command(self, command: Literal["bdist_rpm"], create: bool | Literal[0, 1] = 1) -> bdist_rpm: ... + @overload + def get_finalized_command(self, command: Literal["build"], create: bool | Literal[0, 1] = 1) -> build: ... + @overload + def get_finalized_command(self, command: Literal["build_clib"], create: bool | Literal[0, 1] = 1) -> build_clib: ... + @overload + def get_finalized_command(self, command: Literal["build_ext"], create: bool | Literal[0, 1] = 1) -> build_ext: ... + @overload + def get_finalized_command(self, command: Literal["build_py"], create: bool | Literal[0, 1] = 1) -> build_py: ... + @overload + def get_finalized_command(self, command: Literal["build_scripts"], create: bool | Literal[0, 1] = 1) -> build_scripts: ... + @overload + def get_finalized_command(self, command: Literal["check"], create: bool | Literal[0, 1] = 1) -> check: ... + @overload + def get_finalized_command(self, command: Literal["clean"], create: bool | Literal[0, 1] = 1) -> clean: ... + @overload + def get_finalized_command(self, command: Literal["config"], create: bool | Literal[0, 1] = 1) -> config: ... + @overload + def get_finalized_command(self, command: Literal["install"], create: bool | Literal[0, 1] = 1) -> install: ... + @overload + def get_finalized_command(self, command: Literal["install_data"], create: bool | Literal[0, 1] = 1) -> install_data: ... + @overload + def get_finalized_command( + self, command: Literal["install_egg_info"], create: bool | Literal[0, 1] = 1 + ) -> install_egg_info: ... + @overload + def get_finalized_command(self, command: Literal["install_headers"], create: bool | Literal[0, 1] = 1) -> install_headers: ... + @overload + def get_finalized_command(self, command: Literal["install_lib"], create: bool | Literal[0, 1] = 1) -> install_lib: ... + @overload + def get_finalized_command(self, command: Literal["install_scripts"], create: bool | Literal[0, 1] = 1) -> install_scripts: ... + @overload + def get_finalized_command(self, command: Literal["register"], create: bool | Literal[0, 1] = 1) -> register: ... + @overload + def get_finalized_command(self, command: Literal["sdist"], create: bool | Literal[0, 1] = 1) -> sdist: ... + @overload + def get_finalized_command(self, command: Literal["upload"], create: bool | Literal[0, 1] = 1) -> upload: ... + @overload + def get_finalized_command(self, command: str, create: bool | Literal[0, 1] = 1) -> Command: ... + @overload + def reinitialize_command(self, command: Literal["bdist"], reinit_subcommands: bool | Literal[0, 1] = 0) -> bdist: ... + @overload + def reinitialize_command( + self, command: Literal["bdist_dumb"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> bdist_dumb: ... + @overload + def reinitialize_command(self, command: Literal["bdist_rpm"], reinit_subcommands: bool | Literal[0, 1] = 0) -> bdist_rpm: ... + @overload + def reinitialize_command(self, command: Literal["build"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build: ... + @overload + def reinitialize_command( + self, command: Literal["build_clib"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> build_clib: ... + @overload + def reinitialize_command(self, command: Literal["build_ext"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build_ext: ... + @overload + def reinitialize_command(self, command: Literal["build_py"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build_py: ... + @overload + def reinitialize_command( + self, command: Literal["build_scripts"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> build_scripts: ... + @overload + def reinitialize_command(self, command: Literal["check"], reinit_subcommands: bool | Literal[0, 1] = 0) -> check: ... + @overload + def reinitialize_command(self, command: Literal["clean"], reinit_subcommands: bool | Literal[0, 1] = 0) -> clean: ... + @overload + def reinitialize_command(self, command: Literal["config"], reinit_subcommands: bool | Literal[0, 1] = 0) -> config: ... + @overload + def reinitialize_command(self, command: Literal["install"], reinit_subcommands: bool | Literal[0, 1] = 0) -> install: ... + @overload + def reinitialize_command( + self, command: Literal["install_data"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_data: ... + @overload + def reinitialize_command( + self, command: Literal["install_egg_info"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_egg_info: ... + @overload + def reinitialize_command( + self, command: Literal["install_headers"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_headers: ... + @overload + def reinitialize_command( + self, command: Literal["install_lib"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_lib: ... + @overload + def reinitialize_command( + self, command: Literal["install_scripts"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_scripts: ... + @overload + def reinitialize_command(self, command: Literal["register"], reinit_subcommands: bool | Literal[0, 1] = 0) -> register: ... + @overload + def reinitialize_command(self, command: Literal["sdist"], reinit_subcommands: bool | Literal[0, 1] = 0) -> sdist: ... + @overload + def reinitialize_command(self, command: Literal["upload"], reinit_subcommands: bool | Literal[0, 1] = 0) -> upload: ... + @overload + def reinitialize_command(self, command: str, reinit_subcommands: bool | Literal[0, 1] = 0) -> Command: ... + @overload + def reinitialize_command(self, command: _CommandT, reinit_subcommands: bool | Literal[0, 1] = 0) -> _CommandT: ... + def run_command(self, command: str) -> None: ... + def get_sub_commands(self) -> list[str]: ... + def warn(self, msg: str) -> None: ... + def execute( + self, func: Callable[[Unpack[_Ts]], Unused], args: tuple[Unpack[_Ts]], msg: str | None = None, level: int = 1 + ) -> None: ... + def mkpath(self, name: str, mode: int = 0o777) -> None: ... + @overload + def copy_file( + self, + infile: StrPath, + outfile: _StrPathT, + preserve_mode: bool | Literal[0, 1] = 1, + preserve_times: bool | Literal[0, 1] = 1, + link: str | None = None, + level: Unused = 1, + ) -> tuple[_StrPathT | str, bool]: ... + @overload + def copy_file( + self, + infile: BytesPath, + outfile: _BytesPathT, + preserve_mode: bool | Literal[0, 1] = 1, + preserve_times: bool | Literal[0, 1] = 1, + link: str | None = None, + level: Unused = 1, + ) -> tuple[_BytesPathT | bytes, bool]: ... + def copy_tree( + self, + infile: StrPath, + outfile: str, + preserve_mode: bool | Literal[0, 1] = 1, + preserve_times: bool | Literal[0, 1] = 1, + preserve_symlinks: bool | Literal[0, 1] = 0, + level: Unused = 1, + ) -> list[str]: ... + @overload + def move_file(self, src: StrPath, dst: _StrPathT, level: Unused = 1) -> _StrPathT | str: ... + @overload + def move_file(self, src: BytesPath, dst: _BytesPathT, level: Unused = 1) -> _BytesPathT | bytes: ... + def spawn(self, cmd: Iterable[str], search_path: bool | Literal[0, 1] = 1, level: Unused = 1) -> None: ... + @overload + def make_archive( + self, + base_name: str, + format: str, + root_dir: StrOrBytesPath | None = None, + base_dir: str | None = None, + owner: str | None = None, + group: str | None = None, + ) -> str: ... + @overload + def make_archive( + self, + base_name: StrPath, + format: str, + root_dir: StrOrBytesPath, + base_dir: str | None = None, + owner: str | None = None, + group: str | None = None, + ) -> str: ... + def make_file( + self, + infiles: str | list[str] | tuple[str, ...], + outfile: StrOrBytesPath, + func: Callable[[Unpack[_Ts]], Unused], + args: tuple[Unpack[_Ts]], + exec_msg: str | None = None, + skip_msg: str | None = None, + level: Unused = 1, + ) -> None: ... + def ensure_finalized(self) -> None: ... + def dump_options(self, header: Incomplete | None = None, indent: str = "") -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/__init__.pyi b/mypy/typeshed/stdlib/distutils/command/__init__.pyi new file mode 100644 index 000000000000..4d7372858af3 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/__init__.pyi @@ -0,0 +1,48 @@ +import sys + +from . import ( + bdist, + bdist_dumb, + bdist_rpm, + build, + build_clib, + build_ext, + build_py, + build_scripts, + check, + clean, + install, + install_data, + install_headers, + install_lib, + install_scripts, + register, + sdist, + upload, +) + +__all__ = [ + "build", + "build_py", + "build_ext", + "build_clib", + "build_scripts", + "clean", + "install", + "install_lib", + "install_headers", + "install_scripts", + "install_data", + "sdist", + "register", + "bdist", + "bdist_dumb", + "bdist_rpm", + "check", + "upload", +] + +if sys.version_info < (3, 10): + from . import bdist_wininst + + __all__ += ["bdist_wininst"] diff --git a/mypy/typeshed/stdlib/distutils/command/bdist.pyi b/mypy/typeshed/stdlib/distutils/command/bdist.pyi new file mode 100644 index 000000000000..6f996207077e --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/bdist.pyi @@ -0,0 +1,27 @@ +from _typeshed import Incomplete, Unused +from collections.abc import Callable +from typing import ClassVar + +from ..cmd import Command + +def show_formats() -> None: ... + +class bdist(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] + no_format_option: ClassVar[tuple[str, ...]] + default_format: ClassVar[dict[str, str]] + format_commands: ClassVar[list[str]] + format_command: ClassVar[dict[str, tuple[str, str]]] + bdist_base: Incomplete + plat_name: Incomplete + formats: Incomplete + dist_dir: Incomplete + skip_build: int + group: Incomplete + owner: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi new file mode 100644 index 000000000000..297a0c39ed43 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi @@ -0,0 +1,22 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command + +class bdist_dumb(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + default_format: ClassVar[dict[str, str]] + bdist_dir: Incomplete + plat_name: Incomplete + format: Incomplete + keep_temp: int + dist_dir: Incomplete + skip_build: Incomplete + relative: int + owner: Incomplete + group: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi new file mode 100644 index 000000000000..baeee7d3eccb --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi @@ -0,0 +1,46 @@ +import sys +from _typeshed import Incomplete +from typing import ClassVar, Literal + +from ..cmd import Command + +if sys.platform == "win32": + from msilib import Control, Dialog + + class PyDialog(Dialog): + def __init__(self, *args, **kw) -> None: ... + def title(self, title) -> None: ... + def back(self, title, next, name: str = "Back", active: bool | Literal[0, 1] = 1) -> Control: ... + def cancel(self, title, next, name: str = "Cancel", active: bool | Literal[0, 1] = 1) -> Control: ... + def next(self, title, next, name: str = "Next", active: bool | Literal[0, 1] = 1) -> Control: ... + def xbutton(self, name, title, next, xpos) -> Control: ... + + class bdist_msi(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + all_versions: Incomplete + other_version: str + if sys.version_info >= (3, 9): + def __init__(self, *args, **kw) -> None: ... + bdist_dir: Incomplete + plat_name: Incomplete + keep_temp: int + no_target_compile: int + no_target_optimize: int + target_version: Incomplete + dist_dir: Incomplete + skip_build: Incomplete + install_script: Incomplete + pre_install_script: Incomplete + versions: Incomplete + def initialize_options(self) -> None: ... + install_script_key: Incomplete + def finalize_options(self) -> None: ... + db: Incomplete + def run(self) -> None: ... + def add_files(self) -> None: ... + def add_find_python(self) -> None: ... + def add_scripts(self) -> None: ... + def add_ui(self) -> None: ... + def get_installer_filename(self, fullname): ... diff --git a/stubs/3.2/bz2.py b/mypy/typeshed/stdlib/distutils/command/bdist_packager.pyi similarity index 100% rename from stubs/3.2/bz2.py rename to mypy/typeshed/stdlib/distutils/command/bdist_packager.pyi diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi new file mode 100644 index 000000000000..83b4161094c5 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi @@ -0,0 +1,53 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command + +class bdist_rpm(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] + bdist_base: Incomplete + rpm_base: Incomplete + dist_dir: Incomplete + python: Incomplete + fix_python: Incomplete + spec_only: Incomplete + binary_only: Incomplete + source_only: Incomplete + use_bzip2: Incomplete + distribution_name: Incomplete + group: Incomplete + release: Incomplete + serial: Incomplete + vendor: Incomplete + packager: Incomplete + doc_files: Incomplete + changelog: Incomplete + icon: Incomplete + prep_script: Incomplete + build_script: Incomplete + install_script: Incomplete + clean_script: Incomplete + verify_script: Incomplete + pre_install: Incomplete + post_install: Incomplete + pre_uninstall: Incomplete + post_uninstall: Incomplete + prep: Incomplete + provides: Incomplete + requires: Incomplete + conflicts: Incomplete + build_requires: Incomplete + obsoletes: Incomplete + keep_temp: int + use_rpm_opt_flags: int + rpm3_mode: int + no_autoreq: int + force_arch: Incomplete + quiet: int + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def finalize_package_data(self) -> None: ... + def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_wininst.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_wininst.pyi new file mode 100644 index 000000000000..cf333bc5400d --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/bdist_wininst.pyi @@ -0,0 +1,16 @@ +from _typeshed import StrOrBytesPath +from distutils.cmd import Command +from typing import ClassVar + +class bdist_wininst(Command): + description: ClassVar[str] + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def get_inidata(self) -> str: ... + def create_exe(self, arcname: StrOrBytesPath, fullname: str, bitmap: StrOrBytesPath | None = None) -> None: ... + def get_installer_filename(self, fullname: str) -> str: ... + def get_exe_bytes(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/distutils/command/build.pyi b/mypy/typeshed/stdlib/distutils/command/build.pyi new file mode 100644 index 000000000000..3ec0c9614d62 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/build.pyi @@ -0,0 +1,34 @@ +from _typeshed import Incomplete, Unused +from collections.abc import Callable +from typing import Any, ClassVar + +from ..cmd import Command + +def show_compilers() -> None: ... + +class build(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] + build_base: str + build_purelib: Incomplete + build_platlib: Incomplete + build_lib: Incomplete + build_temp: Incomplete + build_scripts: Incomplete + compiler: Incomplete + plat_name: Incomplete + debug: Incomplete + force: int + executable: Incomplete + parallel: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def has_pure_modules(self): ... + def has_c_libraries(self): ... + def has_ext_modules(self): ... + def has_scripts(self): ... + # Any to work around variance issues + sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] diff --git a/mypy/typeshed/stdlib/distutils/command/build_clib.pyi b/mypy/typeshed/stdlib/distutils/command/build_clib.pyi new file mode 100644 index 000000000000..69cfbe7120d8 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/build_clib.pyi @@ -0,0 +1,29 @@ +from _typeshed import Incomplete, Unused +from collections.abc import Callable +from typing import ClassVar + +from ..cmd import Command + +def show_compilers() -> None: ... + +class build_clib(Command): + description: str + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] + build_clib: Incomplete + build_temp: Incomplete + libraries: Incomplete + include_dirs: Incomplete + define: Incomplete + undef: Incomplete + debug: Incomplete + force: int + compiler: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def check_library_list(self, libraries) -> None: ... + def get_library_names(self): ... + def get_source_files(self): ... + def build_libraries(self, libraries) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/build_ext.pyi b/mypy/typeshed/stdlib/distutils/command/build_ext.pyi new file mode 100644 index 000000000000..c5a9b5d508f0 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/build_ext.pyi @@ -0,0 +1,52 @@ +from _typeshed import Incomplete, Unused +from collections.abc import Callable +from typing import ClassVar + +from ..cmd import Command + +extension_name_re: Incomplete + +def show_compilers() -> None: ... + +class build_ext(Command): + description: str + sep_by: Incomplete + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] + extensions: Incomplete + build_lib: Incomplete + plat_name: Incomplete + build_temp: Incomplete + inplace: int + package: Incomplete + include_dirs: Incomplete + define: Incomplete + undef: Incomplete + libraries: Incomplete + library_dirs: Incomplete + rpath: Incomplete + link_objects: Incomplete + debug: Incomplete + force: Incomplete + compiler: Incomplete + swig: Incomplete + swig_cpp: Incomplete + swig_opts: Incomplete + user: Incomplete + parallel: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def check_extensions_list(self, extensions) -> None: ... + def get_source_files(self): ... + def get_outputs(self): ... + def build_extensions(self) -> None: ... + def build_extension(self, ext) -> None: ... + def swig_sources(self, sources, extension): ... + def find_swig(self): ... + def get_ext_fullpath(self, ext_name: str) -> str: ... + def get_ext_fullname(self, ext_name: str) -> str: ... + def get_ext_filename(self, ext_name: str) -> str: ... + def get_export_symbols(self, ext): ... + def get_libraries(self, ext): ... diff --git a/mypy/typeshed/stdlib/distutils/command/build_py.pyi b/mypy/typeshed/stdlib/distutils/command/build_py.pyi new file mode 100644 index 000000000000..23ed230bb2d8 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/build_py.pyi @@ -0,0 +1,45 @@ +from _typeshed import Incomplete +from typing import ClassVar, Literal + +from ..cmd import Command +from ..util import Mixin2to3 as Mixin2to3 + +class build_py(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] + build_lib: Incomplete + py_modules: Incomplete + package: Incomplete + package_data: Incomplete + package_dir: Incomplete + compile: int + optimize: int + force: Incomplete + def initialize_options(self) -> None: ... + packages: Incomplete + data_files: Incomplete + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def get_data_files(self): ... + def find_data_files(self, package, src_dir): ... + def build_package_data(self) -> None: ... + def get_package_dir(self, package): ... + def check_package(self, package, package_dir): ... + def check_module(self, module, module_file): ... + def find_package_modules(self, package, package_dir): ... + def find_modules(self): ... + def find_all_modules(self): ... + def get_source_files(self): ... + def get_module_outfile(self, build_dir, package, module): ... + def get_outputs(self, include_bytecode: bool | Literal[0, 1] = 1) -> list[str]: ... + def build_module(self, module, module_file, package): ... + def build_modules(self) -> None: ... + def build_packages(self) -> None: ... + def byte_compile(self, files) -> None: ... + +class build_py_2to3(build_py, Mixin2to3): + updated_files: Incomplete + def run(self) -> None: ... + def build_module(self, module, module_file, package): ... diff --git a/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi b/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi new file mode 100644 index 000000000000..8372919bbd53 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi @@ -0,0 +1,25 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command +from ..util import Mixin2to3 as Mixin2to3 + +first_line_re: Incomplete + +class build_scripts(Command): + description: str + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] + build_dir: Incomplete + scripts: Incomplete + force: Incomplete + executable: Incomplete + outfiles: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def get_source_files(self): ... + def run(self) -> None: ... + def copy_scripts(self): ... + +class build_scripts_2to3(build_scripts, Mixin2to3): + def copy_scripts(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/check.pyi b/mypy/typeshed/stdlib/distutils/command/check.pyi new file mode 100644 index 000000000000..2c807fd2c439 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/check.pyi @@ -0,0 +1,40 @@ +from _typeshed import Incomplete +from typing import Any, ClassVar, Final, Literal +from typing_extensions import TypeAlias + +from ..cmd import Command + +_Reporter: TypeAlias = Any # really docutils.utils.Reporter + +# Only defined if docutils is installed. +# Depends on a third-party stub. Since distutils is deprecated anyway, +# it's easier to just suppress the "any subclassing" error. +class SilentReporter(_Reporter): + messages: Incomplete + def __init__( + self, + source, + report_level, + halt_level, + stream: Incomplete | None = ..., + debug: bool | Literal[0, 1] = 0, + encoding: str = ..., + error_handler: str = ..., + ) -> None: ... + def system_message(self, level, message, *children, **kwargs): ... + +HAS_DOCUTILS: Final[bool] + +class check(Command): + description: str + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] + restructuredtext: int + metadata: int + strict: int + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def warn(self, msg): ... + def run(self) -> None: ... + def check_metadata(self) -> None: ... + def check_restructuredtext(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/clean.pyi b/mypy/typeshed/stdlib/distutils/command/clean.pyi new file mode 100644 index 000000000000..0f3768d6dcf4 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/clean.pyi @@ -0,0 +1,18 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command + +class clean(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + build_base: Incomplete + build_lib: Incomplete + build_temp: Incomplete + build_scripts: Incomplete + bdist_base: Incomplete + all: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/config.pyi b/mypy/typeshed/stdlib/distutils/command/config.pyi new file mode 100644 index 000000000000..562ff3a5271f --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/config.pyi @@ -0,0 +1,84 @@ +from _typeshed import Incomplete, StrOrBytesPath +from collections.abc import Sequence +from re import Pattern +from typing import ClassVar, Final, Literal + +from ..ccompiler import CCompiler +from ..cmd import Command + +LANG_EXT: Final[dict[str, str]] + +class config(Command): + description: str + # Tuple is full name, short name, description + user_options: ClassVar[list[tuple[str, str | None, str]]] + compiler: str | CCompiler + cc: str | None + include_dirs: Sequence[str] | None + libraries: Sequence[str] | None + library_dirs: Sequence[str] | None + noisy: int + dump_source: int + temp_files: Sequence[str] + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def try_cpp( + self, + body: str | None = None, + headers: Sequence[str] | None = None, + include_dirs: Sequence[str] | None = None, + lang: str = "c", + ) -> bool: ... + def search_cpp( + self, + pattern: Pattern[str] | str, + body: str | None = None, + headers: Sequence[str] | None = None, + include_dirs: Sequence[str] | None = None, + lang: str = "c", + ) -> bool: ... + def try_compile( + self, body: str, headers: Sequence[str] | None = None, include_dirs: Sequence[str] | None = None, lang: str = "c" + ) -> bool: ... + def try_link( + self, + body: str, + headers: Sequence[str] | None = None, + include_dirs: Sequence[str] | None = None, + libraries: Sequence[str] | None = None, + library_dirs: Sequence[str] | None = None, + lang: str = "c", + ) -> bool: ... + def try_run( + self, + body: str, + headers: Sequence[str] | None = None, + include_dirs: Sequence[str] | None = None, + libraries: Sequence[str] | None = None, + library_dirs: Sequence[str] | None = None, + lang: str = "c", + ) -> bool: ... + def check_func( + self, + func: str, + headers: Sequence[str] | None = None, + include_dirs: Sequence[str] | None = None, + libraries: Sequence[str] | None = None, + library_dirs: Sequence[str] | None = None, + decl: bool | Literal[0, 1] = 0, + call: bool | Literal[0, 1] = 0, + ) -> bool: ... + def check_lib( + self, + library: str, + library_dirs: Sequence[str] | None = None, + headers: Sequence[str] | None = None, + include_dirs: Sequence[str] | None = None, + other_libraries: list[str] = [], + ) -> bool: ... + def check_header( + self, header: str, include_dirs: Sequence[str] | None = None, library_dirs: Sequence[str] | None = None, lang: str = "c" + ) -> bool: ... + +def dump_file(filename: StrOrBytesPath, head: Incomplete | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/install.pyi b/mypy/typeshed/stdlib/distutils/command/install.pyi new file mode 100644 index 000000000000..1714e01a2c28 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/install.pyi @@ -0,0 +1,71 @@ +import sys +from _typeshed import Incomplete +from collections.abc import Callable +from typing import Any, ClassVar, Final, Literal + +from ..cmd import Command + +HAS_USER_SITE: Final[bool] + +SCHEME_KEYS: Final[tuple[Literal["purelib"], Literal["platlib"], Literal["headers"], Literal["scripts"], Literal["data"]]] +INSTALL_SCHEMES: Final[dict[str, dict[str, str]]] + +if sys.version_info < (3, 10): + WINDOWS_SCHEME: Final[dict[str, str]] + +class install(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] + prefix: str | None + exec_prefix: Incomplete + home: str | None + user: bool + install_base: Incomplete + install_platbase: Incomplete + root: str | None + install_purelib: Incomplete + install_platlib: Incomplete + install_headers: Incomplete + install_lib: str | None + install_scripts: Incomplete + install_data: Incomplete + install_userbase: Incomplete + install_usersite: Incomplete + compile: Incomplete + optimize: Incomplete + extra_path: Incomplete + install_path_file: int + force: int + skip_build: int + warn_dir: int + build_base: Incomplete + build_lib: Incomplete + record: Incomplete + def initialize_options(self) -> None: ... + config_vars: Incomplete + install_libbase: Incomplete + def finalize_options(self) -> None: ... + def dump_dirs(self, msg) -> None: ... + def finalize_unix(self) -> None: ... + def finalize_other(self) -> None: ... + def select_scheme(self, name) -> None: ... + def expand_basedirs(self) -> None: ... + def expand_dirs(self) -> None: ... + def convert_paths(self, *names) -> None: ... + path_file: Incomplete + extra_dirs: Incomplete + def handle_extra_path(self) -> None: ... + def change_roots(self, *names) -> None: ... + def create_home_path(self) -> None: ... + def run(self) -> None: ... + def create_path_file(self) -> None: ... + def get_outputs(self): ... + def get_inputs(self): ... + def has_lib(self): ... + def has_headers(self): ... + def has_scripts(self): ... + def has_data(self): ... + # Any to work around variance issues + sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] diff --git a/mypy/typeshed/stdlib/distutils/command/install_data.pyi b/mypy/typeshed/stdlib/distutils/command/install_data.pyi new file mode 100644 index 000000000000..609de62b04b5 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/install_data.pyi @@ -0,0 +1,20 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command + +class install_data(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + install_dir: Incomplete + outfiles: Incomplete + root: Incomplete + force: int + data_files: Incomplete + warn_dir: int + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def get_inputs(self): ... + def get_outputs(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi b/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi new file mode 100644 index 000000000000..75bb906ce582 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi @@ -0,0 +1,19 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command + +class install_egg_info(Command): + description: ClassVar[str] + user_options: ClassVar[list[tuple[str, str, str]]] + install_dir: Incomplete + def initialize_options(self) -> None: ... + target: Incomplete + outputs: Incomplete + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def get_outputs(self) -> list[str]: ... + +def safe_name(name): ... +def safe_version(version): ... +def to_filename(name): ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_headers.pyi b/mypy/typeshed/stdlib/distutils/command/install_headers.pyi new file mode 100644 index 000000000000..3caad8a07dca --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/install_headers.pyi @@ -0,0 +1,17 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command + +class install_headers(Command): + description: str + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] + install_dir: Incomplete + force: int + outfiles: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def get_inputs(self): ... + def get_outputs(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_lib.pyi b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi new file mode 100644 index 000000000000..a537e254904a --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi @@ -0,0 +1,26 @@ +from _typeshed import Incomplete +from typing import ClassVar, Final + +from ..cmd import Command + +PYTHON_SOURCE_EXTENSION: Final = ".py" + +class install_lib(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] + install_dir: Incomplete + build_dir: Incomplete + force: int + compile: Incomplete + optimize: Incomplete + skip_build: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def build(self) -> None: ... + def install(self): ... + def byte_compile(self, files) -> None: ... + def get_outputs(self): ... + def get_inputs(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi b/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi new file mode 100644 index 000000000000..658594f32e43 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi @@ -0,0 +1,19 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..cmd import Command + +class install_scripts(Command): + description: str + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + install_dir: Incomplete + force: int + build_dir: Incomplete + skip_build: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + outfiles: Incomplete + def run(self) -> None: ... + def get_inputs(self): ... + def get_outputs(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/register.pyi b/mypy/typeshed/stdlib/distutils/command/register.pyi new file mode 100644 index 000000000000..cf98e178a9ba --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/register.pyi @@ -0,0 +1,21 @@ +from _typeshed import Incomplete +from collections.abc import Callable +from typing import Any, ClassVar + +from ..config import PyPIRCCommand + +class register(PyPIRCCommand): + description: str + # Any to work around variance issues + sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] + list_classifiers: int + strict: int + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def check_metadata(self) -> None: ... + def classifiers(self) -> None: ... + def verify_metadata(self) -> None: ... + def send_metadata(self) -> None: ... + def build_post_data(self, action): ... + def post_to_server(self, data, auth: Incomplete | None = None): ... diff --git a/mypy/typeshed/stdlib/distutils/command/sdist.pyi b/mypy/typeshed/stdlib/distutils/command/sdist.pyi new file mode 100644 index 000000000000..48a140714dda --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/sdist.pyi @@ -0,0 +1,45 @@ +from _typeshed import Incomplete, Unused +from collections.abc import Callable +from typing import Any, ClassVar + +from ..cmd import Command + +def show_formats() -> None: ... + +class sdist(Command): + description: str + def checking_metadata(self): ... + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] + negative_opt: ClassVar[dict[str, str]] + # Any to work around variance issues + sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] + READMES: ClassVar[tuple[str, ...]] + template: Incomplete + manifest: Incomplete + use_defaults: int + prune: int + manifest_only: int + force_manifest: int + formats: Incomplete + keep_temp: int + dist_dir: Incomplete + archive_files: Incomplete + metadata_check: int + owner: Incomplete + group: Incomplete + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + filelist: Incomplete + def run(self) -> None: ... + def check_metadata(self) -> None: ... + def get_file_list(self) -> None: ... + def add_defaults(self) -> None: ... + def read_template(self) -> None: ... + def prune_file_list(self) -> None: ... + def write_manifest(self) -> None: ... + def read_manifest(self) -> None: ... + def make_release_tree(self, base_dir, files) -> None: ... + def make_distribution(self) -> None: ... + def get_archive_files(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/upload.pyi b/mypy/typeshed/stdlib/distutils/command/upload.pyi new file mode 100644 index 000000000000..afcfbaf48677 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/command/upload.pyi @@ -0,0 +1,18 @@ +from _typeshed import Incomplete +from typing import ClassVar + +from ..config import PyPIRCCommand + +class upload(PyPIRCCommand): + description: ClassVar[str] + username: str + password: str + show_response: int + sign: bool + identity: Incomplete + def initialize_options(self) -> None: ... + repository: Incomplete + realm: Incomplete + def finalize_options(self) -> None: ... + def run(self) -> None: ... + def upload_file(self, command: str, pyversion: str, filename: str) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/config.pyi b/mypy/typeshed/stdlib/distutils/config.pyi new file mode 100644 index 000000000000..5814a82841cc --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/config.pyi @@ -0,0 +1,17 @@ +from abc import abstractmethod +from distutils.cmd import Command +from typing import ClassVar + +DEFAULT_PYPIRC: str + +class PyPIRCCommand(Command): + DEFAULT_REPOSITORY: ClassVar[str] + DEFAULT_REALM: ClassVar[str] + repository: None + realm: None + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + def initialize_options(self) -> None: ... + def finalize_options(self) -> None: ... + @abstractmethod + def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/core.pyi b/mypy/typeshed/stdlib/distutils/core.pyi new file mode 100644 index 000000000000..174f24991351 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/core.pyi @@ -0,0 +1,58 @@ +from _typeshed import Incomplete, StrOrBytesPath +from collections.abc import Mapping +from distutils.cmd import Command as Command +from distutils.dist import Distribution as Distribution +from distutils.extension import Extension as Extension +from typing import Any, Final, Literal + +USAGE: Final[str] + +def gen_usage(script_name: StrOrBytesPath) -> str: ... + +setup_keywords: tuple[str, ...] +extension_keywords: tuple[str, ...] + +def setup( + *, + name: str = ..., + version: str = ..., + description: str = ..., + long_description: str = ..., + author: str = ..., + author_email: str = ..., + maintainer: str = ..., + maintainer_email: str = ..., + url: str = ..., + download_url: str = ..., + packages: list[str] = ..., + py_modules: list[str] = ..., + scripts: list[str] = ..., + ext_modules: list[Extension] = ..., + classifiers: list[str] = ..., + distclass: type[Distribution] = ..., + script_name: str = ..., + script_args: list[str] = ..., + options: Mapping[str, Incomplete] = ..., + license: str = ..., + keywords: list[str] | str = ..., + platforms: list[str] | str = ..., + cmdclass: Mapping[str, type[Command]] = ..., + data_files: list[tuple[str, list[str]]] = ..., + package_dir: Mapping[str, str] = ..., + obsoletes: list[str] = ..., + provides: list[str] = ..., + requires: list[str] = ..., + command_packages: list[str] = ..., + command_options: Mapping[str, Mapping[str, tuple[Incomplete, Incomplete]]] = ..., + package_data: Mapping[str, list[str]] = ..., + include_package_data: bool | Literal[0, 1] = ..., + libraries: list[str] = ..., + headers: list[str] = ..., + ext_package: str = ..., + include_dirs: list[str] = ..., + password: str = ..., + fullname: str = ..., + # Custom Distributions could accept more params + **attrs: Any, +) -> Distribution: ... +def run_setup(script_name: str, script_args: list[str] | None = None, stop_after: str = "run") -> Distribution: ... diff --git a/mypy/typeshed/stdlib/distutils/cygwinccompiler.pyi b/mypy/typeshed/stdlib/distutils/cygwinccompiler.pyi new file mode 100644 index 000000000000..80924d63e471 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/cygwinccompiler.pyi @@ -0,0 +1,20 @@ +from distutils.unixccompiler import UnixCCompiler +from distutils.version import LooseVersion +from re import Pattern +from typing import Final, Literal + +def get_msvcr() -> list[str] | None: ... + +class CygwinCCompiler(UnixCCompiler): ... +class Mingw32CCompiler(CygwinCCompiler): ... + +CONFIG_H_OK: Final = "ok" +CONFIG_H_NOTOK: Final = "not ok" +CONFIG_H_UNCERTAIN: Final = "uncertain" + +def check_config_h() -> tuple[Literal["ok", "not ok", "uncertain"], str]: ... + +RE_VERSION: Final[Pattern[bytes]] + +def get_versions() -> tuple[LooseVersion | None, ...]: ... +def is_cygwingcc() -> bool: ... diff --git a/mypy/typeshed/stdlib/distutils/debug.pyi b/mypy/typeshed/stdlib/distutils/debug.pyi new file mode 100644 index 000000000000..30095883b064 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/debug.pyi @@ -0,0 +1,3 @@ +from typing import Final + +DEBUG: Final[str | None] diff --git a/mypy/typeshed/stdlib/distutils/dep_util.pyi b/mypy/typeshed/stdlib/distutils/dep_util.pyi new file mode 100644 index 000000000000..058377accabc --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/dep_util.pyi @@ -0,0 +1,14 @@ +from _typeshed import StrOrBytesPath, SupportsLenAndGetItem +from collections.abc import Iterable +from typing import Literal, TypeVar + +_SourcesT = TypeVar("_SourcesT", bound=StrOrBytesPath) +_TargetsT = TypeVar("_TargetsT", bound=StrOrBytesPath) + +def newer(source: StrOrBytesPath, target: StrOrBytesPath) -> bool | Literal[1]: ... +def newer_pairwise( + sources: SupportsLenAndGetItem[_SourcesT], targets: SupportsLenAndGetItem[_TargetsT] +) -> tuple[list[_SourcesT], list[_TargetsT]]: ... +def newer_group( + sources: Iterable[StrOrBytesPath], target: StrOrBytesPath, missing: Literal["error", "ignore", "newer"] = "error" +) -> Literal[0, 1]: ... diff --git a/mypy/typeshed/stdlib/distutils/dir_util.pyi b/mypy/typeshed/stdlib/distutils/dir_util.pyi new file mode 100644 index 000000000000..23e2c3bc28b9 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/dir_util.pyi @@ -0,0 +1,23 @@ +from _typeshed import StrOrBytesPath, StrPath +from collections.abc import Iterable +from typing import Literal + +def mkpath(name: str, mode: int = 0o777, verbose: bool | Literal[0, 1] = 1, dry_run: bool | Literal[0, 1] = 0) -> list[str]: ... +def create_tree( + base_dir: StrPath, + files: Iterable[StrPath], + mode: int = 0o777, + verbose: bool | Literal[0, 1] = 1, + dry_run: bool | Literal[0, 1] = 0, +) -> None: ... +def copy_tree( + src: StrPath, + dst: str, + preserve_mode: bool | Literal[0, 1] = 1, + preserve_times: bool | Literal[0, 1] = 1, + preserve_symlinks: bool | Literal[0, 1] = 0, + update: bool | Literal[0, 1] = 0, + verbose: bool | Literal[0, 1] = 1, + dry_run: bool | Literal[0, 1] = 0, +) -> list[str]: ... +def remove_tree(directory: StrOrBytesPath, verbose: bool | Literal[0, 1] = 1, dry_run: bool | Literal[0, 1] = 0) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/dist.pyi b/mypy/typeshed/stdlib/distutils/dist.pyi new file mode 100644 index 000000000000..75fc7dbb388d --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/dist.pyi @@ -0,0 +1,317 @@ +from _typeshed import Incomplete, StrOrBytesPath, StrPath, SupportsWrite +from collections.abc import Iterable, MutableMapping +from distutils.cmd import Command +from distutils.command.bdist import bdist +from distutils.command.bdist_dumb import bdist_dumb +from distutils.command.bdist_rpm import bdist_rpm +from distutils.command.build import build +from distutils.command.build_clib import build_clib +from distutils.command.build_ext import build_ext +from distutils.command.build_py import build_py +from distutils.command.build_scripts import build_scripts +from distutils.command.check import check +from distutils.command.clean import clean +from distutils.command.config import config +from distutils.command.install import install +from distutils.command.install_data import install_data +from distutils.command.install_egg_info import install_egg_info +from distutils.command.install_headers import install_headers +from distutils.command.install_lib import install_lib +from distutils.command.install_scripts import install_scripts +from distutils.command.register import register +from distutils.command.sdist import sdist +from distutils.command.upload import upload +from re import Pattern +from typing import IO, ClassVar, Literal, TypeVar, overload +from typing_extensions import TypeAlias + +command_re: Pattern[str] + +_OptionsList: TypeAlias = list[tuple[str, str | None, str, int] | tuple[str, str | None, str]] +_CommandT = TypeVar("_CommandT", bound=Command) + +class DistributionMetadata: + def __init__(self, path: StrOrBytesPath | None = None) -> None: ... + name: str | None + version: str | None + author: str | None + author_email: str | None + maintainer: str | None + maintainer_email: str | None + url: str | None + license: str | None + description: str | None + long_description: str | None + keywords: str | list[str] | None + platforms: str | list[str] | None + classifiers: str | list[str] | None + download_url: str | None + provides: list[str] | None + requires: list[str] | None + obsoletes: list[str] | None + def read_pkg_file(self, file: IO[str]) -> None: ... + def write_pkg_info(self, base_dir: StrPath) -> None: ... + def write_pkg_file(self, file: SupportsWrite[str]) -> None: ... + def get_name(self) -> str: ... + def get_version(self) -> str: ... + def get_fullname(self) -> str: ... + def get_author(self) -> str: ... + def get_author_email(self) -> str: ... + def get_maintainer(self) -> str: ... + def get_maintainer_email(self) -> str: ... + def get_contact(self) -> str: ... + def get_contact_email(self) -> str: ... + def get_url(self) -> str: ... + def get_license(self) -> str: ... + def get_licence(self) -> str: ... + def get_description(self) -> str: ... + def get_long_description(self) -> str: ... + def get_keywords(self) -> str | list[str]: ... + def get_platforms(self) -> str | list[str]: ... + def get_classifiers(self) -> str | list[str]: ... + def get_download_url(self) -> str: ... + def get_requires(self) -> list[str]: ... + def set_requires(self, value: Iterable[str]) -> None: ... + def get_provides(self) -> list[str]: ... + def set_provides(self, value: Iterable[str]) -> None: ... + def get_obsoletes(self) -> list[str]: ... + def set_obsoletes(self, value: Iterable[str]) -> None: ... + +class Distribution: + cmdclass: dict[str, type[Command]] + metadata: DistributionMetadata + def __init__(self, attrs: MutableMapping[str, Incomplete] | None = None) -> None: ... + def get_option_dict(self, command: str) -> dict[str, tuple[str, str]]: ... + def parse_config_files(self, filenames: Iterable[str] | None = None) -> None: ... + global_options: ClassVar[_OptionsList] + common_usage: ClassVar[str] + display_options: ClassVar[_OptionsList] + display_option_names: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] + verbose: Literal[0, 1] + dry_run: Literal[0, 1] + help: Literal[0, 1] + command_packages: list[str] | None + script_name: str | None + script_args: list[str] | None + command_options: dict[str, dict[str, tuple[str, str]]] + dist_files: list[tuple[str, str, str]] + packages: Incomplete + package_data: dict[str, list[str]] + package_dir: Incomplete + py_modules: Incomplete + libraries: Incomplete + headers: Incomplete + ext_modules: Incomplete + ext_package: Incomplete + include_dirs: Incomplete + extra_path: Incomplete + scripts: Incomplete + data_files: Incomplete + password: str + command_obj: Incomplete + have_run: Incomplete + want_user_cfg: bool + def dump_option_dicts( + self, header: Incomplete | None = None, commands: Incomplete | None = None, indent: str = "" + ) -> None: ... + def find_config_files(self): ... + commands: Incomplete + def parse_command_line(self): ... + def finalize_options(self) -> None: ... + def handle_display_options(self, option_order): ... + def print_command_list(self, commands, header, max_length) -> None: ... + def print_commands(self) -> None: ... + def get_command_list(self): ... + def get_command_packages(self): ... + # NOTE: This list comes directly from the distutils/command folder. Minus bdist_msi and bdist_wininst. + @overload + def get_command_obj(self, command: Literal["bdist"], create: Literal[1, True] = 1) -> bdist: ... + @overload + def get_command_obj(self, command: Literal["bdist_dumb"], create: Literal[1, True] = 1) -> bdist_dumb: ... + @overload + def get_command_obj(self, command: Literal["bdist_rpm"], create: Literal[1, True] = 1) -> bdist_rpm: ... + @overload + def get_command_obj(self, command: Literal["build"], create: Literal[1, True] = 1) -> build: ... + @overload + def get_command_obj(self, command: Literal["build_clib"], create: Literal[1, True] = 1) -> build_clib: ... + @overload + def get_command_obj(self, command: Literal["build_ext"], create: Literal[1, True] = 1) -> build_ext: ... + @overload + def get_command_obj(self, command: Literal["build_py"], create: Literal[1, True] = 1) -> build_py: ... + @overload + def get_command_obj(self, command: Literal["build_scripts"], create: Literal[1, True] = 1) -> build_scripts: ... + @overload + def get_command_obj(self, command: Literal["check"], create: Literal[1, True] = 1) -> check: ... + @overload + def get_command_obj(self, command: Literal["clean"], create: Literal[1, True] = 1) -> clean: ... + @overload + def get_command_obj(self, command: Literal["config"], create: Literal[1, True] = 1) -> config: ... + @overload + def get_command_obj(self, command: Literal["install"], create: Literal[1, True] = 1) -> install: ... + @overload + def get_command_obj(self, command: Literal["install_data"], create: Literal[1, True] = 1) -> install_data: ... + @overload + def get_command_obj(self, command: Literal["install_egg_info"], create: Literal[1, True] = 1) -> install_egg_info: ... + @overload + def get_command_obj(self, command: Literal["install_headers"], create: Literal[1, True] = 1) -> install_headers: ... + @overload + def get_command_obj(self, command: Literal["install_lib"], create: Literal[1, True] = 1) -> install_lib: ... + @overload + def get_command_obj(self, command: Literal["install_scripts"], create: Literal[1, True] = 1) -> install_scripts: ... + @overload + def get_command_obj(self, command: Literal["register"], create: Literal[1, True] = 1) -> register: ... + @overload + def get_command_obj(self, command: Literal["sdist"], create: Literal[1, True] = 1) -> sdist: ... + @overload + def get_command_obj(self, command: Literal["upload"], create: Literal[1, True] = 1) -> upload: ... + @overload + def get_command_obj(self, command: str, create: Literal[1, True] = 1) -> Command: ... + # Not replicating the overloads for "Command | None", user may use "isinstance" + @overload + def get_command_obj(self, command: str, create: Literal[0, False]) -> Command | None: ... + @overload + def get_command_class(self, command: Literal["bdist"]) -> type[bdist]: ... + @overload + def get_command_class(self, command: Literal["bdist_dumb"]) -> type[bdist_dumb]: ... + @overload + def get_command_class(self, command: Literal["bdist_rpm"]) -> type[bdist_rpm]: ... + @overload + def get_command_class(self, command: Literal["build"]) -> type[build]: ... + @overload + def get_command_class(self, command: Literal["build_clib"]) -> type[build_clib]: ... + @overload + def get_command_class(self, command: Literal["build_ext"]) -> type[build_ext]: ... + @overload + def get_command_class(self, command: Literal["build_py"]) -> type[build_py]: ... + @overload + def get_command_class(self, command: Literal["build_scripts"]) -> type[build_scripts]: ... + @overload + def get_command_class(self, command: Literal["check"]) -> type[check]: ... + @overload + def get_command_class(self, command: Literal["clean"]) -> type[clean]: ... + @overload + def get_command_class(self, command: Literal["config"]) -> type[config]: ... + @overload + def get_command_class(self, command: Literal["install"]) -> type[install]: ... + @overload + def get_command_class(self, command: Literal["install_data"]) -> type[install_data]: ... + @overload + def get_command_class(self, command: Literal["install_egg_info"]) -> type[install_egg_info]: ... + @overload + def get_command_class(self, command: Literal["install_headers"]) -> type[install_headers]: ... + @overload + def get_command_class(self, command: Literal["install_lib"]) -> type[install_lib]: ... + @overload + def get_command_class(self, command: Literal["install_scripts"]) -> type[install_scripts]: ... + @overload + def get_command_class(self, command: Literal["register"]) -> type[register]: ... + @overload + def get_command_class(self, command: Literal["sdist"]) -> type[sdist]: ... + @overload + def get_command_class(self, command: Literal["upload"]) -> type[upload]: ... + @overload + def get_command_class(self, command: str) -> type[Command]: ... + @overload + def reinitialize_command(self, command: Literal["bdist"], reinit_subcommands: bool = False) -> bdist: ... + @overload + def reinitialize_command(self, command: Literal["bdist_dumb"], reinit_subcommands: bool = False) -> bdist_dumb: ... + @overload + def reinitialize_command(self, command: Literal["bdist_rpm"], reinit_subcommands: bool = False) -> bdist_rpm: ... + @overload + def reinitialize_command(self, command: Literal["build"], reinit_subcommands: bool = False) -> build: ... + @overload + def reinitialize_command(self, command: Literal["build_clib"], reinit_subcommands: bool = False) -> build_clib: ... + @overload + def reinitialize_command(self, command: Literal["build_ext"], reinit_subcommands: bool = False) -> build_ext: ... + @overload + def reinitialize_command(self, command: Literal["build_py"], reinit_subcommands: bool = False) -> build_py: ... + @overload + def reinitialize_command(self, command: Literal["build_scripts"], reinit_subcommands: bool = False) -> build_scripts: ... + @overload + def reinitialize_command(self, command: Literal["check"], reinit_subcommands: bool = False) -> check: ... + @overload + def reinitialize_command(self, command: Literal["clean"], reinit_subcommands: bool = False) -> clean: ... + @overload + def reinitialize_command(self, command: Literal["config"], reinit_subcommands: bool = False) -> config: ... + @overload + def reinitialize_command(self, command: Literal["install"], reinit_subcommands: bool = False) -> install: ... + @overload + def reinitialize_command(self, command: Literal["install_data"], reinit_subcommands: bool = False) -> install_data: ... + @overload + def reinitialize_command( + self, command: Literal["install_egg_info"], reinit_subcommands: bool = False + ) -> install_egg_info: ... + @overload + def reinitialize_command(self, command: Literal["install_headers"], reinit_subcommands: bool = False) -> install_headers: ... + @overload + def reinitialize_command(self, command: Literal["install_lib"], reinit_subcommands: bool = False) -> install_lib: ... + @overload + def reinitialize_command(self, command: Literal["install_scripts"], reinit_subcommands: bool = False) -> install_scripts: ... + @overload + def reinitialize_command(self, command: Literal["register"], reinit_subcommands: bool = False) -> register: ... + @overload + def reinitialize_command(self, command: Literal["sdist"], reinit_subcommands: bool = False) -> sdist: ... + @overload + def reinitialize_command(self, command: Literal["upload"], reinit_subcommands: bool = False) -> upload: ... + @overload + def reinitialize_command(self, command: str, reinit_subcommands: bool = False) -> Command: ... + @overload + def reinitialize_command(self, command: _CommandT, reinit_subcommands: bool = False) -> _CommandT: ... + def announce(self, msg, level: int = 2) -> None: ... + def run_commands(self) -> None: ... + def run_command(self, command: str) -> None: ... + def has_pure_modules(self) -> bool: ... + def has_ext_modules(self) -> bool: ... + def has_c_libraries(self) -> bool: ... + def has_modules(self) -> bool: ... + def has_headers(self) -> bool: ... + def has_scripts(self) -> bool: ... + def has_data_files(self) -> bool: ... + def is_pure(self) -> bool: ... + + # Default getter methods generated in __init__ from self.metadata._METHOD_BASENAMES + def get_name(self) -> str: ... + def get_version(self) -> str: ... + def get_fullname(self) -> str: ... + def get_author(self) -> str: ... + def get_author_email(self) -> str: ... + def get_maintainer(self) -> str: ... + def get_maintainer_email(self) -> str: ... + def get_contact(self) -> str: ... + def get_contact_email(self) -> str: ... + def get_url(self) -> str: ... + def get_license(self) -> str: ... + def get_licence(self) -> str: ... + def get_description(self) -> str: ... + def get_long_description(self) -> str: ... + def get_keywords(self) -> str | list[str]: ... + def get_platforms(self) -> str | list[str]: ... + def get_classifiers(self) -> str | list[str]: ... + def get_download_url(self) -> str: ... + def get_requires(self) -> list[str]: ... + def get_provides(self) -> list[str]: ... + def get_obsoletes(self) -> list[str]: ... + + # Default attributes generated in __init__ from self.display_option_names + help_commands: bool | Literal[0] + name: str | Literal[0] + version: str | Literal[0] + fullname: str | Literal[0] + author: str | Literal[0] + author_email: str | Literal[0] + maintainer: str | Literal[0] + maintainer_email: str | Literal[0] + contact: str | Literal[0] + contact_email: str | Literal[0] + url: str | Literal[0] + license: str | Literal[0] + licence: str | Literal[0] + description: str | Literal[0] + long_description: str | Literal[0] + platforms: str | list[str] | Literal[0] + classifiers: str | list[str] | Literal[0] + keywords: str | list[str] | Literal[0] + provides: list[str] | Literal[0] + requires: list[str] | Literal[0] + obsoletes: list[str] | Literal[0] diff --git a/mypy/typeshed/stdlib/distutils/errors.pyi b/mypy/typeshed/stdlib/distutils/errors.pyi new file mode 100644 index 000000000000..e483362bfbf1 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/errors.pyi @@ -0,0 +1,19 @@ +class DistutilsError(Exception): ... +class DistutilsModuleError(DistutilsError): ... +class DistutilsClassError(DistutilsError): ... +class DistutilsGetoptError(DistutilsError): ... +class DistutilsArgError(DistutilsError): ... +class DistutilsFileError(DistutilsError): ... +class DistutilsOptionError(DistutilsError): ... +class DistutilsSetupError(DistutilsError): ... +class DistutilsPlatformError(DistutilsError): ... +class DistutilsExecError(DistutilsError): ... +class DistutilsInternalError(DistutilsError): ... +class DistutilsTemplateError(DistutilsError): ... +class DistutilsByteCompileError(DistutilsError): ... +class CCompilerError(Exception): ... +class PreprocessError(CCompilerError): ... +class CompileError(CCompilerError): ... +class LibError(CCompilerError): ... +class LinkError(CCompilerError): ... +class UnknownFileError(CCompilerError): ... diff --git a/mypy/typeshed/stdlib/distutils/extension.pyi b/mypy/typeshed/stdlib/distutils/extension.pyi new file mode 100644 index 000000000000..789bbf6ec3d1 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/extension.pyi @@ -0,0 +1,36 @@ +class Extension: + name: str + sources: list[str] + include_dirs: list[str] + define_macros: list[tuple[str, str | None]] + undef_macros: list[str] + library_dirs: list[str] + libraries: list[str] + runtime_library_dirs: list[str] + extra_objects: list[str] + extra_compile_args: list[str] + extra_link_args: list[str] + export_symbols: list[str] + swig_opts: list[str] + depends: list[str] + language: str | None + optional: bool | None + def __init__( + self, + name: str, + sources: list[str], + include_dirs: list[str] | None = None, + define_macros: list[tuple[str, str | None]] | None = None, + undef_macros: list[str] | None = None, + library_dirs: list[str] | None = None, + libraries: list[str] | None = None, + runtime_library_dirs: list[str] | None = None, + extra_objects: list[str] | None = None, + extra_compile_args: list[str] | None = None, + extra_link_args: list[str] | None = None, + export_symbols: list[str] | None = None, + swig_opts: list[str] | None = None, + depends: list[str] | None = None, + language: str | None = None, + optional: bool | None = None, + ) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/fancy_getopt.pyi b/mypy/typeshed/stdlib/distutils/fancy_getopt.pyi new file mode 100644 index 000000000000..c4d37419ed06 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/fancy_getopt.pyi @@ -0,0 +1,34 @@ +from collections.abc import Iterable, Mapping +from re import Pattern +from typing import Any, Final, overload +from typing_extensions import TypeAlias + +_Option: TypeAlias = tuple[str, str | None, str] +_GR: TypeAlias = tuple[list[str], OptionDummy] + +longopt_pat: Final = r"[a-zA-Z](?:[a-zA-Z0-9-]*)" +longopt_re: Final[Pattern[str]] +neg_alias_re: Final[Pattern[str]] +longopt_xlate: Final[dict[int, int]] + +class FancyGetopt: + def __init__(self, option_table: list[_Option] | None = None) -> None: ... + # TODO kinda wrong, `getopt(object=object())` is invalid + @overload + def getopt(self, args: list[str] | None = None) -> _GR: ... + @overload + def getopt(self, args: list[str] | None, object: Any) -> list[str]: ... + def get_option_order(self) -> list[tuple[str, str]]: ... + def generate_help(self, header: str | None = None) -> list[str]: ... + +def fancy_getopt( + options: list[_Option], negative_opt: Mapping[_Option, _Option], object: Any, args: list[str] | None +) -> list[str] | _GR: ... + +WS_TRANS: Final[dict[int, str]] + +def wrap_text(text: str, width: int) -> list[str]: ... +def translate_longopt(opt: str) -> str: ... + +class OptionDummy: + def __init__(self, options: Iterable[str] = []) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/file_util.pyi b/mypy/typeshed/stdlib/distutils/file_util.pyi new file mode 100644 index 000000000000..873d23ea7e50 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/file_util.pyi @@ -0,0 +1,38 @@ +from _typeshed import BytesPath, StrOrBytesPath, StrPath +from collections.abc import Iterable +from typing import Literal, TypeVar, overload + +_StrPathT = TypeVar("_StrPathT", bound=StrPath) +_BytesPathT = TypeVar("_BytesPathT", bound=BytesPath) + +@overload +def copy_file( + src: StrPath, + dst: _StrPathT, + preserve_mode: bool | Literal[0, 1] = 1, + preserve_times: bool | Literal[0, 1] = 1, + update: bool | Literal[0, 1] = 0, + link: str | None = None, + verbose: bool | Literal[0, 1] = 1, + dry_run: bool | Literal[0, 1] = 0, +) -> tuple[_StrPathT | str, bool]: ... +@overload +def copy_file( + src: BytesPath, + dst: _BytesPathT, + preserve_mode: bool | Literal[0, 1] = 1, + preserve_times: bool | Literal[0, 1] = 1, + update: bool | Literal[0, 1] = 0, + link: str | None = None, + verbose: bool | Literal[0, 1] = 1, + dry_run: bool | Literal[0, 1] = 0, +) -> tuple[_BytesPathT | bytes, bool]: ... +@overload +def move_file( + src: StrPath, dst: _StrPathT, verbose: bool | Literal[0, 1] = 0, dry_run: bool | Literal[0, 1] = 0 +) -> _StrPathT | str: ... +@overload +def move_file( + src: BytesPath, dst: _BytesPathT, verbose: bool | Literal[0, 1] = 0, dry_run: bool | Literal[0, 1] = 0 +) -> _BytesPathT | bytes: ... +def write_file(filename: StrOrBytesPath, contents: Iterable[str]) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/filelist.pyi b/mypy/typeshed/stdlib/distutils/filelist.pyi new file mode 100644 index 000000000000..607a78a1fbac --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/filelist.pyi @@ -0,0 +1,58 @@ +from collections.abc import Iterable +from re import Pattern +from typing import Literal, overload + +# class is entirely undocumented +class FileList: + allfiles: Iterable[str] | None + files: list[str] + def __init__(self, warn: None = None, debug_print: None = None) -> None: ... + def set_allfiles(self, allfiles: Iterable[str]) -> None: ... + def findall(self, dir: str = ".") -> None: ... + def debug_print(self, msg: str) -> None: ... + def append(self, item: str) -> None: ... + def extend(self, items: Iterable[str]) -> None: ... + def sort(self) -> None: ... + def remove_duplicates(self) -> None: ... + def process_template_line(self, line: str) -> None: ... + @overload + def include_pattern( + self, pattern: str, anchor: bool | Literal[0, 1] = 1, prefix: str | None = None, is_regex: Literal[0, False] = 0 + ) -> bool: ... + @overload + def include_pattern(self, pattern: str | Pattern[str], *, is_regex: Literal[True, 1]) -> bool: ... + @overload + def include_pattern( + self, + pattern: str | Pattern[str], + anchor: bool | Literal[0, 1] = 1, + prefix: str | None = None, + is_regex: bool | Literal[0, 1] = 0, + ) -> bool: ... + @overload + def exclude_pattern( + self, pattern: str, anchor: bool | Literal[0, 1] = 1, prefix: str | None = None, is_regex: Literal[0, False] = 0 + ) -> bool: ... + @overload + def exclude_pattern(self, pattern: str | Pattern[str], *, is_regex: Literal[True, 1]) -> bool: ... + @overload + def exclude_pattern( + self, + pattern: str | Pattern[str], + anchor: bool | Literal[0, 1] = 1, + prefix: str | None = None, + is_regex: bool | Literal[0, 1] = 0, + ) -> bool: ... + +def findall(dir: str = ".") -> list[str]: ... +def glob_to_re(pattern: str) -> str: ... +@overload +def translate_pattern( + pattern: str, anchor: bool | Literal[0, 1] = 1, prefix: str | None = None, is_regex: Literal[False, 0] = 0 +) -> Pattern[str]: ... +@overload +def translate_pattern(pattern: str | Pattern[str], *, is_regex: Literal[True, 1]) -> Pattern[str]: ... +@overload +def translate_pattern( + pattern: str | Pattern[str], anchor: bool | Literal[0, 1] = 1, prefix: str | None = None, is_regex: bool | Literal[0, 1] = 0 +) -> Pattern[str]: ... diff --git a/mypy/typeshed/stdlib/distutils/log.pyi b/mypy/typeshed/stdlib/distutils/log.pyi new file mode 100644 index 000000000000..7246dd6be0cd --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/log.pyi @@ -0,0 +1,26 @@ +from typing import Any, Final + +DEBUG: Final = 1 +INFO: Final = 2 +WARN: Final = 3 +ERROR: Final = 4 +FATAL: Final = 5 + +class Log: + def __init__(self, threshold: int = 3) -> None: ... + # Arbitrary msg args' type depends on the format method + def log(self, level: int, msg: str, *args: Any) -> None: ... + def debug(self, msg: str, *args: Any) -> None: ... + def info(self, msg: str, *args: Any) -> None: ... + def warn(self, msg: str, *args: Any) -> None: ... + def error(self, msg: str, *args: Any) -> None: ... + def fatal(self, msg: str, *args: Any) -> None: ... + +def log(level: int, msg: str, *args: Any) -> None: ... +def debug(msg: str, *args: Any) -> None: ... +def info(msg: str, *args: Any) -> None: ... +def warn(msg: str, *args: Any) -> None: ... +def error(msg: str, *args: Any) -> None: ... +def fatal(msg: str, *args: Any) -> None: ... +def set_threshold(level: int) -> int: ... +def set_verbosity(v: int) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/msvccompiler.pyi b/mypy/typeshed/stdlib/distutils/msvccompiler.pyi new file mode 100644 index 000000000000..80872a6b739f --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/msvccompiler.pyi @@ -0,0 +1,3 @@ +from distutils.ccompiler import CCompiler + +class MSVCCompiler(CCompiler): ... diff --git a/mypy/typeshed/stdlib/distutils/spawn.pyi b/mypy/typeshed/stdlib/distutils/spawn.pyi new file mode 100644 index 000000000000..ae07a49504fe --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/spawn.pyi @@ -0,0 +1,10 @@ +from collections.abc import Iterable +from typing import Literal + +def spawn( + cmd: Iterable[str], + search_path: bool | Literal[0, 1] = 1, + verbose: bool | Literal[0, 1] = 0, + dry_run: bool | Literal[0, 1] = 0, +) -> None: ... +def find_executable(executable: str, path: str | None = None) -> str | None: ... diff --git a/mypy/typeshed/stdlib/distutils/sysconfig.pyi b/mypy/typeshed/stdlib/distutils/sysconfig.pyi new file mode 100644 index 000000000000..4a9c45eb562a --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/sysconfig.pyi @@ -0,0 +1,33 @@ +import sys +from collections.abc import Mapping +from distutils.ccompiler import CCompiler +from typing import Final, Literal, overload +from typing_extensions import deprecated + +PREFIX: Final[str] +EXEC_PREFIX: Final[str] +BASE_PREFIX: Final[str] +BASE_EXEC_PREFIX: Final[str] +project_base: Final[str] +python_build: Final[bool] + +def expand_makefile_vars(s: str, vars: Mapping[str, str]) -> str: ... +@overload +@deprecated("SO is deprecated, use EXT_SUFFIX. Support is removed in Python 3.11") +def get_config_var(name: Literal["SO"]) -> int | str | None: ... +@overload +def get_config_var(name: str) -> int | str | None: ... +@overload +def get_config_vars() -> dict[str, str | int]: ... +@overload +def get_config_vars(arg: str, /, *args: str) -> list[str | int]: ... +def get_config_h_filename() -> str: ... +def get_makefile_filename() -> str: ... +def get_python_inc(plat_specific: bool | Literal[0, 1] = 0, prefix: str | None = None) -> str: ... +def get_python_lib( + plat_specific: bool | Literal[0, 1] = 0, standard_lib: bool | Literal[0, 1] = 0, prefix: str | None = None +) -> str: ... +def customize_compiler(compiler: CCompiler) -> None: ... + +if sys.version_info < (3, 10): + def get_python_version() -> str: ... diff --git a/mypy/typeshed/stdlib/distutils/text_file.pyi b/mypy/typeshed/stdlib/distutils/text_file.pyi new file mode 100644 index 000000000000..54951af7e55d --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/text_file.pyi @@ -0,0 +1,21 @@ +from typing import IO, Literal + +class TextFile: + def __init__( + self, + filename: str | None = None, + file: IO[str] | None = None, + *, + strip_comments: bool | Literal[0, 1] = ..., + lstrip_ws: bool | Literal[0, 1] = ..., + rstrip_ws: bool | Literal[0, 1] = ..., + skip_blanks: bool | Literal[0, 1] = ..., + join_lines: bool | Literal[0, 1] = ..., + collapse_join: bool | Literal[0, 1] = ..., + ) -> None: ... + def open(self, filename: str) -> None: ... + def close(self) -> None: ... + def warn(self, msg: str, line: list[int] | tuple[int, int] | int | None = None) -> None: ... + def readline(self) -> str | None: ... + def readlines(self) -> list[str]: ... + def unreadline(self, line: str) -> str: ... diff --git a/mypy/typeshed/stdlib/distutils/unixccompiler.pyi b/mypy/typeshed/stdlib/distutils/unixccompiler.pyi new file mode 100644 index 000000000000..e1d443471af3 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/unixccompiler.pyi @@ -0,0 +1,3 @@ +from distutils.ccompiler import CCompiler + +class UnixCCompiler(CCompiler): ... diff --git a/mypy/typeshed/stdlib/distutils/util.pyi b/mypy/typeshed/stdlib/distutils/util.pyi new file mode 100644 index 000000000000..0e1bb4165d99 --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/util.pyi @@ -0,0 +1,53 @@ +from _typeshed import StrPath, Unused +from collections.abc import Callable, Container, Iterable, Mapping +from typing import Any, Literal +from typing_extensions import TypeVarTuple, Unpack + +_Ts = TypeVarTuple("_Ts") + +def get_host_platform() -> str: ... +def get_platform() -> str: ... +def convert_path(pathname: str) -> str: ... +def change_root(new_root: StrPath, pathname: StrPath) -> str: ... +def check_environ() -> None: ... +def subst_vars(s: str, local_vars: Mapping[str, str]) -> None: ... +def split_quoted(s: str) -> list[str]: ... +def execute( + func: Callable[[Unpack[_Ts]], Unused], + args: tuple[Unpack[_Ts]], + msg: str | None = None, + verbose: bool | Literal[0, 1] = 0, + dry_run: bool | Literal[0, 1] = 0, +) -> None: ... +def strtobool(val: str) -> Literal[0, 1]: ... +def byte_compile( + py_files: list[str], + optimize: int = 0, + force: bool | Literal[0, 1] = 0, + prefix: str | None = None, + base_dir: str | None = None, + verbose: bool | Literal[0, 1] = 1, + dry_run: bool | Literal[0, 1] = 0, + direct: bool | None = None, +) -> None: ... +def rfc822_escape(header: str) -> str: ... +def run_2to3( + files: Iterable[str], + fixer_names: Iterable[str] | None = None, + options: Mapping[str, Any] | None = None, + explicit: Unused = None, +) -> None: ... +def copydir_run_2to3( + src: StrPath, + dest: StrPath, + template: str | None = None, + fixer_names: Iterable[str] | None = None, + options: Mapping[str, Any] | None = None, + explicit: Container[str] | None = None, +) -> list[str]: ... + +class Mixin2to3: + fixer_names: Iterable[str] | None + options: Mapping[str, Any] | None + explicit: Container[str] | None + def run_2to3(self, files: Iterable[str]) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/version.pyi b/mypy/typeshed/stdlib/distutils/version.pyi new file mode 100644 index 000000000000..47da65ef87aa --- /dev/null +++ b/mypy/typeshed/stdlib/distutils/version.pyi @@ -0,0 +1,36 @@ +from abc import abstractmethod +from re import Pattern +from typing_extensions import Self + +class Version: + def __eq__(self, other: object) -> bool: ... + def __lt__(self, other: Self | str) -> bool: ... + def __le__(self, other: Self | str) -> bool: ... + def __gt__(self, other: Self | str) -> bool: ... + def __ge__(self, other: Self | str) -> bool: ... + @abstractmethod + def __init__(self, vstring: str | None = None) -> None: ... + @abstractmethod + def parse(self, vstring: str) -> Self: ... + @abstractmethod + def __str__(self) -> str: ... + @abstractmethod + def _cmp(self, other: Self | str) -> bool: ... + +class StrictVersion(Version): + version_re: Pattern[str] + version: tuple[int, int, int] + prerelease: tuple[str, int] | None + def __init__(self, vstring: str | None = None) -> None: ... + def parse(self, vstring: str) -> Self: ... + def __str__(self) -> str: ... # noqa: Y029 + def _cmp(self, other: Self | str) -> bool: ... + +class LooseVersion(Version): + component_re: Pattern[str] + vstring: str + version: tuple[str | int, ...] + def __init__(self, vstring: str | None = None) -> None: ... + def parse(self, vstring: str) -> Self: ... + def __str__(self) -> str: ... # noqa: Y029 + def _cmp(self, other: Self | str) -> bool: ... diff --git a/mypy/typeshed/stdlib/doctest.pyi b/mypy/typeshed/stdlib/doctest.pyi new file mode 100644 index 000000000000..562b5a5bdac9 --- /dev/null +++ b/mypy/typeshed/stdlib/doctest.pyi @@ -0,0 +1,262 @@ +import sys +import types +import unittest +from _typeshed import ExcInfo +from collections.abc import Callable +from typing import Any, NamedTuple, type_check_only +from typing_extensions import Self, TypeAlias + +__all__ = [ + "register_optionflag", + "DONT_ACCEPT_TRUE_FOR_1", + "DONT_ACCEPT_BLANKLINE", + "NORMALIZE_WHITESPACE", + "ELLIPSIS", + "SKIP", + "IGNORE_EXCEPTION_DETAIL", + "COMPARISON_FLAGS", + "REPORT_UDIFF", + "REPORT_CDIFF", + "REPORT_NDIFF", + "REPORT_ONLY_FIRST_FAILURE", + "REPORTING_FLAGS", + "FAIL_FAST", + "Example", + "DocTest", + "DocTestParser", + "DocTestFinder", + "DocTestRunner", + "OutputChecker", + "DocTestFailure", + "UnexpectedException", + "DebugRunner", + "testmod", + "testfile", + "run_docstring_examples", + "DocTestSuite", + "DocFileSuite", + "set_unittest_reportflags", + "script_from_examples", + "testsource", + "debug_src", + "debug", +] + +if sys.version_info >= (3, 13): + @type_check_only + class _TestResultsBase(NamedTuple): + failed: int + attempted: int + + class TestResults(_TestResultsBase): + def __new__(cls, failed: int, attempted: int, *, skipped: int = 0) -> Self: ... + skipped: int + +else: + class TestResults(NamedTuple): + failed: int + attempted: int + +OPTIONFLAGS_BY_NAME: dict[str, int] + +def register_optionflag(name: str) -> int: ... + +DONT_ACCEPT_TRUE_FOR_1: int +DONT_ACCEPT_BLANKLINE: int +NORMALIZE_WHITESPACE: int +ELLIPSIS: int +SKIP: int +IGNORE_EXCEPTION_DETAIL: int + +COMPARISON_FLAGS: int + +REPORT_UDIFF: int +REPORT_CDIFF: int +REPORT_NDIFF: int +REPORT_ONLY_FIRST_FAILURE: int +FAIL_FAST: int + +REPORTING_FLAGS: int + +BLANKLINE_MARKER: str +ELLIPSIS_MARKER: str + +class Example: + source: str + want: str + exc_msg: str | None + lineno: int + indent: int + options: dict[int, bool] + def __init__( + self, + source: str, + want: str, + exc_msg: str | None = None, + lineno: int = 0, + indent: int = 0, + options: dict[int, bool] | None = None, + ) -> None: ... + def __hash__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + +class DocTest: + examples: list[Example] + globs: dict[str, Any] + name: str + filename: str | None + lineno: int | None + docstring: str | None + def __init__( + self, + examples: list[Example], + globs: dict[str, Any], + name: str, + filename: str | None, + lineno: int | None, + docstring: str | None, + ) -> None: ... + def __hash__(self) -> int: ... + def __lt__(self, other: DocTest) -> bool: ... + def __eq__(self, other: object) -> bool: ... + +class DocTestParser: + def parse(self, string: str, name: str = "") -> list[str | Example]: ... + def get_doctest(self, string: str, globs: dict[str, Any], name: str, filename: str | None, lineno: int | None) -> DocTest: ... + def get_examples(self, string: str, name: str = "") -> list[Example]: ... + +class DocTestFinder: + def __init__( + self, verbose: bool = False, parser: DocTestParser = ..., recurse: bool = True, exclude_empty: bool = True + ) -> None: ... + def find( + self, + obj: object, + name: str | None = None, + module: None | bool | types.ModuleType = None, + globs: dict[str, Any] | None = None, + extraglobs: dict[str, Any] | None = None, + ) -> list[DocTest]: ... + +_Out: TypeAlias = Callable[[str], object] + +class DocTestRunner: + DIVIDER: str + optionflags: int + original_optionflags: int + tries: int + failures: int + if sys.version_info >= (3, 13): + skips: int + test: DocTest + def __init__(self, checker: OutputChecker | None = None, verbose: bool | None = None, optionflags: int = 0) -> None: ... + def report_start(self, out: _Out, test: DocTest, example: Example) -> None: ... + def report_success(self, out: _Out, test: DocTest, example: Example, got: str) -> None: ... + def report_failure(self, out: _Out, test: DocTest, example: Example, got: str) -> None: ... + def report_unexpected_exception(self, out: _Out, test: DocTest, example: Example, exc_info: ExcInfo) -> None: ... + def run( + self, test: DocTest, compileflags: int | None = None, out: _Out | None = None, clear_globs: bool = True + ) -> TestResults: ... + def summarize(self, verbose: bool | None = None) -> TestResults: ... + def merge(self, other: DocTestRunner) -> None: ... + +class OutputChecker: + def check_output(self, want: str, got: str, optionflags: int) -> bool: ... + def output_difference(self, example: Example, got: str, optionflags: int) -> str: ... + +class DocTestFailure(Exception): + test: DocTest + example: Example + got: str + def __init__(self, test: DocTest, example: Example, got: str) -> None: ... + +class UnexpectedException(Exception): + test: DocTest + example: Example + exc_info: ExcInfo + def __init__(self, test: DocTest, example: Example, exc_info: ExcInfo) -> None: ... + +class DebugRunner(DocTestRunner): ... + +master: DocTestRunner | None + +def testmod( + m: types.ModuleType | None = None, + name: str | None = None, + globs: dict[str, Any] | None = None, + verbose: bool | None = None, + report: bool = True, + optionflags: int = 0, + extraglobs: dict[str, Any] | None = None, + raise_on_error: bool = False, + exclude_empty: bool = False, +) -> TestResults: ... +def testfile( + filename: str, + module_relative: bool = True, + name: str | None = None, + package: None | str | types.ModuleType = None, + globs: dict[str, Any] | None = None, + verbose: bool | None = None, + report: bool = True, + optionflags: int = 0, + extraglobs: dict[str, Any] | None = None, + raise_on_error: bool = False, + parser: DocTestParser = ..., + encoding: str | None = None, +) -> TestResults: ... +def run_docstring_examples( + f: object, + globs: dict[str, Any], + verbose: bool = False, + name: str = "NoName", + compileflags: int | None = None, + optionflags: int = 0, +) -> None: ... +def set_unittest_reportflags(flags: int) -> int: ... + +class DocTestCase(unittest.TestCase): + def __init__( + self, + test: DocTest, + optionflags: int = 0, + setUp: Callable[[DocTest], object] | None = None, + tearDown: Callable[[DocTest], object] | None = None, + checker: OutputChecker | None = None, + ) -> None: ... + def runTest(self) -> None: ... + def format_failure(self, err: str) -> str: ... + def __hash__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + +class SkipDocTestCase(DocTestCase): + def __init__(self, module: types.ModuleType) -> None: ... + def test_skip(self) -> None: ... + +class _DocTestSuite(unittest.TestSuite): ... + +def DocTestSuite( + module: None | str | types.ModuleType = None, + globs: dict[str, Any] | None = None, + extraglobs: dict[str, Any] | None = None, + test_finder: DocTestFinder | None = None, + **options: Any, +) -> _DocTestSuite: ... + +class DocFileCase(DocTestCase): ... + +def DocFileTest( + path: str, + module_relative: bool = True, + package: None | str | types.ModuleType = None, + globs: dict[str, Any] | None = None, + parser: DocTestParser = ..., + encoding: str | None = None, + **options: Any, +) -> DocFileCase: ... +def DocFileSuite(*paths: str, **kw: Any) -> _DocTestSuite: ... +def script_from_examples(s: str) -> str: ... +def testsource(module: None | str | types.ModuleType, name: str) -> str: ... +def debug_src(src: str, pm: bool = False, globs: dict[str, Any] | None = None) -> None: ... +def debug_script(src: str, pm: bool = False, globs: dict[str, Any] | None = None) -> None: ... +def debug(module: None | str | types.ModuleType, name: str, pm: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/dummy_threading.pyi b/mypy/typeshed/stdlib/dummy_threading.pyi new file mode 100644 index 000000000000..757cb8d4bd4c --- /dev/null +++ b/mypy/typeshed/stdlib/dummy_threading.pyi @@ -0,0 +1,2 @@ +from _dummy_threading import * +from _dummy_threading import __all__ as __all__ diff --git a/mypy/typeshed/stdlib/email/__init__.pyi b/mypy/typeshed/stdlib/email/__init__.pyi new file mode 100644 index 000000000000..f564ced105bd --- /dev/null +++ b/mypy/typeshed/stdlib/email/__init__.pyi @@ -0,0 +1,37 @@ +from collections.abc import Callable +from email.message import Message +from email.policy import Policy +from typing import IO +from typing_extensions import TypeAlias + +# At runtime, listing submodules in __all__ without them being imported is +# valid, and causes them to be included in a star import. See #6523 + +__all__ = [ # noqa: F822 # Undefined names in __all__ + "base64mime", # pyright: ignore[reportUnsupportedDunderAll] + "charset", # pyright: ignore[reportUnsupportedDunderAll] + "encoders", # pyright: ignore[reportUnsupportedDunderAll] + "errors", # pyright: ignore[reportUnsupportedDunderAll] + "feedparser", # pyright: ignore[reportUnsupportedDunderAll] + "generator", # pyright: ignore[reportUnsupportedDunderAll] + "header", # pyright: ignore[reportUnsupportedDunderAll] + "iterators", # pyright: ignore[reportUnsupportedDunderAll] + "message", # pyright: ignore[reportUnsupportedDunderAll] + "message_from_file", + "message_from_binary_file", + "message_from_string", + "message_from_bytes", + "mime", # pyright: ignore[reportUnsupportedDunderAll] + "parser", # pyright: ignore[reportUnsupportedDunderAll] + "quoprimime", # pyright: ignore[reportUnsupportedDunderAll] + "utils", # pyright: ignore[reportUnsupportedDunderAll] +] + +# Definitions imported by multiple submodules in typeshed +_ParamType: TypeAlias = str | tuple[str | None, str | None, str] # noqa: Y047 +_ParamsType: TypeAlias = str | None | tuple[str, str | None, str] # noqa: Y047 + +def message_from_string(s: str, _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ... +def message_from_bytes(s: bytes | bytearray, _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ... +def message_from_file(fp: IO[str], _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ... +def message_from_binary_file(fp: IO[bytes], _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ... diff --git a/mypy/typeshed/stdlib/email/_header_value_parser.pyi b/mypy/typeshed/stdlib/email/_header_value_parser.pyi new file mode 100644 index 000000000000..ff405a8b61d2 --- /dev/null +++ b/mypy/typeshed/stdlib/email/_header_value_parser.pyi @@ -0,0 +1,396 @@ +from collections.abc import Iterable, Iterator +from email.errors import HeaderParseError, MessageDefect +from email.policy import Policy +from re import Pattern +from typing import Any, Final +from typing_extensions import Self + +WSP: Final[set[str]] +CFWS_LEADER: Final[set[str]] +SPECIALS: Final[set[str]] +ATOM_ENDS: Final[set[str]] +DOT_ATOM_ENDS: Final[set[str]] +PHRASE_ENDS: Final[set[str]] +TSPECIALS: Final[set[str]] +TOKEN_ENDS: Final[set[str]] +ASPECIALS: Final[set[str]] +ATTRIBUTE_ENDS: Final[set[str]] +EXTENDED_ATTRIBUTE_ENDS: Final[set[str]] +# Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +NLSET: Final[set[str]] +# Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +SPECIALSNL: Final[set[str]] + +def quote_string(value: Any) -> str: ... + +rfc2047_matcher: Pattern[str] + +class TokenList(list[TokenList | Terminal]): + token_type: str | None + syntactic_break: bool + ew_combine_allowed: bool + defects: list[MessageDefect] + def __init__(self, *args: Any, **kw: Any) -> None: ... + @property + def value(self) -> str: ... + @property + def all_defects(self) -> list[MessageDefect]: ... + def startswith_fws(self) -> bool: ... + @property + def as_ew_allowed(self) -> bool: ... + @property + def comments(self) -> list[str]: ... + def fold(self, *, policy: Policy) -> str: ... + def pprint(self, indent: str = "") -> None: ... + def ppstr(self, indent: str = "") -> str: ... + +class WhiteSpaceTokenList(TokenList): ... + +class UnstructuredTokenList(TokenList): + token_type: str + +class Phrase(TokenList): + token_type: str + +class Word(TokenList): + token_type: str + +class CFWSList(WhiteSpaceTokenList): + token_type: str + +class Atom(TokenList): + token_type: str + +class Token(TokenList): + token_type: str + encode_as_ew: bool + +class EncodedWord(TokenList): + token_type: str + cte: str | None + charset: str | None + lang: str | None + +class QuotedString(TokenList): + token_type: str + @property + def content(self) -> str: ... + @property + def quoted_value(self) -> str: ... + @property + def stripped_value(self) -> str: ... + +class BareQuotedString(QuotedString): + token_type: str + +class Comment(WhiteSpaceTokenList): + token_type: str + def quote(self, value: Any) -> str: ... + @property + def content(self) -> str: ... + +class AddressList(TokenList): + token_type: str + @property + def addresses(self) -> list[Address]: ... + @property + def mailboxes(self) -> list[Mailbox]: ... + @property + def all_mailboxes(self) -> list[Mailbox]: ... + +class Address(TokenList): + token_type: str + @property + def display_name(self) -> str: ... + @property + def mailboxes(self) -> list[Mailbox]: ... + @property + def all_mailboxes(self) -> list[Mailbox]: ... + +class MailboxList(TokenList): + token_type: str + @property + def mailboxes(self) -> list[Mailbox]: ... + @property + def all_mailboxes(self) -> list[Mailbox]: ... + +class GroupList(TokenList): + token_type: str + @property + def mailboxes(self) -> list[Mailbox]: ... + @property + def all_mailboxes(self) -> list[Mailbox]: ... + +class Group(TokenList): + token_type: str + @property + def mailboxes(self) -> list[Mailbox]: ... + @property + def all_mailboxes(self) -> list[Mailbox]: ... + @property + def display_name(self) -> str: ... + +class NameAddr(TokenList): + token_type: str + @property + def display_name(self) -> str: ... + @property + def local_part(self) -> str: ... + @property + def domain(self) -> str: ... + @property + def route(self) -> list[Domain] | None: ... + @property + def addr_spec(self) -> str: ... + +class AngleAddr(TokenList): + token_type: str + @property + def local_part(self) -> str: ... + @property + def domain(self) -> str: ... + @property + def route(self) -> list[Domain] | None: ... + @property + def addr_spec(self) -> str: ... + +class ObsRoute(TokenList): + token_type: str + @property + def domains(self) -> list[Domain]: ... + +class Mailbox(TokenList): + token_type: str + @property + def display_name(self) -> str: ... + @property + def local_part(self) -> str: ... + @property + def domain(self) -> str: ... + @property + def route(self) -> list[str]: ... + @property + def addr_spec(self) -> str: ... + +class InvalidMailbox(TokenList): + token_type: str + @property + def display_name(self) -> None: ... + @property + def local_part(self) -> None: ... + @property + def domain(self) -> None: ... + @property + def route(self) -> None: ... + @property + def addr_spec(self) -> None: ... + +class Domain(TokenList): + token_type: str + as_ew_allowed: bool + @property + def domain(self) -> str: ... + +class DotAtom(TokenList): + token_type: str + +class DotAtomText(TokenList): + token_type: str + as_ew_allowed: bool + +class NoFoldLiteral(TokenList): + token_type: str + as_ew_allowed: bool + +class AddrSpec(TokenList): + token_type: str + as_ew_allowed: bool + @property + def local_part(self) -> str: ... + @property + def domain(self) -> str: ... + @property + def addr_spec(self) -> str: ... + +class ObsLocalPart(TokenList): + token_type: str + as_ew_allowed: bool + +class DisplayName(Phrase): + token_type: str + @property + def display_name(self) -> str: ... + +class LocalPart(TokenList): + token_type: str + as_ew_allowed: bool + @property + def local_part(self) -> str: ... + +class DomainLiteral(TokenList): + token_type: str + as_ew_allowed: bool + @property + def domain(self) -> str: ... + @property + def ip(self) -> str: ... + +class MIMEVersion(TokenList): + token_type: str + major: int | None + minor: int | None + +class Parameter(TokenList): + token_type: str + sectioned: bool + extended: bool + charset: str + @property + def section_number(self) -> int: ... + @property + def param_value(self) -> str: ... + +class InvalidParameter(Parameter): + token_type: str + +class Attribute(TokenList): + token_type: str + @property + def stripped_value(self) -> str: ... + +class Section(TokenList): + token_type: str + number: int | None + +class Value(TokenList): + token_type: str + @property + def stripped_value(self) -> str: ... + +class MimeParameters(TokenList): + token_type: str + syntactic_break: bool + @property + def params(self) -> Iterator[tuple[str, str]]: ... + +class ParameterizedHeaderValue(TokenList): + syntactic_break: bool + @property + def params(self) -> Iterable[tuple[str, str]]: ... + +class ContentType(ParameterizedHeaderValue): + token_type: str + as_ew_allowed: bool + maintype: str + subtype: str + +class ContentDisposition(ParameterizedHeaderValue): + token_type: str + as_ew_allowed: bool + content_disposition: Any + +class ContentTransferEncoding(TokenList): + token_type: str + as_ew_allowed: bool + cte: str + +class HeaderLabel(TokenList): + token_type: str + as_ew_allowed: bool + +class MsgID(TokenList): + token_type: str + as_ew_allowed: bool + def fold(self, policy: Policy) -> str: ... + +class MessageID(MsgID): + token_type: str + +class InvalidMessageID(MessageID): + token_type: str + +class Header(TokenList): + token_type: str + +class Terminal(str): + as_ew_allowed: bool + ew_combine_allowed: bool + syntactic_break: bool + token_type: str + defects: list[MessageDefect] + def __new__(cls, value: str, token_type: str) -> Self: ... + def pprint(self) -> None: ... + @property + def all_defects(self) -> list[MessageDefect]: ... + def pop_trailing_ws(self) -> None: ... + @property + def comments(self) -> list[str]: ... + def __getnewargs__(self) -> tuple[str, str]: ... # type: ignore[override] + +class WhiteSpaceTerminal(Terminal): + @property + def value(self) -> str: ... + def startswith_fws(self) -> bool: ... + +class ValueTerminal(Terminal): + @property + def value(self) -> ValueTerminal: ... + def startswith_fws(self) -> bool: ... + +class EWWhiteSpaceTerminal(WhiteSpaceTerminal): ... +class _InvalidEwError(HeaderParseError): ... + +DOT: Final[ValueTerminal] +ListSeparator: Final[ValueTerminal] +RouteComponentMarker: Final[ValueTerminal] + +def get_fws(value: str) -> tuple[WhiteSpaceTerminal, str]: ... +def get_encoded_word(value: str) -> tuple[EncodedWord, str]: ... +def get_unstructured(value: str) -> UnstructuredTokenList: ... +def get_qp_ctext(value: str) -> tuple[WhiteSpaceTerminal, str]: ... +def get_qcontent(value: str) -> tuple[ValueTerminal, str]: ... +def get_atext(value: str) -> tuple[ValueTerminal, str]: ... +def get_bare_quoted_string(value: str) -> tuple[BareQuotedString, str]: ... +def get_comment(value: str) -> tuple[Comment, str]: ... +def get_cfws(value: str) -> tuple[CFWSList, str]: ... +def get_quoted_string(value: str) -> tuple[QuotedString, str]: ... +def get_atom(value: str) -> tuple[Atom, str]: ... +def get_dot_atom_text(value: str) -> tuple[DotAtomText, str]: ... +def get_dot_atom(value: str) -> tuple[DotAtom, str]: ... +def get_word(value: str) -> tuple[Any, str]: ... +def get_phrase(value: str) -> tuple[Phrase, str]: ... +def get_local_part(value: str) -> tuple[LocalPart, str]: ... +def get_obs_local_part(value: str) -> tuple[ObsLocalPart, str]: ... +def get_dtext(value: str) -> tuple[ValueTerminal, str]: ... +def get_domain_literal(value: str) -> tuple[DomainLiteral, str]: ... +def get_domain(value: str) -> tuple[Domain, str]: ... +def get_addr_spec(value: str) -> tuple[AddrSpec, str]: ... +def get_obs_route(value: str) -> tuple[ObsRoute, str]: ... +def get_angle_addr(value: str) -> tuple[AngleAddr, str]: ... +def get_display_name(value: str) -> tuple[DisplayName, str]: ... +def get_name_addr(value: str) -> tuple[NameAddr, str]: ... +def get_mailbox(value: str) -> tuple[Mailbox, str]: ... +def get_invalid_mailbox(value: str, endchars: str) -> tuple[InvalidMailbox, str]: ... +def get_mailbox_list(value: str) -> tuple[MailboxList, str]: ... +def get_group_list(value: str) -> tuple[GroupList, str]: ... +def get_group(value: str) -> tuple[Group, str]: ... +def get_address(value: str) -> tuple[Address, str]: ... +def get_address_list(value: str) -> tuple[AddressList, str]: ... +def get_no_fold_literal(value: str) -> tuple[NoFoldLiteral, str]: ... +def get_msg_id(value: str) -> tuple[MsgID, str]: ... +def parse_message_id(value: str) -> MessageID: ... +def parse_mime_version(value: str) -> MIMEVersion: ... +def get_invalid_parameter(value: str) -> tuple[InvalidParameter, str]: ... +def get_ttext(value: str) -> tuple[ValueTerminal, str]: ... +def get_token(value: str) -> tuple[Token, str]: ... +def get_attrtext(value: str) -> tuple[ValueTerminal, str]: ... +def get_attribute(value: str) -> tuple[Attribute, str]: ... +def get_extended_attrtext(value: str) -> tuple[ValueTerminal, str]: ... +def get_extended_attribute(value: str) -> tuple[Attribute, str]: ... +def get_section(value: str) -> tuple[Section, str]: ... +def get_value(value: str) -> tuple[Value, str]: ... +def get_parameter(value: str) -> tuple[Parameter, str]: ... +def parse_mime_parameters(value: str) -> MimeParameters: ... +def parse_content_type_header(value: str) -> ContentType: ... +def parse_content_disposition_header(value: str) -> ContentDisposition: ... +def parse_content_transfer_encoding_header(value: str) -> ContentTransferEncoding: ... diff --git a/mypy/typeshed/stdlib/email/_policybase.pyi b/mypy/typeshed/stdlib/email/_policybase.pyi new file mode 100644 index 000000000000..f5dbbd96da14 --- /dev/null +++ b/mypy/typeshed/stdlib/email/_policybase.pyi @@ -0,0 +1,77 @@ +from abc import ABCMeta, abstractmethod +from email.errors import MessageDefect +from email.header import Header +from email.message import Message +from typing import Generic, Protocol, TypeVar, type_check_only +from typing_extensions import Self + +__all__ = ["Policy", "Compat32", "compat32"] + +_MessageT = TypeVar("_MessageT", bound=Message, default=Message) + +@type_check_only +class _MessageFactory(Protocol[_MessageT]): + def __call__(self, policy: Policy[_MessageT]) -> _MessageT: ... + +# Policy below is the only known direct subclass of _PolicyBase. We therefore +# assume that the __init__ arguments and attributes of _PolicyBase are +# the same as those of Policy. +class _PolicyBase(Generic[_MessageT]): + max_line_length: int | None + linesep: str + cte_type: str + raise_on_defect: bool + mangle_from_: bool + message_factory: _MessageFactory[_MessageT] | None + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool + + def __init__( + self, + *, + max_line_length: int | None = 78, + linesep: str = "\n", + cte_type: str = "8bit", + raise_on_defect: bool = False, + mangle_from_: bool = ..., # default depends on sub-class + message_factory: _MessageFactory[_MessageT] | None = None, + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool = True, + ) -> None: ... + def clone( + self, + *, + max_line_length: int | None = ..., + linesep: str = ..., + cte_type: str = ..., + raise_on_defect: bool = ..., + mangle_from_: bool = ..., + message_factory: _MessageFactory[_MessageT] | None = ..., + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool = ..., + ) -> Self: ... + def __add__(self, other: Policy) -> Self: ... + +class Policy(_PolicyBase[_MessageT], metaclass=ABCMeta): + def handle_defect(self, obj: _MessageT, defect: MessageDefect) -> None: ... + def register_defect(self, obj: _MessageT, defect: MessageDefect) -> None: ... + def header_max_count(self, name: str) -> int | None: ... + @abstractmethod + def header_source_parse(self, sourcelines: list[str]) -> tuple[str, str]: ... + @abstractmethod + def header_store_parse(self, name: str, value: str) -> tuple[str, str]: ... + @abstractmethod + def header_fetch_parse(self, name: str, value: str) -> str: ... + @abstractmethod + def fold(self, name: str, value: str) -> str: ... + @abstractmethod + def fold_binary(self, name: str, value: str) -> bytes: ... + +class Compat32(Policy[_MessageT]): + def header_source_parse(self, sourcelines: list[str]) -> tuple[str, str]: ... + def header_store_parse(self, name: str, value: str) -> tuple[str, str]: ... + def header_fetch_parse(self, name: str, value: str) -> str | Header: ... # type: ignore[override] + def fold(self, name: str, value: str) -> str: ... + def fold_binary(self, name: str, value: str) -> bytes: ... + +compat32: Compat32[Message] diff --git a/mypy/typeshed/stdlib/email/base64mime.pyi b/mypy/typeshed/stdlib/email/base64mime.pyi new file mode 100644 index 000000000000..563cd7f669a2 --- /dev/null +++ b/mypy/typeshed/stdlib/email/base64mime.pyi @@ -0,0 +1,13 @@ +__all__ = ["body_decode", "body_encode", "decode", "decodestring", "header_encode", "header_length"] + +from _typeshed import ReadableBuffer + +def header_length(bytearray: str | bytes | bytearray) -> int: ... +def header_encode(header_bytes: str | ReadableBuffer, charset: str = "iso-8859-1") -> str: ... + +# First argument should be a buffer that supports slicing and len(). +def body_encode(s: bytes | bytearray, maxlinelen: int = 76, eol: str = "\n") -> str: ... +def decode(string: str | ReadableBuffer) -> bytes: ... + +body_decode = decode +decodestring = decode diff --git a/mypy/typeshed/stdlib/email/charset.pyi b/mypy/typeshed/stdlib/email/charset.pyi new file mode 100644 index 000000000000..2939192c9526 --- /dev/null +++ b/mypy/typeshed/stdlib/email/charset.pyi @@ -0,0 +1,34 @@ +from collections.abc import Callable, Iterator +from email.message import Message +from typing import Final, overload + +__all__ = ["Charset", "add_alias", "add_charset", "add_codec"] + +QP: Final[int] # undocumented +BASE64: Final[int] # undocumented +SHORTEST: Final[int] # undocumented + +class Charset: + input_charset: str + header_encoding: int + body_encoding: int + output_charset: str | None + input_codec: str | None + output_codec: str | None + def __init__(self, input_charset: str = "us-ascii") -> None: ... + def get_body_encoding(self) -> str | Callable[[Message], None]: ... + def get_output_charset(self) -> str | None: ... + def header_encode(self, string: str) -> str: ... + def header_encode_lines(self, string: str, maxlengths: Iterator[int]) -> list[str | None]: ... + @overload + def body_encode(self, string: None) -> None: ... + @overload + def body_encode(self, string: str | bytes) -> str: ... + def __eq__(self, other: object) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + +def add_charset( + charset: str, header_enc: int | None = None, body_enc: int | None = None, output_charset: str | None = None +) -> None: ... +def add_alias(alias: str, canonical: str) -> None: ... +def add_codec(charset: str, codecname: str) -> None: ... diff --git a/mypy/typeshed/stdlib/email/contentmanager.pyi b/mypy/typeshed/stdlib/email/contentmanager.pyi new file mode 100644 index 000000000000..3214f1a4781d --- /dev/null +++ b/mypy/typeshed/stdlib/email/contentmanager.pyi @@ -0,0 +1,11 @@ +from collections.abc import Callable +from email.message import Message +from typing import Any + +class ContentManager: + def get_content(self, msg: Message, *args: Any, **kw: Any) -> Any: ... + def set_content(self, msg: Message, obj: Any, *args: Any, **kw: Any) -> Any: ... + def add_get_handler(self, key: str, handler: Callable[..., Any]) -> None: ... + def add_set_handler(self, typekey: type, handler: Callable[..., Any]) -> None: ... + +raw_data_manager: ContentManager diff --git a/mypy/typeshed/stdlib/email/encoders.pyi b/mypy/typeshed/stdlib/email/encoders.pyi new file mode 100644 index 000000000000..55223bdc0762 --- /dev/null +++ b/mypy/typeshed/stdlib/email/encoders.pyi @@ -0,0 +1,8 @@ +from email.message import Message + +__all__ = ["encode_7or8bit", "encode_base64", "encode_noop", "encode_quopri"] + +def encode_base64(msg: Message) -> None: ... +def encode_quopri(msg: Message) -> None: ... +def encode_7or8bit(msg: Message) -> None: ... +def encode_noop(msg: Message) -> None: ... diff --git a/mypy/typeshed/stdlib/email/errors.pyi b/mypy/typeshed/stdlib/email/errors.pyi new file mode 100644 index 000000000000..f105576c5ee4 --- /dev/null +++ b/mypy/typeshed/stdlib/email/errors.pyi @@ -0,0 +1,42 @@ +import sys + +class MessageError(Exception): ... +class MessageParseError(MessageError): ... +class HeaderParseError(MessageParseError): ... +class BoundaryError(MessageParseError): ... +class MultipartConversionError(MessageError, TypeError): ... +class CharsetError(MessageError): ... + +# Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +class HeaderWriteError(MessageError): ... + +class MessageDefect(ValueError): + def __init__(self, line: str | None = None) -> None: ... + +class NoBoundaryInMultipartDefect(MessageDefect): ... +class StartBoundaryNotFoundDefect(MessageDefect): ... +class FirstHeaderLineIsContinuationDefect(MessageDefect): ... +class MisplacedEnvelopeHeaderDefect(MessageDefect): ... +class MultipartInvariantViolationDefect(MessageDefect): ... +class InvalidMultipartContentTransferEncodingDefect(MessageDefect): ... +class UndecodableBytesDefect(MessageDefect): ... +class InvalidBase64PaddingDefect(MessageDefect): ... +class InvalidBase64CharactersDefect(MessageDefect): ... +class InvalidBase64LengthDefect(MessageDefect): ... +class CloseBoundaryNotFoundDefect(MessageDefect): ... +class MissingHeaderBodySeparatorDefect(MessageDefect): ... + +MalformedHeaderDefect = MissingHeaderBodySeparatorDefect + +class HeaderDefect(MessageDefect): ... +class InvalidHeaderDefect(HeaderDefect): ... +class HeaderMissingRequiredValue(HeaderDefect): ... + +class NonPrintableDefect(HeaderDefect): + def __init__(self, non_printables: str | None) -> None: ... + +class ObsoleteHeaderDefect(HeaderDefect): ... +class NonASCIILocalPartDefect(HeaderDefect): ... + +if sys.version_info >= (3, 10): + class InvalidDateDefect(HeaderDefect): ... diff --git a/mypy/typeshed/stdlib/email/feedparser.pyi b/mypy/typeshed/stdlib/email/feedparser.pyi new file mode 100644 index 000000000000..8c268ca1ae18 --- /dev/null +++ b/mypy/typeshed/stdlib/email/feedparser.pyi @@ -0,0 +1,23 @@ +from collections.abc import Callable +from email.message import Message +from email.policy import Policy +from typing import Generic, TypeVar, overload + +__all__ = ["FeedParser", "BytesFeedParser"] + +_MessageT = TypeVar("_MessageT", bound=Message, default=Message) + +class FeedParser(Generic[_MessageT]): + @overload + def __init__(self: FeedParser[Message], _factory: None = None, *, policy: Policy[Message] = ...) -> None: ... + @overload + def __init__(self, _factory: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... + def feed(self, data: str) -> None: ... + def close(self) -> _MessageT: ... + +class BytesFeedParser(FeedParser[_MessageT]): + @overload + def __init__(self: BytesFeedParser[Message], _factory: None = None, *, policy: Policy[Message] = ...) -> None: ... + @overload + def __init__(self, _factory: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... + def feed(self, data: bytes | bytearray) -> None: ... # type: ignore[override] diff --git a/mypy/typeshed/stdlib/email/generator.pyi b/mypy/typeshed/stdlib/email/generator.pyi new file mode 100644 index 000000000000..dfa0604a20a9 --- /dev/null +++ b/mypy/typeshed/stdlib/email/generator.pyi @@ -0,0 +1,77 @@ +from _typeshed import SupportsWrite +from email.message import Message +from email.policy import Policy +from typing import Any, Generic, TypeVar, overload +from typing_extensions import Self + +__all__ = ["Generator", "DecodedGenerator", "BytesGenerator"] + +# By default, generators do not have a message policy. +_MessageT = TypeVar("_MessageT", bound=Message, default=Any) + +class Generator(Generic[_MessageT]): + maxheaderlen: int | None + policy: Policy[_MessageT] | None + @overload + def __init__( + self: Generator[Any], # The Policy of the message is used. + outfp: SupportsWrite[str], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + *, + policy: None = None, + ) -> None: ... + @overload + def __init__( + self, + outfp: SupportsWrite[str], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + *, + policy: Policy[_MessageT], + ) -> None: ... + def write(self, s: str) -> None: ... + def flatten(self, msg: _MessageT, unixfrom: bool = False, linesep: str | None = None) -> None: ... + def clone(self, fp: SupportsWrite[str]) -> Self: ... + +class BytesGenerator(Generator[_MessageT]): + @overload + def __init__( + self: BytesGenerator[Any], # The Policy of the message is used. + outfp: SupportsWrite[bytes], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + *, + policy: None = None, + ) -> None: ... + @overload + def __init__( + self, + outfp: SupportsWrite[bytes], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + *, + policy: Policy[_MessageT], + ) -> None: ... + +class DecodedGenerator(Generator[_MessageT]): + @overload + def __init__( + self: DecodedGenerator[Any], # The Policy of the message is used. + outfp: SupportsWrite[str], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + fmt: str | None = None, + *, + policy: None = None, + ) -> None: ... + @overload + def __init__( + self, + outfp: SupportsWrite[str], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + fmt: str | None = None, + *, + policy: Policy[_MessageT], + ) -> None: ... diff --git a/mypy/typeshed/stdlib/email/header.pyi b/mypy/typeshed/stdlib/email/header.pyi new file mode 100644 index 000000000000..212132c6be18 --- /dev/null +++ b/mypy/typeshed/stdlib/email/header.pyi @@ -0,0 +1,31 @@ +from collections.abc import Iterable +from email.charset import Charset +from typing import Any + +__all__ = ["Header", "decode_header", "make_header"] + +class Header: + def __init__( + self, + s: bytes | bytearray | str | None = None, + charset: Charset | str | None = None, + maxlinelen: int | None = None, + header_name: str | None = None, + continuation_ws: str = " ", + errors: str = "strict", + ) -> None: ... + def append(self, s: bytes | bytearray | str, charset: Charset | str | None = None, errors: str = "strict") -> None: ... + def encode(self, splitchars: str = ";, \t", maxlinelen: int | None = None, linesep: str = "\n") -> str: ... + def __eq__(self, other: object) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + +# decode_header() either returns list[tuple[str, None]] if the header +# contains no encoded parts, or list[tuple[bytes, str | None]] if the header +# contains at least one encoded part. +def decode_header(header: Header | str) -> list[tuple[Any, Any | None]]: ... +def make_header( + decoded_seq: Iterable[tuple[bytes | bytearray | str, str | None]], + maxlinelen: int | None = None, + header_name: str | None = None, + continuation_ws: str = " ", +) -> Header: ... diff --git a/mypy/typeshed/stdlib/email/headerregistry.pyi b/mypy/typeshed/stdlib/email/headerregistry.pyi new file mode 100644 index 000000000000..2ffdca9b2f22 --- /dev/null +++ b/mypy/typeshed/stdlib/email/headerregistry.pyi @@ -0,0 +1,178 @@ +import types +from collections.abc import Iterable, Mapping +from datetime import datetime as _datetime +from email._header_value_parser import ( + AddressList, + ContentDisposition, + ContentTransferEncoding, + ContentType, + MessageID, + MIMEVersion, + TokenList, + UnstructuredTokenList, +) +from email.errors import MessageDefect +from email.policy import Policy +from typing import Any, ClassVar, Literal, Protocol +from typing_extensions import Self + +class BaseHeader(str): + # max_count is actually more of an abstract ClassVar (not defined on the base class, but expected to be defined in subclasses) + max_count: ClassVar[Literal[1] | None] + @property + def name(self) -> str: ... + @property + def defects(self) -> tuple[MessageDefect, ...]: ... + def __new__(cls, name: str, value: Any) -> Self: ... + def init(self, name: str, *, parse_tree: TokenList, defects: Iterable[MessageDefect]) -> None: ... + def fold(self, *, policy: Policy) -> str: ... + +class UnstructuredHeader: + max_count: ClassVar[Literal[1] | None] + @staticmethod + def value_parser(value: str) -> UnstructuredTokenList: ... + @classmethod + def parse(cls, value: str, kwds: dict[str, Any]) -> None: ... + +class UniqueUnstructuredHeader(UnstructuredHeader): + max_count: ClassVar[Literal[1]] + +class DateHeader: + max_count: ClassVar[Literal[1] | None] + def init(self, name: str, *, parse_tree: TokenList, defects: Iterable[MessageDefect], datetime: _datetime) -> None: ... + @property + def datetime(self) -> _datetime: ... + @staticmethod + def value_parser(value: str) -> UnstructuredTokenList: ... + @classmethod + def parse(cls, value: str | _datetime, kwds: dict[str, Any]) -> None: ... + +class UniqueDateHeader(DateHeader): + max_count: ClassVar[Literal[1]] + +class AddressHeader: + max_count: ClassVar[Literal[1] | None] + def init(self, name: str, *, parse_tree: TokenList, defects: Iterable[MessageDefect], groups: Iterable[Group]) -> None: ... + @property + def groups(self) -> tuple[Group, ...]: ... + @property + def addresses(self) -> tuple[Address, ...]: ... + @staticmethod + def value_parser(value: str) -> AddressList: ... + @classmethod + def parse(cls, value: str, kwds: dict[str, Any]) -> None: ... + +class UniqueAddressHeader(AddressHeader): + max_count: ClassVar[Literal[1]] + +class SingleAddressHeader(AddressHeader): + @property + def address(self) -> Address: ... + +class UniqueSingleAddressHeader(SingleAddressHeader): + max_count: ClassVar[Literal[1]] + +class MIMEVersionHeader: + max_count: ClassVar[Literal[1]] + def init( + self, + name: str, + *, + parse_tree: TokenList, + defects: Iterable[MessageDefect], + version: str | None, + major: int | None, + minor: int | None, + ) -> None: ... + @property + def version(self) -> str | None: ... + @property + def major(self) -> int | None: ... + @property + def minor(self) -> int | None: ... + @staticmethod + def value_parser(value: str) -> MIMEVersion: ... + @classmethod + def parse(cls, value: str, kwds: dict[str, Any]) -> None: ... + +class ParameterizedMIMEHeader: + max_count: ClassVar[Literal[1]] + def init(self, name: str, *, parse_tree: TokenList, defects: Iterable[MessageDefect], params: Mapping[str, Any]) -> None: ... + @property + def params(self) -> types.MappingProxyType[str, Any]: ... + @classmethod + def parse(cls, value: str, kwds: dict[str, Any]) -> None: ... + +class ContentTypeHeader(ParameterizedMIMEHeader): + @property + def content_type(self) -> str: ... + @property + def maintype(self) -> str: ... + @property + def subtype(self) -> str: ... + @staticmethod + def value_parser(value: str) -> ContentType: ... + +class ContentDispositionHeader(ParameterizedMIMEHeader): + # init is redefined but has the same signature as parent class, so is omitted from the stub + @property + def content_disposition(self) -> str | None: ... + @staticmethod + def value_parser(value: str) -> ContentDisposition: ... + +class ContentTransferEncodingHeader: + max_count: ClassVar[Literal[1]] + def init(self, name: str, *, parse_tree: TokenList, defects: Iterable[MessageDefect]) -> None: ... + @property + def cte(self) -> str: ... + @classmethod + def parse(cls, value: str, kwds: dict[str, Any]) -> None: ... + @staticmethod + def value_parser(value: str) -> ContentTransferEncoding: ... + +class MessageIDHeader: + max_count: ClassVar[Literal[1]] + @classmethod + def parse(cls, value: str, kwds: dict[str, Any]) -> None: ... + @staticmethod + def value_parser(value: str) -> MessageID: ... + +class _HeaderParser(Protocol): + max_count: ClassVar[Literal[1] | None] + @staticmethod + def value_parser(value: str, /) -> TokenList: ... + @classmethod + def parse(cls, value: str, kwds: dict[str, Any], /) -> None: ... + +class HeaderRegistry: + registry: dict[str, type[_HeaderParser]] + base_class: type[BaseHeader] + default_class: type[_HeaderParser] + def __init__( + self, base_class: type[BaseHeader] = ..., default_class: type[_HeaderParser] = ..., use_default_map: bool = True + ) -> None: ... + def map_to_type(self, name: str, cls: type[BaseHeader]) -> None: ... + def __getitem__(self, name: str) -> type[BaseHeader]: ... + def __call__(self, name: str, value: Any) -> BaseHeader: ... + +class Address: + @property + def display_name(self) -> str: ... + @property + def username(self) -> str: ... + @property + def domain(self) -> str: ... + @property + def addr_spec(self) -> str: ... + def __init__( + self, display_name: str = "", username: str | None = "", domain: str | None = "", addr_spec: str | None = None + ) -> None: ... + def __eq__(self, other: object) -> bool: ... + +class Group: + @property + def display_name(self) -> str | None: ... + @property + def addresses(self) -> tuple[Address, ...]: ... + def __init__(self, display_name: str | None = None, addresses: Iterable[Address] | None = None) -> None: ... + def __eq__(self, other: object) -> bool: ... diff --git a/mypy/typeshed/stdlib/email/iterators.pyi b/mypy/typeshed/stdlib/email/iterators.pyi new file mode 100644 index 000000000000..d964d6843833 --- /dev/null +++ b/mypy/typeshed/stdlib/email/iterators.pyi @@ -0,0 +1,12 @@ +from _typeshed import SupportsWrite +from collections.abc import Iterator +from email.message import Message + +__all__ = ["body_line_iterator", "typed_subpart_iterator", "walk"] + +def body_line_iterator(msg: Message, decode: bool = False) -> Iterator[str]: ... +def typed_subpart_iterator(msg: Message, maintype: str = "text", subtype: str | None = None) -> Iterator[str]: ... +def walk(self: Message) -> Iterator[Message]: ... + +# We include the seemingly private function because it is documented in the stdlib documentation. +def _structure(msg: Message, fp: SupportsWrite[str] | None = None, level: int = 0, include_default: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/email/message.pyi b/mypy/typeshed/stdlib/email/message.pyi new file mode 100644 index 000000000000..ebad05a1cf7b --- /dev/null +++ b/mypy/typeshed/stdlib/email/message.pyi @@ -0,0 +1,172 @@ +from _typeshed import MaybeNone +from collections.abc import Generator, Iterator, Sequence +from email import _ParamsType, _ParamType +from email.charset import Charset +from email.contentmanager import ContentManager +from email.errors import MessageDefect +from email.policy import Policy +from typing import Any, Generic, Literal, Protocol, TypeVar, overload +from typing_extensions import Self, TypeAlias + +__all__ = ["Message", "EmailMessage"] + +_T = TypeVar("_T") +# Type returned by Policy.header_fetch_parse, often str or Header. +_HeaderT = TypeVar("_HeaderT", default=str) +_HeaderParamT = TypeVar("_HeaderParamT", default=str) +# Represents headers constructed by HeaderRegistry. Those are sub-classes +# of BaseHeader and another header type. +_HeaderRegistryT = TypeVar("_HeaderRegistryT", default=Any) +_HeaderRegistryParamT = TypeVar("_HeaderRegistryParamT", default=Any) + +_PayloadType: TypeAlias = Message | str +_EncodedPayloadType: TypeAlias = Message | bytes +_MultipartPayloadType: TypeAlias = list[_PayloadType] +_CharsetType: TypeAlias = Charset | str | None + +class _SupportsEncodeToPayload(Protocol): + def encode(self, encoding: str, /) -> _PayloadType | _MultipartPayloadType | _SupportsDecodeToPayload: ... + +class _SupportsDecodeToPayload(Protocol): + def decode(self, encoding: str, errors: str, /) -> _PayloadType | _MultipartPayloadType: ... + +class Message(Generic[_HeaderT, _HeaderParamT]): + # The policy attributes and arguments in this class and its subclasses + # would ideally use Policy[Self], but this is not possible. + policy: Policy[Any] # undocumented + preamble: str | None + epilogue: str | None + defects: list[MessageDefect] + def __init__(self, policy: Policy[Any] = ...) -> None: ... + def is_multipart(self) -> bool: ... + def set_unixfrom(self, unixfrom: str) -> None: ... + def get_unixfrom(self) -> str | None: ... + def attach(self, payload: _PayloadType) -> None: ... + # `i: int` without a multipart payload results in an error + # `| MaybeNone` acts like `| Any`: can be None for cleared or unset payload, but annoying to check + @overload # multipart + def get_payload(self, i: int, decode: Literal[True]) -> None: ... + @overload # multipart + def get_payload(self, i: int, decode: Literal[False] = False) -> _PayloadType | MaybeNone: ... + @overload # either + def get_payload(self, i: None = None, decode: Literal[False] = False) -> _PayloadType | _MultipartPayloadType | MaybeNone: ... + @overload # not multipart + def get_payload(self, i: None = None, *, decode: Literal[True]) -> _EncodedPayloadType | MaybeNone: ... + @overload # not multipart, IDEM but w/o kwarg + def get_payload(self, i: None, decode: Literal[True]) -> _EncodedPayloadType | MaybeNone: ... + # If `charset=None` and payload supports both `encode` AND `decode`, + # then an invalid payload could be passed, but this is unlikely + # Not[_SupportsEncodeToPayload] + @overload + def set_payload( + self, payload: _SupportsDecodeToPayload | _PayloadType | _MultipartPayloadType, charset: None = None + ) -> None: ... + @overload + def set_payload( + self, + payload: _SupportsEncodeToPayload | _SupportsDecodeToPayload | _PayloadType | _MultipartPayloadType, + charset: Charset | str, + ) -> None: ... + def set_charset(self, charset: _CharsetType) -> None: ... + def get_charset(self) -> _CharsetType: ... + def __len__(self) -> int: ... + def __contains__(self, name: str) -> bool: ... + def __iter__(self) -> Iterator[str]: ... + # Same as `get` with `failobj=None`, but with the expectation that it won't return None in most scenarios + # This is important for protocols using __getitem__, like SupportsKeysAndGetItem + # Morally, the return type should be `AnyOf[_HeaderType, None]`, + # so using "the Any trick" instead. + def __getitem__(self, name: str) -> _HeaderT | MaybeNone: ... + def __setitem__(self, name: str, val: _HeaderParamT) -> None: ... + def __delitem__(self, name: str) -> None: ... + def keys(self) -> list[str]: ... + def values(self) -> list[_HeaderT]: ... + def items(self) -> list[tuple[str, _HeaderT]]: ... + @overload + def get(self, name: str, failobj: None = None) -> _HeaderT | None: ... + @overload + def get(self, name: str, failobj: _T) -> _HeaderT | _T: ... + @overload + def get_all(self, name: str, failobj: None = None) -> list[_HeaderT] | None: ... + @overload + def get_all(self, name: str, failobj: _T) -> list[_HeaderT] | _T: ... + def add_header(self, _name: str, _value: str, **_params: _ParamsType) -> None: ... + def replace_header(self, _name: str, _value: _HeaderParamT) -> None: ... + def get_content_type(self) -> str: ... + def get_content_maintype(self) -> str: ... + def get_content_subtype(self) -> str: ... + def get_default_type(self) -> str: ... + def set_default_type(self, ctype: str) -> None: ... + @overload + def get_params( + self, failobj: None = None, header: str = "content-type", unquote: bool = True + ) -> list[tuple[str, str]] | None: ... + @overload + def get_params(self, failobj: _T, header: str = "content-type", unquote: bool = True) -> list[tuple[str, str]] | _T: ... + @overload + def get_param( + self, param: str, failobj: None = None, header: str = "content-type", unquote: bool = True + ) -> _ParamType | None: ... + @overload + def get_param(self, param: str, failobj: _T, header: str = "content-type", unquote: bool = True) -> _ParamType | _T: ... + def del_param(self, param: str, header: str = "content-type", requote: bool = True) -> None: ... + def set_type(self, type: str, header: str = "Content-Type", requote: bool = True) -> None: ... + @overload + def get_filename(self, failobj: None = None) -> str | None: ... + @overload + def get_filename(self, failobj: _T) -> str | _T: ... + @overload + def get_boundary(self, failobj: None = None) -> str | None: ... + @overload + def get_boundary(self, failobj: _T) -> str | _T: ... + def set_boundary(self, boundary: str) -> None: ... + @overload + def get_content_charset(self) -> str | None: ... + @overload + def get_content_charset(self, failobj: _T) -> str | _T: ... + @overload + def get_charsets(self, failobj: None = None) -> list[str | None]: ... + @overload + def get_charsets(self, failobj: _T) -> list[str | _T]: ... + def walk(self) -> Generator[Self, None, None]: ... + def get_content_disposition(self) -> str | None: ... + def as_string(self, unixfrom: bool = False, maxheaderlen: int = 0, policy: Policy[Any] | None = None) -> str: ... + def as_bytes(self, unixfrom: bool = False, policy: Policy[Any] | None = None) -> bytes: ... + def __bytes__(self) -> bytes: ... + def set_param( + self, + param: str, + value: str, + header: str = "Content-Type", + requote: bool = True, + charset: str | None = None, + language: str = "", + replace: bool = False, + ) -> None: ... + # The following two methods are undocumented, but a source code comment states that they are public API + def set_raw(self, name: str, value: _HeaderParamT) -> None: ... + def raw_items(self) -> Iterator[tuple[str, _HeaderT]]: ... + +class MIMEPart(Message[_HeaderRegistryT, _HeaderRegistryParamT]): + def __init__(self, policy: Policy[Any] | None = None) -> None: ... + def get_body(self, preferencelist: Sequence[str] = ("related", "html", "plain")) -> MIMEPart[_HeaderRegistryT] | None: ... + def attach(self, payload: Self) -> None: ... # type: ignore[override] + # The attachments are created via type(self) in the attach method. It's theoretically + # possible to sneak other attachment types into a MIMEPart instance, but could cause + # cause unforseen consequences. + def iter_attachments(self) -> Iterator[Self]: ... + def iter_parts(self) -> Iterator[MIMEPart[_HeaderRegistryT]]: ... + def get_content(self, *args: Any, content_manager: ContentManager | None = None, **kw: Any) -> Any: ... + def set_content(self, *args: Any, content_manager: ContentManager | None = None, **kw: Any) -> None: ... + def make_related(self, boundary: str | None = None) -> None: ... + def make_alternative(self, boundary: str | None = None) -> None: ... + def make_mixed(self, boundary: str | None = None) -> None: ... + def add_related(self, *args: Any, content_manager: ContentManager | None = ..., **kw: Any) -> None: ... + def add_alternative(self, *args: Any, content_manager: ContentManager | None = ..., **kw: Any) -> None: ... + def add_attachment(self, *args: Any, content_manager: ContentManager | None = ..., **kw: Any) -> None: ... + def clear(self) -> None: ... + def clear_content(self) -> None: ... + def as_string(self, unixfrom: bool = False, maxheaderlen: int | None = None, policy: Policy[Any] | None = None) -> str: ... + def is_attachment(self) -> bool: ... + +class EmailMessage(MIMEPart): ... diff --git a/stubs/3.2/distutils/__init__.py b/mypy/typeshed/stdlib/email/mime/__init__.pyi similarity index 100% rename from stubs/3.2/distutils/__init__.py rename to mypy/typeshed/stdlib/email/mime/__init__.pyi diff --git a/mypy/typeshed/stdlib/email/mime/application.pyi b/mypy/typeshed/stdlib/email/mime/application.pyi new file mode 100644 index 000000000000..a7ab9dc75ce2 --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/application.pyi @@ -0,0 +1,17 @@ +from collections.abc import Callable +from email import _ParamsType +from email.mime.nonmultipart import MIMENonMultipart +from email.policy import Policy + +__all__ = ["MIMEApplication"] + +class MIMEApplication(MIMENonMultipart): + def __init__( + self, + _data: str | bytes | bytearray, + _subtype: str = "octet-stream", + _encoder: Callable[[MIMEApplication], object] = ..., + *, + policy: Policy | None = None, + **_params: _ParamsType, + ) -> None: ... diff --git a/mypy/typeshed/stdlib/email/mime/audio.pyi b/mypy/typeshed/stdlib/email/mime/audio.pyi new file mode 100644 index 000000000000..090dfb960db6 --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/audio.pyi @@ -0,0 +1,17 @@ +from collections.abc import Callable +from email import _ParamsType +from email.mime.nonmultipart import MIMENonMultipart +from email.policy import Policy + +__all__ = ["MIMEAudio"] + +class MIMEAudio(MIMENonMultipart): + def __init__( + self, + _audiodata: str | bytes | bytearray, + _subtype: str | None = None, + _encoder: Callable[[MIMEAudio], object] = ..., + *, + policy: Policy | None = None, + **_params: _ParamsType, + ) -> None: ... diff --git a/mypy/typeshed/stdlib/email/mime/base.pyi b/mypy/typeshed/stdlib/email/mime/base.pyi new file mode 100644 index 000000000000..b733709f1b5a --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/base.pyi @@ -0,0 +1,8 @@ +import email.message +from email import _ParamsType +from email.policy import Policy + +__all__ = ["MIMEBase"] + +class MIMEBase(email.message.Message): + def __init__(self, _maintype: str, _subtype: str, *, policy: Policy | None = None, **_params: _ParamsType) -> None: ... diff --git a/mypy/typeshed/stdlib/email/mime/image.pyi b/mypy/typeshed/stdlib/email/mime/image.pyi new file mode 100644 index 000000000000..b47afa6ce592 --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/image.pyi @@ -0,0 +1,17 @@ +from collections.abc import Callable +from email import _ParamsType +from email.mime.nonmultipart import MIMENonMultipart +from email.policy import Policy + +__all__ = ["MIMEImage"] + +class MIMEImage(MIMENonMultipart): + def __init__( + self, + _imagedata: str | bytes | bytearray, + _subtype: str | None = None, + _encoder: Callable[[MIMEImage], object] = ..., + *, + policy: Policy | None = None, + **_params: _ParamsType, + ) -> None: ... diff --git a/mypy/typeshed/stdlib/email/mime/message.pyi b/mypy/typeshed/stdlib/email/mime/message.pyi new file mode 100644 index 000000000000..23cf58619ad9 --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/message.pyi @@ -0,0 +1,8 @@ +from email.message import Message +from email.mime.nonmultipart import MIMENonMultipart +from email.policy import Policy + +__all__ = ["MIMEMessage"] + +class MIMEMessage(MIMENonMultipart): + def __init__(self, _msg: Message, _subtype: str = "rfc822", *, policy: Policy | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/email/mime/multipart.pyi b/mypy/typeshed/stdlib/email/mime/multipart.pyi new file mode 100644 index 000000000000..6163810ed94a --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/multipart.pyi @@ -0,0 +1,18 @@ +from collections.abc import Sequence +from email import _ParamsType +from email.message import Message +from email.mime.base import MIMEBase +from email.policy import Policy + +__all__ = ["MIMEMultipart"] + +class MIMEMultipart(MIMEBase): + def __init__( + self, + _subtype: str = "mixed", + boundary: str | None = None, + _subparts: Sequence[Message] | None = None, + *, + policy: Policy | None = None, + **_params: _ParamsType, + ) -> None: ... diff --git a/mypy/typeshed/stdlib/email/mime/nonmultipart.pyi b/mypy/typeshed/stdlib/email/mime/nonmultipart.pyi new file mode 100644 index 000000000000..5497d89b1072 --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/nonmultipart.pyi @@ -0,0 +1,5 @@ +from email.mime.base import MIMEBase + +__all__ = ["MIMENonMultipart"] + +class MIMENonMultipart(MIMEBase): ... diff --git a/mypy/typeshed/stdlib/email/mime/text.pyi b/mypy/typeshed/stdlib/email/mime/text.pyi new file mode 100644 index 000000000000..74d5ef4c5cae --- /dev/null +++ b/mypy/typeshed/stdlib/email/mime/text.pyi @@ -0,0 +1,9 @@ +from email.mime.nonmultipart import MIMENonMultipart +from email.policy import Policy + +__all__ = ["MIMEText"] + +class MIMEText(MIMENonMultipart): + def __init__( + self, _text: str, _subtype: str = "plain", _charset: str | None = None, *, policy: Policy | None = None + ) -> None: ... diff --git a/mypy/typeshed/stdlib/email/parser.pyi b/mypy/typeshed/stdlib/email/parser.pyi new file mode 100644 index 000000000000..a1a57b4eef4b --- /dev/null +++ b/mypy/typeshed/stdlib/email/parser.pyi @@ -0,0 +1,38 @@ +from _typeshed import SupportsRead +from collections.abc import Callable +from email.feedparser import BytesFeedParser as BytesFeedParser, FeedParser as FeedParser +from email.message import Message +from email.policy import Policy +from io import _WrappedBuffer +from typing import Generic, TypeVar, overload + +__all__ = ["Parser", "HeaderParser", "BytesParser", "BytesHeaderParser", "FeedParser", "BytesFeedParser"] + +_MessageT = TypeVar("_MessageT", bound=Message, default=Message) + +class Parser(Generic[_MessageT]): + @overload + def __init__(self: Parser[Message[str, str]], _class: None = None, *, policy: Policy[Message[str, str]] = ...) -> None: ... + @overload + def __init__(self, _class: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... + def parse(self, fp: SupportsRead[str], headersonly: bool = False) -> _MessageT: ... + def parsestr(self, text: str, headersonly: bool = False) -> _MessageT: ... + +class HeaderParser(Parser[_MessageT]): + def parse(self, fp: SupportsRead[str], headersonly: bool = True) -> _MessageT: ... + def parsestr(self, text: str, headersonly: bool = True) -> _MessageT: ... + +class BytesParser(Generic[_MessageT]): + parser: Parser[_MessageT] + @overload + def __init__( + self: BytesParser[Message[str, str]], _class: None = None, *, policy: Policy[Message[str, str]] = ... + ) -> None: ... + @overload + def __init__(self, _class: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... + def parse(self, fp: _WrappedBuffer, headersonly: bool = False) -> _MessageT: ... + def parsebytes(self, text: bytes | bytearray, headersonly: bool = False) -> _MessageT: ... + +class BytesHeaderParser(BytesParser[_MessageT]): + def parse(self, fp: _WrappedBuffer, headersonly: bool = True) -> _MessageT: ... + def parsebytes(self, text: bytes | bytearray, headersonly: bool = True) -> _MessageT: ... diff --git a/mypy/typeshed/stdlib/email/policy.pyi b/mypy/typeshed/stdlib/email/policy.pyi new file mode 100644 index 000000000000..5b145bcf2318 --- /dev/null +++ b/mypy/typeshed/stdlib/email/policy.pyi @@ -0,0 +1,77 @@ +from collections.abc import Callable +from email._policybase import Compat32 as Compat32, Policy as Policy, _MessageFactory, compat32 as compat32 +from email.contentmanager import ContentManager +from email.message import EmailMessage, Message +from typing import Any, TypeVar, overload +from typing_extensions import Self + +__all__ = ["Compat32", "compat32", "Policy", "EmailPolicy", "default", "strict", "SMTP", "HTTP"] + +_MessageT = TypeVar("_MessageT", bound=Message, default=Message) + +class EmailPolicy(Policy[_MessageT]): + utf8: bool + refold_source: str + header_factory: Callable[[str, Any], Any] + content_manager: ContentManager + @overload + def __init__( + self: EmailPolicy[EmailMessage], + *, + max_line_length: int | None = ..., + linesep: str = ..., + cte_type: str = ..., + raise_on_defect: bool = ..., + mangle_from_: bool = ..., + message_factory: None = None, + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool = ..., + utf8: bool = ..., + refold_source: str = ..., + header_factory: Callable[[str, str], str] = ..., + content_manager: ContentManager = ..., + ) -> None: ... + @overload + def __init__( + self, + *, + max_line_length: int | None = ..., + linesep: str = ..., + cte_type: str = ..., + raise_on_defect: bool = ..., + mangle_from_: bool = ..., + message_factory: _MessageFactory[_MessageT] | None = ..., + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool = ..., + utf8: bool = ..., + refold_source: str = ..., + header_factory: Callable[[str, str], str] = ..., + content_manager: ContentManager = ..., + ) -> None: ... + def header_source_parse(self, sourcelines: list[str]) -> tuple[str, str]: ... + def header_store_parse(self, name: str, value: Any) -> tuple[str, Any]: ... + def header_fetch_parse(self, name: str, value: str) -> Any: ... + def fold(self, name: str, value: str) -> Any: ... + def fold_binary(self, name: str, value: str) -> bytes: ... + def clone( + self, + *, + max_line_length: int | None = ..., + linesep: str = ..., + cte_type: str = ..., + raise_on_defect: bool = ..., + mangle_from_: bool = ..., + message_factory: _MessageFactory[_MessageT] | None = ..., + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool = ..., + utf8: bool = ..., + refold_source: str = ..., + header_factory: Callable[[str, str], str] = ..., + content_manager: ContentManager = ..., + ) -> Self: ... + +default: EmailPolicy[EmailMessage] +SMTP: EmailPolicy[EmailMessage] +SMTPUTF8: EmailPolicy[EmailMessage] +HTTP: EmailPolicy[EmailMessage] +strict: EmailPolicy[EmailMessage] diff --git a/mypy/typeshed/stdlib/email/quoprimime.pyi b/mypy/typeshed/stdlib/email/quoprimime.pyi new file mode 100644 index 000000000000..87d08eecc70c --- /dev/null +++ b/mypy/typeshed/stdlib/email/quoprimime.pyi @@ -0,0 +1,28 @@ +from collections.abc import Iterable + +__all__ = [ + "body_decode", + "body_encode", + "body_length", + "decode", + "decodestring", + "header_decode", + "header_encode", + "header_length", + "quote", + "unquote", +] + +def header_check(octet: int) -> bool: ... +def body_check(octet: int) -> bool: ... +def header_length(bytearray: Iterable[int]) -> int: ... +def body_length(bytearray: Iterable[int]) -> int: ... +def unquote(s: str | bytes | bytearray) -> str: ... +def quote(c: str | bytes | bytearray) -> str: ... +def header_encode(header_bytes: bytes | bytearray, charset: str = "iso-8859-1") -> str: ... +def body_encode(body: str, maxlinelen: int = 76, eol: str = "\n") -> str: ... +def decode(encoded: str, eol: str = "\n") -> str: ... +def header_decode(s: str) -> str: ... + +body_decode = decode +decodestring = decode diff --git a/mypy/typeshed/stdlib/email/utils.pyi b/mypy/typeshed/stdlib/email/utils.pyi new file mode 100644 index 000000000000..dc3eecb5ef7f --- /dev/null +++ b/mypy/typeshed/stdlib/email/utils.pyi @@ -0,0 +1,78 @@ +import datetime +import sys +from _typeshed import Unused +from collections.abc import Iterable +from email import _ParamType +from email.charset import Charset +from typing import overload +from typing_extensions import TypeAlias, deprecated + +__all__ = [ + "collapse_rfc2231_value", + "decode_params", + "decode_rfc2231", + "encode_rfc2231", + "formataddr", + "formatdate", + "format_datetime", + "getaddresses", + "make_msgid", + "mktime_tz", + "parseaddr", + "parsedate", + "parsedate_tz", + "parsedate_to_datetime", + "unquote", +] + +_PDTZ: TypeAlias = tuple[int, int, int, int, int, int, int, int, int, int | None] + +def quote(str: str) -> str: ... +def unquote(str: str) -> str: ... + +# `strict` parameter added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +def parseaddr(addr: str | list[str], *, strict: bool = True) -> tuple[str, str]: ... +def formataddr(pair: tuple[str | None, str], charset: str | Charset = "utf-8") -> str: ... + +# `strict` parameter added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +def getaddresses(fieldvalues: Iterable[str], *, strict: bool = True) -> list[tuple[str, str]]: ... +@overload +def parsedate(data: None) -> None: ... +@overload +def parsedate(data: str) -> tuple[int, int, int, int, int, int, int, int, int] | None: ... +@overload +def parsedate_tz(data: None) -> None: ... +@overload +def parsedate_tz(data: str) -> _PDTZ | None: ... + +if sys.version_info >= (3, 10): + @overload + def parsedate_to_datetime(data: None) -> None: ... + @overload + def parsedate_to_datetime(data: str) -> datetime.datetime: ... + +else: + def parsedate_to_datetime(data: str) -> datetime.datetime: ... + +def mktime_tz(data: _PDTZ) -> int: ... +def formatdate(timeval: float | None = None, localtime: bool = False, usegmt: bool = False) -> str: ... +def format_datetime(dt: datetime.datetime, usegmt: bool = False) -> str: ... + +if sys.version_info >= (3, 14): + def localtime(dt: datetime.datetime | None = None) -> datetime.datetime: ... + +elif sys.version_info >= (3, 12): + @overload + def localtime(dt: datetime.datetime | None = None) -> datetime.datetime: ... + @overload + @deprecated("The `isdst` parameter does nothing and will be removed in Python 3.14.") + def localtime(dt: datetime.datetime | None = None, isdst: Unused = None) -> datetime.datetime: ... + +else: + def localtime(dt: datetime.datetime | None = None, isdst: int = -1) -> datetime.datetime: ... + +def make_msgid(idstring: str | None = None, domain: str | None = None) -> str: ... +def decode_rfc2231(s: str) -> tuple[str | None, str | None, str]: ... # May return list[str]. See issue #10431 for details. +def encode_rfc2231(s: str, charset: str | None = None, language: str | None = None) -> str: ... +def collapse_rfc2231_value(value: _ParamType, errors: str = "replace", fallback_charset: str = "us-ascii") -> str: ... +def decode_params(params: list[tuple[str, str]]) -> list[tuple[str, _ParamType]]: ... diff --git a/mypy/typeshed/stdlib/encodings/__init__.pyi b/mypy/typeshed/stdlib/encodings/__init__.pyi new file mode 100644 index 000000000000..2e83f0f65a71 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/__init__.pyi @@ -0,0 +1,10 @@ +from _typeshed import Incomplete +from codecs import CodecInfo + +class CodecRegistryError(LookupError, SystemError): ... + +def normalize_encoding(encoding: str | bytes) -> str: ... +def search_function(encoding: str) -> CodecInfo | None: ... + +# Needed for submodules +def __getattr__(name: str) -> Incomplete: ... diff --git a/mypy/typeshed/stdlib/encodings/aliases.pyi b/mypy/typeshed/stdlib/encodings/aliases.pyi new file mode 100644 index 000000000000..079af85d51ee --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/aliases.pyi @@ -0,0 +1 @@ +aliases: dict[str, str] diff --git a/mypy/typeshed/stdlib/encodings/ascii.pyi b/mypy/typeshed/stdlib/encodings/ascii.pyi new file mode 100644 index 000000000000..a85585af32ed --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/ascii.pyi @@ -0,0 +1,30 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + # At runtime, this is codecs.ascii_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + # At runtime, this is codecs.ascii_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +# Note: encode being a decode function and decode being an encode function is accurate to runtime. +class StreamConverter(StreamWriter, StreamReader): # type: ignore[misc] # incompatible methods in base classes + # At runtime, this is codecs.ascii_decode + @staticmethod + def encode(data: ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... # type: ignore[override] + # At runtime, this is codecs.ascii_encode + @staticmethod + def decode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... # type: ignore[override] + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/base64_codec.pyi b/mypy/typeshed/stdlib/encodings/base64_codec.pyi new file mode 100644 index 000000000000..0c4f1cb1fe59 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/base64_codec.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer +from typing import ClassVar + +# This codec is bytes to bytes. + +def base64_encode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... +def base64_decode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... + +class Codec(codecs.Codec): + def encode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): + charbuffertype: ClassVar[type] = ... + +class StreamReader(Codec, codecs.StreamReader): + charbuffertype: ClassVar[type] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/big5.pyi b/mypy/typeshed/stdlib/encodings/big5.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/big5.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/big5hkscs.pyi b/mypy/typeshed/stdlib/encodings/big5hkscs.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/big5hkscs.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/bz2_codec.pyi b/mypy/typeshed/stdlib/encodings/bz2_codec.pyi new file mode 100644 index 000000000000..468346a93da9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/bz2_codec.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer +from typing import ClassVar + +# This codec is bytes to bytes. + +def bz2_encode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... +def bz2_decode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... + +class Codec(codecs.Codec): + def encode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): + charbuffertype: ClassVar[type] = ... + +class StreamReader(Codec, codecs.StreamReader): + charbuffertype: ClassVar[type] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/charmap.pyi b/mypy/typeshed/stdlib/encodings/charmap.pyi new file mode 100644 index 000000000000..a971a15860b5 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/charmap.pyi @@ -0,0 +1,33 @@ +import codecs +from _codecs import _CharMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + # At runtime, this is codecs.charmap_encode + @staticmethod + def encode(str: str, errors: str | None = None, mapping: _CharMap | None = None, /) -> tuple[bytes, int]: ... + # At runtime, this is codecs.charmap_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, mapping: _CharMap | None = None, /) -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + mapping: _CharMap | None + def __init__(self, errors: str = "strict", mapping: _CharMap | None = None) -> None: ... + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + mapping: _CharMap | None + def __init__(self, errors: str = "strict", mapping: _CharMap | None = None) -> None: ... + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): + mapping: _CharMap | None + def __init__(self, stream: codecs._WritableStream, errors: str = "strict", mapping: _CharMap | None = None) -> None: ... + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + +class StreamReader(Codec, codecs.StreamReader): + mapping: _CharMap | None + def __init__(self, stream: codecs._ReadableStream, errors: str = "strict", mapping: _CharMap | None = None) -> None: ... + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[str, int]: ... # type: ignore[override] + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/cp037.pyi b/mypy/typeshed/stdlib/encodings/cp037.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp037.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1006.pyi b/mypy/typeshed/stdlib/encodings/cp1006.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1006.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1026.pyi b/mypy/typeshed/stdlib/encodings/cp1026.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1026.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1125.pyi b/mypy/typeshed/stdlib/encodings/cp1125.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1125.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp1140.pyi b/mypy/typeshed/stdlib/encodings/cp1140.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1140.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1250.pyi b/mypy/typeshed/stdlib/encodings/cp1250.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1250.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1251.pyi b/mypy/typeshed/stdlib/encodings/cp1251.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1251.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1252.pyi b/mypy/typeshed/stdlib/encodings/cp1252.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1252.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1253.pyi b/mypy/typeshed/stdlib/encodings/cp1253.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1253.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1254.pyi b/mypy/typeshed/stdlib/encodings/cp1254.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1254.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1255.pyi b/mypy/typeshed/stdlib/encodings/cp1255.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1255.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1256.pyi b/mypy/typeshed/stdlib/encodings/cp1256.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1256.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1257.pyi b/mypy/typeshed/stdlib/encodings/cp1257.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1257.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp1258.pyi b/mypy/typeshed/stdlib/encodings/cp1258.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp1258.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp273.pyi b/mypy/typeshed/stdlib/encodings/cp273.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp273.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp424.pyi b/mypy/typeshed/stdlib/encodings/cp424.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp424.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp437.pyi b/mypy/typeshed/stdlib/encodings/cp437.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp437.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp500.pyi b/mypy/typeshed/stdlib/encodings/cp500.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp500.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp720.pyi b/mypy/typeshed/stdlib/encodings/cp720.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp720.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp737.pyi b/mypy/typeshed/stdlib/encodings/cp737.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp737.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp775.pyi b/mypy/typeshed/stdlib/encodings/cp775.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp775.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp850.pyi b/mypy/typeshed/stdlib/encodings/cp850.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp850.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp852.pyi b/mypy/typeshed/stdlib/encodings/cp852.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp852.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp855.pyi b/mypy/typeshed/stdlib/encodings/cp855.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp855.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp856.pyi b/mypy/typeshed/stdlib/encodings/cp856.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp856.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp857.pyi b/mypy/typeshed/stdlib/encodings/cp857.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp857.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp858.pyi b/mypy/typeshed/stdlib/encodings/cp858.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp858.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp860.pyi b/mypy/typeshed/stdlib/encodings/cp860.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp860.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp861.pyi b/mypy/typeshed/stdlib/encodings/cp861.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp861.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp862.pyi b/mypy/typeshed/stdlib/encodings/cp862.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp862.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp863.pyi b/mypy/typeshed/stdlib/encodings/cp863.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp863.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp864.pyi b/mypy/typeshed/stdlib/encodings/cp864.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp864.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp865.pyi b/mypy/typeshed/stdlib/encodings/cp865.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp865.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp866.pyi b/mypy/typeshed/stdlib/encodings/cp866.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp866.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp869.pyi b/mypy/typeshed/stdlib/encodings/cp869.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp869.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/cp874.pyi b/mypy/typeshed/stdlib/encodings/cp874.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp874.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp875.pyi b/mypy/typeshed/stdlib/encodings/cp875.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp875.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/cp932.pyi b/mypy/typeshed/stdlib/encodings/cp932.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp932.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/cp949.pyi b/mypy/typeshed/stdlib/encodings/cp949.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp949.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/cp950.pyi b/mypy/typeshed/stdlib/encodings/cp950.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/cp950.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/euc_jis_2004.pyi b/mypy/typeshed/stdlib/encodings/euc_jis_2004.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/euc_jis_2004.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/euc_jisx0213.pyi b/mypy/typeshed/stdlib/encodings/euc_jisx0213.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/euc_jisx0213.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/euc_jp.pyi b/mypy/typeshed/stdlib/encodings/euc_jp.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/euc_jp.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/euc_kr.pyi b/mypy/typeshed/stdlib/encodings/euc_kr.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/euc_kr.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/gb18030.pyi b/mypy/typeshed/stdlib/encodings/gb18030.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/gb18030.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/gb2312.pyi b/mypy/typeshed/stdlib/encodings/gb2312.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/gb2312.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/gbk.pyi b/mypy/typeshed/stdlib/encodings/gbk.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/gbk.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/hex_codec.pyi b/mypy/typeshed/stdlib/encodings/hex_codec.pyi new file mode 100644 index 000000000000..3fd4fe38898a --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/hex_codec.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer +from typing import ClassVar + +# This codec is bytes to bytes. + +def hex_encode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... +def hex_decode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... + +class Codec(codecs.Codec): + def encode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): + charbuffertype: ClassVar[type] = ... + +class StreamReader(Codec, codecs.StreamReader): + charbuffertype: ClassVar[type] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/hp_roman8.pyi b/mypy/typeshed/stdlib/encodings/hp_roman8.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/hp_roman8.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/hz.pyi b/mypy/typeshed/stdlib/encodings/hz.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/hz.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/idna.pyi b/mypy/typeshed/stdlib/encodings/idna.pyi new file mode 100644 index 000000000000..3e2c8baf1cb2 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/idna.pyi @@ -0,0 +1,26 @@ +import codecs +import re +from _typeshed import ReadableBuffer + +dots: re.Pattern[str] +ace_prefix: bytes +sace_prefix: str + +def nameprep(label: str) -> str: ... +def ToASCII(label: str) -> bytes: ... +def ToUnicode(label: bytes | str) -> str: ... + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: ReadableBuffer | str, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.BufferedIncrementalEncoder): + def _buffer_encode(self, input: str, errors: str, final: bool) -> tuple[bytes, int]: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input: ReadableBuffer | str, errors: str, final: bool) -> tuple[str, int]: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso2022_jp.pyi b/mypy/typeshed/stdlib/encodings/iso2022_jp.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso2022_jp.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso2022_jp_1.pyi b/mypy/typeshed/stdlib/encodings/iso2022_jp_1.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso2022_jp_1.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso2022_jp_2.pyi b/mypy/typeshed/stdlib/encodings/iso2022_jp_2.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso2022_jp_2.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso2022_jp_2004.pyi b/mypy/typeshed/stdlib/encodings/iso2022_jp_2004.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso2022_jp_2004.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso2022_jp_3.pyi b/mypy/typeshed/stdlib/encodings/iso2022_jp_3.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso2022_jp_3.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso2022_jp_ext.pyi b/mypy/typeshed/stdlib/encodings/iso2022_jp_ext.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso2022_jp_ext.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso2022_kr.pyi b/mypy/typeshed/stdlib/encodings/iso2022_kr.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso2022_kr.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/iso8859_1.pyi b/mypy/typeshed/stdlib/encodings/iso8859_1.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_1.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_10.pyi b/mypy/typeshed/stdlib/encodings/iso8859_10.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_10.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_11.pyi b/mypy/typeshed/stdlib/encodings/iso8859_11.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_11.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_13.pyi b/mypy/typeshed/stdlib/encodings/iso8859_13.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_13.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_14.pyi b/mypy/typeshed/stdlib/encodings/iso8859_14.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_14.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_15.pyi b/mypy/typeshed/stdlib/encodings/iso8859_15.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_15.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_16.pyi b/mypy/typeshed/stdlib/encodings/iso8859_16.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_16.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_2.pyi b/mypy/typeshed/stdlib/encodings/iso8859_2.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_2.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_3.pyi b/mypy/typeshed/stdlib/encodings/iso8859_3.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_3.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_4.pyi b/mypy/typeshed/stdlib/encodings/iso8859_4.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_4.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_5.pyi b/mypy/typeshed/stdlib/encodings/iso8859_5.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_5.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_6.pyi b/mypy/typeshed/stdlib/encodings/iso8859_6.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_6.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_7.pyi b/mypy/typeshed/stdlib/encodings/iso8859_7.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_7.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_8.pyi b/mypy/typeshed/stdlib/encodings/iso8859_8.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_8.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/iso8859_9.pyi b/mypy/typeshed/stdlib/encodings/iso8859_9.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/iso8859_9.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/johab.pyi b/mypy/typeshed/stdlib/encodings/johab.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/johab.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/koi8_r.pyi b/mypy/typeshed/stdlib/encodings/koi8_r.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/koi8_r.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/koi8_t.pyi b/mypy/typeshed/stdlib/encodings/koi8_t.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/koi8_t.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/koi8_u.pyi b/mypy/typeshed/stdlib/encodings/koi8_u.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/koi8_u.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/kz1048.pyi b/mypy/typeshed/stdlib/encodings/kz1048.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/kz1048.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/latin_1.pyi b/mypy/typeshed/stdlib/encodings/latin_1.pyi new file mode 100644 index 000000000000..3b06773eac03 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/latin_1.pyi @@ -0,0 +1,30 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + # At runtime, this is codecs.latin_1_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + # At runtime, this is codecs.latin_1_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +# Note: encode being a decode function and decode being an encode function is accurate to runtime. +class StreamConverter(StreamWriter, StreamReader): # type: ignore[misc] # incompatible methods in base classes + # At runtime, this is codecs.latin_1_decode + @staticmethod + def encode(data: ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... # type: ignore[override] + # At runtime, this is codecs.latin_1_encode + @staticmethod + def decode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... # type: ignore[override] + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/mac_arabic.pyi b/mypy/typeshed/stdlib/encodings/mac_arabic.pyi new file mode 100644 index 000000000000..42781b489298 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_arabic.pyi @@ -0,0 +1,21 @@ +import codecs +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_map: dict[int, int | None] +decoding_table: str +encoding_map: dict[int, int] diff --git a/mypy/typeshed/stdlib/encodings/mac_centeuro.pyi b/mypy/typeshed/stdlib/encodings/mac_centeuro.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_centeuro.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_croatian.pyi b/mypy/typeshed/stdlib/encodings/mac_croatian.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_croatian.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_cyrillic.pyi b/mypy/typeshed/stdlib/encodings/mac_cyrillic.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_cyrillic.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_farsi.pyi b/mypy/typeshed/stdlib/encodings/mac_farsi.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_farsi.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_greek.pyi b/mypy/typeshed/stdlib/encodings/mac_greek.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_greek.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_iceland.pyi b/mypy/typeshed/stdlib/encodings/mac_iceland.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_iceland.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_latin2.pyi b/mypy/typeshed/stdlib/encodings/mac_latin2.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_latin2.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_roman.pyi b/mypy/typeshed/stdlib/encodings/mac_roman.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_roman.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_romanian.pyi b/mypy/typeshed/stdlib/encodings/mac_romanian.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_romanian.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mac_turkish.pyi b/mypy/typeshed/stdlib/encodings/mac_turkish.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mac_turkish.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/mbcs.pyi b/mypy/typeshed/stdlib/encodings/mbcs.pyi new file mode 100644 index 000000000000..2c2917d63f6d --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/mbcs.pyi @@ -0,0 +1,28 @@ +import codecs +import sys +from _typeshed import ReadableBuffer + +if sys.platform == "win32": + encode = codecs.mbcs_encode + + def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.mbcs_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + + class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.mbcs_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + + class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.mbcs_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + + def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/oem.pyi b/mypy/typeshed/stdlib/encodings/oem.pyi new file mode 100644 index 000000000000..376c12c445f4 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/oem.pyi @@ -0,0 +1,28 @@ +import codecs +import sys +from _typeshed import ReadableBuffer + +if sys.platform == "win32": + encode = codecs.oem_encode + + def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.oem_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + + class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.oem_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + + class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.oem_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + + def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/palmos.pyi b/mypy/typeshed/stdlib/encodings/palmos.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/palmos.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/ptcp154.pyi b/mypy/typeshed/stdlib/encodings/ptcp154.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/ptcp154.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/punycode.pyi b/mypy/typeshed/stdlib/encodings/punycode.pyi new file mode 100644 index 000000000000..eb99e667b416 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/punycode.pyi @@ -0,0 +1,33 @@ +import codecs +from typing import Literal + +def segregate(str: str) -> tuple[bytes, list[int]]: ... +def selective_len(str: str, max: int) -> int: ... +def selective_find(str: str, char: str, index: int, pos: int) -> tuple[int, int]: ... +def insertion_unsort(str: str, extended: list[int]) -> list[int]: ... +def T(j: int, bias: int) -> int: ... + +digits: Literal[b"abcdefghijklmnopqrstuvwxyz0123456789"] + +def generate_generalized_integer(N: int, bias: int) -> bytes: ... +def adapt(delta: int, first: bool, numchars: int) -> int: ... +def generate_integers(baselen: int, deltas: list[int]) -> bytes: ... +def punycode_encode(text: str) -> bytes: ... +def decode_generalized_number(extended: bytes, extpos: int, bias: int, errors: str) -> tuple[int, int | None]: ... +def insertion_sort(base: str, extended: bytes, errors: str) -> str: ... +def punycode_decode(text: memoryview | bytes | bytearray | str, errors: str) -> str: ... + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: memoryview | bytes | bytearray | str, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: memoryview | bytes | bytearray | str, final: bool = False) -> str: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/quopri_codec.pyi b/mypy/typeshed/stdlib/encodings/quopri_codec.pyi new file mode 100644 index 000000000000..e9deadd8d463 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/quopri_codec.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer +from typing import ClassVar + +# This codec is bytes to bytes. + +def quopri_encode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... +def quopri_decode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... + +class Codec(codecs.Codec): + def encode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): + charbuffertype: ClassVar[type] = ... + +class StreamReader(Codec, codecs.StreamReader): + charbuffertype: ClassVar[type] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/raw_unicode_escape.pyi b/mypy/typeshed/stdlib/encodings/raw_unicode_escape.pyi new file mode 100644 index 000000000000..74abb4623fab --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/raw_unicode_escape.pyi @@ -0,0 +1,34 @@ +import codecs +import sys +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + # At runtime, this is codecs.raw_unicode_escape_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + # At runtime, this is codecs.raw_unicode_escape_decode + if sys.version_info >= (3, 9): + @staticmethod + def decode(data: str | ReadableBuffer, errors: str | None = None, final: bool = True, /) -> tuple[str, int]: ... + else: + @staticmethod + def decode(data: str | ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +if sys.version_info >= (3, 9): + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input: str | ReadableBuffer, errors: str | None, final: bool) -> tuple[str, int]: ... + +else: + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: str | ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... + +class StreamReader(Codec, codecs.StreamReader): + if sys.version_info >= (3, 9): + def decode(self, input: str | ReadableBuffer, errors: str = "strict") -> tuple[str, int]: ... # type: ignore[override] + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/rot_13.pyi b/mypy/typeshed/stdlib/encodings/rot_13.pyi new file mode 100644 index 000000000000..8d71bc957594 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/rot_13.pyi @@ -0,0 +1,23 @@ +import codecs +from _typeshed import SupportsRead, SupportsWrite + +# This codec is string to string. + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[str, int]: ... # type: ignore[override] + def decode(self, input: str, errors: str = "strict") -> tuple[str, int]: ... # type: ignore[override] + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> str: ... # type: ignore[override] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: str, final: bool = False) -> str: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +rot13_map: dict[int, int] + +def rot13(infile: SupportsRead[str], outfile: SupportsWrite[str]) -> None: ... diff --git a/mypy/typeshed/stdlib/encodings/shift_jis.pyi b/mypy/typeshed/stdlib/encodings/shift_jis.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/shift_jis.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/shift_jis_2004.pyi b/mypy/typeshed/stdlib/encodings/shift_jis_2004.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/shift_jis_2004.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/shift_jisx0213.pyi b/mypy/typeshed/stdlib/encodings/shift_jisx0213.pyi new file mode 100644 index 000000000000..d613026a5a86 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/shift_jisx0213.pyi @@ -0,0 +1,23 @@ +import _multibytecodec as mbc +import codecs +from typing import ClassVar + +codec: mbc._MultibyteCodec + +class Codec(codecs.Codec): + encode = codec.encode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + decode = codec.decode # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + +class IncrementalEncoder(mbc.MultibyteIncrementalEncoder, codecs.IncrementalEncoder): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class IncrementalDecoder(mbc.MultibyteIncrementalDecoder, codecs.IncrementalDecoder): + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader): # type: ignore[misc] + codec: ClassVar[mbc._MultibyteCodec] = ... + +class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter): + codec: ClassVar[mbc._MultibyteCodec] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/tis_620.pyi b/mypy/typeshed/stdlib/encodings/tis_620.pyi new file mode 100644 index 000000000000..f62195662ce9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/tis_620.pyi @@ -0,0 +1,21 @@ +import codecs +from _codecs import _EncodingMap +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: bytes, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... + +decoding_table: str +encoding_table: _EncodingMap diff --git a/mypy/typeshed/stdlib/encodings/undefined.pyi b/mypy/typeshed/stdlib/encodings/undefined.pyi new file mode 100644 index 000000000000..4775dac752f2 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/undefined.pyi @@ -0,0 +1,20 @@ +import codecs +from _typeshed import ReadableBuffer + +# These return types are just to match the base types. In reality, these always +# raise an error. + +class Codec(codecs.Codec): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... +class StreamReader(Codec, codecs.StreamReader): ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/unicode_escape.pyi b/mypy/typeshed/stdlib/encodings/unicode_escape.pyi new file mode 100644 index 000000000000..1e942f57916e --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/unicode_escape.pyi @@ -0,0 +1,34 @@ +import codecs +import sys +from _typeshed import ReadableBuffer + +class Codec(codecs.Codec): + # At runtime, this is codecs.unicode_escape_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + # At runtime, this is codecs.unicode_escape_decode + if sys.version_info >= (3, 9): + @staticmethod + def decode(data: str | ReadableBuffer, errors: str | None = None, final: bool = True, /) -> tuple[str, int]: ... + else: + @staticmethod + def decode(data: str | ReadableBuffer, errors: str | None = None, /) -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +if sys.version_info >= (3, 9): + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input: str | ReadableBuffer, errors: str | None, final: bool) -> tuple[str, int]: ... + +else: + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: str | ReadableBuffer, final: bool = False) -> str: ... + +class StreamWriter(Codec, codecs.StreamWriter): ... + +class StreamReader(Codec, codecs.StreamReader): + if sys.version_info >= (3, 9): + def decode(self, input: str | ReadableBuffer, errors: str = "strict") -> tuple[str, int]: ... # type: ignore[override] + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_16.pyi b/mypy/typeshed/stdlib/encodings/utf_16.pyi new file mode 100644 index 000000000000..3b712cde420a --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_16.pyi @@ -0,0 +1,20 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_16_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input: ReadableBuffer, errors: str, final: bool) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_16_be.pyi b/mypy/typeshed/stdlib/encodings/utf_16_be.pyi new file mode 100644 index 000000000000..cc7d1534fc69 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_16_be.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_16_be_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.utf_16_be_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.utf_16_be_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.utf_16_be_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_16_le.pyi b/mypy/typeshed/stdlib/encodings/utf_16_le.pyi new file mode 100644 index 000000000000..ba103eb088e3 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_16_le.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_16_le_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.utf_16_le_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.utf_16_le_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.utf_16_le_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_32.pyi b/mypy/typeshed/stdlib/encodings/utf_32.pyi new file mode 100644 index 000000000000..c925be712c72 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_32.pyi @@ -0,0 +1,20 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_32_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input: ReadableBuffer, errors: str, final: bool) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + def encode(self, input: str, errors: str = "strict") -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_32_be.pyi b/mypy/typeshed/stdlib/encodings/utf_32_be.pyi new file mode 100644 index 000000000000..9d28f5199c50 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_32_be.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_32_be_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.utf_32_be_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.utf_32_be_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.utf_32_be_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_32_le.pyi b/mypy/typeshed/stdlib/encodings/utf_32_le.pyi new file mode 100644 index 000000000000..5be14a91a3e6 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_32_le.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_32_le_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.utf_32_le_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.utf_32_le_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.utf_32_le_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_7.pyi b/mypy/typeshed/stdlib/encodings/utf_7.pyi new file mode 100644 index 000000000000..dc1162f34c28 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_7.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_7_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.utf_7_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.utf_7_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.utf_7_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_8.pyi b/mypy/typeshed/stdlib/encodings/utf_8.pyi new file mode 100644 index 000000000000..918712d80473 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_8.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer + +encode = codecs.utf_8_encode + +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: str, final: bool = False) -> bytes: ... + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + # At runtime, this is codecs.utf_8_decode + @staticmethod + def _buffer_decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + # At runtime, this is codecs.utf_8_encode + @staticmethod + def encode(str: str, errors: str | None = None, /) -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + # At runtime, this is codecs.utf_8_decode + @staticmethod + def decode(data: ReadableBuffer, errors: str | None = None, final: bool = False, /) -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/utf_8_sig.pyi b/mypy/typeshed/stdlib/encodings/utf_8_sig.pyi new file mode 100644 index 000000000000..af69217d6732 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/utf_8_sig.pyi @@ -0,0 +1,22 @@ +import codecs +from _typeshed import ReadableBuffer + +class IncrementalEncoder(codecs.IncrementalEncoder): + def __init__(self, errors: str = "strict") -> None: ... + def encode(self, input: str, final: bool = False) -> bytes: ... + def getstate(self) -> int: ... + def setstate(self, state: int) -> None: ... # type: ignore[override] + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def __init__(self, errors: str = "strict") -> None: ... + def _buffer_decode(self, input: ReadableBuffer, errors: str | None, final: bool) -> tuple[str, int]: ... + +class StreamWriter(codecs.StreamWriter): + def encode(self, input: str, errors: str | None = "strict") -> tuple[bytes, int]: ... + +class StreamReader(codecs.StreamReader): + def decode(self, input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... + +def getregentry() -> codecs.CodecInfo: ... +def encode(input: str, errors: str | None = "strict") -> tuple[bytes, int]: ... +def decode(input: ReadableBuffer, errors: str | None = "strict") -> tuple[str, int]: ... diff --git a/mypy/typeshed/stdlib/encodings/uu_codec.pyi b/mypy/typeshed/stdlib/encodings/uu_codec.pyi new file mode 100644 index 000000000000..e32ba8ac0a1a --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/uu_codec.pyi @@ -0,0 +1,28 @@ +import codecs +from _typeshed import ReadableBuffer +from typing import ClassVar + +# This codec is bytes to bytes. + +def uu_encode( + input: ReadableBuffer, errors: str = "strict", filename: str = "", mode: int = 0o666 +) -> tuple[bytes, int]: ... +def uu_decode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... + +class Codec(codecs.Codec): + def encode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): + charbuffertype: ClassVar[type] = ... + +class StreamReader(Codec, codecs.StreamReader): + charbuffertype: ClassVar[type] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/encodings/zlib_codec.pyi b/mypy/typeshed/stdlib/encodings/zlib_codec.pyi new file mode 100644 index 000000000000..0f13d0e810e9 --- /dev/null +++ b/mypy/typeshed/stdlib/encodings/zlib_codec.pyi @@ -0,0 +1,26 @@ +import codecs +from _typeshed import ReadableBuffer +from typing import ClassVar + +# This codec is bytes to bytes. + +def zlib_encode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... +def zlib_decode(input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... + +class Codec(codecs.Codec): + def encode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + def decode(self, input: ReadableBuffer, errors: str = "strict") -> tuple[bytes, int]: ... # type: ignore[override] + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input: ReadableBuffer, final: bool = False) -> bytes: ... # type: ignore[override] + +class StreamWriter(Codec, codecs.StreamWriter): + charbuffertype: ClassVar[type] = ... + +class StreamReader(Codec, codecs.StreamReader): + charbuffertype: ClassVar[type] = ... + +def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/ensurepip/__init__.pyi b/mypy/typeshed/stdlib/ensurepip/__init__.pyi new file mode 100644 index 000000000000..332fb1845917 --- /dev/null +++ b/mypy/typeshed/stdlib/ensurepip/__init__.pyi @@ -0,0 +1,12 @@ +__all__ = ["version", "bootstrap"] + +def version() -> str: ... +def bootstrap( + *, + root: str | None = None, + upgrade: bool = False, + user: bool = False, + altinstall: bool = False, + default_pip: bool = False, + verbosity: int = 0, +) -> None: ... diff --git a/mypy/typeshed/stdlib/enum.pyi b/mypy/typeshed/stdlib/enum.pyi new file mode 100644 index 000000000000..3b6c325522d7 --- /dev/null +++ b/mypy/typeshed/stdlib/enum.pyi @@ -0,0 +1,339 @@ +import _typeshed +import sys +import types +from _typeshed import SupportsKeysAndGetItem, Unused +from builtins import property as _builtins_property +from collections.abc import Callable, Iterable, Iterator, Mapping +from typing import Any, Generic, Literal, TypeVar, overload +from typing_extensions import Self, TypeAlias + +__all__ = ["EnumMeta", "Enum", "IntEnum", "Flag", "IntFlag", "auto", "unique"] + +if sys.version_info >= (3, 11): + __all__ += [ + "CONFORM", + "CONTINUOUS", + "EJECT", + "EnumCheck", + "EnumType", + "FlagBoundary", + "KEEP", + "NAMED_FLAGS", + "ReprEnum", + "STRICT", + "StrEnum", + "UNIQUE", + "global_enum", + "global_enum_repr", + "global_flag_repr", + "global_str", + "member", + "nonmember", + "property", + "verify", + "pickle_by_enum_name", + "pickle_by_global_name", + ] + +if sys.version_info >= (3, 13): + __all__ += ["EnumDict"] + +_EnumMemberT = TypeVar("_EnumMemberT") +_EnumerationT = TypeVar("_EnumerationT", bound=type[Enum]) + +# The following all work: +# >>> from enum import Enum +# >>> from string import ascii_lowercase +# >>> Enum('Foo', names='RED YELLOW GREEN') +# +# >>> Enum('Foo', names=[('RED', 1), ('YELLOW, 2)]) +# +# >>> Enum('Foo', names=((x for x in (ascii_lowercase[i], i)) for i in range(5))) +# +# >>> Enum('Foo', names={'RED': 1, 'YELLOW': 2}) +# +_EnumNames: TypeAlias = str | Iterable[str] | Iterable[Iterable[str | Any]] | Mapping[str, Any] + +if sys.version_info >= (3, 11): + class nonmember(Generic[_EnumMemberT]): + value: _EnumMemberT + def __init__(self, value: _EnumMemberT) -> None: ... + + class member(Generic[_EnumMemberT]): + value: _EnumMemberT + def __init__(self, value: _EnumMemberT) -> None: ... + +class _EnumDict(dict[str, Any]): + def __init__(self) -> None: ... + def __setitem__(self, key: str, value: Any) -> None: ... + if sys.version_info >= (3, 11): + # See comment above `typing.MutableMapping.update` + # for why overloads are preferable to a Union here + # + # Unlike with MutableMapping.update(), the first argument is required, + # hence the type: ignore + @overload # type: ignore[override] + def update(self, members: SupportsKeysAndGetItem[str, Any], **more_members: Any) -> None: ... + @overload + def update(self, members: Iterable[tuple[str, Any]], **more_members: Any) -> None: ... + if sys.version_info >= (3, 13): + @property + def member_names(self) -> list[str]: ... + +if sys.version_info >= (3, 13): + EnumDict = _EnumDict + +# Structurally: Iterable[T], Reversible[T], Container[T] where T is the enum itself +class EnumMeta(type): + if sys.version_info >= (3, 11): + def __new__( + metacls: type[_typeshed.Self], + cls: str, + bases: tuple[type, ...], + classdict: _EnumDict, + *, + boundary: FlagBoundary | None = None, + _simple: bool = False, + **kwds: Any, + ) -> _typeshed.Self: ... + elif sys.version_info >= (3, 9): + def __new__( + metacls: type[_typeshed.Self], cls: str, bases: tuple[type, ...], classdict: _EnumDict, **kwds: Any + ) -> _typeshed.Self: ... + else: + def __new__(metacls: type[_typeshed.Self], cls: str, bases: tuple[type, ...], classdict: _EnumDict) -> _typeshed.Self: ... + + if sys.version_info >= (3, 9): + @classmethod + def __prepare__(metacls, cls: str, bases: tuple[type, ...], **kwds: Any) -> _EnumDict: ... # type: ignore[override] + else: + @classmethod + def __prepare__(metacls, cls: str, bases: tuple[type, ...]) -> _EnumDict: ... # type: ignore[override] + + def __iter__(self: type[_EnumMemberT]) -> Iterator[_EnumMemberT]: ... + def __reversed__(self: type[_EnumMemberT]) -> Iterator[_EnumMemberT]: ... + if sys.version_info >= (3, 12): + def __contains__(self: type[Any], value: object) -> bool: ... + elif sys.version_info >= (3, 11): + def __contains__(self: type[Any], member: object) -> bool: ... + elif sys.version_info >= (3, 10): + def __contains__(self: type[Any], obj: object) -> bool: ... + else: + def __contains__(self: type[Any], member: object) -> bool: ... + + def __getitem__(self: type[_EnumMemberT], name: str) -> _EnumMemberT: ... + @_builtins_property + def __members__(self: type[_EnumMemberT]) -> types.MappingProxyType[str, _EnumMemberT]: ... + def __len__(self) -> int: ... + def __bool__(self) -> Literal[True]: ... + def __dir__(self) -> list[str]: ... + + # Overload 1: Value lookup on an already existing enum class (simple case) + @overload + def __call__(cls: type[_EnumMemberT], value: Any, names: None = None) -> _EnumMemberT: ... + + # Overload 2: Functional API for constructing new enum classes. + if sys.version_info >= (3, 11): + @overload + def __call__( + cls, + value: str, + names: _EnumNames, + *, + module: str | None = None, + qualname: str | None = None, + type: type | None = None, + start: int = 1, + boundary: FlagBoundary | None = None, + ) -> type[Enum]: ... + else: + @overload + def __call__( + cls, + value: str, + names: _EnumNames, + *, + module: str | None = None, + qualname: str | None = None, + type: type | None = None, + start: int = 1, + ) -> type[Enum]: ... + + # Overload 3 (py312+ only): Value lookup on an already existing enum class (complex case) + # + # >>> class Foo(enum.Enum): + # ... X = 1, 2, 3 + # >>> Foo(1, 2, 3) + # + # + if sys.version_info >= (3, 12): + @overload + def __call__(cls: type[_EnumMemberT], value: Any, *values: Any) -> _EnumMemberT: ... + + _member_names_: list[str] # undocumented + _member_map_: dict[str, Enum] # undocumented + _value2member_map_: dict[Any, Enum] # undocumented + +if sys.version_info >= (3, 11): + # In 3.11 `EnumMeta` metaclass is renamed to `EnumType`, but old name also exists. + EnumType = EnumMeta + + class property(types.DynamicClassAttribute): + def __set_name__(self, ownerclass: type[Enum], name: str) -> None: ... + name: str + clsname: str + member: Enum | None + + _magic_enum_attr = property +else: + _magic_enum_attr = types.DynamicClassAttribute + +class Enum(metaclass=EnumMeta): + @_magic_enum_attr + def name(self) -> str: ... + @_magic_enum_attr + def value(self) -> Any: ... + _name_: str + _value_: Any + _ignore_: str | list[str] + _order_: str + __order__: str + @classmethod + def _missing_(cls, value: object) -> Any: ... + @staticmethod + def _generate_next_value_(name: str, start: int, count: int, last_values: list[Any]) -> Any: ... + # It's not true that `__new__` will accept any argument type, + # so ideally we'd use `Any` to indicate that the argument type is inexpressible. + # However, using `Any` causes too many false-positives for those using mypy's `--disallow-any-expr` + # (see #7752, #2539, mypy/#5788), + # and in practice using `object` here has the same effect as using `Any`. + def __new__(cls, value: object) -> Self: ... + def __dir__(self) -> list[str]: ... + def __hash__(self) -> int: ... + def __format__(self, format_spec: str) -> str: ... + def __reduce_ex__(self, proto: Unused) -> tuple[Any, ...]: ... + if sys.version_info >= (3, 11): + def __copy__(self) -> Self: ... + def __deepcopy__(self, memo: Any) -> Self: ... + if sys.version_info >= (3, 12): + @classmethod + def __signature__(cls) -> str: ... + +if sys.version_info >= (3, 11): + class ReprEnum(Enum): ... + +if sys.version_info >= (3, 11): + _IntEnumBase = ReprEnum +else: + _IntEnumBase = Enum + +class IntEnum(int, _IntEnumBase): + _value_: int + @_magic_enum_attr + def value(self) -> int: ... + def __new__(cls, value: int) -> Self: ... + +def unique(enumeration: _EnumerationT) -> _EnumerationT: ... + +_auto_null: Any + +class Flag(Enum): + _name_: str | None # type: ignore[assignment] + _value_: int + @_magic_enum_attr + def name(self) -> str | None: ... # type: ignore[override] + @_magic_enum_attr + def value(self) -> int: ... + def __contains__(self, other: Self) -> bool: ... + def __bool__(self) -> bool: ... + def __or__(self, other: Self) -> Self: ... + def __and__(self, other: Self) -> Self: ... + def __xor__(self, other: Self) -> Self: ... + def __invert__(self) -> Self: ... + if sys.version_info >= (3, 11): + def __iter__(self) -> Iterator[Self]: ... + def __len__(self) -> int: ... + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ + +if sys.version_info >= (3, 11): + class StrEnum(str, ReprEnum): + def __new__(cls, value: str) -> Self: ... + _value_: str + @_magic_enum_attr + def value(self) -> str: ... + @staticmethod + def _generate_next_value_(name: str, start: int, count: int, last_values: list[str]) -> str: ... + + class EnumCheck(StrEnum): + CONTINUOUS = "no skipped integer values" + NAMED_FLAGS = "multi-flag aliases may not contain unnamed flags" + UNIQUE = "one name per value" + + CONTINUOUS = EnumCheck.CONTINUOUS + NAMED_FLAGS = EnumCheck.NAMED_FLAGS + UNIQUE = EnumCheck.UNIQUE + + class verify: + def __init__(self, *checks: EnumCheck) -> None: ... + def __call__(self, enumeration: _EnumerationT) -> _EnumerationT: ... + + class FlagBoundary(StrEnum): + STRICT = "strict" + CONFORM = "conform" + EJECT = "eject" + KEEP = "keep" + + STRICT = FlagBoundary.STRICT + CONFORM = FlagBoundary.CONFORM + EJECT = FlagBoundary.EJECT + KEEP = FlagBoundary.KEEP + + def global_str(self: Enum) -> str: ... + def global_enum(cls: _EnumerationT, update_str: bool = False) -> _EnumerationT: ... + def global_enum_repr(self: Enum) -> str: ... + def global_flag_repr(self: Flag) -> str: ... + +if sys.version_info >= (3, 11): + # The body of the class is the same, but the base classes are different. + class IntFlag(int, ReprEnum, Flag, boundary=KEEP): # type: ignore[misc] # complaints about incompatible bases + def __new__(cls, value: int) -> Self: ... + def __or__(self, other: int) -> Self: ... + def __and__(self, other: int) -> Self: ... + def __xor__(self, other: int) -> Self: ... + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ + +else: + class IntFlag(int, Flag): # type: ignore[misc] # complaints about incompatible bases + def __new__(cls, value: int) -> Self: ... + def __or__(self, other: int) -> Self: ... + def __and__(self, other: int) -> Self: ... + def __xor__(self, other: int) -> Self: ... + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ + +class auto: + _value_: Any + @_magic_enum_attr + def value(self) -> Any: ... + def __new__(cls) -> Self: ... + + # These don't exist, but auto is basically immediately replaced with + # either an int or a str depending on the type of the enum. StrEnum's auto + # shouldn't have these, but they're needed for int versions of auto (mostly the __or__). + # Ideally type checkers would special case auto enough to handle this, + # but until then this is a slightly inaccurate helping hand. + def __or__(self, other: int | Self) -> Self: ... + def __and__(self, other: int | Self) -> Self: ... + def __xor__(self, other: int | Self) -> Self: ... + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ + +if sys.version_info >= (3, 11): + def pickle_by_global_name(self: Enum, proto: int) -> str: ... + def pickle_by_enum_name(self: _EnumMemberT, proto: int) -> tuple[Callable[..., Any], tuple[type[_EnumMemberT], str]]: ... diff --git a/mypy/typeshed/stdlib/errno.pyi b/mypy/typeshed/stdlib/errno.pyi new file mode 100644 index 000000000000..84d2b44a6a61 --- /dev/null +++ b/mypy/typeshed/stdlib/errno.pyi @@ -0,0 +1,222 @@ +import sys +from collections.abc import Mapping + +errorcode: Mapping[int, str] + +EPERM: int +ENOENT: int +ESRCH: int +EINTR: int +EIO: int +ENXIO: int +E2BIG: int +ENOEXEC: int +EBADF: int +ECHILD: int +EAGAIN: int +ENOMEM: int +EACCES: int +EFAULT: int +EBUSY: int +EEXIST: int +EXDEV: int +ENODEV: int +ENOTDIR: int +EISDIR: int +EINVAL: int +ENFILE: int +EMFILE: int +ENOTTY: int +ETXTBSY: int +EFBIG: int +ENOSPC: int +ESPIPE: int +EROFS: int +EMLINK: int +EPIPE: int +EDOM: int +ERANGE: int +EDEADLK: int +ENAMETOOLONG: int +ENOLCK: int +ENOSYS: int +ENOTEMPTY: int +ELOOP: int +EWOULDBLOCK: int +ENOMSG: int +EIDRM: int +ENOSTR: int +ENODATA: int +ETIME: int +ENOSR: int +EREMOTE: int +ENOLINK: int +EPROTO: int +EBADMSG: int +EOVERFLOW: int +EILSEQ: int +EUSERS: int +ENOTSOCK: int +EDESTADDRREQ: int +EMSGSIZE: int +EPROTOTYPE: int +ENOPROTOOPT: int +EPROTONOSUPPORT: int +ESOCKTNOSUPPORT: int +ENOTSUP: int +EOPNOTSUPP: int +EPFNOSUPPORT: int +EAFNOSUPPORT: int +EADDRINUSE: int +EADDRNOTAVAIL: int +ENETDOWN: int +ENETUNREACH: int +ENETRESET: int +ECONNABORTED: int +ECONNRESET: int +ENOBUFS: int +EISCONN: int +ENOTCONN: int +ESHUTDOWN: int +ETOOMANYREFS: int +ETIMEDOUT: int +ECONNREFUSED: int +EHOSTDOWN: int +EHOSTUNREACH: int +EALREADY: int +EINPROGRESS: int +ESTALE: int +EDQUOT: int +ECANCELED: int # undocumented +ENOTRECOVERABLE: int # undocumented +EOWNERDEAD: int # undocumented + +if sys.platform == "sunos5" or sys.platform == "solaris": # noqa: Y008 + ELOCKUNMAPPED: int + ENOTACTIVE: int + +if sys.platform != "win32": + ENOTBLK: int + EMULTIHOP: int + +if sys.platform == "darwin": + # All of the below are undocumented + EAUTH: int + EBADARCH: int + EBADEXEC: int + EBADMACHO: int + EBADRPC: int + EDEVERR: int + EFTYPE: int + ENEEDAUTH: int + ENOATTR: int + ENOPOLICY: int + EPROCLIM: int + EPROCUNAVAIL: int + EPROGMISMATCH: int + EPROGUNAVAIL: int + EPWROFF: int + ERPCMISMATCH: int + ESHLIBVERS: int + if sys.version_info >= (3, 11): + EQFULL: int + +if sys.platform != "darwin": + EDEADLOCK: int + +if sys.platform != "win32" and sys.platform != "darwin": + ECHRNG: int + EL2NSYNC: int + EL3HLT: int + EL3RST: int + ELNRNG: int + EUNATCH: int + ENOCSI: int + EL2HLT: int + EBADE: int + EBADR: int + EXFULL: int + ENOANO: int + EBADRQC: int + EBADSLT: int + EBFONT: int + ENONET: int + ENOPKG: int + EADV: int + ESRMNT: int + ECOMM: int + EDOTDOT: int + ENOTUNIQ: int + EBADFD: int + EREMCHG: int + ELIBACC: int + ELIBBAD: int + ELIBSCN: int + ELIBMAX: int + ELIBEXEC: int + ERESTART: int + ESTRPIPE: int + EUCLEAN: int + ENOTNAM: int + ENAVAIL: int + EISNAM: int + EREMOTEIO: int + # All of the below are undocumented + EKEYEXPIRED: int + EKEYREJECTED: int + EKEYREVOKED: int + EMEDIUMTYPE: int + ENOKEY: int + ENOMEDIUM: int + ERFKILL: int + +if sys.platform == "win32": + # All of these are undocumented + WSABASEERR: int + WSAEACCES: int + WSAEADDRINUSE: int + WSAEADDRNOTAVAIL: int + WSAEAFNOSUPPORT: int + WSAEALREADY: int + WSAEBADF: int + WSAECONNABORTED: int + WSAECONNREFUSED: int + WSAECONNRESET: int + WSAEDESTADDRREQ: int + WSAEDISCON: int + WSAEDQUOT: int + WSAEFAULT: int + WSAEHOSTDOWN: int + WSAEHOSTUNREACH: int + WSAEINPROGRESS: int + WSAEINTR: int + WSAEINVAL: int + WSAEISCONN: int + WSAELOOP: int + WSAEMFILE: int + WSAEMSGSIZE: int + WSAENAMETOOLONG: int + WSAENETDOWN: int + WSAENETRESET: int + WSAENETUNREACH: int + WSAENOBUFS: int + WSAENOPROTOOPT: int + WSAENOTCONN: int + WSAENOTEMPTY: int + WSAENOTSOCK: int + WSAEOPNOTSUPP: int + WSAEPFNOSUPPORT: int + WSAEPROCLIM: int + WSAEPROTONOSUPPORT: int + WSAEPROTOTYPE: int + WSAEREMOTE: int + WSAESHUTDOWN: int + WSAESOCKTNOSUPPORT: int + WSAESTALE: int + WSAETIMEDOUT: int + WSAETOOMANYREFS: int + WSAEUSERS: int + WSAEWOULDBLOCK: int + WSANOTINITIALISED: int + WSASYSNOTREADY: int + WSAVERNOTSUPPORTED: int diff --git a/mypy/typeshed/stdlib/faulthandler.pyi b/mypy/typeshed/stdlib/faulthandler.pyi new file mode 100644 index 000000000000..320a8b6fad15 --- /dev/null +++ b/mypy/typeshed/stdlib/faulthandler.pyi @@ -0,0 +1,13 @@ +import sys +from _typeshed import FileDescriptorLike + +def cancel_dump_traceback_later() -> None: ... +def disable() -> None: ... +def dump_traceback(file: FileDescriptorLike = ..., all_threads: bool = ...) -> None: ... +def dump_traceback_later(timeout: float, repeat: bool = ..., file: FileDescriptorLike = ..., exit: bool = ...) -> None: ... +def enable(file: FileDescriptorLike = ..., all_threads: bool = ...) -> None: ... +def is_enabled() -> bool: ... + +if sys.platform != "win32": + def register(signum: int, file: FileDescriptorLike = ..., all_threads: bool = ..., chain: bool = ...) -> None: ... + def unregister(signum: int, /) -> None: ... diff --git a/mypy/typeshed/stdlib/fcntl.pyi b/mypy/typeshed/stdlib/fcntl.pyi new file mode 100644 index 000000000000..71078b3b4579 --- /dev/null +++ b/mypy/typeshed/stdlib/fcntl.pyi @@ -0,0 +1,160 @@ +import sys +from _typeshed import FileDescriptorLike, ReadOnlyBuffer, WriteableBuffer +from typing import Any, Final, Literal, overload +from typing_extensions import Buffer + +if sys.platform != "win32": + FASYNC: int + FD_CLOEXEC: int + F_DUPFD: int + F_DUPFD_CLOEXEC: int + F_GETFD: int + F_GETFL: int + F_GETLK: int + F_GETOWN: int + F_RDLCK: int + F_SETFD: int + F_SETFL: int + F_SETLK: int + F_SETLKW: int + F_SETOWN: int + F_UNLCK: int + F_WRLCK: int + + F_GETLEASE: int + F_SETLEASE: int + if sys.platform == "darwin": + F_FULLFSYNC: int + F_NOCACHE: int + if sys.version_info >= (3, 9): + F_GETPATH: int + if sys.platform == "linux": + F_SETLKW64: int + F_SETSIG: int + F_SHLCK: int + F_SETLK64: int + F_GETSIG: int + F_NOTIFY: int + F_EXLCK: int + F_GETLK64: int + F_ADD_SEALS: int + F_GET_SEALS: int + F_SEAL_GROW: int + F_SEAL_SEAL: int + F_SEAL_SHRINK: int + F_SEAL_WRITE: int + if sys.version_info >= (3, 9): + F_OFD_GETLK: Final[int] + F_OFD_SETLK: Final[int] + F_OFD_SETLKW: Final[int] + + if sys.version_info >= (3, 10): + F_GETPIPE_SZ: int + F_SETPIPE_SZ: int + + DN_ACCESS: int + DN_ATTRIB: int + DN_CREATE: int + DN_DELETE: int + DN_MODIFY: int + DN_MULTISHOT: int + DN_RENAME: int + + LOCK_EX: int + LOCK_NB: int + LOCK_SH: int + LOCK_UN: int + if sys.platform == "linux": + LOCK_MAND: int + LOCK_READ: int + LOCK_RW: int + LOCK_WRITE: int + + if sys.platform == "linux": + # Constants for the POSIX STREAMS interface. Present in glibc until 2.29 (released February 2019). + # Never implemented on BSD, and considered "obsolescent" starting in POSIX 2008. + # Probably still used on Solaris. + I_ATMARK: int + I_CANPUT: int + I_CKBAND: int + I_FDINSERT: int + I_FIND: int + I_FLUSH: int + I_FLUSHBAND: int + I_GETBAND: int + I_GETCLTIME: int + I_GETSIG: int + I_GRDOPT: int + I_GWROPT: int + I_LINK: int + I_LIST: int + I_LOOK: int + I_NREAD: int + I_PEEK: int + I_PLINK: int + I_POP: int + I_PUNLINK: int + I_PUSH: int + I_RECVFD: int + I_SENDFD: int + I_SETCLTIME: int + I_SETSIG: int + I_SRDOPT: int + I_STR: int + I_SWROPT: int + I_UNLINK: int + + if sys.version_info >= (3, 12) and sys.platform == "linux": + FICLONE: int + FICLONERANGE: int + + if sys.version_info >= (3, 13) and sys.platform == "linux": + F_OWNER_TID: Final = 0 + F_OWNER_PID: Final = 1 + F_OWNER_PGRP: Final = 2 + F_SETOWN_EX: Final = 15 + F_GETOWN_EX: Final = 16 + F_SEAL_FUTURE_WRITE: Final = 16 + F_GET_RW_HINT: Final = 1035 + F_SET_RW_HINT: Final = 1036 + F_GET_FILE_RW_HINT: Final = 1037 + F_SET_FILE_RW_HINT: Final = 1038 + RWH_WRITE_LIFE_NOT_SET: Final = 0 + RWH_WRITE_LIFE_NONE: Final = 1 + RWH_WRITE_LIFE_SHORT: Final = 2 + RWH_WRITE_LIFE_MEDIUM: Final = 3 + RWH_WRITE_LIFE_LONG: Final = 4 + RWH_WRITE_LIFE_EXTREME: Final = 5 + + if sys.version_info >= (3, 11) and sys.platform == "darwin": + F_OFD_SETLK: Final = 90 + F_OFD_SETLKW: Final = 91 + F_OFD_GETLK: Final = 92 + + if sys.version_info >= (3, 13) and sys.platform != "linux": + # OSx and NetBSD + F_GETNOSIGPIPE: Final[int] + F_SETNOSIGPIPE: Final[int] + # OSx and FreeBSD + F_RDAHEAD: Final[int] + + @overload + def fcntl(fd: FileDescriptorLike, cmd: int, arg: int = 0, /) -> int: ... + @overload + def fcntl(fd: FileDescriptorLike, cmd: int, arg: str | ReadOnlyBuffer, /) -> bytes: ... + # If arg is an int, return int + @overload + def ioctl(fd: FileDescriptorLike, request: int, arg: int = 0, mutate_flag: bool = True, /) -> int: ... + # The return type works as follows: + # - If arg is a read-write buffer, return int if mutate_flag is True, otherwise bytes + # - If arg is a read-only buffer, return bytes (and ignore the value of mutate_flag) + # We can't represent that precisely as we can't distinguish between read-write and read-only + # buffers, so we add overloads for a few unambiguous cases and use Any for the rest. + @overload + def ioctl(fd: FileDescriptorLike, request: int, arg: bytes, mutate_flag: bool = True, /) -> bytes: ... + @overload + def ioctl(fd: FileDescriptorLike, request: int, arg: WriteableBuffer, mutate_flag: Literal[False], /) -> bytes: ... + @overload + def ioctl(fd: FileDescriptorLike, request: int, arg: Buffer, mutate_flag: bool = True, /) -> Any: ... + def flock(fd: FileDescriptorLike, operation: int, /) -> None: ... + def lockf(fd: FileDescriptorLike, cmd: int, len: int = 0, start: int = 0, whence: int = 0, /) -> Any: ... diff --git a/mypy/typeshed/stdlib/filecmp.pyi b/mypy/typeshed/stdlib/filecmp.pyi new file mode 100644 index 000000000000..cb7b94596077 --- /dev/null +++ b/mypy/typeshed/stdlib/filecmp.pyi @@ -0,0 +1,68 @@ +import sys +from _typeshed import GenericPath, StrOrBytesPath +from collections.abc import Callable, Iterable, Sequence +from typing import Any, AnyStr, Final, Generic, Literal + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["clear_cache", "cmp", "dircmp", "cmpfiles", "DEFAULT_IGNORES"] + +DEFAULT_IGNORES: list[str] +BUFSIZE: Final = 8192 + +def cmp(f1: StrOrBytesPath, f2: StrOrBytesPath, shallow: bool | Literal[0, 1] = True) -> bool: ... +def cmpfiles( + a: GenericPath[AnyStr], b: GenericPath[AnyStr], common: Iterable[GenericPath[AnyStr]], shallow: bool | Literal[0, 1] = True +) -> tuple[list[AnyStr], list[AnyStr], list[AnyStr]]: ... + +class dircmp(Generic[AnyStr]): + if sys.version_info >= (3, 13): + def __init__( + self, + a: GenericPath[AnyStr], + b: GenericPath[AnyStr], + ignore: Sequence[AnyStr] | None = None, + hide: Sequence[AnyStr] | None = None, + *, + shallow: bool = True, + ) -> None: ... + else: + def __init__( + self, + a: GenericPath[AnyStr], + b: GenericPath[AnyStr], + ignore: Sequence[AnyStr] | None = None, + hide: Sequence[AnyStr] | None = None, + ) -> None: ... + left: AnyStr + right: AnyStr + hide: Sequence[AnyStr] + ignore: Sequence[AnyStr] + # These properties are created at runtime by __getattr__ + subdirs: dict[AnyStr, dircmp[AnyStr]] + same_files: list[AnyStr] + diff_files: list[AnyStr] + funny_files: list[AnyStr] + common_dirs: list[AnyStr] + common_files: list[AnyStr] + common_funny: list[AnyStr] + common: list[AnyStr] + left_only: list[AnyStr] + right_only: list[AnyStr] + left_list: list[AnyStr] + right_list: list[AnyStr] + def report(self) -> None: ... + def report_partial_closure(self) -> None: ... + def report_full_closure(self) -> None: ... + methodmap: dict[str, Callable[[], None]] + def phase0(self) -> None: ... + def phase1(self) -> None: ... + def phase2(self) -> None: ... + def phase3(self) -> None: ... + def phase4(self) -> None: ... + def phase4_closure(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +def clear_cache() -> None: ... diff --git a/mypy/typeshed/stdlib/fileinput.pyi b/mypy/typeshed/stdlib/fileinput.pyi new file mode 100644 index 000000000000..bf6daad0aea7 --- /dev/null +++ b/mypy/typeshed/stdlib/fileinput.pyi @@ -0,0 +1,213 @@ +import sys +from _typeshed import AnyStr_co, StrOrBytesPath +from collections.abc import Callable, Iterable +from types import TracebackType +from typing import IO, Any, AnyStr, Generic, Literal, Protocol, overload +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "input", + "close", + "nextfile", + "filename", + "lineno", + "filelineno", + "fileno", + "isfirstline", + "isstdin", + "FileInput", + "hook_compressed", + "hook_encoded", +] + +if sys.version_info >= (3, 11): + _TextMode: TypeAlias = Literal["r"] +else: + _TextMode: TypeAlias = Literal["r", "rU", "U"] + +class _HasReadlineAndFileno(Protocol[AnyStr_co]): + def readline(self) -> AnyStr_co: ... + def fileno(self) -> int: ... + +if sys.version_info >= (3, 10): + # encoding and errors are added + @overload + def input( + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: _TextMode = "r", + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[str]] | None = None, + encoding: str | None = None, + errors: str | None = None, + ) -> FileInput[str]: ... + @overload + def input( + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: Literal["rb"], + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[bytes]] | None = None, + encoding: None = None, + errors: None = None, + ) -> FileInput[bytes]: ... + @overload + def input( + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: str, + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[Any]] | None = None, + encoding: str | None = None, + errors: str | None = None, + ) -> FileInput[Any]: ... + +else: + # bufsize is dropped and mode and openhook become keyword-only + @overload + def input( + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: _TextMode = "r", + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[str]] | None = None, + ) -> FileInput[str]: ... + @overload + def input( + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: Literal["rb"], + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[bytes]] | None = None, + ) -> FileInput[bytes]: ... + @overload + def input( + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: str, + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[Any]] | None = None, + ) -> FileInput[Any]: ... + +def close() -> None: ... +def nextfile() -> None: ... +def filename() -> str: ... +def lineno() -> int: ... +def filelineno() -> int: ... +def fileno() -> int: ... +def isfirstline() -> bool: ... +def isstdin() -> bool: ... + +class FileInput(Generic[AnyStr]): + if sys.version_info >= (3, 10): + # encoding and errors are added + @overload + def __init__( + self: FileInput[str], + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: _TextMode = "r", + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[str]] | None = None, + encoding: str | None = None, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self: FileInput[bytes], + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: Literal["rb"], + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[bytes]] | None = None, + encoding: None = None, + errors: None = None, + ) -> None: ... + @overload + def __init__( + self: FileInput[Any], + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: str, + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[Any]] | None = None, + encoding: str | None = None, + errors: str | None = None, + ) -> None: ... + + else: + # bufsize is dropped and mode and openhook become keyword-only + @overload + def __init__( + self: FileInput[str], + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: _TextMode = "r", + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[str]] | None = None, + ) -> None: ... + @overload + def __init__( + self: FileInput[bytes], + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: Literal["rb"], + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[bytes]] | None = None, + ) -> None: ... + @overload + def __init__( + self: FileInput[Any], + files: StrOrBytesPath | Iterable[StrOrBytesPath] | None = None, + inplace: bool = False, + backup: str = "", + *, + mode: str, + openhook: Callable[[StrOrBytesPath, str], _HasReadlineAndFileno[Any]] | None = None, + ) -> None: ... + + def __del__(self) -> None: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> AnyStr: ... + if sys.version_info < (3, 11): + def __getitem__(self, i: int) -> AnyStr: ... + + def nextfile(self) -> None: ... + def readline(self) -> AnyStr: ... + def filename(self) -> str: ... + def lineno(self) -> int: ... + def filelineno(self) -> int: ... + def fileno(self) -> int: ... + def isfirstline(self) -> bool: ... + def isstdin(self) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +if sys.version_info >= (3, 10): + def hook_compressed( + filename: StrOrBytesPath, mode: str, *, encoding: str | None = None, errors: str | None = None + ) -> IO[Any]: ... + +else: + def hook_compressed(filename: StrOrBytesPath, mode: str) -> IO[Any]: ... + +def hook_encoded(encoding: str, errors: str | None = None) -> Callable[[StrOrBytesPath, str], IO[Any]]: ... diff --git a/mypy/typeshed/stdlib/fnmatch.pyi b/mypy/typeshed/stdlib/fnmatch.pyi new file mode 100644 index 000000000000..7051c999c430 --- /dev/null +++ b/mypy/typeshed/stdlib/fnmatch.pyi @@ -0,0 +1,9 @@ +from collections.abc import Iterable +from typing import AnyStr + +__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] + +def fnmatch(name: AnyStr, pat: AnyStr) -> bool: ... +def fnmatchcase(name: AnyStr, pat: AnyStr) -> bool: ... +def filter(names: Iterable[AnyStr], pat: AnyStr) -> list[AnyStr]: ... +def translate(pat: str) -> str: ... diff --git a/mypy/typeshed/stdlib/formatter.pyi b/mypy/typeshed/stdlib/formatter.pyi new file mode 100644 index 000000000000..05c3c8b3dd41 --- /dev/null +++ b/mypy/typeshed/stdlib/formatter.pyi @@ -0,0 +1,88 @@ +from collections.abc import Iterable +from typing import IO, Any +from typing_extensions import TypeAlias + +AS_IS: None +_FontType: TypeAlias = tuple[str, bool, bool, bool] +_StylesType: TypeAlias = tuple[Any, ...] + +class NullFormatter: + writer: NullWriter | None + def __init__(self, writer: NullWriter | None = None) -> None: ... + def end_paragraph(self, blankline: int) -> None: ... + def add_line_break(self) -> None: ... + def add_hor_rule(self, *args: Any, **kw: Any) -> None: ... + def add_label_data(self, format: str, counter: int, blankline: int | None = None) -> None: ... + def add_flowing_data(self, data: str) -> None: ... + def add_literal_data(self, data: str) -> None: ... + def flush_softspace(self) -> None: ... + def push_alignment(self, align: str | None) -> None: ... + def pop_alignment(self) -> None: ... + def push_font(self, x: _FontType) -> None: ... + def pop_font(self) -> None: ... + def push_margin(self, margin: int) -> None: ... + def pop_margin(self) -> None: ... + def set_spacing(self, spacing: str | None) -> None: ... + def push_style(self, *styles: _StylesType) -> None: ... + def pop_style(self, n: int = 1) -> None: ... + def assert_line_data(self, flag: int = 1) -> None: ... + +class AbstractFormatter: + writer: NullWriter + align: str | None + align_stack: list[str | None] + font_stack: list[_FontType] + margin_stack: list[int] + spacing: str | None + style_stack: Any + nospace: int + softspace: int + para_end: int + parskip: int + hard_break: int + have_label: int + def __init__(self, writer: NullWriter) -> None: ... + def end_paragraph(self, blankline: int) -> None: ... + def add_line_break(self) -> None: ... + def add_hor_rule(self, *args: Any, **kw: Any) -> None: ... + def add_label_data(self, format: str, counter: int, blankline: int | None = None) -> None: ... + def format_counter(self, format: Iterable[str], counter: int) -> str: ... + def format_letter(self, case: str, counter: int) -> str: ... + def format_roman(self, case: str, counter: int) -> str: ... + def add_flowing_data(self, data: str) -> None: ... + def add_literal_data(self, data: str) -> None: ... + def flush_softspace(self) -> None: ... + def push_alignment(self, align: str | None) -> None: ... + def pop_alignment(self) -> None: ... + def push_font(self, font: _FontType) -> None: ... + def pop_font(self) -> None: ... + def push_margin(self, margin: int) -> None: ... + def pop_margin(self) -> None: ... + def set_spacing(self, spacing: str | None) -> None: ... + def push_style(self, *styles: _StylesType) -> None: ... + def pop_style(self, n: int = 1) -> None: ... + def assert_line_data(self, flag: int = 1) -> None: ... + +class NullWriter: + def flush(self) -> None: ... + def new_alignment(self, align: str | None) -> None: ... + def new_font(self, font: _FontType) -> None: ... + def new_margin(self, margin: int, level: int) -> None: ... + def new_spacing(self, spacing: str | None) -> None: ... + def new_styles(self, styles: tuple[Any, ...]) -> None: ... + def send_paragraph(self, blankline: int) -> None: ... + def send_line_break(self) -> None: ... + def send_hor_rule(self, *args: Any, **kw: Any) -> None: ... + def send_label_data(self, data: str) -> None: ... + def send_flowing_data(self, data: str) -> None: ... + def send_literal_data(self, data: str) -> None: ... + +class AbstractWriter(NullWriter): ... + +class DumbWriter(NullWriter): + file: IO[str] + maxcol: int + def __init__(self, file: IO[str] | None = None, maxcol: int = 72) -> None: ... + def reset(self) -> None: ... + +def test(file: str | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/fractions.pyi b/mypy/typeshed/stdlib/fractions.pyi new file mode 100644 index 000000000000..fbcfa868cc1b --- /dev/null +++ b/mypy/typeshed/stdlib/fractions.pyi @@ -0,0 +1,158 @@ +import sys +from collections.abc import Callable +from decimal import Decimal +from numbers import Integral, Rational, Real +from typing import Any, Literal, Protocol, SupportsIndex, overload +from typing_extensions import Self, TypeAlias + +_ComparableNum: TypeAlias = int | float | Decimal | Real + +if sys.version_info >= (3, 9): + __all__ = ["Fraction"] +else: + __all__ = ["Fraction", "gcd"] + @overload + def gcd(a: int, b: int) -> int: ... + @overload + def gcd(a: Integral, b: int) -> Integral: ... + @overload + def gcd(a: int, b: Integral) -> Integral: ... + @overload + def gcd(a: Integral, b: Integral) -> Integral: ... + +class _ConvertibleToIntegerRatio(Protocol): + def as_integer_ratio(self) -> tuple[int | Rational, int | Rational]: ... + +class Fraction(Rational): + @overload + def __new__(cls, numerator: int | Rational = 0, denominator: int | Rational | None = None) -> Self: ... + @overload + def __new__(cls, value: float | Decimal | str, /) -> Self: ... + + if sys.version_info >= (3, 14): + @overload + def __new__(cls, value: _ConvertibleToIntegerRatio) -> Self: ... + + @classmethod + def from_float(cls, f: float) -> Self: ... + @classmethod + def from_decimal(cls, dec: Decimal) -> Self: ... + def limit_denominator(self, max_denominator: int = 1000000) -> Fraction: ... + def as_integer_ratio(self) -> tuple[int, int]: ... + if sys.version_info >= (3, 12): + def is_integer(self) -> bool: ... + + @property + def numerator(a) -> int: ... + @property + def denominator(a) -> int: ... + @overload + def __add__(a, b: int | Fraction) -> Fraction: ... + @overload + def __add__(a, b: float) -> float: ... + @overload + def __add__(a, b: complex) -> complex: ... + @overload + def __radd__(b, a: int | Fraction) -> Fraction: ... + @overload + def __radd__(b, a: float) -> float: ... + @overload + def __radd__(b, a: complex) -> complex: ... + @overload + def __sub__(a, b: int | Fraction) -> Fraction: ... + @overload + def __sub__(a, b: float) -> float: ... + @overload + def __sub__(a, b: complex) -> complex: ... + @overload + def __rsub__(b, a: int | Fraction) -> Fraction: ... + @overload + def __rsub__(b, a: float) -> float: ... + @overload + def __rsub__(b, a: complex) -> complex: ... + @overload + def __mul__(a, b: int | Fraction) -> Fraction: ... + @overload + def __mul__(a, b: float) -> float: ... + @overload + def __mul__(a, b: complex) -> complex: ... + @overload + def __rmul__(b, a: int | Fraction) -> Fraction: ... + @overload + def __rmul__(b, a: float) -> float: ... + @overload + def __rmul__(b, a: complex) -> complex: ... + @overload + def __truediv__(a, b: int | Fraction) -> Fraction: ... + @overload + def __truediv__(a, b: float) -> float: ... + @overload + def __truediv__(a, b: complex) -> complex: ... + @overload + def __rtruediv__(b, a: int | Fraction) -> Fraction: ... + @overload + def __rtruediv__(b, a: float) -> float: ... + @overload + def __rtruediv__(b, a: complex) -> complex: ... + @overload + def __floordiv__(a, b: int | Fraction) -> int: ... + @overload + def __floordiv__(a, b: float) -> float: ... + @overload + def __rfloordiv__(b, a: int | Fraction) -> int: ... + @overload + def __rfloordiv__(b, a: float) -> float: ... + @overload + def __mod__(a, b: int | Fraction) -> Fraction: ... + @overload + def __mod__(a, b: float) -> float: ... + @overload + def __rmod__(b, a: int | Fraction) -> Fraction: ... + @overload + def __rmod__(b, a: float) -> float: ... + @overload + def __divmod__(a, b: int | Fraction) -> tuple[int, Fraction]: ... + @overload + def __divmod__(a, b: float) -> tuple[float, Fraction]: ... + @overload + def __rdivmod__(a, b: int | Fraction) -> tuple[int, Fraction]: ... + @overload + def __rdivmod__(a, b: float) -> tuple[float, Fraction]: ... + @overload + def __pow__(a, b: int) -> Fraction: ... + @overload + def __pow__(a, b: float | Fraction) -> float: ... + @overload + def __pow__(a, b: complex) -> complex: ... + @overload + def __rpow__(b, a: float | Fraction) -> float: ... + @overload + def __rpow__(b, a: complex) -> complex: ... + def __pos__(a) -> Fraction: ... + def __neg__(a) -> Fraction: ... + def __abs__(a) -> Fraction: ... + def __trunc__(a) -> int: ... + def __floor__(a) -> int: ... + def __ceil__(a) -> int: ... + @overload + def __round__(self, ndigits: None = None) -> int: ... + @overload + def __round__(self, ndigits: int) -> Fraction: ... + def __hash__(self) -> int: ... + def __eq__(a, b: object) -> bool: ... + def __lt__(a, b: _ComparableNum) -> bool: ... + def __gt__(a, b: _ComparableNum) -> bool: ... + def __le__(a, b: _ComparableNum) -> bool: ... + def __ge__(a, b: _ComparableNum) -> bool: ... + def __bool__(a) -> bool: ... + def __copy__(self) -> Self: ... + def __deepcopy__(self, memo: Any) -> Self: ... + if sys.version_info >= (3, 11): + def __int__(a, _index: Callable[[SupportsIndex], int] = ...) -> int: ... + # Not actually defined within fractions.py, but provides more useful + # overrides + @property + def real(self) -> Fraction: ... + @property + def imag(self) -> Literal[0]: ... + def conjugate(self) -> Fraction: ... diff --git a/mypy/typeshed/stdlib/ftplib.pyi b/mypy/typeshed/stdlib/ftplib.pyi new file mode 100644 index 000000000000..3693d7c52a26 --- /dev/null +++ b/mypy/typeshed/stdlib/ftplib.pyi @@ -0,0 +1,178 @@ +import sys +from _typeshed import SupportsRead, SupportsReadline +from collections.abc import Callable, Iterable, Iterator +from socket import socket +from ssl import SSLContext +from types import TracebackType +from typing import Any, Final, Literal, TextIO +from typing_extensions import Self + +__all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", "all_errors", "FTP_TLS"] + +MSG_OOB: Final = 1 +FTP_PORT: Final = 21 +MAXLINE: Final = 8192 +CRLF: Final = "\r\n" +B_CRLF: Final = b"\r\n" + +class Error(Exception): ... +class error_reply(Error): ... +class error_temp(Error): ... +class error_perm(Error): ... +class error_proto(Error): ... + +all_errors: tuple[type[Exception], ...] + +class FTP: + debugging: int + host: str + port: int + maxline: int + sock: socket | None + welcome: str | None + passiveserver: int + timeout: float | None + af: int + lastresp: str + file: TextIO | None + encoding: str + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + source_address: tuple[str, int] | None + if sys.version_info >= (3, 9): + def __init__( + self, + host: str = "", + user: str = "", + passwd: str = "", + acct: str = "", + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + *, + encoding: str = "utf-8", + ) -> None: ... + else: + def __init__( + self, + host: str = "", + user: str = "", + passwd: str = "", + acct: str = "", + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + ) -> None: ... + + def connect( + self, host: str = "", port: int = 0, timeout: float = -999, source_address: tuple[str, int] | None = None + ) -> str: ... + def getwelcome(self) -> str: ... + def set_debuglevel(self, level: int) -> None: ... + def debug(self, level: int) -> None: ... + def set_pasv(self, val: bool | Literal[0, 1]) -> None: ... + def sanitize(self, s: str) -> str: ... + def putline(self, line: str) -> None: ... + def putcmd(self, line: str) -> None: ... + def getline(self) -> str: ... + def getmultiline(self) -> str: ... + def getresp(self) -> str: ... + def voidresp(self) -> str: ... + def abort(self) -> str: ... + def sendcmd(self, cmd: str) -> str: ... + def voidcmd(self, cmd: str) -> str: ... + def sendport(self, host: str, port: int) -> str: ... + def sendeprt(self, host: str, port: int) -> str: ... + def makeport(self) -> socket: ... + def makepasv(self) -> tuple[str, int]: ... + def login(self, user: str = "", passwd: str = "", acct: str = "") -> str: ... + # In practice, `rest` can actually be anything whose str() is an integer sequence, so to make it simple we allow integers + def ntransfercmd(self, cmd: str, rest: int | str | None = None) -> tuple[socket, int | None]: ... + def transfercmd(self, cmd: str, rest: int | str | None = None) -> socket: ... + def retrbinary( + self, cmd: str, callback: Callable[[bytes], object], blocksize: int = 8192, rest: int | str | None = None + ) -> str: ... + def storbinary( + self, + cmd: str, + fp: SupportsRead[bytes], + blocksize: int = 8192, + callback: Callable[[bytes], object] | None = None, + rest: int | str | None = None, + ) -> str: ... + def retrlines(self, cmd: str, callback: Callable[[str], object] | None = None) -> str: ... + def storlines(self, cmd: str, fp: SupportsReadline[bytes], callback: Callable[[bytes], object] | None = None) -> str: ... + def acct(self, password: str) -> str: ... + def nlst(self, *args: str) -> list[str]: ... + # Technically only the last arg can be a Callable but ... + def dir(self, *args: str | Callable[[str], object]) -> None: ... + def mlsd(self, path: str = "", facts: Iterable[str] = []) -> Iterator[tuple[str, dict[str, str]]]: ... + def rename(self, fromname: str, toname: str) -> str: ... + def delete(self, filename: str) -> str: ... + def cwd(self, dirname: str) -> str: ... + def size(self, filename: str) -> int | None: ... + def mkd(self, dirname: str) -> str: ... + def rmd(self, dirname: str) -> str: ... + def pwd(self) -> str: ... + def quit(self) -> str: ... + def close(self) -> None: ... + +class FTP_TLS(FTP): + if sys.version_info >= (3, 12): + def __init__( + self, + host: str = "", + user: str = "", + passwd: str = "", + acct: str = "", + *, + context: SSLContext | None = None, + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + encoding: str = "utf-8", + ) -> None: ... + elif sys.version_info >= (3, 9): + def __init__( + self, + host: str = "", + user: str = "", + passwd: str = "", + acct: str = "", + keyfile: str | None = None, + certfile: str | None = None, + context: SSLContext | None = None, + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + *, + encoding: str = "utf-8", + ) -> None: ... + else: + def __init__( + self, + host: str = "", + user: str = "", + passwd: str = "", + acct: str = "", + keyfile: str | None = None, + certfile: str | None = None, + context: SSLContext | None = None, + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + ) -> None: ... + ssl_version: int + keyfile: str | None + certfile: str | None + context: SSLContext + def login(self, user: str = "", passwd: str = "", acct: str = "", secure: bool = True) -> str: ... + def auth(self) -> str: ... + def prot_p(self) -> str: ... + def prot_c(self) -> str: ... + def ccc(self) -> str: ... + +def parse150(resp: str) -> int | None: ... # undocumented +def parse227(resp: str) -> tuple[str, int]: ... # undocumented +def parse229(resp: str, peer: Any) -> tuple[str, int]: ... # undocumented +def parse257(resp: str) -> str: ... # undocumented +def ftpcp( + source: FTP, sourcename: str, target: FTP, targetname: str = "", type: Literal["A", "I"] = "I" +) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/functools.pyi b/mypy/typeshed/stdlib/functools.pyi new file mode 100644 index 000000000000..9957fa8f1634 --- /dev/null +++ b/mypy/typeshed/stdlib/functools.pyi @@ -0,0 +1,213 @@ +import sys +import types +from _typeshed import SupportsAllComparisons, SupportsItems +from collections.abc import Callable, Hashable, Iterable, Sequence, Sized +from typing import Any, Generic, Literal, NamedTuple, TypedDict, TypeVar, final, overload +from typing_extensions import ParamSpec, Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "update_wrapper", + "wraps", + "WRAPPER_ASSIGNMENTS", + "WRAPPER_UPDATES", + "total_ordering", + "cmp_to_key", + "lru_cache", + "reduce", + "partial", + "partialmethod", + "singledispatch", + "cached_property", + "singledispatchmethod", +] + +if sys.version_info >= (3, 9): + __all__ += ["cache"] + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_S = TypeVar("_S") +_PWrapped = ParamSpec("_PWrapped") +_RWrapped = TypeVar("_RWrapped") +_PWrapper = ParamSpec("_PWrapper") +_RWrapper = TypeVar("_RWrapper") + +@overload +def reduce(function: Callable[[_T, _S], _T], sequence: Iterable[_S], initial: _T, /) -> _T: ... +@overload +def reduce(function: Callable[[_T, _T], _T], sequence: Iterable[_T], /) -> _T: ... + +class _CacheInfo(NamedTuple): + hits: int + misses: int + maxsize: int | None + currsize: int + +if sys.version_info >= (3, 9): + class _CacheParameters(TypedDict): + maxsize: int + typed: bool + +@final +class _lru_cache_wrapper(Generic[_T]): + __wrapped__: Callable[..., _T] + def __call__(self, *args: Hashable, **kwargs: Hashable) -> _T: ... + def cache_info(self) -> _CacheInfo: ... + def cache_clear(self) -> None: ... + if sys.version_info >= (3, 9): + def cache_parameters(self) -> _CacheParameters: ... + + def __copy__(self) -> _lru_cache_wrapper[_T]: ... + def __deepcopy__(self, memo: Any, /) -> _lru_cache_wrapper[_T]: ... + +@overload +def lru_cache(maxsize: int | None = 128, typed: bool = False) -> Callable[[Callable[..., _T]], _lru_cache_wrapper[_T]]: ... +@overload +def lru_cache(maxsize: Callable[..., _T], typed: bool = False) -> _lru_cache_wrapper[_T]: ... + +if sys.version_info >= (3, 12): + WRAPPER_ASSIGNMENTS: tuple[ + Literal["__module__"], + Literal["__name__"], + Literal["__qualname__"], + Literal["__doc__"], + Literal["__annotations__"], + Literal["__type_params__"], + ] +else: + WRAPPER_ASSIGNMENTS: tuple[ + Literal["__module__"], Literal["__name__"], Literal["__qualname__"], Literal["__doc__"], Literal["__annotations__"] + ] +WRAPPER_UPDATES: tuple[Literal["__dict__"]] + +class _Wrapped(Generic[_PWrapped, _RWrapped, _PWrapper, _RWrapper]): + __wrapped__: Callable[_PWrapped, _RWrapped] + def __call__(self, *args: _PWrapper.args, **kwargs: _PWrapper.kwargs) -> _RWrapper: ... + # as with ``Callable``, we'll assume that these attributes exist + __name__: str + __qualname__: str + +class _Wrapper(Generic[_PWrapped, _RWrapped]): + def __call__(self, f: Callable[_PWrapper, _RWrapper]) -> _Wrapped[_PWrapped, _RWrapped, _PWrapper, _RWrapper]: ... + +if sys.version_info >= (3, 12): + def update_wrapper( + wrapper: Callable[_PWrapper, _RWrapper], + wrapped: Callable[_PWrapped, _RWrapped], + assigned: Sequence[str] = ("__module__", "__name__", "__qualname__", "__doc__", "__annotations__", "__type_params__"), + updated: Sequence[str] = ("__dict__",), + ) -> _Wrapped[_PWrapped, _RWrapped, _PWrapper, _RWrapper]: ... + def wraps( + wrapped: Callable[_PWrapped, _RWrapped], + assigned: Sequence[str] = ("__module__", "__name__", "__qualname__", "__doc__", "__annotations__", "__type_params__"), + updated: Sequence[str] = ("__dict__",), + ) -> _Wrapper[_PWrapped, _RWrapped]: ... + +else: + def update_wrapper( + wrapper: Callable[_PWrapper, _RWrapper], + wrapped: Callable[_PWrapped, _RWrapped], + assigned: Sequence[str] = ("__module__", "__name__", "__qualname__", "__doc__", "__annotations__"), + updated: Sequence[str] = ("__dict__",), + ) -> _Wrapped[_PWrapped, _RWrapped, _PWrapper, _RWrapper]: ... + def wraps( + wrapped: Callable[_PWrapped, _RWrapped], + assigned: Sequence[str] = ("__module__", "__name__", "__qualname__", "__doc__", "__annotations__"), + updated: Sequence[str] = ("__dict__",), + ) -> _Wrapper[_PWrapped, _RWrapped]: ... + +def total_ordering(cls: type[_T]) -> type[_T]: ... +def cmp_to_key(mycmp: Callable[[_T, _T], int]) -> Callable[[_T], SupportsAllComparisons]: ... + +class partial(Generic[_T]): + @property + def func(self) -> Callable[..., _T]: ... + @property + def args(self) -> tuple[Any, ...]: ... + @property + def keywords(self) -> dict[str, Any]: ... + def __new__(cls, func: Callable[..., _T], /, *args: Any, **kwargs: Any) -> Self: ... + def __call__(self, /, *args: Any, **kwargs: Any) -> _T: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +# With protocols, this could change into a generic protocol that defines __get__ and returns _T +_Descriptor: TypeAlias = Any + +class partialmethod(Generic[_T]): + func: Callable[..., _T] | _Descriptor + args: tuple[Any, ...] + keywords: dict[str, Any] + @overload + def __init__(self, func: Callable[..., _T], /, *args: Any, **keywords: Any) -> None: ... + @overload + def __init__(self, func: _Descriptor, /, *args: Any, **keywords: Any) -> None: ... + def __get__(self, obj: Any, cls: type[Any] | None = None) -> Callable[..., _T]: ... + @property + def __isabstractmethod__(self) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class _SingleDispatchCallable(Generic[_T]): + registry: types.MappingProxyType[Any, Callable[..., _T]] + def dispatch(self, cls: Any) -> Callable[..., _T]: ... + # @fun.register(complex) + # def _(arg, verbose=False): ... + @overload + def register(self, cls: type[Any], func: None = None) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... + # @fun.register + # def _(arg: int, verbose=False): + @overload + def register(self, cls: Callable[..., _T], func: None = None) -> Callable[..., _T]: ... + # fun.register(int, lambda x: x) + @overload + def register(self, cls: type[Any], func: Callable[..., _T]) -> Callable[..., _T]: ... + def _clear_cache(self) -> None: ... + def __call__(self, /, *args: Any, **kwargs: Any) -> _T: ... + +def singledispatch(func: Callable[..., _T]) -> _SingleDispatchCallable[_T]: ... + +class singledispatchmethod(Generic[_T]): + dispatcher: _SingleDispatchCallable[_T] + func: Callable[..., _T] + def __init__(self, func: Callable[..., _T]) -> None: ... + @property + def __isabstractmethod__(self) -> bool: ... + @overload + def register(self, cls: type[Any], method: None = None) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... + @overload + def register(self, cls: Callable[..., _T], method: None = None) -> Callable[..., _T]: ... + @overload + def register(self, cls: type[Any], method: Callable[..., _T]) -> Callable[..., _T]: ... + def __get__(self, obj: _S, cls: type[_S] | None = None) -> Callable[..., _T]: ... + +class cached_property(Generic[_T_co]): + func: Callable[[Any], _T_co] + attrname: str | None + def __init__(self, func: Callable[[Any], _T_co]) -> None: ... + @overload + def __get__(self, instance: None, owner: type[Any] | None = None) -> Self: ... + @overload + def __get__(self, instance: object, owner: type[Any] | None = None) -> _T_co: ... + def __set_name__(self, owner: type[Any], name: str) -> None: ... + # __set__ is not defined at runtime, but @cached_property is designed to be settable + def __set__(self, instance: object, value: _T_co) -> None: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +if sys.version_info >= (3, 9): + def cache(user_function: Callable[..., _T], /) -> _lru_cache_wrapper[_T]: ... + +def _make_key( + args: tuple[Hashable, ...], + kwds: SupportsItems[Any, Any], + typed: bool, + kwd_mark: tuple[object, ...] = ..., + fasttypes: set[type] = ..., + tuple: type = ..., + type: Any = ..., + len: Callable[[Sized], int] = ..., +) -> Hashable: ... diff --git a/mypy/typeshed/stdlib/gc.pyi b/mypy/typeshed/stdlib/gc.pyi new file mode 100644 index 000000000000..9d34e0d6213a --- /dev/null +++ b/mypy/typeshed/stdlib/gc.pyi @@ -0,0 +1,37 @@ +import sys +from collections.abc import Callable +from typing import Any, Final, Literal +from typing_extensions import TypeAlias + +DEBUG_COLLECTABLE: Final = 2 +DEBUG_LEAK: Final = 38 +DEBUG_SAVEALL: Final = 32 +DEBUG_STATS: Final = 1 +DEBUG_UNCOLLECTABLE: Final = 4 + +_CallbackType: TypeAlias = Callable[[Literal["start", "stop"], dict[str, int]], object] + +callbacks: list[_CallbackType] +garbage: list[Any] + +def collect(generation: int = 2) -> int: ... +def disable() -> None: ... +def enable() -> None: ... +def get_count() -> tuple[int, int, int]: ... +def get_debug() -> int: ... +def get_objects(generation: int | None = None) -> list[Any]: ... +def freeze() -> None: ... +def unfreeze() -> None: ... +def get_freeze_count() -> int: ... +def get_referents(*objs: Any) -> list[Any]: ... +def get_referrers(*objs: Any) -> list[Any]: ... +def get_stats() -> list[dict[str, Any]]: ... +def get_threshold() -> tuple[int, int, int]: ... +def is_tracked(obj: Any, /) -> bool: ... + +if sys.version_info >= (3, 9): + def is_finalized(obj: Any, /) -> bool: ... + +def isenabled() -> bool: ... +def set_debug(flags: int, /) -> None: ... +def set_threshold(threshold0: int, threshold1: int = ..., threshold2: int = ..., /) -> None: ... diff --git a/mypy/typeshed/stdlib/genericpath.pyi b/mypy/typeshed/stdlib/genericpath.pyi new file mode 100644 index 000000000000..9d87c48fd520 --- /dev/null +++ b/mypy/typeshed/stdlib/genericpath.pyi @@ -0,0 +1,59 @@ +import os +import sys +from _typeshed import BytesPath, FileDescriptorOrPath, StrOrBytesPath, StrPath, SupportsRichComparisonT +from collections.abc import Sequence +from typing import Literal, overload +from typing_extensions import LiteralString + +__all__ = [ + "commonprefix", + "exists", + "getatime", + "getctime", + "getmtime", + "getsize", + "isdir", + "isfile", + "samefile", + "sameopenfile", + "samestat", +] +if sys.version_info >= (3, 12): + __all__ += ["islink"] +if sys.version_info >= (3, 13): + __all__ += ["isjunction", "isdevdrive", "lexists"] + +# All overloads can return empty string. Ideally, Literal[""] would be a valid +# Iterable[T], so that list[T] | Literal[""] could be used as a return +# type. But because this only works when T is str, we need Sequence[T] instead. +@overload +def commonprefix(m: Sequence[LiteralString]) -> LiteralString: ... +@overload +def commonprefix(m: Sequence[StrPath]) -> str: ... +@overload +def commonprefix(m: Sequence[BytesPath]) -> bytes | Literal[""]: ... +@overload +def commonprefix(m: Sequence[list[SupportsRichComparisonT]]) -> Sequence[SupportsRichComparisonT]: ... +@overload +def commonprefix(m: Sequence[tuple[SupportsRichComparisonT, ...]]) -> Sequence[SupportsRichComparisonT]: ... +def exists(path: FileDescriptorOrPath) -> bool: ... +def getsize(filename: FileDescriptorOrPath) -> int: ... +def isfile(path: FileDescriptorOrPath) -> bool: ... +def isdir(s: FileDescriptorOrPath) -> bool: ... + +if sys.version_info >= (3, 12): + def islink(path: StrOrBytesPath) -> bool: ... + +# These return float if os.stat_float_times() == True, +# but int is a subclass of float. +def getatime(filename: FileDescriptorOrPath) -> float: ... +def getmtime(filename: FileDescriptorOrPath) -> float: ... +def getctime(filename: FileDescriptorOrPath) -> float: ... +def samefile(f1: FileDescriptorOrPath, f2: FileDescriptorOrPath) -> bool: ... +def sameopenfile(fp1: int, fp2: int) -> bool: ... +def samestat(s1: os.stat_result, s2: os.stat_result) -> bool: ... + +if sys.version_info >= (3, 13): + def isjunction(path: StrOrBytesPath) -> bool: ... + def isdevdrive(path: StrOrBytesPath) -> bool: ... + def lexists(path: StrOrBytesPath) -> bool: ... diff --git a/mypy/typeshed/stdlib/getopt.pyi b/mypy/typeshed/stdlib/getopt.pyi new file mode 100644 index 000000000000..bcc8d9750b19 --- /dev/null +++ b/mypy/typeshed/stdlib/getopt.pyi @@ -0,0 +1,15 @@ +from collections.abc import Iterable + +__all__ = ["GetoptError", "error", "getopt", "gnu_getopt"] + +def getopt(args: list[str], shortopts: str, longopts: Iterable[str] | str = []) -> tuple[list[tuple[str, str]], list[str]]: ... +def gnu_getopt( + args: list[str], shortopts: str, longopts: Iterable[str] | str = [] +) -> tuple[list[tuple[str, str]], list[str]]: ... + +class GetoptError(Exception): + msg: str + opt: str + def __init__(self, msg: str, opt: str = "") -> None: ... + +error = GetoptError diff --git a/mypy/typeshed/stdlib/getpass.pyi b/mypy/typeshed/stdlib/getpass.pyi new file mode 100644 index 000000000000..6104e0dedfee --- /dev/null +++ b/mypy/typeshed/stdlib/getpass.pyi @@ -0,0 +1,8 @@ +from typing import TextIO + +__all__ = ["getpass", "getuser", "GetPassWarning"] + +def getpass(prompt: str = "Password: ", stream: TextIO | None = None) -> str: ... +def getuser() -> str: ... + +class GetPassWarning(UserWarning): ... diff --git a/mypy/typeshed/stdlib/gettext.pyi b/mypy/typeshed/stdlib/gettext.pyi new file mode 100644 index 000000000000..d8fd92a00e13 --- /dev/null +++ b/mypy/typeshed/stdlib/gettext.pyi @@ -0,0 +1,171 @@ +import io +import sys +from _typeshed import StrPath +from collections.abc import Callable, Container, Iterable, Sequence +from typing import Any, Final, Literal, Protocol, TypeVar, overload + +__all__ = [ + "NullTranslations", + "GNUTranslations", + "Catalog", + "find", + "translation", + "install", + "textdomain", + "bindtextdomain", + "dgettext", + "dngettext", + "gettext", + "ngettext", + "dnpgettext", + "dpgettext", + "npgettext", + "pgettext", +] + +if sys.version_info < (3, 11): + __all__ += ["bind_textdomain_codeset", "ldgettext", "ldngettext", "lgettext", "lngettext"] + +class _TranslationsReader(Protocol): + def read(self) -> bytes: ... + # optional: + # name: str + +class NullTranslations: + def __init__(self, fp: _TranslationsReader | None = None) -> None: ... + def _parse(self, fp: _TranslationsReader) -> None: ... + def add_fallback(self, fallback: NullTranslations) -> None: ... + def gettext(self, message: str) -> str: ... + def ngettext(self, msgid1: str, msgid2: str, n: int) -> str: ... + def pgettext(self, context: str, message: str) -> str: ... + def npgettext(self, context: str, msgid1: str, msgid2: str, n: int) -> str: ... + def info(self) -> dict[str, str]: ... + def charset(self) -> str | None: ... + if sys.version_info < (3, 11): + def output_charset(self) -> str | None: ... + def set_output_charset(self, charset: str) -> None: ... + def lgettext(self, message: str) -> str: ... + def lngettext(self, msgid1: str, msgid2: str, n: int) -> str: ... + + def install(self, names: Container[str] | None = None) -> None: ... + +class GNUTranslations(NullTranslations): + LE_MAGIC: Final[int] + BE_MAGIC: Final[int] + CONTEXT: str + VERSIONS: Sequence[int] + +@overload +def find( + domain: str, localedir: StrPath | None = None, languages: Iterable[str] | None = None, all: Literal[False] = False +) -> str | None: ... +@overload +def find( + domain: str, localedir: StrPath | None = None, languages: Iterable[str] | None = None, *, all: Literal[True] +) -> list[str]: ... +@overload +def find(domain: str, localedir: StrPath | None, languages: Iterable[str] | None, all: Literal[True]) -> list[str]: ... +@overload +def find(domain: str, localedir: StrPath | None = None, languages: Iterable[str] | None = None, all: bool = False) -> Any: ... + +_NullTranslationsT = TypeVar("_NullTranslationsT", bound=NullTranslations) + +if sys.version_info >= (3, 11): + @overload + def translation( + domain: str, + localedir: StrPath | None = None, + languages: Iterable[str] | None = None, + class_: None = None, + fallback: Literal[False] = False, + ) -> GNUTranslations: ... + @overload + def translation( + domain: str, + localedir: StrPath | None = None, + languages: Iterable[str] | None = None, + *, + class_: Callable[[io.BufferedReader], _NullTranslationsT], + fallback: Literal[False] = False, + ) -> _NullTranslationsT: ... + @overload + def translation( + domain: str, + localedir: StrPath | None, + languages: Iterable[str] | None, + class_: Callable[[io.BufferedReader], _NullTranslationsT], + fallback: Literal[False] = False, + ) -> _NullTranslationsT: ... + @overload + def translation( + domain: str, + localedir: StrPath | None = None, + languages: Iterable[str] | None = None, + class_: Callable[[io.BufferedReader], NullTranslations] | None = None, + fallback: bool = False, + ) -> NullTranslations: ... + def install(domain: str, localedir: StrPath | None = None, *, names: Container[str] | None = None) -> None: ... + +else: + @overload + def translation( + domain: str, + localedir: StrPath | None = None, + languages: Iterable[str] | None = None, + class_: None = None, + fallback: Literal[False] = False, + codeset: str | None = None, + ) -> GNUTranslations: ... + @overload + def translation( + domain: str, + localedir: StrPath | None = None, + languages: Iterable[str] | None = None, + *, + class_: Callable[[io.BufferedReader], _NullTranslationsT], + fallback: Literal[False] = False, + codeset: str | None = None, + ) -> _NullTranslationsT: ... + @overload + def translation( + domain: str, + localedir: StrPath | None, + languages: Iterable[str] | None, + class_: Callable[[io.BufferedReader], _NullTranslationsT], + fallback: Literal[False] = False, + codeset: str | None = None, + ) -> _NullTranslationsT: ... + @overload + def translation( + domain: str, + localedir: StrPath | None = None, + languages: Iterable[str] | None = None, + class_: Callable[[io.BufferedReader], NullTranslations] | None = None, + fallback: bool = False, + codeset: str | None = None, + ) -> NullTranslations: ... + def install( + domain: str, localedir: StrPath | None = None, codeset: str | None = None, names: Container[str] | None = None + ) -> None: ... + +def textdomain(domain: str | None = None) -> str: ... +def bindtextdomain(domain: str, localedir: StrPath | None = None) -> str: ... +def dgettext(domain: str, message: str) -> str: ... +def dngettext(domain: str, msgid1: str, msgid2: str, n: int) -> str: ... +def gettext(message: str) -> str: ... +def ngettext(msgid1: str, msgid2: str, n: int) -> str: ... +def pgettext(context: str, message: str) -> str: ... +def dpgettext(domain: str, context: str, message: str) -> str: ... +def npgettext(context: str, msgid1: str, msgid2: str, n: int) -> str: ... +def dnpgettext(domain: str, context: str, msgid1: str, msgid2: str, n: int) -> str: ... + +if sys.version_info < (3, 11): + def lgettext(message: str) -> str: ... + def ldgettext(domain: str, message: str) -> str: ... + def lngettext(msgid1: str, msgid2: str, n: int) -> str: ... + def ldngettext(domain: str, msgid1: str, msgid2: str, n: int) -> str: ... + def bind_textdomain_codeset(domain: str, codeset: str | None = None) -> str: ... + +Catalog = translation + +def c2py(plural: str) -> Callable[[int], int]: ... diff --git a/mypy/typeshed/stdlib/glob.pyi b/mypy/typeshed/stdlib/glob.pyi new file mode 100644 index 000000000000..03cb5418e256 --- /dev/null +++ b/mypy/typeshed/stdlib/glob.pyi @@ -0,0 +1,50 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Iterator, Sequence +from typing import AnyStr + +__all__ = ["escape", "glob", "iglob"] + +if sys.version_info >= (3, 13): + __all__ += ["translate"] + +def glob0(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... +def glob1(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... + +if sys.version_info >= (3, 11): + def glob( + pathname: AnyStr, + *, + root_dir: StrOrBytesPath | None = None, + dir_fd: int | None = None, + recursive: bool = False, + include_hidden: bool = False, + ) -> list[AnyStr]: ... + def iglob( + pathname: AnyStr, + *, + root_dir: StrOrBytesPath | None = None, + dir_fd: int | None = None, + recursive: bool = False, + include_hidden: bool = False, + ) -> Iterator[AnyStr]: ... + +elif sys.version_info >= (3, 10): + def glob( + pathname: AnyStr, *, root_dir: StrOrBytesPath | None = None, dir_fd: int | None = None, recursive: bool = False + ) -> list[AnyStr]: ... + def iglob( + pathname: AnyStr, *, root_dir: StrOrBytesPath | None = None, dir_fd: int | None = None, recursive: bool = False + ) -> Iterator[AnyStr]: ... + +else: + def glob(pathname: AnyStr, *, recursive: bool = False) -> list[AnyStr]: ... + def iglob(pathname: AnyStr, *, recursive: bool = False) -> Iterator[AnyStr]: ... + +def escape(pathname: AnyStr) -> AnyStr: ... +def has_magic(s: str | bytes) -> bool: ... # undocumented + +if sys.version_info >= (3, 13): + def translate( + pat: str, *, recursive: bool = False, include_hidden: bool = False, seps: Sequence[str] | None = None + ) -> str: ... diff --git a/mypy/typeshed/stdlib/graphlib.pyi b/mypy/typeshed/stdlib/graphlib.pyi new file mode 100644 index 000000000000..1ca8cbe12b08 --- /dev/null +++ b/mypy/typeshed/stdlib/graphlib.pyi @@ -0,0 +1,28 @@ +import sys +from _typeshed import SupportsItems +from collections.abc import Iterable +from typing import Any, Generic, TypeVar, overload + +__all__ = ["TopologicalSorter", "CycleError"] + +_T = TypeVar("_T") + +if sys.version_info >= (3, 11): + from types import GenericAlias + +class TopologicalSorter(Generic[_T]): + @overload + def __init__(self, graph: None = None) -> None: ... + @overload + def __init__(self, graph: SupportsItems[_T, Iterable[_T]]) -> None: ... + def add(self, node: _T, *predecessors: _T) -> None: ... + def prepare(self) -> None: ... + def is_active(self) -> bool: ... + def __bool__(self) -> bool: ... + def done(self, *nodes: _T) -> None: ... + def get_ready(self) -> tuple[_T, ...]: ... + def static_order(self) -> Iterable[_T]: ... + if sys.version_info >= (3, 11): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class CycleError(ValueError): ... diff --git a/mypy/typeshed/stdlib/grp.pyi b/mypy/typeshed/stdlib/grp.pyi new file mode 100644 index 000000000000..965ecece2a56 --- /dev/null +++ b/mypy/typeshed/stdlib/grp.pyi @@ -0,0 +1,22 @@ +import sys +from _typeshed import structseq +from typing import Any, Final, final + +if sys.platform != "win32": + @final + class struct_group(structseq[Any], tuple[str, str | None, int, list[str]]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("gr_name", "gr_passwd", "gr_gid", "gr_mem") + + @property + def gr_name(self) -> str: ... + @property + def gr_passwd(self) -> str | None: ... + @property + def gr_gid(self) -> int: ... + @property + def gr_mem(self) -> list[str]: ... + + def getgrall() -> list[struct_group]: ... + def getgrgid(id: int) -> struct_group: ... + def getgrnam(name: str) -> struct_group: ... diff --git a/mypy/typeshed/stdlib/gzip.pyi b/mypy/typeshed/stdlib/gzip.pyi new file mode 100644 index 000000000000..9b32008dcbf6 --- /dev/null +++ b/mypy/typeshed/stdlib/gzip.pyi @@ -0,0 +1,160 @@ +import _compression +import sys +import zlib +from _typeshed import ReadableBuffer, SizedBuffer, StrOrBytesPath +from io import FileIO +from typing import Final, Literal, Protocol, TextIO, overload +from typing_extensions import TypeAlias + +__all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"] + +_ReadBinaryMode: TypeAlias = Literal["r", "rb"] +_WriteBinaryMode: TypeAlias = Literal["a", "ab", "w", "wb", "x", "xb"] +_OpenTextMode: TypeAlias = Literal["rt", "at", "wt", "xt"] + +READ: Final[object] # undocumented +WRITE: Final[object] # undocumented + +FTEXT: Final[int] # actually Literal[1] # undocumented +FHCRC: Final[int] # actually Literal[2] # undocumented +FEXTRA: Final[int] # actually Literal[4] # undocumented +FNAME: Final[int] # actually Literal[8] # undocumented +FCOMMENT: Final[int] # actually Literal[16] # undocumented + +class _ReadableFileobj(Protocol): + def read(self, n: int, /) -> bytes: ... + def seek(self, n: int, /) -> object: ... + # The following attributes and methods are optional: + # name: str + # mode: str + # def fileno() -> int: ... + +class _WritableFileobj(Protocol): + def write(self, b: bytes, /) -> object: ... + def flush(self) -> object: ... + # The following attributes and methods are optional: + # name: str + # mode: str + # def fileno() -> int: ... + +@overload +def open( + filename: StrOrBytesPath | _ReadableFileobj, + mode: _ReadBinaryMode = "rb", + compresslevel: int = 9, + encoding: None = None, + errors: None = None, + newline: None = None, +) -> GzipFile: ... +@overload +def open( + filename: StrOrBytesPath | _WritableFileobj, + mode: _WriteBinaryMode, + compresslevel: int = 9, + encoding: None = None, + errors: None = None, + newline: None = None, +) -> GzipFile: ... +@overload +def open( + filename: StrOrBytesPath, + mode: _OpenTextMode, + compresslevel: int = 9, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIO: ... +@overload +def open( + filename: StrOrBytesPath | _ReadableFileobj | _WritableFileobj, + mode: str, + compresslevel: int = 9, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> GzipFile | TextIO: ... + +class _PaddedFile: + file: _ReadableFileobj + def __init__(self, f: _ReadableFileobj, prepend: bytes = b"") -> None: ... + def read(self, size: int) -> bytes: ... + def prepend(self, prepend: bytes = b"") -> None: ... + def seek(self, off: int) -> int: ... + def seekable(self) -> bool: ... + +class BadGzipFile(OSError): ... + +class GzipFile(_compression.BaseStream): + myfileobj: FileIO | None + mode: object + name: str + compress: zlib._Compress + fileobj: _ReadableFileobj | _WritableFileobj + @overload + def __init__( + self, + filename: StrOrBytesPath | None, + mode: _ReadBinaryMode, + compresslevel: int = 9, + fileobj: _ReadableFileobj | None = None, + mtime: float | None = None, + ) -> None: ... + @overload + def __init__( + self, + *, + mode: _ReadBinaryMode, + compresslevel: int = 9, + fileobj: _ReadableFileobj | None = None, + mtime: float | None = None, + ) -> None: ... + @overload + def __init__( + self, + filename: StrOrBytesPath | None, + mode: _WriteBinaryMode, + compresslevel: int = 9, + fileobj: _WritableFileobj | None = None, + mtime: float | None = None, + ) -> None: ... + @overload + def __init__( + self, + *, + mode: _WriteBinaryMode, + compresslevel: int = 9, + fileobj: _WritableFileobj | None = None, + mtime: float | None = None, + ) -> None: ... + @overload + def __init__( + self, + filename: StrOrBytesPath | None = None, + mode: str | None = None, + compresslevel: int = 9, + fileobj: _ReadableFileobj | _WritableFileobj | None = None, + mtime: float | None = None, + ) -> None: ... + if sys.version_info < (3, 12): + @property + def filename(self) -> str: ... + + @property + def mtime(self) -> int | None: ... + crc: int + def write(self, data: ReadableBuffer) -> int: ... + def read(self, size: int | None = -1) -> bytes: ... + def read1(self, size: int = -1) -> bytes: ... + def peek(self, n: int) -> bytes: ... + def close(self) -> None: ... + def flush(self, zlib_mode: int = 2) -> None: ... + def fileno(self) -> int: ... + def rewind(self) -> None: ... + def seek(self, offset: int, whence: int = 0) -> int: ... + def readline(self, size: int | None = -1) -> bytes: ... + +class _GzipReader(_compression.DecompressReader): + def __init__(self, fp: _ReadableFileobj) -> None: ... + +def compress(data: SizedBuffer, compresslevel: int = 9, *, mtime: float | None = None) -> bytes: ... +def decompress(data: ReadableBuffer) -> bytes: ... diff --git a/mypy/typeshed/stdlib/hashlib.pyi b/mypy/typeshed/stdlib/hashlib.pyi new file mode 100644 index 000000000000..db6f8635054d --- /dev/null +++ b/mypy/typeshed/stdlib/hashlib.pyi @@ -0,0 +1,104 @@ +import sys +from _blake2 import blake2b as blake2b, blake2s as blake2s +from _hashlib import ( + HASH, + openssl_md5 as md5, + openssl_sha1 as sha1, + openssl_sha224 as sha224, + openssl_sha256 as sha256, + openssl_sha384 as sha384, + openssl_sha512 as sha512, + pbkdf2_hmac as pbkdf2_hmac, + scrypt as scrypt, +) +from _typeshed import ReadableBuffer +from collections.abc import Callable, Set as AbstractSet +from typing import Protocol, type_check_only + +if sys.version_info >= (3, 11): + __all__ = ( + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "blake2b", + "blake2s", + "sha3_224", + "sha3_256", + "sha3_384", + "sha3_512", + "shake_128", + "shake_256", + "new", + "algorithms_guaranteed", + "algorithms_available", + "pbkdf2_hmac", + "file_digest", + ) +else: + __all__ = ( + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "blake2b", + "blake2s", + "sha3_224", + "sha3_256", + "sha3_384", + "sha3_512", + "shake_128", + "shake_256", + "new", + "algorithms_guaranteed", + "algorithms_available", + "pbkdf2_hmac", + ) + +if sys.version_info >= (3, 9): + def new(name: str, data: ReadableBuffer = b"", *, usedforsecurity: bool = ...) -> HASH: ... + from _hashlib import ( + openssl_sha3_224 as sha3_224, + openssl_sha3_256 as sha3_256, + openssl_sha3_384 as sha3_384, + openssl_sha3_512 as sha3_512, + openssl_shake_128 as shake_128, + openssl_shake_256 as shake_256, + ) + +else: + @type_check_only + class _VarLenHash(HASH): + def digest(self, length: int) -> bytes: ... # type: ignore[override] + def hexdigest(self, length: int) -> str: ... # type: ignore[override] + + def new(name: str, data: ReadableBuffer = b"") -> HASH: ... + # At runtime these aren't functions but classes imported from _sha3 + def sha3_224(string: ReadableBuffer = b"") -> HASH: ... + def sha3_256(string: ReadableBuffer = b"") -> HASH: ... + def sha3_384(string: ReadableBuffer = b"") -> HASH: ... + def sha3_512(string: ReadableBuffer = b"") -> HASH: ... + def shake_128(string: ReadableBuffer = b"") -> _VarLenHash: ... + def shake_256(string: ReadableBuffer = b"") -> _VarLenHash: ... + +algorithms_guaranteed: AbstractSet[str] +algorithms_available: AbstractSet[str] + +if sys.version_info >= (3, 11): + class _BytesIOLike(Protocol): + def getbuffer(self) -> ReadableBuffer: ... + + class _FileDigestFileObj(Protocol): + def readinto(self, buf: bytearray, /) -> int: ... + def readable(self) -> bool: ... + + def file_digest( + fileobj: _BytesIOLike | _FileDigestFileObj, digest: str | Callable[[], HASH], /, *, _bufsize: int = 262144 + ) -> HASH: ... + +# Legacy typing-only alias +_Hash = HASH diff --git a/mypy/typeshed/stdlib/heapq.pyi b/mypy/typeshed/stdlib/heapq.pyi new file mode 100644 index 000000000000..7a3aa8b442a5 --- /dev/null +++ b/mypy/typeshed/stdlib/heapq.pyi @@ -0,0 +1,17 @@ +from _heapq import * +from _typeshed import SupportsRichComparison +from collections.abc import Callable, Iterable +from typing import Any, Final, TypeVar + +__all__ = ["heappush", "heappop", "heapify", "heapreplace", "merge", "nlargest", "nsmallest", "heappushpop"] + +_S = TypeVar("_S") + +__about__: Final[str] + +def merge( + *iterables: Iterable[_S], key: Callable[[_S], SupportsRichComparison] | None = None, reverse: bool = False +) -> Iterable[_S]: ... +def nlargest(n: int, iterable: Iterable[_S], key: Callable[[_S], SupportsRichComparison] | None = None) -> list[_S]: ... +def nsmallest(n: int, iterable: Iterable[_S], key: Callable[[_S], SupportsRichComparison] | None = None) -> list[_S]: ... +def _heapify_max(heap: list[Any], /) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/hmac.pyi b/mypy/typeshed/stdlib/hmac.pyi new file mode 100644 index 000000000000..efd649ec39a8 --- /dev/null +++ b/mypy/typeshed/stdlib/hmac.pyi @@ -0,0 +1,42 @@ +import sys +from _hashlib import HASH as _HashlibHash +from _typeshed import ReadableBuffer, SizedBuffer +from collections.abc import Callable +from types import ModuleType +from typing import AnyStr, overload +from typing_extensions import TypeAlias + +_DigestMod: TypeAlias = str | Callable[[], _HashlibHash] | ModuleType + +trans_5C: bytes +trans_36: bytes + +digest_size: None + +# In reality digestmod has a default value, but the function always throws an error +# if the argument is not given, so we pretend it is a required argument. +@overload +def new(key: bytes | bytearray, msg: ReadableBuffer | None, digestmod: _DigestMod) -> HMAC: ... +@overload +def new(key: bytes | bytearray, *, digestmod: _DigestMod) -> HMAC: ... + +class HMAC: + digest_size: int + block_size: int + @property + def name(self) -> str: ... + def __init__(self, key: bytes | bytearray, msg: ReadableBuffer | None = None, digestmod: _DigestMod = "") -> None: ... + def update(self, msg: ReadableBuffer) -> None: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + def copy(self) -> HMAC: ... + +def digest(key: SizedBuffer, msg: ReadableBuffer, digest: _DigestMod) -> bytes: ... + +if sys.version_info >= (3, 9): + from _hashlib import compare_digest as compare_digest +else: + @overload + def compare_digest(a: ReadableBuffer, b: ReadableBuffer, /) -> bool: ... + @overload + def compare_digest(a: AnyStr, b: AnyStr, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/html/__init__.pyi b/mypy/typeshed/stdlib/html/__init__.pyi new file mode 100644 index 000000000000..afba90832535 --- /dev/null +++ b/mypy/typeshed/stdlib/html/__init__.pyi @@ -0,0 +1,6 @@ +from typing import AnyStr + +__all__ = ["escape", "unescape"] + +def escape(s: AnyStr, quote: bool = True) -> AnyStr: ... +def unescape(s: AnyStr) -> AnyStr: ... diff --git a/mypy/typeshed/stdlib/html/entities.pyi b/mypy/typeshed/stdlib/html/entities.pyi new file mode 100644 index 000000000000..be83fd1135be --- /dev/null +++ b/mypy/typeshed/stdlib/html/entities.pyi @@ -0,0 +1,6 @@ +__all__ = ["html5", "name2codepoint", "codepoint2name", "entitydefs"] + +name2codepoint: dict[str, int] +html5: dict[str, str] +codepoint2name: dict[int, str] +entitydefs: dict[str, str] diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi new file mode 100644 index 000000000000..d322ade965d9 --- /dev/null +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -0,0 +1,34 @@ +from _markupbase import ParserBase +from re import Pattern + +__all__ = ["HTMLParser"] + +class HTMLParser(ParserBase): + def __init__(self, *, convert_charrefs: bool = True) -> None: ... + def feed(self, data: str) -> None: ... + def close(self) -> None: ... + def get_starttag_text(self) -> str | None: ... + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: ... + def handle_endtag(self, tag: str) -> None: ... + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: ... + def handle_data(self, data: str) -> None: ... + def handle_entityref(self, name: str) -> None: ... + def handle_charref(self, name: str) -> None: ... + def handle_comment(self, data: str) -> None: ... + def handle_decl(self, decl: str) -> None: ... + def handle_pi(self, data: str) -> None: ... + CDATA_CONTENT_ELEMENTS: tuple[str, ...] + def check_for_whole_start_tag(self, i: int) -> int: ... # undocumented + def clear_cdata_mode(self) -> None: ... # undocumented + def goahead(self, end: bool) -> None: ... # undocumented + def parse_bogus_comment(self, i: int, report: bool = ...) -> int: ... # undocumented + def parse_endtag(self, i: int) -> int: ... # undocumented + def parse_html_declaration(self, i: int) -> int: ... # undocumented + def parse_pi(self, i: int) -> int: ... # undocumented + def parse_starttag(self, i: int) -> int: ... # undocumented + def set_cdata_mode(self, elem: str) -> None: ... # undocumented + rawdata: str # undocumented + cdata_elem: str | None # undocumented + convert_charrefs: bool # undocumented + interesting: Pattern[str] # undocumented + lasttag: str # undocumented diff --git a/mypy/typeshed/stdlib/http/__init__.pyi b/mypy/typeshed/stdlib/http/__init__.pyi new file mode 100644 index 000000000000..d455283948d1 --- /dev/null +++ b/mypy/typeshed/stdlib/http/__init__.pyi @@ -0,0 +1,112 @@ +import sys +from enum import IntEnum + +if sys.version_info >= (3, 11): + from enum import StrEnum + +if sys.version_info >= (3, 11): + __all__ = ["HTTPStatus", "HTTPMethod"] +else: + __all__ = ["HTTPStatus"] + +class HTTPStatus(IntEnum): + @property + def phrase(self) -> str: ... + @property + def description(self) -> str: ... + CONTINUE = 100 + SWITCHING_PROTOCOLS = 101 + PROCESSING = 102 + OK = 200 + CREATED = 201 + ACCEPTED = 202 + NON_AUTHORITATIVE_INFORMATION = 203 + NO_CONTENT = 204 + RESET_CONTENT = 205 + PARTIAL_CONTENT = 206 + MULTI_STATUS = 207 + ALREADY_REPORTED = 208 + IM_USED = 226 + MULTIPLE_CHOICES = 300 + MOVED_PERMANENTLY = 301 + FOUND = 302 + SEE_OTHER = 303 + NOT_MODIFIED = 304 + USE_PROXY = 305 + TEMPORARY_REDIRECT = 307 + PERMANENT_REDIRECT = 308 + BAD_REQUEST = 400 + UNAUTHORIZED = 401 + PAYMENT_REQUIRED = 402 + FORBIDDEN = 403 + NOT_FOUND = 404 + METHOD_NOT_ALLOWED = 405 + NOT_ACCEPTABLE = 406 + PROXY_AUTHENTICATION_REQUIRED = 407 + REQUEST_TIMEOUT = 408 + CONFLICT = 409 + GONE = 410 + LENGTH_REQUIRED = 411 + PRECONDITION_FAILED = 412 + if sys.version_info >= (3, 13): + CONTENT_TOO_LARGE = 413 + REQUEST_ENTITY_TOO_LARGE = 413 + if sys.version_info >= (3, 13): + URI_TOO_LONG = 414 + REQUEST_URI_TOO_LONG = 414 + UNSUPPORTED_MEDIA_TYPE = 415 + if sys.version_info >= (3, 13): + RANGE_NOT_SATISFIABLE = 416 + REQUESTED_RANGE_NOT_SATISFIABLE = 416 + EXPECTATION_FAILED = 417 + if sys.version_info >= (3, 13): + UNPROCESSABLE_CONTENT = 422 + UNPROCESSABLE_ENTITY = 422 + LOCKED = 423 + FAILED_DEPENDENCY = 424 + UPGRADE_REQUIRED = 426 + PRECONDITION_REQUIRED = 428 + TOO_MANY_REQUESTS = 429 + REQUEST_HEADER_FIELDS_TOO_LARGE = 431 + INTERNAL_SERVER_ERROR = 500 + NOT_IMPLEMENTED = 501 + BAD_GATEWAY = 502 + SERVICE_UNAVAILABLE = 503 + GATEWAY_TIMEOUT = 504 + HTTP_VERSION_NOT_SUPPORTED = 505 + VARIANT_ALSO_NEGOTIATES = 506 + INSUFFICIENT_STORAGE = 507 + LOOP_DETECTED = 508 + NOT_EXTENDED = 510 + NETWORK_AUTHENTICATION_REQUIRED = 511 + MISDIRECTED_REQUEST = 421 + UNAVAILABLE_FOR_LEGAL_REASONS = 451 + if sys.version_info >= (3, 9): + EARLY_HINTS = 103 + IM_A_TEAPOT = 418 + TOO_EARLY = 425 + if sys.version_info >= (3, 12): + @property + def is_informational(self) -> bool: ... + @property + def is_success(self) -> bool: ... + @property + def is_redirection(self) -> bool: ... + @property + def is_client_error(self) -> bool: ... + @property + def is_server_error(self) -> bool: ... + +if sys.version_info >= (3, 11): + class HTTPMethod(StrEnum): + @property + def description(self) -> str: ... + CONNECT = "CONNECT" + DELETE = "DELETE" + GET = "GET" + HEAD = "HEAD" + OPTIONS = "OPTIONS" + PATCH = "PATCH" + POST = "POST" + PUT = "PUT" + TRACE = "TRACE" diff --git a/mypy/typeshed/stdlib/http/client.pyi b/mypy/typeshed/stdlib/http/client.pyi new file mode 100644 index 000000000000..3db764ef1e7c --- /dev/null +++ b/mypy/typeshed/stdlib/http/client.pyi @@ -0,0 +1,246 @@ +import email.message +import io +import ssl +import sys +import types +from _typeshed import MaybeNone, ReadableBuffer, SupportsRead, SupportsReadline, WriteableBuffer +from collections.abc import Callable, Iterable, Iterator, Mapping +from socket import socket +from typing import BinaryIO, TypeVar, overload +from typing_extensions import Self, TypeAlias + +__all__ = [ + "HTTPResponse", + "HTTPConnection", + "HTTPException", + "NotConnected", + "UnknownProtocol", + "UnknownTransferEncoding", + "UnimplementedFileMode", + "IncompleteRead", + "InvalidURL", + "ImproperConnectionState", + "CannotSendRequest", + "CannotSendHeader", + "ResponseNotReady", + "BadStatusLine", + "LineTooLong", + "RemoteDisconnected", + "error", + "responses", + "HTTPSConnection", +] + +_DataType: TypeAlias = SupportsRead[bytes] | Iterable[ReadableBuffer] | ReadableBuffer +_T = TypeVar("_T") +_MessageT = TypeVar("_MessageT", bound=email.message.Message) +_HeaderValue: TypeAlias = ReadableBuffer | str | int + +HTTP_PORT: int +HTTPS_PORT: int + +CONTINUE: int +SWITCHING_PROTOCOLS: int +PROCESSING: int + +OK: int +CREATED: int +ACCEPTED: int +NON_AUTHORITATIVE_INFORMATION: int +NO_CONTENT: int +RESET_CONTENT: int +PARTIAL_CONTENT: int +MULTI_STATUS: int +IM_USED: int + +MULTIPLE_CHOICES: int +MOVED_PERMANENTLY: int +FOUND: int +SEE_OTHER: int +NOT_MODIFIED: int +USE_PROXY: int +TEMPORARY_REDIRECT: int + +BAD_REQUEST: int +UNAUTHORIZED: int +PAYMENT_REQUIRED: int +FORBIDDEN: int +NOT_FOUND: int +METHOD_NOT_ALLOWED: int +NOT_ACCEPTABLE: int +PROXY_AUTHENTICATION_REQUIRED: int +REQUEST_TIMEOUT: int +CONFLICT: int +GONE: int +LENGTH_REQUIRED: int +PRECONDITION_FAILED: int +REQUEST_ENTITY_TOO_LARGE: int +REQUEST_URI_TOO_LONG: int +UNSUPPORTED_MEDIA_TYPE: int +REQUESTED_RANGE_NOT_SATISFIABLE: int +EXPECTATION_FAILED: int +UNPROCESSABLE_ENTITY: int +LOCKED: int +FAILED_DEPENDENCY: int +UPGRADE_REQUIRED: int +PRECONDITION_REQUIRED: int +TOO_MANY_REQUESTS: int +REQUEST_HEADER_FIELDS_TOO_LARGE: int + +INTERNAL_SERVER_ERROR: int +NOT_IMPLEMENTED: int +BAD_GATEWAY: int +SERVICE_UNAVAILABLE: int +GATEWAY_TIMEOUT: int +HTTP_VERSION_NOT_SUPPORTED: int +INSUFFICIENT_STORAGE: int +NOT_EXTENDED: int +NETWORK_AUTHENTICATION_REQUIRED: int + +responses: dict[int, str] + +class HTTPMessage(email.message.Message[str, str]): + def getallmatchingheaders(self, name: str) -> list[str]: ... # undocumented + +@overload +def parse_headers(fp: SupportsReadline[bytes], _class: Callable[[], _MessageT]) -> _MessageT: ... +@overload +def parse_headers(fp: SupportsReadline[bytes]) -> HTTPMessage: ... + +class HTTPResponse(io.BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible method definitions in the base classes + msg: HTTPMessage + headers: HTTPMessage + version: int + debuglevel: int + fp: io.BufferedReader + closed: bool + status: int + reason: str + chunked: bool + chunk_left: int | None + length: int | None + will_close: bool + # url is set on instances of the class in urllib.request.AbstractHTTPHandler.do_open + # to match urllib.response.addinfourl's interface. + # It's not set in HTTPResponse.__init__ or any other method on the class + url: str + def __init__(self, sock: socket, debuglevel: int = 0, method: str | None = None, url: str | None = None) -> None: ... + def peek(self, n: int = -1) -> bytes: ... + def read(self, amt: int | None = None) -> bytes: ... + def read1(self, n: int = -1) -> bytes: ... + def readinto(self, b: WriteableBuffer) -> int: ... + def readline(self, limit: int = -1) -> bytes: ... # type: ignore[override] + @overload + def getheader(self, name: str) -> str | None: ... + @overload + def getheader(self, name: str, default: _T) -> str | _T: ... + def getheaders(self) -> list[tuple[str, str]]: ... + def isclosed(self) -> bool: ... + def __iter__(self) -> Iterator[bytes]: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def info(self) -> email.message.Message: ... + def geturl(self) -> str: ... + def getcode(self) -> int: ... + def begin(self) -> None: ... + +class HTTPConnection: + auto_open: int # undocumented + debuglevel: int + default_port: int # undocumented + response_class: type[HTTPResponse] # undocumented + timeout: float | None + host: str + port: int + sock: socket | MaybeNone # can be `None` if `.connect()` was not called + def __init__( + self, + host: str, + port: int | None = None, + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + blocksize: int = 8192, + ) -> None: ... + def request( + self, + method: str, + url: str, + body: _DataType | str | None = None, + headers: Mapping[str, _HeaderValue] = {}, + *, + encode_chunked: bool = False, + ) -> None: ... + def getresponse(self) -> HTTPResponse: ... + def set_debuglevel(self, level: int) -> None: ... + if sys.version_info >= (3, 12): + def get_proxy_response_headers(self) -> HTTPMessage | None: ... + + def set_tunnel(self, host: str, port: int | None = None, headers: Mapping[str, str] | None = None) -> None: ... + def connect(self) -> None: ... + def close(self) -> None: ... + def putrequest(self, method: str, url: str, skip_host: bool = False, skip_accept_encoding: bool = False) -> None: ... + def putheader(self, header: str | bytes, *values: _HeaderValue) -> None: ... + def endheaders(self, message_body: _DataType | None = None, *, encode_chunked: bool = False) -> None: ... + def send(self, data: _DataType | str) -> None: ... + +class HTTPSConnection(HTTPConnection): + # Can be `None` if `.connect()` was not called: + sock: ssl.SSLSocket | MaybeNone + if sys.version_info >= (3, 12): + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + context: ssl.SSLContext | None = None, + blocksize: int = 8192, + ) -> None: ... + else: + def __init__( + self, + host: str, + port: int | None = None, + key_file: str | None = None, + cert_file: str | None = None, + timeout: float | None = ..., + source_address: tuple[str, int] | None = None, + *, + context: ssl.SSLContext | None = None, + check_hostname: bool | None = None, + blocksize: int = 8192, + ) -> None: ... + +class HTTPException(Exception): ... + +error = HTTPException + +class NotConnected(HTTPException): ... +class InvalidURL(HTTPException): ... + +class UnknownProtocol(HTTPException): + def __init__(self, version: str) -> None: ... + +class UnknownTransferEncoding(HTTPException): ... +class UnimplementedFileMode(HTTPException): ... + +class IncompleteRead(HTTPException): + def __init__(self, partial: bytes, expected: int | None = None) -> None: ... + partial: bytes + expected: int | None + +class ImproperConnectionState(HTTPException): ... +class CannotSendRequest(ImproperConnectionState): ... +class CannotSendHeader(ImproperConnectionState): ... +class ResponseNotReady(ImproperConnectionState): ... + +class BadStatusLine(HTTPException): + def __init__(self, line: str) -> None: ... + +class LineTooLong(HTTPException): + def __init__(self, line_type: str) -> None: ... + +class RemoteDisconnected(ConnectionResetError, BadStatusLine): ... diff --git a/mypy/typeshed/stdlib/http/cookiejar.pyi b/mypy/typeshed/stdlib/http/cookiejar.pyi new file mode 100644 index 000000000000..31e1d3fc8378 --- /dev/null +++ b/mypy/typeshed/stdlib/http/cookiejar.pyi @@ -0,0 +1,159 @@ +import sys +from _typeshed import StrPath +from collections.abc import Iterator, Sequence +from http.client import HTTPResponse +from re import Pattern +from typing import ClassVar, TypeVar, overload +from urllib.request import Request + +__all__ = [ + "Cookie", + "CookieJar", + "CookiePolicy", + "DefaultCookiePolicy", + "FileCookieJar", + "LWPCookieJar", + "LoadError", + "MozillaCookieJar", +] + +_T = TypeVar("_T") + +class LoadError(OSError): ... + +class CookieJar: + non_word_re: ClassVar[Pattern[str]] # undocumented + quote_re: ClassVar[Pattern[str]] # undocumented + strict_domain_re: ClassVar[Pattern[str]] # undocumented + domain_re: ClassVar[Pattern[str]] # undocumented + dots_re: ClassVar[Pattern[str]] # undocumented + magic_re: ClassVar[Pattern[str]] # undocumented + def __init__(self, policy: CookiePolicy | None = None) -> None: ... + def add_cookie_header(self, request: Request) -> None: ... + def extract_cookies(self, response: HTTPResponse, request: Request) -> None: ... + def set_policy(self, policy: CookiePolicy) -> None: ... + def make_cookies(self, response: HTTPResponse, request: Request) -> Sequence[Cookie]: ... + def set_cookie(self, cookie: Cookie) -> None: ... + def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: ... + def clear(self, domain: str | None = None, path: str | None = None, name: str | None = None) -> None: ... + def clear_session_cookies(self) -> None: ... + def clear_expired_cookies(self) -> None: ... # undocumented + def __iter__(self) -> Iterator[Cookie]: ... + def __len__(self) -> int: ... + +class FileCookieJar(CookieJar): + filename: str | None + delayload: bool + def __init__(self, filename: StrPath | None = None, delayload: bool = False, policy: CookiePolicy | None = None) -> None: ... + def save(self, filename: str | None = None, ignore_discard: bool = False, ignore_expires: bool = False) -> None: ... + def load(self, filename: str | None = None, ignore_discard: bool = False, ignore_expires: bool = False) -> None: ... + def revert(self, filename: str | None = None, ignore_discard: bool = False, ignore_expires: bool = False) -> None: ... + +class MozillaCookieJar(FileCookieJar): + if sys.version_info < (3, 10): + header: ClassVar[str] # undocumented + +class LWPCookieJar(FileCookieJar): + def as_lwp_str(self, ignore_discard: bool = True, ignore_expires: bool = True) -> str: ... # undocumented + +class CookiePolicy: + netscape: bool + rfc2965: bool + hide_cookie2: bool + def set_ok(self, cookie: Cookie, request: Request) -> bool: ... + def return_ok(self, cookie: Cookie, request: Request) -> bool: ... + def domain_return_ok(self, domain: str, request: Request) -> bool: ... + def path_return_ok(self, path: str, request: Request) -> bool: ... + +class DefaultCookiePolicy(CookiePolicy): + rfc2109_as_netscape: bool + strict_domain: bool + strict_rfc2965_unverifiable: bool + strict_ns_unverifiable: bool + strict_ns_domain: int + strict_ns_set_initial_dollar: bool + strict_ns_set_path: bool + DomainStrictNoDots: ClassVar[int] + DomainStrictNonDomain: ClassVar[int] + DomainRFC2965Match: ClassVar[int] + DomainLiberal: ClassVar[int] + DomainStrict: ClassVar[int] + def __init__( + self, + blocked_domains: Sequence[str] | None = None, + allowed_domains: Sequence[str] | None = None, + netscape: bool = True, + rfc2965: bool = False, + rfc2109_as_netscape: bool | None = None, + hide_cookie2: bool = False, + strict_domain: bool = False, + strict_rfc2965_unverifiable: bool = True, + strict_ns_unverifiable: bool = False, + strict_ns_domain: int = 0, + strict_ns_set_initial_dollar: bool = False, + strict_ns_set_path: bool = False, + secure_protocols: Sequence[str] = ("https", "wss"), + ) -> None: ... + def blocked_domains(self) -> tuple[str, ...]: ... + def set_blocked_domains(self, blocked_domains: Sequence[str]) -> None: ... + def is_blocked(self, domain: str) -> bool: ... + def allowed_domains(self) -> tuple[str, ...] | None: ... + def set_allowed_domains(self, allowed_domains: Sequence[str] | None) -> None: ... + def is_not_allowed(self, domain: str) -> bool: ... + def set_ok_version(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def set_ok_verifiability(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def set_ok_name(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def set_ok_path(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def set_ok_domain(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def set_ok_port(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def return_ok_version(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def return_ok_verifiability(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def return_ok_secure(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def return_ok_expires(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def return_ok_port(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + def return_ok_domain(self, cookie: Cookie, request: Request) -> bool: ... # undocumented + +class Cookie: + version: int | None + name: str + value: str | None + port: str | None + path: str + path_specified: bool + secure: bool + expires: int | None + discard: bool + comment: str | None + comment_url: str | None + rfc2109: bool + port_specified: bool + domain: str # undocumented + domain_specified: bool + domain_initial_dot: bool + def __init__( + self, + version: int | None, + name: str, + value: str | None, # undocumented + port: str | None, + port_specified: bool, + domain: str, + domain_specified: bool, + domain_initial_dot: bool, + path: str, + path_specified: bool, + secure: bool, + expires: int | None, + discard: bool, + comment: str | None, + comment_url: str | None, + rest: dict[str, str], + rfc2109: bool = False, + ) -> None: ... + def has_nonstandard_attr(self, name: str) -> bool: ... + @overload + def get_nonstandard_attr(self, name: str) -> str | None: ... + @overload + def get_nonstandard_attr(self, name: str, default: _T) -> str | _T: ... + def set_nonstandard_attr(self, name: str, value: str) -> None: ... + def is_expired(self, now: int | None = None) -> bool: ... diff --git a/mypy/typeshed/stdlib/http/cookies.pyi b/mypy/typeshed/stdlib/http/cookies.pyi new file mode 100644 index 000000000000..c4af5256b5d8 --- /dev/null +++ b/mypy/typeshed/stdlib/http/cookies.pyi @@ -0,0 +1,60 @@ +import sys +from collections.abc import Iterable, Mapping +from typing import Any, Generic, TypeVar, overload +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["CookieError", "BaseCookie", "SimpleCookie"] + +_DataType: TypeAlias = str | Mapping[str, str | Morsel[Any]] +_T = TypeVar("_T") + +@overload +def _quote(str: None) -> None: ... +@overload +def _quote(str: str) -> str: ... +@overload +def _unquote(str: None) -> None: ... +@overload +def _unquote(str: str) -> str: ... + +class CookieError(Exception): ... + +class Morsel(dict[str, Any], Generic[_T]): + @property + def value(self) -> str: ... + @property + def coded_value(self) -> _T: ... + @property + def key(self) -> str: ... + def __init__(self) -> None: ... + def set(self, key: str, val: str, coded_val: _T) -> None: ... + def setdefault(self, key: str, val: str | None = None) -> str: ... + # The dict update can also get a keywords argument so this is incompatible + @overload # type: ignore[override] + def update(self, values: Mapping[str, str]) -> None: ... + @overload + def update(self, values: Iterable[tuple[str, str]]) -> None: ... + def isReservedKey(self, K: str) -> bool: ... + def output(self, attrs: list[str] | None = None, header: str = "Set-Cookie:") -> str: ... + __str__ = output + def js_output(self, attrs: list[str] | None = None) -> str: ... + def OutputString(self, attrs: list[str] | None = None) -> str: ... + def __eq__(self, morsel: object) -> bool: ... + def __setitem__(self, K: str, V: Any) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class BaseCookie(dict[str, Morsel[_T]], Generic[_T]): + def __init__(self, input: _DataType | None = None) -> None: ... + def value_decode(self, val: str) -> tuple[_T, str]: ... + def value_encode(self, val: _T) -> tuple[_T, str]: ... + def output(self, attrs: list[str] | None = None, header: str = "Set-Cookie:", sep: str = "\r\n") -> str: ... + __str__ = output + def js_output(self, attrs: list[str] | None = None) -> str: ... + def load(self, rawdata: _DataType) -> None: ... + def __setitem__(self, key: str, value: str | Morsel[_T]) -> None: ... + +class SimpleCookie(BaseCookie[str]): ... diff --git a/mypy/typeshed/stdlib/http/server.pyi b/mypy/typeshed/stdlib/http/server.pyi new file mode 100644 index 000000000000..07cde553c1df --- /dev/null +++ b/mypy/typeshed/stdlib/http/server.pyi @@ -0,0 +1,83 @@ +import _socket +import email.message +import io +import socketserver +import sys +from _typeshed import StrPath, SupportsRead, SupportsWrite +from collections.abc import Mapping, Sequence +from typing import Any, AnyStr, BinaryIO, ClassVar + +__all__ = ["HTTPServer", "ThreadingHTTPServer", "BaseHTTPRequestHandler", "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler"] + +class HTTPServer(socketserver.TCPServer): + server_name: str + server_port: int + +class ThreadingHTTPServer(socketserver.ThreadingMixIn, HTTPServer): ... + +class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): + client_address: tuple[str, int] + close_connection: bool + requestline: str + command: str + path: str + request_version: str + headers: email.message.Message + server_version: str + sys_version: str + error_message_format: str + error_content_type: str + protocol_version: str + MessageClass: type + responses: Mapping[int, tuple[str, str]] + default_request_version: str # undocumented + weekdayname: ClassVar[Sequence[str]] # undocumented + monthname: ClassVar[Sequence[str | None]] # undocumented + def handle_one_request(self) -> None: ... + def handle_expect_100(self) -> bool: ... + def send_error(self, code: int, message: str | None = None, explain: str | None = None) -> None: ... + def send_response(self, code: int, message: str | None = None) -> None: ... + def send_header(self, keyword: str, value: str) -> None: ... + def send_response_only(self, code: int, message: str | None = None) -> None: ... + def end_headers(self) -> None: ... + def flush_headers(self) -> None: ... + def log_request(self, code: int | str = "-", size: int | str = "-") -> None: ... + def log_error(self, format: str, *args: Any) -> None: ... + def log_message(self, format: str, *args: Any) -> None: ... + def version_string(self) -> str: ... + def date_time_string(self, timestamp: float | None = None) -> str: ... + def log_date_time_string(self) -> str: ... + def address_string(self) -> str: ... + def parse_request(self) -> bool: ... # undocumented + +class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + extensions_map: dict[str, str] + if sys.version_info >= (3, 12): + index_pages: ClassVar[tuple[str, ...]] + directory: str + def __init__( + self, + request: socketserver._RequestType, + client_address: _socket._RetAddress, + server: socketserver.BaseServer, + *, + directory: str | None = None, + ) -> None: ... + def do_GET(self) -> None: ... + def do_HEAD(self) -> None: ... + def send_head(self) -> io.BytesIO | BinaryIO | None: ... # undocumented + def list_directory(self, path: StrPath) -> io.BytesIO | None: ... # undocumented + def translate_path(self, path: str) -> str: ... # undocumented + def copyfile(self, source: SupportsRead[AnyStr], outputfile: SupportsWrite[AnyStr]) -> None: ... # undocumented + def guess_type(self, path: StrPath) -> str: ... # undocumented + +def executable(path: StrPath) -> bool: ... # undocumented + +class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): + cgi_directories: list[str] + have_fork: bool # undocumented + def do_POST(self) -> None: ... + def is_cgi(self) -> bool: ... # undocumented + def is_executable(self, path: StrPath) -> bool: ... # undocumented + def is_python(self, path: StrPath) -> bool: ... # undocumented + def run_cgi(self) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/imaplib.pyi b/mypy/typeshed/stdlib/imaplib.pyi new file mode 100644 index 000000000000..6a4d8b2e720a --- /dev/null +++ b/mypy/typeshed/stdlib/imaplib.pyi @@ -0,0 +1,168 @@ +import subprocess +import sys +import time +from _typeshed import ReadableBuffer, SizedBuffer +from builtins import list as _list # conflicts with a method named "list" +from collections.abc import Callable +from datetime import datetime +from re import Pattern +from socket import socket as _socket +from ssl import SSLContext, SSLSocket +from types import TracebackType +from typing import IO, Any, Literal, SupportsAbs, SupportsInt +from typing_extensions import Self, TypeAlias + +__all__ = ["IMAP4", "IMAP4_stream", "Internaldate2tuple", "Int2AP", "ParseFlags", "Time2Internaldate", "IMAP4_SSL"] + +# TODO: Commands should use their actual return types, not this type alias. +# E.g. Tuple[Literal["OK"], List[bytes]] +_CommandResults: TypeAlias = tuple[str, list[Any]] + +_AnyResponseData: TypeAlias = list[None] | list[bytes | tuple[bytes, bytes]] + +Commands: dict[str, tuple[str, ...]] + +class IMAP4: + class error(Exception): ... + class abort(error): ... + class readonly(abort): ... + mustquote: Pattern[str] + debug: int + state: str + literal: str | None + tagged_commands: dict[bytes, _list[bytes] | None] + untagged_responses: dict[str, _list[bytes | tuple[bytes, bytes]]] + continuation_response: str + is_readonly: bool + tagnum: int + tagpre: str + tagre: Pattern[str] + welcome: bytes + capabilities: tuple[str, ...] + PROTOCOL_VERSION: str + if sys.version_info >= (3, 9): + def __init__(self, host: str = "", port: int = 143, timeout: float | None = None) -> None: ... + def open(self, host: str = "", port: int = 143, timeout: float | None = None) -> None: ... + else: + def __init__(self, host: str = "", port: int = 143) -> None: ... + def open(self, host: str = "", port: int = 143) -> None: ... + + def __getattr__(self, attr: str) -> Any: ... + host: str + port: int + sock: _socket + file: IO[str] | IO[bytes] + def read(self, size: int) -> bytes: ... + def readline(self) -> bytes: ... + def send(self, data: ReadableBuffer) -> None: ... + def shutdown(self) -> None: ... + def socket(self) -> _socket: ... + def recent(self) -> _CommandResults: ... + def response(self, code: str) -> _CommandResults: ... + def append(self, mailbox: str, flags: str, date_time: str, message: ReadableBuffer) -> str: ... + def authenticate(self, mechanism: str, authobject: Callable[[bytes], bytes | None]) -> tuple[str, str]: ... + def capability(self) -> _CommandResults: ... + def check(self) -> _CommandResults: ... + def close(self) -> _CommandResults: ... + def copy(self, message_set: str, new_mailbox: str) -> _CommandResults: ... + def create(self, mailbox: str) -> _CommandResults: ... + def delete(self, mailbox: str) -> _CommandResults: ... + def deleteacl(self, mailbox: str, who: str) -> _CommandResults: ... + def enable(self, capability: str) -> _CommandResults: ... + def __enter__(self) -> Self: ... + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... + def expunge(self) -> _CommandResults: ... + def fetch(self, message_set: str, message_parts: str) -> tuple[str, _AnyResponseData]: ... + def getacl(self, mailbox: str) -> _CommandResults: ... + def getannotation(self, mailbox: str, entry: str, attribute: str) -> _CommandResults: ... + def getquota(self, root: str) -> _CommandResults: ... + def getquotaroot(self, mailbox: str) -> _CommandResults: ... + def list(self, directory: str = '""', pattern: str = "*") -> tuple[str, _AnyResponseData]: ... + def login(self, user: str, password: str) -> tuple[Literal["OK"], _list[bytes]]: ... + def login_cram_md5(self, user: str, password: str) -> _CommandResults: ... + def logout(self) -> tuple[str, _AnyResponseData]: ... + def lsub(self, directory: str = '""', pattern: str = "*") -> _CommandResults: ... + def myrights(self, mailbox: str) -> _CommandResults: ... + def namespace(self) -> _CommandResults: ... + def noop(self) -> tuple[str, _list[bytes]]: ... + def partial(self, message_num: str, message_part: str, start: str, length: str) -> _CommandResults: ... + def proxyauth(self, user: str) -> _CommandResults: ... + def rename(self, oldmailbox: str, newmailbox: str) -> _CommandResults: ... + def search(self, charset: str | None, *criteria: str) -> _CommandResults: ... + def select(self, mailbox: str = "INBOX", readonly: bool = False) -> tuple[str, _list[bytes | None]]: ... + def setacl(self, mailbox: str, who: str, what: str) -> _CommandResults: ... + def setannotation(self, *args: str) -> _CommandResults: ... + def setquota(self, root: str, limits: str) -> _CommandResults: ... + def sort(self, sort_criteria: str, charset: str, *search_criteria: str) -> _CommandResults: ... + def starttls(self, ssl_context: Any | None = None) -> tuple[Literal["OK"], _list[None]]: ... + def status(self, mailbox: str, names: str) -> _CommandResults: ... + def store(self, message_set: str, command: str, flags: str) -> _CommandResults: ... + def subscribe(self, mailbox: str) -> _CommandResults: ... + def thread(self, threading_algorithm: str, charset: str, *search_criteria: str) -> _CommandResults: ... + def uid(self, command: str, *args: str) -> _CommandResults: ... + def unsubscribe(self, mailbox: str) -> _CommandResults: ... + if sys.version_info >= (3, 9): + def unselect(self) -> _CommandResults: ... + + def xatom(self, name: str, *args: str) -> _CommandResults: ... + def print_log(self) -> None: ... + +class IMAP4_SSL(IMAP4): + if sys.version_info < (3, 12): + keyfile: str + certfile: str + if sys.version_info >= (3, 12): + def __init__( + self, host: str = "", port: int = 993, *, ssl_context: SSLContext | None = None, timeout: float | None = None + ) -> None: ... + elif sys.version_info >= (3, 9): + def __init__( + self, + host: str = "", + port: int = 993, + keyfile: str | None = None, + certfile: str | None = None, + ssl_context: SSLContext | None = None, + timeout: float | None = None, + ) -> None: ... + else: + def __init__( + self, + host: str = "", + port: int = 993, + keyfile: str | None = None, + certfile: str | None = None, + ssl_context: SSLContext | None = None, + ) -> None: ... + sslobj: SSLSocket + file: IO[Any] + if sys.version_info >= (3, 9): + def open(self, host: str = "", port: int | None = 993, timeout: float | None = None) -> None: ... + else: + def open(self, host: str = "", port: int | None = 993) -> None: ... + + def ssl(self) -> SSLSocket: ... + +class IMAP4_stream(IMAP4): + command: str + def __init__(self, command: str) -> None: ... + file: IO[Any] + process: subprocess.Popen[bytes] + writefile: IO[Any] + readfile: IO[Any] + if sys.version_info >= (3, 9): + def open(self, host: str | None = None, port: int | None = None, timeout: float | None = None) -> None: ... + else: + def open(self, host: str | None = None, port: int | None = None) -> None: ... + +class _Authenticator: + mech: Callable[[bytes], bytes | bytearray | memoryview | str | None] + def __init__(self, mechinst: Callable[[bytes], bytes | bytearray | memoryview | str | None]) -> None: ... + def process(self, data: str) -> str: ... + def encode(self, inp: bytes | bytearray | memoryview) -> str: ... + def decode(self, inp: str | SizedBuffer) -> bytes: ... + +def Internaldate2tuple(resp: ReadableBuffer) -> time.struct_time | None: ... +def Int2AP(num: SupportsAbs[SupportsInt]) -> bytes: ... +def ParseFlags(resp: ReadableBuffer) -> tuple[bytes, ...]: ... +def Time2Internaldate(date_time: float | time.struct_time | time._TimeTuple | datetime | str) -> str: ... diff --git a/mypy/typeshed/stdlib/imghdr.pyi b/mypy/typeshed/stdlib/imghdr.pyi new file mode 100644 index 000000000000..6e1b858b8f32 --- /dev/null +++ b/mypy/typeshed/stdlib/imghdr.pyi @@ -0,0 +1,17 @@ +from _typeshed import StrPath +from collections.abc import Callable +from typing import Any, BinaryIO, Protocol, overload + +__all__ = ["what"] + +class _ReadableBinary(Protocol): + def tell(self) -> int: ... + def read(self, size: int, /) -> bytes: ... + def seek(self, offset: int, /) -> Any: ... + +@overload +def what(file: StrPath | _ReadableBinary, h: None = None) -> str | None: ... +@overload +def what(file: Any, h: bytes) -> str | None: ... + +tests: list[Callable[[bytes, BinaryIO | None], str | None]] diff --git a/mypy/typeshed/stdlib/imp.pyi b/mypy/typeshed/stdlib/imp.pyi new file mode 100644 index 000000000000..ee5a0cd7bc72 --- /dev/null +++ b/mypy/typeshed/stdlib/imp.pyi @@ -0,0 +1,62 @@ +import types +from _imp import ( + acquire_lock as acquire_lock, + create_dynamic as create_dynamic, + get_frozen_object as get_frozen_object, + init_frozen as init_frozen, + is_builtin as is_builtin, + is_frozen as is_frozen, + is_frozen_package as is_frozen_package, + lock_held as lock_held, + release_lock as release_lock, +) +from _typeshed import StrPath +from os import PathLike +from types import TracebackType +from typing import IO, Any, Protocol + +SEARCH_ERROR: int +PY_SOURCE: int +PY_COMPILED: int +C_EXTENSION: int +PY_RESOURCE: int +PKG_DIRECTORY: int +C_BUILTIN: int +PY_FROZEN: int +PY_CODERESOURCE: int +IMP_HOOK: int + +def new_module(name: str) -> types.ModuleType: ... +def get_magic() -> bytes: ... +def get_tag() -> str: ... +def cache_from_source(path: StrPath, debug_override: bool | None = None) -> str: ... +def source_from_cache(path: StrPath) -> str: ... +def get_suffixes() -> list[tuple[str, str, int]]: ... + +class NullImporter: + def __init__(self, path: StrPath) -> None: ... + def find_module(self, fullname: Any) -> None: ... + +# Technically, a text file has to support a slightly different set of operations than a binary file, +# but we ignore that here. +class _FileLike(Protocol): + closed: bool + mode: str + def read(self) -> str | bytes: ... + def close(self) -> Any: ... + def __enter__(self) -> Any: ... + def __exit__(self, typ: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None, /) -> Any: ... + +# PathLike doesn't work for the pathname argument here +def load_source(name: str, pathname: str, file: _FileLike | None = None) -> types.ModuleType: ... +def load_compiled(name: str, pathname: str, file: _FileLike | None = None) -> types.ModuleType: ... +def load_package(name: str, path: StrPath) -> types.ModuleType: ... +def load_module(name: str, file: _FileLike | None, filename: str, details: tuple[str, str, int]) -> types.ModuleType: ... + +# IO[Any] is a TextIOWrapper if name is a .py file, and a FileIO otherwise. +def find_module( + name: str, path: None | list[str] | list[PathLike[str]] | list[StrPath] = None +) -> tuple[IO[Any], str, tuple[str, str, int]]: ... +def reload(module: types.ModuleType) -> types.ModuleType: ... +def init_builtin(name: str) -> types.ModuleType | None: ... +def load_dynamic(name: str, path: str, file: Any = None) -> types.ModuleType: ... # file argument is ignored diff --git a/mypy/typeshed/stdlib/importlib/__init__.pyi b/mypy/typeshed/stdlib/importlib/__init__.pyi new file mode 100644 index 000000000000..cab81512e92f --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/__init__.pyi @@ -0,0 +1,15 @@ +import sys +from importlib._bootstrap import __import__ as __import__ +from importlib.abc import Loader +from types import ModuleType + +__all__ = ["__import__", "import_module", "invalidate_caches", "reload"] + +# `importlib.import_module` return type should be kept the same as `builtins.__import__` +def import_module(name: str, package: str | None = None) -> ModuleType: ... + +if sys.version_info < (3, 12): + def find_loader(name: str, path: str | None = None) -> Loader | None: ... + +def invalidate_caches() -> None: ... +def reload(module: ModuleType) -> ModuleType: ... diff --git a/mypy/typeshed/stdlib/importlib/_abc.pyi b/mypy/typeshed/stdlib/importlib/_abc.pyi new file mode 100644 index 000000000000..1a21b9a72cd8 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/_abc.pyi @@ -0,0 +1,15 @@ +import sys +import types +from abc import ABCMeta +from importlib.machinery import ModuleSpec + +if sys.version_info >= (3, 10): + class Loader(metaclass=ABCMeta): + def load_module(self, fullname: str) -> types.ModuleType: ... + if sys.version_info < (3, 12): + def module_repr(self, module: types.ModuleType) -> str: ... + + def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: ... + # Not defined on the actual class for backwards-compatibility reasons, + # but expected in new code. + def exec_module(self, module: types.ModuleType) -> None: ... diff --git a/mypy/typeshed/stdlib/importlib/_bootstrap.pyi b/mypy/typeshed/stdlib/importlib/_bootstrap.pyi new file mode 100644 index 000000000000..02427ff42062 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/_bootstrap.pyi @@ -0,0 +1,2 @@ +from _frozen_importlib import * +from _frozen_importlib import __import__ as __import__, _init_module_attrs as _init_module_attrs diff --git a/mypy/typeshed/stdlib/importlib/_bootstrap_external.pyi b/mypy/typeshed/stdlib/importlib/_bootstrap_external.pyi new file mode 100644 index 000000000000..6210ce7083af --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/_bootstrap_external.pyi @@ -0,0 +1,2 @@ +from _frozen_importlib_external import * +from _frozen_importlib_external import _NamespaceLoader as _NamespaceLoader diff --git a/mypy/typeshed/stdlib/importlib/abc.pyi b/mypy/typeshed/stdlib/importlib/abc.pyi new file mode 100644 index 000000000000..588377d7d871 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/abc.pyi @@ -0,0 +1,173 @@ +import _ast +import sys +import types +from _typeshed import ReadableBuffer, StrPath +from abc import ABCMeta, abstractmethod +from collections.abc import Iterator, Mapping, Sequence +from importlib import _bootstrap_external +from importlib.machinery import ModuleSpec +from io import BufferedReader +from typing import IO, Any, Literal, Protocol, overload, runtime_checkable + +if sys.version_info >= (3, 11): + __all__ = [ + "Loader", + "MetaPathFinder", + "PathEntryFinder", + "ResourceLoader", + "InspectLoader", + "ExecutionLoader", + "FileLoader", + "SourceLoader", + ] + + if sys.version_info < (3, 12): + __all__ += ["Finder", "ResourceReader", "Traversable", "TraversableResources"] + +if sys.version_info >= (3, 10): + from importlib._abc import Loader as Loader +else: + class Loader(metaclass=ABCMeta): + def load_module(self, fullname: str) -> types.ModuleType: ... + def module_repr(self, module: types.ModuleType) -> str: ... + def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: ... + # Not defined on the actual class for backwards-compatibility reasons, + # but expected in new code. + def exec_module(self, module: types.ModuleType) -> None: ... + +if sys.version_info < (3, 12): + class Finder(metaclass=ABCMeta): ... + +class ResourceLoader(Loader): + @abstractmethod + def get_data(self, path: str) -> bytes: ... + +class InspectLoader(Loader): + def is_package(self, fullname: str) -> bool: ... + def get_code(self, fullname: str) -> types.CodeType | None: ... + @abstractmethod + def get_source(self, fullname: str) -> str | None: ... + def exec_module(self, module: types.ModuleType) -> None: ... + @staticmethod + def source_to_code( + data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: ReadableBuffer | StrPath = "" + ) -> types.CodeType: ... + +class ExecutionLoader(InspectLoader): + @abstractmethod + def get_filename(self, fullname: str) -> str: ... + +class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader, metaclass=ABCMeta): # type: ignore[misc] # incompatible definitions of source_to_code in the base classes + def path_mtime(self, path: str) -> float: ... + def set_data(self, path: str, data: bytes) -> None: ... + def get_source(self, fullname: str) -> str | None: ... + def path_stats(self, path: str) -> Mapping[str, Any]: ... + +# The base classes differ starting in 3.10: +if sys.version_info >= (3, 10): + # Please keep in sync with _typeshed.importlib.MetaPathFinderProtocol + class MetaPathFinder(metaclass=ABCMeta): + if sys.version_info < (3, 12): + def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ... + + def invalidate_caches(self) -> None: ... + # Not defined on the actual class, but expected to exist. + def find_spec( + self, fullname: str, path: Sequence[str] | None, target: types.ModuleType | None = ..., / + ) -> ModuleSpec | None: ... + + class PathEntryFinder(metaclass=ABCMeta): + if sys.version_info < (3, 12): + def find_module(self, fullname: str) -> Loader | None: ... + def find_loader(self, fullname: str) -> tuple[Loader | None, Sequence[str]]: ... + + def invalidate_caches(self) -> None: ... + # Not defined on the actual class, but expected to exist. + def find_spec(self, fullname: str, target: types.ModuleType | None = ...) -> ModuleSpec | None: ... + +else: + # Please keep in sync with _typeshed.importlib.MetaPathFinderProtocol + class MetaPathFinder(Finder): + def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ... + def invalidate_caches(self) -> None: ... + # Not defined on the actual class, but expected to exist. + def find_spec( + self, fullname: str, path: Sequence[str] | None, target: types.ModuleType | None = ..., / + ) -> ModuleSpec | None: ... + + class PathEntryFinder(Finder): + def find_module(self, fullname: str) -> Loader | None: ... + def find_loader(self, fullname: str) -> tuple[Loader | None, Sequence[str]]: ... + def invalidate_caches(self) -> None: ... + # Not defined on the actual class, but expected to exist. + def find_spec(self, fullname: str, target: types.ModuleType | None = ...) -> ModuleSpec | None: ... + +class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader, metaclass=ABCMeta): + name: str + path: str + def __init__(self, fullname: str, path: str) -> None: ... + def get_data(self, path: str) -> bytes: ... + def get_filename(self, name: str | None = None) -> str: ... + def load_module(self, name: str | None = None) -> types.ModuleType: ... + +class ResourceReader(metaclass=ABCMeta): + @abstractmethod + def open_resource(self, resource: str) -> IO[bytes]: ... + @abstractmethod + def resource_path(self, resource: str) -> str: ... + if sys.version_info >= (3, 10): + @abstractmethod + def is_resource(self, path: str) -> bool: ... + else: + @abstractmethod + def is_resource(self, name: str) -> bool: ... + + @abstractmethod + def contents(self) -> Iterator[str]: ... + +if sys.version_info >= (3, 9): + @runtime_checkable + class Traversable(Protocol): + @abstractmethod + def is_dir(self) -> bool: ... + @abstractmethod + def is_file(self) -> bool: ... + @abstractmethod + def iterdir(self) -> Iterator[Traversable]: ... + if sys.version_info >= (3, 11): + @abstractmethod + def joinpath(self, *descendants: str) -> Traversable: ... + else: + @abstractmethod + def joinpath(self, child: str, /) -> Traversable: ... + + # The documentation and runtime protocol allows *args, **kwargs arguments, + # but this would mean that all implementers would have to support them, + # which is not the case. + @overload + @abstractmethod + def open(self, mode: Literal["r"] = "r", *, encoding: str | None = None, errors: str | None = None) -> IO[str]: ... + @overload + @abstractmethod + def open(self, mode: Literal["rb"]) -> IO[bytes]: ... + @property + @abstractmethod + def name(self) -> str: ... + if sys.version_info >= (3, 10): + def __truediv__(self, child: str, /) -> Traversable: ... + else: + @abstractmethod + def __truediv__(self, child: str, /) -> Traversable: ... + + @abstractmethod + def read_bytes(self) -> bytes: ... + @abstractmethod + def read_text(self, encoding: str | None = None) -> str: ... + + class TraversableResources(ResourceReader): + @abstractmethod + def files(self) -> Traversable: ... + def open_resource(self, resource: str) -> BufferedReader: ... + def resource_path(self, resource: Any) -> str: ... + def is_resource(self, path: str) -> bool: ... + def contents(self) -> Iterator[str]: ... diff --git a/mypy/typeshed/stdlib/importlib/machinery.pyi b/mypy/typeshed/stdlib/importlib/machinery.pyi new file mode 100644 index 000000000000..bb1a6f93d0e0 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/machinery.pyi @@ -0,0 +1,20 @@ +import sys +from importlib._bootstrap import BuiltinImporter as BuiltinImporter, FrozenImporter as FrozenImporter, ModuleSpec as ModuleSpec +from importlib._bootstrap_external import ( + BYTECODE_SUFFIXES as BYTECODE_SUFFIXES, + DEBUG_BYTECODE_SUFFIXES as DEBUG_BYTECODE_SUFFIXES, + EXTENSION_SUFFIXES as EXTENSION_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES as OPTIMIZED_BYTECODE_SUFFIXES, + SOURCE_SUFFIXES as SOURCE_SUFFIXES, + ExtensionFileLoader as ExtensionFileLoader, + FileFinder as FileFinder, + PathFinder as PathFinder, + SourceFileLoader as SourceFileLoader, + SourcelessFileLoader as SourcelessFileLoader, + WindowsRegistryFinder as WindowsRegistryFinder, +) + +if sys.version_info >= (3, 11): + from importlib._bootstrap_external import NamespaceLoader as NamespaceLoader + +def all_suffixes() -> list[str]: ... diff --git a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi new file mode 100644 index 000000000000..5e26f8987277 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi @@ -0,0 +1,291 @@ +import abc +import pathlib +import sys +import types +from _collections_abc import dict_keys, dict_values +from _typeshed import StrPath +from collections.abc import Iterable, Iterator, Mapping +from email.message import Message +from importlib.abc import MetaPathFinder +from os import PathLike +from pathlib import Path +from re import Pattern +from typing import Any, ClassVar, Generic, NamedTuple, TypeVar, overload +from typing_extensions import Self, TypeAlias + +_T = TypeVar("_T") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") + +__all__ = [ + "Distribution", + "DistributionFinder", + "PackageNotFoundError", + "distribution", + "distributions", + "entry_points", + "files", + "metadata", + "requires", + "version", +] + +if sys.version_info >= (3, 10): + __all__ += ["PackageMetadata", "packages_distributions"] + +if sys.version_info >= (3, 10): + from importlib.metadata._meta import PackageMetadata as PackageMetadata, SimplePath + def packages_distributions() -> Mapping[str, list[str]]: ... + + _SimplePath: TypeAlias = SimplePath + +else: + _SimplePath: TypeAlias = Path + +class PackageNotFoundError(ModuleNotFoundError): + @property + def name(self) -> str: ... # type: ignore[override] + +if sys.version_info >= (3, 13): + _EntryPointBase = object +elif sys.version_info >= (3, 11): + class DeprecatedTuple: + def __getitem__(self, item: int) -> str: ... + + _EntryPointBase = DeprecatedTuple +else: + class _EntryPointBase(NamedTuple): + name: str + value: str + group: str + +class EntryPoint(_EntryPointBase): + pattern: ClassVar[Pattern[str]] + if sys.version_info >= (3, 11): + name: str + value: str + group: str + + def __init__(self, name: str, value: str, group: str) -> None: ... + + def load(self) -> Any: ... # Callable[[], Any] or an importable module + @property + def extras(self) -> list[str]: ... + if sys.version_info >= (3, 9): + @property + def module(self) -> str: ... + @property + def attr(self) -> str: ... + if sys.version_info >= (3, 10): + dist: ClassVar[Distribution | None] + def matches( + self, + *, + name: str = ..., + value: str = ..., + group: str = ..., + module: str = ..., + attr: str = ..., + extras: list[str] = ..., + ) -> bool: ... # undocumented + + def __hash__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + if sys.version_info >= (3, 11): + def __lt__(self, other: object) -> bool: ... + if sys.version_info < (3, 12): + def __iter__(self) -> Iterator[Any]: ... # result of iter((str, Self)), really + +if sys.version_info >= (3, 12): + class EntryPoints(tuple[EntryPoint, ...]): + def __getitem__(self, name: str) -> EntryPoint: ... # type: ignore[override] + def select( + self, + *, + name: str = ..., + value: str = ..., + group: str = ..., + module: str = ..., + attr: str = ..., + extras: list[str] = ..., + ) -> EntryPoints: ... + @property + def names(self) -> set[str]: ... + @property + def groups(self) -> set[str]: ... + +elif sys.version_info >= (3, 10): + class DeprecatedList(list[_T]): ... + + class EntryPoints(DeprecatedList[EntryPoint]): # use as list is deprecated since 3.10 + # int argument is deprecated since 3.10 + def __getitem__(self, name: int | str) -> EntryPoint: ... # type: ignore[override] + def select( + self, + *, + name: str = ..., + value: str = ..., + group: str = ..., + module: str = ..., + attr: str = ..., + extras: list[str] = ..., + ) -> EntryPoints: ... + @property + def names(self) -> set[str]: ... + @property + def groups(self) -> set[str]: ... + +if sys.version_info >= (3, 10) and sys.version_info < (3, 12): + class Deprecated(Generic[_KT, _VT]): + def __getitem__(self, name: _KT) -> _VT: ... + @overload + def get(self, name: _KT) -> _VT | None: ... + @overload + def get(self, name: _KT, default: _T) -> _VT | _T: ... + def __iter__(self) -> Iterator[_KT]: ... + def __contains__(self, *args: object) -> bool: ... + def keys(self) -> dict_keys[_KT, _VT]: ... + def values(self) -> dict_values[_KT, _VT]: ... + + class SelectableGroups(Deprecated[str, EntryPoints], dict[str, EntryPoints]): # use as dict is deprecated since 3.10 + @classmethod + def load(cls, eps: Iterable[EntryPoint]) -> Self: ... + @property + def groups(self) -> set[str]: ... + @property + def names(self) -> set[str]: ... + @overload + def select(self) -> Self: ... + @overload + def select( + self, + *, + name: str = ..., + value: str = ..., + group: str = ..., + module: str = ..., + attr: str = ..., + extras: list[str] = ..., + ) -> EntryPoints: ... + +class PackagePath(pathlib.PurePosixPath): + def read_text(self, encoding: str = "utf-8") -> str: ... + def read_binary(self) -> bytes: ... + def locate(self) -> PathLike[str]: ... + # The following attributes are not defined on PackagePath, but are dynamically added by Distribution.files: + hash: FileHash | None + size: int | None + dist: Distribution + +class FileHash: + mode: str + value: str + def __init__(self, spec: str) -> None: ... + +if sys.version_info >= (3, 12): + class DeprecatedNonAbstract: ... + _distribution_parent = DeprecatedNonAbstract +else: + _distribution_parent = object + +class Distribution(_distribution_parent): + @abc.abstractmethod + def read_text(self, filename: str) -> str | None: ... + @abc.abstractmethod + def locate_file(self, path: StrPath) -> _SimplePath: ... + @classmethod + def from_name(cls, name: str) -> Distribution: ... + @overload + @classmethod + def discover(cls, *, context: DistributionFinder.Context) -> Iterable[Distribution]: ... + @overload + @classmethod + def discover( + cls, *, context: None = None, name: str | None = ..., path: list[str] = ..., **kwargs: Any + ) -> Iterable[Distribution]: ... + @staticmethod + def at(path: StrPath) -> PathDistribution: ... + + if sys.version_info >= (3, 10): + @property + def metadata(self) -> PackageMetadata: ... + @property + def entry_points(self) -> EntryPoints: ... + else: + @property + def metadata(self) -> Message: ... + @property + def entry_points(self) -> list[EntryPoint]: ... + + @property + def version(self) -> str: ... + @property + def files(self) -> list[PackagePath] | None: ... + @property + def requires(self) -> list[str] | None: ... + if sys.version_info >= (3, 10): + @property + def name(self) -> str: ... + if sys.version_info >= (3, 13): + @property + def origin(self) -> types.SimpleNamespace: ... + +class DistributionFinder(MetaPathFinder): + class Context: + name: str | None + def __init__(self, *, name: str | None = ..., path: list[str] = ..., **kwargs: Any) -> None: ... + @property + def path(self) -> list[str]: ... + + @abc.abstractmethod + def find_distributions(self, context: DistributionFinder.Context = ...) -> Iterable[Distribution]: ... + +class MetadataPathFinder(DistributionFinder): + @classmethod + def find_distributions(cls, context: DistributionFinder.Context = ...) -> Iterable[PathDistribution]: ... + if sys.version_info >= (3, 11): + @classmethod + def invalidate_caches(cls) -> None: ... + elif sys.version_info >= (3, 10): + # Yes, this is an instance method that has a parameter named "cls" + def invalidate_caches(cls) -> None: ... + +class PathDistribution(Distribution): + _path: _SimplePath + def __init__(self, path: _SimplePath) -> None: ... + def read_text(self, filename: StrPath) -> str | None: ... + def locate_file(self, path: StrPath) -> _SimplePath: ... + +def distribution(distribution_name: str) -> Distribution: ... +@overload +def distributions(*, context: DistributionFinder.Context) -> Iterable[Distribution]: ... +@overload +def distributions( + *, context: None = None, name: str | None = ..., path: list[str] = ..., **kwargs: Any +) -> Iterable[Distribution]: ... + +if sys.version_info >= (3, 10): + def metadata(distribution_name: str) -> PackageMetadata: ... + +else: + def metadata(distribution_name: str) -> Message: ... + +if sys.version_info >= (3, 12): + def entry_points( + *, name: str = ..., value: str = ..., group: str = ..., module: str = ..., attr: str = ..., extras: list[str] = ... + ) -> EntryPoints: ... + +elif sys.version_info >= (3, 10): + @overload + def entry_points() -> SelectableGroups: ... + @overload + def entry_points( + *, name: str = ..., value: str = ..., group: str = ..., module: str = ..., attr: str = ..., extras: list[str] = ... + ) -> EntryPoints: ... + +else: + def entry_points() -> dict[str, list[EntryPoint]]: ... + +def version(distribution_name: str) -> str: ... +def files(distribution_name: str) -> list[PackagePath] | None: ... +def requires(distribution_name: str) -> list[str] | None: ... diff --git a/mypy/typeshed/stdlib/importlib/metadata/_meta.pyi b/mypy/typeshed/stdlib/importlib/metadata/_meta.pyi new file mode 100644 index 000000000000..9f791dab254f --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/metadata/_meta.pyi @@ -0,0 +1,63 @@ +import sys +from _typeshed import StrPath +from collections.abc import Iterator +from os import PathLike +from typing import Any, Protocol, overload +from typing_extensions import TypeVar + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True, default=Any) + +class PackageMetadata(Protocol): + def __len__(self) -> int: ... + def __contains__(self, item: str) -> bool: ... + def __getitem__(self, key: str) -> str: ... + def __iter__(self) -> Iterator[str]: ... + @property + def json(self) -> dict[str, str | list[str]]: ... + @overload + def get_all(self, name: str, failobj: None = None) -> list[Any] | None: ... + @overload + def get_all(self, name: str, failobj: _T) -> list[Any] | _T: ... + if sys.version_info >= (3, 12): + @overload + def get(self, name: str, failobj: None = None) -> str | None: ... + @overload + def get(self, name: str, failobj: _T) -> _T | str: ... + +if sys.version_info >= (3, 13): + class SimplePath(Protocol): + def joinpath(self, other: StrPath, /) -> SimplePath: ... + def __truediv__(self, other: StrPath, /) -> SimplePath: ... + # Incorrect at runtime + @property + def parent(self) -> PathLike[str]: ... + def read_text(self, encoding: str | None = None) -> str: ... + def read_bytes(self) -> bytes: ... + def exists(self) -> bool: ... + +elif sys.version_info >= (3, 12): + class SimplePath(Protocol[_T_co]): + # At runtime this is defined as taking `str | _T`, but that causes trouble. + # See #11436. + def joinpath(self, other: str, /) -> _T_co: ... + @property + def parent(self) -> _T_co: ... + def read_text(self) -> str: ... + # As with joinpath(), this is annotated as taking `str | _T` at runtime. + def __truediv__(self, other: str, /) -> _T_co: ... + +else: + class SimplePath(Protocol): + # Actually takes only self at runtime, but that's clearly wrong + def joinpath(self, other: Any, /) -> SimplePath: ... + # Not defined as a property at runtime, but it should be + @property + def parent(self) -> Any: ... + def read_text(self) -> str: ... + # There was a bug in `SimplePath` definition in cpython, see #8451 + # Strictly speaking `__div__` was defined in 3.10, not __truediv__, + # but it should have always been `__truediv__`. + # Also, the runtime defines this method as taking no arguments, + # which is obviously wrong. + def __truediv__(self, other: Any, /) -> SimplePath: ... diff --git a/mypy/typeshed/stdlib/importlib/metadata/diagnose.pyi b/mypy/typeshed/stdlib/importlib/metadata/diagnose.pyi new file mode 100644 index 000000000000..565872fd976f --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/metadata/diagnose.pyi @@ -0,0 +1,2 @@ +def inspect(path: str) -> None: ... +def run() -> None: ... diff --git a/mypy/typeshed/stdlib/importlib/readers.pyi b/mypy/typeshed/stdlib/importlib/readers.pyi new file mode 100644 index 000000000000..41d7af966d58 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/readers.pyi @@ -0,0 +1,68 @@ +# On py311+, things are actually defined in importlib.resources.readers, +# and re-exported here, +# but doing it this way leads to less code duplication for us + +import pathlib +import sys +import zipfile +from _typeshed import Incomplete, StrPath +from collections.abc import Iterable, Iterator +from io import BufferedReader +from typing import Literal, NoReturn, TypeVar +from typing_extensions import Never + +if sys.version_info >= (3, 11): + import importlib.resources.abc as abc +else: + import importlib.abc as abc + +if sys.version_info >= (3, 10): + if sys.version_info >= (3, 11): + __all__ = ["FileReader", "ZipReader", "MultiplexedPath", "NamespaceReader"] + + if sys.version_info < (3, 11): + _T = TypeVar("_T") + + def remove_duplicates(items: Iterable[_T]) -> Iterator[_T]: ... + + class FileReader(abc.TraversableResources): + path: pathlib.Path + def __init__(self, loader) -> None: ... + def resource_path(self, resource: StrPath) -> str: ... + def files(self) -> pathlib.Path: ... + + class ZipReader(abc.TraversableResources): + prefix: str + archive: Incomplete + def __init__(self, loader, module: str) -> None: ... + def open_resource(self, resource: str) -> BufferedReader: ... + def is_resource(self, path: StrPath) -> bool: ... + def files(self) -> zipfile.Path: ... + + class MultiplexedPath(abc.Traversable): + def __init__(self, *paths: abc.Traversable) -> None: ... + def iterdir(self) -> Iterator[abc.Traversable]: ... + def read_bytes(self) -> NoReturn: ... + def read_text(self, *args: Never, **kwargs: Never) -> NoReturn: ... # type: ignore[override] + def is_dir(self) -> Literal[True]: ... + def is_file(self) -> Literal[False]: ... + + if sys.version_info >= (3, 12): + def joinpath(self, *descendants: str) -> abc.Traversable: ... + elif sys.version_info >= (3, 11): + def joinpath(self, child: str) -> abc.Traversable: ... # type: ignore[override] + else: + def joinpath(self, child: str) -> abc.Traversable: ... + + if sys.version_info < (3, 12): + __truediv__ = joinpath + + def open(self, *args: Never, **kwargs: Never) -> NoReturn: ... # type: ignore[override] + @property + def name(self) -> str: ... + + class NamespaceReader(abc.TraversableResources): + path: MultiplexedPath + def __init__(self, namespace_path) -> None: ... + def resource_path(self, resource: str) -> str: ... + def files(self) -> MultiplexedPath: ... diff --git a/mypy/typeshed/stdlib/importlib/resources/__init__.pyi b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi new file mode 100644 index 000000000000..f82df8c591fa --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi @@ -0,0 +1,70 @@ +import os +import sys +from collections.abc import Iterator +from contextlib import AbstractContextManager +from pathlib import Path +from types import ModuleType +from typing import Any, BinaryIO, TextIO +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 11): + from importlib.resources._common import Package as Package +else: + Package: TypeAlias = str | ModuleType + +if sys.version_info >= (3, 9): + from importlib.abc import Traversable + +__all__ = ["Package", "contents", "is_resource", "open_binary", "open_text", "path", "read_binary", "read_text"] + +if sys.version_info >= (3, 9): + __all__ += ["as_file", "files"] + +if sys.version_info >= (3, 10): + __all__ += ["ResourceReader"] + +if sys.version_info < (3, 13): + __all__ += ["Resource"] + +if sys.version_info < (3, 11): + Resource: TypeAlias = str | os.PathLike[Any] +elif sys.version_info < (3, 13): + Resource: TypeAlias = str + +if sys.version_info >= (3, 13): + from importlib.resources._common import Anchor as Anchor + + __all__ += ["Anchor"] + + from importlib.resources._functional import ( + contents as contents, + is_resource as is_resource, + open_binary as open_binary, + open_text as open_text, + path as path, + read_binary as read_binary, + read_text as read_text, + ) + +else: + def open_binary(package: Package, resource: Resource) -> BinaryIO: ... + def open_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> TextIO: ... + def read_binary(package: Package, resource: Resource) -> bytes: ... + def read_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> str: ... + def path(package: Package, resource: Resource) -> AbstractContextManager[Path]: ... + def is_resource(package: Package, name: str) -> bool: ... + def contents(package: Package) -> Iterator[str]: ... + +if sys.version_info >= (3, 11): + from importlib.resources._common import as_file as as_file +elif sys.version_info >= (3, 9): + def as_file(path: Traversable) -> AbstractContextManager[Path]: ... + +if sys.version_info >= (3, 11): + from importlib.resources._common import files as files + +elif sys.version_info >= (3, 9): + def files(package: Package) -> Traversable: ... + +if sys.version_info >= (3, 10): + from importlib.abc import ResourceReader as ResourceReader diff --git a/mypy/typeshed/stdlib/importlib/resources/_common.pyi b/mypy/typeshed/stdlib/importlib/resources/_common.pyi new file mode 100644 index 000000000000..f04f70f25e23 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/_common.pyi @@ -0,0 +1,42 @@ +import sys + +# Even though this file is 3.11+ only, Pyright will complain in stubtest for older versions. +if sys.version_info >= (3, 11): + import types + from collections.abc import Callable + from contextlib import AbstractContextManager + from importlib.abc import ResourceReader, Traversable + from pathlib import Path + from typing import overload + from typing_extensions import TypeAlias, deprecated + + Package: TypeAlias = str | types.ModuleType + + if sys.version_info >= (3, 12): + Anchor: TypeAlias = Package + + def package_to_anchor( + func: Callable[[Anchor | None], Traversable] + ) -> Callable[[Anchor | None, Anchor | None], Traversable]: ... + @overload + def files(anchor: Anchor | None = None) -> Traversable: ... + @overload + @deprecated("First parameter to files is renamed to 'anchor'") + def files(package: Anchor | None = None) -> Traversable: ... + + else: + def files(package: Package) -> Traversable: ... + + def get_resource_reader(package: types.ModuleType) -> ResourceReader | None: ... + + if sys.version_info >= (3, 12): + def resolve(cand: Anchor | None) -> types.ModuleType: ... + + else: + def resolve(cand: Package) -> types.ModuleType: ... + + if sys.version_info < (3, 12): + def get_package(package: Package) -> types.ModuleType: ... + + def from_package(package: types.ModuleType) -> Traversable: ... + def as_file(path: Traversable) -> AbstractContextManager[Path]: ... diff --git a/mypy/typeshed/stdlib/importlib/resources/_functional.pyi b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi new file mode 100644 index 000000000000..97e46bdf0a53 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi @@ -0,0 +1,30 @@ +import sys + +# Even though this file is 3.13+ only, Pyright will complain in stubtest for older versions. +if sys.version_info >= (3, 13): + from _typeshed import StrPath + from collections.abc import Iterator + from contextlib import AbstractContextManager + from importlib.resources._common import Anchor + from io import TextIOWrapper + from pathlib import Path + from typing import BinaryIO, overload + from typing_extensions import Unpack + + def open_binary(anchor: Anchor, *path_names: StrPath) -> BinaryIO: ... + @overload + def open_text( + anchor: Anchor, *path_names: Unpack[tuple[StrPath]], encoding: str | None = "utf-8", errors: str | None = "strict" + ) -> TextIOWrapper: ... + @overload + def open_text(anchor: Anchor, *path_names: StrPath, encoding: str | None, errors: str | None = "strict") -> TextIOWrapper: ... + def read_binary(anchor: Anchor, *path_names: StrPath) -> bytes: ... + @overload + def read_text( + anchor: Anchor, *path_names: Unpack[tuple[StrPath]], encoding: str | None = "utf-8", errors: str | None = "strict" + ) -> str: ... + @overload + def read_text(anchor: Anchor, *path_names: StrPath, encoding: str | None, errors: str | None = "strict") -> str: ... + def path(anchor: Anchor, *path_names: StrPath) -> AbstractContextManager[Path]: ... + def is_resource(anchor: Anchor, *path_names: StrPath) -> bool: ... + def contents(anchor: Anchor, *path_names: StrPath) -> Iterator[str]: ... diff --git a/mypy/typeshed/stdlib/importlib/resources/abc.pyi b/mypy/typeshed/stdlib/importlib/resources/abc.pyi new file mode 100644 index 000000000000..ad80605f7c71 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/abc.pyi @@ -0,0 +1,14 @@ +import sys + +if sys.version_info >= (3, 11): + # These are all actually defined in this file on 3.11+, + # and re-exported from importlib.abc, + # but it's much less code duplication for typeshed if we pretend that they're still defined + # in importlib.abc on 3.11+, and re-exported from this file + from importlib.abc import ( + ResourceReader as ResourceReader, + Traversable as Traversable, + TraversableResources as TraversableResources, + ) + + __all__ = ["ResourceReader", "Traversable", "TraversableResources"] diff --git a/mypy/typeshed/stdlib/importlib/resources/readers.pyi b/mypy/typeshed/stdlib/importlib/resources/readers.pyi new file mode 100644 index 000000000000..0ab21fd29114 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/readers.pyi @@ -0,0 +1,14 @@ +# On py311+, things are actually defined here +# and re-exported from importlib.readers, +# but doing it this way leads to less code duplication for us + +import sys +from collections.abc import Iterable, Iterator +from typing import TypeVar + +if sys.version_info >= (3, 11): + from importlib.readers import * + + _T = TypeVar("_T") + + def remove_duplicates(items: Iterable[_T]) -> Iterator[_T]: ... diff --git a/mypy/typeshed/stdlib/importlib/resources/simple.pyi b/mypy/typeshed/stdlib/importlib/resources/simple.pyi new file mode 100644 index 000000000000..c4c758111c2d --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/simple.pyi @@ -0,0 +1,56 @@ +import abc +import sys +from collections.abc import Iterator +from io import TextIOWrapper +from typing import IO, Any, BinaryIO, Literal, NoReturn, overload +from typing_extensions import Never + +if sys.version_info >= (3, 11): + from .abc import Traversable, TraversableResources + + class SimpleReader(abc.ABC): + @property + @abc.abstractmethod + def package(self) -> str: ... + @abc.abstractmethod + def children(self) -> list[SimpleReader]: ... + @abc.abstractmethod + def resources(self) -> list[str]: ... + @abc.abstractmethod + def open_binary(self, resource: str) -> BinaryIO: ... + @property + def name(self) -> str: ... + + class ResourceHandle(Traversable, metaclass=abc.ABCMeta): + parent: ResourceContainer + def __init__(self, parent: ResourceContainer, name: str) -> None: ... + def is_file(self) -> Literal[True]: ... + def is_dir(self) -> Literal[False]: ... + @overload + def open( + self, + mode: Literal["r"] = "r", + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool = False, + write_through: bool = False, + ) -> TextIOWrapper: ... + @overload + def open(self, mode: Literal["rb"]) -> BinaryIO: ... + @overload + def open(self, mode: str) -> IO[Any]: ... + def joinpath(self, name: Never) -> NoReturn: ... # type: ignore[override] + + class ResourceContainer(Traversable, metaclass=abc.ABCMeta): + reader: SimpleReader + def __init__(self, reader: SimpleReader) -> None: ... + def is_dir(self) -> Literal[True]: ... + def is_file(self) -> Literal[False]: ... + def iterdir(self) -> Iterator[ResourceHandle | ResourceContainer]: ... + def open(self, *args: Never, **kwargs: Never) -> NoReturn: ... # type: ignore[override] + if sys.version_info < (3, 12): + def joinpath(self, *descendants: str) -> Traversable: ... + + class TraversableReader(TraversableResources, SimpleReader, metaclass=abc.ABCMeta): + def files(self) -> ResourceContainer: ... diff --git a/mypy/typeshed/stdlib/importlib/simple.pyi b/mypy/typeshed/stdlib/importlib/simple.pyi new file mode 100644 index 000000000000..58d8c6617082 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/simple.pyi @@ -0,0 +1,11 @@ +import sys + +if sys.version_info >= (3, 11): + from .resources.simple import ( + ResourceContainer as ResourceContainer, + ResourceHandle as ResourceHandle, + SimpleReader as SimpleReader, + TraversableReader as TraversableReader, + ) + + __all__ = ["SimpleReader", "ResourceHandle", "ResourceContainer", "TraversableReader"] diff --git a/mypy/typeshed/stdlib/importlib/util.pyi b/mypy/typeshed/stdlib/importlib/util.pyi new file mode 100644 index 000000000000..cc1c98ae4d0e --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/util.pyi @@ -0,0 +1,33 @@ +import importlib.abc +import importlib.machinery +import sys +import types +from _typeshed import ReadableBuffer +from collections.abc import Callable +from importlib._bootstrap import module_from_spec as module_from_spec, spec_from_loader as spec_from_loader +from importlib._bootstrap_external import ( + MAGIC_NUMBER as MAGIC_NUMBER, + cache_from_source as cache_from_source, + decode_source as decode_source, + source_from_cache as source_from_cache, + spec_from_file_location as spec_from_file_location, +) +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") + +if sys.version_info < (3, 12): + def module_for_loader(fxn: Callable[_P, types.ModuleType]) -> Callable[_P, types.ModuleType]: ... + def set_loader(fxn: Callable[_P, types.ModuleType]) -> Callable[_P, types.ModuleType]: ... + def set_package(fxn: Callable[_P, types.ModuleType]) -> Callable[_P, types.ModuleType]: ... + +def resolve_name(name: str, package: str | None) -> str: ... +def find_spec(name: str, package: str | None = None) -> importlib.machinery.ModuleSpec | None: ... + +class LazyLoader(importlib.abc.Loader): + def __init__(self, loader: importlib.abc.Loader) -> None: ... + @classmethod + def factory(cls, loader: importlib.abc.Loader) -> Callable[..., LazyLoader]: ... + def exec_module(self, module: types.ModuleType) -> None: ... + +def source_hash(source_bytes: ReadableBuffer) -> bytes: ... diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi new file mode 100644 index 000000000000..1eb9fc502e12 --- /dev/null +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -0,0 +1,642 @@ +import dis +import enum +import sys +import types +from _typeshed import StrPath +from collections import OrderedDict +from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence, Set as AbstractSet +from types import ( + AsyncGeneratorType, + BuiltinFunctionType, + BuiltinMethodType, + ClassMethodDescriptorType, + CodeType, + CoroutineType, + FrameType, + FunctionType, + GeneratorType, + GetSetDescriptorType, + LambdaType, + MemberDescriptorType, + MethodDescriptorType, + MethodType, + MethodWrapperType, + ModuleType, + TracebackType, + WrapperDescriptorType, +) +from typing import Any, ClassVar, Final, Literal, NamedTuple, Protocol, TypeVar, overload +from typing_extensions import ParamSpec, Self, TypeAlias, TypeGuard, TypeIs + +if sys.version_info >= (3, 11): + __all__ = [ + "ArgInfo", + "Arguments", + "Attribute", + "BlockFinder", + "BoundArguments", + "CORO_CLOSED", + "CORO_CREATED", + "CORO_RUNNING", + "CORO_SUSPENDED", + "CO_ASYNC_GENERATOR", + "CO_COROUTINE", + "CO_GENERATOR", + "CO_ITERABLE_COROUTINE", + "CO_NESTED", + "CO_NEWLOCALS", + "CO_NOFREE", + "CO_OPTIMIZED", + "CO_VARARGS", + "CO_VARKEYWORDS", + "ClassFoundException", + "ClosureVars", + "EndOfBlock", + "FrameInfo", + "FullArgSpec", + "GEN_CLOSED", + "GEN_CREATED", + "GEN_RUNNING", + "GEN_SUSPENDED", + "Parameter", + "Signature", + "TPFLAGS_IS_ABSTRACT", + "Traceback", + "classify_class_attrs", + "cleandoc", + "currentframe", + "findsource", + "formatannotation", + "formatannotationrelativeto", + "formatargvalues", + "get_annotations", + "getabsfile", + "getargs", + "getargvalues", + "getattr_static", + "getblock", + "getcallargs", + "getclasstree", + "getclosurevars", + "getcomments", + "getcoroutinelocals", + "getcoroutinestate", + "getdoc", + "getfile", + "getframeinfo", + "getfullargspec", + "getgeneratorlocals", + "getgeneratorstate", + "getinnerframes", + "getlineno", + "getmembers", + "getmembers_static", + "getmodule", + "getmodulename", + "getmro", + "getouterframes", + "getsource", + "getsourcefile", + "getsourcelines", + "indentsize", + "isabstract", + "isasyncgen", + "isasyncgenfunction", + "isawaitable", + "isbuiltin", + "isclass", + "iscode", + "iscoroutine", + "iscoroutinefunction", + "isdatadescriptor", + "isframe", + "isfunction", + "isgenerator", + "isgeneratorfunction", + "isgetsetdescriptor", + "ismemberdescriptor", + "ismethod", + "ismethoddescriptor", + "ismethodwrapper", + "ismodule", + "isroutine", + "istraceback", + "signature", + "stack", + "trace", + "unwrap", + "walktree", + ] + + if sys.version_info >= (3, 12): + __all__ += [ + "markcoroutinefunction", + "AGEN_CLOSED", + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", + "getasyncgenlocals", + "getasyncgenstate", + "BufferFlags", + ] + +_P = ParamSpec("_P") +_T = TypeVar("_T") +_F = TypeVar("_F", bound=Callable[..., Any]) +_T_cont = TypeVar("_T_cont", contravariant=True) +_V_cont = TypeVar("_V_cont", contravariant=True) + +# +# Types and members +# +class EndOfBlock(Exception): ... + +class BlockFinder: + indent: int + islambda: bool + started: bool + passline: bool + indecorator: bool + decoratorhasargs: bool + last: int + def tokeneater(self, type: int, token: str, srowcol: tuple[int, int], erowcol: tuple[int, int], line: str) -> None: ... + +CO_OPTIMIZED: Final = 1 +CO_NEWLOCALS: Final = 2 +CO_VARARGS: Final = 4 +CO_VARKEYWORDS: Final = 8 +CO_NESTED: Final = 16 +CO_GENERATOR: Final = 32 +CO_NOFREE: Final = 64 +CO_COROUTINE: Final = 128 +CO_ITERABLE_COROUTINE: Final = 256 +CO_ASYNC_GENERATOR: Final = 512 +TPFLAGS_IS_ABSTRACT: Final = 1048576 + +modulesbyfile: dict[str, Any] + +_GetMembersPredicateTypeGuard: TypeAlias = Callable[[Any], TypeGuard[_T]] +_GetMembersPredicateTypeIs: TypeAlias = Callable[[Any], TypeIs[_T]] +_GetMembersPredicate: TypeAlias = Callable[[Any], bool] +_GetMembersReturn: TypeAlias = list[tuple[str, _T]] + +@overload +def getmembers(object: object, predicate: _GetMembersPredicateTypeGuard[_T]) -> _GetMembersReturn[_T]: ... +@overload +def getmembers(object: object, predicate: _GetMembersPredicateTypeIs[_T]) -> _GetMembersReturn[_T]: ... +@overload +def getmembers(object: object, predicate: _GetMembersPredicate | None = None) -> _GetMembersReturn[Any]: ... + +if sys.version_info >= (3, 11): + @overload + def getmembers_static(object: object, predicate: _GetMembersPredicateTypeGuard[_T]) -> _GetMembersReturn[_T]: ... + @overload + def getmembers_static(object: object, predicate: _GetMembersPredicateTypeIs[_T]) -> _GetMembersReturn[_T]: ... + @overload + def getmembers_static(object: object, predicate: _GetMembersPredicate | None = None) -> _GetMembersReturn[Any]: ... + +def getmodulename(path: StrPath) -> str | None: ... +def ismodule(object: object) -> TypeIs[ModuleType]: ... +def isclass(object: object) -> TypeIs[type[Any]]: ... +def ismethod(object: object) -> TypeIs[MethodType]: ... +def isfunction(object: object) -> TypeIs[FunctionType]: ... + +if sys.version_info >= (3, 12): + def markcoroutinefunction(func: _F) -> _F: ... + +@overload +def isgeneratorfunction(obj: Callable[..., Generator[Any, Any, Any]]) -> bool: ... +@overload +def isgeneratorfunction(obj: Callable[_P, Any]) -> TypeGuard[Callable[_P, GeneratorType[Any, Any, Any]]]: ... +@overload +def isgeneratorfunction(obj: object) -> TypeGuard[Callable[..., GeneratorType[Any, Any, Any]]]: ... +@overload +def iscoroutinefunction(obj: Callable[..., Coroutine[Any, Any, Any]]) -> bool: ... +@overload +def iscoroutinefunction(obj: Callable[_P, Awaitable[_T]]) -> TypeGuard[Callable[_P, CoroutineType[Any, Any, _T]]]: ... +@overload +def iscoroutinefunction(obj: Callable[_P, object]) -> TypeGuard[Callable[_P, CoroutineType[Any, Any, Any]]]: ... +@overload +def iscoroutinefunction(obj: object) -> TypeGuard[Callable[..., CoroutineType[Any, Any, Any]]]: ... +def isgenerator(object: object) -> TypeIs[GeneratorType[Any, Any, Any]]: ... +def iscoroutine(object: object) -> TypeIs[CoroutineType[Any, Any, Any]]: ... +def isawaitable(object: object) -> TypeIs[Awaitable[Any]]: ... +@overload +def isasyncgenfunction(obj: Callable[..., AsyncGenerator[Any, Any]]) -> bool: ... +@overload +def isasyncgenfunction(obj: Callable[_P, Any]) -> TypeGuard[Callable[_P, AsyncGeneratorType[Any, Any]]]: ... +@overload +def isasyncgenfunction(obj: object) -> TypeGuard[Callable[..., AsyncGeneratorType[Any, Any]]]: ... + +class _SupportsSet(Protocol[_T_cont, _V_cont]): + def __set__(self, instance: _T_cont, value: _V_cont, /) -> None: ... + +class _SupportsDelete(Protocol[_T_cont]): + def __delete__(self, instance: _T_cont, /) -> None: ... + +def isasyncgen(object: object) -> TypeIs[AsyncGeneratorType[Any, Any]]: ... +def istraceback(object: object) -> TypeIs[TracebackType]: ... +def isframe(object: object) -> TypeIs[FrameType]: ... +def iscode(object: object) -> TypeIs[CodeType]: ... +def isbuiltin(object: object) -> TypeIs[BuiltinFunctionType]: ... + +if sys.version_info >= (3, 11): + def ismethodwrapper(object: object) -> TypeIs[MethodWrapperType]: ... + +def isroutine( + object: object, +) -> TypeIs[ + FunctionType + | LambdaType + | MethodType + | BuiltinFunctionType + | BuiltinMethodType + | WrapperDescriptorType + | MethodDescriptorType + | ClassMethodDescriptorType +]: ... +def ismethoddescriptor(object: object) -> TypeIs[MethodDescriptorType]: ... +def ismemberdescriptor(object: object) -> TypeIs[MemberDescriptorType]: ... +def isabstract(object: object) -> bool: ... +def isgetsetdescriptor(object: object) -> TypeIs[GetSetDescriptorType]: ... +def isdatadescriptor(object: object) -> TypeIs[_SupportsSet[Any, Any] | _SupportsDelete[Any]]: ... + +# +# Retrieving source code +# +_SourceObjectType: TypeAlias = ( + ModuleType | type[Any] | MethodType | FunctionType | TracebackType | FrameType | CodeType | Callable[..., Any] +) + +def findsource(object: _SourceObjectType) -> tuple[list[str], int]: ... +def getabsfile(object: _SourceObjectType, _filename: str | None = None) -> str: ... + +# Special-case the two most common input types here +# to avoid the annoyingly vague `Sequence[str]` return type +@overload +def getblock(lines: list[str]) -> list[str]: ... +@overload +def getblock(lines: tuple[str, ...]) -> tuple[str, ...]: ... +@overload +def getblock(lines: Sequence[str]) -> Sequence[str]: ... +def getdoc(object: object) -> str | None: ... +def getcomments(object: object) -> str | None: ... +def getfile(object: _SourceObjectType) -> str: ... +def getmodule(object: object, _filename: str | None = None) -> ModuleType | None: ... +def getsourcefile(object: _SourceObjectType) -> str | None: ... +def getsourcelines(object: _SourceObjectType) -> tuple[list[str], int]: ... +def getsource(object: _SourceObjectType) -> str: ... +def cleandoc(doc: str) -> str: ... +def indentsize(line: str) -> int: ... + +_IntrospectableCallable: TypeAlias = Callable[..., Any] + +# +# Introspecting callables with the Signature object +# +if sys.version_info >= (3, 10): + def signature( + obj: _IntrospectableCallable, + *, + follow_wrapped: bool = True, + globals: Mapping[str, Any] | None = None, + locals: Mapping[str, Any] | None = None, + eval_str: bool = False, + ) -> Signature: ... + +else: + def signature(obj: _IntrospectableCallable, *, follow_wrapped: bool = True) -> Signature: ... + +class _void: ... +class _empty: ... + +class Signature: + def __init__( + self, parameters: Sequence[Parameter] | None = None, *, return_annotation: Any = ..., __validate_parameters__: bool = True + ) -> None: ... + empty = _empty + @property + def parameters(self) -> types.MappingProxyType[str, Parameter]: ... + @property + def return_annotation(self) -> Any: ... + def bind(self, *args: Any, **kwargs: Any) -> BoundArguments: ... + def bind_partial(self, *args: Any, **kwargs: Any) -> BoundArguments: ... + def replace(self, *, parameters: Sequence[Parameter] | type[_void] | None = ..., return_annotation: Any = ...) -> Self: ... + __replace__ = replace + if sys.version_info >= (3, 10): + @classmethod + def from_callable( + cls, + obj: _IntrospectableCallable, + *, + follow_wrapped: bool = True, + globals: Mapping[str, Any] | None = None, + locals: Mapping[str, Any] | None = None, + eval_str: bool = False, + ) -> Self: ... + else: + @classmethod + def from_callable(cls, obj: _IntrospectableCallable, *, follow_wrapped: bool = True) -> Self: ... + if sys.version_info >= (3, 13): + def format(self, *, max_width: int | None = None) -> str: ... + + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +if sys.version_info >= (3, 10): + def get_annotations( + obj: Callable[..., object] | type[Any] | ModuleType, + *, + globals: Mapping[str, Any] | None = None, + locals: Mapping[str, Any] | None = None, + eval_str: bool = False, + ) -> dict[str, Any]: ... + +# The name is the same as the enum's name in CPython +class _ParameterKind(enum.IntEnum): + POSITIONAL_ONLY = 0 + POSITIONAL_OR_KEYWORD = 1 + VAR_POSITIONAL = 2 + KEYWORD_ONLY = 3 + VAR_KEYWORD = 4 + + @property + def description(self) -> str: ... + +if sys.version_info >= (3, 12): + AGEN_CREATED: Final = "AGEN_CREATED" + AGEN_RUNNING: Final = "AGEN_RUNNING" + AGEN_SUSPENDED: Final = "AGEN_SUSPENDED" + AGEN_CLOSED: Final = "AGEN_CLOSED" + + def getasyncgenstate( + agen: AsyncGenerator[Any, Any] + ) -> Literal["AGEN_CREATED", "AGEN_RUNNING", "AGEN_SUSPENDED", "AGEN_CLOSED"]: ... + def getasyncgenlocals(agen: AsyncGeneratorType[Any, Any]) -> dict[str, Any]: ... + +class Parameter: + def __init__(self, name: str, kind: _ParameterKind, *, default: Any = ..., annotation: Any = ...) -> None: ... + empty = _empty + + POSITIONAL_ONLY: ClassVar[Literal[_ParameterKind.POSITIONAL_ONLY]] + POSITIONAL_OR_KEYWORD: ClassVar[Literal[_ParameterKind.POSITIONAL_OR_KEYWORD]] + VAR_POSITIONAL: ClassVar[Literal[_ParameterKind.VAR_POSITIONAL]] + KEYWORD_ONLY: ClassVar[Literal[_ParameterKind.KEYWORD_ONLY]] + VAR_KEYWORD: ClassVar[Literal[_ParameterKind.VAR_KEYWORD]] + @property + def name(self) -> str: ... + @property + def default(self) -> Any: ... + @property + def kind(self) -> _ParameterKind: ... + @property + def annotation(self) -> Any: ... + def replace( + self, + *, + name: str | type[_void] = ..., + kind: _ParameterKind | type[_void] = ..., + default: Any = ..., + annotation: Any = ..., + ) -> Self: ... + if sys.version_info >= (3, 13): + __replace__ = replace + + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +class BoundArguments: + arguments: OrderedDict[str, Any] + @property + def args(self) -> tuple[Any, ...]: ... + @property + def kwargs(self) -> dict[str, Any]: ... + @property + def signature(self) -> Signature: ... + def __init__(self, signature: Signature, arguments: OrderedDict[str, Any]) -> None: ... + def apply_defaults(self) -> None: ... + def __eq__(self, other: object) -> bool: ... + +# +# Classes and functions +# + +# TODO: The actual return type should be list[_ClassTreeItem] but mypy doesn't +# seem to be supporting this at the moment: +# _ClassTreeItem = list[_ClassTreeItem] | Tuple[type, Tuple[type, ...]] +def getclasstree(classes: list[type], unique: bool = False) -> list[Any]: ... +def walktree(classes: list[type], children: Mapping[type[Any], list[type]], parent: type[Any] | None) -> list[Any]: ... + +class Arguments(NamedTuple): + args: list[str] + varargs: str | None + varkw: str | None + +def getargs(co: CodeType) -> Arguments: ... + +if sys.version_info < (3, 11): + class ArgSpec(NamedTuple): + args: list[str] + varargs: str | None + keywords: str | None + defaults: tuple[Any, ...] + + def getargspec(func: object) -> ArgSpec: ... + +class FullArgSpec(NamedTuple): + args: list[str] + varargs: str | None + varkw: str | None + defaults: tuple[Any, ...] | None + kwonlyargs: list[str] + kwonlydefaults: dict[str, Any] | None + annotations: dict[str, Any] + +def getfullargspec(func: object) -> FullArgSpec: ... + +class ArgInfo(NamedTuple): + args: list[str] + varargs: str | None + keywords: str | None + locals: dict[str, Any] + +def getargvalues(frame: FrameType) -> ArgInfo: ... +def formatannotation(annotation: object, base_module: str | None = None) -> str: ... +def formatannotationrelativeto(object: object) -> Callable[[object], str]: ... + +if sys.version_info < (3, 11): + def formatargspec( + args: list[str], + varargs: str | None = None, + varkw: str | None = None, + defaults: tuple[Any, ...] | None = None, + kwonlyargs: Sequence[str] | None = (), + kwonlydefaults: Mapping[str, Any] | None = {}, + annotations: Mapping[str, Any] = {}, + formatarg: Callable[[str], str] = ..., + formatvarargs: Callable[[str], str] = ..., + formatvarkw: Callable[[str], str] = ..., + formatvalue: Callable[[Any], str] = ..., + formatreturns: Callable[[Any], str] = ..., + formatannotation: Callable[[Any], str] = ..., + ) -> str: ... + +def formatargvalues( + args: list[str], + varargs: str | None, + varkw: str | None, + locals: Mapping[str, Any] | None, + formatarg: Callable[[str], str] | None = ..., + formatvarargs: Callable[[str], str] | None = ..., + formatvarkw: Callable[[str], str] | None = ..., + formatvalue: Callable[[Any], str] | None = ..., +) -> str: ... +def getmro(cls: type) -> tuple[type, ...]: ... +def getcallargs(func: Callable[_P, Any], /, *args: _P.args, **kwds: _P.kwargs) -> dict[str, Any]: ... + +class ClosureVars(NamedTuple): + nonlocals: Mapping[str, Any] + globals: Mapping[str, Any] + builtins: Mapping[str, Any] + unbound: AbstractSet[str] + +def getclosurevars(func: _IntrospectableCallable) -> ClosureVars: ... +def unwrap(func: Callable[..., Any], *, stop: Callable[[Callable[..., Any]], Any] | None = None) -> Any: ... + +# +# The interpreter stack +# + +if sys.version_info >= (3, 11): + class _Traceback(NamedTuple): + filename: str + lineno: int + function: str + code_context: list[str] | None + index: int | None # type: ignore[assignment] + + class Traceback(_Traceback): + positions: dis.Positions | None + def __new__( + cls, + filename: str, + lineno: int, + function: str, + code_context: list[str] | None, + index: int | None, + *, + positions: dis.Positions | None = None, + ) -> Self: ... + + class _FrameInfo(NamedTuple): + frame: FrameType + filename: str + lineno: int + function: str + code_context: list[str] | None + index: int | None # type: ignore[assignment] + + class FrameInfo(_FrameInfo): + positions: dis.Positions | None + def __new__( + cls, + frame: FrameType, + filename: str, + lineno: int, + function: str, + code_context: list[str] | None, + index: int | None, + *, + positions: dis.Positions | None = None, + ) -> Self: ... + +else: + class Traceback(NamedTuple): + filename: str + lineno: int + function: str + code_context: list[str] | None + index: int | None # type: ignore[assignment] + + class FrameInfo(NamedTuple): + frame: FrameType + filename: str + lineno: int + function: str + code_context: list[str] | None + index: int | None # type: ignore[assignment] + +def getframeinfo(frame: FrameType | TracebackType, context: int = 1) -> Traceback: ... +def getouterframes(frame: Any, context: int = 1) -> list[FrameInfo]: ... +def getinnerframes(tb: TracebackType, context: int = 1) -> list[FrameInfo]: ... +def getlineno(frame: FrameType) -> int: ... +def currentframe() -> FrameType | None: ... +def stack(context: int = 1) -> list[FrameInfo]: ... +def trace(context: int = 1) -> list[FrameInfo]: ... + +# +# Fetching attributes statically +# + +def getattr_static(obj: object, attr: str, default: Any | None = ...) -> Any: ... + +# +# Current State of Generators and Coroutines +# + +GEN_CREATED: Final = "GEN_CREATED" +GEN_RUNNING: Final = "GEN_RUNNING" +GEN_SUSPENDED: Final = "GEN_SUSPENDED" +GEN_CLOSED: Final = "GEN_CLOSED" + +def getgeneratorstate( + generator: Generator[Any, Any, Any] +) -> Literal["GEN_CREATED", "GEN_RUNNING", "GEN_SUSPENDED", "GEN_CLOSED"]: ... + +CORO_CREATED: Final = "CORO_CREATED" +CORO_RUNNING: Final = "CORO_RUNNING" +CORO_SUSPENDED: Final = "CORO_SUSPENDED" +CORO_CLOSED: Final = "CORO_CLOSED" + +def getcoroutinestate( + coroutine: Coroutine[Any, Any, Any] +) -> Literal["CORO_CREATED", "CORO_RUNNING", "CORO_SUSPENDED", "CORO_CLOSED"]: ... +def getgeneratorlocals(generator: Generator[Any, Any, Any]) -> dict[str, Any]: ... +def getcoroutinelocals(coroutine: Coroutine[Any, Any, Any]) -> dict[str, Any]: ... + +# Create private type alias to avoid conflict with symbol of same +# name created in Attribute class. +_Object: TypeAlias = object + +class Attribute(NamedTuple): + name: str + kind: Literal["class method", "static method", "property", "method", "data"] + defining_class: type + object: _Object + +def classify_class_attrs(cls: type) -> list[Attribute]: ... + +if sys.version_info >= (3, 9): + class ClassFoundException(Exception): ... + +if sys.version_info >= (3, 12): + class BufferFlags(enum.IntFlag): + SIMPLE = 0 + WRITABLE = 1 + FORMAT = 4 + ND = 8 + STRIDES = 24 + C_CONTIGUOUS = 56 + F_CONTIGUOUS = 88 + ANY_CONTIGUOUS = 152 + INDIRECT = 280 + CONTIG = 9 + CONTIG_RO = 8 + STRIDED = 25 + STRIDED_RO = 24 + RECORDS = 29 + RECORDS_RO = 28 + FULL = 285 + FULL_RO = 284 + READ = 256 + WRITE = 512 diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi new file mode 100644 index 000000000000..5c26cb245a2f --- /dev/null +++ b/mypy/typeshed/stdlib/io.pyi @@ -0,0 +1,60 @@ +import abc +import sys +from _io import ( + DEFAULT_BUFFER_SIZE as DEFAULT_BUFFER_SIZE, + BlockingIOError as BlockingIOError, + BufferedRandom as BufferedRandom, + BufferedReader as BufferedReader, + BufferedRWPair as BufferedRWPair, + BufferedWriter as BufferedWriter, + BytesIO as BytesIO, + FileIO as FileIO, + IncrementalNewlineDecoder as IncrementalNewlineDecoder, + StringIO as StringIO, + TextIOWrapper as TextIOWrapper, + _BufferedIOBase, + _IOBase, + _RawIOBase, + _TextIOBase, + _WrappedBuffer as _WrappedBuffer, # used elsewhere in typeshed + open as open, + open_code as open_code, +) +from typing import Final + +__all__ = [ + "BlockingIOError", + "open", + "open_code", + "IOBase", + "RawIOBase", + "FileIO", + "BytesIO", + "StringIO", + "BufferedIOBase", + "BufferedReader", + "BufferedWriter", + "BufferedRWPair", + "BufferedRandom", + "TextIOBase", + "TextIOWrapper", + "UnsupportedOperation", + "SEEK_SET", + "SEEK_CUR", + "SEEK_END", +] + +if sys.version_info >= (3, 11): + from _io import text_encoding as text_encoding + + __all__ += ["DEFAULT_BUFFER_SIZE", "IncrementalNewlineDecoder", "text_encoding"] + +SEEK_SET: Final = 0 +SEEK_CUR: Final = 1 +SEEK_END: Final = 2 + +class UnsupportedOperation(OSError, ValueError): ... +class IOBase(_IOBase, metaclass=abc.ABCMeta): ... +class RawIOBase(_RawIOBase, IOBase): ... +class BufferedIOBase(_BufferedIOBase, IOBase): ... +class TextIOBase(_TextIOBase, IOBase): ... diff --git a/mypy/typeshed/stdlib/ipaddress.pyi b/mypy/typeshed/stdlib/ipaddress.pyi new file mode 100644 index 000000000000..f5cee43d6b32 --- /dev/null +++ b/mypy/typeshed/stdlib/ipaddress.pyi @@ -0,0 +1,234 @@ +import sys +from collections.abc import Iterable, Iterator +from typing import Any, Final, Generic, Literal, TypeVar, overload +from typing_extensions import Self, TypeAlias + +# Undocumented length constants +IPV4LENGTH: Final = 32 +IPV6LENGTH: Final = 128 + +_A = TypeVar("_A", IPv4Address, IPv6Address) +_N = TypeVar("_N", IPv4Network, IPv6Network) + +_RawIPAddress: TypeAlias = int | str | bytes | IPv4Address | IPv6Address +_RawNetworkPart: TypeAlias = IPv4Network | IPv6Network | IPv4Interface | IPv6Interface + +def ip_address(address: _RawIPAddress) -> IPv4Address | IPv6Address: ... +def ip_network( + address: _RawIPAddress | _RawNetworkPart | tuple[_RawIPAddress] | tuple[_RawIPAddress, int], strict: bool = True +) -> IPv4Network | IPv6Network: ... +def ip_interface( + address: _RawIPAddress | _RawNetworkPart | tuple[_RawIPAddress] | tuple[_RawIPAddress, int] +) -> IPv4Interface | IPv6Interface: ... + +class _IPAddressBase: + @property + def compressed(self) -> str: ... + @property + def exploded(self) -> str: ... + @property + def reverse_pointer(self) -> str: ... + @property + def version(self) -> int: ... + +class _BaseAddress(_IPAddressBase): + def __init__(self, address: object) -> None: ... + def __add__(self, other: int) -> Self: ... + def __hash__(self) -> int: ... + def __int__(self) -> int: ... + def __sub__(self, other: int) -> Self: ... + if sys.version_info >= (3, 9): + def __format__(self, fmt: str) -> str: ... + + def __eq__(self, other: object) -> bool: ... + def __lt__(self, other: Self) -> bool: ... + if sys.version_info >= (3, 11): + def __ge__(self, other: Self) -> bool: ... + def __gt__(self, other: Self) -> bool: ... + def __le__(self, other: Self) -> bool: ... + else: + def __ge__(self, other: Self, NotImplemented: Any = ...) -> bool: ... + def __gt__(self, other: Self, NotImplemented: Any = ...) -> bool: ... + def __le__(self, other: Self, NotImplemented: Any = ...) -> bool: ... + +class _BaseNetwork(_IPAddressBase, Generic[_A]): + network_address: _A + netmask: _A + def __init__(self, address: object, strict: bool = ...) -> None: ... + def __contains__(self, other: Any) -> bool: ... + def __getitem__(self, n: int) -> _A: ... + def __iter__(self) -> Iterator[_A]: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def __lt__(self, other: Self) -> bool: ... + if sys.version_info >= (3, 11): + def __ge__(self, other: Self) -> bool: ... + def __gt__(self, other: Self) -> bool: ... + def __le__(self, other: Self) -> bool: ... + else: + def __ge__(self, other: Self, NotImplemented: Any = ...) -> bool: ... + def __gt__(self, other: Self, NotImplemented: Any = ...) -> bool: ... + def __le__(self, other: Self, NotImplemented: Any = ...) -> bool: ... + + def address_exclude(self, other: Self) -> Iterator[Self]: ... + @property + def broadcast_address(self) -> _A: ... + def compare_networks(self, other: Self) -> int: ... + def hosts(self) -> Iterator[_A]: ... + @property + def is_global(self) -> bool: ... + @property + def is_link_local(self) -> bool: ... + @property + def is_loopback(self) -> bool: ... + @property + def is_multicast(self) -> bool: ... + @property + def is_private(self) -> bool: ... + @property + def is_reserved(self) -> bool: ... + @property + def is_unspecified(self) -> bool: ... + @property + def num_addresses(self) -> int: ... + def overlaps(self, other: _BaseNetwork[IPv4Address] | _BaseNetwork[IPv6Address]) -> bool: ... + @property + def prefixlen(self) -> int: ... + def subnet_of(self, other: Self) -> bool: ... + def supernet_of(self, other: Self) -> bool: ... + def subnets(self, prefixlen_diff: int = 1, new_prefix: int | None = None) -> Iterator[Self]: ... + def supernet(self, prefixlen_diff: int = 1, new_prefix: int | None = None) -> Self: ... + @property + def with_hostmask(self) -> str: ... + @property + def with_netmask(self) -> str: ... + @property + def with_prefixlen(self) -> str: ... + @property + def hostmask(self) -> _A: ... + +class _BaseV4: + @property + def version(self) -> Literal[4]: ... + @property + def max_prefixlen(self) -> Literal[32]: ... + +class IPv4Address(_BaseV4, _BaseAddress): + @property + def is_global(self) -> bool: ... + @property + def is_link_local(self) -> bool: ... + @property + def is_loopback(self) -> bool: ... + @property + def is_multicast(self) -> bool: ... + @property + def is_private(self) -> bool: ... + @property + def is_reserved(self) -> bool: ... + @property + def is_unspecified(self) -> bool: ... + @property + def packed(self) -> bytes: ... + if sys.version_info >= (3, 13): + @property + def ipv6_mapped(self) -> IPv6Address: ... + +class IPv4Network(_BaseV4, _BaseNetwork[IPv4Address]): ... + +class IPv4Interface(IPv4Address): + netmask: IPv4Address + network: IPv4Network + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + @property + def hostmask(self) -> IPv4Address: ... + @property + def ip(self) -> IPv4Address: ... + @property + def with_hostmask(self) -> str: ... + @property + def with_netmask(self) -> str: ... + @property + def with_prefixlen(self) -> str: ... + +class _BaseV6: + @property + def version(self) -> Literal[6]: ... + @property + def max_prefixlen(self) -> Literal[128]: ... + +class IPv6Address(_BaseV6, _BaseAddress): + @property + def is_global(self) -> bool: ... + @property + def is_link_local(self) -> bool: ... + @property + def is_loopback(self) -> bool: ... + @property + def is_multicast(self) -> bool: ... + @property + def is_private(self) -> bool: ... + @property + def is_reserved(self) -> bool: ... + @property + def is_unspecified(self) -> bool: ... + @property + def packed(self) -> bytes: ... + @property + def ipv4_mapped(self) -> IPv4Address | None: ... + @property + def is_site_local(self) -> bool: ... + @property + def sixtofour(self) -> IPv4Address | None: ... + @property + def teredo(self) -> tuple[IPv4Address, IPv4Address] | None: ... + if sys.version_info >= (3, 9): + @property + def scope_id(self) -> str | None: ... + + def __hash__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + +class IPv6Network(_BaseV6, _BaseNetwork[IPv6Address]): + @property + def is_site_local(self) -> bool: ... + +class IPv6Interface(IPv6Address): + netmask: IPv6Address + network: IPv6Network + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + @property + def hostmask(self) -> IPv6Address: ... + @property + def ip(self) -> IPv6Address: ... + @property + def with_hostmask(self) -> str: ... + @property + def with_netmask(self) -> str: ... + @property + def with_prefixlen(self) -> str: ... + +def v4_int_to_packed(address: int) -> bytes: ... +def v6_int_to_packed(address: int) -> bytes: ... + +# Third overload is technically incorrect, but convenient when first and last are return values of ip_address() +@overload +def summarize_address_range(first: IPv4Address, last: IPv4Address) -> Iterator[IPv4Network]: ... +@overload +def summarize_address_range(first: IPv6Address, last: IPv6Address) -> Iterator[IPv6Network]: ... +@overload +def summarize_address_range( + first: IPv4Address | IPv6Address, last: IPv4Address | IPv6Address +) -> Iterator[IPv4Network] | Iterator[IPv6Network]: ... +def collapse_addresses(addresses: Iterable[_N]) -> Iterator[_N]: ... +@overload +def get_mixed_type_key(obj: _A) -> tuple[int, _A]: ... +@overload +def get_mixed_type_key(obj: IPv4Network) -> tuple[int, IPv4Address, IPv4Address]: ... +@overload +def get_mixed_type_key(obj: IPv6Network) -> tuple[int, IPv6Address, IPv6Address]: ... + +class AddressValueError(ValueError): ... +class NetmaskValueError(ValueError): ... diff --git a/mypy/typeshed/stdlib/itertools.pyi b/mypy/typeshed/stdlib/itertools.pyi new file mode 100644 index 000000000000..013c3cba120f --- /dev/null +++ b/mypy/typeshed/stdlib/itertools.pyi @@ -0,0 +1,336 @@ +import sys +from _typeshed import MaybeNone +from collections.abc import Callable, Iterable, Iterator +from typing import Any, Generic, Literal, SupportsComplex, SupportsFloat, SupportsIndex, SupportsInt, TypeVar, overload +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_S = TypeVar("_S") +_N = TypeVar("_N", int, float, SupportsFloat, SupportsInt, SupportsIndex, SupportsComplex) +_T_co = TypeVar("_T_co", covariant=True) +_S_co = TypeVar("_S_co", covariant=True) +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_T3 = TypeVar("_T3") +_T4 = TypeVar("_T4") +_T5 = TypeVar("_T5") +_T6 = TypeVar("_T6") +_T7 = TypeVar("_T7") +_T8 = TypeVar("_T8") +_T9 = TypeVar("_T9") +_T10 = TypeVar("_T10") + +_Step: TypeAlias = SupportsFloat | SupportsInt | SupportsIndex | SupportsComplex + +_Predicate: TypeAlias = Callable[[_T], object] + +# Technically count can take anything that implements a number protocol and has an add method +# but we can't enforce the add method +class count(Generic[_N]): + @overload + def __new__(cls) -> count[int]: ... + @overload + def __new__(cls, start: _N, step: _Step = ...) -> count[_N]: ... + @overload + def __new__(cls, *, step: _N) -> count[_N]: ... + def __next__(self) -> _N: ... + def __iter__(self) -> Self: ... + +class cycle(Generic[_T]): + def __init__(self, iterable: Iterable[_T], /) -> None: ... + def __next__(self) -> _T: ... + def __iter__(self) -> Self: ... + +class repeat(Generic[_T]): + @overload + def __init__(self, object: _T) -> None: ... + @overload + def __init__(self, object: _T, times: int) -> None: ... + def __next__(self) -> _T: ... + def __iter__(self) -> Self: ... + def __length_hint__(self) -> int: ... + +class accumulate(Generic[_T]): + @overload + def __init__(self, iterable: Iterable[_T], func: None = None, *, initial: _T | None = ...) -> None: ... + @overload + def __init__(self, iterable: Iterable[_S], func: Callable[[_T, _S], _T], *, initial: _T | None = ...) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + +class chain(Generic[_T]): + def __init__(self, *iterables: Iterable[_T]) -> None: ... + def __next__(self) -> _T: ... + def __iter__(self) -> Self: ... + @classmethod + # We use type[Any] and not type[_S] to not lose the type inference from __iterable + def from_iterable(cls: type[Any], iterable: Iterable[Iterable[_S]], /) -> chain[_S]: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class compress(Generic[_T]): + def __init__(self, data: Iterable[_T], selectors: Iterable[Any]) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + +class dropwhile(Generic[_T]): + def __init__(self, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + +class filterfalse(Generic[_T]): + def __init__(self, predicate: _Predicate[_T] | None, iterable: Iterable[_T], /) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + +class groupby(Generic[_T_co, _S_co]): + @overload + def __new__(cls, iterable: Iterable[_T1], key: None = None) -> groupby[_T1, _T1]: ... + @overload + def __new__(cls, iterable: Iterable[_T1], key: Callable[[_T1], _T2]) -> groupby[_T2, _T1]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> tuple[_T_co, Iterator[_S_co]]: ... + +class islice(Generic[_T]): + @overload + def __init__(self, iterable: Iterable[_T], stop: int | None, /) -> None: ... + @overload + def __init__(self, iterable: Iterable[_T], start: int | None, stop: int | None, step: int | None = ..., /) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + +class starmap(Generic[_T_co]): + def __new__(cls, function: Callable[..., _T], iterable: Iterable[Iterable[Any]], /) -> starmap[_T]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +class takewhile(Generic[_T]): + def __init__(self, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T: ... + +def tee(iterable: Iterable[_T], n: int = 2, /) -> tuple[Iterator[_T], ...]: ... + +class zip_longest(Generic[_T_co]): + # one iterable (fillvalue doesn't matter) + @overload + def __new__(cls, iter1: Iterable[_T1], /, *, fillvalue: object = ...) -> zip_longest[tuple[_T1]]: ... + # two iterables + @overload + # In the overloads without fillvalue, all of the tuple members could theoretically be None, + # but we return Any instead to avoid false positives for code where we know one of the iterables + # is longer. + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /) -> zip_longest[tuple[_T1 | MaybeNone, _T2 | MaybeNone]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /, *, fillvalue: _T + ) -> zip_longest[tuple[_T1 | _T, _T2 | _T]]: ... + # three iterables + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], / + ) -> zip_longest[tuple[_T1 | MaybeNone, _T2 | MaybeNone, _T3 | MaybeNone]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /, *, fillvalue: _T + ) -> zip_longest[tuple[_T1 | _T, _T2 | _T, _T3 | _T]]: ... + # four iterables + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], / + ) -> zip_longest[tuple[_T1 | MaybeNone, _T2 | MaybeNone, _T3 | MaybeNone, _T4 | MaybeNone]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], /, *, fillvalue: _T + ) -> zip_longest[tuple[_T1 | _T, _T2 | _T, _T3 | _T, _T4 | _T]]: ... + # five iterables + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], iter5: Iterable[_T5], / + ) -> zip_longest[tuple[_T1 | MaybeNone, _T2 | MaybeNone, _T3 | MaybeNone, _T4 | MaybeNone, _T5 | MaybeNone]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + /, + *, + fillvalue: _T, + ) -> zip_longest[tuple[_T1 | _T, _T2 | _T, _T3 | _T, _T4 | _T, _T5 | _T]]: ... + # six or more iterables + @overload + def __new__( + cls, + iter1: Iterable[_T], + iter2: Iterable[_T], + iter3: Iterable[_T], + iter4: Iterable[_T], + iter5: Iterable[_T], + iter6: Iterable[_T], + /, + *iterables: Iterable[_T], + ) -> zip_longest[tuple[_T | MaybeNone, ...]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T], + iter2: Iterable[_T], + iter3: Iterable[_T], + iter4: Iterable[_T], + iter5: Iterable[_T], + iter6: Iterable[_T], + /, + *iterables: Iterable[_T], + fillvalue: _T, + ) -> zip_longest[tuple[_T, ...]]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +class product(Generic[_T_co]): + @overload + def __new__(cls, iter1: Iterable[_T1], /) -> product[tuple[_T1]]: ... + @overload + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /) -> product[tuple[_T1, _T2]]: ... + @overload + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /) -> product[tuple[_T1, _T2, _T3]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], / + ) -> product[tuple[_T1, _T2, _T3, _T4]]: ... + @overload + def __new__( + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], iter5: Iterable[_T5], / + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + iter8: Iterable[_T8], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7, _T8]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + iter8: Iterable[_T8], + iter9: Iterable[_T9], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7, _T8, _T9]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + iter8: Iterable[_T8], + iter9: Iterable[_T9], + iter10: Iterable[_T10], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7, _T8, _T9, _T10]]: ... + @overload + def __new__(cls, *iterables: Iterable[_T1], repeat: int = 1) -> product[tuple[_T1, ...]]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +class permutations(Generic[_T_co]): + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> permutations[tuple[_T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[3]) -> permutations[tuple[_T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[4]) -> permutations[tuple[_T, _T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[5]) -> permutations[tuple[_T, _T, _T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: int | None = ...) -> permutations[tuple[_T, ...]]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +class combinations(Generic[_T_co]): + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations[tuple[_T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[3]) -> combinations[tuple[_T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[4]) -> combinations[tuple[_T, _T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[5]) -> combinations[tuple[_T, _T, _T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: int) -> combinations[tuple[_T, ...]]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +class combinations_with_replacement(Generic[_T_co]): + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations_with_replacement[tuple[_T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[3]) -> combinations_with_replacement[tuple[_T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[4]) -> combinations_with_replacement[tuple[_T, _T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: Literal[5]) -> combinations_with_replacement[tuple[_T, _T, _T, _T, _T]]: ... + @overload + def __new__(cls, iterable: Iterable[_T], r: int) -> combinations_with_replacement[tuple[_T, ...]]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +if sys.version_info >= (3, 10): + class pairwise(Generic[_T_co]): + def __new__(cls, iterable: Iterable[_T], /) -> pairwise[tuple[_T, _T]]: ... + def __iter__(self) -> Self: ... + def __next__(self) -> _T_co: ... + +if sys.version_info >= (3, 12): + class batched(Generic[_T_co]): + if sys.version_info >= (3, 13): + def __new__(cls, iterable: Iterable[_T_co], n: int, *, strict: bool = False) -> Self: ... + else: + def __new__(cls, iterable: Iterable[_T_co], n: int) -> Self: ... + + def __iter__(self) -> Self: ... + def __next__(self) -> tuple[_T_co, ...]: ... diff --git a/mypy/typeshed/stdlib/json/__init__.pyi b/mypy/typeshed/stdlib/json/__init__.pyi new file mode 100644 index 000000000000..63e9718ee151 --- /dev/null +++ b/mypy/typeshed/stdlib/json/__init__.pyi @@ -0,0 +1,61 @@ +from _typeshed import SupportsRead, SupportsWrite +from collections.abc import Callable +from typing import Any + +from .decoder import JSONDecodeError as JSONDecodeError, JSONDecoder as JSONDecoder +from .encoder import JSONEncoder as JSONEncoder + +__all__ = ["dump", "dumps", "load", "loads", "JSONDecoder", "JSONDecodeError", "JSONEncoder"] + +def dumps( + obj: Any, + *, + skipkeys: bool = False, + ensure_ascii: bool = True, + check_circular: bool = True, + allow_nan: bool = True, + cls: type[JSONEncoder] | None = None, + indent: None | int | str = None, + separators: tuple[str, str] | None = None, + default: Callable[[Any], Any] | None = None, + sort_keys: bool = False, + **kwds: Any, +) -> str: ... +def dump( + obj: Any, + fp: SupportsWrite[str], + *, + skipkeys: bool = False, + ensure_ascii: bool = True, + check_circular: bool = True, + allow_nan: bool = True, + cls: type[JSONEncoder] | None = None, + indent: None | int | str = None, + separators: tuple[str, str] | None = None, + default: Callable[[Any], Any] | None = None, + sort_keys: bool = False, + **kwds: Any, +) -> None: ... +def loads( + s: str | bytes | bytearray, + *, + cls: type[JSONDecoder] | None = None, + object_hook: Callable[[dict[Any, Any]], Any] | None = None, + parse_float: Callable[[str], Any] | None = None, + parse_int: Callable[[str], Any] | None = None, + parse_constant: Callable[[str], Any] | None = None, + object_pairs_hook: Callable[[list[tuple[Any, Any]]], Any] | None = None, + **kwds: Any, +) -> Any: ... +def load( + fp: SupportsRead[str | bytes], + *, + cls: type[JSONDecoder] | None = None, + object_hook: Callable[[dict[Any, Any]], Any] | None = None, + parse_float: Callable[[str], Any] | None = None, + parse_int: Callable[[str], Any] | None = None, + parse_constant: Callable[[str], Any] | None = None, + object_pairs_hook: Callable[[list[tuple[Any, Any]]], Any] | None = None, + **kwds: Any, +) -> Any: ... +def detect_encoding(b: bytes | bytearray) -> str: ... # undocumented diff --git a/mypy/typeshed/stdlib/json/decoder.pyi b/mypy/typeshed/stdlib/json/decoder.pyi new file mode 100644 index 000000000000..8debfe6cd65a --- /dev/null +++ b/mypy/typeshed/stdlib/json/decoder.pyi @@ -0,0 +1,32 @@ +from collections.abc import Callable +from typing import Any + +__all__ = ["JSONDecoder", "JSONDecodeError"] + +class JSONDecodeError(ValueError): + msg: str + doc: str + pos: int + lineno: int + colno: int + def __init__(self, msg: str, doc: str, pos: int) -> None: ... + +class JSONDecoder: + object_hook: Callable[[dict[str, Any]], Any] + parse_float: Callable[[str], Any] + parse_int: Callable[[str], Any] + parse_constant: Callable[[str], Any] + strict: bool + object_pairs_hook: Callable[[list[tuple[str, Any]]], Any] + def __init__( + self, + *, + object_hook: Callable[[dict[str, Any]], Any] | None = None, + parse_float: Callable[[str], Any] | None = None, + parse_int: Callable[[str], Any] | None = None, + parse_constant: Callable[[str], Any] | None = None, + strict: bool = True, + object_pairs_hook: Callable[[list[tuple[str, Any]]], Any] | None = None, + ) -> None: ... + def decode(self, s: str, _w: Callable[..., Any] = ...) -> Any: ... # _w is undocumented + def raw_decode(self, s: str, idx: int = 0) -> tuple[Any, int]: ... diff --git a/mypy/typeshed/stdlib/json/encoder.pyi b/mypy/typeshed/stdlib/json/encoder.pyi new file mode 100644 index 000000000000..aa4a3bdf61d4 --- /dev/null +++ b/mypy/typeshed/stdlib/json/encoder.pyi @@ -0,0 +1,40 @@ +from collections.abc import Callable, Iterator +from re import Pattern +from typing import Any, Final + +ESCAPE: Final[Pattern[str]] +ESCAPE_ASCII: Final[Pattern[str]] +HAS_UTF8: Final[Pattern[bytes]] +ESCAPE_DCT: Final[dict[str, str]] +INFINITY: Final[float] + +def py_encode_basestring(s: str) -> str: ... # undocumented +def py_encode_basestring_ascii(s: str) -> str: ... # undocumented +def encode_basestring(s: str, /) -> str: ... # undocumented +def encode_basestring_ascii(s: str, /) -> str: ... # undocumented + +class JSONEncoder: + item_separator: str + key_separator: str + + skipkeys: bool + ensure_ascii: bool + check_circular: bool + allow_nan: bool + sort_keys: bool + indent: int | str + def __init__( + self, + *, + skipkeys: bool = False, + ensure_ascii: bool = True, + check_circular: bool = True, + allow_nan: bool = True, + sort_keys: bool = False, + indent: int | str | None = None, + separators: tuple[str, str] | None = None, + default: Callable[..., Any] | None = None, + ) -> None: ... + def default(self, o: Any) -> Any: ... + def encode(self, o: Any) -> str: ... + def iterencode(self, o: Any, _one_shot: bool = False) -> Iterator[str]: ... diff --git a/mypy/typeshed/stdlib/json/scanner.pyi b/mypy/typeshed/stdlib/json/scanner.pyi new file mode 100644 index 000000000000..f3b98996b752 --- /dev/null +++ b/mypy/typeshed/stdlib/json/scanner.pyi @@ -0,0 +1,3 @@ +from _json import make_scanner as make_scanner + +__all__ = ["make_scanner"] diff --git a/mypy/typeshed/stdlib/json/tool.pyi b/mypy/typeshed/stdlib/json/tool.pyi new file mode 100644 index 000000000000..7e7363e797f3 --- /dev/null +++ b/mypy/typeshed/stdlib/json/tool.pyi @@ -0,0 +1 @@ +def main() -> None: ... diff --git a/mypy/typeshed/stdlib/keyword.pyi b/mypy/typeshed/stdlib/keyword.pyi new file mode 100644 index 000000000000..960dfd2fa155 --- /dev/null +++ b/mypy/typeshed/stdlib/keyword.pyi @@ -0,0 +1,21 @@ +import sys +from collections.abc import Sequence +from typing import Final + +if sys.version_info >= (3, 9): + __all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"] +else: + __all__ = ["iskeyword", "kwlist"] + +def iskeyword(s: str, /) -> bool: ... + +# a list at runtime, but you're not meant to mutate it; +# type it as a sequence +kwlist: Final[Sequence[str]] + +if sys.version_info >= (3, 9): + def issoftkeyword(s: str, /) -> bool: ... + + # a list at runtime, but you're not meant to mutate it; + # type it as a sequence + softkwlist: Final[Sequence[str]] diff --git a/stubs/3.2/docutils/parsers/rst/__init__.py b/mypy/typeshed/stdlib/lib2to3/__init__.pyi similarity index 100% rename from stubs/3.2/docutils/parsers/rst/__init__.py rename to mypy/typeshed/stdlib/lib2to3/__init__.pyi diff --git a/mypy/typeshed/stdlib/lib2to3/btm_matcher.pyi b/mypy/typeshed/stdlib/lib2to3/btm_matcher.pyi new file mode 100644 index 000000000000..4c87b664eb20 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/btm_matcher.pyi @@ -0,0 +1,28 @@ +from _typeshed import Incomplete, SupportsGetItem +from collections import defaultdict +from collections.abc import Iterable + +from .fixer_base import BaseFix +from .pytree import Leaf, Node + +class BMNode: + count: Incomplete + transition_table: Incomplete + fixers: Incomplete + id: Incomplete + content: str + def __init__(self) -> None: ... + +class BottomMatcher: + match: Incomplete + root: Incomplete + nodes: Incomplete + fixers: Incomplete + logger: Incomplete + def __init__(self) -> None: ... + def add_fixer(self, fixer: BaseFix) -> None: ... + def add(self, pattern: SupportsGetItem[int | slice, Incomplete] | None, start: BMNode) -> list[BMNode]: ... + def run(self, leaves: Iterable[Leaf]) -> defaultdict[BaseFix, list[Node | Leaf]]: ... + def print_ac(self) -> None: ... + +def type_repr(type_num: int) -> str | int: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixer_base.pyi b/mypy/typeshed/stdlib/lib2to3/fixer_base.pyi new file mode 100644 index 000000000000..06813c94308a --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixer_base.pyi @@ -0,0 +1,42 @@ +from _typeshed import Incomplete, StrPath +from abc import ABCMeta, abstractmethod +from collections.abc import MutableMapping +from typing import ClassVar, Literal, TypeVar + +from .pytree import Base, Leaf, Node + +_N = TypeVar("_N", bound=Base) + +class BaseFix: + PATTERN: ClassVar[str | None] + pattern: Incomplete | None + pattern_tree: Incomplete | None + options: Incomplete | None + filename: Incomplete | None + numbers: Incomplete + used_names: Incomplete + order: ClassVar[Literal["post", "pre"]] + explicit: ClassVar[bool] + run_order: ClassVar[int] + keep_line_order: ClassVar[bool] + BM_compatible: ClassVar[bool] + syms: Incomplete + log: Incomplete + def __init__(self, options: MutableMapping[str, Incomplete], log: list[str]) -> None: ... + def compile_pattern(self) -> None: ... + def set_filename(self, filename: StrPath) -> None: ... + def match(self, node: _N) -> Literal[False] | dict[str, _N]: ... + @abstractmethod + def transform(self, node: Base, results: dict[str, Base]) -> Node | Leaf | None: ... + def new_name(self, template: str = "xxx_todo_changeme") -> str: ... + first_log: bool + def log_message(self, message: str) -> None: ... + def cannot_convert(self, node: Base, reason: str | None = None) -> None: ... + def warning(self, node: Base, reason: str) -> None: ... + def start_tree(self, tree: Node, filename: StrPath) -> None: ... + def finish_tree(self, tree: Node, filename: StrPath) -> None: ... + +class ConditionalFix(BaseFix, metaclass=ABCMeta): + skip_on: ClassVar[str | None] + def start_tree(self, tree: Node, filename: StrPath, /) -> None: ... + def should_skip(self, node: Base) -> bool: ... diff --git a/stubs/3.2/urllib/__init__.py b/mypy/typeshed/stdlib/lib2to3/fixes/__init__.pyi similarity index 100% rename from stubs/3.2/urllib/__init__.py rename to mypy/typeshed/stdlib/lib2to3/fixes/__init__.pyi diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_apply.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_apply.pyi new file mode 100644 index 000000000000..e53e3dd86457 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_apply.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixApply(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_asserts.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_asserts.pyi new file mode 100644 index 000000000000..1bf7db2f76e9 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_asserts.pyi @@ -0,0 +1,10 @@ +from typing import ClassVar, Final, Literal + +from ..fixer_base import BaseFix + +NAMES: Final[dict[str, str]] + +class FixAsserts(BaseFix): + BM_compatible: ClassVar[Literal[False]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_basestring.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_basestring.pyi new file mode 100644 index 000000000000..8ed5ccaa7fd3 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_basestring.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixBasestring(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[Literal["'basestring'"]] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_buffer.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_buffer.pyi new file mode 100644 index 000000000000..1efca6228ea2 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_buffer.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixBuffer(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_dict.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_dict.pyi new file mode 100644 index 000000000000..08c54c3bc376 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_dict.pyi @@ -0,0 +1,16 @@ +from _typeshed import Incomplete +from typing import ClassVar, Literal + +from .. import fixer_base + +iter_exempt: set[str] + +class FixDict(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... + P1: ClassVar[str] + p1: ClassVar[Incomplete] + P2: ClassVar[str] + p2: ClassVar[Incomplete] + def in_special_context(self, node, isiter): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_except.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_except.pyi new file mode 100644 index 000000000000..30930a2c381e --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_except.pyi @@ -0,0 +1,14 @@ +from collections.abc import Generator, Iterable +from typing import ClassVar, Literal, TypeVar + +from .. import fixer_base +from ..pytree import Base + +_N = TypeVar("_N", bound=Base) + +def find_excepts(nodes: Iterable[_N]) -> Generator[tuple[_N, _N], None, None]: ... + +class FixExcept(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_exec.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_exec.pyi new file mode 100644 index 000000000000..71e2a820a564 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_exec.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixExec(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_execfile.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_execfile.pyi new file mode 100644 index 000000000000..8122a6389b12 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_execfile.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixExecfile(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_exitfunc.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_exitfunc.pyi new file mode 100644 index 000000000000..7fc910c0a1bc --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_exitfunc.pyi @@ -0,0 +1,13 @@ +from _typeshed import Incomplete, StrPath +from lib2to3 import fixer_base +from typing import ClassVar, Literal + +from ..pytree import Node + +class FixExitfunc(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def __init__(self, *args) -> None: ... + sys_import: Incomplete | None + def start_tree(self, tree: Node, filename: StrPath) -> None: ... + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_filter.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_filter.pyi new file mode 100644 index 000000000000..638889be8b65 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_filter.pyi @@ -0,0 +1,9 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixFilter(fixer_base.ConditionalFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + skip_on: ClassVar[Literal["future_builtins.filter"]] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_funcattrs.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_funcattrs.pyi new file mode 100644 index 000000000000..60487bb1f2a6 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_funcattrs.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixFuncattrs(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_future.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_future.pyi new file mode 100644 index 000000000000..12ed93f21223 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_future.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixFuture(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_getcwdu.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_getcwdu.pyi new file mode 100644 index 000000000000..aa3ccf50be9e --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_getcwdu.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixGetcwdu(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_has_key.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_has_key.pyi new file mode 100644 index 000000000000..f6f5a072e21b --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_has_key.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixHasKey(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_idioms.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_idioms.pyi new file mode 100644 index 000000000000..6b2723d09d43 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_idioms.pyi @@ -0,0 +1,15 @@ +from typing import ClassVar, Final, Literal + +from .. import fixer_base + +CMP: Final[str] +TYPE: Final[str] + +class FixIdioms(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[False]] + PATTERN: ClassVar[str] + def match(self, node): ... + def transform(self, node, results): ... + def transform_isinstance(self, node, results): ... + def transform_while(self, node, results) -> None: ... + def transform_sort(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_import.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_import.pyi new file mode 100644 index 000000000000..bf4b2d00925e --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_import.pyi @@ -0,0 +1,16 @@ +from _typeshed import StrPath +from collections.abc import Generator +from typing import ClassVar, Literal + +from .. import fixer_base +from ..pytree import Node + +def traverse_imports(names) -> Generator[str, None, None]: ... + +class FixImport(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + skip: bool + def start_tree(self, tree: Node, name: StrPath) -> None: ... + def transform(self, node, results): ... + def probably_a_local_import(self, imp_name): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports.pyi new file mode 100644 index 000000000000..c747af529f44 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports.pyi @@ -0,0 +1,21 @@ +from _typeshed import StrPath +from collections.abc import Generator +from typing import ClassVar, Final, Literal + +from .. import fixer_base +from ..pytree import Node + +MAPPING: Final[dict[str, str]] + +def alternates(members): ... +def build_pattern(mapping=...) -> Generator[str, None, None]: ... + +class FixImports(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + mapping = MAPPING + def build_pattern(self): ... + def compile_pattern(self) -> None: ... + def match(self, node): ... + replace: dict[str, str] + def start_tree(self, tree: Node, filename: StrPath) -> None: ... + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports2.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports2.pyi new file mode 100644 index 000000000000..618ecd0424d8 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports2.pyi @@ -0,0 +1,8 @@ +from typing import Final + +from . import fix_imports + +MAPPING: Final[dict[str, str]] + +class FixImports2(fix_imports.FixImports): + mapping = MAPPING diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_input.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_input.pyi new file mode 100644 index 000000000000..fc1279535bed --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_input.pyi @@ -0,0 +1,11 @@ +from _typeshed import Incomplete +from typing import ClassVar, Literal + +from .. import fixer_base + +context: Incomplete + +class FixInput(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_intern.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_intern.pyi new file mode 100644 index 000000000000..804b7b2517a5 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_intern.pyi @@ -0,0 +1,9 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixIntern(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + order: ClassVar[Literal["pre"]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_isinstance.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_isinstance.pyi new file mode 100644 index 000000000000..31eefd625317 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_isinstance.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixIsinstance(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_itertools.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_itertools.pyi new file mode 100644 index 000000000000..229d86ee71bb --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_itertools.pyi @@ -0,0 +1,9 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixItertools(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + it_funcs: str + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_itertools_imports.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_itertools_imports.pyi new file mode 100644 index 000000000000..39a4da506867 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_itertools_imports.pyi @@ -0,0 +1,7 @@ +from lib2to3 import fixer_base +from typing import ClassVar, Literal + +class FixItertoolsImports(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_long.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_long.pyi new file mode 100644 index 000000000000..9ccf2711d7d1 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_long.pyi @@ -0,0 +1,7 @@ +from lib2to3 import fixer_base +from typing import ClassVar, Literal + +class FixLong(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[Literal["'long'"]] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_map.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_map.pyi new file mode 100644 index 000000000000..6e60282cf0be --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_map.pyi @@ -0,0 +1,9 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixMap(fixer_base.ConditionalFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + skip_on: ClassVar[Literal["future_builtins.map"]] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_metaclass.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_metaclass.pyi new file mode 100644 index 000000000000..1b1ec82032b4 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_metaclass.pyi @@ -0,0 +1,17 @@ +from collections.abc import Generator +from typing import ClassVar, Literal + +from .. import fixer_base +from ..pytree import Base + +def has_metaclass(parent): ... +def fixup_parse_tree(cls_node) -> None: ... +def fixup_simple_stmt(parent, i, stmt_node) -> None: ... +def remove_trailing_newline(node) -> None: ... +def find_metas(cls_node) -> Generator[tuple[Base, int, Base], None, None]: ... +def fixup_indent(suite) -> None: ... + +class FixMetaclass(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_methodattrs.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_methodattrs.pyi new file mode 100644 index 000000000000..ca9b71e43f85 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_methodattrs.pyi @@ -0,0 +1,10 @@ +from typing import ClassVar, Final, Literal + +from .. import fixer_base + +MAP: Final[dict[str, str]] + +class FixMethodattrs(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_ne.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_ne.pyi new file mode 100644 index 000000000000..6ff1220b0472 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_ne.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixNe(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[False]] + def match(self, node): ... + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_next.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_next.pyi new file mode 100644 index 000000000000..b13914ae8c01 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_next.pyi @@ -0,0 +1,19 @@ +from _typeshed import StrPath +from typing import ClassVar, Literal + +from .. import fixer_base +from ..pytree import Node + +bind_warning: str + +class FixNext(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + order: ClassVar[Literal["pre"]] + shadowed_next: bool + def start_tree(self, tree: Node, filename: StrPath) -> None: ... + def transform(self, node, results) -> None: ... + +def is_assign_target(node): ... +def find_assign(node): ... +def is_subtree(root, node): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_nonzero.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_nonzero.pyi new file mode 100644 index 000000000000..5c37fc12ef08 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_nonzero.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixNonzero(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_numliterals.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_numliterals.pyi new file mode 100644 index 000000000000..113145e395f6 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_numliterals.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixNumliterals(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[False]] + def match(self, node): ... + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_operator.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_operator.pyi new file mode 100644 index 000000000000..b9863d38347b --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_operator.pyi @@ -0,0 +1,12 @@ +from lib2to3 import fixer_base +from typing import ClassVar, Literal + +def invocation(s): ... + +class FixOperator(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + order: ClassVar[Literal["pre"]] + methods: str + obj: str + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_paren.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_paren.pyi new file mode 100644 index 000000000000..237df6c5ff2c --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_paren.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixParen(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_print.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_print.pyi new file mode 100644 index 000000000000..e9564b04ac75 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_print.pyi @@ -0,0 +1,12 @@ +from _typeshed import Incomplete +from typing import ClassVar, Literal + +from .. import fixer_base + +parend_expr: Incomplete + +class FixPrint(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... + def add_kwarg(self, l_nodes, s_kwd, n_expr) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_raise.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_raise.pyi new file mode 100644 index 000000000000..e02c3080f409 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_raise.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixRaise(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_raw_input.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_raw_input.pyi new file mode 100644 index 000000000000..d1a0eb0e0a7e --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_raw_input.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixRawInput(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_reduce.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_reduce.pyi new file mode 100644 index 000000000000..f8ad876c21a6 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_reduce.pyi @@ -0,0 +1,8 @@ +from lib2to3 import fixer_base +from typing import ClassVar, Literal + +class FixReduce(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + order: ClassVar[Literal["pre"]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_reload.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_reload.pyi new file mode 100644 index 000000000000..820075438eca --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_reload.pyi @@ -0,0 +1,9 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixReload(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + order: ClassVar[Literal["pre"]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_renames.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_renames.pyi new file mode 100644 index 000000000000..652d8f15ea1a --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_renames.pyi @@ -0,0 +1,17 @@ +from collections.abc import Generator +from typing import ClassVar, Final, Literal + +from .. import fixer_base + +MAPPING: Final[dict[str, dict[str, str]]] +LOOKUP: Final[dict[tuple[str, str], str]] + +def alternates(members): ... +def build_pattern() -> Generator[str, None, None]: ... + +class FixRenames(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + order: ClassVar[Literal["pre"]] + PATTERN: ClassVar[str] + def match(self, node): ... + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_repr.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_repr.pyi new file mode 100644 index 000000000000..3b192d396dd6 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_repr.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixRepr(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_set_literal.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_set_literal.pyi new file mode 100644 index 000000000000..6962ff326f56 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_set_literal.pyi @@ -0,0 +1,7 @@ +from lib2to3 import fixer_base +from typing import ClassVar, Literal + +class FixSetLiteral(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_standarderror.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_standarderror.pyi new file mode 100644 index 000000000000..ba914bcab5d6 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_standarderror.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixStandarderror(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_sys_exc.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_sys_exc.pyi new file mode 100644 index 000000000000..0fa1a4787087 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_sys_exc.pyi @@ -0,0 +1,9 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixSysExc(fixer_base.BaseFix): + exc_info: ClassVar[list[str]] + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_throw.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_throw.pyi new file mode 100644 index 000000000000..4c99855e5c37 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_throw.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixThrow(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_tuple_params.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_tuple_params.pyi new file mode 100644 index 000000000000..bfaa9970c996 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_tuple_params.pyi @@ -0,0 +1,17 @@ +from _typeshed import Incomplete +from typing import ClassVar, Literal + +from .. import fixer_base + +def is_docstring(stmt): ... + +class FixTupleParams(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... + def transform_lambda(self, node, results) -> None: ... + +def simplify_args(node): ... +def find_params(node): ... +def map_to_index(param_list, prefix=..., d: Incomplete | None = ...): ... +def tuple_name(param_list): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_types.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_types.pyi new file mode 100644 index 000000000000..e26dbec71a97 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_types.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixTypes(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi new file mode 100644 index 000000000000..85d1315213b9 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi @@ -0,0 +1,12 @@ +from _typeshed import StrPath +from typing import ClassVar, Literal + +from .. import fixer_base +from ..pytree import Node + +class FixUnicode(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + unicode_literals: bool + def start_tree(self, tree: Node, filename: StrPath) -> None: ... + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_urllib.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_urllib.pyi new file mode 100644 index 000000000000..abdcc0f62970 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_urllib.pyi @@ -0,0 +1,15 @@ +from collections.abc import Generator +from typing import Final, Literal + +from .fix_imports import FixImports + +MAPPING: Final[dict[str, list[tuple[Literal["urllib.request", "urllib.parse", "urllib.error"], list[str]]]]] + +def build_pattern() -> Generator[str, None, None]: ... + +class FixUrllib(FixImports): + def build_pattern(self): ... + def transform_import(self, node, results) -> None: ... + def transform_member(self, node, results): ... + def transform_dot(self, node, results) -> None: ... + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_ws_comma.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_ws_comma.pyi new file mode 100644 index 000000000000..4ce5cb2c4ac1 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_ws_comma.pyi @@ -0,0 +1,12 @@ +from typing import ClassVar, Literal + +from .. import fixer_base +from ..pytree import Leaf + +class FixWsComma(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[False]] + PATTERN: ClassVar[str] + COMMA: Leaf + COLON: Leaf + SEPS: tuple[Leaf, Leaf] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_xrange.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_xrange.pyi new file mode 100644 index 000000000000..71318b7660b6 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_xrange.pyi @@ -0,0 +1,20 @@ +from _typeshed import Incomplete, StrPath +from typing import ClassVar, Literal + +from .. import fixer_base +from ..pytree import Node + +class FixXrange(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + transformed_xranges: set[Incomplete] | None + def start_tree(self, tree: Node, filename: StrPath) -> None: ... + def finish_tree(self, tree: Node, filename: StrPath) -> None: ... + def transform(self, node, results): ... + def transform_xrange(self, node, results) -> None: ... + def transform_range(self, node, results): ... + P1: ClassVar[str] + p1: ClassVar[Incomplete] + P2: ClassVar[str] + p2: ClassVar[Incomplete] + def in_special_context(self, node): ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_xreadlines.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_xreadlines.pyi new file mode 100644 index 000000000000..b4794143a003 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_xreadlines.pyi @@ -0,0 +1,8 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixXreadlines(fixer_base.BaseFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + def transform(self, node, results) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_zip.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_zip.pyi new file mode 100644 index 000000000000..805886ee3180 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_zip.pyi @@ -0,0 +1,9 @@ +from typing import ClassVar, Literal + +from .. import fixer_base + +class FixZip(fixer_base.ConditionalFix): + BM_compatible: ClassVar[Literal[True]] + PATTERN: ClassVar[str] + skip_on: ClassVar[Literal["future_builtins.zip"]] + def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/lib2to3/main.pyi b/mypy/typeshed/stdlib/lib2to3/main.pyi new file mode 100644 index 000000000000..5b7fdfca5d65 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/main.pyi @@ -0,0 +1,42 @@ +from _typeshed import FileDescriptorOrPath +from collections.abc import Container, Iterable, Iterator, Mapping, Sequence +from logging import _ExcInfoType +from typing import AnyStr, Literal + +from . import refactor as refactor + +def diff_texts(a: str, b: str, filename: str) -> Iterator[str]: ... + +class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): + nobackups: bool + show_diffs: bool + def __init__( + self, + fixers: Iterable[str], + options: Mapping[str, object] | None, + explicit: Container[str] | None, + nobackups: bool, + show_diffs: bool, + input_base_dir: str = "", + output_dir: str = "", + append_suffix: str = "", + ) -> None: ... + # Same as super.log_error and Logger.error + def log_error( # type: ignore[override] + self, + msg: str, + *args: Iterable[str], + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + # Same as super.write_file but without default values + def write_file( # type: ignore[override] + self, new_text: str, filename: FileDescriptorOrPath, old_text: str, encoding: str | None + ) -> None: ... + # filename has to be str + def print_output(self, old: str, new: str, filename: str, equal: bool) -> None: ... # type: ignore[override] + +def warn(msg: object) -> None: ... +def main(fixer_pkg: str, args: Sequence[AnyStr] | None = None) -> Literal[0, 1, 2]: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/__init__.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/__init__.pyi new file mode 100644 index 000000000000..de8a874f434d --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/__init__.pyi @@ -0,0 +1,9 @@ +from collections.abc import Callable +from typing import Any +from typing_extensions import TypeAlias + +from ..pytree import _RawNode +from .grammar import Grammar + +# This is imported in several lib2to3/pgen2 submodules +_Convert: TypeAlias = Callable[[Grammar, _RawNode], Any] # noqa: Y047 diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/driver.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/driver.pyi new file mode 100644 index 000000000000..dea13fb9d0f8 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/driver.pyi @@ -0,0 +1,27 @@ +from _typeshed import StrPath +from collections.abc import Iterable +from logging import Logger +from typing import IO + +from ..pytree import _NL +from . import _Convert +from .grammar import Grammar + +__all__ = ["Driver", "load_grammar"] + +class Driver: + grammar: Grammar + logger: Logger + convert: _Convert + def __init__(self, grammar: Grammar, convert: _Convert | None = None, logger: Logger | None = None) -> None: ... + def parse_tokens( + self, tokens: Iterable[tuple[int, str, tuple[int, int], tuple[int, int], str]], debug: bool = False + ) -> _NL: ... + def parse_stream_raw(self, stream: IO[str], debug: bool = False) -> _NL: ... + def parse_stream(self, stream: IO[str], debug: bool = False) -> _NL: ... + def parse_file(self, filename: StrPath, encoding: str | None = None, debug: bool = False) -> _NL: ... + def parse_string(self, text: str, debug: bool = False) -> _NL: ... + +def load_grammar( + gt: str = "Grammar.txt", gp: str | None = None, save: bool = True, force: bool = False, logger: Logger | None = None +) -> Grammar: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/grammar.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/grammar.pyi new file mode 100644 index 000000000000..bef0a7922683 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/grammar.pyi @@ -0,0 +1,24 @@ +from _typeshed import StrPath +from typing_extensions import Self, TypeAlias + +_Label: TypeAlias = tuple[int, str | None] +_DFA: TypeAlias = list[list[tuple[int, int]]] +_DFAS: TypeAlias = tuple[_DFA, dict[int, int]] + +class Grammar: + symbol2number: dict[str, int] + number2symbol: dict[int, str] + states: list[_DFA] + dfas: dict[int, _DFAS] + labels: list[_Label] + keywords: dict[str, int] + tokens: dict[int, int] + symbol2label: dict[str, int] + start: int + def dump(self, filename: StrPath) -> None: ... + def load(self, filename: StrPath) -> None: ... + def copy(self) -> Self: ... + def report(self) -> None: ... + +opmap_raw: str +opmap: dict[str, str] diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/literals.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/literals.pyi new file mode 100644 index 000000000000..c3fabe8a5177 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/literals.pyi @@ -0,0 +1,7 @@ +from re import Match + +simple_escapes: dict[str, str] + +def escape(m: Match[str]) -> str: ... +def evalString(s: str) -> str: ... +def test() -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/parse.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/parse.pyi new file mode 100644 index 000000000000..320c5f018d43 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/parse.pyi @@ -0,0 +1,30 @@ +from _typeshed import Incomplete +from collections.abc import Sequence +from typing_extensions import TypeAlias + +from ..pytree import _NL, _RawNode +from . import _Convert +from .grammar import _DFAS, Grammar + +_Context: TypeAlias = Sequence[Incomplete] + +class ParseError(Exception): + msg: str + type: int + value: str | None + context: _Context + def __init__(self, msg: str, type: int, value: str | None, context: _Context) -> None: ... + +class Parser: + grammar: Grammar + convert: _Convert + stack: list[tuple[_DFAS, int, _RawNode]] + rootnode: _NL | None + used_names: set[str] + def __init__(self, grammar: Grammar, convert: _Convert | None = None) -> None: ... + def setup(self, start: int | None = None) -> None: ... + def addtoken(self, type: int, value: str | None, context: _Context) -> bool: ... + def classify(self, type: int, value: str | None, context: _Context) -> int: ... + def shift(self, type: int, value: str | None, newstate: int, context: _Context) -> None: ... + def push(self, type: int, newdfa: _DFAS, newstate: int, context: _Context) -> None: ... + def pop(self) -> None: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/pgen.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/pgen.pyi new file mode 100644 index 000000000000..6d9f776c61ae --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/pgen.pyi @@ -0,0 +1,50 @@ +from _typeshed import Incomplete, StrPath +from collections.abc import Iterable, Iterator +from typing import IO, NoReturn, overload + +from . import grammar +from .tokenize import _TokenInfo + +class PgenGrammar(grammar.Grammar): ... + +class ParserGenerator: + filename: StrPath + stream: IO[str] + generator: Iterator[_TokenInfo] + first: dict[str, dict[str, int]] + def __init__(self, filename: StrPath, stream: IO[str] | None = None) -> None: ... + def make_grammar(self) -> PgenGrammar: ... + def make_first(self, c: PgenGrammar, name: str) -> dict[int, int]: ... + def make_label(self, c: PgenGrammar, label: str) -> int: ... + def addfirstsets(self) -> None: ... + def calcfirst(self, name: str) -> None: ... + def parse(self) -> tuple[dict[str, list[DFAState]], str]: ... + def make_dfa(self, start: NFAState, finish: NFAState) -> list[DFAState]: ... + def dump_nfa(self, name: str, start: NFAState, finish: NFAState) -> list[DFAState]: ... + def dump_dfa(self, name: str, dfa: Iterable[DFAState]) -> None: ... + def simplify_dfa(self, dfa: list[DFAState]) -> None: ... + def parse_rhs(self) -> tuple[NFAState, NFAState]: ... + def parse_alt(self) -> tuple[NFAState, NFAState]: ... + def parse_item(self) -> tuple[NFAState, NFAState]: ... + def parse_atom(self) -> tuple[NFAState, NFAState]: ... + def expect(self, type: int, value: str | None = None) -> str: ... + def gettoken(self) -> None: ... + @overload + def raise_error(self, msg: object) -> NoReturn: ... + @overload + def raise_error(self, msg: str, *args: object) -> NoReturn: ... + +class NFAState: + arcs: list[tuple[str | None, NFAState]] + def addarc(self, next: NFAState, label: str | None = None) -> None: ... + +class DFAState: + nfaset: dict[NFAState, Incomplete] + isfinal: bool + arcs: dict[str, DFAState] + def __init__(self, nfaset: dict[NFAState, Incomplete], final: NFAState) -> None: ... + def addarc(self, next: DFAState, label: str) -> None: ... + def unifystate(self, old: DFAState, new: DFAState) -> None: ... + def __eq__(self, other: DFAState) -> bool: ... # type: ignore[override] + +def generate_grammar(filename: StrPath = "Grammar.txt") -> PgenGrammar: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi new file mode 100644 index 000000000000..6898517acee6 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi @@ -0,0 +1,69 @@ +from typing import Final + +ENDMARKER: Final[int] +NAME: Final[int] +NUMBER: Final[int] +STRING: Final[int] +NEWLINE: Final[int] +INDENT: Final[int] +DEDENT: Final[int] +LPAR: Final[int] +RPAR: Final[int] +LSQB: Final[int] +RSQB: Final[int] +COLON: Final[int] +COMMA: Final[int] +SEMI: Final[int] +PLUS: Final[int] +MINUS: Final[int] +STAR: Final[int] +SLASH: Final[int] +VBAR: Final[int] +AMPER: Final[int] +LESS: Final[int] +GREATER: Final[int] +EQUAL: Final[int] +DOT: Final[int] +PERCENT: Final[int] +BACKQUOTE: Final[int] +LBRACE: Final[int] +RBRACE: Final[int] +EQEQUAL: Final[int] +NOTEQUAL: Final[int] +LESSEQUAL: Final[int] +GREATEREQUAL: Final[int] +TILDE: Final[int] +CIRCUMFLEX: Final[int] +LEFTSHIFT: Final[int] +RIGHTSHIFT: Final[int] +DOUBLESTAR: Final[int] +PLUSEQUAL: Final[int] +MINEQUAL: Final[int] +STAREQUAL: Final[int] +SLASHEQUAL: Final[int] +PERCENTEQUAL: Final[int] +AMPEREQUAL: Final[int] +VBAREQUAL: Final[int] +CIRCUMFLEXEQUAL: Final[int] +LEFTSHIFTEQUAL: Final[int] +RIGHTSHIFTEQUAL: Final[int] +DOUBLESTAREQUAL: Final[int] +DOUBLESLASH: Final[int] +DOUBLESLASHEQUAL: Final[int] +OP: Final[int] +COMMENT: Final[int] +NL: Final[int] +RARROW: Final[int] +AT: Final[int] +ATEQUAL: Final[int] +AWAIT: Final[int] +ASYNC: Final[int] +ERRORTOKEN: Final[int] +COLONEQUAL: Final[int] +N_TOKENS: Final[int] +NT_OFFSET: Final[int] +tok_name: dict[int, str] + +def ISTERMINAL(x: int) -> bool: ... +def ISNONTERMINAL(x: int) -> bool: ... +def ISEOF(x: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/tokenize.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/tokenize.pyi new file mode 100644 index 000000000000..af54de1b51d3 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/tokenize.pyi @@ -0,0 +1,96 @@ +from collections.abc import Callable, Iterable, Iterator +from typing_extensions import TypeAlias + +from .token import * + +__all__ = [ + "AMPER", + "AMPEREQUAL", + "ASYNC", + "AT", + "ATEQUAL", + "AWAIT", + "BACKQUOTE", + "CIRCUMFLEX", + "CIRCUMFLEXEQUAL", + "COLON", + "COMMA", + "COMMENT", + "DEDENT", + "DOT", + "DOUBLESLASH", + "DOUBLESLASHEQUAL", + "DOUBLESTAR", + "DOUBLESTAREQUAL", + "ENDMARKER", + "EQEQUAL", + "EQUAL", + "ERRORTOKEN", + "GREATER", + "GREATEREQUAL", + "INDENT", + "ISEOF", + "ISNONTERMINAL", + "ISTERMINAL", + "LBRACE", + "LEFTSHIFT", + "LEFTSHIFTEQUAL", + "LESS", + "LESSEQUAL", + "LPAR", + "LSQB", + "MINEQUAL", + "MINUS", + "NAME", + "NEWLINE", + "NL", + "NOTEQUAL", + "NT_OFFSET", + "NUMBER", + "N_TOKENS", + "OP", + "PERCENT", + "PERCENTEQUAL", + "PLUS", + "PLUSEQUAL", + "RARROW", + "RBRACE", + "RIGHTSHIFT", + "RIGHTSHIFTEQUAL", + "RPAR", + "RSQB", + "SEMI", + "SLASH", + "SLASHEQUAL", + "STAR", + "STAREQUAL", + "STRING", + "TILDE", + "VBAR", + "VBAREQUAL", + "tok_name", + "tokenize", + "generate_tokens", + "untokenize", + "COLONEQUAL", +] + +_Coord: TypeAlias = tuple[int, int] +_TokenEater: TypeAlias = Callable[[int, str, _Coord, _Coord, str], object] +_TokenInfo: TypeAlias = tuple[int, str, _Coord, _Coord, str] + +class TokenError(Exception): ... +class StopTokenizing(Exception): ... + +def tokenize(readline: Callable[[], str], tokeneater: _TokenEater = ...) -> None: ... + +class Untokenizer: + tokens: list[str] + prev_row: int + prev_col: int + def add_whitespace(self, start: _Coord) -> None: ... + def untokenize(self, iterable: Iterable[_TokenInfo]) -> str: ... + def compat(self, token: tuple[int, str], iterable: Iterable[_TokenInfo]) -> None: ... + +def untokenize(iterable: Iterable[_TokenInfo]) -> str: ... +def generate_tokens(readline: Callable[[], str]) -> Iterator[_TokenInfo]: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pygram.pyi b/mypy/typeshed/stdlib/lib2to3/pygram.pyi new file mode 100644 index 000000000000..86c74b54888a --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pygram.pyi @@ -0,0 +1,114 @@ +from .pgen2.grammar import Grammar + +class Symbols: + def __init__(self, grammar: Grammar) -> None: ... + +class python_symbols(Symbols): + and_expr: int + and_test: int + annassign: int + arglist: int + argument: int + arith_expr: int + assert_stmt: int + async_funcdef: int + async_stmt: int + atom: int + augassign: int + break_stmt: int + classdef: int + comp_for: int + comp_if: int + comp_iter: int + comp_op: int + comparison: int + compound_stmt: int + continue_stmt: int + decorated: int + decorator: int + decorators: int + del_stmt: int + dictsetmaker: int + dotted_as_name: int + dotted_as_names: int + dotted_name: int + encoding_decl: int + eval_input: int + except_clause: int + exec_stmt: int + expr: int + expr_stmt: int + exprlist: int + factor: int + file_input: int + flow_stmt: int + for_stmt: int + funcdef: int + global_stmt: int + if_stmt: int + import_as_name: int + import_as_names: int + import_from: int + import_name: int + import_stmt: int + lambdef: int + listmaker: int + not_test: int + old_lambdef: int + old_test: int + or_test: int + parameters: int + pass_stmt: int + power: int + print_stmt: int + raise_stmt: int + return_stmt: int + shift_expr: int + simple_stmt: int + single_input: int + sliceop: int + small_stmt: int + star_expr: int + stmt: int + subscript: int + subscriptlist: int + suite: int + term: int + test: int + testlist: int + testlist1: int + testlist_gexp: int + testlist_safe: int + testlist_star_expr: int + tfpdef: int + tfplist: int + tname: int + trailer: int + try_stmt: int + typedargslist: int + varargslist: int + vfpdef: int + vfplist: int + vname: int + while_stmt: int + with_item: int + with_stmt: int + with_var: int + xor_expr: int + yield_arg: int + yield_expr: int + yield_stmt: int + +class pattern_symbols(Symbols): + Alternative: int + Alternatives: int + Details: int + Matcher: int + NegatedUnit: int + Repeater: int + Unit: int + +python_grammar: Grammar +python_grammar_no_print_statement: Grammar +python_grammar_no_print_and_exec_statement: Grammar +pattern_grammar: Grammar diff --git a/mypy/typeshed/stdlib/lib2to3/pytree.pyi b/mypy/typeshed/stdlib/lib2to3/pytree.pyi new file mode 100644 index 000000000000..138333bd58af --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/pytree.pyi @@ -0,0 +1,117 @@ +from _typeshed import Incomplete, SupportsGetItem, SupportsLenAndGetItem, Unused +from abc import abstractmethod +from collections.abc import Iterable, Iterator, MutableSequence +from typing import Final +from typing_extensions import Self, TypeAlias + +from .fixer_base import BaseFix +from .pgen2.grammar import Grammar + +_NL: TypeAlias = Node | Leaf +_Context: TypeAlias = tuple[str, int, int] +_Results: TypeAlias = dict[str, _NL] +_RawNode: TypeAlias = tuple[int, str, _Context, list[_NL] | None] + +HUGE: Final = 0x7FFFFFFF + +def type_repr(type_num: int) -> str | int: ... + +class Base: + type: int + parent: Node | None + prefix: str + children: list[_NL] + was_changed: bool + was_checked: bool + def __eq__(self, other: object) -> bool: ... + @abstractmethod + def _eq(self, other: Base) -> bool: ... + @abstractmethod + def clone(self) -> Self: ... + @abstractmethod + def post_order(self) -> Iterator[Self]: ... + @abstractmethod + def pre_order(self) -> Iterator[Self]: ... + def replace(self, new: _NL | list[_NL]) -> None: ... + def get_lineno(self) -> int: ... + def changed(self) -> None: ... + def remove(self) -> int | None: ... + @property + def next_sibling(self) -> _NL | None: ... + @property + def prev_sibling(self) -> _NL | None: ... + def leaves(self) -> Iterator[Leaf]: ... + def depth(self) -> int: ... + def get_suffix(self) -> str: ... + +class Node(Base): + fixers_applied: MutableSequence[BaseFix] | None + # Is Unbound until set in refactor.RefactoringTool + future_features: frozenset[Incomplete] + # Is Unbound until set in pgen2.parse.Parser.pop + used_names: set[str] + def __init__( + self, + type: int, + children: Iterable[_NL], + context: Unused = None, + prefix: str | None = None, + fixers_applied: MutableSequence[BaseFix] | None = None, + ) -> None: ... + def _eq(self, other: Base) -> bool: ... + def clone(self) -> Node: ... + def post_order(self) -> Iterator[Self]: ... + def pre_order(self) -> Iterator[Self]: ... + def set_child(self, i: int, child: _NL) -> None: ... + def insert_child(self, i: int, child: _NL) -> None: ... + def append_child(self, child: _NL) -> None: ... + def __unicode__(self) -> str: ... + +class Leaf(Base): + lineno: int + column: int + value: str + fixers_applied: MutableSequence[BaseFix] + def __init__( + self, + type: int, + value: str, + context: _Context | None = None, + prefix: str | None = None, + fixers_applied: MutableSequence[BaseFix] = [], + ) -> None: ... + def _eq(self, other: Base) -> bool: ... + def clone(self) -> Leaf: ... + def post_order(self) -> Iterator[Self]: ... + def pre_order(self) -> Iterator[Self]: ... + def __unicode__(self) -> str: ... + +def convert(gr: Grammar, raw_node: _RawNode) -> _NL: ... + +class BasePattern: + type: int + content: str | None + name: str | None + def optimize(self) -> BasePattern: ... # sic, subclasses are free to optimize themselves into different patterns + def match(self, node: _NL, results: _Results | None = None) -> bool: ... + def match_seq(self, nodes: SupportsLenAndGetItem[_NL], results: _Results | None = None) -> bool: ... + def generate_matches(self, nodes: SupportsGetItem[int, _NL]) -> Iterator[tuple[int, _Results]]: ... + +class LeafPattern(BasePattern): + def __init__(self, type: int | None = None, content: str | None = None, name: str | None = None) -> None: ... + +class NodePattern(BasePattern): + wildcards: bool + def __init__(self, type: int | None = None, content: str | None = None, name: str | None = None) -> None: ... + +class WildcardPattern(BasePattern): + min: int + max: int + def __init__(self, content: str | None = None, min: int = 0, max: int = 0x7FFFFFFF, name: str | None = None) -> None: ... + +class NegatedPattern(BasePattern): + def __init__(self, content: str | None = None) -> None: ... + +def generate_matches( + patterns: SupportsGetItem[int | slice, BasePattern] | None, nodes: SupportsGetItem[int | slice, _NL] +) -> Iterator[tuple[int, _Results]]: ... diff --git a/mypy/typeshed/stdlib/lib2to3/refactor.pyi b/mypy/typeshed/stdlib/lib2to3/refactor.pyi new file mode 100644 index 000000000000..a7f382540648 --- /dev/null +++ b/mypy/typeshed/stdlib/lib2to3/refactor.pyi @@ -0,0 +1,82 @@ +from _typeshed import FileDescriptorOrPath, StrPath, SupportsGetItem +from collections.abc import Container, Generator, Iterable, Mapping +from logging import Logger, _ExcInfoType +from multiprocessing import JoinableQueue +from multiprocessing.synchronize import Lock +from typing import Any, ClassVar, Final, NoReturn, overload + +from .btm_matcher import BottomMatcher +from .fixer_base import BaseFix +from .pgen2.driver import Driver +from .pgen2.grammar import Grammar +from .pytree import Node + +def get_all_fix_names(fixer_pkg: str, remove_prefix: bool = True) -> list[str]: ... +def get_fixers_from_package(pkg_name: str) -> list[str]: ... + +class FixerError(Exception): ... + +class RefactoringTool: + CLASS_PREFIX: ClassVar[str] + FILE_PREFIX: ClassVar[str] + fixers: Iterable[str] + explicit: Container[str] + options: dict[str, Any] + grammar: Grammar + write_unchanged_files: bool + errors: list[tuple[str, Iterable[str], dict[str, _ExcInfoType]]] + logger: Logger + fixer_log: list[str] + wrote: bool + driver: Driver + pre_order: list[BaseFix] + post_order: list[BaseFix] + files: list[StrPath] + BM: BottomMatcher + bmi_pre_order: list[BaseFix] + bmi_post_order: list[BaseFix] + def __init__( + self, fixer_names: Iterable[str], options: Mapping[str, object] | None = None, explicit: Container[str] | None = None + ) -> None: ... + def get_fixers(self) -> tuple[list[BaseFix], list[BaseFix]]: ... + def log_error(self, msg: str, *args: Iterable[str], **kwargs: _ExcInfoType) -> NoReturn: ... + @overload + def log_message(self, msg: object) -> None: ... + @overload + def log_message(self, msg: str, *args: object) -> None: ... + @overload + def log_debug(self, msg: object) -> None: ... + @overload + def log_debug(self, msg: str, *args: object) -> None: ... + def print_output(self, old_text: str, new_text: str, filename: StrPath, equal: bool) -> None: ... + def refactor(self, items: Iterable[str], write: bool = False, doctests_only: bool = False) -> None: ... + def refactor_dir(self, dir_name: str, write: bool = False, doctests_only: bool = False) -> None: ... + def _read_python_source(self, filename: FileDescriptorOrPath) -> tuple[str, str]: ... + def refactor_file(self, filename: StrPath, write: bool = False, doctests_only: bool = False) -> None: ... + def refactor_string(self, data: str, name: str) -> Node | None: ... + def refactor_stdin(self, doctests_only: bool = False) -> None: ... + def refactor_tree(self, tree: Node, name: str) -> bool: ... + def traverse_by(self, fixers: SupportsGetItem[int, Iterable[BaseFix]] | None, traversal: Iterable[Node]) -> None: ... + def processed_file( + self, new_text: str, filename: StrPath, old_text: str | None = None, write: bool = False, encoding: str | None = None + ) -> None: ... + def write_file(self, new_text: str, filename: FileDescriptorOrPath, old_text: str, encoding: str | None = None) -> None: ... + PS1: Final = ">>> " + PS2: Final = "... " + def refactor_docstring(self, input: str, filename: StrPath) -> str: ... + def refactor_doctest(self, block: list[str], lineno: int, indent: int, filename: StrPath) -> list[str]: ... + def summarize(self) -> None: ... + def parse_block(self, block: Iterable[str], lineno: int, indent: int) -> Node: ... + def wrap_toks( + self, block: Iterable[str], lineno: int, indent: int + ) -> Generator[tuple[int, str, tuple[int, int], tuple[int, int], str], None, None]: ... + def gen_lines(self, block: Iterable[str], indent: int) -> Generator[str, None, None]: ... + +class MultiprocessingUnsupported(Exception): ... + +class MultiprocessRefactoringTool(RefactoringTool): + queue: JoinableQueue[None | tuple[Iterable[str], bool | int]] | None + output_lock: Lock | None + def refactor( + self, items: Iterable[str], write: bool = False, doctests_only: bool = False, num_processes: int = 1 + ) -> None: ... diff --git a/mypy/typeshed/stdlib/linecache.pyi b/mypy/typeshed/stdlib/linecache.pyi new file mode 100644 index 000000000000..2e050e13b621 --- /dev/null +++ b/mypy/typeshed/stdlib/linecache.pyi @@ -0,0 +1,23 @@ +import sys +from collections.abc import Callable +from typing import Any +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 9): + __all__ = ["getline", "clearcache", "checkcache", "lazycache"] +else: + __all__ = ["getline", "clearcache", "checkcache"] + +_ModuleGlobals: TypeAlias = dict[str, Any] +_ModuleMetadata: TypeAlias = tuple[int, float | None, list[str], str] + +_SourceLoader: TypeAlias = tuple[Callable[[], str | None]] + +cache: dict[str, _SourceLoader | _ModuleMetadata] # undocumented + +def getline(filename: str, lineno: int, module_globals: _ModuleGlobals | None = None) -> str: ... +def clearcache() -> None: ... +def getlines(filename: str, module_globals: _ModuleGlobals | None = None) -> list[str]: ... +def checkcache(filename: str | None = None) -> None: ... +def updatecache(filename: str, module_globals: _ModuleGlobals | None = None) -> list[str]: ... +def lazycache(filename: str, module_globals: _ModuleGlobals) -> bool: ... diff --git a/mypy/typeshed/stdlib/locale.pyi b/mypy/typeshed/stdlib/locale.pyi new file mode 100644 index 000000000000..58de65449572 --- /dev/null +++ b/mypy/typeshed/stdlib/locale.pyi @@ -0,0 +1,156 @@ +import sys +from _locale import ( + CHAR_MAX as CHAR_MAX, + LC_ALL as LC_ALL, + LC_COLLATE as LC_COLLATE, + LC_CTYPE as LC_CTYPE, + LC_MONETARY as LC_MONETARY, + LC_NUMERIC as LC_NUMERIC, + LC_TIME as LC_TIME, + localeconv as localeconv, + strcoll as strcoll, + strxfrm as strxfrm, +) + +# This module defines a function "str()", which is why "str" can't be used +# as a type annotation or type alias. +from builtins import str as _str +from collections.abc import Callable, Iterable +from decimal import Decimal +from typing import Any + +if sys.version_info >= (3, 11): + from _locale import getencoding as getencoding + +# Some parts of the `_locale` module are platform-specific: +if sys.platform != "win32": + from _locale import ( + ABDAY_1 as ABDAY_1, + ABDAY_2 as ABDAY_2, + ABDAY_3 as ABDAY_3, + ABDAY_4 as ABDAY_4, + ABDAY_5 as ABDAY_5, + ABDAY_6 as ABDAY_6, + ABDAY_7 as ABDAY_7, + ABMON_1 as ABMON_1, + ABMON_2 as ABMON_2, + ABMON_3 as ABMON_3, + ABMON_4 as ABMON_4, + ABMON_5 as ABMON_5, + ABMON_6 as ABMON_6, + ABMON_7 as ABMON_7, + ABMON_8 as ABMON_8, + ABMON_9 as ABMON_9, + ABMON_10 as ABMON_10, + ABMON_11 as ABMON_11, + ABMON_12 as ABMON_12, + ALT_DIGITS as ALT_DIGITS, + AM_STR as AM_STR, + CODESET as CODESET, + CRNCYSTR as CRNCYSTR, + D_FMT as D_FMT, + D_T_FMT as D_T_FMT, + DAY_1 as DAY_1, + DAY_2 as DAY_2, + DAY_3 as DAY_3, + DAY_4 as DAY_4, + DAY_5 as DAY_5, + DAY_6 as DAY_6, + DAY_7 as DAY_7, + ERA as ERA, + ERA_D_FMT as ERA_D_FMT, + ERA_D_T_FMT as ERA_D_T_FMT, + ERA_T_FMT as ERA_T_FMT, + LC_MESSAGES as LC_MESSAGES, + MON_1 as MON_1, + MON_2 as MON_2, + MON_3 as MON_3, + MON_4 as MON_4, + MON_5 as MON_5, + MON_6 as MON_6, + MON_7 as MON_7, + MON_8 as MON_8, + MON_9 as MON_9, + MON_10 as MON_10, + MON_11 as MON_11, + MON_12 as MON_12, + NOEXPR as NOEXPR, + PM_STR as PM_STR, + RADIXCHAR as RADIXCHAR, + T_FMT as T_FMT, + T_FMT_AMPM as T_FMT_AMPM, + THOUSEP as THOUSEP, + YESEXPR as YESEXPR, + bind_textdomain_codeset as bind_textdomain_codeset, + bindtextdomain as bindtextdomain, + dcgettext as dcgettext, + dgettext as dgettext, + gettext as gettext, + nl_langinfo as nl_langinfo, + textdomain as textdomain, + ) + +__all__ = [ + "getlocale", + "getdefaultlocale", + "getpreferredencoding", + "Error", + "setlocale", + "localeconv", + "strcoll", + "strxfrm", + "str", + "atof", + "atoi", + "format_string", + "currency", + "normalize", + "LC_CTYPE", + "LC_COLLATE", + "LC_TIME", + "LC_MONETARY", + "LC_NUMERIC", + "LC_ALL", + "CHAR_MAX", +] + +if sys.version_info >= (3, 11): + __all__ += ["getencoding"] + +if sys.version_info < (3, 12): + __all__ += ["format"] + +if sys.version_info < (3, 13): + __all__ += ["resetlocale"] + +if sys.platform != "win32": + __all__ += ["LC_MESSAGES"] + +class Error(Exception): ... + +def getdefaultlocale( + envvars: tuple[_str, ...] = ("LC_ALL", "LC_CTYPE", "LANG", "LANGUAGE") +) -> tuple[_str | None, _str | None]: ... +def getlocale(category: int = ...) -> tuple[_str | None, _str | None]: ... +def setlocale(category: int, locale: _str | Iterable[_str | None] | None = None) -> _str: ... +def getpreferredencoding(do_setlocale: bool = True) -> _str: ... +def normalize(localename: _str) -> _str: ... + +if sys.version_info < (3, 13): + def resetlocale(category: int = ...) -> None: ... + +if sys.version_info < (3, 12): + def format( + percent: _str, value: float | Decimal, grouping: bool = False, monetary: bool = False, *additional: Any + ) -> _str: ... + +def format_string(f: _str, val: Any, grouping: bool = False, monetary: bool = False) -> _str: ... +def currency(val: float | Decimal, symbol: bool = True, grouping: bool = False, international: bool = False) -> _str: ... +def delocalize(string: _str) -> _str: ... +def atof(string: _str, func: Callable[[_str], float] = ...) -> float: ... +def atoi(string: _str) -> int: ... +def str(val: float) -> _str: ... + +locale_alias: dict[_str, _str] # undocumented +locale_encoding_alias: dict[_str, _str] # undocumented +windows_locale: dict[int, _str] # undocumented diff --git a/mypy/typeshed/stdlib/logging/__init__.pyi b/mypy/typeshed/stdlib/logging/__init__.pyi new file mode 100644 index 000000000000..9a4827a8f626 --- /dev/null +++ b/mypy/typeshed/stdlib/logging/__init__.pyi @@ -0,0 +1,684 @@ +import sys +import threading +from _typeshed import StrPath, SupportsWrite +from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence +from io import TextIOWrapper +from re import Pattern +from string import Template +from time import struct_time +from types import FrameType, TracebackType +from typing import Any, ClassVar, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload +from typing_extensions import Self, TypeAlias, deprecated + +if sys.version_info >= (3, 11): + from types import GenericAlias + +__all__ = [ + "BASIC_FORMAT", + "BufferingFormatter", + "CRITICAL", + "DEBUG", + "ERROR", + "FATAL", + "FileHandler", + "Filter", + "Formatter", + "Handler", + "INFO", + "LogRecord", + "Logger", + "LoggerAdapter", + "NOTSET", + "NullHandler", + "StreamHandler", + "WARN", + "WARNING", + "addLevelName", + "basicConfig", + "captureWarnings", + "critical", + "debug", + "disable", + "error", + "exception", + "fatal", + "getLevelName", + "getLogger", + "getLoggerClass", + "info", + "log", + "makeLogRecord", + "setLoggerClass", + "shutdown", + "warning", + "getLogRecordFactory", + "setLogRecordFactory", + "lastResort", + "raiseExceptions", + "warn", +] + +if sys.version_info >= (3, 11): + __all__ += ["getLevelNamesMapping"] +if sys.version_info >= (3, 12): + __all__ += ["getHandlerByName", "getHandlerNames"] + +_SysExcInfoType: TypeAlias = tuple[type[BaseException], BaseException, TracebackType | None] | tuple[None, None, None] +_ExcInfoType: TypeAlias = None | bool | _SysExcInfoType | BaseException +_ArgsType: TypeAlias = tuple[object, ...] | Mapping[str, object] +_Level: TypeAlias = int | str +_FormatStyle: TypeAlias = Literal["%", "{", "$"] + +if sys.version_info >= (3, 12): + class _SupportsFilter(Protocol): + def filter(self, record: LogRecord, /) -> bool | LogRecord: ... + + _FilterType: TypeAlias = Filter | Callable[[LogRecord], bool | LogRecord] | _SupportsFilter +else: + class _SupportsFilter(Protocol): + def filter(self, record: LogRecord, /) -> bool: ... + + _FilterType: TypeAlias = Filter | Callable[[LogRecord], bool] | _SupportsFilter + +raiseExceptions: bool +logThreads: bool +logMultiprocessing: bool +logProcesses: bool +_srcfile: str | None + +def currentframe() -> FrameType: ... + +_levelToName: dict[int, str] +_nameToLevel: dict[str, int] + +class Filterer: + filters: list[_FilterType] + def addFilter(self, filter: _FilterType) -> None: ... + def removeFilter(self, filter: _FilterType) -> None: ... + if sys.version_info >= (3, 12): + def filter(self, record: LogRecord) -> bool | LogRecord: ... + else: + def filter(self, record: LogRecord) -> bool: ... + +class Manager: # undocumented + root: RootLogger + disable: int + emittedNoHandlerWarning: bool + loggerDict: dict[str, Logger | PlaceHolder] + loggerClass: type[Logger] | None + logRecordFactory: Callable[..., LogRecord] | None + def __init__(self, rootnode: RootLogger) -> None: ... + def getLogger(self, name: str) -> Logger: ... + def setLoggerClass(self, klass: type[Logger]) -> None: ... + def setLogRecordFactory(self, factory: Callable[..., LogRecord]) -> None: ... + +class Logger(Filterer): + name: str # undocumented + level: int # undocumented + parent: Logger | None # undocumented + propagate: bool + handlers: list[Handler] # undocumented + disabled: bool # undocumented + root: ClassVar[RootLogger] # undocumented + manager: Manager # undocumented + def __init__(self, name: str, level: _Level = 0) -> None: ... + def setLevel(self, level: _Level) -> None: ... + def isEnabledFor(self, level: int) -> bool: ... + def getEffectiveLevel(self) -> int: ... + def getChild(self, suffix: str) -> Self: ... # see python/typing#980 + if sys.version_info >= (3, 12): + def getChildren(self) -> set[Logger]: ... + + def debug( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + def info( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + def warning( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + @deprecated("Deprecated; use warning() instead.") + def warn( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + def error( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + def exception( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = True, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + def critical( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + def log( + self, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... + def _log( + self, + level: int, + msg: object, + args: _ArgsType, + exc_info: _ExcInfoType | None = None, + extra: Mapping[str, object] | None = None, + stack_info: bool = False, + stacklevel: int = 1, + ) -> None: ... # undocumented + fatal = critical + def addHandler(self, hdlr: Handler) -> None: ... + def removeHandler(self, hdlr: Handler) -> None: ... + def findCaller(self, stack_info: bool = False, stacklevel: int = 1) -> tuple[str, int, str, str | None]: ... + def handle(self, record: LogRecord) -> None: ... + def makeRecord( + self, + name: str, + level: int, + fn: str, + lno: int, + msg: object, + args: _ArgsType, + exc_info: _SysExcInfoType | None, + func: str | None = None, + extra: Mapping[str, object] | None = None, + sinfo: str | None = None, + ) -> LogRecord: ... + def hasHandlers(self) -> bool: ... + def callHandlers(self, record: LogRecord) -> None: ... # undocumented + +CRITICAL: Final = 50 +FATAL: Final = CRITICAL +ERROR: Final = 40 +WARNING: Final = 30 +WARN: Final = WARNING +INFO: Final = 20 +DEBUG: Final = 10 +NOTSET: Final = 0 + +class Handler(Filterer): + level: int # undocumented + formatter: Formatter | None # undocumented + lock: threading.Lock | None # undocumented + name: str | None # undocumented + def __init__(self, level: _Level = 0) -> None: ... + def get_name(self) -> str: ... # undocumented + def set_name(self, name: str) -> None: ... # undocumented + def createLock(self) -> None: ... + def acquire(self) -> None: ... + def release(self) -> None: ... + def setLevel(self, level: _Level) -> None: ... + def setFormatter(self, fmt: Formatter | None) -> None: ... + def flush(self) -> None: ... + def close(self) -> None: ... + def handle(self, record: LogRecord) -> bool: ... + def handleError(self, record: LogRecord) -> None: ... + def format(self, record: LogRecord) -> str: ... + def emit(self, record: LogRecord) -> None: ... + +if sys.version_info >= (3, 12): + def getHandlerByName(name: str) -> Handler | None: ... + def getHandlerNames() -> frozenset[str]: ... + +class Formatter: + converter: Callable[[float | None], struct_time] + _fmt: str | None # undocumented + datefmt: str | None # undocumented + _style: PercentStyle # undocumented + default_time_format: str + if sys.version_info >= (3, 9): + default_msec_format: str | None + else: + default_msec_format: str + + if sys.version_info >= (3, 10): + def __init__( + self, + fmt: str | None = None, + datefmt: str | None = None, + style: _FormatStyle = "%", + validate: bool = True, + *, + defaults: Mapping[str, Any] | None = None, + ) -> None: ... + else: + def __init__( + self, fmt: str | None = None, datefmt: str | None = None, style: _FormatStyle = "%", validate: bool = True + ) -> None: ... + + def format(self, record: LogRecord) -> str: ... + def formatTime(self, record: LogRecord, datefmt: str | None = None) -> str: ... + def formatException(self, ei: _SysExcInfoType) -> str: ... + def formatMessage(self, record: LogRecord) -> str: ... # undocumented + def formatStack(self, stack_info: str) -> str: ... + def usesTime(self) -> bool: ... # undocumented + +class BufferingFormatter: + linefmt: Formatter + def __init__(self, linefmt: Formatter | None = None) -> None: ... + def formatHeader(self, records: Sequence[LogRecord]) -> str: ... + def formatFooter(self, records: Sequence[LogRecord]) -> str: ... + def format(self, records: Sequence[LogRecord]) -> str: ... + +class Filter: + name: str # undocumented + nlen: int # undocumented + def __init__(self, name: str = "") -> None: ... + if sys.version_info >= (3, 12): + def filter(self, record: LogRecord) -> bool | LogRecord: ... + else: + def filter(self, record: LogRecord) -> bool: ... + +class LogRecord: + # args can be set to None by logging.handlers.QueueHandler + # (see https://bugs.python.org/issue44473) + args: _ArgsType | None + asctime: str + created: float + exc_info: _SysExcInfoType | None + exc_text: str | None + filename: str + funcName: str + levelname: str + levelno: int + lineno: int + module: str + msecs: float + # Only created when logging.Formatter.format is called. See #6132. + message: str + msg: str | Any # The runtime accepts any object, but will be a str in 99% of cases + name: str + pathname: str + process: int | None + processName: str | None + relativeCreated: float + stack_info: str | None + thread: int | None + threadName: str | None + if sys.version_info >= (3, 12): + taskName: str | None + + def __init__( + self, + name: str, + level: int, + pathname: str, + lineno: int, + msg: object, + args: _ArgsType | None, + exc_info: _SysExcInfoType | None, + func: str | None = None, + sinfo: str | None = None, + ) -> None: ... + def getMessage(self) -> str: ... + # Allows setting contextual information on LogRecord objects as per the docs, see #7833 + def __setattr__(self, name: str, value: Any, /) -> None: ... + +_L = TypeVar("_L", bound=Logger | LoggerAdapter[Any]) + +class LoggerAdapter(Generic[_L]): + logger: _L + manager: Manager # undocumented + + if sys.version_info >= (3, 13): + def __init__(self, logger: _L, extra: Mapping[str, object] | None = None, merge_extra: bool = False) -> None: ... + elif sys.version_info >= (3, 10): + def __init__(self, logger: _L, extra: Mapping[str, object] | None = None) -> None: ... + else: + def __init__(self, logger: _L, extra: Mapping[str, object]) -> None: ... + + if sys.version_info >= (3, 10): + extra: Mapping[str, object] | None + else: + extra: Mapping[str, object] + + def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> tuple[Any, MutableMapping[str, Any]]: ... + def debug( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + def info( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + def warning( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + @deprecated("Deprecated; use warning() instead.") + def warn( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + def error( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + def exception( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = True, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + def critical( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + def log( + self, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... + def isEnabledFor(self, level: int) -> bool: ... + def getEffectiveLevel(self) -> int: ... + def setLevel(self, level: _Level) -> None: ... + def hasHandlers(self) -> bool: ... + if sys.version_info >= (3, 11): + def _log( + self, + level: int, + msg: object, + args: _ArgsType, + *, + exc_info: _ExcInfoType | None = None, + extra: Mapping[str, object] | None = None, + stack_info: bool = False, + ) -> None: ... # undocumented + else: + def _log( + self, + level: int, + msg: object, + args: _ArgsType, + exc_info: _ExcInfoType | None = None, + extra: Mapping[str, object] | None = None, + stack_info: bool = False, + ) -> None: ... # undocumented + + @property + def name(self) -> str: ... # undocumented + if sys.version_info >= (3, 11): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +def getLogger(name: str | None = None) -> Logger: ... +def getLoggerClass() -> type[Logger]: ... +def getLogRecordFactory() -> Callable[..., LogRecord]: ... +def debug( + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... +def info( + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... +def warning( + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... +@deprecated("Deprecated; use warning() instead.") +def warn( + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... +def error( + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... +def critical( + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... +def exception( + msg: object, + *args: object, + exc_info: _ExcInfoType = True, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... +def log( + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... + +fatal = critical + +def disable(level: int = 50) -> None: ... +def addLevelName(level: int, levelName: str) -> None: ... +@overload +def getLevelName(level: int) -> str: ... +@overload +@deprecated("The str -> int case is considered a mistake.") +def getLevelName(level: str) -> Any: ... + +if sys.version_info >= (3, 11): + def getLevelNamesMapping() -> dict[str, int]: ... + +def makeLogRecord(dict: Mapping[str, object]) -> LogRecord: ... + +if sys.version_info >= (3, 9): + def basicConfig( + *, + filename: StrPath | None = ..., + filemode: str = ..., + format: str = ..., + datefmt: str | None = ..., + style: _FormatStyle = ..., + level: _Level | None = ..., + stream: SupportsWrite[str] | None = ..., + handlers: Iterable[Handler] | None = ..., + force: bool | None = ..., + encoding: str | None = ..., + errors: str | None = ..., + ) -> None: ... + +else: + def basicConfig( + *, + filename: StrPath | None = ..., + filemode: str = ..., + format: str = ..., + datefmt: str | None = ..., + style: _FormatStyle = ..., + level: _Level | None = ..., + stream: SupportsWrite[str] | None = ..., + handlers: Iterable[Handler] | None = ..., + force: bool = ..., + ) -> None: ... + +def shutdown(handlerList: Sequence[Any] = ...) -> None: ... # handlerList is undocumented +def setLoggerClass(klass: type[Logger]) -> None: ... +def captureWarnings(capture: bool) -> None: ... +def setLogRecordFactory(factory: Callable[..., LogRecord]) -> None: ... + +lastResort: Handler | None + +_StreamT = TypeVar("_StreamT", bound=SupportsWrite[str]) + +class StreamHandler(Handler, Generic[_StreamT]): + stream: _StreamT # undocumented + terminator: str + @overload + def __init__(self: StreamHandler[TextIO], stream: None = None) -> None: ... + @overload + def __init__(self: StreamHandler[_StreamT], stream: _StreamT) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] #11780 + def setStream(self, stream: _StreamT) -> _StreamT | None: ... + if sys.version_info >= (3, 11): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class FileHandler(StreamHandler[TextIOWrapper]): + baseFilename: str # undocumented + mode: str # undocumented + encoding: str | None # undocumented + delay: bool # undocumented + if sys.version_info >= (3, 9): + errors: str | None # undocumented + def __init__( + self, filename: StrPath, mode: str = "a", encoding: str | None = None, delay: bool = False, errors: str | None = None + ) -> None: ... + else: + def __init__(self, filename: StrPath, mode: str = "a", encoding: str | None = None, delay: bool = False) -> None: ... + + def _open(self) -> TextIOWrapper: ... # undocumented + +class NullHandler(Handler): ... + +class PlaceHolder: # undocumented + loggerMap: dict[Logger, None] + def __init__(self, alogger: Logger) -> None: ... + def append(self, alogger: Logger) -> None: ... + +# Below aren't in module docs but still visible + +class RootLogger(Logger): + def __init__(self, level: int) -> None: ... + +root: RootLogger + +class PercentStyle: # undocumented + default_format: str + asctime_format: str + asctime_search: str + validation_pattern: Pattern[str] + _fmt: str + if sys.version_info >= (3, 10): + def __init__(self, fmt: str, *, defaults: Mapping[str, Any] | None = None) -> None: ... + else: + def __init__(self, fmt: str) -> None: ... + + def usesTime(self) -> bool: ... + def validate(self) -> None: ... + def format(self, record: Any) -> str: ... + +class StrFormatStyle(PercentStyle): # undocumented + fmt_spec: Pattern[str] + field_spec: Pattern[str] + +class StringTemplateStyle(PercentStyle): # undocumented + _tpl: Template + +_STYLES: Final[dict[str, tuple[PercentStyle, str]]] + +BASIC_FORMAT: Final[str] diff --git a/mypy/typeshed/stdlib/logging/config.pyi b/mypy/typeshed/stdlib/logging/config.pyi new file mode 100644 index 000000000000..5c444e66c4c7 --- /dev/null +++ b/mypy/typeshed/stdlib/logging/config.pyi @@ -0,0 +1,134 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence +from configparser import RawConfigParser +from re import Pattern +from threading import Thread +from typing import IO, Any, Final, Literal, SupportsIndex, TypedDict, overload +from typing_extensions import Required, TypeAlias + +from . import Filter, Filterer, Formatter, Handler, Logger, _FilterType, _FormatStyle, _Level + +DEFAULT_LOGGING_CONFIG_PORT: int +RESET_ERROR: Final[int] # undocumented +IDENTIFIER: Final[Pattern[str]] # undocumented + +if sys.version_info >= (3, 11): + class _RootLoggerConfiguration(TypedDict, total=False): + level: _Level + filters: Sequence[str | _FilterType] + handlers: Sequence[str] + +else: + class _RootLoggerConfiguration(TypedDict, total=False): + level: _Level + filters: Sequence[str] + handlers: Sequence[str] + +class _LoggerConfiguration(_RootLoggerConfiguration, TypedDict, total=False): + propagate: bool + +_FormatterConfigurationTypedDict = TypedDict( + "_FormatterConfigurationTypedDict", {"class": str, "format": str, "datefmt": str, "style": _FormatStyle}, total=False +) + +class _FilterConfigurationTypedDict(TypedDict): + name: str + +# Formatter and filter configs can specify custom factories via the special `()` key. +# If that is the case, the dictionary can contain any additional keys +# https://docs.python.org/3/library/logging.config.html#user-defined-objects +_FormatterConfiguration: TypeAlias = _FormatterConfigurationTypedDict | dict[str, Any] +_FilterConfiguration: TypeAlias = _FilterConfigurationTypedDict | dict[str, Any] +# Handler config can have additional keys even when not providing a custom factory so we just use `dict`. +_HandlerConfiguration: TypeAlias = dict[str, Any] + +class _DictConfigArgs(TypedDict, total=False): + version: Required[Literal[1]] + formatters: dict[str, _FormatterConfiguration] + filters: dict[str, _FilterConfiguration] + handlers: dict[str, _HandlerConfiguration] + loggers: dict[str, _LoggerConfiguration] + root: _RootLoggerConfiguration + incremental: bool + disable_existing_loggers: bool + +# Accept dict[str, Any] to avoid false positives if called with a dict +# type, since dict types are not compatible with TypedDicts. +# +# Also accept a TypedDict type, to allow callers to use TypedDict +# types, and for somewhat stricter type checking of dict literals. +def dictConfig(config: _DictConfigArgs | dict[str, Any]) -> None: ... + +if sys.version_info >= (3, 10): + def fileConfig( + fname: StrOrBytesPath | IO[str] | RawConfigParser, + defaults: Mapping[str, str] | None = None, + disable_existing_loggers: bool = True, + encoding: str | None = None, + ) -> None: ... + +else: + def fileConfig( + fname: StrOrBytesPath | IO[str] | RawConfigParser, + defaults: Mapping[str, str] | None = None, + disable_existing_loggers: bool = True, + ) -> None: ... + +def valid_ident(s: str) -> Literal[True]: ... # undocumented +def listen(port: int = 9030, verify: Callable[[bytes], bytes | None] | None = None) -> Thread: ... +def stopListening() -> None: ... + +class ConvertingMixin: # undocumented + def convert_with_key(self, key: Any, value: Any, replace: bool = True) -> Any: ... + def convert(self, value: Any) -> Any: ... + +class ConvertingDict(dict[Hashable, Any], ConvertingMixin): # undocumented + def __getitem__(self, key: Hashable) -> Any: ... + def get(self, key: Hashable, default: Any = None) -> Any: ... + def pop(self, key: Hashable, default: Any = None) -> Any: ... + +class ConvertingList(list[Any], ConvertingMixin): # undocumented + @overload + def __getitem__(self, key: SupportsIndex) -> Any: ... + @overload + def __getitem__(self, key: slice) -> Any: ... + def pop(self, idx: SupportsIndex = -1) -> Any: ... + +class ConvertingTuple(tuple[Any, ...], ConvertingMixin): # undocumented + @overload + def __getitem__(self, key: SupportsIndex) -> Any: ... + @overload + def __getitem__(self, key: slice) -> Any: ... + +class BaseConfigurator: # undocumented + CONVERT_PATTERN: Pattern[str] + WORD_PATTERN: Pattern[str] + DOT_PATTERN: Pattern[str] + INDEX_PATTERN: Pattern[str] + DIGIT_PATTERN: Pattern[str] + value_converters: dict[str, str] + importer: Callable[..., Any] + + def __init__(self, config: _DictConfigArgs | dict[str, Any]) -> None: ... + def resolve(self, s: str) -> Any: ... + def ext_convert(self, value: str) -> Any: ... + def cfg_convert(self, value: str) -> Any: ... + def convert(self, value: Any) -> Any: ... + def configure_custom(self, config: dict[str, Any]) -> Any: ... + def as_tuple(self, value: list[Any] | tuple[Any, ...]) -> tuple[Any, ...]: ... + +class DictConfigurator(BaseConfigurator): + def configure(self) -> None: ... # undocumented + def configure_formatter(self, config: _FormatterConfiguration) -> Formatter | Any: ... # undocumented + def configure_filter(self, config: _FilterConfiguration) -> Filter | Any: ... # undocumented + def add_filters(self, filterer: Filterer, filters: Iterable[_FilterType]) -> None: ... # undocumented + def configure_handler(self, config: _HandlerConfiguration) -> Handler | Any: ... # undocumented + def add_handlers(self, logger: Logger, handlers: Iterable[str]) -> None: ... # undocumented + def common_logger_config( + self, logger: Logger, config: _LoggerConfiguration, incremental: bool = False + ) -> None: ... # undocumented + def configure_logger(self, name: str, config: _LoggerConfiguration, incremental: bool = False) -> None: ... # undocumented + def configure_root(self, config: _LoggerConfiguration, incremental: bool = False) -> None: ... # undocumented + +dictConfigClass = DictConfigurator diff --git a/mypy/typeshed/stdlib/logging/handlers.pyi b/mypy/typeshed/stdlib/logging/handlers.pyi new file mode 100644 index 000000000000..d594d6569a7e --- /dev/null +++ b/mypy/typeshed/stdlib/logging/handlers.pyi @@ -0,0 +1,277 @@ +import datetime +import http.client +import ssl +import sys +from _typeshed import ReadableBuffer, StrPath +from collections.abc import Callable +from logging import FileHandler, Handler, LogRecord +from re import Pattern +from socket import SocketKind, socket +from threading import Thread +from typing import Any, ClassVar, Final, Protocol, TypeVar + +_T = TypeVar("_T") + +DEFAULT_TCP_LOGGING_PORT: Final[int] +DEFAULT_UDP_LOGGING_PORT: Final[int] +DEFAULT_HTTP_LOGGING_PORT: Final[int] +DEFAULT_SOAP_LOGGING_PORT: Final[int] +SYSLOG_UDP_PORT: Final[int] +SYSLOG_TCP_PORT: Final[int] + +class WatchedFileHandler(FileHandler): + dev: int # undocumented + ino: int # undocumented + if sys.version_info >= (3, 9): + def __init__( + self, filename: StrPath, mode: str = "a", encoding: str | None = None, delay: bool = False, errors: str | None = None + ) -> None: ... + else: + def __init__(self, filename: StrPath, mode: str = "a", encoding: str | None = None, delay: bool = False) -> None: ... + + def _statstream(self) -> None: ... # undocumented + def reopenIfNeeded(self) -> None: ... + +class BaseRotatingHandler(FileHandler): + namer: Callable[[str], str] | None + rotator: Callable[[str, str], None] | None + if sys.version_info >= (3, 9): + def __init__( + self, filename: StrPath, mode: str, encoding: str | None = None, delay: bool = False, errors: str | None = None + ) -> None: ... + else: + def __init__(self, filename: StrPath, mode: str, encoding: str | None = None, delay: bool = False) -> None: ... + + def rotation_filename(self, default_name: str) -> str: ... + def rotate(self, source: str, dest: str) -> None: ... + +class RotatingFileHandler(BaseRotatingHandler): + maxBytes: int # undocumented + backupCount: int # undocumented + if sys.version_info >= (3, 9): + def __init__( + self, + filename: StrPath, + mode: str = "a", + maxBytes: int = 0, + backupCount: int = 0, + encoding: str | None = None, + delay: bool = False, + errors: str | None = None, + ) -> None: ... + else: + def __init__( + self, + filename: StrPath, + mode: str = "a", + maxBytes: int = 0, + backupCount: int = 0, + encoding: str | None = None, + delay: bool = False, + ) -> None: ... + + def doRollover(self) -> None: ... + def shouldRollover(self, record: LogRecord) -> int: ... # undocumented + +class TimedRotatingFileHandler(BaseRotatingHandler): + when: str # undocumented + backupCount: int # undocumented + utc: bool # undocumented + atTime: datetime.time | None # undocumented + interval: int # undocumented + suffix: str # undocumented + dayOfWeek: int # undocumented + rolloverAt: int # undocumented + extMatch: Pattern[str] # undocumented + if sys.version_info >= (3, 9): + def __init__( + self, + filename: StrPath, + when: str = "h", + interval: int = 1, + backupCount: int = 0, + encoding: str | None = None, + delay: bool = False, + utc: bool = False, + atTime: datetime.time | None = None, + errors: str | None = None, + ) -> None: ... + else: + def __init__( + self, + filename: StrPath, + when: str = "h", + interval: int = 1, + backupCount: int = 0, + encoding: str | None = None, + delay: bool = False, + utc: bool = False, + atTime: datetime.time | None = None, + ) -> None: ... + + def doRollover(self) -> None: ... + def shouldRollover(self, record: LogRecord) -> int: ... # undocumented + def computeRollover(self, currentTime: int) -> int: ... # undocumented + def getFilesToDelete(self) -> list[str]: ... # undocumented + +class SocketHandler(Handler): + host: str # undocumented + port: int | None # undocumented + address: tuple[str, int] | str # undocumented + sock: socket | None # undocumented + closeOnError: bool # undocumented + retryTime: float | None # undocumented + retryStart: float # undocumented + retryFactor: float # undocumented + retryMax: float # undocumented + def __init__(self, host: str, port: int | None) -> None: ... + def makeSocket(self, timeout: float = 1) -> socket: ... # timeout is undocumented + def makePickle(self, record: LogRecord) -> bytes: ... + def send(self, s: ReadableBuffer) -> None: ... + def createSocket(self) -> None: ... + +class DatagramHandler(SocketHandler): + def makeSocket(self) -> socket: ... # type: ignore[override] + +class SysLogHandler(Handler): + LOG_EMERG: int + LOG_ALERT: int + LOG_CRIT: int + LOG_ERR: int + LOG_WARNING: int + LOG_NOTICE: int + LOG_INFO: int + LOG_DEBUG: int + + LOG_KERN: int + LOG_USER: int + LOG_MAIL: int + LOG_DAEMON: int + LOG_AUTH: int + LOG_SYSLOG: int + LOG_LPR: int + LOG_NEWS: int + LOG_UUCP: int + LOG_CRON: int + LOG_AUTHPRIV: int + LOG_FTP: int + + if sys.version_info >= (3, 9): + LOG_NTP: int + LOG_SECURITY: int + LOG_CONSOLE: int + LOG_SOLCRON: int + + LOG_LOCAL0: int + LOG_LOCAL1: int + LOG_LOCAL2: int + LOG_LOCAL3: int + LOG_LOCAL4: int + LOG_LOCAL5: int + LOG_LOCAL6: int + LOG_LOCAL7: int + address: tuple[str, int] | str # undocumented + unixsocket: bool # undocumented + socktype: SocketKind # undocumented + ident: str # undocumented + append_nul: bool # undocumented + facility: int # undocumented + priority_names: ClassVar[dict[str, int]] # undocumented + facility_names: ClassVar[dict[str, int]] # undocumented + priority_map: ClassVar[dict[str, str]] # undocumented + def __init__( + self, address: tuple[str, int] | str = ("localhost", 514), facility: str | int = 1, socktype: SocketKind | None = None + ) -> None: ... + if sys.version_info >= (3, 11): + def createSocket(self) -> None: ... + + def encodePriority(self, facility: int | str, priority: int | str) -> int: ... + def mapPriority(self, levelName: str) -> str: ... + +class NTEventLogHandler(Handler): + def __init__(self, appname: str, dllname: str | None = None, logtype: str = "Application") -> None: ... + def getEventCategory(self, record: LogRecord) -> int: ... + # TODO correct return value? + def getEventType(self, record: LogRecord) -> int: ... + def getMessageID(self, record: LogRecord) -> int: ... + +class SMTPHandler(Handler): + mailhost: str # undocumented + mailport: int | None # undocumented + username: str | None # undocumented + # password only exists as an attribute if passed credentials is a tuple or list + password: str # undocumented + fromaddr: str # undocumented + toaddrs: list[str] # undocumented + subject: str # undocumented + secure: tuple[()] | tuple[str] | tuple[str, str] | None # undocumented + timeout: float # undocumented + def __init__( + self, + mailhost: str | tuple[str, int], + fromaddr: str, + toaddrs: str | list[str], + subject: str, + credentials: tuple[str, str] | None = None, + secure: tuple[()] | tuple[str] | tuple[str, str] | None = None, + timeout: float = 5.0, + ) -> None: ... + def getSubject(self, record: LogRecord) -> str: ... + +class BufferingHandler(Handler): + capacity: int # undocumented + buffer: list[LogRecord] # undocumented + def __init__(self, capacity: int) -> None: ... + def shouldFlush(self, record: LogRecord) -> bool: ... + +class MemoryHandler(BufferingHandler): + flushLevel: int # undocumented + target: Handler | None # undocumented + flushOnClose: bool # undocumented + def __init__(self, capacity: int, flushLevel: int = 40, target: Handler | None = None, flushOnClose: bool = True) -> None: ... + def setTarget(self, target: Handler | None) -> None: ... + +class HTTPHandler(Handler): + host: str # undocumented + url: str # undocumented + method: str # undocumented + secure: bool # undocumented + credentials: tuple[str, str] | None # undocumented + context: ssl.SSLContext | None # undocumented + def __init__( + self, + host: str, + url: str, + method: str = "GET", + secure: bool = False, + credentials: tuple[str, str] | None = None, + context: ssl.SSLContext | None = None, + ) -> None: ... + def mapLogRecord(self, record: LogRecord) -> dict[str, Any]: ... + if sys.version_info >= (3, 9): + def getConnection(self, host: str, secure: bool) -> http.client.HTTPConnection: ... # undocumented + +class _QueueLike(Protocol[_T]): + def get(self) -> _T: ... + def put_nowait(self, item: _T, /) -> None: ... + +class QueueHandler(Handler): + queue: _QueueLike[Any] + def __init__(self, queue: _QueueLike[Any]) -> None: ... + def prepare(self, record: LogRecord) -> Any: ... + def enqueue(self, record: LogRecord) -> None: ... + if sys.version_info >= (3, 12): + listener: QueueListener | None + +class QueueListener: + handlers: tuple[Handler, ...] # undocumented + respect_handler_level: bool # undocumented + queue: _QueueLike[Any] # undocumented + _thread: Thread | None # undocumented + def __init__(self, queue: _QueueLike[Any], *handlers: Handler, respect_handler_level: bool = False) -> None: ... + def dequeue(self, block: bool) -> LogRecord: ... + def prepare(self, record: LogRecord) -> Any: ... + def start(self) -> None: ... + def stop(self) -> None: ... + def enqueue_sentinel(self) -> None: ... + def handle(self, record: LogRecord) -> None: ... diff --git a/mypy/typeshed/stdlib/lzma.pyi b/mypy/typeshed/stdlib/lzma.pyi new file mode 100644 index 000000000000..2f0279f5986b --- /dev/null +++ b/mypy/typeshed/stdlib/lzma.pyi @@ -0,0 +1,174 @@ +from _compression import BaseStream +from _lzma import ( + CHECK_CRC32 as CHECK_CRC32, + CHECK_CRC64 as CHECK_CRC64, + CHECK_ID_MAX as CHECK_ID_MAX, + CHECK_NONE as CHECK_NONE, + CHECK_SHA256 as CHECK_SHA256, + CHECK_UNKNOWN as CHECK_UNKNOWN, + FILTER_ARM as FILTER_ARM, + FILTER_ARMTHUMB as FILTER_ARMTHUMB, + FILTER_DELTA as FILTER_DELTA, + FILTER_IA64 as FILTER_IA64, + FILTER_LZMA1 as FILTER_LZMA1, + FILTER_LZMA2 as FILTER_LZMA2, + FILTER_POWERPC as FILTER_POWERPC, + FILTER_SPARC as FILTER_SPARC, + FILTER_X86 as FILTER_X86, + FORMAT_ALONE as FORMAT_ALONE, + FORMAT_AUTO as FORMAT_AUTO, + FORMAT_RAW as FORMAT_RAW, + FORMAT_XZ as FORMAT_XZ, + MF_BT2 as MF_BT2, + MF_BT3 as MF_BT3, + MF_BT4 as MF_BT4, + MF_HC3 as MF_HC3, + MF_HC4 as MF_HC4, + MODE_FAST as MODE_FAST, + MODE_NORMAL as MODE_NORMAL, + PRESET_DEFAULT as PRESET_DEFAULT, + PRESET_EXTREME as PRESET_EXTREME, + LZMACompressor as LZMACompressor, + LZMADecompressor as LZMADecompressor, + LZMAError as LZMAError, + _FilterChain, + is_check_supported as is_check_supported, +) +from _typeshed import ReadableBuffer, StrOrBytesPath +from typing import IO, Literal, TextIO, overload +from typing_extensions import Self, TypeAlias + +__all__ = [ + "CHECK_NONE", + "CHECK_CRC32", + "CHECK_CRC64", + "CHECK_SHA256", + "CHECK_ID_MAX", + "CHECK_UNKNOWN", + "FILTER_LZMA1", + "FILTER_LZMA2", + "FILTER_DELTA", + "FILTER_X86", + "FILTER_IA64", + "FILTER_ARM", + "FILTER_ARMTHUMB", + "FILTER_POWERPC", + "FILTER_SPARC", + "FORMAT_AUTO", + "FORMAT_XZ", + "FORMAT_ALONE", + "FORMAT_RAW", + "MF_HC3", + "MF_HC4", + "MF_BT2", + "MF_BT3", + "MF_BT4", + "MODE_FAST", + "MODE_NORMAL", + "PRESET_DEFAULT", + "PRESET_EXTREME", + "LZMACompressor", + "LZMADecompressor", + "LZMAFile", + "LZMAError", + "open", + "compress", + "decompress", + "is_check_supported", +] + +_OpenBinaryWritingMode: TypeAlias = Literal["w", "wb", "x", "xb", "a", "ab"] +_OpenTextWritingMode: TypeAlias = Literal["wt", "xt", "at"] + +_PathOrFile: TypeAlias = StrOrBytesPath | IO[bytes] + +class LZMAFile(BaseStream, IO[bytes]): # type: ignore[misc] # incompatible definitions of writelines in the base classes + def __init__( + self, + filename: _PathOrFile | None = None, + mode: str = "r", + *, + format: int | None = None, + check: int = -1, + preset: int | None = None, + filters: _FilterChain | None = None, + ) -> None: ... + def __enter__(self) -> Self: ... + def peek(self, size: int = -1) -> bytes: ... + def read(self, size: int | None = -1) -> bytes: ... + def read1(self, size: int = -1) -> bytes: ... + def readline(self, size: int | None = -1) -> bytes: ... + def write(self, data: ReadableBuffer) -> int: ... + def seek(self, offset: int, whence: int = 0) -> int: ... + +@overload +def open( + filename: _PathOrFile, + mode: Literal["r", "rb"] = "rb", + *, + format: int | None = None, + check: Literal[-1] = -1, + preset: None = None, + filters: _FilterChain | None = None, + encoding: None = None, + errors: None = None, + newline: None = None, +) -> LZMAFile: ... +@overload +def open( + filename: _PathOrFile, + mode: _OpenBinaryWritingMode, + *, + format: int | None = None, + check: int = -1, + preset: int | None = None, + filters: _FilterChain | None = None, + encoding: None = None, + errors: None = None, + newline: None = None, +) -> LZMAFile: ... +@overload +def open( + filename: StrOrBytesPath, + mode: Literal["rt"], + *, + format: int | None = None, + check: Literal[-1] = -1, + preset: None = None, + filters: _FilterChain | None = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIO: ... +@overload +def open( + filename: StrOrBytesPath, + mode: _OpenTextWritingMode, + *, + format: int | None = None, + check: int = -1, + preset: int | None = None, + filters: _FilterChain | None = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIO: ... +@overload +def open( + filename: _PathOrFile, + mode: str, + *, + format: int | None = None, + check: int = -1, + preset: int | None = None, + filters: _FilterChain | None = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> LZMAFile | TextIO: ... +def compress( + data: ReadableBuffer, format: int = 1, check: int = -1, preset: int | None = None, filters: _FilterChain | None = None +) -> bytes: ... +def decompress( + data: ReadableBuffer, format: int = 0, memlimit: int | None = None, filters: _FilterChain | None = None +) -> bytes: ... diff --git a/mypy/typeshed/stdlib/mailbox.pyi b/mypy/typeshed/stdlib/mailbox.pyi new file mode 100644 index 000000000000..a98a00a42853 --- /dev/null +++ b/mypy/typeshed/stdlib/mailbox.pyi @@ -0,0 +1,264 @@ +import email.message +import io +import sys +from _typeshed import StrPath, SupportsNoArgReadline, SupportsRead +from abc import ABCMeta, abstractmethod +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from types import TracebackType +from typing import IO, Any, AnyStr, Generic, Literal, Protocol, TypeVar, overload +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "Mailbox", + "Maildir", + "mbox", + "MH", + "Babyl", + "MMDF", + "Message", + "MaildirMessage", + "mboxMessage", + "MHMessage", + "BabylMessage", + "MMDFMessage", + "Error", + "NoSuchMailboxError", + "NotEmptyError", + "ExternalClashError", + "FormatError", +] + +_T = TypeVar("_T") +_MessageT = TypeVar("_MessageT", bound=Message) + +class _SupportsReadAndReadline(SupportsRead[bytes], SupportsNoArgReadline[bytes], Protocol): ... + +_MessageData: TypeAlias = email.message.Message | bytes | str | io.StringIO | _SupportsReadAndReadline + +class _HasIteritems(Protocol): + def iteritems(self) -> Iterator[tuple[str, _MessageData]]: ... + +class _HasItems(Protocol): + def items(self) -> Iterator[tuple[str, _MessageData]]: ... + +linesep: bytes + +class Mailbox(Generic[_MessageT]): + _path: str # undocumented + _factory: Callable[[IO[Any]], _MessageT] | None # undocumented + @overload + def __init__(self, path: StrPath, factory: Callable[[IO[Any]], _MessageT], create: bool = True) -> None: ... + @overload + def __init__(self, path: StrPath, factory: None = None, create: bool = True) -> None: ... + @abstractmethod + def add(self, message: _MessageData) -> str: ... + @abstractmethod + def remove(self, key: str) -> None: ... + def __delitem__(self, key: str) -> None: ... + def discard(self, key: str) -> None: ... + @abstractmethod + def __setitem__(self, key: str, message: _MessageData) -> None: ... + @overload + def get(self, key: str, default: None = None) -> _MessageT | None: ... + @overload + def get(self, key: str, default: _T) -> _MessageT | _T: ... + def __getitem__(self, key: str) -> _MessageT: ... + @abstractmethod + def get_message(self, key: str) -> _MessageT: ... + def get_string(self, key: str) -> str: ... + @abstractmethod + def get_bytes(self, key: str) -> bytes: ... + # As '_ProxyFile' doesn't implement the full IO spec, and BytesIO is incompatible with it, get_file return is Any here + @abstractmethod + def get_file(self, key: str) -> Any: ... + @abstractmethod + def iterkeys(self) -> Iterator[str]: ... + def keys(self) -> list[str]: ... + def itervalues(self) -> Iterator[_MessageT]: ... + def __iter__(self) -> Iterator[_MessageT]: ... + def values(self) -> list[_MessageT]: ... + def iteritems(self) -> Iterator[tuple[str, _MessageT]]: ... + def items(self) -> list[tuple[str, _MessageT]]: ... + @abstractmethod + def __contains__(self, key: str) -> bool: ... + @abstractmethod + def __len__(self) -> int: ... + def clear(self) -> None: ... + @overload + def pop(self, key: str, default: None = None) -> _MessageT | None: ... + @overload + def pop(self, key: str, default: _T) -> _MessageT | _T: ... + def popitem(self) -> tuple[str, _MessageT]: ... + def update(self, arg: _HasIteritems | _HasItems | Iterable[tuple[str, _MessageData]] | None = None) -> None: ... + @abstractmethod + def flush(self) -> None: ... + @abstractmethod + def lock(self) -> None: ... + @abstractmethod + def unlock(self) -> None: ... + @abstractmethod + def close(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class Maildir(Mailbox[MaildirMessage]): + colon: str + def __init__( + self, dirname: StrPath, factory: Callable[[IO[Any]], MaildirMessage] | None = None, create: bool = True + ) -> None: ... + def add(self, message: _MessageData) -> str: ... + def remove(self, key: str) -> None: ... + def __setitem__(self, key: str, message: _MessageData) -> None: ... + def get_message(self, key: str) -> MaildirMessage: ... + def get_bytes(self, key: str) -> bytes: ... + def get_file(self, key: str) -> _ProxyFile[bytes]: ... + if sys.version_info >= (3, 13): + def get_info(self, key: str) -> str: ... + def set_info(self, key: str, info: str) -> None: ... + def get_flags(self, key: str) -> str: ... + def set_flags(self, key: str, flags: str) -> None: ... + def add_flag(self, key: str, flag: str) -> None: ... + def remove_flag(self, key: str, flag: str) -> None: ... + + def iterkeys(self) -> Iterator[str]: ... + def __contains__(self, key: str) -> bool: ... + def __len__(self) -> int: ... + def flush(self) -> None: ... + def lock(self) -> None: ... + def unlock(self) -> None: ... + def close(self) -> None: ... + def list_folders(self) -> list[str]: ... + def get_folder(self, folder: str) -> Maildir: ... + def add_folder(self, folder: str) -> Maildir: ... + def remove_folder(self, folder: str) -> None: ... + def clean(self) -> None: ... + def next(self) -> str | None: ... + +class _singlefileMailbox(Mailbox[_MessageT], metaclass=ABCMeta): + def add(self, message: _MessageData) -> str: ... + def remove(self, key: str) -> None: ... + def __setitem__(self, key: str, message: _MessageData) -> None: ... + def iterkeys(self) -> Iterator[str]: ... + def __contains__(self, key: str) -> bool: ... + def __len__(self) -> int: ... + def lock(self) -> None: ... + def unlock(self) -> None: ... + def flush(self) -> None: ... + def close(self) -> None: ... + +class _mboxMMDF(_singlefileMailbox[_MessageT]): + def get_message(self, key: str) -> _MessageT: ... + def get_file(self, key: str, from_: bool = False) -> _PartialFile[bytes]: ... + def get_bytes(self, key: str, from_: bool = False) -> bytes: ... + def get_string(self, key: str, from_: bool = False) -> str: ... + +class mbox(_mboxMMDF[mboxMessage]): + def __init__(self, path: StrPath, factory: Callable[[IO[Any]], mboxMessage] | None = None, create: bool = True) -> None: ... + +class MMDF(_mboxMMDF[MMDFMessage]): + def __init__(self, path: StrPath, factory: Callable[[IO[Any]], MMDFMessage] | None = None, create: bool = True) -> None: ... + +class MH(Mailbox[MHMessage]): + def __init__(self, path: StrPath, factory: Callable[[IO[Any]], MHMessage] | None = None, create: bool = True) -> None: ... + def add(self, message: _MessageData) -> str: ... + def remove(self, key: str) -> None: ... + def __setitem__(self, key: str, message: _MessageData) -> None: ... + def get_message(self, key: str) -> MHMessage: ... + def get_bytes(self, key: str) -> bytes: ... + def get_file(self, key: str) -> _ProxyFile[bytes]: ... + def iterkeys(self) -> Iterator[str]: ... + def __contains__(self, key: str) -> bool: ... + def __len__(self) -> int: ... + def flush(self) -> None: ... + def lock(self) -> None: ... + def unlock(self) -> None: ... + def close(self) -> None: ... + def list_folders(self) -> list[str]: ... + def get_folder(self, folder: StrPath) -> MH: ... + def add_folder(self, folder: StrPath) -> MH: ... + def remove_folder(self, folder: StrPath) -> None: ... + def get_sequences(self) -> dict[str, list[int]]: ... + def set_sequences(self, sequences: Mapping[str, Sequence[int]]) -> None: ... + def pack(self) -> None: ... + +class Babyl(_singlefileMailbox[BabylMessage]): + def __init__(self, path: StrPath, factory: Callable[[IO[Any]], BabylMessage] | None = None, create: bool = True) -> None: ... + def get_message(self, key: str) -> BabylMessage: ... + def get_bytes(self, key: str) -> bytes: ... + def get_file(self, key: str) -> IO[bytes]: ... + def get_labels(self) -> list[str]: ... + +class Message(email.message.Message): + def __init__(self, message: _MessageData | None = None) -> None: ... + +class MaildirMessage(Message): + def get_subdir(self) -> str: ... + def set_subdir(self, subdir: Literal["new", "cur"]) -> None: ... + def get_flags(self) -> str: ... + def set_flags(self, flags: Iterable[str]) -> None: ... + def add_flag(self, flag: str) -> None: ... + def remove_flag(self, flag: str) -> None: ... + def get_date(self) -> int: ... + def set_date(self, date: float) -> None: ... + def get_info(self) -> str: ... + def set_info(self, info: str) -> None: ... + +class _mboxMMDFMessage(Message): + def get_from(self) -> str: ... + def set_from(self, from_: str, time_: bool | tuple[int, int, int, int, int, int, int, int, int] | None = None) -> None: ... + def get_flags(self) -> str: ... + def set_flags(self, flags: Iterable[str]) -> None: ... + def add_flag(self, flag: str) -> None: ... + def remove_flag(self, flag: str) -> None: ... + +class mboxMessage(_mboxMMDFMessage): ... + +class MHMessage(Message): + def get_sequences(self) -> list[str]: ... + def set_sequences(self, sequences: Iterable[str]) -> None: ... + def add_sequence(self, sequence: str) -> None: ... + def remove_sequence(self, sequence: str) -> None: ... + +class BabylMessage(Message): + def get_labels(self) -> list[str]: ... + def set_labels(self, labels: Iterable[str]) -> None: ... + def add_label(self, label: str) -> None: ... + def remove_label(self, label: str) -> None: ... + def get_visible(self) -> Message: ... + def set_visible(self, visible: _MessageData) -> None: ... + def update_visible(self) -> None: ... + +class MMDFMessage(_mboxMMDFMessage): ... + +class _ProxyFile(Generic[AnyStr]): + def __init__(self, f: IO[AnyStr], pos: int | None = None) -> None: ... + def read(self, size: int | None = None) -> AnyStr: ... + def read1(self, size: int | None = None) -> AnyStr: ... + def readline(self, size: int | None = None) -> AnyStr: ... + def readlines(self, sizehint: int | None = None) -> list[AnyStr]: ... + def __iter__(self) -> Iterator[AnyStr]: ... + def tell(self) -> int: ... + def seek(self, offset: int, whence: int = 0) -> None: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None) -> None: ... + def readable(self) -> bool: ... + def writable(self) -> bool: ... + def seekable(self) -> bool: ... + def flush(self) -> None: ... + @property + def closed(self) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class _PartialFile(_ProxyFile[AnyStr]): + def __init__(self, f: IO[AnyStr], start: int | None = None, stop: int | None = None) -> None: ... + +class Error(Exception): ... +class NoSuchMailboxError(Error): ... +class NotEmptyError(Error): ... +class ExternalClashError(Error): ... +class FormatError(Error): ... diff --git a/mypy/typeshed/stdlib/mailcap.pyi b/mypy/typeshed/stdlib/mailcap.pyi new file mode 100644 index 000000000000..ce549e01f528 --- /dev/null +++ b/mypy/typeshed/stdlib/mailcap.pyi @@ -0,0 +1,11 @@ +from collections.abc import Mapping, Sequence +from typing_extensions import TypeAlias + +_Cap: TypeAlias = dict[str, str | int] + +__all__ = ["getcaps", "findmatch"] + +def findmatch( + caps: Mapping[str, list[_Cap]], MIMEtype: str, key: str = "view", filename: str = "/dev/null", plist: Sequence[str] = [] +) -> tuple[str | None, _Cap | None]: ... +def getcaps() -> dict[str, list[_Cap]]: ... diff --git a/mypy/typeshed/stdlib/marshal.pyi b/mypy/typeshed/stdlib/marshal.pyi new file mode 100644 index 000000000000..6ab202637dda --- /dev/null +++ b/mypy/typeshed/stdlib/marshal.pyi @@ -0,0 +1,41 @@ +import builtins +import sys +import types +from _typeshed import ReadableBuffer, SupportsRead, SupportsWrite +from typing import Any +from typing_extensions import TypeAlias + +version: int + +_Marshallable: TypeAlias = ( + # handled in w_object() in marshal.c + None + | type[StopIteration] + | builtins.ellipsis + | bool + # handled in w_complex_object() in marshal.c + | int + | float + | complex + | bytes + | str + | tuple[_Marshallable, ...] + | list[Any] + | dict[Any, Any] + | set[Any] + | frozenset[_Marshallable] + | types.CodeType + | ReadableBuffer +) + +if sys.version_info >= (3, 13): + def dump(value: _Marshallable, file: SupportsWrite[bytes], version: int = 4, /, *, allow_code: bool = True) -> None: ... + def load(file: SupportsRead[bytes], /, *, allow_code: bool = True) -> Any: ... + def dumps(value: _Marshallable, version: int = 4, /, *, allow_code: bool = True) -> bytes: ... + def loads(bytes: ReadableBuffer, /, *, allow_code: bool = True) -> Any: ... + +else: + def dump(value: _Marshallable, file: SupportsWrite[bytes], version: int = 4, /) -> None: ... + def load(file: SupportsRead[bytes], /) -> Any: ... + def dumps(value: _Marshallable, version: int = 4, /) -> bytes: ... + def loads(bytes: ReadableBuffer, /) -> Any: ... diff --git a/mypy/typeshed/stdlib/math.pyi b/mypy/typeshed/stdlib/math.pyi new file mode 100644 index 000000000000..2bb61e0669b4 --- /dev/null +++ b/mypy/typeshed/stdlib/math.pyi @@ -0,0 +1,128 @@ +import sys +from collections.abc import Iterable +from typing import Protocol, SupportsFloat, SupportsIndex, TypeVar, overload +from typing_extensions import TypeAlias + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) + +_SupportsFloatOrIndex: TypeAlias = SupportsFloat | SupportsIndex + +e: float +pi: float +inf: float +nan: float +tau: float + +def acos(x: _SupportsFloatOrIndex, /) -> float: ... +def acosh(x: _SupportsFloatOrIndex, /) -> float: ... +def asin(x: _SupportsFloatOrIndex, /) -> float: ... +def asinh(x: _SupportsFloatOrIndex, /) -> float: ... +def atan(x: _SupportsFloatOrIndex, /) -> float: ... +def atan2(y: _SupportsFloatOrIndex, x: _SupportsFloatOrIndex, /) -> float: ... +def atanh(x: _SupportsFloatOrIndex, /) -> float: ... + +if sys.version_info >= (3, 11): + def cbrt(x: _SupportsFloatOrIndex, /) -> float: ... + +class _SupportsCeil(Protocol[_T_co]): + def __ceil__(self) -> _T_co: ... + +@overload +def ceil(x: _SupportsCeil[_T], /) -> _T: ... +@overload +def ceil(x: _SupportsFloatOrIndex, /) -> int: ... +def comb(n: SupportsIndex, k: SupportsIndex, /) -> int: ... +def copysign(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, /) -> float: ... +def cos(x: _SupportsFloatOrIndex, /) -> float: ... +def cosh(x: _SupportsFloatOrIndex, /) -> float: ... +def degrees(x: _SupportsFloatOrIndex, /) -> float: ... +def dist(p: Iterable[_SupportsFloatOrIndex], q: Iterable[_SupportsFloatOrIndex], /) -> float: ... +def erf(x: _SupportsFloatOrIndex, /) -> float: ... +def erfc(x: _SupportsFloatOrIndex, /) -> float: ... +def exp(x: _SupportsFloatOrIndex, /) -> float: ... + +if sys.version_info >= (3, 11): + def exp2(x: _SupportsFloatOrIndex, /) -> float: ... + +def expm1(x: _SupportsFloatOrIndex, /) -> float: ... +def fabs(x: _SupportsFloatOrIndex, /) -> float: ... +def factorial(x: SupportsIndex, /) -> int: ... + +class _SupportsFloor(Protocol[_T_co]): + def __floor__(self) -> _T_co: ... + +@overload +def floor(x: _SupportsFloor[_T], /) -> _T: ... +@overload +def floor(x: _SupportsFloatOrIndex, /) -> int: ... +def fmod(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, /) -> float: ... +def frexp(x: _SupportsFloatOrIndex, /) -> tuple[float, int]: ... +def fsum(seq: Iterable[_SupportsFloatOrIndex], /) -> float: ... +def gamma(x: _SupportsFloatOrIndex, /) -> float: ... + +if sys.version_info >= (3, 9): + def gcd(*integers: SupportsIndex) -> int: ... + +else: + def gcd(x: SupportsIndex, y: SupportsIndex, /) -> int: ... + +def hypot(*coordinates: _SupportsFloatOrIndex) -> float: ... +def isclose( + a: _SupportsFloatOrIndex, + b: _SupportsFloatOrIndex, + *, + rel_tol: _SupportsFloatOrIndex = 1e-09, + abs_tol: _SupportsFloatOrIndex = 0.0, +) -> bool: ... +def isinf(x: _SupportsFloatOrIndex, /) -> bool: ... +def isfinite(x: _SupportsFloatOrIndex, /) -> bool: ... +def isnan(x: _SupportsFloatOrIndex, /) -> bool: ... +def isqrt(n: SupportsIndex, /) -> int: ... + +if sys.version_info >= (3, 9): + def lcm(*integers: SupportsIndex) -> int: ... + +def ldexp(x: _SupportsFloatOrIndex, i: int, /) -> float: ... +def lgamma(x: _SupportsFloatOrIndex, /) -> float: ... +def log(x: _SupportsFloatOrIndex, base: _SupportsFloatOrIndex = ...) -> float: ... +def log10(x: _SupportsFloatOrIndex, /) -> float: ... +def log1p(x: _SupportsFloatOrIndex, /) -> float: ... +def log2(x: _SupportsFloatOrIndex, /) -> float: ... +def modf(x: _SupportsFloatOrIndex, /) -> tuple[float, float]: ... + +if sys.version_info >= (3, 12): + def nextafter(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, /, *, steps: SupportsIndex | None = None) -> float: ... + +elif sys.version_info >= (3, 9): + def nextafter(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, /) -> float: ... + +def perm(n: SupportsIndex, k: SupportsIndex | None = None, /) -> int: ... +def pow(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, /) -> float: ... +@overload +def prod(iterable: Iterable[SupportsIndex], /, *, start: SupportsIndex = 1) -> int: ... # type: ignore[overload-overlap] +@overload +def prod(iterable: Iterable[_SupportsFloatOrIndex], /, *, start: _SupportsFloatOrIndex = 1) -> float: ... +def radians(x: _SupportsFloatOrIndex, /) -> float: ... +def remainder(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, /) -> float: ... +def sin(x: _SupportsFloatOrIndex, /) -> float: ... +def sinh(x: _SupportsFloatOrIndex, /) -> float: ... + +if sys.version_info >= (3, 12): + def sumprod(p: Iterable[float], q: Iterable[float], /) -> float: ... + +def sqrt(x: _SupportsFloatOrIndex, /) -> float: ... +def tan(x: _SupportsFloatOrIndex, /) -> float: ... +def tanh(x: _SupportsFloatOrIndex, /) -> float: ... + +# Is different from `_typeshed.SupportsTrunc`, which is not generic +class _SupportsTrunc(Protocol[_T_co]): + def __trunc__(self) -> _T_co: ... + +def trunc(x: _SupportsTrunc[_T], /) -> _T: ... + +if sys.version_info >= (3, 9): + def ulp(x: _SupportsFloatOrIndex, /) -> float: ... + +if sys.version_info >= (3, 13): + def fma(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, z: _SupportsFloatOrIndex, /) -> float: ... diff --git a/mypy/typeshed/stdlib/mimetypes.pyi b/mypy/typeshed/stdlib/mimetypes.pyi new file mode 100644 index 000000000000..9914a34a2d6a --- /dev/null +++ b/mypy/typeshed/stdlib/mimetypes.pyi @@ -0,0 +1,56 @@ +import sys +from _typeshed import StrPath +from collections.abc import Sequence +from typing import IO + +__all__ = [ + "knownfiles", + "inited", + "MimeTypes", + "guess_type", + "guess_all_extensions", + "guess_extension", + "add_type", + "init", + "read_mime_types", + "suffix_map", + "encodings_map", + "types_map", + "common_types", +] + +if sys.version_info >= (3, 13): + __all__ += ["guess_file_type"] + +def guess_type(url: StrPath, strict: bool = True) -> tuple[str | None, str | None]: ... +def guess_all_extensions(type: str, strict: bool = True) -> list[str]: ... +def guess_extension(type: str, strict: bool = True) -> str | None: ... +def init(files: Sequence[str] | None = None) -> None: ... +def read_mime_types(file: str) -> dict[str, str] | None: ... +def add_type(type: str, ext: str, strict: bool = True) -> None: ... + +if sys.version_info >= (3, 13): + def guess_file_type(path: StrPath, *, strict: bool = True) -> tuple[str | None, str | None]: ... + +inited: bool +knownfiles: list[str] +suffix_map: dict[str, str] +encodings_map: dict[str, str] +types_map: dict[str, str] +common_types: dict[str, str] + +class MimeTypes: + suffix_map: dict[str, str] + encodings_map: dict[str, str] + types_map: tuple[dict[str, str], dict[str, str]] + types_map_inv: tuple[dict[str, str], dict[str, str]] + def __init__(self, filenames: tuple[str, ...] = (), strict: bool = True) -> None: ... + def add_type(self, type: str, ext: str, strict: bool = True) -> None: ... + def guess_extension(self, type: str, strict: bool = True) -> str | None: ... + def guess_type(self, url: StrPath, strict: bool = True) -> tuple[str | None, str | None]: ... + def guess_all_extensions(self, type: str, strict: bool = True) -> list[str]: ... + def read(self, filename: str, strict: bool = True) -> None: ... + def readfp(self, fp: IO[str], strict: bool = True) -> None: ... + def read_windows_registry(self, strict: bool = True) -> None: ... + if sys.version_info >= (3, 13): + def guess_file_type(self, path: StrPath, *, strict: bool = True) -> tuple[str | None, str | None]: ... diff --git a/mypy/typeshed/stdlib/mmap.pyi b/mypy/typeshed/stdlib/mmap.pyi new file mode 100644 index 000000000000..f94e876237d1 --- /dev/null +++ b/mypy/typeshed/stdlib/mmap.pyi @@ -0,0 +1,146 @@ +import sys +from _typeshed import ReadableBuffer, Unused +from collections.abc import Iterator +from typing import Final, Literal, NoReturn, overload +from typing_extensions import Self + +ACCESS_DEFAULT: int +ACCESS_READ: int +ACCESS_WRITE: int +ACCESS_COPY: int + +ALLOCATIONGRANULARITY: int + +if sys.platform == "linux": + MAP_DENYWRITE: int + MAP_EXECUTABLE: int + if sys.version_info >= (3, 10): + MAP_POPULATE: int +if sys.version_info >= (3, 11) and sys.platform != "win32" and sys.platform != "darwin": + MAP_STACK: int + +if sys.platform != "win32": + MAP_ANON: int + MAP_ANONYMOUS: int + MAP_PRIVATE: int + MAP_SHARED: int + PROT_EXEC: int + PROT_READ: int + PROT_WRITE: int + +PAGESIZE: int + +class mmap: + if sys.platform == "win32": + def __init__(self, fileno: int, length: int, tagname: str | None = ..., access: int = ..., offset: int = ...) -> None: ... + else: + if sys.version_info >= (3, 13): + def __init__( + self, + fileno: int, + length: int, + flags: int = ..., + prot: int = ..., + access: int = ..., + offset: int = ..., + *, + trackfd: bool = True, + ) -> None: ... + else: + def __init__( + self, fileno: int, length: int, flags: int = ..., prot: int = ..., access: int = ..., offset: int = ... + ) -> None: ... + + def close(self) -> None: ... + def flush(self, offset: int = ..., size: int = ...) -> None: ... + def move(self, dest: int, src: int, count: int) -> None: ... + def read_byte(self) -> int: ... + def readline(self) -> bytes: ... + def resize(self, newsize: int) -> None: ... + def seek(self, pos: int, whence: int = ...) -> None: ... + def size(self) -> int: ... + def tell(self) -> int: ... + def write_byte(self, byte: int) -> None: ... + def __len__(self) -> int: ... + closed: bool + if sys.platform != "win32": + def madvise(self, option: int, start: int = ..., length: int = ...) -> None: ... + + def find(self, sub: ReadableBuffer, start: int = ..., stop: int = ...) -> int: ... + def rfind(self, sub: ReadableBuffer, start: int = ..., stop: int = ...) -> int: ... + def read(self, n: int | None = ...) -> bytes: ... + def write(self, bytes: ReadableBuffer) -> int: ... + @overload + def __getitem__(self, key: int, /) -> int: ... + @overload + def __getitem__(self, key: slice, /) -> bytes: ... + def __delitem__(self, key: int | slice, /) -> NoReturn: ... + @overload + def __setitem__(self, key: int, value: int, /) -> None: ... + @overload + def __setitem__(self, key: slice, value: ReadableBuffer, /) -> None: ... + # Doesn't actually exist, but the object actually supports "in" because it has __getitem__, + # so we claim that there is also a __contains__ to help type checkers. + def __contains__(self, o: object, /) -> bool: ... + # Doesn't actually exist, but the object is actually iterable because it has __getitem__ and __len__, + # so we claim that there is also an __iter__ to help type checkers. + def __iter__(self) -> Iterator[int]: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def __buffer__(self, flags: int, /) -> memoryview: ... + def __release_buffer__(self, buffer: memoryview, /) -> None: ... + if sys.version_info >= (3, 13): + def seekable(self) -> Literal[True]: ... + +if sys.platform != "win32": + MADV_NORMAL: int + MADV_RANDOM: int + MADV_SEQUENTIAL: int + MADV_WILLNEED: int + MADV_DONTNEED: int + MADV_FREE: int + +if sys.platform == "linux": + MADV_REMOVE: int + MADV_DONTFORK: int + MADV_DOFORK: int + MADV_HWPOISON: int + MADV_MERGEABLE: int + MADV_UNMERGEABLE: int + # Seems like this constant is not defined in glibc. + # See https://github.com/python/typeshed/pull/5360 for details + # MADV_SOFT_OFFLINE: int + MADV_HUGEPAGE: int + MADV_NOHUGEPAGE: int + MADV_DONTDUMP: int + MADV_DODUMP: int + +# This Values are defined for FreeBSD but type checkers do not support conditions for these +if sys.platform != "linux" and sys.platform != "darwin" and sys.platform != "win32": + MADV_NOSYNC: int + MADV_AUTOSYNC: int + MADV_NOCORE: int + MADV_CORE: int + MADV_PROTECT: int + +if sys.version_info >= (3, 10) and sys.platform == "darwin": + MADV_FREE_REUSABLE: int + MADV_FREE_REUSE: int + +if sys.version_info >= (3, 13) and sys.platform != "win32": + MAP_32BIT: Final = 32768 + +if sys.version_info >= (3, 13) and sys.platform == "darwin": + MAP_NORESERVE: Final = 64 + MAP_NOEXTEND: Final = 256 + MAP_HASSEMAPHORE: Final = 512 + MAP_NOCACHE: Final = 1024 + MAP_JIT: Final = 2048 + MAP_RESILIENT_CODESIGN: Final = 8192 + MAP_RESILIENT_MEDIA: Final = 16384 + MAP_TRANSLATED_ALLOW_EXECUTE: Final = 131072 + MAP_UNIX03: Final = 262144 + MAP_TPRO: Final = 524288 + +if sys.version_info >= (3, 13) and sys.platform == "linux": + MAP_NORESERVE: Final = 16384 diff --git a/mypy/typeshed/stdlib/modulefinder.pyi b/mypy/typeshed/stdlib/modulefinder.pyi new file mode 100644 index 000000000000..6db665a18e69 --- /dev/null +++ b/mypy/typeshed/stdlib/modulefinder.pyi @@ -0,0 +1,68 @@ +import sys +from collections.abc import Container, Iterable, Iterator, Sequence +from types import CodeType +from typing import IO, Any, Final + +if sys.version_info < (3, 11): + LOAD_CONST: Final[int] # undocumented + IMPORT_NAME: Final[int] # undocumented + STORE_NAME: Final[int] # undocumented + STORE_GLOBAL: Final[int] # undocumented + STORE_OPS: Final[tuple[int, int]] # undocumented + EXTENDED_ARG: Final[int] # undocumented + +packagePathMap: dict[str, list[str]] # undocumented + +def AddPackagePath(packagename: str, path: str) -> None: ... + +replacePackageMap: dict[str, str] # undocumented + +def ReplacePackage(oldname: str, newname: str) -> None: ... + +class Module: # undocumented + def __init__(self, name: str, file: str | None = None, path: str | None = None) -> None: ... + +class ModuleFinder: + modules: dict[str, Module] + path: list[str] # undocumented + badmodules: dict[str, dict[str, int]] # undocumented + debug: int # undocumented + indent: int # undocumented + excludes: Container[str] # undocumented + replace_paths: Sequence[tuple[str, str]] # undocumented + + def __init__( + self, + path: list[str] | None = None, + debug: int = 0, + excludes: Container[str] | None = None, + replace_paths: Sequence[tuple[str, str]] | None = None, + ) -> None: ... + def msg(self, level: int, str: str, *args: Any) -> None: ... # undocumented + def msgin(self, *args: Any) -> None: ... # undocumented + def msgout(self, *args: Any) -> None: ... # undocumented + def run_script(self, pathname: str) -> None: ... + def load_file(self, pathname: str) -> None: ... # undocumented + def import_hook( + self, name: str, caller: Module | None = None, fromlist: list[str] | None = None, level: int = -1 + ) -> Module | None: ... # undocumented + def determine_parent(self, caller: Module | None, level: int = -1) -> Module | None: ... # undocumented + def find_head_package(self, parent: Module, name: str) -> tuple[Module, str]: ... # undocumented + def load_tail(self, q: Module, tail: str) -> Module: ... # undocumented + def ensure_fromlist(self, m: Module, fromlist: Iterable[str], recursive: int = 0) -> None: ... # undocumented + def find_all_submodules(self, m: Module) -> Iterable[str]: ... # undocumented + def import_module(self, partname: str, fqname: str, parent: Module) -> Module | None: ... # undocumented + def load_module(self, fqname: str, fp: IO[str], pathname: str, file_info: tuple[str, str, str]) -> Module: ... # undocumented + def scan_opcodes(self, co: CodeType) -> Iterator[tuple[str, tuple[Any, ...]]]: ... # undocumented + def scan_code(self, co: CodeType, m: Module) -> None: ... # undocumented + def load_package(self, fqname: str, pathname: str) -> Module: ... # undocumented + def add_module(self, fqname: str) -> Module: ... # undocumented + def find_module( + self, name: str, path: str | None, parent: Module | None = None + ) -> tuple[IO[Any] | None, str | None, tuple[str, str, int]]: ... # undocumented + def report(self) -> None: ... + def any_missing(self) -> list[str]: ... # undocumented + def any_missing_maybe(self) -> tuple[list[str], list[str]]: ... # undocumented + def replace_paths_in_code(self, co: CodeType) -> CodeType: ... # undocumented + +def test() -> ModuleFinder | None: ... # undocumented diff --git a/mypy/typeshed/stdlib/msilib/__init__.pyi b/mypy/typeshed/stdlib/msilib/__init__.pyi new file mode 100644 index 000000000000..3e43cbc44f52 --- /dev/null +++ b/mypy/typeshed/stdlib/msilib/__init__.pyi @@ -0,0 +1,177 @@ +import sys +from collections.abc import Container, Iterable, Sequence +from types import ModuleType +from typing import Any, Literal + +if sys.platform == "win32": + from _msi import * + from _msi import _Database + + AMD64: bool + Win64: bool + + datasizemask: Literal[0x00FF] + type_valid: Literal[0x0100] + type_localizable: Literal[0x0200] + typemask: Literal[0x0C00] + type_long: Literal[0x0000] + type_short: Literal[0x0400] + type_string: Literal[0x0C00] + type_binary: Literal[0x0800] + type_nullable: Literal[0x1000] + type_key: Literal[0x2000] + knownbits: Literal[0x3FFF] + + class Table: + name: str + fields: list[tuple[int, str, int]] + def __init__(self, name: str) -> None: ... + def add_field(self, index: int, name: str, type: int) -> None: ... + def sql(self) -> str: ... + def create(self, db: _Database) -> None: ... + + class _Unspecified: ... + + def change_sequence( + seq: Sequence[tuple[str, str | None, int]], + action: str, + seqno: int | type[_Unspecified] = ..., + cond: str | type[_Unspecified] = ..., + ) -> None: ... + def add_data(db: _Database, table: str, values: Iterable[tuple[Any, ...]]) -> None: ... + def add_stream(db: _Database, name: str, path: str) -> None: ... + def init_database( + name: str, schema: ModuleType, ProductName: str, ProductCode: str, ProductVersion: str, Manufacturer: str + ) -> _Database: ... + def add_tables(db: _Database, module: ModuleType) -> None: ... + def make_id(str: str) -> str: ... + def gen_uuid() -> str: ... + + class CAB: + name: str + files: list[tuple[str, str]] + filenames: set[str] + index: int + def __init__(self, name: str) -> None: ... + def gen_id(self, file: str) -> str: ... + def append(self, full: str, file: str, logical: str) -> tuple[int, str]: ... + def commit(self, db: _Database) -> None: ... + + _directories: set[str] + + class Directory: + db: _Database + cab: CAB + basedir: str + physical: str + logical: str + component: str | None + short_names: set[str] + ids: set[str] + keyfiles: dict[str, str] + componentflags: int | None + absolute: str + def __init__( + self, + db: _Database, + cab: CAB, + basedir: str, + physical: str, + _logical: str, + default: str, + componentflags: int | None = None, + ) -> None: ... + def start_component( + self, + component: str | None = None, + feature: Feature | None = None, + flags: int | None = None, + keyfile: str | None = None, + uuid: str | None = None, + ) -> None: ... + def make_short(self, file: str) -> str: ... + def add_file(self, file: str, src: str | None = None, version: str | None = None, language: str | None = None) -> str: ... + def glob(self, pattern: str, exclude: Container[str] | None = None) -> list[str]: ... + def remove_pyc(self) -> None: ... + + class Binary: + name: str + def __init__(self, fname: str) -> None: ... + + class Feature: + id: str + def __init__( + self, + db: _Database, + id: str, + title: str, + desc: str, + display: int, + level: int = 1, + parent: Feature | None = None, + directory: str | None = None, + attributes: int = 0, + ) -> None: ... + def set_current(self) -> None: ... + + class Control: + dlg: Dialog + name: str + def __init__(self, dlg: Dialog, name: str) -> None: ... + def event(self, event: str, argument: str, condition: str = "1", ordering: int | None = None) -> None: ... + def mapping(self, event: str, attribute: str) -> None: ... + def condition(self, action: str, condition: str) -> None: ... + + class RadioButtonGroup(Control): + property: str + index: int + def __init__(self, dlg: Dialog, name: str, property: str) -> None: ... + def add(self, name: str, x: int, y: int, w: int, h: int, text: str, value: str | None = None) -> None: ... + + class Dialog: + db: _Database + name: str + x: int + y: int + w: int + h: int + def __init__( + self, + db: _Database, + name: str, + x: int, + y: int, + w: int, + h: int, + attr: int, + title: str, + first: str, + default: str, + cancel: str, + ) -> None: ... + def control( + self, + name: str, + type: str, + x: int, + y: int, + w: int, + h: int, + attr: int, + prop: str | None, + text: str | None, + next: str | None, + help: str | None, + ) -> Control: ... + def text(self, name: str, x: int, y: int, w: int, h: int, attr: int, text: str | None) -> Control: ... + def bitmap(self, name: str, x: int, y: int, w: int, h: int, text: str | None) -> Control: ... + def line(self, name: str, x: int, y: int, w: int, h: int) -> Control: ... + def pushbutton( + self, name: str, x: int, y: int, w: int, h: int, attr: int, text: str | None, next: str | None + ) -> Control: ... + def radiogroup( + self, name: str, x: int, y: int, w: int, h: int, attr: int, prop: str | None, text: str | None, next: str | None + ) -> RadioButtonGroup: ... + def checkbox( + self, name: str, x: int, y: int, w: int, h: int, attr: int, prop: str | None, text: str | None, next: str | None + ) -> Control: ... diff --git a/mypy/typeshed/stdlib/msilib/schema.pyi b/mypy/typeshed/stdlib/msilib/schema.pyi new file mode 100644 index 000000000000..4ad9a1783fcd --- /dev/null +++ b/mypy/typeshed/stdlib/msilib/schema.pyi @@ -0,0 +1,94 @@ +import sys + +if sys.platform == "win32": + from . import Table + + _Validation: Table + ActionText: Table + AdminExecuteSequence: Table + Condition: Table + AdminUISequence: Table + AdvtExecuteSequence: Table + AdvtUISequence: Table + AppId: Table + AppSearch: Table + Property: Table + BBControl: Table + Billboard: Table + Feature: Table + Binary: Table + BindImage: Table + File: Table + CCPSearch: Table + CheckBox: Table + Class: Table + Component: Table + Icon: Table + ProgId: Table + ComboBox: Table + CompLocator: Table + Complus: Table + Directory: Table + Control: Table + Dialog: Table + ControlCondition: Table + ControlEvent: Table + CreateFolder: Table + CustomAction: Table + DrLocator: Table + DuplicateFile: Table + Environment: Table + Error: Table + EventMapping: Table + Extension: Table + MIME: Table + FeatureComponents: Table + FileSFPCatalog: Table + SFPCatalog: Table + Font: Table + IniFile: Table + IniLocator: Table + InstallExecuteSequence: Table + InstallUISequence: Table + IsolatedComponent: Table + LaunchCondition: Table + ListBox: Table + ListView: Table + LockPermissions: Table + Media: Table + MoveFile: Table + MsiAssembly: Table + MsiAssemblyName: Table + MsiDigitalCertificate: Table + MsiDigitalSignature: Table + MsiFileHash: Table + MsiPatchHeaders: Table + ODBCAttribute: Table + ODBCDriver: Table + ODBCDataSource: Table + ODBCSourceAttribute: Table + ODBCTranslator: Table + Patch: Table + PatchPackage: Table + PublishComponent: Table + RadioButton: Table + Registry: Table + RegLocator: Table + RemoveFile: Table + RemoveIniFile: Table + RemoveRegistry: Table + ReserveCost: Table + SelfReg: Table + ServiceControl: Table + ServiceInstall: Table + Shortcut: Table + Signature: Table + TextStyle: Table + TypeLib: Table + UIText: Table + Upgrade: Table + Verb: Table + + tables: list[Table] + + _Validation_records: list[tuple[str, str, str, int | None, int | None, str | None, int | None, str | None, str | None, str]] diff --git a/mypy/typeshed/stdlib/msilib/sequence.pyi b/mypy/typeshed/stdlib/msilib/sequence.pyi new file mode 100644 index 000000000000..b8af09f46e65 --- /dev/null +++ b/mypy/typeshed/stdlib/msilib/sequence.pyi @@ -0,0 +1,13 @@ +import sys +from typing_extensions import TypeAlias + +if sys.platform == "win32": + _SequenceType: TypeAlias = list[tuple[str, str | None, int]] + + AdminExecuteSequence: _SequenceType + AdminUISequence: _SequenceType + AdvtExecuteSequence: _SequenceType + InstallExecuteSequence: _SequenceType + InstallUISequence: _SequenceType + + tables: list[str] diff --git a/mypy/typeshed/stdlib/msilib/text.pyi b/mypy/typeshed/stdlib/msilib/text.pyi new file mode 100644 index 000000000000..441c843ca6cf --- /dev/null +++ b/mypy/typeshed/stdlib/msilib/text.pyi @@ -0,0 +1,7 @@ +import sys + +if sys.platform == "win32": + ActionText: list[tuple[str, str, str | None]] + UIText: list[tuple[str, str | None]] + dirname: str + tables: list[str] diff --git a/mypy/typeshed/stdlib/msvcrt.pyi b/mypy/typeshed/stdlib/msvcrt.pyi new file mode 100644 index 000000000000..403a5d933522 --- /dev/null +++ b/mypy/typeshed/stdlib/msvcrt.pyi @@ -0,0 +1,32 @@ +import sys +from typing import Final + +# This module is only available on Windows +if sys.platform == "win32": + CRT_ASSEMBLY_VERSION: Final[str] + LK_UNLCK: Final = 0 + LK_LOCK: Final = 1 + LK_NBLCK: Final = 2 + LK_RLCK: Final = 3 + LK_NBRLCK: Final = 4 + SEM_FAILCRITICALERRORS: int + SEM_NOALIGNMENTFAULTEXCEPT: int + SEM_NOGPFAULTERRORBOX: int + SEM_NOOPENFILEERRORBOX: int + def locking(fd: int, mode: int, nbytes: int, /) -> None: ... + def setmode(fd: int, mode: int, /) -> int: ... + def open_osfhandle(handle: int, flags: int, /) -> int: ... + def get_osfhandle(fd: int, /) -> int: ... + def kbhit() -> bool: ... + def getch() -> bytes: ... + def getwch() -> str: ... + def getche() -> bytes: ... + def getwche() -> str: ... + def putch(char: bytes | bytearray, /) -> None: ... + def putwch(unicode_char: str, /) -> None: ... + def ungetch(char: bytes | bytearray, /) -> None: ... + def ungetwch(unicode_char: str, /) -> None: ... + def heapmin() -> None: ... + def SetErrorMode(mode: int, /) -> int: ... + if sys.version_info >= (3, 10): + def GetErrorMode() -> int: ... # undocumented diff --git a/mypy/typeshed/stdlib/multiprocessing/__init__.pyi b/mypy/typeshed/stdlib/multiprocessing/__init__.pyi new file mode 100644 index 000000000000..2bd6e2883ddb --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/__init__.pyi @@ -0,0 +1,90 @@ +from multiprocessing import context, reduction as reducer +from multiprocessing.context import ( + AuthenticationError as AuthenticationError, + BufferTooShort as BufferTooShort, + Process as Process, + ProcessError as ProcessError, + TimeoutError as TimeoutError, +) +from multiprocessing.process import ( + active_children as active_children, + current_process as current_process, + parent_process as parent_process, +) + +# These are technically functions that return instances of these Queue classes. +# The stub here doesn't reflect reality exactly -- +# while e.g. `multiprocessing.queues.Queue` is a class, +# `multiprocessing.Queue` is actually a function at runtime. +# Avoid using `multiprocessing.Queue` as a type annotation; +# use imports from multiprocessing.queues instead. +# See #4266 and #8450 for discussion. +from multiprocessing.queues import JoinableQueue as JoinableQueue, Queue as Queue, SimpleQueue as SimpleQueue +from multiprocessing.spawn import freeze_support as freeze_support + +__all__ = [ + "Array", + "AuthenticationError", + "Barrier", + "BoundedSemaphore", + "BufferTooShort", + "Condition", + "Event", + "JoinableQueue", + "Lock", + "Manager", + "Pipe", + "Pool", + "Process", + "ProcessError", + "Queue", + "RLock", + "RawArray", + "RawValue", + "Semaphore", + "SimpleQueue", + "TimeoutError", + "Value", + "active_children", + "allow_connection_pickling", + "cpu_count", + "current_process", + "freeze_support", + "get_all_start_methods", + "get_context", + "get_logger", + "get_start_method", + "log_to_stderr", + "parent_process", + "reducer", + "set_executable", + "set_forkserver_preload", + "set_start_method", +] + +# These functions (really bound methods) +# are all autogenerated at runtime here: https://github.com/python/cpython/blob/600c65c094b0b48704d8ec2416930648052ba715/Lib/multiprocessing/__init__.py#L23 +RawValue = context._default_context.RawValue +RawArray = context._default_context.RawArray +Value = context._default_context.Value +Array = context._default_context.Array +Barrier = context._default_context.Barrier +BoundedSemaphore = context._default_context.BoundedSemaphore +Condition = context._default_context.Condition +Event = context._default_context.Event +Lock = context._default_context.Lock +RLock = context._default_context.RLock +Semaphore = context._default_context.Semaphore +Pipe = context._default_context.Pipe +Pool = context._default_context.Pool +allow_connection_pickling = context._default_context.allow_connection_pickling +cpu_count = context._default_context.cpu_count +get_logger = context._default_context.get_logger +log_to_stderr = context._default_context.log_to_stderr +Manager = context._default_context.Manager +set_executable = context._default_context.set_executable +set_forkserver_preload = context._default_context.set_forkserver_preload +get_all_start_methods = context._default_context.get_all_start_methods +get_start_method = context._default_context.get_start_method +set_start_method = context._default_context.set_start_method +get_context = context._default_context.get_context diff --git a/mypy/typeshed/stdlib/multiprocessing/connection.pyi b/mypy/typeshed/stdlib/multiprocessing/connection.pyi new file mode 100644 index 000000000000..9998239d3119 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/connection.pyi @@ -0,0 +1,83 @@ +import socket +import sys +from _typeshed import Incomplete, ReadableBuffer +from collections.abc import Iterable +from types import TracebackType +from typing import Any, Generic, SupportsIndex, TypeVar +from typing_extensions import Self, TypeAlias + +__all__ = ["Client", "Listener", "Pipe", "wait"] + +# https://docs.python.org/3/library/multiprocessing.html#address-formats +_Address: TypeAlias = str | tuple[str, int] + +# Defaulting to Any to avoid forcing generics on a lot of pre-existing code +_SendT = TypeVar("_SendT", contravariant=True, default=Any) +_RecvT = TypeVar("_RecvT", covariant=True, default=Any) + +class _ConnectionBase(Generic[_SendT, _RecvT]): + def __init__(self, handle: SupportsIndex, readable: bool = True, writable: bool = True) -> None: ... + @property + def closed(self) -> bool: ... # undocumented + @property + def readable(self) -> bool: ... # undocumented + @property + def writable(self) -> bool: ... # undocumented + def fileno(self) -> int: ... + def close(self) -> None: ... + def send_bytes(self, buf: ReadableBuffer, offset: int = 0, size: int | None = None) -> None: ... + def send(self, obj: _SendT) -> None: ... + def recv_bytes(self, maxlength: int | None = None) -> bytes: ... + def recv_bytes_into(self, buf: Any, offset: int = 0) -> int: ... + def recv(self) -> _RecvT: ... + def poll(self, timeout: float | None = 0.0) -> bool: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def __del__(self) -> None: ... + +class Connection(_ConnectionBase[_SendT, _RecvT]): ... + +if sys.platform == "win32": + class PipeConnection(_ConnectionBase[_SendT, _RecvT]): ... + +class Listener: + def __init__( + self, address: _Address | None = None, family: str | None = None, backlog: int = 1, authkey: bytes | None = None + ) -> None: ... + def accept(self) -> Connection[Incomplete, Incomplete]: ... + def close(self) -> None: ... + @property + def address(self) -> _Address: ... + @property + def last_accepted(self) -> _Address | None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + +# Any: send and recv methods unused +if sys.version_info >= (3, 12): + def deliver_challenge(connection: Connection[Any, Any], authkey: bytes, digest_name: str = "sha256") -> None: ... + +else: + def deliver_challenge(connection: Connection[Any, Any], authkey: bytes) -> None: ... + +def answer_challenge(connection: Connection[Any, Any], authkey: bytes) -> None: ... +def wait( + object_list: Iterable[Connection[_SendT, _RecvT] | socket.socket | int], timeout: float | None = None +) -> list[Connection[_SendT, _RecvT] | socket.socket | int]: ... +def Client(address: _Address, family: str | None = None, authkey: bytes | None = None) -> Connection[Any, Any]: ... + +# N.B. Keep this in sync with multiprocessing.context.BaseContext.Pipe. +# _ConnectionBase is the common base class of Connection and PipeConnection +# and can be used in cross-platform code. +# +# The two connections should have the same generic types but inverted (Connection[_T1, _T2], Connection[_T2, _T1]). +# However, TypeVars scoped entirely within a return annotation is unspecified in the spec. +if sys.platform != "win32": + def Pipe(duplex: bool = True) -> tuple[Connection[Any, Any], Connection[Any, Any]]: ... + +else: + def Pipe(duplex: bool = True) -> tuple[PipeConnection[Any, Any], PipeConnection[Any, Any]]: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/context.pyi b/mypy/typeshed/stdlib/multiprocessing/context.pyi new file mode 100644 index 000000000000..03d1d2e5c220 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/context.pyi @@ -0,0 +1,206 @@ +import ctypes +import sys +from _ctypes import _CData +from collections.abc import Callable, Iterable, Sequence +from ctypes import _SimpleCData, c_char +from logging import Logger, _Level as _LoggingLevel +from multiprocessing import popen_fork, popen_forkserver, popen_spawn_posix, popen_spawn_win32, queues, synchronize +from multiprocessing.managers import SyncManager +from multiprocessing.pool import Pool as _Pool +from multiprocessing.process import BaseProcess +from multiprocessing.sharedctypes import Synchronized, SynchronizedArray, SynchronizedString +from typing import Any, ClassVar, Literal, TypeVar, overload +from typing_extensions import TypeAlias + +if sys.platform != "win32": + from multiprocessing.connection import Connection +else: + from multiprocessing.connection import PipeConnection + +__all__ = () + +_LockLike: TypeAlias = synchronize.Lock | synchronize.RLock +_T = TypeVar("_T") +_CT = TypeVar("_CT", bound=_CData) + +class ProcessError(Exception): ... +class BufferTooShort(ProcessError): ... +class TimeoutError(ProcessError): ... +class AuthenticationError(ProcessError): ... + +class BaseContext: + ProcessError: ClassVar[type[ProcessError]] + BufferTooShort: ClassVar[type[BufferTooShort]] + TimeoutError: ClassVar[type[TimeoutError]] + AuthenticationError: ClassVar[type[AuthenticationError]] + + # N.B. The methods below are applied at runtime to generate + # multiprocessing.*, so the signatures should be identical (modulo self). + @staticmethod + def current_process() -> BaseProcess: ... + @staticmethod + def parent_process() -> BaseProcess | None: ... + @staticmethod + def active_children() -> list[BaseProcess]: ... + def cpu_count(self) -> int: ... + def Manager(self) -> SyncManager: ... + + # N.B. Keep this in sync with multiprocessing.connection.Pipe. + # _ConnectionBase is the common base class of Connection and PipeConnection + # and can be used in cross-platform code. + # + # The two connections should have the same generic types but inverted (Connection[_T1, _T2], Connection[_T2, _T1]). + # However, TypeVars scoped entirely within a return annotation is unspecified in the spec. + if sys.platform != "win32": + def Pipe(self, duplex: bool = True) -> tuple[Connection[Any, Any], Connection[Any, Any]]: ... + else: + def Pipe(self, duplex: bool = True) -> tuple[PipeConnection[Any, Any], PipeConnection[Any, Any]]: ... + + def Barrier( + self, parties: int, action: Callable[..., object] | None = None, timeout: float | None = None + ) -> synchronize.Barrier: ... + def BoundedSemaphore(self, value: int = 1) -> synchronize.BoundedSemaphore: ... + def Condition(self, lock: _LockLike | None = None) -> synchronize.Condition: ... + def Event(self) -> synchronize.Event: ... + def Lock(self) -> synchronize.Lock: ... + def RLock(self) -> synchronize.RLock: ... + def Semaphore(self, value: int = 1) -> synchronize.Semaphore: ... + def Queue(self, maxsize: int = 0) -> queues.Queue[Any]: ... + def JoinableQueue(self, maxsize: int = 0) -> queues.JoinableQueue[Any]: ... + def SimpleQueue(self) -> queues.SimpleQueue[Any]: ... + def Pool( + self, + processes: int | None = None, + initializer: Callable[..., object] | None = None, + initargs: Iterable[Any] = (), + maxtasksperchild: int | None = None, + ) -> _Pool: ... + @overload + def RawValue(self, typecode_or_type: type[_CT], *args: Any) -> _CT: ... + @overload + def RawValue(self, typecode_or_type: str, *args: Any) -> Any: ... + @overload + def RawArray(self, typecode_or_type: type[_CT], size_or_initializer: int | Sequence[Any]) -> ctypes.Array[_CT]: ... + @overload + def RawArray(self, typecode_or_type: str, size_or_initializer: int | Sequence[Any]) -> Any: ... + @overload + def Value( + self, typecode_or_type: type[_SimpleCData[_T]], *args: Any, lock: Literal[True] | _LockLike = True + ) -> Synchronized[_T]: ... + @overload + def Value(self, typecode_or_type: type[_CT], *args: Any, lock: Literal[False]) -> Synchronized[_CT]: ... + @overload + def Value(self, typecode_or_type: type[_CT], *args: Any, lock: Literal[True] | _LockLike = True) -> Synchronized[_CT]: ... + @overload + def Value(self, typecode_or_type: str, *args: Any, lock: Literal[True] | _LockLike = True) -> Synchronized[Any]: ... + @overload + def Value(self, typecode_or_type: str | type[_CData], *args: Any, lock: bool | _LockLike = True) -> Any: ... + @overload + def Array( + self, typecode_or_type: type[_SimpleCData[_T]], size_or_initializer: int | Sequence[Any], *, lock: Literal[False] + ) -> SynchronizedArray[_T]: ... + @overload + def Array( + self, typecode_or_type: type[c_char], size_or_initializer: int | Sequence[Any], *, lock: Literal[True] | _LockLike = True + ) -> SynchronizedString: ... + @overload + def Array( + self, + typecode_or_type: type[_SimpleCData[_T]], + size_or_initializer: int | Sequence[Any], + *, + lock: Literal[True] | _LockLike = True, + ) -> SynchronizedArray[_T]: ... + @overload + def Array( + self, typecode_or_type: str, size_or_initializer: int | Sequence[Any], *, lock: Literal[True] | _LockLike = True + ) -> SynchronizedArray[Any]: ... + @overload + def Array( + self, typecode_or_type: str | type[_CData], size_or_initializer: int | Sequence[Any], *, lock: bool | _LockLike = True + ) -> Any: ... + def freeze_support(self) -> None: ... + def get_logger(self) -> Logger: ... + def log_to_stderr(self, level: _LoggingLevel | None = None) -> Logger: ... + def allow_connection_pickling(self) -> None: ... + def set_executable(self, executable: str) -> None: ... + def set_forkserver_preload(self, module_names: list[str]) -> None: ... + if sys.platform != "win32": + @overload + def get_context(self, method: None = None) -> DefaultContext: ... + @overload + def get_context(self, method: Literal["spawn"]) -> SpawnContext: ... + @overload + def get_context(self, method: Literal["fork"]) -> ForkContext: ... + @overload + def get_context(self, method: Literal["forkserver"]) -> ForkServerContext: ... + @overload + def get_context(self, method: str) -> BaseContext: ... + else: + @overload + def get_context(self, method: None = None) -> DefaultContext: ... + @overload + def get_context(self, method: Literal["spawn"]) -> SpawnContext: ... + @overload + def get_context(self, method: str) -> BaseContext: ... + + @overload + def get_start_method(self, allow_none: Literal[False] = False) -> str: ... + @overload + def get_start_method(self, allow_none: bool) -> str | None: ... + def set_start_method(self, method: str | None, force: bool = False) -> None: ... + @property + def reducer(self) -> str: ... + @reducer.setter + def reducer(self, reduction: str) -> None: ... + def _check_available(self) -> None: ... + +class Process(BaseProcess): + _start_method: str | None + @staticmethod + def _Popen(process_obj: BaseProcess) -> DefaultContext: ... + +class DefaultContext(BaseContext): + Process: ClassVar[type[Process]] + def __init__(self, context: BaseContext) -> None: ... + def get_start_method(self, allow_none: bool = False) -> str: ... + def get_all_start_methods(self) -> list[str]: ... + +_default_context: DefaultContext + +class SpawnProcess(BaseProcess): + _start_method: str + if sys.platform != "win32": + @staticmethod + def _Popen(process_obj: BaseProcess) -> popen_spawn_posix.Popen: ... + else: + @staticmethod + def _Popen(process_obj: BaseProcess) -> popen_spawn_win32.Popen: ... + +class SpawnContext(BaseContext): + _name: str + Process: ClassVar[type[SpawnProcess]] + +if sys.platform != "win32": + class ForkProcess(BaseProcess): + _start_method: str + @staticmethod + def _Popen(process_obj: BaseProcess) -> popen_fork.Popen: ... + + class ForkServerProcess(BaseProcess): + _start_method: str + @staticmethod + def _Popen(process_obj: BaseProcess) -> popen_forkserver.Popen: ... + + class ForkContext(BaseContext): + _name: str + Process: ClassVar[type[ForkProcess]] + + class ForkServerContext(BaseContext): + _name: str + Process: ClassVar[type[ForkServerProcess]] + +def _force_start_method(method: str) -> None: ... +def get_spawning_popen() -> Any | None: ... +def set_spawning_popen(popen: Any) -> None: ... +def assert_spawning(obj: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/dummy/__init__.pyi b/mypy/typeshed/stdlib/multiprocessing/dummy/__init__.pyi new file mode 100644 index 000000000000..3cbeeb057791 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/dummy/__init__.pyi @@ -0,0 +1,77 @@ +import array +import threading +import weakref +from collections.abc import Callable, Iterable, Mapping, Sequence +from queue import Queue as Queue +from threading import ( + Barrier as Barrier, + BoundedSemaphore as BoundedSemaphore, + Condition as Condition, + Event as Event, + Lock as Lock, + RLock as RLock, + Semaphore as Semaphore, +) +from typing import Any, Literal + +from .connection import Pipe as Pipe + +__all__ = [ + "Process", + "current_process", + "active_children", + "freeze_support", + "Lock", + "RLock", + "Semaphore", + "BoundedSemaphore", + "Condition", + "Event", + "Barrier", + "Queue", + "Manager", + "Pipe", + "Pool", + "JoinableQueue", +] + +JoinableQueue = Queue + +class DummyProcess(threading.Thread): + _children: weakref.WeakKeyDictionary[Any, Any] + _parent: threading.Thread + _pid: None + _start_called: int + @property + def exitcode(self) -> Literal[0] | None: ... + def __init__( + self, + group: Any = None, + target: Callable[..., object] | None = None, + name: str | None = None, + args: Iterable[Any] = (), + kwargs: Mapping[str, Any] = {}, + ) -> None: ... + +Process = DummyProcess + +class Namespace: + def __init__(self, **kwds: Any) -> None: ... + def __getattr__(self, name: str, /) -> Any: ... + def __setattr__(self, name: str, value: Any, /) -> None: ... + +class Value: + _typecode: Any + _value: Any + value: Any + def __init__(self, typecode: Any, value: Any, lock: Any = True) -> None: ... + +def Array(typecode: Any, sequence: Sequence[Any], lock: Any = True) -> array.array[Any]: ... +def Manager() -> Any: ... +def Pool(processes: int | None = None, initializer: Callable[..., object] | None = None, initargs: Iterable[Any] = ()) -> Any: ... +def active_children() -> list[Any]: ... + +current_process = threading.current_thread + +def freeze_support() -> None: ... +def shutdown() -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/dummy/connection.pyi b/mypy/typeshed/stdlib/multiprocessing/dummy/connection.pyi new file mode 100644 index 000000000000..d7e982129466 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/dummy/connection.pyi @@ -0,0 +1,39 @@ +from multiprocessing.connection import _Address +from queue import Queue +from types import TracebackType +from typing import Any +from typing_extensions import Self + +__all__ = ["Client", "Listener", "Pipe"] + +families: list[None] + +class Connection: + _in: Any + _out: Any + recv: Any + recv_bytes: Any + send: Any + send_bytes: Any + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def __init__(self, _in: Any, _out: Any) -> None: ... + def close(self) -> None: ... + def poll(self, timeout: float = 0.0) -> bool: ... + +class Listener: + _backlog_queue: Queue[Any] | None + @property + def address(self) -> Queue[Any] | None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def __init__(self, address: _Address | None = None, family: int | None = None, backlog: int = 1) -> None: ... + def accept(self) -> Connection: ... + def close(self) -> None: ... + +def Client(address: _Address) -> Connection: ... +def Pipe(duplex: bool = True) -> tuple[Connection, Connection]: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi b/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi new file mode 100644 index 000000000000..31b982856355 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi @@ -0,0 +1,31 @@ +from _typeshed import FileDescriptorLike, Unused +from collections.abc import Sequence +from struct import Struct +from typing import Any, Final + +__all__ = ["ensure_running", "get_inherited_fds", "connect_to_new_process", "set_forkserver_preload"] + +MAXFDS_TO_SEND: Final = 256 +SIGNED_STRUCT: Final[Struct] + +class ForkServer: + def set_forkserver_preload(self, modules_names: list[str]) -> None: ... + def get_inherited_fds(self) -> list[int] | None: ... + def connect_to_new_process(self, fds: Sequence[int]) -> tuple[int, int]: ... + def ensure_running(self) -> None: ... + +def main( + listener_fd: int | None, + alive_r: FileDescriptorLike, + preload: Sequence[str], + main_path: str | None = None, + sys_path: Unused = None, +) -> None: ... +def read_signed(fd: int) -> Any: ... +def write_signed(fd: int, n: int) -> None: ... + +_forkserver: ForkServer +ensure_running = _forkserver.ensure_running +get_inherited_fds = _forkserver.get_inherited_fds +connect_to_new_process = _forkserver.connect_to_new_process +set_forkserver_preload = _forkserver.set_forkserver_preload diff --git a/mypy/typeshed/stdlib/multiprocessing/heap.pyi b/mypy/typeshed/stdlib/multiprocessing/heap.pyi new file mode 100644 index 000000000000..b5e2ced5e8ee --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/heap.pyi @@ -0,0 +1,36 @@ +import sys +from _typeshed import Incomplete +from collections.abc import Callable +from mmap import mmap +from typing import Protocol +from typing_extensions import TypeAlias + +__all__ = ["BufferWrapper"] + +class Arena: + size: int + buffer: mmap + if sys.platform == "win32": + name: str + def __init__(self, size: int) -> None: ... + else: + fd: int + def __init__(self, size: int, fd: int = -1) -> None: ... + +_Block: TypeAlias = tuple[Arena, int, int] + +if sys.platform != "win32": + class _SupportsDetach(Protocol): + def detach(self) -> int: ... + + def reduce_arena(a: Arena) -> tuple[Callable[[int, _SupportsDetach], Arena], tuple[int, Incomplete]]: ... + def rebuild_arena(size: int, dupfd: _SupportsDetach) -> Arena: ... + +class Heap: + def __init__(self, size: int = ...) -> None: ... + def free(self, block: _Block) -> None: ... + def malloc(self, size: int) -> _Block: ... + +class BufferWrapper: + def __init__(self, size: int) -> None: ... + def create_memoryview(self) -> memoryview: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/managers.pyi b/mypy/typeshed/stdlib/multiprocessing/managers.pyi new file mode 100644 index 000000000000..1669c5f09f97 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/managers.pyi @@ -0,0 +1,246 @@ +import queue +import sys +import threading +from _typeshed import Incomplete, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT +from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence +from types import TracebackType +from typing import Any, AnyStr, ClassVar, Generic, SupportsIndex, TypeVar, overload +from typing_extensions import Self, TypeAlias + +from .connection import Connection +from .context import BaseContext +from .shared_memory import _SLT, ShareableList as _ShareableList, SharedMemory as _SharedMemory +from .util import Finalize as _Finalize + +__all__ = ["BaseManager", "SyncManager", "BaseProxy", "Token", "SharedMemoryManager"] + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") + +class Namespace: + def __init__(self, **kwds: Any) -> None: ... + def __getattr__(self, name: str, /) -> Any: ... + def __setattr__(self, name: str, value: Any, /) -> None: ... + +_Namespace: TypeAlias = Namespace + +class Token: + typeid: str | bytes | None + address: tuple[str | bytes, int] + id: str | bytes | int | None + def __init__(self, typeid: bytes | str | None, address: tuple[str | bytes, int], id: str | bytes | int | None) -> None: ... + def __getstate__(self) -> tuple[str | bytes | None, tuple[str | bytes, int], str | bytes | int | None]: ... + def __setstate__(self, state: tuple[str | bytes | None, tuple[str | bytes, int], str | bytes | int | None]) -> None: ... + +class BaseProxy: + _address_to_local: dict[Any, Any] + _mutex: Any + def __init__( + self, + token: Any, + serializer: str, + manager: Any = None, + authkey: AnyStr | None = None, + exposed: Any = None, + incref: bool = True, + manager_owned: bool = False, + ) -> None: ... + def __deepcopy__(self, memo: Any | None) -> Any: ... + def _callmethod(self, methodname: str, args: tuple[Any, ...] = (), kwds: dict[Any, Any] = {}) -> None: ... + def _getvalue(self) -> Any: ... + def __reduce__(self) -> tuple[Any, tuple[Any, Any, str, dict[Any, Any]]]: ... + +class ValueProxy(BaseProxy, Generic[_T]): + def get(self) -> _T: ... + def set(self, value: _T) -> None: ... + value: _T + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +if sys.version_info >= (3, 13): + class _BaseDictProxy(BaseProxy, MutableMapping[_KT, _VT]): + __builtins__: ClassVar[dict[str, Any]] + def __len__(self) -> int: ... + def __getitem__(self, key: _KT, /) -> _VT: ... + def __setitem__(self, key: _KT, value: _VT, /) -> None: ... + def __delitem__(self, key: _KT, /) -> None: ... + def __iter__(self) -> Iterator[_KT]: ... + def copy(self) -> dict[_KT, _VT]: ... + @overload # type: ignore[override] + def get(self, key: _KT, /) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _VT, /) -> _VT: ... + @overload + def get(self, key: _KT, default: _T, /) -> _VT | _T: ... + @overload + def pop(self, key: _KT, /) -> _VT: ... + @overload + def pop(self, key: _KT, default: _VT, /) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T, /) -> _VT | _T: ... + def keys(self) -> list[_KT]: ... # type: ignore[override] + def items(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override] + def values(self) -> list[_VT]: ... # type: ignore[override] + + class DictProxy(_BaseDictProxy[_KT, _VT]): + def __class_getitem__(cls, args: Any, /) -> GenericAlias: ... + +else: + class DictProxy(BaseProxy, MutableMapping[_KT, _VT]): + __builtins__: ClassVar[dict[str, Any]] + def __len__(self) -> int: ... + def __getitem__(self, key: _KT, /) -> _VT: ... + def __setitem__(self, key: _KT, value: _VT, /) -> None: ... + def __delitem__(self, key: _KT, /) -> None: ... + def __iter__(self) -> Iterator[_KT]: ... + def copy(self) -> dict[_KT, _VT]: ... + @overload # type: ignore[override] + def get(self, key: _KT, /) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _VT, /) -> _VT: ... + @overload + def get(self, key: _KT, default: _T, /) -> _VT | _T: ... + @overload + def pop(self, key: _KT, /) -> _VT: ... + @overload + def pop(self, key: _KT, default: _VT, /) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T, /) -> _VT | _T: ... + def keys(self) -> list[_KT]: ... # type: ignore[override] + def items(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override] + def values(self) -> list[_VT]: ... # type: ignore[override] + +class BaseListProxy(BaseProxy, MutableSequence[_T]): + __builtins__: ClassVar[dict[str, Any]] + def __len__(self) -> int: ... + def __add__(self, x: list[_T], /) -> list[_T]: ... + def __delitem__(self, i: SupportsIndex | slice, /) -> None: ... + @overload + def __getitem__(self, i: SupportsIndex, /) -> _T: ... + @overload + def __getitem__(self, s: slice, /) -> list[_T]: ... + @overload + def __setitem__(self, i: SupportsIndex, o: _T, /) -> None: ... + @overload + def __setitem__(self, s: slice, o: Iterable[_T], /) -> None: ... + def __mul__(self, n: SupportsIndex, /) -> list[_T]: ... + def __rmul__(self, n: SupportsIndex, /) -> list[_T]: ... + def __reversed__(self) -> Iterator[_T]: ... + def append(self, object: _T, /) -> None: ... + def extend(self, iterable: Iterable[_T], /) -> None: ... + def pop(self, index: SupportsIndex = ..., /) -> _T: ... + def index(self, value: _T, start: SupportsIndex = ..., stop: SupportsIndex = ..., /) -> int: ... + def count(self, value: _T, /) -> int: ... + def insert(self, index: SupportsIndex, object: _T, /) -> None: ... + def remove(self, value: _T, /) -> None: ... + # Use BaseListProxy[SupportsRichComparisonT] for the first overload rather than [SupportsRichComparison] + # to work around invariance + @overload + def sort(self: BaseListProxy[SupportsRichComparisonT], *, key: None = None, reverse: bool = ...) -> None: ... + @overload + def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = ...) -> None: ... + +class ListProxy(BaseListProxy[_T]): + def __iadd__(self, value: Iterable[_T], /) -> Self: ... # type: ignore[override] + def __imul__(self, value: SupportsIndex, /) -> Self: ... # type: ignore[override] + if sys.version_info >= (3, 13): + def __class_getitem__(cls, args: Any, /) -> Any: ... + +# Returned by BaseManager.get_server() +class Server: + address: Any + def __init__( + self, registry: dict[str, tuple[Callable[..., Any], Any, Any, Any]], address: Any, authkey: bytes, serializer: str + ) -> None: ... + def serve_forever(self) -> None: ... + def accept_connection( + self, c: Connection[tuple[str, str | None], tuple[str, str, Iterable[Incomplete], Mapping[str, Incomplete]]], name: str + ) -> None: ... + +class BaseManager: + if sys.version_info >= (3, 11): + def __init__( + self, + address: Any | None = None, + authkey: bytes | None = None, + serializer: str = "pickle", + ctx: BaseContext | None = None, + *, + shutdown_timeout: float = 1.0, + ) -> None: ... + else: + def __init__( + self, + address: Any | None = None, + authkey: bytes | None = None, + serializer: str = "pickle", + ctx: BaseContext | None = None, + ) -> None: ... + + def get_server(self) -> Server: ... + def connect(self) -> None: ... + def start(self, initializer: Callable[..., object] | None = None, initargs: Iterable[Any] = ()) -> None: ... + shutdown: _Finalize # only available after start() was called + def join(self, timeout: float | None = None) -> None: ... # undocumented + @property + def address(self) -> Any: ... + @classmethod + def register( + cls, + typeid: str, + callable: Callable[..., object] | None = None, + proxytype: Any = None, + exposed: Sequence[str] | None = None, + method_to_typeid: Mapping[str, str] | None = None, + create_method: bool = True, + ) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + +class SyncManager(BaseManager): + def BoundedSemaphore(self, value: Any = ...) -> threading.BoundedSemaphore: ... + def Condition(self, lock: Any = ...) -> threading.Condition: ... + def Event(self) -> threading.Event: ... + def Lock(self) -> threading.Lock: ... + def Namespace(self) -> _Namespace: ... + def Queue(self, maxsize: int = ...) -> queue.Queue[Any]: ... + def RLock(self) -> threading.RLock: ... + def Semaphore(self, value: Any = ...) -> threading.Semaphore: ... + def Array(self, typecode: Any, sequence: Sequence[_T]) -> Sequence[_T]: ... + def Value(self, typecode: Any, value: _T) -> ValueProxy[_T]: ... + # Overloads are copied from builtins.dict.__init__ + @overload + def dict(self) -> DictProxy[Any, Any]: ... + @overload + def dict(self, **kwargs: _VT) -> DictProxy[str, _VT]: ... + @overload + def dict(self, map: SupportsKeysAndGetItem[_KT, _VT], /) -> DictProxy[_KT, _VT]: ... + @overload + def dict(self, map: SupportsKeysAndGetItem[str, _VT], /, **kwargs: _VT) -> DictProxy[str, _VT]: ... + @overload + def dict(self, iterable: Iterable[tuple[_KT, _VT]], /) -> DictProxy[_KT, _VT]: ... + @overload + def dict(self, iterable: Iterable[tuple[str, _VT]], /, **kwargs: _VT) -> DictProxy[str, _VT]: ... + @overload + def dict(self, iterable: Iterable[list[str]], /) -> DictProxy[str, str]: ... + @overload + def dict(self, iterable: Iterable[list[bytes]], /) -> DictProxy[bytes, bytes]: ... + @overload + def list(self, sequence: Sequence[_T], /) -> ListProxy[_T]: ... + @overload + def list(self) -> ListProxy[Any]: ... + +class RemoteError(Exception): ... +class SharedMemoryServer(Server): ... + +class SharedMemoryManager(BaseManager): + def get_server(self) -> SharedMemoryServer: ... + def SharedMemory(self, size: int) -> _SharedMemory: ... + def ShareableList(self, sequence: Iterable[_SLT] | None) -> _ShareableList[_SLT]: ... + def __del__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/pool.pyi b/mypy/typeshed/stdlib/multiprocessing/pool.pyi new file mode 100644 index 000000000000..61d6d0781213 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/pool.pyi @@ -0,0 +1,103 @@ +import sys +from collections.abc import Callable, Iterable, Mapping +from types import TracebackType +from typing import Any, Final, Generic, TypeVar +from typing_extensions import Self + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["Pool", "ThreadPool"] + +_S = TypeVar("_S") +_T = TypeVar("_T") + +class ApplyResult(Generic[_T]): + def __init__( + self, pool: Pool, callback: Callable[[_T], object] | None, error_callback: Callable[[BaseException], object] | None + ) -> None: ... + def get(self, timeout: float | None = None) -> _T: ... + def wait(self, timeout: float | None = None) -> None: ... + def ready(self) -> bool: ... + def successful(self) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +# alias created during issue #17805 +AsyncResult = ApplyResult + +class MapResult(ApplyResult[list[_T]]): + def __init__( + self, + pool: Pool, + chunksize: int, + length: int, + callback: Callable[[list[_T]], object] | None, + error_callback: Callable[[BaseException], object] | None, + ) -> None: ... + +class IMapIterator(Generic[_T]): + def __init__(self, pool: Pool) -> None: ... + def __iter__(self) -> Self: ... + def next(self, timeout: float | None = None) -> _T: ... + def __next__(self, timeout: float | None = None) -> _T: ... + +class IMapUnorderedIterator(IMapIterator[_T]): ... + +class Pool: + def __init__( + self, + processes: int | None = None, + initializer: Callable[..., object] | None = None, + initargs: Iterable[Any] = (), + maxtasksperchild: int | None = None, + context: Any | None = None, + ) -> None: ... + def apply(self, func: Callable[..., _T], args: Iterable[Any] = (), kwds: Mapping[str, Any] = {}) -> _T: ... + def apply_async( + self, + func: Callable[..., _T], + args: Iterable[Any] = (), + kwds: Mapping[str, Any] = {}, + callback: Callable[[_T], object] | None = None, + error_callback: Callable[[BaseException], object] | None = None, + ) -> AsyncResult[_T]: ... + def map(self, func: Callable[[_S], _T], iterable: Iterable[_S], chunksize: int | None = None) -> list[_T]: ... + def map_async( + self, + func: Callable[[_S], _T], + iterable: Iterable[_S], + chunksize: int | None = None, + callback: Callable[[list[_T]], object] | None = None, + error_callback: Callable[[BaseException], object] | None = None, + ) -> MapResult[_T]: ... + def imap(self, func: Callable[[_S], _T], iterable: Iterable[_S], chunksize: int | None = 1) -> IMapIterator[_T]: ... + def imap_unordered(self, func: Callable[[_S], _T], iterable: Iterable[_S], chunksize: int | None = 1) -> IMapIterator[_T]: ... + def starmap(self, func: Callable[..., _T], iterable: Iterable[Iterable[Any]], chunksize: int | None = None) -> list[_T]: ... + def starmap_async( + self, + func: Callable[..., _T], + iterable: Iterable[Iterable[Any]], + chunksize: int | None = None, + callback: Callable[[list[_T]], object] | None = None, + error_callback: Callable[[BaseException], object] | None = None, + ) -> AsyncResult[list[_T]]: ... + def close(self) -> None: ... + def terminate(self) -> None: ... + def join(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def __del__(self) -> None: ... + +class ThreadPool(Pool): + def __init__( + self, processes: int | None = None, initializer: Callable[..., object] | None = None, initargs: Iterable[Any] = () + ) -> None: ... + +# undocumented +INIT: Final = "INIT" +RUN: Final = "RUN" +CLOSE: Final = "CLOSE" +TERMINATE: Final = "TERMINATE" diff --git a/mypy/typeshed/stdlib/multiprocessing/popen_fork.pyi b/mypy/typeshed/stdlib/multiprocessing/popen_fork.pyi new file mode 100644 index 000000000000..4fcbfd99a8d0 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/popen_fork.pyi @@ -0,0 +1,23 @@ +import sys +from typing import ClassVar + +from .process import BaseProcess +from .util import Finalize + +if sys.platform != "win32": + __all__ = ["Popen"] + + class Popen: + finalizer: Finalize | None + method: ClassVar[str] + pid: int + returncode: int | None + sentinel: int # doesn't exist if os.fork in _launch returns 0 + + def __init__(self, process_obj: BaseProcess) -> None: ... + def duplicate_for_child(self, fd: int) -> int: ... + def poll(self, flag: int = 1) -> int | None: ... + def wait(self, timeout: float | None = None) -> int | None: ... + def terminate(self) -> None: ... + def kill(self) -> None: ... + def close(self) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/popen_forkserver.pyi b/mypy/typeshed/stdlib/multiprocessing/popen_forkserver.pyi new file mode 100644 index 000000000000..f7d53bbb3e41 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/popen_forkserver.pyi @@ -0,0 +1,16 @@ +import sys +from typing import ClassVar + +from . import popen_fork +from .util import Finalize + +if sys.platform != "win32": + __all__ = ["Popen"] + + class _DupFd: + def __init__(self, ind: int) -> None: ... + def detach(self) -> int: ... + + class Popen(popen_fork.Popen): + DupFd: ClassVar[type[_DupFd]] + finalizer: Finalize diff --git a/mypy/typeshed/stdlib/multiprocessing/popen_spawn_posix.pyi b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_posix.pyi new file mode 100644 index 000000000000..7e81d39600ad --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_posix.pyi @@ -0,0 +1,20 @@ +import sys +from typing import ClassVar + +from . import popen_fork +from .util import Finalize + +if sys.platform != "win32": + __all__ = ["Popen"] + + class _DupFd: + fd: int + + def __init__(self, fd: int) -> None: ... + def detach(self) -> int: ... + + class Popen(popen_fork.Popen): + DupFd: ClassVar[type[_DupFd]] + finalizer: Finalize + pid: int # may not exist if _launch raises in second try / except + sentinel: int # may not exist if _launch raises in second try / except diff --git a/mypy/typeshed/stdlib/multiprocessing/popen_spawn_win32.pyi b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_win32.pyi new file mode 100644 index 000000000000..481b9eec5a37 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_win32.pyi @@ -0,0 +1,30 @@ +import sys +from multiprocessing.process import BaseProcess +from typing import ClassVar, Final + +from .util import Finalize + +if sys.platform == "win32": + __all__ = ["Popen"] + + TERMINATE: Final[int] + WINEXE: Final[bool] + WINSERVICE: Final[bool] + WINENV: Final[bool] + + class Popen: + finalizer: Finalize + method: ClassVar[str] + pid: int + returncode: int | None + sentinel: int + + def __init__(self, process_obj: BaseProcess) -> None: ... + def duplicate_for_child(self, handle: int) -> int: ... + def wait(self, timeout: float | None = None) -> int | None: ... + def poll(self) -> int | None: ... + def terminate(self) -> None: ... + + kill = terminate + + def close(self) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/process.pyi b/mypy/typeshed/stdlib/multiprocessing/process.pyi new file mode 100644 index 000000000000..4d129b27b0e8 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/process.pyi @@ -0,0 +1,39 @@ +from collections.abc import Callable, Iterable, Mapping +from typing import Any + +__all__ = ["BaseProcess", "current_process", "active_children", "parent_process"] + +class BaseProcess: + name: str + daemon: bool + authkey: bytes + _identity: tuple[int, ...] # undocumented + def __init__( + self, + group: None = None, + target: Callable[..., object] | None = None, + name: str | None = None, + args: Iterable[Any] = (), + kwargs: Mapping[str, Any] = {}, + *, + daemon: bool | None = None, + ) -> None: ... + def run(self) -> None: ... + def start(self) -> None: ... + def terminate(self) -> None: ... + def kill(self) -> None: ... + def close(self) -> None: ... + def join(self, timeout: float | None = None) -> None: ... + def is_alive(self) -> bool: ... + @property + def exitcode(self) -> int | None: ... + @property + def ident(self) -> int | None: ... + @property + def pid(self) -> int | None: ... + @property + def sentinel(self) -> int: ... + +def current_process() -> BaseProcess: ... +def active_children() -> list[BaseProcess]: ... +def parent_process() -> BaseProcess | None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/queues.pyi b/mypy/typeshed/stdlib/multiprocessing/queues.pyi new file mode 100644 index 000000000000..581a46ea0bc8 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/queues.pyi @@ -0,0 +1,41 @@ +import sys +from typing import Any, Generic, TypeVar + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["Queue", "SimpleQueue", "JoinableQueue"] + +_T = TypeVar("_T") + +class Queue(Generic[_T]): + # FIXME: `ctx` is a circular dependency and it's not actually optional. + # It's marked as such to be able to use the generic Queue in __init__.pyi. + def __init__(self, maxsize: int = 0, *, ctx: Any = ...) -> None: ... + def put(self, obj: _T, block: bool = True, timeout: float | None = None) -> None: ... + def get(self, block: bool = True, timeout: float | None = None) -> _T: ... + def qsize(self) -> int: ... + def empty(self) -> bool: ... + def full(self) -> bool: ... + def get_nowait(self) -> _T: ... + def put_nowait(self, obj: _T) -> None: ... + def close(self) -> None: ... + def join_thread(self) -> None: ... + def cancel_join_thread(self) -> None: ... + if sys.version_info >= (3, 12): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class JoinableQueue(Queue[_T]): + def task_done(self) -> None: ... + def join(self) -> None: ... + +class SimpleQueue(Generic[_T]): + def __init__(self, *, ctx: Any = ...) -> None: ... + if sys.version_info >= (3, 9): + def close(self) -> None: ... + + def empty(self) -> bool: ... + def get(self) -> _T: ... + def put(self, obj: _T) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi new file mode 100644 index 000000000000..473e90936d71 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi @@ -0,0 +1,87 @@ +import pickle +import sys +from _typeshed import HasFileno, SupportsWrite, Unused +from abc import ABCMeta +from builtins import type as Type # alias to avoid name clash +from collections.abc import Callable +from copyreg import _DispatchTableType +from multiprocessing import connection +from pickle import _ReducedType +from socket import socket +from typing import Any, Final + +if sys.platform == "win32": + __all__ = ["send_handle", "recv_handle", "ForkingPickler", "register", "dump", "DupHandle", "duplicate", "steal_handle"] +else: + __all__ = ["send_handle", "recv_handle", "ForkingPickler", "register", "dump", "DupFd", "sendfds", "recvfds"] + +HAVE_SEND_HANDLE: Final[bool] + +class ForkingPickler(pickle.Pickler): + dispatch_table: _DispatchTableType + def __init__(self, file: SupportsWrite[bytes], protocol: int | None = ...) -> None: ... + @classmethod + def register(cls, type: Type, reduce: Callable[[Any], _ReducedType]) -> None: ... + @classmethod + def dumps(cls, obj: Any, protocol: int | None = None) -> memoryview: ... + loads = pickle.loads + +register = ForkingPickler.register + +def dump(obj: Any, file: SupportsWrite[bytes], protocol: int | None = None) -> None: ... + +if sys.platform == "win32": + def duplicate( + handle: int, target_process: int | None = None, inheritable: bool = False, *, source_process: int | None = None + ) -> int: ... + def steal_handle(source_pid: int, handle: int) -> int: ... + def send_handle(conn: connection.PipeConnection[DupHandle, Any], handle: int, destination_pid: int) -> None: ... + def recv_handle(conn: connection.PipeConnection[Any, DupHandle]) -> int: ... + + class DupHandle: + def __init__(self, handle: int, access: int, pid: int | None = None) -> None: ... + def detach(self) -> int: ... + +else: + ACKNOWLEDGE: Final[bool] + + def recvfds(sock: socket, size: int) -> list[int]: ... + def send_handle(conn: HasFileno, handle: int, destination_pid: Unused) -> None: ... + def recv_handle(conn: HasFileno) -> int: ... + def sendfds(sock: socket, fds: list[int]) -> None: ... + def DupFd(fd: int) -> Any: ... # Return type is really hard to get right + +# These aliases are to work around pyright complaints. +# Pyright doesn't like it when a class object is defined as an alias +# of a global object with the same name. +_ForkingPickler = ForkingPickler +_register = register +_dump = dump +_send_handle = send_handle +_recv_handle = recv_handle + +if sys.platform == "win32": + _steal_handle = steal_handle + _duplicate = duplicate + _DupHandle = DupHandle +else: + _sendfds = sendfds + _recvfds = recvfds + _DupFd = DupFd + +class AbstractReducer(metaclass=ABCMeta): + ForkingPickler = _ForkingPickler + register = _register + dump = _dump + send_handle = _send_handle + recv_handle = _recv_handle + if sys.platform == "win32": + steal_handle = _steal_handle + duplicate = _duplicate + DupHandle = _DupHandle + else: + sendfds = _sendfds + recvfds = _recvfds + DupFd = _DupFd + + def __init__(self, *args: Unused) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/resource_sharer.pyi b/mypy/typeshed/stdlib/multiprocessing/resource_sharer.pyi new file mode 100644 index 000000000000..5fee7cf31e17 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/resource_sharer.pyi @@ -0,0 +1,20 @@ +import sys +from socket import socket + +__all__ = ["stop"] + +if sys.platform == "win32": + __all__ += ["DupSocket"] + + class DupSocket: + def __init__(self, sock: socket) -> None: ... + def detach(self) -> socket: ... + +else: + __all__ += ["DupFd"] + + class DupFd: + def __init__(self, fd: int) -> None: ... + def detach(self) -> int: ... + +def stop(timeout: float | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/resource_tracker.pyi b/mypy/typeshed/stdlib/multiprocessing/resource_tracker.pyi new file mode 100644 index 000000000000..61da7fdf1ceb --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/resource_tracker.pyi @@ -0,0 +1,18 @@ +from _typeshed import FileDescriptorOrPath +from collections.abc import Sized + +__all__ = ["ensure_running", "register", "unregister"] + +class ResourceTracker: + def getfd(self) -> int | None: ... + def ensure_running(self) -> None: ... + def register(self, name: Sized, rtype: str) -> None: ... + def unregister(self, name: Sized, rtype: str) -> None: ... + +_resource_tracker: ResourceTracker +ensure_running = _resource_tracker.ensure_running +register = _resource_tracker.register +unregister = _resource_tracker.unregister +getfd = _resource_tracker.getfd + +def main(fd: FileDescriptorOrPath) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi new file mode 100644 index 000000000000..b63cedf85867 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi @@ -0,0 +1,44 @@ +import sys +from collections.abc import Iterable +from typing import Any, Generic, TypeVar, overload +from typing_extensions import Self + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["SharedMemory", "ShareableList"] + +_SLT = TypeVar("_SLT", int, float, bool, str, bytes, None) + +class SharedMemory: + if sys.version_info >= (3, 13): + def __init__(self, name: str | None = None, create: bool = False, size: int = 0, *, track: bool = True) -> None: ... + else: + def __init__(self, name: str | None = None, create: bool = False, size: int = 0) -> None: ... + + @property + def buf(self) -> memoryview: ... + @property + def name(self) -> str: ... + @property + def size(self) -> int: ... + def close(self) -> None: ... + def unlink(self) -> None: ... + def __del__(self) -> None: ... + +class ShareableList(Generic[_SLT]): + shm: SharedMemory + @overload + def __init__(self, sequence: None = None, *, name: str | None = None) -> None: ... + @overload + def __init__(self, sequence: Iterable[_SLT], *, name: str | None = None) -> None: ... + def __getitem__(self, position: int) -> _SLT: ... + def __setitem__(self, position: int, value: _SLT) -> None: ... + def __reduce__(self) -> tuple[Self, tuple[_SLT, ...]]: ... + def __len__(self) -> int: ... + @property + def format(self) -> str: ... + def count(self, value: _SLT) -> int: ... + def index(self, value: _SLT) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi new file mode 100644 index 000000000000..5283445d8545 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi @@ -0,0 +1,129 @@ +import ctypes +from _ctypes import _CData +from collections.abc import Callable, Iterable, Sequence +from ctypes import _SimpleCData, c_char +from multiprocessing.context import BaseContext +from multiprocessing.synchronize import _LockLike +from types import TracebackType +from typing import Any, Generic, Literal, Protocol, TypeVar, overload + +__all__ = ["RawValue", "RawArray", "Value", "Array", "copy", "synchronized"] + +_T = TypeVar("_T") +_CT = TypeVar("_CT", bound=_CData) + +@overload +def RawValue(typecode_or_type: type[_CT], *args: Any) -> _CT: ... +@overload +def RawValue(typecode_or_type: str, *args: Any) -> Any: ... +@overload +def RawArray(typecode_or_type: type[_CT], size_or_initializer: int | Sequence[Any]) -> ctypes.Array[_CT]: ... +@overload +def RawArray(typecode_or_type: str, size_or_initializer: int | Sequence[Any]) -> Any: ... +@overload +def Value(typecode_or_type: type[_CT], *args: Any, lock: Literal[False], ctx: BaseContext | None = None) -> _CT: ... +@overload +def Value( + typecode_or_type: type[_CT], *args: Any, lock: Literal[True] | _LockLike = True, ctx: BaseContext | None = None +) -> SynchronizedBase[_CT]: ... +@overload +def Value( + typecode_or_type: str, *args: Any, lock: Literal[True] | _LockLike = True, ctx: BaseContext | None = None +) -> SynchronizedBase[Any]: ... +@overload +def Value( + typecode_or_type: str | type[_CData], *args: Any, lock: bool | _LockLike = True, ctx: BaseContext | None = None +) -> Any: ... +@overload +def Array( + typecode_or_type: type[_CT], size_or_initializer: int | Sequence[Any], *, lock: Literal[False], ctx: BaseContext | None = None +) -> _CT: ... +@overload +def Array( + typecode_or_type: type[c_char], + size_or_initializer: int | Sequence[Any], + *, + lock: Literal[True] | _LockLike = True, + ctx: BaseContext | None = None, +) -> SynchronizedString: ... +@overload +def Array( + typecode_or_type: type[_SimpleCData[_T]], + size_or_initializer: int | Sequence[Any], + *, + lock: Literal[True] | _LockLike = True, + ctx: BaseContext | None = None, +) -> SynchronizedArray[_T]: ... +@overload +def Array( + typecode_or_type: str, + size_or_initializer: int | Sequence[Any], + *, + lock: Literal[True] | _LockLike = True, + ctx: BaseContext | None = None, +) -> SynchronizedArray[Any]: ... +@overload +def Array( + typecode_or_type: str | type[_CData], + size_or_initializer: int | Sequence[Any], + *, + lock: bool | _LockLike = True, + ctx: BaseContext | None = None, +) -> Any: ... +def copy(obj: _CT) -> _CT: ... +@overload +def synchronized(obj: _SimpleCData[_T], lock: _LockLike | None = None, ctx: Any | None = None) -> Synchronized[_T]: ... +@overload +def synchronized(obj: ctypes.Array[c_char], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedString: ... +@overload +def synchronized( + obj: ctypes.Array[_SimpleCData[_T]], lock: _LockLike | None = None, ctx: Any | None = None +) -> SynchronizedArray[_T]: ... +@overload +def synchronized(obj: _CT, lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedBase[_CT]: ... + +class _AcquireFunc(Protocol): + def __call__(self, block: bool = ..., timeout: float | None = ..., /) -> bool: ... + +class SynchronizedBase(Generic[_CT]): + acquire: _AcquireFunc + release: Callable[[], None] + def __init__(self, obj: Any, lock: _LockLike | None = None, ctx: Any | None = None) -> None: ... + def __reduce__(self) -> tuple[Callable[[Any, _LockLike], SynchronizedBase[Any]], tuple[Any, _LockLike]]: ... + def get_obj(self) -> _CT: ... + def get_lock(self) -> _LockLike: ... + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, / + ) -> None: ... + +class Synchronized(SynchronizedBase[_SimpleCData[_T]], Generic[_T]): + value: _T + +class SynchronizedArray(SynchronizedBase[ctypes.Array[_SimpleCData[_T]]], Generic[_T]): + def __len__(self) -> int: ... + @overload + def __getitem__(self, i: slice) -> list[_T]: ... + @overload + def __getitem__(self, i: int) -> _T: ... + @overload + def __setitem__(self, i: slice, value: Iterable[_T]) -> None: ... + @overload + def __setitem__(self, i: int, value: _T) -> None: ... + def __getslice__(self, start: int, stop: int) -> list[_T]: ... + def __setslice__(self, start: int, stop: int, values: Iterable[_T]) -> None: ... + +class SynchronizedString(SynchronizedArray[bytes]): + @overload # type: ignore[override] + def __getitem__(self, i: slice) -> bytes: ... + @overload + def __getitem__(self, i: int) -> bytes: ... + @overload # type: ignore[override] + def __setitem__(self, i: slice, value: bytes) -> None: ... + @overload + def __setitem__(self, i: int, value: bytes) -> None: ... + def __getslice__(self, start: int, stop: int) -> bytes: ... # type: ignore[override] + def __setslice__(self, start: int, stop: int, values: bytes) -> None: ... # type: ignore[override] + + value: bytes + raw: bytes diff --git a/mypy/typeshed/stdlib/multiprocessing/spawn.pyi b/mypy/typeshed/stdlib/multiprocessing/spawn.pyi new file mode 100644 index 000000000000..4a9753222897 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/spawn.pyi @@ -0,0 +1,32 @@ +from collections.abc import Mapping, Sequence +from types import ModuleType +from typing import Any, Final + +__all__ = [ + "_main", + "freeze_support", + "set_executable", + "get_executable", + "get_preparation_data", + "get_command_line", + "import_main_path", +] + +WINEXE: Final[bool] +WINSERVICE: Final[bool] + +def set_executable(exe: str) -> None: ... +def get_executable() -> str: ... +def is_forking(argv: Sequence[str]) -> bool: ... +def freeze_support() -> None: ... +def get_command_line(**kwds: Any) -> list[str]: ... +def spawn_main(pipe_handle: int, parent_pid: int | None = None, tracker_fd: int | None = None) -> None: ... + +# undocumented +def _main(fd: int, parent_sentinel: int) -> int: ... +def get_preparation_data(name: str) -> dict[str, Any]: ... + +old_main_modules: list[ModuleType] + +def prepare(data: Mapping[str, Any]) -> None: ... +def import_main_path(main_path: str) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/synchronize.pyi b/mypy/typeshed/stdlib/multiprocessing/synchronize.pyi new file mode 100644 index 000000000000..e3cbfbc0ec82 --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/synchronize.pyi @@ -0,0 +1,59 @@ +import threading +from collections.abc import Callable +from multiprocessing.context import BaseContext +from types import TracebackType +from typing_extensions import TypeAlias + +__all__ = ["Lock", "RLock", "Semaphore", "BoundedSemaphore", "Condition", "Event"] + +_LockLike: TypeAlias = Lock | RLock + +class Barrier(threading.Barrier): + def __init__( + self, parties: int, action: Callable[[], object] | None = None, timeout: float | None = None, *, ctx: BaseContext + ) -> None: ... + +class Condition: + def __init__(self, lock: _LockLike | None = None, *, ctx: BaseContext) -> None: ... + def notify(self, n: int = 1) -> None: ... + def notify_all(self) -> None: ... + def wait(self, timeout: float | None = None) -> bool: ... + def wait_for(self, predicate: Callable[[], bool], timeout: float | None = None) -> bool: ... + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, / + ) -> None: ... + # These methods are copied from the lock passed to the constructor, or an + # instance of ctx.RLock() if lock was None. + def acquire(self, block: bool = True, timeout: float | None = None) -> bool: ... + def release(self) -> None: ... + +class Event: + def __init__(self, *, ctx: BaseContext) -> None: ... + def is_set(self) -> bool: ... + def set(self) -> None: ... + def clear(self) -> None: ... + def wait(self, timeout: float | None = None) -> bool: ... + +# Not part of public API +class SemLock: + def __init__(self, kind: int, value: int, maxvalue: int, *, ctx: BaseContext | None) -> None: ... + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, / + ) -> None: ... + # These methods are copied from the wrapped _multiprocessing.SemLock object + def acquire(self, block: bool = True, timeout: float | None = None) -> bool: ... + def release(self) -> None: ... + +class Lock(SemLock): + def __init__(self, *, ctx: BaseContext) -> None: ... + +class RLock(SemLock): + def __init__(self, *, ctx: BaseContext) -> None: ... + +class Semaphore(SemLock): + def __init__(self, value: int = 1, *, ctx: BaseContext) -> None: ... + +class BoundedSemaphore(Semaphore): + def __init__(self, value: int = 1, *, ctx: BaseContext) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/util.pyi b/mypy/typeshed/stdlib/multiprocessing/util.pyi new file mode 100644 index 000000000000..d5b6384afd5e --- /dev/null +++ b/mypy/typeshed/stdlib/multiprocessing/util.pyi @@ -0,0 +1,98 @@ +import threading +from _typeshed import ConvertibleToInt, Incomplete, Unused +from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence +from logging import Logger, _Level as _LoggingLevel +from typing import Any, Final, Generic, TypeVar, overload + +__all__ = [ + "sub_debug", + "debug", + "info", + "sub_warning", + "get_logger", + "log_to_stderr", + "get_temp_dir", + "register_after_fork", + "is_exiting", + "Finalize", + "ForkAwareThreadLock", + "ForkAwareLocal", + "close_all_fds_except", + "SUBDEBUG", + "SUBWARNING", +] + +_T = TypeVar("_T") +_R_co = TypeVar("_R_co", default=Any, covariant=True) + +NOTSET: Final[int] +SUBDEBUG: Final[int] +DEBUG: Final[int] +INFO: Final[int] +SUBWARNING: Final[int] + +LOGGER_NAME: Final[str] +DEFAULT_LOGGING_FORMAT: Final[str] + +def sub_debug(msg: object, *args: object) -> None: ... +def debug(msg: object, *args: object) -> None: ... +def info(msg: object, *args: object) -> None: ... +def sub_warning(msg: object, *args: object) -> None: ... +def get_logger() -> Logger: ... +def log_to_stderr(level: _LoggingLevel | None = None) -> Logger: ... +def is_abstract_socket_namespace(address: str | bytes | None) -> bool: ... + +abstract_sockets_supported: bool + +def get_temp_dir() -> str: ... +def register_after_fork(obj: _T, func: Callable[[_T], object]) -> None: ... + +class Finalize(Generic[_R_co]): + # "args" and "kwargs" are passed as arguments to "callback". + @overload + def __init__( + self, + obj: None, + callback: Callable[..., _R_co], + *, + args: Sequence[Any] = (), + kwargs: Mapping[str, Any] | None = None, + exitpriority: int, + ) -> None: ... + @overload + def __init__( + self, obj: None, callback: Callable[..., _R_co], args: Sequence[Any], kwargs: Mapping[str, Any] | None, exitpriority: int + ) -> None: ... + @overload + def __init__( + self, + obj: Any, + callback: Callable[..., _R_co], + args: Sequence[Any] = (), + kwargs: Mapping[str, Any] | None = None, + exitpriority: int | None = None, + ) -> None: ... + def __call__( + self, + wr: Unused = None, + _finalizer_registry: MutableMapping[Incomplete, Incomplete] = {}, + sub_debug: Callable[..., object] = ..., + getpid: Callable[[], int] = ..., + ) -> _R_co: ... + def cancel(self) -> None: ... + def still_active(self) -> bool: ... + +def is_exiting() -> bool: ... + +class ForkAwareThreadLock: + acquire: Callable[[bool, float], bool] + release: Callable[[], None] + def __enter__(self) -> bool: ... + def __exit__(self, *args: Unused) -> None: ... + +class ForkAwareLocal(threading.local): ... + +MAXFD: Final[int] + +def close_all_fds_except(fds: Iterable[int]) -> None: ... +def spawnv_passfds(path: bytes, args: Sequence[ConvertibleToInt], passfds: Sequence[int]) -> int: ... diff --git a/mypy/typeshed/stdlib/netrc.pyi b/mypy/typeshed/stdlib/netrc.pyi new file mode 100644 index 000000000000..480f55a46d64 --- /dev/null +++ b/mypy/typeshed/stdlib/netrc.pyi @@ -0,0 +1,23 @@ +import sys +from _typeshed import StrOrBytesPath +from typing_extensions import TypeAlias + +__all__ = ["netrc", "NetrcParseError"] + +class NetrcParseError(Exception): + filename: str | None + lineno: int | None + msg: str + def __init__(self, msg: str, filename: StrOrBytesPath | None = None, lineno: int | None = None) -> None: ... + +# (login, account, password) tuple +if sys.version_info >= (3, 11): + _NetrcTuple: TypeAlias = tuple[str, str, str] +else: + _NetrcTuple: TypeAlias = tuple[str, str | None, str | None] + +class netrc: + hosts: dict[str, _NetrcTuple] + macros: dict[str, list[str]] + def __init__(self, file: StrOrBytesPath | None = None) -> None: ... + def authenticators(self, host: str) -> _NetrcTuple | None: ... diff --git a/mypy/typeshed/stdlib/nis.pyi b/mypy/typeshed/stdlib/nis.pyi new file mode 100644 index 000000000000..10eef2336a83 --- /dev/null +++ b/mypy/typeshed/stdlib/nis.pyi @@ -0,0 +1,9 @@ +import sys + +if sys.platform != "win32": + def cat(map: str, domain: str = ...) -> dict[str, str]: ... + def get_default_domain() -> str: ... + def maps(domain: str = ...) -> list[str]: ... + def match(key: str, map: str, domain: str = ...) -> str: ... + + class error(Exception): ... diff --git a/mypy/typeshed/stdlib/nntplib.pyi b/mypy/typeshed/stdlib/nntplib.pyi new file mode 100644 index 000000000000..85dfbff1cb50 --- /dev/null +++ b/mypy/typeshed/stdlib/nntplib.pyi @@ -0,0 +1,125 @@ +import datetime +import socket +import ssl +import sys +from _typeshed import Unused +from builtins import list as _list # conflicts with a method named "list" +from collections.abc import Iterable +from typing import IO, Any, Final, NamedTuple +from typing_extensions import Self, TypeAlias + +__all__ = [ + "NNTP", + "NNTPError", + "NNTPReplyError", + "NNTPTemporaryError", + "NNTPPermanentError", + "NNTPProtocolError", + "NNTPDataError", + "decode_header", + "NNTP_SSL", +] + +_File: TypeAlias = IO[bytes] | bytes | str | None + +class NNTPError(Exception): + response: str + +class NNTPReplyError(NNTPError): ... +class NNTPTemporaryError(NNTPError): ... +class NNTPPermanentError(NNTPError): ... +class NNTPProtocolError(NNTPError): ... +class NNTPDataError(NNTPError): ... + +NNTP_PORT: Final = 119 +NNTP_SSL_PORT: Final = 563 + +class GroupInfo(NamedTuple): + group: str + last: str + first: str + flag: str + +class ArticleInfo(NamedTuple): + number: int + message_id: str + lines: list[bytes] + +def decode_header(header_str: str) -> str: ... + +class NNTP: + encoding: str + errors: str + + host: str + port: int + sock: socket.socket + file: IO[bytes] + debugging: int + welcome: str + readermode_afterauth: bool + tls_on: bool + authenticated: bool + nntp_implementation: str + nntp_version: int + def __init__( + self, + host: str, + port: int = 119, + user: str | None = None, + password: str | None = None, + readermode: bool | None = None, + usenetrc: bool = False, + timeout: float = ..., + ) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def getwelcome(self) -> str: ... + def getcapabilities(self) -> dict[str, _list[str]]: ... + def set_debuglevel(self, level: int) -> None: ... + def debug(self, level: int) -> None: ... + def capabilities(self) -> tuple[str, dict[str, _list[str]]]: ... + def newgroups(self, date: datetime.date | datetime.datetime, *, file: _File = None) -> tuple[str, _list[str]]: ... + def newnews(self, group: str, date: datetime.date | datetime.datetime, *, file: _File = None) -> tuple[str, _list[str]]: ... + def list(self, group_pattern: str | None = None, *, file: _File = None) -> tuple[str, _list[str]]: ... + def description(self, group: str) -> str: ... + def descriptions(self, group_pattern: str) -> tuple[str, dict[str, str]]: ... + def group(self, name: str) -> tuple[str, int, int, int, str]: ... + def help(self, *, file: _File = None) -> tuple[str, _list[str]]: ... + def stat(self, message_spec: Any = None) -> tuple[str, int, str]: ... + def next(self) -> tuple[str, int, str]: ... + def last(self) -> tuple[str, int, str]: ... + def head(self, message_spec: Any = None, *, file: _File = None) -> tuple[str, ArticleInfo]: ... + def body(self, message_spec: Any = None, *, file: _File = None) -> tuple[str, ArticleInfo]: ... + def article(self, message_spec: Any = None, *, file: _File = None) -> tuple[str, ArticleInfo]: ... + def slave(self) -> str: ... + def xhdr(self, hdr: str, str: Any, *, file: _File = None) -> tuple[str, _list[str]]: ... + def xover(self, start: int, end: int, *, file: _File = None) -> tuple[str, _list[tuple[int, dict[str, str]]]]: ... + def over( + self, message_spec: None | str | _list[Any] | tuple[Any, ...], *, file: _File = None + ) -> tuple[str, _list[tuple[int, dict[str, str]]]]: ... + if sys.version_info < (3, 9): + def xgtitle(self, group: str, *, file: _File = None) -> tuple[str, _list[tuple[str, str]]]: ... + def xpath(self, id: Any) -> tuple[str, str]: ... + + def date(self) -> tuple[str, datetime.datetime]: ... + def post(self, data: bytes | Iterable[bytes]) -> str: ... + def ihave(self, message_id: Any, data: bytes | Iterable[bytes]) -> str: ... + def quit(self) -> str: ... + def login(self, user: str | None = None, password: str | None = None, usenetrc: bool = True) -> None: ... + def starttls(self, context: ssl.SSLContext | None = None) -> None: ... + +class NNTP_SSL(NNTP): + ssl_context: ssl.SSLContext | None + sock: ssl.SSLSocket + def __init__( + self, + host: str, + port: int = 563, + user: str | None = None, + password: str | None = None, + ssl_context: ssl.SSLContext | None = None, + readermode: bool | None = None, + usenetrc: bool = False, + timeout: float = ..., + ) -> None: ... diff --git a/mypy/typeshed/stdlib/nt.pyi b/mypy/typeshed/stdlib/nt.pyi new file mode 100644 index 000000000000..e1d57d09a9bd --- /dev/null +++ b/mypy/typeshed/stdlib/nt.pyi @@ -0,0 +1,113 @@ +import sys + +if sys.platform == "win32": + # Actually defined here and re-exported from os at runtime, + # but this leads to less code duplication + from os import ( + F_OK as F_OK, + O_APPEND as O_APPEND, + O_BINARY as O_BINARY, + O_CREAT as O_CREAT, + O_EXCL as O_EXCL, + O_NOINHERIT as O_NOINHERIT, + O_RANDOM as O_RANDOM, + O_RDONLY as O_RDONLY, + O_RDWR as O_RDWR, + O_SEQUENTIAL as O_SEQUENTIAL, + O_SHORT_LIVED as O_SHORT_LIVED, + O_TEMPORARY as O_TEMPORARY, + O_TEXT as O_TEXT, + O_TRUNC as O_TRUNC, + O_WRONLY as O_WRONLY, + P_DETACH as P_DETACH, + P_NOWAIT as P_NOWAIT, + P_NOWAITO as P_NOWAITO, + P_OVERLAY as P_OVERLAY, + P_WAIT as P_WAIT, + R_OK as R_OK, + TMP_MAX as TMP_MAX, + W_OK as W_OK, + X_OK as X_OK, + DirEntry as DirEntry, + abort as abort, + access as access, + chdir as chdir, + chmod as chmod, + close as close, + closerange as closerange, + cpu_count as cpu_count, + device_encoding as device_encoding, + dup as dup, + dup2 as dup2, + error as error, + execv as execv, + execve as execve, + fspath as fspath, + fstat as fstat, + fsync as fsync, + ftruncate as ftruncate, + get_handle_inheritable as get_handle_inheritable, + get_inheritable as get_inheritable, + get_terminal_size as get_terminal_size, + getcwd as getcwd, + getcwdb as getcwdb, + getlogin as getlogin, + getpid as getpid, + getppid as getppid, + isatty as isatty, + kill as kill, + link as link, + listdir as listdir, + lseek as lseek, + lstat as lstat, + mkdir as mkdir, + open as open, + pipe as pipe, + putenv as putenv, + read as read, + readlink as readlink, + remove as remove, + rename as rename, + replace as replace, + rmdir as rmdir, + scandir as scandir, + set_handle_inheritable as set_handle_inheritable, + set_inheritable as set_inheritable, + spawnv as spawnv, + spawnve as spawnve, + startfile as startfile, + stat as stat, + stat_result as stat_result, + statvfs_result as statvfs_result, + strerror as strerror, + symlink as symlink, + system as system, + terminal_size as terminal_size, + times as times, + times_result as times_result, + truncate as truncate, + umask as umask, + uname_result as uname_result, + unlink as unlink, + urandom as urandom, + utime as utime, + waitpid as waitpid, + write as write, + ) + + if sys.version_info >= (3, 9): + from os import unsetenv as unsetenv, waitstatus_to_exitcode as waitstatus_to_exitcode + if sys.version_info >= (3, 11): + from os import EX_OK as EX_OK + if sys.version_info >= (3, 12): + from os import ( + get_blocking as get_blocking, + listdrives as listdrives, + listmounts as listmounts, + listvolumes as listvolumes, + set_blocking as set_blocking, + ) + if sys.version_info >= (3, 13): + from os import fchmod as fchmod, lchmod as lchmod + + environ: dict[str, str] diff --git a/mypy/typeshed/stdlib/ntpath.pyi b/mypy/typeshed/stdlib/ntpath.pyi new file mode 100644 index 000000000000..ebe305ef708c --- /dev/null +++ b/mypy/typeshed/stdlib/ntpath.pyi @@ -0,0 +1,126 @@ +import sys +from _typeshed import BytesPath, StrOrBytesPath, StrPath +from genericpath import ( + commonprefix as commonprefix, + exists as exists, + getatime as getatime, + getctime as getctime, + getmtime as getmtime, + getsize as getsize, + isdir as isdir, + isfile as isfile, + samefile as samefile, + sameopenfile as sameopenfile, + samestat as samestat, +) +from os import PathLike + +# Re-export common definitions from posixpath to reduce duplication +from posixpath import ( + abspath as abspath, + basename as basename, + commonpath as commonpath, + curdir as curdir, + defpath as defpath, + devnull as devnull, + dirname as dirname, + expanduser as expanduser, + expandvars as expandvars, + extsep as extsep, + isabs as isabs, + islink as islink, + ismount as ismount, + lexists as lexists, + normcase as normcase, + normpath as normpath, + pardir as pardir, + pathsep as pathsep, + relpath as relpath, + sep as sep, + split as split, + splitdrive as splitdrive, + splitext as splitext, + supports_unicode_filenames as supports_unicode_filenames, +) +from typing import AnyStr, overload +from typing_extensions import LiteralString + +if sys.version_info >= (3, 12): + from posixpath import isjunction as isjunction, splitroot as splitroot +if sys.version_info >= (3, 13): + from genericpath import isdevdrive as isdevdrive + +__all__ = [ + "normcase", + "isabs", + "join", + "splitdrive", + "split", + "splitext", + "basename", + "dirname", + "commonprefix", + "getsize", + "getmtime", + "getatime", + "getctime", + "islink", + "exists", + "lexists", + "isdir", + "isfile", + "ismount", + "expanduser", + "expandvars", + "normpath", + "abspath", + "curdir", + "pardir", + "sep", + "pathsep", + "defpath", + "altsep", + "extsep", + "devnull", + "realpath", + "supports_unicode_filenames", + "relpath", + "samefile", + "sameopenfile", + "samestat", + "commonpath", +] +if sys.version_info >= (3, 12): + __all__ += ["isjunction", "splitroot"] +if sys.version_info >= (3, 13): + __all__ += ["isdevdrive", "isreserved"] + +altsep: LiteralString + +# First parameter is not actually pos-only, +# but must be defined as pos-only in the stub or cross-platform code doesn't type-check, +# as the parameter name is different in posixpath.join() +@overload +def join(path: LiteralString, /, *paths: LiteralString) -> LiteralString: ... +@overload +def join(path: StrPath, /, *paths: StrPath) -> str: ... +@overload +def join(path: BytesPath, /, *paths: BytesPath) -> bytes: ... + +if sys.platform == "win32": + if sys.version_info >= (3, 10): + @overload + def realpath(path: PathLike[AnyStr], *, strict: bool = False) -> AnyStr: ... + @overload + def realpath(path: AnyStr, *, strict: bool = False) -> AnyStr: ... + else: + @overload + def realpath(path: PathLike[AnyStr]) -> AnyStr: ... + @overload + def realpath(path: AnyStr) -> AnyStr: ... + +else: + realpath = abspath + +if sys.version_info >= (3, 13): + def isreserved(path: StrOrBytesPath) -> bool: ... diff --git a/mypy/typeshed/stdlib/nturl2path.pyi b/mypy/typeshed/stdlib/nturl2path.pyi new file mode 100644 index 000000000000..b8ad8d682155 --- /dev/null +++ b/mypy/typeshed/stdlib/nturl2path.pyi @@ -0,0 +1,2 @@ +def url2pathname(url: str) -> str: ... +def pathname2url(p: str) -> str: ... diff --git a/mypy/typeshed/stdlib/numbers.pyi b/mypy/typeshed/stdlib/numbers.pyi new file mode 100644 index 000000000000..e129de2cdc67 --- /dev/null +++ b/mypy/typeshed/stdlib/numbers.pyi @@ -0,0 +1,209 @@ +# Note: these stubs are incomplete. The more complex type +# signatures are currently omitted. +# +# Use _ComplexLike, _RealLike and _IntegralLike for return types in this module +# rather than `numbers.Complex`, `numbers.Real` and `numbers.Integral`, +# to avoid an excessive number of `type: ignore`s in subclasses of these ABCs +# (since type checkers don't see `complex` as a subtype of `numbers.Complex`, +# nor `float` as a subtype of `numbers.Real`, etc.) + +from _typeshed import Incomplete +from abc import ABCMeta, abstractmethod +from typing import Literal, Protocol, overload + +__all__ = ["Number", "Complex", "Real", "Rational", "Integral"] + +############################ +# Protocols for return types +############################ + +# `_ComplexLike` is a structural-typing approximation +# of the `Complex` ABC, which is not (and cannot be) a protocol +# +# NOTE: We can't include `__complex__` here, +# as we want `int` to be seen as a subtype of `_ComplexLike`, +# and `int.__complex__` does not exist :( +class _ComplexLike(Protocol): + def __neg__(self) -> _ComplexLike: ... + def __pos__(self) -> _ComplexLike: ... + def __abs__(self) -> _RealLike: ... + +# _RealLike is a structural-typing approximation +# of the `Real` ABC, which is not (and cannot be) a protocol +class _RealLike(_ComplexLike, Protocol): + def __trunc__(self) -> _IntegralLike: ... + def __floor__(self) -> _IntegralLike: ... + def __ceil__(self) -> _IntegralLike: ... + def __float__(self) -> float: ... + # Overridden from `_ComplexLike` + # for a more precise return type: + def __neg__(self) -> _RealLike: ... + def __pos__(self) -> _RealLike: ... + +# _IntegralLike is a structural-typing approximation +# of the `Integral` ABC, which is not (and cannot be) a protocol +class _IntegralLike(_RealLike, Protocol): + def __invert__(self) -> _IntegralLike: ... + def __int__(self) -> int: ... + def __index__(self) -> int: ... + # Overridden from `_ComplexLike` + # for a more precise return type: + def __abs__(self) -> _IntegralLike: ... + # Overridden from `RealLike` + # for a more precise return type: + def __neg__(self) -> _IntegralLike: ... + def __pos__(self) -> _IntegralLike: ... + +################# +# Module "proper" +################# + +class Number(metaclass=ABCMeta): + @abstractmethod + def __hash__(self) -> int: ... + +# See comment at the top of the file +# for why some of these return types are purposefully vague +class Complex(Number, _ComplexLike): + @abstractmethod + def __complex__(self) -> complex: ... + def __bool__(self) -> bool: ... + @property + @abstractmethod + def real(self) -> _RealLike: ... + @property + @abstractmethod + def imag(self) -> _RealLike: ... + @abstractmethod + def __add__(self, other) -> _ComplexLike: ... + @abstractmethod + def __radd__(self, other) -> _ComplexLike: ... + @abstractmethod + def __neg__(self) -> _ComplexLike: ... + @abstractmethod + def __pos__(self) -> _ComplexLike: ... + def __sub__(self, other) -> _ComplexLike: ... + def __rsub__(self, other) -> _ComplexLike: ... + @abstractmethod + def __mul__(self, other) -> _ComplexLike: ... + @abstractmethod + def __rmul__(self, other) -> _ComplexLike: ... + @abstractmethod + def __truediv__(self, other) -> _ComplexLike: ... + @abstractmethod + def __rtruediv__(self, other) -> _ComplexLike: ... + @abstractmethod + def __pow__(self, exponent) -> _ComplexLike: ... + @abstractmethod + def __rpow__(self, base) -> _ComplexLike: ... + @abstractmethod + def __abs__(self) -> _RealLike: ... + @abstractmethod + def conjugate(self) -> _ComplexLike: ... + @abstractmethod + def __eq__(self, other: object) -> bool: ... + +# See comment at the top of the file +# for why some of these return types are purposefully vague +class Real(Complex, _RealLike): + @abstractmethod + def __float__(self) -> float: ... + @abstractmethod + def __trunc__(self) -> _IntegralLike: ... + @abstractmethod + def __floor__(self) -> _IntegralLike: ... + @abstractmethod + def __ceil__(self) -> _IntegralLike: ... + @abstractmethod + @overload + def __round__(self, ndigits: None = None) -> _IntegralLike: ... + @abstractmethod + @overload + def __round__(self, ndigits: int) -> _RealLike: ... + def __divmod__(self, other) -> tuple[_RealLike, _RealLike]: ... + def __rdivmod__(self, other) -> tuple[_RealLike, _RealLike]: ... + @abstractmethod + def __floordiv__(self, other) -> _RealLike: ... + @abstractmethod + def __rfloordiv__(self, other) -> _RealLike: ... + @abstractmethod + def __mod__(self, other) -> _RealLike: ... + @abstractmethod + def __rmod__(self, other) -> _RealLike: ... + @abstractmethod + def __lt__(self, other) -> bool: ... + @abstractmethod + def __le__(self, other) -> bool: ... + def __complex__(self) -> complex: ... + @property + def real(self) -> _RealLike: ... + @property + def imag(self) -> Literal[0]: ... + def conjugate(self) -> _RealLike: ... + # Not actually overridden at runtime, + # but we override these in the stub to give them more precise return types: + @abstractmethod + def __pos__(self) -> _RealLike: ... + @abstractmethod + def __neg__(self) -> _RealLike: ... + +# See comment at the top of the file +# for why some of these return types are purposefully vague +class Rational(Real): + @property + @abstractmethod + def numerator(self) -> _IntegralLike: ... + @property + @abstractmethod + def denominator(self) -> _IntegralLike: ... + def __float__(self) -> float: ... + +# See comment at the top of the file +# for why some of these return types are purposefully vague +class Integral(Rational, _IntegralLike): + @abstractmethod + def __int__(self) -> int: ... + def __index__(self) -> int: ... + @abstractmethod + def __pow__(self, exponent, modulus: Incomplete | None = None) -> _IntegralLike: ... + @abstractmethod + def __lshift__(self, other) -> _IntegralLike: ... + @abstractmethod + def __rlshift__(self, other) -> _IntegralLike: ... + @abstractmethod + def __rshift__(self, other) -> _IntegralLike: ... + @abstractmethod + def __rrshift__(self, other) -> _IntegralLike: ... + @abstractmethod + def __and__(self, other) -> _IntegralLike: ... + @abstractmethod + def __rand__(self, other) -> _IntegralLike: ... + @abstractmethod + def __xor__(self, other) -> _IntegralLike: ... + @abstractmethod + def __rxor__(self, other) -> _IntegralLike: ... + @abstractmethod + def __or__(self, other) -> _IntegralLike: ... + @abstractmethod + def __ror__(self, other) -> _IntegralLike: ... + @abstractmethod + def __invert__(self) -> _IntegralLike: ... + def __float__(self) -> float: ... + @property + def numerator(self) -> _IntegralLike: ... + @property + def denominator(self) -> Literal[1]: ... + # Not actually overridden at runtime, + # but we override these in the stub to give them more precise return types: + @abstractmethod + def __pos__(self) -> _IntegralLike: ... + @abstractmethod + def __neg__(self) -> _IntegralLike: ... + @abstractmethod + def __abs__(self) -> _IntegralLike: ... + @abstractmethod + @overload + def __round__(self, ndigits: None = None) -> _IntegralLike: ... + @abstractmethod + @overload + def __round__(self, ndigits: int) -> _IntegralLike: ... diff --git a/mypy/typeshed/stdlib/opcode.pyi b/mypy/typeshed/stdlib/opcode.pyi new file mode 100644 index 000000000000..f9f76962f876 --- /dev/null +++ b/mypy/typeshed/stdlib/opcode.pyi @@ -0,0 +1,63 @@ +import sys +from typing import Literal + +__all__ = [ + "cmp_op", + "hasconst", + "hasname", + "hasjrel", + "hasjabs", + "haslocal", + "hascompare", + "hasfree", + "opname", + "opmap", + "HAVE_ARGUMENT", + "EXTENDED_ARG", + "stack_effect", +] +if sys.version_info >= (3, 12): + __all__ += ["hasarg", "hasexc"] +else: + __all__ += ["hasnargs"] +if sys.version_info >= (3, 13): + __all__ += ["hasjump"] + +if sys.version_info >= (3, 9): + cmp_op: tuple[Literal["<"], Literal["<="], Literal["=="], Literal["!="], Literal[">"], Literal[">="]] +else: + cmp_op: tuple[ + Literal["<"], + Literal["<="], + Literal["=="], + Literal["!="], + Literal[">"], + Literal[">="], + Literal["in"], + Literal["not in"], + Literal["is"], + Literal["is not"], + Literal["exception match"], + Literal["BAD"], + ] +hasconst: list[int] +hasname: list[int] +hasjrel: list[int] +hasjabs: list[int] +haslocal: list[int] +hascompare: list[int] +hasfree: list[int] +if sys.version_info >= (3, 12): + hasarg: list[int] + hasexc: list[int] +else: + hasnargs: list[int] +if sys.version_info >= (3, 13): + hasjump: list[int] +opname: list[str] + +opmap: dict[str, int] +HAVE_ARGUMENT: int +EXTENDED_ARG: int + +def stack_effect(opcode: int, oparg: int | None = None, /, *, jump: bool | None = None) -> int: ... diff --git a/mypy/typeshed/stdlib/operator.pyi b/mypy/typeshed/stdlib/operator.pyi new file mode 100644 index 000000000000..b73e037f3ed9 --- /dev/null +++ b/mypy/typeshed/stdlib/operator.pyi @@ -0,0 +1,215 @@ +import sys +from _operator import ( + abs as abs, + add as add, + and_ as and_, + concat as concat, + contains as contains, + countOf as countOf, + delitem as delitem, + eq as eq, + floordiv as floordiv, + ge as ge, + getitem as getitem, + gt as gt, + iadd as iadd, + iand as iand, + iconcat as iconcat, + ifloordiv as ifloordiv, + ilshift as ilshift, + imatmul as imatmul, + imod as imod, + imul as imul, + index as index, + indexOf as indexOf, + inv as inv, + invert as invert, + ior as ior, + ipow as ipow, + irshift as irshift, + is_ as is_, + is_not as is_not, + isub as isub, + itruediv as itruediv, + ixor as ixor, + le as le, + length_hint as length_hint, + lshift as lshift, + lt as lt, + matmul as matmul, + mod as mod, + mul as mul, + ne as ne, + neg as neg, + not_ as not_, + or_ as or_, + pos as pos, + pow as pow, + rshift as rshift, + setitem as setitem, + sub as sub, + truediv as truediv, + truth as truth, + xor as xor, +) +from _typeshed import SupportsGetItem +from typing import Any, Generic, TypeVar, final, overload +from typing_extensions import TypeVarTuple, Unpack + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_Ts = TypeVarTuple("_Ts") + +__all__ = [ + "abs", + "add", + "and_", + "attrgetter", + "concat", + "contains", + "countOf", + "delitem", + "eq", + "floordiv", + "ge", + "getitem", + "gt", + "iadd", + "iand", + "iconcat", + "ifloordiv", + "ilshift", + "imatmul", + "imod", + "imul", + "index", + "indexOf", + "inv", + "invert", + "ior", + "ipow", + "irshift", + "is_", + "is_not", + "isub", + "itemgetter", + "itruediv", + "ixor", + "le", + "length_hint", + "lshift", + "lt", + "matmul", + "methodcaller", + "mod", + "mul", + "ne", + "neg", + "not_", + "or_", + "pos", + "pow", + "rshift", + "setitem", + "sub", + "truediv", + "truth", + "xor", +] + +if sys.version_info >= (3, 11): + from _operator import call as call + + __all__ += ["call"] + +if sys.version_info >= (3, 14): + from _operator import is_none as is_none, is_not_none as is_not_none + + __all__ += ["is_none", "is_not_none"] + +__lt__ = lt +__le__ = le +__eq__ = eq +__ne__ = ne +__ge__ = ge +__gt__ = gt +__not__ = not_ +__abs__ = abs +__add__ = add +__and__ = and_ +__floordiv__ = floordiv +__index__ = index +__inv__ = inv +__invert__ = invert +__lshift__ = lshift +__mod__ = mod +__mul__ = mul +__matmul__ = matmul +__neg__ = neg +__or__ = or_ +__pos__ = pos +__pow__ = pow +__rshift__ = rshift +__sub__ = sub +__truediv__ = truediv +__xor__ = xor +__concat__ = concat +__contains__ = contains +__delitem__ = delitem +__getitem__ = getitem +__setitem__ = setitem +__iadd__ = iadd +__iand__ = iand +__iconcat__ = iconcat +__ifloordiv__ = ifloordiv +__ilshift__ = ilshift +__imod__ = imod +__imul__ = imul +__imatmul__ = imatmul +__ior__ = ior +__ipow__ = ipow +__irshift__ = irshift +__isub__ = isub +__itruediv__ = itruediv +__ixor__ = ixor +if sys.version_info >= (3, 11): + __call__ = call + +# At runtime, these classes are implemented in C as part of the _operator module +# However, they consider themselves to live in the operator module, so we'll put +# them here. +@final +class attrgetter(Generic[_T_co]): + @overload + def __new__(cls, attr: str, /) -> attrgetter[Any]: ... + @overload + def __new__(cls, attr: str, attr2: str, /) -> attrgetter[tuple[Any, Any]]: ... + @overload + def __new__(cls, attr: str, attr2: str, attr3: str, /) -> attrgetter[tuple[Any, Any, Any]]: ... + @overload + def __new__(cls, attr: str, attr2: str, attr3: str, attr4: str, /) -> attrgetter[tuple[Any, Any, Any, Any]]: ... + @overload + def __new__(cls, attr: str, /, *attrs: str) -> attrgetter[tuple[Any, ...]]: ... + def __call__(self, obj: Any, /) -> _T_co: ... + +@final +class itemgetter(Generic[_T_co]): + @overload + def __new__(cls, item: _T, /) -> itemgetter[_T]: ... + @overload + def __new__(cls, item1: _T1, item2: _T2, /, *items: Unpack[_Ts]) -> itemgetter[tuple[_T1, _T2, Unpack[_Ts]]]: ... + # __key: _KT_contra in SupportsGetItem seems to be causing variance issues, ie: + # TypeVar "_KT_contra@SupportsGetItem" is contravariant + # "tuple[int, int]" is incompatible with protocol "SupportsIndex" + # preventing [_T_co, ...] instead of [Any, ...] + # + # A suspected mypy issue prevents using [..., _T] instead of [..., Any] here. + # https://github.com/python/mypy/issues/14032 + def __call__(self, obj: SupportsGetItem[Any, Any]) -> Any: ... + +@final +class methodcaller: + def __init__(self, name: str, /, *args: Any, **kwargs: Any) -> None: ... + def __call__(self, obj: Any) -> Any: ... diff --git a/mypy/typeshed/stdlib/optparse.pyi b/mypy/typeshed/stdlib/optparse.pyi new file mode 100644 index 000000000000..d6db7a06f291 --- /dev/null +++ b/mypy/typeshed/stdlib/optparse.pyi @@ -0,0 +1,255 @@ +from _typeshed import Incomplete, MaybeNone +from abc import abstractmethod +from collections.abc import Callable, Iterable, Mapping, Sequence +from typing import IO, Any, AnyStr, Literal, NoReturn, overload + +__all__ = [ + "Option", + "make_option", + "SUPPRESS_HELP", + "SUPPRESS_USAGE", + "Values", + "OptionContainer", + "OptionGroup", + "OptionParser", + "HelpFormatter", + "IndentedHelpFormatter", + "TitledHelpFormatter", + "OptParseError", + "OptionError", + "OptionConflictError", + "OptionValueError", + "BadOptionError", + "check_choice", +] + +NO_DEFAULT: tuple[str, ...] +SUPPRESS_HELP: str +SUPPRESS_USAGE: str + +def check_builtin(option: Option, opt, value: str): ... +def check_choice(option: Option, opt, value: str) -> str: ... + +class OptParseError(Exception): + msg: str + def __init__(self, msg: str) -> None: ... + +class BadOptionError(OptParseError): + opt_str: str + def __init__(self, opt_str: str) -> None: ... + +class AmbiguousOptionError(BadOptionError): + possibilities: Iterable[str] + def __init__(self, opt_str: str, possibilities: Sequence[str]) -> None: ... + +class OptionError(OptParseError): + option_id: str + def __init__(self, msg: str, option: Option) -> None: ... + +class OptionConflictError(OptionError): ... +class OptionValueError(OptParseError): ... + +class HelpFormatter: + NO_DEFAULT_VALUE: str + _long_opt_fmt: str + _short_opt_fmt: str + current_indent: int + default_tag: str + help_position: int + help_width: int | MaybeNone # initialized as None and computed later as int when storing option strings + indent_increment: int + level: int + max_help_position: int + option_strings: dict[Option, str] + parser: OptionParser + short_first: Incomplete + width: int + def __init__(self, indent_increment: int, max_help_position: int, width: int | None, short_first: int) -> None: ... + def dedent(self) -> None: ... + def expand_default(self, option: Option) -> str: ... + def format_description(self, description: str | None) -> str: ... + def format_epilog(self, epilog: str | None) -> str: ... + @abstractmethod + def format_heading(self, heading: str) -> str: ... + def format_option(self, option: Option) -> str: ... + def format_option_strings(self, option: Option) -> str: ... + @abstractmethod + def format_usage(self, usage: str) -> str: ... + def indent(self) -> None: ... + def set_long_opt_delimiter(self, delim: str) -> None: ... + def set_parser(self, parser: OptionParser) -> None: ... + def set_short_opt_delimiter(self, delim: str) -> None: ... + def store_option_strings(self, parser: OptionParser) -> None: ... + +class IndentedHelpFormatter(HelpFormatter): + def __init__( + self, indent_increment: int = 2, max_help_position: int = 24, width: int | None = None, short_first: int = 1 + ) -> None: ... + def format_heading(self, heading: str) -> str: ... + def format_usage(self, usage: str) -> str: ... + +class TitledHelpFormatter(HelpFormatter): + def __init__( + self, indent_increment: int = 0, max_help_position: int = 24, width: int | None = None, short_first: int = 0 + ) -> None: ... + def format_heading(self, heading: str) -> str: ... + def format_usage(self, usage: str) -> str: ... + +class Option: + ACTIONS: tuple[str, ...] + ALWAYS_TYPED_ACTIONS: tuple[str, ...] + ATTRS: list[str] + CHECK_METHODS: list[Callable[..., Incomplete]] | None + CONST_ACTIONS: tuple[str, ...] + STORE_ACTIONS: tuple[str, ...] + TYPED_ACTIONS: tuple[str, ...] + TYPES: tuple[str, ...] + TYPE_CHECKER: dict[str, Callable[[Option, str, Incomplete], Any]] + _long_opts: list[str] + _short_opts: list[str] + action: str + dest: str | None + default: Incomplete + nargs: int + type: Incomplete + callback: Callable[..., Incomplete] | None + callback_args: tuple[Incomplete, ...] | None + callback_kwargs: dict[str, Incomplete] | None + help: str | None + metavar: str | None + def __init__(self, *opts: str | None, **attrs) -> None: ... + def _check_action(self) -> None: ... + def _check_callback(self) -> None: ... + def _check_choice(self) -> None: ... + def _check_const(self) -> None: ... + def _check_dest(self) -> None: ... + def _check_nargs(self) -> None: ... + def _check_opt_strings(self, opts: Iterable[str | None]) -> list[str]: ... + def _check_type(self) -> None: ... + def _set_attrs(self, attrs: dict[str, Incomplete]) -> None: ... + def _set_opt_strings(self, opts: Iterable[str]) -> None: ... + def check_value(self, opt: str, value): ... + def convert_value(self, opt: str, value): ... + def get_opt_string(self) -> str: ... + def process(self, opt, value, values, parser: OptionParser) -> int: ... + def take_action(self, action: str, dest: str, opt, value, values, parser: OptionParser) -> int: ... + def takes_value(self) -> bool: ... + +make_option = Option + +class OptionContainer: + _long_opt: dict[str, Option] + _short_opt: dict[str, Option] + conflict_handler: str + defaults: dict[str, Incomplete] + description: str | None + option_class: type[Option] + def __init__( + self, option_class: type[Option], conflict_handler: Literal["error", "resolve"], description: str | None + ) -> None: ... + def _check_conflict(self, option: Option) -> None: ... + def _create_option_mappings(self) -> None: ... + def _share_option_mappings(self, parser: OptionParser) -> None: ... + @overload + def add_option(self, opt: Option, /) -> Option: ... + @overload + def add_option(self, arg: str, /, *args: str | None, **kwargs) -> Option: ... + def add_options(self, option_list: Iterable[Option]) -> None: ... + def destroy(self) -> None: ... + def format_option_help(self, formatter: HelpFormatter) -> str: ... + def format_description(self, formatter: HelpFormatter) -> str: ... + def format_help(self, formatter: HelpFormatter) -> str: ... + def get_description(self) -> str | None: ... + def get_option(self, opt_str: str) -> Option | None: ... + def has_option(self, opt_str: str) -> bool: ... + def remove_option(self, opt_str: str) -> None: ... + def set_conflict_handler(self, handler: Literal["error", "resolve"]) -> None: ... + def set_description(self, description: str | None) -> None: ... + +class OptionGroup(OptionContainer): + option_list: list[Option] + parser: OptionParser + title: str + def __init__(self, parser: OptionParser, title: str, description: str | None = None) -> None: ... + def _create_option_list(self) -> None: ... + def set_title(self, title: str) -> None: ... + +class Values: + def __init__(self, defaults: Mapping[str, Incomplete] | None = None) -> None: ... + def _update(self, dict: Mapping[str, Incomplete], mode) -> None: ... + def _update_careful(self, dict: Mapping[str, Incomplete]) -> None: ... + def _update_loose(self, dict: Mapping[str, Incomplete]) -> None: ... + def ensure_value(self, attr: str, value): ... + def read_file(self, filename: str, mode: str = "careful") -> None: ... + def read_module(self, modname: str, mode: str = "careful") -> None: ... + def __getattr__(self, name: str): ... + def __setattr__(self, name: str, value, /) -> None: ... + def __eq__(self, other: object) -> bool: ... + +class OptionParser(OptionContainer): + allow_interspersed_args: bool + epilog: str | None + formatter: HelpFormatter + largs: list[str] | None + option_groups: list[OptionGroup] + option_list: list[Option] + process_default_values: bool + prog: str | None + rargs: list[str] | None + standard_option_list: list[Option] + usage: str | None + values: Values | None + version: str + def __init__( + self, + usage: str | None = None, + option_list: Iterable[Option] | None = None, + option_class: type[Option] = ..., + version: str | None = None, + conflict_handler: str = "error", + description: str | None = None, + formatter: HelpFormatter | None = None, + add_help_option: bool = True, + prog: str | None = None, + epilog: str | None = None, + ) -> None: ... + def _add_help_option(self) -> None: ... + def _add_version_option(self) -> None: ... + def _create_option_list(self) -> None: ... + def _get_all_options(self) -> list[Option]: ... + def _get_args(self, args: Iterable[Incomplete]) -> list[Incomplete]: ... + def _init_parsing_state(self) -> None: ... + def _match_long_opt(self, opt: str) -> str: ... + def _populate_option_list(self, option_list: Iterable[Option], add_help: bool = True) -> None: ... + def _process_args(self, largs: list[Incomplete], rargs: list[Incomplete], values: Values) -> None: ... + def _process_long_opt(self, rargs: list[Incomplete], values) -> None: ... + def _process_short_opts(self, rargs: list[Incomplete], values) -> None: ... + @overload + def add_option_group(self, opt_group: OptionGroup, /) -> OptionGroup: ... + @overload + def add_option_group(self, *args, **kwargs) -> OptionGroup: ... + def check_values(self, values: Values, args: list[str]) -> tuple[Values, list[str]]: ... + def disable_interspersed_args(self) -> None: ... + def enable_interspersed_args(self) -> None: ... + def error(self, msg: str) -> NoReturn: ... + def exit(self, status: int = 0, msg: str | None = None) -> NoReturn: ... + def expand_prog_name(self, s: str) -> str: ... + def format_epilog(self, formatter: HelpFormatter) -> str: ... + def format_help(self, formatter: HelpFormatter | None = None) -> str: ... + def format_option_help(self, formatter: HelpFormatter | None = None) -> str: ... + def get_default_values(self) -> Values: ... + def get_option_group(self, opt_str: str) -> OptionGroup | None: ... + def get_prog_name(self) -> str: ... + def get_usage(self) -> str: ... + def get_version(self) -> str: ... + @overload + def parse_args(self, args: None = None, values: Values | None = None) -> tuple[Values, list[str]]: ... + @overload + def parse_args(self, args: Sequence[AnyStr], values: Values | None = None) -> tuple[Values, list[AnyStr]]: ... + def print_usage(self, file: IO[str] | None = None) -> None: ... + def print_help(self, file: IO[str] | None = None) -> None: ... + def print_version(self, file: IO[str] | None = None) -> None: ... + def set_default(self, dest, value) -> None: ... + def set_defaults(self, **kwargs) -> None: ... + def set_process_default_values(self, process) -> None: ... + def set_usage(self, usage: str) -> None: ... diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi new file mode 100644 index 000000000000..98260b14e7ed --- /dev/null +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -0,0 +1,1675 @@ +import sys +from _typeshed import ( + AnyStr_co, + BytesPath, + FileDescriptor, + FileDescriptorLike, + FileDescriptorOrPath, + GenericPath, + OpenBinaryMode, + OpenBinaryModeReading, + OpenBinaryModeUpdating, + OpenBinaryModeWriting, + OpenTextMode, + ReadableBuffer, + StrOrBytesPath, + StrPath, + SupportsLenAndGetItem, + Unused, + WriteableBuffer, + structseq, +) +from abc import ABC, abstractmethod +from builtins import OSError +from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence +from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper +from subprocess import Popen +from types import TracebackType +from typing import ( + IO, + Any, + AnyStr, + BinaryIO, + Final, + Generic, + Literal, + NoReturn, + Protocol, + TypeVar, + final, + overload, + runtime_checkable, +) +from typing_extensions import Self, TypeAlias, Unpack, deprecated + +from . import path as _path + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "F_OK", + "O_APPEND", + "O_CREAT", + "O_EXCL", + "O_RDONLY", + "O_RDWR", + "O_TRUNC", + "O_WRONLY", + "P_NOWAIT", + "P_NOWAITO", + "P_WAIT", + "R_OK", + "SEEK_CUR", + "SEEK_END", + "SEEK_SET", + "TMP_MAX", + "W_OK", + "X_OK", + "DirEntry", + "_exit", + "abort", + "access", + "altsep", + "chdir", + "chmod", + "close", + "closerange", + "cpu_count", + "curdir", + "defpath", + "device_encoding", + "devnull", + "dup", + "dup2", + "environ", + "error", + "execl", + "execle", + "execlp", + "execlpe", + "execv", + "execve", + "execvp", + "execvpe", + "extsep", + "fdopen", + "fsdecode", + "fsencode", + "fspath", + "fstat", + "fsync", + "ftruncate", + "get_exec_path", + "get_inheritable", + "get_terminal_size", + "getcwd", + "getcwdb", + "getenv", + "getlogin", + "getpid", + "getppid", + "isatty", + "kill", + "linesep", + "link", + "listdir", + "lseek", + "lstat", + "makedirs", + "mkdir", + "name", + "open", + "pardir", + "path", + "pathsep", + "pipe", + "popen", + "putenv", + "read", + "readlink", + "remove", + "removedirs", + "rename", + "renames", + "replace", + "rmdir", + "scandir", + "sep", + "set_inheritable", + "spawnl", + "spawnle", + "spawnv", + "spawnve", + "stat", + "stat_result", + "statvfs_result", + "strerror", + "supports_bytes_environ", + "symlink", + "system", + "terminal_size", + "times", + "times_result", + "truncate", + "umask", + "uname_result", + "unlink", + "urandom", + "utime", + "waitpid", + "walk", + "write", +] +if sys.version_info >= (3, 9): + __all__ += ["waitstatus_to_exitcode"] +if sys.platform == "darwin" and sys.version_info >= (3, 12): + __all__ += ["PRIO_DARWIN_BG", "PRIO_DARWIN_NONUI", "PRIO_DARWIN_PROCESS", "PRIO_DARWIN_THREAD"] +if sys.platform == "darwin" and sys.version_info >= (3, 10): + __all__ += ["O_EVTONLY", "O_NOFOLLOW_ANY", "O_SYMLINK"] +if sys.platform == "linux": + __all__ += [ + "GRND_NONBLOCK", + "GRND_RANDOM", + "MFD_ALLOW_SEALING", + "MFD_CLOEXEC", + "MFD_HUGETLB", + "MFD_HUGE_16GB", + "MFD_HUGE_16MB", + "MFD_HUGE_1GB", + "MFD_HUGE_1MB", + "MFD_HUGE_256MB", + "MFD_HUGE_2GB", + "MFD_HUGE_2MB", + "MFD_HUGE_32MB", + "MFD_HUGE_512KB", + "MFD_HUGE_512MB", + "MFD_HUGE_64KB", + "MFD_HUGE_8MB", + "MFD_HUGE_MASK", + "MFD_HUGE_SHIFT", + "O_DIRECT", + "O_LARGEFILE", + "O_NOATIME", + "O_PATH", + "O_RSYNC", + "O_TMPFILE", + "RTLD_DEEPBIND", + "SCHED_BATCH", + "SCHED_IDLE", + "SCHED_RESET_ON_FORK", + "XATTR_CREATE", + "XATTR_REPLACE", + "XATTR_SIZE_MAX", + "copy_file_range", + "getrandom", + "getxattr", + "listxattr", + "memfd_create", + "removexattr", + "setxattr", + ] +if sys.platform == "linux" and sys.version_info >= (3, 13): + __all__ += [ + "POSIX_SPAWN_CLOSEFROM", + "TFD_CLOEXEC", + "TFD_NONBLOCK", + "TFD_TIMER_ABSTIME", + "TFD_TIMER_CANCEL_ON_SET", + "timerfd_create", + "timerfd_gettime", + "timerfd_gettime_ns", + "timerfd_settime", + "timerfd_settime_ns", + ] +if sys.platform == "linux" and sys.version_info >= (3, 12): + __all__ += [ + "CLONE_FILES", + "CLONE_FS", + "CLONE_NEWCGROUP", + "CLONE_NEWIPC", + "CLONE_NEWNET", + "CLONE_NEWNS", + "CLONE_NEWPID", + "CLONE_NEWUSER", + "CLONE_NEWUTS", + "CLONE_SIGHAND", + "CLONE_SYSVSEM", + "CLONE_THREAD", + "CLONE_VM", + "setns", + "unshare", + ] +if sys.platform == "linux" and sys.version_info >= (3, 10): + __all__ += [ + "EFD_CLOEXEC", + "EFD_NONBLOCK", + "EFD_SEMAPHORE", + "RWF_APPEND", + "SPLICE_F_MORE", + "SPLICE_F_MOVE", + "SPLICE_F_NONBLOCK", + "eventfd", + "eventfd_read", + "eventfd_write", + "splice", + ] +if sys.platform == "linux" and sys.version_info >= (3, 9): + __all__ += ["P_PIDFD", "pidfd_open"] +if sys.platform == "win32": + __all__ += [ + "O_BINARY", + "O_NOINHERIT", + "O_RANDOM", + "O_SEQUENTIAL", + "O_SHORT_LIVED", + "O_TEMPORARY", + "O_TEXT", + "P_DETACH", + "P_OVERLAY", + "get_handle_inheritable", + "set_handle_inheritable", + "startfile", + ] +if sys.platform == "win32" and sys.version_info >= (3, 12): + __all__ += ["listdrives", "listmounts", "listvolumes"] +if sys.platform != "win32": + __all__ += [ + "CLD_CONTINUED", + "CLD_DUMPED", + "CLD_EXITED", + "CLD_TRAPPED", + "EX_CANTCREAT", + "EX_CONFIG", + "EX_DATAERR", + "EX_IOERR", + "EX_NOHOST", + "EX_NOINPUT", + "EX_NOPERM", + "EX_NOUSER", + "EX_OSERR", + "EX_OSFILE", + "EX_PROTOCOL", + "EX_SOFTWARE", + "EX_TEMPFAIL", + "EX_UNAVAILABLE", + "EX_USAGE", + "F_LOCK", + "F_TEST", + "F_TLOCK", + "F_ULOCK", + "NGROUPS_MAX", + "O_ACCMODE", + "O_ASYNC", + "O_CLOEXEC", + "O_DIRECTORY", + "O_DSYNC", + "O_NDELAY", + "O_NOCTTY", + "O_NOFOLLOW", + "O_NONBLOCK", + "O_SYNC", + "POSIX_SPAWN_CLOSE", + "POSIX_SPAWN_DUP2", + "POSIX_SPAWN_OPEN", + "PRIO_PGRP", + "PRIO_PROCESS", + "PRIO_USER", + "P_ALL", + "P_PGID", + "P_PID", + "RTLD_GLOBAL", + "RTLD_LAZY", + "RTLD_LOCAL", + "RTLD_NODELETE", + "RTLD_NOLOAD", + "RTLD_NOW", + "SCHED_FIFO", + "SCHED_OTHER", + "SCHED_RR", + "SEEK_DATA", + "SEEK_HOLE", + "ST_NOSUID", + "ST_RDONLY", + "WCONTINUED", + "WCOREDUMP", + "WEXITED", + "WEXITSTATUS", + "WIFCONTINUED", + "WIFEXITED", + "WIFSIGNALED", + "WIFSTOPPED", + "WNOHANG", + "WNOWAIT", + "WSTOPPED", + "WSTOPSIG", + "WTERMSIG", + "WUNTRACED", + "chown", + "chroot", + "confstr", + "confstr_names", + "ctermid", + "environb", + "fchdir", + "fchown", + "fork", + "forkpty", + "fpathconf", + "fstatvfs", + "fwalk", + "getegid", + "getenvb", + "geteuid", + "getgid", + "getgrouplist", + "getgroups", + "getloadavg", + "getpgid", + "getpgrp", + "getpriority", + "getsid", + "getuid", + "initgroups", + "killpg", + "lchown", + "lockf", + "major", + "makedev", + "minor", + "mkfifo", + "mknod", + "nice", + "openpty", + "pathconf", + "pathconf_names", + "posix_spawn", + "posix_spawnp", + "pread", + "preadv", + "pwrite", + "pwritev", + "readv", + "register_at_fork", + "sched_get_priority_max", + "sched_get_priority_min", + "sched_yield", + "sendfile", + "setegid", + "seteuid", + "setgid", + "setgroups", + "setpgid", + "setpgrp", + "setpriority", + "setregid", + "setreuid", + "setsid", + "setuid", + "spawnlp", + "spawnlpe", + "spawnvp", + "spawnvpe", + "statvfs", + "sync", + "sysconf", + "sysconf_names", + "tcgetpgrp", + "tcsetpgrp", + "ttyname", + "uname", + "wait", + "wait3", + "wait4", + "writev", + ] +if sys.platform != "win32" and sys.version_info >= (3, 13): + __all__ += ["grantpt", "posix_openpt", "ptsname", "unlockpt"] +if sys.platform != "win32" and sys.version_info >= (3, 11): + __all__ += ["login_tty"] +if sys.platform != "win32" and sys.version_info >= (3, 10): + __all__ += ["O_FSYNC"] +if sys.platform != "win32" and sys.version_info >= (3, 9): + __all__ += ["CLD_KILLED", "CLD_STOPPED"] +if sys.platform != "darwin" and sys.platform != "win32": + __all__ += [ + "POSIX_FADV_DONTNEED", + "POSIX_FADV_NOREUSE", + "POSIX_FADV_NORMAL", + "POSIX_FADV_RANDOM", + "POSIX_FADV_SEQUENTIAL", + "POSIX_FADV_WILLNEED", + "RWF_DSYNC", + "RWF_HIPRI", + "RWF_NOWAIT", + "RWF_SYNC", + "ST_APPEND", + "ST_MANDLOCK", + "ST_NOATIME", + "ST_NODEV", + "ST_NODIRATIME", + "ST_NOEXEC", + "ST_RELATIME", + "ST_SYNCHRONOUS", + "ST_WRITE", + "fdatasync", + "getresgid", + "getresuid", + "pipe2", + "posix_fadvise", + "posix_fallocate", + "sched_getaffinity", + "sched_getparam", + "sched_getscheduler", + "sched_param", + "sched_rr_get_interval", + "sched_setaffinity", + "sched_setparam", + "sched_setscheduler", + "setresgid", + "setresuid", + ] +if sys.platform != "linux" and sys.platform != "win32": + __all__ += ["O_EXLOCK", "O_SHLOCK", "chflags", "lchflags"] +if sys.platform != "linux" and sys.platform != "win32" and sys.version_info >= (3, 13): + __all__ += ["O_EXEC", "O_SEARCH"] +if sys.platform != "darwin" or sys.version_info >= (3, 13): + if sys.platform != "win32": + __all__ += ["waitid", "waitid_result"] +if sys.platform != "win32" or sys.version_info >= (3, 13): + __all__ += ["fchmod"] + if sys.platform != "linux": + __all__ += ["lchmod"] +if sys.platform != "win32" or sys.version_info >= (3, 12): + __all__ += ["get_blocking", "set_blocking"] +if sys.platform != "win32" or sys.version_info >= (3, 11): + __all__ += ["EX_OK"] +if sys.platform != "win32" or sys.version_info >= (3, 9): + __all__ += ["unsetenv"] + +# This unnecessary alias is to work around various errors +path = _path + +_T = TypeVar("_T") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") + +# ----- os variables ----- + +error = OSError + +supports_bytes_environ: bool + +supports_dir_fd: set[Callable[..., Any]] +supports_fd: set[Callable[..., Any]] +supports_effective_ids: set[Callable[..., Any]] +supports_follow_symlinks: set[Callable[..., Any]] + +if sys.platform != "win32": + # Unix only + PRIO_PROCESS: int + PRIO_PGRP: int + PRIO_USER: int + + F_LOCK: int + F_TLOCK: int + F_ULOCK: int + F_TEST: int + + if sys.platform != "darwin": + POSIX_FADV_NORMAL: int + POSIX_FADV_SEQUENTIAL: int + POSIX_FADV_RANDOM: int + POSIX_FADV_NOREUSE: int + POSIX_FADV_WILLNEED: int + POSIX_FADV_DONTNEED: int + + if sys.platform != "linux" and sys.platform != "darwin": + # In the os-module docs, these are marked as being available + # on "Unix, not Emscripten, not WASI." + # However, in the source code, a comment indicates they're "FreeBSD constants". + # sys.platform could have one of many values on a FreeBSD Python build, + # so the sys-module docs recommend doing `if sys.platform.startswith('freebsd')` + # to detect FreeBSD builds. Unfortunately that would be too dynamic + # for type checkers, however. + SF_NODISKIO: int + SF_MNOWAIT: int + SF_SYNC: int + + if sys.version_info >= (3, 11): + SF_NOCACHE: int + + if sys.platform == "linux": + XATTR_SIZE_MAX: int + XATTR_CREATE: int + XATTR_REPLACE: int + + P_PID: int + P_PGID: int + P_ALL: int + + if sys.platform == "linux" and sys.version_info >= (3, 9): + P_PIDFD: int + + WEXITED: int + WSTOPPED: int + WNOWAIT: int + + CLD_EXITED: int + CLD_DUMPED: int + CLD_TRAPPED: int + CLD_CONTINUED: int + + if sys.version_info >= (3, 9): + CLD_KILLED: int + CLD_STOPPED: int + + SCHED_OTHER: int + SCHED_FIFO: int + SCHED_RR: int + if sys.platform != "darwin" and sys.platform != "linux": + SCHED_SPORADIC: int + +if sys.platform == "linux": + SCHED_BATCH: int + SCHED_IDLE: int + SCHED_RESET_ON_FORK: int + +if sys.platform != "win32": + RTLD_LAZY: int + RTLD_NOW: int + RTLD_GLOBAL: int + RTLD_LOCAL: int + RTLD_NODELETE: int + RTLD_NOLOAD: int + +if sys.platform == "linux": + RTLD_DEEPBIND: int + GRND_NONBLOCK: int + GRND_RANDOM: int + +if sys.platform == "darwin" and sys.version_info >= (3, 12): + PRIO_DARWIN_BG: int + PRIO_DARWIN_NONUI: int + PRIO_DARWIN_PROCESS: int + PRIO_DARWIN_THREAD: int + +SEEK_SET: int +SEEK_CUR: int +SEEK_END: int +if sys.platform != "win32": + SEEK_DATA: int + SEEK_HOLE: int + +O_RDONLY: int +O_WRONLY: int +O_RDWR: int +O_APPEND: int +O_CREAT: int +O_EXCL: int +O_TRUNC: int +if sys.platform == "win32": + O_BINARY: int + O_NOINHERIT: int + O_SHORT_LIVED: int + O_TEMPORARY: int + O_RANDOM: int + O_SEQUENTIAL: int + O_TEXT: int + +if sys.platform != "win32": + O_DSYNC: int + O_SYNC: int + O_NDELAY: int + O_NONBLOCK: int + O_NOCTTY: int + O_CLOEXEC: int + O_ASYNC: int # Gnu extension if in C library + O_DIRECTORY: int # Gnu extension if in C library + O_NOFOLLOW: int # Gnu extension if in C library + O_ACCMODE: int # TODO: when does this exist? + +if sys.platform == "linux": + O_RSYNC: int + O_DIRECT: int # Gnu extension if in C library + O_NOATIME: int # Gnu extension if in C library + O_PATH: int # Gnu extension if in C library + O_TMPFILE: int # Gnu extension if in C library + O_LARGEFILE: int # Gnu extension if in C library + +if sys.platform != "linux" and sys.platform != "win32": + O_SHLOCK: int + O_EXLOCK: int + +if sys.platform == "darwin" and sys.version_info >= (3, 10): + O_EVTONLY: int + O_NOFOLLOW_ANY: int + O_SYMLINK: int + +if sys.platform != "win32" and sys.version_info >= (3, 10): + O_FSYNC: int + +if sys.platform != "linux" and sys.platform != "win32" and sys.version_info >= (3, 13): + O_EXEC: int + O_SEARCH: int + +if sys.platform != "win32" and sys.platform != "darwin": + # posix, but apparently missing on macos + ST_APPEND: int + ST_MANDLOCK: int + ST_NOATIME: int + ST_NODEV: int + ST_NODIRATIME: int + ST_NOEXEC: int + ST_RELATIME: int + ST_SYNCHRONOUS: int + ST_WRITE: int + +if sys.platform != "win32": + NGROUPS_MAX: int + ST_NOSUID: int + ST_RDONLY: int + +curdir: str +pardir: str +sep: str +if sys.platform == "win32": + altsep: str +else: + altsep: str | None +extsep: str +pathsep: str +defpath: str +linesep: str +devnull: str +name: str + +F_OK: int +R_OK: int +W_OK: int +X_OK: int + +_EnvironCodeFunc: TypeAlias = Callable[[AnyStr], AnyStr] + +class _Environ(MutableMapping[AnyStr, AnyStr], Generic[AnyStr]): + encodekey: _EnvironCodeFunc[AnyStr] + decodekey: _EnvironCodeFunc[AnyStr] + encodevalue: _EnvironCodeFunc[AnyStr] + decodevalue: _EnvironCodeFunc[AnyStr] + if sys.version_info >= (3, 9): + def __init__( + self, + data: MutableMapping[AnyStr, AnyStr], + encodekey: _EnvironCodeFunc[AnyStr], + decodekey: _EnvironCodeFunc[AnyStr], + encodevalue: _EnvironCodeFunc[AnyStr], + decodevalue: _EnvironCodeFunc[AnyStr], + ) -> None: ... + else: + putenv: Callable[[AnyStr, AnyStr], object] + unsetenv: Callable[[AnyStr, AnyStr], object] + def __init__( + self, + data: MutableMapping[AnyStr, AnyStr], + encodekey: _EnvironCodeFunc[AnyStr], + decodekey: _EnvironCodeFunc[AnyStr], + encodevalue: _EnvironCodeFunc[AnyStr], + decodevalue: _EnvironCodeFunc[AnyStr], + putenv: Callable[[AnyStr, AnyStr], object], + unsetenv: Callable[[AnyStr, AnyStr], object], + ) -> None: ... + + def setdefault(self, key: AnyStr, value: AnyStr) -> AnyStr: ... + def copy(self) -> dict[AnyStr, AnyStr]: ... + def __delitem__(self, key: AnyStr) -> None: ... + def __getitem__(self, key: AnyStr) -> AnyStr: ... + def __setitem__(self, key: AnyStr, value: AnyStr) -> None: ... + def __iter__(self) -> Iterator[AnyStr]: ... + def __len__(self) -> int: ... + if sys.version_info >= (3, 9): + def __or__(self, other: Mapping[_T1, _T2]) -> dict[AnyStr | _T1, AnyStr | _T2]: ... + def __ror__(self, other: Mapping[_T1, _T2]) -> dict[AnyStr | _T1, AnyStr | _T2]: ... + # We use @overload instead of a Union for reasons similar to those given for + # overloading MutableMapping.update in stdlib/typing.pyi + # The type: ignore is needed due to incompatible __or__/__ior__ signatures + @overload # type: ignore[misc] + def __ior__(self, other: Mapping[AnyStr, AnyStr]) -> Self: ... + @overload + def __ior__(self, other: Iterable[tuple[AnyStr, AnyStr]]) -> Self: ... + +environ: _Environ[str] +if sys.platform != "win32": + environb: _Environ[bytes] + +if sys.version_info >= (3, 11) or sys.platform != "win32": + EX_OK: int + +if sys.platform != "win32": + confstr_names: dict[str, int] + pathconf_names: dict[str, int] + sysconf_names: dict[str, int] + + EX_USAGE: int + EX_DATAERR: int + EX_NOINPUT: int + EX_NOUSER: int + EX_NOHOST: int + EX_UNAVAILABLE: int + EX_SOFTWARE: int + EX_OSERR: int + EX_OSFILE: int + EX_CANTCREAT: int + EX_IOERR: int + EX_TEMPFAIL: int + EX_PROTOCOL: int + EX_NOPERM: int + EX_CONFIG: int + +# Exists on some Unix platforms, e.g. Solaris. +if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + EX_NOTFOUND: int + +P_NOWAIT: int +P_NOWAITO: int +P_WAIT: int +if sys.platform == "win32": + P_DETACH: int + P_OVERLAY: int + +# wait()/waitpid() options +if sys.platform != "win32": + WNOHANG: int # Unix only + WCONTINUED: int # some Unix systems + WUNTRACED: int # Unix only + +TMP_MAX: int # Undocumented, but used by tempfile + +# ----- os classes (structures) ----- +@final +class stat_result(structseq[float], tuple[int, int, int, int, int, int, int, float, float, float]): + # The constructor of this class takes an iterable of variable length (though it must be at least 10). + # + # However, this class behaves like a tuple of 10 elements, + # no matter how long the iterable supplied to the constructor is. + # https://github.com/python/typeshed/pull/6560#discussion_r767162532 + # + # The 10 elements always present are st_mode, st_ino, st_dev, st_nlink, + # st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime. + # + # More items may be added at the end by some implementations. + if sys.version_info >= (3, 10): + __match_args__: Final = ("st_mode", "st_ino", "st_dev", "st_nlink", "st_uid", "st_gid", "st_size") + + @property + def st_mode(self) -> int: ... # protection bits, + @property + def st_ino(self) -> int: ... # inode number, + @property + def st_dev(self) -> int: ... # device, + @property + def st_nlink(self) -> int: ... # number of hard links, + @property + def st_uid(self) -> int: ... # user id of owner, + @property + def st_gid(self) -> int: ... # group id of owner, + @property + def st_size(self) -> int: ... # size of file, in bytes, + @property + def st_atime(self) -> float: ... # time of most recent access, + @property + def st_mtime(self) -> float: ... # time of most recent content modification, + # platform dependent (time of most recent metadata change on Unix, or the time of creation on Windows) + if sys.version_info >= (3, 12) and sys.platform == "win32": + @property + @deprecated( + """\ +Use st_birthtime instead to retrieve the file creation time. \ +In the future, this property will contain the last metadata change time.""" + ) + def st_ctime(self) -> float: ... + else: + @property + def st_ctime(self) -> float: ... + + @property + def st_atime_ns(self) -> int: ... # time of most recent access, in nanoseconds + @property + def st_mtime_ns(self) -> int: ... # time of most recent content modification in nanoseconds + # platform dependent (time of most recent metadata change on Unix, or the time of creation on Windows) in nanoseconds + @property + def st_ctime_ns(self) -> int: ... + if sys.platform == "win32": + @property + def st_file_attributes(self) -> int: ... + @property + def st_reparse_tag(self) -> int: ... + if sys.version_info >= (3, 12): + @property + def st_birthtime(self) -> float: ... # time of file creation in seconds + @property + def st_birthtime_ns(self) -> int: ... # time of file creation in nanoseconds + else: + @property + def st_blocks(self) -> int: ... # number of blocks allocated for file + @property + def st_blksize(self) -> int: ... # filesystem blocksize + @property + def st_rdev(self) -> int: ... # type of device if an inode device + if sys.platform != "linux": + # These properties are available on MacOS, but not Ubuntu. + # On other Unix systems (such as FreeBSD), the following attributes may be + # available (but may be only filled out if root tries to use them): + @property + def st_gen(self) -> int: ... # file generation number + @property + def st_birthtime(self) -> float: ... # time of file creation in seconds + if sys.platform == "darwin": + @property + def st_flags(self) -> int: ... # user defined flags for file + # Attributes documented as sometimes appearing, but deliberately omitted from the stub: `st_creator`, `st_rsize`, `st_type`. + # See https://github.com/python/typeshed/pull/6560#issuecomment-991253327 + +# mypy and pyright object to this being both ABC and Protocol. +# At runtime it inherits from ABC and is not a Protocol, but it will be +# on the allowlist for use as a Protocol starting in 3.14. +@runtime_checkable +class PathLike(ABC, Protocol[AnyStr_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + @abstractmethod + def __fspath__(self) -> AnyStr_co: ... + +@overload +def listdir(path: StrPath | None = None) -> list[str]: ... +@overload +def listdir(path: BytesPath) -> list[bytes]: ... +@overload +def listdir(path: int) -> list[str]: ... +@final +class DirEntry(Generic[AnyStr]): + # This is what the scandir iterator yields + # The constructor is hidden + + @property + def name(self) -> AnyStr: ... + @property + def path(self) -> AnyStr: ... + def inode(self) -> int: ... + def is_dir(self, *, follow_symlinks: bool = True) -> bool: ... + def is_file(self, *, follow_symlinks: bool = True) -> bool: ... + def is_symlink(self) -> bool: ... + def stat(self, *, follow_symlinks: bool = True) -> stat_result: ... + def __fspath__(self) -> AnyStr: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + if sys.version_info >= (3, 12): + def is_junction(self) -> bool: ... + +@final +class statvfs_result(structseq[int], tuple[int, int, int, int, int, int, int, int, int, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ( + "f_bsize", + "f_frsize", + "f_blocks", + "f_bfree", + "f_bavail", + "f_files", + "f_ffree", + "f_favail", + "f_flag", + "f_namemax", + ) + + @property + def f_bsize(self) -> int: ... + @property + def f_frsize(self) -> int: ... + @property + def f_blocks(self) -> int: ... + @property + def f_bfree(self) -> int: ... + @property + def f_bavail(self) -> int: ... + @property + def f_files(self) -> int: ... + @property + def f_ffree(self) -> int: ... + @property + def f_favail(self) -> int: ... + @property + def f_flag(self) -> int: ... + @property + def f_namemax(self) -> int: ... + @property + def f_fsid(self) -> int: ... + +# ----- os function stubs ----- +def fsencode(filename: StrOrBytesPath) -> bytes: ... +def fsdecode(filename: StrOrBytesPath) -> str: ... +@overload +def fspath(path: str) -> str: ... +@overload +def fspath(path: bytes) -> bytes: ... +@overload +def fspath(path: PathLike[AnyStr]) -> AnyStr: ... +def get_exec_path(env: Mapping[str, str] | None = None) -> list[str]: ... +def getlogin() -> str: ... +def getpid() -> int: ... +def getppid() -> int: ... +def strerror(code: int, /) -> str: ... +def umask(mask: int, /) -> int: ... +@final +class uname_result(structseq[str], tuple[str, str, str, str, str]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("sysname", "nodename", "release", "version", "machine") + + @property + def sysname(self) -> str: ... + @property + def nodename(self) -> str: ... + @property + def release(self) -> str: ... + @property + def version(self) -> str: ... + @property + def machine(self) -> str: ... + +if sys.platform != "win32": + def ctermid() -> str: ... + def getegid() -> int: ... + def geteuid() -> int: ... + def getgid() -> int: ... + def getgrouplist(user: str, group: int, /) -> list[int]: ... + def getgroups() -> list[int]: ... # Unix only, behaves differently on Mac + def initgroups(username: str, gid: int, /) -> None: ... + def getpgid(pid: int) -> int: ... + def getpgrp() -> int: ... + def getpriority(which: int, who: int) -> int: ... + def setpriority(which: int, who: int, priority: int) -> None: ... + if sys.platform != "darwin": + def getresuid() -> tuple[int, int, int]: ... + def getresgid() -> tuple[int, int, int]: ... + + def getuid() -> int: ... + def setegid(egid: int, /) -> None: ... + def seteuid(euid: int, /) -> None: ... + def setgid(gid: int, /) -> None: ... + def setgroups(groups: Sequence[int], /) -> None: ... + def setpgrp() -> None: ... + def setpgid(pid: int, pgrp: int, /) -> None: ... + def setregid(rgid: int, egid: int, /) -> None: ... + if sys.platform != "darwin": + def setresgid(rgid: int, egid: int, sgid: int, /) -> None: ... + def setresuid(ruid: int, euid: int, suid: int, /) -> None: ... + + def setreuid(ruid: int, euid: int, /) -> None: ... + def getsid(pid: int, /) -> int: ... + def setsid() -> None: ... + def setuid(uid: int, /) -> None: ... + def uname() -> uname_result: ... + +@overload +def getenv(key: str) -> str | None: ... +@overload +def getenv(key: str, default: _T) -> str | _T: ... + +if sys.platform != "win32": + @overload + def getenvb(key: bytes) -> bytes | None: ... + @overload + def getenvb(key: bytes, default: _T) -> bytes | _T: ... + def putenv(name: StrOrBytesPath, value: StrOrBytesPath, /) -> None: ... + def unsetenv(name: StrOrBytesPath, /) -> None: ... + +else: + def putenv(name: str, value: str, /) -> None: ... + + if sys.version_info >= (3, 9): + def unsetenv(name: str, /) -> None: ... + +_Opener: TypeAlias = Callable[[str, int], int] + +@overload +def fdopen( + fd: int, + mode: OpenTextMode = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = ..., + newline: str | None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> TextIOWrapper: ... +@overload +def fdopen( + fd: int, + mode: OpenBinaryMode, + buffering: Literal[0], + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = ..., + opener: _Opener | None = ..., +) -> FileIO: ... +@overload +def fdopen( + fd: int, + mode: OpenBinaryModeUpdating, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = ..., + opener: _Opener | None = ..., +) -> BufferedRandom: ... +@overload +def fdopen( + fd: int, + mode: OpenBinaryModeWriting, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = ..., + opener: _Opener | None = ..., +) -> BufferedWriter: ... +@overload +def fdopen( + fd: int, + mode: OpenBinaryModeReading, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = ..., + opener: _Opener | None = ..., +) -> BufferedReader: ... +@overload +def fdopen( + fd: int, + mode: OpenBinaryMode, + buffering: int = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = ..., + opener: _Opener | None = ..., +) -> BinaryIO: ... +@overload +def fdopen( + fd: int, + mode: str, + buffering: int = -1, + encoding: str | None = None, + errors: str | None = ..., + newline: str | None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> IO[Any]: ... +def close(fd: int) -> None: ... +def closerange(fd_low: int, fd_high: int, /) -> None: ... +def device_encoding(fd: int) -> str | None: ... +def dup(fd: int, /) -> int: ... +def dup2(fd: int, fd2: int, inheritable: bool = True) -> int: ... +def fstat(fd: int) -> stat_result: ... +def ftruncate(fd: int, length: int, /) -> None: ... +def fsync(fd: FileDescriptorLike) -> None: ... +def isatty(fd: int, /) -> bool: ... + +if sys.platform != "win32" and sys.version_info >= (3, 11): + def login_tty(fd: int, /) -> None: ... + +if sys.version_info >= (3, 11): + def lseek(fd: int, position: int, whence: int, /) -> int: ... + +else: + def lseek(fd: int, position: int, how: int, /) -> int: ... + +def open(path: StrOrBytesPath, flags: int, mode: int = 0o777, *, dir_fd: int | None = None) -> int: ... +def pipe() -> tuple[int, int]: ... +def read(fd: int, length: int, /) -> bytes: ... + +if sys.version_info >= (3, 12) or sys.platform != "win32": + def get_blocking(fd: int, /) -> bool: ... + def set_blocking(fd: int, blocking: bool, /) -> None: ... + +if sys.platform != "win32": + def fchown(fd: int, uid: int, gid: int) -> None: ... + def fpathconf(fd: int, name: str | int, /) -> int: ... + def fstatvfs(fd: int, /) -> statvfs_result: ... + def lockf(fd: int, command: int, length: int, /) -> None: ... + def openpty() -> tuple[int, int]: ... # some flavors of Unix + if sys.platform != "darwin": + def fdatasync(fd: FileDescriptorLike) -> None: ... + def pipe2(flags: int, /) -> tuple[int, int]: ... # some flavors of Unix + def posix_fallocate(fd: int, offset: int, length: int, /) -> None: ... + def posix_fadvise(fd: int, offset: int, length: int, advice: int, /) -> None: ... + + def pread(fd: int, length: int, offset: int, /) -> bytes: ... + def pwrite(fd: int, buffer: ReadableBuffer, offset: int, /) -> int: ... + # In CI, stubtest sometimes reports that these are available on MacOS, sometimes not + def preadv(fd: int, buffers: SupportsLenAndGetItem[WriteableBuffer], offset: int, flags: int = 0, /) -> int: ... + def pwritev(fd: int, buffers: SupportsLenAndGetItem[ReadableBuffer], offset: int, flags: int = 0, /) -> int: ... + if sys.platform != "darwin": + if sys.version_info >= (3, 10): + RWF_APPEND: int # docs say available on 3.7+, stubtest says otherwise + RWF_DSYNC: int + RWF_SYNC: int + RWF_HIPRI: int + RWF_NOWAIT: int + + if sys.platform == "linux": + def sendfile(out_fd: FileDescriptor, in_fd: FileDescriptor, offset: int | None, count: int) -> int: ... + else: + def sendfile( + out_fd: FileDescriptor, + in_fd: FileDescriptor, + offset: int, + count: int, + headers: Sequence[ReadableBuffer] = ..., + trailers: Sequence[ReadableBuffer] = ..., + flags: int = 0, + ) -> int: ... # FreeBSD and Mac OS X only + + def readv(fd: int, buffers: SupportsLenAndGetItem[WriteableBuffer], /) -> int: ... + def writev(fd: int, buffers: SupportsLenAndGetItem[ReadableBuffer], /) -> int: ... + +@final +class terminal_size(structseq[int], tuple[int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("columns", "lines") + + @property + def columns(self) -> int: ... + @property + def lines(self) -> int: ... + +def get_terminal_size(fd: int = ..., /) -> terminal_size: ... +def get_inheritable(fd: int, /) -> bool: ... +def set_inheritable(fd: int, inheritable: bool, /) -> None: ... + +if sys.platform == "win32": + def get_handle_inheritable(handle: int, /) -> bool: ... + def set_handle_inheritable(handle: int, inheritable: bool, /) -> None: ... + +if sys.platform != "win32": + # Unix only + def tcgetpgrp(fd: int, /) -> int: ... + def tcsetpgrp(fd: int, pgid: int, /) -> None: ... + def ttyname(fd: int, /) -> str: ... + +def write(fd: int, data: ReadableBuffer, /) -> int: ... +def access( + path: FileDescriptorOrPath, mode: int, *, dir_fd: int | None = None, effective_ids: bool = False, follow_symlinks: bool = True +) -> bool: ... +def chdir(path: FileDescriptorOrPath) -> None: ... + +if sys.platform != "win32": + def fchdir(fd: FileDescriptorLike) -> None: ... + +def getcwd() -> str: ... +def getcwdb() -> bytes: ... +def chmod(path: FileDescriptorOrPath, mode: int, *, dir_fd: int | None = None, follow_symlinks: bool = ...) -> None: ... + +if sys.platform != "win32" and sys.platform != "linux": + def chflags(path: StrOrBytesPath, flags: int, follow_symlinks: bool = True) -> None: ... # some flavors of Unix + def lchflags(path: StrOrBytesPath, flags: int) -> None: ... + +if sys.platform != "win32": + def chroot(path: StrOrBytesPath) -> None: ... + def chown( + path: FileDescriptorOrPath, uid: int, gid: int, *, dir_fd: int | None = None, follow_symlinks: bool = True + ) -> None: ... + def lchown(path: StrOrBytesPath, uid: int, gid: int) -> None: ... + +def link( + src: StrOrBytesPath, + dst: StrOrBytesPath, + *, + src_dir_fd: int | None = None, + dst_dir_fd: int | None = None, + follow_symlinks: bool = True, +) -> None: ... +def lstat(path: StrOrBytesPath, *, dir_fd: int | None = None) -> stat_result: ... +def mkdir(path: StrOrBytesPath, mode: int = 0o777, *, dir_fd: int | None = None) -> None: ... + +if sys.platform != "win32": + def mkfifo(path: StrOrBytesPath, mode: int = 0o666, *, dir_fd: int | None = None) -> None: ... # Unix only + +def makedirs(name: StrOrBytesPath, mode: int = 0o777, exist_ok: bool = False) -> None: ... + +if sys.platform != "win32": + def mknod(path: StrOrBytesPath, mode: int = 0o600, device: int = 0, *, dir_fd: int | None = None) -> None: ... + def major(device: int, /) -> int: ... + def minor(device: int, /) -> int: ... + def makedev(major: int, minor: int, /) -> int: ... + def pathconf(path: FileDescriptorOrPath, name: str | int) -> int: ... # Unix only + +def readlink(path: GenericPath[AnyStr], *, dir_fd: int | None = None) -> AnyStr: ... +def remove(path: StrOrBytesPath, *, dir_fd: int | None = None) -> None: ... +def removedirs(name: StrOrBytesPath) -> None: ... +def rename(src: StrOrBytesPath, dst: StrOrBytesPath, *, src_dir_fd: int | None = None, dst_dir_fd: int | None = None) -> None: ... +def renames(old: StrOrBytesPath, new: StrOrBytesPath) -> None: ... +def replace( + src: StrOrBytesPath, dst: StrOrBytesPath, *, src_dir_fd: int | None = None, dst_dir_fd: int | None = None +) -> None: ... +def rmdir(path: StrOrBytesPath, *, dir_fd: int | None = None) -> None: ... +@final +class _ScandirIterator(Generic[AnyStr]): + def __del__(self) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> DirEntry[AnyStr]: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def close(self) -> None: ... + +@overload +def scandir(path: None = None) -> _ScandirIterator[str]: ... +@overload +def scandir(path: int) -> _ScandirIterator[str]: ... +@overload +def scandir(path: GenericPath[AnyStr]) -> _ScandirIterator[AnyStr]: ... +def stat(path: FileDescriptorOrPath, *, dir_fd: int | None = None, follow_symlinks: bool = True) -> stat_result: ... + +if sys.platform != "win32": + def statvfs(path: FileDescriptorOrPath) -> statvfs_result: ... # Unix only + +def symlink( + src: StrOrBytesPath, dst: StrOrBytesPath, target_is_directory: bool = False, *, dir_fd: int | None = None +) -> None: ... + +if sys.platform != "win32": + def sync() -> None: ... # Unix only + +def truncate(path: FileDescriptorOrPath, length: int) -> None: ... # Unix only up to version 3.4 +def unlink(path: StrOrBytesPath, *, dir_fd: int | None = None) -> None: ... +def utime( + path: FileDescriptorOrPath, + times: tuple[int, int] | tuple[float, float] | None = None, + *, + ns: tuple[int, int] = ..., + dir_fd: int | None = None, + follow_symlinks: bool = True, +) -> None: ... + +_OnError: TypeAlias = Callable[[OSError], object] + +def walk( + top: GenericPath[AnyStr], topdown: bool = True, onerror: _OnError | None = None, followlinks: bool = False +) -> Iterator[tuple[AnyStr, list[AnyStr], list[AnyStr]]]: ... + +if sys.platform != "win32": + @overload + def fwalk( + top: StrPath = ".", + topdown: bool = True, + onerror: _OnError | None = None, + *, + follow_symlinks: bool = False, + dir_fd: int | None = None, + ) -> Iterator[tuple[str, list[str], list[str], int]]: ... + @overload + def fwalk( + top: BytesPath, + topdown: bool = True, + onerror: _OnError | None = None, + *, + follow_symlinks: bool = False, + dir_fd: int | None = None, + ) -> Iterator[tuple[bytes, list[bytes], list[bytes], int]]: ... + if sys.platform == "linux": + def getxattr(path: FileDescriptorOrPath, attribute: StrOrBytesPath, *, follow_symlinks: bool = True) -> bytes: ... + def listxattr(path: FileDescriptorOrPath | None = None, *, follow_symlinks: bool = True) -> list[str]: ... + def removexattr(path: FileDescriptorOrPath, attribute: StrOrBytesPath, *, follow_symlinks: bool = True) -> None: ... + def setxattr( + path: FileDescriptorOrPath, + attribute: StrOrBytesPath, + value: ReadableBuffer, + flags: int = 0, + *, + follow_symlinks: bool = True, + ) -> None: ... + +def abort() -> NoReturn: ... + +# These are defined as execl(file, *args) but the first *arg is mandatory. +def execl(file: StrOrBytesPath, *args: Unpack[tuple[StrOrBytesPath, Unpack[tuple[StrOrBytesPath, ...]]]]) -> NoReturn: ... +def execlp(file: StrOrBytesPath, *args: Unpack[tuple[StrOrBytesPath, Unpack[tuple[StrOrBytesPath, ...]]]]) -> NoReturn: ... + +# These are: execle(file, *args, env) but env is pulled from the last element of the args. +def execle( + file: StrOrBytesPath, *args: Unpack[tuple[StrOrBytesPath, Unpack[tuple[StrOrBytesPath, ...]], _ExecEnv]] +) -> NoReturn: ... +def execlpe( + file: StrOrBytesPath, *args: Unpack[tuple[StrOrBytesPath, Unpack[tuple[StrOrBytesPath, ...]], _ExecEnv]] +) -> NoReturn: ... + +# The docs say `args: tuple or list of strings` +# The implementation enforces tuple or list so we can't use Sequence. +# Not separating out PathLike[str] and PathLike[bytes] here because it doesn't make much difference +# in practice, and doing so would explode the number of combinations in this already long union. +# All these combinations are necessary due to list being invariant. +_ExecVArgs: TypeAlias = ( + tuple[StrOrBytesPath, ...] + | list[bytes] + | list[str] + | list[PathLike[Any]] + | list[bytes | str] + | list[bytes | PathLike[Any]] + | list[str | PathLike[Any]] + | list[bytes | str | PathLike[Any]] +) +# Depending on the OS, the keys and values are passed either to +# PyUnicode_FSDecoder (which accepts str | ReadableBuffer) or to +# PyUnicode_FSConverter (which accepts StrOrBytesPath). For simplicity, +# we limit to str | bytes. +_ExecEnv: TypeAlias = Mapping[bytes, bytes | str] | Mapping[str, bytes | str] + +def execv(path: StrOrBytesPath, argv: _ExecVArgs, /) -> NoReturn: ... +def execve(path: FileDescriptorOrPath, argv: _ExecVArgs, env: _ExecEnv) -> NoReturn: ... +def execvp(file: StrOrBytesPath, args: _ExecVArgs) -> NoReturn: ... +def execvpe(file: StrOrBytesPath, args: _ExecVArgs, env: _ExecEnv) -> NoReturn: ... +def _exit(status: int) -> NoReturn: ... +def kill(pid: int, signal: int, /) -> None: ... + +if sys.platform != "win32": + # Unix only + def fork() -> int: ... + def forkpty() -> tuple[int, int]: ... # some flavors of Unix + def killpg(pgid: int, signal: int, /) -> None: ... + def nice(increment: int, /) -> int: ... + if sys.platform != "darwin" and sys.platform != "linux": + def plock(op: int, /) -> None: ... + +class _wrap_close: + def __init__(self, stream: TextIOWrapper, proc: Popen[str]) -> None: ... + def close(self) -> int | None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def __iter__(self) -> Iterator[str]: ... + # Methods below here don't exist directly on the _wrap_close object, but + # are copied from the wrapped TextIOWrapper object via __getattr__. + # The full set of TextIOWrapper methods are technically available this way, + # but undocumented. Only a subset are currently included here. + def read(self, size: int | None = -1, /) -> str: ... + def readable(self) -> bool: ... + def readline(self, size: int = -1, /) -> str: ... + def readlines(self, hint: int = -1, /) -> list[str]: ... + def writable(self) -> bool: ... + def write(self, s: str, /) -> int: ... + def writelines(self, lines: Iterable[str], /) -> None: ... + +def popen(cmd: str, mode: str = "r", buffering: int = -1) -> _wrap_close: ... +def spawnl(mode: int, file: StrOrBytesPath, arg0: StrOrBytesPath, *args: StrOrBytesPath) -> int: ... +def spawnle(mode: int, file: StrOrBytesPath, arg0: StrOrBytesPath, *args: Any) -> int: ... # Imprecise sig + +if sys.platform != "win32": + def spawnv(mode: int, file: StrOrBytesPath, args: _ExecVArgs) -> int: ... + def spawnve(mode: int, file: StrOrBytesPath, args: _ExecVArgs, env: _ExecEnv) -> int: ... + +else: + def spawnv(mode: int, path: StrOrBytesPath, argv: _ExecVArgs, /) -> int: ... + def spawnve(mode: int, path: StrOrBytesPath, argv: _ExecVArgs, env: _ExecEnv, /) -> int: ... + +def system(command: StrOrBytesPath) -> int: ... +@final +class times_result(structseq[float], tuple[float, float, float, float, float]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("user", "system", "children_user", "children_system", "elapsed") + + @property + def user(self) -> float: ... + @property + def system(self) -> float: ... + @property + def children_user(self) -> float: ... + @property + def children_system(self) -> float: ... + @property + def elapsed(self) -> float: ... + +def times() -> times_result: ... +def waitpid(pid: int, options: int, /) -> tuple[int, int]: ... + +if sys.platform == "win32": + if sys.version_info >= (3, 10): + def startfile( + filepath: StrOrBytesPath, + operation: str = ..., + arguments: str = "", + cwd: StrOrBytesPath | None = None, + show_cmd: int = 1, + ) -> None: ... + else: + def startfile(filepath: StrOrBytesPath, operation: str = ...) -> None: ... + +else: + def spawnlp(mode: int, file: StrOrBytesPath, arg0: StrOrBytesPath, *args: StrOrBytesPath) -> int: ... + def spawnlpe(mode: int, file: StrOrBytesPath, arg0: StrOrBytesPath, *args: Any) -> int: ... # Imprecise signature + def spawnvp(mode: int, file: StrOrBytesPath, args: _ExecVArgs) -> int: ... + def spawnvpe(mode: int, file: StrOrBytesPath, args: _ExecVArgs, env: _ExecEnv) -> int: ... + def wait() -> tuple[int, int]: ... # Unix only + # Added to MacOS in 3.13 + if sys.platform != "darwin" or sys.version_info >= (3, 13): + @final + class waitid_result(structseq[int], tuple[int, int, int, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("si_pid", "si_uid", "si_signo", "si_status", "si_code") + + @property + def si_pid(self) -> int: ... + @property + def si_uid(self) -> int: ... + @property + def si_signo(self) -> int: ... + @property + def si_status(self) -> int: ... + @property + def si_code(self) -> int: ... + + def waitid(idtype: int, ident: int, options: int, /) -> waitid_result | None: ... + + from resource import struct_rusage + + def wait3(options: int) -> tuple[int, int, struct_rusage]: ... + def wait4(pid: int, options: int) -> tuple[int, int, struct_rusage]: ... + def WCOREDUMP(status: int, /) -> bool: ... + def WIFCONTINUED(status: int) -> bool: ... + def WIFSTOPPED(status: int) -> bool: ... + def WIFSIGNALED(status: int) -> bool: ... + def WIFEXITED(status: int) -> bool: ... + def WEXITSTATUS(status: int) -> int: ... + def WSTOPSIG(status: int) -> int: ... + def WTERMSIG(status: int) -> int: ... + def posix_spawn( + path: StrOrBytesPath, + argv: _ExecVArgs, + env: _ExecEnv, + /, + *, + file_actions: Sequence[tuple[Any, ...]] | None = ..., + setpgroup: int | None = ..., + resetids: bool = ..., + setsid: bool = ..., + setsigmask: Iterable[int] = ..., + setsigdef: Iterable[int] = ..., + scheduler: tuple[Any, sched_param] | None = ..., + ) -> int: ... + def posix_spawnp( + path: StrOrBytesPath, + argv: _ExecVArgs, + env: _ExecEnv, + /, + *, + file_actions: Sequence[tuple[Any, ...]] | None = ..., + setpgroup: int | None = ..., + resetids: bool = ..., + setsid: bool = ..., + setsigmask: Iterable[int] = ..., + setsigdef: Iterable[int] = ..., + scheduler: tuple[Any, sched_param] | None = ..., + ) -> int: ... + POSIX_SPAWN_OPEN: int + POSIX_SPAWN_CLOSE: int + POSIX_SPAWN_DUP2: int + +if sys.platform != "win32": + @final + class sched_param(structseq[int], tuple[int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("sched_priority",) + + def __new__(cls, sched_priority: int) -> Self: ... + @property + def sched_priority(self) -> int: ... + + def sched_get_priority_min(policy: int) -> int: ... # some flavors of Unix + def sched_get_priority_max(policy: int) -> int: ... # some flavors of Unix + def sched_yield() -> None: ... # some flavors of Unix + if sys.platform != "darwin": + def sched_setscheduler(pid: int, policy: int, param: sched_param, /) -> None: ... # some flavors of Unix + def sched_getscheduler(pid: int, /) -> int: ... # some flavors of Unix + def sched_rr_get_interval(pid: int, /) -> float: ... # some flavors of Unix + def sched_setparam(pid: int, param: sched_param, /) -> None: ... # some flavors of Unix + def sched_getparam(pid: int, /) -> sched_param: ... # some flavors of Unix + def sched_setaffinity(pid: int, mask: Iterable[int], /) -> None: ... # some flavors of Unix + def sched_getaffinity(pid: int, /) -> set[int]: ... # some flavors of Unix + +def cpu_count() -> int | None: ... + +if sys.version_info >= (3, 13): + # Documented to return `int | None`, but falls back to `len(sched_getaffinity(0))` when + # available. See https://github.com/python/cpython/blob/417c130/Lib/os.py#L1175-L1186. + if sys.platform != "win32" and sys.platform != "darwin": + def process_cpu_count() -> int: ... + else: + def process_cpu_count() -> int | None: ... + +if sys.platform != "win32": + # Unix only + def confstr(name: str | int, /) -> str | None: ... + def getloadavg() -> tuple[float, float, float]: ... + def sysconf(name: str | int, /) -> int: ... + +if sys.platform == "linux": + def getrandom(size: int, flags: int = 0) -> bytes: ... + +def urandom(size: int, /) -> bytes: ... + +if sys.platform != "win32": + def register_at_fork( + *, + before: Callable[..., Any] | None = ..., + after_in_parent: Callable[..., Any] | None = ..., + after_in_child: Callable[..., Any] | None = ..., + ) -> None: ... + +if sys.platform == "win32": + class _AddedDllDirectory: + path: str | None + def __init__(self, path: str | None, cookie: _T, remove_dll_directory: Callable[[_T], object]) -> None: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + + def add_dll_directory(path: str) -> _AddedDllDirectory: ... + +if sys.platform == "linux": + MFD_CLOEXEC: int + MFD_ALLOW_SEALING: int + MFD_HUGETLB: int + MFD_HUGE_SHIFT: int + MFD_HUGE_MASK: int + MFD_HUGE_64KB: int + MFD_HUGE_512KB: int + MFD_HUGE_1MB: int + MFD_HUGE_2MB: int + MFD_HUGE_8MB: int + MFD_HUGE_16MB: int + MFD_HUGE_32MB: int + MFD_HUGE_256MB: int + MFD_HUGE_512MB: int + MFD_HUGE_1GB: int + MFD_HUGE_2GB: int + MFD_HUGE_16GB: int + def memfd_create(name: str, flags: int = ...) -> int: ... + def copy_file_range(src: int, dst: int, count: int, offset_src: int | None = ..., offset_dst: int | None = ...) -> int: ... + +if sys.version_info >= (3, 9): + def waitstatus_to_exitcode(status: int) -> int: ... + + if sys.platform == "linux": + def pidfd_open(pid: int, flags: int = ...) -> int: ... + +if sys.version_info >= (3, 12) and sys.platform == "win32": + def listdrives() -> list[str]: ... + def listmounts(volume: str) -> list[str]: ... + def listvolumes() -> list[str]: ... + +if sys.version_info >= (3, 10) and sys.platform == "linux": + EFD_CLOEXEC: int + EFD_NONBLOCK: int + EFD_SEMAPHORE: int + SPLICE_F_MORE: int + SPLICE_F_MOVE: int + SPLICE_F_NONBLOCK: int + def eventfd(initval: int, flags: int = 524288) -> FileDescriptor: ... + def eventfd_read(fd: FileDescriptor) -> int: ... + def eventfd_write(fd: FileDescriptor, value: int) -> None: ... + def splice( + src: FileDescriptor, + dst: FileDescriptor, + count: int, + offset_src: int | None = ..., + offset_dst: int | None = ..., + flags: int = 0, + ) -> int: ... + +if sys.version_info >= (3, 12) and sys.platform == "linux": + CLONE_FILES: int + CLONE_FS: int + CLONE_NEWCGROUP: int # Linux 4.6+ + CLONE_NEWIPC: int # Linux 2.6.19+ + CLONE_NEWNET: int # Linux 2.6.24+ + CLONE_NEWNS: int + CLONE_NEWPID: int # Linux 3.8+ + CLONE_NEWTIME: int # Linux 5.6+ + CLONE_NEWUSER: int # Linux 3.8+ + CLONE_NEWUTS: int # Linux 2.6.19+ + CLONE_SIGHAND: int + CLONE_SYSVSEM: int # Linux 2.6.26+ + CLONE_THREAD: int + CLONE_VM: int + def unshare(flags: int) -> None: ... + def setns(fd: FileDescriptorLike, nstype: int = 0) -> None: ... + +if sys.version_info >= (3, 13) and sys.platform != "win32": + def posix_openpt(oflag: int, /) -> int: ... + def grantpt(fd: FileDescriptorLike, /) -> None: ... + def unlockpt(fd: FileDescriptorLike, /) -> None: ... + def ptsname(fd: FileDescriptorLike, /) -> str: ... + +if sys.version_info >= (3, 13) and sys.platform == "linux": + TFD_TIMER_ABSTIME: Final = 1 + TFD_TIMER_CANCEL_ON_SET: Final = 2 + TFD_NONBLOCK: Final[int] + TFD_CLOEXEC: Final[int] + POSIX_SPAWN_CLOSEFROM: Final[int] + + def timerfd_create(clockid: int, /, *, flags: int = 0) -> int: ... + def timerfd_settime( + fd: FileDescriptor, /, *, flags: int = 0, initial: float = 0.0, interval: float = 0.0 + ) -> tuple[float, float]: ... + def timerfd_settime_ns(fd: FileDescriptor, /, *, flags: int = 0, initial: int = 0, interval: int = 0) -> tuple[int, int]: ... + def timerfd_gettime(fd: FileDescriptor, /) -> tuple[float, float]: ... + def timerfd_gettime_ns(fd: FileDescriptor, /) -> tuple[int, int]: ... + +if sys.version_info >= (3, 13) or sys.platform != "win32": + # Added to Windows in 3.13. + def fchmod(fd: int, mode: int) -> None: ... + +if sys.platform != "linux": + if sys.version_info >= (3, 13) or sys.platform != "win32": + # Added to Windows in 3.13. + def lchmod(path: StrOrBytesPath, mode: int) -> None: ... diff --git a/mypy/typeshed/stdlib/os/path.pyi b/mypy/typeshed/stdlib/os/path.pyi new file mode 100644 index 000000000000..dc688a9f877f --- /dev/null +++ b/mypy/typeshed/stdlib/os/path.pyi @@ -0,0 +1,8 @@ +import sys + +if sys.platform == "win32": + from ntpath import * + from ntpath import __all__ as __all__ +else: + from posixpath import * + from posixpath import __all__ as __all__ diff --git a/mypy/typeshed/stdlib/ossaudiodev.pyi b/mypy/typeshed/stdlib/ossaudiodev.pyi new file mode 100644 index 000000000000..b9ee3edab033 --- /dev/null +++ b/mypy/typeshed/stdlib/ossaudiodev.pyi @@ -0,0 +1,131 @@ +import sys +from typing import Any, Literal, overload + +if sys.platform != "win32" and sys.platform != "darwin": + AFMT_AC3: int + AFMT_A_LAW: int + AFMT_IMA_ADPCM: int + AFMT_MPEG: int + AFMT_MU_LAW: int + AFMT_QUERY: int + AFMT_S16_BE: int + AFMT_S16_LE: int + AFMT_S16_NE: int + AFMT_S8: int + AFMT_U16_BE: int + AFMT_U16_LE: int + AFMT_U8: int + SNDCTL_COPR_HALT: int + SNDCTL_COPR_LOAD: int + SNDCTL_COPR_RCODE: int + SNDCTL_COPR_RCVMSG: int + SNDCTL_COPR_RDATA: int + SNDCTL_COPR_RESET: int + SNDCTL_COPR_RUN: int + SNDCTL_COPR_SENDMSG: int + SNDCTL_COPR_WCODE: int + SNDCTL_COPR_WDATA: int + SNDCTL_DSP_BIND_CHANNEL: int + SNDCTL_DSP_CHANNELS: int + SNDCTL_DSP_GETBLKSIZE: int + SNDCTL_DSP_GETCAPS: int + SNDCTL_DSP_GETCHANNELMASK: int + SNDCTL_DSP_GETFMTS: int + SNDCTL_DSP_GETIPTR: int + SNDCTL_DSP_GETISPACE: int + SNDCTL_DSP_GETODELAY: int + SNDCTL_DSP_GETOPTR: int + SNDCTL_DSP_GETOSPACE: int + SNDCTL_DSP_GETSPDIF: int + SNDCTL_DSP_GETTRIGGER: int + SNDCTL_DSP_MAPINBUF: int + SNDCTL_DSP_MAPOUTBUF: int + SNDCTL_DSP_NONBLOCK: int + SNDCTL_DSP_POST: int + SNDCTL_DSP_PROFILE: int + SNDCTL_DSP_RESET: int + SNDCTL_DSP_SAMPLESIZE: int + SNDCTL_DSP_SETDUPLEX: int + SNDCTL_DSP_SETFMT: int + SNDCTL_DSP_SETFRAGMENT: int + SNDCTL_DSP_SETSPDIF: int + SNDCTL_DSP_SETSYNCRO: int + SNDCTL_DSP_SETTRIGGER: int + SNDCTL_DSP_SPEED: int + SNDCTL_DSP_STEREO: int + SNDCTL_DSP_SUBDIVIDE: int + SNDCTL_DSP_SYNC: int + SNDCTL_FM_4OP_ENABLE: int + SNDCTL_FM_LOAD_INSTR: int + SNDCTL_MIDI_INFO: int + SNDCTL_MIDI_MPUCMD: int + SNDCTL_MIDI_MPUMODE: int + SNDCTL_MIDI_PRETIME: int + SNDCTL_SEQ_CTRLRATE: int + SNDCTL_SEQ_GETINCOUNT: int + SNDCTL_SEQ_GETOUTCOUNT: int + SNDCTL_SEQ_GETTIME: int + SNDCTL_SEQ_NRMIDIS: int + SNDCTL_SEQ_NRSYNTHS: int + SNDCTL_SEQ_OUTOFBAND: int + SNDCTL_SEQ_PANIC: int + SNDCTL_SEQ_PERCMODE: int + SNDCTL_SEQ_RESET: int + SNDCTL_SEQ_RESETSAMPLES: int + SNDCTL_SEQ_SYNC: int + SNDCTL_SEQ_TESTMIDI: int + SNDCTL_SEQ_THRESHOLD: int + SNDCTL_SYNTH_CONTROL: int + SNDCTL_SYNTH_ID: int + SNDCTL_SYNTH_INFO: int + SNDCTL_SYNTH_MEMAVL: int + SNDCTL_SYNTH_REMOVESAMPLE: int + SNDCTL_TMR_CONTINUE: int + SNDCTL_TMR_METRONOME: int + SNDCTL_TMR_SELECT: int + SNDCTL_TMR_SOURCE: int + SNDCTL_TMR_START: int + SNDCTL_TMR_STOP: int + SNDCTL_TMR_TEMPO: int + SNDCTL_TMR_TIMEBASE: int + SOUND_MIXER_ALTPCM: int + SOUND_MIXER_BASS: int + SOUND_MIXER_CD: int + SOUND_MIXER_DIGITAL1: int + SOUND_MIXER_DIGITAL2: int + SOUND_MIXER_DIGITAL3: int + SOUND_MIXER_IGAIN: int + SOUND_MIXER_IMIX: int + SOUND_MIXER_LINE: int + SOUND_MIXER_LINE1: int + SOUND_MIXER_LINE2: int + SOUND_MIXER_LINE3: int + SOUND_MIXER_MIC: int + SOUND_MIXER_MONITOR: int + SOUND_MIXER_NRDEVICES: int + SOUND_MIXER_OGAIN: int + SOUND_MIXER_PCM: int + SOUND_MIXER_PHONEIN: int + SOUND_MIXER_PHONEOUT: int + SOUND_MIXER_RADIO: int + SOUND_MIXER_RECLEV: int + SOUND_MIXER_SPEAKER: int + SOUND_MIXER_SYNTH: int + SOUND_MIXER_TREBLE: int + SOUND_MIXER_VIDEO: int + SOUND_MIXER_VOLUME: int + + control_labels: list[str] + control_names: list[str] + + # TODO: oss_audio_device return type + @overload + def open(mode: Literal["r", "w", "rw"]) -> Any: ... + @overload + def open(device: str, mode: Literal["r", "w", "rw"]) -> Any: ... + + # TODO: oss_mixer_device return type + def openmixer(device: str = ...) -> Any: ... + + class OSSAudioError(Exception): ... + error = OSSAudioError diff --git a/mypy/typeshed/stdlib/parser.pyi b/mypy/typeshed/stdlib/parser.pyi new file mode 100644 index 000000000000..bafc8015fed9 --- /dev/null +++ b/mypy/typeshed/stdlib/parser.pyi @@ -0,0 +1,24 @@ +from _typeshed import StrOrBytesPath +from collections.abc import Sequence +from types import CodeType +from typing import Any, final + +def expr(source: str) -> STType: ... +def suite(source: str) -> STType: ... +def sequence2st(sequence: Sequence[Any]) -> STType: ... +def tuple2st(sequence: Sequence[Any]) -> STType: ... +def st2list(st: STType, line_info: bool = ..., col_info: bool = ...) -> list[Any]: ... +def st2tuple(st: STType, line_info: bool = ..., col_info: bool = ...) -> tuple[Any, ...]: ... +def compilest(st: STType, filename: StrOrBytesPath = ...) -> CodeType: ... +def isexpr(st: STType) -> bool: ... +def issuite(st: STType) -> bool: ... + +class ParserError(Exception): ... + +@final +class STType: + def compile(self, filename: StrOrBytesPath = ...) -> CodeType: ... + def isexpr(self) -> bool: ... + def issuite(self) -> bool: ... + def tolist(self, line_info: bool = ..., col_info: bool = ...) -> list[Any]: ... + def totuple(self, line_info: bool = ..., col_info: bool = ...) -> tuple[Any, ...]: ... diff --git a/mypy/typeshed/stdlib/pathlib.pyi b/mypy/typeshed/stdlib/pathlib.pyi new file mode 100644 index 000000000000..bdca375f626d --- /dev/null +++ b/mypy/typeshed/stdlib/pathlib.pyi @@ -0,0 +1,298 @@ +import sys +import types +from _typeshed import ( + OpenBinaryMode, + OpenBinaryModeReading, + OpenBinaryModeUpdating, + OpenBinaryModeWriting, + OpenTextMode, + ReadableBuffer, + StrOrBytesPath, + StrPath, + Unused, +) +from collections.abc import Callable, Generator, Iterator, Sequence +from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper +from os import PathLike, stat_result +from types import TracebackType +from typing import IO, Any, BinaryIO, ClassVar, Literal, overload +from typing_extensions import Self, deprecated + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["PurePath", "PurePosixPath", "PureWindowsPath", "Path", "PosixPath", "WindowsPath"] + +if sys.version_info >= (3, 13): + __all__ += ["UnsupportedOperation"] + +class PurePath(PathLike[str]): + if sys.version_info >= (3, 13): + parser: ClassVar[types.ModuleType] + def full_match(self, pattern: StrPath, *, case_sensitive: bool | None = None) -> bool: ... + + @property + def parts(self) -> tuple[str, ...]: ... + @property + def drive(self) -> str: ... + @property + def root(self) -> str: ... + @property + def anchor(self) -> str: ... + @property + def name(self) -> str: ... + @property + def suffix(self) -> str: ... + @property + def suffixes(self) -> list[str]: ... + @property + def stem(self) -> str: ... + if sys.version_info >= (3, 12): + def __new__(cls, *args: StrPath, **kwargs: Unused) -> Self: ... + def __init__(self, *args: StrPath) -> None: ... # pyright: ignore[reportInconsistentConstructor] + else: + def __new__(cls, *args: StrPath) -> Self: ... + + def __hash__(self) -> int: ... + def __fspath__(self) -> str: ... + def __lt__(self, other: PurePath) -> bool: ... + def __le__(self, other: PurePath) -> bool: ... + def __gt__(self, other: PurePath) -> bool: ... + def __ge__(self, other: PurePath) -> bool: ... + def __truediv__(self, key: StrPath) -> Self: ... + def __rtruediv__(self, key: StrPath) -> Self: ... + def __bytes__(self) -> bytes: ... + def as_posix(self) -> str: ... + def as_uri(self) -> str: ... + def is_absolute(self) -> bool: ... + def is_reserved(self) -> bool: ... + if sys.version_info >= (3, 12): + def is_relative_to(self, other: StrPath, /, *_deprecated: StrPath) -> bool: ... + elif sys.version_info >= (3, 9): + def is_relative_to(self, *other: StrPath) -> bool: ... + + if sys.version_info >= (3, 12): + def match(self, path_pattern: str, *, case_sensitive: bool | None = None) -> bool: ... + else: + def match(self, path_pattern: str) -> bool: ... + + if sys.version_info >= (3, 12): + def relative_to(self, other: StrPath, /, *_deprecated: StrPath, walk_up: bool = False) -> Self: ... + else: + def relative_to(self, *other: StrPath) -> Self: ... + + def with_name(self, name: str) -> Self: ... + if sys.version_info >= (3, 9): + def with_stem(self, stem: str) -> Self: ... + + def with_suffix(self, suffix: str) -> Self: ... + def joinpath(self, *other: StrPath) -> Self: ... + @property + def parents(self) -> Sequence[Self]: ... + @property + def parent(self) -> Self: ... + if sys.version_info >= (3, 9) and sys.version_info < (3, 11): + def __class_getitem__(cls, type: Any) -> GenericAlias: ... + + if sys.version_info >= (3, 12): + def with_segments(self, *args: StrPath) -> Self: ... + +class PurePosixPath(PurePath): ... +class PureWindowsPath(PurePath): ... + +class Path(PurePath): + if sys.version_info >= (3, 12): + def __new__(cls, *args: StrPath, **kwargs: Unused) -> Self: ... # pyright: ignore[reportInconsistentConstructor] + else: + def __new__(cls, *args: StrPath, **kwargs: Unused) -> Self: ... + + @classmethod + def cwd(cls) -> Self: ... + if sys.version_info >= (3, 10): + def stat(self, *, follow_symlinks: bool = True) -> stat_result: ... + def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: ... + else: + def stat(self) -> stat_result: ... + def chmod(self, mode: int) -> None: ... + + if sys.version_info >= (3, 13): + @classmethod + def from_uri(cls, uri: str) -> Self: ... + def is_dir(self, *, follow_symlinks: bool = True) -> bool: ... + def is_file(self, *, follow_symlinks: bool = True) -> bool: ... + def read_text(self, encoding: str | None = None, errors: str | None = None, newline: str | None = None) -> str: ... + else: + def __enter__(self) -> Self: ... + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... + def is_dir(self) -> bool: ... + def is_file(self) -> bool: ... + def read_text(self, encoding: str | None = None, errors: str | None = None) -> str: ... + + if sys.version_info >= (3, 13): + def glob( + self, pattern: str, *, case_sensitive: bool | None = None, recurse_symlinks: bool = False + ) -> Generator[Self, None, None]: ... + def rglob( + self, pattern: str, *, case_sensitive: bool | None = None, recurse_symlinks: bool = False + ) -> Generator[Self, None, None]: ... + elif sys.version_info >= (3, 12): + def glob(self, pattern: str, *, case_sensitive: bool | None = None) -> Generator[Self, None, None]: ... + def rglob(self, pattern: str, *, case_sensitive: bool | None = None) -> Generator[Self, None, None]: ... + else: + def glob(self, pattern: str) -> Generator[Self, None, None]: ... + def rglob(self, pattern: str) -> Generator[Self, None, None]: ... + + if sys.version_info >= (3, 12): + def exists(self, *, follow_symlinks: bool = True) -> bool: ... + else: + def exists(self) -> bool: ... + + def is_symlink(self) -> bool: ... + def is_socket(self) -> bool: ... + def is_fifo(self) -> bool: ... + def is_block_device(self) -> bool: ... + def is_char_device(self) -> bool: ... + if sys.version_info >= (3, 12): + def is_junction(self) -> bool: ... + + def iterdir(self) -> Generator[Self, None, None]: ... + def lchmod(self, mode: int) -> None: ... + def lstat(self) -> stat_result: ... + def mkdir(self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False) -> None: ... + + if sys.version_info >= (3, 14): + def copy(self, target: StrPath, *, follow_symlinks: bool = True, preserve_metadata: bool = False) -> None: ... + def copytree( + self, + target: StrPath, + *, + follow_symlinks: bool = True, + preserve_metadata: bool = False, + dirs_exist_ok: bool = False, + ignore: Callable[[Self], bool] | None = None, + on_error: Callable[[OSError], object] | None = None, + ) -> None: ... + + # Adapted from builtins.open + # Text mode: always returns a TextIOWrapper + # The Traversable .open in stdlib/importlib/abc.pyi should be kept in sync with this. + @overload + def open( + self, + mode: OpenTextMode = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> TextIOWrapper: ... + # Unbuffered binary mode: returns a FileIO + @overload + def open( + self, mode: OpenBinaryMode, buffering: Literal[0], encoding: None = None, errors: None = None, newline: None = None + ) -> FileIO: ... + # Buffering is on: return BufferedRandom, BufferedReader, or BufferedWriter + @overload + def open( + self, + mode: OpenBinaryModeUpdating, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> BufferedRandom: ... + @overload + def open( + self, + mode: OpenBinaryModeWriting, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> BufferedWriter: ... + @overload + def open( + self, + mode: OpenBinaryModeReading, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> BufferedReader: ... + # Buffering cannot be determined: fall back to BinaryIO + @overload + def open( + self, mode: OpenBinaryMode, buffering: int = -1, encoding: None = None, errors: None = None, newline: None = None + ) -> BinaryIO: ... + # Fallback if mode is not specified + @overload + def open( + self, mode: str, buffering: int = -1, encoding: str | None = None, errors: str | None = None, newline: str | None = None + ) -> IO[Any]: ... + if sys.platform != "win32": + # These methods do "exist" on Windows, but they always raise NotImplementedError, + # so it's safer to pretend they don't exist + if sys.version_info >= (3, 13): + def owner(self, *, follow_symlinks: bool = True) -> str: ... + def group(self, *, follow_symlinks: bool = True) -> str: ... + else: + def owner(self) -> str: ... + def group(self) -> str: ... + + # This method does "exist" on Windows on <3.12, but always raises NotImplementedError + # On py312+, it works properly on Windows, as with all other platforms + if sys.platform != "win32" or sys.version_info >= (3, 12): + def is_mount(self) -> bool: ... + + if sys.version_info >= (3, 9): + def readlink(self) -> Self: ... + + if sys.version_info >= (3, 10): + def rename(self, target: StrPath) -> Self: ... + def replace(self, target: StrPath) -> Self: ... + else: + def rename(self, target: str | PurePath) -> Self: ... + def replace(self, target: str | PurePath) -> Self: ... + + def resolve(self, strict: bool = False) -> Self: ... + def rmdir(self) -> None: ... + if sys.version_info >= (3, 14): + def delete(self, ignore_errors: bool = False, on_error: Callable[[OSError], object] | None = None) -> None: ... + + def symlink_to(self, target: StrOrBytesPath, target_is_directory: bool = False) -> None: ... + if sys.version_info >= (3, 10): + def hardlink_to(self, target: StrOrBytesPath) -> None: ... + + def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: ... + def unlink(self, missing_ok: bool = False) -> None: ... + @classmethod + def home(cls) -> Self: ... + def absolute(self) -> Self: ... + def expanduser(self) -> Self: ... + def read_bytes(self) -> bytes: ... + def samefile(self, other_path: StrPath) -> bool: ... + def write_bytes(self, data: ReadableBuffer) -> int: ... + if sys.version_info >= (3, 10): + def write_text( + self, data: str, encoding: str | None = None, errors: str | None = None, newline: str | None = None + ) -> int: ... + else: + def write_text(self, data: str, encoding: str | None = None, errors: str | None = None) -> int: ... + if sys.version_info < (3, 12): + if sys.version_info >= (3, 10): + @deprecated("Deprecated as of Python 3.10 and removed in Python 3.12. Use hardlink_to() instead.") + def link_to(self, target: StrOrBytesPath) -> None: ... + else: + def link_to(self, target: StrOrBytesPath) -> None: ... + if sys.version_info >= (3, 12): + def walk( + self, top_down: bool = ..., on_error: Callable[[OSError], object] | None = ..., follow_symlinks: bool = ... + ) -> Iterator[tuple[Self, list[str], list[str]]]: ... + + if sys.version_info >= (3, 14): + def rmtree(self, ignore_errors: bool = False, on_error: Callable[[OSError], object] | None = None) -> None: ... + +class PosixPath(Path, PurePosixPath): ... +class WindowsPath(Path, PureWindowsPath): ... + +if sys.version_info >= (3, 13): + class UnsupportedOperation(NotImplementedError): ... diff --git a/mypy/typeshed/stdlib/pdb.pyi b/mypy/typeshed/stdlib/pdb.pyi new file mode 100644 index 000000000000..61e8b7176e84 --- /dev/null +++ b/mypy/typeshed/stdlib/pdb.pyi @@ -0,0 +1,206 @@ +import signal +import sys +from bdb import Bdb +from cmd import Cmd +from collections.abc import Callable, Iterable, Mapping, Sequence +from inspect import _SourceObjectType +from types import CodeType, FrameType, TracebackType +from typing import IO, Any, ClassVar, Final, TypeVar +from typing_extensions import ParamSpec, Self + +__all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +line_prefix: str # undocumented + +class Restart(Exception): ... + +def run(statement: str, globals: dict[str, Any] | None = None, locals: Mapping[str, Any] | None = None) -> None: ... +def runeval(expression: str, globals: dict[str, Any] | None = None, locals: Mapping[str, Any] | None = None) -> Any: ... +def runctx(statement: str, globals: dict[str, Any], locals: Mapping[str, Any]) -> None: ... +def runcall(func: Callable[_P, _T], *args: _P.args, **kwds: _P.kwargs) -> _T | None: ... +def set_trace(*, header: str | None = None) -> None: ... +def post_mortem(t: TracebackType | None = None) -> None: ... +def pm() -> None: ... + +class Pdb(Bdb, Cmd): + # Everything here is undocumented, except for __init__ + + commands_resuming: ClassVar[list[str]] + + if sys.version_info >= (3, 13): + MAX_CHAINED_EXCEPTION_DEPTH: Final = 999 + + aliases: dict[str, str] + mainpyfile: str + _wait_for_mainpyfile: bool + rcLines: list[str] + commands: dict[int, list[str]] + commands_doprompt: dict[int, bool] + commands_silent: dict[int, bool] + commands_defining: bool + commands_bnum: int | None + lineno: int | None + stack: list[tuple[FrameType, int]] + curindex: int + curframe: FrameType | None + curframe_locals: Mapping[str, Any] + def __init__( + self, + completekey: str = "tab", + stdin: IO[str] | None = None, + stdout: IO[str] | None = None, + skip: Iterable[str] | None = None, + nosigint: bool = False, + readrc: bool = True, + ) -> None: ... + def forget(self) -> None: ... + def setup(self, f: FrameType | None, tb: TracebackType | None) -> None: ... + if sys.version_info < (3, 11): + def execRcLines(self) -> None: ... + + if sys.version_info >= (3, 13): + user_opcode = Bdb.user_line + + def bp_commands(self, frame: FrameType) -> bool: ... + + if sys.version_info >= (3, 13): + def interaction(self, frame: FrameType | None, tb_or_exc: TracebackType | BaseException | None) -> None: ... + else: + def interaction(self, frame: FrameType | None, traceback: TracebackType | None) -> None: ... + + def displayhook(self, obj: object) -> None: ... + def handle_command_def(self, line: str) -> bool: ... + def defaultFile(self) -> str: ... + def lineinfo(self, identifier: str) -> tuple[None, None, None] | tuple[str, str, int]: ... + def checkline(self, filename: str, lineno: int) -> int: ... + def _getval(self, arg: str) -> object: ... + def print_stack_trace(self) -> None: ... + def print_stack_entry(self, frame_lineno: tuple[FrameType, int], prompt_prefix: str = "\n-> ") -> None: ... + def lookupmodule(self, filename: str) -> str | None: ... + if sys.version_info < (3, 11): + def _runscript(self, filename: str) -> None: ... + + if sys.version_info >= (3, 13): + def completedefault(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... + + def do_commands(self, arg: str) -> bool | None: ... + def do_break(self, arg: str, temporary: bool = ...) -> bool | None: ... + def do_tbreak(self, arg: str) -> bool | None: ... + def do_enable(self, arg: str) -> bool | None: ... + def do_disable(self, arg: str) -> bool | None: ... + def do_condition(self, arg: str) -> bool | None: ... + def do_ignore(self, arg: str) -> bool | None: ... + def do_clear(self, arg: str) -> bool | None: ... + def do_where(self, arg: str) -> bool | None: ... + if sys.version_info >= (3, 13): + def do_exceptions(self, arg: str) -> bool | None: ... + + def do_up(self, arg: str) -> bool | None: ... + def do_down(self, arg: str) -> bool | None: ... + def do_until(self, arg: str) -> bool | None: ... + def do_step(self, arg: str) -> bool | None: ... + def do_next(self, arg: str) -> bool | None: ... + def do_run(self, arg: str) -> bool | None: ... + def do_return(self, arg: str) -> bool | None: ... + def do_continue(self, arg: str) -> bool | None: ... + def do_jump(self, arg: str) -> bool | None: ... + def do_debug(self, arg: str) -> bool | None: ... + def do_quit(self, arg: str) -> bool | None: ... + def do_EOF(self, arg: str) -> bool | None: ... + def do_args(self, arg: str) -> bool | None: ... + def do_retval(self, arg: str) -> bool | None: ... + def do_p(self, arg: str) -> bool | None: ... + def do_pp(self, arg: str) -> bool | None: ... + def do_list(self, arg: str) -> bool | None: ... + def do_whatis(self, arg: str) -> bool | None: ... + def do_alias(self, arg: str) -> bool | None: ... + def do_unalias(self, arg: str) -> bool | None: ... + def do_help(self, arg: str) -> bool | None: ... + do_b = do_break + do_cl = do_clear + do_w = do_where + do_bt = do_where + do_u = do_up + do_d = do_down + do_unt = do_until + do_s = do_step + do_n = do_next + do_restart = do_run + do_r = do_return + do_c = do_continue + do_cont = do_continue + do_j = do_jump + do_q = do_quit + do_exit = do_quit + do_a = do_args + do_rv = do_retval + do_l = do_list + do_h = do_help + def help_exec(self) -> None: ... + def help_pdb(self) -> None: ... + def sigint_handler(self, signum: signal.Signals, frame: FrameType) -> None: ... + if sys.version_info >= (3, 13): + def message(self, msg: str, end: str = "\n") -> None: ... + else: + def message(self, msg: str) -> None: ... + + def error(self, msg: str) -> None: ... + if sys.version_info >= (3, 13): + def completenames(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... # type: ignore[override] + if sys.version_info >= (3, 12): + def set_convenience_variable(self, frame: FrameType, name: str, value: Any) -> None: ... + + def _select_frame(self, number: int) -> None: ... + def _getval_except(self, arg: str, frame: FrameType | None = None) -> object: ... + def _print_lines( + self, lines: Sequence[str], start: int, breaks: Sequence[int] = (), frame: FrameType | None = None + ) -> None: ... + def _cmdloop(self) -> None: ... + def do_display(self, arg: str) -> bool | None: ... + def do_interact(self, arg: str) -> bool | None: ... + def do_longlist(self, arg: str) -> bool | None: ... + def do_source(self, arg: str) -> bool | None: ... + def do_undisplay(self, arg: str) -> bool | None: ... + do_ll = do_longlist + def _complete_location(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... + def _complete_bpnumber(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... + def _complete_expression(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... + def complete_undisplay(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... + def complete_unalias(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... + complete_commands = _complete_bpnumber + complete_break = _complete_location + complete_b = _complete_location + complete_tbreak = _complete_location + complete_enable = _complete_bpnumber + complete_disable = _complete_bpnumber + complete_condition = _complete_bpnumber + complete_ignore = _complete_bpnumber + complete_clear = _complete_location + complete_cl = _complete_location + complete_debug = _complete_expression + complete_print = _complete_expression + complete_p = _complete_expression + complete_pp = _complete_expression + complete_source = _complete_expression + complete_whatis = _complete_expression + complete_display = _complete_expression + + if sys.version_info < (3, 11): + def _runmodule(self, module_name: str) -> None: ... + +# undocumented + +def find_function(funcname: str, filename: str) -> tuple[str, str, int] | None: ... +def main() -> None: ... +def help() -> None: ... + +if sys.version_info < (3, 10): + def getsourcelines(obj: _SourceObjectType) -> tuple[list[str], int]: ... + +def lasti2lineno(code: CodeType, lasti: int) -> int: ... + +class _rstr(str): + def __repr__(self) -> Self: ... diff --git a/mypy/typeshed/stdlib/pickle.pyi b/mypy/typeshed/stdlib/pickle.pyi new file mode 100644 index 000000000000..9bea92ef1c9e --- /dev/null +++ b/mypy/typeshed/stdlib/pickle.pyi @@ -0,0 +1,271 @@ +from _typeshed import ReadableBuffer, SupportsWrite +from collections.abc import Callable, Iterable, Iterator, Mapping +from typing import Any, ClassVar, Protocol, SupportsBytes, SupportsIndex, final +from typing_extensions import TypeAlias + +__all__ = [ + "PickleBuffer", + "PickleError", + "PicklingError", + "UnpicklingError", + "Pickler", + "Unpickler", + "dump", + "dumps", + "load", + "loads", + "ADDITEMS", + "APPEND", + "APPENDS", + "BINBYTES", + "BINBYTES8", + "BINFLOAT", + "BINGET", + "BININT", + "BININT1", + "BININT2", + "BINPERSID", + "BINPUT", + "BINSTRING", + "BINUNICODE", + "BINUNICODE8", + "BUILD", + "BYTEARRAY8", + "DEFAULT_PROTOCOL", + "DICT", + "DUP", + "EMPTY_DICT", + "EMPTY_LIST", + "EMPTY_SET", + "EMPTY_TUPLE", + "EXT1", + "EXT2", + "EXT4", + "FALSE", + "FLOAT", + "FRAME", + "FROZENSET", + "GET", + "GLOBAL", + "HIGHEST_PROTOCOL", + "INST", + "INT", + "LIST", + "LONG", + "LONG1", + "LONG4", + "LONG_BINGET", + "LONG_BINPUT", + "MARK", + "MEMOIZE", + "NEWFALSE", + "NEWOBJ", + "NEWOBJ_EX", + "NEWTRUE", + "NEXT_BUFFER", + "NONE", + "OBJ", + "PERSID", + "POP", + "POP_MARK", + "PROTO", + "PUT", + "READONLY_BUFFER", + "REDUCE", + "SETITEM", + "SETITEMS", + "SHORT_BINBYTES", + "SHORT_BINSTRING", + "SHORT_BINUNICODE", + "STACK_GLOBAL", + "STOP", + "STRING", + "TRUE", + "TUPLE", + "TUPLE1", + "TUPLE2", + "TUPLE3", + "UNICODE", +] + +HIGHEST_PROTOCOL: int +DEFAULT_PROTOCOL: int + +bytes_types: tuple[type[Any], ...] # undocumented + +class _ReadableFileobj(Protocol): + def read(self, n: int, /) -> bytes: ... + def readline(self) -> bytes: ... + +@final +class PickleBuffer: + def __init__(self, buffer: ReadableBuffer) -> None: ... + def raw(self) -> memoryview: ... + def release(self) -> None: ... + def __buffer__(self, flags: int, /) -> memoryview: ... + def __release_buffer__(self, buffer: memoryview, /) -> None: ... + +_BufferCallback: TypeAlias = Callable[[PickleBuffer], Any] | None + +def dump( + obj: Any, + file: SupportsWrite[bytes], + protocol: int | None = None, + *, + fix_imports: bool = True, + buffer_callback: _BufferCallback = None, +) -> None: ... +def dumps( + obj: Any, protocol: int | None = None, *, fix_imports: bool = True, buffer_callback: _BufferCallback = None +) -> bytes: ... +def load( + file: _ReadableFileobj, + *, + fix_imports: bool = True, + encoding: str = "ASCII", + errors: str = "strict", + buffers: Iterable[Any] | None = (), +) -> Any: ... +def loads( + data: ReadableBuffer, + /, + *, + fix_imports: bool = True, + encoding: str = "ASCII", + errors: str = "strict", + buffers: Iterable[Any] | None = (), +) -> Any: ... + +class PickleError(Exception): ... +class PicklingError(PickleError): ... +class UnpicklingError(PickleError): ... + +_ReducedType: TypeAlias = ( + str + | tuple[Callable[..., Any], tuple[Any, ...]] + | tuple[Callable[..., Any], tuple[Any, ...], Any] + | tuple[Callable[..., Any], tuple[Any, ...], Any, Iterator[Any] | None] + | tuple[Callable[..., Any], tuple[Any, ...], Any, Iterator[Any] | None, Iterator[Any] | None] +) + +class Pickler: + fast: bool + dispatch_table: Mapping[type, Callable[[Any], _ReducedType]] + bin: bool # undocumented + dispatch: ClassVar[dict[type, Callable[[Unpickler, Any], None]]] # undocumented, _Pickler only + + def __init__( + self, + file: SupportsWrite[bytes], + protocol: int | None = None, + *, + fix_imports: bool = True, + buffer_callback: _BufferCallback = None, + ) -> None: ... + def reducer_override(self, obj: Any) -> Any: ... + def dump(self, obj: Any, /) -> None: ... + def clear_memo(self) -> None: ... + def persistent_id(self, obj: Any) -> Any: ... + +class Unpickler: + dispatch: ClassVar[dict[int, Callable[[Unpickler], None]]] # undocumented, _Unpickler only + + def __init__( + self, + file: _ReadableFileobj, + *, + fix_imports: bool = True, + encoding: str = "ASCII", + errors: str = "strict", + buffers: Iterable[Any] | None = (), + ) -> None: ... + def load(self) -> Any: ... + def find_class(self, module_name: str, global_name: str, /) -> Any: ... + def persistent_load(self, pid: Any) -> Any: ... + +MARK: bytes +STOP: bytes +POP: bytes +POP_MARK: bytes +DUP: bytes +FLOAT: bytes +INT: bytes +BININT: bytes +BININT1: bytes +LONG: bytes +BININT2: bytes +NONE: bytes +PERSID: bytes +BINPERSID: bytes +REDUCE: bytes +STRING: bytes +BINSTRING: bytes +SHORT_BINSTRING: bytes +UNICODE: bytes +BINUNICODE: bytes +APPEND: bytes +BUILD: bytes +GLOBAL: bytes +DICT: bytes +EMPTY_DICT: bytes +APPENDS: bytes +GET: bytes +BINGET: bytes +INST: bytes +LONG_BINGET: bytes +LIST: bytes +EMPTY_LIST: bytes +OBJ: bytes +PUT: bytes +BINPUT: bytes +LONG_BINPUT: bytes +SETITEM: bytes +TUPLE: bytes +EMPTY_TUPLE: bytes +SETITEMS: bytes +BINFLOAT: bytes + +TRUE: bytes +FALSE: bytes + +# protocol 2 +PROTO: bytes +NEWOBJ: bytes +EXT1: bytes +EXT2: bytes +EXT4: bytes +TUPLE1: bytes +TUPLE2: bytes +TUPLE3: bytes +NEWTRUE: bytes +NEWFALSE: bytes +LONG1: bytes +LONG4: bytes + +# protocol 3 +BINBYTES: bytes +SHORT_BINBYTES: bytes + +# protocol 4 +SHORT_BINUNICODE: bytes +BINUNICODE8: bytes +BINBYTES8: bytes +EMPTY_SET: bytes +ADDITEMS: bytes +FROZENSET: bytes +NEWOBJ_EX: bytes +STACK_GLOBAL: bytes +MEMOIZE: bytes +FRAME: bytes + +# protocol 5 +BYTEARRAY8: bytes +NEXT_BUFFER: bytes +READONLY_BUFFER: bytes + +def encode_long(x: int) -> bytes: ... # undocumented +def decode_long(data: Iterable[SupportsIndex] | SupportsBytes | ReadableBuffer) -> int: ... # undocumented + +# pure-Python implementations +_Pickler = Pickler # undocumented +_Unpickler = Unpickler # undocumented diff --git a/mypy/typeshed/stdlib/pickletools.pyi b/mypy/typeshed/stdlib/pickletools.pyi new file mode 100644 index 000000000000..542172814926 --- /dev/null +++ b/mypy/typeshed/stdlib/pickletools.pyi @@ -0,0 +1,167 @@ +from collections.abc import Callable, Iterator, MutableMapping +from typing import IO, Any +from typing_extensions import TypeAlias + +__all__ = ["dis", "genops", "optimize"] + +_Reader: TypeAlias = Callable[[IO[bytes]], Any] +bytes_types: tuple[type[Any], ...] + +UP_TO_NEWLINE: int +TAKEN_FROM_ARGUMENT1: int +TAKEN_FROM_ARGUMENT4: int +TAKEN_FROM_ARGUMENT4U: int +TAKEN_FROM_ARGUMENT8U: int + +class ArgumentDescriptor: + name: str + n: int + reader: _Reader + doc: str + def __init__(self, name: str, n: int, reader: _Reader, doc: str) -> None: ... + +def read_uint1(f: IO[bytes]) -> int: ... + +uint1: ArgumentDescriptor + +def read_uint2(f: IO[bytes]) -> int: ... + +uint2: ArgumentDescriptor + +def read_int4(f: IO[bytes]) -> int: ... + +int4: ArgumentDescriptor + +def read_uint4(f: IO[bytes]) -> int: ... + +uint4: ArgumentDescriptor + +def read_uint8(f: IO[bytes]) -> int: ... + +uint8: ArgumentDescriptor + +def read_stringnl(f: IO[bytes], decode: bool = True, stripquotes: bool = True) -> bytes | str: ... + +stringnl: ArgumentDescriptor + +def read_stringnl_noescape(f: IO[bytes]) -> str: ... + +stringnl_noescape: ArgumentDescriptor + +def read_stringnl_noescape_pair(f: IO[bytes]) -> str: ... + +stringnl_noescape_pair: ArgumentDescriptor + +def read_string1(f: IO[bytes]) -> str: ... + +string1: ArgumentDescriptor + +def read_string4(f: IO[bytes]) -> str: ... + +string4: ArgumentDescriptor + +def read_bytes1(f: IO[bytes]) -> bytes: ... + +bytes1: ArgumentDescriptor + +def read_bytes4(f: IO[bytes]) -> bytes: ... + +bytes4: ArgumentDescriptor + +def read_bytes8(f: IO[bytes]) -> bytes: ... + +bytes8: ArgumentDescriptor + +def read_unicodestringnl(f: IO[bytes]) -> str: ... + +unicodestringnl: ArgumentDescriptor + +def read_unicodestring1(f: IO[bytes]) -> str: ... + +unicodestring1: ArgumentDescriptor + +def read_unicodestring4(f: IO[bytes]) -> str: ... + +unicodestring4: ArgumentDescriptor + +def read_unicodestring8(f: IO[bytes]) -> str: ... + +unicodestring8: ArgumentDescriptor + +def read_decimalnl_short(f: IO[bytes]) -> int: ... +def read_decimalnl_long(f: IO[bytes]) -> int: ... + +decimalnl_short: ArgumentDescriptor +decimalnl_long: ArgumentDescriptor + +def read_floatnl(f: IO[bytes]) -> float: ... + +floatnl: ArgumentDescriptor + +def read_float8(f: IO[bytes]) -> float: ... + +float8: ArgumentDescriptor + +def read_long1(f: IO[bytes]) -> int: ... + +long1: ArgumentDescriptor + +def read_long4(f: IO[bytes]) -> int: ... + +long4: ArgumentDescriptor + +class StackObject: + name: str + obtype: type[Any] | tuple[type[Any], ...] + doc: str + def __init__(self, name: str, obtype: type[Any] | tuple[type[Any], ...], doc: str) -> None: ... + +pyint: StackObject +pylong: StackObject +pyinteger_or_bool: StackObject +pybool: StackObject +pyfloat: StackObject +pybytes_or_str: StackObject +pystring: StackObject +pybytes: StackObject +pyunicode: StackObject +pynone: StackObject +pytuple: StackObject +pylist: StackObject +pydict: StackObject +pyset: StackObject +pyfrozenset: StackObject +anyobject: StackObject +markobject: StackObject +stackslice: StackObject + +class OpcodeInfo: + name: str + code: str + arg: ArgumentDescriptor | None + stack_before: list[StackObject] + stack_after: list[StackObject] + proto: int + doc: str + def __init__( + self, + name: str, + code: str, + arg: ArgumentDescriptor | None, + stack_before: list[StackObject], + stack_after: list[StackObject], + proto: int, + doc: str, + ) -> None: ... + +opcodes: list[OpcodeInfo] + +def genops(pickle: bytes | bytearray | IO[bytes]) -> Iterator[tuple[OpcodeInfo, Any | None, int | None]]: ... +def optimize(p: bytes | bytearray | IO[bytes]) -> bytes: ... +def dis( + pickle: bytes | bytearray | IO[bytes], + out: IO[str] | None = None, + memo: MutableMapping[int, Any] | None = None, + indentlevel: int = 4, + annotate: int = 0, +) -> None: ... diff --git a/mypy/typeshed/stdlib/pipes.pyi b/mypy/typeshed/stdlib/pipes.pyi new file mode 100644 index 000000000000..fe680bfddf5f --- /dev/null +++ b/mypy/typeshed/stdlib/pipes.pyi @@ -0,0 +1,16 @@ +import os + +__all__ = ["Template"] + +class Template: + def reset(self) -> None: ... + def clone(self) -> Template: ... + def debug(self, flag: bool) -> None: ... + def append(self, cmd: str, kind: str) -> None: ... + def prepend(self, cmd: str, kind: str) -> None: ... + def open(self, file: str, rw: str) -> os._wrap_close: ... + def copy(self, infile: str, outfile: str) -> int: ... + +# Not documented, but widely used. +# Documented as shlex.quote since 3.3. +def quote(s: str) -> str: ... diff --git a/mypy/typeshed/stdlib/pkgutil.pyi b/mypy/typeshed/stdlib/pkgutil.pyi new file mode 100644 index 000000000000..7e7fa4fda9a1 --- /dev/null +++ b/mypy/typeshed/stdlib/pkgutil.pyi @@ -0,0 +1,53 @@ +import sys +from _typeshed import SupportsRead +from _typeshed.importlib import LoaderProtocol, MetaPathFinderProtocol, PathEntryFinderProtocol +from collections.abc import Callable, Iterable, Iterator +from typing import IO, Any, NamedTuple, TypeVar +from typing_extensions import deprecated + +__all__ = [ + "get_importer", + "iter_importers", + "get_loader", + "find_loader", + "walk_packages", + "iter_modules", + "get_data", + "read_code", + "extend_path", + "ModuleInfo", +] +if sys.version_info < (3, 12): + __all__ += ["ImpImporter", "ImpLoader"] + +_PathT = TypeVar("_PathT", bound=Iterable[str]) + +class ModuleInfo(NamedTuple): + module_finder: MetaPathFinderProtocol | PathEntryFinderProtocol + name: str + ispkg: bool + +def extend_path(path: _PathT, name: str) -> _PathT: ... + +if sys.version_info < (3, 12): + class ImpImporter: + def __init__(self, path: str | None = None) -> None: ... + + class ImpLoader: + def __init__(self, fullname: str, file: IO[str], filename: str, etc: tuple[str, str, int]) -> None: ... + +@deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") +def find_loader(fullname: str) -> LoaderProtocol | None: ... +def get_importer(path_item: str) -> PathEntryFinderProtocol | None: ... +@deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") +def get_loader(module_or_name: str) -> LoaderProtocol | None: ... +def iter_importers(fullname: str = "") -> Iterator[MetaPathFinderProtocol | PathEntryFinderProtocol]: ... +def iter_modules(path: Iterable[str] | None = None, prefix: str = "") -> Iterator[ModuleInfo]: ... +def read_code(stream: SupportsRead[bytes]) -> Any: ... # undocumented +def walk_packages( + path: Iterable[str] | None = None, prefix: str = "", onerror: Callable[[str], object] | None = None +) -> Iterator[ModuleInfo]: ... +def get_data(package: str, resource: str) -> bytes | None: ... + +if sys.version_info >= (3, 9): + def resolve_name(name: str) -> Any: ... diff --git a/mypy/typeshed/stdlib/platform.pyi b/mypy/typeshed/stdlib/platform.pyi new file mode 100644 index 000000000000..73393eada02c --- /dev/null +++ b/mypy/typeshed/stdlib/platform.pyi @@ -0,0 +1,95 @@ +import sys +from typing import NamedTuple, type_check_only +from typing_extensions import Self + +def libc_ver(executable: str | None = None, lib: str = "", version: str = "", chunksize: int = 16384) -> tuple[str, str]: ... +def win32_ver(release: str = "", version: str = "", csd: str = "", ptype: str = "") -> tuple[str, str, str, str]: ... +def win32_edition() -> str: ... +def win32_is_iot() -> bool: ... +def mac_ver( + release: str = "", versioninfo: tuple[str, str, str] = ("", "", ""), machine: str = "" +) -> tuple[str, tuple[str, str, str], str]: ... +def java_ver( + release: str = "", vendor: str = "", vminfo: tuple[str, str, str] = ("", "", ""), osinfo: tuple[str, str, str] = ("", "", "") +) -> tuple[str, str, tuple[str, str, str], tuple[str, str, str]]: ... +def system_alias(system: str, release: str, version: str) -> tuple[str, str, str]: ... +def architecture(executable: str = sys.executable, bits: str = "", linkage: str = "") -> tuple[str, str]: ... + +if sys.version_info >= (3, 9): + # This class is not exposed. It calls itself platform.uname_result_base. + # At runtime it only has 5 fields. + @type_check_only + class _uname_result_base(NamedTuple): + system: str + node: str + release: str + version: str + machine: str + # This base class doesn't have this field at runtime, but claiming it + # does is the least bad way to handle the situation. Nobody really + # sees this class anyway. See #13068 + processor: str + + # uname_result emulates a 6-field named tuple, but the processor field + # is lazily evaluated rather than being passed in to the constructor. + class uname_result(_uname_result_base): + if sys.version_info >= (3, 10): + __match_args__ = ("system", "node", "release", "version", "machine") # pyright: ignore[reportAssignmentType] + + def __new__(_cls, system: str, node: str, release: str, version: str, machine: str) -> Self: ... + @property + def processor(self) -> str: ... + +else: + # On 3.8, uname_result is actually just a regular NamedTuple. + class uname_result(NamedTuple): + system: str + node: str + release: str + version: str + machine: str + processor: str + +def uname() -> uname_result: ... +def system() -> str: ... +def node() -> str: ... +def release() -> str: ... +def version() -> str: ... +def machine() -> str: ... +def processor() -> str: ... +def python_implementation() -> str: ... +def python_version() -> str: ... +def python_version_tuple() -> tuple[str, str, str]: ... +def python_branch() -> str: ... +def python_revision() -> str: ... +def python_build() -> tuple[str, str]: ... +def python_compiler() -> str: ... +def platform(aliased: bool = ..., terse: bool = ...) -> str: ... + +if sys.version_info >= (3, 10): + def freedesktop_os_release() -> dict[str, str]: ... + +if sys.version_info >= (3, 13): + class AndroidVer(NamedTuple): + release: str + api_level: int + manufacturer: str + model: str + device: str + is_emulator: bool + + class IOSVersionInfo(NamedTuple): + system: str + release: str + model: str + is_simulator: bool + + def android_ver( + release: str = "", + api_level: int = 0, + manufacturer: str = "", + model: str = "", + device: str = "", + is_emulator: bool = False, + ) -> AndroidVer: ... + def ios_ver(system: str = "", release: str = "", model: str = "", is_simulator: bool = False) -> IOSVersionInfo: ... diff --git a/mypy/typeshed/stdlib/plistlib.pyi b/mypy/typeshed/stdlib/plistlib.pyi new file mode 100644 index 000000000000..09637673ce21 --- /dev/null +++ b/mypy/typeshed/stdlib/plistlib.pyi @@ -0,0 +1,113 @@ +import sys +from _typeshed import ReadableBuffer +from collections.abc import Mapping, MutableMapping +from datetime import datetime +from enum import Enum +from typing import IO, Any +from typing_extensions import Self + +__all__ = ["InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID"] +if sys.version_info < (3, 9): + __all__ += ["readPlist", "writePlist", "readPlistFromBytes", "writePlistToBytes", "Data"] + +class PlistFormat(Enum): + FMT_XML = 1 + FMT_BINARY = 2 + +FMT_XML = PlistFormat.FMT_XML +FMT_BINARY = PlistFormat.FMT_BINARY +if sys.version_info >= (3, 13): + def load( + fp: IO[bytes], + *, + fmt: PlistFormat | None = None, + dict_type: type[MutableMapping[str, Any]] = ..., + aware_datetime: bool = False, + ) -> Any: ... + def loads( + value: ReadableBuffer | str, + *, + fmt: PlistFormat | None = None, + dict_type: type[MutableMapping[str, Any]] = ..., + aware_datetime: bool = False, + ) -> Any: ... + +elif sys.version_info >= (3, 9): + def load(fp: IO[bytes], *, fmt: PlistFormat | None = None, dict_type: type[MutableMapping[str, Any]] = ...) -> Any: ... + def loads( + value: ReadableBuffer, *, fmt: PlistFormat | None = None, dict_type: type[MutableMapping[str, Any]] = ... + ) -> Any: ... + +else: + def load( + fp: IO[bytes], + *, + fmt: PlistFormat | None = None, + use_builtin_types: bool = True, + dict_type: type[MutableMapping[str, Any]] = ..., + ) -> Any: ... + def loads( + value: ReadableBuffer, + *, + fmt: PlistFormat | None = None, + use_builtin_types: bool = True, + dict_type: type[MutableMapping[str, Any]] = ..., + ) -> Any: ... + +if sys.version_info >= (3, 13): + def dump( + value: Mapping[str, Any] | list[Any] | tuple[Any, ...] | str | bool | float | bytes | bytearray | datetime, + fp: IO[bytes], + *, + fmt: PlistFormat = ..., + sort_keys: bool = True, + skipkeys: bool = False, + aware_datetime: bool = False, + ) -> None: ... + def dumps( + value: Mapping[str, Any] | list[Any] | tuple[Any, ...] | str | bool | float | bytes | bytearray | datetime, + *, + fmt: PlistFormat = ..., + skipkeys: bool = False, + sort_keys: bool = True, + aware_datetime: bool = False, + ) -> bytes: ... + +else: + def dump( + value: Mapping[str, Any] | list[Any] | tuple[Any, ...] | str | bool | float | bytes | bytearray | datetime, + fp: IO[bytes], + *, + fmt: PlistFormat = ..., + sort_keys: bool = True, + skipkeys: bool = False, + ) -> None: ... + def dumps( + value: Mapping[str, Any] | list[Any] | tuple[Any, ...] | str | bool | float | bytes | bytearray | datetime, + *, + fmt: PlistFormat = ..., + skipkeys: bool = False, + sort_keys: bool = True, + ) -> bytes: ... + +if sys.version_info < (3, 9): + def readPlist(pathOrFile: str | IO[bytes]) -> Any: ... + def writePlist(value: Mapping[str, Any], pathOrFile: str | IO[bytes]) -> None: ... + def readPlistFromBytes(data: ReadableBuffer) -> Any: ... + def writePlistToBytes(value: Mapping[str, Any]) -> bytes: ... + +if sys.version_info < (3, 9): + class Data: + data: bytes + def __init__(self, data: bytes) -> None: ... + +class UID: + data: int + def __init__(self, data: int) -> None: ... + def __index__(self) -> int: ... + def __reduce__(self) -> tuple[type[Self], tuple[int]]: ... + def __hash__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + +class InvalidFileException(ValueError): + def __init__(self, message: str = "Invalid file") -> None: ... diff --git a/mypy/typeshed/stdlib/poplib.pyi b/mypy/typeshed/stdlib/poplib.pyi new file mode 100644 index 000000000000..a1e41be86a7f --- /dev/null +++ b/mypy/typeshed/stdlib/poplib.pyi @@ -0,0 +1,72 @@ +import socket +import ssl +import sys +from builtins import list as _list # conflicts with a method named "list" +from re import Pattern +from typing import Any, BinaryIO, Final, NoReturn, overload +from typing_extensions import TypeAlias + +__all__ = ["POP3", "error_proto", "POP3_SSL"] + +_LongResp: TypeAlias = tuple[bytes, list[bytes], int] + +class error_proto(Exception): ... + +POP3_PORT: Final = 110 +POP3_SSL_PORT: Final = 995 +CR: Final = b"\r" +LF: Final = b"\n" +CRLF: Final = b"\r\n" +HAVE_SSL: bool + +class POP3: + encoding: str + host: str + port: int + sock: socket.socket + file: BinaryIO + welcome: bytes + def __init__(self, host: str, port: int = 110, timeout: float = ...) -> None: ... + def getwelcome(self) -> bytes: ... + def set_debuglevel(self, level: int) -> None: ... + def user(self, user: str) -> bytes: ... + def pass_(self, pswd: str) -> bytes: ... + def stat(self) -> tuple[int, int]: ... + def list(self, which: Any | None = None) -> _LongResp: ... + def retr(self, which: Any) -> _LongResp: ... + def dele(self, which: Any) -> bytes: ... + def noop(self) -> bytes: ... + def rset(self) -> bytes: ... + def quit(self) -> bytes: ... + def close(self) -> None: ... + def rpop(self, user: str) -> bytes: ... + timestamp: Pattern[str] + def apop(self, user: str, password: str) -> bytes: ... + def top(self, which: Any, howmuch: int) -> _LongResp: ... + @overload + def uidl(self) -> _LongResp: ... + @overload + def uidl(self, which: Any) -> bytes: ... + def utf8(self) -> bytes: ... + def capa(self) -> dict[str, _list[str]]: ... + def stls(self, context: ssl.SSLContext | None = None) -> bytes: ... + +class POP3_SSL(POP3): + if sys.version_info >= (3, 12): + def __init__( + self, host: str, port: int = 995, *, timeout: float = ..., context: ssl.SSLContext | None = None + ) -> None: ... + def stls(self, context: Any = None) -> NoReturn: ... + else: + def __init__( + self, + host: str, + port: int = 995, + keyfile: str | None = None, + certfile: str | None = None, + timeout: float = ..., + context: ssl.SSLContext | None = None, + ) -> None: ... + # "context" is actually the last argument, + # but that breaks LSP and it doesn't really matter because all the arguments are ignored + def stls(self, context: Any = None, keyfile: Any = None, certfile: Any = None) -> NoReturn: ... diff --git a/mypy/typeshed/stdlib/posix.pyi b/mypy/typeshed/stdlib/posix.pyi new file mode 100644 index 000000000000..7a4d6cb4bdbe --- /dev/null +++ b/mypy/typeshed/stdlib/posix.pyi @@ -0,0 +1,399 @@ +import sys + +if sys.platform != "win32": + # Actually defined here, but defining in os allows sharing code with windows + from os import ( + CLD_CONTINUED as CLD_CONTINUED, + CLD_DUMPED as CLD_DUMPED, + CLD_EXITED as CLD_EXITED, + CLD_TRAPPED as CLD_TRAPPED, + EX_CANTCREAT as EX_CANTCREAT, + EX_CONFIG as EX_CONFIG, + EX_DATAERR as EX_DATAERR, + EX_IOERR as EX_IOERR, + EX_NOHOST as EX_NOHOST, + EX_NOINPUT as EX_NOINPUT, + EX_NOPERM as EX_NOPERM, + EX_NOUSER as EX_NOUSER, + EX_OK as EX_OK, + EX_OSERR as EX_OSERR, + EX_OSFILE as EX_OSFILE, + EX_PROTOCOL as EX_PROTOCOL, + EX_SOFTWARE as EX_SOFTWARE, + EX_TEMPFAIL as EX_TEMPFAIL, + EX_UNAVAILABLE as EX_UNAVAILABLE, + EX_USAGE as EX_USAGE, + F_LOCK as F_LOCK, + F_OK as F_OK, + F_TEST as F_TEST, + F_TLOCK as F_TLOCK, + F_ULOCK as F_ULOCK, + NGROUPS_MAX as NGROUPS_MAX, + O_ACCMODE as O_ACCMODE, + O_APPEND as O_APPEND, + O_ASYNC as O_ASYNC, + O_CLOEXEC as O_CLOEXEC, + O_CREAT as O_CREAT, + O_DIRECTORY as O_DIRECTORY, + O_DSYNC as O_DSYNC, + O_EXCL as O_EXCL, + O_NDELAY as O_NDELAY, + O_NOCTTY as O_NOCTTY, + O_NOFOLLOW as O_NOFOLLOW, + O_NONBLOCK as O_NONBLOCK, + O_RDONLY as O_RDONLY, + O_RDWR as O_RDWR, + O_SYNC as O_SYNC, + O_TRUNC as O_TRUNC, + O_WRONLY as O_WRONLY, + P_ALL as P_ALL, + P_PGID as P_PGID, + P_PID as P_PID, + POSIX_SPAWN_CLOSE as POSIX_SPAWN_CLOSE, + POSIX_SPAWN_DUP2 as POSIX_SPAWN_DUP2, + POSIX_SPAWN_OPEN as POSIX_SPAWN_OPEN, + PRIO_PGRP as PRIO_PGRP, + PRIO_PROCESS as PRIO_PROCESS, + PRIO_USER as PRIO_USER, + R_OK as R_OK, + RTLD_GLOBAL as RTLD_GLOBAL, + RTLD_LAZY as RTLD_LAZY, + RTLD_LOCAL as RTLD_LOCAL, + RTLD_NODELETE as RTLD_NODELETE, + RTLD_NOLOAD as RTLD_NOLOAD, + RTLD_NOW as RTLD_NOW, + SCHED_FIFO as SCHED_FIFO, + SCHED_OTHER as SCHED_OTHER, + SCHED_RR as SCHED_RR, + SEEK_DATA as SEEK_DATA, + SEEK_HOLE as SEEK_HOLE, + ST_NOSUID as ST_NOSUID, + ST_RDONLY as ST_RDONLY, + TMP_MAX as TMP_MAX, + W_OK as W_OK, + WCONTINUED as WCONTINUED, + WCOREDUMP as WCOREDUMP, + WEXITED as WEXITED, + WEXITSTATUS as WEXITSTATUS, + WIFCONTINUED as WIFCONTINUED, + WIFEXITED as WIFEXITED, + WIFSIGNALED as WIFSIGNALED, + WIFSTOPPED as WIFSTOPPED, + WNOHANG as WNOHANG, + WNOWAIT as WNOWAIT, + WSTOPPED as WSTOPPED, + WSTOPSIG as WSTOPSIG, + WTERMSIG as WTERMSIG, + WUNTRACED as WUNTRACED, + X_OK as X_OK, + DirEntry as DirEntry, + _exit as _exit, + abort as abort, + access as access, + chdir as chdir, + chmod as chmod, + chown as chown, + chroot as chroot, + close as close, + closerange as closerange, + confstr as confstr, + confstr_names as confstr_names, + cpu_count as cpu_count, + ctermid as ctermid, + device_encoding as device_encoding, + dup as dup, + dup2 as dup2, + error as error, + execv as execv, + execve as execve, + fchdir as fchdir, + fchmod as fchmod, + fchown as fchown, + fork as fork, + forkpty as forkpty, + fpathconf as fpathconf, + fspath as fspath, + fstat as fstat, + fstatvfs as fstatvfs, + fsync as fsync, + ftruncate as ftruncate, + get_blocking as get_blocking, + get_inheritable as get_inheritable, + get_terminal_size as get_terminal_size, + getcwd as getcwd, + getcwdb as getcwdb, + getegid as getegid, + geteuid as geteuid, + getgid as getgid, + getgrouplist as getgrouplist, + getgroups as getgroups, + getloadavg as getloadavg, + getlogin as getlogin, + getpgid as getpgid, + getpgrp as getpgrp, + getpid as getpid, + getppid as getppid, + getpriority as getpriority, + getsid as getsid, + getuid as getuid, + initgroups as initgroups, + isatty as isatty, + kill as kill, + killpg as killpg, + lchown as lchown, + link as link, + listdir as listdir, + lockf as lockf, + lseek as lseek, + lstat as lstat, + major as major, + makedev as makedev, + minor as minor, + mkdir as mkdir, + mkfifo as mkfifo, + mknod as mknod, + nice as nice, + open as open, + openpty as openpty, + pathconf as pathconf, + pathconf_names as pathconf_names, + pipe as pipe, + posix_spawn as posix_spawn, + posix_spawnp as posix_spawnp, + pread as pread, + preadv as preadv, + putenv as putenv, + pwrite as pwrite, + pwritev as pwritev, + read as read, + readlink as readlink, + readv as readv, + register_at_fork as register_at_fork, + remove as remove, + rename as rename, + replace as replace, + rmdir as rmdir, + scandir as scandir, + sched_get_priority_max as sched_get_priority_max, + sched_get_priority_min as sched_get_priority_min, + sched_param as sched_param, + sched_yield as sched_yield, + sendfile as sendfile, + set_blocking as set_blocking, + set_inheritable as set_inheritable, + setegid as setegid, + seteuid as seteuid, + setgid as setgid, + setgroups as setgroups, + setpgid as setpgid, + setpgrp as setpgrp, + setpriority as setpriority, + setregid as setregid, + setreuid as setreuid, + setsid as setsid, + setuid as setuid, + stat as stat, + stat_result as stat_result, + statvfs as statvfs, + statvfs_result as statvfs_result, + strerror as strerror, + symlink as symlink, + sync as sync, + sysconf as sysconf, + sysconf_names as sysconf_names, + system as system, + tcgetpgrp as tcgetpgrp, + tcsetpgrp as tcsetpgrp, + terminal_size as terminal_size, + times as times, + times_result as times_result, + truncate as truncate, + ttyname as ttyname, + umask as umask, + uname as uname, + uname_result as uname_result, + unlink as unlink, + unsetenv as unsetenv, + urandom as urandom, + utime as utime, + wait as wait, + wait3 as wait3, + wait4 as wait4, + waitpid as waitpid, + write as write, + writev as writev, + ) + + if sys.version_info >= (3, 9): + from os import CLD_KILLED as CLD_KILLED, CLD_STOPPED as CLD_STOPPED, waitstatus_to_exitcode as waitstatus_to_exitcode + + if sys.version_info >= (3, 10): + from os import O_FSYNC as O_FSYNC + + if sys.version_info >= (3, 11): + from os import login_tty as login_tty + + if sys.version_info >= (3, 13): + from os import grantpt as grantpt, posix_openpt as posix_openpt, ptsname as ptsname, unlockpt as unlockpt + + if sys.version_info >= (3, 13) and sys.platform == "linux": + from os import ( + POSIX_SPAWN_CLOSEFROM as POSIX_SPAWN_CLOSEFROM, + TFD_CLOEXEC as TFD_CLOEXEC, + TFD_NONBLOCK as TFD_NONBLOCK, + TFD_TIMER_ABSTIME as TFD_TIMER_ABSTIME, + TFD_TIMER_CANCEL_ON_SET as TFD_TIMER_CANCEL_ON_SET, + timerfd_create as timerfd_create, + timerfd_gettime as timerfd_gettime, + timerfd_gettime_ns as timerfd_gettime_ns, + timerfd_settime as timerfd_settime, + timerfd_settime_ns as timerfd_settime_ns, + ) + + if sys.platform != "linux": + from os import O_EXLOCK as O_EXLOCK, O_SHLOCK as O_SHLOCK, chflags as chflags, lchflags as lchflags, lchmod as lchmod + + if sys.platform != "linux" and sys.platform != "darwin": + from os import EX_NOTFOUND as EX_NOTFOUND, SCHED_SPORADIC as SCHED_SPORADIC + + if sys.platform != "linux" and sys.version_info >= (3, 13): + from os import O_EXEC as O_EXEC, O_SEARCH as O_SEARCH + + if sys.platform != "darwin": + from os import ( + POSIX_FADV_DONTNEED as POSIX_FADV_DONTNEED, + POSIX_FADV_NOREUSE as POSIX_FADV_NOREUSE, + POSIX_FADV_NORMAL as POSIX_FADV_NORMAL, + POSIX_FADV_RANDOM as POSIX_FADV_RANDOM, + POSIX_FADV_SEQUENTIAL as POSIX_FADV_SEQUENTIAL, + POSIX_FADV_WILLNEED as POSIX_FADV_WILLNEED, + RWF_DSYNC as RWF_DSYNC, + RWF_HIPRI as RWF_HIPRI, + RWF_NOWAIT as RWF_NOWAIT, + RWF_SYNC as RWF_SYNC, + ST_APPEND as ST_APPEND, + ST_MANDLOCK as ST_MANDLOCK, + ST_NOATIME as ST_NOATIME, + ST_NODEV as ST_NODEV, + ST_NODIRATIME as ST_NODIRATIME, + ST_NOEXEC as ST_NOEXEC, + ST_RELATIME as ST_RELATIME, + ST_SYNCHRONOUS as ST_SYNCHRONOUS, + ST_WRITE as ST_WRITE, + fdatasync as fdatasync, + getresgid as getresgid, + getresuid as getresuid, + pipe2 as pipe2, + posix_fadvise as posix_fadvise, + posix_fallocate as posix_fallocate, + sched_getaffinity as sched_getaffinity, + sched_getparam as sched_getparam, + sched_getscheduler as sched_getscheduler, + sched_rr_get_interval as sched_rr_get_interval, + sched_setaffinity as sched_setaffinity, + sched_setparam as sched_setparam, + sched_setscheduler as sched_setscheduler, + setresgid as setresgid, + setresuid as setresuid, + ) + + if sys.version_info >= (3, 10): + from os import RWF_APPEND as RWF_APPEND + + if sys.platform != "darwin" or sys.version_info >= (3, 13): + from os import waitid as waitid, waitid_result as waitid_result + + if sys.platform == "linux": + from os import ( + GRND_NONBLOCK as GRND_NONBLOCK, + GRND_RANDOM as GRND_RANDOM, + MFD_ALLOW_SEALING as MFD_ALLOW_SEALING, + MFD_CLOEXEC as MFD_CLOEXEC, + MFD_HUGE_1GB as MFD_HUGE_1GB, + MFD_HUGE_1MB as MFD_HUGE_1MB, + MFD_HUGE_2GB as MFD_HUGE_2GB, + MFD_HUGE_2MB as MFD_HUGE_2MB, + MFD_HUGE_8MB as MFD_HUGE_8MB, + MFD_HUGE_16GB as MFD_HUGE_16GB, + MFD_HUGE_16MB as MFD_HUGE_16MB, + MFD_HUGE_32MB as MFD_HUGE_32MB, + MFD_HUGE_64KB as MFD_HUGE_64KB, + MFD_HUGE_256MB as MFD_HUGE_256MB, + MFD_HUGE_512KB as MFD_HUGE_512KB, + MFD_HUGE_512MB as MFD_HUGE_512MB, + MFD_HUGE_MASK as MFD_HUGE_MASK, + MFD_HUGE_SHIFT as MFD_HUGE_SHIFT, + MFD_HUGETLB as MFD_HUGETLB, + O_DIRECT as O_DIRECT, + O_LARGEFILE as O_LARGEFILE, + O_NOATIME as O_NOATIME, + O_PATH as O_PATH, + O_RSYNC as O_RSYNC, + O_TMPFILE as O_TMPFILE, + RTLD_DEEPBIND as RTLD_DEEPBIND, + SCHED_BATCH as SCHED_BATCH, + SCHED_IDLE as SCHED_IDLE, + SCHED_RESET_ON_FORK as SCHED_RESET_ON_FORK, + XATTR_CREATE as XATTR_CREATE, + XATTR_REPLACE as XATTR_REPLACE, + XATTR_SIZE_MAX as XATTR_SIZE_MAX, + copy_file_range as copy_file_range, + getrandom as getrandom, + getxattr as getxattr, + listxattr as listxattr, + memfd_create as memfd_create, + removexattr as removexattr, + setxattr as setxattr, + ) + + if sys.version_info >= (3, 9): + from os import P_PIDFD as P_PIDFD, pidfd_open as pidfd_open + + if sys.version_info >= (3, 10): + from os import ( + EFD_CLOEXEC as EFD_CLOEXEC, + EFD_NONBLOCK as EFD_NONBLOCK, + EFD_SEMAPHORE as EFD_SEMAPHORE, + SPLICE_F_MORE as SPLICE_F_MORE, + SPLICE_F_MOVE as SPLICE_F_MOVE, + SPLICE_F_NONBLOCK as SPLICE_F_NONBLOCK, + eventfd as eventfd, + eventfd_read as eventfd_read, + eventfd_write as eventfd_write, + splice as splice, + ) + + if sys.version_info >= (3, 12): + from os import ( + CLONE_FILES as CLONE_FILES, + CLONE_FS as CLONE_FS, + CLONE_NEWCGROUP as CLONE_NEWCGROUP, + CLONE_NEWIPC as CLONE_NEWIPC, + CLONE_NEWNET as CLONE_NEWNET, + CLONE_NEWNS as CLONE_NEWNS, + CLONE_NEWPID as CLONE_NEWPID, + CLONE_NEWTIME as CLONE_NEWTIME, + CLONE_NEWUSER as CLONE_NEWUSER, + CLONE_NEWUTS as CLONE_NEWUTS, + CLONE_SIGHAND as CLONE_SIGHAND, + CLONE_SYSVSEM as CLONE_SYSVSEM, + CLONE_THREAD as CLONE_THREAD, + CLONE_VM as CLONE_VM, + setns as setns, + unshare as unshare, + ) + + if sys.platform == "darwin": + if sys.version_info >= (3, 12): + from os import ( + PRIO_DARWIN_BG as PRIO_DARWIN_BG, + PRIO_DARWIN_NONUI as PRIO_DARWIN_NONUI, + PRIO_DARWIN_PROCESS as PRIO_DARWIN_PROCESS, + PRIO_DARWIN_THREAD as PRIO_DARWIN_THREAD, + ) + if sys.platform == "darwin" and sys.version_info >= (3, 10): + from os import O_EVTONLY as O_EVTONLY, O_NOFOLLOW_ANY as O_NOFOLLOW_ANY, O_SYMLINK as O_SYMLINK + + # Not same as os.environ or os.environb + # Because of this variable, we can't do "from posix import *" in os/__init__.pyi + environ: dict[bytes, bytes] diff --git a/mypy/typeshed/stdlib/posixpath.pyi b/mypy/typeshed/stdlib/posixpath.pyi new file mode 100644 index 000000000000..3313667f1781 --- /dev/null +++ b/mypy/typeshed/stdlib/posixpath.pyi @@ -0,0 +1,166 @@ +import sys +from _typeshed import AnyOrLiteralStr, BytesPath, FileDescriptorOrPath, StrOrBytesPath, StrPath +from collections.abc import Iterable +from genericpath import ( + commonprefix as commonprefix, + exists as exists, + getatime as getatime, + getctime as getctime, + getmtime as getmtime, + getsize as getsize, + isdir as isdir, + isfile as isfile, + samefile as samefile, + sameopenfile as sameopenfile, + samestat as samestat, +) + +if sys.version_info >= (3, 13): + from genericpath import isdevdrive as isdevdrive +from os import PathLike +from typing import AnyStr, overload +from typing_extensions import LiteralString + +__all__ = [ + "normcase", + "isabs", + "join", + "splitdrive", + "split", + "splitext", + "basename", + "dirname", + "commonprefix", + "getsize", + "getmtime", + "getatime", + "getctime", + "islink", + "exists", + "lexists", + "isdir", + "isfile", + "ismount", + "expanduser", + "expandvars", + "normpath", + "abspath", + "samefile", + "sameopenfile", + "samestat", + "curdir", + "pardir", + "sep", + "pathsep", + "defpath", + "altsep", + "extsep", + "devnull", + "realpath", + "supports_unicode_filenames", + "relpath", + "commonpath", +] +if sys.version_info >= (3, 12): + __all__ += ["isjunction", "splitroot"] +if sys.version_info >= (3, 13): + __all__ += ["isdevdrive"] + +supports_unicode_filenames: bool +# aliases (also in os) +curdir: LiteralString +pardir: LiteralString +sep: LiteralString +altsep: LiteralString | None +extsep: LiteralString +pathsep: LiteralString +defpath: LiteralString +devnull: LiteralString + +# Overloads are necessary to work around python/mypy#17952 & python/mypy#11880 +@overload +def abspath(path: PathLike[AnyStr]) -> AnyStr: ... +@overload +def abspath(path: AnyStr) -> AnyStr: ... +@overload +def basename(p: PathLike[AnyStr]) -> AnyStr: ... +@overload +def basename(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... +@overload +def dirname(p: PathLike[AnyStr]) -> AnyStr: ... +@overload +def dirname(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... +@overload +def expanduser(path: PathLike[AnyStr]) -> AnyStr: ... +@overload +def expanduser(path: AnyStr) -> AnyStr: ... +@overload +def expandvars(path: PathLike[AnyStr]) -> AnyStr: ... +@overload +def expandvars(path: AnyStr) -> AnyStr: ... +@overload +def normcase(s: PathLike[AnyStr]) -> AnyStr: ... +@overload +def normcase(s: AnyOrLiteralStr) -> AnyOrLiteralStr: ... +@overload +def normpath(path: PathLike[AnyStr]) -> AnyStr: ... +@overload +def normpath(path: AnyOrLiteralStr) -> AnyOrLiteralStr: ... +@overload +def commonpath(paths: Iterable[LiteralString]) -> LiteralString: ... +@overload +def commonpath(paths: Iterable[StrPath]) -> str: ... +@overload +def commonpath(paths: Iterable[BytesPath]) -> bytes: ... + +# First parameter is not actually pos-only, +# but must be defined as pos-only in the stub or cross-platform code doesn't type-check, +# as the parameter name is different in ntpath.join() +@overload +def join(a: LiteralString, /, *paths: LiteralString) -> LiteralString: ... +@overload +def join(a: StrPath, /, *paths: StrPath) -> str: ... +@overload +def join(a: BytesPath, /, *paths: BytesPath) -> bytes: ... + +if sys.version_info >= (3, 10): + @overload + def realpath(filename: PathLike[AnyStr], *, strict: bool = False) -> AnyStr: ... + @overload + def realpath(filename: AnyStr, *, strict: bool = False) -> AnyStr: ... + +else: + @overload + def realpath(filename: PathLike[AnyStr]) -> AnyStr: ... + @overload + def realpath(filename: AnyStr) -> AnyStr: ... + +@overload +def relpath(path: LiteralString, start: LiteralString | None = None) -> LiteralString: ... +@overload +def relpath(path: BytesPath, start: BytesPath | None = None) -> bytes: ... +@overload +def relpath(path: StrPath, start: StrPath | None = None) -> str: ... +@overload +def split(p: PathLike[AnyStr]) -> tuple[AnyStr, AnyStr]: ... +@overload +def split(p: AnyOrLiteralStr) -> tuple[AnyOrLiteralStr, AnyOrLiteralStr]: ... +@overload +def splitdrive(p: PathLike[AnyStr]) -> tuple[AnyStr, AnyStr]: ... +@overload +def splitdrive(p: AnyOrLiteralStr) -> tuple[AnyOrLiteralStr, AnyOrLiteralStr]: ... +@overload +def splitext(p: PathLike[AnyStr]) -> tuple[AnyStr, AnyStr]: ... +@overload +def splitext(p: AnyOrLiteralStr) -> tuple[AnyOrLiteralStr, AnyOrLiteralStr]: ... +def isabs(s: StrOrBytesPath) -> bool: ... +def islink(path: FileDescriptorOrPath) -> bool: ... +def ismount(path: FileDescriptorOrPath) -> bool: ... +def lexists(path: FileDescriptorOrPath) -> bool: ... + +if sys.version_info >= (3, 12): + def isjunction(path: StrOrBytesPath) -> bool: ... + @overload + def splitroot(p: AnyOrLiteralStr) -> tuple[AnyOrLiteralStr, AnyOrLiteralStr, AnyOrLiteralStr]: ... + @overload + def splitroot(p: PathLike[AnyStr]) -> tuple[AnyStr, AnyStr, AnyStr]: ... diff --git a/mypy/typeshed/stdlib/pprint.pyi b/mypy/typeshed/stdlib/pprint.pyi new file mode 100644 index 000000000000..171878f4165d --- /dev/null +++ b/mypy/typeshed/stdlib/pprint.pyi @@ -0,0 +1,112 @@ +import sys +from typing import IO + +__all__ = ["pprint", "pformat", "isreadable", "isrecursive", "saferepr", "PrettyPrinter", "pp"] + +if sys.version_info >= (3, 10): + def pformat( + object: object, + indent: int = 1, + width: int = 80, + depth: int | None = None, + *, + compact: bool = False, + sort_dicts: bool = True, + underscore_numbers: bool = False, + ) -> str: ... + +else: + def pformat( + object: object, + indent: int = 1, + width: int = 80, + depth: int | None = None, + *, + compact: bool = False, + sort_dicts: bool = True, + ) -> str: ... + +if sys.version_info >= (3, 10): + def pp( + object: object, + stream: IO[str] | None = ..., + indent: int = ..., + width: int = ..., + depth: int | None = ..., + *, + compact: bool = ..., + sort_dicts: bool = False, + underscore_numbers: bool = ..., + ) -> None: ... + +else: + def pp( + object: object, + stream: IO[str] | None = ..., + indent: int = ..., + width: int = ..., + depth: int | None = ..., + *, + compact: bool = ..., + sort_dicts: bool = False, + ) -> None: ... + +if sys.version_info >= (3, 10): + def pprint( + object: object, + stream: IO[str] | None = None, + indent: int = 1, + width: int = 80, + depth: int | None = None, + *, + compact: bool = False, + sort_dicts: bool = True, + underscore_numbers: bool = False, + ) -> None: ... + +else: + def pprint( + object: object, + stream: IO[str] | None = None, + indent: int = 1, + width: int = 80, + depth: int | None = None, + *, + compact: bool = False, + sort_dicts: bool = True, + ) -> None: ... + +def isreadable(object: object) -> bool: ... +def isrecursive(object: object) -> bool: ... +def saferepr(object: object) -> str: ... + +class PrettyPrinter: + if sys.version_info >= (3, 10): + def __init__( + self, + indent: int = 1, + width: int = 80, + depth: int | None = None, + stream: IO[str] | None = None, + *, + compact: bool = False, + sort_dicts: bool = True, + underscore_numbers: bool = False, + ) -> None: ... + else: + def __init__( + self, + indent: int = 1, + width: int = 80, + depth: int | None = None, + stream: IO[str] | None = None, + *, + compact: bool = False, + sort_dicts: bool = True, + ) -> None: ... + + def pformat(self, object: object) -> str: ... + def pprint(self, object: object) -> None: ... + def isreadable(self, object: object) -> bool: ... + def isrecursive(self, object: object) -> bool: ... + def format(self, object: object, context: dict[int, int], maxlevels: int, level: int) -> tuple[str, bool, bool]: ... diff --git a/mypy/typeshed/stdlib/profile.pyi b/mypy/typeshed/stdlib/profile.pyi new file mode 100644 index 000000000000..696193d9dc16 --- /dev/null +++ b/mypy/typeshed/stdlib/profile.pyi @@ -0,0 +1,31 @@ +from _typeshed import StrOrBytesPath +from collections.abc import Callable, Mapping +from typing import Any, TypeVar +from typing_extensions import ParamSpec, Self, TypeAlias + +__all__ = ["run", "runctx", "Profile"] + +def run(statement: str, filename: str | None = None, sort: str | int = -1) -> None: ... +def runctx( + statement: str, globals: dict[str, Any], locals: Mapping[str, Any], filename: str | None = None, sort: str | int = -1 +) -> None: ... + +_T = TypeVar("_T") +_P = ParamSpec("_P") +_Label: TypeAlias = tuple[str, int, str] + +class Profile: + bias: int + stats: dict[_Label, tuple[int, int, int, int, dict[_Label, tuple[int, int, int, int]]]] # undocumented + def __init__(self, timer: Callable[[], float] | None = None, bias: int | None = None) -> None: ... + def set_cmd(self, cmd: str) -> None: ... + def simulate_call(self, name: str) -> None: ... + def simulate_cmd_complete(self) -> None: ... + def print_stats(self, sort: str | int = -1) -> None: ... + def dump_stats(self, file: StrOrBytesPath) -> None: ... + def create_stats(self) -> None: ... + def snapshot_stats(self) -> None: ... + def run(self, cmd: str) -> Self: ... + def runctx(self, cmd: str, globals: dict[str, Any], locals: Mapping[str, Any]) -> Self: ... + def runcall(self, func: Callable[_P, _T], /, *args: _P.args, **kw: _P.kwargs) -> _T: ... + def calibrate(self, m: int, verbose: int = 0) -> float: ... diff --git a/mypy/typeshed/stdlib/pstats.pyi b/mypy/typeshed/stdlib/pstats.pyi new file mode 100644 index 000000000000..d41fa202cf77 --- /dev/null +++ b/mypy/typeshed/stdlib/pstats.pyi @@ -0,0 +1,98 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Iterable +from cProfile import Profile as _cProfile +from profile import Profile +from typing import IO, Any, Literal, overload +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from enum import Enum + +if sys.version_info >= (3, 9): + __all__ = ["Stats", "SortKey", "FunctionProfile", "StatsProfile"] +else: + __all__ = ["Stats", "SortKey"] + +_Selector: TypeAlias = str | float | int + +if sys.version_info >= (3, 11): + class SortKey(StrEnum): + CALLS = "calls" + CUMULATIVE = "cumulative" + FILENAME = "filename" + LINE = "line" + NAME = "name" + NFL = "nfl" + PCALLS = "pcalls" + STDNAME = "stdname" + TIME = "time" + +else: + class SortKey(str, Enum): + CALLS = "calls" + CUMULATIVE = "cumulative" + FILENAME = "filename" + LINE = "line" + NAME = "name" + NFL = "nfl" + PCALLS = "pcalls" + STDNAME = "stdname" + TIME = "time" + +if sys.version_info >= (3, 9): + from dataclasses import dataclass + + @dataclass(unsafe_hash=True) + class FunctionProfile: + ncalls: str + tottime: float + percall_tottime: float + cumtime: float + percall_cumtime: float + file_name: str + line_number: int + + @dataclass(unsafe_hash=True) + class StatsProfile: + total_tt: float + func_profiles: dict[str, FunctionProfile] + +_SortArgDict: TypeAlias = dict[str, tuple[tuple[tuple[int, int], ...], str]] + +class Stats: + sort_arg_dict_default: _SortArgDict + def __init__( + self, + arg: None | str | Profile | _cProfile = ..., + /, + *args: None | str | Profile | _cProfile | Self, + stream: IO[Any] | None = None, + ) -> None: ... + def init(self, arg: None | str | Profile | _cProfile) -> None: ... + def load_stats(self, arg: None | str | Profile | _cProfile) -> None: ... + def get_top_level_stats(self) -> None: ... + def add(self, *arg_list: None | str | Profile | _cProfile | Self) -> Self: ... + def dump_stats(self, filename: StrOrBytesPath) -> None: ... + def get_sort_arg_defs(self) -> _SortArgDict: ... + @overload + def sort_stats(self, field: Literal[-1, 0, 1, 2]) -> Self: ... + @overload + def sort_stats(self, *field: str) -> Self: ... + def reverse_order(self) -> Self: ... + def strip_dirs(self) -> Self: ... + def calc_callees(self) -> None: ... + def eval_print_amount(self, sel: _Selector, list: list[str], msg: str) -> tuple[list[str], str]: ... + if sys.version_info >= (3, 9): + def get_stats_profile(self) -> StatsProfile: ... + + def get_print_list(self, sel_list: Iterable[_Selector]) -> tuple[int, list[str]]: ... + def print_stats(self, *amount: _Selector) -> Self: ... + def print_callees(self, *amount: _Selector) -> Self: ... + def print_callers(self, *amount: _Selector) -> Self: ... + def print_call_heading(self, name_size: int, column_title: str) -> None: ... + def print_call_line(self, name_size: int, source: str, call_dict: dict[str, Any], arrow: str = "->") -> None: ... + def print_title(self) -> None: ... + def print_line(self, func: str) -> None: ... diff --git a/mypy/typeshed/stdlib/pty.pyi b/mypy/typeshed/stdlib/pty.pyi new file mode 100644 index 000000000000..941915179c4a --- /dev/null +++ b/mypy/typeshed/stdlib/pty.pyi @@ -0,0 +1,24 @@ +import sys +from collections.abc import Callable, Iterable +from typing import Final +from typing_extensions import TypeAlias, deprecated + +if sys.platform != "win32": + __all__ = ["openpty", "fork", "spawn"] + _Reader: TypeAlias = Callable[[int], bytes] + + STDIN_FILENO: Final = 0 + STDOUT_FILENO: Final = 1 + STDERR_FILENO: Final = 2 + + CHILD: Final = 0 + def openpty() -> tuple[int, int]: ... + + if sys.version_info < (3, 14): + @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead") + def master_open() -> tuple[int, str]: ... + @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead") + def slave_open(tty_name: str) -> int: ... + + def fork() -> tuple[int, int]: ... + def spawn(argv: str | Iterable[str], master_read: _Reader = ..., stdin_read: _Reader = ...) -> int: ... diff --git a/mypy/typeshed/stdlib/pwd.pyi b/mypy/typeshed/stdlib/pwd.pyi new file mode 100644 index 000000000000..a84ba324718a --- /dev/null +++ b/mypy/typeshed/stdlib/pwd.pyi @@ -0,0 +1,28 @@ +import sys +from _typeshed import structseq +from typing import Any, Final, final + +if sys.platform != "win32": + @final + class struct_passwd(structseq[Any], tuple[str, str, int, int, str, str, str]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("pw_name", "pw_passwd", "pw_uid", "pw_gid", "pw_gecos", "pw_dir", "pw_shell") + + @property + def pw_name(self) -> str: ... + @property + def pw_passwd(self) -> str: ... + @property + def pw_uid(self) -> int: ... + @property + def pw_gid(self) -> int: ... + @property + def pw_gecos(self) -> str: ... + @property + def pw_dir(self) -> str: ... + @property + def pw_shell(self) -> str: ... + + def getpwall() -> list[struct_passwd]: ... + def getpwuid(uid: int, /) -> struct_passwd: ... + def getpwnam(name: str, /) -> struct_passwd: ... diff --git a/mypy/typeshed/stdlib/py_compile.pyi b/mypy/typeshed/stdlib/py_compile.pyi new file mode 100644 index 000000000000..334ce79b5dd0 --- /dev/null +++ b/mypy/typeshed/stdlib/py_compile.pyi @@ -0,0 +1,34 @@ +import enum +import sys +from typing import AnyStr + +__all__ = ["compile", "main", "PyCompileError", "PycInvalidationMode"] + +class PyCompileError(Exception): + exc_type_name: str + exc_value: BaseException + file: str + msg: str + def __init__(self, exc_type: type[BaseException], exc_value: BaseException, file: str, msg: str = "") -> None: ... + +class PycInvalidationMode(enum.Enum): + TIMESTAMP = 1 + CHECKED_HASH = 2 + UNCHECKED_HASH = 3 + +def _get_default_invalidation_mode() -> PycInvalidationMode: ... +def compile( + file: AnyStr, + cfile: AnyStr | None = None, + dfile: AnyStr | None = None, + doraise: bool = False, + optimize: int = -1, + invalidation_mode: PycInvalidationMode | None = None, + quiet: int = 0, +) -> AnyStr | None: ... + +if sys.version_info >= (3, 10): + def main() -> None: ... + +else: + def main(args: list[str] | None = None) -> int: ... diff --git a/mypy/typeshed/stdlib/pyclbr.pyi b/mypy/typeshed/stdlib/pyclbr.pyi new file mode 100644 index 000000000000..504a5d5f115a --- /dev/null +++ b/mypy/typeshed/stdlib/pyclbr.pyi @@ -0,0 +1,74 @@ +import sys +from collections.abc import Mapping, Sequence + +__all__ = ["readmodule", "readmodule_ex", "Class", "Function"] + +class _Object: + module: str + name: str + file: int + lineno: int + + if sys.version_info >= (3, 10): + end_lineno: int | None + + parent: _Object | None + + # This is a dict at runtime, but we're typing it as Mapping to + # avoid variance issues in the subclasses + children: Mapping[str, _Object] + + if sys.version_info >= (3, 10): + def __init__( + self, module: str, name: str, file: str, lineno: int, end_lineno: int | None, parent: _Object | None + ) -> None: ... + else: + def __init__(self, module: str, name: str, file: str, lineno: int, parent: _Object | None) -> None: ... + +class Function(_Object): + if sys.version_info >= (3, 10): + is_async: bool + + parent: Function | Class | None + children: dict[str, Class | Function] + + if sys.version_info >= (3, 10): + def __init__( + self, + module: str, + name: str, + file: str, + lineno: int, + parent: Function | Class | None = None, + is_async: bool = False, + *, + end_lineno: int | None = None, + ) -> None: ... + else: + def __init__(self, module: str, name: str, file: str, lineno: int, parent: Function | Class | None = None) -> None: ... + +class Class(_Object): + super: list[Class | str] | None + methods: dict[str, int] + parent: Class | None + children: dict[str, Class | Function] + + if sys.version_info >= (3, 10): + def __init__( + self, + module: str, + name: str, + super_: list[Class | str] | None, + file: str, + lineno: int, + parent: Class | None = None, + *, + end_lineno: int | None = None, + ) -> None: ... + else: + def __init__( + self, module: str, name: str, super: list[Class | str] | None, file: str, lineno: int, parent: Class | None = None + ) -> None: ... + +def readmodule(module: str, path: Sequence[str] | None = None) -> dict[str, Class]: ... +def readmodule_ex(module: str, path: Sequence[str] | None = None) -> dict[str, Class | Function | list[str]]: ... diff --git a/mypy/typeshed/stdlib/pydoc.pyi b/mypy/typeshed/stdlib/pydoc.pyi new file mode 100644 index 000000000000..144f782acad5 --- /dev/null +++ b/mypy/typeshed/stdlib/pydoc.pyi @@ -0,0 +1,333 @@ +import sys +from _typeshed import OptExcInfo, SupportsWrite, Unused +from abc import abstractmethod +from builtins import list as _list # "list" conflicts with method name +from collections.abc import Callable, Container, Mapping, MutableMapping +from reprlib import Repr +from types import MethodType, ModuleType, TracebackType +from typing import IO, Any, AnyStr, Final, NoReturn, Protocol, TypeVar +from typing_extensions import TypeGuard + +__all__ = ["help"] + +_T = TypeVar("_T") + +__author__: Final[str] +__date__: Final[str] +__version__: Final[str] +__credits__: Final[str] + +class _Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: ... + +def pathdirs() -> list[str]: ... +def getdoc(object: object) -> str: ... +def splitdoc(doc: AnyStr) -> tuple[AnyStr, AnyStr]: ... +def classname(object: object, modname: str) -> str: ... +def isdata(object: object) -> bool: ... +def replace(text: AnyStr, *pairs: AnyStr) -> AnyStr: ... +def cram(text: str, maxlen: int) -> str: ... +def stripid(text: str) -> str: ... +def allmethods(cl: type) -> MutableMapping[str, MethodType]: ... +def visiblename(name: str, all: Container[str] | None = None, obj: object = None) -> bool: ... +def classify_class_attrs(object: object) -> list[tuple[str, str, type, str]]: ... +def ispackage(path: str) -> bool: ... +def source_synopsis(file: IO[AnyStr]) -> AnyStr | None: ... +def synopsis(filename: str, cache: MutableMapping[str, tuple[int, str]] = {}) -> str | None: ... + +class ErrorDuringImport(Exception): + filename: str + exc: type[BaseException] | None + value: BaseException | None + tb: TracebackType | None + def __init__(self, filename: str, exc_info: OptExcInfo) -> None: ... + +def importfile(path: str) -> ModuleType: ... +def safeimport(path: str, forceload: bool = ..., cache: MutableMapping[str, ModuleType] = {}) -> ModuleType | None: ... + +class Doc: + PYTHONDOCS: str + def document(self, object: object, name: str | None = None, *args: Any) -> str: ... + def fail(self, object: object, name: str | None = None, *args: Any) -> NoReturn: ... + @abstractmethod + def docmodule(self, object: object, name: str | None = None, *args: Any) -> str: ... + @abstractmethod + def docclass(self, object: object, name: str | None = None, *args: Any) -> str: ... + @abstractmethod + def docroutine(self, object: object, name: str | None = None, *args: Any) -> str: ... + @abstractmethod + def docother(self, object: object, name: str | None = None, *args: Any) -> str: ... + @abstractmethod + def docproperty(self, object: object, name: str | None = None, *args: Any) -> str: ... + @abstractmethod + def docdata(self, object: object, name: str | None = None, *args: Any) -> str: ... + def getdocloc(self, object: object, basedir: str = ...) -> str | None: ... + +class HTMLRepr(Repr): + def __init__(self) -> None: ... + def escape(self, text: str) -> str: ... + def repr(self, object: object) -> str: ... + def repr1(self, x: object, level: complex) -> str: ... + def repr_string(self, x: str, level: complex) -> str: ... + def repr_str(self, x: str, level: complex) -> str: ... + def repr_instance(self, x: object, level: complex) -> str: ... + def repr_unicode(self, x: AnyStr, level: complex) -> str: ... + +class HTMLDoc(Doc): + _repr_instance: HTMLRepr + repr = _repr_instance.repr + escape = _repr_instance.escape + def page(self, title: str, contents: str) -> str: ... + if sys.version_info >= (3, 11): + def heading(self, title: str, extras: str = "") -> str: ... + def section( + self, + title: str, + cls: str, + contents: str, + width: int = 6, + prelude: str = "", + marginalia: str | None = None, + gap: str = " ", + ) -> str: ... + def multicolumn(self, list: list[_T], format: Callable[[_T], str]) -> str: ... + else: + def heading(self, title: str, fgcol: str, bgcol: str, extras: str = "") -> str: ... + def section( + self, + title: str, + fgcol: str, + bgcol: str, + contents: str, + width: int = 6, + prelude: str = "", + marginalia: str | None = None, + gap: str = " ", + ) -> str: ... + def multicolumn(self, list: list[_T], format: Callable[[_T], str], cols: int = 4) -> str: ... + + def bigsection(self, title: str, *args: Any) -> str: ... + def preformat(self, text: str) -> str: ... + def grey(self, text: str) -> str: ... + def namelink(self, name: str, *dicts: MutableMapping[str, str]) -> str: ... + def classlink(self, object: object, modname: str) -> str: ... + def modulelink(self, object: object) -> str: ... + def modpkglink(self, modpkginfo: tuple[str, str, bool, bool]) -> str: ... + def markup( + self, + text: str, + escape: Callable[[str], str] | None = None, + funcs: Mapping[str, str] = {}, + classes: Mapping[str, str] = {}, + methods: Mapping[str, str] = {}, + ) -> str: ... + def formattree( + self, tree: list[tuple[type, tuple[type, ...]] | list[Any]], modname: str, parent: type | None = None + ) -> str: ... + def docmodule(self, object: object, name: str | None = None, mod: str | None = None, *ignored: Unused) -> str: ... + def docclass( + self, + object: object, + name: str | None = None, + mod: str | None = None, + funcs: Mapping[str, str] = {}, + classes: Mapping[str, str] = {}, + *ignored: Unused, + ) -> str: ... + def formatvalue(self, object: object) -> str: ... + def docother(self, object: object, name: str | None = None, mod: Any | None = None, *ignored: Unused) -> str: ... + if sys.version_info >= (3, 11): + def docroutine( # type: ignore[override] + self, + object: object, + name: str | None = None, + mod: str | None = None, + funcs: Mapping[str, str] = {}, + classes: Mapping[str, str] = {}, + methods: Mapping[str, str] = {}, + cl: type | None = None, + homecls: type | None = None, + ) -> str: ... + def docproperty( + self, object: object, name: str | None = None, mod: str | None = None, cl: Any | None = None, *ignored: Unused + ) -> str: ... + def docdata( + self, object: object, name: str | None = None, mod: Any | None = None, cl: Any | None = None, *ignored: Unused + ) -> str: ... + else: + def docroutine( # type: ignore[override] + self, + object: object, + name: str | None = None, + mod: str | None = None, + funcs: Mapping[str, str] = {}, + classes: Mapping[str, str] = {}, + methods: Mapping[str, str] = {}, + cl: type | None = None, + ) -> str: ... + def docproperty(self, object: object, name: str | None = None, mod: str | None = None, cl: Any | None = None) -> str: ... # type: ignore[override] + def docdata(self, object: object, name: str | None = None, mod: Any | None = None, cl: Any | None = None) -> str: ... # type: ignore[override] + if sys.version_info >= (3, 11): + def parentlink(self, object: type | ModuleType, modname: str) -> str: ... + + def index(self, dir: str, shadowed: MutableMapping[str, bool] | None = None) -> str: ... + def filelink(self, url: str, path: str) -> str: ... + +class TextRepr(Repr): + def __init__(self) -> None: ... + def repr1(self, x: object, level: complex) -> str: ... + def repr_string(self, x: str, level: complex) -> str: ... + def repr_str(self, x: str, level: complex) -> str: ... + def repr_instance(self, x: object, level: complex) -> str: ... + +class TextDoc(Doc): + _repr_instance: TextRepr + repr = _repr_instance.repr + def bold(self, text: str) -> str: ... + def indent(self, text: str, prefix: str = " ") -> str: ... + def section(self, title: str, contents: str) -> str: ... + def formattree( + self, tree: list[tuple[type, tuple[type, ...]] | list[Any]], modname: str, parent: type | None = None, prefix: str = "" + ) -> str: ... + def docclass(self, object: object, name: str | None = None, mod: str | None = None, *ignored: Unused) -> str: ... + def formatvalue(self, object: object) -> str: ... + if sys.version_info >= (3, 11): + def docroutine( # type: ignore[override] + self, + object: object, + name: str | None = None, + mod: str | None = None, + cl: Any | None = None, + homecls: Any | None = None, + ) -> str: ... + def docmodule(self, object: object, name: str | None = None, mod: Any | None = None, *ignored: Unused) -> str: ... + def docproperty( + self, object: object, name: str | None = None, mod: Any | None = None, cl: Any | None = None, *ignored: Unused + ) -> str: ... + def docdata( + self, object: object, name: str | None = None, mod: str | None = None, cl: Any | None = None, *ignored: Unused + ) -> str: ... + def docother( + self, + object: object, + name: str | None = None, + mod: str | None = None, + parent: str | None = None, + *ignored: Unused, + maxlen: int | None = None, + doc: Any | None = None, + ) -> str: ... + else: + def docroutine(self, object: object, name: str | None = None, mod: str | None = None, cl: Any | None = None) -> str: ... # type: ignore[override] + def docmodule(self, object: object, name: str | None = None, mod: Any | None = None) -> str: ... # type: ignore[override] + def docproperty(self, object: object, name: str | None = None, mod: Any | None = None, cl: Any | None = None) -> str: ... # type: ignore[override] + def docdata(self, object: object, name: str | None = None, mod: str | None = None, cl: Any | None = None) -> str: ... # type: ignore[override] + def docother( # type: ignore[override] + self, + object: object, + name: str | None = None, + mod: str | None = None, + parent: str | None = None, + maxlen: int | None = None, + doc: Any | None = None, + ) -> str: ... + +if sys.version_info >= (3, 13): + def pager(text: str, title: str = "") -> None: ... + +else: + def pager(text: str) -> None: ... + +def plain(text: str) -> str: ... +def describe(thing: Any) -> str: ... +def locate(path: str, forceload: bool = ...) -> object: ... + +if sys.version_info >= (3, 13): + def get_pager() -> _Pager: ... + def pipe_pager(text: str, cmd: str, title: str = "") -> None: ... + def tempfile_pager(text: str, cmd: str, title: str = "") -> None: ... + def tty_pager(text: str, title: str = "") -> None: ... + def plain_pager(text: str, title: str = "") -> None: ... + + # For backwards compatibility. + getpager = get_pager + pipepager = pipe_pager + tempfilepager = tempfile_pager + ttypager = tty_pager + plainpager = plain_pager +else: + def getpager() -> Callable[[str], None]: ... + def pipepager(text: str, cmd: str) -> None: ... + def tempfilepager(text: str, cmd: str) -> None: ... + def ttypager(text: str) -> None: ... + def plainpager(text: str) -> None: ... + +text: TextDoc +html: HTMLDoc + +def resolve(thing: str | object, forceload: bool = ...) -> tuple[object, str] | None: ... +def render_doc( + thing: str | object, title: str = "Python Library Documentation: %s", forceload: bool = ..., renderer: Doc | None = None +) -> str: ... + +if sys.version_info >= (3, 11): + def doc( + thing: str | object, + title: str = "Python Library Documentation: %s", + forceload: bool = ..., + output: SupportsWrite[str] | None = None, + is_cli: bool = False, + ) -> None: ... + +else: + def doc( + thing: str | object, + title: str = "Python Library Documentation: %s", + forceload: bool = ..., + output: SupportsWrite[str] | None = None, + ) -> None: ... + +def writedoc(thing: str | object, forceload: bool = ...) -> None: ... +def writedocs(dir: str, pkgpath: str = "", done: Any | None = None) -> None: ... + +class Helper: + keywords: dict[str, str | tuple[str, str]] + symbols: dict[str, str] + topics: dict[str, str | tuple[str, ...]] + def __init__(self, input: IO[str] | None = None, output: IO[str] | None = None) -> None: ... + @property + def input(self) -> IO[str]: ... + @property + def output(self) -> IO[str]: ... + def __call__(self, request: str | Helper | object = ...) -> None: ... + def interact(self) -> None: ... + def getline(self, prompt: str) -> str: ... + if sys.version_info >= (3, 11): + def help(self, request: Any, is_cli: bool = False) -> None: ... + else: + def help(self, request: Any) -> None: ... + + def intro(self) -> None: ... + def list(self, items: _list[str], columns: int = 4, width: int = 80) -> None: ... + def listkeywords(self) -> None: ... + def listsymbols(self) -> None: ... + def listtopics(self) -> None: ... + def showtopic(self, topic: str, more_xrefs: str = "") -> None: ... + def showsymbol(self, symbol: str) -> None: ... + def listmodules(self, key: str = "") -> None: ... + +help: Helper + +class ModuleScanner: + quit: bool + def run( + self, + callback: Callable[[str | None, str, str], object], + key: str | None = None, + completer: Callable[[], object] | None = None, + onerror: Callable[[str], object] | None = None, + ) -> None: ... + +def apropos(key: str) -> None: ... +def ispath(x: object) -> TypeGuard[str]: ... +def cli() -> None: ... diff --git a/stubs/3.3/do-not-remove b/mypy/typeshed/stdlib/pydoc_data/__init__.pyi similarity index 100% rename from stubs/3.3/do-not-remove rename to mypy/typeshed/stdlib/pydoc_data/__init__.pyi diff --git a/mypy/typeshed/stdlib/pydoc_data/topics.pyi b/mypy/typeshed/stdlib/pydoc_data/topics.pyi new file mode 100644 index 000000000000..091d34300106 --- /dev/null +++ b/mypy/typeshed/stdlib/pydoc_data/topics.pyi @@ -0,0 +1 @@ +topics: dict[str, str] diff --git a/mypy/typeshed/stdlib/pyexpat/__init__.pyi b/mypy/typeshed/stdlib/pyexpat/__init__.pyi new file mode 100644 index 000000000000..21e676052098 --- /dev/null +++ b/mypy/typeshed/stdlib/pyexpat/__init__.pyi @@ -0,0 +1,82 @@ +from _typeshed import ReadableBuffer, SupportsRead +from collections.abc import Callable +from pyexpat import errors as errors, model as model +from typing import Any, Final, final +from typing_extensions import CapsuleType, TypeAlias +from xml.parsers.expat import ExpatError as ExpatError + +EXPAT_VERSION: Final[str] # undocumented +version_info: tuple[int, int, int] # undocumented +native_encoding: str # undocumented +features: list[tuple[str, int]] # undocumented + +error = ExpatError +XML_PARAM_ENTITY_PARSING_NEVER: Final = 0 +XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE: Final = 1 +XML_PARAM_ENTITY_PARSING_ALWAYS: Final = 2 + +_Model: TypeAlias = tuple[int, int, str | None, tuple[Any, ...]] + +@final +class XMLParserType: + def Parse(self, data: str | ReadableBuffer, isfinal: bool = False, /) -> int: ... + def ParseFile(self, file: SupportsRead[bytes], /) -> int: ... + def SetBase(self, base: str, /) -> None: ... + def GetBase(self) -> str | None: ... + def GetInputContext(self) -> bytes | None: ... + def ExternalEntityParserCreate(self, context: str | None, encoding: str = ..., /) -> XMLParserType: ... + def SetParamEntityParsing(self, flag: int, /) -> int: ... + def UseForeignDTD(self, flag: bool = True, /) -> None: ... + def GetReparseDeferralEnabled(self) -> bool: ... + def SetReparseDeferralEnabled(self, enabled: bool, /) -> None: ... + @property + def intern(self) -> dict[str, str]: ... + buffer_size: int + buffer_text: bool + buffer_used: int + namespace_prefixes: bool # undocumented + ordered_attributes: bool + specified_attributes: bool + ErrorByteIndex: int + ErrorCode: int + ErrorColumnNumber: int + ErrorLineNumber: int + CurrentByteIndex: int + CurrentColumnNumber: int + CurrentLineNumber: int + XmlDeclHandler: Callable[[str, str | None, int], Any] | None + StartDoctypeDeclHandler: Callable[[str, str | None, str | None, bool], Any] | None + EndDoctypeDeclHandler: Callable[[], Any] | None + ElementDeclHandler: Callable[[str, _Model], Any] | None + AttlistDeclHandler: Callable[[str, str, str, str | None, bool], Any] | None + StartElementHandler: ( + Callable[[str, dict[str, str]], Any] + | Callable[[str, list[str]], Any] + | Callable[[str, dict[str, str], list[str]], Any] + | None + ) + EndElementHandler: Callable[[str], Any] | None + ProcessingInstructionHandler: Callable[[str, str], Any] | None + CharacterDataHandler: Callable[[str], Any] | None + UnparsedEntityDeclHandler: Callable[[str, str | None, str, str | None, str], Any] | None + EntityDeclHandler: Callable[[str, bool, str | None, str | None, str, str | None, str | None], Any] | None + NotationDeclHandler: Callable[[str, str | None, str, str | None], Any] | None + StartNamespaceDeclHandler: Callable[[str, str], Any] | None + EndNamespaceDeclHandler: Callable[[str], Any] | None + CommentHandler: Callable[[str], Any] | None + StartCdataSectionHandler: Callable[[], Any] | None + EndCdataSectionHandler: Callable[[], Any] | None + DefaultHandler: Callable[[str], Any] | None + DefaultHandlerExpand: Callable[[str], Any] | None + NotStandaloneHandler: Callable[[], int] | None + ExternalEntityRefHandler: Callable[[str, str | None, str | None, str | None], int] | None + SkippedEntityHandler: Callable[[str, bool], Any] | None + +def ErrorString(code: int, /) -> str: ... + +# intern is undocumented +def ParserCreate( + encoding: str | None = None, namespace_separator: str | None = None, intern: dict[str, Any] | None = None +) -> XMLParserType: ... + +expat_CAPI: CapsuleType diff --git a/mypy/typeshed/stdlib/pyexpat/errors.pyi b/mypy/typeshed/stdlib/pyexpat/errors.pyi new file mode 100644 index 000000000000..cae4da089161 --- /dev/null +++ b/mypy/typeshed/stdlib/pyexpat/errors.pyi @@ -0,0 +1,51 @@ +import sys +from typing import Final +from typing_extensions import LiteralString + +codes: dict[str, int] +messages: dict[int, str] + +XML_ERROR_ABORTED: Final[LiteralString] +XML_ERROR_ASYNC_ENTITY: Final[LiteralString] +XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF: Final[LiteralString] +XML_ERROR_BAD_CHAR_REF: Final[LiteralString] +XML_ERROR_BINARY_ENTITY_REF: Final[LiteralString] +XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING: Final[LiteralString] +XML_ERROR_DUPLICATE_ATTRIBUTE: Final[LiteralString] +XML_ERROR_ENTITY_DECLARED_IN_PE: Final[LiteralString] +XML_ERROR_EXTERNAL_ENTITY_HANDLING: Final[LiteralString] +XML_ERROR_FEATURE_REQUIRES_XML_DTD: Final[LiteralString] +XML_ERROR_FINISHED: Final[LiteralString] +XML_ERROR_INCOMPLETE_PE: Final[LiteralString] +XML_ERROR_INCORRECT_ENCODING: Final[LiteralString] +XML_ERROR_INVALID_TOKEN: Final[LiteralString] +XML_ERROR_JUNK_AFTER_DOC_ELEMENT: Final[LiteralString] +XML_ERROR_MISPLACED_XML_PI: Final[LiteralString] +XML_ERROR_NOT_STANDALONE: Final[LiteralString] +XML_ERROR_NOT_SUSPENDED: Final[LiteralString] +XML_ERROR_NO_ELEMENTS: Final[LiteralString] +XML_ERROR_NO_MEMORY: Final[LiteralString] +XML_ERROR_PARAM_ENTITY_REF: Final[LiteralString] +XML_ERROR_PARTIAL_CHAR: Final[LiteralString] +XML_ERROR_PUBLICID: Final[LiteralString] +XML_ERROR_RECURSIVE_ENTITY_REF: Final[LiteralString] +XML_ERROR_SUSPENDED: Final[LiteralString] +XML_ERROR_SUSPEND_PE: Final[LiteralString] +XML_ERROR_SYNTAX: Final[LiteralString] +XML_ERROR_TAG_MISMATCH: Final[LiteralString] +XML_ERROR_TEXT_DECL: Final[LiteralString] +XML_ERROR_UNBOUND_PREFIX: Final[LiteralString] +XML_ERROR_UNCLOSED_CDATA_SECTION: Final[LiteralString] +XML_ERROR_UNCLOSED_TOKEN: Final[LiteralString] +XML_ERROR_UNDECLARING_PREFIX: Final[LiteralString] +XML_ERROR_UNDEFINED_ENTITY: Final[LiteralString] +XML_ERROR_UNEXPECTED_STATE: Final[LiteralString] +XML_ERROR_UNKNOWN_ENCODING: Final[LiteralString] +XML_ERROR_XML_DECL: Final[LiteralString] +if sys.version_info >= (3, 11): + XML_ERROR_RESERVED_PREFIX_XML: Final[LiteralString] + XML_ERROR_RESERVED_PREFIX_XMLNS: Final[LiteralString] + XML_ERROR_RESERVED_NAMESPACE_URI: Final[LiteralString] + XML_ERROR_INVALID_ARGUMENT: Final[LiteralString] + XML_ERROR_NO_BUFFER: Final[LiteralString] + XML_ERROR_AMPLIFICATION_LIMIT_BREACH: Final[LiteralString] diff --git a/mypy/typeshed/stdlib/pyexpat/model.pyi b/mypy/typeshed/stdlib/pyexpat/model.pyi new file mode 100644 index 000000000000..bac8f3692ce5 --- /dev/null +++ b/mypy/typeshed/stdlib/pyexpat/model.pyi @@ -0,0 +1,13 @@ +from typing import Final + +XML_CTYPE_ANY: Final = 2 +XML_CTYPE_EMPTY: Final = 1 +XML_CTYPE_MIXED: Final = 3 +XML_CTYPE_NAME: Final = 4 +XML_CTYPE_CHOICE: Final = 5 +XML_CTYPE_SEQ: Final = 6 + +XML_CQUANT_NONE: Final = 0 +XML_CQUANT_OPT: Final = 1 +XML_CQUANT_REP: Final = 2 +XML_CQUANT_PLUS: Final = 3 diff --git a/mypy/typeshed/stdlib/queue.pyi b/mypy/typeshed/stdlib/queue.pyi new file mode 100644 index 000000000000..4fb49cb6102b --- /dev/null +++ b/mypy/typeshed/stdlib/queue.pyi @@ -0,0 +1,57 @@ +import sys +from _queue import Empty as Empty, SimpleQueue as SimpleQueue +from threading import Condition, Lock +from typing import Any, Generic, TypeVar + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = ["Empty", "Full", "Queue", "PriorityQueue", "LifoQueue", "SimpleQueue"] +if sys.version_info >= (3, 13): + __all__ += ["ShutDown"] + +_T = TypeVar("_T") + +class Full(Exception): ... + +if sys.version_info >= (3, 13): + class ShutDown(Exception): ... + +class Queue(Generic[_T]): + maxsize: int + + mutex: Lock # undocumented + not_empty: Condition # undocumented + not_full: Condition # undocumented + all_tasks_done: Condition # undocumented + unfinished_tasks: int # undocumented + if sys.version_info >= (3, 13): + is_shutdown: bool # undocumented + # Despite the fact that `queue` has `deque` type, + # we treat it as `Any` to allow different implementations in subtypes. + queue: Any # undocumented + def __init__(self, maxsize: int = 0) -> None: ... + def _init(self, maxsize: int) -> None: ... + def empty(self) -> bool: ... + def full(self) -> bool: ... + def get(self, block: bool = True, timeout: float | None = None) -> _T: ... + def get_nowait(self) -> _T: ... + if sys.version_info >= (3, 13): + def shutdown(self, immediate: bool = False) -> None: ... + + def _get(self) -> _T: ... + def put(self, item: _T, block: bool = True, timeout: float | None = None) -> None: ... + def put_nowait(self, item: _T) -> None: ... + def _put(self, item: _T) -> None: ... + def join(self) -> None: ... + def qsize(self) -> int: ... + def _qsize(self) -> int: ... + def task_done(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class PriorityQueue(Queue[_T]): + queue: list[_T] + +class LifoQueue(Queue[_T]): + queue: list[_T] diff --git a/mypy/typeshed/stdlib/quopri.pyi b/mypy/typeshed/stdlib/quopri.pyi new file mode 100644 index 000000000000..b652e139bd0e --- /dev/null +++ b/mypy/typeshed/stdlib/quopri.pyi @@ -0,0 +1,11 @@ +from _typeshed import ReadableBuffer, SupportsNoArgReadline, SupportsRead, SupportsWrite +from typing import Protocol + +__all__ = ["encode", "decode", "encodestring", "decodestring"] + +class _Input(SupportsRead[bytes], SupportsNoArgReadline[bytes], Protocol): ... + +def encode(input: _Input, output: SupportsWrite[bytes], quotetabs: int, header: bool = False) -> None: ... +def encodestring(s: ReadableBuffer, quotetabs: bool = False, header: bool = False) -> bytes: ... +def decode(input: _Input, output: SupportsWrite[bytes], header: bool = False) -> None: ... +def decodestring(s: str | ReadableBuffer, header: bool = False) -> bytes: ... diff --git a/mypy/typeshed/stdlib/random.pyi b/mypy/typeshed/stdlib/random.pyi new file mode 100644 index 000000000000..e7320369c377 --- /dev/null +++ b/mypy/typeshed/stdlib/random.pyi @@ -0,0 +1,141 @@ +import _random +import sys +from _typeshed import SupportsLenAndGetItem +from collections.abc import Callable, Iterable, MutableSequence, Sequence, Set as AbstractSet +from fractions import Fraction +from typing import Any, ClassVar, NoReturn, TypeVar + +__all__ = [ + "Random", + "seed", + "random", + "uniform", + "randint", + "choice", + "sample", + "randrange", + "shuffle", + "normalvariate", + "lognormvariate", + "expovariate", + "vonmisesvariate", + "gammavariate", + "triangular", + "gauss", + "betavariate", + "paretovariate", + "weibullvariate", + "getstate", + "setstate", + "getrandbits", + "choices", + "SystemRandom", +] + +if sys.version_info >= (3, 9): + __all__ += ["randbytes"] +if sys.version_info >= (3, 12): + __all__ += ["binomialvariate"] + +_T = TypeVar("_T") + +class Random(_random.Random): + VERSION: ClassVar[int] + if sys.version_info >= (3, 9): + def __init__(self, x: int | float | str | bytes | bytearray | None = None) -> None: ... # noqa: Y041 + else: + def __init__(self, x: Any = None) -> None: ... + # Using other `seed` types is deprecated since 3.9 and removed in 3.11 + # Ignore Y041, since random.seed doesn't treat int like a float subtype. Having an explicit + # int better documents conventional usage of random.seed. + if sys.version_info >= (3, 9): + def seed(self, a: int | float | str | bytes | bytearray | None = None, version: int = 2) -> None: ... # type: ignore[override] # noqa: Y041 + else: + def seed(self, a: Any = None, version: int = 2) -> None: ... + + def getstate(self) -> tuple[Any, ...]: ... + def setstate(self, state: tuple[Any, ...]) -> None: ... + def randrange(self, start: int, stop: int | None = None, step: int = 1) -> int: ... + def randint(self, a: int, b: int) -> int: ... + if sys.version_info >= (3, 9): + def randbytes(self, n: int) -> bytes: ... + + def choice(self, seq: SupportsLenAndGetItem[_T]) -> _T: ... + def choices( + self, + population: SupportsLenAndGetItem[_T], + weights: Sequence[float | Fraction] | None = None, + *, + cum_weights: Sequence[float | Fraction] | None = None, + k: int = 1, + ) -> list[_T]: ... + if sys.version_info >= (3, 11): + def shuffle(self, x: MutableSequence[Any]) -> None: ... + else: + def shuffle(self, x: MutableSequence[Any], random: Callable[[], float] | None = None) -> None: ... + if sys.version_info >= (3, 11): + def sample(self, population: Sequence[_T], k: int, *, counts: Iterable[int] | None = None) -> list[_T]: ... + elif sys.version_info >= (3, 9): + def sample( + self, population: Sequence[_T] | AbstractSet[_T], k: int, *, counts: Iterable[int] | None = None + ) -> list[_T]: ... + else: + def sample(self, population: Sequence[_T] | AbstractSet[_T], k: int) -> list[_T]: ... + + def uniform(self, a: float, b: float) -> float: ... + def triangular(self, low: float = 0.0, high: float = 1.0, mode: float | None = None) -> float: ... + if sys.version_info >= (3, 12): + def binomialvariate(self, n: int = 1, p: float = 0.5) -> int: ... + + def betavariate(self, alpha: float, beta: float) -> float: ... + if sys.version_info >= (3, 12): + def expovariate(self, lambd: float = 1.0) -> float: ... + else: + def expovariate(self, lambd: float) -> float: ... + + def gammavariate(self, alpha: float, beta: float) -> float: ... + if sys.version_info >= (3, 11): + def gauss(self, mu: float = 0.0, sigma: float = 1.0) -> float: ... + def normalvariate(self, mu: float = 0.0, sigma: float = 1.0) -> float: ... + else: + def gauss(self, mu: float, sigma: float) -> float: ... + def normalvariate(self, mu: float, sigma: float) -> float: ... + + def lognormvariate(self, mu: float, sigma: float) -> float: ... + def vonmisesvariate(self, mu: float, kappa: float) -> float: ... + def paretovariate(self, alpha: float) -> float: ... + def weibullvariate(self, alpha: float, beta: float) -> float: ... + +# SystemRandom is not implemented for all OS's; good on Windows & Linux +class SystemRandom(Random): + def getrandbits(self, k: int) -> int: ... # k can be passed by keyword + def getstate(self, *args: Any, **kwds: Any) -> NoReturn: ... + def setstate(self, *args: Any, **kwds: Any) -> NoReturn: ... + +_inst: Random +seed = _inst.seed +random = _inst.random +uniform = _inst.uniform +triangular = _inst.triangular +randint = _inst.randint +choice = _inst.choice +randrange = _inst.randrange +sample = _inst.sample +shuffle = _inst.shuffle +choices = _inst.choices +normalvariate = _inst.normalvariate +lognormvariate = _inst.lognormvariate +expovariate = _inst.expovariate +vonmisesvariate = _inst.vonmisesvariate +gammavariate = _inst.gammavariate +gauss = _inst.gauss +if sys.version_info >= (3, 12): + binomialvariate = _inst.binomialvariate +betavariate = _inst.betavariate +paretovariate = _inst.paretovariate +weibullvariate = _inst.weibullvariate +getstate = _inst.getstate +setstate = _inst.setstate +getrandbits = _inst.getrandbits +if sys.version_info >= (3, 9): + randbytes = _inst.randbytes diff --git a/mypy/typeshed/stdlib/re.pyi b/mypy/typeshed/stdlib/re.pyi new file mode 100644 index 000000000000..b8fe2e9e1a46 --- /dev/null +++ b/mypy/typeshed/stdlib/re.pyi @@ -0,0 +1,314 @@ +import enum +import sre_compile +import sre_constants +import sys +from _typeshed import MaybeNone, ReadableBuffer +from collections.abc import Callable, Iterator, Mapping +from typing import Any, AnyStr, Generic, Literal, TypeVar, final, overload +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "match", + "fullmatch", + "search", + "sub", + "subn", + "split", + "findall", + "finditer", + "compile", + "purge", + "escape", + "error", + "A", + "I", + "L", + "M", + "S", + "X", + "U", + "ASCII", + "IGNORECASE", + "LOCALE", + "MULTILINE", + "DOTALL", + "VERBOSE", + "UNICODE", + "Match", + "Pattern", +] +if sys.version_info < (3, 13): + __all__ += ["template"] + +if sys.version_info >= (3, 11): + __all__ += ["NOFLAG", "RegexFlag"] + +if sys.version_info >= (3, 13): + __all__ += ["PatternError"] + + PatternError = sre_constants.error + +_T = TypeVar("_T") + +# The implementation defines this in re._constants (version_info >= 3, 11) or +# sre_constants. Typeshed has it here because its __module__ attribute is set to "re". +class error(Exception): + msg: str + pattern: str | bytes | None + pos: int | None + lineno: int + colno: int + def __init__(self, msg: str, pattern: str | bytes | None = None, pos: int | None = None) -> None: ... + +@final +class Match(Generic[AnyStr]): + @property + def pos(self) -> int: ... + @property + def endpos(self) -> int: ... + @property + def lastindex(self) -> int | None: ... + @property + def lastgroup(self) -> str | None: ... + @property + def string(self) -> AnyStr: ... + + # The regular expression object whose match() or search() method produced + # this match instance. + @property + def re(self) -> Pattern[AnyStr]: ... + @overload + def expand(self: Match[str], template: str) -> str: ... + @overload + def expand(self: Match[bytes], template: ReadableBuffer) -> bytes: ... + @overload + def expand(self, template: AnyStr) -> AnyStr: ... + # group() returns "AnyStr" or "AnyStr | None", depending on the pattern. + @overload + def group(self, group: Literal[0] = 0, /) -> AnyStr: ... + @overload + def group(self, group: str | int, /) -> AnyStr | MaybeNone: ... + @overload + def group(self, group1: str | int, group2: str | int, /, *groups: str | int) -> tuple[AnyStr | MaybeNone, ...]: ... + # Each item of groups()'s return tuple is either "AnyStr" or + # "AnyStr | None", depending on the pattern. + @overload + def groups(self) -> tuple[AnyStr | MaybeNone, ...]: ... + @overload + def groups(self, default: _T) -> tuple[AnyStr | _T, ...]: ... + # Each value in groupdict()'s return dict is either "AnyStr" or + # "AnyStr | None", depending on the pattern. + @overload + def groupdict(self) -> dict[str, AnyStr | MaybeNone]: ... + @overload + def groupdict(self, default: _T) -> dict[str, AnyStr | _T]: ... + def start(self, group: int | str = 0, /) -> int: ... + def end(self, group: int | str = 0, /) -> int: ... + def span(self, group: int | str = 0, /) -> tuple[int, int]: ... + @property + def regs(self) -> tuple[tuple[int, int], ...]: ... # undocumented + # __getitem__() returns "AnyStr" or "AnyStr | None", depending on the pattern. + @overload + def __getitem__(self, key: Literal[0], /) -> AnyStr: ... + @overload + def __getitem__(self, key: int | str, /) -> AnyStr | MaybeNone: ... + def __copy__(self) -> Match[AnyStr]: ... + def __deepcopy__(self, memo: Any, /) -> Match[AnyStr]: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +@final +class Pattern(Generic[AnyStr]): + @property + def flags(self) -> int: ... + @property + def groupindex(self) -> Mapping[str, int]: ... + @property + def groups(self) -> int: ... + @property + def pattern(self) -> AnyStr: ... + @overload + def search(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ... + @overload + def search(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ... + @overload + def search(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ... + @overload + def match(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ... + @overload + def match(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ... + @overload + def match(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ... + @overload + def fullmatch(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ... + @overload + def fullmatch( + self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize + ) -> Match[bytes] | None: ... + @overload + def fullmatch(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ... + @overload + def split(self: Pattern[str], string: str, maxsplit: int = 0) -> list[str | MaybeNone]: ... + @overload + def split(self: Pattern[bytes], string: ReadableBuffer, maxsplit: int = 0) -> list[bytes | MaybeNone]: ... + @overload + def split(self, string: AnyStr, maxsplit: int = 0) -> list[AnyStr | MaybeNone]: ... + # return type depends on the number of groups in the pattern + @overload + def findall(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> list[Any]: ... + @overload + def findall(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> list[Any]: ... + @overload + def findall(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> list[AnyStr]: ... + @overload + def finditer(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[str]]: ... + @overload + def finditer( + self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize + ) -> Iterator[Match[bytes]]: ... + @overload + def finditer(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[AnyStr]]: ... + @overload + def sub(self: Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0) -> str: ... + @overload + def sub( + self: Pattern[bytes], + repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer], + string: ReadableBuffer, + count: int = 0, + ) -> bytes: ... + @overload + def sub(self, repl: AnyStr | Callable[[Match[AnyStr]], AnyStr], string: AnyStr, count: int = 0) -> AnyStr: ... + @overload + def subn(self: Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0) -> tuple[str, int]: ... + @overload + def subn( + self: Pattern[bytes], + repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer], + string: ReadableBuffer, + count: int = 0, + ) -> tuple[bytes, int]: ... + @overload + def subn(self, repl: AnyStr | Callable[[Match[AnyStr]], AnyStr], string: AnyStr, count: int = 0) -> tuple[AnyStr, int]: ... + def __copy__(self) -> Pattern[AnyStr]: ... + def __deepcopy__(self, memo: Any, /) -> Pattern[AnyStr]: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +# ----- re variables and constants ----- + +class RegexFlag(enum.IntFlag): + A = sre_compile.SRE_FLAG_ASCII + ASCII = A + DEBUG = sre_compile.SRE_FLAG_DEBUG + I = sre_compile.SRE_FLAG_IGNORECASE + IGNORECASE = I + L = sre_compile.SRE_FLAG_LOCALE + LOCALE = L + M = sre_compile.SRE_FLAG_MULTILINE + MULTILINE = M + S = sre_compile.SRE_FLAG_DOTALL + DOTALL = S + X = sre_compile.SRE_FLAG_VERBOSE + VERBOSE = X + U = sre_compile.SRE_FLAG_UNICODE + UNICODE = U + if sys.version_info < (3, 13): + T = sre_compile.SRE_FLAG_TEMPLATE + TEMPLATE = T + if sys.version_info >= (3, 11): + NOFLAG = 0 + +A = RegexFlag.A +ASCII = RegexFlag.ASCII +DEBUG = RegexFlag.DEBUG +I = RegexFlag.I +IGNORECASE = RegexFlag.IGNORECASE +L = RegexFlag.L +LOCALE = RegexFlag.LOCALE +M = RegexFlag.M +MULTILINE = RegexFlag.MULTILINE +S = RegexFlag.S +DOTALL = RegexFlag.DOTALL +X = RegexFlag.X +VERBOSE = RegexFlag.VERBOSE +U = RegexFlag.U +UNICODE = RegexFlag.UNICODE +if sys.version_info < (3, 13): + T = RegexFlag.T + TEMPLATE = RegexFlag.TEMPLATE +if sys.version_info >= (3, 11): + NOFLAG = RegexFlag.NOFLAG +_FlagsType: TypeAlias = int | RegexFlag + +# Type-wise the compile() overloads are unnecessary, they could also be modeled using +# unions in the parameter types. However mypy has a bug regarding TypeVar +# constraints (https://github.com/python/mypy/issues/11880), +# which limits us here because AnyStr is a constrained TypeVar. + +# pattern arguments do *not* accept arbitrary buffers such as bytearray, +# because the pattern must be hashable. +@overload +def compile(pattern: AnyStr, flags: _FlagsType = 0) -> Pattern[AnyStr]: ... +@overload +def compile(pattern: Pattern[AnyStr], flags: _FlagsType = 0) -> Pattern[AnyStr]: ... +@overload +def search(pattern: str | Pattern[str], string: str, flags: _FlagsType = 0) -> Match[str] | None: ... +@overload +def search(pattern: bytes | Pattern[bytes], string: ReadableBuffer, flags: _FlagsType = 0) -> Match[bytes] | None: ... +@overload +def match(pattern: str | Pattern[str], string: str, flags: _FlagsType = 0) -> Match[str] | None: ... +@overload +def match(pattern: bytes | Pattern[bytes], string: ReadableBuffer, flags: _FlagsType = 0) -> Match[bytes] | None: ... +@overload +def fullmatch(pattern: str | Pattern[str], string: str, flags: _FlagsType = 0) -> Match[str] | None: ... +@overload +def fullmatch(pattern: bytes | Pattern[bytes], string: ReadableBuffer, flags: _FlagsType = 0) -> Match[bytes] | None: ... +@overload +def split(pattern: str | Pattern[str], string: str, maxsplit: int = 0, flags: _FlagsType = 0) -> list[str | MaybeNone]: ... +@overload +def split( + pattern: bytes | Pattern[bytes], string: ReadableBuffer, maxsplit: int = 0, flags: _FlagsType = 0 +) -> list[bytes | MaybeNone]: ... +@overload +def findall(pattern: str | Pattern[str], string: str, flags: _FlagsType = 0) -> list[Any]: ... +@overload +def findall(pattern: bytes | Pattern[bytes], string: ReadableBuffer, flags: _FlagsType = 0) -> list[Any]: ... +@overload +def finditer(pattern: str | Pattern[str], string: str, flags: _FlagsType = 0) -> Iterator[Match[str]]: ... +@overload +def finditer(pattern: bytes | Pattern[bytes], string: ReadableBuffer, flags: _FlagsType = 0) -> Iterator[Match[bytes]]: ... +@overload +def sub( + pattern: str | Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0, flags: _FlagsType = 0 +) -> str: ... +@overload +def sub( + pattern: bytes | Pattern[bytes], + repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer], + string: ReadableBuffer, + count: int = 0, + flags: _FlagsType = 0, +) -> bytes: ... +@overload +def subn( + pattern: str | Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0, flags: _FlagsType = 0 +) -> tuple[str, int]: ... +@overload +def subn( + pattern: bytes | Pattern[bytes], + repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer], + string: ReadableBuffer, + count: int = 0, + flags: _FlagsType = 0, +) -> tuple[bytes, int]: ... +def escape(pattern: AnyStr) -> AnyStr: ... +def purge() -> None: ... + +if sys.version_info < (3, 13): + def template(pattern: AnyStr | Pattern[AnyStr], flags: _FlagsType = 0) -> Pattern[AnyStr]: ... diff --git a/mypy/typeshed/stdlib/readline.pyi b/mypy/typeshed/stdlib/readline.pyi new file mode 100644 index 000000000000..7325c267b32c --- /dev/null +++ b/mypy/typeshed/stdlib/readline.pyi @@ -0,0 +1,40 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Callable, Sequence +from typing import Literal +from typing_extensions import TypeAlias + +if sys.platform != "win32": + _Completer: TypeAlias = Callable[[str, int], str | None] + _CompDisp: TypeAlias = Callable[[str, Sequence[str], int], None] + + def parse_and_bind(string: str, /) -> None: ... + def read_init_file(filename: StrOrBytesPath | None = None, /) -> None: ... + def get_line_buffer() -> str: ... + def insert_text(string: str, /) -> None: ... + def redisplay() -> None: ... + def read_history_file(filename: StrOrBytesPath | None = None, /) -> None: ... + def write_history_file(filename: StrOrBytesPath | None = None, /) -> None: ... + def append_history_file(nelements: int, filename: StrOrBytesPath | None = None, /) -> None: ... + def get_history_length() -> int: ... + def set_history_length(length: int, /) -> None: ... + def clear_history() -> None: ... + def get_current_history_length() -> int: ... + def get_history_item(index: int, /) -> str: ... + def remove_history_item(pos: int, /) -> None: ... + def replace_history_item(pos: int, line: str, /) -> None: ... + def add_history(string: str, /) -> None: ... + def set_auto_history(enabled: bool, /) -> None: ... + def set_startup_hook(function: Callable[[], object] | None = None, /) -> None: ... + def set_pre_input_hook(function: Callable[[], object] | None = None, /) -> None: ... + def set_completer(function: _Completer | None = None, /) -> None: ... + def get_completer() -> _Completer | None: ... + def get_completion_type() -> int: ... + def get_begidx() -> int: ... + def get_endidx() -> int: ... + def set_completer_delims(string: str, /) -> None: ... + def get_completer_delims() -> str: ... + def set_completion_display_matches_hook(function: _CompDisp | None = None, /) -> None: ... + + if sys.version_info >= (3, 13): + backend: Literal["readline", "editline"] diff --git a/mypy/typeshed/stdlib/reprlib.pyi b/mypy/typeshed/stdlib/reprlib.pyi new file mode 100644 index 000000000000..68ada6569348 --- /dev/null +++ b/mypy/typeshed/stdlib/reprlib.pyi @@ -0,0 +1,65 @@ +import sys +from array import array +from collections import deque +from collections.abc import Callable +from typing import Any +from typing_extensions import TypeAlias + +__all__ = ["Repr", "repr", "recursive_repr"] + +_ReprFunc: TypeAlias = Callable[[Any], str] + +def recursive_repr(fillvalue: str = "...") -> Callable[[_ReprFunc], _ReprFunc]: ... + +class Repr: + maxlevel: int + maxdict: int + maxlist: int + maxtuple: int + maxset: int + maxfrozenset: int + maxdeque: int + maxarray: int + maxlong: int + maxstring: int + maxother: int + if sys.version_info >= (3, 11): + fillvalue: str + if sys.version_info >= (3, 12): + indent: str | int | None + + if sys.version_info >= (3, 12): + def __init__( + self, + *, + maxlevel: int = 6, + maxtuple: int = 6, + maxlist: int = 6, + maxarray: int = 5, + maxdict: int = 4, + maxset: int = 6, + maxfrozenset: int = 6, + maxdeque: int = 6, + maxstring: int = 30, + maxlong: int = 40, + maxother: int = 30, + fillvalue: str = "...", + indent: str | int | None = None, + ) -> None: ... + + def repr(self, x: Any) -> str: ... + def repr1(self, x: Any, level: int) -> str: ... + def repr_tuple(self, x: tuple[Any, ...], level: int) -> str: ... + def repr_list(self, x: list[Any], level: int) -> str: ... + def repr_array(self, x: array[Any], level: int) -> str: ... + def repr_set(self, x: set[Any], level: int) -> str: ... + def repr_frozenset(self, x: frozenset[Any], level: int) -> str: ... + def repr_deque(self, x: deque[Any], level: int) -> str: ... + def repr_dict(self, x: dict[Any, Any], level: int) -> str: ... + def repr_str(self, x: str, level: int) -> str: ... + def repr_int(self, x: int, level: int) -> str: ... + def repr_instance(self, x: Any, level: int) -> str: ... + +aRepr: Repr + +def repr(x: object) -> str: ... diff --git a/mypy/typeshed/stdlib/resource.pyi b/mypy/typeshed/stdlib/resource.pyi new file mode 100644 index 000000000000..5e468c2cead5 --- /dev/null +++ b/mypy/typeshed/stdlib/resource.pyi @@ -0,0 +1,94 @@ +import sys +from _typeshed import structseq +from typing import Final, final + +if sys.platform != "win32": + RLIMIT_AS: int + RLIMIT_CORE: int + RLIMIT_CPU: int + RLIMIT_DATA: int + RLIMIT_FSIZE: int + RLIMIT_MEMLOCK: int + RLIMIT_NOFILE: int + RLIMIT_NPROC: int + RLIMIT_RSS: int + RLIMIT_STACK: int + RLIM_INFINITY: int + RUSAGE_CHILDREN: int + RUSAGE_SELF: int + if sys.platform == "linux": + RLIMIT_MSGQUEUE: int + RLIMIT_NICE: int + RLIMIT_OFILE: int + RLIMIT_RTPRIO: int + RLIMIT_RTTIME: int + RLIMIT_SIGPENDING: int + RUSAGE_THREAD: int + + @final + class struct_rusage( + structseq[float], tuple[float, float, int, int, int, int, int, int, int, int, int, int, int, int, int, int] + ): + if sys.version_info >= (3, 10): + __match_args__: Final = ( + "ru_utime", + "ru_stime", + "ru_maxrss", + "ru_ixrss", + "ru_idrss", + "ru_isrss", + "ru_minflt", + "ru_majflt", + "ru_nswap", + "ru_inblock", + "ru_oublock", + "ru_msgsnd", + "ru_msgrcv", + "ru_nsignals", + "ru_nvcsw", + "ru_nivcsw", + ) + + @property + def ru_utime(self) -> float: ... + @property + def ru_stime(self) -> float: ... + @property + def ru_maxrss(self) -> int: ... + @property + def ru_ixrss(self) -> int: ... + @property + def ru_idrss(self) -> int: ... + @property + def ru_isrss(self) -> int: ... + @property + def ru_minflt(self) -> int: ... + @property + def ru_majflt(self) -> int: ... + @property + def ru_nswap(self) -> int: ... + @property + def ru_inblock(self) -> int: ... + @property + def ru_oublock(self) -> int: ... + @property + def ru_msgsnd(self) -> int: ... + @property + def ru_msgrcv(self) -> int: ... + @property + def ru_nsignals(self) -> int: ... + @property + def ru_nvcsw(self) -> int: ... + @property + def ru_nivcsw(self) -> int: ... + + def getpagesize() -> int: ... + def getrlimit(resource: int, /) -> tuple[int, int]: ... + def getrusage(who: int, /) -> struct_rusage: ... + def setrlimit(resource: int, limits: tuple[int, int], /) -> None: ... + if sys.platform == "linux": + if sys.version_info >= (3, 12): + def prlimit(pid: int, resource: int, limits: tuple[int, int] | None = None, /) -> tuple[int, int]: ... + else: + def prlimit(pid: int, resource: int, limits: tuple[int, int] = ..., /) -> tuple[int, int]: ... + error = OSError diff --git a/mypy/typeshed/stdlib/rlcompleter.pyi b/mypy/typeshed/stdlib/rlcompleter.pyi new file mode 100644 index 000000000000..8d9477e3ee45 --- /dev/null +++ b/mypy/typeshed/stdlib/rlcompleter.pyi @@ -0,0 +1,9 @@ +from typing import Any + +__all__ = ["Completer"] + +class Completer: + def __init__(self, namespace: dict[str, Any] | None = None) -> None: ... + def complete(self, text: str, state: int) -> str | None: ... + def attr_matches(self, text: str) -> list[str]: ... + def global_matches(self, text: str) -> list[str]: ... diff --git a/mypy/typeshed/stdlib/runpy.pyi b/mypy/typeshed/stdlib/runpy.pyi new file mode 100644 index 000000000000..d4406ea4ac41 --- /dev/null +++ b/mypy/typeshed/stdlib/runpy.pyi @@ -0,0 +1,24 @@ +from _typeshed import Unused +from types import ModuleType +from typing import Any +from typing_extensions import Self + +__all__ = ["run_module", "run_path"] + +class _TempModule: + mod_name: str + module: ModuleType + def __init__(self, mod_name: str) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + +class _ModifiedArgv0: + value: Any + def __init__(self, value: Any) -> None: ... + def __enter__(self) -> None: ... + def __exit__(self, *args: Unused) -> None: ... + +def run_module( + mod_name: str, init_globals: dict[str, Any] | None = None, run_name: str | None = None, alter_sys: bool = False +) -> dict[str, Any]: ... +def run_path(path_name: str, init_globals: dict[str, Any] | None = None, run_name: str | None = None) -> dict[str, Any]: ... diff --git a/mypy/typeshed/stdlib/sched.pyi b/mypy/typeshed/stdlib/sched.pyi new file mode 100644 index 000000000000..ace501430847 --- /dev/null +++ b/mypy/typeshed/stdlib/sched.pyi @@ -0,0 +1,45 @@ +import sys +from collections.abc import Callable +from typing import Any, NamedTuple, type_check_only +from typing_extensions import TypeAlias + +__all__ = ["scheduler"] + +_ActionCallback: TypeAlias = Callable[..., Any] + +if sys.version_info >= (3, 10): + class Event(NamedTuple): + time: float + priority: Any + sequence: int + action: _ActionCallback + argument: tuple[Any, ...] + kwargs: dict[str, Any] + +else: + @type_check_only + class _EventBase(NamedTuple): + time: float + priority: Any + action: _ActionCallback + argument: tuple[Any, ...] + kwargs: dict[str, Any] + + class Event(_EventBase): ... + +class scheduler: + timefunc: Callable[[], float] + delayfunc: Callable[[float], object] + + def __init__(self, timefunc: Callable[[], float] = ..., delayfunc: Callable[[float], object] = ...) -> None: ... + def enterabs( + self, time: float, priority: Any, action: _ActionCallback, argument: tuple[Any, ...] = (), kwargs: dict[str, Any] = ... + ) -> Event: ... + def enter( + self, delay: float, priority: Any, action: _ActionCallback, argument: tuple[Any, ...] = (), kwargs: dict[str, Any] = ... + ) -> Event: ... + def run(self, blocking: bool = True) -> float | None: ... + def cancel(self, event: Event) -> None: ... + def empty(self) -> bool: ... + @property + def queue(self) -> list[Event]: ... diff --git a/mypy/typeshed/stdlib/secrets.pyi b/mypy/typeshed/stdlib/secrets.pyi new file mode 100644 index 000000000000..4861b6f09340 --- /dev/null +++ b/mypy/typeshed/stdlib/secrets.pyi @@ -0,0 +1,15 @@ +from _typeshed import SupportsLenAndGetItem +from hmac import compare_digest as compare_digest +from random import SystemRandom as SystemRandom +from typing import TypeVar + +__all__ = ["choice", "randbelow", "randbits", "SystemRandom", "token_bytes", "token_hex", "token_urlsafe", "compare_digest"] + +_T = TypeVar("_T") + +def randbelow(exclusive_upper_bound: int) -> int: ... +def randbits(k: int) -> int: ... +def choice(seq: SupportsLenAndGetItem[_T]) -> _T: ... +def token_bytes(nbytes: int | None = None) -> bytes: ... +def token_hex(nbytes: int | None = None) -> str: ... +def token_urlsafe(nbytes: int | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/select.pyi b/mypy/typeshed/stdlib/select.pyi new file mode 100644 index 000000000000..6d4c8d8f4c15 --- /dev/null +++ b/mypy/typeshed/stdlib/select.pyi @@ -0,0 +1,155 @@ +import sys +from _typeshed import FileDescriptorLike +from collections.abc import Iterable +from types import TracebackType +from typing import Any, final +from typing_extensions import Self + +if sys.platform != "win32": + PIPE_BUF: int + POLLERR: int + POLLHUP: int + POLLIN: int + POLLMSG: int + POLLNVAL: int + POLLOUT: int + POLLPRI: int + POLLRDBAND: int + if sys.platform == "linux": + POLLRDHUP: int + POLLRDNORM: int + POLLWRBAND: int + POLLWRNORM: int + +class poll: + def register(self, fd: FileDescriptorLike, eventmask: int = ...) -> None: ... + def modify(self, fd: FileDescriptorLike, eventmask: int) -> None: ... + def unregister(self, fd: FileDescriptorLike) -> None: ... + def poll(self, timeout: float | None = ...) -> list[tuple[int, int]]: ... + +def select( + rlist: Iterable[Any], wlist: Iterable[Any], xlist: Iterable[Any], timeout: float | None = None, / +) -> tuple[list[Any], list[Any], list[Any]]: ... + +error = OSError + +if sys.platform != "linux" and sys.platform != "win32": + # BSD only + @final + class kevent: + data: Any + fflags: int + filter: int + flags: int + ident: int + udata: Any + def __init__( + self, + ident: FileDescriptorLike, + filter: int = ..., + flags: int = ..., + fflags: int = ..., + data: Any = ..., + udata: Any = ..., + ) -> None: ... + + # BSD only + @final + class kqueue: + closed: bool + def __init__(self) -> None: ... + def close(self) -> None: ... + def control( + self, changelist: Iterable[kevent] | None, maxevents: int, timeout: float | None = None, / + ) -> list[kevent]: ... + def fileno(self) -> int: ... + @classmethod + def fromfd(cls, fd: FileDescriptorLike, /) -> kqueue: ... + + KQ_EV_ADD: int + KQ_EV_CLEAR: int + KQ_EV_DELETE: int + KQ_EV_DISABLE: int + KQ_EV_ENABLE: int + KQ_EV_EOF: int + KQ_EV_ERROR: int + KQ_EV_FLAG1: int + KQ_EV_ONESHOT: int + KQ_EV_SYSFLAGS: int + KQ_FILTER_AIO: int + KQ_FILTER_NETDEV: int + KQ_FILTER_PROC: int + KQ_FILTER_READ: int + KQ_FILTER_SIGNAL: int + KQ_FILTER_TIMER: int + KQ_FILTER_VNODE: int + KQ_FILTER_WRITE: int + KQ_NOTE_ATTRIB: int + KQ_NOTE_CHILD: int + KQ_NOTE_DELETE: int + KQ_NOTE_EXEC: int + KQ_NOTE_EXIT: int + KQ_NOTE_EXTEND: int + KQ_NOTE_FORK: int + KQ_NOTE_LINK: int + if sys.platform != "darwin": + KQ_NOTE_LINKDOWN: int + KQ_NOTE_LINKINV: int + KQ_NOTE_LINKUP: int + KQ_NOTE_LOWAT: int + KQ_NOTE_PCTRLMASK: int + KQ_NOTE_PDATAMASK: int + KQ_NOTE_RENAME: int + KQ_NOTE_REVOKE: int + KQ_NOTE_TRACK: int + KQ_NOTE_TRACKERR: int + KQ_NOTE_WRITE: int + +if sys.platform == "linux": + @final + class epoll: + def __init__(self, sizehint: int = ..., flags: int = ...) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = ..., + exc_tb: TracebackType | None = None, + /, + ) -> None: ... + def close(self) -> None: ... + closed: bool + def fileno(self) -> int: ... + def register(self, fd: FileDescriptorLike, eventmask: int = ...) -> None: ... + def modify(self, fd: FileDescriptorLike, eventmask: int) -> None: ... + def unregister(self, fd: FileDescriptorLike) -> None: ... + def poll(self, timeout: float | None = None, maxevents: int = -1) -> list[tuple[int, int]]: ... + @classmethod + def fromfd(cls, fd: FileDescriptorLike, /) -> epoll: ... + + EPOLLERR: int + EPOLLEXCLUSIVE: int + EPOLLET: int + EPOLLHUP: int + EPOLLIN: int + EPOLLMSG: int + EPOLLONESHOT: int + EPOLLOUT: int + EPOLLPRI: int + EPOLLRDBAND: int + EPOLLRDHUP: int + EPOLLRDNORM: int + EPOLLWRBAND: int + EPOLLWRNORM: int + EPOLL_CLOEXEC: int + +if sys.platform != "linux" and sys.platform != "darwin" and sys.platform != "win32": + # Solaris only + class devpoll: + def close(self) -> None: ... + closed: bool + def fileno(self) -> int: ... + def register(self, fd: FileDescriptorLike, eventmask: int = ...) -> None: ... + def modify(self, fd: FileDescriptorLike, eventmask: int = ...) -> None: ... + def unregister(self, fd: FileDescriptorLike) -> None: ... + def poll(self, timeout: float | None = ...) -> list[tuple[int, int]]: ... diff --git a/mypy/typeshed/stdlib/selectors.pyi b/mypy/typeshed/stdlib/selectors.pyi new file mode 100644 index 000000000000..a857d0e242ab --- /dev/null +++ b/mypy/typeshed/stdlib/selectors.pyi @@ -0,0 +1,67 @@ +import sys +from _typeshed import FileDescriptor, FileDescriptorLike, Unused +from abc import ABCMeta, abstractmethod +from collections.abc import Mapping +from typing import Any, NamedTuple +from typing_extensions import Self, TypeAlias + +_EventMask: TypeAlias = int + +EVENT_READ: _EventMask +EVENT_WRITE: _EventMask + +class SelectorKey(NamedTuple): + fileobj: FileDescriptorLike + fd: FileDescriptor + events: _EventMask + data: Any + +class BaseSelector(metaclass=ABCMeta): + @abstractmethod + def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... + @abstractmethod + def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... + def modify(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... + @abstractmethod + def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + def close(self) -> None: ... + def get_key(self, fileobj: FileDescriptorLike) -> SelectorKey: ... + @abstractmethod + def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + +class _BaseSelectorImpl(BaseSelector, metaclass=ABCMeta): + def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... + def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... + def modify(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... + def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... + +class SelectSelector(_BaseSelectorImpl): + def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + +class _PollLikeSelector(_BaseSelectorImpl): + def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + +if sys.platform != "win32": + class PollSelector(_PollLikeSelector): ... + +if sys.platform == "linux": + class EpollSelector(_PollLikeSelector): + def fileno(self) -> int: ... + +class DevpollSelector(_PollLikeSelector): + def fileno(self) -> int: ... + +if sys.platform != "win32": + class KqueueSelector(_BaseSelectorImpl): + def fileno(self) -> int: ... + def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + +# Not a real class at runtime, it is just a conditional alias to other real selectors. +# The runtime logic is more fine-grained than a `sys.platform` check; +# not really expressible in the stubs +class DefaultSelector(_BaseSelectorImpl): + def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + if sys.platform != "win32": + def fileno(self) -> int: ... diff --git a/mypy/typeshed/stdlib/shelve.pyi b/mypy/typeshed/stdlib/shelve.pyi new file mode 100644 index 000000000000..654c2ea097f7 --- /dev/null +++ b/mypy/typeshed/stdlib/shelve.pyi @@ -0,0 +1,59 @@ +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Iterator, MutableMapping +from dbm import _TFlags +from types import TracebackType +from typing import Any, TypeVar, overload +from typing_extensions import Self + +__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] + +_T = TypeVar("_T") +_VT = TypeVar("_VT") + +class Shelf(MutableMapping[str, _VT]): + def __init__( + self, dict: MutableMapping[bytes, bytes], protocol: int | None = None, writeback: bool = False, keyencoding: str = "utf-8" + ) -> None: ... + def __iter__(self) -> Iterator[str]: ... + def __len__(self) -> int: ... + @overload # type: ignore[override] + def get(self, key: str, default: None = None) -> _VT | None: ... + @overload + def get(self, key: str, default: _VT) -> _VT: ... + @overload + def get(self, key: str, default: _T) -> _VT | _T: ... + def __getitem__(self, key: str) -> _VT: ... + def __setitem__(self, key: str, value: _VT) -> None: ... + def __delitem__(self, key: str) -> None: ... + def __contains__(self, key: str) -> bool: ... # type: ignore[override] + def __enter__(self) -> Self: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + def __del__(self) -> None: ... + def close(self) -> None: ... + def sync(self) -> None: ... + +class BsdDbShelf(Shelf[_VT]): + def set_location(self, key: str) -> tuple[str, _VT]: ... + def next(self) -> tuple[str, _VT]: ... + def previous(self) -> tuple[str, _VT]: ... + def first(self) -> tuple[str, _VT]: ... + def last(self) -> tuple[str, _VT]: ... + +class DbfilenameShelf(Shelf[_VT]): + if sys.version_info >= (3, 11): + def __init__( + self, filename: StrOrBytesPath, flag: _TFlags = "c", protocol: int | None = None, writeback: bool = False + ) -> None: ... + else: + def __init__(self, filename: str, flag: _TFlags = "c", protocol: int | None = None, writeback: bool = False) -> None: ... + +if sys.version_info >= (3, 11): + def open( + filename: StrOrBytesPath, flag: _TFlags = "c", protocol: int | None = None, writeback: bool = False + ) -> Shelf[Any]: ... + +else: + def open(filename: str, flag: _TFlags = "c", protocol: int | None = None, writeback: bool = False) -> Shelf[Any]: ... diff --git a/mypy/typeshed/stdlib/shlex.pyi b/mypy/typeshed/stdlib/shlex.pyi new file mode 100644 index 000000000000..1c27483782fb --- /dev/null +++ b/mypy/typeshed/stdlib/shlex.pyi @@ -0,0 +1,63 @@ +import sys +from collections import deque +from collections.abc import Iterable +from io import TextIOWrapper +from typing import Literal, Protocol, overload, type_check_only +from typing_extensions import Self, deprecated + +__all__ = ["shlex", "split", "quote", "join"] + +@type_check_only +class _ShlexInstream(Protocol): + def read(self, size: Literal[1], /) -> str: ... + def readline(self) -> object: ... + def close(self) -> object: ... + +if sys.version_info >= (3, 12): + def split(s: str | _ShlexInstream, comments: bool = False, posix: bool = True) -> list[str]: ... + +else: + @overload + def split(s: str | _ShlexInstream, comments: bool = False, posix: bool = True) -> list[str]: ... + @overload + @deprecated("Passing None for 's' to shlex.split() is deprecated and will raise an error in Python 3.12.") + def split(s: None, comments: bool = False, posix: bool = True) -> list[str]: ... + +def join(split_command: Iterable[str]) -> str: ... +def quote(s: str) -> str: ... + +# TODO: Make generic over infile once PEP 696 is implemented. +class shlex: + commenters: str + wordchars: str + whitespace: str + escape: str + quotes: str + escapedquotes: str + whitespace_split: bool + infile: str | None + instream: _ShlexInstream + source: str + debug: int + lineno: int + token: str + filestack: deque[tuple[str | None, _ShlexInstream, int]] + eof: str | None + @property + def punctuation_chars(self) -> str: ... + def __init__( + self, + instream: str | _ShlexInstream | None = None, + infile: str | None = None, + posix: bool = False, + punctuation_chars: bool | str = False, + ) -> None: ... + def get_token(self) -> str | None: ... + def push_token(self, tok: str) -> None: ... + def read_token(self) -> str | None: ... + def sourcehook(self, newfile: str) -> tuple[str, TextIOWrapper] | None: ... + def push_source(self, newstream: str | _ShlexInstream, newfile: str | None = None) -> None: ... + def pop_source(self) -> None: ... + def error_leader(self, infile: str | None = None, lineno: int | None = None) -> str: ... + def __iter__(self) -> Self: ... + def __next__(self) -> str: ... diff --git a/mypy/typeshed/stdlib/shutil.pyi b/mypy/typeshed/stdlib/shutil.pyi new file mode 100644 index 000000000000..dcff18d110bd --- /dev/null +++ b/mypy/typeshed/stdlib/shutil.pyi @@ -0,0 +1,230 @@ +import os +import sys +from _typeshed import BytesPath, ExcInfo, FileDescriptorOrPath, StrOrBytesPath, StrPath, SupportsRead, SupportsWrite +from collections.abc import Callable, Iterable, Sequence +from tarfile import _TarfileFilter +from typing import Any, AnyStr, NamedTuple, Protocol, TypeVar, overload +from typing_extensions import TypeAlias, deprecated + +__all__ = [ + "copyfileobj", + "copyfile", + "copymode", + "copystat", + "copy", + "copy2", + "copytree", + "move", + "rmtree", + "Error", + "SpecialFileError", + "ExecError", + "make_archive", + "get_archive_formats", + "register_archive_format", + "unregister_archive_format", + "get_unpack_formats", + "register_unpack_format", + "unregister_unpack_format", + "unpack_archive", + "ignore_patterns", + "chown", + "which", + "get_terminal_size", + "SameFileError", + "disk_usage", +] + +_StrOrBytesPathT = TypeVar("_StrOrBytesPathT", bound=StrOrBytesPath) +_StrPathT = TypeVar("_StrPathT", bound=StrPath) +# Return value of some functions that may either return a path-like object that was passed in or +# a string +_PathReturn: TypeAlias = Any + +class Error(OSError): ... +class SameFileError(Error): ... +class SpecialFileError(OSError): ... +class ExecError(OSError): ... +class ReadError(OSError): ... +class RegistryError(Exception): ... + +def copyfileobj(fsrc: SupportsRead[AnyStr], fdst: SupportsWrite[AnyStr], length: int = 0) -> None: ... +def copyfile(src: StrOrBytesPath, dst: _StrOrBytesPathT, *, follow_symlinks: bool = True) -> _StrOrBytesPathT: ... +def copymode(src: StrOrBytesPath, dst: StrOrBytesPath, *, follow_symlinks: bool = True) -> None: ... +def copystat(src: StrOrBytesPath, dst: StrOrBytesPath, *, follow_symlinks: bool = True) -> None: ... +@overload +def copy(src: StrPath, dst: StrPath, *, follow_symlinks: bool = True) -> _PathReturn: ... +@overload +def copy(src: BytesPath, dst: BytesPath, *, follow_symlinks: bool = True) -> _PathReturn: ... +@overload +def copy2(src: StrPath, dst: StrPath, *, follow_symlinks: bool = True) -> _PathReturn: ... +@overload +def copy2(src: BytesPath, dst: BytesPath, *, follow_symlinks: bool = True) -> _PathReturn: ... +def ignore_patterns(*patterns: StrPath) -> Callable[[Any, list[str]], set[str]]: ... +def copytree( + src: StrPath, + dst: StrPath, + symlinks: bool = False, + ignore: None | Callable[[str, list[str]], Iterable[str]] | Callable[[StrPath, list[str]], Iterable[str]] = None, + copy_function: Callable[[str, str], object] = ..., + ignore_dangling_symlinks: bool = False, + dirs_exist_ok: bool = False, +) -> _PathReturn: ... + +_OnErrorCallback: TypeAlias = Callable[[Callable[..., Any], str, ExcInfo], object] +_OnExcCallback: TypeAlias = Callable[[Callable[..., Any], str, BaseException], object] + +class _RmtreeType(Protocol): + avoids_symlink_attacks: bool + if sys.version_info >= (3, 12): + @overload + @deprecated("The `onerror` parameter is deprecated. Use `onexc` instead.") + def __call__( + self, + path: StrOrBytesPath, + ignore_errors: bool, + onerror: _OnErrorCallback, + *, + onexc: None = None, + dir_fd: int | None = None, + ) -> None: ... + @overload + @deprecated("The `onerror` parameter is deprecated. Use `onexc` instead.") + def __call__( + self, + path: StrOrBytesPath, + ignore_errors: bool = False, + *, + onerror: _OnErrorCallback, + onexc: None = None, + dir_fd: int | None = None, + ) -> None: ... + @overload + def __call__( + self, + path: StrOrBytesPath, + ignore_errors: bool = False, + *, + onexc: _OnExcCallback | None = None, + dir_fd: int | None = None, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __call__( + self, + path: StrOrBytesPath, + ignore_errors: bool = False, + onerror: _OnErrorCallback | None = None, + *, + dir_fd: int | None = None, + ) -> None: ... + + else: + def __call__( + self, path: StrOrBytesPath, ignore_errors: bool = False, onerror: _OnErrorCallback | None = None + ) -> None: ... + +rmtree: _RmtreeType + +_CopyFn: TypeAlias = Callable[[str, str], object] | Callable[[StrPath, StrPath], object] + +# N.B. shutil.move appears to take bytes arguments, however, +# this does not work when dst is (or is within) an existing directory. +# (#6832) +if sys.version_info >= (3, 9): + def move(src: StrPath, dst: StrPath, copy_function: _CopyFn = ...) -> _PathReturn: ... + +else: + # See https://bugs.python.org/issue32689 + def move(src: str, dst: StrPath, copy_function: _CopyFn = ...) -> _PathReturn: ... + +class _ntuple_diskusage(NamedTuple): + total: int + used: int + free: int + +def disk_usage(path: FileDescriptorOrPath) -> _ntuple_diskusage: ... + +# While chown can be imported on Windows, it doesn't actually work; +# see https://bugs.python.org/issue33140. We keep it here because it's +# in __all__. +if sys.version_info >= (3, 13): + @overload + def chown( + path: FileDescriptorOrPath, + user: str | int, + group: None = None, + *, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> None: ... + @overload + def chown( + path: FileDescriptorOrPath, + user: None = None, + *, + group: str | int, + dir_fd: int | None = None, + follow_symlinks: bool = True, + ) -> None: ... + @overload + def chown( + path: FileDescriptorOrPath, user: None, group: str | int, *, dir_fd: int | None = None, follow_symlinks: bool = True + ) -> None: ... + @overload + def chown( + path: FileDescriptorOrPath, user: str | int, group: str | int, *, dir_fd: int | None = None, follow_symlinks: bool = True + ) -> None: ... + +else: + @overload + def chown(path: FileDescriptorOrPath, user: str | int, group: None = None) -> None: ... + @overload + def chown(path: FileDescriptorOrPath, user: None = None, *, group: str | int) -> None: ... + @overload + def chown(path: FileDescriptorOrPath, user: None, group: str | int) -> None: ... + @overload + def chown(path: FileDescriptorOrPath, user: str | int, group: str | int) -> None: ... + +@overload +def which(cmd: _StrPathT, mode: int = 1, path: StrPath | None = None) -> str | _StrPathT | None: ... +@overload +def which(cmd: bytes, mode: int = 1, path: StrPath | None = None) -> bytes | None: ... +def make_archive( + base_name: str, + format: str, + root_dir: StrPath | None = None, + base_dir: StrPath | None = None, + verbose: bool = ..., + dry_run: bool = ..., + owner: str | None = None, + group: str | None = None, + logger: Any | None = None, +) -> str: ... +def get_archive_formats() -> list[tuple[str, str]]: ... +@overload +def register_archive_format( + name: str, function: Callable[..., object], extra_args: Sequence[tuple[str, Any] | list[Any]], description: str = "" +) -> None: ... +@overload +def register_archive_format( + name: str, function: Callable[[str, str], object], extra_args: None = None, description: str = "" +) -> None: ... +def unregister_archive_format(name: str) -> None: ... +def unpack_archive( + filename: StrPath, extract_dir: StrPath | None = None, format: str | None = None, *, filter: _TarfileFilter | None = None +) -> None: ... +@overload +def register_unpack_format( + name: str, + extensions: list[str], + function: Callable[..., object], + extra_args: Sequence[tuple[str, Any]], + description: str = "", +) -> None: ... +@overload +def register_unpack_format( + name: str, extensions: list[str], function: Callable[[str, str], object], extra_args: None = None, description: str = "" +) -> None: ... +def unregister_unpack_format(name: str) -> None: ... +def get_unpack_formats() -> list[tuple[str, list[str], str]]: ... +def get_terminal_size(fallback: tuple[int, int] = (80, 24)) -> os.terminal_size: ... diff --git a/mypy/typeshed/stdlib/signal.pyi b/mypy/typeshed/stdlib/signal.pyi new file mode 100644 index 000000000000..2e3ac5bf24c3 --- /dev/null +++ b/mypy/typeshed/stdlib/signal.pyi @@ -0,0 +1,188 @@ +import sys +from _typeshed import structseq +from collections.abc import Callable, Iterable +from enum import IntEnum +from types import FrameType +from typing import Any, Final, final +from typing_extensions import Never, TypeAlias + +NSIG: int + +class Signals(IntEnum): + SIGABRT = 6 + SIGFPE = 8 + SIGILL = 4 + SIGINT = 2 + SIGSEGV = 11 + SIGTERM = 15 + + if sys.platform == "win32": + SIGBREAK = 21 + CTRL_C_EVENT = 0 + CTRL_BREAK_EVENT = 1 + else: + SIGALRM = 14 + SIGBUS = 7 + SIGCHLD = 17 + SIGCONT = 18 + SIGHUP = 1 + SIGIO = 29 + SIGIOT = 6 + SIGKILL = 9 + SIGPIPE = 13 + SIGPROF = 27 + SIGQUIT = 3 + SIGSTOP = 19 + SIGSYS = 31 + SIGTRAP = 5 + SIGTSTP = 20 + SIGTTIN = 21 + SIGTTOU = 22 + SIGURG = 23 + SIGUSR1 = 10 + SIGUSR2 = 12 + SIGVTALRM = 26 + SIGWINCH = 28 + SIGXCPU = 24 + SIGXFSZ = 25 + if sys.platform != "linux": + SIGEMT = 7 + SIGINFO = 29 + if sys.platform != "darwin": + SIGCLD = 17 + SIGPOLL = 29 + SIGPWR = 30 + SIGRTMAX = 64 + SIGRTMIN = 34 + if sys.version_info >= (3, 11): + SIGSTKFLT = 16 + +class Handlers(IntEnum): + SIG_DFL = 0 + SIG_IGN = 1 + +SIG_DFL: Handlers +SIG_IGN: Handlers + +_SIGNUM: TypeAlias = int | Signals +_HANDLER: TypeAlias = Callable[[int, FrameType | None], Any] | int | Handlers | None + +def default_int_handler(signalnum: int, frame: FrameType | None, /) -> Never: ... + +if sys.version_info >= (3, 10): # arguments changed in 3.10.2 + def getsignal(signalnum: _SIGNUM) -> _HANDLER: ... + def signal(signalnum: _SIGNUM, handler: _HANDLER) -> _HANDLER: ... + +else: + def getsignal(signalnum: _SIGNUM, /) -> _HANDLER: ... + def signal(signalnum: _SIGNUM, handler: _HANDLER, /) -> _HANDLER: ... + +SIGABRT: Signals +SIGFPE: Signals +SIGILL: Signals +SIGINT: Signals +SIGSEGV: Signals +SIGTERM: Signals + +if sys.platform == "win32": + SIGBREAK: Signals + CTRL_C_EVENT: Signals + CTRL_BREAK_EVENT: Signals +else: + if sys.platform != "linux": + SIGINFO: Signals + SIGEMT: Signals + SIGALRM: Signals + SIGBUS: Signals + SIGCHLD: Signals + SIGCONT: Signals + SIGHUP: Signals + SIGIO: Signals + SIGIOT: Signals + SIGKILL: Signals + SIGPIPE: Signals + SIGPROF: Signals + SIGQUIT: Signals + SIGSTOP: Signals + SIGSYS: Signals + SIGTRAP: Signals + SIGTSTP: Signals + SIGTTIN: Signals + SIGTTOU: Signals + SIGURG: Signals + SIGUSR1: Signals + SIGUSR2: Signals + SIGVTALRM: Signals + SIGWINCH: Signals + SIGXCPU: Signals + SIGXFSZ: Signals + + class ItimerError(OSError): ... + ITIMER_PROF: int + ITIMER_REAL: int + ITIMER_VIRTUAL: int + + class Sigmasks(IntEnum): + SIG_BLOCK = 0 + SIG_UNBLOCK = 1 + SIG_SETMASK = 2 + + SIG_BLOCK = Sigmasks.SIG_BLOCK + SIG_UNBLOCK = Sigmasks.SIG_UNBLOCK + SIG_SETMASK = Sigmasks.SIG_SETMASK + def alarm(seconds: int, /) -> int: ... + def getitimer(which: int, /) -> tuple[float, float]: ... + def pause() -> None: ... + def pthread_kill(thread_id: int, signalnum: int, /) -> None: ... + if sys.version_info >= (3, 10): # arguments changed in 3.10.2 + def pthread_sigmask(how: int, mask: Iterable[int]) -> set[_SIGNUM]: ... + else: + def pthread_sigmask(how: int, mask: Iterable[int], /) -> set[_SIGNUM]: ... + + def setitimer(which: int, seconds: float, interval: float = 0.0, /) -> tuple[float, float]: ... + def siginterrupt(signalnum: int, flag: bool, /) -> None: ... + def sigpending() -> Any: ... + if sys.version_info >= (3, 10): # argument changed in 3.10.2 + def sigwait(sigset: Iterable[int]) -> _SIGNUM: ... + else: + def sigwait(sigset: Iterable[int], /) -> _SIGNUM: ... + if sys.platform != "darwin": + SIGCLD: Signals + SIGPOLL: Signals + SIGPWR: Signals + SIGRTMAX: Signals + SIGRTMIN: Signals + if sys.version_info >= (3, 11): + SIGSTKFLT: Signals + + @final + class struct_siginfo(structseq[int], tuple[int, int, int, int, int, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("si_signo", "si_code", "si_errno", "si_pid", "si_uid", "si_status", "si_band") + + @property + def si_signo(self) -> int: ... + @property + def si_code(self) -> int: ... + @property + def si_errno(self) -> int: ... + @property + def si_pid(self) -> int: ... + @property + def si_uid(self) -> int: ... + @property + def si_status(self) -> int: ... + @property + def si_band(self) -> int: ... + + def sigtimedwait(sigset: Iterable[int], timeout: float, /) -> struct_siginfo | None: ... + def sigwaitinfo(sigset: Iterable[int], /) -> struct_siginfo: ... + +def strsignal(signalnum: _SIGNUM, /) -> str | None: ... +def valid_signals() -> set[Signals]: ... +def raise_signal(signalnum: _SIGNUM, /) -> None: ... +def set_wakeup_fd(fd: int, /, *, warn_on_full_buffer: bool = ...) -> int: ... + +if sys.version_info >= (3, 9): + if sys.platform == "linux": + def pidfd_send_signal(pidfd: int, sig: int, siginfo: None = None, flags: int = ..., /) -> None: ... diff --git a/mypy/typeshed/stdlib/site.pyi b/mypy/typeshed/stdlib/site.pyi new file mode 100644 index 000000000000..6e39677aaea0 --- /dev/null +++ b/mypy/typeshed/stdlib/site.pyi @@ -0,0 +1,36 @@ +import sys +from _typeshed import StrPath +from collections.abc import Iterable + +PREFIXES: list[str] +ENABLE_USER_SITE: bool | None +USER_SITE: str | None +USER_BASE: str | None + +def main() -> None: ... +def abs_paths() -> None: ... # undocumented +def addpackage(sitedir: StrPath, name: StrPath, known_paths: set[str] | None) -> set[str] | None: ... # undocumented +def addsitedir(sitedir: str, known_paths: set[str] | None = None) -> None: ... +def addsitepackages(known_paths: set[str] | None, prefixes: Iterable[str] | None = None) -> set[str] | None: ... # undocumented +def addusersitepackages(known_paths: set[str] | None) -> set[str] | None: ... # undocumented +def check_enableusersite() -> bool | None: ... # undocumented + +if sys.version_info >= (3, 13): + def gethistoryfile() -> str: ... # undocumented + +def enablerlcompleter() -> None: ... # undocumented + +if sys.version_info >= (3, 13): + def register_readline() -> None: ... # undocumented + +def execsitecustomize() -> None: ... # undocumented +def execusercustomize() -> None: ... # undocumented +def getsitepackages(prefixes: Iterable[str] | None = None) -> list[str]: ... +def getuserbase() -> str: ... +def getusersitepackages() -> str: ... +def makepath(*paths: StrPath) -> tuple[str, str]: ... # undocumented +def removeduppaths() -> set[str]: ... # undocumented +def setcopyright() -> None: ... # undocumented +def sethelper() -> None: ... # undocumented +def setquit() -> None: ... # undocumented +def venv(known_paths: set[str] | None) -> set[str] | None: ... # undocumented diff --git a/mypy/typeshed/stdlib/smtpd.pyi b/mypy/typeshed/stdlib/smtpd.pyi new file mode 100644 index 000000000000..7392bd51627d --- /dev/null +++ b/mypy/typeshed/stdlib/smtpd.pyi @@ -0,0 +1,91 @@ +import asynchat +import asyncore +import socket +import sys +from collections import defaultdict +from typing import Any +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 11): + __all__ = ["SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy"] +else: + __all__ = ["SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", "MailmanProxy"] + +_Address: TypeAlias = tuple[str, int] # (host, port) + +class SMTPChannel(asynchat.async_chat): + COMMAND: int + DATA: int + + command_size_limits: defaultdict[str, int] + smtp_server: SMTPServer + conn: socket.socket + addr: Any + received_lines: list[str] + smtp_state: int + seen_greeting: str + mailfrom: str + rcpttos: list[str] + received_data: str + fqdn: str + peer: str + + command_size_limit: int + data_size_limit: int + + enable_SMTPUTF8: bool + @property + def max_command_size_limit(self) -> int: ... + def __init__( + self, + server: SMTPServer, + conn: socket.socket, + addr: Any, + data_size_limit: int = 33554432, + map: asyncore._MapType | None = None, + enable_SMTPUTF8: bool = False, + decode_data: bool = False, + ) -> None: ... + # base asynchat.async_chat.push() accepts bytes + def push(self, msg: str) -> None: ... # type: ignore[override] + def collect_incoming_data(self, data: bytes) -> None: ... + def found_terminator(self) -> None: ... + def smtp_HELO(self, arg: str) -> None: ... + def smtp_NOOP(self, arg: str) -> None: ... + def smtp_QUIT(self, arg: str) -> None: ... + def smtp_MAIL(self, arg: str) -> None: ... + def smtp_RCPT(self, arg: str) -> None: ... + def smtp_RSET(self, arg: str) -> None: ... + def smtp_DATA(self, arg: str) -> None: ... + def smtp_EHLO(self, arg: str) -> None: ... + def smtp_HELP(self, arg: str) -> None: ... + def smtp_VRFY(self, arg: str) -> None: ... + def smtp_EXPN(self, arg: str) -> None: ... + +class SMTPServer(asyncore.dispatcher): + channel_class: type[SMTPChannel] + + data_size_limit: int + enable_SMTPUTF8: bool + def __init__( + self, + localaddr: _Address, + remoteaddr: _Address, + data_size_limit: int = 33554432, + map: asyncore._MapType | None = None, + enable_SMTPUTF8: bool = False, + decode_data: bool = False, + ) -> None: ... + def handle_accepted(self, conn: socket.socket, addr: Any) -> None: ... + def process_message( + self, peer: _Address, mailfrom: str, rcpttos: list[str], data: bytes | str, **kwargs: Any + ) -> str | None: ... + +class DebuggingServer(SMTPServer): ... + +class PureProxy(SMTPServer): + def process_message(self, peer: _Address, mailfrom: str, rcpttos: list[str], data: bytes | str) -> str | None: ... # type: ignore[override] + +if sys.version_info < (3, 11): + class MailmanProxy(PureProxy): + def process_message(self, peer: _Address, mailfrom: str, rcpttos: list[str], data: bytes | str) -> str | None: ... # type: ignore[override] diff --git a/mypy/typeshed/stdlib/smtplib.pyi b/mypy/typeshed/stdlib/smtplib.pyi new file mode 100644 index 000000000000..a762427bcab3 --- /dev/null +++ b/mypy/typeshed/stdlib/smtplib.pyi @@ -0,0 +1,204 @@ +import sys +from _socket import _Address as _SourceAddress +from _typeshed import ReadableBuffer, SizedBuffer +from collections.abc import Sequence +from email.message import Message as _Message +from re import Pattern +from socket import socket +from ssl import SSLContext +from types import TracebackType +from typing import Any, Protocol, overload +from typing_extensions import Self, TypeAlias + +__all__ = [ + "SMTPException", + "SMTPServerDisconnected", + "SMTPResponseException", + "SMTPSenderRefused", + "SMTPRecipientsRefused", + "SMTPDataError", + "SMTPConnectError", + "SMTPHeloError", + "SMTPAuthenticationError", + "quoteaddr", + "quotedata", + "SMTP", + "SMTP_SSL", + "SMTPNotSupportedError", +] + +_Reply: TypeAlias = tuple[int, bytes] +_SendErrs: TypeAlias = dict[str, _Reply] + +SMTP_PORT: int +SMTP_SSL_PORT: int +CRLF: str +bCRLF: bytes + +OLDSTYLE_AUTH: Pattern[str] + +class SMTPException(OSError): ... +class SMTPNotSupportedError(SMTPException): ... +class SMTPServerDisconnected(SMTPException): ... + +class SMTPResponseException(SMTPException): + smtp_code: int + smtp_error: bytes | str + args: tuple[int, bytes | str] | tuple[int, bytes, str] + def __init__(self, code: int, msg: bytes | str) -> None: ... + +class SMTPSenderRefused(SMTPResponseException): + smtp_error: bytes + sender: str + args: tuple[int, bytes, str] + def __init__(self, code: int, msg: bytes, sender: str) -> None: ... + +class SMTPRecipientsRefused(SMTPException): + recipients: _SendErrs + args: tuple[_SendErrs] + def __init__(self, recipients: _SendErrs) -> None: ... + +class SMTPDataError(SMTPResponseException): ... +class SMTPConnectError(SMTPResponseException): ... +class SMTPHeloError(SMTPResponseException): ... +class SMTPAuthenticationError(SMTPResponseException): ... + +def quoteaddr(addrstring: str) -> str: ... +def quotedata(data: str) -> str: ... + +class _AuthObject(Protocol): + @overload + def __call__(self, challenge: None = None, /) -> str | None: ... + @overload + def __call__(self, challenge: bytes, /) -> str: ... + +class SMTP: + debuglevel: int + sock: socket | None + # Type of file should match what socket.makefile() returns + file: Any | None + helo_resp: bytes | None + ehlo_msg: str + ehlo_resp: bytes | None + does_esmtp: bool + default_port: int + timeout: float + esmtp_features: dict[str, str] + command_encoding: str + source_address: _SourceAddress | None + local_hostname: str + def __init__( + self, + host: str = "", + port: int = 0, + local_hostname: str | None = None, + timeout: float = ..., + source_address: _SourceAddress | None = None, + ) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, tb: TracebackType | None + ) -> None: ... + def set_debuglevel(self, debuglevel: int) -> None: ... + def connect(self, host: str = "localhost", port: int = 0, source_address: _SourceAddress | None = None) -> _Reply: ... + def send(self, s: ReadableBuffer | str) -> None: ... + def putcmd(self, cmd: str, args: str = "") -> None: ... + def getreply(self) -> _Reply: ... + def docmd(self, cmd: str, args: str = "") -> _Reply: ... + def helo(self, name: str = "") -> _Reply: ... + def ehlo(self, name: str = "") -> _Reply: ... + def has_extn(self, opt: str) -> bool: ... + def help(self, args: str = "") -> bytes: ... + def rset(self) -> _Reply: ... + def noop(self) -> _Reply: ... + def mail(self, sender: str, options: Sequence[str] = ()) -> _Reply: ... + def rcpt(self, recip: str, options: Sequence[str] = ()) -> _Reply: ... + def data(self, msg: ReadableBuffer | str) -> _Reply: ... + def verify(self, address: str) -> _Reply: ... + vrfy = verify + def expn(self, address: str) -> _Reply: ... + def ehlo_or_helo_if_needed(self) -> None: ... + user: str + password: str + def auth(self, mechanism: str, authobject: _AuthObject, *, initial_response_ok: bool = True) -> _Reply: ... + @overload + def auth_cram_md5(self, challenge: None = None) -> None: ... + @overload + def auth_cram_md5(self, challenge: ReadableBuffer) -> str: ... + def auth_plain(self, challenge: ReadableBuffer | None = None) -> str: ... + def auth_login(self, challenge: ReadableBuffer | None = None) -> str: ... + def login(self, user: str, password: str, *, initial_response_ok: bool = True) -> _Reply: ... + if sys.version_info >= (3, 12): + def starttls(self, *, context: SSLContext | None = None) -> _Reply: ... + else: + def starttls( + self, keyfile: str | None = None, certfile: str | None = None, context: SSLContext | None = None + ) -> _Reply: ... + + def sendmail( + self, + from_addr: str, + to_addrs: str | Sequence[str], + msg: SizedBuffer | str, + mail_options: Sequence[str] = (), + rcpt_options: Sequence[str] = (), + ) -> _SendErrs: ... + def send_message( + self, + msg: _Message, + from_addr: str | None = None, + to_addrs: str | Sequence[str] | None = None, + mail_options: Sequence[str] = (), + rcpt_options: Sequence[str] = (), + ) -> _SendErrs: ... + def close(self) -> None: ... + def quit(self) -> _Reply: ... + +class SMTP_SSL(SMTP): + keyfile: str | None + certfile: str | None + context: SSLContext + if sys.version_info >= (3, 12): + def __init__( + self, + host: str = "", + port: int = 0, + local_hostname: str | None = None, + *, + timeout: float = ..., + source_address: _SourceAddress | None = None, + context: SSLContext | None = None, + ) -> None: ... + else: + def __init__( + self, + host: str = "", + port: int = 0, + local_hostname: str | None = None, + keyfile: str | None = None, + certfile: str | None = None, + timeout: float = ..., + source_address: _SourceAddress | None = None, + context: SSLContext | None = None, + ) -> None: ... + +LMTP_PORT: int + +class LMTP(SMTP): + if sys.version_info >= (3, 9): + def __init__( + self, + host: str = "", + port: int = 2003, + local_hostname: str | None = None, + source_address: _SourceAddress | None = None, + timeout: float = ..., + ) -> None: ... + else: + def __init__( + self, + host: str = "", + port: int = 2003, + local_hostname: str | None = None, + source_address: _SourceAddress | None = None, + ) -> None: ... diff --git a/mypy/typeshed/stdlib/sndhdr.pyi b/mypy/typeshed/stdlib/sndhdr.pyi new file mode 100644 index 000000000000..f4d487607fbb --- /dev/null +++ b/mypy/typeshed/stdlib/sndhdr.pyi @@ -0,0 +1,14 @@ +from _typeshed import StrOrBytesPath +from typing import NamedTuple + +__all__ = ["what", "whathdr"] + +class SndHeaders(NamedTuple): + filetype: str + framerate: int + nchannels: int + nframes: int + sampwidth: int | str + +def what(filename: StrOrBytesPath) -> SndHeaders | None: ... +def whathdr(filename: StrOrBytesPath) -> SndHeaders | None: ... diff --git a/mypy/typeshed/stdlib/socket.pyi b/mypy/typeshed/stdlib/socket.pyi new file mode 100644 index 000000000000..e42bba757fc3 --- /dev/null +++ b/mypy/typeshed/stdlib/socket.pyi @@ -0,0 +1,1403 @@ +# Ideally, we'd just do "from _socket import *". Unfortunately, socket +# overrides some definitions from _socket incompatibly. mypy incorrectly +# prefers the definitions from _socket over those defined here. +import _socket +import sys +from _socket import ( + CAPI as CAPI, + EAI_AGAIN as EAI_AGAIN, + EAI_BADFLAGS as EAI_BADFLAGS, + EAI_FAIL as EAI_FAIL, + EAI_FAMILY as EAI_FAMILY, + EAI_MEMORY as EAI_MEMORY, + EAI_NODATA as EAI_NODATA, + EAI_NONAME as EAI_NONAME, + EAI_SERVICE as EAI_SERVICE, + EAI_SOCKTYPE as EAI_SOCKTYPE, + INADDR_ALLHOSTS_GROUP as INADDR_ALLHOSTS_GROUP, + INADDR_ANY as INADDR_ANY, + INADDR_BROADCAST as INADDR_BROADCAST, + INADDR_LOOPBACK as INADDR_LOOPBACK, + INADDR_MAX_LOCAL_GROUP as INADDR_MAX_LOCAL_GROUP, + INADDR_NONE as INADDR_NONE, + INADDR_UNSPEC_GROUP as INADDR_UNSPEC_GROUP, + IP_ADD_MEMBERSHIP as IP_ADD_MEMBERSHIP, + IP_DROP_MEMBERSHIP as IP_DROP_MEMBERSHIP, + IP_HDRINCL as IP_HDRINCL, + IP_MULTICAST_IF as IP_MULTICAST_IF, + IP_MULTICAST_LOOP as IP_MULTICAST_LOOP, + IP_MULTICAST_TTL as IP_MULTICAST_TTL, + IP_OPTIONS as IP_OPTIONS, + IP_TOS as IP_TOS, + IP_TTL as IP_TTL, + IPPORT_RESERVED as IPPORT_RESERVED, + IPPORT_USERRESERVED as IPPORT_USERRESERVED, + IPPROTO_AH as IPPROTO_AH, + IPPROTO_DSTOPTS as IPPROTO_DSTOPTS, + IPPROTO_EGP as IPPROTO_EGP, + IPPROTO_ESP as IPPROTO_ESP, + IPPROTO_FRAGMENT as IPPROTO_FRAGMENT, + IPPROTO_HOPOPTS as IPPROTO_HOPOPTS, + IPPROTO_ICMP as IPPROTO_ICMP, + IPPROTO_ICMPV6 as IPPROTO_ICMPV6, + IPPROTO_IDP as IPPROTO_IDP, + IPPROTO_IGMP as IPPROTO_IGMP, + IPPROTO_IP as IPPROTO_IP, + IPPROTO_IPV6 as IPPROTO_IPV6, + IPPROTO_NONE as IPPROTO_NONE, + IPPROTO_PIM as IPPROTO_PIM, + IPPROTO_PUP as IPPROTO_PUP, + IPPROTO_RAW as IPPROTO_RAW, + IPPROTO_ROUTING as IPPROTO_ROUTING, + IPPROTO_SCTP as IPPROTO_SCTP, + IPPROTO_TCP as IPPROTO_TCP, + IPPROTO_UDP as IPPROTO_UDP, + IPV6_CHECKSUM as IPV6_CHECKSUM, + IPV6_JOIN_GROUP as IPV6_JOIN_GROUP, + IPV6_LEAVE_GROUP as IPV6_LEAVE_GROUP, + IPV6_MULTICAST_HOPS as IPV6_MULTICAST_HOPS, + IPV6_MULTICAST_IF as IPV6_MULTICAST_IF, + IPV6_MULTICAST_LOOP as IPV6_MULTICAST_LOOP, + IPV6_RECVTCLASS as IPV6_RECVTCLASS, + IPV6_TCLASS as IPV6_TCLASS, + IPV6_UNICAST_HOPS as IPV6_UNICAST_HOPS, + IPV6_V6ONLY as IPV6_V6ONLY, + NI_DGRAM as NI_DGRAM, + NI_MAXHOST as NI_MAXHOST, + NI_MAXSERV as NI_MAXSERV, + NI_NAMEREQD as NI_NAMEREQD, + NI_NOFQDN as NI_NOFQDN, + NI_NUMERICHOST as NI_NUMERICHOST, + NI_NUMERICSERV as NI_NUMERICSERV, + SHUT_RD as SHUT_RD, + SHUT_RDWR as SHUT_RDWR, + SHUT_WR as SHUT_WR, + SO_ACCEPTCONN as SO_ACCEPTCONN, + SO_BROADCAST as SO_BROADCAST, + SO_DEBUG as SO_DEBUG, + SO_DONTROUTE as SO_DONTROUTE, + SO_ERROR as SO_ERROR, + SO_KEEPALIVE as SO_KEEPALIVE, + SO_LINGER as SO_LINGER, + SO_OOBINLINE as SO_OOBINLINE, + SO_RCVBUF as SO_RCVBUF, + SO_RCVLOWAT as SO_RCVLOWAT, + SO_RCVTIMEO as SO_RCVTIMEO, + SO_REUSEADDR as SO_REUSEADDR, + SO_SNDBUF as SO_SNDBUF, + SO_SNDLOWAT as SO_SNDLOWAT, + SO_SNDTIMEO as SO_SNDTIMEO, + SO_TYPE as SO_TYPE, + SOL_IP as SOL_IP, + SOL_SOCKET as SOL_SOCKET, + SOL_TCP as SOL_TCP, + SOL_UDP as SOL_UDP, + SOMAXCONN as SOMAXCONN, + TCP_FASTOPEN as TCP_FASTOPEN, + TCP_KEEPCNT as TCP_KEEPCNT, + TCP_KEEPINTVL as TCP_KEEPINTVL, + TCP_MAXSEG as TCP_MAXSEG, + TCP_NODELAY as TCP_NODELAY, + SocketType as SocketType, + _Address as _Address, + _RetAddress as _RetAddress, + close as close, + dup as dup, + getdefaulttimeout as getdefaulttimeout, + gethostbyaddr as gethostbyaddr, + gethostbyname as gethostbyname, + gethostbyname_ex as gethostbyname_ex, + gethostname as gethostname, + getnameinfo as getnameinfo, + getprotobyname as getprotobyname, + getservbyname as getservbyname, + getservbyport as getservbyport, + has_ipv6 as has_ipv6, + htonl as htonl, + htons as htons, + if_indextoname as if_indextoname, + if_nameindex as if_nameindex, + if_nametoindex as if_nametoindex, + inet_aton as inet_aton, + inet_ntoa as inet_ntoa, + inet_ntop as inet_ntop, + inet_pton as inet_pton, + ntohl as ntohl, + ntohs as ntohs, + setdefaulttimeout as setdefaulttimeout, +) +from _typeshed import ReadableBuffer, Unused, WriteableBuffer +from collections.abc import Iterable +from enum import IntEnum, IntFlag +from io import BufferedReader, BufferedRWPair, BufferedWriter, IOBase, RawIOBase, TextIOWrapper +from typing import Any, Literal, Protocol, SupportsIndex, overload +from typing_extensions import Self + +__all__ = [ + "fromfd", + "getfqdn", + "create_connection", + "create_server", + "has_dualstack_ipv6", + "AddressFamily", + "SocketKind", + "AF_APPLETALK", + "AF_DECnet", + "AF_INET", + "AF_INET6", + "AF_IPX", + "AF_SNA", + "AF_UNSPEC", + "AI_ADDRCONFIG", + "AI_ALL", + "AI_CANONNAME", + "AI_NUMERICHOST", + "AI_NUMERICSERV", + "AI_PASSIVE", + "AI_V4MAPPED", + "CAPI", + "EAI_AGAIN", + "EAI_BADFLAGS", + "EAI_FAIL", + "EAI_FAMILY", + "EAI_MEMORY", + "EAI_NODATA", + "EAI_NONAME", + "EAI_SERVICE", + "EAI_SOCKTYPE", + "INADDR_ALLHOSTS_GROUP", + "INADDR_ANY", + "INADDR_BROADCAST", + "INADDR_LOOPBACK", + "INADDR_MAX_LOCAL_GROUP", + "INADDR_NONE", + "INADDR_UNSPEC_GROUP", + "IPPORT_RESERVED", + "IPPORT_USERRESERVED", + "IPPROTO_AH", + "IPPROTO_DSTOPTS", + "IPPROTO_EGP", + "IPPROTO_ESP", + "IPPROTO_FRAGMENT", + "IPPROTO_HOPOPTS", + "IPPROTO_ICMP", + "IPPROTO_ICMPV6", + "IPPROTO_IDP", + "IPPROTO_IGMP", + "IPPROTO_IP", + "IPPROTO_IPV6", + "IPPROTO_NONE", + "IPPROTO_PIM", + "IPPROTO_PUP", + "IPPROTO_RAW", + "IPPROTO_ROUTING", + "IPPROTO_SCTP", + "IPPROTO_TCP", + "IPPROTO_UDP", + "IPV6_CHECKSUM", + "IPV6_JOIN_GROUP", + "IPV6_LEAVE_GROUP", + "IPV6_MULTICAST_HOPS", + "IPV6_MULTICAST_IF", + "IPV6_MULTICAST_LOOP", + "IPV6_RECVTCLASS", + "IPV6_TCLASS", + "IPV6_UNICAST_HOPS", + "IPV6_V6ONLY", + "IP_ADD_MEMBERSHIP", + "IP_DROP_MEMBERSHIP", + "IP_HDRINCL", + "IP_MULTICAST_IF", + "IP_MULTICAST_LOOP", + "IP_MULTICAST_TTL", + "IP_OPTIONS", + "IP_TOS", + "IP_TTL", + "MSG_CTRUNC", + "MSG_DONTROUTE", + "MSG_OOB", + "MSG_PEEK", + "MSG_TRUNC", + "MSG_WAITALL", + "NI_DGRAM", + "NI_MAXHOST", + "NI_MAXSERV", + "NI_NAMEREQD", + "NI_NOFQDN", + "NI_NUMERICHOST", + "NI_NUMERICSERV", + "SHUT_RD", + "SHUT_RDWR", + "SHUT_WR", + "SOCK_DGRAM", + "SOCK_RAW", + "SOCK_RDM", + "SOCK_SEQPACKET", + "SOCK_STREAM", + "SOL_IP", + "SOL_SOCKET", + "SOL_TCP", + "SOL_UDP", + "SOMAXCONN", + "SO_ACCEPTCONN", + "SO_BROADCAST", + "SO_DEBUG", + "SO_DONTROUTE", + "SO_ERROR", + "SO_KEEPALIVE", + "SO_LINGER", + "SO_OOBINLINE", + "SO_RCVBUF", + "SO_RCVLOWAT", + "SO_RCVTIMEO", + "SO_REUSEADDR", + "SO_SNDBUF", + "SO_SNDLOWAT", + "SO_SNDTIMEO", + "SO_TYPE", + "SocketType", + "TCP_FASTOPEN", + "TCP_KEEPCNT", + "TCP_KEEPINTVL", + "TCP_MAXSEG", + "TCP_NODELAY", + "close", + "dup", + "error", + "gaierror", + "getaddrinfo", + "getdefaulttimeout", + "gethostbyaddr", + "gethostbyname", + "gethostbyname_ex", + "gethostname", + "getnameinfo", + "getprotobyname", + "getservbyname", + "getservbyport", + "has_ipv6", + "herror", + "htonl", + "htons", + "if_indextoname", + "if_nameindex", + "if_nametoindex", + "inet_aton", + "inet_ntoa", + "inet_ntop", + "inet_pton", + "ntohl", + "ntohs", + "setdefaulttimeout", + "socket", + "socketpair", + "timeout", +] + +if sys.platform == "win32": + from _socket import ( + IPPROTO_CBT as IPPROTO_CBT, + IPPROTO_ICLFXBM as IPPROTO_ICLFXBM, + IPPROTO_IGP as IPPROTO_IGP, + IPPROTO_L2TP as IPPROTO_L2TP, + IPPROTO_PGM as IPPROTO_PGM, + IPPROTO_RDP as IPPROTO_RDP, + IPPROTO_ST as IPPROTO_ST, + RCVALL_MAX as RCVALL_MAX, + RCVALL_OFF as RCVALL_OFF, + RCVALL_ON as RCVALL_ON, + RCVALL_SOCKETLEVELONLY as RCVALL_SOCKETLEVELONLY, + SIO_KEEPALIVE_VALS as SIO_KEEPALIVE_VALS, + SIO_LOOPBACK_FAST_PATH as SIO_LOOPBACK_FAST_PATH, + SIO_RCVALL as SIO_RCVALL, + SO_EXCLUSIVEADDRUSE as SO_EXCLUSIVEADDRUSE, + ) + + __all__ += [ + "IPPROTO_CBT", + "IPPROTO_ICLFXBM", + "IPPROTO_IGP", + "IPPROTO_L2TP", + "IPPROTO_PGM", + "IPPROTO_RDP", + "IPPROTO_ST", + "RCVALL_MAX", + "RCVALL_OFF", + "RCVALL_ON", + "RCVALL_SOCKETLEVELONLY", + "SIO_KEEPALIVE_VALS", + "SIO_LOOPBACK_FAST_PATH", + "SIO_RCVALL", + "SO_EXCLUSIVEADDRUSE", + "fromshare", + "errorTab", + "MSG_BCAST", + "MSG_MCAST", + ] + +if sys.platform != "darwin" or sys.version_info >= (3, 9): + from _socket import ( + IPV6_DONTFRAG as IPV6_DONTFRAG, + IPV6_HOPLIMIT as IPV6_HOPLIMIT, + IPV6_HOPOPTS as IPV6_HOPOPTS, + IPV6_PKTINFO as IPV6_PKTINFO, + IPV6_RECVRTHDR as IPV6_RECVRTHDR, + IPV6_RTHDR as IPV6_RTHDR, + ) + + __all__ += ["IPV6_DONTFRAG", "IPV6_HOPLIMIT", "IPV6_HOPOPTS", "IPV6_PKTINFO", "IPV6_RECVRTHDR", "IPV6_RTHDR"] + +if sys.platform == "darwin": + from _socket import PF_SYSTEM as PF_SYSTEM, SYSPROTO_CONTROL as SYSPROTO_CONTROL + + __all__ += ["PF_SYSTEM", "SYSPROTO_CONTROL", "AF_SYSTEM"] + +if sys.platform != "darwin": + from _socket import TCP_KEEPIDLE as TCP_KEEPIDLE + + __all__ += ["TCP_KEEPIDLE", "AF_IRDA", "MSG_ERRQUEUE"] + +if sys.version_info >= (3, 10): + from _socket import IP_RECVTOS as IP_RECVTOS + + __all__ += ["IP_RECVTOS"] + +if sys.platform != "win32" and sys.platform != "darwin": + from _socket import ( + IP_TRANSPARENT as IP_TRANSPARENT, + IPX_TYPE as IPX_TYPE, + SCM_CREDENTIALS as SCM_CREDENTIALS, + SO_BINDTODEVICE as SO_BINDTODEVICE, + SO_DOMAIN as SO_DOMAIN, + SO_MARK as SO_MARK, + SO_PASSCRED as SO_PASSCRED, + SO_PASSSEC as SO_PASSSEC, + SO_PEERCRED as SO_PEERCRED, + SO_PEERSEC as SO_PEERSEC, + SO_PRIORITY as SO_PRIORITY, + SO_PROTOCOL as SO_PROTOCOL, + SOL_ATALK as SOL_ATALK, + SOL_AX25 as SOL_AX25, + SOL_HCI as SOL_HCI, + SOL_IPX as SOL_IPX, + SOL_NETROM as SOL_NETROM, + SOL_ROSE as SOL_ROSE, + TCP_CONGESTION as TCP_CONGESTION, + TCP_CORK as TCP_CORK, + TCP_DEFER_ACCEPT as TCP_DEFER_ACCEPT, + TCP_INFO as TCP_INFO, + TCP_LINGER2 as TCP_LINGER2, + TCP_QUICKACK as TCP_QUICKACK, + TCP_SYNCNT as TCP_SYNCNT, + TCP_USER_TIMEOUT as TCP_USER_TIMEOUT, + TCP_WINDOW_CLAMP as TCP_WINDOW_CLAMP, + ) + + __all__ += [ + "IP_TRANSPARENT", + "SCM_CREDENTIALS", + "SO_BINDTODEVICE", + "SO_DOMAIN", + "SO_MARK", + "SO_PASSCRED", + "SO_PASSSEC", + "SO_PEERCRED", + "SO_PEERSEC", + "SO_PRIORITY", + "SO_PROTOCOL", + "TCP_CONGESTION", + "TCP_CORK", + "TCP_DEFER_ACCEPT", + "TCP_INFO", + "TCP_LINGER2", + "TCP_QUICKACK", + "TCP_SYNCNT", + "TCP_USER_TIMEOUT", + "TCP_WINDOW_CLAMP", + "AF_ASH", + "AF_ATMPVC", + "AF_ATMSVC", + "AF_AX25", + "AF_BRIDGE", + "AF_ECONET", + "AF_KEY", + "AF_LLC", + "AF_NETBEUI", + "AF_NETROM", + "AF_PPPOX", + "AF_ROSE", + "AF_SECURITY", + "AF_WANPIPE", + "AF_X25", + "MSG_CMSG_CLOEXEC", + "MSG_CONFIRM", + "MSG_FASTOPEN", + "MSG_MORE", + ] + +if sys.platform != "win32" and sys.platform != "darwin" and sys.version_info >= (3, 11): + from _socket import IP_BIND_ADDRESS_NO_PORT as IP_BIND_ADDRESS_NO_PORT + + __all__ += ["IP_BIND_ADDRESS_NO_PORT"] + +if sys.platform != "win32": + from _socket import ( + CMSG_LEN as CMSG_LEN, + CMSG_SPACE as CMSG_SPACE, + EAI_ADDRFAMILY as EAI_ADDRFAMILY, + EAI_OVERFLOW as EAI_OVERFLOW, + EAI_SYSTEM as EAI_SYSTEM, + IP_DEFAULT_MULTICAST_LOOP as IP_DEFAULT_MULTICAST_LOOP, + IP_DEFAULT_MULTICAST_TTL as IP_DEFAULT_MULTICAST_TTL, + IP_MAX_MEMBERSHIPS as IP_MAX_MEMBERSHIPS, + IP_RECVOPTS as IP_RECVOPTS, + IP_RECVRETOPTS as IP_RECVRETOPTS, + IP_RETOPTS as IP_RETOPTS, + IPPROTO_GRE as IPPROTO_GRE, + IPPROTO_IPIP as IPPROTO_IPIP, + IPPROTO_RSVP as IPPROTO_RSVP, + IPPROTO_TP as IPPROTO_TP, + IPV6_RTHDR_TYPE_0 as IPV6_RTHDR_TYPE_0, + SCM_RIGHTS as SCM_RIGHTS, + SO_REUSEPORT as SO_REUSEPORT, + TCP_NOTSENT_LOWAT as TCP_NOTSENT_LOWAT, + sethostname as sethostname, + ) + + __all__ += [ + "CMSG_LEN", + "CMSG_SPACE", + "EAI_ADDRFAMILY", + "EAI_OVERFLOW", + "EAI_SYSTEM", + "IP_DEFAULT_MULTICAST_LOOP", + "IP_DEFAULT_MULTICAST_TTL", + "IP_MAX_MEMBERSHIPS", + "IP_RECVOPTS", + "IP_RECVRETOPTS", + "IP_RETOPTS", + "IPPROTO_GRE", + "IPPROTO_IPIP", + "IPPROTO_RSVP", + "IPPROTO_TP", + "IPV6_RTHDR_TYPE_0", + "SCM_RIGHTS", + "SO_REUSEPORT", + "TCP_NOTSENT_LOWAT", + "sethostname", + "AF_ROUTE", + "AF_UNIX", + "MSG_DONTWAIT", + "MSG_EOR", + "MSG_NOSIGNAL", + ] + + if sys.platform != "darwin" or sys.version_info >= (3, 9): + from _socket import ( + IPV6_DSTOPTS as IPV6_DSTOPTS, + IPV6_NEXTHOP as IPV6_NEXTHOP, + IPV6_PATHMTU as IPV6_PATHMTU, + IPV6_RECVDSTOPTS as IPV6_RECVDSTOPTS, + IPV6_RECVHOPLIMIT as IPV6_RECVHOPLIMIT, + IPV6_RECVHOPOPTS as IPV6_RECVHOPOPTS, + IPV6_RECVPATHMTU as IPV6_RECVPATHMTU, + IPV6_RECVPKTINFO as IPV6_RECVPKTINFO, + IPV6_RTHDRDSTOPTS as IPV6_RTHDRDSTOPTS, + ) + + __all__ += [ + "IPV6_DSTOPTS", + "IPV6_NEXTHOP", + "IPV6_PATHMTU", + "IPV6_RECVDSTOPTS", + "IPV6_RECVHOPLIMIT", + "IPV6_RECVHOPOPTS", + "IPV6_RECVPATHMTU", + "IPV6_RECVPKTINFO", + "IPV6_RTHDRDSTOPTS", + ] + +if sys.platform != "darwin" and sys.platform != "linux": + if sys.platform != "win32" or sys.version_info >= (3, 9): + from _socket import BDADDR_ANY as BDADDR_ANY, BDADDR_LOCAL as BDADDR_LOCAL, BTPROTO_RFCOMM as BTPROTO_RFCOMM + + __all__ += ["BDADDR_ANY", "BDADDR_LOCAL", "BTPROTO_RFCOMM"] + +if sys.platform == "darwin" and sys.version_info >= (3, 10): + from _socket import TCP_KEEPALIVE as TCP_KEEPALIVE + + __all__ += ["TCP_KEEPALIVE"] + +if sys.platform == "darwin" and sys.version_info >= (3, 11): + from _socket import TCP_CONNECTION_INFO as TCP_CONNECTION_INFO + + __all__ += ["TCP_CONNECTION_INFO"] + +if sys.platform == "linux": + from _socket import ( + ALG_OP_DECRYPT as ALG_OP_DECRYPT, + ALG_OP_ENCRYPT as ALG_OP_ENCRYPT, + ALG_OP_SIGN as ALG_OP_SIGN, + ALG_OP_VERIFY as ALG_OP_VERIFY, + ALG_SET_AEAD_ASSOCLEN as ALG_SET_AEAD_ASSOCLEN, + ALG_SET_AEAD_AUTHSIZE as ALG_SET_AEAD_AUTHSIZE, + ALG_SET_IV as ALG_SET_IV, + ALG_SET_KEY as ALG_SET_KEY, + ALG_SET_OP as ALG_SET_OP, + ALG_SET_PUBKEY as ALG_SET_PUBKEY, + CAN_BCM as CAN_BCM, + CAN_BCM_CAN_FD_FRAME as CAN_BCM_CAN_FD_FRAME, + CAN_BCM_RX_ANNOUNCE_RESUME as CAN_BCM_RX_ANNOUNCE_RESUME, + CAN_BCM_RX_CHANGED as CAN_BCM_RX_CHANGED, + CAN_BCM_RX_CHECK_DLC as CAN_BCM_RX_CHECK_DLC, + CAN_BCM_RX_DELETE as CAN_BCM_RX_DELETE, + CAN_BCM_RX_FILTER_ID as CAN_BCM_RX_FILTER_ID, + CAN_BCM_RX_NO_AUTOTIMER as CAN_BCM_RX_NO_AUTOTIMER, + CAN_BCM_RX_READ as CAN_BCM_RX_READ, + CAN_BCM_RX_RTR_FRAME as CAN_BCM_RX_RTR_FRAME, + CAN_BCM_RX_SETUP as CAN_BCM_RX_SETUP, + CAN_BCM_RX_STATUS as CAN_BCM_RX_STATUS, + CAN_BCM_RX_TIMEOUT as CAN_BCM_RX_TIMEOUT, + CAN_BCM_SETTIMER as CAN_BCM_SETTIMER, + CAN_BCM_STARTTIMER as CAN_BCM_STARTTIMER, + CAN_BCM_TX_ANNOUNCE as CAN_BCM_TX_ANNOUNCE, + CAN_BCM_TX_COUNTEVT as CAN_BCM_TX_COUNTEVT, + CAN_BCM_TX_CP_CAN_ID as CAN_BCM_TX_CP_CAN_ID, + CAN_BCM_TX_DELETE as CAN_BCM_TX_DELETE, + CAN_BCM_TX_EXPIRED as CAN_BCM_TX_EXPIRED, + CAN_BCM_TX_READ as CAN_BCM_TX_READ, + CAN_BCM_TX_RESET_MULTI_IDX as CAN_BCM_TX_RESET_MULTI_IDX, + CAN_BCM_TX_SEND as CAN_BCM_TX_SEND, + CAN_BCM_TX_SETUP as CAN_BCM_TX_SETUP, + CAN_BCM_TX_STATUS as CAN_BCM_TX_STATUS, + CAN_EFF_FLAG as CAN_EFF_FLAG, + CAN_EFF_MASK as CAN_EFF_MASK, + CAN_ERR_FLAG as CAN_ERR_FLAG, + CAN_ERR_MASK as CAN_ERR_MASK, + CAN_ISOTP as CAN_ISOTP, + CAN_RAW as CAN_RAW, + CAN_RAW_FD_FRAMES as CAN_RAW_FD_FRAMES, + CAN_RAW_FILTER as CAN_RAW_FILTER, + CAN_RAW_LOOPBACK as CAN_RAW_LOOPBACK, + CAN_RAW_RECV_OWN_MSGS as CAN_RAW_RECV_OWN_MSGS, + CAN_RTR_FLAG as CAN_RTR_FLAG, + CAN_SFF_MASK as CAN_SFF_MASK, + IOCTL_VM_SOCKETS_GET_LOCAL_CID as IOCTL_VM_SOCKETS_GET_LOCAL_CID, + NETLINK_CRYPTO as NETLINK_CRYPTO, + NETLINK_DNRTMSG as NETLINK_DNRTMSG, + NETLINK_FIREWALL as NETLINK_FIREWALL, + NETLINK_IP6_FW as NETLINK_IP6_FW, + NETLINK_NFLOG as NETLINK_NFLOG, + NETLINK_ROUTE as NETLINK_ROUTE, + NETLINK_USERSOCK as NETLINK_USERSOCK, + NETLINK_XFRM as NETLINK_XFRM, + PACKET_BROADCAST as PACKET_BROADCAST, + PACKET_FASTROUTE as PACKET_FASTROUTE, + PACKET_HOST as PACKET_HOST, + PACKET_LOOPBACK as PACKET_LOOPBACK, + PACKET_MULTICAST as PACKET_MULTICAST, + PACKET_OTHERHOST as PACKET_OTHERHOST, + PACKET_OUTGOING as PACKET_OUTGOING, + PF_CAN as PF_CAN, + PF_PACKET as PF_PACKET, + PF_RDS as PF_RDS, + RDS_CANCEL_SENT_TO as RDS_CANCEL_SENT_TO, + RDS_CMSG_RDMA_ARGS as RDS_CMSG_RDMA_ARGS, + RDS_CMSG_RDMA_DEST as RDS_CMSG_RDMA_DEST, + RDS_CMSG_RDMA_MAP as RDS_CMSG_RDMA_MAP, + RDS_CMSG_RDMA_STATUS as RDS_CMSG_RDMA_STATUS, + RDS_CONG_MONITOR as RDS_CONG_MONITOR, + RDS_FREE_MR as RDS_FREE_MR, + RDS_GET_MR as RDS_GET_MR, + RDS_GET_MR_FOR_DEST as RDS_GET_MR_FOR_DEST, + RDS_RDMA_DONTWAIT as RDS_RDMA_DONTWAIT, + RDS_RDMA_FENCE as RDS_RDMA_FENCE, + RDS_RDMA_INVALIDATE as RDS_RDMA_INVALIDATE, + RDS_RDMA_NOTIFY_ME as RDS_RDMA_NOTIFY_ME, + RDS_RDMA_READWRITE as RDS_RDMA_READWRITE, + RDS_RDMA_SILENT as RDS_RDMA_SILENT, + RDS_RDMA_USE_ONCE as RDS_RDMA_USE_ONCE, + RDS_RECVERR as RDS_RECVERR, + SO_VM_SOCKETS_BUFFER_MAX_SIZE as SO_VM_SOCKETS_BUFFER_MAX_SIZE, + SO_VM_SOCKETS_BUFFER_MIN_SIZE as SO_VM_SOCKETS_BUFFER_MIN_SIZE, + SO_VM_SOCKETS_BUFFER_SIZE as SO_VM_SOCKETS_BUFFER_SIZE, + SOL_ALG as SOL_ALG, + SOL_CAN_BASE as SOL_CAN_BASE, + SOL_CAN_RAW as SOL_CAN_RAW, + SOL_RDS as SOL_RDS, + SOL_TIPC as SOL_TIPC, + TIPC_ADDR_ID as TIPC_ADDR_ID, + TIPC_ADDR_NAME as TIPC_ADDR_NAME, + TIPC_ADDR_NAMESEQ as TIPC_ADDR_NAMESEQ, + TIPC_CFG_SRV as TIPC_CFG_SRV, + TIPC_CLUSTER_SCOPE as TIPC_CLUSTER_SCOPE, + TIPC_CONN_TIMEOUT as TIPC_CONN_TIMEOUT, + TIPC_CRITICAL_IMPORTANCE as TIPC_CRITICAL_IMPORTANCE, + TIPC_DEST_DROPPABLE as TIPC_DEST_DROPPABLE, + TIPC_HIGH_IMPORTANCE as TIPC_HIGH_IMPORTANCE, + TIPC_IMPORTANCE as TIPC_IMPORTANCE, + TIPC_LOW_IMPORTANCE as TIPC_LOW_IMPORTANCE, + TIPC_MEDIUM_IMPORTANCE as TIPC_MEDIUM_IMPORTANCE, + TIPC_NODE_SCOPE as TIPC_NODE_SCOPE, + TIPC_PUBLISHED as TIPC_PUBLISHED, + TIPC_SRC_DROPPABLE as TIPC_SRC_DROPPABLE, + TIPC_SUB_CANCEL as TIPC_SUB_CANCEL, + TIPC_SUB_PORTS as TIPC_SUB_PORTS, + TIPC_SUB_SERVICE as TIPC_SUB_SERVICE, + TIPC_SUBSCR_TIMEOUT as TIPC_SUBSCR_TIMEOUT, + TIPC_TOP_SRV as TIPC_TOP_SRV, + TIPC_WAIT_FOREVER as TIPC_WAIT_FOREVER, + TIPC_WITHDRAWN as TIPC_WITHDRAWN, + TIPC_ZONE_SCOPE as TIPC_ZONE_SCOPE, + VM_SOCKETS_INVALID_VERSION as VM_SOCKETS_INVALID_VERSION, + VMADDR_CID_ANY as VMADDR_CID_ANY, + VMADDR_CID_HOST as VMADDR_CID_HOST, + VMADDR_PORT_ANY as VMADDR_PORT_ANY, + ) + + __all__ += [ + "ALG_OP_DECRYPT", + "ALG_OP_ENCRYPT", + "ALG_OP_SIGN", + "ALG_OP_VERIFY", + "ALG_SET_AEAD_ASSOCLEN", + "ALG_SET_AEAD_AUTHSIZE", + "ALG_SET_IV", + "ALG_SET_KEY", + "ALG_SET_OP", + "ALG_SET_PUBKEY", + "CAN_BCM", + "CAN_BCM_CAN_FD_FRAME", + "CAN_BCM_RX_ANNOUNCE_RESUME", + "CAN_BCM_RX_CHANGED", + "CAN_BCM_RX_CHECK_DLC", + "CAN_BCM_RX_DELETE", + "CAN_BCM_RX_FILTER_ID", + "CAN_BCM_RX_NO_AUTOTIMER", + "CAN_BCM_RX_READ", + "CAN_BCM_RX_RTR_FRAME", + "CAN_BCM_RX_SETUP", + "CAN_BCM_RX_STATUS", + "CAN_BCM_RX_TIMEOUT", + "CAN_BCM_SETTIMER", + "CAN_BCM_STARTTIMER", + "CAN_BCM_TX_ANNOUNCE", + "CAN_BCM_TX_COUNTEVT", + "CAN_BCM_TX_CP_CAN_ID", + "CAN_BCM_TX_DELETE", + "CAN_BCM_TX_EXPIRED", + "CAN_BCM_TX_READ", + "CAN_BCM_TX_RESET_MULTI_IDX", + "CAN_BCM_TX_SEND", + "CAN_BCM_TX_SETUP", + "CAN_BCM_TX_STATUS", + "CAN_EFF_FLAG", + "CAN_EFF_MASK", + "CAN_ERR_FLAG", + "CAN_ERR_MASK", + "CAN_ISOTP", + "CAN_RAW", + "CAN_RAW_FD_FRAMES", + "CAN_RAW_FILTER", + "CAN_RAW_LOOPBACK", + "CAN_RAW_RECV_OWN_MSGS", + "CAN_RTR_FLAG", + "CAN_SFF_MASK", + "IOCTL_VM_SOCKETS_GET_LOCAL_CID", + "NETLINK_CRYPTO", + "NETLINK_DNRTMSG", + "NETLINK_FIREWALL", + "NETLINK_IP6_FW", + "NETLINK_NFLOG", + "NETLINK_ROUTE", + "NETLINK_USERSOCK", + "NETLINK_XFRM", + "PACKET_BROADCAST", + "PACKET_FASTROUTE", + "PACKET_HOST", + "PACKET_LOOPBACK", + "PACKET_MULTICAST", + "PACKET_OTHERHOST", + "PACKET_OUTGOING", + "PF_CAN", + "PF_PACKET", + "PF_RDS", + "SO_VM_SOCKETS_BUFFER_MAX_SIZE", + "SO_VM_SOCKETS_BUFFER_MIN_SIZE", + "SO_VM_SOCKETS_BUFFER_SIZE", + "SOL_ALG", + "SOL_CAN_BASE", + "SOL_CAN_RAW", + "SOL_RDS", + "SOL_TIPC", + "TIPC_ADDR_ID", + "TIPC_ADDR_NAME", + "TIPC_ADDR_NAMESEQ", + "TIPC_CFG_SRV", + "TIPC_CLUSTER_SCOPE", + "TIPC_CONN_TIMEOUT", + "TIPC_CRITICAL_IMPORTANCE", + "TIPC_DEST_DROPPABLE", + "TIPC_HIGH_IMPORTANCE", + "TIPC_IMPORTANCE", + "TIPC_LOW_IMPORTANCE", + "TIPC_MEDIUM_IMPORTANCE", + "TIPC_NODE_SCOPE", + "TIPC_PUBLISHED", + "TIPC_SRC_DROPPABLE", + "TIPC_SUB_CANCEL", + "TIPC_SUB_PORTS", + "TIPC_SUB_SERVICE", + "TIPC_SUBSCR_TIMEOUT", + "TIPC_TOP_SRV", + "TIPC_WAIT_FOREVER", + "TIPC_WITHDRAWN", + "TIPC_ZONE_SCOPE", + "VM_SOCKETS_INVALID_VERSION", + "VMADDR_CID_ANY", + "VMADDR_CID_HOST", + "VMADDR_PORT_ANY", + "AF_CAN", + "AF_PACKET", + "AF_RDS", + "AF_TIPC", + "AF_ALG", + "AF_NETLINK", + "AF_VSOCK", + "AF_QIPCRTR", + "SOCK_CLOEXEC", + "SOCK_NONBLOCK", + ] + + if sys.version_info < (3, 11): + from _socket import CAN_RAW_ERR_FILTER as CAN_RAW_ERR_FILTER + + __all__ += ["CAN_RAW_ERR_FILTER"] + +if sys.platform == "linux" and sys.version_info >= (3, 9): + from _socket import ( + CAN_J1939 as CAN_J1939, + CAN_RAW_JOIN_FILTERS as CAN_RAW_JOIN_FILTERS, + IPPROTO_UDPLITE as IPPROTO_UDPLITE, + J1939_EE_INFO_NONE as J1939_EE_INFO_NONE, + J1939_EE_INFO_TX_ABORT as J1939_EE_INFO_TX_ABORT, + J1939_FILTER_MAX as J1939_FILTER_MAX, + J1939_IDLE_ADDR as J1939_IDLE_ADDR, + J1939_MAX_UNICAST_ADDR as J1939_MAX_UNICAST_ADDR, + J1939_NLA_BYTES_ACKED as J1939_NLA_BYTES_ACKED, + J1939_NLA_PAD as J1939_NLA_PAD, + J1939_NO_ADDR as J1939_NO_ADDR, + J1939_NO_NAME as J1939_NO_NAME, + J1939_NO_PGN as J1939_NO_PGN, + J1939_PGN_ADDRESS_CLAIMED as J1939_PGN_ADDRESS_CLAIMED, + J1939_PGN_ADDRESS_COMMANDED as J1939_PGN_ADDRESS_COMMANDED, + J1939_PGN_MAX as J1939_PGN_MAX, + J1939_PGN_PDU1_MAX as J1939_PGN_PDU1_MAX, + J1939_PGN_REQUEST as J1939_PGN_REQUEST, + SCM_J1939_DEST_ADDR as SCM_J1939_DEST_ADDR, + SCM_J1939_DEST_NAME as SCM_J1939_DEST_NAME, + SCM_J1939_ERRQUEUE as SCM_J1939_ERRQUEUE, + SCM_J1939_PRIO as SCM_J1939_PRIO, + SO_J1939_ERRQUEUE as SO_J1939_ERRQUEUE, + SO_J1939_FILTER as SO_J1939_FILTER, + SO_J1939_PROMISC as SO_J1939_PROMISC, + SO_J1939_SEND_PRIO as SO_J1939_SEND_PRIO, + UDPLITE_RECV_CSCOV as UDPLITE_RECV_CSCOV, + UDPLITE_SEND_CSCOV as UDPLITE_SEND_CSCOV, + ) + + __all__ += [ + "CAN_J1939", + "CAN_RAW_JOIN_FILTERS", + "IPPROTO_UDPLITE", + "J1939_EE_INFO_NONE", + "J1939_EE_INFO_TX_ABORT", + "J1939_FILTER_MAX", + "J1939_IDLE_ADDR", + "J1939_MAX_UNICAST_ADDR", + "J1939_NLA_BYTES_ACKED", + "J1939_NLA_PAD", + "J1939_NO_ADDR", + "J1939_NO_NAME", + "J1939_NO_PGN", + "J1939_PGN_ADDRESS_CLAIMED", + "J1939_PGN_ADDRESS_COMMANDED", + "J1939_PGN_MAX", + "J1939_PGN_PDU1_MAX", + "J1939_PGN_REQUEST", + "SCM_J1939_DEST_ADDR", + "SCM_J1939_DEST_NAME", + "SCM_J1939_ERRQUEUE", + "SCM_J1939_PRIO", + "SO_J1939_ERRQUEUE", + "SO_J1939_FILTER", + "SO_J1939_PROMISC", + "SO_J1939_SEND_PRIO", + "UDPLITE_RECV_CSCOV", + "UDPLITE_SEND_CSCOV", + ] +if sys.platform == "linux" and sys.version_info >= (3, 10): + from _socket import IPPROTO_MPTCP as IPPROTO_MPTCP + + __all__ += ["IPPROTO_MPTCP"] +if sys.platform == "linux" and sys.version_info >= (3, 11): + from _socket import SO_INCOMING_CPU as SO_INCOMING_CPU + + __all__ += ["SO_INCOMING_CPU"] +if sys.platform == "linux" and sys.version_info >= (3, 12): + from _socket import ( + TCP_CC_INFO as TCP_CC_INFO, + TCP_FASTOPEN_CONNECT as TCP_FASTOPEN_CONNECT, + TCP_FASTOPEN_KEY as TCP_FASTOPEN_KEY, + TCP_FASTOPEN_NO_COOKIE as TCP_FASTOPEN_NO_COOKIE, + TCP_INQ as TCP_INQ, + TCP_MD5SIG as TCP_MD5SIG, + TCP_MD5SIG_EXT as TCP_MD5SIG_EXT, + TCP_QUEUE_SEQ as TCP_QUEUE_SEQ, + TCP_REPAIR as TCP_REPAIR, + TCP_REPAIR_OPTIONS as TCP_REPAIR_OPTIONS, + TCP_REPAIR_QUEUE as TCP_REPAIR_QUEUE, + TCP_REPAIR_WINDOW as TCP_REPAIR_WINDOW, + TCP_SAVE_SYN as TCP_SAVE_SYN, + TCP_SAVED_SYN as TCP_SAVED_SYN, + TCP_THIN_DUPACK as TCP_THIN_DUPACK, + TCP_THIN_LINEAR_TIMEOUTS as TCP_THIN_LINEAR_TIMEOUTS, + TCP_TIMESTAMP as TCP_TIMESTAMP, + TCP_TX_DELAY as TCP_TX_DELAY, + TCP_ULP as TCP_ULP, + TCP_ZEROCOPY_RECEIVE as TCP_ZEROCOPY_RECEIVE, + ) + + __all__ += [ + "TCP_CC_INFO", + "TCP_FASTOPEN_CONNECT", + "TCP_FASTOPEN_KEY", + "TCP_FASTOPEN_NO_COOKIE", + "TCP_INQ", + "TCP_MD5SIG", + "TCP_MD5SIG_EXT", + "TCP_QUEUE_SEQ", + "TCP_REPAIR", + "TCP_REPAIR_OPTIONS", + "TCP_REPAIR_QUEUE", + "TCP_REPAIR_WINDOW", + "TCP_SAVED_SYN", + "TCP_SAVE_SYN", + "TCP_THIN_DUPACK", + "TCP_THIN_LINEAR_TIMEOUTS", + "TCP_TIMESTAMP", + "TCP_TX_DELAY", + "TCP_ULP", + "TCP_ZEROCOPY_RECEIVE", + ] + +if sys.platform == "linux" and sys.version_info >= (3, 13): + from _socket import NI_IDN as NI_IDN, SO_BINDTOIFINDEX as SO_BINDTOIFINDEX + + __all__ += ["NI_IDN", "SO_BINDTOIFINDEX"] + +if sys.version_info >= (3, 12): + from _socket import ( + IP_ADD_SOURCE_MEMBERSHIP as IP_ADD_SOURCE_MEMBERSHIP, + IP_BLOCK_SOURCE as IP_BLOCK_SOURCE, + IP_DROP_SOURCE_MEMBERSHIP as IP_DROP_SOURCE_MEMBERSHIP, + IP_PKTINFO as IP_PKTINFO, + IP_UNBLOCK_SOURCE as IP_UNBLOCK_SOURCE, + ) + + __all__ += ["IP_ADD_SOURCE_MEMBERSHIP", "IP_BLOCK_SOURCE", "IP_DROP_SOURCE_MEMBERSHIP", "IP_PKTINFO", "IP_UNBLOCK_SOURCE"] + + if sys.platform == "win32": + from _socket import ( + HV_GUID_BROADCAST as HV_GUID_BROADCAST, + HV_GUID_CHILDREN as HV_GUID_CHILDREN, + HV_GUID_LOOPBACK as HV_GUID_LOOPBACK, + HV_GUID_PARENT as HV_GUID_PARENT, + HV_GUID_WILDCARD as HV_GUID_WILDCARD, + HV_GUID_ZERO as HV_GUID_ZERO, + HV_PROTOCOL_RAW as HV_PROTOCOL_RAW, + HVSOCKET_ADDRESS_FLAG_PASSTHRU as HVSOCKET_ADDRESS_FLAG_PASSTHRU, + HVSOCKET_CONNECT_TIMEOUT as HVSOCKET_CONNECT_TIMEOUT, + HVSOCKET_CONNECT_TIMEOUT_MAX as HVSOCKET_CONNECT_TIMEOUT_MAX, + HVSOCKET_CONNECTED_SUSPEND as HVSOCKET_CONNECTED_SUSPEND, + ) + + __all__ += [ + "HV_GUID_BROADCAST", + "HV_GUID_CHILDREN", + "HV_GUID_LOOPBACK", + "HV_GUID_PARENT", + "HV_GUID_WILDCARD", + "HV_GUID_ZERO", + "HV_PROTOCOL_RAW", + "HVSOCKET_ADDRESS_FLAG_PASSTHRU", + "HVSOCKET_CONNECT_TIMEOUT", + "HVSOCKET_CONNECT_TIMEOUT_MAX", + "HVSOCKET_CONNECTED_SUSPEND", + ] + else: + from _socket import ( + ETHERTYPE_ARP as ETHERTYPE_ARP, + ETHERTYPE_IP as ETHERTYPE_IP, + ETHERTYPE_IPV6 as ETHERTYPE_IPV6, + ETHERTYPE_VLAN as ETHERTYPE_VLAN, + ) + + __all__ += ["ETHERTYPE_ARP", "ETHERTYPE_IP", "ETHERTYPE_IPV6", "ETHERTYPE_VLAN"] + + if sys.platform == "linux": + from _socket import ETH_P_ALL as ETH_P_ALL + + __all__ += ["ETH_P_ALL"] + + if sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin": + # FreeBSD >= 14.0 + from _socket import PF_DIVERT as PF_DIVERT + + __all__ += ["PF_DIVERT", "AF_DIVERT"] + +if sys.platform != "win32" and sys.version_info >= (3, 9): + __all__ += ["send_fds", "recv_fds"] + +if sys.platform != "win32" or sys.version_info >= (3, 9): + if sys.platform != "linux": + __all__ += ["AF_LINK"] + if sys.platform != "darwin" and sys.platform != "linux": + __all__ += ["AF_BLUETOOTH"] + +if sys.platform == "win32" and sys.version_info >= (3, 12): + __all__ += ["AF_HYPERV"] + +if sys.platform != "win32" and sys.platform != "linux": + from _socket import ( + EAI_BADHINTS as EAI_BADHINTS, + EAI_MAX as EAI_MAX, + EAI_PROTOCOL as EAI_PROTOCOL, + IPPROTO_EON as IPPROTO_EON, + IPPROTO_HELLO as IPPROTO_HELLO, + IPPROTO_IPCOMP as IPPROTO_IPCOMP, + IPPROTO_XTP as IPPROTO_XTP, + LOCAL_PEERCRED as LOCAL_PEERCRED, + SCM_CREDS as SCM_CREDS, + ) + + __all__ += [ + "EAI_BADHINTS", + "EAI_MAX", + "EAI_PROTOCOL", + "IPPROTO_EON", + "IPPROTO_HELLO", + "IPPROTO_IPCOMP", + "IPPROTO_XTP", + "LOCAL_PEERCRED", + "SCM_CREDS", + "AI_DEFAULT", + "AI_MASK", + "AI_V4MAPPED_CFG", + "MSG_EOF", + ] + if sys.platform != "darwin" or sys.version_info >= (3, 9): + from _socket import IPV6_USE_MIN_MTU as IPV6_USE_MIN_MTU + + __all__ += ["IPV6_USE_MIN_MTU"] + +if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + from _socket import ( + IPPROTO_BIP as IPPROTO_BIP, + IPPROTO_MOBILE as IPPROTO_MOBILE, + IPPROTO_VRRP as IPPROTO_VRRP, + MSG_BTAG as MSG_BTAG, + MSG_ETAG as MSG_ETAG, + SO_SETFIB as SO_SETFIB, + ) + + __all__ += ["SO_SETFIB", "MSG_BTAG", "MSG_ETAG", "IPPROTO_BIP", "IPPROTO_MOBILE", "IPPROTO_VRRP", "MSG_NOTIFICATION"] + +if sys.platform != "linux": + from _socket import ( + IP_RECVDSTADDR as IP_RECVDSTADDR, + IPPROTO_GGP as IPPROTO_GGP, + IPPROTO_IPV4 as IPPROTO_IPV4, + IPPROTO_MAX as IPPROTO_MAX, + IPPROTO_ND as IPPROTO_ND, + SO_USELOOPBACK as SO_USELOOPBACK, + ) + + __all__ += ["IPPROTO_GGP", "IPPROTO_IPV4", "IPPROTO_MAX", "IPPROTO_ND", "IP_RECVDSTADDR", "SO_USELOOPBACK"] + +# Re-exported from errno +EBADF: int +EAGAIN: int +EWOULDBLOCK: int + +# These errors are implemented in _socket at runtime +# but they consider themselves to live in socket so we'll put them here. +error = OSError + +class herror(error): ... +class gaierror(error): ... + +if sys.version_info >= (3, 10): + timeout = TimeoutError +else: + class timeout(error): ... + +class AddressFamily(IntEnum): + AF_INET = 2 + AF_INET6 = 10 + AF_APPLETALK = 5 + AF_DECnet = ... + AF_IPX = 4 + AF_SNA = 22 + AF_UNSPEC = 0 + if sys.platform != "darwin": + AF_IRDA = 23 + if sys.platform != "win32": + AF_ROUTE = 16 + AF_UNIX = 1 + if sys.platform == "darwin": + AF_SYSTEM = 32 + if sys.platform != "win32" and sys.platform != "darwin": + AF_ASH = 18 + AF_ATMPVC = 8 + AF_ATMSVC = 20 + AF_AX25 = 3 + AF_BRIDGE = 7 + AF_ECONET = 19 + AF_KEY = 15 + AF_LLC = 26 + AF_NETBEUI = 13 + AF_NETROM = 6 + AF_PPPOX = 24 + AF_ROSE = 11 + AF_SECURITY = 14 + AF_WANPIPE = 25 + AF_X25 = 9 + if sys.platform == "linux": + AF_CAN = 29 + AF_PACKET = 17 + AF_RDS = 21 + AF_TIPC = 30 + AF_ALG = 38 + AF_NETLINK = 16 + AF_VSOCK = 40 + AF_QIPCRTR = 42 + if sys.platform != "win32" or sys.version_info >= (3, 9): + if sys.platform != "linux": + AF_LINK = 33 + if sys.platform != "darwin" and sys.platform != "linux": + AF_BLUETOOTH = 32 + if sys.platform == "win32" and sys.version_info >= (3, 12): + AF_HYPERV = 34 + if sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin" and sys.version_info >= (3, 12): + # FreeBSD >= 14.0 + AF_DIVERT = 44 + +AF_INET = AddressFamily.AF_INET +AF_INET6 = AddressFamily.AF_INET6 +AF_APPLETALK = AddressFamily.AF_APPLETALK +AF_DECnet = AddressFamily.AF_DECnet +AF_IPX = AddressFamily.AF_IPX +AF_SNA = AddressFamily.AF_SNA +AF_UNSPEC = AddressFamily.AF_UNSPEC + +if sys.platform != "darwin": + AF_IRDA = AddressFamily.AF_IRDA + +if sys.platform != "win32": + AF_ROUTE = AddressFamily.AF_ROUTE + AF_UNIX = AddressFamily.AF_UNIX + +if sys.platform == "darwin": + AF_SYSTEM = AddressFamily.AF_SYSTEM + +if sys.platform != "win32" and sys.platform != "darwin": + AF_ASH = AddressFamily.AF_ASH + AF_ATMPVC = AddressFamily.AF_ATMPVC + AF_ATMSVC = AddressFamily.AF_ATMSVC + AF_AX25 = AddressFamily.AF_AX25 + AF_BRIDGE = AddressFamily.AF_BRIDGE + AF_ECONET = AddressFamily.AF_ECONET + AF_KEY = AddressFamily.AF_KEY + AF_LLC = AddressFamily.AF_LLC + AF_NETBEUI = AddressFamily.AF_NETBEUI + AF_NETROM = AddressFamily.AF_NETROM + AF_PPPOX = AddressFamily.AF_PPPOX + AF_ROSE = AddressFamily.AF_ROSE + AF_SECURITY = AddressFamily.AF_SECURITY + AF_WANPIPE = AddressFamily.AF_WANPIPE + AF_X25 = AddressFamily.AF_X25 + +if sys.platform == "linux": + AF_CAN = AddressFamily.AF_CAN + AF_PACKET = AddressFamily.AF_PACKET + AF_RDS = AddressFamily.AF_RDS + AF_TIPC = AddressFamily.AF_TIPC + AF_ALG = AddressFamily.AF_ALG + AF_NETLINK = AddressFamily.AF_NETLINK + AF_VSOCK = AddressFamily.AF_VSOCK + AF_QIPCRTR = AddressFamily.AF_QIPCRTR + +if sys.platform != "win32" or sys.version_info >= (3, 9): + if sys.platform != "linux": + AF_LINK = AddressFamily.AF_LINK + if sys.platform != "darwin" and sys.platform != "linux": + AF_BLUETOOTH = AddressFamily.AF_BLUETOOTH + +if sys.platform == "win32" and sys.version_info >= (3, 12): + AF_HYPERV = AddressFamily.AF_HYPERV +if sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin" and sys.version_info >= (3, 12): + # FreeBSD >= 14.0 + AF_DIVERT = AddressFamily.AF_DIVERT + +class SocketKind(IntEnum): + SOCK_STREAM = 1 + SOCK_DGRAM = 2 + SOCK_RAW = 3 + SOCK_RDM = 4 + SOCK_SEQPACKET = 5 + if sys.platform == "linux": + SOCK_CLOEXEC = 524288 + SOCK_NONBLOCK = 2048 + +SOCK_STREAM = SocketKind.SOCK_STREAM +SOCK_DGRAM = SocketKind.SOCK_DGRAM +SOCK_RAW = SocketKind.SOCK_RAW +SOCK_RDM = SocketKind.SOCK_RDM +SOCK_SEQPACKET = SocketKind.SOCK_SEQPACKET +if sys.platform == "linux": + SOCK_CLOEXEC = SocketKind.SOCK_CLOEXEC + SOCK_NONBLOCK = SocketKind.SOCK_NONBLOCK + +class MsgFlag(IntFlag): + MSG_CTRUNC = 8 + MSG_DONTROUTE = 4 + MSG_OOB = 1 + MSG_PEEK = 2 + MSG_TRUNC = 32 + MSG_WAITALL = 256 + if sys.platform == "win32": + MSG_BCAST = 1024 + MSG_MCAST = 2048 + + if sys.platform != "darwin": + MSG_ERRQUEUE = 8192 + + if sys.platform != "win32" and sys.platform != "darwin": + MSG_CMSG_CLOEXEC = 1073741821 + MSG_CONFIRM = 2048 + MSG_FASTOPEN = 536870912 + MSG_MORE = 32768 + + if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + MSG_NOTIFICATION = 8192 + + if sys.platform != "win32": + MSG_DONTWAIT = 64 + MSG_EOR = 128 + MSG_NOSIGNAL = 16384 # sometimes this exists on darwin, sometimes not + if sys.platform != "win32" and sys.platform != "linux": + MSG_EOF = 256 + +MSG_CTRUNC = MsgFlag.MSG_CTRUNC +MSG_DONTROUTE = MsgFlag.MSG_DONTROUTE +MSG_OOB = MsgFlag.MSG_OOB +MSG_PEEK = MsgFlag.MSG_PEEK +MSG_TRUNC = MsgFlag.MSG_TRUNC +MSG_WAITALL = MsgFlag.MSG_WAITALL + +if sys.platform == "win32": + MSG_BCAST = MsgFlag.MSG_BCAST + MSG_MCAST = MsgFlag.MSG_MCAST + +if sys.platform != "darwin": + MSG_ERRQUEUE = MsgFlag.MSG_ERRQUEUE + +if sys.platform != "win32": + MSG_DONTWAIT = MsgFlag.MSG_DONTWAIT + MSG_EOR = MsgFlag.MSG_EOR + MSG_NOSIGNAL = MsgFlag.MSG_NOSIGNAL # Sometimes this exists on darwin, sometimes not + +if sys.platform != "win32" and sys.platform != "darwin": + MSG_CMSG_CLOEXEC = MsgFlag.MSG_CMSG_CLOEXEC + MSG_CONFIRM = MsgFlag.MSG_CONFIRM + MSG_FASTOPEN = MsgFlag.MSG_FASTOPEN + MSG_MORE = MsgFlag.MSG_MORE + +if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": + MSG_NOTIFICATION = MsgFlag.MSG_NOTIFICATION + +if sys.platform != "win32" and sys.platform != "linux": + MSG_EOF = MsgFlag.MSG_EOF + +class AddressInfo(IntFlag): + AI_ADDRCONFIG = 32 + AI_ALL = 16 + AI_CANONNAME = 2 + AI_NUMERICHOST = 4 + AI_NUMERICSERV = 1024 + AI_PASSIVE = 1 + AI_V4MAPPED = 8 + if sys.platform != "win32" and sys.platform != "linux": + AI_DEFAULT = 1536 + AI_MASK = 5127 + AI_V4MAPPED_CFG = 512 + +AI_ADDRCONFIG = AddressInfo.AI_ADDRCONFIG +AI_ALL = AddressInfo.AI_ALL +AI_CANONNAME = AddressInfo.AI_CANONNAME +AI_NUMERICHOST = AddressInfo.AI_NUMERICHOST +AI_NUMERICSERV = AddressInfo.AI_NUMERICSERV +AI_PASSIVE = AddressInfo.AI_PASSIVE +AI_V4MAPPED = AddressInfo.AI_V4MAPPED + +if sys.platform != "win32" and sys.platform != "linux": + AI_DEFAULT = AddressInfo.AI_DEFAULT + AI_MASK = AddressInfo.AI_MASK + AI_V4MAPPED_CFG = AddressInfo.AI_V4MAPPED_CFG + +if sys.platform == "win32": + errorTab: dict[int, str] # undocumented + +class _SendableFile(Protocol): + def read(self, size: int, /) -> bytes: ... + def seek(self, offset: int, /) -> object: ... + + # optional fields: + # + # @property + # def mode(self) -> str: ... + # def fileno(self) -> int: ... + +class socket(_socket.socket): + def __init__( + self, family: AddressFamily | int = -1, type: SocketKind | int = -1, proto: int = -1, fileno: int | None = None + ) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def dup(self) -> Self: ... + def accept(self) -> tuple[socket, _RetAddress]: ... + # Note that the makefile's documented windows-specific behavior is not represented + # mode strings with duplicates are intentionally excluded + @overload + def makefile( + self, + mode: Literal["b", "rb", "br", "wb", "bw", "rwb", "rbw", "wrb", "wbr", "brw", "bwr"], + buffering: Literal[0], + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> SocketIO: ... + @overload + def makefile( + self, + mode: Literal["rwb", "rbw", "wrb", "wbr", "brw", "bwr"], + buffering: Literal[-1, 1] | None = None, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> BufferedRWPair: ... + @overload + def makefile( + self, + mode: Literal["rb", "br"], + buffering: Literal[-1, 1] | None = None, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> BufferedReader: ... + @overload + def makefile( + self, + mode: Literal["wb", "bw"], + buffering: Literal[-1, 1] | None = None, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> BufferedWriter: ... + @overload + def makefile( + self, + mode: Literal["b", "rb", "br", "wb", "bw", "rwb", "rbw", "wrb", "wbr", "brw", "bwr"], + buffering: int, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> IOBase: ... + @overload + def makefile( + self, + mode: Literal["r", "w", "rw", "wr", ""] = "r", + buffering: int | None = None, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> TextIOWrapper: ... + def sendfile(self, file: _SendableFile, offset: int = 0, count: int | None = None) -> int: ... + @property + def family(self) -> AddressFamily: ... + @property + def type(self) -> SocketKind: ... + def get_inheritable(self) -> bool: ... + def set_inheritable(self, inheritable: bool) -> None: ... + +def fromfd(fd: SupportsIndex, family: AddressFamily | int, type: SocketKind | int, proto: int = 0) -> socket: ... + +if sys.platform != "win32": + if sys.version_info >= (3, 9): + def send_fds( + sock: socket, buffers: Iterable[ReadableBuffer], fds: Iterable[int], flags: Unused = 0, address: Unused = None + ) -> int: ... + def recv_fds(sock: socket, bufsize: int, maxfds: int, flags: int = 0) -> tuple[bytes, list[int], int, Any]: ... + +if sys.platform == "win32": + def fromshare(info: bytes) -> socket: ... + +if sys.platform == "win32": + def socketpair(family: int = ..., type: int = ..., proto: int = 0) -> tuple[socket, socket]: ... + +else: + def socketpair( + family: int | AddressFamily | None = None, type: SocketType | int = ..., proto: int = 0 + ) -> tuple[socket, socket]: ... + +class SocketIO(RawIOBase): + def __init__(self, sock: socket, mode: Literal["r", "w", "rw", "rb", "wb", "rwb"]) -> None: ... + def readinto(self, b: WriteableBuffer) -> int | None: ... + def write(self, b: ReadableBuffer) -> int | None: ... + @property + def name(self) -> int: ... # return value is really "int" + @property + def mode(self) -> Literal["rb", "wb", "rwb"]: ... + +def getfqdn(name: str = "") -> str: ... + +if sys.version_info >= (3, 11): + def create_connection( + address: tuple[str | None, int], + timeout: float | None = ..., + source_address: _Address | None = None, + *, + all_errors: bool = False, + ) -> socket: ... + +else: + def create_connection( + address: tuple[str | None, int], timeout: float | None = ..., source_address: _Address | None = None + ) -> socket: ... + +def has_dualstack_ipv6() -> bool: ... +def create_server( + address: _Address, *, family: int = ..., backlog: int | None = None, reuse_port: bool = False, dualstack_ipv6: bool = False +) -> socket: ... + +# the 5th tuple item is an address +def getaddrinfo( + host: bytes | str | None, port: bytes | str | int | None, family: int = 0, type: int = 0, proto: int = 0, flags: int = 0 +) -> list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int] | tuple[str, int, int, int]]]: ... diff --git a/mypy/typeshed/stdlib/socketserver.pyi b/mypy/typeshed/stdlib/socketserver.pyi new file mode 100644 index 000000000000..ae6575d85082 --- /dev/null +++ b/mypy/typeshed/stdlib/socketserver.pyi @@ -0,0 +1,169 @@ +import sys +import types +from _socket import _Address, _RetAddress +from _typeshed import ReadableBuffer +from collections.abc import Callable +from io import BufferedIOBase +from socket import socket as _socket +from typing import Any, ClassVar +from typing_extensions import Self, TypeAlias + +__all__ = [ + "BaseServer", + "TCPServer", + "UDPServer", + "ThreadingUDPServer", + "ThreadingTCPServer", + "BaseRequestHandler", + "StreamRequestHandler", + "DatagramRequestHandler", + "ThreadingMixIn", +] +if sys.platform != "win32": + __all__ += [ + "ForkingMixIn", + "ForkingTCPServer", + "ForkingUDPServer", + "ThreadingUnixDatagramServer", + "ThreadingUnixStreamServer", + "UnixDatagramServer", + "UnixStreamServer", + ] + if sys.version_info >= (3, 12): + __all__ += ["ForkingUnixStreamServer", "ForkingUnixDatagramServer"] + +_RequestType: TypeAlias = _socket | tuple[bytes, _socket] +_AfUnixAddress: TypeAlias = str | ReadableBuffer # address acceptable for an AF_UNIX socket +_AfInetAddress: TypeAlias = tuple[str | bytes | bytearray, int] # address acceptable for an AF_INET socket + +# This can possibly be generic at some point: +class BaseServer: + address_family: int + server_address: _Address + socket: _socket + allow_reuse_address: bool + request_queue_size: int + socket_type: int + timeout: float | None + RequestHandlerClass: Callable[[Any, _RetAddress, Self], BaseRequestHandler] + def __init__( + self, server_address: _Address, RequestHandlerClass: Callable[[Any, _RetAddress, Self], BaseRequestHandler] + ) -> None: ... + def fileno(self) -> int: ... + def handle_request(self) -> None: ... + def serve_forever(self, poll_interval: float = 0.5) -> None: ... + def shutdown(self) -> None: ... + def server_close(self) -> None: ... + def finish_request(self, request: _RequestType, client_address: _RetAddress) -> None: ... + def get_request(self) -> tuple[Any, Any]: ... + def handle_error(self, request: _RequestType, client_address: _RetAddress) -> None: ... + def handle_timeout(self) -> None: ... + def process_request(self, request: _RequestType, client_address: _RetAddress) -> None: ... + def server_activate(self) -> None: ... + def server_bind(self) -> None: ... + def verify_request(self, request: _RequestType, client_address: _RetAddress) -> bool: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def service_actions(self) -> None: ... + def shutdown_request(self, request: _RequestType) -> None: ... # undocumented + def close_request(self, request: _RequestType) -> None: ... # undocumented + +class TCPServer(BaseServer): + if sys.version_info >= (3, 11): + allow_reuse_port: bool + server_address: _AfInetAddress + def __init__( + self, + server_address: _AfInetAddress, + RequestHandlerClass: Callable[[Any, _RetAddress, Self], BaseRequestHandler], + bind_and_activate: bool = True, + ) -> None: ... + def get_request(self) -> tuple[_socket, _RetAddress]: ... + +class UDPServer(TCPServer): + max_packet_size: ClassVar[int] + def get_request(self) -> tuple[tuple[bytes, _socket], _RetAddress]: ... # type: ignore[override] + +if sys.platform != "win32": + class UnixStreamServer(TCPServer): + server_address: _AfUnixAddress # type: ignore[assignment] + def __init__( + self, + server_address: _AfUnixAddress, + RequestHandlerClass: Callable[[Any, _RetAddress, Self], BaseRequestHandler], + bind_and_activate: bool = True, + ) -> None: ... + + class UnixDatagramServer(UDPServer): + server_address: _AfUnixAddress # type: ignore[assignment] + def __init__( + self, + server_address: _AfUnixAddress, + RequestHandlerClass: Callable[[Any, _RetAddress, Self], BaseRequestHandler], + bind_and_activate: bool = True, + ) -> None: ... + +if sys.platform != "win32": + class ForkingMixIn: + timeout: float | None # undocumented + active_children: set[int] | None # undocumented + max_children: int # undocumented + block_on_close: bool + def collect_children(self, *, blocking: bool = False) -> None: ... # undocumented + def handle_timeout(self) -> None: ... # undocumented + def service_actions(self) -> None: ... # undocumented + def process_request(self, request: _RequestType, client_address: _RetAddress) -> None: ... + def server_close(self) -> None: ... + +class ThreadingMixIn: + daemon_threads: bool + block_on_close: bool + def process_request_thread(self, request: _RequestType, client_address: _RetAddress) -> None: ... # undocumented + def process_request(self, request: _RequestType, client_address: _RetAddress) -> None: ... + def server_close(self) -> None: ... + +if sys.platform != "win32": + class ForkingTCPServer(ForkingMixIn, TCPServer): ... + class ForkingUDPServer(ForkingMixIn, UDPServer): ... + if sys.version_info >= (3, 12): + class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer): ... + class ForkingUnixDatagramServer(ForkingMixIn, UnixDatagramServer): ... + +class ThreadingTCPServer(ThreadingMixIn, TCPServer): ... +class ThreadingUDPServer(ThreadingMixIn, UDPServer): ... + +if sys.platform != "win32": + class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): ... + class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): ... + +class BaseRequestHandler: + # `request` is technically of type _RequestType, + # but there are some concerns that having a union here would cause + # too much inconvenience to people using it (see + # https://github.com/python/typeshed/pull/384#issuecomment-234649696) + # + # Note also that _RetAddress is also just an alias for `Any` + request: Any + client_address: _RetAddress + server: BaseServer + def __init__(self, request: _RequestType, client_address: _RetAddress, server: BaseServer) -> None: ... + def setup(self) -> None: ... + def handle(self) -> None: ... + def finish(self) -> None: ... + +class StreamRequestHandler(BaseRequestHandler): + rbufsize: ClassVar[int] # undocumented + wbufsize: ClassVar[int] # undocumented + timeout: ClassVar[float | None] # undocumented + disable_nagle_algorithm: ClassVar[bool] # undocumented + connection: Any # undocumented + rfile: BufferedIOBase + wfile: BufferedIOBase + +class DatagramRequestHandler(BaseRequestHandler): + packet: bytes # undocumented + socket: _socket # undocumented + rfile: BufferedIOBase + wfile: BufferedIOBase diff --git a/mypy/typeshed/stdlib/spwd.pyi b/mypy/typeshed/stdlib/spwd.pyi new file mode 100644 index 000000000000..3a5d39997dcc --- /dev/null +++ b/mypy/typeshed/stdlib/spwd.pyi @@ -0,0 +1,46 @@ +import sys +from _typeshed import structseq +from typing import Any, Final, final + +if sys.platform != "win32": + @final + class struct_spwd(structseq[Any], tuple[str, str, int, int, int, int, int, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ( + "sp_namp", + "sp_pwdp", + "sp_lstchg", + "sp_min", + "sp_max", + "sp_warn", + "sp_inact", + "sp_expire", + "sp_flag", + ) + + @property + def sp_namp(self) -> str: ... + @property + def sp_pwdp(self) -> str: ... + @property + def sp_lstchg(self) -> int: ... + @property + def sp_min(self) -> int: ... + @property + def sp_max(self) -> int: ... + @property + def sp_warn(self) -> int: ... + @property + def sp_inact(self) -> int: ... + @property + def sp_expire(self) -> int: ... + @property + def sp_flag(self) -> int: ... + # Deprecated aliases below. + @property + def sp_nam(self) -> str: ... + @property + def sp_pwd(self) -> str: ... + + def getspall() -> list[struct_spwd]: ... + def getspnam(arg: str, /) -> struct_spwd: ... diff --git a/mypy/typeshed/stdlib/sqlite3/__init__.pyi b/mypy/typeshed/stdlib/sqlite3/__init__.pyi new file mode 100644 index 000000000000..bc0ff6469d5e --- /dev/null +++ b/mypy/typeshed/stdlib/sqlite3/__init__.pyi @@ -0,0 +1,467 @@ +import sys +from _typeshed import MaybeNone, ReadableBuffer, StrOrBytesPath, SupportsLenAndGetItem, Unused +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence +from sqlite3.dbapi2 import ( + PARSE_COLNAMES as PARSE_COLNAMES, + PARSE_DECLTYPES as PARSE_DECLTYPES, + SQLITE_ALTER_TABLE as SQLITE_ALTER_TABLE, + SQLITE_ANALYZE as SQLITE_ANALYZE, + SQLITE_ATTACH as SQLITE_ATTACH, + SQLITE_CREATE_INDEX as SQLITE_CREATE_INDEX, + SQLITE_CREATE_TABLE as SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_INDEX as SQLITE_CREATE_TEMP_INDEX, + SQLITE_CREATE_TEMP_TABLE as SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_TEMP_TRIGGER as SQLITE_CREATE_TEMP_TRIGGER, + SQLITE_CREATE_TEMP_VIEW as SQLITE_CREATE_TEMP_VIEW, + SQLITE_CREATE_TRIGGER as SQLITE_CREATE_TRIGGER, + SQLITE_CREATE_VIEW as SQLITE_CREATE_VIEW, + SQLITE_CREATE_VTABLE as SQLITE_CREATE_VTABLE, + SQLITE_DELETE as SQLITE_DELETE, + SQLITE_DENY as SQLITE_DENY, + SQLITE_DETACH as SQLITE_DETACH, + SQLITE_DONE as SQLITE_DONE, + SQLITE_DROP_INDEX as SQLITE_DROP_INDEX, + SQLITE_DROP_TABLE as SQLITE_DROP_TABLE, + SQLITE_DROP_TEMP_INDEX as SQLITE_DROP_TEMP_INDEX, + SQLITE_DROP_TEMP_TABLE as SQLITE_DROP_TEMP_TABLE, + SQLITE_DROP_TEMP_TRIGGER as SQLITE_DROP_TEMP_TRIGGER, + SQLITE_DROP_TEMP_VIEW as SQLITE_DROP_TEMP_VIEW, + SQLITE_DROP_TRIGGER as SQLITE_DROP_TRIGGER, + SQLITE_DROP_VIEW as SQLITE_DROP_VIEW, + SQLITE_DROP_VTABLE as SQLITE_DROP_VTABLE, + SQLITE_FUNCTION as SQLITE_FUNCTION, + SQLITE_IGNORE as SQLITE_IGNORE, + SQLITE_INSERT as SQLITE_INSERT, + SQLITE_OK as SQLITE_OK, + SQLITE_PRAGMA as SQLITE_PRAGMA, + SQLITE_READ as SQLITE_READ, + SQLITE_RECURSIVE as SQLITE_RECURSIVE, + SQLITE_REINDEX as SQLITE_REINDEX, + SQLITE_SAVEPOINT as SQLITE_SAVEPOINT, + SQLITE_SELECT as SQLITE_SELECT, + SQLITE_TRANSACTION as SQLITE_TRANSACTION, + SQLITE_UPDATE as SQLITE_UPDATE, + Binary as Binary, + Date as Date, + DateFromTicks as DateFromTicks, + Time as Time, + TimeFromTicks as TimeFromTicks, + TimestampFromTicks as TimestampFromTicks, + adapt as adapt, + adapters as adapters, + apilevel as apilevel, + complete_statement as complete_statement, + connect as connect, + converters as converters, + enable_callback_tracebacks as enable_callback_tracebacks, + paramstyle as paramstyle, + register_adapter as register_adapter, + register_converter as register_converter, + sqlite_version as sqlite_version, + sqlite_version_info as sqlite_version_info, + threadsafety as threadsafety, + version_info as version_info, +) +from types import TracebackType +from typing import Any, Literal, Protocol, SupportsIndex, TypeVar, final, overload, type_check_only +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 12): + from sqlite3.dbapi2 import ( + LEGACY_TRANSACTION_CONTROL as LEGACY_TRANSACTION_CONTROL, + SQLITE_DBCONFIG_DEFENSIVE as SQLITE_DBCONFIG_DEFENSIVE, + SQLITE_DBCONFIG_DQS_DDL as SQLITE_DBCONFIG_DQS_DDL, + SQLITE_DBCONFIG_DQS_DML as SQLITE_DBCONFIG_DQS_DML, + SQLITE_DBCONFIG_ENABLE_FKEY as SQLITE_DBCONFIG_ENABLE_FKEY, + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER as SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION as SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, + SQLITE_DBCONFIG_ENABLE_QPSG as SQLITE_DBCONFIG_ENABLE_QPSG, + SQLITE_DBCONFIG_ENABLE_TRIGGER as SQLITE_DBCONFIG_ENABLE_TRIGGER, + SQLITE_DBCONFIG_ENABLE_VIEW as SQLITE_DBCONFIG_ENABLE_VIEW, + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE as SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT as SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE as SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + SQLITE_DBCONFIG_RESET_DATABASE as SQLITE_DBCONFIG_RESET_DATABASE, + SQLITE_DBCONFIG_TRIGGER_EQP as SQLITE_DBCONFIG_TRIGGER_EQP, + SQLITE_DBCONFIG_TRUSTED_SCHEMA as SQLITE_DBCONFIG_TRUSTED_SCHEMA, + SQLITE_DBCONFIG_WRITABLE_SCHEMA as SQLITE_DBCONFIG_WRITABLE_SCHEMA, + ) + +if sys.version_info >= (3, 11): + from sqlite3.dbapi2 import ( + SQLITE_ABORT as SQLITE_ABORT, + SQLITE_ABORT_ROLLBACK as SQLITE_ABORT_ROLLBACK, + SQLITE_AUTH as SQLITE_AUTH, + SQLITE_AUTH_USER as SQLITE_AUTH_USER, + SQLITE_BUSY as SQLITE_BUSY, + SQLITE_BUSY_RECOVERY as SQLITE_BUSY_RECOVERY, + SQLITE_BUSY_SNAPSHOT as SQLITE_BUSY_SNAPSHOT, + SQLITE_BUSY_TIMEOUT as SQLITE_BUSY_TIMEOUT, + SQLITE_CANTOPEN as SQLITE_CANTOPEN, + SQLITE_CANTOPEN_CONVPATH as SQLITE_CANTOPEN_CONVPATH, + SQLITE_CANTOPEN_DIRTYWAL as SQLITE_CANTOPEN_DIRTYWAL, + SQLITE_CANTOPEN_FULLPATH as SQLITE_CANTOPEN_FULLPATH, + SQLITE_CANTOPEN_ISDIR as SQLITE_CANTOPEN_ISDIR, + SQLITE_CANTOPEN_NOTEMPDIR as SQLITE_CANTOPEN_NOTEMPDIR, + SQLITE_CANTOPEN_SYMLINK as SQLITE_CANTOPEN_SYMLINK, + SQLITE_CONSTRAINT as SQLITE_CONSTRAINT, + SQLITE_CONSTRAINT_CHECK as SQLITE_CONSTRAINT_CHECK, + SQLITE_CONSTRAINT_COMMITHOOK as SQLITE_CONSTRAINT_COMMITHOOK, + SQLITE_CONSTRAINT_FOREIGNKEY as SQLITE_CONSTRAINT_FOREIGNKEY, + SQLITE_CONSTRAINT_FUNCTION as SQLITE_CONSTRAINT_FUNCTION, + SQLITE_CONSTRAINT_NOTNULL as SQLITE_CONSTRAINT_NOTNULL, + SQLITE_CONSTRAINT_PINNED as SQLITE_CONSTRAINT_PINNED, + SQLITE_CONSTRAINT_PRIMARYKEY as SQLITE_CONSTRAINT_PRIMARYKEY, + SQLITE_CONSTRAINT_ROWID as SQLITE_CONSTRAINT_ROWID, + SQLITE_CONSTRAINT_TRIGGER as SQLITE_CONSTRAINT_TRIGGER, + SQLITE_CONSTRAINT_UNIQUE as SQLITE_CONSTRAINT_UNIQUE, + SQLITE_CONSTRAINT_VTAB as SQLITE_CONSTRAINT_VTAB, + SQLITE_CORRUPT as SQLITE_CORRUPT, + SQLITE_CORRUPT_INDEX as SQLITE_CORRUPT_INDEX, + SQLITE_CORRUPT_SEQUENCE as SQLITE_CORRUPT_SEQUENCE, + SQLITE_CORRUPT_VTAB as SQLITE_CORRUPT_VTAB, + SQLITE_EMPTY as SQLITE_EMPTY, + SQLITE_ERROR as SQLITE_ERROR, + SQLITE_ERROR_MISSING_COLLSEQ as SQLITE_ERROR_MISSING_COLLSEQ, + SQLITE_ERROR_RETRY as SQLITE_ERROR_RETRY, + SQLITE_ERROR_SNAPSHOT as SQLITE_ERROR_SNAPSHOT, + SQLITE_FORMAT as SQLITE_FORMAT, + SQLITE_FULL as SQLITE_FULL, + SQLITE_INTERNAL as SQLITE_INTERNAL, + SQLITE_INTERRUPT as SQLITE_INTERRUPT, + SQLITE_IOERR as SQLITE_IOERR, + SQLITE_IOERR_ACCESS as SQLITE_IOERR_ACCESS, + SQLITE_IOERR_AUTH as SQLITE_IOERR_AUTH, + SQLITE_IOERR_BEGIN_ATOMIC as SQLITE_IOERR_BEGIN_ATOMIC, + SQLITE_IOERR_BLOCKED as SQLITE_IOERR_BLOCKED, + SQLITE_IOERR_CHECKRESERVEDLOCK as SQLITE_IOERR_CHECKRESERVEDLOCK, + SQLITE_IOERR_CLOSE as SQLITE_IOERR_CLOSE, + SQLITE_IOERR_COMMIT_ATOMIC as SQLITE_IOERR_COMMIT_ATOMIC, + SQLITE_IOERR_CONVPATH as SQLITE_IOERR_CONVPATH, + SQLITE_IOERR_CORRUPTFS as SQLITE_IOERR_CORRUPTFS, + SQLITE_IOERR_DATA as SQLITE_IOERR_DATA, + SQLITE_IOERR_DELETE as SQLITE_IOERR_DELETE, + SQLITE_IOERR_DELETE_NOENT as SQLITE_IOERR_DELETE_NOENT, + SQLITE_IOERR_DIR_CLOSE as SQLITE_IOERR_DIR_CLOSE, + SQLITE_IOERR_DIR_FSYNC as SQLITE_IOERR_DIR_FSYNC, + SQLITE_IOERR_FSTAT as SQLITE_IOERR_FSTAT, + SQLITE_IOERR_FSYNC as SQLITE_IOERR_FSYNC, + SQLITE_IOERR_GETTEMPPATH as SQLITE_IOERR_GETTEMPPATH, + SQLITE_IOERR_LOCK as SQLITE_IOERR_LOCK, + SQLITE_IOERR_MMAP as SQLITE_IOERR_MMAP, + SQLITE_IOERR_NOMEM as SQLITE_IOERR_NOMEM, + SQLITE_IOERR_RDLOCK as SQLITE_IOERR_RDLOCK, + SQLITE_IOERR_READ as SQLITE_IOERR_READ, + SQLITE_IOERR_ROLLBACK_ATOMIC as SQLITE_IOERR_ROLLBACK_ATOMIC, + SQLITE_IOERR_SEEK as SQLITE_IOERR_SEEK, + SQLITE_IOERR_SHMLOCK as SQLITE_IOERR_SHMLOCK, + SQLITE_IOERR_SHMMAP as SQLITE_IOERR_SHMMAP, + SQLITE_IOERR_SHMOPEN as SQLITE_IOERR_SHMOPEN, + SQLITE_IOERR_SHMSIZE as SQLITE_IOERR_SHMSIZE, + SQLITE_IOERR_SHORT_READ as SQLITE_IOERR_SHORT_READ, + SQLITE_IOERR_TRUNCATE as SQLITE_IOERR_TRUNCATE, + SQLITE_IOERR_UNLOCK as SQLITE_IOERR_UNLOCK, + SQLITE_IOERR_VNODE as SQLITE_IOERR_VNODE, + SQLITE_IOERR_WRITE as SQLITE_IOERR_WRITE, + SQLITE_LIMIT_ATTACHED as SQLITE_LIMIT_ATTACHED, + SQLITE_LIMIT_COLUMN as SQLITE_LIMIT_COLUMN, + SQLITE_LIMIT_COMPOUND_SELECT as SQLITE_LIMIT_COMPOUND_SELECT, + SQLITE_LIMIT_EXPR_DEPTH as SQLITE_LIMIT_EXPR_DEPTH, + SQLITE_LIMIT_FUNCTION_ARG as SQLITE_LIMIT_FUNCTION_ARG, + SQLITE_LIMIT_LENGTH as SQLITE_LIMIT_LENGTH, + SQLITE_LIMIT_LIKE_PATTERN_LENGTH as SQLITE_LIMIT_LIKE_PATTERN_LENGTH, + SQLITE_LIMIT_SQL_LENGTH as SQLITE_LIMIT_SQL_LENGTH, + SQLITE_LIMIT_TRIGGER_DEPTH as SQLITE_LIMIT_TRIGGER_DEPTH, + SQLITE_LIMIT_VARIABLE_NUMBER as SQLITE_LIMIT_VARIABLE_NUMBER, + SQLITE_LIMIT_VDBE_OP as SQLITE_LIMIT_VDBE_OP, + SQLITE_LIMIT_WORKER_THREADS as SQLITE_LIMIT_WORKER_THREADS, + SQLITE_LOCKED as SQLITE_LOCKED, + SQLITE_LOCKED_SHAREDCACHE as SQLITE_LOCKED_SHAREDCACHE, + SQLITE_LOCKED_VTAB as SQLITE_LOCKED_VTAB, + SQLITE_MISMATCH as SQLITE_MISMATCH, + SQLITE_MISUSE as SQLITE_MISUSE, + SQLITE_NOLFS as SQLITE_NOLFS, + SQLITE_NOMEM as SQLITE_NOMEM, + SQLITE_NOTADB as SQLITE_NOTADB, + SQLITE_NOTFOUND as SQLITE_NOTFOUND, + SQLITE_NOTICE as SQLITE_NOTICE, + SQLITE_NOTICE_RECOVER_ROLLBACK as SQLITE_NOTICE_RECOVER_ROLLBACK, + SQLITE_NOTICE_RECOVER_WAL as SQLITE_NOTICE_RECOVER_WAL, + SQLITE_OK_LOAD_PERMANENTLY as SQLITE_OK_LOAD_PERMANENTLY, + SQLITE_OK_SYMLINK as SQLITE_OK_SYMLINK, + SQLITE_PERM as SQLITE_PERM, + SQLITE_PROTOCOL as SQLITE_PROTOCOL, + SQLITE_RANGE as SQLITE_RANGE, + SQLITE_READONLY as SQLITE_READONLY, + SQLITE_READONLY_CANTINIT as SQLITE_READONLY_CANTINIT, + SQLITE_READONLY_CANTLOCK as SQLITE_READONLY_CANTLOCK, + SQLITE_READONLY_DBMOVED as SQLITE_READONLY_DBMOVED, + SQLITE_READONLY_DIRECTORY as SQLITE_READONLY_DIRECTORY, + SQLITE_READONLY_RECOVERY as SQLITE_READONLY_RECOVERY, + SQLITE_READONLY_ROLLBACK as SQLITE_READONLY_ROLLBACK, + SQLITE_ROW as SQLITE_ROW, + SQLITE_SCHEMA as SQLITE_SCHEMA, + SQLITE_TOOBIG as SQLITE_TOOBIG, + SQLITE_WARNING as SQLITE_WARNING, + SQLITE_WARNING_AUTOINDEX as SQLITE_WARNING_AUTOINDEX, + ) + +if sys.version_info < (3, 12): + from sqlite3.dbapi2 import enable_shared_cache as enable_shared_cache, version as version + +if sys.version_info < (3, 10): + from sqlite3.dbapi2 import OptimizedUnicode as OptimizedUnicode + +_CursorT = TypeVar("_CursorT", bound=Cursor) +_SqliteData: TypeAlias = str | ReadableBuffer | int | float | None +# Data that is passed through adapters can be of any type accepted by an adapter. +_AdaptedInputData: TypeAlias = _SqliteData | Any +# The Mapping must really be a dict, but making it invariant is too annoying. +_Parameters: TypeAlias = SupportsLenAndGetItem[_AdaptedInputData] | Mapping[str, _AdaptedInputData] + +class _AnyParamWindowAggregateClass(Protocol): + def step(self, *args: Any) -> object: ... + def inverse(self, *args: Any) -> object: ... + def value(self) -> _SqliteData: ... + def finalize(self) -> _SqliteData: ... + +class _WindowAggregateClass(Protocol): + step: Callable[..., object] + inverse: Callable[..., object] + def value(self) -> _SqliteData: ... + def finalize(self) -> _SqliteData: ... + +class _AggregateProtocol(Protocol): + def step(self, value: int, /) -> object: ... + def finalize(self) -> int: ... + +class _SingleParamWindowAggregateClass(Protocol): + def step(self, param: Any, /) -> object: ... + def inverse(self, param: Any, /) -> object: ... + def value(self) -> _SqliteData: ... + def finalize(self) -> _SqliteData: ... + +# These classes are implemented in the C module _sqlite3. At runtime, they're imported +# from there into sqlite3.dbapi2 and from that module to here. However, they +# consider themselves to live in the sqlite3.* namespace, so we'll define them here. + +class Error(Exception): + if sys.version_info >= (3, 11): + sqlite_errorcode: int + sqlite_errorname: str + +class DatabaseError(Error): ... +class DataError(DatabaseError): ... +class IntegrityError(DatabaseError): ... +class InterfaceError(Error): ... +class InternalError(DatabaseError): ... +class NotSupportedError(DatabaseError): ... +class OperationalError(DatabaseError): ... +class ProgrammingError(DatabaseError): ... +class Warning(Exception): ... + +class Connection: + @property + def DataError(self) -> type[DataError]: ... + @property + def DatabaseError(self) -> type[DatabaseError]: ... + @property + def Error(self) -> type[Error]: ... + @property + def IntegrityError(self) -> type[IntegrityError]: ... + @property + def InterfaceError(self) -> type[InterfaceError]: ... + @property + def InternalError(self) -> type[InternalError]: ... + @property + def NotSupportedError(self) -> type[NotSupportedError]: ... + @property + def OperationalError(self) -> type[OperationalError]: ... + @property + def ProgrammingError(self) -> type[ProgrammingError]: ... + @property + def Warning(self) -> type[Warning]: ... + @property + def in_transaction(self) -> bool: ... + isolation_level: str | None # one of '', 'DEFERRED', 'IMMEDIATE' or 'EXCLUSIVE' + @property + def total_changes(self) -> int: ... + if sys.version_info >= (3, 12): + @property + def autocommit(self) -> int: ... + @autocommit.setter + def autocommit(self, val: int) -> None: ... + row_factory: Any + text_factory: Any + if sys.version_info >= (3, 12): + def __init__( + self, + database: StrOrBytesPath, + timeout: float = ..., + detect_types: int = ..., + isolation_level: str | None = ..., + check_same_thread: bool = ..., + factory: type[Connection] | None = ..., + cached_statements: int = ..., + uri: bool = ..., + autocommit: bool = ..., + ) -> None: ... + else: + def __init__( + self, + database: StrOrBytesPath, + timeout: float = ..., + detect_types: int = ..., + isolation_level: str | None = ..., + check_same_thread: bool = ..., + factory: type[Connection] | None = ..., + cached_statements: int = ..., + uri: bool = ..., + ) -> None: ... + + def close(self) -> None: ... + if sys.version_info >= (3, 11): + def blobopen(self, table: str, column: str, row: int, /, *, readonly: bool = False, name: str = "main") -> Blob: ... + + def commit(self) -> None: ... + def create_aggregate(self, name: str, n_arg: int, aggregate_class: Callable[[], _AggregateProtocol]) -> None: ... + if sys.version_info >= (3, 11): + # num_params determines how many params will be passed to the aggregate class. We provide an overload + # for the case where num_params = 1, which is expected to be the common case. + @overload + def create_window_function( + self, name: str, num_params: Literal[1], aggregate_class: Callable[[], _SingleParamWindowAggregateClass] | None, / + ) -> None: ... + # And for num_params = -1, which means the aggregate must accept any number of parameters. + @overload + def create_window_function( + self, name: str, num_params: Literal[-1], aggregate_class: Callable[[], _AnyParamWindowAggregateClass] | None, / + ) -> None: ... + @overload + def create_window_function( + self, name: str, num_params: int, aggregate_class: Callable[[], _WindowAggregateClass] | None, / + ) -> None: ... + + def create_collation(self, name: str, callback: Callable[[str, str], int | SupportsIndex] | None, /) -> None: ... + def create_function( + self, name: str, narg: int, func: Callable[..., _SqliteData] | None, *, deterministic: bool = False + ) -> None: ... + @overload + def cursor(self, factory: None = None) -> Cursor: ... + @overload + def cursor(self, factory: Callable[[Connection], _CursorT]) -> _CursorT: ... + def execute(self, sql: str, parameters: _Parameters = ..., /) -> Cursor: ... + def executemany(self, sql: str, parameters: Iterable[_Parameters], /) -> Cursor: ... + def executescript(self, sql_script: str, /) -> Cursor: ... + def interrupt(self) -> None: ... + if sys.version_info >= (3, 13): + def iterdump(self, *, filter: str | None = None) -> Generator[str, None, None]: ... + else: + def iterdump(self) -> Generator[str, None, None]: ... + + def rollback(self) -> None: ... + def set_authorizer( + self, authorizer_callback: Callable[[int, str | None, str | None, str | None, str | None], int] | None + ) -> None: ... + def set_progress_handler(self, progress_handler: Callable[[], int | None] | None, n: int) -> None: ... + def set_trace_callback(self, trace_callback: Callable[[str], object] | None) -> None: ... + # enable_load_extension and load_extension is not available on python distributions compiled + # without sqlite3 loadable extension support. see footnotes https://docs.python.org/3/library/sqlite3.html#f1 + def enable_load_extension(self, enable: bool, /) -> None: ... + if sys.version_info >= (3, 12): + def load_extension(self, name: str, /, *, entrypoint: str | None = None) -> None: ... + else: + def load_extension(self, name: str, /) -> None: ... + + def backup( + self, + target: Connection, + *, + pages: int = -1, + progress: Callable[[int, int, int], object] | None = None, + name: str = "main", + sleep: float = 0.25, + ) -> None: ... + if sys.version_info >= (3, 11): + def setlimit(self, category: int, limit: int, /) -> int: ... + def getlimit(self, category: int, /) -> int: ... + def serialize(self, *, name: str = "main") -> bytes: ... + def deserialize(self, data: ReadableBuffer, /, *, name: str = "main") -> None: ... + if sys.version_info >= (3, 12): + def getconfig(self, op: int, /) -> bool: ... + def setconfig(self, op: int, enable: bool = True, /) -> bool: ... + + def __call__(self, sql: str, /) -> _Statement: ... + def __enter__(self) -> Self: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None, / + ) -> Literal[False]: ... + +class Cursor: + arraysize: int + @property + def connection(self) -> Connection: ... + # May be None, but using `| MaybeNone` (`| Any`) instead to avoid slightly annoying false positives. + @property + def description(self) -> tuple[tuple[str, None, None, None, None, None, None], ...] | MaybeNone: ... + @property + def lastrowid(self) -> int | None: ... + row_factory: Callable[[Cursor, Row], object] | None + @property + def rowcount(self) -> int: ... + def __init__(self, cursor: Connection, /) -> None: ... + def close(self) -> None: ... + def execute(self, sql: str, parameters: _Parameters = (), /) -> Self: ... + def executemany(self, sql: str, seq_of_parameters: Iterable[_Parameters], /) -> Self: ... + def executescript(self, sql_script: str, /) -> Cursor: ... + def fetchall(self) -> list[Any]: ... + def fetchmany(self, size: int | None = 1) -> list[Any]: ... + # Returns either a row (as created by the row_factory) or None, but + # putting None in the return annotation causes annoying false positives. + def fetchone(self) -> Any: ... + def setinputsizes(self, sizes: Unused, /) -> None: ... # does nothing + def setoutputsize(self, size: Unused, column: Unused = None, /) -> None: ... # does nothing + def __iter__(self) -> Self: ... + def __next__(self) -> Any: ... + +@final +class PrepareProtocol: + def __init__(self, *args: object, **kwargs: object) -> None: ... + +class Row(Sequence[Any]): + def __init__(self, cursor: Cursor, data: tuple[Any, ...], /) -> None: ... + def keys(self) -> list[str]: ... + @overload + def __getitem__(self, key: int | str, /) -> Any: ... + @overload + def __getitem__(self, key: slice, /) -> tuple[Any, ...]: ... + def __hash__(self) -> int: ... + def __iter__(self) -> Iterator[Any]: ... + def __len__(self) -> int: ... + # These return NotImplemented for anything that is not a Row. + def __eq__(self, value: object, /) -> bool: ... + def __ge__(self, value: object, /) -> bool: ... + def __gt__(self, value: object, /) -> bool: ... + def __le__(self, value: object, /) -> bool: ... + def __lt__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + +# This class is not exposed. It calls itself sqlite3.Statement. +@final +@type_check_only +class _Statement: ... + +if sys.version_info >= (3, 11): + @final + class Blob: + def close(self) -> None: ... + def read(self, length: int = -1, /) -> bytes: ... + def write(self, data: ReadableBuffer, /) -> None: ... + def tell(self) -> int: ... + # whence must be one of os.SEEK_SET, os.SEEK_CUR, os.SEEK_END + def seek(self, offset: int, origin: int = 0, /) -> None: ... + def __len__(self) -> int: ... + def __enter__(self) -> Self: ... + def __exit__(self, type: object, val: object, tb: object, /) -> Literal[False]: ... + def __getitem__(self, key: SupportsIndex | slice, /) -> int: ... + def __setitem__(self, key: SupportsIndex | slice, value: int, /) -> None: ... diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi new file mode 100644 index 000000000000..d3ea3ef0e896 --- /dev/null +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -0,0 +1,241 @@ +import sys +from _sqlite3 import ( + PARSE_COLNAMES as PARSE_COLNAMES, + PARSE_DECLTYPES as PARSE_DECLTYPES, + SQLITE_ALTER_TABLE as SQLITE_ALTER_TABLE, + SQLITE_ANALYZE as SQLITE_ANALYZE, + SQLITE_ATTACH as SQLITE_ATTACH, + SQLITE_CREATE_INDEX as SQLITE_CREATE_INDEX, + SQLITE_CREATE_TABLE as SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_INDEX as SQLITE_CREATE_TEMP_INDEX, + SQLITE_CREATE_TEMP_TABLE as SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_TEMP_TRIGGER as SQLITE_CREATE_TEMP_TRIGGER, + SQLITE_CREATE_TEMP_VIEW as SQLITE_CREATE_TEMP_VIEW, + SQLITE_CREATE_TRIGGER as SQLITE_CREATE_TRIGGER, + SQLITE_CREATE_VIEW as SQLITE_CREATE_VIEW, + SQLITE_CREATE_VTABLE as SQLITE_CREATE_VTABLE, + SQLITE_DELETE as SQLITE_DELETE, + SQLITE_DENY as SQLITE_DENY, + SQLITE_DETACH as SQLITE_DETACH, + SQLITE_DONE as SQLITE_DONE, + SQLITE_DROP_INDEX as SQLITE_DROP_INDEX, + SQLITE_DROP_TABLE as SQLITE_DROP_TABLE, + SQLITE_DROP_TEMP_INDEX as SQLITE_DROP_TEMP_INDEX, + SQLITE_DROP_TEMP_TABLE as SQLITE_DROP_TEMP_TABLE, + SQLITE_DROP_TEMP_TRIGGER as SQLITE_DROP_TEMP_TRIGGER, + SQLITE_DROP_TEMP_VIEW as SQLITE_DROP_TEMP_VIEW, + SQLITE_DROP_TRIGGER as SQLITE_DROP_TRIGGER, + SQLITE_DROP_VIEW as SQLITE_DROP_VIEW, + SQLITE_DROP_VTABLE as SQLITE_DROP_VTABLE, + SQLITE_FUNCTION as SQLITE_FUNCTION, + SQLITE_IGNORE as SQLITE_IGNORE, + SQLITE_INSERT as SQLITE_INSERT, + SQLITE_OK as SQLITE_OK, + SQLITE_PRAGMA as SQLITE_PRAGMA, + SQLITE_READ as SQLITE_READ, + SQLITE_RECURSIVE as SQLITE_RECURSIVE, + SQLITE_REINDEX as SQLITE_REINDEX, + SQLITE_SAVEPOINT as SQLITE_SAVEPOINT, + SQLITE_SELECT as SQLITE_SELECT, + SQLITE_TRANSACTION as SQLITE_TRANSACTION, + SQLITE_UPDATE as SQLITE_UPDATE, + adapt as adapt, + adapters as adapters, + complete_statement as complete_statement, + connect as connect, + converters as converters, + enable_callback_tracebacks as enable_callback_tracebacks, + register_adapter as register_adapter, + register_converter as register_converter, + sqlite_version as sqlite_version, +) +from datetime import date, datetime, time +from sqlite3 import ( + Connection as Connection, + Cursor as Cursor, + DatabaseError as DatabaseError, + DataError as DataError, + Error as Error, + IntegrityError as IntegrityError, + InterfaceError as InterfaceError, + InternalError as InternalError, + NotSupportedError as NotSupportedError, + OperationalError as OperationalError, + PrepareProtocol as PrepareProtocol, + ProgrammingError as ProgrammingError, + Row as Row, + Warning as Warning, +) + +if sys.version_info >= (3, 12): + from _sqlite3 import ( + LEGACY_TRANSACTION_CONTROL as LEGACY_TRANSACTION_CONTROL, + SQLITE_DBCONFIG_DEFENSIVE as SQLITE_DBCONFIG_DEFENSIVE, + SQLITE_DBCONFIG_DQS_DDL as SQLITE_DBCONFIG_DQS_DDL, + SQLITE_DBCONFIG_DQS_DML as SQLITE_DBCONFIG_DQS_DML, + SQLITE_DBCONFIG_ENABLE_FKEY as SQLITE_DBCONFIG_ENABLE_FKEY, + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER as SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION as SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, + SQLITE_DBCONFIG_ENABLE_QPSG as SQLITE_DBCONFIG_ENABLE_QPSG, + SQLITE_DBCONFIG_ENABLE_TRIGGER as SQLITE_DBCONFIG_ENABLE_TRIGGER, + SQLITE_DBCONFIG_ENABLE_VIEW as SQLITE_DBCONFIG_ENABLE_VIEW, + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE as SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT as SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE as SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + SQLITE_DBCONFIG_RESET_DATABASE as SQLITE_DBCONFIG_RESET_DATABASE, + SQLITE_DBCONFIG_TRIGGER_EQP as SQLITE_DBCONFIG_TRIGGER_EQP, + SQLITE_DBCONFIG_TRUSTED_SCHEMA as SQLITE_DBCONFIG_TRUSTED_SCHEMA, + SQLITE_DBCONFIG_WRITABLE_SCHEMA as SQLITE_DBCONFIG_WRITABLE_SCHEMA, + ) + +if sys.version_info >= (3, 11): + from _sqlite3 import ( + SQLITE_ABORT as SQLITE_ABORT, + SQLITE_ABORT_ROLLBACK as SQLITE_ABORT_ROLLBACK, + SQLITE_AUTH as SQLITE_AUTH, + SQLITE_AUTH_USER as SQLITE_AUTH_USER, + SQLITE_BUSY as SQLITE_BUSY, + SQLITE_BUSY_RECOVERY as SQLITE_BUSY_RECOVERY, + SQLITE_BUSY_SNAPSHOT as SQLITE_BUSY_SNAPSHOT, + SQLITE_BUSY_TIMEOUT as SQLITE_BUSY_TIMEOUT, + SQLITE_CANTOPEN as SQLITE_CANTOPEN, + SQLITE_CANTOPEN_CONVPATH as SQLITE_CANTOPEN_CONVPATH, + SQLITE_CANTOPEN_DIRTYWAL as SQLITE_CANTOPEN_DIRTYWAL, + SQLITE_CANTOPEN_FULLPATH as SQLITE_CANTOPEN_FULLPATH, + SQLITE_CANTOPEN_ISDIR as SQLITE_CANTOPEN_ISDIR, + SQLITE_CANTOPEN_NOTEMPDIR as SQLITE_CANTOPEN_NOTEMPDIR, + SQLITE_CANTOPEN_SYMLINK as SQLITE_CANTOPEN_SYMLINK, + SQLITE_CONSTRAINT as SQLITE_CONSTRAINT, + SQLITE_CONSTRAINT_CHECK as SQLITE_CONSTRAINT_CHECK, + SQLITE_CONSTRAINT_COMMITHOOK as SQLITE_CONSTRAINT_COMMITHOOK, + SQLITE_CONSTRAINT_FOREIGNKEY as SQLITE_CONSTRAINT_FOREIGNKEY, + SQLITE_CONSTRAINT_FUNCTION as SQLITE_CONSTRAINT_FUNCTION, + SQLITE_CONSTRAINT_NOTNULL as SQLITE_CONSTRAINT_NOTNULL, + SQLITE_CONSTRAINT_PINNED as SQLITE_CONSTRAINT_PINNED, + SQLITE_CONSTRAINT_PRIMARYKEY as SQLITE_CONSTRAINT_PRIMARYKEY, + SQLITE_CONSTRAINT_ROWID as SQLITE_CONSTRAINT_ROWID, + SQLITE_CONSTRAINT_TRIGGER as SQLITE_CONSTRAINT_TRIGGER, + SQLITE_CONSTRAINT_UNIQUE as SQLITE_CONSTRAINT_UNIQUE, + SQLITE_CONSTRAINT_VTAB as SQLITE_CONSTRAINT_VTAB, + SQLITE_CORRUPT as SQLITE_CORRUPT, + SQLITE_CORRUPT_INDEX as SQLITE_CORRUPT_INDEX, + SQLITE_CORRUPT_SEQUENCE as SQLITE_CORRUPT_SEQUENCE, + SQLITE_CORRUPT_VTAB as SQLITE_CORRUPT_VTAB, + SQLITE_EMPTY as SQLITE_EMPTY, + SQLITE_ERROR as SQLITE_ERROR, + SQLITE_ERROR_MISSING_COLLSEQ as SQLITE_ERROR_MISSING_COLLSEQ, + SQLITE_ERROR_RETRY as SQLITE_ERROR_RETRY, + SQLITE_ERROR_SNAPSHOT as SQLITE_ERROR_SNAPSHOT, + SQLITE_FORMAT as SQLITE_FORMAT, + SQLITE_FULL as SQLITE_FULL, + SQLITE_INTERNAL as SQLITE_INTERNAL, + SQLITE_INTERRUPT as SQLITE_INTERRUPT, + SQLITE_IOERR as SQLITE_IOERR, + SQLITE_IOERR_ACCESS as SQLITE_IOERR_ACCESS, + SQLITE_IOERR_AUTH as SQLITE_IOERR_AUTH, + SQLITE_IOERR_BEGIN_ATOMIC as SQLITE_IOERR_BEGIN_ATOMIC, + SQLITE_IOERR_BLOCKED as SQLITE_IOERR_BLOCKED, + SQLITE_IOERR_CHECKRESERVEDLOCK as SQLITE_IOERR_CHECKRESERVEDLOCK, + SQLITE_IOERR_CLOSE as SQLITE_IOERR_CLOSE, + SQLITE_IOERR_COMMIT_ATOMIC as SQLITE_IOERR_COMMIT_ATOMIC, + SQLITE_IOERR_CONVPATH as SQLITE_IOERR_CONVPATH, + SQLITE_IOERR_CORRUPTFS as SQLITE_IOERR_CORRUPTFS, + SQLITE_IOERR_DATA as SQLITE_IOERR_DATA, + SQLITE_IOERR_DELETE as SQLITE_IOERR_DELETE, + SQLITE_IOERR_DELETE_NOENT as SQLITE_IOERR_DELETE_NOENT, + SQLITE_IOERR_DIR_CLOSE as SQLITE_IOERR_DIR_CLOSE, + SQLITE_IOERR_DIR_FSYNC as SQLITE_IOERR_DIR_FSYNC, + SQLITE_IOERR_FSTAT as SQLITE_IOERR_FSTAT, + SQLITE_IOERR_FSYNC as SQLITE_IOERR_FSYNC, + SQLITE_IOERR_GETTEMPPATH as SQLITE_IOERR_GETTEMPPATH, + SQLITE_IOERR_LOCK as SQLITE_IOERR_LOCK, + SQLITE_IOERR_MMAP as SQLITE_IOERR_MMAP, + SQLITE_IOERR_NOMEM as SQLITE_IOERR_NOMEM, + SQLITE_IOERR_RDLOCK as SQLITE_IOERR_RDLOCK, + SQLITE_IOERR_READ as SQLITE_IOERR_READ, + SQLITE_IOERR_ROLLBACK_ATOMIC as SQLITE_IOERR_ROLLBACK_ATOMIC, + SQLITE_IOERR_SEEK as SQLITE_IOERR_SEEK, + SQLITE_IOERR_SHMLOCK as SQLITE_IOERR_SHMLOCK, + SQLITE_IOERR_SHMMAP as SQLITE_IOERR_SHMMAP, + SQLITE_IOERR_SHMOPEN as SQLITE_IOERR_SHMOPEN, + SQLITE_IOERR_SHMSIZE as SQLITE_IOERR_SHMSIZE, + SQLITE_IOERR_SHORT_READ as SQLITE_IOERR_SHORT_READ, + SQLITE_IOERR_TRUNCATE as SQLITE_IOERR_TRUNCATE, + SQLITE_IOERR_UNLOCK as SQLITE_IOERR_UNLOCK, + SQLITE_IOERR_VNODE as SQLITE_IOERR_VNODE, + SQLITE_IOERR_WRITE as SQLITE_IOERR_WRITE, + SQLITE_LIMIT_ATTACHED as SQLITE_LIMIT_ATTACHED, + SQLITE_LIMIT_COLUMN as SQLITE_LIMIT_COLUMN, + SQLITE_LIMIT_COMPOUND_SELECT as SQLITE_LIMIT_COMPOUND_SELECT, + SQLITE_LIMIT_EXPR_DEPTH as SQLITE_LIMIT_EXPR_DEPTH, + SQLITE_LIMIT_FUNCTION_ARG as SQLITE_LIMIT_FUNCTION_ARG, + SQLITE_LIMIT_LENGTH as SQLITE_LIMIT_LENGTH, + SQLITE_LIMIT_LIKE_PATTERN_LENGTH as SQLITE_LIMIT_LIKE_PATTERN_LENGTH, + SQLITE_LIMIT_SQL_LENGTH as SQLITE_LIMIT_SQL_LENGTH, + SQLITE_LIMIT_TRIGGER_DEPTH as SQLITE_LIMIT_TRIGGER_DEPTH, + SQLITE_LIMIT_VARIABLE_NUMBER as SQLITE_LIMIT_VARIABLE_NUMBER, + SQLITE_LIMIT_VDBE_OP as SQLITE_LIMIT_VDBE_OP, + SQLITE_LIMIT_WORKER_THREADS as SQLITE_LIMIT_WORKER_THREADS, + SQLITE_LOCKED as SQLITE_LOCKED, + SQLITE_LOCKED_SHAREDCACHE as SQLITE_LOCKED_SHAREDCACHE, + SQLITE_LOCKED_VTAB as SQLITE_LOCKED_VTAB, + SQLITE_MISMATCH as SQLITE_MISMATCH, + SQLITE_MISUSE as SQLITE_MISUSE, + SQLITE_NOLFS as SQLITE_NOLFS, + SQLITE_NOMEM as SQLITE_NOMEM, + SQLITE_NOTADB as SQLITE_NOTADB, + SQLITE_NOTFOUND as SQLITE_NOTFOUND, + SQLITE_NOTICE as SQLITE_NOTICE, + SQLITE_NOTICE_RECOVER_ROLLBACK as SQLITE_NOTICE_RECOVER_ROLLBACK, + SQLITE_NOTICE_RECOVER_WAL as SQLITE_NOTICE_RECOVER_WAL, + SQLITE_OK_LOAD_PERMANENTLY as SQLITE_OK_LOAD_PERMANENTLY, + SQLITE_OK_SYMLINK as SQLITE_OK_SYMLINK, + SQLITE_PERM as SQLITE_PERM, + SQLITE_PROTOCOL as SQLITE_PROTOCOL, + SQLITE_RANGE as SQLITE_RANGE, + SQLITE_READONLY as SQLITE_READONLY, + SQLITE_READONLY_CANTINIT as SQLITE_READONLY_CANTINIT, + SQLITE_READONLY_CANTLOCK as SQLITE_READONLY_CANTLOCK, + SQLITE_READONLY_DBMOVED as SQLITE_READONLY_DBMOVED, + SQLITE_READONLY_DIRECTORY as SQLITE_READONLY_DIRECTORY, + SQLITE_READONLY_RECOVERY as SQLITE_READONLY_RECOVERY, + SQLITE_READONLY_ROLLBACK as SQLITE_READONLY_ROLLBACK, + SQLITE_ROW as SQLITE_ROW, + SQLITE_SCHEMA as SQLITE_SCHEMA, + SQLITE_TOOBIG as SQLITE_TOOBIG, + SQLITE_WARNING as SQLITE_WARNING, + SQLITE_WARNING_AUTOINDEX as SQLITE_WARNING_AUTOINDEX, + ) + from sqlite3 import Blob as Blob + +if sys.version_info < (3, 14): + # Deprecated and removed from _sqlite3 in 3.12, but removed from here in 3.14. + version: str + +if sys.version_info < (3, 12): + if sys.version_info >= (3, 10): + # deprecation wrapper that has a different name for the argument... + def enable_shared_cache(enable: int) -> None: ... + else: + from _sqlite3 import enable_shared_cache as enable_shared_cache + +if sys.version_info < (3, 10): + from _sqlite3 import OptimizedUnicode as OptimizedUnicode + +paramstyle: str +threadsafety: int +apilevel: str +Date = date +Time = time +Timestamp = datetime + +def DateFromTicks(ticks: float) -> Date: ... +def TimeFromTicks(ticks: float) -> Time: ... +def TimestampFromTicks(ticks: float) -> Timestamp: ... + +if sys.version_info < (3, 14): + # Deprecated in 3.12, removed in 3.14. + version_info: tuple[int, int, int] + +sqlite_version_info: tuple[int, int, int] +Binary = memoryview diff --git a/mypy/typeshed/stdlib/sqlite3/dump.pyi b/mypy/typeshed/stdlib/sqlite3/dump.pyi new file mode 100644 index 000000000000..ed95fa46e1c7 --- /dev/null +++ b/mypy/typeshed/stdlib/sqlite3/dump.pyi @@ -0,0 +1,2 @@ +# This file is intentionally empty. The runtime module contains only +# private functions. diff --git a/mypy/typeshed/stdlib/sre_compile.pyi b/mypy/typeshed/stdlib/sre_compile.pyi new file mode 100644 index 000000000000..2d04a886c931 --- /dev/null +++ b/mypy/typeshed/stdlib/sre_compile.pyi @@ -0,0 +1,11 @@ +from re import Pattern +from sre_constants import * +from sre_constants import _NamedIntConstant +from sre_parse import SubPattern +from typing import Any + +MAXCODE: int + +def dis(code: list[_NamedIntConstant]) -> None: ... +def isstring(obj: Any) -> bool: ... +def compile(p: str | bytes | SubPattern, flags: int = 0) -> Pattern[Any]: ... diff --git a/mypy/typeshed/stdlib/sre_constants.pyi b/mypy/typeshed/stdlib/sre_constants.pyi new file mode 100644 index 000000000000..383f0f7eb8bd --- /dev/null +++ b/mypy/typeshed/stdlib/sre_constants.pyi @@ -0,0 +1,124 @@ +import sys +from re import error as error +from typing import Any +from typing_extensions import Self + +MAXGROUPS: int + +MAGIC: int + +class _NamedIntConstant(int): + name: Any + def __new__(cls, value: int, name: str) -> Self: ... + +MAXREPEAT: _NamedIntConstant +OPCODES: list[_NamedIntConstant] +ATCODES: list[_NamedIntConstant] +CHCODES: list[_NamedIntConstant] +OP_IGNORE: dict[_NamedIntConstant, _NamedIntConstant] +OP_LOCALE_IGNORE: dict[_NamedIntConstant, _NamedIntConstant] +OP_UNICODE_IGNORE: dict[_NamedIntConstant, _NamedIntConstant] +AT_MULTILINE: dict[_NamedIntConstant, _NamedIntConstant] +AT_LOCALE: dict[_NamedIntConstant, _NamedIntConstant] +AT_UNICODE: dict[_NamedIntConstant, _NamedIntConstant] +CH_LOCALE: dict[_NamedIntConstant, _NamedIntConstant] +CH_UNICODE: dict[_NamedIntConstant, _NamedIntConstant] +if sys.version_info < (3, 13): + SRE_FLAG_TEMPLATE: int +SRE_FLAG_IGNORECASE: int +SRE_FLAG_LOCALE: int +SRE_FLAG_MULTILINE: int +SRE_FLAG_DOTALL: int +SRE_FLAG_UNICODE: int +SRE_FLAG_VERBOSE: int +SRE_FLAG_DEBUG: int +SRE_FLAG_ASCII: int +SRE_INFO_PREFIX: int +SRE_INFO_LITERAL: int +SRE_INFO_CHARSET: int + +# Stubgen above; manually defined constants below (dynamic at runtime) + +# from OPCODES +FAILURE: _NamedIntConstant +SUCCESS: _NamedIntConstant +ANY: _NamedIntConstant +ANY_ALL: _NamedIntConstant +ASSERT: _NamedIntConstant +ASSERT_NOT: _NamedIntConstant +AT: _NamedIntConstant +BRANCH: _NamedIntConstant +if sys.version_info < (3, 11): + CALL: _NamedIntConstant +CATEGORY: _NamedIntConstant +CHARSET: _NamedIntConstant +BIGCHARSET: _NamedIntConstant +GROUPREF: _NamedIntConstant +GROUPREF_EXISTS: _NamedIntConstant +GROUPREF_IGNORE: _NamedIntConstant +IN: _NamedIntConstant +IN_IGNORE: _NamedIntConstant +INFO: _NamedIntConstant +JUMP: _NamedIntConstant +LITERAL: _NamedIntConstant +LITERAL_IGNORE: _NamedIntConstant +MARK: _NamedIntConstant +MAX_UNTIL: _NamedIntConstant +MIN_UNTIL: _NamedIntConstant +NOT_LITERAL: _NamedIntConstant +NOT_LITERAL_IGNORE: _NamedIntConstant +NEGATE: _NamedIntConstant +RANGE: _NamedIntConstant +REPEAT: _NamedIntConstant +REPEAT_ONE: _NamedIntConstant +SUBPATTERN: _NamedIntConstant +MIN_REPEAT_ONE: _NamedIntConstant +if sys.version_info >= (3, 11): + ATOMIC_GROUP: _NamedIntConstant + POSSESSIVE_REPEAT: _NamedIntConstant + POSSESSIVE_REPEAT_ONE: _NamedIntConstant +RANGE_UNI_IGNORE: _NamedIntConstant +GROUPREF_LOC_IGNORE: _NamedIntConstant +GROUPREF_UNI_IGNORE: _NamedIntConstant +IN_LOC_IGNORE: _NamedIntConstant +IN_UNI_IGNORE: _NamedIntConstant +LITERAL_LOC_IGNORE: _NamedIntConstant +LITERAL_UNI_IGNORE: _NamedIntConstant +NOT_LITERAL_LOC_IGNORE: _NamedIntConstant +NOT_LITERAL_UNI_IGNORE: _NamedIntConstant +MIN_REPEAT: _NamedIntConstant +MAX_REPEAT: _NamedIntConstant + +# from ATCODES +AT_BEGINNING: _NamedIntConstant +AT_BEGINNING_LINE: _NamedIntConstant +AT_BEGINNING_STRING: _NamedIntConstant +AT_BOUNDARY: _NamedIntConstant +AT_NON_BOUNDARY: _NamedIntConstant +AT_END: _NamedIntConstant +AT_END_LINE: _NamedIntConstant +AT_END_STRING: _NamedIntConstant +AT_LOC_BOUNDARY: _NamedIntConstant +AT_LOC_NON_BOUNDARY: _NamedIntConstant +AT_UNI_BOUNDARY: _NamedIntConstant +AT_UNI_NON_BOUNDARY: _NamedIntConstant + +# from CHCODES +CATEGORY_DIGIT: _NamedIntConstant +CATEGORY_NOT_DIGIT: _NamedIntConstant +CATEGORY_SPACE: _NamedIntConstant +CATEGORY_NOT_SPACE: _NamedIntConstant +CATEGORY_WORD: _NamedIntConstant +CATEGORY_NOT_WORD: _NamedIntConstant +CATEGORY_LINEBREAK: _NamedIntConstant +CATEGORY_NOT_LINEBREAK: _NamedIntConstant +CATEGORY_LOC_WORD: _NamedIntConstant +CATEGORY_LOC_NOT_WORD: _NamedIntConstant +CATEGORY_UNI_DIGIT: _NamedIntConstant +CATEGORY_UNI_NOT_DIGIT: _NamedIntConstant +CATEGORY_UNI_SPACE: _NamedIntConstant +CATEGORY_UNI_NOT_SPACE: _NamedIntConstant +CATEGORY_UNI_WORD: _NamedIntConstant +CATEGORY_UNI_NOT_WORD: _NamedIntConstant +CATEGORY_UNI_LINEBREAK: _NamedIntConstant +CATEGORY_UNI_NOT_LINEBREAK: _NamedIntConstant diff --git a/mypy/typeshed/stdlib/sre_parse.pyi b/mypy/typeshed/stdlib/sre_parse.pyi new file mode 100644 index 000000000000..c242bd2a065f --- /dev/null +++ b/mypy/typeshed/stdlib/sre_parse.pyi @@ -0,0 +1,104 @@ +import sys +from collections.abc import Iterable +from re import Match, Pattern as _Pattern +from sre_constants import * +from sre_constants import _NamedIntConstant as _NIC, error as _Error +from typing import Any, overload +from typing_extensions import TypeAlias + +SPECIAL_CHARS: str +REPEAT_CHARS: str +DIGITS: frozenset[str] +OCTDIGITS: frozenset[str] +HEXDIGITS: frozenset[str] +ASCIILETTERS: frozenset[str] +WHITESPACE: frozenset[str] +ESCAPES: dict[str, tuple[_NIC, int]] +CATEGORIES: dict[str, tuple[_NIC, _NIC] | tuple[_NIC, list[tuple[_NIC, _NIC]]]] +FLAGS: dict[str, int] +TYPE_FLAGS: int +GLOBAL_FLAGS: int + +if sys.version_info >= (3, 11): + MAXWIDTH: int + +if sys.version_info < (3, 11): + class Verbose(Exception): ... + +_OpSubpatternType: TypeAlias = tuple[int | None, int, int, SubPattern] +_OpGroupRefExistsType: TypeAlias = tuple[int, SubPattern, SubPattern] +_OpInType: TypeAlias = list[tuple[_NIC, int]] +_OpBranchType: TypeAlias = tuple[None, list[SubPattern]] +_AvType: TypeAlias = _OpInType | _OpBranchType | Iterable[SubPattern] | _OpGroupRefExistsType | _OpSubpatternType +_CodeType: TypeAlias = tuple[_NIC, _AvType] + +class State: + flags: int + groupdict: dict[str, int] + groupwidths: list[int | None] + lookbehindgroups: int | None + @property + def groups(self) -> int: ... + def opengroup(self, name: str | None = ...) -> int: ... + def closegroup(self, gid: int, p: SubPattern) -> None: ... + def checkgroup(self, gid: int) -> bool: ... + def checklookbehindgroup(self, gid: int, source: Tokenizer) -> None: ... + +class SubPattern: + data: list[_CodeType] + width: int | None + state: State + + def __init__(self, state: State, data: list[_CodeType] | None = None) -> None: ... + def dump(self, level: int = 0) -> None: ... + def __len__(self) -> int: ... + def __delitem__(self, index: int | slice) -> None: ... + def __getitem__(self, index: int | slice) -> SubPattern | _CodeType: ... + def __setitem__(self, index: int | slice, code: _CodeType) -> None: ... + def insert(self, index: int, code: _CodeType) -> None: ... + def append(self, code: _CodeType) -> None: ... + def getwidth(self) -> tuple[int, int]: ... + +class Tokenizer: + istext: bool + string: Any + decoded_string: str + index: int + next: str | None + def __init__(self, string: Any) -> None: ... + def match(self, char: str) -> bool: ... + def get(self) -> str | None: ... + def getwhile(self, n: int, charset: Iterable[str]) -> str: ... + def getuntil(self, terminator: str, name: str) -> str: ... + @property + def pos(self) -> int: ... + def tell(self) -> int: ... + def seek(self, index: int) -> None: ... + def error(self, msg: str, offset: int = 0) -> _Error: ... + + if sys.version_info >= (3, 12): + def checkgroupname(self, name: str, offset: int) -> None: ... + elif sys.version_info >= (3, 11): + def checkgroupname(self, name: str, offset: int, nested: int) -> None: ... + +def fix_flags(src: str | bytes, flags: int) -> int: ... + +_TemplateType: TypeAlias = tuple[list[tuple[int, int]], list[str | None]] +_TemplateByteType: TypeAlias = tuple[list[tuple[int, int]], list[bytes | None]] + +if sys.version_info >= (3, 12): + @overload + def parse_template(source: str, pattern: _Pattern[Any]) -> _TemplateType: ... + @overload + def parse_template(source: bytes, pattern: _Pattern[Any]) -> _TemplateByteType: ... + +else: + @overload + def parse_template(source: str, state: _Pattern[Any]) -> _TemplateType: ... + @overload + def parse_template(source: bytes, state: _Pattern[Any]) -> _TemplateByteType: ... + +def parse(str: str, flags: int = 0, state: State | None = None) -> SubPattern: ... + +if sys.version_info < (3, 12): + def expand_template(template: _TemplateType, match: Match[Any]) -> str: ... diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi new file mode 100644 index 000000000000..f587b51d5ac0 --- /dev/null +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -0,0 +1,517 @@ +import enum +import socket +import sys +from _ssl import ( + _DEFAULT_CIPHERS as _DEFAULT_CIPHERS, + _OPENSSL_API_VERSION as _OPENSSL_API_VERSION, + HAS_ALPN as HAS_ALPN, + HAS_ECDH as HAS_ECDH, + HAS_NPN as HAS_NPN, + HAS_SNI as HAS_SNI, + OPENSSL_VERSION as OPENSSL_VERSION, + OPENSSL_VERSION_INFO as OPENSSL_VERSION_INFO, + OPENSSL_VERSION_NUMBER as OPENSSL_VERSION_NUMBER, + HAS_SSLv2 as HAS_SSLv2, + HAS_SSLv3 as HAS_SSLv3, + HAS_TLSv1 as HAS_TLSv1, + HAS_TLSv1_1 as HAS_TLSv1_1, + HAS_TLSv1_2 as HAS_TLSv1_2, + HAS_TLSv1_3 as HAS_TLSv1_3, + MemoryBIO as MemoryBIO, + RAND_add as RAND_add, + RAND_bytes as RAND_bytes, + RAND_status as RAND_status, + SSLSession as SSLSession, + _PasswordType as _PasswordType, # typeshed only, but re-export for other type stubs to use + _SSLContext, +) +from _typeshed import ReadableBuffer, StrOrBytesPath, WriteableBuffer +from collections.abc import Callable, Iterable +from typing import Any, Literal, NamedTuple, TypedDict, overload, type_check_only +from typing_extensions import Never, Self, TypeAlias + +if sys.version_info >= (3, 13): + from _ssl import HAS_PSK as HAS_PSK + +if sys.version_info < (3, 12): + from _ssl import RAND_pseudo_bytes as RAND_pseudo_bytes + +if sys.version_info < (3, 10): + from _ssl import RAND_egd as RAND_egd + +if sys.platform == "win32": + from _ssl import enum_certificates as enum_certificates, enum_crls as enum_crls + +_PCTRTT: TypeAlias = tuple[tuple[str, str], ...] +_PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] +_PeerCertRetDictType: TypeAlias = dict[str, str | _PCTRTTT | _PCTRTT] +_PeerCertRetType: TypeAlias = _PeerCertRetDictType | bytes | None +_SrvnmeCbType: TypeAlias = Callable[[SSLSocket | SSLObject, str | None, SSLSocket], int | None] + +socket_error = OSError + +class _Cipher(TypedDict): + aead: bool + alg_bits: int + auth: str + description: str + digest: str | None + id: int + kea: str + name: str + protocol: str + strength_bits: int + symmetric: str + +class SSLError(OSError): + library: str + reason: str + +class SSLZeroReturnError(SSLError): ... +class SSLWantReadError(SSLError): ... +class SSLWantWriteError(SSLError): ... +class SSLSyscallError(SSLError): ... +class SSLEOFError(SSLError): ... + +class SSLCertVerificationError(SSLError, ValueError): + verify_code: int + verify_message: str + +CertificateError = SSLCertVerificationError + +if sys.version_info < (3, 12): + def wrap_socket( + sock: socket.socket, + keyfile: StrOrBytesPath | None = None, + certfile: StrOrBytesPath | None = None, + server_side: bool = False, + cert_reqs: int = ..., + ssl_version: int = ..., + ca_certs: str | None = None, + do_handshake_on_connect: bool = True, + suppress_ragged_eofs: bool = True, + ciphers: str | None = None, + ) -> SSLSocket: ... + +def create_default_context( + purpose: Purpose = ..., + *, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, +) -> SSLContext: ... + +if sys.version_info >= (3, 10): + def _create_unverified_context( + protocol: int | None = None, + *, + cert_reqs: int = ..., + check_hostname: bool = False, + purpose: Purpose = ..., + certfile: StrOrBytesPath | None = None, + keyfile: StrOrBytesPath | None = None, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, + ) -> SSLContext: ... + +else: + def _create_unverified_context( + protocol: int = ..., + *, + cert_reqs: int = ..., + check_hostname: bool = False, + purpose: Purpose = ..., + certfile: StrOrBytesPath | None = None, + keyfile: StrOrBytesPath | None = None, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, + ) -> SSLContext: ... + +_create_default_https_context: Callable[..., SSLContext] + +if sys.version_info < (3, 12): + def match_hostname(cert: _PeerCertRetDictType, hostname: str) -> None: ... + +def cert_time_to_seconds(cert_time: str) -> int: ... + +if sys.version_info >= (3, 10): + def get_server_certificate( + addr: tuple[str, int], ssl_version: int = ..., ca_certs: str | None = None, timeout: float = ... + ) -> str: ... + +else: + def get_server_certificate(addr: tuple[str, int], ssl_version: int = ..., ca_certs: str | None = None) -> str: ... + +def DER_cert_to_PEM_cert(der_cert_bytes: ReadableBuffer) -> str: ... +def PEM_cert_to_DER_cert(pem_cert_string: str) -> bytes: ... + +class DefaultVerifyPaths(NamedTuple): + cafile: str + capath: str + openssl_cafile_env: str + openssl_cafile: str + openssl_capath_env: str + openssl_capath: str + +def get_default_verify_paths() -> DefaultVerifyPaths: ... + +class VerifyMode(enum.IntEnum): + CERT_NONE = 0 + CERT_OPTIONAL = 1 + CERT_REQUIRED = 2 + +CERT_NONE: VerifyMode +CERT_OPTIONAL: VerifyMode +CERT_REQUIRED: VerifyMode + +class VerifyFlags(enum.IntFlag): + VERIFY_DEFAULT = 0 + VERIFY_CRL_CHECK_LEAF = 4 + VERIFY_CRL_CHECK_CHAIN = 12 + VERIFY_X509_STRICT = 32 + VERIFY_X509_TRUSTED_FIRST = 32768 + if sys.version_info >= (3, 10): + VERIFY_ALLOW_PROXY_CERTS = 64 + VERIFY_X509_PARTIAL_CHAIN = 524288 + +VERIFY_DEFAULT: VerifyFlags +VERIFY_CRL_CHECK_LEAF: VerifyFlags +VERIFY_CRL_CHECK_CHAIN: VerifyFlags +VERIFY_X509_STRICT: VerifyFlags +VERIFY_X509_TRUSTED_FIRST: VerifyFlags + +if sys.version_info >= (3, 10): + VERIFY_ALLOW_PROXY_CERTS: VerifyFlags + VERIFY_X509_PARTIAL_CHAIN: VerifyFlags + +class _SSLMethod(enum.IntEnum): + PROTOCOL_SSLv23 = 2 + PROTOCOL_SSLv2 = ... + PROTOCOL_SSLv3 = ... + PROTOCOL_TLSv1 = 3 + PROTOCOL_TLSv1_1 = 4 + PROTOCOL_TLSv1_2 = 5 + PROTOCOL_TLS = 2 + PROTOCOL_TLS_CLIENT = 16 + PROTOCOL_TLS_SERVER = 17 + +PROTOCOL_SSLv23: _SSLMethod +PROTOCOL_SSLv2: _SSLMethod +PROTOCOL_SSLv3: _SSLMethod +PROTOCOL_TLSv1: _SSLMethod +PROTOCOL_TLSv1_1: _SSLMethod +PROTOCOL_TLSv1_2: _SSLMethod +PROTOCOL_TLS: _SSLMethod +PROTOCOL_TLS_CLIENT: _SSLMethod +PROTOCOL_TLS_SERVER: _SSLMethod + +class Options(enum.IntFlag): + OP_ALL = 2147483728 + OP_NO_SSLv2 = 0 + OP_NO_SSLv3 = 33554432 + OP_NO_TLSv1 = 67108864 + OP_NO_TLSv1_1 = 268435456 + OP_NO_TLSv1_2 = 134217728 + OP_NO_TLSv1_3 = 536870912 + OP_CIPHER_SERVER_PREFERENCE = 4194304 + OP_SINGLE_DH_USE = 0 + OP_SINGLE_ECDH_USE = 0 + OP_NO_COMPRESSION = 131072 + OP_NO_TICKET = 16384 + OP_NO_RENEGOTIATION = 1073741824 + OP_ENABLE_MIDDLEBOX_COMPAT = 1048576 + if sys.version_info >= (3, 12): + OP_LEGACY_SERVER_CONNECT = 4 + OP_ENABLE_KTLS = 8 + if sys.version_info >= (3, 11) or sys.platform == "linux": + OP_IGNORE_UNEXPECTED_EOF = 128 + +OP_ALL: Options +OP_NO_SSLv2: Options +OP_NO_SSLv3: Options +OP_NO_TLSv1: Options +OP_NO_TLSv1_1: Options +OP_NO_TLSv1_2: Options +OP_NO_TLSv1_3: Options +OP_CIPHER_SERVER_PREFERENCE: Options +OP_SINGLE_DH_USE: Options +OP_SINGLE_ECDH_USE: Options +OP_NO_COMPRESSION: Options +OP_NO_TICKET: Options +OP_NO_RENEGOTIATION: Options +OP_ENABLE_MIDDLEBOX_COMPAT: Options +if sys.version_info >= (3, 12): + OP_LEGACY_SERVER_CONNECT: Options + OP_ENABLE_KTLS: Options +if sys.version_info >= (3, 11) or sys.platform == "linux": + OP_IGNORE_UNEXPECTED_EOF: Options + +HAS_NEVER_CHECK_COMMON_NAME: bool + +CHANNEL_BINDING_TYPES: list[str] + +class AlertDescription(enum.IntEnum): + ALERT_DESCRIPTION_ACCESS_DENIED = 49 + ALERT_DESCRIPTION_BAD_CERTIFICATE = 42 + ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE = 114 + ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE = 113 + ALERT_DESCRIPTION_BAD_RECORD_MAC = 20 + ALERT_DESCRIPTION_CERTIFICATE_EXPIRED = 45 + ALERT_DESCRIPTION_CERTIFICATE_REVOKED = 44 + ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN = 46 + ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE = 111 + ALERT_DESCRIPTION_CLOSE_NOTIFY = 0 + ALERT_DESCRIPTION_DECODE_ERROR = 50 + ALERT_DESCRIPTION_DECOMPRESSION_FAILURE = 30 + ALERT_DESCRIPTION_DECRYPT_ERROR = 51 + ALERT_DESCRIPTION_HANDSHAKE_FAILURE = 40 + ALERT_DESCRIPTION_ILLEGAL_PARAMETER = 47 + ALERT_DESCRIPTION_INSUFFICIENT_SECURITY = 71 + ALERT_DESCRIPTION_INTERNAL_ERROR = 80 + ALERT_DESCRIPTION_NO_RENEGOTIATION = 100 + ALERT_DESCRIPTION_PROTOCOL_VERSION = 70 + ALERT_DESCRIPTION_RECORD_OVERFLOW = 22 + ALERT_DESCRIPTION_UNEXPECTED_MESSAGE = 10 + ALERT_DESCRIPTION_UNKNOWN_CA = 48 + ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY = 115 + ALERT_DESCRIPTION_UNRECOGNIZED_NAME = 112 + ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE = 43 + ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION = 110 + ALERT_DESCRIPTION_USER_CANCELLED = 90 + +ALERT_DESCRIPTION_HANDSHAKE_FAILURE: AlertDescription +ALERT_DESCRIPTION_INTERNAL_ERROR: AlertDescription +ALERT_DESCRIPTION_ACCESS_DENIED: AlertDescription +ALERT_DESCRIPTION_BAD_CERTIFICATE: AlertDescription +ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: AlertDescription +ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: AlertDescription +ALERT_DESCRIPTION_BAD_RECORD_MAC: AlertDescription +ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: AlertDescription +ALERT_DESCRIPTION_CERTIFICATE_REVOKED: AlertDescription +ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: AlertDescription +ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: AlertDescription +ALERT_DESCRIPTION_CLOSE_NOTIFY: AlertDescription +ALERT_DESCRIPTION_DECODE_ERROR: AlertDescription +ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: AlertDescription +ALERT_DESCRIPTION_DECRYPT_ERROR: AlertDescription +ALERT_DESCRIPTION_ILLEGAL_PARAMETER: AlertDescription +ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: AlertDescription +ALERT_DESCRIPTION_NO_RENEGOTIATION: AlertDescription +ALERT_DESCRIPTION_PROTOCOL_VERSION: AlertDescription +ALERT_DESCRIPTION_RECORD_OVERFLOW: AlertDescription +ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: AlertDescription +ALERT_DESCRIPTION_UNKNOWN_CA: AlertDescription +ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: AlertDescription +ALERT_DESCRIPTION_UNRECOGNIZED_NAME: AlertDescription +ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: AlertDescription +ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: AlertDescription +ALERT_DESCRIPTION_USER_CANCELLED: AlertDescription + +# This class is not exposed. It calls itself ssl._ASN1Object. +@type_check_only +class _ASN1ObjectBase(NamedTuple): + nid: int + shortname: str + longname: str + oid: str + +class _ASN1Object(_ASN1ObjectBase): + def __new__(cls, oid: str) -> Self: ... + @classmethod + def fromnid(cls, nid: int) -> Self: ... + @classmethod + def fromname(cls, name: str) -> Self: ... + +class Purpose(_ASN1Object, enum.Enum): + SERVER_AUTH = (129, "serverAuth", "TLS Web Server Authentication", "1.3.6.1.5.5.7.3.2") # pyright: ignore[reportCallIssue] + CLIENT_AUTH = (130, "clientAuth", "TLS Web Client Authentication", "1.3.6.1.5.5.7.3.1") # pyright: ignore[reportCallIssue] + +class SSLSocket(socket.socket): + context: SSLContext + server_side: bool + server_hostname: str | None + session: SSLSession | None + @property + def session_reused(self) -> bool | None: ... + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + def connect(self, addr: socket._Address) -> None: ... + def connect_ex(self, addr: socket._Address) -> int: ... + def recv(self, buflen: int = 1024, flags: int = 0) -> bytes: ... + def recv_into(self, buffer: WriteableBuffer, nbytes: int | None = None, flags: int = 0) -> int: ... + def recvfrom(self, buflen: int = 1024, flags: int = 0) -> tuple[bytes, socket._RetAddress]: ... + def recvfrom_into( + self, buffer: WriteableBuffer, nbytes: int | None = None, flags: int = 0 + ) -> tuple[int, socket._RetAddress]: ... + def send(self, data: ReadableBuffer, flags: int = 0) -> int: ... + def sendall(self, data: ReadableBuffer, flags: int = 0) -> None: ... + @overload + def sendto(self, data: ReadableBuffer, flags_or_addr: socket._Address, addr: None = None) -> int: ... + @overload + def sendto(self, data: ReadableBuffer, flags_or_addr: int, addr: socket._Address) -> int: ... + def shutdown(self, how: int) -> None: ... + def read(self, len: int = 1024, buffer: bytearray | None = None) -> bytes: ... + def write(self, data: ReadableBuffer) -> int: ... + def do_handshake(self, block: bool = False) -> None: ... # block is undocumented + @overload + def getpeercert(self, binary_form: Literal[False] = False) -> _PeerCertRetDictType | None: ... + @overload + def getpeercert(self, binary_form: Literal[True]) -> bytes | None: ... + @overload + def getpeercert(self, binary_form: bool) -> _PeerCertRetType: ... + def cipher(self) -> tuple[str, str, int] | None: ... + def shared_ciphers(self) -> list[tuple[str, str, int]] | None: ... + def compression(self) -> str | None: ... + def get_channel_binding(self, cb_type: str = "tls-unique") -> bytes | None: ... + def selected_alpn_protocol(self) -> str | None: ... + def selected_npn_protocol(self) -> str | None: ... + def accept(self) -> tuple[SSLSocket, socket._RetAddress]: ... + def unwrap(self) -> socket.socket: ... + def version(self) -> str | None: ... + def pending(self) -> int: ... + def verify_client_post_handshake(self) -> None: ... + # These methods always raise `NotImplementedError`: + def recvmsg(self, *args: Never, **kwargs: Never) -> Never: ... # type: ignore[override] + def recvmsg_into(self, *args: Never, **kwargs: Never) -> Never: ... # type: ignore[override] + def sendmsg(self, *args: Never, **kwargs: Never) -> Never: ... # type: ignore[override] + if sys.version_info >= (3, 13): + def get_verified_chain(self) -> list[bytes]: ... + def get_unverified_chain(self) -> list[bytes]: ... + +class TLSVersion(enum.IntEnum): + MINIMUM_SUPPORTED = -2 + MAXIMUM_SUPPORTED = -1 + SSLv3 = 768 + TLSv1 = 769 + TLSv1_1 = 770 + TLSv1_2 = 771 + TLSv1_3 = 772 + +class SSLContext(_SSLContext): + options: Options + verify_flags: VerifyFlags + verify_mode: VerifyMode + @property + def protocol(self) -> _SSLMethod: ... # type: ignore[override] + hostname_checks_common_name: bool + maximum_version: TLSVersion + minimum_version: TLSVersion + # The following two attributes have class-level defaults. + # However, the docs explicitly state that it's OK to override these attributes on instances, + # so making these ClassVars wouldn't be appropriate + sslobject_class: type[SSLObject] + sslsocket_class: type[SSLSocket] + keylog_filename: str + post_handshake_auth: bool + if sys.version_info >= (3, 10): + security_level: int + if sys.version_info >= (3, 10): + # Using the default (None) for the `protocol` parameter is deprecated, + # but there isn't a good way of marking that in the stub unless/until PEP 702 is accepted + def __new__(cls, protocol: int | None = None, *args: Any, **kwargs: Any) -> Self: ... + else: + def __new__(cls, protocol: int = ..., *args: Any, **kwargs: Any) -> Self: ... + + def load_default_certs(self, purpose: Purpose = ...) -> None: ... + def load_verify_locations( + self, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, + ) -> None: ... + @overload + def get_ca_certs(self, binary_form: Literal[False] = False) -> list[_PeerCertRetDictType]: ... + @overload + def get_ca_certs(self, binary_form: Literal[True]) -> list[bytes]: ... + @overload + def get_ca_certs(self, binary_form: bool = False) -> Any: ... + def get_ciphers(self) -> list[_Cipher]: ... + def set_default_verify_paths(self) -> None: ... + def set_ciphers(self, cipherlist: str, /) -> None: ... + def set_alpn_protocols(self, alpn_protocols: Iterable[str]) -> None: ... + def set_npn_protocols(self, npn_protocols: Iterable[str]) -> None: ... + def set_servername_callback(self, server_name_callback: _SrvnmeCbType | None) -> None: ... + def load_dh_params(self, path: str, /) -> None: ... + def set_ecdh_curve(self, name: str, /) -> None: ... + def wrap_socket( + self, + sock: socket.socket, + server_side: bool = False, + do_handshake_on_connect: bool = True, + suppress_ragged_eofs: bool = True, + server_hostname: str | bytes | None = None, + session: SSLSession | None = None, + ) -> SSLSocket: ... + def wrap_bio( + self, + incoming: MemoryBIO, + outgoing: MemoryBIO, + server_side: bool = False, + server_hostname: str | bytes | None = None, + session: SSLSession | None = None, + ) -> SSLObject: ... + +class SSLObject: + context: SSLContext + @property + def server_side(self) -> bool: ... + @property + def server_hostname(self) -> str | None: ... + session: SSLSession | None + @property + def session_reused(self) -> bool: ... + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + def read(self, len: int = 1024, buffer: bytearray | None = None) -> bytes: ... + def write(self, data: ReadableBuffer) -> int: ... + @overload + def getpeercert(self, binary_form: Literal[False] = False) -> _PeerCertRetDictType | None: ... + @overload + def getpeercert(self, binary_form: Literal[True]) -> bytes | None: ... + @overload + def getpeercert(self, binary_form: bool) -> _PeerCertRetType: ... + def selected_alpn_protocol(self) -> str | None: ... + def selected_npn_protocol(self) -> str | None: ... + def cipher(self) -> tuple[str, str, int] | None: ... + def shared_ciphers(self) -> list[tuple[str, str, int]] | None: ... + def compression(self) -> str | None: ... + def pending(self) -> int: ... + def do_handshake(self) -> None: ... + def unwrap(self) -> None: ... + def version(self) -> str | None: ... + def get_channel_binding(self, cb_type: str = "tls-unique") -> bytes | None: ... + def verify_client_post_handshake(self) -> None: ... + if sys.version_info >= (3, 13): + def get_verified_chain(self) -> list[bytes]: ... + def get_unverified_chain(self) -> list[bytes]: ... + +class SSLErrorNumber(enum.IntEnum): + SSL_ERROR_EOF = 8 + SSL_ERROR_INVALID_ERROR_CODE = 10 + SSL_ERROR_SSL = 1 + SSL_ERROR_SYSCALL = 5 + SSL_ERROR_WANT_CONNECT = 7 + SSL_ERROR_WANT_READ = 2 + SSL_ERROR_WANT_WRITE = 3 + SSL_ERROR_WANT_X509_LOOKUP = 4 + SSL_ERROR_ZERO_RETURN = 6 + +SSL_ERROR_EOF: SSLErrorNumber # undocumented +SSL_ERROR_INVALID_ERROR_CODE: SSLErrorNumber # undocumented +SSL_ERROR_SSL: SSLErrorNumber # undocumented +SSL_ERROR_SYSCALL: SSLErrorNumber # undocumented +SSL_ERROR_WANT_CONNECT: SSLErrorNumber # undocumented +SSL_ERROR_WANT_READ: SSLErrorNumber # undocumented +SSL_ERROR_WANT_WRITE: SSLErrorNumber # undocumented +SSL_ERROR_WANT_X509_LOOKUP: SSLErrorNumber # undocumented +SSL_ERROR_ZERO_RETURN: SSLErrorNumber # undocumented + +def get_protocol_name(protocol_code: int) -> str: ... + +if sys.version_info < (3, 9): + AF_INET: int +PEM_FOOTER: str +PEM_HEADER: str +SOCK_STREAM: int +SOL_SOCKET: int +SO_TYPE: int diff --git a/mypy/typeshed/stdlib/stat.pyi b/mypy/typeshed/stdlib/stat.pyi new file mode 100644 index 000000000000..face28ab0cbb --- /dev/null +++ b/mypy/typeshed/stdlib/stat.pyi @@ -0,0 +1,7 @@ +import sys +from _stat import * +from typing import Final + +if sys.version_info >= (3, 13): + # https://github.com/python/cpython/issues/114081#issuecomment-2119017790 + SF_RESTRICTED: Final = 0x00080000 diff --git a/mypy/typeshed/stdlib/statistics.pyi b/mypy/typeshed/stdlib/statistics.pyi new file mode 100644 index 000000000000..c8ecbbceab1a --- /dev/null +++ b/mypy/typeshed/stdlib/statistics.pyi @@ -0,0 +1,161 @@ +import sys +from _typeshed import SupportsRichComparisonT +from collections.abc import Callable, Hashable, Iterable, Sequence +from decimal import Decimal +from fractions import Fraction +from typing import Any, Literal, NamedTuple, SupportsFloat, TypeVar +from typing_extensions import Self, TypeAlias + +__all__ = [ + "StatisticsError", + "fmean", + "geometric_mean", + "mean", + "harmonic_mean", + "pstdev", + "pvariance", + "stdev", + "variance", + "median", + "median_low", + "median_high", + "median_grouped", + "mode", + "multimode", + "NormalDist", + "quantiles", +] + +if sys.version_info >= (3, 10): + __all__ += ["covariance", "correlation", "linear_regression"] +if sys.version_info >= (3, 13): + __all__ += ["kde", "kde_random"] + +# Most functions in this module accept homogeneous collections of one of these types +_Number: TypeAlias = float | Decimal | Fraction +_NumberT = TypeVar("_NumberT", float, Decimal, Fraction) + +# Used in mode, multimode +_HashableT = TypeVar("_HashableT", bound=Hashable) + +class StatisticsError(ValueError): ... + +if sys.version_info >= (3, 11): + def fmean(data: Iterable[SupportsFloat], weights: Iterable[SupportsFloat] | None = None) -> float: ... + +else: + def fmean(data: Iterable[SupportsFloat]) -> float: ... + +def geometric_mean(data: Iterable[SupportsFloat]) -> float: ... +def mean(data: Iterable[_NumberT]) -> _NumberT: ... + +if sys.version_info >= (3, 10): + def harmonic_mean(data: Iterable[_NumberT], weights: Iterable[_Number] | None = None) -> _NumberT: ... + +else: + def harmonic_mean(data: Iterable[_NumberT]) -> _NumberT: ... + +def median(data: Iterable[_NumberT]) -> _NumberT: ... +def median_low(data: Iterable[SupportsRichComparisonT]) -> SupportsRichComparisonT: ... +def median_high(data: Iterable[SupportsRichComparisonT]) -> SupportsRichComparisonT: ... + +if sys.version_info >= (3, 11): + def median_grouped(data: Iterable[SupportsFloat], interval: SupportsFloat = 1.0) -> float: ... + +else: + def median_grouped(data: Iterable[_NumberT], interval: _NumberT | float = 1) -> _NumberT | float: ... + +def mode(data: Iterable[_HashableT]) -> _HashableT: ... +def multimode(data: Iterable[_HashableT]) -> list[_HashableT]: ... +def pstdev(data: Iterable[_NumberT], mu: _NumberT | None = None) -> _NumberT: ... +def pvariance(data: Iterable[_NumberT], mu: _NumberT | None = None) -> _NumberT: ... +def quantiles( + data: Iterable[_NumberT], *, n: int = 4, method: Literal["inclusive", "exclusive"] = "exclusive" +) -> list[_NumberT]: ... +def stdev(data: Iterable[_NumberT], xbar: _NumberT | None = None) -> _NumberT: ... +def variance(data: Iterable[_NumberT], xbar: _NumberT | None = None) -> _NumberT: ... + +class NormalDist: + def __init__(self, mu: float = 0.0, sigma: float = 1.0) -> None: ... + @property + def mean(self) -> float: ... + @property + def median(self) -> float: ... + @property + def mode(self) -> float: ... + @property + def stdev(self) -> float: ... + @property + def variance(self) -> float: ... + @classmethod + def from_samples(cls, data: Iterable[SupportsFloat]) -> Self: ... + def samples(self, n: int, *, seed: Any | None = None) -> list[float]: ... + def pdf(self, x: float) -> float: ... + def cdf(self, x: float) -> float: ... + def inv_cdf(self, p: float) -> float: ... + def overlap(self, other: NormalDist) -> float: ... + def quantiles(self, n: int = 4) -> list[float]: ... + if sys.version_info >= (3, 9): + def zscore(self, x: float) -> float: ... + + def __eq__(self, x2: object) -> bool: ... + def __add__(self, x2: float | NormalDist) -> NormalDist: ... + def __sub__(self, x2: float | NormalDist) -> NormalDist: ... + def __mul__(self, x2: float) -> NormalDist: ... + def __truediv__(self, x2: float) -> NormalDist: ... + def __pos__(self) -> NormalDist: ... + def __neg__(self) -> NormalDist: ... + __radd__ = __add__ + def __rsub__(self, x2: float | NormalDist) -> NormalDist: ... + __rmul__ = __mul__ + def __hash__(self) -> int: ... + +if sys.version_info >= (3, 12): + def correlation( + x: Sequence[_Number], y: Sequence[_Number], /, *, method: Literal["linear", "ranked"] = "linear" + ) -> float: ... + +elif sys.version_info >= (3, 10): + def correlation(x: Sequence[_Number], y: Sequence[_Number], /) -> float: ... + +if sys.version_info >= (3, 10): + def covariance(x: Sequence[_Number], y: Sequence[_Number], /) -> float: ... + + class LinearRegression(NamedTuple): + slope: float + intercept: float + +if sys.version_info >= (3, 11): + def linear_regression( + regressor: Sequence[_Number], dependent_variable: Sequence[_Number], /, *, proportional: bool = False + ) -> LinearRegression: ... + +elif sys.version_info >= (3, 10): + def linear_regression(regressor: Sequence[_Number], dependent_variable: Sequence[_Number], /) -> LinearRegression: ... + +if sys.version_info >= (3, 13): + _Kernel: TypeAlias = Literal[ + "normal", + "gauss", + "logistic", + "sigmoid", + "rectangular", + "uniform", + "triangular", + "parabolic", + "epanechnikov", + "quartic", + "biweight", + "triweight", + "cosine", + ] + def kde( + data: Sequence[float], h: float, kernel: _Kernel = "normal", *, cumulative: bool = False + ) -> Callable[[float], float]: ... + def kde_random( + data: Sequence[float], + h: float, + kernel: _Kernel = "normal", + *, + seed: int | float | str | bytes | bytearray | None = None, # noqa: Y041 + ) -> Callable[[], float]: ... diff --git a/mypy/typeshed/stdlib/string.pyi b/mypy/typeshed/stdlib/string.pyi new file mode 100644 index 000000000000..35a76e9c8628 --- /dev/null +++ b/mypy/typeshed/stdlib/string.pyi @@ -0,0 +1,83 @@ +import sys +from _typeshed import StrOrLiteralStr +from collections.abc import Iterable, Mapping, Sequence +from re import Pattern, RegexFlag +from typing import Any, ClassVar, overload +from typing_extensions import LiteralString, TypeAlias + +__all__ = [ + "ascii_letters", + "ascii_lowercase", + "ascii_uppercase", + "capwords", + "digits", + "hexdigits", + "octdigits", + "printable", + "punctuation", + "whitespace", + "Formatter", + "Template", +] + +ascii_letters: LiteralString +ascii_lowercase: LiteralString +ascii_uppercase: LiteralString +digits: LiteralString +hexdigits: LiteralString +octdigits: LiteralString +punctuation: LiteralString +printable: LiteralString +whitespace: LiteralString + +def capwords(s: StrOrLiteralStr, sep: StrOrLiteralStr | None = None) -> StrOrLiteralStr: ... + +if sys.version_info >= (3, 9): + _TemplateMetaclass: TypeAlias = type +else: + class _TemplateMetaclass(type): + pattern: ClassVar[str] + def __init__(cls, name: str, bases: tuple[type, ...], dct: dict[str, Any]) -> None: ... + +class Template(metaclass=_TemplateMetaclass): + template: str + delimiter: ClassVar[str] + idpattern: ClassVar[str] + braceidpattern: ClassVar[str | None] + flags: ClassVar[RegexFlag] + pattern: ClassVar[Pattern[str]] + def __init__(self, template: str) -> None: ... + def substitute(self, mapping: Mapping[str, object] = {}, /, **kwds: object) -> str: ... + def safe_substitute(self, mapping: Mapping[str, object] = {}, /, **kwds: object) -> str: ... + if sys.version_info >= (3, 11): + def get_identifiers(self) -> list[str]: ... + def is_valid(self) -> bool: ... + +class Formatter: + @overload + def format(self, format_string: LiteralString, /, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... + @overload + def format(self, format_string: str, /, *args: Any, **kwargs: Any) -> str: ... + @overload + def vformat( + self, format_string: LiteralString, args: Sequence[LiteralString], kwargs: Mapping[LiteralString, LiteralString] + ) -> LiteralString: ... + @overload + def vformat(self, format_string: str, args: Sequence[Any], kwargs: Mapping[str, Any]) -> str: ... + def _vformat( # undocumented + self, + format_string: str, + args: Sequence[Any], + kwargs: Mapping[str, Any], + used_args: set[int | str], + recursion_depth: int, + auto_arg_index: int = 0, + ) -> tuple[str, int]: ... + def parse( + self, format_string: StrOrLiteralStr + ) -> Iterable[tuple[StrOrLiteralStr, StrOrLiteralStr | None, StrOrLiteralStr | None, StrOrLiteralStr | None]]: ... + def get_field(self, field_name: str, args: Sequence[Any], kwargs: Mapping[str, Any]) -> Any: ... + def get_value(self, key: int | str, args: Sequence[Any], kwargs: Mapping[str, Any]) -> Any: ... + def check_unused_args(self, used_args: set[int | str], args: Sequence[Any], kwargs: Mapping[str, Any]) -> None: ... + def format_field(self, value: Any, format_spec: str) -> Any: ... + def convert_field(self, value: Any, conversion: str | None) -> Any: ... diff --git a/mypy/typeshed/stdlib/stringprep.pyi b/mypy/typeshed/stdlib/stringprep.pyi new file mode 100644 index 000000000000..fc28c027ca9b --- /dev/null +++ b/mypy/typeshed/stdlib/stringprep.pyi @@ -0,0 +1,27 @@ +b1_set: set[int] +b3_exceptions: dict[int, str] +c22_specials: set[int] +c6_set: set[int] +c7_set: set[int] +c8_set: set[int] +c9_set: set[int] + +def in_table_a1(code: str) -> bool: ... +def in_table_b1(code: str) -> bool: ... +def map_table_b3(code: str) -> str: ... +def map_table_b2(a: str) -> str: ... +def in_table_c11(code: str) -> bool: ... +def in_table_c12(code: str) -> bool: ... +def in_table_c11_c12(code: str) -> bool: ... +def in_table_c21(code: str) -> bool: ... +def in_table_c22(code: str) -> bool: ... +def in_table_c21_c22(code: str) -> bool: ... +def in_table_c3(code: str) -> bool: ... +def in_table_c4(code: str) -> bool: ... +def in_table_c5(code: str) -> bool: ... +def in_table_c6(code: str) -> bool: ... +def in_table_c7(code: str) -> bool: ... +def in_table_c8(code: str) -> bool: ... +def in_table_c9(code: str) -> bool: ... +def in_table_d1(code: str) -> bool: ... +def in_table_d2(code: str) -> bool: ... diff --git a/mypy/typeshed/stdlib/struct.pyi b/mypy/typeshed/stdlib/struct.pyi new file mode 100644 index 000000000000..2c26908746ec --- /dev/null +++ b/mypy/typeshed/stdlib/struct.pyi @@ -0,0 +1,5 @@ +from _struct import * + +__all__ = ["calcsize", "pack", "pack_into", "unpack", "unpack_from", "iter_unpack", "Struct", "error"] + +class error(Exception): ... diff --git a/mypy/typeshed/stdlib/subprocess.pyi b/mypy/typeshed/stdlib/subprocess.pyi new file mode 100644 index 000000000000..fef35b56945a --- /dev/null +++ b/mypy/typeshed/stdlib/subprocess.pyi @@ -0,0 +1,2628 @@ +import sys +from _typeshed import MaybeNone, ReadableBuffer, StrOrBytesPath +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence +from types import TracebackType +from typing import IO, Any, AnyStr, Final, Generic, Literal, TypeVar, overload +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "Popen", + "PIPE", + "STDOUT", + "call", + "check_call", + "getstatusoutput", + "getoutput", + "check_output", + "run", + "CalledProcessError", + "DEVNULL", + "SubprocessError", + "TimeoutExpired", + "CompletedProcess", +] + +if sys.platform == "win32": + __all__ += [ + "CREATE_NEW_CONSOLE", + "CREATE_NEW_PROCESS_GROUP", + "STARTF_USESHOWWINDOW", + "STARTF_USESTDHANDLES", + "STARTUPINFO", + "STD_ERROR_HANDLE", + "STD_INPUT_HANDLE", + "STD_OUTPUT_HANDLE", + "SW_HIDE", + "ABOVE_NORMAL_PRIORITY_CLASS", + "BELOW_NORMAL_PRIORITY_CLASS", + "CREATE_BREAKAWAY_FROM_JOB", + "CREATE_DEFAULT_ERROR_MODE", + "CREATE_NO_WINDOW", + "DETACHED_PROCESS", + "HIGH_PRIORITY_CLASS", + "IDLE_PRIORITY_CLASS", + "NORMAL_PRIORITY_CLASS", + "REALTIME_PRIORITY_CLASS", + ] + +# We prefer to annotate inputs to methods (eg subprocess.check_call) with these +# union types. +# For outputs we use laborious literal based overloads to try to determine +# which specific return types to use, and prefer to fall back to Any when +# this does not work, so the caller does not have to use an assertion to confirm +# which type. +# +# For example: +# +# try: +# x = subprocess.check_output(["ls", "-l"]) +# reveal_type(x) # bytes, based on the overloads +# except TimeoutError as e: +# reveal_type(e.cmd) # Any, but morally is _CMD +_FILE: TypeAlias = None | int | IO[Any] +_InputString: TypeAlias = ReadableBuffer | str +_CMD: TypeAlias = StrOrBytesPath | Sequence[StrOrBytesPath] +if sys.platform == "win32": + _ENV: TypeAlias = Mapping[str, str] +else: + _ENV: TypeAlias = Mapping[bytes, StrOrBytesPath] | Mapping[str, StrOrBytesPath] + +_T = TypeVar("_T") + +# These two are private but documented +if sys.version_info >= (3, 11): + _USE_VFORK: Final[bool] +_USE_POSIX_SPAWN: Final[bool] + +class CompletedProcess(Generic[_T]): + # morally: _CMD + args: Any + returncode: int + # These can both be None, but requiring checks for None would be tedious + # and writing all the overloads would be horrific. + stdout: _T + stderr: _T + def __init__(self, args: _CMD, returncode: int, stdout: _T | None = None, stderr: _T | None = None) -> None: ... + def check_returncode(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +if sys.version_info >= (3, 11): + # 3.11 adds "process_group" argument + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: Literal[True], + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the *real* keyword only args start + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: None = None, + errors: None = None, + input: ReadableBuffer | None = None, + text: Literal[False] | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> CompletedProcess[bytes]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: _InputString | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> CompletedProcess[Any]: ... + +elif sys.version_info >= (3, 10): + # 3.10 adds "pipesize" argument + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: Literal[True], + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the *real* keyword only args start + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: None = None, + errors: None = None, + input: ReadableBuffer | None = None, + text: Literal[False] | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> CompletedProcess[bytes]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: _InputString | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> CompletedProcess[Any]: ... + +elif sys.version_info >= (3, 9): + # 3.9 adds arguments "user", "group", "extra_groups" and "umask" + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: Literal[True], + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the *real* keyword only args start + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: None = None, + errors: None = None, + input: ReadableBuffer | None = None, + text: Literal[False] | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> CompletedProcess[bytes]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: _InputString | None = None, + text: bool | None = None, + timeout: float | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> CompletedProcess[Any]: ... + +else: + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: Literal[True], + timeout: float | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the *real* keyword only args start + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: str | None = None, + text: bool | None = None, + timeout: float | None = None, + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: None = None, + errors: None = None, + input: ReadableBuffer | None = None, + text: Literal[False] | None = None, + timeout: float | None = None, + ) -> CompletedProcess[bytes]: ... + @overload + def run( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + errors: str | None = None, + input: _InputString | None = None, + text: bool | None = None, + timeout: float | None = None, + ) -> CompletedProcess[Any]: ... + +# Same args as Popen.__init__ +if sys.version_info >= (3, 11): + # 3.11 adds "process_group" argument + def call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + encoding: str | None = None, + timeout: float | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> int: ... + +elif sys.version_info >= (3, 10): + # 3.10 adds "pipesize" argument + def call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + encoding: str | None = None, + timeout: float | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> int: ... + +elif sys.version_info >= (3, 9): + # 3.9 adds arguments "user", "group", "extra_groups" and "umask" + def call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + encoding: str | None = None, + timeout: float | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> int: ... + +else: + def call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + encoding: str | None = None, + timeout: float | None = None, + text: bool | None = None, + ) -> int: ... + +# Same args as Popen.__init__ +if sys.version_info >= (3, 11): + # 3.11 adds "process_group" argument + def check_call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + timeout: float | None = ..., + *, + encoding: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> int: ... + +elif sys.version_info >= (3, 10): + # 3.10 adds "pipesize" argument + def check_call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + timeout: float | None = ..., + *, + encoding: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> int: ... + +elif sys.version_info >= (3, 9): + # 3.9 adds arguments "user", "group", "extra_groups" and "umask" + def check_call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + timeout: float | None = ..., + *, + encoding: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> int: ... + +else: + def check_call( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stdout: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + timeout: float | None = ..., + *, + encoding: str | None = None, + text: bool | None = None, + ) -> int: ... + +if sys.version_info >= (3, 11): + # 3.11 adds "process_group" argument + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: Literal[True], + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the real keyword only ones start + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> bytes: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> Any: ... # morally: -> str | bytes + +elif sys.version_info >= (3, 10): + # 3.10 adds "pipesize" argument + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: Literal[True], + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the real keyword only ones start + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> bytes: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> Any: ... # morally: -> str | bytes + +elif sys.version_info >= (3, 9): + # 3.9 adds arguments "user", "group", "extra_groups" and "umask" + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: Literal[True], + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the real keyword only ones start + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> bytes: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> Any: ... # morally: -> str | bytes + +else: + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: Literal[True], + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str, + errors: str | None = None, + text: bool | None = None, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str, + text: bool | None = None, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + # where the real keyword only ones start + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: None = None, + errors: None = None, + text: Literal[False] | None = None, + ) -> bytes: ... + @overload + def check_output( + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE = None, + stderr: _FILE = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = ..., + *, + timeout: float | None = None, + input: _InputString | None = ..., + encoding: str | None = None, + errors: str | None = None, + text: bool | None = None, + ) -> Any: ... # morally: -> str | bytes + +PIPE: Final[int] +STDOUT: Final[int] +DEVNULL: Final[int] + +class SubprocessError(Exception): ... + +class TimeoutExpired(SubprocessError): + def __init__( + self, cmd: _CMD, timeout: float, output: str | bytes | None = None, stderr: str | bytes | None = None + ) -> None: ... + # morally: _CMD + cmd: Any + timeout: float + # morally: str | bytes | None + output: Any + stdout: bytes | None + stderr: bytes | None + +class CalledProcessError(SubprocessError): + returncode: int + # morally: _CMD + cmd: Any + # morally: str | bytes | None + output: Any + + # morally: str | bytes | None + stdout: Any + stderr: Any + def __init__( + self, returncode: int, cmd: _CMD, output: str | bytes | None = None, stderr: str | bytes | None = None + ) -> None: ... + +class Popen(Generic[AnyStr]): + args: _CMD + stdin: IO[AnyStr] | None + stdout: IO[AnyStr] | None + stderr: IO[AnyStr] | None + pid: int + returncode: int | MaybeNone + universal_newlines: bool + + if sys.version_info >= (3, 11): + # process_group is added in 3.11 + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + # where the *real* keyword only args start + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[True], + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[bytes], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[False] | None = None, + encoding: None = None, + errors: None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[Any], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + process_group: int | None = None, + ) -> None: ... + elif sys.version_info >= (3, 10): + # pipesize is added in 3.10 + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + # where the *real* keyword only args start + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[True], + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[bytes], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[False] | None = None, + encoding: None = None, + errors: None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[Any], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + pipesize: int = -1, + ) -> None: ... + elif sys.version_info >= (3, 9): + # user, group, extra_groups, umask were added in 3.9 + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + # where the *real* keyword only args start + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[True], + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[bytes], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[False] | None = None, + encoding: None = None, + errors: None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> None: ... + @overload + def __init__( + self: Popen[Any], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + ) -> None: ... + else: + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + *, + universal_newlines: Literal[True], + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + # where the *real* keyword only args start + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[str], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[True], + encoding: str | None = None, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[bytes], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: Literal[False] | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: Literal[False] | None = None, + encoding: None = None, + errors: None = None, + ) -> None: ... + @overload + def __init__( + self: Popen[Any], + args: _CMD, + bufsize: int = -1, + executable: StrOrBytesPath | None = None, + stdin: _FILE | None = None, + stdout: _FILE | None = None, + stderr: _FILE | None = None, + preexec_fn: Callable[[], Any] | None = None, + close_fds: bool = True, + shell: bool = False, + cwd: StrOrBytesPath | None = None, + env: _ENV | None = None, + universal_newlines: bool | None = None, + startupinfo: Any | None = None, + creationflags: int = 0, + restore_signals: bool = True, + start_new_session: bool = False, + pass_fds: Collection[int] = (), + *, + text: bool | None = None, + encoding: str | None = None, + errors: str | None = None, + ) -> None: ... + + def poll(self) -> int | None: ... + def wait(self, timeout: float | None = None) -> int: ... + # morally the members of the returned tuple should be optional + # TODO this should allow ReadableBuffer for Popen[bytes], but adding + # overloads for that runs into a mypy bug (python/mypy#14070). + def communicate(self, input: AnyStr | None = None, timeout: float | None = None) -> tuple[AnyStr, AnyStr]: ... + def send_signal(self, sig: int) -> None: ... + def terminate(self) -> None: ... + def kill(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + def __del__(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +# The result really is always a str. +if sys.version_info >= (3, 11): + def getstatusoutput(cmd: _CMD, *, encoding: str | None = None, errors: str | None = None) -> tuple[int, str]: ... + def getoutput(cmd: _CMD, *, encoding: str | None = None, errors: str | None = None) -> str: ... + +else: + def getstatusoutput(cmd: _CMD) -> tuple[int, str]: ... + def getoutput(cmd: _CMD) -> str: ... + +def list2cmdline(seq: Iterable[StrOrBytesPath]) -> str: ... # undocumented + +if sys.platform == "win32": + if sys.version_info >= (3, 13): + from _winapi import STARTF_FORCEOFFFEEDBACK, STARTF_FORCEONFEEDBACK + + __all__ += ["STARTF_FORCEOFFFEEDBACK", "STARTF_FORCEONFEEDBACK"] + + class STARTUPINFO: + def __init__( + self, + *, + dwFlags: int = 0, + hStdInput: Any | None = None, + hStdOutput: Any | None = None, + hStdError: Any | None = None, + wShowWindow: int = 0, + lpAttributeList: Mapping[str, Any] | None = None, + ) -> None: ... + dwFlags: int + hStdInput: Any | None + hStdOutput: Any | None + hStdError: Any | None + wShowWindow: int + lpAttributeList: Mapping[str, Any] + def copy(self) -> STARTUPINFO: ... + + from _winapi import ( + ABOVE_NORMAL_PRIORITY_CLASS as ABOVE_NORMAL_PRIORITY_CLASS, + BELOW_NORMAL_PRIORITY_CLASS as BELOW_NORMAL_PRIORITY_CLASS, + CREATE_BREAKAWAY_FROM_JOB as CREATE_BREAKAWAY_FROM_JOB, + CREATE_DEFAULT_ERROR_MODE as CREATE_DEFAULT_ERROR_MODE, + CREATE_NEW_CONSOLE as CREATE_NEW_CONSOLE, + CREATE_NEW_PROCESS_GROUP as CREATE_NEW_PROCESS_GROUP, + CREATE_NO_WINDOW as CREATE_NO_WINDOW, + DETACHED_PROCESS as DETACHED_PROCESS, + HIGH_PRIORITY_CLASS as HIGH_PRIORITY_CLASS, + IDLE_PRIORITY_CLASS as IDLE_PRIORITY_CLASS, + NORMAL_PRIORITY_CLASS as NORMAL_PRIORITY_CLASS, + REALTIME_PRIORITY_CLASS as REALTIME_PRIORITY_CLASS, + STARTF_USESHOWWINDOW as STARTF_USESHOWWINDOW, + STARTF_USESTDHANDLES as STARTF_USESTDHANDLES, + STD_ERROR_HANDLE as STD_ERROR_HANDLE, + STD_INPUT_HANDLE as STD_INPUT_HANDLE, + STD_OUTPUT_HANDLE as STD_OUTPUT_HANDLE, + SW_HIDE as SW_HIDE, + ) diff --git a/mypy/typeshed/stdlib/sunau.pyi b/mypy/typeshed/stdlib/sunau.pyi new file mode 100644 index 000000000000..9b051e82b64b --- /dev/null +++ b/mypy/typeshed/stdlib/sunau.pyi @@ -0,0 +1,86 @@ +import sys +from _typeshed import Unused +from typing import IO, Any, Literal, NamedTuple, NoReturn, overload +from typing_extensions import Self, TypeAlias + +_File: TypeAlias = str | IO[bytes] + +class Error(Exception): ... + +AUDIO_FILE_MAGIC: int +AUDIO_FILE_ENCODING_MULAW_8: int +AUDIO_FILE_ENCODING_LINEAR_8: int +AUDIO_FILE_ENCODING_LINEAR_16: int +AUDIO_FILE_ENCODING_LINEAR_24: int +AUDIO_FILE_ENCODING_LINEAR_32: int +AUDIO_FILE_ENCODING_FLOAT: int +AUDIO_FILE_ENCODING_DOUBLE: int +AUDIO_FILE_ENCODING_ADPCM_G721: int +AUDIO_FILE_ENCODING_ADPCM_G722: int +AUDIO_FILE_ENCODING_ADPCM_G723_3: int +AUDIO_FILE_ENCODING_ADPCM_G723_5: int +AUDIO_FILE_ENCODING_ALAW_8: int +AUDIO_UNKNOWN_SIZE: int + +class _sunau_params(NamedTuple): + nchannels: int + sampwidth: int + framerate: int + nframes: int + comptype: str + compname: str + +class Au_read: + def __init__(self, f: _File) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def __del__(self) -> None: ... + def getfp(self) -> IO[bytes] | None: ... + def rewind(self) -> None: ... + def close(self) -> None: ... + def tell(self) -> int: ... + def getnchannels(self) -> int: ... + def getnframes(self) -> int: ... + def getsampwidth(self) -> int: ... + def getframerate(self) -> int: ... + def getcomptype(self) -> str: ... + def getcompname(self) -> str: ... + def getparams(self) -> _sunau_params: ... + def getmarkers(self) -> None: ... + def getmark(self, id: Any) -> NoReturn: ... + def setpos(self, pos: int) -> None: ... + def readframes(self, nframes: int) -> bytes | None: ... + +class Au_write: + def __init__(self, f: _File) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def __del__(self) -> None: ... + def setnchannels(self, nchannels: int) -> None: ... + def getnchannels(self) -> int: ... + def setsampwidth(self, sampwidth: int) -> None: ... + def getsampwidth(self) -> int: ... + def setframerate(self, framerate: float) -> None: ... + def getframerate(self) -> int: ... + def setnframes(self, nframes: int) -> None: ... + def getnframes(self) -> int: ... + def setcomptype(self, type: str, name: str) -> None: ... + def getcomptype(self) -> str: ... + def getcompname(self) -> str: ... + def setparams(self, params: _sunau_params) -> None: ... + def getparams(self) -> _sunau_params: ... + def tell(self) -> int: ... + # should be any bytes-like object after 3.4, but we don't have a type for that + def writeframesraw(self, data: bytes) -> None: ... + def writeframes(self, data: bytes) -> None: ... + def close(self) -> None: ... + +@overload +def open(f: _File, mode: Literal["r", "rb"]) -> Au_read: ... +@overload +def open(f: _File, mode: Literal["w", "wb"]) -> Au_write: ... +@overload +def open(f: _File, mode: str | None = None) -> Any: ... + +if sys.version_info < (3, 9): + openfp = open diff --git a/mypy/typeshed/stdlib/symbol.pyi b/mypy/typeshed/stdlib/symbol.pyi new file mode 100644 index 000000000000..48ae3567a1a5 --- /dev/null +++ b/mypy/typeshed/stdlib/symbol.pyi @@ -0,0 +1,93 @@ +single_input: int +file_input: int +eval_input: int +decorator: int +decorators: int +decorated: int +async_funcdef: int +funcdef: int +parameters: int +typedargslist: int +tfpdef: int +varargslist: int +vfpdef: int +stmt: int +simple_stmt: int +small_stmt: int +expr_stmt: int +annassign: int +testlist_star_expr: int +augassign: int +del_stmt: int +pass_stmt: int +flow_stmt: int +break_stmt: int +continue_stmt: int +return_stmt: int +yield_stmt: int +raise_stmt: int +import_stmt: int +import_name: int +import_from: int +import_as_name: int +dotted_as_name: int +import_as_names: int +dotted_as_names: int +dotted_name: int +global_stmt: int +nonlocal_stmt: int +assert_stmt: int +compound_stmt: int +async_stmt: int +if_stmt: int +while_stmt: int +for_stmt: int +try_stmt: int +with_stmt: int +with_item: int +except_clause: int +suite: int +test: int +test_nocond: int +lambdef: int +lambdef_nocond: int +or_test: int +and_test: int +not_test: int +comparison: int +comp_op: int +star_expr: int +expr: int +xor_expr: int +and_expr: int +shift_expr: int +arith_expr: int +term: int +factor: int +power: int +atom_expr: int +atom: int +testlist_comp: int +trailer: int +subscriptlist: int +subscript: int +sliceop: int +exprlist: int +testlist: int +dictorsetmaker: int +classdef: int +arglist: int +argument: int +comp_iter: int +comp_for: int +comp_if: int +encoding_decl: int +yield_expr: int +yield_arg: int +sync_comp_for: int +func_body_suite: int +func_type: int +func_type_input: int +namedexpr_test: int +typelist: int +sym_name: dict[int, str] diff --git a/mypy/typeshed/stdlib/symtable.pyi b/mypy/typeshed/stdlib/symtable.pyi new file mode 100644 index 000000000000..ee0a1eb2f1cb --- /dev/null +++ b/mypy/typeshed/stdlib/symtable.pyi @@ -0,0 +1,90 @@ +import sys +from _collections_abc import dict_keys +from collections.abc import Sequence +from typing import Any +from typing_extensions import deprecated + +__all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] + +if sys.version_info >= (3, 13): + __all__ += ["SymbolTableType"] + +def symtable(code: str, filename: str, compile_type: str) -> SymbolTable: ... + +if sys.version_info >= (3, 13): + from enum import StrEnum + + class SymbolTableType(StrEnum): + MODULE = "module" + FUNCTION = "function" + CLASS = "class" + ANNOTATION = "annotation" + TYPE_ALIAS = "type alias" + TYPE_PARAMETERS = "type parameters" + TYPE_VARIABLE = "type variable" + +class SymbolTable: + def __init__(self, raw_table: Any, filename: str) -> None: ... + if sys.version_info >= (3, 13): + def get_type(self) -> SymbolTableType: ... + else: + def get_type(self) -> str: ... + + def get_id(self) -> int: ... + def get_name(self) -> str: ... + def get_lineno(self) -> int: ... + def is_optimized(self) -> bool: ... + def is_nested(self) -> bool: ... + def has_children(self) -> bool: ... + if sys.version_info < (3, 9): + def has_exec(self) -> bool: ... + + def get_identifiers(self) -> dict_keys[str, int]: ... + def lookup(self, name: str) -> Symbol: ... + def get_symbols(self) -> list[Symbol]: ... + def get_children(self) -> list[SymbolTable]: ... + +class Function(SymbolTable): + def get_parameters(self) -> tuple[str, ...]: ... + def get_locals(self) -> tuple[str, ...]: ... + def get_globals(self) -> tuple[str, ...]: ... + def get_frees(self) -> tuple[str, ...]: ... + def get_nonlocals(self) -> tuple[str, ...]: ... + +class Class(SymbolTable): + if sys.version_info < (3, 16): + @deprecated("deprecated in Python 3.14, will be removed in Python 3.16") + def get_methods(self) -> tuple[str, ...]: ... + +class Symbol: + def __init__( + self, name: str, flags: int, namespaces: Sequence[SymbolTable] | None = None, *, module_scope: bool = False + ) -> None: ... + def is_nonlocal(self) -> bool: ... + def get_name(self) -> str: ... + def is_referenced(self) -> bool: ... + def is_parameter(self) -> bool: ... + if sys.version_info >= (3, 14): + def is_type_parameter(self) -> bool: ... + + def is_global(self) -> bool: ... + def is_declared_global(self) -> bool: ... + def is_local(self) -> bool: ... + def is_annotated(self) -> bool: ... + def is_free(self) -> bool: ... + if sys.version_info >= (3, 14): + def is_free_class(self) -> bool: ... + + def is_imported(self) -> bool: ... + def is_assigned(self) -> bool: ... + if sys.version_info >= (3, 14): + def is_comp_iter(self) -> bool: ... + def is_comp_cell(self) -> bool: ... + + def is_namespace(self) -> bool: ... + def get_namespaces(self) -> Sequence[SymbolTable]: ... + def get_namespace(self) -> SymbolTable: ... + +class SymbolTableFactory: + def new(self, table: Any, filename: str) -> SymbolTable: ... + def __call__(self, table: Any, filename: str) -> SymbolTable: ... diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi new file mode 100644 index 000000000000..c4b1adca9bc6 --- /dev/null +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -0,0 +1,475 @@ +import sys +from _typeshed import MaybeNone, OptExcInfo, ProfileFunction, TraceFunction, structseq +from _typeshed.importlib import MetaPathFinderProtocol, PathEntryFinderProtocol +from builtins import object as _object +from collections.abc import AsyncGenerator, Callable, Sequence +from io import TextIOWrapper +from types import FrameType, ModuleType, TracebackType +from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, type_check_only +from typing_extensions import TypeAlias + +_T = TypeVar("_T") + +# see https://github.com/python/typeshed/issues/8513#issue-1333671093 for the rationale behind this alias +_ExitCode: TypeAlias = str | int | None + +# ----- sys variables ----- +if sys.platform != "win32": + abiflags: str +argv: list[str] +base_exec_prefix: str +base_prefix: str +byteorder: Literal["little", "big"] +builtin_module_names: Sequence[str] # actually a tuple of strings +copyright: str +if sys.platform == "win32": + dllhandle: int +dont_write_bytecode: bool +displayhook: Callable[[object], Any] +excepthook: Callable[[type[BaseException], BaseException, TracebackType | None], Any] +exec_prefix: str +executable: str +float_repr_style: Literal["short", "legacy"] +hexversion: int +last_type: type[BaseException] | None +last_value: BaseException | None +last_traceback: TracebackType | None +if sys.version_info >= (3, 12): + last_exc: BaseException # or undefined. +maxsize: int +maxunicode: int +meta_path: list[MetaPathFinderProtocol] +modules: dict[str, ModuleType] +if sys.version_info >= (3, 10): + orig_argv: list[str] +path: list[str] +path_hooks: list[Callable[[str], PathEntryFinderProtocol]] +path_importer_cache: dict[str, PathEntryFinderProtocol | None] +platform: str +if sys.version_info >= (3, 9): + platlibdir: str +prefix: str +pycache_prefix: str | None +ps1: object +ps2: object + +# TextIO is used instead of more specific types for the standard streams, +# since they are often monkeypatched at runtime. At startup, the objects +# are initialized to instances of TextIOWrapper, but can also be None under +# some circumstances. +# +# To use methods from TextIOWrapper, use an isinstance check to ensure that +# the streams have not been overridden: +# +# if isinstance(sys.stdout, io.TextIOWrapper): +# sys.stdout.reconfigure(...) +stdin: TextIO | MaybeNone +stdout: TextIO | MaybeNone +stderr: TextIO | MaybeNone + +if sys.version_info >= (3, 10): + stdlib_module_names: frozenset[str] + +__stdin__: Final[TextIOWrapper | None] # Contains the original value of stdin +__stdout__: Final[TextIOWrapper | None] # Contains the original value of stdout +__stderr__: Final[TextIOWrapper | None] # Contains the original value of stderr +tracebacklimit: int +version: str +api_version: int +warnoptions: Any +# Each entry is a tuple of the form (action, message, category, module, +# lineno) +if sys.platform == "win32": + winver: str +_xoptions: dict[Any, Any] + +# Type alias used as a mixin for structseq classes that cannot be instantiated at runtime +# This can't be represented in the type system, so we just use `structseq[Any]` +_UninstantiableStructseq: TypeAlias = structseq[Any] + +flags: _flags + +# This class is not exposed at runtime. It calls itself sys.flags. +# As a tuple, it can have a length between 15 and 18. We don't model +# the exact length here because that varies by patch version due to +# the backported security fix int_max_str_digits. The exact length shouldn't +# be relied upon. See #13031 +# This can be re-visited when typeshed drops support for 3.10, +# at which point all supported versions will include int_max_str_digits +# in all patch versions. +# 3.8 and 3.9 are 15 or 16-tuple +# 3.10 is 16 or 17-tuple +# 3.11+ is an 18-tuple. +@final +@type_check_only +class _flags(_UninstantiableStructseq, tuple[int, ...]): + # `safe_path` was added in py311 + if sys.version_info >= (3, 11): + __match_args__: Final = ( + "debug", + "inspect", + "interactive", + "optimize", + "dont_write_bytecode", + "no_user_site", + "no_site", + "ignore_environment", + "verbose", + "bytes_warning", + "quiet", + "hash_randomization", + "isolated", + "dev_mode", + "utf8_mode", + "warn_default_encoding", + "safe_path", + "int_max_str_digits", + ) + elif sys.version_info >= (3, 10): + __match_args__: Final = ( + "debug", + "inspect", + "interactive", + "optimize", + "dont_write_bytecode", + "no_user_site", + "no_site", + "ignore_environment", + "verbose", + "bytes_warning", + "quiet", + "hash_randomization", + "isolated", + "dev_mode", + "utf8_mode", + "warn_default_encoding", + "int_max_str_digits", + ) + + @property + def debug(self) -> int: ... + @property + def inspect(self) -> int: ... + @property + def interactive(self) -> int: ... + @property + def optimize(self) -> int: ... + @property + def dont_write_bytecode(self) -> int: ... + @property + def no_user_site(self) -> int: ... + @property + def no_site(self) -> int: ... + @property + def ignore_environment(self) -> int: ... + @property + def verbose(self) -> int: ... + @property + def bytes_warning(self) -> int: ... + @property + def quiet(self) -> int: ... + @property + def hash_randomization(self) -> int: ... + @property + def isolated(self) -> int: ... + @property + def dev_mode(self) -> bool: ... + @property + def utf8_mode(self) -> int: ... + if sys.version_info >= (3, 10): + @property + def warn_default_encoding(self) -> int: ... + if sys.version_info >= (3, 11): + @property + def safe_path(self) -> bool: ... + # Whether or not this exists on lower versions of Python + # may depend on which patch release you're using + # (it was backported to all Python versions on 3.8+ as a security fix) + # Added in: 3.8.14, 3.9.14, 3.10.7 + # and present in all versions of 3.11 and later. + @property + def int_max_str_digits(self) -> int: ... + +float_info: _float_info + +# This class is not exposed at runtime. It calls itself sys.float_info. +@final +@type_check_only +class _float_info(structseq[float], tuple[float, int, int, float, int, int, int, int, float, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ( + "max", + "max_exp", + "max_10_exp", + "min", + "min_exp", + "min_10_exp", + "dig", + "mant_dig", + "epsilon", + "radix", + "rounds", + ) + + @property + def max(self) -> float: ... # DBL_MAX + @property + def max_exp(self) -> int: ... # DBL_MAX_EXP + @property + def max_10_exp(self) -> int: ... # DBL_MAX_10_EXP + @property + def min(self) -> float: ... # DBL_MIN + @property + def min_exp(self) -> int: ... # DBL_MIN_EXP + @property + def min_10_exp(self) -> int: ... # DBL_MIN_10_EXP + @property + def dig(self) -> int: ... # DBL_DIG + @property + def mant_dig(self) -> int: ... # DBL_MANT_DIG + @property + def epsilon(self) -> float: ... # DBL_EPSILON + @property + def radix(self) -> int: ... # FLT_RADIX + @property + def rounds(self) -> int: ... # FLT_ROUNDS + +hash_info: _hash_info + +# This class is not exposed at runtime. It calls itself sys.hash_info. +@final +@type_check_only +class _hash_info(structseq[Any | int], tuple[int, int, int, int, int, str, int, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("width", "modulus", "inf", "nan", "imag", "algorithm", "hash_bits", "seed_bits", "cutoff") + + @property + def width(self) -> int: ... + @property + def modulus(self) -> int: ... + @property + def inf(self) -> int: ... + @property + def nan(self) -> int: ... + @property + def imag(self) -> int: ... + @property + def algorithm(self) -> str: ... + @property + def hash_bits(self) -> int: ... + @property + def seed_bits(self) -> int: ... + @property + def cutoff(self) -> int: ... # undocumented + +implementation: _implementation + +# This class isn't really a thing. At runtime, implementation is an instance +# of types.SimpleNamespace. This allows for better typing. +@type_check_only +class _implementation: + name: str + version: _version_info + hexversion: int + cache_tag: str + # Define __getattr__, as the documentation states: + # > sys.implementation may contain additional attributes specific to the Python implementation. + # > These non-standard attributes must start with an underscore, and are not described here. + def __getattr__(self, name: str) -> Any: ... + +int_info: _int_info + +# This class is not exposed at runtime. It calls itself sys.int_info. +@final +@type_check_only +class _int_info(structseq[int], tuple[int, int, int, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("bits_per_digit", "sizeof_digit", "default_max_str_digits", "str_digits_check_threshold") + + @property + def bits_per_digit(self) -> int: ... + @property + def sizeof_digit(self) -> int: ... + @property + def default_max_str_digits(self) -> int: ... + @property + def str_digits_check_threshold(self) -> int: ... + +_ThreadInfoName: TypeAlias = Literal["nt", "pthread", "pthread-stubs", "solaris"] +_ThreadInfoLock: TypeAlias = Literal["semaphore", "mutex+cond"] | None + +# This class is not exposed at runtime. It calls itself sys.thread_info. +@final +@type_check_only +class _thread_info(_UninstantiableStructseq, tuple[_ThreadInfoName, _ThreadInfoLock, str | None]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("name", "lock", "version") + + @property + def name(self) -> _ThreadInfoName: ... + @property + def lock(self) -> _ThreadInfoLock: ... + @property + def version(self) -> str | None: ... + +thread_info: _thread_info +_ReleaseLevel: TypeAlias = Literal["alpha", "beta", "candidate", "final"] + +# This class is not exposed at runtime. It calls itself sys.version_info. +@final +@type_check_only +class _version_info(_UninstantiableStructseq, tuple[int, int, int, _ReleaseLevel, int]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("major", "minor", "micro", "releaselevel", "serial") + + @property + def major(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def micro(self) -> int: ... + @property + def releaselevel(self) -> _ReleaseLevel: ... + @property + def serial(self) -> int: ... + +version_info: _version_info + +def call_tracing(func: Callable[..., _T], args: Any, /) -> _T: ... +def _clear_type_cache() -> None: ... +def _current_frames() -> dict[int, FrameType]: ... +def _getframe(depth: int = 0, /) -> FrameType: ... +def _debugmallocstats() -> None: ... +def __displayhook__(object: object, /) -> None: ... +def __excepthook__(exctype: type[BaseException], value: BaseException, traceback: TracebackType | None, /) -> None: ... +def exc_info() -> OptExcInfo: ... + +if sys.version_info >= (3, 11): + def exception() -> BaseException | None: ... + +def exit(status: _ExitCode = None, /) -> NoReturn: ... +def getallocatedblocks() -> int: ... +def getdefaultencoding() -> str: ... + +if sys.platform != "win32": + def getdlopenflags() -> int: ... + +def getfilesystemencoding() -> str: ... +def getfilesystemencodeerrors() -> str: ... +def getrefcount(object: Any, /) -> int: ... +def getrecursionlimit() -> int: ... +def getsizeof(obj: object, default: int = ...) -> int: ... +def getswitchinterval() -> float: ... +def getprofile() -> ProfileFunction | None: ... +def setprofile(function: ProfileFunction | None, /) -> None: ... +def gettrace() -> TraceFunction | None: ... +def settrace(function: TraceFunction | None, /) -> None: ... + +if sys.platform == "win32": + # A tuple of length 5, even though it has more than 5 attributes. + @final + class _WinVersion(_UninstantiableStructseq, tuple[int, int, int, int, str]): + @property + def major(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def build(self) -> int: ... + @property + def platform(self) -> int: ... + @property + def service_pack(self) -> str: ... + @property + def service_pack_minor(self) -> int: ... + @property + def service_pack_major(self) -> int: ... + @property + def suite_mask(self) -> int: ... + @property + def product_type(self) -> int: ... + @property + def platform_version(self) -> tuple[int, int, int]: ... + + def getwindowsversion() -> _WinVersion: ... + +def intern(string: str, /) -> str: ... +def is_finalizing() -> bool: ... +def breakpointhook(*args: Any, **kwargs: Any) -> Any: ... + +__breakpointhook__ = breakpointhook # Contains the original value of breakpointhook + +if sys.platform != "win32": + def setdlopenflags(flags: int, /) -> None: ... + +def setrecursionlimit(limit: int, /) -> None: ... +def setswitchinterval(interval: float, /) -> None: ... +def gettotalrefcount() -> int: ... # Debug builds only + +if sys.version_info < (3, 9): + def getcheckinterval() -> int: ... # deprecated + def setcheckinterval(n: int, /) -> None: ... # deprecated + +if sys.version_info < (3, 9): + # An 11-tuple or None + def callstats() -> tuple[int, int, int, int, int, int, int, int, int, int, int] | None: ... + +# Doesn't exist at runtime, but exported in the stubs so pytest etc. can annotate their code more easily. +@type_check_only +class UnraisableHookArgs(Protocol): + exc_type: type[BaseException] + exc_value: BaseException | None + exc_traceback: TracebackType | None + err_msg: str | None + object: _object + +unraisablehook: Callable[[UnraisableHookArgs], Any] + +def __unraisablehook__(unraisable: UnraisableHookArgs, /) -> Any: ... +def addaudithook(hook: Callable[[str, tuple[Any, ...]], Any]) -> None: ... +def audit(event: str, /, *args: Any) -> None: ... + +_AsyncgenHook: TypeAlias = Callable[[AsyncGenerator[Any, Any]], None] | None + +# This class is not exposed at runtime. It calls itself builtins.asyncgen_hooks. +@final +@type_check_only +class _asyncgen_hooks(structseq[_AsyncgenHook], tuple[_AsyncgenHook, _AsyncgenHook]): + if sys.version_info >= (3, 10): + __match_args__: Final = ("firstiter", "finalizer") + + @property + def firstiter(self) -> _AsyncgenHook: ... + @property + def finalizer(self) -> _AsyncgenHook: ... + +def get_asyncgen_hooks() -> _asyncgen_hooks: ... +def set_asyncgen_hooks(firstiter: _AsyncgenHook = ..., finalizer: _AsyncgenHook = ...) -> None: ... + +if sys.platform == "win32": + def _enablelegacywindowsfsencoding() -> None: ... + +def get_coroutine_origin_tracking_depth() -> int: ... +def set_coroutine_origin_tracking_depth(depth: int) -> None: ... + +# The following two functions were added in 3.11.0, 3.10.7, 3.9.14, and 3.8.14, +# as part of the response to CVE-2020-10735 +def set_int_max_str_digits(maxdigits: int) -> None: ... +def get_int_max_str_digits() -> int: ... + +if sys.version_info >= (3, 12): + if sys.version_info >= (3, 13): + def getunicodeinternedsize(*, _only_immortal: bool = False) -> int: ... + else: + def getunicodeinternedsize() -> int: ... + + def deactivate_stack_trampoline() -> None: ... + def is_stack_trampoline_active() -> bool: ... + # It always exists, but raises on non-linux platforms: + if sys.platform == "linux": + def activate_stack_trampoline(backend: str, /) -> None: ... + else: + def activate_stack_trampoline(backend: str, /) -> NoReturn: ... + + from . import _monitoring + + monitoring = _monitoring diff --git a/mypy/typeshed/stdlib/sys/_monitoring.pyi b/mypy/typeshed/stdlib/sys/_monitoring.pyi new file mode 100644 index 000000000000..0507eeedc26d --- /dev/null +++ b/mypy/typeshed/stdlib/sys/_monitoring.pyi @@ -0,0 +1,52 @@ +# This py312+ module provides annotations for `sys.monitoring`. +# It's named `sys._monitoring` in typeshed, +# because trying to import `sys.monitoring` will fail at runtime! +# At runtime, `sys.monitoring` has the unique status +# of being a `types.ModuleType` instance that cannot be directly imported, +# and exists in the `sys`-module namespace despite `sys` not being a package. + +from collections.abc import Callable +from types import CodeType +from typing import Any + +DEBUGGER_ID: int +COVERAGE_ID: int +PROFILER_ID: int +OPTIMIZER_ID: int + +def use_tool_id(tool_id: int, name: str, /) -> None: ... +def free_tool_id(tool_id: int, /) -> None: ... +def get_tool(tool_id: int, /) -> str | None: ... + +events: _events + +class _events: + BRANCH: int + CALL: int + C_RAISE: int + C_RETURN: int + EXCEPTION_HANDLED: int + INSTRUCTION: int + JUMP: int + LINE: int + NO_EVENTS: int + PY_RESUME: int + PY_RETURN: int + PY_START: int + PY_THROW: int + PY_UNWIND: int + PY_YIELD: int + RAISE: int + RERAISE: int + STOP_ITERATION: int + +def get_events(tool_id: int, /) -> int: ... +def set_events(tool_id: int, event_set: int, /) -> None: ... +def get_local_events(tool_id: int, code: CodeType, /) -> int: ... +def set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> int: ... +def restart_events() -> None: ... + +DISABLE: object +MISSING: object + +def register_callback(tool_id: int, event: int, func: Callable[..., Any] | None, /) -> Callable[..., Any] | None: ... diff --git a/mypy/typeshed/stdlib/sysconfig.pyi b/mypy/typeshed/stdlib/sysconfig.pyi new file mode 100644 index 000000000000..807a979050e8 --- /dev/null +++ b/mypy/typeshed/stdlib/sysconfig.pyi @@ -0,0 +1,48 @@ +import sys +from typing import IO, Any, Literal, overload +from typing_extensions import deprecated + +__all__ = [ + "get_config_h_filename", + "get_config_var", + "get_config_vars", + "get_makefile_filename", + "get_path", + "get_path_names", + "get_paths", + "get_platform", + "get_python_version", + "get_scheme_names", + "parse_config_h", +] + +@overload +@deprecated("SO is deprecated, use EXT_SUFFIX. Support is removed in Python 3.11") +def get_config_var(name: Literal["SO"]) -> Any: ... +@overload +def get_config_var(name: str) -> Any: ... +@overload +def get_config_vars() -> dict[str, Any]: ... +@overload +def get_config_vars(arg: str, /, *args: str) -> list[Any]: ... +def get_scheme_names() -> tuple[str, ...]: ... + +if sys.version_info >= (3, 10): + def get_default_scheme() -> str: ... + def get_preferred_scheme(key: Literal["prefix", "home", "user"]) -> str: ... + +def get_path_names() -> tuple[str, ...]: ... +def get_path(name: str, scheme: str = ..., vars: dict[str, Any] | None = None, expand: bool = True) -> str: ... +def get_paths(scheme: str = ..., vars: dict[str, Any] | None = None, expand: bool = True) -> dict[str, str]: ... +def get_python_version() -> str: ... +def get_platform() -> str: ... + +if sys.version_info >= (3, 11): + def is_python_build(check_home: object = None) -> bool: ... + +else: + def is_python_build(check_home: bool = False) -> bool: ... + +def parse_config_h(fp: IO[Any], vars: dict[str, Any] | None = None) -> dict[str, Any]: ... +def get_config_h_filename() -> str: ... +def get_makefile_filename() -> str: ... diff --git a/mypy/typeshed/stdlib/syslog.pyi b/mypy/typeshed/stdlib/syslog.pyi new file mode 100644 index 000000000000..1e0d0d383902 --- /dev/null +++ b/mypy/typeshed/stdlib/syslog.pyi @@ -0,0 +1,57 @@ +import sys +from typing import Final, overload + +if sys.platform != "win32": + LOG_ALERT: Final = 1 + LOG_AUTH: Final = 32 + LOG_AUTHPRIV: Final = 80 + LOG_CONS: Final = 2 + LOG_CRIT: Final = 2 + LOG_CRON: Final = 72 + LOG_DAEMON: Final = 24 + LOG_DEBUG: Final = 7 + LOG_EMERG: Final = 0 + LOG_ERR: Final = 3 + LOG_INFO: Final = 6 + LOG_KERN: Final = 0 + LOG_LOCAL0: Final = 128 + LOG_LOCAL1: Final = 136 + LOG_LOCAL2: Final = 144 + LOG_LOCAL3: Final = 152 + LOG_LOCAL4: Final = 160 + LOG_LOCAL5: Final = 168 + LOG_LOCAL6: Final = 176 + LOG_LOCAL7: Final = 184 + LOG_LPR: Final = 48 + LOG_MAIL: Final = 16 + LOG_NDELAY: Final = 8 + LOG_NEWS: Final = 56 + LOG_NOTICE: Final = 5 + LOG_NOWAIT: Final = 16 + LOG_ODELAY: Final = 4 + LOG_PERROR: Final = 32 + LOG_PID: Final = 1 + LOG_SYSLOG: Final = 40 + LOG_USER: Final = 8 + LOG_UUCP: Final = 64 + LOG_WARNING: Final = 4 + + if sys.version_info >= (3, 13): + LOG_FTP: Final = 88 + + if sys.platform == "darwin": + LOG_INSTALL: Final = 112 + LOG_LAUNCHD: Final = 192 + LOG_NETINFO: Final = 96 + LOG_RAS: Final = 120 + LOG_REMOTEAUTH: Final = 104 + + def LOG_MASK(pri: int, /) -> int: ... + def LOG_UPTO(pri: int, /) -> int: ... + def closelog() -> None: ... + def openlog(ident: str = ..., logoption: int = ..., facility: int = ...) -> None: ... + def setlogmask(maskpri: int, /) -> int: ... + @overload + def syslog(priority: int, message: str) -> None: ... + @overload + def syslog(message: str) -> None: ... diff --git a/mypy/typeshed/stdlib/tabnanny.pyi b/mypy/typeshed/stdlib/tabnanny.pyi new file mode 100644 index 000000000000..8a8592f44124 --- /dev/null +++ b/mypy/typeshed/stdlib/tabnanny.pyi @@ -0,0 +1,16 @@ +from _typeshed import StrOrBytesPath +from collections.abc import Iterable + +__all__ = ["check", "NannyNag", "process_tokens"] + +verbose: int +filename_only: int + +class NannyNag(Exception): + def __init__(self, lineno: int, msg: str, line: str) -> None: ... + def get_lineno(self) -> int: ... + def get_msg(self) -> str: ... + def get_line(self) -> str: ... + +def check(file: StrOrBytesPath) -> None: ... +def process_tokens(tokens: Iterable[tuple[int, str, tuple[int, int], tuple[int, int], str]]) -> None: ... diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi new file mode 100644 index 000000000000..a7135d8150ee --- /dev/null +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -0,0 +1,589 @@ +import bz2 +import io +import sys +from _typeshed import StrOrBytesPath, StrPath, SupportsRead +from builtins import list as _list # aliases to avoid name clashes with fields named "type" or "list" +from collections.abc import Callable, Iterable, Iterator, Mapping +from gzip import _ReadableFileobj as _GzipReadableFileobj, _WritableFileobj as _GzipWritableFileobj +from types import TracebackType +from typing import IO, ClassVar, Literal, Protocol, overload +from typing_extensions import Self, TypeAlias + +__all__ = [ + "TarFile", + "TarInfo", + "is_tarfile", + "TarError", + "ReadError", + "CompressionError", + "StreamError", + "ExtractError", + "HeaderError", + "ENCODING", + "USTAR_FORMAT", + "GNU_FORMAT", + "PAX_FORMAT", + "DEFAULT_FORMAT", + "open", +] +if sys.version_info >= (3, 12): + __all__ += [ + "fully_trusted_filter", + "data_filter", + "tar_filter", + "FilterError", + "AbsoluteLinkError", + "OutsideDestinationError", + "SpecialFileError", + "AbsolutePathError", + "LinkOutsideDestinationError", + ] + +_FilterFunction: TypeAlias = Callable[[TarInfo, str], TarInfo | None] +_TarfileFilter: TypeAlias = Literal["fully_trusted", "tar", "data"] | _FilterFunction + +class _Fileobj(Protocol): + def read(self, size: int, /) -> bytes: ... + def write(self, b: bytes, /) -> object: ... + def tell(self) -> int: ... + def seek(self, pos: int, /) -> object: ... + def close(self) -> object: ... + # Optional fields: + # name: str | bytes + # mode: Literal["rb", "r+b", "wb", "xb"] + +class _Bz2ReadableFileobj(bz2._ReadableFileobj): + def close(self) -> object: ... + +class _Bz2WritableFileobj(bz2._WritableFileobj): + def close(self) -> object: ... + +# tar constants +NUL: bytes +BLOCKSIZE: int +RECORDSIZE: int +GNU_MAGIC: bytes +POSIX_MAGIC: bytes + +LENGTH_NAME: int +LENGTH_LINK: int +LENGTH_PREFIX: int + +REGTYPE: bytes +AREGTYPE: bytes +LNKTYPE: bytes +SYMTYPE: bytes +CONTTYPE: bytes +BLKTYPE: bytes +DIRTYPE: bytes +FIFOTYPE: bytes +CHRTYPE: bytes + +GNUTYPE_LONGNAME: bytes +GNUTYPE_LONGLINK: bytes +GNUTYPE_SPARSE: bytes + +XHDTYPE: bytes +XGLTYPE: bytes +SOLARIS_XHDTYPE: bytes + +USTAR_FORMAT: int +GNU_FORMAT: int +PAX_FORMAT: int +DEFAULT_FORMAT: int + +# tarfile constants + +SUPPORTED_TYPES: tuple[bytes, ...] +REGULAR_TYPES: tuple[bytes, ...] +GNU_TYPES: tuple[bytes, ...] +PAX_FIELDS: tuple[str, ...] +PAX_NUMBER_FIELDS: dict[str, type] +PAX_NAME_FIELDS: set[str] + +ENCODING: str + +@overload +def open( + name: StrOrBytesPath | None = None, + mode: Literal["r", "r:*", "r:", "r:gz", "r:bz2", "r:xz"] = "r", + fileobj: IO[bytes] | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None, + mode: Literal["x", "x:", "a", "a:", "w", "w:"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None = None, + *, + mode: Literal["x", "x:", "a", "a:", "w", "w:"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None, + mode: Literal["x:gz", "x:bz2", "w:gz", "w:bz2"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + compresslevel: int = 9, +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None = None, + *, + mode: Literal["x:gz", "x:bz2", "w:gz", "w:bz2"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + compresslevel: int = 9, +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None, + mode: Literal["x:xz", "w:xz"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + preset: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = ..., +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None = None, + *, + mode: Literal["x:xz", "w:xz"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + preset: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = ..., +) -> TarFile: ... + +# TODO: Temporary fallback for modes containing pipe characters. These don't +# work with mypy 1.10, but this should be fixed with mypy 1.11. +# https://github.com/python/typeshed/issues/12182 +@overload +def open( + name: StrOrBytesPath | None = None, + *, + mode: str, + fileobj: IO[bytes] | None = None, + bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + preset: int | None = ..., +) -> TarFile: ... + +class ExFileObject(io.BufferedReader): + def __init__(self, tarfile: TarFile, tarinfo: TarInfo) -> None: ... + +class TarFile: + OPEN_METH: ClassVar[Mapping[str, str]] + name: StrOrBytesPath | None + mode: Literal["r", "a", "w", "x"] + fileobj: _Fileobj | None + format: int | None + tarinfo: type[TarInfo] + dereference: bool | None + ignore_zeros: bool | None + encoding: str | None + errors: str + fileobject: type[ExFileObject] + pax_headers: Mapping[str, str] | None + debug: int | None + errorlevel: int | None + offset: int # undocumented + extraction_filter: _FilterFunction | None + if sys.version_info >= (3, 13): + stream: bool + def __init__( + self, + name: StrOrBytesPath | None = None, + mode: Literal["r", "a", "w", "x"] = "r", + fileobj: _Fileobj | None = None, + format: int | None = None, + tarinfo: type[TarInfo] | None = None, + dereference: bool | None = None, + ignore_zeros: bool | None = None, + encoding: str | None = None, + errors: str = "surrogateescape", + pax_headers: Mapping[str, str] | None = None, + debug: int | None = None, + errorlevel: int | None = None, + copybufsize: int | None = None, # undocumented + stream: bool = False, + ) -> None: ... + else: + def __init__( + self, + name: StrOrBytesPath | None = None, + mode: Literal["r", "a", "w", "x"] = "r", + fileobj: _Fileobj | None = None, + format: int | None = None, + tarinfo: type[TarInfo] | None = None, + dereference: bool | None = None, + ignore_zeros: bool | None = None, + encoding: str | None = None, + errors: str = "surrogateescape", + pax_headers: Mapping[str, str] | None = None, + debug: int | None = None, + errorlevel: int | None = None, + copybufsize: int | None = None, # undocumented + ) -> None: ... + + def __enter__(self) -> Self: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + def __iter__(self) -> Iterator[TarInfo]: ... + @classmethod + def open( + cls, + name: StrOrBytesPath | None = None, + mode: str = "r", + fileobj: IO[bytes] | None = None, # depends on mode + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + @classmethod + def taropen( + cls, + name: StrOrBytesPath | None, + mode: Literal["r", "a", "w", "x"] = "r", + fileobj: _Fileobj | None = None, + *, + compresslevel: int = ..., + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + @overload + @classmethod + def gzopen( + cls, + name: StrOrBytesPath | None, + mode: Literal["r"] = "r", + fileobj: _GzipReadableFileobj | None = None, + compresslevel: int = 9, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + @overload + @classmethod + def gzopen( + cls, + name: StrOrBytesPath | None, + mode: Literal["w", "x"], + fileobj: _GzipWritableFileobj | None = None, + compresslevel: int = 9, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + @overload + @classmethod + def bz2open( + cls, + name: StrOrBytesPath | None, + mode: Literal["w", "x"], + fileobj: _Bz2WritableFileobj | None = None, + compresslevel: int = 9, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + @overload + @classmethod + def bz2open( + cls, + name: StrOrBytesPath | None, + mode: Literal["r"] = "r", + fileobj: _Bz2ReadableFileobj | None = None, + compresslevel: int = 9, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + @classmethod + def xzopen( + cls, + name: StrOrBytesPath | None, + mode: Literal["r", "w", "x"] = "r", + fileobj: IO[bytes] | None = None, + preset: int | None = None, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + def getmember(self, name: str) -> TarInfo: ... + def getmembers(self) -> _list[TarInfo]: ... + def getnames(self) -> _list[str]: ... + def list(self, verbose: bool = True, *, members: _list[TarInfo] | None = None) -> None: ... + def next(self) -> TarInfo | None: ... + # Calling this method without `filter` is deprecated, but it may be set either on the class or in an + # individual call, so we can't mark it as @deprecated here. + def extractall( + self, + path: StrOrBytesPath = ".", + members: Iterable[TarInfo] | None = None, + *, + numeric_owner: bool = False, + filter: _TarfileFilter | None = ..., + ) -> None: ... + # Same situation as for `extractall`. + def extract( + self, + member: str | TarInfo, + path: StrOrBytesPath = "", + set_attrs: bool = True, + *, + numeric_owner: bool = False, + filter: _TarfileFilter | None = ..., + ) -> None: ... + def _extract_member( + self, tarinfo: TarInfo, targetpath: str, set_attrs: bool = True, numeric_owner: bool = False + ) -> None: ... # undocumented + def extractfile(self, member: str | TarInfo) -> IO[bytes] | None: ... + def makedir(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def makefile(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def makeunknown(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def makefifo(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def makedev(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def makelink(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def chown(self, tarinfo: TarInfo, targetpath: StrOrBytesPath, numeric_owner: bool) -> None: ... # undocumented + def chmod(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def utime(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def add( + self, + name: StrPath, + arcname: StrPath | None = None, + recursive: bool = True, + *, + filter: Callable[[TarInfo], TarInfo | None] | None = None, + ) -> None: ... + def addfile(self, tarinfo: TarInfo, fileobj: SupportsRead[bytes] | None = None) -> None: ... + def gettarinfo( + self, name: StrOrBytesPath | None = None, arcname: str | None = None, fileobj: IO[bytes] | None = None + ) -> TarInfo: ... + def close(self) -> None: ... + +if sys.version_info >= (3, 9): + def is_tarfile(name: StrOrBytesPath | IO[bytes]) -> bool: ... + +else: + def is_tarfile(name: StrOrBytesPath) -> bool: ... + +class TarError(Exception): ... +class ReadError(TarError): ... +class CompressionError(TarError): ... +class StreamError(TarError): ... +class ExtractError(TarError): ... +class HeaderError(TarError): ... + +class FilterError(TarError): + # This attribute is only set directly on the subclasses, but the documentation guarantees + # that it is always present on FilterError. + tarinfo: TarInfo + +class AbsolutePathError(FilterError): + def __init__(self, tarinfo: TarInfo) -> None: ... + +class OutsideDestinationError(FilterError): + def __init__(self, tarinfo: TarInfo, path: str) -> None: ... + +class SpecialFileError(FilterError): + def __init__(self, tarinfo: TarInfo) -> None: ... + +class AbsoluteLinkError(FilterError): + def __init__(self, tarinfo: TarInfo) -> None: ... + +class LinkOutsideDestinationError(FilterError): + def __init__(self, tarinfo: TarInfo, path: str) -> None: ... + +def fully_trusted_filter(member: TarInfo, dest_path: str) -> TarInfo: ... +def tar_filter(member: TarInfo, dest_path: str) -> TarInfo: ... +def data_filter(member: TarInfo, dest_path: str) -> TarInfo: ... + +class TarInfo: + name: str + path: str + size: int + mtime: int | float + chksum: int + devmajor: int + devminor: int + offset: int + offset_data: int + sparse: bytes | None + tarfile: TarFile | None + mode: int + type: bytes + linkname: str + uid: int + gid: int + uname: str + gname: str + pax_headers: Mapping[str, str] + def __init__(self, name: str = "") -> None: ... + @classmethod + def frombuf(cls, buf: bytes | bytearray, encoding: str, errors: str) -> Self: ... + @classmethod + def fromtarfile(cls, tarfile: TarFile) -> Self: ... + @property + def linkpath(self) -> str: ... + @linkpath.setter + def linkpath(self, linkname: str) -> None: ... + def replace( + self, + *, + name: str = ..., + mtime: int = ..., + mode: int = ..., + linkname: str = ..., + uid: int = ..., + gid: int = ..., + uname: str = ..., + gname: str = ..., + deep: bool = True, + ) -> Self: ... + def get_info(self) -> Mapping[str, str | int | bytes | Mapping[str, str]]: ... + def tobuf(self, format: int | None = 2, encoding: str | None = "utf-8", errors: str = "surrogateescape") -> bytes: ... + def create_ustar_header( + self, info: Mapping[str, str | int | bytes | Mapping[str, str]], encoding: str, errors: str + ) -> bytes: ... + def create_gnu_header( + self, info: Mapping[str, str | int | bytes | Mapping[str, str]], encoding: str, errors: str + ) -> bytes: ... + def create_pax_header(self, info: Mapping[str, str | int | bytes | Mapping[str, str]], encoding: str) -> bytes: ... + @classmethod + def create_pax_global_header(cls, pax_headers: Mapping[str, str]) -> bytes: ... + def isfile(self) -> bool: ... + def isreg(self) -> bool: ... + def issparse(self) -> bool: ... + def isdir(self) -> bool: ... + def issym(self) -> bool: ... + def islnk(self) -> bool: ... + def ischr(self) -> bool: ... + def isblk(self) -> bool: ... + def isfifo(self) -> bool: ... + def isdev(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/telnetlib.pyi b/mypy/typeshed/stdlib/telnetlib.pyi new file mode 100644 index 000000000000..294a1cb12b63 --- /dev/null +++ b/mypy/typeshed/stdlib/telnetlib.pyi @@ -0,0 +1,123 @@ +import socket +from collections.abc import Callable, Sequence +from re import Match, Pattern +from types import TracebackType +from typing import Any +from typing_extensions import Self + +__all__ = ["Telnet"] + +DEBUGLEVEL: int +TELNET_PORT: int + +IAC: bytes +DONT: bytes +DO: bytes +WONT: bytes +WILL: bytes +theNULL: bytes + +SE: bytes +NOP: bytes +DM: bytes +BRK: bytes +IP: bytes +AO: bytes +AYT: bytes +EC: bytes +EL: bytes +GA: bytes +SB: bytes + +BINARY: bytes +ECHO: bytes +RCP: bytes +SGA: bytes +NAMS: bytes +STATUS: bytes +TM: bytes +RCTE: bytes +NAOL: bytes +NAOP: bytes +NAOCRD: bytes +NAOHTS: bytes +NAOHTD: bytes +NAOFFD: bytes +NAOVTS: bytes +NAOVTD: bytes +NAOLFD: bytes +XASCII: bytes +LOGOUT: bytes +BM: bytes +DET: bytes +SUPDUP: bytes +SUPDUPOUTPUT: bytes +SNDLOC: bytes +TTYPE: bytes +EOR: bytes +TUID: bytes +OUTMRK: bytes +TTYLOC: bytes +VT3270REGIME: bytes +X3PAD: bytes +NAWS: bytes +TSPEED: bytes +LFLOW: bytes +LINEMODE: bytes +XDISPLOC: bytes +OLD_ENVIRON: bytes +AUTHENTICATION: bytes +ENCRYPT: bytes +NEW_ENVIRON: bytes + +TN3270E: bytes +XAUTH: bytes +CHARSET: bytes +RSP: bytes +COM_PORT_OPTION: bytes +SUPPRESS_LOCAL_ECHO: bytes +TLS: bytes +KERMIT: bytes +SEND_URL: bytes +FORWARD_X: bytes +PRAGMA_LOGON: bytes +SSPI_LOGON: bytes +PRAGMA_HEARTBEAT: bytes +EXOPL: bytes +NOOPT: bytes + +class Telnet: + host: str | None # undocumented + sock: socket.socket | None # undocumented + def __init__(self, host: str | None = None, port: int = 0, timeout: float = ...) -> None: ... + def open(self, host: str, port: int = 0, timeout: float = ...) -> None: ... + def msg(self, msg: str, *args: Any) -> None: ... + def set_debuglevel(self, debuglevel: int) -> None: ... + def close(self) -> None: ... + def get_socket(self) -> socket.socket: ... + def fileno(self) -> int: ... + def write(self, buffer: bytes) -> None: ... + def read_until(self, match: bytes, timeout: float | None = None) -> bytes: ... + def read_all(self) -> bytes: ... + def read_some(self) -> bytes: ... + def read_very_eager(self) -> bytes: ... + def read_eager(self) -> bytes: ... + def read_lazy(self) -> bytes: ... + def read_very_lazy(self) -> bytes: ... + def read_sb_data(self) -> bytes: ... + def set_option_negotiation_callback(self, callback: Callable[[socket.socket, bytes, bytes], object] | None) -> None: ... + def process_rawq(self) -> None: ... + def rawq_getchar(self) -> bytes: ... + def fill_rawq(self) -> None: ... + def sock_avail(self) -> bool: ... + def interact(self) -> None: ... + def mt_interact(self) -> None: ... + def listener(self) -> None: ... + def expect( + self, list: Sequence[Pattern[bytes] | bytes], timeout: float | None = None + ) -> tuple[int, Match[bytes] | None, bytes]: ... + def __enter__(self) -> Self: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + def __del__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/tempfile.pyi b/mypy/typeshed/stdlib/tempfile.pyi new file mode 100644 index 000000000000..0c19d56fc7a6 --- /dev/null +++ b/mypy/typeshed/stdlib/tempfile.pyi @@ -0,0 +1,483 @@ +import io +import sys +from _typeshed import ( + BytesPath, + GenericPath, + OpenBinaryMode, + OpenBinaryModeReading, + OpenBinaryModeUpdating, + OpenBinaryModeWriting, + OpenTextMode, + ReadableBuffer, + StrPath, + WriteableBuffer, +) +from collections.abc import Iterable, Iterator +from types import TracebackType +from typing import IO, Any, AnyStr, Generic, Literal, overload +from typing_extensions import Self + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "NamedTemporaryFile", + "TemporaryFile", + "SpooledTemporaryFile", + "TemporaryDirectory", + "mkstemp", + "mkdtemp", + "mktemp", + "TMP_MAX", + "gettempprefix", + "tempdir", + "gettempdir", + "gettempprefixb", + "gettempdirb", +] + +# global variables +TMP_MAX: int +tempdir: str | None +template: str + +if sys.version_info >= (3, 12): + @overload + def NamedTemporaryFile( + mode: OpenTextMode, + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + delete: bool = True, + *, + errors: str | None = None, + delete_on_close: bool = True, + ) -> _TemporaryFileWrapper[str]: ... + @overload + def NamedTemporaryFile( + mode: OpenBinaryMode = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + delete: bool = True, + *, + errors: str | None = None, + delete_on_close: bool = True, + ) -> _TemporaryFileWrapper[bytes]: ... + @overload + def NamedTemporaryFile( + mode: str = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + delete: bool = True, + *, + errors: str | None = None, + delete_on_close: bool = True, + ) -> _TemporaryFileWrapper[Any]: ... + +else: + @overload + def NamedTemporaryFile( + mode: OpenTextMode, + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + delete: bool = True, + *, + errors: str | None = None, + ) -> _TemporaryFileWrapper[str]: ... + @overload + def NamedTemporaryFile( + mode: OpenBinaryMode = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + delete: bool = True, + *, + errors: str | None = None, + ) -> _TemporaryFileWrapper[bytes]: ... + @overload + def NamedTemporaryFile( + mode: str = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + delete: bool = True, + *, + errors: str | None = None, + ) -> _TemporaryFileWrapper[Any]: ... + +if sys.platform == "win32": + TemporaryFile = NamedTemporaryFile +else: + # See the comments for builtins.open() for an explanation of the overloads. + @overload + def TemporaryFile( + mode: OpenTextMode, + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + *, + errors: str | None = None, + ) -> io.TextIOWrapper: ... + @overload + def TemporaryFile( + mode: OpenBinaryMode, + buffering: Literal[0], + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + *, + errors: str | None = None, + ) -> io.FileIO: ... + @overload + def TemporaryFile( + *, + buffering: Literal[0], + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + errors: str | None = None, + ) -> io.FileIO: ... + @overload + def TemporaryFile( + mode: OpenBinaryModeWriting, + buffering: Literal[-1, 1] = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + *, + errors: str | None = None, + ) -> io.BufferedWriter: ... + @overload + def TemporaryFile( + mode: OpenBinaryModeReading, + buffering: Literal[-1, 1] = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + *, + errors: str | None = None, + ) -> io.BufferedReader: ... + @overload + def TemporaryFile( + mode: OpenBinaryModeUpdating = "w+b", + buffering: Literal[-1, 1] = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + *, + errors: str | None = None, + ) -> io.BufferedRandom: ... + @overload + def TemporaryFile( + mode: str = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: GenericPath[AnyStr] | None = None, + *, + errors: str | None = None, + ) -> IO[Any]: ... + +class _TemporaryFileWrapper(IO[AnyStr]): + file: IO[AnyStr] # io.TextIOWrapper, io.BufferedReader or io.BufferedWriter + name: str + delete: bool + if sys.version_info >= (3, 12): + def __init__(self, file: IO[AnyStr], name: str, delete: bool = True, delete_on_close: bool = True) -> None: ... + else: + def __init__(self, file: IO[AnyStr], name: str, delete: bool = True) -> None: ... + + def __enter__(self) -> Self: ... + def __exit__(self, exc: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None) -> None: ... + def __getattr__(self, name: str) -> Any: ... + def close(self) -> None: ... + # These methods don't exist directly on this object, but + # are delegated to the underlying IO object through __getattr__. + # We need to add them here so that this class is concrete. + def __iter__(self) -> Iterator[AnyStr]: ... + # FIXME: __next__ doesn't actually exist on this class and should be removed: + # see also https://github.com/python/typeshed/pull/5456#discussion_r633068648 + # >>> import tempfile + # >>> ntf=tempfile.NamedTemporaryFile() + # >>> next(ntf) + # Traceback (most recent call last): + # File "", line 1, in + # TypeError: '_TemporaryFileWrapper' object is not an iterator + def __next__(self) -> AnyStr: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + def read(self, n: int = ...) -> AnyStr: ... + def readable(self) -> bool: ... + def readline(self, limit: int = ...) -> AnyStr: ... + def readlines(self, hint: int = ...) -> list[AnyStr]: ... + def seek(self, offset: int, whence: int = ...) -> int: ... + def seekable(self) -> bool: ... + def tell(self) -> int: ... + def truncate(self, size: int | None = ...) -> int: ... + def writable(self) -> bool: ... + @overload + def write(self: _TemporaryFileWrapper[str], s: str, /) -> int: ... + @overload + def write(self: _TemporaryFileWrapper[bytes], s: ReadableBuffer, /) -> int: ... + @overload + def write(self, s: AnyStr, /) -> int: ... + @overload + def writelines(self: _TemporaryFileWrapper[str], lines: Iterable[str]) -> None: ... + @overload + def writelines(self: _TemporaryFileWrapper[bytes], lines: Iterable[ReadableBuffer]) -> None: ... + @overload + def writelines(self, lines: Iterable[AnyStr]) -> None: ... + @property + def closed(self) -> bool: ... + +if sys.version_info >= (3, 11): + _SpooledTemporaryFileBase = io.IOBase +else: + _SpooledTemporaryFileBase = object + +# It does not actually derive from IO[AnyStr], but it does mostly behave +# like one. +class SpooledTemporaryFile(IO[AnyStr], _SpooledTemporaryFileBase): + _file: IO[AnyStr] + @property + def encoding(self) -> str: ... # undocumented + @property + def newlines(self) -> str | tuple[str, ...] | None: ... # undocumented + # bytes needs to go first, as default mode is to open as bytes + @overload + def __init__( + self: SpooledTemporaryFile[bytes], + max_size: int = 0, + mode: OpenBinaryMode = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + *, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self: SpooledTemporaryFile[str], + max_size: int, + mode: OpenTextMode, + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + *, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self: SpooledTemporaryFile[str], + max_size: int = 0, + *, + mode: OpenTextMode, + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self, + max_size: int, + mode: str, + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + *, + errors: str | None = None, + ) -> None: ... + @overload + def __init__( + self, + max_size: int = 0, + *, + mode: str, + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + errors: str | None = None, + ) -> None: ... + @property + def errors(self) -> str | None: ... + def rollover(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None) -> None: ... + # These methods are copied from the abstract methods of IO, because + # SpooledTemporaryFile implements IO. + # See also https://github.com/python/typeshed/pull/2452#issuecomment-420657918. + def close(self) -> None: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + if sys.version_info >= (3, 11): + # These three work only if the SpooledTemporaryFile is opened in binary mode, + # because the underlying object in text mode does not have these methods. + def read1(self, size: int = ..., /) -> AnyStr: ... + def readinto(self, b: WriteableBuffer) -> int: ... + def readinto1(self, b: WriteableBuffer) -> int: ... + def detach(self) -> io.RawIOBase: ... + + def read(self, n: int = ..., /) -> AnyStr: ... + def readline(self, limit: int | None = ..., /) -> AnyStr: ... # type: ignore[override] + def readlines(self, hint: int = ..., /) -> list[AnyStr]: ... # type: ignore[override] + def seek(self, offset: int, whence: int = ...) -> int: ... + def tell(self) -> int: ... + if sys.version_info >= (3, 11): + def truncate(self, size: int | None = None) -> int: ... + else: + def truncate(self, size: int | None = None) -> None: ... # type: ignore[override] + + @overload + def write(self: SpooledTemporaryFile[str], s: str) -> int: ... + @overload + def write(self: SpooledTemporaryFile[bytes], s: ReadableBuffer) -> int: ... + @overload + def write(self, s: AnyStr) -> int: ... + @overload + def writelines(self: SpooledTemporaryFile[str], iterable: Iterable[str]) -> None: ... + @overload + def writelines(self: SpooledTemporaryFile[bytes], iterable: Iterable[ReadableBuffer]) -> None: ... + @overload + def writelines(self, iterable: Iterable[AnyStr]) -> None: ... + def __iter__(self) -> Iterator[AnyStr]: ... # type: ignore[override] + # These exist at runtime only on 3.11+. + def readable(self) -> bool: ... + def seekable(self) -> bool: ... + def writable(self) -> bool: ... + def __next__(self) -> AnyStr: ... # type: ignore[override] + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class TemporaryDirectory(Generic[AnyStr]): + name: AnyStr + if sys.version_info >= (3, 12): + @overload + def __init__( + self: TemporaryDirectory[str], + suffix: str | None = None, + prefix: str | None = None, + dir: StrPath | None = None, + ignore_cleanup_errors: bool = False, + *, + delete: bool = True, + ) -> None: ... + @overload + def __init__( + self: TemporaryDirectory[bytes], + suffix: bytes | None = None, + prefix: bytes | None = None, + dir: BytesPath | None = None, + ignore_cleanup_errors: bool = False, + *, + delete: bool = True, + ) -> None: ... + elif sys.version_info >= (3, 10): + @overload + def __init__( + self: TemporaryDirectory[str], + suffix: str | None = None, + prefix: str | None = None, + dir: StrPath | None = None, + ignore_cleanup_errors: bool = False, + ) -> None: ... + @overload + def __init__( + self: TemporaryDirectory[bytes], + suffix: bytes | None = None, + prefix: bytes | None = None, + dir: BytesPath | None = None, + ignore_cleanup_errors: bool = False, + ) -> None: ... + else: + @overload + def __init__( + self: TemporaryDirectory[str], suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None + ) -> None: ... + @overload + def __init__( + self: TemporaryDirectory[bytes], + suffix: bytes | None = None, + prefix: bytes | None = None, + dir: BytesPath | None = None, + ) -> None: ... + + def cleanup(self) -> None: ... + def __enter__(self) -> AnyStr: ... + def __exit__(self, exc: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +# The overloads overlap, but they should still work fine. +@overload +def mkstemp( + suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None, text: bool = False +) -> tuple[int, str]: ... +@overload +def mkstemp( + suffix: bytes | None = None, prefix: bytes | None = None, dir: BytesPath | None = None, text: bool = False +) -> tuple[int, bytes]: ... + +# The overloads overlap, but they should still work fine. +@overload +def mkdtemp(suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None) -> str: ... +@overload +def mkdtemp(suffix: bytes | None = None, prefix: bytes | None = None, dir: BytesPath | None = None) -> bytes: ... +def mktemp(suffix: str = "", prefix: str = "tmp", dir: StrPath | None = None) -> str: ... +def gettempdirb() -> bytes: ... +def gettempprefixb() -> bytes: ... +def gettempdir() -> str: ... +def gettempprefix() -> str: ... diff --git a/mypy/typeshed/stdlib/termios.pyi b/mypy/typeshed/stdlib/termios.pyi new file mode 100644 index 000000000000..5a5a1f53be3c --- /dev/null +++ b/mypy/typeshed/stdlib/termios.pyi @@ -0,0 +1,303 @@ +import sys +from _typeshed import FileDescriptorLike +from typing import Any +from typing_extensions import TypeAlias + +# Must be a list of length 7, containing 6 ints and a list of NCCS 1-character bytes or ints. +_Attr: TypeAlias = list[int | list[bytes | int]] | list[int | list[bytes]] | list[int | list[int]] +# Same as _Attr for return types; we use Any to avoid a union. +_AttrReturn: TypeAlias = list[Any] + +if sys.platform != "win32": + B0: int + B110: int + B115200: int + B1200: int + B134: int + B150: int + B1800: int + B19200: int + B200: int + B230400: int + B2400: int + B300: int + B38400: int + B4800: int + B50: int + B57600: int + B600: int + B75: int + B9600: int + BRKINT: int + BS0: int + BS1: int + BSDLY: int + CDSUSP: int + CEOF: int + CEOL: int + CEOT: int + CERASE: int + CFLUSH: int + CINTR: int + CKILL: int + CLNEXT: int + CLOCAL: int + CQUIT: int + CR0: int + CR1: int + CR2: int + CR3: int + CRDLY: int + CREAD: int + CRPRNT: int + CRTSCTS: int + CS5: int + CS6: int + CS7: int + CS8: int + CSIZE: int + CSTART: int + CSTOP: int + CSTOPB: int + CSUSP: int + CWERASE: int + ECHO: int + ECHOCTL: int + ECHOE: int + ECHOK: int + ECHOKE: int + ECHONL: int + ECHOPRT: int + EXTA: int + EXTB: int + FF0: int + FF1: int + FFDLY: int + FIOASYNC: int + FIOCLEX: int + FIONBIO: int + FIONCLEX: int + FIONREAD: int + FLUSHO: int + HUPCL: int + ICANON: int + ICRNL: int + IEXTEN: int + IGNBRK: int + IGNCR: int + IGNPAR: int + IMAXBEL: int + INLCR: int + INPCK: int + ISIG: int + ISTRIP: int + IXANY: int + IXOFF: int + IXON: int + NCCS: int + NL0: int + NL1: int + NLDLY: int + NOFLSH: int + OCRNL: int + OFDEL: int + OFILL: int + ONLCR: int + ONLRET: int + ONOCR: int + OPOST: int + PARENB: int + PARMRK: int + PARODD: int + PENDIN: int + TAB0: int + TAB1: int + TAB2: int + TAB3: int + TABDLY: int + TCIFLUSH: int + TCIOFF: int + TCIOFLUSH: int + TCION: int + TCOFLUSH: int + TCOOFF: int + TCOON: int + TCSADRAIN: int + TCSAFLUSH: int + TCSANOW: int + TIOCCONS: int + TIOCEXCL: int + TIOCGETD: int + TIOCGPGRP: int + TIOCGWINSZ: int + TIOCM_CAR: int + TIOCM_CD: int + TIOCM_CTS: int + TIOCM_DSR: int + TIOCM_DTR: int + TIOCM_LE: int + TIOCM_RI: int + TIOCM_RNG: int + TIOCM_RTS: int + TIOCM_SR: int + TIOCM_ST: int + TIOCMBIC: int + TIOCMBIS: int + TIOCMGET: int + TIOCMSET: int + TIOCNOTTY: int + TIOCNXCL: int + TIOCOUTQ: int + TIOCPKT_DATA: int + TIOCPKT_DOSTOP: int + TIOCPKT_FLUSHREAD: int + TIOCPKT_FLUSHWRITE: int + TIOCPKT_NOSTOP: int + TIOCPKT_START: int + TIOCPKT_STOP: int + TIOCPKT: int + TIOCSCTTY: int + TIOCSETD: int + TIOCSPGRP: int + TIOCSTI: int + TIOCSWINSZ: int + TOSTOP: int + VDISCARD: int + VEOF: int + VEOL: int + VEOL2: int + VERASE: int + VINTR: int + VKILL: int + VLNEXT: int + VMIN: int + VQUIT: int + VREPRINT: int + VSTART: int + VSTOP: int + VSUSP: int + VT0: int + VT1: int + VTDLY: int + VTIME: int + VWERASE: int + + if sys.version_info >= (3, 13): + EXTPROC: int + IUTF8: int + + if sys.platform == "darwin" and sys.version_info >= (3, 13): + ALTWERASE: int + B14400: int + B28800: int + B7200: int + B76800: int + CCAR_OFLOW: int + CCTS_OFLOW: int + CDSR_OFLOW: int + CDTR_IFLOW: int + CIGNORE: int + CRTS_IFLOW: int + MDMBUF: int + NL2: int + NL3: int + NOKERNINFO: int + ONOEOT: int + OXTABS: int + VDSUSP: int + VSTATUS: int + + if sys.platform == "darwin" and sys.version_info >= (3, 11): + TIOCGSIZE: int + TIOCSSIZE: int + + if sys.platform == "linux": + B1152000: int + B576000: int + CBAUD: int + CBAUDEX: int + CIBAUD: int + IOCSIZE_MASK: int + IOCSIZE_SHIFT: int + IUCLC: int + N_MOUSE: int + N_PPP: int + N_SLIP: int + N_STRIP: int + N_TTY: int + NCC: int + OLCUC: int + TCFLSH: int + TCGETA: int + TCGETS: int + TCSBRK: int + TCSBRKP: int + TCSETA: int + TCSETAF: int + TCSETAW: int + TCSETS: int + TCSETSF: int + TCSETSW: int + TCXONC: int + TIOCGICOUNT: int + TIOCGLCKTRMIOS: int + TIOCGSERIAL: int + TIOCGSOFTCAR: int + TIOCINQ: int + TIOCLINUX: int + TIOCMIWAIT: int + TIOCTTYGSTRUCT: int + TIOCSER_TEMT: int + TIOCSERCONFIG: int + TIOCSERGETLSR: int + TIOCSERGETMULTI: int + TIOCSERGSTRUCT: int + TIOCSERGWILD: int + TIOCSERSETMULTI: int + TIOCSERSWILD: int + TIOCSLCKTRMIOS: int + TIOCSSERIAL: int + TIOCSSOFTCAR: int + VSWTC: int + VSWTCH: int + XCASE: int + XTABS: int + + if sys.platform != "darwin": + B1000000: int + B1500000: int + B2000000: int + B2500000: int + B3000000: int + B3500000: int + B4000000: int + B460800: int + B500000: int + B921600: int + + if sys.platform != "linux": + TCSASOFT: int + + if sys.platform != "darwin" and sys.platform != "linux": + # not available on FreeBSD either. + CDEL: int + CEOL2: int + CESC: int + CNUL: int + COMMON: int + CSWTCH: int + IBSHIFT: int + INIT_C_CC: int + NSWTCH: int + + def tcgetattr(fd: FileDescriptorLike, /) -> _AttrReturn: ... + def tcsetattr(fd: FileDescriptorLike, when: int, attributes: _Attr, /) -> None: ... + def tcsendbreak(fd: FileDescriptorLike, duration: int, /) -> None: ... + def tcdrain(fd: FileDescriptorLike, /) -> None: ... + def tcflush(fd: FileDescriptorLike, queue: int, /) -> None: ... + def tcflow(fd: FileDescriptorLike, action: int, /) -> None: ... + if sys.version_info >= (3, 11): + def tcgetwinsize(fd: FileDescriptorLike, /) -> tuple[int, int]: ... + def tcsetwinsize(fd: FileDescriptorLike, winsize: tuple[int, int], /) -> None: ... + + class error(Exception): ... diff --git a/mypy/typeshed/stdlib/textwrap.pyi b/mypy/typeshed/stdlib/textwrap.pyi new file mode 100644 index 000000000000..c00cce3c2d57 --- /dev/null +++ b/mypy/typeshed/stdlib/textwrap.pyi @@ -0,0 +1,103 @@ +from collections.abc import Callable +from re import Pattern + +__all__ = ["TextWrapper", "wrap", "fill", "dedent", "indent", "shorten"] + +class TextWrapper: + width: int + initial_indent: str + subsequent_indent: str + expand_tabs: bool + replace_whitespace: bool + fix_sentence_endings: bool + drop_whitespace: bool + break_long_words: bool + break_on_hyphens: bool + tabsize: int + max_lines: int | None + placeholder: str + + # Attributes not present in documentation + sentence_end_re: Pattern[str] + wordsep_re: Pattern[str] + wordsep_simple_re: Pattern[str] + whitespace_trans: str + unicode_whitespace_trans: dict[int, int] + uspace: int + x: str # leaked loop variable + def __init__( + self, + width: int = 70, + initial_indent: str = "", + subsequent_indent: str = "", + expand_tabs: bool = True, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + drop_whitespace: bool = True, + break_on_hyphens: bool = True, + tabsize: int = 8, + *, + max_lines: int | None = None, + placeholder: str = " [...]", + ) -> None: ... + # Private methods *are* part of the documented API for subclasses. + def _munge_whitespace(self, text: str) -> str: ... + def _split(self, text: str) -> list[str]: ... + def _fix_sentence_endings(self, chunks: list[str]) -> None: ... + def _handle_long_word(self, reversed_chunks: list[str], cur_line: list[str], cur_len: int, width: int) -> None: ... + def _wrap_chunks(self, chunks: list[str]) -> list[str]: ... + def _split_chunks(self, text: str) -> list[str]: ... + def wrap(self, text: str) -> list[str]: ... + def fill(self, text: str) -> str: ... + +def wrap( + text: str, + width: int = 70, + *, + initial_indent: str = "", + subsequent_indent: str = "", + expand_tabs: bool = True, + tabsize: int = 8, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + break_on_hyphens: bool = True, + drop_whitespace: bool = True, + max_lines: int | None = None, + placeholder: str = " [...]", +) -> list[str]: ... +def fill( + text: str, + width: int = 70, + *, + initial_indent: str = "", + subsequent_indent: str = "", + expand_tabs: bool = True, + tabsize: int = 8, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + break_on_hyphens: bool = True, + drop_whitespace: bool = True, + max_lines: int | None = None, + placeholder: str = " [...]", +) -> str: ... +def shorten( + text: str, + width: int, + *, + initial_indent: str = "", + subsequent_indent: str = "", + expand_tabs: bool = True, + tabsize: int = 8, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + break_on_hyphens: bool = True, + drop_whitespace: bool = True, + # Omit `max_lines: int = None`, it is forced to 1 here. + placeholder: str = " [...]", +) -> str: ... +def dedent(text: str) -> str: ... +def indent(text: str, prefix: str, predicate: Callable[[str], bool] | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/this.pyi b/mypy/typeshed/stdlib/this.pyi new file mode 100644 index 000000000000..8de996b04aec --- /dev/null +++ b/mypy/typeshed/stdlib/this.pyi @@ -0,0 +1,2 @@ +s: str +d: dict[str, str] diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi new file mode 100644 index 000000000000..c441a04681e2 --- /dev/null +++ b/mypy/typeshed/stdlib/threading.pyi @@ -0,0 +1,190 @@ +import _thread +import sys +from _thread import _excepthook, _ExceptHookArgs, get_native_id as get_native_id +from _typeshed import ProfileFunction, TraceFunction +from collections.abc import Callable, Iterable, Mapping +from types import TracebackType +from typing import Any, TypeVar, final + +_T = TypeVar("_T") + +__all__ = [ + "get_ident", + "active_count", + "Condition", + "current_thread", + "enumerate", + "main_thread", + "TIMEOUT_MAX", + "Event", + "Lock", + "RLock", + "Semaphore", + "BoundedSemaphore", + "Thread", + "Barrier", + "BrokenBarrierError", + "Timer", + "ThreadError", + "ExceptHookArgs", + "setprofile", + "settrace", + "local", + "stack_size", + "excepthook", + "get_native_id", +] + +if sys.version_info >= (3, 10): + __all__ += ["getprofile", "gettrace"] + +if sys.version_info >= (3, 12): + __all__ += ["setprofile_all_threads", "settrace_all_threads"] + +_profile_hook: ProfileFunction | None + +def active_count() -> int: ... +def activeCount() -> int: ... # deprecated alias for active_count() +def current_thread() -> Thread: ... +def currentThread() -> Thread: ... # deprecated alias for current_thread() +def get_ident() -> int: ... +def enumerate() -> list[Thread]: ... +def main_thread() -> Thread: ... +def settrace(func: TraceFunction) -> None: ... +def setprofile(func: ProfileFunction | None) -> None: ... + +if sys.version_info >= (3, 12): + def setprofile_all_threads(func: ProfileFunction | None) -> None: ... + def settrace_all_threads(func: TraceFunction) -> None: ... + +if sys.version_info >= (3, 10): + def gettrace() -> TraceFunction | None: ... + def getprofile() -> ProfileFunction | None: ... + +def stack_size(size: int = 0, /) -> int: ... + +TIMEOUT_MAX: float + +ThreadError = _thread.error +local = _thread._local + +class Thread: + name: str + @property + def ident(self) -> int | None: ... + daemon: bool + def __init__( + self, + group: None = None, + target: Callable[..., object] | None = None, + name: str | None = None, + args: Iterable[Any] = (), + kwargs: Mapping[str, Any] | None = None, + *, + daemon: bool | None = None, + ) -> None: ... + def start(self) -> None: ... + def run(self) -> None: ... + def join(self, timeout: float | None = None) -> None: ... + @property + def native_id(self) -> int | None: ... # only available on some platforms + def is_alive(self) -> bool: ... + if sys.version_info < (3, 9): + def isAlive(self) -> bool: ... + # the following methods are all deprecated + def getName(self) -> str: ... + def setName(self, name: str) -> None: ... + def isDaemon(self) -> bool: ... + def setDaemon(self, daemonic: bool) -> None: ... + +class _DummyThread(Thread): + def __init__(self) -> None: ... + +@final +class Lock: + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def acquire(self, blocking: bool = ..., timeout: float = ...) -> bool: ... + def release(self) -> None: ... + def locked(self) -> bool: ... + def acquire_lock(self, blocking: bool = ..., timeout: float = ...) -> bool: ... # undocumented + def release_lock(self) -> None: ... # undocumented + def locked_lock(self) -> bool: ... # undocumented + +@final +class _RLock: + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: ... + def release(self) -> None: ... + __enter__ = acquire + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... + +RLock = _RLock + +class Condition: + def __init__(self, lock: Lock | _RLock | None = None) -> None: ... + def __enter__(self) -> bool: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def acquire(self, blocking: bool = ..., timeout: float = ...) -> bool: ... + def release(self) -> None: ... + def wait(self, timeout: float | None = None) -> bool: ... + def wait_for(self, predicate: Callable[[], _T], timeout: float | None = None) -> _T: ... + def notify(self, n: int = 1) -> None: ... + def notify_all(self) -> None: ... + def notifyAll(self) -> None: ... # deprecated alias for notify_all() + +class Semaphore: + _value: int + def __init__(self, value: int = 1) -> None: ... + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... + def acquire(self, blocking: bool = True, timeout: float | None = None) -> bool: ... + def __enter__(self, blocking: bool = True, timeout: float | None = None) -> bool: ... + if sys.version_info >= (3, 9): + def release(self, n: int = 1) -> None: ... + else: + def release(self) -> None: ... + +class BoundedSemaphore(Semaphore): ... + +class Event: + def is_set(self) -> bool: ... + def isSet(self) -> bool: ... # deprecated alias for is_set() + def set(self) -> None: ... + def clear(self) -> None: ... + def wait(self, timeout: float | None = None) -> bool: ... + +excepthook = _excepthook +ExceptHookArgs = _ExceptHookArgs + +class Timer(Thread): + args: Iterable[Any] # undocumented + finished: Event # undocumented + function: Callable[..., Any] # undocumented + interval: float # undocumented + kwargs: Mapping[str, Any] # undocumented + + def __init__( + self, + interval: float, + function: Callable[..., object], + args: Iterable[Any] | None = None, + kwargs: Mapping[str, Any] | None = None, + ) -> None: ... + def cancel(self) -> None: ... + +class Barrier: + @property + def parties(self) -> int: ... + @property + def n_waiting(self) -> int: ... + @property + def broken(self) -> bool: ... + def __init__(self, parties: int, action: Callable[[], None] | None = None, timeout: float | None = None) -> None: ... + def wait(self, timeout: float | None = None) -> int: ... + def reset(self) -> None: ... + def abort(self) -> None: ... + +class BrokenBarrierError(RuntimeError): ... diff --git a/mypy/typeshed/stdlib/time.pyi b/mypy/typeshed/stdlib/time.pyi new file mode 100644 index 000000000000..71cdc4d78fdc --- /dev/null +++ b/mypy/typeshed/stdlib/time.pyi @@ -0,0 +1,111 @@ +import sys +from _typeshed import structseq +from typing import Any, Final, Literal, Protocol, final +from typing_extensions import TypeAlias + +_TimeTuple: TypeAlias = tuple[int, int, int, int, int, int, int, int, int] + +altzone: int +daylight: int +timezone: int +tzname: tuple[str, str] + +if sys.platform == "linux": + CLOCK_BOOTTIME: int +if sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin": + CLOCK_PROF: int # FreeBSD, NetBSD, OpenBSD + CLOCK_UPTIME: int # FreeBSD, OpenBSD + +if sys.platform != "win32": + CLOCK_MONOTONIC: int + CLOCK_MONOTONIC_RAW: int + CLOCK_PROCESS_CPUTIME_ID: int + CLOCK_REALTIME: int + CLOCK_THREAD_CPUTIME_ID: int + if sys.platform != "linux" and sys.platform != "darwin": + CLOCK_HIGHRES: int # Solaris only + +if sys.platform == "darwin": + CLOCK_UPTIME_RAW: int + if sys.version_info >= (3, 13): + CLOCK_UPTIME_RAW_APPROX: int + CLOCK_MONOTONIC_RAW_APPROX: int + +if sys.version_info >= (3, 9) and sys.platform == "linux": + CLOCK_TAI: int + +# Constructor takes an iterable of any type, of length between 9 and 11 elements. +# However, it always *behaves* like a tuple of 9 elements, +# even if an iterable with length >9 is passed. +# https://github.com/python/typeshed/pull/6560#discussion_r767162532 +@final +class struct_time(structseq[Any | int], _TimeTuple): + if sys.version_info >= (3, 10): + __match_args__: Final = ("tm_year", "tm_mon", "tm_mday", "tm_hour", "tm_min", "tm_sec", "tm_wday", "tm_yday", "tm_isdst") + + @property + def tm_year(self) -> int: ... + @property + def tm_mon(self) -> int: ... + @property + def tm_mday(self) -> int: ... + @property + def tm_hour(self) -> int: ... + @property + def tm_min(self) -> int: ... + @property + def tm_sec(self) -> int: ... + @property + def tm_wday(self) -> int: ... + @property + def tm_yday(self) -> int: ... + @property + def tm_isdst(self) -> int: ... + # These final two properties only exist if a 10- or 11-item sequence was passed to the constructor. + @property + def tm_zone(self) -> str: ... + @property + def tm_gmtoff(self) -> int: ... + +def asctime(time_tuple: _TimeTuple | struct_time = ..., /) -> str: ... +def ctime(seconds: float | None = None, /) -> str: ... +def gmtime(seconds: float | None = None, /) -> struct_time: ... +def localtime(seconds: float | None = None, /) -> struct_time: ... +def mktime(time_tuple: _TimeTuple | struct_time, /) -> float: ... +def sleep(seconds: float, /) -> None: ... +def strftime(format: str, time_tuple: _TimeTuple | struct_time = ..., /) -> str: ... +def strptime(data_string: str, format: str = "%a %b %d %H:%M:%S %Y", /) -> struct_time: ... +def time() -> float: ... + +if sys.platform != "win32": + def tzset() -> None: ... # Unix only + +class _ClockInfo(Protocol): + adjustable: bool + implementation: str + monotonic: bool + resolution: float + +def get_clock_info(name: Literal["monotonic", "perf_counter", "process_time", "time", "thread_time"], /) -> _ClockInfo: ... +def monotonic() -> float: ... +def perf_counter() -> float: ... +def process_time() -> float: ... + +if sys.platform != "win32": + def clock_getres(clk_id: int, /) -> float: ... # Unix only + def clock_gettime(clk_id: int, /) -> float: ... # Unix only + def clock_settime(clk_id: int, time: float, /) -> None: ... # Unix only + +if sys.platform != "win32": + def clock_gettime_ns(clk_id: int, /) -> int: ... + def clock_settime_ns(clock_id: int, time: int, /) -> int: ... + +if sys.platform == "linux": + def pthread_getcpuclockid(thread_id: int, /) -> int: ... + +def monotonic_ns() -> int: ... +def perf_counter_ns() -> int: ... +def process_time_ns() -> int: ... +def time_ns() -> int: ... +def thread_time() -> float: ... +def thread_time_ns() -> int: ... diff --git a/mypy/typeshed/stdlib/timeit.pyi b/mypy/typeshed/stdlib/timeit.pyi new file mode 100644 index 000000000000..a5da943c8484 --- /dev/null +++ b/mypy/typeshed/stdlib/timeit.pyi @@ -0,0 +1,32 @@ +from collections.abc import Callable, Sequence +from typing import IO, Any +from typing_extensions import TypeAlias + +__all__ = ["Timer", "timeit", "repeat", "default_timer"] + +_Timer: TypeAlias = Callable[[], float] +_Stmt: TypeAlias = str | Callable[[], object] + +default_timer: _Timer + +class Timer: + def __init__( + self, stmt: _Stmt = "pass", setup: _Stmt = "pass", timer: _Timer = ..., globals: dict[str, Any] | None = None + ) -> None: ... + def print_exc(self, file: IO[str] | None = None) -> None: ... + def timeit(self, number: int = 1000000) -> float: ... + def repeat(self, repeat: int = 5, number: int = 1000000) -> list[float]: ... + def autorange(self, callback: Callable[[int, float], object] | None = None) -> tuple[int, float]: ... + +def timeit( + stmt: _Stmt = "pass", setup: _Stmt = "pass", timer: _Timer = ..., number: int = 1000000, globals: dict[str, Any] | None = None +) -> float: ... +def repeat( + stmt: _Stmt = "pass", + setup: _Stmt = "pass", + timer: _Timer = ..., + repeat: int = 5, + number: int = 1000000, + globals: dict[str, Any] | None = None, +) -> list[float]: ... +def main(args: Sequence[str] | None = None, *, _wrap_timer: Callable[[_Timer], _Timer] | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi new file mode 100644 index 000000000000..d6a234d67919 --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -0,0 +1,3898 @@ +import _tkinter +import sys +from _typeshed import Incomplete, MaybeNone, StrOrBytesPath +from collections.abc import Callable, Iterable, Mapping, Sequence +from tkinter.constants import * +from tkinter.font import _FontDescription +from types import TracebackType +from typing import Any, Generic, Literal, NamedTuple, Protocol, TypedDict, TypeVar, overload, type_check_only +from typing_extensions import TypeAlias, TypeVarTuple, Unpack, deprecated + +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from enum import Enum + +if sys.version_info >= (3, 9): + __all__ = [ + "TclError", + "NO", + "FALSE", + "OFF", + "YES", + "TRUE", + "ON", + "N", + "S", + "W", + "E", + "NW", + "SW", + "NE", + "SE", + "NS", + "EW", + "NSEW", + "CENTER", + "NONE", + "X", + "Y", + "BOTH", + "LEFT", + "TOP", + "RIGHT", + "BOTTOM", + "RAISED", + "SUNKEN", + "FLAT", + "RIDGE", + "GROOVE", + "SOLID", + "HORIZONTAL", + "VERTICAL", + "NUMERIC", + "CHAR", + "WORD", + "BASELINE", + "INSIDE", + "OUTSIDE", + "SEL", + "SEL_FIRST", + "SEL_LAST", + "END", + "INSERT", + "CURRENT", + "ANCHOR", + "ALL", + "NORMAL", + "DISABLED", + "ACTIVE", + "HIDDEN", + "CASCADE", + "CHECKBUTTON", + "COMMAND", + "RADIOBUTTON", + "SEPARATOR", + "SINGLE", + "BROWSE", + "MULTIPLE", + "EXTENDED", + "DOTBOX", + "UNDERLINE", + "PIESLICE", + "CHORD", + "ARC", + "FIRST", + "LAST", + "BUTT", + "PROJECTING", + "ROUND", + "BEVEL", + "MITER", + "MOVETO", + "SCROLL", + "UNITS", + "PAGES", + "TkVersion", + "TclVersion", + "READABLE", + "WRITABLE", + "EXCEPTION", + "EventType", + "Event", + "NoDefaultRoot", + "Variable", + "StringVar", + "IntVar", + "DoubleVar", + "BooleanVar", + "mainloop", + "getint", + "getdouble", + "getboolean", + "Misc", + "CallWrapper", + "XView", + "YView", + "Wm", + "Tk", + "Tcl", + "Pack", + "Place", + "Grid", + "BaseWidget", + "Widget", + "Toplevel", + "Button", + "Canvas", + "Checkbutton", + "Entry", + "Frame", + "Label", + "Listbox", + "Menu", + "Menubutton", + "Message", + "Radiobutton", + "Scale", + "Scrollbar", + "Text", + "OptionMenu", + "Image", + "PhotoImage", + "BitmapImage", + "image_names", + "image_types", + "Spinbox", + "LabelFrame", + "PanedWindow", + ] + +# Using anything from tkinter.font in this file means that 'import tkinter' +# seems to also load tkinter.font. That's not how it actually works, but +# unfortunately not much can be done about it. https://github.com/python/typeshed/pull/4346 + +TclError = _tkinter.TclError +wantobjects: int +TkVersion: float +TclVersion: float +READABLE = _tkinter.READABLE +WRITABLE = _tkinter.WRITABLE +EXCEPTION = _tkinter.EXCEPTION + +# Quick guide for figuring out which widget class to choose: +# - Misc: any widget (don't use BaseWidget because Tk doesn't inherit from BaseWidget) +# - Widget: anything that is meant to be put into another widget with e.g. pack or grid +# +# Don't trust tkinter's docstrings, because they have been created by copy/pasting from +# Tk's manual pages more than 10 years ago. Use the latest manual pages instead: +# +# $ sudo apt install tk-doc tcl-doc +# $ man 3tk label # tkinter.Label +# $ man 3tk ttk_label # tkinter.ttk.Label +# $ man 3tcl after # tkinter.Misc.after +# +# You can also read the manual pages online: https://www.tcl.tk/doc/ + +# Some widgets have an option named -compound that accepts different values +# than the _Compound defined here. Many other options have similar things. +_Anchor: TypeAlias = Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] # manual page: Tk_GetAnchor +_ButtonCommand: TypeAlias = str | Callable[[], Any] # accepts string of tcl code, return value is returned from Button.invoke() +_Compound: TypeAlias = Literal["top", "left", "center", "right", "bottom", "none"] # -compound in manual page named 'options' +# manual page: Tk_GetCursor +_Cursor: TypeAlias = str | tuple[str] | tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str] +# example when it's sequence: entry['invalidcommand'] = [entry.register(print), '%P'] +_EntryValidateCommand: TypeAlias = str | list[str] | tuple[str, ...] | Callable[[], bool] +_ImageSpec: TypeAlias = _Image | str # str can be from e.g. tkinter.image_names() +_Relief: TypeAlias = Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] # manual page: Tk_GetRelief +_ScreenUnits: TypeAlias = str | float # Often the right type instead of int. Manual page: Tk_GetPixels +# -xscrollcommand and -yscrollcommand in 'options' manual page +_XYScrollCommand: TypeAlias = str | Callable[[float, float], object] +_TakeFocusValue: TypeAlias = bool | Literal[0, 1, ""] | Callable[[str], bool | None] # -takefocus in manual page named 'options' + +if sys.version_info >= (3, 11): + @type_check_only + class _VersionInfoTypeBase(NamedTuple): + major: int + minor: int + micro: int + releaselevel: str + serial: int + + class _VersionInfoType(_VersionInfoTypeBase): ... + +if sys.version_info >= (3, 11): + class EventType(StrEnum): + Activate = "36" + ButtonPress = "4" + Button = ButtonPress + ButtonRelease = "5" + Circulate = "26" + CirculateRequest = "27" + ClientMessage = "33" + Colormap = "32" + Configure = "22" + ConfigureRequest = "23" + Create = "16" + Deactivate = "37" + Destroy = "17" + Enter = "7" + Expose = "12" + FocusIn = "9" + FocusOut = "10" + GraphicsExpose = "13" + Gravity = "24" + KeyPress = "2" + Key = "2" + KeyRelease = "3" + Keymap = "11" + Leave = "8" + Map = "19" + MapRequest = "20" + Mapping = "34" + Motion = "6" + MouseWheel = "38" + NoExpose = "14" + Property = "28" + Reparent = "21" + ResizeRequest = "25" + Selection = "31" + SelectionClear = "29" + SelectionRequest = "30" + Unmap = "18" + VirtualEvent = "35" + Visibility = "15" + +else: + class EventType(str, Enum): + Activate = "36" + ButtonPress = "4" + Button = ButtonPress + ButtonRelease = "5" + Circulate = "26" + CirculateRequest = "27" + ClientMessage = "33" + Colormap = "32" + Configure = "22" + ConfigureRequest = "23" + Create = "16" + Deactivate = "37" + Destroy = "17" + Enter = "7" + Expose = "12" + FocusIn = "9" + FocusOut = "10" + GraphicsExpose = "13" + Gravity = "24" + KeyPress = "2" + Key = "2" + KeyRelease = "3" + Keymap = "11" + Leave = "8" + Map = "19" + MapRequest = "20" + Mapping = "34" + Motion = "6" + MouseWheel = "38" + NoExpose = "14" + Property = "28" + Reparent = "21" + ResizeRequest = "25" + Selection = "31" + SelectionClear = "29" + SelectionRequest = "30" + Unmap = "18" + VirtualEvent = "35" + Visibility = "15" + +_W = TypeVar("_W", bound=Misc) +# Events considered covariant because you should never assign to event.widget. +_W_co = TypeVar("_W_co", covariant=True, bound=Misc) + +class Event(Generic[_W_co]): + serial: int + num: int + focus: bool + height: int + width: int + keycode: int + state: int | str + time: int + x: int + y: int + x_root: int + y_root: int + char: str + send_event: bool + keysym: str + keysym_num: int + type: EventType + widget: _W_co + delta: int + +def NoDefaultRoot() -> None: ... + +class Variable: + def __init__(self, master: Misc | None = None, value: Incomplete | None = None, name: str | None = None) -> None: ... + def set(self, value) -> None: ... + initialize = set + def get(self): ... + def trace_add(self, mode: Literal["array", "read", "write", "unset"], callback: Callable[[str, str, str], object]) -> str: ... + def trace_remove(self, mode: Literal["array", "read", "write", "unset"], cbname: str) -> None: ... + def trace_info(self) -> list[tuple[tuple[Literal["array", "read", "write", "unset"], ...], str]]: ... + @deprecated("use trace_add() instead of trace()") + def trace(self, mode, callback): ... + @deprecated("use trace_add() instead of trace_variable()") + def trace_variable(self, mode, callback): ... + @deprecated("use trace_remove() instead of trace_vdelete()") + def trace_vdelete(self, mode, cbname) -> None: ... + @deprecated("use trace_info() instead of trace_vinfo()") + def trace_vinfo(self): ... + def __eq__(self, other: object) -> bool: ... + def __del__(self) -> None: ... + +class StringVar(Variable): + def __init__(self, master: Misc | None = None, value: str | None = None, name: str | None = None) -> None: ... + def set(self, value: str) -> None: ... + initialize = set + def get(self) -> str: ... + +class IntVar(Variable): + def __init__(self, master: Misc | None = None, value: int | None = None, name: str | None = None) -> None: ... + def set(self, value: int) -> None: ... + initialize = set + def get(self) -> int: ... + +class DoubleVar(Variable): + def __init__(self, master: Misc | None = None, value: float | None = None, name: str | None = None) -> None: ... + def set(self, value: float) -> None: ... + initialize = set + def get(self) -> float: ... + +class BooleanVar(Variable): + def __init__(self, master: Misc | None = None, value: bool | None = None, name: str | None = None) -> None: ... + def set(self, value: bool) -> None: ... + initialize = set + def get(self) -> bool: ... + +def mainloop(n: int = 0) -> None: ... + +getint: Incomplete +getdouble: Incomplete + +def getboolean(s): ... + +_Ts = TypeVarTuple("_Ts") + +class _GridIndexInfo(TypedDict, total=False): + minsize: _ScreenUnits + pad: _ScreenUnits + uniform: str | None + weight: int + +class Misc: + master: Misc | None + tk: _tkinter.TkappType + children: dict[str, Widget] + def destroy(self) -> None: ... + def deletecommand(self, name: str) -> None: ... + def tk_strictMotif(self, boolean: Incomplete | None = None): ... + def tk_bisque(self) -> None: ... + def tk_setPalette(self, *args, **kw) -> None: ... + def wait_variable(self, name: str | Variable = "PY_VAR") -> None: ... + waitvar = wait_variable + def wait_window(self, window: Misc | None = None) -> None: ... + def wait_visibility(self, window: Misc | None = None) -> None: ... + def setvar(self, name: str = "PY_VAR", value: str = "1") -> None: ... + def getvar(self, name: str = "PY_VAR"): ... + def getint(self, s): ... + def getdouble(self, s): ... + def getboolean(self, s): ... + def focus_set(self) -> None: ... + focus = focus_set + def focus_force(self) -> None: ... + def focus_get(self) -> Misc | None: ... + def focus_displayof(self) -> Misc | None: ... + def focus_lastfor(self) -> Misc | None: ... + def tk_focusFollowsMouse(self) -> None: ... + def tk_focusNext(self) -> Misc | None: ... + def tk_focusPrev(self) -> Misc | None: ... + # .after() can be called without the "func" argument, but it is basically never what you want. + # It behaves like time.sleep() and freezes the GUI app. + def after(self, ms: int | Literal["idle"], func: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> str: ... + # after_idle is essentially partialmethod(after, "idle") + def after_idle(self, func: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> str: ... + def after_cancel(self, id: str) -> None: ... + def bell(self, displayof: Literal[0] | Misc | None = 0) -> None: ... + def clipboard_get(self, *, displayof: Misc = ..., type: str = ...) -> str: ... + def clipboard_clear(self, *, displayof: Misc = ...) -> None: ... + def clipboard_append(self, string: str, *, displayof: Misc = ..., format: str = ..., type: str = ...) -> None: ... + def grab_current(self): ... + def grab_release(self) -> None: ... + def grab_set(self) -> None: ... + def grab_set_global(self) -> None: ... + def grab_status(self) -> Literal["local", "global"] | None: ... + def option_add( + self, pattern, value, priority: int | Literal["widgetDefault", "startupFile", "userDefault", "interactive"] | None = None + ) -> None: ... + def option_clear(self) -> None: ... + def option_get(self, name, className): ... + def option_readfile(self, fileName, priority: Incomplete | None = None) -> None: ... + def selection_clear(self, **kw) -> None: ... + def selection_get(self, **kw): ... + def selection_handle(self, command, **kw) -> None: ... + def selection_own(self, **kw) -> None: ... + def selection_own_get(self, **kw): ... + def send(self, interp, cmd, *args): ... + def lower(self, belowThis: Incomplete | None = None) -> None: ... + def tkraise(self, aboveThis: Incomplete | None = None) -> None: ... + lift = tkraise + if sys.version_info >= (3, 11): + def info_patchlevel(self) -> _VersionInfoType: ... + + def winfo_atom(self, name: str, displayof: Literal[0] | Misc | None = 0) -> int: ... + def winfo_atomname(self, id: int, displayof: Literal[0] | Misc | None = 0) -> str: ... + def winfo_cells(self) -> int: ... + def winfo_children(self) -> list[Widget]: ... # Widget because it can't be Toplevel or Tk + def winfo_class(self) -> str: ... + def winfo_colormapfull(self) -> bool: ... + def winfo_containing(self, rootX: int, rootY: int, displayof: Literal[0] | Misc | None = 0) -> Misc | None: ... + def winfo_depth(self) -> int: ... + def winfo_exists(self) -> bool: ... + def winfo_fpixels(self, number: _ScreenUnits) -> float: ... + def winfo_geometry(self) -> str: ... + def winfo_height(self) -> int: ... + def winfo_id(self) -> int: ... + def winfo_interps(self, displayof: Literal[0] | Misc | None = 0) -> tuple[str, ...]: ... + def winfo_ismapped(self) -> bool: ... + def winfo_manager(self) -> str: ... + def winfo_name(self) -> str: ... + def winfo_parent(self) -> str: ... # return value needs nametowidget() + def winfo_pathname(self, id: int, displayof: Literal[0] | Misc | None = 0): ... + def winfo_pixels(self, number: _ScreenUnits) -> int: ... + def winfo_pointerx(self) -> int: ... + def winfo_pointerxy(self) -> tuple[int, int]: ... + def winfo_pointery(self) -> int: ... + def winfo_reqheight(self) -> int: ... + def winfo_reqwidth(self) -> int: ... + def winfo_rgb(self, color: str) -> tuple[int, int, int]: ... + def winfo_rootx(self) -> int: ... + def winfo_rooty(self) -> int: ... + def winfo_screen(self) -> str: ... + def winfo_screencells(self) -> int: ... + def winfo_screendepth(self) -> int: ... + def winfo_screenheight(self) -> int: ... + def winfo_screenmmheight(self) -> int: ... + def winfo_screenmmwidth(self) -> int: ... + def winfo_screenvisual(self) -> str: ... + def winfo_screenwidth(self) -> int: ... + def winfo_server(self) -> str: ... + def winfo_toplevel(self) -> Tk | Toplevel: ... + def winfo_viewable(self) -> bool: ... + def winfo_visual(self) -> str: ... + def winfo_visualid(self) -> str: ... + def winfo_visualsavailable(self, includeids: bool = False) -> list[tuple[str, int]]: ... + def winfo_vrootheight(self) -> int: ... + def winfo_vrootwidth(self) -> int: ... + def winfo_vrootx(self) -> int: ... + def winfo_vrooty(self) -> int: ... + def winfo_width(self) -> int: ... + def winfo_x(self) -> int: ... + def winfo_y(self) -> int: ... + def update(self) -> None: ... + def update_idletasks(self) -> None: ... + @overload + def bindtags(self, tagList: None = None) -> tuple[str, ...]: ... + @overload + def bindtags(self, tagList: list[str] | tuple[str, ...]) -> None: ... + # bind with isinstance(func, str) doesn't return anything, but all other + # binds do. The default value of func is not str. + @overload + def bind( + self, + sequence: str | None = None, + func: Callable[[Event[Misc]], object] | None = None, + add: Literal["", "+"] | bool | None = None, + ) -> str: ... + @overload + def bind(self, sequence: str | None, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + @overload + def bind(self, *, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + # There's no way to know what type of widget bind_all and bind_class + # callbacks will get, so those are Misc. + @overload + def bind_all( + self, + sequence: str | None = None, + func: Callable[[Event[Misc]], object] | None = None, + add: Literal["", "+"] | bool | None = None, + ) -> str: ... + @overload + def bind_all(self, sequence: str | None, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + @overload + def bind_all(self, *, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + @overload + def bind_class( + self, + className: str, + sequence: str | None = None, + func: Callable[[Event[Misc]], object] | None = None, + add: Literal["", "+"] | bool | None = None, + ) -> str: ... + @overload + def bind_class(self, className: str, sequence: str | None, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + @overload + def bind_class(self, className: str, *, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + def unbind(self, sequence: str, funcid: str | None = None) -> None: ... + def unbind_all(self, sequence: str) -> None: ... + def unbind_class(self, className: str, sequence: str) -> None: ... + def mainloop(self, n: int = 0) -> None: ... + def quit(self) -> None: ... + @property + def _windowingsystem(self) -> Literal["win32", "aqua", "x11"]: ... + def nametowidget(self, name: str | Misc | _tkinter.Tcl_Obj) -> Any: ... + def register( + self, func: Callable[..., object], subst: Callable[..., Sequence[Any]] | None = None, needcleanup: int = 1 + ) -> str: ... + def keys(self) -> list[str]: ... + @overload + def pack_propagate(self, flag: bool) -> bool | None: ... + @overload + def pack_propagate(self) -> None: ... + propagate = pack_propagate + def grid_anchor(self, anchor: _Anchor | None = None) -> None: ... + anchor = grid_anchor + @overload + def grid_bbox( + self, column: None = None, row: None = None, col2: None = None, row2: None = None + ) -> tuple[int, int, int, int] | None: ... + @overload + def grid_bbox(self, column: int, row: int, col2: None = None, row2: None = None) -> tuple[int, int, int, int] | None: ... + @overload + def grid_bbox(self, column: int, row: int, col2: int, row2: int) -> tuple[int, int, int, int] | None: ... + bbox = grid_bbox + def grid_columnconfigure( + self, + index: int | str | list[int] | tuple[int, ...], + cnf: _GridIndexInfo = {}, + *, + minsize: _ScreenUnits = ..., + pad: _ScreenUnits = ..., + uniform: str = ..., + weight: int = ..., + ) -> _GridIndexInfo | MaybeNone: ... # can be None but annoying to check + def grid_rowconfigure( + self, + index: int | str | list[int] | tuple[int, ...], + cnf: _GridIndexInfo = {}, + *, + minsize: _ScreenUnits = ..., + pad: _ScreenUnits = ..., + uniform: str = ..., + weight: int = ..., + ) -> _GridIndexInfo | MaybeNone: ... # can be None but annoying to check + columnconfigure = grid_columnconfigure + rowconfigure = grid_rowconfigure + def grid_location(self, x: _ScreenUnits, y: _ScreenUnits) -> tuple[int, int]: ... + @overload + def grid_propagate(self, flag: bool) -> None: ... + @overload + def grid_propagate(self) -> bool: ... + def grid_size(self) -> tuple[int, int]: ... + size = grid_size + # Widget because Toplevel or Tk is never a slave + def pack_slaves(self) -> list[Widget]: ... + def grid_slaves(self, row: int | None = None, column: int | None = None) -> list[Widget]: ... + def place_slaves(self) -> list[Widget]: ... + slaves = pack_slaves + def event_add(self, virtual: str, *sequences: str) -> None: ... + def event_delete(self, virtual: str, *sequences: str) -> None: ... + def event_generate( + self, + sequence: str, + *, + above: Misc | int = ..., + borderwidth: _ScreenUnits = ..., + button: int = ..., + count: int = ..., + data: Any = ..., # anything with usable str() value + delta: int = ..., + detail: str = ..., + focus: bool = ..., + height: _ScreenUnits = ..., + keycode: int = ..., + keysym: str = ..., + mode: str = ..., + override: bool = ..., + place: Literal["PlaceOnTop", "PlaceOnBottom"] = ..., + root: Misc | int = ..., + rootx: _ScreenUnits = ..., + rooty: _ScreenUnits = ..., + sendevent: bool = ..., + serial: int = ..., + state: int | str = ..., + subwindow: Misc | int = ..., + time: int = ..., + warp: bool = ..., + width: _ScreenUnits = ..., + when: Literal["now", "tail", "head", "mark"] = ..., + x: _ScreenUnits = ..., + y: _ScreenUnits = ..., + ) -> None: ... + def event_info(self, virtual: str | None = None) -> tuple[str, ...]: ... + def image_names(self) -> tuple[str, ...]: ... + def image_types(self) -> tuple[str, ...]: ... + # See #4363 and #4891 + def __setitem__(self, key: str, value: Any) -> None: ... + def __getitem__(self, key: str) -> Any: ... + def cget(self, key: str) -> Any: ... + def configure(self, cnf: Any = None) -> Any: ... + # TODO: config is an alias of configure, but adding that here creates + # conflict with the type of config in the subclasses. See #13149 + +class CallWrapper: + func: Incomplete + subst: Incomplete + widget: Incomplete + def __init__(self, func, subst, widget) -> None: ... + def __call__(self, *args): ... + +class XView: + @overload + def xview(self) -> tuple[float, float]: ... + @overload + def xview(self, *args): ... + def xview_moveto(self, fraction: float) -> None: ... + @overload + def xview_scroll(self, number: int, what: Literal["units", "pages"]) -> None: ... + @overload + def xview_scroll(self, number: _ScreenUnits, what: Literal["pixels"]) -> None: ... + +class YView: + @overload + def yview(self) -> tuple[float, float]: ... + @overload + def yview(self, *args): ... + def yview_moveto(self, fraction: float) -> None: ... + @overload + def yview_scroll(self, number: int, what: Literal["units", "pages"]) -> None: ... + @overload + def yview_scroll(self, number: _ScreenUnits, what: Literal["pixels"]) -> None: ... + +class Wm: + @overload + def wm_aspect(self, minNumer: int, minDenom: int, maxNumer: int, maxDenom: int) -> None: ... + @overload + def wm_aspect( + self, minNumer: None = None, minDenom: None = None, maxNumer: None = None, maxDenom: None = None + ) -> tuple[int, int, int, int] | None: ... + aspect = wm_aspect + @overload + def wm_attributes(self) -> tuple[Any, ...]: ... + @overload + def wm_attributes(self, option: str, /): ... + @overload + def wm_attributes(self, option: str, value, /, *__other_option_value_pairs: Any) -> None: ... + attributes = wm_attributes + def wm_client(self, name: str | None = None) -> str: ... + client = wm_client + @overload + def wm_colormapwindows(self) -> list[Misc]: ... + @overload + def wm_colormapwindows(self, wlist: list[Misc] | tuple[Misc, ...], /) -> None: ... + @overload + def wm_colormapwindows(self, first_wlist_item: Misc, /, *other_wlist_items: Misc) -> None: ... + colormapwindows = wm_colormapwindows + def wm_command(self, value: str | None = None) -> str: ... + command = wm_command + # Some of these always return empty string, but return type is set to None to prevent accidentally using it + def wm_deiconify(self) -> None: ... + deiconify = wm_deiconify + def wm_focusmodel(self, model: Literal["active", "passive"] | None = None) -> Literal["active", "passive", ""]: ... + focusmodel = wm_focusmodel + def wm_forget(self, window: Wm) -> None: ... + forget = wm_forget + def wm_frame(self) -> str: ... + frame = wm_frame + @overload + def wm_geometry(self, newGeometry: None = None) -> str: ... + @overload + def wm_geometry(self, newGeometry: str) -> None: ... + geometry = wm_geometry + def wm_grid( + self, + baseWidth: Incomplete | None = None, + baseHeight: Incomplete | None = None, + widthInc: Incomplete | None = None, + heightInc: Incomplete | None = None, + ): ... + grid = wm_grid + def wm_group(self, pathName: Incomplete | None = None): ... + group = wm_group + def wm_iconbitmap(self, bitmap: Incomplete | None = None, default: Incomplete | None = None): ... + iconbitmap = wm_iconbitmap + def wm_iconify(self) -> None: ... + iconify = wm_iconify + def wm_iconmask(self, bitmap: Incomplete | None = None): ... + iconmask = wm_iconmask + def wm_iconname(self, newName: Incomplete | None = None) -> str: ... + iconname = wm_iconname + def wm_iconphoto(self, default: bool, image1: _PhotoImageLike | str, /, *args: _PhotoImageLike | str) -> None: ... + iconphoto = wm_iconphoto + def wm_iconposition(self, x: int | None = None, y: int | None = None) -> tuple[int, int] | None: ... + iconposition = wm_iconposition + def wm_iconwindow(self, pathName: Incomplete | None = None): ... + iconwindow = wm_iconwindow + def wm_manage(self, widget) -> None: ... + manage = wm_manage + @overload + def wm_maxsize(self, width: None = None, height: None = None) -> tuple[int, int]: ... + @overload + def wm_maxsize(self, width: int, height: int) -> None: ... + maxsize = wm_maxsize + @overload + def wm_minsize(self, width: None = None, height: None = None) -> tuple[int, int]: ... + @overload + def wm_minsize(self, width: int, height: int) -> None: ... + minsize = wm_minsize + @overload + def wm_overrideredirect(self, boolean: None = None) -> bool | None: ... # returns True or None + @overload + def wm_overrideredirect(self, boolean: bool) -> None: ... + overrideredirect = wm_overrideredirect + def wm_positionfrom(self, who: Literal["program", "user"] | None = None) -> Literal["", "program", "user"]: ... + positionfrom = wm_positionfrom + @overload + def wm_protocol(self, name: str, func: Callable[[], object] | str) -> None: ... + @overload + def wm_protocol(self, name: str, func: None = None) -> str: ... + @overload + def wm_protocol(self, name: None = None, func: None = None) -> tuple[str, ...]: ... + protocol = wm_protocol + @overload + def wm_resizable(self, width: None = None, height: None = None) -> tuple[bool, bool]: ... + @overload + def wm_resizable(self, width: bool, height: bool) -> None: ... + resizable = wm_resizable + def wm_sizefrom(self, who: Literal["program", "user"] | None = None) -> Literal["", "program", "user"]: ... + sizefrom = wm_sizefrom + @overload + def wm_state(self, newstate: None = None) -> str: ... + @overload + def wm_state(self, newstate: str) -> None: ... + state = wm_state + @overload + def wm_title(self, string: None = None) -> str: ... + @overload + def wm_title(self, string: str) -> None: ... + title = wm_title + @overload + def wm_transient(self, master: None = None) -> _tkinter.Tcl_Obj: ... + @overload + def wm_transient(self, master: Wm | _tkinter.Tcl_Obj) -> None: ... + transient = wm_transient + def wm_withdraw(self) -> None: ... + withdraw = wm_withdraw + +class Tk(Misc, Wm): + master: None + def __init__( + # Make sure to keep in sync with other functions that use the same + # args. + # use `git grep screenName` to find them + self, + screenName: str | None = None, + baseName: str | None = None, + className: str = "Tk", + useTk: bool = True, + sync: bool = False, + use: str | None = None, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + menu: Menu = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + takefocus: _TakeFocusValue = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def destroy(self) -> None: ... + def readprofile(self, baseName: str, className: str) -> None: ... + report_callback_exception: Callable[[type[BaseException], BaseException, TracebackType | None], object] + # Tk has __getattr__ so that tk_instance.foo falls back to tk_instance.tk.foo + # Please keep in sync with _tkinter.TkappType. + # Some methods are intentionally missing because they are inherited from Misc instead. + def adderrorinfo(self, msg, /): ... + def call(self, command: Any, /, *args: Any) -> Any: ... + def createcommand(self, name, func, /): ... + if sys.platform != "win32": + def createfilehandler(self, file, mask, func, /): ... + def deletefilehandler(self, file, /): ... + + def createtimerhandler(self, milliseconds, func, /): ... + def dooneevent(self, flags: int = ..., /): ... + def eval(self, script: str, /) -> str: ... + def evalfile(self, fileName, /): ... + def exprboolean(self, s, /): ... + def exprdouble(self, s, /): ... + def exprlong(self, s, /): ... + def exprstring(self, s, /): ... + def globalgetvar(self, *args, **kwargs): ... + def globalsetvar(self, *args, **kwargs): ... + def globalunsetvar(self, *args, **kwargs): ... + def interpaddr(self): ... + def loadtk(self) -> None: ... + def record(self, script, /): ... + if sys.version_info < (3, 11): + def split(self, arg, /): ... + + def splitlist(self, arg, /): ... + def unsetvar(self, *args, **kwargs): ... + def wantobjects(self, *args, **kwargs): ... + def willdispatch(self): ... + +def Tcl(screenName: str | None = None, baseName: str | None = None, className: str = "Tk", useTk: bool = False) -> Tk: ... + +_InMiscTotal = TypedDict("_InMiscTotal", {"in": Misc}) +_InMiscNonTotal = TypedDict("_InMiscNonTotal", {"in": Misc}, total=False) + +class _PackInfo(_InMiscTotal): + # 'before' and 'after' never appear in _PackInfo + anchor: _Anchor + expand: bool + fill: Literal["none", "x", "y", "both"] + side: Literal["left", "right", "top", "bottom"] + # Paddings come out as int or tuple of int, even though any _ScreenUnits + # can be specified in pack(). + ipadx: int + ipady: int + padx: int | tuple[int, int] + pady: int | tuple[int, int] + +class Pack: + # _PackInfo is not the valid type for cnf because pad stuff accepts any + # _ScreenUnits instead of int only. I didn't bother to create another + # TypedDict for cnf because it appears to be a legacy thing that was + # replaced by **kwargs. + def pack_configure( + self, + cnf: Mapping[str, Any] | None = {}, + *, + after: Misc = ..., + anchor: _Anchor = ..., + before: Misc = ..., + expand: bool | Literal[0, 1] = 0, + fill: Literal["none", "x", "y", "both"] = ..., + side: Literal["left", "right", "top", "bottom"] = ..., + ipadx: _ScreenUnits = ..., + ipady: _ScreenUnits = ..., + padx: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + pady: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + in_: Misc = ..., + **kw: Any, # allow keyword argument named 'in', see #4836 + ) -> None: ... + def pack_forget(self) -> None: ... + def pack_info(self) -> _PackInfo: ... # errors if widget hasn't been packed + pack = pack_configure + forget = pack_forget + propagate = Misc.pack_propagate + +class _PlaceInfo(_InMiscNonTotal): # empty dict if widget hasn't been placed + anchor: _Anchor + bordermode: Literal["inside", "outside", "ignore"] + width: str # can be int()ed (even after e.g. widget.place(height='2.3c') or similar) + height: str # can be int()ed + x: str # can be int()ed + y: str # can be int()ed + relheight: str # can be float()ed if not empty string + relwidth: str # can be float()ed if not empty string + relx: str # can be float()ed if not empty string + rely: str # can be float()ed if not empty string + +class Place: + def place_configure( + self, + cnf: Mapping[str, Any] | None = {}, + *, + anchor: _Anchor = ..., + bordermode: Literal["inside", "outside", "ignore"] = ..., + width: _ScreenUnits = ..., + height: _ScreenUnits = ..., + x: _ScreenUnits = ..., + y: _ScreenUnits = ..., + # str allowed for compatibility with place_info() + relheight: str | float = ..., + relwidth: str | float = ..., + relx: str | float = ..., + rely: str | float = ..., + in_: Misc = ..., + **kw: Any, # allow keyword argument named 'in', see #4836 + ) -> None: ... + def place_forget(self) -> None: ... + def place_info(self) -> _PlaceInfo: ... + place = place_configure + info = place_info + +class _GridInfo(_InMiscNonTotal): # empty dict if widget hasn't been gridded + column: int + columnspan: int + row: int + rowspan: int + ipadx: int + ipady: int + padx: int | tuple[int, int] + pady: int | tuple[int, int] + sticky: str # consists of letters 'n', 's', 'w', 'e', no repeats, may be empty + +class Grid: + def grid_configure( + self, + cnf: Mapping[str, Any] | None = {}, + *, + column: int = ..., + columnspan: int = ..., + row: int = ..., + rowspan: int = ..., + ipadx: _ScreenUnits = ..., + ipady: _ScreenUnits = ..., + padx: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + pady: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + sticky: str = ..., # consists of letters 'n', 's', 'w', 'e', may contain repeats, may be empty + in_: Misc = ..., + **kw: Any, # allow keyword argument named 'in', see #4836 + ) -> None: ... + def grid_forget(self) -> None: ... + def grid_remove(self) -> None: ... + def grid_info(self) -> _GridInfo: ... + grid = grid_configure + location = Misc.grid_location + size = Misc.grid_size + +class BaseWidget(Misc): + master: Misc + widgetName: Incomplete + def __init__(self, master, widgetName, cnf={}, kw={}, extra=()) -> None: ... + def destroy(self) -> None: ... + +# This class represents any widget except Toplevel or Tk. +class Widget(BaseWidget, Pack, Place, Grid): + # Allow bind callbacks to take e.g. Event[Label] instead of Event[Misc]. + # Tk and Toplevel get notified for their child widgets' events, but other + # widgets don't. + @overload + def bind( + self: _W, + sequence: str | None = None, + func: Callable[[Event[_W]], object] | None = None, + add: Literal["", "+"] | bool | None = None, + ) -> str: ... + @overload + def bind(self, sequence: str | None, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + @overload + def bind(self, *, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + +class Toplevel(BaseWidget, Wm): + # Toplevel and Tk have the same options because they correspond to the same + # Tcl/Tk toplevel widget. For some reason, config and configure must be + # copy/pasted here instead of aliasing as 'config = Tk.config'. + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + background: str = ..., + bd: _ScreenUnits = 0, + bg: str = ..., + border: _ScreenUnits = 0, + borderwidth: _ScreenUnits = 0, + class_: str = "Toplevel", + colormap: Literal["new", ""] | Misc = "", + container: bool = False, + cursor: _Cursor = "", + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 0, + menu: Menu = ..., + name: str = ..., + padx: _ScreenUnits = 0, + pady: _ScreenUnits = 0, + relief: _Relief = "flat", + screen: str = "", # can't be changed after creating widget + takefocus: _TakeFocusValue = 0, + use: int = ..., + visual: str | tuple[str, int] = "", + width: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + menu: Menu = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + takefocus: _TakeFocusValue = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Button(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = "center", + background: str = ..., + bd: _ScreenUnits = ..., # same as borderwidth + bg: str = ..., # same as background + bitmap: str = "", + border: _ScreenUnits = ..., # same as borderwidth + borderwidth: _ScreenUnits = ..., + command: _ButtonCommand = "", + compound: _Compound = "none", + cursor: _Cursor = "", + default: Literal["normal", "active", "disabled"] = "disabled", + disabledforeground: str = ..., + fg: str = ..., # same as foreground + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + # width and height must be int for buttons containing just text, but + # ints are also valid _ScreenUnits + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 1, + image: _ImageSpec = "", + justify: Literal["left", "center", "right"] = "center", + name: str = ..., + overrelief: _Relief | Literal[""] = "", + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + repeatdelay: int = ..., + repeatinterval: int = ..., + state: Literal["normal", "active", "disabled"] = "normal", + takefocus: _TakeFocusValue = "", + text: float | str = "", + # We allow the textvariable to be any Variable, not necessarily + # StringVar. This is useful for e.g. a button that displays the value + # of an IntVar. + textvariable: Variable = ..., + underline: int = -1, + width: _ScreenUnits = 0, + wraplength: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + command: _ButtonCommand = ..., + compound: _Compound = ..., + cursor: _Cursor = ..., + default: Literal["normal", "active", "disabled"] = ..., + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + image: _ImageSpec = ..., + justify: Literal["left", "center", "right"] = ..., + overrelief: _Relief | Literal[""] = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + repeatdelay: int = ..., + repeatinterval: int = ..., + state: Literal["normal", "active", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + text: float | str = ..., + textvariable: Variable = ..., + underline: int = ..., + width: _ScreenUnits = ..., + wraplength: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def flash(self) -> None: ... + def invoke(self) -> Any: ... + +class Canvas(Widget, XView, YView): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + background: str = ..., + bd: _ScreenUnits = 0, + bg: str = ..., + border: _ScreenUnits = 0, + borderwidth: _ScreenUnits = 0, + closeenough: float = 1.0, + confine: bool = True, + cursor: _Cursor = "", + # canvas manual page has a section named COORDINATES, and the first + # part of it describes _ScreenUnits. + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = 0, + insertofftime: int = 300, + insertontime: int = 600, + insertwidth: _ScreenUnits = 2, + name: str = ..., + offset=..., # undocumented + relief: _Relief = "flat", + # Setting scrollregion to None doesn't reset it back to empty, + # but setting it to () does. + scrollregion: tuple[_ScreenUnits, _ScreenUnits, _ScreenUnits, _ScreenUnits] | tuple[()] = (), + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = 1, + selectforeground: str = ..., + # man page says that state can be 'hidden', but it can't + state: Literal["normal", "disabled"] = "normal", + takefocus: _TakeFocusValue = "", + width: _ScreenUnits = ..., + xscrollcommand: _XYScrollCommand = "", + xscrollincrement: _ScreenUnits = 0, + yscrollcommand: _XYScrollCommand = "", + yscrollincrement: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + closeenough: float = ..., + confine: bool = ..., + cursor: _Cursor = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = ..., + insertofftime: int = ..., + insertontime: int = ..., + insertwidth: _ScreenUnits = ..., + offset=..., # undocumented + relief: _Relief = ..., + scrollregion: tuple[_ScreenUnits, _ScreenUnits, _ScreenUnits, _ScreenUnits] | tuple[()] = ..., + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + state: Literal["normal", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + width: _ScreenUnits = ..., + xscrollcommand: _XYScrollCommand = ..., + xscrollincrement: _ScreenUnits = ..., + yscrollcommand: _XYScrollCommand = ..., + yscrollincrement: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def addtag(self, *args): ... # internal method + def addtag_above(self, newtag: str, tagOrId: str | int) -> None: ... + def addtag_all(self, newtag: str) -> None: ... + def addtag_below(self, newtag: str, tagOrId: str | int) -> None: ... + def addtag_closest( + self, newtag: str, x: _ScreenUnits, y: _ScreenUnits, halo: _ScreenUnits | None = None, start: str | int | None = None + ) -> None: ... + def addtag_enclosed(self, newtag: str, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> None: ... + def addtag_overlapping(self, newtag: str, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> None: ... + def addtag_withtag(self, newtag: str, tagOrId: str | int) -> None: ... + def find(self, *args): ... # internal method + def find_above(self, tagOrId: str | int) -> tuple[int, ...]: ... + def find_all(self) -> tuple[int, ...]: ... + def find_below(self, tagOrId: str | int) -> tuple[int, ...]: ... + def find_closest( + self, x: _ScreenUnits, y: _ScreenUnits, halo: _ScreenUnits | None = None, start: str | int | None = None + ) -> tuple[int, ...]: ... + def find_enclosed(self, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> tuple[int, ...]: ... + def find_overlapping(self, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: float) -> tuple[int, ...]: ... + def find_withtag(self, tagOrId: str | int) -> tuple[int, ...]: ... + # Incompatible with Misc.bbox(), tkinter violates LSP + def bbox(self, *args: str | int) -> tuple[int, int, int, int]: ... # type: ignore[override] + @overload + def tag_bind( + self, + tagOrId: str | int, + sequence: str | None = None, + func: Callable[[Event[Canvas]], object] | None = None, + add: Literal["", "+"] | bool | None = None, + ) -> str: ... + @overload + def tag_bind( + self, tagOrId: str | int, sequence: str | None, func: str, add: Literal["", "+"] | bool | None = None + ) -> None: ... + @overload + def tag_bind(self, tagOrId: str | int, *, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + def tag_unbind(self, tagOrId: str | int, sequence: str, funcid: str | None = None) -> None: ... + def canvasx(self, screenx, gridspacing: Incomplete | None = None): ... + def canvasy(self, screeny, gridspacing: Incomplete | None = None): ... + @overload + def coords(self, tagOrId: str | int, /) -> list[float]: ... + @overload + def coords(self, tagOrId: str | int, args: list[int] | list[float] | tuple[float, ...], /) -> None: ... + @overload + def coords(self, tagOrId: str | int, x1: float, y1: float, /, *args: float) -> None: ... + # create_foo() methods accept coords as a list or tuple, or as separate arguments. + # Lists and tuples can be flat as in [1, 2, 3, 4], or nested as in [(1, 2), (3, 4)]. + # Keyword arguments should be the same in all overloads of each method. + def create_arc(self, *args, **kw) -> int: ... + def create_bitmap(self, *args, **kw) -> int: ... + def create_image(self, *args, **kw) -> int: ... + @overload + def create_line( + self, + x0: float, + y0: float, + x1: float, + y1: float, + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + arrow: Literal["first", "last", "both"] = ..., + arrowshape: tuple[float, float, float] = ..., + capstyle: Literal["round", "projecting", "butt"] = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + joinstyle: Literal["round", "bevel", "miter"] = ..., + offset: _ScreenUnits = ..., + smooth: bool = ..., + splinesteps: float = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_line( + self, + xy_pair_0: tuple[float, float], + xy_pair_1: tuple[float, float], + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + arrow: Literal["first", "last", "both"] = ..., + arrowshape: tuple[float, float, float] = ..., + capstyle: Literal["round", "projecting", "butt"] = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + joinstyle: Literal["round", "bevel", "miter"] = ..., + offset: _ScreenUnits = ..., + smooth: bool = ..., + splinesteps: float = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_line( + self, + coords: ( + tuple[float, float, float, float] + | tuple[tuple[float, float], tuple[float, float]] + | list[int] + | list[float] + | list[tuple[int, int]] + | list[tuple[float, float]] + ), + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + arrow: Literal["first", "last", "both"] = ..., + arrowshape: tuple[float, float, float] = ..., + capstyle: Literal["round", "projecting", "butt"] = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + joinstyle: Literal["round", "bevel", "miter"] = ..., + offset: _ScreenUnits = ..., + smooth: bool = ..., + splinesteps: float = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_oval( + self, + x0: float, + y0: float, + x1: float, + y1: float, + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_oval( + self, + xy_pair_0: tuple[float, float], + xy_pair_1: tuple[float, float], + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_oval( + self, + coords: ( + tuple[float, float, float, float] + | tuple[tuple[float, float], tuple[float, float]] + | list[int] + | list[float] + | list[tuple[int, int]] + | list[tuple[float, float]] + ), + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_polygon( + self, + x0: float, + y0: float, + x1: float, + y1: float, + /, + *xy_pairs: float, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + joinstyle: Literal["round", "bevel", "miter"] = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + smooth: bool = ..., + splinesteps: float = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_polygon( + self, + xy_pair_0: tuple[float, float], + xy_pair_1: tuple[float, float], + /, + *xy_pairs: tuple[float, float], + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + joinstyle: Literal["round", "bevel", "miter"] = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + smooth: bool = ..., + splinesteps: float = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_polygon( + self, + coords: ( + tuple[float, ...] + | tuple[tuple[float, float], ...] + | list[int] + | list[float] + | list[tuple[int, int]] + | list[tuple[float, float]] + ), + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + joinstyle: Literal["round", "bevel", "miter"] = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + smooth: bool = ..., + splinesteps: float = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_rectangle( + self, + x0: float, + y0: float, + x1: float, + y1: float, + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_rectangle( + self, + xy_pair_0: tuple[float, float], + xy_pair_1: tuple[float, float], + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_rectangle( + self, + coords: ( + tuple[float, float, float, float] + | tuple[tuple[float, float], tuple[float, float]] + | list[int] + | list[float] + | list[tuple[int, int]] + | list[tuple[float, float]] + ), + /, + *, + activedash: str | int | list[int] | tuple[int, ...] = ..., + activefill: str = ..., + activeoutline: str = ..., + activeoutlinestipple: str = ..., + activestipple: str = ..., + activewidth: _ScreenUnits = ..., + dash: str | int | list[int] | tuple[int, ...] = ..., + dashoffset: _ScreenUnits = ..., + disableddash: str | int | list[int] | tuple[int, ...] = ..., + disabledfill: str = ..., + disabledoutline: str = ..., + disabledoutlinestipple: str = ..., + disabledstipple: str = ..., + disabledwidth: _ScreenUnits = ..., + fill: str = ..., + offset: _ScreenUnits = ..., + outline: str = ..., + outlineoffset: _ScreenUnits = ..., + outlinestipple: str = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_text( + self, + x: float, + y: float, + /, + *, + activefill: str = ..., + activestipple: str = ..., + anchor: _Anchor = ..., + angle: float | str = ..., + disabledfill: str = ..., + disabledstipple: str = ..., + fill: str = ..., + font: _FontDescription = ..., + justify: Literal["left", "center", "right"] = ..., + offset: _ScreenUnits = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + text: float | str = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_text( + self, + coords: tuple[float, float] | list[int] | list[float], + /, + *, + activefill: str = ..., + activestipple: str = ..., + anchor: _Anchor = ..., + angle: float | str = ..., + disabledfill: str = ..., + disabledstipple: str = ..., + fill: str = ..., + font: _FontDescription = ..., + justify: Literal["left", "center", "right"] = ..., + offset: _ScreenUnits = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + stipple: str = ..., + tags: str | list[str] | tuple[str, ...] = ..., + text: float | str = ..., + width: _ScreenUnits = ..., + ) -> int: ... + @overload + def create_window( + self, + x: float, + y: float, + /, + *, + anchor: _Anchor = ..., + height: _ScreenUnits = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + window: Widget = ..., + ) -> int: ... + @overload + def create_window( + self, + coords: tuple[float, float] | list[int] | list[float], + /, + *, + anchor: _Anchor = ..., + height: _ScreenUnits = ..., + state: Literal["normal", "hidden", "disabled"] = ..., + tags: str | list[str] | tuple[str, ...] = ..., + width: _ScreenUnits = ..., + window: Widget = ..., + ) -> int: ... + def dchars(self, *args) -> None: ... + def delete(self, *tagsOrCanvasIds: str | int) -> None: ... + @overload + def dtag(self, tag: str, tag_to_delete: str | None = ..., /) -> None: ... + @overload + def dtag(self, id: int, tag_to_delete: str, /) -> None: ... + def focus(self, *args): ... + def gettags(self, tagOrId: str | int, /) -> tuple[str, ...]: ... + def icursor(self, *args) -> None: ... + def index(self, *args): ... + def insert(self, *args) -> None: ... + def itemcget(self, tagOrId, option): ... + # itemconfigure kwargs depend on item type, which is not known when type checking + def itemconfigure( + self, tagOrId: str | int, cnf: dict[str, Any] | None = None, **kw: Any + ) -> dict[str, tuple[str, str, str, str, str]] | None: ... + itemconfig = itemconfigure + def move(self, *args) -> None: ... + def moveto(self, tagOrId: str | int, x: Literal[""] | float = "", y: Literal[""] | float = "") -> None: ... + def postscript(self, cnf={}, **kw): ... + # tkinter does: + # lower = tag_lower + # lift = tkraise = tag_raise + # + # But mypy doesn't like aliasing here (maybe because Misc defines the same names) + def tag_lower(self, first: str | int, second: str | int | None = ..., /) -> None: ... + def lower(self, first: str | int, second: str | int | None = ..., /) -> None: ... # type: ignore[override] + def tag_raise(self, first: str | int, second: str | int | None = ..., /) -> None: ... + def tkraise(self, first: str | int, second: str | int | None = ..., /) -> None: ... # type: ignore[override] + def lift(self, first: str | int, second: str | int | None = ..., /) -> None: ... # type: ignore[override] + def scale( + self, tagOrId: str | int, xOrigin: _ScreenUnits, yOrigin: _ScreenUnits, xScale: float, yScale: float, / + ) -> None: ... + def scan_mark(self, x, y) -> None: ... + def scan_dragto(self, x, y, gain: int = 10) -> None: ... + def select_adjust(self, tagOrId, index) -> None: ... + def select_clear(self) -> None: ... + def select_from(self, tagOrId, index) -> None: ... + def select_item(self): ... + def select_to(self, tagOrId, index) -> None: ... + def type(self, tagOrId: str | int) -> int | None: ... + +class Checkbutton(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = "center", + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = "", + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + command: _ButtonCommand = "", + compound: _Compound = "none", + cursor: _Cursor = "", + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 1, + image: _ImageSpec = "", + indicatoron: bool = True, + justify: Literal["left", "center", "right"] = "center", + name: str = ..., + offrelief: _Relief = ..., + # The checkbutton puts a value to its variable when it's checked or + # unchecked. We don't restrict the type of that value here, so + # Any-typing is fine. + # + # I think Checkbutton shouldn't be generic, because then specifying + # "any checkbutton regardless of what variable it uses" would be + # difficult, and we might run into issues just like how list[float] + # and list[int] are incompatible. Also, we would need a way to + # specify "Checkbutton not associated with any variable", which is + # done by setting variable to empty string (the default). + offvalue: Any = 0, + onvalue: Any = 1, + overrelief: _Relief | Literal[""] = "", + padx: _ScreenUnits = 1, + pady: _ScreenUnits = 1, + relief: _Relief = "flat", + selectcolor: str = ..., + selectimage: _ImageSpec = "", + state: Literal["normal", "active", "disabled"] = "normal", + takefocus: _TakeFocusValue = "", + text: float | str = "", + textvariable: Variable = ..., + tristateimage: _ImageSpec = "", + tristatevalue: Any = "", + underline: int = -1, + variable: Variable | Literal[""] = ..., + width: _ScreenUnits = 0, + wraplength: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + command: _ButtonCommand = ..., + compound: _Compound = ..., + cursor: _Cursor = ..., + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + image: _ImageSpec = ..., + indicatoron: bool = ..., + justify: Literal["left", "center", "right"] = ..., + offrelief: _Relief = ..., + offvalue: Any = ..., + onvalue: Any = ..., + overrelief: _Relief | Literal[""] = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + selectcolor: str = ..., + selectimage: _ImageSpec = ..., + state: Literal["normal", "active", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + text: float | str = ..., + textvariable: Variable = ..., + tristateimage: _ImageSpec = ..., + tristatevalue: Any = ..., + underline: int = ..., + variable: Variable | Literal[""] = ..., + width: _ScreenUnits = ..., + wraplength: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def deselect(self) -> None: ... + def flash(self) -> None: ... + def invoke(self) -> Any: ... + def select(self) -> None: ... + def toggle(self) -> None: ... + +class Entry(Widget, XView): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = "xterm", + disabledbackground: str = ..., + disabledforeground: str = ..., + exportselection: bool = True, + fg: str = ..., + font: _FontDescription = "TkTextFont", + foreground: str = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = 0, + insertofftime: int = 300, + insertontime: int = 600, + insertwidth: _ScreenUnits = ..., + invalidcommand: _EntryValidateCommand = "", + invcmd: _EntryValidateCommand = "", # same as invalidcommand + justify: Literal["left", "center", "right"] = "left", + name: str = ..., + readonlybackground: str = ..., + relief: _Relief = "sunken", + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + show: str = "", + state: Literal["normal", "disabled", "readonly"] = "normal", + takefocus: _TakeFocusValue = "", + textvariable: Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", + validatecommand: _EntryValidateCommand = "", + vcmd: _EntryValidateCommand = "", # same as validatecommand + width: int = 20, + xscrollcommand: _XYScrollCommand = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + disabledbackground: str = ..., + disabledforeground: str = ..., + exportselection: bool = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = ..., + insertofftime: int = ..., + insertontime: int = ..., + insertwidth: _ScreenUnits = ..., + invalidcommand: _EntryValidateCommand = ..., + invcmd: _EntryValidateCommand = ..., + justify: Literal["left", "center", "right"] = ..., + readonlybackground: str = ..., + relief: _Relief = ..., + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + show: str = ..., + state: Literal["normal", "disabled", "readonly"] = ..., + takefocus: _TakeFocusValue = ..., + textvariable: Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., + validatecommand: _EntryValidateCommand = ..., + vcmd: _EntryValidateCommand = ..., + width: int = ..., + xscrollcommand: _XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def delete(self, first: str | int, last: str | int | None = None) -> None: ... + def get(self) -> str: ... + def icursor(self, index: str | int) -> None: ... + def index(self, index: str | int) -> int: ... + def insert(self, index: str | int, string: str) -> None: ... + def scan_mark(self, x) -> None: ... + def scan_dragto(self, x) -> None: ... + def selection_adjust(self, index: str | int) -> None: ... + def selection_clear(self) -> None: ... # type: ignore[override] + def selection_from(self, index: str | int) -> None: ... + def selection_present(self) -> bool: ... + def selection_range(self, start: str | int, end: str | int) -> None: ... + def selection_to(self, index: str | int) -> None: ... + select_adjust = selection_adjust + select_clear = selection_clear + select_from = selection_from + select_present = selection_present + select_range = selection_range + select_to = selection_to + +class Frame(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + background: str = ..., + bd: _ScreenUnits = 0, + bg: str = ..., + border: _ScreenUnits = 0, + borderwidth: _ScreenUnits = 0, + class_: str = "Frame", # can't be changed with configure() + colormap: Literal["new", ""] | Misc = "", # can't be changed with configure() + container: bool = False, # can't be changed with configure() + cursor: _Cursor = "", + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 0, + name: str = ..., + padx: _ScreenUnits = 0, + pady: _ScreenUnits = 0, + relief: _Relief = "flat", + takefocus: _TakeFocusValue = 0, + visual: str | tuple[str, int] = "", # can't be changed with configure() + width: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + takefocus: _TakeFocusValue = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Label(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = "center", + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = "", + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + compound: _Compound = "none", + cursor: _Cursor = "", + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 0, + image: _ImageSpec = "", + justify: Literal["left", "center", "right"] = "center", + name: str = ..., + padx: _ScreenUnits = 1, + pady: _ScreenUnits = 1, + relief: _Relief = "flat", + state: Literal["normal", "active", "disabled"] = "normal", + takefocus: _TakeFocusValue = 0, + text: float | str = "", + textvariable: Variable = ..., + underline: int = -1, + width: _ScreenUnits = 0, + wraplength: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + compound: _Compound = ..., + cursor: _Cursor = ..., + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + image: _ImageSpec = ..., + justify: Literal["left", "center", "right"] = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + state: Literal["normal", "active", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + text: float | str = ..., + textvariable: Variable = ..., + underline: int = ..., + width: _ScreenUnits = ..., + wraplength: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Listbox(Widget, XView, YView): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activestyle: Literal["dotbox", "none", "underline"] = ..., + background: str = ..., + bd: _ScreenUnits = 1, + bg: str = ..., + border: _ScreenUnits = 1, + borderwidth: _ScreenUnits = 1, + cursor: _Cursor = "", + disabledforeground: str = ..., + exportselection: bool | Literal[0, 1] = 1, + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: int = 10, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + justify: Literal["left", "center", "right"] = "left", + # There's no tkinter.ListVar, but seems like bare tkinter.Variable + # actually works for this: + # + # >>> import tkinter + # >>> lb = tkinter.Listbox() + # >>> var = lb['listvariable'] = tkinter.Variable() + # >>> var.set(['foo', 'bar', 'baz']) + # >>> lb.get(0, 'end') + # ('foo', 'bar', 'baz') + listvariable: Variable = ..., + name: str = ..., + relief: _Relief = ..., + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = 0, + selectforeground: str = ..., + # from listbox man page: "The value of the [selectmode] option may be + # arbitrary, but the default bindings expect it to be either single, + # browse, multiple, or extended" + # + # I have never seen anyone setting this to something else than what + # "the default bindings expect", but let's support it anyway. + selectmode: str | Literal["single", "browse", "multiple", "extended"] = "browse", # noqa: Y051 + setgrid: bool = False, + state: Literal["normal", "disabled"] = "normal", + takefocus: _TakeFocusValue = "", + width: int = 20, + xscrollcommand: _XYScrollCommand = "", + yscrollcommand: _XYScrollCommand = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activestyle: Literal["dotbox", "none", "underline"] = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + disabledforeground: str = ..., + exportselection: bool = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: int = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + justify: Literal["left", "center", "right"] = ..., + listvariable: Variable = ..., + relief: _Relief = ..., + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + selectmode: str | Literal["single", "browse", "multiple", "extended"] = ..., # noqa: Y051 + setgrid: bool = ..., + state: Literal["normal", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + width: int = ..., + xscrollcommand: _XYScrollCommand = ..., + yscrollcommand: _XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def activate(self, index: str | int) -> None: ... + def bbox(self, index: str | int) -> tuple[int, int, int, int] | None: ... # type: ignore[override] + def curselection(self): ... + def delete(self, first: str | int, last: str | int | None = None) -> None: ... + def get(self, first: str | int, last: str | int | None = None): ... + def index(self, index: str | int) -> int: ... + def insert(self, index: str | int, *elements: str | float) -> None: ... + def nearest(self, y): ... + def scan_mark(self, x, y) -> None: ... + def scan_dragto(self, x, y) -> None: ... + def see(self, index: str | int) -> None: ... + def selection_anchor(self, index: str | int) -> None: ... + select_anchor = selection_anchor + def selection_clear(self, first: str | int, last: str | int | None = None) -> None: ... # type: ignore[override] + select_clear = selection_clear + def selection_includes(self, index: str | int): ... + select_includes = selection_includes + def selection_set(self, first: str | int, last: str | int | None = None) -> None: ... + select_set = selection_set + def size(self) -> int: ... # type: ignore[override] + def itemcget(self, index: str | int, option): ... + def itemconfigure(self, index: str | int, cnf: Incomplete | None = None, **kw): ... + itemconfig = itemconfigure + +class Menu(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + activeborderwidth: _ScreenUnits = ..., + activeforeground: str = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = "arrow", + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + name: str = ..., + postcommand: Callable[[], object] | str = "", + relief: _Relief = ..., + selectcolor: str = ..., + takefocus: _TakeFocusValue = 0, + tearoff: bool | Literal[0, 1] = 1, + # I guess tearoffcommand arguments are supposed to be widget objects, + # but they are widget name strings. Use nametowidget() to handle the + # arguments of tearoffcommand. + tearoffcommand: Callable[[str, str], object] | str = "", + title: str = "", + type: Literal["menubar", "tearoff", "normal"] = "normal", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + activeborderwidth: _ScreenUnits = ..., + activeforeground: str = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + postcommand: Callable[[], object] | str = ..., + relief: _Relief = ..., + selectcolor: str = ..., + takefocus: _TakeFocusValue = ..., + tearoff: bool = ..., + tearoffcommand: Callable[[str, str], object] | str = ..., + title: str = ..., + type: Literal["menubar", "tearoff", "normal"] = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def tk_popup(self, x: int, y: int, entry: str | int = "") -> None: ... + def activate(self, index: str | int) -> None: ... + def add(self, itemType, cnf={}, **kw): ... # docstring says "Internal function." + def insert(self, index, itemType, cnf={}, **kw): ... # docstring says "Internal function." + def add_cascade( + self, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + label: str = ..., + menu: Menu = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + ) -> None: ... + def add_checkbutton( + self, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + indicatoron: bool = ..., + label: str = ..., + offvalue: Any = ..., + onvalue: Any = ..., + selectcolor: str = ..., + selectimage: _ImageSpec = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + variable: Variable = ..., + ) -> None: ... + def add_command( + self, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + label: str = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + ) -> None: ... + def add_radiobutton( + self, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + indicatoron: bool = ..., + label: str = ..., + selectcolor: str = ..., + selectimage: _ImageSpec = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + value: Any = ..., + variable: Variable = ..., + ) -> None: ... + def add_separator(self, cnf: dict[str, Any] | None = {}, *, background: str = ...) -> None: ... + def insert_cascade( + self, + index: str | int, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + label: str = ..., + menu: Menu = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + ) -> None: ... + def insert_checkbutton( + self, + index: str | int, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + indicatoron: bool = ..., + label: str = ..., + offvalue: Any = ..., + onvalue: Any = ..., + selectcolor: str = ..., + selectimage: _ImageSpec = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + variable: Variable = ..., + ) -> None: ... + def insert_command( + self, + index: str | int, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + label: str = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + ) -> None: ... + def insert_radiobutton( + self, + index: str | int, + cnf: dict[str, Any] | None = {}, + *, + accelerator: str = ..., + activebackground: str = ..., + activeforeground: str = ..., + background: str = ..., + bitmap: str = ..., + columnbreak: int = ..., + command: Callable[[], object] | str = ..., + compound: _Compound = ..., + font: _FontDescription = ..., + foreground: str = ..., + hidemargin: bool = ..., + image: _ImageSpec = ..., + indicatoron: bool = ..., + label: str = ..., + selectcolor: str = ..., + selectimage: _ImageSpec = ..., + state: Literal["normal", "active", "disabled"] = ..., + underline: int = ..., + value: Any = ..., + variable: Variable = ..., + ) -> None: ... + def insert_separator(self, index: str | int, cnf: dict[str, Any] | None = {}, *, background: str = ...) -> None: ... + def delete(self, index1: str | int, index2: str | int | None = None) -> None: ... + def entrycget(self, index: str | int, option: str) -> Any: ... + def entryconfigure( + self, index: str | int, cnf: dict[str, Any] | None = None, **kw: Any + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + entryconfig = entryconfigure + def index(self, index: str | int) -> int | None: ... + def invoke(self, index: str | int) -> Any: ... + def post(self, x: int, y: int) -> None: ... + def type(self, index: str | int) -> Literal["cascade", "checkbutton", "command", "radiobutton", "separator"]: ... + def unpost(self) -> None: ... + def xposition(self, index: str | int) -> int: ... + def yposition(self, index: str | int) -> int: ... + +class Menubutton(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = "", + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + compound: _Compound = "none", + cursor: _Cursor = "", + direction: Literal["above", "below", "left", "right", "flush"] = "below", + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 0, + image: _ImageSpec = "", + indicatoron: bool = ..., + justify: Literal["left", "center", "right"] = ..., + menu: Menu = ..., + name: str = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = "flat", + state: Literal["normal", "active", "disabled"] = "normal", + takefocus: _TakeFocusValue = 0, + text: float | str = "", + textvariable: Variable = ..., + underline: int = -1, + width: _ScreenUnits = 0, + wraplength: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + compound: _Compound = ..., + cursor: _Cursor = ..., + direction: Literal["above", "below", "left", "right", "flush"] = ..., + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + image: _ImageSpec = ..., + indicatoron: bool = ..., + justify: Literal["left", "center", "right"] = ..., + menu: Menu = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + state: Literal["normal", "active", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + text: float | str = ..., + textvariable: Variable = ..., + underline: int = ..., + width: _ScreenUnits = ..., + wraplength: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Message(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + anchor: _Anchor = "center", + aspect: int = 150, + background: str = ..., + bd: _ScreenUnits = 1, + bg: str = ..., + border: _ScreenUnits = 1, + borderwidth: _ScreenUnits = 1, + cursor: _Cursor = "", + fg: str = ..., + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 0, + justify: Literal["left", "center", "right"] = "left", + name: str = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = "flat", + takefocus: _TakeFocusValue = 0, + text: float | str = "", + textvariable: Variable = ..., + # there's width but no height + width: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + anchor: _Anchor = ..., + aspect: int = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + justify: Literal["left", "center", "right"] = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + takefocus: _TakeFocusValue = ..., + text: float | str = ..., + textvariable: Variable = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Radiobutton(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = "center", + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = "", + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + command: _ButtonCommand = "", + compound: _Compound = "none", + cursor: _Cursor = "", + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 1, + image: _ImageSpec = "", + indicatoron: bool = True, + justify: Literal["left", "center", "right"] = "center", + name: str = ..., + offrelief: _Relief = ..., + overrelief: _Relief | Literal[""] = "", + padx: _ScreenUnits = 1, + pady: _ScreenUnits = 1, + relief: _Relief = "flat", + selectcolor: str = ..., + selectimage: _ImageSpec = "", + state: Literal["normal", "active", "disabled"] = "normal", + takefocus: _TakeFocusValue = "", + text: float | str = "", + textvariable: Variable = ..., + tristateimage: _ImageSpec = "", + tristatevalue: Any = "", + underline: int = -1, + value: Any = "", + variable: Variable | Literal[""] = ..., + width: _ScreenUnits = 0, + wraplength: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + activeforeground: str = ..., + anchor: _Anchor = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bitmap: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + command: _ButtonCommand = ..., + compound: _Compound = ..., + cursor: _Cursor = ..., + disabledforeground: str = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + image: _ImageSpec = ..., + indicatoron: bool = ..., + justify: Literal["left", "center", "right"] = ..., + offrelief: _Relief = ..., + overrelief: _Relief | Literal[""] = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + selectcolor: str = ..., + selectimage: _ImageSpec = ..., + state: Literal["normal", "active", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + text: float | str = ..., + textvariable: Variable = ..., + tristateimage: _ImageSpec = ..., + tristatevalue: Any = ..., + underline: int = ..., + value: Any = ..., + variable: Variable | Literal[""] = ..., + width: _ScreenUnits = ..., + wraplength: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def deselect(self) -> None: ... + def flash(self) -> None: ... + def invoke(self) -> Any: ... + def select(self) -> None: ... + +class Scale(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + background: str = ..., + bd: _ScreenUnits = 1, + bg: str = ..., + bigincrement: float = 0.0, + border: _ScreenUnits = 1, + borderwidth: _ScreenUnits = 1, + # don't know why the callback gets string instead of float + command: str | Callable[[str], object] = "", + cursor: _Cursor = "", + digits: int = 0, + fg: str = ..., + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + from_: float = 0.0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + label: str = "", + length: _ScreenUnits = 100, + name: str = ..., + orient: Literal["horizontal", "vertical"] = "vertical", + relief: _Relief = "flat", + repeatdelay: int = 300, + repeatinterval: int = 100, + resolution: float = 1.0, + showvalue: bool = True, + sliderlength: _ScreenUnits = 30, + sliderrelief: _Relief = "raised", + state: Literal["normal", "active", "disabled"] = "normal", + takefocus: _TakeFocusValue = "", + tickinterval: float = 0.0, + to: float = 100.0, + troughcolor: str = ..., + variable: IntVar | DoubleVar = ..., + width: _ScreenUnits = 15, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + bigincrement: float = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + command: str | Callable[[str], object] = ..., + cursor: _Cursor = ..., + digits: int = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + from_: float = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + label: str = ..., + length: _ScreenUnits = ..., + orient: Literal["horizontal", "vertical"] = ..., + relief: _Relief = ..., + repeatdelay: int = ..., + repeatinterval: int = ..., + resolution: float = ..., + showvalue: bool = ..., + sliderlength: _ScreenUnits = ..., + sliderrelief: _Relief = ..., + state: Literal["normal", "active", "disabled"] = ..., + takefocus: _TakeFocusValue = ..., + tickinterval: float = ..., + to: float = ..., + troughcolor: str = ..., + variable: IntVar | DoubleVar = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def get(self) -> float: ... + def set(self, value) -> None: ... + def coords(self, value: float | None = None) -> tuple[int, int]: ... + def identify(self, x, y) -> Literal["", "slider", "trough1", "trough2"]: ... + +class Scrollbar(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + activerelief: _Relief = "raised", + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + # There are many ways how the command may get called. Search for + # 'SCROLLING COMMANDS' in scrollbar man page. There doesn't seem to + # be any way to specify an overloaded callback function, so we say + # that it can take any args while it can't in reality. + command: Callable[..., tuple[float, float] | None] | str = "", + cursor: _Cursor = "", + elementborderwidth: _ScreenUnits = -1, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 0, + jump: bool = False, + name: str = ..., + orient: Literal["horizontal", "vertical"] = "vertical", + relief: _Relief = ..., + repeatdelay: int = 300, + repeatinterval: int = 100, + takefocus: _TakeFocusValue = "", + troughcolor: str = ..., + width: _ScreenUnits = ..., + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + activerelief: _Relief = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + command: Callable[..., tuple[float, float] | None] | str = ..., + cursor: _Cursor = ..., + elementborderwidth: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + jump: bool = ..., + orient: Literal["horizontal", "vertical"] = ..., + relief: _Relief = ..., + repeatdelay: int = ..., + repeatinterval: int = ..., + takefocus: _TakeFocusValue = ..., + troughcolor: str = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def activate(self, index: Incomplete | None = None): ... + def delta(self, deltax: int, deltay: int) -> float: ... + def fraction(self, x: int, y: int) -> float: ... + def identify(self, x: int, y: int) -> Literal["arrow1", "arrow2", "slider", "trough1", "trough2", ""]: ... + def get(self) -> tuple[float, float, float, float] | tuple[float, float]: ... + def set(self, first: float | str, last: float | str) -> None: ... + +_TextIndex: TypeAlias = _tkinter.Tcl_Obj | str | float | Misc +_WhatToCount: TypeAlias = Literal[ + "chars", "displaychars", "displayindices", "displaylines", "indices", "lines", "xpixels", "ypixels" +] + +class Text(Widget, XView, YView): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + autoseparators: bool = True, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + blockcursor: bool = False, + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = "xterm", + endline: int | Literal[""] = "", + exportselection: bool = True, + fg: str = ..., + font: _FontDescription = "TkFixedFont", + foreground: str = ..., + # width is always int, but height is allowed to be ScreenUnits. + # This doesn't make any sense to me, and this isn't documented. + # The docs seem to say that both should be integers. + height: _ScreenUnits = 24, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + inactiveselectbackground: str = ..., + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = 0, + insertofftime: int = 300, + insertontime: int = 600, + insertunfocussed: Literal["none", "hollow", "solid"] = "none", + insertwidth: _ScreenUnits = ..., + maxundo: int = 0, + name: str = ..., + padx: _ScreenUnits = 1, + pady: _ScreenUnits = 1, + relief: _Relief = ..., + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + setgrid: bool = False, + spacing1: _ScreenUnits = 0, + spacing2: _ScreenUnits = 0, + spacing3: _ScreenUnits = 0, + startline: int | Literal[""] = "", + state: Literal["normal", "disabled"] = "normal", + # Literal inside Tuple doesn't actually work + tabs: _ScreenUnits | str | tuple[_ScreenUnits | str, ...] = "", + tabstyle: Literal["tabular", "wordprocessor"] = "tabular", + takefocus: _TakeFocusValue = "", + undo: bool = False, + width: int = 80, + wrap: Literal["none", "char", "word"] = "char", + xscrollcommand: _XYScrollCommand = "", + yscrollcommand: _XYScrollCommand = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + autoseparators: bool = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + blockcursor: bool = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + endline: int | Literal[""] = ..., + exportselection: bool = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + inactiveselectbackground: str = ..., + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = ..., + insertofftime: int = ..., + insertontime: int = ..., + insertunfocussed: Literal["none", "hollow", "solid"] = ..., + insertwidth: _ScreenUnits = ..., + maxundo: int = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + setgrid: bool = ..., + spacing1: _ScreenUnits = ..., + spacing2: _ScreenUnits = ..., + spacing3: _ScreenUnits = ..., + startline: int | Literal[""] = ..., + state: Literal["normal", "disabled"] = ..., + tabs: _ScreenUnits | str | tuple[_ScreenUnits | str, ...] = ..., + tabstyle: Literal["tabular", "wordprocessor"] = ..., + takefocus: _TakeFocusValue = ..., + undo: bool = ..., + width: int = ..., + wrap: Literal["none", "char", "word"] = ..., + xscrollcommand: _XYScrollCommand = ..., + yscrollcommand: _XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def bbox(self, index: _TextIndex) -> tuple[int, int, int, int] | None: ... # type: ignore[override] + def compare(self, index1: _TextIndex, op: Literal["<", "<=", "==", ">=", ">", "!="], index2: _TextIndex) -> bool: ... + if sys.version_info >= (3, 13): + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[True]) -> int: ... + @overload + def count( + self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], /, *, return_ints: Literal[True] + ) -> int: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: Literal["update"], + arg2: _WhatToCount, + /, + *, + return_ints: Literal[True], + ) -> int: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount, + arg2: Literal["update"], + /, + *, + return_ints: Literal[True], + ) -> int: ... + @overload + def count( + self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /, *, return_ints: Literal[True] + ) -> tuple[int, int]: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount | Literal["update"], + arg2: _WhatToCount | Literal["update"], + arg3: _WhatToCount | Literal["update"], + /, + *args: _WhatToCount | Literal["update"], + return_ints: Literal[True], + ) -> tuple[int, ...]: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[False] = False) -> tuple[int] | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg: _WhatToCount | Literal["update"], + /, + *, + return_ints: Literal[False] = False, + ) -> tuple[int] | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: Literal["update"], + arg2: _WhatToCount, + /, + *, + return_ints: Literal[False] = False, + ) -> int | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount, + arg2: Literal["update"], + /, + *, + return_ints: Literal[False] = False, + ) -> int | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount, + arg2: _WhatToCount, + /, + *, + return_ints: Literal[False] = False, + ) -> tuple[int, int]: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount | Literal["update"], + arg2: _WhatToCount | Literal["update"], + arg3: _WhatToCount | Literal["update"], + /, + *args: _WhatToCount | Literal["update"], + return_ints: Literal[False] = False, + ) -> tuple[int, ...]: ... + else: + @overload + def count(self, index1: _TextIndex, index2: _TextIndex) -> tuple[int] | None: ... + @overload + def count( + self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], / + ) -> tuple[int] | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: Literal["update"], arg2: _WhatToCount, /) -> int | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: Literal["update"], /) -> int | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /) -> tuple[int, int]: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount | Literal["update"], + arg2: _WhatToCount | Literal["update"], + arg3: _WhatToCount | Literal["update"], + /, + *args: _WhatToCount | Literal["update"], + ) -> tuple[int, ...]: ... + + @overload + def debug(self, boolean: None = None) -> bool: ... + @overload + def debug(self, boolean: bool) -> None: ... + def delete(self, index1: _TextIndex, index2: _TextIndex | None = None) -> None: ... + def dlineinfo(self, index: _TextIndex) -> tuple[int, int, int, int, int] | None: ... + @overload + def dump( + self, + index1: _TextIndex, + index2: _TextIndex | None = None, + command: None = None, + *, + all: bool = ..., + image: bool = ..., + mark: bool = ..., + tag: bool = ..., + text: bool = ..., + window: bool = ..., + ) -> list[tuple[str, str, str]]: ... + @overload + def dump( + self, + index1: _TextIndex, + index2: _TextIndex | None, + command: Callable[[str, str, str], object] | str, + *, + all: bool = ..., + image: bool = ..., + mark: bool = ..., + tag: bool = ..., + text: bool = ..., + window: bool = ..., + ) -> None: ... + @overload + def dump( + self, + index1: _TextIndex, + index2: _TextIndex | None = None, + *, + command: Callable[[str, str, str], object] | str, + all: bool = ..., + image: bool = ..., + mark: bool = ..., + tag: bool = ..., + text: bool = ..., + window: bool = ..., + ) -> None: ... + def edit(self, *args): ... # docstring says "Internal method" + @overload + def edit_modified(self, arg: None = None) -> bool: ... # actually returns Literal[0, 1] + @overload + def edit_modified(self, arg: bool) -> None: ... # actually returns empty string + def edit_redo(self) -> None: ... # actually returns empty string + def edit_reset(self) -> None: ... # actually returns empty string + def edit_separator(self) -> None: ... # actually returns empty string + def edit_undo(self) -> None: ... # actually returns empty string + def get(self, index1: _TextIndex, index2: _TextIndex | None = None) -> str: ... + @overload + def image_cget(self, index: _TextIndex, option: Literal["image", "name"]) -> str: ... + @overload + def image_cget(self, index: _TextIndex, option: Literal["padx", "pady"]) -> int: ... + @overload + def image_cget(self, index: _TextIndex, option: Literal["align"]) -> Literal["baseline", "bottom", "center", "top"]: ... + @overload + def image_cget(self, index: _TextIndex, option: str) -> Any: ... + @overload + def image_configure(self, index: _TextIndex, cnf: str) -> tuple[str, str, str, str, str | int]: ... + @overload + def image_configure( + self, + index: _TextIndex, + cnf: dict[str, Any] | None = {}, + *, + align: Literal["baseline", "bottom", "center", "top"] = ..., + image: _ImageSpec = ..., + name: str = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... + def image_create( + self, + index: _TextIndex, + cnf: dict[str, Any] | None = {}, + *, + align: Literal["baseline", "bottom", "center", "top"] = ..., + image: _ImageSpec = ..., + name: str = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + ) -> str: ... + def image_names(self) -> tuple[str, ...]: ... + def index(self, index: _TextIndex) -> str: ... + def insert(self, index: _TextIndex, chars: str, *args: str | list[str] | tuple[str, ...]) -> None: ... + @overload + def mark_gravity(self, markName: str, direction: None = None) -> Literal["left", "right"]: ... + @overload + def mark_gravity(self, markName: str, direction: Literal["left", "right"]) -> None: ... # actually returns empty string + def mark_names(self) -> tuple[str, ...]: ... + def mark_set(self, markName: str, index: _TextIndex) -> None: ... + def mark_unset(self, *markNames: str) -> None: ... + def mark_next(self, index: _TextIndex) -> str | None: ... + def mark_previous(self, index: _TextIndex) -> str | None: ... + # **kw of peer_create is same as the kwargs of Text.__init__ + def peer_create(self, newPathName: str | Text, cnf: dict[str, Any] = {}, **kw) -> None: ... + def peer_names(self) -> tuple[_tkinter.Tcl_Obj, ...]: ... + def replace(self, index1: _TextIndex, index2: _TextIndex, chars: str, *args: str | list[str] | tuple[str, ...]) -> None: ... + def scan_mark(self, x: int, y: int) -> None: ... + def scan_dragto(self, x: int, y: int) -> None: ... + def search( + self, + pattern: str, + index: _TextIndex, + stopindex: _TextIndex | None = None, + forwards: bool | None = None, + backwards: bool | None = None, + exact: bool | None = None, + regexp: bool | None = None, + nocase: bool | None = None, + count: Variable | None = None, + elide: bool | None = None, + ) -> str: ... # returns empty string for not found + def see(self, index: _TextIndex) -> None: ... + def tag_add(self, tagName: str, index1: _TextIndex, *args: _TextIndex) -> None: ... + # tag_bind stuff is very similar to Canvas + @overload + def tag_bind( + self, + tagName: str, + sequence: str | None, + func: Callable[[Event[Text]], object] | None, + add: Literal["", "+"] | bool | None = None, + ) -> str: ... + @overload + def tag_bind(self, tagName: str, sequence: str | None, func: str, add: Literal["", "+"] | bool | None = None) -> None: ... + def tag_unbind(self, tagName: str, sequence: str, funcid: str | None = None) -> None: ... + # allowing any string for cget instead of just Literals because there's no other way to look up tag options + def tag_cget(self, tagName: str, option: str): ... + @overload + def tag_configure( + self, + tagName: str, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bgstipple: str = ..., + borderwidth: _ScreenUnits = ..., + border: _ScreenUnits = ..., # alias for borderwidth + elide: bool = ..., + fgstipple: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + justify: Literal["left", "right", "center"] = ..., + lmargin1: _ScreenUnits = ..., + lmargin2: _ScreenUnits = ..., + lmargincolor: str = ..., + offset: _ScreenUnits = ..., + overstrike: bool = ..., + overstrikefg: str = ..., + relief: _Relief = ..., + rmargin: _ScreenUnits = ..., + rmargincolor: str = ..., + selectbackground: str = ..., + selectforeground: str = ..., + spacing1: _ScreenUnits = ..., + spacing2: _ScreenUnits = ..., + spacing3: _ScreenUnits = ..., + tabs: Any = ..., # the exact type is kind of complicated, see manual page + tabstyle: Literal["tabular", "wordprocessor"] = ..., + underline: bool = ..., + underlinefg: str = ..., + wrap: Literal["none", "char", "word"] = ..., # be careful with "none" vs None + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def tag_configure(self, tagName: str, cnf: str) -> tuple[str, str, str, Any, Any]: ... + tag_config = tag_configure + def tag_delete(self, first_tag_name: str, /, *tagNames: str) -> None: ... # error if no tag names given + def tag_lower(self, tagName: str, belowThis: str | None = None) -> None: ... + def tag_names(self, index: _TextIndex | None = None) -> tuple[str, ...]: ... + def tag_nextrange( + self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None + ) -> tuple[str, str] | tuple[()]: ... + def tag_prevrange( + self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None + ) -> tuple[str, str] | tuple[()]: ... + def tag_raise(self, tagName: str, aboveThis: str | None = None) -> None: ... + def tag_ranges(self, tagName: str) -> tuple[_tkinter.Tcl_Obj, ...]: ... + # tag_remove and tag_delete are different + def tag_remove(self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None) -> None: ... + @overload + def window_cget(self, index: _TextIndex, option: Literal["padx", "pady"]) -> int: ... + @overload + def window_cget(self, index: _TextIndex, option: Literal["stretch"]) -> bool: ... # actually returns Literal[0, 1] + @overload + def window_cget(self, index: _TextIndex, option: Literal["align"]) -> Literal["baseline", "bottom", "center", "top"]: ... + @overload # window is set to a widget, but read as the string name. + def window_cget(self, index: _TextIndex, option: Literal["create", "window"]) -> str: ... + @overload + def window_cget(self, index: _TextIndex, option: str) -> Any: ... + @overload + def window_configure(self, index: _TextIndex, cnf: str) -> tuple[str, str, str, str, str | int]: ... + @overload + def window_configure( + self, + index: _TextIndex, + cnf: dict[str, Any] | None = None, + *, + align: Literal["baseline", "bottom", "center", "top"] = ..., + create: str = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + stretch: bool | Literal[0, 1] = ..., + window: Misc | str = ..., + ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... + window_config = window_configure + def window_create( + self, + index: _TextIndex, + cnf: dict[str, Any] | None = {}, + *, + align: Literal["baseline", "bottom", "center", "top"] = ..., + create: str = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + stretch: bool | Literal[0, 1] = ..., + window: Misc | str = ..., + ) -> None: ... + def window_names(self) -> tuple[str, ...]: ... + def yview_pickplace(self, *what): ... # deprecated + +class _setit: + def __init__(self, var, value, callback: Incomplete | None = None) -> None: ... + def __call__(self, *args) -> None: ... + +# manual page: tk_optionMenu +class OptionMenu(Menubutton): + widgetName: Incomplete + menuname: Incomplete + def __init__( + # differs from other widgets + self, + master: Misc | None, + variable: StringVar, + value: str, + *values: str, + # kwarg only from now on + command: Callable[[StringVar], object] | None = ..., + ) -> None: ... + # configure, config, cget are inherited from Menubutton + # destroy and __getitem__ are overridden, signature does not change + +# This matches tkinter's image classes (PhotoImage and BitmapImage) +# and PIL's tkinter-compatible class (PIL.ImageTk.PhotoImage), +# but not a plain PIL image that isn't tkinter compatible. +# The reason is that PIL has width and height attributes, not methods. +@type_check_only +class _Image(Protocol): + def width(self) -> int: ... + def height(self) -> int: ... + +@type_check_only +class _BitmapImageLike(_Image): ... + +@type_check_only +class _PhotoImageLike(_Image): ... + +class Image(_Image): + name: Incomplete + tk: _tkinter.TkappType + def __init__( + self, imgtype, name: Incomplete | None = None, cnf={}, master: Misc | _tkinter.TkappType | None = None, **kw + ) -> None: ... + def __del__(self) -> None: ... + def __setitem__(self, key, value) -> None: ... + def __getitem__(self, key): ... + configure: Incomplete + config: Incomplete + def type(self): ... + +class PhotoImage(Image, _PhotoImageLike): + # This should be kept in sync with PIL.ImageTK.PhotoImage.__init__() + def __init__( + self, + name: str | None = None, + cnf: dict[str, Any] = {}, + master: Misc | _tkinter.TkappType | None = None, + *, + data: str | bytes = ..., # not same as data argument of put() + format: str = ..., + file: StrOrBytesPath = ..., + gamma: float = ..., + height: int = ..., + palette: int | str = ..., + width: int = ..., + ) -> None: ... + def configure( + self, + *, + data: str | bytes = ..., + format: str = ..., + file: StrOrBytesPath = ..., + gamma: float = ..., + height: int = ..., + palette: int | str = ..., + width: int = ..., + ) -> None: ... + config = configure + def blank(self) -> None: ... + def cget(self, option: str) -> str: ... + def __getitem__(self, key: str) -> str: ... # always string: image['height'] can be '0' + if sys.version_info >= (3, 13): + def copy( + self, + *, + from_coords: Iterable[int] | None = None, + zoom: int | tuple[int, int] | list[int] | None = None, + subsample: int | tuple[int, int] | list[int] | None = None, + ) -> PhotoImage: ... + def subsample(self, x: int, y: Literal[""] = "", *, from_coords: Iterable[int] | None = None) -> PhotoImage: ... + def zoom(self, x: int, y: Literal[""] = "", *, from_coords: Iterable[int] | None = None) -> PhotoImage: ... + def copy_replace( + self, + sourceImage: PhotoImage | str, + *, + from_coords: Iterable[int] | None = None, + to: Iterable[int] | None = None, + shrink: bool = False, + zoom: int | tuple[int, int] | list[int] | None = None, + subsample: int | tuple[int, int] | list[int] | None = None, + # `None` defaults to overlay. + compositingrule: Literal["overlay", "set"] | None = None, + ) -> None: ... + else: + def copy(self) -> PhotoImage: ... + def zoom(self, x: int, y: int | Literal[""] = "") -> PhotoImage: ... + def subsample(self, x: int, y: int | Literal[""] = "") -> PhotoImage: ... + + def get(self, x: int, y: int) -> tuple[int, int, int]: ... + def put( + self, + data: ( + str + | list[str] + | list[list[str]] + | list[tuple[str, ...]] + | tuple[str, ...] + | tuple[list[str], ...] + | tuple[tuple[str, ...], ...] + ), + to: tuple[int, int] | None = None, + ) -> None: ... + if sys.version_info >= (3, 13): + def read( + self, + filename: StrOrBytesPath, + format: str | None = None, + *, + from_coords: Iterable[int] | None = None, + to: Iterable[int] | None = None, + shrink: bool = False, + ) -> None: ... + def write( + self, + filename: StrOrBytesPath, + format: str | None = None, + from_coords: Iterable[int] | None = None, + *, + background: str | None = None, + grayscale: bool = False, + ) -> None: ... + @overload + def data( + self, format: str, *, from_coords: Iterable[int] | None = None, background: str | None = None, grayscale: bool = False + ) -> bytes: ... + @overload + def data( + self, + format: None = None, + *, + from_coords: Iterable[int] | None = None, + background: str | None = None, + grayscale: bool = False, + ) -> tuple[str, ...]: ... + + else: + def write( + self, filename: StrOrBytesPath, format: str | None = None, from_coords: tuple[int, int] | None = None + ) -> None: ... + + def transparency_get(self, x: int, y: int) -> bool: ... + def transparency_set(self, x: int, y: int, boolean: bool) -> None: ... + +class BitmapImage(Image, _BitmapImageLike): + # This should be kept in sync with PIL.ImageTK.BitmapImage.__init__() + def __init__( + self, + name: Incomplete | None = None, + cnf: dict[str, Any] = {}, + master: Misc | _tkinter.TkappType | None = None, + *, + background: str = ..., + data: str | bytes = ..., + file: StrOrBytesPath = ..., + foreground: str = ..., + maskdata: str = ..., + maskfile: StrOrBytesPath = ..., + ) -> None: ... + +def image_names() -> tuple[str, ...]: ... +def image_types() -> tuple[str, ...]: ... + +class Spinbox(Widget, XView): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + activebackground: str = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + buttonbackground: str = ..., + buttoncursor: _Cursor = "", + buttondownrelief: _Relief = ..., + buttonuprelief: _Relief = ..., + # percent substitutions don't seem to be supported, it's similar to Entry's validation stuff + command: Callable[[], object] | str | list[str] | tuple[str, ...] = "", + cursor: _Cursor = "xterm", + disabledbackground: str = ..., + disabledforeground: str = ..., + exportselection: bool = True, + fg: str = ..., + font: _FontDescription = "TkTextFont", + foreground: str = ..., + format: str = "", + from_: float = 0.0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + increment: float = 1.0, + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = 0, + insertofftime: int = 300, + insertontime: int = 600, + insertwidth: _ScreenUnits = ..., + invalidcommand: _EntryValidateCommand = "", + invcmd: _EntryValidateCommand = "", + justify: Literal["left", "center", "right"] = "left", + name: str = ..., + readonlybackground: str = ..., + relief: _Relief = "sunken", + repeatdelay: int = 400, + repeatinterval: int = 100, + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + state: Literal["normal", "disabled", "readonly"] = "normal", + takefocus: _TakeFocusValue = "", + textvariable: Variable = ..., + to: float = 0.0, + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", + validatecommand: _EntryValidateCommand = "", + vcmd: _EntryValidateCommand = "", + values: list[str] | tuple[str, ...] = ..., + width: int = 20, + wrap: bool = False, + xscrollcommand: _XYScrollCommand = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + activebackground: str = ..., + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + buttonbackground: str = ..., + buttoncursor: _Cursor = ..., + buttondownrelief: _Relief = ..., + buttonuprelief: _Relief = ..., + command: Callable[[], object] | str | list[str] | tuple[str, ...] = ..., + cursor: _Cursor = ..., + disabledbackground: str = ..., + disabledforeground: str = ..., + exportselection: bool = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + format: str = ..., + from_: float = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + increment: float = ..., + insertbackground: str = ..., + insertborderwidth: _ScreenUnits = ..., + insertofftime: int = ..., + insertontime: int = ..., + insertwidth: _ScreenUnits = ..., + invalidcommand: _EntryValidateCommand = ..., + invcmd: _EntryValidateCommand = ..., + justify: Literal["left", "center", "right"] = ..., + readonlybackground: str = ..., + relief: _Relief = ..., + repeatdelay: int = ..., + repeatinterval: int = ..., + selectbackground: str = ..., + selectborderwidth: _ScreenUnits = ..., + selectforeground: str = ..., + state: Literal["normal", "disabled", "readonly"] = ..., + takefocus: _TakeFocusValue = ..., + textvariable: Variable = ..., + to: float = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., + validatecommand: _EntryValidateCommand = ..., + vcmd: _EntryValidateCommand = ..., + values: list[str] | tuple[str, ...] = ..., + width: int = ..., + wrap: bool = ..., + xscrollcommand: _XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def bbox(self, index) -> tuple[int, int, int, int] | None: ... # type: ignore[override] + def delete(self, first, last: Incomplete | None = None) -> Literal[""]: ... + def get(self) -> str: ... + def icursor(self, index): ... + def identify(self, x: int, y: int) -> Literal["", "buttondown", "buttonup", "entry"]: ... + def index(self, index: str | int) -> int: ... + def insert(self, index: str | int, s: str) -> Literal[""]: ... + # spinbox.invoke("asdf") gives error mentioning .invoke("none"), but it's not documented + def invoke(self, element: Literal["none", "buttonup", "buttondown"]) -> Literal[""]: ... + def scan(self, *args): ... + def scan_mark(self, x): ... + def scan_dragto(self, x): ... + def selection(self, *args) -> tuple[int, ...]: ... + def selection_adjust(self, index): ... + def selection_clear(self): ... # type: ignore[override] + def selection_element(self, element: Incomplete | None = None): ... + def selection_from(self, index: int) -> None: ... + def selection_present(self) -> None: ... + def selection_range(self, start: int, end: int) -> None: ... + def selection_to(self, index: int) -> None: ... + +class LabelFrame(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + background: str = ..., + bd: _ScreenUnits = 2, + bg: str = ..., + border: _ScreenUnits = 2, + borderwidth: _ScreenUnits = 2, + class_: str = "Labelframe", # can't be changed with configure() + colormap: Literal["new", ""] | Misc = "", # can't be changed with configure() + container: bool = False, # undocumented, can't be changed with configure() + cursor: _Cursor = "", + fg: str = ..., + font: _FontDescription = "TkDefaultFont", + foreground: str = ..., + height: _ScreenUnits = 0, + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = 0, + # 'ne' and 'en' are valid labelanchors, but only 'ne' is a valid _Anchor. + labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = "nw", + labelwidget: Misc = ..., + name: str = ..., + padx: _ScreenUnits = 0, + pady: _ScreenUnits = 0, + relief: _Relief = "groove", + takefocus: _TakeFocusValue = 0, + text: float | str = "", + visual: str | tuple[str, int] = "", # can't be changed with configure() + width: _ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + fg: str = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: _ScreenUnits = ..., + highlightbackground: str = ..., + highlightcolor: str = ..., + highlightthickness: _ScreenUnits = ..., + labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., + labelwidget: Misc = ..., + padx: _ScreenUnits = ..., + pady: _ScreenUnits = ..., + relief: _Relief = ..., + takefocus: _TakeFocusValue = ..., + text: float | str = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class PanedWindow(Widget): + def __init__( + self, + master: Misc | None = None, + cnf: dict[str, Any] | None = {}, + *, + background: str = ..., + bd: _ScreenUnits = 1, + bg: str = ..., + border: _ScreenUnits = 1, + borderwidth: _ScreenUnits = 1, + cursor: _Cursor = "", + handlepad: _ScreenUnits = 8, + handlesize: _ScreenUnits = 8, + height: _ScreenUnits = "", + name: str = ..., + opaqueresize: bool = True, + orient: Literal["horizontal", "vertical"] = "horizontal", + proxybackground: str = "", + proxyborderwidth: _ScreenUnits = 2, + proxyrelief: _Relief = "flat", + relief: _Relief = "flat", + sashcursor: _Cursor = "", + sashpad: _ScreenUnits = 0, + sashrelief: _Relief = "flat", + sashwidth: _ScreenUnits = 3, + showhandle: bool = False, + width: _ScreenUnits = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + bd: _ScreenUnits = ..., + bg: str = ..., + border: _ScreenUnits = ..., + borderwidth: _ScreenUnits = ..., + cursor: _Cursor = ..., + handlepad: _ScreenUnits = ..., + handlesize: _ScreenUnits = ..., + height: _ScreenUnits = ..., + opaqueresize: bool = ..., + orient: Literal["horizontal", "vertical"] = ..., + proxybackground: str = ..., + proxyborderwidth: _ScreenUnits = ..., + proxyrelief: _Relief = ..., + relief: _Relief = ..., + sashcursor: _Cursor = ..., + sashpad: _ScreenUnits = ..., + sashrelief: _Relief = ..., + sashwidth: _ScreenUnits = ..., + showhandle: bool = ..., + width: _ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def add(self, child: Widget, **kw) -> None: ... + def remove(self, child) -> None: ... + forget: Incomplete + def identify(self, x: int, y: int): ... + def proxy(self, *args): ... + def proxy_coord(self): ... + def proxy_forget(self): ... + def proxy_place(self, x, y): ... + def sash(self, *args): ... + def sash_coord(self, index): ... + def sash_mark(self, index): ... + def sash_place(self, index, x, y): ... + def panecget(self, child, option): ... + def paneconfigure(self, tagOrId, cnf: Incomplete | None = None, **kw): ... + paneconfig: Incomplete + def panes(self): ... + +def _test() -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/colorchooser.pyi b/mypy/typeshed/stdlib/tkinter/colorchooser.pyi new file mode 100644 index 000000000000..09bc8cbb4f1e --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/colorchooser.pyi @@ -0,0 +1,20 @@ +import sys +from tkinter import Misc +from tkinter.commondialog import Dialog +from typing import ClassVar + +if sys.version_info >= (3, 9): + __all__ = ["Chooser", "askcolor"] + +class Chooser(Dialog): + command: ClassVar[str] + +if sys.version_info >= (3, 9): + def askcolor( + color: str | bytes | None = None, *, initialcolor: str = ..., parent: Misc = ..., title: str = ... + ) -> tuple[None, None] | tuple[tuple[int, int, int], str]: ... + +else: + def askcolor( + color: str | bytes | None = None, *, initialcolor: str = ..., parent: Misc = ..., title: str = ... + ) -> tuple[None, None] | tuple[tuple[float, float, float], str]: ... diff --git a/mypy/typeshed/stdlib/tkinter/commondialog.pyi b/mypy/typeshed/stdlib/tkinter/commondialog.pyi new file mode 100644 index 000000000000..d06c08df5b76 --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/commondialog.pyi @@ -0,0 +1,14 @@ +import sys +from _typeshed import Incomplete +from collections.abc import Mapping +from typing import ClassVar + +if sys.version_info >= (3, 9): + __all__ = ["Dialog"] + +class Dialog: + command: ClassVar[str | None] + master: Incomplete | None + options: Mapping[str, Incomplete] + def __init__(self, master: Incomplete | None = None, **options) -> None: ... + def show(self, **options): ... diff --git a/mypy/typeshed/stdlib/tkinter/constants.pyi b/mypy/typeshed/stdlib/tkinter/constants.pyi new file mode 100644 index 000000000000..fbfe8b49b997 --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/constants.pyi @@ -0,0 +1,80 @@ +from typing import Final + +# These are not actually bools. See #4669 +NO: Final[bool] +YES: Final[bool] +TRUE: Final[bool] +FALSE: Final[bool] +ON: Final[bool] +OFF: Final[bool] +N: Final = "n" +S: Final = "s" +W: Final = "w" +E: Final = "e" +NW: Final = "nw" +SW: Final = "sw" +NE: Final = "ne" +SE: Final = "se" +NS: Final = "ns" +EW: Final = "ew" +NSEW: Final = "nsew" +CENTER: Final = "center" +NONE: Final = "none" +X: Final = "x" +Y: Final = "y" +BOTH: Final = "both" +LEFT: Final = "left" +TOP: Final = "top" +RIGHT: Final = "right" +BOTTOM: Final = "bottom" +RAISED: Final = "raised" +SUNKEN: Final = "sunken" +FLAT: Final = "flat" +RIDGE: Final = "ridge" +GROOVE: Final = "groove" +SOLID: Final = "solid" +HORIZONTAL: Final = "horizontal" +VERTICAL: Final = "vertical" +NUMERIC: Final = "numeric" +CHAR: Final = "char" +WORD: Final = "word" +BASELINE: Final = "baseline" +INSIDE: Final = "inside" +OUTSIDE: Final = "outside" +SEL: Final = "sel" +SEL_FIRST: Final = "sel.first" +SEL_LAST: Final = "sel.last" +END: Final = "end" +INSERT: Final = "insert" +CURRENT: Final = "current" +ANCHOR: Final = "anchor" +ALL: Final = "all" +NORMAL: Final = "normal" +DISABLED: Final = "disabled" +ACTIVE: Final = "active" +HIDDEN: Final = "hidden" +CASCADE: Final = "cascade" +CHECKBUTTON: Final = "checkbutton" +COMMAND: Final = "command" +RADIOBUTTON: Final = "radiobutton" +SEPARATOR: Final = "separator" +SINGLE: Final = "single" +BROWSE: Final = "browse" +MULTIPLE: Final = "multiple" +EXTENDED: Final = "extended" +DOTBOX: Final = "dotbox" +UNDERLINE: Final = "underline" +PIESLICE: Final = "pieslice" +CHORD: Final = "chord" +ARC: Final = "arc" +FIRST: Final = "first" +LAST: Final = "last" +BUTT: Final = "butt" +PROJECTING: Final = "projecting" +ROUND: Final = "round" +BEVEL: Final = "bevel" +MITER: Final = "miter" +MOVETO: Final = "moveto" +SCROLL: Final = "scroll" +UNITS: Final = "units" +PAGES: Final = "pages" diff --git a/mypy/typeshed/stdlib/tkinter/dialog.pyi b/mypy/typeshed/stdlib/tkinter/dialog.pyi new file mode 100644 index 000000000000..b7d74c0fa71e --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/dialog.pyi @@ -0,0 +1,16 @@ +import sys +from _typeshed import Incomplete +from collections.abc import Mapping +from tkinter import Widget +from typing import Any, Final + +if sys.version_info >= (3, 9): + __all__ = ["Dialog"] + +DIALOG_ICON: Final = "questhead" + +class Dialog(Widget): + widgetName: str + num: int + def __init__(self, master: Incomplete | None = None, cnf: Mapping[str, Any] = {}, **kw) -> None: ... + def destroy(self) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/dnd.pyi b/mypy/typeshed/stdlib/tkinter/dnd.pyi new file mode 100644 index 000000000000..d806be74068e --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/dnd.pyi @@ -0,0 +1,20 @@ +import sys +from tkinter import Event, Misc, Tk, Widget +from typing import ClassVar, Protocol + +if sys.version_info >= (3, 9): + __all__ = ["dnd_start", "DndHandler"] + +class _DndSource(Protocol): + def dnd_end(self, target: Widget | None, event: Event[Misc] | None, /) -> None: ... + +class DndHandler: + root: ClassVar[Tk | None] + def __init__(self, source: _DndSource, event: Event[Misc]) -> None: ... + def cancel(self, event: Event[Misc] | None = None) -> None: ... + def finish(self, event: Event[Misc] | None, commit: int = 0) -> None: ... + def on_motion(self, event: Event[Misc]) -> None: ... + def on_release(self, event: Event[Misc]) -> None: ... + def __del__(self) -> None: ... + +def dnd_start(source: _DndSource, event: Event[Misc]) -> DndHandler | None: ... diff --git a/mypy/typeshed/stdlib/tkinter/filedialog.pyi b/mypy/typeshed/stdlib/tkinter/filedialog.pyi new file mode 100644 index 000000000000..3d62f079178e --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/filedialog.pyi @@ -0,0 +1,151 @@ +import sys +from _typeshed import Incomplete, StrOrBytesPath +from collections.abc import Iterable +from tkinter import Button, Entry, Frame, Listbox, Misc, Scrollbar, StringVar, Toplevel, commondialog +from typing import IO, ClassVar, Literal + +if sys.version_info >= (3, 9): + __all__ = [ + "FileDialog", + "LoadFileDialog", + "SaveFileDialog", + "Open", + "SaveAs", + "Directory", + "askopenfilename", + "asksaveasfilename", + "askopenfilenames", + "askopenfile", + "askopenfiles", + "asksaveasfile", + "askdirectory", + ] + +dialogstates: dict[Incomplete, tuple[Incomplete, Incomplete]] + +class FileDialog: + title: str + master: Incomplete + directory: Incomplete | None + top: Toplevel + botframe: Frame + selection: Entry + filter: Entry + midframe: Entry + filesbar: Scrollbar + files: Listbox + dirsbar: Scrollbar + dirs: Listbox + ok_button: Button + filter_button: Button + cancel_button: Button + def __init__( + self, master, title: Incomplete | None = None + ) -> None: ... # title is usually a str or None, but e.g. int doesn't raise en exception either + how: Incomplete | None + def go(self, dir_or_file=".", pattern: str = "*", default: str = "", key: Incomplete | None = None): ... + def quit(self, how: Incomplete | None = None) -> None: ... + def dirs_double_event(self, event) -> None: ... + def dirs_select_event(self, event) -> None: ... + def files_double_event(self, event) -> None: ... + def files_select_event(self, event) -> None: ... + def ok_event(self, event) -> None: ... + def ok_command(self) -> None: ... + def filter_command(self, event: Incomplete | None = None) -> None: ... + def get_filter(self): ... + def get_selection(self): ... + def cancel_command(self, event: Incomplete | None = None) -> None: ... + def set_filter(self, dir, pat) -> None: ... + def set_selection(self, file) -> None: ... + +class LoadFileDialog(FileDialog): + title: str + def ok_command(self) -> None: ... + +class SaveFileDialog(FileDialog): + title: str + def ok_command(self) -> None: ... + +class _Dialog(commondialog.Dialog): ... + +class Open(_Dialog): + command: ClassVar[str] + +class SaveAs(_Dialog): + command: ClassVar[str] + +class Directory(commondialog.Dialog): + command: ClassVar[str] + +# TODO: command kwarg available on macos +def asksaveasfilename( + *, + confirmoverwrite: bool | None = ..., + defaultextension: str | None = ..., + filetypes: Iterable[tuple[str, str | list[str] | tuple[str, ...]]] | None = ..., + initialdir: StrOrBytesPath | None = ..., + initialfile: StrOrBytesPath | None = ..., + parent: Misc | None = ..., + title: str | None = ..., + typevariable: StringVar | str | None = ..., +) -> str: ... # can be empty string +def askopenfilename( + *, + defaultextension: str | None = ..., + filetypes: Iterable[tuple[str, str | list[str] | tuple[str, ...]]] | None = ..., + initialdir: StrOrBytesPath | None = ..., + initialfile: StrOrBytesPath | None = ..., + parent: Misc | None = ..., + title: str | None = ..., + typevariable: StringVar | str | None = ..., +) -> str: ... # can be empty string +def askopenfilenames( + *, + defaultextension: str | None = ..., + filetypes: Iterable[tuple[str, str | list[str] | tuple[str, ...]]] | None = ..., + initialdir: StrOrBytesPath | None = ..., + initialfile: StrOrBytesPath | None = ..., + parent: Misc | None = ..., + title: str | None = ..., + typevariable: StringVar | str | None = ..., +) -> Literal[""] | tuple[str, ...]: ... +def askdirectory( + *, initialdir: StrOrBytesPath | None = ..., mustexist: bool | None = ..., parent: Misc | None = ..., title: str | None = ... +) -> str: ... # can be empty string + +# TODO: If someone actually uses these, overload to have the actual return type of open(..., mode) +def asksaveasfile( + mode: str = "w", + *, + confirmoverwrite: bool | None = ..., + defaultextension: str | None = ..., + filetypes: Iterable[tuple[str, str | list[str] | tuple[str, ...]]] | None = ..., + initialdir: StrOrBytesPath | None = ..., + initialfile: StrOrBytesPath | None = ..., + parent: Misc | None = ..., + title: str | None = ..., + typevariable: StringVar | str | None = ..., +) -> IO[Incomplete] | None: ... +def askopenfile( + mode: str = "r", + *, + defaultextension: str | None = ..., + filetypes: Iterable[tuple[str, str | list[str] | tuple[str, ...]]] | None = ..., + initialdir: StrOrBytesPath | None = ..., + initialfile: StrOrBytesPath | None = ..., + parent: Misc | None = ..., + title: str | None = ..., + typevariable: StringVar | str | None = ..., +) -> IO[Incomplete] | None: ... +def askopenfiles( + mode: str = "r", + *, + defaultextension: str | None = ..., + filetypes: Iterable[tuple[str, str | list[str] | tuple[str, ...]]] | None = ..., + initialdir: StrOrBytesPath | None = ..., + initialfile: StrOrBytesPath | None = ..., + parent: Misc | None = ..., + title: str | None = ..., + typevariable: StringVar | str | None = ..., +) -> tuple[IO[Incomplete], ...]: ... # can be empty tuple +def test() -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/font.pyi b/mypy/typeshed/stdlib/tkinter/font.pyi new file mode 100644 index 000000000000..097c2e4b4382 --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/font.pyi @@ -0,0 +1,118 @@ +import _tkinter +import itertools +import sys +import tkinter +from typing import Any, ClassVar, Final, Literal, TypedDict, overload +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 9): + __all__ = ["NORMAL", "ROMAN", "BOLD", "ITALIC", "nametofont", "Font", "families", "names"] + +NORMAL: Final = "normal" +ROMAN: Final = "roman" +BOLD: Final = "bold" +ITALIC: Final = "italic" + +_FontDescription: TypeAlias = ( + str # "Helvetica 12" + | Font # A font object constructed in Python + | list[Any] # ["Helvetica", 12, BOLD] + | tuple[str] # ("Liberation Sans",) needs wrapping in tuple/list to handle spaces + | tuple[str, int] # ("Liberation Sans", 12) + | tuple[str, int, str] # ("Liberation Sans", 12, "bold") + | tuple[str, int, list[str] | tuple[str, ...]] # e.g. bold and italic + | _tkinter.Tcl_Obj # A font object constructed in Tcl +) + +class _FontDict(TypedDict): + family: str + size: int + weight: Literal["normal", "bold"] + slant: Literal["roman", "italic"] + underline: bool + overstrike: bool + +class _MetricsDict(TypedDict): + ascent: int + descent: int + linespace: int + fixed: bool + +class Font: + name: str + delete_font: bool + counter: ClassVar[itertools.count[int]] # undocumented + def __init__( + self, + # In tkinter, 'root' refers to tkinter.Tk by convention, but the code + # actually works with any tkinter widget so we use tkinter.Misc. + root: tkinter.Misc | None = None, + font: _FontDescription | None = None, + name: str | None = None, + exists: bool = False, + *, + family: str = ..., + size: int = ..., + weight: Literal["normal", "bold"] = ..., + slant: Literal["roman", "italic"] = ..., + underline: bool = ..., + overstrike: bool = ..., + ) -> None: ... + def __setitem__(self, key: str, value: Any) -> None: ... + @overload + def cget(self, option: Literal["family"]) -> str: ... + @overload + def cget(self, option: Literal["size"]) -> int: ... + @overload + def cget(self, option: Literal["weight"]) -> Literal["normal", "bold"]: ... + @overload + def cget(self, option: Literal["slant"]) -> Literal["roman", "italic"]: ... + @overload + def cget(self, option: Literal["underline", "overstrike"]) -> bool: ... + @overload + def cget(self, option: str) -> Any: ... + __getitem__ = cget + @overload + def actual(self, option: Literal["family"], displayof: tkinter.Misc | None = None) -> str: ... + @overload + def actual(self, option: Literal["size"], displayof: tkinter.Misc | None = None) -> int: ... + @overload + def actual(self, option: Literal["weight"], displayof: tkinter.Misc | None = None) -> Literal["normal", "bold"]: ... + @overload + def actual(self, option: Literal["slant"], displayof: tkinter.Misc | None = None) -> Literal["roman", "italic"]: ... + @overload + def actual(self, option: Literal["underline", "overstrike"], displayof: tkinter.Misc | None = None) -> bool: ... + @overload + def actual(self, option: None, displayof: tkinter.Misc | None = None) -> _FontDict: ... + @overload + def actual(self, *, displayof: tkinter.Misc | None = None) -> _FontDict: ... + def config( + self, + *, + family: str = ..., + size: int = ..., + weight: Literal["normal", "bold"] = ..., + slant: Literal["roman", "italic"] = ..., + underline: bool = ..., + overstrike: bool = ..., + ) -> _FontDict | None: ... + configure = config + def copy(self) -> Font: ... + @overload + def metrics(self, option: Literal["ascent", "descent", "linespace"], /, *, displayof: tkinter.Misc | None = ...) -> int: ... + @overload + def metrics(self, option: Literal["fixed"], /, *, displayof: tkinter.Misc | None = ...) -> bool: ... + @overload + def metrics(self, *, displayof: tkinter.Misc | None = ...) -> _MetricsDict: ... + def measure(self, text: str, displayof: tkinter.Misc | None = None) -> int: ... + def __eq__(self, other: object) -> bool: ... + def __del__(self) -> None: ... + +def families(root: tkinter.Misc | None = None, displayof: tkinter.Misc | None = None) -> tuple[str, ...]: ... +def names(root: tkinter.Misc | None = None) -> tuple[str, ...]: ... + +if sys.version_info >= (3, 10): + def nametofont(name: str, root: tkinter.Misc | None = None) -> Font: ... + +else: + def nametofont(name: str) -> Font: ... diff --git a/mypy/typeshed/stdlib/tkinter/messagebox.pyi b/mypy/typeshed/stdlib/tkinter/messagebox.pyi new file mode 100644 index 000000000000..5cdfe512f9b7 --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/messagebox.pyi @@ -0,0 +1,44 @@ +import sys +from tkinter.commondialog import Dialog +from typing import ClassVar, Final + +if sys.version_info >= (3, 9): + __all__ = [ + "showinfo", + "showwarning", + "showerror", + "askquestion", + "askokcancel", + "askyesno", + "askyesnocancel", + "askretrycancel", + ] + +ERROR: Final = "error" +INFO: Final = "info" +QUESTION: Final = "question" +WARNING: Final = "warning" +ABORTRETRYIGNORE: Final = "abortretryignore" +OK: Final = "ok" +OKCANCEL: Final = "okcancel" +RETRYCANCEL: Final = "retrycancel" +YESNO: Final = "yesno" +YESNOCANCEL: Final = "yesnocancel" +ABORT: Final = "abort" +RETRY: Final = "retry" +IGNORE: Final = "ignore" +CANCEL: Final = "cancel" +YES: Final = "yes" +NO: Final = "no" + +class Message(Dialog): + command: ClassVar[str] + +def showinfo(title: str | None = None, message: str | None = None, **options) -> str: ... +def showwarning(title: str | None = None, message: str | None = None, **options) -> str: ... +def showerror(title: str | None = None, message: str | None = None, **options) -> str: ... +def askquestion(title: str | None = None, message: str | None = None, **options) -> str: ... +def askokcancel(title: str | None = None, message: str | None = None, **options) -> bool: ... +def askyesno(title: str | None = None, message: str | None = None, **options) -> bool: ... +def askyesnocancel(title: str | None = None, message: str | None = None, **options) -> bool | None: ... +def askretrycancel(title: str | None = None, message: str | None = None, **options) -> bool: ... diff --git a/mypy/typeshed/stdlib/tkinter/scrolledtext.pyi b/mypy/typeshed/stdlib/tkinter/scrolledtext.pyi new file mode 100644 index 000000000000..6f1abc714487 --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/scrolledtext.pyi @@ -0,0 +1,9 @@ +from tkinter import Frame, Misc, Scrollbar, Text + +__all__ = ["ScrolledText"] + +# The methods from Pack, Place, and Grid are dynamically added over the parent's impls +class ScrolledText(Text): + frame: Frame + vbar: Scrollbar + def __init__(self, master: Misc | None = None, **kwargs) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/simpledialog.pyi b/mypy/typeshed/stdlib/tkinter/simpledialog.pyi new file mode 100644 index 000000000000..45dce21a6b1c --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/simpledialog.pyi @@ -0,0 +1,54 @@ +from tkinter import Event, Frame, Misc, Toplevel + +class Dialog(Toplevel): + def __init__(self, parent: Misc | None, title: str | None = None) -> None: ... + def body(self, master: Frame) -> Misc | None: ... + def buttonbox(self) -> None: ... + def ok(self, event: Event[Misc] | None = None) -> None: ... + def cancel(self, event: Event[Misc] | None = None) -> None: ... + def validate(self) -> bool: ... + def apply(self) -> None: ... + +class SimpleDialog: + def __init__( + self, + master: Misc | None, + text: str = "", + buttons: list[str] = [], + default: int | None = None, + cancel: int | None = None, + title: str | None = None, + class_: str | None = None, + ) -> None: ... + def go(self) -> int | None: ... + def return_event(self, event: Event[Misc]) -> None: ... + def wm_delete_window(self) -> None: ... + def done(self, num: int) -> None: ... + +def askfloat( + title: str | None, + prompt: str, + *, + initialvalue: float | None = ..., + minvalue: float | None = ..., + maxvalue: float | None = ..., + parent: Misc | None = ..., +) -> float | None: ... +def askinteger( + title: str | None, + prompt: str, + *, + initialvalue: int | None = ..., + minvalue: int | None = ..., + maxvalue: int | None = ..., + parent: Misc | None = ..., +) -> int | None: ... +def askstring( + title: str | None, + prompt: str, + *, + initialvalue: str | None = ..., + show: str | None = ..., + # minvalue/maxvalue is accepted but not useful. + parent: Misc | None = ..., +) -> str | None: ... diff --git a/mypy/typeshed/stdlib/tkinter/tix.pyi b/mypy/typeshed/stdlib/tkinter/tix.pyi new file mode 100644 index 000000000000..7891364fa02c --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/tix.pyi @@ -0,0 +1,299 @@ +import tkinter +from _typeshed import Incomplete +from typing import Any, Final + +WINDOW: Final = "window" +TEXT: Final = "text" +STATUS: Final = "status" +IMMEDIATE: Final = "immediate" +IMAGE: Final = "image" +IMAGETEXT: Final = "imagetext" +BALLOON: Final = "balloon" +AUTO: Final = "auto" +ACROSSTOP: Final = "acrosstop" + +ASCII: Final = "ascii" +CELL: Final = "cell" +COLUMN: Final = "column" +DECREASING: Final = "decreasing" +INCREASING: Final = "increasing" +INTEGER: Final = "integer" +MAIN: Final = "main" +MAX: Final = "max" +REAL: Final = "real" +ROW: Final = "row" +S_REGION: Final = "s-region" +X_REGION: Final = "x-region" +Y_REGION: Final = "y-region" + +# These should be kept in sync with _tkinter constants, except TCL_ALL_EVENTS which doesn't match ALL_EVENTS +TCL_DONT_WAIT: Final = 2 +TCL_WINDOW_EVENTS: Final = 4 +TCL_FILE_EVENTS: Final = 8 +TCL_TIMER_EVENTS: Final = 16 +TCL_IDLE_EVENTS: Final = 32 +TCL_ALL_EVENTS: Final = 0 + +class tixCommand: + def tix_addbitmapdir(self, directory: str) -> None: ... + def tix_cget(self, option: str) -> Any: ... + def tix_configure(self, cnf: dict[str, Any] | None = None, **kw: Any) -> Any: ... + def tix_filedialog(self, dlgclass: str | None = None) -> str: ... + def tix_getbitmap(self, name: str) -> str: ... + def tix_getimage(self, name: str) -> str: ... + def tix_option_get(self, name: str) -> Any: ... + def tix_resetoptions(self, newScheme: str, newFontSet: str, newScmPrio: str | None = None) -> None: ... + +class Tk(tkinter.Tk, tixCommand): + def __init__(self, screenName: str | None = None, baseName: str | None = None, className: str = "Tix") -> None: ... + +class TixWidget(tkinter.Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + widgetName: str | None = None, + static_options: list[str] | None = None, + cnf: dict[str, Any] = {}, + kw: dict[str, Any] = {}, + ) -> None: ... + def __getattr__(self, name: str): ... + def set_silent(self, value: str) -> None: ... + def subwidget(self, name: str) -> tkinter.Widget: ... + def subwidgets_all(self) -> list[tkinter.Widget]: ... + def config_all(self, option: Any, value: Any) -> None: ... + def image_create(self, imgtype: str, cnf: dict[str, Any] = {}, master: tkinter.Widget | None = None, **kw) -> None: ... + def image_delete(self, imgname: str) -> None: ... + +class TixSubWidget(TixWidget): + def __init__(self, master: tkinter.Widget, name: str, destroy_physically: int = 1, check_intermediate: int = 1) -> None: ... + +class DisplayStyle: + def __init__(self, itemtype: str, cnf: dict[str, Any] = {}, *, master: tkinter.Widget | None = None, **kw) -> None: ... + def __getitem__(self, key: str): ... + def __setitem__(self, key: str, value: Any) -> None: ... + def delete(self) -> None: ... + def config(self, cnf: dict[str, Any] = {}, **kw): ... + +class Balloon(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def bind_widget(self, widget: tkinter.Widget, cnf: dict[str, Any] = {}, **kw) -> None: ... + def unbind_widget(self, widget: tkinter.Widget) -> None: ... + +class ButtonBox(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add(self, name: str, cnf: dict[str, Any] = {}, **kw) -> tkinter.Widget: ... + def invoke(self, name: str) -> None: ... + +class ComboBox(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add_history(self, str: str) -> None: ... + def append_history(self, str: str) -> None: ... + def insert(self, index: int, str: str) -> None: ... + def pick(self, index: int) -> None: ... + +class Control(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def decrement(self) -> None: ... + def increment(self) -> None: ... + def invoke(self) -> None: ... + +class LabelEntry(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + +class LabelFrame(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + +class Meter(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + +class OptionMenu(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add_command(self, name: str, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add_separator(self, name: str, cnf: dict[str, Any] = {}, **kw) -> None: ... + def delete(self, name: str) -> None: ... + def disable(self, name: str) -> None: ... + def enable(self, name: str) -> None: ... + +class PopupMenu(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def bind_widget(self, widget: tkinter.Widget) -> None: ... + def unbind_widget(self, widget: tkinter.Widget) -> None: ... + def post_widget(self, widget: tkinter.Widget, x: int, y: int) -> None: ... + +class Select(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add(self, name: str, cnf: dict[str, Any] = {}, **kw) -> tkinter.Widget: ... + def invoke(self, name: str) -> None: ... + +class StdButtonBox(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def invoke(self, name: str) -> None: ... + +class DirList(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def chdir(self, dir: str) -> None: ... + +class DirTree(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def chdir(self, dir: str) -> None: ... + +class DirSelectDialog(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def popup(self) -> None: ... + def popdown(self) -> None: ... + +class DirSelectBox(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + +class ExFileSelectBox(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def filter(self) -> None: ... + def invoke(self) -> None: ... + +class FileSelectBox(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def apply_filter(self) -> None: ... + def invoke(self) -> None: ... + +class FileEntry(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def invoke(self) -> None: ... + def file_dialog(self) -> None: ... + +class HList(TixWidget, tkinter.XView, tkinter.YView): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add(self, entry: str, cnf: dict[str, Any] = {}, **kw) -> tkinter.Widget: ... + def add_child(self, parent: str | None = None, cnf: dict[str, Any] = {}, **kw) -> tkinter.Widget: ... + def anchor_set(self, entry: str) -> None: ... + def anchor_clear(self) -> None: ... + # FIXME: Overload, certain combos return, others don't + def column_width(self, col: int = 0, width: int | None = None, chars: int | None = None) -> int | None: ... + def delete_all(self) -> None: ... + def delete_entry(self, entry: str) -> None: ... + def delete_offsprings(self, entry: str) -> None: ... + def delete_siblings(self, entry: str) -> None: ... + def dragsite_set(self, index: int) -> None: ... + def dragsite_clear(self) -> None: ... + def dropsite_set(self, index: int) -> None: ... + def dropsite_clear(self) -> None: ... + def header_create(self, col: int, cnf: dict[str, Any] = {}, **kw) -> None: ... + def header_configure(self, col: int, cnf: dict[str, Any] = {}, **kw) -> Incomplete | None: ... + def header_cget(self, col: int, opt): ... + def header_exists(self, col: int) -> bool: ... + def header_exist(self, col: int) -> bool: ... + def header_delete(self, col: int) -> None: ... + def header_size(self, col: int) -> int: ... + def hide_entry(self, entry: str) -> None: ... + def indicator_create(self, entry: str, cnf: dict[str, Any] = {}, **kw) -> None: ... + def indicator_configure(self, entry: str, cnf: dict[str, Any] = {}, **kw) -> Incomplete | None: ... + def indicator_cget(self, entry: str, opt): ... + def indicator_exists(self, entry: str) -> bool: ... + def indicator_delete(self, entry: str) -> None: ... + def indicator_size(self, entry: str) -> int: ... + def info_anchor(self) -> str: ... + def info_bbox(self, entry: str) -> tuple[int, int, int, int]: ... + def info_children(self, entry: str | None = None) -> tuple[str, ...]: ... + def info_data(self, entry: str) -> Any: ... + def info_dragsite(self) -> str: ... + def info_dropsite(self) -> str: ... + def info_exists(self, entry: str) -> bool: ... + def info_hidden(self, entry: str) -> bool: ... + def info_next(self, entry: str) -> str: ... + def info_parent(self, entry: str) -> str: ... + def info_prev(self, entry: str) -> str: ... + def info_selection(self) -> tuple[str, ...]: ... + def item_cget(self, entry: str, col: int, opt): ... + def item_configure(self, entry: str, col: int, cnf: dict[str, Any] = {}, **kw) -> Incomplete | None: ... + def item_create(self, entry: str, col: int, cnf: dict[str, Any] = {}, **kw) -> None: ... + def item_exists(self, entry: str, col: int) -> bool: ... + def item_delete(self, entry: str, col: int) -> None: ... + def entrycget(self, entry: str, opt): ... + def entryconfigure(self, entry: str, cnf: dict[str, Any] = {}, **kw) -> Incomplete | None: ... + def nearest(self, y: int) -> str: ... + def see(self, entry: str) -> None: ... + def selection_clear(self, cnf: dict[str, Any] = {}, **kw) -> None: ... + def selection_includes(self, entry: str) -> bool: ... + def selection_set(self, first: str, last: str | None = None) -> None: ... + def show_entry(self, entry: str) -> None: ... + +class CheckList(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def autosetmode(self) -> None: ... + def close(self, entrypath: str) -> None: ... + def getmode(self, entrypath: str) -> str: ... + def open(self, entrypath: str) -> None: ... + def getselection(self, mode: str = "on") -> tuple[str, ...]: ... + def getstatus(self, entrypath: str) -> str: ... + def setstatus(self, entrypath: str, mode: str = "on") -> None: ... + +class Tree(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def autosetmode(self) -> None: ... + def close(self, entrypath: str) -> None: ... + def getmode(self, entrypath: str) -> str: ... + def open(self, entrypath: str) -> None: ... + def setmode(self, entrypath: str, mode: str = "none") -> None: ... + +class TList(TixWidget, tkinter.XView, tkinter.YView): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def active_set(self, index: int) -> None: ... + def active_clear(self) -> None: ... + def anchor_set(self, index: int) -> None: ... + def anchor_clear(self) -> None: ... + def delete(self, from_: int, to: int | None = None) -> None: ... + def dragsite_set(self, index: int) -> None: ... + def dragsite_clear(self) -> None: ... + def dropsite_set(self, index: int) -> None: ... + def dropsite_clear(self) -> None: ... + def insert(self, index: int, cnf: dict[str, Any] = {}, **kw) -> None: ... + def info_active(self) -> int: ... + def info_anchor(self) -> int: ... + def info_down(self, index: int) -> int: ... + def info_left(self, index: int) -> int: ... + def info_right(self, index: int) -> int: ... + def info_selection(self) -> tuple[int, ...]: ... + def info_size(self) -> int: ... + def info_up(self, index: int) -> int: ... + def nearest(self, x: int, y: int) -> int: ... + def see(self, index: int) -> None: ... + def selection_clear(self, cnf: dict[str, Any] = {}, **kw) -> None: ... + def selection_includes(self, index: int) -> bool: ... + def selection_set(self, first: int, last: int | None = None) -> None: ... + +class PanedWindow(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add(self, name: str, cnf: dict[str, Any] = {}, **kw) -> None: ... + def delete(self, name: str) -> None: ... + def forget(self, name: str) -> None: ... # type: ignore[override] + def panecget(self, entry: str, opt): ... + def paneconfigure(self, entry: str, cnf: dict[str, Any] = {}, **kw) -> Incomplete | None: ... + def panes(self) -> list[tkinter.Widget]: ... + +class ListNoteBook(TixWidget): + def __init__(self, master: tkinter.Widget | None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add(self, name: str, cnf: dict[str, Any] = {}, **kw) -> None: ... + def page(self, name: str) -> tkinter.Widget: ... + def pages(self) -> list[tkinter.Widget]: ... + def raise_page(self, name: str) -> None: ... + +class NoteBook(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + def add(self, name: str, cnf: dict[str, Any] = {}, **kw) -> None: ... + def delete(self, name: str) -> None: ... + def page(self, name: str) -> tkinter.Widget: ... + def pages(self) -> list[tkinter.Widget]: ... + def raise_page(self, name: str) -> None: ... + def raised(self) -> bool: ... + +class InputOnly(TixWidget): + def __init__(self, master: tkinter.Widget | None = None, cnf: dict[str, Any] = {}, **kw) -> None: ... + +class Form: + def __setitem__(self, key: str, value: Any) -> None: ... + def config(self, cnf: dict[str, Any] = {}, **kw) -> None: ... + def form(self, cnf: dict[str, Any] = {}, **kw) -> None: ... + def check(self) -> bool: ... + def forget(self) -> None: ... + def grid(self, xsize: int = 0, ysize: int = 0) -> tuple[int, int] | None: ... + def info(self, option: str | None = None): ... + def slaves(self) -> list[tkinter.Widget]: ... diff --git a/mypy/typeshed/stdlib/tkinter/ttk.pyi b/mypy/typeshed/stdlib/tkinter/ttk.pyi new file mode 100644 index 000000000000..dacef0620b22 --- /dev/null +++ b/mypy/typeshed/stdlib/tkinter/ttk.pyi @@ -0,0 +1,1207 @@ +import _tkinter +import tkinter +from _typeshed import Incomplete, MaybeNone +from collections.abc import Callable +from tkinter.font import _FontDescription +from typing import Any, Literal, TypedDict, overload +from typing_extensions import TypeAlias + +__all__ = [ + "Button", + "Checkbutton", + "Combobox", + "Entry", + "Frame", + "Label", + "Labelframe", + "LabelFrame", + "Menubutton", + "Notebook", + "Panedwindow", + "PanedWindow", + "Progressbar", + "Radiobutton", + "Scale", + "Scrollbar", + "Separator", + "Sizegrip", + "Style", + "Treeview", + "LabeledScale", + "OptionMenu", + "tclobjs_to_py", + "setup_master", + "Spinbox", +] + +def tclobjs_to_py(adict: dict[Any, Any]) -> dict[Any, Any]: ... +def setup_master(master: Incomplete | None = None): ... + +_Padding: TypeAlias = ( + tkinter._ScreenUnits + | tuple[tkinter._ScreenUnits] + | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits] + | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits] + | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits] +) + +# from ttk_widget (aka ttk::widget) manual page, differs from tkinter._Compound +_TtkCompound: TypeAlias = Literal["", "text", "image", tkinter._Compound] + +class Style: + master: Incomplete + tk: _tkinter.TkappType + def __init__(self, master: tkinter.Misc | None = None) -> None: ... + def configure(self, style, query_opt: Incomplete | None = None, **kw): ... + def map(self, style, query_opt: Incomplete | None = None, **kw): ... + def lookup(self, style, option, state: Incomplete | None = None, default: Incomplete | None = None): ... + def layout(self, style, layoutspec: Incomplete | None = None): ... + def element_create(self, elementname, etype, *args, **kw) -> None: ... + def element_names(self): ... + def element_options(self, elementname): ... + def theme_create(self, themename, parent: Incomplete | None = None, settings: Incomplete | None = None) -> None: ... + def theme_settings(self, themename, settings) -> None: ... + def theme_names(self) -> tuple[str, ...]: ... + @overload + def theme_use(self, themename: str) -> None: ... + @overload + def theme_use(self, themename: None = None) -> str: ... + +class Widget(tkinter.Widget): + def __init__(self, master: tkinter.Misc | None, widgetname, kw: Incomplete | None = None) -> None: ... + def identify(self, x: int, y: int) -> str: ... + def instate(self, statespec, callback: Incomplete | None = None, *args, **kw): ... + def state(self, statespec: Incomplete | None = None): ... + +class Button(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + command: tkinter._ButtonCommand = "", + compound: _TtkCompound = "", + cursor: tkinter._Cursor = "", + default: Literal["normal", "active", "disabled"] = "normal", + image: tkinter._ImageSpec = "", + name: str = ..., + padding=..., # undocumented + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = "", + textvariable: tkinter.Variable = ..., + underline: int = -1, + width: int | Literal[""] = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + command: tkinter._ButtonCommand = ..., + compound: _TtkCompound = ..., + cursor: tkinter._Cursor = ..., + default: Literal["normal", "active", "disabled"] = ..., + image: tkinter._ImageSpec = ..., + padding=..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = ..., + textvariable: tkinter.Variable = ..., + underline: int = ..., + width: int | Literal[""] = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def invoke(self) -> Any: ... + +class Checkbutton(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + command: tkinter._ButtonCommand = "", + compound: _TtkCompound = "", + cursor: tkinter._Cursor = "", + image: tkinter._ImageSpec = "", + name: str = ..., + offvalue: Any = 0, + onvalue: Any = 1, + padding=..., # undocumented + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = "", + textvariable: tkinter.Variable = ..., + underline: int = -1, + # Seems like variable can be empty string, but actually setting it to + # empty string segfaults before Tcl 8.6.9. Search for ttk::checkbutton + # here: https://sourceforge.net/projects/tcl/files/Tcl/8.6.9/tcltk-release-notes-8.6.9.txt/view + variable: tkinter.Variable = ..., + width: int | Literal[""] = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + command: tkinter._ButtonCommand = ..., + compound: _TtkCompound = ..., + cursor: tkinter._Cursor = ..., + image: tkinter._ImageSpec = ..., + offvalue: Any = ..., + onvalue: Any = ..., + padding=..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = ..., + textvariable: tkinter.Variable = ..., + underline: int = ..., + variable: tkinter.Variable = ..., + width: int | Literal[""] = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def invoke(self) -> Any: ... + +class Entry(Widget, tkinter.Entry): + def __init__( + self, + master: tkinter.Misc | None = None, + widget: str | None = None, + *, + background: str = ..., # undocumented + class_: str = "", + cursor: tkinter._Cursor = ..., + exportselection: bool = True, + font: _FontDescription = "TkTextFont", + foreground: str = "", + invalidcommand: tkinter._EntryValidateCommand = "", + justify: Literal["left", "center", "right"] = "left", + name: str = ..., + show: str = "", + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", + validatecommand: tkinter._EntryValidateCommand = "", + width: int = 20, + xscrollcommand: tkinter._XYScrollCommand = "", + ) -> None: ... + @overload # type: ignore[override] + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + cursor: tkinter._Cursor = ..., + exportselection: bool = ..., + font: _FontDescription = ..., + foreground: str = ..., + invalidcommand: tkinter._EntryValidateCommand = ..., + justify: Literal["left", "center", "right"] = ..., + show: str = ..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., + validatecommand: tkinter._EntryValidateCommand = ..., + width: int = ..., + xscrollcommand: tkinter._XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + # config must be copy/pasted, otherwise ttk.Entry().config is mypy error (don't know why) + @overload # type: ignore[override] + def config( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + cursor: tkinter._Cursor = ..., + exportselection: bool = ..., + font: _FontDescription = ..., + foreground: str = ..., + invalidcommand: tkinter._EntryValidateCommand = ..., + justify: Literal["left", "center", "right"] = ..., + show: str = ..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., + validatecommand: tkinter._EntryValidateCommand = ..., + width: int = ..., + xscrollcommand: tkinter._XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + def bbox(self, index) -> tuple[int, int, int, int]: ... # type: ignore[override] + def identify(self, x: int, y: int) -> str: ... + def validate(self): ... + +class Combobox(Entry): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + background: str = ..., # undocumented + class_: str = "", + cursor: tkinter._Cursor = "", + exportselection: bool = True, + font: _FontDescription = ..., # undocumented + foreground: str = ..., # undocumented + height: int = 10, + invalidcommand: tkinter._EntryValidateCommand = ..., # undocumented + justify: Literal["left", "center", "right"] = "left", + name: str = ..., + postcommand: Callable[[], object] | str = "", + show=..., # undocumented + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., # undocumented + validatecommand: tkinter._EntryValidateCommand = ..., # undocumented + values: list[str] | tuple[str, ...] = ..., + width: int = 20, + xscrollcommand: tkinter._XYScrollCommand = ..., # undocumented + ) -> None: ... + @overload # type: ignore[override] + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + cursor: tkinter._Cursor = ..., + exportselection: bool = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: int = ..., + invalidcommand: tkinter._EntryValidateCommand = ..., + justify: Literal["left", "center", "right"] = ..., + postcommand: Callable[[], object] | str = ..., + show=..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., + validatecommand: tkinter._EntryValidateCommand = ..., + values: list[str] | tuple[str, ...] = ..., + width: int = ..., + xscrollcommand: tkinter._XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + # config must be copy/pasted, otherwise ttk.Combobox().config is mypy error (don't know why) + @overload # type: ignore[override] + def config( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + cursor: tkinter._Cursor = ..., + exportselection: bool = ..., + font: _FontDescription = ..., + foreground: str = ..., + height: int = ..., + invalidcommand: tkinter._EntryValidateCommand = ..., + justify: Literal["left", "center", "right"] = ..., + postcommand: Callable[[], object] | str = ..., + show=..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., + validatecommand: tkinter._EntryValidateCommand = ..., + values: list[str] | tuple[str, ...] = ..., + width: int = ..., + xscrollcommand: tkinter._XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + def current(self, newindex: int | None = None) -> int: ... + def set(self, value: Any) -> None: ... + +class Frame(Widget): + # This should be kept in sync with tkinter.ttk.LabeledScale.__init__() + # (all of these keyword-only arguments are also present there) + def __init__( + self, + master: tkinter.Misc | None = None, + *, + border: tkinter._ScreenUnits = ..., + borderwidth: tkinter._ScreenUnits = ..., + class_: str = "", + cursor: tkinter._Cursor = "", + height: tkinter._ScreenUnits = 0, + name: str = ..., + padding: _Padding = ..., + relief: tkinter._Relief = ..., + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + width: tkinter._ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + border: tkinter._ScreenUnits = ..., + borderwidth: tkinter._ScreenUnits = ..., + cursor: tkinter._Cursor = ..., + height: tkinter._ScreenUnits = ..., + padding: _Padding = ..., + relief: tkinter._Relief = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + width: tkinter._ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Label(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + anchor: tkinter._Anchor = ..., + background: str = "", + border: tkinter._ScreenUnits = ..., # alias for borderwidth + borderwidth: tkinter._ScreenUnits = ..., # undocumented + class_: str = "", + compound: _TtkCompound = "", + cursor: tkinter._Cursor = "", + font: _FontDescription = ..., + foreground: str = "", + image: tkinter._ImageSpec = "", + justify: Literal["left", "center", "right"] = ..., + name: str = ..., + padding: _Padding = ..., + relief: tkinter._Relief = ..., + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + text: float | str = "", + textvariable: tkinter.Variable = ..., + underline: int = -1, + width: int | Literal[""] = "", + wraplength: tkinter._ScreenUnits = ..., + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + anchor: tkinter._Anchor = ..., + background: str = ..., + border: tkinter._ScreenUnits = ..., + borderwidth: tkinter._ScreenUnits = ..., + compound: _TtkCompound = ..., + cursor: tkinter._Cursor = ..., + font: _FontDescription = ..., + foreground: str = ..., + image: tkinter._ImageSpec = ..., + justify: Literal["left", "center", "right"] = ..., + padding: _Padding = ..., + relief: tkinter._Relief = ..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = ..., + textvariable: tkinter.Variable = ..., + underline: int = ..., + width: int | Literal[""] = ..., + wraplength: tkinter._ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Labelframe(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + border: tkinter._ScreenUnits = ..., + borderwidth: tkinter._ScreenUnits = ..., # undocumented + class_: str = "", + cursor: tkinter._Cursor = "", + height: tkinter._ScreenUnits = 0, + labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., + labelwidget: tkinter.Misc = ..., + name: str = ..., + padding: _Padding = ..., + relief: tkinter._Relief = ..., # undocumented + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + text: float | str = "", + underline: int = -1, + width: tkinter._ScreenUnits = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + border: tkinter._ScreenUnits = ..., + borderwidth: tkinter._ScreenUnits = ..., + cursor: tkinter._Cursor = ..., + height: tkinter._ScreenUnits = ..., + labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., + labelwidget: tkinter.Misc = ..., + padding: _Padding = ..., + relief: tkinter._Relief = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = ..., + underline: int = ..., + width: tkinter._ScreenUnits = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +LabelFrame = Labelframe + +class Menubutton(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + compound: _TtkCompound = "", + cursor: tkinter._Cursor = "", + direction: Literal["above", "below", "left", "right", "flush"] = "below", + image: tkinter._ImageSpec = "", + menu: tkinter.Menu = ..., + name: str = ..., + padding=..., # undocumented + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = "", + textvariable: tkinter.Variable = ..., + underline: int = -1, + width: int | Literal[""] = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + compound: _TtkCompound = ..., + cursor: tkinter._Cursor = ..., + direction: Literal["above", "below", "left", "right", "flush"] = ..., + image: tkinter._ImageSpec = ..., + menu: tkinter.Menu = ..., + padding=..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = ..., + textvariable: tkinter.Variable = ..., + underline: int = ..., + width: int | Literal[""] = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Notebook(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + cursor: tkinter._Cursor = "", + height: int = 0, + name: str = ..., + padding: _Padding = ..., + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + width: int = 0, + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + cursor: tkinter._Cursor = ..., + height: int = ..., + padding: _Padding = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + width: int = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def add( + self, + child: tkinter.Widget, + *, + state: Literal["normal", "disabled", "hidden"] = ..., + sticky: str = ..., # consists of letters 'n', 's', 'w', 'e', no repeats, may be empty + padding: _Padding = ..., + text: str = ..., + # `image` is a sequence of an image name, followed by zero or more + # (sequences of one or more state names followed by an image name) + image=..., + compound: tkinter._Compound = ..., + underline: int = ..., + ) -> None: ... + def forget(self, tab_id) -> None: ... + def hide(self, tab_id) -> None: ... + def identify(self, x: int, y: int) -> str: ... + def index(self, tab_id): ... + def insert(self, pos, child, **kw) -> None: ... + def select(self, tab_id: Incomplete | None = None): ... + def tab(self, tab_id, option: Incomplete | None = None, **kw): ... + def tabs(self): ... + def enable_traversal(self) -> None: ... + +class Panedwindow(Widget, tkinter.PanedWindow): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + cursor: tkinter._Cursor = "", + # width and height for tkinter.ttk.Panedwindow are int but for tkinter.PanedWindow they are screen units + height: int = 0, + name: str = ..., + orient: Literal["vertical", "horizontal"] = "vertical", # can't be changed with configure() + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + width: int = 0, + ) -> None: ... + def add(self, child: tkinter.Widget, *, weight: int = ..., **kw) -> None: ... + @overload # type: ignore[override] + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + cursor: tkinter._Cursor = ..., + height: int = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + width: int = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + # config must be copy/pasted, otherwise ttk.Panedwindow().config is mypy error (don't know why) + @overload # type: ignore[override] + def config( + self, + cnf: dict[str, Any] | None = None, + *, + cursor: tkinter._Cursor = ..., + height: int = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + width: int = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + forget: Incomplete + def insert(self, pos, child, **kw) -> None: ... + def pane(self, pane, option: Incomplete | None = None, **kw): ... + def sashpos(self, index, newpos: Incomplete | None = None): ... + +PanedWindow = Panedwindow + +class Progressbar(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + cursor: tkinter._Cursor = "", + length: tkinter._ScreenUnits = 100, + maximum: float = 100, + mode: Literal["determinate", "indeterminate"] = "determinate", + name: str = ..., + orient: Literal["horizontal", "vertical"] = "horizontal", + phase: int = 0, # docs say read-only but assigning int to this works + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + value: float = 0.0, + variable: tkinter.IntVar | tkinter.DoubleVar = ..., + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + cursor: tkinter._Cursor = ..., + length: tkinter._ScreenUnits = ..., + maximum: float = ..., + mode: Literal["determinate", "indeterminate"] = ..., + orient: Literal["horizontal", "vertical"] = ..., + phase: int = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + value: float = ..., + variable: tkinter.IntVar | tkinter.DoubleVar = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def start(self, interval: Literal["idle"] | int | None = None) -> None: ... + def step(self, amount: float | None = None) -> None: ... + def stop(self) -> None: ... + +class Radiobutton(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + command: tkinter._ButtonCommand = "", + compound: _TtkCompound = "", + cursor: tkinter._Cursor = "", + image: tkinter._ImageSpec = "", + name: str = ..., + padding=..., # undocumented + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = "", + textvariable: tkinter.Variable = ..., + underline: int = -1, + value: Any = "1", + variable: tkinter.Variable | Literal[""] = ..., + width: int | Literal[""] = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + command: tkinter._ButtonCommand = ..., + compound: _TtkCompound = ..., + cursor: tkinter._Cursor = ..., + image: tkinter._ImageSpec = ..., + padding=..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + text: float | str = ..., + textvariable: tkinter.Variable = ..., + underline: int = ..., + value: Any = ..., + variable: tkinter.Variable | Literal[""] = ..., + width: int | Literal[""] = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def invoke(self) -> Any: ... + +# type ignore, because identify() methods of Widget and tkinter.Scale are incompatible +class Scale(Widget, tkinter.Scale): # type: ignore[misc] + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + command: str | Callable[[str], object] = "", + cursor: tkinter._Cursor = "", + from_: float = 0, + length: tkinter._ScreenUnits = 100, + name: str = ..., + orient: Literal["horizontal", "vertical"] = "horizontal", + state: str = ..., # undocumented + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + to: float = 1.0, + value: float = 0, + variable: tkinter.IntVar | tkinter.DoubleVar = ..., + ) -> None: ... + @overload # type: ignore[override] + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + command: str | Callable[[str], object] = ..., + cursor: tkinter._Cursor = ..., + from_: float = ..., + length: tkinter._ScreenUnits = ..., + orient: Literal["horizontal", "vertical"] = ..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + to: float = ..., + value: float = ..., + variable: tkinter.IntVar | tkinter.DoubleVar = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + # config must be copy/pasted, otherwise ttk.Scale().config is mypy error (don't know why) + @overload # type: ignore[override] + def config( + self, + cnf: dict[str, Any] | None = None, + *, + command: str | Callable[[str], object] = ..., + cursor: tkinter._Cursor = ..., + from_: float = ..., + length: tkinter._ScreenUnits = ..., + orient: Literal["horizontal", "vertical"] = ..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + to: float = ..., + value: float = ..., + variable: tkinter.IntVar | tkinter.DoubleVar = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + def get(self, x: int | None = None, y: int | None = None) -> float: ... + +# type ignore, because identify() methods of Widget and tkinter.Scale are incompatible +class Scrollbar(Widget, tkinter.Scrollbar): # type: ignore[misc] + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + command: Callable[..., tuple[float, float] | None] | str = "", + cursor: tkinter._Cursor = "", + name: str = ..., + orient: Literal["horizontal", "vertical"] = "vertical", + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + ) -> None: ... + @overload # type: ignore[override] + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + command: Callable[..., tuple[float, float] | None] | str = ..., + cursor: tkinter._Cursor = ..., + orient: Literal["horizontal", "vertical"] = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + # config must be copy/pasted, otherwise ttk.Scrollbar().config is mypy error (don't know why) + @overload # type: ignore[override] + def config( + self, + cnf: dict[str, Any] | None = None, + *, + command: Callable[..., tuple[float, float] | None] | str = ..., + cursor: tkinter._Cursor = ..., + orient: Literal["horizontal", "vertical"] = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + +class Separator(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + cursor: tkinter._Cursor = "", + name: str = ..., + orient: Literal["horizontal", "vertical"] = "horizontal", + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + cursor: tkinter._Cursor = ..., + orient: Literal["horizontal", "vertical"] = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Sizegrip(Widget): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + cursor: tkinter._Cursor = ..., + name: str = ..., + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + cursor: tkinter._Cursor = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + +class Spinbox(Entry): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + background: str = ..., # undocumented + class_: str = "", + command: Callable[[], object] | str | list[str] | tuple[str, ...] = "", + cursor: tkinter._Cursor = "", + exportselection: bool = ..., # undocumented + font: _FontDescription = ..., # undocumented + foreground: str = ..., # undocumented + format: str = "", + from_: float = 0, + increment: float = 1, + invalidcommand: tkinter._EntryValidateCommand = ..., # undocumented + justify: Literal["left", "center", "right"] = ..., # undocumented + name: str = ..., + show=..., # undocumented + state: str = "normal", + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., # undocumented + to: float = 0, + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", + validatecommand: tkinter._EntryValidateCommand = "", + values: list[str] | tuple[str, ...] = ..., + width: int = ..., # undocumented + wrap: bool = False, + xscrollcommand: tkinter._XYScrollCommand = "", + ) -> None: ... + @overload # type: ignore[override] + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + background: str = ..., + command: Callable[[], object] | str | list[str] | tuple[str, ...] = ..., + cursor: tkinter._Cursor = ..., + exportselection: bool = ..., + font: _FontDescription = ..., + foreground: str = ..., + format: str = ..., + from_: float = ..., + increment: float = ..., + invalidcommand: tkinter._EntryValidateCommand = ..., + justify: Literal["left", "center", "right"] = ..., + show=..., + state: str = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + textvariable: tkinter.Variable = ..., + to: float = ..., + validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., + validatecommand: tkinter._EntryValidateCommand = ..., + values: list[str] | tuple[str, ...] = ..., + width: int = ..., + wrap: bool = ..., + xscrollcommand: tkinter._XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure # type: ignore[assignment] + def set(self, value: Any) -> None: ... + +class _TreeviewItemDict(TypedDict): + text: str + image: list[str] | Literal[""] # no idea why it's wrapped in list + values: list[Any] | Literal[""] + open: bool # actually 0 or 1 + tags: list[str] | Literal[""] + +class _TreeviewTagDict(TypedDict): + # There is also 'text' and 'anchor', but they don't seem to do anything, using them is likely a bug + foreground: str + background: str + font: _FontDescription + image: str # not wrapped in list :D + +class _TreeviewHeaderDict(TypedDict): + text: str + image: list[str] | Literal[""] + anchor: tkinter._Anchor + command: str + state: str # Doesn't seem to appear anywhere else than in these dicts + +class _TreeviewColumnDict(TypedDict): + width: int + minwidth: int + stretch: bool # actually 0 or 1 + anchor: tkinter._Anchor + id: str + +class Treeview(Widget, tkinter.XView, tkinter.YView): + def __init__( + self, + master: tkinter.Misc | None = None, + *, + class_: str = "", + columns: str | list[str] | list[int] | list[str | int] | tuple[str | int, ...] = "", + cursor: tkinter._Cursor = "", + displaycolumns: str | int | list[str] | tuple[str, ...] | list[int] | tuple[int, ...] = ("#all",), + height: int = 10, + name: str = ..., + padding: _Padding = ..., + selectmode: Literal["extended", "browse", "none"] = "extended", + # list/tuple of Literal don't actually work in mypy + # + # 'tree headings' is same as ['tree', 'headings'], and I wouldn't be + # surprised if someone is using it. + show: Literal["tree", "headings", "tree headings", ""] | list[str] | tuple[str, ...] = ("tree", "headings"), + style: str = "", + takefocus: tkinter._TakeFocusValue = ..., + xscrollcommand: tkinter._XYScrollCommand = "", + yscrollcommand: tkinter._XYScrollCommand = "", + ) -> None: ... + @overload + def configure( + self, + cnf: dict[str, Any] | None = None, + *, + columns: str | list[str] | list[int] | list[str | int] | tuple[str | int, ...] = ..., + cursor: tkinter._Cursor = ..., + displaycolumns: str | int | list[str] | tuple[str, ...] | list[int] | tuple[int, ...] = ..., + height: int = ..., + padding: _Padding = ..., + selectmode: Literal["extended", "browse", "none"] = ..., + show: Literal["tree", "headings", "tree headings", ""] | list[str] | tuple[str, ...] = ..., + style: str = ..., + takefocus: tkinter._TakeFocusValue = ..., + xscrollcommand: tkinter._XYScrollCommand = ..., + yscrollcommand: tkinter._XYScrollCommand = ..., + ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... + @overload + def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... + config = configure + def bbox(self, item: str | int, column: str | int | None = None) -> tuple[int, int, int, int] | Literal[""]: ... # type: ignore[override] + def get_children(self, item: str | int | None = None) -> tuple[str, ...]: ... + def set_children(self, item: str | int, *newchildren: str | int) -> None: ... + @overload + def column(self, column: str | int, option: Literal["width", "minwidth"]) -> int: ... + @overload + def column(self, column: str | int, option: Literal["stretch"]) -> bool: ... # actually 0 or 1 + @overload + def column(self, column: str | int, option: Literal["anchor"]) -> _tkinter.Tcl_Obj: ... + @overload + def column(self, column: str | int, option: Literal["id"]) -> str: ... + @overload + def column(self, column: str | int, option: str) -> Any: ... + @overload + def column( + self, + column: str | int, + option: None = None, + *, + width: int = ..., + minwidth: int = ..., + stretch: bool = ..., + anchor: tkinter._Anchor = ..., + # id is read-only + ) -> _TreeviewColumnDict | None: ... + def delete(self, *items: str | int) -> None: ... + def detach(self, *items: str | int) -> None: ... + def exists(self, item: str | int) -> bool: ... + @overload # type: ignore[override] + def focus(self, item: None = None) -> str: ... # can return empty string + @overload + def focus(self, item: str | int) -> Literal[""]: ... + @overload + def heading(self, column: str | int, option: Literal["text"]) -> str: ... + @overload + def heading(self, column: str | int, option: Literal["image"]) -> tuple[str] | str: ... + @overload + def heading(self, column: str | int, option: Literal["anchor"]) -> _tkinter.Tcl_Obj: ... + @overload + def heading(self, column: str | int, option: Literal["command"]) -> str: ... + @overload + def heading(self, column: str | int, option: str) -> Any: ... + @overload + def heading(self, column: str | int, option: None = None) -> _TreeviewHeaderDict: ... + @overload + def heading( + self, + column: str | int, + option: None = None, + *, + text: str = ..., + image: tkinter._ImageSpec = ..., + anchor: tkinter._Anchor = ..., + command: str | Callable[[], object] = ..., + ) -> None: ... + # Internal Method. Leave untyped: + def identify(self, component, x, y): ... # type: ignore[override] + def identify_row(self, y: int) -> str: ... + def identify_column(self, x: int) -> str: ... + def identify_region(self, x: int, y: int) -> Literal["heading", "separator", "tree", "cell", "nothing"]: ... + def identify_element(self, x: int, y: int) -> str: ... # don't know what possible return values are + def index(self, item: str | int) -> int: ... + def insert( + self, + parent: str, + index: int | Literal["end"], + iid: str | int | None = None, + *, + id: str | int = ..., # same as iid + text: str = ..., + image: tkinter._ImageSpec = ..., + values: list[Any] | tuple[Any, ...] = ..., + open: bool = ..., + tags: str | list[str] | tuple[str, ...] = ..., + ) -> str: ... + @overload + def item(self, item: str | int, option: Literal["text"]) -> str: ... + @overload + def item(self, item: str | int, option: Literal["image"]) -> tuple[str] | Literal[""]: ... + @overload + def item(self, item: str | int, option: Literal["values"]) -> tuple[Any, ...] | Literal[""]: ... + @overload + def item(self, item: str | int, option: Literal["open"]) -> bool: ... # actually 0 or 1 + @overload + def item(self, item: str | int, option: Literal["tags"]) -> tuple[str, ...] | Literal[""]: ... + @overload + def item(self, item: str | int, option: str) -> Any: ... + @overload + def item(self, item: str | int, option: None = None) -> _TreeviewItemDict: ... + @overload + def item( + self, + item: str | int, + option: None = None, + *, + text: str = ..., + image: tkinter._ImageSpec = ..., + values: list[Any] | tuple[Any, ...] | Literal[""] = ..., + open: bool = ..., + tags: str | list[str] | tuple[str, ...] = ..., + ) -> None: ... + def move(self, item: str | int, parent: str, index: int) -> None: ... + reattach = move + def next(self, item: str | int) -> str: ... # returning empty string means last item + def parent(self, item: str | int) -> str: ... + def prev(self, item: str | int) -> str: ... # returning empty string means first item + def see(self, item: str | int) -> None: ... + def selection(self) -> tuple[str, ...]: ... + @overload + def selection_set(self, items: list[str] | tuple[str, ...] | list[int] | tuple[int, ...], /) -> None: ... + @overload + def selection_set(self, *items: str | int) -> None: ... + @overload + def selection_add(self, items: list[str] | tuple[str, ...] | list[int] | tuple[int, ...], /) -> None: ... + @overload + def selection_add(self, *items: str | int) -> None: ... + @overload + def selection_remove(self, items: list[str] | tuple[str, ...] | list[int] | tuple[int, ...], /) -> None: ... + @overload + def selection_remove(self, *items: str | int) -> None: ... + @overload + def selection_toggle(self, items: list[str] | tuple[str, ...] | list[int] | tuple[int, ...], /) -> None: ... + @overload + def selection_toggle(self, *items: str | int) -> None: ... + @overload + def set(self, item: str | int, column: None = None, value: None = None) -> dict[str, Any]: ... + @overload + def set(self, item: str | int, column: str | int, value: None = None) -> Any: ... + @overload + def set(self, item: str | int, column: str | int, value: Any) -> Literal[""]: ... + # There's no tag_unbind() or 'add' argument for whatever reason. + # Also, it's 'callback' instead of 'func' here. + @overload + def tag_bind( + self, tagname: str, sequence: str | None = None, callback: Callable[[tkinter.Event[Treeview]], object] | None = None + ) -> str: ... + @overload + def tag_bind(self, tagname: str, sequence: str | None, callback: str) -> None: ... + @overload + def tag_bind(self, tagname: str, *, callback: str) -> None: ... + @overload + def tag_configure(self, tagname: str, option: Literal["foreground", "background"]) -> str: ... + @overload + def tag_configure(self, tagname: str, option: Literal["font"]) -> _FontDescription: ... + @overload + def tag_configure(self, tagname: str, option: Literal["image"]) -> str: ... + @overload + def tag_configure( + self, + tagname: str, + option: None = None, + *, + # There is also 'text' and 'anchor', but they don't seem to do anything, using them is likely a bug + foreground: str = ..., + background: str = ..., + font: _FontDescription = ..., + image: tkinter._ImageSpec = ..., + ) -> _TreeviewTagDict | MaybeNone: ... # can be None but annoying to check + @overload + def tag_has(self, tagname: str, item: None = None) -> tuple[str, ...]: ... + @overload + def tag_has(self, tagname: str, item: str | int) -> bool: ... + +class LabeledScale(Frame): + label: Label + scale: Scale + # This should be kept in sync with tkinter.ttk.Frame.__init__() + # (all the keyword-only args except compound are from there) + def __init__( + self, + master: tkinter.Misc | None = None, + variable: tkinter.IntVar | tkinter.DoubleVar | None = None, + from_: float = 0, + to: float = 10, + *, + border: tkinter._ScreenUnits = ..., + borderwidth: tkinter._ScreenUnits = ..., + class_: str = "", + compound: Literal["top", "bottom"] = "top", + cursor: tkinter._Cursor = "", + height: tkinter._ScreenUnits = 0, + name: str = ..., + padding: _Padding = ..., + relief: tkinter._Relief = ..., + style: str = "", + takefocus: tkinter._TakeFocusValue = "", + width: tkinter._ScreenUnits = 0, + ) -> None: ... + # destroy is overridden, signature does not change + value: Any + +class OptionMenu(Menubutton): + def __init__( + self, + master: tkinter.Misc | None, + variable: tkinter.StringVar, + default: str | None = None, + *values: str, + # rest of these are keyword-only because *args syntax used above + style: str = "", + direction: Literal["above", "below", "left", "right", "flush"] = "below", + command: Callable[[tkinter.StringVar], object] | None = None, + ) -> None: ... + # configure, config, cget, destroy are inherited from Menubutton + # destroy and __setitem__ are overridden, signature does not change + def set_menu(self, default: str | None = None, *values: str) -> None: ... diff --git a/mypy/typeshed/stdlib/token.pyi b/mypy/typeshed/stdlib/token.pyi new file mode 100644 index 000000000000..668987d7c2bf --- /dev/null +++ b/mypy/typeshed/stdlib/token.pyi @@ -0,0 +1,160 @@ +import sys + +__all__ = [ + "AMPER", + "AMPEREQUAL", + "AT", + "ATEQUAL", + "CIRCUMFLEX", + "CIRCUMFLEXEQUAL", + "COLON", + "COLONEQUAL", + "COMMA", + "DEDENT", + "DOT", + "DOUBLESLASH", + "DOUBLESLASHEQUAL", + "DOUBLESTAR", + "DOUBLESTAREQUAL", + "ELLIPSIS", + "ENDMARKER", + "EQEQUAL", + "EQUAL", + "ERRORTOKEN", + "GREATER", + "GREATEREQUAL", + "INDENT", + "ISEOF", + "ISNONTERMINAL", + "ISTERMINAL", + "LBRACE", + "LEFTSHIFT", + "LEFTSHIFTEQUAL", + "LESS", + "LESSEQUAL", + "LPAR", + "LSQB", + "MINEQUAL", + "MINUS", + "NAME", + "NEWLINE", + "NOTEQUAL", + "NT_OFFSET", + "NUMBER", + "N_TOKENS", + "OP", + "PERCENT", + "PERCENTEQUAL", + "PLUS", + "PLUSEQUAL", + "RARROW", + "RBRACE", + "RIGHTSHIFT", + "RIGHTSHIFTEQUAL", + "RPAR", + "RSQB", + "SEMI", + "SLASH", + "SLASHEQUAL", + "STAR", + "STAREQUAL", + "STRING", + "TILDE", + "TYPE_COMMENT", + "TYPE_IGNORE", + "VBAR", + "VBAREQUAL", + "tok_name", + "ENCODING", + "NL", + "COMMENT", +] +if sys.version_info < (3, 13): + __all__ += ["ASYNC", "AWAIT"] + +if sys.version_info >= (3, 10): + __all__ += ["SOFT_KEYWORD"] + +if sys.version_info >= (3, 12): + __all__ += ["EXCLAMATION", "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"] + +ENDMARKER: int +NAME: int +NUMBER: int +STRING: int +NEWLINE: int +INDENT: int +DEDENT: int +LPAR: int +RPAR: int +LSQB: int +RSQB: int +COLON: int +COMMA: int +SEMI: int +PLUS: int +MINUS: int +STAR: int +SLASH: int +VBAR: int +AMPER: int +LESS: int +GREATER: int +EQUAL: int +DOT: int +PERCENT: int +LBRACE: int +RBRACE: int +EQEQUAL: int +NOTEQUAL: int +LESSEQUAL: int +GREATEREQUAL: int +TILDE: int +CIRCUMFLEX: int +LEFTSHIFT: int +RIGHTSHIFT: int +DOUBLESTAR: int +PLUSEQUAL: int +MINEQUAL: int +STAREQUAL: int +SLASHEQUAL: int +PERCENTEQUAL: int +AMPEREQUAL: int +VBAREQUAL: int +CIRCUMFLEXEQUAL: int +LEFTSHIFTEQUAL: int +RIGHTSHIFTEQUAL: int +DOUBLESTAREQUAL: int +DOUBLESLASH: int +DOUBLESLASHEQUAL: int +AT: int +RARROW: int +ELLIPSIS: int +ATEQUAL: int +if sys.version_info < (3, 13): + AWAIT: int + ASYNC: int +OP: int +ERRORTOKEN: int +N_TOKENS: int +NT_OFFSET: int +tok_name: dict[int, str] +COMMENT: int +NL: int +ENCODING: int +TYPE_COMMENT: int +TYPE_IGNORE: int +COLONEQUAL: int +EXACT_TOKEN_TYPES: dict[str, int] +if sys.version_info >= (3, 10): + SOFT_KEYWORD: int + +if sys.version_info >= (3, 12): + EXCLAMATION: int + FSTRING_END: int + FSTRING_MIDDLE: int + FSTRING_START: int + +def ISTERMINAL(x: int) -> bool: ... +def ISNONTERMINAL(x: int) -> bool: ... +def ISEOF(x: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/tokenize.pyi b/mypy/typeshed/stdlib/tokenize.pyi new file mode 100644 index 000000000000..7e9a945cdc46 --- /dev/null +++ b/mypy/typeshed/stdlib/tokenize.pyi @@ -0,0 +1,186 @@ +import sys +from _typeshed import FileDescriptorOrPath +from collections.abc import Callable, Generator, Iterable, Sequence +from re import Pattern +from token import * +from token import EXACT_TOKEN_TYPES as EXACT_TOKEN_TYPES +from typing import Any, NamedTuple, TextIO, type_check_only +from typing_extensions import TypeAlias + +__all__ = [ + "AMPER", + "AMPEREQUAL", + "AT", + "ATEQUAL", + "CIRCUMFLEX", + "CIRCUMFLEXEQUAL", + "COLON", + "COLONEQUAL", + "COMMA", + "COMMENT", + "DEDENT", + "DOT", + "DOUBLESLASH", + "DOUBLESLASHEQUAL", + "DOUBLESTAR", + "DOUBLESTAREQUAL", + "ELLIPSIS", + "ENCODING", + "ENDMARKER", + "EQEQUAL", + "EQUAL", + "ERRORTOKEN", + "GREATER", + "GREATEREQUAL", + "INDENT", + "ISEOF", + "ISNONTERMINAL", + "ISTERMINAL", + "LBRACE", + "LEFTSHIFT", + "LEFTSHIFTEQUAL", + "LESS", + "LESSEQUAL", + "LPAR", + "LSQB", + "MINEQUAL", + "MINUS", + "NAME", + "NEWLINE", + "NL", + "NOTEQUAL", + "NT_OFFSET", + "NUMBER", + "N_TOKENS", + "OP", + "PERCENT", + "PERCENTEQUAL", + "PLUS", + "PLUSEQUAL", + "RARROW", + "RBRACE", + "RIGHTSHIFT", + "RIGHTSHIFTEQUAL", + "RPAR", + "RSQB", + "SEMI", + "SLASH", + "SLASHEQUAL", + "STAR", + "STAREQUAL", + "STRING", + "TILDE", + "TYPE_COMMENT", + "TYPE_IGNORE", + "TokenInfo", + "VBAR", + "VBAREQUAL", + "detect_encoding", + "generate_tokens", + "tok_name", + "tokenize", + "untokenize", +] +if sys.version_info < (3, 13): + __all__ += ["ASYNC", "AWAIT"] + +if sys.version_info >= (3, 10): + __all__ += ["SOFT_KEYWORD"] + +if sys.version_info >= (3, 12): + __all__ += ["EXCLAMATION", "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"] + +if sys.version_info >= (3, 13): + __all__ += ["TokenError", "open"] + +cookie_re: Pattern[str] +blank_re: Pattern[bytes] + +_Position: TypeAlias = tuple[int, int] + +# This class is not exposed. It calls itself tokenize.TokenInfo. +@type_check_only +class _TokenInfo(NamedTuple): + type: int + string: str + start: _Position + end: _Position + line: str + +class TokenInfo(_TokenInfo): + @property + def exact_type(self) -> int: ... + +# Backwards compatible tokens can be sequences of a shorter length too +_Token: TypeAlias = TokenInfo | Sequence[int | str | _Position] + +class TokenError(Exception): ... + +if sys.version_info < (3, 13): + class StopTokenizing(Exception): ... # undocumented + +class Untokenizer: + tokens: list[str] + prev_row: int + prev_col: int + encoding: str | None + def add_whitespace(self, start: _Position) -> None: ... + def untokenize(self, iterable: Iterable[_Token]) -> str: ... + def compat(self, token: Sequence[int | str], iterable: Iterable[_Token]) -> None: ... + if sys.version_info >= (3, 12): + def escape_brackets(self, token: str) -> str: ... + +# the docstring says "returns bytes" but is incorrect -- +# if the ENCODING token is missing, it skips the encode +def untokenize(iterable: Iterable[_Token]) -> Any: ... +def detect_encoding(readline: Callable[[], bytes | bytearray]) -> tuple[str, Sequence[bytes]]: ... +def tokenize(readline: Callable[[], bytes | bytearray]) -> Generator[TokenInfo, None, None]: ... +def generate_tokens(readline: Callable[[], str]) -> Generator[TokenInfo, None, None]: ... +def open(filename: FileDescriptorOrPath) -> TextIO: ... +def group(*choices: str) -> str: ... # undocumented +def any(*choices: str) -> str: ... # undocumented +def maybe(*choices: str) -> str: ... # undocumented + +Whitespace: str # undocumented +Comment: str # undocumented +Ignore: str # undocumented +Name: str # undocumented + +Hexnumber: str # undocumented +Binnumber: str # undocumented +Octnumber: str # undocumented +Decnumber: str # undocumented +Intnumber: str # undocumented +Exponent: str # undocumented +Pointfloat: str # undocumented +Expfloat: str # undocumented +Floatnumber: str # undocumented +Imagnumber: str # undocumented +Number: str # undocumented + +def _all_string_prefixes() -> set[str]: ... # undocumented + +StringPrefix: str # undocumented + +Single: str # undocumented +Double: str # undocumented +Single3: str # undocumented +Double3: str # undocumented +Triple: str # undocumented +String: str # undocumented + +Special: str # undocumented +Funny: str # undocumented + +PlainToken: str # undocumented +Token: str # undocumented + +ContStr: str # undocumented +PseudoExtras: str # undocumented +PseudoToken: str # undocumented + +endpats: dict[str, str] # undocumented +single_quoted: set[str] # undocumented +triple_quoted: set[str] # undocumented + +tabsize: int # undocumented diff --git a/mypy/typeshed/stdlib/tomllib.pyi b/mypy/typeshed/stdlib/tomllib.pyi new file mode 100644 index 000000000000..d559568b912b --- /dev/null +++ b/mypy/typeshed/stdlib/tomllib.pyi @@ -0,0 +1,10 @@ +from _typeshed import SupportsRead +from collections.abc import Callable +from typing import Any + +__all__ = ("loads", "load", "TOMLDecodeError") + +class TOMLDecodeError(ValueError): ... + +def load(fp: SupportsRead[bytes], /, *, parse_float: Callable[[str], Any] = ...) -> dict[str, Any]: ... +def loads(s: str, /, *, parse_float: Callable[[str], Any] = ...) -> dict[str, Any]: ... diff --git a/mypy/typeshed/stdlib/trace.pyi b/mypy/typeshed/stdlib/trace.pyi new file mode 100644 index 000000000000..04390f119195 --- /dev/null +++ b/mypy/typeshed/stdlib/trace.pyi @@ -0,0 +1,90 @@ +import sys +import types +from _typeshed import Incomplete, StrPath, TraceFunction +from collections.abc import Callable, Iterable, Mapping, Sequence +from typing import Any, TypeVar +from typing_extensions import ParamSpec, TypeAlias + +__all__ = ["Trace", "CoverageResults"] + +_T = TypeVar("_T") +_P = ParamSpec("_P") +_FileModuleFunction: TypeAlias = tuple[str, str | None, str] + +class CoverageResults: + counts: dict[tuple[str, int], int] + counter: dict[tuple[str, int], int] + calledfuncs: dict[_FileModuleFunction, int] + callers: dict[tuple[_FileModuleFunction, _FileModuleFunction], int] + inifile: StrPath | None + outfile: StrPath | None + def __init__( + self, + counts: dict[tuple[str, int], int] | None = None, + calledfuncs: dict[_FileModuleFunction, int] | None = None, + infile: StrPath | None = None, + callers: dict[tuple[_FileModuleFunction, _FileModuleFunction], int] | None = None, + outfile: StrPath | None = None, + ) -> None: ... # undocumented + def update(self, other: CoverageResults) -> None: ... + if sys.version_info >= (3, 13): + def write_results( + self, + show_missing: bool = True, + summary: bool = False, + coverdir: StrPath | None = None, + *, + ignore_missing_files: bool = False, + ) -> None: ... + else: + def write_results(self, show_missing: bool = True, summary: bool = False, coverdir: StrPath | None = None) -> None: ... + + def write_results_file( + self, path: StrPath, lines: Sequence[str], lnotab: Any, lines_hit: Mapping[int, int], encoding: str | None = None + ) -> tuple[int, int]: ... + def is_ignored_filename(self, filename: str) -> bool: ... # undocumented + +class _Ignore: + def __init__(self, modules: Iterable[str] | None = None, dirs: Iterable[StrPath] | None = None) -> None: ... + def names(self, filename: str, modulename: str) -> int: ... + +class Trace: + inifile: StrPath | None + outfile: StrPath | None + ignore: _Ignore + counts: dict[str, int] + pathtobasename: dict[Incomplete, Incomplete] + donothing: int + trace: int + start_time: int | None + globaltrace: TraceFunction + localtrace: TraceFunction + def __init__( + self, + count: int = 1, + trace: int = 1, + countfuncs: int = 0, + countcallers: int = 0, + ignoremods: Sequence[str] = (), + ignoredirs: Sequence[str] = (), + infile: StrPath | None = None, + outfile: StrPath | None = None, + timing: bool = False, + ) -> None: ... + def run(self, cmd: str | types.CodeType) -> None: ... + def runctx( + self, cmd: str | types.CodeType, globals: Mapping[str, Any] | None = None, locals: Mapping[str, Any] | None = None + ) -> None: ... + if sys.version_info >= (3, 9): + def runfunc(self, func: Callable[_P, _T], /, *args: _P.args, **kw: _P.kwargs) -> _T: ... + else: + def runfunc(self, func: Callable[_P, _T], *args: _P.args, **kw: _P.kwargs) -> _T: ... + + def file_module_function_of(self, frame: types.FrameType) -> _FileModuleFunction: ... + def globaltrace_trackcallers(self, frame: types.FrameType, why: str, arg: Any) -> None: ... + def globaltrace_countfuncs(self, frame: types.FrameType, why: str, arg: Any) -> None: ... + def globaltrace_lt(self, frame: types.FrameType, why: str, arg: Any) -> None: ... + def localtrace_trace_and_count(self, frame: types.FrameType, why: str, arg: Any) -> TraceFunction: ... + def localtrace_trace(self, frame: types.FrameType, why: str, arg: Any) -> TraceFunction: ... + def localtrace_count(self, frame: types.FrameType, why: str, arg: Any) -> TraceFunction: ... + def results(self) -> CoverageResults: ... diff --git a/mypy/typeshed/stdlib/traceback.pyi b/mypy/typeshed/stdlib/traceback.pyi new file mode 100644 index 000000000000..1c4a59de66aa --- /dev/null +++ b/mypy/typeshed/stdlib/traceback.pyi @@ -0,0 +1,300 @@ +import sys +from _typeshed import SupportsWrite, Unused +from collections.abc import Generator, Iterable, Iterator, Mapping +from types import FrameType, TracebackType +from typing import Any, Literal, overload +from typing_extensions import Self, TypeAlias, deprecated + +__all__ = [ + "extract_stack", + "extract_tb", + "format_exception", + "format_exception_only", + "format_list", + "format_stack", + "format_tb", + "print_exc", + "format_exc", + "print_exception", + "print_last", + "print_stack", + "print_tb", + "clear_frames", + "FrameSummary", + "StackSummary", + "TracebackException", + "walk_stack", + "walk_tb", +] + +_FrameSummaryTuple: TypeAlias = tuple[str, int, str, str | None] + +def print_tb(tb: TracebackType | None, limit: int | None = None, file: SupportsWrite[str] | None = None) -> None: ... + +if sys.version_info >= (3, 10): + @overload + def print_exception( + exc: type[BaseException] | None, + /, + value: BaseException | None = ..., + tb: TracebackType | None = ..., + limit: int | None = None, + file: SupportsWrite[str] | None = None, + chain: bool = True, + ) -> None: ... + @overload + def print_exception( + exc: BaseException, /, *, limit: int | None = None, file: SupportsWrite[str] | None = None, chain: bool = True + ) -> None: ... + @overload + def format_exception( + exc: type[BaseException] | None, + /, + value: BaseException | None = ..., + tb: TracebackType | None = ..., + limit: int | None = None, + chain: bool = True, + ) -> list[str]: ... + @overload + def format_exception(exc: BaseException, /, *, limit: int | None = None, chain: bool = True) -> list[str]: ... + +else: + def print_exception( + etype: type[BaseException] | None, + value: BaseException | None, + tb: TracebackType | None, + limit: int | None = None, + file: SupportsWrite[str] | None = None, + chain: bool = True, + ) -> None: ... + def format_exception( + etype: type[BaseException] | None, + value: BaseException | None, + tb: TracebackType | None, + limit: int | None = None, + chain: bool = True, + ) -> list[str]: ... + +def print_exc(limit: int | None = None, file: SupportsWrite[str] | None = None, chain: bool = True) -> None: ... +def print_last(limit: int | None = None, file: SupportsWrite[str] | None = None, chain: bool = True) -> None: ... +def print_stack(f: FrameType | None = None, limit: int | None = None, file: SupportsWrite[str] | None = None) -> None: ... +def extract_tb(tb: TracebackType | None, limit: int | None = None) -> StackSummary: ... +def extract_stack(f: FrameType | None = None, limit: int | None = None) -> StackSummary: ... +def format_list(extracted_list: Iterable[FrameSummary | _FrameSummaryTuple]) -> list[str]: ... + +# undocumented +def print_list(extracted_list: Iterable[FrameSummary | _FrameSummaryTuple], file: SupportsWrite[str] | None = None) -> None: ... + +if sys.version_info >= (3, 13): + @overload + def format_exception_only(exc: BaseException | None, /, *, show_group: bool = False) -> list[str]: ... + @overload + def format_exception_only(exc: Unused, /, value: BaseException | None, *, show_group: bool = False) -> list[str]: ... + +elif sys.version_info >= (3, 10): + @overload + def format_exception_only(exc: BaseException | None, /) -> list[str]: ... + @overload + def format_exception_only(exc: Unused, /, value: BaseException | None) -> list[str]: ... + +else: + def format_exception_only(etype: type[BaseException] | None, value: BaseException | None) -> list[str]: ... + +def format_exc(limit: int | None = None, chain: bool = True) -> str: ... +def format_tb(tb: TracebackType | None, limit: int | None = None) -> list[str]: ... +def format_stack(f: FrameType | None = None, limit: int | None = None) -> list[str]: ... +def clear_frames(tb: TracebackType | None) -> None: ... +def walk_stack(f: FrameType | None) -> Iterator[tuple[FrameType, int]]: ... +def walk_tb(tb: TracebackType | None) -> Iterator[tuple[FrameType, int]]: ... + +if sys.version_info >= (3, 11): + class _ExceptionPrintContext: + def indent(self) -> str: ... + def emit(self, text_gen: str | Iterable[str], margin_char: str | None = None) -> Generator[str, None, None]: ... + +class TracebackException: + __cause__: TracebackException + __context__: TracebackException + __suppress_context__: bool + stack: StackSummary + filename: str + lineno: int + text: str + offset: int + msg: str + if sys.version_info >= (3, 13): + @property + def exc_type_str(self) -> str: ... + @property + @deprecated("Deprecated in 3.13. Use exc_type_str instead.") + def exc_type(self) -> type[BaseException] | None: ... + else: + exc_type: type[BaseException] + if sys.version_info >= (3, 13): + def __init__( + self, + exc_type: type[BaseException], + exc_value: BaseException, + exc_traceback: TracebackType | None, + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + compact: bool = False, + max_group_width: int = 15, + max_group_depth: int = 10, + save_exc_type: bool = True, + _seen: set[int] | None = None, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __init__( + self, + exc_type: type[BaseException], + exc_value: BaseException, + exc_traceback: TracebackType | None, + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + compact: bool = False, + max_group_width: int = 15, + max_group_depth: int = 10, + _seen: set[int] | None = None, + ) -> None: ... + elif sys.version_info >= (3, 10): + def __init__( + self, + exc_type: type[BaseException], + exc_value: BaseException, + exc_traceback: TracebackType | None, + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + compact: bool = False, + _seen: set[int] | None = None, + ) -> None: ... + else: + def __init__( + self, + exc_type: type[BaseException], + exc_value: BaseException, + exc_traceback: TracebackType | None, + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + _seen: set[int] | None = None, + ) -> None: ... + + if sys.version_info >= (3, 11): + @classmethod + def from_exception( + cls, + exc: BaseException, + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + compact: bool = False, + max_group_width: int = 15, + max_group_depth: int = 10, + ) -> Self: ... + elif sys.version_info >= (3, 10): + @classmethod + def from_exception( + cls, + exc: BaseException, + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + compact: bool = False, + ) -> Self: ... + else: + @classmethod + def from_exception( + cls, exc: BaseException, *, limit: int | None = None, lookup_lines: bool = True, capture_locals: bool = False + ) -> Self: ... + + def __eq__(self, other: object) -> bool: ... + if sys.version_info >= (3, 11): + def format(self, *, chain: bool = True, _ctx: _ExceptionPrintContext | None = None) -> Generator[str, None, None]: ... + else: + def format(self, *, chain: bool = True) -> Generator[str, None, None]: ... + + if sys.version_info >= (3, 13): + def format_exception_only(self, *, show_group: bool = False, _depth: int = 0) -> Generator[str, None, None]: ... + else: + def format_exception_only(self) -> Generator[str, None, None]: ... + + if sys.version_info >= (3, 11): + def print(self, *, file: SupportsWrite[str] | None = None, chain: bool = True) -> None: ... + +class FrameSummary: + if sys.version_info >= (3, 11): + def __init__( + self, + filename: str, + lineno: int | None, + name: str, + *, + lookup_line: bool = True, + locals: Mapping[str, str] | None = None, + line: str | None = None, + end_lineno: int | None = None, + colno: int | None = None, + end_colno: int | None = None, + ) -> None: ... + end_lineno: int | None + colno: int | None + end_colno: int | None + else: + def __init__( + self, + filename: str, + lineno: int | None, + name: str, + *, + lookup_line: bool = True, + locals: Mapping[str, str] | None = None, + line: str | None = None, + ) -> None: ... + filename: str + lineno: int | None + name: str + locals: dict[str, str] | None + @property + def line(self) -> str | None: ... + @overload + def __getitem__(self, pos: Literal[0]) -> str: ... + @overload + def __getitem__(self, pos: Literal[1]) -> int: ... + @overload + def __getitem__(self, pos: Literal[2]) -> str: ... + @overload + def __getitem__(self, pos: Literal[3]) -> str | None: ... + @overload + def __getitem__(self, pos: int) -> Any: ... + @overload + def __getitem__(self, pos: slice) -> tuple[Any, ...]: ... + def __iter__(self) -> Iterator[Any]: ... + def __eq__(self, other: object) -> bool: ... + def __len__(self) -> Literal[4]: ... + +class StackSummary(list[FrameSummary]): + @classmethod + def extract( + cls, + frame_gen: Iterable[tuple[FrameType, int]], + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + ) -> StackSummary: ... + @classmethod + def from_list(cls, a_list: Iterable[FrameSummary | _FrameSummaryTuple]) -> StackSummary: ... + if sys.version_info >= (3, 11): + def format_frame_summary(self, frame_summary: FrameSummary) -> str: ... + + def format(self) -> list[str]: ... diff --git a/mypy/typeshed/stdlib/tracemalloc.pyi b/mypy/typeshed/stdlib/tracemalloc.pyi new file mode 100644 index 000000000000..e721e414138b --- /dev/null +++ b/mypy/typeshed/stdlib/tracemalloc.pyi @@ -0,0 +1,124 @@ +import sys +from _tracemalloc import * +from collections.abc import Sequence +from typing import Any, SupportsIndex, overload +from typing_extensions import TypeAlias + +def get_object_traceback(obj: object) -> Traceback | None: ... +def take_snapshot() -> Snapshot: ... + +class BaseFilter: + inclusive: bool + def __init__(self, inclusive: bool) -> None: ... + +class DomainFilter(BaseFilter): + @property + def domain(self) -> int: ... + def __init__(self, inclusive: bool, domain: int) -> None: ... + +class Filter(BaseFilter): + domain: int | None + lineno: int | None + @property + def filename_pattern(self) -> str: ... + all_frames: bool + def __init__( + self, + inclusive: bool, + filename_pattern: str, + lineno: int | None = None, + all_frames: bool = False, + domain: int | None = None, + ) -> None: ... + +class Statistic: + count: int + size: int + traceback: Traceback + def __init__(self, traceback: Traceback, size: int, count: int) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +class StatisticDiff: + count: int + count_diff: int + size: int + size_diff: int + traceback: Traceback + def __init__(self, traceback: Traceback, size: int, size_diff: int, count: int, count_diff: int) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +_FrameTuple: TypeAlias = tuple[str, int] + +class Frame: + @property + def filename(self) -> str: ... + @property + def lineno(self) -> int: ... + def __init__(self, frame: _FrameTuple) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def __lt__(self, other: Frame) -> bool: ... + if sys.version_info >= (3, 11): + def __gt__(self, other: Frame) -> bool: ... + def __ge__(self, other: Frame) -> bool: ... + def __le__(self, other: Frame) -> bool: ... + else: + def __gt__(self, other: Frame, NotImplemented: Any = ...) -> bool: ... + def __ge__(self, other: Frame, NotImplemented: Any = ...) -> bool: ... + def __le__(self, other: Frame, NotImplemented: Any = ...) -> bool: ... + +if sys.version_info >= (3, 9): + _TraceTuple: TypeAlias = tuple[int, int, Sequence[_FrameTuple], int | None] | tuple[int, int, Sequence[_FrameTuple]] +else: + _TraceTuple: TypeAlias = tuple[int, int, Sequence[_FrameTuple]] + +class Trace: + @property + def domain(self) -> int: ... + @property + def size(self) -> int: ... + @property + def traceback(self) -> Traceback: ... + def __init__(self, trace: _TraceTuple) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +class Traceback(Sequence[Frame]): + if sys.version_info >= (3, 9): + @property + def total_nframe(self) -> int | None: ... + def __init__(self, frames: Sequence[_FrameTuple], total_nframe: int | None = None) -> None: ... + else: + def __init__(self, frames: Sequence[_FrameTuple]) -> None: ... + + def format(self, limit: int | None = None, most_recent_first: bool = False) -> list[str]: ... + @overload + def __getitem__(self, index: SupportsIndex) -> Frame: ... + @overload + def __getitem__(self, index: slice) -> Sequence[Frame]: ... + def __contains__(self, frame: Frame) -> bool: ... # type: ignore[override] + def __len__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def __lt__(self, other: Traceback) -> bool: ... + if sys.version_info >= (3, 11): + def __gt__(self, other: Traceback) -> bool: ... + def __ge__(self, other: Traceback) -> bool: ... + def __le__(self, other: Traceback) -> bool: ... + else: + def __gt__(self, other: Traceback, NotImplemented: Any = ...) -> bool: ... + def __ge__(self, other: Traceback, NotImplemented: Any = ...) -> bool: ... + def __le__(self, other: Traceback, NotImplemented: Any = ...) -> bool: ... + +class Snapshot: + def __init__(self, traces: Sequence[_TraceTuple], traceback_limit: int) -> None: ... + def compare_to(self, old_snapshot: Snapshot, key_type: str, cumulative: bool = False) -> list[StatisticDiff]: ... + def dump(self, filename: str) -> None: ... + def filter_traces(self, filters: Sequence[DomainFilter | Filter]) -> Snapshot: ... + @staticmethod + def load(filename: str) -> Snapshot: ... + def statistics(self, key_type: str, cumulative: bool = False) -> list[Statistic]: ... + traceback_limit: int + traces: Sequence[Trace] diff --git a/mypy/typeshed/stdlib/tty.pyi b/mypy/typeshed/stdlib/tty.pyi new file mode 100644 index 000000000000..0611879cf1b2 --- /dev/null +++ b/mypy/typeshed/stdlib/tty.pyi @@ -0,0 +1,30 @@ +import sys +import termios +from typing import IO, Final +from typing_extensions import TypeAlias + +if sys.platform != "win32": + __all__ = ["setraw", "setcbreak"] + if sys.version_info >= (3, 12): + __all__ += ["cfmakeraw", "cfmakecbreak"] + + _ModeSetterReturn: TypeAlias = termios._AttrReturn + else: + _ModeSetterReturn: TypeAlias = None + + _FD: TypeAlias = int | IO[str] + + # XXX: Undocumented integer constants + IFLAG: Final[int] + OFLAG: Final[int] + CFLAG: Final[int] + LFLAG: Final[int] + ISPEED: Final[int] + OSPEED: Final[int] + CC: Final[int] + def setraw(fd: _FD, when: int = 2) -> _ModeSetterReturn: ... + def setcbreak(fd: _FD, when: int = 2) -> _ModeSetterReturn: ... + + if sys.version_info >= (3, 12): + def cfmakeraw(mode: termios._Attr) -> None: ... + def cfmakecbreak(mode: termios._Attr) -> None: ... diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi new file mode 100644 index 000000000000..a2ab728de943 --- /dev/null +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -0,0 +1,732 @@ +import sys +from collections.abc import Callable, Sequence +from tkinter import Canvas, Frame, Misc, PhotoImage, Scrollbar +from typing import Any, ClassVar, Literal, TypedDict, overload +from typing_extensions import Self, TypeAlias + +__all__ = [ + "ScrolledCanvas", + "TurtleScreen", + "Screen", + "RawTurtle", + "Turtle", + "RawPen", + "Pen", + "Shape", + "Vec2D", + "addshape", + "bgcolor", + "bgpic", + "bye", + "clearscreen", + "colormode", + "delay", + "exitonclick", + "getcanvas", + "getshapes", + "listen", + "mainloop", + "mode", + "numinput", + "onkey", + "onkeypress", + "onkeyrelease", + "onscreenclick", + "ontimer", + "register_shape", + "resetscreen", + "screensize", + "setup", + "setworldcoordinates", + "textinput", + "title", + "tracer", + "turtles", + "update", + "window_height", + "window_width", + "back", + "backward", + "begin_fill", + "begin_poly", + "bk", + "circle", + "clear", + "clearstamp", + "clearstamps", + "clone", + "color", + "degrees", + "distance", + "dot", + "down", + "end_fill", + "end_poly", + "fd", + "fillcolor", + "filling", + "forward", + "get_poly", + "getpen", + "getscreen", + "get_shapepoly", + "getturtle", + "goto", + "heading", + "hideturtle", + "home", + "ht", + "isdown", + "isvisible", + "left", + "lt", + "onclick", + "ondrag", + "onrelease", + "pd", + "pen", + "pencolor", + "pendown", + "pensize", + "penup", + "pos", + "position", + "pu", + "radians", + "right", + "reset", + "resizemode", + "rt", + "seth", + "setheading", + "setpos", + "setposition", + "setundobuffer", + "setx", + "sety", + "shape", + "shapesize", + "shapetransform", + "shearfactor", + "showturtle", + "speed", + "st", + "stamp", + "tilt", + "tiltangle", + "towards", + "turtlesize", + "undo", + "undobufferentries", + "up", + "width", + "write", + "xcor", + "ycor", + "write_docstringdict", + "done", + "Terminator", +] + +if sys.version_info >= (3, 12): + __all__ += ["teleport"] + +if sys.version_info < (3, 13): + __all__ += ["settiltangle"] + +# Note: '_Color' is the alias we use for arguments and _AnyColor is the +# alias we use for return types. Really, these two aliases should be the +# same, but as per the "no union returns" typeshed policy, we'll return +# Any instead. +_Color: TypeAlias = str | tuple[float, float, float] +_AnyColor: TypeAlias = Any + +class _PenState(TypedDict): + shown: bool + pendown: bool + pencolor: _Color + fillcolor: _Color + pensize: int + speed: int + resizemode: Literal["auto", "user", "noresize"] + stretchfactor: tuple[float, float] + shearfactor: float + outline: int + tilt: float + +_Speed: TypeAlias = str | float +_PolygonCoords: TypeAlias = Sequence[tuple[float, float]] + +class Vec2D(tuple[float, float]): + def __new__(cls, x: float, y: float) -> Self: ... + def __add__(self, other: tuple[float, float]) -> Vec2D: ... # type: ignore[override] + @overload # type: ignore[override] + def __mul__(self, other: Vec2D) -> float: ... + @overload + def __mul__(self, other: float) -> Vec2D: ... + def __rmul__(self, other: float) -> Vec2D: ... # type: ignore[override] + def __sub__(self, other: tuple[float, float]) -> Vec2D: ... + def __neg__(self) -> Vec2D: ... + def __abs__(self) -> float: ... + def rotate(self, angle: float) -> Vec2D: ... + +# Does not actually inherit from Canvas, but dynamically gets all methods of Canvas +class ScrolledCanvas(Canvas, Frame): # type: ignore[misc] + bg: str + hscroll: Scrollbar + vscroll: Scrollbar + def __init__( + self, master: Misc | None, width: int = 500, height: int = 350, canvwidth: int = 600, canvheight: int = 500 + ) -> None: ... + canvwidth: int + canvheight: int + def reset(self, canvwidth: int | None = None, canvheight: int | None = None, bg: str | None = None) -> None: ... + +class TurtleScreenBase: + cv: Canvas + canvwidth: int + canvheight: int + xscale: float + yscale: float + def __init__(self, cv: Canvas) -> None: ... + def mainloop(self) -> None: ... + def textinput(self, title: str, prompt: str) -> str | None: ... + def numinput( + self, title: str, prompt: str, default: float | None = None, minval: float | None = None, maxval: float | None = None + ) -> float | None: ... + +class Terminator(Exception): ... +class TurtleGraphicsError(Exception): ... + +class Shape: + def __init__(self, type_: str, data: _PolygonCoords | PhotoImage | None = None) -> None: ... + def addcomponent(self, poly: _PolygonCoords, fill: _Color, outline: _Color | None = None) -> None: ... + +class TurtleScreen(TurtleScreenBase): + def __init__(self, cv: Canvas, mode: str = "standard", colormode: float = 1.0, delay: int = 10) -> None: ... + def clear(self) -> None: ... + @overload + def mode(self, mode: None = None) -> str: ... + @overload + def mode(self, mode: str) -> None: ... + def setworldcoordinates(self, llx: float, lly: float, urx: float, ury: float) -> None: ... + def register_shape(self, name: str, shape: _PolygonCoords | Shape | None = None) -> None: ... + @overload + def colormode(self, cmode: None = None) -> float: ... + @overload + def colormode(self, cmode: float) -> None: ... + def reset(self) -> None: ... + def turtles(self) -> list[Turtle]: ... + @overload + def bgcolor(self) -> _AnyColor: ... + @overload + def bgcolor(self, color: _Color) -> None: ... + @overload + def bgcolor(self, r: float, g: float, b: float) -> None: ... + @overload + def tracer(self, n: None = None) -> int: ... + @overload + def tracer(self, n: int, delay: int | None = None) -> None: ... + @overload + def delay(self, delay: None = None) -> int: ... + @overload + def delay(self, delay: int) -> None: ... + def update(self) -> None: ... + def window_width(self) -> int: ... + def window_height(self) -> int: ... + def getcanvas(self) -> Canvas: ... + def getshapes(self) -> list[str]: ... + def onclick(self, fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... + def onkey(self, fun: Callable[[], object], key: str) -> None: ... + def listen(self, xdummy: float | None = None, ydummy: float | None = None) -> None: ... + def ontimer(self, fun: Callable[[], object], t: int = 0) -> None: ... + @overload + def bgpic(self, picname: None = None) -> str: ... + @overload + def bgpic(self, picname: str) -> None: ... + @overload + def screensize(self, canvwidth: None = None, canvheight: None = None, bg: None = None) -> tuple[int, int]: ... + # Looks like if self.cv is not a ScrolledCanvas, this could return a tuple as well + @overload + def screensize(self, canvwidth: int, canvheight: int, bg: _Color | None = None) -> None: ... + onscreenclick = onclick + resetscreen = reset + clearscreen = clear + addshape = register_shape + def onkeypress(self, fun: Callable[[], object], key: str | None = None) -> None: ... + onkeyrelease = onkey + +class TNavigator: + START_ORIENTATION: dict[str, Vec2D] + DEFAULT_MODE: str + DEFAULT_ANGLEOFFSET: int + DEFAULT_ANGLEORIENT: int + def __init__(self, mode: str = "standard") -> None: ... + def reset(self) -> None: ... + def degrees(self, fullcircle: float = 360.0) -> None: ... + def radians(self) -> None: ... + if sys.version_info >= (3, 12): + def teleport(self, x: float | None = None, y: float | None = None, *, fill_gap: bool = False) -> None: ... + + def forward(self, distance: float) -> None: ... + def back(self, distance: float) -> None: ... + def right(self, angle: float) -> None: ... + def left(self, angle: float) -> None: ... + def pos(self) -> Vec2D: ... + def xcor(self) -> float: ... + def ycor(self) -> float: ... + @overload + def goto(self, x: tuple[float, float], y: None = None) -> None: ... + @overload + def goto(self, x: float, y: float) -> None: ... + def home(self) -> None: ... + def setx(self, x: float) -> None: ... + def sety(self, y: float) -> None: ... + @overload + def distance(self, x: TNavigator | tuple[float, float], y: None = None) -> float: ... + @overload + def distance(self, x: float, y: float) -> float: ... + @overload + def towards(self, x: TNavigator | tuple[float, float], y: None = None) -> float: ... + @overload + def towards(self, x: float, y: float) -> float: ... + def heading(self) -> float: ... + def setheading(self, to_angle: float) -> None: ... + def circle(self, radius: float, extent: float | None = None, steps: int | None = None) -> None: ... + def speed(self, s: int | None = 0) -> int | None: ... + fd = forward + bk = back + backward = back + rt = right + lt = left + position = pos + setpos = goto + setposition = goto + seth = setheading + +class TPen: + def __init__(self, resizemode: str = "noresize") -> None: ... + @overload + def resizemode(self, rmode: None = None) -> str: ... + @overload + def resizemode(self, rmode: str) -> None: ... + @overload + def pensize(self, width: None = None) -> int: ... + @overload + def pensize(self, width: int) -> None: ... + def penup(self) -> None: ... + def pendown(self) -> None: ... + def isdown(self) -> bool: ... + @overload + def speed(self, speed: None = None) -> int: ... + @overload + def speed(self, speed: _Speed) -> None: ... + @overload + def pencolor(self) -> _AnyColor: ... + @overload + def pencolor(self, color: _Color) -> None: ... + @overload + def pencolor(self, r: float, g: float, b: float) -> None: ... + @overload + def fillcolor(self) -> _AnyColor: ... + @overload + def fillcolor(self, color: _Color) -> None: ... + @overload + def fillcolor(self, r: float, g: float, b: float) -> None: ... + @overload + def color(self) -> tuple[_AnyColor, _AnyColor]: ... + @overload + def color(self, color: _Color) -> None: ... + @overload + def color(self, r: float, g: float, b: float) -> None: ... + @overload + def color(self, color1: _Color, color2: _Color) -> None: ... + if sys.version_info >= (3, 12): + def teleport(self, x: float | None = None, y: float | None = None, *, fill_gap: bool = False) -> None: ... + + def showturtle(self) -> None: ... + def hideturtle(self) -> None: ... + def isvisible(self) -> bool: ... + # Note: signatures 1 and 2 overlap unsafely when no arguments are provided + @overload + def pen(self) -> _PenState: ... + @overload + def pen( + self, + pen: _PenState | None = None, + *, + shown: bool = ..., + pendown: bool = ..., + pencolor: _Color = ..., + fillcolor: _Color = ..., + pensize: int = ..., + speed: int = ..., + resizemode: str = ..., + stretchfactor: tuple[float, float] = ..., + outline: int = ..., + tilt: float = ..., + ) -> None: ... + width = pensize + up = penup + pu = penup + pd = pendown + down = pendown + st = showturtle + ht = hideturtle + +class RawTurtle(TPen, TNavigator): # type: ignore[misc] # Conflicting methods in base classes + screen: TurtleScreen + screens: ClassVar[list[TurtleScreen]] + def __init__( + self, + canvas: Canvas | TurtleScreen | None = None, + shape: str = "classic", + undobuffersize: int = 1000, + visible: bool = True, + ) -> None: ... + def reset(self) -> None: ... + def setundobuffer(self, size: int | None) -> None: ... + def undobufferentries(self) -> int: ... + def clear(self) -> None: ... + def clone(self) -> Self: ... + @overload + def shape(self, name: None = None) -> str: ... + @overload + def shape(self, name: str) -> None: ... + # Unsafely overlaps when no arguments are provided + @overload + def shapesize(self) -> tuple[float, float, float]: ... + @overload + def shapesize( + self, stretch_wid: float | None = None, stretch_len: float | None = None, outline: float | None = None + ) -> None: ... + @overload + def shearfactor(self, shear: None = None) -> float: ... + @overload + def shearfactor(self, shear: float) -> None: ... + # Unsafely overlaps when no arguments are provided + @overload + def shapetransform(self) -> tuple[float, float, float, float]: ... + @overload + def shapetransform( + self, t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None + ) -> None: ... + def get_shapepoly(self) -> _PolygonCoords | None: ... + + if sys.version_info < (3, 13): + def settiltangle(self, angle: float) -> None: ... + + @overload + def tiltangle(self, angle: None = None) -> float: ... + @overload + def tiltangle(self, angle: float) -> None: ... + def tilt(self, angle: float) -> None: ... + # Can return either 'int' or Tuple[int, ...] based on if the stamp is + # a compound stamp or not. So, as per the "no Union return" policy, + # we return Any. + def stamp(self) -> Any: ... + def clearstamp(self, stampid: int | tuple[int, ...]) -> None: ... + def clearstamps(self, n: int | None = None) -> None: ... + def filling(self) -> bool: ... + def begin_fill(self) -> None: ... + def end_fill(self) -> None: ... + def dot(self, size: int | None = None, *color: _Color) -> None: ... + def write( + self, arg: object, move: bool = False, align: str = "left", font: tuple[str, int, str] = ("Arial", 8, "normal") + ) -> None: ... + def begin_poly(self) -> None: ... + def end_poly(self) -> None: ... + def get_poly(self) -> _PolygonCoords | None: ... + def getscreen(self) -> TurtleScreen: ... + def getturtle(self) -> Self: ... + getpen = getturtle + def onclick(self, fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... + def onrelease(self, fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... + def ondrag(self, fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... + def undo(self) -> None: ... + turtlesize = shapesize + +class _Screen(TurtleScreen): + def __init__(self) -> None: ... + # Note int and float are interpreted differently, hence the Union instead of just float + def setup( + self, + width: int | float = 0.5, # noqa: Y041 + height: int | float = 0.75, # noqa: Y041 + startx: int | None = None, + starty: int | None = None, + ) -> None: ... + def title(self, titlestring: str) -> None: ... + def bye(self) -> None: ... + def exitonclick(self) -> None: ... + +class Turtle(RawTurtle): + def __init__(self, shape: str = "classic", undobuffersize: int = 1000, visible: bool = True) -> None: ... + +RawPen = RawTurtle +Pen = Turtle + +def write_docstringdict(filename: str = "turtle_docstringdict") -> None: ... + +# Note: it's somewhat unfortunate that we have to copy the function signatures. +# It would be nice if we could partially reduce the redundancy by doing something +# like the following: +# +# _screen: Screen +# clear = _screen.clear +# +# However, it seems pytype does not support this type of syntax in pyi files. + +# Functions copied from TurtleScreenBase: + +# Note: mainloop() was always present in the global scope, but was added to +# TurtleScreenBase in Python 3.0 +def mainloop() -> None: ... +def textinput(title: str, prompt: str) -> str | None: ... +def numinput( + title: str, prompt: str, default: float | None = None, minval: float | None = None, maxval: float | None = None +) -> float | None: ... + +# Functions copied from TurtleScreen: + +def clear() -> None: ... +@overload +def mode(mode: None = None) -> str: ... +@overload +def mode(mode: str) -> None: ... +def setworldcoordinates(llx: float, lly: float, urx: float, ury: float) -> None: ... +def register_shape(name: str, shape: _PolygonCoords | Shape | None = None) -> None: ... +@overload +def colormode(cmode: None = None) -> float: ... +@overload +def colormode(cmode: float) -> None: ... +def reset() -> None: ... +def turtles() -> list[Turtle]: ... +@overload +def bgcolor() -> _AnyColor: ... +@overload +def bgcolor(color: _Color) -> None: ... +@overload +def bgcolor(r: float, g: float, b: float) -> None: ... +@overload +def tracer(n: None = None) -> int: ... +@overload +def tracer(n: int, delay: int | None = None) -> None: ... +@overload +def delay(delay: None = None) -> int: ... +@overload +def delay(delay: int) -> None: ... +def update() -> None: ... +def window_width() -> int: ... +def window_height() -> int: ... +def getcanvas() -> Canvas: ... +def getshapes() -> list[str]: ... +def onclick(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... +def onkey(fun: Callable[[], object], key: str) -> None: ... +def listen(xdummy: float | None = None, ydummy: float | None = None) -> None: ... +def ontimer(fun: Callable[[], object], t: int = 0) -> None: ... +@overload +def bgpic(picname: None = None) -> str: ... +@overload +def bgpic(picname: str) -> None: ... +@overload +def screensize(canvwidth: None = None, canvheight: None = None, bg: None = None) -> tuple[int, int]: ... +@overload +def screensize(canvwidth: int, canvheight: int, bg: _Color | None = None) -> None: ... + +onscreenclick = onclick +resetscreen = reset +clearscreen = clear +addshape = register_shape + +def onkeypress(fun: Callable[[], object], key: str | None = None) -> None: ... + +onkeyrelease = onkey + +# Functions copied from _Screen: + +def setup(width: float = 0.5, height: float = 0.75, startx: int | None = None, starty: int | None = None) -> None: ... +def title(titlestring: str) -> None: ... +def bye() -> None: ... +def exitonclick() -> None: ... +def Screen() -> _Screen: ... + +# Functions copied from TNavigator: + +def degrees(fullcircle: float = 360.0) -> None: ... +def radians() -> None: ... +def forward(distance: float) -> None: ... +def back(distance: float) -> None: ... +def right(angle: float) -> None: ... +def left(angle: float) -> None: ... +def pos() -> Vec2D: ... +def xcor() -> float: ... +def ycor() -> float: ... +@overload +def goto(x: tuple[float, float], y: None = None) -> None: ... +@overload +def goto(x: float, y: float) -> None: ... +def home() -> None: ... +def setx(x: float) -> None: ... +def sety(y: float) -> None: ... +@overload +def distance(x: TNavigator | tuple[float, float], y: None = None) -> float: ... +@overload +def distance(x: float, y: float) -> float: ... +@overload +def towards(x: TNavigator | tuple[float, float], y: None = None) -> float: ... +@overload +def towards(x: float, y: float) -> float: ... +def heading() -> float: ... +def setheading(to_angle: float) -> None: ... +def circle(radius: float, extent: float | None = None, steps: int | None = None) -> None: ... + +fd = forward +bk = back +backward = back +rt = right +lt = left +position = pos +setpos = goto +setposition = goto +seth = setheading + +# Functions copied from TPen: +@overload +def resizemode(rmode: None = None) -> str: ... +@overload +def resizemode(rmode: str) -> None: ... +@overload +def pensize(width: None = None) -> int: ... +@overload +def pensize(width: int) -> None: ... +def penup() -> None: ... +def pendown() -> None: ... +def isdown() -> bool: ... +@overload +def speed(speed: None = None) -> int: ... +@overload +def speed(speed: _Speed) -> None: ... +@overload +def pencolor() -> _AnyColor: ... +@overload +def pencolor(color: _Color) -> None: ... +@overload +def pencolor(r: float, g: float, b: float) -> None: ... +@overload +def fillcolor() -> _AnyColor: ... +@overload +def fillcolor(color: _Color) -> None: ... +@overload +def fillcolor(r: float, g: float, b: float) -> None: ... +@overload +def color() -> tuple[_AnyColor, _AnyColor]: ... +@overload +def color(color: _Color) -> None: ... +@overload +def color(r: float, g: float, b: float) -> None: ... +@overload +def color(color1: _Color, color2: _Color) -> None: ... +def showturtle() -> None: ... +def hideturtle() -> None: ... +def isvisible() -> bool: ... + +# Note: signatures 1 and 2 overlap unsafely when no arguments are provided +@overload +def pen() -> _PenState: ... +@overload +def pen( + pen: _PenState | None = None, + *, + shown: bool = ..., + pendown: bool = ..., + pencolor: _Color = ..., + fillcolor: _Color = ..., + pensize: int = ..., + speed: int = ..., + resizemode: str = ..., + stretchfactor: tuple[float, float] = ..., + outline: int = ..., + tilt: float = ..., +) -> None: ... + +width = pensize +up = penup +pu = penup +pd = pendown +down = pendown +st = showturtle +ht = hideturtle + +# Functions copied from RawTurtle: + +def setundobuffer(size: int | None) -> None: ... +def undobufferentries() -> int: ... +@overload +def shape(name: None = None) -> str: ... +@overload +def shape(name: str) -> None: ... + +if sys.version_info >= (3, 12): + def teleport(x: float | None = None, y: float | None = None, *, fill_gap: bool = False) -> None: ... + +# Unsafely overlaps when no arguments are provided +@overload +def shapesize() -> tuple[float, float, float]: ... +@overload +def shapesize(stretch_wid: float | None = None, stretch_len: float | None = None, outline: float | None = None) -> None: ... +@overload +def shearfactor(shear: None = None) -> float: ... +@overload +def shearfactor(shear: float) -> None: ... + +# Unsafely overlaps when no arguments are provided +@overload +def shapetransform() -> tuple[float, float, float, float]: ... +@overload +def shapetransform( + t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None +) -> None: ... +def get_shapepoly() -> _PolygonCoords | None: ... + +if sys.version_info < (3, 13): + def settiltangle(angle: float) -> None: ... + +@overload +def tiltangle(angle: None = None) -> float: ... +@overload +def tiltangle(angle: float) -> None: ... +def tilt(angle: float) -> None: ... + +# Can return either 'int' or Tuple[int, ...] based on if the stamp is +# a compound stamp or not. So, as per the "no Union return" policy, +# we return Any. +def stamp() -> Any: ... +def clearstamp(stampid: int | tuple[int, ...]) -> None: ... +def clearstamps(n: int | None = None) -> None: ... +def filling() -> bool: ... +def begin_fill() -> None: ... +def end_fill() -> None: ... +def dot(size: int | None = None, *color: _Color) -> None: ... +def write(arg: object, move: bool = False, align: str = "left", font: tuple[str, int, str] = ("Arial", 8, "normal")) -> None: ... +def begin_poly() -> None: ... +def end_poly() -> None: ... +def get_poly() -> _PolygonCoords | None: ... +def getscreen() -> TurtleScreen: ... +def getturtle() -> Turtle: ... + +getpen = getturtle + +def onrelease(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... +def ondrag(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... +def undo() -> None: ... + +turtlesize = shapesize + +# Functions copied from RawTurtle with a few tweaks: + +def clone() -> Turtle: ... + +# Extra functions present only in the global scope: + +done = mainloop diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi new file mode 100644 index 000000000000..b513bd77468a --- /dev/null +++ b/mypy/typeshed/stdlib/types.pyi @@ -0,0 +1,645 @@ +import sys +from _typeshed import MaybeNone, SupportsKeysAndGetItem +from _typeshed.importlib import LoaderProtocol +from collections.abc import ( + AsyncGenerator, + Awaitable, + Callable, + Coroutine, + Generator, + ItemsView, + Iterable, + Iterator, + KeysView, + MutableSequence, + ValuesView, +) +from importlib.machinery import ModuleSpec + +# pytype crashes if types.MappingProxyType inherits from collections.abc.Mapping instead of typing.Mapping +from typing import Any, ClassVar, Literal, Mapping, TypeVar, final, overload # noqa: Y022 +from typing_extensions import ParamSpec, Self, TypeVarTuple, deprecated + +__all__ = [ + "FunctionType", + "LambdaType", + "CodeType", + "MappingProxyType", + "SimpleNamespace", + "GeneratorType", + "CoroutineType", + "AsyncGeneratorType", + "MethodType", + "BuiltinFunctionType", + "ModuleType", + "TracebackType", + "FrameType", + "GetSetDescriptorType", + "MemberDescriptorType", + "new_class", + "prepare_class", + "DynamicClassAttribute", + "coroutine", + "BuiltinMethodType", + "ClassMethodDescriptorType", + "MethodDescriptorType", + "MethodWrapperType", + "WrapperDescriptorType", + "resolve_bases", + "CellType", +] + +if sys.version_info >= (3, 9): + __all__ += ["GenericAlias"] + +if sys.version_info >= (3, 10): + __all__ += ["EllipsisType", "NoneType", "NotImplementedType", "UnionType"] + +if sys.version_info >= (3, 12): + __all__ += ["get_original_bases"] + +if sys.version_info >= (3, 13): + __all__ += ["CapsuleType"] + +# Note, all classes "defined" here require special handling. + +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_KT = TypeVar("_KT") +_VT_co = TypeVar("_VT_co", covariant=True) + +# Make sure this class definition stays roughly in line with `builtins.function` +@final +class FunctionType: + @property + def __closure__(self) -> tuple[CellType, ...] | None: ... + __code__: CodeType + __defaults__: tuple[Any, ...] | None + __dict__: dict[str, Any] + @property + def __globals__(self) -> dict[str, Any]: ... + __name__: str + __qualname__: str + __annotations__: dict[str, Any] + __kwdefaults__: dict[str, Any] | None + if sys.version_info >= (3, 10): + @property + def __builtins__(self) -> dict[str, Any]: ... + if sys.version_info >= (3, 12): + __type_params__: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] + + __module__: str + def __new__( + cls, + code: CodeType, + globals: dict[str, Any], + name: str | None = ..., + argdefs: tuple[object, ...] | None = ..., + closure: tuple[CellType, ...] | None = ..., + ) -> Self: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + @overload + def __get__(self, instance: None, owner: type, /) -> FunctionType: ... + @overload + def __get__(self, instance: object, owner: type | None = None, /) -> MethodType: ... + +LambdaType = FunctionType + +@final +class CodeType: + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + @property + def co_argcount(self) -> int: ... + @property + def co_posonlyargcount(self) -> int: ... + @property + def co_kwonlyargcount(self) -> int: ... + @property + def co_nlocals(self) -> int: ... + @property + def co_stacksize(self) -> int: ... + @property + def co_flags(self) -> int: ... + @property + def co_code(self) -> bytes: ... + @property + def co_consts(self) -> tuple[Any, ...]: ... + @property + def co_names(self) -> tuple[str, ...]: ... + @property + def co_varnames(self) -> tuple[str, ...]: ... + @property + def co_filename(self) -> str: ... + @property + def co_name(self) -> str: ... + @property + def co_firstlineno(self) -> int: ... + if sys.version_info >= (3, 10): + @property + @deprecated("Will be removed in Python 3.14. Use the co_lines() method instead.") + def co_lnotab(self) -> bytes: ... + else: + @property + def co_lnotab(self) -> bytes: ... + + @property + def co_freevars(self) -> tuple[str, ...]: ... + @property + def co_cellvars(self) -> tuple[str, ...]: ... + if sys.version_info >= (3, 10): + @property + def co_linetable(self) -> bytes: ... + def co_lines(self) -> Iterator[tuple[int, int, int | None]]: ... + if sys.version_info >= (3, 11): + @property + def co_exceptiontable(self) -> bytes: ... + @property + def co_qualname(self) -> str: ... + def co_positions(self) -> Iterable[tuple[int | None, int | None, int | None, int | None]]: ... + + if sys.version_info >= (3, 11): + def __new__( + cls, + argcount: int, + posonlyargcount: int, + kwonlyargcount: int, + nlocals: int, + stacksize: int, + flags: int, + codestring: bytes, + constants: tuple[object, ...], + names: tuple[str, ...], + varnames: tuple[str, ...], + filename: str, + name: str, + qualname: str, + firstlineno: int, + linetable: bytes, + exceptiontable: bytes, + freevars: tuple[str, ...] = ..., + cellvars: tuple[str, ...] = ..., + /, + ) -> Self: ... + elif sys.version_info >= (3, 10): + def __new__( + cls, + argcount: int, + posonlyargcount: int, + kwonlyargcount: int, + nlocals: int, + stacksize: int, + flags: int, + codestring: bytes, + constants: tuple[object, ...], + names: tuple[str, ...], + varnames: tuple[str, ...], + filename: str, + name: str, + firstlineno: int, + linetable: bytes, + freevars: tuple[str, ...] = ..., + cellvars: tuple[str, ...] = ..., + /, + ) -> Self: ... + else: + def __new__( + cls, + argcount: int, + posonlyargcount: int, + kwonlyargcount: int, + nlocals: int, + stacksize: int, + flags: int, + codestring: bytes, + constants: tuple[object, ...], + names: tuple[str, ...], + varnames: tuple[str, ...], + filename: str, + name: str, + firstlineno: int, + lnotab: bytes, + freevars: tuple[str, ...] = ..., + cellvars: tuple[str, ...] = ..., + /, + ) -> Self: ... + if sys.version_info >= (3, 11): + def replace( + self, + *, + co_argcount: int = -1, + co_posonlyargcount: int = -1, + co_kwonlyargcount: int = -1, + co_nlocals: int = -1, + co_stacksize: int = -1, + co_flags: int = -1, + co_firstlineno: int = -1, + co_code: bytes = ..., + co_consts: tuple[object, ...] = ..., + co_names: tuple[str, ...] = ..., + co_varnames: tuple[str, ...] = ..., + co_freevars: tuple[str, ...] = ..., + co_cellvars: tuple[str, ...] = ..., + co_filename: str = ..., + co_name: str = ..., + co_qualname: str = ..., + co_linetable: bytes = ..., + co_exceptiontable: bytes = ..., + ) -> Self: ... + elif sys.version_info >= (3, 10): + def replace( + self, + *, + co_argcount: int = -1, + co_posonlyargcount: int = -1, + co_kwonlyargcount: int = -1, + co_nlocals: int = -1, + co_stacksize: int = -1, + co_flags: int = -1, + co_firstlineno: int = -1, + co_code: bytes = ..., + co_consts: tuple[object, ...] = ..., + co_names: tuple[str, ...] = ..., + co_varnames: tuple[str, ...] = ..., + co_freevars: tuple[str, ...] = ..., + co_cellvars: tuple[str, ...] = ..., + co_filename: str = ..., + co_name: str = ..., + co_linetable: bytes = ..., + ) -> Self: ... + else: + def replace( + self, + *, + co_argcount: int = -1, + co_posonlyargcount: int = -1, + co_kwonlyargcount: int = -1, + co_nlocals: int = -1, + co_stacksize: int = -1, + co_flags: int = -1, + co_firstlineno: int = -1, + co_code: bytes = ..., + co_consts: tuple[object, ...] = ..., + co_names: tuple[str, ...] = ..., + co_varnames: tuple[str, ...] = ..., + co_freevars: tuple[str, ...] = ..., + co_cellvars: tuple[str, ...] = ..., + co_filename: str = ..., + co_name: str = ..., + co_lnotab: bytes = ..., + ) -> Self: ... + + if sys.version_info >= (3, 13): + __replace__ = replace + +@final +class MappingProxyType(Mapping[_KT, _VT_co]): + __hash__: ClassVar[None] # type: ignore[assignment] + def __new__(cls, mapping: SupportsKeysAndGetItem[_KT, _VT_co]) -> Self: ... + def __getitem__(self, key: _KT, /) -> _VT_co: ... + def __iter__(self) -> Iterator[_KT]: ... + def __len__(self) -> int: ... + def __eq__(self, value: object, /) -> bool: ... + def copy(self) -> dict[_KT, _VT_co]: ... + def keys(self) -> KeysView[_KT]: ... + def values(self) -> ValuesView[_VT_co]: ... + def items(self) -> ItemsView[_KT, _VT_co]: ... + @overload + def get(self, key: _KT, /) -> _VT_co | None: ... + @overload + def get(self, key: _KT, default: _VT_co | _T2, /) -> _VT_co | _T2: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + def __reversed__(self) -> Iterator[_KT]: ... + def __or__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... + def __ror__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... + +class SimpleNamespace: + __hash__: ClassVar[None] # type: ignore[assignment] + if sys.version_info >= (3, 13): + def __init__(self, mapping_or_iterable: Mapping[str, Any] | Iterable[tuple[str, Any]] = (), /, **kwargs: Any) -> None: ... + else: + def __init__(self, **kwargs: Any) -> None: ... + + def __eq__(self, value: object, /) -> bool: ... + def __getattribute__(self, name: str, /) -> Any: ... + def __setattr__(self, name: str, value: Any, /) -> None: ... + def __delattr__(self, name: str, /) -> None: ... + if sys.version_info >= (3, 13): + def __replace__(self, **kwargs: Any) -> Self: ... + +class ModuleType: + __name__: str + __file__: str | None + @property + def __dict__(self) -> dict[str, Any]: ... # type: ignore[override] + __loader__: LoaderProtocol | None + __package__: str | None + __path__: MutableSequence[str] + __spec__: ModuleSpec | None + # N.B. Although this is the same type as `builtins.object.__doc__`, + # it is deliberately redeclared here. Most symbols declared in the namespace + # of `types.ModuleType` are available as "implicit globals" within a module's + # namespace, but this is not true for symbols declared in the namespace of `builtins.object`. + # Redeclaring `__doc__` here helps some type checkers understand that `__doc__` is available + # as an implicit global in all modules, similar to `__name__`, `__file__`, `__spec__`, etc. + __doc__: str | None + def __init__(self, name: str, doc: str | None = ...) -> None: ... + # __getattr__ doesn't exist at runtime, + # but having it here in typeshed makes dynamic imports + # using `builtins.__import__` or `importlib.import_module` less painful + def __getattr__(self, name: str) -> Any: ... + +@final +class CellType: + def __new__(cls, contents: object = ..., /) -> Self: ... + __hash__: ClassVar[None] # type: ignore[assignment] + cell_contents: Any + +_YieldT_co = TypeVar("_YieldT_co", covariant=True) +_SendT_contra = TypeVar("_SendT_contra", contravariant=True) +_ReturnT_co = TypeVar("_ReturnT_co", covariant=True) + +@final +class GeneratorType(Generator[_YieldT_co, _SendT_contra, _ReturnT_co]): + @property + def gi_yieldfrom(self) -> GeneratorType[_YieldT_co, _SendT_contra, Any] | None: ... + if sys.version_info >= (3, 11): + @property + def gi_suspended(self) -> bool: ... + __name__: str + __qualname__: str + def __iter__(self) -> Self: ... + def __next__(self) -> _YieldT_co: ... + def send(self, arg: _SendT_contra, /) -> _YieldT_co: ... + @overload + def throw( + self, typ: type[BaseException], val: BaseException | object = ..., tb: TracebackType | None = ..., / + ) -> _YieldT_co: ... + @overload + def throw(self, typ: BaseException, val: None = None, tb: TracebackType | None = ..., /) -> _YieldT_co: ... + if sys.version_info >= (3, 13): + def __class_getitem__(cls, item: Any, /) -> Any: ... + +@final +class AsyncGeneratorType(AsyncGenerator[_YieldT_co, _SendT_contra]): + @property + def ag_await(self) -> Awaitable[Any] | None: ... + __name__: str + __qualname__: str + if sys.version_info >= (3, 12): + @property + def ag_suspended(self) -> bool: ... + + def __aiter__(self) -> Self: ... + def __anext__(self) -> Coroutine[Any, Any, _YieldT_co]: ... + def asend(self, val: _SendT_contra, /) -> Coroutine[Any, Any, _YieldT_co]: ... + @overload + async def athrow( + self, typ: type[BaseException], val: BaseException | object = ..., tb: TracebackType | None = ..., / + ) -> _YieldT_co: ... + @overload + async def athrow(self, typ: BaseException, val: None = None, tb: TracebackType | None = ..., /) -> _YieldT_co: ... + def aclose(self) -> Coroutine[Any, Any, None]: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +@final +class CoroutineType(Coroutine[_YieldT_co, _SendT_contra, _ReturnT_co]): + __name__: str + __qualname__: str + @property + def cr_origin(self) -> tuple[tuple[str, int, str], ...] | None: ... + if sys.version_info >= (3, 11): + @property + def cr_suspended(self) -> bool: ... + + def close(self) -> None: ... + def __await__(self) -> Generator[Any, None, _ReturnT_co]: ... + def send(self, arg: _SendT_contra, /) -> _YieldT_co: ... + @overload + def throw( + self, typ: type[BaseException], val: BaseException | object = ..., tb: TracebackType | None = ..., / + ) -> _YieldT_co: ... + @overload + def throw(self, typ: BaseException, val: None = None, tb: TracebackType | None = ..., /) -> _YieldT_co: ... + if sys.version_info >= (3, 13): + def __class_getitem__(cls, item: Any, /) -> Any: ... + +@final +class MethodType: + @property + def __closure__(self) -> tuple[CellType, ...] | None: ... # inherited from the added function + @property + def __code__(self) -> CodeType: ... # inherited from the added function + @property + def __defaults__(self) -> tuple[Any, ...] | None: ... # inherited from the added function + @property + def __func__(self) -> Callable[..., Any]: ... + @property + def __self__(self) -> object: ... + @property + def __name__(self) -> str: ... # inherited from the added function + @property + def __qualname__(self) -> str: ... # inherited from the added function + def __new__(cls, func: Callable[..., Any], obj: object, /) -> Self: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + +@final +class BuiltinFunctionType: + @property + def __self__(self) -> object | ModuleType: ... + @property + def __name__(self) -> str: ... + @property + def __qualname__(self) -> str: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + +BuiltinMethodType = BuiltinFunctionType + +@final +class WrapperDescriptorType: + @property + def __name__(self) -> str: ... + @property + def __qualname__(self) -> str: ... + @property + def __objclass__(self) -> type: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + def __get__(self, instance: Any, owner: type | None = None, /) -> Any: ... + +@final +class MethodWrapperType: + @property + def __self__(self) -> object: ... + @property + def __name__(self) -> str: ... + @property + def __qualname__(self) -> str: ... + @property + def __objclass__(self) -> type: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + +@final +class MethodDescriptorType: + @property + def __name__(self) -> str: ... + @property + def __qualname__(self) -> str: ... + @property + def __objclass__(self) -> type: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + def __get__(self, instance: Any, owner: type | None = None, /) -> Any: ... + +@final +class ClassMethodDescriptorType: + @property + def __name__(self) -> str: ... + @property + def __qualname__(self) -> str: ... + @property + def __objclass__(self) -> type: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + def __get__(self, instance: Any, owner: type | None = None, /) -> Any: ... + +@final +class TracebackType: + def __new__(cls, tb_next: TracebackType | None, tb_frame: FrameType, tb_lasti: int, tb_lineno: int) -> Self: ... + tb_next: TracebackType | None + # the rest are read-only + @property + def tb_frame(self) -> FrameType: ... + @property + def tb_lasti(self) -> int: ... + @property + def tb_lineno(self) -> int: ... + +@final +class FrameType: + @property + def f_back(self) -> FrameType | None: ... + @property + def f_builtins(self) -> dict[str, Any]: ... + @property + def f_code(self) -> CodeType: ... + @property + def f_globals(self) -> dict[str, Any]: ... + @property + def f_lasti(self) -> int: ... + # see discussion in #6769: f_lineno *can* sometimes be None, + # but you should probably file a bug report with CPython if you encounter it being None in the wild. + # An `int | None` annotation here causes too many false-positive errors, so applying `int | Any`. + @property + def f_lineno(self) -> int | MaybeNone: ... + @property + def f_locals(self) -> dict[str, Any]: ... + f_trace: Callable[[FrameType, str, Any], Any] | None + f_trace_lines: bool + f_trace_opcodes: bool + def clear(self) -> None: ... + +@final +class GetSetDescriptorType: + @property + def __name__(self) -> str: ... + @property + def __qualname__(self) -> str: ... + @property + def __objclass__(self) -> type: ... + def __get__(self, instance: Any, owner: type | None = None, /) -> Any: ... + def __set__(self, instance: Any, value: Any, /) -> None: ... + def __delete__(self, instance: Any, /) -> None: ... + +@final +class MemberDescriptorType: + @property + def __name__(self) -> str: ... + @property + def __qualname__(self) -> str: ... + @property + def __objclass__(self) -> type: ... + def __get__(self, instance: Any, owner: type | None = None, /) -> Any: ... + def __set__(self, instance: Any, value: Any, /) -> None: ... + def __delete__(self, instance: Any, /) -> None: ... + +def new_class( + name: str, + bases: Iterable[object] = (), + kwds: dict[str, Any] | None = None, + exec_body: Callable[[dict[str, Any]], object] | None = None, +) -> type: ... +def resolve_bases(bases: Iterable[object]) -> tuple[Any, ...]: ... +def prepare_class( + name: str, bases: tuple[type, ...] = (), kwds: dict[str, Any] | None = None +) -> tuple[type, dict[str, Any], dict[str, Any]]: ... + +if sys.version_info >= (3, 12): + def get_original_bases(cls: type, /) -> tuple[Any, ...]: ... + +# Actually a different type, but `property` is special and we want that too. +DynamicClassAttribute = property + +_Fn = TypeVar("_Fn", bound=Callable[..., object]) +_R = TypeVar("_R") +_P = ParamSpec("_P") + +# it's not really an Awaitable, but can be used in an await expression. Real type: Generator & Awaitable +@overload +def coroutine(func: Callable[_P, Generator[Any, Any, _R]]) -> Callable[_P, Awaitable[_R]]: ... +@overload +def coroutine(func: _Fn) -> _Fn: ... + +if sys.version_info >= (3, 9): + class GenericAlias: + @property + def __origin__(self) -> type: ... + @property + def __args__(self) -> tuple[Any, ...]: ... + @property + def __parameters__(self) -> tuple[Any, ...]: ... + def __new__(cls, origin: type, args: Any) -> Self: ... + def __getitem__(self, typeargs: Any, /) -> GenericAlias: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + if sys.version_info >= (3, 11): + @property + def __unpacked__(self) -> bool: ... + @property + def __typing_unpacked_tuple_args__(self) -> tuple[Any, ...] | None: ... + if sys.version_info >= (3, 10): + def __or__(self, value: Any, /) -> UnionType: ... + def __ror__(self, value: Any, /) -> UnionType: ... + + # GenericAlias delegates attr access to `__origin__` + def __getattr__(self, name: str) -> Any: ... + +if sys.version_info >= (3, 10): + @final + class NoneType: + def __bool__(self) -> Literal[False]: ... + + @final + class EllipsisType: ... + + from builtins import _NotImplementedType + + NotImplementedType = _NotImplementedType + @final + class UnionType: + @property + def __args__(self) -> tuple[Any, ...]: ... + def __or__(self, value: Any, /) -> UnionType: ... + def __ror__(self, value: Any, /) -> UnionType: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + +if sys.version_info >= (3, 13): + @final + class CapsuleType: ... diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi new file mode 100644 index 000000000000..8f0d4fbb6a02 --- /dev/null +++ b/mypy/typeshed/stdlib/typing.pyi @@ -0,0 +1,1068 @@ +# Since this module defines "overload" it is not recognized by Ruff as typing.overload +# ruff: noqa: F811 +# TODO: The collections import is required, otherwise mypy crashes. +# https://github.com/python/mypy/issues/16744 +import collections # noqa: F401 # pyright: ignore[reportUnusedImport] +import sys +import typing_extensions +from _collections_abc import dict_items, dict_keys, dict_values +from _typeshed import IdentityFunction, ReadableBuffer, SupportsKeysAndGetItem +from abc import ABCMeta, abstractmethod +from re import Match as Match, Pattern as Pattern +from types import ( + BuiltinFunctionType, + CodeType, + FrameType, + FunctionType, + MethodDescriptorType, + MethodType, + MethodWrapperType, + ModuleType, + TracebackType, + WrapperDescriptorType, +) +from typing_extensions import Never as _Never, ParamSpec as _ParamSpec, deprecated + +if sys.version_info >= (3, 9): + from types import GenericAlias +if sys.version_info >= (3, 10): + from types import UnionType + +__all__ = [ + "AbstractSet", + "Any", + "AnyStr", + "AsyncContextManager", + "AsyncGenerator", + "AsyncIterable", + "AsyncIterator", + "Awaitable", + "ByteString", + "Callable", + "ChainMap", + "ClassVar", + "Collection", + "Container", + "ContextManager", + "Coroutine", + "Counter", + "DefaultDict", + "Deque", + "Dict", + "Final", + "FrozenSet", + "Generator", + "Generic", + "Hashable", + "ItemsView", + "Iterable", + "Iterator", + "KeysView", + "List", + "Literal", + "Mapping", + "MappingView", + "MutableMapping", + "MutableSequence", + "MutableSet", + "NamedTuple", + "NewType", + "Optional", + "Protocol", + "Reversible", + "Sequence", + "Set", + "Sized", + "SupportsAbs", + "SupportsBytes", + "SupportsComplex", + "SupportsFloat", + "SupportsIndex", + "SupportsInt", + "SupportsRound", + "Text", + "Tuple", + "Type", + "TypeVar", + "TypedDict", + "Union", + "ValuesView", + "TYPE_CHECKING", + "cast", + "final", + "get_args", + "get_origin", + "get_type_hints", + "no_type_check", + "no_type_check_decorator", + "overload", + "runtime_checkable", + "ForwardRef", + "NoReturn", + "OrderedDict", +] + +if sys.version_info >= (3, 9): + __all__ += ["Annotated", "BinaryIO", "IO", "Match", "Pattern", "TextIO"] + +if sys.version_info >= (3, 10): + __all__ += ["Concatenate", "ParamSpec", "ParamSpecArgs", "ParamSpecKwargs", "TypeAlias", "TypeGuard", "is_typeddict"] + +if sys.version_info >= (3, 11): + __all__ += [ + "LiteralString", + "Never", + "NotRequired", + "Required", + "Self", + "TypeVarTuple", + "Unpack", + "assert_never", + "assert_type", + "clear_overloads", + "dataclass_transform", + "get_overloads", + "reveal_type", + ] + +if sys.version_info >= (3, 12): + __all__ += ["TypeAliasType", "override"] + +if sys.version_info >= (3, 13): + __all__ += ["get_protocol_members", "is_protocol", "NoDefault", "TypeIs", "ReadOnly"] + +Any = object() + +class _Final: ... + +def final(f: _T) -> _T: ... +@final +class TypeVar: + @property + def __name__(self) -> str: ... + @property + def __bound__(self) -> Any | None: ... + @property + def __constraints__(self) -> tuple[Any, ...]: ... + @property + def __covariant__(self) -> bool: ... + @property + def __contravariant__(self) -> bool: ... + if sys.version_info >= (3, 12): + @property + def __infer_variance__(self) -> bool: ... + if sys.version_info >= (3, 13): + @property + def __default__(self) -> Any: ... + if sys.version_info >= (3, 13): + def __init__( + self, + name: str, + *constraints: Any, + bound: Any | None = None, + contravariant: bool = False, + covariant: bool = False, + infer_variance: bool = False, + default: Any = ..., + ) -> None: ... + elif sys.version_info >= (3, 12): + def __init__( + self, + name: str, + *constraints: Any, + bound: Any | None = None, + covariant: bool = False, + contravariant: bool = False, + infer_variance: bool = False, + ) -> None: ... + else: + def __init__( + self, name: str, *constraints: Any, bound: Any | None = None, covariant: bool = False, contravariant: bool = False + ) -> None: ... + if sys.version_info >= (3, 10): + def __or__(self, right: Any) -> _SpecialForm: ... + def __ror__(self, left: Any) -> _SpecialForm: ... + if sys.version_info >= (3, 11): + def __typing_subst__(self, arg: Any) -> Any: ... + if sys.version_info >= (3, 13): + def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + def has_default(self) -> bool: ... + +# Used for an undocumented mypy feature. Does not exist at runtime. +_promote = object() + +# N.B. Keep this definition in sync with typing_extensions._SpecialForm +@final +class _SpecialForm(_Final): + def __getitem__(self, parameters: Any) -> object: ... + if sys.version_info >= (3, 10): + def __or__(self, other: Any) -> _SpecialForm: ... + def __ror__(self, other: Any) -> _SpecialForm: ... + +Union: _SpecialForm +Generic: _SpecialForm +# Protocol is only present in 3.8 and later, but mypy needs it unconditionally +Protocol: _SpecialForm +Callable: _SpecialForm +Type: _SpecialForm +NoReturn: _SpecialForm +ClassVar: _SpecialForm + +Optional: _SpecialForm +Tuple: _SpecialForm +Final: _SpecialForm + +Literal: _SpecialForm +TypedDict: _SpecialForm + +if sys.version_info >= (3, 11): + Self: _SpecialForm + Never: _SpecialForm + Unpack: _SpecialForm + Required: _SpecialForm + NotRequired: _SpecialForm + LiteralString: _SpecialForm + + @final + class TypeVarTuple: + @property + def __name__(self) -> str: ... + if sys.version_info >= (3, 13): + @property + def __default__(self) -> Any: ... + def has_default(self) -> bool: ... + if sys.version_info >= (3, 13): + def __init__(self, name: str, *, default: Any = ...) -> None: ... + else: + def __init__(self, name: str) -> None: ... + + def __iter__(self) -> Any: ... + def __typing_subst__(self, arg: Never) -> Never: ... + def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + +if sys.version_info >= (3, 10): + @final + class ParamSpecArgs: + @property + def __origin__(self) -> ParamSpec: ... + def __init__(self, origin: ParamSpec) -> None: ... + def __eq__(self, other: object) -> bool: ... + + @final + class ParamSpecKwargs: + @property + def __origin__(self) -> ParamSpec: ... + def __init__(self, origin: ParamSpec) -> None: ... + def __eq__(self, other: object) -> bool: ... + + @final + class ParamSpec: + @property + def __name__(self) -> str: ... + @property + def __bound__(self) -> Any | None: ... + @property + def __covariant__(self) -> bool: ... + @property + def __contravariant__(self) -> bool: ... + if sys.version_info >= (3, 12): + @property + def __infer_variance__(self) -> bool: ... + if sys.version_info >= (3, 13): + @property + def __default__(self) -> Any: ... + if sys.version_info >= (3, 13): + def __init__( + self, + name: str, + *, + bound: Any | None = None, + contravariant: bool = False, + covariant: bool = False, + infer_variance: bool = False, + default: Any = ..., + ) -> None: ... + elif sys.version_info >= (3, 12): + def __init__( + self, + name: str, + *, + bound: Any | None = None, + contravariant: bool = False, + covariant: bool = False, + infer_variance: bool = False, + ) -> None: ... + else: + def __init__( + self, name: str, *, bound: Any | None = None, contravariant: bool = False, covariant: bool = False + ) -> None: ... + + @property + def args(self) -> ParamSpecArgs: ... + @property + def kwargs(self) -> ParamSpecKwargs: ... + if sys.version_info >= (3, 11): + def __typing_subst__(self, arg: Any) -> Any: ... + def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + + def __or__(self, right: Any) -> _SpecialForm: ... + def __ror__(self, left: Any) -> _SpecialForm: ... + if sys.version_info >= (3, 13): + def has_default(self) -> bool: ... + + Concatenate: _SpecialForm + TypeAlias: _SpecialForm + TypeGuard: _SpecialForm + + class NewType: + def __init__(self, name: str, tp: Any) -> None: ... + if sys.version_info >= (3, 11): + @staticmethod + def __call__(x: _T, /) -> _T: ... + else: + def __call__(self, x: _T) -> _T: ... + + def __or__(self, other: Any) -> _SpecialForm: ... + def __ror__(self, other: Any) -> _SpecialForm: ... + __supertype__: type | NewType + +else: + def NewType(name: str, tp: Any) -> Any: ... + +_F = TypeVar("_F", bound=Callable[..., Any]) +_P = _ParamSpec("_P") +_T = TypeVar("_T") + +# These type variables are used by the container types. +_S = TypeVar("_S") +_KT = TypeVar("_KT") # Key type. +_VT = TypeVar("_VT") # Value type. +_T_co = TypeVar("_T_co", covariant=True) # Any type covariant containers. +_KT_co = TypeVar("_KT_co", covariant=True) # Key type covariant containers. +_VT_co = TypeVar("_VT_co", covariant=True) # Value type covariant containers. +_TC = TypeVar("_TC", bound=type[object]) + +def overload(func: _F) -> _F: ... +def no_type_check(arg: _F) -> _F: ... +def no_type_check_decorator(decorator: Callable[_P, _T]) -> Callable[_P, _T]: ... + +# This itself is only available during type checking +def type_check_only(func_or_cls: _F) -> _F: ... + +# Type aliases and type constructors + +class _Alias: + # Class for defining generic aliases for library types. + def __getitem__(self, typeargs: Any) -> Any: ... + +List = _Alias() +Dict = _Alias() +DefaultDict = _Alias() +Set = _Alias() +FrozenSet = _Alias() +Counter = _Alias() +Deque = _Alias() +ChainMap = _Alias() + +OrderedDict = _Alias() + +if sys.version_info >= (3, 9): + Annotated: _SpecialForm + +# Predefined type variables. +AnyStr = TypeVar("AnyStr", str, bytes) # noqa: Y001 + +class _ProtocolMeta(ABCMeta): + if sys.version_info >= (3, 12): + def __init__(cls, *args: Any, **kwargs: Any) -> None: ... + +# Abstract base classes. + +def runtime_checkable(cls: _TC) -> _TC: ... +@runtime_checkable +class SupportsInt(Protocol, metaclass=ABCMeta): + @abstractmethod + def __int__(self) -> int: ... + +@runtime_checkable +class SupportsFloat(Protocol, metaclass=ABCMeta): + @abstractmethod + def __float__(self) -> float: ... + +@runtime_checkable +class SupportsComplex(Protocol, metaclass=ABCMeta): + @abstractmethod + def __complex__(self) -> complex: ... + +@runtime_checkable +class SupportsBytes(Protocol, metaclass=ABCMeta): + @abstractmethod + def __bytes__(self) -> bytes: ... + +@runtime_checkable +class SupportsIndex(Protocol, metaclass=ABCMeta): + @abstractmethod + def __index__(self) -> int: ... + +@runtime_checkable +class SupportsAbs(Protocol[_T_co]): + @abstractmethod + def __abs__(self) -> _T_co: ... + +@runtime_checkable +class SupportsRound(Protocol[_T_co]): + @overload + @abstractmethod + def __round__(self) -> int: ... + @overload + @abstractmethod + def __round__(self, ndigits: int, /) -> _T_co: ... + +@runtime_checkable +class Sized(Protocol, metaclass=ABCMeta): + @abstractmethod + def __len__(self) -> int: ... + +@runtime_checkable +class Hashable(Protocol, metaclass=ABCMeta): + # TODO: This is special, in that a subclass of a hashable class may not be hashable + # (for example, list vs. object). It's not obvious how to represent this. This class + # is currently mostly useless for static checking. + @abstractmethod + def __hash__(self) -> int: ... + +@runtime_checkable +class Iterable(Protocol[_T_co]): + @abstractmethod + def __iter__(self) -> Iterator[_T_co]: ... + +@runtime_checkable +class Iterator(Iterable[_T_co], Protocol[_T_co]): + @abstractmethod + def __next__(self) -> _T_co: ... + def __iter__(self) -> Iterator[_T_co]: ... + +@runtime_checkable +class Reversible(Iterable[_T_co], Protocol[_T_co]): + @abstractmethod + def __reversed__(self) -> Iterator[_T_co]: ... + +_YieldT_co = TypeVar("_YieldT_co", covariant=True) +_SendT_contra = TypeVar("_SendT_contra", contravariant=True, default=None) +_ReturnT_co = TypeVar("_ReturnT_co", covariant=True, default=None) + +class Generator(Iterator[_YieldT_co], Generic[_YieldT_co, _SendT_contra, _ReturnT_co]): + def __next__(self) -> _YieldT_co: ... + @abstractmethod + def send(self, value: _SendT_contra, /) -> _YieldT_co: ... + @overload + @abstractmethod + def throw( + self, typ: type[BaseException], val: BaseException | object = None, tb: TracebackType | None = None, / + ) -> _YieldT_co: ... + @overload + @abstractmethod + def throw(self, typ: BaseException, val: None = None, tb: TracebackType | None = None, /) -> _YieldT_co: ... + if sys.version_info >= (3, 13): + def close(self) -> _ReturnT_co | None: ... + else: + def close(self) -> None: ... + + def __iter__(self) -> Generator[_YieldT_co, _SendT_contra, _ReturnT_co]: ... + @property + def gi_code(self) -> CodeType: ... + @property + def gi_frame(self) -> FrameType: ... + @property + def gi_running(self) -> bool: ... + @property + def gi_yieldfrom(self) -> Generator[Any, Any, Any] | None: ... + +# NOTE: Prior to Python 3.13 these aliases are lacking the second _ExitT_co parameter +if sys.version_info >= (3, 13): + from contextlib import AbstractAsyncContextManager as AsyncContextManager, AbstractContextManager as ContextManager +else: + from contextlib import AbstractAsyncContextManager, AbstractContextManager + + @runtime_checkable + class ContextManager(AbstractContextManager[_T_co, bool | None], Protocol[_T_co]): ... + + @runtime_checkable + class AsyncContextManager(AbstractAsyncContextManager[_T_co, bool | None], Protocol[_T_co]): ... + +@runtime_checkable +class Awaitable(Protocol[_T_co]): + @abstractmethod + def __await__(self) -> Generator[Any, Any, _T_co]: ... + +# Non-default variations to accommodate couroutines, and `AwaitableGenerator` having a 4th type parameter. +_SendT_contra_nd = TypeVar("_SendT_contra_nd", contravariant=True) +_ReturnT_co_nd = TypeVar("_ReturnT_co_nd", covariant=True) + +class Coroutine(Awaitable[_ReturnT_co_nd], Generic[_YieldT_co, _SendT_contra_nd, _ReturnT_co_nd]): + __name__: str + __qualname__: str + @property + def cr_await(self) -> Any | None: ... + @property + def cr_code(self) -> CodeType: ... + @property + def cr_frame(self) -> FrameType | None: ... + @property + def cr_running(self) -> bool: ... + @abstractmethod + def send(self, value: _SendT_contra_nd, /) -> _YieldT_co: ... + @overload + @abstractmethod + def throw( + self, typ: type[BaseException], val: BaseException | object = None, tb: TracebackType | None = None, / + ) -> _YieldT_co: ... + @overload + @abstractmethod + def throw(self, typ: BaseException, val: None = None, tb: TracebackType | None = None, /) -> _YieldT_co: ... + @abstractmethod + def close(self) -> None: ... + +# NOTE: This type does not exist in typing.py or PEP 484 but mypy needs it to exist. +# The parameters correspond to Generator, but the 4th is the original type. +@type_check_only +class AwaitableGenerator( + Awaitable[_ReturnT_co_nd], + Generator[_YieldT_co, _SendT_contra_nd, _ReturnT_co_nd], + Generic[_YieldT_co, _SendT_contra_nd, _ReturnT_co_nd, _S], + metaclass=ABCMeta, +): ... + +@runtime_checkable +class AsyncIterable(Protocol[_T_co]): + @abstractmethod + def __aiter__(self) -> AsyncIterator[_T_co]: ... + +@runtime_checkable +class AsyncIterator(AsyncIterable[_T_co], Protocol[_T_co]): + @abstractmethod + def __anext__(self) -> Awaitable[_T_co]: ... + def __aiter__(self) -> AsyncIterator[_T_co]: ... + +class AsyncGenerator(AsyncIterator[_YieldT_co], Generic[_YieldT_co, _SendT_contra]): + def __anext__(self) -> Coroutine[Any, Any, _YieldT_co]: ... + @abstractmethod + def asend(self, value: _SendT_contra, /) -> Coroutine[Any, Any, _YieldT_co]: ... + @overload + @abstractmethod + def athrow( + self, typ: type[BaseException], val: BaseException | object = None, tb: TracebackType | None = None, / + ) -> Coroutine[Any, Any, _YieldT_co]: ... + @overload + @abstractmethod + def athrow( + self, typ: BaseException, val: None = None, tb: TracebackType | None = None, / + ) -> Coroutine[Any, Any, _YieldT_co]: ... + def aclose(self) -> Coroutine[Any, Any, None]: ... + @property + def ag_await(self) -> Any: ... + @property + def ag_code(self) -> CodeType: ... + @property + def ag_frame(self) -> FrameType: ... + @property + def ag_running(self) -> bool: ... + +@runtime_checkable +class Container(Protocol[_T_co]): + # This is generic more on vibes than anything else + @abstractmethod + def __contains__(self, x: object, /) -> bool: ... + +@runtime_checkable +class Collection(Iterable[_T_co], Container[_T_co], Protocol[_T_co]): + # Implement Sized (but don't have it as a base class). + @abstractmethod + def __len__(self) -> int: ... + +class Sequence(Reversible[_T_co], Collection[_T_co]): + @overload + @abstractmethod + def __getitem__(self, index: int) -> _T_co: ... + @overload + @abstractmethod + def __getitem__(self, index: slice) -> Sequence[_T_co]: ... + # Mixin methods + def index(self, value: Any, start: int = 0, stop: int = ...) -> int: ... + def count(self, value: Any) -> int: ... + def __contains__(self, value: object) -> bool: ... + def __iter__(self) -> Iterator[_T_co]: ... + def __reversed__(self) -> Iterator[_T_co]: ... + +class MutableSequence(Sequence[_T]): + @abstractmethod + def insert(self, index: int, value: _T) -> None: ... + @overload + @abstractmethod + def __getitem__(self, index: int) -> _T: ... + @overload + @abstractmethod + def __getitem__(self, index: slice) -> MutableSequence[_T]: ... + @overload + @abstractmethod + def __setitem__(self, index: int, value: _T) -> None: ... + @overload + @abstractmethod + def __setitem__(self, index: slice, value: Iterable[_T]) -> None: ... + @overload + @abstractmethod + def __delitem__(self, index: int) -> None: ... + @overload + @abstractmethod + def __delitem__(self, index: slice) -> None: ... + # Mixin methods + def append(self, value: _T) -> None: ... + def clear(self) -> None: ... + def extend(self, values: Iterable[_T]) -> None: ... + def reverse(self) -> None: ... + def pop(self, index: int = -1) -> _T: ... + def remove(self, value: _T) -> None: ... + def __iadd__(self, values: Iterable[_T]) -> typing_extensions.Self: ... + +class AbstractSet(Collection[_T_co]): + @abstractmethod + def __contains__(self, x: object) -> bool: ... + def _hash(self) -> int: ... + # Mixin methods + def __le__(self, other: AbstractSet[Any]) -> bool: ... + def __lt__(self, other: AbstractSet[Any]) -> bool: ... + def __gt__(self, other: AbstractSet[Any]) -> bool: ... + def __ge__(self, other: AbstractSet[Any]) -> bool: ... + def __and__(self, other: AbstractSet[Any]) -> AbstractSet[_T_co]: ... + def __or__(self, other: AbstractSet[_T]) -> AbstractSet[_T_co | _T]: ... + def __sub__(self, other: AbstractSet[Any]) -> AbstractSet[_T_co]: ... + def __xor__(self, other: AbstractSet[_T]) -> AbstractSet[_T_co | _T]: ... + def __eq__(self, other: object) -> bool: ... + def isdisjoint(self, other: Iterable[Any]) -> bool: ... + +class MutableSet(AbstractSet[_T]): + @abstractmethod + def add(self, value: _T) -> None: ... + @abstractmethod + def discard(self, value: _T) -> None: ... + # Mixin methods + def clear(self) -> None: ... + def pop(self) -> _T: ... + def remove(self, value: _T) -> None: ... + def __ior__(self, it: AbstractSet[_T]) -> typing_extensions.Self: ... # type: ignore[override,misc] + def __iand__(self, it: AbstractSet[Any]) -> typing_extensions.Self: ... + def __ixor__(self, it: AbstractSet[_T]) -> typing_extensions.Self: ... # type: ignore[override,misc] + def __isub__(self, it: AbstractSet[Any]) -> typing_extensions.Self: ... + +class MappingView(Sized): + def __init__(self, mapping: Mapping[Any, Any]) -> None: ... # undocumented + def __len__(self) -> int: ... + +class ItemsView(MappingView, AbstractSet[tuple[_KT_co, _VT_co]], Generic[_KT_co, _VT_co]): + def __init__(self, mapping: Mapping[_KT_co, _VT_co]) -> None: ... # undocumented + def __and__(self, other: Iterable[Any]) -> set[tuple[_KT_co, _VT_co]]: ... + def __rand__(self, other: Iterable[_T]) -> set[_T]: ... + def __contains__(self, item: object) -> bool: ... + def __iter__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... + def __or__(self, other: Iterable[_T]) -> set[tuple[_KT_co, _VT_co] | _T]: ... + def __ror__(self, other: Iterable[_T]) -> set[tuple[_KT_co, _VT_co] | _T]: ... + def __sub__(self, other: Iterable[Any]) -> set[tuple[_KT_co, _VT_co]]: ... + def __rsub__(self, other: Iterable[_T]) -> set[_T]: ... + def __xor__(self, other: Iterable[_T]) -> set[tuple[_KT_co, _VT_co] | _T]: ... + def __rxor__(self, other: Iterable[_T]) -> set[tuple[_KT_co, _VT_co] | _T]: ... + +class KeysView(MappingView, AbstractSet[_KT_co]): + def __init__(self, mapping: Mapping[_KT_co, Any]) -> None: ... # undocumented + def __and__(self, other: Iterable[Any]) -> set[_KT_co]: ... + def __rand__(self, other: Iterable[_T]) -> set[_T]: ... + def __contains__(self, key: object) -> bool: ... + def __iter__(self) -> Iterator[_KT_co]: ... + def __or__(self, other: Iterable[_T]) -> set[_KT_co | _T]: ... + def __ror__(self, other: Iterable[_T]) -> set[_KT_co | _T]: ... + def __sub__(self, other: Iterable[Any]) -> set[_KT_co]: ... + def __rsub__(self, other: Iterable[_T]) -> set[_T]: ... + def __xor__(self, other: Iterable[_T]) -> set[_KT_co | _T]: ... + def __rxor__(self, other: Iterable[_T]) -> set[_KT_co | _T]: ... + +class ValuesView(MappingView, Collection[_VT_co]): + def __init__(self, mapping: Mapping[Any, _VT_co]) -> None: ... # undocumented + def __contains__(self, value: object) -> bool: ... + def __iter__(self) -> Iterator[_VT_co]: ... + +class Mapping(Collection[_KT], Generic[_KT, _VT_co]): + # TODO: We wish the key type could also be covariant, but that doesn't work, + # see discussion in https://github.com/python/typing/pull/273. + @abstractmethod + def __getitem__(self, key: _KT, /) -> _VT_co: ... + # Mixin methods + @overload + def get(self, key: _KT, /) -> _VT_co | None: ... + @overload + def get(self, key: _KT, /, default: _VT_co | _T) -> _VT_co | _T: ... + def items(self) -> ItemsView[_KT, _VT_co]: ... + def keys(self) -> KeysView[_KT]: ... + def values(self) -> ValuesView[_VT_co]: ... + def __contains__(self, key: object, /) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... + +class MutableMapping(Mapping[_KT, _VT]): + @abstractmethod + def __setitem__(self, key: _KT, value: _VT, /) -> None: ... + @abstractmethod + def __delitem__(self, key: _KT, /) -> None: ... + def clear(self) -> None: ... + @overload + def pop(self, key: _KT, /) -> _VT: ... + @overload + def pop(self, key: _KT, /, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT, /, default: _T) -> _VT | _T: ... + def popitem(self) -> tuple[_KT, _VT]: ... + # This overload should be allowed only if the value type is compatible with None. + # + # Keep the following methods in line with MutableMapping.setdefault, modulo positional-only differences: + # -- collections.OrderedDict.setdefault + # -- collections.ChainMap.setdefault + # -- weakref.WeakKeyDictionary.setdefault + @overload + def setdefault(self: MutableMapping[_KT, _T | None], key: _KT, default: None = None, /) -> _T | None: ... + @overload + def setdefault(self, key: _KT, default: _VT, /) -> _VT: ... + # 'update' used to take a Union, but using overloading is better. + # The second overloaded type here is a bit too general, because + # Mapping[tuple[_KT, _VT], W] is a subclass of Iterable[tuple[_KT, _VT]], + # but will always have the behavior of the first overloaded type + # at runtime, leading to keys of a mix of types _KT and tuple[_KT, _VT]. + # We don't currently have any way of forcing all Mappings to use + # the first overload, but by using overloading rather than a Union, + # mypy will commit to using the first overload when the argument is + # known to be a Mapping with unknown type parameters, which is closer + # to the behavior we want. See mypy issue #1430. + # + # Various mapping classes have __ior__ methods that should be kept roughly in line with .update(): + # -- dict.__ior__ + # -- os._Environ.__ior__ + # -- collections.UserDict.__ior__ + # -- collections.ChainMap.__ior__ + # -- peewee.attrdict.__add__ + # -- peewee.attrdict.__iadd__ + # -- weakref.WeakValueDictionary.__ior__ + # -- weakref.WeakKeyDictionary.__ior__ + @overload + def update(self, m: SupportsKeysAndGetItem[_KT, _VT], /, **kwargs: _VT) -> None: ... + @overload + def update(self, m: Iterable[tuple[_KT, _VT]], /, **kwargs: _VT) -> None: ... + @overload + def update(self, **kwargs: _VT) -> None: ... + +Text = str + +TYPE_CHECKING: bool + +# In stubs, the arguments of the IO class are marked as positional-only. +# This differs from runtime, but better reflects the fact that in reality +# classes deriving from IO use different names for the arguments. +class IO(Generic[AnyStr]): + # At runtime these are all abstract properties, + # but making them abstract in the stub is hugely disruptive, for not much gain. + # See #8726 + @property + def mode(self) -> str: ... + # Usually str, but may be bytes if a bytes path was passed to open(). See #10737. + # If PEP 696 becomes available, we may want to use a defaulted TypeVar here. + @property + def name(self) -> str | Any: ... + @abstractmethod + def close(self) -> None: ... + @property + def closed(self) -> bool: ... + @abstractmethod + def fileno(self) -> int: ... + @abstractmethod + def flush(self) -> None: ... + @abstractmethod + def isatty(self) -> bool: ... + @abstractmethod + def read(self, n: int = -1, /) -> AnyStr: ... + @abstractmethod + def readable(self) -> bool: ... + @abstractmethod + def readline(self, limit: int = -1, /) -> AnyStr: ... + @abstractmethod + def readlines(self, hint: int = -1, /) -> list[AnyStr]: ... + @abstractmethod + def seek(self, offset: int, whence: int = 0, /) -> int: ... + @abstractmethod + def seekable(self) -> bool: ... + @abstractmethod + def tell(self) -> int: ... + @abstractmethod + def truncate(self, size: int | None = None, /) -> int: ... + @abstractmethod + def writable(self) -> bool: ... + @abstractmethod + @overload + def write(self: IO[bytes], s: ReadableBuffer, /) -> int: ... + @abstractmethod + @overload + def write(self, s: AnyStr, /) -> int: ... + @abstractmethod + @overload + def writelines(self: IO[bytes], lines: Iterable[ReadableBuffer], /) -> None: ... + @abstractmethod + @overload + def writelines(self, lines: Iterable[AnyStr], /) -> None: ... + @abstractmethod + def __next__(self) -> AnyStr: ... + @abstractmethod + def __iter__(self) -> Iterator[AnyStr]: ... + @abstractmethod + def __enter__(self) -> IO[AnyStr]: ... + @abstractmethod + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None, / + ) -> None: ... + +class BinaryIO(IO[bytes]): + @abstractmethod + def __enter__(self) -> BinaryIO: ... + +class TextIO(IO[str]): + # See comment regarding the @properties in the `IO` class + @property + def buffer(self) -> BinaryIO: ... + @property + def encoding(self) -> str: ... + @property + def errors(self) -> str | None: ... + @property + def line_buffering(self) -> int: ... # int on PyPy, bool on CPython + @property + def newlines(self) -> Any: ... # None, str or tuple + @abstractmethod + def __enter__(self) -> TextIO: ... + +if sys.version_info < (3, 14): + ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview + +# Functions + +_get_type_hints_obj_allowed_types: typing_extensions.TypeAlias = ( # noqa: Y042 + object + | Callable[..., Any] + | FunctionType + | BuiltinFunctionType + | MethodType + | ModuleType + | WrapperDescriptorType + | MethodWrapperType + | MethodDescriptorType +) + +if sys.version_info >= (3, 9): + def get_type_hints( + obj: _get_type_hints_obj_allowed_types, + globalns: dict[str, Any] | None = None, + localns: Mapping[str, Any] | None = None, + include_extras: bool = False, + ) -> dict[str, Any]: ... + +else: + def get_type_hints( + obj: _get_type_hints_obj_allowed_types, globalns: dict[str, Any] | None = None, localns: Mapping[str, Any] | None = None + ) -> dict[str, Any]: ... + +def get_args(tp: Any) -> tuple[Any, ...]: ... + +if sys.version_info >= (3, 10): + @overload + def get_origin(tp: ParamSpecArgs | ParamSpecKwargs) -> ParamSpec: ... + @overload + def get_origin(tp: UnionType) -> type[UnionType]: ... + +if sys.version_info >= (3, 9): + @overload + def get_origin(tp: GenericAlias) -> type: ... + @overload + def get_origin(tp: Any) -> Any | None: ... + +else: + def get_origin(tp: Any) -> Any | None: ... + +@overload +def cast(typ: type[_T], val: Any) -> _T: ... +@overload +def cast(typ: str, val: Any) -> Any: ... +@overload +def cast(typ: object, val: Any) -> Any: ... + +if sys.version_info >= (3, 11): + def reveal_type(obj: _T, /) -> _T: ... + def assert_never(arg: Never, /) -> Never: ... + def assert_type(val: _T, typ: Any, /) -> _T: ... + def clear_overloads() -> None: ... + def get_overloads(func: Callable[..., object]) -> Sequence[Callable[..., object]]: ... + def dataclass_transform( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + frozen_default: bool = False, # on 3.11, runtime accepts it as part of kwargs + field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), + **kwargs: Any, + ) -> IdentityFunction: ... + +# Type constructors + +class NamedTuple(tuple[Any, ...]): + if sys.version_info < (3, 9): + _field_types: ClassVar[dict[str, type]] + _field_defaults: ClassVar[dict[str, Any]] + _fields: ClassVar[tuple[str, ...]] + # __orig_bases__ sometimes exists on <3.12, but not consistently + # So we only add it to the stub on 3.12+. + if sys.version_info >= (3, 12): + __orig_bases__: ClassVar[tuple[Any, ...]] + + @overload + def __init__(self, typename: str, fields: Iterable[tuple[str, Any]], /) -> None: ... + @overload + @typing_extensions.deprecated( + "Creating a typing.NamedTuple using keyword arguments is deprecated and support will be removed in Python 3.15" + ) + def __init__(self, typename: str, fields: None = None, /, **kwargs: Any) -> None: ... + @classmethod + def _make(cls, iterable: Iterable[Any]) -> typing_extensions.Self: ... + def _asdict(self) -> dict[str, Any]: ... + def _replace(self, **kwargs: Any) -> typing_extensions.Self: ... + if sys.version_info >= (3, 13): + def __replace__(self, **kwargs: Any) -> typing_extensions.Self: ... + +# Internal mypy fallback type for all typed dicts (does not exist at runtime) +# N.B. Keep this mostly in sync with typing_extensions._TypedDict/mypy_extensions._TypedDict +@type_check_only +class _TypedDict(Mapping[str, object], metaclass=ABCMeta): + __total__: ClassVar[bool] + if sys.version_info >= (3, 9): + __required_keys__: ClassVar[frozenset[str]] + __optional_keys__: ClassVar[frozenset[str]] + # __orig_bases__ sometimes exists on <3.12, but not consistently, + # so we only add it to the stub on 3.12+ + if sys.version_info >= (3, 12): + __orig_bases__: ClassVar[tuple[Any, ...]] + + def copy(self) -> typing_extensions.Self: ... + # Using Never so that only calls using mypy plugin hook that specialize the signature + # can go through. + def setdefault(self, k: _Never, default: object) -> object: ... + # Mypy plugin hook for 'pop' expects that 'default' has a type variable type. + def pop(self, k: _Never, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] + def update(self: _T, m: _T, /) -> None: ... + def __delitem__(self, k: _Never) -> None: ... + def items(self) -> dict_items[str, object]: ... + def keys(self) -> dict_keys[str, object]: ... + def values(self) -> dict_values[str, object]: ... + if sys.version_info >= (3, 9): + @overload + def __or__(self, value: typing_extensions.Self, /) -> typing_extensions.Self: ... + @overload + def __or__(self, value: dict[str, Any], /) -> dict[str, object]: ... + @overload + def __ror__(self, value: typing_extensions.Self, /) -> typing_extensions.Self: ... + @overload + def __ror__(self, value: dict[str, Any], /) -> dict[str, object]: ... + # supposedly incompatible definitions of __or__ and __ior__ + def __ior__(self, value: typing_extensions.Self, /) -> typing_extensions.Self: ... # type: ignore[misc] + +@final +class ForwardRef(_Final): + __forward_arg__: str + __forward_code__: CodeType + __forward_evaluated__: bool + __forward_value__: Any | None + __forward_is_argument__: bool + __forward_is_class__: bool + __forward_module__: Any | None + if sys.version_info >= (3, 9): + # The module and is_class arguments were added in later Python 3.9 versions. + def __init__(self, arg: str, is_argument: bool = True, module: Any | None = None, *, is_class: bool = False) -> None: ... + else: + def __init__(self, arg: str, is_argument: bool = True) -> None: ... + + if sys.version_info >= (3, 13): + @overload + @deprecated( + "Failing to pass a value to the 'type_params' parameter of ForwardRef._evaluate() is deprecated, " + "as it leads to incorrect behaviour when evaluating a stringified annotation " + "that references a PEP 695 type parameter. It will be disallowed in Python 3.15." + ) + def _evaluate( + self, globalns: dict[str, Any] | None, localns: Mapping[str, Any] | None, *, recursive_guard: frozenset[str] + ) -> Any | None: ... + @overload + def _evaluate( + self, + globalns: dict[str, Any] | None, + localns: Mapping[str, Any] | None, + type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...], + *, + recursive_guard: frozenset[str], + ) -> Any | None: ... + elif sys.version_info >= (3, 12): + def _evaluate( + self, + globalns: dict[str, Any] | None, + localns: Mapping[str, Any] | None, + type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] | None = None, + *, + recursive_guard: frozenset[str], + ) -> Any | None: ... + elif sys.version_info >= (3, 9): + def _evaluate( + self, globalns: dict[str, Any] | None, localns: Mapping[str, Any] | None, recursive_guard: frozenset[str] + ) -> Any | None: ... + else: + def _evaluate(self, globalns: dict[str, Any] | None, localns: Mapping[str, Any] | None) -> Any | None: ... + + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + if sys.version_info >= (3, 11): + def __or__(self, other: Any) -> _SpecialForm: ... + def __ror__(self, other: Any) -> _SpecialForm: ... + +if sys.version_info >= (3, 10): + def is_typeddict(tp: object) -> bool: ... + +def _type_repr(obj: object) -> str: ... + +if sys.version_info >= (3, 12): + def override(method: _F, /) -> _F: ... + @final + class TypeAliasType: + def __init__( + self, name: str, value: Any, *, type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] = () + ) -> None: ... + @property + def __value__(self) -> Any: ... + @property + def __type_params__(self) -> tuple[TypeVar | ParamSpec | TypeVarTuple, ...]: ... + @property + def __parameters__(self) -> tuple[Any, ...]: ... + @property + def __name__(self) -> str: ... + # It's writable on types, but not on instances of TypeAliasType. + @property + def __module__(self) -> str | None: ... # type: ignore[override] + def __getitem__(self, parameters: Any) -> GenericAlias: ... + def __or__(self, right: Any) -> _SpecialForm: ... + def __ror__(self, left: Any) -> _SpecialForm: ... + +if sys.version_info >= (3, 13): + def is_protocol(tp: type, /) -> bool: ... + def get_protocol_members(tp: type, /) -> frozenset[str]: ... + @final + class _NoDefaultType: ... + + NoDefault: _NoDefaultType + TypeIs: _SpecialForm + ReadOnly: _SpecialForm diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi new file mode 100644 index 000000000000..a6b606e6b670 --- /dev/null +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -0,0 +1,536 @@ +import abc +import sys +import typing +from _collections_abc import dict_items, dict_keys, dict_values +from _typeshed import IdentityFunction +from contextlib import AbstractAsyncContextManager as AsyncContextManager, AbstractContextManager as ContextManager +from typing import ( # noqa: Y022,Y037,Y038,Y039 + IO as IO, + TYPE_CHECKING as TYPE_CHECKING, + AbstractSet as AbstractSet, + Any as Any, + AnyStr as AnyStr, + AsyncGenerator as AsyncGenerator, + AsyncIterable as AsyncIterable, + AsyncIterator as AsyncIterator, + Awaitable as Awaitable, + BinaryIO as BinaryIO, + Callable as Callable, + ChainMap as ChainMap, + ClassVar as ClassVar, + Collection as Collection, + Container as Container, + Coroutine as Coroutine, + Counter as Counter, + DefaultDict as DefaultDict, + Deque as Deque, + Dict as Dict, + ForwardRef as ForwardRef, + FrozenSet as FrozenSet, + Generator as Generator, + Generic as Generic, + Hashable as Hashable, + ItemsView as ItemsView, + Iterable as Iterable, + Iterator as Iterator, + KeysView as KeysView, + List as List, + Mapping as Mapping, + MappingView as MappingView, + Match as Match, + MutableMapping as MutableMapping, + MutableSequence as MutableSequence, + MutableSet as MutableSet, + NoReturn as NoReturn, + Optional as Optional, + Pattern as Pattern, + Reversible as Reversible, + Sequence as Sequence, + Set as Set, + Sized as Sized, + SupportsAbs as SupportsAbs, + SupportsBytes as SupportsBytes, + SupportsComplex as SupportsComplex, + SupportsFloat as SupportsFloat, + SupportsInt as SupportsInt, + SupportsRound as SupportsRound, + Text as Text, + TextIO as TextIO, + Tuple as Tuple, + Type as Type, + TypedDict as TypedDict, + Union as Union, + ValuesView as ValuesView, + _Alias, + cast as cast, + no_type_check as no_type_check, + no_type_check_decorator as no_type_check_decorator, + overload as overload, + type_check_only, +) + +if sys.version_info >= (3, 10): + from types import UnionType +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "Any", + "Buffer", + "ClassVar", + "Concatenate", + "Final", + "LiteralString", + "ParamSpec", + "ParamSpecArgs", + "ParamSpecKwargs", + "Self", + "Type", + "TypeVar", + "TypeVarTuple", + "Unpack", + "Awaitable", + "AsyncIterator", + "AsyncIterable", + "Coroutine", + "AsyncGenerator", + "AsyncContextManager", + "CapsuleType", + "ChainMap", + "ContextManager", + "Counter", + "Deque", + "DefaultDict", + "NamedTuple", + "OrderedDict", + "TypedDict", + "SupportsIndex", + "SupportsAbs", + "SupportsRound", + "SupportsBytes", + "SupportsComplex", + "SupportsFloat", + "SupportsInt", + "Annotated", + "assert_never", + "assert_type", + "dataclass_transform", + "deprecated", + "final", + "IntVar", + "is_typeddict", + "Literal", + "NewType", + "overload", + "override", + "Protocol", + "reveal_type", + "runtime", + "runtime_checkable", + "Text", + "TypeAlias", + "TypeAliasType", + "TypeGuard", + "TYPE_CHECKING", + "Never", + "NoReturn", + "Required", + "NotRequired", + "clear_overloads", + "get_args", + "get_origin", + "get_original_bases", + "get_overloads", + "get_type_hints", + "AbstractSet", + "AnyStr", + "BinaryIO", + "Callable", + "Collection", + "Container", + "Dict", + "Doc", + "ForwardRef", + "FrozenSet", + "Generator", + "Generic", + "Hashable", + "IO", + "ItemsView", + "Iterable", + "Iterator", + "KeysView", + "List", + "Mapping", + "MappingView", + "Match", + "MutableMapping", + "MutableSequence", + "MutableSet", + "NoDefault", + "Optional", + "Pattern", + "Reversible", + "Sequence", + "Set", + "Sized", + "TextIO", + "Tuple", + "Union", + "ValuesView", + "cast", + "get_protocol_members", + "is_protocol", + "no_type_check", + "no_type_check_decorator", + "ReadOnly", + "TypeIs", +] + +_T = typing.TypeVar("_T") +_F = typing.TypeVar("_F", bound=Callable[..., Any]) +_TC = typing.TypeVar("_TC", bound=type[object]) + +class _Final: ... # This should be imported from typing but that breaks pytype + +# unfortunately we have to duplicate this class definition from typing.pyi or we break pytype +class _SpecialForm(_Final): + def __getitem__(self, parameters: Any) -> object: ... + if sys.version_info >= (3, 10): + def __or__(self, other: Any) -> _SpecialForm: ... + def __ror__(self, other: Any) -> _SpecialForm: ... + +# Do not import (and re-export) Protocol or runtime_checkable from +# typing module because type checkers need to be able to distinguish +# typing.Protocol and typing_extensions.Protocol so they can properly +# warn users about potential runtime exceptions when using typing.Protocol +# on older versions of Python. +Protocol: _SpecialForm + +def runtime_checkable(cls: _TC) -> _TC: ... + +# This alias for above is kept here for backwards compatibility. +runtime = runtime_checkable +Final: _SpecialForm + +def final(f: _F) -> _F: ... + +Literal: _SpecialForm + +def IntVar(name: str) -> Any: ... # returns a new TypeVar + +# Internal mypy fallback type for all typed dicts (does not exist at runtime) +# N.B. Keep this mostly in sync with typing._TypedDict/mypy_extensions._TypedDict +@type_check_only +class _TypedDict(Mapping[str, object], metaclass=abc.ABCMeta): + __required_keys__: ClassVar[frozenset[str]] + __optional_keys__: ClassVar[frozenset[str]] + __total__: ClassVar[bool] + __orig_bases__: ClassVar[tuple[Any, ...]] + # PEP 705 + __readonly_keys__: ClassVar[frozenset[str]] + __mutable_keys__: ClassVar[frozenset[str]] + # PEP 728 + __closed__: ClassVar[bool] + __extra_items__: ClassVar[Any] + def copy(self) -> Self: ... + # Using Never so that only calls using mypy plugin hook that specialize the signature + # can go through. + def setdefault(self, k: Never, default: object) -> object: ... + # Mypy plugin hook for 'pop' expects that 'default' has a type variable type. + def pop(self, k: Never, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] + def update(self: _T, m: _T, /) -> None: ... + def items(self) -> dict_items[str, object]: ... + def keys(self) -> dict_keys[str, object]: ... + def values(self) -> dict_values[str, object]: ... + def __delitem__(self, k: Never) -> None: ... + if sys.version_info >= (3, 9): + @overload + def __or__(self, value: Self, /) -> Self: ... + @overload + def __or__(self, value: dict[str, Any], /) -> dict[str, object]: ... + @overload + def __ror__(self, value: Self, /) -> Self: ... + @overload + def __ror__(self, value: dict[str, Any], /) -> dict[str, object]: ... + # supposedly incompatible definitions of `__ior__` and `__or__`: + def __ior__(self, value: Self, /) -> Self: ... # type: ignore[misc] + +OrderedDict = _Alias() + +def get_type_hints( + obj: Callable[..., Any], + globalns: dict[str, Any] | None = None, + localns: Mapping[str, Any] | None = None, + include_extras: bool = False, +) -> dict[str, Any]: ... +def get_args(tp: Any) -> tuple[Any, ...]: ... + +if sys.version_info >= (3, 10): + @overload + def get_origin(tp: UnionType) -> type[UnionType]: ... + +if sys.version_info >= (3, 9): + @overload + def get_origin(tp: GenericAlias) -> type: ... + +@overload +def get_origin(tp: ParamSpecArgs | ParamSpecKwargs) -> ParamSpec: ... +@overload +def get_origin(tp: Any) -> Any | None: ... + +Annotated: _SpecialForm +_AnnotatedAlias: Any # undocumented + +@runtime_checkable +class SupportsIndex(Protocol, metaclass=abc.ABCMeta): + @abc.abstractmethod + def __index__(self) -> int: ... + +# New and changed things in 3.10 +if sys.version_info >= (3, 10): + from typing import ( + Concatenate as Concatenate, + ParamSpecArgs as ParamSpecArgs, + ParamSpecKwargs as ParamSpecKwargs, + TypeAlias as TypeAlias, + TypeGuard as TypeGuard, + is_typeddict as is_typeddict, + ) +else: + @final + class ParamSpecArgs: + @property + def __origin__(self) -> ParamSpec: ... + def __init__(self, origin: ParamSpec) -> None: ... + + @final + class ParamSpecKwargs: + @property + def __origin__(self) -> ParamSpec: ... + def __init__(self, origin: ParamSpec) -> None: ... + + Concatenate: _SpecialForm + TypeAlias: _SpecialForm + TypeGuard: _SpecialForm + def is_typeddict(tp: object) -> bool: ... + +# New and changed things in 3.11 +if sys.version_info >= (3, 11): + from typing import ( + LiteralString as LiteralString, + NamedTuple as NamedTuple, + Never as Never, + NewType as NewType, + NotRequired as NotRequired, + Required as Required, + Self as Self, + Unpack as Unpack, + assert_never as assert_never, + assert_type as assert_type, + clear_overloads as clear_overloads, + dataclass_transform as dataclass_transform, + get_overloads as get_overloads, + reveal_type as reveal_type, + ) +else: + Self: _SpecialForm + Never: _SpecialForm + def reveal_type(obj: _T, /) -> _T: ... + def assert_never(arg: Never, /) -> Never: ... + def assert_type(val: _T, typ: Any, /) -> _T: ... + def clear_overloads() -> None: ... + def get_overloads(func: Callable[..., object]) -> Sequence[Callable[..., object]]: ... + + Required: _SpecialForm + NotRequired: _SpecialForm + LiteralString: _SpecialForm + Unpack: _SpecialForm + + def dataclass_transform( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + frozen_default: bool = False, + field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), + **kwargs: object, + ) -> IdentityFunction: ... + + class NamedTuple(tuple[Any, ...]): + if sys.version_info < (3, 9): + _field_types: ClassVar[dict[str, type]] + _field_defaults: ClassVar[dict[str, Any]] + _fields: ClassVar[tuple[str, ...]] + __orig_bases__: ClassVar[tuple[Any, ...]] + @overload + def __init__(self, typename: str, fields: Iterable[tuple[str, Any]] = ...) -> None: ... + @overload + def __init__(self, typename: str, fields: None = None, **kwargs: Any) -> None: ... + @classmethod + def _make(cls, iterable: Iterable[Any]) -> Self: ... + def _asdict(self) -> dict[str, Any]: ... + def _replace(self, **kwargs: Any) -> Self: ... + + class NewType: + def __init__(self, name: str, tp: Any) -> None: ... + def __call__(self, obj: _T, /) -> _T: ... + __supertype__: type | NewType + if sys.version_info >= (3, 10): + def __or__(self, other: Any) -> _SpecialForm: ... + def __ror__(self, other: Any) -> _SpecialForm: ... + +if sys.version_info >= (3, 12): + from collections.abc import Buffer as Buffer + from types import get_original_bases as get_original_bases + from typing import TypeAliasType as TypeAliasType, override as override +else: + def override(arg: _F, /) -> _F: ... + def get_original_bases(cls: type, /) -> tuple[Any, ...]: ... + @final + class TypeAliasType: + def __init__( + self, name: str, value: Any, *, type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] = () + ) -> None: ... + @property + def __value__(self) -> Any: ... + @property + def __type_params__(self) -> tuple[TypeVar | ParamSpec | TypeVarTuple, ...]: ... + @property + def __parameters__(self) -> tuple[Any, ...]: ... + @property + def __name__(self) -> str: ... + # It's writable on types, but not on instances of TypeAliasType. + @property + def __module__(self) -> str | None: ... # type: ignore[override] + # Returns typing._GenericAlias, which isn't stubbed. + def __getitem__(self, parameters: Any) -> Any: ... + if sys.version_info >= (3, 10): + def __or__(self, right: Any) -> _SpecialForm: ... + def __ror__(self, left: Any) -> _SpecialForm: ... + + # mypy and pyright object to this being both ABC and Protocol. + # At runtime it inherits from ABC and is not a Protocol, but it is on the + # allowlist for use as a Protocol. + @runtime_checkable + class Buffer(Protocol, abc.ABC): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + # Not actually a Protocol at runtime; see + # https://github.com/python/typeshed/issues/10224 for why we're defining it this way + def __buffer__(self, flags: int, /) -> memoryview: ... + +if sys.version_info >= (3, 13): + from types import CapsuleType as CapsuleType + from typing import ( + NoDefault as NoDefault, + ParamSpec as ParamSpec, + ReadOnly as ReadOnly, + TypeIs as TypeIs, + TypeVar as TypeVar, + TypeVarTuple as TypeVarTuple, + get_protocol_members as get_protocol_members, + is_protocol as is_protocol, + ) + from warnings import deprecated as deprecated +else: + def is_protocol(tp: type, /) -> bool: ... + def get_protocol_members(tp: type, /) -> frozenset[str]: ... + @final + class _NoDefaultType: ... + + NoDefault: _NoDefaultType + @final + class CapsuleType: ... + + class deprecated: + message: LiteralString + category: type[Warning] | None + stacklevel: int + def __init__(self, message: LiteralString, /, *, category: type[Warning] | None = ..., stacklevel: int = 1) -> None: ... + def __call__(self, arg: _T, /) -> _T: ... + + @final + class TypeVar: + @property + def __name__(self) -> str: ... + @property + def __bound__(self) -> Any | None: ... + @property + def __constraints__(self) -> tuple[Any, ...]: ... + @property + def __covariant__(self) -> bool: ... + @property + def __contravariant__(self) -> bool: ... + @property + def __infer_variance__(self) -> bool: ... + @property + def __default__(self) -> Any: ... + def __init__( + self, + name: str, + *constraints: Any, + bound: Any | None = None, + covariant: bool = False, + contravariant: bool = False, + default: Any = ..., + infer_variance: bool = False, + ) -> None: ... + def has_default(self) -> bool: ... + def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + if sys.version_info >= (3, 10): + def __or__(self, right: Any) -> _SpecialForm: ... + def __ror__(self, left: Any) -> _SpecialForm: ... + if sys.version_info >= (3, 11): + def __typing_subst__(self, arg: Any) -> Any: ... + + @final + class ParamSpec: + @property + def __name__(self) -> str: ... + @property + def __bound__(self) -> Any | None: ... + @property + def __covariant__(self) -> bool: ... + @property + def __contravariant__(self) -> bool: ... + @property + def __infer_variance__(self) -> bool: ... + @property + def __default__(self) -> Any: ... + def __init__( + self, + name: str, + *, + bound: None | type[Any] | str = None, + contravariant: bool = False, + covariant: bool = False, + default: Any = ..., + ) -> None: ... + @property + def args(self) -> ParamSpecArgs: ... + @property + def kwargs(self) -> ParamSpecKwargs: ... + def has_default(self) -> bool: ... + def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + if sys.version_info >= (3, 10): + def __or__(self, right: Any) -> _SpecialForm: ... + def __ror__(self, left: Any) -> _SpecialForm: ... + + @final + class TypeVarTuple: + @property + def __name__(self) -> str: ... + @property + def __default__(self) -> Any: ... + def __init__(self, name: str, *, default: Any = ...) -> None: ... + def __iter__(self) -> Any: ... # Unpack[Self] + def has_default(self) -> bool: ... + def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + + ReadOnly: _SpecialForm + TypeIs: _SpecialForm + +class Doc: + documentation: str + def __init__(self, documentation: str, /) -> None: ... + def __hash__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... diff --git a/mypy/typeshed/stdlib/unicodedata.pyi b/mypy/typeshed/stdlib/unicodedata.pyi new file mode 100644 index 000000000000..77d69edf06af --- /dev/null +++ b/mypy/typeshed/stdlib/unicodedata.pyi @@ -0,0 +1,73 @@ +import sys +from _typeshed import ReadOnlyBuffer +from typing import Any, Literal, TypeVar, final, overload +from typing_extensions import TypeAlias + +ucd_3_2_0: UCD +unidata_version: str + +if sys.version_info < (3, 10): + ucnhash_CAPI: Any + +_T = TypeVar("_T") + +_NormalizationForm: TypeAlias = Literal["NFC", "NFD", "NFKC", "NFKD"] + +def bidirectional(chr: str, /) -> str: ... +def category(chr: str, /) -> str: ... +def combining(chr: str, /) -> int: ... +@overload +def decimal(chr: str, /) -> int: ... +@overload +def decimal(chr: str, default: _T, /) -> int | _T: ... +def decomposition(chr: str, /) -> str: ... +@overload +def digit(chr: str, /) -> int: ... +@overload +def digit(chr: str, default: _T, /) -> int | _T: ... + +_EastAsianWidth: TypeAlias = Literal["F", "H", "W", "Na", "A", "N"] + +def east_asian_width(chr: str, /) -> _EastAsianWidth: ... +def is_normalized(form: _NormalizationForm, unistr: str, /) -> bool: ... +def lookup(name: str | ReadOnlyBuffer, /) -> str: ... +def mirrored(chr: str, /) -> int: ... +@overload +def name(chr: str, /) -> str: ... +@overload +def name(chr: str, default: _T, /) -> str | _T: ... +def normalize(form: _NormalizationForm, unistr: str, /) -> str: ... +@overload +def numeric(chr: str, /) -> float: ... +@overload +def numeric(chr: str, default: _T, /) -> float | _T: ... +@final +class UCD: + # The methods below are constructed from the same array in C + # (unicodedata_functions) and hence identical to the functions above. + unidata_version: str + def bidirectional(self, chr: str, /) -> str: ... + def category(self, chr: str, /) -> str: ... + def combining(self, chr: str, /) -> int: ... + @overload + def decimal(self, chr: str, /) -> int: ... + @overload + def decimal(self, chr: str, default: _T, /) -> int | _T: ... + def decomposition(self, chr: str, /) -> str: ... + @overload + def digit(self, chr: str, /) -> int: ... + @overload + def digit(self, chr: str, default: _T, /) -> int | _T: ... + def east_asian_width(self, chr: str, /) -> _EastAsianWidth: ... + def is_normalized(self, form: _NormalizationForm, unistr: str, /) -> bool: ... + def lookup(self, name: str | ReadOnlyBuffer, /) -> str: ... + def mirrored(self, chr: str, /) -> int: ... + @overload + def name(self, chr: str, /) -> str: ... + @overload + def name(self, chr: str, default: _T, /) -> str | _T: ... + def normalize(self, form: _NormalizationForm, unistr: str, /) -> str: ... + @overload + def numeric(self, chr: str, /) -> float: ... + @overload + def numeric(self, chr: str, default: _T, /) -> float | _T: ... diff --git a/mypy/typeshed/stdlib/unittest/__init__.pyi b/mypy/typeshed/stdlib/unittest/__init__.pyi new file mode 100644 index 000000000000..546ea77bb4ca --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/__init__.pyi @@ -0,0 +1,63 @@ +import sys +from unittest.async_case import * + +from .case import ( + FunctionTestCase as FunctionTestCase, + SkipTest as SkipTest, + TestCase as TestCase, + addModuleCleanup as addModuleCleanup, + expectedFailure as expectedFailure, + skip as skip, + skipIf as skipIf, + skipUnless as skipUnless, +) +from .loader import TestLoader as TestLoader, defaultTestLoader as defaultTestLoader +from .main import TestProgram as TestProgram, main as main +from .result import TestResult as TestResult +from .runner import TextTestResult as TextTestResult, TextTestRunner as TextTestRunner +from .signals import ( + installHandler as installHandler, + registerResult as registerResult, + removeHandler as removeHandler, + removeResult as removeResult, +) +from .suite import BaseTestSuite as BaseTestSuite, TestSuite as TestSuite + +if sys.version_info >= (3, 11): + from .case import doModuleCleanups as doModuleCleanups, enterModuleContext as enterModuleContext + +__all__ = [ + "IsolatedAsyncioTestCase", + "TestResult", + "TestCase", + "TestSuite", + "TextTestRunner", + "TestLoader", + "FunctionTestCase", + "main", + "defaultTestLoader", + "SkipTest", + "skip", + "skipIf", + "skipUnless", + "expectedFailure", + "TextTestResult", + "installHandler", + "registerResult", + "removeResult", + "removeHandler", + "addModuleCleanup", +] + +if sys.version_info < (3, 13): + from .loader import findTestCases as findTestCases, getTestCaseNames as getTestCaseNames, makeSuite as makeSuite + + __all__ += ["getTestCaseNames", "makeSuite", "findTestCases"] + +if sys.version_info >= (3, 11): + __all__ += ["enterModuleContext", "doModuleCleanups"] + +if sys.version_info < (3, 12): + def load_tests(loader: TestLoader, tests: TestSuite, pattern: str | None) -> TestSuite: ... + +def __dir__() -> set[str]: ... diff --git a/mypy/typeshed/stdlib/unittest/_log.pyi b/mypy/typeshed/stdlib/unittest/_log.pyi new file mode 100644 index 000000000000..011a970d8bbc --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/_log.pyi @@ -0,0 +1,27 @@ +import logging +import sys +from types import TracebackType +from typing import ClassVar, Generic, NamedTuple, TypeVar +from unittest.case import TestCase, _BaseTestCaseContext + +_L = TypeVar("_L", None, _LoggingWatcher) + +class _LoggingWatcher(NamedTuple): + records: list[logging.LogRecord] + output: list[str] + +class _AssertLogsContext(_BaseTestCaseContext, Generic[_L]): + LOGGING_FORMAT: ClassVar[str] + logger_name: str + level: int + msg: None + if sys.version_info >= (3, 10): + def __init__(self, test_case: TestCase, logger_name: str, level: int, no_logs: bool) -> None: ... + no_logs: bool + else: + def __init__(self, test_case: TestCase, logger_name: str, level: int) -> None: ... + + def __enter__(self) -> _L: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, tb: TracebackType | None + ) -> bool | None: ... diff --git a/mypy/typeshed/stdlib/unittest/async_case.pyi b/mypy/typeshed/stdlib/unittest/async_case.pyi new file mode 100644 index 000000000000..565dd91c0fda --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/async_case.pyi @@ -0,0 +1,25 @@ +import sys +from asyncio.events import AbstractEventLoop +from collections.abc import Awaitable, Callable +from typing import TypeVar +from typing_extensions import ParamSpec + +from .case import TestCase + +if sys.version_info >= (3, 11): + from contextlib import AbstractAsyncContextManager + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +class IsolatedAsyncioTestCase(TestCase): + if sys.version_info >= (3, 13): + loop_factory: Callable[[], AbstractEventLoop] | None = None + + async def asyncSetUp(self) -> None: ... + async def asyncTearDown(self) -> None: ... + def addAsyncCleanup(self, func: Callable[_P, Awaitable[object]], /, *args: _P.args, **kwargs: _P.kwargs) -> None: ... + if sys.version_info >= (3, 11): + async def enterAsyncContext(self, cm: AbstractAsyncContextManager[_T]) -> _T: ... + if sys.version_info >= (3, 9): + def __del__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/case.pyi b/mypy/typeshed/stdlib/unittest/case.pyi new file mode 100644 index 000000000000..a92f03f9745f --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/case.pyi @@ -0,0 +1,355 @@ +import logging +import sys +import unittest.result +from _typeshed import SupportsDunderGE, SupportsDunderGT, SupportsDunderLE, SupportsDunderLT, SupportsRSub, SupportsSub +from collections.abc import Callable, Container, Iterable, Mapping, Sequence, Set as AbstractSet +from contextlib import AbstractContextManager +from re import Pattern +from types import TracebackType +from typing import ( + Any, + AnyStr, + ClassVar, + Final, + Generic, + NamedTuple, + NoReturn, + Protocol, + SupportsAbs, + SupportsRound, + TypeVar, + overload, +) +from typing_extensions import ParamSpec, Self, TypeAlias +from warnings import WarningMessage + +if sys.version_info >= (3, 9): + from types import GenericAlias + +if sys.version_info >= (3, 10): + from types import UnionType + +_T = TypeVar("_T") +_S = TypeVar("_S", bound=SupportsSub[Any, Any]) +_E = TypeVar("_E", bound=BaseException) +_FT = TypeVar("_FT", bound=Callable[..., Any]) +_P = ParamSpec("_P") + +DIFF_OMITTED: Final[str] + +class _BaseTestCaseContext: + test_case: TestCase + def __init__(self, test_case: TestCase) -> None: ... + +class _AssertRaisesBaseContext(_BaseTestCaseContext): + expected: type[BaseException] | tuple[type[BaseException], ...] + expected_regex: Pattern[str] | None + obj_name: str | None + msg: str | None + + def __init__( + self, + expected: type[BaseException] | tuple[type[BaseException], ...], + test_case: TestCase, + expected_regex: str | Pattern[str] | None = None, + ) -> None: ... + + # This returns Self if args is the empty list, and None otherwise. + # but it's not possible to construct an overload which expresses that + def handle(self, name: str, args: list[Any], kwargs: dict[str, Any]) -> Any: ... + +if sys.version_info >= (3, 9): + from unittest._log import _AssertLogsContext, _LoggingWatcher +else: + # Unused dummy for _AssertLogsContext. Starting with Python 3.10, + # this is generic over the logging watcher, but in lower versions + # the watcher is hard-coded. + _L = TypeVar("_L") + + class _LoggingWatcher(NamedTuple): + records: list[logging.LogRecord] + output: list[str] + + class _AssertLogsContext(_BaseTestCaseContext, Generic[_L]): + LOGGING_FORMAT: ClassVar[str] + logger_name: str + level: int + msg: None + def __init__(self, test_case: TestCase, logger_name: str, level: int) -> None: ... + def __enter__(self) -> _LoggingWatcher: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, tb: TracebackType | None + ) -> bool | None: ... + +def addModuleCleanup(function: Callable[_P, object], /, *args: _P.args, **kwargs: _P.kwargs) -> None: ... +def doModuleCleanups() -> None: ... + +if sys.version_info >= (3, 11): + def enterModuleContext(cm: AbstractContextManager[_T]) -> _T: ... + +def expectedFailure(test_item: _FT) -> _FT: ... +def skip(reason: str) -> Callable[[_FT], _FT]: ... +def skipIf(condition: object, reason: str) -> Callable[[_FT], _FT]: ... +def skipUnless(condition: object, reason: str) -> Callable[[_FT], _FT]: ... + +class SkipTest(Exception): + def __init__(self, reason: str) -> None: ... + +class _SupportsAbsAndDunderGE(SupportsDunderGE[Any], SupportsAbs[Any], Protocol): ... + +# Keep this alias in sync with builtins._ClassInfo +# We can't import it from builtins or pytype crashes, +# due to the fact that pytype uses a custom builtins stub rather than typeshed's builtins stub +if sys.version_info >= (3, 10): + _ClassInfo: TypeAlias = type | UnionType | tuple[_ClassInfo, ...] +else: + _ClassInfo: TypeAlias = type | tuple[_ClassInfo, ...] + +class TestCase: + failureException: type[BaseException] + longMessage: bool + maxDiff: int | None + # undocumented + _testMethodName: str + # undocumented + _testMethodDoc: str + def __init__(self, methodName: str = "runTest") -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def setUp(self) -> None: ... + def tearDown(self) -> None: ... + @classmethod + def setUpClass(cls) -> None: ... + @classmethod + def tearDownClass(cls) -> None: ... + def run(self, result: unittest.result.TestResult | None = None) -> unittest.result.TestResult | None: ... + def __call__(self, result: unittest.result.TestResult | None = ...) -> unittest.result.TestResult | None: ... + def skipTest(self, reason: Any) -> NoReturn: ... + def subTest(self, msg: Any = ..., **params: Any) -> AbstractContextManager[None]: ... + def debug(self) -> None: ... + if sys.version_info < (3, 11): + def _addSkip(self, result: unittest.result.TestResult, test_case: TestCase, reason: str) -> None: ... + + def assertEqual(self, first: Any, second: Any, msg: Any = None) -> None: ... + def assertNotEqual(self, first: Any, second: Any, msg: Any = None) -> None: ... + def assertTrue(self, expr: Any, msg: Any = None) -> None: ... + def assertFalse(self, expr: Any, msg: Any = None) -> None: ... + def assertIs(self, expr1: object, expr2: object, msg: Any = None) -> None: ... + def assertIsNot(self, expr1: object, expr2: object, msg: Any = None) -> None: ... + def assertIsNone(self, obj: object, msg: Any = None) -> None: ... + def assertIsNotNone(self, obj: object, msg: Any = None) -> None: ... + def assertIn(self, member: Any, container: Iterable[Any] | Container[Any], msg: Any = None) -> None: ... + def assertNotIn(self, member: Any, container: Iterable[Any] | Container[Any], msg: Any = None) -> None: ... + def assertIsInstance(self, obj: object, cls: _ClassInfo, msg: Any = None) -> None: ... + def assertNotIsInstance(self, obj: object, cls: _ClassInfo, msg: Any = None) -> None: ... + @overload + def assertGreater(self, a: SupportsDunderGT[_T], b: _T, msg: Any = None) -> None: ... + @overload + def assertGreater(self, a: _T, b: SupportsDunderLT[_T], msg: Any = None) -> None: ... + @overload + def assertGreaterEqual(self, a: SupportsDunderGE[_T], b: _T, msg: Any = None) -> None: ... + @overload + def assertGreaterEqual(self, a: _T, b: SupportsDunderLE[_T], msg: Any = None) -> None: ... + @overload + def assertLess(self, a: SupportsDunderLT[_T], b: _T, msg: Any = None) -> None: ... + @overload + def assertLess(self, a: _T, b: SupportsDunderGT[_T], msg: Any = None) -> None: ... + @overload + def assertLessEqual(self, a: SupportsDunderLE[_T], b: _T, msg: Any = None) -> None: ... + @overload + def assertLessEqual(self, a: _T, b: SupportsDunderGE[_T], msg: Any = None) -> None: ... + # `assertRaises`, `assertRaisesRegex`, and `assertRaisesRegexp` + # are not using `ParamSpec` intentionally, + # because they might be used with explicitly wrong arg types to raise some error in tests. + @overload + def assertRaises( + self, + expected_exception: type[BaseException] | tuple[type[BaseException], ...], + callable: Callable[..., object], + *args: Any, + **kwargs: Any, + ) -> None: ... + @overload + def assertRaises( + self, expected_exception: type[_E] | tuple[type[_E], ...], *, msg: Any = ... + ) -> _AssertRaisesContext[_E]: ... + @overload + def assertRaisesRegex( + self, + expected_exception: type[BaseException] | tuple[type[BaseException], ...], + expected_regex: str | Pattern[str], + callable: Callable[..., object], + *args: Any, + **kwargs: Any, + ) -> None: ... + @overload + def assertRaisesRegex( + self, expected_exception: type[_E] | tuple[type[_E], ...], expected_regex: str | Pattern[str], *, msg: Any = ... + ) -> _AssertRaisesContext[_E]: ... + @overload + def assertWarns( + self, + expected_warning: type[Warning] | tuple[type[Warning], ...], + callable: Callable[_P, object], + *args: _P.args, + **kwargs: _P.kwargs, + ) -> None: ... + @overload + def assertWarns( + self, expected_warning: type[Warning] | tuple[type[Warning], ...], *, msg: Any = ... + ) -> _AssertWarnsContext: ... + @overload + def assertWarnsRegex( + self, + expected_warning: type[Warning] | tuple[type[Warning], ...], + expected_regex: str | Pattern[str], + callable: Callable[_P, object], + *args: _P.args, + **kwargs: _P.kwargs, + ) -> None: ... + @overload + def assertWarnsRegex( + self, expected_warning: type[Warning] | tuple[type[Warning], ...], expected_regex: str | Pattern[str], *, msg: Any = ... + ) -> _AssertWarnsContext: ... + def assertLogs( + self, logger: str | logging.Logger | None = None, level: int | str | None = None + ) -> _AssertLogsContext[_LoggingWatcher]: ... + if sys.version_info >= (3, 10): + def assertNoLogs( + self, logger: str | logging.Logger | None = None, level: int | str | None = None + ) -> _AssertLogsContext[None]: ... + + @overload + def assertAlmostEqual(self, first: _S, second: _S, places: None, msg: Any, delta: _SupportsAbsAndDunderGE) -> None: ... + @overload + def assertAlmostEqual( + self, first: _S, second: _S, places: None = None, msg: Any = None, *, delta: _SupportsAbsAndDunderGE + ) -> None: ... + @overload + def assertAlmostEqual( + self, + first: SupportsSub[_T, SupportsAbs[SupportsRound[object]]], + second: _T, + places: int | None = None, + msg: Any = None, + delta: None = None, + ) -> None: ... + @overload + def assertAlmostEqual( + self, + first: _T, + second: SupportsRSub[_T, SupportsAbs[SupportsRound[object]]], + places: int | None = None, + msg: Any = None, + delta: None = None, + ) -> None: ... + @overload + def assertNotAlmostEqual(self, first: _S, second: _S, places: None, msg: Any, delta: _SupportsAbsAndDunderGE) -> None: ... + @overload + def assertNotAlmostEqual( + self, first: _S, second: _S, places: None = None, msg: Any = None, *, delta: _SupportsAbsAndDunderGE + ) -> None: ... + @overload + def assertNotAlmostEqual( + self, + first: SupportsSub[_T, SupportsAbs[SupportsRound[object]]], + second: _T, + places: int | None = None, + msg: Any = None, + delta: None = None, + ) -> None: ... + @overload + def assertNotAlmostEqual( + self, + first: _T, + second: SupportsRSub[_T, SupportsAbs[SupportsRound[object]]], + places: int | None = None, + msg: Any = None, + delta: None = None, + ) -> None: ... + def assertRegex(self, text: AnyStr, expected_regex: AnyStr | Pattern[AnyStr], msg: Any = None) -> None: ... + def assertNotRegex(self, text: AnyStr, unexpected_regex: AnyStr | Pattern[AnyStr], msg: Any = None) -> None: ... + def assertCountEqual(self, first: Iterable[Any], second: Iterable[Any], msg: Any = None) -> None: ... + def addTypeEqualityFunc(self, typeobj: type[Any], function: Callable[..., None]) -> None: ... + def assertMultiLineEqual(self, first: str, second: str, msg: Any = None) -> None: ... + def assertSequenceEqual( + self, seq1: Sequence[Any], seq2: Sequence[Any], msg: Any = None, seq_type: type[Sequence[Any]] | None = None + ) -> None: ... + def assertListEqual(self, list1: list[Any], list2: list[Any], msg: Any = None) -> None: ... + def assertTupleEqual(self, tuple1: tuple[Any, ...], tuple2: tuple[Any, ...], msg: Any = None) -> None: ... + def assertSetEqual(self, set1: AbstractSet[object], set2: AbstractSet[object], msg: Any = None) -> None: ... + # assertDictEqual accepts only true dict instances. We can't use that here, since that would make + # assertDictEqual incompatible with TypedDict. + def assertDictEqual(self, d1: Mapping[Any, object], d2: Mapping[Any, object], msg: Any = None) -> None: ... + def fail(self, msg: Any = None) -> NoReturn: ... + def countTestCases(self) -> int: ... + def defaultTestResult(self) -> unittest.result.TestResult: ... + def id(self) -> str: ... + def shortDescription(self) -> str | None: ... + def addCleanup(self, function: Callable[_P, object], /, *args: _P.args, **kwargs: _P.kwargs) -> None: ... + + if sys.version_info >= (3, 11): + def enterContext(self, cm: AbstractContextManager[_T]) -> _T: ... + + def doCleanups(self) -> None: ... + @classmethod + def addClassCleanup(cls, function: Callable[_P, object], /, *args: _P.args, **kwargs: _P.kwargs) -> None: ... + @classmethod + def doClassCleanups(cls) -> None: ... + + if sys.version_info >= (3, 11): + @classmethod + def enterClassContext(cls, cm: AbstractContextManager[_T]) -> _T: ... + + def _formatMessage(self, msg: str | None, standardMsg: str) -> str: ... # undocumented + def _getAssertEqualityFunc(self, first: Any, second: Any) -> Callable[..., None]: ... # undocumented + if sys.version_info < (3, 12): + failUnlessEqual = assertEqual + assertEquals = assertEqual + failIfEqual = assertNotEqual + assertNotEquals = assertNotEqual + failUnless = assertTrue + assert_ = assertTrue + failIf = assertFalse + failUnlessRaises = assertRaises + failUnlessAlmostEqual = assertAlmostEqual + assertAlmostEquals = assertAlmostEqual + failIfAlmostEqual = assertNotAlmostEqual + assertNotAlmostEquals = assertNotAlmostEqual + assertRegexpMatches = assertRegex + assertNotRegexpMatches = assertNotRegex + assertRaisesRegexp = assertRaisesRegex + def assertDictContainsSubset( + self, subset: Mapping[Any, Any], dictionary: Mapping[Any, Any], msg: object = None + ) -> None: ... + +class FunctionTestCase(TestCase): + def __init__( + self, + testFunc: Callable[[], object], + setUp: Callable[[], object] | None = None, + tearDown: Callable[[], object] | None = None, + description: str | None = None, + ) -> None: ... + def runTest(self) -> None: ... + def __hash__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + +class _AssertRaisesContext(_AssertRaisesBaseContext, Generic[_E]): + exception: _E + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, tb: TracebackType | None + ) -> bool: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class _AssertWarnsContext(_AssertRaisesBaseContext): + warning: WarningMessage + filename: str + lineno: int + warnings: list[WarningMessage] + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, tb: TracebackType | None + ) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/loader.pyi b/mypy/typeshed/stdlib/unittest/loader.pyi new file mode 100644 index 000000000000..598e3cd84a5e --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/loader.pyi @@ -0,0 +1,55 @@ +import sys +import unittest.case +import unittest.suite +from collections.abc import Callable, Sequence +from re import Pattern +from types import ModuleType +from typing import Any, Final +from typing_extensions import TypeAlias, deprecated + +_SortComparisonMethod: TypeAlias = Callable[[str, str], int] +_SuiteClass: TypeAlias = Callable[[list[unittest.case.TestCase]], unittest.suite.TestSuite] + +VALID_MODULE_NAME: Final[Pattern[str]] + +class TestLoader: + errors: list[type[BaseException]] + testMethodPrefix: str + sortTestMethodsUsing: _SortComparisonMethod + testNamePatterns: list[str] | None + suiteClass: _SuiteClass + def loadTestsFromTestCase(self, testCaseClass: type[unittest.case.TestCase]) -> unittest.suite.TestSuite: ... + if sys.version_info >= (3, 12): + def loadTestsFromModule(self, module: ModuleType, *, pattern: str | None = None) -> unittest.suite.TestSuite: ... + else: + def loadTestsFromModule(self, module: ModuleType, *args: Any, pattern: str | None = None) -> unittest.suite.TestSuite: ... + + def loadTestsFromName(self, name: str, module: ModuleType | None = None) -> unittest.suite.TestSuite: ... + def loadTestsFromNames(self, names: Sequence[str], module: ModuleType | None = None) -> unittest.suite.TestSuite: ... + def getTestCaseNames(self, testCaseClass: type[unittest.case.TestCase]) -> Sequence[str]: ... + def discover( + self, start_dir: str, pattern: str = "test*.py", top_level_dir: str | None = None + ) -> unittest.suite.TestSuite: ... + def _match_path(self, path: str, full_path: str, pattern: str) -> bool: ... + +defaultTestLoader: TestLoader + +if sys.version_info < (3, 13): + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def getTestCaseNames( + testCaseClass: type[unittest.case.TestCase], + prefix: str, + sortUsing: _SortComparisonMethod = ..., + testNamePatterns: list[str] | None = None, + ) -> Sequence[str]: ... + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def makeSuite( + testCaseClass: type[unittest.case.TestCase], + prefix: str = "test", + sortUsing: _SortComparisonMethod = ..., + suiteClass: _SuiteClass = ..., + ) -> unittest.suite.TestSuite: ... + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def findTestCases( + module: ModuleType, prefix: str = "test", sortUsing: _SortComparisonMethod = ..., suiteClass: _SuiteClass = ... + ) -> unittest.suite.TestSuite: ... diff --git a/mypy/typeshed/stdlib/unittest/main.pyi b/mypy/typeshed/stdlib/unittest/main.pyi new file mode 100644 index 000000000000..22f2ec10634d --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/main.pyi @@ -0,0 +1,73 @@ +import sys +import unittest.case +import unittest.loader +import unittest.result +import unittest.suite +from collections.abc import Iterable +from types import ModuleType +from typing import Any, Final, Protocol +from typing_extensions import deprecated + +MAIN_EXAMPLES: Final[str] +MODULE_EXAMPLES: Final[str] + +class _TestRunner(Protocol): + def run(self, test: unittest.suite.TestSuite | unittest.case.TestCase, /) -> unittest.result.TestResult: ... + +# not really documented +class TestProgram: + result: unittest.result.TestResult + module: None | str | ModuleType + verbosity: int + failfast: bool | None + catchbreak: bool | None + buffer: bool | None + progName: str | None + warnings: str | None + testNamePatterns: list[str] | None + if sys.version_info >= (3, 12): + durations: unittest.result._DurationsType | None + def __init__( + self, + module: None | str | ModuleType = "__main__", + defaultTest: str | Iterable[str] | None = None, + argv: list[str] | None = None, + testRunner: type[_TestRunner] | _TestRunner | None = None, + testLoader: unittest.loader.TestLoader = ..., + exit: bool = True, + verbosity: int = 1, + failfast: bool | None = None, + catchbreak: bool | None = None, + buffer: bool | None = None, + warnings: str | None = None, + *, + tb_locals: bool = False, + durations: unittest.result._DurationsType | None = None, + ) -> None: ... + else: + def __init__( + self, + module: None | str | ModuleType = "__main__", + defaultTest: str | Iterable[str] | None = None, + argv: list[str] | None = None, + testRunner: type[_TestRunner] | _TestRunner | None = None, + testLoader: unittest.loader.TestLoader = ..., + exit: bool = True, + verbosity: int = 1, + failfast: bool | None = None, + catchbreak: bool | None = None, + buffer: bool | None = None, + warnings: str | None = None, + *, + tb_locals: bool = False, + ) -> None: ... + + if sys.version_info < (3, 13): + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def usageExit(self, msg: Any = None) -> None: ... + + def parseArgs(self, argv: list[str]) -> None: ... + def createTests(self, from_discovery: bool = False, Loader: unittest.loader.TestLoader | None = None) -> None: ... + def runTests(self) -> None: ... # undocumented + +main = TestProgram diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi new file mode 100644 index 000000000000..193a4123c395 --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -0,0 +1,466 @@ +import sys +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Mapping, Sequence +from contextlib import _GeneratorContextManager +from types import TracebackType +from typing import Any, Final, Generic, Literal, TypeVar, overload +from typing_extensions import ParamSpec, Self, TypeAlias + +_T = TypeVar("_T") +_TT = TypeVar("_TT", bound=type[Any]) +_R = TypeVar("_R") +_F = TypeVar("_F", bound=Callable[..., Any]) +_AF = TypeVar("_AF", bound=Callable[..., Coroutine[Any, Any, Any]]) +_P = ParamSpec("_P") + +if sys.version_info >= (3, 13): + # ThreadingMock added in 3.13 + __all__ = ( + "Mock", + "MagicMock", + "patch", + "sentinel", + "DEFAULT", + "ANY", + "call", + "create_autospec", + "ThreadingMock", + "AsyncMock", + "FILTER_DIR", + "NonCallableMock", + "NonCallableMagicMock", + "mock_open", + "PropertyMock", + "seal", + ) +else: + __all__ = ( + "Mock", + "MagicMock", + "patch", + "sentinel", + "DEFAULT", + "ANY", + "call", + "create_autospec", + "AsyncMock", + "FILTER_DIR", + "NonCallableMock", + "NonCallableMagicMock", + "mock_open", + "PropertyMock", + "seal", + ) + +if sys.version_info < (3, 9): + __version__: Final[str] + +FILTER_DIR: Any + +class _SentinelObject: + name: Any + def __init__(self, name: Any) -> None: ... + +class _Sentinel: + def __getattr__(self, name: str) -> Any: ... + +sentinel: Any +DEFAULT: Any + +_ArgsKwargs: TypeAlias = tuple[tuple[Any, ...], Mapping[str, Any]] +_NameArgsKwargs: TypeAlias = tuple[str, tuple[Any, ...], Mapping[str, Any]] +_CallValue: TypeAlias = str | tuple[Any, ...] | Mapping[str, Any] | _ArgsKwargs | _NameArgsKwargs + +class _Call(tuple[Any, ...]): + def __new__( + cls, value: _CallValue = (), name: str | None = "", parent: Any | None = None, two: bool = False, from_kall: bool = True + ) -> Self: ... + name: Any + parent: Any + from_kall: Any + def __init__( + self, + value: _CallValue = (), + name: str | None = None, + parent: Any | None = None, + two: bool = False, + from_kall: bool = True, + ) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __call__(self, *args: Any, **kwargs: Any) -> _Call: ... + def __getattr__(self, attr: str) -> Any: ... + def __getattribute__(self, attr: str) -> Any: ... + @property + def args(self) -> tuple[Any, ...]: ... + @property + def kwargs(self) -> Mapping[str, Any]: ... + def call_list(self) -> Any: ... + +call: _Call + +class _CallList(list[_Call]): + def __contains__(self, value: Any) -> bool: ... + +class Base: + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + +# We subclass with "Any" because mocks are explicitly designed to stand in for other types, +# something that can't be expressed with our static type system. +class NonCallableMock(Base, Any): + if sys.version_info >= (3, 12): + def __new__( + cls, + spec: list[str] | object | type[object] | None = None, + wraps: Any | None = None, + name: str | None = None, + spec_set: list[str] | object | type[object] | None = None, + parent: NonCallableMock | None = None, + _spec_state: Any | None = None, + _new_name: str = "", + _new_parent: NonCallableMock | None = None, + _spec_as_instance: bool = False, + _eat_self: bool | None = None, + unsafe: bool = False, + **kwargs: Any, + ) -> Self: ... + else: + def __new__(cls, /, *args: Any, **kw: Any) -> Self: ... + + def __init__( + self, + spec: list[str] | object | type[object] | None = None, + wraps: Any | None = None, + name: str | None = None, + spec_set: list[str] | object | type[object] | None = None, + parent: NonCallableMock | None = None, + _spec_state: Any | None = None, + _new_name: str = "", + _new_parent: NonCallableMock | None = None, + _spec_as_instance: bool = False, + _eat_self: bool | None = None, + unsafe: bool = False, + **kwargs: Any, + ) -> None: ... + def __getattr__(self, name: str) -> Any: ... + def __delattr__(self, name: str) -> None: ... + def __setattr__(self, name: str, value: Any) -> None: ... + def __dir__(self) -> list[str]: ... + def assert_called_with(self, *args: Any, **kwargs: Any) -> None: ... + def assert_not_called(self) -> None: ... + def assert_called_once_with(self, *args: Any, **kwargs: Any) -> None: ... + def _format_mock_failure_message(self, args: Any, kwargs: Any, action: str = "call") -> str: ... + def assert_called(self) -> None: ... + def assert_called_once(self) -> None: ... + def reset_mock(self, visited: Any = None, *, return_value: bool = False, side_effect: bool = False) -> None: ... + def _extract_mock_name(self) -> str: ... + def _get_call_signature_from_name(self, name: str) -> Any: ... + def assert_any_call(self, *args: Any, **kwargs: Any) -> None: ... + def assert_has_calls(self, calls: Sequence[_Call], any_order: bool = False) -> None: ... + def mock_add_spec(self, spec: Any, spec_set: bool = False) -> None: ... + def _mock_add_spec(self, spec: Any, spec_set: bool, _spec_as_instance: bool = False, _eat_self: bool = False) -> None: ... + def attach_mock(self, mock: NonCallableMock, attribute: str) -> None: ... + def configure_mock(self, **kwargs: Any) -> None: ... + return_value: Any + side_effect: Any + called: bool + call_count: int + call_args: Any + call_args_list: _CallList + mock_calls: _CallList + def _format_mock_call_signature(self, args: Any, kwargs: Any) -> str: ... + def _call_matcher(self, _call: tuple[_Call, ...]) -> _Call: ... + def _get_child_mock(self, **kw: Any) -> NonCallableMock: ... + if sys.version_info >= (3, 13): + def _calls_repr(self) -> str: ... + else: + def _calls_repr(self, prefix: str = "Calls") -> str: ... + +class CallableMixin(Base): + side_effect: Any + def __init__( + self, + spec: Any | None = None, + side_effect: Any | None = None, + return_value: Any = ..., + wraps: Any | None = None, + name: Any | None = None, + spec_set: Any | None = None, + parent: Any | None = None, + _spec_state: Any | None = None, + _new_name: Any = "", + _new_parent: Any | None = None, + **kwargs: Any, + ) -> None: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + +class Mock(CallableMixin, NonCallableMock): ... + +class _patch(Generic[_T]): + attribute_name: Any + getter: Callable[[], Any] + attribute: str + new: _T + new_callable: Any + spec: Any + create: bool + has_local: Any + spec_set: Any + autospec: Any + kwargs: Mapping[str, Any] + additional_patchers: Any + # If new==DEFAULT, self is _patch[Any]. Ideally we'd be able to add an overload for it so that self is _patch[MagicMock], + # but that's impossible with the current type system. + if sys.version_info >= (3, 10): + def __init__( + self: _patch[_T], # pyright: ignore[reportInvalidTypeVarUse] #11780 + getter: Callable[[], Any], + attribute: str, + new: _T, + spec: Any | None, + create: bool, + spec_set: Any | None, + autospec: Any | None, + new_callable: Any | None, + kwargs: Mapping[str, Any], + *, + unsafe: bool = False, + ) -> None: ... + else: + def __init__( + self: _patch[_T], # pyright: ignore[reportInvalidTypeVarUse] #11780 + getter: Callable[[], Any], + attribute: str, + new: _T, + spec: Any | None, + create: bool, + spec_set: Any | None, + autospec: Any | None, + new_callable: Any | None, + kwargs: Mapping[str, Any], + ) -> None: ... + + def copy(self) -> _patch[_T]: ... + @overload + def __call__(self, func: _TT) -> _TT: ... + # If new==DEFAULT, this should add a MagicMock parameter to the function + # arguments. See the _patch_default_new class below for this functionality. + @overload + def __call__(self, func: Callable[_P, _R]) -> Callable[_P, _R]: ... + def decoration_helper( + self, patched: _patch[Any], args: Sequence[Any], keywargs: Any + ) -> _GeneratorContextManager[tuple[Sequence[Any], Any]]: ... + def decorate_class(self, klass: _TT) -> _TT: ... + def decorate_callable(self, func: Callable[..., _R]) -> Callable[..., _R]: ... + def decorate_async_callable(self, func: Callable[..., Awaitable[_R]]) -> Callable[..., Awaitable[_R]]: ... + def get_original(self) -> tuple[Any, bool]: ... + target: Any + temp_original: Any + is_local: bool + def __enter__(self) -> _T: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, / + ) -> None: ... + def start(self) -> _T: ... + def stop(self) -> None: ... + +# This class does not exist at runtime, it's a hack to make this work: +# @patch("foo") +# def bar(..., mock: MagicMock) -> None: ... +class _patch_default_new(_patch[MagicMock | AsyncMock]): + @overload + def __call__(self, func: _TT) -> _TT: ... + # Can't use the following as ParamSpec is only allowed as last parameter: + # def __call__(self, func: Callable[_P, _R]) -> Callable[Concatenate[_P, MagicMock], _R]: ... + @overload + def __call__(self, func: Callable[..., _R]) -> Callable[..., _R]: ... + +class _patch_dict: + in_dict: Any + values: Any + clear: Any + def __init__(self, in_dict: Any, values: Any = (), clear: Any = False, **kwargs: Any) -> None: ... + def __call__(self, f: Any) -> Any: ... + if sys.version_info >= (3, 10): + def decorate_callable(self, f: _F) -> _F: ... + def decorate_async_callable(self, f: _AF) -> _AF: ... + + def decorate_class(self, klass: Any) -> Any: ... + def __enter__(self) -> Any: ... + def __exit__(self, *args: object) -> Any: ... + start: Any + stop: Any + +# This class does not exist at runtime, it's a hack to add methods to the +# patch() function. +class _patcher: + TEST_PREFIX: str + dict: type[_patch_dict] + # This overload also covers the case, where new==DEFAULT. In this case, the return type is _patch[Any]. + # Ideally we'd be able to add an overload for it so that the return type is _patch[MagicMock], + # but that's impossible with the current type system. + @overload + def __call__( + self, + target: str, + new: _T, + spec: Any | None = ..., + create: bool = ..., + spec_set: Any | None = ..., + autospec: Any | None = ..., + new_callable: Any | None = ..., + **kwargs: Any, + ) -> _patch[_T]: ... + @overload + def __call__( + self, + target: str, + *, + spec: Any | None = ..., + create: bool = ..., + spec_set: Any | None = ..., + autospec: Any | None = ..., + new_callable: Any | None = ..., + **kwargs: Any, + ) -> _patch_default_new: ... + @overload + @staticmethod + def object( + target: Any, + attribute: str, + new: _T, + spec: Any | None = ..., + create: bool = ..., + spec_set: Any | None = ..., + autospec: Any | None = ..., + new_callable: Any | None = ..., + **kwargs: Any, + ) -> _patch[_T]: ... + @overload + @staticmethod + def object( + target: Any, + attribute: str, + *, + spec: Any | None = ..., + create: bool = ..., + spec_set: Any | None = ..., + autospec: Any | None = ..., + new_callable: Any | None = ..., + **kwargs: Any, + ) -> _patch[MagicMock | AsyncMock]: ... + @staticmethod + def multiple( + target: Any, + spec: Any | None = ..., + create: bool = ..., + spec_set: Any | None = ..., + autospec: Any | None = ..., + new_callable: Any | None = ..., + **kwargs: Any, + ) -> _patch[Any]: ... + @staticmethod + def stopall() -> None: ... + +patch: _patcher + +class MagicMixin(Base): + def __init__(self, *args: Any, **kw: Any) -> None: ... + +class NonCallableMagicMock(MagicMixin, NonCallableMock): ... +class MagicMock(MagicMixin, Mock): ... + +class AsyncMockMixin(Base): + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + async def _execute_mock_call(self, *args: Any, **kwargs: Any) -> Any: ... + def assert_awaited(self) -> None: ... + def assert_awaited_once(self) -> None: ... + def assert_awaited_with(self, *args: Any, **kwargs: Any) -> None: ... + def assert_awaited_once_with(self, *args: Any, **kwargs: Any) -> None: ... + def assert_any_await(self, *args: Any, **kwargs: Any) -> None: ... + def assert_has_awaits(self, calls: Iterable[_Call], any_order: bool = False) -> None: ... + def assert_not_awaited(self) -> None: ... + def reset_mock(self, *args: Any, **kwargs: Any) -> None: ... + await_count: int + await_args: _Call | None + await_args_list: _CallList + +class AsyncMagicMixin(MagicMixin): + def __init__(self, *args: Any, **kw: Any) -> None: ... + +class AsyncMock(AsyncMockMixin, AsyncMagicMixin, Mock): + # Improving the `reset_mock` signature. + # It is defined on `AsyncMockMixin` with `*args, **kwargs`, which is not ideal. + # But, `NonCallableMock` super-class has the better version. + def reset_mock(self, visited: Any = None, *, return_value: bool = False, side_effect: bool = False) -> None: ... + +class MagicProxy(Base): + name: str + parent: Any + def __init__(self, name: str, parent: Any) -> None: ... + def create_mock(self) -> Any: ... + def __get__(self, obj: Any, _type: Any | None = None) -> Any: ... + +class _ANY: + def __eq__(self, other: object) -> Literal[True]: ... + def __ne__(self, other: object) -> Literal[False]: ... + +ANY: Any + +if sys.version_info >= (3, 10): + def create_autospec( + spec: Any, + spec_set: Any = False, + instance: Any = False, + _parent: Any | None = None, + _name: Any | None = None, + *, + unsafe: bool = False, + **kwargs: Any, + ) -> Any: ... + +else: + def create_autospec( + spec: Any, + spec_set: Any = False, + instance: Any = False, + _parent: Any | None = None, + _name: Any | None = None, + **kwargs: Any, + ) -> Any: ... + +class _SpecState: + spec: Any + ids: Any + spec_set: Any + parent: Any + instance: Any + name: Any + def __init__( + self, + spec: Any, + spec_set: Any = False, + parent: Any | None = None, + name: Any | None = None, + ids: Any | None = None, + instance: Any = False, + ) -> None: ... + +def mock_open(mock: Any | None = None, read_data: Any = "") -> Any: ... + +class PropertyMock(Mock): + def __get__(self, obj: _T, obj_type: type[_T] | None = None) -> Self: ... + def __set__(self, obj: Any, val: Any) -> None: ... + +if sys.version_info >= (3, 13): + class ThreadingMixin(Base): + DEFAULT_TIMEOUT: Final[float | None] = None + + def __init__(self, /, *args: Any, timeout: float | None | _SentinelObject = ..., **kwargs: Any) -> None: ... + # Same as `NonCallableMock.reset_mock.` + def reset_mock(self, visited: Any = None, *, return_value: bool = False, side_effect: bool = False) -> None: ... + def wait_until_called(self, *, timeout: float | None | _SentinelObject = ...) -> None: ... + def wait_until_any_call_with(self, *args: Any, **kwargs: Any) -> None: ... + + class ThreadingMock(ThreadingMixin, MagicMixin, Mock): ... + +def seal(mock: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/result.pyi b/mypy/typeshed/stdlib/unittest/result.pyi new file mode 100644 index 000000000000..0761baaa2830 --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/result.pyi @@ -0,0 +1,47 @@ +import sys +import unittest.case +from _typeshed import OptExcInfo +from collections.abc import Callable +from typing import Any, Final, TextIO, TypeVar +from typing_extensions import TypeAlias + +_F = TypeVar("_F", bound=Callable[..., Any]) +_DurationsType: TypeAlias = list[tuple[str, float]] + +STDOUT_LINE: Final[str] +STDERR_LINE: Final[str] + +# undocumented +def failfast(method: _F) -> _F: ... + +class TestResult: + errors: list[tuple[unittest.case.TestCase, str]] + failures: list[tuple[unittest.case.TestCase, str]] + skipped: list[tuple[unittest.case.TestCase, str]] + expectedFailures: list[tuple[unittest.case.TestCase, str]] + unexpectedSuccesses: list[unittest.case.TestCase] + shouldStop: bool + testsRun: int + buffer: bool + failfast: bool + tb_locals: bool + if sys.version_info >= (3, 12): + collectedDurations: _DurationsType + + def __init__(self, stream: TextIO | None = None, descriptions: bool | None = None, verbosity: int | None = None) -> None: ... + def printErrors(self) -> None: ... + def wasSuccessful(self) -> bool: ... + def stop(self) -> None: ... + def startTest(self, test: unittest.case.TestCase) -> None: ... + def stopTest(self, test: unittest.case.TestCase) -> None: ... + def startTestRun(self) -> None: ... + def stopTestRun(self) -> None: ... + def addError(self, test: unittest.case.TestCase, err: OptExcInfo) -> None: ... + def addFailure(self, test: unittest.case.TestCase, err: OptExcInfo) -> None: ... + def addSuccess(self, test: unittest.case.TestCase) -> None: ... + def addSkip(self, test: unittest.case.TestCase, reason: str) -> None: ... + def addExpectedFailure(self, test: unittest.case.TestCase, err: OptExcInfo) -> None: ... + def addUnexpectedSuccess(self, test: unittest.case.TestCase) -> None: ... + def addSubTest(self, test: unittest.case.TestCase, subtest: unittest.case.TestCase, err: OptExcInfo | None) -> None: ... + if sys.version_info >= (3, 12): + def addDuration(self, test: unittest.case.TestCase, elapsed: float) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/runner.pyi b/mypy/typeshed/stdlib/unittest/runner.pyi new file mode 100644 index 000000000000..393d03dfa0fc --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/runner.pyi @@ -0,0 +1,92 @@ +import sys +import unittest.case +import unittest.result +import unittest.suite +from _typeshed import SupportsFlush, SupportsWrite +from collections.abc import Callable, Iterable +from typing import Any, Generic, Protocol, TypeVar +from typing_extensions import Never, TypeAlias + +_ResultClassType: TypeAlias = Callable[[_TextTestStream, bool, int], TextTestResult] + +class _SupportsWriteAndFlush(SupportsWrite[str], SupportsFlush, Protocol): ... + +# All methods used by unittest.runner.TextTestResult's stream +class _TextTestStream(_SupportsWriteAndFlush, Protocol): + def writeln(self, arg: str | None = None, /) -> str: ... + +# _WritelnDecorator should have all the same attrs as its stream param. +# But that's not feasible to do Generically +# We can expand the attributes if requested +class _WritelnDecorator: + def __init__(self, stream: _TextTestStream) -> None: ... + def writeln(self, arg: str | None = None) -> str: ... + def __getattr__(self, attr: str) -> Any: ... # Any attribute from the stream type passed to __init__ + # These attributes are prevented by __getattr__ + stream: Never + __getstate__: Never + # Methods proxied from the wrapped stream object via __getattr__ + def flush(self) -> object: ... + def write(self, s: str, /) -> object: ... + +_StreamT = TypeVar("_StreamT", bound=_TextTestStream, default=_WritelnDecorator) + +class TextTestResult(unittest.result.TestResult, Generic[_StreamT]): + descriptions: bool # undocumented + dots: bool # undocumented + separator1: str + separator2: str + showAll: bool # undocumented + stream: _StreamT # undocumented + if sys.version_info >= (3, 12): + durations: unittest.result._DurationsType | None + def __init__( + self, stream: _StreamT, descriptions: bool, verbosity: int, *, durations: unittest.result._DurationsType | None = None + ) -> None: ... + else: + def __init__(self, stream: _StreamT, descriptions: bool, verbosity: int) -> None: ... + + def getDescription(self, test: unittest.case.TestCase) -> str: ... + def printErrorList(self, flavour: str, errors: Iterable[tuple[unittest.case.TestCase, str]]) -> None: ... + +class TextTestRunner: + resultclass: _ResultClassType + stream: _WritelnDecorator + descriptions: bool + verbosity: int + failfast: bool + buffer: bool + warnings: str | None + tb_locals: bool + + if sys.version_info >= (3, 12): + durations: unittest.result._DurationsType | None + def __init__( + self, + stream: _SupportsWriteAndFlush | None = None, + descriptions: bool = True, + verbosity: int = 1, + failfast: bool = False, + buffer: bool = False, + resultclass: _ResultClassType | None = None, + warnings: str | None = None, + *, + tb_locals: bool = False, + durations: unittest.result._DurationsType | None = None, + ) -> None: ... + else: + def __init__( + self, + stream: _SupportsWriteAndFlush | None = None, + descriptions: bool = True, + verbosity: int = 1, + failfast: bool = False, + buffer: bool = False, + resultclass: _ResultClassType | None = None, + warnings: str | None = None, + *, + tb_locals: bool = False, + ) -> None: ... + + def _makeResult(self) -> TextTestResult: ... + def run(self, test: unittest.suite.TestSuite | unittest.case.TestCase) -> TextTestResult: ... diff --git a/mypy/typeshed/stdlib/unittest/signals.pyi b/mypy/typeshed/stdlib/unittest/signals.pyi new file mode 100644 index 000000000000..a60133ada9d9 --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/signals.pyi @@ -0,0 +1,15 @@ +import unittest.result +from collections.abc import Callable +from typing import TypeVar, overload +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_T = TypeVar("_T") + +def installHandler() -> None: ... +def registerResult(result: unittest.result.TestResult) -> None: ... +def removeResult(result: unittest.result.TestResult) -> bool: ... +@overload +def removeHandler(method: None = None) -> None: ... +@overload +def removeHandler(method: Callable[_P, _T]) -> Callable[_P, _T]: ... diff --git a/mypy/typeshed/stdlib/unittest/suite.pyi b/mypy/typeshed/stdlib/unittest/suite.pyi new file mode 100644 index 000000000000..ff583d0766a0 --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/suite.pyi @@ -0,0 +1,22 @@ +import unittest.case +import unittest.result +from collections.abc import Iterable, Iterator +from typing_extensions import TypeAlias + +_TestType: TypeAlias = unittest.case.TestCase | TestSuite + +class BaseTestSuite: + _tests: list[unittest.case.TestCase] + _removed_tests: int + def __init__(self, tests: Iterable[_TestType] = ()) -> None: ... + def __call__(self, result: unittest.result.TestResult) -> unittest.result.TestResult: ... + def addTest(self, test: _TestType) -> None: ... + def addTests(self, tests: Iterable[_TestType]) -> None: ... + def run(self, result: unittest.result.TestResult) -> unittest.result.TestResult: ... + def debug(self) -> None: ... + def countTestCases(self) -> int: ... + def __iter__(self) -> Iterator[_TestType]: ... + def __eq__(self, other: object) -> bool: ... + +class TestSuite(BaseTestSuite): + def run(self, result: unittest.result.TestResult, debug: bool = False) -> unittest.result.TestResult: ... diff --git a/mypy/typeshed/stdlib/unittest/util.pyi b/mypy/typeshed/stdlib/unittest/util.pyi new file mode 100644 index 000000000000..945b0cecfed0 --- /dev/null +++ b/mypy/typeshed/stdlib/unittest/util.pyi @@ -0,0 +1,23 @@ +from collections.abc import MutableSequence, Sequence +from typing import Any, Final, TypeVar +from typing_extensions import TypeAlias + +_T = TypeVar("_T") +_Mismatch: TypeAlias = tuple[_T, _T, int] + +_MAX_LENGTH: Final[int] +_PLACEHOLDER_LEN: Final[int] +_MIN_BEGIN_LEN: Final[int] +_MIN_END_LEN: Final[int] +_MIN_COMMON_LEN: Final[int] +_MIN_DIFF_LEN: Final[int] + +def _shorten(s: str, prefixlen: int, suffixlen: int) -> str: ... +def _common_shorten_repr(*args: str) -> tuple[str, ...]: ... +def safe_repr(obj: object, short: bool = False) -> str: ... +def strclass(cls: type) -> str: ... +def sorted_list_difference(expected: Sequence[_T], actual: Sequence[_T]) -> tuple[list[_T], list[_T]]: ... +def unorderable_list_difference(expected: MutableSequence[_T], actual: MutableSequence[_T]) -> tuple[list[_T], list[_T]]: ... +def three_way_cmp(x: Any, y: Any) -> int: ... +def _count_diff_all_purpose(actual: Sequence[_T], expected: Sequence[_T]) -> list[_Mismatch[_T]]: ... +def _count_diff_hashable(actual: Sequence[_T], expected: Sequence[_T]) -> list[_Mismatch[_T]]: ... diff --git a/mypy/typeshed/stdlib/urllib/__init__.pyi b/mypy/typeshed/stdlib/urllib/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypy/typeshed/stdlib/urllib/error.pyi b/mypy/typeshed/stdlib/urllib/error.pyi new file mode 100644 index 000000000000..89cec9bf289c --- /dev/null +++ b/mypy/typeshed/stdlib/urllib/error.pyi @@ -0,0 +1,23 @@ +from email.message import Message +from typing import IO +from urllib.response import addinfourl + +__all__ = ["URLError", "HTTPError", "ContentTooShortError"] + +class URLError(OSError): + reason: str | BaseException + def __init__(self, reason: str | BaseException, filename: str | None = None) -> None: ... + +class HTTPError(URLError, addinfourl): + @property + def headers(self) -> Message: ... + @headers.setter + def headers(self, headers: Message) -> None: ... + @property + def reason(self) -> str: ... # type: ignore[override] + code: int + def __init__(self, url: str, code: int, msg: str, hdrs: Message, fp: IO[bytes] | None) -> None: ... + +class ContentTooShortError(URLError): + content: tuple[str, Message] + def __init__(self, message: str, content: tuple[str, Message]) -> None: ... diff --git a/mypy/typeshed/stdlib/urllib/parse.pyi b/mypy/typeshed/stdlib/urllib/parse.pyi new file mode 100644 index 000000000000..785bb9678ec7 --- /dev/null +++ b/mypy/typeshed/stdlib/urllib/parse.pyi @@ -0,0 +1,210 @@ +import sys +from collections.abc import Callable, Iterable, Mapping, Sequence +from typing import Any, AnyStr, Generic, Literal, NamedTuple, TypeVar, overload +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "urlparse", + "urlunparse", + "urljoin", + "urldefrag", + "urlsplit", + "urlunsplit", + "urlencode", + "parse_qs", + "parse_qsl", + "quote", + "quote_plus", + "quote_from_bytes", + "unquote", + "unquote_plus", + "unquote_to_bytes", + "DefragResult", + "ParseResult", + "SplitResult", + "DefragResultBytes", + "ParseResultBytes", + "SplitResultBytes", +] + +uses_relative: list[str] +uses_netloc: list[str] +uses_params: list[str] +non_hierarchical: list[str] +uses_query: list[str] +uses_fragment: list[str] +scheme_chars: str +if sys.version_info < (3, 11): + MAX_CACHE_SIZE: int + +class _ResultMixinStr: + def encode(self, encoding: str = "ascii", errors: str = "strict") -> _ResultMixinBytes: ... + +class _ResultMixinBytes: + def decode(self, encoding: str = "ascii", errors: str = "strict") -> _ResultMixinStr: ... + +class _NetlocResultMixinBase(Generic[AnyStr]): + @property + def username(self) -> AnyStr | None: ... + @property + def password(self) -> AnyStr | None: ... + @property + def hostname(self) -> AnyStr | None: ... + @property + def port(self) -> int | None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +class _NetlocResultMixinStr(_NetlocResultMixinBase[str], _ResultMixinStr): ... +class _NetlocResultMixinBytes(_NetlocResultMixinBase[bytes], _ResultMixinBytes): ... + +class _DefragResultBase(NamedTuple, Generic[AnyStr]): + url: AnyStr + fragment: AnyStr + +class _SplitResultBase(NamedTuple, Generic[AnyStr]): + scheme: AnyStr + netloc: AnyStr + path: AnyStr + query: AnyStr + fragment: AnyStr + +class _ParseResultBase(NamedTuple, Generic[AnyStr]): + scheme: AnyStr + netloc: AnyStr + path: AnyStr + params: AnyStr + query: AnyStr + fragment: AnyStr + +# Structured result objects for string data +class DefragResult(_DefragResultBase[str], _ResultMixinStr): + def geturl(self) -> str: ... + +class SplitResult(_SplitResultBase[str], _NetlocResultMixinStr): + def geturl(self) -> str: ... + +class ParseResult(_ParseResultBase[str], _NetlocResultMixinStr): + def geturl(self) -> str: ... + +# Structured result objects for bytes data +class DefragResultBytes(_DefragResultBase[bytes], _ResultMixinBytes): + def geturl(self) -> bytes: ... + +class SplitResultBytes(_SplitResultBase[bytes], _NetlocResultMixinBytes): + def geturl(self) -> bytes: ... + +class ParseResultBytes(_ParseResultBase[bytes], _NetlocResultMixinBytes): + def geturl(self) -> bytes: ... + +def parse_qs( + qs: AnyStr | None, + keep_blank_values: bool = False, + strict_parsing: bool = False, + encoding: str = "utf-8", + errors: str = "replace", + max_num_fields: int | None = None, + separator: str = "&", +) -> dict[AnyStr, list[AnyStr]]: ... +def parse_qsl( + qs: AnyStr | None, + keep_blank_values: bool = False, + strict_parsing: bool = False, + encoding: str = "utf-8", + errors: str = "replace", + max_num_fields: int | None = None, + separator: str = "&", +) -> list[tuple[AnyStr, AnyStr]]: ... +@overload +def quote(string: str, safe: str | Iterable[int] = "/", encoding: str | None = None, errors: str | None = None) -> str: ... +@overload +def quote(string: bytes | bytearray, safe: str | Iterable[int] = "/") -> str: ... +def quote_from_bytes(bs: bytes | bytearray, safe: str | Iterable[int] = "/") -> str: ... +@overload +def quote_plus(string: str, safe: str | Iterable[int] = "", encoding: str | None = None, errors: str | None = None) -> str: ... +@overload +def quote_plus(string: bytes | bytearray, safe: str | Iterable[int] = "") -> str: ... + +if sys.version_info >= (3, 9): + def unquote(string: str | bytes, encoding: str = "utf-8", errors: str = "replace") -> str: ... + +else: + def unquote(string: str, encoding: str = "utf-8", errors: str = "replace") -> str: ... + +def unquote_to_bytes(string: str | bytes | bytearray) -> bytes: ... +def unquote_plus(string: str, encoding: str = "utf-8", errors: str = "replace") -> str: ... +@overload +def urldefrag(url: str) -> DefragResult: ... +@overload +def urldefrag(url: bytes | bytearray | None) -> DefragResultBytes: ... + +_Q = TypeVar("_Q", bound=str | Iterable[int]) +_QueryType: TypeAlias = ( + Mapping[Any, Any] | Mapping[Any, Sequence[Any]] | Sequence[tuple[Any, Any]] | Sequence[tuple[Any, Sequence[Any]]] +) + +@overload +def urlencode( + query: _QueryType, + doseq: bool = False, + safe: str = "", + encoding: str | None = None, + errors: str | None = None, + quote_via: Callable[[AnyStr, str, str, str], str] = ..., +) -> str: ... +@overload +def urlencode( + query: _QueryType, + doseq: bool, + safe: _Q, + encoding: str | None = None, + errors: str | None = None, + quote_via: Callable[[AnyStr, _Q, str, str], str] = ..., +) -> str: ... +@overload +def urlencode( + query: _QueryType, + doseq: bool = False, + *, + safe: _Q, + encoding: str | None = None, + errors: str | None = None, + quote_via: Callable[[AnyStr, _Q, str, str], str] = ..., +) -> str: ... +def urljoin(base: AnyStr, url: AnyStr | None, allow_fragments: bool = True) -> AnyStr: ... +@overload +def urlparse(url: str, scheme: str = "", allow_fragments: bool = True) -> ParseResult: ... +@overload +def urlparse( + url: bytes | bytearray | None, scheme: bytes | bytearray | None | Literal[""] = "", allow_fragments: bool = True +) -> ParseResultBytes: ... +@overload +def urlsplit(url: str, scheme: str = "", allow_fragments: bool = True) -> SplitResult: ... + +if sys.version_info >= (3, 11): + @overload + def urlsplit( + url: bytes | None, scheme: bytes | None | Literal[""] = "", allow_fragments: bool = True + ) -> SplitResultBytes: ... + +else: + @overload + def urlsplit( + url: bytes | bytearray | None, scheme: bytes | bytearray | None | Literal[""] = "", allow_fragments: bool = True + ) -> SplitResultBytes: ... + +# Requires an iterable of length 6 +@overload +def urlunparse(components: Iterable[None]) -> Literal[b""]: ... # type: ignore[overload-overlap] +@overload +def urlunparse(components: Iterable[AnyStr | None]) -> AnyStr: ... + +# Requires an iterable of length 5 +@overload +def urlunsplit(components: Iterable[None]) -> Literal[b""]: ... # type: ignore[overload-overlap] +@overload +def urlunsplit(components: Iterable[AnyStr | None]) -> AnyStr: ... +def unwrap(url: str) -> str: ... diff --git a/mypy/typeshed/stdlib/urllib/request.pyi b/mypy/typeshed/stdlib/urllib/request.pyi new file mode 100644 index 000000000000..ad4f91fc31ae --- /dev/null +++ b/mypy/typeshed/stdlib/urllib/request.pyi @@ -0,0 +1,408 @@ +import ssl +import sys +from _typeshed import ReadableBuffer, StrOrBytesPath, SupportsRead +from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence +from email.message import Message +from http.client import HTTPConnection, HTTPMessage, HTTPResponse +from http.cookiejar import CookieJar +from re import Pattern +from typing import IO, Any, ClassVar, NoReturn, Protocol, TypeVar, overload +from typing_extensions import TypeAlias +from urllib.error import HTTPError as HTTPError +from urllib.response import addclosehook, addinfourl + +__all__ = [ + "Request", + "OpenerDirector", + "BaseHandler", + "HTTPDefaultErrorHandler", + "HTTPRedirectHandler", + "HTTPCookieProcessor", + "ProxyHandler", + "HTTPPasswordMgr", + "HTTPPasswordMgrWithDefaultRealm", + "HTTPPasswordMgrWithPriorAuth", + "AbstractBasicAuthHandler", + "HTTPBasicAuthHandler", + "ProxyBasicAuthHandler", + "AbstractDigestAuthHandler", + "HTTPDigestAuthHandler", + "ProxyDigestAuthHandler", + "HTTPHandler", + "FileHandler", + "FTPHandler", + "CacheFTPHandler", + "DataHandler", + "UnknownHandler", + "HTTPErrorProcessor", + "urlopen", + "install_opener", + "build_opener", + "pathname2url", + "url2pathname", + "getproxies", + "urlretrieve", + "urlcleanup", + "URLopener", + "FancyURLopener", + "HTTPSHandler", +] + +_T = TypeVar("_T") +_UrlopenRet: TypeAlias = Any +_DataType: TypeAlias = ReadableBuffer | SupportsRead[bytes] | Iterable[bytes] | None + +if sys.version_info >= (3, 13): + def urlopen( + url: str | Request, data: _DataType | None = None, timeout: float | None = ..., *, context: ssl.SSLContext | None = None + ) -> _UrlopenRet: ... + +else: + def urlopen( + url: str | Request, + data: _DataType | None = None, + timeout: float | None = ..., + *, + cafile: str | None = None, + capath: str | None = None, + cadefault: bool = False, + context: ssl.SSLContext | None = None, + ) -> _UrlopenRet: ... + +def install_opener(opener: OpenerDirector) -> None: ... +def build_opener(*handlers: BaseHandler | Callable[[], BaseHandler]) -> OpenerDirector: ... + +if sys.platform == "win32": + from nturl2path import pathname2url as pathname2url, url2pathname as url2pathname +else: + def url2pathname(pathname: str) -> str: ... + def pathname2url(pathname: str) -> str: ... + +def getproxies() -> dict[str, str]: ... +def getproxies_environment() -> dict[str, str]: ... +def parse_http_list(s: str) -> list[str]: ... +def parse_keqv_list(l: list[str]) -> dict[str, str]: ... + +if sys.platform == "win32" or sys.platform == "darwin": + def proxy_bypass(host: str) -> Any: ... # undocumented + +else: + def proxy_bypass(host: str, proxies: Mapping[str, str] | None = None) -> Any: ... # undocumented + +class Request: + @property + def full_url(self) -> str: ... + @full_url.setter + def full_url(self, value: str) -> None: ... + @full_url.deleter + def full_url(self) -> None: ... + type: str + host: str + origin_req_host: str + selector: str + data: _DataType + headers: MutableMapping[str, str] + unredirected_hdrs: dict[str, str] + unverifiable: bool + method: str | None + timeout: float | None # Undocumented, only set after __init__() by OpenerDirector.open() + def __init__( + self, + url: str, + data: _DataType = None, + headers: MutableMapping[str, str] = {}, + origin_req_host: str | None = None, + unverifiable: bool = False, + method: str | None = None, + ) -> None: ... + def get_method(self) -> str: ... + def add_header(self, key: str, val: str) -> None: ... + def add_unredirected_header(self, key: str, val: str) -> None: ... + def has_header(self, header_name: str) -> bool: ... + def remove_header(self, header_name: str) -> None: ... + def get_full_url(self) -> str: ... + def set_proxy(self, host: str, type: str) -> None: ... + @overload + def get_header(self, header_name: str) -> str | None: ... + @overload + def get_header(self, header_name: str, default: _T) -> str | _T: ... + def header_items(self) -> list[tuple[str, str]]: ... + def has_proxy(self) -> bool: ... + +class OpenerDirector: + addheaders: list[tuple[str, str]] + def add_handler(self, handler: BaseHandler) -> None: ... + def open(self, fullurl: str | Request, data: _DataType = None, timeout: float | None = ...) -> _UrlopenRet: ... + def error(self, proto: str, *args: Any) -> _UrlopenRet: ... + def close(self) -> None: ... + +class BaseHandler: + handler_order: ClassVar[int] + parent: OpenerDirector + def add_parent(self, parent: OpenerDirector) -> None: ... + def close(self) -> None: ... + def __lt__(self, other: object) -> bool: ... + +class HTTPDefaultErrorHandler(BaseHandler): + def http_error_default( + self, req: Request, fp: IO[bytes], code: int, msg: str, hdrs: HTTPMessage + ) -> HTTPError: ... # undocumented + +class HTTPRedirectHandler(BaseHandler): + max_redirections: ClassVar[int] # undocumented + max_repeats: ClassVar[int] # undocumented + inf_msg: ClassVar[str] # undocumented + def redirect_request( + self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage, newurl: str + ) -> Request | None: ... + def http_error_301(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + def http_error_302(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + def http_error_303(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + def http_error_307(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + if sys.version_info >= (3, 11): + def http_error_308( + self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage + ) -> _UrlopenRet | None: ... + +class HTTPCookieProcessor(BaseHandler): + cookiejar: CookieJar + def __init__(self, cookiejar: CookieJar | None = None) -> None: ... + def http_request(self, request: Request) -> Request: ... # undocumented + def http_response(self, request: Request, response: HTTPResponse) -> HTTPResponse: ... # undocumented + def https_request(self, request: Request) -> Request: ... # undocumented + def https_response(self, request: Request, response: HTTPResponse) -> HTTPResponse: ... # undocumented + +class ProxyHandler(BaseHandler): + def __init__(self, proxies: dict[str, str] | None = None) -> None: ... + def proxy_open(self, req: Request, proxy: str, type: str) -> _UrlopenRet | None: ... # undocumented + # TODO add a method for every (common) proxy protocol + +class HTTPPasswordMgr: + def add_password(self, realm: str, uri: str | Sequence[str], user: str, passwd: str) -> None: ... + def find_user_password(self, realm: str, authuri: str) -> tuple[str | None, str | None]: ... + def is_suburi(self, base: str, test: str) -> bool: ... # undocumented + def reduce_uri(self, uri: str, default_port: bool = True) -> tuple[str, str]: ... # undocumented + +class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr): + def add_password(self, realm: str | None, uri: str | Sequence[str], user: str, passwd: str) -> None: ... + def find_user_password(self, realm: str | None, authuri: str) -> tuple[str | None, str | None]: ... + +class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm): + def add_password( + self, realm: str | None, uri: str | Sequence[str], user: str, passwd: str, is_authenticated: bool = False + ) -> None: ... + def update_authenticated(self, uri: str | Sequence[str], is_authenticated: bool = False) -> None: ... + def is_authenticated(self, authuri: str) -> bool | None: ... + +class AbstractBasicAuthHandler: + rx: ClassVar[Pattern[str]] # undocumented + passwd: HTTPPasswordMgr + add_password: Callable[[str, str | Sequence[str], str, str], None] + def __init__(self, password_mgr: HTTPPasswordMgr | None = None) -> None: ... + def http_error_auth_reqed(self, authreq: str, host: str, req: Request, headers: HTTPMessage) -> None: ... + def http_request(self, req: Request) -> Request: ... # undocumented + def http_response(self, req: Request, response: HTTPResponse) -> HTTPResponse: ... # undocumented + def https_request(self, req: Request) -> Request: ... # undocumented + def https_response(self, req: Request, response: HTTPResponse) -> HTTPResponse: ... # undocumented + def retry_http_basic_auth(self, host: str, req: Request, realm: str) -> _UrlopenRet | None: ... # undocumented + +class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler): + auth_header: ClassVar[str] # undocumented + def http_error_401(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + +class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler): + auth_header: ClassVar[str] + def http_error_407(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + +class AbstractDigestAuthHandler: + def __init__(self, passwd: HTTPPasswordMgr | None = None) -> None: ... + def reset_retry_count(self) -> None: ... + def http_error_auth_reqed(self, auth_header: str, host: str, req: Request, headers: HTTPMessage) -> None: ... + def retry_http_digest_auth(self, req: Request, auth: str) -> _UrlopenRet | None: ... + def get_cnonce(self, nonce: str) -> str: ... + def get_authorization(self, req: Request, chal: Mapping[str, str]) -> str | None: ... + def get_algorithm_impls(self, algorithm: str) -> tuple[Callable[[str], str], Callable[[str, str], str]]: ... + def get_entity_digest(self, data: ReadableBuffer | None, chal: Mapping[str, str]) -> str | None: ... + +class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler): + auth_header: ClassVar[str] # undocumented + def http_error_401(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + +class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler): + auth_header: ClassVar[str] # undocumented + def http_error_407(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... + +class _HTTPConnectionProtocol(Protocol): + def __call__( + self, + host: str, + /, + *, + port: int | None = ..., + timeout: float = ..., + source_address: tuple[str, int] | None = ..., + blocksize: int = ..., + ) -> HTTPConnection: ... + +class AbstractHTTPHandler(BaseHandler): # undocumented + if sys.version_info >= (3, 12): + def __init__(self, debuglevel: int | None = None) -> None: ... + else: + def __init__(self, debuglevel: int = 0) -> None: ... + + def set_http_debuglevel(self, level: int) -> None: ... + def do_request_(self, request: Request) -> Request: ... + def do_open(self, http_class: _HTTPConnectionProtocol, req: Request, **http_conn_args: Any) -> HTTPResponse: ... + +class HTTPHandler(AbstractHTTPHandler): + def http_open(self, req: Request) -> HTTPResponse: ... + def http_request(self, request: Request) -> Request: ... # undocumented + +class HTTPSHandler(AbstractHTTPHandler): + if sys.version_info >= (3, 12): + def __init__( + self, debuglevel: int | None = None, context: ssl.SSLContext | None = None, check_hostname: bool | None = None + ) -> None: ... + else: + def __init__( + self, debuglevel: int = 0, context: ssl.SSLContext | None = None, check_hostname: bool | None = None + ) -> None: ... + + def https_open(self, req: Request) -> HTTPResponse: ... + def https_request(self, request: Request) -> Request: ... # undocumented + +class FileHandler(BaseHandler): + names: ClassVar[tuple[str, ...] | None] # undocumented + def file_open(self, req: Request) -> addinfourl: ... + def get_names(self) -> tuple[str, ...]: ... # undocumented + def open_local_file(self, req: Request) -> addinfourl: ... # undocumented + +class DataHandler(BaseHandler): + def data_open(self, req: Request) -> addinfourl: ... + +class ftpwrapper: # undocumented + def __init__( + self, user: str, passwd: str, host: str, port: int, dirs: str, timeout: float | None = None, persistent: bool = True + ) -> None: ... + def close(self) -> None: ... + def endtransfer(self) -> None: ... + def file_close(self) -> None: ... + def init(self) -> None: ... + def real_close(self) -> None: ... + def retrfile(self, file: str, type: str) -> tuple[addclosehook, int | None]: ... + +class FTPHandler(BaseHandler): + def ftp_open(self, req: Request) -> addinfourl: ... + def connect_ftp( + self, user: str, passwd: str, host: str, port: int, dirs: str, timeout: float + ) -> ftpwrapper: ... # undocumented + +class CacheFTPHandler(FTPHandler): + def setTimeout(self, t: float) -> None: ... + def setMaxConns(self, m: int) -> None: ... + def check_cache(self) -> None: ... # undocumented + def clear_cache(self) -> None: ... # undocumented + +class UnknownHandler(BaseHandler): + def unknown_open(self, req: Request) -> NoReturn: ... + +class HTTPErrorProcessor(BaseHandler): + def http_response(self, request: Request, response: HTTPResponse) -> _UrlopenRet: ... + def https_response(self, request: Request, response: HTTPResponse) -> _UrlopenRet: ... + +def urlretrieve( + url: str, + filename: StrOrBytesPath | None = None, + reporthook: Callable[[int, int, int], object] | None = None, + data: _DataType = None, +) -> tuple[str, HTTPMessage]: ... +def urlcleanup() -> None: ... + +class URLopener: + version: ClassVar[str] + def __init__(self, proxies: dict[str, str] | None = None, **x509: str) -> None: ... + def open(self, fullurl: str, data: ReadableBuffer | None = None) -> _UrlopenRet: ... + def open_unknown(self, fullurl: str, data: ReadableBuffer | None = None) -> _UrlopenRet: ... + def retrieve( + self, + url: str, + filename: str | None = None, + reporthook: Callable[[int, int, int], object] | None = None, + data: ReadableBuffer | None = None, + ) -> tuple[str, Message | None]: ... + def addheader(self, *args: tuple[str, str]) -> None: ... # undocumented + def cleanup(self) -> None: ... # undocumented + def close(self) -> None: ... # undocumented + def http_error( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: bytes | None = None + ) -> _UrlopenRet: ... # undocumented + def http_error_default( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage + ) -> _UrlopenRet: ... # undocumented + def open_data(self, url: str, data: ReadableBuffer | None = None) -> addinfourl: ... # undocumented + def open_file(self, url: str) -> addinfourl: ... # undocumented + def open_ftp(self, url: str) -> addinfourl: ... # undocumented + def open_http(self, url: str, data: ReadableBuffer | None = None) -> _UrlopenRet: ... # undocumented + def open_https(self, url: str, data: ReadableBuffer | None = None) -> _UrlopenRet: ... # undocumented + def open_local_file(self, url: str) -> addinfourl: ... # undocumented + def open_unknown_proxy(self, proxy: str, fullurl: str, data: ReadableBuffer | None = None) -> None: ... # undocumented + def __del__(self) -> None: ... + +class FancyURLopener(URLopener): + def prompt_user_passwd(self, host: str, realm: str) -> tuple[str, str]: ... + def get_user_passwd(self, host: str, realm: str, clear_cache: int = 0) -> tuple[str, str]: ... # undocumented + def http_error_301( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = None + ) -> _UrlopenRet | addinfourl | None: ... # undocumented + def http_error_302( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = None + ) -> _UrlopenRet | addinfourl | None: ... # undocumented + def http_error_303( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = None + ) -> _UrlopenRet | addinfourl | None: ... # undocumented + def http_error_307( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = None + ) -> _UrlopenRet | addinfourl | None: ... # undocumented + if sys.version_info >= (3, 11): + def http_error_308( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None = None + ) -> _UrlopenRet | addinfourl | None: ... # undocumented + + def http_error_401( + self, + url: str, + fp: IO[bytes], + errcode: int, + errmsg: str, + headers: HTTPMessage, + data: ReadableBuffer | None = None, + retry: bool = False, + ) -> _UrlopenRet | None: ... # undocumented + def http_error_407( + self, + url: str, + fp: IO[bytes], + errcode: int, + errmsg: str, + headers: HTTPMessage, + data: ReadableBuffer | None = None, + retry: bool = False, + ) -> _UrlopenRet | None: ... # undocumented + def http_error_default( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage + ) -> addinfourl: ... # undocumented + def redirect_internal( + self, url: str, fp: IO[bytes], errcode: int, errmsg: str, headers: HTTPMessage, data: ReadableBuffer | None + ) -> _UrlopenRet | None: ... # undocumented + def retry_http_basic_auth( + self, url: str, realm: str, data: ReadableBuffer | None = None + ) -> _UrlopenRet | None: ... # undocumented + def retry_https_basic_auth( + self, url: str, realm: str, data: ReadableBuffer | None = None + ) -> _UrlopenRet | None: ... # undocumented + def retry_proxy_http_basic_auth( + self, url: str, realm: str, data: ReadableBuffer | None = None + ) -> _UrlopenRet | None: ... # undocumented + def retry_proxy_https_basic_auth( + self, url: str, realm: str, data: ReadableBuffer | None = None + ) -> _UrlopenRet | None: ... # undocumented diff --git a/mypy/typeshed/stdlib/urllib/response.pyi b/mypy/typeshed/stdlib/urllib/response.pyi new file mode 100644 index 000000000000..bbec4cacc750 --- /dev/null +++ b/mypy/typeshed/stdlib/urllib/response.pyi @@ -0,0 +1,43 @@ +import sys +import tempfile +from _typeshed import ReadableBuffer +from collections.abc import Callable, Iterable +from email.message import Message +from types import TracebackType +from typing import IO, Any + +__all__ = ["addbase", "addclosehook", "addinfo", "addinfourl"] + +class addbase(tempfile._TemporaryFileWrapper[bytes]): + fp: IO[bytes] + def __init__(self, fp: IO[bytes]) -> None: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + # These methods don't actually exist, but the class inherits at runtime from + # tempfile._TemporaryFileWrapper, which uses __getattr__ to delegate to the + # underlying file object. To satisfy the BinaryIO interface, we pretend that this + # class has these additional methods. + def write(self, s: ReadableBuffer) -> int: ... + def writelines(self, lines: Iterable[ReadableBuffer]) -> None: ... + +class addclosehook(addbase): + closehook: Callable[..., object] + hookargs: tuple[Any, ...] + def __init__(self, fp: IO[bytes], closehook: Callable[..., object], *hookargs: Any) -> None: ... + +class addinfo(addbase): + headers: Message + def __init__(self, fp: IO[bytes], headers: Message) -> None: ... + def info(self) -> Message: ... + +class addinfourl(addinfo): + url: str + code: int | None + if sys.version_info >= (3, 9): + @property + def status(self) -> int | None: ... + + def __init__(self, fp: IO[bytes], headers: Message, url: str, code: int | None = None) -> None: ... + def geturl(self) -> str: ... + def getcode(self) -> int | None: ... diff --git a/mypy/typeshed/stdlib/urllib/robotparser.pyi b/mypy/typeshed/stdlib/urllib/robotparser.pyi new file mode 100644 index 000000000000..14ceef550dab --- /dev/null +++ b/mypy/typeshed/stdlib/urllib/robotparser.pyi @@ -0,0 +1,20 @@ +from collections.abc import Iterable +from typing import NamedTuple + +__all__ = ["RobotFileParser"] + +class RequestRate(NamedTuple): + requests: int + seconds: int + +class RobotFileParser: + def __init__(self, url: str = "") -> None: ... + def set_url(self, url: str) -> None: ... + def read(self) -> None: ... + def parse(self, lines: Iterable[str]) -> None: ... + def can_fetch(self, useragent: str, url: str) -> bool: ... + def mtime(self) -> int: ... + def modified(self) -> None: ... + def crawl_delay(self, useragent: str) -> str | None: ... + def request_rate(self, useragent: str) -> RequestRate | None: ... + def site_maps(self) -> list[str] | None: ... diff --git a/mypy/typeshed/stdlib/uu.pyi b/mypy/typeshed/stdlib/uu.pyi new file mode 100644 index 000000000000..324053e04337 --- /dev/null +++ b/mypy/typeshed/stdlib/uu.pyi @@ -0,0 +1,13 @@ +from typing import BinaryIO +from typing_extensions import TypeAlias + +__all__ = ["Error", "encode", "decode"] + +_File: TypeAlias = str | BinaryIO + +class Error(Exception): ... + +def encode( + in_file: _File, out_file: _File, name: str | None = None, mode: int | None = None, *, backtick: bool = False +) -> None: ... +def decode(in_file: _File, out_file: _File | None = None, mode: int | None = None, quiet: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/uuid.pyi b/mypy/typeshed/stdlib/uuid.pyi new file mode 100644 index 000000000000..1be7a5ef009f --- /dev/null +++ b/mypy/typeshed/stdlib/uuid.pyi @@ -0,0 +1,100 @@ +import builtins +import sys +from _typeshed import Unused +from enum import Enum +from typing_extensions import TypeAlias + +_FieldsType: TypeAlias = tuple[int, int, int, int, int, int] + +class SafeUUID(Enum): + safe = 0 + unsafe = -1 + unknown = None + +class UUID: + def __init__( + self, + hex: str | None = None, + bytes: builtins.bytes | None = None, + bytes_le: builtins.bytes | None = None, + fields: _FieldsType | None = None, + int: builtins.int | None = None, + version: builtins.int | None = None, + *, + is_safe: SafeUUID = ..., + ) -> None: ... + @property + def is_safe(self) -> SafeUUID: ... + @property + def bytes(self) -> builtins.bytes: ... + @property + def bytes_le(self) -> builtins.bytes: ... + @property + def clock_seq(self) -> builtins.int: ... + @property + def clock_seq_hi_variant(self) -> builtins.int: ... + @property + def clock_seq_low(self) -> builtins.int: ... + @property + def fields(self) -> _FieldsType: ... + @property + def hex(self) -> str: ... + @property + def int(self) -> builtins.int: ... + @property + def node(self) -> builtins.int: ... + @property + def time(self) -> builtins.int: ... + @property + def time_hi_version(self) -> builtins.int: ... + @property + def time_low(self) -> builtins.int: ... + @property + def time_mid(self) -> builtins.int: ... + @property + def urn(self) -> str: ... + @property + def variant(self) -> str: ... + @property + def version(self) -> builtins.int | None: ... + def __int__(self) -> builtins.int: ... + def __eq__(self, other: object) -> bool: ... + def __lt__(self, other: UUID) -> bool: ... + def __le__(self, other: UUID) -> bool: ... + def __gt__(self, other: UUID) -> bool: ... + def __ge__(self, other: UUID) -> bool: ... + def __hash__(self) -> builtins.int: ... + +if sys.version_info >= (3, 9): + def getnode() -> int: ... + +else: + def getnode(*, getters: Unused = None) -> int: ... # undocumented + +def uuid1(node: int | None = None, clock_seq: int | None = None) -> UUID: ... + +if sys.version_info >= (3, 12): + def uuid3(namespace: UUID, name: str | bytes) -> UUID: ... + +else: + def uuid3(namespace: UUID, name: str) -> UUID: ... + +def uuid4() -> UUID: ... + +if sys.version_info >= (3, 12): + def uuid5(namespace: UUID, name: str | bytes) -> UUID: ... + +else: + def uuid5(namespace: UUID, name: str) -> UUID: ... + +NAMESPACE_DNS: UUID +NAMESPACE_URL: UUID +NAMESPACE_OID: UUID +NAMESPACE_X500: UUID +RESERVED_NCS: str +RFC_4122: str +RESERVED_MICROSOFT: str +RESERVED_FUTURE: str + +if sys.version_info >= (3, 12): + def main() -> None: ... diff --git a/mypy/typeshed/stdlib/venv/__init__.pyi b/mypy/typeshed/stdlib/venv/__init__.pyi new file mode 100644 index 000000000000..0490c35b44f2 --- /dev/null +++ b/mypy/typeshed/stdlib/venv/__init__.pyi @@ -0,0 +1,107 @@ +import logging +import sys +from _typeshed import StrOrBytesPath +from collections.abc import Iterable, Sequence +from types import SimpleNamespace + +logger: logging.Logger + +if sys.version_info >= (3, 9): + CORE_VENV_DEPS: tuple[str, ...] + +class EnvBuilder: + system_site_packages: bool + clear: bool + symlinks: bool + upgrade: bool + with_pip: bool + prompt: str | None + + if sys.version_info >= (3, 13): + def __init__( + self, + system_site_packages: bool = False, + clear: bool = False, + symlinks: bool = False, + upgrade: bool = False, + with_pip: bool = False, + prompt: str | None = None, + upgrade_deps: bool = False, + *, + scm_ignore_files: Iterable[str] = ..., + ) -> None: ... + elif sys.version_info >= (3, 9): + def __init__( + self, + system_site_packages: bool = False, + clear: bool = False, + symlinks: bool = False, + upgrade: bool = False, + with_pip: bool = False, + prompt: str | None = None, + upgrade_deps: bool = False, + ) -> None: ... + else: + def __init__( + self, + system_site_packages: bool = False, + clear: bool = False, + symlinks: bool = False, + upgrade: bool = False, + with_pip: bool = False, + prompt: str | None = None, + ) -> None: ... + + def create(self, env_dir: StrOrBytesPath) -> None: ... + def clear_directory(self, path: StrOrBytesPath) -> None: ... # undocumented + def ensure_directories(self, env_dir: StrOrBytesPath) -> SimpleNamespace: ... + def create_configuration(self, context: SimpleNamespace) -> None: ... + def symlink_or_copy( + self, src: StrOrBytesPath, dst: StrOrBytesPath, relative_symlinks_ok: bool = False + ) -> None: ... # undocumented + def setup_python(self, context: SimpleNamespace) -> None: ... + def _setup_pip(self, context: SimpleNamespace) -> None: ... # undocumented + def setup_scripts(self, context: SimpleNamespace) -> None: ... + def post_setup(self, context: SimpleNamespace) -> None: ... + def replace_variables(self, text: str, context: SimpleNamespace) -> str: ... # undocumented + def install_scripts(self, context: SimpleNamespace, path: str) -> None: ... + if sys.version_info >= (3, 9): + def upgrade_dependencies(self, context: SimpleNamespace) -> None: ... + if sys.version_info >= (3, 13): + def create_git_ignore_file(self, context: SimpleNamespace) -> None: ... + +if sys.version_info >= (3, 13): + def create( + env_dir: StrOrBytesPath, + system_site_packages: bool = False, + clear: bool = False, + symlinks: bool = False, + with_pip: bool = False, + prompt: str | None = None, + upgrade_deps: bool = False, + *, + scm_ignore_files: Iterable[str] = ..., + ) -> None: ... + +elif sys.version_info >= (3, 9): + def create( + env_dir: StrOrBytesPath, + system_site_packages: bool = False, + clear: bool = False, + symlinks: bool = False, + with_pip: bool = False, + prompt: str | None = None, + upgrade_deps: bool = False, + ) -> None: ... + +else: + def create( + env_dir: StrOrBytesPath, + system_site_packages: bool = False, + clear: bool = False, + symlinks: bool = False, + with_pip: bool = False, + prompt: str | None = None, + ) -> None: ... + +def main(args: Sequence[str] | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/warnings.pyi b/mypy/typeshed/stdlib/warnings.pyi new file mode 100644 index 000000000000..533a36817506 --- /dev/null +++ b/mypy/typeshed/stdlib/warnings.pyi @@ -0,0 +1,127 @@ +import re +import sys +from _warnings import warn as warn, warn_explicit as warn_explicit +from collections.abc import Sequence +from types import ModuleType, TracebackType +from typing import Any, Generic, Literal, TextIO, TypeVar, overload +from typing_extensions import LiteralString, TypeAlias + +__all__ = [ + "warn", + "warn_explicit", + "showwarning", + "formatwarning", + "filterwarnings", + "simplefilter", + "resetwarnings", + "catch_warnings", +] + +if sys.version_info >= (3, 13): + __all__ += ["deprecated"] + +_T = TypeVar("_T") +_W = TypeVar("_W", bound=list[WarningMessage] | None) +if sys.version_info >= (3, 14): + _ActionKind: TypeAlias = Literal["default", "error", "ignore", "always", "module", "once"] +else: + _ActionKind: TypeAlias = Literal["default", "error", "ignore", "always", "all", "module", "once"] +filters: Sequence[tuple[str, re.Pattern[str] | None, type[Warning], re.Pattern[str] | None, int]] # undocumented, do not mutate + +def showwarning( + message: Warning | str, + category: type[Warning], + filename: str, + lineno: int, + file: TextIO | None = None, + line: str | None = None, +) -> None: ... +def formatwarning( + message: Warning | str, category: type[Warning], filename: str, lineno: int, line: str | None = None +) -> str: ... +def filterwarnings( + action: _ActionKind, message: str = "", category: type[Warning] = ..., module: str = "", lineno: int = 0, append: bool = False +) -> None: ... +def simplefilter(action: _ActionKind, category: type[Warning] = ..., lineno: int = 0, append: bool = False) -> None: ... +def resetwarnings() -> None: ... + +class _OptionError(Exception): ... + +class WarningMessage: + message: Warning | str + category: type[Warning] + filename: str + lineno: int + file: TextIO | None + line: str | None + source: Any | None + def __init__( + self, + message: Warning | str, + category: type[Warning], + filename: str, + lineno: int, + file: TextIO | None = None, + line: str | None = None, + source: Any | None = None, + ) -> None: ... + +class catch_warnings(Generic[_W]): + if sys.version_info >= (3, 11): + @overload + def __init__( + self: catch_warnings[None], + *, + record: Literal[False] = False, + module: ModuleType | None = None, + action: _ActionKind | None = None, + category: type[Warning] = ..., + lineno: int = 0, + append: bool = False, + ) -> None: ... + @overload + def __init__( + self: catch_warnings[list[WarningMessage]], + *, + record: Literal[True], + module: ModuleType | None = None, + action: _ActionKind | None = None, + category: type[Warning] = ..., + lineno: int = 0, + append: bool = False, + ) -> None: ... + @overload + def __init__( + self: catch_warnings[list[WarningMessage] | None], + *, + record: bool, + module: ModuleType | None = None, + action: _ActionKind | None = None, + category: type[Warning] = ..., + lineno: int = 0, + append: bool = False, + ) -> None: ... + else: + @overload + def __init__(self: catch_warnings[None], *, record: Literal[False] = False, module: ModuleType | None = None) -> None: ... + @overload + def __init__( + self: catch_warnings[list[WarningMessage]], *, record: Literal[True], module: ModuleType | None = None + ) -> None: ... + @overload + def __init__( + self: catch_warnings[list[WarningMessage] | None], *, record: bool, module: ModuleType | None = None + ) -> None: ... + + def __enter__(self) -> _W: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + +if sys.version_info >= (3, 13): + class deprecated: + message: LiteralString + category: type[Warning] | None + stacklevel: int + def __init__(self, message: LiteralString, /, *, category: type[Warning] | None = ..., stacklevel: int = 1) -> None: ... + def __call__(self, arg: _T, /) -> _T: ... diff --git a/mypy/typeshed/stdlib/wave.pyi b/mypy/typeshed/stdlib/wave.pyi new file mode 100644 index 000000000000..9319d5347c79 --- /dev/null +++ b/mypy/typeshed/stdlib/wave.pyi @@ -0,0 +1,85 @@ +import sys +from _typeshed import ReadableBuffer, Unused +from typing import IO, Any, BinaryIO, Final, Literal, NamedTuple, NoReturn, overload +from typing_extensions import Self, TypeAlias, deprecated + +if sys.version_info >= (3, 9): + __all__ = ["open", "Error", "Wave_read", "Wave_write"] +else: + __all__ = ["open", "openfp", "Error", "Wave_read", "Wave_write"] + +_File: TypeAlias = str | IO[bytes] + +class Error(Exception): ... + +WAVE_FORMAT_PCM: Final = 1 + +class _wave_params(NamedTuple): + nchannels: int + sampwidth: int + framerate: int + nframes: int + comptype: str + compname: str + +class Wave_read: + def __init__(self, f: _File) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def __del__(self) -> None: ... + def getfp(self) -> BinaryIO | None: ... + def rewind(self) -> None: ... + def close(self) -> None: ... + def tell(self) -> int: ... + def getnchannels(self) -> int: ... + def getnframes(self) -> int: ... + def getsampwidth(self) -> int: ... + def getframerate(self) -> int: ... + def getcomptype(self) -> str: ... + def getcompname(self) -> str: ... + def getparams(self) -> _wave_params: ... + @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") + def getmarkers(self) -> None: ... + @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") + def getmark(self, id: Any) -> NoReturn: ... + def setpos(self, pos: int) -> None: ... + def readframes(self, nframes: int) -> bytes: ... + +class Wave_write: + def __init__(self, f: _File) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + def __del__(self) -> None: ... + def setnchannels(self, nchannels: int) -> None: ... + def getnchannels(self) -> int: ... + def setsampwidth(self, sampwidth: int) -> None: ... + def getsampwidth(self) -> int: ... + def setframerate(self, framerate: float) -> None: ... + def getframerate(self) -> int: ... + def setnframes(self, nframes: int) -> None: ... + def getnframes(self) -> int: ... + def setcomptype(self, comptype: str, compname: str) -> None: ... + def getcomptype(self) -> str: ... + def getcompname(self) -> str: ... + def setparams(self, params: _wave_params | tuple[int, int, int, int, str, str]) -> None: ... + def getparams(self) -> _wave_params: ... + @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") + def setmark(self, id: Any, pos: Any, name: Any) -> NoReturn: ... + @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") + def getmark(self, id: Any) -> NoReturn: ... + @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") + def getmarkers(self) -> None: ... + def tell(self) -> int: ... + def writeframesraw(self, data: ReadableBuffer) -> None: ... + def writeframes(self, data: ReadableBuffer) -> None: ... + def close(self) -> None: ... + +@overload +def open(f: _File, mode: Literal["r", "rb"]) -> Wave_read: ... +@overload +def open(f: _File, mode: Literal["w", "wb"]) -> Wave_write: ... +@overload +def open(f: _File, mode: str | None = None) -> Any: ... + +if sys.version_info < (3, 9): + openfp = open diff --git a/mypy/typeshed/stdlib/weakref.pyi b/mypy/typeshed/stdlib/weakref.pyi new file mode 100644 index 000000000000..853caf3e8abb --- /dev/null +++ b/mypy/typeshed/stdlib/weakref.pyi @@ -0,0 +1,182 @@ +import sys +from _typeshed import SupportsKeysAndGetItem +from _weakref import getweakrefcount as getweakrefcount, getweakrefs as getweakrefs, proxy as proxy +from _weakrefset import WeakSet as WeakSet +from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping +from typing import Any, Generic, TypeVar, final, overload +from typing_extensions import ParamSpec, Self + +if sys.version_info >= (3, 9): + from types import GenericAlias + +__all__ = [ + "ref", + "proxy", + "getweakrefcount", + "getweakrefs", + "WeakKeyDictionary", + "ReferenceType", + "ProxyType", + "CallableProxyType", + "ProxyTypes", + "WeakValueDictionary", + "WeakSet", + "WeakMethod", + "finalize", +] + +_T = TypeVar("_T") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_CallableT = TypeVar("_CallableT", bound=Callable[..., Any]) +_P = ParamSpec("_P") + +ProxyTypes: tuple[type[Any], ...] + +# These classes are implemented in C and imported from _weakref at runtime. However, +# they consider themselves to live in the weakref module for sys.version_info >= (3, 11), +# so defining their stubs here means we match their __module__ value. +# Prior to 3.11 they did not declare a module for themselves and ended up looking like they +# came from the builtin module at runtime, which was just wrong, and we won't attempt to +# duplicate that. + +@final +class CallableProxyType(Generic[_CallableT]): # "weakcallableproxy" + def __eq__(self, value: object, /) -> bool: ... + def __getattr__(self, attr: str) -> Any: ... + __call__: _CallableT + +@final +class ProxyType(Generic[_T]): # "weakproxy" + def __eq__(self, value: object, /) -> bool: ... + def __getattr__(self, attr: str) -> Any: ... + +class ReferenceType(Generic[_T]): # "weakref" + __callback__: Callable[[Self], Any] + def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... + def __call__(self) -> _T | None: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +ref = ReferenceType + +# everything below here is implemented in weakref.py + +class WeakMethod(ref[_CallableT]): + def __new__(cls, meth: _CallableT, callback: Callable[[Self], Any] | None = None) -> Self: ... + def __call__(self) -> _CallableT | None: ... + def __eq__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +class WeakValueDictionary(MutableMapping[_KT, _VT]): + @overload + def __init__(self) -> None: ... + @overload + def __init__( + self: WeakValueDictionary[_KT, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + other: Mapping[_KT, _VT] | Iterable[tuple[_KT, _VT]], + /, + ) -> None: ... + @overload + def __init__( + self: WeakValueDictionary[str, _VT], # pyright: ignore[reportInvalidTypeVarUse] #11780 + other: Mapping[str, _VT] | Iterable[tuple[str, _VT]] = (), + /, + **kwargs: _VT, + ) -> None: ... + def __len__(self) -> int: ... + def __getitem__(self, key: _KT) -> _VT: ... + def __setitem__(self, key: _KT, value: _VT) -> None: ... + def __delitem__(self, key: _KT) -> None: ... + def __contains__(self, key: object) -> bool: ... + def __iter__(self) -> Iterator[_KT]: ... + def copy(self) -> WeakValueDictionary[_KT, _VT]: ... + __copy__ = copy + def __deepcopy__(self, memo: Any) -> Self: ... + @overload + def get(self, key: _KT, default: None = None) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _T) -> _VT | _T: ... + # These are incompatible with Mapping + def keys(self) -> Iterator[_KT]: ... # type: ignore[override] + def values(self) -> Iterator[_VT]: ... # type: ignore[override] + def items(self) -> Iterator[tuple[_KT, _VT]]: ... # type: ignore[override] + def itervaluerefs(self) -> Iterator[KeyedRef[_KT, _VT]]: ... + def valuerefs(self) -> list[KeyedRef[_KT, _VT]]: ... + def setdefault(self, key: _KT, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T) -> _VT | _T: ... + if sys.version_info >= (3, 9): + def __or__(self, other: Mapping[_T1, _T2]) -> WeakValueDictionary[_KT | _T1, _VT | _T2]: ... + def __ror__(self, other: Mapping[_T1, _T2]) -> WeakValueDictionary[_KT | _T1, _VT | _T2]: ... + # WeakValueDictionary.__ior__ should be kept roughly in line with MutableMapping.update() + @overload # type: ignore[misc] + def __ior__(self, other: SupportsKeysAndGetItem[_KT, _VT]) -> Self: ... + @overload + def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... + +class KeyedRef(ref[_T], Generic[_KT, _T]): + key: _KT + def __new__(type, ob: _T, callback: Callable[[Self], Any], key: _KT) -> Self: ... + def __init__(self, ob: _T, callback: Callable[[Self], Any], key: _KT) -> None: ... + +class WeakKeyDictionary(MutableMapping[_KT, _VT]): + @overload + def __init__(self, dict: None = None) -> None: ... + @overload + def __init__(self, dict: Mapping[_KT, _VT] | Iterable[tuple[_KT, _VT]]) -> None: ... + def __len__(self) -> int: ... + def __getitem__(self, key: _KT) -> _VT: ... + def __setitem__(self, key: _KT, value: _VT) -> None: ... + def __delitem__(self, key: _KT) -> None: ... + def __contains__(self, key: object) -> bool: ... + def __iter__(self) -> Iterator[_KT]: ... + def copy(self) -> WeakKeyDictionary[_KT, _VT]: ... + __copy__ = copy + def __deepcopy__(self, memo: Any) -> Self: ... + @overload + def get(self, key: _KT, default: None = None) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _T) -> _VT | _T: ... + # These are incompatible with Mapping + def keys(self) -> Iterator[_KT]: ... # type: ignore[override] + def values(self) -> Iterator[_VT]: ... # type: ignore[override] + def items(self) -> Iterator[tuple[_KT, _VT]]: ... # type: ignore[override] + def keyrefs(self) -> list[ref[_KT]]: ... + # Keep WeakKeyDictionary.setdefault in line with MutableMapping.setdefault, modulo positional-only differences + @overload + def setdefault(self: WeakKeyDictionary[_KT, _VT | None], key: _KT, default: None = None) -> _VT: ... + @overload + def setdefault(self, key: _KT, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _VT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T) -> _VT | _T: ... + if sys.version_info >= (3, 9): + def __or__(self, other: Mapping[_T1, _T2]) -> WeakKeyDictionary[_KT | _T1, _VT | _T2]: ... + def __ror__(self, other: Mapping[_T1, _T2]) -> WeakKeyDictionary[_KT | _T1, _VT | _T2]: ... + # WeakKeyDictionary.__ior__ should be kept roughly in line with MutableMapping.update() + @overload # type: ignore[misc] + def __ior__(self, other: SupportsKeysAndGetItem[_KT, _VT]) -> Self: ... + @overload + def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... + +class finalize: # TODO: This is a good candidate for to be a `Generic[_P, _T]` class + def __init__(self, obj: object, func: Callable[_P, Any], /, *args: _P.args, **kwargs: _P.kwargs) -> None: ... + def __call__(self, _: Any = None) -> Any | None: ... + def detach(self) -> tuple[Any, Any, tuple[Any, ...], dict[str, Any]] | None: ... + def peek(self) -> tuple[Any, Any, tuple[Any, ...], dict[str, Any]] | None: ... + @property + def alive(self) -> bool: ... + atexit: bool diff --git a/mypy/typeshed/stdlib/webbrowser.pyi b/mypy/typeshed/stdlib/webbrowser.pyi new file mode 100644 index 000000000000..773786c24821 --- /dev/null +++ b/mypy/typeshed/stdlib/webbrowser.pyi @@ -0,0 +1,78 @@ +import sys +from abc import abstractmethod +from collections.abc import Callable, Sequence +from typing import Literal +from typing_extensions import deprecated + +__all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] + +class Error(Exception): ... + +def register( + name: str, klass: Callable[[], BaseBrowser] | None, instance: BaseBrowser | None = None, *, preferred: bool = False +) -> None: ... +def get(using: str | None = None) -> BaseBrowser: ... +def open(url: str, new: int = 0, autoraise: bool = True) -> bool: ... +def open_new(url: str) -> bool: ... +def open_new_tab(url: str) -> bool: ... + +class BaseBrowser: + args: list[str] + name: str + basename: str + def __init__(self, name: str = "") -> None: ... + @abstractmethod + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + def open_new(self, url: str) -> bool: ... + def open_new_tab(self, url: str) -> bool: ... + +class GenericBrowser(BaseBrowser): + def __init__(self, name: str | Sequence[str]) -> None: ... + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + +class BackgroundBrowser(GenericBrowser): ... + +class UnixBrowser(BaseBrowser): + def open(self, url: str, new: Literal[0, 1, 2] = 0, autoraise: bool = True) -> bool: ... # type: ignore[override] + raise_opts: list[str] | None + background: bool + redirect_stdout: bool + remote_args: list[str] + remote_action: str + remote_action_newwin: str + remote_action_newtab: str + +class Mozilla(UnixBrowser): ... + +if sys.version_info < (3, 12): + class Galeon(UnixBrowser): + raise_opts: list[str] + + class Grail(BaseBrowser): + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + +class Chrome(UnixBrowser): ... +class Opera(UnixBrowser): ... +class Elinks(UnixBrowser): ... + +class Konqueror(BaseBrowser): + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + +if sys.platform == "win32": + class WindowsDefault(BaseBrowser): + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + +if sys.platform == "darwin": + if sys.version_info < (3, 13): + @deprecated("Deprecated in 3.11, to be removed in 3.13.") + class MacOSX(BaseBrowser): + def __init__(self, name: str) -> None: ... + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + + class MacOSXOSAScript(BaseBrowser): # In runtime this class does not have `name` and `basename` + if sys.version_info >= (3, 11): + def __init__(self, name: str = "default") -> None: ... + else: + def __init__(self, name: str) -> None: ... + + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... diff --git a/mypy/typeshed/stdlib/winreg.pyi b/mypy/typeshed/stdlib/winreg.pyi new file mode 100644 index 000000000000..d4d04817d7e0 --- /dev/null +++ b/mypy/typeshed/stdlib/winreg.pyi @@ -0,0 +1,132 @@ +import sys +from _typeshed import ReadableBuffer, Unused +from types import TracebackType +from typing import Any, Final, Literal, final, overload +from typing_extensions import Self, TypeAlias + +if sys.platform == "win32": + _KeyType: TypeAlias = HKEYType | int + def CloseKey(hkey: _KeyType, /) -> None: ... + def ConnectRegistry(computer_name: str | None, key: _KeyType, /) -> HKEYType: ... + def CreateKey(key: _KeyType, sub_key: str | None, /) -> HKEYType: ... + def CreateKeyEx(key: _KeyType, sub_key: str | None, reserved: int = 0, access: int = 131078) -> HKEYType: ... + def DeleteKey(key: _KeyType, sub_key: str, /) -> None: ... + def DeleteKeyEx(key: _KeyType, sub_key: str, access: int = 256, reserved: int = 0) -> None: ... + def DeleteValue(key: _KeyType, value: str, /) -> None: ... + def EnumKey(key: _KeyType, index: int, /) -> str: ... + def EnumValue(key: _KeyType, index: int, /) -> tuple[str, Any, int]: ... + def ExpandEnvironmentStrings(string: str, /) -> str: ... + def FlushKey(key: _KeyType, /) -> None: ... + def LoadKey(key: _KeyType, sub_key: str, file_name: str, /) -> None: ... + def OpenKey(key: _KeyType, sub_key: str, reserved: int = 0, access: int = 131097) -> HKEYType: ... + def OpenKeyEx(key: _KeyType, sub_key: str, reserved: int = 0, access: int = 131097) -> HKEYType: ... + def QueryInfoKey(key: _KeyType, /) -> tuple[int, int, int]: ... + def QueryValue(key: _KeyType, sub_key: str | None, /) -> str: ... + def QueryValueEx(key: _KeyType, name: str, /) -> tuple[Any, int]: ... + def SaveKey(key: _KeyType, file_name: str, /) -> None: ... + def SetValue(key: _KeyType, sub_key: str, type: int, value: str, /) -> None: ... + @overload # type=REG_DWORD|REG_QWORD + def SetValueEx( + key: _KeyType, value_name: str | None, reserved: Unused, type: Literal[4, 5], value: int | None, / + ) -> None: ... + @overload # type=REG_SZ|REG_EXPAND_SZ + def SetValueEx( + key: _KeyType, value_name: str | None, reserved: Unused, type: Literal[1, 2], value: str | None, / + ) -> None: ... + @overload # type=REG_MULTI_SZ + def SetValueEx( + key: _KeyType, value_name: str | None, reserved: Unused, type: Literal[7], value: list[str] | None, / + ) -> None: ... + @overload # type=REG_BINARY and everything else + def SetValueEx( + key: _KeyType, + value_name: str | None, + reserved: Unused, + type: Literal[0, 3, 8, 9, 10, 11], + value: ReadableBuffer | None, + /, + ) -> None: ... + @overload # Unknown or undocumented + def SetValueEx( + key: _KeyType, + value_name: str | None, + reserved: Unused, + type: int, + value: int | str | list[str] | ReadableBuffer | None, + /, + ) -> None: ... + def DisableReflectionKey(key: _KeyType, /) -> None: ... + def EnableReflectionKey(key: _KeyType, /) -> None: ... + def QueryReflectionKey(key: _KeyType, /) -> bool: ... + + HKEY_CLASSES_ROOT: int + HKEY_CURRENT_USER: int + HKEY_LOCAL_MACHINE: int + HKEY_USERS: int + HKEY_PERFORMANCE_DATA: int + HKEY_CURRENT_CONFIG: int + HKEY_DYN_DATA: int + + KEY_ALL_ACCESS: Final = 983103 + KEY_WRITE: Final = 131078 + KEY_READ: Final = 131097 + KEY_EXECUTE: Final = 131097 + KEY_QUERY_VALUE: Final = 1 + KEY_SET_VALUE: Final = 2 + KEY_CREATE_SUB_KEY: Final = 4 + KEY_ENUMERATE_SUB_KEYS: Final = 8 + KEY_NOTIFY: Final = 16 + KEY_CREATE_LINK: Final = 32 + + KEY_WOW64_64KEY: Final = 256 + KEY_WOW64_32KEY: Final = 512 + + REG_BINARY: Final = 3 + REG_DWORD: Final = 4 + REG_DWORD_LITTLE_ENDIAN: Final = 4 + REG_DWORD_BIG_ENDIAN: Final = 5 + REG_EXPAND_SZ: Final = 2 + REG_LINK: Final = 6 + REG_MULTI_SZ: Final = 7 + REG_NONE: Final = 0 + REG_QWORD: Final = 11 + REG_QWORD_LITTLE_ENDIAN: Final = 11 + REG_RESOURCE_LIST: Final = 8 + REG_FULL_RESOURCE_DESCRIPTOR: Final = 9 + REG_RESOURCE_REQUIREMENTS_LIST: Final = 10 + REG_SZ: Final = 1 + + REG_CREATED_NEW_KEY: Final = 1 # undocumented + REG_LEGAL_CHANGE_FILTER: Final = 268435471 # undocumented + REG_LEGAL_OPTION: Final = 31 # undocumented + REG_NOTIFY_CHANGE_ATTRIBUTES: Final = 2 # undocumented + REG_NOTIFY_CHANGE_LAST_SET: Final = 4 # undocumented + REG_NOTIFY_CHANGE_NAME: Final = 1 # undocumented + REG_NOTIFY_CHANGE_SECURITY: Final = 8 # undocumented + REG_NO_LAZY_FLUSH: Final = 4 # undocumented + REG_OPENED_EXISTING_KEY: Final = 2 # undocumented + REG_OPTION_BACKUP_RESTORE: Final = 4 # undocumented + REG_OPTION_CREATE_LINK: Final = 2 # undocumented + REG_OPTION_NON_VOLATILE: Final = 0 # undocumented + REG_OPTION_OPEN_LINK: Final = 8 # undocumented + REG_OPTION_RESERVED: Final = 0 # undocumented + REG_OPTION_VOLATILE: Final = 1 # undocumented + REG_REFRESH_HIVE: Final = 2 # undocumented + REG_WHOLE_HIVE_VOLATILE: Final = 1 # undocumented + + error = OSError + + # Though this class has a __name__ of PyHKEY, it's exposed as HKEYType for some reason + @final + class HKEYType: + def __bool__(self) -> bool: ... + def __int__(self) -> int: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None + ) -> bool | None: ... + def Close(self) -> None: ... + def Detach(self) -> int: ... + def __hash__(self) -> int: ... + @property + def handle(self) -> int: ... diff --git a/mypy/typeshed/stdlib/winsound.pyi b/mypy/typeshed/stdlib/winsound.pyi new file mode 100644 index 000000000000..a20e81f94f98 --- /dev/null +++ b/mypy/typeshed/stdlib/winsound.pyi @@ -0,0 +1,28 @@ +import sys +from _typeshed import ReadableBuffer +from typing import Final, Literal, overload + +if sys.platform == "win32": + SND_APPLICATION: Final = 128 + SND_FILENAME: Final = 131072 + SND_ALIAS: Final = 65536 + SND_LOOP: Final = 8 + SND_MEMORY: Final = 4 + SND_PURGE: Final = 64 + SND_ASYNC: Final = 1 + SND_NODEFAULT: Final = 2 + SND_NOSTOP: Final = 16 + SND_NOWAIT: Final = 8192 + + MB_ICONASTERISK: Final = 64 + MB_ICONEXCLAMATION: Final = 48 + MB_ICONHAND: Final = 16 + MB_ICONQUESTION: Final = 32 + MB_OK: Final = 0 + def Beep(frequency: int, duration: int) -> None: ... + # Can actually accept anything ORed with 4, and if not it's definitely str, but that's inexpressible + @overload + def PlaySound(sound: ReadableBuffer | None, flags: Literal[4]) -> None: ... + @overload + def PlaySound(sound: str | ReadableBuffer | None, flags: int) -> None: ... + def MessageBeep(type: int = 0) -> None: ... diff --git a/mypy/typeshed/stdlib/wsgiref/__init__.pyi b/mypy/typeshed/stdlib/wsgiref/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypy/typeshed/stdlib/wsgiref/handlers.pyi b/mypy/typeshed/stdlib/wsgiref/handlers.pyi new file mode 100644 index 000000000000..ebead540018e --- /dev/null +++ b/mypy/typeshed/stdlib/wsgiref/handlers.pyi @@ -0,0 +1,91 @@ +from _typeshed import OptExcInfo +from _typeshed.wsgi import ErrorStream, InputStream, StartResponse, WSGIApplication, WSGIEnvironment +from abc import abstractmethod +from collections.abc import Callable, MutableMapping +from typing import IO + +from .headers import Headers +from .util import FileWrapper + +__all__ = ["BaseHandler", "SimpleHandler", "BaseCGIHandler", "CGIHandler", "IISCGIHandler", "read_environ"] + +def format_date_time(timestamp: float | None) -> str: ... # undocumented +def read_environ() -> dict[str, str]: ... + +class BaseHandler: + wsgi_version: tuple[int, int] # undocumented + wsgi_multithread: bool + wsgi_multiprocess: bool + wsgi_run_once: bool + + origin_server: bool + http_version: str + server_software: str | None + + os_environ: MutableMapping[str, str] + + wsgi_file_wrapper: type[FileWrapper] | None + headers_class: type[Headers] # undocumented + + traceback_limit: int | None + error_status: str + error_headers: list[tuple[str, str]] + error_body: bytes + def run(self, application: WSGIApplication) -> None: ... + def setup_environ(self) -> None: ... + def finish_response(self) -> None: ... + def get_scheme(self) -> str: ... + def set_content_length(self) -> None: ... + def cleanup_headers(self) -> None: ... + def start_response( + self, status: str, headers: list[tuple[str, str]], exc_info: OptExcInfo | None = None + ) -> Callable[[bytes], None]: ... + def send_preamble(self) -> None: ... + def write(self, data: bytes) -> None: ... + def sendfile(self) -> bool: ... + def finish_content(self) -> None: ... + def close(self) -> None: ... + def send_headers(self) -> None: ... + def result_is_file(self) -> bool: ... + def client_is_modern(self) -> bool: ... + def log_exception(self, exc_info: OptExcInfo) -> None: ... + def handle_error(self) -> None: ... + def error_output(self, environ: WSGIEnvironment, start_response: StartResponse) -> list[bytes]: ... + @abstractmethod + def _write(self, data: bytes) -> None: ... + @abstractmethod + def _flush(self) -> None: ... + @abstractmethod + def get_stdin(self) -> InputStream: ... + @abstractmethod + def get_stderr(self) -> ErrorStream: ... + @abstractmethod + def add_cgi_vars(self) -> None: ... + +class SimpleHandler(BaseHandler): + stdin: InputStream + stdout: IO[bytes] + stderr: ErrorStream + base_env: MutableMapping[str, str] + def __init__( + self, + stdin: InputStream, + stdout: IO[bytes], + stderr: ErrorStream, + environ: MutableMapping[str, str], + multithread: bool = True, + multiprocess: bool = False, + ) -> None: ... + def get_stdin(self) -> InputStream: ... + def get_stderr(self) -> ErrorStream: ... + def add_cgi_vars(self) -> None: ... + def _write(self, data: bytes) -> None: ... + def _flush(self) -> None: ... + +class BaseCGIHandler(SimpleHandler): ... + +class CGIHandler(BaseCGIHandler): + def __init__(self) -> None: ... + +class IISCGIHandler(BaseCGIHandler): + def __init__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/wsgiref/headers.pyi b/mypy/typeshed/stdlib/wsgiref/headers.pyi new file mode 100644 index 000000000000..2654d79bf4e5 --- /dev/null +++ b/mypy/typeshed/stdlib/wsgiref/headers.pyi @@ -0,0 +1,26 @@ +from re import Pattern +from typing import overload +from typing_extensions import TypeAlias + +_HeaderList: TypeAlias = list[tuple[str, str]] + +tspecials: Pattern[str] # undocumented + +class Headers: + def __init__(self, headers: _HeaderList | None = None) -> None: ... + def __len__(self) -> int: ... + def __setitem__(self, name: str, val: str) -> None: ... + def __delitem__(self, name: str) -> None: ... + def __getitem__(self, name: str) -> str | None: ... + def __contains__(self, name: str) -> bool: ... + def get_all(self, name: str) -> list[str]: ... + @overload + def get(self, name: str, default: str) -> str: ... + @overload + def get(self, name: str, default: str | None = None) -> str | None: ... + def keys(self) -> list[str]: ... + def values(self) -> list[str]: ... + def items(self) -> _HeaderList: ... + def __bytes__(self) -> bytes: ... + def setdefault(self, name: str, value: str) -> str: ... + def add_header(self, _name: str, _value: str | None, **_params: str | None) -> None: ... diff --git a/mypy/typeshed/stdlib/wsgiref/simple_server.pyi b/mypy/typeshed/stdlib/wsgiref/simple_server.pyi new file mode 100644 index 000000000000..547f562cc1d4 --- /dev/null +++ b/mypy/typeshed/stdlib/wsgiref/simple_server.pyi @@ -0,0 +1,37 @@ +from _typeshed.wsgi import ErrorStream, StartResponse, WSGIApplication, WSGIEnvironment +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import TypeVar, overload + +from .handlers import SimpleHandler + +__all__ = ["WSGIServer", "WSGIRequestHandler", "demo_app", "make_server"] + +server_version: str # undocumented +sys_version: str # undocumented +software_version: str # undocumented + +class ServerHandler(SimpleHandler): # undocumented + server_software: str + +class WSGIServer(HTTPServer): + application: WSGIApplication | None + base_environ: WSGIEnvironment # only available after call to setup_environ() + def setup_environ(self) -> None: ... + def get_app(self) -> WSGIApplication | None: ... + def set_app(self, application: WSGIApplication | None) -> None: ... + +class WSGIRequestHandler(BaseHTTPRequestHandler): + server_version: str + def get_environ(self) -> WSGIEnvironment: ... + def get_stderr(self) -> ErrorStream: ... + +def demo_app(environ: WSGIEnvironment, start_response: StartResponse) -> list[bytes]: ... + +_S = TypeVar("_S", bound=WSGIServer) + +@overload +def make_server(host: str, port: int, app: WSGIApplication, *, handler_class: type[WSGIRequestHandler] = ...) -> WSGIServer: ... +@overload +def make_server( + host: str, port: int, app: WSGIApplication, server_class: type[_S], handler_class: type[WSGIRequestHandler] = ... +) -> _S: ... diff --git a/mypy/typeshed/stdlib/wsgiref/types.pyi b/mypy/typeshed/stdlib/wsgiref/types.pyi new file mode 100644 index 000000000000..57276fd05ea8 --- /dev/null +++ b/mypy/typeshed/stdlib/wsgiref/types.pyi @@ -0,0 +1,32 @@ +from _typeshed import OptExcInfo +from collections.abc import Callable, Iterable, Iterator +from typing import Any, Protocol +from typing_extensions import TypeAlias + +__all__ = ["StartResponse", "WSGIEnvironment", "WSGIApplication", "InputStream", "ErrorStream", "FileWrapper"] + +class StartResponse(Protocol): + def __call__( + self, status: str, headers: list[tuple[str, str]], exc_info: OptExcInfo | None = ..., / + ) -> Callable[[bytes], object]: ... + +WSGIEnvironment: TypeAlias = dict[str, Any] +WSGIApplication: TypeAlias = Callable[[WSGIEnvironment, StartResponse], Iterable[bytes]] + +class InputStream(Protocol): + def read(self, size: int = ..., /) -> bytes: ... + def readline(self, size: int = ..., /) -> bytes: ... + def readlines(self, hint: int = ..., /) -> list[bytes]: ... + def __iter__(self) -> Iterator[bytes]: ... + +class ErrorStream(Protocol): + def flush(self) -> object: ... + def write(self, s: str, /) -> object: ... + def writelines(self, seq: list[str], /) -> object: ... + +class _Readable(Protocol): + def read(self, size: int = ..., /) -> bytes: ... + # Optional: def close(self) -> object: ... + +class FileWrapper(Protocol): + def __call__(self, file: _Readable, block_size: int = ..., /) -> Iterable[bytes]: ... diff --git a/mypy/typeshed/stdlib/wsgiref/util.pyi b/mypy/typeshed/stdlib/wsgiref/util.pyi new file mode 100644 index 000000000000..3966e17b0d28 --- /dev/null +++ b/mypy/typeshed/stdlib/wsgiref/util.pyi @@ -0,0 +1,26 @@ +import sys +from _typeshed.wsgi import WSGIEnvironment +from collections.abc import Callable +from typing import IO, Any + +__all__ = ["FileWrapper", "guess_scheme", "application_uri", "request_uri", "shift_path_info", "setup_testing_defaults"] +if sys.version_info >= (3, 13): + __all__ += ["is_hop_by_hop"] + +class FileWrapper: + filelike: IO[bytes] + blksize: int + close: Callable[[], None] # only exists if filelike.close exists + def __init__(self, filelike: IO[bytes], blksize: int = 8192) -> None: ... + if sys.version_info < (3, 11): + def __getitem__(self, key: Any) -> bytes: ... + + def __iter__(self) -> FileWrapper: ... + def __next__(self) -> bytes: ... + +def guess_scheme(environ: WSGIEnvironment) -> str: ... +def application_uri(environ: WSGIEnvironment) -> str: ... +def request_uri(environ: WSGIEnvironment, include_query: bool = True) -> str: ... +def shift_path_info(environ: WSGIEnvironment) -> str | None: ... +def setup_testing_defaults(environ: WSGIEnvironment) -> None: ... +def is_hop_by_hop(header_name: str) -> bool: ... diff --git a/mypy/typeshed/stdlib/wsgiref/validate.pyi b/mypy/typeshed/stdlib/wsgiref/validate.pyi new file mode 100644 index 000000000000..fa8a6bbb8d03 --- /dev/null +++ b/mypy/typeshed/stdlib/wsgiref/validate.pyi @@ -0,0 +1,50 @@ +from _typeshed.wsgi import ErrorStream, InputStream, WSGIApplication +from collections.abc import Callable, Iterable, Iterator +from typing import Any, NoReturn +from typing_extensions import TypeAlias + +__all__ = ["validator"] + +class WSGIWarning(Warning): ... + +def validator(application: WSGIApplication) -> WSGIApplication: ... + +class InputWrapper: + input: InputStream + def __init__(self, wsgi_input: InputStream) -> None: ... + def read(self, size: int) -> bytes: ... + def readline(self, size: int = ...) -> bytes: ... + def readlines(self, hint: int = ...) -> bytes: ... + def __iter__(self) -> Iterator[bytes]: ... + def close(self) -> NoReturn: ... + +class ErrorWrapper: + errors: ErrorStream + def __init__(self, wsgi_errors: ErrorStream) -> None: ... + def write(self, s: str) -> None: ... + def flush(self) -> None: ... + def writelines(self, seq: Iterable[str]) -> None: ... + def close(self) -> NoReturn: ... + +_WriterCallback: TypeAlias = Callable[[bytes], Any] + +class WriteWrapper: + writer: _WriterCallback + def __init__(self, wsgi_writer: _WriterCallback) -> None: ... + def __call__(self, s: bytes) -> None: ... + +class PartialIteratorWrapper: + iterator: Iterator[bytes] + def __init__(self, wsgi_iterator: Iterator[bytes]) -> None: ... + def __iter__(self) -> IteratorWrapper: ... + +class IteratorWrapper: + original_iterator: Iterator[bytes] + iterator: Iterator[bytes] + closed: bool + check_start_response: bool | None + def __init__(self, wsgi_iterator: Iterator[bytes], check_start_response: bool | None) -> None: ... + def __iter__(self) -> IteratorWrapper: ... + def __next__(self) -> bytes: ... + def close(self) -> None: ... + def __del__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/xdrlib.pyi b/mypy/typeshed/stdlib/xdrlib.pyi new file mode 100644 index 000000000000..78f3ecec8d78 --- /dev/null +++ b/mypy/typeshed/stdlib/xdrlib.pyi @@ -0,0 +1,57 @@ +from collections.abc import Callable, Sequence +from typing import TypeVar + +__all__ = ["Error", "Packer", "Unpacker", "ConversionError"] + +_T = TypeVar("_T") + +class Error(Exception): + msg: str + def __init__(self, msg: str) -> None: ... + +class ConversionError(Error): ... + +class Packer: + def reset(self) -> None: ... + def get_buffer(self) -> bytes: ... + def get_buf(self) -> bytes: ... + def pack_uint(self, x: int) -> None: ... + def pack_int(self, x: int) -> None: ... + def pack_enum(self, x: int) -> None: ... + def pack_bool(self, x: bool) -> None: ... + def pack_uhyper(self, x: int) -> None: ... + def pack_hyper(self, x: int) -> None: ... + def pack_float(self, x: float) -> None: ... + def pack_double(self, x: float) -> None: ... + def pack_fstring(self, n: int, s: bytes) -> None: ... + def pack_fopaque(self, n: int, s: bytes) -> None: ... + def pack_string(self, s: bytes) -> None: ... + def pack_opaque(self, s: bytes) -> None: ... + def pack_bytes(self, s: bytes) -> None: ... + def pack_list(self, list: Sequence[_T], pack_item: Callable[[_T], object]) -> None: ... + def pack_farray(self, n: int, list: Sequence[_T], pack_item: Callable[[_T], object]) -> None: ... + def pack_array(self, list: Sequence[_T], pack_item: Callable[[_T], object]) -> None: ... + +class Unpacker: + def __init__(self, data: bytes) -> None: ... + def reset(self, data: bytes) -> None: ... + def get_position(self) -> int: ... + def set_position(self, position: int) -> None: ... + def get_buffer(self) -> bytes: ... + def done(self) -> None: ... + def unpack_uint(self) -> int: ... + def unpack_int(self) -> int: ... + def unpack_enum(self) -> int: ... + def unpack_bool(self) -> bool: ... + def unpack_uhyper(self) -> int: ... + def unpack_hyper(self) -> int: ... + def unpack_float(self) -> float: ... + def unpack_double(self) -> float: ... + def unpack_fstring(self, n: int) -> bytes: ... + def unpack_fopaque(self, n: int) -> bytes: ... + def unpack_string(self) -> bytes: ... + def unpack_opaque(self) -> bytes: ... + def unpack_bytes(self) -> bytes: ... + def unpack_list(self, unpack_item: Callable[[], _T]) -> list[_T]: ... + def unpack_farray(self, n: int, unpack_item: Callable[[], _T]) -> list[_T]: ... + def unpack_array(self, unpack_item: Callable[[], _T]) -> list[_T]: ... diff --git a/mypy/typeshed/stdlib/xml/__init__.pyi b/mypy/typeshed/stdlib/xml/__init__.pyi new file mode 100644 index 000000000000..7a240965136e --- /dev/null +++ b/mypy/typeshed/stdlib/xml/__init__.pyi @@ -0,0 +1,3 @@ +# At runtime, listing submodules in __all__ without them being imported is +# valid, and causes them to be included in a star import. See #6523 +__all__ = ["dom", "parsers", "sax", "etree"] # noqa: F822 # pyright: ignore[reportUnsupportedDunderAll] diff --git a/mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi b/mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi new file mode 100644 index 000000000000..80fb73d23433 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi @@ -0,0 +1,19 @@ +class NodeFilter: + FILTER_ACCEPT: int + FILTER_REJECT: int + FILTER_SKIP: int + + SHOW_ALL: int + SHOW_ELEMENT: int + SHOW_ATTRIBUTE: int + SHOW_TEXT: int + SHOW_CDATA_SECTION: int + SHOW_ENTITY_REFERENCE: int + SHOW_ENTITY: int + SHOW_PROCESSING_INSTRUCTION: int + SHOW_COMMENT: int + SHOW_DOCUMENT: int + SHOW_DOCUMENT_TYPE: int + SHOW_DOCUMENT_FRAGMENT: int + SHOW_NOTATION: int + def acceptNode(self, node) -> int: ... diff --git a/mypy/typeshed/stdlib/xml/dom/__init__.pyi b/mypy/typeshed/stdlib/xml/dom/__init__.pyi new file mode 100644 index 000000000000..8738015638a9 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/__init__.pyi @@ -0,0 +1,69 @@ +from typing import Any, Final + +from .domreg import getDOMImplementation as getDOMImplementation, registerDOMImplementation as registerDOMImplementation + +class Node: + ELEMENT_NODE: int + ATTRIBUTE_NODE: int + TEXT_NODE: int + CDATA_SECTION_NODE: int + ENTITY_REFERENCE_NODE: int + ENTITY_NODE: int + PROCESSING_INSTRUCTION_NODE: int + COMMENT_NODE: int + DOCUMENT_NODE: int + DOCUMENT_TYPE_NODE: int + DOCUMENT_FRAGMENT_NODE: int + NOTATION_NODE: int + +# ExceptionCode +INDEX_SIZE_ERR: Final[int] +DOMSTRING_SIZE_ERR: Final[int] +HIERARCHY_REQUEST_ERR: Final[int] +WRONG_DOCUMENT_ERR: Final[int] +INVALID_CHARACTER_ERR: Final[int] +NO_DATA_ALLOWED_ERR: Final[int] +NO_MODIFICATION_ALLOWED_ERR: Final[int] +NOT_FOUND_ERR: Final[int] +NOT_SUPPORTED_ERR: Final[int] +INUSE_ATTRIBUTE_ERR: Final[int] +INVALID_STATE_ERR: Final[int] +SYNTAX_ERR: Final[int] +INVALID_MODIFICATION_ERR: Final[int] +NAMESPACE_ERR: Final[int] +INVALID_ACCESS_ERR: Final[int] +VALIDATION_ERR: Final[int] + +class DOMException(Exception): + code: int + def __init__(self, *args: Any, **kw: Any) -> None: ... + def _get_code(self) -> int: ... + +class IndexSizeErr(DOMException): ... +class DomstringSizeErr(DOMException): ... +class HierarchyRequestErr(DOMException): ... +class WrongDocumentErr(DOMException): ... +class InvalidCharacterErr(DOMException): ... +class NoDataAllowedErr(DOMException): ... +class NoModificationAllowedErr(DOMException): ... +class NotFoundErr(DOMException): ... +class NotSupportedErr(DOMException): ... +class InuseAttributeErr(DOMException): ... +class InvalidStateErr(DOMException): ... +class SyntaxErr(DOMException): ... +class InvalidModificationErr(DOMException): ... +class NamespaceErr(DOMException): ... +class InvalidAccessErr(DOMException): ... +class ValidationErr(DOMException): ... + +class UserDataHandler: + NODE_CLONED: int + NODE_IMPORTED: int + NODE_DELETED: int + NODE_RENAMED: int + +XML_NAMESPACE: Final[str] +XMLNS_NAMESPACE: Final[str] +XHTML_NAMESPACE: Final[str] +EMPTY_NAMESPACE: Final[None] +EMPTY_PREFIX: Final[None] diff --git a/mypy/typeshed/stdlib/xml/dom/domreg.pyi b/mypy/typeshed/stdlib/xml/dom/domreg.pyi new file mode 100644 index 000000000000..346a4bf63bd4 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/domreg.pyi @@ -0,0 +1,8 @@ +from _typeshed.xml import DOMImplementation +from collections.abc import Callable, Iterable + +well_known_implementations: dict[str, str] +registered: dict[str, Callable[[], DOMImplementation]] + +def registerDOMImplementation(name: str, factory: Callable[[], DOMImplementation]) -> None: ... +def getDOMImplementation(name: str | None = None, features: str | Iterable[tuple[str, str | None]] = ()) -> DOMImplementation: ... diff --git a/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi b/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi new file mode 100644 index 000000000000..45f0af7aa979 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi @@ -0,0 +1,100 @@ +from _typeshed import Incomplete, ReadableBuffer, SupportsRead +from typing import Any, NoReturn +from xml.dom.minidom import Document, DOMImplementation, Node, TypeInfo +from xml.dom.xmlbuilder import DOMBuilderFilter, Options + +TEXT_NODE = Node.TEXT_NODE +CDATA_SECTION_NODE = Node.CDATA_SECTION_NODE +DOCUMENT_NODE = Node.DOCUMENT_NODE +FILTER_ACCEPT = DOMBuilderFilter.FILTER_ACCEPT +FILTER_REJECT = DOMBuilderFilter.FILTER_REJECT +FILTER_SKIP = DOMBuilderFilter.FILTER_SKIP +FILTER_INTERRUPT = DOMBuilderFilter.FILTER_INTERRUPT +theDOMImplementation: DOMImplementation | None + +class ElementInfo: + tagName: Incomplete + def __init__(self, tagName, model: Incomplete | None = None) -> None: ... + def getAttributeType(self, aname) -> TypeInfo: ... + def getAttributeTypeNS(self, namespaceURI, localName) -> TypeInfo: ... + def isElementContent(self) -> bool: ... + def isEmpty(self) -> bool: ... + def isId(self, aname) -> bool: ... + def isIdNS(self, euri, ename, auri, aname) -> bool: ... + +class ExpatBuilder: + document: Document # Created in self.reset() + curNode: Incomplete # Created in self.reset() + def __init__(self, options: Options | None = None) -> None: ... + def createParser(self): ... + def getParser(self): ... + def reset(self) -> None: ... + def install(self, parser) -> None: ... + def parseFile(self, file: SupportsRead[ReadableBuffer | str]) -> Document: ... + def parseString(self, string: str | ReadableBuffer) -> Document: ... + def start_doctype_decl_handler(self, doctypeName, systemId, publicId, has_internal_subset) -> None: ... + def end_doctype_decl_handler(self) -> None: ... + def pi_handler(self, target, data) -> None: ... + def character_data_handler_cdata(self, data) -> None: ... + def character_data_handler(self, data) -> None: ... + def start_cdata_section_handler(self) -> None: ... + def end_cdata_section_handler(self) -> None: ... + def entity_decl_handler(self, entityName, is_parameter_entity, value, base, systemId, publicId, notationName) -> None: ... + def notation_decl_handler(self, notationName, base, systemId, publicId) -> None: ... + def comment_handler(self, data) -> None: ... + def external_entity_ref_handler(self, context, base, systemId, publicId) -> int: ... + def first_element_handler(self, name, attributes) -> None: ... + def start_element_handler(self, name, attributes) -> None: ... + def end_element_handler(self, name) -> None: ... + def element_decl_handler(self, name, model) -> None: ... + def attlist_decl_handler(self, elem, name, type, default, required) -> None: ... + def xml_decl_handler(self, version, encoding, standalone) -> None: ... + +class FilterVisibilityController: + filter: DOMBuilderFilter + def __init__(self, filter: DOMBuilderFilter) -> None: ... + def startContainer(self, node: Node) -> int: ... + def acceptNode(self, node: Node) -> int: ... + +class FilterCrutch: + def __init__(self, builder) -> None: ... + +class Rejecter(FilterCrutch): + def start_element_handler(self, *args: Any) -> None: ... + def end_element_handler(self, *args: Any) -> None: ... + +class Skipper(FilterCrutch): + def start_element_handler(self, *args: Any) -> None: ... + def end_element_handler(self, *args: Any) -> None: ... + +class FragmentBuilder(ExpatBuilder): + fragment: Incomplete | None + originalDocument: Incomplete + context: Incomplete + def __init__(self, context, options: Options | None = None) -> None: ... + +class Namespaces: + def createParser(self): ... + def install(self, parser) -> None: ... + def start_namespace_decl_handler(self, prefix, uri) -> None: ... + def start_element_handler(self, name, attributes) -> None: ... + def end_element_handler(self, name) -> None: ... + +class ExpatBuilderNS(Namespaces, ExpatBuilder): ... +class FragmentBuilderNS(Namespaces, FragmentBuilder): ... +class ParseEscape(Exception): ... + +class InternalSubsetExtractor(ExpatBuilder): + subset: Any | None + def getSubset(self) -> Any | None: ... + def parseFile(self, file: SupportsRead[ReadableBuffer | str]) -> None: ... # type: ignore[override] + def parseString(self, string: str | ReadableBuffer) -> None: ... # type: ignore[override] + def start_doctype_decl_handler(self, name, publicId, systemId, has_internal_subset) -> None: ... # type: ignore[override] + def end_doctype_decl_handler(self) -> NoReturn: ... + def start_element_handler(self, name, attrs) -> NoReturn: ... + +def parse(file: str | SupportsRead[ReadableBuffer | str], namespaces: bool = True): ... +def parseString(string: str | ReadableBuffer, namespaces: bool = True): ... +def parseFragment(file, context, namespaces: bool = True): ... +def parseFragmentString(string: str, context, namespaces: bool = True): ... +def makeBuilder(options: Options) -> ExpatBuilderNS | ExpatBuilder: ... diff --git a/mypy/typeshed/stdlib/xml/dom/minicompat.pyi b/mypy/typeshed/stdlib/xml/dom/minicompat.pyi new file mode 100644 index 000000000000..162f60254a58 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/minicompat.pyi @@ -0,0 +1,22 @@ +from collections.abc import Iterable +from typing import Any, Literal, TypeVar + +__all__ = ["NodeList", "EmptyNodeList", "StringTypes", "defproperty"] + +_T = TypeVar("_T") + +StringTypes: tuple[type[str]] + +class NodeList(list[_T]): + @property + def length(self) -> int: ... + def item(self, index: int) -> _T | None: ... + +class EmptyNodeList(tuple[()]): + @property + def length(self) -> Literal[0]: ... + def item(self, index: int) -> None: ... + def __add__(self, other: Iterable[_T]) -> NodeList[_T]: ... # type: ignore[override] + def __radd__(self, other: Iterable[_T]) -> NodeList[_T]: ... + +def defproperty(klass: type[Any], name: str, doc: str) -> None: ... diff --git a/mypy/typeshed/stdlib/xml/dom/minidom.pyi b/mypy/typeshed/stdlib/xml/dom/minidom.pyi new file mode 100644 index 000000000000..fae2c4d98714 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/minidom.pyi @@ -0,0 +1,404 @@ +import sys +import xml.dom +from _typeshed import Incomplete, ReadableBuffer, SupportsRead, SupportsWrite +from typing import Literal, NoReturn, TypeVar, overload +from typing_extensions import Self +from xml.dom.minicompat import NodeList +from xml.dom.xmlbuilder import DocumentLS, DOMImplementationLS +from xml.sax.xmlreader import XMLReader + +_N = TypeVar("_N", bound=Node) + +def parse( + file: str | SupportsRead[ReadableBuffer | str], parser: XMLReader | None = None, bufsize: int | None = None +) -> Document: ... +def parseString(string: str | ReadableBuffer, parser: XMLReader | None = None) -> Document: ... +def getDOMImplementation(features=None) -> DOMImplementation | None: ... + +class Node(xml.dom.Node): + namespaceURI: str | None + parentNode: Incomplete + ownerDocument: Incomplete + nextSibling: Incomplete + previousSibling: Incomplete + prefix: Incomplete + @property + def firstChild(self) -> Node | None: ... + @property + def lastChild(self) -> Node | None: ... + @property + def localName(self) -> str | None: ... + def __bool__(self) -> Literal[True]: ... + if sys.version_info >= (3, 9): + @overload + def toxml(self, encoding: str, standalone: bool | None = None) -> bytes: ... + @overload + def toxml(self, encoding: None = None, standalone: bool | None = None) -> str: ... + @overload + def toprettyxml( + self, + indent: str = "\t", + newl: str = "\n", + # Handle any case where encoding is not provided or where it is passed with None + encoding: None = None, + standalone: bool | None = None, + ) -> str: ... + @overload + def toprettyxml( + self, + indent: str, + newl: str, + # Handle cases where encoding is passed as str *positionally* + encoding: str, + standalone: bool | None = None, + ) -> bytes: ... + @overload + def toprettyxml( + self, + indent: str = "\t", + newl: str = "\n", + # Handle all cases where encoding is passed as a keyword argument; because standalone + # comes after, it will also have to be a keyword arg if encoding is + *, + encoding: str, + standalone: bool | None = None, + ) -> bytes: ... + else: + @overload + def toxml(self, encoding: str) -> bytes: ... + @overload + def toxml(self, encoding: None = None) -> str: ... + @overload + def toprettyxml( + self, + indent: str = "\t", + newl: str = "\n", + # Handle any case where encoding is not provided or where it is passed with None + encoding: None = None, + ) -> str: ... + @overload + def toprettyxml( + self, + indent: str, + newl: str, + # Handle cases where encoding is passed as str *positionally* + encoding: str, + ) -> bytes: ... + @overload + def toprettyxml( + self, + indent: str = "\t", + newl: str = "\n", + # Handle all cases where encoding is passed as a keyword argument + *, + encoding: str, + ) -> bytes: ... + + def hasChildNodes(self) -> bool: ... + def insertBefore(self, newChild, refChild): ... + def appendChild(self, node: _N) -> _N: ... + def replaceChild(self, newChild, oldChild): ... + def removeChild(self, oldChild): ... + def normalize(self) -> None: ... + def cloneNode(self, deep): ... + def isSupported(self, feature, version): ... + def isSameNode(self, other): ... + def getInterface(self, feature): ... + def getUserData(self, key): ... + def setUserData(self, key, data, handler): ... + childNodes: Incomplete + def unlink(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, et, ev, tb) -> None: ... + +class DocumentFragment(Node): + nodeType: int + nodeName: str + nodeValue: Incomplete + attributes: Incomplete + parentNode: Incomplete + childNodes: Incomplete + def __init__(self) -> None: ... + +class Attr(Node): + name: str + nodeType: int + attributes: Incomplete + specified: bool + ownerElement: Incomplete + namespaceURI: str | None + childNodes: Incomplete + nodeName: Incomplete + nodeValue: str + value: str + prefix: Incomplete + def __init__( + self, qName: str, namespaceURI: str | None = None, localName: str | None = None, prefix: Incomplete | None = None + ) -> None: ... + def unlink(self) -> None: ... + @property + def isId(self) -> bool: ... + @property + def schemaType(self): ... + +class NamedNodeMap: + def __init__(self, attrs, attrsNS, ownerElement) -> None: ... + def item(self, index): ... + def items(self): ... + def itemsNS(self): ... + def __contains__(self, key): ... + def keys(self): ... + def keysNS(self): ... + def values(self): ... + def get(self, name: str, value: Incomplete | None = None): ... + def __len__(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: NamedNodeMap) -> bool: ... + def __gt__(self, other: NamedNodeMap) -> bool: ... + def __le__(self, other: NamedNodeMap) -> bool: ... + def __lt__(self, other: NamedNodeMap) -> bool: ... + def __getitem__(self, attname_or_tuple: tuple[str, str | None] | str): ... + def __setitem__(self, attname: str, value: Attr | str) -> None: ... + def getNamedItem(self, name: str) -> Attr | None: ... + def getNamedItemNS(self, namespaceURI: str, localName: str | None) -> Attr | None: ... + def removeNamedItem(self, name: str) -> Attr: ... + def removeNamedItemNS(self, namespaceURI: str, localName: str | None): ... + def setNamedItem(self, node: Attr) -> Attr: ... + def setNamedItemNS(self, node: Attr) -> Attr: ... + def __delitem__(self, attname_or_tuple: tuple[str, str | None] | str) -> None: ... + @property + def length(self) -> int: ... + +AttributeList = NamedNodeMap + +class TypeInfo: + namespace: Incomplete | None + name: str + def __init__(self, namespace: Incomplete | None, name: str) -> None: ... + +class Element(Node): + nodeType: int + nodeValue: Incomplete + schemaType: Incomplete + parentNode: Incomplete + tagName: str + nodeName: str + prefix: Incomplete + namespaceURI: str | None + childNodes: Incomplete + nextSibling: Incomplete + def __init__( + self, tagName, namespaceURI: str | None = None, prefix: Incomplete | None = None, localName: Incomplete | None = None + ) -> None: ... + def unlink(self) -> None: ... + def getAttribute(self, attname: str) -> str: ... + def getAttributeNS(self, namespaceURI: str, localName): ... + def setAttribute(self, attname: str, value: str) -> None: ... + def setAttributeNS(self, namespaceURI: str, qualifiedName: str, value) -> None: ... + def getAttributeNode(self, attrname: str): ... + def getAttributeNodeNS(self, namespaceURI: str, localName): ... + def setAttributeNode(self, attr): ... + setAttributeNodeNS: Incomplete + def removeAttribute(self, name: str) -> None: ... + def removeAttributeNS(self, namespaceURI: str, localName) -> None: ... + def removeAttributeNode(self, node): ... + removeAttributeNodeNS: Incomplete + def hasAttribute(self, name: str) -> bool: ... + def hasAttributeNS(self, namespaceURI: str, localName) -> bool: ... + def getElementsByTagName(self, name: str) -> NodeList[Element]: ... + def getElementsByTagNameNS(self, namespaceURI: str, localName: str) -> NodeList[Element]: ... + def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... + def hasAttributes(self) -> bool: ... + def setIdAttribute(self, name) -> None: ... + def setIdAttributeNS(self, namespaceURI: str, localName) -> None: ... + def setIdAttributeNode(self, idAttr) -> None: ... + @property + def attributes(self) -> NamedNodeMap: ... + +class Childless: + attributes: Incomplete + childNodes: Incomplete + firstChild: Incomplete + lastChild: Incomplete + def appendChild(self, node) -> NoReturn: ... + def hasChildNodes(self) -> bool: ... + def insertBefore(self, newChild, refChild) -> NoReturn: ... + def removeChild(self, oldChild) -> NoReturn: ... + def normalize(self) -> None: ... + def replaceChild(self, newChild, oldChild) -> NoReturn: ... + +class ProcessingInstruction(Childless, Node): + nodeType: int + target: Incomplete + data: Incomplete + def __init__(self, target, data) -> None: ... + nodeValue: Incomplete + nodeName: Incomplete + def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... + +class CharacterData(Childless, Node): + ownerDocument: Incomplete + previousSibling: Incomplete + def __init__(self) -> None: ... + def __len__(self) -> int: ... + data: str + nodeValue: Incomplete + def substringData(self, offset: int, count: int) -> str: ... + def appendData(self, arg: str) -> None: ... + def insertData(self, offset: int, arg: str) -> None: ... + def deleteData(self, offset: int, count: int) -> None: ... + def replaceData(self, offset: int, count: int, arg: str) -> None: ... + @property + def length(self) -> int: ... + +class Text(CharacterData): + nodeType: int + nodeName: str + attributes: Incomplete + data: Incomplete + def splitText(self, offset: int) -> Self: ... + def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... + def replaceWholeText(self, content) -> Self | None: ... + @property + def isWhitespaceInElementContent(self) -> bool: ... + @property + def wholeText(self) -> str: ... + +class Comment(CharacterData): + nodeType: int + nodeName: str + def __init__(self, data) -> None: ... + def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... + +class CDATASection(Text): + nodeType: int + nodeName: str + def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... + +class ReadOnlySequentialNamedNodeMap: + def __init__(self, seq=()) -> None: ... + def __len__(self) -> int: ... + def getNamedItem(self, name): ... + def getNamedItemNS(self, namespaceURI: str, localName): ... + def __getitem__(self, name_or_tuple): ... + def item(self, index): ... + def removeNamedItem(self, name) -> None: ... + def removeNamedItemNS(self, namespaceURI: str, localName) -> None: ... + def setNamedItem(self, node) -> None: ... + def setNamedItemNS(self, node) -> None: ... + @property + def length(self) -> int: ... + +class Identified: + publicId: Incomplete + systemId: Incomplete + +class DocumentType(Identified, Childless, Node): + nodeType: int + nodeValue: Incomplete + name: Incomplete + internalSubset: Incomplete + entities: Incomplete + notations: Incomplete + nodeName: Incomplete + def __init__(self, qualifiedName: str) -> None: ... + def cloneNode(self, deep): ... + def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... + +class Entity(Identified, Node): + attributes: Incomplete + nodeType: int + nodeValue: Incomplete + actualEncoding: Incomplete + encoding: Incomplete + version: Incomplete + nodeName: Incomplete + notationName: Incomplete + childNodes: Incomplete + def __init__(self, name, publicId, systemId, notation) -> None: ... + def appendChild(self, newChild) -> NoReturn: ... + def insertBefore(self, newChild, refChild) -> NoReturn: ... + def removeChild(self, oldChild) -> NoReturn: ... + def replaceChild(self, newChild, oldChild) -> NoReturn: ... + +class Notation(Identified, Childless, Node): + nodeType: int + nodeValue: Incomplete + nodeName: Incomplete + def __init__(self, name, publicId, systemId) -> None: ... + +class DOMImplementation(DOMImplementationLS): + def hasFeature(self, feature: str, version: str | None) -> bool: ... + def createDocument(self, namespaceURI: str | None, qualifiedName: str | None, doctype: DocumentType | None) -> Document: ... + def createDocumentType(self, qualifiedName: str | None, publicId: str, systemId: str) -> DocumentType: ... + def getInterface(self, feature: str) -> Self | None: ... + +class ElementInfo: + tagName: Incomplete + def __init__(self, name) -> None: ... + def getAttributeType(self, aname): ... + def getAttributeTypeNS(self, namespaceURI: str, localName): ... + def isElementContent(self): ... + def isEmpty(self): ... + def isId(self, aname): ... + def isIdNS(self, namespaceURI: str, localName): ... + +class Document(Node, DocumentLS): + implementation: Incomplete + nodeType: int + nodeName: str + nodeValue: Incomplete + attributes: Incomplete + parentNode: Incomplete + previousSibling: Incomplete + nextSibling: Incomplete + actualEncoding: Incomplete + encoding: str | None + standalone: bool | None + version: Incomplete + strictErrorChecking: bool + errorHandler: Incomplete + documentURI: Incomplete + doctype: DocumentType | None + childNodes: Incomplete + def __init__(self) -> None: ... + def appendChild(self, node: _N) -> _N: ... + documentElement: Incomplete + def removeChild(self, oldChild): ... + def unlink(self) -> None: ... + def cloneNode(self, deep): ... + def createDocumentFragment(self) -> DocumentFragment: ... + def createElement(self, tagName: str) -> Element: ... + def createTextNode(self, data: str) -> Text: ... + def createCDATASection(self, data: str) -> CDATASection: ... + def createComment(self, data: str) -> Comment: ... + def createProcessingInstruction(self, target, data): ... + def createAttribute(self, qName) -> Attr: ... + def createElementNS(self, namespaceURI: str, qualifiedName: str): ... + def createAttributeNS(self, namespaceURI: str, qualifiedName: str) -> Attr: ... + def getElementById(self, id: str) -> Element | None: ... + def getElementsByTagName(self, name: str) -> NodeList[Element]: ... + def getElementsByTagNameNS(self, namespaceURI: str, localName: str) -> NodeList[Element]: ... + def isSupported(self, feature: str, version: str | None) -> bool: ... + def importNode(self, node, deep): ... + if sys.version_info >= (3, 9): + def writexml( + self, + writer: SupportsWrite[str], + indent: str = "", + addindent: str = "", + newl: str = "", + encoding: str | None = None, + standalone: bool | None = None, + ) -> None: ... + else: + def writexml( + self, + writer: SupportsWrite[str], + indent: str = "", + addindent: str = "", + newl: str = "", + encoding: Incomplete | None = None, + ) -> None: ... + + def renameNode(self, n, namespaceURI: str, name): ... diff --git a/mypy/typeshed/stdlib/xml/dom/pulldom.pyi b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi new file mode 100644 index 000000000000..50250de5cb2f --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi @@ -0,0 +1,94 @@ +import sys +from _typeshed import Incomplete, SupportsRead +from collections.abc import Sequence +from typing import Final, Literal +from typing_extensions import TypeAlias +from xml.dom.minidom import Document, DOMImplementation, Element, Text +from xml.sax.handler import ContentHandler +from xml.sax.xmlreader import XMLReader + +START_ELEMENT: Final = "START_ELEMENT" +END_ELEMENT: Final = "END_ELEMENT" +COMMENT: Final = "COMMENT" +START_DOCUMENT: Final = "START_DOCUMENT" +END_DOCUMENT: Final = "END_DOCUMENT" +PROCESSING_INSTRUCTION: Final = "PROCESSING_INSTRUCTION" +IGNORABLE_WHITESPACE: Final = "IGNORABLE_WHITESPACE" +CHARACTERS: Final = "CHARACTERS" + +_DocumentFactory: TypeAlias = DOMImplementation | None +_Node: TypeAlias = Document | Element | Text + +_Event: TypeAlias = tuple[ + Literal[ + Literal["START_ELEMENT"], + Literal["END_ELEMENT"], + Literal["COMMENT"], + Literal["START_DOCUMENT"], + Literal["END_DOCUMENT"], + Literal["PROCESSING_INSTRUCTION"], + Literal["IGNORABLE_WHITESPACE"], + Literal["CHARACTERS"], + ], + _Node, +] + +class PullDOM(ContentHandler): + document: Document | None + documentFactory: _DocumentFactory + firstEvent: Incomplete + lastEvent: Incomplete + elementStack: Sequence[Incomplete] + pending_events: Sequence[Incomplete] + def __init__(self, documentFactory: _DocumentFactory = None) -> None: ... + def pop(self) -> Element: ... + def setDocumentLocator(self, locator) -> None: ... + def startPrefixMapping(self, prefix, uri) -> None: ... + def endPrefixMapping(self, prefix) -> None: ... + def startElementNS(self, name, tagName, attrs) -> None: ... + def endElementNS(self, name, tagName) -> None: ... + def startElement(self, name, attrs) -> None: ... + def endElement(self, name) -> None: ... + def comment(self, s) -> None: ... + def processingInstruction(self, target, data) -> None: ... + def ignorableWhitespace(self, chars) -> None: ... + def characters(self, chars) -> None: ... + def startDocument(self) -> None: ... + def buildDocument(self, uri, tagname): ... + def endDocument(self) -> None: ... + def clear(self) -> None: ... + +class ErrorHandler: + def warning(self, exception) -> None: ... + def error(self, exception) -> None: ... + def fatalError(self, exception) -> None: ... + +class DOMEventStream: + stream: SupportsRead[bytes] | SupportsRead[str] + parser: XMLReader + bufsize: int + def __init__(self, stream: SupportsRead[bytes] | SupportsRead[str], parser: XMLReader, bufsize: int) -> None: ... + pulldom: Incomplete + if sys.version_info < (3, 11): + def __getitem__(self, pos): ... + + def __next__(self): ... + def __iter__(self): ... + def getEvent(self) -> _Event: ... + def expandNode(self, node: _Node) -> None: ... + def reset(self) -> None: ... + def clear(self) -> None: ... + +class SAX2DOM(PullDOM): + def startElementNS(self, name, tagName, attrs) -> None: ... + def startElement(self, name, attrs) -> None: ... + def processingInstruction(self, target, data) -> None: ... + def ignorableWhitespace(self, chars) -> None: ... + def characters(self, chars) -> None: ... + +default_bufsize: int + +def parse( + stream_or_string: str | SupportsRead[bytes] | SupportsRead[str], parser: XMLReader | None = None, bufsize: int | None = None +) -> DOMEventStream: ... +def parseString(string: str, parser: XMLReader | None = None) -> DOMEventStream: ... diff --git a/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi b/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi new file mode 100644 index 000000000000..ab76d362e23f --- /dev/null +++ b/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi @@ -0,0 +1,108 @@ +from _typeshed import Incomplete, Unused +from typing import Any, Literal, NoReturn +from typing_extensions import TypeAlias +from urllib.request import OpenerDirector +from xml.dom.expatbuilder import ExpatBuilder, ExpatBuilderNS +from xml.dom.minidom import Node + +__all__ = ["DOMBuilder", "DOMEntityResolver", "DOMInputSource"] + +# UNKNOWN TYPES: +# - `Options.errorHandler`. +# The same as `_DOMBuilderErrorHandlerType`? +# Maybe `xml.sax.handler.ErrorHandler`? +# - Return type of DOMBuilder.getFeature(). +# We could get rid of the `Incomplete` if we knew more +# about `Options.errorHandler`. + +# ALIASES REPRESENTING MORE UNKNOWN TYPES: + +# probably the same as `Options.errorHandler`? +# Maybe `xml.sax.handler.ErrorHandler`? +_DOMBuilderErrorHandlerType: TypeAlias = Incomplete | None +# probably some kind of IO... +_DOMInputSourceCharacterStreamType: TypeAlias = Incomplete | None +# probably a string?? +_DOMInputSourceStringDataType: TypeAlias = Incomplete | None +# probably a string?? +_DOMInputSourceEncodingType: TypeAlias = Incomplete | None + +class Options: + namespaces: int + namespace_declarations: bool + validation: bool + external_parameter_entities: bool + external_general_entities: bool + external_dtd_subset: bool + validate_if_schema: bool + validate: bool + datatype_normalization: bool + create_entity_ref_nodes: bool + entities: bool + whitespace_in_element_content: bool + cdata_sections: bool + comments: bool + charset_overrides_xml_encoding: bool + infoset: bool + supported_mediatypes_only: bool + errorHandler: Any | None + filter: DOMBuilderFilter | None # a guess, but seems likely + +class DOMBuilder: + entityResolver: DOMEntityResolver | None # a guess, but seems likely + errorHandler: _DOMBuilderErrorHandlerType + filter: DOMBuilderFilter | None # a guess, but seems likely + ACTION_REPLACE: Literal[1] + ACTION_APPEND_AS_CHILDREN: Literal[2] + ACTION_INSERT_AFTER: Literal[3] + ACTION_INSERT_BEFORE: Literal[4] + def setFeature(self, name: str, state: int) -> None: ... + def supportsFeature(self, name: str) -> bool: ... + def canSetFeature(self, name: str, state: int) -> bool: ... + # getFeature could return any attribute from an instance of `Options` + def getFeature(self, name: str) -> Any: ... + def parseURI(self, uri: str) -> ExpatBuilder | ExpatBuilderNS: ... + def parse(self, input: DOMInputSource) -> ExpatBuilder | ExpatBuilderNS: ... + # `input` and `cnode` argtypes for `parseWithContext` are unknowable + # as the function does nothing with them, and always raises an exception. + # But `input` is *probably* `DOMInputSource`? + def parseWithContext(self, input: Unused, cnode: Unused, action: Literal[1, 2, 3, 4]) -> NoReturn: ... + +class DOMEntityResolver: + def resolveEntity(self, publicId: str | None, systemId: str) -> DOMInputSource: ... + +class DOMInputSource: + byteStream: OpenerDirector | None + characterStream: _DOMInputSourceCharacterStreamType + stringData: _DOMInputSourceStringDataType + encoding: _DOMInputSourceEncodingType + publicId: str | None + systemId: str | None + baseURI: str | None + +class DOMBuilderFilter: + FILTER_ACCEPT: Literal[1] + FILTER_REJECT: Literal[2] + FILTER_SKIP: Literal[3] + FILTER_INTERRUPT: Literal[4] + whatToShow: int + def acceptNode(self, element: Unused) -> Literal[1]: ... + def startContainer(self, element: Unused) -> Literal[1]: ... + +class DocumentLS: + async_: bool + def abort(self) -> NoReturn: ... + # `load()` and `loadXML()` always raise exceptions + # so the argtypes of `uri` and `source` are unknowable. + # `source` is *probably* `DOMInputSource`? + # `uri` is *probably* a str? (see DOMBuilder.parseURI()) + def load(self, uri: Unused) -> NoReturn: ... + def loadXML(self, source: Unused) -> NoReturn: ... + def saveXML(self, snode: Node | None) -> str: ... + +class DOMImplementationLS: + MODE_SYNCHRONOUS: Literal[1] + MODE_ASYNCHRONOUS: Literal[2] + def createDOMBuilder(self, mode: Literal[1], schemaType: None) -> DOMBuilder: ... + def createDOMWriter(self) -> NoReturn: ... + def createDOMInputSource(self) -> DOMInputSource: ... diff --git a/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi new file mode 100644 index 000000000000..5a15772ec2a9 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi @@ -0,0 +1,29 @@ +import sys +from _typeshed import FileDescriptorOrPath +from collections.abc import Callable +from typing import Final +from xml.etree.ElementTree import Element + +XINCLUDE: Final[str] +XINCLUDE_INCLUDE: Final[str] +XINCLUDE_FALLBACK: Final[str] + +if sys.version_info >= (3, 9): + DEFAULT_MAX_INCLUSION_DEPTH: Final = 6 + +class FatalIncludeError(SyntaxError): ... + +def default_loader(href: FileDescriptorOrPath, parse: str, encoding: str | None = None) -> str | Element: ... + +# TODO: loader is of type default_loader ie it takes a callable that has the +# same signature as default_loader. But default_loader has a keyword argument +# Which can't be represented using Callable... +if sys.version_info >= (3, 9): + def include( + elem: Element, loader: Callable[..., str | Element] | None = None, base_url: str | None = None, max_depth: int | None = 6 + ) -> None: ... + + class LimitedRecursiveIncludeError(FatalIncludeError): ... + +else: + def include(elem: Element, loader: Callable[..., str | Element] | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/xml/etree/ElementPath.pyi b/mypy/typeshed/stdlib/xml/etree/ElementPath.pyi new file mode 100644 index 000000000000..c3f6207ea241 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/etree/ElementPath.pyi @@ -0,0 +1,34 @@ +from collections.abc import Callable, Generator +from re import Pattern +from typing import TypeVar +from typing_extensions import TypeAlias +from xml.etree.ElementTree import Element + +xpath_tokenizer_re: Pattern[str] + +_Token: TypeAlias = tuple[str, str] +_Next: TypeAlias = Callable[[], _Token] +_Callback: TypeAlias = Callable[[_SelectorContext, list[Element]], Generator[Element, None, None]] + +def xpath_tokenizer(pattern: str, namespaces: dict[str, str] | None = None) -> Generator[_Token, None, None]: ... +def get_parent_map(context: _SelectorContext) -> dict[Element, Element]: ... +def prepare_child(next: _Next, token: _Token) -> _Callback: ... +def prepare_star(next: _Next, token: _Token) -> _Callback: ... +def prepare_self(next: _Next, token: _Token) -> _Callback: ... +def prepare_descendant(next: _Next, token: _Token) -> _Callback: ... +def prepare_parent(next: _Next, token: _Token) -> _Callback: ... +def prepare_predicate(next: _Next, token: _Token) -> _Callback: ... + +ops: dict[str, Callable[[_Next, _Token], _Callback]] + +class _SelectorContext: + parent_map: dict[Element, Element] | None + root: Element + def __init__(self, root: Element) -> None: ... + +_T = TypeVar("_T") + +def iterfind(elem: Element, path: str, namespaces: dict[str, str] | None = None) -> Generator[Element, None, None]: ... +def find(elem: Element, path: str, namespaces: dict[str, str] | None = None) -> Element | None: ... +def findall(elem: Element, path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ... +def findtext(elem: Element, path: str, default: _T | None = None, namespaces: dict[str, str] | None = None) -> _T | str: ... diff --git a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi new file mode 100644 index 000000000000..64ebbd3ee63f --- /dev/null +++ b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -0,0 +1,333 @@ +import sys +from _collections_abc import dict_keys +from _typeshed import FileDescriptorOrPath, ReadableBuffer, SupportsRead, SupportsWrite +from collections.abc import Callable, Generator, ItemsView, Iterable, Iterator, Mapping, Sequence +from typing import Any, Final, Literal, SupportsIndex, TypeVar, overload +from typing_extensions import TypeAlias, TypeGuard, deprecated + +__all__ = [ + "C14NWriterTarget", + "Comment", + "dump", + "Element", + "ElementTree", + "canonicalize", + "fromstring", + "fromstringlist", + "iselement", + "iterparse", + "parse", + "ParseError", + "PI", + "ProcessingInstruction", + "QName", + "SubElement", + "tostring", + "tostringlist", + "TreeBuilder", + "VERSION", + "XML", + "XMLID", + "XMLParser", + "XMLPullParser", + "register_namespace", +] + +if sys.version_info >= (3, 9): + __all__ += ["indent"] + +_T = TypeVar("_T") +_FileRead: TypeAlias = FileDescriptorOrPath | SupportsRead[bytes] | SupportsRead[str] +_FileWriteC14N: TypeAlias = FileDescriptorOrPath | SupportsWrite[bytes] +_FileWrite: TypeAlias = _FileWriteC14N | SupportsWrite[str] + +VERSION: Final[str] + +class ParseError(SyntaxError): + code: int + position: tuple[int, int] + +# In reality it works based on `.tag` attribute duck typing. +def iselement(element: object) -> TypeGuard[Element]: ... +@overload +def canonicalize( + xml_data: str | ReadableBuffer | None = None, + *, + out: None = None, + from_file: _FileRead | None = None, + with_comments: bool = False, + strip_text: bool = False, + rewrite_prefixes: bool = False, + qname_aware_tags: Iterable[str] | None = None, + qname_aware_attrs: Iterable[str] | None = None, + exclude_attrs: Iterable[str] | None = None, + exclude_tags: Iterable[str] | None = None, +) -> str: ... +@overload +def canonicalize( + xml_data: str | ReadableBuffer | None = None, + *, + out: SupportsWrite[str], + from_file: _FileRead | None = None, + with_comments: bool = False, + strip_text: bool = False, + rewrite_prefixes: bool = False, + qname_aware_tags: Iterable[str] | None = None, + qname_aware_attrs: Iterable[str] | None = None, + exclude_attrs: Iterable[str] | None = None, + exclude_tags: Iterable[str] | None = None, +) -> None: ... + +class Element: + tag: str + attrib: dict[str, str] + text: str | None + tail: str | None + def __init__(self, tag: str, attrib: dict[str, str] = ..., **extra: str) -> None: ... + def append(self, subelement: Element, /) -> None: ... + def clear(self) -> None: ... + def extend(self, elements: Iterable[Element], /) -> None: ... + def find(self, path: str, namespaces: dict[str, str] | None = None) -> Element | None: ... + def findall(self, path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ... + @overload + def findtext(self, path: str, default: None = None, namespaces: dict[str, str] | None = None) -> str | None: ... + @overload + def findtext(self, path: str, default: _T, namespaces: dict[str, str] | None = None) -> _T | str: ... + @overload + def get(self, key: str, default: None = None) -> str | None: ... + @overload + def get(self, key: str, default: _T) -> str | _T: ... + def insert(self, index: int, subelement: Element, /) -> None: ... + def items(self) -> ItemsView[str, str]: ... + def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ... + def iterfind(self, path: str, namespaces: dict[str, str] | None = None) -> Generator[Element, None, None]: ... + def itertext(self) -> Generator[str, None, None]: ... + def keys(self) -> dict_keys[str, str]: ... + # makeelement returns the type of self in Python impl, but not in C impl + def makeelement(self, tag: str, attrib: dict[str, str], /) -> Element: ... + def remove(self, subelement: Element, /) -> None: ... + def set(self, key: str, value: str, /) -> None: ... + def __copy__(self) -> Element: ... # returns the type of self in Python impl, but not in C impl + def __deepcopy__(self, memo: Any, /) -> Element: ... # Only exists in C impl + def __delitem__(self, key: SupportsIndex | slice, /) -> None: ... + @overload + def __getitem__(self, key: SupportsIndex, /) -> Element: ... + @overload + def __getitem__(self, key: slice, /) -> list[Element]: ... + def __len__(self) -> int: ... + # Doesn't actually exist at runtime, but instance of the class are indeed iterable due to __getitem__. + def __iter__(self) -> Iterator[Element]: ... + @overload + def __setitem__(self, key: SupportsIndex, value: Element, /) -> None: ... + @overload + def __setitem__(self, key: slice, value: Iterable[Element], /) -> None: ... + + # Doesn't really exist in earlier versions, where __len__ is called implicitly instead + @deprecated("Testing an element's truth value is deprecated.") + def __bool__(self) -> bool: ... + if sys.version_info < (3, 9): + def getchildren(self) -> list[Element]: ... + def getiterator(self, tag: str | None = None) -> list[Element]: ... + +def SubElement(parent: Element, tag: str, attrib: dict[str, str] = ..., **extra: str) -> Element: ... +def Comment(text: str | None = None) -> Element: ... +def ProcessingInstruction(target: str, text: str | None = None) -> Element: ... + +PI = ProcessingInstruction + +class QName: + text: str + def __init__(self, text_or_uri: str, tag: str | None = None) -> None: ... + def __lt__(self, other: QName | str) -> bool: ... + def __le__(self, other: QName | str) -> bool: ... + def __gt__(self, other: QName | str) -> bool: ... + def __ge__(self, other: QName | str) -> bool: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + +class ElementTree: + def __init__(self, element: Element | None = None, file: _FileRead | None = None) -> None: ... + def getroot(self) -> Element | Any: ... + def parse(self, source: _FileRead, parser: XMLParser | None = None) -> Element: ... + def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ... + if sys.version_info < (3, 9): + def getiterator(self, tag: str | None = None) -> list[Element]: ... + + def find(self, path: str, namespaces: dict[str, str] | None = None) -> Element | None: ... + @overload + def findtext(self, path: str, default: None = None, namespaces: dict[str, str] | None = None) -> str | None: ... + @overload + def findtext(self, path: str, default: _T, namespaces: dict[str, str] | None = None) -> _T | str: ... + def findall(self, path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ... + def iterfind(self, path: str, namespaces: dict[str, str] | None = None) -> Generator[Element, None, None]: ... + def write( + self, + file_or_filename: _FileWrite, + encoding: str | None = None, + xml_declaration: bool | None = None, + default_namespace: str | None = None, + method: str | None = None, + *, + short_empty_elements: bool = True, + ) -> None: ... + def write_c14n(self, file: _FileWriteC14N) -> None: ... + +def register_namespace(prefix: str, uri: str) -> None: ... +@overload +def tostring( + element: Element, + encoding: None = None, + method: str | None = None, + *, + xml_declaration: bool | None = None, + default_namespace: str | None = None, + short_empty_elements: bool = True, +) -> bytes: ... +@overload +def tostring( + element: Element, + encoding: Literal["unicode"], + method: str | None = None, + *, + xml_declaration: bool | None = None, + default_namespace: str | None = None, + short_empty_elements: bool = True, +) -> str: ... +@overload +def tostring( + element: Element, + encoding: str, + method: str | None = None, + *, + xml_declaration: bool | None = None, + default_namespace: str | None = None, + short_empty_elements: bool = True, +) -> Any: ... +@overload +def tostringlist( + element: Element, + encoding: None = None, + method: str | None = None, + *, + xml_declaration: bool | None = None, + default_namespace: str | None = None, + short_empty_elements: bool = True, +) -> list[bytes]: ... +@overload +def tostringlist( + element: Element, + encoding: Literal["unicode"], + method: str | None = None, + *, + xml_declaration: bool | None = None, + default_namespace: str | None = None, + short_empty_elements: bool = True, +) -> list[str]: ... +@overload +def tostringlist( + element: Element, + encoding: str, + method: str | None = None, + *, + xml_declaration: bool | None = None, + default_namespace: str | None = None, + short_empty_elements: bool = True, +) -> list[Any]: ... +def dump(elem: Element) -> None: ... + +if sys.version_info >= (3, 9): + def indent(tree: Element | ElementTree, space: str = " ", level: int = 0) -> None: ... + +def parse(source: _FileRead, parser: XMLParser | None = None) -> ElementTree: ... + +class _IterParseIterator(Iterator[tuple[str, Any]]): + def __next__(self) -> tuple[str, Any]: ... + if sys.version_info >= (3, 13): + def close(self) -> None: ... + if sys.version_info >= (3, 11): + def __del__(self) -> None: ... + +def iterparse(source: _FileRead, events: Sequence[str] | None = None, parser: XMLParser | None = None) -> _IterParseIterator: ... + +class XMLPullParser: + def __init__(self, events: Sequence[str] | None = None, *, _parser: XMLParser | None = None) -> None: ... + def feed(self, data: str | ReadableBuffer) -> None: ... + def close(self) -> None: ... + # Second element in the tuple could be `Element`, `tuple[str, str]` or `None`. + # Use `Any` to avoid false-positive errors. + def read_events(self) -> Iterator[tuple[str, Any]]: ... + def flush(self) -> None: ... + +def XML(text: str | ReadableBuffer, parser: XMLParser | None = None) -> Element: ... +def XMLID(text: str | ReadableBuffer, parser: XMLParser | None = None) -> tuple[Element, dict[str, Element]]: ... + +# This is aliased to XML in the source. +fromstring = XML + +def fromstringlist(sequence: Sequence[str | ReadableBuffer], parser: XMLParser | None = None) -> Element: ... + +# This type is both not precise enough and too precise. The TreeBuilder +# requires the elementfactory to accept tag and attrs in its args and produce +# some kind of object that has .text and .tail properties. +# I've chosen to constrain the ElementFactory to always produce an Element +# because that is how almost everyone will use it. +# Unfortunately, the type of the factory arguments is dependent on how +# TreeBuilder is called by client code (they could pass strs, bytes or whatever); +# but we don't want to use a too-broad type, or it would be too hard to write +# elementfactories. +_ElementFactory: TypeAlias = Callable[[Any, dict[Any, Any]], Element] + +class TreeBuilder: + # comment_factory can take None because passing None to Comment is not an error + def __init__( + self, + element_factory: _ElementFactory | None = ..., + *, + comment_factory: Callable[[str | None], Element] | None = ..., + pi_factory: Callable[[str, str | None], Element] | None = ..., + insert_comments: bool = ..., + insert_pis: bool = ..., + ) -> None: ... + insert_comments: bool + insert_pis: bool + + def close(self) -> Element: ... + def data(self, data: str, /) -> None: ... + # tag and attrs are passed to the element_factory, so they could be anything + # depending on what the particular factory supports. + def start(self, tag: Any, attrs: dict[Any, Any], /) -> Element: ... + def end(self, tag: str, /) -> Element: ... + # These two methods have pos-only parameters in the C implementation + def comment(self, text: str | None, /) -> Element: ... + def pi(self, target: str, text: str | None = None, /) -> Element: ... + +class C14NWriterTarget: + def __init__( + self, + write: Callable[[str], object], + *, + with_comments: bool = False, + strip_text: bool = False, + rewrite_prefixes: bool = False, + qname_aware_tags: Iterable[str] | None = None, + qname_aware_attrs: Iterable[str] | None = None, + exclude_attrs: Iterable[str] | None = None, + exclude_tags: Iterable[str] | None = None, + ) -> None: ... + def data(self, data: str) -> None: ... + def start_ns(self, prefix: str, uri: str) -> None: ... + def start(self, tag: str, attrs: Mapping[str, str]) -> None: ... + def end(self, tag: str) -> None: ... + def comment(self, text: str) -> None: ... + def pi(self, target: str, data: str) -> None: ... + +class XMLParser: + parser: Any + target: Any + # TODO-what is entity used for??? + entity: Any + version: str + def __init__(self, *, target: Any = ..., encoding: str | None = ...) -> None: ... + def close(self) -> Any: ... + def feed(self, data: str | ReadableBuffer, /) -> None: ... + def flush(self) -> None: ... diff --git a/mypy/typeshed/stdlib/xml/etree/__init__.pyi b/mypy/typeshed/stdlib/xml/etree/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypy/typeshed/stdlib/xml/etree/cElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/cElementTree.pyi new file mode 100644 index 000000000000..02272d803c18 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/etree/cElementTree.pyi @@ -0,0 +1 @@ +from xml.etree.ElementTree import * diff --git a/mypy/typeshed/stdlib/xml/parsers/__init__.pyi b/mypy/typeshed/stdlib/xml/parsers/__init__.pyi new file mode 100644 index 000000000000..cebdb6a30014 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/parsers/__init__.pyi @@ -0,0 +1 @@ +from xml.parsers import expat as expat diff --git a/mypy/typeshed/stdlib/xml/parsers/expat/__init__.pyi b/mypy/typeshed/stdlib/xml/parsers/expat/__init__.pyi new file mode 100644 index 000000000000..d9b7ea536999 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/parsers/expat/__init__.pyi @@ -0,0 +1,7 @@ +from pyexpat import * + +# This is actually implemented in the C module pyexpat, but considers itself to live here. +class ExpatError(Exception): + code: int + lineno: int + offset: int diff --git a/mypy/typeshed/stdlib/xml/parsers/expat/errors.pyi b/mypy/typeshed/stdlib/xml/parsers/expat/errors.pyi new file mode 100644 index 000000000000..e22d769ec340 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/parsers/expat/errors.pyi @@ -0,0 +1 @@ +from pyexpat.errors import * diff --git a/mypy/typeshed/stdlib/xml/parsers/expat/model.pyi b/mypy/typeshed/stdlib/xml/parsers/expat/model.pyi new file mode 100644 index 000000000000..d8f44b47c51b --- /dev/null +++ b/mypy/typeshed/stdlib/xml/parsers/expat/model.pyi @@ -0,0 +1 @@ +from pyexpat.model import * diff --git a/mypy/typeshed/stdlib/xml/sax/__init__.pyi b/mypy/typeshed/stdlib/xml/sax/__init__.pyi new file mode 100644 index 000000000000..a2eecc5a7864 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/sax/__init__.pyi @@ -0,0 +1,25 @@ +from _typeshed import ReadableBuffer, StrPath, SupportsRead, _T_co +from collections.abc import Iterable +from typing import Protocol +from typing_extensions import TypeAlias +from xml.sax._exceptions import ( + SAXException as SAXException, + SAXNotRecognizedException as SAXNotRecognizedException, + SAXNotSupportedException as SAXNotSupportedException, + SAXParseException as SAXParseException, + SAXReaderNotAvailable as SAXReaderNotAvailable, +) +from xml.sax.handler import ContentHandler as ContentHandler, ErrorHandler as ErrorHandler +from xml.sax.xmlreader import XMLReader + +class _SupportsReadClose(SupportsRead[_T_co], Protocol[_T_co]): + def close(self) -> None: ... + +_Source: TypeAlias = StrPath | _SupportsReadClose[bytes] | _SupportsReadClose[str] + +default_parser_list: list[str] + +def make_parser(parser_list: Iterable[str] = ()) -> XMLReader: ... +def parse(source: _Source, handler: ContentHandler, errorHandler: ErrorHandler = ...) -> None: ... +def parseString(string: ReadableBuffer | str, handler: ContentHandler, errorHandler: ErrorHandler | None = ...) -> None: ... +def _create_parser(parser_name: str) -> XMLReader: ... diff --git a/mypy/typeshed/stdlib/xml/sax/_exceptions.pyi b/mypy/typeshed/stdlib/xml/sax/_exceptions.pyi new file mode 100644 index 000000000000..8a437a971f13 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/sax/_exceptions.pyi @@ -0,0 +1,19 @@ +from typing import NoReturn +from xml.sax.xmlreader import Locator + +class SAXException(Exception): + def __init__(self, msg: str, exception: Exception | None = None) -> None: ... + def getMessage(self) -> str: ... + def getException(self) -> Exception: ... + def __getitem__(self, ix: object) -> NoReturn: ... + +class SAXParseException(SAXException): + def __init__(self, msg: str, exception: Exception | None, locator: Locator) -> None: ... + def getColumnNumber(self) -> int: ... + def getLineNumber(self) -> int: ... + def getPublicId(self): ... + def getSystemId(self): ... + +class SAXNotRecognizedException(SAXException): ... +class SAXNotSupportedException(SAXException): ... +class SAXReaderNotAvailable(SAXNotSupportedException): ... diff --git a/mypy/typeshed/stdlib/xml/sax/handler.pyi b/mypy/typeshed/stdlib/xml/sax/handler.pyi new file mode 100644 index 000000000000..7b7c69048efd --- /dev/null +++ b/mypy/typeshed/stdlib/xml/sax/handler.pyi @@ -0,0 +1,55 @@ +import sys +from typing import NoReturn +from xml.sax import xmlreader + +version: str + +class ErrorHandler: + def error(self, exception: BaseException) -> NoReturn: ... + def fatalError(self, exception: BaseException) -> NoReturn: ... + def warning(self, exception: BaseException) -> None: ... + +class ContentHandler: + def setDocumentLocator(self, locator: xmlreader.Locator) -> None: ... + def startDocument(self) -> None: ... + def endDocument(self) -> None: ... + def startPrefixMapping(self, prefix: str | None, uri: str) -> None: ... + def endPrefixMapping(self, prefix: str | None) -> None: ... + def startElement(self, name: str, attrs: xmlreader.AttributesImpl) -> None: ... + def endElement(self, name: str) -> None: ... + def startElementNS(self, name: tuple[str, str], qname: str, attrs: xmlreader.AttributesNSImpl) -> None: ... + def endElementNS(self, name: tuple[str, str], qname: str) -> None: ... + def characters(self, content: str) -> None: ... + def ignorableWhitespace(self, whitespace: str) -> None: ... + def processingInstruction(self, target: str, data: str) -> None: ... + def skippedEntity(self, name: str) -> None: ... + +class DTDHandler: + def notationDecl(self, name, publicId, systemId): ... + def unparsedEntityDecl(self, name, publicId, systemId, ndata): ... + +class EntityResolver: + def resolveEntity(self, publicId, systemId): ... + +feature_namespaces: str +feature_namespace_prefixes: str +feature_string_interning: str +feature_validation: str +feature_external_ges: str +feature_external_pes: str +all_features: list[str] +property_lexical_handler: str +property_declaration_handler: str +property_dom_node: str +property_xml_string: str +property_encoding: str +property_interning_dict: str +all_properties: list[str] + +if sys.version_info >= (3, 10): + class LexicalHandler: + def comment(self, content: str) -> object: ... + def startDTD(self, name: str, public_id: str | None, system_id: str | None) -> object: ... + def endDTD(self) -> object: ... + def startCDATA(self) -> object: ... + def endCDATA(self) -> object: ... diff --git a/mypy/typeshed/stdlib/xml/sax/saxutils.pyi b/mypy/typeshed/stdlib/xml/sax/saxutils.pyi new file mode 100644 index 000000000000..528f35963947 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/sax/saxutils.pyi @@ -0,0 +1,60 @@ +from _typeshed import SupportsWrite +from codecs import StreamReaderWriter, StreamWriter +from collections.abc import Mapping +from io import RawIOBase, TextIOBase +from xml.sax import _Source, handler, xmlreader + +def escape(data: str, entities: Mapping[str, str] = {}) -> str: ... +def unescape(data: str, entities: Mapping[str, str] = {}) -> str: ... +def quoteattr(data: str, entities: Mapping[str, str] = {}) -> str: ... + +class XMLGenerator(handler.ContentHandler): + def __init__( + self, + out: TextIOBase | RawIOBase | StreamWriter | StreamReaderWriter | SupportsWrite[bytes] | None = None, + encoding: str = "iso-8859-1", + short_empty_elements: bool = False, + ) -> None: ... + def startDocument(self) -> None: ... + def endDocument(self) -> None: ... + def startPrefixMapping(self, prefix: str | None, uri: str) -> None: ... + def endPrefixMapping(self, prefix: str | None) -> None: ... + def startElement(self, name: str, attrs: xmlreader.AttributesImpl) -> None: ... + def endElement(self, name: str) -> None: ... + def startElementNS(self, name: tuple[str, str], qname: str, attrs: xmlreader.AttributesNSImpl) -> None: ... + def endElementNS(self, name: tuple[str, str], qname: str) -> None: ... + def characters(self, content: str) -> None: ... + def ignorableWhitespace(self, content: str) -> None: ... + def processingInstruction(self, target: str, data: str) -> None: ... + +class XMLFilterBase(xmlreader.XMLReader): + def __init__(self, parent: xmlreader.XMLReader | None = None) -> None: ... + def error(self, exception): ... + def fatalError(self, exception): ... + def warning(self, exception): ... + def setDocumentLocator(self, locator: xmlreader.Locator) -> None: ... + def startDocument(self) -> None: ... + def endDocument(self) -> None: ... + def startPrefixMapping(self, prefix: str | None, uri: str) -> None: ... + def endPrefixMapping(self, prefix: str | None) -> None: ... + def startElement(self, name: str, attrs: xmlreader.AttributesImpl) -> None: ... + def endElement(self, name: str) -> None: ... + def startElementNS(self, name: tuple[str, str], qname: str, attrs: xmlreader.AttributesNSImpl) -> None: ... + def endElementNS(self, name: tuple[str, str], qname: str) -> None: ... + def characters(self, content: str) -> None: ... + def ignorableWhitespace(self, chars: str) -> None: ... + def processingInstruction(self, target: str, data: str) -> None: ... + def skippedEntity(self, name: str) -> None: ... + def notationDecl(self, name, publicId, systemId): ... + def unparsedEntityDecl(self, name, publicId, systemId, ndata): ... + def resolveEntity(self, publicId, systemId): ... + def parse(self, source: _Source) -> None: ... + def setLocale(self, locale): ... + def getFeature(self, name: str) -> object: ... + def setFeature(self, name: str, state: object) -> None: ... + def getProperty(self, name: str) -> object: ... + def setProperty(self, name: str, value: object) -> None: ... + def getParent(self) -> xmlreader.XMLReader: ... + def setParent(self, parent: xmlreader.XMLReader) -> None: ... + +def prepare_input_source(source, base=""): ... diff --git a/mypy/typeshed/stdlib/xml/sax/xmlreader.pyi b/mypy/typeshed/stdlib/xml/sax/xmlreader.pyi new file mode 100644 index 000000000000..2ccbc95bbef0 --- /dev/null +++ b/mypy/typeshed/stdlib/xml/sax/xmlreader.pyi @@ -0,0 +1,87 @@ +from collections.abc import Mapping +from typing import overload +from typing_extensions import Self, TypeAlias +from xml.sax.handler import ContentHandler, DTDHandler, EntityResolver, ErrorHandler + +class XMLReader: + def parse(self, source): ... + def getContentHandler(self) -> ContentHandler: ... + def setContentHandler(self, handler: ContentHandler) -> None: ... + def getDTDHandler(self) -> DTDHandler: ... + def setDTDHandler(self, handler: DTDHandler) -> None: ... + def getEntityResolver(self) -> EntityResolver: ... + def setEntityResolver(self, resolver: EntityResolver) -> None: ... + def getErrorHandler(self) -> ErrorHandler: ... + def setErrorHandler(self, handler: ErrorHandler) -> None: ... + def setLocale(self, locale): ... + def getFeature(self, name: str) -> object: ... + def setFeature(self, name: str, state: object) -> None: ... + def getProperty(self, name: str) -> object: ... + def setProperty(self, name: str, value: object) -> None: ... + +class IncrementalParser(XMLReader): + def __init__(self, bufsize: int = 65536) -> None: ... + def parse(self, source): ... + def feed(self, data): ... + def prepareParser(self, source): ... + def close(self): ... + def reset(self): ... + +class Locator: + def getColumnNumber(self): ... + def getLineNumber(self): ... + def getPublicId(self): ... + def getSystemId(self): ... + +class InputSource: + def __init__(self, system_id: str | None = None) -> None: ... + def setPublicId(self, public_id): ... + def getPublicId(self): ... + def setSystemId(self, system_id): ... + def getSystemId(self): ... + def setEncoding(self, encoding): ... + def getEncoding(self): ... + def setByteStream(self, bytefile): ... + def getByteStream(self): ... + def setCharacterStream(self, charfile): ... + def getCharacterStream(self): ... + +class AttributesImpl: + def __init__(self, attrs: Mapping[str, str]) -> None: ... + def getLength(self) -> int: ... + def getType(self, name: str) -> str: ... + def getValue(self, name: str) -> str: ... + def getValueByQName(self, name: str) -> str: ... + def getNameByQName(self, name: str) -> str: ... + def getQNameByName(self, name: str) -> str: ... + def getNames(self) -> list[str]: ... + def getQNames(self) -> list[str]: ... + def __len__(self) -> int: ... + def __getitem__(self, name: str) -> str: ... + def keys(self) -> list[str]: ... + def __contains__(self, name: str) -> bool: ... + @overload + def get(self, name: str, alternative: None = None) -> str | None: ... + @overload + def get(self, name: str, alternative: str) -> str: ... + def copy(self) -> Self: ... + def items(self) -> list[tuple[str, str]]: ... + def values(self) -> list[str]: ... + +_NSName: TypeAlias = tuple[str | None, str] + +class AttributesNSImpl(AttributesImpl): + def __init__(self, attrs: Mapping[_NSName, str], qnames: Mapping[_NSName, str]) -> None: ... + def getType(self, name: _NSName) -> str: ... # type: ignore[override] + def getValue(self, name: _NSName) -> str: ... # type: ignore[override] + def getNameByQName(self, name: str) -> _NSName: ... # type: ignore[override] + def getQNameByName(self, name: _NSName) -> str: ... # type: ignore[override] + def getNames(self) -> list[_NSName]: ... # type: ignore[override] + def __getitem__(self, name: _NSName) -> str: ... # type: ignore[override] + def keys(self) -> list[_NSName]: ... # type: ignore[override] + def __contains__(self, name: _NSName) -> bool: ... # type: ignore[override] + @overload # type: ignore[override] + def get(self, name: _NSName, alternative: None = None) -> str | None: ... + @overload + def get(self, name: _NSName, alternative: str) -> str: ... + def items(self) -> list[tuple[_NSName, str]]: ... # type: ignore[override] diff --git a/mypy/typeshed/stdlib/xmlrpc/__init__.pyi b/mypy/typeshed/stdlib/xmlrpc/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypy/typeshed/stdlib/xmlrpc/client.pyi b/mypy/typeshed/stdlib/xmlrpc/client.pyi new file mode 100644 index 000000000000..5899d1d72a38 --- /dev/null +++ b/mypy/typeshed/stdlib/xmlrpc/client.pyi @@ -0,0 +1,296 @@ +import gzip +import http.client +import time +from _typeshed import ReadableBuffer, SizedBuffer, SupportsRead, SupportsWrite +from collections.abc import Callable, Iterable, Mapping +from datetime import datetime +from io import BytesIO +from types import TracebackType +from typing import Any, Final, Literal, Protocol, overload +from typing_extensions import Self, TypeAlias + +class _SupportsTimeTuple(Protocol): + def timetuple(self) -> time.struct_time: ... + +_DateTimeComparable: TypeAlias = DateTime | datetime | str | _SupportsTimeTuple +_Marshallable: TypeAlias = ( + bool + | int + | float + | str + | bytes + | bytearray + | None + | tuple[_Marshallable, ...] + # Ideally we'd use _Marshallable for list and dict, but invariance makes that impractical + | list[Any] + | dict[str, Any] + | datetime + | DateTime + | Binary +) +_XMLDate: TypeAlias = int | datetime | tuple[int, ...] | time.struct_time +_HostType: TypeAlias = tuple[str, dict[str, str]] | str + +def escape(s: str) -> str: ... # undocumented + +MAXINT: Final[int] # undocumented +MININT: Final[int] # undocumented + +PARSE_ERROR: Final[int] # undocumented +SERVER_ERROR: Final[int] # undocumented +APPLICATION_ERROR: Final[int] # undocumented +SYSTEM_ERROR: Final[int] # undocumented +TRANSPORT_ERROR: Final[int] # undocumented + +NOT_WELLFORMED_ERROR: Final[int] # undocumented +UNSUPPORTED_ENCODING: Final[int] # undocumented +INVALID_ENCODING_CHAR: Final[int] # undocumented +INVALID_XMLRPC: Final[int] # undocumented +METHOD_NOT_FOUND: Final[int] # undocumented +INVALID_METHOD_PARAMS: Final[int] # undocumented +INTERNAL_ERROR: Final[int] # undocumented + +class Error(Exception): ... + +class ProtocolError(Error): + url: str + errcode: int + errmsg: str + headers: dict[str, str] + def __init__(self, url: str, errcode: int, errmsg: str, headers: dict[str, str]) -> None: ... + +class ResponseError(Error): ... + +class Fault(Error): + faultCode: int + faultString: str + def __init__(self, faultCode: int, faultString: str, **extra: Any) -> None: ... + +boolean = bool +Boolean = bool + +def _iso8601_format(value: datetime) -> str: ... # undocumented +def _strftime(value: _XMLDate) -> str: ... # undocumented + +class DateTime: + value: str # undocumented + def __init__(self, value: int | str | datetime | time.struct_time | tuple[int, ...] = 0) -> None: ... + def __lt__(self, other: _DateTimeComparable) -> bool: ... + def __le__(self, other: _DateTimeComparable) -> bool: ... + def __gt__(self, other: _DateTimeComparable) -> bool: ... + def __ge__(self, other: _DateTimeComparable) -> bool: ... + def __eq__(self, other: _DateTimeComparable) -> bool: ... # type: ignore[override] + def make_comparable(self, other: _DateTimeComparable) -> tuple[str, str]: ... # undocumented + def timetuple(self) -> time.struct_time: ... # undocumented + def decode(self, data: Any) -> None: ... + def encode(self, out: SupportsWrite[str]) -> None: ... + +def _datetime(data: Any) -> DateTime: ... # undocumented +def _datetime_type(data: str) -> datetime: ... # undocumented + +class Binary: + data: bytes + def __init__(self, data: bytes | bytearray | None = None) -> None: ... + def decode(self, data: ReadableBuffer) -> None: ... + def encode(self, out: SupportsWrite[str]) -> None: ... + def __eq__(self, other: object) -> bool: ... + +def _binary(data: ReadableBuffer) -> Binary: ... # undocumented + +WRAPPERS: Final[tuple[type[DateTime], type[Binary]]] # undocumented + +class ExpatParser: # undocumented + def __init__(self, target: Unmarshaller) -> None: ... + def feed(self, data: str | ReadableBuffer) -> None: ... + def close(self) -> None: ... + +_WriteCallback: TypeAlias = Callable[[str], object] + +class Marshaller: + # TODO: Replace 'Any' with some kind of binding + dispatch: dict[type[Any], Callable[[Marshaller, Any, _WriteCallback], None]] + memo: dict[Any, None] + data: None + encoding: str | None + allow_none: bool + def __init__(self, encoding: str | None = None, allow_none: bool = False) -> None: ... + def dumps(self, values: Fault | Iterable[_Marshallable]) -> str: ... + def __dump(self, value: _Marshallable, write: _WriteCallback) -> None: ... # undocumented + def dump_nil(self, value: None, write: _WriteCallback) -> None: ... + def dump_bool(self, value: bool, write: _WriteCallback) -> None: ... + def dump_long(self, value: int, write: _WriteCallback) -> None: ... + def dump_int(self, value: int, write: _WriteCallback) -> None: ... + def dump_double(self, value: float, write: _WriteCallback) -> None: ... + def dump_unicode(self, value: str, write: _WriteCallback, escape: Callable[[str], str] = ...) -> None: ... + def dump_bytes(self, value: ReadableBuffer, write: _WriteCallback) -> None: ... + def dump_array(self, value: Iterable[_Marshallable], write: _WriteCallback) -> None: ... + def dump_struct( + self, value: Mapping[str, _Marshallable], write: _WriteCallback, escape: Callable[[str], str] = ... + ) -> None: ... + def dump_datetime(self, value: _XMLDate, write: _WriteCallback) -> None: ... + def dump_instance(self, value: object, write: _WriteCallback) -> None: ... + +class Unmarshaller: + dispatch: dict[str, Callable[[Unmarshaller, str], None]] + + _type: str | None + _stack: list[_Marshallable] + _marks: list[int] + _data: list[str] + _value: bool + _methodname: str | None + _encoding: str + append: Callable[[Any], None] + _use_datetime: bool + _use_builtin_types: bool + def __init__(self, use_datetime: bool = False, use_builtin_types: bool = False) -> None: ... + def close(self) -> tuple[_Marshallable, ...]: ... + def getmethodname(self) -> str | None: ... + def xml(self, encoding: str, standalone: Any) -> None: ... # Standalone is ignored + def start(self, tag: str, attrs: dict[str, str]) -> None: ... + def data(self, text: str) -> None: ... + def end(self, tag: str) -> None: ... + def end_dispatch(self, tag: str, data: str) -> None: ... + def end_nil(self, data: str) -> None: ... + def end_boolean(self, data: str) -> None: ... + def end_int(self, data: str) -> None: ... + def end_double(self, data: str) -> None: ... + def end_bigdecimal(self, data: str) -> None: ... + def end_string(self, data: str) -> None: ... + def end_array(self, data: str) -> None: ... + def end_struct(self, data: str) -> None: ... + def end_base64(self, data: str) -> None: ... + def end_dateTime(self, data: str) -> None: ... + def end_value(self, data: str) -> None: ... + def end_params(self, data: str) -> None: ... + def end_fault(self, data: str) -> None: ... + def end_methodName(self, data: str) -> None: ... + +class _MultiCallMethod: # undocumented + __call_list: list[tuple[str, tuple[_Marshallable, ...]]] + __name: str + def __init__(self, call_list: list[tuple[str, _Marshallable]], name: str) -> None: ... + def __getattr__(self, name: str) -> _MultiCallMethod: ... + def __call__(self, *args: _Marshallable) -> None: ... + +class MultiCallIterator: # undocumented + results: list[list[_Marshallable]] + def __init__(self, results: list[list[_Marshallable]]) -> None: ... + def __getitem__(self, i: int) -> _Marshallable: ... + +class MultiCall: + __server: ServerProxy + __call_list: list[tuple[str, tuple[_Marshallable, ...]]] + def __init__(self, server: ServerProxy) -> None: ... + def __getattr__(self, name: str) -> _MultiCallMethod: ... + def __call__(self) -> MultiCallIterator: ... + +# A little white lie +FastMarshaller: Marshaller | None +FastParser: ExpatParser | None +FastUnmarshaller: Unmarshaller | None + +def getparser(use_datetime: bool = False, use_builtin_types: bool = False) -> tuple[ExpatParser, Unmarshaller]: ... +def dumps( + params: Fault | tuple[_Marshallable, ...], + methodname: str | None = None, + methodresponse: bool | None = None, + encoding: str | None = None, + allow_none: bool = False, +) -> str: ... +def loads( + data: str | ReadableBuffer, use_datetime: bool = False, use_builtin_types: bool = False +) -> tuple[tuple[_Marshallable, ...], str | None]: ... +def gzip_encode(data: ReadableBuffer) -> bytes: ... # undocumented +def gzip_decode(data: ReadableBuffer, max_decode: int = 20971520) -> bytes: ... # undocumented + +class GzipDecodedResponse(gzip.GzipFile): # undocumented + io: BytesIO + def __init__(self, response: SupportsRead[ReadableBuffer]) -> None: ... + +class _Method: # undocumented + __send: Callable[[str, tuple[_Marshallable, ...]], _Marshallable] + __name: str + def __init__(self, send: Callable[[str, tuple[_Marshallable, ...]], _Marshallable], name: str) -> None: ... + def __getattr__(self, name: str) -> _Method: ... + def __call__(self, *args: _Marshallable) -> _Marshallable: ... + +class Transport: + user_agent: str + accept_gzip_encoding: bool + encode_threshold: int | None + + _use_datetime: bool + _use_builtin_types: bool + _connection: tuple[_HostType | None, http.client.HTTPConnection | None] + _headers: list[tuple[str, str]] + _extra_headers: list[tuple[str, str]] + + def __init__( + self, use_datetime: bool = False, use_builtin_types: bool = False, *, headers: Iterable[tuple[str, str]] = () + ) -> None: ... + def request( + self, host: _HostType, handler: str, request_body: SizedBuffer, verbose: bool = False + ) -> tuple[_Marshallable, ...]: ... + def single_request( + self, host: _HostType, handler: str, request_body: SizedBuffer, verbose: bool = False + ) -> tuple[_Marshallable, ...]: ... + def getparser(self) -> tuple[ExpatParser, Unmarshaller]: ... + def get_host_info(self, host: _HostType) -> tuple[str, list[tuple[str, str]], dict[str, str]]: ... + def make_connection(self, host: _HostType) -> http.client.HTTPConnection: ... + def close(self) -> None: ... + def send_request( + self, host: _HostType, handler: str, request_body: SizedBuffer, debug: bool + ) -> http.client.HTTPConnection: ... + def send_headers(self, connection: http.client.HTTPConnection, headers: list[tuple[str, str]]) -> None: ... + def send_content(self, connection: http.client.HTTPConnection, request_body: SizedBuffer) -> None: ... + def parse_response(self, response: http.client.HTTPResponse) -> tuple[_Marshallable, ...]: ... + +class SafeTransport(Transport): + def __init__( + self, + use_datetime: bool = False, + use_builtin_types: bool = False, + *, + headers: Iterable[tuple[str, str]] = (), + context: Any | None = None, + ) -> None: ... + def make_connection(self, host: _HostType) -> http.client.HTTPSConnection: ... + +class ServerProxy: + __host: str + __handler: str + __transport: Transport + __encoding: str + __verbose: bool + __allow_none: bool + + def __init__( + self, + uri: str, + transport: Transport | None = None, + encoding: str | None = None, + verbose: bool = False, + allow_none: bool = False, + use_datetime: bool = False, + use_builtin_types: bool = False, + *, + headers: Iterable[tuple[str, str]] = (), + context: Any | None = None, + ) -> None: ... + def __getattr__(self, name: str) -> _Method: ... + @overload + def __call__(self, attr: Literal["close"]) -> Callable[[], None]: ... + @overload + def __call__(self, attr: Literal["transport"]) -> Transport: ... + @overload + def __call__(self, attr: str) -> Callable[[], None] | Transport: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def __close(self) -> None: ... # undocumented + def __request(self, methodname: str, params: tuple[_Marshallable, ...]) -> tuple[_Marshallable, ...]: ... # undocumented + +Server = ServerProxy diff --git a/mypy/typeshed/stdlib/xmlrpc/server.pyi b/mypy/typeshed/stdlib/xmlrpc/server.pyi new file mode 100644 index 000000000000..5f497aa7190e --- /dev/null +++ b/mypy/typeshed/stdlib/xmlrpc/server.pyi @@ -0,0 +1,144 @@ +import http.server +import pydoc +import socketserver +from _typeshed import ReadableBuffer +from collections.abc import Callable, Iterable, Mapping +from re import Pattern +from typing import Any, ClassVar, Protocol +from typing_extensions import TypeAlias +from xmlrpc.client import Fault, _Marshallable + +# The dispatch accepts anywhere from 0 to N arguments, no easy way to allow this in mypy +class _DispatchArity0(Protocol): + def __call__(self) -> _Marshallable: ... + +class _DispatchArity1(Protocol): + def __call__(self, arg1: _Marshallable, /) -> _Marshallable: ... + +class _DispatchArity2(Protocol): + def __call__(self, arg1: _Marshallable, arg2: _Marshallable, /) -> _Marshallable: ... + +class _DispatchArity3(Protocol): + def __call__(self, arg1: _Marshallable, arg2: _Marshallable, arg3: _Marshallable, /) -> _Marshallable: ... + +class _DispatchArity4(Protocol): + def __call__( + self, arg1: _Marshallable, arg2: _Marshallable, arg3: _Marshallable, arg4: _Marshallable, / + ) -> _Marshallable: ... + +class _DispatchArityN(Protocol): + def __call__(self, *args: _Marshallable) -> _Marshallable: ... + +_DispatchProtocol: TypeAlias = ( + _DispatchArity0 | _DispatchArity1 | _DispatchArity2 | _DispatchArity3 | _DispatchArity4 | _DispatchArityN +) + +def resolve_dotted_attribute(obj: Any, attr: str, allow_dotted_names: bool = True) -> Any: ... # undocumented +def list_public_methods(obj: Any) -> list[str]: ... # undocumented + +class SimpleXMLRPCDispatcher: # undocumented + funcs: dict[str, _DispatchProtocol] + instance: Any | None + allow_none: bool + encoding: str + use_builtin_types: bool + def __init__(self, allow_none: bool = False, encoding: str | None = None, use_builtin_types: bool = False) -> None: ... + def register_instance(self, instance: Any, allow_dotted_names: bool = False) -> None: ... + def register_function(self, function: _DispatchProtocol | None = None, name: str | None = None) -> Callable[..., Any]: ... + def register_introspection_functions(self) -> None: ... + def register_multicall_functions(self) -> None: ... + def _marshaled_dispatch( + self, + data: str | ReadableBuffer, + dispatch_method: Callable[[str, tuple[_Marshallable, ...]], Fault | tuple[_Marshallable, ...]] | None = None, + path: Any | None = None, + ) -> str: ... # undocumented + def system_listMethods(self) -> list[str]: ... # undocumented + def system_methodSignature(self, method_name: str) -> str: ... # undocumented + def system_methodHelp(self, method_name: str) -> str: ... # undocumented + def system_multicall(self, call_list: list[dict[str, _Marshallable]]) -> list[_Marshallable]: ... # undocumented + def _dispatch(self, method: str, params: Iterable[_Marshallable]) -> _Marshallable: ... # undocumented + +class SimpleXMLRPCRequestHandler(http.server.BaseHTTPRequestHandler): + rpc_paths: ClassVar[tuple[str, ...]] + encode_threshold: int # undocumented + aepattern: Pattern[str] # undocumented + def accept_encodings(self) -> dict[str, float]: ... + def is_rpc_path_valid(self) -> bool: ... + def do_POST(self) -> None: ... + def decode_request_content(self, data: bytes) -> bytes | None: ... + def report_404(self) -> None: ... + +class SimpleXMLRPCServer(socketserver.TCPServer, SimpleXMLRPCDispatcher): + _send_traceback_handler: bool + def __init__( + self, + addr: tuple[str, int], + requestHandler: type[SimpleXMLRPCRequestHandler] = ..., + logRequests: bool = True, + allow_none: bool = False, + encoding: str | None = None, + bind_and_activate: bool = True, + use_builtin_types: bool = False, + ) -> None: ... + +class MultiPathXMLRPCServer(SimpleXMLRPCServer): # undocumented + dispatchers: dict[str, SimpleXMLRPCDispatcher] + def __init__( + self, + addr: tuple[str, int], + requestHandler: type[SimpleXMLRPCRequestHandler] = ..., + logRequests: bool = True, + allow_none: bool = False, + encoding: str | None = None, + bind_and_activate: bool = True, + use_builtin_types: bool = False, + ) -> None: ... + def add_dispatcher(self, path: str, dispatcher: SimpleXMLRPCDispatcher) -> SimpleXMLRPCDispatcher: ... + def get_dispatcher(self, path: str) -> SimpleXMLRPCDispatcher: ... + +class CGIXMLRPCRequestHandler(SimpleXMLRPCDispatcher): + def __init__(self, allow_none: bool = False, encoding: str | None = None, use_builtin_types: bool = False) -> None: ... + def handle_xmlrpc(self, request_text: str) -> None: ... + def handle_get(self) -> None: ... + def handle_request(self, request_text: str | None = None) -> None: ... + +class ServerHTMLDoc(pydoc.HTMLDoc): # undocumented + def docroutine( # type: ignore[override] + self, + object: object, + name: str, + mod: str | None = None, + funcs: Mapping[str, str] = {}, + classes: Mapping[str, str] = {}, + methods: Mapping[str, str] = {}, + cl: type | None = None, + ) -> str: ... + def docserver(self, server_name: str, package_documentation: str, methods: dict[str, str]) -> str: ... + +class XMLRPCDocGenerator: # undocumented + server_name: str + server_documentation: str + server_title: str + def set_server_title(self, server_title: str) -> None: ... + def set_server_name(self, server_name: str) -> None: ... + def set_server_documentation(self, server_documentation: str) -> None: ... + def generate_html_documentation(self) -> str: ... + +class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + def do_GET(self) -> None: ... + +class DocXMLRPCServer(SimpleXMLRPCServer, XMLRPCDocGenerator): + def __init__( + self, + addr: tuple[str, int], + requestHandler: type[SimpleXMLRPCRequestHandler] = ..., + logRequests: bool = True, + allow_none: bool = False, + encoding: str | None = None, + bind_and_activate: bool = True, + use_builtin_types: bool = False, + ) -> None: ... + +class DocCGIXMLRPCRequestHandler(CGIXMLRPCRequestHandler, XMLRPCDocGenerator): + def __init__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/xxlimited.pyi b/mypy/typeshed/stdlib/xxlimited.pyi new file mode 100644 index 000000000000..6bae87a8db2a --- /dev/null +++ b/mypy/typeshed/stdlib/xxlimited.pyi @@ -0,0 +1,22 @@ +import sys +from typing import Any, final + +class Str(str): ... + +@final +class Xxo: + def demo(self) -> None: ... + if sys.version_info >= (3, 11) and sys.platform != "win32": + x_exports: int + +def foo(i: int, j: int, /) -> Any: ... +def new() -> Xxo: ... + +if sys.version_info >= (3, 10): + class Error(Exception): ... + +else: + class error(Exception): ... + class Null: ... + + def roj(b: Any, /) -> None: ... diff --git a/mypy/typeshed/stdlib/zipapp.pyi b/mypy/typeshed/stdlib/zipapp.pyi new file mode 100644 index 000000000000..c7cf1704b135 --- /dev/null +++ b/mypy/typeshed/stdlib/zipapp.pyi @@ -0,0 +1,20 @@ +from collections.abc import Callable +from pathlib import Path +from typing import BinaryIO +from typing_extensions import TypeAlias + +__all__ = ["ZipAppError", "create_archive", "get_interpreter"] + +_Path: TypeAlias = str | Path | BinaryIO + +class ZipAppError(ValueError): ... + +def create_archive( + source: _Path, + target: _Path | None = None, + interpreter: str | None = None, + main: str | None = None, + filter: Callable[[Path], bool] | None = None, + compressed: bool = False, +) -> None: ... +def get_interpreter(archive: _Path) -> str: ... diff --git a/mypy/typeshed/stdlib/zipfile/__init__.pyi b/mypy/typeshed/stdlib/zipfile/__init__.pyi new file mode 100644 index 000000000000..5b8f02f61bce --- /dev/null +++ b/mypy/typeshed/stdlib/zipfile/__init__.pyi @@ -0,0 +1,384 @@ +import io +import sys +from _typeshed import SizedBuffer, StrOrBytesPath, StrPath +from collections.abc import Callable, Iterable, Iterator +from io import TextIOWrapper +from os import PathLike +from types import TracebackType +from typing import IO, Final, Literal, Protocol, overload +from typing_extensions import Self, TypeAlias + +__all__ = [ + "BadZipFile", + "BadZipfile", + "Path", + "error", + "ZIP_STORED", + "ZIP_DEFLATED", + "ZIP_BZIP2", + "ZIP_LZMA", + "is_zipfile", + "ZipInfo", + "ZipFile", + "PyZipFile", + "LargeZipFile", +] + +# TODO: use TypeAlias for these two when mypy bugs are fixed +# https://github.com/python/mypy/issues/16581 +_DateTuple = tuple[int, int, int, int, int, int] # noqa: Y026 +_ZipFileMode = Literal["r", "w", "x", "a"] # noqa: Y026 + +_ReadWriteMode: TypeAlias = Literal["r", "w"] +_ReadWriteBinaryMode: TypeAlias = Literal["r", "w", "rb", "wb"] + +class BadZipFile(Exception): ... + +BadZipfile = BadZipFile +error = BadZipfile + +class LargeZipFile(Exception): ... + +class _ZipStream(Protocol): + def read(self, n: int, /) -> bytes: ... + # The following methods are optional: + # def seekable(self) -> bool: ... + # def tell(self) -> int: ... + # def seek(self, n: int, /) -> object: ... + +# Stream shape as required by _EndRecData() and _EndRecData64(). +class _SupportsReadSeekTell(Protocol): + def read(self, n: int = ..., /) -> bytes: ... + def seek(self, cookie: int, whence: int, /) -> object: ... + def tell(self) -> int: ... + +class _ClosableZipStream(_ZipStream, Protocol): + def close(self) -> object: ... + +class ZipExtFile(io.BufferedIOBase): + MAX_N: int + MIN_READ_SIZE: int + MAX_SEEK_READ: int + newlines: list[bytes] | None + mode: _ReadWriteMode + name: str + @overload + def __init__( + self, fileobj: _ClosableZipStream, mode: _ReadWriteMode, zipinfo: ZipInfo, pwd: bytes | None, close_fileobj: Literal[True] + ) -> None: ... + @overload + def __init__( + self, + fileobj: _ClosableZipStream, + mode: _ReadWriteMode, + zipinfo: ZipInfo, + pwd: bytes | None = None, + *, + close_fileobj: Literal[True], + ) -> None: ... + @overload + def __init__( + self, + fileobj: _ZipStream, + mode: _ReadWriteMode, + zipinfo: ZipInfo, + pwd: bytes | None = None, + close_fileobj: Literal[False] = False, + ) -> None: ... + def read(self, n: int | None = -1) -> bytes: ... + def readline(self, limit: int = -1) -> bytes: ... # type: ignore[override] + def peek(self, n: int = 1) -> bytes: ... + def read1(self, n: int | None) -> bytes: ... # type: ignore[override] + def seek(self, offset: int, whence: int = 0) -> int: ... + +class _Writer(Protocol): + def write(self, s: str, /) -> object: ... + +class _ZipReadable(Protocol): + def seek(self, offset: int, whence: int = 0, /) -> int: ... + def read(self, n: int = -1, /) -> bytes: ... + +class _ZipTellable(Protocol): + def tell(self) -> int: ... + +class _ZipReadableTellable(_ZipReadable, _ZipTellable, Protocol): ... + +class _ZipWritable(Protocol): + def flush(self) -> None: ... + def close(self) -> None: ... + def write(self, b: bytes, /) -> int: ... + +class ZipFile: + filename: str | None + debug: int + comment: bytes + filelist: list[ZipInfo] + fp: IO[bytes] | None + NameToInfo: dict[str, ZipInfo] + start_dir: int # undocumented + compression: int # undocumented + compresslevel: int | None # undocumented + mode: _ZipFileMode # undocumented + pwd: bytes | None # undocumented + # metadata_encoding is new in 3.11 + if sys.version_info >= (3, 11): + @overload + def __init__( + self, + file: StrPath | IO[bytes], + mode: _ZipFileMode = "r", + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + metadata_encoding: str | None = None, + ) -> None: ... + # metadata_encoding is only allowed for read mode + @overload + def __init__( + self, + file: StrPath | _ZipReadable, + mode: Literal["r"] = "r", + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + metadata_encoding: str | None = None, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipWritable, + mode: Literal["w", "x"] = ..., + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + metadata_encoding: None = None, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipReadableTellable, + mode: Literal["a"] = ..., + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + metadata_encoding: None = None, + ) -> None: ... + else: + @overload + def __init__( + self, + file: StrPath | IO[bytes], + mode: _ZipFileMode = "r", + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipReadable, + mode: Literal["r"] = "r", + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipWritable, + mode: Literal["w", "x"] = ..., + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipReadableTellable, + mode: Literal["a"] = ..., + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + ) -> None: ... + + def __enter__(self) -> Self: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None + ) -> None: ... + def close(self) -> None: ... + def getinfo(self, name: str) -> ZipInfo: ... + def infolist(self) -> list[ZipInfo]: ... + def namelist(self) -> list[str]: ... + def open( + self, name: str | ZipInfo, mode: _ReadWriteMode = "r", pwd: bytes | None = None, *, force_zip64: bool = False + ) -> IO[bytes]: ... + def extract(self, member: str | ZipInfo, path: StrPath | None = None, pwd: bytes | None = None) -> str: ... + def extractall( + self, path: StrPath | None = None, members: Iterable[str | ZipInfo] | None = None, pwd: bytes | None = None + ) -> None: ... + def printdir(self, file: _Writer | None = None) -> None: ... + def setpassword(self, pwd: bytes) -> None: ... + def read(self, name: str | ZipInfo, pwd: bytes | None = None) -> bytes: ... + def testzip(self) -> str | None: ... + def write( + self, + filename: StrPath, + arcname: StrPath | None = None, + compress_type: int | None = None, + compresslevel: int | None = None, + ) -> None: ... + def writestr( + self, + zinfo_or_arcname: str | ZipInfo, + data: SizedBuffer | str, + compress_type: int | None = None, + compresslevel: int | None = None, + ) -> None: ... + if sys.version_info >= (3, 11): + def mkdir(self, zinfo_or_directory_name: str | ZipInfo, mode: int = 0o777) -> None: ... + + def __del__(self) -> None: ... + +class PyZipFile(ZipFile): + def __init__( + self, file: str | IO[bytes], mode: _ZipFileMode = "r", compression: int = 0, allowZip64: bool = True, optimize: int = -1 + ) -> None: ... + def writepy(self, pathname: str, basename: str = "", filterfunc: Callable[[str], bool] | None = None) -> None: ... + +class ZipInfo: + filename: str + date_time: _DateTuple + compress_type: int + comment: bytes + extra: bytes + create_system: int + create_version: int + extract_version: int + reserved: int + flag_bits: int + volume: int + internal_attr: int + external_attr: int + header_offset: int + CRC: int + compress_size: int + file_size: int + orig_filename: str # undocumented + if sys.version_info >= (3, 13): + compress_level: int | None + + def __init__(self, filename: str = "NoName", date_time: _DateTuple = (1980, 1, 1, 0, 0, 0)) -> None: ... + @classmethod + def from_file(cls, filename: StrPath, arcname: StrPath | None = None, *, strict_timestamps: bool = True) -> Self: ... + def is_dir(self) -> bool: ... + def FileHeader(self, zip64: bool | None = None) -> bytes: ... + +if sys.version_info >= (3, 12): + from zipfile._path import CompleteDirs as CompleteDirs, Path as Path + +else: + class CompleteDirs(ZipFile): + def resolve_dir(self, name: str) -> str: ... + @overload + @classmethod + def make(cls, source: ZipFile) -> CompleteDirs: ... + @overload + @classmethod + def make(cls, source: StrPath | IO[bytes]) -> Self: ... + + class Path: + root: CompleteDirs + at: str + def __init__(self, root: ZipFile | StrPath | IO[bytes], at: str = "") -> None: ... + @property + def name(self) -> str: ... + @property + def parent(self) -> PathLike[str]: ... # undocumented + if sys.version_info >= (3, 10): + @property + def filename(self) -> PathLike[str]: ... # undocumented + if sys.version_info >= (3, 11): + @property + def suffix(self) -> str: ... + @property + def suffixes(self) -> list[str]: ... + @property + def stem(self) -> str: ... + + if sys.version_info >= (3, 9): + @overload + def open( + self, + mode: Literal["r", "w"] = "r", + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool = ..., + write_through: bool = ..., + *, + pwd: bytes | None = None, + ) -> TextIOWrapper: ... + @overload + def open(self, mode: Literal["rb", "wb"], *, pwd: bytes | None = None) -> IO[bytes]: ... + else: + def open( + self, mode: _ReadWriteBinaryMode = "r", pwd: bytes | None = None, *, force_zip64: bool = False + ) -> IO[bytes]: ... + + if sys.version_info >= (3, 10): + def iterdir(self) -> Iterator[Self]: ... + else: + def iterdir(self) -> Iterator[Path]: ... + + def is_dir(self) -> bool: ... + def is_file(self) -> bool: ... + def exists(self) -> bool: ... + def read_text( + self, + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + line_buffering: bool = ..., + write_through: bool = ..., + ) -> str: ... + def read_bytes(self) -> bytes: ... + if sys.version_info >= (3, 10): + def joinpath(self, *other: StrPath) -> Path: ... + else: + def joinpath(self, add: StrPath) -> Path: ... # undocumented + if sys.version_info >= (3, 12): + def glob(self, pattern: str) -> Iterator[Self]: ... + def rglob(self, pattern: str) -> Iterator[Self]: ... + def is_symlink(self) -> Literal[False]: ... + def relative_to(self, other: Path, *extra: StrPath) -> str: ... + def match(self, path_pattern: str) -> bool: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + + def __truediv__(self, add: StrPath) -> Path: ... + +def is_zipfile(filename: StrOrBytesPath | _SupportsReadSeekTell) -> bool: ... + +ZIP_STORED: Final[int] +ZIP_DEFLATED: Final[int] +ZIP64_LIMIT: Final[int] +ZIP_FILECOUNT_LIMIT: Final[int] +ZIP_MAX_COMMENT: Final[int] +ZIP_BZIP2: Final[int] +ZIP_LZMA: Final[int] diff --git a/mypy/typeshed/stdlib/zipfile/_path.pyi b/mypy/typeshed/stdlib/zipfile/_path.pyi new file mode 100644 index 000000000000..a7248ba7ab72 --- /dev/null +++ b/mypy/typeshed/stdlib/zipfile/_path.pyi @@ -0,0 +1,103 @@ +import sys +from _typeshed import StrPath +from collections.abc import Iterator, Sequence +from io import TextIOWrapper +from os import PathLike +from typing import IO, Literal, TypeVar, overload +from typing_extensions import Self, TypeAlias +from zipfile import ZipFile + +_ReadWriteBinaryMode: TypeAlias = Literal["r", "w", "rb", "wb"] + +_ZF = TypeVar("_ZF", bound=ZipFile) + +if sys.version_info >= (3, 12): + __all__ = ["Path"] + + class InitializedState: + def __init__(self, *args: object, **kwargs: object) -> None: ... + def __getstate__(self) -> tuple[list[object], dict[object, object]]: ... + def __setstate__(self, state: Sequence[tuple[list[object], dict[object, object]]]) -> None: ... + + class CompleteDirs(InitializedState, ZipFile): + def resolve_dir(self, name: str) -> str: ... + @overload + @classmethod + def make(cls, source: ZipFile) -> CompleteDirs: ... + @overload + @classmethod + def make(cls, source: StrPath | IO[bytes]) -> Self: ... + if sys.version_info >= (3, 13): + @classmethod + def inject(cls, zf: _ZF) -> _ZF: ... + + class Path: + root: CompleteDirs + at: str + def __init__(self, root: ZipFile | StrPath | IO[bytes], at: str = "") -> None: ... + @property + def name(self) -> str: ... + @property + def parent(self) -> PathLike[str]: ... # undocumented + if sys.version_info >= (3, 10): + @property + def filename(self) -> PathLike[str]: ... # undocumented + if sys.version_info >= (3, 11): + @property + def suffix(self) -> str: ... + @property + def suffixes(self) -> list[str]: ... + @property + def stem(self) -> str: ... + + if sys.version_info >= (3, 9): + @overload + def open( + self, + mode: Literal["r", "w"] = "r", + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool = ..., + write_through: bool = ..., + *, + pwd: bytes | None = None, + ) -> TextIOWrapper: ... + @overload + def open(self, mode: Literal["rb", "wb"], *, pwd: bytes | None = None) -> IO[bytes]: ... + else: + def open( + self, mode: _ReadWriteBinaryMode = "r", pwd: bytes | None = None, *, force_zip64: bool = False + ) -> IO[bytes]: ... + + if sys.version_info >= (3, 10): + def iterdir(self) -> Iterator[Self]: ... + else: + def iterdir(self) -> Iterator[Path]: ... + + def is_dir(self) -> bool: ... + def is_file(self) -> bool: ... + def exists(self) -> bool: ... + def read_text( + self, + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + line_buffering: bool = ..., + write_through: bool = ..., + ) -> str: ... + def read_bytes(self) -> bytes: ... + if sys.version_info >= (3, 10): + def joinpath(self, *other: StrPath) -> Path: ... + else: + def joinpath(self, add: StrPath) -> Path: ... # undocumented + if sys.version_info >= (3, 12): + def glob(self, pattern: str) -> Iterator[Self]: ... + def rglob(self, pattern: str) -> Iterator[Self]: ... + def is_symlink(self) -> Literal[False]: ... + def relative_to(self, other: Path, *extra: StrPath) -> str: ... + def match(self, path_pattern: str) -> bool: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + + def __truediv__(self, add: StrPath) -> Path: ... diff --git a/mypy/typeshed/stdlib/zipimport.pyi b/mypy/typeshed/stdlib/zipimport.pyi new file mode 100644 index 000000000000..3e94c681b7a2 --- /dev/null +++ b/mypy/typeshed/stdlib/zipimport.pyi @@ -0,0 +1,41 @@ +import sys +from _typeshed import StrOrBytesPath +from importlib.abc import ResourceReader +from importlib.machinery import ModuleSpec +from types import CodeType, ModuleType +from typing_extensions import deprecated + +if sys.version_info >= (3, 10): + from _frozen_importlib_external import _LoaderBasics +else: + _LoaderBasics = object + +__all__ = ["ZipImportError", "zipimporter"] + +class ZipImportError(ImportError): ... + +class zipimporter(_LoaderBasics): + archive: str + prefix: str + if sys.version_info >= (3, 11): + def __init__(self, path: str) -> None: ... + else: + def __init__(self, path: StrOrBytesPath) -> None: ... + + if sys.version_info < (3, 12): + def find_loader(self, fullname: str, path: str | None = None) -> tuple[zipimporter | None, list[str]]: ... # undocumented + def find_module(self, fullname: str, path: str | None = None) -> zipimporter | None: ... + + def get_code(self, fullname: str) -> CodeType: ... + def get_data(self, pathname: str) -> bytes: ... + def get_filename(self, fullname: str) -> str: ... + def get_resource_reader(self, fullname: str) -> ResourceReader | None: ... # undocumented + def get_source(self, fullname: str) -> str | None: ... + def is_package(self, fullname: str) -> bool: ... + @deprecated("Deprecated since 3.10; use exec_module() instead") + def load_module(self, fullname: str) -> ModuleType: ... + if sys.version_info >= (3, 10): + def exec_module(self, module: ModuleType) -> None: ... + def create_module(self, spec: ModuleSpec) -> None: ... + def find_spec(self, fullname: str, target: ModuleType | None = None) -> ModuleSpec | None: ... + def invalidate_caches(self) -> None: ... diff --git a/mypy/typeshed/stdlib/zlib.pyi b/mypy/typeshed/stdlib/zlib.pyi new file mode 100644 index 000000000000..7cafb44b34a7 --- /dev/null +++ b/mypy/typeshed/stdlib/zlib.pyi @@ -0,0 +1,70 @@ +import sys +from _typeshed import ReadableBuffer +from typing import Any, Final, final, type_check_only +from typing_extensions import Self + +DEFLATED: Final = 8 +DEF_MEM_LEVEL: int # can change +DEF_BUF_SIZE: Final = 16384 +MAX_WBITS: int +ZLIB_VERSION: str # can change +ZLIB_RUNTIME_VERSION: str # can change +Z_NO_COMPRESSION: Final = 0 +Z_PARTIAL_FLUSH: Final = 1 +Z_BEST_COMPRESSION: Final = 9 +Z_BEST_SPEED: Final = 1 +Z_BLOCK: Final = 5 +Z_DEFAULT_COMPRESSION: Final = -1 +Z_DEFAULT_STRATEGY: Final = 0 +Z_FILTERED: Final = 1 +Z_FINISH: Final = 4 +Z_FIXED: Final = 4 +Z_FULL_FLUSH: Final = 3 +Z_HUFFMAN_ONLY: Final = 2 +Z_NO_FLUSH: Final = 0 +Z_RLE: Final = 3 +Z_SYNC_FLUSH: Final = 2 +Z_TREES: Final = 6 + +class error(Exception): ... + +# This class is not exposed at runtime. It calls itself zlib.Compress. +@final +@type_check_only +class _Compress: + def __copy__(self) -> Self: ... + def __deepcopy__(self, memo: Any, /) -> Self: ... + def compress(self, data: ReadableBuffer, /) -> bytes: ... + def flush(self, mode: int = 4, /) -> bytes: ... + def copy(self) -> _Compress: ... + +# This class is not exposed at runtime. It calls itself zlib.Decompress. +@final +@type_check_only +class _Decompress: + @property + def unused_data(self) -> bytes: ... + @property + def unconsumed_tail(self) -> bytes: ... + @property + def eof(self) -> bool: ... + def __copy__(self) -> Self: ... + def __deepcopy__(self, memo: Any, /) -> Self: ... + def decompress(self, data: ReadableBuffer, /, max_length: int = 0) -> bytes: ... + def flush(self, length: int = 16384, /) -> bytes: ... + def copy(self) -> _Decompress: ... + +def adler32(data: ReadableBuffer, value: int = 1, /) -> int: ... + +if sys.version_info >= (3, 11): + def compress(data: ReadableBuffer, /, level: int = -1, wbits: int = 15) -> bytes: ... + +else: + def compress(data: ReadableBuffer, /, level: int = -1) -> bytes: ... + +def compressobj( + level: int = -1, method: int = 8, wbits: int = 15, memLevel: int = 8, strategy: int = 0, zdict: ReadableBuffer | None = None +) -> _Compress: ... +def crc32(data: ReadableBuffer, value: int = 0, /) -> int: ... +def decompress(data: ReadableBuffer, /, wbits: int = 15, bufsize: int = 16384) -> bytes: ... +def decompressobj(wbits: int = 15, zdict: ReadableBuffer = b"") -> _Decompress: ... diff --git a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi new file mode 100644 index 000000000000..cc483afad9ff --- /dev/null +++ b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi @@ -0,0 +1,35 @@ +import sys +from collections.abc import Iterable +from datetime import datetime, timedelta, tzinfo +from typing_extensions import Self + +# TODO: remove this version check +# In theory we shouldn't need this version check. Pyright complains about the imports +# from zoneinfo.* when run on 3.8 and 3.7 without this. Updates to typeshed's +# pyright test script are probably needed, see #11189 +if sys.version_info >= (3, 9): + from zoneinfo._common import ZoneInfoNotFoundError as ZoneInfoNotFoundError, _IOBytes + from zoneinfo._tzpath import ( + TZPATH as TZPATH, + InvalidTZPathWarning as InvalidTZPathWarning, + available_timezones as available_timezones, + reset_tzpath as reset_tzpath, + ) + + __all__ = ["ZoneInfo", "reset_tzpath", "available_timezones", "TZPATH", "ZoneInfoNotFoundError", "InvalidTZPathWarning"] + + class ZoneInfo(tzinfo): + @property + def key(self) -> str: ... + def __init__(self, key: str) -> None: ... + @classmethod + def no_cache(cls, key: str) -> Self: ... + @classmethod + def from_file(cls, fobj: _IOBytes, /, key: str | None = None) -> Self: ... + @classmethod + def clear_cache(cls, *, only_keys: Iterable[str] | None = None) -> None: ... + def tzname(self, dt: datetime | None, /) -> str | None: ... + def utcoffset(self, dt: datetime | None, /) -> timedelta | None: ... + def dst(self, dt: datetime | None, /) -> timedelta | None: ... + + def __dir__() -> list[str]: ... diff --git a/mypy/typeshed/stdlib/zoneinfo/_common.pyi b/mypy/typeshed/stdlib/zoneinfo/_common.pyi new file mode 100644 index 000000000000..a2f29f2d14f0 --- /dev/null +++ b/mypy/typeshed/stdlib/zoneinfo/_common.pyi @@ -0,0 +1,13 @@ +import io +from typing import Any, Protocol + +class _IOBytes(Protocol): + def read(self, size: int, /) -> bytes: ... + def seek(self, size: int, whence: int = ..., /) -> Any: ... + +def load_tzdata(key: str) -> io.BufferedReader: ... +def load_data( + fobj: _IOBytes, +) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[str, ...], bytes | None]: ... + +class ZoneInfoNotFoundError(KeyError): ... diff --git a/mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi b/mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi new file mode 100644 index 000000000000..0ef78d03e5f4 --- /dev/null +++ b/mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi @@ -0,0 +1,13 @@ +from _typeshed import StrPath +from collections.abc import Sequence + +# Note: Both here and in clear_cache, the types allow the use of `str` where +# a sequence of strings is required. This should be remedied if a solution +# to this typing bug is found: https://github.com/python/typing/issues/256 +def reset_tzpath(to: Sequence[StrPath] | None = None) -> None: ... +def find_tzfile(key: str) -> str | None: ... +def available_timezones() -> set[str]: ... + +TZPATH: tuple[str, ...] + +class InvalidTZPathWarning(RuntimeWarning): ... diff --git a/mypy/typeshed/stubs/mypy-extensions/@tests/stubtest_allowlist.txt b/mypy/typeshed/stubs/mypy-extensions/@tests/stubtest_allowlist.txt new file mode 100644 index 000000000000..bffaebc697dc --- /dev/null +++ b/mypy/typeshed/stubs/mypy-extensions/@tests/stubtest_allowlist.txt @@ -0,0 +1,2 @@ +mypy_extensions.FlexibleAlias +mypy_extensions.TypedDict diff --git a/mypy/typeshed/stubs/mypy-extensions/METADATA.toml b/mypy/typeshed/stubs/mypy-extensions/METADATA.toml new file mode 100644 index 000000000000..516f11f6b9e2 --- /dev/null +++ b/mypy/typeshed/stubs/mypy-extensions/METADATA.toml @@ -0,0 +1,4 @@ +version = "1.0.*" + +[tool.stubtest] +ignore_missing_stub = false diff --git a/mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi b/mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi new file mode 100644 index 000000000000..b6358a0022f3 --- /dev/null +++ b/mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi @@ -0,0 +1,218 @@ +# These stubs are forked from typeshed, since we use some definitions that only make +# sense in the context of mypy/mypyc (in particular, native int types such as i64). + +import abc +import sys +from _collections_abc import dict_items, dict_keys, dict_values +from _typeshed import IdentityFunction, Self +from collections.abc import Mapping +from typing import Any, ClassVar, Generic, SupportsInt, TypeVar, overload, type_check_only +from typing_extensions import Never, SupportsIndex +from _typeshed import ReadableBuffer, SupportsTrunc + +_T = TypeVar("_T") +_U = TypeVar("_U") + +# Internal mypy fallback type for all typed dicts (does not exist at runtime) +# N.B. Keep this mostly in sync with typing(_extensions)._TypedDict +@type_check_only +class _TypedDict(Mapping[str, object], metaclass=abc.ABCMeta): + __total__: ClassVar[bool] + # Unlike typing(_extensions).TypedDict, + # subclasses of mypy_extensions.TypedDict do NOT have the __required_keys__ and __optional_keys__ ClassVars + def copy(self: Self) -> Self: ... + # Using Never so that only calls using mypy plugin hook that specialize the signature + # can go through. + def setdefault(self, k: Never, default: object) -> object: ... + # Mypy plugin hook for 'pop' expects that 'default' has a type variable type. + def pop(self, k: Never, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] + def update(self: Self, __m: Self) -> None: ... + def items(self) -> dict_items[str, object]: ... + def keys(self) -> dict_keys[str, object]: ... + def values(self) -> dict_values[str, object]: ... + def __delitem__(self, k: Never) -> None: ... + if sys.version_info >= (3, 9): + def __or__(self: Self, __other: Self) -> Self: ... + def __ior__(self: Self, __other: Self) -> Self: ... + +def TypedDict(typename: str, fields: dict[str, type[Any]], total: bool = ...) -> type[dict[str, Any]]: ... +@overload +def Arg(type: _T, name: str | None = ...) -> _T: ... +@overload +def Arg(*, name: str | None = ...) -> Any: ... +@overload +def DefaultArg(type: _T, name: str | None = ...) -> _T: ... +@overload +def DefaultArg(*, name: str | None = ...) -> Any: ... +@overload +def NamedArg(type: _T, name: str | None = ...) -> _T: ... +@overload +def NamedArg(*, name: str | None = ...) -> Any: ... +@overload +def DefaultNamedArg(type: _T, name: str | None = ...) -> _T: ... +@overload +def DefaultNamedArg(*, name: str | None = ...) -> Any: ... +@overload +def VarArg(type: _T) -> _T: ... +@overload +def VarArg() -> Any: ... +@overload +def KwArg(type: _T) -> _T: ... +@overload +def KwArg() -> Any: ... + +# Return type that indicates a function does not return. +# Deprecated: Use typing.NoReturn instead. +class NoReturn: ... + +# This is consistent with implementation. Usage intends for this as +# a class decorator, but mypy does not support type[_T] for abstract +# classes until this issue is resolved, https://github.com/python/mypy/issues/4717. +def trait(cls: _T) -> _T: ... +def mypyc_attr(*attrs: str, **kwattrs: object) -> IdentityFunction: ... + +class FlexibleAlias(Generic[_T, _U]): ... + +# Native int types such as i64 are magical and support implicit +# coercions to/from int using special logic in mypy. We generally only +# include operations here for which we have specialized primitives. + +class i64: + @overload + def __new__(cls, __x: str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> i64: ... + @overload + def __new__(cls, __x: str | bytes | bytearray, base: SupportsIndex) -> i64: ... + + def __add__(self, x: i64) -> i64: ... + def __radd__(self, x: i64) -> i64: ... + def __sub__(self, x: i64) -> i64: ... + def __rsub__(self, x: i64) -> i64: ... + def __mul__(self, x: i64) -> i64: ... + def __rmul__(self, x: i64) -> i64: ... + def __floordiv__(self, x: i64) -> i64: ... + def __rfloordiv__(self, x: i64) -> i64: ... + def __mod__(self, x: i64) -> i64: ... + def __rmod__(self, x: i64) -> i64: ... + def __and__(self, x: i64) -> i64: ... + def __rand__(self, x: i64) -> i64: ... + def __or__(self, x: i64) -> i64: ... + def __ror__(self, x: i64) -> i64: ... + def __xor__(self, x: i64) -> i64: ... + def __rxor__(self, x: i64) -> i64: ... + def __lshift__(self, x: i64) -> i64: ... + def __rlshift__(self, x: i64) -> i64: ... + def __rshift__(self, x: i64) -> i64: ... + def __rrshift__(self, x: i64) -> i64: ... + def __neg__(self) -> i64: ... + def __invert__(self) -> i64: ... + def __pos__(self) -> i64: ... + def __lt__(self, x: i64) -> bool: ... + def __le__(self, x: i64) -> bool: ... + def __ge__(self, x: i64) -> bool: ... + def __gt__(self, x: i64) -> bool: ... + def __index__(self) -> int: ... + +class i32: + @overload + def __new__(cls, __x: str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> i32: ... + @overload + def __new__(cls, __x: str | bytes | bytearray, base: SupportsIndex) -> i32: ... + + def __add__(self, x: i32) -> i32: ... + def __radd__(self, x: i32) -> i32: ... + def __sub__(self, x: i32) -> i32: ... + def __rsub__(self, x: i32) -> i32: ... + def __mul__(self, x: i32) -> i32: ... + def __rmul__(self, x: i32) -> i32: ... + def __floordiv__(self, x: i32) -> i32: ... + def __rfloordiv__(self, x: i32) -> i32: ... + def __mod__(self, x: i32) -> i32: ... + def __rmod__(self, x: i32) -> i32: ... + def __and__(self, x: i32) -> i32: ... + def __rand__(self, x: i32) -> i32: ... + def __or__(self, x: i32) -> i32: ... + def __ror__(self, x: i32) -> i32: ... + def __xor__(self, x: i32) -> i32: ... + def __rxor__(self, x: i32) -> i32: ... + def __lshift__(self, x: i32) -> i32: ... + def __rlshift__(self, x: i32) -> i32: ... + def __rshift__(self, x: i32) -> i32: ... + def __rrshift__(self, x: i32) -> i32: ... + def __neg__(self) -> i32: ... + def __invert__(self) -> i32: ... + def __pos__(self) -> i32: ... + def __lt__(self, x: i32) -> bool: ... + def __le__(self, x: i32) -> bool: ... + def __ge__(self, x: i32) -> bool: ... + def __gt__(self, x: i32) -> bool: ... + def __index__(self) -> int: ... + +class i16: + @overload + def __new__(cls, __x: str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> i16: ... + @overload + def __new__(cls, __x: str | bytes | bytearray, base: SupportsIndex) -> i16: ... + + def __add__(self, x: i16) -> i16: ... + def __radd__(self, x: i16) -> i16: ... + def __sub__(self, x: i16) -> i16: ... + def __rsub__(self, x: i16) -> i16: ... + def __mul__(self, x: i16) -> i16: ... + def __rmul__(self, x: i16) -> i16: ... + def __floordiv__(self, x: i16) -> i16: ... + def __rfloordiv__(self, x: i16) -> i16: ... + def __mod__(self, x: i16) -> i16: ... + def __rmod__(self, x: i16) -> i16: ... + def __and__(self, x: i16) -> i16: ... + def __rand__(self, x: i16) -> i16: ... + def __or__(self, x: i16) -> i16: ... + def __ror__(self, x: i16) -> i16: ... + def __xor__(self, x: i16) -> i16: ... + def __rxor__(self, x: i16) -> i16: ... + def __lshift__(self, x: i16) -> i16: ... + def __rlshift__(self, x: i16) -> i16: ... + def __rshift__(self, x: i16) -> i16: ... + def __rrshift__(self, x: i16) -> i16: ... + def __neg__(self) -> i16: ... + def __invert__(self) -> i16: ... + def __pos__(self) -> i16: ... + def __lt__(self, x: i16) -> bool: ... + def __le__(self, x: i16) -> bool: ... + def __ge__(self, x: i16) -> bool: ... + def __gt__(self, x: i16) -> bool: ... + def __index__(self) -> int: ... + +class u8: + @overload + def __new__(cls, __x: str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> u8: ... + @overload + def __new__(cls, __x: str | bytes | bytearray, base: SupportsIndex) -> u8: ... + + def __add__(self, x: u8) -> u8: ... + def __radd__(self, x: u8) -> u8: ... + def __sub__(self, x: u8) -> u8: ... + def __rsub__(self, x: u8) -> u8: ... + def __mul__(self, x: u8) -> u8: ... + def __rmul__(self, x: u8) -> u8: ... + def __floordiv__(self, x: u8) -> u8: ... + def __rfloordiv__(self, x: u8) -> u8: ... + def __mod__(self, x: u8) -> u8: ... + def __rmod__(self, x: u8) -> u8: ... + def __and__(self, x: u8) -> u8: ... + def __rand__(self, x: u8) -> u8: ... + def __or__(self, x: u8) -> u8: ... + def __ror__(self, x: u8) -> u8: ... + def __xor__(self, x: u8) -> u8: ... + def __rxor__(self, x: u8) -> u8: ... + def __lshift__(self, x: u8) -> u8: ... + def __rlshift__(self, x: u8) -> u8: ... + def __rshift__(self, x: u8) -> u8: ... + def __rrshift__(self, x: u8) -> u8: ... + def __neg__(self) -> u8: ... + def __invert__(self) -> u8: ... + def __pos__(self) -> u8: ... + def __lt__(self, x: u8) -> bool: ... + def __le__(self, x: u8) -> bool: ... + def __ge__(self, x: u8) -> bool: ... + def __gt__(self, x: u8) -> bool: ... + def __index__(self) -> int: ... diff --git a/mypy/typestate.py b/mypy/typestate.py new file mode 100644 index 000000000000..0082c5564705 --- /dev/null +++ b/mypy/typestate.py @@ -0,0 +1,329 @@ +""" +A shared state for all TypeInfos that holds global cache and dependency information, +and potentially other mutable TypeInfo state. This module contains mutable global state. +""" + +from __future__ import annotations + +from typing import Dict, Final, Set, Tuple +from typing_extensions import TypeAlias as _TypeAlias + +from mypy.nodes import VARIANCE_NOT_READY, TypeInfo +from mypy.server.trigger import make_trigger +from mypy.types import Instance, Type, TypeVarId, TypeVarType, get_proper_type + +MAX_NEGATIVE_CACHE_TYPES: Final = 1000 +MAX_NEGATIVE_CACHE_ENTRIES: Final = 10000 + +# Represents that the 'left' instance is a subtype of the 'right' instance +SubtypeRelationship: _TypeAlias = Tuple[Instance, Instance] + +# A tuple encoding the specific conditions under which we performed the subtype check. +# (e.g. did we want a proper subtype? A regular subtype while ignoring variance?) +SubtypeKind: _TypeAlias = Tuple[bool, ...] + +# A cache that keeps track of whether the given TypeInfo is a part of a particular +# subtype relationship +SubtypeCache: _TypeAlias = Dict[TypeInfo, Dict[SubtypeKind, Set[SubtypeRelationship]]] + + +class TypeState: + """This class provides subtype caching to improve performance of subtype checks. + It also holds protocol fine grained dependencies. + + Note: to avoid leaking global state, 'reset_all_subtype_caches()' should be called + after a build has finished and after a daemon shutdown. This subtype cache only exists for + performance reasons, resetting subtype caches for a class has no semantic effect. + The protocol dependencies however are only stored here, and shouldn't be deleted unless + not needed any more (e.g. during daemon shutdown). + """ + + # '_subtype_caches' keeps track of (subtype, supertype) pairs where supertypes are + # instances of the given TypeInfo. The cache also keeps track of whether the check + # was done in strict optional mode and of the specific *kind* of subtyping relationship, + # which we represent as an arbitrary hashable tuple. + # We need the caches, since subtype checks for structural types are very slow. + _subtype_caches: Final[SubtypeCache] + + # Same as above but for negative subtyping results. + _negative_subtype_caches: Final[SubtypeCache] + + # This contains protocol dependencies generated after running a full build, + # or after an update. These dependencies are special because: + # * They are a global property of the program; i.e. some dependencies for imported + # classes can be generated in the importing modules. + # * Because of the above, they are serialized separately, after a full run, + # or a full update. + # `proto_deps` can be None if after deserialization it turns out that they are + # inconsistent with the other cache files (or an error occurred during deserialization). + # A blocking error will be generated in this case, since we can't proceed safely. + # For the description of kinds of protocol dependencies and corresponding examples, + # see _snapshot_protocol_deps. + proto_deps: dict[str, set[str]] | None + + # Protocols (full names) a given class attempted to implement. + # Used to calculate fine grained protocol dependencies and optimize protocol + # subtype cache invalidation in fine grained mode. For example, if we pass a value + # of type a.A to a function expecting something compatible with protocol p.P, + # we'd have 'a.A' -> {'p.P', ...} in the map. This map is flushed after every incremental + # update. + _attempted_protocols: Final[dict[str, set[str]]] + # We also snapshot protocol members of the above protocols. For example, if we pass + # a value of type a.A to a function expecting something compatible with Iterable, we'd have + # 'a.A' -> {'__iter__', ...} in the map. This map is also flushed after every incremental + # update. This map is needed to only generate dependencies like -> + # instead of a wildcard to avoid unnecessarily invalidating classes. + _checked_against_members: Final[dict[str, set[str]]] + # TypeInfos that appeared as a left type (subtype) in a subtype check since latest + # dependency snapshot update. This is an optimisation for fine grained mode; during a full + # run we only take a dependency snapshot at the very end, so this set will contain all + # subtype-checked TypeInfos. After a fine grained update however, we can gather only new + # dependencies generated from (typically) few TypeInfos that were subtype-checked + # (i.e. appeared as r.h.s. in an assignment or an argument in a function call in + # a re-checked target) during the update. + _rechecked_types: Final[set[TypeInfo]] + + # The two attributes below are assumption stacks for subtyping relationships between + # recursive type aliases. Normally, one would pass type assumptions as an additional + # arguments to is_subtype(), but this would mean updating dozens of related functions + # threading this through all callsites (see also comment for TypeInfo.assuming). + _assuming: Final[list[tuple[Type, Type]]] + _assuming_proper: Final[list[tuple[Type, Type]]] + # Ditto for inference of generic constraints against recursive type aliases. + inferring: Final[list[tuple[Type, Type]]] + # Whether to use joins or unions when solving constraints, see checkexpr.py for details. + infer_unions: bool + # Whether to use new type inference algorithm that can infer polymorphic types. + # This is temporary and will be removed soon when new algorithm is more polished. + infer_polymorphic: bool + + # N.B: We do all of the accesses to these properties through + # TypeState, instead of making these classmethods and accessing + # via the cls parameter, since mypyc can optimize accesses to + # Final attributes of a directly referenced type. + + def __init__(self) -> None: + self._subtype_caches = {} + self._negative_subtype_caches = {} + self.proto_deps = {} + self._attempted_protocols = {} + self._checked_against_members = {} + self._rechecked_types = set() + self._assuming = [] + self._assuming_proper = [] + self.inferring = [] + self.infer_unions = False + self.infer_polymorphic = False + + def is_assumed_subtype(self, left: Type, right: Type) -> bool: + for l, r in reversed(self._assuming): + if get_proper_type(l) == get_proper_type(left) and get_proper_type( + r + ) == get_proper_type(right): + return True + return False + + def is_assumed_proper_subtype(self, left: Type, right: Type) -> bool: + for l, r in reversed(self._assuming_proper): + if get_proper_type(l) == get_proper_type(left) and get_proper_type( + r + ) == get_proper_type(right): + return True + return False + + def get_assumptions(self, is_proper: bool) -> list[tuple[Type, Type]]: + if is_proper: + return self._assuming_proper + return self._assuming + + def reset_all_subtype_caches(self) -> None: + """Completely reset all known subtype caches.""" + self._subtype_caches.clear() + self._negative_subtype_caches.clear() + + def reset_subtype_caches_for(self, info: TypeInfo) -> None: + """Reset subtype caches (if any) for a given supertype TypeInfo.""" + if info in self._subtype_caches: + self._subtype_caches[info].clear() + if info in self._negative_subtype_caches: + self._negative_subtype_caches[info].clear() + + def reset_all_subtype_caches_for(self, info: TypeInfo) -> None: + """Reset subtype caches (if any) for a given supertype TypeInfo and its MRO.""" + for item in info.mro: + self.reset_subtype_caches_for(item) + + def is_cached_subtype_check(self, kind: SubtypeKind, left: Instance, right: Instance) -> bool: + if left.last_known_value is not None or right.last_known_value is not None: + # If there is a literal last known value, give up. There + # will be an unbounded number of potential types to cache, + # making caching less effective. + return False + info = right.type + cache = self._subtype_caches.get(info) + if cache is None: + return False + subcache = cache.get(kind) + if subcache is None: + return False + return (left, right) in subcache + + def is_cached_negative_subtype_check( + self, kind: SubtypeKind, left: Instance, right: Instance + ) -> bool: + if left.last_known_value is not None or right.last_known_value is not None: + # If there is a literal last known value, give up. There + # will be an unbounded number of potential types to cache, + # making caching less effective. + return False + info = right.type + cache = self._negative_subtype_caches.get(info) + if cache is None: + return False + subcache = cache.get(kind) + if subcache is None: + return False + return (left, right) in subcache + + def record_subtype_cache_entry( + self, kind: SubtypeKind, left: Instance, right: Instance + ) -> None: + if left.last_known_value is not None or right.last_known_value is not None: + # These are unlikely to match, due to the large space of + # possible values. Avoid uselessly increasing cache sizes. + return + if any( + (isinstance(tv, TypeVarType) and tv.variance == VARIANCE_NOT_READY) + for tv in right.type.defn.type_vars + ): + # Variance indeterminate -- don't know the result + return + cache = self._subtype_caches.setdefault(right.type, {}) + cache.setdefault(kind, set()).add((left, right)) + + def record_negative_subtype_cache_entry( + self, kind: SubtypeKind, left: Instance, right: Instance + ) -> None: + if left.last_known_value is not None or right.last_known_value is not None: + # These are unlikely to match, due to the large space of + # possible values. Avoid uselessly increasing cache sizes. + return + if len(self._negative_subtype_caches) > MAX_NEGATIVE_CACHE_TYPES: + self._negative_subtype_caches.clear() + cache = self._negative_subtype_caches.setdefault(right.type, {}) + subcache = cache.setdefault(kind, set()) + if len(subcache) > MAX_NEGATIVE_CACHE_ENTRIES: + subcache.clear() + cache.setdefault(kind, set()).add((left, right)) + + def reset_protocol_deps(self) -> None: + """Reset dependencies after a full run or before a daemon shutdown.""" + self.proto_deps = {} + self._attempted_protocols.clear() + self._checked_against_members.clear() + self._rechecked_types.clear() + + def record_protocol_subtype_check(self, left_type: TypeInfo, right_type: TypeInfo) -> None: + assert right_type.is_protocol + self._rechecked_types.add(left_type) + self._attempted_protocols.setdefault(left_type.fullname, set()).add(right_type.fullname) + self._checked_against_members.setdefault(left_type.fullname, set()).update( + right_type.protocol_members + ) + + def _snapshot_protocol_deps(self) -> dict[str, set[str]]: + """Collect protocol attribute dependencies found so far from registered subtype checks. + + There are three kinds of protocol dependencies. For example, after a subtype check: + + x: Proto = C() + + the following dependencies will be generated: + 1. ..., , -> + 2. ..., , -> [for every attr in Proto members] + 3. -> Proto # this one to invalidate the subtype cache + + The first kind is generated immediately per-module in deps.py (see also an example there + for motivation why it is needed). While two other kinds are generated here after all + modules are type checked and we have recorded all the subtype checks. To understand these + two kinds, consider a simple example: + + class A: + def __iter__(self) -> Iterator[int]: + ... + + it: Iterable[int] = A() + + We add -> to invalidate the assignment (module target in this case), + whenever the signature of a.A.__iter__ changes. We also add -> typing.Iterable, + to invalidate the subtype caches of the latter. (Note that the same logic applies to + proper subtype checks, and calculating meets and joins, if this involves calling + 'subtypes.is_protocol_implementation'). + """ + deps: dict[str, set[str]] = {} + for info in self._rechecked_types: + for attr in self._checked_against_members[info.fullname]: + # The need for full MRO here is subtle, during an update, base classes of + # a concrete class may not be reprocessed, so not all -> deps + # are added. + for base_info in info.mro[:-1]: + trigger = make_trigger(f"{base_info.fullname}.{attr}") + if "typing" in trigger or "builtins" in trigger: + # TODO: avoid everything from typeshed + continue + deps.setdefault(trigger, set()).add(make_trigger(info.fullname)) + for proto in self._attempted_protocols[info.fullname]: + trigger = make_trigger(info.fullname) + if "typing" in trigger or "builtins" in trigger: + continue + # If any class that was checked against a protocol changes, + # we need to reset the subtype cache for the protocol. + # + # Note: strictly speaking, the protocol doesn't need to be + # re-checked, we only need to reset the cache, and its uses + # elsewhere are still valid (unless invalidated by other deps). + deps.setdefault(trigger, set()).add(proto) + return deps + + def update_protocol_deps(self, second_map: dict[str, set[str]] | None = None) -> None: + """Update global protocol dependency map. + + We update the global map incrementally, using a snapshot only from recently + type checked types. If second_map is given, update it as well. This is currently used + by FineGrainedBuildManager that maintains normal (non-protocol) dependencies. + """ + assert self.proto_deps is not None, "This should not be called after failed cache load" + new_deps = self._snapshot_protocol_deps() + for trigger, targets in new_deps.items(): + self.proto_deps.setdefault(trigger, set()).update(targets) + if second_map is not None: + for trigger, targets in new_deps.items(): + second_map.setdefault(trigger, set()).update(targets) + self._rechecked_types.clear() + self._attempted_protocols.clear() + self._checked_against_members.clear() + + def add_all_protocol_deps(self, deps: dict[str, set[str]]) -> None: + """Add all known protocol dependencies to deps. + + This is used by tests and debug output, and also when collecting + all collected or loaded dependencies as part of build. + """ + self.update_protocol_deps() # just in case + if self.proto_deps is not None: + for trigger, targets in self.proto_deps.items(): + deps.setdefault(trigger, set()).update(targets) + + +type_state: Final = TypeState() + + +def reset_global_state() -> None: + """Reset most existing global state. + + Currently most of it is in this module. Few exceptions are strict optional status + and functools.lru_cache. + """ + type_state.reset_all_subtype_caches() + type_state.reset_protocol_deps() + TypeVarId.next_raw_id = 1 diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py new file mode 100644 index 000000000000..a28bbf422b61 --- /dev/null +++ b/mypy/typetraverser.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from typing import Iterable + +from mypy_extensions import trait + +from mypy.types import ( + AnyType, + CallableArgument, + CallableType, + DeletedType, + EllipsisType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PartialType, + PlaceholderType, + RawExpressionType, + SyntheticTypeVisitor, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeList, + TypeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, +) + + +@trait +class TypeTraverserVisitor(SyntheticTypeVisitor[None]): + """Visitor that traverses all components of a type""" + + # Atomic types + + def visit_any(self, t: AnyType) -> None: + pass + + def visit_uninhabited_type(self, t: UninhabitedType) -> None: + pass + + def visit_none_type(self, t: NoneType) -> None: + pass + + def visit_erased_type(self, t: ErasedType) -> None: + pass + + def visit_deleted_type(self, t: DeletedType) -> None: + pass + + def visit_type_var(self, t: TypeVarType) -> None: + # Note that type variable values and upper bound aren't treated as + # components, since they are components of the type variable + # definition. We want to traverse everything just once. + t.default.accept(self) + + def visit_param_spec(self, t: ParamSpecType) -> None: + t.default.accept(self) + + def visit_parameters(self, t: Parameters) -> None: + self.traverse_types(t.arg_types) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + t.default.accept(self) + + def visit_literal_type(self, t: LiteralType) -> None: + t.fallback.accept(self) + + # Composite types + + def visit_instance(self, t: Instance) -> None: + self.traverse_types(t.args) + + def visit_callable_type(self, t: CallableType) -> None: + # FIX generics + self.traverse_types(t.arg_types) + t.ret_type.accept(self) + t.fallback.accept(self) + + if t.type_guard is not None: + t.type_guard.accept(self) + + if t.type_is is not None: + t.type_is.accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + self.traverse_types(t.items) + t.partial_fallback.accept(self) + + def visit_typeddict_type(self, t: TypedDictType) -> None: + self.traverse_types(t.items.values()) + t.fallback.accept(self) + + def visit_union_type(self, t: UnionType) -> None: + self.traverse_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> None: + self.traverse_types(t.items) + + def visit_type_type(self, t: TypeType) -> None: + t.item.accept(self) + + # Special types (not real types) + + def visit_callable_argument(self, t: CallableArgument) -> None: + t.typ.accept(self) + + def visit_unbound_type(self, t: UnboundType) -> None: + self.traverse_types(t.args) + + def visit_type_list(self, t: TypeList) -> None: + self.traverse_types(t.items) + + def visit_ellipsis_type(self, t: EllipsisType) -> None: + pass + + def visit_placeholder_type(self, t: PlaceholderType) -> None: + self.traverse_types(t.args) + + def visit_partial_type(self, t: PartialType) -> None: + pass + + def visit_raw_expression_type(self, t: RawExpressionType) -> None: + pass + + def visit_type_alias_type(self, t: TypeAliasType) -> None: + # TODO: sometimes we want to traverse target as well + # We need to find a way to indicate explicitly the intent, + # maybe make this method abstract (like for TypeTranslator)? + self.traverse_types(t.args) + + def visit_unpack_type(self, t: UnpackType) -> None: + t.type.accept(self) + + # Helpers + + def traverse_types(self, types: Iterable[Type]) -> None: + for typ in types: + typ.accept(self) diff --git a/mypy/typevars.py b/mypy/typevars.py new file mode 100644 index 000000000000..e871973104a2 --- /dev/null +++ b/mypy/typevars.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from mypy.erasetype import erase_typevars +from mypy.nodes import TypeInfo +from mypy.types import ( + Instance, + ParamSpecType, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnpackType, +) +from mypy.typevartuples import erased_vars + + +def fill_typevars(typ: TypeInfo) -> Instance | TupleType: + """For a non-generic type, return instance type representing the type. + + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tvs: list[Type] = [] + # TODO: why do we need to keep both typ.type_vars and typ.defn.type_vars? + for i in range(len(typ.defn.type_vars)): + tv: TypeVarLikeType | UnpackType = typ.defn.type_vars[i] + # Change the line number + if isinstance(tv, TypeVarType): + tv = tv.copy_modified(line=-1, column=-1) + elif isinstance(tv, TypeVarTupleType): + tv = UnpackType( + TypeVarTupleType( + tv.name, + tv.fullname, + tv.id, + tv.upper_bound, + tv.tuple_fallback, + tv.default, + line=-1, + column=-1, + ) + ) + else: + assert isinstance(tv, ParamSpecType) + tv = ParamSpecType( + tv.name, + tv.fullname, + tv.id, + tv.flavor, + tv.upper_bound, + tv.default, + line=-1, + column=-1, + ) + tvs.append(tv) + inst = Instance(typ, tvs) + # TODO: do we need to also handle typeddict_type here and below? + if typ.tuple_type is None: + return inst + return typ.tuple_type.copy_modified(fallback=inst) + + +def fill_typevars_with_any(typ: TypeInfo) -> Instance | TupleType: + """Apply a correct number of Any's as type arguments to a type.""" + inst = Instance(typ, erased_vars(typ.defn.type_vars, TypeOfAny.special_form)) + if typ.tuple_type is None: + return inst + erased_tuple_type = erase_typevars(typ.tuple_type, {tv.id for tv in typ.defn.type_vars}) + assert isinstance(erased_tuple_type, ProperType) + if isinstance(erased_tuple_type, TupleType): + return typ.tuple_type.copy_modified(fallback=inst) + return inst + + +def has_no_typevars(typ: Type) -> bool: + # We test if a type contains type variables by erasing all type variables + # and comparing the result to the original type. We use comparison by equality that + # in turn uses `__eq__` defined for types. Note: we can't use `is_same_type` because + # it is not safe with unresolved forward references, while this function may be called + # before forward references resolution patch pass. Note also that it is not safe to use + # `is` comparison because `erase_typevars` doesn't preserve type identity. + return typ == erase_typevars(typ) diff --git a/mypy/typevartuples.py b/mypy/typevartuples.py new file mode 100644 index 000000000000..3bc67dc55ef3 --- /dev/null +++ b/mypy/typevartuples.py @@ -0,0 +1,36 @@ +"""Helpers for interacting with type var tuples.""" + +from __future__ import annotations + +from typing import Sequence + +from mypy.types import ( + AnyType, + Instance, + Type, + TypeVarLikeType, + TypeVarTupleType, + UnpackType, + split_with_prefix_and_suffix, +) + + +def split_with_instance( + typ: Instance, +) -> tuple[tuple[Type, ...], tuple[Type, ...], tuple[Type, ...]]: + assert typ.type.type_var_tuple_prefix is not None + assert typ.type.type_var_tuple_suffix is not None + return split_with_prefix_and_suffix( + typ.args, typ.type.type_var_tuple_prefix, typ.type.type_var_tuple_suffix + ) + + +def erased_vars(type_vars: Sequence[TypeVarLikeType], type_of_any: int) -> list[Type]: + args: list[Type] = [] + for tv in type_vars: + # Valid erasure for *Ts is *tuple[Any, ...], not just Any. + if isinstance(tv, TypeVarTupleType): + args.append(UnpackType(tv.tuple_fallback.copy_modified(args=[AnyType(type_of_any)]))) + else: + args.append(AnyType(type_of_any)) + return args diff --git a/mypy/util.py b/mypy/util.py index 0261432dd979..e0a9cf9ce1b2 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -1,9 +1,112 @@ -"""Utility functions.""" +"""Utility functions with no non-trivial dependencies.""" -from typing import typevar, List, Any +from __future__ import annotations +import hashlib +import io +import json +import os +import re +import shutil +import sys +import time +from importlib import resources as importlib_resources +from typing import IO, Any, Callable, Container, Final, Iterable, Sequence, Sized, TypeVar +from typing_extensions import Literal -T = typevar('T') +orjson: Any +try: + import orjson # type: ignore[import-not-found, no-redef, unused-ignore] +except ImportError: + orjson = None + +try: + import _curses # noqa: F401 + import curses + + CURSES_ENABLED = True +except ImportError: + CURSES_ENABLED = False + +T = TypeVar("T") + +if sys.version_info >= (3, 9): + TYPESHED_DIR: Final = str(importlib_resources.files("mypy") / "typeshed") +else: + with importlib_resources.path( + "mypy", # mypy-c doesn't support __package__ + "py.typed", # a marker file for type information, we assume typeshed to live in the same dir + ) as _resource: + TYPESHED_DIR = str(_resource.parent / "typeshed") + + +ENCODING_RE: Final = re.compile(rb"([ \t\v]*#.*(\r\n?|\n))??[ \t\v]*#.*coding[:=][ \t]*([-\w.]+)") + +DEFAULT_SOURCE_OFFSET: Final = 4 +DEFAULT_COLUMNS: Final = 80 + +# At least this number of columns will be shown on each side of +# error location when printing source code snippet. +MINIMUM_WIDTH: Final = 20 + +# VT100 color code processing was added in Windows 10, but only the second major update, +# Threshold 2. Fortunately, everyone (even on LTSB, Long Term Support Branch) should +# have a version of Windows 10 newer than this. Note that Windows 8 and below are not +# supported, but are either going out of support, or make up only a few % of the market. +MINIMUM_WINDOWS_MAJOR_VT100: Final = 10 +MINIMUM_WINDOWS_BUILD_VT100: Final = 10586 + +SPECIAL_DUNDERS: Final = frozenset( + ("__init__", "__new__", "__call__", "__init_subclass__", "__class_getitem__") +) + + +def is_dunder(name: str, exclude_special: bool = False) -> bool: + """Returns whether name is a dunder name. + + Args: + exclude_special: Whether to return False for a couple special dunder methods. + + """ + if exclude_special and name in SPECIAL_DUNDERS: + return False + return name.startswith("__") and name.endswith("__") + + +def is_sunder(name: str) -> bool: + return not is_dunder(name) and name.startswith("_") and name.endswith("_") + + +def split_module_names(mod_name: str) -> list[str]: + """Return the module and all parent module names. + + So, if `mod_name` is 'a.b.c', this function will return + ['a.b.c', 'a.b', and 'a']. + """ + out = [mod_name] + while "." in mod_name: + mod_name = mod_name.rsplit(".", 1)[0] + out.append(mod_name) + return out + + +def module_prefix(modules: Iterable[str], target: str) -> str | None: + result = split_target(modules, target) + if result is None: + return None + return result[0] + + +def split_target(modules: Iterable[str], target: str) -> tuple[str, str] | None: + remaining: list[str] = [] + while True: + if target in modules: + return target, ".".join(remaining) + components = target.rsplit(".", 1) + if len(components) == 1: + return None + target = components[0] + remaining.insert(0, components[1]) def short_type(obj: object) -> str: @@ -12,50 +115,839 @@ def short_type(obj: object) -> str: If obj is None, return 'nil'. For example, if obj is 1, return 'int'. """ if obj is None: - return 'nil' + return "nil" t = str(type(obj)) - return t.split('.')[-1].rstrip("'>") - - -def indent(s: str, n: int) -> str: - """Indent all the lines in s (separated by Newlines) by n spaces.""" - s = ' ' * n + s - s = s.replace('\n', '\n' + ' ' * n) - return s - - -def array_repr(a: List[T]) -> List[str]: - """Return the items of an array converted to strings using Repr.""" - aa = [] # type: List[str] - for x in a: - aa.append(repr(x)) - return aa - - -def dump_tagged(nodes: List[Any], tag: str) -> str: - """Convert an array into a pretty-printed multiline string representation. - - The format is - tag( - item1.. - itemN) - Individual items are formatted like this: - - arrays are flattened - - pairs (str : array) are converted recursively, so that str is the tag - - other items are converted to strings and indented - """ - a = [] # type: List[str] - if tag: - a.append(tag + '(') - for n in nodes: - if isinstance(n, list): - if n: - a.append(dump_tagged(n, None)) - elif isinstance(n, tuple): - s = dump_tagged(n[1], n[0]) - a.append(indent(s, 2)) - elif n: - a.append(indent(str(n), 2)) - if tag: - a[-1] += ')' - return '\n'.join(a) + return t.split(".")[-1].rstrip("'>") + + +def find_python_encoding(text: bytes) -> tuple[str, int]: + """PEP-263 for detecting Python file encoding""" + result = ENCODING_RE.match(text) + if result: + line = 2 if result.group(1) else 1 + encoding = result.group(3).decode("ascii") + # Handle some aliases that Python is happy to accept and that are used in the wild. + if encoding.startswith(("iso-latin-1-", "latin-1-")) or encoding == "iso-latin-1": + encoding = "latin-1" + return encoding, line + else: + default_encoding = "utf8" + return default_encoding, -1 + + +def bytes_to_human_readable_repr(b: bytes) -> str: + """Converts bytes into some human-readable representation. Unprintable + bytes such as the nul byte are escaped. For example: + + >>> b = bytes([102, 111, 111, 10, 0]) + >>> s = bytes_to_human_readable_repr(b) + >>> print(s) + foo\n\x00 + >>> print(repr(s)) + 'foo\\n\\x00' + """ + return repr(b)[2:-1] + + +class DecodeError(Exception): + """Exception raised when a file cannot be decoded due to an unknown encoding type. + + Essentially a wrapper for the LookupError raised by `bytearray.decode` + """ + + +def decode_python_encoding(source: bytes) -> str: + """Read the Python file with while obeying PEP-263 encoding detection. + + Returns the source as a string. + """ + # check for BOM UTF-8 encoding and strip it out if present + if source.startswith(b"\xef\xbb\xbf"): + encoding = "utf8" + source = source[3:] + else: + # look at first two lines and check if PEP-263 coding is present + encoding, _ = find_python_encoding(source) + + try: + source_text = source.decode(encoding) + except LookupError as lookuperr: + raise DecodeError(str(lookuperr)) from lookuperr + return source_text + + +def read_py_file(path: str, read: Callable[[str], bytes]) -> list[str] | None: + """Try reading a Python file as list of source lines. + + Return None if something goes wrong. + """ + try: + source = read(path) + except OSError: + return None + else: + try: + source_lines = decode_python_encoding(source).splitlines() + except DecodeError: + return None + return source_lines + + +def trim_source_line(line: str, max_len: int, col: int, min_width: int) -> tuple[str, int]: + """Trim a line of source code to fit into max_len. + + Show 'min_width' characters on each side of 'col' (an error location). If either + start or end is trimmed, this is indicated by adding '...' there. + A typical result looks like this: + ...some_variable = function_to_call(one_arg, other_arg) or... + + Return the trimmed string and the column offset to adjust error location. + """ + if max_len < 2 * min_width + 1: + # In case the window is too tiny it is better to still show something. + max_len = 2 * min_width + 1 + + # Trivial case: line already fits in. + if len(line) <= max_len: + return line, 0 + + # If column is not too large so that there is still min_width after it, + # the line doesn't need to be trimmed at the start. + if col + min_width < max_len: + return line[:max_len] + "...", 0 + + # Otherwise, if the column is not too close to the end, trim both sides. + if col < len(line) - min_width - 1: + offset = col - max_len + min_width + 1 + return "..." + line[offset : col + min_width + 1] + "...", offset - 3 + + # Finally, if the column is near the end, just trim the start. + return "..." + line[-max_len:], len(line) - max_len - 3 + + +def get_mypy_comments(source: str) -> list[tuple[int, str]]: + PREFIX = "# mypy: " + # Don't bother splitting up the lines unless we know it is useful + if PREFIX not in source: + return [] + lines = source.split("\n") + results = [] + for i, line in enumerate(lines): + if line.startswith(PREFIX): + results.append((i + 1, line[len(PREFIX) :])) + + return results + + +JUNIT_HEADER_TEMPLATE: Final = """ + +""" + +JUNIT_TESTCASE_FAIL_TEMPLATE: Final = """ + {text} + +""" + +JUNIT_ERROR_TEMPLATE: Final = """ + {text} + +""" + +JUNIT_TESTCASE_PASS_TEMPLATE: Final = """ + +""" + +JUNIT_FOOTER: Final = """ +""" + + +def _generate_junit_contents( + dt: float, + serious: bool, + messages_by_file: dict[str | None, list[str]], + version: str, + platform: str, +) -> str: + from xml.sax.saxutils import escape + + if serious: + failures = 0 + errors = len(messages_by_file) + else: + failures = len(messages_by_file) + errors = 0 + + xml = JUNIT_HEADER_TEMPLATE.format( + errors=errors, + failures=failures, + time=dt, + # If there are no messages, we still write one "test" indicating success. + tests=len(messages_by_file) or 1, + ) + + if not messages_by_file: + xml += JUNIT_TESTCASE_PASS_TEMPLATE.format(time=dt, ver=version, platform=platform) + else: + for filename, messages in messages_by_file.items(): + if filename is not None: + xml += JUNIT_TESTCASE_FAIL_TEMPLATE.format( + text=escape("\n".join(messages)), + filename=filename, + time=dt, + name="mypy-py{ver}-{platform} {filename}".format( + ver=version, platform=platform, filename=filename + ), + ) + else: + xml += JUNIT_TESTCASE_FAIL_TEMPLATE.format( + text=escape("\n".join(messages)), + filename="mypy", + time=dt, + name=f"mypy-py{version}-{platform}", + ) + + xml += JUNIT_FOOTER + + return xml + + +def write_junit_xml( + dt: float, + serious: bool, + messages_by_file: dict[str | None, list[str]], + path: str, + version: str, + platform: str, +) -> None: + xml = _generate_junit_contents(dt, serious, messages_by_file, version, platform) + + # creates folders if needed + xml_dirs = os.path.dirname(os.path.abspath(path)) + os.makedirs(xml_dirs, exist_ok=True) + + with open(path, "wb") as f: + f.write(xml.encode("utf-8")) + + +class IdMapper: + """Generate integer ids for objects. + + Unlike id(), these start from 0 and increment by 1, and ids won't + get reused across the life-time of IdMapper. + + Assume objects don't redefine __eq__ or __hash__. + """ + + def __init__(self) -> None: + self.id_map: dict[object, int] = {} + self.next_id = 0 + + def id(self, o: object) -> int: + if o not in self.id_map: + self.id_map[o] = self.next_id + self.next_id += 1 + return self.id_map[o] + + +def get_prefix(fullname: str) -> str: + """Drop the final component of a qualified name (e.g. ('x.y' -> 'x').""" + return fullname.rsplit(".", 1)[0] + + +def correct_relative_import( + cur_mod_id: str, relative: int, target: str, is_cur_package_init_file: bool +) -> tuple[str, bool]: + if relative == 0: + return target, True + parts = cur_mod_id.split(".") + rel = relative + if is_cur_package_init_file: + rel -= 1 + ok = len(parts) >= rel + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + return cur_mod_id + (("." + target) if target else ""), ok + + +fields_cache: Final[dict[type[object], list[str]]] = {} + + +def get_class_descriptors(cls: type[object]) -> Sequence[str]: + import inspect # Lazy import for minor startup speed win + + # Maintain a cache of type -> attributes defined by descriptors in the class + # (that is, attributes from __slots__ and C extension classes) + if cls not in fields_cache: + members = inspect.getmembers( + cls, lambda o: inspect.isgetsetdescriptor(o) or inspect.ismemberdescriptor(o) + ) + fields_cache[cls] = [x for x, y in members if x != "__weakref__" and x != "__dict__"] + return fields_cache[cls] + + +def replace_object_state( + new: object, old: object, copy_dict: bool = False, skip_slots: tuple[str, ...] = () +) -> None: + """Copy state of old node to the new node. + + This handles cases where there is __dict__ and/or attribute descriptors + (either from slots or because the type is defined in a C extension module). + + Assume that both objects have the same __class__. + """ + if hasattr(old, "__dict__"): + if copy_dict: + new.__dict__ = dict(old.__dict__) + else: + new.__dict__ = old.__dict__ + + for attr in get_class_descriptors(old.__class__): + if attr in skip_slots: + continue + try: + if hasattr(old, attr): + setattr(new, attr, getattr(old, attr)) + elif hasattr(new, attr): + delattr(new, attr) + # There is no way to distinguish getsetdescriptors that allow + # writes from ones that don't (I think?), so we just ignore + # AttributeErrors if we need to. + # TODO: What about getsetdescriptors that act like properties??? + except AttributeError: + pass + + +def is_sub_path_normabs(path: str, dir: str) -> bool: + """Given two paths, return if path is a sub-path of dir. + + Moral equivalent of: Path(dir) in Path(path).parents + + Similar to the pathlib version: + - Treats paths case-sensitively + - Does not fully handle unnormalised paths (e.g. paths with "..") + - Does not handle a mix of absolute and relative paths + Unlike the pathlib version: + - Fast + - On Windows, assumes input has been slash normalised + - Handles even fewer unnormalised paths (e.g. paths with "." and "//") + + As a result, callers should ensure that inputs have had os.path.abspath called on them + (note that os.path.abspath will normalise) + """ + if not dir.endswith(os.sep): + dir += os.sep + return path.startswith(dir) + + +if sys.platform == "linux" or sys.platform == "darwin": + + def os_path_join(path: str, b: str) -> str: + # Based off of os.path.join, but simplified to str-only, 2 args and mypyc can compile it. + if b.startswith("/") or not path: + return b + elif path.endswith("/"): + return path + b + else: + return path + "/" + b + +else: + + def os_path_join(a: str, p: str) -> str: + return os.path.join(a, p) + + +def hard_exit(status: int = 0) -> None: + """Kill the current process without fully cleaning up. + + This can be quite a bit faster than a normal exit() since objects are not freed. + """ + sys.stdout.flush() + sys.stderr.flush() + os._exit(status) + + +def unmangle(name: str) -> str: + """Remove internal suffixes from a short name.""" + return name.rstrip("'") + + +def get_unique_redefinition_name(name: str, existing: Container[str]) -> str: + """Get a simple redefinition name not present among existing. + + For example, for name 'foo' we try 'foo-redefinition', 'foo-redefinition2', + 'foo-redefinition3', etc. until we find one that is not in existing. + """ + r_name = name + "-redefinition" + if r_name not in existing: + return r_name + + i = 2 + while r_name + str(i) in existing: + i += 1 + return r_name + str(i) + + +def check_python_version(program: str) -> None: + """Report issues with the Python used to run mypy, dmypy, or stubgen""" + # Check for known bad Python versions. + if sys.version_info[:2] < (3, 8): # noqa: UP036 + sys.exit( + "Running {name} with Python 3.7 or lower is not supported; " + "please upgrade to 3.8 or newer".format(name=program) + ) + + +def count_stats(messages: list[str]) -> tuple[int, int, int]: + """Count total number of errors, notes and error_files in message list.""" + errors = [e for e in messages if ": error:" in e] + error_files = {e.split(":")[0] for e in errors} + notes = [e for e in messages if ": note:" in e] + return len(errors), len(notes), len(error_files) + + +def split_words(msg: str) -> list[str]: + """Split line of text into words (but not within quoted groups).""" + next_word = "" + res: list[str] = [] + allow_break = True + for c in msg: + if c == " " and allow_break: + res.append(next_word) + next_word = "" + continue + if c == '"': + allow_break = not allow_break + next_word += c + res.append(next_word) + return res + + +def get_terminal_width() -> int: + """Get current terminal width if possible, otherwise return the default one.""" + return ( + int(os.getenv("MYPY_FORCE_TERMINAL_WIDTH", "0")) + or shutil.get_terminal_size().columns + or DEFAULT_COLUMNS + ) + + +def soft_wrap(msg: str, max_len: int, first_offset: int, num_indent: int = 0) -> str: + """Wrap a long error message into few lines. + + Breaks will only happen between words, and never inside a quoted group + (to avoid breaking types such as "Union[int, str]"). The 'first_offset' is + the width before the start of first line. + + Pad every next line with 'num_indent' spaces. Every line will be at most 'max_len' + characters, except if it is a single word or quoted group. + + For example: + first_offset + ------------------------ + path/to/file: error: 58: Some very long error message + that needs to be split in separate lines. + "Long[Type, Names]" are never split. + ^^^^-------------------------------------------------- + num_indent max_len + """ + words = split_words(msg) + next_line = words.pop(0) + lines: list[str] = [] + while words: + next_word = words.pop(0) + max_line_len = max_len - num_indent if lines else max_len - first_offset + # Add 1 to account for space between words. + if len(next_line) + len(next_word) + 1 <= max_line_len: + next_line += " " + next_word + else: + lines.append(next_line) + next_line = next_word + lines.append(next_line) + padding = "\n" + " " * num_indent + return padding.join(lines) + + +def hash_digest(data: bytes) -> str: + """Compute a hash digest of some data. + + We use a cryptographic hash because we want a low probability of + accidental collision, but we don't really care about any of the + cryptographic properties. + """ + return hashlib.sha1(data).hexdigest() + + +def parse_gray_color(cup: bytes) -> str: + """Reproduce a gray color in ANSI escape sequence""" + if sys.platform == "win32": + assert False, "curses is not available on Windows" + set_color = "".join([cup[:-1].decode(), "m"]) + gray = curses.tparm(set_color.encode("utf-8"), 1, 9).decode() + return gray + + +def should_force_color() -> bool: + env_var = os.getenv("MYPY_FORCE_COLOR", os.getenv("FORCE_COLOR", "0")) + try: + return bool(int(env_var)) + except ValueError: + return bool(env_var) + + +class FancyFormatter: + """Apply color and bold font to terminal output. + + This currently only works on Linux and Mac. + """ + + def __init__( + self, f_out: IO[str], f_err: IO[str], hide_error_codes: bool, hide_success: bool = False + ) -> None: + self.hide_error_codes = hide_error_codes + self.hide_success = hide_success + + # Check if we are in a human-facing terminal on a supported platform. + if sys.platform not in ("linux", "darwin", "win32", "emscripten"): + self.dummy_term = True + return + if not should_force_color() and (not f_out.isatty() or not f_err.isatty()): + self.dummy_term = True + return + if sys.platform == "win32": + self.dummy_term = not self.initialize_win_colors() + elif sys.platform == "emscripten": + self.dummy_term = not self.initialize_vt100_colors() + else: + self.dummy_term = not self.initialize_unix_colors() + if not self.dummy_term: + self.colors = { + "red": self.RED, + "green": self.GREEN, + "blue": self.BLUE, + "yellow": self.YELLOW, + "none": "", + } + + def initialize_vt100_colors(self) -> bool: + """Return True if initialization was successful and we can use colors, False otherwise""" + # Windows and Emscripten can both use ANSI/VT100 escape sequences for color + assert sys.platform in ("win32", "emscripten") + self.BOLD = "\033[1m" + self.UNDER = "\033[4m" + self.BLUE = "\033[94m" + self.GREEN = "\033[92m" + self.RED = "\033[91m" + self.YELLOW = "\033[93m" + self.NORMAL = "\033[0m" + self.DIM = "\033[2m" + return True + + def initialize_win_colors(self) -> bool: + """Return True if initialization was successful and we can use colors, False otherwise""" + # Windows ANSI escape sequences are only supported on Threshold 2 and above. + # we check with an assert at runtime and an if check for mypy, as asserts do not + # yet narrow platform + assert sys.platform == "win32" + if sys.platform == "win32": + winver = sys.getwindowsversion() + if ( + winver.major < MINIMUM_WINDOWS_MAJOR_VT100 + or winver.build < MINIMUM_WINDOWS_BUILD_VT100 + ): + return False + import ctypes + + kernel32 = ctypes.windll.kernel32 + ENABLE_PROCESSED_OUTPUT = 0x1 + ENABLE_WRAP_AT_EOL_OUTPUT = 0x2 + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x4 + STD_OUTPUT_HANDLE = -11 + kernel32.SetConsoleMode( + kernel32.GetStdHandle(STD_OUTPUT_HANDLE), + ENABLE_PROCESSED_OUTPUT + | ENABLE_WRAP_AT_EOL_OUTPUT + | ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + self.initialize_vt100_colors() + return True + return False + + def initialize_unix_colors(self) -> bool: + """Return True if initialization was successful and we can use colors, False otherwise""" + if sys.platform == "win32" or not CURSES_ENABLED: + return False + try: + # setupterm wants a fd to potentially write an "initialization sequence". + # We override sys.stdout for the daemon API so if stdout doesn't have an fd, + # just give it /dev/null. + try: + fd = sys.stdout.fileno() + except io.UnsupportedOperation: + with open("/dev/null", "rb") as f: + curses.setupterm(fd=f.fileno()) + else: + curses.setupterm(fd=fd) + except curses.error: + # Most likely terminfo not found. + return False + bold = curses.tigetstr("bold") + under = curses.tigetstr("smul") + set_color = curses.tigetstr("setaf") + set_eseq = curses.tigetstr("cup") + normal = curses.tigetstr("sgr0") + + if not (bold and under and set_color and set_eseq and normal): + return False + + self.NORMAL = normal.decode() + self.BOLD = bold.decode() + self.UNDER = under.decode() + self.DIM = parse_gray_color(set_eseq) + self.BLUE = curses.tparm(set_color, curses.COLOR_BLUE).decode() + self.GREEN = curses.tparm(set_color, curses.COLOR_GREEN).decode() + self.RED = curses.tparm(set_color, curses.COLOR_RED).decode() + self.YELLOW = curses.tparm(set_color, curses.COLOR_YELLOW).decode() + return True + + def style( + self, + text: str, + color: Literal["red", "green", "blue", "yellow", "none"], + bold: bool = False, + underline: bool = False, + dim: bool = False, + ) -> str: + """Apply simple color and style (underlined or bold).""" + if self.dummy_term: + return text + if bold: + start = self.BOLD + else: + start = "" + if underline: + start += self.UNDER + if dim: + start += self.DIM + return start + self.colors[color] + text + self.NORMAL + + def fit_in_terminal( + self, messages: list[str], fixed_terminal_width: int | None = None + ) -> list[str]: + """Improve readability by wrapping error messages and trimming source code.""" + width = fixed_terminal_width or get_terminal_width() + new_messages = messages.copy() + for i, error in enumerate(messages): + if ": error:" in error: + loc, msg = error.split("error:", maxsplit=1) + msg = soft_wrap(msg, width, first_offset=len(loc) + len("error: ")) + new_messages[i] = loc + "error:" + msg + if error.startswith(" " * DEFAULT_SOURCE_OFFSET) and "^" not in error: + # TODO: detecting source code highlights through an indent can be surprising. + # Restore original error message and error location. + error = error[DEFAULT_SOURCE_OFFSET:] + marker_line = messages[i + 1] + marker_column = marker_line.index("^") + column = marker_column - DEFAULT_SOURCE_OFFSET + if "~" not in marker_line: + marker = "^" + else: + # +1 because both ends are included + marker = marker_line[marker_column : marker_line.rindex("~") + 1] + + # Let source have some space also on the right side, plus 6 + # to accommodate ... on each side. + max_len = width - DEFAULT_SOURCE_OFFSET - 6 + source_line, offset = trim_source_line(error, max_len, column, MINIMUM_WIDTH) + + new_messages[i] = " " * DEFAULT_SOURCE_OFFSET + source_line + # Also adjust the error marker position and trim error marker is needed. + new_marker_line = " " * (DEFAULT_SOURCE_OFFSET + column - offset) + marker + if len(new_marker_line) > len(new_messages[i]) and len(marker) > 3: + new_marker_line = new_marker_line[: len(new_messages[i]) - 3] + "..." + new_messages[i + 1] = new_marker_line + return new_messages + + def colorize(self, error: str) -> str: + """Colorize an output line by highlighting the status and error code.""" + if ": error:" in error: + loc, msg = error.split("error:", maxsplit=1) + if self.hide_error_codes: + return ( + loc + self.style("error:", "red", bold=True) + self.highlight_quote_groups(msg) + ) + codepos = msg.rfind("[") + if codepos != -1: + code = msg[codepos:] + msg = msg[:codepos] + else: + code = "" # no error code specified + return ( + loc + + self.style("error:", "red", bold=True) + + self.highlight_quote_groups(msg) + + self.style(code, "yellow") + ) + elif ": note:" in error: + loc, msg = error.split("note:", maxsplit=1) + formatted = self.highlight_quote_groups(self.underline_link(msg)) + return loc + self.style("note:", "blue") + formatted + elif error.startswith(" " * DEFAULT_SOURCE_OFFSET): + # TODO: detecting source code highlights through an indent can be surprising. + if "^" not in error: + return self.style(error, "none", dim=True) + return self.style(error, "red") + else: + return error + + def highlight_quote_groups(self, msg: str) -> str: + """Make groups quoted with double quotes bold (including quotes). + + This is used to highlight types, attribute names etc. + """ + if msg.count('"') % 2: + # Broken error message, don't do any formatting. + return msg + parts = msg.split('"') + out = "" + for i, part in enumerate(parts): + if i % 2 == 0: + out += self.style(part, "none") + else: + out += self.style('"' + part + '"', "none", bold=True) + return out + + def underline_link(self, note: str) -> str: + """Underline a link in a note message (if any). + + This assumes there is at most one link in the message. + """ + match = re.search(r"https?://\S*", note) + if not match: + return note + start = match.start() + end = match.end() + return note[:start] + self.style(note[start:end], "none", underline=True) + note[end:] + + def format_success(self, n_sources: int, use_color: bool = True) -> str: + """Format short summary in case of success. + + n_sources is total number of files passed directly on command line, + i.e. excluding stubs and followed imports. + """ + if self.hide_success: + return "" + + msg = f"Success: no issues found in {n_sources} source file{plural_s(n_sources)}" + if not use_color: + return msg + return self.style(msg, "green", bold=True) + + def format_error( + self, + n_errors: int, + n_files: int, + n_sources: int, + *, + blockers: bool = False, + use_color: bool = True, + ) -> str: + """Format a short summary in case of errors.""" + msg = f"Found {n_errors} error{plural_s(n_errors)} in {n_files} file{plural_s(n_files)}" + if blockers: + msg += " (errors prevented further checking)" + else: + msg += f" (checked {n_sources} source file{plural_s(n_sources)})" + if not use_color: + return msg + return self.style(msg, "red", bold=True) + + +def is_typeshed_file(typeshed_dir: str | None, file: str) -> bool: + typeshed_dir = typeshed_dir if typeshed_dir is not None else TYPESHED_DIR + try: + return os.path.commonpath((typeshed_dir, os.path.abspath(file))) == typeshed_dir + except ValueError: # Different drives on Windows + return False + + +def is_stdlib_file(typeshed_dir: str | None, file: str) -> bool: + if "stdlib" not in file: + # Fast path + return False + typeshed_dir = typeshed_dir if typeshed_dir is not None else TYPESHED_DIR + stdlib_dir = os.path.join(typeshed_dir, "stdlib") + try: + return os.path.commonpath((stdlib_dir, os.path.abspath(file))) == stdlib_dir + except ValueError: # Different drives on Windows + return False + + +def is_stub_package_file(file: str) -> bool: + # Use hacky heuristics to check whether file is part of a PEP 561 stub package. + if not file.endswith(".pyi"): + return False + return any(component.endswith("-stubs") for component in os.path.split(os.path.abspath(file))) + + +def unnamed_function(name: str | None) -> bool: + return name is not None and name == "_" + + +time_ref = time.perf_counter_ns + + +def time_spent_us(t0: int) -> int: + return int((time.perf_counter_ns() - t0) / 1000) + + +def plural_s(s: int | Sized) -> str: + count = s if isinstance(s, int) else len(s) + if count != 1: + return "s" + else: + return "" + + +def quote_docstring(docstr: str) -> str: + """Returns docstring correctly encapsulated in a single or double quoted form.""" + # Uses repr to get hint on the correct quotes and escape everything properly. + # Creating multiline string for prettier output. + docstr_repr = "\n".join(re.split(r"(?<=[^\\])\\n", repr(docstr))) + + if docstr_repr.startswith("'"): + # Enforce double quotes when it's safe to do so. + # That is when double quotes are not in the string + # or when it doesn't end with a single quote. + if '"' not in docstr_repr[1:-1] and docstr_repr[-2] != "'": + return f'"""{docstr_repr[1:-1]}"""' + return f"''{docstr_repr}''" + else: + return f'""{docstr_repr}""' + + +def json_dumps(obj: object, debug: bool = False) -> bytes: + if orjson is not None: + if debug: + dumps_option = orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS + else: + # TODO: If we don't sort keys here, testIncrementalInternalScramble fails + # We should document exactly what is going on there + dumps_option = orjson.OPT_SORT_KEYS + + try: + return orjson.dumps(obj, option=dumps_option) # type: ignore[no-any-return] + except TypeError as e: + if str(e) != "Integer exceeds 64-bit range": + raise + + if debug: + return json.dumps(obj, indent=2, sort_keys=True).encode("utf-8") + else: + # See above for sort_keys comment + return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def json_loads(data: bytes) -> Any: + if orjson is not None: + return orjson.loads(data) + return json.loads(data) diff --git a/mypy/version.py b/mypy/version.py new file mode 100644 index 000000000000..eca889c708e0 --- /dev/null +++ b/mypy/version.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import os + +from mypy import git + +# Base version. +# - Release versions have the form "1.2.3". +# - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). +# - Before 1.0 we had the form "0.NNN". +__version__ = "1.14.0" +base_version = __version__ + +mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +if __version__.endswith("+dev") and git.is_git_repo(mypy_dir) and git.have_git(): + __version__ += "." + git.git_revision(mypy_dir).decode("utf-8") + if git.is_dirty(mypy_dir): + __version__ += ".dirty" +del mypy_dir diff --git a/mypy/visitor.py b/mypy/visitor.py index 6ac592000295..340e1af64e00 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -1,230 +1,628 @@ """Generic abstract syntax tree node visitor""" -from typing import typevar, Generic +from __future__ import annotations -import mypy.nodes +from abc import abstractmethod +from typing import TYPE_CHECKING, Generic, TypeVar +from mypy_extensions import mypyc_attr, trait -T = typevar('T') +if TYPE_CHECKING: + # break import cycle only needed for mypy + import mypy.nodes + import mypy.patterns -class NodeVisitor(Generic[T]): +T = TypeVar("T") + + +@trait +@mypyc_attr(allow_interpreted_subclasses=True) +class ExpressionVisitor(Generic[T]): + @abstractmethod + def visit_int_expr(self, o: mypy.nodes.IntExpr) -> T: + pass + + @abstractmethod + def visit_str_expr(self, o: mypy.nodes.StrExpr) -> T: + pass + + @abstractmethod + def visit_bytes_expr(self, o: mypy.nodes.BytesExpr) -> T: + pass + + @abstractmethod + def visit_float_expr(self, o: mypy.nodes.FloatExpr) -> T: + pass + + @abstractmethod + def visit_complex_expr(self, o: mypy.nodes.ComplexExpr) -> T: + pass + + @abstractmethod + def visit_ellipsis(self, o: mypy.nodes.EllipsisExpr) -> T: + pass + + @abstractmethod + def visit_star_expr(self, o: mypy.nodes.StarExpr) -> T: + pass + + @abstractmethod + def visit_name_expr(self, o: mypy.nodes.NameExpr) -> T: + pass + + @abstractmethod + def visit_member_expr(self, o: mypy.nodes.MemberExpr) -> T: + pass + + @abstractmethod + def visit_yield_from_expr(self, o: mypy.nodes.YieldFromExpr) -> T: + pass + + @abstractmethod + def visit_yield_expr(self, o: mypy.nodes.YieldExpr) -> T: + pass + + @abstractmethod + def visit_call_expr(self, o: mypy.nodes.CallExpr) -> T: + pass + + @abstractmethod + def visit_op_expr(self, o: mypy.nodes.OpExpr) -> T: + pass + + @abstractmethod + def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> T: + pass + + @abstractmethod + def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> T: + pass + + @abstractmethod + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> T: + pass + + @abstractmethod + def visit_reveal_expr(self, o: mypy.nodes.RevealExpr) -> T: + pass + + @abstractmethod + def visit_super_expr(self, o: mypy.nodes.SuperExpr) -> T: + pass + + @abstractmethod + def visit_unary_expr(self, o: mypy.nodes.UnaryExpr) -> T: + pass + + @abstractmethod + def visit_assignment_expr(self, o: mypy.nodes.AssignmentExpr) -> T: + pass + + @abstractmethod + def visit_list_expr(self, o: mypy.nodes.ListExpr) -> T: + pass + + @abstractmethod + def visit_dict_expr(self, o: mypy.nodes.DictExpr) -> T: + pass + + @abstractmethod + def visit_tuple_expr(self, o: mypy.nodes.TupleExpr) -> T: + pass + + @abstractmethod + def visit_set_expr(self, o: mypy.nodes.SetExpr) -> T: + pass + + @abstractmethod + def visit_index_expr(self, o: mypy.nodes.IndexExpr) -> T: + pass + + @abstractmethod + def visit_type_application(self, o: mypy.nodes.TypeApplication) -> T: + pass + + @abstractmethod + def visit_lambda_expr(self, o: mypy.nodes.LambdaExpr) -> T: + pass + + @abstractmethod + def visit_list_comprehension(self, o: mypy.nodes.ListComprehension) -> T: + pass + + @abstractmethod + def visit_set_comprehension(self, o: mypy.nodes.SetComprehension) -> T: + pass + + @abstractmethod + def visit_dictionary_comprehension(self, o: mypy.nodes.DictionaryComprehension) -> T: + pass + + @abstractmethod + def visit_generator_expr(self, o: mypy.nodes.GeneratorExpr) -> T: + pass + + @abstractmethod + def visit_slice_expr(self, o: mypy.nodes.SliceExpr) -> T: + pass + + @abstractmethod + def visit_conditional_expr(self, o: mypy.nodes.ConditionalExpr) -> T: + pass + + @abstractmethod + def visit_type_var_expr(self, o: mypy.nodes.TypeVarExpr) -> T: + pass + + @abstractmethod + def visit_paramspec_expr(self, o: mypy.nodes.ParamSpecExpr) -> T: + pass + + @abstractmethod + def visit_type_var_tuple_expr(self, o: mypy.nodes.TypeVarTupleExpr) -> T: + pass + + @abstractmethod + def visit_type_alias_expr(self, o: mypy.nodes.TypeAliasExpr) -> T: + pass + + @abstractmethod + def visit_namedtuple_expr(self, o: mypy.nodes.NamedTupleExpr) -> T: + pass + + @abstractmethod + def visit_enum_call_expr(self, o: mypy.nodes.EnumCallExpr) -> T: + pass + + @abstractmethod + def visit_typeddict_expr(self, o: mypy.nodes.TypedDictExpr) -> T: + pass + + @abstractmethod + def visit_newtype_expr(self, o: mypy.nodes.NewTypeExpr) -> T: + pass + + @abstractmethod + def visit__promote_expr(self, o: mypy.nodes.PromoteExpr) -> T: + pass + + @abstractmethod + def visit_await_expr(self, o: mypy.nodes.AwaitExpr) -> T: + pass + + @abstractmethod + def visit_temp_node(self, o: mypy.nodes.TempNode) -> T: + pass + + +@trait +@mypyc_attr(allow_interpreted_subclasses=True) +class StatementVisitor(Generic[T]): + # Definitions + + @abstractmethod + def visit_assignment_stmt(self, o: mypy.nodes.AssignmentStmt) -> T: + pass + + @abstractmethod + def visit_for_stmt(self, o: mypy.nodes.ForStmt) -> T: + pass + + @abstractmethod + def visit_with_stmt(self, o: mypy.nodes.WithStmt) -> T: + pass + + @abstractmethod + def visit_del_stmt(self, o: mypy.nodes.DelStmt) -> T: + pass + + @abstractmethod + def visit_func_def(self, o: mypy.nodes.FuncDef) -> T: + pass + + @abstractmethod + def visit_overloaded_func_def(self, o: mypy.nodes.OverloadedFuncDef) -> T: + pass + + @abstractmethod + def visit_class_def(self, o: mypy.nodes.ClassDef) -> T: + pass + + @abstractmethod + def visit_global_decl(self, o: mypy.nodes.GlobalDecl) -> T: + pass + + @abstractmethod + def visit_nonlocal_decl(self, o: mypy.nodes.NonlocalDecl) -> T: + pass + + @abstractmethod + def visit_decorator(self, o: mypy.nodes.Decorator) -> T: + pass + + # Module structure + + @abstractmethod + def visit_import(self, o: mypy.nodes.Import) -> T: + pass + + @abstractmethod + def visit_import_from(self, o: mypy.nodes.ImportFrom) -> T: + pass + + @abstractmethod + def visit_import_all(self, o: mypy.nodes.ImportAll) -> T: + pass + + # Statements + + @abstractmethod + def visit_block(self, o: mypy.nodes.Block) -> T: + pass + + @abstractmethod + def visit_expression_stmt(self, o: mypy.nodes.ExpressionStmt) -> T: + pass + + @abstractmethod + def visit_operator_assignment_stmt(self, o: mypy.nodes.OperatorAssignmentStmt) -> T: + pass + + @abstractmethod + def visit_while_stmt(self, o: mypy.nodes.WhileStmt) -> T: + pass + + @abstractmethod + def visit_return_stmt(self, o: mypy.nodes.ReturnStmt) -> T: + pass + + @abstractmethod + def visit_assert_stmt(self, o: mypy.nodes.AssertStmt) -> T: + pass + + @abstractmethod + def visit_if_stmt(self, o: mypy.nodes.IfStmt) -> T: + pass + + @abstractmethod + def visit_break_stmt(self, o: mypy.nodes.BreakStmt) -> T: + pass + + @abstractmethod + def visit_continue_stmt(self, o: mypy.nodes.ContinueStmt) -> T: + pass + + @abstractmethod + def visit_pass_stmt(self, o: mypy.nodes.PassStmt) -> T: + pass + + @abstractmethod + def visit_raise_stmt(self, o: mypy.nodes.RaiseStmt) -> T: + pass + + @abstractmethod + def visit_try_stmt(self, o: mypy.nodes.TryStmt) -> T: + pass + + @abstractmethod + def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> T: + pass + + @abstractmethod + def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> T: + pass + + +@trait +@mypyc_attr(allow_interpreted_subclasses=True) +class PatternVisitor(Generic[T]): + @abstractmethod + def visit_as_pattern(self, o: mypy.patterns.AsPattern) -> T: + pass + + @abstractmethod + def visit_or_pattern(self, o: mypy.patterns.OrPattern) -> T: + pass + + @abstractmethod + def visit_value_pattern(self, o: mypy.patterns.ValuePattern) -> T: + pass + + @abstractmethod + def visit_singleton_pattern(self, o: mypy.patterns.SingletonPattern) -> T: + pass + + @abstractmethod + def visit_sequence_pattern(self, o: mypy.patterns.SequencePattern) -> T: + pass + + @abstractmethod + def visit_starred_pattern(self, o: mypy.patterns.StarredPattern) -> T: + pass + + @abstractmethod + def visit_mapping_pattern(self, o: mypy.patterns.MappingPattern) -> T: + pass + + @abstractmethod + def visit_class_pattern(self, o: mypy.patterns.ClassPattern) -> T: + pass + + +@trait +@mypyc_attr(allow_interpreted_subclasses=True) +class NodeVisitor(Generic[T], ExpressionVisitor[T], StatementVisitor[T], PatternVisitor[T]): """Empty base class for parse tree node visitors. The T type argument specifies the return type of the visit methods. As all methods defined here return None by default, subclasses do not always need to override all the methods. - TODO make the default return value explicit + TODO: make the default return value explicit, then turn on + empty body checking in mypy_self_check.ini. """ - # Module structure + # Not in superclasses: - def visit_mypy_file(self, o: 'mypy.nodes.MypyFile') -> T: + def visit_mypy_file(self, o: mypy.nodes.MypyFile) -> T: pass - def visit_import(self, o: 'mypy.nodes.Import') -> T: + # TODO: We have a visit_var method, but no visit_typeinfo or any + # other non-Statement SymbolNode (accepting those will raise a + # runtime error). Maybe this should be resolved in some direction. + def visit_var(self, o: mypy.nodes.Var) -> T: + pass + + # Module structure + + def visit_import(self, o: mypy.nodes.Import) -> T: pass - def visit_import_from(self, o: 'mypy.nodes.ImportFrom') -> T: + def visit_import_from(self, o: mypy.nodes.ImportFrom) -> T: pass - def visit_import_all(self, o: 'mypy.nodes.ImportAll') -> T: + def visit_import_all(self, o: mypy.nodes.ImportAll) -> T: pass # Definitions - def visit_func_def(self, o: 'mypy.nodes.FuncDef') -> T: + def visit_func_def(self, o: mypy.nodes.FuncDef) -> T: pass - def visit_overloaded_func_def(self, - o: 'mypy.nodes.OverloadedFuncDef') -> T: + def visit_overloaded_func_def(self, o: mypy.nodes.OverloadedFuncDef) -> T: pass - def visit_class_def(self, o: 'mypy.nodes.ClassDef') -> T: + def visit_class_def(self, o: mypy.nodes.ClassDef) -> T: pass - def visit_var_def(self, o: 'mypy.nodes.VarDef') -> T: + def visit_global_decl(self, o: mypy.nodes.GlobalDecl) -> T: pass - def visit_global_decl(self, o: 'mypy.nodes.GlobalDecl') -> T: + def visit_nonlocal_decl(self, o: mypy.nodes.NonlocalDecl) -> T: pass - def visit_nonlocal_decl(self, o: 'mypy.nodes.NonlocalDecl') -> T: + def visit_decorator(self, o: mypy.nodes.Decorator) -> T: pass - def visit_decorator(self, o: 'mypy.nodes.Decorator') -> T: + def visit_type_alias(self, o: mypy.nodes.TypeAlias) -> T: pass - def visit_var(self, o: 'mypy.nodes.Var') -> T: + def visit_placeholder_node(self, o: mypy.nodes.PlaceholderNode) -> T: pass # Statements - def visit_block(self, o: 'mypy.nodes.Block') -> T: + def visit_block(self, o: mypy.nodes.Block) -> T: pass - def visit_expression_stmt(self, o: 'mypy.nodes.ExpressionStmt') -> T: + def visit_expression_stmt(self, o: mypy.nodes.ExpressionStmt) -> T: pass - def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: + def visit_assignment_stmt(self, o: mypy.nodes.AssignmentStmt) -> T: pass - def visit_operator_assignment_stmt(self, - o: 'mypy.nodes.OperatorAssignmentStmt') -> T: + def visit_operator_assignment_stmt(self, o: mypy.nodes.OperatorAssignmentStmt) -> T: pass - def visit_while_stmt(self, o: 'mypy.nodes.WhileStmt') -> T: + def visit_while_stmt(self, o: mypy.nodes.WhileStmt) -> T: pass - def visit_for_stmt(self, o: 'mypy.nodes.ForStmt') -> T: + def visit_for_stmt(self, o: mypy.nodes.ForStmt) -> T: pass - def visit_return_stmt(self, o: 'mypy.nodes.ReturnStmt') -> T: + def visit_return_stmt(self, o: mypy.nodes.ReturnStmt) -> T: pass - def visit_assert_stmt(self, o: 'mypy.nodes.AssertStmt') -> T: + def visit_assert_stmt(self, o: mypy.nodes.AssertStmt) -> T: pass - def visit_yield_stmt(self, o: 'mypy.nodes.YieldStmt') -> T: + def visit_del_stmt(self, o: mypy.nodes.DelStmt) -> T: pass - def visit_yield_from_stmt(self, o: 'mypy.nodes.YieldFromStmt') -> T: + def visit_if_stmt(self, o: mypy.nodes.IfStmt) -> T: pass - def visit_del_stmt(self, o: 'mypy.nodes.DelStmt') -> T: + def visit_break_stmt(self, o: mypy.nodes.BreakStmt) -> T: pass - def visit_if_stmt(self, o: 'mypy.nodes.IfStmt') -> T: + def visit_continue_stmt(self, o: mypy.nodes.ContinueStmt) -> T: pass - def visit_break_stmt(self, o: 'mypy.nodes.BreakStmt') -> T: + def visit_pass_stmt(self, o: mypy.nodes.PassStmt) -> T: pass - def visit_continue_stmt(self, o: 'mypy.nodes.ContinueStmt') -> T: + def visit_raise_stmt(self, o: mypy.nodes.RaiseStmt) -> T: pass - def visit_pass_stmt(self, o: 'mypy.nodes.PassStmt') -> T: + def visit_try_stmt(self, o: mypy.nodes.TryStmt) -> T: pass - def visit_raise_stmt(self, o: 'mypy.nodes.RaiseStmt') -> T: + def visit_with_stmt(self, o: mypy.nodes.WithStmt) -> T: pass - def visit_try_stmt(self, o: 'mypy.nodes.TryStmt') -> T: + def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> T: pass - def visit_with_stmt(self, o: 'mypy.nodes.WithStmt') -> T: + def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> T: pass - def visit_print_stmt(self, o: 'mypy.nodes.PrintStmt') -> T: + # Expressions (default no-op implementation) + + def visit_int_expr(self, o: mypy.nodes.IntExpr) -> T: + pass + + def visit_str_expr(self, o: mypy.nodes.StrExpr) -> T: pass - # Expressions + def visit_bytes_expr(self, o: mypy.nodes.BytesExpr) -> T: + pass + + def visit_float_expr(self, o: mypy.nodes.FloatExpr) -> T: + pass - def visit_int_expr(self, o: 'mypy.nodes.IntExpr') -> T: + def visit_complex_expr(self, o: mypy.nodes.ComplexExpr) -> T: pass - def visit_str_expr(self, o: 'mypy.nodes.StrExpr') -> T: + def visit_ellipsis(self, o: mypy.nodes.EllipsisExpr) -> T: pass - def visit_bytes_expr(self, o: 'mypy.nodes.BytesExpr') -> T: + def visit_star_expr(self, o: mypy.nodes.StarExpr) -> T: pass - def visit_unicode_expr(self, o: 'mypy.nodes.UnicodeExpr') -> T: + def visit_name_expr(self, o: mypy.nodes.NameExpr) -> T: pass - def visit_float_expr(self, o: 'mypy.nodes.FloatExpr') -> T: + def visit_member_expr(self, o: mypy.nodes.MemberExpr) -> T: pass - def visit_complex_expr(self, o: 'mypy.nodes.ComplexExpr') -> T: + def visit_yield_from_expr(self, o: mypy.nodes.YieldFromExpr) -> T: pass - def visit_paren_expr(self, o: 'mypy.nodes.ParenExpr') -> T: + def visit_yield_expr(self, o: mypy.nodes.YieldExpr) -> T: pass - def visit_star_expr(self, o: 'mypy.nodes.StarExpr') -> T: + def visit_call_expr(self, o: mypy.nodes.CallExpr) -> T: pass - def visit_name_expr(self, o: 'mypy.nodes.NameExpr') -> T: + def visit_op_expr(self, o: mypy.nodes.OpExpr) -> T: pass - def visit_member_expr(self, o: 'mypy.nodes.MemberExpr') -> T: + def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> T: pass - def visit_yield_from_expr(self, o: 'mypy.nodes.YieldFromExpr') -> T: + def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> T: pass - def visit_call_expr(self, o: 'mypy.nodes.CallExpr') -> T: + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> T: pass - def visit_op_expr(self, o: 'mypy.nodes.OpExpr') -> T: + def visit_reveal_expr(self, o: mypy.nodes.RevealExpr) -> T: pass - def visit_comparison_expr(self, o: 'mypy.nodes.ComparisonExpr') -> T: + def visit_super_expr(self, o: mypy.nodes.SuperExpr) -> T: pass - def visit_cast_expr(self, o: 'mypy.nodes.CastExpr') -> T: + def visit_assignment_expr(self, o: mypy.nodes.AssignmentExpr) -> T: pass - def visit_super_expr(self, o: 'mypy.nodes.SuperExpr') -> T: + def visit_unary_expr(self, o: mypy.nodes.UnaryExpr) -> T: pass - def visit_unary_expr(self, o: 'mypy.nodes.UnaryExpr') -> T: + def visit_list_expr(self, o: mypy.nodes.ListExpr) -> T: pass - def visit_list_expr(self, o: 'mypy.nodes.ListExpr') -> T: + def visit_dict_expr(self, o: mypy.nodes.DictExpr) -> T: pass - def visit_dict_expr(self, o: 'mypy.nodes.DictExpr') -> T: + def visit_tuple_expr(self, o: mypy.nodes.TupleExpr) -> T: pass - def visit_tuple_expr(self, o: 'mypy.nodes.TupleExpr') -> T: + def visit_set_expr(self, o: mypy.nodes.SetExpr) -> T: pass - def visit_set_expr(self, o: 'mypy.nodes.SetExpr') -> T: + def visit_index_expr(self, o: mypy.nodes.IndexExpr) -> T: pass - def visit_index_expr(self, o: 'mypy.nodes.IndexExpr') -> T: + def visit_type_application(self, o: mypy.nodes.TypeApplication) -> T: pass - def visit_undefined_expr(self, o: 'mypy.nodes.UndefinedExpr') -> T: + def visit_lambda_expr(self, o: mypy.nodes.LambdaExpr) -> T: pass - def visit_type_application(self, o: 'mypy.nodes.TypeApplication') -> T: + def visit_list_comprehension(self, o: mypy.nodes.ListComprehension) -> T: pass - def visit_func_expr(self, o: 'mypy.nodes.FuncExpr') -> T: + def visit_set_comprehension(self, o: mypy.nodes.SetComprehension) -> T: pass - def visit_list_comprehension(self, o: 'mypy.nodes.ListComprehension') -> T: + def visit_dictionary_comprehension(self, o: mypy.nodes.DictionaryComprehension) -> T: pass - def visit_set_comprehension(self, o: 'mypy.nodes.SetComprehension') -> T: + def visit_generator_expr(self, o: mypy.nodes.GeneratorExpr) -> T: pass - def visit_dictionary_comprehension(self, o: 'mypy.nodes.DictionaryComprehension') -> T: + def visit_slice_expr(self, o: mypy.nodes.SliceExpr) -> T: pass - def visit_generator_expr(self, o: 'mypy.nodes.GeneratorExpr') -> T: + def visit_conditional_expr(self, o: mypy.nodes.ConditionalExpr) -> T: + pass + + def visit_type_var_expr(self, o: mypy.nodes.TypeVarExpr) -> T: + pass + + def visit_paramspec_expr(self, o: mypy.nodes.ParamSpecExpr) -> T: + pass + + def visit_type_var_tuple_expr(self, o: mypy.nodes.TypeVarTupleExpr) -> T: + pass + + def visit_type_alias_expr(self, o: mypy.nodes.TypeAliasExpr) -> T: + pass + + def visit_namedtuple_expr(self, o: mypy.nodes.NamedTupleExpr) -> T: + pass + + def visit_enum_call_expr(self, o: mypy.nodes.EnumCallExpr) -> T: + pass + + def visit_typeddict_expr(self, o: mypy.nodes.TypedDictExpr) -> T: + pass + + def visit_newtype_expr(self, o: mypy.nodes.NewTypeExpr) -> T: + pass + + def visit__promote_expr(self, o: mypy.nodes.PromoteExpr) -> T: + pass + + def visit_await_expr(self, o: mypy.nodes.AwaitExpr) -> T: + pass + + def visit_temp_node(self, o: mypy.nodes.TempNode) -> T: + pass + + # Patterns + + def visit_as_pattern(self, o: mypy.patterns.AsPattern) -> T: pass - def visit_slice_expr(self, o: 'mypy.nodes.SliceExpr') -> T: + def visit_or_pattern(self, o: mypy.patterns.OrPattern) -> T: pass - def visit_conditional_expr(self, o: 'mypy.nodes.ConditionalExpr') -> T: + def visit_value_pattern(self, o: mypy.patterns.ValuePattern) -> T: pass - def visit_type_var_expr(self, o: 'mypy.nodes.TypeVarExpr') -> T: + def visit_singleton_pattern(self, o: mypy.patterns.SingletonPattern) -> T: pass - def visit_namedtuple_expr(self, o: 'mypy.nodes.NamedTupleExpr') -> T: + def visit_sequence_pattern(self, o: mypy.patterns.SequencePattern) -> T: pass - def visit_ducktype_expr(self, o: 'mypy.nodes.DucktypeExpr') -> T: + def visit_starred_pattern(self, o: mypy.patterns.StarredPattern) -> T: pass - def visit_disjointclass_expr(self, o: 'mypy.nodes.DisjointclassExpr') -> T: + def visit_mapping_pattern(self, o: mypy.patterns.MappingPattern) -> T: pass - def visit_temp_node(self, o: 'mypy.nodes.TempNode') -> T: + def visit_class_pattern(self, o: mypy.patterns.ClassPattern) -> T: pass diff --git a/mypy/xml/mypy-html.css b/mypy/xml/mypy-html.css new file mode 100644 index 000000000000..ec2bdf9c9668 --- /dev/null +++ b/mypy/xml/mypy-html.css @@ -0,0 +1,104 @@ +/* CSS for type check coverage reports */ + +/* + Used by both summary and file. +*/ +body { + font-family: "Helvetica Neue", sans-serif; +} + +/* + Used only by summary. +*/ + +h1 { + text-align: center; + font-size: 135%; + margin: 20px; +} + +table.summary { + border-collapse: collapse; + margin-left: 7%; + margin-right: 7%; + width: 85%; +} + +table caption { + margin: 1em; +} + +table.summary, tr.summary, th.summary, td.summary { + border: 1px solid #aaa; +} + +th.summary, td.summary { + padding: 0.4em; +} + +td.summary a { + text-decoration: none; +} + +.summary-quality-0 { + background-color: #dfd; +} + +.summary-quality-1 { + background-color: #ffa; +} + +.summary-quality-2 { + background-color: #faa; +} + +td.summary-filename, th.summary-filename { + text-align: left; +} + +td.summary-filename { + width: 50%; +} + +.summary-precision { + text-align: center; +} + +.summary-lines { + text-align: center; +} + +/* + Used only by file. +*/ + +td.table-lines { + text-align: right; + padding-right: 0.5em; +} + +td.table-code { } + +span.lineno { + text-align: right; +} + +a:link.lineno, a:visited.lineno { + color: #999; text-decoration: none; +} + +a:hover.lineno, a:active.lineno { + color: #000; text-decoration: underline; +} + +.line-empty, .line-precise { + background-color: #dfd; +} + +.line-imprecise { + background-color: #ffa; +} + +.line-any, .line-unanalyzed { + background-color: #faa; +} diff --git a/mypy/xml/mypy-html.xslt b/mypy/xml/mypy-html.xslt new file mode 100644 index 000000000000..ddd78c2449ee --- /dev/null +++ b/mypy/xml/mypy-html.xslt @@ -0,0 +1,81 @@ + + + + + + + + + + + + + +

Mypy Type Check Coverage Summary

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Summary from
FileImprecisionLines
Total imprecise LOC
imprecise LOC
+ + +
+ + + + + + +

+ + + + + + + + +
+
+                  
+                    

+                  
+                
+
+
+                  
+                    

+                  
+                
+
+ + +
+
diff --git a/mypy/xml/mypy-txt.xslt b/mypy/xml/mypy-txt.xslt new file mode 100644 index 000000000000..fe12065c1ce3 --- /dev/null +++ b/mypy/xml/mypy-txt.xslt @@ -0,0 +1,100 @@ + + + + + + + Mypy Type Check Coverage Summary + ================================ + + Script: + + + + + + + + + + + + + + +- + + -+- + + -+- + + -+ + + | + + + | + + + | + + + | + + +- + + -+- + + -+- + + -+ + + + + + + + | + + + | + + + | + + + | + + + +- + + -+- + + -+- + + -+ + + + + + + + | + + + | + + + | + + + | + + +- + + -+- + + -+- + + -+ + + diff --git a/mypy/xml/mypy.xsd b/mypy/xml/mypy.xsd new file mode 100644 index 000000000000..77d0737a4f19 --- /dev/null +++ b/mypy/xml/mypy.xsd @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mypy_bootstrap.ini b/mypy_bootstrap.ini new file mode 100644 index 000000000000..6e82f23b0530 --- /dev/null +++ b/mypy_bootstrap.ini @@ -0,0 +1,9 @@ +[mypy] +strict = True +warn_unused_ignores = False +show_traceback = True +always_true = MYPYC + +[mypy-mypy.visitor] +# See docstring for NodeVisitor for motivation. +disable_error_code = empty-body diff --git a/mypy_self_check.ini b/mypy_self_check.ini new file mode 100644 index 000000000000..7f1f9689a757 --- /dev/null +++ b/mypy_self_check.ini @@ -0,0 +1,17 @@ +[mypy] + +strict = True +disallow_any_unimported = True +show_traceback = True +pretty = True +always_false = MYPYC +plugins = mypy.plugins.proper_plugin +python_version = 3.8 +exclude = mypy/typeshed/|mypyc/test-data/|mypyc/lib-rt/ +enable_error_code = ignore-without-code,redundant-expr +enable_incomplete_feature = PreciseTupleTypes +show_error_code_links = True + +[mypy-mypy.visitor] +# See docstring for NodeVisitor for motivation. +disable_error_code = empty-body diff --git a/mypyc/.readthedocs.yaml b/mypyc/.readthedocs.yaml new file mode 100644 index 000000000000..90831dfd7069 --- /dev/null +++ b/mypyc/.readthedocs.yaml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: mypyc/doc/conf.py + +formats: [pdf, htmlzip, epub] + +python: + install: + - requirements: docs/requirements-docs.txt diff --git a/mypyc/README.md b/mypyc/README.md new file mode 100644 index 000000000000..cb6cf5bf225c --- /dev/null +++ b/mypyc/README.md @@ -0,0 +1,133 @@ +mypyc: Mypy to Python C Extension Compiler +========================================== + +**NOTE: We are in the process of moving the mypyc README to the** +**[mypyc repository](https://github.com/mypyc/mypyc)** + +**This may be out of date!** + +Mypyc is a compiler that compiles mypy-annotated, statically typed +Python modules into CPython C extensions. Currently our primary focus +is on making mypy faster through compilation -- the default mypy wheels +are compiled with mypyc. Compiled mypy is about 4x faster than +without compilation. + +Mypyc compiles what is essentially a Python language variant using "strict" +semantics. This means (among some other things): + + * Most type annotations are enforced at runtime (raising ``TypeError`` on mismatch) + + * Classes are compiled into extension classes without ``__dict__`` + (much, but not quite, like if they used ``__slots__``) + + * Monkey patching doesn't work + + * Instance attributes won't fall back to class attributes if undefined + + * Also there are still a bunch of bad bugs and unsupported features :) + +Compiled modules can import arbitrary Python modules, and compiled modules +can be used from other Python modules. Typically mypyc is used to only +compile modules that contain performance bottlenecks. + +You can run compiled modules also as normal, interpreted Python +modules, since mypyc targets valid Python code. This means that +all Python developer tools and debuggers can be used. + +macOS Requirements +------------------ + +* macOS Sierra or later + +* Xcode command line tools + +* Python 3.5+ from python.org (other versions are untested) + +Linux Requirements +------------------ + +* A recent enough C/C++ build environment + +* Python 3.5+ + +Windows Requirements +-------------------- + +* Windows has been tested with Windows 10 and MSVC 2017. + +* Python 3.5+ + +Quick Start for Contributors +---------------------------- + +First clone the mypy git repository: + + $ git clone https://github.com/python/mypy.git + $ cd mypy + +Optionally create a virtualenv (recommended): + + $ python3 -m venv + $ source /bin/activate + +Then install the dependencies: + + $ python3 -m pip install -r test-requirements.txt + +Now you can run the tests: + + $ pytest -q mypyc + +Look at the [issue tracker](https://github.com/mypyc/mypyc/issues) +for things to work on. Please express your interest in working on an +issue by adding a comment before doing any significant work, since +there is a risk of duplicate work. + +Note that the issue tracker is hosted on the mypyc GitHub project, not +with mypy itself. + +Documentation +------------- + +We have some [developer documentation](doc/dev-intro.md). + +Development Status and Roadmap +------------------------------ + +These are the current planned major milestones: + +1. [DONE] Support a smallish but useful Python subset. Focus on compiling + single modules, while the rest of the program is interpreted and does not + need to be type checked. + +2. [DONE] Support compiling multiple modules as a single compilation unit (or + dynamic linking of compiled modules). Without this inter-module + calls will use slower Python-level objects, wrapper functions and + Python namespaces. + +3. [DONE] Mypyc can compile mypy. + +4. [DONE] Optimize some important performance bottlenecks. + +5. [PARTIALLY DONE] Generate useful errors for code that uses unsupported Python + features instead of crashing or generating bad code. + +6. [DONE] Release a version of mypy that includes a compiled mypy. + +7. + 1. More feature/compatibility work. (100% compatibility with Python is distinctly + an anti-goal, but more than we have now is a good idea.) + 2. [DONE] Support compiling Black, which is a prominent tool that could benefit + and has maintainer buy-in. + (Let us know if you maintain another Python tool or library and are + interested in working with us on this!) + 3. More optimization! Code size reductions in particular are likely to + be valuable and will speed up mypyc compilation. + +8. We'll see! Adventure is out there! + +Future +------ + +We have some ideas for +[future improvements and optimizations](doc/future.md). diff --git a/mypyc/__init__.py b/mypyc/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/__main__.py b/mypyc/__main__.py new file mode 100644 index 000000000000..653199e0fb55 --- /dev/null +++ b/mypyc/__main__.py @@ -0,0 +1,58 @@ +"""Mypyc command-line tool. + +Usage: + + $ mypyc foo.py [...] + $ python3 -c 'import foo' # Uses compiled 'foo' + + +This is just a thin wrapper that generates a setup.py file that uses +mypycify, suitable for prototyping and testing. +""" + +from __future__ import annotations + +import os +import os.path +import subprocess +import sys + +base_path = os.path.join(os.path.dirname(__file__), "..") + +setup_format = """\ +from setuptools import setup +from mypyc.build import mypycify + +setup(name='mypyc_output', + ext_modules=mypycify({}, opt_level="{}", debug_level="{}", strict_dunder_typing={}), +) +""" + + +def main() -> None: + build_dir = "build" # can this be overridden?? + try: + os.mkdir(build_dir) + except FileExistsError: + pass + + opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") + debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") + strict_dunder_typing = bool(int(os.getenv("MYPYC_STRICT_DUNDER_TYPING", "0"))) + + setup_file = os.path.join(build_dir, "setup.py") + with open(setup_file, "w") as f: + f.write(setup_format.format(sys.argv[1:], opt_level, debug_level, strict_dunder_typing)) + + # We don't use run_setup (like we do in the test suite) because it throws + # away the error code from distutils, and we don't care about the slight + # performance loss here. + env = os.environ.copy() + base_path = os.path.join(os.path.dirname(__file__), "..") + env["PYTHONPATH"] = base_path + os.pathsep + env.get("PYTHONPATH", "") + cmd = subprocess.run([sys.executable, setup_file, "build_ext", "--inplace"], env=env) + sys.exit(cmd.returncode) + + +if __name__ == "__main__": + main() diff --git a/mypyc/analysis/__init__.py b/mypyc/analysis/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/analysis/attrdefined.py b/mypyc/analysis/attrdefined.py new file mode 100644 index 000000000000..350158246cdb --- /dev/null +++ b/mypyc/analysis/attrdefined.py @@ -0,0 +1,436 @@ +"""Always defined attribute analysis. + +An always defined attribute has some statements in __init__ or the +class body that cause the attribute to be always initialized when an +instance is constructed. It must also not be possible to read the +attribute before initialization, and it can't be deletable. + +We can assume that the value is always defined when reading an always +defined attribute. Otherwise we'll need to raise AttributeError if the +value is undefined (i.e. has the error value). + +We use data flow analysis to figure out attributes that are always +defined. Example: + + class C: + def __init__(self) -> None: + self.x = 0 + if func(): + self.y = 1 + else: + self.y = 2 + self.z = 3 + +In this example, the attributes 'x' and 'y' are always defined, but 'z' +is not. The analysis assumes that we know that there won't be any subclasses. + +The analysis also works if there is a known, closed set of subclasses. +An attribute defined in a base class can only be always defined if it's +also always defined in all subclasses. + +As soon as __init__ contains an op that can 'leak' self to another +function, we will stop inferring always defined attributes, since the +analysis is mostly intra-procedural and only looks at __init__ methods. +The called code could read an uninitialized attribute. Example: + + class C: + def __init__(self) -> None: + self.x = self.foo() + + def foo(self) -> int: + ... + +Now we won't infer 'x' as always defined, since 'foo' might read 'x' +before initialization. + +As an exception to the above limitation, we perform inter-procedural +analysis of super().__init__ calls, since these are very common. + +Our analysis is somewhat optimistic. We assume that nobody calls a +method of a partially uninitialized object through gc.get_objects(), in +particular. Code like this could potentially cause a segfault with a null +pointer dereference. This seems very unlikely to be an issue in practice, +however. + +Accessing an attribute via getattr always checks for undefined attributes +and thus works if the object is partially uninitialized. This can be used +as a workaround if somebody ever needs to inspect partially uninitialized +objects via gc.get_objects(). + +The analysis runs after IR building as a separate pass. Since we only +run this on __init__ methods, this analysis pass will be fairly quick. +""" + +from __future__ import annotations + +from typing import Final, Set, Tuple + +from mypyc.analysis.dataflow import ( + CFG, + MAYBE_ANALYSIS, + AnalysisResult, + BaseAnalysisVisitor, + get_cfg, + run_analysis, +) +from mypyc.analysis.selfleaks import analyze_self_leaks +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.ops import ( + Assign, + AssignMulti, + BasicBlock, + Branch, + Call, + ControlOp, + GetAttr, + Register, + RegisterOp, + Return, + SetAttr, + SetMem, + Unreachable, +) +from mypyc.ir.rtypes import RInstance + +# If True, print out all always-defined attributes of native classes (to aid +# debugging and testing) +dump_always_defined: Final = False + + +def analyze_always_defined_attrs(class_irs: list[ClassIR]) -> None: + """Find always defined attributes all classes of a compilation unit. + + Also tag attribute initialization ops to not decref the previous + value (as this would read a NULL pointer and segfault). + + Update the _always_initialized_attrs, _sometimes_initialized_attrs + and init_self_leak attributes in ClassIR instances. + + This is the main entry point. + """ + seen: set[ClassIR] = set() + + # First pass: only look at target class and classes in MRO + for cl in class_irs: + analyze_always_defined_attrs_in_class(cl, seen) + + # Second pass: look at all derived class + seen = set() + for cl in class_irs: + update_always_defined_attrs_using_subclasses(cl, seen) + + # Final pass: detect attributes that need to use a bitmap to track definedness + seen = set() + for cl in class_irs: + detect_undefined_bitmap(cl, seen) + + +def analyze_always_defined_attrs_in_class(cl: ClassIR, seen: set[ClassIR]) -> None: + if cl in seen: + return + + seen.add(cl) + + if ( + cl.is_trait + or cl.inherits_python + or cl.allow_interpreted_subclasses + or cl.builtin_base is not None + or cl.children is None + or cl.is_serializable() + ): + # Give up -- we can't enforce that attributes are always defined. + return + + # First analyze all base classes. Track seen classes to avoid duplicate work. + for base in cl.mro[1:]: + analyze_always_defined_attrs_in_class(base, seen) + + m = cl.get_method("__init__") + if m is None: + cl._always_initialized_attrs = cl.attrs_with_defaults.copy() + cl._sometimes_initialized_attrs = cl.attrs_with_defaults.copy() + return + self_reg = m.arg_regs[0] + cfg = get_cfg(m.blocks) + dirty = analyze_self_leaks(m.blocks, self_reg, cfg) + maybe_defined = analyze_maybe_defined_attrs_in_init( + m.blocks, self_reg, cl.attrs_with_defaults, cfg + ) + all_attrs: set[str] = set() + for base in cl.mro: + all_attrs.update(base.attributes) + maybe_undefined = analyze_maybe_undefined_attrs_in_init( + m.blocks, self_reg, initial_undefined=all_attrs - cl.attrs_with_defaults, cfg=cfg + ) + + always_defined = find_always_defined_attributes( + m.blocks, self_reg, all_attrs, maybe_defined, maybe_undefined, dirty + ) + always_defined = {a for a in always_defined if not cl.is_deletable(a)} + + cl._always_initialized_attrs = always_defined + if dump_always_defined: + print(cl.name, sorted(always_defined)) + cl._sometimes_initialized_attrs = find_sometimes_defined_attributes( + m.blocks, self_reg, maybe_defined, dirty + ) + + mark_attr_initialiation_ops(m.blocks, self_reg, maybe_defined, dirty) + + # Check if __init__ can run unpredictable code (leak 'self'). + any_dirty = False + for b in m.blocks: + for i, op in enumerate(b.ops): + if dirty.after[b, i] and not isinstance(op, Return): + any_dirty = True + break + cl.init_self_leak = any_dirty + + +def find_always_defined_attributes( + blocks: list[BasicBlock], + self_reg: Register, + all_attrs: set[str], + maybe_defined: AnalysisResult[str], + maybe_undefined: AnalysisResult[str], + dirty: AnalysisResult[None], +) -> set[str]: + """Find attributes that are always initialized in some basic blocks. + + The analysis results are expected to be up-to-date for the blocks. + + Return a set of always defined attributes. + """ + attrs = all_attrs.copy() + for block in blocks: + for i, op in enumerate(block.ops): + # If an attribute we *read* may be undefined, it isn't always defined. + if isinstance(op, GetAttr) and op.obj is self_reg: + if op.attr in maybe_undefined.before[block, i]: + attrs.discard(op.attr) + # If an attribute we *set* may be sometimes undefined and + # sometimes defined, don't consider it always defined. Unlike + # the get case, it's fine for the attribute to be undefined. + # The set operation will then be treated as initialization. + if isinstance(op, SetAttr) and op.obj is self_reg: + if ( + op.attr in maybe_undefined.before[block, i] + and op.attr in maybe_defined.before[block, i] + ): + attrs.discard(op.attr) + # Treat an op that might run arbitrary code as an "exit" + # in terms of the analysis -- we can't do any inference + # afterwards reliably. + if dirty.after[block, i]: + if not dirty.before[block, i]: + attrs = attrs & ( + maybe_defined.after[block, i] - maybe_undefined.after[block, i] + ) + break + if isinstance(op, ControlOp): + for target in op.targets(): + # Gotos/branches can also be "exits". + if not dirty.after[block, i] and dirty.before[target, 0]: + attrs = attrs & ( + maybe_defined.after[target, 0] - maybe_undefined.after[target, 0] + ) + return attrs + + +def find_sometimes_defined_attributes( + blocks: list[BasicBlock], + self_reg: Register, + maybe_defined: AnalysisResult[str], + dirty: AnalysisResult[None], +) -> set[str]: + """Find attributes that are sometimes initialized in some basic blocks.""" + attrs: set[str] = set() + for block in blocks: + for i, op in enumerate(block.ops): + # Only look at possibly defined attributes at exits. + if dirty.after[block, i]: + if not dirty.before[block, i]: + attrs = attrs | maybe_defined.after[block, i] + break + if isinstance(op, ControlOp): + for target in op.targets(): + if not dirty.after[block, i] and dirty.before[target, 0]: + attrs = attrs | maybe_defined.after[target, 0] + return attrs + + +def mark_attr_initialiation_ops( + blocks: list[BasicBlock], + self_reg: Register, + maybe_defined: AnalysisResult[str], + dirty: AnalysisResult[None], +) -> None: + """Tag all SetAttr ops in the basic blocks that initialize attributes. + + Initialization ops assume that the previous attribute value is the error value, + so there's no need to decref or check for definedness. + """ + for block in blocks: + for i, op in enumerate(block.ops): + if isinstance(op, SetAttr) and op.obj is self_reg: + attr = op.attr + if attr not in maybe_defined.before[block, i] and not dirty.after[block, i]: + op.mark_as_initializer() + + +GenAndKill = Tuple[Set[str], Set[str]] + + +def attributes_initialized_by_init_call(op: Call) -> set[str]: + """Calculate attributes that are always initialized by a super().__init__ call.""" + self_type = op.fn.sig.args[0].type + assert isinstance(self_type, RInstance) + cl = self_type.class_ir + return {a for base in cl.mro for a in base.attributes if base.is_always_defined(a)} + + +def attributes_maybe_initialized_by_init_call(op: Call) -> set[str]: + """Calculate attributes that may be initialized by a super().__init__ call.""" + self_type = op.fn.sig.args[0].type + assert isinstance(self_type, RInstance) + cl = self_type.class_ir + return attributes_initialized_by_init_call(op) | cl._sometimes_initialized_attrs + + +class AttributeMaybeDefinedVisitor(BaseAnalysisVisitor[str]): + """Find attributes that may have been defined via some code path. + + Consider initializations in class body and assignments to 'self.x' + and calls to base class '__init__'. + """ + + def __init__(self, self_reg: Register) -> None: + self.self_reg = self_reg + + def visit_branch(self, op: Branch) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_return(self, op: Return) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_unreachable(self, op: Unreachable) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_register_op(self, op: RegisterOp) -> tuple[set[str], set[str]]: + if isinstance(op, SetAttr) and op.obj is self.self_reg: + return {op.attr}, set() + if isinstance(op, Call) and op.fn.class_name and op.fn.name == "__init__": + return attributes_maybe_initialized_by_init_call(op), set() + return set(), set() + + def visit_assign(self, op: Assign) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_assign_multi(self, op: AssignMulti) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_set_mem(self, op: SetMem) -> tuple[set[str], set[str]]: + return set(), set() + + +def analyze_maybe_defined_attrs_in_init( + blocks: list[BasicBlock], self_reg: Register, attrs_with_defaults: set[str], cfg: CFG +) -> AnalysisResult[str]: + return run_analysis( + blocks=blocks, + cfg=cfg, + gen_and_kill=AttributeMaybeDefinedVisitor(self_reg), + initial=attrs_with_defaults, + backward=False, + kind=MAYBE_ANALYSIS, + ) + + +class AttributeMaybeUndefinedVisitor(BaseAnalysisVisitor[str]): + """Find attributes that may be undefined via some code path. + + Consider initializations in class body, assignments to 'self.x' + and calls to base class '__init__'. + """ + + def __init__(self, self_reg: Register) -> None: + self.self_reg = self_reg + + def visit_branch(self, op: Branch) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_return(self, op: Return) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_unreachable(self, op: Unreachable) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_register_op(self, op: RegisterOp) -> tuple[set[str], set[str]]: + if isinstance(op, SetAttr) and op.obj is self.self_reg: + return set(), {op.attr} + if isinstance(op, Call) and op.fn.class_name and op.fn.name == "__init__": + return set(), attributes_initialized_by_init_call(op) + return set(), set() + + def visit_assign(self, op: Assign) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_assign_multi(self, op: AssignMulti) -> tuple[set[str], set[str]]: + return set(), set() + + def visit_set_mem(self, op: SetMem) -> tuple[set[str], set[str]]: + return set(), set() + + +def analyze_maybe_undefined_attrs_in_init( + blocks: list[BasicBlock], self_reg: Register, initial_undefined: set[str], cfg: CFG +) -> AnalysisResult[str]: + return run_analysis( + blocks=blocks, + cfg=cfg, + gen_and_kill=AttributeMaybeUndefinedVisitor(self_reg), + initial=initial_undefined, + backward=False, + kind=MAYBE_ANALYSIS, + ) + + +def update_always_defined_attrs_using_subclasses(cl: ClassIR, seen: set[ClassIR]) -> None: + """Remove attributes not defined in all subclasses from always defined attrs.""" + if cl in seen: + return + if cl.children is None: + # Subclasses are unknown + return + removed = set() + for attr in cl._always_initialized_attrs: + for child in cl.children: + update_always_defined_attrs_using_subclasses(child, seen) + if attr not in child._always_initialized_attrs: + removed.add(attr) + cl._always_initialized_attrs -= removed + seen.add(cl) + + +def detect_undefined_bitmap(cl: ClassIR, seen: set[ClassIR]) -> None: + if cl.is_trait: + return + + if cl in seen: + return + seen.add(cl) + for base in cl.base_mro[1:]: + detect_undefined_bitmap(cl, seen) + + if len(cl.base_mro) > 1: + cl.bitmap_attrs.extend(cl.base_mro[1].bitmap_attrs) + for n, t in cl.attributes.items(): + if t.error_overlap and not cl.is_always_defined(n): + cl.bitmap_attrs.append(n) + + for base in cl.mro[1:]: + if base.is_trait: + for n, t in base.attributes.items(): + if t.error_overlap and not cl.is_always_defined(n) and n not in cl.bitmap_attrs: + cl.bitmap_attrs.append(n) diff --git a/mypyc/analysis/blockfreq.py b/mypyc/analysis/blockfreq.py new file mode 100644 index 000000000000..74a1bc0579c6 --- /dev/null +++ b/mypyc/analysis/blockfreq.py @@ -0,0 +1,32 @@ +"""Find basic blocks that are likely to be executed frequently. + +For example, this would not include blocks that have exception handlers. + +We can use different optimization heuristics for common and rare code. For +example, we can make IR fast to compile instead of fast to execute for rare +code. +""" + +from __future__ import annotations + +from mypyc.ir.ops import BasicBlock, Branch, Goto + + +def frequently_executed_blocks(entry_point: BasicBlock) -> set[BasicBlock]: + result: set[BasicBlock] = set() + worklist = [entry_point] + while worklist: + block = worklist.pop() + if block in result: + continue + result.add(block) + t = block.terminator + if isinstance(t, Goto): + worklist.append(t.label) + elif isinstance(t, Branch): + if t.rare or t.traceback_entry is not None: + worklist.append(t.false) + else: + worklist.append(t.true) + worklist.append(t.false) + return result diff --git a/mypyc/analysis/dataflow.py b/mypyc/analysis/dataflow.py new file mode 100644 index 000000000000..411fc8093404 --- /dev/null +++ b/mypyc/analysis/dataflow.py @@ -0,0 +1,606 @@ +"""Data-flow analyses.""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import Dict, Generic, Iterable, Iterator, Set, Tuple, TypeVar + +from mypyc.ir.ops import ( + Assign, + AssignMulti, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + ControlOp, + Extend, + Float, + FloatComparisonOp, + FloatNeg, + FloatOp, + GetAttr, + GetElementPtr, + Goto, + InitStatic, + Integer, + IntOp, + KeepAlive, + LoadAddress, + LoadErrorValue, + LoadGlobal, + LoadLiteral, + LoadMem, + LoadStatic, + MethodCall, + Op, + OpVisitor, + PrimitiveOp, + RaiseStandardError, + RegisterOp, + Return, + SetAttr, + SetMem, + Truncate, + TupleGet, + TupleSet, + Unborrow, + Unbox, + Unreachable, + Value, +) + + +class CFG: + """Control-flow graph. + + Node 0 is always assumed to be the entry point. There must be a + non-empty set of exits. + """ + + def __init__( + self, + succ: dict[BasicBlock, list[BasicBlock]], + pred: dict[BasicBlock, list[BasicBlock]], + exits: set[BasicBlock], + ) -> None: + assert exits + self.succ = succ + self.pred = pred + self.exits = exits + + def __str__(self) -> str: + exits = sorted(self.exits, key=lambda e: int(e.label)) + return f"exits: {exits}\nsucc: {self.succ}\npred: {self.pred}" + + +def get_cfg(blocks: list[BasicBlock]) -> CFG: + """Calculate basic block control-flow graph. + + The result is a dictionary like this: + + basic block index -> (successors blocks, predecesssor blocks) + """ + succ_map = {} + pred_map: dict[BasicBlock, list[BasicBlock]] = {} + exits = set() + for block in blocks: + assert not any( + isinstance(op, ControlOp) for op in block.ops[:-1] + ), "Control-flow ops must be at the end of blocks" + + succ = list(block.terminator.targets()) + if not succ: + exits.add(block) + + # Errors can occur anywhere inside a block, which means that + # we can't assume that the entire block has executed before + # jumping to the error handler. In our CFG construction, we + # model this as saying that a block can jump to its error + # handler or the error handlers of any of its normal + # successors (to represent an error before that next block + # completes). This works well for analyses like "must + # defined", where it implies that registers assigned in a + # block may be undefined in its error handler, but is in + # general not a precise representation of reality; any + # analyses that require more fidelity must wait until after + # exception insertion. + for error_point in [block] + succ: + if error_point.error_handler: + succ.append(error_point.error_handler) + + succ_map[block] = succ + pred_map[block] = [] + for prev, nxt in succ_map.items(): + for label in nxt: + pred_map[label].append(prev) + return CFG(succ_map, pred_map, exits) + + +def get_real_target(label: BasicBlock) -> BasicBlock: + if len(label.ops) == 1 and isinstance(label.ops[-1], Goto): + label = label.ops[-1].label + return label + + +def cleanup_cfg(blocks: list[BasicBlock]) -> None: + """Cleanup the control flow graph. + + This eliminates obviously dead basic blocks and eliminates blocks that contain + nothing but a single jump. + + There is a lot more that could be done. + """ + changed = True + while changed: + # First collapse any jumps to basic block that only contain a goto + for block in blocks: + for i, tgt in enumerate(block.terminator.targets()): + block.terminator.set_target(i, get_real_target(tgt)) + + # Then delete any blocks that have no predecessors + changed = False + cfg = get_cfg(blocks) + orig_blocks = blocks.copy() + blocks.clear() + for i, block in enumerate(orig_blocks): + if i == 0 or cfg.pred[block]: + blocks.append(block) + else: + changed = True + + +T = TypeVar("T") + +AnalysisDict = Dict[Tuple[BasicBlock, int], Set[T]] + + +class AnalysisResult(Generic[T]): + def __init__(self, before: AnalysisDict[T], after: AnalysisDict[T]) -> None: + self.before = before + self.after = after + + def __str__(self) -> str: + return f"before: {self.before}\nafter: {self.after}\n" + + +GenAndKill = Tuple[Set[T], Set[T]] + + +class BaseAnalysisVisitor(OpVisitor[GenAndKill[T]]): + def visit_goto(self, op: Goto) -> GenAndKill[T]: + return set(), set() + + @abstractmethod + def visit_register_op(self, op: RegisterOp) -> GenAndKill[T]: + raise NotImplementedError + + @abstractmethod + def visit_assign(self, op: Assign) -> GenAndKill[T]: + raise NotImplementedError + + @abstractmethod + def visit_assign_multi(self, op: AssignMulti) -> GenAndKill[T]: + raise NotImplementedError + + @abstractmethod + def visit_set_mem(self, op: SetMem) -> GenAndKill[T]: + raise NotImplementedError + + def visit_call(self, op: Call) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_method_call(self, op: MethodCall) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_load_error_value(self, op: LoadErrorValue) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_load_literal(self, op: LoadLiteral) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_get_attr(self, op: GetAttr) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_set_attr(self, op: SetAttr) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_load_static(self, op: LoadStatic) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_init_static(self, op: InitStatic) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_tuple_get(self, op: TupleGet) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_tuple_set(self, op: TupleSet) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_box(self, op: Box) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_unbox(self, op: Unbox) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_cast(self, op: Cast) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_raise_standard_error(self, op: RaiseStandardError) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_call_c(self, op: CallC) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_primitive_op(self, op: PrimitiveOp) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_truncate(self, op: Truncate) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_extend(self, op: Extend) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_load_global(self, op: LoadGlobal) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_int_op(self, op: IntOp) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_float_op(self, op: FloatOp) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_float_neg(self, op: FloatNeg) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_comparison_op(self, op: ComparisonOp) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_float_comparison_op(self, op: FloatComparisonOp) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_load_mem(self, op: LoadMem) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_load_address(self, op: LoadAddress) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_keep_alive(self, op: KeepAlive) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_unborrow(self, op: Unborrow) -> GenAndKill[T]: + return self.visit_register_op(op) + + +class DefinedVisitor(BaseAnalysisVisitor[Value]): + """Visitor for finding defined registers. + + Note that this only deals with registers and not temporaries, on + the assumption that we never access temporaries when they might be + undefined. + + If strict_errors is True, then we regard any use of LoadErrorValue + as making a register undefined. Otherwise we only do if + `undefines` is set on the error value. + + This lets us only consider the things we care about during + uninitialized variable checking while capturing all possibly + undefined things for refcounting. + """ + + def __init__(self, strict_errors: bool = False) -> None: + self.strict_errors = strict_errors + + def visit_branch(self, op: Branch) -> GenAndKill[Value]: + return set(), set() + + def visit_return(self, op: Return) -> GenAndKill[Value]: + return set(), set() + + def visit_unreachable(self, op: Unreachable) -> GenAndKill[Value]: + return set(), set() + + def visit_register_op(self, op: RegisterOp) -> GenAndKill[Value]: + return set(), set() + + def visit_assign(self, op: Assign) -> GenAndKill[Value]: + # Loading an error value may undefine the register. + if isinstance(op.src, LoadErrorValue) and (op.src.undefines or self.strict_errors): + return set(), {op.dest} + else: + return {op.dest}, set() + + def visit_assign_multi(self, op: AssignMulti) -> GenAndKill[Value]: + # Array registers are special and we don't track the definedness of them. + return set(), set() + + def visit_set_mem(self, op: SetMem) -> GenAndKill[Value]: + return set(), set() + + +def analyze_maybe_defined_regs( + blocks: list[BasicBlock], cfg: CFG, initial_defined: set[Value] +) -> AnalysisResult[Value]: + """Calculate potentially defined registers at each CFG location. + + A register is defined if it has a value along some path from the initial location. + """ + return run_analysis( + blocks=blocks, + cfg=cfg, + gen_and_kill=DefinedVisitor(), + initial=initial_defined, + backward=False, + kind=MAYBE_ANALYSIS, + ) + + +def analyze_must_defined_regs( + blocks: list[BasicBlock], + cfg: CFG, + initial_defined: set[Value], + regs: Iterable[Value], + strict_errors: bool = False, +) -> AnalysisResult[Value]: + """Calculate always defined registers at each CFG location. + + This analysis can work before exception insertion, since it is a + sound assumption that registers defined in a block might not be + initialized in its error handler. + + A register is defined if it has a value along all paths from the + initial location. + """ + return run_analysis( + blocks=blocks, + cfg=cfg, + gen_and_kill=DefinedVisitor(strict_errors=strict_errors), + initial=initial_defined, + backward=False, + kind=MUST_ANALYSIS, + universe=set(regs), + ) + + +class BorrowedArgumentsVisitor(BaseAnalysisVisitor[Value]): + def __init__(self, args: set[Value]) -> None: + self.args = args + + def visit_branch(self, op: Branch) -> GenAndKill[Value]: + return set(), set() + + def visit_return(self, op: Return) -> GenAndKill[Value]: + return set(), set() + + def visit_unreachable(self, op: Unreachable) -> GenAndKill[Value]: + return set(), set() + + def visit_register_op(self, op: RegisterOp) -> GenAndKill[Value]: + return set(), set() + + def visit_assign(self, op: Assign) -> GenAndKill[Value]: + if op.dest in self.args: + return set(), {op.dest} + return set(), set() + + def visit_assign_multi(self, op: AssignMulti) -> GenAndKill[Value]: + return set(), set() + + def visit_set_mem(self, op: SetMem) -> GenAndKill[Value]: + return set(), set() + + +def analyze_borrowed_arguments( + blocks: list[BasicBlock], cfg: CFG, borrowed: set[Value] +) -> AnalysisResult[Value]: + """Calculate arguments that can use references borrowed from the caller. + + When assigning to an argument, it no longer is borrowed. + """ + return run_analysis( + blocks=blocks, + cfg=cfg, + gen_and_kill=BorrowedArgumentsVisitor(borrowed), + initial=borrowed, + backward=False, + kind=MUST_ANALYSIS, + universe=borrowed, + ) + + +class UndefinedVisitor(BaseAnalysisVisitor[Value]): + def visit_branch(self, op: Branch) -> GenAndKill[Value]: + return set(), set() + + def visit_return(self, op: Return) -> GenAndKill[Value]: + return set(), set() + + def visit_unreachable(self, op: Unreachable) -> GenAndKill[Value]: + return set(), set() + + def visit_register_op(self, op: RegisterOp) -> GenAndKill[Value]: + return set(), {op} if not op.is_void else set() + + def visit_assign(self, op: Assign) -> GenAndKill[Value]: + return set(), {op.dest} + + def visit_assign_multi(self, op: AssignMulti) -> GenAndKill[Value]: + return set(), {op.dest} + + def visit_set_mem(self, op: SetMem) -> GenAndKill[Value]: + return set(), set() + + +def non_trivial_sources(op: Op) -> set[Value]: + result = set() + for source in op.sources(): + if not isinstance(source, (Integer, Float)): + result.add(source) + return result + + +class LivenessVisitor(BaseAnalysisVisitor[Value]): + def visit_branch(self, op: Branch) -> GenAndKill[Value]: + return non_trivial_sources(op), set() + + def visit_return(self, op: Return) -> GenAndKill[Value]: + if not isinstance(op.value, (Integer, Float)): + return {op.value}, set() + else: + return set(), set() + + def visit_unreachable(self, op: Unreachable) -> GenAndKill[Value]: + return set(), set() + + def visit_register_op(self, op: RegisterOp) -> GenAndKill[Value]: + gen = non_trivial_sources(op) + if not op.is_void: + return gen, {op} + else: + return gen, set() + + def visit_assign(self, op: Assign) -> GenAndKill[Value]: + return non_trivial_sources(op), {op.dest} + + def visit_assign_multi(self, op: AssignMulti) -> GenAndKill[Value]: + return non_trivial_sources(op), {op.dest} + + def visit_set_mem(self, op: SetMem) -> GenAndKill[Value]: + return non_trivial_sources(op), set() + + +def analyze_live_regs(blocks: list[BasicBlock], cfg: CFG) -> AnalysisResult[Value]: + """Calculate live registers at each CFG location. + + A register is live at a location if it can be read along some CFG path starting + from the location. + """ + return run_analysis( + blocks=blocks, + cfg=cfg, + gen_and_kill=LivenessVisitor(), + initial=set(), + backward=True, + kind=MAYBE_ANALYSIS, + ) + + +# Analysis kinds +MUST_ANALYSIS = 0 +MAYBE_ANALYSIS = 1 + + +def run_analysis( + blocks: list[BasicBlock], + cfg: CFG, + gen_and_kill: OpVisitor[GenAndKill[T]], + initial: set[T], + kind: int, + backward: bool, + universe: set[T] | None = None, +) -> AnalysisResult[T]: + """Run a general set-based data flow analysis. + + Args: + blocks: All basic blocks + cfg: Control-flow graph for the code + gen_and_kill: Implementation of gen and kill functions for each op + initial: Value of analysis for the entry points (for a forward analysis) or the + exit points (for a backward analysis) + kind: MUST_ANALYSIS or MAYBE_ANALYSIS + backward: If False, the analysis is a forward analysis; it's backward otherwise + universe: For a must analysis, the set of all possible values. This is the starting + value for the work list algorithm, which will narrow this down until reaching a + fixed point. For a maybe analysis the iteration always starts from an empty set + and this argument is ignored. + + Return analysis results: (before, after) + """ + block_gen = {} + block_kill = {} + + # Calculate kill and gen sets for entire basic blocks. + for block in blocks: + gen: set[T] = set() + kill: set[T] = set() + ops = block.ops + if backward: + ops = list(reversed(ops)) + for op in ops: + opgen, opkill = op.accept(gen_and_kill) + gen = (gen - opkill) | opgen + kill = (kill - opgen) | opkill + block_gen[block] = gen + block_kill[block] = kill + + # Set up initial state for worklist algorithm. + worklist = list(blocks) + if not backward: + worklist = worklist[::-1] # Reverse for a small performance improvement + workset = set(worklist) + before: dict[BasicBlock, set[T]] = {} + after: dict[BasicBlock, set[T]] = {} + for block in blocks: + if kind == MAYBE_ANALYSIS: + before[block] = set() + after[block] = set() + else: + assert universe is not None, "Universe must be defined for a must analysis" + before[block] = set(universe) + after[block] = set(universe) + + if backward: + pred_map = cfg.succ + succ_map = cfg.pred + else: + pred_map = cfg.pred + succ_map = cfg.succ + + # Run work list algorithm to generate in and out sets for each basic block. + while worklist: + label = worklist.pop() + workset.remove(label) + if pred_map[label]: + new_before: set[T] | None = None + for pred in pred_map[label]: + if new_before is None: + new_before = set(after[pred]) + elif kind == MAYBE_ANALYSIS: + new_before |= after[pred] + else: + new_before &= after[pred] + assert new_before is not None + else: + new_before = set(initial) + before[label] = new_before + new_after = (new_before - block_kill[label]) | block_gen[label] + if new_after != after[label]: + for succ in succ_map[label]: + if succ not in workset: + worklist.append(succ) + workset.add(succ) + after[label] = new_after + + # Run algorithm for each basic block to generate opcode-level sets. + op_before: dict[tuple[BasicBlock, int], set[T]] = {} + op_after: dict[tuple[BasicBlock, int], set[T]] = {} + for block in blocks: + label = block + cur = before[label] + ops_enum: Iterator[tuple[int, Op]] = enumerate(block.ops) + if backward: + ops_enum = reversed(list(ops_enum)) + for idx, op in ops_enum: + op_before[label, idx] = cur + opgen, opkill = op.accept(gen_and_kill) + cur = (cur - opkill) | opgen + op_after[label, idx] = cur + if backward: + op_after, op_before = op_before, op_after + + return AnalysisResult(op_before, op_after) diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py new file mode 100644 index 000000000000..88737ac208de --- /dev/null +++ b/mypyc/analysis/ircheck.py @@ -0,0 +1,433 @@ +"""Utilities for checking that internal ir is valid and consistent.""" + +from __future__ import annotations + +from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncIR +from mypyc.ir.ops import ( + Assign, + AssignMulti, + BaseAssign, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + ControlOp, + DecRef, + Extend, + FloatComparisonOp, + FloatNeg, + FloatOp, + GetAttr, + GetElementPtr, + Goto, + IncRef, + InitStatic, + Integer, + IntOp, + KeepAlive, + LoadAddress, + LoadErrorValue, + LoadGlobal, + LoadLiteral, + LoadMem, + LoadStatic, + MethodCall, + Op, + OpVisitor, + PrimitiveOp, + RaiseStandardError, + Register, + Return, + SetAttr, + SetMem, + Truncate, + TupleGet, + TupleSet, + Unborrow, + Unbox, + Unreachable, + Value, +) +from mypyc.ir.pprint import format_func +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RPrimitive, + RType, + RUnion, + bytes_rprimitive, + dict_rprimitive, + int_rprimitive, + is_float_rprimitive, + is_object_rprimitive, + list_rprimitive, + range_rprimitive, + set_rprimitive, + str_rprimitive, + tuple_rprimitive, +) + + +class FnError: + def __init__(self, source: Op | BasicBlock, desc: str) -> None: + self.source = source + self.desc = desc + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, FnError) and self.source == other.source and self.desc == other.desc + ) + + def __repr__(self) -> str: + return f"FnError(source={self.source}, desc={self.desc})" + + +def check_func_ir(fn: FuncIR) -> list[FnError]: + """Applies validations to a given function ir and returns a list of errors found.""" + errors = [] + + op_set = set() + + for block in fn.blocks: + if not block.terminated: + errors.append( + FnError(source=block.ops[-1] if block.ops else block, desc="Block not terminated") + ) + for op in block.ops[:-1]: + if isinstance(op, ControlOp): + errors.append(FnError(source=op, desc="Block has operations after control op")) + + if op in op_set: + errors.append(FnError(source=op, desc="Func has a duplicate op")) + op_set.add(op) + + errors.extend(check_op_sources_valid(fn)) + if errors: + return errors + + op_checker = OpChecker(fn) + for block in fn.blocks: + for op in block.ops: + op.accept(op_checker) + + return op_checker.errors + + +class IrCheckException(Exception): + pass + + +def assert_func_ir_valid(fn: FuncIR) -> None: + errors = check_func_ir(fn) + if errors: + raise IrCheckException( + "Internal error: Generated invalid IR: \n" + + "\n".join(format_func(fn, [(e.source, e.desc) for e in errors])) + ) + + +def check_op_sources_valid(fn: FuncIR) -> list[FnError]: + errors = [] + valid_ops: set[Op] = set() + valid_registers: set[Register] = set() + + for block in fn.blocks: + valid_ops.update(block.ops) + + for op in block.ops: + if isinstance(op, BaseAssign): + valid_registers.add(op.dest) + elif isinstance(op, LoadAddress) and isinstance(op.src, Register): + valid_registers.add(op.src) + + valid_registers.update(fn.arg_regs) + + for block in fn.blocks: + for op in block.ops: + for source in op.sources(): + if isinstance(source, Integer): + pass + elif isinstance(source, Op): + if source not in valid_ops: + errors.append( + FnError( + source=op, + desc=f"Invalid op reference to op of type {type(source).__name__}", + ) + ) + elif isinstance(source, Register): + if source not in valid_registers: + errors.append( + FnError( + source=op, desc=f"Invalid op reference to register {source.name!r}" + ) + ) + + return errors + + +disjoint_types = { + int_rprimitive.name, + bytes_rprimitive.name, + str_rprimitive.name, + dict_rprimitive.name, + list_rprimitive.name, + set_rprimitive.name, + tuple_rprimitive.name, + range_rprimitive.name, +} + + +def can_coerce_to(src: RType, dest: RType) -> bool: + """Check if src can be assigned to dest_rtype. + + Currently okay to have false positives. + """ + if isinstance(dest, RUnion): + return any(can_coerce_to(src, d) for d in dest.items) + + if isinstance(dest, RPrimitive): + if isinstance(src, RPrimitive): + # If either src or dest is a disjoint type, then they must both be. + if src.name in disjoint_types and dest.name in disjoint_types: + return src.name == dest.name + return src.size == dest.size + if isinstance(src, RInstance): + return is_object_rprimitive(dest) + if isinstance(src, RUnion): + # IR doesn't have the ability to narrow unions based on + # control flow, so cannot be a strict all() here. + return any(can_coerce_to(s, dest) for s in src.items) + return False + + return True + + +class OpChecker(OpVisitor[None]): + def __init__(self, parent_fn: FuncIR) -> None: + self.parent_fn = parent_fn + self.errors: list[FnError] = [] + + def fail(self, source: Op, desc: str) -> None: + self.errors.append(FnError(source=source, desc=desc)) + + def check_control_op_targets(self, op: ControlOp) -> None: + for target in op.targets(): + if target not in self.parent_fn.blocks: + self.fail(source=op, desc=f"Invalid control operation target: {target.label}") + + def check_type_coercion(self, op: Op, src: RType, dest: RType) -> None: + if not can_coerce_to(src, dest): + self.fail( + source=op, desc=f"Cannot coerce source type {src.name} to dest type {dest.name}" + ) + + def check_compatibility(self, op: Op, t: RType, s: RType) -> None: + if not can_coerce_to(t, s) or not can_coerce_to(s, t): + self.fail(source=op, desc=f"{t.name} and {s.name} are not compatible") + + def expect_float(self, op: Op, v: Value) -> None: + if not is_float_rprimitive(v.type): + self.fail(op, f"Float expected (actual type is {v.type})") + + def expect_non_float(self, op: Op, v: Value) -> None: + if is_float_rprimitive(v.type): + self.fail(op, "Float not expected") + + def visit_goto(self, op: Goto) -> None: + self.check_control_op_targets(op) + + def visit_branch(self, op: Branch) -> None: + self.check_control_op_targets(op) + + def visit_return(self, op: Return) -> None: + self.check_type_coercion(op, op.value.type, self.parent_fn.decl.sig.ret_type) + + def visit_unreachable(self, op: Unreachable) -> None: + # Unreachables are checked at a higher level since validation + # requires access to the entire basic block. + pass + + def visit_assign(self, op: Assign) -> None: + self.check_type_coercion(op, op.src.type, op.dest.type) + + def visit_assign_multi(self, op: AssignMulti) -> None: + for src in op.src: + assert isinstance(op.dest.type, RArray) + self.check_type_coercion(op, src.type, op.dest.type.item_type) + + def visit_load_error_value(self, op: LoadErrorValue) -> None: + # Currently it is assumed that all types have an error value. + # Once this is fixed we can validate that the rtype here actually + # has an error value. + pass + + def check_tuple_items_valid_literals(self, op: LoadLiteral, t: tuple[object, ...]) -> None: + for x in t: + if x is not None and not isinstance(x, (str, bytes, bool, int, float, complex, tuple)): + self.fail(op, f"Invalid type for item of tuple literal: {type(x)})") + if isinstance(x, tuple): + self.check_tuple_items_valid_literals(op, x) + + def check_frozenset_items_valid_literals(self, op: LoadLiteral, s: frozenset[object]) -> None: + for x in s: + if x is None or isinstance(x, (str, bytes, bool, int, float, complex)): + pass + elif isinstance(x, tuple): + self.check_tuple_items_valid_literals(op, x) + else: + self.fail(op, f"Invalid type for item of frozenset literal: {type(x)})") + + def visit_load_literal(self, op: LoadLiteral) -> None: + expected_type = None + if op.value is None: + expected_type = "builtins.object" + elif isinstance(op.value, int): + expected_type = "builtins.int" + elif isinstance(op.value, str): + expected_type = "builtins.str" + elif isinstance(op.value, bytes): + expected_type = "builtins.bytes" + elif isinstance(op.value, bool): + expected_type = "builtins.object" + elif isinstance(op.value, float): + expected_type = "builtins.float" + elif isinstance(op.value, complex): + expected_type = "builtins.object" + elif isinstance(op.value, tuple): + expected_type = "builtins.tuple" + self.check_tuple_items_valid_literals(op, op.value) + elif isinstance(op.value, frozenset): + # There's no frozenset_rprimitive type since it'd be pretty useless so we just pretend + # it's a set (when it's really a frozenset). + expected_type = "builtins.set" + self.check_frozenset_items_valid_literals(op, op.value) + + assert expected_type is not None, "Missed a case for LoadLiteral check" + + if op.type.name not in [expected_type, "builtins.object"]: + self.fail( + op, + f"Invalid literal value for type: value has " + f"type {expected_type}, but op has type {op.type.name}", + ) + + def visit_get_attr(self, op: GetAttr) -> None: + # Nothing to do. + pass + + def visit_set_attr(self, op: SetAttr) -> None: + # Nothing to do. + pass + + # Static operations cannot be checked at the function level. + def visit_load_static(self, op: LoadStatic) -> None: + pass + + def visit_init_static(self, op: InitStatic) -> None: + pass + + def visit_tuple_get(self, op: TupleGet) -> None: + # Nothing to do. + pass + + def visit_tuple_set(self, op: TupleSet) -> None: + # Nothing to do. + pass + + def visit_inc_ref(self, op: IncRef) -> None: + # Nothing to do. + pass + + def visit_dec_ref(self, op: DecRef) -> None: + # Nothing to do. + pass + + def visit_call(self, op: Call) -> None: + # Length is checked in constructor, and return type is set + # in a way that can't be incorrect + for arg_value, arg_runtime in zip(op.args, op.fn.sig.args): + self.check_type_coercion(op, arg_value.type, arg_runtime.type) + + def visit_method_call(self, op: MethodCall) -> None: + # Similar to above, but we must look up method first. + method_decl = op.receiver_type.class_ir.method_decl(op.method) + if method_decl.kind == FUNC_STATICMETHOD: + decl_index = 0 + else: + decl_index = 1 + + if len(op.args) + decl_index != len(method_decl.sig.args): + self.fail(op, "Incorrect number of args for method call.") + + # Skip the receiver argument (self) + for arg_value, arg_runtime in zip(op.args, method_decl.sig.args[decl_index:]): + self.check_type_coercion(op, arg_value.type, arg_runtime.type) + + def visit_cast(self, op: Cast) -> None: + pass + + def visit_box(self, op: Box) -> None: + pass + + def visit_unbox(self, op: Unbox) -> None: + pass + + def visit_raise_standard_error(self, op: RaiseStandardError) -> None: + pass + + def visit_call_c(self, op: CallC) -> None: + pass + + def visit_primitive_op(self, op: PrimitiveOp) -> None: + pass + + def visit_truncate(self, op: Truncate) -> None: + pass + + def visit_extend(self, op: Extend) -> None: + pass + + def visit_load_global(self, op: LoadGlobal) -> None: + pass + + def visit_int_op(self, op: IntOp) -> None: + self.expect_non_float(op, op.lhs) + self.expect_non_float(op, op.rhs) + + def visit_comparison_op(self, op: ComparisonOp) -> None: + self.check_compatibility(op, op.lhs.type, op.rhs.type) + self.expect_non_float(op, op.lhs) + self.expect_non_float(op, op.rhs) + + def visit_float_op(self, op: FloatOp) -> None: + self.expect_float(op, op.lhs) + self.expect_float(op, op.rhs) + + def visit_float_neg(self, op: FloatNeg) -> None: + self.expect_float(op, op.src) + + def visit_float_comparison_op(self, op: FloatComparisonOp) -> None: + self.expect_float(op, op.lhs) + self.expect_float(op, op.rhs) + + def visit_load_mem(self, op: LoadMem) -> None: + pass + + def visit_set_mem(self, op: SetMem) -> None: + pass + + def visit_get_element_ptr(self, op: GetElementPtr) -> None: + pass + + def visit_load_address(self, op: LoadAddress) -> None: + pass + + def visit_keep_alive(self, op: KeepAlive) -> None: + pass + + def visit_unborrow(self, op: Unborrow) -> None: + pass diff --git a/mypyc/analysis/selfleaks.py b/mypyc/analysis/selfleaks.py new file mode 100644 index 000000000000..5d89a9bfc7c6 --- /dev/null +++ b/mypyc/analysis/selfleaks.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +from typing import Set, Tuple + +from mypyc.analysis.dataflow import CFG, MAYBE_ANALYSIS, AnalysisResult, run_analysis +from mypyc.ir.ops import ( + Assign, + AssignMulti, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + Extend, + FloatComparisonOp, + FloatNeg, + FloatOp, + GetAttr, + GetElementPtr, + Goto, + InitStatic, + IntOp, + KeepAlive, + LoadAddress, + LoadErrorValue, + LoadGlobal, + LoadLiteral, + LoadMem, + LoadStatic, + MethodCall, + OpVisitor, + PrimitiveOp, + RaiseStandardError, + Register, + RegisterOp, + Return, + SetAttr, + SetMem, + Truncate, + TupleGet, + TupleSet, + Unborrow, + Unbox, + Unreachable, +) +from mypyc.ir.rtypes import RInstance + +GenAndKill = Tuple[Set[None], Set[None]] + +CLEAN: GenAndKill = (set(), set()) +DIRTY: GenAndKill = ({None}, {None}) + + +class SelfLeakedVisitor(OpVisitor[GenAndKill]): + """Analyze whether 'self' may be seen by arbitrary code in '__init__'. + + More formally, the set is not empty if along some path from IR entry point + arbitrary code could have been executed that has access to 'self'. + + (We don't consider access via 'gc.get_objects()'.) + """ + + def __init__(self, self_reg: Register) -> None: + self.self_reg = self_reg + + def visit_goto(self, op: Goto) -> GenAndKill: + return CLEAN + + def visit_branch(self, op: Branch) -> GenAndKill: + return CLEAN + + def visit_return(self, op: Return) -> GenAndKill: + # Consider all exits from the function 'dirty' since they implicitly + # cause 'self' to be returned. + return DIRTY + + def visit_unreachable(self, op: Unreachable) -> GenAndKill: + return CLEAN + + def visit_assign(self, op: Assign) -> GenAndKill: + if op.src is self.self_reg or op.dest is self.self_reg: + return DIRTY + return CLEAN + + def visit_assign_multi(self, op: AssignMulti) -> GenAndKill: + return CLEAN + + def visit_set_mem(self, op: SetMem) -> GenAndKill: + return CLEAN + + def visit_call(self, op: Call) -> GenAndKill: + fn = op.fn + if fn.class_name and fn.name == "__init__": + self_type = op.fn.sig.args[0].type + assert isinstance(self_type, RInstance) + cl = self_type.class_ir + if not cl.init_self_leak: + return CLEAN + return self.check_register_op(op) + + def visit_method_call(self, op: MethodCall) -> GenAndKill: + return self.check_register_op(op) + + def visit_load_error_value(self, op: LoadErrorValue) -> GenAndKill: + return CLEAN + + def visit_load_literal(self, op: LoadLiteral) -> GenAndKill: + return CLEAN + + def visit_get_attr(self, op: GetAttr) -> GenAndKill: + cl = op.class_type.class_ir + if cl.get_method(op.attr): + # Property -- calls a function + return self.check_register_op(op) + return CLEAN + + def visit_set_attr(self, op: SetAttr) -> GenAndKill: + cl = op.class_type.class_ir + if cl.get_method(op.attr): + # Property - calls a function + return self.check_register_op(op) + return CLEAN + + def visit_load_static(self, op: LoadStatic) -> GenAndKill: + return CLEAN + + def visit_init_static(self, op: InitStatic) -> GenAndKill: + return self.check_register_op(op) + + def visit_tuple_get(self, op: TupleGet) -> GenAndKill: + return CLEAN + + def visit_tuple_set(self, op: TupleSet) -> GenAndKill: + return self.check_register_op(op) + + def visit_box(self, op: Box) -> GenAndKill: + return self.check_register_op(op) + + def visit_unbox(self, op: Unbox) -> GenAndKill: + return self.check_register_op(op) + + def visit_cast(self, op: Cast) -> GenAndKill: + return self.check_register_op(op) + + def visit_raise_standard_error(self, op: RaiseStandardError) -> GenAndKill: + return CLEAN + + def visit_call_c(self, op: CallC) -> GenAndKill: + return self.check_register_op(op) + + def visit_primitive_op(self, op: PrimitiveOp) -> GenAndKill: + return self.check_register_op(op) + + def visit_truncate(self, op: Truncate) -> GenAndKill: + return CLEAN + + def visit_extend(self, op: Extend) -> GenAndKill: + return CLEAN + + def visit_load_global(self, op: LoadGlobal) -> GenAndKill: + return CLEAN + + def visit_int_op(self, op: IntOp) -> GenAndKill: + return CLEAN + + def visit_comparison_op(self, op: ComparisonOp) -> GenAndKill: + return CLEAN + + def visit_float_op(self, op: FloatOp) -> GenAndKill: + return CLEAN + + def visit_float_neg(self, op: FloatNeg) -> GenAndKill: + return CLEAN + + def visit_float_comparison_op(self, op: FloatComparisonOp) -> GenAndKill: + return CLEAN + + def visit_load_mem(self, op: LoadMem) -> GenAndKill: + return CLEAN + + def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill: + return CLEAN + + def visit_load_address(self, op: LoadAddress) -> GenAndKill: + return CLEAN + + def visit_keep_alive(self, op: KeepAlive) -> GenAndKill: + return CLEAN + + def visit_unborrow(self, op: Unborrow) -> GenAndKill: + return CLEAN + + def check_register_op(self, op: RegisterOp) -> GenAndKill: + if any(src is self.self_reg for src in op.sources()): + return DIRTY + return CLEAN + + +def analyze_self_leaks( + blocks: list[BasicBlock], self_reg: Register, cfg: CFG +) -> AnalysisResult[None]: + return run_analysis( + blocks=blocks, + cfg=cfg, + gen_and_kill=SelfLeakedVisitor(self_reg), + initial=set(), + backward=False, + kind=MAYBE_ANALYSIS, + ) diff --git a/mypyc/build.py b/mypyc/build.py new file mode 100644 index 000000000000..6d59113ef872 --- /dev/null +++ b/mypyc/build.py @@ -0,0 +1,621 @@ +"""Support for building extensions using mypyc with distutils or setuptools + +The main entry point is mypycify, which produces a list of extension +modules to be passed to setup. A trivial setup.py for a mypyc built +project, then, looks like: + + from setuptools import setup + from mypyc.build import mypycify + + setup(name='test_module', + ext_modules=mypycify(['foo.py']), + ) + +See the mypycify docs for additional arguments. + +mypycify can integrate with either distutils or setuptools, but needs +to know at import-time whether it is using distutils or setuputils. We +hackily decide based on whether setuptools has been imported already. +""" + +from __future__ import annotations + +import hashlib +import os.path +import re +import sys +import time +from typing import TYPE_CHECKING, Any, Dict, Iterable, NoReturn, Union, cast + +from mypy.build import BuildSource +from mypy.errors import CompileError +from mypy.fscache import FileSystemCache +from mypy.main import process_options +from mypy.options import Options +from mypy.util import write_junit_xml +from mypyc.codegen import emitmodule +from mypyc.common import RUNTIME_C_FILES, shared_lib_name +from mypyc.errors import Errors +from mypyc.ir.pprint import format_modules +from mypyc.namegen import exported_name +from mypyc.options import CompilerOptions + +try: + # Import setuptools so that it monkey-patch overrides distutils + import setuptools +except ImportError: + pass + +if TYPE_CHECKING: + if sys.version_info >= (3, 12): + from setuptools import Extension + else: + from distutils.core import Extension as _distutils_Extension + from typing_extensions import TypeAlias + + from setuptools import Extension as _setuptools_Extension + + Extension: TypeAlias = Union[_setuptools_Extension, _distutils_Extension] + +if sys.version_info >= (3, 12): + # From setuptools' monkeypatch + from distutils import ccompiler, sysconfig # type: ignore[import-not-found] +else: + from distutils import ccompiler, sysconfig + + +def get_extension() -> type[Extension]: + # We can work with either setuptools or distutils, and pick setuptools + # if it has been imported. + use_setuptools = "setuptools" in sys.modules + extension_class: type[Extension] + + if sys.version_info < (3, 12) and not use_setuptools: + import distutils.core + + extension_class = distutils.core.Extension + else: + if not use_setuptools: + sys.exit("error: setuptools not installed") + extension_class = setuptools.Extension + + return extension_class + + +def setup_mypycify_vars() -> None: + """Rewrite a bunch of config vars in pretty dubious ways.""" + # There has to be a better approach to this. + + # The vars can contain ints but we only work with str ones + vars = cast(Dict[str, str], sysconfig.get_config_vars()) + if sys.platform == "darwin": + # Disable building 32-bit binaries, since we generate too much code + # for a 32-bit Mach-O object. There has to be a better way to do this. + vars["LDSHARED"] = vars["LDSHARED"].replace("-arch i386", "") + vars["LDFLAGS"] = vars["LDFLAGS"].replace("-arch i386", "") + vars["CFLAGS"] = vars["CFLAGS"].replace("-arch i386", "") + + +def fail(message: str) -> NoReturn: + # TODO: Is there something else we should do to fail? + sys.exit(message) + + +def emit_messages(options: Options, messages: list[str], dt: float, serious: bool = False) -> None: + # ... you know, just in case. + if options.junit_xml: + py_version = f"{options.python_version[0]}_{options.python_version[1]}" + write_junit_xml( + dt, + serious, + {None: messages} if messages else {}, + options.junit_xml, + py_version, + options.platform, + ) + if messages: + print("\n".join(messages)) + + +def get_mypy_config( + mypy_options: list[str], + only_compile_paths: Iterable[str] | None, + compiler_options: CompilerOptions, + fscache: FileSystemCache | None, +) -> tuple[list[BuildSource], list[BuildSource], Options]: + """Construct mypy BuildSources and Options from file and options lists""" + all_sources, options = process_options(mypy_options, fscache=fscache) + if only_compile_paths is not None: + paths_set = set(only_compile_paths) + mypyc_sources = [s for s in all_sources if s.path in paths_set] + else: + mypyc_sources = all_sources + + if compiler_options.separate: + mypyc_sources = [ + src for src in mypyc_sources if src.path and not src.path.endswith("__init__.py") + ] + + if not mypyc_sources: + return mypyc_sources, all_sources, options + + # Override whatever python_version is inferred from the .ini file, + # and set the python_version to be the currently used version. + options.python_version = sys.version_info[:2] + + if options.python_version[0] == 2: + fail("Python 2 not supported") + if not options.strict_optional: + fail("Disabling strict optional checking not supported") + options.show_traceback = True + # Needed to get types for all AST nodes + options.export_types = True + # We use mypy incremental mode when doing separate/incremental mypyc compilation + options.incremental = compiler_options.separate + options.preserve_asts = True + + for source in mypyc_sources: + options.per_module_options.setdefault(source.module, {})["mypyc"] = True + + return mypyc_sources, all_sources, options + + +def generate_c_extension_shim( + full_module_name: str, module_name: str, dir_name: str, group_name: str +) -> str: + """Create a C extension shim with a passthrough PyInit function. + + Arguments: + full_module_name: the dotted full module name + module_name: the final component of the module name + dir_name: the directory to place source code + group_name: the name of the group + """ + cname = "%s.c" % full_module_name.replace(".", os.sep) + cpath = os.path.join(dir_name, cname) + + # We load the C extension shim template from a file. + # (So that the file could be reused as a bazel template also.) + with open(os.path.join(include_dir(), "module_shim.tmpl")) as f: + shim_template = f.read() + + write_file( + cpath, + shim_template.format( + modname=module_name, + libname=shared_lib_name(group_name), + full_modname=exported_name(full_module_name), + ), + ) + + return cpath + + +def group_name(modules: list[str]) -> str: + """Produce a probably unique name for a group from a list of module names.""" + if len(modules) == 1: + return modules[0] + + h = hashlib.sha1() + h.update(",".join(modules).encode()) + return h.hexdigest()[:20] + + +def include_dir() -> str: + """Find the path of the lib-rt dir that needs to be included""" + return os.path.join(os.path.abspath(os.path.dirname(__file__)), "lib-rt") + + +def generate_c( + sources: list[BuildSource], + options: Options, + groups: emitmodule.Groups, + fscache: FileSystemCache, + compiler_options: CompilerOptions, +) -> tuple[list[list[tuple[str, str]]], str]: + """Drive the actual core compilation step. + + The groups argument describes how modules are assigned to C + extension modules. See the comments on the Groups type in + mypyc.emitmodule for details. + + Returns the C source code and (for debugging) the pretty printed IR. + """ + t0 = time.time() + + try: + result = emitmodule.parse_and_typecheck( + sources, options, compiler_options, groups, fscache + ) + except CompileError as e: + emit_messages(options, e.messages, time.time() - t0, serious=(not e.use_stdout)) + sys.exit(1) + + t1 = time.time() + if result.errors: + emit_messages(options, result.errors, t1 - t0) + sys.exit(1) + + if compiler_options.verbose: + print(f"Parsed and typechecked in {t1 - t0:.3f}s") + + errors = Errors(options) + modules, ctext = emitmodule.compile_modules_to_c( + result, compiler_options=compiler_options, errors=errors, groups=groups + ) + t2 = time.time() + emit_messages(options, errors.new_messages(), t2 - t1) + if errors.num_errors: + # No need to stop the build if only warnings were emitted. + sys.exit(1) + + if compiler_options.verbose: + print(f"Compiled to C in {t2 - t1:.3f}s") + + return ctext, "\n".join(format_modules(modules)) + + +def build_using_shared_lib( + sources: list[BuildSource], + group_name: str, + cfiles: list[str], + deps: list[str], + build_dir: str, + extra_compile_args: list[str], +) -> list[Extension]: + """Produce the list of extension modules when a shared library is needed. + + This creates one shared library extension module that all of the + others import and then one shim extension module for each + module in the build, that simply calls an initialization function + in the shared library. + + The shared library (which lib_name is the name of) is a python + extension module that exports the real initialization functions in + Capsules stored in module attributes. + """ + extensions = [ + get_extension()( + shared_lib_name(group_name), + sources=cfiles, + include_dirs=[include_dir(), build_dir], + depends=deps, + extra_compile_args=extra_compile_args, + ) + ] + + for source in sources: + module_name = source.module.split(".")[-1] + shim_file = generate_c_extension_shim(source.module, module_name, build_dir, group_name) + + # We include the __init__ in the "module name" we stick in the Extension, + # since this seems to be needed for it to end up in the right place. + full_module_name = source.module + assert source.path + if os.path.split(source.path)[1] == "__init__.py": + full_module_name += ".__init__" + extensions.append( + get_extension()( + full_module_name, sources=[shim_file], extra_compile_args=extra_compile_args + ) + ) + + return extensions + + +def build_single_module( + sources: list[BuildSource], cfiles: list[str], extra_compile_args: list[str] +) -> list[Extension]: + """Produce the list of extension modules for a standalone extension. + + This contains just one module, since there is no need for a shared module. + """ + return [ + get_extension()( + sources[0].module, + sources=cfiles, + include_dirs=[include_dir()], + extra_compile_args=extra_compile_args, + ) + ] + + +def write_file(path: str, contents: str) -> None: + """Write data into a file. + + If the file already exists and has the same contents we + want to write, skip writing so as to preserve the mtime + and avoid triggering recompilation. + """ + # We encode it ourselves and open the files as binary to avoid windows + # newline translation + encoded_contents = contents.encode("utf-8") + try: + with open(path, "rb") as f: + old_contents: bytes | None = f.read() + except OSError: + old_contents = None + if old_contents != encoded_contents: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as g: + g.write(encoded_contents) + + # Fudge the mtime forward because otherwise when two builds happen close + # together (like in a test) setuptools might not realize the source is newer + # than the new artifact. + # XXX: This is bad though. + new_mtime = os.stat(path).st_mtime + 1 + os.utime(path, times=(new_mtime, new_mtime)) + + +def construct_groups( + sources: list[BuildSource], + separate: bool | list[tuple[list[str], str | None]], + use_shared_lib: bool, +) -> emitmodule.Groups: + """Compute Groups given the input source list and separate configs. + + separate is the user-specified configuration for how to assign + modules to compilation groups (see mypycify docstring for details). + + This takes that and expands it into our internal representation of + group configuration, documented in mypyc.emitmodule's definition + of Group. + """ + + if separate is True: + groups: emitmodule.Groups = [([source], None) for source in sources] + elif isinstance(separate, list): + groups = [] + used_sources = set() + for files, name in separate: + group_sources = [src for src in sources if src.path in files] + groups.append((group_sources, name)) + used_sources.update(group_sources) + unused_sources = [src for src in sources if src not in used_sources] + if unused_sources: + groups.extend([([source], None) for source in unused_sources]) + else: + groups = [(sources, None)] + + # Generate missing names + for i, (group, name) in enumerate(groups): + if use_shared_lib and not name: + name = group_name([source.module for source in group]) + groups[i] = (group, name) + + return groups + + +def get_header_deps(cfiles: list[tuple[str, str]]) -> list[str]: + """Find all the headers used by a group of cfiles. + + We do this by just regexping the source, which is a bit simpler than + properly plumbing the data through. + + Arguments: + cfiles: A list of (file name, file contents) pairs. + """ + headers: set[str] = set() + for _, contents in cfiles: + headers.update(re.findall(r'#include "(.*)"', contents)) + + return sorted(headers) + + +def mypyc_build( + paths: list[str], + compiler_options: CompilerOptions, + *, + separate: bool | list[tuple[list[str], str | None]] = False, + only_compile_paths: Iterable[str] | None = None, + skip_cgen_input: Any | None = None, + always_use_shared_lib: bool = False, +) -> tuple[emitmodule.Groups, list[tuple[list[str], list[str]]]]: + """Do the front and middle end of mypyc building, producing and writing out C source.""" + fscache = FileSystemCache() + mypyc_sources, all_sources, options = get_mypy_config( + paths, only_compile_paths, compiler_options, fscache + ) + + # We generate a shared lib if there are multiple modules or if any + # of the modules are in package. (Because I didn't want to fuss + # around with making the single module code handle packages.) + use_shared_lib = ( + len(mypyc_sources) > 1 + or any("." in x.module for x in mypyc_sources) + or always_use_shared_lib + ) + + groups = construct_groups(mypyc_sources, separate, use_shared_lib) + + # We let the test harness just pass in the c file contents instead + # so that it can do a corner-cutting version without full stubs. + if not skip_cgen_input: + group_cfiles, ops_text = generate_c( + all_sources, options, groups, fscache, compiler_options=compiler_options + ) + # TODO: unique names? + write_file(os.path.join(compiler_options.target_dir, "ops.txt"), ops_text) + else: + group_cfiles = skip_cgen_input + + # Write out the generated C and collect the files for each group + # Should this be here?? + group_cfilenames: list[tuple[list[str], list[str]]] = [] + for cfiles in group_cfiles: + cfilenames = [] + for cfile, ctext in cfiles: + cfile = os.path.join(compiler_options.target_dir, cfile) + write_file(cfile, ctext) + if os.path.splitext(cfile)[1] == ".c": + cfilenames.append(cfile) + + deps = [os.path.join(compiler_options.target_dir, dep) for dep in get_header_deps(cfiles)] + group_cfilenames.append((cfilenames, deps)) + + return groups, group_cfilenames + + +def mypycify( + paths: list[str], + *, + only_compile_paths: Iterable[str] | None = None, + verbose: bool = False, + opt_level: str = "3", + debug_level: str = "1", + strip_asserts: bool = False, + multi_file: bool = False, + separate: bool | list[tuple[list[str], str | None]] = False, + skip_cgen_input: Any | None = None, + target_dir: str | None = None, + include_runtime_files: bool | None = None, + strict_dunder_typing: bool = False, +) -> list[Extension]: + """Main entry point to building using mypyc. + + This produces a list of Extension objects that should be passed as the + ext_modules parameter to setup. + + Arguments: + paths: A list of file paths to build. It may also contain mypy options. + only_compile_paths: If not None, an iterable of paths that are to be + the only modules compiled, even if other modules + appear in the mypy command line given to paths. + (These modules must still be passed to paths.) + + verbose: Should mypyc be more verbose. Defaults to false. + + opt_level: The optimization level, as a string. Defaults to '3' (meaning '-O3'). + debug_level: The debug level, as a string. Defaults to '1' (meaning '-g1'). + strip_asserts: Should asserts be stripped from the generated code. + + multi_file: Should each Python module be compiled into its own C source file. + This can reduce compile time and memory requirements at the likely + cost of runtime performance of compiled code. Defaults to false. + separate: Should compiled modules be placed in separate extension modules. + If False, all modules are placed in a single shared library. + If True, every module is placed in its own library. + Otherwise separate should be a list of + (file name list, optional shared library name) pairs specifying + groups of files that should be placed in the same shared library + (while all other modules will be placed in its own library). + + Each group can be compiled independently, which can + speed up compilation, but calls between groups can + be slower than calls within a group and can't be + inlined. + target_dir: The directory to write C output files. Defaults to 'build'. + include_runtime_files: If not None, whether the mypyc runtime library + should be directly #include'd instead of linked + separately in order to reduce compiler invocations. + Defaults to False in multi_file mode, True otherwise. + strict_dunder_typing: If True, force dunder methods to have the return type + of the method strictly, which can lead to more + optimization opportunities. Defaults to False. + """ + + # Figure out our configuration + compiler_options = CompilerOptions( + strip_asserts=strip_asserts, + multi_file=multi_file, + verbose=verbose, + separate=separate is not False, + target_dir=target_dir, + include_runtime_files=include_runtime_files, + strict_dunder_typing=strict_dunder_typing, + ) + + # Generate all the actual important C code + groups, group_cfilenames = mypyc_build( + paths, + only_compile_paths=only_compile_paths, + compiler_options=compiler_options, + separate=separate, + skip_cgen_input=skip_cgen_input, + ) + + # Mess around with setuptools and actually get the thing built + setup_mypycify_vars() + + # Create a compiler object so we can make decisions based on what + # compiler is being used. typeshed is missing some attributes on the + # compiler object so we give it type Any + compiler: Any = ccompiler.new_compiler() + sysconfig.customize_compiler(compiler) + + build_dir = compiler_options.target_dir + + cflags: list[str] = [] + if compiler.compiler_type == "unix": + cflags += [ + f"-O{opt_level}", + f"-g{debug_level}", + "-Werror", + "-Wno-unused-function", + "-Wno-unused-label", + "-Wno-unreachable-code", + "-Wno-unused-variable", + "-Wno-unused-command-line-argument", + "-Wno-unknown-warning-option", + "-Wno-unused-but-set-variable", + "-Wno-ignored-optimization-argument", + # Disables C Preprocessor (cpp) warnings + # See https://github.com/mypyc/mypyc/issues/956 + "-Wno-cpp", + ] + elif compiler.compiler_type == "msvc": + # msvc doesn't have levels, '/O2' is full and '/Od' is disable + if opt_level == "0": + opt_level = "d" + elif opt_level in ("1", "2", "3"): + opt_level = "2" + if debug_level == "0": + debug_level = "NONE" + elif debug_level == "1": + debug_level = "FASTLINK" + elif debug_level in ("2", "3"): + debug_level = "FULL" + cflags += [ + f"/O{opt_level}", + f"/DEBUG:{debug_level}", + "/wd4102", # unreferenced label + "/wd4101", # unreferenced local variable + "/wd4146", # negating unsigned int + ] + if multi_file: + # Disable whole program optimization in multi-file mode so + # that we actually get the compilation speed and memory + # use wins that multi-file mode is intended for. + cflags += ["/GL-", "/wd9025"] # warning about overriding /GL + + # If configured to (defaults to yes in multi-file mode), copy the + # runtime library in. Otherwise it just gets #included to save on + # compiler invocations. + shared_cfilenames = [] + if not compiler_options.include_runtime_files: + for name in RUNTIME_C_FILES: + rt_file = os.path.join(build_dir, name) + with open(os.path.join(include_dir(), name), encoding="utf-8") as f: + write_file(rt_file, f.read()) + shared_cfilenames.append(rt_file) + + extensions = [] + for (group_sources, lib_name), (cfilenames, deps) in zip(groups, group_cfilenames): + if lib_name: + extensions.extend( + build_using_shared_lib( + group_sources, + lib_name, + cfilenames + shared_cfilenames, + deps, + build_dir, + cflags, + ) + ) + else: + extensions.extend( + build_single_module(group_sources, cfilenames + shared_cfilenames, cflags) + ) + + return extensions diff --git a/mypyc/codegen/__init__.py b/mypyc/codegen/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/codegen/cstring.py b/mypyc/codegen/cstring.py new file mode 100644 index 000000000000..853787f8161d --- /dev/null +++ b/mypyc/codegen/cstring.py @@ -0,0 +1,54 @@ +"""Encode valid C string literals from Python strings. + +If a character is not allowed in C string literals, it is either emitted +as a simple escape sequence (e.g. '\\n'), or an octal escape sequence +with exactly three digits ('\\oXXX'). Question marks are escaped to +prevent trigraphs in the string literal from being interpreted. Note +that '\\?' is an invalid escape sequence in Python. + +Consider the string literal "AB\\xCDEF". As one would expect, Python +parses it as ['A', 'B', 0xCD, 'E', 'F']. However, the C standard +specifies that all hexadecimal digits immediately following '\\x' will +be interpreted as part of the escape sequence. Therefore, it is +unexpectedly parsed as ['A', 'B', 0xCDEF]. + +Emitting ("AB\\xCD" "EF") would avoid this behaviour. However, we opt +for simplicity and use octal escape sequences instead. They do not +suffer from the same issue as they are defined to parse at most three +octal digits. +""" + +from __future__ import annotations + +import string +from typing import Final + +CHAR_MAP: Final = [f"\\{i:03o}" for i in range(256)] + +# It is safe to use string.printable as it always uses the C locale. +for c in string.printable: + CHAR_MAP[ord(c)] = c + +# These assignments must come last because we prioritize simple escape +# sequences over any other representation. +for c in ("'", '"', "\\", "a", "b", "f", "n", "r", "t", "v"): + escaped = f"\\{c}" + decoded = escaped.encode("ascii").decode("unicode_escape") + CHAR_MAP[ord(decoded)] = escaped + +# This escape sequence is invalid in Python. +CHAR_MAP[ord("?")] = r"\?" + + +def encode_bytes_as_c_string(b: bytes) -> str: + """Produce contents of a C string literal for a byte string, without quotes.""" + escaped = "".join([CHAR_MAP[i] for i in b]) + return escaped + + +def c_string_initializer(value: bytes) -> str: + """Create initializer for a C char[]/ char * variable from a string. + + For example, if value if b'foo', the result would be '"foo"'. + """ + return '"' + encode_bytes_as_c_string(value) + '"' diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py new file mode 100644 index 000000000000..fce6896e8d11 --- /dev/null +++ b/mypyc/codegen/emit.py @@ -0,0 +1,1193 @@ +"""Utilities for emitting C code.""" + +from __future__ import annotations + +import pprint +import sys +import textwrap +from typing import Callable, Final + +from mypyc.codegen.literals import Literals +from mypyc.common import ( + ATTR_PREFIX, + BITMAP_BITS, + FAST_ISINSTANCE_MAX_SUBCLASSES, + NATIVE_PREFIX, + REG_PREFIX, + STATIC_PREFIX, + TYPE_PREFIX, + use_vectorcall, +) +from mypyc.ir.class_ir import ClassIR, all_concrete_classes +from mypyc.ir.func_ir import FuncDecl +from mypyc.ir.ops import BasicBlock, Value +from mypyc.ir.rtypes import ( + RInstance, + RPrimitive, + RTuple, + RType, + RUnion, + int_rprimitive, + is_bit_rprimitive, + is_bool_rprimitive, + is_bytes_rprimitive, + is_dict_rprimitive, + is_fixed_width_rtype, + is_float_rprimitive, + is_int16_rprimitive, + is_int32_rprimitive, + is_int64_rprimitive, + is_int_rprimitive, + is_list_rprimitive, + is_none_rprimitive, + is_object_rprimitive, + is_optional_type, + is_range_rprimitive, + is_set_rprimitive, + is_short_int_rprimitive, + is_str_rprimitive, + is_tuple_rprimitive, + is_uint8_rprimitive, + object_rprimitive, + optional_value_type, +) +from mypyc.namegen import NameGenerator, exported_name +from mypyc.sametype import is_same_type + +# Whether to insert debug asserts for all error handling, to quickly +# catch errors propagating without exceptions set. +DEBUG_ERRORS: Final = False + + +class HeaderDeclaration: + """A representation of a declaration in C. + + This is used to generate declarations in header files and + (optionally) definitions in source files. + + Attributes: + decl: C source code for the declaration. + defn: Optionally, C source code for a definition. + dependencies: The names of any objects that must be declared prior. + is_type: Whether the declaration is of a C type. (C types will be declared in + external header files and not marked 'extern'.) + needs_export: Whether the declared object needs to be exported to + other modules in the linking table. + """ + + def __init__( + self, + decl: str | list[str], + defn: list[str] | None = None, + *, + dependencies: set[str] | None = None, + is_type: bool = False, + needs_export: bool = False, + ) -> None: + self.decl = [decl] if isinstance(decl, str) else decl + self.defn = defn + self.dependencies = dependencies or set() + self.is_type = is_type + self.needs_export = needs_export + + +class EmitterContext: + """Shared emitter state for a compilation group.""" + + def __init__( + self, + names: NameGenerator, + group_name: str | None = None, + group_map: dict[str, str | None] | None = None, + ) -> None: + """Setup shared emitter state. + + Args: + names: The name generator to use + group_map: Map from module names to group name + group_name: Current group name + """ + self.temp_counter = 0 + self.names = names + self.group_name = group_name + self.group_map = group_map or {} + # Groups that this group depends on + self.group_deps: set[str] = set() + + # The map below is used for generating declarations and + # definitions at the top of the C file. The main idea is that they can + # be generated at any time during the emit phase. + + # A map of a C identifier to whatever the C identifier declares. Currently this is + # used for declaring structs and the key corresponds to the name of the struct. + # The declaration contains the body of the struct. + self.declarations: dict[str, HeaderDeclaration] = {} + + self.literals = Literals() + + +class ErrorHandler: + """Describes handling errors in unbox/cast operations.""" + + +class AssignHandler(ErrorHandler): + """Assign an error value on error.""" + + +class GotoHandler(ErrorHandler): + """Goto label on error.""" + + def __init__(self, label: str) -> None: + self.label = label + + +class TracebackAndGotoHandler(ErrorHandler): + """Add traceback item and goto label on error.""" + + def __init__( + self, label: str, source_path: str, module_name: str, traceback_entry: tuple[str, int] + ) -> None: + self.label = label + self.source_path = source_path + self.module_name = module_name + self.traceback_entry = traceback_entry + + +class ReturnHandler(ErrorHandler): + """Return a constant value on error.""" + + def __init__(self, value: str) -> None: + self.value = value + + +class Emitter: + """Helper for C code generation.""" + + def __init__( + self, + context: EmitterContext, + value_names: dict[Value, str] | None = None, + capi_version: tuple[int, int] | None = None, + ) -> None: + self.context = context + self.capi_version = capi_version or sys.version_info[:2] + self.names = context.names + self.value_names = value_names or {} + self.fragments: list[str] = [] + self._indent = 0 + + # Low-level operations + + def indent(self) -> None: + self._indent += 4 + + def dedent(self) -> None: + self._indent -= 4 + assert self._indent >= 0 + + def label(self, label: BasicBlock) -> str: + return "CPyL%s" % label.label + + def reg(self, reg: Value) -> str: + return REG_PREFIX + self.value_names[reg] + + def attr(self, name: str) -> str: + return ATTR_PREFIX + name + + def object_annotation(self, obj: object, line: str) -> str: + """Build a C comment with an object's string represention. + + If the comment exceeds the line length limit, it's wrapped into a + multiline string (with the extra lines indented to be aligned with + the first line's comment). + + If it contains illegal characters, an empty string is returned.""" + line_width = self._indent + len(line) + formatted = pprint.pformat(obj, compact=True, width=max(90 - line_width, 20)) + if any(x in formatted for x in ("/*", "*/", "\0")): + return "" + + if "\n" in formatted: + first_line, rest = formatted.split("\n", maxsplit=1) + comment_continued = textwrap.indent(rest, (line_width + 3) * " ") + return f" /* {first_line}\n{comment_continued} */" + else: + return f" /* {formatted} */" + + def emit_line(self, line: str = "", *, ann: object = None) -> None: + if line.startswith("}"): + self.dedent() + comment = self.object_annotation(ann, line) if ann is not None else "" + self.fragments.append(self._indent * " " + line + comment + "\n") + if line.endswith("{"): + self.indent() + + def emit_lines(self, *lines: str) -> None: + for line in lines: + self.emit_line(line) + + def emit_label(self, label: BasicBlock | str) -> None: + if isinstance(label, str): + text = label + else: + if label.label == 0 or not label.referenced: + return + + text = self.label(label) + # Extra semicolon prevents an error when the next line declares a tempvar + self.fragments.append(f"{text}: ;\n") + + def emit_from_emitter(self, emitter: Emitter) -> None: + self.fragments.extend(emitter.fragments) + + def emit_printf(self, fmt: str, *args: str) -> None: + fmt = fmt.replace("\n", "\\n") + self.emit_line("printf(%s);" % ", ".join(['"%s"' % fmt] + list(args))) + self.emit_line("fflush(stdout);") + + def temp_name(self) -> str: + self.context.temp_counter += 1 + return "__tmp%d" % self.context.temp_counter + + def new_label(self) -> str: + self.context.temp_counter += 1 + return "__LL%d" % self.context.temp_counter + + def get_module_group_prefix(self, module_name: str) -> str: + """Get the group prefix for a module (relative to the current group). + + The prefix should be prepended to the object name whenever + accessing an object from this module. + + If the module lives is in the current compilation group, there is + no prefix. But if it lives in a different group (and hence a separate + extension module), we need to access objects from it indirectly via an + export table. + + For example, for code in group `a` to call a function `bar` in group `b`, + it would need to do `exports_b.CPyDef_bar(...)`, while code that is + also in group `b` can simply do `CPyDef_bar(...)`. + + Thus the prefix for a module in group `b` is 'exports_b.' if the current + group is *not* b and just '' if it is. + """ + groups = self.context.group_map + target_group_name = groups.get(module_name) + if target_group_name and target_group_name != self.context.group_name: + self.context.group_deps.add(target_group_name) + return f"exports_{exported_name(target_group_name)}." + else: + return "" + + def get_group_prefix(self, obj: ClassIR | FuncDecl) -> str: + """Get the group prefix for an object.""" + # See docs above + return self.get_module_group_prefix(obj.module_name) + + def static_name(self, id: str, module: str | None, prefix: str = STATIC_PREFIX) -> str: + """Create name of a C static variable. + + These are used for literals and imported modules, among other + things. + + The caller should ensure that the (id, module) pair cannot + overlap with other calls to this method within a compilation + group. + """ + lib_prefix = "" if not module else self.get_module_group_prefix(module) + # If we are accessing static via the export table, we need to dereference + # the pointer also. + star_maybe = "*" if lib_prefix else "" + suffix = self.names.private_name(module or "", id) + return f"{star_maybe}{lib_prefix}{prefix}{suffix}" + + def type_struct_name(self, cl: ClassIR) -> str: + return self.static_name(cl.name, cl.module_name, prefix=TYPE_PREFIX) + + def ctype(self, rtype: RType) -> str: + return rtype._ctype + + def ctype_spaced(self, rtype: RType) -> str: + """Adds a space after ctype for non-pointers.""" + ctype = self.ctype(rtype) + if ctype[-1] == "*": + return ctype + else: + return ctype + " " + + def c_undefined_value(self, rtype: RType) -> str: + if not rtype.is_unboxed: + return "NULL" + elif isinstance(rtype, RPrimitive): + return rtype.c_undefined + elif isinstance(rtype, RTuple): + return self.tuple_undefined_value(rtype) + assert False, rtype + + def c_error_value(self, rtype: RType) -> str: + return self.c_undefined_value(rtype) + + def native_function_name(self, fn: FuncDecl) -> str: + return f"{NATIVE_PREFIX}{fn.cname(self.names)}" + + def tuple_c_declaration(self, rtuple: RTuple) -> list[str]: + result = [ + f"#ifndef MYPYC_DECLARED_{rtuple.struct_name}", + f"#define MYPYC_DECLARED_{rtuple.struct_name}", + f"typedef struct {rtuple.struct_name} {{", + ] + if len(rtuple.types) == 0: # empty tuple + # Empty tuples contain a flag so that they can still indicate + # error values. + result.append("int empty_struct_error_flag;") + else: + i = 0 + for typ in rtuple.types: + result.append(f"{self.ctype_spaced(typ)}f{i};") + i += 1 + result.append(f"}} {rtuple.struct_name};") + result.append("#endif") + result.append("") + + return result + + def bitmap_field(self, index: int) -> str: + """Return C field name used for attribute bitmap.""" + n = index // BITMAP_BITS + if n == 0: + return "bitmap" + return f"bitmap{n + 1}" + + def attr_bitmap_expr(self, obj: str, cl: ClassIR, index: int) -> str: + """Return reference to the attribute definedness bitmap.""" + cast = f"({cl.struct_name(self.names)} *)" + attr = self.bitmap_field(index) + return f"({cast}{obj})->{attr}" + + def emit_attr_bitmap_set( + self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str + ) -> None: + """Mark an attribute as defined in the attribute bitmap. + + Assumes that the attribute is tracked in the bitmap (only some attributes + use the bitmap). If 'value' is not equal to the error value, do nothing. + """ + self._emit_attr_bitmap_update(value, obj, rtype, cl, attr, clear=False) + + def emit_attr_bitmap_clear(self, obj: str, rtype: RType, cl: ClassIR, attr: str) -> None: + """Mark an attribute as undefined in the attribute bitmap. + + Unlike emit_attr_bitmap_set, clear unconditionally. + """ + self._emit_attr_bitmap_update("", obj, rtype, cl, attr, clear=True) + + def _emit_attr_bitmap_update( + self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str, clear: bool + ) -> None: + if value: + check = self.error_value_check(rtype, value, "==") + self.emit_line(f"if (unlikely({check})) {{") + index = cl.bitmap_attrs.index(attr) + mask = 1 << (index & (BITMAP_BITS - 1)) + bitmap = self.attr_bitmap_expr(obj, cl, index) + if clear: + self.emit_line(f"{bitmap} &= ~{mask};") + else: + self.emit_line(f"{bitmap} |= {mask};") + if value: + self.emit_line("}") + + def use_vectorcall(self) -> bool: + return use_vectorcall(self.capi_version) + + def emit_undefined_attr_check( + self, + rtype: RType, + attr_expr: str, + compare: str, + obj: str, + attr: str, + cl: ClassIR, + *, + unlikely: bool = False, + ) -> None: + check = self.error_value_check(rtype, attr_expr, compare) + if unlikely: + check = f"unlikely({check})" + if rtype.error_overlap: + index = cl.bitmap_attrs.index(attr) + bit = 1 << (index & (BITMAP_BITS - 1)) + attr = self.bitmap_field(index) + obj_expr = f"({cl.struct_name(self.names)} *){obj}" + check = f"{check} && !(({obj_expr})->{attr} & {bit})" + self.emit_line(f"if ({check}) {{") + + def error_value_check(self, rtype: RType, value: str, compare: str) -> str: + if isinstance(rtype, RTuple): + return self.tuple_undefined_check_cond( + rtype, value, self.c_error_value, compare, check_exception=False + ) + else: + return f"{value} {compare} {self.c_error_value(rtype)}" + + def tuple_undefined_check_cond( + self, + rtuple: RTuple, + tuple_expr_in_c: str, + c_type_compare_val: Callable[[RType], str], + compare: str, + *, + check_exception: bool = True, + ) -> str: + if len(rtuple.types) == 0: + # empty tuple + return "{}.empty_struct_error_flag {} {}".format( + tuple_expr_in_c, compare, c_type_compare_val(int_rprimitive) + ) + if rtuple.error_overlap: + i = 0 + item_type = rtuple.types[0] + else: + for i, typ in enumerate(rtuple.types): + if not typ.error_overlap: + item_type = rtuple.types[i] + break + else: + assert False, "not expecting tuple with error overlap" + if isinstance(item_type, RTuple): + return self.tuple_undefined_check_cond( + item_type, tuple_expr_in_c + f".f{i}", c_type_compare_val, compare + ) + else: + check = f"{tuple_expr_in_c}.f{i} {compare} {c_type_compare_val(item_type)}" + if rtuple.error_overlap and check_exception: + check += " && PyErr_Occurred()" + return check + + def tuple_undefined_value(self, rtuple: RTuple) -> str: + """Undefined tuple value suitable in an expression.""" + return f"({rtuple.struct_name}) {self.c_initializer_undefined_value(rtuple)}" + + def c_initializer_undefined_value(self, rtype: RType) -> str: + """Undefined value represented in a form suitable for variable initialization.""" + if isinstance(rtype, RTuple): + if not rtype.types: + # Empty tuples contain a flag so that they can still indicate + # error values. + return f"{{ {int_rprimitive.c_undefined} }}" + items = ", ".join([self.c_initializer_undefined_value(t) for t in rtype.types]) + return f"{{ {items} }}" + else: + return self.c_undefined_value(rtype) + + # Higher-level operations + + def declare_tuple_struct(self, tuple_type: RTuple) -> None: + if tuple_type.struct_name not in self.context.declarations: + dependencies = set() + for typ in tuple_type.types: + # XXX other types might eventually need similar behavior + if isinstance(typ, RTuple): + dependencies.add(typ.struct_name) + + self.context.declarations[tuple_type.struct_name] = HeaderDeclaration( + self.tuple_c_declaration(tuple_type), dependencies=dependencies, is_type=True + ) + + def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None: + """Increment reference count of C expression `dest`. + + For composite unboxed structures (e.g. tuples) recursively + increment reference counts for each component. + + If rare is True, optimize for code size and compilation speed. + """ + if is_int_rprimitive(rtype): + if rare: + self.emit_line("CPyTagged_IncRef(%s);" % dest) + else: + self.emit_line("CPyTagged_INCREF(%s);" % dest) + elif isinstance(rtype, RTuple): + for i, item_type in enumerate(rtype.types): + self.emit_inc_ref(f"{dest}.f{i}", item_type) + elif not rtype.is_unboxed: + # Always inline, since this is a simple op + self.emit_line("CPy_INCREF(%s);" % dest) + # Otherwise assume it's an unboxed, pointerless value and do nothing. + + def emit_dec_ref( + self, dest: str, rtype: RType, *, is_xdec: bool = False, rare: bool = False + ) -> None: + """Decrement reference count of C expression `dest`. + + For composite unboxed structures (e.g. tuples) recursively + decrement reference counts for each component. + + If rare is True, optimize for code size and compilation speed. + """ + x = "X" if is_xdec else "" + if is_int_rprimitive(rtype): + if rare: + self.emit_line(f"CPyTagged_{x}DecRef({dest});") + else: + # Inlined + self.emit_line(f"CPyTagged_{x}DECREF({dest});") + elif isinstance(rtype, RTuple): + for i, item_type in enumerate(rtype.types): + self.emit_dec_ref(f"{dest}.f{i}", item_type, is_xdec=is_xdec, rare=rare) + elif not rtype.is_unboxed: + if rare: + self.emit_line(f"CPy_{x}DecRef({dest});") + else: + # Inlined + self.emit_line(f"CPy_{x}DECREF({dest});") + # Otherwise assume it's an unboxed, pointerless value and do nothing. + + def pretty_name(self, typ: RType) -> str: + value_type = optional_value_type(typ) + if value_type is not None: + return "%s or None" % self.pretty_name(value_type) + return str(typ) + + def emit_cast( + self, + src: str, + dest: str, + typ: RType, + *, + declare_dest: bool = False, + error: ErrorHandler | None = None, + raise_exception: bool = True, + optional: bool = False, + src_type: RType | None = None, + likely: bool = True, + ) -> None: + """Emit code for casting a value of given type. + + Somewhat strangely, this supports unboxed types but only + operates on boxed versions. This is necessary to properly + handle types such as Optional[int] in compatibility glue. + + By default, assign NULL (error value) to dest if the value has + an incompatible type and raise TypeError. These can be customized + using 'error' and 'raise_exception'. + + Always copy/steal the reference in 'src'. + + Args: + src: Name of source C variable + dest: Name of target C variable + typ: Type of value + declare_dest: If True, also declare the variable 'dest' + error: What happens on error + raise_exception: If True, also raise TypeError on failure + likely: If the cast is likely to succeed (can be False for unions) + """ + error = error or AssignHandler() + + # Special case casting *from* optional + if src_type and is_optional_type(src_type) and not is_object_rprimitive(typ): + value_type = optional_value_type(src_type) + assert value_type is not None + if is_same_type(value_type, typ): + if declare_dest: + self.emit_line(f"PyObject *{dest};") + check = "({} != Py_None)" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check.format(src), optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") + return + + # TODO: Verify refcount handling. + if ( + is_list_rprimitive(typ) + or is_dict_rprimitive(typ) + or is_set_rprimitive(typ) + or is_str_rprimitive(typ) + or is_range_rprimitive(typ) + or is_float_rprimitive(typ) + or is_int_rprimitive(typ) + or is_bool_rprimitive(typ) + or is_bit_rprimitive(typ) + or is_fixed_width_rtype(typ) + ): + if declare_dest: + self.emit_line(f"PyObject *{dest};") + if is_list_rprimitive(typ): + prefix = "PyList" + elif is_dict_rprimitive(typ): + prefix = "PyDict" + elif is_set_rprimitive(typ): + prefix = "PySet" + elif is_str_rprimitive(typ): + prefix = "PyUnicode" + elif is_range_rprimitive(typ): + prefix = "PyRange" + elif is_float_rprimitive(typ): + prefix = "CPyFloat" + elif is_int_rprimitive(typ) or is_fixed_width_rtype(typ): + # TODO: Range check for fixed-width types? + prefix = "PyLong" + elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + prefix = "PyBool" + else: + assert False, f"unexpected primitive type: {typ}" + check = "({}_Check({}))" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check.format(prefix, src), optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") + elif is_bytes_rprimitive(typ): + if declare_dest: + self.emit_line(f"PyObject *{dest};") + check = "(PyBytes_Check({}) || PyByteArray_Check({}))" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check.format(src, src), optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") + elif is_tuple_rprimitive(typ): + if declare_dest: + self.emit_line(f"{self.ctype(typ)} {dest};") + check = "(PyTuple_Check({}))" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check.format(src), optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") + elif isinstance(typ, RInstance): + if declare_dest: + self.emit_line(f"PyObject *{dest};") + concrete = all_concrete_classes(typ.class_ir) + # If there are too many concrete subclasses or we can't find any + # (meaning the code ought to be dead or we aren't doing global opts), + # fall back to a normal typecheck. + # Otherwise check all the subclasses. + if not concrete or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: + check = "(PyObject_TypeCheck({}, {}))".format( + src, self.type_struct_name(typ.class_ir) + ) + else: + full_str = "(Py_TYPE({src}) == {targets[0]})" + for i in range(1, len(concrete)): + full_str += " || (Py_TYPE({src}) == {targets[%d]})" % i + if len(concrete) > 1: + full_str = "(%s)" % full_str + check = full_str.format( + src=src, targets=[self.type_struct_name(ir) for ir in concrete] + ) + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check, optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") + elif is_none_rprimitive(typ): + if declare_dest: + self.emit_line(f"PyObject *{dest};") + check = "({} == Py_None)" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check.format(src), optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") + elif is_object_rprimitive(typ): + if declare_dest: + self.emit_line(f"PyObject *{dest};") + self.emit_arg_check(src, dest, typ, "", optional) + self.emit_line(f"{dest} = {src};") + if optional: + self.emit_line("}") + elif isinstance(typ, RUnion): + self.emit_union_cast( + src, dest, typ, declare_dest, error, optional, src_type, raise_exception + ) + elif isinstance(typ, RTuple): + assert not optional + self.emit_tuple_cast(src, dest, typ, declare_dest, error, src_type) + else: + assert False, "Cast not implemented: %s" % typ + + def emit_cast_error_handler( + self, error: ErrorHandler, src: str, dest: str, typ: RType, raise_exception: bool + ) -> None: + if raise_exception: + if isinstance(error, TracebackAndGotoHandler): + # Merge raising and emitting traceback entry into a single call. + self.emit_type_error_traceback( + error.source_path, error.module_name, error.traceback_entry, typ=typ, src=src + ) + self.emit_line("goto %s;" % error.label) + return + self.emit_line(f'CPy_TypeError("{self.pretty_name(typ)}", {src}); ') + if isinstance(error, AssignHandler): + self.emit_line("%s = NULL;" % dest) + elif isinstance(error, GotoHandler): + self.emit_line("goto %s;" % error.label) + elif isinstance(error, TracebackAndGotoHandler): + self.emit_line("%s = NULL;" % dest) + self.emit_traceback(error.source_path, error.module_name, error.traceback_entry) + self.emit_line("goto %s;" % error.label) + else: + assert isinstance(error, ReturnHandler) + self.emit_line("return %s;" % error.value) + + def emit_union_cast( + self, + src: str, + dest: str, + typ: RUnion, + declare_dest: bool, + error: ErrorHandler, + optional: bool, + src_type: RType | None, + raise_exception: bool, + ) -> None: + """Emit cast to a union type. + + The arguments are similar to emit_cast. + """ + if declare_dest: + self.emit_line(f"PyObject *{dest};") + good_label = self.new_label() + if optional: + self.emit_line(f"if ({src} == NULL) {{") + self.emit_line(f"{dest} = {self.c_error_value(typ)};") + self.emit_line(f"goto {good_label};") + self.emit_line("}") + for item in typ.items: + self.emit_cast( + src, + dest, + item, + declare_dest=False, + raise_exception=False, + optional=False, + likely=False, + ) + self.emit_line(f"if ({dest} != NULL) goto {good_label};") + # Handle cast failure. + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_label(good_label) + + def emit_tuple_cast( + self, + src: str, + dest: str, + typ: RTuple, + declare_dest: bool, + error: ErrorHandler, + src_type: RType | None, + ) -> None: + """Emit cast to a tuple type. + + The arguments are similar to emit_cast. + """ + if declare_dest: + self.emit_line(f"PyObject *{dest};") + # This reuse of the variable is super dodgy. We don't even + # care about the values except to check whether they are + # invalid. + out_label = self.new_label() + self.emit_lines( + "if (unlikely(!(PyTuple_Check({r}) && PyTuple_GET_SIZE({r}) == {size}))) {{".format( + r=src, size=len(typ.types) + ), + f"{dest} = NULL;", + f"goto {out_label};", + "}", + ) + for i, item in enumerate(typ.types): + # Since we did the checks above this should never fail + self.emit_cast( + f"PyTuple_GET_ITEM({src}, {i})", + dest, + item, + declare_dest=False, + raise_exception=False, + optional=False, + ) + self.emit_line(f"if ({dest} == NULL) goto {out_label};") + + self.emit_line(f"{dest} = {src};") + self.emit_label(out_label) + + def emit_arg_check(self, src: str, dest: str, typ: RType, check: str, optional: bool) -> None: + if optional: + self.emit_line(f"if ({src} == NULL) {{") + self.emit_line(f"{dest} = {self.c_error_value(typ)};") + if check != "": + self.emit_line("{}if {}".format("} else " if optional else "", check)) + elif optional: + self.emit_line("else {") + + def emit_unbox( + self, + src: str, + dest: str, + typ: RType, + *, + declare_dest: bool = False, + error: ErrorHandler | None = None, + raise_exception: bool = True, + optional: bool = False, + borrow: bool = False, + ) -> None: + """Emit code for unboxing a value of given type (from PyObject *). + + By default, assign error value to dest if the value has an + incompatible type and raise TypeError. These can be customized + using 'error' and 'raise_exception'. + + Generate a new reference unless 'borrow' is True. + + Args: + src: Name of source C variable + dest: Name of target C variable + typ: Type of value + declare_dest: If True, also declare the variable 'dest' + error: What happens on error + raise_exception: If True, also raise TypeError on failure + borrow: If True, create a borrowed reference + + """ + error = error or AssignHandler() + # TODO: Verify refcount handling. + if isinstance(error, AssignHandler): + failure = f"{dest} = {self.c_error_value(typ)};" + elif isinstance(error, GotoHandler): + failure = "goto %s;" % error.label + else: + assert isinstance(error, ReturnHandler) + failure = "return %s;" % error.value + if raise_exception: + raise_exc = f'CPy_TypeError("{self.pretty_name(typ)}", {src}); ' + failure = raise_exc + failure + if is_int_rprimitive(typ) or is_short_int_rprimitive(typ): + if declare_dest: + self.emit_line(f"CPyTagged {dest};") + self.emit_arg_check(src, dest, typ, f"(likely(PyLong_Check({src})))", optional) + if borrow: + self.emit_line(f" {dest} = CPyTagged_BorrowFromObject({src});") + else: + self.emit_line(f" {dest} = CPyTagged_FromObject({src});") + self.emit_line("else {") + self.emit_line(failure) + self.emit_line("}") + elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + # Whether we are borrowing or not makes no difference. + if declare_dest: + self.emit_line(f"char {dest};") + self.emit_arg_check(src, dest, typ, f"(unlikely(!PyBool_Check({src}))) {{", optional) + self.emit_line(failure) + self.emit_line("} else") + conversion = f"{src} == Py_True" + self.emit_line(f" {dest} = {conversion};") + elif is_none_rprimitive(typ): + # Whether we are borrowing or not makes no difference. + if declare_dest: + self.emit_line(f"char {dest};") + self.emit_arg_check(src, dest, typ, f"(unlikely({src} != Py_None)) {{", optional) + self.emit_line(failure) + self.emit_line("} else") + self.emit_line(f" {dest} = 1;") + elif is_int64_rprimitive(typ): + # Whether we are borrowing or not makes no difference. + assert not optional # Not supported for overlapping error values + if declare_dest: + self.emit_line(f"int64_t {dest};") + self.emit_line(f"{dest} = CPyLong_AsInt64({src});") + if not isinstance(error, AssignHandler): + self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure) + elif is_int32_rprimitive(typ): + # Whether we are borrowing or not makes no difference. + assert not optional # Not supported for overlapping error values + if declare_dest: + self.emit_line(f"int32_t {dest};") + self.emit_line(f"{dest} = CPyLong_AsInt32({src});") + if not isinstance(error, AssignHandler): + self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure) + elif is_int16_rprimitive(typ): + # Whether we are borrowing or not makes no difference. + assert not optional # Not supported for overlapping error values + if declare_dest: + self.emit_line(f"int16_t {dest};") + self.emit_line(f"{dest} = CPyLong_AsInt16({src});") + if not isinstance(error, AssignHandler): + self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure) + elif is_uint8_rprimitive(typ): + # Whether we are borrowing or not makes no difference. + assert not optional # Not supported for overlapping error values + if declare_dest: + self.emit_line(f"uint8_t {dest};") + self.emit_line(f"{dest} = CPyLong_AsUInt8({src});") + if not isinstance(error, AssignHandler): + self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure) + elif is_float_rprimitive(typ): + assert not optional # Not supported for overlapping error values + if declare_dest: + self.emit_line(f"double {dest};") + # TODO: Don't use __float__ and __index__ + self.emit_line(f"{dest} = PyFloat_AsDouble({src});") + self.emit_lines(f"if ({dest} == -1.0 && PyErr_Occurred()) {{", failure, "}") + elif isinstance(typ, RTuple): + self.declare_tuple_struct(typ) + if declare_dest: + self.emit_line(f"{self.ctype(typ)} {dest};") + # HACK: The error handling for unboxing tuples is busted + # and instead of fixing it I am just wrapping it in the + # cast code which I think is right. This is not good. + if optional: + self.emit_line(f"if ({src} == NULL) {{") + self.emit_line(f"{dest} = {self.c_error_value(typ)};") + self.emit_line("} else {") + + cast_temp = self.temp_name() + self.emit_tuple_cast( + src, cast_temp, typ, declare_dest=True, error=error, src_type=None + ) + self.emit_line(f"if (unlikely({cast_temp} == NULL)) {{") + + # self.emit_arg_check(src, dest, typ, + # '(!PyTuple_Check({}) || PyTuple_Size({}) != {}) {{'.format( + # src, src, len(typ.types)), optional) + self.emit_line(failure) # TODO: Decrease refcount? + self.emit_line("} else {") + if not typ.types: + self.emit_line(f"{dest}.empty_struct_error_flag = 0;") + for i, item_type in enumerate(typ.types): + temp = self.temp_name() + # emit_tuple_cast above checks the size, so this should not fail + self.emit_line(f"PyObject *{temp} = PyTuple_GET_ITEM({src}, {i});") + temp2 = self.temp_name() + # Unbox or check the item. + if item_type.is_unboxed: + self.emit_unbox( + temp, + temp2, + item_type, + raise_exception=raise_exception, + error=error, + declare_dest=True, + borrow=borrow, + ) + else: + if not borrow: + self.emit_inc_ref(temp, object_rprimitive) + self.emit_cast(temp, temp2, item_type, declare_dest=True) + self.emit_line(f"{dest}.f{i} = {temp2};") + self.emit_line("}") + if optional: + self.emit_line("}") + + else: + assert False, "Unboxing not implemented: %s" % typ + + def emit_box( + self, src: str, dest: str, typ: RType, declare_dest: bool = False, can_borrow: bool = False + ) -> None: + """Emit code for boxing a value of given type. + + Generate a simple assignment if no boxing is needed. + + The source reference count is stolen for the result (no need to decref afterwards). + """ + # TODO: Always generate a new reference (if a reference type) + if declare_dest: + declaration = "PyObject *" + else: + declaration = "" + if is_int_rprimitive(typ) or is_short_int_rprimitive(typ): + # Steal the existing reference if it exists. + self.emit_line(f"{declaration}{dest} = CPyTagged_StealAsObject({src});") + elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + # N.B: bool is special cased to produce a borrowed value + # after boxing, so we don't need to increment the refcount + # when this comes directly from a Box op. + self.emit_lines(f"{declaration}{dest} = {src} ? Py_True : Py_False;") + if not can_borrow: + self.emit_inc_ref(dest, object_rprimitive) + elif is_none_rprimitive(typ): + # N.B: None is special cased to produce a borrowed value + # after boxing, so we don't need to increment the refcount + # when this comes directly from a Box op. + self.emit_lines(f"{declaration}{dest} = Py_None;") + if not can_borrow: + self.emit_inc_ref(dest, object_rprimitive) + elif is_int32_rprimitive(typ) or is_int16_rprimitive(typ) or is_uint8_rprimitive(typ): + self.emit_line(f"{declaration}{dest} = PyLong_FromLong({src});") + elif is_int64_rprimitive(typ): + self.emit_line(f"{declaration}{dest} = PyLong_FromLongLong({src});") + elif is_float_rprimitive(typ): + self.emit_line(f"{declaration}{dest} = PyFloat_FromDouble({src});") + elif isinstance(typ, RTuple): + self.declare_tuple_struct(typ) + self.emit_line(f"{declaration}{dest} = PyTuple_New({len(typ.types)});") + self.emit_line(f"if (unlikely({dest} == NULL))") + self.emit_line(" CPyError_OutOfMemory();") + # TODO: Fail if dest is None + for i in range(0, len(typ.types)): + if not typ.is_unboxed: + self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {src}.f{i}") + else: + inner_name = self.temp_name() + self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) + self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") + else: + assert not typ.is_unboxed + # Type is boxed -- trivially just assign. + self.emit_line(f"{declaration}{dest} = {src};") + + def emit_error_check(self, value: str, rtype: RType, failure: str) -> None: + """Emit code for checking a native function return value for uncaught exception.""" + if isinstance(rtype, RTuple): + if len(rtype.types) == 0: + return # empty tuples can't fail. + else: + cond = self.tuple_undefined_check_cond(rtype, value, self.c_error_value, "==") + self.emit_line(f"if ({cond}) {{") + elif rtype.error_overlap: + # The error value is also valid as a normal value, so we need to also check + # for a raised exception. + self.emit_line(f"if ({value} == {self.c_error_value(rtype)} && PyErr_Occurred()) {{") + else: + self.emit_line(f"if ({value} == {self.c_error_value(rtype)}) {{") + self.emit_lines(failure, "}") + + def emit_gc_visit(self, target: str, rtype: RType) -> None: + """Emit code for GC visiting a C variable reference. + + Assume that 'target' represents a C expression that refers to a + struct member, such as 'self->x'. + """ + if not rtype.is_refcounted: + # Not refcounted -> no pointers -> no GC interaction. + return + elif isinstance(rtype, RPrimitive) and rtype.name == "builtins.int": + self.emit_line(f"if (CPyTagged_CheckLong({target})) {{") + self.emit_line(f"Py_VISIT(CPyTagged_LongAsObject({target}));") + self.emit_line("}") + elif isinstance(rtype, RTuple): + for i, item_type in enumerate(rtype.types): + self.emit_gc_visit(f"{target}.f{i}", item_type) + elif self.ctype(rtype) == "PyObject *": + # The simplest case. + self.emit_line(f"Py_VISIT({target});") + else: + assert False, "emit_gc_visit() not implemented for %s" % repr(rtype) + + def emit_gc_clear(self, target: str, rtype: RType) -> None: + """Emit code for clearing a C attribute reference for GC. + + Assume that 'target' represents a C expression that refers to a + struct member, such as 'self->x'. + """ + if not rtype.is_refcounted: + # Not refcounted -> no pointers -> no GC interaction. + return + elif isinstance(rtype, RPrimitive) and rtype.name == "builtins.int": + self.emit_line(f"if (CPyTagged_CheckLong({target})) {{") + self.emit_line(f"CPyTagged __tmp = {target};") + self.emit_line(f"{target} = {self.c_undefined_value(rtype)};") + self.emit_line("Py_XDECREF(CPyTagged_LongAsObject(__tmp));") + self.emit_line("}") + elif isinstance(rtype, RTuple): + for i, item_type in enumerate(rtype.types): + self.emit_gc_clear(f"{target}.f{i}", item_type) + elif self.ctype(rtype) == "PyObject *" and self.c_undefined_value(rtype) == "NULL": + # The simplest case. + self.emit_line(f"Py_CLEAR({target});") + else: + assert False, "emit_gc_clear() not implemented for %s" % repr(rtype) + + def emit_traceback( + self, source_path: str, module_name: str, traceback_entry: tuple[str, int] + ) -> None: + return self._emit_traceback("CPy_AddTraceback", source_path, module_name, traceback_entry) + + def emit_type_error_traceback( + self, + source_path: str, + module_name: str, + traceback_entry: tuple[str, int], + *, + typ: RType, + src: str, + ) -> None: + func = "CPy_TypeErrorTraceback" + type_str = f'"{self.pretty_name(typ)}"' + return self._emit_traceback( + func, source_path, module_name, traceback_entry, type_str=type_str, src=src + ) + + def _emit_traceback( + self, + func: str, + source_path: str, + module_name: str, + traceback_entry: tuple[str, int], + type_str: str = "", + src: str = "", + ) -> None: + globals_static = self.static_name("globals", module_name) + line = '%s("%s", "%s", %d, %s' % ( + func, + source_path.replace("\\", "\\\\"), + traceback_entry[0], + traceback_entry[1], + globals_static, + ) + if type_str: + assert src + line += f", {type_str}, {src}" + line += ");" + self.emit_line(line) + if DEBUG_ERRORS: + self.emit_line('assert(PyErr_Occurred() != NULL && "failure w/o err!");') + + def emit_unbox_failure_with_overlapping_error_value( + self, dest: str, typ: RType, failure: str + ) -> None: + self.emit_line(f"if ({dest} == {self.c_error_value(typ)} && PyErr_Occurred()) {{") + self.emit_line(failure) + self.emit_line("}") + + +def c_array_initializer(components: list[str], *, indented: bool = False) -> str: + """Construct an initializer for a C array variable. + + Components are C expressions valid in an initializer. + + For example, if components are ["1", "2"], the result + would be "{1, 2}", which can be used like this: + + int a[] = {1, 2}; + + If the result is long, split it into multiple lines. + """ + indent = " " * 4 if indented else "" + res = [] + current: list[str] = [] + cur_len = 0 + for c in components: + if not current or cur_len + 2 + len(indent) + len(c) < 70: + current.append(c) + cur_len += len(c) + 2 + else: + res.append(indent + ", ".join(current)) + current = [c] + cur_len = len(c) + if not res: + # Result fits on a single line + return "{%s}" % ", ".join(current) + # Multi-line result + res.append(indent + ", ".join(current)) + return "{\n " + ",\n ".join(res) + "\n" + indent + "}" diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py new file mode 100644 index 000000000000..d1a9ad3bace1 --- /dev/null +++ b/mypyc/codegen/emitclass.py @@ -0,0 +1,1060 @@ +"""Code generation for native classes and related wrappers.""" + +from __future__ import annotations + +from typing import Callable, Mapping, Tuple + +from mypyc.codegen.emit import Emitter, HeaderDeclaration, ReturnHandler +from mypyc.codegen.emitfunc import native_function_header +from mypyc.codegen.emitwrapper import ( + generate_bin_op_wrapper, + generate_bool_wrapper, + generate_contains_wrapper, + generate_dunder_wrapper, + generate_get_wrapper, + generate_hash_wrapper, + generate_ipow_wrapper, + generate_len_wrapper, + generate_richcompare_wrapper, + generate_set_del_item_wrapper, +) +from mypyc.common import BITMAP_BITS, BITMAP_TYPE, NATIVE_PREFIX, PREFIX, REG_PREFIX +from mypyc.ir.class_ir import ClassIR, VTableEntries +from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR +from mypyc.ir.rtypes import RTuple, RType, object_rprimitive +from mypyc.namegen import NameGenerator +from mypyc.sametype import is_same_type + + +def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + return f"{NATIVE_PREFIX}{fn.cname(emitter.names)}" + + +def wrapper_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + return f"{PREFIX}{fn.cname(emitter.names)}" + + +# We maintain a table from dunder function names to struct slots they +# correspond to and functions that generate a wrapper (if necessary) +# and return the function name to stick in the slot. +# TODO: Add remaining dunder methods +SlotGenerator = Callable[[ClassIR, FuncIR, Emitter], str] +SlotTable = Mapping[str, Tuple[str, SlotGenerator]] + +SLOT_DEFS: SlotTable = { + "__init__": ("tp_init", lambda c, t, e: generate_init_for_class(c, t, e)), + "__call__": ("tp_call", lambda c, t, e: generate_call_wrapper(c, t, e)), + "__str__": ("tp_str", native_slot), + "__repr__": ("tp_repr", native_slot), + "__next__": ("tp_iternext", native_slot), + "__iter__": ("tp_iter", native_slot), + "__hash__": ("tp_hash", generate_hash_wrapper), + "__get__": ("tp_descr_get", generate_get_wrapper), +} + +AS_MAPPING_SLOT_DEFS: SlotTable = { + "__getitem__": ("mp_subscript", generate_dunder_wrapper), + "__setitem__": ("mp_ass_subscript", generate_set_del_item_wrapper), + "__delitem__": ("mp_ass_subscript", generate_set_del_item_wrapper), + "__len__": ("mp_length", generate_len_wrapper), +} + +AS_SEQUENCE_SLOT_DEFS: SlotTable = {"__contains__": ("sq_contains", generate_contains_wrapper)} + +AS_NUMBER_SLOT_DEFS: SlotTable = { + # Unary operations. + "__bool__": ("nb_bool", generate_bool_wrapper), + "__int__": ("nb_int", generate_dunder_wrapper), + "__float__": ("nb_float", generate_dunder_wrapper), + "__neg__": ("nb_negative", generate_dunder_wrapper), + "__pos__": ("nb_positive", generate_dunder_wrapper), + "__abs__": ("nb_absolute", generate_dunder_wrapper), + "__invert__": ("nb_invert", generate_dunder_wrapper), + # Binary operations. + "__add__": ("nb_add", generate_bin_op_wrapper), + "__radd__": ("nb_add", generate_bin_op_wrapper), + "__sub__": ("nb_subtract", generate_bin_op_wrapper), + "__rsub__": ("nb_subtract", generate_bin_op_wrapper), + "__mul__": ("nb_multiply", generate_bin_op_wrapper), + "__rmul__": ("nb_multiply", generate_bin_op_wrapper), + "__mod__": ("nb_remainder", generate_bin_op_wrapper), + "__rmod__": ("nb_remainder", generate_bin_op_wrapper), + "__truediv__": ("nb_true_divide", generate_bin_op_wrapper), + "__rtruediv__": ("nb_true_divide", generate_bin_op_wrapper), + "__floordiv__": ("nb_floor_divide", generate_bin_op_wrapper), + "__rfloordiv__": ("nb_floor_divide", generate_bin_op_wrapper), + "__divmod__": ("nb_divmod", generate_bin_op_wrapper), + "__rdivmod__": ("nb_divmod", generate_bin_op_wrapper), + "__lshift__": ("nb_lshift", generate_bin_op_wrapper), + "__rlshift__": ("nb_lshift", generate_bin_op_wrapper), + "__rshift__": ("nb_rshift", generate_bin_op_wrapper), + "__rrshift__": ("nb_rshift", generate_bin_op_wrapper), + "__and__": ("nb_and", generate_bin_op_wrapper), + "__rand__": ("nb_and", generate_bin_op_wrapper), + "__or__": ("nb_or", generate_bin_op_wrapper), + "__ror__": ("nb_or", generate_bin_op_wrapper), + "__xor__": ("nb_xor", generate_bin_op_wrapper), + "__rxor__": ("nb_xor", generate_bin_op_wrapper), + "__matmul__": ("nb_matrix_multiply", generate_bin_op_wrapper), + "__rmatmul__": ("nb_matrix_multiply", generate_bin_op_wrapper), + # In-place binary operations. + "__iadd__": ("nb_inplace_add", generate_dunder_wrapper), + "__isub__": ("nb_inplace_subtract", generate_dunder_wrapper), + "__imul__": ("nb_inplace_multiply", generate_dunder_wrapper), + "__imod__": ("nb_inplace_remainder", generate_dunder_wrapper), + "__itruediv__": ("nb_inplace_true_divide", generate_dunder_wrapper), + "__ifloordiv__": ("nb_inplace_floor_divide", generate_dunder_wrapper), + "__ilshift__": ("nb_inplace_lshift", generate_dunder_wrapper), + "__irshift__": ("nb_inplace_rshift", generate_dunder_wrapper), + "__iand__": ("nb_inplace_and", generate_dunder_wrapper), + "__ior__": ("nb_inplace_or", generate_dunder_wrapper), + "__ixor__": ("nb_inplace_xor", generate_dunder_wrapper), + "__imatmul__": ("nb_inplace_matrix_multiply", generate_dunder_wrapper), + # Ternary operations. (yes, really) + # These are special cased in generate_bin_op_wrapper(). + "__pow__": ("nb_power", generate_bin_op_wrapper), + "__rpow__": ("nb_power", generate_bin_op_wrapper), + "__ipow__": ("nb_inplace_power", generate_ipow_wrapper), +} + +AS_ASYNC_SLOT_DEFS: SlotTable = { + "__await__": ("am_await", native_slot), + "__aiter__": ("am_aiter", native_slot), + "__anext__": ("am_anext", native_slot), +} + +SIDE_TABLES = [ + ("as_mapping", "PyMappingMethods", AS_MAPPING_SLOT_DEFS), + ("as_sequence", "PySequenceMethods", AS_SEQUENCE_SLOT_DEFS), + ("as_number", "PyNumberMethods", AS_NUMBER_SLOT_DEFS), + ("as_async", "PyAsyncMethods", AS_ASYNC_SLOT_DEFS), +] + +# Slots that need to always be filled in because they don't get +# inherited right. +ALWAYS_FILL = {"__hash__"} + + +def generate_call_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + if emitter.use_vectorcall(): + # Use vectorcall wrapper if supported (PEP 590). + return "PyVectorcall_Call" + else: + # On older Pythons use the legacy wrapper. + return wrapper_slot(cl, fn, emitter) + + +def slot_key(attr: str) -> str: + """Map dunder method name to sort key. + + Sort reverse operator methods and __delitem__ after others ('x' > '_'). + """ + if (attr.startswith("__r") and attr != "__rshift__") or attr == "__delitem__": + return "x" + attr + return attr + + +def generate_slots(cl: ClassIR, table: SlotTable, emitter: Emitter) -> dict[str, str]: + fields: dict[str, str] = {} + generated: dict[str, str] = {} + # Sort for determinism on Python 3.5 + for name, (slot, generator) in sorted(table.items(), key=lambda x: slot_key(x[0])): + method_cls = cl.get_method_and_class(name) + if method_cls and (method_cls[1] == cl or name in ALWAYS_FILL): + if slot in generated: + # Reuse previously generated wrapper. + fields[slot] = generated[slot] + else: + # Generate new wrapper. + name = generator(cl, method_cls[0], emitter) + fields[slot] = name + generated[slot] = name + + return fields + + +def generate_class_type_decl( + cl: ClassIR, c_emitter: Emitter, external_emitter: Emitter, emitter: Emitter +) -> None: + context = c_emitter.context + name = emitter.type_struct_name(cl) + context.declarations[name] = HeaderDeclaration( + f"PyTypeObject *{emitter.type_struct_name(cl)};", needs_export=True + ) + + # If this is a non-extension class, all we want is the type object decl. + if not cl.is_ext_class: + return + + generate_object_struct(cl, external_emitter) + generate_full = not cl.is_trait and not cl.builtin_base + if generate_full: + context.declarations[emitter.native_function_name(cl.ctor)] = HeaderDeclaration( + f"{native_function_header(cl.ctor, emitter)};", needs_export=True + ) + + +def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: + """Generate C code for a class. + + This is the main entry point to the module. + """ + name = cl.name + name_prefix = cl.name_prefix(emitter.names) + + setup_name = f"{name_prefix}_setup" + new_name = f"{name_prefix}_new" + members_name = f"{name_prefix}_members" + getseters_name = f"{name_prefix}_getseters" + vtable_name = f"{name_prefix}_vtable" + traverse_name = f"{name_prefix}_traverse" + clear_name = f"{name_prefix}_clear" + dealloc_name = f"{name_prefix}_dealloc" + methods_name = f"{name_prefix}_methods" + vtable_setup_name = f"{name_prefix}_trait_vtable_setup" + + fields: dict[str, str] = {"tp_name": f'"{name}"'} + + generate_full = not cl.is_trait and not cl.builtin_base + needs_getseters = cl.needs_getseters or not cl.is_generated or cl.has_dict + + if not cl.builtin_base: + fields["tp_new"] = new_name + + if generate_full: + fields["tp_dealloc"] = f"(destructor){name_prefix}_dealloc" + fields["tp_traverse"] = f"(traverseproc){name_prefix}_traverse" + fields["tp_clear"] = f"(inquiry){name_prefix}_clear" + if needs_getseters: + fields["tp_getset"] = getseters_name + fields["tp_methods"] = methods_name + + def emit_line() -> None: + emitter.emit_line() + + emit_line() + + # If the class has a method to initialize default attribute + # values, we need to call it during initialization. + defaults_fn = cl.get_method("__mypyc_defaults_setup") + + # If there is a __init__ method, we'll use it in the native constructor. + init_fn = cl.get_method("__init__") + + # Fill out slots in the type object from dunder methods. + fields.update(generate_slots(cl, SLOT_DEFS, emitter)) + + # Fill out dunder methods that live in tables hanging off the side. + for table_name, type, slot_defs in SIDE_TABLES: + slots = generate_slots(cl, slot_defs, emitter) + if slots: + table_struct_name = generate_side_table_for_class(cl, table_name, type, slots, emitter) + fields[f"tp_{table_name}"] = f"&{table_struct_name}" + + richcompare_name = generate_richcompare_wrapper(cl, emitter) + if richcompare_name: + fields["tp_richcompare"] = richcompare_name + + # If the class inherits from python, make space for a __dict__ + struct_name = cl.struct_name(emitter.names) + if cl.builtin_base: + base_size = f"sizeof({cl.builtin_base})" + elif cl.is_trait: + base_size = "sizeof(PyObject)" + else: + base_size = f"sizeof({struct_name})" + # Since our types aren't allocated using type() we need to + # populate these fields ourselves if we want them to have correct + # values. PyType_Ready will inherit the offsets from tp_base but + # that isn't what we want. + + # XXX: there is no reason for the __weakref__ stuff to be mixed up with __dict__ + if cl.has_dict and not has_managed_dict(cl, emitter): + # __dict__ lives right after the struct and __weakref__ lives right after that + # TODO: They should get members in the struct instead of doing this nonsense. + weak_offset = f"{base_size} + sizeof(PyObject *)" + emitter.emit_lines( + f"PyMemberDef {members_name}[] = {{", + f'{{"__dict__", T_OBJECT_EX, {base_size}, 0, NULL}},', + f'{{"__weakref__", T_OBJECT_EX, {weak_offset}, 0, NULL}},', + "{0}", + "};", + ) + + fields["tp_members"] = members_name + fields["tp_basicsize"] = f"{base_size} + 2*sizeof(PyObject *)" + if emitter.capi_version < (3, 12): + fields["tp_dictoffset"] = base_size + fields["tp_weaklistoffset"] = weak_offset + else: + fields["tp_basicsize"] = base_size + + if generate_full: + # Declare setup method that allocates and initializes an object. type is the + # type of the class being initialized, which could be another class if there + # is an interpreted subclass. + emitter.emit_line(f"static PyObject *{setup_name}(PyTypeObject *type);") + assert cl.ctor is not None + emitter.emit_line(native_function_header(cl.ctor, emitter) + ";") + + emit_line() + init_fn = cl.get_method("__init__") + generate_new_for_class(cl, new_name, vtable_name, setup_name, init_fn, emitter) + emit_line() + generate_traverse_for_class(cl, traverse_name, emitter) + emit_line() + generate_clear_for_class(cl, clear_name, emitter) + emit_line() + generate_dealloc_for_class(cl, dealloc_name, clear_name, emitter) + emit_line() + + if cl.allow_interpreted_subclasses: + shadow_vtable_name: str | None = generate_vtables( + cl, vtable_setup_name + "_shadow", vtable_name + "_shadow", emitter, shadow=True + ) + emit_line() + else: + shadow_vtable_name = None + vtable_name = generate_vtables(cl, vtable_setup_name, vtable_name, emitter, shadow=False) + emit_line() + if needs_getseters: + generate_getseter_declarations(cl, emitter) + emit_line() + generate_getseters_table(cl, getseters_name, emitter) + emit_line() + + if cl.is_trait: + generate_new_for_trait(cl, new_name, emitter) + + generate_methods_table(cl, methods_name, emitter) + emit_line() + + flags = ["Py_TPFLAGS_DEFAULT", "Py_TPFLAGS_HEAPTYPE", "Py_TPFLAGS_BASETYPE"] + if generate_full: + flags.append("Py_TPFLAGS_HAVE_GC") + if cl.has_method("__call__") and emitter.use_vectorcall(): + fields["tp_vectorcall_offset"] = "offsetof({}, vectorcall)".format( + cl.struct_name(emitter.names) + ) + flags.append("_Py_TPFLAGS_HAVE_VECTORCALL") + if not fields.get("tp_vectorcall"): + # This is just a placeholder to please CPython. It will be + # overridden during setup. + fields["tp_call"] = "PyVectorcall_Call" + if has_managed_dict(cl, emitter): + flags.append("Py_TPFLAGS_MANAGED_DICT") + fields["tp_flags"] = " | ".join(flags) + + emitter.emit_line(f"static PyTypeObject {emitter.type_struct_name(cl)}_template_ = {{") + emitter.emit_line("PyVarObject_HEAD_INIT(NULL, 0)") + for field, value in fields.items(): + emitter.emit_line(f".{field} = {value},") + emitter.emit_line("};") + emitter.emit_line( + "static PyTypeObject *{t}_template = &{t}_template_;".format( + t=emitter.type_struct_name(cl) + ) + ) + + emitter.emit_line() + if generate_full: + generate_setup_for_class( + cl, setup_name, defaults_fn, vtable_name, shadow_vtable_name, emitter + ) + emitter.emit_line() + generate_constructor_for_class(cl, cl.ctor, init_fn, setup_name, vtable_name, emitter) + emitter.emit_line() + if needs_getseters: + generate_getseters(cl, emitter) + + +def getter_name(cl: ClassIR, attribute: str, names: NameGenerator) -> str: + return names.private_name(cl.module_name, f"{cl.name}_get_{attribute}") + + +def setter_name(cl: ClassIR, attribute: str, names: NameGenerator) -> str: + return names.private_name(cl.module_name, f"{cl.name}_set_{attribute}") + + +def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None: + seen_attrs: set[tuple[str, RType]] = set() + lines: list[str] = [] + lines += ["typedef struct {", "PyObject_HEAD", "CPyVTableItem *vtable;"] + if cl.has_method("__call__") and emitter.use_vectorcall(): + lines.append("vectorcallfunc vectorcall;") + bitmap_attrs = [] + for base in reversed(cl.base_mro): + if not base.is_trait: + if base.bitmap_attrs: + # Do we need another attribute bitmap field? + if emitter.bitmap_field(len(base.bitmap_attrs) - 1) not in bitmap_attrs: + for i in range(0, len(base.bitmap_attrs), BITMAP_BITS): + attr = emitter.bitmap_field(i) + if attr not in bitmap_attrs: + lines.append(f"{BITMAP_TYPE} {attr};") + bitmap_attrs.append(attr) + for attr, rtype in base.attributes.items(): + if (attr, rtype) not in seen_attrs: + lines.append(f"{emitter.ctype_spaced(rtype)}{emitter.attr(attr)};") + seen_attrs.add((attr, rtype)) + + if isinstance(rtype, RTuple): + emitter.declare_tuple_struct(rtype) + + lines.append(f"}} {cl.struct_name(emitter.names)};") + lines.append("") + emitter.context.declarations[cl.struct_name(emitter.names)] = HeaderDeclaration( + lines, is_type=True + ) + + +def generate_vtables( + base: ClassIR, vtable_setup_name: str, vtable_name: str, emitter: Emitter, shadow: bool +) -> str: + """Emit the vtables and vtable setup functions for a class. + + This includes both the primary vtable and any trait implementation vtables. + The trait vtables go before the main vtable, and have the following layout: + { + CPyType_T1, // pointer to type object + C_T1_trait_vtable, // pointer to array of method pointers + C_T1_offset_table, // pointer to array of attribute offsets + CPyType_T2, + C_T2_trait_vtable, + C_T2_offset_table, + ... + } + The method implementations are calculated at the end of IR pass, attribute + offsets are {offsetof(native__C, _x1), offsetof(native__C, _y1), ...}. + + To account for both dynamic loading and dynamic class creation, + vtables are populated dynamically at class creation time, so we + emit empty array definitions to store the vtables and a function to + populate them. + + If shadow is True, generate "shadow vtables" that point to the + shadow glue methods (which should dispatch via the Python C-API). + + Returns the expression to use to refer to the vtable, which might be + different than the name, if there are trait vtables. + """ + + def trait_vtable_name(trait: ClassIR) -> str: + return "{}_{}_trait_vtable{}".format( + base.name_prefix(emitter.names), + trait.name_prefix(emitter.names), + "_shadow" if shadow else "", + ) + + def trait_offset_table_name(trait: ClassIR) -> str: + return "{}_{}_offset_table".format( + base.name_prefix(emitter.names), trait.name_prefix(emitter.names) + ) + + # Emit array definitions with enough space for all the entries + emitter.emit_line( + "static CPyVTableItem {}[{}];".format( + vtable_name, max(1, len(base.vtable_entries) + 3 * len(base.trait_vtables)) + ) + ) + + for trait, vtable in base.trait_vtables.items(): + # Trait methods entry (vtable index -> method implementation). + emitter.emit_line( + f"static CPyVTableItem {trait_vtable_name(trait)}[{max(1, len(vtable))}];" + ) + # Trait attributes entry (attribute number in trait -> offset in actual struct). + emitter.emit_line( + "static size_t {}[{}];".format( + trait_offset_table_name(trait), max(1, len(trait.attributes)) + ) + ) + + # Emit vtable setup function + emitter.emit_line("static bool") + emitter.emit_line(f"{NATIVE_PREFIX}{vtable_setup_name}(void)") + emitter.emit_line("{") + + if base.allow_interpreted_subclasses and not shadow: + emitter.emit_line(f"{NATIVE_PREFIX}{vtable_setup_name}_shadow();") + + subtables = [] + for trait, vtable in base.trait_vtables.items(): + name = trait_vtable_name(trait) + offset_name = trait_offset_table_name(trait) + generate_vtable(vtable, name, emitter, [], shadow) + generate_offset_table(offset_name, emitter, trait, base) + subtables.append((trait, name, offset_name)) + + generate_vtable(base.vtable_entries, vtable_name, emitter, subtables, shadow) + + emitter.emit_line("return 1;") + emitter.emit_line("}") + + return vtable_name if not subtables else f"{vtable_name} + {len(subtables) * 3}" + + +def generate_offset_table( + trait_offset_table_name: str, emitter: Emitter, trait: ClassIR, cl: ClassIR +) -> None: + """Generate attribute offset row of a trait vtable.""" + emitter.emit_line(f"size_t {trait_offset_table_name}_scratch[] = {{") + for attr in trait.attributes: + emitter.emit_line(f"offsetof({cl.struct_name(emitter.names)}, {emitter.attr(attr)}),") + if not trait.attributes: + # This is for msvc. + emitter.emit_line("0") + emitter.emit_line("};") + emitter.emit_line( + "memcpy({name}, {name}_scratch, sizeof({name}));".format(name=trait_offset_table_name) + ) + + +def generate_vtable( + entries: VTableEntries, + vtable_name: str, + emitter: Emitter, + subtables: list[tuple[ClassIR, str, str]], + shadow: bool, +) -> None: + emitter.emit_line(f"CPyVTableItem {vtable_name}_scratch[] = {{") + if subtables: + emitter.emit_line("/* Array of trait vtables */") + for trait, table, offset_table in subtables: + emitter.emit_line( + "(CPyVTableItem){}, (CPyVTableItem){}, (CPyVTableItem){},".format( + emitter.type_struct_name(trait), table, offset_table + ) + ) + emitter.emit_line("/* Start of real vtable */") + + for entry in entries: + method = entry.shadow_method if shadow and entry.shadow_method else entry.method + emitter.emit_line( + "(CPyVTableItem){}{}{},".format( + emitter.get_group_prefix(entry.method.decl), + NATIVE_PREFIX, + method.cname(emitter.names), + ) + ) + + # msvc doesn't allow empty arrays; maybe allowing them at all is an extension? + if not entries: + emitter.emit_line("NULL") + emitter.emit_line("};") + emitter.emit_line("memcpy({name}, {name}_scratch, sizeof({name}));".format(name=vtable_name)) + + +def generate_setup_for_class( + cl: ClassIR, + func_name: str, + defaults_fn: FuncIR | None, + vtable_name: str, + shadow_vtable_name: str | None, + emitter: Emitter, +) -> None: + """Generate a native function that allocates an instance of a class.""" + emitter.emit_line("static PyObject *") + emitter.emit_line(f"{func_name}(PyTypeObject *type)") + emitter.emit_line("{") + emitter.emit_line(f"{cl.struct_name(emitter.names)} *self;") + emitter.emit_line(f"self = ({cl.struct_name(emitter.names)} *)type->tp_alloc(type, 0);") + emitter.emit_line("if (self == NULL)") + emitter.emit_line(" return NULL;") + + if shadow_vtable_name: + emitter.emit_line(f"if (type != {emitter.type_struct_name(cl)}) {{") + emitter.emit_line(f"self->vtable = {shadow_vtable_name};") + emitter.emit_line("} else {") + emitter.emit_line(f"self->vtable = {vtable_name};") + emitter.emit_line("}") + else: + emitter.emit_line(f"self->vtable = {vtable_name};") + + for i in range(0, len(cl.bitmap_attrs), BITMAP_BITS): + field = emitter.bitmap_field(i) + emitter.emit_line(f"self->{field} = 0;") + + if cl.has_method("__call__") and emitter.use_vectorcall(): + name = cl.method_decl("__call__").cname(emitter.names) + emitter.emit_line(f"self->vectorcall = {PREFIX}{name};") + + for base in reversed(cl.base_mro): + for attr, rtype in base.attributes.items(): + value = emitter.c_undefined_value(rtype) + + # We don't need to set this field to NULL since tp_alloc() already + # zero-initializes `self`. + if value != "NULL": + emitter.emit_line(rf"self->{emitter.attr(attr)} = {value};") + + # Initialize attributes to default values, if necessary + if defaults_fn is not None: + emitter.emit_lines( + "if ({}{}((PyObject *)self) == 0) {{".format( + NATIVE_PREFIX, defaults_fn.cname(emitter.names) + ), + "Py_DECREF(self);", + "return NULL;", + "}", + ) + + emitter.emit_line("return (PyObject *)self;") + emitter.emit_line("}") + + +def generate_constructor_for_class( + cl: ClassIR, + fn: FuncDecl, + init_fn: FuncIR | None, + setup_name: str, + vtable_name: str, + emitter: Emitter, +) -> None: + """Generate a native function that allocates and initializes an instance of a class.""" + emitter.emit_line(f"{native_function_header(fn, emitter)}") + emitter.emit_line("{") + emitter.emit_line(f"PyObject *self = {setup_name}({emitter.type_struct_name(cl)});") + emitter.emit_line("if (self == NULL)") + emitter.emit_line(" return NULL;") + args = ", ".join(["self"] + [REG_PREFIX + arg.name for arg in fn.sig.args]) + if init_fn is not None: + emitter.emit_line( + "char res = {}{}{}({});".format( + emitter.get_group_prefix(init_fn.decl), + NATIVE_PREFIX, + init_fn.cname(emitter.names), + args, + ) + ) + emitter.emit_line("if (res == 2) {") + emitter.emit_line("Py_DECREF(self);") + emitter.emit_line("return NULL;") + emitter.emit_line("}") + + # If there is a nontrivial ctor that we didn't define, invoke it via tp_init + elif len(fn.sig.args) > 1: + emitter.emit_line(f"int res = {emitter.type_struct_name(cl)}->tp_init({args});") + + emitter.emit_line("if (res < 0) {") + emitter.emit_line("Py_DECREF(self);") + emitter.emit_line("return NULL;") + emitter.emit_line("}") + + emitter.emit_line("return self;") + emitter.emit_line("}") + + +def generate_init_for_class(cl: ClassIR, init_fn: FuncIR, emitter: Emitter) -> str: + """Generate an init function suitable for use as tp_init. + + tp_init needs to be a function that returns an int, and our + __init__ methods return a PyObject. Translate NULL to -1, + everything else to 0. + """ + func_name = f"{cl.name_prefix(emitter.names)}_init" + + emitter.emit_line("static int") + emitter.emit_line(f"{func_name}(PyObject *self, PyObject *args, PyObject *kwds)") + emitter.emit_line("{") + if cl.allow_interpreted_subclasses or cl.builtin_base: + emitter.emit_line( + "return {}{}(self, args, kwds) != NULL ? 0 : -1;".format( + PREFIX, init_fn.cname(emitter.names) + ) + ) + else: + emitter.emit_line("return 0;") + emitter.emit_line("}") + + return func_name + + +def generate_new_for_class( + cl: ClassIR, + func_name: str, + vtable_name: str, + setup_name: str, + init_fn: FuncIR | None, + emitter: Emitter, +) -> None: + emitter.emit_line("static PyObject *") + emitter.emit_line(f"{func_name}(PyTypeObject *type, PyObject *args, PyObject *kwds)") + emitter.emit_line("{") + # TODO: Check and unbox arguments + if not cl.allow_interpreted_subclasses: + emitter.emit_line(f"if (type != {emitter.type_struct_name(cl)}) {{") + emitter.emit_line( + 'PyErr_SetString(PyExc_TypeError, "interpreted classes cannot inherit from compiled");' + ) + emitter.emit_line("return NULL;") + emitter.emit_line("}") + + if not init_fn or cl.allow_interpreted_subclasses or cl.builtin_base or cl.is_serializable(): + # Match Python semantics -- __new__ doesn't call __init__. + emitter.emit_line(f"return {setup_name}(type);") + else: + # __new__ of a native class implicitly calls __init__ so that we + # can enforce that instances are always properly initialized. This + # is needed to support always defined attributes. + emitter.emit_line(f"PyObject *self = {setup_name}(type);") + emitter.emit_lines("if (self == NULL)", " return NULL;") + emitter.emit_line( + f"PyObject *ret = {PREFIX}{init_fn.cname(emitter.names)}(self, args, kwds);" + ) + emitter.emit_lines("if (ret == NULL)", " return NULL;") + emitter.emit_line("return self;") + emitter.emit_line("}") + + +def generate_new_for_trait(cl: ClassIR, func_name: str, emitter: Emitter) -> None: + emitter.emit_line("static PyObject *") + emitter.emit_line(f"{func_name}(PyTypeObject *type, PyObject *args, PyObject *kwds)") + emitter.emit_line("{") + emitter.emit_line(f"if (type != {emitter.type_struct_name(cl)}) {{") + emitter.emit_line( + "PyErr_SetString(PyExc_TypeError, " + '"interpreted classes cannot inherit from compiled traits");' + ) + emitter.emit_line("} else {") + emitter.emit_line('PyErr_SetString(PyExc_TypeError, "traits may not be directly created");') + emitter.emit_line("}") + emitter.emit_line("return NULL;") + emitter.emit_line("}") + + +def generate_traverse_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> None: + """Emit function that performs cycle GC traversal of an instance.""" + emitter.emit_line("static int") + emitter.emit_line( + f"{func_name}({cl.struct_name(emitter.names)} *self, visitproc visit, void *arg)" + ) + emitter.emit_line("{") + for base in reversed(cl.base_mro): + for attr, rtype in base.attributes.items(): + emitter.emit_gc_visit(f"self->{emitter.attr(attr)}", rtype) + if has_managed_dict(cl, emitter): + emitter.emit_line("PyObject_VisitManagedDict((PyObject *)self, visit, arg);") + elif cl.has_dict: + struct_name = cl.struct_name(emitter.names) + # __dict__ lives right after the struct and __weakref__ lives right after that + emitter.emit_gc_visit( + f"*((PyObject **)((char *)self + sizeof({struct_name})))", object_rprimitive + ) + emitter.emit_gc_visit( + f"*((PyObject **)((char *)self + sizeof(PyObject *) + sizeof({struct_name})))", + object_rprimitive, + ) + emitter.emit_line("return 0;") + emitter.emit_line("}") + + +def generate_clear_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> None: + emitter.emit_line("static int") + emitter.emit_line(f"{func_name}({cl.struct_name(emitter.names)} *self)") + emitter.emit_line("{") + for base in reversed(cl.base_mro): + for attr, rtype in base.attributes.items(): + emitter.emit_gc_clear(f"self->{emitter.attr(attr)}", rtype) + if has_managed_dict(cl, emitter): + emitter.emit_line("PyObject_ClearManagedDict((PyObject *)self);") + elif cl.has_dict: + struct_name = cl.struct_name(emitter.names) + # __dict__ lives right after the struct and __weakref__ lives right after that + emitter.emit_gc_clear( + f"*((PyObject **)((char *)self + sizeof({struct_name})))", object_rprimitive + ) + emitter.emit_gc_clear( + f"*((PyObject **)((char *)self + sizeof(PyObject *) + sizeof({struct_name})))", + object_rprimitive, + ) + emitter.emit_line("return 0;") + emitter.emit_line("}") + + +def generate_dealloc_for_class( + cl: ClassIR, dealloc_func_name: str, clear_func_name: str, emitter: Emitter +) -> None: + emitter.emit_line("static void") + emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)") + emitter.emit_line("{") + emitter.emit_line("PyObject_GC_UnTrack(self);") + # The trashcan is needed to handle deep recursive deallocations + emitter.emit_line(f"CPy_TRASHCAN_BEGIN(self, {dealloc_func_name})") + emitter.emit_line(f"{clear_func_name}(self);") + emitter.emit_line("Py_TYPE(self)->tp_free((PyObject *)self);") + emitter.emit_line("CPy_TRASHCAN_END(self)") + emitter.emit_line("}") + + +def generate_methods_table(cl: ClassIR, name: str, emitter: Emitter) -> None: + emitter.emit_line(f"static PyMethodDef {name}[] = {{") + for fn in cl.methods.values(): + if fn.decl.is_prop_setter or fn.decl.is_prop_getter: + continue + emitter.emit_line(f'{{"{fn.name}",') + emitter.emit_line(f" (PyCFunction){PREFIX}{fn.cname(emitter.names)},") + flags = ["METH_FASTCALL", "METH_KEYWORDS"] + if fn.decl.kind == FUNC_STATICMETHOD: + flags.append("METH_STATIC") + elif fn.decl.kind == FUNC_CLASSMETHOD: + flags.append("METH_CLASS") + + emitter.emit_line(" {}, NULL}},".format(" | ".join(flags))) + + # Provide a default __getstate__ and __setstate__ + if not cl.has_method("__setstate__") and not cl.has_method("__getstate__"): + emitter.emit_lines( + '{"__setstate__", (PyCFunction)CPyPickle_SetState, METH_O, NULL},', + '{"__getstate__", (PyCFunction)CPyPickle_GetState, METH_NOARGS, NULL},', + ) + + emitter.emit_line("{NULL} /* Sentinel */") + emitter.emit_line("};") + + +def generate_side_table_for_class( + cl: ClassIR, name: str, type: str, slots: dict[str, str], emitter: Emitter +) -> str | None: + name = f"{cl.name_prefix(emitter.names)}_{name}" + emitter.emit_line(f"static {type} {name} = {{") + for field, value in slots.items(): + emitter.emit_line(f".{field} = {value},") + emitter.emit_line("};") + return name + + +def generate_getseter_declarations(cl: ClassIR, emitter: Emitter) -> None: + if not cl.is_trait: + for attr in cl.attributes: + emitter.emit_line("static PyObject *") + emitter.emit_line( + "{}({} *self, void *closure);".format( + getter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) + ) + emitter.emit_line("static int") + emitter.emit_line( + "{}({} *self, PyObject *value, void *closure);".format( + setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) + ) + + for prop, (getter, setter) in cl.properties.items(): + if getter.decl.implicit: + continue + + # Generate getter declaration + emitter.emit_line("static PyObject *") + emitter.emit_line( + "{}({} *self, void *closure);".format( + getter_name(cl, prop, emitter.names), cl.struct_name(emitter.names) + ) + ) + + # Generate property setter declaration if a setter exists + if setter: + emitter.emit_line("static int") + emitter.emit_line( + "{}({} *self, PyObject *value, void *closure);".format( + setter_name(cl, prop, emitter.names), cl.struct_name(emitter.names) + ) + ) + + +def generate_getseters_table(cl: ClassIR, name: str, emitter: Emitter) -> None: + emitter.emit_line(f"static PyGetSetDef {name}[] = {{") + if not cl.is_trait: + for attr in cl.attributes: + emitter.emit_line(f'{{"{attr}",') + emitter.emit_line( + " (getter){}, (setter){},".format( + getter_name(cl, attr, emitter.names), setter_name(cl, attr, emitter.names) + ) + ) + emitter.emit_line(" NULL, NULL},") + for prop, (getter, setter) in cl.properties.items(): + if getter.decl.implicit: + continue + + emitter.emit_line(f'{{"{prop}",') + emitter.emit_line(f" (getter){getter_name(cl, prop, emitter.names)},") + + if setter: + emitter.emit_line(f" (setter){setter_name(cl, prop, emitter.names)},") + emitter.emit_line("NULL, NULL},") + else: + emitter.emit_line("NULL, NULL, NULL},") + + if cl.has_dict: + emitter.emit_line('{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},') + + emitter.emit_line("{NULL} /* Sentinel */") + emitter.emit_line("};") + + +def generate_getseters(cl: ClassIR, emitter: Emitter) -> None: + if not cl.is_trait: + for i, (attr, rtype) in enumerate(cl.attributes.items()): + generate_getter(cl, attr, rtype, emitter) + emitter.emit_line("") + generate_setter(cl, attr, rtype, emitter) + if i < len(cl.attributes) - 1: + emitter.emit_line("") + for prop, (getter, setter) in cl.properties.items(): + if getter.decl.implicit: + continue + + rtype = getter.sig.ret_type + emitter.emit_line("") + generate_readonly_getter(cl, prop, rtype, getter, emitter) + if setter: + arg_type = setter.sig.args[1].type + emitter.emit_line("") + generate_property_setter(cl, prop, arg_type, setter, emitter) + + +def generate_getter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> None: + attr_field = emitter.attr(attr) + emitter.emit_line("static PyObject *") + emitter.emit_line( + "{}({} *self, void *closure)".format( + getter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) + ) + emitter.emit_line("{") + attr_expr = f"self->{attr_field}" + + # HACK: Don't consider refcounted values as always defined, since it's possible to + # access uninitialized values via 'gc.get_objects()'. Accessing non-refcounted + # values is benign. + always_defined = cl.is_always_defined(attr) and not rtype.is_refcounted + + if not always_defined: + emitter.emit_undefined_attr_check(rtype, attr_expr, "==", "self", attr, cl, unlikely=True) + emitter.emit_line("PyErr_SetString(PyExc_AttributeError,") + emitter.emit_line(f' "attribute {repr(attr)} of {repr(cl.name)} undefined");') + emitter.emit_line("return NULL;") + emitter.emit_line("}") + emitter.emit_inc_ref(f"self->{attr_field}", rtype) + emitter.emit_box(f"self->{attr_field}", "retval", rtype, declare_dest=True) + emitter.emit_line("return retval;") + emitter.emit_line("}") + + +def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> None: + attr_field = emitter.attr(attr) + emitter.emit_line("static int") + emitter.emit_line( + "{}({} *self, PyObject *value, void *closure)".format( + setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) + ) + emitter.emit_line("{") + + deletable = cl.is_deletable(attr) + if not deletable: + emitter.emit_line("if (value == NULL) {") + emitter.emit_line("PyErr_SetString(PyExc_AttributeError,") + emitter.emit_line( + f' "{repr(cl.name)} object attribute {repr(attr)} cannot be deleted");' + ) + emitter.emit_line("return -1;") + emitter.emit_line("}") + + # HACK: Don't consider refcounted values as always defined, since it's possible to + # access uninitialized values via 'gc.get_objects()'. Accessing non-refcounted + # values is benign. + always_defined = cl.is_always_defined(attr) and not rtype.is_refcounted + + if rtype.is_refcounted: + attr_expr = f"self->{attr_field}" + if not always_defined: + emitter.emit_undefined_attr_check(rtype, attr_expr, "!=", "self", attr, cl) + emitter.emit_dec_ref(f"self->{attr_field}", rtype) + if not always_defined: + emitter.emit_line("}") + + if deletable: + emitter.emit_line("if (value != NULL) {") + + if rtype.is_unboxed: + emitter.emit_unbox("value", "tmp", rtype, error=ReturnHandler("-1"), declare_dest=True) + elif is_same_type(rtype, object_rprimitive): + emitter.emit_line("PyObject *tmp = value;") + else: + emitter.emit_cast("value", "tmp", rtype, declare_dest=True) + emitter.emit_lines("if (!tmp)", " return -1;") + emitter.emit_inc_ref("tmp", rtype) + emitter.emit_line(f"self->{attr_field} = tmp;") + if rtype.error_overlap and not always_defined: + emitter.emit_attr_bitmap_set("tmp", "self", rtype, cl, attr) + + if deletable: + emitter.emit_line("} else") + emitter.emit_line(f" self->{attr_field} = {emitter.c_undefined_value(rtype)};") + if rtype.error_overlap: + emitter.emit_attr_bitmap_clear("self", rtype, cl, attr) + emitter.emit_line("return 0;") + emitter.emit_line("}") + + +def generate_readonly_getter( + cl: ClassIR, attr: str, rtype: RType, func_ir: FuncIR, emitter: Emitter +) -> None: + emitter.emit_line("static PyObject *") + emitter.emit_line( + "{}({} *self, void *closure)".format( + getter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) + ) + emitter.emit_line("{") + if rtype.is_unboxed: + emitter.emit_line( + "{}retval = {}{}((PyObject *) self);".format( + emitter.ctype_spaced(rtype), NATIVE_PREFIX, func_ir.cname(emitter.names) + ) + ) + emitter.emit_error_check("retval", rtype, "return NULL;") + emitter.emit_box("retval", "retbox", rtype, declare_dest=True) + emitter.emit_line("return retbox;") + else: + emitter.emit_line( + f"return {NATIVE_PREFIX}{func_ir.cname(emitter.names)}((PyObject *) self);" + ) + emitter.emit_line("}") + + +def generate_property_setter( + cl: ClassIR, attr: str, arg_type: RType, func_ir: FuncIR, emitter: Emitter +) -> None: + emitter.emit_line("static int") + emitter.emit_line( + "{}({} *self, PyObject *value, void *closure)".format( + setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) + ) + emitter.emit_line("{") + if arg_type.is_unboxed: + emitter.emit_unbox("value", "tmp", arg_type, error=ReturnHandler("-1"), declare_dest=True) + emitter.emit_line( + f"{NATIVE_PREFIX}{func_ir.cname(emitter.names)}((PyObject *) self, tmp);" + ) + else: + emitter.emit_line( + f"{NATIVE_PREFIX}{func_ir.cname(emitter.names)}((PyObject *) self, value);" + ) + emitter.emit_line("return 0;") + emitter.emit_line("}") + + +def has_managed_dict(cl: ClassIR, emitter: Emitter) -> bool: + """Should the class get the Py_TPFLAGS_MANAGED_DICT flag?""" + # On 3.11 and earlier the flag doesn't exist and we use + # tp_dictoffset instead. If a class inherits from Exception, the + # flag conflicts with tp_dictoffset set in the base class. + return ( + emitter.capi_version >= (3, 12) + and cl.has_dict + and cl.builtin_base != "PyBaseExceptionObject" + ) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py new file mode 100644 index 000000000000..6088fb06dd32 --- /dev/null +++ b/mypyc/codegen/emitfunc.py @@ -0,0 +1,867 @@ +"""Code generation for native function bodies.""" + +from __future__ import annotations + +from typing import Final + +from mypyc.analysis.blockfreq import frequently_executed_blocks +from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer +from mypyc.common import ( + MODULE_PREFIX, + NATIVE_PREFIX, + REG_PREFIX, + STATIC_PREFIX, + TYPE_PREFIX, + TYPE_VAR_PREFIX, +) +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values +from mypyc.ir.ops import ( + ERR_FALSE, + NAMESPACE_MODULE, + NAMESPACE_STATIC, + NAMESPACE_TYPE, + NAMESPACE_TYPE_VAR, + Assign, + AssignMulti, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + ControlOp, + DecRef, + Extend, + Float, + FloatComparisonOp, + FloatNeg, + FloatOp, + GetAttr, + GetElementPtr, + Goto, + IncRef, + InitStatic, + Integer, + IntOp, + KeepAlive, + LoadAddress, + LoadErrorValue, + LoadGlobal, + LoadLiteral, + LoadMem, + LoadStatic, + MethodCall, + Op, + OpVisitor, + PrimitiveOp, + RaiseStandardError, + Register, + Return, + SetAttr, + SetMem, + Truncate, + TupleGet, + TupleSet, + Unborrow, + Unbox, + Unreachable, + Value, +) +from mypyc.ir.pprint import generate_names_for_ir +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RStruct, + RTuple, + RType, + is_int32_rprimitive, + is_int64_rprimitive, + is_int_rprimitive, + is_pointer_rprimitive, + is_tagged, +) + + +def native_function_type(fn: FuncIR, emitter: Emitter) -> str: + args = ", ".join(emitter.ctype(arg.type) for arg in fn.args) or "void" + ret = emitter.ctype(fn.ret_type) + return f"{ret} (*)({args})" + + +def native_function_header(fn: FuncDecl, emitter: Emitter) -> str: + args = [] + for arg in fn.sig.args: + args.append(f"{emitter.ctype_spaced(arg.type)}{REG_PREFIX}{arg.name}") + + return "{ret_type}{name}({args})".format( + ret_type=emitter.ctype_spaced(fn.sig.ret_type), + name=emitter.native_function_name(fn), + args=", ".join(args) or "void", + ) + + +def generate_native_function( + fn: FuncIR, emitter: Emitter, source_path: str, module_name: str +) -> None: + declarations = Emitter(emitter.context) + names = generate_names_for_ir(fn.arg_regs, fn.blocks) + body = Emitter(emitter.context, names) + visitor = FunctionEmitterVisitor(body, declarations, source_path, module_name) + + declarations.emit_line(f"{native_function_header(fn.decl, emitter)} {{") + body.indent() + + for r in all_values(fn.arg_regs, fn.blocks): + if isinstance(r.type, RTuple): + emitter.declare_tuple_struct(r.type) + if isinstance(r.type, RArray): + continue # Special: declared on first assignment + + if r in fn.arg_regs: + continue # Skip the arguments + + ctype = emitter.ctype_spaced(r.type) + init = "" + declarations.emit_line( + "{ctype}{prefix}{name}{init};".format( + ctype=ctype, prefix=REG_PREFIX, name=names[r], init=init + ) + ) + + # Before we emit the blocks, give them all labels + blocks = fn.blocks + for i, block in enumerate(blocks): + block.label = i + + # Find blocks that are never jumped to or are only jumped to from the + # block directly above it. This allows for more labels and gotos to be + # eliminated during code generation. + for block in fn.blocks: + terminator = block.terminator + assert isinstance(terminator, ControlOp) + + for target in terminator.targets(): + is_next_block = target.label == block.label + 1 + + # Always emit labels for GetAttr error checks since the emit code that + # generates them will add instructions between the branch and the + # next label, causing the label to be wrongly removed. A better + # solution would be to change the IR so that it adds a basic block + # inbetween the calls. + is_problematic_op = isinstance(terminator, Branch) and any( + isinstance(s, GetAttr) for s in terminator.sources() + ) + + if not is_next_block or is_problematic_op: + fn.blocks[target.label].referenced = True + + common = frequently_executed_blocks(fn.blocks[0]) + + for i in range(len(blocks)): + block = blocks[i] + visitor.rare = block not in common + next_block = None + if i + 1 < len(blocks): + next_block = blocks[i + 1] + body.emit_label(block) + visitor.next_block = next_block + + ops = block.ops + visitor.ops = ops + visitor.op_index = 0 + while visitor.op_index < len(ops): + ops[visitor.op_index].accept(visitor) + visitor.op_index += 1 + + body.emit_line("}") + + emitter.emit_from_emitter(declarations) + emitter.emit_from_emitter(body) + + +class FunctionEmitterVisitor(OpVisitor[None]): + def __init__( + self, emitter: Emitter, declarations: Emitter, source_path: str, module_name: str + ) -> None: + self.emitter = emitter + self.names = emitter.names + self.declarations = declarations + self.source_path = source_path + self.module_name = module_name + self.literals = emitter.context.literals + self.rare = False + # Next basic block to be processed after the current one (if any), set by caller + self.next_block: BasicBlock | None = None + # Ops in the basic block currently being processed, set by caller + self.ops: list[Op] = [] + # Current index within ops; visit methods can increment this to skip/merge ops + self.op_index = 0 + + def temp_name(self) -> str: + return self.emitter.temp_name() + + def visit_goto(self, op: Goto) -> None: + if op.label is not self.next_block: + self.emit_line("goto %s;" % self.label(op.label)) + + def visit_branch(self, op: Branch) -> None: + true, false = op.true, op.false + negated = op.negated + negated_rare = False + if true is self.next_block and op.traceback_entry is None: + # Switch true/false since it avoids an else block. + true, false = false, true + negated = not negated + negated_rare = True + + neg = "!" if negated else "" + cond = "" + if op.op == Branch.BOOL: + expr_result = self.reg(op.value) + cond = f"{neg}{expr_result}" + elif op.op == Branch.IS_ERROR: + typ = op.value.type + compare = "!=" if negated else "==" + if isinstance(typ, RTuple): + # TODO: What about empty tuple? + cond = self.emitter.tuple_undefined_check_cond( + typ, self.reg(op.value), self.c_error_value, compare + ) + else: + cond = f"{self.reg(op.value)} {compare} {self.c_error_value(typ)}" + else: + assert False, "Invalid branch" + + # For error checks, tell the compiler the branch is unlikely + if op.traceback_entry is not None or op.rare: + if not negated_rare: + cond = f"unlikely({cond})" + else: + cond = f"likely({cond})" + + if false is self.next_block: + if op.traceback_entry is None: + if true is not self.next_block: + self.emit_line(f"if ({cond}) goto {self.label(true)};") + else: + self.emit_line(f"if ({cond}) {{") + self.emit_traceback(op) + self.emit_lines("goto %s;" % self.label(true), "}") + else: + self.emit_line(f"if ({cond}) {{") + self.emit_traceback(op) + + if true is not self.next_block: + self.emit_line("goto %s;" % self.label(true)) + + self.emit_lines("} else", " goto %s;" % self.label(false)) + + def visit_return(self, op: Return) -> None: + value_str = self.reg(op.value) + self.emit_line("return %s;" % value_str) + + def visit_tuple_set(self, op: TupleSet) -> None: + dest = self.reg(op) + tuple_type = op.tuple_type + self.emitter.declare_tuple_struct(tuple_type) + if len(op.items) == 0: # empty tuple + self.emit_line(f"{dest}.empty_struct_error_flag = 0;") + else: + for i, item in enumerate(op.items): + self.emit_line(f"{dest}.f{i} = {self.reg(item)};") + + def visit_assign(self, op: Assign) -> None: + dest = self.reg(op.dest) + src = self.reg(op.src) + # clang whines about self assignment (which we might generate + # for some casts), so don't emit it. + if dest != src: + # We sometimes assign from an integer prepresentation of a pointer + # to a real pointer, and C compilers insist on a cast. + if op.src.type.is_unboxed and not op.dest.type.is_unboxed: + src = f"(void *){src}" + self.emit_line(f"{dest} = {src};") + + def visit_assign_multi(self, op: AssignMulti) -> None: + typ = op.dest.type + assert isinstance(typ, RArray) + dest = self.reg(op.dest) + # RArray values can only be assigned to once, so we can always + # declare them on initialization. + self.emit_line( + "%s%s[%d] = %s;" + % ( + self.emitter.ctype_spaced(typ.item_type), + dest, + len(op.src), + c_array_initializer([self.reg(s) for s in op.src], indented=True), + ) + ) + + def visit_load_error_value(self, op: LoadErrorValue) -> None: + if isinstance(op.type, RTuple): + values = [self.c_undefined_value(item) for item in op.type.types] + tmp = self.temp_name() + self.emit_line("{} {} = {{ {} }};".format(self.ctype(op.type), tmp, ", ".join(values))) + self.emit_line(f"{self.reg(op)} = {tmp};") + else: + self.emit_line(f"{self.reg(op)} = {self.c_error_value(op.type)};") + + def visit_load_literal(self, op: LoadLiteral) -> None: + index = self.literals.literal_index(op.value) + if not is_int_rprimitive(op.type): + self.emit_line("%s = CPyStatics[%d];" % (self.reg(op), index), ann=op.value) + else: + self.emit_line( + "%s = (CPyTagged)CPyStatics[%d] | 1;" % (self.reg(op), index), ann=op.value + ) + + def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> str: + """Generate attribute accessor for normal (non-property) access. + + This either has a form like obj->attr_name for attributes defined in non-trait + classes, and *(obj + attr_offset) for attributes defined by traits. We also + insert all necessary C casts here. + """ + cast = f"({op.class_type.struct_name(self.emitter.names)} *)" + if decl_cl.is_trait and op.class_type.class_ir.is_trait: + # For pure trait access find the offset first, offsets + # are ordered by attribute position in the cl.attributes dict. + # TODO: pre-calculate the mapping to make this faster. + trait_attr_index = list(decl_cl.attributes).index(op.attr) + # TODO: reuse these names somehow? + offset = self.emitter.temp_name() + self.declarations.emit_line(f"size_t {offset};") + self.emitter.emit_line( + "{} = {};".format( + offset, + "CPy_FindAttrOffset({}, {}, {})".format( + self.emitter.type_struct_name(decl_cl), + f"({cast}{obj})->vtable", + trait_attr_index, + ), + ) + ) + attr_cast = f"({self.ctype(op.class_type.attr_type(op.attr))} *)" + return f"*{attr_cast}((char *){obj} + {offset})" + else: + # Cast to something non-trait. Note: for this to work, all struct + # members for non-trait classes must obey monotonic linear growth. + if op.class_type.class_ir.is_trait: + assert not decl_cl.is_trait + cast = f"({decl_cl.struct_name(self.emitter.names)} *)" + return f"({cast}{obj})->{self.emitter.attr(op.attr)}" + + def visit_get_attr(self, op: GetAttr) -> None: + dest = self.reg(op) + obj = self.reg(op.obj) + rtype = op.class_type + cl = rtype.class_ir + attr_rtype, decl_cl = cl.attr_details(op.attr) + prefer_method = cl.is_trait and attr_rtype.error_overlap + if cl.get_method(op.attr, prefer_method=prefer_method): + # Properties are essentially methods, so use vtable access for them. + if cl.is_method_final(op.attr): + self.emit_method_call(f"{dest} = ", op.obj, op.attr, []) + else: + version = "_TRAIT" if cl.is_trait else "" + self.emit_line( + "%s = CPY_GET_ATTR%s(%s, %s, %d, %s, %s); /* %s */" + % ( + dest, + version, + obj, + self.emitter.type_struct_name(rtype.class_ir), + rtype.getter_index(op.attr), + rtype.struct_name(self.names), + self.ctype(rtype.attr_type(op.attr)), + op.attr, + ) + ) + else: + # Otherwise, use direct or offset struct access. + attr_expr = self.get_attr_expr(obj, op, decl_cl) + self.emitter.emit_line(f"{dest} = {attr_expr};") + always_defined = cl.is_always_defined(op.attr) + merged_branch = None + if not always_defined: + self.emitter.emit_undefined_attr_check( + attr_rtype, dest, "==", obj, op.attr, cl, unlikely=True + ) + branch = self.next_branch() + if branch is not None: + if ( + branch.value is op + and branch.op == Branch.IS_ERROR + and branch.traceback_entry is not None + and not branch.negated + ): + # Generate code for the following branch here to avoid + # redundant branches in the generated code. + self.emit_attribute_error(branch, cl.name, op.attr) + self.emit_line("goto %s;" % self.label(branch.true)) + merged_branch = branch + self.emitter.emit_line("}") + if not merged_branch: + exc_class = "PyExc_AttributeError" + self.emitter.emit_line( + 'PyErr_SetString({}, "attribute {} of {} undefined");'.format( + exc_class, repr(op.attr), repr(cl.name) + ) + ) + + if attr_rtype.is_refcounted and not op.is_borrowed: + if not merged_branch and not always_defined: + self.emitter.emit_line("} else {") + self.emitter.emit_inc_ref(dest, attr_rtype) + if merged_branch: + if merged_branch.false is not self.next_block: + self.emit_line("goto %s;" % self.label(merged_branch.false)) + self.op_index += 1 + elif not always_defined: + self.emitter.emit_line("}") + + def next_branch(self) -> Branch | None: + if self.op_index + 1 < len(self.ops): + next_op = self.ops[self.op_index + 1] + if isinstance(next_op, Branch): + return next_op + return None + + def visit_set_attr(self, op: SetAttr) -> None: + if op.error_kind == ERR_FALSE: + dest = self.reg(op) + obj = self.reg(op.obj) + src = self.reg(op.src) + rtype = op.class_type + cl = rtype.class_ir + attr_rtype, decl_cl = cl.attr_details(op.attr) + if cl.get_method(op.attr): + # Again, use vtable access for properties... + assert not op.is_init and op.error_kind == ERR_FALSE, "%s %d %d %s" % ( + op.attr, + op.is_init, + op.error_kind, + rtype, + ) + version = "_TRAIT" if cl.is_trait else "" + self.emit_line( + "%s = CPY_SET_ATTR%s(%s, %s, %d, %s, %s, %s); /* %s */" + % ( + dest, + version, + obj, + self.emitter.type_struct_name(rtype.class_ir), + rtype.setter_index(op.attr), + src, + rtype.struct_name(self.names), + self.ctype(rtype.attr_type(op.attr)), + op.attr, + ) + ) + else: + # ...and struct access for normal attributes. + attr_expr = self.get_attr_expr(obj, op, decl_cl) + if not op.is_init and attr_rtype.is_refcounted: + # This is not an initialization (where we know that the attribute was + # previously undefined), so decref the old value. + always_defined = cl.is_always_defined(op.attr) + if not always_defined: + self.emitter.emit_undefined_attr_check( + attr_rtype, attr_expr, "!=", obj, op.attr, cl + ) + self.emitter.emit_dec_ref(attr_expr, attr_rtype) + if not always_defined: + self.emitter.emit_line("}") + elif attr_rtype.error_overlap and not cl.is_always_defined(op.attr): + # If there is overlap with the error value, update bitmap to mark + # attribute as defined. + self.emitter.emit_attr_bitmap_set(src, obj, attr_rtype, cl, op.attr) + + # This steals the reference to src, so we don't need to increment the arg + self.emitter.emit_line(f"{attr_expr} = {src};") + if op.error_kind == ERR_FALSE: + self.emitter.emit_line(f"{dest} = 1;") + + PREFIX_MAP: Final = { + NAMESPACE_STATIC: STATIC_PREFIX, + NAMESPACE_TYPE: TYPE_PREFIX, + NAMESPACE_MODULE: MODULE_PREFIX, + NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX, + } + + def visit_load_static(self, op: LoadStatic) -> None: + dest = self.reg(op) + prefix = self.PREFIX_MAP[op.namespace] + name = self.emitter.static_name(op.identifier, op.module_name, prefix) + if op.namespace == NAMESPACE_TYPE: + name = "(PyObject *)%s" % name + self.emit_line(f"{dest} = {name};", ann=op.ann) + + def visit_init_static(self, op: InitStatic) -> None: + value = self.reg(op.value) + prefix = self.PREFIX_MAP[op.namespace] + name = self.emitter.static_name(op.identifier, op.module_name, prefix) + if op.namespace == NAMESPACE_TYPE: + value = "(PyTypeObject *)%s" % value + self.emit_line(f"{name} = {value};") + self.emit_inc_ref(name, op.value.type) + + def visit_tuple_get(self, op: TupleGet) -> None: + dest = self.reg(op) + src = self.reg(op.src) + self.emit_line(f"{dest} = {src}.f{op.index};") + if not op.is_borrowed: + self.emit_inc_ref(dest, op.type) + + def get_dest_assign(self, dest: Value) -> str: + if not dest.is_void: + return self.reg(dest) + " = " + else: + return "" + + def visit_call(self, op: Call) -> None: + """Call native function.""" + dest = self.get_dest_assign(op) + args = ", ".join(self.reg(arg) for arg in op.args) + lib = self.emitter.get_group_prefix(op.fn) + cname = op.fn.cname(self.names) + self.emit_line(f"{dest}{lib}{NATIVE_PREFIX}{cname}({args});") + + def visit_method_call(self, op: MethodCall) -> None: + """Call native method.""" + dest = self.get_dest_assign(op) + self.emit_method_call(dest, op.obj, op.method, op.args) + + def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Value]) -> None: + obj = self.reg(op_obj) + rtype = op_obj.type + assert isinstance(rtype, RInstance) + class_ir = rtype.class_ir + method = rtype.class_ir.get_method(name) + assert method is not None + + # Can we call the method directly, bypassing vtable? + is_direct = class_ir.is_method_final(name) + + # The first argument gets omitted for static methods and + # turned into the class for class methods + obj_args = ( + [] + if method.decl.kind == FUNC_STATICMETHOD + else [f"(PyObject *)Py_TYPE({obj})"] if method.decl.kind == FUNC_CLASSMETHOD else [obj] + ) + args = ", ".join(obj_args + [self.reg(arg) for arg in op_args]) + mtype = native_function_type(method, self.emitter) + version = "_TRAIT" if rtype.class_ir.is_trait else "" + if is_direct: + # Directly call method, without going through the vtable. + lib = self.emitter.get_group_prefix(method.decl) + self.emit_line(f"{dest}{lib}{NATIVE_PREFIX}{method.cname(self.names)}({args});") + else: + # Call using vtable. + method_idx = rtype.method_index(name) + self.emit_line( + "{}CPY_GET_METHOD{}({}, {}, {}, {}, {})({}); /* {} */".format( + dest, + version, + obj, + self.emitter.type_struct_name(rtype.class_ir), + method_idx, + rtype.struct_name(self.names), + mtype, + args, + name, + ) + ) + + def visit_inc_ref(self, op: IncRef) -> None: + src = self.reg(op.src) + self.emit_inc_ref(src, op.src.type) + + def visit_dec_ref(self, op: DecRef) -> None: + src = self.reg(op.src) + self.emit_dec_ref(src, op.src.type, is_xdec=op.is_xdec) + + def visit_box(self, op: Box) -> None: + self.emitter.emit_box(self.reg(op.src), self.reg(op), op.src.type, can_borrow=True) + + def visit_cast(self, op: Cast) -> None: + branch = self.next_branch() + handler = None + if branch is not None: + if ( + branch.value is op + and branch.op == Branch.IS_ERROR + and branch.traceback_entry is not None + and not branch.negated + and branch.false is self.next_block + ): + # Generate code also for the following branch here to avoid + # redundant branches in the generated code. + handler = TracebackAndGotoHandler( + self.label(branch.true), + self.source_path, + self.module_name, + branch.traceback_entry, + ) + self.op_index += 1 + + self.emitter.emit_cast( + self.reg(op.src), self.reg(op), op.type, src_type=op.src.type, error=handler + ) + + def visit_unbox(self, op: Unbox) -> None: + self.emitter.emit_unbox(self.reg(op.src), self.reg(op), op.type) + + def visit_unreachable(self, op: Unreachable) -> None: + self.emitter.emit_line("CPy_Unreachable();") + + def visit_raise_standard_error(self, op: RaiseStandardError) -> None: + # TODO: Better escaping of backspaces and such + if op.value is not None: + if isinstance(op.value, str): + message = op.value.replace('"', '\\"') + self.emitter.emit_line(f'PyErr_SetString(PyExc_{op.class_name}, "{message}");') + elif isinstance(op.value, Value): + self.emitter.emit_line( + "PyErr_SetObject(PyExc_{}, {});".format( + op.class_name, self.emitter.reg(op.value) + ) + ) + else: + assert False, "op value type must be either str or Value" + else: + self.emitter.emit_line(f"PyErr_SetNone(PyExc_{op.class_name});") + self.emitter.emit_line(f"{self.reg(op)} = 0;") + + def visit_call_c(self, op: CallC) -> None: + if op.is_void: + dest = "" + else: + dest = self.get_dest_assign(op) + args = ", ".join(self.reg(arg) for arg in op.args) + self.emitter.emit_line(f"{dest}{op.function_name}({args});") + + def visit_primitive_op(self, op: PrimitiveOp) -> None: + raise RuntimeError( + f"unexpected PrimitiveOp {op.desc.name}: they must be lowered before codegen" + ) + + def visit_truncate(self, op: Truncate) -> None: + dest = self.reg(op) + value = self.reg(op.src) + # for C backend the generated code are straight assignments + self.emit_line(f"{dest} = {value};") + + def visit_extend(self, op: Extend) -> None: + dest = self.reg(op) + value = self.reg(op.src) + if op.signed: + src_cast = self.emit_signed_int_cast(op.src.type) + else: + src_cast = self.emit_unsigned_int_cast(op.src.type) + self.emit_line(f"{dest} = {src_cast}{value};") + + def visit_load_global(self, op: LoadGlobal) -> None: + dest = self.reg(op) + self.emit_line(f"{dest} = {op.identifier};", ann=op.ann) + + def visit_int_op(self, op: IntOp) -> None: + dest = self.reg(op) + lhs = self.reg(op.lhs) + rhs = self.reg(op.rhs) + if op.op == IntOp.RIGHT_SHIFT: + # Signed right shift + lhs = self.emit_signed_int_cast(op.lhs.type) + lhs + rhs = self.emit_signed_int_cast(op.rhs.type) + rhs + self.emit_line(f"{dest} = {lhs} {op.op_str[op.op]} {rhs};") + + def visit_comparison_op(self, op: ComparisonOp) -> None: + dest = self.reg(op) + lhs = self.reg(op.lhs) + rhs = self.reg(op.rhs) + lhs_cast = "" + rhs_cast = "" + if op.op in (ComparisonOp.SLT, ComparisonOp.SGT, ComparisonOp.SLE, ComparisonOp.SGE): + # Always signed comparison op + lhs_cast = self.emit_signed_int_cast(op.lhs.type) + rhs_cast = self.emit_signed_int_cast(op.rhs.type) + elif op.op in (ComparisonOp.ULT, ComparisonOp.UGT, ComparisonOp.ULE, ComparisonOp.UGE): + # Always unsigned comparison op + lhs_cast = self.emit_unsigned_int_cast(op.lhs.type) + rhs_cast = self.emit_unsigned_int_cast(op.rhs.type) + elif isinstance(op.lhs, Integer) and op.lhs.value < 0: + # Force signed ==/!= with negative operand + rhs_cast = self.emit_signed_int_cast(op.rhs.type) + elif isinstance(op.rhs, Integer) and op.rhs.value < 0: + # Force signed ==/!= with negative operand + lhs_cast = self.emit_signed_int_cast(op.lhs.type) + self.emit_line(f"{dest} = {lhs_cast}{lhs} {op.op_str[op.op]} {rhs_cast}{rhs};") + + def visit_float_op(self, op: FloatOp) -> None: + dest = self.reg(op) + lhs = self.reg(op.lhs) + rhs = self.reg(op.rhs) + if op.op != FloatOp.MOD: + self.emit_line(f"{dest} = {lhs} {op.op_str[op.op]} {rhs};") + else: + # TODO: This may set errno as a side effect, that is a little sketchy. + self.emit_line(f"{dest} = fmod({lhs}, {rhs});") + + def visit_float_neg(self, op: FloatNeg) -> None: + dest = self.reg(op) + src = self.reg(op.src) + self.emit_line(f"{dest} = -{src};") + + def visit_float_comparison_op(self, op: FloatComparisonOp) -> None: + dest = self.reg(op) + lhs = self.reg(op.lhs) + rhs = self.reg(op.rhs) + self.emit_line(f"{dest} = {lhs} {op.op_str[op.op]} {rhs};") + + def visit_load_mem(self, op: LoadMem) -> None: + dest = self.reg(op) + src = self.reg(op.src) + # TODO: we shouldn't dereference to type that are pointer type so far + type = self.ctype(op.type) + self.emit_line(f"{dest} = *({type} *){src};") + + def visit_set_mem(self, op: SetMem) -> None: + dest = self.reg(op.dest) + src = self.reg(op.src) + dest_type = self.ctype(op.dest_type) + # clang whines about self assignment (which we might generate + # for some casts), so don't emit it. + if dest != src: + self.emit_line(f"*({dest_type} *){dest} = {src};") + + def visit_get_element_ptr(self, op: GetElementPtr) -> None: + dest = self.reg(op) + src = self.reg(op.src) + # TODO: support tuple type + assert isinstance(op.src_type, RStruct) + assert op.field in op.src_type.names, "Invalid field name." + self.emit_line( + "{} = ({})&(({} *){})->{};".format( + dest, op.type._ctype, op.src_type.name, src, op.field + ) + ) + + def visit_load_address(self, op: LoadAddress) -> None: + typ = op.type + dest = self.reg(op) + if isinstance(op.src, Register): + src = self.reg(op.src) + elif isinstance(op.src, LoadStatic): + prefix = self.PREFIX_MAP[op.src.namespace] + src = self.emitter.static_name(op.src.identifier, op.src.module_name, prefix) + else: + src = op.src + self.emit_line(f"{dest} = ({typ._ctype})&{src};") + + def visit_keep_alive(self, op: KeepAlive) -> None: + # This is a no-op. + pass + + def visit_unborrow(self, op: Unborrow) -> None: + # This is a no-op that propagates the source value. + dest = self.reg(op) + src = self.reg(op.src) + self.emit_line(f"{dest} = {src};") + + # Helpers + + def label(self, label: BasicBlock) -> str: + return self.emitter.label(label) + + def reg(self, reg: Value) -> str: + if isinstance(reg, Integer): + val = reg.value + if val == 0 and is_pointer_rprimitive(reg.type): + return "NULL" + s = str(val) + if val >= (1 << 31): + # Avoid overflowing signed 32-bit int + if val >= (1 << 63): + s += "ULL" + else: + s += "LL" + elif val == -(1 << 63): + # Avoid overflowing C integer literal + s = "(-9223372036854775807LL - 1)" + elif val <= -(1 << 31): + s += "LL" + return s + elif isinstance(reg, Float): + r = repr(reg.value) + if r == "inf": + return "INFINITY" + elif r == "-inf": + return "-INFINITY" + elif r == "nan": + return "NAN" + return r + else: + return self.emitter.reg(reg) + + def ctype(self, rtype: RType) -> str: + return self.emitter.ctype(rtype) + + def c_error_value(self, rtype: RType) -> str: + return self.emitter.c_error_value(rtype) + + def c_undefined_value(self, rtype: RType) -> str: + return self.emitter.c_undefined_value(rtype) + + def emit_line(self, line: str, *, ann: object = None) -> None: + self.emitter.emit_line(line, ann=ann) + + def emit_lines(self, *lines: str) -> None: + self.emitter.emit_lines(*lines) + + def emit_inc_ref(self, dest: str, rtype: RType) -> None: + self.emitter.emit_inc_ref(dest, rtype, rare=self.rare) + + def emit_dec_ref(self, dest: str, rtype: RType, is_xdec: bool) -> None: + self.emitter.emit_dec_ref(dest, rtype, is_xdec=is_xdec, rare=self.rare) + + def emit_declaration(self, line: str) -> None: + self.declarations.emit_line(line) + + def emit_traceback(self, op: Branch) -> None: + if op.traceback_entry is not None: + self.emitter.emit_traceback(self.source_path, self.module_name, op.traceback_entry) + + def emit_attribute_error(self, op: Branch, class_name: str, attr: str) -> None: + assert op.traceback_entry is not None + globals_static = self.emitter.static_name("globals", self.module_name) + self.emit_line( + 'CPy_AttributeError("%s", "%s", "%s", "%s", %d, %s);' + % ( + self.source_path.replace("\\", "\\\\"), + op.traceback_entry[0], + class_name, + attr, + op.traceback_entry[1], + globals_static, + ) + ) + if DEBUG_ERRORS: + self.emit_line('assert(PyErr_Occurred() != NULL && "failure w/o err!");') + + def emit_signed_int_cast(self, type: RType) -> str: + if is_tagged(type): + return "(Py_ssize_t)" + else: + return "" + + def emit_unsigned_int_cast(self, type: RType) -> str: + if is_int32_rprimitive(type): + return "(uint32_t)" + elif is_int64_rprimitive(type): + return "(uint64_t)" + else: + return "" diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py new file mode 100644 index 000000000000..5b2812c2293a --- /dev/null +++ b/mypyc/codegen/emitmodule.py @@ -0,0 +1,1132 @@ +"""Generate C code for a Python C extension module from Python source code.""" + +# FIXME: Basically nothing in this file operates on the level of a +# single module and it should be renamed. + +from __future__ import annotations + +import json +import os +from typing import Iterable, List, Optional, Tuple, TypeVar + +from mypy.build import ( + BuildResult, + BuildSource, + State, + build, + compute_hash, + create_metastore, + get_cache_names, + sorted_components, +) +from mypy.errors import CompileError +from mypy.fscache import FileSystemCache +from mypy.nodes import MypyFile +from mypy.options import Options +from mypy.plugin import Plugin, ReportConfigContext +from mypy.util import hash_digest, json_dumps +from mypyc.codegen.cstring import c_string_initializer +from mypyc.codegen.emit import Emitter, EmitterContext, HeaderDeclaration, c_array_initializer +from mypyc.codegen.emitclass import generate_class, generate_class_type_decl +from mypyc.codegen.emitfunc import generate_native_function, native_function_header +from mypyc.codegen.emitwrapper import ( + generate_legacy_wrapper_function, + generate_wrapper_function, + legacy_wrapper_function_header, + wrapper_function_header, +) +from mypyc.codegen.literals import Literals +from mypyc.common import ( + MODULE_PREFIX, + PREFIX, + RUNTIME_C_FILES, + TOP_LEVEL_NAME, + TYPE_VAR_PREFIX, + shared_lib_name, + short_id_from_name, + use_vectorcall, +) +from mypyc.errors import Errors +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.module_ir import ModuleIR, ModuleIRs, deserialize_modules +from mypyc.ir.ops import DeserMaps, LoadLiteral +from mypyc.ir.rtypes import RType +from mypyc.irbuild.main import build_ir +from mypyc.irbuild.mapper import Mapper +from mypyc.irbuild.prepare import load_type_map +from mypyc.namegen import NameGenerator, exported_name +from mypyc.options import CompilerOptions +from mypyc.transform.copy_propagation import do_copy_propagation +from mypyc.transform.exceptions import insert_exception_handling +from mypyc.transform.flag_elimination import do_flag_elimination +from mypyc.transform.lower import lower_ir +from mypyc.transform.refcount import insert_ref_count_opcodes +from mypyc.transform.uninit import insert_uninit_checks + +# All of the modules being compiled are divided into "groups". A group +# is a set of modules that are placed into the same shared library. +# Two common configurations are that every module is placed in a group +# by itself (fully separate compilation) and that every module is +# placed in the same group (fully whole-program compilation), but we +# support finer-grained control of the group as well. +# +# In fully whole-program compilation, we will generate N+1 extension +# modules: one shim per module and one shared library containing all +# the actual code. +# In fully separate compilation, we (unfortunately) will generate 2*N +# extension modules: one shim per module and also one library containing +# each module's actual code. (This might be fixable in the future, +# but allows a clean separation between setup of the export tables +# (see generate_export_table) and running module top levels.) +# +# A group is represented as a list of BuildSources containing all of +# its modules along with the name of the group. (Which can be None +# only if we are compiling only a single group with a single file in it +# and not using shared libraries). +Group = Tuple[List[BuildSource], Optional[str]] +Groups = List[Group] + +# A list of (file name, file contents) pairs. +FileContents = List[Tuple[str, str]] + + +class MarkedDeclaration: + """Add a mark, useful for topological sort.""" + + def __init__(self, declaration: HeaderDeclaration, mark: bool) -> None: + self.declaration = declaration + self.mark = False + + +class MypycPlugin(Plugin): + """Plugin for making mypyc interoperate properly with mypy incremental mode. + + Basically the point of this plugin is to force mypy to recheck things + based on the demands of mypyc in a couple situations: + * Any modules in the same group must be compiled together, so we + tell mypy that modules depend on all their groupmates. + * If the IR metadata is missing or stale or any of the generated + C source files associated missing or stale, then we need to + recompile the module so we mark it as stale. + """ + + def __init__( + self, options: Options, compiler_options: CompilerOptions, groups: Groups + ) -> None: + super().__init__(options) + self.group_map: dict[str, tuple[str | None, list[str]]] = {} + for sources, name in groups: + modules = sorted(source.module for source in sources) + for id in modules: + self.group_map[id] = (name, modules) + + self.compiler_options = compiler_options + self.metastore = create_metastore(options) + + def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list[str]] | None: + # The config data we report is the group map entry for the module. + # If the data is being used to check validity, we do additional checks + # that the IR cache exists and matches the metadata cache and all + # output source files exist and are up to date. + + id, path, is_check = ctx.id, ctx.path, ctx.is_check + + if id not in self.group_map: + return None + + # If we aren't doing validity checks, just return the cache data + if not is_check: + return self.group_map[id] + + # Load the metadata and IR cache + meta_path, _, _ = get_cache_names(id, path, self.options) + ir_path = get_ir_cache_name(id, path, self.options) + try: + meta_json = self.metastore.read(meta_path) + ir_json = self.metastore.read(ir_path) + except FileNotFoundError: + # This could happen if mypyc failed after mypy succeeded + # in the previous run or if some cache files got + # deleted. No big deal, just fail to load the cache. + return None + + ir_data = json.loads(ir_json) + + # Check that the IR cache matches the metadata cache + if hash_digest(meta_json) != ir_data["meta_hash"]: + return None + + # Check that all of the source files are present and as + # expected. The main situation where this would come up is the + # user deleting the build directory without deleting + # .mypy_cache, which we should handle gracefully. + for path, hash in ir_data["src_hashes"].items(): + try: + with open(os.path.join(self.compiler_options.target_dir, path), "rb") as f: + contents = f.read() + except FileNotFoundError: + return None + real_hash = hash_digest(contents) + if hash != real_hash: + return None + + return self.group_map[id] + + def get_additional_deps(self, file: MypyFile) -> list[tuple[int, str, int]]: + # Report dependency on modules in the module's group + return [(10, id, -1) for id in self.group_map.get(file.fullname, (None, []))[1]] + + +def parse_and_typecheck( + sources: list[BuildSource], + options: Options, + compiler_options: CompilerOptions, + groups: Groups, + fscache: FileSystemCache | None = None, + alt_lib_path: str | None = None, +) -> BuildResult: + assert options.strict_optional, "strict_optional must be turned on" + result = build( + sources=sources, + options=options, + alt_lib_path=alt_lib_path, + fscache=fscache, + extra_plugins=[MypycPlugin(options, compiler_options, groups)], + ) + if result.errors: + raise CompileError(result.errors) + return result + + +def compile_scc_to_ir( + scc: list[MypyFile], + result: BuildResult, + mapper: Mapper, + compiler_options: CompilerOptions, + errors: Errors, +) -> ModuleIRs: + """Compile an SCC into ModuleIRs. + + Any modules that this SCC depends on must have either compiled or + loaded from a cache into mapper. + + Arguments: + scc: The list of MypyFiles to compile + result: The BuildResult from the mypy front-end + mapper: The Mapper object mapping mypy ASTs to class and func IRs + compiler_options: The compilation options + errors: Where to report any errors encountered + + Returns the IR of the modules. + """ + + if compiler_options.verbose: + print("Compiling {}".format(", ".join(x.name for x in scc))) + + # Generate basic IR, with missing exception and refcount handling. + modules = build_ir(scc, result.graph, result.types, mapper, compiler_options, errors) + if errors.num_errors > 0: + return modules + + for module in modules.values(): + for fn in module.functions: + # Insert uninit checks. + insert_uninit_checks(fn) + # Insert exception handling. + insert_exception_handling(fn) + # Insert refcount handling. + insert_ref_count_opcodes(fn) + # Switch to lower abstraction level IR. + lower_ir(fn, compiler_options) + # Perform optimizations. + do_copy_propagation(fn, compiler_options) + do_flag_elimination(fn, compiler_options) + + return modules + + +def compile_modules_to_ir( + result: BuildResult, mapper: Mapper, compiler_options: CompilerOptions, errors: Errors +) -> ModuleIRs: + """Compile a collection of modules into ModuleIRs. + + The modules to compile are specified as part of mapper's group_map. + + Returns the IR of the modules. + """ + deser_ctx = DeserMaps({}, {}) + modules = {} + + # Process the graph by SCC in topological order, like we do in mypy.build + for scc in sorted_components(result.graph): + scc_states = [result.graph[id] for id in scc] + trees = [st.tree for st in scc_states if st.id in mapper.group_map and st.tree] + + if not trees: + continue + + fresh = all(id not in result.manager.rechecked_modules for id in scc) + if fresh: + load_scc_from_cache(trees, result, mapper, deser_ctx) + else: + scc_ir = compile_scc_to_ir(trees, result, mapper, compiler_options, errors) + modules.update(scc_ir) + + return modules + + +def compile_ir_to_c( + groups: Groups, + modules: ModuleIRs, + result: BuildResult, + mapper: Mapper, + compiler_options: CompilerOptions, +) -> dict[str | None, list[tuple[str, str]]]: + """Compile a collection of ModuleIRs to C source text. + + Returns a dictionary mapping group names to a list of (file name, + file text) pairs. + """ + source_paths = { + source.module: result.graph[source.module].xpath + for sources, _ in groups + for source in sources + } + + names = NameGenerator([[source.module for source in sources] for sources, _ in groups]) + + # Generate C code for each compilation group. Each group will be + # compiled into a separate extension module. + ctext: dict[str | None, list[tuple[str, str]]] = {} + for group_sources, group_name in groups: + group_modules = { + source.module: modules[source.module] + for source in group_sources + if source.module in modules + } + if not group_modules: + ctext[group_name] = [] + continue + generator = GroupGenerator( + group_modules, source_paths, group_name, mapper.group_map, names, compiler_options + ) + ctext[group_name] = generator.generate_c_for_modules() + + return ctext + + +def get_ir_cache_name(id: str, path: str, options: Options) -> str: + meta_path, _, _ = get_cache_names(id, path, options) + return meta_path.replace(".meta.json", ".ir.json") + + +def get_state_ir_cache_name(state: State) -> str: + return get_ir_cache_name(state.id, state.xpath, state.options) + + +def write_cache( + modules: ModuleIRs, + result: BuildResult, + group_map: dict[str, str | None], + ctext: dict[str | None, list[tuple[str, str]]], +) -> None: + """Write out the cache information for modules. + + Each module has the following cache information written (which is + in addition to the cache information written by mypy itself): + * A serialized version of its mypyc IR, minus the bodies of + functions. This allows code that depends on it to use + these serialized data structures when compiling against it + instead of needing to recompile it. (Compiling against a + module requires access to both its mypy and mypyc data + structures.) + * The hash of the mypy metadata cache file for the module. + This is used to ensure that the mypyc cache and the mypy + cache are in sync and refer to the same version of the code. + This is particularly important if mypyc crashes/errors/is + stopped after mypy has written its cache but before mypyc has. + * The hashes of all of the source file outputs for the group + the module is in. This is so that the module will be + recompiled if the source outputs are missing. + """ + + hashes = {} + for name, files in ctext.items(): + hashes[name] = {file: compute_hash(data) for file, data in files} + + # Write out cache data + for id, module in modules.items(): + st = result.graph[id] + + meta_path, _, _ = get_cache_names(id, st.xpath, result.manager.options) + # If the metadata isn't there, skip writing the cache. + try: + meta_data = result.manager.metastore.read(meta_path) + except OSError: + continue + + newpath = get_state_ir_cache_name(st) + ir_data = { + "ir": module.serialize(), + "meta_hash": hash_digest(meta_data), + "src_hashes": hashes[group_map[id]], + } + + result.manager.metastore.write(newpath, json_dumps(ir_data)) + + result.manager.metastore.commit() + + +def load_scc_from_cache( + scc: list[MypyFile], result: BuildResult, mapper: Mapper, ctx: DeserMaps +) -> ModuleIRs: + """Load IR for an SCC of modules from the cache. + + Arguments and return are as compile_scc_to_ir. + """ + cache_data = { + k.fullname: json.loads( + result.manager.metastore.read(get_state_ir_cache_name(result.graph[k.fullname])) + )["ir"] + for k in scc + } + modules = deserialize_modules(cache_data, ctx) + load_type_map(mapper, scc, ctx) + return modules + + +def compile_modules_to_c( + result: BuildResult, compiler_options: CompilerOptions, errors: Errors, groups: Groups +) -> tuple[ModuleIRs, list[FileContents]]: + """Compile Python module(s) to the source of Python C extension modules. + + This generates the source code for the "shared library" module + for each group. The shim modules are generated in mypyc.build. + Each shared library module provides, for each module in its group, + a PyCapsule containing an initialization function. + Additionally, it provides a capsule containing an export table of + pointers to all of the group's functions and static variables. + + Arguments: + result: The BuildResult from the mypy front-end + compiler_options: The compilation options + errors: Where to report any errors encountered + groups: The groups that we are compiling. See documentation of Groups type above. + + Returns the IR of the modules and a list containing the generated files for each group. + """ + # Construct a map from modules to what group they belong to + group_map = {source.module: lib_name for group, lib_name in groups for source in group} + mapper = Mapper(group_map) + + # Sometimes when we call back into mypy, there might be errors. + # We don't want to crash when that happens. + result.manager.errors.set_file( + "", module=None, scope=None, options=result.manager.options + ) + + modules = compile_modules_to_ir(result, mapper, compiler_options, errors) + if errors.num_errors > 0: + return {}, [] + + ctext = compile_ir_to_c(groups, modules, result, mapper, compiler_options) + write_cache(modules, result, group_map, ctext) + + return modules, [ctext[name] for _, name in groups] + + +def generate_function_declaration(fn: FuncIR, emitter: Emitter) -> None: + emitter.context.declarations[emitter.native_function_name(fn.decl)] = HeaderDeclaration( + f"{native_function_header(fn.decl, emitter)};", needs_export=True + ) + if fn.name != TOP_LEVEL_NAME: + if is_fastcall_supported(fn, emitter.capi_version): + emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration( + f"{wrapper_function_header(fn, emitter.names)};" + ) + else: + emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration( + f"{legacy_wrapper_function_header(fn, emitter.names)};" + ) + + +def pointerize(decl: str, name: str) -> str: + """Given a C decl and its name, modify it to be a declaration to a pointer.""" + # This doesn't work in general but does work for all our types... + if "(" in decl: + # Function pointer. Stick an * in front of the name and wrap it in parens. + return decl.replace(name, f"(*{name})") + else: + # Non-function pointer. Just stick an * in front of the name. + return decl.replace(name, f"*{name}") + + +def group_dir(group_name: str) -> str: + """Given a group name, return the relative directory path for it.""" + return os.sep.join(group_name.split(".")[:-1]) + + +class GroupGenerator: + def __init__( + self, + modules: dict[str, ModuleIR], + source_paths: dict[str, str], + group_name: str | None, + group_map: dict[str, str | None], + names: NameGenerator, + compiler_options: CompilerOptions, + ) -> None: + """Generator for C source for a compilation group. + + The code for a compilation group contains an internal and an + external .h file, and then one .c if not in multi_file mode or + one .c file per module if in multi_file mode.) + + Arguments: + modules: (name, ir) pairs for each module in the group + source_paths: Map from module names to source file paths + group_name: The name of the group (or None if this is single-module compilation) + group_map: A map of modules to their group names + names: The name generator for the compilation + multi_file: Whether to put each module in its own source file regardless + of group structure. + """ + self.modules = modules + self.source_paths = source_paths + self.context = EmitterContext(names, group_name, group_map) + self.names = names + # Initializations of globals to simple values that we can't + # do statically because the windows loader is bad. + self.simple_inits: list[tuple[str, str]] = [] + self.group_name = group_name + self.use_shared_lib = group_name is not None + self.compiler_options = compiler_options + self.multi_file = compiler_options.multi_file + + @property + def group_suffix(self) -> str: + return "_" + exported_name(self.group_name) if self.group_name else "" + + @property + def short_group_suffix(self) -> str: + return "_" + exported_name(self.group_name.split(".")[-1]) if self.group_name else "" + + def generate_c_for_modules(self) -> list[tuple[str, str]]: + file_contents = [] + multi_file = self.use_shared_lib and self.multi_file + + # Collect all literal refs in IR. + for module in self.modules.values(): + for fn in module.functions: + collect_literals(fn, self.context.literals) + + base_emitter = Emitter(self.context) + # Optionally just include the runtime library c files to + # reduce the number of compiler invocations needed + if self.compiler_options.include_runtime_files: + for name in RUNTIME_C_FILES: + base_emitter.emit_line(f'#include "{name}"') + base_emitter.emit_line(f'#include "__native{self.short_group_suffix}.h"') + base_emitter.emit_line(f'#include "__native_internal{self.short_group_suffix}.h"') + emitter = base_emitter + + self.generate_literal_tables() + + for module_name, module in self.modules.items(): + if multi_file: + emitter = Emitter(self.context) + emitter.emit_line(f'#include "__native{self.short_group_suffix}.h"') + emitter.emit_line(f'#include "__native_internal{self.short_group_suffix}.h"') + + self.declare_module(module_name, emitter) + self.declare_internal_globals(module_name, emitter) + self.declare_imports(module.imports, emitter) + + for cl in module.classes: + if cl.is_ext_class: + generate_class(cl, module_name, emitter) + + # Generate Python extension module definitions and module initialization functions. + self.generate_module_def(emitter, module_name, module) + + for fn in module.functions: + emitter.emit_line() + generate_native_function(fn, emitter, self.source_paths[module_name], module_name) + if fn.name != TOP_LEVEL_NAME: + emitter.emit_line() + if is_fastcall_supported(fn, emitter.capi_version): + generate_wrapper_function( + fn, emitter, self.source_paths[module_name], module_name + ) + else: + generate_legacy_wrapper_function( + fn, emitter, self.source_paths[module_name], module_name + ) + if multi_file: + name = f"__native_{emitter.names.private_name(module_name)}.c" + file_contents.append((name, "".join(emitter.fragments))) + + # The external header file contains type declarations while + # the internal contains declarations of functions and objects + # (which are shared between shared libraries via dynamic + # exports tables and not accessed directly.) + ext_declarations = Emitter(self.context) + ext_declarations.emit_line(f"#ifndef MYPYC_NATIVE{self.group_suffix}_H") + ext_declarations.emit_line(f"#define MYPYC_NATIVE{self.group_suffix}_H") + ext_declarations.emit_line("#include ") + ext_declarations.emit_line("#include ") + + declarations = Emitter(self.context) + declarations.emit_line(f"#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") + declarations.emit_line(f"#define MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") + declarations.emit_line("#include ") + declarations.emit_line("#include ") + declarations.emit_line(f'#include "__native{self.short_group_suffix}.h"') + declarations.emit_line() + declarations.emit_line("int CPyGlobalsInit(void);") + declarations.emit_line() + + for module_name, module in self.modules.items(): + self.declare_finals(module_name, module.final_names, declarations) + for cl in module.classes: + generate_class_type_decl(cl, emitter, ext_declarations, declarations) + self.declare_type_vars(module_name, module.type_var_names, declarations) + for fn in module.functions: + generate_function_declaration(fn, declarations) + + for lib in sorted(self.context.group_deps): + elib = exported_name(lib) + short_lib = exported_name(lib.split(".")[-1]) + declarations.emit_lines( + "#include <{}>".format(os.path.join(group_dir(lib), f"__native_{short_lib}.h")), + f"struct export_table_{elib} exports_{elib};", + ) + + sorted_decls = self.toposort_declarations() + + emitter = base_emitter + self.generate_globals_init(emitter) + + emitter.emit_line() + + for declaration in sorted_decls: + decls = ext_declarations if declaration.is_type else declarations + if not declaration.is_type: + decls.emit_lines(f"extern {declaration.decl[0]}", *declaration.decl[1:]) + # If there is a definition, emit it. Otherwise repeat the declaration + # (without an extern). + if declaration.defn: + emitter.emit_lines(*declaration.defn) + else: + emitter.emit_lines(*declaration.decl) + else: + decls.emit_lines(*declaration.decl) + + if self.group_name: + self.generate_export_table(ext_declarations, emitter) + + self.generate_shared_lib_init(emitter) + + ext_declarations.emit_line("#endif") + declarations.emit_line("#endif") + + output_dir = group_dir(self.group_name) if self.group_name else "" + return file_contents + [ + ( + os.path.join(output_dir, f"__native{self.short_group_suffix}.c"), + "".join(emitter.fragments), + ), + ( + os.path.join(output_dir, f"__native_internal{self.short_group_suffix}.h"), + "".join(declarations.fragments), + ), + ( + os.path.join(output_dir, f"__native{self.short_group_suffix}.h"), + "".join(ext_declarations.fragments), + ), + ] + + def generate_literal_tables(self) -> None: + """Generate tables containing descriptions of Python literals to construct. + + We will store the constructed literals in a single array that contains + literals of all types. This way we can refer to an arbitrary literal by + its index. + """ + literals = self.context.literals + # During module initialization we store all the constructed objects here + self.declare_global("PyObject *[%d]" % literals.num_literals(), "CPyStatics") + # Descriptions of str literals + init_str = c_string_array_initializer(literals.encoded_str_values()) + self.declare_global("const char * const []", "CPyLit_Str", initializer=init_str) + # Descriptions of bytes literals + init_bytes = c_string_array_initializer(literals.encoded_bytes_values()) + self.declare_global("const char * const []", "CPyLit_Bytes", initializer=init_bytes) + # Descriptions of int literals + init_int = c_string_array_initializer(literals.encoded_int_values()) + self.declare_global("const char * const []", "CPyLit_Int", initializer=init_int) + # Descriptions of float literals + init_floats = c_array_initializer(literals.encoded_float_values()) + self.declare_global("const double []", "CPyLit_Float", initializer=init_floats) + # Descriptions of complex literals + init_complex = c_array_initializer(literals.encoded_complex_values()) + self.declare_global("const double []", "CPyLit_Complex", initializer=init_complex) + # Descriptions of tuple literals + init_tuple = c_array_initializer(literals.encoded_tuple_values()) + self.declare_global("const int []", "CPyLit_Tuple", initializer=init_tuple) + # Descriptions of frozenset literals + init_frozenset = c_array_initializer(literals.encoded_frozenset_values()) + self.declare_global("const int []", "CPyLit_FrozenSet", initializer=init_frozenset) + + def generate_export_table(self, decl_emitter: Emitter, code_emitter: Emitter) -> None: + """Generate the declaration and definition of the group's export struct. + + To avoid needing to deal with deeply platform specific issues + involving dynamic library linking (and some possibly + insurmountable issues involving cyclic dependencies), compiled + code accesses functions and data in other compilation groups + via an explicit "export struct". + + Each group declares a struct type that contains a pointer to + every function and static variable it exports. It then + populates this struct and stores a pointer to it in a capsule + stored as an attribute named 'exports' on the group's shared + library's python module. + + On load, a group's init function will import all of its + dependencies' exports tables using the capsule mechanism and + copy the contents into a local copy of the table (to eliminate + the need for a pointer indirection when accessing it). + + Then, all calls to functions in another group and accesses to statics + from another group are done indirectly via the export table. + + For example, a group containing a module b, where b contains a class B + and a function bar, would declare an export table like: + struct export_table_b { + PyTypeObject **CPyType_B; + PyObject *(*CPyDef_B)(CPyTagged cpy_r_x); + CPyTagged (*CPyDef_B___foo)(PyObject *cpy_r_self, CPyTagged cpy_r_y); + tuple_T2OI (*CPyDef_bar)(PyObject *cpy_r_x); + char (*CPyDef___top_level__)(void); + }; + that would be initialized with: + static struct export_table_b exports = { + &CPyType_B, + &CPyDef_B, + &CPyDef_B___foo, + &CPyDef_bar, + &CPyDef___top_level__, + }; + To call `b.foo`, then, a function in another group would do + `exports_b.CPyDef_bar(...)`. + """ + + decls = decl_emitter.context.declarations + + decl_emitter.emit_lines("", f"struct export_table{self.group_suffix} {{") + for name, decl in decls.items(): + if decl.needs_export: + decl_emitter.emit_line(pointerize("\n".join(decl.decl), name)) + + decl_emitter.emit_line("};") + + code_emitter.emit_lines("", f"static struct export_table{self.group_suffix} exports = {{") + for name, decl in decls.items(): + if decl.needs_export: + code_emitter.emit_line(f"&{name},") + + code_emitter.emit_line("};") + + def generate_shared_lib_init(self, emitter: Emitter) -> None: + """Generate the init function for a shared library. + + A shared library contains all of the actual code for a + compilation group. + + The init function is responsible for creating Capsules that + wrap pointers to the initialization function of all the real + init functions for modules in this shared library as well as + the export table containing all of the exported functions and + values from all the modules. + + These capsules are stored in attributes of the shared library. + """ + assert self.group_name is not None + + emitter.emit_line() + emitter.emit_lines( + "PyMODINIT_FUNC PyInit_{}(void)".format( + shared_lib_name(self.group_name).split(".")[-1] + ), + "{", + ( + 'static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'.format( + shared_lib_name(self.group_name) + ) + ), + "int res;", + "PyObject *capsule;", + "PyObject *tmp;", + "static PyObject *module;", + "if (module) {", + "Py_INCREF(module);", + "return module;", + "}", + "module = PyModule_Create(&def);", + "if (!module) {", + "goto fail;", + "}", + "", + ) + + emitter.emit_lines( + 'capsule = PyCapsule_New(&exports, "{}.exports", NULL);'.format( + shared_lib_name(self.group_name) + ), + "if (!capsule) {", + "goto fail;", + "}", + 'res = PyObject_SetAttrString(module, "exports", capsule);', + "Py_DECREF(capsule);", + "if (res < 0) {", + "goto fail;", + "}", + "", + ) + + for mod in self.modules: + name = exported_name(mod) + emitter.emit_lines( + f"extern PyObject *CPyInit_{name}(void);", + 'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.init_{}", NULL);'.format( + name, shared_lib_name(self.group_name), name + ), + "if (!capsule) {", + "goto fail;", + "}", + f'res = PyObject_SetAttrString(module, "init_{name}", capsule);', + "Py_DECREF(capsule);", + "if (res < 0) {", + "goto fail;", + "}", + "", + ) + + for group in sorted(self.context.group_deps): + egroup = exported_name(group) + emitter.emit_lines( + 'tmp = PyImport_ImportModule("{}"); if (!tmp) goto fail; Py_DECREF(tmp);'.format( + shared_lib_name(group) + ), + 'struct export_table_{} *pexports_{} = PyCapsule_Import("{}.exports", 0);'.format( + egroup, egroup, shared_lib_name(group) + ), + f"if (!pexports_{egroup}) {{", + "goto fail;", + "}", + "memcpy(&exports_{group}, pexports_{group}, sizeof(exports_{group}));".format( + group=egroup + ), + "", + ) + + emitter.emit_lines("return module;", "fail:", "Py_XDECREF(module);", "return NULL;", "}") + + def generate_globals_init(self, emitter: Emitter) -> None: + emitter.emit_lines( + "", + "int CPyGlobalsInit(void)", + "{", + "static int is_initialized = 0;", + "if (is_initialized) return 0;", + "", + ) + + emitter.emit_line("CPy_Init();") + for symbol, fixup in self.simple_inits: + emitter.emit_line(f"{symbol} = {fixup};") + + values = "CPyLit_Str, CPyLit_Bytes, CPyLit_Int, CPyLit_Float, CPyLit_Complex, CPyLit_Tuple, CPyLit_FrozenSet" + emitter.emit_lines( + f"if (CPyStatics_Initialize(CPyStatics, {values}) < 0) {{", "return -1;", "}" + ) + + emitter.emit_lines("is_initialized = 1;", "return 0;", "}") + + def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None: + """Emit the PyModuleDef struct for a module and the module init function.""" + # Emit module methods + module_prefix = emitter.names.private_name(module_name) + emitter.emit_line(f"static PyMethodDef {module_prefix}module_methods[] = {{") + for fn in module.functions: + if fn.class_name is not None or fn.name == TOP_LEVEL_NAME: + continue + name = short_id_from_name(fn.name, fn.decl.shortname, fn.line) + if is_fastcall_supported(fn, emitter.capi_version): + flag = "METH_FASTCALL" + else: + flag = "METH_VARARGS" + emitter.emit_line( + ( + '{{"{name}", (PyCFunction){prefix}{cname}, {flag} | METH_KEYWORDS, ' + "NULL /* docstring */}}," + ).format(name=name, cname=fn.cname(emitter.names), prefix=PREFIX, flag=flag) + ) + emitter.emit_line("{NULL, NULL, 0, NULL}") + emitter.emit_line("};") + emitter.emit_line() + + # Emit module definition struct + emitter.emit_lines( + f"static struct PyModuleDef {module_prefix}module = {{", + "PyModuleDef_HEAD_INIT,", + f'"{module_name}",', + "NULL, /* docstring */", + "-1, /* size of per-interpreter state of the module,", + " or -1 if the module keeps state in global variables. */", + f"{module_prefix}module_methods", + "};", + ) + emitter.emit_line() + # Emit module init function. If we are compiling just one module, this + # will be the C API init function. If we are compiling 2+ modules, we + # generate a shared library for the modules and shims that call into + # the shared library, and in this case we use an internal module + # initialized function that will be called by the shim. + if not self.use_shared_lib: + declaration = f"PyMODINIT_FUNC PyInit_{module_name}(void)" + else: + declaration = f"PyObject *CPyInit_{exported_name(module_name)}(void)" + emitter.emit_lines(declaration, "{") + emitter.emit_line("PyObject* modname = NULL;") + # Store the module reference in a static and return it when necessary. + # This is separate from the *global* reference to the module that will + # be populated when it is imported by a compiled module. We want that + # reference to only be populated when the module has been successfully + # imported, whereas this we want to have to stop a circular import. + module_static = self.module_internal_static_name(module_name, emitter) + + emitter.emit_lines( + f"if ({module_static}) {{", + f"Py_INCREF({module_static});", + f"return {module_static};", + "}", + ) + + emitter.emit_lines( + f"{module_static} = PyModule_Create(&{module_prefix}module);", + f"if (unlikely({module_static} == NULL))", + " goto fail;", + ) + emitter.emit_line( + f'modname = PyObject_GetAttrString((PyObject *){module_static}, "__name__");' + ) + + module_globals = emitter.static_name("globals", module_name) + emitter.emit_lines( + f"{module_globals} = PyModule_GetDict({module_static});", + f"if (unlikely({module_globals} == NULL))", + " goto fail;", + ) + + # HACK: Manually instantiate generated classes here + type_structs: list[str] = [] + for cl in module.classes: + type_struct = emitter.type_struct_name(cl) + type_structs.append(type_struct) + if cl.is_generated: + emitter.emit_lines( + "{t} = (PyTypeObject *)CPyType_FromTemplate(" + "(PyObject *){t}_template, NULL, modname);".format(t=type_struct) + ) + emitter.emit_lines(f"if (unlikely(!{type_struct}))", " goto fail;") + + emitter.emit_lines("if (CPyGlobalsInit() < 0)", " goto fail;") + + self.generate_top_level_call(module, emitter) + + emitter.emit_lines("Py_DECREF(modname);") + + emitter.emit_line(f"return {module_static};") + emitter.emit_lines("fail:", f"Py_CLEAR({module_static});", "Py_CLEAR(modname);") + for name, typ in module.final_names: + static_name = emitter.static_name(name, module_name) + emitter.emit_dec_ref(static_name, typ, is_xdec=True) + undef = emitter.c_undefined_value(typ) + emitter.emit_line(f"{static_name} = {undef};") + # the type objects returned from CPyType_FromTemplate are all new references + # so we have to decref them + for t in type_structs: + emitter.emit_line(f"Py_CLEAR({t});") + emitter.emit_line("return NULL;") + emitter.emit_line("}") + + def generate_top_level_call(self, module: ModuleIR, emitter: Emitter) -> None: + """Generate call to function representing module top level.""" + # Optimization: we tend to put the top level last, so reverse iterate + for fn in reversed(module.functions): + if fn.name == TOP_LEVEL_NAME: + emitter.emit_lines( + f"char result = {emitter.native_function_name(fn.decl)}();", + "if (result == 2)", + " goto fail;", + ) + break + + def toposort_declarations(self) -> list[HeaderDeclaration]: + """Topologically sort the declaration dict by dependencies. + + Declarations can require other declarations to come prior in C (such as declaring structs). + In order to guarantee that the C output will compile the declarations will thus need to + be properly ordered. This simple DFS guarantees that we have a proper ordering. + + This runs in O(V + E). + """ + result = [] + marked_declarations: dict[str, MarkedDeclaration] = {} + for k, v in self.context.declarations.items(): + marked_declarations[k] = MarkedDeclaration(v, False) + + def _toposort_visit(name: str) -> None: + decl = marked_declarations[name] + if decl.mark: + return + + for child in decl.declaration.dependencies: + _toposort_visit(child) + + result.append(decl.declaration) + decl.mark = True + + for name in marked_declarations: + _toposort_visit(name) + + return result + + def declare_global( + self, type_spaced: str, name: str, *, initializer: str | None = None + ) -> None: + if "[" not in type_spaced: + base = f"{type_spaced}{name}" + else: + a, b = type_spaced.split("[", 1) + base = f"{a}{name}[{b}" + + if not initializer: + defn = None + else: + defn = [f"{base} = {initializer};"] + if name not in self.context.declarations: + self.context.declarations[name] = HeaderDeclaration(f"{base};", defn=defn) + + def declare_internal_globals(self, module_name: str, emitter: Emitter) -> None: + static_name = emitter.static_name("globals", module_name) + self.declare_global("PyObject *", static_name) + + def module_internal_static_name(self, module_name: str, emitter: Emitter) -> str: + return emitter.static_name(module_name + "_internal", None, prefix=MODULE_PREFIX) + + def declare_module(self, module_name: str, emitter: Emitter) -> None: + # We declare two globals for each compiled module: + # one used internally in the implementation of module init to cache results + # and prevent infinite recursion in import cycles, and one used + # by other modules to refer to it. + if module_name in self.modules: + internal_static_name = self.module_internal_static_name(module_name, emitter) + self.declare_global("CPyModule *", internal_static_name, initializer="NULL") + static_name = emitter.static_name(module_name, None, prefix=MODULE_PREFIX) + self.declare_global("CPyModule *", static_name) + self.simple_inits.append((static_name, "Py_None")) + + def declare_imports(self, imps: Iterable[str], emitter: Emitter) -> None: + for imp in imps: + self.declare_module(imp, emitter) + + def declare_finals( + self, module: str, final_names: Iterable[tuple[str, RType]], emitter: Emitter + ) -> None: + for name, typ in final_names: + static_name = emitter.static_name(name, module) + emitter.context.declarations[static_name] = HeaderDeclaration( + f"{emitter.ctype_spaced(typ)}{static_name};", + [self.final_definition(module, name, typ, emitter)], + needs_export=True, + ) + + def final_definition(self, module: str, name: str, typ: RType, emitter: Emitter) -> str: + static_name = emitter.static_name(name, module) + # Here we rely on the fact that undefined value and error value are always the same + undefined = emitter.c_initializer_undefined_value(typ) + return f"{emitter.ctype_spaced(typ)}{static_name} = {undefined};" + + def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None: + symbol = emitter.static_name(identifier, None) + self.declare_global("PyObject *", symbol) + + def declare_type_vars(self, module: str, type_var_names: list[str], emitter: Emitter) -> None: + for name in type_var_names: + static_name = emitter.static_name(name, module, prefix=TYPE_VAR_PREFIX) + emitter.context.declarations[static_name] = HeaderDeclaration( + f"PyObject *{static_name};", + [f"PyObject *{static_name} = NULL;"], + needs_export=False, + ) + + +T = TypeVar("T") + + +def toposort(deps: dict[T, set[T]]) -> list[T]: + """Topologically sort a dict from item to dependencies. + + This runs in O(V + E). + """ + result = [] + visited: set[T] = set() + + def visit(item: T) -> None: + if item in visited: + return + + for child in deps[item]: + visit(child) + + result.append(item) + visited.add(item) + + for item in deps: + visit(item) + + return result + + +def is_fastcall_supported(fn: FuncIR, capi_version: tuple[int, int]) -> bool: + if fn.class_name is not None: + if fn.name == "__call__": + # We can use vectorcalls (PEP 590) when supported + return use_vectorcall(capi_version) + # TODO: Support fastcall for __init__. + return fn.name != "__init__" + return True + + +def collect_literals(fn: FuncIR, literals: Literals) -> None: + """Store all Python literal object refs in fn. + + Collecting literals must happen only after we have the final IR. + This way we won't include literals that have been optimized away. + """ + for block in fn.blocks: + for op in block.ops: + if isinstance(op, LoadLiteral): + literals.record_literal(op.value) + + +def c_string_array_initializer(components: list[bytes]) -> str: + result = [] + result.append("{\n") + for s in components: + result.append(" " + c_string_initializer(s) + ",\n") + result.append("}") + return "".join(result) diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py new file mode 100644 index 000000000000..45c6c7a05867 --- /dev/null +++ b/mypyc/codegen/emitwrapper.py @@ -0,0 +1,979 @@ +"""Generate CPython API wrapper functions for native functions. + +The wrapper functions are used by the CPython runtime when calling +native functions from interpreted code, and when the called function +can't be determined statically in compiled code. They validate, match, +unbox and type check function arguments, and box return values as +needed. All wrappers accept and return 'PyObject *' (boxed) values. + +The wrappers aren't used for most calls between two native functions +or methods in a single compilation unit. +""" + +from __future__ import annotations + +from typing import Sequence + +from mypy.nodes import ARG_NAMED, ARG_NAMED_OPT, ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, ArgKind +from mypy.operators import op_methods_to_symbols, reverse_op_method_names, reverse_op_methods +from mypyc.codegen.emit import AssignHandler, Emitter, ErrorHandler, GotoHandler, ReturnHandler +from mypyc.common import ( + BITMAP_BITS, + BITMAP_TYPE, + DUNDER_PREFIX, + NATIVE_PREFIX, + PREFIX, + bitmap_name, + use_vectorcall, +) +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncIR, RuntimeArg +from mypyc.ir.rtypes import ( + RInstance, + RType, + is_bool_rprimitive, + is_int_rprimitive, + is_object_rprimitive, + object_rprimitive, +) +from mypyc.namegen import NameGenerator + +# Generic vectorcall wrapper functions (Python 3.7+) +# +# A wrapper function has a signature like this: +# +# PyObject *fn(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +# +# The function takes a self object, pointer to an array of arguments, +# the number of positional arguments, and a tuple of keyword argument +# names (that are stored starting in args[nargs]). +# +# It returns the returned object, or NULL on an exception. +# +# These are more efficient than legacy wrapper functions, since +# usually no tuple or dict objects need to be created for the +# arguments. Vectorcalls also use pre-constructed str objects for +# keyword argument names and other pre-computed information, instead +# of processing the argument format string on each call. + + +def wrapper_function_header(fn: FuncIR, names: NameGenerator) -> str: + """Return header of a vectorcall wrapper function. + + See comment above for a summary of the arguments. + """ + return ( + "PyObject *{prefix}{name}(" + "PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames)" + ).format(prefix=PREFIX, name=fn.cname(names)) + + +def generate_traceback_code( + fn: FuncIR, emitter: Emitter, source_path: str, module_name: str +) -> str: + # If we hit an error while processing arguments, then we emit a + # traceback frame to make it possible to debug where it happened. + # Unlike traceback frames added for exceptions seen in IR, we do this + # even if there is no `traceback_name`. This is because the error will + # have originated here and so we need it in the traceback. + globals_static = emitter.static_name("globals", module_name) + traceback_code = 'CPy_AddTraceback("%s", "%s", %d, %s);' % ( + source_path.replace("\\", "\\\\"), + fn.traceback_name or fn.name, + fn.line, + globals_static, + ) + return traceback_code + + +def make_arg_groups(args: list[RuntimeArg]) -> dict[ArgKind, list[RuntimeArg]]: + """Group arguments by kind.""" + return {k: [arg for arg in args if arg.kind == k] for k in ArgKind} + + +def reorder_arg_groups(groups: dict[ArgKind, list[RuntimeArg]]) -> list[RuntimeArg]: + """Reorder argument groups to match their order in a format string.""" + return groups[ARG_POS] + groups[ARG_OPT] + groups[ARG_NAMED_OPT] + groups[ARG_NAMED] + + +def make_static_kwlist(args: list[RuntimeArg]) -> str: + arg_names = "".join(f'"{arg.name}", ' for arg in args) + return f"static const char * const kwlist[] = {{{arg_names}0}};" + + +def make_format_string(func_name: str | None, groups: dict[ArgKind, list[RuntimeArg]]) -> str: + """Return a format string that specifies the accepted arguments. + + The format string is an extended subset of what is supported by + PyArg_ParseTupleAndKeywords(). Only the type 'O' is used, and we + also support some extensions: + + - Required keyword-only arguments are introduced after '@' + - If the function receives *args or **kwargs, we add a '%' prefix + + Each group requires the previous groups' delimiters to be present + first. + + These are used by both vectorcall and legacy wrapper functions. + """ + format = "" + if groups[ARG_STAR] or groups[ARG_STAR2]: + format += "%" + format += "O" * len(groups[ARG_POS]) + if groups[ARG_OPT] or groups[ARG_NAMED_OPT] or groups[ARG_NAMED]: + format += "|" + "O" * len(groups[ARG_OPT]) + if groups[ARG_NAMED_OPT] or groups[ARG_NAMED]: + format += "$" + "O" * len(groups[ARG_NAMED_OPT]) + if groups[ARG_NAMED]: + format += "@" + "O" * len(groups[ARG_NAMED]) + if func_name is not None: + format += f":{func_name}" + return format + + +def generate_wrapper_function( + fn: FuncIR, emitter: Emitter, source_path: str, module_name: str +) -> None: + """Generate a CPython-compatible vectorcall wrapper for a native function. + + In particular, this handles unboxing the arguments, calling the native function, and + then boxing the return value. + """ + emitter.emit_line(f"{wrapper_function_header(fn, emitter.names)} {{") + + # If fn is a method, then the first argument is a self param + real_args = list(fn.args) + if fn.sig.num_bitmap_args: + real_args = real_args[: -fn.sig.num_bitmap_args] + if fn.class_name and fn.decl.kind != FUNC_STATICMETHOD: + arg = real_args.pop(0) + emitter.emit_line(f"PyObject *obj_{arg.name} = self;") + + # Need to order args as: required, optional, kwonly optional, kwonly required + # This is because CPyArg_ParseStackAndKeywords format string requires + # them grouped in that way. + groups = make_arg_groups(real_args) + reordered_args = reorder_arg_groups(groups) + + emitter.emit_line(make_static_kwlist(reordered_args)) + fmt = make_format_string(fn.name, groups) + # Define the arguments the function accepts (but no types yet) + emitter.emit_line(f'static CPyArg_Parser parser = {{"{fmt}", kwlist, 0}};') + + for arg in real_args: + emitter.emit_line( + "PyObject *obj_{}{};".format(arg.name, " = NULL" if arg.optional else "") + ) + + cleanups = [f"CPy_DECREF(obj_{arg.name});" for arg in groups[ARG_STAR] + groups[ARG_STAR2]] + + arg_ptrs: list[str] = [] + if groups[ARG_STAR] or groups[ARG_STAR2]: + arg_ptrs += [f"&obj_{groups[ARG_STAR][0].name}" if groups[ARG_STAR] else "NULL"] + arg_ptrs += [f"&obj_{groups[ARG_STAR2][0].name}" if groups[ARG_STAR2] else "NULL"] + arg_ptrs += [f"&obj_{arg.name}" for arg in reordered_args] + + if fn.name == "__call__" and use_vectorcall(emitter.capi_version): + nargs = "PyVectorcall_NARGS(nargs)" + else: + nargs = "nargs" + parse_fn = "CPyArg_ParseStackAndKeywords" + # Special case some common signatures + if not real_args: + # No args + parse_fn = "CPyArg_ParseStackAndKeywordsNoArgs" + elif len(real_args) == 1 and len(groups[ARG_POS]) == 1: + # Single positional arg + parse_fn = "CPyArg_ParseStackAndKeywordsOneArg" + elif len(real_args) == len(groups[ARG_POS]) + len(groups[ARG_OPT]): + # No keyword-only args, *args or **kwargs + parse_fn = "CPyArg_ParseStackAndKeywordsSimple" + emitter.emit_lines( + "if (!{}(args, {}, kwnames, &parser{})) {{".format( + parse_fn, nargs, "".join(", " + n for n in arg_ptrs) + ), + "return NULL;", + "}", + ) + for i in range(fn.sig.num_bitmap_args): + name = bitmap_name(i) + emitter.emit_line(f"{BITMAP_TYPE} {name} = 0;") + traceback_code = generate_traceback_code(fn, emitter, source_path, module_name) + generate_wrapper_core( + fn, + emitter, + groups[ARG_OPT] + groups[ARG_NAMED_OPT], + cleanups=cleanups, + traceback_code=traceback_code, + ) + + emitter.emit_line("}") + + +# Legacy generic wrapper functions +# +# These take a self object, a Python tuple of positional arguments, +# and a dict of keyword arguments. These are a lot slower than +# vectorcall wrappers, especially in calls involving keyword +# arguments. + + +def legacy_wrapper_function_header(fn: FuncIR, names: NameGenerator) -> str: + return "PyObject *{prefix}{name}(PyObject *self, PyObject *args, PyObject *kw)".format( + prefix=PREFIX, name=fn.cname(names) + ) + + +def generate_legacy_wrapper_function( + fn: FuncIR, emitter: Emitter, source_path: str, module_name: str +) -> None: + """Generates a CPython-compatible legacy wrapper for a native function. + + In particular, this handles unboxing the arguments, calling the native function, and + then boxing the return value. + """ + emitter.emit_line(f"{legacy_wrapper_function_header(fn, emitter.names)} {{") + + # If fn is a method, then the first argument is a self param + real_args = list(fn.args) + if fn.sig.num_bitmap_args: + real_args = real_args[: -fn.sig.num_bitmap_args] + if fn.class_name and fn.decl.kind != FUNC_STATICMETHOD: + arg = real_args.pop(0) + emitter.emit_line(f"PyObject *obj_{arg.name} = self;") + + # Need to order args as: required, optional, kwonly optional, kwonly required + # This is because CPyArg_ParseTupleAndKeywords format string requires + # them grouped in that way. + groups = make_arg_groups(real_args) + reordered_args = reorder_arg_groups(groups) + + emitter.emit_line(make_static_kwlist(reordered_args)) + for arg in real_args: + emitter.emit_line( + "PyObject *obj_{}{};".format(arg.name, " = NULL" if arg.optional else "") + ) + + cleanups = [f"CPy_DECREF(obj_{arg.name});" for arg in groups[ARG_STAR] + groups[ARG_STAR2]] + + arg_ptrs: list[str] = [] + if groups[ARG_STAR] or groups[ARG_STAR2]: + arg_ptrs += [f"&obj_{groups[ARG_STAR][0].name}" if groups[ARG_STAR] else "NULL"] + arg_ptrs += [f"&obj_{groups[ARG_STAR2][0].name}" if groups[ARG_STAR2] else "NULL"] + arg_ptrs += [f"&obj_{arg.name}" for arg in reordered_args] + + emitter.emit_lines( + 'if (!CPyArg_ParseTupleAndKeywords(args, kw, "{}", "{}", kwlist{})) {{'.format( + make_format_string(None, groups), fn.name, "".join(", " + n for n in arg_ptrs) + ), + "return NULL;", + "}", + ) + for i in range(fn.sig.num_bitmap_args): + name = bitmap_name(i) + emitter.emit_line(f"{BITMAP_TYPE} {name} = 0;") + traceback_code = generate_traceback_code(fn, emitter, source_path, module_name) + generate_wrapper_core( + fn, + emitter, + groups[ARG_OPT] + groups[ARG_NAMED_OPT], + cleanups=cleanups, + traceback_code=traceback_code, + ) + + emitter.emit_line("}") + + +# Specialized wrapper functions + + +def generate_dunder_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __dunder__ methods to be able to fit into the mapping + protocol slot. This specifically means that the arguments are taken as *PyObjects and returned + as *PyObjects. + """ + gen = WrapperGenerator(cl, emitter) + gen.set_target(fn) + gen.emit_header() + gen.emit_arg_processing() + gen.emit_call() + gen.finish() + return gen.wrapper_name() + + +def generate_ipow_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generate a wrapper for native __ipow__. + + Since __ipow__ fills a ternary slot, but almost no one defines __ipow__ to take three + arguments, the wrapper needs to tweaked to force it to accept three arguments. + """ + gen = WrapperGenerator(cl, emitter) + gen.set_target(fn) + assert len(fn.args) in (2, 3), "__ipow__ should only take 2 or 3 arguments" + gen.arg_names = ["self", "exp", "mod"] + gen.emit_header() + gen.emit_arg_processing() + handle_third_pow_argument( + fn, + emitter, + gen, + if_unsupported=[ + 'PyErr_SetString(PyExc_TypeError, "__ipow__ takes 2 positional arguments but 3 were given");', + "return NULL;", + ], + ) + gen.emit_call() + gen.finish() + return gen.wrapper_name() + + +def generate_bin_op_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for a native binary dunder method. + + The same wrapper that handles the forward method (e.g. __add__) also handles + the corresponding reverse method (e.g. __radd__), if defined. + + Both arguments and the return value are PyObject *. + """ + gen = WrapperGenerator(cl, emitter) + gen.set_target(fn) + if fn.name in ("__pow__", "__rpow__"): + gen.arg_names = ["left", "right", "mod"] + else: + gen.arg_names = ["left", "right"] + wrapper_name = gen.wrapper_name() + + gen.emit_header() + if fn.name not in reverse_op_methods and fn.name in reverse_op_method_names: + # There's only a reverse operator method. + generate_bin_op_reverse_only_wrapper(fn, emitter, gen) + else: + rmethod = reverse_op_methods[fn.name] + fn_rev = cl.get_method(rmethod) + if fn_rev is None: + # There's only a forward operator method. + generate_bin_op_forward_only_wrapper(fn, emitter, gen) + else: + # There's both a forward and a reverse operator method. + generate_bin_op_both_wrappers(cl, fn, fn_rev, emitter, gen) + return wrapper_name + + +def generate_bin_op_forward_only_wrapper( + fn: FuncIR, emitter: Emitter, gen: WrapperGenerator +) -> None: + gen.emit_arg_processing(error=GotoHandler("typefail"), raise_exception=False) + handle_third_pow_argument(fn, emitter, gen, if_unsupported=["goto typefail;"]) + gen.emit_call(not_implemented_handler="goto typefail;") + gen.emit_error_handling() + emitter.emit_label("typefail") + # If some argument has an incompatible type, treat this the same as + # returning NotImplemented, and try to call the reverse operator method. + # + # Note that in normal Python you'd instead of an explicit + # return of NotImplemented, but it doesn't generally work here + # the body won't be executed at all if there is an argument + # type check failure. + # + # The recommended way is to still use a type check in the + # body. This will only be used in interpreted mode: + # + # def __add__(self, other: int) -> Foo: + # if not isinstance(other, int): + # return NotImplemented + # ... + generate_bin_op_reverse_dunder_call(fn, emitter, reverse_op_methods[fn.name]) + gen.finish() + + +def generate_bin_op_reverse_only_wrapper( + fn: FuncIR, emitter: Emitter, gen: WrapperGenerator +) -> None: + gen.arg_names = ["right", "left"] + gen.emit_arg_processing(error=GotoHandler("typefail"), raise_exception=False) + handle_third_pow_argument(fn, emitter, gen, if_unsupported=["goto typefail;"]) + gen.emit_call() + gen.emit_error_handling() + emitter.emit_label("typefail") + emitter.emit_line("Py_INCREF(Py_NotImplemented);") + emitter.emit_line("return Py_NotImplemented;") + gen.finish() + + +def generate_bin_op_both_wrappers( + cl: ClassIR, fn: FuncIR, fn_rev: FuncIR, emitter: Emitter, gen: WrapperGenerator +) -> None: + # There's both a forward and a reverse operator method. First + # check if we should try calling the forward one. If the + # argument type check fails, fall back to the reverse method. + # + # Similar to above, we can't perfectly match Python semantics. + # In regular Python code you'd return NotImplemented if the + # operand has the wrong type, but in compiled code we'll never + # get to execute the type check. + emitter.emit_line( + "if (PyObject_IsInstance(obj_left, (PyObject *){})) {{".format( + emitter.type_struct_name(cl) + ) + ) + gen.emit_arg_processing(error=GotoHandler("typefail"), raise_exception=False) + handle_third_pow_argument(fn, emitter, gen, if_unsupported=["goto typefail2;"]) + # Ternary __rpow__ calls aren't a thing so immediately bail + # if ternary __pow__ returns NotImplemented. + if fn.name == "__pow__" and len(fn.args) == 3: + fwd_not_implemented_handler = "goto typefail2;" + else: + fwd_not_implemented_handler = "goto typefail;" + gen.emit_call(not_implemented_handler=fwd_not_implemented_handler) + gen.emit_error_handling() + emitter.emit_line("}") + emitter.emit_label("typefail") + emitter.emit_line( + "if (PyObject_IsInstance(obj_right, (PyObject *){})) {{".format( + emitter.type_struct_name(cl) + ) + ) + gen.set_target(fn_rev) + gen.arg_names = ["right", "left"] + gen.emit_arg_processing(error=GotoHandler("typefail2"), raise_exception=False) + handle_third_pow_argument(fn_rev, emitter, gen, if_unsupported=["goto typefail2;"]) + gen.emit_call() + gen.emit_error_handling() + emitter.emit_line("} else {") + generate_bin_op_reverse_dunder_call(fn, emitter, fn_rev.name) + emitter.emit_line("}") + emitter.emit_label("typefail2") + emitter.emit_line("Py_INCREF(Py_NotImplemented);") + emitter.emit_line("return Py_NotImplemented;") + gen.finish() + + +def generate_bin_op_reverse_dunder_call(fn: FuncIR, emitter: Emitter, rmethod: str) -> None: + if fn.name in ("__pow__", "__rpow__"): + # Ternary pow() will never call the reverse dunder. + emitter.emit_line("if (obj_mod == Py_None) {") + emitter.emit_line(f"_Py_IDENTIFIER({rmethod});") + emitter.emit_line( + 'return CPy_CallReverseOpMethod(obj_left, obj_right, "{}", &PyId_{});'.format( + op_methods_to_symbols[fn.name], rmethod + ) + ) + if fn.name in ("__pow__", "__rpow__"): + emitter.emit_line("} else {") + emitter.emit_line("Py_INCREF(Py_NotImplemented);") + emitter.emit_line("return Py_NotImplemented;") + emitter.emit_line("}") + + +def handle_third_pow_argument( + fn: FuncIR, emitter: Emitter, gen: WrapperGenerator, *, if_unsupported: list[str] +) -> None: + if fn.name not in ("__pow__", "__rpow__", "__ipow__"): + return + + if (fn.name in ("__pow__", "__ipow__") and len(fn.args) == 2) or fn.name == "__rpow__": + # If the power dunder only supports two arguments and the third + # argument (AKA mod) is set to a non-default value, simply bail. + # + # Importantly, this prevents any ternary __rpow__ calls from + # happening (as per the language specification). + emitter.emit_line("if (obj_mod != Py_None) {") + for line in if_unsupported: + emitter.emit_line(line) + emitter.emit_line("}") + # The slot wrapper will receive three arguments, but the call only + # supports two so make sure that the third argument isn't passed + # along. This is needed as two-argument __(i)pow__ is allowed and + # rather common. + if len(gen.arg_names) == 3: + gen.arg_names.pop() + + +RICHCOMPARE_OPS = { + "__lt__": "Py_LT", + "__gt__": "Py_GT", + "__le__": "Py_LE", + "__ge__": "Py_GE", + "__eq__": "Py_EQ", + "__ne__": "Py_NE", +} + + +def generate_richcompare_wrapper(cl: ClassIR, emitter: Emitter) -> str | None: + """Generates a wrapper for richcompare dunder methods.""" + # Sort for determinism on Python 3.5 + matches = sorted(name for name in RICHCOMPARE_OPS if cl.has_method(name)) + if not matches: + return None + + name = f"{DUNDER_PREFIX}_RichCompare_{cl.name_prefix(emitter.names)}" + emitter.emit_line( + "static PyObject *{name}(PyObject *obj_lhs, PyObject *obj_rhs, int op) {{".format( + name=name + ) + ) + emitter.emit_line("switch (op) {") + for func in matches: + emitter.emit_line(f"case {RICHCOMPARE_OPS[func]}: {{") + method = cl.get_method(func) + assert method is not None + generate_wrapper_core(method, emitter, arg_names=["lhs", "rhs"]) + emitter.emit_line("}") + emitter.emit_line("}") + + emitter.emit_line("Py_INCREF(Py_NotImplemented);") + emitter.emit_line("return Py_NotImplemented;") + + emitter.emit_line("}") + + return name + + +def generate_get_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __get__ methods.""" + name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" + emitter.emit_line( + "static PyObject *{name}(PyObject *self, PyObject *instance, PyObject *owner) {{".format( + name=name + ) + ) + emitter.emit_line("instance = instance ? instance : Py_None;") + emitter.emit_line(f"return {NATIVE_PREFIX}{fn.cname(emitter.names)}(self, instance, owner);") + emitter.emit_line("}") + + return name + + +def generate_hash_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __hash__ methods.""" + name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" + emitter.emit_line(f"static Py_ssize_t {name}(PyObject *self) {{") + emitter.emit_line( + "{}retval = {}{}{}(self);".format( + emitter.ctype_spaced(fn.ret_type), + emitter.get_group_prefix(fn.decl), + NATIVE_PREFIX, + fn.cname(emitter.names), + ) + ) + emitter.emit_error_check("retval", fn.ret_type, "return -1;") + if is_int_rprimitive(fn.ret_type): + emitter.emit_line("Py_ssize_t val = CPyTagged_AsSsize_t(retval);") + else: + emitter.emit_line("Py_ssize_t val = PyLong_AsSsize_t(retval);") + emitter.emit_dec_ref("retval", fn.ret_type) + emitter.emit_line("if (PyErr_Occurred()) return -1;") + # We can't return -1 from a hash function.. + emitter.emit_line("if (val == -1) return -2;") + emitter.emit_line("return val;") + emitter.emit_line("}") + + return name + + +def generate_len_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __len__ methods.""" + name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" + emitter.emit_line(f"static Py_ssize_t {name}(PyObject *self) {{") + emitter.emit_line( + "{}retval = {}{}{}(self);".format( + emitter.ctype_spaced(fn.ret_type), + emitter.get_group_prefix(fn.decl), + NATIVE_PREFIX, + fn.cname(emitter.names), + ) + ) + emitter.emit_error_check("retval", fn.ret_type, "return -1;") + if is_int_rprimitive(fn.ret_type): + emitter.emit_line("Py_ssize_t val = CPyTagged_AsSsize_t(retval);") + else: + emitter.emit_line("Py_ssize_t val = PyLong_AsSsize_t(retval);") + emitter.emit_dec_ref("retval", fn.ret_type) + emitter.emit_line("if (PyErr_Occurred()) return -1;") + emitter.emit_line("return val;") + emitter.emit_line("}") + + return name + + +def generate_bool_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __bool__ methods.""" + name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" + emitter.emit_line(f"static int {name}(PyObject *self) {{") + emitter.emit_line( + "{}val = {}{}(self);".format( + emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names) + ) + ) + emitter.emit_error_check("val", fn.ret_type, "return -1;") + # This wouldn't be that hard to fix but it seems unimportant and + # getting error handling and unboxing right would be fiddly. (And + # way easier to do in IR!) + assert is_bool_rprimitive(fn.ret_type), "Only bool return supported for __bool__" + emitter.emit_line("return val;") + emitter.emit_line("}") + + return name + + +def generate_del_item_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __delitem__. + + This is only called from a combined __delitem__/__setitem__ wrapper. + """ + name = "{}{}{}".format(DUNDER_PREFIX, "__delitem__", cl.name_prefix(emitter.names)) + input_args = ", ".join(f"PyObject *obj_{arg.name}" for arg in fn.args) + emitter.emit_line(f"static int {name}({input_args}) {{") + generate_set_del_item_wrapper_inner(fn, emitter, fn.args) + return name + + +def generate_set_del_item_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __setitem__ method (also works for __delitem__). + + This is used with the mapping protocol slot. Arguments are taken as *PyObjects and we + return a negative C int on error. + + Create a separate wrapper function for __delitem__ as needed and have the + __setitem__ wrapper call it if the value is NULL. Return the name + of the outer (__setitem__) wrapper. + """ + method_cls = cl.get_method_and_class("__delitem__") + del_name = None + if method_cls and method_cls[1] == cl: + # Generate a separate wrapper for __delitem__ + del_name = generate_del_item_wrapper(cl, method_cls[0], emitter) + + args = fn.args + if fn.name == "__delitem__": + # Add an extra argument for value that we expect to be NULL. + args = list(args) + [RuntimeArg("___value", object_rprimitive, ARG_POS)] + + name = "{}{}{}".format(DUNDER_PREFIX, "__setitem__", cl.name_prefix(emitter.names)) + input_args = ", ".join(f"PyObject *obj_{arg.name}" for arg in args) + emitter.emit_line(f"static int {name}({input_args}) {{") + + # First check if this is __delitem__ + emitter.emit_line(f"if (obj_{args[2].name} == NULL) {{") + if del_name is not None: + # We have a native implementation, so call it + emitter.emit_line(f"return {del_name}(obj_{args[0].name}, obj_{args[1].name});") + else: + # Try to call superclass method instead + emitter.emit_line(f"PyObject *super = CPy_Super(CPyModule_builtins, obj_{args[0].name});") + emitter.emit_line("if (super == NULL) return -1;") + emitter.emit_line( + 'PyObject *result = PyObject_CallMethod(super, "__delitem__", "O", obj_{});'.format( + args[1].name + ) + ) + emitter.emit_line("Py_DECREF(super);") + emitter.emit_line("Py_XDECREF(result);") + emitter.emit_line("return result == NULL ? -1 : 0;") + emitter.emit_line("}") + + method_cls = cl.get_method_and_class("__setitem__") + if method_cls and method_cls[1] == cl: + generate_set_del_item_wrapper_inner(fn, emitter, args) + else: + emitter.emit_line(f"PyObject *super = CPy_Super(CPyModule_builtins, obj_{args[0].name});") + emitter.emit_line("if (super == NULL) return -1;") + emitter.emit_line("PyObject *result;") + + if method_cls is None and cl.builtin_base is None: + msg = f"'{cl.name}' object does not support item assignment" + emitter.emit_line(f'PyErr_SetString(PyExc_TypeError, "{msg}");') + emitter.emit_line("result = NULL;") + else: + # A base class may have __setitem__ + emitter.emit_line( + 'result = PyObject_CallMethod(super, "__setitem__", "OO", obj_{}, obj_{});'.format( + args[1].name, args[2].name + ) + ) + emitter.emit_line("Py_DECREF(super);") + emitter.emit_line("Py_XDECREF(result);") + emitter.emit_line("return result == NULL ? -1 : 0;") + emitter.emit_line("}") + return name + + +def generate_set_del_item_wrapper_inner( + fn: FuncIR, emitter: Emitter, args: Sequence[RuntimeArg] +) -> None: + for arg in args: + generate_arg_check(arg.name, arg.type, emitter, GotoHandler("fail")) + native_args = ", ".join(f"arg_{arg.name}" for arg in args) + emitter.emit_line( + "{}val = {}{}({});".format( + emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names), native_args + ) + ) + emitter.emit_error_check("val", fn.ret_type, "goto fail;") + emitter.emit_dec_ref("val", fn.ret_type) + emitter.emit_line("return 0;") + emitter.emit_label("fail") + emitter.emit_line("return -1;") + emitter.emit_line("}") + + +def generate_contains_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for a native __contains__ method.""" + name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" + emitter.emit_line(f"static int {name}(PyObject *self, PyObject *obj_item) {{") + generate_arg_check("item", fn.args[1].type, emitter, ReturnHandler("-1")) + emitter.emit_line( + "{}val = {}{}(self, arg_item);".format( + emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names) + ) + ) + emitter.emit_error_check("val", fn.ret_type, "return -1;") + if is_bool_rprimitive(fn.ret_type): + emitter.emit_line("return val;") + else: + emitter.emit_line("int boolval = PyObject_IsTrue(val);") + emitter.emit_dec_ref("val", fn.ret_type) + emitter.emit_line("return boolval;") + emitter.emit_line("}") + + return name + + +# Helpers + + +def generate_wrapper_core( + fn: FuncIR, + emitter: Emitter, + optional_args: list[RuntimeArg] | None = None, + arg_names: list[str] | None = None, + cleanups: list[str] | None = None, + traceback_code: str | None = None, +) -> None: + """Generates the core part of a wrapper function for a native function. + + This expects each argument as a PyObject * named obj_{arg} as a precondition. + It converts the PyObject *s to the necessary types, checking and unboxing if necessary, + makes the call, then boxes the result if necessary and returns it. + """ + gen = WrapperGenerator(None, emitter) + gen.set_target(fn) + if arg_names: + gen.arg_names = arg_names + gen.cleanups = cleanups or [] + gen.optional_args = optional_args or [] + gen.traceback_code = traceback_code or "" + + error = ReturnHandler("NULL") if not gen.use_goto() else GotoHandler("fail") + gen.emit_arg_processing(error=error) + gen.emit_call() + gen.emit_error_handling() + + +def generate_arg_check( + name: str, + typ: RType, + emitter: Emitter, + error: ErrorHandler | None = None, + *, + optional: bool = False, + raise_exception: bool = True, + bitmap_arg_index: int = 0, +) -> None: + """Insert a runtime check for argument and unbox if necessary. + + The object is named PyObject *obj_{}. This is expected to generate + a value of name arg_{} (unboxed if necessary). For each primitive a runtime + check ensures the correct type. + """ + error = error or AssignHandler() + if typ.is_unboxed: + if typ.error_overlap and optional: + # Update bitmap is value is provided. + init = emitter.c_undefined_value(typ) + emitter.emit_line(f"{emitter.ctype(typ)} arg_{name} = {init};") + emitter.emit_line(f"if (obj_{name} != NULL) {{") + bitmap = bitmap_name(bitmap_arg_index // BITMAP_BITS) + emitter.emit_line(f"{bitmap} |= 1 << {bitmap_arg_index & (BITMAP_BITS - 1)};") + emitter.emit_unbox( + f"obj_{name}", + f"arg_{name}", + typ, + declare_dest=False, + raise_exception=raise_exception, + error=error, + borrow=True, + ) + emitter.emit_line("}") + else: + # Borrow when unboxing to avoid reference count manipulation. + emitter.emit_unbox( + f"obj_{name}", + f"arg_{name}", + typ, + declare_dest=True, + raise_exception=raise_exception, + error=error, + borrow=True, + optional=optional, + ) + elif is_object_rprimitive(typ): + # Object is trivial since any object is valid + if optional: + emitter.emit_line(f"PyObject *arg_{name};") + emitter.emit_line(f"if (obj_{name} == NULL) {{") + emitter.emit_line(f"arg_{name} = {emitter.c_error_value(typ)};") + emitter.emit_lines("} else {", f"arg_{name} = obj_{name}; ", "}") + else: + emitter.emit_line(f"PyObject *arg_{name} = obj_{name};") + else: + emitter.emit_cast( + f"obj_{name}", + f"arg_{name}", + typ, + declare_dest=True, + raise_exception=raise_exception, + error=error, + optional=optional, + ) + + +class WrapperGenerator: + """Helper that simplifies the generation of wrapper functions.""" + + # TODO: Use this for more wrappers + + def __init__(self, cl: ClassIR | None, emitter: Emitter) -> None: + self.cl = cl + self.emitter = emitter + self.cleanups: list[str] = [] + self.optional_args: list[RuntimeArg] = [] + self.traceback_code = "" + + def set_target(self, fn: FuncIR) -> None: + """Set the wrapped function. + + It's fine to modify the attributes initialized here later to customize + the wrapper function. + """ + self.target_name = fn.name + self.target_cname = fn.cname(self.emitter.names) + self.num_bitmap_args = fn.sig.num_bitmap_args + if self.num_bitmap_args: + self.args = fn.args[: -self.num_bitmap_args] + else: + self.args = fn.args + self.arg_names = [arg.name for arg in self.args] + self.ret_type = fn.ret_type + + def wrapper_name(self) -> str: + """Return the name of the wrapper function.""" + return "{}{}{}".format( + DUNDER_PREFIX, + self.target_name, + self.cl.name_prefix(self.emitter.names) if self.cl else "", + ) + + def use_goto(self) -> bool: + """Do we use a goto for error handling (instead of straight return)?""" + return bool(self.cleanups or self.traceback_code) + + def emit_header(self) -> None: + """Emit the function header of the wrapper implementation.""" + input_args = ", ".join(f"PyObject *obj_{arg}" for arg in self.arg_names) + self.emitter.emit_line( + "static PyObject *{name}({input_args}) {{".format( + name=self.wrapper_name(), input_args=input_args + ) + ) + + def emit_arg_processing( + self, error: ErrorHandler | None = None, raise_exception: bool = True + ) -> None: + """Emit validation and unboxing of arguments.""" + error = error or self.error() + bitmap_arg_index = 0 + for arg_name, arg in zip(self.arg_names, self.args): + # Suppress the argument check for *args/**kwargs, since we know it must be right. + typ = arg.type if arg.kind not in (ARG_STAR, ARG_STAR2) else object_rprimitive + optional = arg in self.optional_args + generate_arg_check( + arg_name, + typ, + self.emitter, + error, + raise_exception=raise_exception, + optional=optional, + bitmap_arg_index=bitmap_arg_index, + ) + if optional and typ.error_overlap: + bitmap_arg_index += 1 + + def emit_call(self, not_implemented_handler: str = "") -> None: + """Emit call to the wrapper function. + + If not_implemented_handler is non-empty, use this C code to handle + a NotImplemented return value (if it's possible based on the return type). + """ + native_args = ", ".join(f"arg_{arg}" for arg in self.arg_names) + if self.num_bitmap_args: + bitmap_args = ", ".join( + [bitmap_name(i) for i in reversed(range(self.num_bitmap_args))] + ) + native_args = f"{native_args}, {bitmap_args}" + + ret_type = self.ret_type + emitter = self.emitter + if ret_type.is_unboxed or self.use_goto(): + # TODO: The Py_RETURN macros return the correct PyObject * with reference count + # handling. Are they relevant? + emitter.emit_line( + "{}retval = {}{}({});".format( + emitter.ctype_spaced(ret_type), NATIVE_PREFIX, self.target_cname, native_args + ) + ) + emitter.emit_lines(*self.cleanups) + if ret_type.is_unboxed: + emitter.emit_error_check("retval", ret_type, "return NULL;") + emitter.emit_box("retval", "retbox", ret_type, declare_dest=True) + + emitter.emit_line("return {};".format("retbox" if ret_type.is_unboxed else "retval")) + else: + if not_implemented_handler and not isinstance(ret_type, RInstance): + # The return value type may overlap with NotImplemented. + emitter.emit_line( + "PyObject *retbox = {}{}({});".format( + NATIVE_PREFIX, self.target_cname, native_args + ) + ) + emitter.emit_lines( + "if (retbox == Py_NotImplemented) {", + not_implemented_handler, + "}", + "return retbox;", + ) + else: + emitter.emit_line(f"return {NATIVE_PREFIX}{self.target_cname}({native_args});") + # TODO: Tracebacks? + + def error(self) -> ErrorHandler: + """Figure out how to deal with errors in the wrapper.""" + if self.cleanups or self.traceback_code: + # We'll have a label at the end with error handling code. + return GotoHandler("fail") + else: + # Nothing special needs to done to handle errors, so just return. + return ReturnHandler("NULL") + + def emit_error_handling(self) -> None: + """Emit error handling block at the end of the wrapper, if needed.""" + emitter = self.emitter + if self.use_goto(): + emitter.emit_label("fail") + emitter.emit_lines(*self.cleanups) + if self.traceback_code: + emitter.emit_line(self.traceback_code) + emitter.emit_line("return NULL;") + + def finish(self) -> None: + self.emitter.emit_line("}") diff --git a/mypyc/codegen/literals.py b/mypyc/codegen/literals.py new file mode 100644 index 000000000000..2c4ab0c1dc2e --- /dev/null +++ b/mypyc/codegen/literals.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +from typing import Final, FrozenSet, Tuple, Union +from typing_extensions import TypeGuard + +# Supported Python literal types. All tuple / frozenset items must have supported +# literal types as well, but we can't represent the type precisely. +LiteralValue = Union[ + str, bytes, int, bool, float, complex, Tuple[object, ...], FrozenSet[object], None +] + + +def _is_literal_value(obj: object) -> TypeGuard[LiteralValue]: + return isinstance(obj, (str, bytes, int, float, complex, tuple, frozenset, type(None))) + + +# Some literals are singletons and handled specially (None, False and True) +NUM_SINGLETONS: Final = 3 + + +class Literals: + """Collection of literal values used in a compilation group and related helpers.""" + + def __init__(self) -> None: + # Each dict maps value to literal index (0, 1, ...) + self.str_literals: dict[str, int] = {} + self.bytes_literals: dict[bytes, int] = {} + self.int_literals: dict[int, int] = {} + self.float_literals: dict[float, int] = {} + self.complex_literals: dict[complex, int] = {} + self.tuple_literals: dict[tuple[object, ...], int] = {} + self.frozenset_literals: dict[frozenset[object], int] = {} + + def record_literal(self, value: LiteralValue) -> None: + """Ensure that the literal value is available in generated code.""" + if value is None or value is True or value is False: + # These are special cased and always present + return + if isinstance(value, str): + str_literals = self.str_literals + if value not in str_literals: + str_literals[value] = len(str_literals) + elif isinstance(value, bytes): + bytes_literals = self.bytes_literals + if value not in bytes_literals: + bytes_literals[value] = len(bytes_literals) + elif isinstance(value, int): + int_literals = self.int_literals + if value not in int_literals: + int_literals[value] = len(int_literals) + elif isinstance(value, float): + float_literals = self.float_literals + if value not in float_literals: + float_literals[value] = len(float_literals) + elif isinstance(value, complex): + complex_literals = self.complex_literals + if value not in complex_literals: + complex_literals[value] = len(complex_literals) + elif isinstance(value, tuple): + tuple_literals = self.tuple_literals + if value not in tuple_literals: + for item in value: + assert _is_literal_value(item) + self.record_literal(item) + tuple_literals[value] = len(tuple_literals) + elif isinstance(value, frozenset): + frozenset_literals = self.frozenset_literals + if value not in frozenset_literals: + for item in value: + assert _is_literal_value(item) + self.record_literal(item) + frozenset_literals[value] = len(frozenset_literals) + else: + assert False, "invalid literal: %r" % value + + def literal_index(self, value: LiteralValue) -> int: + """Return the index to the literals array for given value.""" + # The array contains first None and booleans, followed by all str values, + # followed by bytes values, etc. + if value is None: + return 0 + elif value is False: + return 1 + elif value is True: + return 2 + n = NUM_SINGLETONS + if isinstance(value, str): + return n + self.str_literals[value] + n += len(self.str_literals) + if isinstance(value, bytes): + return n + self.bytes_literals[value] + n += len(self.bytes_literals) + if isinstance(value, int): + return n + self.int_literals[value] + n += len(self.int_literals) + if isinstance(value, float): + return n + self.float_literals[value] + n += len(self.float_literals) + if isinstance(value, complex): + return n + self.complex_literals[value] + n += len(self.complex_literals) + if isinstance(value, tuple): + return n + self.tuple_literals[value] + n += len(self.tuple_literals) + if isinstance(value, frozenset): + return n + self.frozenset_literals[value] + assert False, "invalid literal: %r" % value + + def num_literals(self) -> int: + # The first three are for None, True and False + return ( + NUM_SINGLETONS + + len(self.str_literals) + + len(self.bytes_literals) + + len(self.int_literals) + + len(self.float_literals) + + len(self.complex_literals) + + len(self.tuple_literals) + + len(self.frozenset_literals) + ) + + # The following methods return the C encodings of literal values + # of different types + + def encoded_str_values(self) -> list[bytes]: + return _encode_str_values(self.str_literals) + + def encoded_int_values(self) -> list[bytes]: + return _encode_int_values(self.int_literals) + + def encoded_bytes_values(self) -> list[bytes]: + return _encode_bytes_values(self.bytes_literals) + + def encoded_float_values(self) -> list[str]: + return _encode_float_values(self.float_literals) + + def encoded_complex_values(self) -> list[str]: + return _encode_complex_values(self.complex_literals) + + def encoded_tuple_values(self) -> list[str]: + return self._encode_collection_values(self.tuple_literals) + + def encoded_frozenset_values(self) -> list[str]: + return self._encode_collection_values(self.frozenset_literals) + + def _encode_collection_values( + self, values: dict[tuple[object, ...], int] | dict[frozenset[object], int] + ) -> list[str]: + """Encode tuple/frozenset values into a C array. + + The format of the result is like this: + + + + + ... + + + ... + """ + value_by_index = {index: value for value, index in values.items()} + result = [] + count = len(values) + result.append(str(count)) + for i in range(count): + value = value_by_index[i] + result.append(str(len(value))) + for item in value: + assert _is_literal_value(item) + index = self.literal_index(item) + result.append(str(index)) + return result + + +def _encode_str_values(values: dict[str, int]) -> list[bytes]: + value_by_index = {index: value for value, index in values.items()} + result = [] + line: list[bytes] = [] + line_len = 0 + for i in range(len(values)): + value = value_by_index[i] + c_literal = format_str_literal(value) + c_len = len(c_literal) + if line_len > 0 and line_len + c_len > 70: + result.append(format_int(len(line)) + b"".join(line)) + line = [] + line_len = 0 + line.append(c_literal) + line_len += c_len + if line: + result.append(format_int(len(line)) + b"".join(line)) + result.append(b"") + return result + + +def _encode_bytes_values(values: dict[bytes, int]) -> list[bytes]: + value_by_index = {index: value for value, index in values.items()} + result = [] + line: list[bytes] = [] + line_len = 0 + for i in range(len(values)): + value = value_by_index[i] + c_init = format_int(len(value)) + c_len = len(c_init) + len(value) + if line_len > 0 and line_len + c_len > 70: + result.append(format_int(len(line)) + b"".join(line)) + line = [] + line_len = 0 + line.append(c_init + value) + line_len += c_len + if line: + result.append(format_int(len(line)) + b"".join(line)) + result.append(b"") + return result + + +def format_int(n: int) -> bytes: + """Format an integer using a variable-length binary encoding.""" + if n < 128: + a = [n] + else: + a = [] + while n > 0: + a.insert(0, n & 0x7F) + n >>= 7 + for i in range(len(a) - 1): + # If the highest bit is set, more 7-bit digits follow + a[i] |= 0x80 + return bytes(a) + + +def format_str_literal(s: str) -> bytes: + utf8 = s.encode("utf-8", errors="surrogatepass") + return format_int(len(utf8)) + utf8 + + +def _encode_int_values(values: dict[int, int]) -> list[bytes]: + """Encode int values into C strings. + + Values are stored in base 10 and separated by 0 bytes. + """ + value_by_index = {index: value for value, index in values.items()} + result = [] + line: list[bytes] = [] + line_len = 0 + for i in range(len(values)): + value = value_by_index[i] + encoded = b"%d" % value + if line_len > 0 and line_len + len(encoded) > 70: + result.append(format_int(len(line)) + b"\0".join(line)) + line = [] + line_len = 0 + line.append(encoded) + line_len += len(encoded) + if line: + result.append(format_int(len(line)) + b"\0".join(line)) + result.append(b"") + return result + + +def float_to_c(x: float) -> str: + """Return C literal representation of a float value.""" + s = str(x) + if s == "inf": + return "INFINITY" + elif s == "-inf": + return "-INFINITY" + elif s == "nan": + return "NAN" + return s + + +def _encode_float_values(values: dict[float, int]) -> list[str]: + """Encode float values into a C array values. + + The result contains the number of values followed by individual values. + """ + value_by_index = {index: value for value, index in values.items()} + result = [] + num = len(values) + result.append(str(num)) + for i in range(num): + value = value_by_index[i] + result.append(float_to_c(value)) + return result + + +def _encode_complex_values(values: dict[complex, int]) -> list[str]: + """Encode float values into a C array values. + + The result contains the number of values followed by pairs of doubles + representing complex numbers. + """ + value_by_index = {index: value for value, index in values.items()} + result = [] + num = len(values) + result.append(str(num)) + for i in range(num): + value = value_by_index[i] + result.append(float_to_c(value.real)) + result.append(float_to_c(value.imag)) + return result diff --git a/mypyc/common.py b/mypyc/common.py new file mode 100644 index 000000000000..31567c689c34 --- /dev/null +++ b/mypyc/common.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import sys +import sysconfig +from typing import Any, Dict, Final + +from mypy.util import unnamed_function + +PREFIX: Final = "CPyPy_" # Python wrappers +NATIVE_PREFIX: Final = "CPyDef_" # Native functions etc. +DUNDER_PREFIX: Final = "CPyDunder_" # Wrappers for exposing dunder methods to the API +REG_PREFIX: Final = "cpy_r_" # Registers +STATIC_PREFIX: Final = "CPyStatic_" # Static variables (for literals etc.) +TYPE_PREFIX: Final = "CPyType_" # Type object struct +MODULE_PREFIX: Final = "CPyModule_" # Cached modules +TYPE_VAR_PREFIX: Final = "CPyTypeVar_" # Type variables when using new-style Python 3.12 syntax +ATTR_PREFIX: Final = "_" # Attributes + +ENV_ATTR_NAME: Final = "__mypyc_env__" +NEXT_LABEL_ATTR_NAME: Final = "__mypyc_next_label__" +TEMP_ATTR_NAME: Final = "__mypyc_temp__" +LAMBDA_NAME: Final = "__mypyc_lambda__" +PROPSET_PREFIX: Final = "__mypyc_setter__" +SELF_NAME: Final = "__mypyc_self__" + +# Max short int we accept as a literal is based on 32-bit platforms, +# so that we can just always emit the same code. + +TOP_LEVEL_NAME: Final = "__top_level__" # Special function representing module top level + +# Maximal number of subclasses for a class to trigger fast path in isinstance() checks. +FAST_ISINSTANCE_MAX_SUBCLASSES: Final = 2 + +# Size of size_t, if configured. +SIZEOF_SIZE_T_SYSCONFIG: Final = sysconfig.get_config_var("SIZEOF_SIZE_T") + +SIZEOF_SIZE_T: Final = ( + int(SIZEOF_SIZE_T_SYSCONFIG) + if SIZEOF_SIZE_T_SYSCONFIG is not None + else (sys.maxsize + 1).bit_length() // 8 +) + +IS_32_BIT_PLATFORM: Final = int(SIZEOF_SIZE_T) == 4 + +PLATFORM_SIZE = 4 if IS_32_BIT_PLATFORM else 8 + +# Maximum value for a short tagged integer. +MAX_SHORT_INT: Final = 2 ** (8 * int(SIZEOF_SIZE_T) - 2) - 1 + +# Minimum value for a short tagged integer. +MIN_SHORT_INT: Final = -(MAX_SHORT_INT) - 1 + +# Maximum value for a short tagged integer represented as a C integer literal. +# +# Note: Assume that the compiled code uses the same bit width as mypyc +MAX_LITERAL_SHORT_INT: Final = MAX_SHORT_INT +MIN_LITERAL_SHORT_INT: Final = -MAX_LITERAL_SHORT_INT - 1 + +# Description of the C type used to track the definedness of attributes and +# the presence of argument default values that have types with overlapping +# error values. Each tracked attribute/argument has a dedicated bit in the +# relevant bitmap. +BITMAP_TYPE: Final = "uint32_t" +BITMAP_BITS: Final = 32 + +# Runtime C library files +RUNTIME_C_FILES: Final = [ + "init.c", + "getargs.c", + "getargsfast.c", + "int_ops.c", + "float_ops.c", + "str_ops.c", + "bytes_ops.c", + "list_ops.c", + "dict_ops.c", + "set_ops.c", + "tuple_ops.c", + "exc_ops.c", + "misc_ops.c", + "generic_ops.c", + "pythonsupport.c", +] + + +JsonDict = Dict[str, Any] + + +def shared_lib_name(group_name: str) -> str: + """Given a group name, return the actual name of its extension module. + + (This just adds a suffix to the final component.) + """ + return f"{group_name}__mypyc" + + +def short_name(name: str) -> str: + if name.startswith("builtins."): + return name[9:] + return name + + +def use_vectorcall(capi_version: tuple[int, int]) -> bool: + # We can use vectorcalls to make calls on Python 3.8+ (PEP 590). + return capi_version >= (3, 8) + + +def use_method_vectorcall(capi_version: tuple[int, int]) -> bool: + # We can use a dedicated vectorcall API to call methods on Python 3.9+. + return capi_version >= (3, 9) + + +def get_id_from_name(name: str, fullname: str, line: int) -> str: + """Create a unique id for a function. + + This creates an id that is unique for any given function definition, so that it can be used as + a dictionary key. This is usually the fullname of the function, but this is different in that + it handles the case where the function is named '_', in which case multiple different functions + could have the same name.""" + if unnamed_function(name): + return f"{fullname}.{line}" + else: + return fullname + + +def short_id_from_name(func_name: str, shortname: str, line: int | None) -> str: + if unnamed_function(func_name): + assert line is not None + partial_name = f"{shortname}.{line}" + else: + partial_name = shortname + return partial_name + + +def bitmap_name(index: int) -> str: + if index == 0: + return "__bitmap" + return f"__bitmap{index + 1}" diff --git a/mypyc/crash.py b/mypyc/crash.py new file mode 100644 index 000000000000..19136ea2f1de --- /dev/null +++ b/mypyc/crash.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import sys +import traceback +from contextlib import contextmanager +from typing import Iterator, NoReturn + + +@contextmanager +def catch_errors(module_path: str, line: int) -> Iterator[None]: + try: + yield + except Exception: + crash_report(module_path, line) + + +def crash_report(module_path: str, line: int) -> NoReturn: + # Adapted from report_internal_error in mypy + err = sys.exc_info()[1] + tb = traceback.extract_stack()[:-4] + # Excise all the traceback from the test runner + for i, x in enumerate(tb): + if x.name == "pytest_runtest_call": + tb = tb[i + 1 :] + break + tb2 = traceback.extract_tb(sys.exc_info()[2])[1:] + print("Traceback (most recent call last):") + for s in traceback.format_list(tb + tb2): + print(s.rstrip("\n")) + print(f"{module_path}:{line}: {type(err).__name__}: {err}") + raise SystemExit(2) diff --git a/mypyc/doc/Makefile b/mypyc/doc/Makefile new file mode 100644 index 000000000000..d4bb2cbb9edd --- /dev/null +++ b/mypyc/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/mypyc/doc/bool_operations.rst b/mypyc/doc/bool_operations.rst new file mode 100644 index 000000000000..8e46514b06c4 --- /dev/null +++ b/mypyc/doc/bool_operations.rst @@ -0,0 +1,27 @@ +.. _bool-ops: + +Native boolean operations +========================= + +Operations on ``bool`` values that are listed here have fast, +optimized implementations. + +Construction +------------ + +* ``True`` +* ``False`` +* ``bool(obj)`` + +Operators +--------- + +* ``b1 and b2`` +* ``b1 or b2`` +* ``not b`` + +Functions +--------- + +* ``any(expr for ... in ...)`` +* ``all(expr for ... in ...)`` diff --git a/mypyc/doc/bytes_operations.rst b/mypyc/doc/bytes_operations.rst new file mode 100644 index 000000000000..038da6391949 --- /dev/null +++ b/mypyc/doc/bytes_operations.rst @@ -0,0 +1,46 @@ +.. _bytes-ops: + +Native bytes operations +======================== + +These ``bytes`` operations have fast, optimized implementations. Other +bytes operations use generic implementations that are often slower. + +Construction +------------ + +* Bytes literal +* ``bytes(x: list)`` + +Operators +--------- + +* Concatenation (``b1 + b2``) +* Indexing (``b[n]``) +* Slicing (``b[n:m]``, ``b[n:]``, ``b[:m]``) +* Comparisons (``==``, ``!=``) + +.. _bytes-methods: + +Methods +------- + +* ``b.decode()`` +* ``b.decode(encoding: str)`` +* ``b.decode(encoding: str, errors: str)`` +* ``b.join(x: Iterable)`` + +.. note:: + + :ref:`str.encode() ` is also optimized. + +Formatting +---------- + +A subset of % formatting operations are optimized (``b"..." % (...)``). + +Functions +--------- + +* ``len(b: bytes)`` +* ``ord(b: bytes)`` diff --git a/mypyc/doc/compilation_units.rst b/mypyc/doc/compilation_units.rst new file mode 100644 index 000000000000..732f91723ee1 --- /dev/null +++ b/mypyc/doc/compilation_units.rst @@ -0,0 +1,20 @@ +.. _compilation-units: + +Compilation units +================= + +When you run mypyc to compile a set of modules, these modules form a +*compilation unit*. Mypyc will use early binding for references within +the compilation unit. + +If you run mypyc multiple times to compile multiple sets of modules, +each invocation will result in a new compilation unit. References +between separate compilation units will fall back to late binding, +i.e. looking up names using Python namespace dictionaries. Also, all +calls will use the slower Python calling convention, where arguments +and the return value will be boxed (and potentially unboxed again in +the called function). + +For maximal performance, minimize interactions across compilation +units. The simplest way to achieve this is to compile your entire +program as a single compilation unit. diff --git a/mypyc/doc/conf.py b/mypyc/doc/conf.py new file mode 100644 index 000000000000..fdd98c12a221 --- /dev/null +++ b/mypyc/doc/conf.py @@ -0,0 +1,59 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +from __future__ import annotations + +import os +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("../..")) + +from mypy.version import __version__ as mypy_version + +# -- Project information ----------------------------------------------------- + +project = "mypyc" +copyright = "2020-2022, mypyc team" +author = "mypyc team" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = mypy_version.split("-")[0] +# The full version, including alpha/beta/rc tags. +release = mypy_version + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions: list[str] = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "furo" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] diff --git a/mypyc/doc/cpython-timings.md b/mypyc/doc/cpython-timings.md new file mode 100644 index 000000000000..360d9cfa3961 --- /dev/null +++ b/mypyc/doc/cpython-timings.md @@ -0,0 +1,25 @@ +# Timings of CPython Operations + +Here are some *very rough* approximate timings of CPython interpreter +operations: + +* `f(1)` (empty function body): 70-90ns +* `f(n=1)` (empty function body): 90-110ns +* `o.x`: 30-40ns +* `o.f(1)` (empty method body): 80-160ns +* `Cls(1)` (initialize attribute in `__init__`): 290-330ns +* `x + y` (integers): 20-35ns +* `a[i]` (list) : 20-40ns +* `[i]` (also dealloc): 35-55ns +* `a.append(i)` (list, average over 5 appends): 70ns +* `d[s]` (dict, shared str key): 20ns +* `d[s] = i` (dict, shared str key): 40ns +* `isinstance(x, A)`: 100ns +* `(x, y)`: 20-35ns +* `x, y = t` (tuple expand): 10ns + +Note that these results are very imprecise due to many factors, but +these should give a rough idea of the relative costs of various +operations. + +Details: CPython 3.6.2, Macbook Pro 15" (Mid 2015), macOS Sierra diff --git a/mypyc/doc/dev-intro.md b/mypyc/doc/dev-intro.md new file mode 100644 index 000000000000..461a19d37121 --- /dev/null +++ b/mypyc/doc/dev-intro.md @@ -0,0 +1,552 @@ +# Introduction for Mypyc Contributors + +This is a short introduction aimed at anybody who is interested in +contributing to mypyc, or anybody who is curious to understand how +mypyc works internally. + +## Key Differences from Python + +Code compiled using mypyc is often much faster than CPython since it +does these things differently: + +* Mypyc generates C that is compiled to native code, instead of + compiling to interpreted byte code, which CPython uses. Interpreted + byte code always has some interpreter overhead, which slows things + down. + +* Mypyc doesn't let you arbitrarily monkey patch classes and functions + in compiled modules. This allows *early binding* -- mypyc + statically binds calls to compiled functions, instead of going + through a namespace dictionary. Mypyc can also call methods of + compiled classes using vtables, which are more efficient than + dictionary lookups used by CPython. + +* Mypyc compiles classes to C extension classes, which are generally + more efficient than normal Python classes. They use an efficient, + fixed memory representation (essentially a C struct). This lets us + use direct memory access instead of (typically) two hash table + lookups to access an attribute. + +* As a result of early binding, compiled code can use C calls to call + compiled functions. Keyword arguments can be translated to + positional arguments during compilation. Thus most calls to native + functions and methods directly map to simple C calls. CPython calls + are quite expensive, since mapping of keyword arguments, `*args`, + and so on has to mostly happen at runtime. + +* Compiled code has runtime type checks to ensure that runtimes types + match the declared static types. Compiled code can thus make + assumptions about the types of expressions, resulting in both faster + and smaller code, since many runtime type checks performed by the + CPython interpreter can be omitted. + +* Compiled code can often use unboxed (not heap allocated) + representations for integers, booleans and tuples. + +## Supported Python Features + +Mypyc supports a large subset of Python. Note that if you try to +compile something that is not supported, you may not always get a very +good error message. + +Here are some major things that aren't yet supported in compiled code: + +* Many dunder methods (only some work, such as `__init__` and `__eq__`) +* Monkey patching compiled functions or classes +* General multiple inheritance (a limited form is supported) +* Named tuple defined using the class-based syntax +* Defining protocols + +We are generally happy to accept contributions that implement new Python +features. + +## Development Environment + +First you should set up the mypy development environment as described in +the [mypy docs](https://github.com/python/mypy/blob/master/README.md). +macOS, Linux and Windows are supported. + +## Compiling and Running Programs + +When working on a mypyc feature or a fix, you'll often need to run +compiled code. For example, you may want to do interactive testing or +to run benchmarks. This is also handy if you want to inspect the +generated C code (see Inspecting Generated C). + +Run `mypyc` to compile a module to a C extension using your +development version of mypyc: + +``` +$ mypyc program.py +``` + +This will generate a C extension for `program` in the current working +directory. For example, on a Linux system the generated file may be +called `program.cpython-37m-x86_64-linux-gnu.so`. + +Since C extensions can't be run as programs, use `python3 -c` to run +the compiled module as a program: + +``` +$ python3 -c "import program" +``` + +Note that `__name__` in `program.py` will now be `program`, not +`__main__`! + +You can manually delete the C extension to get back to an interpreted +version (this example works on Linux): + +``` +$ rm program.*.so +``` + +Another option is to invoke mypyc through tests (see Testing below). + +## High-level Overview of Mypyc + +Mypyc compiles a Python module (or a set of modules) to C, and +compiles the generated C to a Python C extension module (or +modules). You can compile only a subset of your program to C -- +compiled and interpreted code can freely and transparently +interact. You can also freely use any Python libraries (including C +extensions) in compiled code. + +Mypyc will only make compiled code faster. To see a significant +speedup, you must make sure that most of the time is spent in compiled +code -- and not in libraries, for example. + +Mypyc has these passes: + +* Type check the code using mypy and infer types for variables and + expressions. This produces a mypy AST (defined in `mypy.nodes`) and + a type map that describes the inferred types (`mypy.types.Type`) of + all expressions (as PEP 484 types). + +* Translate the mypy AST into a mypyc-specific intermediate representation (IR). + * The IR is defined in `mypyc.ir` (see below for an explanation of the IR). + * Various primitive operations used in the IR are defined in `mypyc.primitives`. + * The translation to IR happens in `mypyc.irbuild`. The top-level logic is in + `mypyc.irbuild.main`. + +* Insert checks for uses of potentially uninitialized variables + (`mypyc.transform.uninit`). + +* Insert exception handling (`mypyc.transform.exceptions`). + +* Insert explicit reference count inc/dec opcodes (`mypyc.transform.refcount`). + +* Translate the IR into C (`mypyc.codegen`). + +* Compile the generated C code using a C compiler (`mypyc.build`). + +## Useful Background Information + +Beyond the mypy documentation, here are some things that are helpful to +know for mypyc contributors: + +* Experience with C + ([The C Programming Language](https://en.wikipedia.org/wiki/The_C_Programming_Language) + is a classic book about C) +* Basic familiarity with the Python C API (see + [Python C API documentation](https://docs.python.org/3/c-api/intro.html)). [Extending and Embedding the Python Interpreter](https://docs.python.org/3/extending/index.html) is a good tutorial for beginners. +* Basics of compilers (see the + [mypy wiki](https://github.com/python/mypy/wiki/Learning-Resources) + for some ideas) + +## Mypyc Intermediate Representation (IR) + +The mypyc IR is defined in `mypyc.ir`. It covers several key concepts +that are essential to understand by all mypyc contributors: + +* `mypyc.ir.ops.Op` is an Abstract Base Class for all IR + operations. These are low-level and generally map to simple + fragments of C each. Mypy expressions are translated to + linear sequences of these ops. + +* `mypyc.ir.ops.BasicBlock` is a container of a sequence of ops with a + branch/goto/return at the end, and no branch/goto/return ops in the + middle. Each function is compiled to a bunch of basic blocks. + +* `mypyc.ir.rtypes.RType` and its subclasses are the types used for + everything in the IR. These are lower-level and simpler than mypy or + PEP 484 types. For example, there are no general-purpose generic + types types here. Each `List[X]` type (for any `X`) is represented + by a single `list` type, for example. + +* Primitive types are special RTypes of which mypyc has some special + understanding, and there are typically some specialized + ops. Examples include `int` (referred to as `int_rprimitive` in the + code) and `list` (`list_rprimitive`). Python types for which there + is no specific RType type will be represented by the catch-all + `object_rprimitive` type. + +* Instances of compiled classes are generally represented using the + `RInstance` type. Classes are compiled to C extension classes and + contain vtables for fast method calls and fast attribute access. + +* IR representations of functions and classes live in + `mypyc.ir.func_ir` and `mypyc.ir.class_ir`, respectively. + +Look at the docstrings and comments in `mypyc.ir` for additional +information. See the test cases in +`mypyc/test-data/irbuild-basic.test` for examples of what the IR looks +like in a pretty-printed form. + +## Testing overview + +Most mypyc test cases are defined in the same format (`.test`) as used +for test cases for mypy. Look at mypy developer documentation for a +general overview of how things work. Test cases live under +`mypyc/test-data/`, and you can run all mypyc tests via `pytest +-q mypyc`. If you don't make changes to code under `mypy/`, it's not +important to regularly run mypy tests during development. + +You can use `python runtests.py mypyc-fast` to run a subset of mypyc +tests that covers most functionality but runs significantly quicker +than the entire test suite. + +When you create a PR, we have Continuous Integration jobs set up that +compile mypy using mypyc and run the mypy test suite using the +compiled mypy. This will sometimes catch additional issues not caught +by the mypyc test suite. It's okay to not do this in your local +development environment. + +We discuss writing tests in more detail later in this document. + +## Inspecting Generated IR + +It's often useful to look at the generated IR when debugging issues or +when trying to understand how mypyc compiles some code. When you +compile some module by running `mypyc`, mypyc will write the +pretty-printed IR into `build/ops.txt`. This is the final IR that +includes the output from exception and reference count handling +insertion passes. + +We also have tests that verify the generate IR +(`mypyc/test-data/irbuild-*.text`). + +## Type-checking Mypyc + +`./runtests.py self` type checks mypy and mypyc. This is pretty slow, +however, since it's using an uncompiled mypy. + +Installing a released version of mypy using `pip` (which is compiled) +and using `dmypy` (mypy daemon) is a much, much faster way to type +check mypyc during development. + +## Value Representation + +Mypyc uses a tagged pointer representation for values of type `int` +(`CPyTagged`), `char` for booleans, and C structs for tuples. For most +other objects mypyc uses the CPython `PyObject *`. + +Python integers that fit in 31/63 bits (depending on whether we are on +a 32-bit or 64-bit platform) are represented as C integers +(`CPyTagged`) shifted left by 1. Integers that don't fit in this +representation are represented as pointers to a `PyObject *` (this is +always a Python `int` object) with the least significant bit +set. Tagged integer operations are defined in `mypyc/lib-rt/int_ops.c` +and `mypyc/lib-rt/CPy.h`. + +There are also low-level integer types, such as `int32` (see +`mypyc.ir.rtypes`), that don't use the tagged representation. These +types are not exposed to users, but they are used in generated code. + +## Overview of Generated C + +Mypyc compiles a function into two functions, a native function and +a wrapper function: + +* The native function takes a fixed number of C arguments with the + correct C types. It assumes that all argument have correct types. + +* The wrapper function conforms to the Python C API calling convention + and takes an arbitrary set of arguments. It processes the arguments, + checks their types, unboxes values with special representations and + calls the native function. The return value from the native function + is translated back to a Python object ("boxing"). + +Calls to other compiled functions don't go through the Python module +namespace but directly call the target native C function. This makes +calls very fast compared to CPython. + +The generated code does runtime checking so that it can assume that +values always have the declared types. Whenever accessing CPython +values which might have unexpected types we need to insert a runtime +type check operation. For example, when getting a list item we need to +insert a runtime type check (an unbox or a cast operation), since +Python lists can contain arbitrary objects. + +The generated code uses various helpers defined in +`mypyc/lib-rt/CPy.h`. The implementations are in various `.c` files +under `mypyc/lib-rt`. + +## Inspecting Generated C + +It's often useful to inspect the C code genenerate by mypyc to debug +issues. Mypyc stores the generated C code as `build/__native.c`. +Compiled native functions have the prefix `CPyDef_`, while wrapper +functions used for calling functions from interpreted Python code have +the `CPyPy_` prefix. + +## Other Important Limitations + +All of these limitations will likely be fixed in the future: + +* We don't detect stack overflows. + +* We don't handle Ctrl-C in compiled code. + +## Hints for Implementing Typical Mypyc Features + +This section gives an overview of where to look for and +what to do to implement specific kinds of mypyc features. + +### Testing + +Our bread-and-butter testing strategy is compiling code with mypyc and +running it. There are downsides to this (kind of slow, tests a huge +number of components at once, insensitive to the particular details of +the IR), but there really is no substitute for running code. You can +also write tests that test the generated IR, however. + +### Tests that compile and run code + +Test cases that compile and run code are located in +`mypyc/test-data/run*.test` and the test runner is in +`mypyc.test.test_run`. The code to compile comes after `[case +test]`. The code gets saved into the file `native.py`, and it +gets compiled into the module `native`. + +Each test case uses a non-compiled Python driver that imports the +`native` module and typically calls some compiled functions. Some +tests also perform assertions and print messages in the driver. + +If you don't provide a driver, a default driver is used. The default +driver just calls each module-level function that is prefixed with +`test_` and reports any uncaught exceptions as failures. (Failure to +build or a segfault also count as failures.) `testStringOps` in +`mypyc/test-data/run-strings.test` is an example of a test that uses +the default driver. + +You should usually use the default driver (don't include +`driver.py`). It's the simplest way to write most tests. + +Here's an example test case that uses the default driver: + +``` +[case testConcatenateLists] +def test_concat_lists() -> None: + assert [1, 2] + [5, 6] == [1, 2, 5, 6] + +def test_concat_empty_lists() -> None: + assert [] + [] == [] +``` + +There is one test case, `testConcatenateLists`. It has two sub-cases, +`test_concat_lists` and `test_concat_empty_lists`. Note that you can +use the pytest -k argument to only run `testConcetanateLists`, but you +can't filter tests at the sub-case level. + +It's recommended to have multiple sub-cases per test case, since each +test case has significant fixed overhead. Each test case is run in a +fresh Python subprocess. + +Many of the existing test cases provide a custom driver by having +`[file driver.py]`, followed by the driver implementation. Here the +driver is not compiled, which is useful if you want to test +interactions between compiled and non-compiled code. However, many of +the tests don't have a good reason to use a custom driver -- when they +were written, the default driver wasn't available. + +Test cases can also have a `[out]` section, which specifies the +expected contents of stdout the test case should produce. New test +cases should prefer assert statements to `[out]` sections. + +### IR tests + +If the specifics of the generated IR of a change is important +(because, for example, you want to make sure a particular optimization +is triggering), you should add a `mypyc.irbuild` test as well. Test +cases are located in `mypyc/test-data/irbuild-*.test` and the test +driver is in `mypyc.test.test_irbuild`. IR build tests do a direct +comparison of the IR output, so try to make the test as targeted as +possible so as to capture only the important details. (Many of our +existing IR build tests do not follow this advice, unfortunately!) + +If you pass the `--update-data` flag to pytest, it will automatically +update the expected output of any tests to match the actual +output. This is very useful for changing or creating IR build tests, +but make sure to carefully inspect the diff! + +You may also need to add some definitions to the stubs used for +builtins during tests (`mypyc/test-data/fixtures/ir.py`). We don't use +full typeshed stubs to run tests since they would seriously slow down +tests. + +### Benchmarking + +Many mypyc improvements attempt to make some operations faster. For +any such change, you should run some measurements to verify that +there actually is a measurable performance impact. + +A typical benchmark would initialize some data to be operated on, and +then measure time spent in some function. In particular, you should +not measure time needed to run the entire benchmark program, as this +would include Python startup overhead and other things that aren't +relevant. In general, for microbenchmarks, you want to do as little as +possible in the timed portion. So ideally you'll just have some loops +and the code under test. Be ready to provide your benchmark in code +review so that mypyc developers can check that the benchmark is fine +(writing a good benchmark is non-trivial). + +You should run a benchmark at least five times, in both original and +changed versions, ignore outliers, and report the average +runtime. Actual performance of a typical desktop or laptop computer is +quite variable, due to dynamic CPU clock frequency changes, background +processes, etc. If you observe a high variance in timings, you'll need +to run the benchmark more times. Also try closing most applications, +including web browsers. + +Interleave original and changed runs. Don't run 10 runs with variant A +followed by 10 runs with variant B, but run an A run, a B run, an A +run, etc. Otherwise you risk that the CPU frequency will be different +between variants. You can also try adding a delay of 5 to 20s between +runs to avoid CPU frequency changes. + +Instead of averaging over many measurements, you can try to adjust +your environment to provide more stable measurements. However, this +can be hard to do with some hardware, including many laptops. Victor +Stinner has written a series of blog posts about making measurements +stable: + +* https://vstinner.github.io/journey-to-stable-benchmark-system.html +* https://vstinner.github.io/journey-to-stable-benchmark-average.html + +### Adding C Helpers + +If you add an operation that compiles into a lot of C code, you may +also want to add a C helper function for the operation to make the +generated code smaller. Here is how to do this: + +* Declare the operation in `mypyc/lib-rt/CPy.h`. We avoid macros, and + we generally avoid inline functions to make it easier to target + additional backends in the future. + +* Consider adding a unit test for your C helper in `mypyc/lib-rt/test_capi.cc`. + We use + [Google Test](https://github.com/google/googletest) for writing + tests in C++. The framework is included in the repository under the + directory `googletest/`. The C unit tests are run as part of the + pytest test suite (`test_c_unit_test`). + +### Adding a Specialized Primitive Operation + +Mypyc speeds up operations on primitive types such as `list` and `int` +by having primitive operations specialized for specific types. These +operations are declared in `mypyc.primitives` (and +`mypyc/lib-rt/CPy.h`). For example, `mypyc.primitives.list_ops` +contains primitives that target list objects. + +The operation definitions are data driven: you specify the kind of +operation (such as a call to `builtins.len` or a binary addition) and +the operand types (such as `list_primitive`), and what code should be +generated for the operation. Mypyc does AST matching to find the most +suitable primitive operation automatically. + +Look at the existing primitive definitions and the docstrings in +`mypyc.primitives.registry` for examples and more information. + +### Adding a New Primitive Type + +Some types (typically Python Python built-in types), such as `int` and +`list`, are special cased in mypyc to generate optimized operations +specific to these types. We'll occasionally want to add additional +primitive types. + +Here are some hints about how to add support for a new primitive type +(this may be incomplete): + +* Decide whether the primitive type has an "unboxed" representation (a + representation that is not just `PyObject *`). For most types we'll + use a boxed representation, as it's easier to implement and more + closely matches Python semantics. + +* Create a new instance of `RPrimitive` to support the primitive type + and add it to `mypyc.ir.rtypes`. Make sure all the attributes are + set correctly and also define `_rprimitive` and + `is__rprimitive`. + +* Update `mypyc.irbuild.mapper.Mapper.type_to_rtype()`. + +* If the type is not unboxed, update `emit_cast` in `mypyc.codegen.emit`. + +If the type is unboxed, there are some additional steps: + +* Update `emit_box` in `mypyc.codegen.emit`. + +* Update `emit_unbox` in `mypyc.codegen.emit`. + +* Update `emit_inc_ref` and `emit_dec_ref` in `mypypc.codegen.emit`. + If the unboxed representation does not need reference counting, + these can be no-ops. + +* Update `emit_error_check` in `mypyc.codegen.emit`. + +* Update `emit_gc_visit` and `emit_gc_clear` in `mypyc.codegen.emit` + if the type has an unboxed representation with pointers. + +The above may be enough to allow you to declare variables with the +type, pass values around, perform runtime type checks, and use generic +fallback primitive operations to perform method calls, binary +operations, and so on. You likely also want to add some faster, +specialized primitive operations for the type (see Adding a +Specialized Primitive Operation above for how to do this). + +Add a test case to `mypyc/test-data/run*.test` to test compilation and +running compiled code. Ideas for things to test: + +* Test using the type as an argument. + +* Test using the type as a return value. + +* Test passing a value of the type to a function both within + compiled code and from regular Python code. Also test this + for return values. + +* Test using the type as list item type. Test both getting a list item + and setting a list item. + +### Supporting More Python Syntax + +Mypyc supports most Python syntax, but there are still some gaps. + +Support for syntactic sugar that doesn't need additional IR operations +typically only requires changes to `mypyc.irbuild`. + +Some new syntax also needs new IR primitives to be added to +`mypyc.primitives`. See `mypyc.primitives.registry` for documentation +about how to do this. + +### Other Hints + +* This developer documentation is not aimed to be very complete. Much + of our documentation is in comments and docstring in the code. If + something is unclear, study the code. + +* It can be useful to look through some recent PRs to get an idea of + what typical code changes, test cases, etc. look like. + +* Feel free to open GitHub issues with questions if you need help when + contributing, or ask questions in existing issues. Note that we only + support contributors. Mypyc is not (yet) an end-user product. You + can also ask questions in our Gitter chat + (https://gitter.im/mypyc-dev/community). + +## Undocumented Workflows + +These workflows would be useful for mypyc contributors. We should add +them to mypyc developer documentation: + +* How to inspect the generated IR before some transform passes. diff --git a/mypyc/doc/dict_operations.rst b/mypyc/doc/dict_operations.rst new file mode 100644 index 000000000000..e3104172133a --- /dev/null +++ b/mypyc/doc/dict_operations.rst @@ -0,0 +1,59 @@ +.. _dict-ops: + +Native dict operations +====================== + +These ``dict`` operations have fast, optimized implementations. Other +dictionary operations use generic implementations that are often slower. + +Construction +------------ + +Construct dict from keys and values: + +* ``{key: value, ...}`` + +Construct empty dict: + +* ``{}`` +* ``dict()`` + +Construct dict from another object: + +* ``dict(d: dict)`` +* ``dict(x: Iterable)`` + +Dict comprehensions: + +* ``{...: ... for ... in ...}`` +* ``{...: ... for ... in ... if ...}`` + +Operators +--------- + +* ``d[key]`` +* ``value in d`` + +Statements +---------- + +* ``d[key] = value`` +* ``for key in d:`` + +Methods +------- + +* ``d.get(key)`` +* ``d.get(key, default)`` +* ``d.keys()`` +* ``d.values()`` +* ``d.items()`` +* ``d.copy()`` +* ``d.clear()`` +* ``d1.update(d2: dict)`` +* ``d.update(x: Iterable)`` + +Functions +--------- + +* ``len(d: dict)`` diff --git a/mypyc/doc/differences_from_python.rst b/mypyc/doc/differences_from_python.rst new file mode 100644 index 000000000000..f1d4d05a3a87 --- /dev/null +++ b/mypyc/doc/differences_from_python.rst @@ -0,0 +1,332 @@ +.. _differences-from-python: + +Differences from Python +======================= + +Mypyc aims to be sufficiently compatible with Python semantics so that +migrating code to mypyc often doesn't require major code +changes. There are various differences to enable performance gains +that you need to be aware of, however. + +This section documents notable differences from Python. We discuss +many of them also elsewhere, but it's convenient to have them here in +one place. + +Running compiled modules +------------------------ + +You can't use ``python3 .py`` or ``python3 -m `` +to run compiled modules. Use ``python3 -c "import "`` instead, +or write a wrapper script that imports your module. + +As a side effect, you can't rely on checking the ``__name__`` attribute in compiled +code, like this:: + + if __name__ == "__main__": # Can't be used in compiled code + main() + +Type errors prevent compilation +------------------------------- + +You can't compile code that generates mypy type check errors. You can +sometimes ignore these with a ``# type: ignore`` comment, but this can +result in bad code being generated, and it's considered dangerous. + +.. note:: + + In the future, mypyc may reject ``# type: ignore`` comments that + may be unsafe. + +Runtime type checking +--------------------- + +Non-erased types in annotations will be type checked at runtime. For example, +consider this function:: + + def twice(x: int) -> int: + return x * 2 + +If you try to call this function with a ``float`` or ``str`` argument, +you'll get a type error on the call site, even if the call site is not +being type checked:: + + twice(5) # OK + twice(2.2) # TypeError + twice("blah") # TypeError + +Also, values with *inferred* types will be type checked. For example, +consider a call to the stdlib function ``socket.gethostname()`` in +compiled code. This function is not compiled (no stdlib modules are +compiled with mypyc), but mypyc uses a *library stub file* to infer +the return type as ``str``. Compiled code calling ``gethostname()`` +will fail with ``TypeError`` if ``gethostname()`` would return an +incompatible value, such as ``None``:: + + import socket + + # Fail if returned value is not a str + name = socket.gethostname() + +Note that ``gethostname()`` is defined like this in the stub file for +``socket`` (in typeshed):: + + def gethostname() -> str: ... + +Thus mypyc verifies that library stub files and annotations in +non-compiled code match runtime values. This adds an extra layer of +type safety. + +Casts such as ``cast(str, x)`` will also result in strict type +checks. Consider this example:: + + from typing import cast + ... + x = cast(str, y) + +The last line is essentially equivalent to this Python code when compiled:: + + if not isinstance(y, str): + raise TypeError(...) + x = y + +In interpreted mode ``cast`` does not perform a runtime type check. + +Native classes +-------------- + +Native classes behave differently from Python classes. See +:ref:`native-classes` for the details. + +Primitive types +--------------- + +Some primitive types behave differently in compiled code to improve +performance. + +``int`` objects use an unboxed (non-heap-allocated) representation for small +integer values. A side effect of this is that the exact runtime type of +``int`` values is lost. For example, consider this simple function:: + + def first_int(x: List[int]) -> int: + return x[0] + + print(first_int([True])) # Output is 1, instead of True! + +``bool`` is a subclass of ``int``, so the above code is +valid. However, when the list value is converted to ``int``, ``True`` +is converted to the corresponding ``int`` value, which is ``1``. + +Note that integers still have an arbitrary precision in compiled code, +similar to normal Python integers. + +Fixed-length tuples are unboxed, similar to integers. The exact type +and identity of fixed-length tuples is not preserved, and you can't +reliably use ``is`` checks to compare tuples that are used in compiled +code. + +.. _early-binding: + +Early binding +------------- + +References to functions, types, most attributes, and methods in the +same :ref:`compilation unit ` use *early binding*: +the target of the reference is decided at compile time, whenever +possible. This contrasts with normal Python behavior of *late +binding*, where the target is found by a namespace lookup at +runtime. Omitting these namespace lookups improves performance, but +some Python idioms don't work without changes. + +Note that non-final module-level variables still use late binding. +You may want to avoid these in very performance-critical code. + +Examples of early and late binding:: + + from typing import Final + + import lib # "lib" is not compiled + + x = 0 + y: Final = 1 + + def func() -> None: + pass + + class Cls: + def __init__(self, attr: int) -> None: + self.attr = attr + + def method(self) -> None: + pass + + def example() -> None: + # Early binding: + var = y + func() + o = Cls() + o.x + o.method() + + # Late binding: + var = x # Module-level variable + lib.func() # Accessing library that is not compiled + +Pickling and copying objects +---------------------------- + +Mypyc tries to enforce that instances native classes are properly +initialized by calling ``__init__`` implicitly when constructing +objects, even if objects are constructed through ``pickle``, +``copy.copy`` or ``copy.deepcopy``, for example. + +If a native class doesn't support calling ``__init__`` without arguments, +you can't pickle or copy instances of the class. Use the +``mypy_extensions.mypyc_attr`` class decorator to override this behavior +and enable pickling through the ``serializable`` flag:: + + from mypy_extensions import mypyc_attr + import pickle + + @mypyc_attr(serializable=True) + class Cls: + def __init__(self, n: int) -> None: + self.n = n + + data = pickle.dumps(Cls(5)) + obj = pickle.loads(data) # OK + +Additional notes: + +* All subclasses inherit the ``serializable`` flag. +* If a class has the ``allow_interpreted_subclasses`` attribute, it + implicitly supports serialization. +* Enabling serialization may slow down attribute access, since compiled + code has to be always prepared to raise ``AttributeError`` in case an + attribute is not defined at runtime. +* If you try to pickle an object without setting the ``serializable`` + flag, you'll get a ``TypeError`` about missing arguments to + ``__init__``. + + +Monkey patching +--------------- + +Since mypyc function and class definitions are immutable, you can't +perform arbitrary monkey patching, such as replacing functions or +methods with mocks in tests. + +.. note:: + + Each compiled module has a Python namespace that is initialized to + point to compiled functions and type objects. This namespace is a + regular ``dict`` object, and it *can* be modified. However, + compiled code generally doesn't use this namespace, so any changes + will only be visible to non-compiled code. + +Stack overflows +--------------- + +Compiled code currently doesn't check for stack overflows. Your +program may crash in an unrecoverable fashion if you have too many +nested function calls, typically due to out-of-control recursion. + +.. note:: + + This limitation will be fixed in the future. + +Final values +------------ + +Compiled code replaces a reference to an attribute declared ``Final`` with +the value of the attribute computed at compile time. This is an example of +:ref:`early binding `. Example:: + + MAX: Final = 100 + + def limit_to_max(x: int) -> int: + if x > MAX: + return MAX + return x + +The two references to ``MAX`` don't involve any module namespace lookups, +and are equivalent to this code:: + + def limit_to_max(x: int) -> int: + if x > 100: + return 100 + return x + +When run as interpreted, the first example will execute slower due to +the extra namespace lookups. In interpreted code final attributes can +also be modified. + +Unsupported features +-------------------- + +Some Python features are not supported by mypyc (yet). They can't be +used in compiled code, or there are some limitations. You can +partially work around some of these limitations by running your code +in interpreted mode. + +Nested classes +************** + +Nested classes are not supported. + +Conditional functions or classes +******************************** + +Function and class definitions guarded by an if-statement are not supported. + +Dunder methods +************** + +Native classes **cannot** use these dunders. If defined, they will not +work as expected. + +* ``__del__`` +* ``__index__`` +* ``__getattr__``, ``__getattribute__`` +* ``__setattr__`` +* ``__delattr__`` + +Generator expressions +********************* + +Generator expressions are not supported. To make it easier to compile +existing code, they are implicitly replaced with list comprehensions. +*This does not always produce the same behavior.* + +To work around this limitation, you can usually use a generator +function instead. You can sometimes replace the generator expression +with an explicit list comprehension. + +Descriptors +*********** + +Native classes can't contain arbitrary descriptors. Properties, static +methods and class methods are supported. + +Introspection +************* + +Various methods of introspection may break by using mypyc. Here's an +non-exhaustive list of what won't work: + +- Instance ``__annotations__`` is usually not kept +- Frames of compiled functions can't be inspected using ``inspect`` +- Compiled methods aren't considered methods by ``inspect.ismethod`` +- ``inspect.signature`` chokes on compiled functions + +Profiling hooks and tracing +*************************** + +Compiled functions don't trigger profiling and tracing hooks, such as +when using the ``profile``, ``cProfile``, or ``trace`` modules. + +Debuggers +********* + +You can't set breakpoints in compiled functions or step through +compiled functions using ``pdb``. Often you can debug your code in +interpreted mode instead. diff --git a/mypyc/doc/float_operations.rst b/mypyc/doc/float_operations.rst new file mode 100644 index 000000000000..feae5a806c70 --- /dev/null +++ b/mypyc/doc/float_operations.rst @@ -0,0 +1,50 @@ +.. _float-ops: + +Native float operations +======================== + +These ``float`` operations have fast, optimized implementations. Other +floating point operations use generic implementations that are often +slower. + +Construction +------------ + +* Float literal +* ``float(x: int)`` +* ``float(x: i64)`` +* ``float(x: i32)`` +* ``float(x: i16)`` +* ``float(x: u8)`` +* ``float(x: str)`` +* ``float(x: float)`` (no-op) + +Operators +--------- + +* Arithmetic (``+``, ``-``, ``*``, ``/``, ``//``, ``%``) +* Comparisons (``==``, ``!=``, ``<``, etc.) +* Augmented assignment (``x += y``, etc.) + +Functions +--------- + +* ``int(f)`` +* ``i64(f)`` (convert to 64-bit signed integer) +* ``i32(f)`` (convert to 32-bit signed integer) +* ``i16(f)`` (convert to 16-bit signed integer) +* ``u8(f)`` (convert to 8-bit unsigned integer) +* ``abs(f)`` +* ``math.sin(f)`` +* ``math.cos(f)`` +* ``math.tan(f)`` +* ``math.sqrt(f)`` +* ``math.exp(f)`` +* ``math.log(f)`` +* ``math.floor(f)`` +* ``math.ceil(f)`` +* ``math.fabs(f)`` +* ``math.pow(x, y)`` +* ``math.copysign(x, y)`` +* ``math.isinf(f)`` +* ``math.isnan(f)`` diff --git a/mypyc/doc/future.md b/mypyc/doc/future.md new file mode 100644 index 000000000000..2d8a454e94a0 --- /dev/null +++ b/mypyc/doc/future.md @@ -0,0 +1,42 @@ +# Future + +This document introduces some ideas for future improvements. + +## Basic Optimizations + +Implement basic optimizations such as common subexpression elimination and +loop invariant code motion. + +Importantly, common subexpression elimination could be used to avoid +redundant type checks. + +## Operation-specific Optimizations + +Some operations or combinations of successive operations can be +replaced with more efficient operations. Examples: + +* If `s` is a string, `s[i] == 'x'` doesn't need to construct the + intermediate single-character string object `s[i]` but just compare + the character value to `ord('x')`. + +* `a + ':' + b` (two string concetenations) can be implemented as + single three-operand concatenation that doesn't construct an + intermediate object. + +* `x in {1, 3}` can be translated into `x == 1 or x == 3` (more + generally we need to evaluate all right-hand-side items). + +## Integer Range Analysis + +Implement integer range analysis. This can be used in various ways: + +* Use untagged representations for some registers. +* Use faster integer arithmetic operations for operations that + only deal with short integers or that can't overflow. +* Remove redundant list and string index checks. + +## Always Defined Attributes + +Somehow make it possible to enforce that attributes in a class are always +defined. This makes attribute access faster since we don't need to explicitly +check if the attribute is defined. diff --git a/mypyc/doc/getting_started.rst b/mypyc/doc/getting_started.rst new file mode 100644 index 000000000000..adc617419ffa --- /dev/null +++ b/mypyc/doc/getting_started.rst @@ -0,0 +1,240 @@ +Getting started +=============== + +Here you will learn some basic things you need to know to get started with mypyc. + +Prerequisites +------------- + +You need a Python C extension development environment. The way to set this up +depends on your operating system. + +macOS +***** + +Install Xcode command line tools: + +.. code-block:: + + $ xcode-select --install + +Linux +***** + +You need a C compiler and CPython headers and libraries. The specifics +of how to install these varies by distribution. Here are instructions for +Ubuntu 18.04, for example: + +.. code-block:: + + $ sudo apt install python3-dev + +Windows +******* + +From `Build Tools for Visual Studio 2022 `_, install MSVC C++ build tools for your architecture and a Windows SDK. (latest versions recommended) + +Installation +------------ + +Mypyc is shipped as part of the mypy distribution. Install mypy like +this (you need Python 3.8 or later): + +.. code-block:: + + $ python3 -m pip install -U 'mypy[mypyc]' + +On some systems you need to use this instead: + +.. code-block:: + + $ python -m pip install -U 'mypy[mypyc]' + +Example program +--------------- + +Let's start with a classic micro-benchmark, recursive fibonacci. Save +this file as ``fib.py``: + +.. code-block:: python + + import time + + def fib(n: int) -> int: + if n <= 1: + return n + else: + return fib(n - 2) + fib(n - 1) + + t0 = time.time() + fib(32) + print(time.time() - t0) + +Note that we gave the ``fib`` function a type annotation. Without it, +performance won't be as impressive after compilation. + +.. note:: + + `Mypy documentation + `_ is a good + introduction if you are new to type annotations or mypy. Mypyc uses + mypy to perform type checking and type inference, so some familiarity + with mypy is very useful. + +Compiling and running +--------------------- + +We can run ``fib.py`` as a regular, interpreted program using CPython: + +.. code-block:: console + + $ python3 fib.py + 0.4125328063964844 + +It took about 0.41s to run on my computer. + +Run ``mypyc`` to compile the program to a binary C extension: + +.. code-block:: console + + $ mypyc fib.py + +This will generate a C extension for ``fib`` in the current working +directory. For example, on a Linux system the generated file may be +called ``fib.cpython-37m-x86_64-linux-gnu.so``. + +Since C extensions can't be run as programs, use ``python3 -c`` to run +the compiled module as a program: + +.. code-block:: console + + $ python3 -c "import fib" + 0.04097270965576172 + +After compilation, the program is about 10x faster. Nice! + +.. note:: + + ``__name__`` in ``fib.py`` would now be ``"fib"``, not ``"__main__"``. + +You can also pass most +`mypy command line options `_ +to ``mypyc``. + +Deleting compiled binary +------------------------ + +You can manually delete the C extension to get back to an interpreted +version (this example works on Linux): + +.. code-block:: + + $ rm fib.*.so + +Using setup.py +-------------- + +You can also use ``setup.py`` to compile modules using mypyc. Here is an +example ``setup.py`` file:: + + from setuptools import setup + + from mypyc.build import mypycify + + setup( + name='mylib', + packages=['mylib'], + ext_modules=mypycify([ + 'mylib/__init__.py', + 'mylib/mod.py', + ]), + ) + +We used ``mypycify(...)`` to specify which files to compile using +mypyc. Your ``setup.py`` can include additional Python files outside +``mypycify(...)`` that won't be compiled. + +Now you can build a wheel (.whl) file for the package:: + + python3 setup.py bdist_wheel + +The wheel is created under ``dist/``. + +You can also compile the C extensions in-place, in the current directory (similar +to using ``mypyc`` to compile modules):: + + python3 setup.py build_ext --inplace + +You can include most `mypy command line options +`_ in the +list of arguments passed to ``mypycify()``. For example, here we use +the ``--disallow-untyped-defs`` flag to require that all functions +have type annotations:: + + ... + setup( + name='frobnicate', + packages=['frobnicate'], + ext_modules=mypycify([ + '--disallow-untyped-defs', # Pass a mypy flag + 'frobnicate.py', + ]), + ) + +.. note: + + You may be tempted to use `--check-untyped-defs + `_ + to type check functions without type annotations. Note that this + may reduce performance, due to many transitions between type-checked and unchecked + code. + +Recommended workflow +-------------------- + +A simple way to use mypyc is to always compile your code after any +code changes, but this can get tedious, especially if you have a lot +of code. Instead, you can do most development in interpreted mode. +This development workflow has worked smoothly for developing mypy and +mypyc (often we forget that we aren't working on a vanilla Python +project): + +1. During development, use interpreted mode. This gives you a fast + edit-run cycle. + +2. Use type annotations liberally and use mypy to type check your code + during development. Mypy and tests can find most errors that would + break your compiled code, if you have good type annotation + coverage. (Running mypy is pretty quick.) + +3. After you've implemented a feature or a fix, compile your project + and run tests again, now in compiled mode. Usually nothing will + break here, assuming your type annotation coverage is good. This + can happen locally or in a Continuous Integration (CI) job. If you + have CI, compiling locally may be rarely needed. + +4. Release or deploy a compiled version. Optionally, include a + fallback interpreted version for platforms that mypyc doesn't + support. + +This mypyc workflow only involves minor tweaks to a typical Python +workflow. Most of development, testing and debugging happens in +interpreted mode. Incremental mypy runs, especially when using the +mypy daemon, are very quick (often a few hundred milliseconds). + +Next steps +---------- + +You can sometimes get good results by just annotating your code and +compiling it. If this isn't providing meaningful performance gains, if +you have trouble getting your code to work under mypyc, or if you want +to optimize your code for maximum performance, you should read the +rest of the documentation in some detail. + +Here are some specific recommendations, or you can just read the +documentation in order: + +* :ref:`using-type-annotations` +* :ref:`native-classes` +* :ref:`differences-from-python` +* :ref:`performance-tips` diff --git a/mypyc/doc/index.rst b/mypyc/doc/index.rst new file mode 100644 index 000000000000..584d6739e803 --- /dev/null +++ b/mypyc/doc/index.rst @@ -0,0 +1,62 @@ +.. mypyc documentation master file, created by + sphinx-quickstart on Sun Apr 5 14:01:55 2020. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to mypyc documentation! +=============================== + +Mypyc compiles Python modules to C extensions. It uses standard Python +`type hints +`_ to +generate fast code. + +.. toctree:: + :maxdepth: 2 + :caption: First steps + + introduction + getting_started + +.. toctree:: + :maxdepth: 2 + :caption: Using mypyc + + using_type_annotations + native_classes + differences_from_python + compilation_units + +.. toctree:: + :maxdepth: 2 + :caption: Native operations reference + + native_operations + int_operations + bool_operations + float_operations + str_operations + bytes_operations + list_operations + dict_operations + set_operations + tuple_operations + +.. toctree:: + :maxdepth: 2 + :caption: Advanced topics + + performance_tips_and_tricks + +.. toctree:: + :hidden: + :caption: Project Links + + GitHub + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/mypyc/doc/int_operations.rst b/mypyc/doc/int_operations.rst new file mode 100644 index 000000000000..eb875f5c9452 --- /dev/null +++ b/mypyc/doc/int_operations.rst @@ -0,0 +1,162 @@ +.. _int-ops: + +Native integer operations +========================= + +Mypyc supports these integer types: + +* ``int`` (arbitrary-precision integer) +* ``i64`` (64-bit signed integer) +* ``i32`` (32-bit signed integer) +* ``i16`` (16-bit signed integer) +* ``u8`` (8-bit unsigned integer) + +``i64``, ``i32``, ``i16`` and ``u8`` are *native integer types* and +are available in the ``mypy_extensions`` module. ``int`` corresponds +to the Python ``int`` type, but uses a more efficient runtime +representation (tagged pointer). Native integer types are value types. + +All integer types have optimized primitive operations, but the native +integer types are more efficient than ``int``, since they don't +require range or bounds checks. + +Operations on integers that are listed here have fast, optimized +implementations. Other integer operations use generic implementations +that are generally slower. Some operations involving integers and other +types, such as list indexing, are documented elsewhere. + +Construction +------------ + +``int`` type: + +* Integer literal +* ``int(x: float)`` +* ``int(x: i64)`` +* ``int(x: i32)`` +* ``int(x: i16)`` +* ``int(x: u8)`` +* ``int(x: str)`` +* ``int(x: str, base: int)`` +* ``int(x: int)`` (no-op) + +``i64`` type: + +* ``i64(x: int)`` +* ``i64(x: float)`` +* ``i64(x: i64)`` (no-op) +* ``i64(x: i32)`` +* ``i64(x: i16)`` +* ``i64(x: u8)`` +* ``i64(x: str)`` +* ``i64(x: str, base: int)`` + +``i32`` type: + +* ``i32(x: int)`` +* ``i32(x: float)`` +* ``i32(x: i64)`` (truncate) +* ``i32(x: i32)`` (no-op) +* ``i32(x: i16)`` +* ``i32(x: u8)`` +* ``i32(x: str)`` +* ``i32(x: str, base: int)`` + +``i16`` type: + +* ``i16(x: int)`` +* ``i16(x: float)`` +* ``i16(x: i64)`` (truncate) +* ``i16(x: i32)`` (truncate) +* ``i16(x: i16)`` (no-op) +* ``i16(x: u8)`` +* ``i16(x: str)`` +* ``i16(x: str, base: int)`` + +Conversions from ``int`` to a native integer type raise +``OverflowError`` if the value is too large or small. Conversions from +a wider native integer type to a narrower one truncate the value and never +fail. More generally, operations between native integer types don't +check for overflow. + +Implicit conversions +-------------------- + +``int`` values can be implicitly converted to a native integer type, +for convenience. This means that these are equivalent:: + + from mypy_extensions import i64 + + def implicit() -> None: + # Implicit conversion of 0 (int) to i64 + x: i64 = 0 + + def explicit() -> None: + # Explicit conversion of 0 (int) to i64 + x = i64(0) + +Similarly, a native integer value can be implicitly converted to an +arbitrary-precision integer. These two functions are equivalent:: + + def implicit(x: i64) -> int: + # Implicit conversion from i64 to int + return x + + def explicit(x: i64) -> int: + # Explicit conversion from i64 to int + return int(x) + +Operators +--------- + +* Arithmetic (``+``, ``-``, ``*``, ``//``, ``/``, ``%``) +* Bitwise operations (``&``, ``|``, ``^``, ``<<``, ``>>``, ``~``) +* Comparisons (``==``, ``!=``, ``<``, etc.) +* Augmented assignment (``x += y``, etc.) + +If one of the above native integer operations overflows or underflows +with signed operands, the behavior is undefined. Signed native integer +types should only be used if all possible values are small enough for +the type. For this reason, the arbitrary-precision ``int`` type is +recommended for signed values unless the performance of integer +operations is critical. + +Operations on unsigned integers (``u8``) wrap around on overflow. + +It's a compile-time error to mix different native integer types in a +binary operation such as addition. An explicit conversion is required:: + + from mypy_extensions import i64, i32 + + def add(x: i64, y: i32) -> None: + a = x + y # Error (i64 + i32) + b = x + i64(y) # OK + +You can freely mix a native integer value and an arbitrary-precision +``int`` value in an operation. The native integer type is "sticky" +and the ``int`` operand is coerced to the native integer type:: + + def example(x: i64, y: int) -> None: + a = x * y + # Type of "a" is "i64" + ... + b = 1 - x + # Similarly, type of "b" is "i64" + +Statements +---------- + +For loop over a range is compiled efficiently, if the ``range(...)`` object +is constructed in the for statement (after ``in``): + +* ``for x in range(end)`` +* ``for x in range(start, end)`` +* ``for x in range(start, end, step)`` + +If one of the arguments to ``range`` in a for loop is a native integer +type, the type of the loop variable is inferred to have this native +integer type, instead of ``int``:: + + for x in range(i64(n)): + # Type of "x" is "i64" + ... diff --git a/mypyc/doc/introduction.rst b/mypyc/doc/introduction.rst new file mode 100644 index 000000000000..53c86ecdab1b --- /dev/null +++ b/mypyc/doc/introduction.rst @@ -0,0 +1,150 @@ +Introduction +============ + +Mypyc compiles Python modules to C extensions. It uses standard Python +`type hints +`_ to +generate fast code. + +The compiled language is a strict, *gradually typed* Python variant. It +restricts the use of some dynamic Python features to gain performance, +but it's mostly compatible with standard Python. + +Mypyc uses `mypy `_ to perform type +checking and type inference. Most type system features in the stdlib +`typing `_ module are +supported. + +Compiled modules can import arbitrary Python modules and third-party +libraries. You can compile anything from a single performance-critical +module to your entire codebase. You can run the modules you compile +also as normal, interpreted Python modules. + +Existing code with type annotations is often **1.5x to 5x** faster +when compiled. Code tuned for mypyc can be **5x to 10x** faster. + +Mypyc currently aims to speed up non-numeric code, such as server +applications. Mypyc is also used to compile itself (and mypy). + +Why mypyc? +---------- + +**Easy to get started.** Compiled code has the look and feel of +regular Python code. Mypyc supports familiar Python syntax and idioms. + +**Expressive types.** Mypyc fully supports standard Python type hints. +Mypyc has local type inference, generics, optional types, tuple types, +union types, and more. Type hints act as machine-checked +documentation, making code not only faster but also easier to +understand and modify. + +**Python ecosystem.** Mypyc runs on top of CPython, the +standard Python implementation. You can use any third-party libraries, +including C extensions, installed with pip. Mypyc uses only valid Python +syntax, so all Python editors and IDEs work perfectly. + +**Fast program startup.** Mypyc uses ahead-of-time compilation, so +compilation does not slow down program startup. Slow program startup +is a common issue with JIT compilers. + +**Migration path for existing code.** Existing Python code often +requires only minor changes to compile using mypyc. + +**Waiting for compilation is optional.** Compiled code also runs as +normal Python code. You can use interpreted Python during development, +with familiar and fast workflows. + +**Runtime type safety.** Mypyc protects you from segfaults and memory +corruption. Any unexpected runtime type safety violation is a bug in +mypyc. Runtime values are checked against type annotations. (Without +mypyc, type annotations are ignored at runtime.) + +**Find errors statically.** Mypyc uses mypy for static type checking +that helps catch many bugs. + +Use cases +--------- + +**Fix only performance bottlenecks.** Often most time is spent in a few +Python modules or functions. Add type annotations and compile these +modules for easy performance gains. + +**Compile it all.** During development you can use interpreted mode, +for a quick edit-run cycle. In releases all non-test code is compiled. +This is how mypy achieved a 4x performance improvement over interpreted +Python. + +**Take advantage of existing type hints.** If you already use type +annotations in your code, adopting mypyc will be easier. You've already +done most of the work needed to use mypyc. + +**Alternative to a lower-level language.** Instead of writing +performance-critical code in C, C++, Cython or Rust, you may get good +performance while staying in the comfort of Python. + +**Migrate C extensions.** Maintaining C extensions is not always fun +for a Python developer. With mypyc you may get performance similar to +the original C, with the convenience of Python. + +Differences from Cython +----------------------- + +Mypyc targets many similar use cases as Cython. Mypyc does many things +differently, however: + +* No need to use non-standard syntax, such as ``cpdef``, or extra + decorators to get good performance. Clean, normal-looking + type-annotated Python code can be fast without language extensions. + This makes it practical to compile entire codebases without a + developer productivity hit. + +* Mypyc has first-class support for features in the ``typing`` module, + such as tuple types, union types and generics. + +* Mypyc has powerful type inference, provided by mypy. Variable type + annotations are not needed for optimal performance. + +* Mypyc fully integrates with mypy for robust and seamless static type + checking. + +* Mypyc performs strict enforcement of type annotations at runtime, + resulting in better runtime type safety and easier debugging. + +Unlike Cython, mypyc doesn't directly support interfacing with C libraries +or speeding up numeric code. + +How does it work +---------------- + +Mypyc uses several techniques to produce fast code: + +* Mypyc uses *ahead-of-time compilation* to native code. This removes + CPython interpreter overhead. + +* Mypyc enforces type annotations (and type comments) at runtime, + raising ``TypeError`` if runtime values don't match annotations. + Value types only need to be checked in the boundaries between + dynamic and static typing. + +* Compiled code uses optimized, type-specific primitives. + +* Mypyc uses *early binding* to resolve called functions and name + references at compile time. Mypyc avoids many dynamic namespace + lookups. + +* Classes are compiled to *C extension classes*. They use `vtables + `_ for fast + method calls and attribute access. + +* Mypyc treats compiled functions, classes, and attributes declared + ``Final`` as immutable. + +* Mypyc has memory-efficient, unboxed representations for integers and + booleans. + +Development status +------------------ + +Mypyc is currently alpha software. It's only recommended for +production use cases with careful testing, and if you are willing to +contribute fixes or to work around issues you will encounter. diff --git a/mypyc/doc/list_operations.rst b/mypyc/doc/list_operations.rst new file mode 100644 index 000000000000..5993c0a656bd --- /dev/null +++ b/mypyc/doc/list_operations.rst @@ -0,0 +1,65 @@ +.. _list-ops: + +Native list operations +====================== + +These ``list`` operations have fast, optimized implementations. Other +list operations use generic implementations that are often slower. + +Construction +------------ + +Construct list with specific items: + +* ``[item0, ..., itemN]`` + +Construct empty list: + +* ``[]`` +* ``list()`` + +Construct list from iterable: + +* ``list(x: Iterable)`` + +List comprehensions: + +* ``[... for ... in ...]`` +* ``[... for ... in ... if ...]`` + +Operators +--------- + +* ``lst[n]`` (get item by integer index) +* ``lst[n:m]``, ``lst[n:]``, ``lst[:m]``, ``lst[:]`` (slicing) +* ``lst * n``, ``n * lst`` +* ``obj in lst`` + +Statements +---------- + +Set item by integer index: + +* ``lst[n] = x`` + +For loop over a list: + +* ``for item in lst:`` + +Methods +------- + +* ``lst.append(obj)`` +* ``lst.extend(x: Iterable)`` +* ``lst.insert(index, obj)`` +* ``lst.pop(index=-1)`` +* ``lst.remove(obj)`` +* ``lst.count(obj)`` +* ``lst.index(obj)`` +* ``lst.reverse()`` +* ``lst.sort()`` + +Functions +--------- + +* ``len(lst: list)`` diff --git a/mypyc/doc/make.bat b/mypyc/doc/make.bat new file mode 100644 index 000000000000..153be5e2f6f9 --- /dev/null +++ b/mypyc/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/mypyc/doc/native_classes.rst b/mypyc/doc/native_classes.rst new file mode 100644 index 000000000000..b2935a6f7185 --- /dev/null +++ b/mypyc/doc/native_classes.rst @@ -0,0 +1,206 @@ +.. _native-classes: + +Native classes +============== + +Classes in compiled modules are *native classes* by default (some +exceptions are discussed below). Native classes are compiled to C +extension classes, which have some important differences from normal +Python classes. Native classes are similar in many ways to built-in +types, such as ``int``, ``str``, and ``list``. + +Immutable namespaces +-------------------- + +The type object namespace of native classes is mostly immutable (but +class variables can be assigned to):: + + class Cls: + def method1(self) -> None: + print("method1") + + def method2(self) -> None: + print("method2") + + Cls.method1 = Cls.method2 # Error + Cls.new_method = Cls.method2 # Error + +Only attributes defined within a class definition (or in a base class) +can be assigned to (similar to using ``__slots__``):: + + class Cls: + x: int + + def __init__(self, y: int) -> None: + self.x = 0 + self.y = y + + def method(self) -> None: + self.z = "x" + + o = Cls(0) + print(o.x, o.y) # OK + o.z = "y" # OK + o.extra = 3 # Error: no attribute "extra" + +.. _inheritance: + +Inheritance +----------- + +Only single inheritance is supported (except for :ref:`traits +`). Most non-native classes can't be used as base +classes. + +These non-native classes can be used as base classes of native +classes: + +* ``object`` +* ``dict`` (and ``Dict[k, v]``) +* ``BaseException`` +* ``Exception`` +* ``ValueError`` +* ``IndexError`` +* ``LookupError`` +* ``UserWarning`` +* ``typing.NamedTuple`` +* ``enum.Enum`` + +By default, a non-native class can't inherit a native class, and you +can't inherit from a native class outside the compilation unit that +defines the class. You can enable these through +``mypy_extensions.mypyc_attr``:: + + from mypy_extensions import mypyc_attr + + @mypyc_attr(allow_interpreted_subclasses=True) + class Cls: + ... + +Allowing interpreted subclasses has only minor impact on performance +of instances of the native class. Accessing methods and attributes of +a *non-native* subclass (or a subclass defined in another compilation +unit) will be slower, since it needs to use the normal Python +attribute access mechanism. + +You need to install ``mypy-extensions`` to use ``@mypyc_attr``: + +.. code-block:: text + + pip install --upgrade mypy-extensions + +Class variables +--------------- + +Class variables must be explicitly declared using ``attr: ClassVar`` +or ``attr: ClassVar[]``. You can't assign to a class variable +through an instance. Example:: + + from typing import ClassVar + + class Cls: + cv: ClassVar = 0 + + Cls.cv = 2 # OK + o = Cls() + print(o.cv) # OK (2) + o.cv = 3 # Error! + +.. tip:: + + Constant class variables can be declared using ``typing.Final`` or + ``typing.Final[]``. + +Generic native classes +---------------------- + +Native classes can be generic. Type variables are *erased* at runtime, +and instances don't keep track of type variable values. + +Compiled code thus can't check the values of type variables when +performing runtime type checks. These checks are delayed to when +reading a value with a type variable type:: + + from typing import TypeVar, Generic, cast + + T = TypeVar('T') + + class Box(Generic[T]): + def __init__(self, item: T) -> None: + self.item = item + + x = Box(1) # Box[int] + y = cast(Box[str], x) # OK (type variable value not checked) + y.item # Runtime error: item is "int", but "str" expected + +Metaclasses +----------- + +Most metaclasses aren't supported with native classes, since their +behavior is too dynamic. You can use these metaclasses, however: + +* ``abc.ABCMeta`` +* ``typing.GenericMeta`` (used by ``typing.Generic``) + +.. note:: + + If a class definition uses an unsupported metaclass, *mypyc + compiles the class into a regular Python class*. + +Class decorators +---------------- + +Similar to metaclasses, most class decorators aren't supported with +native classes, as they are usually too dynamic. These class +decorators can be used with native classes, however: + +* ``mypy_extensions.trait`` (for defining :ref:`trait types `) +* ``mypy_extensions.mypyc_attr`` (see :ref:`above `) +* ``dataclasses.dataclass`` +* ``@attr.s(auto_attribs=True)`` + +Dataclasses and attrs classes have partial native support, and they aren't as +efficient as pure native classes. + +.. note:: + + If a class definition uses an unsupported class decorator, *mypyc + compiles the class into a regular Python class*. + +Deleting attributes +------------------- + +By default, attributes defined in native classes can't be deleted. You +can explicitly allow certain attributes to be deleted by using +``__deletable__``:: + + class Cls: + x: int = 0 + y: int = 0 + other: int = 0 + + __deletable__ = ['x', 'y'] # 'x' and 'y' can be deleted + + o = Cls() + del o.x # OK + del o.y # OK + del o.other # Error + +You must initialize the ``__deletable__`` attribute in the class body, +using a list or a tuple expression with only string literal items that +refer to attributes. These are not valid:: + + a = ['x', 'y'] + + class Cls: + x: int + y: int + + __deletable__ = a # Error: cannot use variable 'a' + + __deletable__ = ('a',) # Error: not in a class body + +Other properties +---------------- + +Instances of native classes don't usually have a ``__dict__`` attribute. diff --git a/mypyc/doc/native_operations.rst b/mypyc/doc/native_operations.rst new file mode 100644 index 000000000000..2587e982feac --- /dev/null +++ b/mypyc/doc/native_operations.rst @@ -0,0 +1,55 @@ +Miscellaneous native operations +=============================== + +This is a list of various non-type-specific operations that have +custom native implementations. If an operation has no native +implementation, mypyc will use fallback generic implementations that +are often not as fast. + +.. note:: + + Operations specific to various primitive types are described + in the following sections. + +Operators +--------- + +* ``x is y`` (this is very fast for all types) + +Functions +--------- + +* ``isinstance(obj, type: type)`` +* ``isinstance(obj, type: tuple)`` +* ``cast(, obj)`` +* ``type(obj)`` +* ``len(obj)`` +* ``abs(obj)`` +* ``id(obj)`` +* ``iter(obj)`` +* ``next(iter: Iterator)`` +* ``hash(obj)`` +* ``getattr(obj, attr)`` +* ``getattr(obj, attr, default)`` +* ``setattr(obj, attr, value)`` +* ``hasattr(obj, attr)`` +* ``delattr(obj, name)`` +* ``slice(start, stop, step)`` +* ``globals()`` + +Method decorators +----------------- + +* ``@property`` +* ``@staticmethod`` +* ``@classmethod`` +* ``@abc.abstractmethod`` + +Statements +---------- + +These variants of statements have custom implementations: + +* ``for ... in seq:`` (for loop over a sequence) +* ``for ... in enumerate(...):`` +* ``for ... in zip(...):`` diff --git a/mypyc/doc/performance_tips_and_tricks.rst b/mypyc/doc/performance_tips_and_tricks.rst new file mode 100644 index 000000000000..ae0b2950814c --- /dev/null +++ b/mypyc/doc/performance_tips_and_tricks.rst @@ -0,0 +1,244 @@ +.. _performance-tips: + +Performance tips and tricks +=========================== + +Performance optimization is part art, part science. Just using mypyc +in a simple manner will likely make your code faster, but squeezing +the most performance out of your code requires the use of some +techniques we'll summarize below. + +Profiling +--------- + +If you are speeding up existing code, understanding where time is +spent is important. Mypyc speeds up code that you compile. If most of +the time is spent elsewhere, you may come back disappointed. For +example, if you spend 40% of time outside compiled code, even if +compiled code would go 100x faster, overall performance will only be +2.5x faster. + +A simple (but often effective) approach is to record the time in +various points of program execution using ``time.time()``, and to +print out elapsed time (or to write it to a log file). + +The stdlib modules ``profile`` and ``cProfile`` can provide much more +detailed data. (But these only work well with non-compiled code.) + +Avoiding slow libraries +----------------------- + +If profiling indicates that a lot of time is spent in the stdlib or +third-party libraries, you still have several options. + +First, if most time is spent in a few library features, you can +perhaps easily reimplement them in type-annotated Python, or extract +the relevant code and annotate it. Now it may be easy to compile this +code to speed it up. + +Second, you may be able to avoid the library altogether, or use an +alternative, more efficient library to achieve the same purpose. + +Type annotations +---------------- + +As discussed earlier, type annotations are key to major performance +gains. You should at least consider adding annotations to any +performance-critical functions and classes. It may also be helpful to +annotate code called by this code, even if it's not compiled, since +this may help mypy infer better types in the compile code. If you use +libraries, ensure they have stub files with decent type annotation +coverage. Writing a stub file is often easy, and you only need to +annotate features you use a lot. + +If annotating external code or writing stubs feel too burdensome, a +simple workaround is to annotate variables explicitly. For example, +here we call ``acme.get_items()``, but it has no type annotation. We +can use an explicit type annotation for the variable to which we +assign the result:: + + from typing import List, Tuple + import acme + + def work() -> None: + # Annotate "items" to help mypyc + items: List[Tuple[int, str]] = acme.get_items() + for item in items: + ... # Do some work here + +Without the annotation on ``items``, the type would be ``Any`` (since +``acme`` has no type annotation), resulting in slower, generic +operations being used later in the function. + +Avoiding slow Python features +----------------------------- + +Mypyc can optimize some features more effectively than others. Here +the difference is sometimes big -- some things only get marginally +faster at best, while others can get 10x faster, or more. Avoiding +these slow features in performance-critical parts of your code can +help a lot. + +These are some of the most important things to avoid: + +* Using class decorators or metaclasses in compiled code (that aren't + properly supported by mypyc) + +* Heavy reliance on interpreted Python libraries (C extensions are + usually fine) + +These things also tend to be relatively slow: + +* Using Python classes and instances of Python classes (native classes + are much faster) + +* Calling decorated functions (``@property``, ``@staticmethod``, and + ``@classmethod`` are special cased and thus fast) + +* Calling nested functions + +* Calling functions or methods defined in other compilation units + +* Using ``*args`` or ``**kwargs`` + +* Using generator functions + +* Using callable values (i.e. not leveraging early binding to call + functions or methods) + +Nested functions can often be replaced with module-level functions or +methods of native classes. + +Callable values and nested functions can sometimes be replaced with an +instance of a native class with a single method only, such as +``call(...)``. You can derive the class from an ABC, if there are +multiple possible functions. + +.. note:: + + Some slow features will likely get efficient implementations in the + future. You should check this section every once in a while to see + if some additional operations are fast. + +Using fast native features +-------------------------- + +Some native operations are particularly quick relative to the +corresponding interpreted operations. Using them as much as possible +may allow you to see 10x or more in performance gains. + +Some things are not much (or any) faster in compiled code, such as set +math operations. In contrast, calling a method of a native class is +much faster in compiled code. + +If you are used to optimizing for CPython, you might have replaced +some class instances with dictionaries, as they can be +faster. However, in compiled code, this "optimization" would likely +slow down your code. + +Similarly, caching a frequently called method in a local variable can +help in CPython, but it can slow things down in compiled code, since +the code won't use :ref:`early binding `:: + + def squares(n: int) -> List[int]: + a = [] + append = a.append # Not a good idea in compiled code! + for i in range(n): + append(i * i) + return a + +Here are examples of features that are fast, in no particular order +(this list is *not* exhaustive): + +* Calling compiled functions directly defined in the same compilation + unit (with positional and/or keyword arguments) + +* Calling methods of native classes defined in the same compilation + unit (with positional and/or keyword arguments) + +* Many integer operations + +* Many ``float`` operations + +* Booleans + +* :ref:`Native list operations `, such as indexing, + ``append``, and list comprehensions + +* While loops + +* For loops over ranges and lists, and with ``enumerate`` or ``zip`` + +* Reading dictionary items + +* ``isinstance()`` checks against native classes and instances of + primitive types (and unions of them) + +* Accessing local variables + +* Accessing attributes of native classes + +* Accessing final module-level attributes + +* Comparing strings for equality + +These features are also fast, but somewhat less so (relative to other +related operations): + +* Constructing instances of native classes + +* Constructing dictionaries + +* Setting dictionary items + +* Native :ref:`dict ` and :ref:`set ` operations + +* Accessing module-level variables + +Generally anything documented as a native operation is fast, even if +it's not explicitly mentioned here + +Adjusting garbage collection +---------------------------- + +Compilation does not speed up cyclic garbage collection. If everything +else gets much faster, it's possible that garbage collection will take +a big fraction of time. You can use ``gc.set_threshold()`` to adjust +the garbage collector to run less often:: + + import gc + + # Spend less time in gc; do this before significant computation + gc.set_threshold(150000) + + ... # Actual work happens here + +Fast interpreter shutdown +------------------------- + +If you allocate many objects, it's possible that your program spends a +lot of time cleaning up when the Python runtime shuts down. Mypyc +won't speed up the shutdown of a Python process much. + +You can call ``os._exit(code)`` to immediately terminate the Python +process, skipping normal cleanup. This can give a nice boost to a +batch process or a command-line tool. + +.. note:: + + This can be dangerous and can lose data. You need to ensure + that all streams are flushed and everything is otherwise cleaned up + properly. + +Work smarter +------------ + +Usually there are many things you can do to improve performance, even +if most tweaks will yield only minor gains. The key to being effective +is to focus on things that give a large gain with a small effort. + +For example, low-level optimizations, such as avoiding a nested +function, can be pointless, if you could instead avoid a metaclass -- +to allow a key class to be compiled as a native class. The latter +optimization could speed up numerous method calls and attribute +accesses, just like that. diff --git a/mypyc/doc/set_operations.rst b/mypyc/doc/set_operations.rst new file mode 100644 index 000000000000..433e29ea123a --- /dev/null +++ b/mypyc/doc/set_operations.rst @@ -0,0 +1,47 @@ +.. _set-ops: + +Native set operations +====================== + +These ``set`` operations have fast, optimized implementations. Other +set operations use generic implementations that are often slower. + +Construction +------------ + +Construct set with specific items: + +* ``{item0, ..., itemN}`` + +Construct empty set: + +* ``set()`` + +Construct set from iterable: + +* ``set(x: Iterable)`` + +Set comprehensions: + +* ``{... for ... in ...}`` +* ``{... for ... in ... if ...}`` + +Operators +--------- + +* ``item in s`` + +Methods +------- + +* ``s.add(item)`` +* ``s.remove(item)`` +* ``s.discard(item)`` +* ``s.update(x: Iterable)`` +* ``s.clear()`` +* ``s.pop()`` + +Functions +--------- + +* ``len(s: set)`` diff --git a/mypyc/doc/str_operations.rst b/mypyc/doc/str_operations.rst new file mode 100644 index 000000000000..9e94f1b6d7bb --- /dev/null +++ b/mypyc/doc/str_operations.rst @@ -0,0 +1,59 @@ +.. _str-ops: + +Native string operations +======================== + +These ``str`` operations have fast, optimized implementations. Other +string operations use generic implementations that are often slower. + +Construction +------------ + +* String literal +* ``str(x: int)`` +* ``str(x: object)`` + +Operators +--------- + +* Concatenation (``s1 + s2``) +* Indexing (``s[n]``) +* Slicing (``s[n:m]``, ``s[n:]``, ``s[:m]``) +* Comparisons (``==``, ``!=``) +* Augmented assignment (``s1 += s2``) + +.. _str-methods: + +Methods +------- + +* ``s.encode()`` +* ``s.encode(encoding: str)`` +* ``s.encode(encoding: str, errors: str)`` +* ``s1.endswith(s2: str)`` +* ``s.join(x: Iterable)`` +* ``s.replace(old: str, new: str)`` +* ``s.replace(old: str, new: str, count: int)`` +* ``s.split()`` +* ``s.split(sep: str)`` +* ``s.split(sep: str, maxsplit: int)`` +* ``s1.startswith(s2: str)`` + +.. note:: + + :ref:`bytes.decode() ` is also optimized. + +Formatting +---------- + +A subset of these common string formatting expressions are optimized: + +* F-strings +* ``"...".format(...)`` +* ``"..." % (...)`` + +Functions +--------- + +* ``len(s: str)`` +* ``ord(s: str)`` diff --git a/mypyc/doc/tuple_operations.rst b/mypyc/doc/tuple_operations.rst new file mode 100644 index 000000000000..fca9e63fc210 --- /dev/null +++ b/mypyc/doc/tuple_operations.rst @@ -0,0 +1,33 @@ +.. _tuple-ops: + +Native tuple operations +======================= + +These ``tuple`` operations have fast, optimized implementations. Other +tuple operations use generic implementations that are often slower. + +Unless mentioned otherwise, these operations apply to both fixed-length +tuples and variable-length tuples. + +Construction +------------ + +* ``item0, ..., itemN`` (construct a tuple) +* ``tuple(lst: list)`` (construct a variable-length tuple) +* ``tuple(lst: Iterable)`` (construct a variable-length tuple) + +Operators +--------- + +* ``tup[n]`` (integer index) +* ``tup[n:m]``, ``tup[n:]``, ``tup[:m]`` (slicing) + +Statements +---------- + +* ``item0, ..., itemN = tup`` (for fixed-length tuples) + +Functions +--------- + +* ``len(tup: tuple)`` diff --git a/mypyc/doc/using_type_annotations.rst b/mypyc/doc/using_type_annotations.rst new file mode 100644 index 000000000000..04c923819d54 --- /dev/null +++ b/mypyc/doc/using_type_annotations.rst @@ -0,0 +1,398 @@ +.. _using-type-annotations: + +Using type annotations +====================== + +You will get the most out of mypyc if you compile code with precise +type annotations. Not all type annotations will help performance +equally, however. Using types such as :ref:`primitive types +`, :ref:`native classes `, +:ref:`union types `, :ref:`trait types `, +and :ref:`tuple types ` as much as possible is a key to +major performance gains over CPython. + +In contrast, some other types, including ``Any``, are treated as +:ref:`erased types `. Operations on erased types use +generic operations that work with arbitrary objects, similar to how +the CPython interpreter works. If you only use erased types, the only +notable benefits over CPython will be the removal of interpreter +overhead (from compilation) and a bit of :ref:`early binding +`, which will usually only give minor performance +gains. + +.. _primitive-types: + +Primitive types +--------------- + +The following built-in types are treated as *primitive types* by +mypyc, and many operations on these types have efficient +implementations: + +* ``int`` (:ref:`native operations `) +* ``i64`` (:ref:`documentation `, :ref:`native operations `) +* ``i32`` (:ref:`documentation `, :ref:`native operations `) +* ``i16`` (:ref:`documentation `, :ref:`native operations `) +* ``u8`` (:ref:`documentation `, :ref:`native operations `) +* ``float`` (:ref:`native operations `) +* ``bool`` (:ref:`native operations `) +* ``str`` (:ref:`native operations `) +* ``List[T]`` (:ref:`native operations `) +* ``Dict[K, V]`` (:ref:`native operations `) +* ``Set[T]`` (:ref:`native operations `) +* ``Tuple[T, ...]`` (variable-length tuple; :ref:`native operations `) +* ``None`` + +The link after each type lists all supported native, optimized +operations for the type. You can use all operations supported by +Python, but *native operations* will have custom, optimized +implementations. + +Primitive containers +-------------------- + +Primitive container objects such as ``list`` and ``dict`` don't +maintain knowledge of the item types at runtime -- the item type is +*erased*. + +This means that item types are checked when items are accessed, not +when a container is passed as an argument or assigned to another +variable. For example, here we have a runtime type error on the final +line of ``example`` (the ``Any`` type means an arbitrary, unchecked +value):: + + from typing import List, Any + + def example(a: List[Any]) -> None: + b: List[int] = a # No error -- items are not checked + print(b[0]) # Error here -- got str, but expected int + + example(["x"]) + +.. _native-class-intro: + +Native classes +-------------- + +Classes that get compiled to C extensions are called native +classes. Most common operations on instances of these classes are +optimized, including construction, attribute access and method calls. + +Native class definitions look exactly like normal Python class +definitions. A class is usually native if it's in a compiled module +(though there are some exceptions). + +Consider this example: + +.. code-block:: + + class Point: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + + def shift(p: Point) -> Point: + return Point(p.x + 1, p.y + 1) + +All operations in the above example use native operations, if the file +is compiled. + +Native classes have some notable different from Python classes: + +* Only attributes and methods defined in the class body or methods are + supported. If you try to assign to an undefined attribute outside + the class definition, ``AttributeError`` will be raised. This enables + an efficient memory layout and fast method calls for native classes. + +* Native classes usually don't define the ``__dict__`` attribute (they + don't have an attribute dictionary). This follows from only having + a specific set of attributes. + +* Native classes can't have an arbitrary metaclass or use most class + decorators. + +Native classes only support single inheritance. A limited form of +multiple inheritance is supported through *trait types*. You generally +must inherit from another native class (or ``object``). By default, +you can't inherit a Python class from a native class (but there's +an :ref:`override ` to allow that). + +See :ref:`native-classes` for more details. + +.. _tuple-types: + +Tuple types +----------- + +Fixed-length +`tuple types `_ +such as ``Tuple[int, str]`` are represented +as :ref:`value types ` when stored in variables, +passed as arguments, or returned from functions. Value types are +allocated in the low-level machine stack or in CPU registers, as +opposed to *heap types*, which are allocated dynamically from the +heap. + +Like all value types, tuples will be *boxed*, i.e. converted to +corresponding heap types, when stored in Python containers, or passed +to non-native code. A boxed tuple value will be a regular Python tuple +object. + +.. _union-types: + +Union types +----------- + +`Union types `_ +and +`optional types `_ +that contain primitive types, native class types and +trait types are also efficient. If a union type has +:ref:`erased ` items, accessing items with +non-erased types is often still quite efficient. + +A value with a union types is always :ref:`boxed `, +even if it contains a value that also has an unboxed representation, such +as an integer or a boolean. + +For example, using ``Optional[int]`` is quite efficient, but the value +will always be boxed. A plain ``int`` value will usually be faster, since +it has an unboxed representation. + +.. _trait-types: + +Trait types +----------- + +Trait types enable a form of multiple inheritance for native classes. +A native class can inherit any number of traits. Trait types are +defined as classes using the ``mypy_extensions.trait`` decorator:: + + from mypy_extensions import trait + + @trait + class MyTrait: + def method(self) -> None: + ... + +Traits can define methods, properties and attributes. They often +define abstract methods. Traits can be generic. + +If a class subclasses both a non-trait class and traits, the traits +must be placed at the end of the base class list:: + + class Base: ... + + class Derived(Base, MyTrait, FooTrait): # OK + ... + + class Derived2(MyTrait, FooTrait, Base): + # Error: traits should come last + ... + +Traits have some special properties: + +* You shouldn't create instances of traits (though mypyc does not + prevent it yet). + +* Traits can subclass other traits or native classes, but the MRO must be + linear (just like with native classes). + +* Accessing methods or attributes through a trait type is somewhat + less efficient than through a native class type, but this is much + faster than through Python class types or other + :ref:`erased types `. + +You need to install ``mypy-extensions`` to use ``@trait``: + +.. code-block:: text + + pip install --upgrade mypy-extensions + +.. _erased-types: + +Erased types +------------ + +Mypyc supports many other kinds of types as well, beyond those +described above. However, these types don't have customized +operations, and they are implemented using *type erasure*. Type +erasure means that all other types are equivalent to untyped values at +runtime, i.e. they are the equivalent of the type ``Any``. Erased +types include these: + +* Python classes (including ABCs) + +* Non-mypyc extension types and primitive types (including built-in + types that are not primitives) + +* `Callable types `_ + +* `Type variable types `_ + +* Type `Any `_ + +* Protocol types + +Using erased types can still improve performance, since they can +enable better types to be inferred for expressions that use these +types. For example, a value with type ``Callable[[], int]`` will not +allow native calls. However, the return type is a primitive type, and +we can use fast operations on the return value:: + + from typing import Callable + + def call_and_inc(f: Callable[[], int]) -> int: + # Slow call, since f has an erased type + n = f() + # Fast increment; inferred type of n is int (primitive type) + n += 1 + return n + +If the type of the argument ``f`` was ``Any``, the type of ``n`` would +also be ``Any``, resulting in a generic, slower increment operation +being used. + +Strict runtime type checking +---------------------------- + +Compiled code ensures that any variable or expression with a +non-erased type only has compatible values at runtime. This is in +contrast with using *optional static typing*, such as by using mypy, +when type annotations are not enforced at runtime. Mypyc ensures +type safety both statically and at runtime. + +``Any`` types and erased types in general can compromise type safety, +and this is by design. Inserting strict runtime type checks for all +possible values would be too expensive and against the goal of +high performance. + +.. _value-and-heap-types: + +Value and heap types +-------------------- + +In CPython, memory for all objects is dynamically allocated on the +heap. All Python types are thus *heap types*. In compiled code, some +types are *value types* -- no object is (necessarily) allocated on the +heap. ``bool``, ``float``, ``None``, :ref:`native integer types ` +and fixed-length tuples are value types. + +``int`` is a hybrid. For typical integer values, it is a value +type. Large enough integer values, those that require more than 63 +bits (or 31 bits on 32-bit platforms) to represent, use a heap-based +representation (same as CPython). + +Value types have a few differences from heap types: + +* When an instance of a value type is used in a context that expects a + heap value, for example as a list item, it will transparently switch + to a heap-based representation (boxing) as needed. + +* Similarly, mypyc transparently changes from a heap-based + representation to a value representation (unboxing). + +* Object identity of integers, floating point values and tuples is not + preserved. You should use ``==`` instead of ``is`` if you are comparing + two integers, floats or fixed-length tuples. + +* When an instance of a subclass of a value type is converted to the + base type, it is implicitly converted to an instance of the target + type. For example, a ``bool`` value assigned to a variable with an + ``int`` type will be converted to the corresponding integer. + +The latter conversion is the only implicit type conversion that +happens in mypyc programs. + +Example:: + + def example() -> None: + # A small integer uses the value (unboxed) representation + x = 5 + # A large integer uses the heap (boxed) representation + x = 2**500 + # Lists always contain boxed integers + a = [55] + # When reading from a list, the object is automatically unboxed + x = a[0] + # True is converted to 1 on assignment + x = True + +Since integers and floating point values have a different runtime +representations and neither can represent all the values of the other +type, type narrowing of floating point values through assignment is +disallowed in compiled code. For consistency, mypyc rejects assigning +an integer value to a float variable even in variable initialization. +An explicit conversion is required. + +Examples:: + + def narrowing(n: int) -> None: + # Error: Incompatible value representations in assignment + # (expression has type "int", variable has type "float") + x: float = 0 + + y: float = 0.0 # Ok + + if f(): + y = n # Error + if f(): + y = float(n) # Ok + +.. _native-ints: + +Native integer types +-------------------- + +You can use the native integer types ``i64`` (64-bit signed integer), +``i32`` (32-bit signed integer), ``i16`` (16-bit signed integer), and +``u8`` (8-bit unsigned integer) if you know that integer values will +always fit within fixed bounds. These types are faster than the +arbitrary-precision ``int`` type, since they don't require overflow +checks on operations. They may also use less memory than ``int`` +values. The types are imported from the ``mypy_extensions`` module +(installed via ``pip install mypy_extensions``). + +Example:: + + from mypy_extensions import i64 + + def sum_list(l: list[i64]) -> i64: + s: i64 = 0 + for n in l: + s += n + return s + + # Implicit conversions from int to i64 + print(sum_list([1, 3, 5])) + +.. note:: + + Since there are no overflow checks when performing native integer + arithmetic, the above function could result in an overflow or other + undefined behavior if the sum might not fit within 64 bits. + + The behavior when running as interpreted Python program will be + different if there are overflows. Declaring native integer types + have no effect unless code is compiled. Native integer types are + effectively equivalent to ``int`` when interpreted. + +Native integer types have these additional properties: + +* Values can be implicitly converted between ``int`` and a native + integer type (both ways). + +* Conversions between different native integer types must be explicit. + A conversion to a narrower native integer type truncates the value + without a runtime overflow check. + +* If a binary operation (such as ``+``) or an augmented assignment + (such as ``+=``) mixes native integer and ``int`` values, the + ``int`` operand is implicitly coerced to the native integer type + (native integer types are "sticky"). + +* You can't mix different native integer types in binary + operations. Instead, convert between types explicitly. + +For more information about native integer types, refer to +:ref:`native integer operations `. diff --git a/mypyc/errors.py b/mypyc/errors.py new file mode 100644 index 000000000000..8bc9b2714f75 --- /dev/null +++ b/mypyc/errors.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import mypy.errors +from mypy.options import Options + + +class Errors: + def __init__(self, options: Options) -> None: + self.num_errors = 0 + self.num_warnings = 0 + self._errors = mypy.errors.Errors(options, hide_error_codes=True) + + def error(self, msg: str, path: str, line: int) -> None: + self._errors.report(line, None, msg, severity="error", file=path) + self.num_errors += 1 + + def note(self, msg: str, path: str, line: int) -> None: + self._errors.report(line, None, msg, severity="note", file=path) + + def warning(self, msg: str, path: str, line: int) -> None: + self._errors.report(line, None, msg, severity="warning", file=path) + self.num_warnings += 1 + + def new_messages(self) -> list[str]: + return self._errors.new_messages() + + def flush_errors(self) -> None: + for error in self.new_messages(): + print(error) diff --git a/mypyc/external/googletest/LICENSE b/mypyc/external/googletest/LICENSE new file mode 100644 index 000000000000..1941a11f8ce9 --- /dev/null +++ b/mypyc/external/googletest/LICENSE @@ -0,0 +1,28 @@ +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mypyc/external/googletest/README.md b/mypyc/external/googletest/README.md new file mode 100644 index 000000000000..edd4408054b3 --- /dev/null +++ b/mypyc/external/googletest/README.md @@ -0,0 +1,280 @@ + +### Generic Build Instructions ### + +#### Setup #### + +To build Google Test and your tests that use it, you need to tell your +build system where to find its headers and source files. The exact +way to do it depends on which build system you use, and is usually +straightforward. + +#### Build #### + +Suppose you put Google Test in directory `${GTEST_DIR}`. To build it, +create a library build target (or a project as called by Visual Studio +and Xcode) to compile + + ${GTEST_DIR}/src/gtest-all.cc + +with `${GTEST_DIR}/include` in the system header search path and `${GTEST_DIR}` +in the normal header search path. Assuming a Linux-like system and gcc, +something like the following will do: + + g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} \ + -pthread -c ${GTEST_DIR}/src/gtest-all.cc + ar -rv libgtest.a gtest-all.o + +(We need `-pthread` as Google Test uses threads.) + +Next, you should compile your test source file with +`${GTEST_DIR}/include` in the system header search path, and link it +with gtest and any other necessary libraries: + + g++ -isystem ${GTEST_DIR}/include -pthread path/to/your_test.cc libgtest.a \ + -o your_test + +As an example, the make/ directory contains a Makefile that you can +use to build Google Test on systems where GNU make is available +(e.g. Linux, Mac OS X, and Cygwin). It doesn't try to build Google +Test's own tests. Instead, it just builds the Google Test library and +a sample test. You can use it as a starting point for your own build +script. + +If the default settings are correct for your environment, the +following commands should succeed: + + cd ${GTEST_DIR}/make + make + ./sample1_unittest + +If you see errors, try to tweak the contents of `make/Makefile` to make +them go away. There are instructions in `make/Makefile` on how to do +it. + +### Using CMake ### + +Google Test comes with a CMake build script ( +[CMakeLists.txt](CMakeLists.txt)) that can be used on a wide range of platforms ("C" stands for +cross-platform.). If you don't have CMake installed already, you can +download it for free from . + +CMake works by generating native makefiles or build projects that can +be used in the compiler environment of your choice. The typical +workflow starts with: + + mkdir mybuild # Create a directory to hold the build output. + cd mybuild + cmake ${GTEST_DIR} # Generate native build scripts. + +If you want to build Google Test's samples, you should replace the +last command with + + cmake -Dgtest_build_samples=ON ${GTEST_DIR} + +If you are on a \*nix system, you should now see a Makefile in the +current directory. Just type 'make' to build gtest. + +If you use Windows and have Visual Studio installed, a `gtest.sln` file +and several `.vcproj` files will be created. You can then build them +using Visual Studio. + +On Mac OS X with Xcode installed, a `.xcodeproj` file will be generated. + +### Legacy Build Scripts ### + +Before settling on CMake, we have been providing hand-maintained build +projects/scripts for Visual Studio, Xcode, and Autotools. While we +continue to provide them for convenience, they are not actively +maintained any more. We highly recommend that you follow the +instructions in the previous two sections to integrate Google Test +with your existing build system. + +If you still need to use the legacy build scripts, here's how: + +The msvc\ folder contains two solutions with Visual C++ projects. +Open the `gtest.sln` or `gtest-md.sln` file using Visual Studio, and you +are ready to build Google Test the same way you build any Visual +Studio project. Files that have names ending with -md use DLL +versions of Microsoft runtime libraries (the /MD or the /MDd compiler +option). Files without that suffix use static versions of the runtime +libraries (the /MT or the /MTd option). Please note that one must use +the same option to compile both gtest and the test code. If you use +Visual Studio 2005 or above, we recommend the -md version as /MD is +the default for new projects in these versions of Visual Studio. + +On Mac OS X, open the `gtest.xcodeproj` in the `xcode/` folder using +Xcode. Build the "gtest" target. The universal binary framework will +end up in your selected build directory (selected in the Xcode +"Preferences..." -> "Building" pane and defaults to xcode/build). +Alternatively, at the command line, enter: + + xcodebuild + +This will build the "Release" configuration of gtest.framework in your +default build location. See the "xcodebuild" man page for more +information about building different configurations and building in +different locations. + +If you wish to use the Google Test Xcode project with Xcode 4.x and +above, you need to either: + + * update the SDK configuration options in xcode/Config/General.xconfig. + Comment options `SDKROOT`, `MACOS_DEPLOYMENT_TARGET`, and `GCC_VERSION`. If + you choose this route you lose the ability to target earlier versions + of MacOS X. + * Install an SDK for an earlier version. This doesn't appear to be + supported by Apple, but has been reported to work + (http://stackoverflow.com/questions/5378518). + +### Tweaking Google Test ### + +Google Test can be used in diverse environments. The default +configuration may not work (or may not work well) out of the box in +some environments. However, you can easily tweak Google Test by +defining control macros on the compiler command line. Generally, +these macros are named like `GTEST_XYZ` and you define them to either 1 +or 0 to enable or disable a certain feature. + +We list the most frequently used macros below. For a complete list, +see file [include/gtest/internal/gtest-port.h](include/gtest/internal/gtest-port.h). + +### Choosing a TR1 Tuple Library ### + +Some Google Test features require the C++ Technical Report 1 (TR1) +tuple library, which is not yet available with all compilers. The +good news is that Google Test implements a subset of TR1 tuple that's +enough for its own need, and will automatically use this when the +compiler doesn't provide TR1 tuple. + +Usually you don't need to care about which tuple library Google Test +uses. However, if your project already uses TR1 tuple, you need to +tell Google Test to use the same TR1 tuple library the rest of your +project uses, or the two tuple implementations will clash. To do +that, add + + -DGTEST_USE_OWN_TR1_TUPLE=0 + +to the compiler flags while compiling Google Test and your tests. If +you want to force Google Test to use its own tuple library, just add + + -DGTEST_USE_OWN_TR1_TUPLE=1 + +to the compiler flags instead. + +If you don't want Google Test to use tuple at all, add + + -DGTEST_HAS_TR1_TUPLE=0 + +and all features using tuple will be disabled. + +### Multi-threaded Tests ### + +Google Test is thread-safe where the pthread library is available. +After `#include "gtest/gtest.h"`, you can check the `GTEST_IS_THREADSAFE` +macro to see whether this is the case (yes if the macro is `#defined` to +1, no if it's undefined.). + +If Google Test doesn't correctly detect whether pthread is available +in your environment, you can force it with + + -DGTEST_HAS_PTHREAD=1 + +or + + -DGTEST_HAS_PTHREAD=0 + +When Google Test uses pthread, you may need to add flags to your +compiler and/or linker to select the pthread library, or you'll get +link errors. If you use the CMake script or the deprecated Autotools +script, this is taken care of for you. If you use your own build +script, you'll need to read your compiler and linker's manual to +figure out what flags to add. + +### As a Shared Library (DLL) ### + +Google Test is compact, so most users can build and link it as a +static library for the simplicity. You can choose to use Google Test +as a shared library (known as a DLL on Windows) if you prefer. + +To compile *gtest* as a shared library, add + + -DGTEST_CREATE_SHARED_LIBRARY=1 + +to the compiler flags. You'll also need to tell the linker to produce +a shared library instead - consult your linker's manual for how to do +it. + +To compile your *tests* that use the gtest shared library, add + + -DGTEST_LINKED_AS_SHARED_LIBRARY=1 + +to the compiler flags. + +Note: while the above steps aren't technically necessary today when +using some compilers (e.g. GCC), they may become necessary in the +future, if we decide to improve the speed of loading the library (see + for details). Therefore you are +recommended to always add the above flags when using Google Test as a +shared library. Otherwise a future release of Google Test may break +your build script. + +### Avoiding Macro Name Clashes ### + +In C++, macros don't obey namespaces. Therefore two libraries that +both define a macro of the same name will clash if you `#include` both +definitions. In case a Google Test macro clashes with another +library, you can force Google Test to rename its macro to avoid the +conflict. + +Specifically, if both Google Test and some other code define macro +FOO, you can add + + -DGTEST_DONT_DEFINE_FOO=1 + +to the compiler flags to tell Google Test to change the macro's name +from `FOO` to `GTEST_FOO`. Currently `FOO` can be `FAIL`, `SUCCEED`, +or `TEST`. For example, with `-DGTEST_DONT_DEFINE_TEST=1`, you'll +need to write + + GTEST_TEST(SomeTest, DoesThis) { ... } + +instead of + + TEST(SomeTest, DoesThis) { ... } + +in order to define a test. + +## Developing Google Test ## + +This section discusses how to make your own changes to Google Test. + +### Testing Google Test Itself ### + +To make sure your changes work as intended and don't break existing +functionality, you'll want to compile and run Google Test's own tests. +For that you can use CMake: + + mkdir mybuild + cd mybuild + cmake -Dgtest_build_tests=ON ${GTEST_DIR} + +Make sure you have Python installed, as some of Google Test's tests +are written in Python. If the cmake command complains about not being +able to find Python (`Could NOT find PythonInterp (missing: +PYTHON_EXECUTABLE)`), try telling it explicitly where your Python +executable can be found: + + cmake -DPYTHON_EXECUTABLE=path/to/python -Dgtest_build_tests=ON ${GTEST_DIR} + +Next, you can build Google Test and all of its own tests. On \*nix, +this is usually done by 'make'. To run the tests, do + + make test + +All tests should pass. + +Normally you don't need to worry about regenerating the source files, +unless you need to modify them. In that case, you should modify the +corresponding .pump files instead and run the pump.py Python script to +regenerate them. You can find pump.py in the [scripts/](scripts/) directory. +Read the [Pump manual](docs/PumpManual.md) for how to use it. diff --git a/mypyc/external/googletest/include/gtest/gtest-death-test.h b/mypyc/external/googletest/include/gtest/gtest-death-test.h new file mode 100644 index 000000000000..957a69c6a9e7 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-death-test.h @@ -0,0 +1,294 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the public API for death tests. It is +// #included by gtest.h so a user doesn't need to include this +// directly. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ + +#include "gtest/internal/gtest-death-test-internal.h" + +namespace testing { + +// This flag controls the style of death tests. Valid values are "threadsafe", +// meaning that the death test child process will re-execute the test binary +// from the start, running only a single death test, or "fast", +// meaning that the child process will execute the test logic immediately +// after forking. +GTEST_DECLARE_string_(death_test_style); + +#if GTEST_HAS_DEATH_TEST + +namespace internal { + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +GTEST_API_ bool InDeathTestChild(); + +} // namespace internal + +// The following macros are useful for writing death tests. + +// Here's what happens when an ASSERT_DEATH* or EXPECT_DEATH* is +// executed: +// +// 1. It generates a warning if there is more than one active +// thread. This is because it's safe to fork() or clone() only +// when there is a single thread. +// +// 2. The parent process clone()s a sub-process and runs the death +// test in it; the sub-process exits with code 0 at the end of the +// death test, if it hasn't exited already. +// +// 3. The parent process waits for the sub-process to terminate. +// +// 4. The parent process checks the exit code and error message of +// the sub-process. +// +// Examples: +// +// ASSERT_DEATH(server.SendMessage(56, "Hello"), "Invalid port number"); +// for (int i = 0; i < 5; i++) { +// EXPECT_DEATH(server.ProcessRequest(i), +// "Invalid request .* in ProcessRequest()") +// << "Failed to die on request " << i; +// } +// +// ASSERT_EXIT(server.ExitNow(), ::testing::ExitedWithCode(0), "Exiting"); +// +// bool KilledBySIGHUP(int exit_code) { +// return WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGHUP; +// } +// +// ASSERT_EXIT(client.HangUpServer(), KilledBySIGHUP, "Hanging up!"); +// +// On the regular expressions used in death tests: +// +// On POSIX-compliant systems (*nix), we use the library, +// which uses the POSIX extended regex syntax. +// +// On other platforms (e.g. Windows), we only support a simple regex +// syntax implemented as part of Google Test. This limited +// implementation should be enough most of the time when writing +// death tests; though it lacks many features you can find in PCRE +// or POSIX extended regex syntax. For example, we don't support +// union ("x|y"), grouping ("(xy)"), brackets ("[xy]"), and +// repetition count ("x{5,7}"), among others. +// +// Below is the syntax that we do support. We chose it to be a +// subset of both PCRE and POSIX extended regex, so it's easy to +// learn wherever you come from. In the following: 'A' denotes a +// literal character, period (.), or a single \\ escape sequence; +// 'x' and 'y' denote regular expressions; 'm' and 'n' are for +// natural numbers. +// +// c matches any literal character c +// \\d matches any decimal digit +// \\D matches any character that's not a decimal digit +// \\f matches \f +// \\n matches \n +// \\r matches \r +// \\s matches any ASCII whitespace, including \n +// \\S matches any character that's not a whitespace +// \\t matches \t +// \\v matches \v +// \\w matches any letter, _, or decimal digit +// \\W matches any character that \\w doesn't match +// \\c matches any literal character c, which must be a punctuation +// . matches any single character except \n +// A? matches 0 or 1 occurrences of A +// A* matches 0 or many occurrences of A +// A+ matches 1 or many occurrences of A +// ^ matches the beginning of a string (not that of each line) +// $ matches the end of a string (not that of each line) +// xy matches x followed by y +// +// If you accidentally use PCRE or POSIX extended regex features +// not implemented by us, you will get a run-time failure. In that +// case, please try to rewrite your regular expression within the +// above syntax. +// +// This implementation is *not* meant to be as highly tuned or robust +// as a compiled regex library, but should perform well enough for a +// death test, which already incurs significant overhead by launching +// a child process. +// +// Known caveats: +// +// A "threadsafe" style death test obtains the path to the test +// program from argv[0] and re-executes it in the sub-process. For +// simplicity, the current implementation doesn't search the PATH +// when launching the sub-process. This means that the user must +// invoke the test program via a path that contains at least one +// path separator (e.g. path/to/foo_test and +// /absolute/path/to/bar_test are fine, but foo_test is not). This +// is rarely a problem as people usually don't put the test binary +// directory in PATH. +// +// TODO(wan@google.com): make thread-safe death tests search the PATH. + +// Asserts that a given statement causes the program to exit, with an +// integer exit status that satisfies predicate, and emitting error output +// that matches regex. +# define ASSERT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_FATAL_FAILURE_) + +// Like ASSERT_EXIT, but continues on to successive tests in the +// test case, if any: +# define EXPECT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_NONFATAL_FAILURE_) + +// Asserts that a given statement causes the program to exit, either by +// explicitly exiting with a nonzero exit code or being killed by a +// signal, and emitting error output that matches regex. +# define ASSERT_DEATH(statement, regex) \ + ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Like ASSERT_DEATH, but continues on to successive tests in the +// test case, if any: +# define EXPECT_DEATH(statement, regex) \ + EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Two predicate classes that can be used in {ASSERT,EXPECT}_EXIT*: + +// Tests that an exit code describes a normal exit with a given exit code. +class GTEST_API_ ExitedWithCode { + public: + explicit ExitedWithCode(int exit_code); + bool operator()(int exit_status) const; + private: + // No implementation - assignment is unsupported. + void operator=(const ExitedWithCode& other); + + const int exit_code_; +}; + +# if !GTEST_OS_WINDOWS +// Tests that an exit code describes an exit due to termination by a +// given signal. +class GTEST_API_ KilledBySignal { + public: + explicit KilledBySignal(int signum); + bool operator()(int exit_status) const; + private: + const int signum_; +}; +# endif // !GTEST_OS_WINDOWS + +// EXPECT_DEBUG_DEATH asserts that the given statements die in debug mode. +// The death testing framework causes this to have interesting semantics, +// since the sideeffects of the call are only visible in opt mode, and not +// in debug mode. +// +// In practice, this can be used to test functions that utilize the +// LOG(DFATAL) macro using the following style: +// +// int DieInDebugOr12(int* sideeffect) { +// if (sideeffect) { +// *sideeffect = 12; +// } +// LOG(DFATAL) << "death"; +// return 12; +// } +// +// TEST(TestCase, TestDieOr12WorksInDgbAndOpt) { +// int sideeffect = 0; +// // Only asserts in dbg. +// EXPECT_DEBUG_DEATH(DieInDebugOr12(&sideeffect), "death"); +// +// #ifdef NDEBUG +// // opt-mode has sideeffect visible. +// EXPECT_EQ(12, sideeffect); +// #else +// // dbg-mode no visible sideeffect. +// EXPECT_EQ(0, sideeffect); +// #endif +// } +// +// This will assert that DieInDebugReturn12InOpt() crashes in debug +// mode, usually due to a DCHECK or LOG(DFATAL), but returns the +// appropriate fallback value (12 in this case) in opt mode. If you +// need to test that a function has appropriate side-effects in opt +// mode, include assertions against the side-effects. A general +// pattern for this is: +// +// EXPECT_DEBUG_DEATH({ +// // Side-effects here will have an effect after this statement in +// // opt mode, but none in debug mode. +// EXPECT_EQ(12, DieInDebugOr12(&sideeffect)); +// }, "death"); +// +# ifdef NDEBUG + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + GTEST_EXECUTE_STATEMENT_(statement, regex) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + GTEST_EXECUTE_STATEMENT_(statement, regex) + +# else + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + EXPECT_DEATH(statement, regex) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + ASSERT_DEATH(statement, regex) + +# endif // NDEBUG for EXPECT_DEBUG_DEATH +#endif // GTEST_HAS_DEATH_TEST + +// EXPECT_DEATH_IF_SUPPORTED(statement, regex) and +// ASSERT_DEATH_IF_SUPPORTED(statement, regex) expand to real death tests if +// death tests are supported; otherwise they just issue a warning. This is +// useful when you are combining death test assertions with normal test +// assertions in one test. +#if GTEST_HAS_DEATH_TEST +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH(statement, regex) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + ASSERT_DEATH(statement, regex) +#else +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, ) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, return) +#endif + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest-message.h b/mypyc/external/googletest/include/gtest/gtest-message.h new file mode 100644 index 000000000000..fe879bca7927 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-message.h @@ -0,0 +1,250 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the Message class. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! + +#ifndef GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#define GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ + +#include + +#include "gtest/internal/gtest-port.h" + +// Ensures that there is at least one operator<< in the global namespace. +// See Message& operator<<(...) below for why. +void operator<<(const testing::internal::Secret&, int); + +namespace testing { + +// The Message class works like an ostream repeater. +// +// Typical usage: +// +// 1. You stream a bunch of values to a Message object. +// It will remember the text in a stringstream. +// 2. Then you stream the Message object to an ostream. +// This causes the text in the Message to be streamed +// to the ostream. +// +// For example; +// +// testing::Message foo; +// foo << 1 << " != " << 2; +// std::cout << foo; +// +// will print "1 != 2". +// +// Message is not intended to be inherited from. In particular, its +// destructor is not virtual. +// +// Note that stringstream behaves differently in gcc and in MSVC. You +// can stream a NULL char pointer to it in the former, but not in the +// latter (it causes an access violation if you do). The Message +// class hides this difference by treating a NULL char pointer as +// "(null)". +class GTEST_API_ Message { + private: + // The type of basic IO manipulators (endl, ends, and flush) for + // narrow streams. + typedef std::ostream& (*BasicNarrowIoManip)(std::ostream&); + + public: + // Constructs an empty Message. + Message(); + + // Copy constructor. + Message(const Message& msg) : ss_(new ::std::stringstream) { // NOLINT + *ss_ << msg.GetString(); + } + + // Constructs a Message from a C-string. + explicit Message(const char* str) : ss_(new ::std::stringstream) { + *ss_ << str; + } + +#if GTEST_OS_SYMBIAN + // Streams a value (either a pointer or not) to this object. + template + inline Message& operator <<(const T& value) { + StreamHelper(typename internal::is_pointer::type(), value); + return *this; + } +#else + // Streams a non-pointer value to this object. + template + inline Message& operator <<(const T& val) { + // Some libraries overload << for STL containers. These + // overloads are defined in the global namespace instead of ::std. + // + // C++'s symbol lookup rule (i.e. Koenig lookup) says that these + // overloads are visible in either the std namespace or the global + // namespace, but not other namespaces, including the testing + // namespace which Google Test's Message class is in. + // + // To allow STL containers (and other types that has a << operator + // defined in the global namespace) to be used in Google Test + // assertions, testing::Message must access the custom << operator + // from the global namespace. With this using declaration, + // overloads of << defined in the global namespace and those + // visible via Koenig lookup are both exposed in this function. + using ::operator <<; + *ss_ << val; + return *this; + } + + // Streams a pointer value to this object. + // + // This function is an overload of the previous one. When you + // stream a pointer to a Message, this definition will be used as it + // is more specialized. (The C++ Standard, section + // [temp.func.order].) If you stream a non-pointer, then the + // previous definition will be used. + // + // The reason for this overload is that streaming a NULL pointer to + // ostream is undefined behavior. Depending on the compiler, you + // may get "0", "(nil)", "(null)", or an access violation. To + // ensure consistent result across compilers, we always treat NULL + // as "(null)". + template + inline Message& operator <<(T* const& pointer) { // NOLINT + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + *ss_ << pointer; + } + return *this; + } +#endif // GTEST_OS_SYMBIAN + + // Since the basic IO manipulators are overloaded for both narrow + // and wide streams, we have to provide this specialized definition + // of operator <<, even though its body is the same as the + // templatized version above. Without this definition, streaming + // endl or other basic IO manipulators to Message will confuse the + // compiler. + Message& operator <<(BasicNarrowIoManip val) { + *ss_ << val; + return *this; + } + + // Instead of 1/0, we want to see true/false for bool values. + Message& operator <<(bool b) { + return *this << (b ? "true" : "false"); + } + + // These two overloads allow streaming a wide C string to a Message + // using the UTF-8 encoding. + Message& operator <<(const wchar_t* wide_c_str); + Message& operator <<(wchar_t* wide_c_str); + +#if GTEST_HAS_STD_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::std::wstring& wstr); +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::wstring& wstr); +#endif // GTEST_HAS_GLOBAL_WSTRING + + // Gets the text streamed to this object so far as an std::string. + // Each '\0' character in the buffer is replaced with "\\0". + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + std::string GetString() const; + + private: + +#if GTEST_OS_SYMBIAN + // These are needed as the Nokia Symbian Compiler cannot decide between + // const T& and const T* in a function template. The Nokia compiler _can_ + // decide between class template specializations for T and T*, so a + // tr1::type_traits-like is_pointer works, and we can overload on that. + template + inline void StreamHelper(internal::true_type /*is_pointer*/, T* pointer) { + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + *ss_ << pointer; + } + } + template + inline void StreamHelper(internal::false_type /*is_pointer*/, + const T& value) { + // See the comments in Message& operator <<(const T&) above for why + // we need this using statement. + using ::operator <<; + *ss_ << value; + } +#endif // GTEST_OS_SYMBIAN + + // We'll hold the text streamed to this object here. + const internal::scoped_ptr< ::std::stringstream> ss_; + + // We declare (but don't implement) this to prevent the compiler + // from implementing the assignment operator. + void operator=(const Message&); +}; + +// Streams a Message to an ostream. +inline std::ostream& operator <<(std::ostream& os, const Message& sb) { + return os << sb.GetString(); +} + +namespace internal { + +// Converts a streamable value to an std::string. A NULL pointer is +// converted to "(null)". When the input value is a ::string, +// ::std::string, ::wstring, or ::std::wstring object, each NUL +// character in it is replaced with "\\0". +template +std::string StreamableToString(const T& streamable) { + return (Message() << streamable).GetString(); +} + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest-param-test.h b/mypyc/external/googletest/include/gtest/gtest-param-test.h new file mode 100644 index 000000000000..038f9ba79eb3 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-param-test.h @@ -0,0 +1,1444 @@ +// This file was GENERATED by command: +// pump.py gtest-param-test.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: vladl@google.com (Vlad Losev) +// +// Macros and functions for implementing parameterized tests +// in Google C++ Testing Framework (Google Test) +// +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ + + +// Value-parameterized tests allow you to test your code with different +// parameters without writing multiple copies of the same test. +// +// Here is how you use value-parameterized tests: + +#if 0 + +// To write value-parameterized tests, first you should define a fixture +// class. It is usually derived from testing::TestWithParam (see below for +// another inheritance scheme that's sometimes useful in more complicated +// class hierarchies), where the type of your parameter values. +// TestWithParam is itself derived from testing::Test. T can be any +// copyable type. If it's a raw pointer, you are responsible for managing the +// lifespan of the pointed values. + +class FooTest : public ::testing::TestWithParam { + // You can implement all the usual class fixture members here. +}; + +// Then, use the TEST_P macro to define as many parameterized tests +// for this fixture as you want. The _P suffix is for "parameterized" +// or "pattern", whichever you prefer to think. + +TEST_P(FooTest, DoesBlah) { + // Inside a test, access the test parameter with the GetParam() method + // of the TestWithParam class: + EXPECT_TRUE(foo.Blah(GetParam())); + ... +} + +TEST_P(FooTest, HasBlahBlah) { + ... +} + +// Finally, you can use INSTANTIATE_TEST_CASE_P to instantiate the test +// case with any set of parameters you want. Google Test defines a number +// of functions for generating test parameters. They return what we call +// (surprise!) parameter generators. Here is a summary of them, which +// are all in the testing namespace: +// +// +// Range(begin, end [, step]) - Yields values {begin, begin+step, +// begin+step+step, ...}. The values do not +// include end. step defaults to 1. +// Values(v1, v2, ..., vN) - Yields values {v1, v2, ..., vN}. +// ValuesIn(container) - Yields values from a C-style array, an STL +// ValuesIn(begin,end) container, or an iterator range [begin, end). +// Bool() - Yields sequence {false, true}. +// Combine(g1, g2, ..., gN) - Yields all combinations (the Cartesian product +// for the math savvy) of the values generated +// by the N generators. +// +// For more details, see comments at the definitions of these functions below +// in this file. +// +// The following statement will instantiate tests from the FooTest test case +// each with parameter values "meeny", "miny", and "moe". + +INSTANTIATE_TEST_CASE_P(InstantiationName, + FooTest, + Values("meeny", "miny", "moe")); + +// To distinguish different instances of the pattern, (yes, you +// can instantiate it more then once) the first argument to the +// INSTANTIATE_TEST_CASE_P macro is a prefix that will be added to the +// actual test case name. Remember to pick unique prefixes for different +// instantiations. The tests from the instantiation above will have +// these names: +// +// * InstantiationName/FooTest.DoesBlah/0 for "meeny" +// * InstantiationName/FooTest.DoesBlah/1 for "miny" +// * InstantiationName/FooTest.DoesBlah/2 for "moe" +// * InstantiationName/FooTest.HasBlahBlah/0 for "meeny" +// * InstantiationName/FooTest.HasBlahBlah/1 for "miny" +// * InstantiationName/FooTest.HasBlahBlah/2 for "moe" +// +// You can use these names in --gtest_filter. +// +// This statement will instantiate all tests from FooTest again, each +// with parameter values "cat" and "dog": + +const char* pets[] = {"cat", "dog"}; +INSTANTIATE_TEST_CASE_P(AnotherInstantiationName, FooTest, ValuesIn(pets)); + +// The tests from the instantiation above will have these names: +// +// * AnotherInstantiationName/FooTest.DoesBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.DoesBlah/1 for "dog" +// * AnotherInstantiationName/FooTest.HasBlahBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.HasBlahBlah/1 for "dog" +// +// Please note that INSTANTIATE_TEST_CASE_P will instantiate all tests +// in the given test case, whether their definitions come before or +// AFTER the INSTANTIATE_TEST_CASE_P statement. +// +// Please also note that generator expressions (including parameters to the +// generators) are evaluated in InitGoogleTest(), after main() has started. +// This allows the user on one hand, to adjust generator parameters in order +// to dynamically determine a set of tests to run and on the other hand, +// give the user a chance to inspect the generated tests with Google Test +// reflection API before RUN_ALL_TESTS() is executed. +// +// You can see samples/sample7_unittest.cc and samples/sample8_unittest.cc +// for more examples. +// +// In the future, we plan to publish the API for defining new parameter +// generators. But for now this interface remains part of the internal +// implementation and is subject to change. +// +// +// A parameterized test fixture must be derived from testing::Test and from +// testing::WithParamInterface, where T is the type of the parameter +// values. Inheriting from TestWithParam satisfies that requirement because +// TestWithParam inherits from both Test and WithParamInterface. In more +// complicated hierarchies, however, it is occasionally useful to inherit +// separately from Test and WithParamInterface. For example: + +class BaseTest : public ::testing::Test { + // You can inherit all the usual members for a non-parameterized test + // fixture here. +}; + +class DerivedTest : public BaseTest, public ::testing::WithParamInterface { + // The usual test fixture members go here too. +}; + +TEST_F(BaseTest, HasFoo) { + // This is an ordinary non-parameterized test. +} + +TEST_P(DerivedTest, DoesBlah) { + // GetParam works just the same here as if you inherit from TestWithParam. + EXPECT_TRUE(foo.Blah(GetParam())); +} + +#endif // 0 + +#include "gtest/internal/gtest-port.h" + +#if !GTEST_OS_SYMBIAN +# include +#endif + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-param-util-generated.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Functions producing parameter generators. +// +// Google Test uses these generators to produce parameters for value- +// parameterized tests. When a parameterized test case is instantiated +// with a particular generator, Google Test creates and runs tests +// for each element in the sequence produced by the generator. +// +// In the following sample, tests from test case FooTest are instantiated +// each three times with parameter values 3, 5, and 8: +// +// class FooTest : public TestWithParam { ... }; +// +// TEST_P(FooTest, TestThis) { +// } +// TEST_P(FooTest, TestThat) { +// } +// INSTANTIATE_TEST_CASE_P(TestSequence, FooTest, Values(3, 5, 8)); +// + +// Range() returns generators providing sequences of values in a range. +// +// Synopsis: +// Range(start, end) +// - returns a generator producing a sequence of values {start, start+1, +// start+2, ..., }. +// Range(start, end, step) +// - returns a generator producing a sequence of values {start, start+step, +// start+step+step, ..., }. +// Notes: +// * The generated sequences never include end. For example, Range(1, 5) +// returns a generator producing a sequence {1, 2, 3, 4}. Range(1, 9, 2) +// returns a generator producing {1, 3, 5, 7}. +// * start and end must have the same type. That type may be any integral or +// floating-point type or a user defined type satisfying these conditions: +// * It must be assignable (have operator=() defined). +// * It must have operator+() (operator+(int-compatible type) for +// two-operand version). +// * It must have operator<() defined. +// Elements in the resulting sequences will also have that type. +// * Condition start < end must be satisfied in order for resulting sequences +// to contain any elements. +// +template +internal::ParamGenerator Range(T start, T end, IncrementT step) { + return internal::ParamGenerator( + new internal::RangeGenerator(start, end, step)); +} + +template +internal::ParamGenerator Range(T start, T end) { + return Range(start, end, 1); +} + +// ValuesIn() function allows generation of tests with parameters coming from +// a container. +// +// Synopsis: +// ValuesIn(const T (&array)[N]) +// - returns a generator producing sequences with elements from +// a C-style array. +// ValuesIn(const Container& container) +// - returns a generator producing sequences with elements from +// an STL-style container. +// ValuesIn(Iterator begin, Iterator end) +// - returns a generator producing sequences with elements from +// a range [begin, end) defined by a pair of STL-style iterators. These +// iterators can also be plain C pointers. +// +// Please note that ValuesIn copies the values from the containers +// passed in and keeps them to generate tests in RUN_ALL_TESTS(). +// +// Examples: +// +// This instantiates tests from test case StringTest +// each with C-string values of "foo", "bar", and "baz": +// +// const char* strings[] = {"foo", "bar", "baz"}; +// INSTANTIATE_TEST_CASE_P(StringSequence, SrtingTest, ValuesIn(strings)); +// +// This instantiates tests from test case StlStringTest +// each with STL strings with values "a" and "b": +// +// ::std::vector< ::std::string> GetParameterStrings() { +// ::std::vector< ::std::string> v; +// v.push_back("a"); +// v.push_back("b"); +// return v; +// } +// +// INSTANTIATE_TEST_CASE_P(CharSequence, +// StlStringTest, +// ValuesIn(GetParameterStrings())); +// +// +// This will also instantiate tests from CharTest +// each with parameter values 'a' and 'b': +// +// ::std::list GetParameterChars() { +// ::std::list list; +// list.push_back('a'); +// list.push_back('b'); +// return list; +// } +// ::std::list l = GetParameterChars(); +// INSTANTIATE_TEST_CASE_P(CharSequence2, +// CharTest, +// ValuesIn(l.begin(), l.end())); +// +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end) { + typedef typename ::testing::internal::IteratorTraits + ::value_type ParamType; + return internal::ParamGenerator( + new internal::ValuesInIteratorRangeGenerator(begin, end)); +} + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]) { + return ValuesIn(array, array + N); +} + +template +internal::ParamGenerator ValuesIn( + const Container& container) { + return ValuesIn(container.begin(), container.end()); +} + +// Values() allows generating tests from explicitly specified list of +// parameters. +// +// Synopsis: +// Values(T v1, T v2, ..., T vN) +// - returns a generator producing sequences with elements v1, v2, ..., vN. +// +// For example, this instantiates tests from test case BarTest each +// with values "one", "two", and "three": +// +// INSTANTIATE_TEST_CASE_P(NumSequence, BarTest, Values("one", "two", "three")); +// +// This instantiates tests from test case BazTest each with values 1, 2, 3.5. +// The exact type of values will depend on the type of parameter in BazTest. +// +// INSTANTIATE_TEST_CASE_P(FloatingNumbers, BazTest, Values(1, 2, 3.5)); +// +// Currently, Values() supports from 1 to 50 parameters. +// +template +internal::ValueArray1 Values(T1 v1) { + return internal::ValueArray1(v1); +} + +template +internal::ValueArray2 Values(T1 v1, T2 v2) { + return internal::ValueArray2(v1, v2); +} + +template +internal::ValueArray3 Values(T1 v1, T2 v2, T3 v3) { + return internal::ValueArray3(v1, v2, v3); +} + +template +internal::ValueArray4 Values(T1 v1, T2 v2, T3 v3, T4 v4) { + return internal::ValueArray4(v1, v2, v3, v4); +} + +template +internal::ValueArray5 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5) { + return internal::ValueArray5(v1, v2, v3, v4, v5); +} + +template +internal::ValueArray6 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6) { + return internal::ValueArray6(v1, v2, v3, v4, v5, v6); +} + +template +internal::ValueArray7 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7) { + return internal::ValueArray7(v1, v2, v3, v4, v5, + v6, v7); +} + +template +internal::ValueArray8 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8) { + return internal::ValueArray8(v1, v2, v3, v4, + v5, v6, v7, v8); +} + +template +internal::ValueArray9 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9) { + return internal::ValueArray9(v1, v2, v3, + v4, v5, v6, v7, v8, v9); +} + +template +internal::ValueArray10 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10) { + return internal::ValueArray10(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10); +} + +template +internal::ValueArray11 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) { + return internal::ValueArray11(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11); +} + +template +internal::ValueArray12 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) { + return internal::ValueArray12(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12); +} + +template +internal::ValueArray13 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) { + return internal::ValueArray13(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13); +} + +template +internal::ValueArray14 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) { + return internal::ValueArray14(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14); +} + +template +internal::ValueArray15 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) { + return internal::ValueArray15(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); +} + +template +internal::ValueArray16 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16) { + return internal::ValueArray16(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16); +} + +template +internal::ValueArray17 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17) { + return internal::ValueArray17(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17); +} + +template +internal::ValueArray18 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18) { + return internal::ValueArray18(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18); +} + +template +internal::ValueArray19 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19) { + return internal::ValueArray19(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19); +} + +template +internal::ValueArray20 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20) { + return internal::ValueArray20(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20); +} + +template +internal::ValueArray21 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21) { + return internal::ValueArray21(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21); +} + +template +internal::ValueArray22 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22) { + return internal::ValueArray22(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22); +} + +template +internal::ValueArray23 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23) { + return internal::ValueArray23(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23); +} + +template +internal::ValueArray24 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24) { + return internal::ValueArray24(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24); +} + +template +internal::ValueArray25 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25) { + return internal::ValueArray25(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25); +} + +template +internal::ValueArray26 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) { + return internal::ValueArray26(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26); +} + +template +internal::ValueArray27 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) { + return internal::ValueArray27(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27); +} + +template +internal::ValueArray28 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) { + return internal::ValueArray28(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28); +} + +template +internal::ValueArray29 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) { + return internal::ValueArray29(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29); +} + +template +internal::ValueArray30 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) { + return internal::ValueArray30(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30); +} + +template +internal::ValueArray31 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) { + return internal::ValueArray31(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31); +} + +template +internal::ValueArray32 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32) { + return internal::ValueArray32(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32); +} + +template +internal::ValueArray33 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33) { + return internal::ValueArray33(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33); +} + +template +internal::ValueArray34 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34) { + return internal::ValueArray34(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34); +} + +template +internal::ValueArray35 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35) { + return internal::ValueArray35(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35); +} + +template +internal::ValueArray36 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36) { + return internal::ValueArray36(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36); +} + +template +internal::ValueArray37 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37) { + return internal::ValueArray37(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37); +} + +template +internal::ValueArray38 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38) { + return internal::ValueArray38(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, + v33, v34, v35, v36, v37, v38); +} + +template +internal::ValueArray39 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38, T39 v39) { + return internal::ValueArray39(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, + v32, v33, v34, v35, v36, v37, v38, v39); +} + +template +internal::ValueArray40 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, + T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, + T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) { + return internal::ValueArray40(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, + v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40); +} + +template +internal::ValueArray41 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41) { + return internal::ValueArray41(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, + v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41); +} + +template +internal::ValueArray42 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) { + return internal::ValueArray42(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, + v42); +} + +template +internal::ValueArray43 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) { + return internal::ValueArray43(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, + v41, v42, v43); +} + +template +internal::ValueArray44 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) { + return internal::ValueArray44(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, + v40, v41, v42, v43, v44); +} + +template +internal::ValueArray45 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41, T42 v42, T43 v43, T44 v44, T45 v45) { + return internal::ValueArray45(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, + v39, v40, v41, v42, v43, v44, v45); +} + +template +internal::ValueArray46 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) { + return internal::ValueArray46(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46); +} + +template +internal::ValueArray47 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) { + return internal::ValueArray47(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46, v47); +} + +template +internal::ValueArray48 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, + T48 v48) { + return internal::ValueArray48(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, + v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48); +} + +template +internal::ValueArray49 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, + T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, + T47 v47, T48 v48, T49 v49) { + return internal::ValueArray49(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, + v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49); +} + +template +internal::ValueArray50 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, + T38 v38, T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, + T46 v46, T47 v47, T48 v48, T49 v49, T50 v50) { + return internal::ValueArray50(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, + v48, v49, v50); +} + +// Bool() allows generating tests with parameters in a set of (false, true). +// +// Synopsis: +// Bool() +// - returns a generator producing sequences with elements {false, true}. +// +// It is useful when testing code that depends on Boolean flags. Combinations +// of multiple flags can be tested when several Bool()'s are combined using +// Combine() function. +// +// In the following example all tests in the test case FlagDependentTest +// will be instantiated twice with parameters false and true. +// +// class FlagDependentTest : public testing::TestWithParam { +// virtual void SetUp() { +// external_flag = GetParam(); +// } +// } +// INSTANTIATE_TEST_CASE_P(BoolSequence, FlagDependentTest, Bool()); +// +inline internal::ParamGenerator Bool() { + return Values(false, true); +} + +# if GTEST_HAS_COMBINE +// Combine() allows the user to combine two or more sequences to produce +// values of a Cartesian product of those sequences' elements. +// +// Synopsis: +// Combine(gen1, gen2, ..., genN) +// - returns a generator producing sequences with elements coming from +// the Cartesian product of elements from the sequences generated by +// gen1, gen2, ..., genN. The sequence elements will have a type of +// tuple where T1, T2, ..., TN are the types +// of elements from sequences produces by gen1, gen2, ..., genN. +// +// Combine can have up to 10 arguments. This number is currently limited +// by the maximum number of elements in the tuple implementation used by Google +// Test. +// +// Example: +// +// This will instantiate tests in test case AnimalTest each one with +// the parameter values tuple("cat", BLACK), tuple("cat", WHITE), +// tuple("dog", BLACK), and tuple("dog", WHITE): +// +// enum Color { BLACK, GRAY, WHITE }; +// class AnimalTest +// : public testing::TestWithParam > {...}; +// +// TEST_P(AnimalTest, AnimalLooksNice) {...} +// +// INSTANTIATE_TEST_CASE_P(AnimalVariations, AnimalTest, +// Combine(Values("cat", "dog"), +// Values(BLACK, WHITE))); +// +// This will instantiate tests in FlagDependentTest with all variations of two +// Boolean flags: +// +// class FlagDependentTest +// : public testing::TestWithParam > { +// virtual void SetUp() { +// // Assigns external_flag_1 and external_flag_2 values from the tuple. +// tie(external_flag_1, external_flag_2) = GetParam(); +// } +// }; +// +// TEST_P(FlagDependentTest, TestFeature1) { +// // Test your code using external_flag_1 and external_flag_2 here. +// } +// INSTANTIATE_TEST_CASE_P(TwoBoolSequence, FlagDependentTest, +// Combine(Bool(), Bool())); +// +template +internal::CartesianProductHolder2 Combine( + const Generator1& g1, const Generator2& g2) { + return internal::CartesianProductHolder2( + g1, g2); +} + +template +internal::CartesianProductHolder3 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3) { + return internal::CartesianProductHolder3( + g1, g2, g3); +} + +template +internal::CartesianProductHolder4 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4) { + return internal::CartesianProductHolder4( + g1, g2, g3, g4); +} + +template +internal::CartesianProductHolder5 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5) { + return internal::CartesianProductHolder5( + g1, g2, g3, g4, g5); +} + +template +internal::CartesianProductHolder6 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6) { + return internal::CartesianProductHolder6( + g1, g2, g3, g4, g5, g6); +} + +template +internal::CartesianProductHolder7 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7) { + return internal::CartesianProductHolder7( + g1, g2, g3, g4, g5, g6, g7); +} + +template +internal::CartesianProductHolder8 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8) { + return internal::CartesianProductHolder8( + g1, g2, g3, g4, g5, g6, g7, g8); +} + +template +internal::CartesianProductHolder9 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9) { + return internal::CartesianProductHolder9( + g1, g2, g3, g4, g5, g6, g7, g8, g9); +} + +template +internal::CartesianProductHolder10 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9, + const Generator10& g10) { + return internal::CartesianProductHolder10( + g1, g2, g3, g4, g5, g6, g7, g8, g9, g10); +} +# endif // GTEST_HAS_COMBINE + + + +# define TEST_P(test_case_name, test_name) \ + class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + : public test_case_name { \ + public: \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ + virtual void TestBody(); \ + private: \ + static int AddToRegistry() { \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestPattern(\ + #test_case_name, \ + #test_name, \ + new ::testing::internal::TestMetaFactory< \ + GTEST_TEST_CLASS_NAME_(\ + test_case_name, test_name)>()); \ + return 0; \ + } \ + static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)); \ + }; \ + int GTEST_TEST_CLASS_NAME_(test_case_name, \ + test_name)::gtest_registering_dummy_ = \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +// The optional last argument to INSTANTIATE_TEST_CASE_P allows the user +// to specify a function or functor that generates custom test name suffixes +// based on the test parameters. The function should accept one argument of +// type testing::TestParamInfo, and return std::string. +// +// testing::PrintToStringParamName is a builtin test suffix generator that +// returns the value of testing::PrintToString(GetParam()). It does not work +// for std::string or C strings. +// +// Note: test names must be non-empty, unique, and may only contain ASCII +// alphanumeric characters or underscore. + +# define INSTANTIATE_TEST_CASE_P(prefix, test_case_name, generator, ...) \ + ::testing::internal::ParamGenerator \ + gtest_##prefix##test_case_name##_EvalGenerator_() { return generator; } \ + ::std::string gtest_##prefix##test_case_name##_EvalGenerateName_( \ + const ::testing::TestParamInfo& info) { \ + return ::testing::internal::GetParamNameGen \ + (__VA_ARGS__)(info); \ + } \ + int gtest_##prefix##test_case_name##_dummy_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestCaseInstantiation(\ + #prefix, \ + >est_##prefix##test_case_name##_EvalGenerator_, \ + >est_##prefix##test_case_name##_EvalGenerateName_, \ + __FILE__, __LINE__) + +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest-param-test.h.pump b/mypyc/external/googletest/include/gtest/gtest-param-test.h.pump new file mode 100644 index 000000000000..3078d6d2a19d --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-param-test.h.pump @@ -0,0 +1,510 @@ +$$ -*- mode: c++; -*- +$var n = 50 $$ Maximum length of Values arguments we want to support. +$var maxtuple = 10 $$ Maximum number of Combine arguments we want to support. +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: vladl@google.com (Vlad Losev) +// +// Macros and functions for implementing parameterized tests +// in Google C++ Testing Framework (Google Test) +// +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ + + +// Value-parameterized tests allow you to test your code with different +// parameters without writing multiple copies of the same test. +// +// Here is how you use value-parameterized tests: + +#if 0 + +// To write value-parameterized tests, first you should define a fixture +// class. It is usually derived from testing::TestWithParam (see below for +// another inheritance scheme that's sometimes useful in more complicated +// class hierarchies), where the type of your parameter values. +// TestWithParam is itself derived from testing::Test. T can be any +// copyable type. If it's a raw pointer, you are responsible for managing the +// lifespan of the pointed values. + +class FooTest : public ::testing::TestWithParam { + // You can implement all the usual class fixture members here. +}; + +// Then, use the TEST_P macro to define as many parameterized tests +// for this fixture as you want. The _P suffix is for "parameterized" +// or "pattern", whichever you prefer to think. + +TEST_P(FooTest, DoesBlah) { + // Inside a test, access the test parameter with the GetParam() method + // of the TestWithParam class: + EXPECT_TRUE(foo.Blah(GetParam())); + ... +} + +TEST_P(FooTest, HasBlahBlah) { + ... +} + +// Finally, you can use INSTANTIATE_TEST_CASE_P to instantiate the test +// case with any set of parameters you want. Google Test defines a number +// of functions for generating test parameters. They return what we call +// (surprise!) parameter generators. Here is a summary of them, which +// are all in the testing namespace: +// +// +// Range(begin, end [, step]) - Yields values {begin, begin+step, +// begin+step+step, ...}. The values do not +// include end. step defaults to 1. +// Values(v1, v2, ..., vN) - Yields values {v1, v2, ..., vN}. +// ValuesIn(container) - Yields values from a C-style array, an STL +// ValuesIn(begin,end) container, or an iterator range [begin, end). +// Bool() - Yields sequence {false, true}. +// Combine(g1, g2, ..., gN) - Yields all combinations (the Cartesian product +// for the math savvy) of the values generated +// by the N generators. +// +// For more details, see comments at the definitions of these functions below +// in this file. +// +// The following statement will instantiate tests from the FooTest test case +// each with parameter values "meeny", "miny", and "moe". + +INSTANTIATE_TEST_CASE_P(InstantiationName, + FooTest, + Values("meeny", "miny", "moe")); + +// To distinguish different instances of the pattern, (yes, you +// can instantiate it more then once) the first argument to the +// INSTANTIATE_TEST_CASE_P macro is a prefix that will be added to the +// actual test case name. Remember to pick unique prefixes for different +// instantiations. The tests from the instantiation above will have +// these names: +// +// * InstantiationName/FooTest.DoesBlah/0 for "meeny" +// * InstantiationName/FooTest.DoesBlah/1 for "miny" +// * InstantiationName/FooTest.DoesBlah/2 for "moe" +// * InstantiationName/FooTest.HasBlahBlah/0 for "meeny" +// * InstantiationName/FooTest.HasBlahBlah/1 for "miny" +// * InstantiationName/FooTest.HasBlahBlah/2 for "moe" +// +// You can use these names in --gtest_filter. +// +// This statement will instantiate all tests from FooTest again, each +// with parameter values "cat" and "dog": + +const char* pets[] = {"cat", "dog"}; +INSTANTIATE_TEST_CASE_P(AnotherInstantiationName, FooTest, ValuesIn(pets)); + +// The tests from the instantiation above will have these names: +// +// * AnotherInstantiationName/FooTest.DoesBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.DoesBlah/1 for "dog" +// * AnotherInstantiationName/FooTest.HasBlahBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.HasBlahBlah/1 for "dog" +// +// Please note that INSTANTIATE_TEST_CASE_P will instantiate all tests +// in the given test case, whether their definitions come before or +// AFTER the INSTANTIATE_TEST_CASE_P statement. +// +// Please also note that generator expressions (including parameters to the +// generators) are evaluated in InitGoogleTest(), after main() has started. +// This allows the user on one hand, to adjust generator parameters in order +// to dynamically determine a set of tests to run and on the other hand, +// give the user a chance to inspect the generated tests with Google Test +// reflection API before RUN_ALL_TESTS() is executed. +// +// You can see samples/sample7_unittest.cc and samples/sample8_unittest.cc +// for more examples. +// +// In the future, we plan to publish the API for defining new parameter +// generators. But for now this interface remains part of the internal +// implementation and is subject to change. +// +// +// A parameterized test fixture must be derived from testing::Test and from +// testing::WithParamInterface, where T is the type of the parameter +// values. Inheriting from TestWithParam satisfies that requirement because +// TestWithParam inherits from both Test and WithParamInterface. In more +// complicated hierarchies, however, it is occasionally useful to inherit +// separately from Test and WithParamInterface. For example: + +class BaseTest : public ::testing::Test { + // You can inherit all the usual members for a non-parameterized test + // fixture here. +}; + +class DerivedTest : public BaseTest, public ::testing::WithParamInterface { + // The usual test fixture members go here too. +}; + +TEST_F(BaseTest, HasFoo) { + // This is an ordinary non-parameterized test. +} + +TEST_P(DerivedTest, DoesBlah) { + // GetParam works just the same here as if you inherit from TestWithParam. + EXPECT_TRUE(foo.Blah(GetParam())); +} + +#endif // 0 + +#include "gtest/internal/gtest-port.h" + +#if !GTEST_OS_SYMBIAN +# include +#endif + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-param-util-generated.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Functions producing parameter generators. +// +// Google Test uses these generators to produce parameters for value- +// parameterized tests. When a parameterized test case is instantiated +// with a particular generator, Google Test creates and runs tests +// for each element in the sequence produced by the generator. +// +// In the following sample, tests from test case FooTest are instantiated +// each three times with parameter values 3, 5, and 8: +// +// class FooTest : public TestWithParam { ... }; +// +// TEST_P(FooTest, TestThis) { +// } +// TEST_P(FooTest, TestThat) { +// } +// INSTANTIATE_TEST_CASE_P(TestSequence, FooTest, Values(3, 5, 8)); +// + +// Range() returns generators providing sequences of values in a range. +// +// Synopsis: +// Range(start, end) +// - returns a generator producing a sequence of values {start, start+1, +// start+2, ..., }. +// Range(start, end, step) +// - returns a generator producing a sequence of values {start, start+step, +// start+step+step, ..., }. +// Notes: +// * The generated sequences never include end. For example, Range(1, 5) +// returns a generator producing a sequence {1, 2, 3, 4}. Range(1, 9, 2) +// returns a generator producing {1, 3, 5, 7}. +// * start and end must have the same type. That type may be any integral or +// floating-point type or a user defined type satisfying these conditions: +// * It must be assignable (have operator=() defined). +// * It must have operator+() (operator+(int-compatible type) for +// two-operand version). +// * It must have operator<() defined. +// Elements in the resulting sequences will also have that type. +// * Condition start < end must be satisfied in order for resulting sequences +// to contain any elements. +// +template +internal::ParamGenerator Range(T start, T end, IncrementT step) { + return internal::ParamGenerator( + new internal::RangeGenerator(start, end, step)); +} + +template +internal::ParamGenerator Range(T start, T end) { + return Range(start, end, 1); +} + +// ValuesIn() function allows generation of tests with parameters coming from +// a container. +// +// Synopsis: +// ValuesIn(const T (&array)[N]) +// - returns a generator producing sequences with elements from +// a C-style array. +// ValuesIn(const Container& container) +// - returns a generator producing sequences with elements from +// an STL-style container. +// ValuesIn(Iterator begin, Iterator end) +// - returns a generator producing sequences with elements from +// a range [begin, end) defined by a pair of STL-style iterators. These +// iterators can also be plain C pointers. +// +// Please note that ValuesIn copies the values from the containers +// passed in and keeps them to generate tests in RUN_ALL_TESTS(). +// +// Examples: +// +// This instantiates tests from test case StringTest +// each with C-string values of "foo", "bar", and "baz": +// +// const char* strings[] = {"foo", "bar", "baz"}; +// INSTANTIATE_TEST_CASE_P(StringSequence, SrtingTest, ValuesIn(strings)); +// +// This instantiates tests from test case StlStringTest +// each with STL strings with values "a" and "b": +// +// ::std::vector< ::std::string> GetParameterStrings() { +// ::std::vector< ::std::string> v; +// v.push_back("a"); +// v.push_back("b"); +// return v; +// } +// +// INSTANTIATE_TEST_CASE_P(CharSequence, +// StlStringTest, +// ValuesIn(GetParameterStrings())); +// +// +// This will also instantiate tests from CharTest +// each with parameter values 'a' and 'b': +// +// ::std::list GetParameterChars() { +// ::std::list list; +// list.push_back('a'); +// list.push_back('b'); +// return list; +// } +// ::std::list l = GetParameterChars(); +// INSTANTIATE_TEST_CASE_P(CharSequence2, +// CharTest, +// ValuesIn(l.begin(), l.end())); +// +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end) { + typedef typename ::testing::internal::IteratorTraits + ::value_type ParamType; + return internal::ParamGenerator( + new internal::ValuesInIteratorRangeGenerator(begin, end)); +} + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]) { + return ValuesIn(array, array + N); +} + +template +internal::ParamGenerator ValuesIn( + const Container& container) { + return ValuesIn(container.begin(), container.end()); +} + +// Values() allows generating tests from explicitly specified list of +// parameters. +// +// Synopsis: +// Values(T v1, T v2, ..., T vN) +// - returns a generator producing sequences with elements v1, v2, ..., vN. +// +// For example, this instantiates tests from test case BarTest each +// with values "one", "two", and "three": +// +// INSTANTIATE_TEST_CASE_P(NumSequence, BarTest, Values("one", "two", "three")); +// +// This instantiates tests from test case BazTest each with values 1, 2, 3.5. +// The exact type of values will depend on the type of parameter in BazTest. +// +// INSTANTIATE_TEST_CASE_P(FloatingNumbers, BazTest, Values(1, 2, 3.5)); +// +// Currently, Values() supports from 1 to $n parameters. +// +$range i 1..n +$for i [[ +$range j 1..i + +template <$for j, [[typename T$j]]> +internal::ValueArray$i<$for j, [[T$j]]> Values($for j, [[T$j v$j]]) { + return internal::ValueArray$i<$for j, [[T$j]]>($for j, [[v$j]]); +} + +]] + +// Bool() allows generating tests with parameters in a set of (false, true). +// +// Synopsis: +// Bool() +// - returns a generator producing sequences with elements {false, true}. +// +// It is useful when testing code that depends on Boolean flags. Combinations +// of multiple flags can be tested when several Bool()'s are combined using +// Combine() function. +// +// In the following example all tests in the test case FlagDependentTest +// will be instantiated twice with parameters false and true. +// +// class FlagDependentTest : public testing::TestWithParam { +// virtual void SetUp() { +// external_flag = GetParam(); +// } +// } +// INSTANTIATE_TEST_CASE_P(BoolSequence, FlagDependentTest, Bool()); +// +inline internal::ParamGenerator Bool() { + return Values(false, true); +} + +# if GTEST_HAS_COMBINE +// Combine() allows the user to combine two or more sequences to produce +// values of a Cartesian product of those sequences' elements. +// +// Synopsis: +// Combine(gen1, gen2, ..., genN) +// - returns a generator producing sequences with elements coming from +// the Cartesian product of elements from the sequences generated by +// gen1, gen2, ..., genN. The sequence elements will have a type of +// tuple where T1, T2, ..., TN are the types +// of elements from sequences produces by gen1, gen2, ..., genN. +// +// Combine can have up to $maxtuple arguments. This number is currently limited +// by the maximum number of elements in the tuple implementation used by Google +// Test. +// +// Example: +// +// This will instantiate tests in test case AnimalTest each one with +// the parameter values tuple("cat", BLACK), tuple("cat", WHITE), +// tuple("dog", BLACK), and tuple("dog", WHITE): +// +// enum Color { BLACK, GRAY, WHITE }; +// class AnimalTest +// : public testing::TestWithParam > {...}; +// +// TEST_P(AnimalTest, AnimalLooksNice) {...} +// +// INSTANTIATE_TEST_CASE_P(AnimalVariations, AnimalTest, +// Combine(Values("cat", "dog"), +// Values(BLACK, WHITE))); +// +// This will instantiate tests in FlagDependentTest with all variations of two +// Boolean flags: +// +// class FlagDependentTest +// : public testing::TestWithParam > { +// virtual void SetUp() { +// // Assigns external_flag_1 and external_flag_2 values from the tuple. +// tie(external_flag_1, external_flag_2) = GetParam(); +// } +// }; +// +// TEST_P(FlagDependentTest, TestFeature1) { +// // Test your code using external_flag_1 and external_flag_2 here. +// } +// INSTANTIATE_TEST_CASE_P(TwoBoolSequence, FlagDependentTest, +// Combine(Bool(), Bool())); +// +$range i 2..maxtuple +$for i [[ +$range j 1..i + +template <$for j, [[typename Generator$j]]> +internal::CartesianProductHolder$i<$for j, [[Generator$j]]> Combine( + $for j, [[const Generator$j& g$j]]) { + return internal::CartesianProductHolder$i<$for j, [[Generator$j]]>( + $for j, [[g$j]]); +} + +]] +# endif // GTEST_HAS_COMBINE + + + +# define TEST_P(test_case_name, test_name) \ + class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + : public test_case_name { \ + public: \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ + virtual void TestBody(); \ + private: \ + static int AddToRegistry() { \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestPattern(\ + #test_case_name, \ + #test_name, \ + new ::testing::internal::TestMetaFactory< \ + GTEST_TEST_CLASS_NAME_(\ + test_case_name, test_name)>()); \ + return 0; \ + } \ + static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)); \ + }; \ + int GTEST_TEST_CLASS_NAME_(test_case_name, \ + test_name)::gtest_registering_dummy_ = \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +// The optional last argument to INSTANTIATE_TEST_CASE_P allows the user +// to specify a function or functor that generates custom test name suffixes +// based on the test parameters. The function should accept one argument of +// type testing::TestParamInfo, and return std::string. +// +// testing::PrintToStringParamName is a builtin test suffix generator that +// returns the value of testing::PrintToString(GetParam()). +// +// Note: test names must be non-empty, unique, and may only contain ASCII +// alphanumeric characters or underscore. Because PrintToString adds quotes +// to std::string and C strings, it won't work for these types. + +# define INSTANTIATE_TEST_CASE_P(prefix, test_case_name, generator, ...) \ + ::testing::internal::ParamGenerator \ + gtest_##prefix##test_case_name##_EvalGenerator_() { return generator; } \ + ::std::string gtest_##prefix##test_case_name##_EvalGenerateName_( \ + const ::testing::TestParamInfo& info) { \ + return ::testing::internal::GetParamNameGen \ + (__VA_ARGS__)(info); \ + } \ + int gtest_##prefix##test_case_name##_dummy_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestCaseInstantiation(\ + #prefix, \ + >est_##prefix##test_case_name##_EvalGenerator_, \ + >est_##prefix##test_case_name##_EvalGenerateName_, \ + __FILE__, __LINE__) + +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest-printers.h b/mypyc/external/googletest/include/gtest/gtest-printers.h new file mode 100644 index 000000000000..8a33164cb38a --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-printers.h @@ -0,0 +1,993 @@ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Google Test - The Google C++ Testing Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// A user can teach this function how to print a class type T by +// defining either operator<<() or PrintTo() in the namespace that +// defines T. More specifically, the FIRST defined function in the +// following list will be used (assuming T is defined in namespace +// foo): +// +// 1. foo::PrintTo(const T&, ostream*) +// 2. operator<<(ostream&, const T&) defined in either foo or the +// global namespace. +// +// If none of the above is defined, it will print the debug string of +// the value if it is a protocol buffer, or print the raw bytes in the +// value otherwise. +// +// To aid debugging: when T is a reference type, the address of the +// value is also printed; when T is a (const) char pointer, both the +// pointer value and the NUL-terminated string it points to are +// printed. +// +// We also provide some convenient wrappers: +// +// // Prints a value to a string. For a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// std::string ::testing::PrintToString(const T& value); +// +// // Prints a value tersely: for a reference type, the referenced +// // value (but not the address) is printed; for a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// void ::testing::internal::UniversalTersePrint(const T& value, ostream*); +// +// // Prints value using the type inferred by the compiler. The difference +// // from UniversalTersePrint() is that this function prints both the +// // pointer and the NUL-terminated string for a (const or not) char pointer. +// void ::testing::internal::UniversalPrint(const T& value, ostream*); +// +// // Prints the fields of a tuple tersely to a string vector, one +// // element for each field. Tuple support must be enabled in +// // gtest-port.h. +// std::vector UniversalTersePrintTupleFieldsToStrings( +// const Tuple& value); +// +// Known limitation: +// +// The print primitives print the elements of an STL-style container +// using the compiler-inferred type of *iter where iter is a +// const_iterator of the container. When const_iterator is an input +// iterator but not a forward iterator, this inferred type may not +// match value_type, and the print output may be incorrect. In +// practice, this is rarely a problem as for most containers +// const_iterator is a forward iterator. We'll fix this if there's an +// actual need for it. Note that this fix cannot rely on value_type +// being defined as many user-defined container types don't have +// value_type. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ + +#include // NOLINT +#include +#include +#include +#include +#include "gtest/internal/gtest-port.h" +#include "gtest/internal/gtest-internal.h" + +#if GTEST_HAS_STD_TUPLE_ +# include +#endif + +namespace testing { + +// Definitions in the 'internal' and 'internal2' name spaces are +// subject to change without notice. DO NOT USE THEM IN USER CODE! +namespace internal2 { + +// Prints the given number of bytes in the given object to the given +// ostream. +GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, + size_t count, + ::std::ostream* os); + +// For selecting which printer to use when a given type has neither << +// nor PrintTo(). +enum TypeKind { + kProtobuf, // a protobuf type + kConvertibleToInteger, // a type implicitly convertible to BiggestInt + // (e.g. a named or unnamed enum type) + kOtherType // anything else +}; + +// TypeWithoutFormatter::PrintValue(value, os) is called +// by the universal printer to print a value of type T when neither +// operator<< nor PrintTo() is defined for T, where kTypeKind is the +// "kind" of T as defined by enum TypeKind. +template +class TypeWithoutFormatter { + public: + // This default version is called when kTypeKind is kOtherType. + static void PrintValue(const T& value, ::std::ostream* os) { + PrintBytesInObjectTo(reinterpret_cast(&value), + sizeof(value), os); + } +}; + +// We print a protobuf using its ShortDebugString() when the string +// doesn't exceed this many characters; otherwise we print it using +// DebugString() for better readability. +const size_t kProtobufOneLinerMaxLength = 50; + +template +class TypeWithoutFormatter { + public: + static void PrintValue(const T& value, ::std::ostream* os) { + const ::testing::internal::string short_str = value.ShortDebugString(); + const ::testing::internal::string pretty_str = + short_str.length() <= kProtobufOneLinerMaxLength ? + short_str : ("\n" + value.DebugString()); + *os << ("<" + pretty_str + ">"); + } +}; + +template +class TypeWithoutFormatter { + public: + // Since T has no << operator or PrintTo() but can be implicitly + // converted to BiggestInt, we print it as a BiggestInt. + // + // Most likely T is an enum type (either named or unnamed), in which + // case printing it as an integer is the desired behavior. In case + // T is not an enum, printing it as an integer is the best we can do + // given that it has no user-defined printer. + static void PrintValue(const T& value, ::std::ostream* os) { + const internal::BiggestInt kBigInt = value; + *os << kBigInt; + } +}; + +// Prints the given value to the given ostream. If the value is a +// protocol message, its debug string is printed; if it's an enum or +// of a type implicitly convertible to BiggestInt, it's printed as an +// integer; otherwise the bytes in the value are printed. This is +// what UniversalPrinter::Print() does when it knows nothing about +// type T and T has neither << operator nor PrintTo(). +// +// A user can override this behavior for a class type Foo by defining +// a << operator in the namespace where Foo is defined. +// +// We put this operator in namespace 'internal2' instead of 'internal' +// to simplify the implementation, as much code in 'internal' needs to +// use << in STL, which would conflict with our own << were it defined +// in 'internal'. +// +// Note that this operator<< takes a generic std::basic_ostream type instead of the more restricted std::ostream. If +// we define it to take an std::ostream instead, we'll get an +// "ambiguous overloads" compiler error when trying to print a type +// Foo that supports streaming to std::basic_ostream, as the compiler cannot tell whether +// operator<<(std::ostream&, const T&) or +// operator<<(std::basic_stream, const Foo&) is more +// specific. +template +::std::basic_ostream& operator<<( + ::std::basic_ostream& os, const T& x) { + TypeWithoutFormatter::value ? kProtobuf : + internal::ImplicitlyConvertible::value ? + kConvertibleToInteger : kOtherType)>::PrintValue(x, &os); + return os; +} + +} // namespace internal2 +} // namespace testing + +// This namespace MUST NOT BE NESTED IN ::testing, or the name look-up +// magic needed for implementing UniversalPrinter won't work. +namespace testing_internal { + +// Used to print a value that is not an STL-style container when the +// user doesn't define PrintTo() for it. +template +void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) { + // With the following statement, during unqualified name lookup, + // testing::internal2::operator<< appears as if it was declared in + // the nearest enclosing namespace that contains both + // ::testing_internal and ::testing::internal2, i.e. the global + // namespace. For more details, refer to the C++ Standard section + // 7.3.4-1 [namespace.udir]. This allows us to fall back onto + // testing::internal2::operator<< in case T doesn't come with a << + // operator. + // + // We cannot write 'using ::testing::internal2::operator<<;', which + // gcc 3.3 fails to compile due to a compiler bug. + using namespace ::testing::internal2; // NOLINT + + // Assuming T is defined in namespace foo, in the next statement, + // the compiler will consider all of: + // + // 1. foo::operator<< (thanks to Koenig look-up), + // 2. ::operator<< (as the current namespace is enclosed in ::), + // 3. testing::internal2::operator<< (thanks to the using statement above). + // + // The operator<< whose type matches T best will be picked. + // + // We deliberately allow #2 to be a candidate, as sometimes it's + // impossible to define #1 (e.g. when foo is ::std, defining + // anything in it is undefined behavior unless you are a compiler + // vendor.). + *os << value; +} + +} // namespace testing_internal + +namespace testing { +namespace internal { + +// FormatForComparison::Format(value) formats a +// value of type ToPrint that is an operand of a comparison assertion +// (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in +// the comparison, and is used to help determine the best way to +// format the value. In particular, when the value is a C string +// (char pointer) and the other operand is an STL string object, we +// want to format the C string as a string, since we know it is +// compared by value with the string object. If the value is a char +// pointer but the other operand is not an STL string object, we don't +// know whether the pointer is supposed to point to a NUL-terminated +// string, and thus want to print it as a pointer to be safe. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// The default case. +template +class FormatForComparison { + public: + static ::std::string Format(const ToPrint& value) { + return ::testing::PrintToString(value); + } +}; + +// Array. +template +class FormatForComparison { + public: + static ::std::string Format(const ToPrint* value) { + return FormatForComparison::Format(value); + } +}; + +// By default, print C string as pointers to be safe, as we don't know +// whether they actually point to a NUL-terminated string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(CharType) \ + template \ + class FormatForComparison { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(static_cast(value)); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t); + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_ + +// If a C string is compared with an STL string object, we know it's meant +// to point to a NUL-terminated string, and thus can print it as a string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(CharType, OtherStringType) \ + template <> \ + class FormatForComparison { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(value); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string); + +#if GTEST_HAS_GLOBAL_STRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::string); +#endif + +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::wstring); +#endif + +#if GTEST_HAS_STD_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring); +#endif + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_STRING_ + +// Formats a comparison assertion (e.g. ASSERT_EQ, EXPECT_LT, and etc) +// operand to be used in a failure message. The type (but not value) +// of the other operand may affect the format. This allows us to +// print a char* as a raw pointer when it is compared against another +// char* or void*, and print it as a C string when it is compared +// against an std::string object, for example. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +std::string FormatForComparisonFailureMessage( + const T1& value, const T2& /* other_operand */) { + return FormatForComparison::Format(value); +} + +// UniversalPrinter::Print(value, ostream_ptr) prints the given +// value to the given ostream. The caller must ensure that +// 'ostream_ptr' is not NULL, or the behavior is undefined. +// +// We define UniversalPrinter as a class template (as opposed to a +// function template), as we need to partially specialize it for +// reference types, which cannot be done with function templates. +template +class UniversalPrinter; + +template +void UniversalPrint(const T& value, ::std::ostream* os); + +// Used to print an STL-style container when the user doesn't define +// a PrintTo() for it. +template +void DefaultPrintTo(IsContainer /* dummy */, + false_type /* is not a pointer */, + const C& container, ::std::ostream* os) { + const size_t kMaxCount = 32; // The maximum number of elements to print. + *os << '{'; + size_t count = 0; + for (typename C::const_iterator it = container.begin(); + it != container.end(); ++it, ++count) { + if (count > 0) { + *os << ','; + if (count == kMaxCount) { // Enough has been printed. + *os << " ..."; + break; + } + } + *os << ' '; + // We cannot call PrintTo(*it, os) here as PrintTo() doesn't + // handle *it being a native array. + internal::UniversalPrint(*it, os); + } + + if (count > 0) { + *os << ' '; + } + *os << '}'; +} + +// Used to print a pointer that is neither a char pointer nor a member +// pointer, when the user doesn't define PrintTo() for it. (A member +// variable pointer or member function pointer doesn't really point to +// a location in the address space. Their representation is +// implementation-defined. Therefore they will be printed as raw +// bytes.) +template +void DefaultPrintTo(IsNotContainer /* dummy */, + true_type /* is a pointer */, + T* p, ::std::ostream* os) { + if (p == NULL) { + *os << "NULL"; + } else { + // C++ doesn't allow casting from a function pointer to any object + // pointer. + // + // IsTrue() silences warnings: "Condition is always true", + // "unreachable code". + if (IsTrue(ImplicitlyConvertible::value)) { + // T is not a function type. We just call << to print p, + // relying on ADL to pick up user-defined << for their pointer + // types, if any. + *os << p; + } else { + // T is a function type, so '*os << p' doesn't do what we want + // (it just prints p as bool). We want to print p as a const + // void*. However, we cannot cast it to const void* directly, + // even using reinterpret_cast, as earlier versions of gcc + // (e.g. 3.4.5) cannot compile the cast when p is a function + // pointer. Casting to UInt64 first solves the problem. + *os << reinterpret_cast( + reinterpret_cast(p)); + } + } +} + +// Used to print a non-container, non-pointer value when the user +// doesn't define PrintTo() for it. +template +void DefaultPrintTo(IsNotContainer /* dummy */, + false_type /* is not a pointer */, + const T& value, ::std::ostream* os) { + ::testing_internal::DefaultPrintNonContainerTo(value, os); +} + +// Prints the given value using the << operator if it has one; +// otherwise prints the bytes in it. This is what +// UniversalPrinter::Print() does when PrintTo() is not specialized +// or overloaded for type T. +// +// A user can override this behavior for a class type Foo by defining +// an overload of PrintTo() in the namespace where Foo is defined. We +// give the user this option as sometimes defining a << operator for +// Foo is not desirable (e.g. the coding style may prevent doing it, +// or there is already a << operator but it doesn't do what the user +// wants). +template +void PrintTo(const T& value, ::std::ostream* os) { + // DefaultPrintTo() is overloaded. The type of its first two + // arguments determine which version will be picked. If T is an + // STL-style container, the version for container will be called; if + // T is a pointer, the pointer version will be called; otherwise the + // generic version will be called. + // + // Note that we check for container types here, prior to we check + // for protocol message types in our operator<<. The rationale is: + // + // For protocol messages, we want to give people a chance to + // override Google Mock's format by defining a PrintTo() or + // operator<<. For STL containers, other formats can be + // incompatible with Google Mock's format for the container + // elements; therefore we check for container types here to ensure + // that our format is used. + // + // The second argument of DefaultPrintTo() is needed to bypass a bug + // in Symbian's C++ compiler that prevents it from picking the right + // overload between: + // + // PrintTo(const T& x, ...); + // PrintTo(T* x, ...); + DefaultPrintTo(IsContainerTest(0), is_pointer(), value, os); +} + +// The following list of PrintTo() overloads tells +// UniversalPrinter::Print() how to print standard types (built-in +// types, strings, plain arrays, and pointers). + +// Overloads for various char types. +GTEST_API_ void PrintTo(unsigned char c, ::std::ostream* os); +GTEST_API_ void PrintTo(signed char c, ::std::ostream* os); +inline void PrintTo(char c, ::std::ostream* os) { + // When printing a plain char, we always treat it as unsigned. This + // way, the output won't be affected by whether the compiler thinks + // char is signed or not. + PrintTo(static_cast(c), os); +} + +// Overloads for other simple built-in types. +inline void PrintTo(bool x, ::std::ostream* os) { + *os << (x ? "true" : "false"); +} + +// Overload for wchar_t type. +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its decimal code (except for L'\0'). +// The L'\0' char is printed as "L'\\0'". The decimal code is printed +// as signed integer when wchar_t is implemented by the compiler +// as a signed type and is printed as an unsigned integer when wchar_t +// is implemented as an unsigned type. +GTEST_API_ void PrintTo(wchar_t wc, ::std::ostream* os); + +// Overloads for C strings. +GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); +inline void PrintTo(char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// signed/unsigned char is often used for representing binary data, so +// we print pointers to it as void* to be safe. +inline void PrintTo(const signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(const unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// MSVC can be configured to define wchar_t as a typedef of unsigned +// short. It defines _NATIVE_WCHAR_T_DEFINED when wchar_t is a native +// type. When wchar_t is a typedef, defining an overload for const +// wchar_t* would cause unsigned short* be printed as a wide string, +// possibly causing invalid memory accesses. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Overloads for wide C strings +GTEST_API_ void PrintTo(const wchar_t* s, ::std::ostream* os); +inline void PrintTo(wchar_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +#endif + +// Overload for C arrays. Multi-dimensional arrays are printed +// properly. + +// Prints the given number of elements in an array, without printing +// the curly braces. +template +void PrintRawArrayTo(const T a[], size_t count, ::std::ostream* os) { + UniversalPrint(a[0], os); + for (size_t i = 1; i != count; i++) { + *os << ", "; + UniversalPrint(a[i], os); + } +} + +// Overloads for ::string and ::std::string. +#if GTEST_HAS_GLOBAL_STRING +GTEST_API_ void PrintStringTo(const ::string&s, ::std::ostream* os); +inline void PrintTo(const ::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_STRING + +GTEST_API_ void PrintStringTo(const ::std::string&s, ::std::ostream* os); +inline void PrintTo(const ::std::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} + +// Overloads for ::wstring and ::std::wstring. +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_API_ void PrintWideStringTo(const ::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ void PrintWideStringTo(const ::std::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ +// Helper function for printing a tuple. T must be instantiated with +// a tuple type. +template +void PrintTupleTo(const T& t, ::std::ostream* os); +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE +// Overload for ::std::tr1::tuple. Needed for printing function arguments, +// which are packed as tuples. + +// Overloaded PrintTo() for tuples of various arities. We support +// tuples of up-to 10 fields. The following implementation works +// regardless of whether tr1::tuple is implemented using the +// non-standard variadic template feature or not. + +inline void PrintTo(const ::std::tr1::tuple<>& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo( + const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_TR1_TUPLE + +#if GTEST_HAS_STD_TUPLE_ +template +void PrintTo(const ::std::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_STD_TUPLE_ + +// Overload for std::pair. +template +void PrintTo(const ::std::pair& value, ::std::ostream* os) { + *os << '('; + // We cannot use UniversalPrint(value.first, os) here, as T1 may be + // a reference type. The same for printing value.second. + UniversalPrinter::Print(value.first, os); + *os << ", "; + UniversalPrinter::Print(value.second, os); + *os << ')'; +} + +// Implements printing a non-reference type T by letting the compiler +// pick the right overload of PrintTo() for T. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) + + // Note: we deliberately don't call this PrintTo(), as that name + // conflicts with ::testing::internal::PrintTo in the body of the + // function. + static void Print(const T& value, ::std::ostream* os) { + // By default, ::testing::internal::PrintTo() is used for printing + // the value. + // + // Thanks to Koenig look-up, if T is a class and has its own + // PrintTo() function defined in its namespace, that function will + // be visible here. Since it is more specific than the generic ones + // in ::testing::internal, it will be picked by the compiler in the + // following statement - exactly what we want. + PrintTo(value, os); + } + + GTEST_DISABLE_MSC_WARNINGS_POP_() +}; + +// UniversalPrintArray(begin, len, os) prints an array of 'len' +// elements, starting at address 'begin'. +template +void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { + if (len == 0) { + *os << "{}"; + } else { + *os << "{ "; + const size_t kThreshold = 18; + const size_t kChunkSize = 8; + // If the array has more than kThreshold elements, we'll have to + // omit some details by printing only the first and the last + // kChunkSize elements. + // TODO(wan@google.com): let the user control the threshold using a flag. + if (len <= kThreshold) { + PrintRawArrayTo(begin, len, os); + } else { + PrintRawArrayTo(begin, kChunkSize, os); + *os << ", ..., "; + PrintRawArrayTo(begin + len - kChunkSize, kChunkSize, os); + } + *os << " }"; + } +} +// This overload prints a (const) char array compactly. +GTEST_API_ void UniversalPrintArray( + const char* begin, size_t len, ::std::ostream* os); + +// This overload prints a (const) wchar_t array compactly. +GTEST_API_ void UniversalPrintArray( + const wchar_t* begin, size_t len, ::std::ostream* os); + +// Implements printing an array type T[N]. +template +class UniversalPrinter { + public: + // Prints the given array, omitting some elements when there are too + // many. + static void Print(const T (&a)[N], ::std::ostream* os) { + UniversalPrintArray(a, N, os); + } +}; + +// Implements printing a reference type T&. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) + + static void Print(const T& value, ::std::ostream* os) { + // Prints the address of the value. We use reinterpret_cast here + // as static_cast doesn't compile when T is a function type. + *os << "@" << reinterpret_cast(&value) << " "; + + // Then prints the value itself. + UniversalPrint(value, os); + } + + GTEST_DISABLE_MSC_WARNINGS_POP_() +}; + +// Prints a value tersely: for a reference type, the referenced value +// (but not the address) is printed; for a (const) char pointer, the +// NUL-terminated string (but not the pointer) is printed. + +template +class UniversalTersePrinter { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); + } +}; +template +class UniversalTersePrinter { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); + } +}; +template +class UniversalTersePrinter { + public: + static void Print(const T (&value)[N], ::std::ostream* os) { + UniversalPrinter::Print(value, os); + } +}; +template <> +class UniversalTersePrinter { + public: + static void Print(const char* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(string(str), os); + } + } +}; +template <> +class UniversalTersePrinter { + public: + static void Print(char* str, ::std::ostream* os) { + UniversalTersePrinter::Print(str, os); + } +}; + +#if GTEST_HAS_STD_WSTRING +template <> +class UniversalTersePrinter { + public: + static void Print(const wchar_t* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(::std::wstring(str), os); + } + } +}; +#endif + +template <> +class UniversalTersePrinter { + public: + static void Print(wchar_t* str, ::std::ostream* os) { + UniversalTersePrinter::Print(str, os); + } +}; + +template +void UniversalTersePrint(const T& value, ::std::ostream* os) { + UniversalTersePrinter::Print(value, os); +} + +// Prints a value using the type inferred by the compiler. The +// difference between this and UniversalTersePrint() is that for a +// (const) char pointer, this prints both the pointer and the +// NUL-terminated string. +template +void UniversalPrint(const T& value, ::std::ostream* os) { + // A workarond for the bug in VC++ 7.1 that prevents us from instantiating + // UniversalPrinter with T directly. + typedef T T1; + UniversalPrinter::Print(value, os); +} + +typedef ::std::vector Strings; + +// TuplePolicy must provide: +// - tuple_size +// size of tuple TupleT. +// - get(const TupleT& t) +// static function extracting element I of tuple TupleT. +// - tuple_element::type +// type of element I of tuple TupleT. +template +struct TuplePolicy; + +#if GTEST_HAS_TR1_TUPLE +template +struct TuplePolicy { + typedef TupleT Tuple; + static const size_t tuple_size = ::std::tr1::tuple_size::value; + + template + struct tuple_element : ::std::tr1::tuple_element {}; + + template + static typename AddReference< + const typename ::std::tr1::tuple_element::type>::type get( + const Tuple& tuple) { + return ::std::tr1::get(tuple); + } +}; +template +const size_t TuplePolicy::tuple_size; +#endif // GTEST_HAS_TR1_TUPLE + +#if GTEST_HAS_STD_TUPLE_ +template +struct TuplePolicy< ::std::tuple > { + typedef ::std::tuple Tuple; + static const size_t tuple_size = ::std::tuple_size::value; + + template + struct tuple_element : ::std::tuple_element {}; + + template + static const typename ::std::tuple_element::type& get( + const Tuple& tuple) { + return ::std::get(tuple); + } +}; +template +const size_t TuplePolicy< ::std::tuple >::tuple_size; +#endif // GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ +// This helper template allows PrintTo() for tuples and +// UniversalTersePrintTupleFieldsToStrings() to be defined by +// induction on the number of tuple fields. The idea is that +// TuplePrefixPrinter::PrintPrefixTo(t, os) prints the first N +// fields in tuple t, and can be defined in terms of +// TuplePrefixPrinter. +// +// The inductive case. +template +struct TuplePrefixPrinter { + // Prints the first N fields of a tuple. + template + static void PrintPrefixTo(const Tuple& t, ::std::ostream* os) { + TuplePrefixPrinter::PrintPrefixTo(t, os); + GTEST_INTENTIONAL_CONST_COND_PUSH_() + if (N > 1) { + GTEST_INTENTIONAL_CONST_COND_POP_() + *os << ", "; + } + UniversalPrinter< + typename TuplePolicy::template tuple_element::type> + ::Print(TuplePolicy::template get(t), os); + } + + // Tersely prints the first N fields of a tuple to a string vector, + // one element for each field. + template + static void TersePrintPrefixToStrings(const Tuple& t, Strings* strings) { + TuplePrefixPrinter::TersePrintPrefixToStrings(t, strings); + ::std::stringstream ss; + UniversalTersePrint(TuplePolicy::template get(t), &ss); + strings->push_back(ss.str()); + } +}; + +// Base case. +template <> +struct TuplePrefixPrinter<0> { + template + static void PrintPrefixTo(const Tuple&, ::std::ostream*) {} + + template + static void TersePrintPrefixToStrings(const Tuple&, Strings*) {} +}; + +// Helper function for printing a tuple. +// Tuple must be either std::tr1::tuple or std::tuple type. +template +void PrintTupleTo(const Tuple& t, ::std::ostream* os) { + *os << "("; + TuplePrefixPrinter::tuple_size>::PrintPrefixTo(t, os); + *os << ")"; +} + +// Prints the fields of a tuple tersely to a string vector, one +// element for each field. See the comment before +// UniversalTersePrint() for how we define "tersely". +template +Strings UniversalTersePrintTupleFieldsToStrings(const Tuple& value) { + Strings result; + TuplePrefixPrinter::tuple_size>:: + TersePrintPrefixToStrings(value, &result); + return result; +} +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ + +} // namespace internal + +template +::std::string PrintToString(const T& value) { + ::std::stringstream ss; + internal::UniversalTersePrinter::Print(value, &ss); + return ss.str(); +} + +} // namespace testing + +// Include any custom printer added by the local installation. +// We must include this header at the end to make sure it can use the +// declarations from this file. +#include "gtest/internal/custom/gtest-printers.h" + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest-spi.h b/mypyc/external/googletest/include/gtest/gtest-spi.h new file mode 100644 index 000000000000..f63fa9a1b2a8 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-spi.h @@ -0,0 +1,232 @@ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// Utilities for testing Google Test itself and code that uses Google Test +// (e.g. frameworks built on top of Google Test). + +#ifndef GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#define GTEST_INCLUDE_GTEST_GTEST_SPI_H_ + +#include "gtest/gtest.h" + +namespace testing { + +// This helper class can be used to mock out Google Test failure reporting +// so that we can test Google Test or code that builds on Google Test. +// +// An object of this class appends a TestPartResult object to the +// TestPartResultArray object given in the constructor whenever a Google Test +// failure is reported. It can either intercept only failures that are +// generated in the same thread that created this object or it can intercept +// all generated failures. The scope of this mock object can be controlled with +// the second argument to the two arguments constructor. +class GTEST_API_ ScopedFakeTestPartResultReporter + : public TestPartResultReporterInterface { + public: + // The two possible mocking modes of this object. + enum InterceptMode { + INTERCEPT_ONLY_CURRENT_THREAD, // Intercepts only thread local failures. + INTERCEPT_ALL_THREADS // Intercepts all failures. + }; + + // The c'tor sets this object as the test part result reporter used + // by Google Test. The 'result' parameter specifies where to report the + // results. This reporter will only catch failures generated in the current + // thread. DEPRECATED + explicit ScopedFakeTestPartResultReporter(TestPartResultArray* result); + + // Same as above, but you can choose the interception scope of this object. + ScopedFakeTestPartResultReporter(InterceptMode intercept_mode, + TestPartResultArray* result); + + // The d'tor restores the previous test part result reporter. + virtual ~ScopedFakeTestPartResultReporter(); + + // Appends the TestPartResult object to the TestPartResultArray + // received in the constructor. + // + // This method is from the TestPartResultReporterInterface + // interface. + virtual void ReportTestPartResult(const TestPartResult& result); + private: + void Init(); + + const InterceptMode intercept_mode_; + TestPartResultReporterInterface* old_reporter_; + TestPartResultArray* const result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedFakeTestPartResultReporter); +}; + +namespace internal { + +// A helper class for implementing EXPECT_FATAL_FAILURE() and +// EXPECT_NONFATAL_FAILURE(). Its destructor verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +class GTEST_API_ SingleFailureChecker { + public: + // The constructor remembers the arguments. + SingleFailureChecker(const TestPartResultArray* results, + TestPartResult::Type type, + const string& substr); + ~SingleFailureChecker(); + private: + const TestPartResultArray* const results_; + const TestPartResult::Type type_; + const string substr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SingleFailureChecker); +}; + +} // namespace internal + +} // namespace testing + +// A set of macros for testing Google Test assertions or code that's expected +// to generate Google Test fatal failures. It verifies that the given +// statement will cause exactly one fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_FATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_FATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - 'statement' cannot reference local non-static variables or +// non-static members of the current object. +// - 'statement' cannot return a value. +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. The AcceptsMacroThatExpandsToUnprotectedComma test in +// gtest_unittest.cc will fail to compile if we do that. +#define EXPECT_FATAL_FAILURE(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_FATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ALL_THREADS, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +// A macro for testing Google Test assertions or code that's expected to +// generate Google Test non-fatal failures. It asserts that the given +// statement will cause exactly one non-fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_NONFATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// 'statement' is allowed to reference local variables and members of +// the current object. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. If we do that, the code won't compile when the user gives +// EXPECT_NONFATAL_FAILURE() a statement that contains a macro that +// expands to code containing an unprotected comma. The +// AcceptsMacroThatExpandsToUnprotectedComma test in gtest_unittest.cc +// catches that. +// +// For the same reason, we have to write +// if (::testing::internal::AlwaysTrue()) { statement; } +// instead of +// GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) +// to avoid an MSVC warning on unreachable code. +#define EXPECT_NONFATAL_FAILURE(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS, \ + >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#endif // GTEST_INCLUDE_GTEST_GTEST_SPI_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest-test-part.h b/mypyc/external/googletest/include/gtest/gtest-test-part.h new file mode 100644 index 000000000000..77eb844839d8 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-test-part.h @@ -0,0 +1,179 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ + +#include +#include +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-string.h" + +namespace testing { + +// A copyable object representing the result of a test part (i.e. an +// assertion or an explicit FAIL(), ADD_FAILURE(), or SUCCESS()). +// +// Don't inherit from TestPartResult as its destructor is not virtual. +class GTEST_API_ TestPartResult { + public: + // The possible outcomes of a test part (i.e. an assertion or an + // explicit SUCCEED(), FAIL(), or ADD_FAILURE()). + enum Type { + kSuccess, // Succeeded. + kNonFatalFailure, // Failed but the test can continue. + kFatalFailure // Failed and the test should be terminated. + }; + + // C'tor. TestPartResult does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestPartResult object. + TestPartResult(Type a_type, + const char* a_file_name, + int a_line_number, + const char* a_message) + : type_(a_type), + file_name_(a_file_name == NULL ? "" : a_file_name), + line_number_(a_line_number), + summary_(ExtractSummary(a_message)), + message_(a_message) { + } + + // Gets the outcome of the test part. + Type type() const { return type_; } + + // Gets the name of the source file where the test part took place, or + // NULL if it's unknown. + const char* file_name() const { + return file_name_.empty() ? NULL : file_name_.c_str(); + } + + // Gets the line in the source file where the test part took place, + // or -1 if it's unknown. + int line_number() const { return line_number_; } + + // Gets the summary of the failure message. + const char* summary() const { return summary_.c_str(); } + + // Gets the message associated with the test part. + const char* message() const { return message_.c_str(); } + + // Returns true iff the test part passed. + bool passed() const { return type_ == kSuccess; } + + // Returns true iff the test part failed. + bool failed() const { return type_ != kSuccess; } + + // Returns true iff the test part non-fatally failed. + bool nonfatally_failed() const { return type_ == kNonFatalFailure; } + + // Returns true iff the test part fatally failed. + bool fatally_failed() const { return type_ == kFatalFailure; } + + private: + Type type_; + + // Gets the summary of the failure message by omitting the stack + // trace in it. + static std::string ExtractSummary(const char* message); + + // The name of the source file where the test part took place, or + // "" if the source file is unknown. + std::string file_name_; + // The line in the source file where the test part took place, or -1 + // if the line number is unknown. + int line_number_; + std::string summary_; // The test failure summary. + std::string message_; // The test failure message. +}; + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result); + +// An array of TestPartResult objects. +// +// Don't inherit from TestPartResultArray as its destructor is not +// virtual. +class GTEST_API_ TestPartResultArray { + public: + TestPartResultArray() {} + + // Appends the given TestPartResult to the array. + void Append(const TestPartResult& result); + + // Returns the TestPartResult at the given index (0-based). + const TestPartResult& GetTestPartResult(int index) const; + + // Returns the number of TestPartResult objects in the array. + int size() const; + + private: + std::vector array_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestPartResultArray); +}; + +// This interface knows how to report a test part result. +class TestPartResultReporterInterface { + public: + virtual ~TestPartResultReporterInterface() {} + + virtual void ReportTestPartResult(const TestPartResult& result) = 0; +}; + +namespace internal { + +// This helper class is used by {ASSERT|EXPECT}_NO_FATAL_FAILURE to check if a +// statement generates new fatal failures. To do so it registers itself as the +// current test part result reporter. Besides checking if fatal failures were +// reported, it only delegates the reporting to the former result reporter. +// The original result reporter is restored in the destructor. +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +class GTEST_API_ HasNewFatalFailureHelper + : public TestPartResultReporterInterface { + public: + HasNewFatalFailureHelper(); + virtual ~HasNewFatalFailureHelper(); + virtual void ReportTestPartResult(const TestPartResult& result); + bool has_new_fatal_failure() const { return has_new_fatal_failure_; } + private: + bool has_new_fatal_failure_; + TestPartResultReporterInterface* original_reporter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(HasNewFatalFailureHelper); +}; + +} // namespace internal + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest-typed-test.h b/mypyc/external/googletest/include/gtest/gtest-typed-test.h new file mode 100644 index 000000000000..5f69d5678eaf --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest-typed-test.h @@ -0,0 +1,263 @@ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ + +// This header implements typed tests and type-parameterized tests. + +// Typed (aka type-driven) tests repeat the same test for types in a +// list. You must know which types you want to test with when writing +// typed tests. Here's how you do it: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + public: + ... + typedef std::list List; + static T shared_; + T value_; +}; + +// Next, associate a list of types with the test case, which will be +// repeated for each type in the list. The typedef is necessary for +// the macro to parse correctly. +typedef testing::Types MyTypes; +TYPED_TEST_CASE(FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// TYPED_TEST_CASE(FooTest, int); + +// Then, use TYPED_TEST() instead of TEST_F() to define as many typed +// tests for this test case as you want. +TYPED_TEST(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + // Since we are inside a derived class template, C++ requires use to + // visit the members of FooTest via 'this'. + TypeParam n = this->value_; + + // To visit static members of the fixture, add the TestFixture:: + // prefix. + n += TestFixture::shared_; + + // To refer to typedefs in the fixture, add the "typename + // TestFixture::" prefix. + typename TestFixture::List values; + values.push_back(n); + ... +} + +TYPED_TEST(FooTest, HasPropertyA) { ... } + +#endif // 0 + +// Type-parameterized tests are abstract test patterns parameterized +// by a type. Compared with typed tests, type-parameterized tests +// allow you to define the test pattern without knowing what the type +// parameters are. The defined pattern can be instantiated with +// different types any number of times, in any number of translation +// units. +// +// If you are designing an interface or concept, you can define a +// suite of type-parameterized tests to verify properties that any +// valid implementation of the interface/concept should have. Then, +// each implementation can easily instantiate the test suite to verify +// that it conforms to the requirements, without having to write +// similar tests repeatedly. Here's an example: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + ... +}; + +// Next, declare that you will define a type-parameterized test case +// (the _P suffix is for "parameterized" or "pattern", whichever you +// prefer): +TYPED_TEST_CASE_P(FooTest); + +// Then, use TYPED_TEST_P() to define as many type-parameterized tests +// for this type-parameterized test case as you want. +TYPED_TEST_P(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + TypeParam n = 0; + ... +} + +TYPED_TEST_P(FooTest, HasPropertyA) { ... } + +// Now the tricky part: you need to register all test patterns before +// you can instantiate them. The first argument of the macro is the +// test case name; the rest are the names of the tests in this test +// case. +REGISTER_TYPED_TEST_CASE_P(FooTest, + DoesBlah, HasPropertyA); + +// Finally, you are free to instantiate the pattern with the types you +// want. If you put the above code in a header file, you can #include +// it in multiple C++ source files and instantiate it multiple times. +// +// To distinguish different instances of the pattern, the first +// argument to the INSTANTIATE_* macro is a prefix that will be added +// to the actual test case name. Remember to pick unique prefixes for +// different instances. +typedef testing::Types MyTypes; +INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, int); + +#endif // 0 + +#include "gtest/internal/gtest-port.h" +#include "gtest/internal/gtest-type-util.h" + +// Implements typed tests. + +#if GTEST_HAS_TYPED_TEST + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the typedef for the type parameters of the +// given test case. +# define GTEST_TYPE_PARAMS_(TestCaseName) gtest_type_params_##TestCaseName##_ + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define TYPED_TEST_CASE(CaseName, Types) \ + typedef ::testing::internal::TypeList< Types >::type \ + GTEST_TYPE_PARAMS_(CaseName) + +# define TYPED_TEST(CaseName, TestName) \ + template \ + class GTEST_TEST_CLASS_NAME_(CaseName, TestName) \ + : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + bool gtest_##CaseName##_##TestName##_registered_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTest< \ + CaseName, \ + ::testing::internal::TemplateSel< \ + GTEST_TEST_CLASS_NAME_(CaseName, TestName)>, \ + GTEST_TYPE_PARAMS_(CaseName)>::Register(\ + "", ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + #CaseName, #TestName, 0); \ + template \ + void GTEST_TEST_CLASS_NAME_(CaseName, TestName)::TestBody() + +#endif // GTEST_HAS_TYPED_TEST + +// Implements type-parameterized tests. + +#if GTEST_HAS_TYPED_TEST_P + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the namespace name that the type-parameterized tests for +// the given type-parameterized test case are defined in. The exact +// name of the namespace is subject to change without notice. +# define GTEST_CASE_NAMESPACE_(TestCaseName) \ + gtest_case_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the variable used to remember the names of +// the defined tests in the given test case. +# define GTEST_TYPED_TEST_CASE_P_STATE_(TestCaseName) \ + gtest_typed_test_case_p_state_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE DIRECTLY. +// +// Expands to the name of the variable used to remember the names of +// the registered tests in the given test case. +# define GTEST_REGISTERED_TEST_NAMES_(TestCaseName) \ + gtest_registered_test_names_##TestCaseName##_ + +// The variables defined in the type-parameterized test macros are +// static as typically these macros are used in a .h file that can be +// #included in multiple translation units linked together. +# define TYPED_TEST_CASE_P(CaseName) \ + static ::testing::internal::TypedTestCasePState \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName) + +# define TYPED_TEST_P(CaseName, TestName) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + template \ + class TestName : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + static bool gtest_##TestName##_defined_ GTEST_ATTRIBUTE_UNUSED_ = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).AddTestName(\ + __FILE__, __LINE__, #CaseName, #TestName); \ + } \ + template \ + void GTEST_CASE_NAMESPACE_(CaseName)::TestName::TestBody() + +# define REGISTER_TYPED_TEST_CASE_P(CaseName, ...) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + typedef ::testing::internal::Templates<__VA_ARGS__>::type gtest_AllTests_; \ + } \ + static const char* const GTEST_REGISTERED_TEST_NAMES_(CaseName) = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).VerifyRegisteredTestNames(\ + __FILE__, __LINE__, #__VA_ARGS__) + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define INSTANTIATE_TYPED_TEST_CASE_P(Prefix, CaseName, Types) \ + bool gtest_##Prefix##_##CaseName GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTestCase::type>::Register(\ + #Prefix, \ + ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + >EST_TYPED_TEST_CASE_P_STATE_(CaseName), \ + #CaseName, GTEST_REGISTERED_TEST_NAMES_(CaseName)) + +#endif // GTEST_HAS_TYPED_TEST_P + +#endif // GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest.h b/mypyc/external/googletest/include/gtest/gtest.h new file mode 100644 index 000000000000..f846c5bd6696 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest.h @@ -0,0 +1,2236 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the public API for Google Test. It should be +// included by any test program that uses Google Test. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! +// +// Acknowledgment: Google Test borrowed the idea of automatic test +// registration from Barthelemy Dagenais' (barthelemy@prologique.com) +// easyUnit framework. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_H_ + +#include +#include +#include + +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-string.h" +#include "gtest/gtest-death-test.h" +#include "gtest/gtest-message.h" +#include "gtest/gtest-param-test.h" +#include "gtest/gtest-printers.h" +#include "gtest/gtest_prod.h" +#include "gtest/gtest-test-part.h" +#include "gtest/gtest-typed-test.h" + +// Depending on the platform, different string classes are available. +// On Linux, in addition to ::std::string, Google also makes use of +// class ::string, which has the same interface as ::std::string, but +// has a different implementation. +// +// You can define GTEST_HAS_GLOBAL_STRING to 1 to indicate that +// ::string is available AND is a distinct type to ::std::string, or +// define it to 0 to indicate otherwise. +// +// If ::std::string and ::string are the same class on your platform +// due to aliasing, you should define GTEST_HAS_GLOBAL_STRING to 0. +// +// If you do not define GTEST_HAS_GLOBAL_STRING, it is defined +// heuristically. + +namespace testing { + +// Declares the flags. + +// This flag temporary enables the disabled tests. +GTEST_DECLARE_bool_(also_run_disabled_tests); + +// This flag brings the debugger on an assertion failure. +GTEST_DECLARE_bool_(break_on_failure); + +// This flag controls whether Google Test catches all test-thrown exceptions +// and logs them as failures. +GTEST_DECLARE_bool_(catch_exceptions); + +// This flag enables using colors in terminal output. Available values are +// "yes" to enable colors, "no" (disable colors), or "auto" (the default) +// to let Google Test decide. +GTEST_DECLARE_string_(color); + +// This flag sets up the filter to select by name using a glob pattern +// the tests to run. If the filter is not given all tests are executed. +GTEST_DECLARE_string_(filter); + +// This flag causes the Google Test to list tests. None of the tests listed +// are actually run if the flag is provided. +GTEST_DECLARE_bool_(list_tests); + +// This flag controls whether Google Test emits a detailed XML report to a file +// in addition to its normal textual output. +GTEST_DECLARE_string_(output); + +// This flags control whether Google Test prints the elapsed time for each +// test. +GTEST_DECLARE_bool_(print_time); + +// This flag specifies the random number seed. +GTEST_DECLARE_int32_(random_seed); + +// This flag sets how many times the tests are repeated. The default value +// is 1. If the value is -1 the tests are repeating forever. +GTEST_DECLARE_int32_(repeat); + +// This flag controls whether Google Test includes Google Test internal +// stack frames in failure stack traces. +GTEST_DECLARE_bool_(show_internal_stack_frames); + +// When this flag is specified, tests' order is randomized on every iteration. +GTEST_DECLARE_bool_(shuffle); + +// This flag specifies the maximum number of stack frames to be +// printed in a failure message. +GTEST_DECLARE_int32_(stack_trace_depth); + +// When this flag is specified, a failed assertion will throw an +// exception if exceptions are enabled, or exit the program with a +// non-zero code otherwise. +GTEST_DECLARE_bool_(throw_on_failure); + +// When this flag is set with a "host:port" string, on supported +// platforms test results are streamed to the specified port on +// the specified host machine. +GTEST_DECLARE_string_(stream_result_to); + +// The upper limit for valid stack trace depths. +const int kMaxStackTraceDepth = 100; + +namespace internal { + +class AssertHelper; +class DefaultGlobalTestPartResultReporter; +class ExecDeathTest; +class NoExecDeathTest; +class FinalSuccessChecker; +class GTestFlagSaver; +class StreamingListenerTest; +class TestResultAccessor; +class TestEventListenersAccessor; +class TestEventRepeater; +class UnitTestRecordPropertyTestHelper; +class WindowsDeathTest; +class UnitTestImpl* GetUnitTestImpl(); +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const std::string& message); + +} // namespace internal + +// The friend relationship of some of these classes is cyclic. +// If we don't forward declare them the compiler might confuse the classes +// in friendship clauses with same named classes on the scope. +class Test; +class TestCase; +class TestInfo; +class UnitTest; + +// A class for indicating whether an assertion was successful. When +// the assertion wasn't successful, the AssertionResult object +// remembers a non-empty message that describes how it failed. +// +// To create an instance of this class, use one of the factory functions +// (AssertionSuccess() and AssertionFailure()). +// +// This class is useful for two purposes: +// 1. Defining predicate functions to be used with Boolean test assertions +// EXPECT_TRUE/EXPECT_FALSE and their ASSERT_ counterparts +// 2. Defining predicate-format functions to be +// used with predicate assertions (ASSERT_PRED_FORMAT*, etc). +// +// For example, if you define IsEven predicate: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then the failed expectation EXPECT_TRUE(IsEven(Fib(5))) +// will print the message +// +// Value of: IsEven(Fib(5)) +// Actual: false (5 is odd) +// Expected: true +// +// instead of a more opaque +// +// Value of: IsEven(Fib(5)) +// Actual: false +// Expected: true +// +// in case IsEven is a simple Boolean predicate. +// +// If you expect your predicate to be reused and want to support informative +// messages in EXPECT_FALSE and ASSERT_FALSE (negative assertions show up +// about half as often as positive ones in our tests), supply messages for +// both success and failure cases: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess() << n << " is even"; +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then a statement EXPECT_FALSE(IsEven(Fib(6))) will print +// +// Value of: IsEven(Fib(6)) +// Actual: true (8 is even) +// Expected: false +// +// NB: Predicates that support negative Boolean assertions have reduced +// performance in positive ones so be careful not to use them in tests +// that have lots (tens of thousands) of positive Boolean assertions. +// +// To use this class with EXPECT_PRED_FORMAT assertions such as: +// +// // Verifies that Foo() returns an even number. +// EXPECT_PRED_FORMAT1(IsEven, Foo()); +// +// you need to define: +// +// testing::AssertionResult IsEven(const char* expr, int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() +// << "Expected: " << expr << " is even\n Actual: it's " << n; +// } +// +// If Foo() returns 5, you will see the following message: +// +// Expected: Foo() is even +// Actual: it's 5 +// +class GTEST_API_ AssertionResult { + public: + // Copy constructor. + // Used in EXPECT_TRUE/FALSE(assertion_result). + AssertionResult(const AssertionResult& other); + + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 /* forcing value to bool */) + + // Used in the EXPECT_TRUE/FALSE(bool_expression). + // + // T must be contextually convertible to bool. + // + // The second parameter prevents this overload from being considered if + // the argument is implicitly convertible to AssertionResult. In that case + // we want AssertionResult's copy constructor to be used. + template + explicit AssertionResult( + const T& success, + typename internal::EnableIf< + !internal::ImplicitlyConvertible::value>::type* + /*enabler*/ = NULL) + : success_(success) {} + + GTEST_DISABLE_MSC_WARNINGS_POP_() + + // Assignment operator. + AssertionResult& operator=(AssertionResult other) { + swap(other); + return *this; + } + + // Returns true iff the assertion succeeded. + operator bool() const { return success_; } // NOLINT + + // Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. + AssertionResult operator!() const; + + // Returns the text streamed into this AssertionResult. Test assertions + // use it when they fail (i.e., the predicate's outcome doesn't match the + // assertion's expectation). When nothing has been streamed into the + // object, returns an empty string. + const char* message() const { + return message_.get() != NULL ? message_->c_str() : ""; + } + // TODO(vladl@google.com): Remove this after making sure no clients use it. + // Deprecated; please use message() instead. + const char* failure_message() const { return message(); } + + // Streams a custom failure message into this object. + template AssertionResult& operator<<(const T& value) { + AppendMessage(Message() << value); + return *this; + } + + // Allows streaming basic output manipulators such as endl or flush into + // this object. + AssertionResult& operator<<( + ::std::ostream& (*basic_manipulator)(::std::ostream& stream)) { + AppendMessage(Message() << basic_manipulator); + return *this; + } + + private: + // Appends the contents of message to message_. + void AppendMessage(const Message& a_message) { + if (message_.get() == NULL) + message_.reset(new ::std::string); + message_->append(a_message.GetString().c_str()); + } + + // Swap the contents of this AssertionResult with other. + void swap(AssertionResult& other); + + // Stores result of the assertion predicate. + bool success_; + // Stores the message describing the condition in case the expectation + // construct is not satisfied with the predicate's outcome. + // Referenced via a pointer to avoid taking too much stack frame space + // with test assertions. + internal::scoped_ptr< ::std::string> message_; +}; + +// Makes a successful assertion result. +GTEST_API_ AssertionResult AssertionSuccess(); + +// Makes a failed assertion result. +GTEST_API_ AssertionResult AssertionFailure(); + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << msg. +GTEST_API_ AssertionResult AssertionFailure(const Message& msg); + +// The abstract class that all tests inherit from. +// +// In Google Test, a unit test program contains one or many TestCases, and +// each TestCase contains one or many Tests. +// +// When you define a test using the TEST macro, you don't need to +// explicitly derive from Test - the TEST macro automatically does +// this for you. +// +// The only time you derive from Test is when defining a test fixture +// to be used a TEST_F. For example: +// +// class FooTest : public testing::Test { +// protected: +// void SetUp() override { ... } +// void TearDown() override { ... } +// ... +// }; +// +// TEST_F(FooTest, Bar) { ... } +// TEST_F(FooTest, Baz) { ... } +// +// Test is not copyable. +class GTEST_API_ Test { + public: + friend class TestInfo; + + // Defines types for pointers to functions that set up and tear down + // a test case. + typedef internal::SetUpTestCaseFunc SetUpTestCaseFunc; + typedef internal::TearDownTestCaseFunc TearDownTestCaseFunc; + + // The d'tor is virtual as we intend to inherit from Test. + virtual ~Test(); + + // Sets up the stuff shared by all tests in this test case. + // + // Google Test will call Foo::SetUpTestCase() before running the first + // test in test case Foo. Hence a sub-class can define its own + // SetUpTestCase() method to shadow the one defined in the super + // class. + static void SetUpTestCase() {} + + // Tears down the stuff shared by all tests in this test case. + // + // Google Test will call Foo::TearDownTestCase() after running the last + // test in test case Foo. Hence a sub-class can define its own + // TearDownTestCase() method to shadow the one defined in the super + // class. + static void TearDownTestCase() {} + + // Returns true iff the current test has a fatal failure. + static bool HasFatalFailure(); + + // Returns true iff the current test has a non-fatal failure. + static bool HasNonfatalFailure(); + + // Returns true iff the current test has a (either fatal or + // non-fatal) failure. + static bool HasFailure() { return HasFatalFailure() || HasNonfatalFailure(); } + + // Logs a property for the current test, test case, or for the entire + // invocation of the test program when used outside of the context of a + // test case. Only the last value for a given key is remembered. These + // are public static so they can be called from utility functions that are + // not members of the test fixture. Calls to RecordProperty made during + // lifespan of the test (from the moment its constructor starts to the + // moment its destructor finishes) will be output in XML as attributes of + // the element. Properties recorded from fixture's + // SetUpTestCase or TearDownTestCase are logged as attributes of the + // corresponding element. Calls to RecordProperty made in the + // global context (before or after invocation of RUN_ALL_TESTS and from + // SetUp/TearDown method of Environment objects registered with Google + // Test) will be output as attributes of the element. + static void RecordProperty(const std::string& key, const std::string& value); + static void RecordProperty(const std::string& key, int value); + + protected: + // Creates a Test object. + Test(); + + // Sets up the test fixture. + virtual void SetUp(); + + // Tears down the test fixture. + virtual void TearDown(); + + private: + // Returns true iff the current test has the same fixture class as + // the first test in the current test case. + static bool HasSameFixtureClass(); + + // Runs the test after the test fixture has been set up. + // + // A sub-class must implement this to define the test logic. + // + // DO NOT OVERRIDE THIS FUNCTION DIRECTLY IN A USER PROGRAM. + // Instead, use the TEST or TEST_F macro. + virtual void TestBody() = 0; + + // Sets up, executes, and tears down the test. + void Run(); + + // Deletes self. We deliberately pick an unusual name for this + // internal method to avoid clashing with names used in user TESTs. + void DeleteSelf_() { delete this; } + + const internal::scoped_ptr< GTEST_FLAG_SAVER_ > gtest_flag_saver_; + + // Often a user misspells SetUp() as Setup() and spends a long time + // wondering why it is never called by Google Test. The declaration of + // the following method is solely for catching such an error at + // compile time: + // + // - The return type is deliberately chosen to be not void, so it + // will be a conflict if void Setup() is declared in the user's + // test fixture. + // + // - This method is private, so it will be another compiler error + // if the method is called from the user's test fixture. + // + // DO NOT OVERRIDE THIS FUNCTION. + // + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } + + // We disallow copying Tests. + GTEST_DISALLOW_COPY_AND_ASSIGN_(Test); +}; + +typedef internal::TimeInMillis TimeInMillis; + +// A copyable object representing a user specified test property which can be +// output as a key/value string pair. +// +// Don't inherit from TestProperty as its destructor is not virtual. +class TestProperty { + public: + // C'tor. TestProperty does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestProperty object. + TestProperty(const std::string& a_key, const std::string& a_value) : + key_(a_key), value_(a_value) { + } + + // Gets the user supplied key. + const char* key() const { + return key_.c_str(); + } + + // Gets the user supplied value. + const char* value() const { + return value_.c_str(); + } + + // Sets a new value, overriding the one supplied in the constructor. + void SetValue(const std::string& new_value) { + value_ = new_value; + } + + private: + // The key supplied by the user. + std::string key_; + // The value supplied by the user. + std::string value_; +}; + +// The result of a single Test. This includes a list of +// TestPartResults, a list of TestProperties, a count of how many +// death tests there are in the Test, and how much time it took to run +// the Test. +// +// TestResult is not copyable. +class GTEST_API_ TestResult { + public: + // Creates an empty TestResult. + TestResult(); + + // D'tor. Do not inherit from TestResult. + ~TestResult(); + + // Gets the number of all test parts. This is the sum of the number + // of successful test parts and the number of failed test parts. + int total_part_count() const; + + // Returns the number of the test properties. + int test_property_count() const; + + // Returns true iff the test passed (i.e. no test part failed). + bool Passed() const { return !Failed(); } + + // Returns true iff the test failed. + bool Failed() const; + + // Returns true iff the test fatally failed. + bool HasFatalFailure() const; + + // Returns true iff the test has a non-fatal failure. + bool HasNonfatalFailure() const; + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test part result among all the results. i can range + // from 0 to test_property_count() - 1. If i is not in that range, aborts + // the program. + const TestPartResult& GetTestPartResult(int i) const; + + // Returns the i-th test property. i can range from 0 to + // test_property_count() - 1. If i is not in that range, aborts the + // program. + const TestProperty& GetTestProperty(int i) const; + + private: + friend class TestInfo; + friend class TestCase; + friend class UnitTest; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::ExecDeathTest; + friend class internal::TestResultAccessor; + friend class internal::UnitTestImpl; + friend class internal::WindowsDeathTest; + + // Gets the vector of TestPartResults. + const std::vector& test_part_results() const { + return test_part_results_; + } + + // Gets the vector of TestProperties. + const std::vector& test_properties() const { + return test_properties_; + } + + // Sets the elapsed time. + void set_elapsed_time(TimeInMillis elapsed) { elapsed_time_ = elapsed; } + + // Adds a test property to the list. The property is validated and may add + // a non-fatal failure if invalid (e.g., if it conflicts with reserved + // key names). If a property is already recorded for the same key, the + // value will be updated, rather than storing multiple values for the same + // key. xml_element specifies the element for which the property is being + // recorded and is used for validation. + void RecordProperty(const std::string& xml_element, + const TestProperty& test_property); + + // Adds a failure if the key is a reserved attribute of Google Test + // testcase tags. Returns true if the property is valid. + // TODO(russr): Validate attribute names are legal and human readable. + static bool ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property); + + // Adds a test part result to the list. + void AddTestPartResult(const TestPartResult& test_part_result); + + // Returns the death test count. + int death_test_count() const { return death_test_count_; } + + // Increments the death test count, returning the new count. + int increment_death_test_count() { return ++death_test_count_; } + + // Clears the test part results. + void ClearTestPartResults(); + + // Clears the object. + void Clear(); + + // Protects mutable state of the property vector and of owned + // properties, whose values may be updated. + internal::Mutex test_properites_mutex_; + + // The vector of TestPartResults + std::vector test_part_results_; + // The vector of TestProperties + std::vector test_properties_; + // Running count of death tests. + int death_test_count_; + // The elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + + // We disallow copying TestResult. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestResult); +}; // class TestResult + +// A TestInfo object stores the following information about a test: +// +// Test case name +// Test name +// Whether the test should be run +// A function pointer that creates the test object when invoked +// Test result +// +// The constructor of TestInfo registers itself with the UnitTest +// singleton such that the RUN_ALL_TESTS() macro knows which tests to +// run. +class GTEST_API_ TestInfo { + public: + // Destructs a TestInfo object. This function is not virtual, so + // don't inherit from TestInfo. + ~TestInfo(); + + // Returns the test case name. + const char* test_case_name() const { return test_case_name_.c_str(); } + + // Returns the test name. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a typed + // or a type-parameterized test. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns the text representation of the value parameter, or NULL if this + // is not a value-parameterized test. + const char* value_param() const { + if (value_param_.get() != NULL) + return value_param_->c_str(); + return NULL; + } + + // Returns the file name where this test is defined. + const char* file() const { return location_.file.c_str(); } + + // Returns the line where this test is defined. + int line() const { return location_.line; } + + // Returns true if this test should run, that is if the test is not + // disabled (or it is disabled but the also_run_disabled_tests flag has + // been specified) and its full name matches the user-specified filter. + // + // Google Test allows the user to filter the tests by their full names. + // The full name of a test Bar in test case Foo is defined as + // "Foo.Bar". Only the tests that match the filter will run. + // + // A filter is a colon-separated list of glob (not regex) patterns, + // optionally followed by a '-' and a colon-separated list of + // negative patterns (tests to exclude). A test is run if it + // matches one of the positive patterns and does not match any of + // the negative patterns. + // + // For example, *A*:Foo.* is a filter that matches any string that + // contains the character 'A' or starts with "Foo.". + bool should_run() const { return should_run_; } + + // Returns true iff this test will appear in the XML report. + bool is_reportable() const { + // For now, the XML report includes all tests matching the filter. + // In the future, we may trim tests that are excluded because of + // sharding. + return matches_filter_; + } + + // Returns the result of the test. + const TestResult* result() const { return &result_; } + + private: +#if GTEST_HAS_DEATH_TEST + friend class internal::DefaultDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + friend class Test; + friend class TestCase; + friend class internal::UnitTestImpl; + friend class internal::StreamingListenerTest; + friend TestInfo* internal::MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + internal::CodeLocation code_location, + internal::TypeId fixture_class_id, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + internal::TestFactoryBase* factory); + + // Constructs a TestInfo object. The newly constructed instance assumes + // ownership of the factory object. + TestInfo(const std::string& test_case_name, + const std::string& name, + const char* a_type_param, // NULL if not a type-parameterized test + const char* a_value_param, // NULL if not a value-parameterized test + internal::CodeLocation a_code_location, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory); + + // Increments the number of death tests encountered in this test so + // far. + int increment_death_test_count() { + return result_.increment_death_test_count(); + } + + // Creates the test object, runs it, records its result, and then + // deletes it. + void Run(); + + static void ClearTestResult(TestInfo* test_info) { + test_info->result_.Clear(); + } + + // These fields are immutable properties of the test. + const std::string test_case_name_; // Test case name + const std::string name_; // Test name + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // Text representation of the value parameter, or NULL if this is not a + // value-parameterized test. + const internal::scoped_ptr value_param_; + internal::CodeLocation location_; + const internal::TypeId fixture_class_id_; // ID of the test fixture class + bool should_run_; // True iff this test should run + bool is_disabled_; // True iff this test is disabled + bool matches_filter_; // True if this test matches the + // user-specified filter. + internal::TestFactoryBase* const factory_; // The factory that creates + // the test object + + // This field is mutable and needs to be reset before running the + // test for the second time. + TestResult result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestInfo); +}; + +// A test case, which consists of a vector of TestInfos. +// +// TestCase is not copyable. +class GTEST_API_ TestCase { + public: + // Creates a TestCase with the given name. + // + // TestCase does NOT have a default constructor. Always use this + // constructor to create a TestCase object. + // + // Arguments: + // + // name: name of the test case + // a_type_param: the name of the test's type parameter, or NULL if + // this is not a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase(const char* name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Destructor of TestCase. + virtual ~TestCase(); + + // Gets the name of the TestCase. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a + // type-parameterized test case. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns true if any test in this test case should run. + bool should_run() const { return should_run_; } + + // Gets the number of successful tests in this test case. + int successful_test_count() const; + + // Gets the number of failed tests in this test case. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests in this test case. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Get the number of tests in this test case that should run. + int test_to_run_count() const; + + // Gets the number of all tests in this test case. + int total_test_count() const; + + // Returns true iff the test case passed. + bool Passed() const { return !Failed(); } + + // Returns true iff the test case failed. + bool Failed() const { return failed_test_count() > 0; } + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + const TestInfo* GetTestInfo(int i) const; + + // Returns the TestResult that holds test properties recorded during + // execution of SetUpTestCase and TearDownTestCase. + const TestResult& ad_hoc_test_result() const { return ad_hoc_test_result_; } + + private: + friend class Test; + friend class internal::UnitTestImpl; + + // Gets the (mutable) vector of TestInfos in this TestCase. + std::vector& test_info_list() { return test_info_list_; } + + // Gets the (immutable) vector of TestInfos in this TestCase. + const std::vector& test_info_list() const { + return test_info_list_; + } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + TestInfo* GetMutableTestInfo(int i); + + // Sets the should_run member. + void set_should_run(bool should) { should_run_ = should; } + + // Adds a TestInfo to this test case. Will delete the TestInfo upon + // destruction of the TestCase object. + void AddTestInfo(TestInfo * test_info); + + // Clears the results of all tests in this test case. + void ClearResult(); + + // Clears the results of all tests in the given test case. + static void ClearTestCaseResult(TestCase* test_case) { + test_case->ClearResult(); + } + + // Runs every test in this TestCase. + void Run(); + + // Runs SetUpTestCase() for this TestCase. This wrapper is needed + // for catching exceptions thrown from SetUpTestCase(). + void RunSetUpTestCase() { (*set_up_tc_)(); } + + // Runs TearDownTestCase() for this TestCase. This wrapper is + // needed for catching exceptions thrown from TearDownTestCase(). + void RunTearDownTestCase() { (*tear_down_tc_)(); } + + // Returns true iff test passed. + static bool TestPassed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Passed(); + } + + // Returns true iff test failed. + static bool TestFailed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Failed(); + } + + // Returns true iff the test is disabled and will be reported in the XML + // report. + static bool TestReportableDisabled(const TestInfo* test_info) { + return test_info->is_reportable() && test_info->is_disabled_; + } + + // Returns true iff test is disabled. + static bool TestDisabled(const TestInfo* test_info) { + return test_info->is_disabled_; + } + + // Returns true iff this test will appear in the XML report. + static bool TestReportable(const TestInfo* test_info) { + return test_info->is_reportable(); + } + + // Returns true if the given test should run. + static bool ShouldRunTest(const TestInfo* test_info) { + return test_info->should_run(); + } + + // Shuffles the tests in this test case. + void ShuffleTests(internal::Random* random); + + // Restores the test order to before the first shuffle. + void UnshuffleTests(); + + // Name of the test case. + std::string name_; + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // The vector of TestInfos in their original order. It owns the + // elements in the vector. + std::vector test_info_list_; + // Provides a level of indirection for the test list to allow easy + // shuffling and restoring the test order. The i-th element in this + // vector is the index of the i-th test in the shuffled test list. + std::vector test_indices_; + // Pointer to the function that sets up the test case. + Test::SetUpTestCaseFunc set_up_tc_; + // Pointer to the function that tears down the test case. + Test::TearDownTestCaseFunc tear_down_tc_; + // True iff any test in this test case should run. + bool should_run_; + // Elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + // Holds test properties recorded during execution of SetUpTestCase and + // TearDownTestCase. + TestResult ad_hoc_test_result_; + + // We disallow copying TestCases. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestCase); +}; + +// An Environment object is capable of setting up and tearing down an +// environment. You should subclass this to define your own +// environment(s). +// +// An Environment object does the set-up and tear-down in virtual +// methods SetUp() and TearDown() instead of the constructor and the +// destructor, as: +// +// 1. You cannot safely throw from a destructor. This is a problem +// as in some cases Google Test is used where exceptions are enabled, and +// we may want to implement ASSERT_* using exceptions where they are +// available. +// 2. You cannot use ASSERT_* directly in a constructor or +// destructor. +class Environment { + public: + // The d'tor is virtual as we need to subclass Environment. + virtual ~Environment() {} + + // Override this to define how to set up the environment. + virtual void SetUp() {} + + // Override this to define how to tear down the environment. + virtual void TearDown() {} + private: + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } +}; + +// The interface for tracing execution of tests. The methods are organized in +// the order the corresponding events are fired. +class TestEventListener { + public: + virtual ~TestEventListener() {} + + // Fired before any test activity starts. + virtual void OnTestProgramStart(const UnitTest& unit_test) = 0; + + // Fired before each iteration of tests starts. There may be more than + // one iteration if GTEST_FLAG(repeat) is set. iteration is the iteration + // index, starting from 0. + virtual void OnTestIterationStart(const UnitTest& unit_test, + int iteration) = 0; + + // Fired before environment set-up for each iteration of tests starts. + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test) = 0; + + // Fired after environment set-up for each iteration of tests ends. + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test) = 0; + + // Fired before the test case starts. + virtual void OnTestCaseStart(const TestCase& test_case) = 0; + + // Fired before the test starts. + virtual void OnTestStart(const TestInfo& test_info) = 0; + + // Fired after a failed assertion or a SUCCEED() invocation. + virtual void OnTestPartResult(const TestPartResult& test_part_result) = 0; + + // Fired after the test ends. + virtual void OnTestEnd(const TestInfo& test_info) = 0; + + // Fired after the test case ends. + virtual void OnTestCaseEnd(const TestCase& test_case) = 0; + + // Fired before environment tear-down for each iteration of tests starts. + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test) = 0; + + // Fired after environment tear-down for each iteration of tests ends. + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test) = 0; + + // Fired after each iteration of tests finishes. + virtual void OnTestIterationEnd(const UnitTest& unit_test, + int iteration) = 0; + + // Fired after all test activities have ended. + virtual void OnTestProgramEnd(const UnitTest& unit_test) = 0; +}; + +// The convenience class for users who need to override just one or two +// methods and are not concerned that a possible change to a signature of +// the methods they override will not be caught during the build. For +// comments about each method please see the definition of TestEventListener +// above. +class EmptyTestEventListener : public TestEventListener { + public: + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& /*test_case*/) {} + virtual void OnTestStart(const TestInfo& /*test_info*/) {} + virtual void OnTestPartResult(const TestPartResult& /*test_part_result*/) {} + virtual void OnTestEnd(const TestInfo& /*test_info*/) {} + virtual void OnTestCaseEnd(const TestCase& /*test_case*/) {} + virtual void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} +}; + +// TestEventListeners lets users add listeners to track events in Google Test. +class GTEST_API_ TestEventListeners { + public: + TestEventListeners(); + ~TestEventListeners(); + + // Appends an event listener to the end of the list. Google Test assumes + // the ownership of the listener (i.e. it will delete the listener when + // the test program finishes). + void Append(TestEventListener* listener); + + // Removes the given event listener from the list and returns it. It then + // becomes the caller's responsibility to delete the listener. Returns + // NULL if the listener is not found in the list. + TestEventListener* Release(TestEventListener* listener); + + // Returns the standard listener responsible for the default console + // output. Can be removed from the listeners list to shut down default + // console output. Note that removing this object from the listener list + // with Release transfers its ownership to the caller and makes this + // function return NULL the next time. + TestEventListener* default_result_printer() const { + return default_result_printer_; + } + + // Returns the standard listener responsible for the default XML output + // controlled by the --gtest_output=xml flag. Can be removed from the + // listeners list by users who want to shut down the default XML output + // controlled by this flag and substitute it with custom one. Note that + // removing this object from the listener list with Release transfers its + // ownership to the caller and makes this function return NULL the next + // time. + TestEventListener* default_xml_generator() const { + return default_xml_generator_; + } + + private: + friend class TestCase; + friend class TestInfo; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::NoExecDeathTest; + friend class internal::TestEventListenersAccessor; + friend class internal::UnitTestImpl; + + // Returns repeater that broadcasts the TestEventListener events to all + // subscribers. + TestEventListener* repeater(); + + // Sets the default_result_printer attribute to the provided listener. + // The listener is also added to the listener list and previous + // default_result_printer is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultResultPrinter(TestEventListener* listener); + + // Sets the default_xml_generator attribute to the provided listener. The + // listener is also added to the listener list and previous + // default_xml_generator is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultXmlGenerator(TestEventListener* listener); + + // Controls whether events will be forwarded by the repeater to the + // listeners in the list. + bool EventForwardingEnabled() const; + void SuppressEventForwarding(); + + // The actual list of listeners. + internal::TestEventRepeater* repeater_; + // Listener responsible for the standard result output. + TestEventListener* default_result_printer_; + // Listener responsible for the creation of the XML output file. + TestEventListener* default_xml_generator_; + + // We disallow copying TestEventListeners. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventListeners); +}; + +// A UnitTest consists of a vector of TestCases. +// +// This is a singleton class. The only instance of UnitTest is +// created when UnitTest::GetInstance() is first called. This +// instance is never deleted. +// +// UnitTest is not copyable. +// +// This class is thread-safe as long as the methods are called +// according to their specification. +class GTEST_API_ UnitTest { + public: + // Gets the singleton UnitTest object. The first time this method + // is called, a UnitTest object is constructed and returned. + // Consecutive calls will return the same object. + static UnitTest* GetInstance(); + + // Runs all tests in this UnitTest object and prints the result. + // Returns 0 if successful, or 1 otherwise. + // + // This method can only be called from the main thread. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + int Run() GTEST_MUST_USE_RESULT_; + + // Returns the working directory when the first TEST() or TEST_F() + // was executed. The UnitTest object owns the string. + const char* original_working_dir() const; + + // Returns the TestCase object for the test that's currently running, + // or NULL if no test is running. + const TestCase* current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_); + + // Returns the TestInfo object for the test that's currently running, + // or NULL if no test is running. + const TestInfo* current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_); + + // Returns the random seed used at the start of the current test run. + int random_seed() const; + +#if GTEST_HAS_PARAM_TEST + // Returns the ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_); +#endif // GTEST_HAS_PARAM_TEST + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const; + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const; + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const; + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const; + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const; + + // Returns the TestResult containing information on test failures and + // properties logged outside of individual test cases. + const TestResult& ad_hoc_test_result() const; + + // Returns the list of event listeners that can be used to track events + // inside Google Test. + TestEventListeners& listeners(); + + private: + // Registers and returns a global test environment. When a test + // program is run, all global test environments will be set-up in + // the order they were registered. After all tests in the program + // have finished, all global test environments will be torn-down in + // the *reverse* order they were registered. + // + // The UnitTest object takes ownership of the given environment. + // + // This method can only be called from the main thread. + Environment* AddEnvironment(Environment* env); + + // Adds a TestPartResult to the current TestResult object. All + // Google Test assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) + // eventually call this to report their results. The user code + // should use the assertion macros instead of calling this directly. + void AddTestPartResult(TestPartResult::Type result_type, + const char* file_name, + int line_number, + const std::string& message, + const std::string& os_stack_trace) + GTEST_LOCK_EXCLUDED_(mutex_); + + // Adds a TestProperty to the current TestResult object when invoked from + // inside a test, to current TestCase's ad_hoc_test_result_ when invoked + // from SetUpTestCase or TearDownTestCase, or to the global property set + // when invoked elsewhere. If the result already contains a property with + // the same key, the value will be updated. + void RecordProperty(const std::string& key, const std::string& value); + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i); + + // Accessors for the implementation object. + internal::UnitTestImpl* impl() { return impl_; } + const internal::UnitTestImpl* impl() const { return impl_; } + + // These classes and funcions are friends as they need to access private + // members of UnitTest. + friend class Test; + friend class internal::AssertHelper; + friend class internal::ScopedTrace; + friend class internal::StreamingListenerTest; + friend class internal::UnitTestRecordPropertyTestHelper; + friend Environment* AddGlobalTestEnvironment(Environment* env); + friend internal::UnitTestImpl* internal::GetUnitTestImpl(); + friend void internal::ReportFailureInUnknownLocation( + TestPartResult::Type result_type, + const std::string& message); + + // Creates an empty UnitTest. + UnitTest(); + + // D'tor + virtual ~UnitTest(); + + // Pushes a trace defined by SCOPED_TRACE() on to the per-thread + // Google Test trace stack. + void PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_); + + // Pops a trace from the per-thread Google Test trace stack. + void PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_); + + // Protects mutable state in *impl_. This is mutable as some const + // methods need to lock it too. + mutable internal::Mutex mutex_; + + // Opaque implementation object. This field is never changed once + // the object is constructed. We don't mark it as const here, as + // doing so will cause a warning in the constructor of UnitTest. + // Mutable state in *impl_ is protected by mutex_. + internal::UnitTestImpl* impl_; + + // We disallow copying UnitTest. + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTest); +}; + +// A convenient wrapper for adding an environment for the test +// program. +// +// You should call this before RUN_ALL_TESTS() is called, probably in +// main(). If you use gtest_main, you need to call this before main() +// starts for it to take effect. For example, you can define a global +// variable like this: +// +// testing::Environment* const foo_env = +// testing::AddGlobalTestEnvironment(new FooEnvironment); +// +// However, we strongly recommend you to write your own main() and +// call AddGlobalTestEnvironment() there, as relying on initialization +// of global variables makes the code harder to read and may cause +// problems when you register multiple environments from different +// translation units and the environments have dependencies among them +// (remember that the compiler doesn't guarantee the order in which +// global variables from different translation units are initialized). +inline Environment* AddGlobalTestEnvironment(Environment* env) { + return UnitTest::GetInstance()->AddEnvironment(env); +} + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +GTEST_API_ void InitGoogleTest(int* argc, char** argv); + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +GTEST_API_ void InitGoogleTest(int* argc, wchar_t** argv); + +namespace internal { + +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperEQ. This helps reduce the overhead of some sanitizers +// when calling EXPECT_* in a tight loop. +template +AssertionResult CmpHelperEQFailure(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, const T2& rhs) { + return EqFailure(lhs_expression, + rhs_expression, + FormatForComparisonFailureMessage(lhs, rhs), + FormatForComparisonFailureMessage(rhs, lhs), + false); +} + +// The helper function for {ASSERT|EXPECT}_EQ. +template +AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs) { +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4389 /* signed/unsigned mismatch */) + if (lhs == rhs) { + return AssertionSuccess(); + } +GTEST_DISABLE_MSC_WARNINGS_POP_() + + return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs); +} + +// With this overloaded version, we allow anonymous enums to be used +// in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous enums +// can be implicitly cast to BiggestInt. +GTEST_API_ AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs); + +// The helper class for {ASSERT|EXPECT}_EQ. The template argument +// lhs_is_null_literal is true iff the first argument to ASSERT_EQ() +// is a null pointer literal. The following default implementation is +// for lhs_is_null_literal being false. +template +class EqHelper { + public: + // This templatized version is for the general case. + template + static AssertionResult Compare(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } + + // With this overloaded version, we allow anonymous enums to be used + // in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous + // enums can be implicitly cast to BiggestInt. + // + // Even though its body looks the same as the above version, we + // cannot merge the two, as it will make anonymous enums unhappy. + static AssertionResult Compare(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } +}; + +// This specialization is used when the first argument to ASSERT_EQ() +// is a null pointer literal, like NULL, false, or 0. +template <> +class EqHelper { + public: + // We define two overloaded versions of Compare(). The first + // version will be picked when the second argument to ASSERT_EQ() is + // NOT a pointer, e.g. ASSERT_EQ(0, AnIntFunction()) or + // EXPECT_EQ(false, a_bool). + template + static AssertionResult Compare( + const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs, + // The following line prevents this overload from being considered if T2 + // is not a pointer type. We need this because ASSERT_EQ(NULL, my_ptr) + // expands to Compare("", "", NULL, my_ptr), which requires a conversion + // to match the Secret* in the other overload, which would otherwise make + // this template match better. + typename EnableIf::value>::type* = 0) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } + + // This version will be picked when the second argument to ASSERT_EQ() is a + // pointer, e.g. ASSERT_EQ(NULL, a_pointer). + template + static AssertionResult Compare( + const char* lhs_expression, + const char* rhs_expression, + // We used to have a second template parameter instead of Secret*. That + // template parameter would deduce to 'long', making this a better match + // than the first overload even without the first overload's EnableIf. + // Unfortunately, gcc with -Wconversion-null warns when "passing NULL to + // non-pointer argument" (even a deduced integral argument), so the old + // implementation caused warnings in user code. + Secret* /* lhs (NULL) */, + T* rhs) { + // We already know that 'lhs' is a null pointer. + return CmpHelperEQ(lhs_expression, rhs_expression, + static_cast(NULL), rhs); + } +}; + +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperOP. This helps reduce the overhead of some sanitizers +// when calling EXPECT_OP in a tight loop. +template +AssertionResult CmpHelperOpFailure(const char* expr1, const char* expr2, + const T1& val1, const T2& val2, + const char* op) { + return AssertionFailure() + << "Expected: (" << expr1 << ") " << op << " (" << expr2 + << "), actual: " << FormatForComparisonFailureMessage(val1, val2) + << " vs " << FormatForComparisonFailureMessage(val2, val1); +} + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste +// of similar code. +// +// For each templatized helper function, we also define an overloaded +// version for BiggestInt in order to reduce code bloat and allow +// anonymous enums to be used with {ASSERT|EXPECT}_?? when compiled +// with gcc 4. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +template \ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + const T1& val1, const T2& val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return CmpHelperOpFailure(expr1, expr2, val1, val2, #op);\ + }\ +}\ +GTEST_API_ AssertionResult CmpHelper##op_name(\ + const char* expr1, const char* expr2, BiggestInt val1, BiggestInt val2) + +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// Implements the helper function for {ASSERT|EXPECT}_NE +GTEST_IMPL_CMP_HELPER_(NE, !=); +// Implements the helper function for {ASSERT|EXPECT}_LE +GTEST_IMPL_CMP_HELPER_(LE, <=); +// Implements the helper function for {ASSERT|EXPECT}_LT +GTEST_IMPL_CMP_HELPER_(LT, <); +// Implements the helper function for {ASSERT|EXPECT}_GE +GTEST_IMPL_CMP_HELPER_(GE, >=); +// Implements the helper function for {ASSERT|EXPECT}_GT +GTEST_IMPL_CMP_HELPER_(GT, >); + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASEEQ(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRNE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + + +// Helper function for *_STREQ on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2); + +// Helper function for *_STRNE on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2); + +} // namespace internal + +// IsSubstring() and IsNotSubstring() are intended to be used as the +// first argument to {EXPECT,ASSERT}_PRED_FORMAT2(), not by +// themselves. They check whether needle is a substring of haystack +// (NULL is considered a substring of itself only), and return an +// appropriate error message when they fail. +// +// The {needle,haystack}_expr arguments are the stringified +// expressions that generated the two real arguments. +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +// Helper template function for comparing floating-points. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +AssertionResult CmpHelperFloatingPointEQ(const char* lhs_expression, + const char* rhs_expression, + RawType lhs_value, + RawType rhs_value) { + const FloatingPoint lhs(lhs_value), rhs(rhs_value); + + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + ::std::stringstream lhs_ss; + lhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << lhs_value; + + ::std::stringstream rhs_ss; + rhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << rhs_value; + + return EqFailure(lhs_expression, + rhs_expression, + StringStreamToString(&lhs_ss), + StringStreamToString(&rhs_ss), + false); +} + +// Helper function for implementing ASSERT_NEAR. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error); + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// A class that enables one to stream messages to assertion macros +class GTEST_API_ AssertHelper { + public: + // Constructor. + AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message); + ~AssertHelper(); + + // Message assignment is a semantic trick to enable assertion + // streaming; see the GTEST_MESSAGE_ macro below. + void operator=(const Message& message) const; + + private: + // We put our data in a struct so that the size of the AssertHelper class can + // be as small as possible. This is important because gcc is incapable of + // re-using stack space even for temporary variables, so every EXPECT_EQ + // reserves stack space for another AssertHelper. + struct AssertHelperData { + AssertHelperData(TestPartResult::Type t, + const char* srcfile, + int line_num, + const char* msg) + : type(t), file(srcfile), line(line_num), message(msg) { } + + TestPartResult::Type const type; + const char* const file; + int const line; + std::string const message; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelperData); + }; + + AssertHelperData* const data_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelper); +}; + +} // namespace internal + +#if GTEST_HAS_PARAM_TEST +// The pure interface class that all value-parameterized tests inherit from. +// A value-parameterized class must inherit from both ::testing::Test and +// ::testing::WithParamInterface. In most cases that just means inheriting +// from ::testing::TestWithParam, but more complicated test hierarchies +// may need to inherit from Test and WithParamInterface at different levels. +// +// This interface has support for accessing the test parameter value via +// the GetParam() method. +// +// Use it with one of the parameter generator defining functions, like Range(), +// Values(), ValuesIn(), Bool(), and Combine(). +// +// class FooTest : public ::testing::TestWithParam { +// protected: +// FooTest() { +// // Can use GetParam() here. +// } +// virtual ~FooTest() { +// // Can use GetParam() here. +// } +// virtual void SetUp() { +// // Can use GetParam() here. +// } +// virtual void TearDown { +// // Can use GetParam() here. +// } +// }; +// TEST_P(FooTest, DoesBar) { +// // Can use GetParam() method here. +// Foo foo; +// ASSERT_TRUE(foo.DoesBar(GetParam())); +// } +// INSTANTIATE_TEST_CASE_P(OneToTenRange, FooTest, ::testing::Range(1, 10)); + +template +class WithParamInterface { + public: + typedef T ParamType; + virtual ~WithParamInterface() {} + + // The current parameter value. Is also available in the test fixture's + // constructor. This member function is non-static, even though it only + // references static data, to reduce the opportunity for incorrect uses + // like writing 'WithParamInterface::GetParam()' for a test that + // uses a fixture whose parameter type is int. + const ParamType& GetParam() const { + GTEST_CHECK_(parameter_ != NULL) + << "GetParam() can only be called inside a value-parameterized test " + << "-- did you intend to write TEST_P instead of TEST_F?"; + return *parameter_; + } + + private: + // Sets parameter value. The caller is responsible for making sure the value + // remains alive and unchanged throughout the current test. + static void SetParam(const ParamType* parameter) { + parameter_ = parameter; + } + + // Static value used for accessing parameter during a test lifetime. + static const ParamType* parameter_; + + // TestClass must be a subclass of WithParamInterface and Test. + template friend class internal::ParameterizedTestFactory; +}; + +template +const T* WithParamInterface::parameter_ = NULL; + +// Most value-parameterized classes can ignore the existence of +// WithParamInterface, and can just inherit from ::testing::TestWithParam. + +template +class TestWithParam : public Test, public WithParamInterface { +}; + +#endif // GTEST_HAS_PARAM_TEST + +// Macros for indicating success/failure in test code. + +// ADD_FAILURE unconditionally adds a failure to the current test. +// SUCCEED generates a success - it doesn't automatically make the +// current test successful, as a test is only successful when it has +// no failure. +// +// EXPECT_* verifies that a certain condition is satisfied. If not, +// it behaves like ADD_FAILURE. In particular: +// +// EXPECT_TRUE verifies that a Boolean condition is true. +// EXPECT_FALSE verifies that a Boolean condition is false. +// +// FAIL and ASSERT_* are similar to ADD_FAILURE and EXPECT_*, except +// that they will also abort the current function on failure. People +// usually want the fail-fast behavior of FAIL and ASSERT_*, but those +// writing data-driven tests often find themselves using ADD_FAILURE +// and EXPECT_* more. + +// Generates a nonfatal failure with a generic message. +#define ADD_FAILURE() GTEST_NONFATAL_FAILURE_("Failed") + +// Generates a nonfatal failure at the given source file location with +// a generic message. +#define ADD_FAILURE_AT(file, line) \ + GTEST_MESSAGE_AT_(file, line, "Failed", \ + ::testing::TestPartResult::kNonFatalFailure) + +// Generates a fatal failure with a generic message. +#define GTEST_FAIL() GTEST_FATAL_FAILURE_("Failed") + +// Define this macro to 1 to omit the definition of FAIL(), which is a +// generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_FAIL +# define FAIL() GTEST_FAIL() +#endif + +// Generates a success with a generic message. +#define GTEST_SUCCEED() GTEST_SUCCESS_("Succeeded") + +// Define this macro to 1 to omit the definition of SUCCEED(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_SUCCEED +# define SUCCEED() GTEST_SUCCEED() +#endif + +// Macros for testing exceptions. +// +// * {ASSERT|EXPECT}_THROW(statement, expected_exception): +// Tests that the statement throws the expected exception. +// * {ASSERT|EXPECT}_NO_THROW(statement): +// Tests that the statement doesn't throw any exception. +// * {ASSERT|EXPECT}_ANY_THROW(statement): +// Tests that the statement throws an exception. + +#define EXPECT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_NONFATAL_FAILURE_) +#define EXPECT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define EXPECT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define ASSERT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_FATAL_FAILURE_) +#define ASSERT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_FATAL_FAILURE_) +#define ASSERT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_FATAL_FAILURE_) + +// Boolean assertions. Condition can be either a Boolean expression or an +// AssertionResult. For more information on how to use AssertionResult with +// these macros see comments on that class. +#define EXPECT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_((condition), #condition, false, true, \ + GTEST_NONFATAL_FAILURE_) +#define EXPECT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_NONFATAL_FAILURE_) +#define ASSERT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_((condition), #condition, false, true, \ + GTEST_FATAL_FAILURE_) +#define ASSERT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_FATAL_FAILURE_) + +// Includes the auto-generated header that implements a family of +// generic predicate assertion macros. +#include "gtest/gtest_pred_impl.h" + +// Macros for testing equalities and inequalities. +// +// * {ASSERT|EXPECT}_EQ(v1, v2): Tests that v1 == v2 +// * {ASSERT|EXPECT}_NE(v1, v2): Tests that v1 != v2 +// * {ASSERT|EXPECT}_LT(v1, v2): Tests that v1 < v2 +// * {ASSERT|EXPECT}_LE(v1, v2): Tests that v1 <= v2 +// * {ASSERT|EXPECT}_GT(v1, v2): Tests that v1 > v2 +// * {ASSERT|EXPECT}_GE(v1, v2): Tests that v1 >= v2 +// +// When they are not, Google Test prints both the tested expressions and +// their actual values. The values must be compatible built-in types, +// or you will get a compiler error. By "compatible" we mean that the +// values can be compared by the respective operator. +// +// Note: +// +// 1. It is possible to make a user-defined type work with +// {ASSERT|EXPECT}_??(), but that requires overloading the +// comparison operators and is thus discouraged by the Google C++ +// Usage Guide. Therefore, you are advised to use the +// {ASSERT|EXPECT}_TRUE() macro to assert that two objects are +// equal. +// +// 2. The {ASSERT|EXPECT}_??() macros do pointer comparisons on +// pointers (in particular, C strings). Therefore, if you use it +// with two C strings, you are testing how their locations in memory +// are related, not how their content is related. To compare two C +// strings by content, use {ASSERT|EXPECT}_STR*(). +// +// 3. {ASSERT|EXPECT}_EQ(v1, v2) is preferred to +// {ASSERT|EXPECT}_TRUE(v1 == v2), as the former tells you +// what the actual value is when it fails, and similarly for the +// other comparisons. +// +// 4. Do not depend on the order in which {ASSERT|EXPECT}_??() +// evaluate their arguments, which is undefined. +// +// 5. These macros evaluate their arguments exactly once. +// +// Examples: +// +// EXPECT_NE(5, Foo()); +// EXPECT_EQ(NULL, a_pointer); +// ASSERT_LT(i, array_size); +// ASSERT_GT(records.size(), 0) << "There is no record left."; + +#define EXPECT_EQ(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + val1, val2) +#define EXPECT_NE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) +#define EXPECT_LE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define EXPECT_LT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define EXPECT_GE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define EXPECT_GT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +#define GTEST_ASSERT_EQ(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + val1, val2) +#define GTEST_ASSERT_NE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) +#define GTEST_ASSERT_LE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define GTEST_ASSERT_LT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define GTEST_ASSERT_GE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define GTEST_ASSERT_GT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +// Define macro GTEST_DONT_DEFINE_ASSERT_XY to 1 to omit the definition of +// ASSERT_XY(), which clashes with some users' own code. + +#if !GTEST_DONT_DEFINE_ASSERT_EQ +# define ASSERT_EQ(val1, val2) GTEST_ASSERT_EQ(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_NE +# define ASSERT_NE(val1, val2) GTEST_ASSERT_NE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LE +# define ASSERT_LE(val1, val2) GTEST_ASSERT_LE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LT +# define ASSERT_LT(val1, val2) GTEST_ASSERT_LT(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GE +# define ASSERT_GE(val1, val2) GTEST_ASSERT_GE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GT +# define ASSERT_GT(val1, val2) GTEST_ASSERT_GT(val1, val2) +#endif + +// C-string Comparisons. All tests treat NULL and any non-NULL string +// as different. Two NULLs are equal. +// +// * {ASSERT|EXPECT}_STREQ(s1, s2): Tests that s1 == s2 +// * {ASSERT|EXPECT}_STRNE(s1, s2): Tests that s1 != s2 +// * {ASSERT|EXPECT}_STRCASEEQ(s1, s2): Tests that s1 == s2, ignoring case +// * {ASSERT|EXPECT}_STRCASENE(s1, s2): Tests that s1 != s2, ignoring case +// +// For wide or narrow string objects, you can use the +// {ASSERT|EXPECT}_??() macros. +// +// Don't depend on the order in which the arguments are evaluated, +// which is undefined. +// +// These macros evaluate their arguments exactly once. + +#define EXPECT_STREQ(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) +#define EXPECT_STRNE(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define EXPECT_STRCASEEQ(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) +#define EXPECT_STRCASENE(s1, s2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +#define ASSERT_STREQ(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) +#define ASSERT_STRNE(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define ASSERT_STRCASEEQ(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) +#define ASSERT_STRCASENE(s1, s2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +// Macros for comparing floating-point numbers. +// +// * {ASSERT|EXPECT}_FLOAT_EQ(val1, val2): +// Tests that two float values are almost equal. +// * {ASSERT|EXPECT}_DOUBLE_EQ(val1, val2): +// Tests that two double values are almost equal. +// * {ASSERT|EXPECT}_NEAR(v1, v2, abs_error): +// Tests that v1 and v2 are within the given distance to each other. +// +// Google Test uses ULP-based comparison to automatically pick a default +// error bound that is appropriate for the operands. See the +// FloatingPoint template class in gtest-internal.h if you are +// interested in the implementation details. + +#define EXPECT_FLOAT_EQ(val1, val2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define EXPECT_DOUBLE_EQ(val1, val2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define ASSERT_FLOAT_EQ(val1, val2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define ASSERT_DOUBLE_EQ(val1, val2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define EXPECT_NEAR(val1, val2, abs_error)\ + EXPECT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +#define ASSERT_NEAR(val1, val2, abs_error)\ + ASSERT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +// These predicate format functions work on floating-point values, and +// can be used in {ASSERT|EXPECT}_PRED_FORMAT2*(), e.g. +// +// EXPECT_PRED_FORMAT2(testing::DoubleLE, Foo(), 5.0); + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +GTEST_API_ AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2); +GTEST_API_ AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2); + + +#if GTEST_OS_WINDOWS + +// Macros that test for HRESULT failure and success, these are only useful +// on Windows, and rely on Windows SDK macros and APIs to compile. +// +// * {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED}(expr) +// +// When expr unexpectedly fails or succeeds, Google Test prints the +// expected result and the actual result with both a human-readable +// string representation of the error, if available, as well as the +// hex result code. +# define EXPECT_HRESULT_SUCCEEDED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define ASSERT_HRESULT_SUCCEEDED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define EXPECT_HRESULT_FAILED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +# define ASSERT_HRESULT_FAILED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +#endif // GTEST_OS_WINDOWS + +// Macros that execute statement and check that it doesn't generate new fatal +// failures in the current thread. +// +// * {ASSERT|EXPECT}_NO_FATAL_FAILURE(statement); +// +// Examples: +// +// EXPECT_NO_FATAL_FAILURE(Process()); +// ASSERT_NO_FATAL_FAILURE(Process()) << "Process() failed"; +// +#define ASSERT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_FATAL_FAILURE_) +#define EXPECT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_NONFATAL_FAILURE_) + +// Causes a trace (including the source file path, the current line +// number, and the given message) to be included in every test failure +// message generated by code in the current scope. The effect is +// undone when the control leaves the current scope. +// +// The message argument can be anything streamable to std::ostream. +// +// In the implementation, we include the current line number as part +// of the dummy variable name, thus allowing multiple SCOPED_TRACE()s +// to appear in the same block - as long as they are on different +// lines. +#define SCOPED_TRACE(message) \ + ::testing::internal::ScopedTrace GTEST_CONCAT_TOKEN_(gtest_trace_, __LINE__)(\ + __FILE__, __LINE__, ::testing::Message() << (message)) + +// Compile-time assertion for type equality. +// StaticAssertTypeEq() compiles iff type1 and type2 are +// the same type. The value it returns is not interesting. +// +// Instead of making StaticAssertTypeEq a class template, we make it a +// function template that invokes a helper class template. This +// prevents a user from misusing StaticAssertTypeEq by +// defining objects of that type. +// +// CAVEAT: +// +// When used inside a method of a class template, +// StaticAssertTypeEq() is effective ONLY IF the method is +// instantiated. For example, given: +// +// template class Foo { +// public: +// void Bar() { testing::StaticAssertTypeEq(); } +// }; +// +// the code: +// +// void Test1() { Foo foo; } +// +// will NOT generate a compiler error, as Foo::Bar() is never +// actually instantiated. Instead, you need: +// +// void Test2() { Foo foo; foo.Bar(); } +// +// to cause a compiler error. +template +bool StaticAssertTypeEq() { + (void)internal::StaticAssertTypeEqHelper(); + return true; +} + +// Defines a test. +// +// The first parameter is the name of the test case, and the second +// parameter is the name of the test within the test case. +// +// The convention is to end the test case name with "Test". For +// example, a test case for the Foo class can be named FooTest. +// +// Test code should appear between braces after an invocation of +// this macro. Example: +// +// TEST(FooTest, InitializesCorrectly) { +// Foo foo; +// EXPECT_TRUE(foo.StatusIsOK()); +// } + +// Note that we call GetTestTypeId() instead of GetTypeId< +// ::testing::Test>() here to get the type ID of testing::Test. This +// is to work around a suspected linker bug when using Google Test as +// a framework on Mac OS X. The bug causes GetTypeId< +// ::testing::Test>() to return different values depending on whether +// the call is from the Google Test framework itself or from user test +// code. GetTestTypeId() is guaranteed to always return the same +// value, as it always calls GetTypeId<>() from the Google Test +// framework. +#define GTEST_TEST(test_case_name, test_name)\ + GTEST_TEST_(test_case_name, test_name, \ + ::testing::Test, ::testing::internal::GetTestTypeId()) + +// Define this macro to 1 to omit the definition of TEST(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_TEST +# define TEST(test_case_name, test_name) GTEST_TEST(test_case_name, test_name) +#endif + +// Defines a test that uses a test fixture. +// +// The first parameter is the name of the test fixture class, which +// also doubles as the test case name. The second parameter is the +// name of the test within the test case. +// +// A test fixture class must be declared earlier. The user should put +// his test code between braces after using this macro. Example: +// +// class FooTest : public testing::Test { +// protected: +// virtual void SetUp() { b_.AddElement(3); } +// +// Foo a_; +// Foo b_; +// }; +// +// TEST_F(FooTest, InitializesCorrectly) { +// EXPECT_TRUE(a_.StatusIsOK()); +// } +// +// TEST_F(FooTest, ReturnsElementCountCorrectly) { +// EXPECT_EQ(0, a_.size()); +// EXPECT_EQ(1, b_.size()); +// } + +#define TEST_F(test_fixture, test_name)\ + GTEST_TEST_(test_fixture, test_name, test_fixture, \ + ::testing::internal::GetTypeId()) + +} // namespace testing + +// Use this function in main() to run all tests. It returns 0 if all +// tests are successful, or 1 otherwise. +// +// RUN_ALL_TESTS() should be invoked after the command line has been +// parsed by InitGoogleTest(). +// +// This function was formerly a macro; thus, it is in the global +// namespace and has an all-caps name. +int RUN_ALL_TESTS() GTEST_MUST_USE_RESULT_; + +inline int RUN_ALL_TESTS() { + return ::testing::UnitTest::GetInstance()->Run(); +} + +#endif // GTEST_INCLUDE_GTEST_GTEST_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest_pred_impl.h b/mypyc/external/googletest/include/gtest/gtest_pred_impl.h new file mode 100644 index 000000000000..30ae712f50ed --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest_pred_impl.h @@ -0,0 +1,358 @@ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is AUTOMATICALLY GENERATED on 10/31/2011 by command +// 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! +// +// Implements a family of generic predicate assertion macros. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ + +// Makes sure this header is not included before gtest.h. +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +# error Do not include gtest_pred_impl.h directly. Include gtest.h instead. +#endif // GTEST_INCLUDE_GTEST_GTEST_H_ + +// This header implements a family of generic predicate assertion +// macros: +// +// ASSERT_PRED_FORMAT1(pred_format, v1) +// ASSERT_PRED_FORMAT2(pred_format, v1, v2) +// ... +// +// where pred_format is a function or functor that takes n (in the +// case of ASSERT_PRED_FORMATn) values and their source expression +// text, and returns a testing::AssertionResult. See the definition +// of ASSERT_EQ in gtest.h for an example. +// +// If you don't care about formatting, you can use the more +// restrictive version: +// +// ASSERT_PRED1(pred, v1) +// ASSERT_PRED2(pred, v1, v2) +// ... +// +// where pred is an n-ary function or functor that returns bool, +// and the values v1, v2, ..., must support the << operator for +// streaming to std::ostream. +// +// We also define the EXPECT_* variations. +// +// For now we only support predicates whose arity is at most 5. +// Please email googletestframework@googlegroups.com if you need +// support for higher arities. + +// GTEST_ASSERT_ is the basic statement to which all of the assertions +// in this file reduce. Don't use this in your code. + +#define GTEST_ASSERT_(expression, on_failure) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar = (expression)) \ + ; \ + else \ + on_failure(gtest_ar.failure_message()) + + +// Helper function for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +template +AssertionResult AssertPred1Helper(const char* pred_text, + const char* e1, + Pred pred, + const T1& v1) { + if (pred(v1)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT1. +// Don't use this in your code. +#define GTEST_PRED_FORMAT1_(pred_format, v1, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, v1), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +#define GTEST_PRED1_(pred, v1, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred1Helper(#pred, \ + #v1, \ + pred, \ + v1), on_failure) + +// Unary predicate assertion macros. +#define EXPECT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +template +AssertionResult AssertPred2Helper(const char* pred_text, + const char* e1, + const char* e2, + Pred pred, + const T1& v1, + const T2& v2) { + if (pred(v1, v2)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT2. +// Don't use this in your code. +#define GTEST_PRED_FORMAT2_(pred_format, v1, v2, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, v1, v2), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +#define GTEST_PRED2_(pred, v1, v2, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred2Helper(#pred, \ + #v1, \ + #v2, \ + pred, \ + v1, \ + v2), on_failure) + +// Binary predicate assertion macros. +#define EXPECT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +template +AssertionResult AssertPred3Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3) { + if (pred(v1, v2, v3)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT3. +// Don't use this in your code. +#define GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, v1, v2, v3), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +#define GTEST_PRED3_(pred, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred3Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + pred, \ + v1, \ + v2, \ + v3), on_failure) + +// Ternary predicate assertion macros. +#define EXPECT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +template +AssertionResult AssertPred4Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4) { + if (pred(v1, v2, v3, v4)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT4. +// Don't use this in your code. +#define GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, v1, v2, v3, v4), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +#define GTEST_PRED4_(pred, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred4Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4), on_failure) + +// 4-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +template +AssertionResult AssertPred5Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + const char* e5, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4, + const T5& v5) { + if (pred(v1, v2, v3, v4, v5)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ", " + << e5 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4 + << "\n" << e5 << " evaluates to " << v5; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT5. +// Don't use this in your code. +#define GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, #v5, v1, v2, v3, v4, v5), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +#define GTEST_PRED5_(pred, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred5Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + #v5, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4, \ + v5), on_failure) + +// 5-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) + + + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ diff --git a/mypyc/external/googletest/include/gtest/gtest_prod.h b/mypyc/external/googletest/include/gtest/gtest_prod.h new file mode 100644 index 000000000000..da80ddc6c70e --- /dev/null +++ b/mypyc/external/googletest/include/gtest/gtest_prod.h @@ -0,0 +1,58 @@ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// Google C++ Testing Framework definitions useful in production code. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ + +// When you need to test the private or protected members of a class, +// use the FRIEND_TEST macro to declare your tests as friends of the +// class. For example: +// +// class MyClass { +// private: +// void MyMethod(); +// FRIEND_TEST(MyClassTest, MyMethod); +// }; +// +// class MyClassTest : public testing::Test { +// // ... +// }; +// +// TEST_F(MyClassTest, MyMethod) { +// // Can call MyClass::MyMethod() here. +// } + +#define FRIEND_TEST(test_case_name, test_name)\ +friend class test_case_name##_##test_name##_Test + +#endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/custom/gtest-port.h b/mypyc/external/googletest/include/gtest/internal/custom/gtest-port.h new file mode 100644 index 000000000000..7e744bd3bb38 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/custom/gtest-port.h @@ -0,0 +1,69 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Injection point for custom user configurations. +// The following macros can be defined: +// +// Flag related macros: +// GTEST_FLAG(flag_name) +// GTEST_USE_OWN_FLAGFILE_FLAG_ - Define to 0 when the system provides its +// own flagfile flag parsing. +// GTEST_DECLARE_bool_(name) +// GTEST_DECLARE_int32_(name) +// GTEST_DECLARE_string_(name) +// GTEST_DEFINE_bool_(name, default_val, doc) +// GTEST_DEFINE_int32_(name, default_val, doc) +// GTEST_DEFINE_string_(name, default_val, doc) +// +// Test filtering: +// GTEST_TEST_FILTER_ENV_VAR_ - The name of an environment variable that +// will be used if --GTEST_FLAG(test_filter) +// is not provided. +// +// Logging: +// GTEST_LOG_(severity) +// GTEST_CHECK_(condition) +// Functions LogToStderr() and FlushInfoLog() have to be provided too. +// +// Threading: +// GTEST_HAS_NOTIFICATION_ - Enabled if Notification is already provided. +// GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ - Enabled if Mutex and ThreadLocal are +// already provided. +// Must also provide GTEST_DECLARE_STATIC_MUTEX_(mutex) and +// GTEST_DEFINE_STATIC_MUTEX_(mutex) +// +// GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks) +// GTEST_LOCK_EXCLUDED_(locks) +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/custom/gtest-printers.h b/mypyc/external/googletest/include/gtest/internal/custom/gtest-printers.h new file mode 100644 index 000000000000..60c1ea050b61 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/custom/gtest-printers.h @@ -0,0 +1,42 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// This file provides an injection point for custom printers in a local +// installation of gTest. +// It will be included from gtest-printers.h and the overrides in this file +// will be visible to everyone. +// See documentation at gtest/gtest-printers.h for details on how to define a +// custom printer. +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/custom/gtest.h b/mypyc/external/googletest/include/gtest/internal/custom/gtest.h new file mode 100644 index 000000000000..c27412a89818 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/custom/gtest.h @@ -0,0 +1,41 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Injection point for custom user configurations. +// The following macros can be defined: +// +// GTEST_OS_STACK_TRACE_GETTER_ - The name of an implementation of +// OsStackTraceGetterInterface. +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-death-test-internal.h b/mypyc/external/googletest/include/gtest/internal/gtest-death-test-internal.h new file mode 100644 index 000000000000..2b3a78f5bf86 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-death-test-internal.h @@ -0,0 +1,319 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines internal utilities needed for implementing +// death tests. They are subject to change without notice. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ + +#include "gtest/internal/gtest-internal.h" + +#include + +namespace testing { +namespace internal { + +GTEST_DECLARE_string_(internal_run_death_test); + +// Names of the flags (needed for parsing Google Test flags). +const char kDeathTestStyleFlag[] = "death_test_style"; +const char kDeathTestUseFork[] = "death_test_use_fork"; +const char kInternalRunDeathTestFlag[] = "internal_run_death_test"; + +#if GTEST_HAS_DEATH_TEST + +// DeathTest is a class that hides much of the complexity of the +// GTEST_DEATH_TEST_ macro. It is abstract; its static Create method +// returns a concrete class that depends on the prevailing death test +// style, as defined by the --gtest_death_test_style and/or +// --gtest_internal_run_death_test flags. + +// In describing the results of death tests, these terms are used with +// the corresponding definitions: +// +// exit status: The integer exit information in the format specified +// by wait(2) +// exit code: The integer code passed to exit(3), _exit(2), or +// returned from main() +class GTEST_API_ DeathTest { + public: + // Create returns false if there was an error determining the + // appropriate action to take for the current death test; for example, + // if the gtest_death_test_style flag is set to an invalid value. + // The LastMessage method will return a more detailed message in that + // case. Otherwise, the DeathTest pointer pointed to by the "test" + // argument is set. If the death test should be skipped, the pointer + // is set to NULL; otherwise, it is set to the address of a new concrete + // DeathTest object that controls the execution of the current test. + static bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); + DeathTest(); + virtual ~DeathTest() { } + + // A helper class that aborts a death test when it's deleted. + class ReturnSentinel { + public: + explicit ReturnSentinel(DeathTest* test) : test_(test) { } + ~ReturnSentinel() { test_->Abort(TEST_ENCOUNTERED_RETURN_STATEMENT); } + private: + DeathTest* const test_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ReturnSentinel); + } GTEST_ATTRIBUTE_UNUSED_; + + // An enumeration of possible roles that may be taken when a death + // test is encountered. EXECUTE means that the death test logic should + // be executed immediately. OVERSEE means that the program should prepare + // the appropriate environment for a child process to execute the death + // test, then wait for it to complete. + enum TestRole { OVERSEE_TEST, EXECUTE_TEST }; + + // An enumeration of the three reasons that a test might be aborted. + enum AbortReason { + TEST_ENCOUNTERED_RETURN_STATEMENT, + TEST_THREW_EXCEPTION, + TEST_DID_NOT_DIE + }; + + // Assumes one of the above roles. + virtual TestRole AssumeRole() = 0; + + // Waits for the death test to finish and returns its status. + virtual int Wait() = 0; + + // Returns true if the death test passed; that is, the test process + // exited during the test, its exit status matches a user-supplied + // predicate, and its stderr output matches a user-supplied regular + // expression. + // The user-supplied predicate may be a macro expression rather + // than a function pointer or functor, or else Wait and Passed could + // be combined. + virtual bool Passed(bool exit_status_ok) = 0; + + // Signals that the death test did not die as expected. + virtual void Abort(AbortReason reason) = 0; + + // Returns a human-readable outcome message regarding the outcome of + // the last death test. + static const char* LastMessage(); + + static void set_last_death_test_message(const std::string& message); + + private: + // A string containing a description of the outcome of the last death test. + static std::string last_death_test_message_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DeathTest); +}; + +// Factory interface for death tests. May be mocked out for testing. +class DeathTestFactory { + public: + virtual ~DeathTestFactory() { } + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) = 0; +}; + +// A concrete DeathTestFactory implementation for normal use. +class DefaultDeathTestFactory : public DeathTestFactory { + public: + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); +}; + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +GTEST_API_ bool ExitedUnsuccessfully(int exit_status); + +// Traps C++ exceptions escaping statement and reports them as test +// failures. Note that trapping SEH exceptions is not implemented here. +# if GTEST_HAS_EXCEPTIONS +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } catch (const ::std::exception& gtest_exception) { \ + fprintf(\ + stderr, \ + "\n%s: Caught std::exception-derived exception escaping the " \ + "death test statement. Exception message: %s\n", \ + ::testing::internal::FormatFileLocation(__FILE__, __LINE__).c_str(), \ + gtest_exception.what()); \ + fflush(stderr); \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } catch (...) { \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } + +# else +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) + +# endif + +// This macro is for implementing ASSERT_DEATH*, EXPECT_DEATH*, +// ASSERT_EXIT*, and EXPECT_EXIT*. +# define GTEST_DEATH_TEST_(statement, predicate, regex, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + const ::testing::internal::RE& gtest_regex = (regex); \ + ::testing::internal::DeathTest* gtest_dt; \ + if (!::testing::internal::DeathTest::Create(#statement, >est_regex, \ + __FILE__, __LINE__, >est_dt)) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + if (gtest_dt != NULL) { \ + ::testing::internal::scoped_ptr< ::testing::internal::DeathTest> \ + gtest_dt_ptr(gtest_dt); \ + switch (gtest_dt->AssumeRole()) { \ + case ::testing::internal::DeathTest::OVERSEE_TEST: \ + if (!gtest_dt->Passed(predicate(gtest_dt->Wait()))) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + break; \ + case ::testing::internal::DeathTest::EXECUTE_TEST: { \ + ::testing::internal::DeathTest::ReturnSentinel \ + gtest_sentinel(gtest_dt); \ + GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, gtest_dt); \ + gtest_dt->Abort(::testing::internal::DeathTest::TEST_DID_NOT_DIE); \ + break; \ + } \ + default: \ + break; \ + } \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__): \ + fail(::testing::internal::DeathTest::LastMessage()) +// The symbol "fail" here expands to something into which a message +// can be streamed. + +// This macro is for implementing ASSERT/EXPECT_DEBUG_DEATH when compiled in +// NDEBUG mode. In this case we need the statements to be executed, the regex is +// ignored, and the macro must accept a streamed message even though the message +// is never printed. +# define GTEST_EXECUTE_STATEMENT_(statement, regex) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } else \ + ::testing::Message() + +// A class representing the parsed contents of the +// --gtest_internal_run_death_test flag, as it existed when +// RUN_ALL_TESTS was called. +class InternalRunDeathTestFlag { + public: + InternalRunDeathTestFlag(const std::string& a_file, + int a_line, + int an_index, + int a_write_fd) + : file_(a_file), line_(a_line), index_(an_index), + write_fd_(a_write_fd) {} + + ~InternalRunDeathTestFlag() { + if (write_fd_ >= 0) + posix::Close(write_fd_); + } + + const std::string& file() const { return file_; } + int line() const { return line_; } + int index() const { return index_; } + int write_fd() const { return write_fd_; } + + private: + std::string file_; + int line_; + int index_; + int write_fd_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InternalRunDeathTestFlag); +}; + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag(); + +#else // GTEST_HAS_DEATH_TEST + +// This macro is used for implementing macros such as +// EXPECT_DEATH_IF_SUPPORTED and ASSERT_DEATH_IF_SUPPORTED on systems where +// death tests are not supported. Those macros must compile on such systems +// iff EXPECT_DEATH and ASSERT_DEATH compile with the same parameters on +// systems that support death tests. This allows one to write such a macro +// on a system that does not support death tests and be sure that it will +// compile on a death-test supporting system. +// +// Parameters: +// statement - A statement that a macro such as EXPECT_DEATH would test +// for program termination. This macro has to make sure this +// statement is compiled but not executed, to ensure that +// EXPECT_DEATH_IF_SUPPORTED compiles with a certain +// parameter iff EXPECT_DEATH compiles with it. +// regex - A regex that a macro such as EXPECT_DEATH would use to test +// the output of statement. This parameter has to be +// compiled but not evaluated by this macro, to ensure that +// this macro only accepts expressions that a macro such as +// EXPECT_DEATH would accept. +// terminator - Must be an empty statement for EXPECT_DEATH_IF_SUPPORTED +// and a return statement for ASSERT_DEATH_IF_SUPPORTED. +// This ensures that ASSERT_DEATH_IF_SUPPORTED will not +// compile inside functions where ASSERT_DEATH doesn't +// compile. +// +// The branch that has an always false condition is used to ensure that +// statement and regex are compiled (and thus syntactically correct) but +// never executed. The unreachable code macro protects the terminator +// statement from generating an 'unreachable code' warning in case +// statement unconditionally returns or throws. The Message constructor at +// the end allows the syntax of streaming additional messages into the +// macro, for compilational compatibility with EXPECT_DEATH/ASSERT_DEATH. +# define GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, terminator) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_LOG_(WARNING) \ + << "Death tests are not supported on this platform.\n" \ + << "Statement '" #statement "' cannot be verified."; \ + } else if (::testing::internal::AlwaysFalse()) { \ + ::testing::internal::RE::PartialMatch(".*", (regex)); \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + terminator; \ + } else \ + ::testing::Message() + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-filepath.h b/mypyc/external/googletest/include/gtest/internal/gtest-filepath.h new file mode 100644 index 000000000000..7a13b4b0de60 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-filepath.h @@ -0,0 +1,206 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: keith.ray@gmail.com (Keith Ray) +// +// Google Test filepath utilities +// +// This header file declares classes and functions used internally by +// Google Test. They are subject to change without notice. +// +// This file is #included in . +// Do not include this header file separately! + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ + +#include "gtest/internal/gtest-string.h" + +namespace testing { +namespace internal { + +// FilePath - a class for file and directory pathname manipulation which +// handles platform-specific conventions (like the pathname separator). +// Used for helper functions for naming files in a directory for xml output. +// Except for Set methods, all methods are const or static, which provides an +// "immutable value object" -- useful for peace of mind. +// A FilePath with a value ending in a path separator ("like/this/") represents +// a directory, otherwise it is assumed to represent a file. In either case, +// it may or may not represent an actual file or directory in the file system. +// Names are NOT checked for syntax correctness -- no checking for illegal +// characters, malformed paths, etc. + +class GTEST_API_ FilePath { + public: + FilePath() : pathname_("") { } + FilePath(const FilePath& rhs) : pathname_(rhs.pathname_) { } + + explicit FilePath(const std::string& pathname) : pathname_(pathname) { + Normalize(); + } + + FilePath& operator=(const FilePath& rhs) { + Set(rhs); + return *this; + } + + void Set(const FilePath& rhs) { + pathname_ = rhs.pathname_; + } + + const std::string& string() const { return pathname_; } + const char* c_str() const { return pathname_.c_str(); } + + // Returns the current working directory, or "" if unsuccessful. + static FilePath GetCurrentDir(); + + // Given directory = "dir", base_name = "test", number = 0, + // extension = "xml", returns "dir/test.xml". If number is greater + // than zero (e.g., 12), returns "dir/test_12.xml". + // On Windows platform, uses \ as the separator rather than /. + static FilePath MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension); + + // Given directory = "dir", relative_path = "test.xml", + // returns "dir/test.xml". + // On Windows, uses \ as the separator rather than /. + static FilePath ConcatPaths(const FilePath& directory, + const FilePath& relative_path); + + // Returns a pathname for a file that does not currently exist. The pathname + // will be directory/base_name.extension or + // directory/base_name_.extension if directory/base_name.extension + // already exists. The number will be incremented until a pathname is found + // that does not already exist. + // Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. + // There could be a race condition if two or more processes are calling this + // function at the same time -- they could both pick the same filename. + static FilePath GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension); + + // Returns true iff the path is "". + bool IsEmpty() const { return pathname_.empty(); } + + // If input name has a trailing separator character, removes it and returns + // the name, otherwise return the name string unmodified. + // On Windows platform, uses \ as the separator, other platforms use /. + FilePath RemoveTrailingPathSeparator() const; + + // Returns a copy of the FilePath with the directory part removed. + // Example: FilePath("path/to/file").RemoveDirectoryName() returns + // FilePath("file"). If there is no directory part ("just_a_file"), it returns + // the FilePath unmodified. If there is no file part ("just_a_dir/") it + // returns an empty FilePath (""). + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveDirectoryName() const; + + // RemoveFileName returns the directory path with the filename removed. + // Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". + // If the FilePath is "a_file" or "/a_file", RemoveFileName returns + // FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does + // not have a file, like "just/a/dir/", it returns the FilePath unmodified. + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveFileName() const; + + // Returns a copy of the FilePath with the case-insensitive extension removed. + // Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns + // FilePath("dir/file"). If a case-insensitive extension is not + // found, returns a copy of the original FilePath. + FilePath RemoveExtension(const char* extension) const; + + // Creates directories so that path exists. Returns true if successful or if + // the directories already exist; returns false if unable to create + // directories for any reason. Will also return false if the FilePath does + // not represent a directory (that is, it doesn't end with a path separator). + bool CreateDirectoriesRecursively() const; + + // Create the directory so that path exists. Returns true if successful or + // if the directory already exists; returns false if unable to create the + // directory for any reason, including if the parent directory does not + // exist. Not named "CreateDirectory" because that's a macro on Windows. + bool CreateFolder() const; + + // Returns true if FilePath describes something in the file-system, + // either a file, directory, or whatever, and that something exists. + bool FileOrDirectoryExists() const; + + // Returns true if pathname describes a directory in the file-system + // that exists. + bool DirectoryExists() const; + + // Returns true if FilePath ends with a path separator, which indicates that + // it is intended to represent a directory. Returns false otherwise. + // This does NOT check that a directory (or file) actually exists. + bool IsDirectory() const; + + // Returns true if pathname describes a root directory. (Windows has one + // root directory per disk drive.) + bool IsRootDirectory() const; + + // Returns true if pathname describes an absolute path. + bool IsAbsolutePath() const; + + private: + // Replaces multiple consecutive separators with a single separator. + // For example, "bar///foo" becomes "bar/foo". Does not eliminate other + // redundancies that might be in a pathname involving "." or "..". + // + // A pathname with multiple consecutive separators may occur either through + // user error or as a result of some scripts or APIs that generate a pathname + // with a trailing separator. On other platforms the same API or script + // may NOT generate a pathname with a trailing "/". Then elsewhere that + // pathname may have another "/" and pathname components added to it, + // without checking for the separator already being there. + // The script language and operating system may allow paths like "foo//bar" + // but some of the functions in FilePath will not handle that correctly. In + // particular, RemoveTrailingPathSeparator() only removes one separator, and + // it is called in CreateDirectoriesRecursively() assuming that it will change + // a pathname from directory syntax (trailing separator) to filename syntax. + // + // On Windows this method also replaces the alternate path separator '/' with + // the primary path separator '\\', so that for example "bar\\/\\foo" becomes + // "bar\\foo". + + void Normalize(); + + // Returns a pointer to the last occurence of a valid path separator in + // the FilePath. On Windows, for example, both '/' and '\' are valid path + // separators. Returns NULL if no path separator was found. + const char* FindLastPathSeparator() const; + + std::string pathname_; +}; // class FilePath + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-internal.h b/mypyc/external/googletest/include/gtest/internal/gtest-internal.h new file mode 100644 index 000000000000..ebd1cf615de0 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-internal.h @@ -0,0 +1,1238 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file declares functions and macros used internally by +// Google Test. They are subject to change without notice. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ + +#include "gtest/internal/gtest-port.h" + +#if GTEST_OS_LINUX +# include +# include +# include +# include +#endif // GTEST_OS_LINUX + +#if GTEST_HAS_EXCEPTIONS +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-string.h" +#include "gtest/internal/gtest-filepath.h" +#include "gtest/internal/gtest-type-util.h" + +// Due to C++ preprocessor weirdness, we need double indirection to +// concatenate two tokens when one of them is __LINE__. Writing +// +// foo ## __LINE__ +// +// will result in the token foo__LINE__, instead of foo followed by +// the current line number. For more details, see +// http://www.parashift.com/c++-faq-lite/misc-technical-issues.html#faq-39.6 +#define GTEST_CONCAT_TOKEN_(foo, bar) GTEST_CONCAT_TOKEN_IMPL_(foo, bar) +#define GTEST_CONCAT_TOKEN_IMPL_(foo, bar) foo ## bar + +class ProtocolMessage; +namespace proto2 { class Message; } + +namespace testing { + +// Forward declarations. + +class AssertionResult; // Result of an assertion. +class Message; // Represents a failure message. +class Test; // Represents a test. +class TestInfo; // Information about a test. +class TestPartResult; // Result of a test part. +class UnitTest; // A collection of test cases. + +template +::std::string PrintToString(const T& value); + +namespace internal { + +struct TraceInfo; // Information about a trace point. +class ScopedTrace; // Implements scoped trace. +class TestInfoImpl; // Opaque implementation of TestInfo +class UnitTestImpl; // Opaque implementation of UnitTest + +// The text used in failure messages to indicate the start of the +// stack trace. +GTEST_API_ extern const char kStackTraceMarker[]; + +// Two overloaded helpers for checking at compile time whether an +// expression is a null pointer literal (i.e. NULL or any 0-valued +// compile-time integral constant). Their return values have +// different sizes, so we can use sizeof() to test which version is +// picked by the compiler. These helpers have no implementations, as +// we only need their signatures. +// +// Given IsNullLiteralHelper(x), the compiler will pick the first +// version if x can be implicitly converted to Secret*, and pick the +// second version otherwise. Since Secret is a secret and incomplete +// type, the only expression a user can write that has type Secret* is +// a null pointer literal. Therefore, we know that x is a null +// pointer literal if and only if the first version is picked by the +// compiler. +char IsNullLiteralHelper(Secret* p); +char (&IsNullLiteralHelper(...))[2]; // NOLINT + +// A compile-time bool constant that is true if and only if x is a +// null pointer literal (i.e. NULL or any 0-valued compile-time +// integral constant). +#ifdef GTEST_ELLIPSIS_NEEDS_POD_ +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_IS_NULL_LITERAL_(x) false +#else +# define GTEST_IS_NULL_LITERAL_(x) \ + (sizeof(::testing::internal::IsNullLiteralHelper(x)) == 1) +#endif // GTEST_ELLIPSIS_NEEDS_POD_ + +// Appends the user-supplied message to the Google-Test-generated message. +GTEST_API_ std::string AppendUserMessage( + const std::string& gtest_msg, const Message& user_msg); + +#if GTEST_HAS_EXCEPTIONS + +// This exception is thrown by (and only by) a failed Google Test +// assertion when GTEST_FLAG(throw_on_failure) is true (if exceptions +// are enabled). We derive it from std::runtime_error, which is for +// errors presumably detectable only at run time. Since +// std::runtime_error inherits from std::exception, many testing +// frameworks know how to extract and print the message inside it. +class GTEST_API_ GoogleTestFailureException : public ::std::runtime_error { + public: + explicit GoogleTestFailureException(const TestPartResult& failure); +}; + +#endif // GTEST_HAS_EXCEPTIONS + +// A helper class for creating scoped traces in user programs. +class GTEST_API_ ScopedTrace { + public: + // The c'tor pushes the given source file location and message onto + // a trace stack maintained by Google Test. + ScopedTrace(const char* file, int line, const Message& message); + + // The d'tor pops the info pushed by the c'tor. + // + // Note that the d'tor is not virtual in order to be efficient. + // Don't inherit from ScopedTrace! + ~ScopedTrace(); + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedTrace); +} GTEST_ATTRIBUTE_UNUSED_; // A ScopedTrace object does its job in its + // c'tor and d'tor. Therefore it doesn't + // need to be used otherwise. + +namespace edit_distance { +// Returns the optimal edits to go from 'left' to 'right'. +// All edits cost the same, with replace having lower priority than +// add/remove. +// Simple implementation of the Wagner–Fischer algorithm. +// See http://en.wikipedia.org/wiki/Wagner-Fischer_algorithm +enum EditType { kMatch, kAdd, kRemove, kReplace }; +GTEST_API_ std::vector CalculateOptimalEdits( + const std::vector& left, const std::vector& right); + +// Same as above, but the input is represented as strings. +GTEST_API_ std::vector CalculateOptimalEdits( + const std::vector& left, + const std::vector& right); + +// Create a diff of the input strings in Unified diff format. +GTEST_API_ std::string CreateUnifiedDiff(const std::vector& left, + const std::vector& right, + size_t context = 2); + +} // namespace edit_distance + +// Calculate the diff between 'left' and 'right' and return it in unified diff +// format. +// If not null, stores in 'total_line_count' the total number of lines found +// in left + right. +GTEST_API_ std::string DiffStrings(const std::string& left, + const std::string& right, + size_t* total_line_count); + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// expected_expression: "foo" +// actual_expression: "bar" +// expected_value: "5" +// actual_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string " (ignoring case)" will +// be inserted into the message. +GTEST_API_ AssertionResult EqFailure(const char* expected_expression, + const char* actual_expression, + const std::string& expected_value, + const std::string& actual_value, + bool ignoring_case); + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +GTEST_API_ std::string GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value); + +// This template class represents an IEEE floating-point number +// (either single-precision or double-precision, depending on the +// template parameters). +// +// The purpose of this class is to do more sophisticated number +// comparison. (Due to round-off error, etc, it's very unlikely that +// two floating-points will be equal exactly. Hence a naive +// comparison by the == operation often doesn't work.) +// +// Format of IEEE floating-point: +// +// The most-significant bit being the leftmost, an IEEE +// floating-point looks like +// +// sign_bit exponent_bits fraction_bits +// +// Here, sign_bit is a single bit that designates the sign of the +// number. +// +// For float, there are 8 exponent bits and 23 fraction bits. +// +// For double, there are 11 exponent bits and 52 fraction bits. +// +// More details can be found at +// http://en.wikipedia.org/wiki/IEEE_floating-point_standard. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +template +class FloatingPoint { + public: + // Defines the unsigned integer type that has the same size as the + // floating point number. + typedef typename TypeWithSize::UInt Bits; + + // Constants. + + // # of bits in a number. + static const size_t kBitCount = 8*sizeof(RawType); + + // # of fraction bits in a number. + static const size_t kFractionBitCount = + std::numeric_limits::digits - 1; + + // # of exponent bits in a number. + static const size_t kExponentBitCount = kBitCount - 1 - kFractionBitCount; + + // The mask for the sign bit. + static const Bits kSignBitMask = static_cast(1) << (kBitCount - 1); + + // The mask for the fraction bits. + static const Bits kFractionBitMask = + ~static_cast(0) >> (kExponentBitCount + 1); + + // The mask for the exponent bits. + static const Bits kExponentBitMask = ~(kSignBitMask | kFractionBitMask); + + // How many ULP's (Units in the Last Place) we want to tolerate when + // comparing two numbers. The larger the value, the more error we + // allow. A 0 value means that two numbers must be exactly the same + // to be considered equal. + // + // The maximum error of a single floating-point operation is 0.5 + // units in the last place. On Intel CPU's, all floating-point + // calculations are done with 80-bit precision, while double has 64 + // bits. Therefore, 4 should be enough for ordinary use. + // + // See the following article for more details on ULP: + // http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ + static const size_t kMaxUlps = 4; + + // Constructs a FloatingPoint from a raw floating-point number. + // + // On an Intel CPU, passing a non-normalized NAN (Not a Number) + // around may change its bits, although the new value is guaranteed + // to be also a NAN. Therefore, don't expect this constructor to + // preserve the bits in x when x is a NAN. + explicit FloatingPoint(const RawType& x) { u_.value_ = x; } + + // Static methods + + // Reinterprets a bit pattern as a floating-point number. + // + // This function is needed to test the AlmostEquals() method. + static RawType ReinterpretBits(const Bits bits) { + FloatingPoint fp(0); + fp.u_.bits_ = bits; + return fp.u_.value_; + } + + // Returns the floating-point number that represent positive infinity. + static RawType Infinity() { + return ReinterpretBits(kExponentBitMask); + } + + // Returns the maximum representable finite floating-point number. + static RawType Max(); + + // Non-static methods + + // Returns the bits that represents this number. + const Bits &bits() const { return u_.bits_; } + + // Returns the exponent bits of this number. + Bits exponent_bits() const { return kExponentBitMask & u_.bits_; } + + // Returns the fraction bits of this number. + Bits fraction_bits() const { return kFractionBitMask & u_.bits_; } + + // Returns the sign bit of this number. + Bits sign_bit() const { return kSignBitMask & u_.bits_; } + + // Returns true iff this is NAN (not a number). + bool is_nan() const { + // It's a NAN if the exponent bits are all ones and the fraction + // bits are not entirely zeros. + return (exponent_bits() == kExponentBitMask) && (fraction_bits() != 0); + } + + // Returns true iff this number is at most kMaxUlps ULP's away from + // rhs. In particular, this function: + // + // - returns false if either number is (or both are) NAN. + // - treats really large numbers as almost equal to infinity. + // - thinks +0.0 and -0.0 are 0 DLP's apart. + bool AlmostEquals(const FloatingPoint& rhs) const { + // The IEEE standard says that any comparison operation involving + // a NAN must return false. + if (is_nan() || rhs.is_nan()) return false; + + return DistanceBetweenSignAndMagnitudeNumbers(u_.bits_, rhs.u_.bits_) + <= kMaxUlps; + } + + private: + // The data type used to store the actual floating-point number. + union FloatingPointUnion { + RawType value_; // The raw floating-point number. + Bits bits_; // The bits that represent the number. + }; + + // Converts an integer from the sign-and-magnitude representation to + // the biased representation. More precisely, let N be 2 to the + // power of (kBitCount - 1), an integer x is represented by the + // unsigned number x + N. + // + // For instance, + // + // -N + 1 (the most negative number representable using + // sign-and-magnitude) is represented by 1; + // 0 is represented by N; and + // N - 1 (the biggest number representable using + // sign-and-magnitude) is represented by 2N - 1. + // + // Read http://en.wikipedia.org/wiki/Signed_number_representations + // for more details on signed number representations. + static Bits SignAndMagnitudeToBiased(const Bits &sam) { + if (kSignBitMask & sam) { + // sam represents a negative number. + return ~sam + 1; + } else { + // sam represents a positive number. + return kSignBitMask | sam; + } + } + + // Given two numbers in the sign-and-magnitude representation, + // returns the distance between them as an unsigned number. + static Bits DistanceBetweenSignAndMagnitudeNumbers(const Bits &sam1, + const Bits &sam2) { + const Bits biased1 = SignAndMagnitudeToBiased(sam1); + const Bits biased2 = SignAndMagnitudeToBiased(sam2); + return (biased1 >= biased2) ? (biased1 - biased2) : (biased2 - biased1); + } + + FloatingPointUnion u_; +}; + +// We cannot use std::numeric_limits::max() as it clashes with the max() +// macro defined by . +template <> +inline float FloatingPoint::Max() { return FLT_MAX; } +template <> +inline double FloatingPoint::Max() { return DBL_MAX; } + +// Typedefs the instances of the FloatingPoint template class that we +// care to use. +typedef FloatingPoint Float; +typedef FloatingPoint Double; + +// In order to catch the mistake of putting tests that use different +// test fixture classes in the same test case, we need to assign +// unique IDs to fixture classes and compare them. The TypeId type is +// used to hold such IDs. The user should treat TypeId as an opaque +// type: the only operation allowed on TypeId values is to compare +// them for equality using the == operator. +typedef const void* TypeId; + +template +class TypeIdHelper { + public: + // dummy_ must not have a const type. Otherwise an overly eager + // compiler (e.g. MSVC 7.1 & 8.0) may try to merge + // TypeIdHelper::dummy_ for different Ts as an "optimization". + static bool dummy_; +}; + +template +bool TypeIdHelper::dummy_ = false; + +// GetTypeId() returns the ID of type T. Different values will be +// returned for different types. Calling the function twice with the +// same type argument is guaranteed to return the same ID. +template +TypeId GetTypeId() { + // The compiler is required to allocate a different + // TypeIdHelper::dummy_ variable for each T used to instantiate + // the template. Therefore, the address of dummy_ is guaranteed to + // be unique. + return &(TypeIdHelper::dummy_); +} + +// Returns the type ID of ::testing::Test. Always call this instead +// of GetTypeId< ::testing::Test>() to get the type ID of +// ::testing::Test, as the latter may give the wrong result due to a +// suspected linker bug when compiling Google Test as a Mac OS X +// framework. +GTEST_API_ TypeId GetTestTypeId(); + +// Defines the abstract factory interface that creates instances +// of a Test object. +class TestFactoryBase { + public: + virtual ~TestFactoryBase() {} + + // Creates a test instance to run. The instance is both created and destroyed + // within TestInfoImpl::Run() + virtual Test* CreateTest() = 0; + + protected: + TestFactoryBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestFactoryBase); +}; + +// This class provides implementation of TeastFactoryBase interface. +// It is used in TEST and TEST_F macros. +template +class TestFactoryImpl : public TestFactoryBase { + public: + virtual Test* CreateTest() { return new TestClass; } +}; + +#if GTEST_OS_WINDOWS + +// Predicate-formatters for implementing the HRESULT checking macros +// {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED} +// We pass a long instead of HRESULT to avoid causing an +// include dependency for the HRESULT type. +GTEST_API_ AssertionResult IsHRESULTSuccess(const char* expr, + long hr); // NOLINT +GTEST_API_ AssertionResult IsHRESULTFailure(const char* expr, + long hr); // NOLINT + +#endif // GTEST_OS_WINDOWS + +// Types of SetUpTestCase() and TearDownTestCase() functions. +typedef void (*SetUpTestCaseFunc)(); +typedef void (*TearDownTestCaseFunc)(); + +struct CodeLocation { + CodeLocation(const string& a_file, int a_line) : file(a_file), line(a_line) {} + + string file; + int line; +}; + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param text representation of the test's value parameter, +// or NULL if this is not a type-parameterized test. +// code_location: code location where the test is defined +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +GTEST_API_ TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + CodeLocation code_location, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory); + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +GTEST_API_ bool SkipPrefix(const char* prefix, const char** pstr); + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// State of the definition of a type-parameterized test case. +class GTEST_API_ TypedTestCasePState { + public: + TypedTestCasePState() : registered_(false) {} + + // Adds the given test name to defined_test_names_ and return true + // if the test case hasn't been registered; otherwise aborts the + // program. + bool AddTestName(const char* file, int line, const char* case_name, + const char* test_name) { + if (registered_) { + fprintf(stderr, "%s Test %s must be defined before " + "REGISTER_TYPED_TEST_CASE_P(%s, ...).\n", + FormatFileLocation(file, line).c_str(), test_name, case_name); + fflush(stderr); + posix::Abort(); + } + registered_tests_.insert( + ::std::make_pair(test_name, CodeLocation(file, line))); + return true; + } + + bool TestExists(const std::string& test_name) const { + return registered_tests_.count(test_name) > 0; + } + + const CodeLocation& GetCodeLocation(const std::string& test_name) const { + RegisteredTestsMap::const_iterator it = registered_tests_.find(test_name); + GTEST_CHECK_(it != registered_tests_.end()); + return it->second; + } + + // Verifies that registered_tests match the test names in + // defined_test_names_; returns registered_tests if successful, or + // aborts the program otherwise. + const char* VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests); + + private: + typedef ::std::map RegisteredTestsMap; + + bool registered_; + RegisteredTestsMap registered_tests_; +}; + +// Skips to the first non-space char after the first comma in 'str'; +// returns NULL if no comma is found in 'str'. +inline const char* SkipComma(const char* str) { + const char* comma = strchr(str, ','); + if (comma == NULL) { + return NULL; + } + while (IsSpace(*(++comma))) {} + return comma; +} + +// Returns the prefix of 'str' before the first comma in it; returns +// the entire string if it contains no comma. +inline std::string GetPrefixUntilComma(const char* str) { + const char* comma = strchr(str, ','); + return comma == NULL ? str : std::string(str, comma); +} + +// Splits a given string on a given delimiter, populating a given +// vector with the fields. +void SplitString(const ::std::string& str, char delimiter, + ::std::vector< ::std::string>* dest); + +// TypeParameterizedTest::Register() +// registers a list of type-parameterized tests with Google Test. The +// return value is insignificant - we just need to return something +// such that we can call this function in a namespace scope. +// +// Implementation note: The GTEST_TEMPLATE_ macro declares a template +// template parameter. It's defined in gtest-type-util.h. +template +class TypeParameterizedTest { + public: + // 'index' is the index of the test in the type list 'Types' + // specified in INSTANTIATE_TYPED_TEST_CASE_P(Prefix, TestCase, + // Types). Valid values for 'index' are [0, N - 1] where N is the + // length of Types. + static bool Register(const char* prefix, + CodeLocation code_location, + const char* case_name, const char* test_names, + int index) { + typedef typename Types::Head Type; + typedef Fixture FixtureClass; + typedef typename GTEST_BIND_(TestSel, Type) TestClass; + + // First, registers the first type-parameterized test in the type + // list. + MakeAndRegisterTestInfo( + (std::string(prefix) + (prefix[0] == '\0' ? "" : "/") + case_name + "/" + + StreamableToString(index)).c_str(), + StripTrailingSpaces(GetPrefixUntilComma(test_names)).c_str(), + GetTypeName().c_str(), + NULL, // No value parameter. + code_location, + GetTypeId(), + TestClass::SetUpTestCase, + TestClass::TearDownTestCase, + new TestFactoryImpl); + + // Next, recurses (at compile time) with the tail of the type list. + return TypeParameterizedTest + ::Register(prefix, code_location, case_name, test_names, index + 1); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTest { + public: + static bool Register(const char* /*prefix*/, CodeLocation, + const char* /*case_name*/, const char* /*test_names*/, + int /*index*/) { + return true; + } +}; + +// TypeParameterizedTestCase::Register() +// registers *all combinations* of 'Tests' and 'Types' with Google +// Test. The return value is insignificant - we just need to return +// something such that we can call this function in a namespace scope. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* prefix, CodeLocation code_location, + const TypedTestCasePState* state, + const char* case_name, const char* test_names) { + std::string test_name = StripTrailingSpaces( + GetPrefixUntilComma(test_names)); + if (!state->TestExists(test_name)) { + fprintf(stderr, "Failed to get code location for test %s.%s at %s.", + case_name, test_name.c_str(), + FormatFileLocation(code_location.file.c_str(), + code_location.line).c_str()); + fflush(stderr); + posix::Abort(); + } + const CodeLocation& test_location = state->GetCodeLocation(test_name); + + typedef typename Tests::Head Head; + + // First, register the first test in 'Test' for each type in 'Types'. + TypeParameterizedTest::Register( + prefix, test_location, case_name, test_names, 0); + + // Next, recurses (at compile time) with the tail of the test list. + return TypeParameterizedTestCase + ::Register(prefix, code_location, state, + case_name, SkipComma(test_names)); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* /*prefix*/, CodeLocation, + const TypedTestCasePState* /*state*/, + const char* /*case_name*/, const char* /*test_names*/) { + return true; + } +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +GTEST_API_ std::string GetCurrentOsStackTraceExceptTop( + UnitTest* unit_test, int skip_count); + +// Helpers for suppressing warnings on unreachable code or constant +// condition. + +// Always returns true. +GTEST_API_ bool AlwaysTrue(); + +// Always returns false. +inline bool AlwaysFalse() { return !AlwaysTrue(); } + +// Helper for suppressing false warning from Clang on a const char* +// variable declared in a conditional expression always being NULL in +// the else branch. +struct GTEST_API_ ConstCharPtr { + ConstCharPtr(const char* str) : value(str) {} + operator bool() const { return true; } + const char* value; +}; + +// A simple Linear Congruential Generator for generating random +// numbers with a uniform distribution. Unlike rand() and srand(), it +// doesn't use global state (and therefore can't interfere with user +// code). Unlike rand_r(), it's portable. An LCG isn't very random, +// but it's good enough for our purposes. +class GTEST_API_ Random { + public: + static const UInt32 kMaxRange = 1u << 31; + + explicit Random(UInt32 seed) : state_(seed) {} + + void Reseed(UInt32 seed) { state_ = seed; } + + // Generates a random number from [0, range). Crashes if 'range' is + // 0 or greater than kMaxRange. + UInt32 Generate(UInt32 range); + + private: + UInt32 state_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(Random); +}; + +// Defining a variable of type CompileAssertTypesEqual will cause a +// compiler error iff T1 and T2 are different types. +template +struct CompileAssertTypesEqual; + +template +struct CompileAssertTypesEqual { +}; + +// Removes the reference from a type if it is a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::remove_reference, which is not widely available yet. +template +struct RemoveReference { typedef T type; }; // NOLINT +template +struct RemoveReference { typedef T type; }; // NOLINT + +// A handy wrapper around RemoveReference that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_REFERENCE_(T) \ + typename ::testing::internal::RemoveReference::type + +// Removes const from a type if it is a const type, otherwise leaves +// it unchanged. This is the same as tr1::remove_const, which is not +// widely available yet. +template +struct RemoveConst { typedef T type; }; // NOLINT +template +struct RemoveConst { typedef T type; }; // NOLINT + +// MSVC 8.0, Sun C++, and IBM XL C++ have a bug which causes the above +// definition to fail to remove the const in 'const int[3]' and 'const +// char[3][4]'. The following specialization works around the bug. +template +struct RemoveConst { + typedef typename RemoveConst::type type[N]; +}; + +#if defined(_MSC_VER) && _MSC_VER < 1400 +// This is the only specialization that allows VC++ 7.1 to remove const in +// 'const int[3] and 'const int[3][4]'. However, it causes trouble with GCC +// and thus needs to be conditionally compiled. +template +struct RemoveConst { + typedef typename RemoveConst::type type[N]; +}; +#endif + +// A handy wrapper around RemoveConst that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_CONST_(T) \ + typename ::testing::internal::RemoveConst::type + +// Turns const U&, U&, const U, and U all into U. +#define GTEST_REMOVE_REFERENCE_AND_CONST_(T) \ + GTEST_REMOVE_CONST_(GTEST_REMOVE_REFERENCE_(T)) + +// Adds reference to a type if it is not a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::add_reference, which is not widely available yet. +template +struct AddReference { typedef T& type; }; // NOLINT +template +struct AddReference { typedef T& type; }; // NOLINT + +// A handy wrapper around AddReference that works when the argument T +// depends on template parameters. +#define GTEST_ADD_REFERENCE_(T) \ + typename ::testing::internal::AddReference::type + +// Adds a reference to const on top of T as necessary. For example, +// it transforms +// +// char ==> const char& +// const char ==> const char& +// char& ==> const char& +// const char& ==> const char& +// +// The argument T must depend on some template parameters. +#define GTEST_REFERENCE_TO_CONST_(T) \ + GTEST_ADD_REFERENCE_(const GTEST_REMOVE_REFERENCE_(T)) + +// ImplicitlyConvertible::value is a compile-time bool +// constant that's true iff type From can be implicitly converted to +// type To. +template +class ImplicitlyConvertible { + private: + // We need the following helper functions only for their types. + // They have no implementations. + + // MakeFrom() is an expression whose type is From. We cannot simply + // use From(), as the type From may not have a public default + // constructor. + static typename AddReference::type MakeFrom(); + + // These two functions are overloaded. Given an expression + // Helper(x), the compiler will pick the first version if x can be + // implicitly converted to type To; otherwise it will pick the + // second version. + // + // The first version returns a value of size 1, and the second + // version returns a value of size 2. Therefore, by checking the + // size of Helper(x), which can be done at compile time, we can tell + // which version of Helper() is used, and hence whether x can be + // implicitly converted to type To. + static char Helper(To); + static char (&Helper(...))[2]; // NOLINT + + // We have to put the 'public' section after the 'private' section, + // or MSVC refuses to compile the code. + public: +#if defined(__BORLANDC__) + // C++Builder cannot use member overload resolution during template + // instantiation. The simplest workaround is to use its C++0x type traits + // functions (C++Builder 2009 and above only). + static const bool value = __is_convertible(From, To); +#else + // MSVC warns about implicitly converting from double to int for + // possible loss of data, so we need to temporarily disable the + // warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4244) + static const bool value = + sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1; + GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif // __BORLANDC__ +}; +template +const bool ImplicitlyConvertible::value; + +// IsAProtocolMessage::value is a compile-time bool constant that's +// true iff T is type ProtocolMessage, proto2::Message, or a subclass +// of those. +template +struct IsAProtocolMessage + : public bool_constant< + ImplicitlyConvertible::value || + ImplicitlyConvertible::value> { +}; + +// When the compiler sees expression IsContainerTest(0), if C is an +// STL-style container class, the first overload of IsContainerTest +// will be viable (since both C::iterator* and C::const_iterator* are +// valid types and NULL can be implicitly converted to them). It will +// be picked over the second overload as 'int' is a perfect match for +// the type of argument 0. If C::iterator or C::const_iterator is not +// a valid type, the first overload is not viable, and the second +// overload will be picked. Therefore, we can determine whether C is +// a container class by checking the type of IsContainerTest(0). +// The value of the expression is insignificant. +// +// Note that we look for both C::iterator and C::const_iterator. The +// reason is that C++ injects the name of a class as a member of the +// class itself (e.g. you can refer to class iterator as either +// 'iterator' or 'iterator::iterator'). If we look for C::iterator +// only, for example, we would mistakenly think that a class named +// iterator is an STL container. +// +// Also note that the simpler approach of overloading +// IsContainerTest(typename C::const_iterator*) and +// IsContainerTest(...) doesn't work with Visual Age C++ and Sun C++. +typedef int IsContainer; +template +IsContainer IsContainerTest(int /* dummy */, + typename C::iterator* /* it */ = NULL, + typename C::const_iterator* /* const_it */ = NULL) { + return 0; +} + +typedef char IsNotContainer; +template +IsNotContainer IsContainerTest(long /* dummy */) { return '\0'; } + +// EnableIf::type is void when 'Cond' is true, and +// undefined when 'Cond' is false. To use SFINAE to make a function +// overload only apply when a particular expression is true, add +// "typename EnableIf::type* = 0" as the last parameter. +template struct EnableIf; +template<> struct EnableIf { typedef void type; }; // NOLINT + +// Utilities for native arrays. + +// ArrayEq() compares two k-dimensional native arrays using the +// elements' operator==, where k can be any integer >= 0. When k is +// 0, ArrayEq() degenerates into comparing a single pair of values. + +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs); + +// This generic version is used when k is 0. +template +inline bool ArrayEq(const T& lhs, const U& rhs) { return lhs == rhs; } + +// This overload is used when k >= 1. +template +inline bool ArrayEq(const T(&lhs)[N], const U(&rhs)[N]) { + return internal::ArrayEq(lhs, N, rhs); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous ArrayEq() function, arrays with different sizes would +// lead to different copies of the template code. +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs) { + for (size_t i = 0; i != size; i++) { + if (!internal::ArrayEq(lhs[i], rhs[i])) + return false; + } + return true; +} + +// Finds the first element in the iterator range [begin, end) that +// equals elem. Element may be a native array type itself. +template +Iter ArrayAwareFind(Iter begin, Iter end, const Element& elem) { + for (Iter it = begin; it != end; ++it) { + if (internal::ArrayEq(*it, elem)) + return it; + } + return end; +} + +// CopyArray() copies a k-dimensional native array using the elements' +// operator=, where k can be any integer >= 0. When k is 0, +// CopyArray() degenerates into copying a single value. + +template +void CopyArray(const T* from, size_t size, U* to); + +// This generic version is used when k is 0. +template +inline void CopyArray(const T& from, U* to) { *to = from; } + +// This overload is used when k >= 1. +template +inline void CopyArray(const T(&from)[N], U(*to)[N]) { + internal::CopyArray(from, N, *to); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous CopyArray() function, arrays with different sizes +// would lead to different copies of the template code. +template +void CopyArray(const T* from, size_t size, U* to) { + for (size_t i = 0; i != size; i++) { + internal::CopyArray(from[i], to + i); + } +} + +// The relation between an NativeArray object (see below) and the +// native array it represents. +// We use 2 different structs to allow non-copyable types to be used, as long +// as RelationToSourceReference() is passed. +struct RelationToSourceReference {}; +struct RelationToSourceCopy {}; + +// Adapts a native array to a read-only STL-style container. Instead +// of the complete STL container concept, this adaptor only implements +// members useful for Google Mock's container matchers. New members +// should be added as needed. To simplify the implementation, we only +// support Element being a raw type (i.e. having no top-level const or +// reference modifier). It's the client's responsibility to satisfy +// this requirement. Element can be an array type itself (hence +// multi-dimensional arrays are supported). +template +class NativeArray { + public: + // STL-style container typedefs. + typedef Element value_type; + typedef Element* iterator; + typedef const Element* const_iterator; + + // Constructs from a native array. References the source. + NativeArray(const Element* array, size_t count, RelationToSourceReference) { + InitRef(array, count); + } + + // Constructs from a native array. Copies the source. + NativeArray(const Element* array, size_t count, RelationToSourceCopy) { + InitCopy(array, count); + } + + // Copy constructor. + NativeArray(const NativeArray& rhs) { + (this->*rhs.clone_)(rhs.array_, rhs.size_); + } + + ~NativeArray() { + if (clone_ != &NativeArray::InitRef) + delete[] array_; + } + + // STL-style container methods. + size_t size() const { return size_; } + const_iterator begin() const { return array_; } + const_iterator end() const { return array_ + size_; } + bool operator==(const NativeArray& rhs) const { + return size() == rhs.size() && + ArrayEq(begin(), size(), rhs.begin()); + } + + private: + enum { + kCheckTypeIsNotConstOrAReference = StaticAssertTypeEqHelper< + Element, GTEST_REMOVE_REFERENCE_AND_CONST_(Element)>::value, + }; + + // Initializes this object with a copy of the input. + void InitCopy(const Element* array, size_t a_size) { + Element* const copy = new Element[a_size]; + CopyArray(array, a_size, copy); + array_ = copy; + size_ = a_size; + clone_ = &NativeArray::InitCopy; + } + + // Initializes this object with a reference of the input. + void InitRef(const Element* array, size_t a_size) { + array_ = array; + size_ = a_size; + clone_ = &NativeArray::InitRef; + } + + const Element* array_; + size_t size_; + void (NativeArray::*clone_)(const Element*, size_t); + + GTEST_DISALLOW_ASSIGN_(NativeArray); +}; + +} // namespace internal +} // namespace testing + +#define GTEST_MESSAGE_AT_(file, line, message, result_type) \ + ::testing::internal::AssertHelper(result_type, file, line, message) \ + = ::testing::Message() + +#define GTEST_MESSAGE_(message, result_type) \ + GTEST_MESSAGE_AT_(__FILE__, __LINE__, message, result_type) + +#define GTEST_FATAL_FAILURE_(message) \ + return GTEST_MESSAGE_(message, ::testing::TestPartResult::kFatalFailure) + +#define GTEST_NONFATAL_FAILURE_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kNonFatalFailure) + +#define GTEST_SUCCESS_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) + +// Suppresses MSVC warnings 4072 (unreachable code) for the code following +// statement if it returns or throws (or doesn't return or throw in some +// situations). +#define GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) \ + if (::testing::internal::AlwaysTrue()) { statement; } + +#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::ConstCharPtr gtest_msg = "") { \ + bool gtest_caught_expected = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (expected_exception const&) { \ + gtest_caught_expected = true; \ + } \ + catch (...) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws a different type."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + if (!gtest_caught_expected) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws nothing."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__): \ + fail(gtest_msg.value) + +#define GTEST_TEST_NO_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__): \ + fail("Expected: " #statement " doesn't throw an exception.\n" \ + " Actual: it throws.") + +#define GTEST_TEST_ANY_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + bool gtest_caught_any = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + gtest_caught_any = true; \ + } \ + if (!gtest_caught_any) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__): \ + fail("Expected: " #statement " throws an exception.\n" \ + " Actual: it doesn't.") + + +// Implements Boolean test assertions such as EXPECT_TRUE. expression can be +// either a boolean expression or an AssertionResult. text is a textual +// represenation of expression as it was passed into the EXPECT_TRUE. +#define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar_ = \ + ::testing::AssertionResult(expression)) \ + ; \ + else \ + fail(::testing::internal::GetBoolAssertionFailureMessage(\ + gtest_ar_, text, #actual, #expected).c_str()) + +#define GTEST_TEST_NO_FATAL_FAILURE_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + ::testing::internal::HasNewFatalFailureHelper gtest_fatal_failure_checker; \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + if (gtest_fatal_failure_checker.has_new_fatal_failure()) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__): \ + fail("Expected: " #statement " doesn't generate new fatal " \ + "failures in the current thread.\n" \ + " Actual: it does.") + +// Expands to the name of the class that implements the given test. +#define GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + test_case_name##_##test_name##_Test + +// Helper macro for defining tests. +#define GTEST_TEST_(test_case_name, test_name, parent_class, parent_id)\ +class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) : public parent_class {\ + public:\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {}\ + private:\ + virtual void TestBody();\ + static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_;\ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name));\ +};\ +\ +::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_case_name, test_name)\ + ::test_info_ =\ + ::testing::internal::MakeAndRegisterTestInfo(\ + #test_case_name, #test_name, NULL, NULL, \ + ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + (parent_id), \ + parent_class::SetUpTestCase, \ + parent_class::TearDownTestCase, \ + new ::testing::internal::TestFactoryImpl<\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>);\ +void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ + diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-linked_ptr.h b/mypyc/external/googletest/include/gtest/internal/gtest-linked_ptr.h new file mode 100644 index 000000000000..36029422174f --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-linked_ptr.h @@ -0,0 +1,243 @@ +// Copyright 2003 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: Dan Egnor (egnor@google.com) +// +// A "smart" pointer type with reference tracking. Every pointer to a +// particular object is kept on a circular linked list. When the last pointer +// to an object is destroyed or reassigned, the object is deleted. +// +// Used properly, this deletes the object when the last reference goes away. +// There are several caveats: +// - Like all reference counting schemes, cycles lead to leaks. +// - Each smart pointer is actually two pointers (8 bytes instead of 4). +// - Every time a pointer is assigned, the entire list of pointers to that +// object is traversed. This class is therefore NOT SUITABLE when there +// will often be more than two or three pointers to a particular object. +// - References are only tracked as long as linked_ptr<> objects are copied. +// If a linked_ptr<> is converted to a raw pointer and back, BAD THINGS +// will happen (double deletion). +// +// A good use of this class is storing object references in STL containers. +// You can safely put linked_ptr<> in a vector<>. +// Other uses may not be as good. +// +// Note: If you use an incomplete type with linked_ptr<>, the class +// *containing* linked_ptr<> must have a constructor and destructor (even +// if they do nothing!). +// +// Bill Gibbons suggested we use something like this. +// +// Thread Safety: +// Unlike other linked_ptr implementations, in this implementation +// a linked_ptr object is thread-safe in the sense that: +// - it's safe to copy linked_ptr objects concurrently, +// - it's safe to copy *from* a linked_ptr and read its underlying +// raw pointer (e.g. via get()) concurrently, and +// - it's safe to write to two linked_ptrs that point to the same +// shared object concurrently. +// TODO(wan@google.com): rename this to safe_linked_ptr to avoid +// confusion with normal linked_ptr. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ + +#include +#include + +#include "gtest/internal/gtest-port.h" + +namespace testing { +namespace internal { + +// Protects copying of all linked_ptr objects. +GTEST_API_ GTEST_DECLARE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// This is used internally by all instances of linked_ptr<>. It needs to be +// a non-template class because different types of linked_ptr<> can refer to +// the same object (linked_ptr(obj) vs linked_ptr(obj)). +// So, it needs to be possible for different types of linked_ptr to participate +// in the same circular linked list, so we need a single class type here. +// +// DO NOT USE THIS CLASS DIRECTLY YOURSELF. Use linked_ptr. +class linked_ptr_internal { + public: + // Create a new circle that includes only this instance. + void join_new() { + next_ = this; + } + + // Many linked_ptr operations may change p.link_ for some linked_ptr + // variable p in the same circle as this object. Therefore we need + // to prevent two such operations from occurring concurrently. + // + // Note that different types of linked_ptr objects can coexist in a + // circle (e.g. linked_ptr, linked_ptr, and + // linked_ptr). Therefore we must use a single mutex to + // protect all linked_ptr objects. This can create serious + // contention in production code, but is acceptable in a testing + // framework. + + // Join an existing circle. + void join(linked_ptr_internal const* ptr) + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { + MutexLock lock(&g_linked_ptr_mutex); + + linked_ptr_internal const* p = ptr; + while (p->next_ != ptr) { + assert(p->next_ != this && + "Trying to join() a linked ring we are already in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } + p->next_ = this; + next_ = ptr; + } + + // Leave whatever circle we're part of. Returns true if we were the + // last member of the circle. Once this is done, you can join() another. + bool depart() + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { + MutexLock lock(&g_linked_ptr_mutex); + + if (next_ == this) return true; + linked_ptr_internal const* p = next_; + while (p->next_ != this) { + assert(p->next_ != next_ && + "Trying to depart() a linked ring we are not in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } + p->next_ = next_; + return false; + } + + private: + mutable linked_ptr_internal const* next_; +}; + +template +class linked_ptr { + public: + typedef T element_type; + + // Take over ownership of a raw pointer. This should happen as soon as + // possible after the object is created. + explicit linked_ptr(T* ptr = NULL) { capture(ptr); } + ~linked_ptr() { depart(); } + + // Copy an existing linked_ptr<>, adding ourselves to the list of references. + template linked_ptr(linked_ptr const& ptr) { copy(&ptr); } + linked_ptr(linked_ptr const& ptr) { // NOLINT + assert(&ptr != this); + copy(&ptr); + } + + // Assignment releases the old value and acquires the new. + template linked_ptr& operator=(linked_ptr const& ptr) { + depart(); + copy(&ptr); + return *this; + } + + linked_ptr& operator=(linked_ptr const& ptr) { + if (&ptr != this) { + depart(); + copy(&ptr); + } + return *this; + } + + // Smart pointer members. + void reset(T* ptr = NULL) { + depart(); + capture(ptr); + } + T* get() const { return value_; } + T* operator->() const { return value_; } + T& operator*() const { return *value_; } + + bool operator==(T* p) const { return value_ == p; } + bool operator!=(T* p) const { return value_ != p; } + template + bool operator==(linked_ptr const& ptr) const { + return value_ == ptr.get(); + } + template + bool operator!=(linked_ptr const& ptr) const { + return value_ != ptr.get(); + } + + private: + template + friend class linked_ptr; + + T* value_; + linked_ptr_internal link_; + + void depart() { + if (link_.depart()) delete value_; + } + + void capture(T* ptr) { + value_ = ptr; + link_.join_new(); + } + + template void copy(linked_ptr const* ptr) { + value_ = ptr->get(); + if (value_) + link_.join(&ptr->link_); + else + link_.join_new(); + } +}; + +template inline +bool operator==(T* ptr, const linked_ptr& x) { + return ptr == x.get(); +} + +template inline +bool operator!=(T* ptr, const linked_ptr& x) { + return ptr != x.get(); +} + +// A function to convert T* into linked_ptr +// Doing e.g. make_linked_ptr(new FooBarBaz(arg)) is a shorter notation +// for linked_ptr >(new FooBarBaz(arg)) +template +linked_ptr make_linked_ptr(T* ptr) { + return linked_ptr(ptr); +} + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-param-util-generated.h b/mypyc/external/googletest/include/gtest/internal/gtest-param-util-generated.h new file mode 100644 index 000000000000..4d1d81d20ffc --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-param-util-generated.h @@ -0,0 +1,5146 @@ +// This file was GENERATED by command: +// pump.py gtest-param-util-generated.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently Google Test supports at most 50 arguments in Values, +// and at most 10 arguments in Combine. Please contact +// googletestframework@googlegroups.com if you need more. +// Please note that the number of arguments to Combine is limited +// by the maximum arity of the implementation of tuple which is +// currently set at 10. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-port.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Forward declarations of ValuesIn(), which is implemented in +// include/gtest/gtest-param-test.h. +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end); + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]); + +template +internal::ParamGenerator ValuesIn( + const Container& container); + +namespace internal { + +// Used in the Values() function to provide polymorphic capabilities. +template +class ValueArray1 { + public: + explicit ValueArray1(T1 v1) : v1_(v1) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray1& other); + + const T1 v1_; +}; + +template +class ValueArray2 { + public: + ValueArray2(T1 v1, T2 v2) : v1_(v1), v2_(v2) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray2& other); + + const T1 v1_; + const T2 v2_; +}; + +template +class ValueArray3 { + public: + ValueArray3(T1 v1, T2 v2, T3 v3) : v1_(v1), v2_(v2), v3_(v3) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray3& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; +}; + +template +class ValueArray4 { + public: + ValueArray4(T1 v1, T2 v2, T3 v3, T4 v4) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray4& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; +}; + +template +class ValueArray5 { + public: + ValueArray5(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray5& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; +}; + +template +class ValueArray6 { + public: + ValueArray6(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray6& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; +}; + +template +class ValueArray7 { + public: + ValueArray7(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray7& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; +}; + +template +class ValueArray8 { + public: + ValueArray8(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray8& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; +}; + +template +class ValueArray9 { + public: + ValueArray9(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray9& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; +}; + +template +class ValueArray10 { + public: + ValueArray10(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray10& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; +}; + +template +class ValueArray11 { + public: + ValueArray11(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray11& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; +}; + +template +class ValueArray12 { + public: + ValueArray12(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray12& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; +}; + +template +class ValueArray13 { + public: + ValueArray13(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray13& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; +}; + +template +class ValueArray14 { + public: + ValueArray14(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray14& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; +}; + +template +class ValueArray15 { + public: + ValueArray15(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray15& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; +}; + +template +class ValueArray16 { + public: + ValueArray16(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray16& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; +}; + +template +class ValueArray17 { + public: + ValueArray17(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray17& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; +}; + +template +class ValueArray18 { + public: + ValueArray18(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray18& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; +}; + +template +class ValueArray19 { + public: + ValueArray19(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray19& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; +}; + +template +class ValueArray20 { + public: + ValueArray20(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray20& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; +}; + +template +class ValueArray21 { + public: + ValueArray21(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray21& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; +}; + +template +class ValueArray22 { + public: + ValueArray22(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray22& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; +}; + +template +class ValueArray23 { + public: + ValueArray23(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray23& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; +}; + +template +class ValueArray24 { + public: + ValueArray24(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray24& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; +}; + +template +class ValueArray25 { + public: + ValueArray25(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray25& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; +}; + +template +class ValueArray26 { + public: + ValueArray26(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray26& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; +}; + +template +class ValueArray27 { + public: + ValueArray27(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray27& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; +}; + +template +class ValueArray28 { + public: + ValueArray28(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray28& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; +}; + +template +class ValueArray29 { + public: + ValueArray29(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray29& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; +}; + +template +class ValueArray30 { + public: + ValueArray30(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray30& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; +}; + +template +class ValueArray31 { + public: + ValueArray31(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray31& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; +}; + +template +class ValueArray32 { + public: + ValueArray32(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray32& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; +}; + +template +class ValueArray33 { + public: + ValueArray33(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray33& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; +}; + +template +class ValueArray34 { + public: + ValueArray34(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray34& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; +}; + +template +class ValueArray35 { + public: + ValueArray35(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray35& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; +}; + +template +class ValueArray36 { + public: + ValueArray36(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray36& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; +}; + +template +class ValueArray37 { + public: + ValueArray37(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray37& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; +}; + +template +class ValueArray38 { + public: + ValueArray38(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray38& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; +}; + +template +class ValueArray39 { + public: + ValueArray39(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray39& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; +}; + +template +class ValueArray40 { + public: + ValueArray40(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray40& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; +}; + +template +class ValueArray41 { + public: + ValueArray41(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray41& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; +}; + +template +class ValueArray42 { + public: + ValueArray42(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray42& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; +}; + +template +class ValueArray43 { + public: + ValueArray43(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), + v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray43& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; +}; + +template +class ValueArray44 { + public: + ValueArray44(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), + v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), + v43_(v43), v44_(v44) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray44& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; +}; + +template +class ValueArray45 { + public: + ValueArray45(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), + v42_(v42), v43_(v43), v44_(v44), v45_(v45) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray45& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; +}; + +template +class ValueArray46 { + public: + ValueArray46(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray46& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; +}; + +template +class ValueArray47 { + public: + ValueArray47(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46), + v47_(v47) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray47& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; +}; + +template +class ValueArray48 { + public: + ValueArray48(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), + v46_(v46), v47_(v47), v48_(v48) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray48& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; +}; + +template +class ValueArray49 { + public: + ValueArray49(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, + T49 v49) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_), static_cast(v49_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray49& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; +}; + +template +class ValueArray50 { + public: + ValueArray50(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, T49 v49, + T50 v50) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49), v50_(v50) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_), static_cast(v49_), static_cast(v50_)}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray50& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; + const T50 v50_; +}; + +# if GTEST_HAS_COMBINE +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Generates values from the Cartesian product of values produced +// by the argument generators. +// +template +class CartesianProductGenerator2 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator2(const ParamGenerator& g1, + const ParamGenerator& g2) + : g1_(g1), g2_(g2) {} + virtual ~CartesianProductGenerator2() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current2_; + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + ParamType current_value_; + }; // class CartesianProductGenerator2::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator2& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; +}; // class CartesianProductGenerator2 + + +template +class CartesianProductGenerator3 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator3(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + virtual ~CartesianProductGenerator3() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current3_; + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + ParamType current_value_; + }; // class CartesianProductGenerator3::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator3& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; +}; // class CartesianProductGenerator3 + + +template +class CartesianProductGenerator4 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator4(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + virtual ~CartesianProductGenerator4() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current4_; + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + ParamType current_value_; + }; // class CartesianProductGenerator4::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator4& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; +}; // class CartesianProductGenerator4 + + +template +class CartesianProductGenerator5 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator5(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + virtual ~CartesianProductGenerator5() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current5_; + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + ParamType current_value_; + }; // class CartesianProductGenerator5::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator5& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; +}; // class CartesianProductGenerator5 + + +template +class CartesianProductGenerator6 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator6(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + virtual ~CartesianProductGenerator6() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current6_; + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + ParamType current_value_; + }; // class CartesianProductGenerator6::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator6& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; +}; // class CartesianProductGenerator6 + + +template +class CartesianProductGenerator7 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator7(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + virtual ~CartesianProductGenerator7() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current7_; + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + ParamType current_value_; + }; // class CartesianProductGenerator7::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator7& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; +}; // class CartesianProductGenerator7 + + +template +class CartesianProductGenerator8 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator8(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + virtual ~CartesianProductGenerator8() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current8_; + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + ParamType current_value_; + }; // class CartesianProductGenerator8::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator8& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; +}; // class CartesianProductGenerator8 + + +template +class CartesianProductGenerator9 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator9(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + virtual ~CartesianProductGenerator9() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current9_; + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + ParamType current_value_; + }; // class CartesianProductGenerator9::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator9& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; +}; // class CartesianProductGenerator9 + + +template +class CartesianProductGenerator10 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator10(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9, + const ParamGenerator& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + virtual ~CartesianProductGenerator10() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin(), g10_, g10_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end(), g10_, g10_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9, + const ParamGenerator& g10, + const typename ParamGenerator::iterator& current10) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9), + begin10_(g10.begin()), end10_(g10.end()), current10_(current10) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current10_; + if (current10_ == end10_) { + current10_ = begin10_; + ++current9_; + } + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_ && + current10_ == typed_other->current10_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_), + begin10_(other.begin10_), + end10_(other.end10_), + current10_(other.current10_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_, *current10_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_ || + current10_ == end10_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + const typename ParamGenerator::iterator begin10_; + const typename ParamGenerator::iterator end10_; + typename ParamGenerator::iterator current10_; + ParamType current_value_; + }; // class CartesianProductGenerator10::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator10& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; + const ParamGenerator g10_; +}; // class CartesianProductGenerator10 + + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Helper classes providing Combine() with polymorphic features. They allow +// casting CartesianProductGeneratorN to ParamGenerator if T is +// convertible to U. +// +template +class CartesianProductHolder2 { + public: +CartesianProductHolder2(const Generator1& g1, const Generator2& g2) + : g1_(g1), g2_(g2) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator2( + static_cast >(g1_), + static_cast >(g2_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder2& other); + + const Generator1 g1_; + const Generator2 g2_; +}; // class CartesianProductHolder2 + +template +class CartesianProductHolder3 { + public: +CartesianProductHolder3(const Generator1& g1, const Generator2& g2, + const Generator3& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator3( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder3& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; +}; // class CartesianProductHolder3 + +template +class CartesianProductHolder4 { + public: +CartesianProductHolder4(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator4( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder4& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; +}; // class CartesianProductHolder4 + +template +class CartesianProductHolder5 { + public: +CartesianProductHolder5(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator5( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder5& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; +}; // class CartesianProductHolder5 + +template +class CartesianProductHolder6 { + public: +CartesianProductHolder6(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator6( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder6& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; +}; // class CartesianProductHolder6 + +template +class CartesianProductHolder7 { + public: +CartesianProductHolder7(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator7( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder7& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; +}; // class CartesianProductHolder7 + +template +class CartesianProductHolder8 { + public: +CartesianProductHolder8(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator8( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder8& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; +}; // class CartesianProductHolder8 + +template +class CartesianProductHolder9 { + public: +CartesianProductHolder9(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator9( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder9& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; +}; // class CartesianProductHolder9 + +template +class CartesianProductHolder10 { + public: +CartesianProductHolder10(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9, const Generator10& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator10( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_), + static_cast >(g10_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder10& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; + const Generator10 g10_; +}; // class CartesianProductHolder10 + +# endif // GTEST_HAS_COMBINE + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-param-util-generated.h.pump b/mypyc/external/googletest/include/gtest/internal/gtest-param-util-generated.h.pump new file mode 100644 index 000000000000..5c7c47af0bb1 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-param-util-generated.h.pump @@ -0,0 +1,286 @@ +$$ -*- mode: c++; -*- +$var n = 50 $$ Maximum length of Values arguments we want to support. +$var maxtuple = 10 $$ Maximum number of Combine arguments we want to support. +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently Google Test supports at most $n arguments in Values, +// and at most $maxtuple arguments in Combine. Please contact +// googletestframework@googlegroups.com if you need more. +// Please note that the number of arguments to Combine is limited +// by the maximum arity of the implementation of tuple which is +// currently set at $maxtuple. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-param-util.h" +#include "gtest/internal/gtest-port.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Forward declarations of ValuesIn(), which is implemented in +// include/gtest/gtest-param-test.h. +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end); + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]); + +template +internal::ParamGenerator ValuesIn( + const Container& container); + +namespace internal { + +// Used in the Values() function to provide polymorphic capabilities. +$range i 1..n +$for i [[ +$range j 1..i + +template <$for j, [[typename T$j]]> +class ValueArray$i { + public: + $if i==1 [[explicit ]]ValueArray$i($for j, [[T$j v$j]]) : $for j, [[v$(j)_(v$j)]] {} + + template + operator ParamGenerator() const { + const T array[] = {$for j, [[static_cast(v$(j)_)]]}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray$i& other); + +$for j [[ + + const T$j v$(j)_; +]] + +}; + +]] + +# if GTEST_HAS_COMBINE +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Generates values from the Cartesian product of values produced +// by the argument generators. +// +$range i 2..maxtuple +$for i [[ +$range j 1..i +$range k 2..i + +template <$for j, [[typename T$j]]> +class CartesianProductGenerator$i + : public ParamGeneratorInterface< ::testing::tuple<$for j, [[T$j]]> > { + public: + typedef ::testing::tuple<$for j, [[T$j]]> ParamType; + + CartesianProductGenerator$i($for j, [[const ParamGenerator& g$j]]) + : $for j, [[g$(j)_(g$j)]] {} + virtual ~CartesianProductGenerator$i() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, $for j, [[g$(j)_, g$(j)_.begin()]]); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, $for j, [[g$(j)_, g$(j)_.end()]]); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, $for j, [[ + + const ParamGenerator& g$j, + const typename ParamGenerator::iterator& current$(j)]]) + : base_(base), +$for j, [[ + + begin$(j)_(g$j.begin()), end$(j)_(g$j.end()), current$(j)_(current$j) +]] { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current$(i)_; + +$for k [[ + if (current$(i+2-k)_ == end$(i+2-k)_) { + current$(i+2-k)_ = begin$(i+2-k)_; + ++current$(i+2-k-1)_; + } + +]] + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ($for j && [[ + + current$(j)_ == typed_other->current$(j)_ +]]); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), $for j, [[ + + begin$(j)_(other.begin$(j)_), + end$(j)_(other.end$(j)_), + current$(j)_(other.current$(j)_) +]] { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType($for j, [[*current$(j)_]]); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return +$for j || [[ + + current$(j)_ == end$(j)_ +]]; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. +$for j [[ + + const typename ParamGenerator::iterator begin$(j)_; + const typename ParamGenerator::iterator end$(j)_; + typename ParamGenerator::iterator current$(j)_; +]] + + ParamType current_value_; + }; // class CartesianProductGenerator$i::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator$i& other); + + +$for j [[ + const ParamGenerator g$(j)_; + +]] +}; // class CartesianProductGenerator$i + + +]] + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Helper classes providing Combine() with polymorphic features. They allow +// casting CartesianProductGeneratorN to ParamGenerator if T is +// convertible to U. +// +$range i 2..maxtuple +$for i [[ +$range j 1..i + +template <$for j, [[class Generator$j]]> +class CartesianProductHolder$i { + public: +CartesianProductHolder$i($for j, [[const Generator$j& g$j]]) + : $for j, [[g$(j)_(g$j)]] {} + template <$for j, [[typename T$j]]> + operator ParamGenerator< ::testing::tuple<$for j, [[T$j]]> >() const { + return ParamGenerator< ::testing::tuple<$for j, [[T$j]]> >( + new CartesianProductGenerator$i<$for j, [[T$j]]>( +$for j,[[ + + static_cast >(g$(j)_) +]])); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder$i& other); + + +$for j [[ + const Generator$j g$(j)_; + +]] +}; // class CartesianProductHolder$i + +]] + +# endif // GTEST_HAS_COMBINE + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-param-util.h b/mypyc/external/googletest/include/gtest/internal/gtest-param-util.h new file mode 100644 index 000000000000..82cab9b0201b --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-param-util.h @@ -0,0 +1,731 @@ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ + +#include + +#include +#include +#include +#include + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-linked_ptr.h" +#include "gtest/internal/gtest-port.h" +#include "gtest/gtest-printers.h" + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Input to a parameterized test name generator, describing a test parameter. +// Consists of the parameter value and the integer parameter index. +template +struct TestParamInfo { + TestParamInfo(const ParamType& a_param, size_t an_index) : + param(a_param), + index(an_index) {} + ParamType param; + size_t index; +}; + +// A builtin parameterized test name generator which returns the result of +// testing::PrintToString. +struct PrintToStringParamName { + template + std::string operator()(const TestParamInfo& info) const { + return PrintToString(info.param); + } +}; + +namespace internal { + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Outputs a message explaining invalid registration of different +// fixture class for the same test case. This may happen when +// TEST_P macro is used to define two tests with the same name +// but in different namespaces. +GTEST_API_ void ReportInvalidTestCaseType(const char* test_case_name, + CodeLocation code_location); + +template class ParamGeneratorInterface; +template class ParamGenerator; + +// Interface for iterating over elements provided by an implementation +// of ParamGeneratorInterface. +template +class ParamIteratorInterface { + public: + virtual ~ParamIteratorInterface() {} + // A pointer to the base generator instance. + // Used only for the purposes of iterator comparison + // to make sure that two iterators belong to the same generator. + virtual const ParamGeneratorInterface* BaseGenerator() const = 0; + // Advances iterator to point to the next element + // provided by the generator. The caller is responsible + // for not calling Advance() on an iterator equal to + // BaseGenerator()->End(). + virtual void Advance() = 0; + // Clones the iterator object. Used for implementing copy semantics + // of ParamIterator. + virtual ParamIteratorInterface* Clone() const = 0; + // Dereferences the current iterator and provides (read-only) access + // to the pointed value. It is the caller's responsibility not to call + // Current() on an iterator equal to BaseGenerator()->End(). + // Used for implementing ParamGenerator::operator*(). + virtual const T* Current() const = 0; + // Determines whether the given iterator and other point to the same + // element in the sequence generated by the generator. + // Used for implementing ParamGenerator::operator==(). + virtual bool Equals(const ParamIteratorInterface& other) const = 0; +}; + +// Class iterating over elements provided by an implementation of +// ParamGeneratorInterface. It wraps ParamIteratorInterface +// and implements the const forward iterator concept. +template +class ParamIterator { + public: + typedef T value_type; + typedef const T& reference; + typedef ptrdiff_t difference_type; + + // ParamIterator assumes ownership of the impl_ pointer. + ParamIterator(const ParamIterator& other) : impl_(other.impl_->Clone()) {} + ParamIterator& operator=(const ParamIterator& other) { + if (this != &other) + impl_.reset(other.impl_->Clone()); + return *this; + } + + const T& operator*() const { return *impl_->Current(); } + const T* operator->() const { return impl_->Current(); } + // Prefix version of operator++. + ParamIterator& operator++() { + impl_->Advance(); + return *this; + } + // Postfix version of operator++. + ParamIterator operator++(int /*unused*/) { + ParamIteratorInterface* clone = impl_->Clone(); + impl_->Advance(); + return ParamIterator(clone); + } + bool operator==(const ParamIterator& other) const { + return impl_.get() == other.impl_.get() || impl_->Equals(*other.impl_); + } + bool operator!=(const ParamIterator& other) const { + return !(*this == other); + } + + private: + friend class ParamGenerator; + explicit ParamIterator(ParamIteratorInterface* impl) : impl_(impl) {} + scoped_ptr > impl_; +}; + +// ParamGeneratorInterface is the binary interface to access generators +// defined in other translation units. +template +class ParamGeneratorInterface { + public: + typedef T ParamType; + + virtual ~ParamGeneratorInterface() {} + + // Generator interface definition + virtual ParamIteratorInterface* Begin() const = 0; + virtual ParamIteratorInterface* End() const = 0; +}; + +// Wraps ParamGeneratorInterface and provides general generator syntax +// compatible with the STL Container concept. +// This class implements copy initialization semantics and the contained +// ParamGeneratorInterface instance is shared among all copies +// of the original object. This is possible because that instance is immutable. +template +class ParamGenerator { + public: + typedef ParamIterator iterator; + + explicit ParamGenerator(ParamGeneratorInterface* impl) : impl_(impl) {} + ParamGenerator(const ParamGenerator& other) : impl_(other.impl_) {} + + ParamGenerator& operator=(const ParamGenerator& other) { + impl_ = other.impl_; + return *this; + } + + iterator begin() const { return iterator(impl_->Begin()); } + iterator end() const { return iterator(impl_->End()); } + + private: + linked_ptr > impl_; +}; + +// Generates values from a range of two comparable values. Can be used to +// generate sequences of user-defined types that implement operator+() and +// operator<(). +// This class is used in the Range() function. +template +class RangeGenerator : public ParamGeneratorInterface { + public: + RangeGenerator(T begin, T end, IncrementT step) + : begin_(begin), end_(end), + step_(step), end_index_(CalculateEndIndex(begin, end, step)) {} + virtual ~RangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, begin_, 0, step_); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, end_, end_index_, step_); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, T value, int index, + IncrementT step) + : base_(base), value_(value), index_(index), step_(step) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + value_ = static_cast(value_ + step_); + index_++; + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const T* Current() const { return &value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const int other_index = + CheckedDowncastToActualType(&other)->index_; + return index_ == other_index; + } + + private: + Iterator(const Iterator& other) + : ParamIteratorInterface(), + base_(other.base_), value_(other.value_), index_(other.index_), + step_(other.step_) {} + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + T value_; + int index_; + const IncrementT step_; + }; // class RangeGenerator::Iterator + + static int CalculateEndIndex(const T& begin, + const T& end, + const IncrementT& step) { + int end_index = 0; + for (T i = begin; i < end; i = static_cast(i + step)) + end_index++; + return end_index; + } + + // No implementation - assignment is unsupported. + void operator=(const RangeGenerator& other); + + const T begin_; + const T end_; + const IncrementT step_; + // The index for the end() iterator. All the elements in the generated + // sequence are indexed (0-based) to aid iterator comparison. + const int end_index_; +}; // class RangeGenerator + + +// Generates values from a pair of STL-style iterators. Used in the +// ValuesIn() function. The elements are copied from the source range +// since the source can be located on the stack, and the generator +// is likely to persist beyond that stack frame. +template +class ValuesInIteratorRangeGenerator : public ParamGeneratorInterface { + public: + template + ValuesInIteratorRangeGenerator(ForwardIterator begin, ForwardIterator end) + : container_(begin, end) {} + virtual ~ValuesInIteratorRangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, container_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, container_.end()); + } + + private: + typedef typename ::std::vector ContainerType; + + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + typename ContainerType::const_iterator iterator) + : base_(base), iterator_(iterator) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + ++iterator_; + value_.reset(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + // We need to use cached value referenced by iterator_ because *iterator_ + // can return a temporary object (and of type other then T), so just + // having "return &*iterator_;" doesn't work. + // value_ is updated here and not in Advance() because Advance() + // can advance iterator_ beyond the end of the range, and we cannot + // detect that fact. The client code, on the other hand, is + // responsible for not calling Current() on an out-of-range iterator. + virtual const T* Current() const { + if (value_.get() == NULL) + value_.reset(new T(*iterator_)); + return value_.get(); + } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + return iterator_ == + CheckedDowncastToActualType(&other)->iterator_; + } + + private: + Iterator(const Iterator& other) + // The explicit constructor call suppresses a false warning + // emitted by gcc when supplied with the -Wextra option. + : ParamIteratorInterface(), + base_(other.base_), + iterator_(other.iterator_) {} + + const ParamGeneratorInterface* const base_; + typename ContainerType::const_iterator iterator_; + // A cached value of *iterator_. We keep it here to allow access by + // pointer in the wrapping iterator's operator->(). + // value_ needs to be mutable to be accessed in Current(). + // Use of scoped_ptr helps manage cached value's lifetime, + // which is bound by the lifespan of the iterator itself. + mutable scoped_ptr value_; + }; // class ValuesInIteratorRangeGenerator::Iterator + + // No implementation - assignment is unsupported. + void operator=(const ValuesInIteratorRangeGenerator& other); + + const ContainerType container_; +}; // class ValuesInIteratorRangeGenerator + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Default parameterized test name generator, returns a string containing the +// integer test parameter index. +template +std::string DefaultParamName(const TestParamInfo& info) { + Message name_stream; + name_stream << info.index; + return name_stream.GetString(); +} + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Parameterized test name overload helpers, which help the +// INSTANTIATE_TEST_CASE_P macro choose between the default parameterized +// test name generator and user param name generator. +template +ParamNameGenFunctor GetParamNameGen(ParamNameGenFunctor func) { + return func; +} + +template +struct ParamNameGenFunc { + typedef std::string Type(const TestParamInfo&); +}; + +template +typename ParamNameGenFunc::Type *GetParamNameGen() { + return DefaultParamName; +} + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Stores a parameter value and later creates tests parameterized with that +// value. +template +class ParameterizedTestFactory : public TestFactoryBase { + public: + typedef typename TestClass::ParamType ParamType; + explicit ParameterizedTestFactory(ParamType parameter) : + parameter_(parameter) {} + virtual Test* CreateTest() { + TestClass::SetParam(¶meter_); + return new TestClass(); + } + + private: + const ParamType parameter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactoryBase is a base class for meta-factories that create +// test factories for passing into MakeAndRegisterTestInfo function. +template +class TestMetaFactoryBase { + public: + virtual ~TestMetaFactoryBase() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) = 0; +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactory creates test factories for passing into +// MakeAndRegisterTestInfo function. Since MakeAndRegisterTestInfo receives +// ownership of test factory pointer, same factory object cannot be passed +// into that method twice. But ParameterizedTestCaseInfo is going to call +// it for each Test/Parameter value combination. Thus it needs meta factory +// creator class. +template +class TestMetaFactory + : public TestMetaFactoryBase { + public: + typedef typename TestCase::ParamType ParamType; + + TestMetaFactory() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) { + return new ParameterizedTestFactory(parameter); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestMetaFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfoBase is a generic interface +// to ParameterizedTestCaseInfo classes. ParameterizedTestCaseInfoBase +// accumulates test information provided by TEST_P macro invocations +// and generators provided by INSTANTIATE_TEST_CASE_P macro invocations +// and uses that information to register all resulting test instances +// in RegisterTests method. The ParameterizeTestCaseRegistry class holds +// a collection of pointers to the ParameterizedTestCaseInfo objects +// and calls RegisterTests() on each of them when asked. +class ParameterizedTestCaseInfoBase { + public: + virtual ~ParameterizedTestCaseInfoBase() {} + + // Base part of test case name for display purposes. + virtual const string& GetTestCaseName() const = 0; + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const = 0; + // UnitTest class invokes this method to register tests in this + // test case right before running them in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + virtual void RegisterTests() = 0; + + protected: + ParameterizedTestCaseInfoBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfoBase); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfo accumulates tests obtained from TEST_P +// macro invocations for a particular test case and generators +// obtained from INSTANTIATE_TEST_CASE_P macro invocations for that +// test case. It registers tests with all values generated by all +// generators when asked. +template +class ParameterizedTestCaseInfo : public ParameterizedTestCaseInfoBase { + public: + // ParamType and GeneratorCreationFunc are private types but are required + // for declarations of public methods AddTestPattern() and + // AddTestCaseInstantiation(). + typedef typename TestCase::ParamType ParamType; + // A function that returns an instance of appropriate generator type. + typedef ParamGenerator(GeneratorCreationFunc)(); + typedef typename ParamNameGenFunc::Type ParamNameGeneratorFunc; + + explicit ParameterizedTestCaseInfo( + const char* name, CodeLocation code_location) + : test_case_name_(name), code_location_(code_location) {} + + // Test case base name for display purposes. + virtual const string& GetTestCaseName() const { return test_case_name_; } + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const { return GetTypeId(); } + // TEST_P macro uses AddTestPattern() to record information + // about a single test in a LocalTestInfo structure. + // test_case_name is the base name of the test case (without invocation + // prefix). test_base_name is the name of an individual test without + // parameter index. For the test SequenceA/FooTest.DoBar/1 FooTest is + // test case base name and DoBar is test base name. + void AddTestPattern(const char* test_case_name, + const char* test_base_name, + TestMetaFactoryBase* meta_factory) { + tests_.push_back(linked_ptr(new TestInfo(test_case_name, + test_base_name, + meta_factory))); + } + // INSTANTIATE_TEST_CASE_P macro uses AddGenerator() to record information + // about a generator. + int AddTestCaseInstantiation(const string& instantiation_name, + GeneratorCreationFunc* func, + ParamNameGeneratorFunc* name_func, + const char* file, + int line) { + instantiations_.push_back( + InstantiationInfo(instantiation_name, func, name_func, file, line)); + return 0; // Return value used only to run this method in namespace scope. + } + // UnitTest class invokes this method to register tests in this test case + // test cases right before running tests in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + // UnitTest has a guard to prevent from calling this method more then once. + virtual void RegisterTests() { + for (typename TestInfoContainer::iterator test_it = tests_.begin(); + test_it != tests_.end(); ++test_it) { + linked_ptr test_info = *test_it; + for (typename InstantiationContainer::iterator gen_it = + instantiations_.begin(); gen_it != instantiations_.end(); + ++gen_it) { + const string& instantiation_name = gen_it->name; + ParamGenerator generator((*gen_it->generator)()); + ParamNameGeneratorFunc* name_func = gen_it->name_func; + const char* file = gen_it->file; + int line = gen_it->line; + + string test_case_name; + if ( !instantiation_name.empty() ) + test_case_name = instantiation_name + "/"; + test_case_name += test_info->test_case_base_name; + + size_t i = 0; + std::set test_param_names; + for (typename ParamGenerator::iterator param_it = + generator.begin(); + param_it != generator.end(); ++param_it, ++i) { + Message test_name_stream; + + std::string param_name = name_func( + TestParamInfo(*param_it, i)); + + GTEST_CHECK_(IsValidParamName(param_name)) + << "Parameterized test name '" << param_name + << "' is invalid, in " << file + << " line " << line << std::endl; + + GTEST_CHECK_(test_param_names.count(param_name) == 0) + << "Duplicate parameterized test name '" << param_name + << "', in " << file << " line " << line << std::endl; + + test_param_names.insert(param_name); + + test_name_stream << test_info->test_base_name << "/" << param_name; + MakeAndRegisterTestInfo( + test_case_name.c_str(), + test_name_stream.GetString().c_str(), + NULL, // No type parameter. + PrintToString(*param_it).c_str(), + code_location_, + GetTestCaseTypeId(), + TestCase::SetUpTestCase, + TestCase::TearDownTestCase, + test_info->test_meta_factory->CreateTestFactory(*param_it)); + } // for param_it + } // for gen_it + } // for test_it + } // RegisterTests + + private: + // LocalTestInfo structure keeps information about a single test registered + // with TEST_P macro. + struct TestInfo { + TestInfo(const char* a_test_case_base_name, + const char* a_test_base_name, + TestMetaFactoryBase* a_test_meta_factory) : + test_case_base_name(a_test_case_base_name), + test_base_name(a_test_base_name), + test_meta_factory(a_test_meta_factory) {} + + const string test_case_base_name; + const string test_base_name; + const scoped_ptr > test_meta_factory; + }; + typedef ::std::vector > TestInfoContainer; + // Records data received from INSTANTIATE_TEST_CASE_P macros: + // + struct InstantiationInfo { + InstantiationInfo(const std::string &name_in, + GeneratorCreationFunc* generator_in, + ParamNameGeneratorFunc* name_func_in, + const char* file_in, + int line_in) + : name(name_in), + generator(generator_in), + name_func(name_func_in), + file(file_in), + line(line_in) {} + + std::string name; + GeneratorCreationFunc* generator; + ParamNameGeneratorFunc* name_func; + const char* file; + int line; + }; + typedef ::std::vector InstantiationContainer; + + static bool IsValidParamName(const std::string& name) { + // Check for empty string + if (name.empty()) + return false; + + // Check for invalid characters + for (std::string::size_type index = 0; index < name.size(); ++index) { + if (!isalnum(name[index]) && name[index] != '_') + return false; + } + + return true; + } + + const string test_case_name_; + CodeLocation code_location_; + TestInfoContainer tests_; + InstantiationContainer instantiations_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfo); +}; // class ParameterizedTestCaseInfo + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseRegistry contains a map of ParameterizedTestCaseInfoBase +// classes accessed by test case names. TEST_P and INSTANTIATE_TEST_CASE_P +// macros use it to locate their corresponding ParameterizedTestCaseInfo +// descriptors. +class ParameterizedTestCaseRegistry { + public: + ParameterizedTestCaseRegistry() {} + ~ParameterizedTestCaseRegistry() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + delete *it; + } + } + + // Looks up or creates and returns a structure containing information about + // tests and instantiations of a particular test case. + template + ParameterizedTestCaseInfo* GetTestCasePatternHolder( + const char* test_case_name, + CodeLocation code_location) { + ParameterizedTestCaseInfo* typed_test_info = NULL; + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + if ((*it)->GetTestCaseName() == test_case_name) { + if ((*it)->GetTestCaseTypeId() != GetTypeId()) { + // Complain about incorrect usage of Google Test facilities + // and terminate the program since we cannot guaranty correct + // test case setup and tear-down in this case. + ReportInvalidTestCaseType(test_case_name, code_location); + posix::Abort(); + } else { + // At this point we are sure that the object we found is of the same + // type we are looking for, so we downcast it to that type + // without further checks. + typed_test_info = CheckedDowncastToActualType< + ParameterizedTestCaseInfo >(*it); + } + break; + } + } + if (typed_test_info == NULL) { + typed_test_info = new ParameterizedTestCaseInfo( + test_case_name, code_location); + test_case_infos_.push_back(typed_test_info); + } + return typed_test_info; + } + void RegisterTests() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + (*it)->RegisterTests(); + } + } + + private: + typedef ::std::vector TestCaseInfoContainer; + + TestCaseInfoContainer test_case_infos_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseRegistry); +}; + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-port-arch.h b/mypyc/external/googletest/include/gtest/internal/gtest-port-arch.h new file mode 100644 index 000000000000..74ab949057c7 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-port-arch.h @@ -0,0 +1,93 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the GTEST_OS_* macro. +// It is separate from gtest-port.h so that custom/gtest-port.h can include it. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ + +// Determines the platform on which Google Test is compiled. +#ifdef __CYGWIN__ +# define GTEST_OS_CYGWIN 1 +#elif defined __SYMBIAN32__ +# define GTEST_OS_SYMBIAN 1 +#elif defined _WIN32 +# define GTEST_OS_WINDOWS 1 +# ifdef _WIN32_WCE +# define GTEST_OS_WINDOWS_MOBILE 1 +# elif defined(__MINGW__) || defined(__MINGW32__) +# define GTEST_OS_WINDOWS_MINGW 1 +# elif defined(WINAPI_FAMILY) +# include +# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +# define GTEST_OS_WINDOWS_DESKTOP 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP) +# define GTEST_OS_WINDOWS_PHONE 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +# define GTEST_OS_WINDOWS_RT 1 +# else + // WINAPI_FAMILY defined but no known partition matched. + // Default to desktop. +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif +# else +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif // _WIN32_WCE +#elif defined __APPLE__ +# define GTEST_OS_MAC 1 +# if TARGET_OS_IPHONE +# define GTEST_OS_IOS 1 +# endif +#elif defined __FreeBSD__ +# define GTEST_OS_FREEBSD 1 +#elif defined __linux__ +# define GTEST_OS_LINUX 1 +# if defined __ANDROID__ +# define GTEST_OS_LINUX_ANDROID 1 +# endif +#elif defined __MVS__ +# define GTEST_OS_ZOS 1 +#elif defined(__sun) && defined(__SVR4) +# define GTEST_OS_SOLARIS 1 +#elif defined(_AIX) +# define GTEST_OS_AIX 1 +#elif defined(__hpux) +# define GTEST_OS_HPUX 1 +#elif defined __native_client__ +# define GTEST_OS_NACL 1 +#elif defined __OpenBSD__ +# define GTEST_OS_OPENBSD 1 +#elif defined __QNX__ +# define GTEST_OS_QNX 1 +#endif // __CYGWIN__ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-port.h b/mypyc/external/googletest/include/gtest/internal/gtest-port.h new file mode 100644 index 000000000000..92f4c11c4f6c --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-port.h @@ -0,0 +1,2560 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan) +// +// Low-level types and utilities for porting Google Test to various +// platforms. All macros ending with _ and symbols defined in an +// internal namespace are subject to change without notice. Code +// outside Google Test MUST NOT USE THEM DIRECTLY. Macros that don't +// end with _ are part of Google Test's public API and can be used by +// code outside Google Test. +// +// This file is fundamental to Google Test. All other Google Test source +// files are expected to #include this. Therefore, it cannot #include +// any other Google Test header. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ + +// Environment-describing macros +// ----------------------------- +// +// Google Test can be used in many different environments. Macros in +// this section tell Google Test what kind of environment it is being +// used in, such that Google Test can provide environment-specific +// features and implementations. +// +// Google Test tries to automatically detect the properties of its +// environment, so users usually don't need to worry about these +// macros. However, the automatic detection is not perfect. +// Sometimes it's necessary for a user to define some of the following +// macros in the build script to override Google Test's decisions. +// +// If the user doesn't define a macro in the list, Google Test will +// provide a default definition. After this header is #included, all +// macros in this list will be defined to either 1 or 0. +// +// Notes to maintainers: +// - Each macro here is a user-tweakable knob; do not grow the list +// lightly. +// - Use #if to key off these macros. Don't use #ifdef or "#if +// defined(...)", which will not work as these macros are ALWAYS +// defined. +// +// GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2) +// is/isn't available. +// GTEST_HAS_EXCEPTIONS - Define it to 1/0 to indicate that exceptions +// are enabled. +// GTEST_HAS_GLOBAL_STRING - Define it to 1/0 to indicate that ::string +// is/isn't available (some systems define +// ::string, which is different to std::string). +// GTEST_HAS_GLOBAL_WSTRING - Define it to 1/0 to indicate that ::string +// is/isn't available (some systems define +// ::wstring, which is different to std::wstring). +// GTEST_HAS_POSIX_RE - Define it to 1/0 to indicate that POSIX regular +// expressions are/aren't available. +// GTEST_HAS_PTHREAD - Define it to 1/0 to indicate that +// is/isn't available. +// GTEST_HAS_RTTI - Define it to 1/0 to indicate that RTTI is/isn't +// enabled. +// GTEST_HAS_STD_WSTRING - Define it to 1/0 to indicate that +// std::wstring does/doesn't work (Google Test can +// be used where std::wstring is unavailable). +// GTEST_HAS_TR1_TUPLE - Define it to 1/0 to indicate tr1::tuple +// is/isn't available. +// GTEST_HAS_SEH - Define it to 1/0 to indicate whether the +// compiler supports Microsoft's "Structured +// Exception Handling". +// GTEST_HAS_STREAM_REDIRECTION +// - Define it to 1/0 to indicate whether the +// platform supports I/O stream redirection using +// dup() and dup2(). +// GTEST_USE_OWN_TR1_TUPLE - Define it to 1/0 to indicate whether Google +// Test's own tr1 tuple implementation should be +// used. Unused when the user sets +// GTEST_HAS_TR1_TUPLE to 0. +// GTEST_LANG_CXX11 - Define it to 1/0 to indicate that Google Test +// is building in C++11/C++98 mode. +// GTEST_LINKED_AS_SHARED_LIBRARY +// - Define to 1 when compiling tests that use +// Google Test as a shared library (known as +// DLL on Windows). +// GTEST_CREATE_SHARED_LIBRARY +// - Define to 1 when compiling Google Test itself +// as a shared library. + +// Platform-indicating macros +// -------------------------- +// +// Macros indicating the platform on which Google Test is being used +// (a macro is defined to 1 if compiled on the given platform; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. +// +// GTEST_OS_AIX - IBM AIX +// GTEST_OS_CYGWIN - Cygwin +// GTEST_OS_FREEBSD - FreeBSD +// GTEST_OS_HPUX - HP-UX +// GTEST_OS_LINUX - Linux +// GTEST_OS_LINUX_ANDROID - Google Android +// GTEST_OS_MAC - Mac OS X +// GTEST_OS_IOS - iOS +// GTEST_OS_NACL - Google Native Client (NaCl) +// GTEST_OS_OPENBSD - OpenBSD +// GTEST_OS_QNX - QNX +// GTEST_OS_SOLARIS - Sun Solaris +// GTEST_OS_SYMBIAN - Symbian +// GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile) +// GTEST_OS_WINDOWS_DESKTOP - Windows Desktop +// GTEST_OS_WINDOWS_MINGW - MinGW +// GTEST_OS_WINDOWS_MOBILE - Windows Mobile +// GTEST_OS_WINDOWS_PHONE - Windows Phone +// GTEST_OS_WINDOWS_RT - Windows Store App/WinRT +// GTEST_OS_ZOS - z/OS +// +// Among the platforms, Cygwin, Linux, Max OS X, and Windows have the +// most stable support. Since core members of the Google Test project +// don't have access to other platforms, support for them may be less +// stable. If you notice any problems on your platform, please notify +// googletestframework@googlegroups.com (patches for fixing them are +// even more welcome!). +// +// It is possible that none of the GTEST_OS_* macros are defined. + +// Feature-indicating macros +// ------------------------- +// +// Macros indicating which Google Test features are available (a macro +// is defined to 1 if the corresponding feature is supported; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. +// +// These macros are public so that portable tests can be written. +// Such tests typically surround code using a feature with an #if +// which controls that code. For example: +// +// #if GTEST_HAS_DEATH_TEST +// EXPECT_DEATH(DoSomethingDeadly()); +// #endif +// +// GTEST_HAS_COMBINE - the Combine() function (for value-parameterized +// tests) +// GTEST_HAS_DEATH_TEST - death tests +// GTEST_HAS_PARAM_TEST - value-parameterized tests +// GTEST_HAS_TYPED_TEST - typed tests +// GTEST_HAS_TYPED_TEST_P - type-parameterized tests +// GTEST_IS_THREADSAFE - Google Test is thread-safe. +// GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with +// GTEST_HAS_POSIX_RE (see above) which users can +// define themselves. +// GTEST_USES_SIMPLE_RE - our own simple regex is used; +// the above two are mutually exclusive. +// GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ(). + +// Misc public macros +// ------------------ +// +// GTEST_FLAG(flag_name) - references the variable corresponding to +// the given Google Test flag. + +// Internal utilities +// ------------------ +// +// The following macros and utilities are for Google Test's INTERNAL +// use only. Code outside Google Test MUST NOT USE THEM DIRECTLY. +// +// Macros for basic C++ coding: +// GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning. +// GTEST_ATTRIBUTE_UNUSED_ - declares that a class' instances or a +// variable don't have to be used. +// GTEST_DISALLOW_ASSIGN_ - disables operator=. +// GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=. +// GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. +// GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is +// suppressed (constant conditional). +// GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127 +// is suppressed. +// +// C++11 feature wrappers: +// +// testing::internal::move - portability wrapper for std::move. +// +// Synchronization: +// Mutex, MutexLock, ThreadLocal, GetThreadCount() +// - synchronization primitives. +// +// Template meta programming: +// is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only. +// IteratorTraits - partial implementation of std::iterator_traits, which +// is not available in libCstd when compiled with Sun C++. +// +// Smart pointers: +// scoped_ptr - as in TR2. +// +// Regular expressions: +// RE - a simple regular expression class using the POSIX +// Extended Regular Expression syntax on UNIX-like +// platforms, or a reduced regular exception syntax on +// other platforms, including Windows. +// +// Logging: +// GTEST_LOG_() - logs messages at the specified severity level. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. +// +// Stdout and stderr capturing: +// CaptureStdout() - starts capturing stdout. +// GetCapturedStdout() - stops capturing stdout and returns the captured +// string. +// CaptureStderr() - starts capturing stderr. +// GetCapturedStderr() - stops capturing stderr and returns the captured +// string. +// +// Integer types: +// TypeWithSize - maps an integer to a int type. +// Int32, UInt32, Int64, UInt64, TimeInMillis +// - integers of known sizes. +// BiggestInt - the biggest signed integer type. +// +// Command-line utilities: +// GTEST_DECLARE_*() - declares a flag. +// GTEST_DEFINE_*() - defines a flag. +// GetInjectableArgvs() - returns the command line as a vector of strings. +// +// Environment variable utilities: +// GetEnv() - gets the value of an environment variable. +// BoolFromGTestEnv() - parses a bool environment variable. +// Int32FromGTestEnv() - parses an Int32 environment variable. +// StringFromGTestEnv() - parses a string environment variable. + +#include // for isspace, etc +#include // for ptrdiff_t +#include +#include +#include +#ifndef _WIN32_WCE +# include +# include +#endif // !_WIN32_WCE + +#if defined __APPLE__ +# include +# include +#endif + +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include +#include // NOLINT + +#include "gtest/internal/gtest-port-arch.h" +#include "gtest/internal/custom/gtest-port.h" + +#if !defined(GTEST_DEV_EMAIL_) +# define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com" +# define GTEST_FLAG_PREFIX_ "gtest_" +# define GTEST_FLAG_PREFIX_DASH_ "gtest-" +# define GTEST_FLAG_PREFIX_UPPER_ "GTEST_" +# define GTEST_NAME_ "Google Test" +# define GTEST_PROJECT_URL_ "https://github.com/google/googletest/" +#endif // !defined(GTEST_DEV_EMAIL_) + +#if !defined(GTEST_INIT_GOOGLE_TEST_NAME_) +# define GTEST_INIT_GOOGLE_TEST_NAME_ "testing::InitGoogleTest" +#endif // !defined(GTEST_INIT_GOOGLE_TEST_NAME_) + +// Determines the version of gcc that is used to compile this. +#ifdef __GNUC__ +// 40302 means version 4.3.2. +# define GTEST_GCC_VER_ \ + (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__) +#endif // __GNUC__ + +// Macros for disabling Microsoft Visual C++ warnings. +// +// GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 4385) +// /* code that triggers warnings C4800 and C4385 */ +// GTEST_DISABLE_MSC_WARNINGS_POP_() +#if _MSC_VER >= 1500 +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) \ + __pragma(warning(push)) \ + __pragma(warning(disable: warnings)) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() \ + __pragma(warning(pop)) +#else +// Older versions of MSVC don't have __pragma. +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif + +#ifndef GTEST_LANG_CXX11 +// gcc and clang define __GXX_EXPERIMENTAL_CXX0X__ when +// -std={c,gnu}++{0x,11} is passed. The C++11 standard specifies a +// value for __cplusplus, and recent versions of clang, gcc, and +// probably other compilers set that too in C++11 mode. +# if __GXX_EXPERIMENTAL_CXX0X__ || __cplusplus >= 201103L +// Compiling in at least C++11 mode. +# define GTEST_LANG_CXX11 1 +# else +# define GTEST_LANG_CXX11 0 +# endif +#endif + +// Distinct from C++11 language support, some environments don't provide +// proper C++11 library support. Notably, it's possible to build in +// C++11 mode when targeting Mac OS X 10.6, which has an old libstdc++ +// with no C++11 support. +// +// libstdc++ has sufficient C++11 support as of GCC 4.6.0, __GLIBCXX__ +// 20110325, but maintenance releases in the 4.4 and 4.5 series followed +// this date, so check for those versions by their date stamps. +// https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning +#if GTEST_LANG_CXX11 && \ + (!defined(__GLIBCXX__) || ( \ + __GLIBCXX__ >= 20110325ul && /* GCC >= 4.6.0 */ \ + /* Blacklist of patch releases of older branches: */ \ + __GLIBCXX__ != 20110416ul && /* GCC 4.4.6 */ \ + __GLIBCXX__ != 20120313ul && /* GCC 4.4.7 */ \ + __GLIBCXX__ != 20110428ul && /* GCC 4.5.3 */ \ + __GLIBCXX__ != 20120702ul)) /* GCC 4.5.4 */ +# define GTEST_STDLIB_CXX11 1 +#endif + +// Only use C++11 library features if the library provides them. +#if GTEST_STDLIB_CXX11 +# define GTEST_HAS_STD_BEGIN_AND_END_ 1 +# define GTEST_HAS_STD_FORWARD_LIST_ 1 +# define GTEST_HAS_STD_FUNCTION_ 1 +# define GTEST_HAS_STD_INITIALIZER_LIST_ 1 +# define GTEST_HAS_STD_MOVE_ 1 +# define GTEST_HAS_STD_SHARED_PTR_ 1 +# define GTEST_HAS_STD_TYPE_TRAITS_ 1 +# define GTEST_HAS_STD_UNIQUE_PTR_ 1 +#endif + +// C++11 specifies that provides std::tuple. +// Some platforms still might not have it, however. +#if GTEST_LANG_CXX11 +# define GTEST_HAS_STD_TUPLE_ 1 +# if defined(__clang__) +// Inspired by http://clang.llvm.org/docs/LanguageExtensions.html#__has_include +# if defined(__has_include) && !__has_include() +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(_MSC_VER) +// Inspired by boost/config/stdlib/dinkumware.hpp +# if defined(_CPPLIB_VER) && _CPPLIB_VER < 520 +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(__GLIBCXX__) +// Inspired by boost/config/stdlib/libstdcpp3.hpp, +// http://gcc.gnu.org/gcc-4.2/changes.html and +// http://gcc.gnu.org/onlinedocs/libstdc++/manual/bk01pt01ch01.html#manual.intro.status.standard.200x +# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 2) +# undef GTEST_HAS_STD_TUPLE_ +# endif +# endif +#endif + +// Brings in definitions for functions used in the testing::internal::posix +// namespace (read, write, close, chdir, isatty, stat). We do not currently +// use them on Windows Mobile. +#if GTEST_OS_WINDOWS +# if !GTEST_OS_WINDOWS_MOBILE +# include +# include +# endif +// In order to avoid having to include , use forward declaration +#if GTEST_OS_WINDOWS_MINGW +// MinGW defined _CRITICAL_SECTION and _RTL_CRITICAL_SECTION as two +// separate (equivalent) structs, instead of using typedef +typedef struct _CRITICAL_SECTION GTEST_CRITICAL_SECTION; +#else +// Assume CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION. +// This assumption is verified by +// WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION. +typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION; +#endif +#else +// This assumes that non-Windows OSes provide unistd.h. For OSes where this +// is not the case, we need to include headers that provide the functions +// mentioned above. +# include +# include +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_LINUX_ANDROID +// Used to define __ANDROID_API__ matching the target NDK API level. +# include // NOLINT +#endif + +// Defines this to true iff Google Test can use POSIX regular expressions. +#ifndef GTEST_HAS_POSIX_RE +# if GTEST_OS_LINUX_ANDROID +// On Android, is only available starting with Gingerbread. +# define GTEST_HAS_POSIX_RE (__ANDROID_API__ >= 9) +# else +# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +# endif +#endif + +#if GTEST_USES_PCRE +// The appropriate headers have already been included. + +#elif GTEST_HAS_POSIX_RE + +// On some platforms, needs someone to define size_t, and +// won't compile otherwise. We can #include it here as we already +// included , which is guaranteed to define size_t through +// . +# include // NOLINT + +# define GTEST_USES_POSIX_RE 1 + +#elif GTEST_OS_WINDOWS + +// is not available on Windows. Use our own simple regex +// implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#else + +// may not be available on this platform. Use our own +// simple regex implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#endif // GTEST_USES_PCRE + +#ifndef GTEST_HAS_EXCEPTIONS +// The user didn't tell us whether exceptions are enabled, so we need +// to figure it out. +# if defined(_MSC_VER) || defined(__BORLANDC__) +// MSVC's and C++Builder's implementations of the STL use the _HAS_EXCEPTIONS +// macro to enable exceptions, so we'll do the same. +// Assumes that exceptions are enabled by default. +# ifndef _HAS_EXCEPTIONS +# define _HAS_EXCEPTIONS 1 +# endif // _HAS_EXCEPTIONS +# define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS +# elif defined(__clang__) +// clang defines __EXCEPTIONS iff exceptions are enabled before clang 220714, +// but iff cleanups are enabled after that. In Obj-C++ files, there can be +// cleanups for ObjC exceptions which also need cleanups, even if C++ exceptions +// are disabled. clang has __has_feature(cxx_exceptions) which checks for C++ +// exceptions starting at clang r206352, but which checked for cleanups prior to +// that. To reliably check for C++ exception availability with clang, check for +// __EXCEPTIONS && __has_feature(cxx_exceptions). +# define GTEST_HAS_EXCEPTIONS (__EXCEPTIONS && __has_feature(cxx_exceptions)) +# elif defined(__GNUC__) && __EXCEPTIONS +// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__SUNPRO_CC) +// Sun Pro CC supports exceptions. However, there is no compile-time way of +// detecting whether they are enabled or not. Therefore, we assume that +// they are enabled unless the user tells us otherwise. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__IBMCPP__) && __EXCEPTIONS +// xlC defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__HP_aCC) +// Exception handling is in effect by default in HP aCC compiler. It has to +// be turned of by +noeh compiler option if desired. +# define GTEST_HAS_EXCEPTIONS 1 +# else +// For other compilers, we assume exceptions are disabled to be +// conservative. +# define GTEST_HAS_EXCEPTIONS 0 +# endif // defined(_MSC_VER) || defined(__BORLANDC__) +#endif // GTEST_HAS_EXCEPTIONS + +#if !defined(GTEST_HAS_STD_STRING) +// Even though we don't use this macro any longer, we keep it in case +// some clients still depend on it. +# define GTEST_HAS_STD_STRING 1 +#elif !GTEST_HAS_STD_STRING +// The user told us that ::std::string isn't available. +# error "Google Test cannot be used where ::std::string isn't available." +#endif // !defined(GTEST_HAS_STD_STRING) + +#ifndef GTEST_HAS_GLOBAL_STRING +// The user didn't tell us whether ::string is available, so we need +// to figure it out. + +# define GTEST_HAS_GLOBAL_STRING 0 + +#endif // GTEST_HAS_GLOBAL_STRING + +#ifndef GTEST_HAS_STD_WSTRING +// The user didn't tell us whether ::std::wstring is available, so we need +// to figure it out. +// TODO(wan@google.com): uses autoconf to detect whether ::std::wstring +// is available. + +// Cygwin 1.7 and below doesn't support ::std::wstring. +// Solaris' libc++ doesn't support it either. Android has +// no support for it at least as recent as Froyo (2.2). +# define GTEST_HAS_STD_WSTRING \ + (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS)) + +#endif // GTEST_HAS_STD_WSTRING + +#ifndef GTEST_HAS_GLOBAL_WSTRING +// The user didn't tell us whether ::wstring is available, so we need +// to figure it out. +# define GTEST_HAS_GLOBAL_WSTRING \ + (GTEST_HAS_STD_WSTRING && GTEST_HAS_GLOBAL_STRING) +#endif // GTEST_HAS_GLOBAL_WSTRING + +// Determines whether RTTI is available. +#ifndef GTEST_HAS_RTTI +// The user didn't tell us whether RTTI is enabled, so we need to +// figure it out. + +# ifdef _MSC_VER + +# ifdef _CPPRTTI // MSVC defines this macro iff RTTI is enabled. +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +// Starting with version 4.3.2, gcc defines __GXX_RTTI iff RTTI is enabled. +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40302) + +# ifdef __GXX_RTTI +// When building against STLport with the Android NDK and with +// -frtti -fno-exceptions, the build fails at link time with undefined +// references to __cxa_bad_typeid. Note sure if STL or toolchain bug, +// so disable RTTI when detected. +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) && \ + !defined(__EXCEPTIONS) +# define GTEST_HAS_RTTI 0 +# else +# define GTEST_HAS_RTTI 1 +# endif // GTEST_OS_LINUX_ANDROID && __STLPORT_MAJOR && !__EXCEPTIONS +# else +# define GTEST_HAS_RTTI 0 +# endif // __GXX_RTTI + +// Clang defines __GXX_RTTI starting with version 3.0, but its manual recommends +// using has_feature instead. has_feature(cxx_rtti) is supported since 2.7, the +// first version with C++ support. +# elif defined(__clang__) + +# define GTEST_HAS_RTTI __has_feature(cxx_rtti) + +// Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if +// both the typeid and dynamic_cast features are present. +# elif defined(__IBMCPP__) && (__IBMCPP__ >= 900) + +# ifdef __RTTI_ALL__ +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +# else + +// For all other compilers, we assume RTTI is enabled. +# define GTEST_HAS_RTTI 1 + +# endif // _MSC_VER + +#endif // GTEST_HAS_RTTI + +// It's this header's responsibility to #include when RTTI +// is enabled. +#if GTEST_HAS_RTTI +# include +#endif + +// Determines whether Google Test can use the pthreads library. +#ifndef GTEST_HAS_PTHREAD +// The user didn't tell us explicitly, so we make reasonable assumptions about +// which platforms have pthreads support. +// +// To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0 +// to your compiler flags. +# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX \ + || GTEST_OS_QNX || GTEST_OS_FREEBSD || GTEST_OS_NACL) +#endif // GTEST_HAS_PTHREAD + +#if GTEST_HAS_PTHREAD +// gtest-port.h guarantees to #include when GTEST_HAS_PTHREAD is +// true. +# include // NOLINT + +// For timespec and nanosleep, used below. +# include // NOLINT +#endif + +// Determines if hash_map/hash_set are available. +// Only used for testing against those containers. +#if !defined(GTEST_HAS_HASH_MAP_) +# if _MSC_VER +# define GTEST_HAS_HASH_MAP_ 1 // Indicates that hash_map is available. +# define GTEST_HAS_HASH_SET_ 1 // Indicates that hash_set is available. +# endif // _MSC_VER +#endif // !defined(GTEST_HAS_HASH_MAP_) + +// Determines whether Google Test can use tr1/tuple. You can define +// this macro to 0 to prevent Google Test from using tuple (any +// feature depending on tuple with be disabled in this mode). +#ifndef GTEST_HAS_TR1_TUPLE +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) +// STLport, provided with the Android NDK, has neither or . +# define GTEST_HAS_TR1_TUPLE 0 +# else +// The user didn't tell us not to do it, so we assume it's OK. +# define GTEST_HAS_TR1_TUPLE 1 +# endif +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether Google Test's own tr1 tuple implementation +// should be used. +#ifndef GTEST_USE_OWN_TR1_TUPLE +// The user didn't tell us, so we need to figure it out. + +// We use our own TR1 tuple if we aren't sure the user has an +// implementation of it already. At this time, libstdc++ 4.0.0+ and +// MSVC 2010 are the only mainstream standard libraries that come +// with a TR1 tuple implementation. NVIDIA's CUDA NVCC compiler +// pretends to be GCC by defining __GNUC__ and friends, but cannot +// compile GCC's tuple implementation. MSVC 2008 (9.0) provides TR1 +// tuple in a 323 MB Feature Pack download, which we cannot assume the +// user has. QNX's QCC compiler is a modified GCC but it doesn't +// support TR1 tuple. libc++ only provides std::tuple, in C++11 mode, +// and it can be used with some compilers that define __GNUC__. +# if (defined(__GNUC__) && !defined(__CUDACC__) && (GTEST_GCC_VER_ >= 40000) \ + && !GTEST_OS_QNX && !defined(_LIBCPP_VERSION)) || _MSC_VER >= 1600 +# define GTEST_ENV_HAS_TR1_TUPLE_ 1 +# endif + +// C++11 specifies that provides std::tuple. Use that if gtest is used +// in C++11 mode and libstdc++ isn't very old (binaries targeting OS X 10.6 +// can build with clang but need to use gcc4.2's libstdc++). +# if GTEST_LANG_CXX11 && (!defined(__GLIBCXX__) || __GLIBCXX__ > 20110325) +# define GTEST_ENV_HAS_STD_TUPLE_ 1 +# endif + +# if GTEST_ENV_HAS_TR1_TUPLE_ || GTEST_ENV_HAS_STD_TUPLE_ +# define GTEST_USE_OWN_TR1_TUPLE 0 +# else +# define GTEST_USE_OWN_TR1_TUPLE 1 +# endif + +#endif // GTEST_USE_OWN_TR1_TUPLE + +// To avoid conditional compilation everywhere, we make it +// gtest-port.h's responsibility to #include the header implementing +// tuple. +#if GTEST_HAS_STD_TUPLE_ +# include // IWYU pragma: export +# define GTEST_TUPLE_NAMESPACE_ ::std +#endif // GTEST_HAS_STD_TUPLE_ + +// We include tr1::tuple even if std::tuple is available to define printers for +// them. +#if GTEST_HAS_TR1_TUPLE +# ifndef GTEST_TUPLE_NAMESPACE_ +# define GTEST_TUPLE_NAMESPACE_ ::std::tr1 +# endif // GTEST_TUPLE_NAMESPACE_ + +# if GTEST_USE_OWN_TR1_TUPLE +# include "gtest/internal/gtest-tuple.h" // IWYU pragma: export // NOLINT +# elif GTEST_ENV_HAS_STD_TUPLE_ +# include +// C++11 puts its tuple into the ::std namespace rather than +// ::std::tr1. gtest expects tuple to live in ::std::tr1, so put it there. +// This causes undefined behavior, but supported compilers react in +// the way we intend. +namespace std { +namespace tr1 { +using ::std::get; +using ::std::make_tuple; +using ::std::tuple; +using ::std::tuple_element; +using ::std::tuple_size; +} +} + +# elif GTEST_OS_SYMBIAN + +// On Symbian, BOOST_HAS_TR1_TUPLE causes Boost's TR1 tuple library to +// use STLport's tuple implementation, which unfortunately doesn't +// work as the copy of STLport distributed with Symbian is incomplete. +// By making sure BOOST_HAS_TR1_TUPLE is undefined, we force Boost to +// use its own tuple implementation. +# ifdef BOOST_HAS_TR1_TUPLE +# undef BOOST_HAS_TR1_TUPLE +# endif // BOOST_HAS_TR1_TUPLE + +// This prevents , which defines +// BOOST_HAS_TR1_TUPLE, from being #included by Boost's . +# define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED +# include // IWYU pragma: export // NOLINT + +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000) +// GCC 4.0+ implements tr1/tuple in the header. This does +// not conform to the TR1 spec, which requires the header to be . + +# if !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 +// Until version 4.3.2, gcc has a bug that causes , +// which is #included by , to not compile when RTTI is +// disabled. _TR1_FUNCTIONAL is the header guard for +// . Hence the following #define is a hack to prevent +// from being included. +# define _TR1_FUNCTIONAL 1 +# include +# undef _TR1_FUNCTIONAL // Allows the user to #include + // if he chooses to. +# else +# include // NOLINT +# endif // !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 + +# else +// If the compiler is not GCC 4.0+, we assume the user is using a +// spec-conforming TR1 implementation. +# include // IWYU pragma: export // NOLINT +# endif // GTEST_USE_OWN_TR1_TUPLE + +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether clone(2) is supported. +// Usually it will only be available on Linux, excluding +// Linux on the Itanium architecture. +// Also see http://linux.die.net/man/2/clone. +#ifndef GTEST_HAS_CLONE +// The user didn't tell us, so we need to figure it out. + +# if GTEST_OS_LINUX && !defined(__ia64__) +# if GTEST_OS_LINUX_ANDROID +// On Android, clone() is only available on ARM starting with Gingerbread. +# if defined(__arm__) && __ANDROID_API__ >= 9 +# define GTEST_HAS_CLONE 1 +# else +# define GTEST_HAS_CLONE 0 +# endif +# else +# define GTEST_HAS_CLONE 1 +# endif +# else +# define GTEST_HAS_CLONE 0 +# endif // GTEST_OS_LINUX && !defined(__ia64__) + +#endif // GTEST_HAS_CLONE + +// Determines whether to support stream redirection. This is used to test +// output correctness and to implement death tests. +#ifndef GTEST_HAS_STREAM_REDIRECTION +// By default, we assume that stream redirection is supported on all +// platforms except known mobile ones. +# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || \ + GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT +# define GTEST_HAS_STREAM_REDIRECTION 0 +# else +# define GTEST_HAS_STREAM_REDIRECTION 1 +# endif // !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_SYMBIAN +#endif // GTEST_HAS_STREAM_REDIRECTION + +// Determines whether to support death tests. +// Google Test does not support death tests for VC 7.1 and earlier as +// abort() in a VC 7.1 application compiled as GUI in debug config +// pops up a dialog window that cannot be suppressed programmatically. +#if (GTEST_OS_LINUX || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ + (GTEST_OS_MAC && !GTEST_OS_IOS) || \ + (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \ + GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX || \ + GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD) +# define GTEST_HAS_DEATH_TEST 1 +#endif + +// We don't support MSVC 7.1 with exceptions disabled now. Therefore +// all the compilers we care about are adequate for supporting +// value-parameterized tests. +#define GTEST_HAS_PARAM_TEST 1 + +// Determines whether to support type-driven tests. + +// Typed tests need and variadic macros, which GCC, VC++ 8.0, +// Sun Pro CC, IBM Visual Age, and HP aCC support. +#if defined(__GNUC__) || (_MSC_VER >= 1400) || defined(__SUNPRO_CC) || \ + defined(__IBMCPP__) || defined(__HP_aCC) +# define GTEST_HAS_TYPED_TEST 1 +# define GTEST_HAS_TYPED_TEST_P 1 +#endif + +// Determines whether to support Combine(). This only makes sense when +// value-parameterized tests are enabled. The implementation doesn't +// work on Sun Studio since it doesn't understand templated conversion +// operators. +#if GTEST_HAS_PARAM_TEST && GTEST_HAS_TR1_TUPLE && !defined(__SUNPRO_CC) +# define GTEST_HAS_COMBINE 1 +#endif + +// Determines whether the system compiler uses UTF-16 for encoding wide strings. +#define GTEST_WIDE_STRING_USES_UTF16_ \ + (GTEST_OS_WINDOWS || GTEST_OS_CYGWIN || GTEST_OS_SYMBIAN || GTEST_OS_AIX) + +// Determines whether test results can be streamed to a socket. +#if GTEST_OS_LINUX +# define GTEST_CAN_STREAM_RESULTS_ 1 +#endif + +// Defines some utility macros. + +// The GNU compiler emits a warning if nested "if" statements are followed by +// an "else" statement and braces are not used to explicitly disambiguate the +// "else" binding. This leads to problems with code like: +// +// if (gate) +// ASSERT_*(condition) << "Some message"; +// +// The "switch (0) case 0:" idiom is used to suppress this. +#ifdef __INTEL_COMPILER +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ +#else +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT +#endif + +// Use this annotation at the end of a struct/class definition to +// prevent the compiler from optimizing away instances that are never +// used. This is useful when all interesting logic happens inside the +// c'tor and / or d'tor. Example: +// +// struct Foo { +// Foo() { ... } +// } GTEST_ATTRIBUTE_UNUSED_; +// +// Also use it after a variable or parameter declaration to tell the +// compiler the variable/parameter does not have to be used. +#if defined(__GNUC__) && !defined(COMPILER_ICC) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +#elif defined(__clang__) +# if __has_attribute(unused) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +# endif +#endif +#ifndef GTEST_ATTRIBUTE_UNUSED_ +# define GTEST_ATTRIBUTE_UNUSED_ +#endif + +// A macro to disallow operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_ASSIGN_(type)\ + void operator=(type const &) + +// A macro to disallow copy constructor and operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_COPY_AND_ASSIGN_(type)\ + type(type const &);\ + GTEST_DISALLOW_ASSIGN_(type) + +// Tell the compiler to warn about unused return values for functions declared +// with this macro. The macro should be used on function declarations +// following the argument list: +// +// Sprocket* AllocateSprocket() GTEST_MUST_USE_RESULT_; +#if defined(__GNUC__) && (GTEST_GCC_VER_ >= 30400) && !defined(COMPILER_ICC) +# define GTEST_MUST_USE_RESULT_ __attribute__ ((warn_unused_result)) +#else +# define GTEST_MUST_USE_RESULT_ +#endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC + +// MS C++ compiler emits warning when a conditional expression is compile time +// constant. In some contexts this warning is false positive and needs to be +// suppressed. Use the following two macros in such cases: +// +// GTEST_INTENTIONAL_CONST_COND_PUSH_() +// while (true) { +// GTEST_INTENTIONAL_CONST_COND_POP_() +// } +# define GTEST_INTENTIONAL_CONST_COND_PUSH_() \ + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4127) +# define GTEST_INTENTIONAL_CONST_COND_POP_() \ + GTEST_DISABLE_MSC_WARNINGS_POP_() + +// Determine whether the compiler supports Microsoft's Structured Exception +// Handling. This is supported by several Windows compilers but generally +// does not exist on any other system. +#ifndef GTEST_HAS_SEH +// The user didn't tell us, so we need to figure it out. + +# if defined(_MSC_VER) || defined(__BORLANDC__) +// These two compilers are known to support SEH. +# define GTEST_HAS_SEH 1 +# else +// Assume no SEH. +# define GTEST_HAS_SEH 0 +# endif + +#define GTEST_IS_THREADSAFE \ + (GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ \ + || (GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT) \ + || GTEST_HAS_PTHREAD) + +#endif // GTEST_HAS_SEH + +#ifdef _MSC_VER +# if GTEST_LINKED_AS_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllimport) +# elif GTEST_CREATE_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllexport) +# endif +#elif __GNUC__ >= 4 || defined(__clang__) +# define GTEST_API_ __attribute__((visibility ("default"))) +#endif // _MSC_VER + +#ifndef GTEST_API_ +# define GTEST_API_ +#endif + +#ifdef __GNUC__ +// Ask the compiler to never inline a given function. +# define GTEST_NO_INLINE_ __attribute__((noinline)) +#else +# define GTEST_NO_INLINE_ +#endif + +// _LIBCPP_VERSION is defined by the libc++ library from the LLVM project. +#if defined(__GLIBCXX__) || defined(_LIBCPP_VERSION) +# define GTEST_HAS_CXXABI_H_ 1 +#else +# define GTEST_HAS_CXXABI_H_ 0 +#endif + +// A function level attribute to disable checking for use of uninitialized +// memory when built with MemorySanitizer. +#if defined(__clang__) +# if __has_feature(memory_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ \ + __attribute__((no_sanitize_memory)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +# endif // __has_feature(memory_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +#endif // __clang__ + +// A function level attribute to disable AddressSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(address_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ \ + __attribute__((no_sanitize_address)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +# endif // __has_feature(address_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +#endif // __clang__ + +// A function level attribute to disable ThreadSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(thread_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ \ + __attribute__((no_sanitize_thread)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +# endif // __has_feature(thread_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +#endif // __clang__ + +namespace testing { + +class Message; + +#if defined(GTEST_TUPLE_NAMESPACE_) +// Import tuple and friends into the ::testing namespace. +// It is part of our interface, having them in ::testing allows us to change +// their types as needed. +using GTEST_TUPLE_NAMESPACE_::get; +using GTEST_TUPLE_NAMESPACE_::make_tuple; +using GTEST_TUPLE_NAMESPACE_::tuple; +using GTEST_TUPLE_NAMESPACE_::tuple_size; +using GTEST_TUPLE_NAMESPACE_::tuple_element; +#endif // defined(GTEST_TUPLE_NAMESPACE_) + +namespace internal { + +// A secret type that Google Test users don't know about. It has no +// definition on purpose. Therefore it's impossible to create a +// Secret object, which is what we want. +class Secret; + +// The GTEST_COMPILE_ASSERT_ macro can be used to verify that a compile time +// expression is true. For example, you could use it to verify the +// size of a static array: +// +// GTEST_COMPILE_ASSERT_(GTEST_ARRAY_SIZE_(names) == NUM_NAMES, +// names_incorrect_size); +// +// or to make sure a struct is smaller than a certain size: +// +// GTEST_COMPILE_ASSERT_(sizeof(foo) < 128, foo_too_large); +// +// The second argument to the macro is the name of the variable. If +// the expression is false, most compilers will issue a warning/error +// containing the name of the variable. + +#if GTEST_LANG_CXX11 +# define GTEST_COMPILE_ASSERT_(expr, msg) static_assert(expr, #msg) +#else // !GTEST_LANG_CXX11 +template + struct CompileAssert { +}; + +# define GTEST_COMPILE_ASSERT_(expr, msg) \ + typedef ::testing::internal::CompileAssert<(static_cast(expr))> \ + msg[static_cast(expr) ? 1 : -1] GTEST_ATTRIBUTE_UNUSED_ +#endif // !GTEST_LANG_CXX11 + +// Implementation details of GTEST_COMPILE_ASSERT_: +// +// (In C++11, we simply use static_assert instead of the following) +// +// - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1 +// elements (and thus is invalid) when the expression is false. +// +// - The simpler definition +// +// #define GTEST_COMPILE_ASSERT_(expr, msg) typedef char msg[(expr) ? 1 : -1] +// +// does not work, as gcc supports variable-length arrays whose sizes +// are determined at run-time (this is gcc's extension and not part +// of the C++ standard). As a result, gcc fails to reject the +// following code with the simple definition: +// +// int foo; +// GTEST_COMPILE_ASSERT_(foo, msg); // not supposed to compile as foo is +// // not a compile-time constant. +// +// - By using the type CompileAssert<(bool(expr))>, we ensures that +// expr is a compile-time constant. (Template arguments must be +// determined at compile-time.) +// +// - The outter parentheses in CompileAssert<(bool(expr))> are necessary +// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written +// +// CompileAssert +// +// instead, these compilers will refuse to compile +// +// GTEST_COMPILE_ASSERT_(5 > 0, some_message); +// +// (They seem to think the ">" in "5 > 0" marks the end of the +// template argument list.) +// +// - The array size is (bool(expr) ? 1 : -1), instead of simply +// +// ((expr) ? 1 : -1). +// +// This is to avoid running into a bug in MS VC 7.1, which +// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1. + +// StaticAssertTypeEqHelper is used by StaticAssertTypeEq defined in gtest.h. +// +// This template is declared, but intentionally undefined. +template +struct StaticAssertTypeEqHelper; + +template +struct StaticAssertTypeEqHelper { + enum { value = true }; +}; + +// Evaluates to the number of elements in 'array'. +#define GTEST_ARRAY_SIZE_(array) (sizeof(array) / sizeof(array[0])) + +#if GTEST_HAS_GLOBAL_STRING +typedef ::string string; +#else +typedef ::std::string string; +#endif // GTEST_HAS_GLOBAL_STRING + +#if GTEST_HAS_GLOBAL_WSTRING +typedef ::wstring wstring; +#elif GTEST_HAS_STD_WSTRING +typedef ::std::wstring wstring; +#endif // GTEST_HAS_GLOBAL_WSTRING + +// A helper for suppressing warnings on constant condition. It just +// returns 'condition'. +GTEST_API_ bool IsTrue(bool condition); + +// Defines scoped_ptr. + +// This implementation of scoped_ptr is PARTIAL - it only contains +// enough stuff to satisfy Google Test's need. +template +class scoped_ptr { + public: + typedef T element_type; + + explicit scoped_ptr(T* p = NULL) : ptr_(p) {} + ~scoped_ptr() { reset(); } + + T& operator*() const { return *ptr_; } + T* operator->() const { return ptr_; } + T* get() const { return ptr_; } + + T* release() { + T* const ptr = ptr_; + ptr_ = NULL; + return ptr; + } + + void reset(T* p = NULL) { + if (p != ptr_) { + if (IsTrue(sizeof(T) > 0)) { // Makes sure T is a complete type. + delete ptr_; + } + ptr_ = p; + } + } + + friend void swap(scoped_ptr& a, scoped_ptr& b) { + using std::swap; + swap(a.ptr_, b.ptr_); + } + + private: + T* ptr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(scoped_ptr); +}; + +// Defines RE. + +// A simple C++ wrapper for . It uses the POSIX Extended +// Regular Expression syntax. +class GTEST_API_ RE { + public: + // A copy constructor is required by the Standard to initialize object + // references from r-values. + RE(const RE& other) { Init(other.pattern()); } + + // Constructs an RE from a string. + RE(const ::std::string& regex) { Init(regex.c_str()); } // NOLINT + +#if GTEST_HAS_GLOBAL_STRING + + RE(const ::string& regex) { Init(regex.c_str()); } // NOLINT + +#endif // GTEST_HAS_GLOBAL_STRING + + RE(const char* regex) { Init(regex); } // NOLINT + ~RE(); + + // Returns the string representation of the regex. + const char* pattern() const { return pattern_; } + + // FullMatch(str, re) returns true iff regular expression re matches + // the entire str. + // PartialMatch(str, re) returns true iff regular expression re + // matches a substring of str (including str itself). + // + // TODO(wan@google.com): make FullMatch() and PartialMatch() work + // when str contains NUL characters. + static bool FullMatch(const ::std::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::std::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +#if GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const ::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +#endif // GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const char* str, const RE& re); + static bool PartialMatch(const char* str, const RE& re); + + private: + void Init(const char* regex); + + // We use a const char* instead of an std::string, as Google Test used to be + // used where std::string is not available. TODO(wan@google.com): change to + // std::string. + const char* pattern_; + bool is_valid_; + +#if GTEST_USES_POSIX_RE + + regex_t full_regex_; // For FullMatch(). + regex_t partial_regex_; // For PartialMatch(). + +#else // GTEST_USES_SIMPLE_RE + + const char* full_pattern_; // For FullMatch(); + +#endif + + GTEST_DISALLOW_ASSIGN_(RE); +}; + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line); + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(const char* file, + int line); + +// Defines logging utilities: +// GTEST_LOG_(severity) - logs messages at the specified severity level. The +// message itself is streamed into the macro. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. + +enum GTestLogSeverity { + GTEST_INFO, + GTEST_WARNING, + GTEST_ERROR, + GTEST_FATAL +}; + +// Formats log entry severity, provides a stream object for streaming the +// log message, and terminates the message with a newline when going out of +// scope. +class GTEST_API_ GTestLog { + public: + GTestLog(GTestLogSeverity severity, const char* file, int line); + + // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. + ~GTestLog(); + + ::std::ostream& GetStream() { return ::std::cerr; } + + private: + const GTestLogSeverity severity_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog); +}; + +#if !defined(GTEST_LOG_) + +# define GTEST_LOG_(severity) \ + ::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \ + __FILE__, __LINE__).GetStream() + +inline void LogToStderr() {} +inline void FlushInfoLog() { fflush(NULL); } + +#endif // !defined(GTEST_LOG_) + +#if !defined(GTEST_CHECK_) +// INTERNAL IMPLEMENTATION - DO NOT USE. +// +// GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition +// is not satisfied. +// Synopsys: +// GTEST_CHECK_(boolean_condition); +// or +// GTEST_CHECK_(boolean_condition) << "Additional message"; +// +// This checks the condition and if the condition is not satisfied +// it prints message about the condition violation, including the +// condition itself, plus additional message streamed into it, if any, +// and then it aborts the program. It aborts the program irrespective of +// whether it is built in the debug mode or not. +# define GTEST_CHECK_(condition) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::IsTrue(condition)) \ + ; \ + else \ + GTEST_LOG_(FATAL) << "Condition " #condition " failed. " +#endif // !defined(GTEST_CHECK_) + +// An all-mode assert to verify that the given POSIX-style function +// call returns 0 (indicating success). Known limitation: this +// doesn't expand to a balanced 'if' statement, so enclose the macro +// in {} if you need to use it as the only statement in an 'if' +// branch. +#define GTEST_CHECK_POSIX_SUCCESS_(posix_call) \ + if (const int gtest_error = (posix_call)) \ + GTEST_LOG_(FATAL) << #posix_call << "failed with error " \ + << gtest_error + +#if GTEST_HAS_STD_MOVE_ +using std::move; +#else // GTEST_HAS_STD_MOVE_ +template +const T& move(const T& t) { + return t; +} +#endif // GTEST_HAS_STD_MOVE_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Use ImplicitCast_ as a safe version of static_cast for upcasting in +// the type hierarchy (e.g. casting a Foo* to a SuperclassOfFoo* or a +// const Foo*). When you use ImplicitCast_, the compiler checks that +// the cast is safe. Such explicit ImplicitCast_s are necessary in +// surprisingly many situations where C++ demands an exact type match +// instead of an argument type convertable to a target type. +// +// The syntax for using ImplicitCast_ is the same as for static_cast: +// +// ImplicitCast_(expr) +// +// ImplicitCast_ would have been part of the C++ standard library, +// but the proposal was submitted too late. It will probably make +// its way into the language in the future. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., implicit_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template +inline To ImplicitCast_(To x) { return x; } + +// When you upcast (that is, cast a pointer from type Foo to type +// SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts +// always succeed. When you downcast (that is, cast a pointer from +// type Foo to type SubclassOfFoo), static_cast<> isn't safe, because +// how do you know the pointer is really of type SubclassOfFoo? It +// could be a bare Foo, or of type DifferentSubclassOfFoo. Thus, +// when you downcast, you should use this macro. In debug mode, we +// use dynamic_cast<> to double-check the downcast is legal (we die +// if it's not). In normal mode, we do the efficient static_cast<> +// instead. Thus, it's important to test in debug mode to make sure +// the cast is legal! +// This is the only place in the code we should use dynamic_cast<>. +// In particular, you SHOULDN'T be using dynamic_cast<> in order to +// do RTTI (eg code like this: +// if (dynamic_cast(foo)) HandleASubclass1Object(foo); +// if (dynamic_cast(foo)) HandleASubclass2Object(foo); +// You should design the code some other way not to need this. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., down_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template // use like this: DownCast_(foo); +inline To DownCast_(From* f) { // so we only accept pointers + // Ensures that To is a sub-type of From *. This test is here only + // for compile-time type checking, and has no overhead in an + // optimized build at run-time, as it will be optimized away + // completely. + GTEST_INTENTIONAL_CONST_COND_PUSH_() + if (false) { + GTEST_INTENTIONAL_CONST_COND_POP_() + const To to = NULL; + ::testing::internal::ImplicitCast_(to); + } + +#if GTEST_HAS_RTTI + // RTTI: debug mode only! + GTEST_CHECK_(f == NULL || dynamic_cast(f) != NULL); +#endif + return static_cast(f); +} + +// Downcasts the pointer of type Base to Derived. +// Derived must be a subclass of Base. The parameter MUST +// point to a class of type Derived, not any subclass of it. +// When RTTI is available, the function performs a runtime +// check to enforce this. +template +Derived* CheckedDowncastToActualType(Base* base) { +#if GTEST_HAS_RTTI + GTEST_CHECK_(typeid(*base) == typeid(Derived)); +#endif + +#if GTEST_HAS_DOWNCAST_ + return ::down_cast(base); +#elif GTEST_HAS_RTTI + return dynamic_cast(base); // NOLINT +#else + return static_cast(base); // Poor man's downcast. +#endif +} + +#if GTEST_HAS_STREAM_REDIRECTION + +// Defines the stderr capturer: +// CaptureStdout - starts capturing stdout. +// GetCapturedStdout - stops capturing stdout and returns the captured string. +// CaptureStderr - starts capturing stderr. +// GetCapturedStderr - stops capturing stderr and returns the captured string. +// +GTEST_API_ void CaptureStdout(); +GTEST_API_ std::string GetCapturedStdout(); +GTEST_API_ void CaptureStderr(); +GTEST_API_ std::string GetCapturedStderr(); + +#endif // GTEST_HAS_STREAM_REDIRECTION + +// Returns a path to temporary directory. +GTEST_API_ std::string TempDir(); + +// Returns the size (in bytes) of a file. +GTEST_API_ size_t GetFileSize(FILE* file); + +// Reads the entire content of a file as a string. +GTEST_API_ std::string ReadEntireFile(FILE* file); + +// All command line arguments. +GTEST_API_ const ::std::vector& GetArgvs(); + +#if GTEST_HAS_DEATH_TEST + +const ::std::vector& GetInjectableArgvs(); +void SetInjectableArgvs(const ::std::vector* + new_argvs); + + +#endif // GTEST_HAS_DEATH_TEST + +// Defines synchronization primitives. +#if GTEST_IS_THREADSAFE +# if GTEST_HAS_PTHREAD +// Sleeps for (roughly) n milliseconds. This function is only for testing +// Google Test's own constructs. Don't use it in user tests, either +// directly or indirectly. +inline void SleepMilliseconds(int n) { + const timespec time = { + 0, // 0 seconds. + n * 1000L * 1000L, // And n ms. + }; + nanosleep(&time, NULL); +} +# endif // GTEST_HAS_PTHREAD + +# if GTEST_HAS_NOTIFICATION_ +// Notification has already been imported into the namespace. +// Nothing to do here. + +# elif GTEST_HAS_PTHREAD +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class Notification { + public: + Notification() : notified_(false) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + } + ~Notification() { + pthread_mutex_destroy(&mutex_); + } + + // Notifies all threads created with this notification to start. Must + // be called from the controller thread. + void Notify() { + pthread_mutex_lock(&mutex_); + notified_ = true; + pthread_mutex_unlock(&mutex_); + } + + // Blocks until the controller thread notifies. Must be called from a test + // thread. + void WaitForNotification() { + for (;;) { + pthread_mutex_lock(&mutex_); + const bool notified = notified_; + pthread_mutex_unlock(&mutex_); + if (notified) + break; + SleepMilliseconds(10); + } + } + + private: + pthread_mutex_t mutex_; + bool notified_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; + +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +GTEST_API_ void SleepMilliseconds(int n); + +// Provides leak-safe Windows kernel handle ownership. +// Used in death tests and in threading support. +class GTEST_API_ AutoHandle { + public: + // Assume that Win32 HANDLE type is equivalent to void*. Doing so allows us to + // avoid including in this header file. Including is + // undesirable because it defines a lot of symbols and macros that tend to + // conflict with client code. This assumption is verified by + // WindowsTypesTest.HANDLEIsVoidStar. + typedef void* Handle; + AutoHandle(); + explicit AutoHandle(Handle handle); + + ~AutoHandle(); + + Handle Get() const; + void Reset(); + void Reset(Handle handle); + + private: + // Returns true iff the handle is a valid handle object that can be closed. + bool IsCloseable() const; + + Handle handle_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle); +}; + +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class GTEST_API_ Notification { + public: + Notification(); + void Notify(); + void WaitForNotification(); + + private: + AutoHandle event_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; +# endif // GTEST_HAS_NOTIFICATION_ + +// On MinGW, we can have both GTEST_OS_WINDOWS and GTEST_HAS_PTHREAD +// defined, but we don't want to use MinGW's pthreads implementation, which +// has conformance problems with some versions of the POSIX standard. +# if GTEST_HAS_PTHREAD && !GTEST_OS_WINDOWS_MINGW + +// As a C-function, ThreadFuncWithCLinkage cannot be templated itself. +// Consequently, it cannot select a correct instantiation of ThreadWithParam +// in order to call its Run(). Introducing ThreadWithParamBase as a +// non-templated base class for ThreadWithParam allows us to bypass this +// problem. +class ThreadWithParamBase { + public: + virtual ~ThreadWithParamBase() {} + virtual void Run() = 0; +}; + +// pthread_create() accepts a pointer to a function type with the C linkage. +// According to the Standard (7.5/1), function types with different linkages +// are different even if they are otherwise identical. Some compilers (for +// example, SunStudio) treat them as different types. Since class methods +// cannot be defined with C-linkage we need to define a free C-function to +// pass into pthread_create(). +extern "C" inline void* ThreadFuncWithCLinkage(void* thread) { + static_cast(thread)->Run(); + return NULL; +} + +// Helper class for testing Google Test's multi-threading constructs. +// To use it, write: +// +// void ThreadFunc(int param) { /* Do things with param */ } +// Notification thread_can_start; +// ... +// // The thread_can_start parameter is optional; you can supply NULL. +// ThreadWithParam thread(&ThreadFunc, 5, &thread_can_start); +// thread_can_start.Notify(); +// +// These classes are only for testing Google Test's own constructs. Do +// not use them in user tests, either directly or indirectly. +template +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void UserThreadFunc(T); + + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) + : func_(func), + param_(param), + thread_can_start_(thread_can_start), + finished_(false) { + ThreadWithParamBase* const base = this; + // The thread can be created only after all fields except thread_ + // have been initialized. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base)); + } + ~ThreadWithParam() { Join(); } + + void Join() { + if (!finished_) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0)); + finished_ = true; + } + } + + virtual void Run() { + if (thread_can_start_ != NULL) + thread_can_start_->WaitForNotification(); + func_(param_); + } + + private: + UserThreadFunc* const func_; // User-supplied thread function. + const T param_; // User-supplied parameter to the thread function. + // When non-NULL, used to block execution until the controller thread + // notifies. + Notification* const thread_can_start_; + bool finished_; // true iff we know that the thread function has finished. + pthread_t thread_; // The native thread object. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; +# endif // !GTEST_OS_WINDOWS && GTEST_HAS_PTHREAD || + // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ + +# if GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ +// Mutex and ThreadLocal have already been imported into the namespace. +// Nothing to do here. + +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +// Mutex implements mutex on Windows platforms. It is used in conjunction +// with class MutexLock: +// +// Mutex mutex; +// ... +// MutexLock lock(&mutex); // Acquires the mutex and releases it at the +// // end of the current scope. +// +// A static Mutex *must* be defined or declared using one of the following +// macros: +// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex); +// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex); +// +// (A non-static Mutex is defined/declared in the usual way). +class GTEST_API_ Mutex { + public: + enum MutexType { kStatic = 0, kDynamic = 1 }; + // We rely on kStaticMutex being 0 as it is to what the linker initializes + // type_ in static mutexes. critical_section_ will be initialized lazily + // in ThreadSafeLazyInit(). + enum StaticConstructorSelector { kStaticMutex = 0 }; + + // This constructor intentionally does nothing. It relies on type_ being + // statically initialized to 0 (effectively setting it to kStatic) and on + // ThreadSafeLazyInit() to lazily initialize the rest of the members. + explicit Mutex(StaticConstructorSelector /*dummy*/) {} + + Mutex(); + ~Mutex(); + + void Lock(); + + void Unlock(); + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld(); + + private: + // Initializes owner_thread_id_ and critical_section_ in static mutexes. + void ThreadSafeLazyInit(); + + // Per http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx, + // we assume that 0 is an invalid value for thread IDs. + unsigned int owner_thread_id_; + + // For static mutexes, we rely on these members being initialized to zeros + // by the linker. + MutexType type_; + long critical_section_init_phase_; // NOLINT + GTEST_CRITICAL_SECTION* critical_section_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::Mutex mutex(::testing::internal::Mutex::kStaticMutex) + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + Mutex* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Base class for ValueHolder. Allows a caller to hold and delete a value +// without knowing its type. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Provides a way for a thread to send notifications to a ThreadLocal +// regardless of its parameter type. +class ThreadLocalBase { + public: + // Creates a new ValueHolder object holding a default value passed to + // this ThreadLocal's constructor and returns it. It is the caller's + // responsibility not to call this when the ThreadLocal instance already + // has a value on the current thread. + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const = 0; + + protected: + ThreadLocalBase() {} + virtual ~ThreadLocalBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocalBase); +}; + +// Maps a thread to a set of ThreadLocals that have values instantiated on that +// thread and notifies them when the thread exits. A ThreadLocal instance is +// expected to persist until all threads it has values on have terminated. +class GTEST_API_ ThreadLocalRegistry { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance); + + // Invoked when a ThreadLocal instance is destroyed. + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance); +}; + +class GTEST_API_ ThreadWithParamBase { + public: + void Join(); + + protected: + class Runnable { + public: + virtual ~Runnable() {} + virtual void Run() = 0; + }; + + ThreadWithParamBase(Runnable *runnable, Notification* thread_can_start); + virtual ~ThreadWithParamBase(); + + private: + AutoHandle thread_; +}; + +// Helper class for testing Google Test's multi-threading constructs. +template +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void UserThreadFunc(T); + + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) + : ThreadWithParamBase(new RunnableImpl(func, param), thread_can_start) { + } + virtual ~ThreadWithParam() {} + + private: + class RunnableImpl : public Runnable { + public: + RunnableImpl(UserThreadFunc* func, T param) + : func_(func), + param_(param) { + } + virtual ~RunnableImpl() {} + virtual void Run() { + func_(param_); + } + + private: + UserThreadFunc* const func_; + const T param_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(RunnableImpl); + }; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; + +// Implements thread-local storage on Windows systems. +// +// // Thread 1 +// ThreadLocal tl(100); // 100 is the default value for each thread. +// +// // Thread 2 +// tl.set(150); // Changes the value for thread 2 only. +// EXPECT_EQ(150, tl.get()); +// +// // Thread 1 +// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value. +// tl.set(200); +// EXPECT_EQ(200, tl.get()); +// +// The template type argument T must have a public copy constructor. +// In addition, the default ThreadLocal constructor requires T to have +// a public default constructor. +// +// The users of a TheadLocal instance have to make sure that all but one +// threads (including the main one) using that instance have exited before +// destroying it. Otherwise, the per-thread objects managed for them by the +// ThreadLocal instance are not guaranteed to be destroyed on all platforms. +// +// Google Test only uses global ThreadLocal objects. That means they +// will die after main() has returned. Therefore, no per-thread +// object managed by Google Test will be leaked as long as all threads +// using Google Test have exited when main() returns. +template +class ThreadLocal : public ThreadLocalBase { + public: + ThreadLocal() : default_factory_(new DefaultValueHolderFactory()) {} + explicit ThreadLocal(const T& value) + : default_factory_(new InstanceValueHolderFactory(value)) {} + + ~ThreadLocal() { ThreadLocalRegistry::OnThreadLocalDestroyed(this); } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of T. Can be deleted via its base class without the caller + // knowing the type of T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + ValueHolder() : value_() {} + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + + T* GetOrCreateValue() const { + return static_cast( + ThreadLocalRegistry::GetValueOnCurrentThread(this))->pointer(); + } + + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const { + return default_factory_->MakeNewHolder(); + } + + class ValueHolderFactory { + public: + ValueHolderFactory() {} + virtual ~ValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const = 0; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); + }; + + class DefaultValueHolderFactory : public ValueHolderFactory { + public: + DefaultValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); + }; + + class InstanceValueHolderFactory : public ValueHolderFactory { + public: + explicit InstanceValueHolderFactory(const T& value) : value_(value) {} + virtual ValueHolder* MakeNewHolder() const { + return new ValueHolder(value_); + } + + private: + const T value_; // The value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); + }; + + scoped_ptr default_factory_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# elif GTEST_HAS_PTHREAD + +// MutexBase and Mutex implement mutex on pthreads-based platforms. +class MutexBase { + public: + // Acquires this mutex. + void Lock() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_)); + owner_ = pthread_self(); + has_owner_ = true; + } + + // Releases this mutex. + void Unlock() { + // Since the lock is being released the owner_ field should no longer be + // considered valid. We don't protect writing to has_owner_ here, as it's + // the caller's responsibility to ensure that the current thread holds the + // mutex when this is called. + has_owner_ = false; + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_)); + } + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld() const { + GTEST_CHECK_(has_owner_ && pthread_equal(owner_, pthread_self())) + << "The current thread is not holding the mutex @" << this; + } + + // A static mutex may be used before main() is entered. It may even + // be used before the dynamic initialization stage. Therefore we + // must be able to initialize a static mutex object at link time. + // This means MutexBase has to be a POD and its member variables + // have to be public. + public: + pthread_mutex_t mutex_; // The underlying pthread mutex. + // has_owner_ indicates whether the owner_ field below contains a valid thread + // ID and is therefore safe to inspect (e.g., to use in pthread_equal()). All + // accesses to the owner_ field should be protected by a check of this field. + // An alternative might be to memset() owner_ to all zeros, but there's no + // guarantee that a zero'd pthread_t is necessarily invalid or even different + // from pthread_self(). + bool has_owner_; + pthread_t owner_; // The thread holding the mutex. +}; + +// Forward-declares a static mutex. +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::MutexBase mutex + +// Defines and statically (i.e. at link time) initializes a static mutex. +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, false, pthread_t() } + +// The Mutex class can only be used for mutexes created at runtime. It +// shares its API with MutexBase otherwise. +class Mutex : public MutexBase { + public: + Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + has_owner_ = false; + } + ~Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_)); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(MutexBase* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + MutexBase* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Helpers for ThreadLocal. + +// pthread_key_create() requires DeleteThreadLocalValue() to have +// C-linkage. Therefore it cannot be templatized to access +// ThreadLocal. Hence the need for class +// ThreadLocalValueHolderBase. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Called by pthread to delete thread-local data stored by +// pthread_setspecific(). +extern "C" inline void DeleteThreadLocalValue(void* value_holder) { + delete static_cast(value_holder); +} + +// Implements thread-local storage on pthreads-based systems. +template +class ThreadLocal { + public: + ThreadLocal() + : key_(CreateKey()), default_factory_(new DefaultValueHolderFactory()) {} + explicit ThreadLocal(const T& value) + : key_(CreateKey()), + default_factory_(new InstanceValueHolderFactory(value)) {} + + ~ThreadLocal() { + // Destroys the managed object for the current thread, if any. + DeleteThreadLocalValue(pthread_getspecific(key_)); + + // Releases resources associated with the key. This will *not* + // delete managed objects for other threads. + GTEST_CHECK_POSIX_SUCCESS_(pthread_key_delete(key_)); + } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of type T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + ValueHolder() : value_() {} + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + static pthread_key_t CreateKey() { + pthread_key_t key; + // When a thread exits, DeleteThreadLocalValue() will be called on + // the object managed for that thread. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_key_create(&key, &DeleteThreadLocalValue)); + return key; + } + + T* GetOrCreateValue() const { + ThreadLocalValueHolderBase* const holder = + static_cast(pthread_getspecific(key_)); + if (holder != NULL) { + return CheckedDowncastToActualType(holder)->pointer(); + } + + ValueHolder* const new_holder = default_factory_->MakeNewHolder(); + ThreadLocalValueHolderBase* const holder_base = new_holder; + GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base)); + return new_holder->pointer(); + } + + class ValueHolderFactory { + public: + ValueHolderFactory() {} + virtual ~ValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const = 0; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); + }; + + class DefaultValueHolderFactory : public ValueHolderFactory { + public: + DefaultValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); + }; + + class InstanceValueHolderFactory : public ValueHolderFactory { + public: + explicit InstanceValueHolderFactory(const T& value) : value_(value) {} + virtual ValueHolder* MakeNewHolder() const { + return new ValueHolder(value_); + } + + private: + const T value_; // The value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); + }; + + // A key pthreads uses for looking up per-thread values. + const pthread_key_t key_; + scoped_ptr default_factory_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# endif // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ + +#else // GTEST_IS_THREADSAFE + +// A dummy implementation of synchronization primitives (mutex, lock, +// and thread-local variable). Necessary for compiling Google Test where +// mutex is not supported - using Google Test in multiple threads is not +// supported on such platforms. + +class Mutex { + public: + Mutex() {} + void Lock() {} + void Unlock() {} + void AssertHeld() const {} +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex*) {} // NOLINT +}; + +typedef GTestMutexLock MutexLock; + +template +class ThreadLocal { + public: + ThreadLocal() : value_() {} + explicit ThreadLocal(const T& value) : value_(value) {} + T* pointer() { return &value_; } + const T* pointer() const { return &value_; } + const T& get() const { return value_; } + void set(const T& value) { value_ = value; } + private: + T value_; +}; + +#endif // GTEST_IS_THREADSAFE + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +GTEST_API_ size_t GetThreadCount(); + +// Passing non-POD classes through ellipsis (...) crashes the ARM +// compiler and generates a warning in Sun Studio. The Nokia Symbian +// and the IBM XL C/C++ compiler try to instantiate a copy constructor +// for objects passed through ellipsis (...), failing for uncopyable +// objects. We define this to ensure that only POD is passed through +// ellipsis on these systems. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) || defined(__SUNPRO_CC) +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_ELLIPSIS_NEEDS_POD_ 1 +#else +# define GTEST_CAN_COMPARE_NULL 1 +#endif + +// The Nokia Symbian and IBM XL C/C++ compilers cannot decide between +// const T& and const T* in a function template. These compilers +// _can_ decide between class template specializations for T and T*, +// so a tr1::type_traits-like is_pointer works. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) +# define GTEST_NEEDS_IS_POINTER_ 1 +#endif + +template +struct bool_constant { + typedef bool_constant type; + static const bool value = bool_value; +}; +template const bool bool_constant::value; + +typedef bool_constant false_type; +typedef bool_constant true_type; + +template +struct is_pointer : public false_type {}; + +template +struct is_pointer : public true_type {}; + +template +struct IteratorTraits { + typedef typename Iterator::value_type value_type; +}; + +template +struct IteratorTraits { + typedef T value_type; +}; + +template +struct IteratorTraits { + typedef T value_type; +}; + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_SEP_ "\\" +# define GTEST_HAS_ALT_PATH_SEP_ 1 +// The biggest signed integer type the compiler supports. +typedef __int64 BiggestInt; +#else +# define GTEST_PATH_SEP_ "/" +# define GTEST_HAS_ALT_PATH_SEP_ 0 +typedef long long BiggestInt; // NOLINT +#endif // GTEST_OS_WINDOWS + +// Utilities for char. + +// isspace(int ch) and friends accept an unsigned char or EOF. char +// may be signed, depending on the compiler (or compiler flags). +// Therefore we need to cast a char to unsigned char before calling +// isspace(), etc. + +inline bool IsAlpha(char ch) { + return isalpha(static_cast(ch)) != 0; +} +inline bool IsAlNum(char ch) { + return isalnum(static_cast(ch)) != 0; +} +inline bool IsDigit(char ch) { + return isdigit(static_cast(ch)) != 0; +} +inline bool IsLower(char ch) { + return islower(static_cast(ch)) != 0; +} +inline bool IsSpace(char ch) { + return isspace(static_cast(ch)) != 0; +} +inline bool IsUpper(char ch) { + return isupper(static_cast(ch)) != 0; +} +inline bool IsXDigit(char ch) { + return isxdigit(static_cast(ch)) != 0; +} +inline bool IsXDigit(wchar_t ch) { + const unsigned char low_byte = static_cast(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} + +inline char ToLower(char ch) { + return static_cast(tolower(static_cast(ch))); +} +inline char ToUpper(char ch) { + return static_cast(toupper(static_cast(ch))); +} + +inline std::string StripTrailingSpaces(std::string str) { + std::string::iterator it = str.end(); + while (it != str.begin() && IsSpace(*--it)) + it = str.erase(it); + return str; +} + +// The testing::internal::posix namespace holds wrappers for common +// POSIX functions. These wrappers hide the differences between +// Windows/MSVC and POSIX systems. Since some compilers define these +// standard functions as macros, the wrapper cannot have the same name +// as the wrapped function. + +namespace posix { + +// Functions with a different name on Windows. + +#if GTEST_OS_WINDOWS + +typedef struct _stat StatStruct; + +# ifdef __BORLANDC__ +inline int IsATTY(int fd) { return isatty(fd); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +# else // !__BORLANDC__ +# if GTEST_OS_WINDOWS_MOBILE +inline int IsATTY(int /* fd */) { return 0; } +# else +inline int IsATTY(int fd) { return _isatty(fd); } +# endif // GTEST_OS_WINDOWS_MOBILE +inline int StrCaseCmp(const char* s1, const char* s2) { + return _stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return _strdup(src); } +# endif // __BORLANDC__ + +# if GTEST_OS_WINDOWS_MOBILE +inline int FileNo(FILE* file) { return reinterpret_cast(_fileno(file)); } +// Stat(), RmDir(), and IsDir() are not needed on Windows CE at this +// time and thus not defined there. +# else +inline int FileNo(FILE* file) { return _fileno(file); } +inline int Stat(const char* path, StatStruct* buf) { return _stat(path, buf); } +inline int RmDir(const char* dir) { return _rmdir(dir); } +inline bool IsDir(const StatStruct& st) { + return (_S_IFDIR & st.st_mode) != 0; +} +# endif // GTEST_OS_WINDOWS_MOBILE + +#else + +typedef struct stat StatStruct; + +inline int FileNo(FILE* file) { return fileno(file); } +inline int IsATTY(int fd) { return isatty(fd); } +inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return strcasecmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +inline int RmDir(const char* dir) { return rmdir(dir); } +inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } + +#endif // GTEST_OS_WINDOWS + +// Functions deprecated by MSVC 8.0. + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996 /* deprecated function */) + +inline const char* StrNCpy(char* dest, const char* src, size_t n) { + return strncpy(dest, src, n); +} + +// ChDir(), FReopen(), FDOpen(), Read(), Write(), Close(), and +// StrError() aren't needed on Windows CE at this time and thus not +// defined there. + +#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT +inline int ChDir(const char* dir) { return chdir(dir); } +#endif +inline FILE* FOpen(const char* path, const char* mode) { + return fopen(path, mode); +} +#if !GTEST_OS_WINDOWS_MOBILE +inline FILE *FReopen(const char* path, const char* mode, FILE* stream) { + return freopen(path, mode, stream); +} +inline FILE* FDOpen(int fd, const char* mode) { return fdopen(fd, mode); } +#endif +inline int FClose(FILE* fp) { return fclose(fp); } +#if !GTEST_OS_WINDOWS_MOBILE +inline int Read(int fd, void* buf, unsigned int count) { + return static_cast(read(fd, buf, count)); +} +inline int Write(int fd, const void* buf, unsigned int count) { + return static_cast(write(fd, buf, count)); +} +inline int Close(int fd) { return close(fd); } +inline const char* StrError(int errnum) { return strerror(errnum); } +#endif +inline const char* GetEnv(const char* name) { +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE | GTEST_OS_WINDOWS_RT + // We are on Windows CE, which has no environment variables. + static_cast(name); // To prevent 'unused argument' warning. + return NULL; +#elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) + // Environment variables which we programmatically clear will be set to the + // empty string rather than unset (NULL). Handle that case. + const char* const env = getenv(name); + return (env != NULL && env[0] != '\0') ? env : NULL; +#else + return getenv(name); +#endif +} + +GTEST_DISABLE_MSC_WARNINGS_POP_() + +#if GTEST_OS_WINDOWS_MOBILE +// Windows CE has no C library. The abort() function is used in +// several places in Google Test. This implementation provides a reasonable +// imitation of standard behaviour. +void Abort(); +#else +inline void Abort() { abort(); } +#endif // GTEST_OS_WINDOWS_MOBILE + +} // namespace posix + +// MSVC "deprecates" snprintf and issues warnings wherever it is used. In +// order to avoid these warnings, we need to use _snprintf or _snprintf_s on +// MSVC-based platforms. We map the GTEST_SNPRINTF_ macro to the appropriate +// function in order to achieve that. We use macro definition here because +// snprintf is a variadic function. +#if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE +// MSVC 2005 and above support variadic macros. +# define GTEST_SNPRINTF_(buffer, size, format, ...) \ + _snprintf_s(buffer, size, size, format, __VA_ARGS__) +#elif defined(_MSC_VER) +// Windows CE does not define _snprintf_s and MSVC prior to 2005 doesn't +// complain about _snprintf. +# define GTEST_SNPRINTF_ _snprintf +#else +# define GTEST_SNPRINTF_ snprintf +#endif + +// The maximum number a BiggestInt can represent. This definition +// works no matter BiggestInt is represented in one's complement or +// two's complement. +// +// We cannot rely on numeric_limits in STL, as __int64 and long long +// are not part of standard C++ and numeric_limits doesn't need to be +// defined for them. +const BiggestInt kMaxBiggestInt = + ~(static_cast(1) << (8*sizeof(BiggestInt) - 1)); + +// This template class serves as a compile-time function from size to +// type. It maps a size in bytes to a primitive type with that +// size. e.g. +// +// TypeWithSize<4>::UInt +// +// is typedef-ed to be unsigned int (unsigned integer made up of 4 +// bytes). +// +// Such functionality should belong to STL, but I cannot find it +// there. +// +// Google Test uses this class in the implementation of floating-point +// comparison. +// +// For now it only handles UInt (unsigned int) as that's all Google Test +// needs. Other types can be easily added in the future if need +// arises. +template +class TypeWithSize { + public: + // This prevents the user from using TypeWithSize with incorrect + // values of N. + typedef void UInt; +}; + +// The specialization for size 4. +template <> +class TypeWithSize<4> { + public: + // unsigned int has size 4 in both gcc and MSVC. + // + // As base/basictypes.h doesn't compile on Windows, we cannot use + // uint32, uint64, and etc here. + typedef int Int; + typedef unsigned int UInt; +}; + +// The specialization for size 8. +template <> +class TypeWithSize<8> { + public: +#if GTEST_OS_WINDOWS + typedef __int64 Int; + typedef unsigned __int64 UInt; +#else + typedef long long Int; // NOLINT + typedef unsigned long long UInt; // NOLINT +#endif // GTEST_OS_WINDOWS +}; + +// Integer types of known sizes. +typedef TypeWithSize<4>::Int Int32; +typedef TypeWithSize<4>::UInt UInt32; +typedef TypeWithSize<8>::Int Int64; +typedef TypeWithSize<8>::UInt UInt64; +typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds. + +// Utilities for command line flags and environment variables. + +// Macro for referencing flags. +#if !defined(GTEST_FLAG) +# define GTEST_FLAG(name) FLAGS_gtest_##name +#endif // !defined(GTEST_FLAG) + +#if !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) +# define GTEST_USE_OWN_FLAGFILE_FLAG_ 1 +#endif // !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) + +#if !defined(GTEST_DECLARE_bool_) +# define GTEST_FLAG_SAVER_ ::testing::internal::GTestFlagSaver + +// Macros for declaring flags. +# define GTEST_DECLARE_bool_(name) GTEST_API_ extern bool GTEST_FLAG(name) +# define GTEST_DECLARE_int32_(name) \ + GTEST_API_ extern ::testing::internal::Int32 GTEST_FLAG(name) +#define GTEST_DECLARE_string_(name) \ + GTEST_API_ extern ::std::string GTEST_FLAG(name) + +// Macros for defining flags. +#define GTEST_DEFINE_bool_(name, default_val, doc) \ + GTEST_API_ bool GTEST_FLAG(name) = (default_val) +#define GTEST_DEFINE_int32_(name, default_val, doc) \ + GTEST_API_ ::testing::internal::Int32 GTEST_FLAG(name) = (default_val) +#define GTEST_DEFINE_string_(name, default_val, doc) \ + GTEST_API_ ::std::string GTEST_FLAG(name) = (default_val) + +#endif // !defined(GTEST_DECLARE_bool_) + +// Thread annotations +#if !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) +# define GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks) +# define GTEST_LOCK_EXCLUDED_(locks) +#endif // !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) + +// Parses 'str' for a 32-bit signed integer. If successful, writes the result +// to *value and returns true; otherwise leaves *value unchanged and returns +// false. +// TODO(chandlerc): Find a better way to refactor flag and environment parsing +// out of both gtest-port.cc and gtest.cc to avoid exporting this utility +// function. +bool ParseInt32(const Message& src_text, const char* str, Int32* value); + +// Parses a bool/Int32/string from the environment variable +// corresponding to the given Google Test flag. +bool BoolFromGTestEnv(const char* flag, bool default_val); +GTEST_API_ Int32 Int32FromGTestEnv(const char* flag, Int32 default_val); +std::string StringFromGTestEnv(const char* flag, const char* default_val); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-string.h b/mypyc/external/googletest/include/gtest/internal/gtest-string.h new file mode 100644 index 000000000000..97f1a7fdd2c0 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-string.h @@ -0,0 +1,167 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file declares the String class and functions used internally by +// Google Test. They are subject to change without notice. They should not used +// by code external to Google Test. +// +// This header file is #included by . +// It should not be #included by other files. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ + +#ifdef __BORLANDC__ +// string.h is not guaranteed to provide strcpy on C++ Builder. +# include +#endif + +#include +#include + +#include "gtest/internal/gtest-port.h" + +namespace testing { +namespace internal { + +// String - an abstract class holding static string utilities. +class GTEST_API_ String { + public: + // Static utility methods + + // Clones a 0-terminated C string, allocating memory using new. The + // caller is responsible for deleting the return value using + // delete[]. Returns the cloned string, or NULL if the input is + // NULL. + // + // This is different from strdup() in string.h, which allocates + // memory using malloc(). + static const char* CloneCString(const char* c_str); + +#if GTEST_OS_WINDOWS_MOBILE + // Windows CE does not have the 'ANSI' versions of Win32 APIs. To be + // able to pass strings to Win32 APIs on CE we need to convert them + // to 'Unicode', UTF-16. + + // Creates a UTF-16 wide string from the given ANSI string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the wide string, or NULL if the + // input is NULL. + // + // The wide string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static LPCWSTR AnsiToUtf16(const char* c_str); + + // Creates an ANSI string from the given wide string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the ANSI string, or NULL if the + // input is NULL. + // + // The returned string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static const char* Utf16ToAnsi(LPCWSTR utf16_str); +#endif + + // Compares two C strings. Returns true iff they have the same content. + // + // Unlike strcmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CStringEquals(const char* lhs, const char* rhs); + + // Converts a wide C string to a String using the UTF-8 encoding. + // NULL will be converted to "(null)". If an error occurred during + // the conversion, "(failed to convert from wide string)" is + // returned. + static std::string ShowWideCString(const wchar_t* wide_c_str); + + // Compares two wide C strings. Returns true iff they have the same + // content. + // + // Unlike wcscmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool WideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); + + // Compares two C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike strcasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CaseInsensitiveCStringEquals(const char* lhs, + const char* rhs); + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. + static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs); + + // Returns true iff the given string ends with the given suffix, ignoring + // case. Any string is considered to end with an empty suffix. + static bool EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix); + + // Formats an int value as "%02d". + static std::string FormatIntWidth2(int value); // "%02d" for width == 2 + + // Formats an int value as "%X". + static std::string FormatHexInt(int value); + + // Formats a byte as "%02X". + static std::string FormatByte(unsigned char value); + + private: + String(); // Not meant to be instantiated. +}; // class String + +// Gets the content of the stringstream's buffer as an std::string. Each '\0' +// character in the buffer is replaced with "\\0". +GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-tuple.h b/mypyc/external/googletest/include/gtest/internal/gtest-tuple.h new file mode 100644 index 000000000000..e9b405340a85 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-tuple.h @@ -0,0 +1,1020 @@ +// This file was GENERATED by command: +// pump.py gtest-tuple.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2009 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Implements a subset of TR1 tuple needed by Google Test and Google Mock. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ + +#include // For ::std::pair. + +// The compiler used in Symbian has a bug that prevents us from declaring the +// tuple template as a friend (it complains that tuple is redefined). This +// hack bypasses the bug by declaring the members that should otherwise be +// private as public. +// Sun Studio versions < 12 also have the above bug. +#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590) +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public: +#else +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \ + template friend class tuple; \ + private: +#endif + +// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict +// with our own definitions. Therefore using our own tuple does not work on +// those compilers. +#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */ +# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \ +GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers." +#endif + +// GTEST_n_TUPLE_(T) is the type of an n-tuple. +#define GTEST_0_TUPLE_(T) tuple<> +#define GTEST_1_TUPLE_(T) tuple +#define GTEST_2_TUPLE_(T) tuple +#define GTEST_3_TUPLE_(T) tuple +#define GTEST_4_TUPLE_(T) tuple +#define GTEST_5_TUPLE_(T) tuple +#define GTEST_6_TUPLE_(T) tuple +#define GTEST_7_TUPLE_(T) tuple +#define GTEST_8_TUPLE_(T) tuple +#define GTEST_9_TUPLE_(T) tuple +#define GTEST_10_TUPLE_(T) tuple + +// GTEST_n_TYPENAMES_(T) declares a list of n typenames. +#define GTEST_0_TYPENAMES_(T) +#define GTEST_1_TYPENAMES_(T) typename T##0 +#define GTEST_2_TYPENAMES_(T) typename T##0, typename T##1 +#define GTEST_3_TYPENAMES_(T) typename T##0, typename T##1, typename T##2 +#define GTEST_4_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3 +#define GTEST_5_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4 +#define GTEST_6_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5 +#define GTEST_7_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6 +#define GTEST_8_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, typename T##7 +#define GTEST_9_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8 +#define GTEST_10_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8, typename T##9 + +// In theory, defining stuff in the ::std namespace is undefined +// behavior. We can do this as we are playing the role of a standard +// library vendor. +namespace std { +namespace tr1 { + +template +class tuple; + +// Anything in namespace gtest_internal is Google Test's INTERNAL +// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code. +namespace gtest_internal { + +// ByRef::type is T if T is a reference; otherwise it's const T&. +template +struct ByRef { typedef const T& type; }; // NOLINT +template +struct ByRef { typedef T& type; }; // NOLINT + +// A handy wrapper for ByRef. +#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef::type + +// AddRef::type is T if T is a reference; otherwise it's T&. This +// is the same as tr1::add_reference::type. +template +struct AddRef { typedef T& type; }; // NOLINT +template +struct AddRef { typedef T& type; }; // NOLINT + +// A handy wrapper for AddRef. +#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef::type + +// A helper for implementing get(). +template class Get; + +// A helper for implementing tuple_element. kIndexValid is true +// iff k < the number of fields in tuple type T. +template +struct TupleElement; + +template +struct TupleElement { + typedef T0 type; +}; + +template +struct TupleElement { + typedef T1 type; +}; + +template +struct TupleElement { + typedef T2 type; +}; + +template +struct TupleElement { + typedef T3 type; +}; + +template +struct TupleElement { + typedef T4 type; +}; + +template +struct TupleElement { + typedef T5 type; +}; + +template +struct TupleElement { + typedef T6 type; +}; + +template +struct TupleElement { + typedef T7 type; +}; + +template +struct TupleElement { + typedef T8 type; +}; + +template +struct TupleElement { + typedef T9 type; +}; + +} // namespace gtest_internal + +template <> +class tuple<> { + public: + tuple() {} + tuple(const tuple& /* t */) {} + tuple& operator=(const tuple& /* t */) { return *this; } +}; + +template +class GTEST_1_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0) : f0_(f0) {} + + tuple(const tuple& t) : f0_(t.f0_) {} + + template + tuple(const GTEST_1_TUPLE_(U)& t) : f0_(t.f0_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_1_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_1_TUPLE_(U)& t) { + f0_ = t.f0_; + return *this; + } + + T0 f0_; +}; + +template +class GTEST_2_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1) : f0_(f0), + f1_(f1) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_) {} + + template + tuple(const GTEST_2_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_) {} + template + tuple(const ::std::pair& p) : f0_(p.first), f1_(p.second) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_2_TUPLE_(U)& t) { + return CopyFrom(t); + } + template + tuple& operator=(const ::std::pair& p) { + f0_ = p.first; + f1_ = p.second; + return *this; + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_2_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + return *this; + } + + T0 f0_; + T1 f1_; +}; + +template +class GTEST_3_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2) : f0_(f0), f1_(f1), f2_(f2) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + template + tuple(const GTEST_3_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_3_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_3_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; +}; + +template +class GTEST_4_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_) {} + + template + tuple(const GTEST_4_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_4_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_4_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; +}; + +template +class GTEST_5_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, + GTEST_BY_REF_(T4) f4) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_) {} + + template + tuple(const GTEST_5_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_5_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_5_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; +}; + +template +class GTEST_6_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_) {} + + template + tuple(const GTEST_6_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_6_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_6_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; +}; + +template +class GTEST_7_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + template + tuple(const GTEST_7_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_7_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_7_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; +}; + +template +class GTEST_8_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, + GTEST_BY_REF_(T7) f7) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + template + tuple(const GTEST_8_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_8_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_8_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; +}; + +template +class GTEST_9_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7), f8_(f8) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + template + tuple(const GTEST_9_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_9_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_9_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; +}; + +template +class tuple { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_(), + f9_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8, GTEST_BY_REF_(T9) f9) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6), f7_(f7), f8_(f8), f9_(f9) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), f9_(t.f9_) {} + + template + tuple(const GTEST_10_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), + f9_(t.f9_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_10_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_10_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + f9_ = t.f9_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; + T9 f9_; +}; + +// 6.1.3.2 Tuple creation functions. + +// Known limitations: we don't support passing an +// std::tr1::reference_wrapper to make_tuple(). And we don't +// implement tie(). + +inline tuple<> make_tuple() { return tuple<>(); } + +template +inline GTEST_1_TUPLE_(T) make_tuple(const T0& f0) { + return GTEST_1_TUPLE_(T)(f0); +} + +template +inline GTEST_2_TUPLE_(T) make_tuple(const T0& f0, const T1& f1) { + return GTEST_2_TUPLE_(T)(f0, f1); +} + +template +inline GTEST_3_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2) { + return GTEST_3_TUPLE_(T)(f0, f1, f2); +} + +template +inline GTEST_4_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3) { + return GTEST_4_TUPLE_(T)(f0, f1, f2, f3); +} + +template +inline GTEST_5_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4) { + return GTEST_5_TUPLE_(T)(f0, f1, f2, f3, f4); +} + +template +inline GTEST_6_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5) { + return GTEST_6_TUPLE_(T)(f0, f1, f2, f3, f4, f5); +} + +template +inline GTEST_7_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6) { + return GTEST_7_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6); +} + +template +inline GTEST_8_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7) { + return GTEST_8_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7); +} + +template +inline GTEST_9_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8) { + return GTEST_9_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8); +} + +template +inline GTEST_10_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8, const T9& f9) { + return GTEST_10_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9); +} + +// 6.1.3.3 Tuple helper classes. + +template struct tuple_size; + +template +struct tuple_size { + static const int value = 0; +}; + +template +struct tuple_size { + static const int value = 1; +}; + +template +struct tuple_size { + static const int value = 2; +}; + +template +struct tuple_size { + static const int value = 3; +}; + +template +struct tuple_size { + static const int value = 4; +}; + +template +struct tuple_size { + static const int value = 5; +}; + +template +struct tuple_size { + static const int value = 6; +}; + +template +struct tuple_size { + static const int value = 7; +}; + +template +struct tuple_size { + static const int value = 8; +}; + +template +struct tuple_size { + static const int value = 9; +}; + +template +struct tuple_size { + static const int value = 10; +}; + +template +struct tuple_element { + typedef typename gtest_internal::TupleElement< + k < (tuple_size::value), k, Tuple>::type type; +}; + +#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element::type + +// 6.1.3.4 Element access. + +namespace gtest_internal { + +template <> +class Get<0> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + Field(Tuple& t) { return t.f0_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + ConstField(const Tuple& t) { return t.f0_; } +}; + +template <> +class Get<1> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + Field(Tuple& t) { return t.f1_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + ConstField(const Tuple& t) { return t.f1_; } +}; + +template <> +class Get<2> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + Field(Tuple& t) { return t.f2_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + ConstField(const Tuple& t) { return t.f2_; } +}; + +template <> +class Get<3> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + Field(Tuple& t) { return t.f3_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + ConstField(const Tuple& t) { return t.f3_; } +}; + +template <> +class Get<4> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + Field(Tuple& t) { return t.f4_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + ConstField(const Tuple& t) { return t.f4_; } +}; + +template <> +class Get<5> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + Field(Tuple& t) { return t.f5_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + ConstField(const Tuple& t) { return t.f5_; } +}; + +template <> +class Get<6> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + Field(Tuple& t) { return t.f6_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + ConstField(const Tuple& t) { return t.f6_; } +}; + +template <> +class Get<7> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + Field(Tuple& t) { return t.f7_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + ConstField(const Tuple& t) { return t.f7_; } +}; + +template <> +class Get<8> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + Field(Tuple& t) { return t.f8_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + ConstField(const Tuple& t) { return t.f8_; } +}; + +template <> +class Get<9> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + Field(Tuple& t) { return t.f9_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + ConstField(const Tuple& t) { return t.f9_; } +}; + +} // namespace gtest_internal + +template +GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::Field(t); +} + +template +GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(const GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::ConstField(t); +} + +// 6.1.3.5 Relational operators + +// We only implement == and !=, as we don't have a need for the rest yet. + +namespace gtest_internal { + +// SameSizeTuplePrefixComparator::Eq(t1, t2) returns true if the +// first k fields of t1 equals the first k fields of t2. +// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if +// k1 != k2. +template +struct SameSizeTuplePrefixComparator; + +template <> +struct SameSizeTuplePrefixComparator<0, 0> { + template + static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) { + return true; + } +}; + +template +struct SameSizeTuplePrefixComparator { + template + static bool Eq(const Tuple1& t1, const Tuple2& t2) { + return SameSizeTuplePrefixComparator::Eq(t1, t2) && + ::std::tr1::get(t1) == ::std::tr1::get(t2); + } +}; + +} // namespace gtest_internal + +template +inline bool operator==(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { + return gtest_internal::SameSizeTuplePrefixComparator< + tuple_size::value, + tuple_size::value>::Eq(t, u); +} + +template +inline bool operator!=(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { return !(t == u); } + +// 6.1.4 Pairs. +// Unimplemented. + +} // namespace tr1 +} // namespace std + +#undef GTEST_0_TUPLE_ +#undef GTEST_1_TUPLE_ +#undef GTEST_2_TUPLE_ +#undef GTEST_3_TUPLE_ +#undef GTEST_4_TUPLE_ +#undef GTEST_5_TUPLE_ +#undef GTEST_6_TUPLE_ +#undef GTEST_7_TUPLE_ +#undef GTEST_8_TUPLE_ +#undef GTEST_9_TUPLE_ +#undef GTEST_10_TUPLE_ + +#undef GTEST_0_TYPENAMES_ +#undef GTEST_1_TYPENAMES_ +#undef GTEST_2_TYPENAMES_ +#undef GTEST_3_TYPENAMES_ +#undef GTEST_4_TYPENAMES_ +#undef GTEST_5_TYPENAMES_ +#undef GTEST_6_TYPENAMES_ +#undef GTEST_7_TYPENAMES_ +#undef GTEST_8_TYPENAMES_ +#undef GTEST_9_TYPENAMES_ +#undef GTEST_10_TYPENAMES_ + +#undef GTEST_DECLARE_TUPLE_AS_FRIEND_ +#undef GTEST_BY_REF_ +#undef GTEST_ADD_REF_ +#undef GTEST_TUPLE_ELEMENT_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-tuple.h.pump b/mypyc/external/googletest/include/gtest/internal/gtest-tuple.h.pump new file mode 100644 index 000000000000..429ddfeecaa7 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-tuple.h.pump @@ -0,0 +1,347 @@ +$$ -*- mode: c++; -*- +$var n = 10 $$ Maximum number of tuple fields we want to support. +$$ This meta comment fixes auto-indentation in Emacs. }} +// Copyright 2009 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Implements a subset of TR1 tuple needed by Google Test and Google Mock. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ + +#include // For ::std::pair. + +// The compiler used in Symbian has a bug that prevents us from declaring the +// tuple template as a friend (it complains that tuple is redefined). This +// hack bypasses the bug by declaring the members that should otherwise be +// private as public. +// Sun Studio versions < 12 also have the above bug. +#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590) +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public: +#else +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \ + template friend class tuple; \ + private: +#endif + +// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict +// with our own definitions. Therefore using our own tuple does not work on +// those compilers. +#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */ +# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \ +GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers." +#endif + + +$range i 0..n-1 +$range j 0..n +$range k 1..n +// GTEST_n_TUPLE_(T) is the type of an n-tuple. +#define GTEST_0_TUPLE_(T) tuple<> + +$for k [[ +$range m 0..k-1 +$range m2 k..n-1 +#define GTEST_$(k)_TUPLE_(T) tuple<$for m, [[T##$m]]$for m2 [[, void]]> + +]] + +// GTEST_n_TYPENAMES_(T) declares a list of n typenames. + +$for j [[ +$range m 0..j-1 +#define GTEST_$(j)_TYPENAMES_(T) $for m, [[typename T##$m]] + + +]] + +// In theory, defining stuff in the ::std namespace is undefined +// behavior. We can do this as we are playing the role of a standard +// library vendor. +namespace std { +namespace tr1 { + +template <$for i, [[typename T$i = void]]> +class tuple; + +// Anything in namespace gtest_internal is Google Test's INTERNAL +// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code. +namespace gtest_internal { + +// ByRef::type is T if T is a reference; otherwise it's const T&. +template +struct ByRef { typedef const T& type; }; // NOLINT +template +struct ByRef { typedef T& type; }; // NOLINT + +// A handy wrapper for ByRef. +#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef::type + +// AddRef::type is T if T is a reference; otherwise it's T&. This +// is the same as tr1::add_reference::type. +template +struct AddRef { typedef T& type; }; // NOLINT +template +struct AddRef { typedef T& type; }; // NOLINT + +// A handy wrapper for AddRef. +#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef::type + +// A helper for implementing get(). +template class Get; + +// A helper for implementing tuple_element. kIndexValid is true +// iff k < the number of fields in tuple type T. +template +struct TupleElement; + + +$for i [[ +template +struct TupleElement { + typedef T$i type; +}; + + +]] +} // namespace gtest_internal + +template <> +class tuple<> { + public: + tuple() {} + tuple(const tuple& /* t */) {} + tuple& operator=(const tuple& /* t */) { return *this; } +}; + + +$for k [[ +$range m 0..k-1 +template +class $if k < n [[GTEST_$(k)_TUPLE_(T)]] $else [[tuple]] { + public: + template friend class gtest_internal::Get; + + tuple() : $for m, [[f$(m)_()]] {} + + explicit tuple($for m, [[GTEST_BY_REF_(T$m) f$m]]) : [[]] +$for m, [[f$(m)_(f$m)]] {} + + tuple(const tuple& t) : $for m, [[f$(m)_(t.f$(m)_)]] {} + + template + tuple(const GTEST_$(k)_TUPLE_(U)& t) : $for m, [[f$(m)_(t.f$(m)_)]] {} + +$if k == 2 [[ + template + tuple(const ::std::pair& p) : f0_(p.first), f1_(p.second) {} + +]] + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_$(k)_TUPLE_(U)& t) { + return CopyFrom(t); + } + +$if k == 2 [[ + template + tuple& operator=(const ::std::pair& p) { + f0_ = p.first; + f1_ = p.second; + return *this; + } + +]] + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_$(k)_TUPLE_(U)& t) { + +$for m [[ + f$(m)_ = t.f$(m)_; + +]] + return *this; + } + + +$for m [[ + T$m f$(m)_; + +]] +}; + + +]] +// 6.1.3.2 Tuple creation functions. + +// Known limitations: we don't support passing an +// std::tr1::reference_wrapper to make_tuple(). And we don't +// implement tie(). + +inline tuple<> make_tuple() { return tuple<>(); } + +$for k [[ +$range m 0..k-1 + +template +inline GTEST_$(k)_TUPLE_(T) make_tuple($for m, [[const T$m& f$m]]) { + return GTEST_$(k)_TUPLE_(T)($for m, [[f$m]]); +} + +]] + +// 6.1.3.3 Tuple helper classes. + +template struct tuple_size; + + +$for j [[ +template +struct tuple_size { + static const int value = $j; +}; + + +]] +template +struct tuple_element { + typedef typename gtest_internal::TupleElement< + k < (tuple_size::value), k, Tuple>::type type; +}; + +#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element::type + +// 6.1.3.4 Element access. + +namespace gtest_internal { + + +$for i [[ +template <> +class Get<$i> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_($i, Tuple)) + Field(Tuple& t) { return t.f$(i)_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_($i, Tuple)) + ConstField(const Tuple& t) { return t.f$(i)_; } +}; + + +]] +} // namespace gtest_internal + +template +GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_$(n)_TUPLE_(T))) +get(GTEST_$(n)_TUPLE_(T)& t) { + return gtest_internal::Get::Field(t); +} + +template +GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_$(n)_TUPLE_(T))) +get(const GTEST_$(n)_TUPLE_(T)& t) { + return gtest_internal::Get::ConstField(t); +} + +// 6.1.3.5 Relational operators + +// We only implement == and !=, as we don't have a need for the rest yet. + +namespace gtest_internal { + +// SameSizeTuplePrefixComparator::Eq(t1, t2) returns true if the +// first k fields of t1 equals the first k fields of t2. +// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if +// k1 != k2. +template +struct SameSizeTuplePrefixComparator; + +template <> +struct SameSizeTuplePrefixComparator<0, 0> { + template + static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) { + return true; + } +}; + +template +struct SameSizeTuplePrefixComparator { + template + static bool Eq(const Tuple1& t1, const Tuple2& t2) { + return SameSizeTuplePrefixComparator::Eq(t1, t2) && + ::std::tr1::get(t1) == ::std::tr1::get(t2); + } +}; + +} // namespace gtest_internal + +template +inline bool operator==(const GTEST_$(n)_TUPLE_(T)& t, + const GTEST_$(n)_TUPLE_(U)& u) { + return gtest_internal::SameSizeTuplePrefixComparator< + tuple_size::value, + tuple_size::value>::Eq(t, u); +} + +template +inline bool operator!=(const GTEST_$(n)_TUPLE_(T)& t, + const GTEST_$(n)_TUPLE_(U)& u) { return !(t == u); } + +// 6.1.4 Pairs. +// Unimplemented. + +} // namespace tr1 +} // namespace std + + +$for j [[ +#undef GTEST_$(j)_TUPLE_ + +]] + + +$for j [[ +#undef GTEST_$(j)_TYPENAMES_ + +]] + +#undef GTEST_DECLARE_TUPLE_AS_FRIEND_ +#undef GTEST_BY_REF_ +#undef GTEST_ADD_REF_ +#undef GTEST_TUPLE_ELEMENT_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-type-util.h b/mypyc/external/googletest/include/gtest/internal/gtest-type-util.h new file mode 100644 index 000000000000..e46f7cfcb483 --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-type-util.h @@ -0,0 +1,3331 @@ +// This file was GENERATED by command: +// pump.py gtest-type-util.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Type utilities needed for implementing typed and type-parameterized +// tests. This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently we support at most 50 types in a list, and at most 50 +// type-parameterized tests in one type-parameterized test case. +// Please contact googletestframework@googlegroups.com if you need +// more. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + +#include "gtest/internal/gtest-port.h" + +// #ifdef __GNUC__ is too general here. It is possible to use gcc without using +// libstdc++ (which is where cxxabi.h comes from). +# if GTEST_HAS_CXXABI_H_ +# include +# elif defined(__HP_aCC) +# include +# endif // GTEST_HASH_CXXABI_H_ + +namespace testing { +namespace internal { + +// GetTypeName() returns a human-readable name of type T. +// NB: This function is also used in Google Mock, so don't move it inside of +// the typed-test-only section below. +template +std::string GetTypeName() { +# if GTEST_HAS_RTTI + + const char* const name = typeid(T).name(); +# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +# if GTEST_HAS_CXXABI_H_ + using abi::__cxa_demangle; +# endif // GTEST_HAS_CXXABI_H_ + char* const readable_name = __cxa_demangle(name, 0, 0, &status); + const std::string name_str(status == 0 ? readable_name : name); + free(readable_name); + return name_str; +# else + return name; +# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC + +# else + + return ""; + +# endif // GTEST_HAS_RTTI +} + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// AssertyTypeEq::type is defined iff T1 and T2 are the same +// type. This can be used as a compile-time assertion to ensure that +// two types are equal. + +template +struct AssertTypeEq; + +template +struct AssertTypeEq { + typedef bool type; +}; + +// A unique type used as the default value for the arguments of class +// template Types. This allows us to simulate variadic templates +// (e.g. Types, Type, and etc), which C++ doesn't +// support directly. +struct None {}; + +// The following family of struct and struct templates are used to +// represent type lists. In particular, TypesN +// represents a type list with N types (T1, T2, ..., and TN) in it. +// Except for Types0, every struct in the family has two member types: +// Head for the first type in the list, and Tail for the rest of the +// list. + +// The empty type list. +struct Types0 {}; + +// Type lists of length 1, 2, 3, and so on. + +template +struct Types1 { + typedef T1 Head; + typedef Types0 Tail; +}; +template +struct Types2 { + typedef T1 Head; + typedef Types1 Tail; +}; + +template +struct Types3 { + typedef T1 Head; + typedef Types2 Tail; +}; + +template +struct Types4 { + typedef T1 Head; + typedef Types3 Tail; +}; + +template +struct Types5 { + typedef T1 Head; + typedef Types4 Tail; +}; + +template +struct Types6 { + typedef T1 Head; + typedef Types5 Tail; +}; + +template +struct Types7 { + typedef T1 Head; + typedef Types6 Tail; +}; + +template +struct Types8 { + typedef T1 Head; + typedef Types7 Tail; +}; + +template +struct Types9 { + typedef T1 Head; + typedef Types8 Tail; +}; + +template +struct Types10 { + typedef T1 Head; + typedef Types9 Tail; +}; + +template +struct Types11 { + typedef T1 Head; + typedef Types10 Tail; +}; + +template +struct Types12 { + typedef T1 Head; + typedef Types11 Tail; +}; + +template +struct Types13 { + typedef T1 Head; + typedef Types12 Tail; +}; + +template +struct Types14 { + typedef T1 Head; + typedef Types13 Tail; +}; + +template +struct Types15 { + typedef T1 Head; + typedef Types14 Tail; +}; + +template +struct Types16 { + typedef T1 Head; + typedef Types15 Tail; +}; + +template +struct Types17 { + typedef T1 Head; + typedef Types16 Tail; +}; + +template +struct Types18 { + typedef T1 Head; + typedef Types17 Tail; +}; + +template +struct Types19 { + typedef T1 Head; + typedef Types18 Tail; +}; + +template +struct Types20 { + typedef T1 Head; + typedef Types19 Tail; +}; + +template +struct Types21 { + typedef T1 Head; + typedef Types20 Tail; +}; + +template +struct Types22 { + typedef T1 Head; + typedef Types21 Tail; +}; + +template +struct Types23 { + typedef T1 Head; + typedef Types22 Tail; +}; + +template +struct Types24 { + typedef T1 Head; + typedef Types23 Tail; +}; + +template +struct Types25 { + typedef T1 Head; + typedef Types24 Tail; +}; + +template +struct Types26 { + typedef T1 Head; + typedef Types25 Tail; +}; + +template +struct Types27 { + typedef T1 Head; + typedef Types26 Tail; +}; + +template +struct Types28 { + typedef T1 Head; + typedef Types27 Tail; +}; + +template +struct Types29 { + typedef T1 Head; + typedef Types28 Tail; +}; + +template +struct Types30 { + typedef T1 Head; + typedef Types29 Tail; +}; + +template +struct Types31 { + typedef T1 Head; + typedef Types30 Tail; +}; + +template +struct Types32 { + typedef T1 Head; + typedef Types31 Tail; +}; + +template +struct Types33 { + typedef T1 Head; + typedef Types32 Tail; +}; + +template +struct Types34 { + typedef T1 Head; + typedef Types33 Tail; +}; + +template +struct Types35 { + typedef T1 Head; + typedef Types34 Tail; +}; + +template +struct Types36 { + typedef T1 Head; + typedef Types35 Tail; +}; + +template +struct Types37 { + typedef T1 Head; + typedef Types36 Tail; +}; + +template +struct Types38 { + typedef T1 Head; + typedef Types37 Tail; +}; + +template +struct Types39 { + typedef T1 Head; + typedef Types38 Tail; +}; + +template +struct Types40 { + typedef T1 Head; + typedef Types39 Tail; +}; + +template +struct Types41 { + typedef T1 Head; + typedef Types40 Tail; +}; + +template +struct Types42 { + typedef T1 Head; + typedef Types41 Tail; +}; + +template +struct Types43 { + typedef T1 Head; + typedef Types42 Tail; +}; + +template +struct Types44 { + typedef T1 Head; + typedef Types43 Tail; +}; + +template +struct Types45 { + typedef T1 Head; + typedef Types44 Tail; +}; + +template +struct Types46 { + typedef T1 Head; + typedef Types45 Tail; +}; + +template +struct Types47 { + typedef T1 Head; + typedef Types46 Tail; +}; + +template +struct Types48 { + typedef T1 Head; + typedef Types47 Tail; +}; + +template +struct Types49 { + typedef T1 Head; + typedef Types48 Tail; +}; + +template +struct Types50 { + typedef T1 Head; + typedef Types49 Tail; +}; + + +} // namespace internal + +// We don't want to require the users to write TypesN<...> directly, +// as that would require them to count the length. Types<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Types +// will appear as Types in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Types, and Google Test will translate +// that to TypesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Types template. +template +struct Types { + typedef internal::Types50 type; +}; + +template <> +struct Types { + typedef internal::Types0 type; +}; +template +struct Types { + typedef internal::Types1 type; +}; +template +struct Types { + typedef internal::Types2 type; +}; +template +struct Types { + typedef internal::Types3 type; +}; +template +struct Types { + typedef internal::Types4 type; +}; +template +struct Types { + typedef internal::Types5 type; +}; +template +struct Types { + typedef internal::Types6 type; +}; +template +struct Types { + typedef internal::Types7 type; +}; +template +struct Types { + typedef internal::Types8 type; +}; +template +struct Types { + typedef internal::Types9 type; +}; +template +struct Types { + typedef internal::Types10 type; +}; +template +struct Types { + typedef internal::Types11 type; +}; +template +struct Types { + typedef internal::Types12 type; +}; +template +struct Types { + typedef internal::Types13 type; +}; +template +struct Types { + typedef internal::Types14 type; +}; +template +struct Types { + typedef internal::Types15 type; +}; +template +struct Types { + typedef internal::Types16 type; +}; +template +struct Types { + typedef internal::Types17 type; +}; +template +struct Types { + typedef internal::Types18 type; +}; +template +struct Types { + typedef internal::Types19 type; +}; +template +struct Types { + typedef internal::Types20 type; +}; +template +struct Types { + typedef internal::Types21 type; +}; +template +struct Types { + typedef internal::Types22 type; +}; +template +struct Types { + typedef internal::Types23 type; +}; +template +struct Types { + typedef internal::Types24 type; +}; +template +struct Types { + typedef internal::Types25 type; +}; +template +struct Types { + typedef internal::Types26 type; +}; +template +struct Types { + typedef internal::Types27 type; +}; +template +struct Types { + typedef internal::Types28 type; +}; +template +struct Types { + typedef internal::Types29 type; +}; +template +struct Types { + typedef internal::Types30 type; +}; +template +struct Types { + typedef internal::Types31 type; +}; +template +struct Types { + typedef internal::Types32 type; +}; +template +struct Types { + typedef internal::Types33 type; +}; +template +struct Types { + typedef internal::Types34 type; +}; +template +struct Types { + typedef internal::Types35 type; +}; +template +struct Types { + typedef internal::Types36 type; +}; +template +struct Types { + typedef internal::Types37 type; +}; +template +struct Types { + typedef internal::Types38 type; +}; +template +struct Types { + typedef internal::Types39 type; +}; +template +struct Types { + typedef internal::Types40 type; +}; +template +struct Types { + typedef internal::Types41 type; +}; +template +struct Types { + typedef internal::Types42 type; +}; +template +struct Types { + typedef internal::Types43 type; +}; +template +struct Types { + typedef internal::Types44 type; +}; +template +struct Types { + typedef internal::Types45 type; +}; +template +struct Types { + typedef internal::Types46 type; +}; +template +struct Types { + typedef internal::Types47 type; +}; +template +struct Types { + typedef internal::Types48 type; +}; +template +struct Types { + typedef internal::Types49 type; +}; + +namespace internal { + +# define GTEST_TEMPLATE_ template class + +// The template "selector" struct TemplateSel is used to +// represent Tmpl, which must be a class template with one type +// parameter, as a type. TemplateSel::Bind::type is defined +// as the type Tmpl. This allows us to actually instantiate the +// template "selected" by TemplateSel. +// +// This trick is necessary for simulating typedef for class templates, +// which C++ doesn't support directly. +template +struct TemplateSel { + template + struct Bind { + typedef Tmpl type; + }; +}; + +# define GTEST_BIND_(TmplSel, T) \ + TmplSel::template Bind::type + +// A unique struct template used as the default value for the +// arguments of class template Templates. This allows us to simulate +// variadic templates (e.g. Templates, Templates, +// and etc), which C++ doesn't support directly. +template +struct NoneT {}; + +// The following family of struct and struct templates are used to +// represent template lists. In particular, TemplatesN represents a list of N templates (T1, T2, ..., and TN). Except +// for Templates0, every struct in the family has two member types: +// Head for the selector of the first template in the list, and Tail +// for the rest of the list. + +// The empty template list. +struct Templates0 {}; + +// Template lists of length 1, 2, 3, and so on. + +template +struct Templates1 { + typedef TemplateSel Head; + typedef Templates0 Tail; +}; +template +struct Templates2 { + typedef TemplateSel Head; + typedef Templates1 Tail; +}; + +template +struct Templates3 { + typedef TemplateSel Head; + typedef Templates2 Tail; +}; + +template +struct Templates4 { + typedef TemplateSel Head; + typedef Templates3 Tail; +}; + +template +struct Templates5 { + typedef TemplateSel Head; + typedef Templates4 Tail; +}; + +template +struct Templates6 { + typedef TemplateSel Head; + typedef Templates5 Tail; +}; + +template +struct Templates7 { + typedef TemplateSel Head; + typedef Templates6 Tail; +}; + +template +struct Templates8 { + typedef TemplateSel Head; + typedef Templates7 Tail; +}; + +template +struct Templates9 { + typedef TemplateSel Head; + typedef Templates8 Tail; +}; + +template +struct Templates10 { + typedef TemplateSel Head; + typedef Templates9 Tail; +}; + +template +struct Templates11 { + typedef TemplateSel Head; + typedef Templates10 Tail; +}; + +template +struct Templates12 { + typedef TemplateSel Head; + typedef Templates11 Tail; +}; + +template +struct Templates13 { + typedef TemplateSel Head; + typedef Templates12 Tail; +}; + +template +struct Templates14 { + typedef TemplateSel Head; + typedef Templates13 Tail; +}; + +template +struct Templates15 { + typedef TemplateSel Head; + typedef Templates14 Tail; +}; + +template +struct Templates16 { + typedef TemplateSel Head; + typedef Templates15 Tail; +}; + +template +struct Templates17 { + typedef TemplateSel Head; + typedef Templates16 Tail; +}; + +template +struct Templates18 { + typedef TemplateSel Head; + typedef Templates17 Tail; +}; + +template +struct Templates19 { + typedef TemplateSel Head; + typedef Templates18 Tail; +}; + +template +struct Templates20 { + typedef TemplateSel Head; + typedef Templates19 Tail; +}; + +template +struct Templates21 { + typedef TemplateSel Head; + typedef Templates20 Tail; +}; + +template +struct Templates22 { + typedef TemplateSel Head; + typedef Templates21 Tail; +}; + +template +struct Templates23 { + typedef TemplateSel Head; + typedef Templates22 Tail; +}; + +template +struct Templates24 { + typedef TemplateSel Head; + typedef Templates23 Tail; +}; + +template +struct Templates25 { + typedef TemplateSel Head; + typedef Templates24 Tail; +}; + +template +struct Templates26 { + typedef TemplateSel Head; + typedef Templates25 Tail; +}; + +template +struct Templates27 { + typedef TemplateSel Head; + typedef Templates26 Tail; +}; + +template +struct Templates28 { + typedef TemplateSel Head; + typedef Templates27 Tail; +}; + +template +struct Templates29 { + typedef TemplateSel Head; + typedef Templates28 Tail; +}; + +template +struct Templates30 { + typedef TemplateSel Head; + typedef Templates29 Tail; +}; + +template +struct Templates31 { + typedef TemplateSel Head; + typedef Templates30 Tail; +}; + +template +struct Templates32 { + typedef TemplateSel Head; + typedef Templates31 Tail; +}; + +template +struct Templates33 { + typedef TemplateSel Head; + typedef Templates32 Tail; +}; + +template +struct Templates34 { + typedef TemplateSel Head; + typedef Templates33 Tail; +}; + +template +struct Templates35 { + typedef TemplateSel Head; + typedef Templates34 Tail; +}; + +template +struct Templates36 { + typedef TemplateSel Head; + typedef Templates35 Tail; +}; + +template +struct Templates37 { + typedef TemplateSel Head; + typedef Templates36 Tail; +}; + +template +struct Templates38 { + typedef TemplateSel Head; + typedef Templates37 Tail; +}; + +template +struct Templates39 { + typedef TemplateSel Head; + typedef Templates38 Tail; +}; + +template +struct Templates40 { + typedef TemplateSel Head; + typedef Templates39 Tail; +}; + +template +struct Templates41 { + typedef TemplateSel Head; + typedef Templates40 Tail; +}; + +template +struct Templates42 { + typedef TemplateSel Head; + typedef Templates41 Tail; +}; + +template +struct Templates43 { + typedef TemplateSel Head; + typedef Templates42 Tail; +}; + +template +struct Templates44 { + typedef TemplateSel Head; + typedef Templates43 Tail; +}; + +template +struct Templates45 { + typedef TemplateSel Head; + typedef Templates44 Tail; +}; + +template +struct Templates46 { + typedef TemplateSel Head; + typedef Templates45 Tail; +}; + +template +struct Templates47 { + typedef TemplateSel Head; + typedef Templates46 Tail; +}; + +template +struct Templates48 { + typedef TemplateSel Head; + typedef Templates47 Tail; +}; + +template +struct Templates49 { + typedef TemplateSel Head; + typedef Templates48 Tail; +}; + +template +struct Templates50 { + typedef TemplateSel Head; + typedef Templates49 Tail; +}; + + +// We don't want to require the users to write TemplatesN<...> directly, +// as that would require them to count the length. Templates<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Templates +// will appear as Templates in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Templates, and Google Test will translate +// that to TemplatesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Templates template. +template +struct Templates { + typedef Templates50 type; +}; + +template <> +struct Templates { + typedef Templates0 type; +}; +template +struct Templates { + typedef Templates1 type; +}; +template +struct Templates { + typedef Templates2 type; +}; +template +struct Templates { + typedef Templates3 type; +}; +template +struct Templates { + typedef Templates4 type; +}; +template +struct Templates { + typedef Templates5 type; +}; +template +struct Templates { + typedef Templates6 type; +}; +template +struct Templates { + typedef Templates7 type; +}; +template +struct Templates { + typedef Templates8 type; +}; +template +struct Templates { + typedef Templates9 type; +}; +template +struct Templates { + typedef Templates10 type; +}; +template +struct Templates { + typedef Templates11 type; +}; +template +struct Templates { + typedef Templates12 type; +}; +template +struct Templates { + typedef Templates13 type; +}; +template +struct Templates { + typedef Templates14 type; +}; +template +struct Templates { + typedef Templates15 type; +}; +template +struct Templates { + typedef Templates16 type; +}; +template +struct Templates { + typedef Templates17 type; +}; +template +struct Templates { + typedef Templates18 type; +}; +template +struct Templates { + typedef Templates19 type; +}; +template +struct Templates { + typedef Templates20 type; +}; +template +struct Templates { + typedef Templates21 type; +}; +template +struct Templates { + typedef Templates22 type; +}; +template +struct Templates { + typedef Templates23 type; +}; +template +struct Templates { + typedef Templates24 type; +}; +template +struct Templates { + typedef Templates25 type; +}; +template +struct Templates { + typedef Templates26 type; +}; +template +struct Templates { + typedef Templates27 type; +}; +template +struct Templates { + typedef Templates28 type; +}; +template +struct Templates { + typedef Templates29 type; +}; +template +struct Templates { + typedef Templates30 type; +}; +template +struct Templates { + typedef Templates31 type; +}; +template +struct Templates { + typedef Templates32 type; +}; +template +struct Templates { + typedef Templates33 type; +}; +template +struct Templates { + typedef Templates34 type; +}; +template +struct Templates { + typedef Templates35 type; +}; +template +struct Templates { + typedef Templates36 type; +}; +template +struct Templates { + typedef Templates37 type; +}; +template +struct Templates { + typedef Templates38 type; +}; +template +struct Templates { + typedef Templates39 type; +}; +template +struct Templates { + typedef Templates40 type; +}; +template +struct Templates { + typedef Templates41 type; +}; +template +struct Templates { + typedef Templates42 type; +}; +template +struct Templates { + typedef Templates43 type; +}; +template +struct Templates { + typedef Templates44 type; +}; +template +struct Templates { + typedef Templates45 type; +}; +template +struct Templates { + typedef Templates46 type; +}; +template +struct Templates { + typedef Templates47 type; +}; +template +struct Templates { + typedef Templates48 type; +}; +template +struct Templates { + typedef Templates49 type; +}; + +// The TypeList template makes it possible to use either a single type +// or a Types<...> list in TYPED_TEST_CASE() and +// INSTANTIATE_TYPED_TEST_CASE_P(). + +template +struct TypeList { + typedef Types1 type; +}; + +template +struct TypeList > { + typedef typename Types::type type; +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ diff --git a/mypyc/external/googletest/include/gtest/internal/gtest-type-util.h.pump b/mypyc/external/googletest/include/gtest/internal/gtest-type-util.h.pump new file mode 100644 index 000000000000..251fdf025b2c --- /dev/null +++ b/mypyc/external/googletest/include/gtest/internal/gtest-type-util.h.pump @@ -0,0 +1,297 @@ +$$ -*- mode: c++; -*- +$var n = 50 $$ Maximum length of type lists we want to support. +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Type utilities needed for implementing typed and type-parameterized +// tests. This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently we support at most $n types in a list, and at most $n +// type-parameterized tests in one type-parameterized test case. +// Please contact googletestframework@googlegroups.com if you need +// more. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + +#include "gtest/internal/gtest-port.h" + +// #ifdef __GNUC__ is too general here. It is possible to use gcc without using +// libstdc++ (which is where cxxabi.h comes from). +# if GTEST_HAS_CXXABI_H_ +# include +# elif defined(__HP_aCC) +# include +# endif // GTEST_HASH_CXXABI_H_ + +namespace testing { +namespace internal { + +// GetTypeName() returns a human-readable name of type T. +// NB: This function is also used in Google Mock, so don't move it inside of +// the typed-test-only section below. +template +std::string GetTypeName() { +# if GTEST_HAS_RTTI + + const char* const name = typeid(T).name(); +# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +# if GTEST_HAS_CXXABI_H_ + using abi::__cxa_demangle; +# endif // GTEST_HAS_CXXABI_H_ + char* const readable_name = __cxa_demangle(name, 0, 0, &status); + const std::string name_str(status == 0 ? readable_name : name); + free(readable_name); + return name_str; +# else + return name; +# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC + +# else + + return ""; + +# endif // GTEST_HAS_RTTI +} + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// AssertyTypeEq::type is defined iff T1 and T2 are the same +// type. This can be used as a compile-time assertion to ensure that +// two types are equal. + +template +struct AssertTypeEq; + +template +struct AssertTypeEq { + typedef bool type; +}; + +// A unique type used as the default value for the arguments of class +// template Types. This allows us to simulate variadic templates +// (e.g. Types, Type, and etc), which C++ doesn't +// support directly. +struct None {}; + +// The following family of struct and struct templates are used to +// represent type lists. In particular, TypesN +// represents a type list with N types (T1, T2, ..., and TN) in it. +// Except for Types0, every struct in the family has two member types: +// Head for the first type in the list, and Tail for the rest of the +// list. + +// The empty type list. +struct Types0 {}; + +// Type lists of length 1, 2, 3, and so on. + +template +struct Types1 { + typedef T1 Head; + typedef Types0 Tail; +}; + +$range i 2..n + +$for i [[ +$range j 1..i +$range k 2..i +template <$for j, [[typename T$j]]> +struct Types$i { + typedef T1 Head; + typedef Types$(i-1)<$for k, [[T$k]]> Tail; +}; + + +]] + +} // namespace internal + +// We don't want to require the users to write TypesN<...> directly, +// as that would require them to count the length. Types<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Types +// will appear as Types in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Types, and Google Test will translate +// that to TypesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Types template. + +$range i 1..n +template <$for i, [[typename T$i = internal::None]]> +struct Types { + typedef internal::Types$n<$for i, [[T$i]]> type; +}; + +template <> +struct Types<$for i, [[internal::None]]> { + typedef internal::Types0 type; +}; + +$range i 1..n-1 +$for i [[ +$range j 1..i +$range k i+1..n +template <$for j, [[typename T$j]]> +struct Types<$for j, [[T$j]]$for k[[, internal::None]]> { + typedef internal::Types$i<$for j, [[T$j]]> type; +}; + +]] + +namespace internal { + +# define GTEST_TEMPLATE_ template class + +// The template "selector" struct TemplateSel is used to +// represent Tmpl, which must be a class template with one type +// parameter, as a type. TemplateSel::Bind::type is defined +// as the type Tmpl. This allows us to actually instantiate the +// template "selected" by TemplateSel. +// +// This trick is necessary for simulating typedef for class templates, +// which C++ doesn't support directly. +template +struct TemplateSel { + template + struct Bind { + typedef Tmpl type; + }; +}; + +# define GTEST_BIND_(TmplSel, T) \ + TmplSel::template Bind::type + +// A unique struct template used as the default value for the +// arguments of class template Templates. This allows us to simulate +// variadic templates (e.g. Templates, Templates, +// and etc), which C++ doesn't support directly. +template +struct NoneT {}; + +// The following family of struct and struct templates are used to +// represent template lists. In particular, TemplatesN represents a list of N templates (T1, T2, ..., and TN). Except +// for Templates0, every struct in the family has two member types: +// Head for the selector of the first template in the list, and Tail +// for the rest of the list. + +// The empty template list. +struct Templates0 {}; + +// Template lists of length 1, 2, 3, and so on. + +template +struct Templates1 { + typedef TemplateSel Head; + typedef Templates0 Tail; +}; + +$range i 2..n + +$for i [[ +$range j 1..i +$range k 2..i +template <$for j, [[GTEST_TEMPLATE_ T$j]]> +struct Templates$i { + typedef TemplateSel Head; + typedef Templates$(i-1)<$for k, [[T$k]]> Tail; +}; + + +]] + +// We don't want to require the users to write TemplatesN<...> directly, +// as that would require them to count the length. Templates<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Templates +// will appear as Templates in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Templates, and Google Test will translate +// that to TemplatesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Templates template. + +$range i 1..n +template <$for i, [[GTEST_TEMPLATE_ T$i = NoneT]]> +struct Templates { + typedef Templates$n<$for i, [[T$i]]> type; +}; + +template <> +struct Templates<$for i, [[NoneT]]> { + typedef Templates0 type; +}; + +$range i 1..n-1 +$for i [[ +$range j 1..i +$range k i+1..n +template <$for j, [[GTEST_TEMPLATE_ T$j]]> +struct Templates<$for j, [[T$j]]$for k[[, NoneT]]> { + typedef Templates$i<$for j, [[T$j]]> type; +}; + +]] + +// The TypeList template makes it possible to use either a single type +// or a Types<...> list in TYPED_TEST_CASE() and +// INSTANTIATE_TYPED_TEST_CASE_P(). + +template +struct TypeList { + typedef Types1 type; +}; + + +$range i 1..n +template <$for i, [[typename T$i]]> +struct TypeList > { + typedef typename Types<$for i, [[T$i]]>::type type; +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ diff --git a/mypyc/external/googletest/make/Makefile b/mypyc/external/googletest/make/Makefile new file mode 100644 index 000000000000..7bbdd79f92e1 --- /dev/null +++ b/mypyc/external/googletest/make/Makefile @@ -0,0 +1,61 @@ +# A sample Makefile for building Google Test and using it in user +# tests. Please tweak it to suit your environment and project. You +# may want to move it to your project's root directory. +# +# SYNOPSIS: +# +# make [all] - makes everything. +# make TARGET - makes the given target. +# make clean - removes all files generated by make. + +# Please tweak the following variable definitions as needed by your +# project, except GTEST_HEADERS, which you can use in your own targets +# but shouldn't modify. + +# Points to the root of Google Test, relative to where this file is. +# Remember to tweak this if you move this file. +GTEST_DIR = .. + +# Flags passed to the preprocessor. +# Set Google Test's header directory as a system directory, such that +# the compiler doesn't generate warnings in Google Test headers. +CPPFLAGS += -isystem $(GTEST_DIR)/include + +# Flags passed to the C++ compiler. +CXXFLAGS += -g -Wall -Wextra -pthread -fPIC + +# All Google Test headers. Usually you shouldn't change this +# definition. +GTEST_HEADERS = $(GTEST_DIR)/include/gtest/*.h \ + $(GTEST_DIR)/include/gtest/internal/*.h + +# House-keeping build targets. + +all : libgtest.a libgtest_main.a + +clean : + rm -f libgtest.a libgtest_main.a *.o + +# Builds libgtest.a and libgtest_main.a. + +# Usually you shouldn't tweak such internal variables, indicated by a +# trailing _. +GTEST_SRCS_ = $(GTEST_DIR)/src/*.cc $(GTEST_DIR)/src/*.h $(GTEST_HEADERS) + +# For simplicity and to avoid depending on Google Test's +# implementation details, the dependencies specified below are +# conservative and not optimized. This is fine as Google Test +# compiles fast and for ordinary users its source rarely changes. +gtest-all.o : $(GTEST_SRCS_) + $(CXX) $(CPPFLAGS) -I$(GTEST_DIR) $(CXXFLAGS) -c \ + $(GTEST_DIR)/src/gtest-all.cc + +gtest_main.o : $(GTEST_SRCS_) + $(CXX) $(CPPFLAGS) -I$(GTEST_DIR) $(CXXFLAGS) -c \ + $(GTEST_DIR)/src/gtest_main.cc + +libgtest.a : gtest-all.o + $(AR) $(ARFLAGS) $@ $^ + +libgtest_main.a : gtest-all.o gtest_main.o + $(AR) $(ARFLAGS) $@ $^ diff --git a/mypyc/external/googletest/src/gtest-all.cc b/mypyc/external/googletest/src/gtest-all.cc new file mode 100644 index 000000000000..0a9cee522333 --- /dev/null +++ b/mypyc/external/googletest/src/gtest-all.cc @@ -0,0 +1,48 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// +// Google C++ Testing Framework (Google Test) +// +// Sometimes it's desirable to build Google Test by compiling a single file. +// This file serves this purpose. + +// This line ensures that gtest.h can be compiled on its own, even +// when it's fused. +#include "gtest/gtest.h" + +// The following lines pull in the real gtest *.cc files. +#include "src/gtest.cc" +#include "src/gtest-death-test.cc" +#include "src/gtest-filepath.cc" +#include "src/gtest-port.cc" +#include "src/gtest-printers.cc" +#include "src/gtest-test-part.cc" +#include "src/gtest-typed-test.cc" diff --git a/mypyc/external/googletest/src/gtest-death-test.cc b/mypyc/external/googletest/src/gtest-death-test.cc new file mode 100644 index 000000000000..a01a36983087 --- /dev/null +++ b/mypyc/external/googletest/src/gtest-death-test.cc @@ -0,0 +1,1342 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan), vladl@google.com (Vlad Losev) +// +// This file implements death tests. + +#include "gtest/gtest-death-test.h" +#include "gtest/internal/gtest-port.h" +#include "gtest/internal/custom/gtest.h" + +#if GTEST_HAS_DEATH_TEST + +# if GTEST_OS_MAC +# include +# endif // GTEST_OS_MAC + +# include +# include +# include + +# if GTEST_OS_LINUX +# include +# endif // GTEST_OS_LINUX + +# include + +# if GTEST_OS_WINDOWS +# include +# else +# include +# include +# endif // GTEST_OS_WINDOWS + +# if GTEST_OS_QNX +# include +# endif // GTEST_OS_QNX + +#endif // GTEST_HAS_DEATH_TEST + +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-string.h" + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +namespace testing { + +// Constants. + +// The default death test style. +static const char kDefaultDeathTestStyle[] = "fast"; + +GTEST_DEFINE_string_( + death_test_style, + internal::StringFromGTestEnv("death_test_style", kDefaultDeathTestStyle), + "Indicates how to run a death test in a forked child process: " + "\"threadsafe\" (child process re-executes the test binary " + "from the beginning, running only the specific death test) or " + "\"fast\" (child process runs the death test immediately " + "after forking)."); + +GTEST_DEFINE_bool_( + death_test_use_fork, + internal::BoolFromGTestEnv("death_test_use_fork", false), + "Instructs to use fork()/_exit() instead of clone() in death tests. " + "Ignored and always uses fork() on POSIX systems where clone() is not " + "implemented. Useful when running under valgrind or similar tools if " + "those do not support clone(). Valgrind 3.3.1 will just fail if " + "it sees an unsupported combination of clone() flags. " + "It is not recommended to use this flag w/o valgrind though it will " + "work in 99% of the cases. Once valgrind is fixed, this flag will " + "most likely be removed."); + +namespace internal { +GTEST_DEFINE_string_( + internal_run_death_test, "", + "Indicates the file, line number, temporal index of " + "the single death test to run, and a file descriptor to " + "which a success code may be sent, all separated by " + "the '|' characters. This flag is specified if and only if the current " + "process is a sub-process launched for running a thread-safe " + "death test. FOR INTERNAL USE ONLY."); +} // namespace internal + +#if GTEST_HAS_DEATH_TEST + +namespace internal { + +// Valid only for fast death tests. Indicates the code is running in the +// child process of a fast style death test. +# if !GTEST_OS_WINDOWS +static bool g_in_fast_death_test_child = false; +# endif + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +bool InDeathTestChild() { +# if GTEST_OS_WINDOWS + + // On Windows, death tests are thread-safe regardless of the value of the + // death_test_style flag. + return !GTEST_FLAG(internal_run_death_test).empty(); + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") + return !GTEST_FLAG(internal_run_death_test).empty(); + else + return g_in_fast_death_test_child; +#endif +} + +} // namespace internal + +// ExitedWithCode constructor. +ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) { +} + +// ExitedWithCode function-call operator. +bool ExitedWithCode::operator()(int exit_status) const { +# if GTEST_OS_WINDOWS + + return exit_status == exit_code_; + +# else + + return WIFEXITED(exit_status) && WEXITSTATUS(exit_status) == exit_code_; + +# endif // GTEST_OS_WINDOWS +} + +# if !GTEST_OS_WINDOWS +// KilledBySignal constructor. +KilledBySignal::KilledBySignal(int signum) : signum_(signum) { +} + +// KilledBySignal function-call operator. +bool KilledBySignal::operator()(int exit_status) const { +# if defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) + { + bool result; + if (GTEST_KILLED_BY_SIGNAL_OVERRIDE_(signum_, exit_status, &result)) { + return result; + } + } +# endif // defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) + return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_; +} +# endif // !GTEST_OS_WINDOWS + +namespace internal { + +// Utilities needed for death tests. + +// Generates a textual description of a given exit code, in the format +// specified by wait(2). +static std::string ExitSummary(int exit_code) { + Message m; + +# if GTEST_OS_WINDOWS + + m << "Exited with exit status " << exit_code; + +# else + + if (WIFEXITED(exit_code)) { + m << "Exited with exit status " << WEXITSTATUS(exit_code); + } else if (WIFSIGNALED(exit_code)) { + m << "Terminated by signal " << WTERMSIG(exit_code); + } +# ifdef WCOREDUMP + if (WCOREDUMP(exit_code)) { + m << " (core dumped)"; + } +# endif +# endif // GTEST_OS_WINDOWS + + return m.GetString(); +} + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +bool ExitedUnsuccessfully(int exit_status) { + return !ExitedWithCode(0)(exit_status); +} + +# if !GTEST_OS_WINDOWS +// Generates a textual failure message when a death test finds more than +// one thread running, or cannot determine the number of threads, prior +// to executing the given statement. It is the responsibility of the +// caller not to pass a thread_count of 1. +static std::string DeathTestThreadWarning(size_t thread_count) { + Message msg; + msg << "Death tests use fork(), which is unsafe particularly" + << " in a threaded context. For this test, " << GTEST_NAME_ << " "; + if (thread_count == 0) + msg << "couldn't detect the number of threads."; + else + msg << "detected " << thread_count << " threads."; + return msg.GetString(); +} +# endif // !GTEST_OS_WINDOWS + +// Flag characters for reporting a death test that did not die. +static const char kDeathTestLived = 'L'; +static const char kDeathTestReturned = 'R'; +static const char kDeathTestThrew = 'T'; +static const char kDeathTestInternalError = 'I'; + +// An enumeration describing all of the possible ways that a death test can +// conclude. DIED means that the process died while executing the test +// code; LIVED means that process lived beyond the end of the test code; +// RETURNED means that the test statement attempted to execute a return +// statement, which is not allowed; THREW means that the test statement +// returned control by throwing an exception. IN_PROGRESS means the test +// has not yet concluded. +// TODO(vladl@google.com): Unify names and possibly values for +// AbortReason, DeathTestOutcome, and flag characters above. +enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW }; + +// Routine for aborting the program which is safe to call from an +// exec-style death test child process, in which case the error +// message is propagated back to the parent process. Otherwise, the +// message is simply printed to stderr. In either case, the program +// then exits with status 1. +void DeathTestAbort(const std::string& message) { + // On a POSIX system, this function may be called from a threadsafe-style + // death test child process, which operates on a very small stack. Use + // the heap for any additional non-minuscule memory requirements. + const InternalRunDeathTestFlag* const flag = + GetUnitTestImpl()->internal_run_death_test_flag(); + if (flag != NULL) { + FILE* parent = posix::FDOpen(flag->write_fd(), "w"); + fputc(kDeathTestInternalError, parent); + fprintf(parent, "%s", message.c_str()); + fflush(parent); + _exit(1); + } else { + fprintf(stderr, "%s", message.c_str()); + fflush(stderr); + posix::Abort(); + } +} + +// A replacement for CHECK that calls DeathTestAbort if the assertion +// fails. +# define GTEST_DEATH_TEST_CHECK_(expression) \ + do { \ + if (!::testing::internal::IsTrue(expression)) { \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// This macro is similar to GTEST_DEATH_TEST_CHECK_, but it is meant for +// evaluating any system call that fulfills two conditions: it must return +// -1 on failure, and set errno to EINTR when it is interrupted and +// should be tried again. The macro expands to a loop that repeatedly +// evaluates the expression as long as it evaluates to -1 and sets +// errno to EINTR. If the expression evaluates to -1 but errno is +// something other than EINTR, DeathTestAbort is called. +# define GTEST_DEATH_TEST_CHECK_SYSCALL_(expression) \ + do { \ + int gtest_retval; \ + do { \ + gtest_retval = (expression); \ + } while (gtest_retval == -1 && errno == EINTR); \ + if (gtest_retval == -1) { \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression + " != -1"); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// Returns the message describing the last system error in errno. +std::string GetLastErrnoDescription() { + return errno == 0 ? "" : posix::StrError(errno); +} + +// This is called from a death test parent process to read a failure +// message from the death test child process and log it with the FATAL +// severity. On Windows, the message is read from a pipe handle. On other +// platforms, it is read from a file descriptor. +static void FailFromInternalError(int fd) { + Message error; + char buffer[256]; + int num_read; + + do { + while ((num_read = posix::Read(fd, buffer, 255)) > 0) { + buffer[num_read] = '\0'; + error << buffer; + } + } while (num_read == -1 && errno == EINTR); + + if (num_read == 0) { + GTEST_LOG_(FATAL) << error.GetString(); + } else { + const int last_error = errno; + GTEST_LOG_(FATAL) << "Error while reading death test internal: " + << GetLastErrnoDescription() << " [" << last_error << "]"; + } +} + +// Death test constructor. Increments the running death test count +// for the current test. +DeathTest::DeathTest() { + TestInfo* const info = GetUnitTestImpl()->current_test_info(); + if (info == NULL) { + DeathTestAbort("Cannot run a death test outside of a TEST or " + "TEST_F construct"); + } +} + +// Creates and returns a death test by dispatching to the current +// death test factory. +bool DeathTest::Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) { + return GetUnitTestImpl()->death_test_factory()->Create( + statement, regex, file, line, test); +} + +const char* DeathTest::LastMessage() { + return last_death_test_message_.c_str(); +} + +void DeathTest::set_last_death_test_message(const std::string& message) { + last_death_test_message_ = message; +} + +std::string DeathTest::last_death_test_message_; + +// Provides cross platform implementation for some death functionality. +class DeathTestImpl : public DeathTest { + protected: + DeathTestImpl(const char* a_statement, const RE* a_regex) + : statement_(a_statement), + regex_(a_regex), + spawned_(false), + status_(-1), + outcome_(IN_PROGRESS), + read_fd_(-1), + write_fd_(-1) {} + + // read_fd_ is expected to be closed and cleared by a derived class. + ~DeathTestImpl() { GTEST_DEATH_TEST_CHECK_(read_fd_ == -1); } + + void Abort(AbortReason reason); + virtual bool Passed(bool status_ok); + + const char* statement() const { return statement_; } + const RE* regex() const { return regex_; } + bool spawned() const { return spawned_; } + void set_spawned(bool is_spawned) { spawned_ = is_spawned; } + int status() const { return status_; } + void set_status(int a_status) { status_ = a_status; } + DeathTestOutcome outcome() const { return outcome_; } + void set_outcome(DeathTestOutcome an_outcome) { outcome_ = an_outcome; } + int read_fd() const { return read_fd_; } + void set_read_fd(int fd) { read_fd_ = fd; } + int write_fd() const { return write_fd_; } + void set_write_fd(int fd) { write_fd_ = fd; } + + // Called in the parent process only. Reads the result code of the death + // test child process via a pipe, interprets it to set the outcome_ + // member, and closes read_fd_. Outputs diagnostics and terminates in + // case of unexpected codes. + void ReadAndInterpretStatusByte(); + + private: + // The textual content of the code this object is testing. This class + // doesn't own this string and should not attempt to delete it. + const char* const statement_; + // The regular expression which test output must match. DeathTestImpl + // doesn't own this object and should not attempt to delete it. + const RE* const regex_; + // True if the death test child process has been successfully spawned. + bool spawned_; + // The exit status of the child process. + int status_; + // How the death test concluded. + DeathTestOutcome outcome_; + // Descriptor to the read end of the pipe to the child process. It is + // always -1 in the child process. The child keeps its write end of the + // pipe in write_fd_. + int read_fd_; + // Descriptor to the child's write end of the pipe to the parent process. + // It is always -1 in the parent process. The parent keeps its end of the + // pipe in read_fd_. + int write_fd_; +}; + +// Called in the parent process only. Reads the result code of the death +// test child process via a pipe, interprets it to set the outcome_ +// member, and closes read_fd_. Outputs diagnostics and terminates in +// case of unexpected codes. +void DeathTestImpl::ReadAndInterpretStatusByte() { + char flag; + int bytes_read; + + // The read() here blocks until data is available (signifying the + // failure of the death test) or until the pipe is closed (signifying + // its success), so it's okay to call this in the parent before + // the child process has exited. + do { + bytes_read = posix::Read(read_fd(), &flag, 1); + } while (bytes_read == -1 && errno == EINTR); + + if (bytes_read == 0) { + set_outcome(DIED); + } else if (bytes_read == 1) { + switch (flag) { + case kDeathTestReturned: + set_outcome(RETURNED); + break; + case kDeathTestThrew: + set_outcome(THREW); + break; + case kDeathTestLived: + set_outcome(LIVED); + break; + case kDeathTestInternalError: + FailFromInternalError(read_fd()); // Does not return. + break; + default: + GTEST_LOG_(FATAL) << "Death test child process reported " + << "unexpected status byte (" + << static_cast(flag) << ")"; + } + } else { + GTEST_LOG_(FATAL) << "Read from death test child process failed: " + << GetLastErrnoDescription(); + } + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Close(read_fd())); + set_read_fd(-1); +} + +// Signals that the death test code which should have exited, didn't. +// Should be called only in a death test child process. +// Writes a status byte to the child's status file descriptor, then +// calls _exit(1). +void DeathTestImpl::Abort(AbortReason reason) { + // The parent process considers the death test to be a failure if + // it finds any data in our pipe. So, here we write a single flag byte + // to the pipe, then exit. + const char status_ch = + reason == TEST_DID_NOT_DIE ? kDeathTestLived : + reason == TEST_THREW_EXCEPTION ? kDeathTestThrew : kDeathTestReturned; + + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Write(write_fd(), &status_ch, 1)); + // We are leaking the descriptor here because on some platforms (i.e., + // when built as Windows DLL), destructors of global objects will still + // run after calling _exit(). On such systems, write_fd_ will be + // indirectly closed from the destructor of UnitTestImpl, causing double + // close if it is also closed here. On debug configurations, double close + // may assert. As there are no in-process buffers to flush here, we are + // relying on the OS to close the descriptor after the process terminates + // when the destructors are not run. + _exit(1); // Exits w/o any normal exit hooks (we were supposed to crash) +} + +// Returns an indented copy of stderr output for a death test. +// This makes distinguishing death test output lines from regular log lines +// much easier. +static ::std::string FormatDeathTestOutput(const ::std::string& output) { + ::std::string ret; + for (size_t at = 0; ; ) { + const size_t line_end = output.find('\n', at); + ret += "[ DEATH ] "; + if (line_end == ::std::string::npos) { + ret += output.substr(at); + break; + } + ret += output.substr(at, line_end + 1 - at); + at = line_end + 1; + } + return ret; +} + +// Assesses the success or failure of a death test, using both private +// members which have previously been set, and one argument: +// +// Private data members: +// outcome: An enumeration describing how the death test +// concluded: DIED, LIVED, THREW, or RETURNED. The death test +// fails in the latter three cases. +// status: The exit status of the child process. On *nix, it is in the +// in the format specified by wait(2). On Windows, this is the +// value supplied to the ExitProcess() API or a numeric code +// of the exception that terminated the program. +// regex: A regular expression object to be applied to +// the test's captured standard error output; the death test +// fails if it does not match. +// +// Argument: +// status_ok: true if exit_status is acceptable in the context of +// this particular death test, which fails if it is false +// +// Returns true iff all of the above conditions are met. Otherwise, the +// first failing condition, in the order given above, is the one that is +// reported. Also sets the last death test message string. +bool DeathTestImpl::Passed(bool status_ok) { + if (!spawned()) + return false; + + const std::string error_message = GetCapturedStderr(); + + bool success = false; + Message buffer; + + buffer << "Death test: " << statement() << "\n"; + switch (outcome()) { + case LIVED: + buffer << " Result: failed to die.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case THREW: + buffer << " Result: threw an exception.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case RETURNED: + buffer << " Result: illegal return in test statement.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case DIED: + if (status_ok) { + const bool matched = RE::PartialMatch(error_message.c_str(), *regex()); + if (matched) { + success = true; + } else { + buffer << " Result: died but not with expected error.\n" + << " Expected: " << regex()->pattern() << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + } else { + buffer << " Result: died but not with expected exit code:\n" + << " " << ExitSummary(status()) << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + break; + case IN_PROGRESS: + default: + GTEST_LOG_(FATAL) + << "DeathTest::Passed somehow called before conclusion of test"; + } + + DeathTest::set_last_death_test_message(buffer.GetString()); + return success; +} + +# if GTEST_OS_WINDOWS +// WindowsDeathTest implements death tests on Windows. Due to the +// specifics of starting new processes on Windows, death tests there are +// always threadsafe, and Google Test considers the +// --gtest_death_test_style=fast setting to be equivalent to +// --gtest_death_test_style=threadsafe there. +// +// A few implementation notes: Like the Linux version, the Windows +// implementation uses pipes for child-to-parent communication. But due to +// the specifics of pipes on Windows, some extra steps are required: +// +// 1. The parent creates a communication pipe and stores handles to both +// ends of it. +// 2. The parent starts the child and provides it with the information +// necessary to acquire the handle to the write end of the pipe. +// 3. The child acquires the write end of the pipe and signals the parent +// using a Windows event. +// 4. Now the parent can release the write end of the pipe on its side. If +// this is done before step 3, the object's reference count goes down to +// 0 and it is destroyed, preventing the child from acquiring it. The +// parent now has to release it, or read operations on the read end of +// the pipe will not return when the child terminates. +// 5. The parent reads child's output through the pipe (outcome code and +// any possible error messages) from the pipe, and its stderr and then +// determines whether to fail the test. +// +// Note: to distinguish Win32 API calls from the local method and function +// calls, the former are explicitly resolved in the global namespace. +// +class WindowsDeathTest : public DeathTestImpl { + public: + WindowsDeathTest(const char* a_statement, + const RE* a_regex, + const char* file, + int line) + : DeathTestImpl(a_statement, a_regex), file_(file), line_(line) {} + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + virtual TestRole AssumeRole(); + + private: + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; + // Handle to the write end of the pipe to the child process. + AutoHandle write_handle_; + // Child process handle. + AutoHandle child_handle_; + // Event the child process uses to signal the parent that it has + // acquired the handle to the write end of the pipe. After seeing this + // event the parent can release its own handles to make sure its + // ReadFile() calls return when the child terminates. + AutoHandle event_handle_; +}; + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int WindowsDeathTest::Wait() { + if (!spawned()) + return 0; + + // Wait until the child either signals that it has acquired the write end + // of the pipe or it dies. + const HANDLE wait_handles[2] = { child_handle_.Get(), event_handle_.Get() }; + switch (::WaitForMultipleObjects(2, + wait_handles, + FALSE, // Waits for any of the handles. + INFINITE)) { + case WAIT_OBJECT_0: + case WAIT_OBJECT_0 + 1: + break; + default: + GTEST_DEATH_TEST_CHECK_(false); // Should not get here. + } + + // The child has acquired the write end of the pipe or exited. + // We release the handle on our side and continue. + write_handle_.Reset(); + event_handle_.Reset(); + + ReadAndInterpretStatusByte(); + + // Waits for the child process to exit if it haven't already. This + // returns immediately if the child has already exited, regardless of + // whether previous calls to WaitForMultipleObjects synchronized on this + // handle or not. + GTEST_DEATH_TEST_CHECK_( + WAIT_OBJECT_0 == ::WaitForSingleObject(child_handle_.Get(), + INFINITE)); + DWORD status_code; + GTEST_DEATH_TEST_CHECK_( + ::GetExitCodeProcess(child_handle_.Get(), &status_code) != FALSE); + child_handle_.Reset(); + set_status(static_cast(status_code)); + return status(); +} + +// The AssumeRole process for a Windows death test. It creates a child +// process with the same executable as the current process to run the +// death test. The child process is given the --gtest_filter and +// --gtest_internal_run_death_test flags such that it knows to run the +// current death test only. +DeathTest::TestRole WindowsDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + // ParseInternalRunDeathTestFlag() has performed all the necessary + // processing. + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + // WindowsDeathTest uses an anonymous pipe to communicate results of + // a death test. + SECURITY_ATTRIBUTES handles_are_inheritable = { + sizeof(SECURITY_ATTRIBUTES), NULL, TRUE }; + HANDLE read_handle, write_handle; + GTEST_DEATH_TEST_CHECK_( + ::CreatePipe(&read_handle, &write_handle, &handles_are_inheritable, + 0) // Default buffer size. + != FALSE); + set_read_fd(::_open_osfhandle(reinterpret_cast(read_handle), + O_RDONLY)); + write_handle_.Reset(write_handle); + event_handle_.Reset(::CreateEvent( + &handles_are_inheritable, + TRUE, // The event will automatically reset to non-signaled state. + FALSE, // The initial state is non-signalled. + NULL)); // The even is unnamed. + GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != NULL); + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + + "=" + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(static_cast(::GetCurrentProcessId())) + + // size_t has the same width as pointers on both 32-bit and 64-bit + // Windows platforms. + // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx. + "|" + StreamableToString(reinterpret_cast(write_handle)) + + "|" + StreamableToString(reinterpret_cast(event_handle_.Get())); + + char executable_path[_MAX_PATH + 1]; // NOLINT + GTEST_DEATH_TEST_CHECK_( + _MAX_PATH + 1 != ::GetModuleFileNameA(NULL, + executable_path, + _MAX_PATH)); + + std::string command_line = + std::string(::GetCommandLineA()) + " " + filter_flag + " \"" + + internal_flag + "\""; + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // Flush the log buffers since the log streams are shared with the child. + FlushInfoLog(); + + // The child process will share the standard handles with the parent. + STARTUPINFOA startup_info; + memset(&startup_info, 0, sizeof(STARTUPINFO)); + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE); + startup_info.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE); + startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); + + PROCESS_INFORMATION process_info; + GTEST_DEATH_TEST_CHECK_(::CreateProcessA( + executable_path, + const_cast(command_line.c_str()), + NULL, // Retuned process handle is not inheritable. + NULL, // Retuned thread handle is not inheritable. + TRUE, // Child inherits all inheritable handles (for write_handle_). + 0x0, // Default creation flags. + NULL, // Inherit the parent's environment. + UnitTest::GetInstance()->original_working_dir(), + &startup_info, + &process_info) != FALSE); + child_handle_.Reset(process_info.hProcess); + ::CloseHandle(process_info.hThread); + set_spawned(true); + return OVERSEE_TEST; +} +# else // We are not on Windows. + +// ForkingDeathTest provides implementations for most of the abstract +// methods of the DeathTest interface. Only the AssumeRole method is +// left undefined. +class ForkingDeathTest : public DeathTestImpl { + public: + ForkingDeathTest(const char* statement, const RE* regex); + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + + protected: + void set_child_pid(pid_t child_pid) { child_pid_ = child_pid; } + + private: + // PID of child process during death test; 0 in the child process itself. + pid_t child_pid_; +}; + +// Constructs a ForkingDeathTest. +ForkingDeathTest::ForkingDeathTest(const char* a_statement, const RE* a_regex) + : DeathTestImpl(a_statement, a_regex), + child_pid_(-1) {} + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int ForkingDeathTest::Wait() { + if (!spawned()) + return 0; + + ReadAndInterpretStatusByte(); + + int status_value; + GTEST_DEATH_TEST_CHECK_SYSCALL_(waitpid(child_pid_, &status_value, 0)); + set_status(status_value); + return status_value; +} + +// A concrete death test class that forks, then immediately runs the test +// in the child process. +class NoExecDeathTest : public ForkingDeathTest { + public: + NoExecDeathTest(const char* a_statement, const RE* a_regex) : + ForkingDeathTest(a_statement, a_regex) { } + virtual TestRole AssumeRole(); +}; + +// The AssumeRole process for a fork-and-run death test. It implements a +// straightforward fork, with a simple pipe to transmit the status byte. +DeathTest::TestRole NoExecDeathTest::AssumeRole() { + const size_t thread_count = GetThreadCount(); + if (thread_count != 1) { + GTEST_LOG_(WARNING) << DeathTestThreadWarning(thread_count); + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + + DeathTest::set_last_death_test_message(""); + CaptureStderr(); + // When we fork the process below, the log file buffers are copied, but the + // file descriptors are shared. We flush all log files here so that closing + // the file descriptors in the child process doesn't throw off the + // synchronization between descriptors and buffers in the parent process. + // This is as close to the fork as possible to avoid a race condition in case + // there are multiple threads running before the death test, and another + // thread writes to the log file. + FlushInfoLog(); + + const pid_t child_pid = fork(); + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + set_child_pid(child_pid); + if (child_pid == 0) { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[0])); + set_write_fd(pipe_fd[1]); + // Redirects all logging to stderr in the child process to prevent + // concurrent writes to the log files. We capture stderr in the parent + // process and append the child process' output to a log. + LogToStderr(); + // Event forwarding to the listeners of event listener API mush be shut + // down in death test subprocesses. + GetUnitTestImpl()->listeners()->SuppressEventForwarding(); + g_in_fast_death_test_child = true; + return EXECUTE_TEST; + } else { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; + } +} + +// A concrete death test class that forks and re-executes the main +// program from the beginning, with command-line flags set that cause +// only this specific death test to be run. +class ExecDeathTest : public ForkingDeathTest { + public: + ExecDeathTest(const char* a_statement, const RE* a_regex, + const char* file, int line) : + ForkingDeathTest(a_statement, a_regex), file_(file), line_(line) { } + virtual TestRole AssumeRole(); + private: + static ::std::vector + GetArgvsForDeathTestChildProcess() { + ::std::vector args = GetInjectableArgvs(); +# if defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) + ::std::vector extra_args = + GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_(); + args.insert(args.end(), extra_args.begin(), extra_args.end()); +# endif // defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) + return args; + } + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; +}; + +// Utility class for accumulating command-line arguments. +class Arguments { + public: + Arguments() { + args_.push_back(NULL); + } + + ~Arguments() { + for (std::vector::iterator i = args_.begin(); i != args_.end(); + ++i) { + free(*i); + } + } + void AddArgument(const char* argument) { + args_.insert(args_.end() - 1, posix::StrDup(argument)); + } + + template + void AddArguments(const ::std::vector& arguments) { + for (typename ::std::vector::const_iterator i = arguments.begin(); + i != arguments.end(); + ++i) { + args_.insert(args_.end() - 1, posix::StrDup(i->c_str())); + } + } + char* const* Argv() { + return &args_[0]; + } + + private: + std::vector args_; +}; + +// A struct that encompasses the arguments to the child process of a +// threadsafe-style death test process. +struct ExecDeathTestArgs { + char* const* argv; // Command-line arguments for the child's call to exec + int close_fd; // File descriptor to close; the read end of a pipe +}; + +# if GTEST_OS_MAC +inline char** GetEnviron() { + // When Google Test is built as a framework on MacOS X, the environ variable + // is unavailable. Apple's documentation (man environ) recommends using + // _NSGetEnviron() instead. + return *_NSGetEnviron(); +} +# else +// Some POSIX platforms expect you to declare environ. extern "C" makes +// it reside in the global namespace. +extern "C" char** environ; +inline char** GetEnviron() { return environ; } +# endif // GTEST_OS_MAC + +# if !GTEST_OS_QNX +// The main function for a threadsafe-style death test child process. +// This function is called in a clone()-ed process and thus must avoid +// any potentially unsafe operations like malloc or libc functions. +static int ExecDeathTestChildMain(void* child_arg) { + ExecDeathTestArgs* const args = static_cast(child_arg); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(args->close_fd)); + + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; + } + + // We can safely call execve() as it's a direct system call. We + // cannot use execvp() as it's a libc function and thus potentially + // unsafe. Since execve() doesn't search the PATH, the user must + // invoke the test program via a valid path that contains at least + // one path separator. + execve(args->argv[0], args->argv, GetEnviron()); + DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " + + original_dir + " failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; +} +# endif // !GTEST_OS_QNX + +// Two utility routines that together determine the direction the stack +// grows. +// This could be accomplished more elegantly by a single recursive +// function, but we want to guard against the unlikely possibility of +// a smart compiler optimizing the recursion away. +// +// GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining +// StackLowerThanAddress into StackGrowsDown, which then doesn't give +// correct answer. +void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_; +void StackLowerThanAddress(const void* ptr, bool* result) { + int dummy; + *result = (&dummy < ptr); +} + +// Make sure AddressSanitizer does not tamper with the stack here. +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +bool StackGrowsDown() { + int dummy; + bool result; + StackLowerThanAddress(&dummy, &result); + return result; +} + +// Spawns a child process with the same executable as the current process in +// a thread-safe manner and instructs it to run the death test. The +// implementation uses fork(2) + exec. On systems where clone(2) is +// available, it is used instead, being slightly more thread-safe. On QNX, +// fork supports only single-threaded environments, so this function uses +// spawn(2) there instead. The function dies with an error message if +// anything goes wrong. +static pid_t ExecDeathTestSpawnChild(char* const* argv, int close_fd) { + ExecDeathTestArgs args = { argv, close_fd }; + pid_t child_pid = -1; + +# if GTEST_OS_QNX + // Obtains the current directory and sets it to be closed in the child + // process. + const int cwd_fd = open(".", O_RDONLY); + GTEST_DEATH_TEST_CHECK_(cwd_fd != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(cwd_fd, F_SETFD, FD_CLOEXEC)); + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; + } + + int fd_flags; + // Set close_fd to be closed after spawn. + GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD)); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD, + fd_flags | FD_CLOEXEC)); + struct inheritance inherit = {0}; + // spawn is a system call. + child_pid = spawn(args.argv[0], 0, NULL, &inherit, args.argv, GetEnviron()); + // Restores the current working directory. + GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd)); + +# else // GTEST_OS_QNX +# if GTEST_OS_LINUX + // When a SIGPROF signal is received while fork() or clone() are executing, + // the process may hang. To avoid this, we ignore SIGPROF here and re-enable + // it after the call to fork()/clone() is complete. + struct sigaction saved_sigprof_action; + struct sigaction ignore_sigprof_action; + memset(&ignore_sigprof_action, 0, sizeof(ignore_sigprof_action)); + sigemptyset(&ignore_sigprof_action.sa_mask); + ignore_sigprof_action.sa_handler = SIG_IGN; + GTEST_DEATH_TEST_CHECK_SYSCALL_(sigaction( + SIGPROF, &ignore_sigprof_action, &saved_sigprof_action)); +# endif // GTEST_OS_LINUX + +# if GTEST_HAS_CLONE + const bool use_fork = GTEST_FLAG(death_test_use_fork); + + if (!use_fork) { + static const bool stack_grows_down = StackGrowsDown(); + const size_t stack_size = getpagesize(); + // MMAP_ANONYMOUS is not defined on Mac, so we use MAP_ANON instead. + void* const stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE, + MAP_ANON | MAP_PRIVATE, -1, 0); + GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED); + + // Maximum stack alignment in bytes: For a downward-growing stack, this + // amount is subtracted from size of the stack space to get an address + // that is within the stack space and is aligned on all systems we care + // about. As far as I know there is no ABI with stack alignment greater + // than 64. We assume stack and stack_size already have alignment of + // kMaxStackAlignment. + const size_t kMaxStackAlignment = 64; + void* const stack_top = + static_cast(stack) + + (stack_grows_down ? stack_size - kMaxStackAlignment : 0); + GTEST_DEATH_TEST_CHECK_(stack_size > kMaxStackAlignment && + reinterpret_cast(stack_top) % kMaxStackAlignment == 0); + + child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args); + + GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1); + } +# else + const bool use_fork = true; +# endif // GTEST_HAS_CLONE + + if (use_fork && (child_pid = fork()) == 0) { + ExecDeathTestChildMain(&args); + _exit(0); + } +# endif // GTEST_OS_QNX +# if GTEST_OS_LINUX + GTEST_DEATH_TEST_CHECK_SYSCALL_( + sigaction(SIGPROF, &saved_sigprof_action, NULL)); +# endif // GTEST_OS_LINUX + + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + return child_pid; +} + +// The AssumeRole process for a fork-and-exec death test. It re-executes the +// main program from the beginning, setting the --gtest_filter +// and --gtest_internal_run_death_test flags to cause only the current +// death test to be re-run. +DeathTest::TestRole ExecDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + // Clear the close-on-exec flag on the write end of the pipe, lest + // it be closed when the child process does an exec: + GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1); + + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(pipe_fd[1]); + Arguments args; + args.AddArguments(GetArgvsForDeathTestChildProcess()); + args.AddArgument(filter_flag.c_str()); + args.AddArgument(internal_flag.c_str()); + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // See the comment in NoExecDeathTest::AssumeRole for why the next line + // is necessary. + FlushInfoLog(); + + const pid_t child_pid = ExecDeathTestSpawnChild(args.Argv(), pipe_fd[0]); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_child_pid(child_pid); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; +} + +# endif // !GTEST_OS_WINDOWS + +// Creates a concrete DeathTest-derived class that depends on the +// --gtest_death_test_style flag, and sets the pointer pointed to +// by the "test" argument to its address. If the test should be +// skipped, sets that pointer to NULL. Returns true, unless the +// flag is set to an invalid value. +bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex, + const char* file, int line, + DeathTest** test) { + UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const int death_test_index = impl->current_test_info() + ->increment_death_test_count(); + + if (flag != NULL) { + if (death_test_index > flag->index()) { + DeathTest::set_last_death_test_message( + "Death test count (" + StreamableToString(death_test_index) + + ") somehow exceeded expected maximum (" + + StreamableToString(flag->index()) + ")"); + return false; + } + + if (!(flag->file() == file && flag->line() == line && + flag->index() == death_test_index)) { + *test = NULL; + return true; + } + } + +# if GTEST_OS_WINDOWS + + if (GTEST_FLAG(death_test_style) == "threadsafe" || + GTEST_FLAG(death_test_style) == "fast") { + *test = new WindowsDeathTest(statement, regex, file, line); + } + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") { + *test = new ExecDeathTest(statement, regex, file, line); + } else if (GTEST_FLAG(death_test_style) == "fast") { + *test = new NoExecDeathTest(statement, regex); + } + +# endif // GTEST_OS_WINDOWS + + else { // NOLINT - this is more readable than unbalanced brackets inside #if. + DeathTest::set_last_death_test_message( + "Unknown death test style \"" + GTEST_FLAG(death_test_style) + + "\" encountered"); + return false; + } + + return true; +} + +# if GTEST_OS_WINDOWS +// Recreates the pipe and event handles from the provided parameters, +// signals the event, and returns a file descriptor wrapped around the pipe +// handle. This function is called in the child process only. +int GetStatusFileDescriptor(unsigned int parent_process_id, + size_t write_handle_as_size_t, + size_t event_handle_as_size_t) { + AutoHandle parent_process_handle(::OpenProcess(PROCESS_DUP_HANDLE, + FALSE, // Non-inheritable. + parent_process_id)); + if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) { + DeathTestAbort("Unable to open parent process " + + StreamableToString(parent_process_id)); + } + + // TODO(vladl@google.com): Replace the following check with a + // compile-time assertion when available. + GTEST_CHECK_(sizeof(HANDLE) <= sizeof(size_t)); + + const HANDLE write_handle = + reinterpret_cast(write_handle_as_size_t); + HANDLE dup_write_handle; + + // The newly initialized handle is accessible only in in the parent + // process. To obtain one accessible within the child, we need to use + // DuplicateHandle. + if (!::DuplicateHandle(parent_process_handle.Get(), write_handle, + ::GetCurrentProcess(), &dup_write_handle, + 0x0, // Requested privileges ignored since + // DUPLICATE_SAME_ACCESS is used. + FALSE, // Request non-inheritable handler. + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort("Unable to duplicate the pipe handle " + + StreamableToString(write_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); + } + + const HANDLE event_handle = reinterpret_cast(event_handle_as_size_t); + HANDLE dup_event_handle; + + if (!::DuplicateHandle(parent_process_handle.Get(), event_handle, + ::GetCurrentProcess(), &dup_event_handle, + 0x0, + FALSE, + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort("Unable to duplicate the event handle " + + StreamableToString(event_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); + } + + const int write_fd = + ::_open_osfhandle(reinterpret_cast(dup_write_handle), O_APPEND); + if (write_fd == -1) { + DeathTestAbort("Unable to convert pipe handle " + + StreamableToString(write_handle_as_size_t) + + " to a file descriptor"); + } + + // Signals the parent that the write end of the pipe has been acquired + // so the parent can release its own write end. + ::SetEvent(dup_event_handle); + + return write_fd; +} +# endif // GTEST_OS_WINDOWS + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() { + if (GTEST_FLAG(internal_run_death_test) == "") return NULL; + + // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we + // can use it here. + int line = -1; + int index = -1; + ::std::vector< ::std::string> fields; + SplitString(GTEST_FLAG(internal_run_death_test).c_str(), '|', &fields); + int write_fd = -1; + +# if GTEST_OS_WINDOWS + + unsigned int parent_process_id = 0; + size_t write_handle_as_size_t = 0; + size_t event_handle_as_size_t = 0; + + if (fields.size() != 6 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &parent_process_id) + || !ParseNaturalNumber(fields[4], &write_handle_as_size_t) + || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) { + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); + } + write_fd = GetStatusFileDescriptor(parent_process_id, + write_handle_as_size_t, + event_handle_as_size_t); +# else + + if (fields.size() != 4 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &write_fd)) { + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); + } + +# endif // GTEST_OS_WINDOWS + + return new InternalRunDeathTestFlag(fields[0], line, index, write_fd); +} + +} // namespace internal + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace testing diff --git a/mypyc/external/googletest/src/gtest-filepath.cc b/mypyc/external/googletest/src/gtest-filepath.cc new file mode 100644 index 000000000000..0292dc11957e --- /dev/null +++ b/mypyc/external/googletest/src/gtest-filepath.cc @@ -0,0 +1,387 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: keith.ray@gmail.com (Keith Ray) + +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-filepath.h" +#include "gtest/internal/gtest-port.h" + +#include + +#if GTEST_OS_WINDOWS_MOBILE +# include +#elif GTEST_OS_WINDOWS +# include +# include +#elif GTEST_OS_SYMBIAN +// Symbian OpenC has PATH_MAX in sys/syslimits.h +# include +#else +# include +# include // Some Linux distributions define PATH_MAX here. +#endif // GTEST_OS_WINDOWS_MOBILE + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_MAX_ _MAX_PATH +#elif defined(PATH_MAX) +# define GTEST_PATH_MAX_ PATH_MAX +#elif defined(_XOPEN_PATH_MAX) +# define GTEST_PATH_MAX_ _XOPEN_PATH_MAX +#else +# define GTEST_PATH_MAX_ _POSIX_PATH_MAX +#endif // GTEST_OS_WINDOWS + +#include "gtest/internal/gtest-string.h" + +namespace testing { +namespace internal { + +#if GTEST_OS_WINDOWS +// On Windows, '\\' is the standard path separator, but many tools and the +// Windows API also accept '/' as an alternate path separator. Unless otherwise +// noted, a file path can contain either kind of path separators, or a mixture +// of them. +const char kPathSeparator = '\\'; +const char kAlternatePathSeparator = '/'; +const char kAlternatePathSeparatorString[] = "/"; +# if GTEST_OS_WINDOWS_MOBILE +// Windows CE doesn't have a current directory. You should not use +// the current directory in tests on Windows CE, but this at least +// provides a reasonable fallback. +const char kCurrentDirectoryString[] = "\\"; +// Windows CE doesn't define INVALID_FILE_ATTRIBUTES +const DWORD kInvalidFileAttributes = 0xffffffff; +# else +const char kCurrentDirectoryString[] = ".\\"; +# endif // GTEST_OS_WINDOWS_MOBILE +#else +const char kPathSeparator = '/'; +const char kCurrentDirectoryString[] = "./"; +#endif // GTEST_OS_WINDOWS + +// Returns whether the given character is a valid path separator. +static bool IsPathSeparator(char c) { +#if GTEST_HAS_ALT_PATH_SEP_ + return (c == kPathSeparator) || (c == kAlternatePathSeparator); +#else + return c == kPathSeparator; +#endif +} + +// Returns the current working directory, or "" if unsuccessful. +FilePath FilePath::GetCurrentDir() { +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + // Windows CE doesn't have a current directory, so we just return + // something reasonable. + return FilePath(kCurrentDirectoryString); +#elif GTEST_OS_WINDOWS + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd); +#else + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + char* result = getcwd(cwd, sizeof(cwd)); +# if GTEST_OS_NACL + // getcwd will likely fail in NaCl due to the sandbox, so return something + // reasonable. The user may have provided a shim implementation for getcwd, + // however, so fallback only when failure is detected. + return FilePath(result == NULL ? kCurrentDirectoryString : cwd); +# endif // GTEST_OS_NACL + return FilePath(result == NULL ? "" : cwd); +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns a copy of the FilePath with the case-insensitive extension removed. +// Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns +// FilePath("dir/file"). If a case-insensitive extension is not +// found, returns a copy of the original FilePath. +FilePath FilePath::RemoveExtension(const char* extension) const { + const std::string dot_extension = std::string(".") + extension; + if (String::EndsWithCaseInsensitive(pathname_, dot_extension)) { + return FilePath(pathname_.substr( + 0, pathname_.length() - dot_extension.length())); + } + return *this; +} + +// Returns a pointer to the last occurence of a valid path separator in +// the FilePath. On Windows, for example, both '/' and '\' are valid path +// separators. Returns NULL if no path separator was found. +const char* FilePath::FindLastPathSeparator() const { + const char* const last_sep = strrchr(c_str(), kPathSeparator); +#if GTEST_HAS_ALT_PATH_SEP_ + const char* const last_alt_sep = strrchr(c_str(), kAlternatePathSeparator); + // Comparing two pointers of which only one is NULL is undefined. + if (last_alt_sep != NULL && + (last_sep == NULL || last_alt_sep > last_sep)) { + return last_alt_sep; + } +#endif + return last_sep; +} + +// Returns a copy of the FilePath with the directory part removed. +// Example: FilePath("path/to/file").RemoveDirectoryName() returns +// FilePath("file"). If there is no directory part ("just_a_file"), it returns +// the FilePath unmodified. If there is no file part ("just_a_dir/") it +// returns an empty FilePath (""). +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveDirectoryName() const { + const char* const last_sep = FindLastPathSeparator(); + return last_sep ? FilePath(last_sep + 1) : *this; +} + +// RemoveFileName returns the directory path with the filename removed. +// Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". +// If the FilePath is "a_file" or "/a_file", RemoveFileName returns +// FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does +// not have a file, like "just/a/dir/", it returns the FilePath unmodified. +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveFileName() const { + const char* const last_sep = FindLastPathSeparator(); + std::string dir; + if (last_sep) { + dir = std::string(c_str(), last_sep + 1 - c_str()); + } else { + dir = kCurrentDirectoryString; + } + return FilePath(dir); +} + +// Helper functions for naming files in a directory for xml output. + +// Given directory = "dir", base_name = "test", number = 0, +// extension = "xml", returns "dir/test.xml". If number is greater +// than zero (e.g., 12), returns "dir/test_12.xml". +// On Windows platform, uses \ as the separator rather than /. +FilePath FilePath::MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension) { + std::string file; + if (number == 0) { + file = base_name.string() + "." + extension; + } else { + file = base_name.string() + "_" + StreamableToString(number) + + "." + extension; + } + return ConcatPaths(directory, FilePath(file)); +} + +// Given directory = "dir", relative_path = "test.xml", returns "dir/test.xml". +// On Windows, uses \ as the separator rather than /. +FilePath FilePath::ConcatPaths(const FilePath& directory, + const FilePath& relative_path) { + if (directory.IsEmpty()) + return relative_path; + const FilePath dir(directory.RemoveTrailingPathSeparator()); + return FilePath(dir.string() + kPathSeparator + relative_path.string()); +} + +// Returns true if pathname describes something findable in the file-system, +// either a file, directory, or whatever. +bool FilePath::FileOrDirectoryExists() const { +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + return attributes != kInvalidFileAttributes; +#else + posix::StatStruct file_stat; + return posix::Stat(pathname_.c_str(), &file_stat) == 0; +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns true if pathname describes a directory in the file-system +// that exists. +bool FilePath::DirectoryExists() const { + bool result = false; +#if GTEST_OS_WINDOWS + // Don't strip off trailing separator if path is a root directory on + // Windows (like "C:\\"). + const FilePath& path(IsRootDirectory() ? *this : + RemoveTrailingPathSeparator()); +#else + const FilePath& path(*this); +#endif + +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(path.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + if ((attributes != kInvalidFileAttributes) && + (attributes & FILE_ATTRIBUTE_DIRECTORY)) { + result = true; + } +#else + posix::StatStruct file_stat; + result = posix::Stat(path.c_str(), &file_stat) == 0 && + posix::IsDir(file_stat); +#endif // GTEST_OS_WINDOWS_MOBILE + + return result; +} + +// Returns true if pathname describes a root directory. (Windows has one +// root directory per disk drive.) +bool FilePath::IsRootDirectory() const { +#if GTEST_OS_WINDOWS + // TODO(wan@google.com): on Windows a network share like + // \\server\share can be a root directory, although it cannot be the + // current directory. Handle this properly. + return pathname_.length() == 3 && IsAbsolutePath(); +#else + return pathname_.length() == 1 && IsPathSeparator(pathname_.c_str()[0]); +#endif +} + +// Returns true if pathname describes an absolute path. +bool FilePath::IsAbsolutePath() const { + const char* const name = pathname_.c_str(); +#if GTEST_OS_WINDOWS + return pathname_.length() >= 3 && + ((name[0] >= 'a' && name[0] <= 'z') || + (name[0] >= 'A' && name[0] <= 'Z')) && + name[1] == ':' && + IsPathSeparator(name[2]); +#else + return IsPathSeparator(name[0]); +#endif +} + +// Returns a pathname for a file that does not currently exist. The pathname +// will be directory/base_name.extension or +// directory/base_name_.extension if directory/base_name.extension +// already exists. The number will be incremented until a pathname is found +// that does not already exist. +// Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. +// There could be a race condition if two or more processes are calling this +// function at the same time -- they could both pick the same filename. +FilePath FilePath::GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension) { + FilePath full_pathname; + int number = 0; + do { + full_pathname.Set(MakeFileName(directory, base_name, number++, extension)); + } while (full_pathname.FileOrDirectoryExists()); + return full_pathname; +} + +// Returns true if FilePath ends with a path separator, which indicates that +// it is intended to represent a directory. Returns false otherwise. +// This does NOT check that a directory (or file) actually exists. +bool FilePath::IsDirectory() const { + return !pathname_.empty() && + IsPathSeparator(pathname_.c_str()[pathname_.length() - 1]); +} + +// Create directories so that path exists. Returns true if successful or if +// the directories already exist; returns false if unable to create directories +// for any reason. +bool FilePath::CreateDirectoriesRecursively() const { + if (!this->IsDirectory()) { + return false; + } + + if (pathname_.length() == 0 || this->DirectoryExists()) { + return true; + } + + const FilePath parent(this->RemoveTrailingPathSeparator().RemoveFileName()); + return parent.CreateDirectoriesRecursively() && this->CreateFolder(); +} + +// Create the directory so that path exists. Returns true if successful or +// if the directory already exists; returns false if unable to create the +// directory for any reason, including if the parent directory does not +// exist. Not named "CreateDirectory" because that's a macro on Windows. +bool FilePath::CreateFolder() const { +#if GTEST_OS_WINDOWS_MOBILE + FilePath removed_sep(this->RemoveTrailingPathSeparator()); + LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); + int result = CreateDirectory(unicode, NULL) ? 0 : -1; + delete [] unicode; +#elif GTEST_OS_WINDOWS + int result = _mkdir(pathname_.c_str()); +#else + int result = mkdir(pathname_.c_str(), 0777); +#endif // GTEST_OS_WINDOWS_MOBILE + + if (result == -1) { + return this->DirectoryExists(); // An error is OK if the directory exists. + } + return true; // No error. +} + +// If input name has a trailing separator character, remove it and return the +// name, otherwise return the name string unmodified. +// On Windows platform, uses \ as the separator, other platforms use /. +FilePath FilePath::RemoveTrailingPathSeparator() const { + return IsDirectory() + ? FilePath(pathname_.substr(0, pathname_.length() - 1)) + : *this; +} + +// Removes any redundant separators that might be in the pathname. +// For example, "bar///foo" becomes "bar/foo". Does not eliminate other +// redundancies that might be in a pathname involving "." or "..". +// TODO(wan@google.com): handle Windows network shares (e.g. \\server\share). +void FilePath::Normalize() { + if (pathname_.c_str() == NULL) { + pathname_ = ""; + return; + } + const char* src = pathname_.c_str(); + char* const dest = new char[pathname_.length() + 1]; + char* dest_ptr = dest; + memset(dest_ptr, 0, pathname_.length() + 1); + + while (*src != '\0') { + *dest_ptr = *src; + if (!IsPathSeparator(*src)) { + src++; + } else { +#if GTEST_HAS_ALT_PATH_SEP_ + if (*dest_ptr == kAlternatePathSeparator) { + *dest_ptr = kPathSeparator; + } +#endif + while (IsPathSeparator(*src)) + src++; + } + dest_ptr++; + } + *dest_ptr = '\0'; + pathname_ = dest; + delete[] dest; +} + +} // namespace internal +} // namespace testing diff --git a/mypyc/external/googletest/src/gtest-internal-inl.h b/mypyc/external/googletest/src/gtest-internal-inl.h new file mode 100644 index 000000000000..ed8a682a964f --- /dev/null +++ b/mypyc/external/googletest/src/gtest-internal-inl.h @@ -0,0 +1,1183 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Utility functions and classes used by the Google C++ testing framework. +// +// Author: wan@google.com (Zhanyong Wan) +// +// This file contains purely Google Test's internal implementation. Please +// DO NOT #INCLUDE IT IN A USER PROGRAM. + +#ifndef GTEST_SRC_GTEST_INTERNAL_INL_H_ +#define GTEST_SRC_GTEST_INTERNAL_INL_H_ + +// GTEST_IMPLEMENTATION_ is defined to 1 iff the current translation unit is +// part of Google Test's implementation; otherwise it's undefined. +#if !GTEST_IMPLEMENTATION_ +// If this file is included from the user's code, just say no. +# error "gtest-internal-inl.h is part of Google Test's internal implementation." +# error "It must not be included except by Google Test itself." +#endif // GTEST_IMPLEMENTATION_ + +#ifndef _WIN32_WCE +# include +#endif // !_WIN32_WCE +#include +#include // For strtoll/_strtoul64/malloc/free. +#include // For memmove. + +#include +#include +#include + +#include "gtest/internal/gtest-port.h" + +#if GTEST_CAN_STREAM_RESULTS_ +# include // NOLINT +# include // NOLINT +#endif + +#if GTEST_OS_WINDOWS +# include // NOLINT +#endif // GTEST_OS_WINDOWS + +#include "gtest/gtest.h" // NOLINT +#include "gtest/gtest-spi.h" + +namespace testing { + +// Declares the flags. +// +// We don't want the users to modify this flag in the code, but want +// Google Test's own unit tests to be able to access it. Therefore we +// declare it here as opposed to in gtest.h. +GTEST_DECLARE_bool_(death_test_use_fork); + +namespace internal { + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +GTEST_API_ extern const TypeId kTestTypeIdInGoogleTest; + +// Names of the flags (needed for parsing Google Test flags). +const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests"; +const char kBreakOnFailureFlag[] = "break_on_failure"; +const char kCatchExceptionsFlag[] = "catch_exceptions"; +const char kColorFlag[] = "color"; +const char kFilterFlag[] = "filter"; +const char kListTestsFlag[] = "list_tests"; +const char kOutputFlag[] = "output"; +const char kPrintTimeFlag[] = "print_time"; +const char kRandomSeedFlag[] = "random_seed"; +const char kRepeatFlag[] = "repeat"; +const char kShuffleFlag[] = "shuffle"; +const char kStackTraceDepthFlag[] = "stack_trace_depth"; +const char kStreamResultToFlag[] = "stream_result_to"; +const char kThrowOnFailureFlag[] = "throw_on_failure"; +const char kFlagfileFlag[] = "flagfile"; + +// A valid random seed must be in [1, kMaxRandomSeed]. +const int kMaxRandomSeed = 99999; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +GTEST_API_ extern bool g_help_flag; + +// Returns the current time in milliseconds. +GTEST_API_ TimeInMillis GetTimeInMillis(); + +// Returns true iff Google Test should use colors in the output. +GTEST_API_ bool ShouldUseColor(bool stdout_is_tty); + +// Formats the given time in milliseconds as seconds. +GTEST_API_ std::string FormatTimeInMillisAsSeconds(TimeInMillis ms); + +// Converts the given time in milliseconds to a date string in the ISO 8601 +// format, without the timezone information. N.B.: due to the use the +// non-reentrant localtime() function, this function is not thread safe. Do +// not use it in any code that can be called from multiple threads. +GTEST_API_ std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms); + +// Parses a string for an Int32 flag, in the form of "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +GTEST_API_ bool ParseInt32Flag( + const char* str, const char* flag, Int32* value); + +// Returns a random seed in range [1, kMaxRandomSeed] based on the +// given --gtest_random_seed flag value. +inline int GetRandomSeedFromFlag(Int32 random_seed_flag) { + const unsigned int raw_seed = (random_seed_flag == 0) ? + static_cast(GetTimeInMillis()) : + static_cast(random_seed_flag); + + // Normalizes the actual seed to range [1, kMaxRandomSeed] such that + // it's easy to type. + const int normalized_seed = + static_cast((raw_seed - 1U) % + static_cast(kMaxRandomSeed)) + 1; + return normalized_seed; +} + +// Returns the first valid random seed after 'seed'. The behavior is +// undefined if 'seed' is invalid. The seed after kMaxRandomSeed is +// considered to be 1. +inline int GetNextRandomSeed(int seed) { + GTEST_CHECK_(1 <= seed && seed <= kMaxRandomSeed) + << "Invalid random seed " << seed << " - must be in [1, " + << kMaxRandomSeed << "]."; + const int next_seed = seed + 1; + return (next_seed > kMaxRandomSeed) ? 1 : next_seed; +} + +// This class saves the values of all Google Test flags in its c'tor, and +// restores them in its d'tor. +class GTestFlagSaver { + public: + // The c'tor. + GTestFlagSaver() { + also_run_disabled_tests_ = GTEST_FLAG(also_run_disabled_tests); + break_on_failure_ = GTEST_FLAG(break_on_failure); + catch_exceptions_ = GTEST_FLAG(catch_exceptions); + color_ = GTEST_FLAG(color); + death_test_style_ = GTEST_FLAG(death_test_style); + death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); + filter_ = GTEST_FLAG(filter); + internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); + list_tests_ = GTEST_FLAG(list_tests); + output_ = GTEST_FLAG(output); + print_time_ = GTEST_FLAG(print_time); + random_seed_ = GTEST_FLAG(random_seed); + repeat_ = GTEST_FLAG(repeat); + shuffle_ = GTEST_FLAG(shuffle); + stack_trace_depth_ = GTEST_FLAG(stack_trace_depth); + stream_result_to_ = GTEST_FLAG(stream_result_to); + throw_on_failure_ = GTEST_FLAG(throw_on_failure); + } + + // The d'tor is not virtual. DO NOT INHERIT FROM THIS CLASS. + ~GTestFlagSaver() { + GTEST_FLAG(also_run_disabled_tests) = also_run_disabled_tests_; + GTEST_FLAG(break_on_failure) = break_on_failure_; + GTEST_FLAG(catch_exceptions) = catch_exceptions_; + GTEST_FLAG(color) = color_; + GTEST_FLAG(death_test_style) = death_test_style_; + GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; + GTEST_FLAG(filter) = filter_; + GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; + GTEST_FLAG(list_tests) = list_tests_; + GTEST_FLAG(output) = output_; + GTEST_FLAG(print_time) = print_time_; + GTEST_FLAG(random_seed) = random_seed_; + GTEST_FLAG(repeat) = repeat_; + GTEST_FLAG(shuffle) = shuffle_; + GTEST_FLAG(stack_trace_depth) = stack_trace_depth_; + GTEST_FLAG(stream_result_to) = stream_result_to_; + GTEST_FLAG(throw_on_failure) = throw_on_failure_; + } + + private: + // Fields for saving the original values of flags. + bool also_run_disabled_tests_; + bool break_on_failure_; + bool catch_exceptions_; + std::string color_; + std::string death_test_style_; + bool death_test_use_fork_; + std::string filter_; + std::string internal_run_death_test_; + bool list_tests_; + std::string output_; + bool print_time_; + internal::Int32 random_seed_; + internal::Int32 repeat_; + bool shuffle_; + internal::Int32 stack_trace_depth_; + std::string stream_result_to_; + bool throw_on_failure_; +} GTEST_ATTRIBUTE_UNUSED_; + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +GTEST_API_ std::string CodePointToUtf8(UInt32 code_point); + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +GTEST_API_ std::string WideStringToUtf8(const wchar_t* str, int num_chars); + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded(); + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (e.g., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +GTEST_API_ bool ShouldShard(const char* total_shards_str, + const char* shard_index_str, + bool in_subprocess_for_death_test); + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error and +// and aborts. +GTEST_API_ Int32 Int32FromEnvOrDie(const char* env_var, Int32 default_val); + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +GTEST_API_ bool ShouldRunTestOnShard( + int total_shards, int shard_index, int test_id); + +// STL container utilities. + +// Returns the number of elements in the given container that satisfy +// the given predicate. +template +inline int CountIf(const Container& c, Predicate predicate) { + // Implemented as an explicit loop since std::count_if() in libCstd on + // Solaris has a non-standard signature. + int count = 0; + for (typename Container::const_iterator it = c.begin(); it != c.end(); ++it) { + if (predicate(*it)) + ++count; + } + return count; +} + +// Applies a function/functor to each element in the container. +template +void ForEach(const Container& c, Functor functor) { + std::for_each(c.begin(), c.end(), functor); +} + +// Returns the i-th element of the vector, or default_value if i is not +// in range [0, v.size()). +template +inline E GetElementOr(const std::vector& v, int i, E default_value) { + return (i < 0 || i >= static_cast(v.size())) ? default_value : v[i]; +} + +// Performs an in-place shuffle of a range of the vector's elements. +// 'begin' and 'end' are element indices as an STL-style range; +// i.e. [begin, end) are shuffled, where 'end' == size() means to +// shuffle to the end of the vector. +template +void ShuffleRange(internal::Random* random, int begin, int end, + std::vector* v) { + const int size = static_cast(v->size()); + GTEST_CHECK_(0 <= begin && begin <= size) + << "Invalid shuffle range start " << begin << ": must be in range [0, " + << size << "]."; + GTEST_CHECK_(begin <= end && end <= size) + << "Invalid shuffle range finish " << end << ": must be in range [" + << begin << ", " << size << "]."; + + // Fisher-Yates shuffle, from + // http://en.wikipedia.org/wiki/Fisher-Yates_shuffle + for (int range_width = end - begin; range_width >= 2; range_width--) { + const int last_in_range = begin + range_width - 1; + const int selected = begin + random->Generate(range_width); + std::swap((*v)[selected], (*v)[last_in_range]); + } +} + +// Performs an in-place shuffle of the vector's elements. +template +inline void Shuffle(internal::Random* random, std::vector* v) { + ShuffleRange(random, 0, static_cast(v->size()), v); +} + +// A function for deleting an object. Handy for being used as a +// functor. +template +static void Delete(T* x) { + delete x; +} + +// A predicate that checks the key of a TestProperty against a known key. +// +// TestPropertyKeyIs is copyable. +class TestPropertyKeyIs { + public: + // Constructor. + // + // TestPropertyKeyIs has NO default constructor. + explicit TestPropertyKeyIs(const std::string& key) : key_(key) {} + + // Returns true iff the test name of test property matches on key_. + bool operator()(const TestProperty& test_property) const { + return test_property.key() == key_; + } + + private: + std::string key_; +}; + +// Class UnitTestOptions. +// +// This class contains functions for processing options the user +// specifies when running the tests. It has only static members. +// +// In most cases, the user can specify an option using either an +// environment variable or a command line flag. E.g. you can set the +// test filter using either GTEST_FILTER or --gtest_filter. If both +// the variable and the flag are present, the latter overrides the +// former. +class GTEST_API_ UnitTestOptions { + public: + // Functions for processing the gtest_output flag. + + // Returns the output format, or "" for normal printed output. + static std::string GetOutputFormat(); + + // Returns the absolute path of the requested output file, or the + // default (test_detail.xml in the original working directory) if + // none was explicitly specified. + static std::string GetAbsolutePathToOutputFile(); + + // Functions for processing the gtest_filter flag. + + // Returns true iff the wildcard pattern matches the string. The + // first ':' or '\0' character in pattern marks the end of it. + // + // This recursive algorithm isn't very efficient, but is clear and + // works well enough for matching test names, which are short. + static bool PatternMatchesString(const char *pattern, const char *str); + + // Returns true iff the user-specified filter matches the test case + // name and the test name. + static bool FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name); + +#if GTEST_OS_WINDOWS + // Function for supporting the gtest_catch_exception flag. + + // Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the + // given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. + // This function is useful as an __except condition. + static int GTestShouldProcessSEH(DWORD exception_code); +#endif // GTEST_OS_WINDOWS + + // Returns true if "name" matches the ':' separated list of glob-style + // filters in "filter". + static bool MatchesFilter(const std::string& name, const char* filter); +}; + +// Returns the current application's name, removing directory path if that +// is present. Used by UnitTestOptions::GetOutputFile. +GTEST_API_ FilePath GetCurrentExecutableName(); + +// The role interface for getting the OS stack trace as a string. +class OsStackTraceGetterInterface { + public: + OsStackTraceGetterInterface() {} + virtual ~OsStackTraceGetterInterface() {} + + // Returns the current OS stack trace as an std::string. Parameters: + // + // max_depth - the maximum number of stack frames to be included + // in the trace. + // skip_count - the number of top frames to be skipped; doesn't count + // against max_depth. + virtual string CurrentStackTrace(int max_depth, int skip_count) = 0; + + // UponLeavingGTest() should be called immediately before Google Test calls + // user code. It saves some information about the current stack that + // CurrentStackTrace() will use to find and hide Google Test stack frames. + virtual void UponLeavingGTest() = 0; + + // This string is inserted in place of stack frames that are part of + // Google Test's implementation. + static const char* const kElidedFramesMarker; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetterInterface); +}; + +// A working implementation of the OsStackTraceGetterInterface interface. +class OsStackTraceGetter : public OsStackTraceGetterInterface { + public: + OsStackTraceGetter() {} + + virtual string CurrentStackTrace(int max_depth, int skip_count); + virtual void UponLeavingGTest(); + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetter); +}; + +// Information about a Google Test trace point. +struct TraceInfo { + const char* file; + int line; + std::string message; +}; + +// This is the default global test part result reporter used in UnitTestImpl. +// This class should only be used by UnitTestImpl. +class DefaultGlobalTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultGlobalTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. Reports the test part + // result in the current test. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultGlobalTestPartResultReporter); +}; + +// This is the default per thread test part result reporter used in +// UnitTestImpl. This class should only be used by UnitTestImpl. +class DefaultPerThreadTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultPerThreadTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. The implementation just + // delegates to the current global test part result reporter of *unit_test_. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultPerThreadTestPartResultReporter); +}; + +// The private implementation of the UnitTest class. We don't protect +// the methods under a mutex, as this class is not accessible by a +// user and the UnitTest class that delegates work to this class does +// proper locking. +class GTEST_API_ UnitTestImpl { + public: + explicit UnitTestImpl(UnitTest* parent); + virtual ~UnitTestImpl(); + + // There are two different ways to register your own TestPartResultReporter. + // You can register your own repoter to listen either only for test results + // from the current thread or for results from all threads. + // By default, each per-thread test result repoter just passes a new + // TestPartResult to the global test result reporter, which registers the + // test part result for the currently running test. + + // Returns the global test part result reporter. + TestPartResultReporterInterface* GetGlobalTestPartResultReporter(); + + // Sets the global test part result reporter. + void SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter); + + // Returns the test part result reporter for the current thread. + TestPartResultReporterInterface* GetTestPartResultReporterForCurrentThread(); + + // Sets the test part result reporter for the current thread. + void SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter); + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const { return start_timestamp_; } + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const { return !Failed(); } + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const { + return failed_test_case_count() > 0 || ad_hoc_test_result()->Failed(); + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[i]; + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i) { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[index]; + } + + // Provides access to the event listener list. + TestEventListeners* listeners() { return &listeners_; } + + // Returns the TestResult for the test that's currently running, or + // the TestResult for the ad hoc test if no test is running. + TestResult* current_test_result(); + + // Returns the TestResult for the ad hoc test. + const TestResult* ad_hoc_test_result() const { return &ad_hoc_test_result_; } + + // Sets the OS stack trace getter. + // + // Does nothing if the input and the current OS stack trace getter + // are the same; otherwise, deletes the old getter and makes the + // input the current getter. + void set_os_stack_trace_getter(OsStackTraceGetterInterface* getter); + + // Returns the current OS stack trace getter if it is not NULL; + // otherwise, creates an OsStackTraceGetter, makes it the current + // getter, and returns it. + OsStackTraceGetterInterface* os_stack_trace_getter(); + + // Returns the current OS stack trace as an std::string. + // + // The maximum number of stack frames to be included is specified by + // the gtest_stack_trace_depth flag. The skip_count parameter + // specifies the number of top frames to be skipped, which doesn't + // count against the number of frames to be included. + // + // For example, if Foo() calls Bar(), which in turn calls + // CurrentOsStackTraceExceptTop(1), Foo() will be included in the + // trace but Bar() and CurrentOsStackTraceExceptTop() won't. + std::string CurrentOsStackTraceExceptTop(int skip_count) GTEST_NO_INLINE_; + + // Finds and returns a TestCase with the given name. If one doesn't + // exist, creates one and returns it. + // + // Arguments: + // + // test_case_name: name of the test case + // type_param: the name of the test's type parameter, or NULL if + // this is not a typed or a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase* GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Adds a TestInfo to the unit test. + // + // Arguments: + // + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + // test_info: the TestInfo object + void AddTestInfo(Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + TestInfo* test_info) { + // In order to support thread-safe death tests, we need to + // remember the original working directory when the test program + // was first invoked. We cannot do this in RUN_ALL_TESTS(), as + // the user may have changed the current directory before calling + // RUN_ALL_TESTS(). Therefore we capture the current directory in + // AddTestInfo(), which is called to register a TEST or TEST_F + // before main() is reached. + if (original_working_dir_.IsEmpty()) { + original_working_dir_.Set(FilePath::GetCurrentDir()); + GTEST_CHECK_(!original_working_dir_.IsEmpty()) + << "Failed to get the current working directory."; + } + + GetTestCase(test_info->test_case_name(), + test_info->type_param(), + set_up_tc, + tear_down_tc)->AddTestInfo(test_info); + } + +#if GTEST_HAS_PARAM_TEST + // Returns ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() { + return parameterized_test_registry_; + } +#endif // GTEST_HAS_PARAM_TEST + + // Sets the TestCase object for the test that's currently running. + void set_current_test_case(TestCase* a_current_test_case) { + current_test_case_ = a_current_test_case; + } + + // Sets the TestInfo object for the test that's currently running. If + // current_test_info is NULL, the assertion results will be stored in + // ad_hoc_test_result_. + void set_current_test_info(TestInfo* a_current_test_info) { + current_test_info_ = a_current_test_info; + } + + // Registers all parameterized tests defined using TEST_P and + // INSTANTIATE_TEST_CASE_P, creating regular tests for each test/parameter + // combination. This method can be called more then once; it has guards + // protecting from registering the tests more then once. If + // value-parameterized tests are disabled, RegisterParameterizedTests is + // present but does nothing. + void RegisterParameterizedTests(); + + // Runs all tests in this UnitTest object, prints the result, and + // returns true if all tests are successful. If any exception is + // thrown during a test, this test is considered to be failed, but + // the rest of the tests will still be run. + bool RunAllTests(); + + // Clears the results of all tests, except the ad hoc tests. + void ClearNonAdHocTestResult() { + ForEach(test_cases_, TestCase::ClearTestCaseResult); + } + + // Clears the results of ad-hoc test assertions. + void ClearAdHocTestResult() { + ad_hoc_test_result_.Clear(); + } + + // Adds a TestProperty to the current TestResult object when invoked in a + // context of a test or a test case, or to the global property set. If the + // result already contains a property with the same key, the value will be + // updated. + void RecordProperty(const TestProperty& test_property); + + enum ReactionToSharding { + HONOR_SHARDING_PROTOCOL, + IGNORE_SHARDING_PROTOCOL + }; + + // Matches the full name of each test against the user-specified + // filter to decide whether the test should run, then records the + // result in each TestCase and TestInfo object. + // If shard_tests == HONOR_SHARDING_PROTOCOL, further filters tests + // based on sharding variables in the environment. + // Returns the number of tests that should run. + int FilterTests(ReactionToSharding shard_tests); + + // Prints the names of the tests matching the user-specified filter flag. + void ListTestsMatchingFilter(); + + const TestCase* current_test_case() const { return current_test_case_; } + TestInfo* current_test_info() { return current_test_info_; } + const TestInfo* current_test_info() const { return current_test_info_; } + + // Returns the vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector& environments() { return environments_; } + + // Getters for the per-thread Google Test trace stack. + std::vector& gtest_trace_stack() { + return *(gtest_trace_stack_.pointer()); + } + const std::vector& gtest_trace_stack() const { + return gtest_trace_stack_.get(); + } + +#if GTEST_HAS_DEATH_TEST + void InitDeathTestSubprocessControlInfo() { + internal_run_death_test_flag_.reset(ParseInternalRunDeathTestFlag()); + } + // Returns a pointer to the parsed --gtest_internal_run_death_test + // flag, or NULL if that flag was not specified. + // This information is useful only in a death test child process. + // Must not be called before a call to InitGoogleTest. + const InternalRunDeathTestFlag* internal_run_death_test_flag() const { + return internal_run_death_test_flag_.get(); + } + + // Returns a pointer to the current death test factory. + internal::DeathTestFactory* death_test_factory() { + return death_test_factory_.get(); + } + + void SuppressTestEventsIfInSubprocess(); + + friend class ReplaceDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + + // Initializes the event listener performing XML output as specified by + // UnitTestOptions. Must not be called before InitGoogleTest. + void ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Initializes the event listener for streaming test results to a socket. + // Must not be called before InitGoogleTest. + void ConfigureStreamingOutput(); +#endif + + // Performs initialization dependent upon flag values obtained in + // ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to + // ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest + // this function is also called from RunAllTests. Since this function can be + // called more than once, it has to be idempotent. + void PostFlagParsingInit(); + + // Gets the random seed used at the start of the current test iteration. + int random_seed() const { return random_seed_; } + + // Gets the random number generator. + internal::Random* random() { return &random_; } + + // Shuffles all test cases, and the tests within each test case, + // making sure that death tests are still run first. + void ShuffleTests(); + + // Restores the test cases and tests to their order before the first shuffle. + void UnshuffleTests(); + + // Returns the value of GTEST_FLAG(catch_exceptions) at the moment + // UnitTest::Run() starts. + bool catch_exceptions() const { return catch_exceptions_; } + + private: + friend class ::testing::UnitTest; + + // Used by UnitTest::Run() to capture the state of + // GTEST_FLAG(catch_exceptions) at the moment it starts. + void set_catch_exceptions(bool value) { catch_exceptions_ = value; } + + // The UnitTest object that owns this implementation object. + UnitTest* const parent_; + + // The working directory when the first TEST() or TEST_F() was + // executed. + internal::FilePath original_working_dir_; + + // The default test part result reporters. + DefaultGlobalTestPartResultReporter default_global_test_part_result_reporter_; + DefaultPerThreadTestPartResultReporter + default_per_thread_test_part_result_reporter_; + + // Points to (but doesn't own) the global test part result reporter. + TestPartResultReporterInterface* global_test_part_result_repoter_; + + // Protects read and write access to global_test_part_result_reporter_. + internal::Mutex global_test_part_result_reporter_mutex_; + + // Points to (but doesn't own) the per-thread test part result reporter. + internal::ThreadLocal + per_thread_test_part_result_reporter_; + + // The vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector environments_; + + // The vector of TestCases in their original order. It owns the + // elements in the vector. + std::vector test_cases_; + + // Provides a level of indirection for the test case list to allow + // easy shuffling and restoring the test case order. The i-th + // element of this vector is the index of the i-th test case in the + // shuffled order. + std::vector test_case_indices_; + +#if GTEST_HAS_PARAM_TEST + // ParameterizedTestRegistry object used to register value-parameterized + // tests. + internal::ParameterizedTestCaseRegistry parameterized_test_registry_; + + // Indicates whether RegisterParameterizedTests() has been called already. + bool parameterized_tests_registered_; +#endif // GTEST_HAS_PARAM_TEST + + // Index of the last death test case registered. Initially -1. + int last_death_test_case_; + + // This points to the TestCase for the currently running test. It + // changes as Google Test goes through one test case after another. + // When no test is running, this is set to NULL and Google Test + // stores assertion results in ad_hoc_test_result_. Initially NULL. + TestCase* current_test_case_; + + // This points to the TestInfo for the currently running test. It + // changes as Google Test goes through one test after another. When + // no test is running, this is set to NULL and Google Test stores + // assertion results in ad_hoc_test_result_. Initially NULL. + TestInfo* current_test_info_; + + // Normally, a user only writes assertions inside a TEST or TEST_F, + // or inside a function called by a TEST or TEST_F. Since Google + // Test keeps track of which test is current running, it can + // associate such an assertion with the test it belongs to. + // + // If an assertion is encountered when no TEST or TEST_F is running, + // Google Test attributes the assertion result to an imaginary "ad hoc" + // test, and records the result in ad_hoc_test_result_. + TestResult ad_hoc_test_result_; + + // The list of event listeners that can be used to track events inside + // Google Test. + TestEventListeners listeners_; + + // The OS stack trace getter. Will be deleted when the UnitTest + // object is destructed. By default, an OsStackTraceGetter is used, + // but the user can set this field to use a custom getter if that is + // desired. + OsStackTraceGetterInterface* os_stack_trace_getter_; + + // True iff PostFlagParsingInit() has been called. + bool post_flag_parse_init_performed_; + + // The random number seed used at the beginning of the test run. + int random_seed_; + + // Our random number generator. + internal::Random random_; + + // The time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp_; + + // How long the test took to run, in milliseconds. + TimeInMillis elapsed_time_; + +#if GTEST_HAS_DEATH_TEST + // The decomposed components of the gtest_internal_run_death_test flag, + // parsed when RUN_ALL_TESTS is called. + internal::scoped_ptr internal_run_death_test_flag_; + internal::scoped_ptr death_test_factory_; +#endif // GTEST_HAS_DEATH_TEST + + // A per-thread stack of traces created by the SCOPED_TRACE() macro. + internal::ThreadLocal > gtest_trace_stack_; + + // The value of GTEST_FLAG(catch_exceptions) at the moment RunAllTests() + // starts. + bool catch_exceptions_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTestImpl); +}; // class UnitTestImpl + +// Convenience function for accessing the global UnitTest +// implementation object. +inline UnitTestImpl* GetUnitTestImpl() { + return UnitTest::GetInstance()->impl(); +} + +#if GTEST_USES_SIMPLE_RE + +// Internal helper functions for implementing the simple regular +// expression matcher. +GTEST_API_ bool IsInSet(char ch, const char* str); +GTEST_API_ bool IsAsciiDigit(char ch); +GTEST_API_ bool IsAsciiPunct(char ch); +GTEST_API_ bool IsRepeat(char ch); +GTEST_API_ bool IsAsciiWhiteSpace(char ch); +GTEST_API_ bool IsAsciiWordChar(char ch); +GTEST_API_ bool IsValidEscape(char ch); +GTEST_API_ bool AtomMatchesChar(bool escaped, char pattern, char ch); +GTEST_API_ bool ValidateRegex(const char* regex); +GTEST_API_ bool MatchRegexAtHead(const char* regex, const char* str); +GTEST_API_ bool MatchRepetitionAndRegexAtHead( + bool escaped, char ch, char repeat, const char* regex, const char* str); +GTEST_API_ bool MatchRegexAnywhere(const char* regex, const char* str); + +#endif // GTEST_USES_SIMPLE_RE + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, char** argv); +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv); + +#if GTEST_HAS_DEATH_TEST + +// Returns the message describing the last system error, regardless of the +// platform. +GTEST_API_ std::string GetLastErrnoDescription(); + +// Attempts to parse a string into a positive integer pointed to by the +// number parameter. Returns true if that is possible. +// GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we can use +// it here. +template +bool ParseNaturalNumber(const ::std::string& str, Integer* number) { + // Fail fast if the given string does not begin with a digit; + // this bypasses strtoXXX's "optional leading whitespace and plus + // or minus sign" semantics, which are undesirable here. + if (str.empty() || !IsDigit(str[0])) { + return false; + } + errno = 0; + + char* end; + // BiggestConvertible is the largest integer type that system-provided + // string-to-number conversion routines can return. + +# if GTEST_OS_WINDOWS && !defined(__GNUC__) + + // MSVC and C++ Builder define __int64 instead of the standard long long. + typedef unsigned __int64 BiggestConvertible; + const BiggestConvertible parsed = _strtoui64(str.c_str(), &end, 10); + +# else + + typedef unsigned long long BiggestConvertible; // NOLINT + const BiggestConvertible parsed = strtoull(str.c_str(), &end, 10); + +# endif // GTEST_OS_WINDOWS && !defined(__GNUC__) + + const bool parse_success = *end == '\0' && errno == 0; + + // TODO(vladl@google.com): Convert this to compile time assertion when it is + // available. + GTEST_CHECK_(sizeof(Integer) <= sizeof(parsed)); + + const Integer result = static_cast(parsed); + if (parse_success && static_cast(result) == parsed) { + *number = result; + return true; + } + return false; +} +#endif // GTEST_HAS_DEATH_TEST + +// TestResult contains some private methods that should be hidden from +// Google Test user but are required for testing. This class allow our tests +// to access them. +// +// This class is supplied only for the purpose of testing Google Test's own +// constructs. Do not use it in user tests, either directly or indirectly. +class TestResultAccessor { + public: + static void RecordProperty(TestResult* test_result, + const std::string& xml_element, + const TestProperty& property) { + test_result->RecordProperty(xml_element, property); + } + + static void ClearTestPartResults(TestResult* test_result) { + test_result->ClearTestPartResults(); + } + + static const std::vector& test_part_results( + const TestResult& test_result) { + return test_result.test_part_results(); + } +}; + +#if GTEST_CAN_STREAM_RESULTS_ + +// Streams test results to the given port on the given host machine. +class GTEST_API_ StreamingListener : public EmptyTestEventListener { + public: + // Abstract base class for writing strings to a socket. + class AbstractSocketWriter { + public: + virtual ~AbstractSocketWriter() {} + + // Sends a string to the socket. + virtual void Send(const string& message) = 0; + + // Closes the socket. + virtual void CloseConnection() {} + + // Sends a string and a newline to the socket. + void SendLn(const string& message) { + Send(message + "\n"); + } + }; + + // Concrete class for actually writing strings to a socket. + class SocketWriter : public AbstractSocketWriter { + public: + SocketWriter(const string& host, const string& port) + : sockfd_(-1), host_name_(host), port_num_(port) { + MakeConnection(); + } + + virtual ~SocketWriter() { + if (sockfd_ != -1) + CloseConnection(); + } + + // Sends a string to the socket. + virtual void Send(const string& message) { + GTEST_CHECK_(sockfd_ != -1) + << "Send() can be called only when there is a connection."; + + const int len = static_cast(message.length()); + if (write(sockfd_, message.c_str(), len) != len) { + GTEST_LOG_(WARNING) + << "stream_result_to: failed to stream to " + << host_name_ << ":" << port_num_; + } + } + + private: + // Creates a client socket and connects to the server. + void MakeConnection(); + + // Closes the socket. + void CloseConnection() { + GTEST_CHECK_(sockfd_ != -1) + << "CloseConnection() can be called only when there is a connection."; + + close(sockfd_); + sockfd_ = -1; + } + + int sockfd_; // socket file descriptor + const string host_name_; + const string port_num_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SocketWriter); + }; // class SocketWriter + + // Escapes '=', '&', '%', and '\n' characters in str as "%xx". + static string UrlEncode(const char* str); + + StreamingListener(const string& host, const string& port) + : socket_writer_(new SocketWriter(host, port)) { Start(); } + + explicit StreamingListener(AbstractSocketWriter* socket_writer) + : socket_writer_(socket_writer) { Start(); } + + void OnTestProgramStart(const UnitTest& /* unit_test */) { + SendLn("event=TestProgramStart"); + } + + void OnTestProgramEnd(const UnitTest& unit_test) { + // Note that Google Test current only report elapsed time for each + // test iteration, not for the entire test program. + SendLn("event=TestProgramEnd&passed=" + FormatBool(unit_test.Passed())); + + // Notify the streaming server to stop. + socket_writer_->CloseConnection(); + } + + void OnTestIterationStart(const UnitTest& /* unit_test */, int iteration) { + SendLn("event=TestIterationStart&iteration=" + + StreamableToString(iteration)); + } + + void OnTestIterationEnd(const UnitTest& unit_test, int /* iteration */) { + SendLn("event=TestIterationEnd&passed=" + + FormatBool(unit_test.Passed()) + "&elapsed_time=" + + StreamableToString(unit_test.elapsed_time()) + "ms"); + } + + void OnTestCaseStart(const TestCase& test_case) { + SendLn(std::string("event=TestCaseStart&name=") + test_case.name()); + } + + void OnTestCaseEnd(const TestCase& test_case) { + SendLn("event=TestCaseEnd&passed=" + FormatBool(test_case.Passed()) + + "&elapsed_time=" + StreamableToString(test_case.elapsed_time()) + + "ms"); + } + + void OnTestStart(const TestInfo& test_info) { + SendLn(std::string("event=TestStart&name=") + test_info.name()); + } + + void OnTestEnd(const TestInfo& test_info) { + SendLn("event=TestEnd&passed=" + + FormatBool((test_info.result())->Passed()) + + "&elapsed_time=" + + StreamableToString((test_info.result())->elapsed_time()) + "ms"); + } + + void OnTestPartResult(const TestPartResult& test_part_result) { + const char* file_name = test_part_result.file_name(); + if (file_name == NULL) + file_name = ""; + SendLn("event=TestPartResult&file=" + UrlEncode(file_name) + + "&line=" + StreamableToString(test_part_result.line_number()) + + "&message=" + UrlEncode(test_part_result.message())); + } + + private: + // Sends the given message and a newline to the socket. + void SendLn(const string& message) { socket_writer_->SendLn(message); } + + // Called at the start of streaming to notify the receiver what + // protocol we are using. + void Start() { SendLn("gtest_streaming_protocol_version=1.0"); } + + string FormatBool(bool value) { return value ? "1" : "0"; } + + const scoped_ptr socket_writer_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamingListener); +}; // class StreamingListener + +#endif // GTEST_CAN_STREAM_RESULTS_ + +} // namespace internal +} // namespace testing + +#endif // GTEST_SRC_GTEST_INTERNAL_INL_H_ diff --git a/mypyc/external/googletest/src/gtest-port.cc b/mypyc/external/googletest/src/gtest-port.cc new file mode 100644 index 000000000000..e5bf3dd2be4b --- /dev/null +++ b/mypyc/external/googletest/src/gtest-port.cc @@ -0,0 +1,1259 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +#include "gtest/internal/gtest-port.h" + +#include +#include +#include +#include +#include + +#if GTEST_OS_WINDOWS +# include +# include +# include +# include // Used in ThreadLocal. +#else +# include +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_MAC +# include +# include +# include +#endif // GTEST_OS_MAC + +#if GTEST_OS_QNX +# include +# include +# include +#endif // GTEST_OS_QNX + +#if GTEST_OS_AIX +# include +# include +#endif // GTEST_OS_AIX + +#include "gtest/gtest-spi.h" +#include "gtest/gtest-message.h" +#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-string.h" + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +namespace testing { +namespace internal { + +#if defined(_MSC_VER) || defined(__BORLANDC__) +// MSVC and C++Builder do not provide a definition of STDERR_FILENO. +const int kStdOutFileno = 1; +const int kStdErrFileno = 2; +#else +const int kStdOutFileno = STDOUT_FILENO; +const int kStdErrFileno = STDERR_FILENO; +#endif // _MSC_VER + +#if GTEST_OS_LINUX + +namespace { +template +T ReadProcFileField(const string& filename, int field) { + std::string dummy; + std::ifstream file(filename.c_str()); + while (field-- > 0) { + file >> dummy; + } + T output = 0; + file >> output; + return output; +} +} // namespace + +// Returns the number of active threads, or 0 when there is an error. +size_t GetThreadCount() { + const string filename = + (Message() << "/proc/" << getpid() << "/stat").GetString(); + return ReadProcFileField(filename, 19); +} + +#elif GTEST_OS_MAC + +size_t GetThreadCount() { + const task_t task = mach_task_self(); + mach_msg_type_number_t thread_count; + thread_act_array_t thread_list; + const kern_return_t status = task_threads(task, &thread_list, &thread_count); + if (status == KERN_SUCCESS) { + // task_threads allocates resources in thread_list and we need to free them + // to avoid leaks. + vm_deallocate(task, + reinterpret_cast(thread_list), + sizeof(thread_t) * thread_count); + return static_cast(thread_count); + } else { + return 0; + } +} + +#elif GTEST_OS_QNX + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +size_t GetThreadCount() { + const int fd = open("/proc/self/as", O_RDONLY); + if (fd < 0) { + return 0; + } + procfs_info process_info; + const int status = + devctl(fd, DCMD_PROC_INFO, &process_info, sizeof(process_info), NULL); + close(fd); + if (status == EOK) { + return static_cast(process_info.num_threads); + } else { + return 0; + } +} + +#elif GTEST_OS_AIX + +size_t GetThreadCount() { + struct procentry64 entry; + pid_t pid = getpid(); + int status = getprocs64(&entry, sizeof(entry), NULL, 0, &pid, 1); + if (status == 1) { + return entry.pi_thcount; + } else { + return 0; + } +} + +#else + +size_t GetThreadCount() { + // There's no portable way to detect the number of threads, so we just + // return 0 to indicate that we cannot detect it. + return 0; +} + +#endif // GTEST_OS_LINUX + +#if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + +void SleepMilliseconds(int n) { + ::Sleep(n); +} + +AutoHandle::AutoHandle() + : handle_(INVALID_HANDLE_VALUE) {} + +AutoHandle::AutoHandle(Handle handle) + : handle_(handle) {} + +AutoHandle::~AutoHandle() { + Reset(); +} + +AutoHandle::Handle AutoHandle::Get() const { + return handle_; +} + +void AutoHandle::Reset() { + Reset(INVALID_HANDLE_VALUE); +} + +void AutoHandle::Reset(HANDLE handle) { + // Resetting with the same handle we already own is invalid. + if (handle_ != handle) { + if (IsCloseable()) { + ::CloseHandle(handle_); + } + handle_ = handle; + } else { + GTEST_CHECK_(!IsCloseable()) + << "Resetting a valid handle to itself is likely a programmer error " + "and thus not allowed."; + } +} + +bool AutoHandle::IsCloseable() const { + // Different Windows APIs may use either of these values to represent an + // invalid handle. + return handle_ != NULL && handle_ != INVALID_HANDLE_VALUE; +} + +Notification::Notification() + : event_(::CreateEvent(NULL, // Default security attributes. + TRUE, // Do not reset automatically. + FALSE, // Initially unset. + NULL)) { // Anonymous event. + GTEST_CHECK_(event_.Get() != NULL); +} + +void Notification::Notify() { + GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE); +} + +void Notification::WaitForNotification() { + GTEST_CHECK_( + ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0); +} + +Mutex::Mutex() + : owner_thread_id_(0), + type_(kDynamic), + critical_section_init_phase_(0), + critical_section_(new CRITICAL_SECTION) { + ::InitializeCriticalSection(critical_section_); +} + +Mutex::~Mutex() { + // Static mutexes are leaked intentionally. It is not thread-safe to try + // to clean them up. + // TODO(yukawa): Switch to Slim Reader/Writer (SRW) Locks, which requires + // nothing to clean it up but is available only on Vista and later. + // http://msdn.microsoft.com/en-us/library/windows/desktop/aa904937.aspx + if (type_ == kDynamic) { + ::DeleteCriticalSection(critical_section_); + delete critical_section_; + critical_section_ = NULL; + } +} + +void Mutex::Lock() { + ThreadSafeLazyInit(); + ::EnterCriticalSection(critical_section_); + owner_thread_id_ = ::GetCurrentThreadId(); +} + +void Mutex::Unlock() { + ThreadSafeLazyInit(); + // We don't protect writing to owner_thread_id_ here, as it's the + // caller's responsibility to ensure that the current thread holds the + // mutex when this is called. + owner_thread_id_ = 0; + ::LeaveCriticalSection(critical_section_); +} + +// Does nothing if the current thread holds the mutex. Otherwise, crashes +// with high probability. +void Mutex::AssertHeld() { + ThreadSafeLazyInit(); + GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId()) + << "The current thread is not holding the mutex @" << this; +} + +// Initializes owner_thread_id_ and critical_section_ in static mutexes. +void Mutex::ThreadSafeLazyInit() { + // Dynamic mutexes are initialized in the constructor. + if (type_ == kStatic) { + switch ( + ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) { + case 0: + // If critical_section_init_phase_ was 0 before the exchange, we + // are the first to test it and need to perform the initialization. + owner_thread_id_ = 0; + critical_section_ = new CRITICAL_SECTION; + ::InitializeCriticalSection(critical_section_); + // Updates the critical_section_init_phase_ to 2 to signal + // initialization complete. + GTEST_CHECK_(::InterlockedCompareExchange( + &critical_section_init_phase_, 2L, 1L) == + 1L); + break; + case 1: + // Somebody else is already initializing the mutex; spin until they + // are done. + while (::InterlockedCompareExchange(&critical_section_init_phase_, + 2L, + 2L) != 2L) { + // Possibly yields the rest of the thread's time slice to other + // threads. + ::Sleep(0); + } + break; + + case 2: + break; // The mutex is already initialized and ready for use. + + default: + GTEST_CHECK_(false) + << "Unexpected value of critical_section_init_phase_ " + << "while initializing a static mutex."; + } + } +} + +namespace { + +class ThreadWithParamSupport : public ThreadWithParamBase { + public: + static HANDLE CreateThread(Runnable* runnable, + Notification* thread_can_start) { + ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start); + DWORD thread_id; + // TODO(yukawa): Consider to use _beginthreadex instead. + HANDLE thread_handle = ::CreateThread( + NULL, // Default security. + 0, // Default stack size. + &ThreadWithParamSupport::ThreadMain, + param, // Parameter to ThreadMainStatic + 0x0, // Default creation flags. + &thread_id); // Need a valid pointer for the call to work under Win98. + GTEST_CHECK_(thread_handle != NULL) << "CreateThread failed with error " + << ::GetLastError() << "."; + if (thread_handle == NULL) { + delete param; + } + return thread_handle; + } + + private: + struct ThreadMainParam { + ThreadMainParam(Runnable* runnable, Notification* thread_can_start) + : runnable_(runnable), + thread_can_start_(thread_can_start) { + } + scoped_ptr runnable_; + // Does not own. + Notification* thread_can_start_; + }; + + static DWORD WINAPI ThreadMain(void* ptr) { + // Transfers ownership. + scoped_ptr param(static_cast(ptr)); + if (param->thread_can_start_ != NULL) + param->thread_can_start_->WaitForNotification(); + param->runnable_->Run(); + return 0; + } + + // Prohibit instantiation. + ThreadWithParamSupport(); + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport); +}; + +} // namespace + +ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable, + Notification* thread_can_start) + : thread_(ThreadWithParamSupport::CreateThread(runnable, + thread_can_start)) { +} + +ThreadWithParamBase::~ThreadWithParamBase() { + Join(); +} + +void ThreadWithParamBase::Join() { + GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0) + << "Failed to join the thread with error " << ::GetLastError() << "."; +} + +// Maps a thread to a set of ThreadIdToThreadLocals that have values +// instantiated on that thread and notifies them when the thread exits. A +// ThreadLocal instance is expected to persist until all threads it has +// values on have terminated. +class ThreadLocalRegistryImpl { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + DWORD current_thread = ::GetCurrentThreadId(); + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(current_thread); + if (thread_local_pos == thread_to_thread_locals->end()) { + thread_local_pos = thread_to_thread_locals->insert( + std::make_pair(current_thread, ThreadLocalValues())).first; + StartWatcherThreadFor(current_thread); + } + ThreadLocalValues& thread_local_values = thread_local_pos->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos == thread_local_values.end()) { + value_pos = + thread_local_values + .insert(std::make_pair( + thread_local_instance, + linked_ptr( + thread_local_instance->NewValueForCurrentThread()))) + .first; + } + return value_pos->second.get(); + } + + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + std::vector > value_holders; + // Clean up the ThreadLocalValues data structure while holding the lock, but + // defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + for (ThreadIdToThreadLocals::iterator it = + thread_to_thread_locals->begin(); + it != thread_to_thread_locals->end(); + ++it) { + ThreadLocalValues& thread_local_values = it->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos != thread_local_values.end()) { + value_holders.push_back(value_pos->second); + thread_local_values.erase(value_pos); + // This 'if' can only be successful at most once, so theoretically we + // could break out of the loop here, but we don't bother doing so. + } + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + static void OnThreadExit(DWORD thread_id) { + GTEST_CHECK_(thread_id != 0) << ::GetLastError(); + std::vector > value_holders; + // Clean up the ThreadIdToThreadLocals data structure while holding the + // lock, but defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(thread_id); + if (thread_local_pos != thread_to_thread_locals->end()) { + ThreadLocalValues& thread_local_values = thread_local_pos->second; + for (ThreadLocalValues::iterator value_pos = + thread_local_values.begin(); + value_pos != thread_local_values.end(); + ++value_pos) { + value_holders.push_back(value_pos->second); + } + thread_to_thread_locals->erase(thread_local_pos); + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + private: + // In a particular thread, maps a ThreadLocal object to its value. + typedef std::map > ThreadLocalValues; + // Stores all ThreadIdToThreadLocals having values in a thread, indexed by + // thread's ID. + typedef std::map ThreadIdToThreadLocals; + + // Holds the thread id and thread handle that we pass from + // StartWatcherThreadFor to WatcherThreadFunc. + typedef std::pair ThreadIdAndHandle; + + static void StartWatcherThreadFor(DWORD thread_id) { + // The returned handle will be kept in thread_map and closed by + // watcher_thread in WatcherThreadFunc. + HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION, + FALSE, + thread_id); + GTEST_CHECK_(thread != NULL); + // We need to to pass a valid thread ID pointer into CreateThread for it + // to work correctly under Win98. + DWORD watcher_thread_id; + HANDLE watcher_thread = ::CreateThread( + NULL, // Default security. + 0, // Default stack size + &ThreadLocalRegistryImpl::WatcherThreadFunc, + reinterpret_cast(new ThreadIdAndHandle(thread_id, thread)), + CREATE_SUSPENDED, + &watcher_thread_id); + GTEST_CHECK_(watcher_thread != NULL); + // Give the watcher thread the same priority as ours to avoid being + // blocked by it. + ::SetThreadPriority(watcher_thread, + ::GetThreadPriority(::GetCurrentThread())); + ::ResumeThread(watcher_thread); + ::CloseHandle(watcher_thread); + } + + // Monitors exit from a given thread and notifies those + // ThreadIdToThreadLocals about thread termination. + static DWORD WINAPI WatcherThreadFunc(LPVOID param) { + const ThreadIdAndHandle* tah = + reinterpret_cast(param); + GTEST_CHECK_( + ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0); + OnThreadExit(tah->first); + ::CloseHandle(tah->second); + delete tah; + return 0; + } + + // Returns map of thread local instances. + static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() { + mutex_.AssertHeld(); + static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals; + return map; + } + + // Protects access to GetThreadLocalsMapLocked() and its return value. + static Mutex mutex_; + // Protects access to GetThreadMapLocked() and its return value. + static Mutex thread_map_mutex_; +}; + +Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); +Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); + +ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + return ThreadLocalRegistryImpl::GetValueOnCurrentThread( + thread_local_instance); +} + +void ThreadLocalRegistry::OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance); +} + +#endif // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + +#if GTEST_USES_POSIX_RE + +// Implements RE. Currently only needed for death tests. + +RE::~RE() { + if (is_valid_) { + // regfree'ing an invalid regex might crash because the content + // of the regex is undefined. Since the regex's are essentially + // the same, one cannot be valid (or invalid) without the other + // being so too. + regfree(&partial_regex_); + regfree(&full_regex_); + } + free(const_cast(pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.full_regex_, str, 1, &match, 0) == 0; +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.partial_regex_, str, 1, &match, 0) == 0; +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = posix::StrDup(regex); + + // Reserves enough bytes to hold the regular expression used for a + // full match. + const size_t full_regex_len = strlen(regex) + 10; + char* const full_pattern = new char[full_regex_len]; + + snprintf(full_pattern, full_regex_len, "^(%s)$", regex); + is_valid_ = regcomp(&full_regex_, full_pattern, REG_EXTENDED) == 0; + // We want to call regcomp(&partial_regex_, ...) even if the + // previous expression returns false. Otherwise partial_regex_ may + // not be properly initialized can may cause trouble when it's + // freed. + // + // Some implementation of POSIX regex (e.g. on at least some + // versions of Cygwin) doesn't accept the empty string as a valid + // regex. We change it to an equivalent form "()" to be safe. + if (is_valid_) { + const char* const partial_regex = (*regex == '\0') ? "()" : regex; + is_valid_ = regcomp(&partial_regex_, partial_regex, REG_EXTENDED) == 0; + } + EXPECT_TRUE(is_valid_) + << "Regular expression \"" << regex + << "\" is not a valid POSIX Extended regular expression."; + + delete[] full_pattern; +} + +#elif GTEST_USES_SIMPLE_RE + +// Returns true iff ch appears anywhere in str (excluding the +// terminating '\0' character). +bool IsInSet(char ch, const char* str) { + return ch != '\0' && strchr(str, ch) != NULL; +} + +// Returns true iff ch belongs to the given classification. Unlike +// similar functions in , these aren't affected by the +// current locale. +bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; } +bool IsAsciiPunct(char ch) { + return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~"); +} +bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); } +bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); } +bool IsAsciiWordChar(char ch) { + return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || + ('0' <= ch && ch <= '9') || ch == '_'; +} + +// Returns true iff "\\c" is a supported escape sequence. +bool IsValidEscape(char c) { + return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW")); +} + +// Returns true iff the given atom (specified by escaped and pattern) +// matches ch. The result is undefined if the atom is invalid. +bool AtomMatchesChar(bool escaped, char pattern_char, char ch) { + if (escaped) { // "\\p" where p is pattern_char. + switch (pattern_char) { + case 'd': return IsAsciiDigit(ch); + case 'D': return !IsAsciiDigit(ch); + case 'f': return ch == '\f'; + case 'n': return ch == '\n'; + case 'r': return ch == '\r'; + case 's': return IsAsciiWhiteSpace(ch); + case 'S': return !IsAsciiWhiteSpace(ch); + case 't': return ch == '\t'; + case 'v': return ch == '\v'; + case 'w': return IsAsciiWordChar(ch); + case 'W': return !IsAsciiWordChar(ch); + } + return IsAsciiPunct(pattern_char) && pattern_char == ch; + } + + return (pattern_char == '.' && ch != '\n') || pattern_char == ch; +} + +// Helper function used by ValidateRegex() to format error messages. +std::string FormatRegexSyntaxError(const char* regex, int index) { + return (Message() << "Syntax error at index " << index + << " in simple regular expression \"" << regex << "\": ").GetString(); +} + +// Generates non-fatal failures and returns false if regex is invalid; +// otherwise returns true. +bool ValidateRegex(const char* regex) { + if (regex == NULL) { + // TODO(wan@google.com): fix the source file location in the + // assertion failures to match where the regex is used in user + // code. + ADD_FAILURE() << "NULL is not a valid simple regular expression."; + return false; + } + + bool is_valid = true; + + // True iff ?, *, or + can follow the previous atom. + bool prev_repeatable = false; + for (int i = 0; regex[i]; i++) { + if (regex[i] == '\\') { // An escape sequence + i++; + if (regex[i] == '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "'\\' cannot appear at the end."; + return false; + } + + if (!IsValidEscape(regex[i])) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "invalid escape sequence \"\\" << regex[i] << "\"."; + is_valid = false; + } + prev_repeatable = true; + } else { // Not an escape sequence. + const char ch = regex[i]; + + if (ch == '^' && i > 0) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'^' can only appear at the beginning."; + is_valid = false; + } else if (ch == '$' && regex[i + 1] != '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'$' can only appear at the end."; + is_valid = false; + } else if (IsInSet(ch, "()[]{}|")) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' is unsupported."; + is_valid = false; + } else if (IsRepeat(ch) && !prev_repeatable) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' can only follow a repeatable token."; + is_valid = false; + } + + prev_repeatable = !IsInSet(ch, "^$?*+"); + } + } + + return is_valid; +} + +// Matches a repeated regex atom followed by a valid simple regular +// expression. The regex atom is defined as c if escaped is false, +// or \c otherwise. repeat is the repetition meta character (?, *, +// or +). The behavior is undefined if str contains too many +// characters to be indexable by size_t, in which case the test will +// probably time out anyway. We are fine with this limitation as +// std::string has it too. +bool MatchRepetitionAndRegexAtHead( + bool escaped, char c, char repeat, const char* regex, + const char* str) { + const size_t min_count = (repeat == '+') ? 1 : 0; + const size_t max_count = (repeat == '?') ? 1 : + static_cast(-1) - 1; + // We cannot call numeric_limits::max() as it conflicts with the + // max() macro on Windows. + + for (size_t i = 0; i <= max_count; ++i) { + // We know that the atom matches each of the first i characters in str. + if (i >= min_count && MatchRegexAtHead(regex, str + i)) { + // We have enough matches at the head, and the tail matches too. + // Since we only care about *whether* the pattern matches str + // (as opposed to *how* it matches), there is no need to find a + // greedy match. + return true; + } + if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i])) + return false; + } + return false; +} + +// Returns true iff regex matches a prefix of str. regex must be a +// valid simple regular expression and not start with "^", or the +// result is undefined. +bool MatchRegexAtHead(const char* regex, const char* str) { + if (*regex == '\0') // An empty regex matches a prefix of anything. + return true; + + // "$" only matches the end of a string. Note that regex being + // valid guarantees that there's nothing after "$" in it. + if (*regex == '$') + return *str == '\0'; + + // Is the first thing in regex an escape sequence? + const bool escaped = *regex == '\\'; + if (escaped) + ++regex; + if (IsRepeat(regex[1])) { + // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so + // here's an indirect recursion. It terminates as the regex gets + // shorter in each recursion. + return MatchRepetitionAndRegexAtHead( + escaped, regex[0], regex[1], regex + 2, str); + } else { + // regex isn't empty, isn't "$", and doesn't start with a + // repetition. We match the first atom of regex with the first + // character of str and recurse. + return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) && + MatchRegexAtHead(regex + 1, str + 1); + } +} + +// Returns true iff regex matches any substring of str. regex must be +// a valid simple regular expression, or the result is undefined. +// +// The algorithm is recursive, but the recursion depth doesn't exceed +// the regex length, so we won't need to worry about running out of +// stack space normally. In rare cases the time complexity can be +// exponential with respect to the regex length + the string length, +// but usually it's must faster (often close to linear). +bool MatchRegexAnywhere(const char* regex, const char* str) { + if (regex == NULL || str == NULL) + return false; + + if (*regex == '^') + return MatchRegexAtHead(regex + 1, str); + + // A successful match can be anywhere in str. + do { + if (MatchRegexAtHead(regex, str)) + return true; + } while (*str++ != '\0'); + return false; +} + +// Implements the RE class. + +RE::~RE() { + free(const_cast(pattern_)); + free(const_cast(full_pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_, str); +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.pattern_, str); +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = full_pattern_ = NULL; + if (regex != NULL) { + pattern_ = posix::StrDup(regex); + } + + is_valid_ = ValidateRegex(regex); + if (!is_valid_) { + // No need to calculate the full pattern when the regex is invalid. + return; + } + + const size_t len = strlen(regex); + // Reserves enough bytes to hold the regular expression used for a + // full match: we need space to prepend a '^', append a '$', and + // terminate the string with '\0'. + char* buffer = static_cast(malloc(len + 3)); + full_pattern_ = buffer; + + if (*regex != '^') + *buffer++ = '^'; // Makes sure full_pattern_ starts with '^'. + + // We don't use snprintf or strncpy, as they trigger a warning when + // compiled with VC++ 8.0. + memcpy(buffer, regex, len); + buffer += len; + + if (len == 0 || regex[len - 1] != '$') + *buffer++ = '$'; // Makes sure full_pattern_ ends with '$'. + + *buffer = '\0'; +} + +#endif // GTEST_USES_POSIX_RE + +const char kUnknownFile[] = "unknown file"; + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) { + const std::string file_name(file == NULL ? kUnknownFile : file); + + if (line < 0) { + return file_name + ":"; + } +#ifdef _MSC_VER + return file_name + "(" + StreamableToString(line) + "):"; +#else + return file_name + ":" + StreamableToString(line) + ":"; +#endif // _MSC_VER +} + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +// Note that FormatCompilerIndependentFileLocation() does NOT append colon +// to the file location it produces, unlike FormatFileLocation(). +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation( + const char* file, int line) { + const std::string file_name(file == NULL ? kUnknownFile : file); + + if (line < 0) + return file_name; + else + return file_name + ":" + StreamableToString(line); +} + +GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line) + : severity_(severity) { + const char* const marker = + severity == GTEST_INFO ? "[ INFO ]" : + severity == GTEST_WARNING ? "[WARNING]" : + severity == GTEST_ERROR ? "[ ERROR ]" : "[ FATAL ]"; + GetStream() << ::std::endl << marker << " " + << FormatFileLocation(file, line).c_str() << ": "; +} + +// Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. +GTestLog::~GTestLog() { + GetStream() << ::std::endl; + if (severity_ == GTEST_FATAL) { + fflush(stderr); + posix::Abort(); + } +} +// Disable Microsoft deprecation warnings for POSIX functions called from +// this class (creat, dup, dup2, and close) +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) + +#if GTEST_HAS_STREAM_REDIRECTION + +// Object that captures an output stream (stdout/stderr). +class CapturedStream { + public: + // The ctor redirects the stream to a temporary file. + explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) { +# if GTEST_OS_WINDOWS + char temp_dir_path[MAX_PATH + 1] = { '\0' }; // NOLINT + char temp_file_path[MAX_PATH + 1] = { '\0' }; // NOLINT + + ::GetTempPathA(sizeof(temp_dir_path), temp_dir_path); + const UINT success = ::GetTempFileNameA(temp_dir_path, + "gtest_redir", + 0, // Generate unique file name. + temp_file_path); + GTEST_CHECK_(success != 0) + << "Unable to create a temporary file in " << temp_dir_path; + const int captured_fd = creat(temp_file_path, _S_IREAD | _S_IWRITE); + GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file " + << temp_file_path; + filename_ = temp_file_path; +# else + // There's no guarantee that a test has write access to the current + // directory, so we create the temporary file in the /tmp directory + // instead. We use /tmp on most systems, and /sdcard on Android. + // That's because Android doesn't have /tmp. +# if GTEST_OS_LINUX_ANDROID + // Note: Android applications are expected to call the framework's + // Context.getExternalStorageDirectory() method through JNI to get + // the location of the world-writable SD Card directory. However, + // this requires a Context handle, which cannot be retrieved + // globally from native code. Doing so also precludes running the + // code as part of a regular standalone executable, which doesn't + // run in a Dalvik process (e.g. when running it through 'adb shell'). + // + // The location /sdcard is directly accessible from native code + // and is the only location (unofficially) supported by the Android + // team. It's generally a symlink to the real SD Card mount point + // which can be /mnt/sdcard, /mnt/sdcard0, /system/media/sdcard, or + // other OEM-customized locations. Never rely on these, and always + // use /sdcard. + char name_template[] = "/sdcard/gtest_captured_stream.XXXXXX"; +# else + char name_template[] = "/tmp/captured_stream.XXXXXX"; +# endif // GTEST_OS_LINUX_ANDROID + const int captured_fd = mkstemp(name_template); + filename_ = name_template; +# endif // GTEST_OS_WINDOWS + fflush(NULL); + dup2(captured_fd, fd_); + close(captured_fd); + } + + ~CapturedStream() { + remove(filename_.c_str()); + } + + std::string GetCapturedString() { + if (uncaptured_fd_ != -1) { + // Restores the original stream. + fflush(NULL); + dup2(uncaptured_fd_, fd_); + close(uncaptured_fd_); + uncaptured_fd_ = -1; + } + + FILE* const file = posix::FOpen(filename_.c_str(), "r"); + const std::string content = ReadEntireFile(file); + posix::FClose(file); + return content; + } + + private: + const int fd_; // A stream to capture. + int uncaptured_fd_; + // Name of the temporary file holding the stderr output. + ::std::string filename_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream); +}; + +GTEST_DISABLE_MSC_WARNINGS_POP_() + +static CapturedStream* g_captured_stderr = NULL; +static CapturedStream* g_captured_stdout = NULL; + +// Starts capturing an output stream (stdout/stderr). +void CaptureStream(int fd, const char* stream_name, CapturedStream** stream) { + if (*stream != NULL) { + GTEST_LOG_(FATAL) << "Only one " << stream_name + << " capturer can exist at a time."; + } + *stream = new CapturedStream(fd); +} + +// Stops capturing the output stream and returns the captured string. +std::string GetCapturedStream(CapturedStream** captured_stream) { + const std::string content = (*captured_stream)->GetCapturedString(); + + delete *captured_stream; + *captured_stream = NULL; + + return content; +} + +// Starts capturing stdout. +void CaptureStdout() { + CaptureStream(kStdOutFileno, "stdout", &g_captured_stdout); +} + +// Starts capturing stderr. +void CaptureStderr() { + CaptureStream(kStdErrFileno, "stderr", &g_captured_stderr); +} + +// Stops capturing stdout and returns the captured string. +std::string GetCapturedStdout() { + return GetCapturedStream(&g_captured_stdout); +} + +// Stops capturing stderr and returns the captured string. +std::string GetCapturedStderr() { + return GetCapturedStream(&g_captured_stderr); +} + +#endif // GTEST_HAS_STREAM_REDIRECTION + +std::string TempDir() { +#if GTEST_OS_WINDOWS_MOBILE + return "\\temp\\"; +#elif GTEST_OS_WINDOWS + const char* temp_dir = posix::GetEnv("TEMP"); + if (temp_dir == NULL || temp_dir[0] == '\0') + return "\\temp\\"; + else if (temp_dir[strlen(temp_dir) - 1] == '\\') + return temp_dir; + else + return std::string(temp_dir) + "\\"; +#elif GTEST_OS_LINUX_ANDROID + return "/sdcard/"; +#else + return "/tmp/"; +#endif // GTEST_OS_WINDOWS_MOBILE +} + +size_t GetFileSize(FILE* file) { + fseek(file, 0, SEEK_END); + return static_cast(ftell(file)); +} + +std::string ReadEntireFile(FILE* file) { + const size_t file_size = GetFileSize(file); + char* const buffer = new char[file_size]; + + size_t bytes_last_read = 0; // # of bytes read in the last fread() + size_t bytes_read = 0; // # of bytes read so far + + fseek(file, 0, SEEK_SET); + + // Keeps reading the file until we cannot read further or the + // pre-determined file size is reached. + do { + bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file); + bytes_read += bytes_last_read; + } while (bytes_last_read > 0 && bytes_read < file_size); + + const std::string content(buffer, bytes_read); + delete[] buffer; + + return content; +} + +#if GTEST_HAS_DEATH_TEST + +static const ::std::vector* g_injected_test_argvs = + NULL; // Owned. + +void SetInjectableArgvs(const ::std::vector* argvs) { + if (g_injected_test_argvs != argvs) + delete g_injected_test_argvs; + g_injected_test_argvs = argvs; +} + +const ::std::vector& GetInjectableArgvs() { + if (g_injected_test_argvs != NULL) { + return *g_injected_test_argvs; + } + return GetArgvs(); +} +#endif // GTEST_HAS_DEATH_TEST + +#if GTEST_OS_WINDOWS_MOBILE +namespace posix { +void Abort() { + DebugBreak(); + TerminateProcess(GetCurrentProcess(), 1); +} +} // namespace posix +#endif // GTEST_OS_WINDOWS_MOBILE + +// Returns the name of the environment variable corresponding to the +// given flag. For example, FlagToEnvVar("foo") will return +// "GTEST_FOO" in the open-source version. +static std::string FlagToEnvVar(const char* flag) { + const std::string full_flag = + (Message() << GTEST_FLAG_PREFIX_ << flag).GetString(); + + Message env_var; + for (size_t i = 0; i != full_flag.length(); i++) { + env_var << ToUpper(full_flag.c_str()[i]); + } + + return env_var.GetString(); +} + +// Parses 'str' for a 32-bit signed integer. If successful, writes +// the result to *value and returns true; otherwise leaves *value +// unchanged and returns false. +bool ParseInt32(const Message& src_text, const char* str, Int32* value) { + // Parses the environment variable as a decimal integer. + char* end = NULL; + const long long_value = strtol(str, &end, 10); // NOLINT + + // Has strtol() consumed all characters in the string? + if (*end != '\0') { + // No - an invalid character was encountered. + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value \"" << str << "\".\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + // Is the parsed value in the range of an Int32? + const Int32 result = static_cast(long_value); + if (long_value == LONG_MAX || long_value == LONG_MIN || + // The parsed value overflows as a long. (strtol() returns + // LONG_MAX or LONG_MIN when the input overflows.) + result != long_value + // The parsed value overflows as an Int32. + ) { + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value " << str << ", which overflows.\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + *value = result; + return true; +} + +// Reads and returns the Boolean environment variable corresponding to +// the given flag; if it's not set, returns default_value. +// +// The value is considered true iff it's not "0". +bool BoolFromGTestEnv(const char* flag, bool default_value) { +#if defined(GTEST_GET_BOOL_FROM_ENV_) + return GTEST_GET_BOOL_FROM_ENV_(flag, default_value); +#endif // defined(GTEST_GET_BOOL_FROM_ENV_) + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + return string_value == NULL ? + default_value : strcmp(string_value, "0") != 0; +} + +// Reads and returns a 32-bit integer stored in the environment +// variable corresponding to the given flag; if it isn't set or +// doesn't represent a valid 32-bit integer, returns default_value. +Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) { +#if defined(GTEST_GET_INT32_FROM_ENV_) + return GTEST_GET_INT32_FROM_ENV_(flag, default_value); +#endif // defined(GTEST_GET_INT32_FROM_ENV_) + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + if (string_value == NULL) { + // The environment variable is not set. + return default_value; + } + + Int32 result = default_value; + if (!ParseInt32(Message() << "Environment variable " << env_var, + string_value, &result)) { + printf("The default value %s is used.\n", + (Message() << default_value).GetString().c_str()); + fflush(stdout); + return default_value; + } + + return result; +} + +// Reads and returns the string environment variable corresponding to +// the given flag; if it's not set, returns default_value. +std::string StringFromGTestEnv(const char* flag, const char* default_value) { +#if defined(GTEST_GET_STRING_FROM_ENV_) + return GTEST_GET_STRING_FROM_ENV_(flag, default_value); +#endif // defined(GTEST_GET_STRING_FROM_ENV_) + const std::string env_var = FlagToEnvVar(flag); + const char* value = posix::GetEnv(env_var.c_str()); + if (value != NULL) { + return value; + } + + // As a special case for the 'output' flag, if GTEST_OUTPUT is not + // set, we look for XML_OUTPUT_FILE, which is set by the Bazel build + // system. The value of XML_OUTPUT_FILE is a filename without the + // "xml:" prefix of GTEST_OUTPUT. + // + // The net priority order after flag processing is thus: + // --gtest_output command line flag + // GTEST_OUTPUT environment variable + // XML_OUTPUT_FILE environment variable + // 'default_value' + if (strcmp(flag, "output") == 0) { + value = posix::GetEnv("XML_OUTPUT_FILE"); + if (value != NULL) { + return std::string("xml:") + value; + } + } + return default_value; +} + +} // namespace internal +} // namespace testing diff --git a/mypyc/external/googletest/src/gtest-printers.cc b/mypyc/external/googletest/src/gtest-printers.cc new file mode 100644 index 000000000000..a2df412f8a20 --- /dev/null +++ b/mypyc/external/googletest/src/gtest-printers.cc @@ -0,0 +1,373 @@ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Google Test - The Google C++ Testing Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// It uses the << operator when possible, and prints the bytes in the +// object otherwise. A user can override its behavior for a class +// type Foo by defining either operator<<(::std::ostream&, const Foo&) +// or void PrintTo(const Foo&, ::std::ostream*) in the namespace that +// defines Foo. + +#include "gtest/gtest-printers.h" +#include +#include +#include +#include // NOLINT +#include +#include "gtest/internal/gtest-port.h" + +namespace testing { + +namespace { + +using ::std::ostream; + +// Prints a segment of bytes in the given object. +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start, + size_t count, ostream* os) { + char text[5] = ""; + for (size_t i = 0; i != count; i++) { + const size_t j = start + i; + if (i != 0) { + // Organizes the bytes into groups of 2 for easy parsing by + // human. + if ((j % 2) == 0) + *os << ' '; + else + *os << '-'; + } + GTEST_SNPRINTF_(text, sizeof(text), "%02X", obj_bytes[j]); + *os << text; + } +} + +// Prints the bytes in the given value to the given ostream. +void PrintBytesInObjectToImpl(const unsigned char* obj_bytes, size_t count, + ostream* os) { + // Tells the user how big the object is. + *os << count << "-byte object <"; + + const size_t kThreshold = 132; + const size_t kChunkSize = 64; + // If the object size is bigger than kThreshold, we'll have to omit + // some details by printing only the first and the last kChunkSize + // bytes. + // TODO(wan): let the user control the threshold using a flag. + if (count < kThreshold) { + PrintByteSegmentInObjectTo(obj_bytes, 0, count, os); + } else { + PrintByteSegmentInObjectTo(obj_bytes, 0, kChunkSize, os); + *os << " ... "; + // Rounds up to 2-byte boundary. + const size_t resume_pos = (count - kChunkSize + 1)/2*2; + PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os); + } + *os << ">"; +} + +} // namespace + +namespace internal2 { + +// Delegates to PrintBytesInObjectToImpl() to print the bytes in the +// given object. The delegation simplifies the implementation, which +// uses the << operator and thus is easier done outside of the +// ::testing::internal namespace, which contains a << operator that +// sometimes conflicts with the one in STL. +void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, + ostream* os) { + PrintBytesInObjectToImpl(obj_bytes, count, os); +} + +} // namespace internal2 + +namespace internal { + +// Depending on the value of a char (or wchar_t), we print it in one +// of three formats: +// - as is if it's a printable ASCII (e.g. 'a', '2', ' '), +// - as a hexidecimal escape sequence (e.g. '\x7F'), or +// - as a special escape sequence (e.g. '\r', '\n'). +enum CharFormat { + kAsIs, + kHexEscape, + kSpecialEscape +}; + +// Returns true if c is a printable ASCII character. We test the +// value of c directly instead of calling isprint(), which is buggy on +// Windows Mobile. +inline bool IsPrintableAscii(wchar_t c) { + return 0x20 <= c && c <= 0x7E; +} + +// Prints a wide or narrow char c as a character literal without the +// quotes, escaping it when necessary; returns how c was formatted. +// The template argument UnsignedChar is the unsigned version of Char, +// which is the type of c. +template +static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { + switch (static_cast(c)) { + case L'\0': + *os << "\\0"; + break; + case L'\'': + *os << "\\'"; + break; + case L'\\': + *os << "\\\\"; + break; + case L'\a': + *os << "\\a"; + break; + case L'\b': + *os << "\\b"; + break; + case L'\f': + *os << "\\f"; + break; + case L'\n': + *os << "\\n"; + break; + case L'\r': + *os << "\\r"; + break; + case L'\t': + *os << "\\t"; + break; + case L'\v': + *os << "\\v"; + break; + default: + if (IsPrintableAscii(c)) { + *os << static_cast(c); + return kAsIs; + } else { + *os << "\\x" + String::FormatHexInt(static_cast(c)); + return kHexEscape; + } + } + return kSpecialEscape; +} + +// Prints a wchar_t c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { + switch (c) { + case L'\'': + *os << "'"; + return kAsIs; + case L'"': + *os << "\\\""; + return kSpecialEscape; + default: + return PrintAsCharLiteralTo(c, os); + } +} + +// Prints a char c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsStringLiteralTo(char c, ostream* os) { + return PrintAsStringLiteralTo( + static_cast(static_cast(c)), os); +} + +// Prints a wide or narrow character c and its code. '\0' is printed +// as "'\\0'", other unprintable characters are also properly escaped +// using the standard C++ escape sequence. The template argument +// UnsignedChar is the unsigned version of Char, which is the type of c. +template +void PrintCharAndCodeTo(Char c, ostream* os) { + // First, print c as a literal in the most readable form we can find. + *os << ((sizeof(c) > 1) ? "L'" : "'"); + const CharFormat format = PrintAsCharLiteralTo(c, os); + *os << "'"; + + // To aid user debugging, we also print c's code in decimal, unless + // it's 0 (in which case c was printed as '\\0', making the code + // obvious). + if (c == 0) + return; + *os << " (" << static_cast(c); + + // For more convenience, we print c's code again in hexidecimal, + // unless c was already printed in the form '\x##' or the code is in + // [1, 9]. + if (format == kHexEscape || (1 <= c && c <= 9)) { + // Do nothing. + } else { + *os << ", 0x" << String::FormatHexInt(static_cast(c)); + } + *os << ")"; +} + +void PrintTo(unsigned char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} +void PrintTo(signed char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} + +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its code. L'\0' is printed as "L'\\0'". +void PrintTo(wchar_t wc, ostream* os) { + PrintCharAndCodeTo(wc, os); +} + +// Prints the given array of characters to the ostream. CharType must be either +// char or wchar_t. +// The array starts at begin, the length is len, it may include '\0' characters +// and may not be NUL-terminated. +template +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static void PrintCharsAsStringTo( + const CharType* begin, size_t len, ostream* os) { + const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\""; + *os << kQuoteBegin; + bool is_previous_hex = false; + for (size_t index = 0; index < len; ++index) { + const CharType cur = begin[index]; + if (is_previous_hex && IsXDigit(cur)) { + // Previous character is of '\x..' form and this character can be + // interpreted as another hexadecimal digit in its number. Break string to + // disambiguate. + *os << "\" " << kQuoteBegin; + } + is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape; + } + *os << "\""; +} + +// Prints a (const) char/wchar_t array of 'len' elements, starting at address +// 'begin'. CharType must be either char or wchar_t. +template +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static void UniversalPrintCharArray( + const CharType* begin, size_t len, ostream* os) { + // The code + // const char kFoo[] = "foo"; + // generates an array of 4, not 3, elements, with the last one being '\0'. + // + // Therefore when printing a char array, we don't print the last element if + // it's '\0', such that the output matches the string literal as it's + // written in the source code. + if (len > 0 && begin[len - 1] == '\0') { + PrintCharsAsStringTo(begin, len - 1, os); + return; + } + + // If, however, the last element in the array is not '\0', e.g. + // const char kFoo[] = { 'f', 'o', 'o' }; + // we must print the entire array. We also print a message to indicate + // that the array is not NUL-terminated. + PrintCharsAsStringTo(begin, len, os); + *os << " (no terminating NUL)"; +} + +// Prints a (const) char array of 'len' elements, starting at address 'begin'. +void UniversalPrintArray(const char* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints a (const) wchar_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints the given C string to the ostream. +void PrintTo(const char* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintCharsAsStringTo(s, strlen(s), os); + } +} + +// MSVC compiler can be configured to define whar_t as a typedef +// of unsigned short. Defining an overload for const wchar_t* in that case +// would cause pointers to unsigned shorts be printed as wide strings, +// possibly accessing more memory than intended and causing invalid +// memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when +// wchar_t is implemented as a native type. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Prints the given wide C string to the ostream. +void PrintTo(const wchar_t* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintCharsAsStringTo(s, std::wcslen(s), os); + } +} +#endif // wchar_t is native + +// Prints a ::string object. +#if GTEST_HAS_GLOBAL_STRING +void PrintStringTo(const ::string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_GLOBAL_STRING + +void PrintStringTo(const ::std::string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} + +// Prints a ::wstring object. +#if GTEST_HAS_GLOBAL_WSTRING +void PrintWideStringTo(const ::wstring& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +void PrintWideStringTo(const ::std::wstring& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_STD_WSTRING + +} // namespace internal + +} // namespace testing diff --git a/mypyc/external/googletest/src/gtest-test-part.cc b/mypyc/external/googletest/src/gtest-test-part.cc new file mode 100644 index 000000000000..fb0e35425e1e --- /dev/null +++ b/mypyc/external/googletest/src/gtest-test-part.cc @@ -0,0 +1,110 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// +// The Google C++ Testing Framework (Google Test) + +#include "gtest/gtest-test-part.h" + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +namespace testing { + +using internal::GetUnitTestImpl; + +// Gets the summary of the failure message by omitting the stack trace +// in it. +std::string TestPartResult::ExtractSummary(const char* message) { + const char* const stack_trace = strstr(message, internal::kStackTraceMarker); + return stack_trace == NULL ? message : + std::string(message, stack_trace); +} + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result) { + return os + << result.file_name() << ":" << result.line_number() << ": " + << (result.type() == TestPartResult::kSuccess ? "Success" : + result.type() == TestPartResult::kFatalFailure ? "Fatal failure" : + "Non-fatal failure") << ":\n" + << result.message() << std::endl; +} + +// Appends a TestPartResult to the array. +void TestPartResultArray::Append(const TestPartResult& result) { + array_.push_back(result); +} + +// Returns the TestPartResult at the given index (0-based). +const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const { + if (index < 0 || index >= size()) { + printf("\nInvalid index (%d) into TestPartResultArray.\n", index); + internal::posix::Abort(); + } + + return array_[index]; +} + +// Returns the number of TestPartResult objects in the array. +int TestPartResultArray::size() const { + return static_cast(array_.size()); +} + +namespace internal { + +HasNewFatalFailureHelper::HasNewFatalFailureHelper() + : has_new_fatal_failure_(false), + original_reporter_(GetUnitTestImpl()-> + GetTestPartResultReporterForCurrentThread()) { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this); +} + +HasNewFatalFailureHelper::~HasNewFatalFailureHelper() { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread( + original_reporter_); +} + +void HasNewFatalFailureHelper::ReportTestPartResult( + const TestPartResult& result) { + if (result.fatally_failed()) + has_new_fatal_failure_ = true; + original_reporter_->ReportTestPartResult(result); +} + +} // namespace internal + +} // namespace testing diff --git a/mypyc/external/googletest/src/gtest-typed-test.cc b/mypyc/external/googletest/src/gtest-typed-test.cc new file mode 100644 index 000000000000..df1eef4754ef --- /dev/null +++ b/mypyc/external/googletest/src/gtest-typed-test.cc @@ -0,0 +1,118 @@ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +#include "gtest/gtest-typed-test.h" +#include "gtest/gtest.h" + +namespace testing { +namespace internal { + +#if GTEST_HAS_TYPED_TEST_P + +// Skips to the first non-space char in str. Returns an empty string if str +// contains only whitespace characters. +static const char* SkipSpaces(const char* str) { + while (IsSpace(*str)) + str++; + return str; +} + +static std::vector SplitIntoTestNames(const char* src) { + std::vector name_vec; + src = SkipSpaces(src); + for (; src != NULL; src = SkipComma(src)) { + name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src))); + } + return name_vec; +} + +// Verifies that registered_tests match the test names in +// registered_tests_; returns registered_tests if successful, or +// aborts the program otherwise. +const char* TypedTestCasePState::VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests) { + typedef RegisteredTestsMap::const_iterator RegisteredTestIter; + registered_ = true; + + std::vector name_vec = SplitIntoTestNames(registered_tests); + + Message errors; + + std::set tests; + for (std::vector::const_iterator name_it = name_vec.begin(); + name_it != name_vec.end(); ++name_it) { + const std::string& name = *name_it; + if (tests.count(name) != 0) { + errors << "Test " << name << " is listed more than once.\n"; + continue; + } + + bool found = false; + for (RegisteredTestIter it = registered_tests_.begin(); + it != registered_tests_.end(); + ++it) { + if (name == it->first) { + found = true; + break; + } + } + + if (found) { + tests.insert(name); + } else { + errors << "No test named " << name + << " can be found in this test case.\n"; + } + } + + for (RegisteredTestIter it = registered_tests_.begin(); + it != registered_tests_.end(); + ++it) { + if (tests.count(it->first) == 0) { + errors << "You forgot to list test " << it->first << ".\n"; + } + } + + const std::string& errors_str = errors.GetString(); + if (errors_str != "") { + fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), + errors_str.c_str()); + fflush(stderr); + posix::Abort(); + } + + return registered_tests; +} + +#endif // GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing diff --git a/mypyc/external/googletest/src/gtest.cc b/mypyc/external/googletest/src/gtest.cc new file mode 100644 index 000000000000..4df3bd6b418a --- /dev/null +++ b/mypyc/external/googletest/src/gtest.cc @@ -0,0 +1,5388 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) + +#include "gtest/gtest.h" +#include "gtest/internal/custom/gtest.h" +#include "gtest/gtest-spi.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include // NOLINT +#include +#include + +#if GTEST_OS_LINUX + +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +# include // NOLINT +# include // NOLINT +# include // NOLINT +// Declares vsnprintf(). This header is not available on Windows. +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include + +#elif GTEST_OS_SYMBIAN +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +#elif GTEST_OS_ZOS +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +// On z/OS we additionally need strings.h for strcasecmp. +# include // NOLINT + +#elif GTEST_OS_WINDOWS_MOBILE // We are on Windows CE. + +# include // NOLINT +# undef min + +#elif GTEST_OS_WINDOWS // We are on Windows proper. + +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT + +# if GTEST_OS_WINDOWS_MINGW +// MinGW has gettimeofday() but not _ftime64(). +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +// TODO(kenton@google.com): There are other ways to get the time on +// Windows, like GetTickCount() or GetSystemTimeAsFileTime(). MinGW +// supports these. consider using them instead. +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT +# endif // GTEST_OS_WINDOWS_MINGW + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT +# undef min + +#else + +// Assume other platforms have gettimeofday(). +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT +# include // NOLINT + +#endif // GTEST_OS_LINUX + +#if GTEST_HAS_EXCEPTIONS +# include +#endif + +#if GTEST_CAN_STREAM_RESULTS_ +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT +#endif + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick is to +// prevent a user from accidentally including gtest-internal-inl.h in +// his code. +#define GTEST_IMPLEMENTATION_ 1 +#include "src/gtest-internal-inl.h" +#undef GTEST_IMPLEMENTATION_ + +#if GTEST_OS_WINDOWS +# define vsnprintf _vsnprintf +#endif // GTEST_OS_WINDOWS + +namespace testing { + +using internal::CountIf; +using internal::ForEach; +using internal::GetElementOr; +using internal::Shuffle; + +// Constants. + +// A test whose test case name or test name matches this filter is +// disabled and not run. +static const char kDisableTestFilter[] = "DISABLED_*:*/DISABLED_*"; + +// A test case whose name matches this filter is considered a death +// test case and will be run before test cases whose name doesn't +// match this filter. +static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*"; + +// A test filter that matches everything. +static const char kUniversalFilter[] = "*"; + +// The default output file for XML output. +static const char kDefaultOutputFile[] = "test_detail.xml"; + +// The environment variable name for the test shard index. +static const char kTestShardIndex[] = "GTEST_SHARD_INDEX"; +// The environment variable name for the total number of test shards. +static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS"; +// The environment variable name for the test shard status file. +static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE"; + +namespace internal { + +// The text used in failure messages to indicate the start of the +// stack trace. +const char kStackTraceMarker[] = "\nStack trace:\n"; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +bool g_help_flag = false; + +} // namespace internal + +static const char* GetDefaultFilter() { +#ifdef GTEST_TEST_FILTER_ENV_VAR_ + const char* const testbridge_test_only = getenv(GTEST_TEST_FILTER_ENV_VAR_); + if (testbridge_test_only != NULL) { + return testbridge_test_only; + } +#endif // GTEST_TEST_FILTER_ENV_VAR_ + return kUniversalFilter; +} + +GTEST_DEFINE_bool_( + also_run_disabled_tests, + internal::BoolFromGTestEnv("also_run_disabled_tests", false), + "Run disabled tests too, in addition to the tests normally being run."); + +GTEST_DEFINE_bool_( + break_on_failure, + internal::BoolFromGTestEnv("break_on_failure", false), + "True iff a failed assertion should be a debugger break-point."); + +GTEST_DEFINE_bool_( + catch_exceptions, + internal::BoolFromGTestEnv("catch_exceptions", true), + "True iff " GTEST_NAME_ + " should catch exceptions and treat them as test failures."); + +GTEST_DEFINE_string_( + color, + internal::StringFromGTestEnv("color", "auto"), + "Whether to use colors in the output. Valid values: yes, no, " + "and auto. 'auto' means to use colors if the output is " + "being sent to a terminal and the TERM environment variable " + "is set to a terminal type that supports colors."); + +GTEST_DEFINE_string_( + filter, + internal::StringFromGTestEnv("filter", GetDefaultFilter()), + "A colon-separated list of glob (not regex) patterns " + "for filtering the tests to run, optionally followed by a " + "'-' and a : separated list of negative patterns (tests to " + "exclude). A test is run if it matches one of the positive " + "patterns and does not match any of the negative patterns."); + +GTEST_DEFINE_bool_(list_tests, false, + "List all tests without running them."); + +GTEST_DEFINE_string_( + output, + internal::StringFromGTestEnv("output", ""), + "A format (currently must be \"xml\"), optionally followed " + "by a colon and an output file name or directory. A directory " + "is indicated by a trailing pathname separator. " + "Examples: \"xml:filename.xml\", \"xml::directoryname/\". " + "If a directory is specified, output files will be created " + "within that directory, with file-names based on the test " + "executable's name and, if necessary, made unique by adding " + "digits."); + +GTEST_DEFINE_bool_( + print_time, + internal::BoolFromGTestEnv("print_time", true), + "True iff " GTEST_NAME_ + " should display elapsed time in text output."); + +GTEST_DEFINE_int32_( + random_seed, + internal::Int32FromGTestEnv("random_seed", 0), + "Random number seed to use when shuffling test orders. Must be in range " + "[1, 99999], or 0 to use a seed based on the current time."); + +GTEST_DEFINE_int32_( + repeat, + internal::Int32FromGTestEnv("repeat", 1), + "How many times to repeat each test. Specify a negative number " + "for repeating forever. Useful for shaking out flaky tests."); + +GTEST_DEFINE_bool_( + show_internal_stack_frames, false, + "True iff " GTEST_NAME_ " should include internal stack frames when " + "printing test failure stack traces."); + +GTEST_DEFINE_bool_( + shuffle, + internal::BoolFromGTestEnv("shuffle", false), + "True iff " GTEST_NAME_ + " should randomize tests' order on every run."); + +GTEST_DEFINE_int32_( + stack_trace_depth, + internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth), + "The maximum number of stack frames to print when an " + "assertion fails. The valid range is 0 through 100, inclusive."); + +GTEST_DEFINE_string_( + stream_result_to, + internal::StringFromGTestEnv("stream_result_to", ""), + "This flag specifies the host name and the port number on which to stream " + "test results. Example: \"localhost:555\". The flag is effective only on " + "Linux."); + +GTEST_DEFINE_bool_( + throw_on_failure, + internal::BoolFromGTestEnv("throw_on_failure", false), + "When this flag is specified, a failed assertion will throw an exception " + "if exceptions are enabled or exit the program with a non-zero code " + "otherwise."); + +#if GTEST_USE_OWN_FLAGFILE_FLAG_ +GTEST_DEFINE_string_( + flagfile, + internal::StringFromGTestEnv("flagfile", ""), + "This flag specifies the flagfile to read command-line flags from."); +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + +namespace internal { + +// Generates a random number from [0, range), using a Linear +// Congruential Generator (LCG). Crashes if 'range' is 0 or greater +// than kMaxRange. +UInt32 Random::Generate(UInt32 range) { + // These constants are the same as are used in glibc's rand(3). + state_ = (1103515245U*state_ + 12345U) % kMaxRange; + + GTEST_CHECK_(range > 0) + << "Cannot generate a number in the range [0, 0)."; + GTEST_CHECK_(range <= kMaxRange) + << "Generation of a number in [0, " << range << ") was requested, " + << "but this can only generate numbers in [0, " << kMaxRange << ")."; + + // Converting via modulus introduces a bit of downward bias, but + // it's simple, and a linear congruential generator isn't too good + // to begin with. + return state_ % range; +} + +// GTestIsInitialized() returns true iff the user has initialized +// Google Test. Useful for catching the user mistake of not initializing +// Google Test before calling RUN_ALL_TESTS(). +static bool GTestIsInitialized() { return GetArgvs().size() > 0; } + +// Iterates over a vector of TestCases, keeping a running sum of the +// results of calling a given int-returning method on each. +// Returns the sum. +static int SumOverTestCaseList(const std::vector& case_list, + int (TestCase::*method)() const) { + int sum = 0; + for (size_t i = 0; i < case_list.size(); i++) { + sum += (case_list[i]->*method)(); + } + return sum; +} + +// Returns true iff the test case passed. +static bool TestCasePassed(const TestCase* test_case) { + return test_case->should_run() && test_case->Passed(); +} + +// Returns true iff the test case failed. +static bool TestCaseFailed(const TestCase* test_case) { + return test_case->should_run() && test_case->Failed(); +} + +// Returns true iff test_case contains at least one test that should +// run. +static bool ShouldRunTestCase(const TestCase* test_case) { + return test_case->should_run(); +} + +// AssertHelper constructor. +AssertHelper::AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message) + : data_(new AssertHelperData(type, file, line, message)) { +} + +AssertHelper::~AssertHelper() { + delete data_; +} + +// Message assignment, for assertion streaming support. +void AssertHelper::operator=(const Message& message) const { + UnitTest::GetInstance()-> + AddTestPartResult(data_->type, data_->file, data_->line, + AppendUserMessage(data_->message, message), + UnitTest::GetInstance()->impl() + ->CurrentOsStackTraceExceptTop(1) + // Skips the stack frame for this function itself. + ); // NOLINT +} + +// Mutex for linked pointers. +GTEST_API_ GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// A copy of all command line arguments. Set by InitGoogleTest(). +::std::vector g_argvs; + +const ::std::vector& GetArgvs() { +#if defined(GTEST_CUSTOM_GET_ARGVS_) + return GTEST_CUSTOM_GET_ARGVS_(); +#else // defined(GTEST_CUSTOM_GET_ARGVS_) + return g_argvs; +#endif // defined(GTEST_CUSTOM_GET_ARGVS_) +} + +// Returns the current application's name, removing directory path if that +// is present. +FilePath GetCurrentExecutableName() { + FilePath result; + +#if GTEST_OS_WINDOWS + result.Set(FilePath(GetArgvs()[0]).RemoveExtension("exe")); +#else + result.Set(FilePath(GetArgvs()[0])); +#endif // GTEST_OS_WINDOWS + + return result.RemoveDirectoryName(); +} + +// Functions for processing the gtest_output flag. + +// Returns the output format, or "" for normal printed output. +std::string UnitTestOptions::GetOutputFormat() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + if (gtest_output_flag == NULL) return std::string(""); + + const char* const colon = strchr(gtest_output_flag, ':'); + return (colon == NULL) ? + std::string(gtest_output_flag) : + std::string(gtest_output_flag, colon - gtest_output_flag); +} + +// Returns the name of the requested output file, or the default if none +// was explicitly specified. +std::string UnitTestOptions::GetAbsolutePathToOutputFile() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + if (gtest_output_flag == NULL) + return ""; + + const char* const colon = strchr(gtest_output_flag, ':'); + if (colon == NULL) + return internal::FilePath::ConcatPaths( + internal::FilePath( + UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(kDefaultOutputFile)).string(); + + internal::FilePath output_name(colon + 1); + if (!output_name.IsAbsolutePath()) + // TODO(wan@google.com): on Windows \some\path is not an absolute + // path (as its meaning depends on the current drive), yet the + // following logic for turning it into an absolute path is wrong. + // Fix it. + output_name = internal::FilePath::ConcatPaths( + internal::FilePath(UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(colon + 1)); + + if (!output_name.IsDirectory()) + return output_name.string(); + + internal::FilePath result(internal::FilePath::GenerateUniqueFileName( + output_name, internal::GetCurrentExecutableName(), + GetOutputFormat().c_str())); + return result.string(); +} + +// Returns true iff the wildcard pattern matches the string. The +// first ':' or '\0' character in pattern marks the end of it. +// +// This recursive algorithm isn't very efficient, but is clear and +// works well enough for matching test names, which are short. +bool UnitTestOptions::PatternMatchesString(const char *pattern, + const char *str) { + switch (*pattern) { + case '\0': + case ':': // Either ':' or '\0' marks the end of the pattern. + return *str == '\0'; + case '?': // Matches any single character. + return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); + case '*': // Matches any string (possibly empty) of characters. + return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || + PatternMatchesString(pattern + 1, str); + default: // Non-special character. Matches itself. + return *pattern == *str && + PatternMatchesString(pattern + 1, str + 1); + } +} + +bool UnitTestOptions::MatchesFilter( + const std::string& name, const char* filter) { + const char *cur_pattern = filter; + for (;;) { + if (PatternMatchesString(cur_pattern, name.c_str())) { + return true; + } + + // Finds the next pattern in the filter. + cur_pattern = strchr(cur_pattern, ':'); + + // Returns if no more pattern can be found. + if (cur_pattern == NULL) { + return false; + } + + // Skips the pattern separater (the ':' character). + cur_pattern++; + } +} + +// Returns true iff the user-specified filter matches the test case +// name and the test name. +bool UnitTestOptions::FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name) { + const std::string& full_name = test_case_name + "." + test_name.c_str(); + + // Split --gtest_filter at '-', if there is one, to separate into + // positive filter and negative filter portions + const char* const p = GTEST_FLAG(filter).c_str(); + const char* const dash = strchr(p, '-'); + std::string positive; + std::string negative; + if (dash == NULL) { + positive = GTEST_FLAG(filter).c_str(); // Whole string is a positive filter + negative = ""; + } else { + positive = std::string(p, dash); // Everything up to the dash + negative = std::string(dash + 1); // Everything after the dash + if (positive.empty()) { + // Treat '-test1' as the same as '*-test1' + positive = kUniversalFilter; + } + } + + // A filter is a colon-separated list of patterns. It matches a + // test if any pattern in it matches the test. + return (MatchesFilter(full_name, positive.c_str()) && + !MatchesFilter(full_name, negative.c_str())); +} + +#if GTEST_HAS_SEH +// Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the +// given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. +// This function is useful as an __except condition. +int UnitTestOptions::GTestShouldProcessSEH(DWORD exception_code) { + // Google Test should handle a SEH exception if: + // 1. the user wants it to, AND + // 2. this is not a breakpoint exception, AND + // 3. this is not a C++ exception (VC++ implements them via SEH, + // apparently). + // + // SEH exception code for C++ exceptions. + // (see http://support.microsoft.com/kb/185294 for more information). + const DWORD kCxxExceptionCode = 0xe06d7363; + + bool should_handle = true; + + if (!GTEST_FLAG(catch_exceptions)) + should_handle = false; + else if (exception_code == EXCEPTION_BREAKPOINT) + should_handle = false; + else if (exception_code == kCxxExceptionCode) + should_handle = false; + + return should_handle ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH; +} +#endif // GTEST_HAS_SEH + +} // namespace internal + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. Intercepts only failures from the current thread. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + TestPartResultArray* result) + : intercept_mode_(INTERCEPT_ONLY_CURRENT_THREAD), + result_(result) { + Init(); +} + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + InterceptMode intercept_mode, TestPartResultArray* result) + : intercept_mode_(intercept_mode), + result_(result) { + Init(); +} + +void ScopedFakeTestPartResultReporter::Init() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + old_reporter_ = impl->GetGlobalTestPartResultReporter(); + impl->SetGlobalTestPartResultReporter(this); + } else { + old_reporter_ = impl->GetTestPartResultReporterForCurrentThread(); + impl->SetTestPartResultReporterForCurrentThread(this); + } +} + +// The d'tor restores the test part result reporter used by Google Test +// before. +ScopedFakeTestPartResultReporter::~ScopedFakeTestPartResultReporter() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + impl->SetGlobalTestPartResultReporter(old_reporter_); + } else { + impl->SetTestPartResultReporterForCurrentThread(old_reporter_); + } +} + +// Increments the test part result count and remembers the result. +// This method is from the TestPartResultReporterInterface interface. +void ScopedFakeTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + result_->Append(result); +} + +namespace internal { + +// Returns the type ID of ::testing::Test. We should always call this +// instead of GetTypeId< ::testing::Test>() to get the type ID of +// testing::Test. This is to work around a suspected linker bug when +// using Google Test as a framework on Mac OS X. The bug causes +// GetTypeId< ::testing::Test>() to return different values depending +// on whether the call is from the Google Test framework itself or +// from user test code. GetTestTypeId() is guaranteed to always +// return the same value, as it always calls GetTypeId<>() from the +// gtest.cc, which is within the Google Test framework. +TypeId GetTestTypeId() { + return GetTypeId(); +} + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +extern const TypeId kTestTypeIdInGoogleTest = GetTestTypeId(); + +// This predicate-formatter checks that 'results' contains a test part +// failure of the given type and that the failure message contains the +// given substring. +AssertionResult HasOneFailure(const char* /* results_expr */, + const char* /* type_expr */, + const char* /* substr_expr */, + const TestPartResultArray& results, + TestPartResult::Type type, + const string& substr) { + const std::string expected(type == TestPartResult::kFatalFailure ? + "1 fatal failure" : + "1 non-fatal failure"); + Message msg; + if (results.size() != 1) { + msg << "Expected: " << expected << "\n" + << " Actual: " << results.size() << " failures"; + for (int i = 0; i < results.size(); i++) { + msg << "\n" << results.GetTestPartResult(i); + } + return AssertionFailure() << msg; + } + + const TestPartResult& r = results.GetTestPartResult(0); + if (r.type() != type) { + return AssertionFailure() << "Expected: " << expected << "\n" + << " Actual:\n" + << r; + } + + if (strstr(r.message(), substr.c_str()) == NULL) { + return AssertionFailure() << "Expected: " << expected << " containing \"" + << substr << "\"\n" + << " Actual:\n" + << r; + } + + return AssertionSuccess(); +} + +// The constructor of SingleFailureChecker remembers where to look up +// test part results, what type of failure we expect, and what +// substring the failure message should contain. +SingleFailureChecker:: SingleFailureChecker( + const TestPartResultArray* results, + TestPartResult::Type type, + const string& substr) + : results_(results), + type_(type), + substr_(substr) {} + +// The destructor of SingleFailureChecker verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +SingleFailureChecker::~SingleFailureChecker() { + EXPECT_PRED_FORMAT3(HasOneFailure, *results_, type_, substr_); +} + +DefaultGlobalTestPartResultReporter::DefaultGlobalTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultGlobalTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->current_test_result()->AddTestPartResult(result); + unit_test_->listeners()->repeater()->OnTestPartResult(result); +} + +DefaultPerThreadTestPartResultReporter::DefaultPerThreadTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultPerThreadTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->GetGlobalTestPartResultReporter()->ReportTestPartResult(result); +} + +// Returns the global test part result reporter. +TestPartResultReporterInterface* +UnitTestImpl::GetGlobalTestPartResultReporter() { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + return global_test_part_result_repoter_; +} + +// Sets the global test part result reporter. +void UnitTestImpl::SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter) { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + global_test_part_result_repoter_ = reporter; +} + +// Returns the test part result reporter for the current thread. +TestPartResultReporterInterface* +UnitTestImpl::GetTestPartResultReporterForCurrentThread() { + return per_thread_test_part_result_reporter_.get(); +} + +// Sets the test part result reporter for the current thread. +void UnitTestImpl::SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter) { + per_thread_test_part_result_reporter_.set(reporter); +} + +// Gets the number of successful test cases. +int UnitTestImpl::successful_test_case_count() const { + return CountIf(test_cases_, TestCasePassed); +} + +// Gets the number of failed test cases. +int UnitTestImpl::failed_test_case_count() const { + return CountIf(test_cases_, TestCaseFailed); +} + +// Gets the number of all test cases. +int UnitTestImpl::total_test_case_count() const { + return static_cast(test_cases_.size()); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTestImpl::test_case_to_run_count() const { + return CountIf(test_cases_, ShouldRunTestCase); +} + +// Gets the number of successful tests. +int UnitTestImpl::successful_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::successful_test_count); +} + +// Gets the number of failed tests. +int UnitTestImpl::failed_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count); +} + +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTestImpl::reportable_disabled_test_count() const { + return SumOverTestCaseList(test_cases_, + &TestCase::reportable_disabled_test_count); +} + +// Gets the number of disabled tests. +int UnitTestImpl::disabled_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count); +} + +// Gets the number of tests to be printed in the XML report. +int UnitTestImpl::reportable_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count); +} + +// Gets the number of all tests. +int UnitTestImpl::total_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::total_test_count); +} + +// Gets the number of tests that should run. +int UnitTestImpl::test_to_run_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::test_to_run_count); +} + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// CurrentOsStackTraceExceptTop(1), Foo() will be included in the +// trace but Bar() and CurrentOsStackTraceExceptTop() won't. +std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) { + return os_stack_trace_getter()->CurrentStackTrace( + static_cast(GTEST_FLAG(stack_trace_depth)), + skip_count + 1 + // Skips the user-specified number of frames plus this function + // itself. + ); // NOLINT +} + +// Returns the current time in milliseconds. +TimeInMillis GetTimeInMillis() { +#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__) + // Difference between 1970-01-01 and 1601-01-01 in milliseconds. + // http://analogous.blogspot.com/2005/04/epoch.html + const TimeInMillis kJavaEpochToWinFileTimeDelta = + static_cast(116444736UL) * 100000UL; + const DWORD kTenthMicrosInMilliSecond = 10000; + + SYSTEMTIME now_systime; + FILETIME now_filetime; + ULARGE_INTEGER now_int64; + // TODO(kenton@google.com): Shouldn't this just use + // GetSystemTimeAsFileTime()? + GetSystemTime(&now_systime); + if (SystemTimeToFileTime(&now_systime, &now_filetime)) { + now_int64.LowPart = now_filetime.dwLowDateTime; + now_int64.HighPart = now_filetime.dwHighDateTime; + now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) - + kJavaEpochToWinFileTimeDelta; + return now_int64.QuadPart; + } + return 0; +#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ + __timeb64 now; + + // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 + // (deprecated function) there. + // TODO(kenton@google.com): Use GetTickCount()? Or use + // SystemTimeToFileTime() + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) + _ftime64(&now); + GTEST_DISABLE_MSC_WARNINGS_POP_() + + return static_cast(now.time) * 1000 + now.millitm; +#elif GTEST_HAS_GETTIMEOFDAY_ + struct timeval now; + gettimeofday(&now, NULL); + return static_cast(now.tv_sec) * 1000 + now.tv_usec / 1000; +#else +# error "Don't know how to get the current time on your system." +#endif +} + +// Utilities + +// class String. + +#if GTEST_OS_WINDOWS_MOBILE +// Creates a UTF-16 wide string from the given ANSI string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the wide string, or NULL if the +// input is NULL. +LPCWSTR String::AnsiToUtf16(const char* ansi) { + if (!ansi) return NULL; + const int length = strlen(ansi); + const int unicode_length = + MultiByteToWideChar(CP_ACP, 0, ansi, length, + NULL, 0); + WCHAR* unicode = new WCHAR[unicode_length + 1]; + MultiByteToWideChar(CP_ACP, 0, ansi, length, + unicode, unicode_length); + unicode[unicode_length] = 0; + return unicode; +} + +// Creates an ANSI string from the given wide string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the ANSI string, or NULL if the +// input is NULL. +const char* String::Utf16ToAnsi(LPCWSTR utf16_str) { + if (!utf16_str) return NULL; + const int ansi_length = + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + NULL, 0, NULL, NULL); + char* ansi = new char[ansi_length + 1]; + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + ansi, ansi_length, NULL, NULL); + ansi[ansi_length] = 0; + return ansi; +} + +#endif // GTEST_OS_WINDOWS_MOBILE + +// Compares two C strings. Returns true iff they have the same content. +// +// Unlike strcmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CStringEquals(const char * lhs, const char * rhs) { + if ( lhs == NULL ) return rhs == NULL; + + if ( rhs == NULL ) return false; + + return strcmp(lhs, rhs) == 0; +} + +#if GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +// Converts an array of wide chars to a narrow string using the UTF-8 +// encoding, and streams the result to the given Message object. +static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length, + Message* msg) { + for (size_t i = 0; i != length; ) { // NOLINT + if (wstr[i] != L'\0') { + *msg << WideStringToUtf8(wstr + i, static_cast(length - i)); + while (i != length && wstr[i] != L'\0') + i++; + } else { + *msg << '\0'; + i++; + } + } +} + +#endif // GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +void SplitString(const ::std::string& str, char delimiter, + ::std::vector< ::std::string>* dest) { + ::std::vector< ::std::string> parsed; + ::std::string::size_type pos = 0; + while (::testing::internal::AlwaysTrue()) { + const ::std::string::size_type colon = str.find(delimiter, pos); + if (colon == ::std::string::npos) { + parsed.push_back(str.substr(pos)); + break; + } else { + parsed.push_back(str.substr(pos, colon - pos)); + pos = colon + 1; + } + } + dest->swap(parsed); +} + +} // namespace internal + +// Constructs an empty Message. +// We allocate the stringstream separately because otherwise each use of +// ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's +// stack frame leading to huge stack frames in some cases; gcc does not reuse +// the stack space. +Message::Message() : ss_(new ::std::stringstream) { + // By default, we want there to be enough precision when printing + // a double to a Message. + *ss_ << std::setprecision(std::numeric_limits::digits10 + 2); +} + +// These two overloads allow streaming a wide C string to a Message +// using the UTF-8 encoding. +Message& Message::operator <<(const wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} +Message& Message::operator <<(wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} + +#if GTEST_HAS_STD_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::std::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +// Gets the text streamed to this object so far as an std::string. +// Each '\0' character in the buffer is replaced with "\\0". +std::string Message::GetString() const { + return internal::StringStreamToString(ss_.get()); +} + +// AssertionResult constructors. +// Used in EXPECT_TRUE/FALSE(assertion_result). +AssertionResult::AssertionResult(const AssertionResult& other) + : success_(other.success_), + message_(other.message_.get() != NULL ? + new ::std::string(*other.message_) : + static_cast< ::std::string*>(NULL)) { +} + +// Swaps two AssertionResults. +void AssertionResult::swap(AssertionResult& other) { + using std::swap; + swap(success_, other.success_); + swap(message_, other.message_); +} + +// Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. +AssertionResult AssertionResult::operator!() const { + AssertionResult negation(!success_); + if (message_.get() != NULL) + negation << *message_; + return negation; +} + +// Makes a successful assertion result. +AssertionResult AssertionSuccess() { + return AssertionResult(true); +} + +// Makes a failed assertion result. +AssertionResult AssertionFailure() { + return AssertionResult(false); +} + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << message. +AssertionResult AssertionFailure(const Message& message) { + return AssertionFailure() << message; +} + +namespace internal { + +namespace edit_distance { +std::vector CalculateOptimalEdits(const std::vector& left, + const std::vector& right) { + std::vector > costs( + left.size() + 1, std::vector(right.size() + 1)); + std::vector > best_move( + left.size() + 1, std::vector(right.size() + 1)); + + // Populate for empty right. + for (size_t l_i = 0; l_i < costs.size(); ++l_i) { + costs[l_i][0] = static_cast(l_i); + best_move[l_i][0] = kRemove; + } + // Populate for empty left. + for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) { + costs[0][r_i] = static_cast(r_i); + best_move[0][r_i] = kAdd; + } + + for (size_t l_i = 0; l_i < left.size(); ++l_i) { + for (size_t r_i = 0; r_i < right.size(); ++r_i) { + if (left[l_i] == right[r_i]) { + // Found a match. Consume it. + costs[l_i + 1][r_i + 1] = costs[l_i][r_i]; + best_move[l_i + 1][r_i + 1] = kMatch; + continue; + } + + const double add = costs[l_i + 1][r_i]; + const double remove = costs[l_i][r_i + 1]; + const double replace = costs[l_i][r_i]; + if (add < remove && add < replace) { + costs[l_i + 1][r_i + 1] = add + 1; + best_move[l_i + 1][r_i + 1] = kAdd; + } else if (remove < add && remove < replace) { + costs[l_i + 1][r_i + 1] = remove + 1; + best_move[l_i + 1][r_i + 1] = kRemove; + } else { + // We make replace a little more expensive than add/remove to lower + // their priority. + costs[l_i + 1][r_i + 1] = replace + 1.00001; + best_move[l_i + 1][r_i + 1] = kReplace; + } + } + } + + // Reconstruct the best path. We do it in reverse order. + std::vector best_path; + for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) { + EditType move = best_move[l_i][r_i]; + best_path.push_back(move); + l_i -= move != kAdd; + r_i -= move != kRemove; + } + std::reverse(best_path.begin(), best_path.end()); + return best_path; +} + +namespace { + +// Helper class to convert string into ids with deduplication. +class InternalStrings { + public: + size_t GetId(const std::string& str) { + IdMap::iterator it = ids_.find(str); + if (it != ids_.end()) return it->second; + size_t id = ids_.size(); + return ids_[str] = id; + } + + private: + typedef std::map IdMap; + IdMap ids_; +}; + +} // namespace + +std::vector CalculateOptimalEdits( + const std::vector& left, + const std::vector& right) { + std::vector left_ids, right_ids; + { + InternalStrings intern_table; + for (size_t i = 0; i < left.size(); ++i) { + left_ids.push_back(intern_table.GetId(left[i])); + } + for (size_t i = 0; i < right.size(); ++i) { + right_ids.push_back(intern_table.GetId(right[i])); + } + } + return CalculateOptimalEdits(left_ids, right_ids); +} + +namespace { + +// Helper class that holds the state for one hunk and prints it out to the +// stream. +// It reorders adds/removes when possible to group all removes before all +// adds. It also adds the hunk header before printint into the stream. +class Hunk { + public: + Hunk(size_t left_start, size_t right_start) + : left_start_(left_start), + right_start_(right_start), + adds_(), + removes_(), + common_() {} + + void PushLine(char edit, const char* line) { + switch (edit) { + case ' ': + ++common_; + FlushEdits(); + hunk_.push_back(std::make_pair(' ', line)); + break; + case '-': + ++removes_; + hunk_removes_.push_back(std::make_pair('-', line)); + break; + case '+': + ++adds_; + hunk_adds_.push_back(std::make_pair('+', line)); + break; + } + } + + void PrintTo(std::ostream* os) { + PrintHeader(os); + FlushEdits(); + for (std::list >::const_iterator it = + hunk_.begin(); + it != hunk_.end(); ++it) { + *os << it->first << it->second << "\n"; + } + } + + bool has_edits() const { return adds_ || removes_; } + + private: + void FlushEdits() { + hunk_.splice(hunk_.end(), hunk_removes_); + hunk_.splice(hunk_.end(), hunk_adds_); + } + + // Print a unified diff header for one hunk. + // The format is + // "@@ -, +, @@" + // where the left/right parts are ommitted if unnecessary. + void PrintHeader(std::ostream* ss) const { + *ss << "@@ "; + if (removes_) { + *ss << "-" << left_start_ << "," << (removes_ + common_); + } + if (removes_ && adds_) { + *ss << " "; + } + if (adds_) { + *ss << "+" << right_start_ << "," << (adds_ + common_); + } + *ss << " @@\n"; + } + + size_t left_start_, right_start_; + size_t adds_, removes_, common_; + std::list > hunk_, hunk_adds_, hunk_removes_; +}; + +} // namespace + +// Create a list of diff hunks in Unified diff format. +// Each hunk has a header generated by PrintHeader above plus a body with +// lines prefixed with ' ' for no change, '-' for deletion and '+' for +// addition. +// 'context' represents the desired unchanged prefix/suffix around the diff. +// If two hunks are close enough that their contexts overlap, then they are +// joined into one hunk. +std::string CreateUnifiedDiff(const std::vector& left, + const std::vector& right, + size_t context) { + const std::vector edits = CalculateOptimalEdits(left, right); + + size_t l_i = 0, r_i = 0, edit_i = 0; + std::stringstream ss; + while (edit_i < edits.size()) { + // Find first edit. + while (edit_i < edits.size() && edits[edit_i] == kMatch) { + ++l_i; + ++r_i; + ++edit_i; + } + + // Find the first line to include in the hunk. + const size_t prefix_context = std::min(l_i, context); + Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1); + for (size_t i = prefix_context; i > 0; --i) { + hunk.PushLine(' ', left[l_i - i].c_str()); + } + + // Iterate the edits until we found enough suffix for the hunk or the input + // is over. + size_t n_suffix = 0; + for (; edit_i < edits.size(); ++edit_i) { + if (n_suffix >= context) { + // Continue only if the next hunk is very close. + std::vector::const_iterator it = edits.begin() + edit_i; + while (it != edits.end() && *it == kMatch) ++it; + if (it == edits.end() || (it - edits.begin()) - edit_i >= context) { + // There is no next edit or it is too far away. + break; + } + } + + EditType edit = edits[edit_i]; + // Reset count when a non match is found. + n_suffix = edit == kMatch ? n_suffix + 1 : 0; + + if (edit == kMatch || edit == kRemove || edit == kReplace) { + hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str()); + } + if (edit == kAdd || edit == kReplace) { + hunk.PushLine('+', right[r_i].c_str()); + } + + // Advance indices, depending on edit type. + l_i += edit != kAdd; + r_i += edit != kRemove; + } + + if (!hunk.has_edits()) { + // We are done. We don't want this hunk. + break; + } + + hunk.PrintTo(&ss); + } + return ss.str(); +} + +} // namespace edit_distance + +namespace { + +// The string representation of the values received in EqFailure() are already +// escaped. Split them on escaped '\n' boundaries. Leave all other escaped +// characters the same. +std::vector SplitEscapedString(const std::string& str) { + std::vector lines; + size_t start = 0, end = str.size(); + if (end > 2 && str[0] == '"' && str[end - 1] == '"') { + ++start; + --end; + } + bool escaped = false; + for (size_t i = start; i + 1 < end; ++i) { + if (escaped) { + escaped = false; + if (str[i] == 'n') { + lines.push_back(str.substr(start, i - start - 1)); + start = i + 1; + } + } else { + escaped = str[i] == '\\'; + } + } + lines.push_back(str.substr(start, end - start)); + return lines; +} + +} // namespace + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// lhs_expression: "foo" +// rhs_expression: "bar" +// lhs_value: "5" +// rhs_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string "Ignoring case" will +// be inserted into the message. +AssertionResult EqFailure(const char* lhs_expression, + const char* rhs_expression, + const std::string& lhs_value, + const std::string& rhs_value, + bool ignoring_case) { + Message msg; + msg << " Expected: " << lhs_expression; + if (lhs_value != lhs_expression) { + msg << "\n Which is: " << lhs_value; + } + msg << "\nTo be equal to: " << rhs_expression; + if (rhs_value != rhs_expression) { + msg << "\n Which is: " << rhs_value; + } + + if (ignoring_case) { + msg << "\nIgnoring case"; + } + + if (!lhs_value.empty() && !rhs_value.empty()) { + const std::vector lhs_lines = + SplitEscapedString(lhs_value); + const std::vector rhs_lines = + SplitEscapedString(rhs_value); + if (lhs_lines.size() > 1 || rhs_lines.size() > 1) { + msg << "\nWith diff:\n" + << edit_distance::CreateUnifiedDiff(lhs_lines, rhs_lines); + } + } + + return AssertionFailure() << msg; +} + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +std::string GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value) { + const char* actual_message = assertion_result.message(); + Message msg; + msg << "Value of: " << expression_text + << "\n Actual: " << actual_predicate_value; + if (actual_message[0] != '\0') + msg << " (" << actual_message << ")"; + msg << "\nExpected: " << expected_predicate_value; + return msg.GetString(); +} + +// Helper function for implementing ASSERT_NEAR. +AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error) { + const double diff = fabs(val1 - val2); + if (diff <= abs_error) return AssertionSuccess(); + + // TODO(wan): do not print the value of an expression if it's + // already a literal. + return AssertionFailure() + << "The difference between " << expr1 << " and " << expr2 + << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n" + << expr1 << " evaluates to " << val1 << ",\n" + << expr2 << " evaluates to " << val2 << ", and\n" + << abs_error_expr << " evaluates to " << abs_error << "."; +} + + +// Helper template for implementing FloatLE() and DoubleLE(). +template +AssertionResult FloatingPointLE(const char* expr1, + const char* expr2, + RawType val1, + RawType val2) { + // Returns success if val1 is less than val2, + if (val1 < val2) { + return AssertionSuccess(); + } + + // or if val1 is almost equal to val2. + const FloatingPoint lhs(val1), rhs(val2); + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + // Note that the above two checks will both fail if either val1 or + // val2 is NaN, as the IEEE floating-point standard requires that + // any predicate involving a NaN must return false. + + ::std::stringstream val1_ss; + val1_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val1; + + ::std::stringstream val2_ss; + val2_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val2; + + return AssertionFailure() + << "Expected: (" << expr1 << ") <= (" << expr2 << ")\n" + << " Actual: " << StringStreamToString(&val1_ss) << " vs " + << StringStreamToString(&val2_ss); +} + +} // namespace internal + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +namespace internal { + +// The helper function for {ASSERT|EXPECT}_EQ with int or enum +// arguments. +AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs) { + if (lhs == rhs) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + FormatForComparisonFailureMessage(lhs, rhs), + FormatForComparisonFailureMessage(rhs, lhs), + false); +} + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_?? with integer or enum arguments. It is here +// just to avoid copy-and-paste of similar code. +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + BiggestInt val1, BiggestInt val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return AssertionFailure() \ + << "Expected: (" << expr1 << ") " #op " (" << expr2\ + << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ + << " vs " << FormatForComparisonFailureMessage(val2, val1);\ + }\ +} + +// Implements the helper function for {ASSERT|EXPECT}_NE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(NE, !=) +// Implements the helper function for {ASSERT|EXPECT}_LE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LE, <=) +// Implements the helper function for {ASSERT|EXPECT}_LT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LT, < ) +// Implements the helper function for {ASSERT|EXPECT}_GE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GE, >=) +// Implements the helper function for {ASSERT|EXPECT}_GT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GT, > ) + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +AssertionResult CmpHelperSTREQ(const char* lhs_expression, + const char* rhs_expression, + const char* lhs, + const char* rhs) { + if (String::CStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + false); +} + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +AssertionResult CmpHelperSTRCASEEQ(const char* lhs_expression, + const char* rhs_expression, + const char* lhs, + const char* rhs) { + if (String::CaseInsensitiveCStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + true); +} + +// The helper function for {ASSERT|EXPECT}_STRNE. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CaseInsensitiveCStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() + << "Expected: (" << s1_expression << ") != (" + << s2_expression << ") (ignoring case), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +} // namespace internal + +namespace { + +// Helper functions for implementing IsSubString() and IsNotSubstring(). + +// This group of overloaded functions return true iff needle is a +// substring of haystack. NULL is considered a substring of itself +// only. + +bool IsSubstringPred(const char* needle, const char* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return strstr(haystack, needle) != NULL; +} + +bool IsSubstringPred(const wchar_t* needle, const wchar_t* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return wcsstr(haystack, needle) != NULL; +} + +// StringType here can be either ::std::string or ::std::wstring. +template +bool IsSubstringPred(const StringType& needle, + const StringType& haystack) { + return haystack.find(needle) != StringType::npos; +} + +// This function implements either IsSubstring() or IsNotSubstring(), +// depending on the value of the expected_to_be_substring parameter. +// StringType here can be const char*, const wchar_t*, ::std::string, +// or ::std::wstring. +template +AssertionResult IsSubstringImpl( + bool expected_to_be_substring, + const char* needle_expr, const char* haystack_expr, + const StringType& needle, const StringType& haystack) { + if (IsSubstringPred(needle, haystack) == expected_to_be_substring) + return AssertionSuccess(); + + const bool is_wide_string = sizeof(needle[0]) > 1; + const char* const begin_string_quote = is_wide_string ? "L\"" : "\""; + return AssertionFailure() + << "Value of: " << needle_expr << "\n" + << " Actual: " << begin_string_quote << needle << "\"\n" + << "Expected: " << (expected_to_be_substring ? "" : "not ") + << "a substring of " << haystack_expr << "\n" + << "Which is: " << begin_string_quote << haystack << "\""; +} + +} // namespace + +// IsSubstring() and IsNotSubstring() check whether needle is a +// substring of haystack (NULL is considered a substring of itself +// only), and return an appropriate error message when they fail. + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +#if GTEST_HAS_STD_WSTRING +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +#if GTEST_OS_WINDOWS + +namespace { + +// Helper function for IsHRESULT{SuccessFailure} predicates +AssertionResult HRESULTFailureHelper(const char* expr, + const char* expected, + long hr) { // NOLINT +# if GTEST_OS_WINDOWS_MOBILE + + // Windows CE doesn't support FormatMessage. + const char error_text[] = ""; + +# else + + // Looks up the human-readable system message for the HRESULT code + // and since we're not passing any params to FormatMessage, we don't + // want inserts expanded. + const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD kBufSize = 4096; + // Gets the system's human readable message string for this HRESULT. + char error_text[kBufSize] = { '\0' }; + DWORD message_length = ::FormatMessageA(kFlags, + 0, // no source, we're asking system + hr, // the error + 0, // no line width restrictions + error_text, // output buffer + kBufSize, // buf size + NULL); // no arguments for inserts + // Trims tailing white space (FormatMessage leaves a trailing CR-LF) + for (; message_length && IsSpace(error_text[message_length - 1]); + --message_length) { + error_text[message_length - 1] = '\0'; + } + +# endif // GTEST_OS_WINDOWS_MOBILE + + const std::string error_hex("0x" + String::FormatHexInt(hr)); + return ::testing::AssertionFailure() + << "Expected: " << expr << " " << expected << ".\n" + << " Actual: " << error_hex << " " << error_text << "\n"; +} + +} // namespace + +AssertionResult IsHRESULTSuccess(const char* expr, long hr) { // NOLINT + if (SUCCEEDED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "succeeds", hr); +} + +AssertionResult IsHRESULTFailure(const char* expr, long hr) { // NOLINT + if (FAILED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "fails", hr); +} + +#endif // GTEST_OS_WINDOWS + +// Utility functions for encoding Unicode text (wide strings) in +// UTF-8. + +// A Unicode code-point can have upto 21 bits, and is encoded in UTF-8 +// like this: +// +// Code-point length Encoding +// 0 - 7 bits 0xxxxxxx +// 8 - 11 bits 110xxxxx 10xxxxxx +// 12 - 16 bits 1110xxxx 10xxxxxx 10xxxxxx +// 17 - 21 bits 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + +// The maximum code-point a one-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint1 = (static_cast(1) << 7) - 1; + +// The maximum code-point a two-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint2 = (static_cast(1) << (5 + 6)) - 1; + +// The maximum code-point a three-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint3 = (static_cast(1) << (4 + 2*6)) - 1; + +// The maximum code-point a four-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint4 = (static_cast(1) << (3 + 3*6)) - 1; + +// Chops off the n lowest bits from a bit pattern. Returns the n +// lowest bits. As a side effect, the original bit pattern will be +// shifted to the right by n bits. +inline UInt32 ChopLowBits(UInt32* bits, int n) { + const UInt32 low_bits = *bits & ((static_cast(1) << n) - 1); + *bits >>= n; + return low_bits; +} + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +std::string CodePointToUtf8(UInt32 code_point) { + if (code_point > kMaxCodePoint4) { + return "(Invalid Unicode 0x" + String::FormatHexInt(code_point) + ")"; + } + + char str[5]; // Big enough for the largest valid code point. + if (code_point <= kMaxCodePoint1) { + str[1] = '\0'; + str[0] = static_cast(code_point); // 0xxxxxxx + } else if (code_point <= kMaxCodePoint2) { + str[2] = '\0'; + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xC0 | code_point); // 110xxxxx + } else if (code_point <= kMaxCodePoint3) { + str[3] = '\0'; + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xE0 | code_point); // 1110xxxx + } else { // code_point <= kMaxCodePoint4 + str[4] = '\0'; + str[3] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xF0 | code_point); // 11110xxx + } + return str; +} + +// The following two functions only make sense if the system +// uses UTF-16 for wide string encoding. All supported systems +// with 16 bit wchar_t (Windows, Cygwin, Symbian OS) do use UTF-16. + +// Determines if the arguments constitute UTF-16 surrogate pair +// and thus should be combined into a single Unicode code point +// using CreateCodePointFromUtf16SurrogatePair. +inline bool IsUtf16SurrogatePair(wchar_t first, wchar_t second) { + return sizeof(wchar_t) == 2 && + (first & 0xFC00) == 0xD800 && (second & 0xFC00) == 0xDC00; +} + +// Creates a Unicode code point from UTF16 surrogate pair. +inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first, + wchar_t second) { + const UInt32 mask = (1 << 10) - 1; + return (sizeof(wchar_t) == 2) ? + (((first & mask) << 10) | (second & mask)) + 0x10000 : + // This function should not be called when the condition is + // false, but we provide a sensible default in case it is. + static_cast(first); +} + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +std::string WideStringToUtf8(const wchar_t* str, int num_chars) { + if (num_chars == -1) + num_chars = static_cast(wcslen(str)); + + ::std::stringstream stream; + for (int i = 0; i < num_chars; ++i) { + UInt32 unicode_code_point; + + if (str[i] == L'\0') { + break; + } else if (i + 1 < num_chars && IsUtf16SurrogatePair(str[i], str[i + 1])) { + unicode_code_point = CreateCodePointFromUtf16SurrogatePair(str[i], + str[i + 1]); + i++; + } else { + unicode_code_point = static_cast(str[i]); + } + + stream << CodePointToUtf8(unicode_code_point); + } + return StringStreamToString(&stream); +} + +// Converts a wide C string to an std::string using the UTF-8 encoding. +// NULL will be converted to "(null)". +std::string String::ShowWideCString(const wchar_t * wide_c_str) { + if (wide_c_str == NULL) return "(null)"; + + return internal::WideStringToUtf8(wide_c_str, -1); +} + +// Compares two wide C strings. Returns true iff they have the same +// content. +// +// Unlike wcscmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::WideCStringEquals(const wchar_t * lhs, const wchar_t * rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + + return wcscmp(lhs, rhs) == 0; +} + +// Helper function for *_STREQ on wide strings. +AssertionResult CmpHelperSTREQ(const char* lhs_expression, + const char* rhs_expression, + const wchar_t* lhs, + const wchar_t* rhs) { + if (String::WideCStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + false); +} + +// Helper function for *_STRNE on wide strings. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2) { + if (!String::WideCStringEquals(s1, s2)) { + return AssertionSuccess(); + } + + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: " + << PrintToString(s1) + << " vs " << PrintToString(s2); +} + +// Compares two C strings, ignoring case. Returns true iff they have +// the same content. +// +// Unlike strcasecmp(), this function can handle NULL argument(s). A +// NULL C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CaseInsensitiveCStringEquals(const char * lhs, const char * rhs) { + if (lhs == NULL) + return rhs == NULL; + if (rhs == NULL) + return false; + return posix::StrCaseCmp(lhs, rhs) == 0; +} + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. +bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + +#if GTEST_OS_WINDOWS + return _wcsicmp(lhs, rhs) == 0; +#elif GTEST_OS_LINUX && !GTEST_OS_LINUX_ANDROID + return wcscasecmp(lhs, rhs) == 0; +#else + // Android, Mac OS X and Cygwin don't define wcscasecmp. + // Other unknown OSes may not define it either. + wint_t left, right; + do { + left = towlower(*lhs++); + right = towlower(*rhs++); + } while (left && left == right); + return left == right; +#endif // OS selector +} + +// Returns true iff str ends with the given suffix, ignoring case. +// Any string is considered to end with an empty suffix. +bool String::EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix) { + const size_t str_len = str.length(); + const size_t suffix_len = suffix.length(); + return (str_len >= suffix_len) && + CaseInsensitiveCStringEquals(str.c_str() + str_len - suffix_len, + suffix.c_str()); +} + +// Formats an int value as "%02d". +std::string String::FormatIntWidth2(int value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << value; + return ss.str(); +} + +// Formats an int value as "%X". +std::string String::FormatHexInt(int value) { + std::stringstream ss; + ss << std::hex << std::uppercase << value; + return ss.str(); +} + +// Formats a byte as "%02X". +std::string String::FormatByte(unsigned char value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << std::hex << std::uppercase + << static_cast(value); + return ss.str(); +} + +// Converts the buffer in a stringstream to an std::string, converting NUL +// bytes to "\\0" along the way. +std::string StringStreamToString(::std::stringstream* ss) { + const ::std::string& str = ss->str(); + const char* const start = str.c_str(); + const char* const end = start + str.length(); + + std::string result; + result.reserve(2 * (end - start)); + for (const char* ch = start; ch != end; ++ch) { + if (*ch == '\0') { + result += "\\0"; // Replaces NUL with "\\0"; + } else { + result += *ch; + } + } + + return result; +} + +// Appends the user-supplied message to the Google-Test-generated message. +std::string AppendUserMessage(const std::string& gtest_msg, + const Message& user_msg) { + // Appends the user message if it's non-empty. + const std::string user_msg_string = user_msg.GetString(); + if (user_msg_string.empty()) { + return gtest_msg; + } + + return gtest_msg + "\n" + user_msg_string; +} + +} // namespace internal + +// class TestResult + +// Creates an empty TestResult. +TestResult::TestResult() + : death_test_count_(0), + elapsed_time_(0) { +} + +// D'tor. +TestResult::~TestResult() { +} + +// Returns the i-th test part result among all the results. i can +// range from 0 to total_part_count() - 1. If i is not in that range, +// aborts the program. +const TestPartResult& TestResult::GetTestPartResult(int i) const { + if (i < 0 || i >= total_part_count()) + internal::posix::Abort(); + return test_part_results_.at(i); +} + +// Returns the i-th test property. i can range from 0 to +// test_property_count() - 1. If i is not in that range, aborts the +// program. +const TestProperty& TestResult::GetTestProperty(int i) const { + if (i < 0 || i >= test_property_count()) + internal::posix::Abort(); + return test_properties_.at(i); +} + +// Clears the test part results. +void TestResult::ClearTestPartResults() { + test_part_results_.clear(); +} + +// Adds a test part result to the list. +void TestResult::AddTestPartResult(const TestPartResult& test_part_result) { + test_part_results_.push_back(test_part_result); +} + +// Adds a test property to the list. If a property with the same key as the +// supplied property is already represented, the value of this test_property +// replaces the old value for that key. +void TestResult::RecordProperty(const std::string& xml_element, + const TestProperty& test_property) { + if (!ValidateTestProperty(xml_element, test_property)) { + return; + } + internal::MutexLock lock(&test_properites_mutex_); + const std::vector::iterator property_with_matching_key = + std::find_if(test_properties_.begin(), test_properties_.end(), + internal::TestPropertyKeyIs(test_property.key())); + if (property_with_matching_key == test_properties_.end()) { + test_properties_.push_back(test_property); + return; + } + property_with_matching_key->SetValue(test_property.value()); +} + +// The list of reserved attributes used in the element of XML +// output. +static const char* const kReservedTestSuitesAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "random_seed", + "tests", + "time", + "timestamp" +}; + +// The list of reserved attributes used in the element of XML +// output. +static const char* const kReservedTestSuiteAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "tests", + "time" +}; + +// The list of reserved attributes used in the element of XML output. +static const char* const kReservedTestCaseAttributes[] = { + "classname", + "name", + "status", + "time", + "type_param", + "value_param" +}; + +template +std::vector ArrayAsVector(const char* const (&array)[kSize]) { + return std::vector(array, array + kSize); +} + +static std::vector GetReservedAttributesForElement( + const std::string& xml_element) { + if (xml_element == "testsuites") { + return ArrayAsVector(kReservedTestSuitesAttributes); + } else if (xml_element == "testsuite") { + return ArrayAsVector(kReservedTestSuiteAttributes); + } else if (xml_element == "testcase") { + return ArrayAsVector(kReservedTestCaseAttributes); + } else { + GTEST_CHECK_(false) << "Unrecognized xml_element provided: " << xml_element; + } + // This code is unreachable but some compilers may not realizes that. + return std::vector(); +} + +static std::string FormatWordList(const std::vector& words) { + Message word_list; + for (size_t i = 0; i < words.size(); ++i) { + if (i > 0 && words.size() > 2) { + word_list << ", "; + } + if (i == words.size() - 1) { + word_list << "and "; + } + word_list << "'" << words[i] << "'"; + } + return word_list.GetString(); +} + +bool ValidateTestPropertyName(const std::string& property_name, + const std::vector& reserved_names) { + if (std::find(reserved_names.begin(), reserved_names.end(), property_name) != + reserved_names.end()) { + ADD_FAILURE() << "Reserved key used in RecordProperty(): " << property_name + << " (" << FormatWordList(reserved_names) + << " are reserved by " << GTEST_NAME_ << ")"; + return false; + } + return true; +} + +// Adds a failure if the key is a reserved attribute of the element named +// xml_element. Returns true if the property is valid. +bool TestResult::ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property) { + return ValidateTestPropertyName(test_property.key(), + GetReservedAttributesForElement(xml_element)); +} + +// Clears the object. +void TestResult::Clear() { + test_part_results_.clear(); + test_properties_.clear(); + death_test_count_ = 0; + elapsed_time_ = 0; +} + +// Returns true iff the test failed. +bool TestResult::Failed() const { + for (int i = 0; i < total_part_count(); ++i) { + if (GetTestPartResult(i).failed()) + return true; + } + return false; +} + +// Returns true iff the test part fatally failed. +static bool TestPartFatallyFailed(const TestPartResult& result) { + return result.fatally_failed(); +} + +// Returns true iff the test fatally failed. +bool TestResult::HasFatalFailure() const { + return CountIf(test_part_results_, TestPartFatallyFailed) > 0; +} + +// Returns true iff the test part non-fatally failed. +static bool TestPartNonfatallyFailed(const TestPartResult& result) { + return result.nonfatally_failed(); +} + +// Returns true iff the test has a non-fatal failure. +bool TestResult::HasNonfatalFailure() const { + return CountIf(test_part_results_, TestPartNonfatallyFailed) > 0; +} + +// Gets the number of all test parts. This is the sum of the number +// of successful test parts and the number of failed test parts. +int TestResult::total_part_count() const { + return static_cast(test_part_results_.size()); +} + +// Returns the number of the test properties. +int TestResult::test_property_count() const { + return static_cast(test_properties_.size()); +} + +// class Test + +// Creates a Test object. + +// The c'tor saves the states of all flags. +Test::Test() + : gtest_flag_saver_(new GTEST_FLAG_SAVER_) { +} + +// The d'tor restores the states of all flags. The actual work is +// done by the d'tor of the gtest_flag_saver_ field, and thus not +// visible here. +Test::~Test() { +} + +// Sets up the test fixture. +// +// A sub-class may override this. +void Test::SetUp() { +} + +// Tears down the test fixture. +// +// A sub-class may override this. +void Test::TearDown() { +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const std::string& key, const std::string& value) { + UnitTest::GetInstance()->RecordProperty(key, value); +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const std::string& key, int value) { + Message value_message; + value_message << value; + RecordProperty(key, value_message.GetString().c_str()); +} + +namespace internal { + +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const std::string& message) { + // This function is a friend of UnitTest and as such has access to + // AddTestPartResult. + UnitTest::GetInstance()->AddTestPartResult( + result_type, + NULL, // No info about the source file where the exception occurred. + -1, // We have no info on which line caused the exception. + message, + ""); // No stack trace, either. +} + +} // namespace internal + +// Google Test requires all tests in the same test case to use the same test +// fixture class. This function checks if the current test has the +// same fixture class as the first test in the current test case. If +// yes, it returns true; otherwise it generates a Google Test failure and +// returns false. +bool Test::HasSameFixtureClass() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + const TestCase* const test_case = impl->current_test_case(); + + // Info about the first test in the current test case. + const TestInfo* const first_test_info = test_case->test_info_list()[0]; + const internal::TypeId first_fixture_id = first_test_info->fixture_class_id_; + const char* const first_test_name = first_test_info->name(); + + // Info about the current test. + const TestInfo* const this_test_info = impl->current_test_info(); + const internal::TypeId this_fixture_id = this_test_info->fixture_class_id_; + const char* const this_test_name = this_test_info->name(); + + if (this_fixture_id != first_fixture_id) { + // Is the first test defined using TEST? + const bool first_is_TEST = first_fixture_id == internal::GetTestTypeId(); + // Is this test defined using TEST? + const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId(); + + if (first_is_TEST || this_is_TEST) { + // Both TEST and TEST_F appear in same test case, which is incorrect. + // Tell the user how to fix this. + + // Gets the name of the TEST and the name of the TEST_F. Note + // that first_is_TEST and this_is_TEST cannot both be true, as + // the fixture IDs are different for the two tests. + const char* const TEST_name = + first_is_TEST ? first_test_name : this_test_name; + const char* const TEST_F_name = + first_is_TEST ? this_test_name : first_test_name; + + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class, so mixing TEST_F and TEST in the same test case is\n" + << "illegal. In test case " << this_test_info->test_case_name() + << ",\n" + << "test " << TEST_F_name << " is defined using TEST_F but\n" + << "test " << TEST_name << " is defined using TEST. You probably\n" + << "want to change the TEST to TEST_F or move it to another test\n" + << "case."; + } else { + // Two fixture classes with the same name appear in two different + // namespaces, which is not allowed. Tell the user how to fix this. + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " + << this_test_info->test_case_name() << ",\n" + << "you defined test " << first_test_name + << " and test " << this_test_name << "\n" + << "using two different test fixture classes. This can happen if\n" + << "the two classes are from different namespaces or translation\n" + << "units and have the same name. You should probably rename one\n" + << "of the classes to put the tests into different test cases."; + } + return false; + } + + return true; +} + +#if GTEST_HAS_SEH + +// Adds an "exception thrown" fatal failure to the current test. This +// function returns its result via an output parameter pointer because VC++ +// prohibits creation of objects with destructors on stack in functions +// using __try (see error C2712). +static std::string* FormatSehExceptionMessage(DWORD exception_code, + const char* location) { + Message message; + message << "SEH exception with code 0x" << std::setbase(16) << + exception_code << std::setbase(10) << " thrown in " << location << "."; + + return new std::string(message.GetString()); +} + +#endif // GTEST_HAS_SEH + +namespace internal { + +#if GTEST_HAS_EXCEPTIONS + +// Adds an "exception thrown" fatal failure to the current test. +static std::string FormatCxxExceptionMessage(const char* description, + const char* location) { + Message message; + if (description != NULL) { + message << "C++ exception with description \"" << description << "\""; + } else { + message << "Unknown C++ exception"; + } + message << " thrown in " << location << "."; + + return message.GetString(); +} + +static std::string PrintTestPartResultToString( + const TestPartResult& test_part_result); + +GoogleTestFailureException::GoogleTestFailureException( + const TestPartResult& failure) + : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {} + +#endif // GTEST_HAS_EXCEPTIONS + +// We put these helper functions in the internal namespace as IBM's xlC +// compiler rejects the code if they were declared static. + +// Runs the given method and handles SEH exceptions it throws, when +// SEH is supported; returns the 0-value for type Result in case of an +// SEH exception. (Microsoft compilers cannot handle SEH and C++ +// exceptions in the same function. Therefore, we provide a separate +// wrapper function for handling SEH exceptions.) +template +Result HandleSehExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { +#if GTEST_HAS_SEH + __try { + return (object->*method)(); + } __except (internal::UnitTestOptions::GTestShouldProcessSEH( // NOLINT + GetExceptionCode())) { + // We create the exception message on the heap because VC++ prohibits + // creation of objects with destructors on stack in functions using __try + // (see error C2712). + std::string* exception_message = FormatSehExceptionMessage( + GetExceptionCode(), location); + internal::ReportFailureInUnknownLocation(TestPartResult::kFatalFailure, + *exception_message); + delete exception_message; + return static_cast(0); + } +#else + (void)location; + return (object->*method)(); +#endif // GTEST_HAS_SEH +} + +// Runs the given method and catches and reports C++ and/or SEH-style +// exceptions, if they are supported; returns the 0-value for type +// Result in case of an SEH exception. +template +Result HandleExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { + // NOTE: The user code can affect the way in which Google Test handles + // exceptions by setting GTEST_FLAG(catch_exceptions), but only before + // RUN_ALL_TESTS() starts. It is technically possible to check the flag + // after the exception is caught and either report or re-throw the + // exception based on the flag's value: + // + // try { + // // Perform the test method. + // } catch (...) { + // if (GTEST_FLAG(catch_exceptions)) + // // Report the exception as failure. + // else + // throw; // Re-throws the original exception. + // } + // + // However, the purpose of this flag is to allow the program to drop into + // the debugger when the exception is thrown. On most platforms, once the + // control enters the catch block, the exception origin information is + // lost and the debugger will stop the program at the point of the + // re-throw in this function -- instead of at the point of the original + // throw statement in the code under test. For this reason, we perform + // the check early, sacrificing the ability to affect Google Test's + // exception handling in the method where the exception is thrown. + if (internal::GetUnitTestImpl()->catch_exceptions()) { +#if GTEST_HAS_EXCEPTIONS + try { + return HandleSehExceptionsInMethodIfSupported(object, method, location); + } catch (const internal::GoogleTestFailureException&) { // NOLINT + // This exception type can only be thrown by a failed Google + // Test assertion with the intention of letting another testing + // framework catch it. Therefore we just re-throw it. + throw; + } catch (const std::exception& e) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(e.what(), location)); + } catch (...) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(NULL, location)); + } + return static_cast(0); +#else + return HandleSehExceptionsInMethodIfSupported(object, method, location); +#endif // GTEST_HAS_EXCEPTIONS + } else { + return (object->*method)(); + } +} + +} // namespace internal + +// Runs the test and updates the test result. +void Test::Run() { + if (!HasSameFixtureClass()) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported(this, &Test::SetUp, "SetUp()"); + // We will run the test only if SetUp() was successful. + if (!HasFatalFailure()) { + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TestBody, "the test body"); + } + + // However, we want to clean up as much as possible. Hence we will + // always call TearDown(), even if SetUp() or the test body has + // failed. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TearDown, "TearDown()"); +} + +// Returns true iff the current test has a fatal failure. +bool Test::HasFatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()->HasFatalFailure(); +} + +// Returns true iff the current test has a non-fatal failure. +bool Test::HasNonfatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()-> + HasNonfatalFailure(); +} + +// class TestInfo + +// Constructs a TestInfo object. It assumes ownership of the test factory +// object. +TestInfo::TestInfo(const std::string& a_test_case_name, + const std::string& a_name, + const char* a_type_param, + const char* a_value_param, + internal::CodeLocation a_code_location, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory) + : test_case_name_(a_test_case_name), + name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + value_param_(a_value_param ? new std::string(a_value_param) : NULL), + location_(a_code_location), + fixture_class_id_(fixture_class_id), + should_run_(false), + is_disabled_(false), + matches_filter_(false), + factory_(factory), + result_() {} + +// Destructs a TestInfo object. +TestInfo::~TestInfo() { delete factory_; } + +namespace internal { + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param: the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param: text representation of the test's value parameter, +// or NULL if this is not a value-parameterized test. +// code_location: code location where the test is defined +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + CodeLocation code_location, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory) { + TestInfo* const test_info = + new TestInfo(test_case_name, name, type_param, value_param, + code_location, fixture_class_id, factory); + GetUnitTestImpl()->AddTestInfo(set_up_tc, tear_down_tc, test_info); + return test_info; +} + +#if GTEST_HAS_PARAM_TEST +void ReportInvalidTestCaseType(const char* test_case_name, + CodeLocation code_location) { + Message errors; + errors + << "Attempted redefinition of test case " << test_case_name << ".\n" + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " << test_case_name << ", you tried\n" + << "to define a test using a fixture class different from the one\n" + << "used earlier. This can happen if the two fixture classes are\n" + << "from different namespaces and have the same name. You should\n" + << "probably rename one of the classes to put the tests into different\n" + << "test cases."; + + fprintf(stderr, "%s %s", + FormatFileLocation(code_location.file.c_str(), + code_location.line).c_str(), + errors.GetString().c_str()); +} +#endif // GTEST_HAS_PARAM_TEST + +} // namespace internal + +namespace { + +// A predicate that checks the test name of a TestInfo against a known +// value. +// +// This is used for implementation of the TestCase class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestNameIs is copyable. +class TestNameIs { + public: + // Constructor. + // + // TestNameIs has NO default constructor. + explicit TestNameIs(const char* name) + : name_(name) {} + + // Returns true iff the test name of test_info matches name_. + bool operator()(const TestInfo * test_info) const { + return test_info && test_info->name() == name_; + } + + private: + std::string name_; +}; + +} // namespace + +namespace internal { + +// This method expands all parameterized tests registered with macros TEST_P +// and INSTANTIATE_TEST_CASE_P into regular tests and registers those. +// This will be done just once during the program runtime. +void UnitTestImpl::RegisterParameterizedTests() { +#if GTEST_HAS_PARAM_TEST + if (!parameterized_tests_registered_) { + parameterized_test_registry_.RegisterTests(); + parameterized_tests_registered_ = true; + } +#endif +} + +} // namespace internal + +// Creates the test object, runs it, records its result, and then +// deletes it. +void TestInfo::Run() { + if (!should_run_) return; + + // Tells UnitTest where to store test result. + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_info(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Notifies the unit test event listeners that a test is about to start. + repeater->OnTestStart(*this); + + const TimeInMillis start = internal::GetTimeInMillis(); + + impl->os_stack_trace_getter()->UponLeavingGTest(); + + // Creates the test object. + Test* const test = internal::HandleExceptionsInMethodIfSupported( + factory_, &internal::TestFactoryBase::CreateTest, + "the test fixture's constructor"); + + // Runs the test only if the test object was created and its + // constructor didn't generate a fatal failure. + if ((test != NULL) && !Test::HasFatalFailure()) { + // This doesn't throw as all user code that can throw are wrapped into + // exception handling code. + test->Run(); + } + + // Deletes the test object. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + test, &Test::DeleteSelf_, "the test fixture's destructor"); + + result_.set_elapsed_time(internal::GetTimeInMillis() - start); + + // Notifies the unit test event listener that a test has just finished. + repeater->OnTestEnd(*this); + + // Tells UnitTest to stop associating assertion results to this + // test. + impl->set_current_test_info(NULL); +} + +// class TestCase + +// Gets the number of successful tests in this test case. +int TestCase::successful_test_count() const { + return CountIf(test_info_list_, TestPassed); +} + +// Gets the number of failed tests in this test case. +int TestCase::failed_test_count() const { + return CountIf(test_info_list_, TestFailed); +} + +// Gets the number of disabled tests that will be reported in the XML report. +int TestCase::reportable_disabled_test_count() const { + return CountIf(test_info_list_, TestReportableDisabled); +} + +// Gets the number of disabled tests in this test case. +int TestCase::disabled_test_count() const { + return CountIf(test_info_list_, TestDisabled); +} + +// Gets the number of tests to be printed in the XML report. +int TestCase::reportable_test_count() const { + return CountIf(test_info_list_, TestReportable); +} + +// Get the number of tests in this test case that should run. +int TestCase::test_to_run_count() const { + return CountIf(test_info_list_, ShouldRunTest); +} + +// Gets the number of all tests. +int TestCase::total_test_count() const { + return static_cast(test_info_list_.size()); +} + +// Creates a TestCase with the given name. +// +// Arguments: +// +// name: name of the test case +// a_type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase::TestCase(const char* a_name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) + : name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + set_up_tc_(set_up_tc), + tear_down_tc_(tear_down_tc), + should_run_(false), + elapsed_time_(0) { +} + +// Destructor of TestCase. +TestCase::~TestCase() { + // Deletes every Test in the collection. + ForEach(test_info_list_, internal::Delete); +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +const TestInfo* TestCase::GetTestInfo(int i) const { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +TestInfo* TestCase::GetMutableTestInfo(int i) { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Adds a test to this test case. Will delete the test upon +// destruction of the TestCase object. +void TestCase::AddTestInfo(TestInfo * test_info) { + test_info_list_.push_back(test_info); + test_indices_.push_back(static_cast(test_indices_.size())); +} + +// Runs every test in this TestCase. +void TestCase::Run() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_case(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + repeater->OnTestCaseStart(*this); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunSetUpTestCase, "SetUpTestCase()"); + + const internal::TimeInMillis start = internal::GetTimeInMillis(); + for (int i = 0; i < total_test_count(); i++) { + GetMutableTestInfo(i)->Run(); + } + elapsed_time_ = internal::GetTimeInMillis() - start; + + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunTearDownTestCase, "TearDownTestCase()"); + + repeater->OnTestCaseEnd(*this); + impl->set_current_test_case(NULL); +} + +// Clears the results of all tests in this test case. +void TestCase::ClearResult() { + ad_hoc_test_result_.Clear(); + ForEach(test_info_list_, TestInfo::ClearTestResult); +} + +// Shuffles the tests in this test case. +void TestCase::ShuffleTests(internal::Random* random) { + Shuffle(random, &test_indices_); +} + +// Restores the test order to before the first shuffle. +void TestCase::UnshuffleTests() { + for (size_t i = 0; i < test_indices_.size(); i++) { + test_indices_[i] = static_cast(i); + } +} + +// Formats a countable noun. Depending on its quantity, either the +// singular form or the plural form is used. e.g. +// +// FormatCountableNoun(1, "formula", "formuli") returns "1 formula". +// FormatCountableNoun(5, "book", "books") returns "5 books". +static std::string FormatCountableNoun(int count, + const char * singular_form, + const char * plural_form) { + return internal::StreamableToString(count) + " " + + (count == 1 ? singular_form : plural_form); +} + +// Formats the count of tests. +static std::string FormatTestCount(int test_count) { + return FormatCountableNoun(test_count, "test", "tests"); +} + +// Formats the count of test cases. +static std::string FormatTestCaseCount(int test_case_count) { + return FormatCountableNoun(test_case_count, "test case", "test cases"); +} + +// Converts a TestPartResult::Type enum to human-friendly string +// representation. Both kNonFatalFailure and kFatalFailure are translated +// to "Failure", as the user usually doesn't care about the difference +// between the two when viewing the test result. +static const char * TestPartResultTypeToString(TestPartResult::Type type) { + switch (type) { + case TestPartResult::kSuccess: + return "Success"; + + case TestPartResult::kNonFatalFailure: + case TestPartResult::kFatalFailure: +#ifdef _MSC_VER + return "error: "; +#else + return "Failure\n"; +#endif + default: + return "Unknown result type"; + } +} + +namespace internal { + +// Prints a TestPartResult to an std::string. +static std::string PrintTestPartResultToString( + const TestPartResult& test_part_result) { + return (Message() + << internal::FormatFileLocation(test_part_result.file_name(), + test_part_result.line_number()) + << " " << TestPartResultTypeToString(test_part_result.type()) + << test_part_result.message()).GetString(); +} + +// Prints a TestPartResult. +static void PrintTestPartResult(const TestPartResult& test_part_result) { + const std::string& result = + PrintTestPartResultToString(test_part_result); + printf("%s\n", result.c_str()); + fflush(stdout); + // If the test program runs in Visual Studio or a debugger, the + // following statements add the test part result message to the Output + // window such that the user can double-click on it to jump to the + // corresponding source code location; otherwise they do nothing. +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + // We don't call OutputDebugString*() on Windows Mobile, as printing + // to stdout is done by OutputDebugString() there already - we don't + // want the same message printed twice. + ::OutputDebugStringA(result.c_str()); + ::OutputDebugStringA("\n"); +#endif +} + +// class PrettyUnitTestResultPrinter + +enum GTestColor { + COLOR_DEFAULT, + COLOR_RED, + COLOR_GREEN, + COLOR_YELLOW +}; + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +// Returns the character attribute for the given color. +WORD GetColorAttribute(GTestColor color) { + switch (color) { + case COLOR_RED: return FOREGROUND_RED; + case COLOR_GREEN: return FOREGROUND_GREEN; + case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; + default: return 0; + } +} + +#else + +// Returns the ANSI color code for the given color. COLOR_DEFAULT is +// an invalid input. +const char* GetAnsiColorCode(GTestColor color) { + switch (color) { + case COLOR_RED: return "1"; + case COLOR_GREEN: return "2"; + case COLOR_YELLOW: return "3"; + default: return NULL; + }; +} + +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + +// Returns true iff Google Test should use colors in the output. +bool ShouldUseColor(bool stdout_is_tty) { + const char* const gtest_color = GTEST_FLAG(color).c_str(); + + if (String::CaseInsensitiveCStringEquals(gtest_color, "auto")) { +#if GTEST_OS_WINDOWS + // On Windows the TERM variable is usually not set, but the + // console there does support colors. + return stdout_is_tty; +#else + // On non-Windows platforms, we rely on the TERM variable. + const char* const term = posix::GetEnv("TERM"); + const bool term_supports_color = + String::CStringEquals(term, "xterm") || + String::CStringEquals(term, "xterm-color") || + String::CStringEquals(term, "xterm-256color") || + String::CStringEquals(term, "screen") || + String::CStringEquals(term, "screen-256color") || + String::CStringEquals(term, "tmux") || + String::CStringEquals(term, "tmux-256color") || + String::CStringEquals(term, "rxvt-unicode") || + String::CStringEquals(term, "rxvt-unicode-256color") || + String::CStringEquals(term, "linux") || + String::CStringEquals(term, "cygwin"); + return stdout_is_tty && term_supports_color; +#endif // GTEST_OS_WINDOWS + } + + return String::CaseInsensitiveCStringEquals(gtest_color, "yes") || + String::CaseInsensitiveCStringEquals(gtest_color, "true") || + String::CaseInsensitiveCStringEquals(gtest_color, "t") || + String::CStringEquals(gtest_color, "1"); + // We take "yes", "true", "t", and "1" as meaning "yes". If the + // value is neither one of these nor "auto", we treat it as "no" to + // be conservative. +} + +// Helpers for printing colored strings to stdout. Note that on Windows, we +// cannot simply emit special characters and have the terminal change colors. +// This routine must actually emit the characters rather than return a string +// that would be colored when printed, as can be done on Linux. +void ColoredPrintf(GTestColor color, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS || \ + GTEST_OS_IOS || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + const bool use_color = AlwaysFalse(); +#else + static const bool in_color_mode = + ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); + const bool use_color = in_color_mode && (color != COLOR_DEFAULT); +#endif // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS + // The '!= 0' comparison is necessary to satisfy MSVC 7.1. + + if (!use_color) { + vprintf(fmt, args); + va_end(args); + return; + } + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); + + // Gets the current text color. + CONSOLE_SCREEN_BUFFER_INFO buffer_info; + GetConsoleScreenBufferInfo(stdout_handle, &buffer_info); + const WORD old_color_attrs = buffer_info.wAttributes; + + // We need to flush the stream buffers into the console before each + // SetConsoleTextAttribute call lest it affect the text that is already + // printed but has not yet reached the console. + fflush(stdout); + SetConsoleTextAttribute(stdout_handle, + GetColorAttribute(color) | FOREGROUND_INTENSITY); + vprintf(fmt, args); + + fflush(stdout); + // Restores the text color. + SetConsoleTextAttribute(stdout_handle, old_color_attrs); +#else + printf("\033[0;3%sm", GetAnsiColorCode(color)); + vprintf(fmt, args); + printf("\033[m"); // Resets the terminal to default. +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + va_end(args); +} + +// Text printed in Google Test's text output and --gunit_list_tests +// output to label the type parameter and value parameter for a test. +static const char kTypeParamLabel[] = "TypeParam"; +static const char kValueParamLabel[] = "GetParam()"; + +void PrintFullTestCommentIfPresent(const TestInfo& test_info) { + const char* const type_param = test_info.type_param(); + const char* const value_param = test_info.value_param(); + + if (type_param != NULL || value_param != NULL) { + printf(", where "); + if (type_param != NULL) { + printf("%s = %s", kTypeParamLabel, type_param); + if (value_param != NULL) + printf(" and "); + } + if (value_param != NULL) { + printf("%s = %s", kValueParamLabel, value_param); + } + } +} + +// This class implements the TestEventListener interface. +// +// Class PrettyUnitTestResultPrinter is copyable. +class PrettyUnitTestResultPrinter : public TestEventListener { + public: + PrettyUnitTestResultPrinter() {} + static void PrintTestName(const char * test_case, const char * test) { + printf("%s.%s", test_case, test); + } + + // The following methods override what's in the TestEventListener class. + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} + + private: + static void PrintFailedTests(const UnitTest& unit_test); +}; + + // Fired before each iteration of tests starts. +void PrettyUnitTestResultPrinter::OnTestIterationStart( + const UnitTest& unit_test, int iteration) { + if (GTEST_FLAG(repeat) != 1) + printf("\nRepeating all tests (iteration %d) . . .\n\n", iteration + 1); + + const char* const filter = GTEST_FLAG(filter).c_str(); + + // Prints the filter if it's not *. This reminds the user that some + // tests may be skipped. + if (!String::CStringEquals(filter, kUniversalFilter)) { + ColoredPrintf(COLOR_YELLOW, + "Note: %s filter = %s\n", GTEST_NAME_, filter); + } + + if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { + const Int32 shard_index = Int32FromEnvOrDie(kTestShardIndex, -1); + ColoredPrintf(COLOR_YELLOW, + "Note: This is test shard %d of %s.\n", + static_cast(shard_index) + 1, + internal::posix::GetEnv(kTestTotalShards)); + } + + if (GTEST_FLAG(shuffle)) { + ColoredPrintf(COLOR_YELLOW, + "Note: Randomizing tests' orders with a seed of %d .\n", + unit_test.random_seed()); + } + + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("Running %s from %s.\n", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment set-up.\n"); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { + const std::string counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s", counts.c_str(), test_case.name()); + if (test_case.type_param() == NULL) { + printf("\n"); + } else { + printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param()); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { + ColoredPrintf(COLOR_GREEN, "[ RUN ] "); + PrintTestName(test_info.test_case_name(), test_info.name()); + printf("\n"); + fflush(stdout); +} + +// Called after an assertion failure. +void PrettyUnitTestResultPrinter::OnTestPartResult( + const TestPartResult& result) { + // If the test part succeeded, we don't need to do anything. + if (result.type() == TestPartResult::kSuccess) + return; + + // Print failure message from the assertion (e.g. expected this and got that). + PrintTestPartResult(result); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { + if (test_info.result()->Passed()) { + ColoredPrintf(COLOR_GREEN, "[ OK ] "); + } else { + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + } + PrintTestName(test_info.test_case_name(), test_info.name()); + if (test_info.result()->Failed()) + PrintFullTestCommentIfPresent(test_info); + + if (GTEST_FLAG(print_time)) { + printf(" (%s ms)\n", internal::StreamableToString( + test_info.result()->elapsed_time()).c_str()); + } else { + printf("\n"); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { + if (!GTEST_FLAG(print_time)) return; + + const std::string counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s (%s ms total)\n\n", + counts.c_str(), test_case.name(), + internal::StreamableToString(test_case.elapsed_time()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment tear-down\n"); + fflush(stdout); +} + +// Internal helper for printing the list of failed tests. +void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) { + const int failed_test_count = unit_test.failed_test_count(); + if (failed_test_count == 0) { + return; + } + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + const TestCase& test_case = *unit_test.GetTestCase(i); + if (!test_case.should_run() || (test_case.failed_test_count() == 0)) { + continue; + } + for (int j = 0; j < test_case.total_test_count(); ++j) { + const TestInfo& test_info = *test_case.GetTestInfo(j); + if (!test_info.should_run() || test_info.result()->Passed()) { + continue; + } + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s.%s", test_case.name(), test_info.name()); + PrintFullTestCommentIfPresent(test_info); + printf("\n"); + } + } +} + +void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("%s from %s ran.", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + if (GTEST_FLAG(print_time)) { + printf(" (%s ms total)", + internal::StreamableToString(unit_test.elapsed_time()).c_str()); + } + printf("\n"); + ColoredPrintf(COLOR_GREEN, "[ PASSED ] "); + printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); + + int num_failures = unit_test.failed_test_count(); + if (!unit_test.Passed()) { + const int failed_test_count = unit_test.failed_test_count(); + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str()); + PrintFailedTests(unit_test); + printf("\n%2d FAILED %s\n", num_failures, + num_failures == 1 ? "TEST" : "TESTS"); + } + + int num_disabled = unit_test.reportable_disabled_test_count(); + if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { + if (!num_failures) { + printf("\n"); // Add a spacer if no FAILURE banner is displayed. + } + ColoredPrintf(COLOR_YELLOW, + " YOU HAVE %d DISABLED %s\n\n", + num_disabled, + num_disabled == 1 ? "TEST" : "TESTS"); + } + // Ensure that Google Test output is printed before, e.g., heapchecker output. + fflush(stdout); +} + +// End PrettyUnitTestResultPrinter + +// class TestEventRepeater +// +// This class forwards events to other event listeners. +class TestEventRepeater : public TestEventListener { + public: + TestEventRepeater() : forwarding_enabled_(true) {} + virtual ~TestEventRepeater(); + void Append(TestEventListener *listener); + TestEventListener* Release(TestEventListener* listener); + + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled() const { return forwarding_enabled_; } + void set_forwarding_enabled(bool enable) { forwarding_enabled_ = enable; } + + virtual void OnTestProgramStart(const UnitTest& unit_test); + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test); + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test); + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& unit_test); + + private: + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled_; + // The list of listeners that receive events. + std::vector listeners_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventRepeater); +}; + +TestEventRepeater::~TestEventRepeater() { + ForEach(listeners_, Delete); +} + +void TestEventRepeater::Append(TestEventListener *listener) { + listeners_.push_back(listener); +} + +// TODO(vladl@google.com): Factor the search functionality into Vector::Find. +TestEventListener* TestEventRepeater::Release(TestEventListener *listener) { + for (size_t i = 0; i < listeners_.size(); ++i) { + if (listeners_[i] == listener) { + listeners_.erase(listeners_.begin() + i); + return listener; + } + } + + return NULL; +} + +// Since most methods are very similar, use macros to reduce boilerplate. +// This defines a member that forwards the call to all listeners. +#define GTEST_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (size_t i = 0; i < listeners_.size(); i++) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} +// This defines a member that forwards the call to all listeners in reverse +// order. +#define GTEST_REVERSE_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} + +GTEST_REPEATER_METHOD_(OnTestProgramStart, UnitTest) +GTEST_REPEATER_METHOD_(OnEnvironmentsSetUpStart, UnitTest) +GTEST_REPEATER_METHOD_(OnTestCaseStart, TestCase) +GTEST_REPEATER_METHOD_(OnTestStart, TestInfo) +GTEST_REPEATER_METHOD_(OnTestPartResult, TestPartResult) +GTEST_REPEATER_METHOD_(OnEnvironmentsTearDownStart, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsSetUpEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsTearDownEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnTestEnd, TestInfo) +GTEST_REVERSE_REPEATER_METHOD_(OnTestCaseEnd, TestCase) +GTEST_REVERSE_REPEATER_METHOD_(OnTestProgramEnd, UnitTest) + +#undef GTEST_REPEATER_METHOD_ +#undef GTEST_REVERSE_REPEATER_METHOD_ + +void TestEventRepeater::OnTestIterationStart(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (size_t i = 0; i < listeners_.size(); i++) { + listeners_[i]->OnTestIterationStart(unit_test, iteration); + } + } +} + +void TestEventRepeater::OnTestIterationEnd(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { + listeners_[i]->OnTestIterationEnd(unit_test, iteration); + } + } +} + +// End TestEventRepeater + +// This class generates an XML output file. +class XmlUnitTestResultPrinter : public EmptyTestEventListener { + public: + explicit XmlUnitTestResultPrinter(const char* output_file); + + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + + private: + // Is c a whitespace character that is normalized to a space character + // when it appears in an XML attribute value? + static bool IsNormalizableWhitespace(char c) { + return c == 0x9 || c == 0xA || c == 0xD; + } + + // May c appear in a well-formed XML document? + static bool IsValidXmlCharacter(char c) { + return IsNormalizableWhitespace(c) || c >= 0x20; + } + + // Returns an XML-escaped copy of the input string str. If + // is_attribute is true, the text is meant to appear as an attribute + // value, and normalizable whitespace is preserved by replacing it + // with character references. + static std::string EscapeXml(const std::string& str, bool is_attribute); + + // Returns the given string with all characters invalid in XML removed. + static std::string RemoveInvalidXmlCharacters(const std::string& str); + + // Convenience wrapper around EscapeXml when str is an attribute value. + static std::string EscapeXmlAttribute(const std::string& str) { + return EscapeXml(str, true); + } + + // Convenience wrapper around EscapeXml when str is not an attribute value. + static std::string EscapeXmlText(const char* str) { + return EscapeXml(str, false); + } + + // Verifies that the given attribute belongs to the given element and + // streams the attribute as XML. + static void OutputXmlAttribute(std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value); + + // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. + static void OutputXmlCDataSection(::std::ostream* stream, const char* data); + + // Streams an XML representation of a TestInfo object. + static void OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info); + + // Prints an XML representation of a TestCase object + static void PrintXmlTestCase(::std::ostream* stream, + const TestCase& test_case); + + // Prints an XML summary of unit_test to output stream out. + static void PrintXmlUnitTest(::std::ostream* stream, + const UnitTest& unit_test); + + // Produces a string representing the test properties in a result as space + // delimited XML attributes based on the property key="value" pairs. + // When the std::string is not empty, it includes a space at the beginning, + // to delimit this attribute from prior attributes. + static std::string TestPropertiesAsXmlAttributes(const TestResult& result); + + // The output file. + const std::string output_file_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(XmlUnitTestResultPrinter); +}; + +// Creates a new XmlUnitTestResultPrinter. +XmlUnitTestResultPrinter::XmlUnitTestResultPrinter(const char* output_file) + : output_file_(output_file) { + if (output_file_.c_str() == NULL || output_file_.empty()) { + fprintf(stderr, "XML output file may not be null\n"); + fflush(stderr); + exit(EXIT_FAILURE); + } +} + +// Called after the unit test ends. +void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + FILE* xmlout = NULL; + FilePath output_file(output_file_); + FilePath output_dir(output_file.RemoveFileName()); + + if (output_dir.CreateDirectoriesRecursively()) { + xmlout = posix::FOpen(output_file_.c_str(), "w"); + } + if (xmlout == NULL) { + // TODO(wan): report the reason of the failure. + // + // We don't do it for now as: + // + // 1. There is no urgent need for it. + // 2. It's a bit involved to make the errno variable thread-safe on + // all three operating systems (Linux, Windows, and Mac OS). + // 3. To interpret the meaning of errno in a thread-safe way, + // we need the strerror_r() function, which is not available on + // Windows. + fprintf(stderr, + "Unable to open file \"%s\"\n", + output_file_.c_str()); + fflush(stderr); + exit(EXIT_FAILURE); + } + std::stringstream stream; + PrintXmlUnitTest(&stream, unit_test); + fprintf(xmlout, "%s", StringStreamToString(&stream).c_str()); + fclose(xmlout); +} + +// Returns an XML-escaped copy of the input string str. If is_attribute +// is true, the text is meant to appear as an attribute value, and +// normalizable whitespace is preserved by replacing it with character +// references. +// +// Invalid XML characters in str, if any, are stripped from the output. +// It is expected that most, if not all, of the text processed by this +// module will consist of ordinary English text. +// If this module is ever modified to produce version 1.1 XML output, +// most invalid characters can be retained using character references. +// TODO(wan): It might be nice to have a minimally invasive, human-readable +// escaping scheme for invalid characters, rather than dropping them. +std::string XmlUnitTestResultPrinter::EscapeXml( + const std::string& str, bool is_attribute) { + Message m; + + for (size_t i = 0; i < str.size(); ++i) { + const char ch = str[i]; + switch (ch) { + case '<': + m << "<"; + break; + case '>': + m << ">"; + break; + case '&': + m << "&"; + break; + case '\'': + if (is_attribute) + m << "'"; + else + m << '\''; + break; + case '"': + if (is_attribute) + m << """; + else + m << '"'; + break; + default: + if (IsValidXmlCharacter(ch)) { + if (is_attribute && IsNormalizableWhitespace(ch)) + m << "&#x" << String::FormatByte(static_cast(ch)) + << ";"; + else + m << ch; + } + break; + } + } + + return m.GetString(); +} + +// Returns the given string with all characters invalid in XML removed. +// Currently invalid characters are dropped from the string. An +// alternative is to replace them with certain characters such as . or ?. +std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters( + const std::string& str) { + std::string output; + output.reserve(str.size()); + for (std::string::const_iterator it = str.begin(); it != str.end(); ++it) + if (IsValidXmlCharacter(*it)) + output.push_back(*it); + + return output; +} + +// The following routines generate an XML representation of a UnitTest +// object. +// +// This is how Google Test concepts map to the DTD: +// +// <-- corresponds to a UnitTest object +// <-- corresponds to a TestCase object +// <-- corresponds to a TestInfo object +// ... +// ... +// ... +// <-- individual assertion failures +// +// +// + +// Formats the given time in milliseconds as seconds. +std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { + ::std::stringstream ss; + ss << (static_cast(ms) * 1e-3); + return ss.str(); +} + +static bool PortableLocaltime(time_t seconds, struct tm* out) { +#if defined(_MSC_VER) + return localtime_s(out, &seconds) == 0; +#elif defined(__MINGW32__) || defined(__MINGW64__) + // MINGW provides neither localtime_r nor localtime_s, but uses + // Windows' localtime(), which has a thread-local tm buffer. + struct tm* tm_ptr = localtime(&seconds); // NOLINT + if (tm_ptr == NULL) + return false; + *out = *tm_ptr; + return true; +#else + return localtime_r(&seconds, out) != NULL; +#endif +} + +// Converts the given epoch time in milliseconds to a date string in the ISO +// 8601 format, without the timezone information. +std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) { + struct tm time_struct; + if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) + return ""; + // YYYY-MM-DDThh:mm:ss + return StreamableToString(time_struct.tm_year + 1900) + "-" + + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + + String::FormatIntWidth2(time_struct.tm_mday) + "T" + + String::FormatIntWidth2(time_struct.tm_hour) + ":" + + String::FormatIntWidth2(time_struct.tm_min) + ":" + + String::FormatIntWidth2(time_struct.tm_sec); +} + +// Streams an XML CDATA section, escaping invalid CDATA sequences as needed. +void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, + const char* data) { + const char* segment = data; + *stream << ""); + if (next_segment != NULL) { + stream->write( + segment, static_cast(next_segment - segment)); + *stream << "]]>]]>"); + } else { + *stream << segment; + break; + } + } + *stream << "]]>"; +} + +void XmlUnitTestResultPrinter::OutputXmlAttribute( + std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value) { + const std::vector& allowed_names = + GetReservedAttributesForElement(element_name); + + GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != + allowed_names.end()) + << "Attribute " << name << " is not allowed for element <" << element_name + << ">."; + + *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\""; +} + +// Prints an XML representation of a TestInfo object. +// TODO(wan): There is also value in printing properties with the plain printer. +void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info) { + const TestResult& result = *test_info.result(); + const std::string kTestcase = "testcase"; + + *stream << " \n"; + } + const string location = internal::FormatCompilerIndependentFileLocation( + part.file_name(), part.line_number()); + const string summary = location + "\n" + part.summary(); + *stream << " "; + const string detail = location + "\n" + part.message(); + OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); + *stream << "\n"; + } + } + + if (failures == 0) + *stream << " />\n"; + else + *stream << " \n"; +} + +// Prints an XML representation of a TestCase object +void XmlUnitTestResultPrinter::PrintXmlTestCase(std::ostream* stream, + const TestCase& test_case) { + const std::string kTestsuite = "testsuite"; + *stream << " <" << kTestsuite; + OutputXmlAttribute(stream, kTestsuite, "name", test_case.name()); + OutputXmlAttribute(stream, kTestsuite, "tests", + StreamableToString(test_case.reportable_test_count())); + OutputXmlAttribute(stream, kTestsuite, "failures", + StreamableToString(test_case.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuite, "disabled", + StreamableToString(test_case.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuite, "errors", "0"); + OutputXmlAttribute(stream, kTestsuite, "time", + FormatTimeInMillisAsSeconds(test_case.elapsed_time())); + *stream << TestPropertiesAsXmlAttributes(test_case.ad_hoc_test_result()) + << ">\n"; + + for (int i = 0; i < test_case.total_test_count(); ++i) { + if (test_case.GetTestInfo(i)->is_reportable()) + OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i)); + } + *stream << " \n"; +} + +// Prints an XML summary of unit_test to output stream out. +void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream, + const UnitTest& unit_test) { + const std::string kTestsuites = "testsuites"; + + *stream << "\n"; + *stream << "<" << kTestsuites; + + OutputXmlAttribute(stream, kTestsuites, "tests", + StreamableToString(unit_test.reportable_test_count())); + OutputXmlAttribute(stream, kTestsuites, "failures", + StreamableToString(unit_test.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuites, "disabled", + StreamableToString(unit_test.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuites, "errors", "0"); + OutputXmlAttribute( + stream, kTestsuites, "timestamp", + FormatEpochTimeInMillisAsIso8601(unit_test.start_timestamp())); + OutputXmlAttribute(stream, kTestsuites, "time", + FormatTimeInMillisAsSeconds(unit_test.elapsed_time())); + + if (GTEST_FLAG(shuffle)) { + OutputXmlAttribute(stream, kTestsuites, "random_seed", + StreamableToString(unit_test.random_seed())); + } + + *stream << TestPropertiesAsXmlAttributes(unit_test.ad_hoc_test_result()); + + OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); + *stream << ">\n"; + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + if (unit_test.GetTestCase(i)->reportable_test_count() > 0) + PrintXmlTestCase(stream, *unit_test.GetTestCase(i)); + } + *stream << "\n"; +} + +// Produces a string representing the test properties in a result as space +// delimited XML attributes based on the property key="value" pairs. +std::string XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( + const TestResult& result) { + Message attributes; + for (int i = 0; i < result.test_property_count(); ++i) { + const TestProperty& property = result.GetTestProperty(i); + attributes << " " << property.key() << "=" + << "\"" << EscapeXmlAttribute(property.value()) << "\""; + } + return attributes.GetString(); +} + +// End XmlUnitTestResultPrinter + +#if GTEST_CAN_STREAM_RESULTS_ + +// Checks if str contains '=', '&', '%' or '\n' characters. If yes, +// replaces them by "%xx" where xx is their hexadecimal value. For +// example, replaces "=" with "%3D". This algorithm is O(strlen(str)) +// in both time and space -- important as the input str may contain an +// arbitrarily long test failure message and stack trace. +string StreamingListener::UrlEncode(const char* str) { + string result; + result.reserve(strlen(str) + 1); + for (char ch = *str; ch != '\0'; ch = *++str) { + switch (ch) { + case '%': + case '=': + case '&': + case '\n': + result.append("%" + String::FormatByte(static_cast(ch))); + break; + default: + result.push_back(ch); + break; + } + } + return result; +} + +void StreamingListener::SocketWriter::MakeConnection() { + GTEST_CHECK_(sockfd_ == -1) + << "MakeConnection() can't be called when there is already a connection."; + + addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; // To allow both IPv4 and IPv6 addresses. + hints.ai_socktype = SOCK_STREAM; + addrinfo* servinfo = NULL; + + // Use the getaddrinfo() to get a linked list of IP addresses for + // the given host name. + const int error_num = getaddrinfo( + host_name_.c_str(), port_num_.c_str(), &hints, &servinfo); + if (error_num != 0) { + GTEST_LOG_(WARNING) << "stream_result_to: getaddrinfo() failed: " + << gai_strerror(error_num); + } + + // Loop through all the results and connect to the first we can. + for (addrinfo* cur_addr = servinfo; sockfd_ == -1 && cur_addr != NULL; + cur_addr = cur_addr->ai_next) { + sockfd_ = socket( + cur_addr->ai_family, cur_addr->ai_socktype, cur_addr->ai_protocol); + if (sockfd_ != -1) { + // Connect the client socket to the server socket. + if (connect(sockfd_, cur_addr->ai_addr, cur_addr->ai_addrlen) == -1) { + close(sockfd_); + sockfd_ = -1; + } + } + } + + freeaddrinfo(servinfo); // all done with this structure + + if (sockfd_ == -1) { + GTEST_LOG_(WARNING) << "stream_result_to: failed to connect to " + << host_name_ << ":" << port_num_; + } +} + +// End of class Streaming Listener +#endif // GTEST_CAN_STREAM_RESULTS__ + +// Class ScopedTrace + +// Pushes the given source file location and message onto a per-thread +// trace stack maintained by Google Test. +ScopedTrace::ScopedTrace(const char* file, int line, const Message& message) + GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { + TraceInfo trace; + trace.file = file; + trace.line = line; + trace.message = message.GetString(); + + UnitTest::GetInstance()->PushGTestTrace(trace); +} + +// Pops the info pushed by the c'tor. +ScopedTrace::~ScopedTrace() + GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { + UnitTest::GetInstance()->PopGTestTrace(); +} + + +// class OsStackTraceGetter + +const char* const OsStackTraceGetterInterface::kElidedFramesMarker = + "... " GTEST_NAME_ " internal frames ..."; + +string OsStackTraceGetter::CurrentStackTrace(int /*max_depth*/, + int /*skip_count*/) { + return ""; +} + +void OsStackTraceGetter::UponLeavingGTest() {} + +// A helper class that creates the premature-exit file in its +// constructor and deletes the file in its destructor. +class ScopedPrematureExitFile { + public: + explicit ScopedPrematureExitFile(const char* premature_exit_filepath) + : premature_exit_filepath_(premature_exit_filepath) { + // If a path to the premature-exit file is specified... + if (premature_exit_filepath != NULL && *premature_exit_filepath != '\0') { + // create the file with a single "0" character in it. I/O + // errors are ignored as there's nothing better we can do and we + // don't want to fail the test because of this. + FILE* pfile = posix::FOpen(premature_exit_filepath, "w"); + fwrite("0", 1, 1, pfile); + fclose(pfile); + } + } + + ~ScopedPrematureExitFile() { + if (premature_exit_filepath_ != NULL && *premature_exit_filepath_ != '\0') { + remove(premature_exit_filepath_); + } + } + + private: + const char* const premature_exit_filepath_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedPrematureExitFile); +}; + +} // namespace internal + +// class TestEventListeners + +TestEventListeners::TestEventListeners() + : repeater_(new internal::TestEventRepeater()), + default_result_printer_(NULL), + default_xml_generator_(NULL) { +} + +TestEventListeners::~TestEventListeners() { delete repeater_; } + +// Returns the standard listener responsible for the default console +// output. Can be removed from the listeners list to shut down default +// console output. Note that removing this object from the listener list +// with Release transfers its ownership to the user. +void TestEventListeners::Append(TestEventListener* listener) { + repeater_->Append(listener); +} + +// Removes the given event listener from the list and returns it. It then +// becomes the caller's responsibility to delete the listener. Returns +// NULL if the listener is not found in the list. +TestEventListener* TestEventListeners::Release(TestEventListener* listener) { + if (listener == default_result_printer_) + default_result_printer_ = NULL; + else if (listener == default_xml_generator_) + default_xml_generator_ = NULL; + return repeater_->Release(listener); +} + +// Returns repeater that broadcasts the TestEventListener events to all +// subscribers. +TestEventListener* TestEventListeners::repeater() { return repeater_; } + +// Sets the default_result_printer attribute to the provided listener. +// The listener is also added to the listener list and previous +// default_result_printer is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultResultPrinter(TestEventListener* listener) { + if (default_result_printer_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_result_printer_); + default_result_printer_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Sets the default_xml_generator attribute to the provided listener. The +// listener is also added to the listener list and previous +// default_xml_generator is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultXmlGenerator(TestEventListener* listener) { + if (default_xml_generator_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_xml_generator_); + default_xml_generator_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Controls whether events will be forwarded by the repeater to the +// listeners in the list. +bool TestEventListeners::EventForwardingEnabled() const { + return repeater_->forwarding_enabled(); +} + +void TestEventListeners::SuppressEventForwarding() { + repeater_->set_forwarding_enabled(false); +} + +// class UnitTest + +// Gets the singleton UnitTest object. The first time this method is +// called, a UnitTest object is constructed and returned. Consecutive +// calls will return the same object. +// +// We don't protect this under mutex_ as a user is not supposed to +// call this before main() starts, from which point on the return +// value will never change. +UnitTest* UnitTest::GetInstance() { + // When compiled with MSVC 7.1 in optimized mode, destroying the + // UnitTest object upon exiting the program messes up the exit code, + // causing successful tests to appear failed. We have to use a + // different implementation in this case to bypass the compiler bug. + // This implementation makes the compiler happy, at the cost of + // leaking the UnitTest object. + + // CodeGear C++Builder insists on a public destructor for the + // default implementation. Use this implementation to keep good OO + // design with private destructor. + +#if (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) + static UnitTest* const instance = new UnitTest; + return instance; +#else + static UnitTest instance; + return &instance; +#endif // (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) +} + +// Gets the number of successful test cases. +int UnitTest::successful_test_case_count() const { + return impl()->successful_test_case_count(); +} + +// Gets the number of failed test cases. +int UnitTest::failed_test_case_count() const { + return impl()->failed_test_case_count(); +} + +// Gets the number of all test cases. +int UnitTest::total_test_case_count() const { + return impl()->total_test_case_count(); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTest::test_case_to_run_count() const { + return impl()->test_case_to_run_count(); +} + +// Gets the number of successful tests. +int UnitTest::successful_test_count() const { + return impl()->successful_test_count(); +} + +// Gets the number of failed tests. +int UnitTest::failed_test_count() const { return impl()->failed_test_count(); } + +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTest::reportable_disabled_test_count() const { + return impl()->reportable_disabled_test_count(); +} + +// Gets the number of disabled tests. +int UnitTest::disabled_test_count() const { + return impl()->disabled_test_count(); +} + +// Gets the number of tests to be printed in the XML report. +int UnitTest::reportable_test_count() const { + return impl()->reportable_test_count(); +} + +// Gets the number of all tests. +int UnitTest::total_test_count() const { return impl()->total_test_count(); } + +// Gets the number of tests that should run. +int UnitTest::test_to_run_count() const { return impl()->test_to_run_count(); } + +// Gets the time of the test program start, in ms from the start of the +// UNIX epoch. +internal::TimeInMillis UnitTest::start_timestamp() const { + return impl()->start_timestamp(); +} + +// Gets the elapsed time, in milliseconds. +internal::TimeInMillis UnitTest::elapsed_time() const { + return impl()->elapsed_time(); +} + +// Returns true iff the unit test passed (i.e. all test cases passed). +bool UnitTest::Passed() const { return impl()->Passed(); } + +// Returns true iff the unit test failed (i.e. some test case failed +// or something outside of all tests failed). +bool UnitTest::Failed() const { return impl()->Failed(); } + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +const TestCase* UnitTest::GetTestCase(int i) const { + return impl()->GetTestCase(i); +} + +// Returns the TestResult containing information on test failures and +// properties logged outside of individual test cases. +const TestResult& UnitTest::ad_hoc_test_result() const { + return *impl()->ad_hoc_test_result(); +} + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +TestCase* UnitTest::GetMutableTestCase(int i) { + return impl()->GetMutableTestCase(i); +} + +// Returns the list of event listeners that can be used to track events +// inside Google Test. +TestEventListeners& UnitTest::listeners() { + return *impl()->listeners(); +} + +// Registers and returns a global test environment. When a test +// program is run, all global test environments will be set-up in the +// order they were registered. After all tests in the program have +// finished, all global test environments will be torn-down in the +// *reverse* order they were registered. +// +// The UnitTest object takes ownership of the given environment. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +Environment* UnitTest::AddEnvironment(Environment* env) { + if (env == NULL) { + return NULL; + } + + impl_->environments().push_back(env); + return env; +} + +// Adds a TestPartResult to the current TestResult object. All Google Test +// assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) eventually call +// this to report their results. The user code should use the +// assertion macros instead of calling this directly. +void UnitTest::AddTestPartResult( + TestPartResult::Type result_type, + const char* file_name, + int line_number, + const std::string& message, + const std::string& os_stack_trace) GTEST_LOCK_EXCLUDED_(mutex_) { + Message msg; + msg << message; + + internal::MutexLock lock(&mutex_); + if (impl_->gtest_trace_stack().size() > 0) { + msg << "\n" << GTEST_NAME_ << " trace:"; + + for (int i = static_cast(impl_->gtest_trace_stack().size()); + i > 0; --i) { + const internal::TraceInfo& trace = impl_->gtest_trace_stack()[i - 1]; + msg << "\n" << internal::FormatFileLocation(trace.file, trace.line) + << " " << trace.message; + } + } + + if (os_stack_trace.c_str() != NULL && !os_stack_trace.empty()) { + msg << internal::kStackTraceMarker << os_stack_trace; + } + + const TestPartResult result = + TestPartResult(result_type, file_name, line_number, + msg.GetString().c_str()); + impl_->GetTestPartResultReporterForCurrentThread()-> + ReportTestPartResult(result); + + if (result_type != TestPartResult::kSuccess) { + // gtest_break_on_failure takes precedence over + // gtest_throw_on_failure. This allows a user to set the latter + // in the code (perhaps in order to use Google Test assertions + // with another testing framework) and specify the former on the + // command line for debugging. + if (GTEST_FLAG(break_on_failure)) { +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + // Using DebugBreak on Windows allows gtest to still break into a debugger + // when a failure happens and both the --gtest_break_on_failure and + // the --gtest_catch_exceptions flags are specified. + DebugBreak(); +#else + // Dereference NULL through a volatile pointer to prevent the compiler + // from removing. We use this rather than abort() or __builtin_trap() for + // portability: Symbian doesn't implement abort() well, and some debuggers + // don't correctly trap abort(). + *static_cast(NULL) = 1; +#endif // GTEST_OS_WINDOWS + } else if (GTEST_FLAG(throw_on_failure)) { +#if GTEST_HAS_EXCEPTIONS + throw internal::GoogleTestFailureException(result); +#else + // We cannot call abort() as it generates a pop-up in debug mode + // that cannot be suppressed in VC 7.1 or below. + exit(1); +#endif + } + } +} + +// Adds a TestProperty to the current TestResult object when invoked from +// inside a test, to current TestCase's ad_hoc_test_result_ when invoked +// from SetUpTestCase or TearDownTestCase, or to the global property set +// when invoked elsewhere. If the result already contains a property with +// the same key, the value will be updated. +void UnitTest::RecordProperty(const std::string& key, + const std::string& value) { + impl_->RecordProperty(TestProperty(key, value)); +} + +// Runs all tests in this UnitTest object and prints the result. +// Returns 0 if successful, or 1 otherwise. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +int UnitTest::Run() { + const bool in_death_test_child_process = + internal::GTEST_FLAG(internal_run_death_test).length() > 0; + + // Google Test implements this protocol for catching that a test + // program exits before returning control to Google Test: + // + // 1. Upon start, Google Test creates a file whose absolute path + // is specified by the environment variable + // TEST_PREMATURE_EXIT_FILE. + // 2. When Google Test has finished its work, it deletes the file. + // + // This allows a test runner to set TEST_PREMATURE_EXIT_FILE before + // running a Google-Test-based test program and check the existence + // of the file at the end of the test execution to see if it has + // exited prematurely. + + // If we are in the child process of a death test, don't + // create/delete the premature exit file, as doing so is unnecessary + // and will confuse the parent process. Otherwise, create/delete + // the file upon entering/leaving this function. If the program + // somehow exits before this function has a chance to return, the + // premature-exit file will be left undeleted, causing a test runner + // that understands the premature-exit-file protocol to report the + // test as having failed. + const internal::ScopedPrematureExitFile premature_exit_file( + in_death_test_child_process ? + NULL : internal::posix::GetEnv("TEST_PREMATURE_EXIT_FILE")); + + // Captures the value of GTEST_FLAG(catch_exceptions). This value will be + // used for the duration of the program. + impl()->set_catch_exceptions(GTEST_FLAG(catch_exceptions)); + +#if GTEST_HAS_SEH + // Either the user wants Google Test to catch exceptions thrown by the + // tests or this is executing in the context of death test child + // process. In either case the user does not want to see pop-up dialogs + // about crashes - they are expected. + if (impl()->catch_exceptions() || in_death_test_child_process) { +# if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + // SetErrorMode doesn't exist on CE. + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT | + SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); +# endif // !GTEST_OS_WINDOWS_MOBILE + +# if (defined(_MSC_VER) || GTEST_OS_WINDOWS_MINGW) && !GTEST_OS_WINDOWS_MOBILE + // Death test children can be terminated with _abort(). On Windows, + // _abort() can show a dialog with a warning message. This forces the + // abort message to go to stderr instead. + _set_error_mode(_OUT_TO_STDERR); +# endif + +# if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE + // In the debug version, Visual Studio pops up a separate dialog + // offering a choice to debug the aborted program. We need to suppress + // this dialog or it will pop up for every EXPECT/ASSERT_DEATH statement + // executed. Google Test will notify the user of any unexpected + // failure via stderr. + // + // VC++ doesn't define _set_abort_behavior() prior to the version 8.0. + // Users of prior VC versions shall suffer the agony and pain of + // clicking through the countless debug dialogs. + // TODO(vladl@google.com): find a way to suppress the abort dialog() in the + // debug mode when compiled with VC 7.1 or lower. + if (!GTEST_FLAG(break_on_failure)) + _set_abort_behavior( + 0x0, // Clear the following flags: + _WRITE_ABORT_MSG | _CALL_REPORTFAULT); // pop-up window, core dump. +# endif + } +#endif // GTEST_HAS_SEH + + return internal::HandleExceptionsInMethodIfSupported( + impl(), + &internal::UnitTestImpl::RunAllTests, + "auxiliary test code (environments or event listeners)") ? 0 : 1; +} + +// Returns the working directory when the first TEST() or TEST_F() was +// executed. +const char* UnitTest::original_working_dir() const { + return impl_->original_working_dir_.c_str(); +} + +// Returns the TestCase object for the test that's currently running, +// or NULL if no test is running. +const TestCase* UnitTest::current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + return impl_->current_test_case(); +} + +// Returns the TestInfo object for the test that's currently running, +// or NULL if no test is running. +const TestInfo* UnitTest::current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + return impl_->current_test_info(); +} + +// Returns the random seed used at the start of the current test run. +int UnitTest::random_seed() const { return impl_->random_seed(); } + +#if GTEST_HAS_PARAM_TEST +// Returns ParameterizedTestCaseRegistry object used to keep track of +// value-parameterized tests and instantiate and register them. +internal::ParameterizedTestCaseRegistry& + UnitTest::parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_) { + return impl_->parameterized_test_registry(); +} +#endif // GTEST_HAS_PARAM_TEST + +// Creates an empty UnitTest. +UnitTest::UnitTest() { + impl_ = new internal::UnitTestImpl(this); +} + +// Destructor of UnitTest. +UnitTest::~UnitTest() { + delete impl_; +} + +// Pushes a trace defined by SCOPED_TRACE() on to the per-thread +// Google Test trace stack. +void UnitTest::PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().push_back(trace); +} + +// Pops a trace from the per-thread Google Test trace stack. +void UnitTest::PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().pop_back(); +} + +namespace internal { + +UnitTestImpl::UnitTestImpl(UnitTest* parent) + : parent_(parent), + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4355 /* using this in initializer */) + default_global_test_part_result_reporter_(this), + default_per_thread_test_part_result_reporter_(this), + GTEST_DISABLE_MSC_WARNINGS_POP_() + global_test_part_result_repoter_( + &default_global_test_part_result_reporter_), + per_thread_test_part_result_reporter_( + &default_per_thread_test_part_result_reporter_), +#if GTEST_HAS_PARAM_TEST + parameterized_test_registry_(), + parameterized_tests_registered_(false), +#endif // GTEST_HAS_PARAM_TEST + last_death_test_case_(-1), + current_test_case_(NULL), + current_test_info_(NULL), + ad_hoc_test_result_(), + os_stack_trace_getter_(NULL), + post_flag_parse_init_performed_(false), + random_seed_(0), // Will be overridden by the flag before first use. + random_(0), // Will be reseeded before first use. + start_timestamp_(0), + elapsed_time_(0), +#if GTEST_HAS_DEATH_TEST + death_test_factory_(new DefaultDeathTestFactory), +#endif + // Will be overridden by the flag before first use. + catch_exceptions_(false) { + listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter); +} + +UnitTestImpl::~UnitTestImpl() { + // Deletes every TestCase. + ForEach(test_cases_, internal::Delete); + + // Deletes every Environment. + ForEach(environments_, internal::Delete); + + delete os_stack_trace_getter_; +} + +// Adds a TestProperty to the current TestResult object when invoked in a +// context of a test, to current test case's ad_hoc_test_result when invoke +// from SetUpTestCase/TearDownTestCase, or to the global property set +// otherwise. If the result already contains a property with the same key, +// the value will be updated. +void UnitTestImpl::RecordProperty(const TestProperty& test_property) { + std::string xml_element; + TestResult* test_result; // TestResult appropriate for property recording. + + if (current_test_info_ != NULL) { + xml_element = "testcase"; + test_result = &(current_test_info_->result_); + } else if (current_test_case_ != NULL) { + xml_element = "testsuite"; + test_result = &(current_test_case_->ad_hoc_test_result_); + } else { + xml_element = "testsuites"; + test_result = &ad_hoc_test_result_; + } + test_result->RecordProperty(xml_element, test_property); +} + +#if GTEST_HAS_DEATH_TEST +// Disables event forwarding if the control is currently in a death test +// subprocess. Must not be called before InitGoogleTest. +void UnitTestImpl::SuppressTestEventsIfInSubprocess() { + if (internal_run_death_test_flag_.get() != NULL) + listeners()->SuppressEventForwarding(); +} +#endif // GTEST_HAS_DEATH_TEST + +// Initializes event listeners performing XML output as specified by +// UnitTestOptions. Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureXmlOutput() { + const std::string& output_format = UnitTestOptions::GetOutputFormat(); + if (output_format == "xml") { + listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); + } else if (output_format != "") { + printf("WARNING: unrecognized output format \"%s\" ignored.\n", + output_format.c_str()); + fflush(stdout); + } +} + +#if GTEST_CAN_STREAM_RESULTS_ +// Initializes event listeners for streaming test results in string form. +// Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureStreamingOutput() { + const std::string& target = GTEST_FLAG(stream_result_to); + if (!target.empty()) { + const size_t pos = target.find(':'); + if (pos != std::string::npos) { + listeners()->Append(new StreamingListener(target.substr(0, pos), + target.substr(pos+1))); + } else { + printf("WARNING: unrecognized streaming target \"%s\" ignored.\n", + target.c_str()); + fflush(stdout); + } + } +} +#endif // GTEST_CAN_STREAM_RESULTS_ + +// Performs initialization dependent upon flag values obtained in +// ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to +// ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest +// this function is also called from RunAllTests. Since this function can be +// called more than once, it has to be idempotent. +void UnitTestImpl::PostFlagParsingInit() { + // Ensures that this function does not execute more than once. + if (!post_flag_parse_init_performed_) { + post_flag_parse_init_performed_ = true; + +#if defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) + // Register to send notifications about key process state changes. + listeners()->Append(new GTEST_CUSTOM_TEST_EVENT_LISTENER_()); +#endif // defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) + +#if GTEST_HAS_DEATH_TEST + InitDeathTestSubprocessControlInfo(); + SuppressTestEventsIfInSubprocess(); +#endif // GTEST_HAS_DEATH_TEST + + // Registers parameterized tests. This makes parameterized tests + // available to the UnitTest reflection API without running + // RUN_ALL_TESTS. + RegisterParameterizedTests(); + + // Configures listeners for XML output. This makes it possible for users + // to shut down the default XML output before invoking RUN_ALL_TESTS. + ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Configures listeners for streaming test results to the specified server. + ConfigureStreamingOutput(); +#endif // GTEST_CAN_STREAM_RESULTS_ + } +} + +// A predicate that checks the name of a TestCase against a known +// value. +// +// This is used for implementation of the UnitTest class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestCaseNameIs is copyable. +class TestCaseNameIs { + public: + // Constructor. + explicit TestCaseNameIs(const std::string& name) + : name_(name) {} + + // Returns true iff the name of test_case matches name_. + bool operator()(const TestCase* test_case) const { + return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; + } + + private: + std::string name_; +}; + +// Finds and returns a TestCase with the given name. If one doesn't +// exist, creates one and returns it. It's the CALLER'S +// RESPONSIBILITY to ensure that this function is only called WHEN THE +// TESTS ARE NOT SHUFFLED. +// +// Arguments: +// +// test_case_name: name of the test case +// type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase* UnitTestImpl::GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) { + // Can we find a TestCase with the given name? + const std::vector::const_iterator test_case = + std::find_if(test_cases_.begin(), test_cases_.end(), + TestCaseNameIs(test_case_name)); + + if (test_case != test_cases_.end()) + return *test_case; + + // No. Let's create one. + TestCase* const new_test_case = + new TestCase(test_case_name, type_param, set_up_tc, tear_down_tc); + + // Is this a death test case? + if (internal::UnitTestOptions::MatchesFilter(test_case_name, + kDeathTestCaseFilter)) { + // Yes. Inserts the test case after the last death test case + // defined so far. This only works when the test cases haven't + // been shuffled. Otherwise we may end up running a death test + // after a non-death test. + ++last_death_test_case_; + test_cases_.insert(test_cases_.begin() + last_death_test_case_, + new_test_case); + } else { + // No. Appends to the end of the list. + test_cases_.push_back(new_test_case); + } + + test_case_indices_.push_back(static_cast(test_case_indices_.size())); + return new_test_case; +} + +// Helpers for setting up / tearing down the given environment. They +// are for use in the ForEach() function. +static void SetUpEnvironment(Environment* env) { env->SetUp(); } +static void TearDownEnvironment(Environment* env) { env->TearDown(); } + +// Runs all tests in this UnitTest object, prints the result, and +// returns true if all tests are successful. If any exception is +// thrown during a test, the test is considered to be failed, but the +// rest of the tests will still be run. +// +// When parameterized tests are enabled, it expands and registers +// parameterized tests first in RegisterParameterizedTests(). +// All other functions called from RunAllTests() may safely assume that +// parameterized tests are ready to be counted and run. +bool UnitTestImpl::RunAllTests() { + // Makes sure InitGoogleTest() was called. + if (!GTestIsInitialized()) { + printf("%s", + "\nThis test program did NOT call ::testing::InitGoogleTest " + "before calling RUN_ALL_TESTS(). Please fix it.\n"); + return false; + } + + // Do not run any test if the --help flag was specified. + if (g_help_flag) + return true; + + // Repeats the call to the post-flag parsing initialization in case the + // user didn't call InitGoogleTest. + PostFlagParsingInit(); + + // Even if sharding is not on, test runners may want to use the + // GTEST_SHARD_STATUS_FILE to query whether the test supports the sharding + // protocol. + internal::WriteToShardStatusFileIfNeeded(); + + // True iff we are in a subprocess for running a thread-safe-style + // death test. + bool in_subprocess_for_death_test = false; + +#if GTEST_HAS_DEATH_TEST + in_subprocess_for_death_test = (internal_run_death_test_flag_.get() != NULL); +# if defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) + if (in_subprocess_for_death_test) { + GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_(); + } +# endif // defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) +#endif // GTEST_HAS_DEATH_TEST + + const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex, + in_subprocess_for_death_test); + + // Compares the full test names with the filter to decide which + // tests to run. + const bool has_tests_to_run = FilterTests(should_shard + ? HONOR_SHARDING_PROTOCOL + : IGNORE_SHARDING_PROTOCOL) > 0; + + // Lists the tests and exits if the --gtest_list_tests flag was specified. + if (GTEST_FLAG(list_tests)) { + // This must be called *after* FilterTests() has been called. + ListTestsMatchingFilter(); + return true; + } + + random_seed_ = GTEST_FLAG(shuffle) ? + GetRandomSeedFromFlag(GTEST_FLAG(random_seed)) : 0; + + // True iff at least one test has failed. + bool failed = false; + + TestEventListener* repeater = listeners()->repeater(); + + start_timestamp_ = GetTimeInMillis(); + repeater->OnTestProgramStart(*parent_); + + // How many times to repeat the tests? We don't want to repeat them + // when we are inside the subprocess of a death test. + const int repeat = in_subprocess_for_death_test ? 1 : GTEST_FLAG(repeat); + // Repeats forever if the repeat count is negative. + const bool forever = repeat < 0; + for (int i = 0; forever || i != repeat; i++) { + // We want to preserve failures generated by ad-hoc test + // assertions executed before RUN_ALL_TESTS(). + ClearNonAdHocTestResult(); + + const TimeInMillis start = GetTimeInMillis(); + + // Shuffles test cases and tests if requested. + if (has_tests_to_run && GTEST_FLAG(shuffle)) { + random()->Reseed(random_seed_); + // This should be done before calling OnTestIterationStart(), + // such that a test event listener can see the actual test order + // in the event. + ShuffleTests(); + } + + // Tells the unit test event listeners that the tests are about to start. + repeater->OnTestIterationStart(*parent_, i); + + // Runs each test case if there is at least one test to run. + if (has_tests_to_run) { + // Sets up all environments beforehand. + repeater->OnEnvironmentsSetUpStart(*parent_); + ForEach(environments_, SetUpEnvironment); + repeater->OnEnvironmentsSetUpEnd(*parent_); + + // Runs the tests only if there was no fatal failure during global + // set-up. + if (!Test::HasFatalFailure()) { + for (int test_index = 0; test_index < total_test_case_count(); + test_index++) { + GetMutableTestCase(test_index)->Run(); + } + } + + // Tears down all environments in reverse order afterwards. + repeater->OnEnvironmentsTearDownStart(*parent_); + std::for_each(environments_.rbegin(), environments_.rend(), + TearDownEnvironment); + repeater->OnEnvironmentsTearDownEnd(*parent_); + } + + elapsed_time_ = GetTimeInMillis() - start; + + // Tells the unit test event listener that the tests have just finished. + repeater->OnTestIterationEnd(*parent_, i); + + // Gets the result and clears it. + if (!Passed()) { + failed = true; + } + + // Restores the original test order after the iteration. This + // allows the user to quickly repro a failure that happens in the + // N-th iteration without repeating the first (N - 1) iterations. + // This is not enclosed in "if (GTEST_FLAG(shuffle)) { ... }", in + // case the user somehow changes the value of the flag somewhere + // (it's always safe to unshuffle the tests). + UnshuffleTests(); + + if (GTEST_FLAG(shuffle)) { + // Picks a new random seed for each iteration. + random_seed_ = GetNextRandomSeed(random_seed_); + } + } + + repeater->OnTestProgramEnd(*parent_); + + return !failed; +} + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded() { + const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile); + if (test_shard_file != NULL) { + FILE* const file = posix::FOpen(test_shard_file, "w"); + if (file == NULL) { + ColoredPrintf(COLOR_RED, + "Could not write to the test shard status file \"%s\" " + "specified by the %s environment variable.\n", + test_shard_file, kTestShardStatusFile); + fflush(stdout); + exit(EXIT_FAILURE); + } + fclose(file); + } +} + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (i.e., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +bool ShouldShard(const char* total_shards_env, + const char* shard_index_env, + bool in_subprocess_for_death_test) { + if (in_subprocess_for_death_test) { + return false; + } + + const Int32 total_shards = Int32FromEnvOrDie(total_shards_env, -1); + const Int32 shard_index = Int32FromEnvOrDie(shard_index_env, -1); + + if (total_shards == -1 && shard_index == -1) { + return false; + } else if (total_shards == -1 && shard_index != -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestShardIndex << " = " << shard_index + << ", but have left " << kTestTotalShards << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (total_shards != -1 && shard_index == -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestTotalShards << " = " << total_shards + << ", but have left " << kTestShardIndex << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (shard_index < 0 || shard_index >= total_shards) { + const Message msg = Message() + << "Invalid environment variables: we require 0 <= " + << kTestShardIndex << " < " << kTestTotalShards + << ", but you have " << kTestShardIndex << "=" << shard_index + << ", " << kTestTotalShards << "=" << total_shards << ".\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } + + return total_shards > 1; +} + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error +// and aborts. +Int32 Int32FromEnvOrDie(const char* var, Int32 default_val) { + const char* str_val = posix::GetEnv(var); + if (str_val == NULL) { + return default_val; + } + + Int32 result; + if (!ParseInt32(Message() << "The value of environment variable " << var, + str_val, &result)) { + exit(EXIT_FAILURE); + } + return result; +} + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) { + return (test_id % total_shards) == shard_index; +} + +// Compares the name of each test with the user-specified filter to +// decide whether the test should be run, then records the result in +// each TestCase and TestInfo object. +// If shard_tests == true, further filters tests based on sharding +// variables in the environment - see +// http://code.google.com/p/googletest/wiki/GoogleTestAdvancedGuide. +// Returns the number of tests that should run. +int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { + const Int32 total_shards = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestTotalShards, -1) : -1; + const Int32 shard_index = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestShardIndex, -1) : -1; + + // num_runnable_tests are the number of tests that will + // run across all shards (i.e., match filter and are not disabled). + // num_selected_tests are the number of tests to be run on + // this shard. + int num_runnable_tests = 0; + int num_selected_tests = 0; + for (size_t i = 0; i < test_cases_.size(); i++) { + TestCase* const test_case = test_cases_[i]; + const std::string &test_case_name = test_case->name(); + test_case->set_should_run(false); + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + TestInfo* const test_info = test_case->test_info_list()[j]; + const std::string test_name(test_info->name()); + // A test is disabled if test case name or test name matches + // kDisableTestFilter. + const bool is_disabled = + internal::UnitTestOptions::MatchesFilter(test_case_name, + kDisableTestFilter) || + internal::UnitTestOptions::MatchesFilter(test_name, + kDisableTestFilter); + test_info->is_disabled_ = is_disabled; + + const bool matches_filter = + internal::UnitTestOptions::FilterMatchesTest(test_case_name, + test_name); + test_info->matches_filter_ = matches_filter; + + const bool is_runnable = + (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) && + matches_filter; + + const bool is_selected = is_runnable && + (shard_tests == IGNORE_SHARDING_PROTOCOL || + ShouldRunTestOnShard(total_shards, shard_index, + num_runnable_tests)); + + num_runnable_tests += is_runnable; + num_selected_tests += is_selected; + + test_info->should_run_ = is_selected; + test_case->set_should_run(test_case->should_run() || is_selected); + } + } + return num_selected_tests; +} + +// Prints the given C-string on a single line by replacing all '\n' +// characters with string "\\n". If the output takes more than +// max_length characters, only prints the first max_length characters +// and "...". +static void PrintOnOneLine(const char* str, int max_length) { + if (str != NULL) { + for (int i = 0; *str != '\0'; ++str) { + if (i >= max_length) { + printf("..."); + break; + } + if (*str == '\n') { + printf("\\n"); + i += 2; + } else { + printf("%c", *str); + ++i; + } + } + } +} + +// Prints the names of the tests matching the user-specified filter flag. +void UnitTestImpl::ListTestsMatchingFilter() { + // Print at most this many characters for each type/value parameter. + const int kMaxParamLength = 250; + + for (size_t i = 0; i < test_cases_.size(); i++) { + const TestCase* const test_case = test_cases_[i]; + bool printed_test_case_name = false; + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + const TestInfo* const test_info = + test_case->test_info_list()[j]; + if (test_info->matches_filter_) { + if (!printed_test_case_name) { + printed_test_case_name = true; + printf("%s.", test_case->name()); + if (test_case->type_param() != NULL) { + printf(" # %s = ", kTypeParamLabel); + // We print the type parameter on a single line to make + // the output easy to parse by a program. + PrintOnOneLine(test_case->type_param(), kMaxParamLength); + } + printf("\n"); + } + printf(" %s", test_info->name()); + if (test_info->value_param() != NULL) { + printf(" # %s = ", kValueParamLabel); + // We print the value parameter on a single line to make the + // output easy to parse by a program. + PrintOnOneLine(test_info->value_param(), kMaxParamLength); + } + printf("\n"); + } + } + } + fflush(stdout); +} + +// Sets the OS stack trace getter. +// +// Does nothing if the input and the current OS stack trace getter are +// the same; otherwise, deletes the old getter and makes the input the +// current getter. +void UnitTestImpl::set_os_stack_trace_getter( + OsStackTraceGetterInterface* getter) { + if (os_stack_trace_getter_ != getter) { + delete os_stack_trace_getter_; + os_stack_trace_getter_ = getter; + } +} + +// Returns the current OS stack trace getter if it is not NULL; +// otherwise, creates an OsStackTraceGetter, makes it the current +// getter, and returns it. +OsStackTraceGetterInterface* UnitTestImpl::os_stack_trace_getter() { + if (os_stack_trace_getter_ == NULL) { +#ifdef GTEST_OS_STACK_TRACE_GETTER_ + os_stack_trace_getter_ = new GTEST_OS_STACK_TRACE_GETTER_; +#else + os_stack_trace_getter_ = new OsStackTraceGetter; +#endif // GTEST_OS_STACK_TRACE_GETTER_ + } + + return os_stack_trace_getter_; +} + +// Returns the TestResult for the test that's currently running, or +// the TestResult for the ad hoc test if no test is running. +TestResult* UnitTestImpl::current_test_result() { + return current_test_info_ ? + &(current_test_info_->result_) : &ad_hoc_test_result_; +} + +// Shuffles all test cases, and the tests within each test case, +// making sure that death tests are still run first. +void UnitTestImpl::ShuffleTests() { + // Shuffles the death test cases. + ShuffleRange(random(), 0, last_death_test_case_ + 1, &test_case_indices_); + + // Shuffles the non-death test cases. + ShuffleRange(random(), last_death_test_case_ + 1, + static_cast(test_cases_.size()), &test_case_indices_); + + // Shuffles the tests inside each test case. + for (size_t i = 0; i < test_cases_.size(); i++) { + test_cases_[i]->ShuffleTests(random()); + } +} + +// Restores the test cases and tests to their order before the first shuffle. +void UnitTestImpl::UnshuffleTests() { + for (size_t i = 0; i < test_cases_.size(); i++) { + // Unshuffles the tests in each test case. + test_cases_[i]->UnshuffleTests(); + // Resets the index of each test case. + test_case_indices_[i] = static_cast(i); + } +} + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +std::string GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/, + int skip_count) { + // We pass skip_count + 1 to skip this wrapper function in addition + // to what the user really wants to skip. + return GetUnitTestImpl()->CurrentOsStackTraceExceptTop(skip_count + 1); +} + +// Used by the GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_ macro to +// suppress unreachable code warnings. +namespace { +class ClassUniqueToAlwaysTrue {}; +} + +bool IsTrue(bool condition) { return condition; } + +bool AlwaysTrue() { +#if GTEST_HAS_EXCEPTIONS + // This condition is always false so AlwaysTrue() never actually throws, + // but it makes the compiler think that it may throw. + if (IsTrue(false)) + throw ClassUniqueToAlwaysTrue(); +#endif // GTEST_HAS_EXCEPTIONS + return true; +} + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +bool SkipPrefix(const char* prefix, const char** pstr) { + const size_t prefix_len = strlen(prefix); + if (strncmp(*pstr, prefix, prefix_len) == 0) { + *pstr += prefix_len; + return true; + } + return false; +} + +// Parses a string as a command line flag. The string should have +// the format "--flag=value". When def_optional is true, the "=value" +// part can be omitted. +// +// Returns the value of the flag, or NULL if the parsing failed. +const char* ParseFlagValue(const char* str, + const char* flag, + bool def_optional) { + // str and flag must not be NULL. + if (str == NULL || flag == NULL) return NULL; + + // The flag must start with "--" followed by GTEST_FLAG_PREFIX_. + const std::string flag_str = std::string("--") + GTEST_FLAG_PREFIX_ + flag; + const size_t flag_len = flag_str.length(); + if (strncmp(str, flag_str.c_str(), flag_len) != 0) return NULL; + + // Skips the flag name. + const char* flag_end = str + flag_len; + + // When def_optional is true, it's OK to not have a "=value" part. + if (def_optional && (flag_end[0] == '\0')) { + return flag_end; + } + + // If def_optional is true and there are more characters after the + // flag name, or if def_optional is false, there must be a '=' after + // the flag name. + if (flag_end[0] != '=') return NULL; + + // Returns the string after "=". + return flag_end + 1; +} + +// Parses a string for a bool flag, in the form of either +// "--flag=value" or "--flag". +// +// In the former case, the value is taken as true as long as it does +// not start with '0', 'f', or 'F'. +// +// In the latter case, the value is taken as true. +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseBoolFlag(const char* str, const char* flag, bool* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, true); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Converts the string value to a bool. + *value = !(*value_str == '0' || *value_str == 'f' || *value_str == 'F'); + return true; +} + +// Parses a string for an Int32 flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseInt32Flag(const char* str, const char* flag, Int32* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + return ParseInt32(Message() << "The value of flag --" << flag, + value_str, value); +} + +// Parses a string for a string flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseStringFlag(const char* str, const char* flag, std::string* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + *value = value_str; + return true; +} + +// Determines whether a string has a prefix that Google Test uses for its +// flags, i.e., starts with GTEST_FLAG_PREFIX_ or GTEST_FLAG_PREFIX_DASH_. +// If Google Test detects that a command line flag has its prefix but is not +// recognized, it will print its help message. Flags starting with +// GTEST_INTERNAL_PREFIX_ followed by "internal_" are considered Google Test +// internal flags and do not trigger the help message. +static bool HasGoogleTestFlagPrefix(const char* str) { + return (SkipPrefix("--", &str) || + SkipPrefix("-", &str) || + SkipPrefix("/", &str)) && + !SkipPrefix(GTEST_FLAG_PREFIX_ "internal_", &str) && + (SkipPrefix(GTEST_FLAG_PREFIX_, &str) || + SkipPrefix(GTEST_FLAG_PREFIX_DASH_, &str)); +} + +// Prints a string containing code-encoded text. The following escape +// sequences can be used in the string to control the text color: +// +// @@ prints a single '@' character. +// @R changes the color to red. +// @G changes the color to green. +// @Y changes the color to yellow. +// @D changes to the default terminal text color. +// +// TODO(wan@google.com): Write tests for this once we add stdout +// capturing to Google Test. +static void PrintColorEncoded(const char* str) { + GTestColor color = COLOR_DEFAULT; // The current color. + + // Conceptually, we split the string into segments divided by escape + // sequences. Then we print one segment at a time. At the end of + // each iteration, the str pointer advances to the beginning of the + // next segment. + for (;;) { + const char* p = strchr(str, '@'); + if (p == NULL) { + ColoredPrintf(color, "%s", str); + return; + } + + ColoredPrintf(color, "%s", std::string(str, p).c_str()); + + const char ch = p[1]; + str = p + 2; + if (ch == '@') { + ColoredPrintf(color, "@"); + } else if (ch == 'D') { + color = COLOR_DEFAULT; + } else if (ch == 'R') { + color = COLOR_RED; + } else if (ch == 'G') { + color = COLOR_GREEN; + } else if (ch == 'Y') { + color = COLOR_YELLOW; + } else { + --str; + } + } +} + +static const char kColorEncodedHelpMessage[] = +"This program contains tests written using " GTEST_NAME_ ". You can use the\n" +"following command line flags to control its behavior:\n" +"\n" +"Test Selection:\n" +" @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n" +" List the names of all tests instead of running them. The name of\n" +" TEST(Foo, Bar) is \"Foo.Bar\".\n" +" @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS" + "[@G-@YNEGATIVE_PATTERNS]@D\n" +" Run only the tests whose name matches one of the positive patterns but\n" +" none of the negative patterns. '?' matches any single character; '*'\n" +" matches any substring; ':' separates two patterns.\n" +" @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n" +" Run all disabled tests too.\n" +"\n" +"Test Execution:\n" +" @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n" +" Run the tests repeatedly; use a negative count to repeat forever.\n" +" @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n" +" Randomize tests' orders on every iteration.\n" +" @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n" +" Random number seed to use for shuffling test orders (between 1 and\n" +" 99999, or 0 to use a seed based on the current time).\n" +"\n" +"Test Output:\n" +" @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" +" Enable/disable colored output. The default is @Gauto@D.\n" +" -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n" +" Don't print the elapsed time of each test.\n" +" @G--" GTEST_FLAG_PREFIX_ "output=xml@Y[@G:@YDIRECTORY_PATH@G" + GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n" +" Generate an XML report in the given directory or with the given file\n" +" name. @YFILE_PATH@D defaults to @Gtest_details.xml@D.\n" +#if GTEST_CAN_STREAM_RESULTS_ +" @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n" +" Stream test results to the given server.\n" +#endif // GTEST_CAN_STREAM_RESULTS_ +"\n" +"Assertion Behavior:\n" +#if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" +" Set the default death test style.\n" +#endif // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n" +" Turn assertion failures into debugger break-points.\n" +" @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n" +" Turn assertion failures into C++ exceptions.\n" +" @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n" +" Do not report exceptions as test failures. Instead, allow them\n" +" to crash the program or throw a pop-up (on Windows).\n" +"\n" +"Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set " + "the corresponding\n" +"environment variable of a flag (all letters in upper-case). For example, to\n" +"disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_ + "color=no@D or set\n" +"the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n" +"\n" +"For more information, please read the " GTEST_NAME_ " documentation at\n" +"@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n" +"(not one in your own code or tests), please report it to\n" +"@G<" GTEST_DEV_EMAIL_ ">@D.\n"; + +bool ParseGoogleTestFlag(const char* const arg) { + return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, + >EST_FLAG(also_run_disabled_tests)) || + ParseBoolFlag(arg, kBreakOnFailureFlag, + >EST_FLAG(break_on_failure)) || + ParseBoolFlag(arg, kCatchExceptionsFlag, + >EST_FLAG(catch_exceptions)) || + ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || + ParseStringFlag(arg, kDeathTestStyleFlag, + >EST_FLAG(death_test_style)) || + ParseBoolFlag(arg, kDeathTestUseFork, + >EST_FLAG(death_test_use_fork)) || + ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || + ParseStringFlag(arg, kInternalRunDeathTestFlag, + >EST_FLAG(internal_run_death_test)) || + ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || + ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || + ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || + ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || + ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || + ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || + ParseInt32Flag(arg, kStackTraceDepthFlag, + >EST_FLAG(stack_trace_depth)) || + ParseStringFlag(arg, kStreamResultToFlag, + >EST_FLAG(stream_result_to)) || + ParseBoolFlag(arg, kThrowOnFailureFlag, + >EST_FLAG(throw_on_failure)); +} + +#if GTEST_USE_OWN_FLAGFILE_FLAG_ +void LoadFlagsFromFile(const std::string& path) { + FILE* flagfile = posix::FOpen(path.c_str(), "r"); + if (!flagfile) { + fprintf(stderr, + "Unable to open file \"%s\"\n", + GTEST_FLAG(flagfile).c_str()); + fflush(stderr); + exit(EXIT_FAILURE); + } + std::string contents(ReadEntireFile(flagfile)); + posix::FClose(flagfile); + std::vector lines; + SplitString(contents, '\n', &lines); + for (size_t i = 0; i < lines.size(); ++i) { + if (lines[i].empty()) + continue; + if (!ParseGoogleTestFlag(lines[i].c_str())) + g_help_flag = true; + } +} +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. The type parameter CharType can be +// instantiated to either char or wchar_t. +template +void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) { + for (int i = 1; i < *argc; i++) { + const std::string arg_string = StreamableToString(argv[i]); + const char* const arg = arg_string.c_str(); + + using internal::ParseBoolFlag; + using internal::ParseInt32Flag; + using internal::ParseStringFlag; + + bool remove_flag = false; + if (ParseGoogleTestFlag(arg)) { + remove_flag = true; +#if GTEST_USE_OWN_FLAGFILE_FLAG_ + } else if (ParseStringFlag(arg, kFlagfileFlag, >EST_FLAG(flagfile))) { + LoadFlagsFromFile(GTEST_FLAG(flagfile)); + remove_flag = true; +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + } else if (arg_string == "--help" || arg_string == "-h" || + arg_string == "-?" || arg_string == "/?" || + HasGoogleTestFlagPrefix(arg)) { + // Both help flag and unrecognized Google Test flags (excluding + // internal ones) trigger help display. + g_help_flag = true; + } + + if (remove_flag) { + // Shift the remainder of the argv list left by one. Note + // that argv has (*argc + 1) elements, the last one always being + // NULL. The following loop moves the trailing NULL element as + // well. + for (int j = i; j != *argc; j++) { + argv[j] = argv[j + 1]; + } + + // Decrements the argument count. + (*argc)--; + + // We also need to decrement the iterator as we just removed + // an element. + i--; + } + } + + if (g_help_flag) { + // We print the help here instead of in RUN_ALL_TESTS(), as the + // latter may not be called at all if the user is using Google + // Test with another testing framework. + PrintColorEncoded(kColorEncodedHelpMessage); + } +} + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +void ParseGoogleTestFlagsOnly(int* argc, char** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); +} +void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); +} + +// The internal implementation of InitGoogleTest(). +// +// The type parameter CharType can be instantiated to either char or +// wchar_t. +template +void InitGoogleTestImpl(int* argc, CharType** argv) { + // We don't want to run the initialization code twice. + if (GTestIsInitialized()) return; + + if (*argc <= 0) return; + + g_argvs.clear(); + for (int i = 0; i != *argc; i++) { + g_argvs.push_back(StreamableToString(argv[i])); + } + + ParseGoogleTestFlagsOnly(argc, argv); + GetUnitTestImpl()->PostFlagParsingInit(); +} + +} // namespace internal + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +void InitGoogleTest(int* argc, char** argv) { +#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); +#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + internal::InitGoogleTestImpl(argc, argv); +#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) +} + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +void InitGoogleTest(int* argc, wchar_t** argv) { +#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); +#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + internal::InitGoogleTestImpl(argc, argv); +#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) +} + +} // namespace testing diff --git a/mypyc/external/googletest/src/gtest_main.cc b/mypyc/external/googletest/src/gtest_main.cc new file mode 100644 index 000000000000..f30282255233 --- /dev/null +++ b/mypyc/external/googletest/src/gtest_main.cc @@ -0,0 +1,38 @@ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include "gtest/gtest.h" + +GTEST_API_ int main(int argc, char **argv) { + printf("Running main() from gtest_main.cc\n"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/mypyc/ir/__init__.py b/mypyc/ir/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py new file mode 100644 index 000000000000..94bf714b28d4 --- /dev/null +++ b/mypyc/ir/class_ir.py @@ -0,0 +1,503 @@ +"""Intermediate representation of classes.""" + +from __future__ import annotations + +from typing import List, NamedTuple + +from mypyc.common import PROPSET_PREFIX, JsonDict +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature +from mypyc.ir.ops import DeserMaps, Value +from mypyc.ir.rtypes import RInstance, RType, deserialize_type +from mypyc.namegen import NameGenerator, exported_name + +# Some notes on the vtable layout: Each concrete class has a vtable +# that contains function pointers for its methods. So that subclasses +# may be efficiently used when their parent class is expected, the +# layout of child vtables must be an extension of their base class's +# vtable. +# +# This makes multiple inheritance tricky, since obviously we cannot be +# an extension of multiple parent classes. We solve this by requiring +# all but one parent to be "traits", which we can operate on in a +# somewhat less efficient way. For each trait implemented by a class, +# we generate a separate vtable for the methods in that trait. +# We then store an array of (trait type, trait vtable) pointers alongside +# a class's main vtable. When we want to call a trait method, we +# (at runtime!) search the array of trait vtables to find the correct one, +# then call through it. +# Trait vtables additionally need entries for attribute getters and setters, +# since they can't always be in the same location. +# +# To keep down the number of indirections necessary, we store the +# array of trait vtables in the memory *before* the class vtable, and +# search it backwards. (This is a trick we can only do once---there +# are only two directions to store data in---but I don't think we'll +# need it again.) +# There are some tricks we could try in the future to store the trait +# vtables inline in the trait table (which would cut down one indirection), +# but this seems good enough for now. +# +# As an example: +# Imagine that we have a class B that inherits from a concrete class A +# and traits T1 and T2, and that A has methods foo() and +# bar() and B overrides bar() with a more specific type. +# Then B's vtable will look something like: +# +# T1 type object +# ptr to B's T1 trait vtable +# T2 type object +# ptr to B's T2 trait vtable +# -> | A.foo +# | Glue function that converts between A.bar's type and B.bar +# B.bar +# B.baz +# +# The arrow points to the "start" of the vtable (what vtable pointers +# point to) and the bars indicate which parts correspond to the parent +# class A's vtable layout. +# +# Classes that allow interpreted code to subclass them also have a +# "shadow vtable" that contains implementations that delegate to +# making a pycall, so that overridden methods in interpreted children +# will be called. (A better strategy could dynamically generate these +# vtables based on which methods are overridden in the children.) + +# Descriptions of method and attribute entries in class vtables. +# The 'cls' field is the class that the method/attr was defined in, +# which might be a parent class. +# The 'shadow_method', if present, contains the method that should be +# placed in the class's shadow vtable (if it has one). + + +class VTableMethod(NamedTuple): + cls: "ClassIR" # noqa: UP037 + name: str + method: FuncIR + shadow_method: FuncIR | None + + +VTableEntries = List[VTableMethod] + + +class ClassIR: + """Intermediate representation of a class. + + This also describes the runtime structure of native instances. + """ + + def __init__( + self, + name: str, + module_name: str, + is_trait: bool = False, + is_generated: bool = False, + is_abstract: bool = False, + is_ext_class: bool = True, + is_final_class: bool = False, + ) -> None: + self.name = name + self.module_name = module_name + self.is_trait = is_trait + self.is_generated = is_generated + self.is_abstract = is_abstract + self.is_ext_class = is_ext_class + self.is_final_class = is_final_class + # An augmented class has additional methods separate from what mypyc generates. + # Right now the only one is dataclasses. + self.is_augmented = False + # Does this inherit from a Python class? + self.inherits_python = False + # Do instances of this class have __dict__? + self.has_dict = False + # Do we allow interpreted subclasses? Derived from a mypyc_attr. + self.allow_interpreted_subclasses = False + # Does this class need getseters to be generated for its attributes? (getseters are also + # added if is_generated is False) + self.needs_getseters = False + # Is this class declared as serializable (supports copy.copy + # and pickle) using @mypyc_attr(serializable=True)? + # + # Additionally, any class with this attribute False but with + # an __init__ that can be called without any arguments is + # *implicitly serializable*. In this case __init__ will be + # called during deserialization without arguments. If this is + # True, we match Python semantics and __init__ won't be called + # during deserialization. + # + # This impacts also all subclasses. Use is_serializable() to + # also consider base classes. + self._serializable = False + # If this a subclass of some built-in python class, the name + # of the object for that class. We currently only support this + # in a few ad-hoc cases. + self.builtin_base: str | None = None + # Default empty constructor + self.ctor = FuncDecl(name, None, module_name, FuncSignature([], RInstance(self))) + # Attributes defined in the class (not inherited) + self.attributes: dict[str, RType] = {} + # Deletable attributes + self.deletable: list[str] = [] + # We populate method_types with the signatures of every method before + # we generate methods, and we rely on this information being present. + self.method_decls: dict[str, FuncDecl] = {} + # Map of methods that are actually present in an extension class + self.methods: dict[str, FuncIR] = {} + # Glue methods for boxing/unboxing when a class changes the type + # while overriding a method. Maps from (parent class overridden, method) + # to IR of glue method. + self.glue_methods: dict[tuple[ClassIR, str], FuncIR] = {} + + # Properties are accessed like attributes, but have behavior like method calls. + # They don't belong in the methods dictionary, since we don't want to expose them to + # Python's method API. But we want to put them into our own vtable as methods, so that + # they are properly handled and overridden. The property dictionary values are a tuple + # containing a property getter and an optional property setter. + self.properties: dict[str, tuple[FuncIR, FuncIR | None]] = {} + # We generate these in prepare_class_def so that we have access to them when generating + # other methods and properties that rely on these types. + self.property_types: dict[str, RType] = {} + + self.vtable: dict[str, int] | None = None + self.vtable_entries: VTableEntries = [] + self.trait_vtables: dict[ClassIR, VTableEntries] = {} + # N.B: base might not actually quite be the direct base. + # It is the nearest concrete base, but we allow a trait in between. + self.base: ClassIR | None = None + self.traits: list[ClassIR] = [] + # Supply a working mro for most generated classes. Real classes will need to + # fix it up. + self.mro: list[ClassIR] = [self] + # base_mro is the chain of concrete (non-trait) ancestors + self.base_mro: list[ClassIR] = [self] + + # Direct subclasses of this class (use subclasses() to also include non-direct ones) + # None if separate compilation prevents this from working. + # + # Often it's better to use has_no_subclasses() or subclasses() instead. + self.children: list[ClassIR] | None = [] + + # Instance attributes that are initialized in the class body. + self.attrs_with_defaults: set[str] = set() + + # Attributes that are always initialized in __init__ or class body + # (inferred in mypyc.analysis.attrdefined using interprocedural analysis) + self._always_initialized_attrs: set[str] = set() + + # Attributes that are sometimes initialized in __init__ + self._sometimes_initialized_attrs: set[str] = set() + + # If True, __init__ can make 'self' visible to unanalyzed/arbitrary code + self.init_self_leak = False + + # Definedness of these attributes is backed by a bitmap. Index in the list + # indicates the bit number. Includes inherited attributes. We need the + # bitmap for types such as native ints that can't have a dedicated error + # value that doesn't overlap a valid value. The bitmap is used if the + # value of an attribute is the same as the error value. + self.bitmap_attrs: list[str] = [] + + def __repr__(self) -> str: + return ( + "ClassIR(" + "name={self.name}, module_name={self.module_name}, " + "is_trait={self.is_trait}, is_generated={self.is_generated}, " + "is_abstract={self.is_abstract}, is_ext_class={self.is_ext_class}, " + "is_final_class={self.is_final_class}" + ")".format(self=self) + ) + + @property + def fullname(self) -> str: + return f"{self.module_name}.{self.name}" + + def real_base(self) -> ClassIR | None: + """Return the actual concrete base class, if there is one.""" + if len(self.mro) > 1 and not self.mro[1].is_trait: + return self.mro[1] + return None + + def vtable_entry(self, name: str) -> int: + assert self.vtable is not None, "vtable not computed yet" + assert name in self.vtable, f"{self.name!r} has no attribute {name!r}" + return self.vtable[name] + + def attr_details(self, name: str) -> tuple[RType, ClassIR]: + for ir in self.mro: + if name in ir.attributes: + return ir.attributes[name], ir + if name in ir.property_types: + return ir.property_types[name], ir + raise KeyError(f"{self.name!r} has no attribute {name!r}") + + def attr_type(self, name: str) -> RType: + return self.attr_details(name)[0] + + def method_decl(self, name: str) -> FuncDecl: + for ir in self.mro: + if name in ir.method_decls: + return ir.method_decls[name] + raise KeyError(f"{self.name!r} has no attribute {name!r}") + + def method_sig(self, name: str) -> FuncSignature: + return self.method_decl(name).sig + + def has_method(self, name: str) -> bool: + try: + self.method_decl(name) + except KeyError: + return False + return True + + def is_method_final(self, name: str) -> bool: + subs = self.subclasses() + if subs is None: + return self.is_final_class + + if self.has_method(name): + method_decl = self.method_decl(name) + for subc in subs: + if subc.method_decl(name) != method_decl: + return False + return True + else: + return not any(subc.has_method(name) for subc in subs) + + def has_attr(self, name: str) -> bool: + try: + self.attr_type(name) + except KeyError: + return False + return True + + def is_deletable(self, name: str) -> bool: + return any(name in ir.deletable for ir in self.mro) + + def is_always_defined(self, name: str) -> bool: + if self.is_deletable(name): + return False + return name in self._always_initialized_attrs + + def name_prefix(self, names: NameGenerator) -> str: + return names.private_name(self.module_name, self.name) + + def struct_name(self, names: NameGenerator) -> str: + return f"{exported_name(self.fullname)}Object" + + def get_method_and_class( + self, name: str, *, prefer_method: bool = False + ) -> tuple[FuncIR, ClassIR] | None: + for ir in self.mro: + if name in ir.methods: + func_ir = ir.methods[name] + if not prefer_method and func_ir.decl.implicit: + # This is an implicit accessor, so there is also an attribute definition + # which the caller prefers. This happens if an attribute overrides a + # property. + return None + return func_ir, ir + + return None + + def get_method(self, name: str, *, prefer_method: bool = False) -> FuncIR | None: + res = self.get_method_and_class(name, prefer_method=prefer_method) + return res[0] if res else None + + def has_method_decl(self, name: str) -> bool: + return any(name in ir.method_decls for ir in self.mro) + + def has_no_subclasses(self) -> bool: + return self.children == [] and not self.allow_interpreted_subclasses + + def subclasses(self) -> set[ClassIR] | None: + """Return all subclasses of this class, both direct and indirect. + + Return None if it is impossible to identify all subclasses, for example + because we are performing separate compilation. + """ + if self.children is None or self.allow_interpreted_subclasses: + return None + result = set(self.children) + for child in self.children: + if child.children: + child_subs = child.subclasses() + if child_subs is None: + return None + result.update(child_subs) + return result + + def concrete_subclasses(self) -> list[ClassIR] | None: + """Return all concrete (i.e. non-trait and non-abstract) subclasses. + + Include both direct and indirect subclasses. Place classes with no children first. + """ + subs = self.subclasses() + if subs is None: + return None + concrete = {c for c in subs if not (c.is_trait or c.is_abstract)} + # We place classes with no children first because they are more likely + # to appear in various isinstance() checks. We then sort leaves by name + # to get stable order. + return sorted(concrete, key=lambda c: (len(c.children or []), c.name)) + + def is_serializable(self) -> bool: + return any(ci._serializable for ci in self.mro) + + def serialize(self) -> JsonDict: + return { + "name": self.name, + "module_name": self.module_name, + "is_trait": self.is_trait, + "is_ext_class": self.is_ext_class, + "is_abstract": self.is_abstract, + "is_generated": self.is_generated, + "is_augmented": self.is_augmented, + "is_final_class": self.is_final_class, + "inherits_python": self.inherits_python, + "has_dict": self.has_dict, + "allow_interpreted_subclasses": self.allow_interpreted_subclasses, + "needs_getseters": self.needs_getseters, + "_serializable": self._serializable, + "builtin_base": self.builtin_base, + "ctor": self.ctor.serialize(), + # We serialize dicts as lists to ensure order is preserved + "attributes": [(k, t.serialize()) for k, t in self.attributes.items()], + # We try to serialize a name reference, but if the decl isn't in methods + # then we can't be sure that will work so we serialize the whole decl. + "method_decls": [ + (k, d.id if k in self.methods else d.serialize()) + for k, d in self.method_decls.items() + ], + # We serialize method fullnames out and put methods in a separate dict + "methods": [(k, m.id) for k, m in self.methods.items()], + "glue_methods": [ + ((cir.fullname, k), m.id) for (cir, k), m in self.glue_methods.items() + ], + # We serialize properties and property_types separately out of an + # abundance of caution about preserving dict ordering... + "property_types": [(k, t.serialize()) for k, t in self.property_types.items()], + "properties": list(self.properties), + "vtable": self.vtable, + "vtable_entries": serialize_vtable(self.vtable_entries), + "trait_vtables": [ + (cir.fullname, serialize_vtable(v)) for cir, v in self.trait_vtables.items() + ], + # References to class IRs are all just names + "base": self.base.fullname if self.base else None, + "traits": [cir.fullname for cir in self.traits], + "mro": [cir.fullname for cir in self.mro], + "base_mro": [cir.fullname for cir in self.base_mro], + "children": ( + [cir.fullname for cir in self.children] if self.children is not None else None + ), + "deletable": self.deletable, + "attrs_with_defaults": sorted(self.attrs_with_defaults), + "_always_initialized_attrs": sorted(self._always_initialized_attrs), + "_sometimes_initialized_attrs": sorted(self._sometimes_initialized_attrs), + "init_self_leak": self.init_self_leak, + } + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: + fullname = data["module_name"] + "." + data["name"] + assert fullname in ctx.classes, "Class %s not in deser class map" % fullname + ir = ctx.classes[fullname] + + ir.is_trait = data["is_trait"] + ir.is_generated = data["is_generated"] + ir.is_abstract = data["is_abstract"] + ir.is_ext_class = data["is_ext_class"] + ir.is_augmented = data["is_augmented"] + ir.is_final_class = data["is_final_class"] + ir.inherits_python = data["inherits_python"] + ir.has_dict = data["has_dict"] + ir.allow_interpreted_subclasses = data["allow_interpreted_subclasses"] + ir.needs_getseters = data["needs_getseters"] + ir._serializable = data["_serializable"] + ir.builtin_base = data["builtin_base"] + ir.ctor = FuncDecl.deserialize(data["ctor"], ctx) + ir.attributes = {k: deserialize_type(t, ctx) for k, t in data["attributes"]} + ir.method_decls = { + k: ctx.functions[v].decl if isinstance(v, str) else FuncDecl.deserialize(v, ctx) + for k, v in data["method_decls"] + } + ir.methods = {k: ctx.functions[v] for k, v in data["methods"]} + ir.glue_methods = { + (ctx.classes[c], k): ctx.functions[v] for (c, k), v in data["glue_methods"] + } + ir.property_types = {k: deserialize_type(t, ctx) for k, t in data["property_types"]} + ir.properties = { + k: (ir.methods[k], ir.methods.get(PROPSET_PREFIX + k)) for k in data["properties"] + } + + ir.vtable = data["vtable"] + ir.vtable_entries = deserialize_vtable(data["vtable_entries"], ctx) + ir.trait_vtables = { + ctx.classes[k]: deserialize_vtable(v, ctx) for k, v in data["trait_vtables"] + } + + base = data["base"] + ir.base = ctx.classes[base] if base else None + ir.traits = [ctx.classes[s] for s in data["traits"]] + ir.mro = [ctx.classes[s] for s in data["mro"]] + ir.base_mro = [ctx.classes[s] for s in data["base_mro"]] + ir.children = data["children"] and [ctx.classes[s] for s in data["children"]] + ir.deletable = data["deletable"] + ir.attrs_with_defaults = set(data["attrs_with_defaults"]) + ir._always_initialized_attrs = set(data["_always_initialized_attrs"]) + ir._sometimes_initialized_attrs = set(data["_sometimes_initialized_attrs"]) + ir.init_self_leak = data["init_self_leak"] + + return ir + + +class NonExtClassInfo: + """Information needed to construct a non-extension class (Python class). + + Includes the class dictionary, a tuple of base classes, + the class annotations dictionary, and the metaclass. + """ + + def __init__(self, dict: Value, bases: Value, anns: Value, metaclass: Value) -> None: + self.dict = dict + self.bases = bases + self.anns = anns + self.metaclass = metaclass + + +def serialize_vtable_entry(entry: VTableMethod) -> JsonDict: + return { + ".class": "VTableMethod", + "cls": entry.cls.fullname, + "name": entry.name, + "method": entry.method.decl.id, + "shadow_method": entry.shadow_method.decl.id if entry.shadow_method else None, + } + + +def serialize_vtable(vtable: VTableEntries) -> list[JsonDict]: + return [serialize_vtable_entry(v) for v in vtable] + + +def deserialize_vtable_entry(data: JsonDict, ctx: DeserMaps) -> VTableMethod: + if data[".class"] == "VTableMethod": + return VTableMethod( + ctx.classes[data["cls"]], + data["name"], + ctx.functions[data["method"]], + ctx.functions[data["shadow_method"]] if data["shadow_method"] else None, + ) + assert False, "Bogus vtable .class: %s" % data[".class"] + + +def deserialize_vtable(data: list[JsonDict], ctx: DeserMaps) -> VTableEntries: + return [deserialize_vtable_entry(x, ctx) for x in data] + + +def all_concrete_classes(class_ir: ClassIR) -> list[ClassIR] | None: + """Return all concrete classes among the class itself and its subclasses.""" + concrete = class_ir.concrete_subclasses() + if concrete is None: + return None + if not (class_ir.is_abstract or class_ir.is_trait): + concrete.append(class_ir) + return concrete diff --git a/mypyc/ir/func_ir.py b/mypyc/ir/func_ir.py new file mode 100644 index 000000000000..44847c7bb0b3 --- /dev/null +++ b/mypyc/ir/func_ir.py @@ -0,0 +1,370 @@ +"""Intermediate representation of functions.""" + +from __future__ import annotations + +from typing import Final, Sequence + +from mypy.nodes import ARG_POS, ArgKind, Block, FuncDef +from mypyc.common import BITMAP_BITS, JsonDict, bitmap_name, get_id_from_name, short_id_from_name +from mypyc.ir.ops import ( + Assign, + AssignMulti, + BasicBlock, + ControlOp, + DeserMaps, + LoadAddress, + Register, + Value, +) +from mypyc.ir.rtypes import RType, bitmap_rprimitive, deserialize_type +from mypyc.namegen import NameGenerator + + +class RuntimeArg: + """Description of a function argument in IR. + + Argument kind is one of ARG_* constants defined in mypy.nodes. + """ + + def __init__( + self, name: str, typ: RType, kind: ArgKind = ARG_POS, pos_only: bool = False + ) -> None: + self.name = name + self.type = typ + self.kind = kind + self.pos_only = pos_only + + @property + def optional(self) -> bool: + return self.kind.is_optional() + + def __repr__(self) -> str: + return "RuntimeArg(name={}, type={}, optional={!r}, pos_only={!r})".format( + self.name, self.type, self.optional, self.pos_only + ) + + def serialize(self) -> JsonDict: + return { + "name": self.name, + "type": self.type.serialize(), + "kind": int(self.kind.value), + "pos_only": self.pos_only, + } + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RuntimeArg: + return RuntimeArg( + data["name"], + deserialize_type(data["type"], ctx), + ArgKind(data["kind"]), + data["pos_only"], + ) + + +class FuncSignature: + """Signature of a function in IR.""" + + # TODO: Track if method? + + def __init__(self, args: Sequence[RuntimeArg], ret_type: RType) -> None: + self.args = tuple(args) + self.ret_type = ret_type + # Bitmap arguments are use to mark default values for arguments that + # have types with overlapping error values. + self.num_bitmap_args = num_bitmap_args(self.args) + if self.num_bitmap_args: + extra = [ + RuntimeArg(bitmap_name(i), bitmap_rprimitive, pos_only=True) + for i in range(self.num_bitmap_args) + ] + self.args = self.args + tuple(reversed(extra)) + + def real_args(self) -> tuple[RuntimeArg, ...]: + """Return arguments without any synthetic bitmap arguments.""" + if self.num_bitmap_args: + return self.args[: -self.num_bitmap_args] + return self.args + + def bound_sig(self) -> FuncSignature: + if self.num_bitmap_args: + return FuncSignature(self.args[1 : -self.num_bitmap_args], self.ret_type) + else: + return FuncSignature(self.args[1:], self.ret_type) + + def __repr__(self) -> str: + return f"FuncSignature(args={self.args!r}, ret={self.ret_type!r})" + + def serialize(self) -> JsonDict: + if self.num_bitmap_args: + args = self.args[: -self.num_bitmap_args] + else: + args = self.args + return {"args": [t.serialize() for t in args], "ret_type": self.ret_type.serialize()} + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncSignature: + return FuncSignature( + [RuntimeArg.deserialize(arg, ctx) for arg in data["args"]], + deserialize_type(data["ret_type"], ctx), + ) + + +def num_bitmap_args(args: tuple[RuntimeArg, ...]) -> int: + n = 0 + for arg in args: + if arg.type.error_overlap and arg.kind.is_optional(): + n += 1 + return (n + (BITMAP_BITS - 1)) // BITMAP_BITS + + +FUNC_NORMAL: Final = 0 +FUNC_STATICMETHOD: Final = 1 +FUNC_CLASSMETHOD: Final = 2 + + +class FuncDecl: + """Declaration of a function in IR (without body or implementation). + + A function can be a regular module-level function, a method, a + static method, a class method, or a property getter/setter. + """ + + def __init__( + self, + name: str, + class_name: str | None, + module_name: str, + sig: FuncSignature, + kind: int = FUNC_NORMAL, + is_prop_setter: bool = False, + is_prop_getter: bool = False, + implicit: bool = False, + ) -> None: + self.name = name + self.class_name = class_name + self.module_name = module_name + self.sig = sig + self.kind = kind + self.is_prop_setter = is_prop_setter + self.is_prop_getter = is_prop_getter + if class_name is None: + self.bound_sig: FuncSignature | None = None + else: + if kind == FUNC_STATICMETHOD: + self.bound_sig = sig + else: + self.bound_sig = sig.bound_sig() + + # If True, not present in the mypy AST and must be synthesized during irbuild + # Currently only supported for property getters/setters + self.implicit = implicit + + # This is optional because this will be set to the line number when the corresponding + # FuncIR is created + self._line: int | None = None + + @property + def line(self) -> int: + assert self._line is not None + return self._line + + @line.setter + def line(self, line: int) -> None: + self._line = line + + @property + def id(self) -> str: + assert self.line is not None + return get_id_from_name(self.name, self.fullname, self.line) + + @staticmethod + def compute_shortname(class_name: str | None, name: str) -> str: + return class_name + "." + name if class_name else name + + @property + def shortname(self) -> str: + return FuncDecl.compute_shortname(self.class_name, self.name) + + @property + def fullname(self) -> str: + return self.module_name + "." + self.shortname + + def cname(self, names: NameGenerator) -> str: + partial_name = short_id_from_name(self.name, self.shortname, self._line) + return names.private_name(self.module_name, partial_name) + + def serialize(self) -> JsonDict: + return { + "name": self.name, + "class_name": self.class_name, + "module_name": self.module_name, + "sig": self.sig.serialize(), + "kind": self.kind, + "is_prop_setter": self.is_prop_setter, + "is_prop_getter": self.is_prop_getter, + "implicit": self.implicit, + } + + # TODO: move this to FuncIR? + @staticmethod + def get_id_from_json(func_ir: JsonDict) -> str: + """Get the id from the serialized FuncIR associated with this FuncDecl""" + decl = func_ir["decl"] + shortname = FuncDecl.compute_shortname(decl["class_name"], decl["name"]) + fullname = decl["module_name"] + "." + shortname + return get_id_from_name(decl["name"], fullname, func_ir["line"]) + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncDecl: + return FuncDecl( + data["name"], + data["class_name"], + data["module_name"], + FuncSignature.deserialize(data["sig"], ctx), + data["kind"], + data["is_prop_setter"], + data["is_prop_getter"], + data["implicit"], + ) + + +class FuncIR: + """Intermediate representation of a function with contextual information. + + Unlike FuncDecl, this includes the IR of the body (basic blocks). + """ + + def __init__( + self, + decl: FuncDecl, + arg_regs: list[Register], + blocks: list[BasicBlock], + line: int = -1, + traceback_name: str | None = None, + ) -> None: + # Declaration of the function, including the signature + self.decl = decl + # Registers for all the arguments to the function + self.arg_regs = arg_regs + # Body of the function + self.blocks = blocks + self.decl.line = line + # The name that should be displayed for tracebacks that + # include this function. Function will be omitted from + # tracebacks if None. + self.traceback_name = traceback_name + + @property + def line(self) -> int: + return self.decl.line + + @property + def args(self) -> Sequence[RuntimeArg]: + return self.decl.sig.args + + @property + def ret_type(self) -> RType: + return self.decl.sig.ret_type + + @property + def class_name(self) -> str | None: + return self.decl.class_name + + @property + def sig(self) -> FuncSignature: + return self.decl.sig + + @property + def name(self) -> str: + return self.decl.name + + @property + def fullname(self) -> str: + return self.decl.fullname + + @property + def id(self) -> str: + return self.decl.id + + def cname(self, names: NameGenerator) -> str: + return self.decl.cname(names) + + def __repr__(self) -> str: + if self.class_name: + return f"" + else: + return f"" + + def serialize(self) -> JsonDict: + # We don't include blocks in the serialized version + return { + "decl": self.decl.serialize(), + "line": self.line, + "traceback_name": self.traceback_name, + } + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncIR: + return FuncIR( + FuncDecl.deserialize(data["decl"], ctx), [], [], data["line"], data["traceback_name"] + ) + + +INVALID_FUNC_DEF: Final = FuncDef("", [], Block([])) + + +def all_values(args: list[Register], blocks: list[BasicBlock]) -> list[Value]: + """Return the set of all values that may be initialized in the blocks. + + This omits registers that are only read. + """ + values: list[Value] = list(args) + seen_registers = set(args) + + for block in blocks: + for op in block.ops: + if not isinstance(op, ControlOp): + if isinstance(op, (Assign, AssignMulti)): + if op.dest not in seen_registers: + values.append(op.dest) + seen_registers.add(op.dest) + elif op.is_void: + continue + else: + # If we take the address of a register, it might get initialized. + if ( + isinstance(op, LoadAddress) + and isinstance(op.src, Register) + and op.src not in seen_registers + ): + values.append(op.src) + seen_registers.add(op.src) + values.append(op) + + return values + + +def all_values_full(args: list[Register], blocks: list[BasicBlock]) -> list[Value]: + """Return set of all values that are initialized or accessed.""" + values: list[Value] = list(args) + seen_registers = set(args) + + for block in blocks: + for op in block.ops: + for source in op.sources(): + # Look for uninitialized registers that are accessed. Ignore + # non-registers since we don't allow ops outside basic blocks. + if isinstance(source, Register) and source not in seen_registers: + values.append(source) + seen_registers.add(source) + if not isinstance(op, ControlOp): + if isinstance(op, (Assign, AssignMulti)): + if op.dest not in seen_registers: + values.append(op.dest) + seen_registers.add(op.dest) + elif op.is_void: + continue + else: + values.append(op) + + return values diff --git a/mypyc/ir/module_ir.py b/mypyc/ir/module_ir.py new file mode 100644 index 000000000000..e3b240629eda --- /dev/null +++ b/mypyc/ir/module_ir.py @@ -0,0 +1,94 @@ +"""Intermediate representation of modules.""" + +from __future__ import annotations + +from typing import Dict + +from mypyc.common import JsonDict +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncDecl, FuncIR +from mypyc.ir.ops import DeserMaps +from mypyc.ir.rtypes import RType, deserialize_type + + +class ModuleIR: + """Intermediate representation of a module.""" + + def __init__( + self, + fullname: str, + imports: list[str], + functions: list[FuncIR], + classes: list[ClassIR], + final_names: list[tuple[str, RType]], + type_var_names: list[str], + ) -> None: + self.fullname = fullname + self.imports = imports.copy() + self.functions = functions + self.classes = classes + self.final_names = final_names + # Names of C statics used for Python 3.12 type variable objects. + # These are only visible in the module that defined them, so no need + # to serialize. + self.type_var_names = type_var_names + + def serialize(self) -> JsonDict: + return { + "fullname": self.fullname, + "imports": self.imports, + "functions": [f.serialize() for f in self.functions], + "classes": [c.serialize() for c in self.classes], + "final_names": [(k, t.serialize()) for k, t in self.final_names], + } + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR: + return ModuleIR( + data["fullname"], + data["imports"], + [ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]], + [ClassIR.deserialize(c, ctx) for c in data["classes"]], + [(k, deserialize_type(t, ctx)) for k, t in data["final_names"]], + [], + ) + + +def deserialize_modules(data: dict[str, JsonDict], ctx: DeserMaps) -> dict[str, ModuleIR]: + """Deserialize a collection of modules. + + The modules can contain dependencies on each other. + + Arguments: + data: A dict containing the modules to deserialize. + ctx: The deserialization maps to use and to populate. + They are populated with information from the deserialized + modules and as a precondition must have been populated by + deserializing any dependencies of the modules being deserialized + (outside of dependencies between the modules themselves). + + Returns a map containing the deserialized modules. + """ + for mod in data.values(): + # First create ClassIRs for every class so that we can construct types and whatnot + for cls in mod["classes"]: + ir = ClassIR(cls["name"], cls["module_name"]) + assert ir.fullname not in ctx.classes, "Class %s already in map" % ir.fullname + ctx.classes[ir.fullname] = ir + + for mod in data.values(): + # Then deserialize all of the functions so that methods are available + # to the class deserialization. + for method in mod["functions"]: + func = FuncIR.deserialize(method, ctx) + assert func.decl.id not in ctx.functions, ( + "Method %s already in map" % func.decl.fullname + ) + ctx.functions[func.decl.id] = func + + return {k: ModuleIR.deserialize(v, ctx) for k, v in data.items()} + + +# ModulesIRs should also always be an *OrderedDict*, but if we +# declared it that way we would need to put it in quotes everywhere... +ModuleIRs = Dict[str, ModuleIR] diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py new file mode 100644 index 000000000000..6e186c4ef0fc --- /dev/null +++ b/mypyc/ir/ops.py @@ -0,0 +1,1744 @@ +"""Low-level opcodes for compiler intermediate representation (IR). + +Opcodes operate on abstract values (Value) in a register machine. Each +value has a type (RType). A value can hold various things, such as: + +- local variables (Register) +- intermediate values of expressions (RegisterOp subclasses) +- condition flags (true/false) +- literals (integer literals, True, False, etc.) +""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import TYPE_CHECKING, Final, Generic, List, NamedTuple, Sequence, TypeVar, Union + +from mypy_extensions import trait + +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RTuple, + RType, + RVoid, + bit_rprimitive, + bool_rprimitive, + float_rprimitive, + int_rprimitive, + is_bit_rprimitive, + is_bool_rprimitive, + is_int_rprimitive, + is_none_rprimitive, + is_pointer_rprimitive, + is_short_int_rprimitive, + object_rprimitive, + pointer_rprimitive, + short_int_rprimitive, + void_rtype, +) + +if TYPE_CHECKING: + from mypyc.codegen.literals import LiteralValue + from mypyc.ir.class_ir import ClassIR + from mypyc.ir.func_ir import FuncDecl, FuncIR + +T = TypeVar("T") + + +class BasicBlock: + """IR basic block. + + Contains a sequence of Ops and ends with a ControlOp (Goto, + Branch, Return or Unreachable). Only the last op can be a + ControlOp. + + All generated Ops live in basic blocks. Basic blocks determine the + order of evaluation and control flow within a function. A basic + block is always associated with a single function/method (FuncIR). + + When building the IR, ops that raise exceptions can be included in + the middle of a basic block, but the exceptions aren't checked. + Afterwards we perform a transform that inserts explicit checks for + all error conditions and splits basic blocks accordingly to preserve + the invariant that a jump, branch or return can only ever appear + as the final op in a block. Manually inserting error checking ops + would be boring and error-prone. + + BasicBlocks have an error_handler attribute that determines where + to jump if an error occurs. If none is specified, an error will + propagate up out of the function. This is compiled away by the + `exceptions` module. + + Block labels are used for pretty printing and emitting C code, and + get filled in by those passes. + + Ops that may terminate the program aren't treated as exits. + """ + + def __init__(self, label: int = -1) -> None: + self.label = label + self.ops: list[Op] = [] + self.error_handler: BasicBlock | None = None + self.referenced = False + + @property + def terminated(self) -> bool: + """Does the block end with a jump, branch or return? + + This should always be true after the basic block has been fully built, but + this is false during construction. + """ + return bool(self.ops) and isinstance(self.ops[-1], ControlOp) + + @property + def terminator(self) -> ControlOp: + """The terminator operation of the block.""" + assert bool(self.ops) and isinstance(self.ops[-1], ControlOp) + return self.ops[-1] + + +# Never generates an exception +ERR_NEVER: Final = 0 +# Generates magic value (c_error_value) based on target RType on exception +ERR_MAGIC: Final = 1 +# Generates false (bool) on exception +ERR_FALSE: Final = 2 +# Always fails +ERR_ALWAYS: Final = 3 +# Like ERR_MAGIC, but the magic return overlaps with a possible return value, and +# an extra PyErr_Occurred() check is also required +ERR_MAGIC_OVERLAPPING: Final = 4 + +# Hack: using this line number for an op will suppress it in tracebacks +NO_TRACEBACK_LINE_NO = -10000 + + +class Value: + """Abstract base class for all IR values. + + These include references to registers, literals, and all + operations (Ops), such as assignments, calls and branches. + + Values are often used as inputs of Ops. Register can be used as an + assignment target. + + A Value is part of the IR being compiled if it's included in a BasicBlock + that is reachable from a FuncIR (i.e., is part of a function). + + See also: Op is a subclass of Value that is the base class of all + operations. + """ + + # Source line number (-1 for no/unknown line) + line = -1 + # Type of the value or the result of the operation + type: RType = void_rtype + is_borrowed = False + + @property + def is_void(self) -> bool: + return isinstance(self.type, RVoid) + + +class Register(Value): + """A Register holds a value of a specific type, and it can be read and mutated. + + A Register is always local to a function. Each local variable maps + to a Register, and they are also used for some (but not all) + temporary values. + + Note that the term 'register' is overloaded and is sometimes used + to refer to arbitrary Values (for example, in RegisterOp). + """ + + def __init__(self, type: RType, name: str = "", is_arg: bool = False, line: int = -1) -> None: + self.type = type + self.name = name + self.is_arg = is_arg + self.is_borrowed = is_arg + self.line = line + + @property + def is_void(self) -> bool: + return False + + def __repr__(self) -> str: + return f"" + + +class Integer(Value): + """Short integer literal. + + Integer literals are treated as constant values and are generally + not included in data flow analyses and such, unlike Register and + Op subclasses. + + Integer can represent multiple types: + + * Short tagged integers (short_int_primitive type; the tag bit is clear) + * Ordinary fixed-width integers (e.g., int32_rprimitive) + * Values of other unboxed primitive types that are represented as integers + (none_rprimitive, bool_rprimitive) + * Null pointers (value 0) of various types, including object_rprimitive + """ + + def __init__(self, value: int, rtype: RType = short_int_rprimitive, line: int = -1) -> None: + if is_short_int_rprimitive(rtype) or is_int_rprimitive(rtype): + self.value = value * 2 + else: + self.value = value + self.type = rtype + self.line = line + + def numeric_value(self) -> int: + if is_short_int_rprimitive(self.type) or is_int_rprimitive(self.type): + return self.value // 2 + return self.value + + +class Float(Value): + """Float literal. + + Floating point literals are treated as constant values and are generally + not included in data flow analyses and such, unlike Register and + Op subclasses. + """ + + def __init__(self, value: float, line: int = -1) -> None: + self.value = value + self.type = float_rprimitive + self.line = line + + +class Op(Value): + """Abstract base class for all IR operations. + + Each operation must be stored in a BasicBlock (in 'ops') to be + active in the IR. This is different from non-Op values, including + Register and Integer, where a reference from an active Op is + sufficient to be considered active. + + In well-formed IR an active Op has no references to inactive ops + or ops used in another function. + """ + + def __init__(self, line: int) -> None: + self.line = line + + def can_raise(self) -> bool: + # Override this is if Op may raise an exception. Note that currently the fact that + # only RegisterOps may raise an exception in hard coded in some places. + return False + + @abstractmethod + def sources(self) -> list[Value]: + """All the values the op may read.""" + + def stolen(self) -> list[Value]: + """Return arguments that have a reference count stolen by this op""" + return [] + + def unique_sources(self) -> list[Value]: + result: list[Value] = [] + for reg in self.sources(): + if reg not in result: + result.append(reg) + return result + + @abstractmethod + def accept(self, visitor: OpVisitor[T]) -> T: + pass + + +class BaseAssign(Op): + """Base class for ops that assign to a register.""" + + def __init__(self, dest: Register, line: int = -1) -> None: + super().__init__(line) + self.dest = dest + + +class Assign(BaseAssign): + """Assign a value to a Register (dest = src).""" + + error_kind = ERR_NEVER + + def __init__(self, dest: Register, src: Value, line: int = -1) -> None: + super().__init__(dest, line) + self.src = src + + def sources(self) -> list[Value]: + return [self.src] + + def stolen(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_assign(self) + + +class AssignMulti(BaseAssign): + """Assign multiple values to a Register (dest = src1, src2, ...). + + This is used to initialize RArray values. It's provided to avoid + very verbose IR for common vectorcall operations. + + Note that this interacts atypically with reference counting. We + assume that each RArray register is initialized exactly once + with this op. + """ + + error_kind = ERR_NEVER + + def __init__(self, dest: Register, src: list[Value], line: int = -1) -> None: + super().__init__(dest, line) + assert src + assert isinstance(dest.type, RArray) + assert dest.type.length == len(src) + self.src = src + + def sources(self) -> list[Value]: + return self.src.copy() + + def stolen(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_assign_multi(self) + + +class ControlOp(Op): + """Control flow operation.""" + + def targets(self) -> Sequence[BasicBlock]: + """Get all basic block targets of the control operation.""" + return () + + def set_target(self, i: int, new: BasicBlock) -> None: + """Update a basic block target.""" + raise AssertionError(f"Invalid set_target({self}, {i})") + + +class Goto(ControlOp): + """Unconditional jump.""" + + error_kind = ERR_NEVER + + def __init__(self, label: BasicBlock, line: int = -1) -> None: + super().__init__(line) + self.label = label + + def targets(self) -> Sequence[BasicBlock]: + return (self.label,) + + def set_target(self, i: int, new: BasicBlock) -> None: + assert i == 0 + self.label = new + + def __repr__(self) -> str: + return "" % self.label.label + + def sources(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_goto(self) + + +class Branch(ControlOp): + """Branch based on a value. + + If op is BOOL, branch based on a bit/bool value: + if [not] r1 goto L1 else goto L2 + + If op is IS_ERROR, branch based on whether there is an error value: + if [not] is_error(r1) goto L1 else goto L2 + """ + + # Branch ops never raise an exception. + error_kind = ERR_NEVER + + BOOL: Final = 100 + IS_ERROR: Final = 101 + + def __init__( + self, + value: Value, + true_label: BasicBlock, + false_label: BasicBlock, + op: int, + line: int = -1, + *, + rare: bool = False, + ) -> None: + super().__init__(line) + # Target value being checked + self.value = value + # Branch here if the condition is true + self.true = true_label + # Branch here if the condition is false + self.false = false_label + # Branch.BOOL (boolean check) or Branch.IS_ERROR (error value check) + self.op = op + # If True, the condition is negated + self.negated = False + # If not None, the true label should generate a traceback entry (func name, line number) + self.traceback_entry: tuple[str, int] | None = None + # If True, we expect to usually take the false branch (for optimization purposes); + # this is implicitly treated as true if there is a traceback entry + self.rare = rare + + def targets(self) -> Sequence[BasicBlock]: + return (self.true, self.false) + + def set_target(self, i: int, new: BasicBlock) -> None: + assert i == 0 or i == 1 + if i == 0: + self.true = new + else: + self.false = new + + def sources(self) -> list[Value]: + return [self.value] + + def invert(self) -> None: + self.negated = not self.negated + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_branch(self) + + +class Return(ControlOp): + """Return a value from a function.""" + + error_kind = ERR_NEVER + + def __init__(self, value: Value, line: int = -1) -> None: + super().__init__(line) + self.value = value + + def sources(self) -> list[Value]: + return [self.value] + + def stolen(self) -> list[Value]: + return [self.value] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_return(self) + + +class Unreachable(ControlOp): + """Mark the end of basic block as unreachable. + + This is sometimes necessary when the end of a basic block is never + reached. This can also be explicitly added to the end of non-None + returning functions (in None-returning function we can just return + None). + + Mypy statically guarantees that the end of the function is not + unreachable if there is not a return statement. + + This prevents the block formatter from being confused due to lack + of a leave and also leaves a nifty note in the IR. It is not + generally processed by visitors. + """ + + error_kind = ERR_NEVER + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def sources(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_unreachable(self) + + +class RegisterOp(Op): + """Abstract base class for operations that can be written as r1 = f(r2, ..., rn). + + Takes some values, performs an operation, and generates an output + (unless the 'type' attribute is void_rtype, which is the default). + Other ops can refer to the result of the Op by referring to the Op + instance. This doesn't do any explicit control flow, but can raise an + error. + + Note that the operands can be arbitrary Values, not just Register + instances, even though the naming may suggest otherwise. + """ + + error_kind = -1 # Can this raise exception and how is it signalled; one of ERR_* + + _type: RType | None = None + + def __init__(self, line: int) -> None: + super().__init__(line) + assert self.error_kind != -1, "error_kind not defined" + + def can_raise(self) -> bool: + return self.error_kind != ERR_NEVER + + +class IncRef(RegisterOp): + """Increase reference count (inc_ref src).""" + + error_kind = ERR_NEVER + + def __init__(self, src: Value, line: int = -1) -> None: + assert src.type.is_refcounted + super().__init__(line) + self.src = src + + def sources(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_inc_ref(self) + + +class DecRef(RegisterOp): + """Decrease reference count and free object if zero (dec_ref src). + + The is_xdec flag says to use an XDECREF, which checks if the + pointer is NULL first. + """ + + error_kind = ERR_NEVER + + def __init__(self, src: Value, is_xdec: bool = False, line: int = -1) -> None: + assert src.type.is_refcounted + super().__init__(line) + self.src = src + self.is_xdec = is_xdec + + def __repr__(self) -> str: + return "<{}DecRef {!r}>".format("X" if self.is_xdec else "", self.src) + + def sources(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_dec_ref(self) + + +class Call(RegisterOp): + """Native call f(arg, ...). + + The call target can be a module-level function or a class. + """ + + def __init__(self, fn: FuncDecl, args: Sequence[Value], line: int) -> None: + self.fn = fn + self.args = list(args) + assert len(self.args) == len(fn.sig.args) + self.type = fn.sig.ret_type + ret_type = fn.sig.ret_type + if not ret_type.error_overlap: + self.error_kind = ERR_MAGIC + else: + self.error_kind = ERR_MAGIC_OVERLAPPING + super().__init__(line) + + def sources(self) -> list[Value]: + return list(self.args.copy()) + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_call(self) + + +class MethodCall(RegisterOp): + """Native method call obj.method(arg, ...)""" + + def __init__(self, obj: Value, method: str, args: list[Value], line: int = -1) -> None: + self.obj = obj + self.method = method + self.args = args + assert isinstance(obj.type, RInstance), "Methods can only be called on instances" + self.receiver_type = obj.type + method_ir = self.receiver_type.class_ir.method_sig(method) + assert method_ir is not None, "{} doesn't have method {}".format( + self.receiver_type.name, method + ) + ret_type = method_ir.ret_type + self.type = ret_type + if not ret_type.error_overlap: + self.error_kind = ERR_MAGIC + else: + self.error_kind = ERR_MAGIC_OVERLAPPING + super().__init__(line) + + def sources(self) -> list[Value]: + return self.args.copy() + [self.obj] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_method_call(self) + + +class PrimitiveDescription: + """Description of a primitive op. + + Primitives get lowered into lower-level ops before code generation. + + If c_function_name is provided, a primitive will be lowered into a CallC op. + Otherwise custom logic will need to be implemented to transform the + primitive into lower-level ops. + """ + + def __init__( + self, + name: str, + arg_types: list[RType], + return_type: RType, # TODO: What about generic? + var_arg_type: RType | None, + truncated_type: RType | None, + c_function_name: str | None, + error_kind: int, + steals: StealsDescription, + is_borrowed: bool, + ordering: list[int] | None, + extra_int_constants: list[tuple[int, RType]], + priority: int, + is_pure: bool, + ) -> None: + # Each primitive much have a distinct name, but otherwise they are arbitrary. + self.name: Final = name + self.arg_types: Final = arg_types + self.return_type: Final = return_type + self.var_arg_type: Final = var_arg_type + self.truncated_type: Final = truncated_type + # If non-None, this will map to a call of a C helper function; if None, + # there must be a custom handler function that gets invoked during the lowering + # pass to generate low-level IR for the primitive (in the mypyc.lower package) + self.c_function_name: Final = c_function_name + self.error_kind: Final = error_kind + self.steals: Final = steals + self.is_borrowed: Final = is_borrowed + self.ordering: Final = ordering + self.extra_int_constants: Final = extra_int_constants + self.priority: Final = priority + # Pure primitives have no side effects, take immutable arguments, and + # never fail. They support additional optimizations. + self.is_pure: Final = is_pure + if is_pure: + assert error_kind == ERR_NEVER + + def __repr__(self) -> str: + return f"" + + +class PrimitiveOp(RegisterOp): + """A higher-level primitive operation. + + Some of these have special compiler support. These will be lowered + (transformed) into lower-level IR ops before code generation, and after + reference counting op insertion. Others will be transformed into CallC + ops. + + Tagged integer equality is a typical primitive op with non-trivial + lowering. It gets transformed into a tag check, followed by different + code paths for short and long representations. + """ + + def __init__(self, args: list[Value], desc: PrimitiveDescription, line: int = -1) -> None: + self.args = args + self.type = desc.return_type + self.error_kind = desc.error_kind + self.desc = desc + + def sources(self) -> list[Value]: + return self.args + + def stolen(self) -> list[Value]: + steals = self.desc.steals + if isinstance(steals, list): + assert len(steals) == len(self.args) + return [arg for arg, steal in zip(self.args, steals) if steal] + else: + return [] if not steals else self.sources() + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_primitive_op(self) + + +class LoadErrorValue(RegisterOp): + """Load an error value. + + Each type has one reserved value that signals an error (exception). This + loads the error value for a specific type. + """ + + error_kind = ERR_NEVER + + def __init__( + self, rtype: RType, line: int = -1, is_borrowed: bool = False, undefines: bool = False + ) -> None: + super().__init__(line) + self.type = rtype + self.is_borrowed = is_borrowed + # Undefines is true if this should viewed by the definedness + # analysis pass as making the register it is assigned to + # undefined (and thus checks should be added on uses). + self.undefines = undefines + + def sources(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_load_error_value(self) + + +class LoadLiteral(RegisterOp): + """Load a Python literal object (dest = 'foo' / b'foo' / ...). + + This is used to load a static PyObject * value corresponding to + a literal of one of the supported types. + + Tuple / frozenset literals must contain only valid literal values as items. + + NOTE: You can use this to load boxed (Python) int objects. Use + Integer to load unboxed, tagged integers or fixed-width, + low-level integers. + + For int literals, both int_rprimitive (CPyTagged) and + object_primitive (PyObject *) are supported as rtype. However, + when using int_rprimitive, the value must *not* be small enough + to fit in an unboxed integer. + """ + + error_kind = ERR_NEVER + is_borrowed = True + + def __init__(self, value: LiteralValue, rtype: RType) -> None: + self.value = value + self.type = rtype + + def sources(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_load_literal(self) + + +class GetAttr(RegisterOp): + """obj.attr (for a native object)""" + + error_kind = ERR_MAGIC + + def __init__(self, obj: Value, attr: str, line: int, *, borrow: bool = False) -> None: + super().__init__(line) + self.obj = obj + self.attr = attr + assert isinstance(obj.type, RInstance), "Attribute access not supported: %s" % obj.type + self.class_type = obj.type + attr_type = obj.type.attr_type(attr) + self.type = attr_type + if attr_type.error_overlap: + self.error_kind = ERR_MAGIC_OVERLAPPING + self.is_borrowed = borrow and attr_type.is_refcounted + + def sources(self) -> list[Value]: + return [self.obj] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_get_attr(self) + + +class SetAttr(RegisterOp): + """obj.attr = src (for a native object) + + Steals the reference to src. + """ + + error_kind = ERR_FALSE + + def __init__(self, obj: Value, attr: str, src: Value, line: int) -> None: + super().__init__(line) + self.obj = obj + self.attr = attr + self.src = src + assert isinstance(obj.type, RInstance), "Attribute access not supported: %s" % obj.type + self.class_type = obj.type + self.type = bool_rprimitive + # If True, we can safely assume that the attribute is previously undefined + # and we don't use a setter + self.is_init = False + + def mark_as_initializer(self) -> None: + self.is_init = True + self.error_kind = ERR_NEVER + self.type = void_rtype + + def sources(self) -> list[Value]: + return [self.obj, self.src] + + def stolen(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_set_attr(self) + + +# Default name space for statics, variables +NAMESPACE_STATIC: Final = "static" + +# Static namespace for pointers to native type objects +NAMESPACE_TYPE: Final = "type" + +# Namespace for modules +NAMESPACE_MODULE: Final = "module" + +# Namespace for Python 3.12 type variable objects (implicitly created TypeVar instances, etc.) +NAMESPACE_TYPE_VAR: Final = "typevar" + + +class LoadStatic(RegisterOp): + """Load a static name (name :: static). + + Load a C static variable/pointer. The namespace for statics is shared + for the entire compilation group. You can optionally provide a module + name and a sub-namespace identifier for additional namespacing to avoid + name conflicts. The static namespace does not overlap with other C names, + since the final C name will get a prefix, so conflicts only must be + avoided with other statics. + """ + + error_kind = ERR_NEVER + is_borrowed = True + + def __init__( + self, + type: RType, + identifier: str, + module_name: str | None = None, + namespace: str = NAMESPACE_STATIC, + line: int = -1, + ann: object = None, + ) -> None: + super().__init__(line) + self.identifier = identifier + self.module_name = module_name + self.namespace = namespace + self.type = type + self.ann = ann # An object to pretty print with the load + + def sources(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_load_static(self) + + +class InitStatic(RegisterOp): + """static = value :: static + + Initialize a C static variable/pointer. See everything in LoadStatic. + """ + + error_kind = ERR_NEVER + + def __init__( + self, + value: Value, + identifier: str, + module_name: str | None = None, + namespace: str = NAMESPACE_STATIC, + line: int = -1, + ) -> None: + super().__init__(line) + self.identifier = identifier + self.module_name = module_name + self.namespace = namespace + self.value = value + + def sources(self) -> list[Value]: + return [self.value] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_init_static(self) + + +class TupleSet(RegisterOp): + """dest = (reg, ...) (for fixed-length tuple)""" + + error_kind = ERR_NEVER + + def __init__(self, items: list[Value], line: int) -> None: + super().__init__(line) + self.items = items + # Don't keep track of the fact that an int is short after it + # is put into a tuple, since we don't properly implement + # runtime subtyping for tuples. + self.tuple_type = RTuple( + [ + arg.type if not is_short_int_rprimitive(arg.type) else int_rprimitive + for arg in items + ] + ) + self.type = self.tuple_type + + def sources(self) -> list[Value]: + return self.items.copy() + + def stolen(self) -> list[Value]: + return self.items.copy() + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_tuple_set(self) + + +class TupleGet(RegisterOp): + """Get item of a fixed-length tuple (src[index]).""" + + error_kind = ERR_NEVER + + def __init__(self, src: Value, index: int, line: int = -1, *, borrow: bool = False) -> None: + super().__init__(line) + self.src = src + self.index = index + assert isinstance(src.type, RTuple), "TupleGet only operates on tuples" + assert index >= 0 + self.type = src.type.types[index] + self.is_borrowed = borrow + + def sources(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_tuple_get(self) + + +class Cast(RegisterOp): + """cast(type, src) + + Perform a runtime type check (no representation or value conversion). + + DO NOT increment reference counts. + """ + + error_kind = ERR_MAGIC + + def __init__(self, src: Value, typ: RType, line: int, *, borrow: bool = False) -> None: + super().__init__(line) + self.src = src + self.type = typ + self.is_borrowed = borrow + + def sources(self) -> list[Value]: + return [self.src] + + def stolen(self) -> list[Value]: + if self.is_borrowed: + return [] + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_cast(self) + + +class Box(RegisterOp): + """box(type, src) + + This converts from a potentially unboxed representation to a straight Python object. + Only supported for types with an unboxed representation. + """ + + error_kind = ERR_NEVER + + def __init__(self, src: Value, line: int = -1) -> None: + super().__init__(line) + self.src = src + self.type = object_rprimitive + # When we box None and bool values, we produce a borrowed result + if ( + is_none_rprimitive(self.src.type) + or is_bool_rprimitive(self.src.type) + or is_bit_rprimitive(self.src.type) + ): + self.is_borrowed = True + + def sources(self) -> list[Value]: + return [self.src] + + def stolen(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_box(self) + + +class Unbox(RegisterOp): + """unbox(type, src) + + This is similar to a cast, but it also changes to a (potentially) unboxed runtime + representation. Only supported for types with an unboxed representation. + """ + + def __init__(self, src: Value, typ: RType, line: int) -> None: + self.src = src + self.type = typ + if not typ.error_overlap: + self.error_kind = ERR_MAGIC + else: + self.error_kind = ERR_MAGIC_OVERLAPPING + super().__init__(line) + + def sources(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_unbox(self) + + +class RaiseStandardError(RegisterOp): + """Raise built-in exception with an optional error string. + + We have a separate opcode for this for convenience and to + generate smaller, more idiomatic C code. + """ + + # TODO: Make it more explicit at IR level that this always raises + + error_kind = ERR_FALSE + + VALUE_ERROR: Final = "ValueError" + ASSERTION_ERROR: Final = "AssertionError" + STOP_ITERATION: Final = "StopIteration" + UNBOUND_LOCAL_ERROR: Final = "UnboundLocalError" + RUNTIME_ERROR: Final = "RuntimeError" + NAME_ERROR: Final = "NameError" + ZERO_DIVISION_ERROR: Final = "ZeroDivisionError" + + def __init__(self, class_name: str, value: str | Value | None, line: int) -> None: + super().__init__(line) + self.class_name = class_name + self.value = value + self.type = bool_rprimitive + + def sources(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_raise_standard_error(self) + + +# True steals all arguments, False steals none, a list steals those in matching positions +StealsDescription = Union[bool, List[bool]] + + +class CallC(RegisterOp): + """result = function(arg0, arg1, ...) + + Call a C function that is not a compiled/native function (for + example, a Python C API function). Use Call to call native + functions. + """ + + def __init__( + self, + function_name: str, + args: list[Value], + ret_type: RType, + steals: StealsDescription, + is_borrowed: bool, + error_kind: int, + line: int, + var_arg_idx: int = -1, + *, + is_pure: bool = False, + ) -> None: + self.error_kind = error_kind + super().__init__(line) + self.function_name = function_name + self.args = args + self.type = ret_type + self.steals = steals + self.is_borrowed = is_borrowed + # The position of the first variable argument in args (if >= 0) + self.var_arg_idx = var_arg_idx + # Is the function pure? Pure functions have no side effects + # and all the arguments are immutable. Pure functions support + # additional optimizations. Pure functions never fail. + self.is_pure = is_pure + if is_pure: + assert error_kind == ERR_NEVER + + def sources(self) -> list[Value]: + return self.args + + def stolen(self) -> list[Value]: + if isinstance(self.steals, list): + assert len(self.steals) == len(self.args) + return [arg for arg, steal in zip(self.args, self.steals) if steal] + else: + return [] if not self.steals else self.sources() + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_call_c(self) + + +class Truncate(RegisterOp): + """result = truncate src from src_type to dst_type + + Truncate a value from type with more bits to type with less bits. + + dst_type and src_type can be native integer types, bools or tagged + integers. Tagged integers should have the tag bit unset. + """ + + error_kind = ERR_NEVER + + def __init__(self, src: Value, dst_type: RType, line: int = -1) -> None: + super().__init__(line) + self.src = src + self.type = dst_type + self.src_type = src.type + + def sources(self) -> list[Value]: + return [self.src] + + def stolen(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_truncate(self) + + +class Extend(RegisterOp): + """result = extend src from src_type to dst_type + + Extend a value from a type with fewer bits to a type with more bits. + + dst_type and src_type can be native integer types, bools or tagged + integers. Tagged integers should have the tag bit unset. + + If 'signed' is true, perform sign extension. Otherwise, the result will be + zero extended. + """ + + error_kind = ERR_NEVER + + def __init__(self, src: Value, dst_type: RType, signed: bool, line: int = -1) -> None: + super().__init__(line) + self.src = src + self.type = dst_type + self.src_type = src.type + self.signed = signed + + def sources(self) -> list[Value]: + return [self.src] + + def stolen(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_extend(self) + + +class LoadGlobal(RegisterOp): + """Load a low-level global variable/pointer. + + Note that can't be used to directly load Python module-level + global variable, since they are stored in a globals dictionary + and accessed using dictionary operations. + """ + + error_kind = ERR_NEVER + is_borrowed = True + + def __init__(self, type: RType, identifier: str, line: int = -1, ann: object = None) -> None: + super().__init__(line) + self.identifier = identifier + self.type = type + self.ann = ann # An object to pretty print with the load + + def sources(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_load_global(self) + + +class IntOp(RegisterOp): + """Binary arithmetic or bitwise op on integer operands (e.g., r1 = r2 + r3). + + These ops are low-level and are similar to the corresponding C + operations. + + The left and right values must have low-level integer types with + compatible representations. Fixed-width integers, short_int_rprimitive, + bool_rprimitive and bit_rprimitive are supported. + + For tagged (arbitrary-precision) integer ops look at mypyc.primitives.int_ops. + """ + + error_kind = ERR_NEVER + + # Arithmetic ops + ADD: Final = 0 + SUB: Final = 1 + MUL: Final = 2 + DIV: Final = 3 + MOD: Final = 4 + + # Bitwise ops + AND: Final = 200 + OR: Final = 201 + XOR: Final = 202 + LEFT_SHIFT: Final = 203 + RIGHT_SHIFT: Final = 204 + + op_str: Final = { + ADD: "+", + SUB: "-", + MUL: "*", + DIV: "/", + MOD: "%", + AND: "&", + OR: "|", + XOR: "^", + LEFT_SHIFT: "<<", + RIGHT_SHIFT: ">>", + } + + def __init__(self, type: RType, lhs: Value, rhs: Value, op: int, line: int = -1) -> None: + super().__init__(line) + self.type = type + self.lhs = lhs + self.rhs = rhs + self.op = op + + def sources(self) -> list[Value]: + return [self.lhs, self.rhs] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_int_op(self) + + +# We can't have this in the IntOp class body, because of +# https://github.com/mypyc/mypyc/issues/932. +int_op_to_id: Final = {op: op_id for op_id, op in IntOp.op_str.items()} + + +class ComparisonOp(RegisterOp): + """Low-level comparison op for integers and pointers. + + Both unsigned and signed comparisons are supported. Supports + comparisons between fixed-width integer types and pointer types. + The operands should have matching sizes. + + The result is always a bit (representing a boolean). + + Python semantics, such as calling __eq__, are not supported. + """ + + # Must be ERR_NEVER or ERR_FALSE. ERR_FALSE means that a false result + # indicates that an exception has been raised and should be propagated. + error_kind = ERR_NEVER + + # S for signed and U for unsigned + EQ: Final = 100 + NEQ: Final = 101 + SLT: Final = 102 + SGT: Final = 103 + SLE: Final = 104 + SGE: Final = 105 + ULT: Final = 106 + UGT: Final = 107 + ULE: Final = 108 + UGE: Final = 109 + + op_str: Final = { + EQ: "==", + NEQ: "!=", + SLT: "<", + SGT: ">", + SLE: "<=", + SGE: ">=", + ULT: "<", + UGT: ">", + ULE: "<=", + UGE: ">=", + } + + signed_ops: Final = {"==": EQ, "!=": NEQ, "<": SLT, ">": SGT, "<=": SLE, ">=": SGE} + unsigned_ops: Final = {"==": EQ, "!=": NEQ, "<": ULT, ">": UGT, "<=": ULE, ">=": UGE} + + def __init__(self, lhs: Value, rhs: Value, op: int, line: int = -1) -> None: + super().__init__(line) + self.type = bit_rprimitive + self.lhs = lhs + self.rhs = rhs + self.op = op + + def sources(self) -> list[Value]: + return [self.lhs, self.rhs] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_comparison_op(self) + + +class FloatOp(RegisterOp): + """Binary float arithmetic op (e.g., r1 = r2 + r3). + + These ops are low-level and are similar to the corresponding C + operations (and somewhat different from Python operations). + + The left and right values must be floats. + """ + + error_kind = ERR_NEVER + + ADD: Final = 0 + SUB: Final = 1 + MUL: Final = 2 + DIV: Final = 3 + MOD: Final = 4 + + op_str: Final = {ADD: "+", SUB: "-", MUL: "*", DIV: "/", MOD: "%"} + + def __init__(self, lhs: Value, rhs: Value, op: int, line: int = -1) -> None: + super().__init__(line) + self.type = float_rprimitive + self.lhs = lhs + self.rhs = rhs + self.op = op + + def sources(self) -> list[Value]: + return [self.lhs, self.rhs] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_float_op(self) + + +# We can't have this in the FloatOp class body, because of +# https://github.com/mypyc/mypyc/issues/932. +float_op_to_id: Final = {op: op_id for op_id, op in FloatOp.op_str.items()} + + +class FloatNeg(RegisterOp): + """Float negation op (r1 = -r2).""" + + error_kind = ERR_NEVER + + def __init__(self, src: Value, line: int = -1) -> None: + super().__init__(line) + self.type = float_rprimitive + self.src = src + + def sources(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_float_neg(self) + + +class FloatComparisonOp(RegisterOp): + """Low-level comparison op for floats.""" + + error_kind = ERR_NEVER + + EQ: Final = 200 + NEQ: Final = 201 + LT: Final = 202 + GT: Final = 203 + LE: Final = 204 + GE: Final = 205 + + op_str: Final = {EQ: "==", NEQ: "!=", LT: "<", GT: ">", LE: "<=", GE: ">="} + + def __init__(self, lhs: Value, rhs: Value, op: int, line: int = -1) -> None: + super().__init__(line) + self.type = bit_rprimitive + self.lhs = lhs + self.rhs = rhs + self.op = op + + def sources(self) -> list[Value]: + return [self.lhs, self.rhs] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_float_comparison_op(self) + + +# We can't have this in the FloatOp class body, because of +# https://github.com/mypyc/mypyc/issues/932. +float_comparison_op_to_id: Final = {op: op_id for op_id, op in FloatComparisonOp.op_str.items()} + + +class LoadMem(RegisterOp): + """Read a memory location: result = *(type *)src. + + Attributes: + type: Type of the read value + src: Pointer to memory to read + """ + + error_kind = ERR_NEVER + + def __init__(self, type: RType, src: Value, line: int = -1) -> None: + super().__init__(line) + self.type = type + # TODO: for now we enforce that the src memory address should be Py_ssize_t + # later we should also support same width unsigned int + assert is_pointer_rprimitive(src.type) + self.src = src + self.is_borrowed = True + + def sources(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_load_mem(self) + + +class SetMem(Op): + """Write to a memory location: *(type *)dest = src + + Attributes: + type: Type of the written value + dest: Pointer to memory to write + src: Source value + """ + + error_kind = ERR_NEVER + + def __init__(self, type: RType, dest: Value, src: Value, line: int = -1) -> None: + super().__init__(line) + self.type = void_rtype + self.dest_type = type + self.src = src + self.dest = dest + + def sources(self) -> list[Value]: + return [self.src, self.dest] + + def stolen(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_set_mem(self) + + +class GetElementPtr(RegisterOp): + """Get the address of a struct element. + + Note that you may need to use KeepAlive to avoid the struct + being freed, if it's reference counted, such as PyObject *. + """ + + error_kind = ERR_NEVER + + def __init__(self, src: Value, src_type: RType, field: str, line: int = -1) -> None: + super().__init__(line) + self.type = pointer_rprimitive + self.src = src + self.src_type = src_type + self.field = field + + def sources(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_get_element_ptr(self) + + +class LoadAddress(RegisterOp): + """Get the address of a value: result = (type)&src + + Attributes: + type: Type of the loaded address(e.g. ptr/object_ptr) + src: Source value (str for globals like 'PyList_Type', + Register for temporary values or locals, LoadStatic + for statics.) + """ + + error_kind = ERR_NEVER + is_borrowed = True + + def __init__(self, type: RType, src: str | Register | LoadStatic, line: int = -1) -> None: + super().__init__(line) + self.type = type + self.src = src + + def sources(self) -> list[Value]: + if isinstance(self.src, Register): + return [self.src] + else: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_load_address(self) + + +class KeepAlive(RegisterOp): + """A no-op operation that ensures source values aren't freed. + + This is sometimes useful to avoid decref when a reference is still + being held but not seen by the compiler. + + A typical use case is like this (C-like pseudocode): + + ptr = &x.item + r = *ptr + keep_alive x # x must not be freed here + # x may be freed here + + If we didn't have "keep_alive x", x could be freed immediately + after taking the address of 'item', resulting in a read after free + on the second line. + + If 'steal' is true, the value is considered to be stolen at + this op, i.e. it won't be decref'd. You need to ensure that + the value is freed otherwise, perhaps by using borrowing + followed by Unborrow. + + Be careful with steal=True -- this can cause memory leaks. + """ + + error_kind = ERR_NEVER + + def __init__(self, src: list[Value], *, steal: bool = False) -> None: + assert src + self.src = src + self.steal = steal + + def sources(self) -> list[Value]: + return self.src.copy() + + def stolen(self) -> list[Value]: + if self.steal: + return self.src.copy() + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_keep_alive(self) + + +class Unborrow(RegisterOp): + """A no-op op to create a regular reference from a borrowed one. + + Borrowed references can only be used temporarily and the reference + counts won't be managed. This value will be refcounted normally. + + This is mainly useful if you split an aggregate value, such as + a tuple, into components using borrowed values (to avoid increfs), + and want to treat the components as sharing the original managed + reference. You'll also need to use KeepAlive with steal=True to + "consume" the original tuple reference: + + # t is a 2-tuple + r0 = borrow t[0] + r1 = borrow t[1] + r2 = unborrow r0 + r3 = unborrow r1 + # now (r2, r3) represent the tuple as separate items, and the + # original tuple can be considered dead and available to be + # stolen + keep_alive steal t + + Be careful with this -- this can easily cause double freeing. + """ + + error_kind = ERR_NEVER + + def __init__(self, src: Value, line: int = -1) -> None: + super().__init__(line) + assert src.is_borrowed + self.src = src + self.type = src.type + + def sources(self) -> list[Value]: + return [self.src] + + def stolen(self) -> list[Value]: + return [] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_unborrow(self) + + +@trait +class OpVisitor(Generic[T]): + """Generic visitor over ops (uses the visitor design pattern).""" + + @abstractmethod + def visit_goto(self, op: Goto) -> T: + raise NotImplementedError + + @abstractmethod + def visit_branch(self, op: Branch) -> T: + raise NotImplementedError + + @abstractmethod + def visit_return(self, op: Return) -> T: + raise NotImplementedError + + @abstractmethod + def visit_unreachable(self, op: Unreachable) -> T: + raise NotImplementedError + + @abstractmethod + def visit_assign(self, op: Assign) -> T: + raise NotImplementedError + + @abstractmethod + def visit_assign_multi(self, op: AssignMulti) -> T: + raise NotImplementedError + + @abstractmethod + def visit_load_error_value(self, op: LoadErrorValue) -> T: + raise NotImplementedError + + @abstractmethod + def visit_load_literal(self, op: LoadLiteral) -> T: + raise NotImplementedError + + @abstractmethod + def visit_get_attr(self, op: GetAttr) -> T: + raise NotImplementedError + + @abstractmethod + def visit_set_attr(self, op: SetAttr) -> T: + raise NotImplementedError + + @abstractmethod + def visit_load_static(self, op: LoadStatic) -> T: + raise NotImplementedError + + @abstractmethod + def visit_init_static(self, op: InitStatic) -> T: + raise NotImplementedError + + @abstractmethod + def visit_tuple_get(self, op: TupleGet) -> T: + raise NotImplementedError + + @abstractmethod + def visit_tuple_set(self, op: TupleSet) -> T: + raise NotImplementedError + + def visit_inc_ref(self, op: IncRef) -> T: + raise NotImplementedError + + def visit_dec_ref(self, op: DecRef) -> T: + raise NotImplementedError + + @abstractmethod + def visit_call(self, op: Call) -> T: + raise NotImplementedError + + @abstractmethod + def visit_method_call(self, op: MethodCall) -> T: + raise NotImplementedError + + @abstractmethod + def visit_cast(self, op: Cast) -> T: + raise NotImplementedError + + @abstractmethod + def visit_box(self, op: Box) -> T: + raise NotImplementedError + + @abstractmethod + def visit_unbox(self, op: Unbox) -> T: + raise NotImplementedError + + @abstractmethod + def visit_raise_standard_error(self, op: RaiseStandardError) -> T: + raise NotImplementedError + + @abstractmethod + def visit_call_c(self, op: CallC) -> T: + raise NotImplementedError + + @abstractmethod + def visit_primitive_op(self, op: PrimitiveOp) -> T: + raise NotImplementedError + + @abstractmethod + def visit_truncate(self, op: Truncate) -> T: + raise NotImplementedError + + @abstractmethod + def visit_extend(self, op: Extend) -> T: + raise NotImplementedError + + @abstractmethod + def visit_load_global(self, op: LoadGlobal) -> T: + raise NotImplementedError + + @abstractmethod + def visit_int_op(self, op: IntOp) -> T: + raise NotImplementedError + + @abstractmethod + def visit_comparison_op(self, op: ComparisonOp) -> T: + raise NotImplementedError + + @abstractmethod + def visit_float_op(self, op: FloatOp) -> T: + raise NotImplementedError + + @abstractmethod + def visit_float_neg(self, op: FloatNeg) -> T: + raise NotImplementedError + + @abstractmethod + def visit_float_comparison_op(self, op: FloatComparisonOp) -> T: + raise NotImplementedError + + @abstractmethod + def visit_load_mem(self, op: LoadMem) -> T: + raise NotImplementedError + + @abstractmethod + def visit_set_mem(self, op: SetMem) -> T: + raise NotImplementedError + + @abstractmethod + def visit_get_element_ptr(self, op: GetElementPtr) -> T: + raise NotImplementedError + + @abstractmethod + def visit_load_address(self, op: LoadAddress) -> T: + raise NotImplementedError + + @abstractmethod + def visit_keep_alive(self, op: KeepAlive) -> T: + raise NotImplementedError + + @abstractmethod + def visit_unborrow(self, op: Unborrow) -> T: + raise NotImplementedError + + +# TODO: Should the following definition live somewhere else? + + +# We do a three-pass deserialization scheme in order to resolve name +# references. +# 1. Create an empty ClassIR for each class in an SCC. +# 2. Deserialize all of the functions, which can contain references +# to ClassIRs in their types +# 3. Deserialize all of the classes, which contain lots of references +# to the functions they contain. (And to other classes.) +# +# Note that this approach differs from how we deserialize ASTs in mypy itself, +# where everything is deserialized in one pass then a second pass cleans up +# 'cross_refs'. We don't follow that approach here because it seems to be more +# code for not a lot of gain since it is easy in mypyc to identify all the objects +# we might need to reference. +# +# Because of these references, we need to maintain maps from class +# names to ClassIRs and func IDs to FuncIRs. +# +# These are tracked in a DeserMaps which is passed to every +# deserialization function. +# +# (Serialization and deserialization *will* be used for incremental +# compilation but so far it is not hooked up to anything.) +class DeserMaps(NamedTuple): + classes: dict[str, ClassIR] + functions: dict[str, FuncIR] diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py new file mode 100644 index 000000000000..59ee994f012d --- /dev/null +++ b/mypyc/ir/pprint.py @@ -0,0 +1,516 @@ +"""Utilities for pretty-printing IR in a human-readable form.""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Any, Final, Sequence, Union + +from mypyc.common import short_name +from mypyc.ir.func_ir import FuncIR, all_values_full +from mypyc.ir.module_ir import ModuleIRs +from mypyc.ir.ops import ( + ERR_NEVER, + Assign, + AssignMulti, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + ControlOp, + DecRef, + Extend, + Float, + FloatComparisonOp, + FloatNeg, + FloatOp, + GetAttr, + GetElementPtr, + Goto, + IncRef, + InitStatic, + Integer, + IntOp, + KeepAlive, + LoadAddress, + LoadErrorValue, + LoadGlobal, + LoadLiteral, + LoadMem, + LoadStatic, + MethodCall, + Op, + OpVisitor, + PrimitiveOp, + RaiseStandardError, + Register, + Return, + SetAttr, + SetMem, + Truncate, + TupleGet, + TupleSet, + Unborrow, + Unbox, + Unreachable, + Value, +) +from mypyc.ir.rtypes import RType, is_bool_rprimitive, is_int_rprimitive + +ErrorSource = Union[BasicBlock, Op] + + +class IRPrettyPrintVisitor(OpVisitor[str]): + """Internal visitor that pretty-prints ops.""" + + def __init__(self, names: dict[Value, str]) -> None: + # This should contain a name for all values that are shown as + # registers in the output. This is not just for Register + # instances -- all Ops that produce values need (generated) names. + self.names = names + + def visit_goto(self, op: Goto) -> str: + return self.format("goto %l", op.label) + + branch_op_names: Final = {Branch.BOOL: ("%r", "bool"), Branch.IS_ERROR: ("is_error(%r)", "")} + + def visit_branch(self, op: Branch) -> str: + fmt, typ = self.branch_op_names[op.op] + if op.negated: + fmt = f"not {fmt}" + + cond = self.format(fmt, op.value) + tb = "" + if op.traceback_entry: + tb = " (error at %s:%d)" % op.traceback_entry + fmt = f"if {cond} goto %l{tb} else goto %l" + if typ: + fmt += f" :: {typ}" + return self.format(fmt, op.true, op.false) + + def visit_return(self, op: Return) -> str: + return self.format("return %r", op.value) + + def visit_unreachable(self, op: Unreachable) -> str: + return "unreachable" + + def visit_assign(self, op: Assign) -> str: + return self.format("%r = %r", op.dest, op.src) + + def visit_assign_multi(self, op: AssignMulti) -> str: + return self.format("%r = [%s]", op.dest, ", ".join(self.format("%r", v) for v in op.src)) + + def visit_load_error_value(self, op: LoadErrorValue) -> str: + return self.format("%r = :: %s", op, op.type) + + def visit_load_literal(self, op: LoadLiteral) -> str: + prefix = "" + # For values that have a potential unboxed representation, make + # it explicit that this is a Python object. + if isinstance(op.value, int): + prefix = "object " + + rvalue = repr(op.value) + if isinstance(op.value, frozenset): + # We need to generate a string representation that won't vary + # run-to-run because sets are unordered, otherwise we may get + # spurious irbuild test failures. + # + # Sorting by the item's string representation is a bit of a + # hack, but it's stable and won't cause TypeErrors. + formatted_items = [repr(i) for i in sorted(op.value, key=str)] + rvalue = "frozenset({" + ", ".join(formatted_items) + "})" + return self.format("%r = %s%s", op, prefix, rvalue) + + def visit_get_attr(self, op: GetAttr) -> str: + return self.format("%r = %s%r.%s", op, self.borrow_prefix(op), op.obj, op.attr) + + def borrow_prefix(self, op: Op) -> str: + if op.is_borrowed: + return "borrow " + return "" + + def visit_set_attr(self, op: SetAttr) -> str: + if op.is_init: + assert op.error_kind == ERR_NEVER + if op.error_kind == ERR_NEVER: + # Initialization and direct struct access can never fail + return self.format("%r.%s = %r", op.obj, op.attr, op.src) + else: + return self.format("%r.%s = %r; %r = is_error", op.obj, op.attr, op.src, op) + + def visit_load_static(self, op: LoadStatic) -> str: + ann = f" ({repr(op.ann)})" if op.ann else "" + name = op.identifier + if op.module_name is not None: + name = f"{op.module_name}.{name}" + return self.format("%r = %s :: %s%s", op, name, op.namespace, ann) + + def visit_init_static(self, op: InitStatic) -> str: + name = op.identifier + if op.module_name is not None: + name = f"{op.module_name}.{name}" + return self.format("%s = %r :: %s", name, op.value, op.namespace) + + def visit_tuple_get(self, op: TupleGet) -> str: + return self.format("%r = %s%r[%d]", op, self.borrow_prefix(op), op.src, op.index) + + def visit_tuple_set(self, op: TupleSet) -> str: + item_str = ", ".join(self.format("%r", item) for item in op.items) + return self.format("%r = (%s)", op, item_str) + + def visit_inc_ref(self, op: IncRef) -> str: + s = self.format("inc_ref %r", op.src) + # TODO: Remove bool check (it's unboxed) + if is_bool_rprimitive(op.src.type) or is_int_rprimitive(op.src.type): + s += f" :: {short_name(op.src.type.name)}" + return s + + def visit_dec_ref(self, op: DecRef) -> str: + s = self.format("%sdec_ref %r", "x" if op.is_xdec else "", op.src) + # TODO: Remove bool check (it's unboxed) + if is_bool_rprimitive(op.src.type) or is_int_rprimitive(op.src.type): + s += f" :: {short_name(op.src.type.name)}" + return s + + def visit_call(self, op: Call) -> str: + args = ", ".join(self.format("%r", arg) for arg in op.args) + # TODO: Display long name? + short_name = op.fn.shortname + s = f"{short_name}({args})" + if not op.is_void: + s = self.format("%r = ", op) + s + return s + + def visit_method_call(self, op: MethodCall) -> str: + args = ", ".join(self.format("%r", arg) for arg in op.args) + s = self.format("%r.%s(%s)", op.obj, op.method, args) + if not op.is_void: + s = self.format("%r = ", op) + s + return s + + def visit_cast(self, op: Cast) -> str: + return self.format("%r = %scast(%s, %r)", op, self.borrow_prefix(op), op.type, op.src) + + def visit_box(self, op: Box) -> str: + return self.format("%r = box(%s, %r)", op, op.src.type, op.src) + + def visit_unbox(self, op: Unbox) -> str: + return self.format("%r = unbox(%s, %r)", op, op.type, op.src) + + def visit_raise_standard_error(self, op: RaiseStandardError) -> str: + if op.value is not None: + if isinstance(op.value, str): + return self.format("%r = raise %s(%s)", op, op.class_name, repr(op.value)) + elif isinstance(op.value, Value): + return self.format("%r = raise %s(%r)", op, op.class_name, op.value) + else: + assert False, "value type must be either str or Value" + else: + return self.format("%r = raise %s", op, op.class_name) + + def visit_call_c(self, op: CallC) -> str: + args_str = ", ".join(self.format("%r", arg) for arg in op.args) + if op.is_void: + return self.format("%s(%s)", op.function_name, args_str) + else: + return self.format("%r = %s(%s)", op, op.function_name, args_str) + + def visit_primitive_op(self, op: PrimitiveOp) -> str: + args = [] + arg_index = 0 + type_arg_index = 0 + for arg_type in zip(op.desc.arg_types): + if arg_type: + args.append(self.format("%r", op.args[arg_index])) + arg_index += 1 + else: + assert op.type_args + args.append(self.format("%r", op.type_args[type_arg_index])) + type_arg_index += 1 + + args_str = ", ".join(args) + if op.is_void: + return self.format("%s %s", op.desc.name, args_str) + else: + return self.format("%r = %s %s", op, op.desc.name, args_str) + + def visit_truncate(self, op: Truncate) -> str: + return self.format("%r = truncate %r: %t to %t", op, op.src, op.src_type, op.type) + + def visit_extend(self, op: Extend) -> str: + if op.signed: + extra = " signed" + else: + extra = "" + return self.format("%r = extend%s %r: %t to %t", op, extra, op.src, op.src_type, op.type) + + def visit_load_global(self, op: LoadGlobal) -> str: + ann = f" ({repr(op.ann)})" if op.ann else "" + return self.format("%r = load_global %s :: static%s", op, op.identifier, ann) + + def visit_int_op(self, op: IntOp) -> str: + return self.format("%r = %r %s %r", op, op.lhs, IntOp.op_str[op.op], op.rhs) + + def visit_comparison_op(self, op: ComparisonOp) -> str: + if op.op in (ComparisonOp.SLT, ComparisonOp.SGT, ComparisonOp.SLE, ComparisonOp.SGE): + sign_format = " :: signed" + elif op.op in (ComparisonOp.ULT, ComparisonOp.UGT, ComparisonOp.ULE, ComparisonOp.UGE): + sign_format = " :: unsigned" + else: + sign_format = "" + return self.format( + "%r = %r %s %r%s", op, op.lhs, ComparisonOp.op_str[op.op], op.rhs, sign_format + ) + + def visit_float_op(self, op: FloatOp) -> str: + return self.format("%r = %r %s %r", op, op.lhs, FloatOp.op_str[op.op], op.rhs) + + def visit_float_neg(self, op: FloatNeg) -> str: + return self.format("%r = -%r", op, op.src) + + def visit_float_comparison_op(self, op: FloatComparisonOp) -> str: + return self.format("%r = %r %s %r", op, op.lhs, op.op_str[op.op], op.rhs) + + def visit_load_mem(self, op: LoadMem) -> str: + return self.format("%r = load_mem %r :: %t*", op, op.src, op.type) + + def visit_set_mem(self, op: SetMem) -> str: + return self.format("set_mem %r, %r :: %t*", op.dest, op.src, op.dest_type) + + def visit_get_element_ptr(self, op: GetElementPtr) -> str: + return self.format("%r = get_element_ptr %r %s :: %t", op, op.src, op.field, op.src_type) + + def visit_load_address(self, op: LoadAddress) -> str: + if isinstance(op.src, Register): + return self.format("%r = load_address %r", op, op.src) + elif isinstance(op.src, LoadStatic): + name = op.src.identifier + if op.src.module_name is not None: + name = f"{op.src.module_name}.{name}" + return self.format("%r = load_address %s :: %s", op, name, op.src.namespace) + else: + return self.format("%r = load_address %s", op, op.src) + + def visit_keep_alive(self, op: KeepAlive) -> str: + if op.steal: + steal = "steal " + else: + steal = "" + return self.format( + "keep_alive {}{}".format(steal, ", ".join(self.format("%r", v) for v in op.src)) + ) + + def visit_unborrow(self, op: Unborrow) -> str: + return self.format("%r = unborrow %r", op, op.src) + + # Helpers + + def format(self, fmt: str, *args: Any) -> str: + """Helper for formatting strings. + + These format sequences are supported in fmt: + + %s: arbitrary object converted to string using str() + %r: name of IR value/register + %d: int + %f: float + %l: BasicBlock (formatted as label 'Ln') + %t: RType + """ + result = [] + i = 0 + arglist = list(args) + while i < len(fmt): + n = fmt.find("%", i) + if n < 0: + n = len(fmt) + result.append(fmt[i:n]) + if n < len(fmt): + typespec = fmt[n + 1] + arg = arglist.pop(0) + if typespec == "r": + # Register/value + assert isinstance(arg, Value) + if isinstance(arg, Integer): + result.append(str(arg.value)) + elif isinstance(arg, Float): + result.append(repr(arg.value)) + else: + result.append(self.names[arg]) + elif typespec == "d": + # Integer + result.append("%d" % arg) + elif typespec == "f": + # Float + result.append("%f" % arg) + elif typespec == "l": + # Basic block (label) + assert isinstance(arg, BasicBlock) + result.append("L%s" % arg.label) + elif typespec == "t": + # RType + assert isinstance(arg, RType) + result.append(arg.name) + elif typespec == "s": + # String + result.append(str(arg)) + else: + raise ValueError(f"Invalid format sequence %{typespec}") + i = n + 2 + else: + i = n + return "".join(result) + + +def format_registers(func_ir: FuncIR, names: dict[Value, str]) -> list[str]: + result = [] + i = 0 + regs = all_values_full(func_ir.arg_regs, func_ir.blocks) + while i < len(regs): + i0 = i + group = [names[regs[i0]]] + while i + 1 < len(regs) and regs[i + 1].type == regs[i0].type: + i += 1 + group.append(names[regs[i]]) + i += 1 + result.append("{} :: {}".format(", ".join(group), regs[i0].type)) + return result + + +def format_blocks( + blocks: list[BasicBlock], + names: dict[Value, str], + source_to_error: dict[ErrorSource, list[str]], +) -> list[str]: + """Format a list of IR basic blocks into a human-readable form.""" + # First label all of the blocks + for i, block in enumerate(blocks): + block.label = i + + handler_map: dict[BasicBlock, list[BasicBlock]] = {} + for b in blocks: + if b.error_handler: + handler_map.setdefault(b.error_handler, []).append(b) + + visitor = IRPrettyPrintVisitor(names) + + lines = [] + for i, block in enumerate(blocks): + handler_msg = "" + if block in handler_map: + labels = sorted("L%d" % b.label for b in handler_map[block]) + handler_msg = " (handler for {})".format(", ".join(labels)) + + lines.append("L%d:%s" % (block.label, handler_msg)) + if block in source_to_error: + for error in source_to_error[block]: + lines.append(f" ERR: {error}") + ops = block.ops + if ( + isinstance(ops[-1], Goto) + and i + 1 < len(blocks) + and ops[-1].label == blocks[i + 1] + and not source_to_error.get(ops[-1], []) + ): + # Hide the last goto if it just goes to the next basic block, + # and there are no assocatiated errors with the op. + ops = ops[:-1] + for op in ops: + line = " " + op.accept(visitor) + lines.append(line) + if op in source_to_error: + for error in source_to_error[op]: + lines.append(f" ERR: {error}") + + if not isinstance(block.ops[-1], (Goto, Branch, Return, Unreachable)): + # Each basic block needs to exit somewhere. + lines.append(" [MISSING BLOCK EXIT OPCODE]") + return lines + + +def format_func(fn: FuncIR, errors: Sequence[tuple[ErrorSource, str]] = ()) -> list[str]: + lines = [] + cls_prefix = fn.class_name + "." if fn.class_name else "" + lines.append( + "def {}{}({}):".format(cls_prefix, fn.name, ", ".join(arg.name for arg in fn.args)) + ) + names = generate_names_for_ir(fn.arg_regs, fn.blocks) + for line in format_registers(fn, names): + lines.append(" " + line) + + source_to_error = defaultdict(list) + for source, error in errors: + source_to_error[source].append(error) + + code = format_blocks(fn.blocks, names, source_to_error) + lines.extend(code) + return lines + + +def format_modules(modules: ModuleIRs) -> list[str]: + ops = [] + for module in modules.values(): + for fn in module.functions: + ops.extend(format_func(fn)) + ops.append("") + return ops + + +def generate_names_for_ir(args: list[Register], blocks: list[BasicBlock]) -> dict[Value, str]: + """Generate unique names for IR values. + + Give names such as 'r5' to temp values in IR which are useful when + pretty-printing or generating C. Ensure generated names are unique. + """ + names: dict[Value, str] = {} + used_names = set() + + temp_index = 0 + + for arg in args: + names[arg] = arg.name + used_names.add(arg.name) + + for block in blocks: + for op in block.ops: + values = [] + + for source in op.sources(): + if source not in names: + values.append(source) + + if isinstance(op, (Assign, AssignMulti)): + values.append(op.dest) + elif isinstance(op, ControlOp) or op.is_void: + continue + elif op not in names: + values.append(op) + + for value in values: + if value in names: + continue + if isinstance(value, Register) and value.name: + name = value.name + elif isinstance(value, (Integer, Float)): + continue + else: + name = "r%d" % temp_index + temp_index += 1 + + # Append _2, _3, ... if needed to make the name unique. + if name in used_names: + n = 2 + while True: + candidate = "%s_%d" % (name, n) + if candidate not in used_names: + name = candidate + break + n += 1 + + names[value] = name + used_names.add(name) + + return names diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py new file mode 100644 index 000000000000..53e3cee74e56 --- /dev/null +++ b/mypyc/ir/rtypes.py @@ -0,0 +1,1038 @@ +"""Types used in the intermediate representation. + +These are runtime types (RTypes), as opposed to mypy Type objects. +The latter are only used during type checking and not directly used at +runtime. Runtime types are derived from mypy types, but there's no +simple one-to-one correspondence. (Here 'runtime' means 'runtime +checked'.) + +The generated IR ensures some runtime type safety properties based on +RTypes. Compiled code can assume that the runtime value matches the +static RType of a value. If the RType of a register is 'builtins.str' +(str_rprimitive), for example, the generated IR will ensure that the +register will have a 'str' object. + +RTypes are simpler and less expressive than mypy (or PEP 484) +types. For example, all mypy types of form 'list[T]' (for arbitrary T) +are erased to the single RType 'builtins.list' (list_rprimitive). + +mypyc.irbuild.mapper.Mapper.type_to_rtype converts mypy Types to mypyc +RTypes. +""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeVar +from typing_extensions import TypeGuard + +from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name +from mypyc.namegen import NameGenerator + +if TYPE_CHECKING: + from mypyc.ir.class_ir import ClassIR + from mypyc.ir.ops import DeserMaps + +T = TypeVar("T") + + +class RType: + """Abstract base class for runtime types (erased, only concrete; no generics).""" + + name: str + # If True, the type has a special unboxed representation. If False, the + # type is represented as PyObject *. Even if True, the representation + # may contain pointers. + is_unboxed = False + # This is the C undefined value for this type. It's used for initialization + # if there's no value yet, and for function return value on error/exception. + # + # TODO: This shouldn't be specific to C or a string + c_undefined: str + # If unboxed: does the unboxed version use reference counting? + is_refcounted = True + # C type; use Emitter.ctype() to access + _ctype: str + # If True, error/undefined value overlaps with a valid value. To + # detect an exception, PyErr_Occurred() must be used in addition + # to checking for error value as the return value of a function. + # + # For example, no i64 value can be reserved for error value, so we + # pick an arbitrary value (e.g. -113) to signal error, but this is + # also a valid non-error value. + error_overlap = False + + @abstractmethod + def accept(self, visitor: RTypeVisitor[T]) -> T: + raise NotImplementedError() + + def short_name(self) -> str: + return short_name(self.name) + + def __str__(self) -> str: + return short_name(self.name) + + def __repr__(self) -> str: + return "<%s>" % self.__class__.__name__ + + def serialize(self) -> JsonDict | str: + raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") + + +def deserialize_type(data: JsonDict | str, ctx: DeserMaps) -> RType: + """Deserialize a JSON-serialized RType. + + Arguments: + data: The decoded JSON of the serialized type + ctx: The deserialization maps to use + """ + # Since there are so few types, we just case on them directly. If + # more get added we should switch to a system like mypy.types + # uses. + if isinstance(data, str): + if data in ctx.classes: + return RInstance(ctx.classes[data]) + elif data in RPrimitive.primitive_map: + return RPrimitive.primitive_map[data] + elif data == "void": + return RVoid() + else: + assert False, f"Can't find class {data}" + elif data[".class"] == "RTuple": + return RTuple.deserialize(data, ctx) + elif data[".class"] == "RUnion": + return RUnion.deserialize(data, ctx) + raise NotImplementedError("unexpected .class {}".format(data[".class"])) + + +class RTypeVisitor(Generic[T]): + """Generic visitor over RTypes (uses the visitor design pattern).""" + + @abstractmethod + def visit_rprimitive(self, typ: RPrimitive) -> T: + raise NotImplementedError + + @abstractmethod + def visit_rinstance(self, typ: RInstance) -> T: + raise NotImplementedError + + @abstractmethod + def visit_runion(self, typ: RUnion) -> T: + raise NotImplementedError + + @abstractmethod + def visit_rtuple(self, typ: RTuple) -> T: + raise NotImplementedError + + @abstractmethod + def visit_rstruct(self, typ: RStruct) -> T: + raise NotImplementedError + + @abstractmethod + def visit_rarray(self, typ: RArray) -> T: + raise NotImplementedError + + @abstractmethod + def visit_rvoid(self, typ: RVoid) -> T: + raise NotImplementedError + + +class RVoid(RType): + """The void type (no value). + + This is a singleton -- use void_rtype (below) to refer to this instead of + constructing a new instance. + """ + + is_unboxed = False + name = "void" + ctype = "void" + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rvoid(self) + + def serialize(self) -> str: + return "void" + + def __eq__(self, other: object) -> bool: + return isinstance(other, RVoid) + + def __hash__(self) -> int: + return hash(RVoid) + + +# Singleton instance of RVoid +void_rtype: Final = RVoid() + + +class RPrimitive(RType): + """Primitive type such as 'object' or 'int'. + + These often have custom ops associated with them. The 'object' + primitive type can be used to hold arbitrary Python objects. + + Different primitive types have different representations, and + primitives may be unboxed or boxed. Primitive types don't need to + directly correspond to Python types, but most do. + + NOTE: All supported primitive types are defined below + (e.g. object_rprimitive). + """ + + # Map from primitive names to primitive types and is used by deserialization + primitive_map: ClassVar[dict[str, RPrimitive]] = {} + + def __init__( + self, + name: str, + *, + is_unboxed: bool, + is_refcounted: bool, + is_native_int: bool = False, + is_signed: bool = False, + ctype: str = "PyObject *", + size: int = PLATFORM_SIZE, + error_overlap: bool = False, + ) -> None: + RPrimitive.primitive_map[name] = self + + self.name = name + self.is_unboxed = is_unboxed + self.is_refcounted = is_refcounted + self.is_native_int = is_native_int + self.is_signed = is_signed + self._ctype = ctype + self.size = size + self.error_overlap = error_overlap + if ctype == "CPyTagged": + self.c_undefined = "CPY_INT_TAG" + elif ctype in ("int16_t", "int32_t", "int64_t"): + # This is basically an arbitrary value that is pretty + # unlikely to overlap with a real value. + self.c_undefined = "-113" + elif ctype == "CPyPtr": + # TODO: Invent an overlapping error value? + self.c_undefined = "0" + elif ctype == "PyObject *": + # Boxed types use the null pointer as the error value. + self.c_undefined = "NULL" + elif ctype == "char": + self.c_undefined = "2" + elif ctype in ("PyObject **", "void *"): + self.c_undefined = "NULL" + elif ctype == "double": + self.c_undefined = "-113.0" + elif ctype in ("uint8_t", "uint16_t", "uint32_t", "uint64_t"): + self.c_undefined = "239" # An arbitrary number + else: + assert False, "Unrecognized ctype: %r" % ctype + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rprimitive(self) + + def serialize(self) -> str: + return self.name + + def __repr__(self) -> str: + return "" % self.name + + def __eq__(self, other: object) -> bool: + return isinstance(other, RPrimitive) and other.name == self.name + + def __hash__(self) -> int: + return hash(self.name) + + +# NOTE: All the supported instances of RPrimitive are defined +# below. Use these instead of creating new instances. + +# Used to represent arbitrary objects and dynamically typed (Any) +# values. There are various ops that let you perform generic, runtime +# checked operations on these (that match Python semantics). See the +# ops in mypyc.primitives.misc_ops, including py_getattr_op, +# py_call_op, and many others. +# +# If there is no more specific RType available for some value, we fall +# back to using this type. +# +# NOTE: Even though this is very flexible, this type should be used as +# little as possible, as generic ops are typically slow. Other types, +# including other primitive types and RInstance, are usually much +# faster. +object_rprimitive: Final = RPrimitive("builtins.object", is_unboxed=False, is_refcounted=True) + +# represents a low level pointer of an object +object_pointer_rprimitive: Final = RPrimitive( + "object_ptr", is_unboxed=False, is_refcounted=False, ctype="PyObject **" +) + +# Arbitrary-precision integer (corresponds to Python 'int'). Small +# enough values are stored unboxed, while large integers are +# represented as a tagged pointer to a Python 'int' PyObject. The +# lowest bit is used as the tag to decide whether it is a signed +# unboxed value (shifted left by one) or a PyObject * pointing to an +# 'int' object. Pointers have the least significant bit set. +# +# The undefined/error value is the null pointer (1 -- only the least +# significant bit is set)). +# +# This cannot represent a subclass of int. An instance of a subclass +# of int is coerced to the corresponding 'int' value. +int_rprimitive: Final = RPrimitive( + "builtins.int", is_unboxed=True, is_refcounted=True, ctype="CPyTagged" +) + +# An unboxed integer. The representation is the same as for unboxed +# int_rprimitive (shifted left by one). These can be used when an +# integer is known to be small enough to fit size_t (CPyTagged). +short_int_rprimitive: Final = RPrimitive( + "short_int", is_unboxed=True, is_refcounted=False, ctype="CPyTagged" +) + +# Low level integer types (correspond to C integer types) + +int16_rprimitive: Final = RPrimitive( + "i16", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=True, + ctype="int16_t", + size=2, + error_overlap=True, +) +int32_rprimitive: Final = RPrimitive( + "i32", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=True, + ctype="int32_t", + size=4, + error_overlap=True, +) +int64_rprimitive: Final = RPrimitive( + "i64", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=True, + ctype="int64_t", + size=8, + error_overlap=True, +) +uint8_rprimitive: Final = RPrimitive( + "u8", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=False, + ctype="uint8_t", + size=1, + error_overlap=True, +) + +# The following unsigned native int types (u16, u32, u64) are not +# exposed to the user. They are for internal use within mypyc only. + +u16_rprimitive: Final = RPrimitive( + "u16", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=False, + ctype="uint16_t", + size=2, + error_overlap=True, +) +uint32_rprimitive: Final = RPrimitive( + "u32", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=False, + ctype="uint32_t", + size=4, + error_overlap=True, +) +uint64_rprimitive: Final = RPrimitive( + "u64", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=False, + ctype="uint64_t", + size=8, + error_overlap=True, +) + +# The C 'int' type +c_int_rprimitive = int32_rprimitive + +if IS_32_BIT_PLATFORM: + c_size_t_rprimitive = uint32_rprimitive + c_pyssize_t_rprimitive = RPrimitive( + "native_int", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=True, + ctype="int32_t", + size=4, + ) +else: + c_size_t_rprimitive = uint64_rprimitive + c_pyssize_t_rprimitive = RPrimitive( + "native_int", + is_unboxed=True, + is_refcounted=False, + is_native_int=True, + is_signed=True, + ctype="int64_t", + size=8, + ) + +# Untyped pointer, represented as integer in the C backend +pointer_rprimitive: Final = RPrimitive("ptr", is_unboxed=True, is_refcounted=False, ctype="CPyPtr") + +# Untyped pointer, represented as void * in the C backend +c_pointer_rprimitive: Final = RPrimitive( + "c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *" +) + +# The type corresponding to mypyc.common.BITMAP_TYPE +bitmap_rprimitive: Final = uint32_rprimitive + +# Floats are represent as 'float' PyObject * values. (In the future +# we'll likely switch to a more efficient, unboxed representation.) +float_rprimitive: Final = RPrimitive( + "builtins.float", + is_unboxed=True, + is_refcounted=False, + ctype="double", + size=8, + error_overlap=True, +) + +# An unboxed Python bool value. This actually has three possible values +# (0 -> False, 1 -> True, 2 -> error). If you only need True/False, use +# bit_rprimitive instead. +bool_rprimitive: Final = RPrimitive( + "builtins.bool", is_unboxed=True, is_refcounted=False, ctype="char", size=1 +) + +# A low-level boolean value with two possible values: 0 and 1. Any +# other value results in undefined behavior. Undefined or error values +# are not supported. +bit_rprimitive: Final = RPrimitive( + "bit", is_unboxed=True, is_refcounted=False, ctype="char", size=1 +) + +# The 'None' value. The possible values are 0 -> None and 2 -> error. +none_rprimitive: Final = RPrimitive( + "builtins.None", is_unboxed=True, is_refcounted=False, ctype="char", size=1 +) + +# Python list object (or an instance of a subclass of list). +list_rprimitive: Final = RPrimitive("builtins.list", is_unboxed=False, is_refcounted=True) + +# Python dict object (or an instance of a subclass of dict). +dict_rprimitive: Final = RPrimitive("builtins.dict", is_unboxed=False, is_refcounted=True) + +# Python set object (or an instance of a subclass of set). +set_rprimitive: Final = RPrimitive("builtins.set", is_unboxed=False, is_refcounted=True) + +# Python str object. At the C layer, str is referred to as unicode +# (PyUnicode). +str_rprimitive: Final = RPrimitive("builtins.str", is_unboxed=False, is_refcounted=True) + +# Python bytes object. +bytes_rprimitive: Final = RPrimitive("builtins.bytes", is_unboxed=False, is_refcounted=True) + +# Tuple of an arbitrary length (corresponds to Tuple[t, ...], with +# explicit '...'). +tuple_rprimitive: Final = RPrimitive("builtins.tuple", is_unboxed=False, is_refcounted=True) + +# Python range object. +range_rprimitive: Final = RPrimitive("builtins.range", is_unboxed=False, is_refcounted=True) + + +def is_tagged(rtype: RType) -> bool: + return rtype is int_rprimitive or rtype is short_int_rprimitive + + +def is_int_rprimitive(rtype: RType) -> bool: + return rtype is int_rprimitive + + +def is_short_int_rprimitive(rtype: RType) -> bool: + return rtype is short_int_rprimitive + + +def is_int16_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: + return rtype is int16_rprimitive + + +def is_int32_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: + return rtype is int32_rprimitive or ( + rtype is c_pyssize_t_rprimitive and rtype._ctype == "int32_t" + ) + + +def is_int64_rprimitive(rtype: RType) -> bool: + return rtype is int64_rprimitive or ( + rtype is c_pyssize_t_rprimitive and rtype._ctype == "int64_t" + ) + + +def is_fixed_width_rtype(rtype: RType) -> TypeGuard[RPrimitive]: + return ( + is_int64_rprimitive(rtype) + or is_int32_rprimitive(rtype) + or is_int16_rprimitive(rtype) + or is_uint8_rprimitive(rtype) + ) + + +def is_uint8_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: + return rtype is uint8_rprimitive + + +def is_uint32_rprimitive(rtype: RType) -> bool: + return rtype is uint32_rprimitive + + +def is_uint64_rprimitive(rtype: RType) -> bool: + return rtype is uint64_rprimitive + + +def is_c_py_ssize_t_rprimitive(rtype: RType) -> bool: + return rtype is c_pyssize_t_rprimitive + + +def is_pointer_rprimitive(rtype: RType) -> bool: + return rtype is pointer_rprimitive + + +def is_float_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.float" + + +def is_bool_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.bool" + + +def is_bit_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "bit" + + +def is_object_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.object" + + +def is_none_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.None" + + +def is_list_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.list" + + +def is_dict_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.dict" + + +def is_set_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.set" + + +def is_str_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.str" + + +def is_bytes_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.bytes" + + +def is_tuple_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.tuple" + + +def is_range_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name == "builtins.range" + + +def is_sequence_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and ( + is_list_rprimitive(rtype) or is_tuple_rprimitive(rtype) or is_str_rprimitive(rtype) + ) + + +class TupleNameVisitor(RTypeVisitor[str]): + """Produce a tuple name based on the concrete representations of types.""" + + def visit_rinstance(self, t: RInstance) -> str: + return "O" + + def visit_runion(self, t: RUnion) -> str: + return "O" + + def visit_rprimitive(self, t: RPrimitive) -> str: + if t._ctype == "CPyTagged": + return "I" + elif t._ctype == "char": + return "C" + elif t._ctype == "int64_t": + return "8" # "8 byte integer" + elif t._ctype == "int32_t": + return "4" # "4 byte integer" + elif t._ctype == "int16_t": + return "2" # "2 byte integer" + elif t._ctype == "uint8_t": + return "U1" # "1 byte unsigned integer" + elif t._ctype == "double": + return "F" + assert not t.is_unboxed, f"{t} unexpected unboxed type" + return "O" + + def visit_rtuple(self, t: RTuple) -> str: + parts = [elem.accept(self) for elem in t.types] + return "T{}{}".format(len(parts), "".join(parts)) + + def visit_rstruct(self, t: RStruct) -> str: + assert False, "RStruct not supported in tuple" + + def visit_rarray(self, t: RArray) -> str: + assert False, "RArray not supported in tuple" + + def visit_rvoid(self, t: RVoid) -> str: + assert False, "rvoid in tuple?" + + +class RTuple(RType): + """Fixed-length unboxed tuple (represented as a C struct). + + These are used to represent mypy TupleType values (fixed-length + Python tuples). Since this is unboxed, the identity of a tuple + object is not preserved within compiled code. If the identity of a + tuple is important, or there is a need to have multiple references + to a single tuple object, a variable-length tuple should be used + (tuple_rprimitive or Tuple[T, ...] with explicit '...'), as they + are boxed. + + These aren't immutable. However, user code won't be able to mutate + individual tuple items. + """ + + is_unboxed = True + + def __init__(self, types: list[RType]) -> None: + self.name = "tuple" + self.types = tuple(types) + self.is_refcounted = any(t.is_refcounted for t in self.types) + # Generate a unique id which is used in naming corresponding C identifiers. + # This is necessary since C does not have anonymous structural type equivalence + # in the same way python can just assign a Tuple[int, bool] to a Tuple[int, bool]. + self.unique_id = self.accept(TupleNameVisitor()) + # Nominally the max c length is 31 chars, but I'm not honestly worried about this. + self.struct_name = f"tuple_{self.unique_id}" + self._ctype = f"{self.struct_name}" + self.error_overlap = all(t.error_overlap for t in self.types) and bool(self.types) + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rtuple(self) + + def __str__(self) -> str: + return "tuple[%s]" % ", ".join(str(typ) for typ in self.types) + + def __repr__(self) -> str: + return "" % ", ".join(repr(typ) for typ in self.types) + + def __eq__(self, other: object) -> bool: + return isinstance(other, RTuple) and self.types == other.types + + def __hash__(self) -> int: + return hash((self.name, self.types)) + + def serialize(self) -> JsonDict: + types = [x.serialize() for x in self.types] + return {".class": "RTuple", "types": types} + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RTuple: + types = [deserialize_type(t, ctx) for t in data["types"]] + return RTuple(types) + + +# Exception tuple: (exception class, exception instance, traceback object) +exc_rtuple = RTuple([object_rprimitive, object_rprimitive, object_rprimitive]) + +# Dictionary iterator tuple: (should continue, internal offset, key, value) +# See mypyc.irbuild.for_helpers.ForDictionaryCommon for more details. +dict_next_rtuple_pair = RTuple( + [bool_rprimitive, short_int_rprimitive, object_rprimitive, object_rprimitive] +) +# Same as above but just for key or value. +dict_next_rtuple_single = RTuple([bool_rprimitive, short_int_rprimitive, object_rprimitive]) + + +def compute_rtype_alignment(typ: RType) -> int: + """Compute alignment of a given type based on platform alignment rule""" + platform_alignment = PLATFORM_SIZE + if isinstance(typ, RPrimitive): + return typ.size + elif isinstance(typ, RInstance): + return platform_alignment + elif isinstance(typ, RUnion): + return platform_alignment + elif isinstance(typ, RArray): + return compute_rtype_alignment(typ.item_type) + else: + if isinstance(typ, RTuple): + items = list(typ.types) + elif isinstance(typ, RStruct): + items = typ.types + else: + assert False, "invalid rtype for computing alignment" + max_alignment = max(compute_rtype_alignment(item) for item in items) + return max_alignment + + +def compute_rtype_size(typ: RType) -> int: + """Compute unaligned size of rtype""" + if isinstance(typ, RPrimitive): + return typ.size + elif isinstance(typ, RTuple): + return compute_aligned_offsets_and_size(list(typ.types))[1] + elif isinstance(typ, RUnion): + return PLATFORM_SIZE + elif isinstance(typ, RStruct): + return compute_aligned_offsets_and_size(typ.types)[1] + elif isinstance(typ, RInstance): + return PLATFORM_SIZE + elif isinstance(typ, RArray): + alignment = compute_rtype_alignment(typ) + aligned_size = (compute_rtype_size(typ.item_type) + (alignment - 1)) & ~(alignment - 1) + return aligned_size * typ.length + else: + assert False, "invalid rtype for computing size" + + +def compute_aligned_offsets_and_size(types: list[RType]) -> tuple[list[int], int]: + """Compute offsets and total size of a list of types after alignment + + Note that the types argument are types of values that are stored + sequentially with platform default alignment. + """ + unaligned_sizes = [compute_rtype_size(typ) for typ in types] + alignments = [compute_rtype_alignment(typ) for typ in types] + + current_offset = 0 + offsets = [] + final_size = 0 + for i in range(len(unaligned_sizes)): + offsets.append(current_offset) + if i + 1 < len(unaligned_sizes): + cur_size = unaligned_sizes[i] + current_offset += cur_size + next_alignment = alignments[i + 1] + # compute aligned offset, + # check https://en.wikipedia.org/wiki/Data_structure_alignment for more information + current_offset = (current_offset + (next_alignment - 1)) & -next_alignment + else: + struct_alignment = max(alignments) + final_size = current_offset + unaligned_sizes[i] + final_size = (final_size + (struct_alignment - 1)) & -struct_alignment + return offsets, final_size + + +class RStruct(RType): + """C struct type""" + + def __init__(self, name: str, names: list[str], types: list[RType]) -> None: + self.name = name + self.names = names + self.types = types + # generate dummy names + if len(self.names) < len(self.types): + for i in range(len(self.types) - len(self.names)): + self.names.append("_item" + str(i)) + self.offsets, self.size = compute_aligned_offsets_and_size(types) + self._ctype = name + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rstruct(self) + + def __str__(self) -> str: + # if not tuple(unnamed structs) + return "{}{{{}}}".format( + self.name, + ", ".join(name + ":" + str(typ) for name, typ in zip(self.names, self.types)), + ) + + def __repr__(self) -> str: + return "".format( + self.name, + ", ".join(name + ":" + repr(typ) for name, typ in zip(self.names, self.types)), + ) + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, RStruct) + and self.name == other.name + and self.names == other.names + and self.types == other.types + ) + + def __hash__(self) -> int: + return hash((self.name, tuple(self.names), tuple(self.types))) + + def serialize(self) -> JsonDict: + assert False + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RStruct: + assert False + + +class RInstance(RType): + """Instance of user-defined class (compiled to C extension class). + + The runtime representation is 'PyObject *', and these are always + boxed and thus reference-counted. + + These support fast method calls and fast attribute access using + vtables, and they usually use a dict-free, struct-based + representation of attributes. Method calls and attribute access + can skip the vtable if we know that there is no overriding. + + These are also sometimes called 'native' types, since these have + the most efficient representation and ops (along with certain + RPrimitive types and RTuple). + """ + + is_unboxed = False + + def __init__(self, class_ir: ClassIR) -> None: + # name is used for formatting the name in messages and debug output + # so we want the fullname for precision. + self.name = class_ir.fullname + self.class_ir = class_ir + self._ctype = "PyObject *" + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rinstance(self) + + def struct_name(self, names: NameGenerator) -> str: + return self.class_ir.struct_name(names) + + def getter_index(self, name: str) -> int: + return self.class_ir.vtable_entry(name) + + def setter_index(self, name: str) -> int: + return self.getter_index(name) + 1 + + def method_index(self, name: str) -> int: + return self.class_ir.vtable_entry(name) + + def attr_type(self, name: str) -> RType: + return self.class_ir.attr_type(name) + + def __repr__(self) -> str: + return "" % self.name + + def __eq__(self, other: object) -> bool: + return isinstance(other, RInstance) and other.name == self.name + + def __hash__(self) -> int: + return hash(self.name) + + def serialize(self) -> str: + return self.name + + +class RUnion(RType): + """union[x, ..., y]""" + + is_unboxed = False + + def __init__(self, items: list[RType]) -> None: + self.name = "union" + self.items = items + self.items_set = frozenset(items) + self._ctype = "PyObject *" + + @staticmethod + def make_simplified_union(items: list[RType]) -> RType: + """Return a normalized union that covers the given items. + + Flatten nested unions and remove duplicate items. + + Overlapping items are *not* simplified. For example, + [object, str] will not be simplified. + """ + items = flatten_nested_unions(items) + assert items + + unique_items = dict.fromkeys(items) + if len(unique_items) > 1: + return RUnion(list(unique_items)) + else: + return next(iter(unique_items)) + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_runion(self) + + def __repr__(self) -> str: + return "" % ", ".join(str(item) for item in self.items) + + def __str__(self) -> str: + return "union[%s]" % ", ".join(str(item) for item in self.items) + + # We compare based on the set because order in a union doesn't matter + def __eq__(self, other: object) -> bool: + return isinstance(other, RUnion) and self.items_set == other.items_set + + def __hash__(self) -> int: + return hash(("union", self.items_set)) + + def serialize(self) -> JsonDict: + types = [x.serialize() for x in self.items] + return {".class": "RUnion", "types": types} + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RUnion: + types = [deserialize_type(t, ctx) for t in data["types"]] + return RUnion(types) + + +def flatten_nested_unions(types: list[RType]) -> list[RType]: + if not any(isinstance(t, RUnion) for t in types): + return types # Fast path + + flat_items: list[RType] = [] + for t in types: + if isinstance(t, RUnion): + flat_items.extend(flatten_nested_unions(t.items)) + else: + flat_items.append(t) + return flat_items + + +def optional_value_type(rtype: RType) -> RType | None: + """If rtype is the union of none_rprimitive and another type X, return X. + + Otherwise return None. + """ + if isinstance(rtype, RUnion) and len(rtype.items) == 2: + if rtype.items[0] == none_rprimitive: + return rtype.items[1] + elif rtype.items[1] == none_rprimitive: + return rtype.items[0] + return None + + +def is_optional_type(rtype: RType) -> bool: + """Is rtype an optional type with exactly two union items?""" + return optional_value_type(rtype) is not None + + +class RArray(RType): + """Fixed-length C array type (for example, int[5]). + + Note that the implementation is a bit limited, and these can basically + be only used for local variables that are initialized in one location. + """ + + def __init__(self, item_type: RType, length: int) -> None: + self.item_type = item_type + # Number of items + self.length = length + self.is_refcounted = False + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rarray(self) + + def __str__(self) -> str: + return f"{self.item_type}[{self.length}]" + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, RArray) + and self.item_type == other.item_type + and self.length == other.length + ) + + def __hash__(self) -> int: + return hash((self.item_type, self.length)) + + def serialize(self) -> JsonDict: + assert False + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RArray: + assert False + + +PyObject = RStruct( + name="PyObject", + names=["ob_refcnt", "ob_type"], + types=[c_pyssize_t_rprimitive, pointer_rprimitive], +) + +PyVarObject = RStruct( + name="PyVarObject", names=["ob_base", "ob_size"], types=[PyObject, c_pyssize_t_rprimitive] +) + +setentry = RStruct( + name="setentry", names=["key", "hash"], types=[pointer_rprimitive, c_pyssize_t_rprimitive] +) + +smalltable = RStruct(name="smalltable", names=[], types=[setentry] * 8) + +PySetObject = RStruct( + name="PySetObject", + names=[ + "ob_base", + "fill", + "used", + "mask", + "table", + "hash", + "finger", + "smalltable", + "weakreflist", + ], + types=[ + PyObject, + c_pyssize_t_rprimitive, + c_pyssize_t_rprimitive, + c_pyssize_t_rprimitive, + pointer_rprimitive, + c_pyssize_t_rprimitive, + c_pyssize_t_rprimitive, + smalltable, + pointer_rprimitive, + ], +) + +PyListObject = RStruct( + name="PyListObject", + names=["ob_base", "ob_item", "allocated"], + types=[PyVarObject, pointer_rprimitive, c_pyssize_t_rprimitive], +) + + +def check_native_int_range(rtype: RPrimitive, n: int) -> bool: + """Is n within the range of a native, fixed-width int type? + + Assume the type is a fixed-width int type. + """ + if not rtype.is_signed: + return 0 <= n < (1 << (8 * rtype.size)) + else: + limit = 1 << (rtype.size * 8 - 1) + return -limit <= n < limit diff --git a/mypyc/irbuild/__init__.py b/mypyc/irbuild/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/irbuild/ast_helpers.py b/mypyc/irbuild/ast_helpers.py new file mode 100644 index 000000000000..3b0f50514594 --- /dev/null +++ b/mypyc/irbuild/ast_helpers.py @@ -0,0 +1,123 @@ +"""IRBuilder AST transform helpers shared between expressions and statements. + +Shared code that is tightly coupled to mypy ASTs can be put here instead of +making mypyc.irbuild.builder larger. +""" + +from __future__ import annotations + +from mypy.nodes import ( + LDEF, + BytesExpr, + ComparisonExpr, + Expression, + FloatExpr, + IntExpr, + MemberExpr, + NameExpr, + OpExpr, + StrExpr, + UnaryExpr, + Var, +) +from mypyc.ir.ops import BasicBlock +from mypyc.ir.rtypes import is_fixed_width_rtype, is_tagged +from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.constant_fold import constant_fold_expr + + +def process_conditional( + self: IRBuilder, e: Expression, true: BasicBlock, false: BasicBlock +) -> None: + if isinstance(e, OpExpr) and e.op in ["and", "or"]: + if e.op == "and": + # Short circuit 'and' in a conditional context. + new = BasicBlock() + process_conditional(self, e.left, new, false) + self.activate_block(new) + process_conditional(self, e.right, true, false) + else: + # Short circuit 'or' in a conditional context. + new = BasicBlock() + process_conditional(self, e.left, true, new) + self.activate_block(new) + process_conditional(self, e.right, true, false) + elif isinstance(e, UnaryExpr) and e.op == "not": + process_conditional(self, e.expr, false, true) + else: + res = maybe_process_conditional_comparison(self, e, true, false) + if res: + return + # Catch-all for arbitrary expressions. + reg = self.accept(e) + self.add_bool_branch(reg, true, false) + + +def maybe_process_conditional_comparison( + self: IRBuilder, e: Expression, true: BasicBlock, false: BasicBlock +) -> bool: + """Transform simple tagged integer comparisons in a conditional context. + + Return True if the operation is supported (and was transformed). Otherwise, + do nothing and return False. + + Args: + self: IR form Builder + e: Arbitrary expression + true: Branch target if comparison is true + false: Branch target if comparison is false + """ + if not isinstance(e, ComparisonExpr) or len(e.operands) != 2: + return False + ltype = self.node_type(e.operands[0]) + rtype = self.node_type(e.operands[1]) + if not ( + (is_tagged(ltype) or is_fixed_width_rtype(ltype)) + and (is_tagged(rtype) or is_fixed_width_rtype(rtype)) + ): + return False + op = e.operators[0] + if op not in ("==", "!=", "<", "<=", ">", ">="): + return False + left_expr = e.operands[0] + right_expr = e.operands[1] + borrow_left = is_borrow_friendly_expr(self, right_expr) + left = self.accept(left_expr, can_borrow=borrow_left) + right = self.accept(right_expr, can_borrow=True) + if is_fixed_width_rtype(ltype) or is_fixed_width_rtype(rtype): + if not is_fixed_width_rtype(ltype): + left = self.coerce(left, rtype, e.line) + elif not is_fixed_width_rtype(rtype): + right = self.coerce(right, ltype, e.line) + reg = self.binary_op(left, right, op, e.line) + self.builder.flush_keep_alives() + self.add_bool_branch(reg, true, false) + else: + # "left op right" for two tagged integers + reg = self.builder.binary_op(left, right, op, e.line) + self.flush_keep_alives() + self.add_bool_branch(reg, true, false) + return True + + +def is_borrow_friendly_expr(self: IRBuilder, expr: Expression) -> bool: + """Can the result of the expression borrowed temporarily? + + Borrowing means keeping a reference without incrementing the reference count. + """ + if isinstance(expr, (IntExpr, FloatExpr, StrExpr, BytesExpr)): + # Literals are immortal and can always be borrowed + return True + if ( + isinstance(expr, (UnaryExpr, OpExpr, NameExpr, MemberExpr)) + and constant_fold_expr(self, expr) is not None + ): + # Literal expressions are similar to literals + return True + if isinstance(expr, NameExpr): + if isinstance(expr.node, Var) and expr.kind == LDEF: + # Local variable reference can be borrowed + return True + if isinstance(expr, MemberExpr) and self.is_native_attr_ref(expr): + return True + return False diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py new file mode 100644 index 000000000000..a0837ba2bfc7 --- /dev/null +++ b/mypyc/irbuild/builder.py @@ -0,0 +1,1457 @@ +"""Builder class used to transform a mypy AST to the IR form. + +The IRBuilder class maintains transformation state and provides access +to various helpers used to implement the transform. + +The top-level transform control logic is in mypyc.irbuild.main. + +mypyc.irbuild.visitor.IRBuilderVisitor is used to dispatch based on mypy +AST node type to code that actually does the bulk of the work. For +example, expressions are transformed in mypyc.irbuild.expression and +functions are transformed in mypyc.irbuild.function. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import Any, Callable, Final, Iterator, Sequence, Union +from typing_extensions import overload + +from mypy.build import Graph +from mypy.maptype import map_instance_to_supertype +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + GDEF, + LDEF, + PARAM_SPEC_KIND, + TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, + ArgKind, + CallExpr, + Decorator, + Expression, + FuncDef, + IndexExpr, + IntExpr, + Lvalue, + MemberExpr, + MypyFile, + NameExpr, + OpExpr, + OverloadedFuncDef, + RefExpr, + StarExpr, + Statement, + SymbolNode, + TupleExpr, + TypeAlias, + TypeInfo, + TypeParam, + UnaryExpr, + Var, +) +from mypy.types import ( + AnyType, + DeletedType, + Instance, + ProperType, + TupleType, + Type, + TypedDictType, + TypeOfAny, + UninhabitedType, + UnionType, + get_proper_type, +) +from mypy.util import module_prefix, split_target +from mypy.visitor import ExpressionVisitor, StatementVisitor +from mypyc.common import BITMAP_BITS, SELF_NAME, TEMP_ATTR_NAME +from mypyc.crash import catch_errors +from mypyc.errors import Errors +from mypyc.ir.class_ir import ClassIR, NonExtClassInfo +from mypyc.ir.func_ir import INVALID_FUNC_DEF, FuncDecl, FuncIR, FuncSignature, RuntimeArg +from mypyc.ir.ops import ( + NAMESPACE_MODULE, + NAMESPACE_TYPE_VAR, + Assign, + BasicBlock, + Branch, + ComparisonOp, + GetAttr, + InitStatic, + Integer, + IntOp, + LoadStatic, + Op, + PrimitiveDescription, + RaiseStandardError, + Register, + SetAttr, + TupleGet, + Unreachable, + Value, +) +from mypyc.ir.rtypes import ( + RInstance, + RTuple, + RType, + RUnion, + bitmap_rprimitive, + c_pyssize_t_rprimitive, + dict_rprimitive, + int_rprimitive, + is_float_rprimitive, + is_list_rprimitive, + is_none_rprimitive, + is_object_rprimitive, + is_tagged, + is_tuple_rprimitive, + none_rprimitive, + object_rprimitive, + str_rprimitive, +) +from mypyc.irbuild.context import FuncInfo, ImplicitClass +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.irbuild.mapper import Mapper +from mypyc.irbuild.nonlocalcontrol import ( + BaseNonlocalControl, + GeneratorNonlocalControl, + LoopNonlocalControl, + NonlocalControl, +) +from mypyc.irbuild.prebuildvisitor import PreBuildVisitor +from mypyc.irbuild.prepare import RegisterImplInfo +from mypyc.irbuild.targets import ( + AssignmentTarget, + AssignmentTargetAttr, + AssignmentTargetIndex, + AssignmentTargetRegister, + AssignmentTargetTuple, +) +from mypyc.irbuild.util import bytes_from_str, is_constant +from mypyc.options import CompilerOptions +from mypyc.primitives.dict_ops import dict_get_item_op, dict_set_item_op +from mypyc.primitives.generic_ops import iter_op, next_op, py_setattr_op +from mypyc.primitives.list_ops import list_get_item_unsafe_op, list_pop_last, to_list +from mypyc.primitives.misc_ops import check_unpack_count_op, get_module_dict_op, import_op +from mypyc.primitives.registry import CFunctionDescription, function_ops + +# These int binary operations can borrow their operands safely, since the +# primitives take this into consideration. +int_borrow_friendly_op: Final = {"+", "-", "==", "!=", "<", "<=", ">", ">="} + + +class IRVisitor(ExpressionVisitor[Value], StatementVisitor[None]): + pass + + +class UnsupportedException(Exception): + pass + + +SymbolTarget = Union[AssignmentTargetRegister, AssignmentTargetAttr] + + +class IRBuilder: + def __init__( + self, + current_module: str, + types: dict[Expression, Type], + graph: Graph, + errors: Errors, + mapper: Mapper, + pbv: PreBuildVisitor, + visitor: IRVisitor, + options: CompilerOptions, + singledispatch_impls: dict[FuncDef, list[RegisterImplInfo]], + ) -> None: + self.builder = LowLevelIRBuilder(errors, options) + self.builders = [self.builder] + self.symtables: list[dict[SymbolNode, SymbolTarget]] = [{}] + self.runtime_args: list[list[RuntimeArg]] = [[]] + self.function_name_stack: list[str] = [] + self.class_ir_stack: list[ClassIR] = [] + # Keep track of whether the next statement in a block is reachable + # or not, separately for each block nesting level + self.block_reachable_stack: list[bool] = [True] + + self.current_module = current_module + self.mapper = mapper + self.types = types + self.graph = graph + self.ret_types: list[RType] = [] + self.functions: list[FuncIR] = [] + self.function_names: set[tuple[str | None, str]] = set() + self.classes: list[ClassIR] = [] + self.final_names: list[tuple[str, RType]] = [] + self.type_var_names: list[str] = [] + self.callable_class_names: set[str] = set() + self.options = options + + # These variables keep track of the number of lambdas, implicit indices, and implicit + # iterators instantiated so we avoid name conflicts. The indices and iterators are + # instantiated from for-loops. + self.lambda_counter = 0 + self.temp_counter = 0 + + # These variables are populated from the first-pass PreBuildVisitor. + self.free_variables = pbv.free_variables + self.prop_setters = pbv.prop_setters + self.encapsulating_funcs = pbv.encapsulating_funcs + self.nested_fitems = pbv.nested_funcs.keys() + self.fdefs_to_decorators = pbv.funcs_to_decorators + self.module_import_groups = pbv.module_import_groups + + self.singledispatch_impls = singledispatch_impls + + self.visitor = visitor + + # This list operates similarly to a function call stack for nested functions. Whenever a + # function definition begins to be generated, a FuncInfo instance is added to the stack, + # and information about that function (e.g. whether it is nested, its environment class to + # be generated) is stored in that FuncInfo instance. When the function is done being + # generated, its corresponding FuncInfo is popped off the stack. + self.fn_info = FuncInfo(INVALID_FUNC_DEF, "", "") + self.fn_infos: list[FuncInfo] = [self.fn_info] + + # This list operates as a stack of constructs that modify the + # behavior of nonlocal control flow constructs. + self.nonlocal_control: list[NonlocalControl] = [] + + self.errors = errors + # Notionally a list of all of the modules imported by the + # module being compiled, but stored as an OrderedDict so we + # can also do quick lookups. + self.imports: dict[str, None] = {} + + self.can_borrow = False + + # High-level control + + def set_module(self, module_name: str, module_path: str) -> None: + """Set the name and path of the current module. + + This must be called before transforming any AST nodes. + """ + self.module_name = module_name + self.module_path = module_path + self.builder.set_module(module_name, module_path) + + @overload + def accept(self, node: Expression, *, can_borrow: bool = False) -> Value: ... + + @overload + def accept(self, node: Statement) -> None: ... + + def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> Value | None: + """Transform an expression or a statement. + + If can_borrow is true, prefer to generate a borrowed reference. + Borrowed references are faster since they don't require reference count + manipulation, but they are only safe to use in specific contexts. + """ + with self.catch_errors(node.line): + if isinstance(node, Expression): + old_can_borrow = self.can_borrow + self.can_borrow = can_borrow + try: + res = node.accept(self.visitor) + res = self.coerce(res, self.node_type(node), node.line) + # If we hit an error during compilation, we want to + # keep trying, so we can produce more error + # messages. Generate a temp of the right type to keep + # from causing more downstream trouble. + except UnsupportedException: + res = Register(self.node_type(node)) + self.can_borrow = old_can_borrow + if not can_borrow: + self.flush_keep_alives() + return res + else: + try: + node.accept(self.visitor) + except UnsupportedException: + pass + return None + + def flush_keep_alives(self) -> None: + self.builder.flush_keep_alives() + + # Pass through methods for the most common low-level builder ops, for convenience. + + def add(self, op: Op) -> Value: + return self.builder.add(op) + + def goto(self, target: BasicBlock) -> None: + self.builder.goto(target) + + def activate_block(self, block: BasicBlock) -> None: + self.builder.activate_block(block) + + def goto_and_activate(self, block: BasicBlock) -> None: + self.builder.goto_and_activate(block) + + def self(self) -> Register: + return self.builder.self() + + def py_get_attr(self, obj: Value, attr: str, line: int) -> Value: + return self.builder.py_get_attr(obj, attr, line) + + def load_str(self, value: str) -> Value: + return self.builder.load_str(value) + + def load_bytes_from_str_literal(self, value: str) -> Value: + """Load bytes object from a string literal. + + The literal characters of BytesExpr (the characters inside b'') + are stored in BytesExpr.value, whose type is 'str' not 'bytes'. + Thus we perform a special conversion here. + """ + return self.builder.load_bytes(bytes_from_str(value)) + + def load_int(self, value: int) -> Value: + return self.builder.load_int(value) + + def load_float(self, value: float) -> Value: + return self.builder.load_float(value) + + def unary_op(self, lreg: Value, expr_op: str, line: int) -> Value: + return self.builder.unary_op(lreg, expr_op, line) + + def binary_op(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> Value: + return self.builder.binary_op(lreg, rreg, expr_op, line) + + def coerce(self, src: Value, target_type: RType, line: int, force: bool = False) -> Value: + return self.builder.coerce(src, target_type, line, force, can_borrow=self.can_borrow) + + def none_object(self) -> Value: + return self.builder.none_object() + + def none(self) -> Value: + return self.builder.none() + + def true(self) -> Value: + return self.builder.true() + + def false(self) -> Value: + return self.builder.false() + + def new_list_op(self, values: list[Value], line: int) -> Value: + return self.builder.new_list_op(values, line) + + def new_set_op(self, values: list[Value], line: int) -> Value: + return self.builder.new_set_op(values, line) + + def translate_is_op(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> Value: + return self.builder.translate_is_op(lreg, rreg, expr_op, line) + + def py_call( + self, + function: Value, + arg_values: list[Value], + line: int, + arg_kinds: list[ArgKind] | None = None, + arg_names: Sequence[str | None] | None = None, + ) -> Value: + return self.builder.py_call(function, arg_values, line, arg_kinds, arg_names) + + def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) -> None: + self.builder.add_bool_branch(value, true, false) + + def load_native_type_object(self, fullname: str) -> Value: + return self.builder.load_native_type_object(fullname) + + def gen_method_call( + self, + base: Value, + name: str, + arg_values: list[Value], + result_type: RType | None, + line: int, + arg_kinds: list[ArgKind] | None = None, + arg_names: list[str | None] | None = None, + ) -> Value: + return self.builder.gen_method_call( + base, name, arg_values, result_type, line, arg_kinds, arg_names, self.can_borrow + ) + + def load_module(self, name: str) -> Value: + return self.builder.load_module(name) + + def call_c(self, desc: CFunctionDescription, args: list[Value], line: int) -> Value: + return self.builder.call_c(desc, args, line) + + def primitive_op(self, desc: PrimitiveDescription, args: list[Value], line: int) -> Value: + return self.builder.primitive_op(desc, args, line) + + def int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int) -> Value: + return self.builder.int_op(type, lhs, rhs, op, line) + + def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: + return self.builder.compare_tuples(lhs, rhs, op, line) + + def builtin_len(self, val: Value, line: int) -> Value: + return self.builder.builtin_len(val, line) + + def new_tuple(self, items: list[Value], line: int) -> Value: + return self.builder.new_tuple(items, line) + + # Helpers for IR building + + def add_to_non_ext_dict( + self, non_ext: NonExtClassInfo, key: str, val: Value, line: int + ) -> None: + # Add an attribute entry into the class dict of a non-extension class. + key_unicode = self.load_str(key) + self.primitive_op(dict_set_item_op, [non_ext.dict, key_unicode, val], line) + + def gen_import(self, id: str, line: int) -> None: + self.imports[id] = None + + needs_import, out = BasicBlock(), BasicBlock() + self.check_if_module_loaded(id, line, needs_import, out) + + self.activate_block(needs_import) + value = self.call_c(import_op, [self.load_str(id)], line) + self.add(InitStatic(value, id, namespace=NAMESPACE_MODULE)) + self.goto_and_activate(out) + + def check_if_module_loaded( + self, id: str, line: int, needs_import: BasicBlock, out: BasicBlock + ) -> None: + """Generate code that checks if the module `id` has been loaded yet. + + Arguments: + id: name of module to check if imported + line: line number that the import occurs on + needs_import: the BasicBlock that is run if the module has not been loaded yet + out: the BasicBlock that is run if the module has already been loaded""" + first_load = self.load_module(id) + comparison = self.translate_is_op(first_load, self.none_object(), "is not", line) + self.add_bool_branch(comparison, out, needs_import) + + def get_module(self, module: str, line: int) -> Value: + # Python 3.7 has a nice 'PyImport_GetModule' function that we can't use :( + mod_dict = self.call_c(get_module_dict_op, [], line) + # Get module object from modules dict. + return self.primitive_op(dict_get_item_op, [mod_dict, self.load_str(module)], line) + + def get_module_attr(self, module: str, attr: str, line: int) -> Value: + """Look up an attribute of a module without storing it in the local namespace. + + For example, get_module_attr('typing', 'TypedDict', line) results in + the value of 'typing.TypedDict'. + + Import the module if needed. + """ + self.gen_import(module, line) + module_obj = self.get_module(module, line) + return self.py_get_attr(module_obj, attr, line) + + def assign_if_null(self, target: Register, get_val: Callable[[], Value], line: int) -> None: + """If target is NULL, assign value produced by get_val to it.""" + error_block, body_block = BasicBlock(), BasicBlock() + self.add(Branch(target, error_block, body_block, Branch.IS_ERROR)) + self.activate_block(error_block) + self.add(Assign(target, self.coerce(get_val(), target.type, line))) + self.goto(body_block) + self.activate_block(body_block) + + def assign_if_bitmap_unset( + self, target: Register, get_val: Callable[[], Value], index: int, line: int + ) -> None: + error_block, body_block = BasicBlock(), BasicBlock() + o = self.int_op( + bitmap_rprimitive, + self.builder.args[-1 - index // BITMAP_BITS], + Integer(1 << (index & (BITMAP_BITS - 1)), bitmap_rprimitive), + IntOp.AND, + line, + ) + b = self.add(ComparisonOp(o, Integer(0, bitmap_rprimitive), ComparisonOp.EQ)) + self.add(Branch(b, error_block, body_block, Branch.BOOL)) + self.activate_block(error_block) + self.add(Assign(target, self.coerce(get_val(), target.type, line))) + self.goto(body_block) + self.activate_block(body_block) + + def maybe_add_implicit_return(self) -> None: + if is_none_rprimitive(self.ret_types[-1]) or is_object_rprimitive(self.ret_types[-1]): + self.add_implicit_return() + else: + self.add_implicit_unreachable() + + def add_implicit_return(self) -> None: + block = self.builder.blocks[-1] + if not block.terminated: + retval = self.coerce(self.builder.none(), self.ret_types[-1], -1) + self.nonlocal_control[-1].gen_return(self, retval, self.fn_info.fitem.line) + + def add_implicit_unreachable(self) -> None: + block = self.builder.blocks[-1] + if not block.terminated: + self.add(Unreachable()) + + def disallow_class_assignments(self, lvalues: list[Lvalue], line: int) -> None: + # Some best-effort attempts to disallow assigning to class + # variables that aren't marked ClassVar, since we blatantly + # miscompile the interaction between instance and class + # variables. + for lvalue in lvalues: + if ( + isinstance(lvalue, MemberExpr) + and isinstance(lvalue.expr, RefExpr) + and isinstance(lvalue.expr.node, TypeInfo) + ): + var = lvalue.expr.node[lvalue.name].node + if isinstance(var, Var) and not var.is_classvar: + self.error("Only class variables defined as ClassVar can be assigned to", line) + + def non_function_scope(self) -> bool: + # Currently the stack always has at least two items: dummy and top-level. + return len(self.fn_infos) <= 2 + + def top_level_fn_info(self) -> FuncInfo | None: + if self.non_function_scope(): + return None + return self.fn_infos[2] + + def init_final_static( + self, + lvalue: Lvalue, + rvalue_reg: Value, + class_name: str | None = None, + *, + type_override: RType | None = None, + ) -> None: + assert isinstance(lvalue, NameExpr) + assert isinstance(lvalue.node, Var) + if lvalue.node.final_value is None: + if class_name is None: + name = lvalue.name + else: + name = f"{class_name}.{lvalue.name}" + assert name is not None, "Full name not set for variable" + coerced = self.coerce(rvalue_reg, type_override or self.node_type(lvalue), lvalue.line) + self.final_names.append((name, coerced.type)) + self.add(InitStatic(coerced, name, self.module_name)) + + def load_final_static( + self, fullname: str, typ: RType, line: int, error_name: str | None = None + ) -> Value: + split_name = split_target(self.graph, fullname) + assert split_name is not None + module, name = split_name + return self.builder.load_static_checked( + typ, + name, + module, + line=line, + error_msg=f'value for final name "{error_name}" was not set', + ) + + def init_type_var(self, value: Value, name: str, line: int) -> None: + unique_name = name + "___" + str(line) + self.type_var_names.append(unique_name) + self.add(InitStatic(value, unique_name, self.module_name, namespace=NAMESPACE_TYPE_VAR)) + + def load_type_var(self, name: str, line: int) -> Value: + return self.add( + LoadStatic( + object_rprimitive, + name + "___" + str(line), + self.module_name, + namespace=NAMESPACE_TYPE_VAR, + ) + ) + + def load_literal_value(self, val: int | str | bytes | float | complex | bool) -> Value: + """Load value of a final name, class-level attribute, or constant folded expression.""" + if isinstance(val, bool): + if val: + return self.true() + else: + return self.false() + elif isinstance(val, int): + return self.builder.load_int(val) + elif isinstance(val, float): + return self.builder.load_float(val) + elif isinstance(val, str): + return self.builder.load_str(val) + elif isinstance(val, bytes): + return self.builder.load_bytes(val) + elif isinstance(val, complex): + return self.builder.load_complex(val) + else: + assert False, "Unsupported literal value" + + def get_assignment_target( + self, lvalue: Lvalue, line: int = -1, *, for_read: bool = False + ) -> AssignmentTarget: + if line == -1: + line = lvalue.line + if isinstance(lvalue, NameExpr): + # If we are visiting a decorator, then the SymbolNode we really want to be looking at + # is the function that is decorated, not the entire Decorator node itself. + symbol = lvalue.node + if isinstance(symbol, Decorator): + symbol = symbol.func + if symbol is None: + # Semantic analyzer doesn't create ad-hoc Vars for special forms. + assert lvalue.is_special_form + symbol = Var(lvalue.name) + if not for_read and isinstance(symbol, Var) and symbol.is_cls: + self.error("Cannot assign to the first argument of classmethod", line) + if lvalue.kind == LDEF: + if symbol not in self.symtables[-1]: + if isinstance(symbol, Var) and not isinstance(symbol.type, DeletedType): + reg_type = self.type_to_rtype(symbol.type) + else: + reg_type = self.node_type(lvalue) + # If the function is a generator function, then first define a new variable + # in the current function's environment class. Next, define a target that + # refers to the newly defined variable in that environment class. Add the + # target to the table containing class environment variables, as well as the + # current environment. + if self.fn_info.is_generator: + return self.add_var_to_env_class( + symbol, reg_type, self.fn_info.generator_class, reassign=False + ) + + # Otherwise define a new local variable. + return self.add_local_reg(symbol, reg_type) + else: + # Assign to a previously defined variable. + return self.lookup(symbol) + elif lvalue.kind == GDEF: + globals_dict = self.load_globals_dict() + name = self.load_str(lvalue.name) + return AssignmentTargetIndex(globals_dict, name) + else: + assert False, lvalue.kind + elif isinstance(lvalue, IndexExpr): + # Indexed assignment x[y] = e + base = self.accept(lvalue.base) + index = self.accept(lvalue.index) + return AssignmentTargetIndex(base, index) + elif isinstance(lvalue, MemberExpr): + # Attribute assignment x.y = e + can_borrow = self.is_native_attr_ref(lvalue) + obj = self.accept(lvalue.expr, can_borrow=can_borrow) + return AssignmentTargetAttr(obj, lvalue.name, can_borrow=can_borrow) + elif isinstance(lvalue, TupleExpr): + # Multiple assignment a, ..., b = e + star_idx: int | None = None + lvalues = [] + for idx, item in enumerate(lvalue.items): + targ = self.get_assignment_target(item) + lvalues.append(targ) + if isinstance(item, StarExpr): + if star_idx is not None: + self.error("Two starred expressions in assignment", line) + star_idx = idx + + return AssignmentTargetTuple(lvalues, star_idx) + + elif isinstance(lvalue, StarExpr): + return self.get_assignment_target(lvalue.expr) + + assert False, "Unsupported lvalue: %r" % lvalue + + def read( + self, target: Value | AssignmentTarget, line: int = -1, can_borrow: bool = False + ) -> Value: + if isinstance(target, Value): + return target + if isinstance(target, AssignmentTargetRegister): + return target.register + if isinstance(target, AssignmentTargetIndex): + reg = self.gen_method_call( + target.base, "__getitem__", [target.index], target.type, line + ) + if reg is not None: + return reg + assert False, target.base.type + if isinstance(target, AssignmentTargetAttr): + if isinstance(target.obj.type, RInstance) and target.obj.type.class_ir.is_ext_class: + borrow = can_borrow and target.can_borrow + return self.add(GetAttr(target.obj, target.attr, line, borrow=borrow)) + else: + return self.py_get_attr(target.obj, target.attr, line) + + assert False, "Unsupported lvalue: %r" % target + + def assign(self, target: Register | AssignmentTarget, rvalue_reg: Value, line: int) -> None: + if isinstance(target, Register): + self.add(Assign(target, self.coerce_rvalue(rvalue_reg, target.type, line))) + elif isinstance(target, AssignmentTargetRegister): + rvalue_reg = self.coerce_rvalue(rvalue_reg, target.type, line) + self.add(Assign(target.register, rvalue_reg)) + elif isinstance(target, AssignmentTargetAttr): + if isinstance(target.obj_type, RInstance): + rvalue_reg = self.coerce_rvalue(rvalue_reg, target.type, line) + self.add(SetAttr(target.obj, target.attr, rvalue_reg, line)) + else: + key = self.load_str(target.attr) + boxed_reg = self.builder.box(rvalue_reg) + self.primitive_op(py_setattr_op, [target.obj, key, boxed_reg], line) + elif isinstance(target, AssignmentTargetIndex): + target_reg2 = self.gen_method_call( + target.base, "__setitem__", [target.index, rvalue_reg], None, line + ) + assert target_reg2 is not None, target.base.type + elif isinstance(target, AssignmentTargetTuple): + if isinstance(rvalue_reg.type, RTuple) and target.star_idx is None: + rtypes = rvalue_reg.type.types + assert len(rtypes) == len(target.items) + for i in range(len(rtypes)): + item_value = self.add(TupleGet(rvalue_reg, i, line)) + self.assign(target.items[i], item_value, line) + elif ( + is_list_rprimitive(rvalue_reg.type) or is_tuple_rprimitive(rvalue_reg.type) + ) and target.star_idx is None: + self.process_sequence_assignment(target, rvalue_reg, line) + else: + self.process_iterator_tuple_assignment(target, rvalue_reg, line) + else: + assert False, "Unsupported assignment target" + + def coerce_rvalue(self, rvalue: Value, rtype: RType, line: int) -> Value: + if is_float_rprimitive(rtype) and is_tagged(rvalue.type): + typename = rvalue.type.short_name() + if typename == "short_int": + typename = "int" + self.error( + "Incompatible value representations in assignment " + + f'(expression has type "{typename}", variable has type "float")', + line, + ) + return self.coerce(rvalue, rtype, line) + + def process_sequence_assignment( + self, target: AssignmentTargetTuple, rvalue: Value, line: int + ) -> None: + """Process assignment like 'x, y = s', where s is a variable-length list or tuple.""" + # Check the length of sequence. + expected_len = Integer(len(target.items), c_pyssize_t_rprimitive) + self.builder.call_c(check_unpack_count_op, [rvalue, expected_len], line) + + # Read sequence items. + values = [] + for i in range(len(target.items)): + item = target.items[i] + index = self.builder.load_int(i) + if is_list_rprimitive(rvalue.type): + item_value = self.call_c(list_get_item_unsafe_op, [rvalue, index], line) + else: + item_value = self.builder.gen_method_call( + rvalue, "__getitem__", [index], item.type, line + ) + values.append(item_value) + + # Assign sequence items to the target lvalues. + for lvalue, value in zip(target.items, values): + self.assign(lvalue, value, line) + + def process_iterator_tuple_assignment_helper( + self, litem: AssignmentTarget, ritem: Value, line: int + ) -> None: + error_block, ok_block = BasicBlock(), BasicBlock() + self.add(Branch(ritem, error_block, ok_block, Branch.IS_ERROR)) + + self.activate_block(error_block) + self.add( + RaiseStandardError(RaiseStandardError.VALUE_ERROR, "not enough values to unpack", line) + ) + self.add(Unreachable()) + + self.activate_block(ok_block) + self.assign(litem, ritem, line) + + def process_iterator_tuple_assignment( + self, target: AssignmentTargetTuple, rvalue_reg: Value, line: int + ) -> None: + iterator = self.primitive_op(iter_op, [rvalue_reg], line) + + # This may be the whole lvalue list if there is no starred value + split_idx = target.star_idx if target.star_idx is not None else len(target.items) + + # Assign values before the first starred value + for litem in target.items[:split_idx]: + ritem = self.call_c(next_op, [iterator], line) + error_block, ok_block = BasicBlock(), BasicBlock() + self.add(Branch(ritem, error_block, ok_block, Branch.IS_ERROR)) + + self.activate_block(error_block) + self.add( + RaiseStandardError( + RaiseStandardError.VALUE_ERROR, "not enough values to unpack", line + ) + ) + self.add(Unreachable()) + + self.activate_block(ok_block) + + self.assign(litem, ritem, line) + + # Assign the starred value and all values after it + if target.star_idx is not None: + post_star_vals = target.items[split_idx + 1 :] + iter_list = self.primitive_op(to_list, [iterator], line) + iter_list_len = self.builtin_len(iter_list, line) + post_star_len = Integer(len(post_star_vals)) + condition = self.binary_op(post_star_len, iter_list_len, "<=", line) + + error_block, ok_block = BasicBlock(), BasicBlock() + self.add(Branch(condition, ok_block, error_block, Branch.BOOL)) + + self.activate_block(error_block) + self.add( + RaiseStandardError( + RaiseStandardError.VALUE_ERROR, "not enough values to unpack", line + ) + ) + self.add(Unreachable()) + + self.activate_block(ok_block) + + for litem in reversed(post_star_vals): + ritem = self.primitive_op(list_pop_last, [iter_list], line) + self.assign(litem, ritem, line) + + # Assign the starred value + self.assign(target.items[target.star_idx], iter_list, line) + + # There is no starred value, so check if there are extra values in rhs that + # have not been assigned. + else: + extra = self.call_c(next_op, [iterator], line) + error_block, ok_block = BasicBlock(), BasicBlock() + self.add(Branch(extra, ok_block, error_block, Branch.IS_ERROR)) + + self.activate_block(error_block) + self.add( + RaiseStandardError( + RaiseStandardError.VALUE_ERROR, "too many values to unpack", line + ) + ) + self.add(Unreachable()) + + self.activate_block(ok_block) + + def push_loop_stack(self, continue_block: BasicBlock, break_block: BasicBlock) -> None: + self.nonlocal_control.append( + LoopNonlocalControl(self.nonlocal_control[-1], continue_block, break_block) + ) + + def pop_loop_stack(self) -> None: + self.nonlocal_control.pop() + + def make_spill_target(self, type: RType) -> AssignmentTarget: + """Moves a given Value instance into the generator class' environment class.""" + name = f"{TEMP_ATTR_NAME}{self.temp_counter}" + self.temp_counter += 1 + target = self.add_var_to_env_class(Var(name), type, self.fn_info.generator_class) + return target + + def spill(self, value: Value) -> AssignmentTarget: + """Moves a given Value instance into the generator class' environment class.""" + target = self.make_spill_target(value.type) + # Shouldn't be able to fail, so -1 for line + self.assign(target, value, -1) + return target + + def maybe_spill(self, value: Value) -> Value | AssignmentTarget: + """ + Moves a given Value instance into the environment class for generator functions. For + non-generator functions, leaves the Value instance as it is. + + Returns an AssignmentTarget associated with the Value for generator functions and the + original Value itself for non-generator functions. + """ + if self.fn_info.is_generator: + return self.spill(value) + return value + + def maybe_spill_assignable(self, value: Value) -> Register | AssignmentTarget: + """ + Moves a given Value instance into the environment class for generator functions. For + non-generator functions, allocate a temporary Register. + + Returns an AssignmentTarget associated with the Value for generator functions and an + assignable Register for non-generator functions. + """ + if self.fn_info.is_generator: + return self.spill(value) + + if isinstance(value, Register): + return value + + # Allocate a temporary register for the assignable value. + reg = Register(value.type) + self.assign(reg, value, -1) + return reg + + def extract_int(self, e: Expression) -> int | None: + if isinstance(e, IntExpr): + return e.value + elif isinstance(e, UnaryExpr) and e.op == "-" and isinstance(e.expr, IntExpr): + return -e.expr.value + else: + return None + + def get_sequence_type(self, expr: Expression) -> RType: + return self.get_sequence_type_from_type(self.types[expr]) + + def get_sequence_type_from_type(self, target_type: Type) -> RType: + target_type = get_proper_type(target_type) + if isinstance(target_type, UnionType): + return RUnion.make_simplified_union( + [self.get_sequence_type_from_type(item) for item in target_type.items] + ) + assert isinstance(target_type, Instance), target_type + if target_type.type.fullname == "builtins.str": + return str_rprimitive + else: + return self.type_to_rtype(target_type.args[0]) + + def get_dict_base_type(self, expr: Expression) -> list[Instance]: + """Find dict type of a dict-like expression. + + This is useful for dict subclasses like SymbolTable. + """ + target_type = get_proper_type(self.types[expr]) + if isinstance(target_type, UnionType): + types = [get_proper_type(item) for item in target_type.items] + else: + types = [target_type] + + dict_types = [] + for t in types: + if isinstance(t, TypedDictType): + t = t.fallback + dict_base = next(base for base in t.type.mro if base.fullname == "typing.Mapping") + else: + assert isinstance(t, Instance), t + dict_base = next(base for base in t.type.mro if base.fullname == "builtins.dict") + dict_types.append(map_instance_to_supertype(t, dict_base)) + return dict_types + + def get_dict_key_type(self, expr: Expression) -> RType: + dict_base_types = self.get_dict_base_type(expr) + if len(dict_base_types) == 1: + return self.type_to_rtype(dict_base_types[0].args[0]) + else: + rtypes = [self.type_to_rtype(t.args[0]) for t in dict_base_types] + return RUnion.make_simplified_union(rtypes) + + def get_dict_value_type(self, expr: Expression) -> RType: + dict_base_types = self.get_dict_base_type(expr) + if len(dict_base_types) == 1: + return self.type_to_rtype(dict_base_types[0].args[1]) + else: + rtypes = [self.type_to_rtype(t.args[1]) for t in dict_base_types] + return RUnion.make_simplified_union(rtypes) + + def get_dict_item_type(self, expr: Expression) -> RType: + key_type = self.get_dict_key_type(expr) + value_type = self.get_dict_value_type(expr) + return RTuple([key_type, value_type]) + + def _analyze_iterable_item_type(self, expr: Expression) -> Type: + """Return the item type given by 'expr' in an iterable context.""" + # This logic is copied from mypy's TypeChecker.analyze_iterable_item_type. + if expr not in self.types: + # Mypy thinks this is unreachable. + iterable: ProperType = AnyType(TypeOfAny.from_error) + else: + iterable = get_proper_type(self.types[expr]) + echk = self.graph[self.module_name].type_checker().expr_checker + iterator = echk.check_method_call_by_name("__iter__", iterable, [], [], expr)[0] + + from mypy.join import join_types + + if isinstance(iterable, TupleType): + joined: Type = UninhabitedType() + for item in iterable.items: + joined = join_types(joined, item) + return joined + else: + # Non-tuple iterable. + return echk.check_method_call_by_name("__next__", iterator, [], [], expr)[0] + + def is_native_module(self, module: str) -> bool: + """Is the given module one compiled by mypyc?""" + return self.mapper.is_native_module(module) + + def is_native_ref_expr(self, expr: RefExpr) -> bool: + return self.mapper.is_native_ref_expr(expr) + + def is_native_module_ref_expr(self, expr: RefExpr) -> bool: + return self.mapper.is_native_module_ref_expr(expr) + + def is_synthetic_type(self, typ: TypeInfo) -> bool: + """Is a type something other than just a class we've created?""" + return typ.is_named_tuple or typ.is_newtype or typ.typeddict_type is not None + + def get_final_ref(self, expr: MemberExpr) -> tuple[str, Var, bool] | None: + """Check if `expr` is a final attribute. + + This needs to be done differently for class and module attributes to + correctly determine fully qualified name. Return a tuple that consists of + the qualified name, the corresponding Var node, and a flag indicating whether + the final name was defined in a compiled module. Return None if `expr` does not + refer to a final attribute. + """ + final_var = None + if isinstance(expr.expr, RefExpr) and isinstance(expr.expr.node, TypeInfo): + # a class attribute + sym = expr.expr.node.get(expr.name) + if sym and isinstance(sym.node, Var): + # Enum attribute are treated as final since they are added to the global cache + expr_fullname = expr.expr.node.bases[0].type.fullname + is_final = sym.node.is_final or expr_fullname == "enum.Enum" + if is_final: + final_var = sym.node + fullname = f"{sym.node.info.fullname}.{final_var.name}" + native = self.is_native_module(expr.expr.node.module_name) + elif self.is_module_member_expr(expr): + # a module attribute + if isinstance(expr.node, Var) and expr.node.is_final: + final_var = expr.node + fullname = expr.node.fullname + native = self.is_native_ref_expr(expr) + if final_var is not None: + return fullname, final_var, native + return None + + def emit_load_final( + self, final_var: Var, fullname: str, name: str, native: bool, typ: Type, line: int + ) -> Value | None: + """Emit code for loading value of a final name (if possible). + + Args: + final_var: Var corresponding to the final name + fullname: its qualified name + name: shorter name to show in errors + native: whether the name was defined in a compiled module + typ: its type + line: line number where loading occurs + """ + if final_var.final_value is not None: # this is safe even for non-native names + return self.load_literal_value(final_var.final_value) + elif native and module_prefix(self.graph, fullname): + return self.load_final_static(fullname, self.mapper.type_to_rtype(typ), line, name) + else: + return None + + def is_module_member_expr(self, expr: MemberExpr) -> bool: + return isinstance(expr.expr, RefExpr) and isinstance(expr.expr.node, MypyFile) + + def call_refexpr_with_args( + self, expr: CallExpr, callee: RefExpr, arg_values: list[Value] + ) -> Value: + # Handle data-driven special-cased primitive call ops. + if callee.fullname and expr.arg_kinds == [ARG_POS] * len(arg_values): + fullname = get_call_target_fullname(callee) + primitive_candidates = function_ops.get(fullname, []) + target = self.builder.matching_primitive_op( + primitive_candidates, arg_values, expr.line, self.node_type(expr) + ) + if target: + return target + + # Standard native call if signature and fullname are good and all arguments are positional + # or named. + callee_node = callee.node + if isinstance(callee_node, OverloadedFuncDef): + callee_node = callee_node.impl + # TODO: use native calls for any decorated functions which have all their decorators + # removed, not just singledispatch functions (which we don't do now just in case those + # decorated functions are callable classes or cannot be called without the python API for + # some other reason) + if ( + isinstance(callee_node, Decorator) + and callee_node.func not in self.fdefs_to_decorators + and callee_node.func in self.singledispatch_impls + ): + callee_node = callee_node.func + if ( + callee_node is not None + and callee.fullname + and callee_node in self.mapper.func_to_decl + and all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds) + ): + decl = self.mapper.func_to_decl[callee_node] + return self.builder.call(decl, arg_values, expr.arg_kinds, expr.arg_names, expr.line) + + # Fall back to a Python call + function = self.accept(callee) + return self.py_call( + function, arg_values, expr.line, arg_kinds=expr.arg_kinds, arg_names=expr.arg_names + ) + + def shortcircuit_expr(self, expr: OpExpr) -> Value: + return self.builder.shortcircuit_helper( + expr.op, + self.node_type(expr), + lambda: self.accept(expr.left), + lambda: self.accept(expr.right), + expr.line, + ) + + # Basic helpers + + def flatten_classes(self, arg: RefExpr | TupleExpr) -> list[ClassIR] | None: + """Flatten classes in isinstance(obj, (A, (B, C))). + + If at least one item is not a reference to a native class, return None. + """ + if isinstance(arg, RefExpr): + if isinstance(arg.node, TypeInfo) and self.is_native_module_ref_expr(arg): + ir = self.mapper.type_to_ir.get(arg.node) + if ir: + return [ir] + return None + else: + res: list[ClassIR] = [] + for item in arg.items: + if isinstance(item, (RefExpr, TupleExpr)): + item_part = self.flatten_classes(item) + if item_part is None: + return None + res.extend(item_part) + else: + return None + return res + + def enter(self, fn_info: FuncInfo | str = "") -> None: + if isinstance(fn_info, str): + fn_info = FuncInfo(name=fn_info) + self.builder = LowLevelIRBuilder(self.errors, self.options) + self.builder.set_module(self.module_name, self.module_path) + self.builders.append(self.builder) + self.symtables.append({}) + self.runtime_args.append([]) + self.fn_info = fn_info + self.fn_infos.append(self.fn_info) + self.ret_types.append(none_rprimitive) + if fn_info.is_generator: + self.nonlocal_control.append(GeneratorNonlocalControl()) + else: + self.nonlocal_control.append(BaseNonlocalControl()) + self.activate_block(BasicBlock()) + + def leave(self) -> tuple[list[Register], list[RuntimeArg], list[BasicBlock], RType, FuncInfo]: + builder = self.builders.pop() + self.symtables.pop() + runtime_args = self.runtime_args.pop() + ret_type = self.ret_types.pop() + fn_info = self.fn_infos.pop() + self.nonlocal_control.pop() + self.builder = self.builders[-1] + self.fn_info = self.fn_infos[-1] + return builder.args, runtime_args, builder.blocks, ret_type, fn_info + + @contextmanager + def enter_method( + self, + class_ir: ClassIR, + name: str, + ret_type: RType, + fn_info: FuncInfo | str = "", + self_type: RType | None = None, + ) -> Iterator[None]: + """Generate IR for a method. + + If the method takes arguments, you should immediately afterwards call + add_argument() for each non-self argument (self is created implicitly). + + Args: + class_ir: Add method to this class + name: Short name of the method + ret_type: Return type of the method + fn_info: Optionally, additional information about the method + self_type: If not None, override default type of the implicit 'self' + argument (by default, derive type from class_ir) + """ + self.enter(fn_info) + self.function_name_stack.append(name) + self.class_ir_stack.append(class_ir) + self.ret_types[-1] = ret_type + if self_type is None: + self_type = RInstance(class_ir) + self.add_argument(SELF_NAME, self_type) + try: + yield + finally: + arg_regs, args, blocks, ret_type, fn_info = self.leave() + sig = FuncSignature(args, ret_type) + name = self.function_name_stack.pop() + class_ir = self.class_ir_stack.pop() + decl = FuncDecl(name, class_ir.name, self.module_name, sig) + ir = FuncIR(decl, arg_regs, blocks) + class_ir.methods[name] = ir + class_ir.method_decls[name] = ir.decl + self.functions.append(ir) + + def add_argument(self, var: str | Var, typ: RType, kind: ArgKind = ARG_POS) -> Register: + """Declare an argument in the current function. + + You should use this instead of directly calling add_local() in new code. + """ + if isinstance(var, str): + var = Var(var) + reg = self.add_local(var, typ, is_arg=True) + self.runtime_args[-1].append(RuntimeArg(var.name, typ, kind)) + return reg + + def lookup(self, symbol: SymbolNode) -> SymbolTarget: + return self.symtables[-1][symbol] + + def add_local(self, symbol: SymbolNode, typ: RType, is_arg: bool = False) -> Register: + """Add register that represents a symbol to the symbol table. + + Args: + is_arg: is this a function argument + """ + assert isinstance(symbol, SymbolNode) + reg = Register( + typ, remangle_redefinition_name(symbol.name), is_arg=is_arg, line=symbol.line + ) + self.symtables[-1][symbol] = AssignmentTargetRegister(reg) + if is_arg: + self.builder.args.append(reg) + return reg + + def add_local_reg( + self, symbol: SymbolNode, typ: RType, is_arg: bool = False + ) -> AssignmentTargetRegister: + """Like add_local, but return an assignment target instead of value.""" + self.add_local(symbol, typ, is_arg) + target = self.symtables[-1][symbol] + assert isinstance(target, AssignmentTargetRegister) + return target + + def add_self_to_env(self, cls: ClassIR) -> AssignmentTargetRegister: + """Low-level function that adds a 'self' argument. + + This is only useful if using enter() instead of enter_method(). + """ + return self.add_local_reg(Var(SELF_NAME), RInstance(cls), is_arg=True) + + def add_target(self, symbol: SymbolNode, target: SymbolTarget) -> SymbolTarget: + self.symtables[-1][symbol] = target + return target + + def type_to_rtype(self, typ: Type | None) -> RType: + return self.mapper.type_to_rtype(typ) + + def node_type(self, node: Expression) -> RType: + if isinstance(node, IntExpr): + # TODO: Don't special case IntExpr + return int_rprimitive + if node not in self.types: + return object_rprimitive + mypy_type = self.types[node] + return self.type_to_rtype(mypy_type) + + def add_var_to_env_class( + self, var: SymbolNode, rtype: RType, base: FuncInfo | ImplicitClass, reassign: bool = False + ) -> AssignmentTarget: + # First, define the variable name as an attribute of the environment class, and then + # construct a target for that attribute. + name = remangle_redefinition_name(var.name) + self.fn_info.env_class.attributes[name] = rtype + attr_target = AssignmentTargetAttr(base.curr_env_reg, name) + + if reassign: + # Read the local definition of the variable, and set the corresponding attribute of + # the environment class' variable to be that value. + reg = self.read(self.lookup(var), self.fn_info.fitem.line) + self.add(SetAttr(base.curr_env_reg, name, reg, self.fn_info.fitem.line)) + + # Override the local definition of the variable to instead point at the variable in + # the environment class. + return self.add_target(var, attr_target) + + def is_builtin_ref_expr(self, expr: RefExpr) -> bool: + assert expr.node, "RefExpr not resolved" + return "." in expr.node.fullname and expr.node.fullname.split(".")[0] == "builtins" + + def load_global(self, expr: NameExpr) -> Value: + """Loads a Python-level global. + + This takes a NameExpr and uses its name as a key to retrieve the corresponding PyObject * + from the _globals dictionary in the C-generated code. + """ + # If the global is from 'builtins', turn it into a module attr load instead + if self.is_builtin_ref_expr(expr): + assert expr.node, "RefExpr not resolved" + return self.load_module_attr_by_fullname(expr.node.fullname, expr.line) + if ( + self.is_native_module_ref_expr(expr) + and isinstance(expr.node, TypeInfo) + and not self.is_synthetic_type(expr.node) + ): + assert expr.fullname + return self.load_native_type_object(expr.fullname) + return self.load_global_str(expr.name, expr.line) + + def load_global_str(self, name: str, line: int) -> Value: + _globals = self.load_globals_dict() + reg = self.load_str(name) + return self.primitive_op(dict_get_item_op, [_globals, reg], line) + + def load_globals_dict(self) -> Value: + return self.add(LoadStatic(dict_rprimitive, "globals", self.module_name)) + + def load_module_attr_by_fullname(self, fullname: str, line: int) -> Value: + module, _, name = fullname.rpartition(".") + left = self.load_module(module) + return self.py_get_attr(left, name, line) + + def is_native_attr_ref(self, expr: MemberExpr) -> bool: + """Is expr a direct reference to a native (struct) attribute of an instance?""" + obj_rtype = self.node_type(expr.expr) + return ( + isinstance(obj_rtype, RInstance) + and obj_rtype.class_ir.is_ext_class + and obj_rtype.class_ir.has_attr(expr.name) + and not obj_rtype.class_ir.get_method(expr.name) + ) + + def mark_block_unreachable(self) -> None: + """Mark statements in the innermost block being processed as unreachable. + + This should be called after a statement that unconditionally leaves the + block, such as 'break' or 'return'. + """ + self.block_reachable_stack[-1] = False + + # Lacks a good type because there wasn't a reasonable type in 3.5 :( + def catch_errors(self, line: int) -> Any: + return catch_errors(self.module_path, line) + + def warning(self, msg: str, line: int) -> None: + self.errors.warning(msg, self.module_path, line) + + def error(self, msg: str, line: int) -> None: + self.errors.error(msg, self.module_path, line) + + def note(self, msg: str, line: int) -> None: + self.errors.note(msg, self.module_path, line) + + def add_function(self, func_ir: FuncIR, line: int) -> None: + name = (func_ir.class_name, func_ir.name) + if name in self.function_names: + self.error(f'Duplicate definition of "{name[1]}" not supported by mypyc', line) + return + self.function_names.add(name) + self.functions.append(func_ir) + + +def gen_arg_defaults(builder: IRBuilder) -> None: + """Generate blocks for arguments that have default values. + + If the passed value is an error value, then assign the default + value to the argument. + """ + fitem = builder.fn_info.fitem + nb = 0 + for arg in fitem.arguments: + if arg.initializer: + target = builder.lookup(arg.variable) + + def get_default() -> Value: + assert arg.initializer is not None + + # If it is constant, don't bother storing it + if is_constant(arg.initializer): + return builder.accept(arg.initializer) + + # Because gen_arg_defaults runs before calculate_arg_defaults, we + # add the static/attribute to final_names/the class here. + elif not builder.fn_info.is_nested: + name = fitem.fullname + "." + arg.variable.name + builder.final_names.append((name, target.type)) + return builder.add(LoadStatic(target.type, name, builder.module_name)) + else: + name = arg.variable.name + builder.fn_info.callable_class.ir.attributes[name] = target.type + return builder.add( + GetAttr(builder.fn_info.callable_class.self_reg, name, arg.line) + ) + + assert isinstance(target, AssignmentTargetRegister) + reg = target.register + if not reg.type.error_overlap: + builder.assign_if_null(target.register, get_default, arg.initializer.line) + else: + builder.assign_if_bitmap_unset( + target.register, get_default, nb, arg.initializer.line + ) + nb += 1 + + +def remangle_redefinition_name(name: str) -> str: + """Remangle names produced by mypy when allow-redefinition is used and a name + is used with multiple types within a single block. + + We only need to do this for locals, because the name is used as the name of the register; + for globals, the name itself is stored in a register for the purpose of doing dict + lookups. + """ + return name.replace("'", "__redef__") + + +def get_call_target_fullname(ref: RefExpr) -> str: + if isinstance(ref.node, TypeAlias): + # Resolve simple type aliases. In calls they evaluate to the type they point to. + target = get_proper_type(ref.node.target) + if isinstance(target, Instance): + return target.type.fullname + return ref.fullname + + +def create_type_params( + builder: IRBuilder, typing_mod: Value, type_args: list[TypeParam], line: int +) -> list[Value]: + """Create objects representing various kinds of Python 3.12 type parameters. + + The "typing_mod" argument is the "_typing" module object. The type objects + are looked up from it. + + The returned list has one item for each "type_args" item, in the same order. + Each item is either a TypeVar, TypeVarTuple or ParamSpec instance. + """ + tvs = [] + type_var_imported: Value | None = None + for type_param in type_args: + if type_param.kind == TYPE_VAR_KIND: + if type_var_imported: + # Reuse previously imported value as a minor optimization + tvt = type_var_imported + else: + tvt = builder.py_get_attr(typing_mod, "TypeVar", line) + type_var_imported = tvt + elif type_param.kind == TYPE_VAR_TUPLE_KIND: + tvt = builder.py_get_attr(typing_mod, "TypeVarTuple", line) + else: + assert type_param.kind == PARAM_SPEC_KIND + tvt = builder.py_get_attr(typing_mod, "ParamSpec", line) + if type_param.kind != TYPE_VAR_TUPLE_KIND: + # To match runtime semantics, pass infer_variance=True + tv = builder.py_call( + tvt, + [builder.load_str(type_param.name), builder.true()], + line, + arg_kinds=[ARG_POS, ARG_NAMED], + arg_names=[None, "infer_variance"], + ) + else: + tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line) + builder.init_type_var(tv, type_param.name, line) + tvs.append(tv) + return tvs diff --git a/mypyc/irbuild/callable_class.py b/mypyc/irbuild/callable_class.py new file mode 100644 index 000000000000..599dbb81f767 --- /dev/null +++ b/mypyc/irbuild/callable_class.py @@ -0,0 +1,173 @@ +"""Generate a class that represents a nested function. + +The class defines __call__ for calling the function and allows access to +non-local variables defined in outer scopes. +""" + +from __future__ import annotations + +from mypyc.common import ENV_ATTR_NAME, SELF_NAME +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg +from mypyc.ir.ops import BasicBlock, Call, Register, Return, SetAttr, Value +from mypyc.ir.rtypes import RInstance, object_rprimitive +from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.context import FuncInfo, ImplicitClass +from mypyc.primitives.misc_ops import method_new_op + + +def setup_callable_class(builder: IRBuilder) -> None: + """Generate an (incomplete) callable class representing a function. + + This can be a nested function or a function within a non-extension + class. Also set up the 'self' variable for that class. + + This takes the most recently visited function and returns a + ClassIR to represent that function. Each callable class contains + an environment attribute which points to another ClassIR + representing the environment class where some of its variables can + be accessed. + + Note that some methods, such as '__call__', are not yet + created here. Use additional functions, such as + add_call_to_callable_class(), to add them. + + Return a newly constructed ClassIR representing the callable + class for the nested function. + """ + # Check to see that the name has not already been taken. If so, + # rename the class. We allow multiple uses of the same function + # name because this is valid in if-else blocks. Example: + # + # if True: + # def foo(): ----> foo_obj() + # return True + # else: + # def foo(): ----> foo_obj_0() + # return False + name = base_name = f"{builder.fn_info.namespaced_name()}_obj" + count = 0 + while name in builder.callable_class_names: + name = base_name + "_" + str(count) + count += 1 + builder.callable_class_names.add(name) + + # Define the actual callable class ClassIR, and set its + # environment to point at the previously defined environment + # class. + callable_class_ir = ClassIR(name, builder.module_name, is_generated=True) + + # The functools @wraps decorator attempts to call setattr on + # nested functions, so we create a dict for these nested + # functions. + # https://github.com/python/cpython/blob/3.7/Lib/functools.py#L58 + if builder.fn_info.is_nested: + callable_class_ir.has_dict = True + + # If the enclosing class doesn't contain nested (which will happen if + # this is a toplevel lambda), don't set up an environment. + if builder.fn_infos[-2].contains_nested: + callable_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class) + callable_class_ir.mro = [callable_class_ir] + builder.fn_info.callable_class = ImplicitClass(callable_class_ir) + builder.classes.append(callable_class_ir) + + # Add a 'self' variable to the environment of the callable class, + # and store that variable in a register to be accessed later. + self_target = builder.add_self_to_env(callable_class_ir) + builder.fn_info.callable_class.self_reg = builder.read(self_target, builder.fn_info.fitem.line) + + +def add_call_to_callable_class( + builder: IRBuilder, + args: list[Register], + blocks: list[BasicBlock], + sig: FuncSignature, + fn_info: FuncInfo, +) -> FuncIR: + """Generate a '__call__' method for a callable class representing a nested function. + + This takes the blocks and signature associated with a function + definition and uses those to build the '__call__' method of a + given callable class, used to represent that function. + """ + # Since we create a method, we also add a 'self' parameter. + nargs = len(sig.args) - sig.num_bitmap_args + sig = FuncSignature( + (RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args[:nargs], sig.ret_type + ) + call_fn_decl = FuncDecl("__call__", fn_info.callable_class.ir.name, builder.module_name, sig) + call_fn_ir = FuncIR( + call_fn_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name + ) + fn_info.callable_class.ir.methods["__call__"] = call_fn_ir + fn_info.callable_class.ir.method_decls["__call__"] = call_fn_decl + return call_fn_ir + + +def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None: + """Generate the '__get__' method for a callable class.""" + line = fn_info.fitem.line + with builder.enter_method( + fn_info.callable_class.ir, + "__get__", + object_rprimitive, + fn_info, + self_type=object_rprimitive, + ): + instance = builder.add_argument("instance", object_rprimitive) + builder.add_argument("owner", object_rprimitive) + + # If accessed through the class, just return the callable + # object. If accessed through an object, create a new bound + # instance method object. + instance_block, class_block = BasicBlock(), BasicBlock() + comparison = builder.translate_is_op( + builder.read(instance), builder.none_object(), "is", line + ) + builder.add_bool_branch(comparison, class_block, instance_block) + + builder.activate_block(class_block) + builder.add(Return(builder.self())) + + builder.activate_block(instance_block) + builder.add( + Return(builder.call_c(method_new_op, [builder.self(), builder.read(instance)], line)) + ) + + +def instantiate_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> Value: + """Create an instance of a callable class for a function. + + Calls to the function will actually call this instance. + + Note that fn_info refers to the function being assigned, whereas + builder.fn_info refers to the function encapsulating the function + being turned into a callable class. + """ + fitem = fn_info.fitem + func_reg = builder.add(Call(fn_info.callable_class.ir.ctor, [], fitem.line)) + + # Set the environment attribute of the callable class to point at + # the environment class defined in the callable class' immediate + # outer scope. Note that there are three possible environment + # class registers we may use. This depends on what the encapsulating + # (parent) function is: + # + # - A nested function: the callable class is instantiated + # from the current callable class' '__call__' function, and hence + # the callable class' environment register is used. + # - A generator function: the callable class is instantiated + # from the '__next__' method of the generator class, and hence the + # environment of the generator class is used. + # - Regular function: we use the environment of the original function. + curr_env_reg = None + if builder.fn_info.is_generator: + curr_env_reg = builder.fn_info.generator_class.curr_env_reg + elif builder.fn_info.is_nested: + curr_env_reg = builder.fn_info.callable_class.curr_env_reg + elif builder.fn_info.contains_nested: + curr_env_reg = builder.fn_info.curr_env_reg + if curr_env_reg: + builder.add(SetAttr(func_reg, ENV_ATTR_NAME, curr_env_reg, fitem.line)) + return func_reg diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py new file mode 100644 index 000000000000..6072efa2c593 --- /dev/null +++ b/mypyc/irbuild/classdef.py @@ -0,0 +1,906 @@ +"""Transform class definitions from the mypy AST form to IR.""" + +from __future__ import annotations + +import typing_extensions +from abc import abstractmethod +from typing import Callable, Final + +from mypy.nodes import ( + EXCLUDED_ENUM_ATTRIBUTES, + TYPE_VAR_TUPLE_KIND, + AssignmentStmt, + CallExpr, + ClassDef, + Decorator, + EllipsisExpr, + ExpressionStmt, + FuncDef, + Lvalue, + MemberExpr, + NameExpr, + OverloadedFuncDef, + PassStmt, + RefExpr, + StrExpr, + TempNode, + TypeInfo, + TypeParam, + is_class_var, +) +from mypy.types import Instance, UnboundType, get_proper_type +from mypyc.common import PROPSET_PREFIX +from mypyc.ir.class_ir import ClassIR, NonExtClassInfo +from mypyc.ir.func_ir import FuncDecl, FuncSignature +from mypyc.ir.ops import ( + NAMESPACE_TYPE, + BasicBlock, + Branch, + Call, + InitStatic, + LoadAddress, + LoadErrorValue, + LoadStatic, + MethodCall, + Register, + Return, + SetAttr, + TupleSet, + Value, +) +from mypyc.ir.rtypes import ( + RType, + bool_rprimitive, + dict_rprimitive, + is_none_rprimitive, + is_object_rprimitive, + is_optional_type, + object_rprimitive, +) +from mypyc.irbuild.builder import IRBuilder, create_type_params +from mypyc.irbuild.function import ( + gen_property_getter_ir, + gen_property_setter_ir, + handle_ext_method, + handle_non_ext_method, + load_type, +) +from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator +from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op +from mypyc.primitives.generic_ops import ( + iter_op, + next_op, + py_get_item_op, + py_hasattr_op, + py_setattr_op, +) +from mypyc.primitives.misc_ops import ( + dataclass_sleight_of_hand, + import_op, + not_implemented_op, + py_calc_meta_op, + pytype_from_template_op, + type_object_op, +) +from mypyc.subtype import is_subtype + + +def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None: + """Create IR for a class definition. + + This can generate both extension (native) and non-extension + classes. These are generated in very different ways. In the + latter case we construct a Python type object at runtime by doing + the equivalent of "type(name, bases, dict)" in IR. Extension + classes are defined via C structs that are generated later in + mypyc.codegen.emitclass. + + This is the main entry point to this module. + """ + ir = builder.mapper.type_to_ir[cdef.info] + + # We do this check here because the base field of parent + # classes aren't necessarily populated yet at + # prepare_class_def time. + if any(ir.base_mro[i].base != ir.base_mro[i + 1] for i in range(len(ir.base_mro) - 1)): + builder.error("Multiple inheritance is not supported (except for traits)", cdef.line) + + if ir.allow_interpreted_subclasses: + for parent in ir.mro: + if not parent.allow_interpreted_subclasses: + builder.error( + 'Base class "{}" does not allow interpreted subclasses'.format( + parent.fullname + ), + cdef.line, + ) + + # Currently, we only create non-extension classes for classes that are + # decorated or inherit from Enum. Classes decorated with @trait do not + # apply here, and are handled in a different way. + if ir.is_ext_class: + cls_type = dataclass_type(cdef) + if cls_type is None: + cls_builder: ClassBuilder = ExtClassBuilder(builder, cdef) + elif cls_type in ["dataclasses", "attr-auto"]: + cls_builder = DataClassBuilder(builder, cdef) + elif cls_type == "attr": + cls_builder = AttrsClassBuilder(builder, cdef) + else: + raise ValueError(cls_type) + else: + cls_builder = NonExtClassBuilder(builder, cdef) + + for stmt in cdef.defs.body: + if isinstance(stmt, OverloadedFuncDef) and stmt.is_property: + if isinstance(cls_builder, NonExtClassBuilder): + # properties with both getters and setters in non_extension + # classes not supported + builder.error("Property setters not supported in non-extension classes", stmt.line) + for item in stmt.items: + with builder.catch_errors(stmt.line): + cls_builder.add_method(get_func_def(item)) + elif isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef)): + # Ignore plugin generated methods (since they have no + # bodies to compile and will need to have the bodies + # provided by some other mechanism.) + if cdef.info.names[stmt.name].plugin_generated: + continue + with builder.catch_errors(stmt.line): + cls_builder.add_method(get_func_def(stmt)) + elif isinstance(stmt, PassStmt) or ( + isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr) + ): + continue + elif isinstance(stmt, AssignmentStmt): + if len(stmt.lvalues) != 1: + builder.error("Multiple assignment in class bodies not supported", stmt.line) + continue + lvalue = stmt.lvalues[0] + if not isinstance(lvalue, NameExpr): + builder.error( + "Only assignment to variables is supported in class bodies", stmt.line + ) + continue + # We want to collect class variables in a dictionary for both real + # non-extension classes and fake dataclass ones. + cls_builder.add_attr(lvalue, stmt) + + elif isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, StrExpr): + # Docstring. Ignore + pass + else: + builder.error("Unsupported statement in class body", stmt.line) + + # Generate implicit property setters/getters + for name, decl in ir.method_decls.items(): + if decl.implicit and decl.is_prop_getter: + getter_ir = gen_property_getter_ir(builder, decl, cdef, ir.is_trait) + builder.functions.append(getter_ir) + ir.methods[getter_ir.decl.name] = getter_ir + + setter_ir = None + setter_name = PROPSET_PREFIX + name + if setter_name in ir.method_decls: + setter_ir = gen_property_setter_ir( + builder, ir.method_decls[setter_name], cdef, ir.is_trait + ) + builder.functions.append(setter_ir) + ir.methods[setter_name] = setter_ir + + ir.properties[name] = (getter_ir, setter_ir) + # TODO: Generate glue method if needed? + # TODO: Do we need interpreted glue methods? Maybe not? + + cls_builder.finalize(ir) + + +class ClassBuilder: + """Create IR for a class definition. + + This is an abstract base class. + """ + + def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None: + self.builder = builder + self.cdef = cdef + self.attrs_to_cache: list[tuple[Lvalue, RType]] = [] + + @abstractmethod + def add_method(self, fdef: FuncDef) -> None: + """Add a method to the class IR""" + + @abstractmethod + def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None: + """Add an attribute to the class IR""" + + @abstractmethod + def finalize(self, ir: ClassIR) -> None: + """Perform any final operations to complete the class IR""" + + +class NonExtClassBuilder(ClassBuilder): + def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None: + super().__init__(builder, cdef) + self.non_ext = self.create_non_ext_info() + + def create_non_ext_info(self) -> NonExtClassInfo: + non_ext_bases = populate_non_ext_bases(self.builder, self.cdef) + non_ext_metaclass = find_non_ext_metaclass(self.builder, self.cdef, non_ext_bases) + non_ext_dict = setup_non_ext_dict( + self.builder, self.cdef, non_ext_metaclass, non_ext_bases + ) + # We populate __annotations__ for non-extension classes + # because dataclasses uses it to determine which attributes to compute on. + # TODO: Maybe generate more precise types for annotations + non_ext_anns = self.builder.call_c(dict_new_op, [], self.cdef.line) + return NonExtClassInfo(non_ext_dict, non_ext_bases, non_ext_anns, non_ext_metaclass) + + def add_method(self, fdef: FuncDef) -> None: + handle_non_ext_method(self.builder, self.non_ext, self.cdef, fdef) + + def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None: + add_non_ext_class_attr_ann(self.builder, self.non_ext, lvalue, stmt) + add_non_ext_class_attr( + self.builder, self.non_ext, lvalue, stmt, self.cdef, self.attrs_to_cache + ) + + def finalize(self, ir: ClassIR) -> None: + # Dynamically create the class via the type constructor + non_ext_class = load_non_ext_class(self.builder, ir, self.non_ext, self.cdef.line) + non_ext_class = load_decorated_class(self.builder, self.cdef, non_ext_class) + + # Save the decorated class + self.builder.add( + InitStatic(non_ext_class, self.cdef.name, self.builder.module_name, NAMESPACE_TYPE) + ) + + # Add the non-extension class to the dict + self.builder.primitive_op( + dict_set_item_op, + [ + self.builder.load_globals_dict(), + self.builder.load_str(self.cdef.name), + non_ext_class, + ], + self.cdef.line, + ) + + # Cache any cacheable class attributes + cache_class_attrs(self.builder, self.attrs_to_cache, self.cdef) + + +class ExtClassBuilder(ClassBuilder): + def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None: + super().__init__(builder, cdef) + # If the class is not decorated, generate an extension class for it. + self.type_obj: Value | None = allocate_class(builder, cdef) + + def skip_attr_default(self, name: str, stmt: AssignmentStmt) -> bool: + """Controls whether to skip generating a default for an attribute.""" + return False + + def add_method(self, fdef: FuncDef) -> None: + handle_ext_method(self.builder, self.cdef, fdef) + + def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None: + # Variable declaration with no body + if isinstance(stmt.rvalue, TempNode): + return + # Only treat marked class variables as class variables. + if not (is_class_var(lvalue) or stmt.is_final_def): + return + typ = self.builder.load_native_type_object(self.cdef.fullname) + value = self.builder.accept(stmt.rvalue) + self.builder.primitive_op( + py_setattr_op, [typ, self.builder.load_str(lvalue.name), value], stmt.line + ) + if self.builder.non_function_scope() and stmt.is_final_def: + self.builder.init_final_static(lvalue, value, self.cdef.name) + + def finalize(self, ir: ClassIR) -> None: + attrs_with_defaults, default_assignments = find_attr_initializers( + self.builder, self.cdef, self.skip_attr_default + ) + ir.attrs_with_defaults.update(attrs_with_defaults) + generate_attr_defaults_init(self.builder, self.cdef, default_assignments) + create_ne_from_eq(self.builder, self.cdef) + + +class DataClassBuilder(ExtClassBuilder): + # controls whether an __annotations__ attribute should be added to the class + # __dict__. This is not desirable for attrs classes where auto_attribs is + # disabled, as attrs will reject it. + add_annotations_to_dict = True + + def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None: + super().__init__(builder, cdef) + self.non_ext = self.create_non_ext_info() + + def create_non_ext_info(self) -> NonExtClassInfo: + """Set up a NonExtClassInfo to track dataclass attributes. + + In addition to setting up a normal extension class for dataclasses, + we also collect its class attributes like a non-extension class so + that we can hand them to the dataclass decorator. + """ + return NonExtClassInfo( + self.builder.call_c(dict_new_op, [], self.cdef.line), + self.builder.add(TupleSet([], self.cdef.line)), + self.builder.call_c(dict_new_op, [], self.cdef.line), + self.builder.add(LoadAddress(type_object_op.type, type_object_op.src, self.cdef.line)), + ) + + def skip_attr_default(self, name: str, stmt: AssignmentStmt) -> bool: + return stmt.type is not None + + def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None: + # We populate __annotations__ because dataclasses uses it to determine + # which attributes to compute on. + ann_type = get_proper_type(stmt.type) + if isinstance(ann_type, Instance): + return ann_type.type + return None + + def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None: + add_non_ext_class_attr_ann( + self.builder, self.non_ext, lvalue, stmt, self.get_type_annotation + ) + add_non_ext_class_attr( + self.builder, self.non_ext, lvalue, stmt, self.cdef, self.attrs_to_cache + ) + super().add_attr(lvalue, stmt) + + def finalize(self, ir: ClassIR) -> None: + """Generate code to finish instantiating a dataclass. + + This works by replacing all of the attributes on the class + (which will be descriptors) with whatever they would be in a + non-extension class, calling dataclass, then switching them back. + + The resulting class is an extension class and instances of it do not + have a __dict__ (unless something else requires it). + All methods written explicitly in the source are compiled and + may be called through the vtable while the methods generated + by dataclasses are interpreted and may not be. + + (If we just called dataclass without doing this, it would think that all + of the descriptors for our attributes are default values and generate an + incorrect constructor. We need to do the switch so that dataclass gets the + appropriate defaults.) + """ + super().finalize(ir) + assert self.type_obj + add_dunders_to_non_ext_dict( + self.builder, self.non_ext, self.cdef.line, self.add_annotations_to_dict + ) + dec = self.builder.accept( + next(d for d in self.cdef.decorators if is_dataclass_decorator(d)) + ) + self.builder.call_c( + dataclass_sleight_of_hand, + [dec, self.type_obj, self.non_ext.dict, self.non_ext.anns], + self.cdef.line, + ) + + +class AttrsClassBuilder(DataClassBuilder): + """Create IR for an attrs class where auto_attribs=False (the default). + + When auto_attribs is enabled, attrs classes behave similarly to dataclasses + (i.e. types are stored as annotations on the class) and are thus handled + by DataClassBuilder, but when auto_attribs is disabled the types are + provided via attr.ib(type=...) + """ + + add_annotations_to_dict = False + + def skip_attr_default(self, name: str, stmt: AssignmentStmt) -> bool: + return True + + def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None: + if isinstance(stmt.rvalue, CallExpr): + # find the type arg in `attr.ib(type=str)` + callee = stmt.rvalue.callee + if ( + isinstance(callee, MemberExpr) + and callee.fullname in ["attr.ib", "attr.attr"] + and "type" in stmt.rvalue.arg_names + ): + index = stmt.rvalue.arg_names.index("type") + type_name = stmt.rvalue.args[index] + if isinstance(type_name, NameExpr) and isinstance(type_name.node, TypeInfo): + lvalue = stmt.lvalues[0] + assert isinstance(lvalue, NameExpr) + return type_name.node + return None + + +def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value: + # OK AND NOW THE FUN PART + base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs + new_style_type_args = cdef.type_args + if new_style_type_args: + bases = [make_generic_base_class(builder, cdef.fullname, new_style_type_args, cdef.line)] + else: + bases = [] + + if base_exprs or new_style_type_args: + bases.extend([builder.accept(x) for x in base_exprs]) + tp_bases = builder.new_tuple(bases, cdef.line) + else: + tp_bases = builder.add(LoadErrorValue(object_rprimitive, is_borrowed=True)) + modname = builder.load_str(builder.module_name) + template = builder.add( + LoadStatic(object_rprimitive, cdef.name + "_template", builder.module_name, NAMESPACE_TYPE) + ) + # Create the class + tp = builder.call_c(pytype_from_template_op, [template, tp_bases, modname], cdef.line) + # Immediately fix up the trait vtables, before doing anything with the class. + ir = builder.mapper.type_to_ir[cdef.info] + if not ir.is_trait and not ir.builtin_base: + builder.add( + Call( + FuncDecl( + cdef.name + "_trait_vtable_setup", + None, + builder.module_name, + FuncSignature([], bool_rprimitive), + ), + [], + -1, + ) + ) + # Populate a '__mypyc_attrs__' field containing the list of attrs + builder.primitive_op( + py_setattr_op, + [ + tp, + builder.load_str("__mypyc_attrs__"), + create_mypyc_attrs_tuple(builder, builder.mapper.type_to_ir[cdef.info], cdef.line), + ], + cdef.line, + ) + + # Save the class + builder.add(InitStatic(tp, cdef.name, builder.module_name, NAMESPACE_TYPE)) + + # Add it to the dict + builder.primitive_op( + dict_set_item_op, [builder.load_globals_dict(), builder.load_str(cdef.name), tp], cdef.line + ) + + return tp + + +def make_generic_base_class( + builder: IRBuilder, fullname: str, type_args: list[TypeParam], line: int +) -> Value: + """Construct Generic[...] base class object for a new-style generic class (Python 3.12).""" + mod = builder.call_c(import_op, [builder.load_str("_typing")], line) + tvs = create_type_params(builder, mod, type_args, line) + args = [] + for tv, type_param in zip(tvs, type_args): + if type_param.kind == TYPE_VAR_TUPLE_KIND: + # Evaluate *Ts for a TypeVarTuple + it = builder.primitive_op(iter_op, [tv], line) + tv = builder.call_c(next_op, [it], line) + args.append(tv) + + gent = builder.py_get_attr(mod, "Generic", line) + if len(args) == 1: + arg = args[0] + else: + arg = builder.new_tuple(args, line) + + base = builder.primitive_op(py_get_item_op, [gent, arg], line) + return base + + +# Mypy uses these internally as base classes of TypedDict classes. These are +# lies and don't have any runtime equivalent. +MAGIC_TYPED_DICT_CLASSES: Final[tuple[str, ...]] = ( + "typing._TypedDict", + "typing_extensions._TypedDict", +) + + +def populate_non_ext_bases(builder: IRBuilder, cdef: ClassDef) -> Value: + """Create base class tuple of a non-extension class. + + The tuple is passed to the metaclass constructor. + """ + is_named_tuple = cdef.info.is_named_tuple + ir = builder.mapper.type_to_ir[cdef.info] + bases = [] + for cls in (b.type for b in cdef.info.bases): + if cls.fullname == "builtins.object": + continue + if is_named_tuple and cls.fullname in ( + "typing.Sequence", + "typing.Iterable", + "typing.Collection", + "typing.Reversible", + "typing.Container", + "typing.Sized", + ): + # HAX: Synthesized base classes added by mypy don't exist at runtime, so skip them. + # This could break if they were added explicitly, though... + continue + # Add the current class to the base classes list of concrete subclasses + if cls in builder.mapper.type_to_ir: + base_ir = builder.mapper.type_to_ir[cls] + if base_ir.children is not None: + base_ir.children.append(ir) + + if cls.fullname in MAGIC_TYPED_DICT_CLASSES: + # HAX: Mypy internally represents TypedDict classes differently from what + # should happen at runtime. Replace with something that works. + module = "typing" + if builder.options.capi_version < (3, 9): + name = "TypedDict" + if builder.options.capi_version < (3, 8): + # TypedDict was added to typing in Python 3.8. + module = "typing_extensions" + # TypedDict is not a real type on typing_extensions 4.7.0+ + name = "_TypedDict" + if isinstance(typing_extensions.TypedDict, type): + raise RuntimeError( + "It looks like you may have an old version " + "of typing_extensions installed. " + "typing_extensions>=4.7.0 is required on Python 3.7." + ) + else: + # In Python 3.9 TypedDict is not a real type. + name = "_TypedDict" + base = builder.get_module_attr(module, name, cdef.line) + elif is_named_tuple and cls.fullname == "builtins.tuple": + if builder.options.capi_version < (3, 9): + name = "NamedTuple" + else: + # This was changed in Python 3.9. + name = "_NamedTuple" + base = builder.get_module_attr("typing", name, cdef.line) + else: + cls_module = cls.fullname.rsplit(".", 1)[0] + if cls_module == builder.current_module: + base = builder.load_global_str(cls.name, cdef.line) + else: + base = builder.load_module_attr_by_fullname(cls.fullname, cdef.line) + bases.append(base) + if cls.fullname in MAGIC_TYPED_DICT_CLASSES: + # The remaining base classes are synthesized by mypy and should be ignored. + break + return builder.new_tuple(bases, cdef.line) + + +def find_non_ext_metaclass(builder: IRBuilder, cdef: ClassDef, bases: Value) -> Value: + """Find the metaclass of a class from its defs and bases.""" + if cdef.metaclass: + declared_metaclass = builder.accept(cdef.metaclass) + else: + if cdef.info.typeddict_type is not None and builder.options.capi_version >= (3, 9): + # In Python 3.9, the metaclass for class-based TypedDict is typing._TypedDictMeta. + # We can't easily calculate it generically, so special case it. + return builder.get_module_attr("typing", "_TypedDictMeta", cdef.line) + elif cdef.info.is_named_tuple and builder.options.capi_version >= (3, 9): + # In Python 3.9, the metaclass for class-based NamedTuple is typing.NamedTupleMeta. + # We can't easily calculate it generically, so special case it. + return builder.get_module_attr("typing", "NamedTupleMeta", cdef.line) + + declared_metaclass = builder.add( + LoadAddress(type_object_op.type, type_object_op.src, cdef.line) + ) + + return builder.call_c(py_calc_meta_op, [declared_metaclass, bases], cdef.line) + + +def setup_non_ext_dict( + builder: IRBuilder, cdef: ClassDef, metaclass: Value, bases: Value +) -> Value: + """Initialize the class dictionary for a non-extension class. + + This class dictionary is passed to the metaclass constructor. + """ + # Check if the metaclass defines a __prepare__ method, and if so, call it. + has_prepare = builder.primitive_op( + py_hasattr_op, [metaclass, builder.load_str("__prepare__")], cdef.line + ) + + non_ext_dict = Register(dict_rprimitive) + + true_block, false_block, exit_block = BasicBlock(), BasicBlock(), BasicBlock() + builder.add_bool_branch(has_prepare, true_block, false_block) + + builder.activate_block(true_block) + cls_name = builder.load_str(cdef.name) + prepare_meth = builder.py_get_attr(metaclass, "__prepare__", cdef.line) + prepare_dict = builder.py_call(prepare_meth, [cls_name, bases], cdef.line) + builder.assign(non_ext_dict, prepare_dict, cdef.line) + builder.goto(exit_block) + + builder.activate_block(false_block) + builder.assign(non_ext_dict, builder.call_c(dict_new_op, [], cdef.line), cdef.line) + builder.goto(exit_block) + builder.activate_block(exit_block) + + return non_ext_dict + + +def add_non_ext_class_attr_ann( + builder: IRBuilder, + non_ext: NonExtClassInfo, + lvalue: NameExpr, + stmt: AssignmentStmt, + get_type_info: Callable[[AssignmentStmt], TypeInfo | None] | None = None, +) -> None: + """Add a class attribute to __annotations__ of a non-extension class.""" + # FIXME: try to better preserve the special forms and type parameters of generics. + typ: Value | None = None + if get_type_info is not None: + type_info = get_type_info(stmt) + if type_info: + typ = load_type(builder, type_info, stmt.line) + + if typ is None: + # FIXME: if get_type_info is not provided, don't fall back to stmt.type? + ann_type = get_proper_type(stmt.type) + if ( + isinstance(stmt.unanalyzed_type, UnboundType) + and stmt.unanalyzed_type.original_str_expr is not None + ): + # Annotation is a forward reference, so don't attempt to load the actual + # type and load the string instead. + # + # TODO: is it possible to determine whether a non-string annotation is + # actually a forward reference due to the __annotations__ future? + typ = builder.load_str(stmt.unanalyzed_type.original_str_expr) + elif isinstance(ann_type, Instance): + typ = load_type(builder, ann_type.type, stmt.line) + else: + typ = builder.add(LoadAddress(type_object_op.type, type_object_op.src, stmt.line)) + + key = builder.load_str(lvalue.name) + builder.primitive_op(dict_set_item_op, [non_ext.anns, key, typ], stmt.line) + + +def add_non_ext_class_attr( + builder: IRBuilder, + non_ext: NonExtClassInfo, + lvalue: NameExpr, + stmt: AssignmentStmt, + cdef: ClassDef, + attr_to_cache: list[tuple[Lvalue, RType]], +) -> None: + """Add a class attribute to __dict__ of a non-extension class.""" + # Only add the attribute to the __dict__ if the assignment is of the form: + # x: type = value (don't add attributes of the form 'x: type' to the __dict__). + if not isinstance(stmt.rvalue, TempNode): + rvalue = builder.accept(stmt.rvalue) + builder.add_to_non_ext_dict(non_ext, lvalue.name, rvalue, stmt.line) + # We cache enum attributes to speed up enum attribute lookup since they + # are final. + if ( + cdef.info.bases + and cdef.info.bases[0].type.is_enum + # Skip these since Enum will remove it + and lvalue.name not in EXCLUDED_ENUM_ATTRIBUTES + ): + # Enum values are always boxed, so use object_rprimitive. + attr_to_cache.append((lvalue, object_rprimitive)) + + +def find_attr_initializers( + builder: IRBuilder, cdef: ClassDef, skip: Callable[[str, AssignmentStmt], bool] | None = None +) -> tuple[set[str], list[AssignmentStmt]]: + """Find initializers of attributes in a class body. + + If provided, the skip arg should be a callable which will return whether + to skip generating a default for an attribute. It will be passed the name of + the attribute and the corresponding AssignmentStmt. + """ + cls = builder.mapper.type_to_ir[cdef.info] + if cls.builtin_base: + return set(), [] + + attrs_with_defaults = set() + + # Pull out all assignments in classes in the mro so we can initialize them + # TODO: Support nested statements + default_assignments = [] + for info in reversed(cdef.info.mro): + if info not in builder.mapper.type_to_ir: + continue + for stmt in info.defn.defs.body: + if ( + isinstance(stmt, AssignmentStmt) + and isinstance(stmt.lvalues[0], NameExpr) + and not is_class_var(stmt.lvalues[0]) + and not isinstance(stmt.rvalue, TempNode) + ): + name = stmt.lvalues[0].name + if name == "__slots__": + continue + + if name == "__deletable__": + check_deletable_declaration(builder, cls, stmt.line) + continue + + if skip is not None and skip(name, stmt): + continue + + attr_type = cls.attr_type(name) + + # If the attribute is initialized to None and type isn't optional, + # doesn't initialize it to anything (special case for "# type:" comments). + if isinstance(stmt.rvalue, RefExpr) and stmt.rvalue.fullname == "builtins.None": + if ( + not is_optional_type(attr_type) + and not is_object_rprimitive(attr_type) + and not is_none_rprimitive(attr_type) + ): + continue + + attrs_with_defaults.add(name) + default_assignments.append(stmt) + + return attrs_with_defaults, default_assignments + + +def generate_attr_defaults_init( + builder: IRBuilder, cdef: ClassDef, default_assignments: list[AssignmentStmt] +) -> None: + """Generate an initialization method for default attr values (from class vars).""" + if not default_assignments: + return + cls = builder.mapper.type_to_ir[cdef.info] + if cls.builtin_base: + return + + with builder.enter_method(cls, "__mypyc_defaults_setup", bool_rprimitive): + self_var = builder.self() + for stmt in default_assignments: + lvalue = stmt.lvalues[0] + assert isinstance(lvalue, NameExpr) + if not stmt.is_final_def and not is_constant(stmt.rvalue): + builder.warning("Unsupported default attribute value", stmt.rvalue.line) + + attr_type = cls.attr_type(lvalue.name) + val = builder.coerce(builder.accept(stmt.rvalue), attr_type, stmt.line) + init = SetAttr(self_var, lvalue.name, val, -1) + init.mark_as_initializer() + builder.add(init) + + builder.add(Return(builder.true())) + + +def check_deletable_declaration(builder: IRBuilder, cl: ClassIR, line: int) -> None: + for attr in cl.deletable: + if attr not in cl.attributes: + if not cl.has_attr(attr): + builder.error(f'Attribute "{attr}" not defined', line) + continue + for base in cl.mro: + if attr in base.property_types: + builder.error(f'Cannot make property "{attr}" deletable', line) + break + else: + _, base = cl.attr_details(attr) + builder.error( + ('Attribute "{}" not defined in "{}" ' + '(defined in "{}")').format( + attr, cl.name, base.name + ), + line, + ) + + +def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None: + """Create a "__ne__" method from a "__eq__" method (if only latter exists).""" + cls = builder.mapper.type_to_ir[cdef.info] + if cls.has_method("__eq__") and not cls.has_method("__ne__"): + gen_glue_ne_method(builder, cls, cdef.line) + + +def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> None: + """Generate a "__ne__" method from a "__eq__" method.""" + func_ir = cls.get_method("__eq__") + assert func_ir + eq_sig = func_ir.decl.sig + strict_typing = builder.options.strict_dunders_typing + with builder.enter_method(cls, "__ne__", eq_sig.ret_type): + rhs_type = eq_sig.args[0].type if strict_typing else object_rprimitive + rhs_arg = builder.add_argument("rhs", rhs_type) + eqval = builder.add(MethodCall(builder.self(), "__eq__", [rhs_arg], line)) + + can_return_not_implemented = is_subtype(not_implemented_op.type, eq_sig.ret_type) + return_bool = is_subtype(eq_sig.ret_type, bool_rprimitive) + + if not strict_typing or can_return_not_implemented: + # If __eq__ returns NotImplemented, then __ne__ should also + not_implemented_block, regular_block = BasicBlock(), BasicBlock() + not_implemented = builder.add( + LoadAddress(not_implemented_op.type, not_implemented_op.src, line) + ) + builder.add( + Branch( + builder.translate_is_op(eqval, not_implemented, "is", line), + not_implemented_block, + regular_block, + Branch.BOOL, + ) + ) + builder.activate_block(regular_block) + rettype = bool_rprimitive if return_bool and strict_typing else object_rprimitive + retval = builder.coerce(builder.unary_op(eqval, "not", line), rettype, line) + builder.add(Return(retval)) + builder.activate_block(not_implemented_block) + builder.add(Return(not_implemented)) + else: + rettype = bool_rprimitive if return_bool and strict_typing else object_rprimitive + retval = builder.coerce(builder.unary_op(eqval, "not", line), rettype, line) + builder.add(Return(retval)) + + +def load_non_ext_class( + builder: IRBuilder, ir: ClassIR, non_ext: NonExtClassInfo, line: int +) -> Value: + cls_name = builder.load_str(ir.name) + + add_dunders_to_non_ext_dict(builder, non_ext, line) + + class_type_obj = builder.py_call( + non_ext.metaclass, [cls_name, non_ext.bases, non_ext.dict], line + ) + return class_type_obj + + +def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> Value: + """Apply class decorators to create a decorated (non-extension) class object. + + Given a decorated ClassDef and a register containing a + non-extension representation of the ClassDef created via the type + constructor, applies the corresponding decorator functions on that + decorated ClassDef and returns a register containing the decorated + ClassDef. + """ + decorators = cdef.decorators + dec_class = type_obj + for d in reversed(decorators): + decorator = d.accept(builder.visitor) + assert isinstance(decorator, Value) + dec_class = builder.py_call(decorator, [dec_class], dec_class.line) + return dec_class + + +def cache_class_attrs( + builder: IRBuilder, attrs_to_cache: list[tuple[Lvalue, RType]], cdef: ClassDef +) -> None: + """Add class attributes to be cached to the global cache.""" + typ = builder.load_native_type_object(cdef.info.fullname) + for lval, rtype in attrs_to_cache: + assert isinstance(lval, NameExpr) + rval = builder.py_get_attr(typ, lval.name, cdef.line) + builder.init_final_static(lval, rval, cdef.name, type_override=rtype) + + +def create_mypyc_attrs_tuple(builder: IRBuilder, ir: ClassIR, line: int) -> Value: + attrs = [name for ancestor in ir.mro for name in ancestor.attributes] + if ir.inherits_python: + attrs.append("__dict__") + items = [builder.load_str(attr) for attr in attrs] + return builder.new_tuple(items, line) + + +def add_dunders_to_non_ext_dict( + builder: IRBuilder, non_ext: NonExtClassInfo, line: int, add_annotations: bool = True +) -> None: + if add_annotations: + # Add __annotations__ to the class dict. + builder.add_to_non_ext_dict(non_ext, "__annotations__", non_ext.anns, line) + + # We add a __doc__ attribute so if the non-extension class is decorated with the + # dataclass decorator, dataclass will not try to look for __text_signature__. + # https://github.com/python/cpython/blob/3.7/Lib/dataclasses.py#L957 + filler_doc_str = "mypyc filler docstring" + builder.add_to_non_ext_dict(non_ext, "__doc__", builder.load_str(filler_doc_str), line) + builder.add_to_non_ext_dict(non_ext, "__module__", builder.load_str(builder.module_name), line) diff --git a/mypyc/irbuild/constant_fold.py b/mypyc/irbuild/constant_fold.py new file mode 100644 index 000000000000..12a4b15dd40c --- /dev/null +++ b/mypyc/irbuild/constant_fold.py @@ -0,0 +1,95 @@ +"""Constant folding of IR values. + +For example, 3 + 5 can be constant folded into 8. + +This is mostly like mypy.constant_fold, but we can bind some additional +NameExpr and MemberExpr references here, since we have more knowledge +about which definitions can be trusted -- we constant fold only references +to other compiled modules in the same compilation unit. +""" + +from __future__ import annotations + +from typing import Final, Union + +from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op +from mypy.nodes import ( + BytesExpr, + ComplexExpr, + Expression, + FloatExpr, + IntExpr, + MemberExpr, + NameExpr, + OpExpr, + StrExpr, + UnaryExpr, + Var, +) +from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.util import bytes_from_str + +# All possible result types of constant folding +ConstantValue = Union[int, float, complex, str, bytes] +CONST_TYPES: Final = (int, float, complex, str, bytes) + + +def constant_fold_expr(builder: IRBuilder, expr: Expression) -> ConstantValue | None: + """Return the constant value of an expression for supported operations. + + Return None otherwise. + """ + if isinstance(expr, IntExpr): + return expr.value + if isinstance(expr, FloatExpr): + return expr.value + if isinstance(expr, StrExpr): + return expr.value + if isinstance(expr, BytesExpr): + return bytes_from_str(expr.value) + if isinstance(expr, ComplexExpr): + return expr.value + elif isinstance(expr, NameExpr): + node = expr.node + if isinstance(node, Var) and node.is_final: + final_value = node.final_value + if isinstance(final_value, (CONST_TYPES)): + return final_value + elif isinstance(expr, MemberExpr): + final = builder.get_final_ref(expr) + if final is not None: + fn, final_var, native = final + if final_var.is_final: + final_value = final_var.final_value + if isinstance(final_value, (CONST_TYPES)): + return final_value + elif isinstance(expr, OpExpr): + left = constant_fold_expr(builder, expr.left) + right = constant_fold_expr(builder, expr.right) + if left is not None and right is not None: + return constant_fold_binary_op_extended(expr.op, left, right) + elif isinstance(expr, UnaryExpr): + value = constant_fold_expr(builder, expr.expr) + if value is not None and not isinstance(value, bytes): + return constant_fold_unary_op(expr.op, value) + return None + + +def constant_fold_binary_op_extended( + op: str, left: ConstantValue, right: ConstantValue +) -> ConstantValue | None: + """Like mypy's constant_fold_binary_op(), but includes bytes support. + + mypy cannot use constant folded bytes easily so it's simpler to only support them in mypyc. + """ + if not isinstance(left, bytes) and not isinstance(right, bytes): + return constant_fold_binary_op(op, left, right) + + if op == "+" and isinstance(left, bytes) and isinstance(right, bytes): + return left + right + elif op == "*" and isinstance(left, bytes) and isinstance(right, int): + return left * right + elif op == "*" and isinstance(left, int) and isinstance(right, bytes): + return left * right + + return None diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py new file mode 100644 index 000000000000..a740f0b821d9 --- /dev/null +++ b/mypyc/irbuild/context.py @@ -0,0 +1,186 @@ +"""Helpers that store information about functions and the related classes.""" + +from __future__ import annotations + +from mypy.nodes import FuncItem +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import INVALID_FUNC_DEF +from mypyc.ir.ops import BasicBlock, Value +from mypyc.irbuild.targets import AssignmentTarget + + +class FuncInfo: + """Contains information about functions as they are generated.""" + + def __init__( + self, + fitem: FuncItem = INVALID_FUNC_DEF, + name: str = "", + class_name: str | None = None, + namespace: str = "", + is_nested: bool = False, + contains_nested: bool = False, + is_decorated: bool = False, + in_non_ext: bool = False, + add_nested_funcs_to_env: bool = False, + ) -> None: + self.fitem = fitem + self.name = name + self.class_name = class_name + self.ns = namespace + # Callable classes implement the '__call__' method, and are used to represent functions + # that are nested inside of other functions. + self._callable_class: ImplicitClass | None = None + # Environment classes are ClassIR instances that contain attributes representing the + # variables in the environment of the function they correspond to. Environment classes are + # generated for functions that contain nested functions. + self._env_class: ClassIR | None = None + # Generator classes implement the '__next__' method, and are used to represent generators + # returned by generator functions. + self._generator_class: GeneratorClass | None = None + # Environment class registers are the local registers associated with instances of an + # environment class, used for getting and setting attributes. curr_env_reg is the register + # associated with the current environment. + self._curr_env_reg: Value | None = None + # These are flags denoting whether a given function is nested, contains a nested function, + # is decorated, or is within a non-extension class. + self.is_nested = is_nested + self.contains_nested = contains_nested + self.is_decorated = is_decorated + self.in_non_ext = in_non_ext + self.add_nested_funcs_to_env = add_nested_funcs_to_env + + # TODO: add field for ret_type: RType = none_rprimitive + + def namespaced_name(self) -> str: + return "_".join(x for x in [self.name, self.class_name, self.ns] if x) + + @property + def is_generator(self) -> bool: + return self.fitem.is_generator or self.fitem.is_coroutine + + @property + def is_coroutine(self) -> bool: + return self.fitem.is_coroutine + + @property + def callable_class(self) -> ImplicitClass: + assert self._callable_class is not None + return self._callable_class + + @callable_class.setter + def callable_class(self, cls: ImplicitClass) -> None: + self._callable_class = cls + + @property + def env_class(self) -> ClassIR: + assert self._env_class is not None + return self._env_class + + @env_class.setter + def env_class(self, ir: ClassIR) -> None: + self._env_class = ir + + @property + def generator_class(self) -> GeneratorClass: + assert self._generator_class is not None + return self._generator_class + + @generator_class.setter + def generator_class(self, cls: GeneratorClass) -> None: + self._generator_class = cls + + @property + def curr_env_reg(self) -> Value: + assert self._curr_env_reg is not None + return self._curr_env_reg + + +class ImplicitClass: + """Contains information regarding implicitly generated classes. + + Implicit classes are generated for nested functions and generator + functions. They are not explicitly defined in the source code. + + NOTE: This is both a concrete class and used as a base class. + """ + + def __init__(self, ir: ClassIR) -> None: + # The ClassIR instance associated with this class. + self.ir = ir + # The register associated with the 'self' instance for this generator class. + self._self_reg: Value | None = None + # Environment class registers are the local registers associated with instances of an + # environment class, used for getting and setting attributes. curr_env_reg is the register + # associated with the current environment. prev_env_reg is the self.__mypyc_env__ field + # associated with the previous environment. + self._curr_env_reg: Value | None = None + self._prev_env_reg: Value | None = None + + @property + def self_reg(self) -> Value: + assert self._self_reg is not None + return self._self_reg + + @self_reg.setter + def self_reg(self, reg: Value) -> None: + self._self_reg = reg + + @property + def curr_env_reg(self) -> Value: + assert self._curr_env_reg is not None + return self._curr_env_reg + + @curr_env_reg.setter + def curr_env_reg(self, reg: Value) -> None: + self._curr_env_reg = reg + + @property + def prev_env_reg(self) -> Value: + assert self._prev_env_reg is not None + return self._prev_env_reg + + @prev_env_reg.setter + def prev_env_reg(self, reg: Value) -> None: + self._prev_env_reg = reg + + +class GeneratorClass(ImplicitClass): + """Contains information about implicit generator function classes.""" + + def __init__(self, ir: ClassIR) -> None: + super().__init__(ir) + # This register holds the label number that the '__next__' function should go to the next + # time it is called. + self._next_label_reg: Value | None = None + self._next_label_target: AssignmentTarget | None = None + + # These registers hold the error values for the generator object for the case that the + # 'throw' function is called. + self.exc_regs: tuple[Value, Value, Value] | None = None + + # Holds the arg passed to send + self.send_arg_reg: Value | None = None + + # The switch block is used to decide which instruction to go using the value held in the + # next-label register. + self.switch_block = BasicBlock() + self.continuation_blocks: list[BasicBlock] = [] + + @property + def next_label_reg(self) -> Value: + assert self._next_label_reg is not None + return self._next_label_reg + + @next_label_reg.setter + def next_label_reg(self, reg: Value) -> None: + self._next_label_reg = reg + + @property + def next_label_target(self) -> AssignmentTarget: + assert self._next_label_target is not None + return self._next_label_target + + @next_label_target.setter + def next_label_target(self, target: AssignmentTarget) -> None: + self._next_label_target = target diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py new file mode 100644 index 000000000000..aa223fe20176 --- /dev/null +++ b/mypyc/irbuild/env_class.py @@ -0,0 +1,223 @@ +"""Generate classes representing function environments (+ related operations). + +If we have a nested function that has non-local (free) variables, access to the +non-locals is via an instance of an environment class. Example: + + def f() -> int: + x = 0 # Make 'x' an attribute of an environment class instance + + def g() -> int: + # We have access to the environment class instance to + # allow accessing 'x' + return x + 2 + + x = x + 1 # Modify the attribute + return g() +""" + +from __future__ import annotations + +from mypy.nodes import Argument, FuncDef, SymbolNode, Var +from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME, bitmap_name +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.ops import Call, GetAttr, SetAttr, Value +from mypyc.ir.rtypes import RInstance, bitmap_rprimitive, object_rprimitive +from mypyc.irbuild.builder import IRBuilder, SymbolTarget +from mypyc.irbuild.context import FuncInfo, GeneratorClass, ImplicitClass +from mypyc.irbuild.targets import AssignmentTargetAttr + + +def setup_env_class(builder: IRBuilder) -> ClassIR: + """Generate a class representing a function environment. + + Note that the variables in the function environment are not + actually populated here. This is because when the environment + class is generated, the function environment has not yet been + visited. This behavior is allowed so that when the compiler visits + nested functions, it can use the returned ClassIR instance to + figure out free variables it needs to access. The remaining + attributes of the environment class are populated when the + environment registers are loaded. + + Return a ClassIR representing an environment for a function + containing a nested function. + """ + env_class = ClassIR( + f"{builder.fn_info.namespaced_name()}_env", builder.module_name, is_generated=True + ) + env_class.attributes[SELF_NAME] = RInstance(env_class) + if builder.fn_info.is_nested: + # If the function is nested, its environment class must contain an environment + # attribute pointing to its encapsulating functions' environment class. + env_class.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class) + env_class.mro = [env_class] + builder.fn_info.env_class = env_class + builder.classes.append(env_class) + return env_class + + +def finalize_env_class(builder: IRBuilder) -> None: + """Generate, instantiate, and set up the environment of an environment class.""" + instantiate_env_class(builder) + + # Iterate through the function arguments and replace local definitions (using registers) + # that were previously added to the environment with references to the function's + # environment class. + if builder.fn_info.is_nested: + add_args_to_env(builder, local=False, base=builder.fn_info.callable_class) + else: + add_args_to_env(builder, local=False, base=builder.fn_info) + + +def instantiate_env_class(builder: IRBuilder) -> Value: + """Assign an environment class to a register named after the given function definition.""" + curr_env_reg = builder.add( + Call(builder.fn_info.env_class.ctor, [], builder.fn_info.fitem.line) + ) + + if builder.fn_info.is_nested: + builder.fn_info.callable_class._curr_env_reg = curr_env_reg + builder.add( + SetAttr( + curr_env_reg, + ENV_ATTR_NAME, + builder.fn_info.callable_class.prev_env_reg, + builder.fn_info.fitem.line, + ) + ) + else: + builder.fn_info._curr_env_reg = curr_env_reg + + return curr_env_reg + + +def load_env_registers(builder: IRBuilder) -> None: + """Load the registers for the current FuncItem being visited. + + Adds the arguments of the FuncItem to the environment. If the + FuncItem is nested inside of another function, then this also + loads all of the outer environments of the FuncItem into registers + so that they can be used when accessing free variables. + """ + add_args_to_env(builder, local=True) + + fn_info = builder.fn_info + fitem = fn_info.fitem + if fn_info.is_nested: + load_outer_envs(builder, fn_info.callable_class) + # If this is a FuncDef, then make sure to load the FuncDef into its own environment + # class so that the function can be called recursively. + if isinstance(fitem, FuncDef) and fn_info.add_nested_funcs_to_env: + setup_func_for_recursive_call(builder, fitem, fn_info.callable_class) + + +def load_outer_env( + builder: IRBuilder, base: Value, outer_env: dict[SymbolNode, SymbolTarget] +) -> Value: + """Load the environment class for a given base into a register. + + Additionally, iterates through all of the SymbolNode and + AssignmentTarget instances of the environment at the given index's + symtable, and adds those instances to the environment of the + current environment. This is done so that the current environment + can access outer environment variables without having to reload + all of the environment registers. + + Returns the register where the environment class was loaded. + """ + env = builder.add(GetAttr(base, ENV_ATTR_NAME, builder.fn_info.fitem.line)) + assert isinstance(env.type, RInstance), f"{env} must be of type RInstance" + + for symbol, target in outer_env.items(): + env.type.class_ir.attributes[symbol.name] = target.type + symbol_target = AssignmentTargetAttr(env, symbol.name) + builder.add_target(symbol, symbol_target) + + return env + + +def load_outer_envs(builder: IRBuilder, base: ImplicitClass) -> None: + index = len(builder.builders) - 2 + + # Load the first outer environment. This one is special because it gets saved in the + # FuncInfo instance's prev_env_reg field. + if index > 1: + # outer_env = builder.fn_infos[index].environment + outer_env = builder.symtables[index] + if isinstance(base, GeneratorClass): + base.prev_env_reg = load_outer_env(builder, base.curr_env_reg, outer_env) + else: + base.prev_env_reg = load_outer_env(builder, base.self_reg, outer_env) + env_reg = base.prev_env_reg + index -= 1 + + # Load the remaining outer environments into registers. + while index > 1: + # outer_env = builder.fn_infos[index].environment + outer_env = builder.symtables[index] + env_reg = load_outer_env(builder, env_reg, outer_env) + index -= 1 + + +def num_bitmap_args(builder: IRBuilder, args: list[Argument]) -> int: + n = 0 + for arg in args: + t = builder.type_to_rtype(arg.variable.type) + if t.error_overlap and arg.kind.is_optional(): + n += 1 + return (n + (BITMAP_BITS - 1)) // BITMAP_BITS + + +def add_args_to_env( + builder: IRBuilder, + local: bool = True, + base: FuncInfo | ImplicitClass | None = None, + reassign: bool = True, +) -> None: + fn_info = builder.fn_info + args = fn_info.fitem.arguments + nb = num_bitmap_args(builder, args) + if local: + for arg in args: + rtype = builder.type_to_rtype(arg.variable.type) + builder.add_local_reg(arg.variable, rtype, is_arg=True) + for i in reversed(range(nb)): + builder.add_local_reg(Var(bitmap_name(i)), bitmap_rprimitive, is_arg=True) + else: + for arg in args: + if is_free_variable(builder, arg.variable) or fn_info.is_generator: + rtype = builder.type_to_rtype(arg.variable.type) + assert base is not None, "base cannot be None for adding nonlocal args" + builder.add_var_to_env_class(arg.variable, rtype, base, reassign=reassign) + + +def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: ImplicitClass) -> None: + """Enable calling a nested function (with a callable class) recursively. + + Adds the instance of the callable class representing the given + FuncDef to a register in the environment so that the function can + be called recursively. Note that this needs to be done only for + nested functions. + """ + # First, set the attribute of the environment class so that GetAttr can be called on it. + prev_env = builder.fn_infos[-2].env_class + prev_env.attributes[fdef.name] = builder.type_to_rtype(fdef.type) + + if isinstance(base, GeneratorClass): + # If we are dealing with a generator class, then we need to first get the register + # holding the current environment class, and load the previous environment class from + # there. + prev_env_reg = builder.add(GetAttr(base.curr_env_reg, ENV_ATTR_NAME, -1)) + else: + prev_env_reg = base.prev_env_reg + + # Obtain the instance of the callable class representing the FuncDef, and add it to the + # current environment. + val = builder.add(GetAttr(prev_env_reg, fdef.name, -1)) + target = builder.add_local_reg(fdef, object_rprimitive) + builder.assign(target, val, -1) + + +def is_free_variable(builder: IRBuilder, symbol: SymbolNode) -> bool: + fitem = builder.fn_info.fitem + return fitem in builder.free_variables and symbol in builder.free_variables[fitem] diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py new file mode 100644 index 000000000000..97cd31af93af --- /dev/null +++ b/mypyc/irbuild/expression.py @@ -0,0 +1,1078 @@ +"""Transform mypy expression ASTs to mypyc IR (Intermediate Representation). + +The top-level AST transformation logic is implemented in mypyc.irbuild.visitor +and mypyc.irbuild.builder. +""" + +from __future__ import annotations + +import math +from typing import Callable, Sequence + +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + LDEF, + AssertTypeExpr, + AssignmentExpr, + BytesExpr, + CallExpr, + CastExpr, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + Expression, + FloatExpr, + GeneratorExpr, + IndexExpr, + IntExpr, + ListComprehension, + ListExpr, + MemberExpr, + MypyFile, + NameExpr, + OpExpr, + RefExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + StrExpr, + SuperExpr, + TupleExpr, + TypeApplication, + TypeInfo, + TypeVarLikeExpr, + UnaryExpr, + Var, +) +from mypy.types import Instance, ProperType, TupleType, TypeType, get_proper_type +from mypyc.common import MAX_SHORT_INT +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD +from mypyc.ir.ops import ( + Assign, + BasicBlock, + ComparisonOp, + Integer, + LoadAddress, + LoadLiteral, + PrimitiveDescription, + RaiseStandardError, + Register, + TupleGet, + TupleSet, + Value, +) +from mypyc.ir.rtypes import ( + RTuple, + bool_rprimitive, + int_rprimitive, + is_fixed_width_rtype, + is_int_rprimitive, + is_list_rprimitive, + is_none_rprimitive, + object_rprimitive, + set_rprimitive, +) +from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional +from mypyc.irbuild.builder import IRBuilder, int_borrow_friendly_op +from mypyc.irbuild.constant_fold import constant_fold_expr +from mypyc.irbuild.for_helpers import ( + comprehension_helper, + raise_error_if_contains_unreachable_names, + translate_list_comprehension, + translate_set_comprehension, +) +from mypyc.irbuild.format_str_tokenizer import ( + convert_format_expr_to_bytes, + convert_format_expr_to_str, + join_formatted_bytes, + join_formatted_strings, + tokenizer_printf_style, +) +from mypyc.irbuild.specialize import apply_function_specialization, apply_method_specialization +from mypyc.primitives.bytes_ops import bytes_slice_op +from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, dict_set_item_op +from mypyc.primitives.generic_ops import iter_op +from mypyc.primitives.list_ops import list_append_op, list_extend_op, list_slice_op +from mypyc.primitives.misc_ops import ellipsis_op, get_module_dict_op, new_slice_op, type_op +from mypyc.primitives.registry import builtin_names +from mypyc.primitives.set_ops import set_add_op, set_in_op, set_update_op +from mypyc.primitives.str_ops import str_slice_op +from mypyc.primitives.tuple_ops import list_tuple_op, tuple_slice_op + +# Name and attribute references + + +def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value: + if isinstance(expr.node, TypeVarLikeExpr) and expr.node.is_new_style: + # Reference to Python 3.12 implicit TypeVar/TupleVarTuple/... object. + # These are stored in C statics and not visible in Python namespaces. + return builder.load_type_var(expr.node.name, expr.node.line) + if expr.node is None: + builder.add( + RaiseStandardError( + RaiseStandardError.RUNTIME_ERROR, + "mypyc internal error: should be unreachable", + expr.line, + ) + ) + return builder.none() + fullname = expr.node.fullname + if fullname in builtin_names: + typ, src = builtin_names[fullname] + return builder.add(LoadAddress(typ, src, expr.line)) + # special cases + if fullname == "builtins.None": + return builder.none() + if fullname == "builtins.True": + return builder.true() + if fullname == "builtins.False": + return builder.false() + if fullname in ("typing.TYPE_CHECKING", "typing_extensions.TYPE_CHECKING"): + return builder.false() + + math_literal = transform_math_literal(builder, fullname) + if math_literal is not None: + return math_literal + + if isinstance(expr.node, Var) and expr.node.is_final: + value = builder.emit_load_final( + expr.node, + fullname, + expr.name, + builder.is_native_ref_expr(expr), + builder.types[expr], + expr.line, + ) + if value is not None: + return value + + if isinstance(expr.node, MypyFile) and expr.node.fullname in builder.imports: + return builder.load_module(expr.node.fullname) + + # If the expression is locally defined, then read the result from the corresponding + # assignment target and return it. Otherwise if the expression is a global, load it from + # the globals dictionary. + # Except for imports, that currently always happens in the global namespace. + if expr.kind == LDEF and not (isinstance(expr.node, Var) and expr.node.is_suppressed_import): + # Try to detect and error when we hit the irritating mypy bug + # where a local variable is cast to None. (#5423) + if ( + isinstance(expr.node, Var) + and is_none_rprimitive(builder.node_type(expr)) + and expr.node.is_inferred + ): + builder.error( + 'Local variable "{}" has inferred type None; add an annotation'.format( + expr.node.name + ), + expr.node.line, + ) + + # TODO: Behavior currently only defined for Var, FuncDef and MypyFile node types. + if isinstance(expr.node, MypyFile): + # Load reference to a module imported inside function from + # the modules dictionary. It would be closer to Python + # semantics to access modules imported inside functions + # via local variables, but this is tricky since the mypy + # AST doesn't include a Var node for the module. We + # instead load the module separately on each access. + mod_dict = builder.call_c(get_module_dict_op, [], expr.line) + obj = builder.primitive_op( + dict_get_item_op, [mod_dict, builder.load_str(expr.node.fullname)], expr.line + ) + return obj + else: + return builder.read(builder.get_assignment_target(expr, for_read=True), expr.line) + + return builder.load_global(expr) + + +def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: + # Special Cases + if expr.fullname in ("typing.TYPE_CHECKING", "typing_extensions.TYPE_CHECKING"): + return builder.false() + + # First check if this is maybe a final attribute. + final = builder.get_final_ref(expr) + if final is not None: + fullname, final_var, native = final + value = builder.emit_load_final( + final_var, fullname, final_var.name, native, builder.types[expr], expr.line + ) + if value is not None: + return value + + math_literal = transform_math_literal(builder, expr.fullname) + if math_literal is not None: + return math_literal + + if isinstance(expr.node, MypyFile) and expr.node.fullname in builder.imports: + return builder.load_module(expr.node.fullname) + + can_borrow = builder.is_native_attr_ref(expr) + obj = builder.accept(expr.expr, can_borrow=can_borrow) + rtype = builder.node_type(expr) + + # Special case: for named tuples transform attribute access to faster index access. + typ = get_proper_type(builder.types.get(expr.expr)) + if isinstance(typ, TupleType) and typ.partial_fallback.type.is_named_tuple: + fields = typ.partial_fallback.type.metadata["namedtuple"]["fields"] + if expr.name in fields: + index = builder.builder.load_int(fields.index(expr.name)) + return builder.gen_method_call(obj, "__getitem__", [index], rtype, expr.line) + + check_instance_attribute_access_through_class(builder, expr, typ) + + borrow = can_borrow and builder.can_borrow + return builder.builder.get_attr(obj, expr.name, rtype, expr.line, borrow=borrow) + + +def check_instance_attribute_access_through_class( + builder: IRBuilder, expr: MemberExpr, typ: ProperType | None +) -> None: + """Report error if accessing an instance attribute through class object.""" + if isinstance(expr.expr, RefExpr): + node = expr.expr.node + if isinstance(typ, TypeType) and isinstance(typ.item, Instance): + # TODO: Handle other item types + node = typ.item.type + if isinstance(node, TypeInfo): + class_ir = builder.mapper.type_to_ir.get(node) + if class_ir is not None and class_ir.is_ext_class: + sym = node.get(expr.name) + if ( + sym is not None + and isinstance(sym.node, Var) + and not sym.node.is_classvar + and not sym.node.is_final + ): + builder.error( + 'Cannot access instance attribute "{}" through class object'.format( + expr.name + ), + expr.line, + ) + builder.note( + '(Hint: Use "x: Final = ..." or "x: ClassVar = ..." to define ' + "a class attribute)", + expr.line, + ) + + +def transform_super_expr(builder: IRBuilder, o: SuperExpr) -> Value: + # warning(builder, 'can not optimize super() expression', o.line) + sup_val = builder.load_module_attr_by_fullname("builtins.super", o.line) + if o.call.args: + args = [builder.accept(arg) for arg in o.call.args] + else: + assert o.info is not None + typ = builder.load_native_type_object(o.info.fullname) + ir = builder.mapper.type_to_ir[o.info] + iter_env = iter(builder.builder.args) + # Grab first argument + vself: Value = next(iter_env) + if builder.fn_info.is_generator: + # grab sixth argument (see comment in translate_super_method_call) + self_targ = list(builder.symtables[-1].values())[6] + vself = builder.read(self_targ, builder.fn_info.fitem.line) + elif not ir.is_ext_class: + vself = next(iter_env) # second argument is self if non_extension class + args = [typ, vself] + res = builder.py_call(sup_val, args, o.line) + return builder.py_get_attr(res, o.name, o.line) + + +# Calls + + +def transform_call_expr(builder: IRBuilder, expr: CallExpr) -> Value: + callee = expr.callee + if isinstance(expr.analyzed, CastExpr): + return translate_cast_expr(builder, expr.analyzed) + elif isinstance(expr.analyzed, AssertTypeExpr): + # Compile to a no-op. + return builder.accept(expr.analyzed.expr) + elif ( + isinstance(callee, (NameExpr, MemberExpr)) + and isinstance(callee.node, TypeInfo) + and callee.node.is_newtype + ): + # A call to a NewType type is a no-op at runtime. + return builder.accept(expr.args[0]) + + if isinstance(callee, IndexExpr) and isinstance(callee.analyzed, TypeApplication): + callee = callee.analyzed.expr # Unwrap type application + + if isinstance(callee, MemberExpr): + if isinstance(callee.expr, RefExpr) and isinstance(callee.expr.node, MypyFile): + # Call a module-level function, not a method. + return translate_call(builder, expr, callee) + return apply_method_specialization(builder, expr, callee) or translate_method_call( + builder, expr, callee + ) + elif isinstance(callee, SuperExpr): + return translate_super_method_call(builder, expr, callee) + else: + return translate_call(builder, expr, callee) + + +def translate_call(builder: IRBuilder, expr: CallExpr, callee: Expression) -> Value: + # The common case of calls is refexprs + if isinstance(callee, RefExpr): + return apply_function_specialization(builder, expr, callee) or translate_refexpr_call( + builder, expr, callee + ) + + function = builder.accept(callee) + args = [builder.accept(arg) for arg in expr.args] + return builder.py_call( + function, args, expr.line, arg_kinds=expr.arg_kinds, arg_names=expr.arg_names + ) + + +def translate_refexpr_call(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value: + """Translate a non-method call.""" + # Gen the argument values + arg_values = [builder.accept(arg) for arg in expr.args] + + return builder.call_refexpr_with_args(expr, callee, arg_values) + + +def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr) -> Value: + """Generate IR for an arbitrary call of form e.m(...). + + This can also deal with calls to module-level functions. + """ + if builder.is_native_ref_expr(callee): + # Call to module-level native function or such + return translate_call(builder, expr, callee) + elif ( + isinstance(callee.expr, RefExpr) + and isinstance(callee.expr.node, TypeInfo) + and callee.expr.node in builder.mapper.type_to_ir + and builder.mapper.type_to_ir[callee.expr.node].has_method(callee.name) + and all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds) + ): + # Call a method via the *class* + assert isinstance(callee.expr.node, TypeInfo) + ir = builder.mapper.type_to_ir[callee.expr.node] + return call_classmethod(builder, ir, expr, callee) + elif builder.is_module_member_expr(callee): + # Fall back to a PyCall for non-native module calls + function = builder.accept(callee) + args = [builder.accept(arg) for arg in expr.args] + return builder.py_call( + function, args, expr.line, arg_kinds=expr.arg_kinds, arg_names=expr.arg_names + ) + else: + if isinstance(callee.expr, RefExpr): + node = callee.expr.node + if isinstance(node, Var) and node.is_cls: + typ = get_proper_type(node.type) + if isinstance(typ, TypeType) and isinstance(typ.item, Instance): + class_ir = builder.mapper.type_to_ir.get(typ.item.type) + if class_ir and class_ir.is_ext_class and class_ir.has_no_subclasses(): + # Call a native classmethod via cls that can be statically bound, + # since the class has no subclasses. + return call_classmethod(builder, class_ir, expr, callee) + + receiver_typ = builder.node_type(callee.expr) + + # If there is a specializer for this method name/type, try calling it. + # We would return the first successful one. + val = apply_method_specialization(builder, expr, callee, receiver_typ) + if val is not None: + return val + + obj = builder.accept(callee.expr) + args = [builder.accept(arg) for arg in expr.args] + return builder.gen_method_call( + obj, + callee.name, + args, + builder.node_type(expr), + expr.line, + expr.arg_kinds, + expr.arg_names, + ) + + +def call_classmethod(builder: IRBuilder, ir: ClassIR, expr: CallExpr, callee: MemberExpr) -> Value: + decl = ir.method_decl(callee.name) + args = [] + arg_kinds, arg_names = expr.arg_kinds.copy(), expr.arg_names.copy() + # Add the class argument for class methods in extension classes + if decl.kind == FUNC_CLASSMETHOD and ir.is_ext_class: + args.append(builder.load_native_type_object(ir.fullname)) + arg_kinds.insert(0, ARG_POS) + arg_names.insert(0, None) + args += [builder.accept(arg) for arg in expr.args] + + if ir.is_ext_class: + return builder.builder.call(decl, args, arg_kinds, arg_names, expr.line) + else: + obj = builder.accept(callee.expr) + return builder.gen_method_call( + obj, + callee.name, + args, + builder.node_type(expr), + expr.line, + expr.arg_kinds, + expr.arg_names, + ) + + +def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: SuperExpr) -> Value: + if callee.info is None or (len(callee.call.args) != 0 and len(callee.call.args) != 2): + return translate_call(builder, expr, callee) + + # We support two-argument super but only when it is super(CurrentClass, self) + # TODO: We could support it when it is a parent class in many cases? + if len(callee.call.args) == 2: + self_arg = callee.call.args[1] + if ( + not isinstance(self_arg, NameExpr) + or not isinstance(self_arg.node, Var) + or not self_arg.node.is_self + ): + return translate_call(builder, expr, callee) + + typ_arg = callee.call.args[0] + if ( + not isinstance(typ_arg, NameExpr) + or not isinstance(typ_arg.node, TypeInfo) + or callee.info is not typ_arg.node + ): + return translate_call(builder, expr, callee) + + ir = builder.mapper.type_to_ir[callee.info] + # Search for the method in the mro, skipping ourselves. We + # determine targets of super calls to native methods statically. + for base in ir.mro[1:]: + if callee.name in base.method_decls: + break + else: + if ( + ir.is_ext_class + and ir.builtin_base is None + and not ir.inherits_python + and callee.name == "__init__" + and len(expr.args) == 0 + ): + # Call translates to object.__init__(self), which is a + # no-op, so omit the call. + return builder.none() + return translate_call(builder, expr, callee) + + decl = base.method_decl(callee.name) + arg_values = [builder.accept(arg) for arg in expr.args] + arg_kinds, arg_names = expr.arg_kinds.copy(), expr.arg_names.copy() + + if decl.kind != FUNC_STATICMETHOD: + # Grab first argument + vself: Value = builder.self() + if decl.kind == FUNC_CLASSMETHOD: + vself = builder.primitive_op(type_op, [vself], expr.line) + elif builder.fn_info.is_generator: + # For generator classes, the self target is the 6th value + # in the symbol table (which is an ordered dict). This is sort + # of ugly, but we can't search by name since the 'self' parameter + # could be named anything, and it doesn't get added to the + # environment indexes. + self_targ = list(builder.symtables[-1].values())[6] + vself = builder.read(self_targ, builder.fn_info.fitem.line) + arg_values.insert(0, vself) + arg_kinds.insert(0, ARG_POS) + arg_names.insert(0, None) + + return builder.builder.call(decl, arg_values, arg_kinds, arg_names, expr.line) + + +def translate_cast_expr(builder: IRBuilder, expr: CastExpr) -> Value: + src = builder.accept(expr.expr) + target_type = builder.type_to_rtype(expr.type) + return builder.coerce(src, target_type, expr.line) + + +# Operators + + +def transform_unary_expr(builder: IRBuilder, expr: UnaryExpr) -> Value: + folded = try_constant_fold(builder, expr) + if folded: + return folded + + return builder.unary_op(builder.accept(expr.expr), expr.op, expr.line) + + +def transform_op_expr(builder: IRBuilder, expr: OpExpr) -> Value: + if expr.op in ("and", "or"): + return builder.shortcircuit_expr(expr) + + # Special case for string formatting + if expr.op == "%" and isinstance(expr.left, (StrExpr, BytesExpr)): + ret = translate_printf_style_formatting(builder, expr.left, expr.right) + if ret is not None: + return ret + + folded = try_constant_fold(builder, expr) + if folded: + return folded + + borrow_left = False + borrow_right = False + + ltype = builder.node_type(expr.left) + rtype = builder.node_type(expr.right) + + # Special case some int ops to allow borrowing operands. + if is_int_rprimitive(ltype) and is_int_rprimitive(rtype): + if expr.op == "//": + expr = try_optimize_int_floor_divide(expr) + if expr.op in int_borrow_friendly_op: + borrow_left = is_borrow_friendly_expr(builder, expr.right) + borrow_right = True + elif is_fixed_width_rtype(ltype) and is_fixed_width_rtype(rtype): + borrow_left = is_borrow_friendly_expr(builder, expr.right) + borrow_right = True + + left = builder.accept(expr.left, can_borrow=borrow_left) + right = builder.accept(expr.right, can_borrow=borrow_right) + return builder.binary_op(left, right, expr.op, expr.line) + + +def try_optimize_int_floor_divide(expr: OpExpr) -> OpExpr: + """Replace // with a power of two with a right shift, if possible.""" + if not isinstance(expr.right, IntExpr): + return expr + divisor = expr.right.value + shift = divisor.bit_length() - 1 + if 0 < shift < 28 and divisor == (1 << shift): + return OpExpr(">>", expr.left, IntExpr(shift)) + return expr + + +def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: + index = expr.index + base_type = builder.node_type(expr.base) + is_list = is_list_rprimitive(base_type) + can_borrow_base = is_list and is_borrow_friendly_expr(builder, index) + + base = builder.accept(expr.base, can_borrow=can_borrow_base) + + if isinstance(base.type, RTuple) and isinstance(index, IntExpr): + return builder.add(TupleGet(base, index.value, expr.line)) + + if isinstance(index, SliceExpr): + value = try_gen_slice_op(builder, base, index) + if value: + return value + + index_reg = builder.accept(expr.index, can_borrow=is_list) + return builder.gen_method_call( + base, "__getitem__", [index_reg], builder.node_type(expr), expr.line + ) + + +def try_constant_fold(builder: IRBuilder, expr: Expression) -> Value | None: + """Return the constant value of an expression if possible. + + Return None otherwise. + """ + value = constant_fold_expr(builder, expr) + if value is not None: + return builder.load_literal_value(value) + return None + + +def try_gen_slice_op(builder: IRBuilder, base: Value, index: SliceExpr) -> Value | None: + """Generate specialized slice op for some index expressions. + + Return None if a specialized op isn't available. + + This supports obj[x:y], obj[:x], and obj[x:] for a few types. + """ + if index.stride: + # We can only handle the default stride of 1. + return None + + if index.begin_index: + begin_type = builder.node_type(index.begin_index) + else: + begin_type = int_rprimitive + if index.end_index: + end_type = builder.node_type(index.end_index) + else: + end_type = int_rprimitive + + # Both begin and end index must be int (or missing). + if is_int_rprimitive(begin_type) and is_int_rprimitive(end_type): + if index.begin_index: + begin = builder.accept(index.begin_index) + else: + begin = builder.load_int(0) + if index.end_index: + end = builder.accept(index.end_index) + else: + # Replace missing end index with the largest short integer + # (a sequence can't be longer). + end = builder.load_int(MAX_SHORT_INT) + candidates = [list_slice_op, tuple_slice_op, str_slice_op, bytes_slice_op] + return builder.builder.matching_call_c(candidates, [base, begin, end], index.line) + + return None + + +def transform_conditional_expr(builder: IRBuilder, expr: ConditionalExpr) -> Value: + if_body, else_body, next_block = BasicBlock(), BasicBlock(), BasicBlock() + + process_conditional(builder, expr.cond, if_body, else_body) + expr_type = builder.node_type(expr) + # Having actual Phi nodes would be really nice here! + target = Register(expr_type) + + builder.activate_block(if_body) + true_value = builder.accept(expr.if_expr) + true_value = builder.coerce(true_value, expr_type, expr.line) + builder.add(Assign(target, true_value)) + builder.goto(next_block) + + builder.activate_block(else_body) + false_value = builder.accept(expr.else_expr) + false_value = builder.coerce(false_value, expr_type, expr.line) + builder.add(Assign(target, false_value)) + builder.goto(next_block) + + builder.activate_block(next_block) + + return target + + +def set_literal_values(builder: IRBuilder, items: Sequence[Expression]) -> list[object] | None: + values: list[object] = [] + for item in items: + const_value = constant_fold_expr(builder, item) + if const_value is not None: + values.append(const_value) + continue + + if isinstance(item, RefExpr): + if item.fullname == "builtins.None": + values.append(None) + elif item.fullname == "builtins.True": + values.append(True) + elif item.fullname == "builtins.False": + values.append(False) + elif isinstance(item, TupleExpr): + tuple_values = set_literal_values(builder, item.items) + if tuple_values is not None: + values.append(tuple(tuple_values)) + + if len(values) != len(items): + # Bail if not all items can be converted into values. + return None + return values + + +def precompute_set_literal(builder: IRBuilder, s: SetExpr) -> Value | None: + """Try to pre-compute a frozenset literal during module initialization. + + Return None if it's not possible. + + Supported items: + - Anything supported by irbuild.constant_fold.constant_fold_expr() + - None, True, and False + - Tuple literals with only items listed above + """ + values = set_literal_values(builder, s.items) + if values is not None: + return builder.add(LoadLiteral(frozenset(values), set_rprimitive)) + + return None + + +def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value: + # x in (...)/[...] + # x not in (...)/[...] + first_op = e.operators[0] + if ( + first_op in ["in", "not in"] + and len(e.operators) == 1 + and isinstance(e.operands[1], (TupleExpr, ListExpr)) + ): + items = e.operands[1].items + n_items = len(items) + # x in y -> x == y[0] or ... or x == y[n] + # x not in y -> x != y[0] and ... and x != y[n] + # 16 is arbitrarily chosen to limit code size + if 1 < n_items < 16: + if e.operators[0] == "in": + bin_op = "or" + cmp_op = "==" + else: + bin_op = "and" + cmp_op = "!=" + lhs = e.operands[0] + mypy_file = builder.graph["builtins"].tree + assert mypy_file is not None + info = mypy_file.names["bool"].node + assert isinstance(info, TypeInfo) + bool_type = Instance(info, []) + exprs = [] + for item in items: + expr = ComparisonExpr([cmp_op], [lhs, item]) + builder.types[expr] = bool_type + exprs.append(expr) + + or_expr: Expression = exprs.pop(0) + for expr in exprs: + or_expr = OpExpr(bin_op, or_expr, expr) + builder.types[or_expr] = bool_type + return builder.accept(or_expr) + # x in [y]/(y) -> x == y + # x not in [y]/(y) -> x != y + elif n_items == 1: + if e.operators[0] == "in": + cmp_op = "==" + else: + cmp_op = "!=" + e.operators = [cmp_op] + e.operands[1] = items[0] + # x in []/() -> False + # x not in []/() -> True + elif n_items == 0: + if e.operators[0] == "in": + return builder.false() + else: + return builder.true() + + # x in {...} + # x not in {...} + if ( + first_op in ("in", "not in") + and len(e.operators) == 1 + and isinstance(e.operands[1], SetExpr) + ): + set_literal = precompute_set_literal(builder, e.operands[1]) + if set_literal is not None: + lhs = e.operands[0] + result = builder.builder.primitive_op( + set_in_op, [builder.accept(lhs), set_literal], e.line, bool_rprimitive + ) + if first_op == "not in": + return builder.unary_op(result, "not", e.line) + return result + + if len(e.operators) == 1: + # Special some common simple cases + if first_op in ("is", "is not"): + right_expr = e.operands[1] + if isinstance(right_expr, NameExpr) and right_expr.fullname == "builtins.None": + # Special case 'is None' / 'is not None'. + return translate_is_none(builder, e.operands[0], negated=first_op != "is") + left_expr = e.operands[0] + if is_int_rprimitive(builder.node_type(left_expr)): + right_expr = e.operands[1] + if is_int_rprimitive(builder.node_type(right_expr)): + if first_op in int_borrow_friendly_op: + borrow_left = is_borrow_friendly_expr(builder, right_expr) + left = builder.accept(left_expr, can_borrow=borrow_left) + right = builder.accept(right_expr, can_borrow=True) + return builder.binary_op(left, right, first_op, e.line) + + # TODO: Don't produce an expression when used in conditional context + # All of the trickiness here is due to support for chained conditionals + # (`e1 < e2 > e3`, etc). `e1 < e2 > e3` is approximately equivalent to + # `e1 < e2 and e2 > e3` except that `e2` is only evaluated once. + expr_type = builder.node_type(e) + + # go(i, prev) generates code for `ei opi e{i+1} op{i+1} ... en`, + # assuming that prev contains the value of `ei`. + def go(i: int, prev: Value) -> Value: + if i == len(e.operators) - 1: + return transform_basic_comparison( + builder, e.operators[i], prev, builder.accept(e.operands[i + 1]), e.line + ) + + next = builder.accept(e.operands[i + 1]) + return builder.builder.shortcircuit_helper( + "and", + expr_type, + lambda: transform_basic_comparison(builder, e.operators[i], prev, next, e.line), + lambda: go(i + 1, next), + e.line, + ) + + return go(0, builder.accept(e.operands[0])) + + +def translate_is_none(builder: IRBuilder, expr: Expression, negated: bool) -> Value: + v = builder.accept(expr, can_borrow=True) + return builder.binary_op(v, builder.none_object(), "is not" if negated else "is", expr.line) + + +def transform_basic_comparison( + builder: IRBuilder, op: str, left: Value, right: Value, line: int +) -> Value: + if is_fixed_width_rtype(left.type) and op in ComparisonOp.signed_ops: + if right.type == left.type: + if left.type.is_signed: + op_id = ComparisonOp.signed_ops[op] + else: + op_id = ComparisonOp.unsigned_ops[op] + return builder.builder.comparison_op(left, right, op_id, line) + elif isinstance(right, Integer): + if left.type.is_signed: + op_id = ComparisonOp.signed_ops[op] + else: + op_id = ComparisonOp.unsigned_ops[op] + return builder.builder.comparison_op( + left, builder.coerce(right, left.type, line), op_id, line + ) + elif ( + is_fixed_width_rtype(right.type) + and op in ComparisonOp.signed_ops + and isinstance(left, Integer) + ): + if right.type.is_signed: + op_id = ComparisonOp.signed_ops[op] + else: + op_id = ComparisonOp.unsigned_ops[op] + return builder.builder.comparison_op( + builder.coerce(left, right.type, line), right, op_id, line + ) + + negate = False + if op == "is not": + op, negate = "is", True + elif op == "not in": + op, negate = "in", True + + target = builder.binary_op(left, right, op, line) + + if negate: + target = builder.unary_op(target, "not", line) + return target + + +def translate_printf_style_formatting( + builder: IRBuilder, format_expr: StrExpr | BytesExpr, rhs: Expression +) -> Value | None: + tokens = tokenizer_printf_style(format_expr.value) + if tokens is not None: + literals, format_ops = tokens + + exprs = [] + if isinstance(rhs, TupleExpr): + exprs = rhs.items + elif isinstance(rhs, Expression): + exprs.append(rhs) + + if isinstance(format_expr, BytesExpr): + substitutions = convert_format_expr_to_bytes( + builder, format_ops, exprs, format_expr.line + ) + if substitutions is not None: + return join_formatted_bytes(builder, literals, substitutions, format_expr.line) + else: + substitutions = convert_format_expr_to_str( + builder, format_ops, exprs, format_expr.line + ) + if substitutions is not None: + return join_formatted_strings(builder, literals, substitutions, format_expr.line) + + return None + + +# Literals + + +def transform_int_expr(builder: IRBuilder, expr: IntExpr) -> Value: + return builder.builder.load_int(expr.value) + + +def transform_float_expr(builder: IRBuilder, expr: FloatExpr) -> Value: + return builder.builder.load_float(expr.value) + + +def transform_complex_expr(builder: IRBuilder, expr: ComplexExpr) -> Value: + return builder.builder.load_complex(expr.value) + + +def transform_str_expr(builder: IRBuilder, expr: StrExpr) -> Value: + return builder.load_str(expr.value) + + +def transform_bytes_expr(builder: IRBuilder, expr: BytesExpr) -> Value: + return builder.load_bytes_from_str_literal(expr.value) + + +def transform_ellipsis(builder: IRBuilder, o: EllipsisExpr) -> Value: + return builder.add(LoadAddress(ellipsis_op.type, ellipsis_op.src, o.line)) + + +# Display expressions + + +def transform_list_expr(builder: IRBuilder, expr: ListExpr) -> Value: + return _visit_list_display(builder, expr.items, expr.line) + + +def _visit_list_display(builder: IRBuilder, items: list[Expression], line: int) -> Value: + return _visit_display( + builder, items, builder.new_list_op, list_append_op, list_extend_op, line, True + ) + + +def transform_tuple_expr(builder: IRBuilder, expr: TupleExpr) -> Value: + if any(isinstance(item, StarExpr) for item in expr.items): + # create a tuple of unknown length + return _visit_tuple_display(builder, expr) + + # create a tuple of fixed length (RTuple) + tuple_type = builder.node_type(expr) + # When handling NamedTuple et. al we might not have proper type info, + # so make some up if we need it. + types = ( + tuple_type.types + if isinstance(tuple_type, RTuple) + else [object_rprimitive] * len(expr.items) + ) + + items = [] + for item_expr, item_type in zip(expr.items, types): + reg = builder.accept(item_expr) + items.append(builder.coerce(reg, item_type, item_expr.line)) + return builder.add(TupleSet(items, expr.line)) + + +def _visit_tuple_display(builder: IRBuilder, expr: TupleExpr) -> Value: + """Create a list, then turn it into a tuple.""" + val_as_list = _visit_list_display(builder, expr.items, expr.line) + return builder.primitive_op(list_tuple_op, [val_as_list], expr.line) + + +def transform_dict_expr(builder: IRBuilder, expr: DictExpr) -> Value: + """First accepts all keys and values, then makes a dict out of them.""" + key_value_pairs = [] + for key_expr, value_expr in expr.items: + key = builder.accept(key_expr) if key_expr is not None else None + value = builder.accept(value_expr) + key_value_pairs.append((key, value)) + + return builder.builder.make_dict(key_value_pairs, expr.line) + + +def transform_set_expr(builder: IRBuilder, expr: SetExpr) -> Value: + return _visit_display( + builder, expr.items, builder.new_set_op, set_add_op, set_update_op, expr.line, False + ) + + +def _visit_display( + builder: IRBuilder, + items: list[Expression], + constructor_op: Callable[[list[Value], int], Value], + append_op: PrimitiveDescription, + extend_op: PrimitiveDescription, + line: int, + is_list: bool, +) -> Value: + accepted_items = [] + for item in items: + if isinstance(item, StarExpr): + accepted_items.append((True, builder.accept(item.expr))) + else: + accepted_items.append((False, builder.accept(item))) + + result: Value | None = None + initial_items = [] + for starred, value in accepted_items: + if result is None and not starred and is_list: + initial_items.append(value) + continue + + if result is None: + result = constructor_op(initial_items, line) + + builder.primitive_op(extend_op if starred else append_op, [result, value], line) + + if result is None: + result = constructor_op(initial_items, line) + + return result + + +# Comprehensions + + +def transform_list_comprehension(builder: IRBuilder, o: ListComprehension) -> Value: + return translate_list_comprehension(builder, o.generator) + + +def transform_set_comprehension(builder: IRBuilder, o: SetComprehension) -> Value: + return translate_set_comprehension(builder, o.generator) + + +def transform_dictionary_comprehension(builder: IRBuilder, o: DictionaryComprehension) -> Value: + if raise_error_if_contains_unreachable_names(builder, o): + return builder.none() + + d = builder.maybe_spill(builder.call_c(dict_new_op, [], o.line)) + loop_params = list(zip(o.indices, o.sequences, o.condlists, o.is_async)) + + def gen_inner_stmts() -> None: + k = builder.accept(o.key) + v = builder.accept(o.value) + builder.primitive_op(dict_set_item_op, [builder.read(d), k, v], o.line) + + comprehension_helper(builder, loop_params, gen_inner_stmts, o.line) + return builder.read(d) + + +# Misc + + +def transform_slice_expr(builder: IRBuilder, expr: SliceExpr) -> Value: + def get_arg(arg: Expression | None) -> Value: + if arg is None: + return builder.none_object() + else: + return builder.accept(arg) + + args = [get_arg(expr.begin_index), get_arg(expr.end_index), get_arg(expr.stride)] + return builder.primitive_op(new_slice_op, args, expr.line) + + +def transform_generator_expr(builder: IRBuilder, o: GeneratorExpr) -> Value: + builder.warning("Treating generator comprehension as list", o.line) + return builder.primitive_op(iter_op, [translate_list_comprehension(builder, o)], o.line) + + +def transform_assignment_expr(builder: IRBuilder, o: AssignmentExpr) -> Value: + value = builder.accept(o.value) + target = builder.get_assignment_target(o.target) + builder.assign(target, value, o.line) + return value + + +def transform_math_literal(builder: IRBuilder, fullname: str) -> Value | None: + if fullname == "math.e": + return builder.load_float(math.e) + if fullname == "math.pi": + return builder.load_float(math.pi) + if fullname == "math.inf": + return builder.load_float(math.inf) + if fullname == "math.nan": + return builder.load_float(math.nan) + if fullname == "math.tau": + return builder.load_float(math.tau) + + return None diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py new file mode 100644 index 000000000000..9b34a094db60 --- /dev/null +++ b/mypyc/irbuild/for_helpers.py @@ -0,0 +1,1076 @@ +"""Helpers for generating for loops and comprehensions. + +We special case certain kinds for loops such as "for x in range(...)" +for better efficiency. Each for loop generator class below deals one +such special case. +""" + +from __future__ import annotations + +from typing import Callable, ClassVar + +from mypy.nodes import ( + ARG_POS, + CallExpr, + DictionaryComprehension, + Expression, + GeneratorExpr, + Lvalue, + MemberExpr, + NameExpr, + RefExpr, + SetExpr, + TupleExpr, + TypeAlias, +) +from mypyc.ir.ops import ( + BasicBlock, + Branch, + Integer, + IntOp, + LoadAddress, + LoadMem, + RaiseStandardError, + Register, + TupleGet, + TupleSet, + Value, +) +from mypyc.ir.rtypes import ( + RTuple, + RType, + bool_rprimitive, + int_rprimitive, + is_dict_rprimitive, + is_fixed_width_rtype, + is_list_rprimitive, + is_sequence_rprimitive, + is_short_int_rprimitive, + is_str_rprimitive, + is_tuple_rprimitive, + pointer_rprimitive, + short_int_rprimitive, +) +from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.targets import AssignmentTarget, AssignmentTargetTuple +from mypyc.primitives.dict_ops import ( + dict_check_size_op, + dict_item_iter_op, + dict_key_iter_op, + dict_next_item_op, + dict_next_key_op, + dict_next_value_op, + dict_value_iter_op, +) +from mypyc.primitives.exc_ops import no_err_occurred_op +from mypyc.primitives.generic_ops import aiter_op, anext_op, iter_op, next_op +from mypyc.primitives.list_ops import list_append_op, list_get_item_unsafe_op, new_list_set_item_op +from mypyc.primitives.misc_ops import stop_async_iteration_op +from mypyc.primitives.registry import CFunctionDescription +from mypyc.primitives.set_ops import set_add_op + +GenFunc = Callable[[], None] + + +def for_loop_helper( + builder: IRBuilder, + index: Lvalue, + expr: Expression, + body_insts: GenFunc, + else_insts: GenFunc | None, + is_async: bool, + line: int, +) -> None: + """Generate IR for a loop. + + Args: + index: the loop index Lvalue + expr: the expression to iterate over + body_insts: a function that generates the body of the loop + else_insts: a function that generates the else block instructions + """ + # Body of the loop + body_block = BasicBlock() + # Block that steps to the next item + step_block = BasicBlock() + # Block for the else clause, if we need it + else_block = BasicBlock() + # Block executed after the loop + exit_block = BasicBlock() + + # Determine where we want to exit, if our condition check fails. + normal_loop_exit = else_block if else_insts is not None else exit_block + + for_gen = make_for_loop_generator( + builder, index, expr, body_block, normal_loop_exit, line, is_async=is_async + ) + + builder.push_loop_stack(step_block, exit_block) + condition_block = BasicBlock() + builder.goto_and_activate(condition_block) + + # Add loop condition check. + for_gen.gen_condition() + + # Generate loop body. + builder.activate_block(body_block) + for_gen.begin_body() + body_insts() + + # We generate a separate step block (which might be empty). + builder.goto_and_activate(step_block) + for_gen.gen_step() + # Go back to loop condition. + builder.goto(condition_block) + + for_gen.add_cleanup(normal_loop_exit) + builder.pop_loop_stack() + + if else_insts is not None: + builder.activate_block(else_block) + else_insts() + builder.goto(exit_block) + + builder.activate_block(exit_block) + + +def for_loop_helper_with_index( + builder: IRBuilder, + index: Lvalue, + expr: Expression, + expr_reg: Value, + body_insts: Callable[[Value], None], + line: int, +) -> None: + """Generate IR for a sequence iteration. + + This function only works for sequence type. Compared to for_loop_helper, + it would feed iteration index to body_insts. + + Args: + index: the loop index Lvalue + expr: the expression to iterate over + body_insts: a function that generates the body of the loop. + It needs a index as parameter. + """ + assert is_sequence_rprimitive(expr_reg.type) + target_type = builder.get_sequence_type(expr) + + body_block = BasicBlock() + step_block = BasicBlock() + exit_block = BasicBlock() + condition_block = BasicBlock() + + for_gen = ForSequence(builder, index, body_block, exit_block, line, False) + for_gen.init(expr_reg, target_type, reverse=False) + + builder.push_loop_stack(step_block, exit_block) + + builder.goto_and_activate(condition_block) + for_gen.gen_condition() + + builder.activate_block(body_block) + for_gen.begin_body() + body_insts(builder.read(for_gen.index_target)) + + builder.goto_and_activate(step_block) + for_gen.gen_step() + builder.goto(condition_block) + + for_gen.add_cleanup(exit_block) + builder.pop_loop_stack() + + builder.activate_block(exit_block) + + +def sequence_from_generator_preallocate_helper( + builder: IRBuilder, + gen: GeneratorExpr, + empty_op_llbuilder: Callable[[Value, int], Value], + set_item_op: CFunctionDescription, +) -> Value | None: + """Generate a new tuple or list from a simple generator expression. + + Currently we only optimize for simplest generator expression, which means that + there is no condition list in the generator and only one original sequence with + one index is allowed. + + e.g. (1) tuple(f(x) for x in a_list/a_tuple) + (2) list(f(x) for x in a_list/a_tuple) + (3) [f(x) for x in a_list/a_tuple] + RTuple as an original sequence is not supported yet. + + Args: + empty_op_llbuilder: A function that can generate an empty sequence op when + passed in length. See `new_list_op_with_length` and `new_tuple_op_with_length` + for detailed implementation. + set_item_op: A primitive that can modify an arbitrary position of a sequence. + The op should have three arguments: + - Self + - Target position + - New Value + See `new_list_set_item_op` and `new_tuple_set_item_op` for detailed + implementation. + """ + if len(gen.sequences) == 1 and len(gen.indices) == 1 and len(gen.condlists[0]) == 0: + rtype = builder.node_type(gen.sequences[0]) + if is_list_rprimitive(rtype) or is_tuple_rprimitive(rtype) or is_str_rprimitive(rtype): + sequence = builder.accept(gen.sequences[0]) + length = builder.builder.builtin_len(sequence, gen.line, use_pyssize_t=True) + target_op = empty_op_llbuilder(length, gen.line) + + def set_item(item_index: Value) -> None: + e = builder.accept(gen.left_expr) + builder.call_c(set_item_op, [target_op, item_index, e], gen.line) + + for_loop_helper_with_index( + builder, gen.indices[0], gen.sequences[0], sequence, set_item, gen.line + ) + + return target_op + return None + + +def translate_list_comprehension(builder: IRBuilder, gen: GeneratorExpr) -> Value: + if raise_error_if_contains_unreachable_names(builder, gen): + return builder.none() + + # Try simplest list comprehension, otherwise fall back to general one + val = sequence_from_generator_preallocate_helper( + builder, + gen, + empty_op_llbuilder=builder.builder.new_list_op_with_length, + set_item_op=new_list_set_item_op, + ) + if val is not None: + return val + + list_ops = builder.maybe_spill(builder.new_list_op([], gen.line)) + + loop_params = list(zip(gen.indices, gen.sequences, gen.condlists, gen.is_async)) + + def gen_inner_stmts() -> None: + e = builder.accept(gen.left_expr) + builder.primitive_op(list_append_op, [builder.read(list_ops), e], gen.line) + + comprehension_helper(builder, loop_params, gen_inner_stmts, gen.line) + return builder.read(list_ops) + + +def raise_error_if_contains_unreachable_names( + builder: IRBuilder, gen: GeneratorExpr | DictionaryComprehension +) -> bool: + """Raise a runtime error and return True if generator contains unreachable names. + + False is returned if the generator can be safely transformed without crashing. + (It may still be unreachable!) + """ + if any(isinstance(s, NameExpr) and s.node is None for s in gen.indices): + error = RaiseStandardError( + RaiseStandardError.RUNTIME_ERROR, + "mypyc internal error: should be unreachable", + gen.line, + ) + builder.add(error) + return True + + return False + + +def translate_set_comprehension(builder: IRBuilder, gen: GeneratorExpr) -> Value: + if raise_error_if_contains_unreachable_names(builder, gen): + return builder.none() + + set_ops = builder.maybe_spill(builder.new_set_op([], gen.line)) + loop_params = list(zip(gen.indices, gen.sequences, gen.condlists, gen.is_async)) + + def gen_inner_stmts() -> None: + e = builder.accept(gen.left_expr) + builder.primitive_op(set_add_op, [builder.read(set_ops), e], gen.line) + + comprehension_helper(builder, loop_params, gen_inner_stmts, gen.line) + return builder.read(set_ops) + + +def comprehension_helper( + builder: IRBuilder, + loop_params: list[tuple[Lvalue, Expression, list[Expression], bool]], + gen_inner_stmts: Callable[[], None], + line: int, +) -> None: + """Helper function for list comprehensions. + + Args: + loop_params: a list of (index, expr, [conditions]) tuples defining nested loops: + - "index" is the Lvalue indexing that loop; + - "expr" is the expression for the object to be iterated over; + - "conditions" is a list of conditions, evaluated in order with short-circuiting, + that must all be true for the loop body to be executed + gen_inner_stmts: function to generate the IR for the body of the innermost loop + """ + + def handle_loop(loop_params: list[tuple[Lvalue, Expression, list[Expression], bool]]) -> None: + """Generate IR for a loop. + + Given a list of (index, expression, [conditions]) tuples, generate IR + for the nested loops the list defines. + """ + index, expr, conds, is_async = loop_params[0] + for_loop_helper( + builder, + index, + expr, + lambda: loop_contents(conds, loop_params[1:]), + None, + is_async=is_async, + line=line, + ) + + def loop_contents( + conds: list[Expression], + remaining_loop_params: list[tuple[Lvalue, Expression, list[Expression], bool]], + ) -> None: + """Generate the body of the loop. + + Args: + conds: a list of conditions to be evaluated (in order, with short circuiting) + to gate the body of the loop + remaining_loop_params: the parameters for any further nested loops; if it's empty + we'll instead evaluate the "gen_inner_stmts" function + """ + # Check conditions, in order, short circuiting them. + for cond in conds: + cond_val = builder.accept(cond) + cont_block, rest_block = BasicBlock(), BasicBlock() + # If the condition is true we'll skip the continue. + builder.add_bool_branch(cond_val, rest_block, cont_block) + builder.activate_block(cont_block) + builder.nonlocal_control[-1].gen_continue(builder, cond.line) + builder.goto_and_activate(rest_block) + + if remaining_loop_params: + # There's another nested level, so the body of this loop is another loop. + return handle_loop(remaining_loop_params) + else: + # We finally reached the actual body of the generator. + # Generate the IR for the inner loop body. + gen_inner_stmts() + + handle_loop(loop_params) + + +def is_range_ref(expr: RefExpr) -> bool: + return ( + expr.fullname == "builtins.range" + or isinstance(expr.node, TypeAlias) + and expr.fullname == "six.moves.xrange" + ) + + +def make_for_loop_generator( + builder: IRBuilder, + index: Lvalue, + expr: Expression, + body_block: BasicBlock, + loop_exit: BasicBlock, + line: int, + is_async: bool = False, + nested: bool = False, +) -> ForGenerator: + """Return helper object for generating a for loop over an iterable. + + If "nested" is True, this is a nested iterator such as "e" in "enumerate(e)". + """ + + # Do an async loop if needed. async is always generic + if is_async: + expr_reg = builder.accept(expr) + async_obj = ForAsyncIterable(builder, index, body_block, loop_exit, line, nested) + item_type = builder._analyze_iterable_item_type(expr) + item_rtype = builder.type_to_rtype(item_type) + async_obj.init(expr_reg, item_rtype) + return async_obj + + rtyp = builder.node_type(expr) + if is_sequence_rprimitive(rtyp): + # Special case "for x in ". + expr_reg = builder.accept(expr) + target_type = builder.get_sequence_type(expr) + + for_list = ForSequence(builder, index, body_block, loop_exit, line, nested) + for_list.init(expr_reg, target_type, reverse=False) + return for_list + + if is_dict_rprimitive(rtyp): + # Special case "for k in ". + expr_reg = builder.accept(expr) + target_type = builder.get_dict_key_type(expr) + + for_dict = ForDictionaryKeys(builder, index, body_block, loop_exit, line, nested) + for_dict.init(expr_reg, target_type) + return for_dict + + if isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr): + if ( + is_range_ref(expr.callee) + and ( + len(expr.args) <= 2 + or (len(expr.args) == 3 and builder.extract_int(expr.args[2]) is not None) + ) + and set(expr.arg_kinds) == {ARG_POS} + ): + # Special case "for x in range(...)". + # We support the 3 arg form but only for int literals, since it doesn't + # seem worth the hassle of supporting dynamically determining which + # direction of comparison to do. + if len(expr.args) == 1: + start_reg: Value = Integer(0) + end_reg = builder.accept(expr.args[0]) + else: + start_reg = builder.accept(expr.args[0]) + end_reg = builder.accept(expr.args[1]) + if len(expr.args) == 3: + step = builder.extract_int(expr.args[2]) + assert step is not None + if step == 0: + builder.error("range() step can't be zero", expr.args[2].line) + else: + step = 1 + + for_range = ForRange(builder, index, body_block, loop_exit, line, nested) + for_range.init(start_reg, end_reg, step) + return for_range + + elif ( + expr.callee.fullname == "builtins.enumerate" + and len(expr.args) == 1 + and expr.arg_kinds == [ARG_POS] + and isinstance(index, TupleExpr) + and len(index.items) == 2 + ): + # Special case "for i, x in enumerate(y)". + lvalue1 = index.items[0] + lvalue2 = index.items[1] + for_enumerate = ForEnumerate(builder, index, body_block, loop_exit, line, nested) + for_enumerate.init(lvalue1, lvalue2, expr.args[0]) + return for_enumerate + + elif ( + expr.callee.fullname == "builtins.zip" + and len(expr.args) >= 2 + and set(expr.arg_kinds) == {ARG_POS} + and isinstance(index, TupleExpr) + and len(index.items) == len(expr.args) + ): + # Special case "for x, y in zip(a, b)". + for_zip = ForZip(builder, index, body_block, loop_exit, line, nested) + for_zip.init(index.items, expr.args) + return for_zip + + if ( + expr.callee.fullname == "builtins.reversed" + and len(expr.args) == 1 + and expr.arg_kinds == [ARG_POS] + and is_sequence_rprimitive(builder.node_type(expr.args[0])) + ): + # Special case "for x in reversed()". + expr_reg = builder.accept(expr.args[0]) + target_type = builder.get_sequence_type(expr) + + for_list = ForSequence(builder, index, body_block, loop_exit, line, nested) + for_list.init(expr_reg, target_type, reverse=True) + return for_list + if isinstance(expr, CallExpr) and isinstance(expr.callee, MemberExpr) and not expr.args: + # Special cases for dictionary iterator methods, like dict.items(). + rtype = builder.node_type(expr.callee.expr) + if is_dict_rprimitive(rtype) and expr.callee.name in ("keys", "values", "items"): + expr_reg = builder.accept(expr.callee.expr) + for_dict_type: type[ForGenerator] | None = None + if expr.callee.name == "keys": + target_type = builder.get_dict_key_type(expr.callee.expr) + for_dict_type = ForDictionaryKeys + elif expr.callee.name == "values": + target_type = builder.get_dict_value_type(expr.callee.expr) + for_dict_type = ForDictionaryValues + else: + target_type = builder.get_dict_item_type(expr.callee.expr) + for_dict_type = ForDictionaryItems + for_dict_gen = for_dict_type(builder, index, body_block, loop_exit, line, nested) + for_dict_gen.init(expr_reg, target_type) + return for_dict_gen + + iterable_expr_reg: Value | None = None + if isinstance(expr, SetExpr): + # Special case "for x in ". + from mypyc.irbuild.expression import precompute_set_literal + + set_literal = precompute_set_literal(builder, expr) + if set_literal is not None: + iterable_expr_reg = set_literal + + # Default to a generic for loop. + if iterable_expr_reg is None: + iterable_expr_reg = builder.accept(expr) + for_obj = ForIterable(builder, index, body_block, loop_exit, line, nested) + item_type = builder._analyze_iterable_item_type(expr) + item_rtype = builder.type_to_rtype(item_type) + for_obj.init(iterable_expr_reg, item_rtype) + return for_obj + + +class ForGenerator: + """Abstract base class for generating for loops.""" + + def __init__( + self, + builder: IRBuilder, + index: Lvalue, + body_block: BasicBlock, + loop_exit: BasicBlock, + line: int, + nested: bool, + ) -> None: + self.builder = builder + self.index = index + self.body_block = body_block + self.line = line + # Some for loops need a cleanup block that we execute at exit. We + # create a cleanup block if needed. However, if we are generating a for + # loop for a nested iterator, such as "e" in "enumerate(e)", the + # outermost generator should generate the cleanup block -- we don't + # need to do it here. + if self.need_cleanup() and not nested: + # Create a new block to handle cleanup after loop exit. + self.loop_exit = BasicBlock() + else: + # Just use the existing loop exit block. + self.loop_exit = loop_exit + + def need_cleanup(self) -> bool: + """If this returns true, we need post-loop cleanup.""" + return False + + def add_cleanup(self, exit_block: BasicBlock) -> None: + """Add post-loop cleanup, if needed.""" + if self.need_cleanup(): + self.builder.activate_block(self.loop_exit) + self.gen_cleanup() + self.builder.goto(exit_block) + + def gen_condition(self) -> None: + """Generate check for loop exit (e.g. exhaustion of iteration).""" + + def begin_body(self) -> None: + """Generate ops at the beginning of the body (if needed).""" + + def gen_step(self) -> None: + """Generate stepping to the next item (if needed).""" + + def gen_cleanup(self) -> None: + """Generate post-loop cleanup (if needed).""" + + def load_len(self, expr: Value | AssignmentTarget) -> Value: + """A helper to get collection length, used by several subclasses.""" + return self.builder.builder.builtin_len(self.builder.read(expr, self.line), self.line) + + +class ForIterable(ForGenerator): + """Generate IR for a for loop over an arbitrary iterable (the general case).""" + + def need_cleanup(self) -> bool: + # Create a new cleanup block for when the loop is finished. + return True + + def init(self, expr_reg: Value, target_type: RType) -> None: + # Define targets to contain the expression, along with the iterator that will be used + # for the for-loop. If we are inside of a generator function, spill these into the + # environment class. + builder = self.builder + iter_reg = builder.primitive_op(iter_op, [expr_reg], self.line) + builder.maybe_spill(expr_reg) + self.iter_target = builder.maybe_spill(iter_reg) + self.target_type = target_type + + def gen_condition(self) -> None: + # We call __next__ on the iterator and check to see if the return value + # is NULL, which signals either the end of the Iterable being traversed + # or an exception being raised. Note that Branch.IS_ERROR checks only + # for NULL (an exception does not necessarily have to be raised). + builder = self.builder + line = self.line + self.next_reg = builder.call_c(next_op, [builder.read(self.iter_target, line)], line) + builder.add(Branch(self.next_reg, self.loop_exit, self.body_block, Branch.IS_ERROR)) + + def begin_body(self) -> None: + # Assign the value obtained from __next__ to the + # lvalue so that it can be referenced by code in the body of the loop. + builder = self.builder + line = self.line + # We unbox here so that iterating with tuple unpacking generates a tuple based + # unpack instead of an iterator based one. + next_reg = builder.coerce(self.next_reg, self.target_type, line) + builder.assign(builder.get_assignment_target(self.index), next_reg, line) + + def gen_step(self) -> None: + # Nothing to do here, since we get the next item as part of gen_condition(). + pass + + def gen_cleanup(self) -> None: + # We set the branch to go here if the conditional evaluates to true. If + # an exception was raised during the loop, then err_reg will be set to + # True. If no_err_occurred_op returns False, then the exception will be + # propagated using the ERR_FALSE flag. + self.builder.call_c(no_err_occurred_op, [], self.line) + + +class ForAsyncIterable(ForGenerator): + """Generate IR for an async for loop.""" + + def init(self, expr_reg: Value, target_type: RType) -> None: + # Define targets to contain the expression, along with the + # iterator that will be used for the for-loop. We are inside + # of a generator function, so we will spill these into + # environment class. + builder = self.builder + iter_reg = builder.call_c(aiter_op, [expr_reg], self.line) + builder.maybe_spill(expr_reg) + self.iter_target = builder.maybe_spill(iter_reg) + self.target_type = target_type + self.stop_reg = Register(bool_rprimitive) + + def gen_condition(self) -> None: + # This does the test and fetches the next value + # try: + # TARGET = await type(iter).__anext__(iter) + # stop = False + # except StopAsyncIteration: + # stop = True + # + # What a pain. + # There are optimizations available here if we punch through some abstractions. + + from mypyc.irbuild.statement import emit_await, transform_try_except + + builder = self.builder + line = self.line + + def except_match() -> Value: + addr = builder.add(LoadAddress(pointer_rprimitive, stop_async_iteration_op.src, line)) + return builder.add(LoadMem(stop_async_iteration_op.type, addr)) + + def try_body() -> None: + awaitable = builder.call_c(anext_op, [builder.read(self.iter_target)], line) + self.next_reg = emit_await(builder, awaitable, line) + builder.assign(self.stop_reg, builder.false(), -1) + + def except_body() -> None: + builder.assign(self.stop_reg, builder.true(), line) + + transform_try_except( + builder, try_body, [((except_match, line), None, except_body)], None, line + ) + + builder.add(Branch(self.stop_reg, self.loop_exit, self.body_block, Branch.BOOL)) + + def begin_body(self) -> None: + # Assign the value obtained from await __anext__ to the + # lvalue so that it can be referenced by code in the body of the loop. + builder = self.builder + line = self.line + # We unbox here so that iterating with tuple unpacking generates a tuple based + # unpack instead of an iterator based one. + next_reg = builder.coerce(self.next_reg, self.target_type, line) + builder.assign(builder.get_assignment_target(self.index), next_reg, line) + + def gen_step(self) -> None: + # Nothing to do here, since we get the next item as part of gen_condition(). + pass + + +def unsafe_index(builder: IRBuilder, target: Value, index: Value, line: int) -> Value: + """Emit a potentially unsafe index into a target.""" + # This doesn't really fit nicely into any of our data-driven frameworks + # since we want to use __getitem__ if we don't have an unsafe version, + # so we just check manually. + if is_list_rprimitive(target.type): + return builder.call_c(list_get_item_unsafe_op, [target, index], line) + else: + return builder.gen_method_call(target, "__getitem__", [index], None, line) + + +class ForSequence(ForGenerator): + """Generate optimized IR for a for loop over a sequence. + + Supports iterating in both forward and reverse. + """ + + def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: + builder = self.builder + self.reverse = reverse + # Define target to contain the expression, along with the index that will be used + # for the for-loop. If we are inside of a generator function, spill these into the + # environment class. + self.expr_target = builder.maybe_spill(expr_reg) + if not reverse: + index_reg: Value = Integer(0) + else: + index_reg = builder.binary_op( + self.load_len(self.expr_target), Integer(1), "-", self.line + ) + self.index_target = builder.maybe_spill_assignable(index_reg) + self.target_type = target_type + + def gen_condition(self) -> None: + builder = self.builder + line = self.line + # TODO: Don't reload the length each time when iterating an immutable sequence? + if self.reverse: + # If we are iterating in reverse order, we obviously need + # to check that the index is still positive. Somewhat less + # obviously we still need to check against the length, + # since it could shrink out from under us. + comparison = builder.binary_op( + builder.read(self.index_target, line), Integer(0), ">=", line + ) + second_check = BasicBlock() + builder.add_bool_branch(comparison, second_check, self.loop_exit) + builder.activate_block(second_check) + # For compatibility with python semantics we recalculate the length + # at every iteration. + len_reg = self.load_len(self.expr_target) + comparison = builder.binary_op(builder.read(self.index_target, line), len_reg, "<", line) + builder.add_bool_branch(comparison, self.body_block, self.loop_exit) + + def begin_body(self) -> None: + builder = self.builder + line = self.line + # Read the next list item. + value_box = unsafe_index( + builder, + builder.read(self.expr_target, line), + builder.read(self.index_target, line), + line, + ) + assert value_box + # We coerce to the type of list elements here so that + # iterating with tuple unpacking generates a tuple based + # unpack instead of an iterator based one. + builder.assign( + builder.get_assignment_target(self.index), + builder.coerce(value_box, self.target_type, line), + line, + ) + + def gen_step(self) -> None: + # Step to the next item. + builder = self.builder + line = self.line + step = 1 if not self.reverse else -1 + add = builder.int_op( + short_int_rprimitive, + builder.read(self.index_target, line), + Integer(step), + IntOp.ADD, + line, + ) + builder.assign(self.index_target, add, line) + + +class ForDictionaryCommon(ForGenerator): + """Generate optimized IR for a for loop over dictionary keys/values. + + The logic is pretty straightforward, we use PyDict_Next() API wrapped in + a tuple, so that we can modify only a single register. The layout of the tuple: + * f0: are there more items (bool) + * f1: current offset (int) + * f2: next key (object) + * f3: next value (object) + For more info see https://docs.python.org/3/c-api/dict.html#c.PyDict_Next. + + Note that for subclasses we fall back to generic PyObject_GetIter() logic, + since they may override some iteration methods in subtly incompatible manner. + The fallback logic is implemented in CPy.h via dynamic type check. + """ + + dict_next_op: ClassVar[CFunctionDescription] + dict_iter_op: ClassVar[CFunctionDescription] + + def need_cleanup(self) -> bool: + # Technically, a dict subclass can raise an unrelated exception + # in __next__(), so we need this. + return True + + def init(self, expr_reg: Value, target_type: RType) -> None: + builder = self.builder + self.target_type = target_type + + # We add some variables to environment class, so they can be read across yield. + self.expr_target = builder.maybe_spill(expr_reg) + offset = Integer(0) + self.offset_target = builder.maybe_spill_assignable(offset) + self.size = builder.maybe_spill(self.load_len(self.expr_target)) + + # For dict class (not a subclass) this is the dictionary itself. + iter_reg = builder.call_c(self.dict_iter_op, [expr_reg], self.line) + self.iter_target = builder.maybe_spill(iter_reg) + + def gen_condition(self) -> None: + """Get next key/value pair, set new offset, and check if we should continue.""" + builder = self.builder + line = self.line + self.next_tuple = self.builder.call_c( + self.dict_next_op, + [builder.read(self.iter_target, line), builder.read(self.offset_target, line)], + line, + ) + + # Do this here instead of in gen_step() to minimize variables in environment. + new_offset = builder.add(TupleGet(self.next_tuple, 1, line)) + builder.assign(self.offset_target, new_offset, line) + + should_continue = builder.add(TupleGet(self.next_tuple, 0, line)) + builder.add(Branch(should_continue, self.body_block, self.loop_exit, Branch.BOOL)) + + def gen_step(self) -> None: + """Check that dictionary didn't change size during iteration. + + Raise RuntimeError if it is not the case to match CPython behavior. + """ + builder = self.builder + line = self.line + # Technically, we don't need a new primitive for this, but it is simpler. + builder.call_c( + dict_check_size_op, + [builder.read(self.expr_target, line), builder.read(self.size, line)], + line, + ) + + def gen_cleanup(self) -> None: + # Same as for generic ForIterable. + self.builder.call_c(no_err_occurred_op, [], self.line) + + +class ForDictionaryKeys(ForDictionaryCommon): + """Generate optimized IR for a for loop over dictionary keys.""" + + dict_next_op = dict_next_key_op + dict_iter_op = dict_key_iter_op + + def begin_body(self) -> None: + builder = self.builder + line = self.line + + # Key is stored at the third place in the tuple. + key = builder.add(TupleGet(self.next_tuple, 2, line)) + builder.assign( + builder.get_assignment_target(self.index), + builder.coerce(key, self.target_type, line), + line, + ) + + +class ForDictionaryValues(ForDictionaryCommon): + """Generate optimized IR for a for loop over dictionary values.""" + + dict_next_op = dict_next_value_op + dict_iter_op = dict_value_iter_op + + def begin_body(self) -> None: + builder = self.builder + line = self.line + + # Value is stored at the third place in the tuple. + value = builder.add(TupleGet(self.next_tuple, 2, line)) + builder.assign( + builder.get_assignment_target(self.index), + builder.coerce(value, self.target_type, line), + line, + ) + + +class ForDictionaryItems(ForDictionaryCommon): + """Generate optimized IR for a for loop over dictionary items.""" + + dict_next_op = dict_next_item_op + dict_iter_op = dict_item_iter_op + + def begin_body(self) -> None: + builder = self.builder + line = self.line + + key = builder.add(TupleGet(self.next_tuple, 2, line)) + value = builder.add(TupleGet(self.next_tuple, 3, line)) + + # Coerce just in case e.g. key is itself a tuple to be unpacked. + assert isinstance(self.target_type, RTuple) + key = builder.coerce(key, self.target_type.types[0], line) + value = builder.coerce(value, self.target_type.types[1], line) + + target = builder.get_assignment_target(self.index) + if isinstance(target, AssignmentTargetTuple): + # Simpler code for common case: for k, v in d.items(). + if len(target.items) != 2: + builder.error("Expected a pair for dict item iteration", line) + builder.assign(target.items[0], key, line) + builder.assign(target.items[1], value, line) + else: + rvalue = builder.add(TupleSet([key, value], line)) + builder.assign(target, rvalue, line) + + +class ForRange(ForGenerator): + """Generate optimized IR for a for loop over an integer range.""" + + def init(self, start_reg: Value, end_reg: Value, step: int) -> None: + builder = self.builder + self.start_reg = start_reg + self.end_reg = end_reg + self.step = step + self.end_target = builder.maybe_spill(end_reg) + if is_short_int_rprimitive(start_reg.type) and is_short_int_rprimitive(end_reg.type): + index_type: RType = short_int_rprimitive + elif is_fixed_width_rtype(end_reg.type): + index_type = end_reg.type + else: + index_type = int_rprimitive + index_reg = Register(index_type) + builder.assign(index_reg, start_reg, -1) + self.index_reg = builder.maybe_spill_assignable(index_reg) + # Initialize loop index to 0. Assert that the index target is assignable. + self.index_target: Register | AssignmentTarget = builder.get_assignment_target(self.index) + builder.assign(self.index_target, builder.read(self.index_reg, self.line), self.line) + + def gen_condition(self) -> None: + builder = self.builder + line = self.line + # Add loop condition check. + cmp = "<" if self.step > 0 else ">" + comparison = builder.binary_op( + builder.read(self.index_reg, line), builder.read(self.end_target, line), cmp, line + ) + builder.add_bool_branch(comparison, self.body_block, self.loop_exit) + + def gen_step(self) -> None: + builder = self.builder + line = self.line + + # Increment index register. If the range is known to fit in short ints, use + # short ints. + if is_short_int_rprimitive(self.start_reg.type) and is_short_int_rprimitive( + self.end_reg.type + ): + new_val = builder.int_op( + short_int_rprimitive, + builder.read(self.index_reg, line), + Integer(self.step), + IntOp.ADD, + line, + ) + + else: + new_val = builder.binary_op( + builder.read(self.index_reg, line), Integer(self.step), "+", line + ) + builder.assign(self.index_reg, new_val, line) + builder.assign(self.index_target, new_val, line) + + +class ForInfiniteCounter(ForGenerator): + """Generate optimized IR for a for loop counting from 0 to infinity.""" + + def init(self) -> None: + builder = self.builder + # Create a register to store the state of the loop index and + # initialize this register along with the loop index to 0. + zero = Integer(0) + self.index_reg = builder.maybe_spill_assignable(zero) + self.index_target: Register | AssignmentTarget = builder.get_assignment_target(self.index) + + def gen_step(self) -> None: + builder = self.builder + line = self.line + # We can safely assume that the integer is short, since we are not going to wrap + # around a 63-bit integer. + # NOTE: This would be questionable if short ints could be 32 bits. + new_val = builder.int_op( + short_int_rprimitive, builder.read(self.index_reg, line), Integer(1), IntOp.ADD, line + ) + builder.assign(self.index_reg, new_val, line) + + def begin_body(self) -> None: + self.builder.assign(self.index_target, self.builder.read(self.index_reg), self.line) + + +class ForEnumerate(ForGenerator): + """Generate optimized IR for a for loop of form "for i, x in enumerate(it)".""" + + def need_cleanup(self) -> bool: + # The wrapped for loop might need cleanup. This might generate a + # redundant cleanup block, but that's okay. + return True + + def init(self, index1: Lvalue, index2: Lvalue, expr: Expression) -> None: + # Count from 0 to infinity (for the index lvalue). + self.index_gen = ForInfiniteCounter( + self.builder, index1, self.body_block, self.loop_exit, self.line, nested=True + ) + self.index_gen.init() + # Iterate over the actual iterable. + self.main_gen = make_for_loop_generator( + self.builder, index2, expr, self.body_block, self.loop_exit, self.line, nested=True + ) + + def gen_condition(self) -> None: + # No need for a check for the index generator, since it's unconditional. + self.main_gen.gen_condition() + + def begin_body(self) -> None: + self.index_gen.begin_body() + self.main_gen.begin_body() + + def gen_step(self) -> None: + self.index_gen.gen_step() + self.main_gen.gen_step() + + def gen_cleanup(self) -> None: + self.index_gen.gen_cleanup() + self.main_gen.gen_cleanup() + + +class ForZip(ForGenerator): + """Generate IR for a for loop of form `for x, ... in zip(a, ...)`.""" + + def need_cleanup(self) -> bool: + # The wrapped for loops might need cleanup. We might generate a + # redundant cleanup block, but that's okay. + return True + + def init(self, indexes: list[Lvalue], exprs: list[Expression]) -> None: + assert len(indexes) == len(exprs) + # Condition check will require multiple basic blocks, since there will be + # multiple conditions to check. + self.cond_blocks = [BasicBlock() for _ in range(len(indexes) - 1)] + [self.body_block] + self.gens: list[ForGenerator] = [] + for index, expr, next_block in zip(indexes, exprs, self.cond_blocks): + gen = make_for_loop_generator( + self.builder, index, expr, next_block, self.loop_exit, self.line, nested=True + ) + self.gens.append(gen) + + def gen_condition(self) -> None: + for i, gen in enumerate(self.gens): + gen.gen_condition() + if i < len(self.gens) - 1: + self.builder.activate_block(self.cond_blocks[i]) + + def begin_body(self) -> None: + for gen in self.gens: + gen.begin_body() + + def gen_step(self) -> None: + for gen in self.gens: + gen.gen_step() + + def gen_cleanup(self) -> None: + for gen in self.gens: + gen.gen_cleanup() diff --git a/mypyc/irbuild/format_str_tokenizer.py b/mypyc/irbuild/format_str_tokenizer.py new file mode 100644 index 000000000000..eaa4027ed768 --- /dev/null +++ b/mypyc/irbuild/format_str_tokenizer.py @@ -0,0 +1,250 @@ +"""Tokenizers for three string formatting methods""" + +from __future__ import annotations + +from enum import Enum, unique +from typing import Final + +from mypy.checkstrformat import ( + ConversionSpecifier, + parse_conversion_specifiers, + parse_format_value, +) +from mypy.errors import Errors +from mypy.messages import MessageBuilder +from mypy.nodes import Context, Expression +from mypy.options import Options +from mypyc.ir.ops import Integer, Value +from mypyc.ir.rtypes import ( + c_pyssize_t_rprimitive, + is_bytes_rprimitive, + is_int_rprimitive, + is_short_int_rprimitive, + is_str_rprimitive, +) +from mypyc.irbuild.builder import IRBuilder +from mypyc.primitives.bytes_ops import bytes_build_op +from mypyc.primitives.int_ops import int_to_str_op +from mypyc.primitives.str_ops import str_build_op, str_op + + +@unique +class FormatOp(Enum): + """FormatOp represents conversion operations of string formatting during + compile time. + + Compare to ConversionSpecifier, FormatOp has fewer attributes. + For example, to mark a conversion from any object to string, + ConversionSpecifier may have several representations, like '%s', '{}' + or '{:{}}'. However, there would only exist one corresponding FormatOp. + """ + + STR = "s" + INT = "d" + BYTES = "b" + + +def generate_format_ops(specifiers: list[ConversionSpecifier]) -> list[FormatOp] | None: + """Convert ConversionSpecifier to FormatOp. + + Different ConversionSpecifiers may share a same FormatOp. + """ + format_ops = [] + for spec in specifiers: + # TODO: Match specifiers instead of using whole_seq + if spec.whole_seq == "%s" or spec.whole_seq == "{:{}}": + format_op = FormatOp.STR + elif spec.whole_seq == "%d": + format_op = FormatOp.INT + elif spec.whole_seq == "%b": + format_op = FormatOp.BYTES + elif spec.whole_seq: + return None + else: + format_op = FormatOp.STR + format_ops.append(format_op) + return format_ops + + +def tokenizer_printf_style(format_str: str) -> tuple[list[str], list[FormatOp]] | None: + """Tokenize a printf-style format string using regex. + + Return: + A list of string literals and a list of FormatOps. + """ + literals: list[str] = [] + specifiers: list[ConversionSpecifier] = parse_conversion_specifiers(format_str) + format_ops = generate_format_ops(specifiers) + if format_ops is None: + return None + + last_end = 0 + for spec in specifiers: + cur_start = spec.start_pos + literals.append(format_str[last_end:cur_start]) + last_end = cur_start + len(spec.whole_seq) + literals.append(format_str[last_end:]) + + return literals, format_ops + + +# The empty Context as an argument for parse_format_value(). +# It wouldn't be used since the code has passed the type-checking. +EMPTY_CONTEXT: Final = Context() + + +def tokenizer_format_call(format_str: str) -> tuple[list[str], list[FormatOp]] | None: + """Tokenize a str.format() format string. + + The core function parse_format_value() is shared with mypy. + With these specifiers, we then parse the literal substrings + of the original format string and convert `ConversionSpecifier` + to `FormatOp`. + + Return: + A list of string literals and a list of FormatOps. The literals + are interleaved with FormatOps and the length of returned literals + should be exactly one more than FormatOps. + Return None if it cannot parse the string. + """ + # Creates an empty MessageBuilder here. + # It wouldn't be used since the code has passed the type-checking. + specifiers = parse_format_value( + format_str, EMPTY_CONTEXT, MessageBuilder(Errors(Options()), {}) + ) + if specifiers is None: + return None + format_ops = generate_format_ops(specifiers) + if format_ops is None: + return None + + literals: list[str] = [] + last_end = 0 + for spec in specifiers: + # Skip { and } + literals.append(format_str[last_end : spec.start_pos - 1]) + last_end = spec.start_pos + len(spec.whole_seq) + 1 + literals.append(format_str[last_end:]) + # Deal with escaped {{ + literals = [x.replace("{{", "{").replace("}}", "}") for x in literals] + + return literals, format_ops + + +def convert_format_expr_to_str( + builder: IRBuilder, format_ops: list[FormatOp], exprs: list[Expression], line: int +) -> list[Value] | None: + """Convert expressions into string literal objects with the guidance + of FormatOps. Return None when fails.""" + if len(format_ops) != len(exprs): + return None + + converted = [] + for x, format_op in zip(exprs, format_ops): + node_type = builder.node_type(x) + if format_op == FormatOp.STR: + if is_str_rprimitive(node_type): + var_str = builder.accept(x) + elif is_int_rprimitive(node_type) or is_short_int_rprimitive(node_type): + var_str = builder.primitive_op(int_to_str_op, [builder.accept(x)], line) + else: + var_str = builder.primitive_op(str_op, [builder.accept(x)], line) + elif format_op == FormatOp.INT: + if is_int_rprimitive(node_type) or is_short_int_rprimitive(node_type): + var_str = builder.primitive_op(int_to_str_op, [builder.accept(x)], line) + else: + return None + else: + return None + converted.append(var_str) + return converted + + +def join_formatted_strings( + builder: IRBuilder, literals: list[str] | None, substitutions: list[Value], line: int +) -> Value: + """Merge the list of literals and the list of substitutions + alternatively using 'str_build_op'. + + `substitutions` is the result value of formatting conversions. + + If the `literals` is set to None, we simply join the substitutions; + Otherwise, the `literals` is the literal substrings of the original + format string and its length should be exactly one more than + substitutions. + + For example: + (1) 'This is a %s and the value is %d' + -> literals: ['This is a ', ' and the value is', ''] + (2) '{} and the value is {}' + -> literals: ['', ' and the value is', ''] + """ + # The first parameter for str_build_op is the total size of + # the following PyObject* + result_list: list[Value] = [Integer(0, c_pyssize_t_rprimitive)] + + if literals is not None: + for a, b in zip(literals, substitutions): + if a: + result_list.append(builder.load_str(a)) + result_list.append(b) + if literals[-1]: + result_list.append(builder.load_str(literals[-1])) + else: + result_list.extend(substitutions) + + # Special case for empty string and literal string + if len(result_list) == 1: + return builder.load_str("") + if not substitutions and len(result_list) == 2: + return result_list[1] + + result_list[0] = Integer(len(result_list) - 1, c_pyssize_t_rprimitive) + return builder.call_c(str_build_op, result_list, line) + + +def convert_format_expr_to_bytes( + builder: IRBuilder, format_ops: list[FormatOp], exprs: list[Expression], line: int +) -> list[Value] | None: + """Convert expressions into bytes literal objects with the guidance + of FormatOps. Return None when fails.""" + if len(format_ops) != len(exprs): + return None + + converted = [] + for x, format_op in zip(exprs, format_ops): + node_type = builder.node_type(x) + # conversion type 's' is an alias of 'b' in bytes formatting + if format_op == FormatOp.BYTES or format_op == FormatOp.STR: + if is_bytes_rprimitive(node_type): + var_bytes = builder.accept(x) + else: + return None + else: + return None + converted.append(var_bytes) + return converted + + +def join_formatted_bytes( + builder: IRBuilder, literals: list[str], substitutions: list[Value], line: int +) -> Value: + """Merge the list of literals and the list of substitutions + alternatively using 'bytes_build_op'.""" + result_list: list[Value] = [Integer(0, c_pyssize_t_rprimitive)] + + for a, b in zip(literals, substitutions): + if a: + result_list.append(builder.load_bytes_from_str_literal(a)) + result_list.append(b) + if literals[-1]: + result_list.append(builder.load_bytes_from_str_literal(literals[-1])) + + # Special case for empty bytes and literal + if len(result_list) == 1: + return builder.load_bytes_from_str_literal("") + if not substitutions and len(result_list) == 2: + return result_list[1] + + result_list[0] = Integer(len(result_list) - 1, c_pyssize_t_rprimitive) + return builder.call_c(bytes_build_op, result_list, line) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py new file mode 100644 index 000000000000..a84db5a08863 --- /dev/null +++ b/mypyc/irbuild/function.py @@ -0,0 +1,1093 @@ +"""Transform mypy AST functions to IR (and related things). + +Normal functions are translated into a list of basic blocks +containing various IR ops (defined in mypyc.ir.ops). + +This also deals with generators, async functions and nested +functions. All of these are transformed into callable classes. These +have a custom __call__ method that implements the call, and state, such +as an environment containing non-local variables, is stored in the +instance of the callable class. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import NamedTuple, Sequence + +from mypy.nodes import ( + ArgKind, + ClassDef, + Decorator, + FuncBase, + FuncDef, + FuncItem, + LambdaExpr, + OverloadedFuncDef, + SymbolNode, + TypeInfo, + Var, +) +from mypy.types import CallableType, get_proper_type +from mypyc.common import LAMBDA_NAME, PROPSET_PREFIX, SELF_NAME +from mypyc.ir.class_ir import ClassIR, NonExtClassInfo +from mypyc.ir.func_ir import ( + FUNC_CLASSMETHOD, + FUNC_NORMAL, + FUNC_STATICMETHOD, + FuncDecl, + FuncIR, + FuncSignature, + RuntimeArg, +) +from mypyc.ir.ops import ( + BasicBlock, + GetAttr, + InitStatic, + Integer, + LoadAddress, + LoadLiteral, + Register, + Return, + SetAttr, + Unbox, + Unreachable, + Value, +) +from mypyc.ir.rtypes import ( + RInstance, + bool_rprimitive, + dict_rprimitive, + int_rprimitive, + object_rprimitive, +) +from mypyc.irbuild.builder import IRBuilder, SymbolTarget, gen_arg_defaults +from mypyc.irbuild.callable_class import ( + add_call_to_callable_class, + add_get_to_callable_class, + instantiate_callable_class, + setup_callable_class, +) +from mypyc.irbuild.context import FuncInfo, ImplicitClass +from mypyc.irbuild.env_class import ( + finalize_env_class, + load_env_registers, + load_outer_envs, + setup_env_class, + setup_func_for_recursive_call, +) +from mypyc.irbuild.generator import ( + add_methods_to_generator_class, + add_raise_exception_blocks_to_generator_class, + create_switch_for_generator_class, + gen_generator_func, + populate_switch_for_generator_class, + setup_env_for_generator_class, +) +from mypyc.irbuild.targets import AssignmentTarget +from mypyc.irbuild.util import is_constant +from mypyc.primitives.dict_ops import dict_get_method_with_none, dict_new_op, dict_set_item_op +from mypyc.primitives.generic_ops import py_setattr_op +from mypyc.primitives.misc_ops import register_function +from mypyc.primitives.registry import builtin_names +from mypyc.sametype import is_same_method_signature, is_same_type + +# Top-level transform functions + + +def transform_func_def(builder: IRBuilder, fdef: FuncDef) -> None: + sig = builder.mapper.fdef_to_sig(fdef, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, fdef, fdef.name, sig) + + # If the function that was visited was a nested function, then either look it up in our + # current environment or define it if it was not already defined. + if func_reg: + builder.assign(get_func_target(builder, fdef), func_reg, fdef.line) + maybe_insert_into_registry_dict(builder, fdef) + builder.add_function(func_ir, fdef.line) + + +def transform_overloaded_func_def(builder: IRBuilder, o: OverloadedFuncDef) -> None: + # Handle regular overload case + assert o.impl + builder.accept(o.impl) + + +def transform_decorator(builder: IRBuilder, dec: Decorator) -> None: + sig = builder.mapper.fdef_to_sig(dec.func, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, dec.func, dec.func.name, sig) + decorated_func: Value | None = None + if func_reg: + decorated_func = load_decorated_func(builder, dec.func, func_reg) + builder.assign(get_func_target(builder, dec.func), decorated_func, dec.func.line) + # If the prebuild pass didn't put this function in the function to decorators map (for example + # if this is a registered singledispatch implementation with no other decorators), we should + # treat this function as a regular function, not a decorated function + elif dec.func in builder.fdefs_to_decorators: + # Obtain the function name in order to construct the name of the helper function. + name = dec.func.fullname.split(".")[-1] + + # Load the callable object representing the non-decorated function, and decorate it. + orig_func = builder.load_global_str(name, dec.line) + decorated_func = load_decorated_func(builder, dec.func, orig_func) + + if decorated_func is not None: + # Set the callable object representing the decorated function as a global. + builder.primitive_op( + dict_set_item_op, + [builder.load_globals_dict(), builder.load_str(dec.func.name), decorated_func], + decorated_func.line, + ) + + maybe_insert_into_registry_dict(builder, dec.func) + + builder.functions.append(func_ir) + + +def transform_lambda_expr(builder: IRBuilder, expr: LambdaExpr) -> Value: + typ = get_proper_type(builder.types[expr]) + assert isinstance(typ, CallableType) + + runtime_args = [] + for arg, arg_type in zip(expr.arguments, typ.arg_types): + arg.variable.type = arg_type + runtime_args.append( + RuntimeArg(arg.variable.name, builder.type_to_rtype(arg_type), arg.kind) + ) + ret_type = builder.type_to_rtype(typ.ret_type) + + fsig = FuncSignature(runtime_args, ret_type) + + fname = f"{LAMBDA_NAME}{builder.lambda_counter}" + builder.lambda_counter += 1 + func_ir, func_reg = gen_func_item(builder, expr, fname, fsig) + assert func_reg is not None + + builder.functions.append(func_ir) + return func_reg + + +# Internal functions + + +def gen_func_item( + builder: IRBuilder, + fitem: FuncItem, + name: str, + sig: FuncSignature, + cdef: ClassDef | None = None, +) -> tuple[FuncIR, Value | None]: + """Generate and return the FuncIR for a given FuncDef. + + If the given FuncItem is a nested function, then we generate a + callable class representing the function and use that instead of + the actual function. if the given FuncItem contains a nested + function, then we generate an environment class so that inner + nested functions can access the environment of the given FuncDef. + + Consider the following nested function: + + def a() -> None: + def b() -> None: + def c() -> None: + return None + return None + return None + + The classes generated would look something like the following. + + has pointer to +-------+ + +--------------------------> | a_env | + | +-------+ + | ^ + | | has pointer to + +-------+ associated with +-------+ + | b_obj | -------------------> | b_env | + +-------+ +-------+ + ^ + | + +-------+ has pointer to | + | c_obj | --------------------------+ + +-------+ + """ + + # TODO: do something about abstract methods. + + func_reg: Value | None = None + + # We treat lambdas as always being nested because we always generate + # a class for lambdas, no matter where they are. (It would probably also + # work to special case toplevel lambdas and generate a non-class function.) + is_nested = fitem in builder.nested_fitems or isinstance(fitem, LambdaExpr) + contains_nested = fitem in builder.encapsulating_funcs.keys() + is_decorated = fitem in builder.fdefs_to_decorators + is_singledispatch = fitem in builder.singledispatch_impls + in_non_ext = False + add_nested_funcs_to_env = has_nested_func_self_reference(builder, fitem) + class_name = None + if cdef: + ir = builder.mapper.type_to_ir[cdef.info] + in_non_ext = not ir.is_ext_class + class_name = cdef.name + + if is_singledispatch: + func_name = singledispatch_main_func_name(name) + else: + func_name = name + builder.enter( + FuncInfo( + fitem=fitem, + name=func_name, + class_name=class_name, + namespace=gen_func_ns(builder), + is_nested=is_nested, + contains_nested=contains_nested, + is_decorated=is_decorated, + in_non_ext=in_non_ext, + add_nested_funcs_to_env=add_nested_funcs_to_env, + ) + ) + + # Functions that contain nested functions need an environment class to store variables that + # are free in their nested functions. Generator functions need an environment class to + # store a variable denoting the next instruction to be executed when the __next__ function + # is called, along with all the variables inside the function itself. + if builder.fn_info.contains_nested or builder.fn_info.is_generator: + setup_env_class(builder) + + if builder.fn_info.is_nested or builder.fn_info.in_non_ext: + setup_callable_class(builder) + + if builder.fn_info.is_generator: + # Do a first-pass and generate a function that just returns a generator object. + gen_generator_func(builder) + args, _, blocks, ret_type, fn_info = builder.leave() + func_ir, func_reg = gen_func_ir( + builder, args, blocks, sig, fn_info, cdef, is_singledispatch + ) + + # Re-enter the FuncItem and visit the body of the function this time. + builder.enter(fn_info) + setup_env_for_generator_class(builder) + load_outer_envs(builder, builder.fn_info.generator_class) + top_level = builder.top_level_fn_info() + if ( + builder.fn_info.is_nested + and isinstance(fitem, FuncDef) + and top_level + and top_level.add_nested_funcs_to_env + ): + setup_func_for_recursive_call(builder, fitem, builder.fn_info.generator_class) + create_switch_for_generator_class(builder) + add_raise_exception_blocks_to_generator_class(builder, fitem.line) + else: + load_env_registers(builder) + gen_arg_defaults(builder) + + if builder.fn_info.contains_nested and not builder.fn_info.is_generator: + finalize_env_class(builder) + + builder.ret_types[-1] = sig.ret_type + + # Add all variables and functions that are declared/defined within this + # function and are referenced in functions nested within this one to this + # function's environment class so the nested functions can reference + # them even if they are declared after the nested function's definition. + # Note that this is done before visiting the body of this function. + + env_for_func: FuncInfo | ImplicitClass = builder.fn_info + if builder.fn_info.is_generator: + env_for_func = builder.fn_info.generator_class + elif builder.fn_info.is_nested or builder.fn_info.in_non_ext: + env_for_func = builder.fn_info.callable_class + + if builder.fn_info.fitem in builder.free_variables: + # Sort the variables to keep things deterministic + for var in sorted(builder.free_variables[builder.fn_info.fitem], key=lambda x: x.name): + if isinstance(var, Var): + rtype = builder.type_to_rtype(var.type) + builder.add_var_to_env_class(var, rtype, env_for_func, reassign=False) + + if builder.fn_info.fitem in builder.encapsulating_funcs: + for nested_fn in builder.encapsulating_funcs[builder.fn_info.fitem]: + if isinstance(nested_fn, FuncDef): + # The return type is 'object' instead of an RInstance of the + # callable class because differently defined functions with + # the same name and signature across conditional blocks + # will generate different callable classes, so the callable + # class that gets instantiated must be generic. + builder.add_var_to_env_class( + nested_fn, object_rprimitive, env_for_func, reassign=False + ) + + builder.accept(fitem.body) + builder.maybe_add_implicit_return() + + if builder.fn_info.is_generator: + populate_switch_for_generator_class(builder) + + # Hang on to the local symbol table for a while, since we use it + # to calculate argument defaults below. + symtable = builder.symtables[-1] + + args, _, blocks, ret_type, fn_info = builder.leave() + + if fn_info.is_generator: + add_methods_to_generator_class(builder, fn_info, sig, args, blocks, fitem.is_coroutine) + else: + func_ir, func_reg = gen_func_ir( + builder, args, blocks, sig, fn_info, cdef, is_singledispatch + ) + + # Evaluate argument defaults in the surrounding scope, since we + # calculate them *once* when the function definition is evaluated. + calculate_arg_defaults(builder, fn_info, func_reg, symtable) + + if is_singledispatch: + # add the generated main singledispatch function + builder.functions.append(func_ir) + # create the dispatch function + assert isinstance(fitem, FuncDef) + return gen_dispatch_func_ir(builder, fitem, fn_info.name, name, sig) + + return func_ir, func_reg + + +def has_nested_func_self_reference(builder: IRBuilder, fitem: FuncItem) -> bool: + """Does a nested function contain a self-reference in its body? + + If a nested function only has references in the surrounding function, + we don't need to add it to the environment. + """ + if any(isinstance(sym, FuncBase) for sym in builder.free_variables.get(fitem, set())): + return True + return any( + has_nested_func_self_reference(builder, nested) + for nested in builder.encapsulating_funcs.get(fitem, []) + ) + + +def gen_func_ir( + builder: IRBuilder, + args: list[Register], + blocks: list[BasicBlock], + sig: FuncSignature, + fn_info: FuncInfo, + cdef: ClassDef | None, + is_singledispatch_main_func: bool = False, +) -> tuple[FuncIR, Value | None]: + """Generate the FuncIR for a function. + + This takes the basic blocks and function info of a particular + function and returns the IR. If the function is nested, + also returns the register containing the instance of the + corresponding callable class. + """ + func_reg: Value | None = None + if fn_info.is_nested or fn_info.in_non_ext: + func_ir = add_call_to_callable_class(builder, args, blocks, sig, fn_info) + add_get_to_callable_class(builder, fn_info) + func_reg = instantiate_callable_class(builder, fn_info) + else: + assert isinstance(fn_info.fitem, FuncDef) + func_decl = builder.mapper.func_to_decl[fn_info.fitem] + if fn_info.is_decorated or is_singledispatch_main_func: + class_name = None if cdef is None else cdef.name + func_decl = FuncDecl( + fn_info.name, + class_name, + builder.module_name, + sig, + func_decl.kind, + func_decl.is_prop_getter, + func_decl.is_prop_setter, + ) + func_ir = FuncIR( + func_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name + ) + else: + func_ir = FuncIR( + func_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name + ) + return (func_ir, func_reg) + + +def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None: + # Perform the function of visit_method for methods inside extension classes. + name = fdef.name + class_ir = builder.mapper.type_to_ir[cdef.info] + sig = builder.mapper.fdef_to_sig(fdef, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, fdef, name, sig, cdef) + builder.functions.append(func_ir) + + if is_decorated(builder, fdef): + # Obtain the function name in order to construct the name of the helper function. + _, _, name = fdef.fullname.rpartition(".") + # Read the PyTypeObject representing the class, get the callable object + # representing the non-decorated method + typ = builder.load_native_type_object(cdef.fullname) + orig_func = builder.py_get_attr(typ, name, fdef.line) + + # Decorate the non-decorated method + decorated_func = load_decorated_func(builder, fdef, orig_func) + + # Set the callable object representing the decorated method as an attribute of the + # extension class. + builder.primitive_op( + py_setattr_op, [typ, builder.load_str(name), decorated_func], fdef.line + ) + + if fdef.is_property: + # If there is a property setter, it will be processed after the getter, + # We populate the optional setter field with none for now. + assert name not in class_ir.properties + class_ir.properties[name] = (func_ir, None) + + elif fdef in builder.prop_setters: + # The respective property getter must have been processed already + assert name in class_ir.properties + getter_ir, _ = class_ir.properties[name] + class_ir.properties[name] = (getter_ir, func_ir) + + class_ir.methods[func_ir.decl.name] = func_ir + + # If this overrides a parent class method with a different type, we need + # to generate a glue method to mediate between them. + for base in class_ir.mro[1:]: + if ( + name in base.method_decls + and name != "__init__" + and not is_same_method_signature( + class_ir.method_decls[name].sig, base.method_decls[name].sig + ) + ): + # TODO: Support contravariant subtyping in the input argument for + # property setters. Need to make a special glue method for handling this, + # similar to gen_glue_property. + + f = gen_glue(builder, base.method_decls[name].sig, func_ir, class_ir, base, fdef) + class_ir.glue_methods[(base, name)] = f + builder.functions.append(f) + + # If the class allows interpreted children, create glue + # methods that dispatch via the Python API. These will go in a + # "shadow vtable" that will be assigned to interpreted + # children. + if class_ir.allow_interpreted_subclasses: + f = gen_glue(builder, func_ir.sig, func_ir, class_ir, class_ir, fdef, do_py_ops=True) + class_ir.glue_methods[(class_ir, name)] = f + builder.functions.append(f) + + +def handle_non_ext_method( + builder: IRBuilder, non_ext: NonExtClassInfo, cdef: ClassDef, fdef: FuncDef +) -> None: + # Perform the function of visit_method for methods inside non-extension classes. + name = fdef.name + sig = builder.mapper.fdef_to_sig(fdef, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, fdef, name, sig, cdef) + assert func_reg is not None + builder.functions.append(func_ir) + + if is_decorated(builder, fdef): + # The undecorated method is a generated callable class + orig_func = func_reg + func_reg = load_decorated_func(builder, fdef, orig_func) + + # TODO: Support property setters in non-extension classes + if fdef.is_property: + prop = builder.load_module_attr_by_fullname("builtins.property", fdef.line) + func_reg = builder.py_call(prop, [func_reg], fdef.line) + + elif builder.mapper.func_to_decl[fdef].kind == FUNC_CLASSMETHOD: + cls_meth = builder.load_module_attr_by_fullname("builtins.classmethod", fdef.line) + func_reg = builder.py_call(cls_meth, [func_reg], fdef.line) + + elif builder.mapper.func_to_decl[fdef].kind == FUNC_STATICMETHOD: + stat_meth = builder.load_module_attr_by_fullname("builtins.staticmethod", fdef.line) + func_reg = builder.py_call(stat_meth, [func_reg], fdef.line) + + builder.add_to_non_ext_dict(non_ext, name, func_reg, fdef.line) + + +def calculate_arg_defaults( + builder: IRBuilder, + fn_info: FuncInfo, + func_reg: Value | None, + symtable: dict[SymbolNode, SymbolTarget], +) -> None: + """Calculate default argument values and store them. + + They are stored in statics for top level functions and in + the function objects for nested functions (while constants are + still stored computed on demand). + """ + fitem = fn_info.fitem + for arg in fitem.arguments: + # Constant values don't get stored but just recomputed + if arg.initializer and not is_constant(arg.initializer): + value = builder.coerce( + builder.accept(arg.initializer), symtable[arg.variable].type, arg.line + ) + if not fn_info.is_nested: + name = fitem.fullname + "." + arg.variable.name + builder.add(InitStatic(value, name, builder.module_name)) + else: + assert func_reg is not None + builder.add(SetAttr(func_reg, arg.variable.name, value, arg.line)) + + +def gen_func_ns(builder: IRBuilder) -> str: + """Generate a namespace for a nested function using its outer function names.""" + return "_".join( + info.name + ("" if not info.class_name else "_" + info.class_name) + for info in builder.fn_infos + if info.name and info.name != "" + ) + + +def load_decorated_func(builder: IRBuilder, fdef: FuncDef, orig_func_reg: Value) -> Value: + """Apply decorators to a function. + + Given a decorated FuncDef and an instance of the callable class + representing that FuncDef, apply the corresponding decorator + functions on that decorated FuncDef and return the decorated + function. + """ + if not is_decorated(builder, fdef): + # If there are no decorators associated with the function, then just return the + # original function. + return orig_func_reg + + decorators = builder.fdefs_to_decorators[fdef] + func_reg = orig_func_reg + for d in reversed(decorators): + decorator = d.accept(builder.visitor) + assert isinstance(decorator, Value) + func_reg = builder.py_call(decorator, [func_reg], func_reg.line) + return func_reg + + +def is_decorated(builder: IRBuilder, fdef: FuncDef) -> bool: + return fdef in builder.fdefs_to_decorators + + +def gen_glue( + builder: IRBuilder, + base_sig: FuncSignature, + target: FuncIR, + cls: ClassIR, + base: ClassIR, + fdef: FuncItem, + *, + do_py_ops: bool = False, +) -> FuncIR: + """Generate glue methods that mediate between different method types in subclasses. + + Works on both properties and methods. See gen_glue_methods below + for more details. + + If do_py_ops is True, then the glue methods should use generic + C API operations instead of direct calls, to enable generating + "shadow" glue methods that work with interpreted subclasses. + """ + if fdef.is_property: + return gen_glue_property(builder, base_sig, target, cls, base, fdef.line, do_py_ops) + else: + return gen_glue_method(builder, base_sig, target, cls, base, fdef.line, do_py_ops) + + +class ArgInfo(NamedTuple): + args: list[Value] + arg_names: list[str | None] + arg_kinds: list[ArgKind] + + +def get_args(builder: IRBuilder, rt_args: Sequence[RuntimeArg], line: int) -> ArgInfo: + # The environment operates on Vars, so we make some up + fake_vars = [(Var(arg.name), arg.type) for arg in rt_args] + args = [ + builder.read(builder.add_local_reg(var, type, is_arg=True), line) + for var, type in fake_vars + ] + arg_names = [ + arg.name if arg.kind.is_named() or (arg.kind.is_optional() and not arg.pos_only) else None + for arg in rt_args + ] + arg_kinds = [arg.kind for arg in rt_args] + return ArgInfo(args, arg_names, arg_kinds) + + +def gen_glue_method( + builder: IRBuilder, + base_sig: FuncSignature, + target: FuncIR, + cls: ClassIR, + base: ClassIR, + line: int, + do_pycall: bool, +) -> FuncIR: + """Generate glue methods that mediate between different method types in subclasses. + + For example, if we have: + + class A: + def f(builder: IRBuilder, x: int) -> object: ... + + then it is totally permissible to have a subclass + + class B(A): + def f(builder: IRBuilder, x: object) -> int: ... + + since '(object) -> int' is a subtype of '(int) -> object' by the usual + contra/co-variant function subtyping rules. + + The trickiness here is that int and object have different + runtime representations in mypyc, so A.f and B.f have + different signatures at the native C level. To deal with this, + we need to generate glue methods that mediate between the + different versions by coercing the arguments and return + values. + + If do_pycall is True, then make the call using the C API + instead of a native call. + """ + check_native_override(builder, base_sig, target.decl.sig, line) + + builder.enter() + builder.ret_types[-1] = base_sig.ret_type + + rt_args = list(base_sig.args) + if target.decl.kind == FUNC_NORMAL: + rt_args[0] = RuntimeArg(base_sig.args[0].name, RInstance(cls)) + + arg_info = get_args(builder, rt_args, line) + args, arg_kinds, arg_names = arg_info.args, arg_info.arg_kinds, arg_info.arg_names + + bitmap_args = None + if base_sig.num_bitmap_args: + args = args[: -base_sig.num_bitmap_args] + arg_kinds = arg_kinds[: -base_sig.num_bitmap_args] + arg_names = arg_names[: -base_sig.num_bitmap_args] + bitmap_args = list(builder.builder.args[-base_sig.num_bitmap_args :]) + + # We can do a passthrough *args/**kwargs with a native call, but if the + # args need to get distributed out to arguments, we just let python handle it + if any(kind.is_star() for kind in arg_kinds) and any( + not arg.kind.is_star() for arg in target.decl.sig.args + ): + do_pycall = True + + if do_pycall: + if target.decl.kind == FUNC_STATICMETHOD: + # FIXME: this won't work if we can do interpreted subclasses + first = builder.builder.get_native_type(cls) + st = 0 + else: + first = args[0] + st = 1 + retval = builder.builder.py_method_call( + first, target.name, args[st:], line, arg_kinds[st:], arg_names[st:] + ) + else: + retval = builder.builder.call( + target.decl, args, arg_kinds, arg_names, line, bitmap_args=bitmap_args + ) + retval = builder.coerce(retval, base_sig.ret_type, line) + builder.add(Return(retval)) + + arg_regs, _, blocks, ret_type, _ = builder.leave() + if base_sig.num_bitmap_args: + rt_args = rt_args[: -base_sig.num_bitmap_args] + return FuncIR( + FuncDecl( + target.name + "__" + base.name + "_glue", + cls.name, + builder.module_name, + FuncSignature(rt_args, ret_type), + target.decl.kind, + ), + arg_regs, + blocks, + ) + + +def check_native_override( + builder: IRBuilder, base_sig: FuncSignature, sub_sig: FuncSignature, line: int +) -> None: + """Report an error if an override changes signature in unsupported ways. + + Glue methods can work around many signature changes but not all of them. + """ + for base_arg, sub_arg in zip(base_sig.real_args(), sub_sig.real_args()): + if base_arg.type.error_overlap: + if not base_arg.optional and sub_arg.optional and base_sig.num_bitmap_args: + # This would change the meanings of bits in the argument defaults + # bitmap, which we don't support. We'd need to do tricky bit + # manipulations to support this generally. + builder.error( + "An argument with type " + + f'"{base_arg.type}" cannot be given a default value in a method override', + line, + ) + if base_arg.type.error_overlap or sub_arg.type.error_overlap: + if not is_same_type(base_arg.type, sub_arg.type): + # This would change from signaling a default via an error value to + # signaling a default via bitmap, which we don't support. + builder.error( + "Incompatible argument type " + + f'"{sub_arg.type}" (base class has type "{base_arg.type}")', + line, + ) + + +def gen_glue_property( + builder: IRBuilder, + sig: FuncSignature, + target: FuncIR, + cls: ClassIR, + base: ClassIR, + line: int, + do_pygetattr: bool, +) -> FuncIR: + """Generate glue methods for properties that mediate between different subclass types. + + Similarly to methods, properties of derived types can be covariantly subtyped. Thus, + properties also require glue. However, this only requires the return type to change. + Further, instead of a method call, an attribute get is performed. + + If do_pygetattr is True, then get the attribute using the Python C + API instead of a native call. + """ + builder.enter() + + rt_arg = RuntimeArg(SELF_NAME, RInstance(cls)) + self_target = builder.add_self_to_env(cls) + arg = builder.read(self_target, line) + builder.ret_types[-1] = sig.ret_type + if do_pygetattr: + retval = builder.py_get_attr(arg, target.name, line) + else: + retval = builder.add(GetAttr(arg, target.name, line)) + retbox = builder.coerce(retval, sig.ret_type, line) + builder.add(Return(retbox)) + + args, _, blocks, return_type, _ = builder.leave() + return FuncIR( + FuncDecl( + target.name + "__" + base.name + "_glue", + cls.name, + builder.module_name, + FuncSignature([rt_arg], return_type), + ), + args, + blocks, + ) + + +def get_func_target(builder: IRBuilder, fdef: FuncDef) -> AssignmentTarget: + """Given a FuncDef, return the target for the instance of its callable class. + + If the function was not already defined somewhere, then define it + and add it to the current environment. + """ + if fdef.original_def: + # Get the target associated with the previously defined FuncDef. + return builder.lookup(fdef.original_def) + + if builder.fn_info.is_generator or builder.fn_info.add_nested_funcs_to_env: + return builder.lookup(fdef) + + return builder.add_local_reg(fdef, object_rprimitive) + + +def load_type(builder: IRBuilder, typ: TypeInfo, line: int) -> Value: + if typ in builder.mapper.type_to_ir: + class_ir = builder.mapper.type_to_ir[typ] + class_obj = builder.builder.get_native_type(class_ir) + elif typ.fullname in builtin_names: + builtin_addr_type, src = builtin_names[typ.fullname] + class_obj = builder.add(LoadAddress(builtin_addr_type, src, line)) + elif typ.module_name in builder.imports: + loaded_module = builder.load_module(typ.module_name) + class_obj = builder.builder.get_attr( + loaded_module, typ.name, object_rprimitive, line, borrow=False + ) + else: + class_obj = builder.load_global_str(typ.name, line) + + return class_obj + + +def load_func(builder: IRBuilder, func_name: str, fullname: str | None, line: int) -> Value: + if fullname and not fullname.startswith(builder.current_module): + # we're calling a function in a different module + + # We can't use load_module_attr_by_fullname here because we need to load the function using + # func_name, not the name specified by fullname (which can be different for underscore + # function) + module = fullname.rsplit(".")[0] + loaded_module = builder.load_module(module) + + func = builder.py_get_attr(loaded_module, func_name, line) + else: + func = builder.load_global_str(func_name, line) + return func + + +def generate_singledispatch_dispatch_function( + builder: IRBuilder, main_singledispatch_function_name: str, fitem: FuncDef +) -> None: + line = fitem.line + current_func_decl = builder.mapper.func_to_decl[fitem] + arg_info = get_args(builder, current_func_decl.sig.args, line) + + dispatch_func_obj = builder.self() + + arg_type = builder.builder.get_type_of_obj(arg_info.args[0], line) + dispatch_cache = builder.builder.get_attr( + dispatch_func_obj, "dispatch_cache", dict_rprimitive, line + ) + call_find_impl, use_cache, call_func = BasicBlock(), BasicBlock(), BasicBlock() + get_result = builder.primitive_op(dict_get_method_with_none, [dispatch_cache, arg_type], line) + is_not_none = builder.translate_is_op(get_result, builder.none_object(), "is not", line) + impl_to_use = Register(object_rprimitive) + builder.add_bool_branch(is_not_none, use_cache, call_find_impl) + + builder.activate_block(use_cache) + builder.assign(impl_to_use, get_result, line) + builder.goto(call_func) + + builder.activate_block(call_find_impl) + find_impl = builder.load_module_attr_by_fullname("functools._find_impl", line) + registry = load_singledispatch_registry(builder, dispatch_func_obj, line) + uncached_impl = builder.py_call(find_impl, [arg_type, registry], line) + builder.primitive_op(dict_set_item_op, [dispatch_cache, arg_type, uncached_impl], line) + builder.assign(impl_to_use, uncached_impl, line) + builder.goto(call_func) + + builder.activate_block(call_func) + gen_calls_to_correct_impl(builder, impl_to_use, arg_info, fitem, line) + + +def gen_calls_to_correct_impl( + builder: IRBuilder, impl_to_use: Value, arg_info: ArgInfo, fitem: FuncDef, line: int +) -> None: + current_func_decl = builder.mapper.func_to_decl[fitem] + + def gen_native_func_call_and_return(fdef: FuncDef) -> None: + func_decl = builder.mapper.func_to_decl[fdef] + ret_val = builder.builder.call( + func_decl, arg_info.args, arg_info.arg_kinds, arg_info.arg_names, line + ) + coerced = builder.coerce(ret_val, current_func_decl.sig.ret_type, line) + builder.add(Return(coerced)) + + typ, src = builtin_names["builtins.int"] + int_type_obj = builder.add(LoadAddress(typ, src, line)) + is_int = builder.builder.type_is_op(impl_to_use, int_type_obj, line) + + native_call, non_native_call = BasicBlock(), BasicBlock() + builder.add_bool_branch(is_int, native_call, non_native_call) + builder.activate_block(native_call) + + passed_id = builder.add(Unbox(impl_to_use, int_rprimitive, line)) + + native_ids = get_native_impl_ids(builder, fitem) + for impl, i in native_ids.items(): + call_impl, next_impl = BasicBlock(), BasicBlock() + + current_id = builder.load_int(i) + cond = builder.binary_op(passed_id, current_id, "==", line) + builder.add_bool_branch(cond, call_impl, next_impl) + + # Call the registered implementation + builder.activate_block(call_impl) + + gen_native_func_call_and_return(impl) + builder.activate_block(next_impl) + + # We've already handled all the possible integer IDs, so we should never get here + builder.add(Unreachable()) + + builder.activate_block(non_native_call) + ret_val = builder.py_call( + impl_to_use, arg_info.args, line, arg_info.arg_kinds, arg_info.arg_names + ) + coerced = builder.coerce(ret_val, current_func_decl.sig.ret_type, line) + builder.add(Return(coerced)) + + +def gen_dispatch_func_ir( + builder: IRBuilder, fitem: FuncDef, main_func_name: str, dispatch_name: str, sig: FuncSignature +) -> tuple[FuncIR, Value]: + """Create a dispatch function (a function that checks the first argument type and dispatches + to the correct implementation) + """ + builder.enter(FuncInfo(fitem, dispatch_name)) + setup_callable_class(builder) + builder.fn_info.callable_class.ir.attributes["registry"] = dict_rprimitive + builder.fn_info.callable_class.ir.attributes["dispatch_cache"] = dict_rprimitive + builder.fn_info.callable_class.ir.has_dict = True + builder.fn_info.callable_class.ir.needs_getseters = True + generate_singledispatch_callable_class_ctor(builder) + + generate_singledispatch_dispatch_function(builder, main_func_name, fitem) + args, _, blocks, _, fn_info = builder.leave() + dispatch_callable_class = add_call_to_callable_class(builder, args, blocks, sig, fn_info) + builder.functions.append(dispatch_callable_class) + add_get_to_callable_class(builder, fn_info) + add_register_method_to_callable_class(builder, fn_info) + func_reg = instantiate_callable_class(builder, fn_info) + dispatch_func_ir = generate_dispatch_glue_native_function( + builder, fitem, dispatch_callable_class.decl, dispatch_name + ) + + return dispatch_func_ir, func_reg + + +def generate_dispatch_glue_native_function( + builder: IRBuilder, fitem: FuncDef, callable_class_decl: FuncDecl, dispatch_name: str +) -> FuncIR: + line = fitem.line + builder.enter() + # We store the callable class in the globals dict for this function + callable_class = builder.load_global_str(dispatch_name, line) + decl = builder.mapper.func_to_decl[fitem] + arg_info = get_args(builder, decl.sig.args, line) + args = [callable_class] + arg_info.args + arg_kinds = [ArgKind.ARG_POS] + arg_info.arg_kinds + arg_names = arg_info.arg_names + arg_names.insert(0, "self") + ret_val = builder.builder.call(callable_class_decl, args, arg_kinds, arg_names, line) + builder.add(Return(ret_val)) + arg_regs, _, blocks, _, fn_info = builder.leave() + return FuncIR(decl, arg_regs, blocks) + + +def generate_singledispatch_callable_class_ctor(builder: IRBuilder) -> None: + """Create an __init__ that sets registry and dispatch_cache to empty dicts""" + line = -1 + class_ir = builder.fn_info.callable_class.ir + with builder.enter_method(class_ir, "__init__", bool_rprimitive): + empty_dict = builder.call_c(dict_new_op, [], line) + builder.add(SetAttr(builder.self(), "registry", empty_dict, line)) + cache_dict = builder.call_c(dict_new_op, [], line) + dispatch_cache_str = builder.load_str("dispatch_cache") + # use the py_setattr_op instead of SetAttr so that it also gets added to our __dict__ + builder.primitive_op(py_setattr_op, [builder.self(), dispatch_cache_str, cache_dict], line) + # the generated C code seems to expect that __init__ returns a char, so just return 1 + builder.add(Return(Integer(1, bool_rprimitive, line), line)) + + +def add_register_method_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None: + line = -1 + with builder.enter_method(fn_info.callable_class.ir, "register", object_rprimitive): + cls_arg = builder.add_argument("cls", object_rprimitive) + func_arg = builder.add_argument("func", object_rprimitive, ArgKind.ARG_OPT) + ret_val = builder.call_c(register_function, [builder.self(), cls_arg, func_arg], line) + builder.add(Return(ret_val, line)) + + +def load_singledispatch_registry(builder: IRBuilder, dispatch_func_obj: Value, line: int) -> Value: + return builder.builder.get_attr(dispatch_func_obj, "registry", dict_rprimitive, line) + + +def singledispatch_main_func_name(orig_name: str) -> str: + return f"__mypyc_singledispatch_main_function_{orig_name}__" + + +def maybe_insert_into_registry_dict(builder: IRBuilder, fitem: FuncDef) -> None: + line = fitem.line + is_singledispatch_main_func = fitem in builder.singledispatch_impls + # dict of singledispatch_func to list of register_types (fitem is the function to register) + to_register: defaultdict[FuncDef, list[TypeInfo]] = defaultdict(list) + for main_func, impls in builder.singledispatch_impls.items(): + for dispatch_type, impl in impls: + if fitem == impl: + to_register[main_func].append(dispatch_type) + + if not to_register and not is_singledispatch_main_func: + return + + if is_singledispatch_main_func: + main_func_name = singledispatch_main_func_name(fitem.name) + main_func_obj = load_func(builder, main_func_name, fitem.fullname, line) + + loaded_object_type = builder.load_module_attr_by_fullname("builtins.object", line) + registry_dict = builder.builder.make_dict([(loaded_object_type, main_func_obj)], line) + + dispatch_func_obj = builder.load_global_str(fitem.name, line) + builder.primitive_op( + py_setattr_op, [dispatch_func_obj, builder.load_str("registry"), registry_dict], line + ) + + for singledispatch_func, types in to_register.items(): + # TODO: avoid recomputing the native IDs for all the functions every time we find a new + # function + native_ids = get_native_impl_ids(builder, singledispatch_func) + if fitem not in native_ids: + to_insert = load_func(builder, fitem.name, fitem.fullname, line) + else: + current_id = native_ids[fitem] + load_literal = LoadLiteral(current_id, object_rprimitive) + to_insert = builder.add(load_literal) + # TODO: avoid reloading the registry here if we just created it + dispatch_func_obj = load_func( + builder, singledispatch_func.name, singledispatch_func.fullname, line + ) + registry = load_singledispatch_registry(builder, dispatch_func_obj, line) + for typ in types: + loaded_type = load_type(builder, typ, line) + builder.primitive_op(dict_set_item_op, [registry, loaded_type, to_insert], line) + dispatch_cache = builder.builder.get_attr( + dispatch_func_obj, "dispatch_cache", dict_rprimitive, line + ) + builder.gen_method_call(dispatch_cache, "clear", [], None, line) + + +def get_native_impl_ids(builder: IRBuilder, singledispatch_func: FuncDef) -> dict[FuncDef, int]: + """Return a dict of registered implementation to native implementation ID for all + implementations + """ + impls = builder.singledispatch_impls[singledispatch_func] + return {impl: i for i, (typ, impl) in enumerate(impls) if not is_decorated(builder, impl)} + + +def gen_property_getter_ir( + builder: IRBuilder, func_decl: FuncDecl, cdef: ClassDef, is_trait: bool +) -> FuncIR: + """Generate an implicit trivial property getter for an attribute. + + These are used if an attribute can also be accessed as a property. + """ + name = func_decl.name + builder.enter(name) + self_reg = builder.add_argument("self", func_decl.sig.args[0].type) + if not is_trait: + value = builder.builder.get_attr(self_reg, name, func_decl.sig.ret_type, -1) + builder.add(Return(value)) + else: + builder.add(Unreachable()) + args, _, blocks, ret_type, fn_info = builder.leave() + return FuncIR(func_decl, args, blocks) + + +def gen_property_setter_ir( + builder: IRBuilder, func_decl: FuncDecl, cdef: ClassDef, is_trait: bool +) -> FuncIR: + """Generate an implicit trivial property setter for an attribute. + + These are used if an attribute can also be accessed as a property. + """ + name = func_decl.name + builder.enter(name) + self_reg = builder.add_argument("self", func_decl.sig.args[0].type) + value_reg = builder.add_argument("value", func_decl.sig.args[1].type) + assert name.startswith(PROPSET_PREFIX) + attr_name = name[len(PROPSET_PREFIX) :] + if not is_trait: + builder.add(SetAttr(self_reg, attr_name, value_reg, -1)) + builder.add(Return(builder.none())) + args, _, blocks, ret_type, fn_info = builder.leave() + return FuncIR(func_decl, args, blocks) diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py new file mode 100644 index 000000000000..92f9abff467c --- /dev/null +++ b/mypyc/irbuild/generator.py @@ -0,0 +1,346 @@ +"""Generate IR for generator functions. + +A generator function is represented by a class that implements the +generator protocol and keeps track of the generator state, including +local variables. + +The top-level logic for dealing with generator functions is in +mypyc.irbuild.function. +""" + +from __future__ import annotations + +from mypy.nodes import ARG_OPT, Var +from mypyc.common import ENV_ATTR_NAME, NEXT_LABEL_ATTR_NAME, SELF_NAME +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg +from mypyc.ir.ops import ( + NO_TRACEBACK_LINE_NO, + BasicBlock, + Branch, + Call, + Goto, + Integer, + MethodCall, + RaiseStandardError, + Register, + Return, + SetAttr, + TupleSet, + Unreachable, + Value, +) +from mypyc.ir.rtypes import RInstance, int_rprimitive, object_rprimitive +from mypyc.irbuild.builder import IRBuilder, gen_arg_defaults +from mypyc.irbuild.context import FuncInfo, GeneratorClass +from mypyc.irbuild.env_class import ( + add_args_to_env, + finalize_env_class, + load_env_registers, + load_outer_env, +) +from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl +from mypyc.primitives.exc_ops import ( + error_catch_op, + exc_matches_op, + raise_exception_with_tb_op, + reraise_exception_op, + restore_exc_info_op, +) + + +def gen_generator_func(builder: IRBuilder) -> None: + setup_generator_class(builder) + load_env_registers(builder) + gen_arg_defaults(builder) + finalize_env_class(builder) + builder.add(Return(instantiate_generator_class(builder))) + + +def instantiate_generator_class(builder: IRBuilder) -> Value: + fitem = builder.fn_info.fitem + generator_reg = builder.add(Call(builder.fn_info.generator_class.ir.ctor, [], fitem.line)) + + # Get the current environment register. If the current function is nested, then the + # generator class gets instantiated from the callable class' '__call__' method, and hence + # we use the callable class' environment register. Otherwise, we use the original + # function's environment register. + if builder.fn_info.is_nested: + curr_env_reg = builder.fn_info.callable_class.curr_env_reg + else: + curr_env_reg = builder.fn_info.curr_env_reg + + # Set the generator class' environment attribute to point at the environment class + # defined in the current scope. + builder.add(SetAttr(generator_reg, ENV_ATTR_NAME, curr_env_reg, fitem.line)) + + # Set the generator class' environment class' NEXT_LABEL_ATTR_NAME attribute to 0. + zero = Integer(0) + builder.add(SetAttr(curr_env_reg, NEXT_LABEL_ATTR_NAME, zero, fitem.line)) + return generator_reg + + +def setup_generator_class(builder: IRBuilder) -> ClassIR: + name = f"{builder.fn_info.namespaced_name()}_gen" + + generator_class_ir = ClassIR(name, builder.module_name, is_generated=True) + generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class) + generator_class_ir.mro = [generator_class_ir] + + builder.classes.append(generator_class_ir) + builder.fn_info.generator_class = GeneratorClass(generator_class_ir) + return generator_class_ir + + +def create_switch_for_generator_class(builder: IRBuilder) -> None: + builder.add(Goto(builder.fn_info.generator_class.switch_block)) + block = BasicBlock() + builder.fn_info.generator_class.continuation_blocks.append(block) + builder.activate_block(block) + + +def populate_switch_for_generator_class(builder: IRBuilder) -> None: + cls = builder.fn_info.generator_class + line = builder.fn_info.fitem.line + + builder.activate_block(cls.switch_block) + for label, true_block in enumerate(cls.continuation_blocks): + false_block = BasicBlock() + comparison = builder.binary_op(cls.next_label_reg, Integer(label), "==", line) + builder.add_bool_branch(comparison, true_block, false_block) + builder.activate_block(false_block) + + builder.add(RaiseStandardError(RaiseStandardError.STOP_ITERATION, None, line)) + builder.add(Unreachable()) + + +def add_raise_exception_blocks_to_generator_class(builder: IRBuilder, line: int) -> None: + """Add error handling blocks to a generator class. + + Generates blocks to check if error flags are set while calling the + helper method for generator functions, and raises an exception if + those flags are set. + """ + cls = builder.fn_info.generator_class + assert cls.exc_regs is not None + exc_type, exc_val, exc_tb = cls.exc_regs + + # Check to see if an exception was raised. + error_block = BasicBlock() + ok_block = BasicBlock() + comparison = builder.translate_is_op(exc_type, builder.none_object(), "is not", line) + builder.add_bool_branch(comparison, error_block, ok_block) + + builder.activate_block(error_block) + builder.call_c(raise_exception_with_tb_op, [exc_type, exc_val, exc_tb], line) + builder.add(Unreachable()) + builder.goto_and_activate(ok_block) + + +def add_methods_to_generator_class( + builder: IRBuilder, + fn_info: FuncInfo, + sig: FuncSignature, + arg_regs: list[Register], + blocks: list[BasicBlock], + is_coroutine: bool, +) -> None: + helper_fn_decl = add_helper_to_generator_class(builder, arg_regs, blocks, sig, fn_info) + add_next_to_generator_class(builder, fn_info, helper_fn_decl, sig) + add_send_to_generator_class(builder, fn_info, helper_fn_decl, sig) + add_iter_to_generator_class(builder, fn_info) + add_throw_to_generator_class(builder, fn_info, helper_fn_decl, sig) + add_close_to_generator_class(builder, fn_info) + if is_coroutine: + add_await_to_generator_class(builder, fn_info) + + +def add_helper_to_generator_class( + builder: IRBuilder, + arg_regs: list[Register], + blocks: list[BasicBlock], + sig: FuncSignature, + fn_info: FuncInfo, +) -> FuncDecl: + """Generates a helper method for a generator class, called by '__next__' and 'throw'.""" + sig = FuncSignature( + ( + RuntimeArg(SELF_NAME, object_rprimitive), + RuntimeArg("type", object_rprimitive), + RuntimeArg("value", object_rprimitive), + RuntimeArg("traceback", object_rprimitive), + RuntimeArg("arg", object_rprimitive), + ), + sig.ret_type, + ) + helper_fn_decl = FuncDecl( + "__mypyc_generator_helper__", fn_info.generator_class.ir.name, builder.module_name, sig + ) + helper_fn_ir = FuncIR( + helper_fn_decl, arg_regs, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name + ) + fn_info.generator_class.ir.methods["__mypyc_generator_helper__"] = helper_fn_ir + builder.functions.append(helper_fn_ir) + return helper_fn_decl + + +def add_iter_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None: + """Generates the '__iter__' method for a generator class.""" + with builder.enter_method(fn_info.generator_class.ir, "__iter__", object_rprimitive, fn_info): + builder.add(Return(builder.self())) + + +def add_next_to_generator_class( + builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl, sig: FuncSignature +) -> None: + """Generates the '__next__' method for a generator class.""" + with builder.enter_method(fn_info.generator_class.ir, "__next__", object_rprimitive, fn_info): + none_reg = builder.none_object() + # Call the helper function with error flags set to Py_None, and return that result. + result = builder.add( + Call( + fn_decl, + [builder.self(), none_reg, none_reg, none_reg, none_reg], + fn_info.fitem.line, + ) + ) + builder.add(Return(result)) + + +def add_send_to_generator_class( + builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl, sig: FuncSignature +) -> None: + """Generates the 'send' method for a generator class.""" + with builder.enter_method(fn_info.generator_class.ir, "send", object_rprimitive, fn_info): + arg = builder.add_argument("arg", object_rprimitive) + none_reg = builder.none_object() + # Call the helper function with error flags set to Py_None, and return that result. + result = builder.add( + Call( + fn_decl, + [builder.self(), none_reg, none_reg, none_reg, builder.read(arg)], + fn_info.fitem.line, + ) + ) + builder.add(Return(result)) + + +def add_throw_to_generator_class( + builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl, sig: FuncSignature +) -> None: + """Generates the 'throw' method for a generator class.""" + with builder.enter_method(fn_info.generator_class.ir, "throw", object_rprimitive, fn_info): + typ = builder.add_argument("type", object_rprimitive) + val = builder.add_argument("value", object_rprimitive, ARG_OPT) + tb = builder.add_argument("traceback", object_rprimitive, ARG_OPT) + + # Because the value and traceback arguments are optional and hence + # can be NULL if not passed in, we have to assign them Py_None if + # they are not passed in. + none_reg = builder.none_object() + builder.assign_if_null(val, lambda: none_reg, builder.fn_info.fitem.line) + builder.assign_if_null(tb, lambda: none_reg, builder.fn_info.fitem.line) + + # Call the helper function using the arguments passed in, and return that result. + result = builder.add( + Call( + fn_decl, + [builder.self(), builder.read(typ), builder.read(val), builder.read(tb), none_reg], + fn_info.fitem.line, + ) + ) + builder.add(Return(result)) + + +def add_close_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None: + """Generates the '__close__' method for a generator class.""" + with builder.enter_method(fn_info.generator_class.ir, "close", object_rprimitive, fn_info): + except_block, else_block = BasicBlock(), BasicBlock() + builder.builder.push_error_handler(except_block) + builder.goto_and_activate(BasicBlock()) + generator_exit = builder.load_module_attr_by_fullname( + "builtins.GeneratorExit", fn_info.fitem.line + ) + builder.add( + MethodCall( + builder.self(), + "throw", + [generator_exit, builder.none_object(), builder.none_object()], + ) + ) + builder.goto(else_block) + builder.builder.pop_error_handler() + + builder.activate_block(except_block) + old_exc = builder.call_c(error_catch_op, [], fn_info.fitem.line) + builder.nonlocal_control.append( + ExceptNonlocalControl(builder.nonlocal_control[-1], old_exc) + ) + stop_iteration = builder.load_module_attr_by_fullname( + "builtins.StopIteration", fn_info.fitem.line + ) + exceptions = builder.add(TupleSet([generator_exit, stop_iteration], fn_info.fitem.line)) + matches = builder.call_c(exc_matches_op, [exceptions], fn_info.fitem.line) + + match_block, non_match_block = BasicBlock(), BasicBlock() + builder.add(Branch(matches, match_block, non_match_block, Branch.BOOL)) + + builder.activate_block(match_block) + builder.call_c(restore_exc_info_op, [builder.read(old_exc)], fn_info.fitem.line) + builder.add(Return(builder.none_object())) + + builder.activate_block(non_match_block) + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + builder.nonlocal_control.pop() + + builder.activate_block(else_block) + builder.add( + RaiseStandardError( + RaiseStandardError.RUNTIME_ERROR, + "generator ignored GeneratorExit", + fn_info.fitem.line, + ) + ) + builder.add(Unreachable()) + + +def add_await_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None: + """Generates the '__await__' method for a generator class.""" + with builder.enter_method(fn_info.generator_class.ir, "__await__", object_rprimitive, fn_info): + builder.add(Return(builder.self())) + + +def setup_env_for_generator_class(builder: IRBuilder) -> None: + """Populates the environment for a generator class.""" + fitem = builder.fn_info.fitem + cls = builder.fn_info.generator_class + self_target = builder.add_self_to_env(cls.ir) + + # Add the type, value, and traceback variables to the environment. + exc_type = builder.add_local(Var("type"), object_rprimitive, is_arg=True) + exc_val = builder.add_local(Var("value"), object_rprimitive, is_arg=True) + exc_tb = builder.add_local(Var("traceback"), object_rprimitive, is_arg=True) + # TODO: Use the right type here instead of object? + exc_arg = builder.add_local(Var("arg"), object_rprimitive, is_arg=True) + + cls.exc_regs = (exc_type, exc_val, exc_tb) + cls.send_arg_reg = exc_arg + + cls.self_reg = builder.read(self_target, fitem.line) + cls.curr_env_reg = load_outer_env(builder, cls.self_reg, builder.symtables[-1]) + + # Define a variable representing the label to go to the next time + # the '__next__' function of the generator is called, and add it + # as an attribute to the environment class. + cls.next_label_target = builder.add_var_to_env_class( + Var(NEXT_LABEL_ATTR_NAME), int_rprimitive, cls, reassign=False + ) + + # Add arguments from the original generator function to the + # environment of the generator class. + add_args_to_env(builder, local=False, base=cls, reassign=False) + + # Set the next label register for the generator class. + cls.next_label_reg = builder.read(cls.next_label_target, fitem.line) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py new file mode 100644 index 000000000000..556d753b89f8 --- /dev/null +++ b/mypyc/irbuild/ll_builder.py @@ -0,0 +1,2424 @@ +"""A "low-level" IR builder class. + +LowLevelIRBuilder provides core abstractions we use for constructing +IR as well as a number of higher-level ones (accessing attributes, +calling functions and methods, and coercing between types, for +example). The core principle of the low-level IR builder is that all +of its facilities operate solely on the IR level and not the AST +level---it has *no knowledge* of mypy types or expressions. +""" + +from __future__ import annotations + +from typing import Callable, Final, Optional, Sequence, Tuple + +from mypy.argmap import map_actuals_to_formals +from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, ArgKind +from mypy.operators import op_methods, unary_op_methods +from mypy.types import AnyType, TypeOfAny +from mypyc.common import ( + BITMAP_BITS, + FAST_ISINSTANCE_MAX_SUBCLASSES, + MAX_LITERAL_SHORT_INT, + MAX_SHORT_INT, + MIN_LITERAL_SHORT_INT, + MIN_SHORT_INT, + PLATFORM_SIZE, + use_method_vectorcall, + use_vectorcall, +) +from mypyc.errors import Errors +from mypyc.ir.class_ir import ClassIR, all_concrete_classes +from mypyc.ir.func_ir import FuncDecl, FuncSignature +from mypyc.ir.ops import ( + ERR_FALSE, + ERR_NEVER, + NAMESPACE_MODULE, + NAMESPACE_STATIC, + NAMESPACE_TYPE, + Assign, + AssignMulti, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + Extend, + Float, + FloatComparisonOp, + FloatNeg, + FloatOp, + GetAttr, + GetElementPtr, + Goto, + Integer, + IntOp, + KeepAlive, + LoadAddress, + LoadErrorValue, + LoadLiteral, + LoadMem, + LoadStatic, + MethodCall, + Op, + PrimitiveDescription, + PrimitiveOp, + RaiseStandardError, + Register, + Truncate, + TupleGet, + TupleSet, + Unbox, + Unreachable, + Value, + float_comparison_op_to_id, + float_op_to_id, + int_op_to_id, +) +from mypyc.ir.rtypes import ( + PyObject, + PySetObject, + RArray, + RInstance, + RPrimitive, + RTuple, + RType, + RUnion, + bit_rprimitive, + bitmap_rprimitive, + bool_rprimitive, + bytes_rprimitive, + c_int_rprimitive, + c_pointer_rprimitive, + c_pyssize_t_rprimitive, + c_size_t_rprimitive, + check_native_int_range, + dict_rprimitive, + float_rprimitive, + int_rprimitive, + is_bit_rprimitive, + is_bool_rprimitive, + is_bytes_rprimitive, + is_dict_rprimitive, + is_fixed_width_rtype, + is_float_rprimitive, + is_int16_rprimitive, + is_int32_rprimitive, + is_int64_rprimitive, + is_int_rprimitive, + is_list_rprimitive, + is_none_rprimitive, + is_set_rprimitive, + is_short_int_rprimitive, + is_str_rprimitive, + is_tagged, + is_tuple_rprimitive, + is_uint8_rprimitive, + list_rprimitive, + none_rprimitive, + object_pointer_rprimitive, + object_rprimitive, + optional_value_type, + pointer_rprimitive, + short_int_rprimitive, + str_rprimitive, +) +from mypyc.irbuild.util import concrete_arg_kind +from mypyc.options import CompilerOptions +from mypyc.primitives.bytes_ops import bytes_compare +from mypyc.primitives.dict_ops import ( + dict_build_op, + dict_new_op, + dict_ssize_t_size_op, + dict_update_in_display_op, +) +from mypyc.primitives.exc_ops import err_occurred_op, keep_propagating_op +from mypyc.primitives.float_ops import copysign_op, int_to_float_op +from mypyc.primitives.generic_ops import ( + generic_len_op, + generic_ssize_t_len_op, + py_call_op, + py_call_with_kwargs_op, + py_getattr_op, + py_method_call_op, + py_vectorcall_method_op, + py_vectorcall_op, +) +from mypyc.primitives.int_ops import ( + int16_divide_op, + int16_mod_op, + int16_overflow, + int32_divide_op, + int32_mod_op, + int32_overflow, + int64_divide_op, + int64_mod_op, + int64_to_int_op, + int_to_int32_op, + int_to_int64_op, + ssize_t_to_int_op, + uint8_overflow, +) +from mypyc.primitives.list_ops import list_build_op, list_extend_op, list_items, new_list_op +from mypyc.primitives.misc_ops import ( + bool_op, + buf_init_item, + fast_isinstance_op, + none_object_op, + not_implemented_op, + var_object_size, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + CFunctionDescription, + binary_ops, + method_call_ops, + unary_ops, +) +from mypyc.primitives.set_ops import new_set_op +from mypyc.primitives.str_ops import str_check_if_true, str_ssize_t_size_op, unicode_compare +from mypyc.primitives.tuple_ops import list_tuple_op, new_tuple_op, new_tuple_with_length_op +from mypyc.rt_subtype import is_runtime_subtype +from mypyc.sametype import is_same_type +from mypyc.subtype import is_subtype + +DictEntry = Tuple[Optional[Value], Value] + +# If the number of items is less than the threshold when initializing +# a list, we would inline the generate IR using SetMem and expanded +# for-loop. Otherwise, we would call `list_build_op` for larger lists. +# TODO: The threshold is a randomly chosen number which needs further +# study on real-world projects for a better balance. +LIST_BUILDING_EXPANSION_THRESHOLD = 10 + +# From CPython +PY_VECTORCALL_ARGUMENTS_OFFSET: Final = 1 << (PLATFORM_SIZE * 8 - 1) + +FIXED_WIDTH_INT_BINARY_OPS: Final = { + "+", + "-", + "*", + "//", + "%", + "&", + "|", + "^", + "<<", + ">>", + "+=", + "-=", + "*=", + "//=", + "%=", + "&=", + "|=", + "^=", + "<<=", + ">>=", +} + +# Binary operations on bools that are specialized and don't just promote operands to int +BOOL_BINARY_OPS: Final = {"&", "&=", "|", "|=", "^", "^=", "==", "!=", "<", "<=", ">", ">="} + + +class LowLevelIRBuilder: + def __init__(self, errors: Errors | None, options: CompilerOptions) -> None: + self.errors = errors + self.options = options + self.args: list[Register] = [] + self.blocks: list[BasicBlock] = [] + # Stack of except handler entry blocks + self.error_handlers: list[BasicBlock | None] = [None] + # Values that we need to keep alive as long as we have borrowed + # temporaries. Use flush_keep_alives() to mark the end of the live range. + self.keep_alives: list[Value] = [] + + def set_module(self, module_name: str, module_path: str) -> None: + """Set the name and path of the current module.""" + self.module_name = module_name + self.module_path = module_path + + # Basic operations + + def add(self, op: Op) -> Value: + """Add an op.""" + assert not self.blocks[-1].terminated, "Can't add to finished block" + self.blocks[-1].ops.append(op) + return op + + def goto(self, target: BasicBlock) -> None: + """Add goto to a basic block.""" + if not self.blocks[-1].terminated: + self.add(Goto(target)) + + def activate_block(self, block: BasicBlock) -> None: + """Add a basic block and make it the active one (target of adds).""" + if self.blocks: + assert self.blocks[-1].terminated + + block.error_handler = self.error_handlers[-1] + self.blocks.append(block) + + def goto_and_activate(self, block: BasicBlock) -> None: + """Add goto a block and make it the active block.""" + self.goto(block) + self.activate_block(block) + + def keep_alive(self, values: list[Value], *, steal: bool = False) -> None: + self.add(KeepAlive(values, steal=steal)) + + def push_error_handler(self, handler: BasicBlock | None) -> None: + self.error_handlers.append(handler) + + def pop_error_handler(self) -> BasicBlock | None: + return self.error_handlers.pop() + + def self(self) -> Register: + """Return reference to the 'self' argument. + + This only works in a method. + """ + return self.args[0] + + def flush_keep_alives(self) -> None: + if self.keep_alives: + self.add(KeepAlive(self.keep_alives.copy())) + self.keep_alives = [] + + # Type conversions + + def box(self, src: Value) -> Value: + if src.type.is_unboxed: + if isinstance(src, Integer) and is_tagged(src.type): + return self.add(LoadLiteral(src.value >> 1, rtype=object_rprimitive)) + return self.add(Box(src)) + else: + return src + + def unbox_or_cast( + self, src: Value, target_type: RType, line: int, *, can_borrow: bool = False + ) -> Value: + if target_type.is_unboxed: + return self.add(Unbox(src, target_type, line)) + else: + if can_borrow: + self.keep_alives.append(src) + return self.add(Cast(src, target_type, line, borrow=can_borrow)) + + def coerce( + self, + src: Value, + target_type: RType, + line: int, + force: bool = False, + *, + can_borrow: bool = False, + ) -> Value: + """Generate a coercion/cast from one type to other (only if needed). + + For example, int -> object boxes the source int; int -> int emits nothing; + object -> int unboxes the object. All conversions preserve object value. + + If force is true, always generate an op (even if it is just an assignment) so + that the result will have exactly target_type as the type. + + Returns the register with the converted value (may be same as src). + """ + src_type = src.type + if src_type.is_unboxed and not target_type.is_unboxed: + # Unboxed -> boxed + return self.box(src) + if (src_type.is_unboxed and target_type.is_unboxed) and not is_runtime_subtype( + src_type, target_type + ): + if ( + isinstance(src, Integer) + and is_short_int_rprimitive(src_type) + and is_fixed_width_rtype(target_type) + ): + value = src.numeric_value() + if not check_native_int_range(target_type, value): + self.error(f'Value {value} is out of range for "{target_type}"', line) + return Integer(src.value >> 1, target_type) + elif is_int_rprimitive(src_type) and is_fixed_width_rtype(target_type): + return self.coerce_int_to_fixed_width(src, target_type, line) + elif is_fixed_width_rtype(src_type) and is_int_rprimitive(target_type): + return self.coerce_fixed_width_to_int(src, line) + elif is_short_int_rprimitive(src_type) and is_fixed_width_rtype(target_type): + return self.coerce_short_int_to_fixed_width(src, target_type, line) + elif ( + isinstance(src_type, RPrimitive) + and isinstance(target_type, RPrimitive) + and src_type.is_native_int + and target_type.is_native_int + and src_type.size == target_type.size + and src_type.is_signed == target_type.is_signed + ): + # Equivalent types + return src + elif (is_bool_rprimitive(src_type) or is_bit_rprimitive(src_type)) and is_tagged( + target_type + ): + shifted = self.int_op( + bool_rprimitive, src, Integer(1, bool_rprimitive), IntOp.LEFT_SHIFT + ) + return self.add(Extend(shifted, target_type, signed=False)) + elif ( + is_bool_rprimitive(src_type) or is_bit_rprimitive(src_type) + ) and is_fixed_width_rtype(target_type): + return self.add(Extend(src, target_type, signed=False)) + elif isinstance(src, Integer) and is_float_rprimitive(target_type): + if is_tagged(src_type): + return Float(float(src.value // 2)) + return Float(float(src.value)) + elif is_tagged(src_type) and is_float_rprimitive(target_type): + return self.int_to_float(src, line) + elif ( + isinstance(src_type, RTuple) + and isinstance(target_type, RTuple) + and len(src_type.types) == len(target_type.types) + ): + # Coerce between two tuple types by coercing each item separately + values = [] + for i in range(len(src_type.types)): + v = None + if isinstance(src, TupleSet): + item = src.items[i] + # We can't reuse register values, since they can be modified. + if not isinstance(item, Register): + v = item + if v is None: + v = TupleGet(src, i) + self.add(v) + values.append(v) + return self.add( + TupleSet( + [self.coerce(v, t, line) for v, t in zip(values, target_type.types)], line + ) + ) + # To go between any other unboxed types, we go through a boxed + # in-between value, for simplicity. + tmp = self.box(src) + return self.unbox_or_cast(tmp, target_type, line) + if (not src_type.is_unboxed and target_type.is_unboxed) or not is_subtype( + src_type, target_type + ): + return self.unbox_or_cast(src, target_type, line, can_borrow=can_borrow) + elif force: + tmp = Register(target_type) + self.add(Assign(tmp, src)) + return tmp + return src + + def coerce_int_to_fixed_width(self, src: Value, target_type: RType, line: int) -> Value: + assert is_fixed_width_rtype(target_type), target_type + assert isinstance(target_type, RPrimitive) + + res = Register(target_type) + + fast, slow, end = BasicBlock(), BasicBlock(), BasicBlock() + + check = self.check_tagged_short_int(src, line) + self.add(Branch(check, fast, slow, Branch.BOOL)) + + self.activate_block(fast) + + size = target_type.size + if size < int_rprimitive.size: + # Add a range check when the target type is smaller than the source tyoe + fast2, fast3 = BasicBlock(), BasicBlock() + upper_bound = 1 << (size * 8 - 1) + if not target_type.is_signed: + upper_bound *= 2 + check2 = self.add(ComparisonOp(src, Integer(upper_bound, src.type), ComparisonOp.SLT)) + self.add(Branch(check2, fast2, slow, Branch.BOOL)) + self.activate_block(fast2) + if target_type.is_signed: + lower_bound = -upper_bound + else: + lower_bound = 0 + check3 = self.add(ComparisonOp(src, Integer(lower_bound, src.type), ComparisonOp.SGE)) + self.add(Branch(check3, fast3, slow, Branch.BOOL)) + self.activate_block(fast3) + tmp = self.int_op( + c_pyssize_t_rprimitive, + src, + Integer(1, c_pyssize_t_rprimitive), + IntOp.RIGHT_SHIFT, + line, + ) + tmp = self.add(Truncate(tmp, target_type)) + else: + if size > int_rprimitive.size: + tmp = self.add(Extend(src, target_type, signed=True)) + else: + tmp = src + tmp = self.int_op(target_type, tmp, Integer(1, target_type), IntOp.RIGHT_SHIFT, line) + + self.add(Assign(res, tmp)) + self.goto(end) + + self.activate_block(slow) + if is_int64_rprimitive(target_type) or ( + is_int32_rprimitive(target_type) and size == int_rprimitive.size + ): + # Slow path calls a library function that handles more complex logic + ptr = self.int_op( + pointer_rprimitive, src, Integer(1, pointer_rprimitive), IntOp.XOR, line + ) + ptr2 = Register(c_pointer_rprimitive) + self.add(Assign(ptr2, ptr)) + if is_int64_rprimitive(target_type): + conv_op = int_to_int64_op + else: + conv_op = int_to_int32_op + tmp = self.call_c(conv_op, [ptr2], line) + self.add(Assign(res, tmp)) + self.add(KeepAlive([src])) + self.goto(end) + elif is_int32_rprimitive(target_type): + # Slow path just always generates an OverflowError + self.call_c(int32_overflow, [], line) + self.add(Unreachable()) + elif is_int16_rprimitive(target_type): + # Slow path just always generates an OverflowError + self.call_c(int16_overflow, [], line) + self.add(Unreachable()) + elif is_uint8_rprimitive(target_type): + # Slow path just always generates an OverflowError + self.call_c(uint8_overflow, [], line) + self.add(Unreachable()) + else: + assert False, target_type + + self.activate_block(end) + return res + + def coerce_short_int_to_fixed_width(self, src: Value, target_type: RType, line: int) -> Value: + if is_int64_rprimitive(target_type): + return self.int_op(target_type, src, Integer(1, target_type), IntOp.RIGHT_SHIFT, line) + # TODO: i32 + assert False, (src.type, target_type) + + def coerce_fixed_width_to_int(self, src: Value, line: int) -> Value: + if ( + (is_int32_rprimitive(src.type) and PLATFORM_SIZE == 8) + or is_int16_rprimitive(src.type) + or is_uint8_rprimitive(src.type) + ): + # Simple case -- just sign extend and shift. + extended = self.add(Extend(src, c_pyssize_t_rprimitive, signed=src.type.is_signed)) + return self.int_op( + int_rprimitive, + extended, + Integer(1, c_pyssize_t_rprimitive), + IntOp.LEFT_SHIFT, + line, + ) + + assert is_fixed_width_rtype(src.type) + assert isinstance(src.type, RPrimitive) + src_type = src.type + + res = Register(int_rprimitive) + + fast, fast2, slow, end = BasicBlock(), BasicBlock(), BasicBlock(), BasicBlock() + + c1 = self.add(ComparisonOp(src, Integer(MAX_SHORT_INT, src_type), ComparisonOp.SLE)) + self.add(Branch(c1, fast, slow, Branch.BOOL)) + + self.activate_block(fast) + c2 = self.add(ComparisonOp(src, Integer(MIN_SHORT_INT, src_type), ComparisonOp.SGE)) + self.add(Branch(c2, fast2, slow, Branch.BOOL)) + + self.activate_block(slow) + if is_int64_rprimitive(src_type): + conv_op = int64_to_int_op + elif is_int32_rprimitive(src_type): + assert PLATFORM_SIZE == 4 + conv_op = ssize_t_to_int_op + else: + assert False, src_type + x = self.call_c(conv_op, [src], line) + self.add(Assign(res, x)) + self.goto(end) + + self.activate_block(fast2) + if int_rprimitive.size < src_type.size: + tmp = self.add(Truncate(src, c_pyssize_t_rprimitive)) + else: + tmp = src + s = self.int_op(int_rprimitive, tmp, Integer(1, tmp.type), IntOp.LEFT_SHIFT, line) + self.add(Assign(res, s)) + self.goto(end) + + self.activate_block(end) + return res + + def coerce_nullable(self, src: Value, target_type: RType, line: int) -> Value: + """Generate a coercion from a potentially null value.""" + if src.type.is_unboxed == target_type.is_unboxed and ( + (target_type.is_unboxed and is_runtime_subtype(src.type, target_type)) + or (not target_type.is_unboxed and is_subtype(src.type, target_type)) + ): + return src + + target = Register(target_type) + + valid, invalid, out = BasicBlock(), BasicBlock(), BasicBlock() + self.add(Branch(src, invalid, valid, Branch.IS_ERROR)) + + self.activate_block(valid) + coerced = self.coerce(src, target_type, line) + self.add(Assign(target, coerced, line)) + self.goto(out) + + self.activate_block(invalid) + error = self.add(LoadErrorValue(target_type)) + self.add(Assign(target, error, line)) + + self.goto_and_activate(out) + return target + + # Attribute access + + def get_attr( + self, obj: Value, attr: str, result_type: RType, line: int, *, borrow: bool = False + ) -> Value: + """Get a native or Python attribute of an object.""" + if ( + isinstance(obj.type, RInstance) + and obj.type.class_ir.is_ext_class + and obj.type.class_ir.has_attr(attr) + ): + op = GetAttr(obj, attr, line, borrow=borrow) + # For non-refcounted attribute types, the borrow might be + # disabled even if requested, so don't check 'borrow'. + if op.is_borrowed: + self.keep_alives.append(obj) + return self.add(op) + elif isinstance(obj.type, RUnion): + return self.union_get_attr(obj, obj.type, attr, result_type, line) + else: + return self.py_get_attr(obj, attr, line) + + def union_get_attr( + self, obj: Value, rtype: RUnion, attr: str, result_type: RType, line: int + ) -> Value: + """Get an attribute of an object with a union type.""" + + def get_item_attr(value: Value) -> Value: + return self.get_attr(value, attr, result_type, line) + + return self.decompose_union_helper(obj, rtype, result_type, get_item_attr, line) + + def py_get_attr(self, obj: Value, attr: str, line: int) -> Value: + """Get a Python attribute (slow). + + Prefer get_attr() which generates optimized code for native classes. + """ + key = self.load_str(attr) + return self.primitive_op(py_getattr_op, [obj, key], line) + + # isinstance() checks + + def isinstance_helper(self, obj: Value, class_irs: list[ClassIR], line: int) -> Value: + """Fast path for isinstance() that checks against a list of native classes.""" + if not class_irs: + return self.false() + ret = self.isinstance_native(obj, class_irs[0], line) + for class_ir in class_irs[1:]: + + def other() -> Value: + return self.isinstance_native(obj, class_ir, line) + + ret = self.shortcircuit_helper("or", bool_rprimitive, lambda: ret, other, line) + return ret + + def get_type_of_obj(self, obj: Value, line: int) -> Value: + ob_type_address = self.add(GetElementPtr(obj, PyObject, "ob_type", line)) + ob_type = self.add(LoadMem(object_rprimitive, ob_type_address)) + self.add(KeepAlive([obj])) + return ob_type + + def type_is_op(self, obj: Value, type_obj: Value, line: int) -> Value: + typ = self.get_type_of_obj(obj, line) + return self.add(ComparisonOp(typ, type_obj, ComparisonOp.EQ, line)) + + def isinstance_native(self, obj: Value, class_ir: ClassIR, line: int) -> Value: + """Fast isinstance() check for a native class. + + If there are three or fewer concrete (non-trait) classes among the class + and all its children, use even faster type comparison checks `type(obj) + is typ`. + """ + concrete = all_concrete_classes(class_ir) + if concrete is None or len(concrete) > FAST_ISINSTANCE_MAX_SUBCLASSES + 1: + return self.primitive_op( + fast_isinstance_op, [obj, self.get_native_type(class_ir)], line + ) + if not concrete: + # There can't be any concrete instance that matches this. + return self.false() + type_obj = self.get_native_type(concrete[0]) + ret = self.type_is_op(obj, type_obj, line) + for c in concrete[1:]: + + def other() -> Value: + return self.type_is_op(obj, self.get_native_type(c), line) + + ret = self.shortcircuit_helper("or", bool_rprimitive, lambda: ret, other, line) + return ret + + # Calls + + def _construct_varargs( + self, + args: Sequence[tuple[Value, ArgKind, str | None]], + line: int, + *, + has_star: bool, + has_star2: bool, + ) -> tuple[Value | None, Value | None]: + """Construct *args and **kwargs from a collection of arguments + + This is pretty complicated, and almost all of the complication here stems from + one of two things (but mostly the second): + * The handling of ARG_STAR/ARG_STAR2. We want to create as much of the args/kwargs + values in one go as we can, so we collect values until our hand is forced, and + then we emit creation of the list/tuple, and expand it from there if needed. + + * Support potentially nullable argument values. This has very narrow applicability, + as this will never be done by our compiled Python code, but is critically used + by gen_glue_method when generating glue methods to mediate between the function + signature of a parent class and its subclasses. + + For named-only arguments, this is quite simple: if it is + null, don't put it in the dict. + + For positional-or-named arguments, things are much more complicated. + * First, anything that was passed as a positional arg + must be forwarded along as a positional arg. It *must + not* be converted to a named arg. This is because mypy + does not enforce that positional-or-named arguments + have the same name in subclasses, and it is not + uncommon for code to have different names in + subclasses (a bunch of mypy's visitors do this, for + example!). This is arguably a bug in both mypy and code doing + this, and they ought to be using positional-only arguments, but + positional-only arguments are new and ugly. + + * On the flip side, we're willing to accept the + infelicity of sometimes turning an argument that was + passed by keyword into a positional argument. It's wrong, + but it's very marginal, and avoiding it would require passing + a bitmask of which arguments were named with every function call, + or something similar. + (See some discussion of this in testComplicatedArgs) + + Thus, our strategy for positional-or-named arguments is to + always pass them as positional, except in the one + situation where we can not, and where we can be absolutely + sure they were passed by name: when an *earlier* + positional argument was missing its value. + + This means that if we have a method `f(self, x: int=..., y: object=...)`: + * x and y present: args=(x, y), kwargs={} + * x present, y missing: args=(x,), kwargs={} + * x missing, y present: args=(), kwargs={'y': y} + + To implement this, when we have multiple optional + positional arguments, we maintain a flag in a register + that tracks whether an argument has been missing, and for + each such optional argument (except the first), we check + the flag to determine whether to append the argument to + the *args list or add it to the **kwargs dict. What a + mess! + + This is what really makes everything here such a tangle; + otherwise the *args and **kwargs code could be separated. + + The arguments has_star and has_star2 indicate whether the target function + takes an ARG_STAR and ARG_STAR2 argument, respectively. + (These will always be true when making a pycall, and be based + on the actual target signature for a native call.) + """ + + star_result: Value | None = None + star2_result: Value | None = None + # We aggregate values that need to go into *args and **kwargs + # in these lists. Once all arguments are processed (in the + # happiest case), or we encounter an ARG_STAR/ARG_STAR2 or a + # nullable arg, then we create the list and/or dict. + star_values: list[Value] = [] + star2_keys: list[Value] = [] + star2_values: list[Value] = [] + + seen_empty_reg: Register | None = None + + for value, kind, name in args: + if kind == ARG_STAR: + if star_result is None: + star_result = self.new_list_op(star_values, line) + self.primitive_op(list_extend_op, [star_result, value], line) + elif kind == ARG_STAR2: + if star2_result is None: + star2_result = self._create_dict(star2_keys, star2_values, line) + + self.call_c(dict_update_in_display_op, [star2_result, value], line=line) + else: + nullable = kind.is_optional() + maybe_pos = kind.is_positional() and has_star + maybe_named = kind.is_named() or (kind.is_optional() and name and has_star2) + + # If the argument is nullable, we need to create the + # relevant args/kwargs objects so that we can + # conditionally modify them. + if nullable: + if maybe_pos and star_result is None: + star_result = self.new_list_op(star_values, line) + if maybe_named and star2_result is None: + star2_result = self._create_dict(star2_keys, star2_values, line) + + # Easy cases: just collect the argument. + if maybe_pos and star_result is None: + star_values.append(value) + continue + + if maybe_named and star2_result is None: + assert name is not None + key = self.load_str(name) + star2_keys.append(key) + star2_values.append(value) + continue + + # OK, anything that is nullable or *after* a nullable arg needs to be here + # TODO: We could try harder to avoid creating basic blocks in the common case + new_seen_empty_reg = seen_empty_reg + + out = BasicBlock() + if nullable: + # If this is the first nullable positional arg we've seen, create + # a register to track whether anything has been null. + # (We won't *check* the register until the next argument, though.) + if maybe_pos and not seen_empty_reg: + new_seen_empty_reg = Register(bool_rprimitive) + self.add(Assign(new_seen_empty_reg, self.false(), line)) + + skip = BasicBlock() if maybe_pos else out + keep = BasicBlock() + self.add(Branch(value, skip, keep, Branch.IS_ERROR)) + self.activate_block(keep) + + # If this could be positional or named and we /might/ have seen a missing + # positional arg, then we need to compile *both* a positional and named + # version! What a pain! + if maybe_pos and maybe_named and seen_empty_reg: + pos_block, named_block = BasicBlock(), BasicBlock() + self.add(Branch(seen_empty_reg, named_block, pos_block, Branch.BOOL)) + else: + pos_block = named_block = BasicBlock() + self.goto(pos_block) + + if maybe_pos: + self.activate_block(pos_block) + assert star_result + self.translate_special_method_call( + star_result, "append", [value], result_type=None, line=line + ) + self.goto(out) + + if maybe_named and (not maybe_pos or seen_empty_reg): + self.activate_block(named_block) + assert name is not None + key = self.load_str(name) + assert star2_result + self.translate_special_method_call( + star2_result, "__setitem__", [key, value], result_type=None, line=line + ) + self.goto(out) + + if nullable and maybe_pos and new_seen_empty_reg: + assert skip is not out + self.activate_block(skip) + self.add(Assign(new_seen_empty_reg, self.true(), line)) + self.goto(out) + + self.activate_block(out) + + seen_empty_reg = new_seen_empty_reg + + assert not (star_result or star_values) or has_star + assert not (star2_result or star2_values) or has_star2 + if has_star: + # If we managed to make it this far without creating a + # *args list, then we can directly create a + # tuple. Otherwise create the tuple from the list. + if star_result is None: + star_result = self.new_tuple(star_values, line) + else: + star_result = self.primitive_op(list_tuple_op, [star_result], line) + if has_star2 and star2_result is None: + star2_result = self._create_dict(star2_keys, star2_values, line) + + return star_result, star2_result + + def py_call( + self, + function: Value, + arg_values: list[Value], + line: int, + arg_kinds: list[ArgKind] | None = None, + arg_names: Sequence[str | None] | None = None, + ) -> Value: + """Call a Python function (non-native and slow). + + Use py_call_op or py_call_with_kwargs_op for Python function call. + """ + if use_vectorcall(self.options.capi_version): + # More recent Python versions support faster vectorcalls. + result = self._py_vector_call(function, arg_values, line, arg_kinds, arg_names) + if result is not None: + return result + + # If all arguments are positional, we can use py_call_op. + if arg_kinds is None or all(kind == ARG_POS for kind in arg_kinds): + return self.call_c(py_call_op, [function] + arg_values, line) + + # Otherwise fallback to py_call_with_kwargs_op. + assert arg_names is not None + + pos_args_tuple, kw_args_dict = self._construct_varargs( + list(zip(arg_values, arg_kinds, arg_names)), line, has_star=True, has_star2=True + ) + assert pos_args_tuple and kw_args_dict + + return self.call_c(py_call_with_kwargs_op, [function, pos_args_tuple, kw_args_dict], line) + + def _py_vector_call( + self, + function: Value, + arg_values: list[Value], + line: int, + arg_kinds: list[ArgKind] | None = None, + arg_names: Sequence[str | None] | None = None, + ) -> Value | None: + """Call function using the vectorcall API if possible. + + Return the return value if successful. Return None if a non-vectorcall + API should be used instead. + """ + # We can do this if all args are positional or named (no *args or **kwargs, not optional). + if arg_kinds is None or all( + not kind.is_star() and not kind.is_optional() for kind in arg_kinds + ): + if arg_values: + # Create a C array containing all arguments as boxed values. + coerced_args = [self.coerce(arg, object_rprimitive, line) for arg in arg_values] + arg_ptr = self.setup_rarray(object_rprimitive, coerced_args, object_ptr=True) + else: + arg_ptr = Integer(0, object_pointer_rprimitive) + num_pos = num_positional_args(arg_values, arg_kinds) + keywords = self._vectorcall_keywords(arg_names) + value = self.call_c( + py_vectorcall_op, + [function, arg_ptr, Integer(num_pos, c_size_t_rprimitive), keywords], + line, + ) + if arg_values: + # Make sure arguments won't be freed until after the call. + # We need this because RArray doesn't support automatic + # memory management. + self.add(KeepAlive(coerced_args)) + return value + return None + + def _vectorcall_keywords(self, arg_names: Sequence[str | None] | None) -> Value: + """Return a reference to a tuple literal with keyword argument names. + + Return null pointer if there are no keyword arguments. + """ + if arg_names: + kw_list = [name for name in arg_names if name is not None] + if kw_list: + return self.add(LoadLiteral(tuple(kw_list), object_rprimitive)) + return Integer(0, object_rprimitive) + + def py_method_call( + self, + obj: Value, + method_name: str, + arg_values: list[Value], + line: int, + arg_kinds: list[ArgKind] | None, + arg_names: Sequence[str | None] | None, + ) -> Value: + """Call a Python method (non-native and slow).""" + if use_method_vectorcall(self.options.capi_version): + # More recent Python versions support faster vectorcalls. + result = self._py_vector_method_call( + obj, method_name, arg_values, line, arg_kinds, arg_names + ) + if result is not None: + return result + + if arg_kinds is None or all(kind == ARG_POS for kind in arg_kinds): + # Use legacy method call API + method_name_reg = self.load_str(method_name) + return self.call_c(py_method_call_op, [obj, method_name_reg] + arg_values, line) + else: + # Use py_call since it supports keyword arguments (and vectorcalls). + method = self.py_get_attr(obj, method_name, line) + return self.py_call(method, arg_values, line, arg_kinds=arg_kinds, arg_names=arg_names) + + def _py_vector_method_call( + self, + obj: Value, + method_name: str, + arg_values: list[Value], + line: int, + arg_kinds: list[ArgKind] | None, + arg_names: Sequence[str | None] | None, + ) -> Value | None: + """Call method using the vectorcall API if possible. + + Return the return value if successful. Return None if a non-vectorcall + API should be used instead. + """ + if arg_kinds is None or all( + not kind.is_star() and not kind.is_optional() for kind in arg_kinds + ): + method_name_reg = self.load_str(method_name) + coerced_args = [ + self.coerce(arg, object_rprimitive, line) for arg in [obj] + arg_values + ] + arg_ptr = self.setup_rarray(object_rprimitive, coerced_args, object_ptr=True) + num_pos = num_positional_args(arg_values, arg_kinds) + keywords = self._vectorcall_keywords(arg_names) + value = self.call_c( + py_vectorcall_method_op, + [ + method_name_reg, + arg_ptr, + Integer((num_pos + 1) | PY_VECTORCALL_ARGUMENTS_OFFSET, c_size_t_rprimitive), + keywords, + ], + line, + ) + # Make sure arguments won't be freed until after the call. + # We need this because RArray doesn't support automatic + # memory management. + self.add(KeepAlive(coerced_args)) + return value + return None + + def call( + self, + decl: FuncDecl, + args: Sequence[Value], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None], + line: int, + *, + bitmap_args: list[Register] | None = None, + ) -> Value: + """Call a native function. + + If bitmap_args is given, they override the values of (some) of the bitmap + arguments used to track the presence of values for certain arguments. By + default, the values of the bitmap arguments are inferred from args. + """ + # Normalize args to positionals. + args = self.native_args_to_positional( + args, arg_kinds, arg_names, decl.sig, line, bitmap_args=bitmap_args + ) + return self.add(Call(decl, args, line)) + + def native_args_to_positional( + self, + args: Sequence[Value], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None], + sig: FuncSignature, + line: int, + *, + bitmap_args: list[Register] | None = None, + ) -> list[Value]: + """Prepare arguments for a native call. + + Given args/kinds/names and a target signature for a native call, map + keyword arguments to their appropriate place in the argument list, + fill in error values for unspecified default arguments, + package arguments that will go into *args/**kwargs into a tuple/dict, + and coerce arguments to the appropriate type. + """ + + sig_args = sig.args + n = sig.num_bitmap_args + if n: + sig_args = sig_args[:-n] + + sig_arg_kinds = [arg.kind for arg in sig_args] + sig_arg_names = [arg.name for arg in sig_args] + + concrete_kinds = [concrete_arg_kind(arg_kind) for arg_kind in arg_kinds] + formal_to_actual = map_actuals_to_formals( + concrete_kinds, + arg_names, + sig_arg_kinds, + sig_arg_names, + lambda n: AnyType(TypeOfAny.special_form), + ) + + # First scan for */** and construct those + has_star = has_star2 = False + star_arg_entries = [] + for lst, arg in zip(formal_to_actual, sig_args): + if arg.kind.is_star(): + star_arg_entries.extend([(args[i], arg_kinds[i], arg_names[i]) for i in lst]) + has_star = has_star or arg.kind == ARG_STAR + has_star2 = has_star2 or arg.kind == ARG_STAR2 + + star_arg, star2_arg = self._construct_varargs( + star_arg_entries, line, has_star=has_star, has_star2=has_star2 + ) + + # Flatten out the arguments, loading error values for default + # arguments, constructing tuples/dicts for star args, and + # coercing everything to the expected type. + output_args: list[Value] = [] + for lst, arg in zip(formal_to_actual, sig_args): + if arg.kind == ARG_STAR: + assert star_arg + output_arg = star_arg + elif arg.kind == ARG_STAR2: + assert star2_arg + output_arg = star2_arg + elif not lst: + if is_fixed_width_rtype(arg.type): + output_arg = Integer(0, arg.type) + elif is_float_rprimitive(arg.type): + output_arg = Float(0.0) + else: + output_arg = self.add(LoadErrorValue(arg.type, is_borrowed=True)) + else: + base_arg = args[lst[0]] + + if arg_kinds[lst[0]].is_optional(): + output_arg = self.coerce_nullable(base_arg, arg.type, line) + else: + output_arg = self.coerce(base_arg, arg.type, line) + + output_args.append(output_arg) + + for i in reversed(range(n)): + if bitmap_args and i < len(bitmap_args): + # Use override provided by caller + output_args.append(bitmap_args[i]) + continue + # Infer values of bitmap args + bitmap = 0 + c = 0 + for lst, arg in zip(formal_to_actual, sig_args): + if arg.kind.is_optional() and arg.type.error_overlap: + if i * BITMAP_BITS <= c < (i + 1) * BITMAP_BITS: + if lst: + bitmap |= 1 << (c & (BITMAP_BITS - 1)) + c += 1 + output_args.append(Integer(bitmap, bitmap_rprimitive)) + + return output_args + + def gen_method_call( + self, + base: Value, + name: str, + arg_values: list[Value], + result_type: RType | None, + line: int, + arg_kinds: list[ArgKind] | None = None, + arg_names: list[str | None] | None = None, + can_borrow: bool = False, + ) -> Value: + """Generate either a native or Python method call.""" + # If we have *args, then fallback to Python method call. + if arg_kinds is not None and any(kind.is_star() for kind in arg_kinds): + return self.py_method_call(base, name, arg_values, base.line, arg_kinds, arg_names) + + # If the base type is one of ours, do a MethodCall + if ( + isinstance(base.type, RInstance) + and base.type.class_ir.is_ext_class + and not base.type.class_ir.builtin_base + ): + if base.type.class_ir.has_method(name): + decl = base.type.class_ir.method_decl(name) + if arg_kinds is None: + assert arg_names is None, "arg_kinds not present but arg_names is" + arg_kinds = [ARG_POS for _ in arg_values] + arg_names = [None for _ in arg_values] + else: + assert arg_names is not None, "arg_kinds present but arg_names is not" + + # Normalize args to positionals. + assert decl.bound_sig + arg_values = self.native_args_to_positional( + arg_values, arg_kinds, arg_names, decl.bound_sig, line + ) + return self.add(MethodCall(base, name, arg_values, line)) + elif base.type.class_ir.has_attr(name): + function = self.add(GetAttr(base, name, line)) + return self.py_call( + function, arg_values, line, arg_kinds=arg_kinds, arg_names=arg_names + ) + + elif isinstance(base.type, RUnion): + return self.union_method_call( + base, base.type, name, arg_values, result_type, line, arg_kinds, arg_names + ) + + # Try to do a special-cased method call + if not arg_kinds or arg_kinds == [ARG_POS] * len(arg_values): + target = self.translate_special_method_call( + base, name, arg_values, result_type, line, can_borrow=can_borrow + ) + if target: + return target + + # Fall back to Python method call + return self.py_method_call(base, name, arg_values, line, arg_kinds, arg_names) + + def union_method_call( + self, + base: Value, + obj_type: RUnion, + name: str, + arg_values: list[Value], + return_rtype: RType | None, + line: int, + arg_kinds: list[ArgKind] | None, + arg_names: list[str | None] | None, + ) -> Value: + """Generate a method call with a union type for the object.""" + # Union method call needs a return_rtype for the type of the output register. + # If we don't have one, use object_rprimitive. + return_rtype = return_rtype or object_rprimitive + + def call_union_item(value: Value) -> Value: + return self.gen_method_call( + value, name, arg_values, return_rtype, line, arg_kinds, arg_names + ) + + return self.decompose_union_helper(base, obj_type, return_rtype, call_union_item, line) + + # Loading various values + + def none(self) -> Value: + """Load unboxed None value (type: none_rprimitive).""" + return Integer(1, none_rprimitive) + + def true(self) -> Value: + """Load unboxed True value (type: bool_rprimitive).""" + return Integer(1, bool_rprimitive) + + def false(self) -> Value: + """Load unboxed False value (type: bool_rprimitive).""" + return Integer(0, bool_rprimitive) + + def none_object(self) -> Value: + """Load Python None value (type: object_rprimitive).""" + return self.add(LoadAddress(none_object_op.type, none_object_op.src, line=-1)) + + def load_int(self, value: int) -> Value: + """Load a tagged (Python) integer literal value.""" + if value > MAX_LITERAL_SHORT_INT or value < MIN_LITERAL_SHORT_INT: + return self.add(LoadLiteral(value, int_rprimitive)) + else: + return Integer(value) + + def load_float(self, value: float) -> Value: + """Load a float literal value.""" + return Float(value) + + def load_str(self, value: str) -> Value: + """Load a str literal value. + + This is useful for more than just str literals; for example, method calls + also require a PyObject * form for the name of the method. + """ + return self.add(LoadLiteral(value, str_rprimitive)) + + def load_bytes(self, value: bytes) -> Value: + """Load a bytes literal value.""" + return self.add(LoadLiteral(value, bytes_rprimitive)) + + def load_complex(self, value: complex) -> Value: + """Load a complex literal value.""" + return self.add(LoadLiteral(value, object_rprimitive)) + + def load_static_checked( + self, + typ: RType, + identifier: str, + module_name: str | None = None, + namespace: str = NAMESPACE_STATIC, + line: int = -1, + error_msg: str | None = None, + ) -> Value: + if error_msg is None: + error_msg = f'name "{identifier}" is not defined' + ok_block, error_block = BasicBlock(), BasicBlock() + value = self.add(LoadStatic(typ, identifier, module_name, namespace, line=line)) + self.add(Branch(value, error_block, ok_block, Branch.IS_ERROR, rare=True)) + self.activate_block(error_block) + self.add(RaiseStandardError(RaiseStandardError.NAME_ERROR, error_msg, line)) + self.add(Unreachable()) + self.activate_block(ok_block) + return value + + def load_module(self, name: str) -> Value: + return self.add(LoadStatic(object_rprimitive, name, namespace=NAMESPACE_MODULE)) + + def get_native_type(self, cls: ClassIR) -> Value: + """Load native type object.""" + fullname = f"{cls.module_name}.{cls.name}" + return self.load_native_type_object(fullname) + + def load_native_type_object(self, fullname: str) -> Value: + module, name = fullname.rsplit(".", 1) + return self.add(LoadStatic(object_rprimitive, name, module, NAMESPACE_TYPE)) + + # Other primitive operations + + def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: + """Perform a binary operation. + + Generate specialized operations based on operand types, with a fallback + to generic operations. + """ + ltype = lreg.type + rtype = rreg.type + + # Special case tuple comparison here so that nested tuples can be supported + if isinstance(ltype, RTuple) and isinstance(rtype, RTuple) and op in ("==", "!="): + return self.compare_tuples(lreg, rreg, op, line) + + # Special case == and != when we can resolve the method call statically + if op in ("==", "!="): + value = self.translate_eq_cmp(lreg, rreg, op, line) + if value is not None: + return value + + # Special case various ops + if op in ("is", "is not"): + return self.translate_is_op(lreg, rreg, op, line) + # TODO: modify 'str' to use same interface as 'compare_bytes' as it avoids + # call to PyErr_Occurred() + if is_str_rprimitive(ltype) and is_str_rprimitive(rtype) and op in ("==", "!="): + return self.compare_strings(lreg, rreg, op, line) + if is_bytes_rprimitive(ltype) and is_bytes_rprimitive(rtype) and op in ("==", "!="): + return self.compare_bytes(lreg, rreg, op, line) + if is_bool_rprimitive(ltype) and is_bool_rprimitive(rtype) and op in BOOL_BINARY_OPS: + if op in ComparisonOp.signed_ops: + return self.bool_comparison_op(lreg, rreg, op, line) + else: + return self.bool_bitwise_op(lreg, rreg, op[0], line) + if isinstance(rtype, RInstance) and op in ("in", "not in"): + return self.translate_instance_contains(rreg, lreg, op, line) + if is_fixed_width_rtype(ltype): + if op in FIXED_WIDTH_INT_BINARY_OPS: + if op.endswith("="): + op = op[:-1] + if op != "//": + op_id = int_op_to_id[op] + else: + op_id = IntOp.DIV + if is_bool_rprimitive(rtype) or is_bit_rprimitive(rtype): + rreg = self.coerce(rreg, ltype, line) + rtype = ltype + if is_fixed_width_rtype(rtype) or is_tagged(rtype): + return self.fixed_width_int_op(ltype, lreg, rreg, op_id, line) + if isinstance(rreg, Integer): + return self.fixed_width_int_op( + ltype, lreg, self.coerce(rreg, ltype, line), op_id, line + ) + elif op in ComparisonOp.signed_ops: + if is_int_rprimitive(rtype): + rreg = self.coerce_int_to_fixed_width(rreg, ltype, line) + elif is_bool_rprimitive(rtype) or is_bit_rprimitive(rtype): + rreg = self.coerce(rreg, ltype, line) + op_id = ComparisonOp.signed_ops[op] + if is_fixed_width_rtype(rreg.type): + return self.comparison_op(lreg, rreg, op_id, line) + if isinstance(rreg, Integer): + return self.comparison_op(lreg, self.coerce(rreg, ltype, line), op_id, line) + elif is_fixed_width_rtype(rtype): + if op in FIXED_WIDTH_INT_BINARY_OPS: + if op.endswith("="): + op = op[:-1] + if op != "//": + op_id = int_op_to_id[op] + else: + op_id = IntOp.DIV + if isinstance(lreg, Integer): + return self.fixed_width_int_op( + rtype, self.coerce(lreg, rtype, line), rreg, op_id, line + ) + if is_tagged(ltype): + return self.fixed_width_int_op(rtype, lreg, rreg, op_id, line) + if is_bool_rprimitive(ltype) or is_bit_rprimitive(ltype): + lreg = self.coerce(lreg, rtype, line) + return self.fixed_width_int_op(rtype, lreg, rreg, op_id, line) + elif op in ComparisonOp.signed_ops: + if is_int_rprimitive(ltype): + lreg = self.coerce_int_to_fixed_width(lreg, rtype, line) + elif is_bool_rprimitive(ltype) or is_bit_rprimitive(ltype): + lreg = self.coerce(lreg, rtype, line) + op_id = ComparisonOp.signed_ops[op] + if isinstance(lreg, Integer): + return self.comparison_op(self.coerce(lreg, rtype, line), rreg, op_id, line) + if is_fixed_width_rtype(lreg.type): + return self.comparison_op(lreg, rreg, op_id, line) + + if is_float_rprimitive(ltype) or is_float_rprimitive(rtype): + if isinstance(lreg, Integer): + lreg = Float(float(lreg.numeric_value())) + elif isinstance(rreg, Integer): + rreg = Float(float(rreg.numeric_value())) + elif is_int_rprimitive(lreg.type): + lreg = self.int_to_float(lreg, line) + elif is_int_rprimitive(rreg.type): + rreg = self.int_to_float(rreg, line) + if is_float_rprimitive(lreg.type) and is_float_rprimitive(rreg.type): + if op in float_comparison_op_to_id: + return self.compare_floats(lreg, rreg, float_comparison_op_to_id[op], line) + if op.endswith("="): + base_op = op[:-1] + else: + base_op = op + if base_op in float_op_to_id: + return self.float_op(lreg, rreg, base_op, line) + + dunder_op = self.dunder_op(lreg, rreg, op, line) + if dunder_op: + return dunder_op + + primitive_ops_candidates = binary_ops.get(op, []) + target = self.matching_primitive_op(primitive_ops_candidates, [lreg, rreg], line) + assert target, "Unsupported binary operation: %s" % op + return target + + def dunder_op(self, lreg: Value, rreg: Value | None, op: str, line: int) -> Value | None: + """ + Dispatch a dunder method if applicable. + For example for `a + b` it will use `a.__add__(b)` which can lead to higher performance + due to the fact that the method could be already compiled and optimized instead of going + all the way through `PyNumber_Add(a, b)` python api (making a jump into the python DL). + """ + ltype = lreg.type + if not isinstance(ltype, RInstance): + return None + + method_name = op_methods.get(op) if rreg else unary_op_methods.get(op) + if method_name is None: + return None + + if not ltype.class_ir.has_method(method_name): + return None + + decl = ltype.class_ir.method_decl(method_name) + if not rreg and len(decl.sig.args) != 1: + return None + + if rreg and (len(decl.sig.args) != 2 or not is_subtype(rreg.type, decl.sig.args[1].type)): + return None + + if rreg and is_subtype(not_implemented_op.type, decl.sig.ret_type): + # If the method is able to return NotImplemented, we should not optimize it. + # We can just let go so it will be handled through the python api. + return None + + args = [rreg] if rreg else [] + return self.gen_method_call(lreg, method_name, args, decl.sig.ret_type, line) + + def check_tagged_short_int(self, val: Value, line: int, negated: bool = False) -> Value: + """Check if a tagged integer is a short integer. + + Return the result of the check (value of type 'bit'). + """ + int_tag = Integer(1, c_pyssize_t_rprimitive, line) + bitwise_and = self.int_op(c_pyssize_t_rprimitive, val, int_tag, IntOp.AND, line) + zero = Integer(0, c_pyssize_t_rprimitive, line) + op = ComparisonOp.NEQ if negated else ComparisonOp.EQ + check = self.comparison_op(bitwise_and, zero, op, line) + return check + + def compare_strings(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: + """Compare two strings""" + compare_result = self.call_c(unicode_compare, [lhs, rhs], line) + error_constant = Integer(-1, c_int_rprimitive, line) + compare_error_check = self.add( + ComparisonOp(compare_result, error_constant, ComparisonOp.EQ, line) + ) + exception_check, propagate, final_compare = BasicBlock(), BasicBlock(), BasicBlock() + branch = Branch(compare_error_check, exception_check, final_compare, Branch.BOOL) + branch.negated = False + self.add(branch) + self.activate_block(exception_check) + check_error_result = self.call_c(err_occurred_op, [], line) + null = Integer(0, pointer_rprimitive, line) + compare_error_check = self.add( + ComparisonOp(check_error_result, null, ComparisonOp.NEQ, line) + ) + branch = Branch(compare_error_check, propagate, final_compare, Branch.BOOL) + branch.negated = False + self.add(branch) + self.activate_block(propagate) + self.call_c(keep_propagating_op, [], line) + self.goto(final_compare) + self.activate_block(final_compare) + op_type = ComparisonOp.EQ if op == "==" else ComparisonOp.NEQ + return self.add(ComparisonOp(compare_result, Integer(0, c_int_rprimitive), op_type, line)) + + def compare_bytes(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: + compare_result = self.call_c(bytes_compare, [lhs, rhs], line) + op_type = ComparisonOp.EQ if op == "==" else ComparisonOp.NEQ + return self.add(ComparisonOp(compare_result, Integer(1, c_int_rprimitive), op_type, line)) + + def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int = -1) -> Value: + """Compare two tuples item by item""" + # type cast to pass mypy check + assert isinstance(lhs.type, RTuple) and isinstance(rhs.type, RTuple) + equal = True if op == "==" else False + result = Register(bool_rprimitive) + # empty tuples + if len(lhs.type.types) == 0 and len(rhs.type.types) == 0: + self.add(Assign(result, self.true() if equal else self.false(), line)) + return result + length = len(lhs.type.types) + false_assign, true_assign, out = BasicBlock(), BasicBlock(), BasicBlock() + check_blocks = [BasicBlock() for _ in range(length)] + lhs_items = [self.add(TupleGet(lhs, i, line)) for i in range(length)] + rhs_items = [self.add(TupleGet(rhs, i, line)) for i in range(length)] + + if equal: + early_stop, final = false_assign, true_assign + else: + early_stop, final = true_assign, false_assign + + for i in range(len(lhs.type.types)): + if i != 0: + self.activate_block(check_blocks[i]) + lhs_item = lhs_items[i] + rhs_item = rhs_items[i] + compare = self.binary_op(lhs_item, rhs_item, op, line) + # Cast to bool if necessary since most types uses comparison returning a object type + # See generic_ops.py for more information + if not is_bool_rprimitive(compare.type): + compare = self.primitive_op(bool_op, [compare], line) + if i < len(lhs.type.types) - 1: + branch = Branch(compare, early_stop, check_blocks[i + 1], Branch.BOOL) + else: + branch = Branch(compare, early_stop, final, Branch.BOOL) + # if op is ==, we branch on false, else branch on true + branch.negated = equal + self.add(branch) + self.activate_block(false_assign) + self.add(Assign(result, self.false(), line)) + self.goto(out) + self.activate_block(true_assign) + self.add(Assign(result, self.true(), line)) + self.goto_and_activate(out) + return result + + def translate_instance_contains(self, inst: Value, item: Value, op: str, line: int) -> Value: + res = self.gen_method_call(inst, "__contains__", [item], None, line) + if not is_bool_rprimitive(res.type): + res = self.primitive_op(bool_op, [res], line) + if op == "not in": + res = self.bool_bitwise_op(res, Integer(1, rtype=bool_rprimitive), "^", line) + return res + + def bool_bitwise_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: + if op == "&": + code = IntOp.AND + elif op == "|": + code = IntOp.OR + elif op == "^": + code = IntOp.XOR + else: + assert False, op + return self.add(IntOp(bool_rprimitive, lreg, rreg, code, line)) + + def bool_comparison_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: + op_id = ComparisonOp.signed_ops[op] + return self.comparison_op(lreg, rreg, op_id, line) + + def unary_not(self, value: Value, line: int) -> Value: + mask = Integer(1, value.type, line) + return self.int_op(value.type, value, mask, IntOp.XOR, line) + + def unary_op(self, value: Value, expr_op: str, line: int) -> Value: + typ = value.type + if is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + if expr_op == "not": + return self.unary_not(value, line) + if expr_op == "+": + return value + if is_fixed_width_rtype(typ): + if expr_op == "-": + # Translate to '0 - x' + return self.int_op(typ, Integer(0, typ), value, IntOp.SUB, line) + elif expr_op == "~": + if typ.is_signed: + # Translate to 'x ^ -1' + return self.int_op(typ, value, Integer(-1, typ), IntOp.XOR, line) + else: + # Translate to 'x ^ 0xff...' + mask = (1 << (typ.size * 8)) - 1 + return self.int_op(typ, value, Integer(mask, typ), IntOp.XOR, line) + elif expr_op == "+": + return value + if is_float_rprimitive(typ): + if expr_op == "-": + return self.add(FloatNeg(value, line)) + elif expr_op == "+": + return value + + if isinstance(value, Integer): + # TODO: Overflow? Unsigned? + num = value.value + if is_short_int_rprimitive(typ): + num >>= 1 + return Integer(-num, typ, value.line) + if is_tagged(typ) and expr_op == "+": + return value + if isinstance(value, Float): + return Float(-value.value, value.line) + if isinstance(typ, RInstance): + result = self.dunder_op(value, None, expr_op, line) + if result is not None: + return result + primitive_ops_candidates = unary_ops.get(expr_op, []) + target = self.matching_primitive_op(primitive_ops_candidates, [value], line) + assert target, "Unsupported unary operation: %s" % expr_op + return target + + def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: + result: Value | None = None + keys: list[Value] = [] + values: list[Value] = [] + for key, value in key_value_pairs: + if key is not None: + # key:value + if result is None: + keys.append(key) + values.append(value) + continue + + self.translate_special_method_call( + result, "__setitem__", [key, value], result_type=None, line=line + ) + else: + # **value + if result is None: + result = self._create_dict(keys, values, line) + + self.call_c(dict_update_in_display_op, [result, value], line=line) + + if result is None: + result = self._create_dict(keys, values, line) + + return result + + def new_list_op_with_length(self, length: Value, line: int) -> Value: + """This function returns an uninitialized list. + + If the length is non-zero, the caller must initialize the list, before + it can be made visible to user code -- otherwise the list object is broken. + You might need further initialization with `new_list_set_item_op` op. + + Args: + length: desired length of the new list. The rtype should be + c_pyssize_t_rprimitive + line: line number + """ + return self.call_c(new_list_op, [length], line) + + def new_list_op(self, values: list[Value], line: int) -> Value: + length: list[Value] = [Integer(len(values), c_pyssize_t_rprimitive, line)] + if len(values) >= LIST_BUILDING_EXPANSION_THRESHOLD: + return self.call_c(list_build_op, length + values, line) + + # If the length of the list is less than the threshold, + # LIST_BUILDING_EXPANSION_THRESHOLD, we directly expand the + # for-loop and inline the SetMem operation, which is faster + # than list_build_op, however generates more code. + result_list = self.call_c(new_list_op, length, line) + if not values: + return result_list + args = [self.coerce(item, object_rprimitive, line) for item in values] + ob_item_base = self.add(PrimitiveOp([result_list], list_items, line)) + for i in range(len(values)): + self.primitive_op( + buf_init_item, [ob_item_base, Integer(i, c_pyssize_t_rprimitive), args[i]], line + ) + self.add(KeepAlive([result_list])) + return result_list + + def new_set_op(self, values: list[Value], line: int) -> Value: + return self.primitive_op(new_set_op, values, line) + + def setup_rarray( + self, item_type: RType, values: Sequence[Value], *, object_ptr: bool = False + ) -> Value: + """Declare and initialize a new RArray, returning its address.""" + array = Register(RArray(item_type, len(values))) + self.add(AssignMulti(array, list(values))) + return self.add( + LoadAddress(object_pointer_rprimitive if object_ptr else c_pointer_rprimitive, array) + ) + + def shortcircuit_helper( + self, + op: str, + expr_type: RType, + left: Callable[[], Value], + right: Callable[[], Value], + line: int, + ) -> Value: + # Having actual Phi nodes would be really nice here! + target = Register(expr_type) + # left_body takes the value of the left side, right_body the right + left_body, right_body, next_block = BasicBlock(), BasicBlock(), BasicBlock() + # true_body is taken if the left is true, false_body if it is false. + # For 'and' the value is the right side if the left is true, and for 'or' + # it is the right side if the left is false. + true_body, false_body = (right_body, left_body) if op == "and" else (left_body, right_body) + + left_value = left() + self.add_bool_branch(left_value, true_body, false_body) + + self.activate_block(left_body) + left_coerced = self.coerce(left_value, expr_type, line) + self.add(Assign(target, left_coerced)) + self.goto(next_block) + + self.activate_block(right_body) + right_value = right() + right_coerced = self.coerce(right_value, expr_type, line) + self.add(Assign(target, right_coerced)) + self.goto(next_block) + + self.activate_block(next_block) + return target + + def bool_value(self, value: Value) -> Value: + """Return bool(value). + + The result type can be bit_rprimitive or bool_rprimitive. + """ + if is_bool_rprimitive(value.type) or is_bit_rprimitive(value.type): + result = value + elif is_runtime_subtype(value.type, int_rprimitive): + zero = Integer(0, short_int_rprimitive) + result = self.comparison_op(value, zero, ComparisonOp.NEQ, value.line) + elif is_fixed_width_rtype(value.type): + zero = Integer(0, value.type) + result = self.add(ComparisonOp(value, zero, ComparisonOp.NEQ)) + elif is_same_type(value.type, str_rprimitive): + result = self.call_c(str_check_if_true, [value], value.line) + elif is_same_type(value.type, list_rprimitive) or is_same_type( + value.type, dict_rprimitive + ): + length = self.builtin_len(value, value.line) + zero = Integer(0) + result = self.binary_op(length, zero, "!=", value.line) + elif ( + isinstance(value.type, RInstance) + and value.type.class_ir.is_ext_class + and value.type.class_ir.has_method("__bool__") + ): + # Directly call the __bool__ method on classes that have it. + result = self.gen_method_call(value, "__bool__", [], bool_rprimitive, value.line) + elif is_float_rprimitive(value.type): + result = self.compare_floats(value, Float(0.0), FloatComparisonOp.NEQ, value.line) + else: + value_type = optional_value_type(value.type) + if value_type is not None: + not_none = self.translate_is_op(value, self.none_object(), "is not", value.line) + always_truthy = False + if isinstance(value_type, RInstance): + # check whether X.__bool__ is always just the default (object.__bool__) + if not value_type.class_ir.has_method( + "__bool__" + ) and value_type.class_ir.is_method_final("__bool__"): + always_truthy = True + + if always_truthy: + result = not_none + else: + # "X | None" where X may be falsey and requires a check + result = Register(bit_rprimitive) + true, false, end = BasicBlock(), BasicBlock(), BasicBlock() + branch = Branch(not_none, true, false, Branch.BOOL) + self.add(branch) + self.activate_block(true) + # unbox_or_cast instead of coerce because we want the + # type to change even if it is a subtype. + remaining = self.unbox_or_cast(value, value_type, value.line) + as_bool = self.bool_value(remaining) + self.add(Assign(result, as_bool)) + self.goto(end) + self.activate_block(false) + self.add(Assign(result, Integer(0, bit_rprimitive))) + self.goto(end) + self.activate_block(end) + else: + result = self.primitive_op(bool_op, [value], value.line) + return result + + def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) -> None: + opt_value_type = optional_value_type(value.type) + if opt_value_type is None: + bool_value = self.bool_value(value) + self.add(Branch(bool_value, true, false, Branch.BOOL)) + else: + # Special-case optional types + is_none = self.translate_is_op(value, self.none_object(), "is not", value.line) + branch = Branch(is_none, true, false, Branch.BOOL) + self.add(branch) + always_truthy = False + if isinstance(opt_value_type, RInstance): + # check whether X.__bool__ is always just the default (object.__bool__) + if not opt_value_type.class_ir.has_method( + "__bool__" + ) and opt_value_type.class_ir.is_method_final("__bool__"): + always_truthy = True + + if not always_truthy: + # Optional[X] where X may be falsey and requires a check + branch.true = BasicBlock() + self.activate_block(branch.true) + # unbox_or_cast instead of coerce because we want the + # type to change even if it is a subtype. + remaining = self.unbox_or_cast(value, opt_value_type, value.line) + self.add_bool_branch(remaining, true, false) + + def call_c( + self, + desc: CFunctionDescription, + args: list[Value], + line: int, + result_type: RType | None = None, + ) -> Value: + """Call function using C/native calling convention (not a Python callable).""" + # Handle void function via singleton RVoid instance + coerced = [] + # Coerce fixed number arguments + for i in range(min(len(args), len(desc.arg_types))): + formal_type = desc.arg_types[i] + arg = args[i] + arg = self.coerce(arg, formal_type, line) + coerced.append(arg) + # Reorder args if necessary + if desc.ordering is not None: + assert desc.var_arg_type is None + coerced = [coerced[i] for i in desc.ordering] + # Coerce any var_arg + var_arg_idx = -1 + if desc.var_arg_type is not None: + var_arg_idx = len(desc.arg_types) + for i in range(len(desc.arg_types), len(args)): + arg = args[i] + arg = self.coerce(arg, desc.var_arg_type, line) + coerced.append(arg) + # Add extra integer constant if any + for item in desc.extra_int_constants: + val, typ = item + extra_int_constant = Integer(val, typ, line) + coerced.append(extra_int_constant) + error_kind = desc.error_kind + if error_kind == ERR_NEG_INT: + # Handled with an explicit comparison + error_kind = ERR_NEVER + target = self.add( + CallC( + desc.c_function_name, + coerced, + desc.return_type, + desc.steals, + desc.is_borrowed, + error_kind, + line, + var_arg_idx, + is_pure=desc.is_pure, + ) + ) + if desc.is_borrowed: + # If the result is borrowed, force the arguments to be + # kept alive afterwards, as otherwise the result might be + # immediately freed, at the risk of a dangling pointer. + for arg in coerced: + if not isinstance(arg, (Integer, LoadLiteral)): + self.keep_alives.append(arg) + if desc.error_kind == ERR_NEG_INT: + comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) + comp.error_kind = ERR_FALSE + self.add(comp) + + if desc.truncated_type is None: + result = target + else: + truncate = self.add(Truncate(target, desc.truncated_type)) + result = truncate + if result_type and not is_runtime_subtype(result.type, result_type): + if is_none_rprimitive(result_type): + # Special case None return. The actual result may actually be a bool + # and so we can't just coerce it. + result = self.none() + else: + result = self.coerce(target, result_type, line, can_borrow=desc.is_borrowed) + return result + + def matching_call_c( + self, + candidates: list[CFunctionDescription], + args: list[Value], + line: int, + result_type: RType | None = None, + can_borrow: bool = False, + ) -> Value | None: + matching: CFunctionDescription | None = None + for desc in candidates: + if len(desc.arg_types) != len(args): + continue + if all( + is_subtype(actual.type, formal) for actual, formal in zip(args, desc.arg_types) + ) and (not desc.is_borrowed or can_borrow): + if matching: + assert matching.priority != desc.priority, "Ambiguous:\n1) {}\n2) {}".format( + matching, desc + ) + if desc.priority > matching.priority: + matching = desc + else: + matching = desc + if matching: + target = self.call_c(matching, args, line, result_type) + return target + return None + + def primitive_op( + self, + desc: PrimitiveDescription, + args: list[Value], + line: int, + result_type: RType | None = None, + ) -> Value: + """Add a primitive op.""" + # Does this primitive map into calling a Python C API + # or an internal mypyc C API function? + if desc.c_function_name: + # TODO: Generate PrimitiveOps here and transform them into CallC + # ops only later in the lowering pass + c_desc = CFunctionDescription( + desc.name, + desc.arg_types, + desc.return_type, + desc.var_arg_type, + desc.truncated_type, + desc.c_function_name, + desc.error_kind, + desc.steals, + desc.is_borrowed, + desc.ordering, + desc.extra_int_constants, + desc.priority, + is_pure=desc.is_pure, + ) + return self.call_c(c_desc, args, line, result_type=result_type) + + # This primitive gets transformed in a lowering pass to + # lower-level IR ops using a custom transform function. + + coerced = [] + # Coerce fixed number arguments + for i in range(min(len(args), len(desc.arg_types))): + formal_type = desc.arg_types[i] + arg = args[i] + assert formal_type is not None # TODO + arg = self.coerce(arg, formal_type, line) + coerced.append(arg) + assert desc.ordering is None + assert desc.var_arg_type is None + assert not desc.extra_int_constants + target = self.add(PrimitiveOp(coerced, desc, line=line)) + if desc.is_borrowed: + # If the result is borrowed, force the arguments to be + # kept alive afterwards, as otherwise the result might be + # immediately freed, at the risk of a dangling pointer. + for arg in coerced: + if not isinstance(arg, (Integer, LoadLiteral)): + self.keep_alives.append(arg) + if desc.error_kind == ERR_NEG_INT: + comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) + comp.error_kind = ERR_FALSE + self.add(comp) + + assert desc.truncated_type is None + result = target + if result_type and not is_runtime_subtype(result.type, result_type): + if is_none_rprimitive(result_type): + # Special case None return. The actual result may actually be a bool + # and so we can't just coerce it. + result = self.none() + else: + result = self.coerce(result, result_type, line, can_borrow=desc.is_borrowed) + return result + + def matching_primitive_op( + self, + candidates: list[PrimitiveDescription], + args: list[Value], + line: int, + result_type: RType | None = None, + can_borrow: bool = False, + ) -> Value | None: + matching: PrimitiveDescription | None = None + for desc in candidates: + if len(desc.arg_types) != len(args): + continue + if all( + # formal is not None and # TODO + is_subtype(actual.type, formal) + for actual, formal in zip(args, desc.arg_types) + ) and (not desc.is_borrowed or can_borrow): + if matching: + assert matching.priority != desc.priority, "Ambiguous:\n1) {}\n2) {}".format( + matching, desc + ) + if desc.priority > matching.priority: + matching = desc + else: + matching = desc + if matching: + return self.primitive_op(matching, args, line=line, result_type=result_type) + return None + + def int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int = -1) -> Value: + """Generate a native integer binary op. + + Use native/C semantics, which sometimes differ from Python + semantics. + + Args: + type: Either int64_rprimitive or int32_rprimitive + op: IntOp.* constant (e.g. IntOp.ADD) + """ + return self.add(IntOp(type, lhs, rhs, op, line)) + + def float_op(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: + """Generate a native float binary arithmetic operation. + + This follows Python semantics (e.g. raise exception on division by zero). + Add a FloatOp directly if you want low-level semantics. + + Args: + op: Binary operator (e.g. '+' or '*') + """ + op_id = float_op_to_id[op] + if op_id in (FloatOp.DIV, FloatOp.MOD): + if not (isinstance(rhs, Float) and rhs.value != 0.0): + c = self.compare_floats(rhs, Float(0.0), FloatComparisonOp.EQ, line) + err, ok = BasicBlock(), BasicBlock() + self.add(Branch(c, err, ok, Branch.BOOL, rare=True)) + self.activate_block(err) + if op_id == FloatOp.DIV: + msg = "float division by zero" + else: + msg = "float modulo" + self.add(RaiseStandardError(RaiseStandardError.ZERO_DIVISION_ERROR, msg, line)) + self.add(Unreachable()) + self.activate_block(ok) + if op_id == FloatOp.MOD: + # Adjust the result to match Python semantics (FloatOp follows C semantics). + return self.float_mod(lhs, rhs, line) + else: + return self.add(FloatOp(lhs, rhs, op_id, line)) + + def float_mod(self, lhs: Value, rhs: Value, line: int) -> Value: + """Perform x % y on floats using Python semantics.""" + mod = self.add(FloatOp(lhs, rhs, FloatOp.MOD, line)) + res = Register(float_rprimitive) + self.add(Assign(res, mod)) + tricky, adjust, copysign, done = BasicBlock(), BasicBlock(), BasicBlock(), BasicBlock() + is_zero = self.add(FloatComparisonOp(res, Float(0.0), FloatComparisonOp.EQ, line)) + self.add(Branch(is_zero, copysign, tricky, Branch.BOOL)) + self.activate_block(tricky) + same_signs = self.is_same_float_signs(lhs, rhs, line) + self.add(Branch(same_signs, done, adjust, Branch.BOOL)) + self.activate_block(adjust) + adj = self.float_op(res, rhs, "+", line) + self.add(Assign(res, adj)) + self.add(Goto(done)) + self.activate_block(copysign) + # If the remainder is zero, CPython ensures the result has the + # same sign as the denominator. + adj = self.primitive_op(copysign_op, [Float(0.0), rhs], line) + self.add(Assign(res, adj)) + self.add(Goto(done)) + self.activate_block(done) + return res + + def compare_floats(self, lhs: Value, rhs: Value, op: int, line: int) -> Value: + return self.add(FloatComparisonOp(lhs, rhs, op, line)) + + def fixed_width_int_op( + self, type: RPrimitive, lhs: Value, rhs: Value, op: int, line: int + ) -> Value: + """Generate a binary op using Python fixed-width integer semantics. + + These may differ in overflow/rounding behavior from native/C ops. + + Args: + type: Either int64_rprimitive or int32_rprimitive + op: IntOp.* constant (e.g. IntOp.ADD) + """ + lhs = self.coerce(lhs, type, line) + rhs = self.coerce(rhs, type, line) + if op == IntOp.DIV: + if isinstance(rhs, Integer) and rhs.value not in (-1, 0): + if not type.is_signed: + return self.int_op(type, lhs, rhs, IntOp.DIV, line) + else: + # Inline simple division by a constant, so that C + # compilers can optimize more + return self.inline_fixed_width_divide(type, lhs, rhs, line) + if is_int64_rprimitive(type): + prim = int64_divide_op + elif is_int32_rprimitive(type): + prim = int32_divide_op + elif is_int16_rprimitive(type): + prim = int16_divide_op + elif is_uint8_rprimitive(type): + self.check_for_zero_division(rhs, type, line) + return self.int_op(type, lhs, rhs, op, line) + else: + assert False, type + return self.call_c(prim, [lhs, rhs], line) + if op == IntOp.MOD: + if isinstance(rhs, Integer) and rhs.value not in (-1, 0): + if not type.is_signed: + return self.int_op(type, lhs, rhs, IntOp.MOD, line) + else: + # Inline simple % by a constant, so that C + # compilers can optimize more + return self.inline_fixed_width_mod(type, lhs, rhs, line) + if is_int64_rprimitive(type): + prim = int64_mod_op + elif is_int32_rprimitive(type): + prim = int32_mod_op + elif is_int16_rprimitive(type): + prim = int16_mod_op + elif is_uint8_rprimitive(type): + self.check_for_zero_division(rhs, type, line) + return self.int_op(type, lhs, rhs, op, line) + else: + assert False, type + return self.call_c(prim, [lhs, rhs], line) + return self.int_op(type, lhs, rhs, op, line) + + def check_for_zero_division(self, rhs: Value, type: RType, line: int) -> None: + err, ok = BasicBlock(), BasicBlock() + is_zero = self.binary_op(rhs, Integer(0, type), "==", line) + self.add(Branch(is_zero, err, ok, Branch.BOOL)) + self.activate_block(err) + self.add( + RaiseStandardError( + RaiseStandardError.ZERO_DIVISION_ERROR, "integer division or modulo by zero", line + ) + ) + self.add(Unreachable()) + self.activate_block(ok) + + def inline_fixed_width_divide(self, type: RType, lhs: Value, rhs: Value, line: int) -> Value: + # Perform floor division (native division truncates) + res = Register(type) + div = self.int_op(type, lhs, rhs, IntOp.DIV, line) + self.add(Assign(res, div)) + same_signs = self.is_same_native_int_signs(type, lhs, rhs, line) + tricky, adjust, done = BasicBlock(), BasicBlock(), BasicBlock() + self.add(Branch(same_signs, done, tricky, Branch.BOOL)) + self.activate_block(tricky) + mul = self.int_op(type, res, rhs, IntOp.MUL, line) + mul_eq = self.add(ComparisonOp(mul, lhs, ComparisonOp.EQ, line)) + self.add(Branch(mul_eq, done, adjust, Branch.BOOL)) + self.activate_block(adjust) + adj = self.int_op(type, res, Integer(1, type), IntOp.SUB, line) + self.add(Assign(res, adj)) + self.add(Goto(done)) + self.activate_block(done) + return res + + def inline_fixed_width_mod(self, type: RType, lhs: Value, rhs: Value, line: int) -> Value: + # Perform floor modulus + res = Register(type) + mod = self.int_op(type, lhs, rhs, IntOp.MOD, line) + self.add(Assign(res, mod)) + same_signs = self.is_same_native_int_signs(type, lhs, rhs, line) + tricky, adjust, done = BasicBlock(), BasicBlock(), BasicBlock() + self.add(Branch(same_signs, done, tricky, Branch.BOOL)) + self.activate_block(tricky) + is_zero = self.add(ComparisonOp(res, Integer(0, type), ComparisonOp.EQ, line)) + self.add(Branch(is_zero, done, adjust, Branch.BOOL)) + self.activate_block(adjust) + adj = self.int_op(type, res, rhs, IntOp.ADD, line) + self.add(Assign(res, adj)) + self.add(Goto(done)) + self.activate_block(done) + return res + + def is_same_native_int_signs(self, type: RType, a: Value, b: Value, line: int) -> Value: + neg1 = self.add(ComparisonOp(a, Integer(0, type), ComparisonOp.SLT, line)) + neg2 = self.add(ComparisonOp(b, Integer(0, type), ComparisonOp.SLT, line)) + return self.add(ComparisonOp(neg1, neg2, ComparisonOp.EQ, line)) + + def is_same_float_signs(self, a: Value, b: Value, line: int) -> Value: + neg1 = self.add(FloatComparisonOp(a, Float(0.0), FloatComparisonOp.LT, line)) + neg2 = self.add(FloatComparisonOp(b, Float(0.0), FloatComparisonOp.LT, line)) + return self.add(ComparisonOp(neg1, neg2, ComparisonOp.EQ, line)) + + def comparison_op(self, lhs: Value, rhs: Value, op: int, line: int) -> Value: + return self.add(ComparisonOp(lhs, rhs, op, line)) + + def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Value: + """Generate len(val). + + Return short_int_rprimitive by default. + Return c_pyssize_t if use_pyssize_t is true (unshifted). + """ + typ = val.type + size_value = None + if is_list_rprimitive(typ) or is_tuple_rprimitive(typ) or is_bytes_rprimitive(typ): + size_value = self.primitive_op(var_object_size, [val], line) + elif is_set_rprimitive(typ): + elem_address = self.add(GetElementPtr(val, PySetObject, "used")) + size_value = self.add(LoadMem(c_pyssize_t_rprimitive, elem_address)) + self.add(KeepAlive([val])) + elif is_dict_rprimitive(typ): + size_value = self.call_c(dict_ssize_t_size_op, [val], line) + elif is_str_rprimitive(typ): + size_value = self.call_c(str_ssize_t_size_op, [val], line) + + if size_value is not None: + if use_pyssize_t: + return size_value + offset = Integer(1, c_pyssize_t_rprimitive, line) + return self.int_op(short_int_rprimitive, size_value, offset, IntOp.LEFT_SHIFT, line) + + if isinstance(typ, RInstance): + # TODO: Support use_pyssize_t + assert not use_pyssize_t + length = self.gen_method_call(val, "__len__", [], int_rprimitive, line) + length = self.coerce(length, int_rprimitive, line) + ok, fail = BasicBlock(), BasicBlock() + cond = self.binary_op(length, Integer(0), ">=", line) + self.add_bool_branch(cond, ok, fail) + self.activate_block(fail) + self.add( + RaiseStandardError( + RaiseStandardError.VALUE_ERROR, "__len__() should return >= 0", line + ) + ) + self.add(Unreachable()) + self.activate_block(ok) + return length + + # generic case + if use_pyssize_t: + return self.call_c(generic_ssize_t_len_op, [val], line) + else: + return self.call_c(generic_len_op, [val], line) + + def new_tuple(self, items: list[Value], line: int) -> Value: + size: Value = Integer(len(items), c_pyssize_t_rprimitive) + return self.call_c(new_tuple_op, [size] + items, line) + + def new_tuple_with_length(self, length: Value, line: int) -> Value: + """This function returns an uninitialized tuple. + + If the length is non-zero, the caller must initialize the tuple, before + it can be made visible to user code -- otherwise the tuple object is broken. + You might need further initialization with `new_tuple_set_item_op` op. + + Args: + length: desired length of the new tuple. The rtype should be + c_pyssize_t_rprimitive + line: line number + """ + return self.call_c(new_tuple_with_length_op, [length], line) + + def int_to_float(self, n: Value, line: int) -> Value: + return self.primitive_op(int_to_float_op, [n], line) + + # Internal helpers + + def decompose_union_helper( + self, + obj: Value, + rtype: RUnion, + result_type: RType, + process_item: Callable[[Value], Value], + line: int, + ) -> Value: + """Generate isinstance() + specialized operations for union items. + + Say, for Union[A, B] generate ops resembling this (pseudocode): + + if isinstance(obj, A): + result = + else: + result = + + Args: + obj: value with a union type + rtype: the union type + result_type: result of the operation + process_item: callback to generate op for a single union item (arg is coerced + to union item type) + line: line number + """ + # TODO: Optimize cases where a single operation can handle multiple union items + # (say a method is implemented in a common base class) + fast_items = [] + rest_items = [] + for item in rtype.items: + if isinstance(item, RInstance): + fast_items.append(item) + else: + # For everything but RInstance we fall back to C API + rest_items.append(item) + exit_block = BasicBlock() + result = Register(result_type) + for i, item in enumerate(fast_items): + more_types = i < len(fast_items) - 1 or rest_items + if more_types: + # We are not at the final item so we need one more branch + op = self.isinstance_native(obj, item.class_ir, line) + true_block, false_block = BasicBlock(), BasicBlock() + self.add_bool_branch(op, true_block, false_block) + self.activate_block(true_block) + coerced = self.coerce(obj, item, line) + temp = process_item(coerced) + temp2 = self.coerce(temp, result_type, line) + self.add(Assign(result, temp2)) + self.goto(exit_block) + if more_types: + self.activate_block(false_block) + if rest_items: + # For everything else we use generic operation. Use force=True to drop the + # union type. + coerced = self.coerce(obj, object_rprimitive, line, force=True) + temp = process_item(coerced) + temp2 = self.coerce(temp, result_type, line) + self.add(Assign(result, temp2)) + self.goto(exit_block) + self.activate_block(exit_block) + return result + + def translate_special_method_call( + self, + base_reg: Value, + name: str, + args: list[Value], + result_type: RType | None, + line: int, + can_borrow: bool = False, + ) -> Value | None: + """Translate a method call which is handled nongenerically. + + These are special in the sense that we have code generated specifically for them. + They tend to be method calls which have equivalents in C that are more direct + than calling with the PyObject api. + + Return None if no translation found; otherwise return the target register. + """ + primitive_ops_candidates = method_call_ops.get(name, []) + primitive_op = self.matching_primitive_op( + primitive_ops_candidates, [base_reg] + args, line, result_type, can_borrow=can_borrow + ) + return primitive_op + + def translate_eq_cmp(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> Value | None: + """Add a equality comparison operation. + + Args: + expr_op: either '==' or '!=' + """ + ltype = lreg.type + rtype = rreg.type + if not (isinstance(ltype, RInstance) and ltype == rtype): + return None + + class_ir = ltype.class_ir + # Check whether any subclasses of the operand redefines __eq__ + # or it might be redefined in a Python parent class or by + # dataclasses + cmp_varies_at_runtime = ( + not class_ir.is_method_final("__eq__") + or not class_ir.is_method_final("__ne__") + or class_ir.inherits_python + or class_ir.is_augmented + ) + + if cmp_varies_at_runtime: + # We might need to call left.__eq__(right) or right.__eq__(left) + # depending on which is the more specific type. + return None + + if not class_ir.has_method("__eq__"): + # There's no __eq__ defined, so just use object identity. + identity_ref_op = "is" if expr_op == "==" else "is not" + return self.translate_is_op(lreg, rreg, identity_ref_op, line) + + return self.gen_method_call(lreg, op_methods[expr_op], [rreg], ltype, line) + + def translate_is_op(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> Value: + """Create equality comparison operation between object identities + + Args: + expr_op: either 'is' or 'is not' + """ + op = ComparisonOp.EQ if expr_op == "is" else ComparisonOp.NEQ + lhs = self.coerce(lreg, object_rprimitive, line) + rhs = self.coerce(rreg, object_rprimitive, line) + return self.add(ComparisonOp(lhs, rhs, op, line)) + + def _create_dict(self, keys: list[Value], values: list[Value], line: int) -> Value: + """Create a dictionary(possibly empty) using keys and values""" + # keys and values should have the same number of items + size = len(keys) + if size > 0: + size_value: Value = Integer(size, c_pyssize_t_rprimitive) + # merge keys and values + items = [i for t in list(zip(keys, values)) for i in t] + return self.call_c(dict_build_op, [size_value] + items, line) + else: + return self.call_c(dict_new_op, [], line) + + def error(self, msg: str, line: int) -> None: + assert self.errors is not None, "cannot generate errors in this compiler phase" + self.errors.error(msg, self.module_path, line) + + +def num_positional_args(arg_values: list[Value], arg_kinds: list[ArgKind] | None) -> int: + if arg_kinds is None: + return len(arg_values) + num_pos = 0 + for kind in arg_kinds: + if kind == ARG_POS: + num_pos += 1 + return num_pos diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py new file mode 100644 index 000000000000..15928d939cbf --- /dev/null +++ b/mypyc/irbuild/main.py @@ -0,0 +1,154 @@ +"""Transform a mypy AST to the IR form (Intermediate Representation). + +For example, consider a function like this: + + def f(x: int) -> int: + return x * 2 + 1 + +It would be translated to something that conceptually looks like this: + + r0 = 2 + r1 = 1 + r2 = x * r0 :: int + r3 = r2 + r1 :: int + return r3 + +This module deals with the module-level IR transformation logic and +putting it all together. The actual IR is implemented in mypyc.ir. + +For the core of the IR transform implementation, look at build_ir() +below, mypyc.irbuild.builder, and mypyc.irbuild.visitor. +""" + +from __future__ import annotations + +from typing import Any, Callable, TypeVar, cast + +from mypy.build import Graph +from mypy.nodes import ClassDef, Expression, MypyFile +from mypy.state import state +from mypy.types import Type +from mypyc.analysis.attrdefined import analyze_always_defined_attrs +from mypyc.common import TOP_LEVEL_NAME +from mypyc.errors import Errors +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature +from mypyc.ir.module_ir import ModuleIR, ModuleIRs +from mypyc.ir.rtypes import none_rprimitive +from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.mapper import Mapper +from mypyc.irbuild.prebuildvisitor import PreBuildVisitor +from mypyc.irbuild.prepare import build_type_map, find_singledispatch_register_impls +from mypyc.irbuild.visitor import IRBuilderVisitor +from mypyc.irbuild.vtable import compute_vtable +from mypyc.options import CompilerOptions + +# The stubs for callable contextmanagers are busted so cast it to the +# right type... +F = TypeVar("F", bound=Callable[..., Any]) +strict_optional_dec = cast(Callable[[F], F], state.strict_optional_set(True)) + + +@strict_optional_dec # Turn on strict optional for any type manipulations we do +def build_ir( + modules: list[MypyFile], + graph: Graph, + types: dict[Expression, Type], + mapper: Mapper, + options: CompilerOptions, + errors: Errors, +) -> ModuleIRs: + """Build basic IR for a set of modules that have been type-checked by mypy. + + The returned IR is not complete and requires additional + transformations, such as the insertion of refcount handling. + """ + + build_type_map(mapper, modules, graph, types, options, errors) + singledispatch_info = find_singledispatch_register_impls(modules, errors) + + result: ModuleIRs = {} + + # Generate IR for all modules. + class_irs = [] + + for module in modules: + # First pass to determine free symbols. + pbv = PreBuildVisitor(errors, module, singledispatch_info.decorators_to_remove) + module.accept(pbv) + + # Construct and configure builder objects (cyclic runtime dependency). + visitor = IRBuilderVisitor() + builder = IRBuilder( + module.fullname, + types, + graph, + errors, + mapper, + pbv, + visitor, + options, + singledispatch_info.singledispatch_impls, + ) + visitor.builder = builder + + # Second pass does the bulk of the work. + transform_mypy_file(builder, module) + module_ir = ModuleIR( + module.fullname, + list(builder.imports), + builder.functions, + builder.classes, + builder.final_names, + builder.type_var_names, + ) + result[module.fullname] = module_ir + class_irs.extend(builder.classes) + + analyze_always_defined_attrs(class_irs) + + # Compute vtables. + for cir in class_irs: + if cir.is_ext_class: + compute_vtable(cir) + + return result + + +def transform_mypy_file(builder: IRBuilder, mypyfile: MypyFile) -> None: + """Generate IR for a single module.""" + + if mypyfile.fullname in ("typing", "abc"): + # These module are special; their contents are currently all + # built-in primitives. + return + + builder.set_module(mypyfile.fullname, mypyfile.path) + + classes = [node for node in mypyfile.defs if isinstance(node, ClassDef)] + + # Collect all classes. + for cls in classes: + ir = builder.mapper.type_to_ir[cls.info] + builder.classes.append(ir) + + builder.enter("") + + # Make sure we have a builtins import + builder.gen_import("builtins", -1) + + # Generate ops. + for node in mypyfile.defs: + builder.accept(node) + + builder.maybe_add_implicit_return() + + # Generate special function representing module top level. + args, _, blocks, ret_type, _ = builder.leave() + sig = FuncSignature([], none_rprimitive) + func_ir = FuncIR( + FuncDecl(TOP_LEVEL_NAME, None, builder.module_name, sig), + args, + blocks, + traceback_name="", + ) + builder.functions.append(func_ir) diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py new file mode 100644 index 000000000000..9cd263c40ae4 --- /dev/null +++ b/mypyc/irbuild/mapper.py @@ -0,0 +1,226 @@ +"""Maintain a mapping from mypy concepts to IR/compiled concepts.""" + +from __future__ import annotations + +from mypy.nodes import ARG_STAR, ARG_STAR2, GDEF, ArgKind, FuncDef, RefExpr, SymbolNode, TypeInfo +from mypy.types import ( + AnyType, + CallableType, + Instance, + LiteralType, + NoneTyp, + Overloaded, + PartialType, + TupleType, + Type, + TypedDictType, + TypeType, + TypeVarLikeType, + UnboundType, + UninhabitedType, + UnionType, + find_unpack_in_list, + get_proper_type, +) +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncDecl, FuncSignature, RuntimeArg +from mypyc.ir.rtypes import ( + RInstance, + RTuple, + RType, + RUnion, + bool_rprimitive, + bytes_rprimitive, + dict_rprimitive, + float_rprimitive, + int16_rprimitive, + int32_rprimitive, + int64_rprimitive, + int_rprimitive, + list_rprimitive, + none_rprimitive, + object_rprimitive, + range_rprimitive, + set_rprimitive, + str_rprimitive, + tuple_rprimitive, + uint8_rprimitive, +) + + +class Mapper: + """Keep track of mappings from mypy concepts to IR concepts. + + For example, we keep track of how the mypy TypeInfos of compiled + classes map to class IR objects. + + This state is shared across all modules being compiled in all + compilation groups. + """ + + def __init__(self, group_map: dict[str, str | None]) -> None: + self.group_map = group_map + self.type_to_ir: dict[TypeInfo, ClassIR] = {} + self.func_to_decl: dict[SymbolNode, FuncDecl] = {} + self.symbol_fullnames: set[str] = set() + + def type_to_rtype(self, typ: Type | None) -> RType: + if typ is None: + return object_rprimitive + + typ = get_proper_type(typ) + if isinstance(typ, Instance): + if typ.type.fullname == "builtins.int": + return int_rprimitive + elif typ.type.fullname == "builtins.float": + return float_rprimitive + elif typ.type.fullname == "builtins.bool": + return bool_rprimitive + elif typ.type.fullname == "builtins.str": + return str_rprimitive + elif typ.type.fullname == "builtins.bytes": + return bytes_rprimitive + elif typ.type.fullname == "builtins.list": + return list_rprimitive + # Dict subclasses are at least somewhat common and we + # specifically support them, so make sure that dict operations + # get optimized on them. + elif any(cls.fullname == "builtins.dict" for cls in typ.type.mro): + return dict_rprimitive + elif typ.type.fullname == "builtins.set": + return set_rprimitive + elif typ.type.fullname == "builtins.tuple": + return tuple_rprimitive # Varying-length tuple + elif typ.type.fullname == "builtins.range": + return range_rprimitive + elif typ.type in self.type_to_ir: + inst = RInstance(self.type_to_ir[typ.type]) + # Treat protocols as Union[protocol, object], so that we can do fast + # method calls in the cases where the protocol is explicitly inherited from + # and fall back to generic operations when it isn't. + if typ.type.is_protocol: + return RUnion([inst, object_rprimitive]) + else: + return inst + elif typ.type.fullname == "mypy_extensions.i64": + return int64_rprimitive + elif typ.type.fullname == "mypy_extensions.i32": + return int32_rprimitive + elif typ.type.fullname == "mypy_extensions.i16": + return int16_rprimitive + elif typ.type.fullname == "mypy_extensions.u8": + return uint8_rprimitive + else: + return object_rprimitive + elif isinstance(typ, TupleType): + # Use our unboxed tuples for raw tuples but fall back to + # being boxed for NamedTuple or for variadic tuples. + if ( + typ.partial_fallback.type.fullname == "builtins.tuple" + and find_unpack_in_list(typ.items) is None + ): + return RTuple([self.type_to_rtype(t) for t in typ.items]) + else: + return tuple_rprimitive + elif isinstance(typ, CallableType): + return object_rprimitive + elif isinstance(typ, NoneTyp): + return none_rprimitive + elif isinstance(typ, UnionType): + return RUnion.make_simplified_union([self.type_to_rtype(item) for item in typ.items]) + elif isinstance(typ, AnyType): + return object_rprimitive + elif isinstance(typ, TypeType): + return object_rprimitive + elif isinstance(typ, TypeVarLikeType): + # Erase type variable to upper bound. + # TODO: Erase to union if object has value restriction? + return self.type_to_rtype(typ.upper_bound) + elif isinstance(typ, PartialType): + assert typ.var.type is not None + return self.type_to_rtype(typ.var.type) + elif isinstance(typ, Overloaded): + return object_rprimitive + elif isinstance(typ, TypedDictType): + return dict_rprimitive + elif isinstance(typ, LiteralType): + return self.type_to_rtype(typ.fallback) + elif isinstance(typ, (UninhabitedType, UnboundType)): + # Sure, whatever! + return object_rprimitive + + # I think we've covered everything that is supposed to + # actually show up, so anything else is a bug somewhere. + assert False, "unexpected type %s" % type(typ) + + def get_arg_rtype(self, typ: Type, kind: ArgKind) -> RType: + if kind == ARG_STAR: + return tuple_rprimitive + elif kind == ARG_STAR2: + return dict_rprimitive + else: + return self.type_to_rtype(typ) + + def fdef_to_sig(self, fdef: FuncDef, strict_dunders_typing: bool) -> FuncSignature: + if isinstance(fdef.type, CallableType): + arg_types = [ + self.get_arg_rtype(typ, kind) + for typ, kind in zip(fdef.type.arg_types, fdef.type.arg_kinds) + ] + arg_pos_onlys = [name is None for name in fdef.type.arg_names] + ret = self.type_to_rtype(fdef.type.ret_type) + else: + # Handle unannotated functions + arg_types = [object_rprimitive for _ in fdef.arguments] + arg_pos_onlys = [arg.pos_only for arg in fdef.arguments] + # We at least know the return type for __init__ methods will be None. + is_init_method = fdef.name == "__init__" and bool(fdef.info) + if is_init_method: + ret = none_rprimitive + else: + ret = object_rprimitive + + # mypyc FuncSignatures (unlike mypy types) want to have a name + # present even when the argument is position only, since it is + # the sole way that FuncDecl arguments are tracked. This is + # generally fine except in some cases (like for computing + # init_sig) we need to produce FuncSignatures from a + # deserialized FuncDef that lacks arguments. We won't ever + # need to use those inside of a FuncIR, so we just make up + # some crap. + if hasattr(fdef, "arguments"): + arg_names = [arg.variable.name for arg in fdef.arguments] + else: + arg_names = [name or "" for name in fdef.arg_names] + + args = [ + RuntimeArg(arg_name, arg_type, arg_kind, arg_pos_only) + for arg_name, arg_kind, arg_type, arg_pos_only in zip( + arg_names, fdef.arg_kinds, arg_types, arg_pos_onlys + ) + ] + + if not strict_dunders_typing: + # We force certain dunder methods to return objects to support letting them + # return NotImplemented. It also avoids some pointless boxing and unboxing, + # since tp_richcompare needs an object anyways. + # However, it also prevents some optimizations. + if fdef.name in ("__eq__", "__ne__", "__lt__", "__gt__", "__le__", "__ge__"): + ret = object_rprimitive + + return FuncSignature(args, ret) + + def is_native_module(self, module: str) -> bool: + """Is the given module one compiled by mypyc?""" + return module in self.group_map + + def is_native_ref_expr(self, expr: RefExpr) -> bool: + if expr.node is None: + return False + if "." in expr.node.fullname: + name = expr.node.fullname.rpartition(".")[0] + return self.is_native_module(name) or name in self.symbol_fullnames + return True + + def is_native_module_ref_expr(self, expr: RefExpr) -> bool: + return self.is_native_ref_expr(expr) and expr.kind == GDEF diff --git a/mypyc/irbuild/match.py b/mypyc/irbuild/match.py new file mode 100644 index 000000000000..976a8810b327 --- /dev/null +++ b/mypyc/irbuild/match.py @@ -0,0 +1,355 @@ +from contextlib import contextmanager +from typing import Generator, List, Optional, Tuple + +from mypy.nodes import MatchStmt, NameExpr, TypeInfo +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + Pattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, +) +from mypy.traverser import TraverserVisitor +from mypy.types import Instance, TupleType, get_proper_type +from mypyc.ir.ops import BasicBlock, Value +from mypyc.ir.rtypes import object_rprimitive +from mypyc.irbuild.builder import IRBuilder +from mypyc.primitives.dict_ops import ( + dict_copy, + dict_del_item, + mapping_has_key, + supports_mapping_protocol, +) +from mypyc.primitives.generic_ops import generic_ssize_t_len_op +from mypyc.primitives.list_ops import ( + sequence_get_item, + sequence_get_slice, + supports_sequence_protocol, +) +from mypyc.primitives.misc_ops import fast_isinstance_op, slow_isinstance_op + +# From: https://peps.python.org/pep-0634/#class-patterns +MATCHABLE_BUILTINS = { + "builtins.bool", + "builtins.bytearray", + "builtins.bytes", + "builtins.dict", + "builtins.float", + "builtins.frozenset", + "builtins.int", + "builtins.list", + "builtins.set", + "builtins.str", + "builtins.tuple", +} + + +class MatchVisitor(TraverserVisitor): + builder: IRBuilder + code_block: BasicBlock + next_block: BasicBlock + final_block: BasicBlock + subject: Value + match: MatchStmt + + as_pattern: Optional[AsPattern] = None + + def __init__(self, builder: IRBuilder, match_node: MatchStmt) -> None: + self.builder = builder + + self.code_block = BasicBlock() + self.next_block = BasicBlock() + self.final_block = BasicBlock() + + self.match = match_node + self.subject = builder.accept(match_node.subject) + + def build_match_body(self, index: int) -> None: + self.builder.activate_block(self.code_block) + + guard = self.match.guards[index] + + if guard: + self.code_block = BasicBlock() + + cond = self.builder.accept(guard) + self.builder.add_bool_branch(cond, self.code_block, self.next_block) + + self.builder.activate_block(self.code_block) + + self.builder.accept(self.match.bodies[index]) + self.builder.goto(self.final_block) + + def visit_match_stmt(self, m: MatchStmt) -> None: + for i, pattern in enumerate(m.patterns): + self.code_block = BasicBlock() + self.next_block = BasicBlock() + + pattern.accept(self) + + self.build_match_body(i) + self.builder.activate_block(self.next_block) + + self.builder.goto_and_activate(self.final_block) + + def visit_value_pattern(self, pattern: ValuePattern) -> None: + value = self.builder.accept(pattern.expr) + + cond = self.builder.binary_op(self.subject, value, "==", pattern.expr.line) + + self.bind_as_pattern(value) + + self.builder.add_bool_branch(cond, self.code_block, self.next_block) + + def visit_or_pattern(self, pattern: OrPattern) -> None: + backup_block = self.next_block + self.next_block = BasicBlock() + + for p in pattern.patterns: + # Hack to ensure the as pattern is bound to each pattern in the + # "or" pattern, but not every subpattern + backup = self.as_pattern + p.accept(self) + self.as_pattern = backup + + self.builder.activate_block(self.next_block) + self.next_block = BasicBlock() + + self.next_block = backup_block + self.builder.goto(self.next_block) + + def visit_class_pattern(self, pattern: ClassPattern) -> None: + # TODO: use faster instance check for native classes (while still + # making sure to account for inheritence) + isinstance_op = ( + fast_isinstance_op + if self.builder.is_builtin_ref_expr(pattern.class_ref) + else slow_isinstance_op + ) + + cond = self.builder.primitive_op( + isinstance_op, [self.subject, self.builder.accept(pattern.class_ref)], pattern.line + ) + + self.builder.add_bool_branch(cond, self.code_block, self.next_block) + + self.bind_as_pattern(self.subject, new_block=True) + + if pattern.positionals: + if pattern.class_ref.fullname in MATCHABLE_BUILTINS: + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + pattern.positionals[0].accept(self) + + return + + node = pattern.class_ref.node + assert isinstance(node, TypeInfo) + + ty = node.names.get("__match_args__") + assert ty + + match_args_type = get_proper_type(ty.type) + assert isinstance(match_args_type, TupleType) + + match_args: List[str] = [] + + for item in match_args_type.items: + proper_item = get_proper_type(item) + assert isinstance(proper_item, Instance) and proper_item.last_known_value + + match_arg = proper_item.last_known_value.value + assert isinstance(match_arg, str) + + match_args.append(match_arg) + + for i, expr in enumerate(pattern.positionals): + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + # TODO: use faster "get_attr" method instead when calling on native or + # builtin objects + positional = self.builder.py_get_attr(self.subject, match_args[i], expr.line) + + with self.enter_subpattern(positional): + expr.accept(self) + + for key, value in zip(pattern.keyword_keys, pattern.keyword_values): + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + # TODO: same as above "get_attr" comment + attr = self.builder.py_get_attr(self.subject, key, value.line) + + with self.enter_subpattern(attr): + value.accept(self) + + def visit_as_pattern(self, pattern: AsPattern) -> None: + if pattern.pattern: + old_pattern = self.as_pattern + self.as_pattern = pattern + pattern.pattern.accept(self) + self.as_pattern = old_pattern + + elif pattern.name: + target = self.builder.get_assignment_target(pattern.name) + + self.builder.assign(target, self.subject, pattern.line) + + self.builder.goto(self.code_block) + + def visit_singleton_pattern(self, pattern: SingletonPattern) -> None: + if pattern.value is None: + obj = self.builder.none_object() + elif pattern.value is True: + obj = self.builder.true() + else: + obj = self.builder.false() + + cond = self.builder.binary_op(self.subject, obj, "is", pattern.line) + + self.builder.add_bool_branch(cond, self.code_block, self.next_block) + + def visit_mapping_pattern(self, pattern: MappingPattern) -> None: + is_dict = self.builder.call_c(supports_mapping_protocol, [self.subject], pattern.line) + + self.builder.add_bool_branch(is_dict, self.code_block, self.next_block) + + keys: List[Value] = [] + + for key, value in zip(pattern.keys, pattern.values): + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + key_value = self.builder.accept(key) + keys.append(key_value) + + exists = self.builder.call_c(mapping_has_key, [self.subject, key_value], pattern.line) + + self.builder.add_bool_branch(exists, self.code_block, self.next_block) + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + item = self.builder.gen_method_call( + self.subject, "__getitem__", [key_value], object_rprimitive, pattern.line + ) + + with self.enter_subpattern(item): + value.accept(self) + + if pattern.rest: + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + rest = self.builder.primitive_op(dict_copy, [self.subject], pattern.rest.line) + + target = self.builder.get_assignment_target(pattern.rest) + + self.builder.assign(target, rest, pattern.rest.line) + + for i, key_name in enumerate(keys): + self.builder.call_c(dict_del_item, [rest, key_name], pattern.keys[i].line) + + self.builder.goto(self.code_block) + + def visit_sequence_pattern(self, seq_pattern: SequencePattern) -> None: + star_index, capture, patterns = prep_sequence_pattern(seq_pattern) + + is_list = self.builder.call_c(supports_sequence_protocol, [self.subject], seq_pattern.line) + + self.builder.add_bool_branch(is_list, self.code_block, self.next_block) + + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + actual_len = self.builder.call_c(generic_ssize_t_len_op, [self.subject], seq_pattern.line) + min_len = len(patterns) + + is_long_enough = self.builder.binary_op( + actual_len, + self.builder.load_int(min_len), + "==" if star_index is None else ">=", + seq_pattern.line, + ) + + self.builder.add_bool_branch(is_long_enough, self.code_block, self.next_block) + + for i, pattern in enumerate(patterns): + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + if star_index is not None and i >= star_index: + current = self.builder.binary_op( + actual_len, self.builder.load_int(min_len - i), "-", pattern.line + ) + + else: + current = self.builder.load_int(i) + + item = self.builder.call_c(sequence_get_item, [self.subject, current], pattern.line) + + with self.enter_subpattern(item): + pattern.accept(self) + + if capture and star_index is not None: + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + capture_end = self.builder.binary_op( + actual_len, self.builder.load_int(min_len - star_index), "-", capture.line + ) + + rest = self.builder.call_c( + sequence_get_slice, + [self.subject, self.builder.load_int(star_index), capture_end], + capture.line, + ) + + target = self.builder.get_assignment_target(capture) + self.builder.assign(target, rest, capture.line) + + self.builder.goto(self.code_block) + + def bind_as_pattern(self, value: Value, new_block: bool = False) -> None: + if self.as_pattern and self.as_pattern.pattern and self.as_pattern.name: + if new_block: + self.builder.activate_block(self.code_block) + self.code_block = BasicBlock() + + target = self.builder.get_assignment_target(self.as_pattern.name) + self.builder.assign(target, value, self.as_pattern.pattern.line) + + self.as_pattern = None + + if new_block: + self.builder.goto(self.code_block) + + @contextmanager + def enter_subpattern(self, subject: Value) -> Generator[None, None, None]: + old_subject = self.subject + self.subject = subject + yield + self.subject = old_subject + + +def prep_sequence_pattern( + seq_pattern: SequencePattern, +) -> Tuple[Optional[int], Optional[NameExpr], List[Pattern]]: + star_index: Optional[int] = None + capture: Optional[NameExpr] = None + patterns: List[Pattern] = [] + + for i, pattern in enumerate(seq_pattern.patterns): + if isinstance(pattern, StarredPattern): + star_index = i + capture = pattern.capture + + else: + patterns.append(pattern) + + return star_index, capture, patterns diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py new file mode 100644 index 000000000000..0ac9bd3cee31 --- /dev/null +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -0,0 +1,197 @@ +"""Helpers for dealing with nonlocal control such as 'break' and 'return'. + +Model how these behave differently in different contexts. +""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import TYPE_CHECKING + +from mypyc.ir.ops import ( + NO_TRACEBACK_LINE_NO, + BasicBlock, + Branch, + Goto, + Integer, + Register, + Return, + Unreachable, + Value, +) +from mypyc.irbuild.targets import AssignmentTarget +from mypyc.primitives.exc_ops import restore_exc_info_op, set_stop_iteration_value + +if TYPE_CHECKING: + from mypyc.irbuild.builder import IRBuilder + + +class NonlocalControl: + """ABC representing a stack frame of constructs that modify nonlocal control flow. + + The nonlocal control flow constructs are break, continue, and + return, and their behavior is modified by a number of other + constructs. The most obvious is loop, which override where break + and continue jump to, but also `except` (which needs to clear + exc_info when left) and (eventually) finally blocks (which need to + ensure that the finally block is always executed when leaving the + try/except blocks). + """ + + @abstractmethod + def gen_break(self, builder: IRBuilder, line: int) -> None: + pass + + @abstractmethod + def gen_continue(self, builder: IRBuilder, line: int) -> None: + pass + + @abstractmethod + def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: + pass + + +class BaseNonlocalControl(NonlocalControl): + """Default nonlocal control outside any statements that affect it.""" + + def gen_break(self, builder: IRBuilder, line: int) -> None: + assert False, "break outside of loop" + + def gen_continue(self, builder: IRBuilder, line: int) -> None: + assert False, "continue outside of loop" + + def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: + builder.add(Return(value)) + + +class LoopNonlocalControl(NonlocalControl): + """Nonlocal control within a loop.""" + + def __init__( + self, outer: NonlocalControl, continue_block: BasicBlock, break_block: BasicBlock + ) -> None: + self.outer = outer + self.continue_block = continue_block + self.break_block = break_block + + def gen_break(self, builder: IRBuilder, line: int) -> None: + builder.add(Goto(self.break_block)) + + def gen_continue(self, builder: IRBuilder, line: int) -> None: + builder.add(Goto(self.continue_block)) + + def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: + self.outer.gen_return(builder, value, line) + + +class GeneratorNonlocalControl(BaseNonlocalControl): + """Default nonlocal control in a generator function outside statements.""" + + def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: + # Assign an invalid next label number so that the next time + # __next__ is called, we jump to the case in which + # StopIteration is raised. + builder.assign(builder.fn_info.generator_class.next_label_target, Integer(-1), line) + + # Raise a StopIteration containing a field for the value that + # should be returned. Before doing so, create a new block + # without an error handler set so that the implicitly thrown + # StopIteration isn't caught by except blocks inside of the + # generator function. + builder.builder.push_error_handler(None) + builder.goto_and_activate(BasicBlock()) + + # Skip creating a traceback frame when we raise here, because + # we don't care about the traceback frame and it is kind of + # expensive since raising StopIteration is an extremely common + # case. Also we call a special internal function to set + # StopIteration instead of using RaiseStandardError because + # the obvious thing doesn't work if the value is a tuple + # (???). + builder.call_c(set_stop_iteration_value, [value], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + builder.builder.pop_error_handler() + + +class CleanupNonlocalControl(NonlocalControl): + """Abstract nonlocal control that runs some cleanup code.""" + + def __init__(self, outer: NonlocalControl) -> None: + self.outer = outer + + @abstractmethod + def gen_cleanup(self, builder: IRBuilder, line: int) -> None: ... + + def gen_break(self, builder: IRBuilder, line: int) -> None: + self.gen_cleanup(builder, line) + self.outer.gen_break(builder, line) + + def gen_continue(self, builder: IRBuilder, line: int) -> None: + self.gen_cleanup(builder, line) + self.outer.gen_continue(builder, line) + + def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: + self.gen_cleanup(builder, line) + self.outer.gen_return(builder, value, line) + + +class TryFinallyNonlocalControl(NonlocalControl): + """Nonlocal control within try/finally.""" + + def __init__(self, target: BasicBlock) -> None: + self.target = target + self.ret_reg: None | Register | AssignmentTarget = None + + def gen_break(self, builder: IRBuilder, line: int) -> None: + builder.error("break inside try/finally block is unimplemented", line) + + def gen_continue(self, builder: IRBuilder, line: int) -> None: + builder.error("continue inside try/finally block is unimplemented", line) + + def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: + if self.ret_reg is None: + if builder.fn_info.is_generator: + self.ret_reg = builder.make_spill_target(builder.ret_types[-1]) + else: + self.ret_reg = Register(builder.ret_types[-1]) + # assert needed because of apparent mypy bug... it loses track of the union + # and infers the type as object + assert isinstance(self.ret_reg, (Register, AssignmentTarget)) + builder.assign(self.ret_reg, value, line) + + builder.add(Goto(self.target)) + + +class ExceptNonlocalControl(CleanupNonlocalControl): + """Nonlocal control for except blocks. + + Just makes sure that sys.exc_info always gets restored when we leave. + This is super annoying. + """ + + def __init__(self, outer: NonlocalControl, saved: Value | AssignmentTarget) -> None: + super().__init__(outer) + self.saved = saved + + def gen_cleanup(self, builder: IRBuilder, line: int) -> None: + builder.call_c(restore_exc_info_op, [builder.read(self.saved)], line) + + +class FinallyNonlocalControl(CleanupNonlocalControl): + """Nonlocal control for finally blocks. + + Just makes sure that sys.exc_info always gets restored when we + leave and the return register is decrefed if it isn't null. + """ + + def __init__(self, outer: NonlocalControl, saved: Value) -> None: + super().__init__(outer) + self.saved = saved + + def gen_cleanup(self, builder: IRBuilder, line: int) -> None: + # Restore the old exc_info + target, cleanup = BasicBlock(), BasicBlock() + builder.add(Branch(self.saved, target, cleanup, Branch.IS_ERROR)) + builder.activate_block(cleanup) + builder.call_c(restore_exc_info_op, [self.saved], line) + builder.goto_and_activate(target) diff --git a/mypyc/irbuild/prebuildvisitor.py b/mypyc/irbuild/prebuildvisitor.py new file mode 100644 index 000000000000..17f907d42111 --- /dev/null +++ b/mypyc/irbuild/prebuildvisitor.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from mypy.nodes import ( + Block, + Decorator, + Expression, + FuncDef, + FuncItem, + Import, + LambdaExpr, + MemberExpr, + MypyFile, + NameExpr, + Node, + SymbolNode, + Var, +) +from mypy.traverser import ExtendedTraverserVisitor +from mypyc.errors import Errors + + +class PreBuildVisitor(ExtendedTraverserVisitor): + """Mypy file AST visitor run before building the IR. + + This collects various things, including: + + * Determine relationships between nested functions and functions that + contain nested functions + * Find non-local variables (free variables) + * Find property setters + * Find decorators of functions + * Find module import groups + + The main IR build pass uses this information. + """ + + def __init__( + self, + errors: Errors, + current_file: MypyFile, + decorators_to_remove: dict[FuncDef, list[int]], + ) -> None: + super().__init__() + # Dict from a function to symbols defined directly in the + # function that are used as non-local (free) variables within a + # nested function. + self.free_variables: dict[FuncItem, set[SymbolNode]] = {} + + # Intermediate data structure used to find the function where + # a SymbolNode is declared. Initially this may point to a + # function nested inside the function with the declaration, + # but we'll eventually update this to refer to the function + # with the declaration. + self.symbols_to_funcs: dict[SymbolNode, FuncItem] = {} + + # Stack representing current function nesting. + self.funcs: list[FuncItem] = [] + + # All property setters encountered so far. + self.prop_setters: set[FuncDef] = set() + + # A map from any function that contains nested functions to + # a set of all the functions that are nested within it. + self.encapsulating_funcs: dict[FuncItem, list[FuncItem]] = {} + + # Map nested function to its parent/encapsulating function. + self.nested_funcs: dict[FuncItem, FuncItem] = {} + + # Map function to its non-special decorators. + self.funcs_to_decorators: dict[FuncDef, list[Expression]] = {} + + # Map function to indices of decorators to remove + self.decorators_to_remove: dict[FuncDef, list[int]] = decorators_to_remove + + # A mapping of import groups (a series of Import nodes with + # nothing inbetween) where each group is keyed by its first + # import node. + self.module_import_groups: dict[Import, list[Import]] = {} + self._current_import_group: Import | None = None + + self.errors: Errors = errors + + self.current_file: MypyFile = current_file + + def visit(self, o: Node) -> bool: + if not isinstance(o, Import): + self._current_import_group = None + return True + + def visit_block(self, block: Block) -> None: + self._current_import_group = None + super().visit_block(block) + self._current_import_group = None + + def visit_decorator(self, dec: Decorator) -> None: + if dec.decorators: + # Only add the function being decorated if there exist + # (ordinary) decorators in the decorator list. Certain + # decorators (such as @property, @abstractmethod) are + # special cased and removed from this list by + # mypy. Functions decorated only by special decorators + # (and property setters) are not treated as decorated + # functions by the IR builder. + if isinstance(dec.decorators[0], MemberExpr) and dec.decorators[0].name == "setter": + # Property setters are not treated as decorated methods. + self.prop_setters.add(dec.func) + else: + decorators_to_store = dec.decorators.copy() + if dec.func in self.decorators_to_remove: + to_remove = self.decorators_to_remove[dec.func] + + for i in reversed(to_remove): + del decorators_to_store[i] + # if all of the decorators are removed, we shouldn't treat this as a decorated + # function because there aren't any decorators to apply + if not decorators_to_store: + return + + self.funcs_to_decorators[dec.func] = decorators_to_store + super().visit_decorator(dec) + + def visit_func_def(self, fdef: FuncDef) -> None: + # TODO: What about overloaded functions? + self.visit_func(fdef) + self.visit_symbol_node(fdef) + + def visit_lambda_expr(self, expr: LambdaExpr) -> None: + self.visit_func(expr) + + def visit_func(self, func: FuncItem) -> None: + # If there were already functions or lambda expressions + # defined in the function stack, then note the previous + # FuncItem as containing a nested function and the current + # FuncItem as being a nested function. + if self.funcs: + # Add the new func to the set of nested funcs within the + # func at top of the func stack. + self.encapsulating_funcs.setdefault(self.funcs[-1], []).append(func) + # Add the func at top of the func stack as the parent of + # new func. + self.nested_funcs[func] = self.funcs[-1] + + self.funcs.append(func) + super().visit_func(func) + self.funcs.pop() + + def visit_import(self, imp: Import) -> None: + if self._current_import_group is not None: + self.module_import_groups[self._current_import_group].append(imp) + else: + self.module_import_groups[imp] = [imp] + self._current_import_group = imp + super().visit_import(imp) + + def visit_name_expr(self, expr: NameExpr) -> None: + if isinstance(expr.node, (Var, FuncDef)): + self.visit_symbol_node(expr.node) + + def visit_var(self, var: Var) -> None: + self.visit_symbol_node(var) + + def visit_symbol_node(self, symbol: SymbolNode) -> None: + if not self.funcs: + # We are not inside a function and hence do not need to do + # anything regarding free variables. + return + + if symbol in self.symbols_to_funcs: + orig_func = self.symbols_to_funcs[symbol] + if self.is_parent(self.funcs[-1], orig_func): + # The function in which the symbol was previously seen is + # nested within the function currently being visited. Thus + # the current function is a better candidate to contain the + # declaration. + self.symbols_to_funcs[symbol] = self.funcs[-1] + # TODO: Remove from the orig_func free_variables set? + self.free_variables.setdefault(self.funcs[-1], set()).add(symbol) + + elif self.is_parent(orig_func, self.funcs[-1]): + # The SymbolNode instance has already been visited + # before in a parent function, thus it's a non-local + # symbol. + self.add_free_variable(symbol) + + else: + # This is the first time the SymbolNode is being + # visited. We map the SymbolNode to the current FuncDef + # being visited to note where it was first visited. + self.symbols_to_funcs[symbol] = self.funcs[-1] + + def is_parent(self, fitem: FuncItem, child: FuncItem) -> bool: + # Check if child is nested within fdef (possibly indirectly + # within multiple nested functions). + if child not in self.nested_funcs: + return False + parent = self.nested_funcs[child] + return parent == fitem or self.is_parent(fitem, parent) + + def add_free_variable(self, symbol: SymbolNode) -> None: + # Find the function where the symbol was (likely) first declared, + # and mark is as a non-local symbol within that function. + func = self.symbols_to_funcs[symbol] + self.free_variables.setdefault(func, set()).add(symbol) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py new file mode 100644 index 000000000000..4b132bb83722 --- /dev/null +++ b/mypyc/irbuild/prepare.py @@ -0,0 +1,645 @@ +"""Prepare for IR transform. + +This needs to run after type checking and before generating IR. + +For example, construct partially initialized FuncIR and ClassIR +objects for all functions and classes. This allows us to bind +references to functions and classes before we've generated full IR for +functions or classes. The actual IR transform will then populate all +the missing bits, such as function bodies (basic blocks). + +Also build a mapping from mypy TypeInfos to ClassIR objects. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Iterable, NamedTuple, Tuple + +from mypy.build import Graph +from mypy.nodes import ( + ARG_STAR, + ARG_STAR2, + CallExpr, + ClassDef, + Decorator, + Expression, + FuncDef, + MemberExpr, + MypyFile, + NameExpr, + OverloadedFuncDef, + RefExpr, + SymbolNode, + TypeInfo, + Var, +) +from mypy.semanal import refers_to_fullname +from mypy.traverser import TraverserVisitor +from mypy.types import Instance, Type, get_proper_type +from mypyc.common import PROPSET_PREFIX, get_id_from_name +from mypyc.crash import catch_errors +from mypyc.errors import Errors +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import ( + FUNC_CLASSMETHOD, + FUNC_NORMAL, + FUNC_STATICMETHOD, + FuncDecl, + FuncSignature, + RuntimeArg, +) +from mypyc.ir.ops import DeserMaps +from mypyc.ir.rtypes import RInstance, RType, dict_rprimitive, none_rprimitive, tuple_rprimitive +from mypyc.irbuild.mapper import Mapper +from mypyc.irbuild.util import ( + get_func_def, + get_mypyc_attrs, + is_dataclass, + is_extension_class, + is_trait, +) +from mypyc.options import CompilerOptions +from mypyc.sametype import is_same_type + + +def build_type_map( + mapper: Mapper, + modules: list[MypyFile], + graph: Graph, + types: dict[Expression, Type], + options: CompilerOptions, + errors: Errors, +) -> None: + # Collect all classes defined in everything we are compiling + classes = [] + for module in modules: + module_classes = [node for node in module.defs if isinstance(node, ClassDef)] + classes.extend([(module, cdef) for cdef in module_classes]) + + # Collect all class mappings so that we can bind arbitrary class name + # references even if there are import cycles. + for module, cdef in classes: + class_ir = ClassIR( + cdef.name, + module.fullname, + is_trait(cdef), + is_abstract=cdef.info.is_abstract, + is_final_class=cdef.info.is_final, + ) + class_ir.is_ext_class = is_extension_class(cdef) + if class_ir.is_ext_class: + class_ir.deletable = cdef.info.deletable_attributes.copy() + # If global optimizations are disabled, turn of tracking of class children + if not options.global_opts: + class_ir.children = None + mapper.type_to_ir[cdef.info] = class_ir + mapper.symbol_fullnames.add(class_ir.fullname) + + # Populate structural information in class IR for extension classes. + for module, cdef in classes: + with catch_errors(module.path, cdef.line): + if mapper.type_to_ir[cdef.info].is_ext_class: + prepare_class_def(module.path, module.fullname, cdef, errors, mapper, options) + else: + prepare_non_ext_class_def( + module.path, module.fullname, cdef, errors, mapper, options + ) + + # Prepare implicit attribute accessors as needed if an attribute overrides a property. + for module, cdef in classes: + class_ir = mapper.type_to_ir[cdef.info] + if class_ir.is_ext_class: + prepare_implicit_property_accessors(cdef.info, class_ir, module.fullname, mapper) + + # Collect all the functions also. We collect from the symbol table + # so that we can easily pick out the right copy of a function that + # is conditionally defined. + for module in modules: + for func in get_module_func_defs(module): + prepare_func_def(module.fullname, None, func, mapper, options) + # TODO: what else? + + # Check for incompatible attribute definitions that were not + # flagged by mypy but can't be supported when compiling. + for module, cdef in classes: + class_ir = mapper.type_to_ir[cdef.info] + for attr in class_ir.attributes: + for base_ir in class_ir.mro[1:]: + if attr in base_ir.attributes: + if not is_same_type(class_ir.attributes[attr], base_ir.attributes[attr]): + node = cdef.info.names[attr].node + assert node is not None + kind = "trait" if base_ir.is_trait else "class" + errors.error( + f'Type of "{attr}" is incompatible with ' + f'definition in {kind} "{base_ir.name}"', + module.path, + node.line, + ) + + +def is_from_module(node: SymbolNode, module: MypyFile) -> bool: + return node.fullname == module.fullname + "." + node.name + + +def load_type_map(mapper: Mapper, modules: list[MypyFile], deser_ctx: DeserMaps) -> None: + """Populate a Mapper with deserialized IR from a list of modules.""" + for module in modules: + for node in module.names.values(): + if isinstance(node.node, TypeInfo) and is_from_module(node.node, module): + ir = deser_ctx.classes[node.node.fullname] + mapper.type_to_ir[node.node] = ir + mapper.symbol_fullnames.add(node.node.fullname) + mapper.func_to_decl[node.node] = ir.ctor + + for module in modules: + for func in get_module_func_defs(module): + func_id = get_id_from_name(func.name, func.fullname, func.line) + mapper.func_to_decl[func] = deser_ctx.functions[func_id].decl + + +def get_module_func_defs(module: MypyFile) -> Iterable[FuncDef]: + """Collect all of the (non-method) functions declared in a module.""" + for node in module.names.values(): + # We need to filter out functions that are imported or + # aliases. The best way to do this seems to be by + # checking that the fullname matches. + if isinstance(node.node, (FuncDef, Decorator, OverloadedFuncDef)) and is_from_module( + node.node, module + ): + yield get_func_def(node.node) + + +def prepare_func_def( + module_name: str, + class_name: str | None, + fdef: FuncDef, + mapper: Mapper, + options: CompilerOptions, +) -> FuncDecl: + kind = ( + FUNC_STATICMETHOD + if fdef.is_static + else (FUNC_CLASSMETHOD if fdef.is_class else FUNC_NORMAL) + ) + sig = mapper.fdef_to_sig(fdef, options.strict_dunders_typing) + decl = FuncDecl(fdef.name, class_name, module_name, sig, kind) + mapper.func_to_decl[fdef] = decl + return decl + + +def prepare_method_def( + ir: ClassIR, + module_name: str, + cdef: ClassDef, + mapper: Mapper, + node: FuncDef | Decorator, + options: CompilerOptions, +) -> None: + if isinstance(node, FuncDef): + ir.method_decls[node.name] = prepare_func_def( + module_name, cdef.name, node, mapper, options + ) + elif isinstance(node, Decorator): + # TODO: do something about abstract methods here. Currently, they are handled just like + # normal methods. + decl = prepare_func_def(module_name, cdef.name, node.func, mapper, options) + if not node.decorators: + ir.method_decls[node.name] = decl + elif isinstance(node.decorators[0], MemberExpr) and node.decorators[0].name == "setter": + # Make property setter name different than getter name so there are no + # name clashes when generating C code, and property lookup at the IR level + # works correctly. + decl.name = PROPSET_PREFIX + decl.name + decl.is_prop_setter = True + # Making the argument implicitly positional-only avoids unnecessary glue methods + decl.sig.args[1].pos_only = True + ir.method_decls[PROPSET_PREFIX + node.name] = decl + + if node.func.is_property: + assert node.func.type, f"Expected return type annotation for property '{node.name}'" + decl.is_prop_getter = True + ir.property_types[node.name] = decl.sig.ret_type + + +def is_valid_multipart_property_def(prop: OverloadedFuncDef) -> bool: + # Checks to ensure supported property decorator semantics + if len(prop.items) != 2: + return False + + getter = prop.items[0] + setter = prop.items[1] + + return ( + isinstance(getter, Decorator) + and isinstance(setter, Decorator) + and getter.func.is_property + and len(setter.decorators) == 1 + and isinstance(setter.decorators[0], MemberExpr) + and setter.decorators[0].name == "setter" + ) + + +def can_subclass_builtin(builtin_base: str) -> bool: + # BaseException and dict are special cased. + return builtin_base in ( + ( + "builtins.Exception", + "builtins.LookupError", + "builtins.IndexError", + "builtins.Warning", + "builtins.UserWarning", + "builtins.ValueError", + "builtins.object", + ) + ) + + +def prepare_class_def( + path: str, + module_name: str, + cdef: ClassDef, + errors: Errors, + mapper: Mapper, + options: CompilerOptions, +) -> None: + """Populate the interface-level information in a class IR. + + This includes attribute and method declarations, and the MRO, among other things, but + method bodies are generated in a later pass. + """ + + ir = mapper.type_to_ir[cdef.info] + info = cdef.info + + attrs = get_mypyc_attrs(cdef) + if attrs.get("allow_interpreted_subclasses") is True: + ir.allow_interpreted_subclasses = True + if attrs.get("serializable") is True: + # Supports copy.copy and pickle (including subclasses) + ir._serializable = True + + # Check for subclassing from builtin types + for cls in info.mro: + # Special case exceptions and dicts + # XXX: How do we handle *other* things?? + if cls.fullname == "builtins.BaseException": + ir.builtin_base = "PyBaseExceptionObject" + elif cls.fullname == "builtins.dict": + ir.builtin_base = "PyDictObject" + elif cls.fullname.startswith("builtins."): + if not can_subclass_builtin(cls.fullname): + # Note that if we try to subclass a C extension class that + # isn't in builtins, bad things will happen and we won't + # catch it here! But this should catch a lot of the most + # common pitfalls. + errors.error( + "Inheriting from most builtin types is unimplemented", path, cdef.line + ) + + # Set up the parent class + bases = [mapper.type_to_ir[base.type] for base in info.bases if base.type in mapper.type_to_ir] + if len(bases) > 1 and any(not c.is_trait for c in bases) and bases[0].is_trait: + # If the first base is a non-trait, don't ever error here. While it is correct + # to error if a trait comes before the next non-trait base (e.g. non-trait, trait, + # non-trait), it's pointless, confusing noise from the bigger issue: multiple + # inheritance is *not* supported. + errors.error("Non-trait base must appear first in parent list", path, cdef.line) + ir.traits = [c for c in bases if c.is_trait] + + mro = [] # All mypyc base classes + base_mro = [] # Non-trait mypyc base classes + for cls in info.mro: + if cls not in mapper.type_to_ir: + if cls.fullname != "builtins.object": + ir.inherits_python = True + continue + base_ir = mapper.type_to_ir[cls] + if not base_ir.is_trait: + base_mro.append(base_ir) + mro.append(base_ir) + + if cls.defn.removed_base_type_exprs or not base_ir.is_ext_class: + ir.inherits_python = True + + base_idx = 1 if not ir.is_trait else 0 + if len(base_mro) > base_idx: + ir.base = base_mro[base_idx] + ir.mro = mro + ir.base_mro = base_mro + + prepare_methods_and_attributes(cdef, ir, path, module_name, errors, mapper, options) + prepare_init_method(cdef, ir, module_name, mapper) + + for base in bases: + if base.children is not None: + base.children.append(ir) + + if is_dataclass(cdef): + ir.is_augmented = True + + +def prepare_methods_and_attributes( + cdef: ClassDef, + ir: ClassIR, + path: str, + module_name: str, + errors: Errors, + mapper: Mapper, + options: CompilerOptions, +) -> None: + """Populate attribute and method declarations.""" + info = cdef.info + for name, node in info.names.items(): + # Currently all plugin generated methods are dummies and not included. + if node.plugin_generated: + continue + + if isinstance(node.node, Var): + assert node.node.type, "Class member %s missing type" % name + if not node.node.is_classvar and name not in ("__slots__", "__deletable__"): + attr_rtype = mapper.type_to_rtype(node.node.type) + if ir.is_trait and attr_rtype.error_overlap: + # Traits don't have attribute definedness bitmaps, so use + # property accessor methods to access attributes that need them. + # We will generate accessor implementations that use the class bitmap + # for any concrete subclasses. + add_getter_declaration(ir, name, attr_rtype, module_name) + add_setter_declaration(ir, name, attr_rtype, module_name) + ir.attributes[name] = attr_rtype + elif isinstance(node.node, (FuncDef, Decorator)): + prepare_method_def(ir, module_name, cdef, mapper, node.node, options) + elif isinstance(node.node, OverloadedFuncDef): + # Handle case for property with both a getter and a setter + if node.node.is_property: + if is_valid_multipart_property_def(node.node): + for item in node.node.items: + prepare_method_def(ir, module_name, cdef, mapper, item, options) + else: + errors.error("Unsupported property decorator semantics", path, cdef.line) + + # Handle case for regular function overload + else: + assert node.node.impl + prepare_method_def(ir, module_name, cdef, mapper, node.node.impl, options) + + if ir.builtin_base: + ir.attributes.clear() + + +def prepare_implicit_property_accessors( + info: TypeInfo, ir: ClassIR, module_name: str, mapper: Mapper +) -> None: + concrete_attributes = set() + for base in ir.base_mro: + for name, attr_rtype in base.attributes.items(): + concrete_attributes.add(name) + add_property_methods_for_attribute_if_needed( + info, ir, name, attr_rtype, module_name, mapper + ) + for base in ir.mro[1:]: + if base.is_trait: + for name, attr_rtype in base.attributes.items(): + if name not in concrete_attributes: + add_property_methods_for_attribute_if_needed( + info, ir, name, attr_rtype, module_name, mapper + ) + + +def add_property_methods_for_attribute_if_needed( + info: TypeInfo, + ir: ClassIR, + attr_name: str, + attr_rtype: RType, + module_name: str, + mapper: Mapper, +) -> None: + """Add getter and/or setter for attribute if defined as property in a base class. + + Only add declarations. The body IR will be synthesized later during irbuild. + """ + for base in info.mro[1:]: + if base in mapper.type_to_ir: + base_ir = mapper.type_to_ir[base] + n = base.names.get(attr_name) + if n is None: + continue + node = n.node + if isinstance(node, Decorator) and node.name not in ir.method_decls: + # Defined as a read-only property in base class/trait + add_getter_declaration(ir, attr_name, attr_rtype, module_name) + elif isinstance(node, OverloadedFuncDef) and is_valid_multipart_property_def(node): + # Defined as a read-write property in base class/trait + add_getter_declaration(ir, attr_name, attr_rtype, module_name) + add_setter_declaration(ir, attr_name, attr_rtype, module_name) + elif base_ir.is_trait and attr_rtype.error_overlap: + add_getter_declaration(ir, attr_name, attr_rtype, module_name) + add_setter_declaration(ir, attr_name, attr_rtype, module_name) + + +def add_getter_declaration( + ir: ClassIR, attr_name: str, attr_rtype: RType, module_name: str +) -> None: + self_arg = RuntimeArg("self", RInstance(ir), pos_only=True) + sig = FuncSignature([self_arg], attr_rtype) + decl = FuncDecl(attr_name, ir.name, module_name, sig, FUNC_NORMAL) + decl.is_prop_getter = True + decl.implicit = True # Triggers synthesization + ir.method_decls[attr_name] = decl + ir.property_types[attr_name] = attr_rtype # TODO: Needed?? + + +def add_setter_declaration( + ir: ClassIR, attr_name: str, attr_rtype: RType, module_name: str +) -> None: + self_arg = RuntimeArg("self", RInstance(ir), pos_only=True) + value_arg = RuntimeArg("value", attr_rtype, pos_only=True) + sig = FuncSignature([self_arg, value_arg], none_rprimitive) + setter_name = PROPSET_PREFIX + attr_name + decl = FuncDecl(setter_name, ir.name, module_name, sig, FUNC_NORMAL) + decl.is_prop_setter = True + decl.implicit = True # Triggers synthesization + ir.method_decls[setter_name] = decl + + +def prepare_init_method(cdef: ClassDef, ir: ClassIR, module_name: str, mapper: Mapper) -> None: + # Set up a constructor decl + init_node = cdef.info["__init__"].node + if not ir.is_trait and not ir.builtin_base and isinstance(init_node, FuncDef): + init_sig = mapper.fdef_to_sig(init_node, True) + + defining_ir = mapper.type_to_ir.get(init_node.info) + # If there is a nontrivial __init__ that wasn't defined in an + # extension class, we need to make the constructor take *args, + # **kwargs so it can call tp_init. + if ( + defining_ir is None + or not defining_ir.is_ext_class + or cdef.info["__init__"].plugin_generated + ) and init_node.info.fullname != "builtins.object": + init_sig = FuncSignature( + [ + init_sig.args[0], + RuntimeArg("args", tuple_rprimitive, ARG_STAR), + RuntimeArg("kwargs", dict_rprimitive, ARG_STAR2), + ], + init_sig.ret_type, + ) + + last_arg = len(init_sig.args) - init_sig.num_bitmap_args + ctor_sig = FuncSignature(init_sig.args[1:last_arg], RInstance(ir)) + ir.ctor = FuncDecl(cdef.name, None, module_name, ctor_sig) + mapper.func_to_decl[cdef.info] = ir.ctor + + +def prepare_non_ext_class_def( + path: str, + module_name: str, + cdef: ClassDef, + errors: Errors, + mapper: Mapper, + options: CompilerOptions, +) -> None: + ir = mapper.type_to_ir[cdef.info] + info = cdef.info + + for node in info.names.values(): + if isinstance(node.node, (FuncDef, Decorator)): + prepare_method_def(ir, module_name, cdef, mapper, node.node, options) + elif isinstance(node.node, OverloadedFuncDef): + # Handle case for property with both a getter and a setter + if node.node.is_property: + if not is_valid_multipart_property_def(node.node): + errors.error("Unsupported property decorator semantics", path, cdef.line) + for item in node.node.items: + prepare_method_def(ir, module_name, cdef, mapper, item, options) + # Handle case for regular function overload + else: + prepare_method_def(ir, module_name, cdef, mapper, get_func_def(node.node), options) + + if any(cls in mapper.type_to_ir and mapper.type_to_ir[cls].is_ext_class for cls in info.mro): + errors.error( + "Non-extension classes may not inherit from extension classes", path, cdef.line + ) + + +RegisterImplInfo = Tuple[TypeInfo, FuncDef] + + +class SingledispatchInfo(NamedTuple): + singledispatch_impls: dict[FuncDef, list[RegisterImplInfo]] + decorators_to_remove: dict[FuncDef, list[int]] + + +def find_singledispatch_register_impls( + modules: list[MypyFile], errors: Errors +) -> SingledispatchInfo: + visitor = SingledispatchVisitor(errors) + for module in modules: + visitor.current_path = module.path + module.accept(visitor) + return SingledispatchInfo(visitor.singledispatch_impls, visitor.decorators_to_remove) + + +class SingledispatchVisitor(TraverserVisitor): + current_path: str + + def __init__(self, errors: Errors) -> None: + super().__init__() + + # Map of main singledispatch function to list of registered implementations + self.singledispatch_impls: defaultdict[FuncDef, list[RegisterImplInfo]] = defaultdict(list) + + # Map of decorated function to the indices of any decorators to remove + self.decorators_to_remove: dict[FuncDef, list[int]] = {} + + self.errors: Errors = errors + + def visit_decorator(self, dec: Decorator) -> None: + if dec.decorators: + decorators_to_store = dec.decorators.copy() + decorators_to_remove: list[int] = [] + # the index of the last non-register decorator before finding a register decorator + # when going through decorators from top to bottom + last_non_register: int | None = None + for i, d in enumerate(decorators_to_store): + impl = get_singledispatch_register_call_info(d, dec.func) + if impl is not None: + self.singledispatch_impls[impl.singledispatch_func].append( + (impl.dispatch_type, dec.func) + ) + decorators_to_remove.append(i) + if last_non_register is not None: + # found a register decorator after a non-register decorator, which we + # don't support because we'd have to make a copy of the function before + # calling the decorator so that we can call it later, which complicates + # the implementation for something that is probably not commonly used + self.errors.error( + "Calling decorator after registering function not supported", + self.current_path, + decorators_to_store[last_non_register].line, + ) + else: + if refers_to_fullname(d, "functools.singledispatch"): + decorators_to_remove.append(i) + # make sure that we still treat the function as a singledispatch function + # even if we don't find any registered implementations (which might happen + # if all registered implementations are registered dynamically) + self.singledispatch_impls.setdefault(dec.func, []) + last_non_register = i + + if decorators_to_remove: + # calling register on a function that tries to dispatch based on type annotations + # raises a TypeError because compiled functions don't have an __annotations__ + # attribute + self.decorators_to_remove[dec.func] = decorators_to_remove + + super().visit_decorator(dec) + + +class RegisteredImpl(NamedTuple): + singledispatch_func: FuncDef + dispatch_type: TypeInfo + + +def get_singledispatch_register_call_info( + decorator: Expression, func: FuncDef +) -> RegisteredImpl | None: + # @fun.register(complex) + # def g(arg): ... + if ( + isinstance(decorator, CallExpr) + and len(decorator.args) == 1 + and isinstance(decorator.args[0], RefExpr) + ): + callee = decorator.callee + dispatch_type = decorator.args[0].node + if not isinstance(dispatch_type, TypeInfo): + return None + + if isinstance(callee, MemberExpr): + return registered_impl_from_possible_register_call(callee, dispatch_type) + # @fun.register + # def g(arg: int): ... + elif isinstance(decorator, MemberExpr): + # we don't know if this is a register call yet, so we can't be sure that the function + # actually has arguments + if not func.arguments: + return None + arg_type = get_proper_type(func.arguments[0].variable.type) + if not isinstance(arg_type, Instance): + return None + info = arg_type.type + return registered_impl_from_possible_register_call(decorator, info) + return None + + +def registered_impl_from_possible_register_call( + expr: MemberExpr, dispatch_type: TypeInfo +) -> RegisteredImpl | None: + if expr.name == "register" and isinstance(expr.expr, NameExpr): + node = expr.expr.node + if isinstance(node, Decorator): + return RegisteredImpl(node.func, dispatch_type) + return None diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py new file mode 100644 index 000000000000..f652449f5289 --- /dev/null +++ b/mypyc/irbuild/specialize.py @@ -0,0 +1,890 @@ +"""Special case IR generation of calls to specific builtin functions. + +Most special cases should be handled using the data driven "primitive +ops" system, but certain operations require special handling that has +access to the AST/IR directly and can make decisions/optimizations +based on it. These special cases can be implemented here. + +For example, we use specializers to statically emit the length of a +fixed length tuple and to emit optimized code for any()/all() calls with +generator comprehensions as the argument. + +See comment below for more documentation. +""" + +from __future__ import annotations + +from typing import Callable, Optional + +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + BytesExpr, + CallExpr, + DictExpr, + Expression, + GeneratorExpr, + IntExpr, + ListExpr, + MemberExpr, + NameExpr, + RefExpr, + StrExpr, + TupleExpr, +) +from mypy.types import AnyType, TypeOfAny +from mypyc.ir.ops import ( + BasicBlock, + Extend, + Integer, + RaiseStandardError, + Register, + Truncate, + Unreachable, + Value, +) +from mypyc.ir.rtypes import ( + RInstance, + RPrimitive, + RTuple, + RType, + bool_rprimitive, + c_int_rprimitive, + dict_rprimitive, + int16_rprimitive, + int32_rprimitive, + int64_rprimitive, + int_rprimitive, + is_bool_rprimitive, + is_dict_rprimitive, + is_fixed_width_rtype, + is_float_rprimitive, + is_int16_rprimitive, + is_int32_rprimitive, + is_int64_rprimitive, + is_int_rprimitive, + is_list_rprimitive, + is_uint8_rprimitive, + list_rprimitive, + set_rprimitive, + str_rprimitive, + uint8_rprimitive, +) +from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.for_helpers import ( + comprehension_helper, + sequence_from_generator_preallocate_helper, + translate_list_comprehension, + translate_set_comprehension, +) +from mypyc.irbuild.format_str_tokenizer import ( + FormatOp, + convert_format_expr_to_str, + join_formatted_strings, + tokenizer_format_call, +) +from mypyc.primitives.dict_ops import ( + dict_items_op, + dict_keys_op, + dict_setdefault_spec_init_op, + dict_values_op, +) +from mypyc.primitives.list_ops import new_list_set_item_op +from mypyc.primitives.str_ops import ( + str_encode_ascii_strict, + str_encode_latin1_strict, + str_encode_utf8_strict, +) +from mypyc.primitives.tuple_ops import new_tuple_set_item_op + +# Specializers are attempted before compiling the arguments to the +# function. Specializers can return None to indicate that they failed +# and the call should be compiled normally. Otherwise they should emit +# code for the call and return a Value containing the result. +# +# Specializers take three arguments: the IRBuilder, the CallExpr being +# compiled, and the RefExpr that is the left hand side of the call. +Specializer = Callable[["IRBuilder", CallExpr, RefExpr], Optional[Value]] + +# Dictionary containing all configured specializers. +# +# Specializers can operate on methods as well, and are keyed on the +# name and RType in that case. +specializers: dict[tuple[str, RType | None], list[Specializer]] = {} + + +def _apply_specialization( + builder: IRBuilder, expr: CallExpr, callee: RefExpr, name: str | None, typ: RType | None = None +) -> Value | None: + # TODO: Allow special cases to have default args or named args. Currently they don't since + # they check that everything in arg_kinds is ARG_POS. + + # If there is a specializer for this function, try calling it. + # Return the first successful one. + if name and (name, typ) in specializers: + for specializer in specializers[name, typ]: + val = specializer(builder, expr, callee) + if val is not None: + return val + return None + + +def apply_function_specialization( + builder: IRBuilder, expr: CallExpr, callee: RefExpr +) -> Value | None: + """Invoke the Specializer callback for a function if one has been registered""" + return _apply_specialization(builder, expr, callee, callee.fullname) + + +def apply_method_specialization( + builder: IRBuilder, expr: CallExpr, callee: MemberExpr, typ: RType | None = None +) -> Value | None: + """Invoke the Specializer callback for a method if one has been registered""" + name = callee.fullname if typ is None else callee.name + return _apply_specialization(builder, expr, callee, name, typ) + + +def specialize_function( + name: str, typ: RType | None = None +) -> Callable[[Specializer], Specializer]: + """Decorator to register a function as being a specializer. + + There may exist multiple specializers for one function. When + translating method calls, the earlier appended specializer has + higher priority. + """ + + def wrapper(f: Specializer) -> Specializer: + specializers.setdefault((name, typ), []).append(f) + return f + + return wrapper + + +@specialize_function("builtins.globals") +def translate_globals(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) == 0: + return builder.load_globals_dict() + return None + + +@specialize_function("builtins.abs") +@specialize_function("builtins.int") +@specialize_function("builtins.float") +@specialize_function("builtins.complex") +@specialize_function("mypy_extensions.i64") +@specialize_function("mypy_extensions.i32") +@specialize_function("mypy_extensions.i16") +@specialize_function("mypy_extensions.u8") +def translate_builtins_with_unary_dunder( + builder: IRBuilder, expr: CallExpr, callee: RefExpr +) -> Value | None: + """Specialize calls on native classes that implement the associated dunder. + + E.g. i64(x) gets specialized to x.__int__() if x is a native instance. + """ + if len(expr.args) == 1 and expr.arg_kinds == [ARG_POS] and isinstance(callee, NameExpr): + arg = expr.args[0] + arg_typ = builder.node_type(arg) + shortname = callee.fullname.split(".")[1] + if shortname in ("i64", "i32", "i16", "u8"): + method = "__int__" + else: + method = f"__{shortname}__" + if isinstance(arg_typ, RInstance) and arg_typ.class_ir.has_method(method): + obj = builder.accept(arg) + return builder.gen_method_call(obj, method, [], None, expr.line) + + return None + + +@specialize_function("builtins.len") +def translate_len(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) == 1 and expr.arg_kinds == [ARG_POS]: + arg = expr.args[0] + expr_rtype = builder.node_type(arg) + if isinstance(expr_rtype, RTuple): + # len() of fixed-length tuple can be trivially determined + # statically, though we still need to evaluate it. + builder.accept(arg) + return Integer(len(expr_rtype.types)) + else: + if is_list_rprimitive(builder.node_type(arg)): + borrow = True + else: + borrow = False + obj = builder.accept(arg, can_borrow=borrow) + return builder.builtin_len(obj, expr.line) + return None + + +@specialize_function("builtins.list") +def dict_methods_fast_path(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Specialize a common case when list() is called on a dictionary + view method call. + + For example: + foo = list(bar.keys()) + """ + if not (len(expr.args) == 1 and expr.arg_kinds == [ARG_POS]): + return None + arg = expr.args[0] + if not (isinstance(arg, CallExpr) and not arg.args and isinstance(arg.callee, MemberExpr)): + return None + base = arg.callee.expr + attr = arg.callee.name + rtype = builder.node_type(base) + if not (is_dict_rprimitive(rtype) and attr in ("keys", "values", "items")): + return None + + obj = builder.accept(base) + # Note that it is not safe to use fast methods on dict subclasses, + # so the corresponding helpers in CPy.h fallback to (inlined) + # generic logic. + if attr == "keys": + return builder.call_c(dict_keys_op, [obj], expr.line) + elif attr == "values": + return builder.call_c(dict_values_op, [obj], expr.line) + else: + return builder.call_c(dict_items_op, [obj], expr.line) + + +@specialize_function("builtins.list") +def translate_list_from_generator_call( + builder: IRBuilder, expr: CallExpr, callee: RefExpr +) -> Value | None: + """Special case for simplest list comprehension. + + For example: + list(f(x) for x in some_list/some_tuple/some_str) + 'translate_list_comprehension()' would take care of other cases + if this fails. + """ + if ( + len(expr.args) == 1 + and expr.arg_kinds[0] == ARG_POS + and isinstance(expr.args[0], GeneratorExpr) + ): + return sequence_from_generator_preallocate_helper( + builder, + expr.args[0], + empty_op_llbuilder=builder.builder.new_list_op_with_length, + set_item_op=new_list_set_item_op, + ) + return None + + +@specialize_function("builtins.tuple") +def translate_tuple_from_generator_call( + builder: IRBuilder, expr: CallExpr, callee: RefExpr +) -> Value | None: + """Special case for simplest tuple creation from a generator. + + For example: + tuple(f(x) for x in some_list/some_tuple/some_str) + 'translate_safe_generator_call()' would take care of other cases + if this fails. + """ + if ( + len(expr.args) == 1 + and expr.arg_kinds[0] == ARG_POS + and isinstance(expr.args[0], GeneratorExpr) + ): + return sequence_from_generator_preallocate_helper( + builder, + expr.args[0], + empty_op_llbuilder=builder.builder.new_tuple_with_length, + set_item_op=new_tuple_set_item_op, + ) + return None + + +@specialize_function("builtins.set") +def translate_set_from_generator_call( + builder: IRBuilder, expr: CallExpr, callee: RefExpr +) -> Value | None: + """Special case for set creation from a generator. + + For example: + set(f(...) for ... in iterator/nested_generators...) + """ + if ( + len(expr.args) == 1 + and expr.arg_kinds[0] == ARG_POS + and isinstance(expr.args[0], GeneratorExpr) + ): + return translate_set_comprehension(builder, expr.args[0]) + return None + + +@specialize_function("builtins.min") +@specialize_function("builtins.max") +def faster_min_max(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if expr.arg_kinds == [ARG_POS, ARG_POS]: + x, y = builder.accept(expr.args[0]), builder.accept(expr.args[1]) + result = Register(builder.node_type(expr)) + # CPython evaluates arguments reversely when calling min(...) or max(...) + if callee.fullname == "builtins.min": + comparison = builder.binary_op(y, x, "<", expr.line) + else: + comparison = builder.binary_op(y, x, ">", expr.line) + + true_block, false_block, next_block = BasicBlock(), BasicBlock(), BasicBlock() + builder.add_bool_branch(comparison, true_block, false_block) + + builder.activate_block(true_block) + builder.assign(result, builder.coerce(y, result.type, expr.line), expr.line) + builder.goto(next_block) + + builder.activate_block(false_block) + builder.assign(result, builder.coerce(x, result.type, expr.line), expr.line) + builder.goto(next_block) + + builder.activate_block(next_block) + return result + return None + + +@specialize_function("builtins.tuple") +@specialize_function("builtins.frozenset") +@specialize_function("builtins.dict") +@specialize_function("builtins.min") +@specialize_function("builtins.max") +@specialize_function("builtins.sorted") +@specialize_function("collections.OrderedDict") +@specialize_function("join", str_rprimitive) +@specialize_function("extend", list_rprimitive) +@specialize_function("update", dict_rprimitive) +@specialize_function("update", set_rprimitive) +def translate_safe_generator_call( + builder: IRBuilder, expr: CallExpr, callee: RefExpr +) -> Value | None: + """Special cases for things that consume iterators where we know we + can safely compile a generator into a list. + """ + if ( + len(expr.args) > 0 + and expr.arg_kinds[0] == ARG_POS + and isinstance(expr.args[0], GeneratorExpr) + ): + if isinstance(callee, MemberExpr): + return builder.gen_method_call( + builder.accept(callee.expr), + callee.name, + ( + [translate_list_comprehension(builder, expr.args[0])] + + [builder.accept(arg) for arg in expr.args[1:]] + ), + builder.node_type(expr), + expr.line, + expr.arg_kinds, + expr.arg_names, + ) + else: + return builder.call_refexpr_with_args( + expr, + callee, + ( + [translate_list_comprehension(builder, expr.args[0])] + + [builder.accept(arg) for arg in expr.args[1:]] + ), + ) + return None + + +@specialize_function("builtins.any") +def translate_any_call(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if ( + len(expr.args) == 1 + and expr.arg_kinds == [ARG_POS] + and isinstance(expr.args[0], GeneratorExpr) + ): + return any_all_helper(builder, expr.args[0], builder.false, lambda x: x, builder.true) + return None + + +@specialize_function("builtins.all") +def translate_all_call(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if ( + len(expr.args) == 1 + and expr.arg_kinds == [ARG_POS] + and isinstance(expr.args[0], GeneratorExpr) + ): + return any_all_helper( + builder, + expr.args[0], + builder.true, + lambda x: builder.unary_op(x, "not", expr.line), + builder.false, + ) + return None + + +def any_all_helper( + builder: IRBuilder, + gen: GeneratorExpr, + initial_value: Callable[[], Value], + modify: Callable[[Value], Value], + new_value: Callable[[], Value], +) -> Value: + retval = Register(bool_rprimitive) + builder.assign(retval, initial_value(), -1) + loop_params = list(zip(gen.indices, gen.sequences, gen.condlists, gen.is_async)) + true_block, false_block, exit_block = BasicBlock(), BasicBlock(), BasicBlock() + + def gen_inner_stmts() -> None: + comparison = modify(builder.accept(gen.left_expr)) + builder.add_bool_branch(comparison, true_block, false_block) + builder.activate_block(true_block) + builder.assign(retval, new_value(), -1) + builder.goto(exit_block) + builder.activate_block(false_block) + + comprehension_helper(builder, loop_params, gen_inner_stmts, gen.line) + builder.goto_and_activate(exit_block) + + return retval + + +@specialize_function("builtins.sum") +def translate_sum_call(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + # specialized implementation is used if: + # - only one or two arguments given (if not, sum() has been given invalid arguments) + # - first argument is a Generator (there is no benefit to optimizing the performance of eg. + # sum([1, 2, 3]), so non-Generator Iterables are not handled) + if not ( + len(expr.args) in (1, 2) + and expr.arg_kinds[0] == ARG_POS + and isinstance(expr.args[0], GeneratorExpr) + ): + return None + + # handle 'start' argument, if given + if len(expr.args) == 2: + # ensure call to sum() was properly constructed + if expr.arg_kinds[1] not in (ARG_POS, ARG_NAMED): + return None + start_expr = expr.args[1] + else: + start_expr = IntExpr(0) + + gen_expr = expr.args[0] + target_type = builder.node_type(expr) + retval = Register(target_type) + builder.assign(retval, builder.coerce(builder.accept(start_expr), target_type, -1), -1) + + def gen_inner_stmts() -> None: + call_expr = builder.accept(gen_expr.left_expr) + builder.assign(retval, builder.binary_op(retval, call_expr, "+", -1), -1) + + loop_params = list( + zip(gen_expr.indices, gen_expr.sequences, gen_expr.condlists, gen_expr.is_async) + ) + comprehension_helper(builder, loop_params, gen_inner_stmts, gen_expr.line) + + return retval + + +@specialize_function("dataclasses.field") +@specialize_function("attr.ib") +@specialize_function("attr.attrib") +@specialize_function("attr.Factory") +def translate_dataclasses_field_call( + builder: IRBuilder, expr: CallExpr, callee: RefExpr +) -> Value | None: + """Special case for 'dataclasses.field', 'attr.attrib', and 'attr.Factory' + function calls because the results of such calls are type-checked + by mypy using the types of the arguments to their respective + functions, resulting in attempted coercions by mypyc that throw a + runtime error. + """ + builder.types[expr] = AnyType(TypeOfAny.from_error) + return None + + +@specialize_function("builtins.next") +def translate_next_call(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Special case for calling next() on a generator expression, an + idiom that shows up some in mypy. + + For example, next(x for x in l if x.id == 12, None) will + generate code that searches l for an element where x.id == 12 + and produce the first such object, or None if no such element + exists. + """ + if not ( + expr.arg_kinds in ([ARG_POS], [ARG_POS, ARG_POS]) + and isinstance(expr.args[0], GeneratorExpr) + ): + return None + + gen = expr.args[0] + retval = Register(builder.node_type(expr)) + default_val = builder.accept(expr.args[1]) if len(expr.args) > 1 else None + exit_block = BasicBlock() + + def gen_inner_stmts() -> None: + # next takes the first element of the generator, so if + # something gets produced, we are done. + builder.assign(retval, builder.accept(gen.left_expr), gen.left_expr.line) + builder.goto(exit_block) + + loop_params = list(zip(gen.indices, gen.sequences, gen.condlists, gen.is_async)) + comprehension_helper(builder, loop_params, gen_inner_stmts, gen.line) + + # Now we need the case for when nothing got hit. If there was + # a default value, we produce it, and otherwise we raise + # StopIteration. + if default_val: + builder.assign(retval, default_val, gen.left_expr.line) + builder.goto(exit_block) + else: + builder.add(RaiseStandardError(RaiseStandardError.STOP_ITERATION, None, expr.line)) + builder.add(Unreachable()) + + builder.activate_block(exit_block) + return retval + + +@specialize_function("builtins.isinstance") +def translate_isinstance(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Special case for builtins.isinstance. + + Prevent coercions on the thing we are checking the instance of - + there is no need to coerce something to a new type before checking + what type it is, and the coercion could lead to bugs. + """ + if ( + len(expr.args) == 2 + and expr.arg_kinds == [ARG_POS, ARG_POS] + and isinstance(expr.args[1], (RefExpr, TupleExpr)) + ): + builder.types[expr.args[0]] = AnyType(TypeOfAny.from_error) + + irs = builder.flatten_classes(expr.args[1]) + if irs is not None: + can_borrow = all( + ir.is_ext_class and not ir.inherits_python and not ir.allow_interpreted_subclasses + for ir in irs + ) + obj = builder.accept(expr.args[0], can_borrow=can_borrow) + return builder.builder.isinstance_helper(obj, irs, expr.line) + return None + + +@specialize_function("setdefault", dict_rprimitive) +def translate_dict_setdefault(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Special case for 'dict.setdefault' which would only construct + default empty collection when needed. + + The dict_setdefault_spec_init_op checks whether the dict contains + the key and would construct the empty collection only once. + + For example, this specializer works for the following cases: + d.setdefault(key, set()).add(value) + d.setdefault(key, []).append(value) + d.setdefault(key, {})[inner_key] = inner_val + """ + if ( + len(expr.args) == 2 + and expr.arg_kinds == [ARG_POS, ARG_POS] + and isinstance(callee, MemberExpr) + ): + arg = expr.args[1] + if isinstance(arg, ListExpr): + if len(arg.items): + return None + data_type = Integer(1, c_int_rprimitive, expr.line) + elif isinstance(arg, DictExpr): + if len(arg.items): + return None + data_type = Integer(2, c_int_rprimitive, expr.line) + elif ( + isinstance(arg, CallExpr) + and isinstance(arg.callee, NameExpr) + and arg.callee.fullname == "builtins.set" + ): + if len(arg.args): + return None + data_type = Integer(3, c_int_rprimitive, expr.line) + else: + return None + + callee_dict = builder.accept(callee.expr) + key_val = builder.accept(expr.args[0]) + return builder.call_c( + dict_setdefault_spec_init_op, [callee_dict, key_val, data_type], expr.line + ) + return None + + +@specialize_function("format", str_rprimitive) +def translate_str_format(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if ( + isinstance(callee, MemberExpr) + and isinstance(callee.expr, StrExpr) + and expr.arg_kinds.count(ARG_POS) == len(expr.arg_kinds) + ): + format_str = callee.expr.value + tokens = tokenizer_format_call(format_str) + if tokens is None: + return None + literals, format_ops = tokens + # Convert variables to strings + substitutions = convert_format_expr_to_str(builder, format_ops, expr.args, expr.line) + if substitutions is None: + return None + return join_formatted_strings(builder, literals, substitutions, expr.line) + return None + + +@specialize_function("join", str_rprimitive) +def translate_fstring(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Special case for f-string, which is translated into str.join() + in mypy AST. + + This specializer optimizes simplest f-strings which don't contain + any format operation. + """ + if ( + isinstance(callee, MemberExpr) + and isinstance(callee.expr, StrExpr) + and callee.expr.value == "" + and expr.arg_kinds == [ARG_POS] + and isinstance(expr.args[0], ListExpr) + ): + for item in expr.args[0].items: + if isinstance(item, StrExpr): + continue + elif isinstance(item, CallExpr): + if not isinstance(item.callee, MemberExpr) or item.callee.name != "format": + return None + elif ( + not isinstance(item.callee.expr, StrExpr) or item.callee.expr.value != "{:{}}" + ): + return None + + if not isinstance(item.args[1], StrExpr) or item.args[1].value != "": + return None + else: + return None + + format_ops = [] + exprs: list[Expression] = [] + + for item in expr.args[0].items: + if isinstance(item, StrExpr) and item.value != "": + format_ops.append(FormatOp.STR) + exprs.append(item) + elif isinstance(item, CallExpr): + format_ops.append(FormatOp.STR) + exprs.append(item.args[0]) + + substitutions = convert_format_expr_to_str(builder, format_ops, exprs, expr.line) + if substitutions is None: + return None + + return join_formatted_strings(builder, None, substitutions, expr.line) + return None + + +@specialize_function("encode", str_rprimitive) +def str_encode_fast_path(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Specialize common cases of str.encode for most used encodings and strict errors.""" + + if not isinstance(callee, MemberExpr): + return None + + # We can only specialize if we have string literals as args + if len(expr.arg_kinds) > 0 and not isinstance(expr.args[0], StrExpr): + return None + if len(expr.arg_kinds) > 1 and not isinstance(expr.args[1], StrExpr): + return None + + encoding = "utf8" + errors = "strict" + if len(expr.arg_kinds) > 0 and isinstance(expr.args[0], StrExpr): + if expr.arg_kinds[0] == ARG_NAMED: + if expr.arg_names[0] == "encoding": + encoding = expr.args[0].value + elif expr.arg_names[0] == "errors": + errors = expr.args[0].value + elif expr.arg_kinds[0] == ARG_POS: + encoding = expr.args[0].value + else: + return None + if len(expr.arg_kinds) > 1 and isinstance(expr.args[1], StrExpr): + if expr.arg_kinds[1] == ARG_NAMED: + if expr.arg_names[1] == "encoding": + encoding = expr.args[1].value + elif expr.arg_names[1] == "errors": + errors = expr.args[1].value + elif expr.arg_kinds[1] == ARG_POS: + errors = expr.args[1].value + else: + return None + + if errors != "strict": + # We can only specialize strict errors + return None + + encoding = encoding.lower().replace("-", "").replace("_", "") # normalize + # Specialized encodings and their accepted aliases + if encoding in ["u8", "utf", "utf8", "cp65001"]: + return builder.call_c(str_encode_utf8_strict, [builder.accept(callee.expr)], expr.line) + elif encoding in ["646", "ascii", "usascii"]: + return builder.call_c(str_encode_ascii_strict, [builder.accept(callee.expr)], expr.line) + elif encoding in ["iso88591", "8859", "cp819", "latin", "latin1", "l1"]: + return builder.call_c(str_encode_latin1_strict, [builder.accept(callee.expr)], expr.line) + + return None + + +@specialize_function("mypy_extensions.i64") +def translate_i64(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if is_int64_rprimitive(arg_type): + return builder.accept(arg) + elif is_int32_rprimitive(arg_type) or is_int16_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Extend(val, int64_rprimitive, signed=True, line=expr.line)) + elif is_uint8_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Extend(val, int64_rprimitive, signed=False, line=expr.line)) + elif is_int_rprimitive(arg_type) or is_bool_rprimitive(arg_type): + val = builder.accept(arg) + return builder.coerce(val, int64_rprimitive, expr.line) + return None + + +@specialize_function("mypy_extensions.i32") +def translate_i32(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if is_int32_rprimitive(arg_type): + return builder.accept(arg) + elif is_int64_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Truncate(val, int32_rprimitive, line=expr.line)) + elif is_int16_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Extend(val, int32_rprimitive, signed=True, line=expr.line)) + elif is_uint8_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Extend(val, int32_rprimitive, signed=False, line=expr.line)) + elif is_int_rprimitive(arg_type) or is_bool_rprimitive(arg_type): + val = builder.accept(arg) + val = truncate_literal(val, int32_rprimitive) + return builder.coerce(val, int32_rprimitive, expr.line) + return None + + +@specialize_function("mypy_extensions.i16") +def translate_i16(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if is_int16_rprimitive(arg_type): + return builder.accept(arg) + elif is_int32_rprimitive(arg_type) or is_int64_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Truncate(val, int16_rprimitive, line=expr.line)) + elif is_uint8_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Extend(val, int16_rprimitive, signed=False, line=expr.line)) + elif is_int_rprimitive(arg_type) or is_bool_rprimitive(arg_type): + val = builder.accept(arg) + val = truncate_literal(val, int16_rprimitive) + return builder.coerce(val, int16_rprimitive, expr.line) + return None + + +@specialize_function("mypy_extensions.u8") +def translate_u8(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if is_uint8_rprimitive(arg_type): + return builder.accept(arg) + elif ( + is_int16_rprimitive(arg_type) + or is_int32_rprimitive(arg_type) + or is_int64_rprimitive(arg_type) + ): + val = builder.accept(arg) + return builder.add(Truncate(val, uint8_rprimitive, line=expr.line)) + elif is_int_rprimitive(arg_type) or is_bool_rprimitive(arg_type): + val = builder.accept(arg) + val = truncate_literal(val, uint8_rprimitive) + return builder.coerce(val, uint8_rprimitive, expr.line) + return None + + +def truncate_literal(value: Value, rtype: RPrimitive) -> Value: + """If value is an integer literal value, truncate it to given native int rtype. + + For example, truncate 256 into 0 if rtype is u8. + """ + if not isinstance(value, Integer): + return value # Not a literal, nothing to do + x = value.numeric_value() + max_unsigned = (1 << (rtype.size * 8)) - 1 + x = x & max_unsigned + if rtype.is_signed and x >= (max_unsigned + 1) // 2: + # Adjust to make it a negative value + x -= max_unsigned + 1 + return Integer(x, rtype) + + +@specialize_function("builtins.int") +def translate_int(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if ( + is_bool_rprimitive(arg_type) + or is_int_rprimitive(arg_type) + or is_fixed_width_rtype(arg_type) + ): + src = builder.accept(arg) + return builder.coerce(src, int_rprimitive, expr.line) + return None + + +@specialize_function("builtins.bool") +def translate_bool(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + src = builder.accept(arg) + return builder.builder.bool_value(src) + + +@specialize_function("builtins.float") +def translate_float(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if is_float_rprimitive(arg_type): + # No-op float conversion. + return builder.accept(arg) + return None + + +@specialize_function("builtins.ord") +def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + if isinstance(arg, (StrExpr, BytesExpr)) and len(arg.value) == 1: + return Integer(ord(arg.value)) + return None diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py new file mode 100644 index 000000000000..bd4acccf077a --- /dev/null +++ b/mypyc/irbuild/statement.py @@ -0,0 +1,1053 @@ +"""Transform mypy statement ASTs to mypyc IR (Intermediate Representation). + +The top-level AST transformation logic is implemented in mypyc.irbuild.visitor +and mypyc.irbuild.builder. + +A few statements are transformed in mypyc.irbuild.function (yield, for example). +""" + +from __future__ import annotations + +import importlib.util +from typing import Callable, Sequence + +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + AssertStmt, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + ContinueStmt, + DelStmt, + Expression, + ExpressionStmt, + ForStmt, + IfStmt, + Import, + ImportAll, + ImportFrom, + ListExpr, + Lvalue, + MatchStmt, + OperatorAssignmentStmt, + RaiseStmt, + ReturnStmt, + StarExpr, + StrExpr, + TempNode, + TryStmt, + TupleExpr, + TypeAliasStmt, + WhileStmt, + WithStmt, + YieldExpr, + YieldFromExpr, +) +from mypyc.ir.ops import ( + NAMESPACE_MODULE, + NO_TRACEBACK_LINE_NO, + Assign, + BasicBlock, + Branch, + InitStatic, + Integer, + LoadAddress, + LoadErrorValue, + LoadLiteral, + LoadStatic, + MethodCall, + PrimitiveDescription, + RaiseStandardError, + Register, + Return, + TupleGet, + Unborrow, + Unreachable, + Value, +) +from mypyc.ir.rtypes import ( + RInstance, + RTuple, + c_pyssize_t_rprimitive, + exc_rtuple, + is_tagged, + none_rprimitive, + object_pointer_rprimitive, + object_rprimitive, +) +from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional +from mypyc.irbuild.builder import IRBuilder, create_type_params, int_borrow_friendly_op +from mypyc.irbuild.for_helpers import for_loop_helper +from mypyc.irbuild.generator import add_raise_exception_blocks_to_generator_class +from mypyc.irbuild.nonlocalcontrol import ( + ExceptNonlocalControl, + FinallyNonlocalControl, + TryFinallyNonlocalControl, +) +from mypyc.irbuild.targets import ( + AssignmentTarget, + AssignmentTargetAttr, + AssignmentTargetIndex, + AssignmentTargetRegister, + AssignmentTargetTuple, +) +from mypyc.primitives.exc_ops import ( + error_catch_op, + exc_matches_op, + get_exc_info_op, + get_exc_value_op, + keep_propagating_op, + raise_exception_op, + reraise_exception_op, + restore_exc_info_op, +) +from mypyc.primitives.generic_ops import iter_op, next_raw_op, py_delattr_op +from mypyc.primitives.misc_ops import ( + check_stop_op, + coro_op, + import_from_many_op, + import_many_op, + import_op, + send_op, + set_type_alias_compute_function_op, + type_op, + yield_from_except_op, +) + +from .match import MatchVisitor + +GenFunc = Callable[[], None] +ValueGenFunc = Callable[[], Value] + + +def transform_block(builder: IRBuilder, block: Block) -> None: + if not block.is_unreachable: + builder.block_reachable_stack.append(True) + for stmt in block.body: + builder.accept(stmt) + if not builder.block_reachable_stack[-1]: + # The rest of the block is unreachable, so skip it + break + builder.block_reachable_stack.pop() + # Raise a RuntimeError if we hit a non-empty unreachable block. + # Don't complain about empty unreachable blocks, since mypy inserts + # those after `if MYPY`. + elif block.body: + builder.add( + RaiseStandardError( + RaiseStandardError.RUNTIME_ERROR, "Reached allegedly unreachable code!", block.line + ) + ) + builder.add(Unreachable()) + + +def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: + if isinstance(stmt.expr, StrExpr): + # Docstring. Ignore + return + # ExpressionStmts do not need to be coerced like other Expressions, so we shouldn't + # call builder.accept here. + stmt.expr.accept(builder.visitor) + builder.flush_keep_alives() + + +def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: + if stmt.expr: + retval = builder.accept(stmt.expr) + else: + retval = builder.builder.none() + retval = builder.coerce(retval, builder.ret_types[-1], stmt.line) + builder.nonlocal_control[-1].gen_return(builder, retval, stmt.line) + + +def transform_assignment_stmt(builder: IRBuilder, stmt: AssignmentStmt) -> None: + lvalues = stmt.lvalues + assert lvalues + builder.disallow_class_assignments(lvalues, stmt.line) + first_lvalue = lvalues[0] + if stmt.type and isinstance(stmt.rvalue, TempNode): + # This is actually a variable annotation without initializer. Don't generate + # an assignment but we need to call get_assignment_target since it adds a + # name binding as a side effect. + builder.get_assignment_target(first_lvalue, stmt.line) + return + + # Special case multiple assignments like 'x, y = e1, e2'. + if ( + isinstance(first_lvalue, (TupleExpr, ListExpr)) + and isinstance(stmt.rvalue, (TupleExpr, ListExpr)) + and len(first_lvalue.items) == len(stmt.rvalue.items) + and all(is_simple_lvalue(item) for item in first_lvalue.items) + and len(lvalues) == 1 + ): + temps = [] + for right in stmt.rvalue.items: + rvalue_reg = builder.accept(right) + temp = Register(rvalue_reg.type) + builder.assign(temp, rvalue_reg, stmt.line) + temps.append(temp) + for left, temp in zip(first_lvalue.items, temps): + assignment_target = builder.get_assignment_target(left) + builder.assign(assignment_target, temp, stmt.line) + builder.flush_keep_alives() + return + + line = stmt.rvalue.line + rvalue_reg = builder.accept(stmt.rvalue) + + if builder.non_function_scope() and stmt.is_final_def: + builder.init_final_static(first_lvalue, rvalue_reg) + + # Special-case multiple assignments like 'x, y = expr' to reduce refcount ops. + if ( + isinstance(first_lvalue, (TupleExpr, ListExpr)) + and isinstance(rvalue_reg.type, RTuple) + and len(rvalue_reg.type.types) == len(first_lvalue.items) + and len(lvalues) == 1 + and all(is_simple_lvalue(item) for item in first_lvalue.items) + and any(t.is_refcounted for t in rvalue_reg.type.types) + ): + n = len(first_lvalue.items) + for i in range(n): + target = builder.get_assignment_target(first_lvalue.items[i]) + rvalue_item = builder.add(TupleGet(rvalue_reg, i, borrow=True)) + rvalue_item = builder.add(Unborrow(rvalue_item)) + builder.assign(target, rvalue_item, line) + builder.builder.keep_alive([rvalue_reg], steal=True) + builder.flush_keep_alives() + return + + for lvalue in lvalues: + target = builder.get_assignment_target(lvalue) + builder.assign(target, rvalue_reg, line) + builder.flush_keep_alives() + + +def is_simple_lvalue(expr: Expression) -> bool: + return not isinstance(expr, (StarExpr, ListExpr, TupleExpr)) + + +def transform_operator_assignment_stmt(builder: IRBuilder, stmt: OperatorAssignmentStmt) -> None: + """Operator assignment statement such as x += 1""" + builder.disallow_class_assignments([stmt.lvalue], stmt.line) + if ( + is_tagged(builder.node_type(stmt.lvalue)) + and is_tagged(builder.node_type(stmt.rvalue)) + and stmt.op in int_borrow_friendly_op + ): + can_borrow = is_borrow_friendly_expr(builder, stmt.rvalue) and is_borrow_friendly_expr( + builder, stmt.lvalue + ) + else: + can_borrow = False + target = builder.get_assignment_target(stmt.lvalue) + target_value = builder.read(target, stmt.line, can_borrow=can_borrow) + rreg = builder.accept(stmt.rvalue, can_borrow=can_borrow) + # the Python parser strips the '=' from operator assignment statements, so re-add it + op = stmt.op + "=" + res = builder.binary_op(target_value, rreg, op, stmt.line) + # usually operator assignments are done in-place + # but when target doesn't support that we need to manually assign + builder.assign(target, res, res.line) + builder.flush_keep_alives() + + +def import_globals_id_and_name(module_id: str, as_name: str | None) -> tuple[str, str]: + """Compute names for updating the globals dict with the appropriate module. + + * For 'import foo.bar as baz' we add 'foo.bar' with the name 'baz' + * For 'import foo.bar' we add 'foo' with the name 'foo' + + Typically we then ignore these entries and access things directly + via the module static, but we will use the globals version for + modules that mypy couldn't find, since it doesn't analyze module + references from those properly.""" + if as_name: + globals_id = module_id + globals_name = as_name + else: + globals_id = globals_name = module_id.split(".")[0] + + return globals_id, globals_name + + +def transform_import(builder: IRBuilder, node: Import) -> None: + if node.is_mypy_only: + return + + # Imports (not from imports!) are processed in an odd way so they can be + # table-driven and compact. Here's how it works: + # + # Import nodes are divided in groups (in the prebuild visitor). Each group + # consists of consecutive Import nodes: + # + # import mod <| group #1 + # import mod2 | + # + # def foo() -> None: + # import mod3 <- group #2 (*) + # + # import mod4 <| group #3 + # import mod5 | + # + # Every time we encounter the first import of a group, build IR to call a + # helper function that will perform all of the group's imports in one go. + if not node.is_top_level: + # (*) Unless the import is within a function. In that case, prioritize + # speed over codesize when generating IR. + globals = builder.load_globals_dict() + for mod_id, as_name in node.ids: + builder.gen_import(mod_id, node.line) + globals_id, globals_name = import_globals_id_and_name(mod_id, as_name) + builder.gen_method_call( + globals, + "__setitem__", + [builder.load_str(globals_name), builder.get_module(globals_id, node.line)], + result_type=None, + line=node.line, + ) + return + + if node not in builder.module_import_groups: + return + + modules = [] + static_ptrs = [] + # To show the right line number on failure, we have to add the traceback + # entry within the helper function (which is admittedly ugly). To drive + # this, we need the line number corresponding to each module. + mod_lines = [] + for import_node in builder.module_import_groups[node]: + for mod_id, as_name in import_node.ids: + builder.imports[mod_id] = None + modules.append((mod_id, *import_globals_id_and_name(mod_id, as_name))) + mod_static = LoadStatic(object_rprimitive, mod_id, namespace=NAMESPACE_MODULE) + static_ptrs.append(builder.add(LoadAddress(object_pointer_rprimitive, mod_static))) + mod_lines.append(Integer(import_node.line, c_pyssize_t_rprimitive)) + + static_array_ptr = builder.builder.setup_rarray(object_pointer_rprimitive, static_ptrs) + import_line_ptr = builder.builder.setup_rarray(c_pyssize_t_rprimitive, mod_lines) + builder.call_c( + import_many_op, + [ + builder.add(LoadLiteral(tuple(modules), object_rprimitive)), + static_array_ptr, + builder.load_globals_dict(), + builder.load_str(builder.module_path), + builder.load_str(builder.fn_info.name), + import_line_ptr, + ], + NO_TRACEBACK_LINE_NO, + ) + + +def transform_import_from(builder: IRBuilder, node: ImportFrom) -> None: + if node.is_mypy_only: + return + + module_state = builder.graph[builder.module_name] + if builder.module_path.endswith("__init__.py"): + module_package = builder.module_name + elif module_state.ancestors is not None and module_state.ancestors: + module_package = module_state.ancestors[0] + else: + module_package = "" + + id = importlib.util.resolve_name("." * node.relative + node.id, module_package) + builder.imports[id] = None + + names = [name for name, _ in node.names] + as_names = [as_name or name for name, as_name in node.names] + names_literal = builder.add(LoadLiteral(tuple(names), object_rprimitive)) + if as_names == names: + # Reuse names tuple to reduce verbosity. + as_names_literal = names_literal + else: + as_names_literal = builder.add(LoadLiteral(tuple(as_names), object_rprimitive)) + # Note that we miscompile import from inside of functions here, + # since that case *shouldn't* load everything into the globals dict. + # This probably doesn't matter much and the code runs basically right. + module = builder.call_c( + import_from_many_op, + [builder.load_str(id), names_literal, as_names_literal, builder.load_globals_dict()], + node.line, + ) + builder.add(InitStatic(module, id, namespace=NAMESPACE_MODULE)) + + +def transform_import_all(builder: IRBuilder, node: ImportAll) -> None: + if node.is_mypy_only: + return + builder.gen_import(node.id, node.line) + + +def transform_if_stmt(builder: IRBuilder, stmt: IfStmt) -> None: + if_body, next = BasicBlock(), BasicBlock() + else_body = BasicBlock() if stmt.else_body else next + + # If statements are normalized + assert len(stmt.expr) == 1 + + process_conditional(builder, stmt.expr[0], if_body, else_body) + builder.activate_block(if_body) + builder.accept(stmt.body[0]) + builder.goto(next) + if stmt.else_body: + builder.activate_block(else_body) + builder.accept(stmt.else_body) + builder.goto(next) + builder.activate_block(next) + + +def transform_while_stmt(builder: IRBuilder, s: WhileStmt) -> None: + body, next, top, else_block = BasicBlock(), BasicBlock(), BasicBlock(), BasicBlock() + normal_loop_exit = else_block if s.else_body is not None else next + + builder.push_loop_stack(top, next) + + # Split block so that we get a handle to the top of the loop. + builder.goto_and_activate(top) + process_conditional(builder, s.expr, body, normal_loop_exit) + + builder.activate_block(body) + builder.accept(s.body) + # Add branch to the top at the end of the body. + builder.goto(top) + + builder.pop_loop_stack() + + if s.else_body is not None: + builder.activate_block(else_block) + builder.accept(s.else_body) + builder.goto(next) + + builder.activate_block(next) + + +def transform_for_stmt(builder: IRBuilder, s: ForStmt) -> None: + def body() -> None: + builder.accept(s.body) + + def else_block() -> None: + assert s.else_body is not None + builder.accept(s.else_body) + + for_loop_helper( + builder, s.index, s.expr, body, else_block if s.else_body else None, s.is_async, s.line + ) + + +def transform_break_stmt(builder: IRBuilder, node: BreakStmt) -> None: + builder.nonlocal_control[-1].gen_break(builder, node.line) + + +def transform_continue_stmt(builder: IRBuilder, node: ContinueStmt) -> None: + builder.nonlocal_control[-1].gen_continue(builder, node.line) + + +def transform_raise_stmt(builder: IRBuilder, s: RaiseStmt) -> None: + if s.expr is None: + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + return + + exc = builder.accept(s.expr) + builder.call_c(raise_exception_op, [exc], s.line) + builder.add(Unreachable()) + + +def transform_try_except( + builder: IRBuilder, + body: GenFunc, + handlers: Sequence[tuple[tuple[ValueGenFunc, int] | None, Expression | None, GenFunc]], + else_body: GenFunc | None, + line: int, +) -> None: + """Generalized try/except/else handling that takes functions to gen the bodies. + + The point of this is to also be able to support with.""" + assert handlers, "try needs except" + + except_entry, exit_block, cleanup_block = BasicBlock(), BasicBlock(), BasicBlock() + double_except_block = BasicBlock() + # If there is an else block, jump there after the try, otherwise just leave + else_block = BasicBlock() if else_body else exit_block + + # Compile the try block with an error handler + builder.builder.push_error_handler(except_entry) + builder.goto_and_activate(BasicBlock()) + body() + builder.goto(else_block) + builder.builder.pop_error_handler() + + # The error handler catches the error and then checks it + # against the except clauses. We compile the error handler + # itself with an error handler so that it can properly restore + # the *old* exc_info if an exception occurs. + # The exception chaining will be done automatically when the + # exception is raised, based on the exception in exc_info. + builder.builder.push_error_handler(double_except_block) + builder.activate_block(except_entry) + old_exc = builder.maybe_spill(builder.call_c(error_catch_op, [], line)) + # Compile the except blocks with the nonlocal control flow overridden to clear exc_info + builder.nonlocal_control.append(ExceptNonlocalControl(builder.nonlocal_control[-1], old_exc)) + + # Process the bodies + for type, var, handler_body in handlers: + next_block = None + if type: + type_f, type_line = type + next_block, body_block = BasicBlock(), BasicBlock() + matches = builder.call_c(exc_matches_op, [type_f()], type_line) + builder.add(Branch(matches, body_block, next_block, Branch.BOOL)) + builder.activate_block(body_block) + if var: + target = builder.get_assignment_target(var) + builder.assign(target, builder.call_c(get_exc_value_op, [], var.line), var.line) + handler_body() + builder.goto(cleanup_block) + if next_block: + builder.activate_block(next_block) + + # Reraise the exception if needed + if next_block: + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + builder.nonlocal_control.pop() + builder.builder.pop_error_handler() + + # Cleanup for if we leave except through normal control flow: + # restore the saved exc_info information and continue propagating + # the exception if it exists. + builder.activate_block(cleanup_block) + builder.call_c(restore_exc_info_op, [builder.read(old_exc)], line) + builder.goto(exit_block) + + # Cleanup for if we leave except through a raised exception: + # restore the saved exc_info information and continue propagating + # the exception. + builder.activate_block(double_except_block) + builder.call_c(restore_exc_info_op, [builder.read(old_exc)], line) + builder.call_c(keep_propagating_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + # If present, compile the else body in the obvious way + if else_body: + builder.activate_block(else_block) + else_body() + builder.goto(exit_block) + + builder.activate_block(exit_block) + + +def transform_try_except_stmt(builder: IRBuilder, t: TryStmt) -> None: + def body() -> None: + builder.accept(t.body) + + # Work around scoping woes + def make_handler(body: Block) -> GenFunc: + return lambda: builder.accept(body) + + def make_entry(type: Expression) -> tuple[ValueGenFunc, int]: + return (lambda: builder.accept(type), type.line) + + handlers = [ + (make_entry(type) if type else None, var, make_handler(body)) + for type, var, body in zip(t.types, t.vars, t.handlers) + ] + else_body = (lambda: builder.accept(t.else_body)) if t.else_body else None + transform_try_except(builder, body, handlers, else_body, t.line) + + +def try_finally_try( + builder: IRBuilder, + err_handler: BasicBlock, + return_entry: BasicBlock, + main_entry: BasicBlock, + try_body: GenFunc, +) -> Register | AssignmentTarget | None: + # Compile the try block with an error handler + control = TryFinallyNonlocalControl(return_entry) + builder.builder.push_error_handler(err_handler) + + builder.nonlocal_control.append(control) + builder.goto_and_activate(BasicBlock()) + try_body() + builder.goto(main_entry) + builder.nonlocal_control.pop() + builder.builder.pop_error_handler() + + return control.ret_reg + + +def try_finally_entry_blocks( + builder: IRBuilder, + err_handler: BasicBlock, + return_entry: BasicBlock, + main_entry: BasicBlock, + finally_block: BasicBlock, + ret_reg: Register | AssignmentTarget | None, +) -> Value: + old_exc = Register(exc_rtuple) + + # Entry block for non-exceptional flow + builder.activate_block(main_entry) + if ret_reg: + builder.assign(ret_reg, builder.add(LoadErrorValue(builder.ret_types[-1])), -1) + builder.goto(return_entry) + + builder.activate_block(return_entry) + builder.add(Assign(old_exc, builder.add(LoadErrorValue(exc_rtuple)))) + builder.goto(finally_block) + + # Entry block for errors + builder.activate_block(err_handler) + if ret_reg: + builder.assign(ret_reg, builder.add(LoadErrorValue(builder.ret_types[-1])), -1) + builder.add(Assign(old_exc, builder.call_c(error_catch_op, [], -1))) + builder.goto(finally_block) + + return old_exc + + +def try_finally_body( + builder: IRBuilder, finally_block: BasicBlock, finally_body: GenFunc, old_exc: Value +) -> tuple[BasicBlock, FinallyNonlocalControl]: + cleanup_block = BasicBlock() + # Compile the finally block with the nonlocal control flow overridden to restore exc_info + builder.builder.push_error_handler(cleanup_block) + finally_control = FinallyNonlocalControl(builder.nonlocal_control[-1], old_exc) + builder.nonlocal_control.append(finally_control) + builder.activate_block(finally_block) + finally_body() + builder.nonlocal_control.pop() + + return cleanup_block, finally_control + + +def try_finally_resolve_control( + builder: IRBuilder, + cleanup_block: BasicBlock, + finally_control: FinallyNonlocalControl, + old_exc: Value, + ret_reg: Register | AssignmentTarget | None, +) -> BasicBlock: + """Resolve the control flow out of a finally block. + + This means returning if there was a return, propagating + exceptions, break/continue (soon), or just continuing on. + """ + reraise, rest = BasicBlock(), BasicBlock() + builder.add(Branch(old_exc, rest, reraise, Branch.IS_ERROR)) + + # Reraise the exception if there was one + builder.activate_block(reraise) + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + builder.builder.pop_error_handler() + + # If there was a return, keep returning + if ret_reg: + builder.activate_block(rest) + return_block, rest = BasicBlock(), BasicBlock() + builder.add(Branch(builder.read(ret_reg), rest, return_block, Branch.IS_ERROR)) + + builder.activate_block(return_block) + builder.nonlocal_control[-1].gen_return(builder, builder.read(ret_reg), -1) + + # TODO: handle break/continue + builder.activate_block(rest) + out_block = BasicBlock() + builder.goto(out_block) + + # If there was an exception, restore again + builder.activate_block(cleanup_block) + finally_control.gen_cleanup(builder, -1) + builder.call_c(keep_propagating_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + return out_block + + +def transform_try_finally_stmt( + builder: IRBuilder, try_body: GenFunc, finally_body: GenFunc +) -> None: + """Generalized try/finally handling that takes functions to gen the bodies. + + The point of this is to also be able to support with.""" + # Finally is a big pain, because there are so many ways that + # exits can occur. We emit 10+ basic blocks for every finally! + + err_handler, main_entry, return_entry, finally_block = ( + BasicBlock(), + BasicBlock(), + BasicBlock(), + BasicBlock(), + ) + + # Compile the body of the try + ret_reg = try_finally_try(builder, err_handler, return_entry, main_entry, try_body) + + # Set up the entry blocks for the finally statement + old_exc = try_finally_entry_blocks( + builder, err_handler, return_entry, main_entry, finally_block, ret_reg + ) + + # Compile the body of the finally + cleanup_block, finally_control = try_finally_body( + builder, finally_block, finally_body, old_exc + ) + + # Resolve the control flow out of the finally block + out_block = try_finally_resolve_control( + builder, cleanup_block, finally_control, old_exc, ret_reg + ) + + builder.activate_block(out_block) + + +def transform_try_stmt(builder: IRBuilder, t: TryStmt) -> None: + # Our compilation strategy for try/except/else/finally is to + # treat try/except/else and try/finally as separate language + # constructs that we compile separately. When we have a + # try/except/else/finally, we treat the try/except/else as the + # body of a try/finally block. + if t.is_star: + builder.error("Exception groups and except* cannot be compiled yet", t.line) + if t.finally_body: + + def transform_try_body() -> None: + if t.handlers: + transform_try_except_stmt(builder, t) + else: + builder.accept(t.body) + + body = t.finally_body + + transform_try_finally_stmt(builder, transform_try_body, lambda: builder.accept(body)) + else: + transform_try_except_stmt(builder, t) + + +def get_sys_exc_info(builder: IRBuilder) -> list[Value]: + exc_info = builder.call_c(get_exc_info_op, [], -1) + return [builder.add(TupleGet(exc_info, i, -1)) for i in range(3)] + + +def transform_with( + builder: IRBuilder, + expr: Expression, + target: Lvalue | None, + body: GenFunc, + is_async: bool, + line: int, +) -> None: + # This is basically a straight transcription of the Python code in PEP 343. + # I don't actually understand why a bunch of it is the way it is. + # We could probably optimize the case where the manager is compiled by us, + # but that is not our common case at all, so. + + al = "a" if is_async else "" + + mgr_v = builder.accept(expr) + is_native = isinstance(mgr_v.type, RInstance) + if is_native: + value = builder.add(MethodCall(mgr_v, f"__{al}enter__", args=[], line=line)) + exit_ = None + else: + typ = builder.primitive_op(type_op, [mgr_v], line) + exit_ = builder.maybe_spill(builder.py_get_attr(typ, f"__{al}exit__", line)) + value = builder.py_call(builder.py_get_attr(typ, f"__{al}enter__", line), [mgr_v], line) + + mgr = builder.maybe_spill(mgr_v) + exc = builder.maybe_spill_assignable(builder.true()) + if is_async: + value = emit_await(builder, value, line) + + def maybe_natively_call_exit(exc_info: bool) -> Value: + if exc_info: + args = get_sys_exc_info(builder) + else: + none = builder.none_object() + args = [none, none, none] + + if is_native: + assert isinstance(mgr_v.type, RInstance) + exit_val = builder.gen_method_call( + builder.read(mgr), + f"__{al}exit__", + arg_values=args, + line=line, + result_type=none_rprimitive, + ) + else: + assert exit_ is not None + exit_val = builder.py_call(builder.read(exit_), [builder.read(mgr)] + args, line) + + if is_async: + return emit_await(builder, exit_val, line) + else: + return exit_val + + def try_body() -> None: + if target: + builder.assign(builder.get_assignment_target(target), value, line) + body() + + def except_body() -> None: + builder.assign(exc, builder.false(), line) + out_block, reraise_block = BasicBlock(), BasicBlock() + builder.add_bool_branch(maybe_natively_call_exit(exc_info=True), out_block, reraise_block) + builder.activate_block(reraise_block) + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + builder.activate_block(out_block) + + def finally_body() -> None: + out_block, exit_block = BasicBlock(), BasicBlock() + builder.add(Branch(builder.read(exc), exit_block, out_block, Branch.BOOL)) + builder.activate_block(exit_block) + + maybe_natively_call_exit(exc_info=False) + builder.goto_and_activate(out_block) + + transform_try_finally_stmt( + builder, + lambda: transform_try_except(builder, try_body, [(None, None, except_body)], None, line), + finally_body, + ) + + +def transform_with_stmt(builder: IRBuilder, o: WithStmt) -> None: + # Generate separate logic for each expr in it, left to right + def generate(i: int) -> None: + if i >= len(o.expr): + builder.accept(o.body) + else: + transform_with( + builder, o.expr[i], o.target[i], lambda: generate(i + 1), o.is_async, o.line + ) + + generate(0) + + +def transform_assert_stmt(builder: IRBuilder, a: AssertStmt) -> None: + if builder.options.strip_asserts: + return + cond = builder.accept(a.expr) + ok_block, error_block = BasicBlock(), BasicBlock() + builder.add_bool_branch(cond, ok_block, error_block) + builder.activate_block(error_block) + if a.msg is None: + # Special case (for simpler generated code) + builder.add(RaiseStandardError(RaiseStandardError.ASSERTION_ERROR, None, a.line)) + elif isinstance(a.msg, StrExpr): + # Another special case + builder.add(RaiseStandardError(RaiseStandardError.ASSERTION_ERROR, a.msg.value, a.line)) + else: + # The general case -- explicitly construct an exception instance + message = builder.accept(a.msg) + exc_type = builder.load_module_attr_by_fullname("builtins.AssertionError", a.line) + exc = builder.py_call(exc_type, [message], a.line) + builder.call_c(raise_exception_op, [exc], a.line) + builder.add(Unreachable()) + builder.activate_block(ok_block) + + +def transform_del_stmt(builder: IRBuilder, o: DelStmt) -> None: + transform_del_item(builder, builder.get_assignment_target(o.expr), o.line) + + +def transform_del_item(builder: IRBuilder, target: AssignmentTarget, line: int) -> None: + if isinstance(target, AssignmentTargetIndex): + builder.gen_method_call( + target.base, "__delitem__", [target.index], result_type=None, line=line + ) + elif isinstance(target, AssignmentTargetAttr): + if isinstance(target.obj_type, RInstance): + cl = target.obj_type.class_ir + if not cl.is_deletable(target.attr): + builder.error(f'"{target.attr}" cannot be deleted', line) + builder.note( + 'Using "__deletable__ = ' + + '[\'\']" in the class body enables "del obj."', + line, + ) + key = builder.load_str(target.attr) + builder.primitive_op(py_delattr_op, [target.obj, key], line) + elif isinstance(target, AssignmentTargetRegister): + # Delete a local by assigning an error value to it, which will + # prompt the insertion of uninit checks. + builder.add( + Assign(target.register, builder.add(LoadErrorValue(target.type, undefines=True))) + ) + elif isinstance(target, AssignmentTargetTuple): + for subtarget in target.items: + transform_del_item(builder, subtarget, line) + + +# yield/yield from/await + +# These are really expressions, not statements... but they depend on try/except/finally + + +def emit_yield(builder: IRBuilder, val: Value, line: int) -> Value: + retval = builder.coerce(val, builder.ret_types[-1], line) + + cls = builder.fn_info.generator_class + # Create a new block for the instructions immediately following the yield expression, and + # set the next label so that the next time '__next__' is called on the generator object, + # the function continues at the new block. + next_block = BasicBlock() + next_label = len(cls.continuation_blocks) + cls.continuation_blocks.append(next_block) + builder.assign(cls.next_label_target, Integer(next_label), line) + builder.add(Return(retval)) + builder.activate_block(next_block) + + add_raise_exception_blocks_to_generator_class(builder, line) + + assert cls.send_arg_reg is not None + return cls.send_arg_reg + + +def emit_yield_from_or_await( + builder: IRBuilder, val: Value, line: int, *, is_await: bool +) -> Value: + # This is basically an implementation of the code in PEP 380. + + # TODO: do we want to use the right types here? + result = Register(object_rprimitive) + to_yield_reg = Register(object_rprimitive) + received_reg = Register(object_rprimitive) + + get_op = coro_op if is_await else iter_op + if isinstance(get_op, PrimitiveDescription): + iter_val = builder.primitive_op(get_op, [val], line) + else: + iter_val = builder.call_c(get_op, [val], line) + + iter_reg = builder.maybe_spill_assignable(iter_val) + + stop_block, main_block, done_block = BasicBlock(), BasicBlock(), BasicBlock() + _y_init = builder.call_c(next_raw_op, [builder.read(iter_reg)], line) + builder.add(Branch(_y_init, stop_block, main_block, Branch.IS_ERROR)) + + # Try extracting a return value from a StopIteration and return it. + # If it wasn't, this reraises the exception. + builder.activate_block(stop_block) + builder.assign(result, builder.call_c(check_stop_op, [], line), line) + builder.goto(done_block) + + builder.activate_block(main_block) + builder.assign(to_yield_reg, _y_init, line) + + # OK Now the main loop! + loop_block = BasicBlock() + builder.goto_and_activate(loop_block) + + def try_body() -> None: + builder.assign(received_reg, emit_yield(builder, builder.read(to_yield_reg), line), line) + + def except_body() -> None: + # The body of the except is all implemented in a C function to + # reduce how much code we need to generate. It returns a value + # indicating whether to break or yield (or raise an exception). + val = Register(object_rprimitive) + val_address = builder.add(LoadAddress(object_pointer_rprimitive, val)) + to_stop = builder.call_c(yield_from_except_op, [builder.read(iter_reg), val_address], line) + + ok, stop = BasicBlock(), BasicBlock() + builder.add(Branch(to_stop, stop, ok, Branch.BOOL)) + + # The exception got swallowed. Continue, yielding the returned value + builder.activate_block(ok) + builder.assign(to_yield_reg, val, line) + builder.nonlocal_control[-1].gen_continue(builder, line) + + # The exception was a StopIteration. Stop iterating. + builder.activate_block(stop) + builder.assign(result, val, line) + builder.nonlocal_control[-1].gen_break(builder, line) + + def else_body() -> None: + # Do a next() or a .send(). It will return NULL on exception + # but it won't automatically propagate. + _y = builder.call_c(send_op, [builder.read(iter_reg), builder.read(received_reg)], line) + ok, stop = BasicBlock(), BasicBlock() + builder.add(Branch(_y, stop, ok, Branch.IS_ERROR)) + + # Everything's fine. Yield it. + builder.activate_block(ok) + builder.assign(to_yield_reg, _y, line) + builder.nonlocal_control[-1].gen_continue(builder, line) + + # Try extracting a return value from a StopIteration and return it. + # If it wasn't, this rereaises the exception. + builder.activate_block(stop) + builder.assign(result, builder.call_c(check_stop_op, [], line), line) + builder.nonlocal_control[-1].gen_break(builder, line) + + builder.push_loop_stack(loop_block, done_block) + transform_try_except(builder, try_body, [(None, None, except_body)], else_body, line) + builder.pop_loop_stack() + + builder.goto_and_activate(done_block) + return builder.read(result) + + +def emit_await(builder: IRBuilder, val: Value, line: int) -> Value: + return emit_yield_from_or_await(builder, val, line, is_await=True) + + +def transform_yield_expr(builder: IRBuilder, expr: YieldExpr) -> Value: + if builder.fn_info.is_coroutine: + builder.error("async generators are unimplemented", expr.line) + + if expr.expr: + retval = builder.accept(expr.expr) + else: + retval = builder.builder.none() + return emit_yield(builder, retval, expr.line) + + +def transform_yield_from_expr(builder: IRBuilder, o: YieldFromExpr) -> Value: + return emit_yield_from_or_await(builder, builder.accept(o.expr), o.line, is_await=False) + + +def transform_await_expr(builder: IRBuilder, o: AwaitExpr) -> Value: + return emit_yield_from_or_await(builder, builder.accept(o.expr), o.line, is_await=True) + + +def transform_match_stmt(builder: IRBuilder, m: MatchStmt) -> None: + m.accept(MatchVisitor(builder, m)) + + +def transform_type_alias_stmt(builder: IRBuilder, s: TypeAliasStmt) -> None: + line = s.line + # Use "_typing" to avoid importing "typing", as the latter can be expensive. + # "_typing" includes everything we need here. + mod = builder.call_c(import_op, [builder.load_str("_typing")], line) + type_params = create_type_params(builder, mod, s.type_args, s.line) + + type_alias_type = builder.py_get_attr(mod, "TypeAliasType", line) + args = [builder.load_str(s.name.name), builder.none()] + arg_names: list[str | None] = [None, None] + arg_kinds = [ARG_POS, ARG_POS] + if s.type_args: + args.append(builder.new_tuple(type_params, line)) + arg_names.append("type_params") + arg_kinds.append(ARG_NAMED) + alias = builder.py_call(type_alias_type, args, line, arg_names=arg_names, arg_kinds=arg_kinds) + + # Use primitive to set function used to lazily compute type alias type value. + # The value needs to be lazily computed to match Python runtime behavior, but + # Python public APIs don't support this, so we use a C primitive. + compute_fn = s.value.accept(builder.visitor) + builder.builder.primitive_op(set_type_alias_compute_function_op, [alias, compute_fn], line) + + target = builder.get_assignment_target(s.name) + builder.assign(target, alias, line) diff --git a/mypyc/irbuild/targets.py b/mypyc/irbuild/targets.py new file mode 100644 index 000000000000..270c2896bc06 --- /dev/null +++ b/mypyc/irbuild/targets.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from mypyc.ir.ops import Register, Value +from mypyc.ir.rtypes import RInstance, RType, object_rprimitive + + +class AssignmentTarget: + """Abstract base class for assignment targets during IR building.""" + + type: RType = object_rprimitive + + +class AssignmentTargetRegister(AssignmentTarget): + """Register as an assignment target. + + This is used for local variables and some temporaries. + """ + + def __init__(self, register: Register) -> None: + self.register = register + self.type = register.type + + +class AssignmentTargetIndex(AssignmentTarget): + """base[index] as assignment target""" + + def __init__(self, base: Value, index: Value) -> None: + self.base = base + self.index = index + # TODO: object_rprimitive won't be right for user-defined classes. Store the + # lvalue type in mypy and use a better type to avoid unneeded boxing. + self.type = object_rprimitive + + +class AssignmentTargetAttr(AssignmentTarget): + """obj.attr as assignment target""" + + def __init__(self, obj: Value, attr: str, can_borrow: bool = False) -> None: + self.obj = obj + self.attr = attr + self.can_borrow = can_borrow + if isinstance(obj.type, RInstance) and obj.type.class_ir.has_attr(attr): + # Native attribute reference + self.obj_type: RType = obj.type + self.type = obj.type.attr_type(attr) + else: + # Python attribute reference + self.obj_type = object_rprimitive + self.type = object_rprimitive + + +class AssignmentTargetTuple(AssignmentTarget): + """x, ..., y as assignment target""" + + def __init__(self, items: list[AssignmentTarget], star_idx: int | None = None) -> None: + self.items = items + self.star_idx = star_idx diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py new file mode 100644 index 000000000000..e27e509ad7fa --- /dev/null +++ b/mypyc/irbuild/util.py @@ -0,0 +1,198 @@ +"""Various utilities that don't depend on other modules in mypyc.irbuild.""" + +from __future__ import annotations + +from typing import Any + +from mypy.nodes import ( + ARG_NAMED, + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + GDEF, + ArgKind, + BytesExpr, + CallExpr, + ClassDef, + Decorator, + Expression, + FloatExpr, + FuncDef, + IntExpr, + NameExpr, + OverloadedFuncDef, + RefExpr, + StrExpr, + TupleExpr, + UnaryExpr, + Var, +) +from mypy.semanal import refers_to_fullname +from mypy.types import FINAL_DECORATOR_NAMES + +DATACLASS_DECORATORS = {"dataclasses.dataclass", "attr.s", "attr.attrs"} + + +def is_final_decorator(d: Expression) -> bool: + return refers_to_fullname(d, FINAL_DECORATOR_NAMES) + + +def is_trait_decorator(d: Expression) -> bool: + return isinstance(d, RefExpr) and d.fullname == "mypy_extensions.trait" + + +def is_trait(cdef: ClassDef) -> bool: + return any(is_trait_decorator(d) for d in cdef.decorators) or cdef.info.is_protocol + + +def dataclass_decorator_type(d: Expression) -> str | None: + if isinstance(d, RefExpr) and d.fullname in DATACLASS_DECORATORS: + return d.fullname.split(".")[0] + elif ( + isinstance(d, CallExpr) + and isinstance(d.callee, RefExpr) + and d.callee.fullname in DATACLASS_DECORATORS + ): + name = d.callee.fullname.split(".")[0] + if name == "attr" and "auto_attribs" in d.arg_names: + # Note: the mypy attrs plugin checks that the value of auto_attribs is + # not computed at runtime, so we don't need to perform that check here + auto = d.args[d.arg_names.index("auto_attribs")] + if isinstance(auto, NameExpr) and auto.name == "True": + return "attr-auto" + return name + else: + return None + + +def is_dataclass_decorator(d: Expression) -> bool: + return dataclass_decorator_type(d) is not None + + +def is_dataclass(cdef: ClassDef) -> bool: + return any(is_dataclass_decorator(d) for d in cdef.decorators) + + +def dataclass_type(cdef: ClassDef) -> str | None: + for d in cdef.decorators: + typ = dataclass_decorator_type(d) + if typ is not None: + return typ + return None + + +def get_mypyc_attr_literal(e: Expression) -> Any: + """Convert an expression from a mypyc_attr decorator to a value. + + Supports a pretty limited range.""" + if isinstance(e, (StrExpr, IntExpr, FloatExpr)): + return e.value + elif isinstance(e, RefExpr) and e.fullname == "builtins.True": + return True + elif isinstance(e, RefExpr) and e.fullname == "builtins.False": + return False + elif isinstance(e, RefExpr) and e.fullname == "builtins.None": + return None + return NotImplemented + + +def get_mypyc_attr_call(d: Expression) -> CallExpr | None: + """Check if an expression is a call to mypyc_attr and return it if so.""" + if ( + isinstance(d, CallExpr) + and isinstance(d.callee, RefExpr) + and d.callee.fullname == "mypy_extensions.mypyc_attr" + ): + return d + return None + + +def get_mypyc_attrs(stmt: ClassDef | Decorator) -> dict[str, Any]: + """Collect all the mypyc_attr attributes on a class definition or a function.""" + attrs: dict[str, Any] = {} + for dec in stmt.decorators: + d = get_mypyc_attr_call(dec) + if d: + for name, arg in zip(d.arg_names, d.args): + if name is None: + if isinstance(arg, StrExpr): + attrs[arg.value] = True + else: + attrs[name] = get_mypyc_attr_literal(arg) + + return attrs + + +def is_extension_class(cdef: ClassDef) -> bool: + if any( + not is_trait_decorator(d) + and not is_dataclass_decorator(d) + and not get_mypyc_attr_call(d) + and not is_final_decorator(d) + for d in cdef.decorators + ): + return False + if cdef.info.typeddict_type: + return False + if cdef.info.is_named_tuple: + return False + if cdef.info.metaclass_type and cdef.info.metaclass_type.type.fullname not in ( + "abc.ABCMeta", + "typing.TypingMeta", + "typing.GenericMeta", + ): + return False + return True + + +def get_func_def(op: FuncDef | Decorator | OverloadedFuncDef) -> FuncDef: + if isinstance(op, OverloadedFuncDef): + assert op.impl + op = op.impl + if isinstance(op, Decorator): + op = op.func + return op + + +def concrete_arg_kind(kind: ArgKind) -> ArgKind: + """Find the concrete version of an arg kind that is being passed.""" + if kind == ARG_OPT: + return ARG_POS + elif kind == ARG_NAMED_OPT: + return ARG_NAMED + else: + return kind + + +def is_constant(e: Expression) -> bool: + """Check whether we allow an expression to appear as a default value. + + We don't currently properly support storing the evaluated + values for default arguments and default attribute values, so + we restrict what expressions we allow. We allow literals of + primitives types, None, and references to Final global + variables. + """ + return ( + isinstance(e, (StrExpr, BytesExpr, IntExpr, FloatExpr)) + or (isinstance(e, UnaryExpr) and e.op == "-" and isinstance(e.expr, (IntExpr, FloatExpr))) + or (isinstance(e, TupleExpr) and all(is_constant(e) for e in e.items)) + or ( + isinstance(e, RefExpr) + and e.kind == GDEF + and ( + e.fullname in ("builtins.True", "builtins.False", "builtins.None") + or (isinstance(e.node, Var) and e.node.is_final) + ) + ) + ) + + +def bytes_from_str(value: str) -> bytes: + """Convert a string representing bytes into actual bytes. + + This is needed because the literal characters of BytesExpr (the + characters inside b'') are stored in BytesExpr.value, whose type is + 'str' not 'bytes'. + """ + return bytes(value, "utf8").decode("unicode-escape").encode("raw-unicode-escape") diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py new file mode 100644 index 000000000000..05a033c3e6ad --- /dev/null +++ b/mypyc/irbuild/visitor.py @@ -0,0 +1,406 @@ +"""Dispatcher used when transforming a mypy AST to the IR form. + +mypyc.irbuild.builder and mypyc.irbuild.main are closely related. +""" + +from __future__ import annotations + +from typing import NoReturn + +from mypy.nodes import ( + AssertStmt, + AssertTypeExpr, + AssignmentExpr, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + BytesExpr, + CallExpr, + CastExpr, + ClassDef, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + ContinueStmt, + Decorator, + DelStmt, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, + ExpressionStmt, + FloatExpr, + ForStmt, + FuncDef, + GeneratorExpr, + GlobalDecl, + IfStmt, + Import, + ImportAll, + ImportFrom, + IndexExpr, + IntExpr, + LambdaExpr, + ListComprehension, + ListExpr, + MatchStmt, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + NewTypeExpr, + NonlocalDecl, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + ParamSpecExpr, + PassStmt, + PromoteExpr, + RaiseStmt, + ReturnStmt, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + StarExpr, + StrExpr, + SuperExpr, + TempNode, + TryStmt, + TupleExpr, + TypeAliasExpr, + TypeAliasStmt, + TypeApplication, + TypedDictExpr, + TypeVarExpr, + TypeVarTupleExpr, + UnaryExpr, + Var, + WhileStmt, + WithStmt, + YieldExpr, + YieldFromExpr, +) +from mypyc.ir.ops import Value +from mypyc.irbuild.builder import IRBuilder, IRVisitor, UnsupportedException +from mypyc.irbuild.classdef import transform_class_def +from mypyc.irbuild.expression import ( + transform_assignment_expr, + transform_bytes_expr, + transform_call_expr, + transform_comparison_expr, + transform_complex_expr, + transform_conditional_expr, + transform_dict_expr, + transform_dictionary_comprehension, + transform_ellipsis, + transform_float_expr, + transform_generator_expr, + transform_index_expr, + transform_int_expr, + transform_list_comprehension, + transform_list_expr, + transform_member_expr, + transform_name_expr, + transform_op_expr, + transform_set_comprehension, + transform_set_expr, + transform_slice_expr, + transform_str_expr, + transform_super_expr, + transform_tuple_expr, + transform_unary_expr, +) +from mypyc.irbuild.function import ( + transform_decorator, + transform_func_def, + transform_lambda_expr, + transform_overloaded_func_def, +) +from mypyc.irbuild.statement import ( + transform_assert_stmt, + transform_assignment_stmt, + transform_await_expr, + transform_block, + transform_break_stmt, + transform_continue_stmt, + transform_del_stmt, + transform_expression_stmt, + transform_for_stmt, + transform_if_stmt, + transform_import, + transform_import_all, + transform_import_from, + transform_match_stmt, + transform_operator_assignment_stmt, + transform_raise_stmt, + transform_return_stmt, + transform_try_stmt, + transform_type_alias_stmt, + transform_while_stmt, + transform_with_stmt, + transform_yield_expr, + transform_yield_from_expr, +) + + +class IRBuilderVisitor(IRVisitor): + """Mypy node visitor that dispatches to node transform implementations. + + This class should have no non-trivial logic. + + This visitor is separated from the rest of code to improve modularity and + to avoid import cycles. + + This is based on the visitor pattern + (https://en.wikipedia.org/wiki/Visitor_pattern). + """ + + # This gets passed to all the implementations and contains all the + # state and many helpers. The attribute is initialized outside + # this class since this class and IRBuilder form a reference loop. + builder: IRBuilder + + def visit_mypy_file(self, mypyfile: MypyFile) -> None: + assert False, "use transform_mypy_file instead" + + def visit_class_def(self, cdef: ClassDef) -> None: + transform_class_def(self.builder, cdef) + + def visit_import(self, node: Import) -> None: + transform_import(self.builder, node) + + def visit_import_from(self, node: ImportFrom) -> None: + transform_import_from(self.builder, node) + + def visit_import_all(self, node: ImportAll) -> None: + transform_import_all(self.builder, node) + + def visit_func_def(self, fdef: FuncDef) -> None: + transform_func_def(self.builder, fdef) + + def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: + transform_overloaded_func_def(self.builder, o) + + def visit_decorator(self, dec: Decorator) -> None: + transform_decorator(self.builder, dec) + + def visit_block(self, block: Block) -> None: + transform_block(self.builder, block) + + # Statements + + def visit_expression_stmt(self, stmt: ExpressionStmt) -> None: + transform_expression_stmt(self.builder, stmt) + + def visit_return_stmt(self, stmt: ReturnStmt) -> None: + transform_return_stmt(self.builder, stmt) + self.builder.mark_block_unreachable() + + def visit_assignment_stmt(self, stmt: AssignmentStmt) -> None: + transform_assignment_stmt(self.builder, stmt) + + def visit_operator_assignment_stmt(self, stmt: OperatorAssignmentStmt) -> None: + transform_operator_assignment_stmt(self.builder, stmt) + + def visit_if_stmt(self, stmt: IfStmt) -> None: + transform_if_stmt(self.builder, stmt) + + def visit_while_stmt(self, stmt: WhileStmt) -> None: + transform_while_stmt(self.builder, stmt) + + def visit_for_stmt(self, stmt: ForStmt) -> None: + transform_for_stmt(self.builder, stmt) + + def visit_break_stmt(self, stmt: BreakStmt) -> None: + transform_break_stmt(self.builder, stmt) + self.builder.mark_block_unreachable() + + def visit_continue_stmt(self, stmt: ContinueStmt) -> None: + transform_continue_stmt(self.builder, stmt) + self.builder.mark_block_unreachable() + + def visit_raise_stmt(self, stmt: RaiseStmt) -> None: + transform_raise_stmt(self.builder, stmt) + self.builder.mark_block_unreachable() + + def visit_try_stmt(self, stmt: TryStmt) -> None: + transform_try_stmt(self.builder, stmt) + + def visit_with_stmt(self, stmt: WithStmt) -> None: + transform_with_stmt(self.builder, stmt) + + def visit_pass_stmt(self, stmt: PassStmt) -> None: + pass + + def visit_assert_stmt(self, stmt: AssertStmt) -> None: + transform_assert_stmt(self.builder, stmt) + + def visit_del_stmt(self, stmt: DelStmt) -> None: + transform_del_stmt(self.builder, stmt) + + def visit_global_decl(self, stmt: GlobalDecl) -> None: + # Pure declaration -- no runtime effect + pass + + def visit_nonlocal_decl(self, stmt: NonlocalDecl) -> None: + # Pure declaration -- no runtime effect + pass + + def visit_match_stmt(self, stmt: MatchStmt) -> None: + transform_match_stmt(self.builder, stmt) + + def visit_type_alias_stmt(self, stmt: TypeAliasStmt) -> None: + transform_type_alias_stmt(self.builder, stmt) + + # Expressions + + def visit_name_expr(self, expr: NameExpr) -> Value: + return transform_name_expr(self.builder, expr) + + def visit_member_expr(self, expr: MemberExpr) -> Value: + return transform_member_expr(self.builder, expr) + + def visit_super_expr(self, expr: SuperExpr) -> Value: + return transform_super_expr(self.builder, expr) + + def visit_call_expr(self, expr: CallExpr) -> Value: + return transform_call_expr(self.builder, expr) + + def visit_unary_expr(self, expr: UnaryExpr) -> Value: + return transform_unary_expr(self.builder, expr) + + def visit_op_expr(self, expr: OpExpr) -> Value: + return transform_op_expr(self.builder, expr) + + def visit_index_expr(self, expr: IndexExpr) -> Value: + return transform_index_expr(self.builder, expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> Value: + return transform_conditional_expr(self.builder, expr) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> Value: + return transform_comparison_expr(self.builder, expr) + + def visit_int_expr(self, expr: IntExpr) -> Value: + return transform_int_expr(self.builder, expr) + + def visit_float_expr(self, expr: FloatExpr) -> Value: + return transform_float_expr(self.builder, expr) + + def visit_complex_expr(self, expr: ComplexExpr) -> Value: + return transform_complex_expr(self.builder, expr) + + def visit_str_expr(self, expr: StrExpr) -> Value: + return transform_str_expr(self.builder, expr) + + def visit_bytes_expr(self, expr: BytesExpr) -> Value: + return transform_bytes_expr(self.builder, expr) + + def visit_ellipsis(self, expr: EllipsisExpr) -> Value: + return transform_ellipsis(self.builder, expr) + + def visit_list_expr(self, expr: ListExpr) -> Value: + return transform_list_expr(self.builder, expr) + + def visit_tuple_expr(self, expr: TupleExpr) -> Value: + return transform_tuple_expr(self.builder, expr) + + def visit_dict_expr(self, expr: DictExpr) -> Value: + return transform_dict_expr(self.builder, expr) + + def visit_set_expr(self, expr: SetExpr) -> Value: + return transform_set_expr(self.builder, expr) + + def visit_list_comprehension(self, expr: ListComprehension) -> Value: + return transform_list_comprehension(self.builder, expr) + + def visit_set_comprehension(self, expr: SetComprehension) -> Value: + return transform_set_comprehension(self.builder, expr) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> Value: + return transform_dictionary_comprehension(self.builder, expr) + + def visit_slice_expr(self, expr: SliceExpr) -> Value: + return transform_slice_expr(self.builder, expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> Value: + return transform_generator_expr(self.builder, expr) + + def visit_lambda_expr(self, expr: LambdaExpr) -> Value: + return transform_lambda_expr(self.builder, expr) + + def visit_yield_expr(self, expr: YieldExpr) -> Value: + return transform_yield_expr(self.builder, expr) + + def visit_yield_from_expr(self, o: YieldFromExpr) -> Value: + return transform_yield_from_expr(self.builder, o) + + def visit_await_expr(self, o: AwaitExpr) -> Value: + return transform_await_expr(self.builder, o) + + def visit_assignment_expr(self, o: AssignmentExpr) -> Value: + return transform_assignment_expr(self.builder, o) + + # Constructs that shouldn't ever show up + + def visit_enum_call_expr(self, o: EnumCallExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit__promote_expr(self, o: PromoteExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_namedtuple_expr(self, o: NamedTupleExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_newtype_expr(self, o: NewTypeExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_temp_node(self, o: TempNode) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_type_alias_expr(self, o: TypeAliasExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_type_application(self, o: TypeApplication) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_type_var_expr(self, o: TypeVarExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_paramspec_expr(self, o: ParamSpecExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_type_var_tuple_expr(self, o: TypeVarTupleExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_typeddict_expr(self, o: TypedDictExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_reveal_expr(self, o: RevealExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_var(self, o: Var) -> None: + assert False, "can't compile Var; should have been handled already?" + + def visit_cast_expr(self, o: CastExpr) -> Value: + assert False, "CastExpr should have been handled in CallExpr" + + def visit_assert_type_expr(self, o: AssertTypeExpr) -> Value: + assert False, "AssertTypeExpr should have been handled in CallExpr" + + def visit_star_expr(self, o: StarExpr) -> Value: + assert False, "should have been handled in Tuple/List/Set/DictExpr or CallExpr" + + # Helpers + + def bail(self, msg: str, line: int) -> NoReturn: + """Reports an error and aborts compilation up until the last accept() call + + (accept() catches the UnsupportedException and keeps on + processing. This allows errors to be non-blocking without always + needing to write handling for them. + """ + self.builder.error(msg, line) + raise UnsupportedException() diff --git a/mypyc/irbuild/vtable.py b/mypyc/irbuild/vtable.py new file mode 100644 index 000000000000..2d4f7261e4ca --- /dev/null +++ b/mypyc/irbuild/vtable.py @@ -0,0 +1,82 @@ +"""Compute vtables of native (extension) classes.""" + +from __future__ import annotations + +import itertools + +from mypyc.ir.class_ir import ClassIR, VTableEntries, VTableMethod +from mypyc.sametype import is_same_method_signature + + +def compute_vtable(cls: ClassIR) -> None: + """Compute the vtable structure for a class.""" + if cls.vtable is not None: + return + + if not cls.is_generated: + cls.has_dict = any(x.inherits_python for x in cls.mro) + + for t in cls.mro[1:]: + # Make sure all ancestors are processed first + compute_vtable(t) + # Merge attributes from traits into the class + if not t.is_trait: + continue + for name, typ in t.attributes.items(): + if not cls.is_trait and not any(name in b.attributes for b in cls.base_mro): + cls.attributes[name] = typ + + cls.vtable = {} + if cls.base: + assert cls.base.vtable is not None + cls.vtable.update(cls.base.vtable) + cls.vtable_entries = specialize_parent_vtable(cls, cls.base) + + # Include the vtable from the parent classes, but handle method overrides. + entries = cls.vtable_entries + + all_traits = [t for t in cls.mro if t.is_trait] + + for t in [cls] + cls.traits: + for fn in itertools.chain(t.methods.values()): + # TODO: don't generate a new entry when we overload without changing the type + if fn == cls.get_method(fn.name, prefer_method=True): + cls.vtable[fn.name] = len(entries) + # If the class contains a glue method referring to itself, that is a + # shadow glue method to support interpreted subclasses. + shadow = cls.glue_methods.get((cls, fn.name)) + entries.append(VTableMethod(t, fn.name, fn, shadow)) + + # Compute vtables for all of the traits that the class implements + if not cls.is_trait: + for trait in all_traits: + compute_vtable(trait) + cls.trait_vtables[trait] = specialize_parent_vtable(cls, trait) + + +def specialize_parent_vtable(cls: ClassIR, parent: ClassIR) -> VTableEntries: + """Generate the part of a vtable corresponding to a parent class or trait""" + updated = [] + for entry in parent.vtable_entries: + # Find the original method corresponding to this vtable entry. + # (This may not be the method in the entry, if it was overridden.) + orig_parent_method = entry.cls.get_method(entry.name, prefer_method=True) + assert orig_parent_method + method_cls = cls.get_method_and_class(entry.name, prefer_method=True) + if method_cls: + child_method, defining_cls = method_cls + # TODO: emit a wrapper for __init__ that raises or something + if ( + is_same_method_signature(orig_parent_method.sig, child_method.sig) + or orig_parent_method.name == "__init__" + ): + entry = VTableMethod(entry.cls, entry.name, child_method, entry.shadow_method) + else: + entry = VTableMethod( + entry.cls, + entry.name, + defining_cls.glue_methods[(entry.cls, entry.name)], + entry.shadow_method, + ) + updated.append(entry) + return updated diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h new file mode 100644 index 000000000000..d3637cde49ff --- /dev/null +++ b/mypyc/lib-rt/CPy.h @@ -0,0 +1,909 @@ +// Mypyc C API + +#ifndef CPY_CPY_H +#define CPY_CPY_H + +#include +#include +#include +#include +#include +#include +#include "pythonsupport.h" +#include "mypyc_util.h" + +#ifdef __cplusplus +extern "C" { +#endif +#if 0 +} // why isn't emacs smart enough to not indent this +#endif + +#define CPYTHON_LARGE_INT_ERRMSG "Python int too large to convert to C ssize_t" + + +// Naming conventions: +// +// Tagged: tagged int +// Long: tagged long int (pointer) +// Short: tagged short int (unboxed) +// Ssize_t: A Py_ssize_t, which ought to be the same width as pointers +// Object: CPython object (PyObject *) + + +// Tuple type definitions needed for API functions + + +#ifndef MYPYC_DECLARED_tuple_T3OOO +#define MYPYC_DECLARED_tuple_T3OOO +typedef struct tuple_T3OOO { + PyObject *f0; + PyObject *f1; + PyObject *f2; +} tuple_T3OOO; +#endif + +// Our return tuple wrapper for dictionary iteration helper. +#ifndef MYPYC_DECLARED_tuple_T3CIO +#define MYPYC_DECLARED_tuple_T3CIO +typedef struct tuple_T3CIO { + char f0; // Should continue? + CPyTagged f1; // Last dict offset + PyObject *f2; // Next dictionary key or value +} tuple_T3CIO; +#endif + +// Same as above but for both key and value. +#ifndef MYPYC_DECLARED_tuple_T4CIOO +#define MYPYC_DECLARED_tuple_T4CIOO +typedef struct tuple_T4CIOO { + char f0; // Should continue? + CPyTagged f1; // Last dict offset + PyObject *f2; // Next dictionary key + PyObject *f3; // Next dictionary value +} tuple_T4CIOO; +#endif + + +// Native object operations + + +// Search backwards through the trait part of a vtable (which sits *before* +// the start of the vtable proper) looking for the subvtable describing a trait +// implementation. We don't do any bounds checking so we'd better be pretty sure +// we know that it is there. +static inline CPyVTableItem *CPy_FindTraitVtable(PyTypeObject *trait, CPyVTableItem *vtable) { + int i; + for (i = -3; ; i -= 3) { + if ((PyTypeObject *)vtable[i] == trait) { + return (CPyVTableItem *)vtable[i + 1]; + } + } +} + +// Use the same logic for offset table. +static inline size_t CPy_FindAttrOffset(PyTypeObject *trait, CPyVTableItem *vtable, size_t index) { + int i; + for (i = -3; ; i -= 3) { + if ((PyTypeObject *)vtable[i] == trait) { + return ((size_t *)vtable[i + 2])[index]; + } + } +} + +// Get attribute value using vtable (may return an undefined value) +#define CPY_GET_ATTR(obj, type, vtable_index, object_type, attr_type) \ + ((attr_type (*)(object_type *))((object_type *)obj)->vtable[vtable_index])((object_type *)obj) + +#define CPY_GET_ATTR_TRAIT(obj, trait, vtable_index, object_type, attr_type) \ + ((attr_type (*)(object_type *))(CPy_FindTraitVtable(trait, ((object_type *)obj)->vtable))[vtable_index])((object_type *)obj) + +// Set attribute value using vtable +#define CPY_SET_ATTR(obj, type, vtable_index, value, object_type, attr_type) \ + ((bool (*)(object_type *, attr_type))((object_type *)obj)->vtable[vtable_index])( \ + (object_type *)obj, value) + +#define CPY_SET_ATTR_TRAIT(obj, trait, vtable_index, value, object_type, attr_type) \ + ((bool (*)(object_type *, attr_type))(CPy_FindTraitVtable(trait, ((object_type *)obj)->vtable))[vtable_index])( \ + (object_type *)obj, value) + +#define CPY_GET_METHOD(obj, type, vtable_index, object_type, method_type) \ + ((method_type)(((object_type *)obj)->vtable[vtable_index])) + +#define CPY_GET_METHOD_TRAIT(obj, trait, vtable_index, object_type, method_type) \ + ((method_type)(CPy_FindTraitVtable(trait, ((object_type *)obj)->vtable)[vtable_index])) + + +// Int operations + + +CPyTagged CPyTagged_FromSsize_t(Py_ssize_t value); +CPyTagged CPyTagged_FromVoidPtr(void *ptr); +CPyTagged CPyTagged_FromInt64(int64_t value); +PyObject *CPyTagged_AsObject(CPyTagged x); +PyObject *CPyTagged_StealAsObject(CPyTagged x); +Py_ssize_t CPyTagged_AsSsize_t(CPyTagged x); +void CPyTagged_IncRef(CPyTagged x); +void CPyTagged_DecRef(CPyTagged x); +void CPyTagged_XDecRef(CPyTagged x); + +bool CPyTagged_IsEq_(CPyTagged left, CPyTagged right); +bool CPyTagged_IsLt_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Negate_(CPyTagged num); +CPyTagged CPyTagged_Invert_(CPyTagged num); +CPyTagged CPyTagged_Add_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Subtract_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Multiply_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_FloorDivide_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Remainder_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_BitwiseLongOp_(CPyTagged a, CPyTagged b, char op); +CPyTagged CPyTagged_Rshift_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Lshift_(CPyTagged left, CPyTagged right); + +PyObject *CPyTagged_Str(CPyTagged n); +CPyTagged CPyTagged_FromFloat(double f); +PyObject *CPyLong_FromStrWithBase(PyObject *o, CPyTagged base); +PyObject *CPyLong_FromStr(PyObject *o); +PyObject *CPyBool_Str(bool b); +int64_t CPyLong_AsInt64_(PyObject *o); +int64_t CPyInt64_Divide(int64_t x, int64_t y); +int64_t CPyInt64_Remainder(int64_t x, int64_t y); +int32_t CPyLong_AsInt32_(PyObject *o); +int32_t CPyInt32_Divide(int32_t x, int32_t y); +int32_t CPyInt32_Remainder(int32_t x, int32_t y); +void CPyInt32_Overflow(void); +int16_t CPyLong_AsInt16_(PyObject *o); +int16_t CPyInt16_Divide(int16_t x, int16_t y); +int16_t CPyInt16_Remainder(int16_t x, int16_t y); +void CPyInt16_Overflow(void); +uint8_t CPyLong_AsUInt8_(PyObject *o); +void CPyUInt8_Overflow(void); +double CPyTagged_TrueDivide(CPyTagged x, CPyTagged y); + +static inline int CPyTagged_CheckLong(CPyTagged x) { + return x & CPY_INT_TAG; +} + +static inline int CPyTagged_CheckShort(CPyTagged x) { + return !CPyTagged_CheckLong(x); +} + +static inline void CPyTagged_INCREF(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + CPyTagged_IncRef(x); + } +} + +static inline void CPyTagged_DECREF(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + CPyTagged_DecRef(x); + } +} + +static inline void CPyTagged_XDECREF(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + CPyTagged_XDecRef(x); + } +} + +static inline Py_ssize_t CPyTagged_ShortAsSsize_t(CPyTagged x) { + // NOTE: Assume that we sign extend. + return (Py_ssize_t)x >> 1; +} + +static inline PyObject *CPyTagged_LongAsObject(CPyTagged x) { + // NOTE: Assume target is not a short int. + return (PyObject *)(x & ~CPY_INT_TAG); +} + +static inline CPyTagged CPyTagged_FromObject(PyObject *object) { + int overflow; + // The overflow check knows about CPyTagged's width + Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); + if (unlikely(overflow != 0)) { + Py_INCREF(object); + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return value << 1; + } +} + +static inline CPyTagged CPyTagged_StealFromObject(PyObject *object) { + int overflow; + // The overflow check knows about CPyTagged's width + Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); + if (unlikely(overflow != 0)) { + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + Py_DECREF(object); + return value << 1; + } +} + +static inline CPyTagged CPyTagged_BorrowFromObject(PyObject *object) { + int overflow; + // The overflow check knows about CPyTagged's width + Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); + if (unlikely(overflow != 0)) { + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return value << 1; + } +} + +static inline bool CPyTagged_TooBig(Py_ssize_t value) { + // Micro-optimized for the common case where it fits. + return (size_t)value > CPY_TAGGED_MAX + && (value >= 0 || value < CPY_TAGGED_MIN); +} + +static inline bool CPyTagged_TooBigInt64(int64_t value) { + // Micro-optimized for the common case where it fits. + return (uint64_t)value > CPY_TAGGED_MAX + && (value >= 0 || value < CPY_TAGGED_MIN); +} + +static inline bool CPyTagged_IsAddOverflow(CPyTagged sum, CPyTagged left, CPyTagged right) { + // This check was copied from some of my old code I believe that it works :-) + return (Py_ssize_t)(sum ^ left) < 0 && (Py_ssize_t)(sum ^ right) < 0; +} + +static inline bool CPyTagged_IsSubtractOverflow(CPyTagged diff, CPyTagged left, CPyTagged right) { + // This check was copied from some of my old code I believe that it works :-) + return (Py_ssize_t)(diff ^ left) < 0 && (Py_ssize_t)(diff ^ right) >= 0; +} + +static inline bool CPyTagged_IsMultiplyOverflow(CPyTagged left, CPyTagged right) { + // This is conservative -- return false only in a small number of all non-overflow cases + return left >= (1U << (CPY_INT_BITS/2 - 1)) || right >= (1U << (CPY_INT_BITS/2 - 1)); +} + +static inline bool CPyTagged_MaybeFloorDivideFault(CPyTagged left, CPyTagged right) { + return right == 0 || left == -((size_t)1 << (CPY_INT_BITS-1)); +} + +static inline bool CPyTagged_MaybeRemainderFault(CPyTagged left, CPyTagged right) { + // Division/modulus can fault when dividing INT_MIN by -1, but we + // do our mods on still-tagged integers with the low-bit clear, so + // -1 is actually represented as -2 and can't overflow. + // Mod by 0 can still fault though. + return right == 0; +} + +static inline bool CPyTagged_IsEq(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left)) { + return left == right; + } else { + return CPyTagged_IsEq_(left, right); + } +} + +static inline bool CPyTagged_IsNe(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left)) { + return left != right; + } else { + return !CPyTagged_IsEq_(left, right); + } +} + +static inline bool CPyTagged_IsLt(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right)) { + return (Py_ssize_t)left < (Py_ssize_t)right; + } else { + return CPyTagged_IsLt_(left, right); + } +} + +static inline bool CPyTagged_IsGe(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right)) { + return (Py_ssize_t)left >= (Py_ssize_t)right; + } else { + return !CPyTagged_IsLt_(left, right); + } +} + +static inline bool CPyTagged_IsGt(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right)) { + return (Py_ssize_t)left > (Py_ssize_t)right; + } else { + return CPyTagged_IsLt_(right, left); + } +} + +static inline bool CPyTagged_IsLe(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right)) { + return (Py_ssize_t)left <= (Py_ssize_t)right; + } else { + return !CPyTagged_IsLt_(right, left); + } +} + +static inline int64_t CPyLong_AsInt64(PyObject *o) { + if (likely(PyLong_Check(o))) { + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = Py_SIZE(lobj); + if (likely(size == 1)) { + // Fast path + return CPY_LONG_DIGIT(lobj, 0); + } else if (likely(size == 0)) { + return 0; + } + } + // Slow path + return CPyLong_AsInt64_(o); +} + +static inline int32_t CPyLong_AsInt32(PyObject *o) { + if (likely(PyLong_Check(o))) { + #if CPY_3_12_FEATURES + PyLongObject *lobj = (PyLongObject *)o; + size_t tag = CPY_LONG_TAG(lobj); + if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { + // Fast path + return CPY_LONG_DIGIT(lobj, 0); + } else if (likely(tag == CPY_SIGN_ZERO)) { + return 0; + } + #else + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = lobj->ob_base.ob_size; + if (likely(size == 1)) { + // Fast path + return CPY_LONG_DIGIT(lobj, 0); + } else if (likely(size == 0)) { + return 0; + } + #endif + } + // Slow path + return CPyLong_AsInt32_(o); +} + +static inline int16_t CPyLong_AsInt16(PyObject *o) { + if (likely(PyLong_Check(o))) { + #if CPY_3_12_FEATURES + PyLongObject *lobj = (PyLongObject *)o; + size_t tag = CPY_LONG_TAG(lobj); + if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { + // Fast path + digit x = CPY_LONG_DIGIT(lobj, 0); + if (x < 0x8000) + return x; + } else if (likely(tag == CPY_SIGN_ZERO)) { + return 0; + } + #else + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = lobj->ob_base.ob_size; + if (likely(size == 1)) { + // Fast path + digit x = lobj->ob_digit[0]; + if (x < 0x8000) + return x; + } else if (likely(size == 0)) { + return 0; + } + #endif + } + // Slow path + return CPyLong_AsInt16_(o); +} + +static inline uint8_t CPyLong_AsUInt8(PyObject *o) { + if (likely(PyLong_Check(o))) { + #if CPY_3_12_FEATURES + PyLongObject *lobj = (PyLongObject *)o; + size_t tag = CPY_LONG_TAG(lobj); + if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { + // Fast path + digit x = CPY_LONG_DIGIT(lobj, 0); + if (x < 256) + return x; + } else if (likely(tag == CPY_SIGN_ZERO)) { + return 0; + } + #else + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = lobj->ob_base.ob_size; + if (likely(size == 1)) { + // Fast path + digit x = lobj->ob_digit[0]; + if (x < 256) + return x; + } else if (likely(size == 0)) { + return 0; + } + #endif + } + // Slow path + return CPyLong_AsUInt8_(o); +} + +static inline CPyTagged CPyTagged_Negate(CPyTagged num) { + if (likely(CPyTagged_CheckShort(num) + && num != (CPyTagged) ((Py_ssize_t)1 << (CPY_INT_BITS - 1)))) { + // The only possibility of an overflow error happening when negating a short is if we + // attempt to negate the most negative number. + return -num; + } + return CPyTagged_Negate_(num); +} + +static inline CPyTagged CPyTagged_Add(CPyTagged left, CPyTagged right) { + // TODO: Use clang/gcc extension __builtin_saddll_overflow instead. + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + CPyTagged sum = left + right; + if (likely(!CPyTagged_IsAddOverflow(sum, left, right))) { + return sum; + } + } + return CPyTagged_Add_(left, right); +} + +static inline CPyTagged CPyTagged_Subtract(CPyTagged left, CPyTagged right) { + // TODO: Use clang/gcc extension __builtin_saddll_overflow instead. + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + CPyTagged diff = left - right; + if (likely(!CPyTagged_IsSubtractOverflow(diff, left, right))) { + return diff; + } + } + return CPyTagged_Subtract_(left, right); +} + +static inline CPyTagged CPyTagged_Multiply(CPyTagged left, CPyTagged right) { + // TODO: Consider using some clang/gcc extension to check for overflow + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right)) { + if (!CPyTagged_IsMultiplyOverflow(left, right)) { + return left * CPyTagged_ShortAsSsize_t(right); + } + } + return CPyTagged_Multiply_(left, right); +} + +static inline CPyTagged CPyTagged_FloorDivide(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) + && CPyTagged_CheckShort(right) + && !CPyTagged_MaybeFloorDivideFault(left, right)) { + Py_ssize_t result = CPyTagged_ShortAsSsize_t(left) / CPyTagged_ShortAsSsize_t(right); + if (((Py_ssize_t)left < 0) != (((Py_ssize_t)right) < 0)) { + if (result * right != left) { + // Round down + result--; + } + } + return result << 1; + } + return CPyTagged_FloorDivide_(left, right); +} + +static inline CPyTagged CPyTagged_Remainder(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right) + && !CPyTagged_MaybeRemainderFault(left, right)) { + Py_ssize_t result = (Py_ssize_t)left % (Py_ssize_t)right; + if (((Py_ssize_t)right < 0) != ((Py_ssize_t)left < 0) && result != 0) { + result += right; + } + return result; + } + return CPyTagged_Remainder_(left, right); +} + +// Bitwise '~' +static inline CPyTagged CPyTagged_Invert(CPyTagged num) { + if (likely(CPyTagged_CheckShort(num) && num != CPY_TAGGED_ABS_MIN)) { + return ~num & ~CPY_INT_TAG; + } + return CPyTagged_Invert_(num); +} + +// Bitwise '&' +static inline CPyTagged CPyTagged_And(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + return left & right; + } + return CPyTagged_BitwiseLongOp_(left, right, '&'); +} + +// Bitwise '|' +static inline CPyTagged CPyTagged_Or(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + return left | right; + } + return CPyTagged_BitwiseLongOp_(left, right, '|'); +} + +// Bitwise '^' +static inline CPyTagged CPyTagged_Xor(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + return left ^ right; + } + return CPyTagged_BitwiseLongOp_(left, right, '^'); +} + +// Bitwise '>>' +static inline CPyTagged CPyTagged_Rshift(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) + && CPyTagged_CheckShort(right) + && (Py_ssize_t)right >= 0)) { + CPyTagged count = CPyTagged_ShortAsSsize_t(right); + if (unlikely(count >= CPY_INT_BITS)) { + if ((Py_ssize_t)left >= 0) { + return 0; + } else { + return CPyTagged_ShortFromInt(-1); + } + } + return ((Py_ssize_t)left >> count) & ~CPY_INT_TAG; + } + return CPyTagged_Rshift_(left, right); +} + +static inline bool IsShortLshiftOverflow(Py_ssize_t short_int, Py_ssize_t shift) { + return ((Py_ssize_t)(short_int << shift) >> shift) != short_int; +} + +// Bitwise '<<' +static inline CPyTagged CPyTagged_Lshift(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) + && CPyTagged_CheckShort(right) + && (Py_ssize_t)right >= 0 + && right < CPY_INT_BITS * 2)) { + CPyTagged shift = CPyTagged_ShortAsSsize_t(right); + if (!IsShortLshiftOverflow(left, shift)) + // Short integers, no overflow + return left << shift; + } + return CPyTagged_Lshift_(left, right); +} + + +// Float operations + + +double CPyFloat_FloorDivide(double x, double y); +double CPyFloat_Pow(double x, double y); +double CPyFloat_Sin(double x); +double CPyFloat_Cos(double x); +double CPyFloat_Tan(double x); +double CPyFloat_Sqrt(double x); +double CPyFloat_Exp(double x); +double CPyFloat_Log(double x); +CPyTagged CPyFloat_Floor(double x); +CPyTagged CPyFloat_Ceil(double x); +double CPyFloat_FromTagged(CPyTagged x); +bool CPyFloat_IsInf(double x); +bool CPyFloat_IsNaN(double x); + + +// Generic operations (that work with arbitrary types) + + +/* We use intentionally non-inlined decrefs in rarely executed code + * paths since it pretty substantially speeds up compile time. We have + * our own copies both to avoid the null check in Py_DecRef and to avoid + * making an indirect PIC call. */ +CPy_NOINLINE +static void CPy_DecRef(PyObject *p) { + CPy_DECREF(p); +} + +CPy_NOINLINE +static void CPy_XDecRef(PyObject *p) { + CPy_XDECREF(p); +} + +static inline CPyTagged CPyObject_Size(PyObject *obj) { + Py_ssize_t s = PyObject_Size(obj); + if (s < 0) { + return CPY_INT_TAG; + } else { + // Technically __len__ could return a really big number, so we + // should allow this to produce a boxed int. In practice it + // shouldn't ever if the data structure actually contains all + // the elements, but... + return CPyTagged_FromSsize_t(s); + } +} + +#ifdef MYPYC_LOG_GETATTR +static void CPy_LogGetAttr(const char *method, PyObject *obj, PyObject *attr) { + PyObject *module = PyImport_ImportModule("getattr_hook"); + if (module) { + PyObject *res = PyObject_CallMethodObjArgs(module, method, obj, attr, NULL); + Py_XDECREF(res); + Py_DECREF(module); + } + PyErr_Clear(); +} +#else +#define CPy_LogGetAttr(method, obj, attr) (void)0 +#endif + +// Intercept a method call and log it. This needs to be a macro +// because there is no API that accepts va_args for making a +// call. Worse, it needs to use the comma operator to return the right +// value. +#define CPyObject_CallMethodObjArgs(obj, attr, ...) \ + (CPy_LogGetAttr("log_method", (obj), (attr)), \ + PyObject_CallMethodObjArgs((obj), (attr), __VA_ARGS__)) + +// This one is a macro for consistency with the above, I guess. +#define CPyObject_GetAttr(obj, attr) \ + (CPy_LogGetAttr("log", (obj), (attr)), \ + PyObject_GetAttr((obj), (attr))) + +CPyTagged CPyObject_Hash(PyObject *o); +PyObject *CPyObject_GetAttr3(PyObject *v, PyObject *name, PyObject *defl); +PyObject *CPyIter_Next(PyObject *iter); +PyObject *CPyNumber_Power(PyObject *base, PyObject *index); +PyObject *CPyNumber_InPlacePower(PyObject *base, PyObject *index); +PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); + + +// List operations + + +PyObject *CPyList_Build(Py_ssize_t len, ...); +PyObject *CPyList_GetItem(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemUnsafe(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); +PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index); +bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); +bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value); +bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); +PyObject *CPyList_PopLast(PyObject *obj); +PyObject *CPyList_Pop(PyObject *obj, CPyTagged index); +CPyTagged CPyList_Count(PyObject *obj, PyObject *value); +int CPyList_Insert(PyObject *list, CPyTagged index, PyObject *value); +PyObject *CPyList_Extend(PyObject *o1, PyObject *o2); +int CPyList_Remove(PyObject *list, PyObject *obj); +CPyTagged CPyList_Index(PyObject *list, PyObject *obj); +PyObject *CPySequence_Multiply(PyObject *seq, CPyTagged t_size); +PyObject *CPySequence_RMultiply(CPyTagged t_size, PyObject *seq); +PyObject *CPyList_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); +int CPySequence_Check(PyObject *obj); + + +// Dict operations + + +PyObject *CPyDict_GetItem(PyObject *dict, PyObject *key); +int CPyDict_SetItem(PyObject *dict, PyObject *key, PyObject *value); +PyObject *CPyDict_Get(PyObject *dict, PyObject *key, PyObject *fallback); +PyObject *CPyDict_GetWithNone(PyObject *dict, PyObject *key); +PyObject *CPyDict_SetDefault(PyObject *dict, PyObject *key, PyObject *value); +PyObject *CPyDict_SetDefaultWithNone(PyObject *dict, PyObject *key); +PyObject *CPyDict_SetDefaultWithEmptyDatatype(PyObject *dict, PyObject *key, int data_type); +PyObject *CPyDict_Build(Py_ssize_t size, ...); +int CPyDict_Update(PyObject *dict, PyObject *stuff); +int CPyDict_UpdateInDisplay(PyObject *dict, PyObject *stuff); +int CPyDict_UpdateFromAny(PyObject *dict, PyObject *stuff); +PyObject *CPyDict_FromAny(PyObject *obj); +PyObject *CPyDict_KeysView(PyObject *dict); +PyObject *CPyDict_ValuesView(PyObject *dict); +PyObject *CPyDict_ItemsView(PyObject *dict); +PyObject *CPyDict_Keys(PyObject *dict); +PyObject *CPyDict_Values(PyObject *dict); +PyObject *CPyDict_Items(PyObject *dict); +char CPyDict_Clear(PyObject *dict); +PyObject *CPyDict_Copy(PyObject *dict); +PyObject *CPyDict_GetKeysIter(PyObject *dict); +PyObject *CPyDict_GetItemsIter(PyObject *dict); +PyObject *CPyDict_GetValuesIter(PyObject *dict); +tuple_T3CIO CPyDict_NextKey(PyObject *dict_or_iter, CPyTagged offset); +tuple_T3CIO CPyDict_NextValue(PyObject *dict_or_iter, CPyTagged offset); +tuple_T4CIOO CPyDict_NextItem(PyObject *dict_or_iter, CPyTagged offset); +int CPyMapping_Check(PyObject *obj); + +// Check that dictionary didn't change size during iteration. +static inline char CPyDict_CheckSize(PyObject *dict, CPyTagged size) { + if (!PyDict_CheckExact(dict)) { + // Dict subclasses will be checked by Python runtime. + return 1; + } + Py_ssize_t py_size = CPyTagged_AsSsize_t(size); + Py_ssize_t dict_size = PyDict_Size(dict); + if (py_size != dict_size) { + PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); + return 0; + } + return 1; +} + + +// Str operations + + +PyObject *CPyStr_Build(Py_ssize_t len, ...); +PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index); +PyObject *CPyStr_Split(PyObject *str, PyObject *sep, CPyTagged max_split); +PyObject *CPyStr_Replace(PyObject *str, PyObject *old_substr, PyObject *new_substr, CPyTagged max_replace); +PyObject *CPyStr_Append(PyObject *o1, PyObject *o2); +PyObject *CPyStr_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); +bool CPyStr_Startswith(PyObject *self, PyObject *subobj); +bool CPyStr_Endswith(PyObject *self, PyObject *subobj); +bool CPyStr_IsTrue(PyObject *obj); +Py_ssize_t CPyStr_Size_size_t(PyObject *str); +PyObject *CPy_Decode(PyObject *obj, PyObject *encoding, PyObject *errors); +PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors); +CPyTagged CPyStr_Ord(PyObject *obj); + + +// Bytes operations + + +PyObject *CPyBytes_Build(Py_ssize_t len, ...); +PyObject *CPyBytes_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); +CPyTagged CPyBytes_GetItem(PyObject *o, CPyTagged index); +PyObject *CPyBytes_Concat(PyObject *a, PyObject *b); +PyObject *CPyBytes_Join(PyObject *sep, PyObject *iter); +CPyTagged CPyBytes_Ord(PyObject *obj); + + +int CPyBytes_Compare(PyObject *left, PyObject *right); + + +// Set operations + + +bool CPySet_Remove(PyObject *set, PyObject *key); + + +// Tuple operations + + +PyObject *CPySequenceTuple_GetItem(PyObject *tuple, CPyTagged index); +PyObject *CPySequenceTuple_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); +bool CPySequenceTuple_SetItemUnsafe(PyObject *tuple, CPyTagged index, PyObject *value); + + +// Exception operations + + +// mypyc is not very good at dealing with refcount management of +// pointers that might be NULL. As a workaround for this, the +// exception APIs that might want to return NULL pointers instead +// return properly refcounted pointers to this dummy object. +struct ExcDummyStruct { PyObject_HEAD }; +extern struct ExcDummyStruct _CPy_ExcDummyStruct; +extern PyObject *_CPy_ExcDummy; + +static inline void _CPy_ToDummy(PyObject **p) { + if (*p == NULL) { + Py_INCREF(_CPy_ExcDummy); + *p = _CPy_ExcDummy; + } +} + +static inline PyObject *_CPy_FromDummy(PyObject *p) { + if (p == _CPy_ExcDummy) return NULL; + Py_INCREF(p); + return p; +} + +static int CPy_NoErrOccured(void) { + return PyErr_Occurred() == NULL; +} + +static inline bool CPy_KeepPropagating(void) { + return 0; +} +// We want to avoid the public PyErr_GetExcInfo API for these because +// it requires a bunch of spurious refcount traffic on the parts of +// the triple we don't care about. +#define CPy_ExcState() PyThreadState_GET()->exc_info + +void CPy_Raise(PyObject *exc); +void CPy_Reraise(void); +void CPyErr_SetObjectAndTraceback(PyObject *type, PyObject *value, PyObject *traceback); +tuple_T3OOO CPy_CatchError(void); +void CPy_RestoreExcInfo(tuple_T3OOO info); +bool CPy_ExceptionMatches(PyObject *type); +PyObject *CPy_GetExcValue(void); +tuple_T3OOO CPy_GetExcInfo(void); +void _CPy_GetExcInfo(PyObject **p_type, PyObject **p_value, PyObject **p_traceback); +void CPyError_OutOfMemory(void); +void CPy_TypeError(const char *expected, PyObject *value); +void CPy_AddTraceback(const char *filename, const char *funcname, int line, PyObject *globals); +void CPy_TypeErrorTraceback(const char *filename, const char *funcname, int line, + PyObject *globals, const char *expected, PyObject *value); +void CPy_AttributeError(const char *filename, const char *funcname, const char *classname, + const char *attrname, int line, PyObject *globals); + + +// Misc operations + +#define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) +#define CPy_TRASHCAN_END(op) Py_TRASHCAN_END + +// Tweaked version of _PyArg_Parser in CPython +typedef struct CPyArg_Parser { + const char *format; + const char * const *keywords; + const char *fname; + const char *custom_msg; + int pos; /* number of positional-only arguments */ + int min; /* minimal number of arguments */ + int max; /* maximal number of positional arguments */ + int has_required_kws; /* are there any keyword-only arguments? */ + int required_kwonly_start; + int varargs; /* does the function accept *args or **kwargs? */ + PyObject *kwtuple; /* tuple of keyword parameter names */ + struct CPyArg_Parser *next; +} CPyArg_Parser; + +// mypy lets ints silently coerce to floats, so a mypyc runtime float +// might be an int also +static inline bool CPyFloat_Check(PyObject *o) { + return PyFloat_Check(o) || PyLong_Check(o); +} + +// TODO: find an unified way to avoid inline functions in non-C back ends that can not +// use inline functions +static inline bool CPy_TypeCheck(PyObject *o, PyObject *type) { + return PyObject_TypeCheck(o, (PyTypeObject *)type); +} + +PyObject *CPy_CalculateMetaclass(PyObject *type, PyObject *o); +PyObject *CPy_GetCoro(PyObject *obj); +PyObject *CPyIter_Send(PyObject *iter, PyObject *val); +int CPy_YieldFromErrorHandle(PyObject *iter, PyObject **outp); +PyObject *CPy_FetchStopIterationValue(void); +PyObject *CPyType_FromTemplate(PyObject *template_, + PyObject *orig_bases, + PyObject *modname); +PyObject *CPyType_FromTemplateWarpper(PyObject *template_, + PyObject *orig_bases, + PyObject *modname); +int CPyDataclass_SleightOfHand(PyObject *dataclass_dec, PyObject *tp, + PyObject *dict, PyObject *annotations); +PyObject *CPyPickle_SetState(PyObject *obj, PyObject *state); +PyObject *CPyPickle_GetState(PyObject *obj); +CPyTagged CPyTagged_Id(PyObject *o); +void CPyDebug_Print(const char *msg); +void CPy_Init(void); +int CPyArg_ParseTupleAndKeywords(PyObject *, PyObject *, + const char *, const char *, const char * const *, ...); +int CPyArg_ParseStackAndKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...); +int CPyArg_ParseStackAndKeywordsNoArgs(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...); +int CPyArg_ParseStackAndKeywordsOneArg(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...); +int CPyArg_ParseStackAndKeywordsSimple(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...); + +int CPySequence_CheckUnpackCount(PyObject *sequence, Py_ssize_t expected); +int CPyStatics_Initialize(PyObject **statics, + const char * const *strings, + const char * const *bytestrings, + const char * const *ints, + const double *floats, + const double *complex_numbers, + const int *tuples, + const int *frozensets); +PyObject *CPy_Super(PyObject *builtins, PyObject *self); +PyObject *CPy_CallReverseOpMethod(PyObject *left, PyObject *right, const char *op, + _Py_Identifier *method); + +bool CPyImport_ImportMany(PyObject *modules, CPyModule **statics[], PyObject *globals, + PyObject *tb_path, PyObject *tb_function, Py_ssize_t *tb_lines); +PyObject *CPyImport_ImportFromMany(PyObject *mod_id, PyObject *names, PyObject *as_names, + PyObject *globals); + +PyObject *CPySingledispatch_RegisterFunction(PyObject *singledispatch_func, PyObject *cls, + PyObject *func); + +PyObject *CPy_GetAIter(PyObject *obj); +PyObject *CPy_GetANext(PyObject *aiter); +void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value); + +#ifdef __cplusplus +} +#endif + +#endif // CPY_CPY_H diff --git a/mypyc/lib-rt/bytes_ops.c b/mypyc/lib-rt/bytes_ops.c new file mode 100644 index 000000000000..5ddf3528211f --- /dev/null +++ b/mypyc/lib-rt/bytes_ops.c @@ -0,0 +1,160 @@ +// Bytes primitive operations +// +// These are registered in mypyc.primitives.bytes_ops. + +#include +#include "CPy.h" + +// Returns -1 on error, 0 on inequality, 1 on equality. +// +// Falls back to PyObject_RichCompareBool. +int CPyBytes_Compare(PyObject *left, PyObject *right) { + if (PyBytes_CheckExact(left) && PyBytes_CheckExact(right)) { + if (left == right) { + return 1; + } + + // Adapted from cpython internal implementation of bytes_compare. + Py_ssize_t len = Py_SIZE(left); + if (Py_SIZE(right) != len) { + return 0; + } + PyBytesObject *left_b = (PyBytesObject *)left; + PyBytesObject *right_b = (PyBytesObject *)right; + if (left_b->ob_sval[0] != right_b->ob_sval[0]) { + return 0; + } + + return memcmp(left_b->ob_sval, right_b->ob_sval, len) == 0; + } + return PyObject_RichCompareBool(left, right, Py_EQ); +} + +CPyTagged CPyBytes_GetItem(PyObject *o, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = ((PyVarObject *)o)->ob_size; + if (n < 0) + n += size; + if (n < 0 || n >= size) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return CPY_INT_TAG; + } + unsigned char num = PyBytes_Check(o) ? ((PyBytesObject *)o)->ob_sval[n] + : ((PyByteArrayObject *)o)->ob_bytes[n]; + return num << 1; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return CPY_INT_TAG; + } +} + +PyObject *CPyBytes_Concat(PyObject *a, PyObject *b) { + if (PyBytes_Check(a) && PyBytes_Check(b)) { + Py_ssize_t a_len = ((PyVarObject *)a)->ob_size; + Py_ssize_t b_len = ((PyVarObject *)b)->ob_size; + PyBytesObject *ret = (PyBytesObject *)PyBytes_FromStringAndSize(NULL, a_len + b_len); + if (ret != NULL) { + memcpy(ret->ob_sval, ((PyBytesObject *)a)->ob_sval, a_len); + memcpy(ret->ob_sval + a_len, ((PyBytesObject *)b)->ob_sval, b_len); + } + return (PyObject *)ret; + } else if (PyByteArray_Check(a)) { + return PyByteArray_Concat(a, b); + } else { + PyBytes_Concat(&a, b); + return a; + } +} + +static inline Py_ssize_t Clamp(Py_ssize_t a, Py_ssize_t b, Py_ssize_t c) { + return a < b ? b : (a >= c ? c : a); +} + +PyObject *CPyBytes_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { + if ((PyBytes_Check(obj) || PyByteArray_Check(obj)) + && CPyTagged_CheckShort(start) && CPyTagged_CheckShort(end)) { + Py_ssize_t startn = CPyTagged_ShortAsSsize_t(start); + Py_ssize_t endn = CPyTagged_ShortAsSsize_t(end); + Py_ssize_t len = ((PyVarObject *)obj)->ob_size; + if (startn < 0) { + startn += len; + } + if (endn < 0) { + endn += len; + } + startn = Clamp(startn, 0, len); + endn = Clamp(endn, 0, len); + Py_ssize_t slice_len = endn - startn; + if (PyBytes_Check(obj)) { + return PyBytes_FromStringAndSize(PyBytes_AS_STRING(obj) + startn, slice_len); + } else { + return PyByteArray_FromStringAndSize(PyByteArray_AS_STRING(obj) + startn, slice_len); + } + } + return CPyObject_GetSlice(obj, start, end); +} + +// Like _PyBytes_Join but fallback to dynamic call if 'sep' is not bytes +// (mostly commonly, for bytearrays) +PyObject *CPyBytes_Join(PyObject *sep, PyObject *iter) { + if (PyBytes_CheckExact(sep)) { + return PyBytes_Join(sep, iter); + } else { + _Py_IDENTIFIER(join); + return _PyObject_CallMethodIdOneArg(sep, &PyId_join, iter); + } +} + +PyObject *CPyBytes_Build(Py_ssize_t len, ...) { + Py_ssize_t i; + Py_ssize_t sz = 0; + + va_list args; + va_start(args, len); + for (i = 0; i < len; i++) { + PyObject *item = va_arg(args, PyObject *); + size_t add_sz = ((PyVarObject *)item)->ob_size; + // Using size_t to avoid overflow during arithmetic calculation + if (add_sz > (size_t)(PY_SSIZE_T_MAX - sz)) { + PyErr_SetString(PyExc_OverflowError, + "join() result is too long for a Python bytes"); + return NULL; + } + sz += add_sz; + } + va_end(args); + + PyBytesObject *ret = (PyBytesObject *)PyBytes_FromStringAndSize(NULL, sz); + if (ret != NULL) { + char *res_data = ret->ob_sval; + va_start(args, len); + for (i = 0; i < len; i++) { + PyObject *item = va_arg(args, PyObject *); + Py_ssize_t item_sz = ((PyVarObject *)item)->ob_size; + memcpy(res_data, ((PyBytesObject *)item)->ob_sval, item_sz); + res_data += item_sz; + } + va_end(args); + assert(res_data == ret->ob_sval + ((PyVarObject *)ret)->ob_size); + } + + return (PyObject *)ret; +} + + +CPyTagged CPyBytes_Ord(PyObject *obj) { + if (PyBytes_Check(obj)) { + Py_ssize_t s = PyBytes_GET_SIZE(obj); + if (s == 1) { + return (unsigned char)(PyBytes_AS_STRING(obj)[0]) << 1; + } + } else if (PyByteArray_Check(obj)) { + Py_ssize_t s = PyByteArray_GET_SIZE(obj); + if (s == 1) { + return (unsigned char)(PyByteArray_AS_STRING(obj)[0]) << 1; + } + } + PyErr_SetString(PyExc_TypeError, "ord() expects a character"); + return CPY_INT_TAG; +} diff --git a/mypyc/lib-rt/dict_ops.c b/mypyc/lib-rt/dict_ops.c new file mode 100644 index 000000000000..b33233521afd --- /dev/null +++ b/mypyc/lib-rt/dict_ops.c @@ -0,0 +1,451 @@ +// Dict primitive operations +// +// These are registered in mypyc.primitives.dict_ops. + +#include +#include "CPy.h" + +#ifndef Py_TPFLAGS_MAPPING +#define Py_TPFLAGS_MAPPING (1 << 6) +#endif + +// Dict subclasses like defaultdict override things in interesting +// ways, so we don't want to just directly use the dict methods. Not +// sure if it is actually worth doing all this stuff, but it saves +// some indirections. +PyObject *CPyDict_GetItem(PyObject *dict, PyObject *key) { + if (PyDict_CheckExact(dict)) { + PyObject *res = PyDict_GetItemWithError(dict, key); + if (!res) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, key); + } + } else { + Py_INCREF(res); + } + return res; + } else { + return PyObject_GetItem(dict, key); + } +} + +PyObject *CPyDict_Build(Py_ssize_t size, ...) { + Py_ssize_t i; + + PyObject *res = _PyDict_NewPresized(size); + if (res == NULL) { + return NULL; + } + + va_list args; + va_start(args, size); + + for (i = 0; i < size; i++) { + PyObject *key = va_arg(args, PyObject *); + PyObject *value = va_arg(args, PyObject *); + if (PyDict_SetItem(res, key, value)) { + Py_DECREF(res); + return NULL; + } + } + + va_end(args); + return res; +} + +PyObject *CPyDict_Get(PyObject *dict, PyObject *key, PyObject *fallback) { + // We are dodgily assuming that get on a subclass doesn't have + // different behavior. + PyObject *res = PyDict_GetItemWithError(dict, key); + if (!res) { + if (PyErr_Occurred()) { + return NULL; + } + res = fallback; + } + Py_INCREF(res); + return res; +} + +PyObject *CPyDict_GetWithNone(PyObject *dict, PyObject *key) { + return CPyDict_Get(dict, key, Py_None); +} + +PyObject *CPyDict_SetDefault(PyObject *dict, PyObject *key, PyObject *value) { + if (PyDict_CheckExact(dict)) { + PyObject* ret = PyDict_SetDefault(dict, key, value); + Py_XINCREF(ret); + return ret; + } + _Py_IDENTIFIER(setdefault); + PyObject *name = _PyUnicode_FromId(&PyId_setdefault); /* borrowed */ + if (name == NULL) { + return NULL; + } + return PyObject_CallMethodObjArgs(dict, name, key, value, NULL); +} + +PyObject *CPyDict_SetDefaultWithNone(PyObject *dict, PyObject *key) { + return CPyDict_SetDefault(dict, key, Py_None); +} + +PyObject *CPyDict_SetDefaultWithEmptyDatatype(PyObject *dict, PyObject *key, + int data_type) { + PyObject *res = CPyDict_GetItem(dict, key); + if (!res) { + // CPyDict_GetItem() would generates a PyExc_KeyError + // when key is not found. + PyErr_Clear(); + + PyObject *new_obj; + if (data_type == 1) { + new_obj = PyList_New(0); + } else if (data_type == 2) { + new_obj = PyDict_New(); + } else if (data_type == 3) { + new_obj = PySet_New(NULL); + } else { + return NULL; + } + + if (CPyDict_SetItem(dict, key, new_obj) == -1) { + return NULL; + } else { + return new_obj; + } + } else { + return res; + } +} + +int CPyDict_SetItem(PyObject *dict, PyObject *key, PyObject *value) { + if (PyDict_CheckExact(dict)) { + return PyDict_SetItem(dict, key, value); + } else { + return PyObject_SetItem(dict, key, value); + } +} + +static inline int CPy_ObjectToStatus(PyObject *obj) { + if (obj) { + Py_DECREF(obj); + return 0; + } else { + return -1; + } +} + +static int CPyDict_UpdateGeneral(PyObject *dict, PyObject *stuff) { + _Py_IDENTIFIER(update); + PyObject *name = _PyUnicode_FromId(&PyId_update); /* borrowed */ + if (name == NULL) { + return -1; + } + PyObject *res = PyObject_CallMethodOneArg(dict, name, stuff); + return CPy_ObjectToStatus(res); +} + +int CPyDict_UpdateInDisplay(PyObject *dict, PyObject *stuff) { + // from https://github.com/python/cpython/blob/55d035113dfb1bd90495c8571758f504ae8d4802/Python/ceval.c#L2710 + int ret = PyDict_Update(dict, stuff); + if (ret < 0) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(stuff)->tp_name); + } + } + return ret; +} + +int CPyDict_Update(PyObject *dict, PyObject *stuff) { + if (PyDict_CheckExact(dict)) { + return PyDict_Update(dict, stuff); + } else { + return CPyDict_UpdateGeneral(dict, stuff); + } +} + +int CPyDict_UpdateFromAny(PyObject *dict, PyObject *stuff) { + if (PyDict_CheckExact(dict)) { + // Argh this sucks + _Py_IDENTIFIER(keys); + if (PyDict_Check(stuff) || _CPyObject_HasAttrId(stuff, &PyId_keys)) { + return PyDict_Update(dict, stuff); + } else { + return PyDict_MergeFromSeq2(dict, stuff, 1); + } + } else { + return CPyDict_UpdateGeneral(dict, stuff); + } +} + +PyObject *CPyDict_FromAny(PyObject *obj) { + if (PyDict_Check(obj)) { + return PyDict_Copy(obj); + } else { + int res; + PyObject *dict = PyDict_New(); + if (!dict) { + return NULL; + } + _Py_IDENTIFIER(keys); + if (_CPyObject_HasAttrId(obj, &PyId_keys)) { + res = PyDict_Update(dict, obj); + } else { + res = PyDict_MergeFromSeq2(dict, obj, 1); + } + if (res < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } +} + +PyObject *CPyDict_KeysView(PyObject *dict) { + if (PyDict_CheckExact(dict)){ + return _CPyDictView_New(dict, &PyDictKeys_Type); + } + _Py_IDENTIFIER(keys); + return _PyObject_CallMethodIdNoArgs(dict, &PyId_keys); +} + +PyObject *CPyDict_ValuesView(PyObject *dict) { + if (PyDict_CheckExact(dict)){ + return _CPyDictView_New(dict, &PyDictValues_Type); + } + _Py_IDENTIFIER(values); + return _PyObject_CallMethodIdNoArgs(dict, &PyId_values); +} + +PyObject *CPyDict_ItemsView(PyObject *dict) { + if (PyDict_CheckExact(dict)){ + return _CPyDictView_New(dict, &PyDictItems_Type); + } + _Py_IDENTIFIER(items); + return _PyObject_CallMethodIdNoArgs(dict, &PyId_items); +} + +PyObject *CPyDict_Keys(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + return PyDict_Keys(dict); + } + // Inline generic fallback logic to also return a list. + PyObject *list = PyList_New(0); + _Py_IDENTIFIER(keys); + PyObject *view = _PyObject_CallMethodIdNoArgs(dict, &PyId_keys); + if (view == NULL) { + return NULL; + } + int res = PyList_Extend(list, view); + Py_DECREF(view); + if (res < 0) { + return NULL; + } + return list; +} + +PyObject *CPyDict_Values(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + return PyDict_Values(dict); + } + // Inline generic fallback logic to also return a list. + PyObject *list = PyList_New(0); + _Py_IDENTIFIER(values); + PyObject *view = _PyObject_CallMethodIdNoArgs(dict, &PyId_values); + if (view == NULL) { + return NULL; + } + int res = PyList_Extend(list, view); + Py_DECREF(view); + if (res < 0) { + return NULL; + } + return list; +} + +PyObject *CPyDict_Items(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + return PyDict_Items(dict); + } + // Inline generic fallback logic to also return a list. + PyObject *list = PyList_New(0); + _Py_IDENTIFIER(items); + PyObject *view = _PyObject_CallMethodIdNoArgs(dict, &PyId_items); + if (view == NULL) { + return NULL; + } + int res = PyList_Extend(list, view); + Py_DECREF(view); + if (res < 0) { + return NULL; + } + return list; +} + +char CPyDict_Clear(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + PyDict_Clear(dict); + } else { + _Py_IDENTIFIER(clear); + PyObject *res = _PyObject_CallMethodIdNoArgs(dict, &PyId_clear); + if (res == NULL) { + return 0; + } + } + return 1; +} + +PyObject *CPyDict_Copy(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + return PyDict_Copy(dict); + } + _Py_IDENTIFIER(copy); + return _PyObject_CallMethodIdNoArgs(dict, &PyId_copy); +} + +PyObject *CPyDict_GetKeysIter(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + // Return dict itself to indicate we can use fast path instead. + Py_INCREF(dict); + return dict; + } + return PyObject_GetIter(dict); +} + +PyObject *CPyDict_GetItemsIter(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + // Return dict itself to indicate we can use fast path instead. + Py_INCREF(dict); + return dict; + } + _Py_IDENTIFIER(items); + PyObject *view = _PyObject_CallMethodIdNoArgs(dict, &PyId_items); + if (view == NULL) { + return NULL; + } + PyObject *iter = PyObject_GetIter(view); + Py_DECREF(view); + return iter; +} + +PyObject *CPyDict_GetValuesIter(PyObject *dict) { + if (PyDict_CheckExact(dict)) { + // Return dict itself to indicate we can use fast path instead. + Py_INCREF(dict); + return dict; + } + _Py_IDENTIFIER(values); + PyObject *view = _PyObject_CallMethodIdNoArgs(dict, &PyId_values); + if (view == NULL) { + return NULL; + } + PyObject *iter = PyObject_GetIter(view); + Py_DECREF(view); + return iter; +} + +static void _CPyDict_FromNext(tuple_T3CIO *ret, PyObject *dict_iter) { + // Get next item from iterator and set "should continue" flag. + ret->f2 = PyIter_Next(dict_iter); + if (ret->f2 == NULL) { + ret->f0 = 0; + Py_INCREF(Py_None); + ret->f2 = Py_None; + } else { + ret->f0 = 1; + } +} + +// Helpers for fast dictionary iteration, return a single tuple +// instead of writing to multiple registers, for exact dicts use +// the fast path, and fall back to generic iterator logic for subclasses. +tuple_T3CIO CPyDict_NextKey(PyObject *dict_or_iter, CPyTagged offset) { + tuple_T3CIO ret; + Py_ssize_t py_offset = CPyTagged_AsSsize_t(offset); + PyObject *dummy; + + if (PyDict_CheckExact(dict_or_iter)) { + ret.f0 = PyDict_Next(dict_or_iter, &py_offset, &ret.f2, &dummy); + if (ret.f0) { + ret.f1 = CPyTagged_FromSsize_t(py_offset); + } else { + // Set key to None, so mypyc can manage refcounts. + ret.f1 = 0; + ret.f2 = Py_None; + } + // PyDict_Next() returns borrowed references. + Py_INCREF(ret.f2); + } else { + // offset is dummy in this case, just use the old value. + ret.f1 = offset; + _CPyDict_FromNext(&ret, dict_or_iter); + } + return ret; +} + +tuple_T3CIO CPyDict_NextValue(PyObject *dict_or_iter, CPyTagged offset) { + tuple_T3CIO ret; + Py_ssize_t py_offset = CPyTagged_AsSsize_t(offset); + PyObject *dummy; + + if (PyDict_CheckExact(dict_or_iter)) { + ret.f0 = PyDict_Next(dict_or_iter, &py_offset, &dummy, &ret.f2); + if (ret.f0) { + ret.f1 = CPyTagged_FromSsize_t(py_offset); + } else { + // Set value to None, so mypyc can manage refcounts. + ret.f1 = 0; + ret.f2 = Py_None; + } + // PyDict_Next() returns borrowed references. + Py_INCREF(ret.f2); + } else { + // offset is dummy in this case, just use the old value. + ret.f1 = offset; + _CPyDict_FromNext(&ret, dict_or_iter); + } + return ret; +} + +tuple_T4CIOO CPyDict_NextItem(PyObject *dict_or_iter, CPyTagged offset) { + tuple_T4CIOO ret; + Py_ssize_t py_offset = CPyTagged_AsSsize_t(offset); + + if (PyDict_CheckExact(dict_or_iter)) { + ret.f0 = PyDict_Next(dict_or_iter, &py_offset, &ret.f2, &ret.f3); + if (ret.f0) { + ret.f1 = CPyTagged_FromSsize_t(py_offset); + } else { + // Set key and value to None, so mypyc can manage refcounts. + ret.f1 = 0; + ret.f2 = Py_None; + ret.f3 = Py_None; + } + } else { + ret.f1 = offset; + PyObject *item = PyIter_Next(dict_or_iter); + if (item == NULL || !PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { + if (item != NULL) { + PyErr_SetString(PyExc_TypeError, "a tuple of length 2 expected"); + } + ret.f0 = 0; + ret.f2 = Py_None; + ret.f3 = Py_None; + } else { + ret.f0 = 1; + ret.f2 = PyTuple_GET_ITEM(item, 0); + ret.f3 = PyTuple_GET_ITEM(item, 1); + Py_DECREF(item); + } + } + // PyDict_Next() returns borrowed references. + Py_INCREF(ret.f2); + Py_INCREF(ret.f3); + return ret; +} + +int CPyMapping_Check(PyObject *obj) { + return Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MAPPING; +} diff --git a/mypyc/lib-rt/exc_ops.c b/mypyc/lib-rt/exc_ops.c new file mode 100644 index 000000000000..d8307ecf21f8 --- /dev/null +++ b/mypyc/lib-rt/exc_ops.c @@ -0,0 +1,259 @@ +// Exception related primitive operations +// +// These are registered in mypyc.primitives.exc_ops. + +#include +#include "CPy.h" + +void CPy_Raise(PyObject *exc) { + if (PyObject_IsInstance(exc, (PyObject *)&PyType_Type)) { + PyObject *obj = PyObject_CallNoArgs(exc); + if (!obj) + return; + PyErr_SetObject(exc, obj); + Py_DECREF(obj); + } else { + PyErr_SetObject((PyObject *)Py_TYPE(exc), exc); + } +} + +void CPy_Reraise(void) { + PyObject *p_type, *p_value, *p_traceback; + PyErr_GetExcInfo(&p_type, &p_value, &p_traceback); + PyErr_Restore(p_type, p_value, p_traceback); +} + +void CPyErr_SetObjectAndTraceback(PyObject *type, PyObject *value, PyObject *traceback) { + if (!PyType_Check(type) && value == Py_None) { + // The first argument must be an exception instance + value = type; + type = (PyObject *)Py_TYPE(value); + } + + // Set the value and traceback of an error. Because calling + // PyErr_Restore takes away a reference to each object passed in + // as an argument, we manually increase the reference count of + // each argument before calling it. + Py_INCREF(type); + Py_INCREF(value); + Py_INCREF(traceback); + PyErr_Restore(type, value, traceback); +} + +tuple_T3OOO CPy_CatchError(void) { + // We need to return the existing sys.exc_info() information, so + // that it can be restored when we finish handling the error we + // are catching now. Grab that triple and convert NULL values to + // the ExcDummy object in order to simplify refcount handling in + // generated code. + tuple_T3OOO ret; + PyErr_GetExcInfo(&ret.f0, &ret.f1, &ret.f2); + _CPy_ToDummy(&ret.f0); + _CPy_ToDummy(&ret.f1); + _CPy_ToDummy(&ret.f2); + + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "CPy_CatchError called with no error!"); + } + + // Retrieve the error info and normalize it so that it looks like + // what python code needs it to be. + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + // Could we avoid always normalizing? + PyErr_NormalizeException(&type, &value, &traceback); + if (traceback != NULL) { + PyException_SetTraceback(value, traceback); + } + // Indicate that we are now handling this exception by stashing it + // in sys.exc_info(). mypyc routines that need access to the + // exception will read it out of there. + PyErr_SetExcInfo(type, value, traceback); + // Clear the error indicator, since the exception isn't + // propagating anymore. + PyErr_Clear(); + + return ret; +} + +void CPy_RestoreExcInfo(tuple_T3OOO info) { + PyErr_SetExcInfo(_CPy_FromDummy(info.f0), _CPy_FromDummy(info.f1), _CPy_FromDummy(info.f2)); +} + +bool CPy_ExceptionMatches(PyObject *type) { + return PyErr_GivenExceptionMatches((PyObject *)Py_TYPE(CPy_ExcState()->exc_value), type); +} + +PyObject *CPy_GetExcValue(void) { + PyObject *exc = CPy_ExcState()->exc_value; + Py_INCREF(exc); + return exc; +} + +static inline void _CPy_ToNone(PyObject **p) { + if (*p == NULL) { + Py_INCREF(Py_None); + *p = Py_None; + } +} + +void _CPy_GetExcInfo(PyObject **p_type, PyObject **p_value, PyObject **p_traceback) { + PyErr_GetExcInfo(p_type, p_value, p_traceback); + _CPy_ToNone(p_type); + _CPy_ToNone(p_value); + _CPy_ToNone(p_traceback); +} + +tuple_T3OOO CPy_GetExcInfo(void) { + tuple_T3OOO ret; + _CPy_GetExcInfo(&ret.f0, &ret.f1, &ret.f2); + return ret; +} + +void CPyError_OutOfMemory(void) { + fprintf(stderr, "fatal: out of memory\n"); + fflush(stderr); + abort(); +} + +// Construct a nicely formatted type name based on __module__ and __name__. +static PyObject *CPy_GetTypeName(PyObject *type) { + PyObject *module = NULL, *name = NULL; + PyObject *full = NULL; + + module = PyObject_GetAttrString(type, "__module__"); + if (!module || !PyUnicode_Check(module)) { + goto out; + } + name = PyObject_GetAttrString(type, "__qualname__"); + if (!name || !PyUnicode_Check(name)) { + goto out; + } + + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + Py_INCREF(name); + full = name; + } else { + full = PyUnicode_FromFormat("%U.%U", module, name); + } + +out: + Py_XDECREF(module); + Py_XDECREF(name); + return full; +} + +// Get the type of a value as a string, expanding tuples to include +// all the element types. +static PyObject *CPy_FormatTypeName(PyObject *value) { + if (Py_IsNone(value)) { + return PyUnicode_FromString("None"); + } + + if (!PyTuple_CheckExact(value)) { + return CPy_GetTypeName((PyObject *)Py_TYPE(value)); + } + + if (PyTuple_GET_SIZE(value) > 10) { + return PyUnicode_FromFormat("tuple[<%d items>]", PyTuple_GET_SIZE(value)); + } + + // Most of the logic is all for tuples, which is the only interesting case + PyObject *output = PyUnicode_FromString("tuple["); + if (!output) { + return NULL; + } + /* This is quadratic but if that ever matters something is really weird. */ + int i; + for (i = 0; i < PyTuple_GET_SIZE(value); i++) { + PyObject *s = CPy_FormatTypeName(PyTuple_GET_ITEM(value, i)); + if (!s) { + Py_DECREF(output); + return NULL; + } + PyObject *next = PyUnicode_FromFormat("%U%U%s", output, s, + i + 1 == PyTuple_GET_SIZE(value) ? "]" : ", "); + Py_DECREF(output); + Py_DECREF(s); + if (!next) { + return NULL; + } + output = next; + } + return output; +} + +CPy_NOINLINE +void CPy_TypeError(const char *expected, PyObject *value) { + PyObject *out = CPy_FormatTypeName(value); + if (out) { + PyErr_Format(PyExc_TypeError, "%s object expected; got %U", expected, out); + Py_DECREF(out); + } else { + PyErr_Format(PyExc_TypeError, "%s object expected; and errored formatting real type!", + expected); + } +} + +// The PyFrameObject type definition (struct _frame) has been moved +// to the internal C API: to the pycore_frame.h header file. +// https://github.com/python/cpython/pull/31530 +#if PY_VERSION_HEX >= 0x030b00a6 +#include "internal/pycore_frame.h" +#endif + +// This function is basically exactly the same with _PyTraceback_Add +// which is available in all the versions we support. +// We're continuing to use this because we'll probably optimize this later. +void CPy_AddTraceback(const char *filename, const char *funcname, int line, PyObject *globals) { + PyObject *exc, *val, *tb; + PyThreadState *thread_state = PyThreadState_GET(); + PyFrameObject *frame_obj; + + // We need to save off the exception state because in 3.8, + // PyFrame_New fails if there is an error set and it fails to look + // up builtins in the globals. (_PyTraceback_Add documents that it + // needs to do it because it decodes the filename according to the + // FS encoding, which could have a decoder in Python. We don't do + // that so *that* doesn't apply to us.) + PyErr_Fetch(&exc, &val, &tb); + PyCodeObject *code_obj = PyCode_NewEmpty(filename, funcname, line); + if (code_obj == NULL) { + goto error; + } + + frame_obj = PyFrame_New(thread_state, code_obj, globals, 0); + if (frame_obj == NULL) { + Py_DECREF(code_obj); + goto error; + } + frame_obj->f_lineno = line; + PyErr_Restore(exc, val, tb); + PyTraceBack_Here(frame_obj); + Py_DECREF(code_obj); + Py_DECREF(frame_obj); + + return; + +error: +#if CPY_3_12_FEATURES + _PyErr_ChainExceptions1(exc); +#else + _PyErr_ChainExceptions(exc, val, tb); +#endif +} + +CPy_NOINLINE +void CPy_TypeErrorTraceback(const char *filename, const char *funcname, int line, + PyObject *globals, const char *expected, PyObject *value) { + CPy_TypeError(expected, value); + CPy_AddTraceback(filename, funcname, line, globals); +} + +void CPy_AttributeError(const char *filename, const char *funcname, const char *classname, + const char *attrname, int line, PyObject *globals) { + char buf[500]; + snprintf(buf, sizeof(buf), "attribute '%.200s' of '%.200s' undefined", attrname, classname); + PyErr_SetString(PyExc_AttributeError, buf); + CPy_AddTraceback(filename, funcname, line, globals); +} diff --git a/mypyc/lib-rt/float_ops.c b/mypyc/lib-rt/float_ops.c new file mode 100644 index 000000000000..d8c6f25955fa --- /dev/null +++ b/mypyc/lib-rt/float_ops.c @@ -0,0 +1,192 @@ +// Float primitive operations +// +// These are registered in mypyc.primitives.float_ops. + +#include +#include "CPy.h" + + +static double CPy_DomainError(void) { + PyErr_SetString(PyExc_ValueError, "math domain error"); + return CPY_FLOAT_ERROR; +} + +static double CPy_MathRangeError(void) { + PyErr_SetString(PyExc_OverflowError, "math range error"); + return CPY_FLOAT_ERROR; +} + +double CPyFloat_FromTagged(CPyTagged x) { + if (CPyTagged_CheckShort(x)) { + return CPyTagged_ShortAsSsize_t(x); + } + double result = PyFloat_AsDouble(CPyTagged_LongAsObject(x)); + if (unlikely(result == -1.0) && PyErr_Occurred()) { + return CPY_FLOAT_ERROR; + } + return result; +} + +double CPyFloat_Sin(double x) { + double v = sin(x); + if (unlikely(isnan(v)) && !isnan(x)) { + return CPy_DomainError(); + } + return v; +} + +double CPyFloat_Cos(double x) { + double v = cos(x); + if (unlikely(isnan(v)) && !isnan(x)) { + return CPy_DomainError(); + } + return v; +} + +double CPyFloat_Tan(double x) { + if (unlikely(isinf(x))) { + return CPy_DomainError(); + } + return tan(x); +} + +double CPyFloat_Sqrt(double x) { + if (x < 0.0) { + return CPy_DomainError(); + } + return sqrt(x); +} + +double CPyFloat_Exp(double x) { + double v = exp(x); + if (unlikely(v == INFINITY) && x != INFINITY) { + return CPy_MathRangeError(); + } + return v; +} + +double CPyFloat_Log(double x) { + if (x <= 0.0) { + return CPy_DomainError(); + } + return log(x); +} + +CPyTagged CPyFloat_Floor(double x) { + double v = floor(x); + return CPyTagged_FromFloat(v); +} + +CPyTagged CPyFloat_Ceil(double x) { + double v = ceil(x); + return CPyTagged_FromFloat(v); +} + +bool CPyFloat_IsInf(double x) { + return isinf(x) != 0; +} + +bool CPyFloat_IsNaN(double x) { + return isnan(x) != 0; +} + +// From CPython 3.10.0, Objects/floatobject.c +static void +_float_div_mod(double vx, double wx, double *floordiv, double *mod) +{ + double div; + *mod = fmod(vx, wx); + /* fmod is typically exact, so vx-mod is *mathematically* an + exact multiple of wx. But this is fp arithmetic, and fp + vx - mod is an approximation; the result is that div may + not be an exact integral value after the division, although + it will always be very close to one. + */ + div = (vx - *mod) / wx; + if (*mod) { + /* ensure the remainder has the same sign as the denominator */ + if ((wx < 0) != (*mod < 0)) { + *mod += wx; + div -= 1.0; + } + } + else { + /* the remainder is zero, and in the presence of signed zeroes + fmod returns different results across platforms; ensure + it has the same sign as the denominator. */ + *mod = copysign(0.0, wx); + } + /* snap quotient to nearest integral value */ + if (div) { + *floordiv = floor(div); + if (div - *floordiv > 0.5) { + *floordiv += 1.0; + } + } + else { + /* div is zero - get the same sign as the true quotient */ + *floordiv = copysign(0.0, vx / wx); /* zero w/ sign of vx/wx */ + } +} + +double CPyFloat_FloorDivide(double x, double y) { + double mod, floordiv; + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "float floor division by zero"); + return CPY_FLOAT_ERROR; + } + _float_div_mod(x, y, &floordiv, &mod); + return floordiv; +} + +// Adapted from CPython 3.10.7 +double CPyFloat_Pow(double x, double y) { + if (!isfinite(x) || !isfinite(y)) { + if (isnan(x)) + return y == 0.0 ? 1.0 : x; /* NaN**0 = 1 */ + else if (isnan(y)) + return x == 1.0 ? 1.0 : y; /* 1**NaN = 1 */ + else if (isinf(x)) { + int odd_y = isfinite(y) && fmod(fabs(y), 2.0) == 1.0; + if (y > 0.0) + return odd_y ? x : fabs(x); + else if (y == 0.0) + return 1.0; + else /* y < 0. */ + return odd_y ? copysign(0.0, x) : 0.0; + } + else if (isinf(y)) { + if (fabs(x) == 1.0) + return 1.0; + else if (y > 0.0 && fabs(x) > 1.0) + return y; + else if (y < 0.0 && fabs(x) < 1.0) { + #if PY_VERSION_HEX < 0x030B0000 + if (x == 0.0) { /* 0**-inf: divide-by-zero */ + return CPy_DomainError(); + } + #endif + return -y; /* result is +inf */ + } else + return 0.0; + } + } + double r = pow(x, y); + if (!isfinite(r)) { + if (isnan(r)) { + return CPy_DomainError(); + } + /* + an infinite result here arises either from: + (A) (+/-0.)**negative (-> divide-by-zero) + (B) overflow of x**y with x and y finite + */ + else if (isinf(r)) { + if (x == 0.0) + return CPy_DomainError(); + else + return CPy_MathRangeError(); + } + } + return r; +} diff --git a/mypyc/lib-rt/generic_ops.c b/mypyc/lib-rt/generic_ops.c new file mode 100644 index 000000000000..260cfec5b360 --- /dev/null +++ b/mypyc/lib-rt/generic_ops.c @@ -0,0 +1,64 @@ +// Generic primitive operations +// +// These are registered in mypyc.primitives.generic_ops. + +#include +#include "CPy.h" + +CPyTagged CPyObject_Hash(PyObject *o) { + Py_hash_t h = PyObject_Hash(o); + if (h == -1) { + return CPY_INT_TAG; + } else { + // This is tragically annoying. The range of hash values in + // 64-bit python covers 64-bits, and our short integers only + // cover 63. This means that half the time we are boxing the + // result for basically no good reason. To add insult to + // injury it is probably about to be immediately unboxed by a + // tp_hash wrapper. + return CPyTagged_FromSsize_t(h); + } +} + +PyObject *CPyObject_GetAttr3(PyObject *v, PyObject *name, PyObject *defl) +{ + PyObject *result = PyObject_GetAttr(v, name); + if (!result && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + Py_INCREF(defl); + result = defl; + } + return result; +} + +PyObject *CPyIter_Next(PyObject *iter) +{ + return (*Py_TYPE(iter)->tp_iternext)(iter); +} + +PyObject *CPyNumber_Power(PyObject *base, PyObject *index) +{ + return PyNumber_Power(base, index, Py_None); +} + +PyObject *CPyNumber_InPlacePower(PyObject *base, PyObject *index) +{ + return PyNumber_InPlacePower(base, index, Py_None); +} + +PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { + PyObject *start_obj = CPyTagged_AsObject(start); + PyObject *end_obj = CPyTagged_AsObject(end); + if (unlikely(start_obj == NULL || end_obj == NULL)) { + return NULL; + } + PyObject *slice = PySlice_New(start_obj, end_obj, NULL); + Py_DECREF(start_obj); + Py_DECREF(end_obj); + if (unlikely(slice == NULL)) { + return NULL; + } + PyObject *result = PyObject_GetItem(obj, slice); + Py_DECREF(slice); + return result; +} diff --git a/mypyc/lib-rt/getargs.c b/mypyc/lib-rt/getargs.c new file mode 100644 index 000000000000..4f2f8aa0be83 --- /dev/null +++ b/mypyc/lib-rt/getargs.c @@ -0,0 +1,454 @@ +/* getargs implementation copied from Python 3.8 and stripped down to only include + * the functions we need. + * We also add support for required kwonly args and accepting *args / **kwargs. + * A good idea would be to also vendor in the Fast versions and get our stuff + * working with *that*. + * Another probably good idea is to strip out all the formatting stuff we don't need + * and then add in custom stuff that we do need. + * + * DOCUMENTATION OF THE EXTENSIONS: + * - Arguments given after a @ format specify are required keyword-only arguments. + * The | and $ specifiers must both appear before @. + * - If the first character of a format string is %, then the function can support + * *args and **kwargs. On seeing a %, the parser will consume two arguments, + * which should be pointers to variables to store the *args and **kwargs, respectively. + * Either pointer can be NULL, in which case the function doesn't take that + * variety of vararg. + * Unlike most format specifiers, the caller takes ownership of these objects + * and is responsible for decrefing them. + * - All arguments must use the 'O' format. + * - There's minimal error checking of format strings. They are generated + * programmatically and can be assumed valid. + */ + +// These macro definitions are copied from pyport.h in Python 3.9 and later +// https://bugs.python.org/issue19569 +#if defined(__clang__) +#define _Py_COMP_DIAG_PUSH _Pragma("clang diagnostic push") +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS \ + _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#define _Py_COMP_DIAG_POP _Pragma("clang diagnostic pop") +#elif defined(__GNUC__) \ + && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 6)) +#define _Py_COMP_DIAG_PUSH _Pragma("GCC diagnostic push") +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS \ + _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#define _Py_COMP_DIAG_POP _Pragma("GCC diagnostic pop") +#elif defined(_MSC_VER) +#define _Py_COMP_DIAG_PUSH __pragma(warning(push)) +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS __pragma(warning(disable: 4996)) +#define _Py_COMP_DIAG_POP __pragma(warning(pop)) +#else +#define _Py_COMP_DIAG_PUSH +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS +#define _Py_COMP_DIAG_POP +#endif + +#include "Python.h" +#include "pythonsupport.h" + +#include +#include + +#ifndef PyDict_GET_SIZE +#define PyDict_GET_SIZE(d) PyDict_Size(d) +#endif + + +#ifdef __cplusplus +extern "C" { +#endif +int CPyArg_ParseTupleAndKeywords(PyObject *, PyObject *, + const char *, const char *, const char * const *, ...); + +/* Forward */ +static int vgetargskeywords(PyObject *, PyObject *, + const char *, const char *, const char * const *, va_list *); +static void skipitem(const char **, va_list *); + +/* Support for keyword arguments donated by + Geoff Philbrick */ + +/* Return false (0) for error, else true. */ +int +CPyArg_ParseTupleAndKeywords(PyObject *args, + PyObject *keywords, + const char *format, + const char *fname, + const char * const *kwlist, ...) +{ + int retval; + va_list va; + + va_start(va, kwlist); + retval = vgetargskeywords(args, keywords, format, fname, kwlist, &va); + va_end(va); + return retval; +} + +#define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') + +static int +vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, + const char *fname, const char * const *kwlist, va_list *p_va) +{ + int min = INT_MAX; + int max = INT_MAX; + int required_kwonly_start = INT_MAX; + int has_required_kws = 0; + int i, pos, len; + int skip = 0; + Py_ssize_t nargs, nkwargs; + PyObject *current_arg; + int bound_pos_args; + + PyObject **p_args = NULL, **p_kwargs = NULL; + + assert(args != NULL && PyTuple_Check(args)); + assert(kwargs == NULL || PyDict_Check(kwargs)); + assert(format != NULL); + assert(kwlist != NULL); + assert(p_va != NULL); + + /* scan kwlist and count the number of positional-only parameters */ + for (pos = 0; kwlist[pos] && !*kwlist[pos]; pos++) { + } + /* scan kwlist and get greatest possible nbr of args */ + for (len = pos; kwlist[len]; len++) { +#ifdef DEBUG + if (!*kwlist[len]) { + PyErr_SetString(PyExc_SystemError, + "Empty keyword parameter name"); + return 0; + } +#endif + } + + if (*format == '%') { + p_args = va_arg(*p_va, PyObject **); + p_kwargs = va_arg(*p_va, PyObject **); + format++; + } + + nargs = PyTuple_GET_SIZE(args); + nkwargs = (kwargs == NULL) ? 0 : PyDict_GET_SIZE(kwargs); + if (unlikely(nargs + nkwargs > len && !p_args && !p_kwargs)) { + /* Adding "keyword" (when nargs == 0) prevents producing wrong error + messages in some special cases (see bpo-31229). */ + PyErr_Format(PyExc_TypeError, + "%.200s%s takes at most %d %sargument%s (%zd given)", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()", + len, + (nargs == 0) ? "keyword " : "", + (len == 1) ? "" : "s", + nargs + nkwargs); + return 0; + } + + /* convert tuple args and keyword args in same loop, using kwlist to drive process */ + for (i = 0; i < len; i++) { + if (*format == '|') { +#ifdef DEBUG + if (min != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string (| specified twice)"); + return 0; + } +#endif + + min = i; + format++; + +#ifdef DEBUG + if (max != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string ($ before |)"); + return 0; + } +#endif + + /* If there are optional args, figure out whether we have + * required keyword arguments so that we don't bail without + * enforcing them. */ + has_required_kws = strchr(format, '@') != NULL; + } + if (*format == '$') { +#ifdef DEBUG + if (max != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string ($ specified twice)"); + return 0; + } +#endif + + max = i; + format++; + +#ifdef DEBUG + if (max < pos) { + PyErr_SetString(PyExc_SystemError, + "Empty parameter name after $"); + return 0; + } +#endif + if (skip) { + /* Now we know the minimal and the maximal numbers of + * positional arguments and can raise an exception with + * informative message (see below). */ + break; + } + if (unlikely(max < nargs && !p_args)) { + if (max == 0) { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes no positional arguments", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()"); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes %s %d positional argument%s" + " (%zd given)", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()", + (min < max) ? "at most" : "exactly", + max, + max == 1 ? "" : "s", + nargs); + } + return 0; + } + } + if (*format == '@') { +#ifdef DEBUG + if (min == INT_MAX && max == INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string " + "(@ without preceding | and $)"); + return 0; + } + if (required_kwonly_start != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string (@ specified twice)"); + return 0; + } +#endif + + required_kwonly_start = i; + format++; + } +#ifdef DEBUG + if (IS_END_OF_FORMAT(*format)) { + PyErr_Format(PyExc_SystemError, + "More keyword list entries (%d) than " + "format specifiers (%d)", len, i); + return 0; + } +#endif + if (!skip) { + if (i < nargs && i < max) { + current_arg = Py_NewRef(PyTuple_GET_ITEM(args, i)); + } + else if (nkwargs && i >= pos) { + int res = PyDict_GetItemStringRef(kwargs, kwlist[i], ¤t_arg); + if (res == 1) { + --nkwargs; + } + else if (res == -1) { + return 0; + } + } + else { + current_arg = NULL; + } + + if (current_arg) { + PyObject **p = va_arg(*p_va, PyObject **); + *p = current_arg; + Py_DECREF(current_arg); + format++; + continue; + } + + if (i < min || i >= required_kwonly_start) { + if (likely(i < pos)) { + assert (min == INT_MAX); + assert (max == INT_MAX); + skip = 1; + /* At that moment we still don't know the minimal and + * the maximal numbers of positional arguments. Raising + * an exception is deferred until we encounter | and $ + * or the end of the format. */ + } + else { + if (i >= max) { + PyErr_Format(PyExc_TypeError, + "%.200s%s missing required " + "keyword-only argument '%s'", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()", + kwlist[i]); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s missing required " + "argument '%s' (pos %d)", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()", + kwlist[i], i+1); + } + return 0; + } + } + /* current code reports success when all required args + * fulfilled and no keyword args left, with no further + * validation. XXX Maybe skip this in debug build ? + */ + if (!nkwargs && !skip && !has_required_kws && + !p_args && !p_kwargs) + { + return 1; + } + } + + /* We are into optional args, skip through to any remaining + * keyword args */ + skipitem(&format, p_va); + } + + if (unlikely(skip)) { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes %s %d positional argument%s" + " (%zd given)", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()", + (Py_MIN(pos, min) < i) ? "at least" : "exactly", + Py_MIN(pos, min), + Py_MIN(pos, min) == 1 ? "" : "s", + nargs); + return 0; + } + +#ifdef DEBUG + if (!IS_END_OF_FORMAT(*format) && + (*format != '|') && (*format != '$') && (*format != '@')) + { + PyErr_Format(PyExc_SystemError, + "more argument specifiers than keyword list entries " + "(remaining format:'%s')", format); + return 0; + } +#endif + + bound_pos_args = Py_MIN(nargs, Py_MIN(max, len)); + if (p_args) { + *p_args = PyTuple_GetSlice(args, bound_pos_args, nargs); + if (!*p_args) { + return 0; + } + } + + if (p_kwargs) { + /* This unfortunately needs to be special cased because if len is 0 then we + * never go through the main loop. */ + if (unlikely(nargs > 0 && len == 0 && !p_args)) { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes no positional arguments", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()"); + + return 0; + } + + *p_kwargs = PyDict_New(); + if (!*p_kwargs) { + goto latefail; + } + } + + if (nkwargs > 0) { + PyObject *key, *value; + Py_ssize_t j; + /* make sure there are no arguments given by name and position */ + for (i = pos; i < bound_pos_args && i < len; i++) { + int res = PyDict_GetItemStringRef(kwargs, kwlist[i], ¤t_arg); + if (res == 1) { + Py_DECREF(current_arg); + } + else if (unlikely(res == 0)) { + /* arg present in tuple and in dict */ + PyErr_Format(PyExc_TypeError, + "argument for %.200s%s given by name ('%s') " + "and position (%d)", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()", + kwlist[i], i+1); + goto latefail; + } + else if (unlikely(res == -1)) { + goto latefail; + } + } + /* make sure there are no extraneous keyword arguments */ + j = 0; + while (PyDict_Next(kwargs, &j, &key, &value)) { + int match = 0; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, + "keywords must be strings"); + goto latefail; + } + for (i = pos; i < len; i++) { + if (PyUnicode_EqualToUTF8(key, kwlist[i])) { + match = 1; + break; + } + } + if (!match) { + if (unlikely(!p_kwargs)) { + PyErr_Format(PyExc_TypeError, + "'%U' is an invalid keyword " + "argument for %.200s%s", + key, + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()"); + goto latefail; + } else { + if (PyDict_SetItem(*p_kwargs, key, value) < 0) { + goto latefail; + } + } + } + } + } + + return 1; + /* Handle failures that have happened after we have tried to + * create *args and **kwargs, if they exist. */ +latefail: + if (p_args) { + Py_XDECREF(*p_args); + } + if (p_kwargs) { + Py_XDECREF(*p_kwargs); + } + return 0; +} + + +static void +skipitem(const char **p_format, va_list *p_va) +{ + const char *format = *p_format; + char c = *format++; + + if (p_va != NULL) { + (void) va_arg(*p_va, PyObject **); + } + + *p_format = format; +} + +#ifdef __cplusplus +}; +#endif diff --git a/mypyc/lib-rt/getargsfast.c b/mypyc/lib-rt/getargsfast.c new file mode 100644 index 000000000000..e5667e22efe3 --- /dev/null +++ b/mypyc/lib-rt/getargsfast.c @@ -0,0 +1,569 @@ +/* getargskeywordsfast implementation copied from Python 3.9 and stripped down to + * only include the functionality we need. + * + * We also add support for required kwonly args and accepting *args / **kwargs. + * + * DOCUMENTATION OF THE EXTENSIONS: + * - Arguments given after a @ format specify required keyword-only arguments. + * The | and $ specifiers must both appear before @. + * - If the first character of a format string is %, then the function can support + * *args and/or **kwargs. In this case the parser will consume two arguments, + * which should be pointers to variables to store the *args and **kwargs, respectively. + * Either pointer can be NULL, in which case the function doesn't take that + * variety of vararg. + * Unlike most format specifiers, the caller takes ownership of these objects + * and is responsible for decrefing them. + */ + +#include +#include "CPy.h" + +#define PARSER_INITED(parser) ((parser)->kwtuple != NULL) + +/* Forward */ +static int +vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject *kwnames, + CPyArg_Parser *parser, + va_list *p_va); +static void skipitem_fast(const char **, va_list *); + +/* Parse args for an arbitrary signature */ +int +CPyArg_ParseStackAndKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + va_start(va, parser); + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va); + va_end(va); + return retval; +} + +/* Parse args for a function that takes no args */ +int +CPyArg_ParseStackAndKeywordsNoArgs(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + va_start(va, parser); + if (nargs == 0 && kwnames == NULL) { + // Fast path: no arguments + retval = 1; + } else { + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va); + } + va_end(va); + return retval; +} + +/* Parse args for a function that takes one arg */ +int +CPyArg_ParseStackAndKeywordsOneArg(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + va_start(va, parser); + if (kwnames == NULL && nargs == 1) { + // Fast path: one positional argument + PyObject **p; + p = va_arg(va, PyObject **); + *p = args[0]; + retval = 1; + } else { + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va); + } + va_end(va); + return retval; +} + +/* Parse args for a function that takes no keyword-only args, *args or **kwargs */ +int +CPyArg_ParseStackAndKeywordsSimple(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + CPyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + va_start(va, parser); + if (kwnames == NULL && PARSER_INITED(parser) && + nargs >= parser->min && nargs <= parser->max) { + // Fast path: correct number of positional arguments only + PyObject **p; + Py_ssize_t i; + for (i = 0; i < nargs; i++) { + p = va_arg(va, PyObject **); + *p = args[i]; + } + retval = 1; + } else { + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va); + } + va_end(va); + return retval; +} + +#define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') + + +/* List of static parsers. */ +static struct CPyArg_Parser *static_arg_parsers = NULL; + +static int +parser_init(CPyArg_Parser *parser) +{ + const char * const *keywords; + const char *format, *msg; + int i, len, min, max, nkw; + PyObject *kwtuple; + + assert(parser->keywords != NULL); + if (PARSER_INITED(parser)) { + return 1; + } + + keywords = parser->keywords; + /* scan keywords and count the number of positional-only parameters */ + for (i = 0; keywords[i] && !*keywords[i]; i++) { + } + parser->pos = i; + /* scan keywords and get greatest possible nbr of args */ + for (; keywords[i]; i++) { + if (!*keywords[i]) { + PyErr_SetString(PyExc_SystemError, + "Empty keyword parameter name"); + return 0; + } + } + len = i; + + parser->required_kwonly_start = INT_MAX; + if (*parser->format == '%') { + parser->format++; + parser->varargs = 1; + } + + format = parser->format; + if (format) { + /* grab the function name or custom error msg first (mutually exclusive) */ + parser->fname = strchr(parser->format, ':'); + if (parser->fname) { + parser->fname++; + parser->custom_msg = NULL; + } + else { + parser->custom_msg = strchr(parser->format,';'); + if (parser->custom_msg) + parser->custom_msg++; + } + + min = max = INT_MAX; + for (i = 0; i < len; i++) { + if (*format == '|') { + if (min != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string (| specified twice)"); + return 0; + } + if (max != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string ($ before |)"); + return 0; + } + min = i; + format++; + } + if (*format == '$') { + if (max != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string ($ specified twice)"); + return 0; + } + if (i < parser->pos) { + PyErr_SetString(PyExc_SystemError, + "Empty parameter name after $"); + return 0; + } + max = i; + format++; + } + if (*format == '@') { + if (parser->required_kwonly_start != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string (@ specified twice)"); + return 0; + } + if (min == INT_MAX && max == INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string " + "(@ without preceding | and $)"); + return 0; + } + format++; + parser->has_required_kws = 1; + parser->required_kwonly_start = i; + } + if (IS_END_OF_FORMAT(*format)) { + PyErr_Format(PyExc_SystemError, + "More keyword list entries (%d) than " + "format specifiers (%d)", len, i); + return 0; + } + + skipitem_fast(&format, NULL); + } + parser->min = Py_MIN(min, len); + parser->max = Py_MIN(max, len); + + if (!IS_END_OF_FORMAT(*format) && (*format != '|') && (*format != '$')) { + PyErr_Format(PyExc_SystemError, + "more argument specifiers than keyword list entries " + "(remaining format:'%s')", format); + return 0; + } + } + + nkw = len - parser->pos; + kwtuple = PyTuple_New(nkw); + if (kwtuple == NULL) { + return 0; + } + keywords = parser->keywords + parser->pos; + for (i = 0; i < nkw; i++) { + PyObject *str = PyUnicode_FromString(keywords[i]); + if (str == NULL) { + Py_DECREF(kwtuple); + return 0; + } + PyUnicode_InternInPlace(&str); + PyTuple_SET_ITEM(kwtuple, i, str); + } + parser->kwtuple = kwtuple; + + assert(parser->next == NULL); + parser->next = static_arg_parsers; + static_arg_parsers = parser; + return 1; +} + +static PyObject* +find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) +{ + Py_ssize_t i, nkwargs; + + nkwargs = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < nkwargs; i++) { + PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); + + /* kwname == key will normally find a match in since keyword keys + should be interned strings; if not retry below in a new loop. */ + if (kwname == key) { + return kwstack[i]; + } + } + + for (i = 0; i < nkwargs; i++) { + PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); + assert(PyUnicode_Check(kwname)); + if (PyUnicode_Equal(kwname, key)) { + return kwstack[i]; + } + } + return NULL; +} + +static int +vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject *kwnames, + CPyArg_Parser *parser, + va_list *p_va) +{ + PyObject *kwtuple; + const char *format; + PyObject *keyword; + int i, pos, len; + Py_ssize_t nkwargs; + PyObject *current_arg; + PyObject *const *kwstack = NULL; + int bound_pos_args; + PyObject **p_args = NULL, **p_kwargs = NULL; + + assert(kwargs == NULL || PyDict_Check(kwargs)); + assert(kwargs == NULL || kwnames == NULL); + assert(p_va != NULL); + + if (!parser_init(parser)) { + return 0; + } + + kwtuple = parser->kwtuple; + pos = parser->pos; + len = pos + (int)PyTuple_GET_SIZE(kwtuple); + + if (parser->varargs) { + p_args = va_arg(*p_va, PyObject **); + p_kwargs = va_arg(*p_va, PyObject **); + } + + if (kwargs != NULL) { + nkwargs = PyDict_GET_SIZE(kwargs); + } + else if (kwnames != NULL) { + nkwargs = PyTuple_GET_SIZE(kwnames); + kwstack = args + nargs; + } + else { + nkwargs = 0; + } + if (nargs + nkwargs > len && !p_args && !p_kwargs) { + /* Adding "keyword" (when nargs == 0) prevents producing wrong error + messages in some special cases (see bpo-31229). */ + PyErr_Format(PyExc_TypeError, + "%.200s%s takes at most %d %sargument%s (%zd given)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + len, + (nargs == 0) ? "keyword " : "", + (len == 1) ? "" : "s", + nargs + nkwargs); + return 0; + } + if (parser->max < nargs && !p_args) { + if (parser->max == 0) { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes no positional arguments", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()"); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes %s %d positional argument%s (%zd given)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + (parser->min < parser->max) ? "at most" : "exactly", + parser->max, + parser->max == 1 ? "" : "s", + nargs); + } + return 0; + } + + format = parser->format; + + /* convert tuple args and keyword args in same loop, using kwtuple to drive process */ + for (i = 0; i < len; i++) { + if (*format == '|') { + format++; + } + if (*format == '$') { + format++; + } + if (*format == '@') { + format++; + } + assert(!IS_END_OF_FORMAT(*format)); + + if (i < nargs && i < parser->max) { + current_arg = args[i]; + } + else if (nkwargs && i >= pos) { + keyword = PyTuple_GET_ITEM(kwtuple, i - pos); + if (kwargs != NULL) { + current_arg = PyDict_GetItemWithError(kwargs, keyword); + if (!current_arg && PyErr_Occurred()) { + return 0; + } + } + else { + current_arg = find_keyword(kwnames, kwstack, keyword); + } + if (current_arg) { + --nkwargs; + } + } + else { + current_arg = NULL; + } + + if (current_arg) { + PyObject **p = va_arg(*p_va, PyObject **); + *p = current_arg; + format++; + continue; + } + + if (i < parser->min || i >= parser->required_kwonly_start) { + /* Less arguments than required */ + if (i < pos) { + Py_ssize_t min = Py_MIN(pos, parser->min); + PyErr_Format(PyExc_TypeError, + "%.200s%s takes %s %d positional argument%s" + " (%zd given)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + min < parser->max ? "at least" : "exactly", + min, + min == 1 ? "" : "s", + nargs); + } + else { + keyword = PyTuple_GET_ITEM(kwtuple, i - pos); + if (i >= parser->max) { + PyErr_Format(PyExc_TypeError, "%.200s%s missing required " + "keyword-only argument '%U'", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + keyword); + } + else { + PyErr_Format(PyExc_TypeError, "%.200s%s missing required " + "argument '%U' (pos %d)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + keyword, i+1); + } + } + return 0; + } + /* current code reports success when all required args + * fulfilled and no keyword args left, with no further + * validation. XXX Maybe skip this in debug build ? + */ + if (!nkwargs && !parser->has_required_kws && !p_args && !p_kwargs) { + return 1; + } + + /* We are into optional args, skip through to any remaining + * keyword args */ + skipitem_fast(&format, p_va); + } + + assert(IS_END_OF_FORMAT(*format) || (*format == '|') || (*format == '$')); + + bound_pos_args = Py_MIN(nargs, Py_MIN(parser->max, len)); + if (p_args) { + *p_args = PyTuple_New(nargs - bound_pos_args); + if (!*p_args) { + return 0; + } + for (i = bound_pos_args; i < nargs; i++) { + PyObject *arg = args[i]; + Py_INCREF(arg); + PyTuple_SET_ITEM(*p_args, i - bound_pos_args, arg); + } + } + + if (p_kwargs) { + /* This unfortunately needs to be special cased because if len is 0 then we + * never go through the main loop. */ + if (nargs > 0 && len == 0 && !p_args) { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes no positional arguments", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()"); + + return 0; + } + + *p_kwargs = PyDict_New(); + if (!*p_kwargs) { + goto latefail; + } + } + + if (nkwargs > 0) { + Py_ssize_t j; + PyObject *value; + /* make sure there are no arguments given by name and position */ + for (i = pos; i < bound_pos_args; i++) { + keyword = PyTuple_GET_ITEM(kwtuple, i - pos); + if (kwargs != NULL) { + current_arg = PyDict_GetItemWithError(kwargs, keyword); + if (!current_arg && PyErr_Occurred()) { + goto latefail; + } + } + else { + current_arg = find_keyword(kwnames, kwstack, keyword); + } + if (current_arg) { + /* arg present in tuple and in dict */ + PyErr_Format(PyExc_TypeError, + "argument for %.200s%s given by name ('%U') " + "and position (%d)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + keyword, i+1); + goto latefail; + } + } + /* make sure there are no extraneous keyword arguments */ + j = 0; + while (1) { + int match; + if (kwargs != NULL) { + if (!PyDict_Next(kwargs, &j, &keyword, &value)) + break; + } + else { + if (j >= PyTuple_GET_SIZE(kwnames)) + break; + keyword = PyTuple_GET_ITEM(kwnames, j); + value = kwstack[j]; + j++; + } + + match = PySequence_Contains(kwtuple, keyword); + if (match <= 0) { + if (!match) { + if (!p_kwargs) { + PyErr_Format(PyExc_TypeError, + "'%S' is an invalid keyword " + "argument for %.200s%s", + keyword, + (parser->fname == NULL) ? "this function" : parser->fname, + (parser->fname == NULL) ? "" : "()"); + goto latefail; + } else { + if (PyDict_SetItem(*p_kwargs, keyword, value) < 0) { + goto latefail; + } + } + } else { + goto latefail; + } + } + } + } + + return 1; + /* Handle failures that have happened after we have tried to + * create *args and **kwargs, if they exist. */ +latefail: + if (p_args) { + Py_XDECREF(*p_args); + } + if (p_kwargs) { + Py_XDECREF(*p_kwargs); + } + return 0; +} + +static void +skipitem_fast(const char **p_format, va_list *p_va) +{ + const char *format = *p_format; + char c = *format++; + + if (p_va != NULL) { + (void) va_arg(*p_va, PyObject **); + } + + *p_format = format; +} diff --git a/mypyc/lib-rt/init.c b/mypyc/lib-rt/init.c new file mode 100644 index 000000000000..01b133233489 --- /dev/null +++ b/mypyc/lib-rt/init.c @@ -0,0 +1,13 @@ +#include +#include "CPy.h" + +struct ExcDummyStruct _CPy_ExcDummyStruct = { PyObject_HEAD_INIT(NULL) }; +PyObject *_CPy_ExcDummy = (PyObject *)&_CPy_ExcDummyStruct; + +// Because its dynamic linker is more restricted than linux/OS X, +// Windows doesn't allow initializing globals with values from +// other dynamic libraries. This means we need to initialize +// things at load time. +void CPy_Init(void) { + _CPy_ExcDummyStruct.ob_base.ob_type = &PyBaseObject_Type; +} diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c new file mode 100644 index 000000000000..9b5d4ef65fb1 --- /dev/null +++ b/mypyc/lib-rt/int_ops.c @@ -0,0 +1,591 @@ +// Int primitive operations (tagged arbitrary-precision integers) +// +// These are registered in mypyc.primitives.int_ops. + +#include +#include "CPy.h" + +#ifndef _WIN32 +// On 64-bit Linux and macOS, ssize_t and long are both 64 bits, and +// PyLong_FromLong is faster than PyLong_FromSsize_t, so use the faster one +#define CPyLong_FromSsize_t PyLong_FromLong +#else +// On 64-bit Windows, ssize_t is 64 bits but long is 32 bits, so we +// can't use the above trick +#define CPyLong_FromSsize_t PyLong_FromSsize_t +#endif + +CPyTagged CPyTagged_FromSsize_t(Py_ssize_t value) { + // We use a Python object if the value shifted left by 1 is too + // large for Py_ssize_t + if (unlikely(CPyTagged_TooBig(value))) { + PyObject *object = PyLong_FromSsize_t(value); + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return value << 1; + } +} + +CPyTagged CPyTagged_FromVoidPtr(void *ptr) { + if ((uintptr_t)ptr > PY_SSIZE_T_MAX) { + PyObject *object = PyLong_FromVoidPtr(ptr); + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return CPyTagged_FromSsize_t((Py_ssize_t)ptr); + } +} + +CPyTagged CPyTagged_FromInt64(int64_t value) { + if (unlikely(CPyTagged_TooBigInt64(value))) { + PyObject *object = PyLong_FromLongLong(value); + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return value << 1; + } +} + +PyObject *CPyTagged_AsObject(CPyTagged x) { + PyObject *value; + if (unlikely(CPyTagged_CheckLong(x))) { + value = CPyTagged_LongAsObject(x); + Py_INCREF(value); + } else { + value = CPyLong_FromSsize_t(CPyTagged_ShortAsSsize_t(x)); + if (value == NULL) { + CPyError_OutOfMemory(); + } + } + return value; +} + +PyObject *CPyTagged_StealAsObject(CPyTagged x) { + PyObject *value; + if (unlikely(CPyTagged_CheckLong(x))) { + value = CPyTagged_LongAsObject(x); + } else { + value = CPyLong_FromSsize_t(CPyTagged_ShortAsSsize_t(x)); + if (value == NULL) { + CPyError_OutOfMemory(); + } + } + return value; +} + +Py_ssize_t CPyTagged_AsSsize_t(CPyTagged x) { + if (likely(CPyTagged_CheckShort(x))) { + return CPyTagged_ShortAsSsize_t(x); + } else { + return PyLong_AsSsize_t(CPyTagged_LongAsObject(x)); + } +} + +CPy_NOINLINE +void CPyTagged_IncRef(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + Py_INCREF(CPyTagged_LongAsObject(x)); + } +} + +CPy_NOINLINE +void CPyTagged_DecRef(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + Py_DECREF(CPyTagged_LongAsObject(x)); + } +} + +CPy_NOINLINE +void CPyTagged_XDecRef(CPyTagged x) { + if (unlikely(CPyTagged_CheckLong(x))) { + Py_XDECREF(CPyTagged_LongAsObject(x)); + } +} + +// Tagged int negation slow path, where the result may be a long integer +CPyTagged CPyTagged_Negate_(CPyTagged num) { + PyObject *num_obj = CPyTagged_AsObject(num); + PyObject *result = PyNumber_Negative(num_obj); + if (result == NULL) { + CPyError_OutOfMemory(); + } + Py_DECREF(num_obj); + return CPyTagged_StealFromObject(result); +} + +// Tagged int addition slow path, where the result may be a long integer +CPyTagged CPyTagged_Add_(CPyTagged left, CPyTagged right) { + PyObject *left_obj = CPyTagged_AsObject(left); + PyObject *right_obj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_Add(left_obj, right_obj); + if (result == NULL) { + CPyError_OutOfMemory(); + } + Py_DECREF(left_obj); + Py_DECREF(right_obj); + return CPyTagged_StealFromObject(result); +} + +// Tagged int subraction slow path, where the result may be a long integer +CPyTagged CPyTagged_Subtract_(CPyTagged left, CPyTagged right) { + PyObject *left_obj = CPyTagged_AsObject(left); + PyObject *right_obj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_Subtract(left_obj, right_obj); + if (result == NULL) { + CPyError_OutOfMemory(); + } + Py_DECREF(left_obj); + Py_DECREF(right_obj); + return CPyTagged_StealFromObject(result); +} + +// Tagged int multiplication slow path, where the result may be a long integer +CPyTagged CPyTagged_Multiply_(CPyTagged left, CPyTagged right) { + PyObject *left_obj = CPyTagged_AsObject(left); + PyObject *right_obj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_Multiply(left_obj, right_obj); + if (result == NULL) { + CPyError_OutOfMemory(); + } + Py_DECREF(left_obj); + Py_DECREF(right_obj); + return CPyTagged_StealFromObject(result); +} + +// Tagged int // slow path, where the result may be a long integer (or raise) +CPyTagged CPyTagged_FloorDivide_(CPyTagged left, CPyTagged right) { + PyObject *left_obj = CPyTagged_AsObject(left); + PyObject *right_obj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_FloorDivide(left_obj, right_obj); + Py_DECREF(left_obj); + Py_DECREF(right_obj); + // Handle exceptions honestly because it could be ZeroDivisionError + if (result == NULL) { + return CPY_INT_TAG; + } else { + return CPyTagged_StealFromObject(result); + } +} + +// Tagged int % slow path, where the result may be a long integer (or raise) +CPyTagged CPyTagged_Remainder_(CPyTagged left, CPyTagged right) { + PyObject *left_obj = CPyTagged_AsObject(left); + PyObject *right_obj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_Remainder(left_obj, right_obj); + Py_DECREF(left_obj); + Py_DECREF(right_obj); + // Handle exceptions honestly because it could be ZeroDivisionError + if (result == NULL) { + return CPY_INT_TAG; + } else { + return CPyTagged_StealFromObject(result); + } +} + +bool CPyTagged_IsEq_(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(right)) { + return false; + } else { + PyObject *left_obj = CPyTagged_AsObject(left); + PyObject *right_obj = CPyTagged_AsObject(right); + int result = PyObject_RichCompareBool(left_obj, right_obj, Py_EQ); + Py_DECREF(left_obj); + Py_DECREF(right_obj); + if (result == -1) { + CPyError_OutOfMemory(); + } + return result; + } +} + +bool CPyTagged_IsLt_(CPyTagged left, CPyTagged right) { + PyObject *left_obj = CPyTagged_AsObject(left); + PyObject *right_obj = CPyTagged_AsObject(right); + int result = PyObject_RichCompareBool(left_obj, right_obj, Py_LT); + Py_DECREF(left_obj); + Py_DECREF(right_obj); + if (result == -1) { + CPyError_OutOfMemory(); + } + return result; +} + +PyObject *CPyLong_FromStrWithBase(PyObject *o, CPyTagged base) { + Py_ssize_t base_size_t = CPyTagged_AsSsize_t(base); + return PyLong_FromUnicodeObject(o, base_size_t); +} + +PyObject *CPyLong_FromStr(PyObject *o) { + CPyTagged base = CPyTagged_FromSsize_t(10); + return CPyLong_FromStrWithBase(o, base); +} + +CPyTagged CPyTagged_FromFloat(double f) { + if (f < ((double)CPY_TAGGED_MAX + 1.0) && f > (CPY_TAGGED_MIN - 1.0)) { + return (Py_ssize_t)f << 1; + } + PyObject *o = PyLong_FromDouble(f); + if (o == NULL) + return CPY_INT_TAG; + return CPyTagged_StealFromObject(o); +} + +PyObject *CPyBool_Str(bool b) { + return PyObject_Str(b ? Py_True : Py_False); +} + +static void CPyLong_NormalizeUnsigned(PyLongObject *v) { + Py_ssize_t i = CPY_LONG_SIZE_UNSIGNED(v); + while (i > 0 && CPY_LONG_DIGIT(v, i - 1) == 0) + i--; + CPyLong_SetUnsignedSize(v, i); +} + +// Bitwise op '&', '|' or '^' using the generic (slow) API +static CPyTagged GenericBitwiseOp(CPyTagged a, CPyTagged b, char op) { + PyObject *aobj = CPyTagged_AsObject(a); + PyObject *bobj = CPyTagged_AsObject(b); + PyObject *r; + if (op == '&') { + r = PyNumber_And(aobj, bobj); + } else if (op == '|') { + r = PyNumber_Or(aobj, bobj); + } else { + r = PyNumber_Xor(aobj, bobj); + } + if (unlikely(r == NULL)) { + CPyError_OutOfMemory(); + } + Py_DECREF(aobj); + Py_DECREF(bobj); + return CPyTagged_StealFromObject(r); +} + +// Return pointer to digits of a PyLong object. If it's a short +// integer, place digits in the buffer buf instead to avoid memory +// allocation (it's assumed to be big enough). Return the number of +// digits in *size. *size is negative if the integer is negative. +static digit *GetIntDigits(CPyTagged n, Py_ssize_t *size, digit *buf) { + if (CPyTagged_CheckShort(n)) { + Py_ssize_t val = CPyTagged_ShortAsSsize_t(n); + bool neg = val < 0; + int len = 1; + if (neg) { + val = -val; + } + buf[0] = val & PyLong_MASK; + if (val > (Py_ssize_t)PyLong_MASK) { + val >>= PyLong_SHIFT; + buf[1] = val & PyLong_MASK; + if (val > (Py_ssize_t)PyLong_MASK) { + buf[2] = val >> PyLong_SHIFT; + len = 3; + } else { + len = 2; + } + } + *size = neg ? -len : len; + return buf; + } else { + PyLongObject *obj = (PyLongObject *)CPyTagged_LongAsObject(n); + *size = CPY_LONG_SIZE_SIGNED(obj); + return &CPY_LONG_DIGIT(obj, 0); + } +} + +// Shared implementation of bitwise '&', '|' and '^' (specified by op) for at least +// one long operand. This is somewhat optimized for performance. +CPyTagged CPyTagged_BitwiseLongOp_(CPyTagged a, CPyTagged b, char op) { + // Directly access the digits, as there is no fast C API function for this. + digit abuf[3]; + digit bbuf[3]; + Py_ssize_t asize; + Py_ssize_t bsize; + digit *adigits = GetIntDigits(a, &asize, abuf); + digit *bdigits = GetIntDigits(b, &bsize, bbuf); + + PyLongObject *r; + if (unlikely(asize < 0 || bsize < 0)) { + // Negative operand. This is slower, but bitwise ops on them are pretty rare. + return GenericBitwiseOp(a, b, op); + } + // Optimized implementation for two non-negative integers. + // Swap a and b as needed to ensure a is no longer than b. + if (asize > bsize) { + digit *tmp = adigits; + adigits = bdigits; + bdigits = tmp; + Py_ssize_t tmp_size = asize; + asize = bsize; + bsize = tmp_size; + } + r = _PyLong_New(op == '&' ? asize : bsize); + if (unlikely(r == NULL)) { + CPyError_OutOfMemory(); + } + Py_ssize_t i; + if (op == '&') { + for (i = 0; i < asize; i++) { + CPY_LONG_DIGIT(r, i) = adigits[i] & bdigits[i]; + } + } else { + if (op == '|') { + for (i = 0; i < asize; i++) { + CPY_LONG_DIGIT(r, i) = adigits[i] | bdigits[i]; + } + } else { + for (i = 0; i < asize; i++) { + CPY_LONG_DIGIT(r, i) = adigits[i] ^ bdigits[i]; + } + } + for (; i < bsize; i++) { + CPY_LONG_DIGIT(r, i) = bdigits[i]; + } + } + CPyLong_NormalizeUnsigned(r); + return CPyTagged_StealFromObject((PyObject *)r); +} + +// Bitwise '~' slow path +CPyTagged CPyTagged_Invert_(CPyTagged num) { + PyObject *obj = CPyTagged_AsObject(num); + PyObject *result = PyNumber_Invert(obj); + if (unlikely(result == NULL)) { + CPyError_OutOfMemory(); + } + Py_DECREF(obj); + return CPyTagged_StealFromObject(result); +} + +// Bitwise '>>' slow path +CPyTagged CPyTagged_Rshift_(CPyTagged left, CPyTagged right) { + // Long integer or negative shift -- use generic op + PyObject *lobj = CPyTagged_AsObject(left); + PyObject *robj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_Rshift(lobj, robj); + Py_DECREF(lobj); + Py_DECREF(robj); + if (result == NULL) { + // Propagate error (could be negative shift count) + return CPY_INT_TAG; + } + return CPyTagged_StealFromObject(result); +} + +// Bitwise '<<' slow path +CPyTagged CPyTagged_Lshift_(CPyTagged left, CPyTagged right) { + // Long integer or out of range shift -- use generic op + PyObject *lobj = CPyTagged_AsObject(left); + PyObject *robj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_Lshift(lobj, robj); + Py_DECREF(lobj); + Py_DECREF(robj); + if (result == NULL) { + // Propagate error (could be negative shift count) + return CPY_INT_TAG; + } + return CPyTagged_StealFromObject(result); +} + +// i64 unboxing slow path +int64_t CPyLong_AsInt64_(PyObject *o) { + int overflow; + int64_t result = PyLong_AsLongLongAndOverflow(o, &overflow); + if (result == -1) { + if (PyErr_Occurred()) { + return CPY_LL_INT_ERROR; + } else if (overflow) { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i64"); + return CPY_LL_INT_ERROR; + } + } + return result; +} + +int64_t CPyInt64_Divide(int64_t x, int64_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + if (y == -1 && x == INT64_MIN) { + PyErr_SetString(PyExc_OverflowError, "integer division overflow"); + return CPY_LL_INT_ERROR; + } + int64_t d = x / y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d * y != x) { + d--; + } + return d; +} + +int64_t CPyInt64_Remainder(int64_t x, int64_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + // Edge case: avoid core dump + if (y == -1 && x == INT64_MIN) { + return 0; + } + int64_t d = x % y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d != 0) { + d += y; + } + return d; +} + +// i32 unboxing slow path +int32_t CPyLong_AsInt32_(PyObject *o) { + int overflow; + long result = PyLong_AsLongAndOverflow(o, &overflow); + if (result > 0x7fffffffLL || result < -0x80000000LL) { + overflow = 1; + result = -1; + } + if (result == -1) { + if (PyErr_Occurred()) { + return CPY_LL_INT_ERROR; + } else if (overflow) { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i32"); + return CPY_LL_INT_ERROR; + } + } + return result; +} + +int32_t CPyInt32_Divide(int32_t x, int32_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + if (y == -1 && x == INT32_MIN) { + PyErr_SetString(PyExc_OverflowError, "integer division overflow"); + return CPY_LL_INT_ERROR; + } + int32_t d = x / y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d * y != x) { + d--; + } + return d; +} + +int32_t CPyInt32_Remainder(int32_t x, int32_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + // Edge case: avoid core dump + if (y == -1 && x == INT32_MIN) { + return 0; + } + int32_t d = x % y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d != 0) { + d += y; + } + return d; +} + +void CPyInt32_Overflow() { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i32"); +} + +// i16 unboxing slow path +int16_t CPyLong_AsInt16_(PyObject *o) { + int overflow; + long result = PyLong_AsLongAndOverflow(o, &overflow); + if (result > 0x7fff || result < -0x8000) { + overflow = 1; + result = -1; + } + if (result == -1) { + if (PyErr_Occurred()) { + return CPY_LL_INT_ERROR; + } else if (overflow) { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i16"); + return CPY_LL_INT_ERROR; + } + } + return result; +} + +int16_t CPyInt16_Divide(int16_t x, int16_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + if (y == -1 && x == INT16_MIN) { + PyErr_SetString(PyExc_OverflowError, "integer division overflow"); + return CPY_LL_INT_ERROR; + } + int16_t d = x / y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d * y != x) { + d--; + } + return d; +} + +int16_t CPyInt16_Remainder(int16_t x, int16_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + // Edge case: avoid core dump + if (y == -1 && x == INT16_MIN) { + return 0; + } + int16_t d = x % y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d != 0) { + d += y; + } + return d; +} + +void CPyInt16_Overflow() { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i16"); +} + +// u8 unboxing slow path +uint8_t CPyLong_AsUInt8_(PyObject *o) { + int overflow; + long result = PyLong_AsLongAndOverflow(o, &overflow); + if (result < 0 || result >= 256) { + overflow = 1; + result = -1; + } + if (result == -1) { + if (PyErr_Occurred()) { + return CPY_LL_UINT_ERROR; + } else if (overflow) { + PyErr_SetString(PyExc_OverflowError, "int too large or small to convert to u8"); + return CPY_LL_UINT_ERROR; + } + } + return result; +} + +void CPyUInt8_Overflow() { + PyErr_SetString(PyExc_OverflowError, "int too large or small to convert to u8"); +} + +double CPyTagged_TrueDivide(CPyTagged x, CPyTagged y) { + if (unlikely(y == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); + return CPY_FLOAT_ERROR; + } + if (likely(!CPyTagged_CheckLong(x) && !CPyTagged_CheckLong(y))) { + return (double)((Py_ssize_t)x >> 1) / (double)((Py_ssize_t)y >> 1); + } else { + PyObject *xo = CPyTagged_AsObject(x); + PyObject *yo = CPyTagged_AsObject(y); + PyObject *result = PyNumber_TrueDivide(xo, yo); + if (result == NULL) { + return CPY_FLOAT_ERROR; + } + return PyFloat_AsDouble(result); + } + return 1.0; +} diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c new file mode 100644 index 000000000000..d297ece8f417 --- /dev/null +++ b/mypyc/lib-rt/list_ops.c @@ -0,0 +1,338 @@ +// List primitive operations +// +// These are registered in mypyc.primitives.list_ops. + +#include +#include "CPy.h" + +#ifndef Py_TPFLAGS_SEQUENCE +#define Py_TPFLAGS_SEQUENCE (1 << 5) +#endif + +PyObject *CPyList_Build(Py_ssize_t len, ...) { + Py_ssize_t i; + + PyObject *res = PyList_New(len); + if (res == NULL) { + return NULL; + } + + va_list args; + va_start(args, len); + for (i = 0; i < len; i++) { + // Steals the reference + PyObject *value = va_arg(args, PyObject *); + PyList_SET_ITEM(res, i, value); + } + va_end(args); + + return res; +} + +PyObject *CPyList_GetItemUnsafe(PyObject *list, CPyTagged index) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + PyObject *result = PyList_GET_ITEM(list, n); + Py_INCREF(result); + return result; +} + +PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + PyObject *result = PyList_GET_ITEM(list, n); + Py_INCREF(result); + return result; +} + +PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + return PyList_GET_ITEM(list, n); +} + +PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + PyObject *result = PyList_GET_ITEM(list, n); + Py_INCREF(result); + return result; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + return PyList_GET_ITEM(list, n); + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { + size_t size = PyList_GET_SIZE(list); + if (likely((uint64_t)index < size)) { + PyObject *result = PyList_GET_ITEM(list, index); + Py_INCREF(result); + return result; + } + if (index >= 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + PyObject *result = PyList_GET_ITEM(list, index); + Py_INCREF(result); + return result; +} + +PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) { + size_t size = PyList_GET_SIZE(list); + if (likely((uint64_t)index < size)) { + return PyList_GET_ITEM(list, index); + } + if (index >= 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + return PyList_GET_ITEM(list, index); +} + +bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return false; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return false; + } + } + // PyList_SET_ITEM doesn't decref the old element, so we do + Py_DECREF(PyList_GET_ITEM(list, n)); + // N.B: Steals reference + PyList_SET_ITEM(list, n, value); + return true; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return false; + } +} + +bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) { + size_t size = PyList_GET_SIZE(list); + if (unlikely((uint64_t)index >= size)) { + if (index > 0) { + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return false; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return false; + } + } + // PyList_SET_ITEM doesn't decref the old element, so we do + Py_DECREF(PyList_GET_ITEM(list, index)); + // N.B: Steals reference + PyList_SET_ITEM(list, index, value); + return true; +} + +// This function should only be used to fill in brand new lists. +bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + PyList_SET_ITEM(list, n, value); + return true; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return false; + } +} + +PyObject *CPyList_PopLast(PyObject *obj) +{ + // I tried a specalized version of pop_impl for just removing the + // last element and it wasn't any faster in microbenchmarks than + // the generic one so I ditched it. + return list_pop_impl((PyListObject *)obj, -1); +} + +PyObject *CPyList_Pop(PyObject *obj, CPyTagged index) +{ + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + return list_pop_impl((PyListObject *)obj, n); + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +CPyTagged CPyList_Count(PyObject *obj, PyObject *value) +{ + return list_count((PyListObject *)obj, value); +} + +int CPyList_Insert(PyObject *list, CPyTagged index, PyObject *value) +{ + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + return PyList_Insert(list, n, value); + } + // The max range doesn't exactly coincide with ssize_t, but we still + // want to keep the error message compatible with CPython. + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return -1; +} + +PyObject *CPyList_Extend(PyObject *o1, PyObject *o2) { + if (PyList_Extend(o1, o2) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +// Return -2 or error, -1 if not found, or index of first match otherwise. +static Py_ssize_t _CPyList_Find(PyObject *list, PyObject *obj) { + Py_ssize_t i; + for (i = 0; i < Py_SIZE(list); i++) { + PyObject *item = PyList_GET_ITEM(list, i); + Py_INCREF(item); + int cmp = PyObject_RichCompareBool(item, obj, Py_EQ); + Py_DECREF(item); + if (cmp != 0) { + if (cmp > 0) { + return i; + } else { + return -2; + } + } + } + return -1; +} + +int CPyList_Remove(PyObject *list, PyObject *obj) { + Py_ssize_t index = _CPyList_Find(list, obj); + if (index == -2) { + return -1; + } + if (index == -1) { + PyErr_SetString(PyExc_ValueError, "list.remove(x): x not in list"); + return -1; + } + return PyList_SetSlice(list, index, index + 1, NULL); +} + +CPyTagged CPyList_Index(PyObject *list, PyObject *obj) { + Py_ssize_t index = _CPyList_Find(list, obj); + if (index == -2) { + return CPY_INT_TAG; + } + if (index == -1) { + PyErr_SetString(PyExc_ValueError, "value is not in list"); + return CPY_INT_TAG; + } + return index << 1; +} + +PyObject *CPySequence_Multiply(PyObject *seq, CPyTagged t_size) { + Py_ssize_t size = CPyTagged_AsSsize_t(t_size); + if (size == -1 && PyErr_Occurred()) { + return NULL; + } + return PySequence_Repeat(seq, size); +} + +PyObject *CPySequence_RMultiply(CPyTagged t_size, PyObject *seq) { + return CPySequence_Multiply(seq, t_size); +} + +PyObject *CPyList_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { + if (likely(PyList_CheckExact(obj) + && CPyTagged_CheckShort(start) && CPyTagged_CheckShort(end))) { + Py_ssize_t startn = CPyTagged_ShortAsSsize_t(start); + Py_ssize_t endn = CPyTagged_ShortAsSsize_t(end); + if (startn < 0) { + startn += PyList_GET_SIZE(obj); + } + if (endn < 0) { + endn += PyList_GET_SIZE(obj); + } + return PyList_GetSlice(obj, startn, endn); + } + return CPyObject_GetSlice(obj, start, end); +} + +int CPySequence_Check(PyObject *obj) { + return Py_TYPE(obj)->tp_flags & Py_TPFLAGS_SEQUENCE; +} diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c new file mode 100644 index 000000000000..d3e8e69ed19b --- /dev/null +++ b/mypyc/lib-rt/misc_ops.c @@ -0,0 +1,1019 @@ +// Misc primitive operations + C helpers +// +// These are registered in mypyc.primitives.misc_ops. + +#include +#include +#include "CPy.h" + +PyObject *CPy_GetCoro(PyObject *obj) +{ + // If the type has an __await__ method, call it, + // otherwise, fallback to calling __iter__. + PyAsyncMethods* async_struct = Py_TYPE(obj)->tp_as_async; + if (async_struct != NULL && async_struct->am_await != NULL) { + return (async_struct->am_await)(obj); + } else { + // TODO: We should check that the type is a generator decorated with + // asyncio.coroutine + return PyObject_GetIter(obj); + } +} + +PyObject *CPyIter_Send(PyObject *iter, PyObject *val) +{ + // Do a send, or a next if second arg is None. + // (This behavior is to match the PEP 380 spec for yield from.) + _Py_IDENTIFIER(send); + if (Py_IsNone(val)) { + return CPyIter_Next(iter); + } else { + return _PyObject_CallMethodIdOneArg(iter, &PyId_send, val); + } +} + +// A somewhat hairy implementation of specifically most of the error handling +// in `yield from` error handling. The point here is to reduce code size. +// +// This implements most of the bodies of the `except` blocks in the +// pseudocode in PEP 380. +// +// Returns true (1) if a StopIteration was received and we should return. +// Returns false (0) if a value should be yielded. +// In both cases the value is stored in outp. +// Signals an error (2) if the an exception should be propagated. +int CPy_YieldFromErrorHandle(PyObject *iter, PyObject **outp) +{ + _Py_IDENTIFIER(close); + _Py_IDENTIFIER(throw); + PyObject *exc_type = (PyObject *)Py_TYPE(CPy_ExcState()->exc_value); + PyObject *type, *value, *traceback; + PyObject *_m; + PyObject *res; + *outp = NULL; + + if (PyErr_GivenExceptionMatches(exc_type, PyExc_GeneratorExit)) { + _m = _PyObject_GetAttrId(iter, &PyId_close); + if (_m) { + res = PyObject_CallNoArgs(_m); + Py_DECREF(_m); + if (!res) + return 2; + Py_DECREF(res); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + return 2; + } + } else { + _m = _PyObject_GetAttrId(iter, &PyId_throw); + if (_m) { + _CPy_GetExcInfo(&type, &value, &traceback); + res = PyObject_CallFunctionObjArgs(_m, type, value, traceback, NULL); + Py_DECREF(type); + Py_DECREF(value); + Py_DECREF(traceback); + Py_DECREF(_m); + if (res) { + *outp = res; + return 0; + } else { + res = CPy_FetchStopIterationValue(); + if (res) { + *outp = res; + return 1; + } + } + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + return 2; + } + } + + CPy_Reraise(); + return 2; +} + +PyObject *CPy_FetchStopIterationValue(void) +{ + PyObject *val = NULL; + _PyGen_FetchStopIterationValue(&val); + return val; +} + +static bool _CPy_IsSafeMetaClass(PyTypeObject *metaclass) { + // mypyc classes can't work with metaclasses in + // general. Through some various nasty hacks we *do* + // manage to work with TypingMeta and its friends. + if (metaclass == &PyType_Type) + return true; + PyObject *module = PyObject_GetAttrString((PyObject *)metaclass, "__module__"); + if (!module) { + PyErr_Clear(); + return false; + } + + bool matches = false; + if (PyUnicode_CompareWithASCIIString(module, "typing") == 0 && + (strcmp(metaclass->tp_name, "TypingMeta") == 0 + || strcmp(metaclass->tp_name, "GenericMeta") == 0 + || strcmp(metaclass->tp_name, "_ProtocolMeta") == 0)) { + matches = true; + } else if (PyUnicode_CompareWithASCIIString(module, "typing_extensions") == 0 && + strcmp(metaclass->tp_name, "_ProtocolMeta") == 0) { + matches = true; + } else if (PyUnicode_CompareWithASCIIString(module, "abc") == 0 && + strcmp(metaclass->tp_name, "ABCMeta") == 0) { + matches = true; + } + Py_DECREF(module); + return matches; +} + +#if CPY_3_13_FEATURES + +// Adapted from CPython 3.13.0b3 +/* Determine the most derived metatype. */ +PyObject *CPy_CalculateMetaclass(PyObject *metatype, PyObject *bases) +{ + Py_ssize_t i, nbases; + PyTypeObject *winner; + PyObject *tmp; + PyTypeObject *tmptype; + + /* Determine the proper metatype to deal with this, + and check for metatype conflicts while we're at it. + Note that if some other metatype wins to contract, + it's possible that its instances are not types. */ + + nbases = PyTuple_GET_SIZE(bases); + winner = (PyTypeObject *)metatype; + for (i = 0; i < nbases; i++) { + tmp = PyTuple_GET_ITEM(bases, i); + tmptype = Py_TYPE(tmp); + if (PyType_IsSubtype(winner, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, winner)) { + winner = tmptype; + continue; + } + /* else: */ + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + return (PyObject *)winner; +} + +#else + +PyObject *CPy_CalculateMetaclass(PyObject *metatype, PyObject *bases) { + return (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)metatype, bases); +} + +#endif + +// Create a heap type based on a template non-heap type. +// This is super hacky and maybe we should suck it up and use PyType_FromSpec instead. +// We allow bases to be NULL to represent just inheriting from object. +// We don't support NULL bases and a non-type metaclass. +PyObject *CPyType_FromTemplate(PyObject *template, + PyObject *orig_bases, + PyObject *modname) { + PyTypeObject *template_ = (PyTypeObject *)template; + PyHeapTypeObject *t = NULL; + PyTypeObject *dummy_class = NULL; + PyObject *name = NULL; + PyObject *bases = NULL; + PyObject *slots; + + // If the type of the class (the metaclass) is NULL, we default it + // to being type. (This allows us to avoid needing to initialize + // it explicitly on windows.) + if (!Py_TYPE(template_)) { + Py_SET_TYPE(template_, &PyType_Type); + } + PyTypeObject *metaclass = Py_TYPE(template_); + + if (orig_bases) { + bases = update_bases(orig_bases); + // update_bases doesn't increment the refcount if nothing changes, + // so we do it to make sure we have distinct "references" to both + if (bases == orig_bases) + Py_INCREF(bases); + + // Find the appropriate metaclass from our base classes. We + // care about this because Generic uses a metaclass prior to + // Python 3.7. + metaclass = (PyTypeObject *)CPy_CalculateMetaclass((PyObject *)metaclass, bases); + if (!metaclass) + goto error; + + if (!_CPy_IsSafeMetaClass(metaclass)) { + PyErr_SetString(PyExc_TypeError, "mypyc classes can't have a metaclass"); + goto error; + } + } + + name = PyUnicode_FromString(template_->tp_name); + if (!name) + goto error; + + // Allocate the type and then copy the main stuff in. + t = (PyHeapTypeObject*)PyType_GenericAlloc(&PyType_Type, 0); + if (!t) + goto error; + memcpy((char *)t + sizeof(PyVarObject), + (char *)template_ + sizeof(PyVarObject), + sizeof(PyTypeObject) - sizeof(PyVarObject)); + + if (bases != orig_bases) { + if (PyObject_SetAttrString((PyObject *)t, "__orig_bases__", orig_bases) < 0) + goto error; + } + + // Having tp_base set is I think required for stuff to get + // inherited in PyType_Ready, which we needed for subclassing + // BaseException. XXX: Taking the first element is wrong I think though. + if (bases) { + t->ht_type.tp_base = (PyTypeObject *)PyTuple_GET_ITEM(bases, 0); + Py_INCREF((PyObject *)t->ht_type.tp_base); + } + + t->ht_name = name; + Py_INCREF(name); + t->ht_qualname = name; + t->ht_type.tp_bases = bases; + // references stolen so NULL these out + bases = name = NULL; + + if (PyType_Ready((PyTypeObject *)t) < 0) + goto error; + + assert(t->ht_type.tp_base != NULL); + + // XXX: This is a terrible hack to work around a cpython check on + // the mro. It was needed for mypy.stats. I need to investigate + // what is actually going on here. + Py_INCREF(metaclass); + Py_SET_TYPE(t, metaclass); + + if (dummy_class) { + if (PyDict_Merge(t->ht_type.tp_dict, dummy_class->tp_dict, 0) != 0) + goto error; + // This is the *really* tasteless bit. GenericMeta's __new__ + // in certain versions of typing sets _gorg to point back to + // the class. We need to override it to keep it from pointing + // to the proxy. + if (PyDict_SetItemString(t->ht_type.tp_dict, "_gorg", (PyObject *)t) < 0) + goto error; + } + + // Reject anything that would give us a nontrivial __slots__, + // because the layout will conflict + slots = PyObject_GetAttrString((PyObject *)t, "__slots__"); + if (slots) { + // don't fail on an empty __slots__ + int is_true = PyObject_IsTrue(slots); + Py_DECREF(slots); + if (is_true > 0) + PyErr_SetString(PyExc_TypeError, "mypyc classes can't have __slots__"); + if (is_true != 0) + goto error; + } else { + PyErr_Clear(); + } + + if (PyObject_SetAttrString((PyObject *)t, "__module__", modname) < 0) + goto error; + + if (init_subclass((PyTypeObject *)t, NULL)) + goto error; + + Py_XDECREF(dummy_class); + +#if PY_MINOR_VERSION == 11 + // This is a hack. Python 3.11 doesn't include good public APIs to work with managed + // dicts, which are the default for heap types. So we try to opt-out until Python 3.12. + t->ht_type.tp_flags &= ~Py_TPFLAGS_MANAGED_DICT; +#endif + return (PyObject *)t; + +error: + Py_XDECREF(t); + Py_XDECREF(bases); + Py_XDECREF(dummy_class); + Py_XDECREF(name); + return NULL; +} + +static int _CPy_UpdateObjFromDict(PyObject *obj, PyObject *dict) +{ + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(dict, &pos, &key, &value)) { + if (PyObject_SetAttr(obj, key, value) != 0) { + return -1; + } + } + return 0; +} + +/* Support for our partial built-in support for dataclasses. + * + * Take a class we want to make a dataclass, remove any descriptors + * for annotated attributes, swap in the actual values of the class + * variables invoke dataclass, and then restore all of the + * descriptors. + * + * The purpose of all this is that dataclasses uses the values of + * class variables to drive which attributes are required and what the + * default values/factories are for optional attributes. This means + * that the class dict needs to contain those values instead of getset + * descriptors for the attributes when we invoke dataclass. + * + * We need to remove descriptors for attributes even when there is no + * default value for them, or else dataclass will think the descriptor + * is the default value. We remove only the attributes, since we don't + * want dataclasses to try generating functions when they are already + * implemented. + * + * Args: + * dataclass_dec: The decorator to apply + * tp: The class we are making a dataclass + * dict: The dictionary containing values that dataclasses needs + * annotations: The type annotation dictionary + */ +int +CPyDataclass_SleightOfHand(PyObject *dataclass_dec, PyObject *tp, + PyObject *dict, PyObject *annotations) { + PyTypeObject *ttp = (PyTypeObject *)tp; + Py_ssize_t pos; + PyObject *res; + + /* Make a copy of the original class __dict__ */ + PyObject *orig_dict = PyDict_Copy(ttp->tp_dict); + if (!orig_dict) { + goto fail; + } + + /* Delete anything that had an annotation */ + pos = 0; + PyObject *key; + while (PyDict_Next(annotations, &pos, &key, NULL)) { + if (PyObject_DelAttr(tp, key) != 0) { + goto fail; + } + } + + /* Copy in all the attributes that we want dataclass to see */ + if (_CPy_UpdateObjFromDict(tp, dict) != 0) { + goto fail; + } + + /* Run the @dataclass descriptor */ + res = PyObject_CallOneArg(dataclass_dec, tp); + if (!res) { + goto fail; + } + Py_DECREF(res); + + /* Copy back the original contents of the dict */ + if (_CPy_UpdateObjFromDict(tp, orig_dict) != 0) { + goto fail; + } + + Py_DECREF(orig_dict); + return 1; + +fail: + Py_XDECREF(orig_dict); + return 0; +} + +// Support for pickling; reusable getstate and setstate functions +PyObject * +CPyPickle_SetState(PyObject *obj, PyObject *state) +{ + if (_CPy_UpdateObjFromDict(obj, state) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyObject * +CPyPickle_GetState(PyObject *obj) +{ + PyObject *attrs = NULL, *state = NULL; + + attrs = PyObject_GetAttrString((PyObject *)Py_TYPE(obj), "__mypyc_attrs__"); + if (!attrs) { + goto fail; + } + if (!PyTuple_Check(attrs)) { + PyErr_SetString(PyExc_TypeError, "__mypyc_attrs__ is not a tuple"); + goto fail; + } + state = PyDict_New(); + if (!state) { + goto fail; + } + + // Collect all the values of attributes in __mypyc_attrs__ + // Attributes that are missing we just ignore + int i; + for (i = 0; i < PyTuple_GET_SIZE(attrs); i++) { + PyObject *key = PyTuple_GET_ITEM(attrs, i); + PyObject *value = PyObject_GetAttr(obj, key); + if (!value) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + continue; + } + goto fail; + } + int result = PyDict_SetItem(state, key, value); + Py_DECREF(value); + if (result != 0) { + goto fail; + } + } + + Py_DECREF(attrs); + + return state; +fail: + Py_XDECREF(attrs); + Py_XDECREF(state); + return NULL; +} + +CPyTagged CPyTagged_Id(PyObject *o) { + return CPyTagged_FromVoidPtr(o); +} + +#define MAX_INT_CHARS 22 +#define _PyUnicode_LENGTH(op) \ + (((PyASCIIObject *)(op))->length) + +// using snprintf or PyUnicode_FromFormat was way slower than +// boxing the int and calling PyObject_Str on it, so we implement our own +static int fmt_ssize_t(char *out, Py_ssize_t n) { + bool neg = n < 0; + if (neg) n = -n; + + // buf gets filled backward and then we copy it forward + char buf[MAX_INT_CHARS]; + int i = 0; + do { + buf[i] = (n % 10) + '0'; + n /= 10; + i++; + } while (n); + + + int len = i; + int j = 0; + if (neg) { + out[j++] = '-'; + len++; + } + + for (; j < len; j++, i--) { + out[j] = buf[i-1]; + } + out[j] = '\0'; + + return len; +} + +static PyObject *CPyTagged_ShortToStr(Py_ssize_t n) { + PyObject *obj = PyUnicode_New(MAX_INT_CHARS, 127); + if (!obj) return NULL; + int len = fmt_ssize_t((char *)PyUnicode_1BYTE_DATA(obj), n); + _PyUnicode_LENGTH(obj) = len; + return obj; +} + +PyObject *CPyTagged_Str(CPyTagged n) { + if (CPyTagged_CheckShort(n)) { + return CPyTagged_ShortToStr(CPyTagged_ShortAsSsize_t(n)); + } else { + return PyObject_Str(CPyTagged_AsObject(n)); + } +} + +void CPyDebug_Print(const char *msg) { + printf("%s\n", msg); + fflush(stdout); +} + +int CPySequence_CheckUnpackCount(PyObject *sequence, Py_ssize_t expected) { + Py_ssize_t actual = Py_SIZE(sequence); + if (unlikely(actual != expected)) { + if (actual < expected) { + PyErr_Format(PyExc_ValueError, "not enough values to unpack (expected %zd, got %zd)", + expected, actual); + } else { + PyErr_Format(PyExc_ValueError, "too many values to unpack (expected %zd)", expected); + } + return -1; + } + return 0; +} + +// Parse an integer (size_t) encoded as a variable-length binary sequence. +static const char *parse_int(const char *s, size_t *len) { + Py_ssize_t n = 0; + while ((unsigned char)*s >= 0x80) { + n = (n << 7) + (*s & 0x7f); + s++; + } + n = (n << 7) | *s++; + *len = n; + return s; +} + +// Initialize static constant array of literal values +int CPyStatics_Initialize(PyObject **statics, + const char * const *strings, + const char * const *bytestrings, + const char * const *ints, + const double *floats, + const double *complex_numbers, + const int *tuples, + const int *frozensets) { + PyObject **result = statics; + // Start with some hard-coded values + *result++ = Py_None; + Py_INCREF(Py_None); + *result++ = Py_False; + Py_INCREF(Py_False); + *result++ = Py_True; + Py_INCREF(Py_True); + if (strings) { + for (; **strings != '\0'; strings++) { + size_t num; + const char *data = *strings; + data = parse_int(data, &num); + while (num-- > 0) { + size_t len; + data = parse_int(data, &len); + PyObject *obj = PyUnicode_DecodeUTF8(data, len, "surrogatepass"); + if (obj == NULL) { + return -1; + } + PyUnicode_InternInPlace(&obj); + *result++ = obj; + data += len; + } + } + } + if (bytestrings) { + for (; **bytestrings != '\0'; bytestrings++) { + size_t num; + const char *data = *bytestrings; + data = parse_int(data, &num); + while (num-- > 0) { + size_t len; + data = parse_int(data, &len); + PyObject *obj = PyBytes_FromStringAndSize(data, len); + if (obj == NULL) { + return -1; + } + *result++ = obj; + data += len; + } + } + } + if (ints) { + for (; **ints != '\0'; ints++) { + size_t num; + const char *data = *ints; + data = parse_int(data, &num); + while (num-- > 0) { + char *end; + PyObject *obj = PyLong_FromString(data, &end, 10); + if (obj == NULL) { + return -1; + } + data = end; + data++; + *result++ = obj; + } + } + } + if (floats) { + size_t num = (size_t)*floats++; + while (num-- > 0) { + PyObject *obj = PyFloat_FromDouble(*floats++); + if (obj == NULL) { + return -1; + } + *result++ = obj; + } + } + if (complex_numbers) { + size_t num = (size_t)*complex_numbers++; + while (num-- > 0) { + double real = *complex_numbers++; + double imag = *complex_numbers++; + PyObject *obj = PyComplex_FromDoubles(real, imag); + if (obj == NULL) { + return -1; + } + *result++ = obj; + } + } + if (tuples) { + int num = *tuples++; + while (num-- > 0) { + int num_items = *tuples++; + PyObject *obj = PyTuple_New(num_items); + if (obj == NULL) { + return -1; + } + int i; + for (i = 0; i < num_items; i++) { + PyObject *item = statics[*tuples++]; + Py_INCREF(item); + PyTuple_SET_ITEM(obj, i, item); + } + *result++ = obj; + } + } + if (frozensets) { + int num = *frozensets++; + while (num-- > 0) { + int num_items = *frozensets++; + PyObject *obj = PyFrozenSet_New(NULL); + if (obj == NULL) { + return -1; + } + for (int i = 0; i < num_items; i++) { + PyObject *item = statics[*frozensets++]; + Py_INCREF(item); + if (PySet_Add(obj, item) == -1) { + return -1; + } + } + *result++ = obj; + } + } + return 0; +} + +// Call super(type(self), self) +PyObject * +CPy_Super(PyObject *builtins, PyObject *self) { + PyObject *super_type = PyObject_GetAttrString(builtins, "super"); + if (!super_type) + return NULL; + PyObject *result = PyObject_CallFunctionObjArgs( + super_type, (PyObject*)Py_TYPE(self), self, NULL); + Py_DECREF(super_type); + return result; +} + +static bool import_single(PyObject *mod_id, PyObject **mod_static, + PyObject *globals_id, PyObject *globals_name, PyObject *globals) { + if (*mod_static == Py_None) { + CPyModule *mod = PyImport_Import(mod_id); + if (mod == NULL) { + return false; + } + *mod_static = mod; + } + + PyObject *mod_dict = PyImport_GetModuleDict(); + CPyModule *globals_mod = CPyDict_GetItem(mod_dict, globals_id); + if (globals_mod == NULL) { + return false; + } + int ret = CPyDict_SetItem(globals, globals_name, globals_mod); + Py_DECREF(globals_mod); + if (ret < 0) { + return false; + } + + return true; +} + +// Table-driven import helper. See transform_import() in irbuild for the details. +bool CPyImport_ImportMany(PyObject *modules, CPyModule **statics[], PyObject *globals, + PyObject *tb_path, PyObject *tb_function, Py_ssize_t *tb_lines) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(modules); i++) { + PyObject *module = PyTuple_GET_ITEM(modules, i); + PyObject *mod_id = PyTuple_GET_ITEM(module, 0); + PyObject *globals_id = PyTuple_GET_ITEM(module, 1); + PyObject *globals_name = PyTuple_GET_ITEM(module, 2); + + if (!import_single(mod_id, statics[i], globals_id, globals_name, globals)) { + assert(PyErr_Occurred() && "error indicator should be set on bad import!"); + PyObject *typ, *val, *tb; + PyErr_Fetch(&typ, &val, &tb); + const char *path = PyUnicode_AsUTF8(tb_path); + if (path == NULL) { + path = ""; + } + const char *function = PyUnicode_AsUTF8(tb_function); + if (function == NULL) { + function = ""; + } + PyErr_Restore(typ, val, tb); + CPy_AddTraceback(path, function, tb_lines[i], globals); + return false; + } + } + return true; +} + +// This helper function is a simplification of cpython/ceval.c/import_from() +static PyObject *CPyImport_ImportFrom(PyObject *module, PyObject *package_name, + PyObject *import_name, PyObject *as_name) { + // check if the imported module has an attribute by that name + PyObject *x = PyObject_GetAttr(module, import_name); + if (x == NULL) { + // if not, attempt to import a submodule with that name + PyObject *fullmodname = PyUnicode_FromFormat("%U.%U", package_name, import_name); + if (fullmodname == NULL) { + goto fail; + } + + // The following code is a simplification of cpython/import.c/PyImport_GetModule() + x = PyObject_GetItem(module, fullmodname); + Py_DECREF(fullmodname); + if (x == NULL) { + goto fail; + } + } + return x; + +fail: + PyErr_Clear(); + PyObject *package_path = PyModule_GetFilenameObject(module); + PyObject *errmsg = PyUnicode_FromFormat("cannot import name %R from %R (%S)", + import_name, package_name, package_path); + // NULL checks for errmsg and package_name done by PyErr_SetImportError. + PyErr_SetImportError(errmsg, package_name, package_path); + Py_DECREF(package_path); + Py_DECREF(errmsg); + return NULL; +} + +PyObject *CPyImport_ImportFromMany(PyObject *mod_id, PyObject *names, PyObject *as_names, + PyObject *globals) { + PyObject *mod = PyImport_ImportModuleLevelObject(mod_id, globals, 0, names, 0); + if (mod == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(names); i++) { + PyObject *name = PyTuple_GET_ITEM(names, i); + PyObject *as_name = PyTuple_GET_ITEM(as_names, i); + PyObject *obj = CPyImport_ImportFrom(mod, mod_id, name, as_name); + if (obj == NULL) { + Py_DECREF(mod); + return NULL; + } + int ret = CPyDict_SetItem(globals, as_name, obj); + Py_DECREF(obj); + if (ret < 0) { + Py_DECREF(mod); + return NULL; + } + } + return mod; +} + +// From CPython +static PyObject * +CPy_BinopTypeError(PyObject *left, PyObject *right, const char *op) { + PyErr_Format(PyExc_TypeError, + "unsupported operand type(s) for %.100s: " + "'%.100s' and '%.100s'", + op, + Py_TYPE(left)->tp_name, + Py_TYPE(right)->tp_name); + return NULL; +} + +PyObject * +CPy_CallReverseOpMethod(PyObject *left, + PyObject *right, + const char *op, + _Py_Identifier *method) { + // Look up reverse method + PyObject *m = _PyObject_GetAttrId(right, method); + if (m == NULL) { + // If reverse method not defined, generate TypeError instead AttributeError + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + CPy_BinopTypeError(left, right, op); + } + return NULL; + } + // Call reverse method + PyObject *result = PyObject_CallOneArg(m, left); + Py_DECREF(m); + return result; +} + +PyObject *CPySingledispatch_RegisterFunction(PyObject *singledispatch_func, + PyObject *cls, + PyObject *func) { + PyObject *registry = PyObject_GetAttrString(singledispatch_func, "registry"); + PyObject *register_func = NULL; + PyObject *typing = NULL; + PyObject *get_type_hints = NULL; + PyObject *type_hints = NULL; + + if (registry == NULL) goto fail; + if (func == NULL) { + // one argument case + if (PyType_Check(cls)) { + // passed a class + // bind cls to the first argument so that register gets called again with both the + // class and the function + register_func = PyObject_GetAttrString(singledispatch_func, "register"); + if (register_func == NULL) goto fail; + return PyMethod_New(register_func, cls); + } + // passed a function + PyObject *annotations = PyFunction_GetAnnotations(cls); + const char *invalid_first_arg_msg = + "Invalid first argument to `register()`: %R. " + "Use either `@register(some_class)` or plain `@register` " + "on an annotated function."; + + if (annotations == NULL) { + PyErr_Format(PyExc_TypeError, invalid_first_arg_msg, cls); + goto fail; + } + + Py_INCREF(annotations); + + func = cls; + typing = PyImport_ImportModule("typing"); + if (typing == NULL) goto fail; + get_type_hints = PyObject_GetAttrString(typing, "get_type_hints"); + + type_hints = PyObject_CallOneArg(get_type_hints, func); + PyObject *argname; + Py_ssize_t pos = 0; + if (!PyDict_Next(type_hints, &pos, &argname, &cls)) { + // the functools implementation raises the same type error if annotations is an empty dict + PyErr_Format(PyExc_TypeError, invalid_first_arg_msg, cls); + goto fail; + } + if (!PyType_Check(cls)) { + const char *invalid_annotation_msg = "Invalid annotation for %R. %R is not a class."; + PyErr_Format(PyExc_TypeError, invalid_annotation_msg, argname, cls); + goto fail; + } + } + if (PyDict_SetItem(registry, cls, func) == -1) { + goto fail; + } + + // clear the cache so we consider the newly added function when dispatching + PyObject *dispatch_cache = PyObject_GetAttrString(singledispatch_func, "dispatch_cache"); + if (dispatch_cache == NULL) goto fail; + PyDict_Clear(dispatch_cache); + + Py_INCREF(func); + return func; + +fail: + Py_XDECREF(registry); + Py_XDECREF(register_func); + Py_XDECREF(typing); + Py_XDECREF(get_type_hints); + Py_XDECREF(type_hints); + return NULL; + +} + +// Adapated from ceval.c GET_AITER +PyObject *CPy_GetAIter(PyObject *obj) +{ + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + + if (getter == NULL) { + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + return NULL; + } + + PyObject *iter = (*getter)(obj); + if (!iter) { + return NULL; + } + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + + PyErr_Format(PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + return NULL; + } + + return iter; +} + +// Adapated from ceval.c GET_ANEXT +PyObject *CPy_GetANext(PyObject *aiter) +{ + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyObject *awaitable = NULL; + PyTypeObject *type = Py_TYPE(aiter); + + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + goto error; + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + goto error; + } + } + else { + PyErr_Format(PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + goto error; + } + + awaitable = CPyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + + Py_DECREF(next_iter); + goto error; + } else { + Py_DECREF(next_iter); + } + } + + return awaitable; +error: + return NULL; +} + +#ifdef CPY_3_12_FEATURES + +// Copied from Python 3.12.3, since this struct is internal to CPython. It defines +// the structure of typing.TypeAliasType objects. We need it since compute_value is +// not part of the public API, and we need to set it to match Python runtime semantics. +// +// IMPORTANT: This needs to be kept in sync with CPython! +typedef struct { + PyObject_HEAD + PyObject *name; + PyObject *type_params; + PyObject *compute_value; + PyObject *value; + PyObject *module; +} typealiasobject; + +void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value) { + typealiasobject *obj = (typealiasobject *)alias; + if (obj->value != NULL) { + Py_DECREF(obj->value); + } + obj->value = NULL; + Py_INCREF(compute_value); + if (obj->compute_value != NULL) { + Py_DECREF(obj->compute_value); + } + obj->compute_value = compute_value; +} + +#endif diff --git a/mypyc/lib-rt/module_shim.tmpl b/mypyc/lib-rt/module_shim.tmpl new file mode 100644 index 000000000000..6e772efd34ec --- /dev/null +++ b/mypyc/lib-rt/module_shim.tmpl @@ -0,0 +1,18 @@ +#include + +PyMODINIT_FUNC +PyInit_{modname}(void) +{{ + PyObject *tmp; + if (!(tmp = PyImport_ImportModule("{libname}"))) return NULL; + Py_DECREF(tmp); + void *init_func = PyCapsule_Import("{libname}.init_{full_modname}", 0); + if (!init_func) {{ + return NULL; + }} + return ((PyObject *(*)(void))init_func)(); +}} + +// distutils sometimes spuriously tells cl to export CPyInit___init__, +// so provide that so it chills out +PyMODINIT_FUNC PyInit___init__(void) {{ return PyInit_{modname}(); }} diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h new file mode 100644 index 000000000000..9967f0a13b4f --- /dev/null +++ b/mypyc/lib-rt/mypyc_util.h @@ -0,0 +1,121 @@ +#ifndef MYPYC_UTIL_H +#define MYPYC_UTIL_H + +#include +#include +#include + +#if defined(__clang__) || defined(__GNUC__) +#define likely(x) __builtin_expect((x),1) +#define unlikely(x) __builtin_expect((x),0) +#define CPy_Unreachable() __builtin_unreachable() +#else +#define likely(x) (x) +#define unlikely(x) (x) +#define CPy_Unreachable() abort() +#endif + +#if defined(__clang__) || defined(__GNUC__) +#define CPy_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +#define CPy_NOINLINE __declspec(noinline) +#else +#define CPy_NOINLINE +#endif + +// INCREF and DECREF that assert the pointer is not NULL. +// asserts are disabled in release builds so there shouldn't be a perf hit. +// I'm honestly kind of surprised that this isn't done by default. +#define CPy_INCREF(p) do { assert(p); Py_INCREF(p); } while (0) +#define CPy_DECREF(p) do { assert(p); Py_DECREF(p); } while (0) +// Here just for consistency +#define CPy_XDECREF(p) Py_XDECREF(p) + +// Tagged integer -- our representation of Python 'int' objects. +// Small enough integers are represented as unboxed integers (shifted +// left by 1); larger integers (larger than 63 bits on a 64-bit +// platform) are stored as a tagged pointer (PyObject *) +// representing a Python int object, with the lowest bit set. +// Tagged integers are always normalized. A small integer *must not* +// have the tag bit set. +typedef size_t CPyTagged; + +typedef size_t CPyPtr; + +#define CPY_INT_BITS (CHAR_BIT * sizeof(CPyTagged)) + +#define CPY_TAGGED_MAX (((Py_ssize_t)1 << (CPY_INT_BITS - 2)) - 1) +#define CPY_TAGGED_MIN (-((Py_ssize_t)1 << (CPY_INT_BITS - 2))) +#define CPY_TAGGED_ABS_MIN (0-(size_t)CPY_TAGGED_MIN) + +typedef PyObject CPyModule; + +// Tag bit used for long integers +#define CPY_INT_TAG 1 + +// Error value for signed fixed-width (low-level) integers +#define CPY_LL_INT_ERROR -113 + +// Error value for unsigned fixed-width (low-level) integers +#define CPY_LL_UINT_ERROR 239 + +// Error value for floats +#define CPY_FLOAT_ERROR -113.0 + +typedef void (*CPyVTableItem)(void); + +static inline CPyTagged CPyTagged_ShortFromInt(int x) { + return x << 1; +} + +static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) { + return x << 1; +} + +// Are we targeting Python 3.12 or newer? +#define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000) + +#if CPY_3_12_FEATURES + +// Same as macros in CPython internal/pycore_long.h, but with a CPY_ prefix +#define CPY_NON_SIZE_BITS 3 +#define CPY_SIGN_ZERO 1 +#define CPY_SIGN_NEGATIVE 2 +#define CPY_SIGN_MASK 3 + +#define CPY_LONG_DIGIT(o, n) ((o)->long_value.ob_digit[n]) + +// Only available on Python 3.12 and later +#define CPY_LONG_TAG(o) ((o)->long_value.lv_tag) +#define CPY_LONG_IS_NEGATIVE(o) (((o)->long_value.lv_tag & CPY_SIGN_MASK) == CPY_SIGN_NEGATIVE) +// Only available on Python 3.12 and later +#define CPY_LONG_SIZE(o) ((o)->long_value.lv_tag >> CPY_NON_SIZE_BITS) +// Number of digits; negative for negative ints +#define CPY_LONG_SIZE_SIGNED(o) (CPY_LONG_IS_NEGATIVE(o) ? -CPY_LONG_SIZE(o) : CPY_LONG_SIZE(o)) +// Number of digits, assuming int is non-negative +#define CPY_LONG_SIZE_UNSIGNED(o) CPY_LONG_SIZE(o) + +static inline void CPyLong_SetUnsignedSize(PyLongObject *o, Py_ssize_t n) { + if (n == 0) + o->long_value.lv_tag = CPY_SIGN_ZERO; + else + o->long_value.lv_tag = n << CPY_NON_SIZE_BITS; +} + +#else + +#define CPY_LONG_DIGIT(o, n) ((o)->ob_digit[n]) +#define CPY_LONG_IS_NEGATIVE(o) (((o)->ob_base.ob_size < 0) +#define CPY_LONG_SIZE_SIGNED(o) ((o)->ob_base.ob_size) +#define CPY_LONG_SIZE_UNSIGNED(o) ((o)->ob_base.ob_size) + +static inline void CPyLong_SetUnsignedSize(PyLongObject *o, Py_ssize_t n) { + o->ob_base.ob_size = n; +} + +#endif + +// Are we targeting Python 3.13 or newer? +#define CPY_3_13_FEATURES (PY_VERSION_HEX >= 0x030d0000) + +#endif diff --git a/mypyc/lib-rt/pythoncapi_compat.h b/mypyc/lib-rt/pythoncapi_compat.h new file mode 100644 index 000000000000..acaadf34bf2e --- /dev/null +++ b/mypyc/lib-rt/pythoncapi_compat.h @@ -0,0 +1,1696 @@ +// Header file providing new C API functions to old Python versions. +// +// File distributed under the Zero Clause BSD (0BSD) license. +// Copyright Contributors to the pythoncapi_compat project. +// +// Homepage: +// https://github.com/python/pythoncapi_compat +// +// Latest version: +// https://raw.githubusercontent.com/python/pythoncapi_compat/master/pythoncapi_compat.h +// +// SPDX-License-Identifier: 0BSD + +#ifndef PYTHONCAPI_COMPAT +#define PYTHONCAPI_COMPAT + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +// Python 3.11.0b4 added PyFrame_Back() to Python.h +#if PY_VERSION_HEX < 0x030b00B4 && !defined(PYPY_VERSION) +# include "frameobject.h" // PyFrameObject, PyFrame_GetBack() +#endif + + +#ifndef _Py_CAST +# define _Py_CAST(type, expr) ((type)(expr)) +#endif + +// Static inline functions should use _Py_NULL rather than using directly NULL +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) +# define _Py_NULL nullptr +#else +# define _Py_NULL NULL +#endif + +// Cast argument to PyObject* type. +#ifndef _PyObject_CAST +# define _PyObject_CAST(op) _Py_CAST(PyObject*, op) +#endif + +#ifndef Py_BUILD_ASSERT +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)sizeof(char [1 - 2 * !(cond)]); \ + } while(0) +#endif + + +// bpo-42262 added Py_NewRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_NewRef) +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} +#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +#endif + + +// bpo-42262 added Py_XNewRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_XNewRef) +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} +#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#endif + + +// bpo-39573 added Py_SET_REFCNT() to Python 3.9.0a4 +#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_REFCNT) +static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) +{ + ob->ob_refcnt = refcnt; +} +#define Py_SET_REFCNT(ob, refcnt) _Py_SET_REFCNT(_PyObject_CAST(ob), refcnt) +#endif + + +// Py_SETREF() and Py_XSETREF() were added to Python 3.5.2. +// It is excluded from the limited C API. +#if (PY_VERSION_HEX < 0x03050200 && !defined(Py_SETREF)) && !defined(Py_LIMITED_API) +#define Py_SETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = _PyObject_CAST(src); \ + Py_DECREF(_tmp_dst); \ + } while (0) + +#define Py_XSETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = _PyObject_CAST(src); \ + Py_XDECREF(_tmp_dst); \ + } while (0) +#endif + + +// bpo-43753 added Py_Is(), Py_IsNone(), Py_IsTrue() and Py_IsFalse() +// to Python 3.10.0b1. +#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_Is) +# define Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsNone) +# define Py_IsNone(x) Py_Is(x, Py_None) +#endif +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsTrue) +# define Py_IsTrue(x) Py_Is(x, Py_True) +#endif +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsFalse) +# define Py_IsFalse(x) Py_Is(x, Py_False) +#endif + + +// bpo-39573 added Py_SET_TYPE() to Python 3.9.0a4 +#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE) +static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) +{ + ob->ob_type = type; +} +#define Py_SET_TYPE(ob, type) _Py_SET_TYPE(_PyObject_CAST(ob), type) +#endif + + +// bpo-39573 added Py_SET_SIZE() to Python 3.9.0a4 +#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE) +static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) +{ + ob->ob_size = size; +} +#define Py_SET_SIZE(ob, size) _Py_SET_SIZE((PyVarObject*)(ob), size) +#endif + + +// bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1 +#if PY_VERSION_HEX < 0x030900B1 || defined(PYPY_VERSION) +static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame) +{ + assert(frame != _Py_NULL); + assert(frame->f_code != _Py_NULL); + return _Py_CAST(PyCodeObject*, Py_NewRef(frame->f_code)); +} +#endif + +static inline PyCodeObject* _PyFrame_GetCodeBorrow(PyFrameObject *frame) +{ + PyCodeObject *code = PyFrame_GetCode(frame); + Py_DECREF(code); + return code; +} + + +// bpo-40421 added PyFrame_GetBack() to Python 3.9.0b1 +#if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) +static inline PyFrameObject* PyFrame_GetBack(PyFrameObject *frame) +{ + assert(frame != _Py_NULL); + return _Py_CAST(PyFrameObject*, Py_XNewRef(frame->f_back)); +} +#endif + +#if !defined(PYPY_VERSION) +static inline PyFrameObject* _PyFrame_GetBackBorrow(PyFrameObject *frame) +{ + PyFrameObject *back = PyFrame_GetBack(frame); + Py_XDECREF(back); + return back; +} +#endif + + +// bpo-40421 added PyFrame_GetLocals() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetLocals(PyFrameObject *frame) +{ +#if PY_VERSION_HEX >= 0x030400B1 + if (PyFrame_FastToLocalsWithError(frame) < 0) { + return NULL; + } +#else + PyFrame_FastToLocals(frame); +#endif + return Py_NewRef(frame->f_locals); +} +#endif + + +// bpo-40421 added PyFrame_GetGlobals() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetGlobals(PyFrameObject *frame) +{ + return Py_NewRef(frame->f_globals); +} +#endif + + +// bpo-40421 added PyFrame_GetBuiltins() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) +{ + return Py_NewRef(frame->f_builtins); +} +#endif + + +// bpo-40421 added PyFrame_GetLasti() to Python 3.11.0b1 +#if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) +static inline int PyFrame_GetLasti(PyFrameObject *frame) +{ +#if PY_VERSION_HEX >= 0x030A00A7 + // bpo-27129: Since Python 3.10.0a7, f_lasti is an instruction offset, + // not a bytes offset anymore. Python uses 16-bit "wordcode" (2 bytes) + // instructions. + if (frame->f_lasti < 0) { + return -1; + } + return frame->f_lasti * 2; +#else + return frame->f_lasti; +#endif +} +#endif + + +// gh-91248 added PyFrame_GetVar() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetVar(PyFrameObject *frame, PyObject *name) +{ + PyObject *locals, *value; + + locals = PyFrame_GetLocals(frame); + if (locals == NULL) { + return NULL; + } +#if PY_VERSION_HEX >= 0x03000000 + value = PyDict_GetItemWithError(locals, name); +#else + value = _PyDict_GetItemWithError(locals, name); +#endif + Py_DECREF(locals); + + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } +#if PY_VERSION_HEX >= 0x03000000 + PyErr_Format(PyExc_NameError, "variable %R does not exist", name); +#else + PyErr_SetString(PyExc_NameError, "variable does not exist"); +#endif + return NULL; + } + return Py_NewRef(value); +} +#endif + + +// gh-91248 added PyFrame_GetVarString() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* +PyFrame_GetVarString(PyFrameObject *frame, const char *name) +{ + PyObject *name_obj, *value; +#if PY_VERSION_HEX >= 0x03000000 + name_obj = PyUnicode_FromString(name); +#else + name_obj = PyString_FromString(name); +#endif + if (name_obj == NULL) { + return NULL; + } + value = PyFrame_GetVar(frame, name_obj); + Py_DECREF(name_obj); + return value; +} +#endif + + +// bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5 +#if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) +static inline PyInterpreterState * +PyThreadState_GetInterpreter(PyThreadState *tstate) +{ + assert(tstate != _Py_NULL); + return tstate->interp; +} +#endif + + +// bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1 +#if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) +static inline PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) +{ + assert(tstate != _Py_NULL); + return _Py_CAST(PyFrameObject *, Py_XNewRef(tstate->frame)); +} +#endif + +#if !defined(PYPY_VERSION) +static inline PyFrameObject* +_PyThreadState_GetFrameBorrow(PyThreadState *tstate) +{ + PyFrameObject *frame = PyThreadState_GetFrame(tstate); + Py_XDECREF(frame); + return frame; +} +#endif + + +// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5 +#if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) +static inline PyInterpreterState* PyInterpreterState_Get(void) +{ + PyThreadState *tstate; + PyInterpreterState *interp; + + tstate = PyThreadState_GET(); + if (tstate == _Py_NULL) { + Py_FatalError("GIL released (tstate is NULL)"); + } + interp = tstate->interp; + if (interp == _Py_NULL) { + Py_FatalError("no current interpreter"); + } + return interp; +} +#endif + + +// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a6 +#if 0x030700A1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) +static inline uint64_t PyThreadState_GetID(PyThreadState *tstate) +{ + assert(tstate != _Py_NULL); + return tstate->id; +} +#endif + +// bpo-43760 added PyThreadState_EnterTracing() to Python 3.11.0a2 +#if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) +static inline void PyThreadState_EnterTracing(PyThreadState *tstate) +{ + tstate->tracing++; +#if PY_VERSION_HEX >= 0x030A00A1 + tstate->cframe->use_tracing = 0; +#else + tstate->use_tracing = 0; +#endif +} +#endif + +// bpo-43760 added PyThreadState_LeaveTracing() to Python 3.11.0a2 +#if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) +static inline void PyThreadState_LeaveTracing(PyThreadState *tstate) +{ + int use_tracing = (tstate->c_tracefunc != _Py_NULL + || tstate->c_profilefunc != _Py_NULL); + tstate->tracing--; +#if PY_VERSION_HEX >= 0x030A00A1 + tstate->cframe->use_tracing = use_tracing; +#else + tstate->use_tracing = use_tracing; +#endif +} +#endif + + +// bpo-37194 added PyObject_CallNoArgs() to Python 3.9.0a1 +// PyObject_CallNoArgs() added to PyPy 3.9.16-v7.3.11 +#if !defined(PyObject_CallNoArgs) && PY_VERSION_HEX < 0x030900A1 +static inline PyObject* PyObject_CallNoArgs(PyObject *func) +{ + return PyObject_CallFunctionObjArgs(func, NULL); +} +#endif + + +// bpo-39245 made PyObject_CallOneArg() public (previously called +// _PyObject_CallOneArg) in Python 3.9.0a4 +// PyObject_CallOneArg() added to PyPy 3.9.16-v7.3.11 +#if !defined(PyObject_CallOneArg) && PY_VERSION_HEX < 0x030900A4 +static inline PyObject* PyObject_CallOneArg(PyObject *func, PyObject *arg) +{ + return PyObject_CallFunctionObjArgs(func, arg, NULL); +} +#endif + + +// bpo-1635741 added PyModule_AddObjectRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 +static inline int +PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) +{ + int res; + + if (!value && !PyErr_Occurred()) { + // PyModule_AddObject() raises TypeError in this case + PyErr_SetString(PyExc_SystemError, + "PyModule_AddObjectRef() must be called " + "with an exception raised if value is NULL"); + return -1; + } + + Py_XINCREF(value); + res = PyModule_AddObject(module, name, value); + if (res < 0) { + Py_XDECREF(value); + } + return res; +} +#endif + + +// bpo-40024 added PyModule_AddType() to Python 3.9.0a5 +#if PY_VERSION_HEX < 0x030900A5 +static inline int PyModule_AddType(PyObject *module, PyTypeObject *type) +{ + const char *name, *dot; + + if (PyType_Ready(type) < 0) { + return -1; + } + + // inline _PyType_Name() + name = type->tp_name; + assert(name != _Py_NULL); + dot = strrchr(name, '.'); + if (dot != _Py_NULL) { + name = dot + 1; + } + + return PyModule_AddObjectRef(module, name, _PyObject_CAST(type)); +} +#endif + + +// bpo-40241 added PyObject_GC_IsTracked() to Python 3.9.0a6. +// bpo-4688 added _PyObject_GC_IS_TRACKED() to Python 2.7.0a2. +#if PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) +static inline int PyObject_GC_IsTracked(PyObject* obj) +{ + return (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)); +} +#endif + +// bpo-40241 added PyObject_GC_IsFinalized() to Python 3.9.0a6. +// bpo-18112 added _PyGCHead_FINALIZED() to Python 3.4.0 final. +#if PY_VERSION_HEX < 0x030900A6 && PY_VERSION_HEX >= 0x030400F0 && !defined(PYPY_VERSION) +static inline int PyObject_GC_IsFinalized(PyObject *obj) +{ + PyGC_Head *gc = _Py_CAST(PyGC_Head*, obj) - 1; + return (PyObject_IS_GC(obj) && _PyGCHead_FINALIZED(gc)); +} +#endif + + +// bpo-39573 added Py_IS_TYPE() to Python 3.9.0a4 +#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_IS_TYPE) +static inline int _Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { + return Py_TYPE(ob) == type; +} +#define Py_IS_TYPE(ob, type) _Py_IS_TYPE(_PyObject_CAST(ob), type) +#endif + + +// bpo-46906 added PyFloat_Pack2() and PyFloat_Unpack2() to Python 3.11a7. +// bpo-11734 added _PyFloat_Pack2() and _PyFloat_Unpack2() to Python 3.6.0b1. +// Python 3.11a2 moved _PyFloat_Pack2() and _PyFloat_Unpack2() to the internal +// C API: Python 3.11a2-3.11a6 versions are not supported. +#if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) +static inline int PyFloat_Pack2(double x, char *p, int le) +{ return _PyFloat_Pack2(x, (unsigned char*)p, le); } + +static inline double PyFloat_Unpack2(const char *p, int le) +{ return _PyFloat_Unpack2((const unsigned char *)p, le); } +#endif + + +// bpo-46906 added PyFloat_Pack4(), PyFloat_Pack8(), PyFloat_Unpack4() and +// PyFloat_Unpack8() to Python 3.11a7. +// Python 3.11a2 moved _PyFloat_Pack4(), _PyFloat_Pack8(), _PyFloat_Unpack4() +// and _PyFloat_Unpack8() to the internal C API: Python 3.11a2-3.11a6 versions +// are not supported. +#if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) +static inline int PyFloat_Pack4(double x, char *p, int le) +{ return _PyFloat_Pack4(x, (unsigned char*)p, le); } + +static inline int PyFloat_Pack8(double x, char *p, int le) +{ return _PyFloat_Pack8(x, (unsigned char*)p, le); } + +static inline double PyFloat_Unpack4(const char *p, int le) +{ return _PyFloat_Unpack4((const unsigned char *)p, le); } + +static inline double PyFloat_Unpack8(const char *p, int le) +{ return _PyFloat_Unpack8((const unsigned char *)p, le); } +#endif + + +// gh-92154 added PyCode_GetCode() to Python 3.11.0b1 +#if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetCode(PyCodeObject *code) +{ + return Py_NewRef(code->co_code); +} +#endif + + +// gh-95008 added PyCode_GetVarnames() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetVarnames(PyCodeObject *code) +{ + return Py_NewRef(code->co_varnames); +} +#endif + +// gh-95008 added PyCode_GetFreevars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetFreevars(PyCodeObject *code) +{ + return Py_NewRef(code->co_freevars); +} +#endif + +// gh-95008 added PyCode_GetCellvars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetCellvars(PyCodeObject *code) +{ + return Py_NewRef(code->co_cellvars); +} +#endif + + +// Py_UNUSED() was added to Python 3.4.0b2. +#if PY_VERSION_HEX < 0x030400B2 && !defined(Py_UNUSED) +# if defined(__GNUC__) || defined(__clang__) +# define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) +# else +# define Py_UNUSED(name) _unused_ ## name +# endif +#endif + + +// gh-105922 added PyImport_AddModuleRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A0 +static inline PyObject* PyImport_AddModuleRef(const char *name) +{ + return Py_XNewRef(PyImport_AddModule(name)); +} +#endif + + +// gh-105927 added PyWeakref_GetRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D0000 +static inline int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) +{ + PyObject *obj; + if (ref != NULL && !PyWeakref_Check(ref)) { + *pobj = NULL; + PyErr_SetString(PyExc_TypeError, "expected a weakref"); + return -1; + } + obj = PyWeakref_GetObject(ref); + if (obj == NULL) { + // SystemError if ref is NULL + *pobj = NULL; + return -1; + } + if (obj == Py_None) { + *pobj = NULL; + return 0; + } + *pobj = Py_NewRef(obj); + return (*pobj != NULL); +} +#endif + + +// bpo-36974 added PY_VECTORCALL_ARGUMENTS_OFFSET to Python 3.8b1 +#ifndef PY_VECTORCALL_ARGUMENTS_OFFSET +# define PY_VECTORCALL_ARGUMENTS_OFFSET (_Py_CAST(size_t, 1) << (8 * sizeof(size_t) - 1)) +#endif + +// bpo-36974 added PyVectorcall_NARGS() to Python 3.8b1 +#if PY_VERSION_HEX < 0x030800B1 +static inline Py_ssize_t PyVectorcall_NARGS(size_t n) +{ + return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET; +} +#endif + + +// gh-105922 added PyObject_Vectorcall() to Python 3.9.0a4 +#if PY_VERSION_HEX < 0x030900A4 +static inline PyObject* +PyObject_Vectorcall(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ +#if PY_VERSION_HEX >= 0x030800B1 && !defined(PYPY_VERSION) + // bpo-36974 added _PyObject_Vectorcall() to Python 3.8.0b1 + return _PyObject_Vectorcall(callable, args, nargsf, kwnames); +#else + PyObject *posargs = NULL, *kwargs = NULL; + PyObject *res; + Py_ssize_t nposargs, nkwargs, i; + + if (nargsf != 0 && args == NULL) { + PyErr_BadInternalCall(); + goto error; + } + if (kwnames != NULL && !PyTuple_Check(kwnames)) { + PyErr_BadInternalCall(); + goto error; + } + + nposargs = (Py_ssize_t)PyVectorcall_NARGS(nargsf); + if (kwnames) { + nkwargs = PyTuple_GET_SIZE(kwnames); + } + else { + nkwargs = 0; + } + + posargs = PyTuple_New(nposargs); + if (posargs == NULL) { + goto error; + } + if (nposargs) { + for (i=0; i < nposargs; i++) { + PyTuple_SET_ITEM(posargs, i, Py_NewRef(*args)); + args++; + } + } + + if (nkwargs) { + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto error; + } + + for (i = 0; i < nkwargs; i++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = *args; + args++; + if (PyDict_SetItem(kwargs, key, value) < 0) { + goto error; + } + } + } + else { + kwargs = NULL; + } + + res = PyObject_Call(callable, posargs, kwargs); + Py_DECREF(posargs); + Py_XDECREF(kwargs); + return res; + +error: + Py_DECREF(posargs); + Py_XDECREF(kwargs); + return NULL; +#endif +} +#endif + + +// gh-106521 added PyObject_GetOptionalAttr() and +// PyObject_GetOptionalAttrString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result) +{ + // bpo-32571 added _PyObject_LookupAttr() to Python 3.7.0b1 +#if PY_VERSION_HEX >= 0x030700B1 && !defined(PYPY_VERSION) + return _PyObject_LookupAttr(obj, attr_name, result); +#else + *result = PyObject_GetAttr(obj, attr_name); + if (*result != NULL) { + return 1; + } + if (!PyErr_Occurred()) { + return 0; + } + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + return 0; + } + return -1; +#endif +} + +static inline int +PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result) +{ + PyObject *name_obj; + int rc; +#if PY_VERSION_HEX >= 0x03000000 + name_obj = PyUnicode_FromString(attr_name); +#else + name_obj = PyString_FromString(attr_name); +#endif + if (name_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyObject_GetOptionalAttr(obj, name_obj, result); + Py_DECREF(name_obj); + return rc; +} +#endif + + +// gh-106307 added PyObject_GetOptionalAttr() and +// PyMapping_GetOptionalItemString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) +{ + *result = PyObject_GetItem(obj, key); + if (*result) { + return 1; + } + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; +} + +static inline int +PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) +{ + PyObject *key_obj; + int rc; +#if PY_VERSION_HEX >= 0x03000000 + key_obj = PyUnicode_FromString(key); +#else + key_obj = PyString_FromString(key); +#endif + if (key_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyMapping_GetOptionalItem(obj, key_obj, result); + Py_DECREF(key_obj); + return rc; +} +#endif + +// gh-108511 added PyMapping_HasKeyWithError() and +// PyMapping_HasKeyStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_HasKeyWithError(PyObject *obj, PyObject *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItem(obj, key, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyMapping_HasKeyStringWithError(PyObject *obj, const char *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItemString(obj, key, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-108511 added PyObject_HasAttrWithError() and +// PyObject_HasAttrStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_HasAttrWithError(PyObject *obj, PyObject *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttr(obj, attr, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyObject_HasAttrStringWithError(PyObject *obj, const char *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttrString(obj, attr, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-106004 added PyDict_GetItemRef() and PyDict_GetItemStringRef() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) +{ +#if PY_VERSION_HEX >= 0x03000000 + PyObject *item = PyDict_GetItemWithError(mp, key); +#else + PyObject *item = _PyDict_GetItemWithError(mp, key); +#endif + if (item != NULL) { + *result = Py_NewRef(item); + return 1; // found + } + if (!PyErr_Occurred()) { + *result = NULL; + return 0; // not found + } + *result = NULL; + return -1; +} + +static inline int +PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result) +{ + int res; +#if PY_VERSION_HEX >= 0x03000000 + PyObject *key_obj = PyUnicode_FromString(key); +#else + PyObject *key_obj = PyString_FromString(key); +#endif + if (key_obj == NULL) { + *result = NULL; + return -1; + } + res = PyDict_GetItemRef(mp, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-106307 added PyModule_Add() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyModule_Add(PyObject *mod, const char *name, PyObject *value) +{ + int res = PyModule_AddObjectRef(mod, name, value); + Py_XDECREF(value); + return res; +} +#endif + + +// gh-108014 added Py_IsFinalizing() to Python 3.13.0a1 +// bpo-1856 added _Py_Finalizing to Python 3.2.1b1. +// _Py_IsFinalizing() was added to PyPy 7.3.0. +#if (0x030201B1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030D00A1) \ + && (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x7030000) +static inline int Py_IsFinalizing(void) +{ +#if PY_VERSION_HEX >= 0x030700A1 + // _Py_IsFinalizing() was added to Python 3.7.0a1. + return _Py_IsFinalizing(); +#else + return (_Py_Finalizing != NULL); +#endif +} +#endif + + +// gh-108323 added PyDict_ContainsString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyDict_ContainsString(PyObject *op, const char *key) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + return -1; + } + int res = PyDict_Contains(op, key_obj); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-108445 added PyLong_AsInt() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyLong_AsInt(PyObject *obj) +{ +#ifdef PYPY_VERSION + long value = PyLong_AsLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + if (value < (long)INT_MIN || (long)INT_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)value; +#else + return _PyLong_AsInt(obj); +#endif +} +#endif + + +// gh-107073 added PyObject_VisitManagedDict() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return -1; + } + Py_VISIT(*dict); + return 0; +} + +static inline void +PyObject_ClearManagedDict(PyObject *obj) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return; + } + Py_CLEAR(*dict); +} +#endif + +// gh-108867 added PyThreadState_GetUnchecked() to Python 3.13.0a1 +// Python 3.5.2 added _PyThreadState_UncheckedGet(). +#if PY_VERSION_HEX >= 0x03050200 && PY_VERSION_HEX < 0x030D00A1 +static inline PyThreadState* +PyThreadState_GetUnchecked(void) +{ + return _PyThreadState_UncheckedGet(); +} +#endif + +// gh-110289 added PyUnicode_EqualToUTF8() and PyUnicode_EqualToUTF8AndSize() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *str, Py_ssize_t str_len) +{ + Py_ssize_t len; + const void *utf8; + PyObject *exc_type, *exc_value, *exc_tb; + int res; + + // API cannot report errors so save/restore the exception + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + // Python 3.3.0a1 added PyUnicode_AsUTF8AndSize() +#if PY_VERSION_HEX >= 0x030300A1 + if (PyUnicode_IS_ASCII(unicode)) { + utf8 = PyUnicode_DATA(unicode); + len = PyUnicode_GET_LENGTH(unicode); + } + else { + utf8 = PyUnicode_AsUTF8AndSize(unicode, &len); + if (utf8 == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + } + + if (len != str_len) { + res = 0; + goto done; + } + res = (memcmp(utf8, str, (size_t)len) == 0); +#else + PyObject *bytes = PyUnicode_AsUTF8String(unicode); + if (bytes == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + +#if PY_VERSION_HEX >= 0x03000000 + len = PyBytes_GET_SIZE(bytes); + utf8 = PyBytes_AS_STRING(bytes); +#else + len = PyString_GET_SIZE(bytes); + utf8 = PyString_AS_STRING(bytes); +#endif + if (len != str_len) { + Py_DECREF(bytes); + res = 0; + goto done; + } + + res = (memcmp(utf8, str, (size_t)len) == 0); + Py_DECREF(bytes); +#endif + +done: + PyErr_Restore(exc_type, exc_value, exc_tb); + return res; +} + +static inline int +PyUnicode_EqualToUTF8(PyObject *unicode, const char *str) +{ + return PyUnicode_EqualToUTF8AndSize(unicode, str, (Py_ssize_t)strlen(str)); +} +#endif + + +// gh-111138 added PyList_Extend() and PyList_Clear() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyList_Extend(PyObject *list, PyObject *iterable) +{ + return PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable); +} + +static inline int +PyList_Clear(PyObject *list) +{ + return PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL); +} +#endif + +// gh-111262 added PyDict_Pop() and PyDict_PopString() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result) +{ + PyObject *value; + + if (!PyDict_Check(dict)) { + PyErr_BadInternalCall(); + if (result) { + *result = NULL; + } + return -1; + } + + // bpo-16991 added _PyDict_Pop() to Python 3.5.0b2. + // Python 3.6.0b3 changed _PyDict_Pop() first argument type to PyObject*. + // Python 3.13.0a1 removed _PyDict_Pop(). +#if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x030500b2 || PY_VERSION_HEX >= 0x030D0000 + value = PyObject_CallMethod(dict, "pop", "O", key); +#elif PY_VERSION_HEX < 0x030600b3 + value = _PyDict_Pop(_Py_CAST(PyDictObject*, dict), key, NULL); +#else + value = _PyDict_Pop(dict, key, NULL); +#endif + if (value == NULL) { + if (result) { + *result = NULL; + } + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; + } + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; +} + +static inline int +PyDict_PopString(PyObject *dict, const char *key, PyObject **result) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + if (result != NULL) { + *result = NULL; + } + return -1; + } + + int res = PyDict_Pop(dict, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +#if PY_VERSION_HEX < 0x030200A4 +// Python 3.2.0a4 added Py_hash_t type +typedef Py_ssize_t Py_hash_t; +#endif + + +// gh-111545 added Py_HashPointer() to Python 3.13.0a3 +#if PY_VERSION_HEX < 0x030D00A3 +static inline Py_hash_t Py_HashPointer(const void *ptr) +{ +#if PY_VERSION_HEX >= 0x030900A4 && !defined(PYPY_VERSION) + return _Py_HashPointer(ptr); +#else + return _Py_HashPointer(_Py_CAST(void*, ptr)); +#endif +} +#endif + + +// Python 3.13a4 added a PyTime API. +// Use the private API added to Python 3.5. +#if PY_VERSION_HEX < 0x030D00A4 && PY_VERSION_HEX >= 0x03050000 +typedef _PyTime_t PyTime_t; +#define PyTime_MIN _PyTime_MIN +#define PyTime_MAX _PyTime_MAX + +static inline double PyTime_AsSecondsDouble(PyTime_t t) +{ return _PyTime_AsSecondsDouble(t); } + +static inline int PyTime_Monotonic(PyTime_t *result) +{ return _PyTime_GetMonotonicClockWithInfo(result, NULL); } + +static inline int PyTime_Time(PyTime_t *result) +{ return _PyTime_GetSystemClockWithInfo(result, NULL); } + +static inline int PyTime_PerfCounter(PyTime_t *result) +{ +#if PY_VERSION_HEX >= 0x03070000 && !defined(PYPY_VERSION) + return _PyTime_GetPerfCounterWithInfo(result, NULL); +#elif PY_VERSION_HEX >= 0x03070000 + // Call time.perf_counter_ns() and convert Python int object to PyTime_t. + // Cache time.perf_counter_ns() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter_ns"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + long long value = PyLong_AsLongLong(res); + Py_DECREF(res); + + if (value == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(value) >= sizeof(PyTime_t)); + *result = (PyTime_t)value; + return 0; +#else + // Call time.perf_counter() and convert C double to PyTime_t. + // Cache time.perf_counter() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + double d = PyFloat_AsDouble(res); + Py_DECREF(res); + + if (d == -1.0 && PyErr_Occurred()) { + return -1; + } + + // Avoid floor() to avoid having to link to libm + *result = (PyTime_t)(d * 1e9); + return 0; +#endif +} + +#endif + +// gh-111389 added hash constants to Python 3.13.0a5. These constants were +// added first as private macros to Python 3.4.0b1 and PyPy 7.3.9. +#if (!defined(PyHASH_BITS) \ + && ((!defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x030400B1) \ + || (defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x03070000 \ + && PYPY_VERSION_NUM >= 0x07090000))) +# define PyHASH_BITS _PyHASH_BITS +# define PyHASH_MODULUS _PyHASH_MODULUS +# define PyHASH_INF _PyHASH_INF +# define PyHASH_IMAG _PyHASH_IMAG +#endif + + +// gh-111545 added Py_GetConstant() and Py_GetConstantBorrowed() +// to Python 3.13.0a6 +#if PY_VERSION_HEX < 0x030D00A6 && !defined(Py_CONSTANT_NONE) + +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 + +static inline PyObject* Py_GetConstant(unsigned int constant_id) +{ + static PyObject* constants[Py_CONSTANT_EMPTY_TUPLE + 1] = {NULL}; + + if (constants[Py_CONSTANT_NONE] == NULL) { + constants[Py_CONSTANT_NONE] = Py_None; + constants[Py_CONSTANT_FALSE] = Py_False; + constants[Py_CONSTANT_TRUE] = Py_True; + constants[Py_CONSTANT_ELLIPSIS] = Py_Ellipsis; + constants[Py_CONSTANT_NOT_IMPLEMENTED] = Py_NotImplemented; + + constants[Py_CONSTANT_ZERO] = PyLong_FromLong(0); + if (constants[Py_CONSTANT_ZERO] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_ONE] = PyLong_FromLong(1); + if (constants[Py_CONSTANT_ONE] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_STR] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_BYTES] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); + if (constants[Py_CONSTANT_EMPTY_TUPLE] == NULL) { + goto fatal_error; + } + // goto dance to avoid compiler warnings about Py_FatalError() + goto init_done; + +fatal_error: + // This case should never happen + Py_FatalError("Py_GetConstant() failed to get constants"); + } + +init_done: + if (constant_id <= Py_CONSTANT_EMPTY_TUPLE) { + return Py_NewRef(constants[constant_id]); + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + +static inline PyObject* Py_GetConstantBorrowed(unsigned int constant_id) +{ + PyObject *obj = Py_GetConstant(constant_id); + Py_XDECREF(obj); + return obj; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t index) +{ + PyObject *item = PyList_GetItem(op, index); + Py_XINCREF(item); + return item; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + PyObject *value; + if (PyDict_GetItemRef(d, key, &value) < 0) { + // get error + if (result) { + *result = NULL; + } + return -1; + } + if (value != NULL) { + // present + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; + } + + // missing: set the item + if (PyDict_SetItem(d, key, default_value) < 0) { + // set error + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = Py_NewRef(default_value); + } + return 0; +} +#endif + +#if PY_VERSION_HEX < 0x030D00B3 +# define Py_BEGIN_CRITICAL_SECTION(op) { +# define Py_END_CRITICAL_SECTION() } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) { +# define Py_END_CRITICAL_SECTION2() } +#endif + +#if PY_VERSION_HEX < 0x030E0000 && PY_VERSION_HEX >= 0x03060000 && !defined(PYPY_VERSION) +typedef struct PyUnicodeWriter PyUnicodeWriter; + +static inline void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + PyMem_Free(writer); +} + +static inline PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == _Py_NULL) { + PyErr_NoMemory(); + return _Py_NULL; + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + return pub_writer; +} + +static inline PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + PyMem_Free(writer); + return str; +} + +static inline int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > 0x10ffff) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + +static inline int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Repr(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)strlen(str); + } + + PyObject *str_obj = PyUnicode_FromStringAndSize(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, + const wchar_t *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)wcslen(str); + } + + PyObject *str_obj = PyUnicode_FromWideChar(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + +static inline int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + PyObject *str = PyUnicode_FromFormatV(format, vargs); + va_end(vargs); + if (str == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} +#endif // PY_VERSION_HEX < 0x030E0000 + +// gh-116560 added PyLong_GetSign() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyLong_GetSign(PyObject *obj, int *sign) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + + *sign = _PyLong_Sign(obj); + return 0; +} +#endif + + +// gh-124502 added PyUnicode_Equal() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyUnicode_Equal(PyObject *str1, PyObject *str2) +{ + if (!PyUnicode_Check(str1)) { + PyErr_Format(PyExc_TypeError, "first argument must be str, not %s", + Py_TYPE(str1)->tp_name); + return -1; + } + if (!PyUnicode_Check(str2)) { + PyErr_Format(PyExc_TypeError, "second argument must be str, not %s", + Py_TYPE(str2)->tp_name); + return -1; + } + +#if PY_VERSION_HEX >= 0x030d0000 && !defined(PYPY_VERSION) + PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *str1, PyObject *str2); + + return _PyUnicode_Equal(str1, str2); +#elif PY_VERSION_HEX >= 0x03060000 && !defined(PYPY_VERSION) + return _PyUnicode_EQ(str1, str2); +#elif PY_VERSION_HEX >= 0x03090000 && defined(PYPY_VERSION) + return _PyUnicode_EQ(str1, str2); +#else + return (PyUnicode_Compare(str1, str2) == 0); +#endif +} +#endif + + +// gh-121645 added PyBytes_Join() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) +{ + return _PyBytes_Join(sep, iterable); +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline Py_hash_t Py_HashBuffer(const void *ptr, Py_ssize_t len) +{ +#if PY_VERSION_HEX >= 0x03000000 && !defined(PYPY_VERSION) + PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void *src, Py_ssize_t len); + + return _Py_HashBytes(ptr, len); +#else + Py_hash_t hash; + PyObject *bytes = PyBytes_FromStringAndSize((const char*)ptr, len); + if (bytes == NULL) { + return -1; + } + hash = PyObject_Hash(bytes); + Py_DECREF(bytes); + return hash; +#endif +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyIter_NextItem(PyObject *iter, PyObject **item) +{ + iternextfunc tp_iternext; + + assert(iter != NULL); + assert(item != NULL); + + tp_iternext = Py_TYPE(iter)->tp_iternext; + if (tp_iternext == NULL) { + *item = NULL; + PyErr_Format(PyExc_TypeError, "expected an iterator, got '%s'", + Py_TYPE(iter)->tp_name); + return -1; + } + + if ((*item = tp_iternext(iter))) { + return 1; + } + if (!PyErr_Occurred()) { + return 0; + } + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + return 0; + } + return -1; +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline PyObject* PyLong_FromInt32(int32_t value) +{ + Py_BUILD_ASSERT(sizeof(long) >= 4); + return PyLong_FromLong(value); +} + +static inline PyObject* PyLong_FromInt64(int64_t value) +{ + Py_BUILD_ASSERT(sizeof(long long) >= 8); + return PyLong_FromLongLong(value); +} + +static inline PyObject* PyLong_FromUInt32(uint32_t value) +{ + Py_BUILD_ASSERT(sizeof(unsigned long) >= 4); + return PyLong_FromUnsignedLong(value); +} + +static inline PyObject* PyLong_FromUInt64(uint64_t value) +{ + Py_BUILD_ASSERT(sizeof(unsigned long long) >= 8); + return PyLong_FromUnsignedLongLong(value); +} + +static inline int PyLong_AsInt32(PyObject *obj, int32_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(int) == 4); + int value = PyLong_AsInt(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (int32_t)value; + return 0; +} + +static inline int PyLong_AsInt64(PyObject *obj, int64_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long long) == 8); + long long value = PyLong_AsLongLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (int64_t)value; + return 0; +} + +static inline int PyLong_AsUInt32(PyObject *obj, uint32_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long) >= 4); + unsigned long value = PyLong_AsUnsignedLong(obj); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } +#if SIZEOF_LONG > 4 + if ((unsigned long)UINT32_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C uint32_t"); + return -1; + } +#endif + *pvalue = (uint32_t)value; + return 0; +} + +static inline int PyLong_AsUInt64(PyObject *obj, uint64_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long long) == 8); + unsigned long long value = PyLong_AsUnsignedLongLong(obj); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (uint64_t)value; + return 0; +} +#endif + + +#ifdef __cplusplus +} +#endif +#endif // PYTHONCAPI_COMPAT diff --git a/mypyc/lib-rt/pythonsupport.c b/mypyc/lib-rt/pythonsupport.c new file mode 100644 index 000000000000..90fb69705a00 --- /dev/null +++ b/mypyc/lib-rt/pythonsupport.c @@ -0,0 +1,106 @@ +// Collects code that was copied in from cpython, for a couple of different reasons: +// * We wanted to modify it to produce a more efficient version for our uses +// * We needed to call it and it was static :( +// * We wanted to call it and needed to backport it + +#include "pythonsupport.h" + +#if CPY_3_12_FEATURES + +// Slow path of CPyLong_AsSsize_tAndOverflow (non-inlined) +Py_ssize_t +CPyLong_AsSsize_tAndOverflow_(PyObject *vv, int *overflow) +{ + PyLongObject *v = (PyLongObject *)vv; + size_t x, prev; + Py_ssize_t res; + Py_ssize_t i; + int sign; + + *overflow = 0; + + res = -1; + i = CPY_LONG_TAG(v); + + sign = 1; + x = 0; + if (i & CPY_SIGN_NEGATIVE) { + sign = -1; + } + i >>= CPY_NON_SIZE_BITS; + while (--i >= 0) { + prev = x; + x = (x << PyLong_SHIFT) + CPY_LONG_DIGIT(v, i); + if ((x >> PyLong_SHIFT) != prev) { + *overflow = sign; + goto exit; + } + } + /* Haven't lost any bits, but casting to long requires extra + * care. + */ + if (x <= (size_t)CPY_TAGGED_MAX) { + res = (Py_ssize_t)x * sign; + } + else if (sign < 0 && x == CPY_TAGGED_ABS_MIN) { + res = CPY_TAGGED_MIN; + } + else { + *overflow = sign; + /* res is already set to -1 */ + } + exit: + return res; +} + +#else + +// Slow path of CPyLong_AsSsize_tAndOverflow (non-inlined, Python 3.11 and earlier) +Py_ssize_t +CPyLong_AsSsize_tAndOverflow_(PyObject *vv, int *overflow) +{ + /* This version by Tim Peters */ + PyLongObject *v = (PyLongObject *)vv; + size_t x, prev; + Py_ssize_t res; + Py_ssize_t i; + int sign; + + *overflow = 0; + + res = -1; + i = Py_SIZE(v); + + sign = 1; + x = 0; + if (i < 0) { + sign = -1; + i = -(i); + } + while (--i >= 0) { + prev = x; + x = (x << PyLong_SHIFT) + CPY_LONG_DIGIT(v, i); + if ((x >> PyLong_SHIFT) != prev) { + *overflow = sign; + goto exit; + } + } + /* Haven't lost any bits, but casting to long requires extra + * care. + */ + if (x <= (size_t)CPY_TAGGED_MAX) { + res = (Py_ssize_t)x * sign; + } + else if (sign < 0 && x == CPY_TAGGED_ABS_MIN) { + res = CPY_TAGGED_MIN; + } + else { + *overflow = sign; + /* res is already set to -1 */ + } + exit: + return res; +} + + +#endif diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h new file mode 100644 index 000000000000..61929f512608 --- /dev/null +++ b/mypyc/lib-rt/pythonsupport.h @@ -0,0 +1,489 @@ +// Collects code that was copied in from cpython, for a couple of different reasons: +// * We wanted to modify it to produce a more efficient version for our uses +// * We needed to call it and it was static :( +// * We wanted to call it and needed to backport it + +#ifndef CPY_PYTHONSUPPORT_H +#define CPY_PYTHONSUPPORT_H + +#include +#include +#include "pythoncapi_compat.h" +#include +#include +#include "mypyc_util.h" + +#if CPY_3_13_FEATURES +#ifndef Py_BUILD_CORE +#define Py_BUILD_CORE +#endif +#include "internal/pycore_call.h" // _PyObject_CallMethodIdNoArgs, _PyObject_CallMethodIdOneArg +#include "internal/pycore_genobject.h" // _PyGen_FetchStopIterationValue +#include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause, _PyErr_SetKeyError +#include "internal/pycore_setobject.h" // _PySet_Update +#endif + +#if CPY_3_12_FEATURES +#include "internal/pycore_frame.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif +#if 0 +} // why isn't emacs smart enough to not indent this +#endif + +///////////////////////////////////////// +// Adapted from bltinmodule.c in Python 3.7.0 +_Py_IDENTIFIER(__mro_entries__); +static PyObject* +update_bases(PyObject *bases) +{ + Py_ssize_t i, j; + PyObject *base, *meth, *new_base, *result, *new_bases = NULL; + PyObject *stack[1] = {bases}; + assert(PyTuple_Check(bases)); + + Py_ssize_t nargs = PyTuple_GET_SIZE(bases); + for (i = 0; i < nargs; i++) { + base = PyTuple_GET_ITEM(bases, i); + if (PyType_Check(base)) { + if (new_bases) { + /* If we already have made a replacement, then we append every normal base, + otherwise just skip it. */ + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + if (PyObject_GetOptionalAttrString(base, PyId___mro_entries__.string, &meth) < 0) { + goto error; + } + if (!meth) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + new_base = _PyObject_Vectorcall(meth, stack, 1, NULL); + Py_DECREF(meth); + if (!new_base) { + goto error; + } + if (!PyTuple_Check(new_base)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(new_base); + goto error; + } + if (!new_bases) { + /* If this is a first successful replacement, create new_bases list and + copy previously encountered bases. */ + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + base = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base); + Py_INCREF(base); + } + } + j = PyList_GET_SIZE(new_bases); + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); + } + if (!new_bases) { + return bases; + } + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); + return result; + +error: + Py_XDECREF(new_bases); + return NULL; +} + +// From Python 3.7's typeobject.c +_Py_IDENTIFIER(__init_subclass__); +static int +init_subclass(PyTypeObject *type, PyObject *kwds) +{ + PyObject *super, *func, *result; + PyObject *args[2] = {(PyObject *)type, (PyObject *)type}; + + super = _PyObject_Vectorcall((PyObject *)&PySuper_Type, args, 2, NULL); + if (super == NULL) { + return -1; + } + + func = _PyObject_GetAttrId(super, &PyId___init_subclass__); + Py_DECREF(super); + if (func == NULL) { + return -1; + } + + result = _PyObject_FastCallDict(func, NULL, 0, kwds); + Py_DECREF(func); + if (result == NULL) { + return -1; + } + + Py_DECREF(result); + return 0; +} + +Py_ssize_t +CPyLong_AsSsize_tAndOverflow_(PyObject *vv, int *overflow); + +#if CPY_3_12_FEATURES + +static inline Py_ssize_t +CPyLong_AsSsize_tAndOverflow(PyObject *vv, int *overflow) +{ + /* This version by Tim Peters */ + PyLongObject *v = (PyLongObject *)vv; + Py_ssize_t res; + Py_ssize_t i; + + *overflow = 0; + + res = -1; + i = CPY_LONG_TAG(v); + + // TODO: Combine zero and non-zero cases helow? + if (likely(i == (1 << CPY_NON_SIZE_BITS))) { + res = CPY_LONG_DIGIT(v, 0); + } else if (likely(i == CPY_SIGN_ZERO)) { + res = 0; + } else if (i == ((1 << CPY_NON_SIZE_BITS) | CPY_SIGN_NEGATIVE)) { + res = -(sdigit)CPY_LONG_DIGIT(v, 0); + } else { + // Slow path is moved to a non-inline helper function to + // limit size of generated code + int overflow_local; + res = CPyLong_AsSsize_tAndOverflow_(vv, &overflow_local); + *overflow = overflow_local; + } + return res; +} + +#else + +// Adapted from longobject.c in Python 3.7.0 + +/* This function adapted from PyLong_AsLongLongAndOverflow, but with + * some safety checks removed and specialized to only work for objects + * that are already longs. + * About half of the win this provides, though, just comes from being + * able to inline the function, which in addition to saving function call + * overhead allows the out-parameter overflow flag to be collapsed into + * control flow. + * Additionally, we check against the possible range of CPyTagged, not of + * Py_ssize_t. */ +static inline Py_ssize_t +CPyLong_AsSsize_tAndOverflow(PyObject *vv, int *overflow) +{ + /* This version by Tim Peters */ + PyLongObject *v = (PyLongObject *)vv; + Py_ssize_t res; + Py_ssize_t i; + + *overflow = 0; + + res = -1; + i = Py_SIZE(v); + + if (likely(i == 1)) { + res = CPY_LONG_DIGIT(v, 0); + } else if (likely(i == 0)) { + res = 0; + } else if (i == -1) { + res = -(sdigit)CPY_LONG_DIGIT(v, 0); + } else { + // Slow path is moved to a non-inline helper function to + // limit size of generated code + int overflow_local; + res = CPyLong_AsSsize_tAndOverflow_(vv, &overflow_local); + *overflow = overflow_local; + } + return res; +} + +#endif + +// Adapted from listobject.c in Python 3.7.0 +static int +list_resize(PyListObject *self, Py_ssize_t newsize) +{ + PyObject **items; + size_t new_allocated, num_allocated_bytes; + Py_ssize_t allocated = self->allocated; + + /* Bypass realloc() when a previous overallocation is large enough + to accommodate the newsize. If the newsize falls lower than half + the allocated size, then proceed with the realloc() to shrink the list. + */ + if (allocated >= newsize && newsize >= (allocated >> 1)) { + assert(self->ob_item != NULL || newsize == 0); + Py_SET_SIZE(self, newsize); + return 0; + } + + /* This over-allocates proportional to the list size, making room + * for additional growth. The over-allocation is mild, but is + * enough to give linear-time amortized behavior over a long + * sequence of appends() in the presence of a poorly-performing + * system realloc(). + * The growth pattern is: 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ... + * Note: new_allocated won't overflow because the largest possible value + * is PY_SSIZE_T_MAX * (9 / 8) + 6 which always fits in a size_t. + */ + new_allocated = (size_t)newsize + (newsize >> 3) + (newsize < 9 ? 3 : 6); + if (new_allocated > (size_t)PY_SSIZE_T_MAX / sizeof(PyObject *)) { + PyErr_NoMemory(); + return -1; + } + + if (newsize == 0) + new_allocated = 0; + num_allocated_bytes = new_allocated * sizeof(PyObject *); + items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes); + if (items == NULL) { + PyErr_NoMemory(); + return -1; + } + self->ob_item = items; + Py_SET_SIZE(self, newsize); + self->allocated = new_allocated; + return 0; +} + +// Changed to use PyList_SetSlice instead of the internal list_ass_slice +static PyObject * +list_pop_impl(PyListObject *self, Py_ssize_t index) +{ + PyObject *v; + int status; + + if (Py_SIZE(self) == 0) { + /* Special-case most common failure cause */ + PyErr_SetString(PyExc_IndexError, "pop from empty list"); + return NULL; + } + if (index < 0) + index += Py_SIZE(self); + if (index < 0 || index >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "pop index out of range"); + return NULL; + } + v = self->ob_item[index]; + if (index == Py_SIZE(self) - 1) { + status = list_resize(self, Py_SIZE(self) - 1); + if (status >= 0) + return v; /* and v now owns the reference the list had */ + else + return NULL; + } + Py_INCREF(v); + status = PyList_SetSlice((PyObject *)self, index, index+1, (PyObject *)NULL); + if (status < 0) { + Py_DECREF(v); + return NULL; + } + return v; +} + +// Tweaked to directly use CPyTagged +static CPyTagged +list_count(PyListObject *self, PyObject *value) +{ + Py_ssize_t count = 0; + Py_ssize_t i; + + for (i = 0; i < Py_SIZE(self); i++) { + int cmp = PyObject_RichCompareBool(self->ob_item[i], value, Py_EQ); + if (cmp > 0) + count++; + else if (cmp < 0) + return CPY_INT_TAG; + } + return CPyTagged_ShortFromSsize_t(count); +} + +// Adapted from genobject.c in Python 3.7.2 +// Copied because it wasn't in 3.5.2 and it is undocumented anyways. +/* + * Set StopIteration with specified value. Value can be arbitrary object + * or NULL. + * + * Returns 0 if StopIteration is set and -1 if any other exception is set. + */ +static int +CPyGen_SetStopIterationValue(PyObject *value) +{ + PyObject *e; + + if (value == NULL || + (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) + { + /* Delay exception instantiation if we can */ + PyErr_SetObject(PyExc_StopIteration, value); + return 0; + } + /* Construct an exception instance manually with + * PyObject_CallOneArg and pass it to PyErr_SetObject. + * + * We do this to handle a situation when "value" is a tuple, in which + * case PyErr_SetObject would set the value of StopIteration to + * the first element of the tuple. + * + * (See PyErr_SetObject/_PyErr_CreateException code for details.) + */ + e = PyObject_CallOneArg(PyExc_StopIteration, value); + if (e == NULL) { + return -1; + } + PyErr_SetObject(PyExc_StopIteration, e); + Py_DECREF(e); + return 0; +} + +// Copied from dictobject.c and dictobject.h, these are not Public before +// Python 3.8. Also remove some error checks that we do in the callers. +typedef struct { + PyObject_HEAD + PyDictObject *dv_dict; +} _CPyDictViewObject; + +static PyObject * +_CPyDictView_New(PyObject *dict, PyTypeObject *type) +{ + _CPyDictViewObject *dv = PyObject_GC_New(_CPyDictViewObject, type); + if (dv == NULL) + return NULL; + Py_INCREF(dict); + dv->dv_dict = (PyDictObject *)dict; + PyObject_GC_Track(dv); + return (PyObject *)dv; +} + +#ifdef __cplusplus +} +#endif + +#if PY_VERSION_HEX >= 0x030A0000 // 3.10 +static int +_CPyObject_HasAttrId(PyObject *v, _Py_Identifier *name) { + PyObject *tmp = NULL; + int result = PyObject_GetOptionalAttrString(v, name->string, &tmp); + if (tmp) { + Py_DECREF(tmp); + } + return result; +} +#else +#define _CPyObject_HasAttrId _PyObject_HasAttrId +#endif + +#if PY_VERSION_HEX < 0x03090000 +// OneArgs and NoArgs functions got added in 3.9 +#define _PyObject_CallMethodIdNoArgs(self, name) \ + _PyObject_CallMethodIdObjArgs((self), (name), NULL) +#define _PyObject_CallMethodIdOneArg(self, name, arg) \ + _PyObject_CallMethodIdObjArgs((self), (name), (arg), NULL) +#define PyObject_CallMethodOneArg(self, name, arg) \ + PyObject_CallMethodObjArgs((self), (name), (arg), NULL) +#endif + +#if CPY_3_12_FEATURES + +// These are copied from genobject.c in Python 3.12 + +static int +gen_is_coroutine(PyObject *o) +{ + if (PyGen_CheckExact(o)) { + PyCodeObject *code = PyGen_GetCode((PyGenObject*)o); + if (code->co_flags & CO_ITERABLE_COROUTINE) { + return 1; + } + } + return 0; +} + +#else + +// Copied from genobject.c in Python 3.10 +static int +gen_is_coroutine(PyObject *o) +{ + if (PyGen_CheckExact(o)) { + PyCodeObject *code = (PyCodeObject *)((PyGenObject*)o)->gi_code; + if (code->co_flags & CO_ITERABLE_COROUTINE) { + return 1; + } + } + return 0; +} + +#endif + +/* + * This helper function returns an awaitable for `o`: + * - `o` if `o` is a coroutine-object; + * - `type(o)->tp_as_async->am_await(o)` + * + * Raises a TypeError if it's not possible to return + * an awaitable and returns NULL. + */ +static PyObject * +CPyCoro_GetAwaitableIter(PyObject *o) +{ + unaryfunc getter = NULL; + PyTypeObject *ot; + + if (PyCoro_CheckExact(o) || gen_is_coroutine(o)) { + /* 'o' is a coroutine. */ + Py_INCREF(o); + return o; + } + + ot = Py_TYPE(o); + if (ot->tp_as_async != NULL) { + getter = ot->tp_as_async->am_await; + } + if (getter != NULL) { + PyObject *res = (*getter)(o); + if (res != NULL) { + if (PyCoro_CheckExact(res) || gen_is_coroutine(res)) { + /* __await__ must return an *iterator*, not + a coroutine or another awaitable (see PEP 492) */ + PyErr_SetString(PyExc_TypeError, + "__await__() returned a coroutine"); + Py_CLEAR(res); + } else if (!PyIter_Check(res)) { + PyErr_Format(PyExc_TypeError, + "__await__() returned non-iterator " + "of type '%.100s'", + Py_TYPE(res)->tp_name); + Py_CLEAR(res); + } + } + return res; + } + + PyErr_Format(PyExc_TypeError, + "object %.100s can't be used in 'await' expression", + ot->tp_name); + return NULL; +} + + +#endif diff --git a/mypyc/lib-rt/set_ops.c b/mypyc/lib-rt/set_ops.c new file mode 100644 index 000000000000..7e769674f985 --- /dev/null +++ b/mypyc/lib-rt/set_ops.c @@ -0,0 +1,17 @@ +// Set primitive operations +// +// These are registered in mypyc.primitives.set_ops. + +#include +#include "CPy.h" + +bool CPySet_Remove(PyObject *set, PyObject *key) { + int success = PySet_Discard(set, key); + if (success == 1) { + return true; + } + if (success == 0) { + _PyErr_SetKeyError(key); + } + return false; +} diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py new file mode 100644 index 000000000000..1faacc8fc136 --- /dev/null +++ b/mypyc/lib-rt/setup.py @@ -0,0 +1,71 @@ +"""Build script for mypyc C runtime library unit tests. + +The tests are written in C++ and use the Google Test framework. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from distutils.command.build_ext import build_ext +from distutils.core import Extension, setup +from typing import Any + +kwargs: dict[str, Any] +if sys.platform == "darwin": + kwargs = {"language": "c++"} + compile_args = [] +else: + kwargs = {} + compile_args = ["--std=c++11"] + + +class build_ext_custom(build_ext): # noqa: N801 + def get_library_names(self): + return ["gtest"] + + def run(self): + gtest_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "external", "googletest") + ) + + os.makedirs(self.build_temp, exist_ok=True) + + # Build Google Test, the C++ framework we use for testing C code. + # The source code for Google Test is copied to this repository. + subprocess.check_call( + ["make", "-f", os.path.join(gtest_dir, "make", "Makefile"), f"GTEST_DIR={gtest_dir}"], + cwd=self.build_temp, + ) + + self.library_dirs = [self.build_temp] + + return build_ext.run(self) + + +setup( + name="test_capi", + version="0.1", + ext_modules=[ + Extension( + "test_capi", + [ + "test_capi.cc", + "init.c", + "int_ops.c", + "float_ops.c", + "list_ops.c", + "exc_ops.c", + "generic_ops.c", + "pythonsupport.c", + ], + depends=["CPy.h", "mypyc_util.h", "pythonsupport.h"], + extra_compile_args=["-Wno-unused-function", "-Wno-sign-compare"] + compile_args, + libraries=["gtest"], + include_dirs=["../external/googletest", "../external/googletest/include"], + **kwargs, + ) + ], + cmdclass={"build_ext": build_ext_custom}, +) diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c new file mode 100644 index 000000000000..68026037502d --- /dev/null +++ b/mypyc/lib-rt/str_ops.c @@ -0,0 +1,257 @@ +// String primitive operations +// +// These are registered in mypyc.primitives.str_ops. + +#include +#include "CPy.h" + +PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { + if (PyUnicode_READY(str) != -1) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyUnicode_GET_LENGTH(str); + if (n < 0) + n += size; + if (n < 0 || n >= size) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return NULL; + } + enum PyUnicode_Kind kind = (enum PyUnicode_Kind)PyUnicode_KIND(str); + void *data = PyUnicode_DATA(str); + Py_UCS4 ch = PyUnicode_READ(kind, data, n); + PyObject *unicode = PyUnicode_New(1, ch); + if (unicode == NULL) + return NULL; + + if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) { + PyUnicode_1BYTE_DATA(unicode)[0] = (Py_UCS1)ch; + } else if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { + PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch; + } else { + assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); + PyUnicode_4BYTE_DATA(unicode)[0] = ch; + } + return unicode; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } + } else { + PyObject *index_obj = CPyTagged_AsObject(index); + return PyObject_GetItem(str, index_obj); + } +} + +// A simplification of _PyUnicode_JoinArray() from CPython 3.9.6 +PyObject *CPyStr_Build(Py_ssize_t len, ...) { + Py_ssize_t i; + va_list args; + + // Calculate the total amount of space and check + // whether all components have the same kind. + Py_ssize_t sz = 0; + Py_UCS4 maxchar = 0; + int use_memcpy = 1; // Use memcpy by default + PyObject *last_obj = NULL; + + va_start(args, len); + for (i = 0; i < len; i++) { + PyObject *item = va_arg(args, PyObject *); + if (!PyUnicode_Check(item)) { + PyErr_Format(PyExc_TypeError, + "sequence item %zd: expected str instance," + " %.80s found", + i, Py_TYPE(item)->tp_name); + return NULL; + } + if (PyUnicode_READY(item) == -1) + return NULL; + + size_t add_sz = PyUnicode_GET_LENGTH(item); + Py_UCS4 item_maxchar = PyUnicode_MAX_CHAR_VALUE(item); + maxchar = Py_MAX(maxchar, item_maxchar); + + // Using size_t to avoid overflow during arithmetic calculation + if (add_sz > (size_t)(PY_SSIZE_T_MAX - sz)) { + PyErr_SetString(PyExc_OverflowError, + "join() result is too long for a Python string"); + return NULL; + } + sz += add_sz; + + // If these strings have different kind, we would call + // _PyUnicode_FastCopyCharacters() in the following part. + if (use_memcpy && last_obj != NULL) { + if (PyUnicode_KIND(last_obj) != PyUnicode_KIND(item)) + use_memcpy = 0; + } + last_obj = item; + } + va_end(args); + + // Construct the string + PyObject *res = PyUnicode_New(sz, maxchar); + if (res == NULL) + return NULL; + + if (use_memcpy) { + unsigned char *res_data = PyUnicode_1BYTE_DATA(res); + unsigned int kind = PyUnicode_KIND(res); + + va_start(args, len); + for (i = 0; i < len; ++i) { + PyObject *item = va_arg(args, PyObject *); + Py_ssize_t itemlen = PyUnicode_GET_LENGTH(item); + if (itemlen != 0) { + memcpy(res_data, PyUnicode_DATA(item), kind * itemlen); + res_data += kind * itemlen; + } + } + va_end(args); + assert(res_data == PyUnicode_1BYTE_DATA(res) + kind * PyUnicode_GET_LENGTH(res)); + } else { + Py_ssize_t res_offset = 0; + + va_start(args, len); + for (i = 0; i < len; ++i) { + PyObject *item = va_arg(args, PyObject *); + Py_ssize_t itemlen = PyUnicode_GET_LENGTH(item); + if (itemlen != 0) { +#if CPY_3_13_FEATURES + PyUnicode_CopyCharacters(res, res_offset, item, 0, itemlen); +#else + _PyUnicode_FastCopyCharacters(res, res_offset, item, 0, itemlen); +#endif + res_offset += itemlen; + } + } + va_end(args); + assert(res_offset == PyUnicode_GET_LENGTH(res)); + } + + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; +} + +PyObject *CPyStr_Split(PyObject *str, PyObject *sep, CPyTagged max_split) { + Py_ssize_t temp_max_split = CPyTagged_AsSsize_t(max_split); + if (temp_max_split == -1 && PyErr_Occurred()) { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } + return PyUnicode_Split(str, sep, temp_max_split); +} + +PyObject *CPyStr_Replace(PyObject *str, PyObject *old_substr, + PyObject *new_substr, CPyTagged max_replace) { + Py_ssize_t temp_max_replace = CPyTagged_AsSsize_t(max_replace); + if (temp_max_replace == -1 && PyErr_Occurred()) { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } + return PyUnicode_Replace(str, old_substr, new_substr, temp_max_replace); +} + +bool CPyStr_Startswith(PyObject *self, PyObject *subobj) { + Py_ssize_t start = 0; + Py_ssize_t end = PyUnicode_GET_LENGTH(self); + return PyUnicode_Tailmatch(self, subobj, start, end, -1); +} + +bool CPyStr_Endswith(PyObject *self, PyObject *subobj) { + Py_ssize_t start = 0; + Py_ssize_t end = PyUnicode_GET_LENGTH(self); + return PyUnicode_Tailmatch(self, subobj, start, end, 1); +} + +/* This does a dodgy attempt to append in place */ +PyObject *CPyStr_Append(PyObject *o1, PyObject *o2) { + PyUnicode_Append(&o1, o2); + return o1; +} + +PyObject *CPyStr_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { + if (likely(PyUnicode_CheckExact(obj) + && CPyTagged_CheckShort(start) && CPyTagged_CheckShort(end))) { + Py_ssize_t startn = CPyTagged_ShortAsSsize_t(start); + Py_ssize_t endn = CPyTagged_ShortAsSsize_t(end); + if (startn < 0) { + startn += PyUnicode_GET_LENGTH(obj); + if (startn < 0) { + startn = 0; + } + } + if (endn < 0) { + endn += PyUnicode_GET_LENGTH(obj); + if (endn < 0) { + endn = 0; + } + } + return PyUnicode_Substring(obj, startn, endn); + } + return CPyObject_GetSlice(obj, start, end); +} + +/* Check if the given string is true (i.e. its length isn't zero) */ +bool CPyStr_IsTrue(PyObject *obj) { + Py_ssize_t length = PyUnicode_GET_LENGTH(obj); + return length != 0; +} + +Py_ssize_t CPyStr_Size_size_t(PyObject *str) { + if (PyUnicode_READY(str) != -1) { + return PyUnicode_GET_LENGTH(str); + } + return -1; +} + +PyObject *CPy_Decode(PyObject *obj, PyObject *encoding, PyObject *errors) { + const char *enc = NULL; + const char *err = NULL; + if (encoding) { + enc = PyUnicode_AsUTF8AndSize(encoding, NULL); + if (!enc) return NULL; + } + if (errors) { + err = PyUnicode_AsUTF8AndSize(errors, NULL); + if (!err) return NULL; + } + if (PyBytes_Check(obj)) { + return PyUnicode_Decode(((PyBytesObject *)obj)->ob_sval, + ((PyVarObject *)obj)->ob_size, + enc, err); + } else { + return PyUnicode_FromEncodedObject(obj, enc, err); + } +} + +PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors) { + const char *enc = NULL; + const char *err = NULL; + if (encoding) { + enc = PyUnicode_AsUTF8AndSize(encoding, NULL); + if (!enc) return NULL; + } + if (errors) { + err = PyUnicode_AsUTF8AndSize(errors, NULL); + if (!err) return NULL; + } + if (PyUnicode_Check(obj)) { + return PyUnicode_AsEncodedString(obj, enc, err); + } else { + PyErr_BadArgument(); + return NULL; + } +} + + +CPyTagged CPyStr_Ord(PyObject *obj) { + Py_ssize_t s = PyUnicode_GET_LENGTH(obj); + if (s == 1) { + int kind = PyUnicode_KIND(obj); + return PyUnicode_READ(kind, PyUnicode_DATA(obj), 0) << 1; + } + PyErr_Format( + PyExc_TypeError, "ord() expected a character, but a string of length %zd found", s); + return CPY_INT_TAG; +} diff --git a/mypyc/lib-rt/test_capi.cc b/mypyc/lib-rt/test_capi.cc new file mode 100644 index 000000000000..4b183de5743e --- /dev/null +++ b/mypyc/lib-rt/test_capi.cc @@ -0,0 +1,585 @@ +// Test cases + +#include +#include +#include "CPy.h" + +static PyObject *moduleDict; + +static PyObject *int_from_str(const char *str) { + return PyLong_FromString(str, 0, 10); +} + +static std::string str_from_object(PyObject *x) { + PyObject *str = PyObject_Str(x); + const char *utf8 = PyUnicode_AsUTF8(str); + return std::string(utf8); +} + +static std::string str_from_int(CPyTagged x) { + return str_from_object(CPyTagged_AsObject(x)); +} + +static bool is_py_equal(PyObject *x, PyObject *y) { + int result = PyObject_RichCompareBool(x, y, Py_EQ); + if (result < 0) { + std::cout << "ERROR: Rich compare failed"; + } + return result == 1; +} + +static bool is_int_equal(CPyTagged x, CPyTagged y) { + if (CPyTagged_CheckShort(x)) { + return x == y; + } else if (CPyTagged_CheckShort(y)) { + return false; + } else { + return is_py_equal(CPyTagged_LongAsObject(x), CPyTagged_LongAsObject(y)); + } +} + +static void fail(std::string message) { + std::cerr << message << "\n"; + exit(1); +} + +static PyObject *eval(std::string expr) { + PyObject *dict = PyDict_New(); + auto result = PyRun_String(expr.c_str(), Py_eval_input, moduleDict, dict); + Py_DECREF(dict); + if (result == 0) { + fail("Python exception"); + } + return result; +} + +static CPyTagged eval_int(std::string expr) { + auto o = eval(expr); + EXPECT_TRUE(PyLong_Check(o)); + return CPyTagged_FromObject(o); +} + +static PyObject *empty_list() { + PyObject *list = PyList_New(0); + EXPECT_TRUE(list); + return list; +} + +static void list_append(PyObject *list, std::string expr) { + PyObject *obj = eval(expr); + int result = PyList_Append(list, obj); + EXPECT_TRUE(result == 0); +} + +class CAPITest : public ::testing::Test { +protected: + PyObject *max_short; + PyObject *min_short; + PyObject *min_pos_long; + PyObject *max_neg_long; + Py_ssize_t c_max_short; + Py_ssize_t c_min_short; + Py_ssize_t c_min_pos_long; + Py_ssize_t c_max_neg_long; + + virtual void SetUp() { + if (!moduleDict) { + fail("Could not find module dictionary"); + } + + c_max_short = CPY_TAGGED_MAX; // 2**62-1 + c_min_pos_long = c_max_short + 1; // 2**62 + c_min_short = CPY_TAGGED_MIN; // -2**62 + c_max_neg_long = c_min_short - 1; // -(2**62+1) + max_short = PyLong_FromSsize_t(c_max_short); + min_pos_long = PyLong_FromSsize_t(c_min_pos_long); + min_short = PyLong_FromSsize_t(c_min_short); + max_neg_long = PyLong_FromSsize_t(c_max_neg_long); + } + + virtual void TearDown() { + Py_DECREF(max_short); + Py_DECREF(min_pos_long); + Py_DECREF(min_short); + Py_DECREF(max_neg_long); + } +}; + +TEST_F(CAPITest, test_cint_conversions) { + EXPECT_EQ(CPyTagged_ShortFromInt(0), 0); + EXPECT_EQ(CPyTagged_ShortFromInt(3), 6); + EXPECT_EQ(CPyTagged_ShortFromInt(-5), -10); + EXPECT_EQ(CPyTagged_ShortAsSsize_t(0), 0); + EXPECT_EQ(CPyTagged_ShortAsSsize_t(6), 3); + EXPECT_EQ(CPyTagged_ShortAsSsize_t(-10), -5); +} + +TEST_F(CAPITest, test_is_long_int) { + EXPECT_TRUE(CPyTagged_CheckLong(1)); + EXPECT_TRUE(CPyTagged_CheckLong(15)); + EXPECT_FALSE(CPyTagged_CheckLong(0)); + EXPECT_FALSE(CPyTagged_CheckLong(6)); + EXPECT_FALSE(CPyTagged_CheckLong(-4)); +} + +TEST_F(CAPITest, test_is_short_int) { + EXPECT_FALSE(CPyTagged_CheckShort(1)); + EXPECT_FALSE(CPyTagged_CheckShort(15)); + EXPECT_TRUE(CPyTagged_CheckShort(0)); + EXPECT_TRUE(CPyTagged_CheckShort(6)); + EXPECT_TRUE(CPyTagged_CheckShort(-4)); +} + +TEST_F(CAPITest, test_obj_to_short_int) { + EXPECT_EQ(CPyTagged_FromObject(int_from_str("0")), CPyTagged_ShortFromInt(0)); + EXPECT_EQ(CPyTagged_FromObject(int_from_str("1234")), CPyTagged_ShortFromInt(1234)); + EXPECT_EQ(CPyTagged_FromObject(int_from_str("-1234")), CPyTagged_ShortFromInt(-1234)); + + EXPECT_EQ(CPyTagged_FromObject(max_short), CPyTagged_ShortFromSsize_t(c_max_short)); + EXPECT_EQ(CPyTagged_FromObject(min_short), CPyTagged_ShortFromSsize_t(c_min_short)); +} + +TEST_F(CAPITest, test_obj_to_long_int) { + // A value larger than 2**64 + PyObject *large = int_from_str("18464758493694263305"); + PyObject *small = int_from_str("-18464758493694263305"); + CPyTagged x; + + x = CPyTagged_FromObject(large); + ASSERT_TRUE(CPyTagged_CheckLong(x)); + EXPECT_TRUE(is_py_equal(large, CPyTagged_LongAsObject(x))); + + x = CPyTagged_FromObject(small); + ASSERT_TRUE(CPyTagged_CheckLong(x)); + EXPECT_TRUE(is_py_equal(small, CPyTagged_LongAsObject(x))); + + x = CPyTagged_FromObject(min_pos_long); + ASSERT_TRUE(CPyTagged_CheckLong(x)); + EXPECT_TRUE(is_py_equal(min_pos_long, CPyTagged_LongAsObject(x))); + + x = CPyTagged_FromObject(max_neg_long); + ASSERT_TRUE(CPyTagged_CheckLong(x)); + EXPECT_TRUE(is_py_equal(max_neg_long, CPyTagged_LongAsObject(x))); +} + +TEST_F(CAPITest, test_short_int_to_obj) { + EXPECT_TRUE(is_py_equal(CPyTagged_AsObject(CPyTagged_ShortFromInt(0)), int_from_str("0"))); + EXPECT_TRUE(is_py_equal(CPyTagged_AsObject(CPyTagged_ShortFromInt(1234)), + int_from_str("1234"))); + EXPECT_TRUE(is_py_equal(CPyTagged_AsObject(CPyTagged_ShortFromInt(-1234)), + int_from_str("-1234"))); + EXPECT_TRUE(is_py_equal(CPyTagged_AsObject(CPyTagged_ShortFromSsize_t(c_max_short)), + max_short)); + EXPECT_TRUE(is_py_equal(CPyTagged_AsObject(CPyTagged_ShortFromSsize_t(c_min_short)), + min_short)); +} + +TEST_F(CAPITest, test_long_int_to_obj) { + // A value larger than 2**64 + PyObject *large = int_from_str("18464758493694263305"); + PyObject *small = int_from_str("-18464758493694263305"); + PyObject *x; + + x = CPyTagged_AsObject(CPyTagged_FromObject(large)); + EXPECT_TRUE(is_py_equal(large, x)); + x = CPyTagged_AsObject(CPyTagged_FromObject(small)); + EXPECT_TRUE(is_py_equal(small, x)); + x = CPyTagged_AsObject(CPyTagged_FromObject(min_pos_long)); + EXPECT_TRUE(is_py_equal(min_pos_long, x)); + x = CPyTagged_AsObject(CPyTagged_FromObject(max_neg_long)); + EXPECT_TRUE(is_py_equal(max_neg_long, x)); +} + +#define EXPECT_INT_EQUAL(x, y) \ + do { \ + if (!is_int_equal(x, y)) \ + std::cout << "Failure: " << str_from_int(x) << " != " << str_from_int(y) << \ + "\n"; \ + EXPECT_TRUE(is_int_equal(x, y)); \ + } while (false) + +#define ASSERT_ADD(x, y, result) \ + EXPECT_TRUE(is_int_equal(CPyTagged_Add(eval_int(x), eval_int(y)), eval_int(result))) + +TEST_F(CAPITest, test_add_short_int) { + ASSERT_ADD("13", "8", "21"); + ASSERT_ADD("-13", "8", "-5"); + ASSERT_ADD("13", "-7", "6"); + ASSERT_ADD("13", "-14", "-1"); + ASSERT_ADD("-3", "-5", "-8"); +} + +TEST_F(CAPITest, test_add_long_ints) { + ASSERT_ADD("2**64", "2**65", "2**64 + 2**65"); + ASSERT_ADD("2**64", "-2**65", "2**64 - 2**65"); +} + +TEST_F(CAPITest, test_add_long_and_short) { + ASSERT_ADD("1", "2**65", "1 + 2**65"); + ASSERT_ADD("2**65", "1", "1 + 2**65"); +} + +TEST_F(CAPITest, test_add_short_overflow) { + // Overfloat + ASSERT_ADD("2**62 - 1", "1", "2**62"); + ASSERT_ADD("-2**62", "-1", "-2**62 - 1"); +} + +TEST_F(CAPITest, test_add_short_edge_cases) { + // Close but not quite overflow + ASSERT_ADD("2**62 - 2", "1", "2**62 - 1"); + ASSERT_ADD("-2**62 + 1", "-1", "-2**62"); + // Max magnitudes + ASSERT_ADD("2**62 - 1", "2**62 - 1", "2**63 - 2"); + ASSERT_ADD("2**62 - 1", "-2**62", "-1"); +} + +#define ASSERT_SUBTRACT(x, y, result) \ + EXPECT_TRUE(is_int_equal(CPyTagged_Subtract(eval_int(x), eval_int(y)), eval_int(result))) + +TEST_F(CAPITest, test_subtract_short_int) { + ASSERT_SUBTRACT("13", "8", "5"); + ASSERT_SUBTRACT("8", "13", "-5"); + ASSERT_SUBTRACT("-13", "8", "-21"); + ASSERT_SUBTRACT("13", "-7", "20"); + ASSERT_SUBTRACT("-3", "-5", "2"); +} + +TEST_F(CAPITest, test_subtract_long_int) { + ASSERT_SUBTRACT("2**65", "2**64", "2**65 - 2**64"); + ASSERT_SUBTRACT("2**65", "-2**64", "2**65 + 2**64"); +} + +TEST_F(CAPITest, test_subtract_long_and_short) { + ASSERT_SUBTRACT("1", "2**65", "1 - 2**65"); + ASSERT_SUBTRACT("2**65", "1", "2**65 - 1"); +} + +TEST_F(CAPITest, test_subtract_short_overflow) { + ASSERT_SUBTRACT("2**62-1", "-1", "2**62"); + ASSERT_SUBTRACT("-2**62", "1", "-2**62 - 1"); + ASSERT_SUBTRACT("0", "-2**62", "2**62"); + ASSERT_SUBTRACT("1", "-2**62 + 1", "2**62"); + ASSERT_SUBTRACT("-2", "2**62 - 1", "-2**62 - 1"); +} + +TEST_F(CAPITest, test_subtract_short_edge_cases) { + // Close but not quite overflow + ASSERT_SUBTRACT("2**62 - 2", "-1", "2**62 - 1"); + ASSERT_SUBTRACT("-2**62 + 1", "1", "-2**62"); + // Max magnitudes + ASSERT_SUBTRACT("2**62 - 1", "-2**62", "2**63 - 1"); + ASSERT_SUBTRACT("-2**62", "2**62 - 1", "-2**63 + 1"); +} + +#define ASSERT_MULTIPLY(x, y, result) \ + EXPECT_TRUE(is_int_equal(CPyTagged_Multiply(eval_int(x), eval_int(y)), eval_int(result))) + +TEST_F(CAPITest, test_multiply_int) { + ASSERT_MULTIPLY("0", "0", "0"); + ASSERT_MULTIPLY("3", "5", "15"); + ASSERT_MULTIPLY("2**40", "2**40", "2**80"); + ASSERT_MULTIPLY("2**30-1", "2**30-1", "(2**30-1)**2"); + ASSERT_MULTIPLY("2**30", "2**30-1", "2**30 * (2**30-1)"); + ASSERT_MULTIPLY("2**30-1", "2**30", "2**30 * (2**30-1)"); + ASSERT_MULTIPLY("2**15", "2**15-1", "2**15 * (2**15-1)"); + ASSERT_MULTIPLY("2**15-1", "2**15", "2**15 * (2**15-1)"); + ASSERT_MULTIPLY("3", "-5", "-15"); + ASSERT_MULTIPLY("-3", "5", "-15"); + ASSERT_MULTIPLY("-3", "-5", "15"); +} + +#define ASSERT_FLOOR_DIV(x, y, result) \ + EXPECT_INT_EQUAL(CPyTagged_FloorDivide(eval_int(x), eval_int(y)), eval_int(result)) + +TEST_F(CAPITest, test_floor_divide_short_int) { + ASSERT_FLOOR_DIV("18", "6", "3"); + ASSERT_FLOOR_DIV("17", "6", "2"); + ASSERT_FLOOR_DIV("12", "6", "2"); + ASSERT_FLOOR_DIV("15", "5", "3"); + ASSERT_FLOOR_DIV("14", "5", "2"); + ASSERT_FLOOR_DIV("11", "5", "2"); + ASSERT_FLOOR_DIV("-18", "6", "-3"); + ASSERT_FLOOR_DIV("-13", "6", "-3"); + ASSERT_FLOOR_DIV("-12", "6", "-2"); + ASSERT_FLOOR_DIV("18", "-6", "-3"); + ASSERT_FLOOR_DIV("13", "-6", "-3"); + ASSERT_FLOOR_DIV("12", "-6", "-2"); + ASSERT_FLOOR_DIV("-3", "-3", "1"); + ASSERT_FLOOR_DIV("-5", "-3", "1"); + ASSERT_FLOOR_DIV("-6", "-3", "2"); + + ASSERT_FLOOR_DIV("2**60", "3", "2**60 // 3"); + ASSERT_FLOOR_DIV("-2**62", "-1", "2**62"); + ASSERT_FLOOR_DIV("-2**62", "1", "-2**62"); + ASSERT_FLOOR_DIV("2**62 - 1", "1", "2**62 - 1"); + ASSERT_FLOOR_DIV("2**62 - 1", "-1", "-2**62 + 1"); + ASSERT_FLOOR_DIV("2**60", "3", "2**60 // 3"); + ASSERT_FLOOR_DIV("-2**30", "-1", "2**30"); + ASSERT_FLOOR_DIV("-2**30", "1", "-2**30"); + ASSERT_FLOOR_DIV("2**30 - 1", "1", "2**30 - 1"); + ASSERT_FLOOR_DIV("2**30 - 1", "-1", "-2**30 + 1"); +} + +TEST_F(CAPITest, test_floor_divide_long_int) { + ASSERT_FLOOR_DIV("2**100", "3", "2**100 // 3"); + ASSERT_FLOOR_DIV("3", "2**100", "0"); + ASSERT_FLOOR_DIV("2**100", "2**70 // 3", "2**100 // (2**70 // 3)"); +} + +#define ASSERT_REMAINDER(x, y, result) \ + EXPECT_INT_EQUAL(CPyTagged_Remainder(eval_int(x), eval_int(y)), eval_int(result)) + +TEST_F(CAPITest, test_remainder_short_int) { + ASSERT_REMAINDER("18", "6", "0"); + ASSERT_REMAINDER("17", "6", "5"); + ASSERT_REMAINDER("13", "6", "1"); + ASSERT_REMAINDER("12", "6", "0"); + ASSERT_REMAINDER("15", "5", "0"); + ASSERT_REMAINDER("14", "5", "4"); + ASSERT_REMAINDER("11", "5", "1"); + ASSERT_REMAINDER("-18", "6", "0"); + ASSERT_REMAINDER("-13", "6", "5"); + ASSERT_REMAINDER("-12", "6", "0"); + ASSERT_REMAINDER("18", "-6", "0"); + ASSERT_REMAINDER("13", "-6", "-5"); + ASSERT_REMAINDER("12", "-6", "0"); + ASSERT_REMAINDER("-3", "-3", "0"); + ASSERT_REMAINDER("-5", "-3", "-2"); + ASSERT_REMAINDER("-6", "-3", "0"); + + ASSERT_REMAINDER("-1", "2**62 - 1", "2**62 - 2"); + ASSERT_REMAINDER("1", "-2**62", "-2**62 + 1"); +} + +TEST_F(CAPITest, test_remainder_long_int) { + ASSERT_REMAINDER("2**100", "3", "2**100 % 3"); + ASSERT_REMAINDER("3", "2**100", "3"); + ASSERT_REMAINDER("2**100", "2**70 // 3", "2**100 % (2**70 // 3)"); +} + +#define INT_EQ(x, y) \ + CPyTagged_IsEq(eval_int(x), eval_int(y)) + +TEST_F(CAPITest, test_int_equality) { + EXPECT_TRUE(INT_EQ("0", "0")); + EXPECT_TRUE(INT_EQ("5", "5")); + EXPECT_TRUE(INT_EQ("-7", "-7")); + EXPECT_TRUE(INT_EQ("2**65 + 0x1234", "2**65 + 0x1234")); + EXPECT_TRUE(INT_EQ("-2**65 + 0x1234", "-2**65 + 0x1234")); + EXPECT_FALSE(INT_EQ("0", "1")); + EXPECT_FALSE(INT_EQ("5", "4")); + EXPECT_FALSE(INT_EQ("-7", "7")); + EXPECT_FALSE(INT_EQ("-7", "-6")); + EXPECT_FALSE(INT_EQ("-7", "-5")); + EXPECT_FALSE(INT_EQ("2**65 + 0x1234", "2**65 + 0x1233")); + EXPECT_FALSE(INT_EQ("2**65 + 0x1234", "2**66 + 0x1234")); + EXPECT_FALSE(INT_EQ("2**65 + 0x1234", "-2**65 - 0x1234")); + EXPECT_FALSE(INT_EQ("-2**65 + 0x1234", "-2**65 + 0x1233")); +} + +#define INT_NE(x, y) \ + CPyTagged_IsNe(eval_int(x), eval_int(y)) + +TEST_F(CAPITest, test_int_non_equality) { + EXPECT_FALSE(INT_NE("0", "0")); + EXPECT_FALSE(INT_NE("5", "5")); + EXPECT_FALSE(INT_NE("-7", "-7")); + EXPECT_FALSE(INT_NE("2**65 + 0x1234", "2**65 + 0x1234")); + EXPECT_FALSE(INT_NE("-2**65 + 0x1234", "-2**65 + 0x1234")); + EXPECT_TRUE(INT_NE("0", "1")); + EXPECT_TRUE(INT_NE("5", "4")); + EXPECT_TRUE(INT_NE("-7", "7")); + EXPECT_TRUE(INT_NE("-7", "-6")); + EXPECT_TRUE(INT_NE("-7", "-5")); + EXPECT_TRUE(INT_NE("2**65 + 0x1234", "2**65 + 0x1233")); + EXPECT_TRUE(INT_NE("2**65 + 0x1234", "2**66 + 0x1234")); + EXPECT_TRUE(INT_NE("2**65 + 0x1234", "-2**65 - 0x1234")); + EXPECT_TRUE(INT_NE("-2**65 + 0x1234", "-2**65 + 0x1233")); +} + +#define INT_LT(x, y) \ + CPyTagged_IsLt(eval_int(x), eval_int(y)) + +TEST_F(CAPITest, test_int_less_than) { + EXPECT_TRUE(INT_LT("0", "5")); + EXPECT_TRUE(INT_LT("4", "5")); + EXPECT_TRUE(INT_LT("-3", "1")); + EXPECT_TRUE(INT_LT("-3", "0")); + EXPECT_TRUE(INT_LT("-3", "-2")); + + EXPECT_FALSE(INT_LT("5", "0")); + EXPECT_FALSE(INT_LT("5", "4")); + EXPECT_FALSE(INT_LT("1", "-3")); + EXPECT_FALSE(INT_LT("0", "-3")); + EXPECT_FALSE(INT_LT("-2", "-3")); + EXPECT_FALSE(INT_LT("-3", "-3")); + EXPECT_FALSE(INT_LT("3", "3")); + + EXPECT_TRUE(INT_LT("5", "2**65")); + EXPECT_TRUE(INT_LT("-2**65", "-5")); + EXPECT_TRUE(INT_LT("-2**66", "2**65")); + EXPECT_TRUE(INT_LT("2**65", "2**66")); + EXPECT_TRUE(INT_LT("-2**66", "-2**65")); + + EXPECT_FALSE(INT_LT("2**65", "5")); + EXPECT_FALSE(INT_LT("-5", "-2**65")); + EXPECT_FALSE(INT_LT("2**65", "-2**66")); + EXPECT_FALSE(INT_LT("2**66", "2**65")); + EXPECT_FALSE(INT_LT("-2**65", "-2**66")); + EXPECT_FALSE(INT_LT("-2**65", "-2**65")); + EXPECT_FALSE(INT_LT("2**65", "2**65")); +} + +#define INT_GE(x, y) \ + CPyTagged_IsGe(eval_int(x), eval_int(y)) + +TEST_F(CAPITest, test_int_greater_than_or_equal) { + EXPECT_TRUE(INT_GE("3", "2")); + EXPECT_TRUE(INT_GE("3", "3")); + EXPECT_FALSE(INT_GE("3", "4")); + EXPECT_TRUE(INT_GE("3", "-4")); + EXPECT_TRUE(INT_GE("-3", "-4")); + EXPECT_TRUE(INT_GE("-3", "-3")); + EXPECT_FALSE(INT_GE("-3", "-2")); + EXPECT_FALSE(INT_GE("-3", "2")); + + EXPECT_TRUE(INT_GE("2**65", "2**65")); + EXPECT_TRUE(INT_GE("2**65", "2**65 - 1")); + EXPECT_FALSE(INT_GE("2**65", "2**65 + 1")); + EXPECT_TRUE(INT_GE("2**65", "2**60")); +} + +#define INT_GT(x, y) \ + CPyTagged_IsGt(eval_int(x), eval_int(y)) + +TEST_F(CAPITest, test_int_greater_than) { + EXPECT_TRUE(INT_GT("5", "0")); + EXPECT_TRUE(INT_GT("5", "4")); + EXPECT_FALSE(INT_GT("5", "5")); + EXPECT_FALSE(INT_GT("5", "6")); + + EXPECT_TRUE(INT_GT("1", "-3")); + EXPECT_FALSE(INT_GT("-3", "1")); + + EXPECT_TRUE(INT_GT("0", "-3")); + EXPECT_TRUE(INT_GT("-2", "-3")); + EXPECT_FALSE(INT_GT("-2", "-2")); + EXPECT_FALSE(INT_GT("-2", "-1")); + + EXPECT_TRUE(INT_GT("2**65", "5")); + EXPECT_TRUE(INT_GT("2**65", "2**65 - 1")); + EXPECT_FALSE(INT_GT("2**65", "2**65")); + EXPECT_FALSE(INT_GT("2**65", "2**65 + 1")); + + EXPECT_FALSE(INT_GT("-2**65", "1")); + EXPECT_TRUE(INT_GT("-2**65", "-2**65 - 1")); + EXPECT_FALSE(INT_GT("-2**65", "-2**65")); + EXPECT_FALSE(INT_GT("-2**65", "-2**65 + 1")); +} + +#define INT_LE(x, y) \ + CPyTagged_IsLe(eval_int(x), eval_int(y)) + +TEST_F(CAPITest, test_int_less_than_or_equal) { + EXPECT_TRUE(INT_LE("0", "5")); + EXPECT_TRUE(INT_LE("5", "6")); + EXPECT_TRUE(INT_LE("5", "5")); + EXPECT_FALSE(INT_LE("5", "4")); + + EXPECT_FALSE(INT_LE("1", "-3")); + EXPECT_TRUE(INT_LE("-3", "1")); + + EXPECT_TRUE(INT_LT("-3", "0")); + EXPECT_TRUE(INT_LE("-2", "-1")); + EXPECT_TRUE(INT_LE("-2", "-2")); + EXPECT_FALSE(INT_LT("-2", "-3")); + + EXPECT_TRUE(INT_LE("5", "2**65")); + EXPECT_FALSE(INT_LE("2**65", "5")); + EXPECT_TRUE(INT_LE("2**65", "2**65 + 1")); + EXPECT_TRUE(INT_LE("2**65", "2**65")); + EXPECT_FALSE(INT_LE("2**65", "2**65 - 1")); + + EXPECT_TRUE(INT_LE("-2**65", "1")); + EXPECT_FALSE(INT_LE("1", "-2**65")); + EXPECT_TRUE(INT_LE("-2**65", "-2**65 + 1")); + EXPECT_TRUE(INT_LE("-2**65", "-2**65")); + EXPECT_FALSE(INT_LE("-2**65", "-2**65 - 1")); +} + +#define list_get_eq(list, index, value) \ + is_py_equal(CPyList_GetItem(list, eval_int(index)), eval(value)) + +TEST_F(CAPITest, test_list_get) { + auto l = empty_list(); + list_append(l, "3"); + list_append(l, "5"); + list_append(l, "7"); + EXPECT_TRUE(list_get_eq(l, "0", "3")); + EXPECT_TRUE(list_get_eq(l, "1", "5")); + EXPECT_TRUE(list_get_eq(l, "2", "7")); + EXPECT_TRUE(list_get_eq(l, "-1", "7")); + EXPECT_TRUE(list_get_eq(l, "-2", "5")); + EXPECT_TRUE(list_get_eq(l, "-3", "3")); +} + +TEST_F(CAPITest, test_tagged_as_long_long) { + auto s = eval_int("3"); + auto neg = eval_int("-1"); + auto l = eval_int("2**128"); + EXPECT_TRUE(CPyTagged_AsSsize_t(s) == 3); + EXPECT_FALSE(PyErr_Occurred()); + EXPECT_TRUE(CPyTagged_AsSsize_t(neg) == -1); + EXPECT_FALSE(PyErr_Occurred()); + EXPECT_TRUE(CPyTagged_AsSsize_t(l) == -1); + EXPECT_TRUE(PyErr_Occurred()); + PyErr_Clear(); +} + + +//// +// Python module glue to drive the C-API tests. +// +// The reason we have this as an extension module instead of a +// standalone binary is because building an extension module is pretty +// well behaved (just specify it with distutils/setuptools and it will +// get compiled and linked against the running python) while linking a +// library against libpython is a huge non-standard +// PITA: python-config locations are janky and it behaves in weird +// ways that I don't understand, while this works very cleanly. + +static PyObject *run_tests(PyObject *dummy, PyObject *should_be_null) { + // Fake command line arguments. We could arrange to actually pass + // in command line arguments (either real ones or ones given as + // arguments) but have not bothered. + int argc = 1; + char asdf[] = "test_capi"; // InitGoogleTest wants char** which means it can't be const... + char *argv[] = {asdf, NULL}; + ::testing::InitGoogleTest(&argc, argv); + return PyLong_FromLong(RUN_ALL_TESTS()); +} + + +static PyMethodDef test_methods[] = { + {"run_tests", run_tests, METH_NOARGS, "Run the C API tests"}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef test_module = { + PyModuleDef_HEAD_INIT, + "test_capi", + NULL, + -1, + test_methods +}; + +PyMODINIT_FUNC +PyInit_test_capi(void) +{ + PyObject *module = PyModule_Create(&test_module); + if (module) { + moduleDict = PyModule_GetDict(module); + } + return module; +} diff --git a/mypyc/lib-rt/tuple_ops.c b/mypyc/lib-rt/tuple_ops.c new file mode 100644 index 000000000000..64418974666f --- /dev/null +++ b/mypyc/lib-rt/tuple_ops.c @@ -0,0 +1,61 @@ +// Tuple primitive operations +// +// These are registered in mypyc.primitives.tuple_ops. + +#include +#include "CPy.h" + +PyObject *CPySequenceTuple_GetItem(PyObject *tuple, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyTuple_GET_SIZE(tuple); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "tuple index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "tuple index out of range"); + return NULL; + } + } + PyObject *result = PyTuple_GET_ITEM(tuple, n); + Py_INCREF(result); + return result; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +PyObject *CPySequenceTuple_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { + if (likely(PyTuple_CheckExact(obj) + && CPyTagged_CheckShort(start) && CPyTagged_CheckShort(end))) { + Py_ssize_t startn = CPyTagged_ShortAsSsize_t(start); + Py_ssize_t endn = CPyTagged_ShortAsSsize_t(end); + if (startn < 0) { + startn += PyTuple_GET_SIZE(obj); + } + if (endn < 0) { + endn += PyTuple_GET_SIZE(obj); + } + return PyTuple_GetSlice(obj, startn, endn); + } + return CPyObject_GetSlice(obj, start, end); +} + +// PyTuple_SET_ITEM does no error checking, +// and should only be used to fill in brand new tuples. +bool CPySequenceTuple_SetItemUnsafe(PyObject *tuple, CPyTagged index, PyObject *value) +{ + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + PyTuple_SET_ITEM(tuple, n, value); + return true; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return false; + } +} diff --git a/mypyc/lower/__init__.py b/mypyc/lower/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/lower/int_ops.py b/mypyc/lower/int_ops.py new file mode 100644 index 000000000000..adfb4c21e2de --- /dev/null +++ b/mypyc/lower/int_ops.py @@ -0,0 +1,113 @@ +"""Convert tagged int primitive ops to lower-level ops.""" + +from __future__ import annotations + +from typing import NamedTuple + +from mypyc.ir.ops import Assign, BasicBlock, Branch, ComparisonOp, Register, Value +from mypyc.ir.rtypes import bool_rprimitive, is_short_int_rprimitive +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.lower.registry import lower_primitive_op +from mypyc.primitives.int_ops import int_equal_, int_less_than_ +from mypyc.primitives.registry import CFunctionDescription + + +# Description for building int comparison ops +# +# Fields: +# binary_op_variant: identify which IntOp to use when operands are short integers +# c_func_description: the C function to call when operands are tagged integers +# c_func_negated: whether to negate the C function call's result +# c_func_swap_operands: whether to swap lhs and rhs when call the function +class IntComparisonOpDescription(NamedTuple): + binary_op_variant: int + c_func_description: CFunctionDescription + c_func_negated: bool + c_func_swap_operands: bool + + +# Provide mapping from textual op to short int's op variant and boxed int's description. +# Note that these are not complete implementations and require extra IR. +int_comparison_op_mapping: dict[str, IntComparisonOpDescription] = { + "==": IntComparisonOpDescription(ComparisonOp.EQ, int_equal_, False, False), + "!=": IntComparisonOpDescription(ComparisonOp.NEQ, int_equal_, True, False), + "<": IntComparisonOpDescription(ComparisonOp.SLT, int_less_than_, False, False), + "<=": IntComparisonOpDescription(ComparisonOp.SLE, int_less_than_, True, True), + ">": IntComparisonOpDescription(ComparisonOp.SGT, int_less_than_, False, True), + ">=": IntComparisonOpDescription(ComparisonOp.SGE, int_less_than_, True, False), +} + + +def compare_tagged(self: LowLevelIRBuilder, lhs: Value, rhs: Value, op: str, line: int) -> Value: + """Compare two tagged integers using given operator (value context).""" + # generate fast binary logic ops on short ints + if (is_short_int_rprimitive(lhs.type) or is_short_int_rprimitive(rhs.type)) and op in ( + "==", + "!=", + ): + quick = True + else: + quick = is_short_int_rprimitive(lhs.type) and is_short_int_rprimitive(rhs.type) + if quick: + return self.comparison_op(lhs, rhs, int_comparison_op_mapping[op][0], line) + op_type, c_func_desc, negate_result, swap_op = int_comparison_op_mapping[op] + result = Register(bool_rprimitive) + short_int_block, int_block, out = BasicBlock(), BasicBlock(), BasicBlock() + check_lhs = self.check_tagged_short_int(lhs, line, negated=True) + if op in ("==", "!="): + self.add(Branch(check_lhs, int_block, short_int_block, Branch.BOOL)) + else: + # for non-equality logical ops (less/greater than, etc.), need to check both sides + short_lhs = BasicBlock() + self.add(Branch(check_lhs, int_block, short_lhs, Branch.BOOL)) + self.activate_block(short_lhs) + check_rhs = self.check_tagged_short_int(rhs, line, negated=True) + self.add(Branch(check_rhs, int_block, short_int_block, Branch.BOOL)) + self.activate_block(int_block) + if swap_op: + args = [rhs, lhs] + else: + args = [lhs, rhs] + call = self.call_c(c_func_desc, args, line) + if negate_result: + # TODO: introduce UnaryIntOp? + call_result = self.unary_op(call, "not", line) + else: + call_result = call + self.add(Assign(result, call_result, line)) + self.goto(out) + self.activate_block(short_int_block) + eq = self.comparison_op(lhs, rhs, op_type, line) + self.add(Assign(result, eq, line)) + self.goto_and_activate(out) + return result + + +@lower_primitive_op("int_eq") +def lower_int_eq(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + return compare_tagged(builder, args[0], args[1], "==", line) + + +@lower_primitive_op("int_ne") +def lower_int_ne(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + return compare_tagged(builder, args[0], args[1], "!=", line) + + +@lower_primitive_op("int_lt") +def lower_int_lt(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + return compare_tagged(builder, args[0], args[1], "<", line) + + +@lower_primitive_op("int_le") +def lower_int_le(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + return compare_tagged(builder, args[0], args[1], "<=", line) + + +@lower_primitive_op("int_gt") +def lower_int_gt(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + return compare_tagged(builder, args[0], args[1], ">", line) + + +@lower_primitive_op("int_ge") +def lower_int_ge(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + return compare_tagged(builder, args[0], args[1], ">=", line) diff --git a/mypyc/lower/list_ops.py b/mypyc/lower/list_ops.py new file mode 100644 index 000000000000..0d2e3e7169d8 --- /dev/null +++ b/mypyc/lower/list_ops.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from mypyc.common import PLATFORM_SIZE +from mypyc.ir.ops import GetElementPtr, Integer, IntOp, LoadMem, SetMem, Value +from mypyc.ir.rtypes import ( + PyListObject, + c_pyssize_t_rprimitive, + object_rprimitive, + pointer_rprimitive, +) +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.lower.registry import lower_primitive_op + + +@lower_primitive_op("buf_init_item") +def buf_init_item(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + """Initialize an item in a buffer of "PyObject *" values at given index. + + This can be used to initialize the data buffer of a freshly allocated list + object. + """ + base = args[0] + index_value = args[1] + value = args[2] + assert isinstance(index_value, Integer) + index = index_value.numeric_value() + if index == 0: + ptr = base + else: + ptr = builder.add( + IntOp( + pointer_rprimitive, + base, + Integer(index * PLATFORM_SIZE, c_pyssize_t_rprimitive), + IntOp.ADD, + line, + ) + ) + return builder.add(SetMem(object_rprimitive, ptr, value, line)) + + +@lower_primitive_op("list_items") +def list_items(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + ob_item_ptr = builder.add(GetElementPtr(args[0], PyListObject, "ob_item", line)) + return builder.add(LoadMem(pointer_rprimitive, ob_item_ptr, line)) diff --git a/mypyc/lower/misc_ops.py b/mypyc/lower/misc_ops.py new file mode 100644 index 000000000000..1effcd4f42ac --- /dev/null +++ b/mypyc/lower/misc_ops.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from mypyc.ir.ops import GetElementPtr, LoadMem, Value +from mypyc.ir.rtypes import PyVarObject, c_pyssize_t_rprimitive +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.lower.registry import lower_primitive_op + + +@lower_primitive_op("var_object_size") +def var_object_size(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + elem_address = builder.add(GetElementPtr(args[0], PyVarObject, "ob_size")) + return builder.add(LoadMem(c_pyssize_t_rprimitive, elem_address)) diff --git a/mypyc/lower/registry.py b/mypyc/lower/registry.py new file mode 100644 index 000000000000..084d57df4608 --- /dev/null +++ b/mypyc/lower/registry.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import Callable, Final, List + +from mypyc.ir.ops import Value +from mypyc.irbuild.ll_builder import LowLevelIRBuilder + +LowerFunc = Callable[[LowLevelIRBuilder, List[Value], int], Value] + + +lowering_registry: Final[dict[str, LowerFunc]] = {} + + +def lower_primitive_op(name: str) -> Callable[[LowerFunc], LowerFunc]: + """Register a handler that generates low-level IR for a primitive op.""" + + def wrapper(f: LowerFunc) -> LowerFunc: + assert name not in lowering_registry + lowering_registry[name] = f + return f + + return wrapper + + +# Import various modules that set up global state. +from mypyc.lower import int_ops, list_ops, misc_ops # noqa: F401 diff --git a/mypyc/namegen.py b/mypyc/namegen.py new file mode 100644 index 000000000000..ce84fde143d1 --- /dev/null +++ b/mypyc/namegen.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from typing import Iterable + + +class NameGenerator: + """Utility for generating distinct C names from Python names. + + Since C names can't use '.' (or unicode), some care is required to + make C names generated from Python names unique. Also, we want to + avoid generating overly long C names since they make the generated + code harder to read. + + Note that we don't restrict ourselves to a 32-character distinguishing + prefix guaranteed by the C standard since all the compilers we care + about at the moment support longer names without issues. + + For names that are exported in a shared library (not static) use + exported_name() instead. + + Summary of the approach: + + * Generate a unique name prefix from suffix of fully-qualified + module name used for static names. If only compiling a single + module, this can be empty. For example, if the modules are + 'foo.bar' and 'foo.baz', the prefixes can be 'bar_' and 'baz_', + respectively. If the modules are 'bar.foo' and 'baz.foo', the + prefixes will be 'bar_foo_' and 'baz_foo_'. + + * Replace '.' in the Python name with '___' in the C name. (And + replace the unlikely but possible '___' with '___3_'. This + collides '___' with '.3_', but this is OK because names + may not start with a digit.) + + The generated should be internal to a build and thus the mapping is + arbitrary. Just generating names '1', '2', ... would be correct, + though not very usable. + """ + + def __init__(self, groups: Iterable[list[str]]) -> None: + """Initialize with a list of modules in each compilation group. + + The names of modules are used to shorten names referring to + modules, for convenience. Arbitrary module + names are supported for generated names, but uncompiled modules + will use long names. + """ + self.module_map: dict[str, str] = {} + for names in groups: + self.module_map.update(make_module_translation_map(names)) + self.translations: dict[tuple[str, str], str] = {} + self.used_names: set[str] = set() + + def private_name(self, module: str, partial_name: str | None = None) -> str: + """Return a C name usable for a static definition. + + Return a distinct result for each (module, partial_name) pair. + + The caller should add a suitable prefix to the name to avoid + conflicts with other C names. Only ensure that the results of + this function are unique, not that they aren't overlapping with + arbitrary names. + + If a name is not specific to any module, the module argument can + be an empty string. + """ + # TODO: Support unicode + if partial_name is None: + return exported_name(self.module_map[module].rstrip(".")) + if (module, partial_name) in self.translations: + return self.translations[module, partial_name] + if module in self.module_map: + module_prefix = self.module_map[module] + elif module: + module_prefix = module + "." + else: + module_prefix = "" + actual = exported_name(f"{module_prefix}{partial_name}") + self.translations[module, partial_name] = actual + return actual + + +def exported_name(fullname: str) -> str: + """Return a C name usable for an exported definition. + + This is like private_name(), but the output only depends on the + 'fullname' argument, so the names are distinct across multiple + builds. + """ + # TODO: Support unicode + return fullname.replace("___", "___3_").replace(".", "___") + + +def make_module_translation_map(names: list[str]) -> dict[str, str]: + num_instances: dict[str, int] = {} + for name in names: + for suffix in candidate_suffixes(name): + num_instances[suffix] = num_instances.get(suffix, 0) + 1 + result = {} + for name in names: + for suffix in candidate_suffixes(name): + if num_instances[suffix] == 1: + break + # Takes the last suffix if none are unique + result[name] = suffix + return result + + +def candidate_suffixes(fullname: str) -> list[str]: + components = fullname.split(".") + result = [""] + for i in range(len(components)): + result.append(".".join(components[-i - 1 :]) + ".") + return result diff --git a/mypyc/options.py b/mypyc/options.py new file mode 100644 index 000000000000..24e68163bb11 --- /dev/null +++ b/mypyc/options.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import sys + + +class CompilerOptions: + def __init__( + self, + strip_asserts: bool = False, + multi_file: bool = False, + verbose: bool = False, + separate: bool = False, + target_dir: str | None = None, + include_runtime_files: bool | None = None, + capi_version: tuple[int, int] | None = None, + python_version: tuple[int, int] | None = None, + strict_dunder_typing: bool = False, + ) -> None: + self.strip_asserts = strip_asserts + self.multi_file = multi_file + self.verbose = verbose + self.separate = separate + self.global_opts = not separate + self.target_dir = target_dir or "build" + self.include_runtime_files = ( + include_runtime_files if include_runtime_files is not None else not multi_file + ) + # The target Python C API version. Overriding this is mostly + # useful in IR tests, since there's no guarantee that + # binaries are backward compatible even if no recent API + # features are used. + self.capi_version = capi_version or sys.version_info[:2] + self.python_version = python_version + # Make possible to inline dunder methods in the generated code. + # Typically, the convention is the dunder methods can return `NotImplemented` + # even when its return type is just `bool`. + # By enabling this option, this convention is no longer valid and the dunder + # will assume the return type of the method strictly, which can lead to + # more optimization opportunities. + self.strict_dunders_typing = strict_dunder_typing diff --git a/mypyc/primitives/__init__.py b/mypyc/primitives/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/primitives/bytes_ops.py b/mypyc/primitives/bytes_ops.py new file mode 100644 index 000000000000..1afd196cff84 --- /dev/null +++ b/mypyc/primitives/bytes_ops.py @@ -0,0 +1,109 @@ +"""Primitive bytes ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_MAGIC +from mypyc.ir.rtypes import ( + RUnion, + bytes_rprimitive, + c_int_rprimitive, + c_pyssize_t_rprimitive, + dict_rprimitive, + int_rprimitive, + list_rprimitive, + object_rprimitive, + str_rprimitive, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + binary_op, + custom_op, + function_op, + load_address_op, + method_op, +) + +# Get the 'bytes' type object. +load_address_op(name="builtins.bytes", type=object_rprimitive, src="PyBytes_Type") + +# bytes(obj) +function_op( + name="builtins.bytes", + arg_types=[RUnion([list_rprimitive, dict_rprimitive, str_rprimitive])], + return_type=bytes_rprimitive, + c_function_name="PyBytes_FromObject", + error_kind=ERR_MAGIC, +) + +# bytearray(obj) +function_op( + name="builtins.bytearray", + arg_types=[object_rprimitive], + return_type=bytes_rprimitive, + c_function_name="PyByteArray_FromObject", + error_kind=ERR_MAGIC, +) + +# bytes ==/!= (return -1/0/1) +bytes_compare = custom_op( + arg_types=[bytes_rprimitive, bytes_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyBytes_Compare", + error_kind=ERR_NEG_INT, +) + +# bytes + bytes +# bytearray + bytearray +binary_op( + name="+", + arg_types=[bytes_rprimitive, bytes_rprimitive], + return_type=bytes_rprimitive, + c_function_name="CPyBytes_Concat", + error_kind=ERR_MAGIC, + steals=[True, False], +) + +# bytes[begin:end] +bytes_slice_op = custom_op( + arg_types=[bytes_rprimitive, int_rprimitive, int_rprimitive], + return_type=bytes_rprimitive, + c_function_name="CPyBytes_GetSlice", + error_kind=ERR_MAGIC, +) + +# bytes[index] +# bytearray[index] +method_op( + name="__getitem__", + arg_types=[bytes_rprimitive, int_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyBytes_GetItem", + error_kind=ERR_MAGIC, +) + +# bytes.join(obj) +method_op( + name="join", + arg_types=[bytes_rprimitive, object_rprimitive], + return_type=bytes_rprimitive, + c_function_name="CPyBytes_Join", + error_kind=ERR_MAGIC, +) + +# Join bytes objects and return a new bytes. +# The first argument is the total number of the following bytes. +bytes_build_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=bytes_rprimitive, + c_function_name="CPyBytes_Build", + error_kind=ERR_MAGIC, + var_arg_type=bytes_rprimitive, +) + +function_op( + name="builtins.ord", + arg_types=[bytes_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyBytes_Ord", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/primitives/dict_ops.py b/mypyc/primitives/dict_ops.py new file mode 100644 index 000000000000..ce7b9bb8d70e --- /dev/null +++ b/mypyc/primitives/dict_ops.py @@ -0,0 +1,325 @@ +"""Primitive dict ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import ( + bit_rprimitive, + bool_rprimitive, + c_int_rprimitive, + c_pyssize_t_rprimitive, + dict_next_rtuple_pair, + dict_next_rtuple_single, + dict_rprimitive, + int_rprimitive, + list_rprimitive, + object_rprimitive, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + binary_op, + custom_op, + function_op, + load_address_op, + method_op, +) + +# Get the 'dict' type object. +load_address_op(name="builtins.dict", type=object_rprimitive, src="PyDict_Type") + +# Construct an empty dictionary via dict(). +function_op( + name="builtins.dict", + arg_types=[], + return_type=dict_rprimitive, + c_function_name="PyDict_New", + error_kind=ERR_MAGIC, +) + +# Construct an empty dictionary. +dict_new_op = custom_op( + arg_types=[], return_type=dict_rprimitive, c_function_name="PyDict_New", error_kind=ERR_MAGIC +) + +# Construct a dictionary from keys and values. +# Positional argument is the number of key-value pairs +# Variable arguments are (key1, value1, ..., keyN, valueN). +dict_build_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=dict_rprimitive, + c_function_name="CPyDict_Build", + error_kind=ERR_MAGIC, + var_arg_type=object_rprimitive, +) + +# Construct a dictionary from another dictionary. +function_op( + name="builtins.dict", + arg_types=[dict_rprimitive], + return_type=dict_rprimitive, + c_function_name="PyDict_Copy", + error_kind=ERR_MAGIC, + priority=2, +) + +# Generic one-argument dict constructor: dict(obj) +dict_copy = function_op( + name="builtins.dict", + arg_types=[object_rprimitive], + return_type=dict_rprimitive, + c_function_name="CPyDict_FromAny", + error_kind=ERR_MAGIC, +) + +# dict[key] +dict_get_item_op = method_op( + name="__getitem__", + arg_types=[dict_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_GetItem", + error_kind=ERR_MAGIC, +) + +# dict[key] = value +dict_set_item_op = method_op( + name="__setitem__", + arg_types=[dict_rprimitive, object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyDict_SetItem", + error_kind=ERR_NEG_INT, +) + +# key in dict +binary_op( + name="in", + arg_types=[object_rprimitive, dict_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyDict_Contains", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + ordering=[1, 0], +) + +# dict1.update(dict2) +dict_update_op = method_op( + name="update", + arg_types=[dict_rprimitive, dict_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyDict_Update", + error_kind=ERR_NEG_INT, + priority=2, +) + +# Operation used for **value in dict displays. +# This is mostly like dict.update(obj), but has customized error handling. +dict_update_in_display_op = custom_op( + arg_types=[dict_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyDict_UpdateInDisplay", + error_kind=ERR_NEG_INT, +) + +# dict.update(obj) +method_op( + name="update", + arg_types=[dict_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyDict_UpdateFromAny", + error_kind=ERR_NEG_INT, +) + +# dict.get(key, default) +method_op( + name="get", + arg_types=[dict_rprimitive, object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_Get", + error_kind=ERR_MAGIC, +) + +# dict.get(key) +dict_get_method_with_none = method_op( + name="get", + arg_types=[dict_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_GetWithNone", + error_kind=ERR_MAGIC, +) + +# dict.setdefault(key, default) +dict_setdefault_op = method_op( + name="setdefault", + arg_types=[dict_rprimitive, object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_SetDefault", + error_kind=ERR_MAGIC, +) + +# dict.setdefault(key) +method_op( + name="setdefault", + arg_types=[dict_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_SetDefaultWithNone", + error_kind=ERR_MAGIC, +) + +# dict.setdefault(key, empty tuple/list/set) +# The third argument marks the data type of the second argument. +# 1: list 2: dict 3: set +# Other number would lead to an error. +dict_setdefault_spec_init_op = custom_op( + arg_types=[dict_rprimitive, object_rprimitive, c_int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_SetDefaultWithEmptyDatatype", + error_kind=ERR_MAGIC, +) + +# dict.keys() +method_op( + name="keys", + arg_types=[dict_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_KeysView", + error_kind=ERR_MAGIC, +) + +# dict.values() +method_op( + name="values", + arg_types=[dict_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_ValuesView", + error_kind=ERR_MAGIC, +) + +# dict.items() +method_op( + name="items", + arg_types=[dict_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_ItemsView", + error_kind=ERR_MAGIC, +) + +# dict.clear() +method_op( + name="clear", + arg_types=[dict_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyDict_Clear", + error_kind=ERR_FALSE, +) + +# dict.copy() +method_op( + name="copy", + arg_types=[dict_rprimitive], + return_type=dict_rprimitive, + c_function_name="CPyDict_Copy", + error_kind=ERR_MAGIC, +) + +# list(dict.keys()) +dict_keys_op = custom_op( + arg_types=[dict_rprimitive], + return_type=list_rprimitive, + c_function_name="CPyDict_Keys", + error_kind=ERR_MAGIC, +) + +# list(dict.values()) +dict_values_op = custom_op( + arg_types=[dict_rprimitive], + return_type=list_rprimitive, + c_function_name="CPyDict_Values", + error_kind=ERR_MAGIC, +) + +# list(dict.items()) +dict_items_op = custom_op( + arg_types=[dict_rprimitive], + return_type=list_rprimitive, + c_function_name="CPyDict_Items", + error_kind=ERR_MAGIC, +) + +# PyDict_Next() fast iteration +dict_key_iter_op = custom_op( + arg_types=[dict_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_GetKeysIter", + error_kind=ERR_MAGIC, +) + +dict_value_iter_op = custom_op( + arg_types=[dict_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_GetValuesIter", + error_kind=ERR_MAGIC, +) + +dict_item_iter_op = custom_op( + arg_types=[dict_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyDict_GetItemsIter", + error_kind=ERR_MAGIC, +) + +dict_next_key_op = custom_op( + arg_types=[object_rprimitive, int_rprimitive], + return_type=dict_next_rtuple_single, + c_function_name="CPyDict_NextKey", + error_kind=ERR_NEVER, +) + +dict_next_value_op = custom_op( + arg_types=[object_rprimitive, int_rprimitive], + return_type=dict_next_rtuple_single, + c_function_name="CPyDict_NextValue", + error_kind=ERR_NEVER, +) + +dict_next_item_op = custom_op( + arg_types=[object_rprimitive, int_rprimitive], + return_type=dict_next_rtuple_pair, + c_function_name="CPyDict_NextItem", + error_kind=ERR_NEVER, +) + +# check that len(dict) == const during iteration +dict_check_size_op = custom_op( + arg_types=[dict_rprimitive, int_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyDict_CheckSize", + error_kind=ERR_FALSE, +) + +dict_ssize_t_size_op = custom_op( + arg_types=[dict_rprimitive], + return_type=c_pyssize_t_rprimitive, + c_function_name="PyDict_Size", + error_kind=ERR_NEVER, +) + +# Delete an item from a dict +dict_del_item = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyDict_DelItem", + error_kind=ERR_NEG_INT, +) + +supports_mapping_protocol = custom_op( + arg_types=[object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyMapping_Check", + error_kind=ERR_NEVER, +) + +mapping_has_key = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyMapping_HasKey", + error_kind=ERR_NEVER, +) diff --git a/mypyc/primitives/exc_ops.py b/mypyc/primitives/exc_ops.py new file mode 100644 index 000000000000..ad105056158a --- /dev/null +++ b/mypyc/primitives/exc_ops.py @@ -0,0 +1,101 @@ +"""Exception-related primitive ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_ALWAYS, ERR_FALSE, ERR_NEVER +from mypyc.ir.rtypes import bit_rprimitive, exc_rtuple, object_rprimitive, void_rtype +from mypyc.primitives.registry import custom_op + +# If the argument is a class, raise an instance of the class. Otherwise, assume +# that the argument is an exception object, and raise it. +raise_exception_op = custom_op( + arg_types=[object_rprimitive], + return_type=void_rtype, + c_function_name="CPy_Raise", + error_kind=ERR_ALWAYS, +) + +# Raise StopIteration exception with the specified value (which can be NULL). +set_stop_iteration_value = custom_op( + arg_types=[object_rprimitive], + return_type=void_rtype, + c_function_name="CPyGen_SetStopIterationValue", + error_kind=ERR_ALWAYS, +) + +# Raise exception with traceback. +# Arguments are (exception type, exception value, traceback). +raise_exception_with_tb_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=void_rtype, + c_function_name="CPyErr_SetObjectAndTraceback", + error_kind=ERR_ALWAYS, +) + +# Reraise the currently raised exception. +reraise_exception_op = custom_op( + arg_types=[], return_type=void_rtype, c_function_name="CPy_Reraise", error_kind=ERR_ALWAYS +) + +# Propagate exception if the CPython error indicator is set (an exception was raised). +no_err_occurred_op = custom_op( + arg_types=[], + return_type=bit_rprimitive, + c_function_name="CPy_NoErrOccured", + error_kind=ERR_FALSE, +) + +err_occurred_op = custom_op( + arg_types=[], + return_type=object_rprimitive, + c_function_name="PyErr_Occurred", + error_kind=ERR_NEVER, + is_borrowed=True, +) + +# Keep propagating a raised exception by unconditionally giving an error value. +# This doesn't actually raise an exception. +keep_propagating_op = custom_op( + arg_types=[], + return_type=bit_rprimitive, + c_function_name="CPy_KeepPropagating", + error_kind=ERR_FALSE, +) + +# Catches a propagating exception and makes it the "currently +# handled exception" (by sticking it into sys.exc_info()). Returns the +# exception that was previously being handled, which must be restored +# later. +error_catch_op = custom_op( + arg_types=[], return_type=exc_rtuple, c_function_name="CPy_CatchError", error_kind=ERR_NEVER +) + +# Restore an old "currently handled exception" returned from. +# error_catch (by sticking it into sys.exc_info()) +restore_exc_info_op = custom_op( + arg_types=[exc_rtuple], + return_type=void_rtype, + c_function_name="CPy_RestoreExcInfo", + error_kind=ERR_NEVER, +) + +# Checks whether the exception currently being handled matches a particular type. +exc_matches_op = custom_op( + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPy_ExceptionMatches", + error_kind=ERR_NEVER, +) + +# Get the value of the exception currently being handled. +get_exc_value_op = custom_op( + arg_types=[], + return_type=object_rprimitive, + c_function_name="CPy_GetExcValue", + error_kind=ERR_NEVER, +) + +# Get exception info (exception type, exception instance, traceback object). +get_exc_info_op = custom_op( + arg_types=[], return_type=exc_rtuple, c_function_name="CPy_GetExcInfo", error_kind=ERR_NEVER +) diff --git a/mypyc/primitives/float_ops.py b/mypyc/primitives/float_ops.py new file mode 100644 index 000000000000..14e8d4caf09c --- /dev/null +++ b/mypyc/primitives/float_ops.py @@ -0,0 +1,168 @@ +"""Primitive float ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ERR_NEVER +from mypyc.ir.rtypes import ( + bool_rprimitive, + float_rprimitive, + int_rprimitive, + object_rprimitive, + str_rprimitive, +) +from mypyc.primitives.registry import binary_op, function_op, load_address_op + +# Get the 'builtins.float' type object. +load_address_op(name="builtins.float", type=object_rprimitive, src="PyFloat_Type") + +binary_op( + name="//", + arg_types=[float_rprimitive, float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_FloorDivide", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# float(int) +int_to_float_op = function_op( + name="builtins.float", + arg_types=[int_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_FromTagged", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# float(str) +function_op( + name="builtins.float", + arg_types=[str_rprimitive], + return_type=object_rprimitive, + c_function_name="PyFloat_FromString", + error_kind=ERR_MAGIC, +) + +# abs(float) +function_op( + name="builtins.abs", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="fabs", + error_kind=ERR_NEVER, +) + +# math.sin(float) +function_op( + name="math.sin", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_Sin", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# math.cos(float) +function_op( + name="math.cos", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_Cos", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# math.tan(float) +function_op( + name="math.tan", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_Tan", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# math.sqrt(float) +function_op( + name="math.sqrt", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_Sqrt", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# math.exp(float) +function_op( + name="math.exp", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_Exp", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# math.log(float) +function_op( + name="math.log", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_Log", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# math.floor(float) +function_op( + name="math.floor", + arg_types=[float_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyFloat_Floor", + error_kind=ERR_MAGIC, +) + +# math.ceil(float) +function_op( + name="math.ceil", + arg_types=[float_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyFloat_Ceil", + error_kind=ERR_MAGIC, +) + +# math.fabs(float) +function_op( + name="math.fabs", + arg_types=[float_rprimitive], + return_type=float_rprimitive, + c_function_name="fabs", + error_kind=ERR_NEVER, +) + +# math.pow(float, float) +pow_op = function_op( + name="math.pow", + arg_types=[float_rprimitive, float_rprimitive], + return_type=float_rprimitive, + c_function_name="CPyFloat_Pow", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# math.copysign(float, float) +copysign_op = function_op( + name="math.copysign", + arg_types=[float_rprimitive, float_rprimitive], + return_type=float_rprimitive, + c_function_name="copysign", + error_kind=ERR_NEVER, +) + +# math.isinf(float) +function_op( + name="math.isinf", + arg_types=[float_rprimitive], + return_type=bool_rprimitive, + c_function_name="CPyFloat_IsInf", + error_kind=ERR_NEVER, +) + +# math.isnan(float) +function_op( + name="math.isnan", + arg_types=[float_rprimitive], + return_type=bool_rprimitive, + c_function_name="CPyFloat_IsNaN", + error_kind=ERR_NEVER, +) diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py new file mode 100644 index 000000000000..fe42767db11e --- /dev/null +++ b/mypyc/primitives/generic_ops.py @@ -0,0 +1,384 @@ +"""Fallback primitive operations that operate on 'object' operands. + +These just call the relevant Python C API function or a thin wrapper +around an API function. Most of these also have faster, specialized +ops that operate on some more specific types. + +Many of these ops are given a low priority (0) so that specialized ops +will take precedence. If your specialized op doesn't seem to be used, +check that the priorities are configured properly. +""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import ( + bool_rprimitive, + c_int_rprimitive, + c_pyssize_t_rprimitive, + c_size_t_rprimitive, + int_rprimitive, + object_pointer_rprimitive, + object_rprimitive, + pointer_rprimitive, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + binary_op, + custom_op, + function_op, + method_op, + unary_op, +) + +# Binary operations + +for op, opid in [ + ("==", 2), # PY_EQ + ("!=", 3), # PY_NE + ("<", 0), # PY_LT + ("<=", 1), # PY_LE + (">", 4), # PY_GT + (">=", 5), +]: # PY_GE + # The result type is 'object' since that's what PyObject_RichCompare returns. + binary_op( + name=op, + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyObject_RichCompare", + error_kind=ERR_MAGIC, + extra_int_constants=[(opid, c_int_rprimitive)], + priority=0, + ) + +for op, funcname in [ + ("+", "PyNumber_Add"), + ("-", "PyNumber_Subtract"), + ("*", "PyNumber_Multiply"), + ("//", "PyNumber_FloorDivide"), + ("/", "PyNumber_TrueDivide"), + ("%", "PyNumber_Remainder"), + ("<<", "PyNumber_Lshift"), + (">>", "PyNumber_Rshift"), + ("&", "PyNumber_And"), + ("^", "PyNumber_Xor"), + ("|", "PyNumber_Or"), + ("@", "PyNumber_MatrixMultiply"), +]: + binary_op( + name=op, + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name=funcname, + error_kind=ERR_MAGIC, + priority=0, + ) + + +function_op( + name="builtins.divmod", + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyNumber_Divmod", + error_kind=ERR_MAGIC, + priority=0, +) + + +for op, funcname in [ + ("+=", "PyNumber_InPlaceAdd"), + ("-=", "PyNumber_InPlaceSubtract"), + ("*=", "PyNumber_InPlaceMultiply"), + ("@=", "PyNumber_InPlaceMatrixMultiply"), + ("//=", "PyNumber_InPlaceFloorDivide"), + ("/=", "PyNumber_InPlaceTrueDivide"), + ("%=", "PyNumber_InPlaceRemainder"), + ("<<=", "PyNumber_InPlaceLshift"), + (">>=", "PyNumber_InPlaceRshift"), + ("&=", "PyNumber_InPlaceAnd"), + ("^=", "PyNumber_InPlaceXor"), + ("|=", "PyNumber_InPlaceOr"), +]: + binary_op( + name=op, + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name=funcname, + error_kind=ERR_MAGIC, + priority=0, + ) + +for op, c_function in (("**", "CPyNumber_Power"), ("**=", "CPyNumber_InPlacePower")): + binary_op( + name=op, + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + error_kind=ERR_MAGIC, + c_function_name=c_function, + priority=0, + ) + +for arg_count, c_function in ((2, "CPyNumber_Power"), (3, "PyNumber_Power")): + function_op( + name="builtins.pow", + arg_types=[object_rprimitive] * arg_count, + return_type=object_rprimitive, + error_kind=ERR_MAGIC, + c_function_name=c_function, + priority=0, + ) + +binary_op( + name="in", + arg_types=[object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PySequence_Contains", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + ordering=[1, 0], + priority=0, +) + + +# Unary operations + +for op, funcname in [ + ("-", "PyNumber_Negative"), + ("+", "PyNumber_Positive"), + ("~", "PyNumber_Invert"), +]: + unary_op( + name=op, + arg_type=object_rprimitive, + return_type=object_rprimitive, + c_function_name=funcname, + error_kind=ERR_MAGIC, + priority=0, + ) + +unary_op( + name="not", + arg_type=object_rprimitive, + return_type=c_int_rprimitive, + c_function_name="PyObject_Not", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + priority=0, +) + +# abs(obj) +function_op( + name="builtins.abs", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyNumber_Absolute", + error_kind=ERR_MAGIC, + priority=0, +) + +# obj1[obj2] +py_get_item_op = method_op( + name="__getitem__", + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyObject_GetItem", + error_kind=ERR_MAGIC, + priority=0, +) + +# obj1[obj2] = obj3 +method_op( + name="__setitem__", + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyObject_SetItem", + error_kind=ERR_NEG_INT, + priority=0, +) + +# del obj1[obj2] +method_op( + name="__delitem__", + arg_types=[object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyObject_DelItem", + error_kind=ERR_NEG_INT, + priority=0, +) + +# hash(obj) +function_op( + name="builtins.hash", + arg_types=[object_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyObject_Hash", + error_kind=ERR_MAGIC, +) + +# getattr(obj, attr) +py_getattr_op = function_op( + name="builtins.getattr", + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyObject_GetAttr", + error_kind=ERR_MAGIC, +) + +# getattr(obj, attr, default) +function_op( + name="builtins.getattr", + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyObject_GetAttr3", + error_kind=ERR_MAGIC, +) + +# setattr(obj, attr, value) +py_setattr_op = function_op( + name="builtins.setattr", + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyObject_SetAttr", + error_kind=ERR_NEG_INT, +) + +# hasattr(obj, attr) +py_hasattr_op = function_op( + name="builtins.hasattr", + arg_types=[object_rprimitive, object_rprimitive], + return_type=bool_rprimitive, + c_function_name="PyObject_HasAttr", + error_kind=ERR_NEVER, +) + +# del obj.attr +py_delattr_op = function_op( + name="builtins.delattr", + arg_types=[object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyObject_DelAttr", + error_kind=ERR_NEG_INT, +) + +# Call callable object with N positional arguments: func(arg1, ..., argN) +# Arguments are (func, arg1, ..., argN). +py_call_op = custom_op( + arg_types=[], + return_type=object_rprimitive, + c_function_name="PyObject_CallFunctionObjArgs", + error_kind=ERR_MAGIC, + var_arg_type=object_rprimitive, + extra_int_constants=[(0, pointer_rprimitive)], +) + +# Call callable object using positional and/or keyword arguments (Python 3.8+) +py_vectorcall_op = custom_op( + arg_types=[ + object_rprimitive, # Callable + object_pointer_rprimitive, # Args (PyObject **) + c_size_t_rprimitive, # Number of positional args + object_rprimitive, + ], # Keyword arg names tuple (or NULL) + return_type=object_rprimitive, + c_function_name="_PyObject_Vectorcall", + error_kind=ERR_MAGIC, +) + +# Call method using positional and/or keyword arguments (Python 3.9+) +py_vectorcall_method_op = custom_op( + arg_types=[ + object_rprimitive, # Method name + object_pointer_rprimitive, # Args, including self (PyObject **) + c_size_t_rprimitive, # Number of positional args, including self + object_rprimitive, + ], # Keyword arg names tuple (or NULL) + return_type=object_rprimitive, + c_function_name="PyObject_VectorcallMethod", + error_kind=ERR_MAGIC, +) + +# Call callable object with positional + keyword args: func(*args, **kwargs) +# Arguments are (func, *args tuple, **kwargs dict). +py_call_with_kwargs_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyObject_Call", + error_kind=ERR_MAGIC, +) + +# Call method with positional arguments: obj.method(arg1, ...) +# Arguments are (object, attribute name, arg1, ...). +py_method_call_op = custom_op( + arg_types=[], + return_type=object_rprimitive, + c_function_name="CPyObject_CallMethodObjArgs", + error_kind=ERR_MAGIC, + var_arg_type=object_rprimitive, + extra_int_constants=[(0, pointer_rprimitive)], +) + +# len(obj) +generic_len_op = custom_op( + arg_types=[object_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyObject_Size", + error_kind=ERR_MAGIC, +) + +# len(obj) +# same as generic_len_op, however return py_ssize_t +generic_ssize_t_len_op = custom_op( + arg_types=[object_rprimitive], + return_type=c_pyssize_t_rprimitive, + c_function_name="PyObject_Size", + error_kind=ERR_NEG_INT, +) + +# iter(obj) +iter_op = function_op( + name="builtins.iter", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyObject_GetIter", + error_kind=ERR_MAGIC, +) +# next(iterator) +# +# Although the error_kind is set to be ERR_NEVER, this can actually +# return NULL, and thus it must be checked using Branch.IS_ERROR. +next_op = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyIter_Next", + error_kind=ERR_NEVER, +) +# next(iterator) +# +# Do a next, don't swallow StopIteration, but also don't propagate an +# error. (N.B: This can still return NULL without an error to +# represent an implicit StopIteration, but if StopIteration is +# *explicitly* raised this will not swallow it.) +# Can return NULL: see next_op. +next_raw_op = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyIter_Next", + error_kind=ERR_NEVER, +) + +# this would be aiter(obj) if it existed +aiter_op = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_GetAIter", + error_kind=ERR_MAGIC, +) + +# this would be anext(obj) if it existed +anext_op = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_GetANext", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py new file mode 100644 index 000000000000..657578d20046 --- /dev/null +++ b/mypyc/primitives/int_ops.py @@ -0,0 +1,298 @@ +"""Arbitrary-precision integer primitive ops. + +These mostly operate on (usually) unboxed integers that use a tagged pointer +representation (CPyTagged) and correspond to the Python 'int' type. + +See also the documentation for mypyc.rtypes.int_rprimitive. + +Use mypyc.ir.ops.IntOp for operations on fixed-width/C integers. +""" + +from __future__ import annotations + +from mypyc.ir.ops import ( + ERR_ALWAYS, + ERR_MAGIC, + ERR_MAGIC_OVERLAPPING, + ERR_NEVER, + PrimitiveDescription, +) +from mypyc.ir.rtypes import ( + RType, + bit_rprimitive, + bool_rprimitive, + c_pyssize_t_rprimitive, + float_rprimitive, + int16_rprimitive, + int32_rprimitive, + int64_rprimitive, + int_rprimitive, + object_rprimitive, + str_rprimitive, + void_rtype, +) +from mypyc.primitives.registry import binary_op, custom_op, function_op, load_address_op, unary_op + +# Constructors for builtins.int and native int types have the same behavior. In +# interpreted mode, native int types are just aliases to 'int'. +for int_name in ( + "builtins.int", + "mypy_extensions.i64", + "mypy_extensions.i32", + "mypy_extensions.i16", + "mypy_extensions.u8", +): + # These int constructors produce object_rprimitives that then need to be unboxed + # I guess unboxing ourselves would save a check and branch though? + + # Get the type object for 'builtins.int' or a native int type. + # For ordinary calls to int() we use a load_address to the type. + # Native ints don't have a separate type object -- we just use 'builtins.int'. + load_address_op(name=int_name, type=object_rprimitive, src="PyLong_Type") + + # int(float). We could do a bit better directly. + function_op( + name=int_name, + arg_types=[float_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyTagged_FromFloat", + error_kind=ERR_MAGIC, + ) + + # int(string) + function_op( + name=int_name, + arg_types=[str_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyLong_FromStr", + error_kind=ERR_MAGIC, + ) + + # int(string, base) + function_op( + name=int_name, + arg_types=[str_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyLong_FromStrWithBase", + error_kind=ERR_MAGIC, + ) + +# str(int) +int_to_str_op = function_op( + name="builtins.str", + arg_types=[int_rprimitive], + return_type=str_rprimitive, + c_function_name="CPyTagged_Str", + error_kind=ERR_MAGIC, + priority=2, +) + +# We need a specialization for str on bools also since the int one is wrong... +function_op( + name="builtins.str", + arg_types=[bool_rprimitive], + return_type=str_rprimitive, + c_function_name="CPyBool_Str", + error_kind=ERR_MAGIC, + priority=3, +) + + +def int_binary_primitive( + op: str, primitive_name: str, return_type: RType = int_rprimitive, error_kind: int = ERR_NEVER +) -> PrimitiveDescription: + return binary_op( + name=op, + arg_types=[int_rprimitive, int_rprimitive], + return_type=return_type, + primitive_name=primitive_name, + error_kind=error_kind, + ) + + +int_eq = int_binary_primitive(op="==", primitive_name="int_eq", return_type=bit_rprimitive) +int_ne = int_binary_primitive(op="!=", primitive_name="int_ne", return_type=bit_rprimitive) +int_lt = int_binary_primitive(op="<", primitive_name="int_lt", return_type=bit_rprimitive) +int_le = int_binary_primitive(op="<=", primitive_name="int_le", return_type=bit_rprimitive) +int_gt = int_binary_primitive(op=">", primitive_name="int_gt", return_type=bit_rprimitive) +int_ge = int_binary_primitive(op=">=", primitive_name="int_ge", return_type=bit_rprimitive) + + +def int_binary_op( + name: str, + c_function_name: str, + return_type: RType = int_rprimitive, + error_kind: int = ERR_NEVER, +) -> None: + binary_op( + name=name, + arg_types=[int_rprimitive, int_rprimitive], + return_type=return_type, + c_function_name=c_function_name, + error_kind=error_kind, + ) + + +# Binary, unary and augmented assignment operations that operate on CPyTagged ints +# are implemented as C functions. + +int_binary_op("+", "CPyTagged_Add") +int_binary_op("-", "CPyTagged_Subtract") +int_binary_op("*", "CPyTagged_Multiply") +int_binary_op("&", "CPyTagged_And") +int_binary_op("|", "CPyTagged_Or") +int_binary_op("^", "CPyTagged_Xor") +# Divide and remainder we honestly propagate errors from because they +# can raise ZeroDivisionError +int_binary_op("//", "CPyTagged_FloorDivide", error_kind=ERR_MAGIC) +int_binary_op("%", "CPyTagged_Remainder", error_kind=ERR_MAGIC) +# Negative shift counts raise an exception +int_binary_op(">>", "CPyTagged_Rshift", error_kind=ERR_MAGIC) +int_binary_op("<<", "CPyTagged_Lshift", error_kind=ERR_MAGIC) + +int_binary_op( + "/", "CPyTagged_TrueDivide", return_type=float_rprimitive, error_kind=ERR_MAGIC_OVERLAPPING +) + +# This should work because assignment operators are parsed differently +# and the code in irbuild that handles it does the assignment +# regardless of whether or not the operator works in place anyway. +int_binary_op("+=", "CPyTagged_Add") +int_binary_op("-=", "CPyTagged_Subtract") +int_binary_op("*=", "CPyTagged_Multiply") +int_binary_op("&=", "CPyTagged_And") +int_binary_op("|=", "CPyTagged_Or") +int_binary_op("^=", "CPyTagged_Xor") +int_binary_op("//=", "CPyTagged_FloorDivide", error_kind=ERR_MAGIC) +int_binary_op("%=", "CPyTagged_Remainder", error_kind=ERR_MAGIC) +int_binary_op(">>=", "CPyTagged_Rshift", error_kind=ERR_MAGIC) +int_binary_op("<<=", "CPyTagged_Lshift", error_kind=ERR_MAGIC) + + +def int_unary_op(name: str, c_function_name: str) -> PrimitiveDescription: + return unary_op( + name=name, + arg_type=int_rprimitive, + return_type=int_rprimitive, + c_function_name=c_function_name, + error_kind=ERR_NEVER, + ) + + +int_neg_op = int_unary_op("-", "CPyTagged_Negate") +int_invert_op = int_unary_op("~", "CPyTagged_Invert") + + +# Primitives related to integer comparison operations: + + +# Equals operation on two boxed tagged integers +int_equal_ = custom_op( + arg_types=[int_rprimitive, int_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyTagged_IsEq_", + error_kind=ERR_NEVER, + is_pure=True, +) + +# Less than operation on two boxed tagged integers +int_less_than_ = custom_op( + arg_types=[int_rprimitive, int_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyTagged_IsLt_", + error_kind=ERR_NEVER, + is_pure=True, +) + +int64_divide_op = custom_op( + arg_types=[int64_rprimitive, int64_rprimitive], + return_type=int64_rprimitive, + c_function_name="CPyInt64_Divide", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +int64_mod_op = custom_op( + arg_types=[int64_rprimitive, int64_rprimitive], + return_type=int64_rprimitive, + c_function_name="CPyInt64_Remainder", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +int32_divide_op = custom_op( + arg_types=[int32_rprimitive, int32_rprimitive], + return_type=int32_rprimitive, + c_function_name="CPyInt32_Divide", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +int32_mod_op = custom_op( + arg_types=[int32_rprimitive, int32_rprimitive], + return_type=int32_rprimitive, + c_function_name="CPyInt32_Remainder", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +int16_divide_op = custom_op( + arg_types=[int16_rprimitive, int16_rprimitive], + return_type=int16_rprimitive, + c_function_name="CPyInt16_Divide", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +int16_mod_op = custom_op( + arg_types=[int16_rprimitive, int16_rprimitive], + return_type=int16_rprimitive, + c_function_name="CPyInt16_Remainder", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +# Convert tagged int (as PyObject *) to i64 +int_to_int64_op = custom_op( + arg_types=[object_rprimitive], + return_type=int64_rprimitive, + c_function_name="CPyLong_AsInt64", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +ssize_t_to_int_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyTagged_FromSsize_t", + error_kind=ERR_MAGIC, +) + +int64_to_int_op = custom_op( + arg_types=[int64_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyTagged_FromInt64", + error_kind=ERR_MAGIC, +) + +# Convert tagged int (as PyObject *) to i32 +int_to_int32_op = custom_op( + arg_types=[object_rprimitive], + return_type=int32_rprimitive, + c_function_name="CPyLong_AsInt32", + error_kind=ERR_MAGIC_OVERLAPPING, +) + +int32_overflow = custom_op( + arg_types=[], + return_type=void_rtype, + c_function_name="CPyInt32_Overflow", + error_kind=ERR_ALWAYS, +) + +int16_overflow = custom_op( + arg_types=[], + return_type=void_rtype, + c_function_name="CPyInt16_Overflow", + error_kind=ERR_ALWAYS, +) + +uint8_overflow = custom_op( + arg_types=[], + return_type=void_rtype, + c_function_name="CPyUInt8_Overflow", + error_kind=ERR_ALWAYS, +) diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py new file mode 100644 index 000000000000..cb75e19a8dea --- /dev/null +++ b/mypyc/primitives/list_ops.py @@ -0,0 +1,310 @@ +"""List primitive ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import ( + bit_rprimitive, + c_int_rprimitive, + c_pyssize_t_rprimitive, + int64_rprimitive, + int_rprimitive, + list_rprimitive, + object_rprimitive, + pointer_rprimitive, + short_int_rprimitive, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + binary_op, + custom_op, + custom_primitive_op, + function_op, + load_address_op, + method_op, +) + +# Get the 'builtins.list' type object. +load_address_op(name="builtins.list", type=object_rprimitive, src="PyList_Type") + +# list(obj) +to_list = function_op( + name="builtins.list", + arg_types=[object_rprimitive], + return_type=list_rprimitive, + c_function_name="PySequence_List", + error_kind=ERR_MAGIC, +) + +# Construct an empty list via list(). +function_op( + name="builtins.list", + arg_types=[], + return_type=list_rprimitive, + c_function_name="PyList_New", + error_kind=ERR_MAGIC, + extra_int_constants=[(0, int_rprimitive)], +) + +new_list_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=list_rprimitive, + c_function_name="PyList_New", + error_kind=ERR_MAGIC, +) + +list_build_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=list_rprimitive, + c_function_name="CPyList_Build", + error_kind=ERR_MAGIC, + var_arg_type=object_rprimitive, + steals=True, +) + +# Get pointer to list items (ob_item PyListObject field) +list_items = custom_primitive_op( + name="list_items", + arg_types=[list_rprimitive], + return_type=pointer_rprimitive, + error_kind=ERR_NEVER, +) + +# list[index] (for an integer index) +list_get_item_op = method_op( + name="__getitem__", + arg_types=[list_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItem", + error_kind=ERR_MAGIC, +) + +# list[index] version with no int tag check for when it is known to be short +method_op( + name="__getitem__", + arg_types=[list_rprimitive, short_int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemShort", + error_kind=ERR_MAGIC, + priority=2, +) + +# list[index] that produces a borrowed result +method_op( + name="__getitem__", + arg_types=[list_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + priority=3, +) + +# list[index] that produces a borrowed result and index is known to be short +method_op( + name="__getitem__", + arg_types=[list_rprimitive, short_int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemShortBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + priority=4, +) + +# Version with native int index +method_op( + name="__getitem__", + arg_types=[list_rprimitive, int64_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemInt64", + error_kind=ERR_MAGIC, + priority=5, +) + +# Version with native int index +method_op( + name="__getitem__", + arg_types=[list_rprimitive, int64_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemInt64Borrow", + is_borrowed=True, + error_kind=ERR_MAGIC, + priority=6, +) + +# This is unsafe because it assumes that the index is a non-negative short integer +# that is in-bounds for the list. +list_get_item_unsafe_op = custom_op( + arg_types=[list_rprimitive, short_int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemUnsafe", + error_kind=ERR_NEVER, +) + +# list[index] = obj +list_set_item_op = method_op( + name="__setitem__", + arg_types=[list_rprimitive, int_rprimitive, object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyList_SetItem", + error_kind=ERR_FALSE, + steals=[False, False, True], +) + +# list[index_i64] = obj +method_op( + name="__setitem__", + arg_types=[list_rprimitive, int64_rprimitive, object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyList_SetItemInt64", + error_kind=ERR_FALSE, + steals=[False, False, True], + priority=2, +) + +# PyList_SET_ITEM does no error checking, +# and should only be used to fill in brand new lists. +new_list_set_item_op = custom_op( + arg_types=[list_rprimitive, int_rprimitive, object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyList_SetItemUnsafe", + error_kind=ERR_FALSE, + steals=[False, False, True], +) + +# list.append(obj) +list_append_op = method_op( + name="append", + arg_types=[list_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyList_Append", + error_kind=ERR_NEG_INT, +) + +# list.extend(obj) +list_extend_op = method_op( + name="extend", + arg_types=[list_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_Extend", + error_kind=ERR_MAGIC, +) + +# list.pop() +list_pop_last = method_op( + name="pop", + arg_types=[list_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_PopLast", + error_kind=ERR_MAGIC, +) + +# list.pop(index) +list_pop = method_op( + name="pop", + arg_types=[list_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_Pop", + error_kind=ERR_MAGIC, +) + +# list.count(obj) +method_op( + name="count", + arg_types=[list_rprimitive, object_rprimitive], + return_type=short_int_rprimitive, + c_function_name="CPyList_Count", + error_kind=ERR_MAGIC, +) + +# list.insert(index, obj) +method_op( + name="insert", + arg_types=[list_rprimitive, int_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyList_Insert", + error_kind=ERR_NEG_INT, +) + +# list.sort() +method_op( + name="sort", + arg_types=[list_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyList_Sort", + error_kind=ERR_NEG_INT, +) + +# list.reverse() +method_op( + name="reverse", + arg_types=[list_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyList_Reverse", + error_kind=ERR_NEG_INT, +) + +# list.remove(obj) +method_op( + name="remove", + arg_types=[list_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyList_Remove", + error_kind=ERR_NEG_INT, +) + +# list.index(obj) +method_op( + name="index", + arg_types=[list_rprimitive, object_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyList_Index", + error_kind=ERR_MAGIC, +) + +# list * int +binary_op( + name="*", + arg_types=[list_rprimitive, int_rprimitive], + return_type=list_rprimitive, + c_function_name="CPySequence_Multiply", + error_kind=ERR_MAGIC, +) + +# int * list +binary_op( + name="*", + arg_types=[int_rprimitive, list_rprimitive], + return_type=list_rprimitive, + c_function_name="CPySequence_RMultiply", + error_kind=ERR_MAGIC, +) + +# list[begin:end] +list_slice_op = custom_op( + arg_types=[list_rprimitive, int_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetSlice", + error_kind=ERR_MAGIC, +) + +supports_sequence_protocol = custom_op( + arg_types=[object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPySequence_Check", + error_kind=ERR_NEVER, +) + +sequence_get_item = custom_op( + arg_types=[object_rprimitive, c_pyssize_t_rprimitive], + return_type=object_rprimitive, + c_function_name="PySequence_GetItem", + error_kind=ERR_NEVER, +) + +sequence_get_slice = custom_op( + arg_types=[object_rprimitive, c_pyssize_t_rprimitive, c_pyssize_t_rprimitive], + return_type=object_rprimitive, + c_function_name="PySequence_GetSlice", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py new file mode 100644 index 000000000000..e9016e24c46d --- /dev/null +++ b/mypyc/primitives/misc_ops.py @@ -0,0 +1,279 @@ +"""Miscellaneous primitive ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import ( + bit_rprimitive, + bool_rprimitive, + c_int_rprimitive, + c_pointer_rprimitive, + c_pyssize_t_rprimitive, + dict_rprimitive, + int_rprimitive, + object_pointer_rprimitive, + object_rprimitive, + pointer_rprimitive, + str_rprimitive, + void_rtype, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + custom_op, + custom_primitive_op, + function_op, + load_address_op, +) + +# Get the 'bool' type object. +load_address_op(name="builtins.bool", type=object_rprimitive, src="PyBool_Type") + +# Get the 'range' type object. +load_address_op(name="builtins.range", type=object_rprimitive, src="PyRange_Type") + +# Get the boxed Python 'None' object +none_object_op = load_address_op(name="Py_None", type=object_rprimitive, src="_Py_NoneStruct") + +# Get the boxed object '...' +ellipsis_op = load_address_op(name="...", type=object_rprimitive, src="_Py_EllipsisObject") + +# Get the boxed NotImplemented object +not_implemented_op = load_address_op( + name="builtins.NotImplemented", type=object_rprimitive, src="_Py_NotImplementedStruct" +) + +# Get the boxed StopAsyncIteration object +stop_async_iteration_op = load_address_op( + name="builtins.StopAsyncIteration", type=object_rprimitive, src="PyExc_StopAsyncIteration" +) + +# id(obj) +function_op( + name="builtins.id", + arg_types=[object_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyTagged_Id", + error_kind=ERR_NEVER, +) + +# Return the result of obj.__await()__ or obj.__iter__() (if no __await__ exists) +coro_op = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_GetCoro", + error_kind=ERR_MAGIC, +) + +# Do obj.send(value), or a next(obj) if second arg is None. +# (This behavior is to match the PEP 380 spec for yield from.) +# Like next_raw_op, don't swallow StopIteration, +# but also don't propagate an error. +# Can return NULL: see next_op. +send_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyIter_Send", + error_kind=ERR_NEVER, +) + +# This is sort of unfortunate but oh well: yield_from_except performs most of the +# error handling logic in `yield from` operations. It returns a bool and passes +# a value by address. +# If the bool is true, then a StopIteration was received and we should return. +# If the bool is false, then the value should be yielded. +# The normal case is probably that it signals an exception, which gets +# propagated. +# Op used for "yield from" error handling. +# See comment in CPy_YieldFromErrorHandle for more information. +yield_from_except_op = custom_op( + arg_types=[object_rprimitive, object_pointer_rprimitive], + return_type=bool_rprimitive, + c_function_name="CPy_YieldFromErrorHandle", + error_kind=ERR_MAGIC, +) + +# Create method object from a callable object and self. +method_new_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyMethod_New", + error_kind=ERR_MAGIC, +) + +# Check if the current exception is a StopIteration and return its value if so. +# Treats "no exception" as StopIteration with a None value. +# If it is a different exception, re-reraise it. +check_stop_op = custom_op( + arg_types=[], + return_type=object_rprimitive, + c_function_name="CPy_FetchStopIterationValue", + error_kind=ERR_MAGIC, +) + +# Determine the most derived metaclass and check for metaclass conflicts. +# Arguments are (metaclass, bases). +py_calc_meta_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_CalculateMetaclass", + error_kind=ERR_MAGIC, + is_borrowed=True, +) + +# Import a module (plain) +import_op = custom_op( + arg_types=[str_rprimitive], + return_type=object_rprimitive, + c_function_name="PyImport_Import", + error_kind=ERR_MAGIC, +) + +# Table-driven import op. +import_many_op = custom_op( + arg_types=[ + object_rprimitive, + c_pointer_rprimitive, + object_rprimitive, + object_rprimitive, + object_rprimitive, + c_pointer_rprimitive, + ], + return_type=bit_rprimitive, + c_function_name="CPyImport_ImportMany", + error_kind=ERR_FALSE, +) + +# From import helper op +import_from_many_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyImport_ImportFromMany", + error_kind=ERR_MAGIC, +) + +# Get the sys.modules dictionary +get_module_dict_op = custom_op( + arg_types=[], + return_type=dict_rprimitive, + c_function_name="PyImport_GetModuleDict", + error_kind=ERR_NEVER, + is_borrowed=True, +) + +# isinstance(obj, cls) +slow_isinstance_op = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyObject_IsInstance", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, +) + +# Faster isinstance(obj, cls) that only works with native classes and doesn't perform +# type checking of the type argument. +fast_isinstance_op = function_op( + "builtins.isinstance", + arg_types=[object_rprimitive, object_rprimitive], + return_type=bool_rprimitive, + c_function_name="CPy_TypeCheck", + error_kind=ERR_NEVER, + priority=0, +) + +# bool(obj) with unboxed result +bool_op = function_op( + name="builtins.bool", + arg_types=[object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyObject_IsTrue", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, +) + +# slice(start, stop, step) +new_slice_op = function_op( + name="builtins.slice", + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + c_function_name="PySlice_New", + return_type=object_rprimitive, + error_kind=ERR_MAGIC, +) + +# type(obj) +type_op = function_op( + name="builtins.type", + arg_types=[object_rprimitive], + c_function_name="PyObject_Type", + return_type=object_rprimitive, + error_kind=ERR_NEVER, +) + +# Get 'builtins.type' (base class of all classes) +type_object_op = load_address_op(name="builtins.type", type=object_rprimitive, src="PyType_Type") + +# Create a heap type based on a template non-heap type. +# See CPyType_FromTemplate for more docs. +pytype_from_template_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive, str_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyType_FromTemplate", + error_kind=ERR_MAGIC, +) + +# Create a dataclass from an extension class. See +# CPyDataclass_SleightOfHand for more docs. +dataclass_sleight_of_hand = custom_op( + arg_types=[object_rprimitive, object_rprimitive, dict_rprimitive, dict_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyDataclass_SleightOfHand", + error_kind=ERR_FALSE, +) + +# Raise ValueError if length of first argument is not equal to the second argument. +# The first argument must be a list or a variable-length tuple. +check_unpack_count_op = custom_op( + arg_types=[object_rprimitive, c_pyssize_t_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPySequence_CheckUnpackCount", + error_kind=ERR_NEG_INT, +) + + +# Register an implementation for a singledispatch function +register_function = custom_op( + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPySingledispatch_RegisterFunction", + error_kind=ERR_MAGIC, +) + + +# Initialize a PyObject * item in a memory buffer (steal the value) +buf_init_item = custom_primitive_op( + name="buf_init_item", + arg_types=[pointer_rprimitive, c_pyssize_t_rprimitive, object_rprimitive], + return_type=void_rtype, + error_kind=ERR_NEVER, + steals=[False, False, True], +) + +# Get length of PyVarObject instance (e.g. list or tuple) +var_object_size = custom_primitive_op( + name="var_object_size", + arg_types=[object_rprimitive], + return_type=c_pyssize_t_rprimitive, + error_kind=ERR_NEVER, +) + +# Set the lazy value compute function of an TypeAliasType instance (Python 3.12+). +# This must only be used as part of initializing the object. Any existing value +# will be cleared. +set_type_alias_compute_function_op = custom_primitive_op( + name="set_type_alias_compute_function", + c_function_name="CPy_SetTypeAliasTypeComputeFunction", + # (alias object, value compute function) + arg_types=[object_rprimitive, object_rprimitive], + return_type=void_rtype, + error_kind=ERR_NEVER, +) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py new file mode 100644 index 000000000000..5e7ecb70f55d --- /dev/null +++ b/mypyc/primitives/registry.py @@ -0,0 +1,374 @@ +"""Utilities for defining primitive ops. + +Most of the ops can be automatically generated by matching against AST +nodes and types. For example, a func_op is automatically generated when +a specific function is called with the specific positional argument +count and argument types. + +Example op definition: + +list_len_op = func_op(name='builtins.len', + arg_types=[list_rprimitive], + result_type=short_int_rprimitive, + error_kind=ERR_NEVER, + emit=emit_len) + +This op is automatically generated for calls to len() with a single +list argument. The result type is short_int_rprimitive, and this +never raises an exception (ERR_NEVER). The function emit_len is used +to generate C for this op. The op can also be manually generated using +"list_len_op". Ops that are only generated automatically don't need to +be assigned to a module attribute. + +Ops defined with custom_op are only explicitly generated in +mypyc.irbuild and won't be generated automatically. They are always +assigned to a module attribute, as otherwise they won't be accessible. + +The actual ops are defined in other submodules of this package, grouped +by category. + +Most operations have fallback implementations that apply to all possible +arguments and types. For example, there are generic implementations of +arbitrary function and method calls, and binary operators. These generic +implementations are typically slower than specialized ones, but we tend +to rely on them for infrequently used ops. It's impractical to have +optimized implementations of all ops. +""" + +from __future__ import annotations + +from typing import Final, NamedTuple + +from mypyc.ir.ops import PrimitiveDescription, StealsDescription +from mypyc.ir.rtypes import RType + +# Error kind for functions that return negative integer on exception. This +# is only used for primitives. We translate it away during IR building. +ERR_NEG_INT: Final = 10 + + +class CFunctionDescription(NamedTuple): + name: str + arg_types: list[RType] + return_type: RType + var_arg_type: RType | None + truncated_type: RType | None + c_function_name: str + error_kind: int + steals: StealsDescription + is_borrowed: bool + ordering: list[int] | None + extra_int_constants: list[tuple[int, RType]] + priority: int + is_pure: bool + + +# A description for C load operations including LoadGlobal and LoadAddress +class LoadAddressDescription(NamedTuple): + name: str + type: RType + src: str # name of the target to load + + +# Primitive ops for method call (such as 'str.join') +method_call_ops: dict[str, list[PrimitiveDescription]] = {} + +# Primitive ops for top level function call (such as 'builtins.list') +function_ops: dict[str, list[PrimitiveDescription]] = {} + +# Primitive ops for binary operations +binary_ops: dict[str, list[PrimitiveDescription]] = {} + +# Primitive ops for unary ops +unary_ops: dict[str, list[PrimitiveDescription]] = {} + +builtin_names: dict[str, tuple[RType, str]] = {} + + +def method_op( + name: str, + arg_types: list[RType], + return_type: RType, + c_function_name: str, + error_kind: int, + var_arg_type: RType | None = None, + truncated_type: RType | None = None, + ordering: list[int] | None = None, + extra_int_constants: list[tuple[int, RType]] | None = None, + steals: StealsDescription = False, + is_borrowed: bool = False, + priority: int = 1, + is_pure: bool = False, +) -> PrimitiveDescription: + """Define a c function call op that replaces a method call. + + This will be automatically generated by matching against the AST. + + Args: + name: short name of the method (for example, 'append') + arg_types: argument types; the receiver is always the first argument + return_type: type of the return value. Use void_rtype to represent void. + c_function_name: name of the C function to call + error_kind: how errors are represented in the result (one of ERR_*) + var_arg_type: type of all variable arguments + truncated_type: type to truncated to(See Truncate for info) + if it's defined both return_type and it should be non-referenced + integer types or bool type + ordering: optional ordering of the arguments, if defined, + reorders the arguments accordingly. + should never be used together with var_arg_type. + all the other arguments(such as arg_types) are in the order + accepted by the python syntax(before reordering) + extra_int_constants: optional extra integer constants as the last arguments to a C call + steals: description of arguments that this steals (ref count wise) + is_borrowed: if True, returned value is borrowed (no need to decrease refcount) + priority: if multiple ops match, the one with the highest priority is picked + is_pure: if True, declare that the C function has no side effects, takes immutable + arguments, and never raises an exception + """ + if extra_int_constants is None: + extra_int_constants = [] + ops = method_call_ops.setdefault(name, []) + desc = PrimitiveDescription( + name, + arg_types, + return_type, + var_arg_type, + truncated_type, + c_function_name, + error_kind, + steals, + is_borrowed, + ordering, + extra_int_constants, + priority, + is_pure=is_pure, + ) + ops.append(desc) + return desc + + +def function_op( + name: str, + arg_types: list[RType], + return_type: RType, + c_function_name: str, + error_kind: int, + var_arg_type: RType | None = None, + truncated_type: RType | None = None, + ordering: list[int] | None = None, + extra_int_constants: list[tuple[int, RType]] | None = None, + steals: StealsDescription = False, + is_borrowed: bool = False, + priority: int = 1, +) -> PrimitiveDescription: + """Define a C function call op that replaces a function call. + + This will be automatically generated by matching against the AST. + + Most arguments are similar to method_op(). + + Args: + name: full name of the function + arg_types: positional argument types for which this applies + """ + if extra_int_constants is None: + extra_int_constants = [] + ops = function_ops.setdefault(name, []) + desc = PrimitiveDescription( + name, + arg_types, + return_type, + var_arg_type=var_arg_type, + truncated_type=truncated_type, + c_function_name=c_function_name, + error_kind=error_kind, + steals=steals, + is_borrowed=is_borrowed, + ordering=ordering, + extra_int_constants=extra_int_constants, + priority=priority, + is_pure=False, + ) + ops.append(desc) + return desc + + +def binary_op( + name: str, + arg_types: list[RType], + return_type: RType, + error_kind: int, + c_function_name: str | None = None, + primitive_name: str | None = None, + var_arg_type: RType | None = None, + truncated_type: RType | None = None, + ordering: list[int] | None = None, + extra_int_constants: list[tuple[int, RType]] | None = None, + steals: StealsDescription = False, + is_borrowed: bool = False, + priority: int = 1, +) -> PrimitiveDescription: + """Define a c function call op for a binary operation. + + This will be automatically generated by matching against the AST. + + Most arguments are similar to method_op(), but exactly two argument types + are expected. + """ + assert c_function_name is not None or primitive_name is not None + assert not (c_function_name is not None and primitive_name is not None) + if extra_int_constants is None: + extra_int_constants = [] + ops = binary_ops.setdefault(name, []) + desc = PrimitiveDescription( + name=primitive_name or name, + arg_types=arg_types, + return_type=return_type, + var_arg_type=var_arg_type, + truncated_type=truncated_type, + c_function_name=c_function_name, + error_kind=error_kind, + steals=steals, + is_borrowed=is_borrowed, + ordering=ordering, + extra_int_constants=extra_int_constants, + priority=priority, + is_pure=False, + ) + ops.append(desc) + return desc + + +def custom_op( + arg_types: list[RType], + return_type: RType, + c_function_name: str, + error_kind: int, + var_arg_type: RType | None = None, + truncated_type: RType | None = None, + ordering: list[int] | None = None, + extra_int_constants: list[tuple[int, RType]] | None = None, + steals: StealsDescription = False, + is_borrowed: bool = False, + *, + is_pure: bool = False, +) -> CFunctionDescription: + """Create a one-off CallC op that can't be automatically generated from the AST. + + Most arguments are similar to method_op(). + """ + if extra_int_constants is None: + extra_int_constants = [] + return CFunctionDescription( + "", + arg_types, + return_type, + var_arg_type, + truncated_type, + c_function_name, + error_kind, + steals, + is_borrowed, + ordering, + extra_int_constants, + 0, + is_pure=is_pure, + ) + + +def custom_primitive_op( + name: str, + arg_types: list[RType], + return_type: RType, + error_kind: int, + c_function_name: str | None = None, + var_arg_type: RType | None = None, + truncated_type: RType | None = None, + ordering: list[int] | None = None, + extra_int_constants: list[tuple[int, RType]] | None = None, + steals: StealsDescription = False, + is_borrowed: bool = False, + is_pure: bool = False, +) -> PrimitiveDescription: + """Define a primitive op that can't be automatically generated based on the AST. + + Most arguments are similar to method_op(). + """ + if extra_int_constants is None: + extra_int_constants = [] + return PrimitiveDescription( + name=name, + arg_types=arg_types, + return_type=return_type, + var_arg_type=var_arg_type, + truncated_type=truncated_type, + c_function_name=c_function_name, + error_kind=error_kind, + steals=steals, + is_borrowed=is_borrowed, + ordering=ordering, + extra_int_constants=extra_int_constants, + priority=0, + is_pure=is_pure, + ) + + +def unary_op( + name: str, + arg_type: RType, + return_type: RType, + c_function_name: str, + error_kind: int, + truncated_type: RType | None = None, + ordering: list[int] | None = None, + extra_int_constants: list[tuple[int, RType]] | None = None, + steals: StealsDescription = False, + is_borrowed: bool = False, + priority: int = 1, + is_pure: bool = False, +) -> PrimitiveDescription: + """Define a primitive op for an unary operation. + + This will be automatically generated by matching against the AST. + + Most arguments are similar to method_op(), but exactly one argument type + is expected. + """ + if extra_int_constants is None: + extra_int_constants = [] + ops = unary_ops.setdefault(name, []) + desc = PrimitiveDescription( + name, + [arg_type], + return_type, + var_arg_type=None, + truncated_type=truncated_type, + c_function_name=c_function_name, + error_kind=error_kind, + steals=steals, + is_borrowed=is_borrowed, + ordering=ordering, + extra_int_constants=extra_int_constants, + priority=priority, + is_pure=is_pure, + ) + ops.append(desc) + return desc + + +def load_address_op(name: str, type: RType, src: str) -> LoadAddressDescription: + assert name not in builtin_names, "already defined: %s" % name + builtin_names[name] = (type, src) + return LoadAddressDescription(name, type, src) + + +# Import various modules that set up global state. +import mypyc.primitives.bytes_ops +import mypyc.primitives.dict_ops +import mypyc.primitives.float_ops +import mypyc.primitives.int_ops +import mypyc.primitives.list_ops +import mypyc.primitives.misc_ops +import mypyc.primitives.str_ops +import mypyc.primitives.tuple_ops # noqa: F401 diff --git a/mypyc/primitives/set_ops.py b/mypyc/primitives/set_ops.py new file mode 100644 index 000000000000..fcfb7847dc7d --- /dev/null +++ b/mypyc/primitives/set_ops.py @@ -0,0 +1,121 @@ +"""Primitive set (and frozenset) ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC +from mypyc.ir.rtypes import ( + bit_rprimitive, + bool_rprimitive, + c_int_rprimitive, + object_rprimitive, + pointer_rprimitive, + set_rprimitive, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + binary_op, + function_op, + load_address_op, + method_op, +) + +# Get the 'builtins.set' type object. +load_address_op(name="builtins.set", type=object_rprimitive, src="PySet_Type") + +# Get the 'builtins.frozenset' tyoe object. +load_address_op(name="builtins.frozenset", type=object_rprimitive, src="PyFrozenSet_Type") + +# Construct an empty set. +new_set_op = function_op( + name="builtins.set", + arg_types=[], + return_type=set_rprimitive, + c_function_name="PySet_New", + error_kind=ERR_MAGIC, + extra_int_constants=[(0, pointer_rprimitive)], +) + +# set(obj) +function_op( + name="builtins.set", + arg_types=[object_rprimitive], + return_type=set_rprimitive, + c_function_name="PySet_New", + error_kind=ERR_MAGIC, +) + +# frozenset(obj) +function_op( + name="builtins.frozenset", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyFrozenSet_New", + error_kind=ERR_MAGIC, +) + +# item in set +set_in_op = binary_op( + name="in", + arg_types=[object_rprimitive, set_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PySet_Contains", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + ordering=[1, 0], +) + +# set.remove(obj) +method_op( + name="remove", + arg_types=[set_rprimitive, object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPySet_Remove", + error_kind=ERR_FALSE, +) + +# set.discard(obj) +method_op( + name="discard", + arg_types=[set_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PySet_Discard", + error_kind=ERR_NEG_INT, +) + +# set.add(obj) +set_add_op = method_op( + name="add", + arg_types=[set_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PySet_Add", + error_kind=ERR_NEG_INT, +) + +# set.update(obj) +# +# This is not a public API but looks like it should be fine. +set_update_op = method_op( + name="update", + arg_types=[set_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="_PySet_Update", + error_kind=ERR_NEG_INT, +) + +# set.clear() +method_op( + name="clear", + arg_types=[set_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PySet_Clear", + error_kind=ERR_NEG_INT, +) + +# set.pop() +method_op( + name="pop", + arg_types=[set_rprimitive], + return_type=object_rprimitive, + c_function_name="PySet_Pop", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py new file mode 100644 index 000000000000..0accffd86a17 --- /dev/null +++ b/mypyc/primitives/str_ops.py @@ -0,0 +1,261 @@ +"""Primitive str ops.""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import ( + RType, + bit_rprimitive, + bool_rprimitive, + bytes_rprimitive, + c_int_rprimitive, + c_pyssize_t_rprimitive, + int_rprimitive, + list_rprimitive, + object_rprimitive, + pointer_rprimitive, + str_rprimitive, +) +from mypyc.primitives.registry import ( + ERR_NEG_INT, + binary_op, + custom_op, + function_op, + load_address_op, + method_op, +) + +# Get the 'str' type object. +load_address_op(name="builtins.str", type=object_rprimitive, src="PyUnicode_Type") + +# str(obj) +str_op = function_op( + name="builtins.str", + arg_types=[object_rprimitive], + return_type=str_rprimitive, + c_function_name="PyObject_Str", + error_kind=ERR_MAGIC, +) + +# str1 + str2 +binary_op( + name="+", + arg_types=[str_rprimitive, str_rprimitive], + return_type=str_rprimitive, + c_function_name="PyUnicode_Concat", + error_kind=ERR_MAGIC, +) + +# str1 += str2 +# +# PyUnicode_Append makes an effort to reuse the LHS when the refcount +# is 1. This is super dodgy but oh well, the interpreter does it. +binary_op( + name="+=", + arg_types=[str_rprimitive, str_rprimitive], + return_type=str_rprimitive, + c_function_name="CPyStr_Append", + error_kind=ERR_MAGIC, + steals=[True, False], +) + +unicode_compare = custom_op( + arg_types=[str_rprimitive, str_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyUnicode_Compare", + error_kind=ERR_NEVER, +) + +# str[index] (for an int index) +method_op( + name="__getitem__", + arg_types=[str_rprimitive, int_rprimitive], + return_type=str_rprimitive, + c_function_name="CPyStr_GetItem", + error_kind=ERR_MAGIC, +) + +# str[begin:end] +str_slice_op = custom_op( + arg_types=[str_rprimitive, int_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyStr_GetSlice", + error_kind=ERR_MAGIC, +) + +# str.join(obj) +method_op( + name="join", + arg_types=[str_rprimitive, object_rprimitive], + return_type=str_rprimitive, + c_function_name="PyUnicode_Join", + error_kind=ERR_MAGIC, +) + +str_build_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=str_rprimitive, + c_function_name="CPyStr_Build", + error_kind=ERR_MAGIC, + var_arg_type=str_rprimitive, +) + +# str.startswith(str) +method_op( + name="startswith", + arg_types=[str_rprimitive, str_rprimitive], + return_type=bool_rprimitive, + c_function_name="CPyStr_Startswith", + error_kind=ERR_NEVER, +) + +# str.endswith(str) +method_op( + name="endswith", + arg_types=[str_rprimitive, str_rprimitive], + return_type=bool_rprimitive, + c_function_name="CPyStr_Endswith", + error_kind=ERR_NEVER, +) + +# str.split(...) +str_split_types: list[RType] = [str_rprimitive, str_rprimitive, int_rprimitive] +str_split_functions = ["PyUnicode_Split", "PyUnicode_Split", "CPyStr_Split"] +str_split_constants: list[list[tuple[int, RType]]] = [ + [(0, pointer_rprimitive), (-1, c_int_rprimitive)], + [(-1, c_int_rprimitive)], + [], +] +for i in range(len(str_split_types)): + method_op( + name="split", + arg_types=str_split_types[0 : i + 1], + return_type=list_rprimitive, + c_function_name=str_split_functions[i], + extra_int_constants=str_split_constants[i], + error_kind=ERR_MAGIC, + ) + +# str.replace(old, new) +method_op( + name="replace", + arg_types=[str_rprimitive, str_rprimitive, str_rprimitive], + return_type=str_rprimitive, + c_function_name="PyUnicode_Replace", + error_kind=ERR_MAGIC, + extra_int_constants=[(-1, c_int_rprimitive)], +) + +# str.replace(old, new, count) +method_op( + name="replace", + arg_types=[str_rprimitive, str_rprimitive, str_rprimitive, int_rprimitive], + return_type=str_rprimitive, + c_function_name="CPyStr_Replace", + error_kind=ERR_MAGIC, +) + +# check if a string is true (isn't an empty string) +str_check_if_true = custom_op( + arg_types=[str_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyStr_IsTrue", + error_kind=ERR_NEVER, +) + +str_ssize_t_size_op = custom_op( + arg_types=[str_rprimitive], + return_type=c_pyssize_t_rprimitive, + c_function_name="CPyStr_Size_size_t", + error_kind=ERR_NEG_INT, +) + +# obj.decode() +method_op( + name="decode", + arg_types=[bytes_rprimitive], + return_type=str_rprimitive, + c_function_name="CPy_Decode", + error_kind=ERR_MAGIC, + extra_int_constants=[(0, pointer_rprimitive), (0, pointer_rprimitive)], +) + +# obj.decode(encoding) +method_op( + name="decode", + arg_types=[bytes_rprimitive, str_rprimitive], + return_type=str_rprimitive, + c_function_name="CPy_Decode", + error_kind=ERR_MAGIC, + extra_int_constants=[(0, pointer_rprimitive)], +) + +# obj.decode(encoding, errors) +method_op( + name="decode", + arg_types=[bytes_rprimitive, str_rprimitive, str_rprimitive], + return_type=str_rprimitive, + c_function_name="CPy_Decode", + error_kind=ERR_MAGIC, +) + +# str.encode() +method_op( + name="encode", + arg_types=[str_rprimitive], + return_type=bytes_rprimitive, + c_function_name="CPy_Encode", + error_kind=ERR_MAGIC, + extra_int_constants=[(0, pointer_rprimitive), (0, pointer_rprimitive)], +) + +# str.encode(encoding) +method_op( + name="encode", + arg_types=[str_rprimitive, str_rprimitive], + return_type=bytes_rprimitive, + c_function_name="CPy_Encode", + error_kind=ERR_MAGIC, + extra_int_constants=[(0, pointer_rprimitive)], +) + +# str.encode(encoding) - utf8 strict specialization +str_encode_utf8_strict = custom_op( + arg_types=[str_rprimitive], + return_type=bytes_rprimitive, + c_function_name="PyUnicode_AsUTF8String", + error_kind=ERR_MAGIC, +) + +# str.encode(encoding) - ascii strict specialization +str_encode_ascii_strict = custom_op( + arg_types=[str_rprimitive], + return_type=bytes_rprimitive, + c_function_name="PyUnicode_AsASCIIString", + error_kind=ERR_MAGIC, +) + +# str.encode(encoding) - latin1 strict specialization +str_encode_latin1_strict = custom_op( + arg_types=[str_rprimitive], + return_type=bytes_rprimitive, + c_function_name="PyUnicode_AsLatin1String", + error_kind=ERR_MAGIC, +) + +# str.encode(encoding, errors) +method_op( + name="encode", + arg_types=[str_rprimitive, str_rprimitive, str_rprimitive], + return_type=bytes_rprimitive, + c_function_name="CPy_Encode", + error_kind=ERR_MAGIC, +) + +function_op( + name="builtins.ord", + arg_types=[str_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyStr_Ord", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/primitives/tuple_ops.py b/mypyc/primitives/tuple_ops.py new file mode 100644 index 000000000000..0ea0243dc18b --- /dev/null +++ b/mypyc/primitives/tuple_ops.py @@ -0,0 +1,83 @@ +"""Primitive tuple ops for *variable-length* tuples. + +Note: Varying-length tuples are represented as boxed Python tuple +objects, i.e. tuple_rprimitive (RPrimitive), not RTuple. +""" + +from __future__ import annotations + +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC +from mypyc.ir.rtypes import ( + bit_rprimitive, + c_pyssize_t_rprimitive, + int_rprimitive, + list_rprimitive, + object_rprimitive, + tuple_rprimitive, +) +from mypyc.primitives.registry import custom_op, function_op, load_address_op, method_op + +# Get the 'builtins.tuple' type object. +load_address_op(name="builtins.tuple", type=object_rprimitive, src="PyTuple_Type") + +# tuple[index] (for an int index) +tuple_get_item_op = method_op( + name="__getitem__", + arg_types=[tuple_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPySequenceTuple_GetItem", + error_kind=ERR_MAGIC, +) + +# Construct a boxed tuple from items: (item1, item2, ...) +new_tuple_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=tuple_rprimitive, + c_function_name="PyTuple_Pack", + error_kind=ERR_MAGIC, + var_arg_type=object_rprimitive, +) + +new_tuple_with_length_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=tuple_rprimitive, + c_function_name="PyTuple_New", + error_kind=ERR_MAGIC, +) + +# PyTuple_SET_ITEM does no error checking, +# and should only be used to fill in brand new tuples. +new_tuple_set_item_op = custom_op( + arg_types=[tuple_rprimitive, int_rprimitive, object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPySequenceTuple_SetItemUnsafe", + error_kind=ERR_FALSE, + steals=[False, False, True], +) + +# Construct tuple from a list. +list_tuple_op = function_op( + name="builtins.tuple", + arg_types=[list_rprimitive], + return_type=tuple_rprimitive, + c_function_name="PyList_AsTuple", + error_kind=ERR_MAGIC, + priority=2, +) + +# Construct tuple from an arbitrary (iterable) object. +function_op( + name="builtins.tuple", + arg_types=[object_rprimitive], + return_type=tuple_rprimitive, + c_function_name="PySequence_Tuple", + error_kind=ERR_MAGIC, +) + +# tuple[begin:end] +tuple_slice_op = custom_op( + arg_types=[tuple_rprimitive, int_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPySequenceTuple_GetSlice", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/rt_subtype.py b/mypyc/rt_subtype.py new file mode 100644 index 000000000000..004e56ed75bc --- /dev/null +++ b/mypyc/rt_subtype.py @@ -0,0 +1,77 @@ +"""'Runtime subtype' check for RTypes. + +A type S is a runtime subtype of T if a value of type S can be used at runtime +when a value of type T is expected without requiring any runtime conversions. + +For boxed types, runtime subtyping is the same as regular subtyping. +Unboxed subtypes, on the other hand, are not runtime subtypes of object +(since they require boxing to be used as an object), but short ints +are runtime subtypes of int. + +Subtyping is used to determine whether an object can be in a +particular place and runtime subtyping is used to determine whether a +coercion is necessary first. +""" + +from __future__ import annotations + +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RPrimitive, + RStruct, + RTuple, + RType, + RTypeVisitor, + RUnion, + RVoid, + is_bit_rprimitive, + is_bool_rprimitive, + is_int_rprimitive, + is_short_int_rprimitive, +) +from mypyc.subtype import is_subtype + + +def is_runtime_subtype(left: RType, right: RType) -> bool: + return left.accept(RTSubtypeVisitor(right)) + + +class RTSubtypeVisitor(RTypeVisitor[bool]): + """Is left a runtime subtype of right? + + A few special cases such as right being 'object' are handled in + is_runtime_subtype and don't need to be covered here. + """ + + def __init__(self, right: RType) -> None: + self.right = right + + def visit_rinstance(self, left: RInstance) -> bool: + return is_subtype(left, self.right) + + def visit_runion(self, left: RUnion) -> bool: + return not self.right.is_unboxed and is_subtype(left, self.right) + + def visit_rprimitive(self, left: RPrimitive) -> bool: + if is_short_int_rprimitive(left) and is_int_rprimitive(self.right): + return True + if is_bit_rprimitive(left) and is_bool_rprimitive(self.right): + return True + return left is self.right + + def visit_rtuple(self, left: RTuple) -> bool: + if isinstance(self.right, RTuple): + return len(self.right.types) == len(left.types) and all( + is_runtime_subtype(t1, t2) for t1, t2 in zip(left.types, self.right.types) + ) + return False + + def visit_rstruct(self, left: RStruct) -> bool: + return isinstance(self.right, RStruct) and self.right.name == left.name + + def visit_rarray(self, left: RArray) -> bool: + return left == self.right + + def visit_rvoid(self, left: RVoid) -> bool: + return isinstance(self.right, RVoid) diff --git a/mypyc/sametype.py b/mypyc/sametype.py new file mode 100644 index 000000000000..1b811d4e9041 --- /dev/null +++ b/mypyc/sametype.py @@ -0,0 +1,83 @@ +"""Same type check for RTypes.""" + +from __future__ import annotations + +from mypyc.ir.func_ir import FuncSignature +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RPrimitive, + RStruct, + RTuple, + RType, + RTypeVisitor, + RUnion, + RVoid, +) + + +def is_same_type(a: RType, b: RType) -> bool: + return a.accept(SameTypeVisitor(b)) + + +def is_same_signature(a: FuncSignature, b: FuncSignature) -> bool: + return ( + len(a.args) == len(b.args) + and is_same_type(a.ret_type, b.ret_type) + and all( + is_same_type(t1.type, t2.type) and t1.name == t2.name for t1, t2 in zip(a.args, b.args) + ) + ) + + +def is_same_method_signature(a: FuncSignature, b: FuncSignature) -> bool: + return ( + len(a.args) == len(b.args) + and is_same_type(a.ret_type, b.ret_type) + and all( + is_same_type(t1.type, t2.type) + and ((t1.pos_only and t2.pos_only) or t1.name == t2.name) + and t1.optional == t2.optional + for t1, t2 in zip(a.args[1:], b.args[1:]) + ) + ) + + +class SameTypeVisitor(RTypeVisitor[bool]): + def __init__(self, right: RType) -> None: + self.right = right + + def visit_rinstance(self, left: RInstance) -> bool: + return isinstance(self.right, RInstance) and left.name == self.right.name + + def visit_runion(self, left: RUnion) -> bool: + if isinstance(self.right, RUnion): + items = list(self.right.items) + for left_item in left.items: + for j, right_item in enumerate(items): + if is_same_type(left_item, right_item): + del items[j] + break + else: + return False + return not items + return False + + def visit_rprimitive(self, left: RPrimitive) -> bool: + return left is self.right + + def visit_rtuple(self, left: RTuple) -> bool: + return ( + isinstance(self.right, RTuple) + and len(self.right.types) == len(left.types) + and all(is_same_type(t1, t2) for t1, t2 in zip(left.types, self.right.types)) + ) + + def visit_rstruct(self, left: RStruct) -> bool: + return isinstance(self.right, RStruct) and self.right.name == left.name + + def visit_rarray(self, left: RArray) -> bool: + return left == self.right + + def visit_rvoid(self, left: RVoid) -> bool: + return isinstance(self.right, RVoid) diff --git a/mypyc/subtype.py b/mypyc/subtype.py new file mode 100644 index 000000000000..726a48d7a01d --- /dev/null +++ b/mypyc/subtype.py @@ -0,0 +1,88 @@ +"""Subtype check for RTypes.""" + +from __future__ import annotations + +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RPrimitive, + RStruct, + RTuple, + RType, + RTypeVisitor, + RUnion, + RVoid, + is_bit_rprimitive, + is_bool_rprimitive, + is_fixed_width_rtype, + is_int_rprimitive, + is_object_rprimitive, + is_short_int_rprimitive, + is_tagged, + is_tuple_rprimitive, +) + + +def is_subtype(left: RType, right: RType) -> bool: + if is_object_rprimitive(right): + return True + elif isinstance(right, RUnion): + if isinstance(left, RUnion): + for left_item in left.items: + if not any(is_subtype(left_item, right_item) for right_item in right.items): + return False + return True + else: + return any(is_subtype(left, item) for item in right.items) + return left.accept(SubtypeVisitor(right)) + + +class SubtypeVisitor(RTypeVisitor[bool]): + """Is left a subtype of right? + + A few special cases such as right being 'object' are handled in + is_subtype and don't need to be covered here. + """ + + def __init__(self, right: RType) -> None: + self.right = right + + def visit_rinstance(self, left: RInstance) -> bool: + return isinstance(self.right, RInstance) and self.right.class_ir in left.class_ir.mro + + def visit_runion(self, left: RUnion) -> bool: + return all(is_subtype(item, self.right) for item in left.items) + + def visit_rprimitive(self, left: RPrimitive) -> bool: + right = self.right + if is_bool_rprimitive(left): + if is_tagged(right) or is_fixed_width_rtype(right): + return True + elif is_bit_rprimitive(left): + if is_bool_rprimitive(right) or is_tagged(right) or is_fixed_width_rtype(right): + return True + elif is_short_int_rprimitive(left): + if is_int_rprimitive(right): + return True + elif is_fixed_width_rtype(left): + if is_int_rprimitive(right): + return True + return left is right + + def visit_rtuple(self, left: RTuple) -> bool: + if is_tuple_rprimitive(self.right): + return True + if isinstance(self.right, RTuple): + return len(self.right.types) == len(left.types) and all( + is_subtype(t1, t2) for t1, t2 in zip(left.types, self.right.types) + ) + return False + + def visit_rstruct(self, left: RStruct) -> bool: + return isinstance(self.right, RStruct) and self.right.name == left.name + + def visit_rarray(self, left: RArray) -> bool: + return left == self.right + + def visit_rvoid(self, left: RVoid) -> bool: + return isinstance(self.right, RVoid) diff --git a/mypyc/test-data/alwaysdefined.test b/mypyc/test-data/alwaysdefined.test new file mode 100644 index 000000000000..e8c44d8fc548 --- /dev/null +++ b/mypyc/test-data/alwaysdefined.test @@ -0,0 +1,732 @@ +-- Test cases for always defined attributes. +-- +-- If class C has attributes x and y that are always defined, the output will +-- have a line like this: +-- +-- C: [x, y] + +[case testAlwaysDefinedSimple] +class C: + def __init__(self, x: int) -> None: + self.x = x +[out] +C: [x] + +[case testAlwaysDefinedFail] +class MethodCall: + def __init__(self, x: int) -> None: + self.f() + self.x = x + + def f(self) -> None: + pass + +class FuncCall: + def __init__(self, x: int) -> None: + f(x) + self.x = x + f(self) + self.y = x + +class GetAttr: + x: int + def __init__(self, x: int) -> None: + a = self.x + self.x = x + +class _Base: + def __init__(self) -> None: + f(self) + +class CallSuper(_Base): + def __init__(self, x: int) -> None: + super().__init__() + self.x = x + +class Lambda: + def __init__(self, x: int) -> None: + f = lambda x: x + 1 + self.x = x + g = lambda x: self + self.y = x + +class If: + def __init__(self, x: int) -> None: + self.a = 1 + if x: + self.x = x + else: + self.y = 1 + +class Deletable: + __deletable__ = ('x', 'y') + + def __init__(self) -> None: + self.x = 0 + self.y = 1 + self.z = 2 + +class PrimitiveWithSelf: + def __init__(self, s: str) -> None: + self.x = getattr(self, s) + +def f(a) -> None: pass +[out] +MethodCall: [] +FuncCall: [x] +GetAttr: [] +CallSuper: [] +Lambda: [] +If: [a] +Deletable: [z] +PrimitiveWithSelf: [] + +[case testAlwaysDefinedConditional] +class IfAlways: + def __init__(self, x: int, y: int) -> None: + if x: + self.x = x + self.y = y + elif y: + self.x = y + self.y = x + else: + self.x = 0 + self.y = 0 + self.z = 0 + +class IfSometimes1: + def __init__(self, x: int, y: int) -> None: + if x: + self.x = x + self.y = y + elif y: + self.z = y + self.y = x + else: + self.y = 0 + self.a = 0 + +class IfSometimes2: + def __init__(self, x: int, y: int) -> None: + if x: + self.x = x + self.y = y + +class IfStopAnalysis1: + def __init__(self, x: int, y: int) -> None: + if x: + self.x = x + f(self) + else: + self.x = x + self.y = y + +class IfStopAnalysis2: + def __init__(self, x: int, y: int) -> None: + if x: + self.x = x + else: + self.x = x + f(self) + self.y = y + +class IfStopAnalysis3: + def __init__(self, x: int, y: int) -> None: + if x: + self.x = x + else: + f(self) + self.x = x + self.y = y + +class IfConditionalAndNonConditional1: + def __init__(self, x: int) -> None: + self.x = 0 + if x: + self.x = x + +class IfConditionalAndNonConditional2: + def __init__(self, x: int) -> None: + # x is not considered always defined, since the second assignment may + # either initialize or update. + if x: + self.x = x + self.x = 0 + +def f(a) -> None: pass +[out] +IfAlways: [x, y, z] +IfSometimes1: [y] +IfSometimes2: [y] +IfStopAnalysis1: [x] +IfStopAnalysis2: [x] +IfStopAnalysis3: [] +IfConditionalAndNonConditional1: [x] +IfConditionalAndNonConditional2: [] + +[case testAlwaysDefinedExpressions] +from typing import Dict, List, Set, Optional, cast +from typing_extensions import Final + +import other + +class C: pass + +class Collections: + def __init__(self, x: int) -> None: + self.l = [x] + self.d: Dict[str, str] = {} + self.s: Set[int] = set() + self.d2 = {'x': x} + self.s2 = {x} + self.l2 = [f(), None] * x + self.t = tuple(self.l2) + +class Comparisons: + def __init__(self, y: int, c: C, s: str, o: Optional[str]) -> None: + self.n1 = y < 5 + self.n2 = y == 5 + self.c1 = y is c + self.c2 = y is not c + self.o1 = o is None + self.o2 = o is not None + self.s = s < 'x' + +class BinaryOps: + def __init__(self, x: int, s: str) -> None: + self.a = x + 2 + self.b = x & 2 + self.c = x * 2 + self.d = -x + self.e = 'x' + s + self.f = x << x + +g = 2 + +class LocalsAndGlobals: + def __init__(self, x: int) -> None: + t = x + 1 + self.a = t - t + self.g = g + +class Booleans: + def __init__(self, x: int, b: bool) -> None: + self.a = True + self.b = False + self.c = not b + self.d = b or b + self.e = b and b + +F: Final = 3 + +class ModuleFinal: + def __init__(self) -> None: + self.a = F + self.b = other.Y + +class ClassFinal: + F: Final = 3 + + def __init__(self) -> None: + self.a = ClassFinal.F + +class Literals: + def __init__(self) -> None: + self.a = 'x' + self.b = b'x' + self.c = 2.2 + +class ListComprehension: + def __init__(self, x: List[int]) -> None: + self.a = [i + 1 for i in x] + +class Helper: + def __init__(self, arg) -> None: + self.x = 0 + + def foo(self, arg) -> int: + return 1 + +class AttrAccess: + def __init__(self, o: Helper) -> None: + self.x = o.x + o.x = o.x + 1 + self.y = o.foo(self.x) + o.foo(self) + self.z = 1 + +class Construct: + def __init__(self) -> None: + self.x = Helper(1) + self.y = Helper(self) + +class IsInstance: + def __init__(self, x: object) -> None: + if isinstance(x, str): + self.x = 0 + elif isinstance(x, Helper): + self.x = 1 + elif isinstance(x, (list, tuple)): + self.x = 2 + else: + self.x = 3 + +class Cast: + def __init__(self, x: object) -> None: + self.x = cast(int, x) + self.s = cast(str, x) + self.c = cast(Cast, x) + +class PropertyAccessGetter: + def __init__(self, other: PropertyAccessGetter) -> None: + self.x = other.p + self.y = 1 + self.z = self.p + + @property + def p(self) -> int: + return 0 + +class PropertyAccessSetter: + def __init__(self, other: PropertyAccessSetter) -> None: + other.p = 1 + self.y = 1 + self.z = self.p + + @property + def p(self) -> int: + return 0 + + @p.setter + def p(self, x: int) -> None: + pass + +def f() -> int: + return 0 + +[file other.py] +# Not compiled +from typing_extensions import Final + +Y: Final = 3 + +[out] +C: [] +Collections: [d, d2, l, l2, s, s2, t] +Comparisons: [c1, c2, n1, n2, o1, o2, s] +BinaryOps: [a, b, c, d, e, f] +LocalsAndGlobals: [a, g] +Booleans: [a, b, c, d, e] +ModuleFinal: [a, b] +ClassFinal: [F, a] +Literals: [a, b, c] +ListComprehension: [a] +Helper: [x] +AttrAccess: [x, y] +Construct: [x] +IsInstance: [x] +Cast: [c, s, x] +PropertyAccessGetter: [x, y] +PropertyAccessSetter: [y] + +[case testAlwaysDefinedExpressions2] +from typing import List, Tuple + +class C: + def __init__(self) -> None: + self.x = 0 + +class AttributeRef: + def __init__(self, c: C) -> None: + self.aa = c.x + self.bb = self.aa + if c is not None: + self.z = 0 + self.cc = 0 + self.dd = self.z + +class ListOps: + def __init__(self, x: List[int], n: int) -> None: + self.a = len(x) + self.b = x[n] + self.c = [y + 1 for y in x] + +class TupleOps: + def __init__(self, t: Tuple[int, str]) -> None: + x, y = t + self.x = x + self.y = t[0] + s = x, y + self.z = s + +class IfExpr: + def __init__(self, x: int) -> None: + self.a = 1 if x < 5 else 2 + +class Base: + def __init__(self, x: int) -> None: + self.x = x + +class Derived1(Base): + def __init__(self, y: int) -> None: + self.aa = y + super().__init__(y) + self.bb = y + +class Derived2(Base): + pass + +class Conditionals: + def __init__(self, b: bool, n: int) -> None: + if not (n == 5 or n >= n + 1): + self.a = b + else: + self.a = not b + if b: + self.b = 2 + else: + self.b = 4 + +[out] +C: [x] +AttributeRef: [aa, bb, cc, dd] +ListOps: [a, b, c] +TupleOps: [x, y, z] +IfExpr: [a] +Base: [x] +Derived1: [aa, bb, x] +Derived2: [x] +Conditionals: [a, b] + +[case testAlwaysDefinedStatements] +from typing import Any, List, Optional, Iterable + +class Return: + def __init__(self, x: int) -> None: + self.x = x + if x > 5: + self.y = 1 + return + self.y = 2 + self.z = x + +class While: + def __init__(self, x: int) -> None: + n = 2 + while x > 0: + n *=2 + x -= 1 + self.a = n + while x < 5: + self.b = 1 + self.b += 1 + +class Try: + def __init__(self, x: List[int]) -> None: + self.a = 0 + try: + self.b = x[0] + except: + self.c = x + self.d = 0 + try: + self.e = x[0] + except: + self.e = 1 + +class TryFinally: + def __init__(self, x: List[int]) -> None: + self.a = 0 + try: + self.b = x[0] + finally: + self.c = x + self.d = 0 + try: + self.e = x[0] + finally: + self.e = 1 + +class Assert: + def __init__(self, x: Optional[str], y: int) -> None: + assert x is not None + assert y < 5 + self.a = x + +class For: + def __init__(self, it: Iterable[int]) -> None: + self.x = 0 + for x in it: + self.x += x + for x in it: + self.y = x + +class Assignment1: + def __init__(self, other: Assignment1) -> None: + self.x = 0 + self = other # Give up after assignment to self + self.y = 1 + +class Assignment2: + def __init__(self) -> None: + self.x = 0 + other = self # Give up after self is aliased + self.y = other.x + +class With: + def __init__(self, x: Any) -> None: + self.a = 0 + with x: + self.b = 1 + self.c = 2 + +def f() -> None: + pass + +[out] +Return: [x, y] +While: [a] +-- We could infer 'e' as always defined, but this is tricky, since always defined attribute +-- analysis must be performed earlier than exception handling transform. This would be +-- easy to infer *after* exception handling transform. +Try: [a, d] +-- Again, 'e' could be always defined, but it would be a bit tricky to do it. +TryFinally: [a, c, d] +Assert: [a] +For: [x] +Assignment1: [x] +Assignment2: [x] +-- TODO: Why is not 'b' included? +With: [a, c] + +[case testAlwaysDefinedAttributeDefaults] +class Basic: + x = 0 + +class ClassBodyAndInit: + x = 0 + s = 'x' + + def __init__(self, n: int) -> None: + self.n = 0 + +class AttrWithDefaultAndInit: + x = 0 + + def __init__(self, x: int) -> None: + self.x = x + +class Base: + x = 0 + y = 1 + +class Derived(Base): + y = 2 + z = 3 +[out] +Basic: [x] +ClassBodyAndInit: [n, s, x] +AttrWithDefaultAndInit: [x] +Base: [x, y] +Derived: [x, y, z] + +[case testAlwaysDefinedWithInheritance] +class Base: + def __init__(self, x: int) -> None: + self.x = x + +class Deriv1(Base): + def __init__(self, x: int, y: str) -> None: + super().__init__(x) + self.y = y + +class Deriv2(Base): + def __init__(self, x: int, y: str) -> None: + self.y = y + super().__init__(x) + +class Deriv22(Deriv2): + def __init__(self, x: int, y: str, z: bool) -> None: + super().__init__(x, y) + self.z = False + +class Deriv3(Base): + def __init__(self) -> None: + super().__init__(1) + +class Deriv4(Base): + def __init__(self) -> None: + self.y = 1 + self.x = 2 + +def f(a): pass + +class BaseUnsafe: + def __init__(self, x: int, y: int) -> None: + self.x = x + f(self) # Unknown function + self.y = y + +class DerivUnsafe(BaseUnsafe): + def __init__(self, z: int, zz: int) -> None: + self.z = z + super().__init__(1, 2) # Calls unknown function + self.zz = zz + +class BaseWithDefault: + x = 1 + + def __init__(self) -> None: + self.y = 1 + +class DerivedWithDefault(BaseWithDefault): + def __init__(self) -> None: + super().__init__() + self.z = 1 + +class AlwaysDefinedInBase: + def __init__(self) -> None: + self.x = 1 + self.y = 1 + +class UndefinedInDerived(AlwaysDefinedInBase): + def __init__(self, x: bool) -> None: + self.x = 1 + if x: + self.y = 2 + +class UndefinedInDerived2(UndefinedInDerived): + def __init__(self, x: bool): + if x: + self.y = 2 +[out] +Base: [x] +Deriv1: [x, y] +Deriv2: [x, y] +Deriv22: [x, y, z] +Deriv3: [x] +Deriv4: [x, y] +BaseUnsafe: [x] +DerivUnsafe: [x, z] +BaseWithDefault: [x, y] +DerivedWithDefault: [x, y, z] +AlwaysDefinedInBase: [] +UndefinedInDerived: [] +UndefinedInDerived2: [] + +[case testAlwaysDefinedWithInheritance2] +from mypy_extensions import trait, mypyc_attr + +from interpreted import PythonBase + +class BasePartiallyDefined: + def __init__(self, x: int) -> None: + self.a = 0 + if x: + self.x = x + +class Derived1(BasePartiallyDefined): + def __init__(self, x: int) -> None: + super().__init__(x) + self.y = x + +class BaseUndefined: + x: int + +class DerivedAlwaysDefined(BaseUndefined): + def __init__(self) -> None: + super().__init__() + self.z = 0 + self.x = 2 + +@trait +class MyTrait: + def f(self) -> None: pass + +class SimpleTraitImpl(MyTrait): + def __init__(self) -> None: + super().__init__() + self.x = 0 + +@trait +class TraitWithAttr: + x: int + y: str + +class TraitWithAttrImpl(TraitWithAttr): + def __init__(self) -> None: + self.y = 'x' + +@trait +class TraitWithAttr2: + z: int + +class TraitWithAttrImpl2(TraitWithAttr, TraitWithAttr2): + def __init__(self) -> None: + self.y = 'x' + self.z = 2 + +@mypyc_attr(allow_interpreted_subclasses=True) +class BaseWithGeneralSubclassing: + x = 0 + y: int + def __init__(self, s: str) -> None: + self.s = s + +class Derived2(BaseWithGeneralSubclassing): + def __init__(self) -> None: + super().__init__('x') + self.z = 0 + +class SubclassPythonclass(PythonBase): + def __init__(self) -> None: + self.y = 1 + +class BaseWithSometimesDefined: + def __init__(self, b: bool) -> None: + if b: + self.x = 0 + +class Derived3(BaseWithSometimesDefined): + def __init__(self, b: bool) -> None: + super().__init__(b) + self.x = 1 + +[file interpreted.py] +class PythonBase: + def __init__(self) -> None: + self.x = 0 + +[out] +BasePartiallyDefined: [a] +Derived1: [a, y] +BaseUndefined: [] +DerivedAlwaysDefined: [x, z] +MyTrait: [] +SimpleTraitImpl: [x] +TraitWithAttr: [] +TraitWithAttrImpl: [y] +TraitWithAttr2: [] +TraitWithAttrImpl2: [y, z] +BaseWithGeneralSubclassing: [] +-- TODO: 's' could also be always defined +Derived2: [x, z] +-- Always defined attribute analysis is turned off when inheriting a non-native class. +SubclassPythonclass: [] +BaseWithSometimesDefined: [] +-- TODO: 'x' could also be always defined, but it is a bit tricky to support +Derived3: [] + +[case testAlwaysDefinedWithNesting] +class NestedFunc: + def __init__(self) -> None: + self.x = 0 + def f() -> None: + self.y = 0 + f() + self.z = 1 +[out] +-- TODO: Support nested functions. +NestedFunc: [] +f___init___NestedFunc_obj: [] diff --git a/mypyc/test-data/analysis.test b/mypyc/test-data/analysis.test new file mode 100644 index 000000000000..35677b8ea56d --- /dev/null +++ b/mypyc/test-data/analysis.test @@ -0,0 +1,470 @@ +-- Test cases for data flow analysis. + +[case testSimple_MaybeDefined] +def f(a: int) -> None: + x = 1 + if x == a: + y = 1 + else: + z = 1 +[out] +def f(a): + a, x :: int + r0 :: bit + y, z :: int +L0: + x = 2 + r0 = int_eq x, a + if r0 goto L1 else goto L2 :: bool +L1: + y = 2 + goto L3 +L2: + z = 2 +L3: + return 1 +(0, 0) {a} {a, x} +(0, 1) {a, x} {a, x} +(0, 2) {a, x} {a, x} +(1, 0) {a, x} {a, x, y} +(1, 1) {a, x, y} {a, x, y} +(2, 0) {a, x} {a, x, z} +(2, 1) {a, x, z} {a, x, z} +(3, 0) {a, x, y, z} {a, x, y, z} + +[case testSimple_Liveness] +def f(a: int) -> int: + x = 1 + if x == 1: + return a + else: + return x +[out] +def f(a): + a, x :: int + r0 :: bit +L0: + x = 2 + r0 = int_eq x, 2 + if r0 goto L1 else goto L2 :: bool +L1: + return a +L2: + return x +L3: + unreachable +(0, 0) {a} {a, x} +(0, 1) {a, x} {a, r0, x} +(0, 2) {a, r0, x} {a, x} +(1, 0) {a} {} +(2, 0) {x} {} +(3, 0) {} {} + +[case testSpecial_Liveness] +def f() -> int: + x = 1 + y = 1 + x = 2 + return x +[out] +def f(): + x, y :: int +L0: + x = 2 + y = 2 + x = 4 + return x +(0, 0) {} {} +(0, 1) {} {} +(0, 2) {} {x} +(0, 3) {x} {} + +[case testSpecial2_Liveness] +def f(a: int) -> int: + a = 1 + a = 2 + a = 3 + return a +[out] +def f(a): + a :: int +L0: + a = 2 + a = 4 + a = 6 + return a +(0, 0) {} {} +(0, 1) {} {} +(0, 2) {} {a} +(0, 3) {a} {} + +[case testSimple_MustDefined] +def f(a: int) -> None: + if a == 1: + y = 1 + x = 2 + else: + x = 2 +[out] +def f(a): + a :: int + r0 :: bit + y, x :: int +L0: + r0 = int_eq a, 2 + if r0 goto L1 else goto L2 :: bool +L1: + y = 2 + x = 4 + goto L3 +L2: + x = 4 +L3: + return 1 +(0, 0) {a} {a} +(0, 1) {a} {a} +(1, 0) {a} {a, y} +(1, 1) {a, y} {a, x, y} +(1, 2) {a, x, y} {a, x, y} +(2, 0) {a} {a, x} +(2, 1) {a, x} {a, x} +(3, 0) {a, x} {a, x} + +[case testTwoArgs_MustDefined] +def f(x: int, y: int) -> int: + return x +[out] +def f(x, y): + x, y :: int +L0: + return x +(0, 0) {x, y} {x, y} + +[case testLoop_MustDefined] +def f(n: int) -> None: + while n < 5: + n = n + 1 + m = n +[out] +def f(n): + n :: int + r0 :: bit + r1, m :: int +L0: +L1: + r0 = int_lt n, 10 + if r0 goto L2 else goto L3 :: bool +L2: + r1 = CPyTagged_Add(n, 2) + n = r1 + m = n + goto L1 +L3: + return 1 +(0, 0) {n} {n} +(1, 0) {n} {n} +(1, 1) {n} {n} +(2, 0) {n} {n} +(2, 1) {n} {n} +(2, 2) {n} {m, n} +(2, 3) {m, n} {m, n} +(3, 0) {n} {n} + +[case testMultiPass_Liveness] +def f(n: int) -> None: + x = 1 + y = 1 + while n < 1: + n = y + while n < 2: + n = 1 + n = x +[out] +def f(n): + n, x, y :: int + r0, r1 :: bit +L0: + x = 2 + y = 2 +L1: + r0 = int_lt n, 2 + if r0 goto L2 else goto L6 :: bool +L2: + n = y +L3: + r1 = int_lt n, 4 + if r1 goto L4 else goto L5 :: bool +L4: + n = 2 + n = x + goto L3 +L5: + goto L1 +L6: + return 1 +(0, 0) {n} {n, x} +(0, 1) {n, x} {n, x, y} +(0, 2) {n, x, y} {n, x, y} +(1, 0) {n, x, y} {r0, x, y} +(1, 1) {r0, x, y} {x, y} +(2, 0) {x, y} {n, x, y} +(2, 1) {n, x, y} {n, x, y} +(3, 0) {n, x, y} {n, r1, x, y} +(3, 1) {n, r1, x, y} {n, x, y} +(4, 0) {x, y} {x, y} +(4, 1) {x, y} {n, x, y} +(4, 2) {n, x, y} {n, x, y} +(5, 0) {n, x, y} {n, x, y} +(6, 0) {} {} + +[case testCall_Liveness] +def f(x: int) -> int: + a = f(1) + return f(a) + a +[out] +def f(x): + x, r0, a, r1, r2, r3 :: int +L0: + r0 = f(2) + if is_error(r0) goto L3 (error at f:2) else goto L1 +L1: + a = r0 + r1 = f(a) + if is_error(r1) goto L3 (error at f:3) else goto L2 +L2: + r2 = CPyTagged_Add(r1, a) + return r2 +L3: + r3 = :: int + return r3 +(0, 0) {} {r0} +(0, 1) {r0} {r0} +(1, 0) {r0} {a} +(1, 1) {a} {a, r1} +(1, 2) {a, r1} {a, r1} +(2, 0) {a, r1} {r2} +(2, 1) {r2} {} +(3, 0) {} {r3} +(3, 1) {r3} {} + +[case testLoop_MaybeDefined] +def f(a: int) -> None: + while a < a: + while a < a: + y = a + x = a +[out] +def f(a): + a :: int + r0, r1 :: bit + y, x :: int +L0: +L1: + r0 = int_lt a, a + if r0 goto L2 else goto L6 :: bool +L2: +L3: + r1 = int_lt a, a + if r1 goto L4 else goto L5 :: bool +L4: + y = a + goto L3 +L5: + x = a + goto L1 +L6: + return 1 +(0, 0) {a} {a} +(1, 0) {a, x, y} {a, x, y} +(1, 1) {a, x, y} {a, x, y} +(2, 0) {a, x, y} {a, x, y} +(3, 0) {a, x, y} {a, x, y} +(3, 1) {a, x, y} {a, x, y} +(4, 0) {a, x, y} {a, x, y} +(4, 1) {a, x, y} {a, x, y} +(5, 0) {a, x, y} {a, x, y} +(5, 1) {a, x, y} {a, x, y} +(6, 0) {a, x, y} {a, x, y} + +[case testTrivial_BorrowedArgument] +def f(a: int, b: int) -> int: + return b +[out] +def f(a, b): + a, b :: int +L0: + return b +(0, 0) {a, b} {a, b} + +[case testSimple_BorrowedArgument] +def f(a: int) -> int: + b = a + a = 1 + return a +[out] +def f(a): + a, b :: int +L0: + b = a + a = 2 + return a +(0, 0) {a} {a} +(0, 1) {a} {} +(0, 2) {} {} + +[case testConditional_BorrowedArgument] +def f(a: int) -> int: + if a == a: + x = 2 + a = 1 + else: + x = 1 + return x +[out] +def f(a): + a :: int + r0 :: bit + x :: int +L0: + r0 = int_eq a, a + if r0 goto L1 else goto L2 :: bool +L1: + x = 4 + a = 2 + goto L3 +L2: + x = 2 +L3: + return x +(0, 0) {a} {a} +(0, 1) {a} {a} +(1, 0) {a} {a} +(1, 1) {a} {} +(1, 2) {} {} +(2, 0) {a} {a} +(2, 1) {a} {a} +(3, 0) {} {} + +[case testLoop_BorrowedArgument] +def f(a: int) -> int: + sum = 0 + i = 0 + while i <= a: + sum = sum + i + i = i + 1 + return sum +[out] +def f(a): + a, sum, i :: int + r0 :: bit + r1, r2 :: int +L0: + sum = 0 + i = 0 +L1: + r0 = int_le i, a + if r0 goto L2 else goto L3 :: bool +L2: + r1 = CPyTagged_Add(sum, i) + sum = r1 + r2 = CPyTagged_Add(i, 2) + i = r2 + goto L1 +L3: + return sum +(0, 0) {a} {a} +(0, 1) {a} {a} +(0, 2) {a} {a} +(1, 0) {a} {a} +(1, 1) {a} {a} +(2, 0) {a} {a} +(2, 1) {a} {a} +(2, 2) {a} {a} +(2, 3) {a} {a} +(2, 4) {a} {a} +(3, 0) {a} {a} + +[case testError] +def f(x: List[int]) -> None: pass # E: Name "List" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import List") + +[case testExceptUndefined_Liveness] +def lol(x: object) -> int: + try: + st = id(x) + except Exception: + return -1 + return st + 1 +[out] +def lol(x): + x :: object + r0, st :: int + r1 :: tuple[object, object, object] + r2 :: object + r3 :: str + r4 :: object + r5, r6 :: bit + r7, r8 :: int +L0: +L1: + r0 = CPyTagged_Id(x) + st = r0 + goto L10 +L2: + r1 = CPy_CatchError() + r2 = builtins :: module + r3 = 'Exception' + r4 = CPyObject_GetAttr(r2, r3) + if is_error(r4) goto L8 (error at lol:4) else goto L3 +L3: + r5 = CPy_ExceptionMatches(r4) + if r5 goto L4 else goto L5 :: bool +L4: + CPy_RestoreExcInfo(r1) + return -2 +L5: + CPy_Reraise() + if not 0 goto L8 else goto L6 :: bool +L6: + unreachable +L7: + CPy_RestoreExcInfo(r1) + goto L10 +L8: + CPy_RestoreExcInfo(r1) + r6 = CPy_KeepPropagating() + if not r6 goto L11 else goto L9 :: bool +L9: + unreachable +L10: + r7 = CPyTagged_Add(st, 2) + return r7 +L11: + r8 = :: int + return r8 +(0, 0) {x} {x} +(1, 0) {x} {r0} +(1, 1) {r0} {st} +(1, 2) {st} {st} +(2, 0) {} {r1} +(2, 1) {r1} {r1, r2} +(2, 2) {r1, r2} {r1, r2, r3} +(2, 3) {r1, r2, r3} {r1, r4} +(2, 4) {r1, r4} {r1, r4} +(3, 0) {r1, r4} {r1, r5} +(3, 1) {r1, r5} {r1} +(4, 0) {r1} {} +(4, 1) {} {} +(5, 0) {r1} {r1} +(5, 1) {r1} {r1} +(6, 0) {} {} +(7, 0) {r1, st} {st} +(7, 1) {st} {st} +(8, 0) {r1} {} +(8, 1) {} {r6} +(8, 2) {r6} {} +(9, 0) {} {} +(10, 0) {st} {r7} +(10, 1) {r7} {} +(11, 0) {} {r8} +(11, 1) {r8} {} diff --git a/mypyc/test-data/commandline.test b/mypyc/test-data/commandline.test new file mode 100644 index 000000000000..672e879fbe1e --- /dev/null +++ b/mypyc/test-data/commandline.test @@ -0,0 +1,245 @@ +-- Test cases for invoking mypyc on the command line +-- +-- These are slow -- do not add test cases unless you have a very good reason to do so. + +[case testCompileMypyc] +# cmd: a.py b.py p/__init__.py p/q.py +import os.path +import p +import p.q +import a +import b +print('
', b.g(a.A())) +try: + a.f('') +except TypeError: + pass +else: + assert False +for x in [a, b, p, p.q]: + assert os.path.splitext(x.__file__)[1] != '.py' +[file z.py] + +[file a.py] +import b +import c +from p import s +from typing import NamedTuple + +print('', ord('A') == 65) # Test full builtins + +class A: + def __init__(self) -> None: + self.x = 4 + +def f(x: int) -> b.B: + return b.B(x) + +class B: + def __init__(self, x: int, y: str) -> None: + self.x = x + +print('', f(5).x) +print('', c.foo()) +assert s.bar(10) == 20 + +class NT(NamedTuple): + x: int + +print(NT(2)) + +[file b.py] +import a +import p.q + +class B: + def __init__(self, x: int) -> None: + self.x = x + +def g(z: 'a.A') -> int: + return p.q.foo(z.x) + +print('', 'here') + +[file c.py] +def foo() -> int: + return 10 + +[file p/__init__.py] + +[file p/q.py] +import p.r +def foo(x: int) -> int: + return x*p.r.foo(x) + +[file p/r.py] +def foo(x: int) -> int: + return x + +[file p/s.py] +def bar(x: int) -> int: + return x*2 + +[out] + here + True + 5 + 10 +NT(x=2) +
16 + +-- This test is here so we can turn it on when we get nervous about +-- this case, but is disabled for speed reasons. +[case testCompileMypycOne-skip] +# cmd: a.py +import os.path +import a +assert os.path.splitext(a.__file__)[1] != '.py' +assert a.f(10) == 100 + +[file a.py] +def f(x: int) -> int: + return x*x + +[case testErrorOutput] +# cmd: test.py + +[file test.py] +from typing import List, Any, AsyncIterable +from typing_extensions import Final +from mypy_extensions import trait, mypyc_attr +from functools import singledispatch + +def busted(b: bool) -> None: + for i in range(1, 10, 0): # E: range() step can't be zero + try: + if i == 5: + break # E: break inside try/finally block is unimplemented + elif i == 4: + continue # E: continue inside try/finally block is unimplemented + finally: + print('oops') + +print(sum([1,2,3])) + +x = [1,2] + +class Foo: + a, b = (10, 20) # E: Only assignment to variables is supported in class bodies + x[0] = 10 # E: Only assignment to variables is supported in class bodies + lol = 20 + l = [10] # W: Unsupported default attribute value + c = d = 50 # E: Multiple assignment in class bodies not supported + + if 1+1 == 2: # E: Unsupported statement in class body + x = 10 + +Foo.lol = 50 # E: Only class variables defined as ClassVar can be assigned to + +def decorator(x: Any) -> Any: + return x + +class NeverMetaclass(type): # E: Inheriting from most builtin types is unimplemented + pass + +class Concrete1: + pass + +@trait +class PureTrait: + pass + +@trait +class Trait1: + pass + +class Concrete2: + pass + +@trait +class Trait2(Concrete2): + pass + +@decorator +class NonExt(Concrete1): # E: Non-extension classes may not inherit from extension classes + pass + + +class NopeMultipleInheritance(Concrete1, Concrete2): # E: Multiple inheritance is not supported (except for traits) + pass + +class NopeMultipleInheritanceAndBadOrder(Concrete1, Trait1, Concrete2): # E: Multiple inheritance is not supported (except for traits) + pass + +class NopeMultipleInheritanceAndBadOrder2(Concrete1, Concrete2, Trait1): # E: Multiple inheritance is not supported (except for traits) + pass + +class NopeMultipleInheritanceAndBadOrder3(Trait1, Concrete1, Concrete2): # E: Non-trait base must appear first in parent list # E: Multiple inheritance is not supported (except for traits) + pass + +class NopeBadOrder(Trait1, Concrete2): # E: Non-trait base must appear first in parent list + pass + + +@decorator +class NonExt2: + @property # E: Property setters not supported in non-extension classes + def test(self) -> int: + return 0 + + @test.setter + def test(self, x: int) -> None: + pass + +iterator_warning = (i+1 for i in range(10)) # W: Treating generator comprehension as list + +# But we don't want warnings for these cases: +tup = tuple(i+1 for i in range(10)) +a_str = " ".join(str(i) for i in range(10)) +wtvr = next(i for i in range(10) if i == 5) + +d1 = {1: 2} + +# Make sure we can produce an error when we hit the awful None case +def f(l: List[object]) -> None: + x = None # E: Local variable "x" has inferred type None; add an annotation + for i in l: + if x is None: + x = i + +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowInterp1(Concrete1): # E: Base class "test.Concrete1" does not allow interpreted subclasses + pass + +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowInterp2(PureTrait): # E: Base class "test.PureTrait" does not allow interpreted subclasses + pass + +async def async_generators() -> AsyncIterable[int]: + yield 1 # E: async generators are unimplemented + +@singledispatch +def a(arg) -> None: + pass + +@decorator # E: Calling decorator after registering function not supported +@a.register +def g(arg: int) -> None: + pass + +@a.register +@decorator +def h(arg: str) -> None: + pass + +@decorator +@decorator # E: Calling decorator after registering function not supported +@a.register +def i(arg: Foo) -> None: + pass + +[case testOnlyWarningOutput] +# cmd: test.py + +[file test.py] +names = (str(v) for v in [1, 2, 3]) # W: Treating generator comprehension as list diff --git a/mypyc/test-data/driver/driver.py b/mypyc/test-data/driver/driver.py new file mode 100644 index 000000000000..c9d179224a30 --- /dev/null +++ b/mypyc/test-data/driver/driver.py @@ -0,0 +1,48 @@ +"""Default driver for run tests (run-*.test). + +This imports the 'native' module (containing the compiled test cases) +and calls each function starting with test_, and reports any +exceptions as failures. + +Test cases can provide a custom driver.py that overrides this file. +""" + +import sys +import native + +failures = [] + +for name in dir(native): + if name.startswith('test_'): + test_func = getattr(native, name) + try: + test_func() + except Exception as e: + failures.append((name, sys.exc_info())) + +if failures: + from traceback import print_exception, format_tb + import re + + def extract_line(tb): + formatted = '\n'.join(format_tb(tb)) + m = re.search('File "(native|driver).py", line ([0-9]+), in (test_|)', formatted) + if m is None: + return "0" + return m.group(1) + + # Sort failures by line number of test function. + failures = sorted(failures, key=lambda e: extract_line(e[1][2])) + + # If there are multiple failures, print stack traces of all but the final failure. + for name, e in failures[:-1]: + print(f'<< {name} >>') + sys.stdout.flush() + print_exception(*e) + print() + sys.stdout.flush() + + # Raise exception for the last failure. Test runner will show the traceback. + print(f'<< {failures[-1][0]} >>') + sys.stdout.flush() + raise failures[-1][1][1] diff --git a/mypyc/test-data/exceptions-freq.test b/mypyc/test-data/exceptions-freq.test new file mode 100644 index 000000000000..a655eed44d90 --- /dev/null +++ b/mypyc/test-data/exceptions-freq.test @@ -0,0 +1,125 @@ +-- Test cases for basic block execution frequency analysis. +-- +-- These test cases are using exception transform test machinery for convenience. +-- +-- NOTE: These must all have the _freq suffix + +[case testSimpleError_freq] +from typing import List +def f(x: List[int]) -> int: + return x[0] +[out] +def f(x): + x :: list + r0 :: object + r1, r2 :: int +L0: + r0 = CPyList_GetItemShort(x, 0) + if is_error(r0) goto L3 (error at f:3) else goto L1 +L1: + r1 = unbox(int, r0) + dec_ref r0 + if is_error(r1) goto L3 (error at f:3) else goto L2 +L2: + return r1 +L3: + r2 = :: int + return r2 +hot blocks: [0, 1, 2] + +[case testHotBranch_freq] +from typing import List +def f(x: bool) -> None: + if x: + y = 1 + else: + y = 2 +[out] +def f(x): + x :: bool + y :: int +L0: + if x goto L1 else goto L2 :: bool +L1: + y = 2 + dec_ref y :: int + goto L3 +L2: + y = 4 + dec_ref y :: int +L3: + return 1 +hot blocks: [0, 1, 2, 3] + +[case testGoto_freq] +from typing import List +def f(x: bool) -> int: + if x: + y = 1 + else: + return 2 + return y +[out] +def f(x): + x :: bool + y :: int +L0: + if x goto L1 else goto L2 :: bool +L1: + y = 2 + goto L3 +L2: + return 4 +L3: + return y +hot blocks: [0, 1, 2, 3] + +[case testFalseOnError_freq] +from typing import List +def f(x: List[int]) -> None: + x[0] = 1 +[out] +def f(x): + x :: list + r0 :: object + r1 :: bit + r2 :: None +L0: + r0 = object 1 + inc_ref r0 + r1 = CPyList_SetItem(x, 0, r0) + if not r1 goto L2 (error at f:3) else goto L1 :: bool +L1: + return 1 +L2: + r2 = :: None + return r2 +hot blocks: [0, 1] + +[case testRareBranch_freq] +from typing_extensions import Final + +x: Final = str() + +def f() -> str: + return x +[out] +def f(): + r0 :: str + r1 :: bool + r2 :: str +L0: + r0 = __main__.x :: static + if is_error(r0) goto L1 else goto L3 +L1: + r1 = raise NameError('value for final name "x" was not set') + if not r1 goto L4 (error at f:6) else goto L2 :: bool +L2: + unreachable +L3: + inc_ref r0 + return r0 +L4: + r2 = :: str + return r2 +hot blocks: [0, 3] diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test new file mode 100644 index 000000000000..1ec03dd9a671 --- /dev/null +++ b/mypyc/test-data/exceptions.test @@ -0,0 +1,699 @@ +-- Test cases for exception handling insertion transform. +-- +-- The result includes refcount handling since these two transforms interact. + +[case testListGetAndUnboxError] +from typing import List +def f(x: List[int]) -> int: + return x[0] +[out] +def f(x): + x :: list + r0 :: object + r1, r2 :: int +L0: + r0 = CPyList_GetItemShort(x, 0) + if is_error(r0) goto L3 (error at f:3) else goto L1 +L1: + r1 = unbox(int, r0) + dec_ref r0 + if is_error(r1) goto L3 (error at f:3) else goto L2 +L2: + return r1 +L3: + r2 = :: int + return r2 + +[case testListAppendAndSetItemError] +from typing import List +def f(x: List[int], y: int, z: int) -> None: + x.append(y) + x[y] = z +[out] +def f(x, y, z): + x :: list + y, z :: int + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: object + r4 :: bit + r5 :: None +L0: + inc_ref y :: int + r0 = box(int, y) + r1 = PyList_Append(x, r0) + dec_ref r0 + r2 = r1 >= 0 :: signed + if not r2 goto L3 (error at f:3) else goto L1 :: bool +L1: + inc_ref z :: int + r3 = box(int, z) + r4 = CPyList_SetItem(x, y, r3) + if not r4 goto L3 (error at f:4) else goto L2 :: bool +L2: + return 1 +L3: + r5 = :: None + return r5 + +[case testOptionalHandling] +from typing import Optional + +class A: pass + +def f(x: Optional[A]) -> int: + if x is None: + return 1 + if x is not None: + return 2 + return 3 +[out] +def f(x): + x :: union[__main__.A, None] + r0 :: object + r1 :: bit + r2 :: __main__.A + r3 :: object + r4 :: bit + r5 :: int +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + r2 = borrow cast(__main__.A, x) + if is_error(r2) goto L6 (error at f:8) else goto L3 +L3: + r3 = load_address _Py_NoneStruct + r4 = r2 != r3 + if r4 goto L4 else goto L5 :: bool +L4: + return 4 +L5: + return 6 +L6: + r5 = :: int + return r5 + +[case testListSum] +from typing import List +def sum(a: List[int], l: int) -> int: + sum = 0 + i = 0 + while i < l: + sum = sum + a[i] + i = i + 1 + return sum +[out] +def sum(a, l): + a :: list + l, sum, i :: int + r0 :: bit + r1 :: object + r2, r3, r4, r5 :: int +L0: + sum = 0 + i = 0 +L1: + r0 = int_lt i, l + if r0 goto L2 else goto L7 :: bool +L2: + r1 = CPyList_GetItemBorrow(a, i) + if is_error(r1) goto L8 (error at sum:6) else goto L3 +L3: + r2 = unbox(int, r1) + if is_error(r2) goto L8 (error at sum:6) else goto L4 +L4: + r3 = CPyTagged_Add(sum, r2) + dec_ref sum :: int + dec_ref r2 :: int + sum = r3 + r4 = CPyTagged_Add(i, 2) + dec_ref i :: int + i = r4 + goto L1 +L5: + return sum +L6: + r5 = :: int + return r5 +L7: + dec_ref i :: int + goto L5 +L8: + dec_ref sum :: int + dec_ref i :: int + goto L6 + +[case testTryExcept] +def g() -> None: + try: + object() + except: + print("weeee") +[out] +def g(): + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: tuple[object, object, object] + r5 :: str + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: bit + r11 :: None +L0: +L1: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + if is_error(r2) goto L3 (error at g:3) else goto L2 +L2: + r3 = PyObject_CallFunctionObjArgs(r2, 0) + dec_ref r2 + if is_error(r3) goto L3 (error at g:3) else goto L10 +L3: + r4 = CPy_CatchError() + r5 = 'weeee' + r6 = builtins :: module + r7 = 'print' + r8 = CPyObject_GetAttr(r6, r7) + if is_error(r8) goto L6 (error at g:5) else goto L4 +L4: + r9 = PyObject_CallFunctionObjArgs(r8, r5, 0) + dec_ref r8 + if is_error(r9) goto L6 (error at g:5) else goto L11 +L5: + CPy_RestoreExcInfo(r4) + dec_ref r4 + goto L8 +L6: + CPy_RestoreExcInfo(r4) + dec_ref r4 + r10 = CPy_KeepPropagating() + if not r10 goto L9 else goto L7 :: bool +L7: + unreachable +L8: + return 1 +L9: + r11 = :: None + return r11 +L10: + dec_ref r3 + goto L8 +L11: + dec_ref r9 + goto L5 + +[case testGenopsTryFinally] +def a() -> str: + try: + print() + return 'hi' + finally: + print('goodbye!') +[out] +def a(): + r0 :: object + r1 :: str + r2, r3 :: object + r4, r5 :: str + r6, r7 :: tuple[object, object, object] + r8 :: str + r9 :: tuple[object, object, object] + r10 :: str + r11 :: object + r12 :: str + r13, r14 :: object + r15 :: bit + r16 :: str +L0: +L1: + r0 = builtins :: module + r1 = 'print' + r2 = CPyObject_GetAttr(r0, r1) + if is_error(r2) goto L5 (error at a:3) else goto L2 +L2: + r3 = PyObject_CallFunctionObjArgs(r2, 0) + dec_ref r2 + if is_error(r3) goto L5 (error at a:3) else goto L19 +L3: + r4 = 'hi' + inc_ref r4 + r5 = r4 +L4: + r6 = :: tuple[object, object, object] + r7 = r6 + goto L6 +L5: + r8 = :: str + r5 = r8 + r9 = CPy_CatchError() + r7 = r9 +L6: + r10 = 'goodbye!' + r11 = builtins :: module + r12 = 'print' + r13 = CPyObject_GetAttr(r11, r12) + if is_error(r13) goto L20 (error at a:6) else goto L7 +L7: + r14 = PyObject_CallFunctionObjArgs(r13, r10, 0) + dec_ref r13 + if is_error(r14) goto L20 (error at a:6) else goto L21 +L8: + if is_error(r7) goto L11 else goto L22 +L9: + CPy_Reraise() + if not 0 goto L13 else goto L23 :: bool +L10: + unreachable +L11: + if is_error(r5) goto L17 else goto L12 +L12: + return r5 +L13: + if is_error(r7) goto L15 else goto L14 +L14: + CPy_RestoreExcInfo(r7) + xdec_ref r7 +L15: + r15 = CPy_KeepPropagating() + if not r15 goto L18 else goto L16 :: bool +L16: + unreachable +L17: + unreachable +L18: + r16 = :: str + return r16 +L19: + dec_ref r3 + goto L3 +L20: + xdec_ref r5 + goto L13 +L21: + dec_ref r14 + goto L8 +L22: + xdec_ref r5 + goto L9 +L23: + xdec_ref r7 + goto L10 + +[case testDocstring1] +def lol() -> None: + """Hello""" + pass +[out] +def lol(): +L0: + return 1 + +[case testExceptUndefined1] +from typing import Any +def lol(x: Any) -> object: + try: + st = x.foo + except: + return '' + # No uninit check should be generated, since the exception branch always returns + return st +[out] +def lol(x): + x :: object + r0 :: str + r1, st :: object + r2 :: tuple[object, object, object] + r3 :: str +L0: +L1: + r0 = 'foo' + r1 = CPyObject_GetAttr(x, r0) + if is_error(r1) goto L3 (error at lol:4) else goto L2 +L2: + st = r1 + goto L4 +L3: + r2 = CPy_CatchError() + r3 = '' + CPy_RestoreExcInfo(r2) + dec_ref r2 + inc_ref r3 + return r3 +L4: + return st + +[case testExceptUndefined2] +from typing import Any +def lol(x: Any) -> object: + try: + a = x.foo + b = x.bar + except: + pass + # uninit checks are needed, since the exception can skip initializing the vars + return a + b +[out] +def lol(x): + x, r0, a, r1, b :: object + r2 :: str + r3 :: object + r4 :: str + r5 :: object + r6 :: tuple[object, object, object] + r7, r8 :: bool + r9, r10 :: object +L0: + r0 = :: object + a = r0 + r1 = :: object + b = r1 +L1: + r2 = 'foo' + r3 = CPyObject_GetAttr(x, r2) + if is_error(r3) goto L4 (error at lol:4) else goto L15 +L2: + a = r3 + r4 = 'bar' + r5 = CPyObject_GetAttr(x, r4) + if is_error(r5) goto L4 (error at lol:5) else goto L16 +L3: + b = r5 + goto L6 +L4: + r6 = CPy_CatchError() +L5: + CPy_RestoreExcInfo(r6) + dec_ref r6 +L6: + if is_error(a) goto L17 else goto L9 +L7: + r7 = raise UnboundLocalError('local variable "a" referenced before assignment') + if not r7 goto L14 (error at lol:9) else goto L8 :: bool +L8: + unreachable +L9: + if is_error(b) goto L18 else goto L12 +L10: + r8 = raise UnboundLocalError('local variable "b" referenced before assignment') + if not r8 goto L14 (error at lol:9) else goto L11 :: bool +L11: + unreachable +L12: + r9 = PyNumber_Add(a, b) + xdec_ref a + xdec_ref b + if is_error(r9) goto L14 (error at lol:9) else goto L13 +L13: + return r9 +L14: + r10 = :: object + return r10 +L15: + xdec_ref a + goto L2 +L16: + xdec_ref b + goto L3 +L17: + xdec_ref b + goto L7 +L18: + xdec_ref a + goto L10 + +[case testMaybeUninitVarExc] +def f(b: bool) -> None: + u = 'a' + while b: + v = 'b' + if v is not u: + break + print(v) +[out] +def f(b): + b :: bool + r0, v, r1, u, r2 :: str + r3, r4 :: bit + r5 :: object + r6 :: str + r7 :: object + r8 :: bool + r9 :: object + r10 :: None +L0: + r0 = :: str + v = r0 + r1 = 'a' + inc_ref r1 + u = r1 +L1: + if b goto L10 else goto L11 :: bool +L2: + r2 = 'b' + inc_ref r2 + v = r2 + r3 = v == u + r4 = r3 ^ 1 + if r4 goto L11 else goto L1 :: bool +L3: + r5 = builtins :: module + r6 = 'print' + r7 = CPyObject_GetAttr(r5, r6) + if is_error(r7) goto L12 (error at f:7) else goto L4 +L4: + if is_error(v) goto L13 else goto L7 +L5: + r8 = raise UnboundLocalError('local variable "v" referenced before assignment') + if not r8 goto L9 (error at f:7) else goto L6 :: bool +L6: + unreachable +L7: + r9 = PyObject_CallFunctionObjArgs(r7, v, 0) + dec_ref r7 + xdec_ref v + if is_error(r9) goto L9 (error at f:7) else goto L14 +L8: + return 1 +L9: + r10 = :: None + return r10 +L10: + xdec_ref v + goto L2 +L11: + dec_ref u + goto L3 +L12: + xdec_ref v + goto L9 +L13: + dec_ref r7 + goto L5 +L14: + dec_ref r9 + goto L8 + +[case testExceptionWithOverlappingErrorValue] +from mypy_extensions import i64 + +def f() -> i64: + return 0 + +def g() -> i64: + return f() +[out] +def f(): +L0: + return 0 +def g(): + r0 :: i64 + r1 :: bit + r2 :: object + r3 :: i64 +L0: + r0 = f() + r1 = r0 == -113 + if r1 goto L2 else goto L1 :: bool +L1: + return r0 +L2: + r2 = PyErr_Occurred() + if not is_error(r2) goto L3 (error at g:7) else goto L1 +L3: + r3 = :: i64 + return r3 + +[case testExceptionWithNativeAttributeGetAndSet] +class C: + def __init__(self, x: int) -> None: + self.x = x + +def foo(c: C, x: int) -> None: + c.x = x - c.x +[out] +def C.__init__(self, x): + self :: __main__.C + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def foo(c, x): + c :: __main__.C + x, r0, r1 :: int + r2 :: bool +L0: + r0 = borrow c.x + r1 = CPyTagged_Subtract(x, r0) + c.x = r1 + return 1 + +[case testExceptionWithOverlappingFloatErrorValue] +def f() -> float: + return 0.0 + +def g() -> float: + return f() +[out] +def f(): +L0: + return 0.0 +def g(): + r0 :: float + r1 :: bit + r2 :: object + r3 :: float +L0: + r0 = f() + r1 = r0 == -113.0 + if r1 goto L2 else goto L1 :: bool +L1: + return r0 +L2: + r2 = PyErr_Occurred() + if not is_error(r2) goto L3 (error at g:5) else goto L1 +L3: + r3 = :: float + return r3 + +[case testExceptionWithLowLevelIntAttribute] +from mypy_extensions import i32, i64 + +class C: + def __init__(self, x: i32, y: i64) -> None: + self.x = x + self.y = y + +def f(c: C) -> None: + c.x + c.y +[out] +def C.__init__(self, x, y): + self :: __main__.C + x :: i32 + y :: i64 +L0: + self.x = x + self.y = y + return 1 +def f(c): + c :: __main__.C + r0 :: i32 + r1 :: i64 +L0: + r0 = c.x + r1 = c.y + return 1 + +[case testConditionallyUndefinedI64] +from mypy_extensions import i64 + +def f(x: i64) -> i64: + if x: + y: i64 = 2 + return y +[out] +def f(x): + x, r0, y :: i64 + __locals_bitmap0 :: u32 + r1 :: bit + r2, r3 :: u32 + r4 :: bit + r5 :: bool + r6 :: i64 +L0: + r0 = :: i64 + y = r0 + __locals_bitmap0 = 0 + r1 = x != 0 + if r1 goto L1 else goto L2 :: bool +L1: + y = 2 + r2 = __locals_bitmap0 | 1 + __locals_bitmap0 = r2 +L2: + r3 = __locals_bitmap0 & 1 + r4 = r3 == 0 + if r4 goto L3 else goto L5 :: bool +L3: + r5 = raise UnboundLocalError('local variable "y" referenced before assignment') + if not r5 goto L6 (error at f:-1) else goto L4 :: bool +L4: + unreachable +L5: + return y +L6: + r6 = :: i64 + return r6 + +[case testExceptionWithFloatAttribute] +class C: + def __init__(self, x: float, y: float) -> None: + self.x = x + if x: + self.y = y + +def f(c: C) -> float: + return c.x + c.y +[out] +def C.__init__(self, x, y): + self :: __main__.C + x, y :: float + r0 :: bit +L0: + self.x = x + r0 = x != 0.0 + if r0 goto L1 else goto L2 :: bool +L1: + self.y = y +L2: + return 1 +def f(c): + c :: __main__.C + r0, r1 :: float + r2 :: bit + r3 :: float + r4 :: object + r5 :: float +L0: + r0 = c.x + r1 = c.y + r2 = r1 == -113.0 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = r0 + r1 + return r3 +L2: + r4 = PyErr_Occurred() + if not is_error(r4) goto L3 (error at f:8) else goto L1 +L3: + r5 = :: float + return r5 diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py new file mode 100644 index 000000000000..be66307286fc --- /dev/null +++ b/mypyc/test-data/fixtures/ir.py @@ -0,0 +1,374 @@ +# These builtins stubs are used implicitly in AST to IR generation +# test cases. + +import _typeshed +from typing import ( + TypeVar, Generic, List, Iterator, Iterable, Dict, Optional, Tuple, Any, Set, + overload, Mapping, Union, Callable, Sequence, FrozenSet, Protocol +) + +_T = TypeVar('_T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) +_S = TypeVar('_S') +_K = TypeVar('_K') # for keys in mapping +_V = TypeVar('_V') # for values in mapping + +class __SupportsAbs(Protocol[T_co]): + def __abs__(self) -> T_co: pass + +class __SupportsDivMod(Protocol[T_contra, T_co]): + def __divmod__(self, other: T_contra) -> T_co: ... + +class __SupportsRDivMod(Protocol[T_contra, T_co]): + def __rdivmod__(self, other: T_contra) -> T_co: ... + +_M = TypeVar("_M", contravariant=True) + +class __SupportsPow2(Protocol[T_contra, T_co]): + def __pow__(self, other: T_contra) -> T_co: ... + +class __SupportsPow3NoneOnly(Protocol[T_contra, T_co]): + def __pow__(self, other: T_contra, modulo: None = ...) -> T_co: ... + +class __SupportsPow3(Protocol[T_contra, _M, T_co]): + def __pow__(self, other: T_contra, modulo: _M) -> T_co: ... + +__SupportsSomeKindOfPow = Union[ + __SupportsPow2[Any, Any], __SupportsPow3NoneOnly[Any, Any] | __SupportsPow3[Any, Any, Any] +] + +class object: + def __init__(self) -> None: pass + def __eq__(self, x: object) -> bool: pass + def __ne__(self, x: object) -> bool: pass + +class type: + def __init__(self, o: object) -> None: ... + def __or__(self, o: object) -> Any: ... + __name__ : str + __annotations__: Dict[str, Any] + +class ellipsis: pass + +# Primitive types are special in generated code. + +class int: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: object, base: int = 10) -> None: pass + def __add__(self, n: int) -> int: pass + def __sub__(self, n: int) -> int: pass + def __mul__(self, n: int) -> int: pass + def __pow__(self, n: int, modulo: Optional[int] = None) -> int: pass + def __floordiv__(self, x: int) -> int: pass + def __truediv__(self, x: float) -> float: pass + def __mod__(self, x: int) -> int: pass + def __divmod__(self, x: float) -> Tuple[float, float]: pass + def __neg__(self) -> int: pass + def __pos__(self) -> int: pass + def __abs__(self) -> int: pass + def __invert__(self) -> int: pass + def __and__(self, n: int) -> int: pass + def __or__(self, n: int) -> int: pass + def __xor__(self, n: int) -> int: pass + def __lshift__(self, x: int) -> int: pass + def __rshift__(self, x: int) -> int: pass + def __eq__(self, n: object) -> bool: pass + def __ne__(self, n: object) -> bool: pass + def __lt__(self, n: int) -> bool: pass + def __gt__(self, n: int) -> bool: pass + def __le__(self, n: int) -> bool: pass + def __ge__(self, n: int) -> bool: pass + +class str: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: object) -> None: pass + def __add__(self, x: str) -> str: pass + def __mul__(self, x: int) -> str: pass + def __rmul__(self, x: int) -> str: pass + def __eq__(self, x: object) -> bool: pass + def __ne__(self, x: object) -> bool: pass + def __lt__(self, x: str) -> bool: ... + def __le__(self, x: str) -> bool: ... + def __gt__(self, x: str) -> bool: ... + def __ge__(self, x: str) -> bool: ... + @overload + def __getitem__(self, i: int) -> str: pass + @overload + def __getitem__(self, i: slice) -> str: pass + def __contains__(self, item: str) -> bool: pass + def __iter__(self) -> Iterator[str]: ... + def split(self, sep: Optional[str] = None, max: Optional[int] = None) -> List[str]: pass + def strip (self, item: str) -> str: pass + def join(self, x: Iterable[str]) -> str: pass + def format(self, *args: Any, **kwargs: Any) -> str: ... + def upper(self) -> str: ... + def startswith(self, x: str, start: int=..., end: int=...) -> bool: ... + def endswith(self, x: str, start: int=..., end: int=...) -> bool: ... + def replace(self, old: str, new: str, maxcount: int=...) -> str: ... + def encode(self, encoding: str=..., errors: str=...) -> bytes: ... + +class float: + def __init__(self, x: object) -> None: pass + def __add__(self, n: float) -> float: pass + def __radd__(self, n: float) -> float: pass + def __sub__(self, n: float) -> float: pass + def __rsub__(self, n: float) -> float: pass + def __mul__(self, n: float) -> float: pass + def __truediv__(self, n: float) -> float: pass + def __floordiv__(self, n: float) -> float: pass + def __mod__(self, n: float) -> float: pass + def __pow__(self, n: float) -> float: pass + def __neg__(self) -> float: pass + def __pos__(self) -> float: pass + def __abs__(self) -> float: pass + def __invert__(self) -> float: pass + def __eq__(self, x: object) -> bool: pass + def __ne__(self, x: object) -> bool: pass + def __lt__(self, x: float) -> bool: ... + def __le__(self, x: float) -> bool: ... + def __gt__(self, x: float) -> bool: ... + def __ge__(self, x: float) -> bool: ... + +class complex: + def __init__(self, x: object, y: object = None) -> None: pass + def __add__(self, n: complex) -> complex: pass + def __radd__(self, n: float) -> complex: pass + def __sub__(self, n: complex) -> complex: pass + def __rsub__(self, n: float) -> complex: pass + def __mul__(self, n: complex) -> complex: pass + def __truediv__(self, n: complex) -> complex: pass + def __neg__(self) -> complex: pass + +class bytes: + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, x: object) -> None: ... + def __add__(self, x: bytes) -> bytes: ... + def __mul__(self, x: int) -> bytes: ... + def __rmul__(self, x: int) -> bytes: ... + def __eq__(self, x: object) -> bool: ... + def __ne__(self, x: object) -> bool: ... + @overload + def __getitem__(self, i: int) -> int: ... + @overload + def __getitem__(self, i: slice) -> bytes: ... + def join(self, x: Iterable[object]) -> bytes: ... + def decode(self, x: str=..., y: str=...) -> str: ... + +class bytearray: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: object) -> None: pass + @overload + def __init__(self, string: str, encoding: str, err: str = ...) -> None: pass + def __add__(self, s: bytes) -> bytearray: ... + def __setitem__(self, i: int, o: int) -> None: ... + def __getitem__(self, i: int) -> int: ... + def decode(self, x: str = ..., y: str = ...) -> str: ... + +class bool(int): + def __init__(self, o: object = ...) -> None: ... + @overload + def __and__(self, n: bool) -> bool: ... + @overload + def __and__(self, n: int) -> int: ... + @overload + def __or__(self, n: bool) -> bool: ... + @overload + def __or__(self, n: int) -> int: ... + @overload + def __xor__(self, n: bool) -> bool: ... + @overload + def __xor__(self, n: int) -> int: ... + +class tuple(Generic[T_co], Sequence[T_co], Iterable[T_co]): + def __init__(self, i: Iterable[T_co]) -> None: pass + @overload + def __getitem__(self, i: int) -> T_co: pass + @overload + def __getitem__(self, i: slice) -> Tuple[T_co, ...]: pass + def __len__(self) -> int: pass + def __iter__(self) -> Iterator[T_co]: ... + def __contains__(self, item: object) -> int: ... + +class function: pass + +class list(Generic[_T], Sequence[_T], Iterable[_T]): + def __init__(self, i: Optional[Iterable[_T]] = None) -> None: pass + @overload + def __getitem__(self, i: int) -> _T: ... + @overload + def __getitem__(self, s: slice) -> List[_T]: ... + def __setitem__(self, i: int, o: _T) -> None: pass + def __delitem__(self, i: int) -> None: pass + def __mul__(self, i: int) -> List[_T]: pass + def __rmul__(self, i: int) -> List[_T]: pass + def __iter__(self) -> Iterator[_T]: pass + def __len__(self) -> int: pass + def __contains__(self, item: object) -> int: ... + def __add__(self, x: List[_T]) -> List[_T]: ... + def append(self, x: _T) -> None: pass + def pop(self, i: int = -1) -> _T: pass + def count(self, _T) -> int: pass + def extend(self, l: Iterable[_T]) -> None: pass + def insert(self, i: int, x: _T) -> None: pass + def sort(self) -> None: pass + def reverse(self) -> None: pass + def remove(self, o: _T) -> None: pass + def index(self, o: _T) -> int: pass + +class dict(Mapping[_K, _V]): + @overload + def __init__(self, **kwargs: _K) -> None: ... + @overload + def __init__(self, map: Mapping[_K, _V], **kwargs: _V) -> None: ... + @overload + def __init__(self, iterable: Iterable[Tuple[_K, _V]], **kwargs: _V) -> None: ... + def __getitem__(self, key: _K) -> _V: pass + def __setitem__(self, k: _K, v: _V) -> None: pass + def __delitem__(self, k: _K) -> None: pass + def __contains__(self, item: object) -> int: pass + def __iter__(self) -> Iterator[_K]: pass + def __len__(self) -> int: pass + @overload + def update(self, __m: Mapping[_K, _V], **kwargs: _V) -> None: pass + @overload + def update(self, __m: Iterable[Tuple[_K, _V]], **kwargs: _V) -> None: ... + @overload + def update(self, **kwargs: _V) -> None: ... + def pop(self, x: int) -> _K: pass + def keys(self) -> Iterable[_K]: pass + def values(self) -> Iterable[_V]: pass + def items(self) -> Iterable[Tuple[_K, _V]]: pass + def clear(self) -> None: pass + def copy(self) -> Dict[_K, _V]: pass + def setdefault(self, key: _K, val: _V = ...) -> _V: pass + +class set(Generic[_T]): + def __init__(self, i: Optional[Iterable[_T]] = None) -> None: pass + def __iter__(self) -> Iterator[_T]: pass + def __len__(self) -> int: pass + def add(self, x: _T) -> None: pass + def remove(self, x: _T) -> None: pass + def discard(self, x: _T) -> None: pass + def clear(self) -> None: pass + def pop(self) -> _T: pass + def update(self, x: Iterable[_S]) -> None: pass + def __or__(self, s: Union[Set[_S], FrozenSet[_S]]) -> Set[Union[_T, _S]]: ... + def __xor__(self, s: Union[Set[_S], FrozenSet[_S]]) -> Set[Union[_T, _S]]: ... + +class frozenset(Generic[_T]): + def __init__(self, i: Optional[Iterable[_T]] = None) -> None: pass + def __iter__(self) -> Iterator[_T]: pass + def __len__(self) -> int: pass + def __or__(self, s: Union[Set[_S], FrozenSet[_S]]) -> FrozenSet[Union[_T, _S]]: ... + def __xor__(self, s: Union[Set[_S], FrozenSet[_S]]) -> FrozenSet[Union[_T, _S]]: ... + +class slice: pass + +class range(Iterable[int]): + def __init__(self, x: int, y: int = ..., z: int = ...) -> None: pass + def __iter__(self) -> Iterator[int]: pass + def __len__(self) -> int: pass + def __next__(self) -> int: pass + +class property: + def __init__(self, fget: Optional[Callable[[Any], Any]] = ..., + fset: Optional[Callable[[Any, Any], None]] = ..., + fdel: Optional[Callable[[Any], None]] = ..., + doc: Optional[str] = ...) -> None: ... + def getter(self, fget: Callable[[Any], Any]) -> property: ... + def setter(self, fset: Callable[[Any, Any], None]) -> property: ... + def deleter(self, fdel: Callable[[Any], None]) -> property: ... + def __get__(self, obj: Any, type: Optional[type] = ...) -> Any: ... + def __set__(self, obj: Any, value: Any) -> None: ... + def __delete__(self, obj: Any) -> None: ... + def fget(self) -> Any: ... + def fset(self, value: Any) -> None: ... + def fdel(self) -> None: ... + +class BaseException: pass + +class Exception(BaseException): + def __init__(self, message: Optional[str] = None) -> None: pass + +class Warning(Exception): pass +class UserWarning(Warning): pass +class TypeError(Exception): pass +class ValueError(Exception): pass +class AttributeError(Exception): pass +class ImportError(Exception): pass +class NameError(Exception): pass +class UnboundLocalError(NameError): pass +class LookupError(Exception): pass +class KeyError(LookupError): pass +class IndexError(LookupError): pass +class RuntimeError(Exception): pass +class UnicodeEncodeError(RuntimeError): pass +class UnicodeDecodeError(RuntimeError): pass +class NotImplementedError(RuntimeError): pass + +class StopIteration(Exception): + value: Any + +class ArithmeticError(Exception): pass +class ZeroDivisionError(ArithmeticError): pass +class OverflowError(ArithmeticError): pass + +class GeneratorExit(BaseException): pass + +def any(i: Iterable[_T]) -> bool: pass +def all(i: Iterable[_T]) -> bool: pass +def sum(i: Iterable[_T]) -> int: pass +def reversed(object: Sequence[_T]) -> Iterator[_T]: ... +def id(o: object) -> int: pass +# This type is obviously wrong but the test stubs don't have Sized anymore +def len(o: object) -> int: pass +def print(*object) -> None: pass +def isinstance(x: object, t: object) -> bool: pass +def iter(i: Iterable[_T]) -> Iterator[_T]: pass +@overload +def next(i: Iterator[_T]) -> _T: pass +@overload +def next(i: Iterator[_T], default: _T) -> _T: pass +def hash(o: object) -> int: ... +def globals() -> Dict[str, Any]: ... +def getattr(obj: object, name: str, default: Any = None) -> Any: ... +def setattr(obj: object, name: str, value: Any) -> None: ... +def enumerate(x: Iterable[_T]) -> Iterator[Tuple[int, _T]]: ... +@overload +def zip(x: Iterable[_T], y: Iterable[_S]) -> Iterator[Tuple[_T, _S]]: ... +@overload +def zip(x: Iterable[_T], y: Iterable[_S], z: Iterable[_V]) -> Iterator[Tuple[_T, _S, _V]]: ... +def eval(e: str) -> Any: ... +def abs(x: __SupportsAbs[_T]) -> _T: ... +@overload +def divmod(x: __SupportsDivMod[T_contra, T_co], y: T_contra) -> T_co: ... +@overload +def divmod(x: T_contra, y: __SupportsRDivMod[T_contra, T_co]) -> T_co: ... +@overload +def pow(base: __SupportsPow2[T_contra, T_co], exp: T_contra, mod: None = None) -> T_co: ... +@overload +def pow(base: __SupportsPow3NoneOnly[T_contra, T_co], exp: T_contra, mod: None = None) -> T_co: ... +@overload +def pow(base: __SupportsPow3[T_contra, _M, T_co], exp: T_contra, mod: _M) -> T_co: ... +def exit() -> None: ... +def min(x: _T, y: _T) -> _T: ... +def max(x: _T, y: _T) -> _T: ... +def repr(o: object) -> str: ... +def ascii(o: object) -> str: ... +def ord(o: object) -> int: ... +def chr(i: int) -> str: ... + +# Dummy definitions. +class classmethod: pass +class staticmethod: pass + +NotImplemented: Any = ... diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py new file mode 100644 index 000000000000..36ec41c8f38b --- /dev/null +++ b/mypyc/test-data/fixtures/testutil.py @@ -0,0 +1,103 @@ +# Simple support library for our run tests. + +from contextlib import contextmanager +from collections.abc import Iterator +import math +from typing import ( + Any, Iterator, TypeVar, Generator, Optional, List, Tuple, Sequence, + Union, Callable, Awaitable, Generic +) +from typing import Final + +FLOAT_MAGIC: Final = -113.0 + +# Various different float values +float_vals = [ + float(n) * 0.25 for n in range(-10, 10) +] + [ + -0.0, + 1.0/3.0, + math.sqrt(2.0), + 1.23e200, + -2.34e200, + 5.43e-100, + -6.532e-200, + float('inf'), + -float('inf'), + float('nan'), + FLOAT_MAGIC, + math.pi, + 2.0 * math.pi, + math.pi / 2.0, + -math.pi / 2.0, + -1.7976931348623158e+308, # Smallest finite value + -2.2250738585072014e-308, # Closest to zero negative normal value + -7.5491e-312, # Arbitrary negative subnormal value + -5e-324, # Closest to zero negative subnormal value + 1.7976931348623158e+308, # Largest finite value + 2.2250738585072014e-308, # Closest to zero positive normal value + -6.3492e-312, # Arbitrary positive subnormal value + 5e-324, # Closest to zero positive subnormal value +] + +@contextmanager +def assertRaises(typ: type, msg: str = '') -> Iterator[None]: + try: + yield + except Exception as e: + assert type(e) is typ, f"{e!r} is not a {typ.__name__}" + assert msg in str(e), f'Message "{e}" does not match "{msg}"' + else: + assert False, f"Expected {typ.__name__} but got no exception" + +def assertDomainError() -> Any: + return assertRaises(ValueError, "math domain error") + +def assertMathRangeError() -> Any: + return assertRaises(OverflowError, "math range error") + +T = TypeVar('T') +U = TypeVar('U') +V = TypeVar('V') + +def run_generator(gen: Generator[T, V, U], + inputs: Optional[List[V]] = None, + p: bool = False) -> Tuple[Sequence[T], Union[U, str]]: + res: List[T] = [] + i = -1 + while True: + try: + if i >= 0 and inputs: + # ... fixtures don't have send + val = gen.send(inputs[i]) # type: ignore + elif not hasattr(gen, '__next__'): # type: ignore + val = gen.send(None) # type: ignore + else: + val = next(gen) + except StopIteration as e: + return (tuple(res), e.value) + except Exception as e: + return (tuple(res), str(e)) + if p: + print(val) + res.append(val) + i += 1 + +F = TypeVar('F', bound=Callable) + + +class async_val(Awaitable[V], Generic[T, V]): + def __init__(self, val: T) -> None: + self.val = val + + def __await__(self) -> Generator[T, V, V]: + z = yield self.val + return z + + +# Wrap a mypyc-generated function in a real python function, to allow it to be +# stuck into classes and the like. +def make_python_function(f: F) -> F: + def g(*args: Any, **kwargs: Any) -> Any: + return f(*args, **kwargs) + return g # type: ignore diff --git a/mypyc/test-data/fixtures/typing-full.pyi b/mypyc/test-data/fixtures/typing-full.pyi new file mode 100644 index 000000000000..6b6aba6802b1 --- /dev/null +++ b/mypyc/test-data/fixtures/typing-full.pyi @@ -0,0 +1,175 @@ +# More complete stub for typing module. +# +# Use [typing fixtures/typing-full.pyi] to use this instead of lib-stub/typing.pyi +# in a particular test case. +# +# Many of the definitions have special handling in the type checker, so they +# can just be initialized to anything. + +from abc import abstractmethod, ABCMeta + +class GenericMeta(type): pass + +class _SpecialForm: + def __getitem__(self, index): ... + +cast = 0 +overload = 0 +Any = object() +Optional = 0 +TypeVar = 0 +Generic = 0 +Protocol = 0 +Tuple = 0 +_promote = 0 +NamedTuple = 0 +Type = 0 +no_type_check = 0 +ClassVar = 0 +Final = 0 +TypedDict = 0 +NoReturn = 0 +NewType = 0 +Callable: _SpecialForm +Union: _SpecialForm +Literal: _SpecialForm + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) +U = TypeVar('U') +V = TypeVar('V') +S = TypeVar('S') + +# Note: definitions below are different from typeshed, variances are declared +# to silence the protocol variance checks. Maybe it is better to use type: ignore? + +@runtime_checkable +class Container(Protocol[T_co]): + @abstractmethod + # Use int because bool isn't in the default test builtins + def __contains__(self, arg: object) -> int: pass + +@runtime_checkable +class Sized(Protocol): + @abstractmethod + def __len__(self) -> int: pass + +@runtime_checkable +class Iterable(Protocol[T_co]): + @abstractmethod + def __iter__(self) -> 'Iterator[T_co]': pass + +@runtime_checkable +class Iterator(Iterable[T_co], Protocol): + @abstractmethod + def __next__(self) -> T_co: pass + +class Generator(Iterator[T], Generic[T, U, V]): + @abstractmethod + def send(self, value: U) -> T: pass + + @abstractmethod + def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass + + @abstractmethod + def close(self) -> None: pass + + @abstractmethod + def __iter__(self) -> 'Generator[T, U, V]': pass + +class AsyncGenerator(AsyncIterator[T], Generic[T, U]): + @abstractmethod + def __anext__(self) -> Awaitable[T]: pass + + @abstractmethod + def asend(self, value: U) -> Awaitable[T]: pass + + @abstractmethod + def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T]: pass + + @abstractmethod + def aclose(self) -> Awaitable[T]: pass + + @abstractmethod + def __aiter__(self) -> 'AsyncGenerator[T, U]': pass + +@runtime_checkable +class Awaitable(Protocol[T]): + @abstractmethod + def __await__(self) -> Generator[Any, Any, T]: pass + +class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S], metaclass=ABCMeta): + pass + +class Coroutine(Awaitable[V], Generic[T, U, V]): + @abstractmethod + def send(self, value: U) -> T: pass + + @abstractmethod + def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass + + @abstractmethod + def close(self) -> None: pass + +@runtime_checkable +class AsyncIterable(Protocol[T]): + @abstractmethod + def __aiter__(self) -> 'AsyncIterator[T]': pass + +@runtime_checkable +class AsyncIterator(AsyncIterable[T], Protocol): + def __aiter__(self) -> 'AsyncIterator[T]': return self + @abstractmethod + def __anext__(self) -> Awaitable[T]: pass + +class Sequence(Iterable[T_co], Container[T_co]): + @abstractmethod + def __getitem__(self, n: Any) -> T_co: pass + +class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + @overload + def get(self, k: T) -> Optional[T_co]: pass + @overload + def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass + def values(self) -> Iterable[T_co]: pass # Approximate return type + def items(self) -> Iterable[Tuple[T, T_co]]: pass # Approximate return type + def __len__(self) -> int: ... + def __contains__(self, arg: object) -> int: pass + +class MutableMapping(Mapping[T, U], metaclass=ABCMeta): + def __setitem__(self, k: T, v: U) -> None: pass + +class SupportsInt(Protocol): + def __int__(self) -> int: pass + +class SupportsFloat(Protocol): + def __float__(self) -> float: pass + +def runtime_checkable(cls: T) -> T: + return cls + +class ContextManager(Generic[T]): + def __enter__(self) -> T: pass + # Use Any because not all the precise types are in the fixtures. + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass + +TYPE_CHECKING = 1 + +# Fallback type for all typed dicts (does not exist at runtime). +class _TypedDict(Mapping[str, object]): + # Needed to make this class non-abstract. It is explicitly declared abstract in + # typeshed, but we don't want to import abc here, as it would slow down the tests. + def __iter__(self) -> Iterator[str]: ... + def copy(self: T) -> T: ... + # Using NoReturn so that only calls using the plugin hook can go through. + def setdefault(self, k: NoReturn, default: object) -> object: ... + # Mypy expects that 'default' has a type variable type. + def pop(self, k: NoReturn, default: T = ...) -> object: ... + def update(self: T, __m: T) -> None: ... + def __delitem__(self, k: NoReturn) -> None: ... + +class TypeAliasType: + pass diff --git a/mypyc/test-data/irbuild-any.test b/mypyc/test-data/irbuild-any.test new file mode 100644 index 000000000000..3bfb1587fb3b --- /dev/null +++ b/mypyc/test-data/irbuild-any.test @@ -0,0 +1,238 @@ +-- Generate ops related to Any types + + +[case testCoerceAnyInCallsAndReturn] +from typing import Any +def f(x: Any) -> Any: + return g(x) +def g(x: int) -> int: + return f(x) +[out] +def f(x): + x :: object + r0, r1 :: int + r2 :: object +L0: + r0 = unbox(int, x) + r1 = g(r0) + r2 = box(int, r1) + return r2 +def g(x): + x :: int + r0, r1 :: object + r2 :: int +L0: + r0 = box(int, x) + r1 = f(r0) + r2 = unbox(int, r1) + return r2 + +[case testCoerceAnyInAssignment] +from typing import Any, Tuple +class C: + a: Any + n: int +def f(a: Any, n: int, c: C) -> None: + c.a = n + c.n = a + a = n + n = a + a.a = n +[out] +def f(a, n, c): + a :: object + n :: int + c :: __main__.C + r0 :: object + r1 :: bool + r2 :: int + r3 :: bool + r4 :: object + r5 :: int + r6 :: str + r7 :: object + r8 :: i32 + r9 :: bit +L0: + r0 = box(int, n) + c.a = r0; r1 = is_error + r2 = unbox(int, a) + c.n = r2; r3 = is_error + r4 = box(int, n) + a = r4 + r5 = unbox(int, a) + n = r5 + r6 = 'a' + r7 = box(int, n) + r8 = PyObject_SetAttr(a, r6, r7) + r9 = r8 >= 0 :: signed + return 1 + +[case testCoerceAnyInOps] +from typing import Any, List +def f1(a: Any, n: int) -> None: + a + n + n + a +def f2(a: Any, n: int, l: List[int]) -> None: + a[n] + l[a] + a[n] = n + l[a] = n + l[n] = a + [a, n] +def f3(a: Any, n: int) -> None: + a += n + n += a +[out] +def f1(a, n): + a :: object + n :: int + r0, r1, r2, r3 :: object +L0: + r0 = box(int, n) + r1 = PyNumber_Add(a, r0) + r2 = box(int, n) + r3 = PyNumber_Add(r2, a) + return 1 +def f2(a, n, l): + a :: object + n :: int + l :: list + r0, r1, r2, r3, r4 :: object + r5 :: i32 + r6 :: bit + r7 :: object + r8 :: i32 + r9, r10 :: bit + r11 :: list + r12 :: object + r13 :: ptr +L0: + r0 = box(int, n) + r1 = PyObject_GetItem(a, r0) + r2 = PyObject_GetItem(l, a) + r3 = box(int, n) + r4 = box(int, n) + r5 = PyObject_SetItem(a, r3, r4) + r6 = r5 >= 0 :: signed + r7 = box(int, n) + r8 = PyObject_SetItem(l, a, r7) + r9 = r8 >= 0 :: signed + r10 = CPyList_SetItem(l, n, a) + r11 = PyList_New(2) + r12 = box(int, n) + r13 = list_items r11 + buf_init_item r13, 0, a + buf_init_item r13, 1, r12 + keep_alive r11 + return 1 +def f3(a, n): + a :: object + n :: int + r0, r1, r2, r3 :: object + r4 :: int +L0: + r0 = box(int, n) + r1 = PyNumber_InPlaceAdd(a, r0) + a = r1 + r2 = box(int, n) + r3 = PyNumber_InPlaceAdd(r2, a) + r4 = unbox(int, r3) + n = r4 + return 1 + +[case testCoerceAnyInConditionalExpr] +from typing import Any +def f4(a: Any, n: int, b: bool) -> None: + a = a if b else n + n = n if b else a +[out] +def f4(a, n, b): + a :: object + n :: int + b :: bool + r0 :: union[object, int] + r1, r2 :: object + r3 :: union[int, object] + r4 :: int +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = a + goto L3 +L2: + r1 = box(int, n) + r0 = r1 +L3: + a = r0 + if b goto L4 else goto L5 :: bool +L4: + r2 = box(int, n) + r3 = r2 + goto L6 +L5: + r3 = a +L6: + r4 = unbox(int, r3) + n = r4 + return 1 + +[case testAbsSpecialization] +# Specialization of native classes that implement __abs__ is checked in +# irbuild-dunders.test +def f() -> None: + a = abs(1) + b = abs(1.1) +[out] +def f(): + r0, r1 :: object + r2, a :: int + r3, b :: float +L0: + r0 = object 1 + r1 = PyNumber_Absolute(r0) + r2 = unbox(int, r1) + a = r2 + r3 = fabs(1.1) + b = r3 + return 1 + +[case testFunctionBasedOps] +def f() -> None: + a = divmod(5, 2) +def f2() -> int: + return pow(2, 5) +def f3() -> float: + return pow(2, 5, 3) +[out] +def f(): + r0, r1, r2 :: object + r3, a :: tuple[float, float] +L0: + r0 = object 5 + r1 = object 2 + r2 = PyNumber_Divmod(r0, r1) + r3 = unbox(tuple[float, float], r2) + a = r3 + return 1 +def f2(): + r0, r1, r2 :: object + r3 :: int +L0: + r0 = object 2 + r1 = object 5 + r2 = CPyNumber_Power(r0, r1) + r3 = unbox(int, r2) + return r3 +def f3(): + r0, r1, r2, r3 :: object + r4 :: int + r5 :: float +L0: + r0 = object 2 + r1 = object 5 + r2 = object 3 + r3 = PyNumber_Power(r0, r1, r2) + r4 = unbox(int, r3) + r5 = CPyFloat_FromTagged(r4) + return r5 diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test new file mode 100644 index 000000000000..11df241b5074 --- /dev/null +++ b/mypyc/test-data/irbuild-basic.test @@ -0,0 +1,3399 @@ +[case testTrivialFunction] +def f() -> int: + return 1 +[out] +def f(): +L0: + return 2 +[case testFunctionArgument] +def f(x: int) -> int: + return x +[out] +def f(x): + x :: int +L0: + return x + + +[case testExplicitNoneReturn] +def f() -> None: + return +[out] +def f(): +L0: + return 1 + +[case testExplicitNoneReturn2] +def f() -> None: + return None +[out] +def f(): +L0: + return 1 + +[case testAssignment] +def f() -> int: + x = 1 + y = x + return y +[out] +def f(): + x, y :: int +L0: + x = 2 + y = x + return y + +[case testAssignmentTwice] +def f(x: int) -> None: + y = 1 + y = x + return +[out] +def f(x): + x, y :: int +L0: + y = 2 + y = x + return 1 + +[case testIntArithmetic] +def f(x: int, y: int) -> int: + return x * (y + 1) +[out] +def f(x, y): + x, y, r0, r1 :: int +L0: + r0 = CPyTagged_Add(y, 2) + r1 = CPyTagged_Multiply(x, r0) + return r1 + +[case testIf] +def f(x: int, y: int) -> int: + if x < y: + x = 1 + return x +[out] +def f(x, y): + x, y :: int + r0 :: bit +L0: + r0 = int_lt x, y + if r0 goto L1 else goto L2 :: bool +L1: + x = 2 +L2: + return x + +[case testIfElse] +def f(x: int, y: int) -> int: + if x < y: + x = 1 + else: + x = 2 + return x +[out] +def f(x, y): + x, y :: int + r0 :: bit +L0: + r0 = int_lt x, y + if r0 goto L1 else goto L2 :: bool +L1: + x = 2 + goto L3 +L2: + x = 4 +L3: + return x + +[case testAnd1] +def f(x: int, y: int) -> int: + if x < y and x > y: + x = 1 + else: + x = 2 + return x +[out] +def f(x, y): + x, y :: int + r0, r1 :: bit +L0: + r0 = int_lt x, y + if r0 goto L1 else goto L3 :: bool +L1: + r1 = int_gt x, y + if r1 goto L2 else goto L3 :: bool +L2: + x = 2 + goto L4 +L3: + x = 4 +L4: + return x + +[case testAnd2] +def f(x: object, y: object) -> str: + return str(x) or str(y) +[out] +def f(x, y): + x, y :: object + r0 :: str + r1 :: bit + r2, r3 :: str +L0: + r0 = PyObject_Str(x) + r1 = CPyStr_IsTrue(r0) + if r1 goto L1 else goto L2 :: bool +L1: + r2 = r0 + goto L3 +L2: + r3 = PyObject_Str(y) + r2 = r3 +L3: + return r2 + +[case testOr] +def f(x: int, y: int) -> int: + if x < y or x > y: + x = 1 + else: + x = 2 + return x +[out] +def f(x, y): + x, y :: int + r0, r1 :: bit +L0: + r0 = int_lt x, y + if r0 goto L2 else goto L1 :: bool +L1: + r1 = int_gt x, y + if r1 goto L2 else goto L3 :: bool +L2: + x = 2 + goto L4 +L3: + x = 4 +L4: + return x + +[case testOr2] +def f(x: object, y: object) -> str: + return str(x) and str(y) +[out] +def f(x, y): + x, y :: object + r0 :: str + r1 :: bit + r2, r3 :: str +L0: + r0 = PyObject_Str(x) + r1 = CPyStr_IsTrue(r0) + if r1 goto L2 else goto L1 :: bool +L1: + r2 = r0 + goto L3 +L2: + r3 = PyObject_Str(y) + r2 = r3 +L3: + return r2 + +[case testSimpleNot] +def f(x: int, y: int) -> int: + if not (x < y): + x = 1 + return x +[out] +def f(x, y): + x, y :: int + r0 :: bit +L0: + r0 = int_lt x, y + if r0 goto L2 else goto L1 :: bool +L1: + x = 2 +L2: + return x + +[case testNotAnd] +def f(x: int, y: int) -> int: + if not (x < y and x > y): + x = 1 + return x +[out] +def f(x, y): + x, y :: int + r0, r1 :: bit +L0: + r0 = int_lt x, y + if r0 goto L1 else goto L2 :: bool +L1: + r1 = int_gt x, y + if r1 goto L3 else goto L2 :: bool +L2: + x = 2 +L3: + return x + +[case testWhile] +def f(x: int, y: int) -> int: + while x > y: + x = x - y + return x +[out] +def f(x, y): + x, y :: int + r0 :: bit + r1 :: int +L0: +L1: + r0 = int_gt x, y + if r0 goto L2 else goto L3 :: bool +L2: + r1 = CPyTagged_Subtract(x, y) + x = r1 + goto L1 +L3: + return x + +[case testWhile2] +def f(x: int, y: int) -> int: + x = 1 + while x > y: + x = x - y + return x +[out] +def f(x, y): + x, y :: int + r0 :: bit + r1 :: int +L0: + x = 2 +L1: + r0 = int_gt x, y + if r0 goto L2 else goto L3 :: bool +L2: + r1 = CPyTagged_Subtract(x, y) + x = r1 + goto L1 +L3: + return x + +[case testImplicitNoneReturn] +def f() -> None: + pass +[out] +def f(): +L0: + return 1 + +[case testImplicitNoneReturn2] +def f() -> None: + x = 1 +[out] +def f(): + x :: int +L0: + x = 2 + return 1 + +[case testImplicitNoneReturnAndIf] +def f(x: int, y: int) -> None: + if x < y: + x = 1 + else: + y = 2 +[out] +def f(x, y): + x, y :: int + r0 :: bit +L0: + r0 = int_lt x, y + if r0 goto L1 else goto L2 :: bool +L1: + x = 2 + goto L3 +L2: + y = 4 +L3: + return 1 + +[case testRecursion] +def f(n: int) -> int: + if n <= 1: + return 1 + else: + return f(n - 1) + f(n - 2) +[out] +def f(n): + n :: int + r0 :: bit + r1, r2, r3, r4, r5 :: int +L0: + r0 = int_le n, 2 + if r0 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + r1 = CPyTagged_Subtract(n, 2) + r2 = f(r1) + r3 = CPyTagged_Subtract(n, 4) + r4 = f(r3) + r5 = CPyTagged_Add(r2, r4) + return r5 +L3: + unreachable + +[case testReportTypeCheckError] +def f() -> None: + return 1 # E: No return value expected + +[case testReportSemanticaAnalysisError1] +def f(x: List[int]) -> None: pass # E: Name "List" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import List") + +[case testReportSemanticaAnalysisError2] +def f() -> None: + x # E: Name "x" is not defined + +[case testElif] +def f(n: int) -> int: + if n < 0: + x = 1 + elif n == 0: + x = 1 + else: + x = 2 + return x +[out] +def f(n): + n :: int + r0 :: bit + x :: int + r1 :: bit +L0: + r0 = int_lt n, 0 + if r0 goto L1 else goto L2 :: bool +L1: + x = 2 + goto L6 +L2: + r1 = int_eq n, 0 + if r1 goto L3 else goto L4 :: bool +L3: + x = 2 + goto L5 +L4: + x = 4 +L5: +L6: + return x + +[case testUnaryMinus] +def f(n: int) -> int: + return -n +[out] +def f(n): + n, r0 :: int +L0: + r0 = CPyTagged_Negate(n) + return r0 + +[case testConditionalExpr] +def f(n: int) -> int: + return 0 if n == 0 else 1 +[out] +def f(n): + n :: int + r0 :: bit + r1 :: int +L0: + r0 = int_eq n, 0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = 0 + goto L3 +L2: + r1 = 2 +L3: + return r1 + +[case testOperatorAssignment] +def f() -> int: + x = 0 + x += 1 + return x +[out] +def f(): + x, r0 :: int +L0: + x = 0 + r0 = CPyTagged_Add(x, 2) + x = r0 + return x + +[case testTrue] +def f() -> bool: + return True +[out] +def f(): +L0: + return 1 + +[case testFalse] +def f() -> bool: + return False +[out] +def f(): +L0: + return 0 +[case testBoolCond] +def f(x: bool) -> bool: + if x: + return False + else: + return True +[out] +def f(x): + x :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + return 0 +L2: + return 1 +L3: + unreachable + +[case testPycall] +import testmodule + +def f(x: int) -> int: + return testmodule.factorial(x) +[file testmodule.py] +def factorial(x: int) -> int: + if x == 0: + return 1 + else: + return x * factorial(x-1) +[out] +def f(x): + x :: int + r0 :: object + r1 :: str + r2, r3, r4 :: object + r5 :: int +L0: + r0 = testmodule :: module + r1 = 'factorial' + r2 = CPyObject_GetAttr(r0, r1) + r3 = box(int, x) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = unbox(int, r4) + return r5 + +[case testImport_toplevel] +import sys +import enum as enum2 +import collections.abc +import collections.abc as abc2 +_ = "filler" +import single +single.hello() + +[file single.py] +def hello() -> None: + print("hello, world") + +[out] +def __top_level__(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4 :: object + r5, r6, r7, r8 :: object_ptr + r9 :: object_ptr[4] + r10 :: c_ptr + r11 :: native_int[4] + r12 :: c_ptr + r13 :: object + r14 :: dict + r15, r16 :: str + r17 :: bit + r18 :: str + r19 :: dict + r20 :: str + r21 :: i32 + r22 :: bit + r23 :: object_ptr + r24 :: object_ptr[1] + r25 :: c_ptr + r26 :: native_int[1] + r27 :: c_ptr + r28 :: object + r29 :: dict + r30, r31 :: str + r32 :: bit + r33 :: object + r34 :: str + r35, r36 :: object +L0: + r0 = builtins :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = 'builtins' + r4 = PyImport_Import(r3) + builtins = r4 :: module +L2: + r5 = load_address sys :: module + r6 = load_address enum :: module + r7 = load_address collections.abc :: module + r8 = load_address collections.abc :: module + r9 = [r5, r6, r7, r8] + r10 = load_address r9 + r11 = [1, 2, 3, 4] + r12 = load_address r11 + r13 = (('sys', 'sys', 'sys'), ('enum', 'enum', 'enum2'), ('collections.abc', 'collections', 'collections'), ('collections.abc', 'collections.abc', 'abc2')) + r14 = __main__.globals :: static + r15 = 'main' + r16 = '' + r17 = CPyImport_ImportMany(r13, r10, r14, r15, r16, r12) + r18 = 'filler' + r19 = __main__.globals :: static + r20 = '_' + r21 = CPyDict_SetItem(r19, r20, r18) + r22 = r21 >= 0 :: signed + r23 = load_address single :: module + r24 = [r23] + r25 = load_address r24 + r26 = [6] + r27 = load_address r26 + r28 = (('single', 'single', 'single'),) + r29 = __main__.globals :: static + r30 = 'main' + r31 = '' + r32 = CPyImport_ImportMany(r28, r25, r29, r30, r31, r27) + r33 = single :: module + r34 = 'hello' + r35 = CPyObject_GetAttr(r33, r34) + r36 = PyObject_CallFunctionObjArgs(r35, 0) + return 1 + +[case testFromImport_toplevel] +from testmodule import g, h +from testmodule import h as two + +def f(x: int) -> int: + return g(x) + h() + two() +[file testmodule.py] +def g(x: int) -> int: + return x + 1 +def h() -> int: + return 2 +[out] +def f(x): + x :: int + r0 :: dict + r1 :: str + r2, r3, r4 :: object + r5 :: int + r6 :: dict + r7 :: str + r8, r9 :: object + r10, r11 :: int + r12 :: dict + r13 :: str + r14, r15 :: object + r16, r17 :: int +L0: + r0 = __main__.globals :: static + r1 = 'g' + r2 = CPyDict_GetItem(r0, r1) + r3 = box(int, x) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = unbox(int, r4) + r6 = __main__.globals :: static + r7 = 'h' + r8 = CPyDict_GetItem(r6, r7) + r9 = PyObject_CallFunctionObjArgs(r8, 0) + r10 = unbox(int, r9) + r11 = CPyTagged_Add(r5, r10) + r12 = __main__.globals :: static + r13 = 'two' + r14 = CPyDict_GetItem(r12, r13) + r15 = PyObject_CallFunctionObjArgs(r14, 0) + r16 = unbox(int, r15) + r17 = CPyTagged_Add(r11, r16) + return r17 +def __top_level__(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4, r5 :: object + r6 :: str + r7 :: dict + r8, r9, r10 :: object + r11 :: str + r12 :: dict + r13 :: object +L0: + r0 = builtins :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = 'builtins' + r4 = PyImport_Import(r3) + builtins = r4 :: module +L2: + r5 = ('g', 'h') + r6 = 'testmodule' + r7 = __main__.globals :: static + r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) + testmodule = r8 :: module + r9 = ('h',) + r10 = ('two',) + r11 = 'testmodule' + r12 = __main__.globals :: static + r13 = CPyImport_ImportFromMany(r11, r9, r10, r12) + testmodule = r13 :: module + return 1 + +[case testPrintFullname] +import builtins +def f(x: int) -> None: + builtins.print(5) +[out] +def f(x): + x :: int + r0 :: object + r1 :: str + r2, r3, r4 :: object +L0: + r0 = builtins :: module + r1 = 'print' + r2 = CPyObject_GetAttr(r0, r1) + r3 = object 5 + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + return 1 + +[case testPrint] +import builtins +def f(x: int) -> None: + print(5) +[out] +def f(x): + x :: int + r0 :: object + r1 :: str + r2, r3, r4 :: object +L0: + r0 = builtins :: module + r1 = 'print' + r2 = CPyObject_GetAttr(r0, r1) + r3 = object 5 + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + return 1 + +[case testUnicodeLiteral] +def f() -> str: + x = "some string" + return "some other string" +[out] +def f(): + r0, x, r1 :: str +L0: + r0 = 'some string' + x = r0 + r1 = 'some other string' + return r1 + +[case testBytesLiteral] +def f() -> bytes: + x = b'\xf0' + return b'1234' +[out] +def f(): + r0, x, r1 :: bytes +L0: + r0 = b'\xf0' + x = r0 + r1 = b'1234' + return r1 + +[case testPyMethodCall1] +from typing import Any +def f(x: Any) -> int: + y: int = x.pop() + return x.pop() +[out] +def f(x): + x :: object + r0 :: str + r1 :: object + r2, y :: int + r3 :: str + r4 :: object + r5 :: int +L0: + r0 = 'pop' + r1 = CPyObject_CallMethodObjArgs(x, r0, 0) + r2 = unbox(int, r1) + y = r2 + r3 = 'pop' + r4 = CPyObject_CallMethodObjArgs(x, r3, 0) + r5 = unbox(int, r4) + return r5 + +[case testObjectType] +def g(y: object) -> None: + g(y) + g([1]) + g(None) +[out] +def g(y): + y :: object + r0 :: None + r1 :: list + r2 :: object + r3 :: ptr + r4 :: None + r5 :: object + r6 :: None +L0: + r0 = g(y) + r1 = PyList_New(1) + r2 = object 1 + r3 = list_items r1 + buf_init_item r3, 0, r2 + keep_alive r1 + r4 = g(r1) + r5 = box(None, 1) + r6 = g(r5) + return 1 + +[case testCoerceToObject1] +def g(y: object) -> object: + g(1) + a = [y] + a[0] = (1, 2) + y = True + return 3 +[out] +def g(y): + y, r0, r1 :: object + r2 :: list + r3 :: ptr + a :: list + r4 :: tuple[int, int] + r5 :: object + r6 :: bit + r7, r8 :: object +L0: + r0 = object 1 + r1 = g(r0) + r2 = PyList_New(1) + r3 = list_items r2 + buf_init_item r3, 0, y + keep_alive r2 + a = r2 + r4 = (2, 4) + r5 = box(tuple[int, int], r4) + r6 = CPyList_SetItem(a, 0, r5) + r7 = box(bool, 1) + y = r7 + r8 = object 3 + return r8 + +[case testCoerceToObject2] +class A: + x: object + n: int +def f(a: A, o: object) -> None: + a.x = 1 + o = a.n +[out] +def f(a, o): + a :: __main__.A + o, r0 :: object + r1 :: bool + r2 :: int + r3 :: object +L0: + r0 = object 1 + a.x = r0; r1 = is_error + r2 = a.n + r3 = box(int, r2) + o = r3 + return 1 + +[case testAssertType] +from typing import assert_type +def f(x: int) -> None: + y = assert_type(x, int) +[out] +def f(x): + x, y :: int +L0: + y = x + return 1 + +[case testDownCast] +from typing import cast, List, Tuple +class A: pass +def f(x: object) -> None: + n = cast(int, x) + b = cast(bool, x) + a = cast(A, x) + l = cast(List[int], x) + t = cast(Tuple[int, A], x) +[out] +def f(x): + x :: object + r0, n :: int + r1, b :: bool + r2, a :: __main__.A + r3, l :: list + r4, t :: tuple[int, __main__.A] +L0: + r0 = unbox(int, x) + n = r0 + r1 = unbox(bool, x) + b = r1 + r2 = cast(__main__.A, x) + a = r2 + r3 = cast(list, x) + l = r3 + r4 = unbox(tuple[int, __main__.A], x) + t = r4 + return 1 + +[case testDownCastSpecialCases] +from typing import cast, Optional, Tuple +class A: pass +def f(o: Optional[A], n: int, t: Tuple[int, ...], tt: Tuple[int, int]) -> None: + a = cast(A, o) + m = cast(bool, n) + t = tt +[out] +def f(o, n, t, tt): + o :: union[__main__.A, None] + n :: int + t :: tuple + tt :: tuple[int, int] + r0, a :: __main__.A + r1 :: object + r2, m :: bool + r3 :: object +L0: + r0 = cast(__main__.A, o) + a = r0 + r1 = box(int, n) + r2 = unbox(bool, r1) + m = r2 + r3 = box(tuple[int, int], tt) + t = r3 + return 1 + +[case testSuccessfulCast] +from typing import cast, Optional, Tuple, List, Dict +class A: pass +def f(o: object, + p: Optional[A], + n: int, + b: bool, + t: Tuple[int, ...], + s: Tuple[int, int], + a: A, + l: List[A], + d: Dict[int, str]) -> None: + o = cast(object, o) + p = cast(Optional[A], p) + n = cast(int, n) + b = cast(bool, b) + t = cast(Tuple[int, ...], t) + s = cast(Tuple[int, int], s) + o = cast(object, n) + a = cast(A, a) + l2 = cast(List[object], l) + d2 = cast(Dict[object, str], d) +[out] +def f(o, p, n, b, t, s, a, l, d): + o :: object + p :: union[__main__.A, None] + n :: int + b :: bool + t :: tuple + s :: tuple[int, int] + a :: __main__.A + l :: list + d :: dict + r0 :: object + l2 :: list + d2 :: dict +L0: + o = o + p = p + n = n + b = b + t = t + s = s + r0 = box(int, n) + o = r0 + a = a + l2 = l + d2 = d + return 1 + +[case testGenericSetItem] +from typing import Any +def f(x: Any, y: Any, z: Any) -> None: + x[y] = z +[out] +def f(x, y, z): + x, y, z :: object + r0 :: i32 + r1 :: bit +L0: + r0 = PyObject_SetItem(x, y, z) + r1 = r0 >= 0 :: signed + return 1 + +[case testLoadFloatSum] +def assign_and_return_float_sum() -> float: + f1 = 1.0 + f2 = 2.0 + f3 = 3.0 + return f1 * f2 + f3 +[out] +def assign_and_return_float_sum(): + f1, f2, f3, r0, r1 :: float +L0: + f1 = 1.0 + f2 = 2.0 + f3 = 3.0 + r0 = f1 * f2 + r1 = r0 + f3 + return r1 + +[case testLoadComplex] +def load() -> complex: + real = 1 + return 5j+real +[out] +def load(): + real :: int + r0, r1, r2 :: object +L0: + real = 2 + r0 = 5j + r1 = box(int, real) + r2 = PyNumber_Add(r0, r1) + return r2 + +[case testBigIntLiteral_64bit] +def big_int() -> None: + a_62_bit = 4611686018427387902 + max_62_bit = 4611686018427387903 + b_63_bit = 4611686018427387904 + c_63_bit = 9223372036854775806 + max_63_bit = 9223372036854775807 + d_64_bit = 9223372036854775808 + max_32_bit = 2147483647 + max_31_bit = 1073741823 +[out] +def big_int(): + a_62_bit, max_62_bit, r0, b_63_bit, r1, c_63_bit, r2, max_63_bit, r3, d_64_bit, max_32_bit, max_31_bit :: int +L0: + a_62_bit = 9223372036854775804 + max_62_bit = 9223372036854775806 + r0 = object 4611686018427387904 + b_63_bit = r0 + r1 = object 9223372036854775806 + c_63_bit = r1 + r2 = object 9223372036854775807 + max_63_bit = r2 + r3 = object 9223372036854775808 + d_64_bit = r3 + max_32_bit = 4294967294 + max_31_bit = 2147483646 + return 1 + +[case testBigIntLiteral_32bit] +def big_int() -> None: + a_62_bit = 4611686018427387902 + max_62_bit = 4611686018427387903 + b_63_bit = 4611686018427387904 + c_63_bit = 9223372036854775806 + max_63_bit = 9223372036854775807 + d_64_bit = 9223372036854775808 + max_32_bit = 2147483647 + max_31_bit = 1073741823 +[out] +def big_int(): + r0, a_62_bit, r1, max_62_bit, r2, b_63_bit, r3, c_63_bit, r4, max_63_bit, r5, d_64_bit, r6, max_32_bit, max_31_bit :: int +L0: + r0 = object 4611686018427387902 + a_62_bit = r0 + r1 = object 4611686018427387903 + max_62_bit = r1 + r2 = object 4611686018427387904 + b_63_bit = r2 + r3 = object 9223372036854775806 + c_63_bit = r3 + r4 = object 9223372036854775807 + max_63_bit = r4 + r5 = object 9223372036854775808 + d_64_bit = r5 + r6 = object 2147483647 + max_32_bit = r6 + max_31_bit = 2147483646 + return 1 + +[case testCallableTypes] +from typing import Callable, Any +from m import f + +def absolute_value(x: int) -> int: + return x if x > 0 else -x + +def call_native_function(x: int) -> int: + return absolute_value(x) + +def call_python_function(x: int) -> int: + return f(x) + +def return_float() -> float: + return 5.0 + +def return_callable_type() -> Callable[[], float]: + return return_float + +def call_callable_type() -> float: + f = return_callable_type() + return f() +[file m.py] +def f(x: int) -> int: + return x +[out] +def absolute_value(x): + x :: int + r0 :: bit + r1, r2 :: int +L0: + r0 = int_gt x, 0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = x + goto L3 +L2: + r2 = CPyTagged_Negate(x) + r1 = r2 +L3: + return r1 +def call_native_function(x): + x, r0 :: int +L0: + r0 = absolute_value(x) + return r0 +def call_python_function(x): + x :: int + r0 :: dict + r1 :: str + r2, r3, r4 :: object + r5 :: int +L0: + r0 = __main__.globals :: static + r1 = 'f' + r2 = CPyDict_GetItem(r0, r1) + r3 = box(int, x) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = unbox(int, r4) + return r5 +def return_float(): +L0: + return 5.0 +def return_callable_type(): + r0 :: dict + r1 :: str + r2 :: object +L0: + r0 = __main__.globals :: static + r1 = 'return_float' + r2 = CPyDict_GetItem(r0, r1) + return r2 +def call_callable_type(): + r0, f, r1 :: object + r2 :: float +L0: + r0 = return_callable_type() + f = r0 + r1 = PyObject_CallFunctionObjArgs(f, 0) + r2 = unbox(float, r1) + return r2 + +[case testCallableTypesWithKeywordArgs] +from typing import List + +def call_python_function_with_keyword_arg(x: str) -> int: + return int(x, base=2) + +def call_python_method_with_keyword_args(xs: List[int], first: int, second: int) -> List[int]: + xs.insert(0, x=first) + xs.insert(x=second, i=1) + return xs + +[out] +def call_python_function_with_keyword_arg(x): + x :: str + r0 :: object + r1 :: str + r2 :: tuple + r3 :: object + r4 :: dict + r5 :: object + r6 :: int +L0: + r0 = load_address PyLong_Type + r1 = 'base' + r2 = PyTuple_Pack(1, x) + r3 = object 2 + r4 = CPyDict_Build(1, r1, r3) + r5 = PyObject_Call(r0, r2, r4) + r6 = unbox(int, r5) + return r6 +def call_python_method_with_keyword_args(xs, first, second): + xs :: list + first, second :: int + r0 :: str + r1 :: object + r2 :: str + r3 :: object + r4 :: tuple + r5 :: object + r6 :: dict + r7 :: object + r8 :: str + r9 :: object + r10, r11 :: str + r12 :: tuple + r13, r14 :: object + r15 :: dict + r16 :: object +L0: + r0 = 'insert' + r1 = CPyObject_GetAttr(xs, r0) + r2 = 'x' + r3 = object 0 + r4 = PyTuple_Pack(1, r3) + r5 = box(int, first) + r6 = CPyDict_Build(1, r2, r5) + r7 = PyObject_Call(r1, r4, r6) + r8 = 'insert' + r9 = CPyObject_GetAttr(xs, r8) + r10 = 'x' + r11 = 'i' + r12 = PyTuple_Pack(0) + r13 = box(int, second) + r14 = object 1 + r15 = CPyDict_Build(2, r10, r13, r11, r14) + r16 = PyObject_Call(r9, r12, r15) + return xs + +[case testObjectAsBoolean] +from typing import List + +def obj(x: object) -> int: + if x: + return 1 + else: + return 0 + +def num(x: int) -> int: + if x: + return 1 + else: + return 0 + +def lst(x: List[int]) -> int: + if x: + return 1 + else: + return 0 +[out] +def obj(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: bool +L0: + r0 = PyObject_IsTrue(x) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + if r2 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 0 +L3: + unreachable +def num(x): + x :: int + r0 :: bit +L0: + r0 = x != 0 + if r0 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 0 +L3: + unreachable +def lst(x): + x :: list + r0 :: native_int + r1 :: short_int + r2 :: bit +L0: + r0 = var_object_size x + r1 = r0 << 1 + r2 = int_ne r1, 0 + if r2 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 0 +L3: + unreachable + +[case testOptionalAsBoolean] +from typing import Optional + +class A: pass + +def opt_int(x: Optional[int]) -> int: + if x: + return 1 + else: + return 0 + +def opt_a(x: Optional[A]) -> int: + if x: + return 1 + else: + return 0 + +def opt_o(x: Optional[object]) -> int: + if x: + return 1 + else: + return 0 +[out] +def opt_int(x): + x :: union[int, None] + r0 :: object + r1 :: bit + r2 :: int + r3 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L3 :: bool +L1: + r2 = unbox(int, x) + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + return 2 +L3: + return 0 +L4: + unreachable +def opt_a(x): + x :: union[__main__.A, None] + r0 :: object + r1 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 0 +L3: + unreachable +def opt_o(x): + x :: union[object, None] + r0 :: object + r1 :: bit + r2 :: object + r3 :: i32 + r4 :: bit + r5 :: bool +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L3 :: bool +L1: + r2 = cast(object, x) + r3 = PyObject_IsTrue(r2) + r4 = r3 >= 0 :: signed + r5 = truncate r3: i32 to builtins.bool + if r5 goto L2 else goto L3 :: bool +L2: + return 2 +L3: + return 0 +L4: + unreachable + +[case testRaise] +def foo() -> None: + raise Exception() + +def bar() -> None: + raise Exception +[out] +def foo(): + r0 :: object + r1 :: str + r2, r3 :: object +L0: + r0 = builtins :: module + r1 = 'Exception' + r2 = CPyObject_GetAttr(r0, r1) + r3 = PyObject_CallFunctionObjArgs(r2, 0) + CPy_Raise(r3) + unreachable +def bar(): + r0 :: object + r1 :: str + r2 :: object +L0: + r0 = builtins :: module + r1 = 'Exception' + r2 = CPyObject_GetAttr(r0, r1) + CPy_Raise(r2) + unreachable + +[case testModuleTopLevel_toplevel] +x = 1 +print(x) + +def f() -> None: + print(x) +[out] +def f(): + r0 :: dict + r1 :: str + r2 :: object + r3 :: int + r4 :: object + r5 :: str + r6, r7, r8 :: object +L0: + r0 = __main__.globals :: static + r1 = 'x' + r2 = CPyDict_GetItem(r0, r1) + r3 = unbox(int, r2) + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = box(int, r3) + r8 = PyObject_CallFunctionObjArgs(r6, r7, 0) + return 1 +def __top_level__(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4 :: object + r5 :: dict + r6 :: str + r7 :: object + r8 :: i32 + r9 :: bit + r10 :: dict + r11 :: str + r12 :: object + r13 :: int + r14 :: object + r15 :: str + r16, r17, r18 :: object +L0: + r0 = builtins :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = 'builtins' + r4 = PyImport_Import(r3) + builtins = r4 :: module +L2: + r5 = __main__.globals :: static + r6 = 'x' + r7 = object 1 + r8 = CPyDict_SetItem(r5, r6, r7) + r9 = r8 >= 0 :: signed + r10 = __main__.globals :: static + r11 = 'x' + r12 = CPyDict_GetItem(r10, r11) + r13 = unbox(int, r12) + r14 = builtins :: module + r15 = 'print' + r16 = CPyObject_GetAttr(r14, r15) + r17 = box(int, r13) + r18 = PyObject_CallFunctionObjArgs(r16, r17, 0) + return 1 + +[case testCallOverloaded] +import m +def f() -> str: + return m.f(1) +[file m.pyi] +from typing import overload +@overload +def f(x: int) -> str: ... +@overload +def f(x: str) -> int: ... +[out] +def f(): + r0 :: object + r1 :: str + r2, r3, r4 :: object + r5 :: str +L0: + r0 = m :: module + r1 = 'f' + r2 = CPyObject_GetAttr(r0, r1) + r3 = object 1 + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = cast(str, r4) + return r5 + +[case testCallOverloadedNative] +from typing import overload, Union + +@overload +def foo(x: int) -> int: ... + +@overload +def foo(x: str) -> str: ... + +def foo(x: Union[int, str]) -> Union[int, str]: + return x + +def main() -> None: + x = foo(0) +[out] +def foo(x): + x :: union[int, str] +L0: + return x +def main(): + r0 :: object + r1 :: union[int, str] + r2, x :: int +L0: + r0 = object 0 + r1 = foo(r0) + r2 = unbox(int, r1) + x = r2 + return 1 + +[case testCallOverloadedNativeSubclass] +from typing import overload, Union + +class A: + x: int +class B(A): + y: int + +@overload +def foo(x: int) -> B: ... + +@overload +def foo(x: Union[int, str]) -> A: ... + +def foo(x: Union[int, str]) -> A: + if isinstance(x, int): + return B() + return A() + +def main() -> None: + x = foo(0) +[out] +def foo(x): + x :: union[int, str] + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: __main__.B + r5 :: __main__.A +L0: + r0 = load_address PyLong_Type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool +L1: + r4 = B() + return r4 +L2: + r5 = A() + return r5 +def main(): + r0 :: object + r1 :: __main__.A + r2, x :: __main__.B +L0: + r0 = object 0 + r1 = foo(r0) + r2 = cast(__main__.B, r1) + x = r2 + return 1 + +[case testFunctionCallWithKeywordArgs] +def f(x: int, y: str) -> None: pass + +def g() -> None: + f(y='a', x=0) + f(1, y='b') +[out] +def f(x, y): + x :: int + y :: str +L0: + return 1 +def g(): + r0 :: str + r1 :: None + r2 :: str + r3 :: None +L0: + r0 = 'a' + r1 = f(0, r0) + r2 = 'b' + r3 = f(2, r2) + return 1 + +[case testMethodCallWithKeywordArgs] +class A: + def f(self, x: int, y: str) -> None: pass + +def g(a: A) -> None: + a.f(y='a', x=0) + a.f(1, y='b') +[out] +def A.f(self, x, y): + self :: __main__.A + x :: int + y :: str +L0: + return 1 +def g(a): + a :: __main__.A + r0 :: str + r1 :: None + r2 :: str + r3 :: None +L0: + r0 = 'a' + r1 = a.f(0, r0) + r2 = 'b' + r3 = a.f(2, r2) + return 1 + +[case testStarArgs] +from typing import Tuple +def f(a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c +def g() -> Tuple[int, int, int]: + return f(*(1, 2, 3)) +def h() -> Tuple[int, int, int]: + return f(1, *(2, 3)) +[out] +def f(a, b, c): + a, b, c :: int + r0 :: tuple[int, int, int] +L0: + r0 = (a, b, c) + return r0 +def g(): + r0 :: tuple[int, int, int] + r1 :: dict + r2 :: str + r3 :: object + r4 :: list + r5, r6 :: object + r7 :: tuple + r8 :: dict + r9 :: object + r10 :: tuple[int, int, int] +L0: + r0 = (2, 4, 6) + r1 = __main__.globals :: static + r2 = 'f' + r3 = CPyDict_GetItem(r1, r2) + r4 = PyList_New(0) + r5 = box(tuple[int, int, int], r0) + r6 = CPyList_Extend(r4, r5) + r7 = PyList_AsTuple(r4) + r8 = PyDict_New() + r9 = PyObject_Call(r3, r7, r8) + r10 = unbox(tuple[int, int, int], r9) + return r10 +def h(): + r0 :: tuple[int, int] + r1 :: dict + r2 :: str + r3 :: object + r4 :: list + r5 :: object + r6 :: ptr + r7, r8 :: object + r9 :: tuple + r10 :: dict + r11 :: object + r12 :: tuple[int, int, int] +L0: + r0 = (4, 6) + r1 = __main__.globals :: static + r2 = 'f' + r3 = CPyDict_GetItem(r1, r2) + r4 = PyList_New(1) + r5 = object 1 + r6 = list_items r4 + buf_init_item r6, 0, r5 + keep_alive r4 + r7 = box(tuple[int, int], r0) + r8 = CPyList_Extend(r4, r7) + r9 = PyList_AsTuple(r4) + r10 = PyDict_New() + r11 = PyObject_Call(r3, r9, r10) + r12 = unbox(tuple[int, int, int], r11) + return r12 + +[case testStar2Args] +from typing import Tuple +def f(a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c +def g() -> Tuple[int, int, int]: + return f(**{'a': 1, 'b': 2, 'c': 3}) +def h() -> Tuple[int, int, int]: + return f(1, **{'b': 2, 'c': 3}) +[out] +def f(a, b, c): + a, b, c :: int + r0 :: tuple[int, int, int] +L0: + r0 = (a, b, c) + return r0 +def g(): + r0, r1, r2 :: str + r3, r4, r5 :: object + r6, r7 :: dict + r8 :: str + r9 :: object + r10 :: dict + r11 :: i32 + r12 :: bit + r13 :: tuple + r14 :: object + r15 :: tuple[int, int, int] +L0: + r0 = 'a' + r1 = 'b' + r2 = 'c' + r3 = object 1 + r4 = object 2 + r5 = object 3 + r6 = CPyDict_Build(3, r0, r3, r1, r4, r2, r5) + r7 = __main__.globals :: static + r8 = 'f' + r9 = CPyDict_GetItem(r7, r8) + r10 = PyDict_New() + r11 = CPyDict_UpdateInDisplay(r10, r6) + r12 = r11 >= 0 :: signed + r13 = PyTuple_Pack(0) + r14 = PyObject_Call(r9, r13, r10) + r15 = unbox(tuple[int, int, int], r14) + return r15 +def h(): + r0, r1 :: str + r2, r3 :: object + r4, r5 :: dict + r6 :: str + r7 :: object + r8 :: dict + r9 :: i32 + r10 :: bit + r11 :: object + r12 :: tuple + r13 :: object + r14 :: tuple[int, int, int] +L0: + r0 = 'b' + r1 = 'c' + r2 = object 2 + r3 = object 3 + r4 = CPyDict_Build(2, r0, r2, r1, r3) + r5 = __main__.globals :: static + r6 = 'f' + r7 = CPyDict_GetItem(r5, r6) + r8 = PyDict_New() + r9 = CPyDict_UpdateInDisplay(r8, r4) + r10 = r9 >= 0 :: signed + r11 = object 1 + r12 = PyTuple_Pack(1, r11) + r13 = PyObject_Call(r7, r12, r8) + r14 = unbox(tuple[int, int, int], r13) + return r14 + +[case testFunctionCallWithDefaultArgs] +def f(x: int, y: int = 3, z: str = "test") -> None: + return None + +def g() -> None: + f(2) + f(y = 3, x = 6) +[out] +def f(x, y, z): + x, y :: int + z, r0 :: str +L0: + if is_error(y) goto L1 else goto L2 +L1: + y = 6 +L2: + if is_error(z) goto L3 else goto L4 +L3: + r0 = 'test' + z = r0 +L4: + return 1 +def g(): + r0 :: int + r1 :: str + r2 :: None + r3 :: str + r4 :: None +L0: + r0 = :: int + r1 = :: str + r2 = f(4, r0, r1) + r3 = :: str + r4 = f(12, 6, r3) + return 1 + +[case testMethodCallWithDefaultArgs] +class A: + def f(self, x: int, y: int = 3, z: str = "test") -> None: + return None + +def g() -> None: + a = A() + a.f(2) + a.f(y = 3, x = 6) +[out] +def A.f(self, x, y, z): + self :: __main__.A + x, y :: int + z, r0 :: str +L0: + if is_error(y) goto L1 else goto L2 +L1: + y = 6 +L2: + if is_error(z) goto L3 else goto L4 +L3: + r0 = 'test' + z = r0 +L4: + return 1 +def g(): + r0, a :: __main__.A + r1 :: int + r2 :: str + r3 :: None + r4 :: str + r5 :: None +L0: + r0 = A() + a = r0 + r1 = :: int + r2 = :: str + r3 = a.f(4, r1, r2) + r4 = :: str + r5 = a.f(12, 6, r4) + return 1 + +[case testListComprehension] +from typing import List + +def f() -> List[int]: + return [x*x for x in [1,2,3] if x != 2 if x != 3] +[out] +def f(): + r0, r1 :: list + r2, r3, r4 :: object + r5 :: ptr + r6 :: short_int + r7 :: native_int + r8 :: short_int + r9 :: bit + r10 :: object + r11, x :: int + r12, r13 :: bit + r14 :: int + r15 :: object + r16 :: i32 + r17 :: bit + r18 :: short_int +L0: + r0 = PyList_New(0) + r1 = PyList_New(3) + r2 = object 1 + r3 = object 2 + r4 = object 3 + r5 = list_items r1 + buf_init_item r5, 0, r2 + buf_init_item r5, 1, r3 + buf_init_item r5, 2, r4 + keep_alive r1 + r6 = 0 +L1: + r7 = var_object_size r1 + r8 = r7 << 1 + r9 = int_lt r6, r8 + if r9 goto L2 else goto L8 :: bool +L2: + r10 = CPyList_GetItemUnsafe(r1, r6) + r11 = unbox(int, r10) + x = r11 + r12 = int_ne x, 4 + if r12 goto L4 else goto L3 :: bool +L3: + goto L7 +L4: + r13 = int_ne x, 6 + if r13 goto L6 else goto L5 :: bool +L5: + goto L7 +L6: + r14 = CPyTagged_Multiply(x, x) + r15 = box(int, r14) + r16 = PyList_Append(r0, r15) + r17 = r16 >= 0 :: signed +L7: + r18 = r6 + 2 + r6 = r18 + goto L1 +L8: + return r0 + +[case testDictComprehension] +from typing import Dict +def f() -> Dict[int, int]: + return {x: x*x for x in [1,2,3] if x != 2 if x != 3} +[out] +def f(): + r0 :: dict + r1 :: list + r2, r3, r4 :: object + r5 :: ptr + r6 :: short_int + r7 :: native_int + r8 :: short_int + r9 :: bit + r10 :: object + r11, x :: int + r12, r13 :: bit + r14 :: int + r15, r16 :: object + r17 :: i32 + r18 :: bit + r19 :: short_int +L0: + r0 = PyDict_New() + r1 = PyList_New(3) + r2 = object 1 + r3 = object 2 + r4 = object 3 + r5 = list_items r1 + buf_init_item r5, 0, r2 + buf_init_item r5, 1, r3 + buf_init_item r5, 2, r4 + keep_alive r1 + r6 = 0 +L1: + r7 = var_object_size r1 + r8 = r7 << 1 + r9 = int_lt r6, r8 + if r9 goto L2 else goto L8 :: bool +L2: + r10 = CPyList_GetItemUnsafe(r1, r6) + r11 = unbox(int, r10) + x = r11 + r12 = int_ne x, 4 + if r12 goto L4 else goto L3 :: bool +L3: + goto L7 +L4: + r13 = int_ne x, 6 + if r13 goto L6 else goto L5 :: bool +L5: + goto L7 +L6: + r14 = CPyTagged_Multiply(x, x) + r15 = box(int, x) + r16 = box(int, r14) + r17 = CPyDict_SetItem(r0, r15, r16) + r18 = r17 >= 0 :: signed +L7: + r19 = r6 + 2 + r6 = r19 + goto L1 +L8: + return r0 + +[case testLoopsMultipleAssign] +from typing import List, Tuple +def f(l: List[Tuple[int, int, int]]) -> List[int]: + for x, y, z in l: + pass + return [x+y+z for x, y, z in l] +[out] +def f(l): + l :: list + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: bit + r4 :: object + r5 :: tuple[int, int, int] + r6, x, r7, y, r8, z :: int + r9 :: short_int + r10 :: native_int + r11 :: list + r12 :: short_int + r13 :: native_int + r14 :: short_int + r15 :: bit + r16 :: object + r17 :: tuple[int, int, int] + r18, x_2, r19, y_2, r20, z_2, r21, r22 :: int + r23 :: object + r24 :: bit + r25 :: short_int +L0: + r0 = 0 +L1: + r1 = var_object_size l + r2 = r1 << 1 + r3 = int_lt r0, r2 + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyList_GetItemUnsafe(l, r0) + r5 = unbox(tuple[int, int, int], r4) + r6 = r5[0] + x = r6 + r7 = r5[1] + y = r7 + r8 = r5[2] + z = r8 +L3: + r9 = r0 + 2 + r0 = r9 + goto L1 +L4: + r10 = var_object_size l + r11 = PyList_New(r10) + r12 = 0 +L5: + r13 = var_object_size l + r14 = r13 << 1 + r15 = int_lt r12, r14 + if r15 goto L6 else goto L8 :: bool +L6: + r16 = CPyList_GetItemUnsafe(l, r12) + r17 = unbox(tuple[int, int, int], r16) + r18 = r17[0] + x_2 = r18 + r19 = r17[1] + y_2 = r19 + r20 = r17[2] + z_2 = r20 + r21 = CPyTagged_Add(x_2, y_2) + r22 = CPyTagged_Add(r21, z_2) + r23 = box(int, r22) + r24 = CPyList_SetItemUnsafe(r11, r12, r23) +L7: + r25 = r12 + 2 + r12 = r25 + goto L5 +L8: + return r11 + +[case testProperty] +class PropertyHolder: + @property + def value(self) -> int: + return self.left + self.right if self.is_add else self.left - self.right + def __init__(self, left: int, right: int, is_add: bool) -> None: + self.left = left + self.right = right + self.is_add = is_add + def twice_value(self) -> int: + return 2 * self.value +[out] +def PropertyHolder.value(self): + self :: __main__.PropertyHolder + r0 :: bool + r1, r2, r3, r4, r5, r6, r7 :: int +L0: + r0 = self.is_add + if r0 goto L1 else goto L2 :: bool +L1: + r1 = borrow self.left + r2 = borrow self.right + r3 = CPyTagged_Add(r1, r2) + keep_alive self, self + r4 = r3 + goto L3 +L2: + r5 = borrow self.left + r6 = borrow self.right + r7 = CPyTagged_Subtract(r5, r6) + keep_alive self, self + r4 = r7 +L3: + return r4 +def PropertyHolder.__init__(self, left, right, is_add): + self :: __main__.PropertyHolder + left, right :: int + is_add :: bool +L0: + self.left = left + self.right = right + self.is_add = is_add + return 1 +def PropertyHolder.twice_value(self): + self :: __main__.PropertyHolder + r0, r1 :: int +L0: + r0 = self.value + r1 = CPyTagged_Multiply(4, r0) + return r1 + +[case testNativeIndex] +from typing import List +class A: + def __getitem__(self, index: int) -> int: pass + +def g(a: A, b: List[int], c: int) -> int: + return a[c] + b[c] +[out] +def A.__getitem__(self, index): + self :: __main__.A + index :: int +L0: + unreachable +def g(a, b, c): + a :: __main__.A + b :: list + c, r0 :: int + r1 :: object + r2, r3 :: int +L0: + r0 = a.__getitem__(c) + r1 = CPyList_GetItemBorrow(b, c) + r2 = unbox(int, r1) + r3 = CPyTagged_Add(r0, r2) + keep_alive b, c + return r3 + +[case testTypeAlias_toplevel] +from typing import List, NewType, NamedTuple +Lol = NamedTuple('Lol', (('a', int), ('b', str))) +x = Lol(1, '') +Foo = List[int] +Bar = NewType('Bar', Foo) +y = Bar([1,2,3]) +[out] +def __top_level__(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4, r5 :: object + r6 :: str + r7 :: dict + r8 :: object + r9, r10 :: str + r11 :: object + r12 :: tuple[str, object] + r13 :: object + r14 :: str + r15 :: object + r16 :: tuple[str, object] + r17 :: object + r18 :: tuple[object, object] + r19 :: object + r20 :: dict + r21 :: str + r22, r23 :: object + r24 :: dict + r25 :: str + r26 :: i32 + r27 :: bit + r28 :: str + r29 :: dict + r30 :: str + r31, r32, r33 :: object + r34 :: tuple + r35 :: dict + r36 :: str + r37 :: i32 + r38 :: bit + r39 :: dict + r40 :: str + r41, r42, r43 :: object + r44 :: dict + r45 :: str + r46 :: i32 + r47 :: bit + r48 :: str + r49 :: dict + r50 :: str + r51 :: object + r52 :: dict + r53 :: str + r54, r55 :: object + r56 :: dict + r57 :: str + r58 :: i32 + r59 :: bit + r60 :: list + r61, r62, r63 :: object + r64 :: ptr + r65 :: dict + r66 :: str + r67 :: i32 + r68 :: bit +L0: + r0 = builtins :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = 'builtins' + r4 = PyImport_Import(r3) + builtins = r4 :: module +L2: + r5 = ('List', 'NewType', 'NamedTuple') + r6 = 'typing' + r7 = __main__.globals :: static + r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) + typing = r8 :: module + r9 = 'Lol' + r10 = 'a' + r11 = load_address PyLong_Type + r12 = (r10, r11) + r13 = box(tuple[str, object], r12) + r14 = 'b' + r15 = load_address PyUnicode_Type + r16 = (r14, r15) + r17 = box(tuple[str, object], r16) + r18 = (r13, r17) + r19 = box(tuple[object, object], r18) + r20 = __main__.globals :: static + r21 = 'NamedTuple' + r22 = CPyDict_GetItem(r20, r21) + r23 = PyObject_CallFunctionObjArgs(r22, r9, r19, 0) + r24 = __main__.globals :: static + r25 = 'Lol' + r26 = CPyDict_SetItem(r24, r25, r23) + r27 = r26 >= 0 :: signed + r28 = '' + r29 = __main__.globals :: static + r30 = 'Lol' + r31 = CPyDict_GetItem(r29, r30) + r32 = object 1 + r33 = PyObject_CallFunctionObjArgs(r31, r32, r28, 0) + r34 = cast(tuple, r33) + r35 = __main__.globals :: static + r36 = 'x' + r37 = CPyDict_SetItem(r35, r36, r34) + r38 = r37 >= 0 :: signed + r39 = __main__.globals :: static + r40 = 'List' + r41 = CPyDict_GetItem(r39, r40) + r42 = load_address PyLong_Type + r43 = PyObject_GetItem(r41, r42) + r44 = __main__.globals :: static + r45 = 'Foo' + r46 = CPyDict_SetItem(r44, r45, r43) + r47 = r46 >= 0 :: signed + r48 = 'Bar' + r49 = __main__.globals :: static + r50 = 'Foo' + r51 = CPyDict_GetItem(r49, r50) + r52 = __main__.globals :: static + r53 = 'NewType' + r54 = CPyDict_GetItem(r52, r53) + r55 = PyObject_CallFunctionObjArgs(r54, r48, r51, 0) + r56 = __main__.globals :: static + r57 = 'Bar' + r58 = CPyDict_SetItem(r56, r57, r55) + r59 = r58 >= 0 :: signed + r60 = PyList_New(3) + r61 = object 1 + r62 = object 2 + r63 = object 3 + r64 = list_items r60 + buf_init_item r64, 0, r61 + buf_init_item r64, 1, r62 + buf_init_item r64, 2, r63 + keep_alive r60 + r65 = __main__.globals :: static + r66 = 'y' + r67 = CPyDict_SetItem(r65, r66, r60) + r68 = r67 >= 0 :: signed + return 1 + +[case testChainedConditional] +def g(x: int) -> int: + return x +def f(x: int, y: int, z: int) -> bool: + return g(x) < g(y) > g(z) +[out] +def g(x): + x :: int +L0: + return x +def f(x, y, z): + x, y, z, r0, r1 :: int + r2 :: bit + r3 :: bool + r4 :: int + r5 :: bit +L0: + r0 = g(x) + r1 = g(y) + r2 = int_lt r0, r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = r2 + goto L3 +L2: + r4 = g(z) + r5 = int_gt r1, r4 + r3 = r5 +L3: + return r3 + +[case testEq] +class A: + def __eq__(self, x: object) -> bool: + return NotImplemented +[out] +def A.__eq__(self, x): + self :: __main__.A + x, r0 :: object +L0: + r0 = load_address _Py_NotImplementedStruct + return r0 +def A.__ne__(__mypyc_self__, rhs): + __mypyc_self__ :: __main__.A + rhs, r0, r1 :: object + r2 :: bit + r3 :: i32 + r4 :: bit + r5 :: bool + r6 :: object +L0: + r0 = __mypyc_self__.__eq__(rhs) + r1 = load_address _Py_NotImplementedStruct + r2 = r0 == r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = PyObject_Not(r0) + r4 = r3 >= 0 :: signed + r5 = truncate r3: i32 to builtins.bool + r6 = box(bool, r5) + return r6 +L2: + return r1 + +[case testDecorators_toplevel] +from typing import Callable + +def a(f: Callable[[], None]) -> Callable[[], None]: + def g() -> None: + print('Entering') + f() + print('Exited') + return g + +def b(f: Callable[[], None]) -> Callable[[], None]: + def g() -> None: + print('---') + f() + print('---') + return g + +@a +@b +def c() -> None: + @a + @b + def d() -> None: + print('d') + print('c') + d() + +[out] +def g_a_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def g_a_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.g_a_obj + r0 :: __main__.a_env + r1 :: str + r2 :: object + r3 :: str + r4, r5, r6, r7 :: object + r8 :: str + r9 :: object + r10 :: str + r11, r12 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = 'Entering' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = PyObject_CallFunctionObjArgs(r4, r1, 0) + r6 = r0.f + r7 = PyObject_CallFunctionObjArgs(r6, 0) + r8 = 'Exited' + r9 = builtins :: module + r10 = 'print' + r11 = CPyObject_GetAttr(r9, r10) + r12 = PyObject_CallFunctionObjArgs(r11, r8, 0) + return 1 +def a(f): + f :: object + r0 :: __main__.a_env + r1 :: bool + r2 :: __main__.g_a_obj + r3 :: bool + g :: object +L0: + r0 = a_env() + r0.f = f; r1 = is_error + r2 = g_a_obj() + r2.__mypyc_env__ = r0; r3 = is_error + g = r2 + return g +def g_b_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def g_b_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.g_b_obj + r0 :: __main__.b_env + r1 :: str + r2 :: object + r3 :: str + r4, r5, r6, r7 :: object + r8 :: str + r9 :: object + r10 :: str + r11, r12 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = '---' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = PyObject_CallFunctionObjArgs(r4, r1, 0) + r6 = r0.f + r7 = PyObject_CallFunctionObjArgs(r6, 0) + r8 = '---' + r9 = builtins :: module + r10 = 'print' + r11 = CPyObject_GetAttr(r9, r10) + r12 = PyObject_CallFunctionObjArgs(r11, r8, 0) + return 1 +def b(f): + f :: object + r0 :: __main__.b_env + r1 :: bool + r2 :: __main__.g_b_obj + r3 :: bool + g :: object +L0: + r0 = b_env() + r0.f = f; r1 = is_error + r2 = g_b_obj() + r2.__mypyc_env__ = r0; r3 = is_error + g = r2 + return g +def d_c_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def d_c_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.d_c_obj + r0 :: __main__.c_env + r1 :: str + r2 :: object + r3 :: str + r4, r5 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = 'd' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = PyObject_CallFunctionObjArgs(r4, r1, 0) + return 1 +def c(): + r0 :: __main__.c_env + r1 :: __main__.d_c_obj + r2 :: bool + r3 :: dict + r4 :: str + r5, r6 :: object + r7 :: dict + r8 :: str + r9, r10, d :: object + r11 :: dict + r12 :: str + r13 :: i32 + r14 :: bit + r15 :: str + r16 :: object + r17 :: str + r18, r19, r20 :: object +L0: + r0 = c_env() + r1 = d_c_obj() + r1.__mypyc_env__ = r0; r2 = is_error + r3 = __main__.globals :: static + r4 = 'b' + r5 = CPyDict_GetItem(r3, r4) + r6 = PyObject_CallFunctionObjArgs(r5, r1, 0) + r7 = __main__.globals :: static + r8 = 'a' + r9 = CPyDict_GetItem(r7, r8) + r10 = PyObject_CallFunctionObjArgs(r9, r6, 0) + d = r10 + r11 = __main__.globals :: static + r12 = 'd' + r13 = CPyDict_SetItem(r11, r12, r10) + r14 = r13 >= 0 :: signed + r15 = 'c' + r16 = builtins :: module + r17 = 'print' + r18 = CPyObject_GetAttr(r16, r17) + r19 = PyObject_CallFunctionObjArgs(r18, r15, 0) + r20 = PyObject_CallFunctionObjArgs(d, 0) + return 1 +def __top_level__(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4, r5 :: object + r6 :: str + r7 :: dict + r8 :: object + r9 :: dict + r10 :: str + r11 :: object + r12 :: dict + r13 :: str + r14, r15 :: object + r16 :: dict + r17 :: str + r18, r19 :: object + r20 :: dict + r21 :: str + r22 :: i32 + r23 :: bit +L0: + r0 = builtins :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = 'builtins' + r4 = PyImport_Import(r3) + builtins = r4 :: module +L2: + r5 = ('Callable',) + r6 = 'typing' + r7 = __main__.globals :: static + r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) + typing = r8 :: module + r9 = __main__.globals :: static + r10 = 'c' + r11 = CPyDict_GetItem(r9, r10) + r12 = __main__.globals :: static + r13 = 'b' + r14 = CPyDict_GetItem(r12, r13) + r15 = PyObject_CallFunctionObjArgs(r14, r11, 0) + r16 = __main__.globals :: static + r17 = 'a' + r18 = CPyDict_GetItem(r16, r17) + r19 = PyObject_CallFunctionObjArgs(r18, r15, 0) + r20 = __main__.globals :: static + r21 = 'c' + r22 = CPyDict_SetItem(r20, r21, r19) + r23 = r22 >= 0 :: signed + return 1 + +[case testDecoratorsSimple_toplevel] +from typing import Callable + +def a(f: Callable[[], None]) -> Callable[[], None]: + def g() -> None: + print('Entering') + f() + print('Exited') + return g + +[out] +def g_a_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def g_a_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.g_a_obj + r0 :: __main__.a_env + r1 :: str + r2 :: object + r3 :: str + r4, r5, r6, r7 :: object + r8 :: str + r9 :: object + r10 :: str + r11, r12 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = 'Entering' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = PyObject_CallFunctionObjArgs(r4, r1, 0) + r6 = r0.f + r7 = PyObject_CallFunctionObjArgs(r6, 0) + r8 = 'Exited' + r9 = builtins :: module + r10 = 'print' + r11 = CPyObject_GetAttr(r9, r10) + r12 = PyObject_CallFunctionObjArgs(r11, r8, 0) + return 1 +def a(f): + f :: object + r0 :: __main__.a_env + r1 :: bool + r2 :: __main__.g_a_obj + r3 :: bool + g :: object +L0: + r0 = a_env() + r0.f = f; r1 = is_error + r2 = g_a_obj() + r2.__mypyc_env__ = r0; r3 = is_error + g = r2 + return g +def __top_level__(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4, r5 :: object + r6 :: str + r7 :: dict + r8 :: object +L0: + r0 = builtins :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = 'builtins' + r4 = PyImport_Import(r3) + builtins = r4 :: module +L2: + r5 = ('Callable',) + r6 = 'typing' + r7 = __main__.globals :: static + r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) + typing = r8 :: module + return 1 + +[case testAnyAllG] +from typing import Iterable + +def call_any(l: Iterable[int]) -> bool: + return any(i == 0 for i in l) + +def call_all(l: Iterable[int]) -> bool: + return all(i == 0 for i in l) + +[out] +def call_any(l): + l :: object + r0 :: bool + r1, r2 :: object + r3, i :: int + r4, r5 :: bit +L0: + r0 = 0 + r1 = PyObject_GetIter(l) +L1: + r2 = PyIter_Next(r1) + if is_error(r2) goto L6 else goto L2 +L2: + r3 = unbox(int, r2) + i = r3 + r4 = int_eq i, 0 + if r4 goto L3 else goto L4 :: bool +L3: + r0 = 1 + goto L8 +L4: +L5: + goto L1 +L6: + r5 = CPy_NoErrOccured() +L7: +L8: + return r0 +def call_all(l): + l :: object + r0 :: bool + r1, r2 :: object + r3, i :: int + r4, r5, r6 :: bit +L0: + r0 = 1 + r1 = PyObject_GetIter(l) +L1: + r2 = PyIter_Next(r1) + if is_error(r2) goto L6 else goto L2 +L2: + r3 = unbox(int, r2) + i = r3 + r4 = int_eq i, 0 + r5 = r4 ^ 1 + if r5 goto L3 else goto L4 :: bool +L3: + r0 = 0 + goto L8 +L4: +L5: + goto L1 +L6: + r6 = CPy_NoErrOccured() +L7: +L8: + return r0 + +[case testSum] +from typing import Callable, Iterable + +def call_sum(l: Iterable[int], comparison: Callable[[int], bool]) -> int: + return sum(comparison(x) for x in l) + +[out] +def call_sum(l, comparison): + l, comparison :: object + r0 :: int + r1, r2 :: object + r3, x :: int + r4, r5 :: object + r6, r7 :: bool + r8, r9 :: int + r10 :: bit +L0: + r0 = 0 + r1 = PyObject_GetIter(l) +L1: + r2 = PyIter_Next(r1) + if is_error(r2) goto L4 else goto L2 +L2: + r3 = unbox(int, r2) + x = r3 + r4 = box(int, x) + r5 = PyObject_CallFunctionObjArgs(comparison, r4, 0) + r6 = unbox(bool, r5) + r7 = r6 << 1 + r8 = extend r7: builtins.bool to builtins.int + r9 = CPyTagged_Add(r0, r8) + r0 = r9 +L3: + goto L1 +L4: + r10 = CPy_NoErrOccured() +L5: + return r0 + +[case testSetAttr1] +from typing import Any, Dict, List +def lol(x: Any): + setattr(x, 'x', '5') + +[out] +def lol(x): + x :: object + r0, r1 :: str + r2 :: i32 + r3 :: bit + r4 :: object +L0: + r0 = 'x' + r1 = '5' + r2 = PyObject_SetAttr(x, r0, r1) + r3 = r2 >= 0 :: signed + r4 = box(None, 1) + return r4 + +[case testFinalModuleInt] +from typing import Final + +x: Final = 1 +y: Final = 2 + +def f(a: bool) -> int: + if a: + return x + else: + return y +[out] +def f(a): + a :: bool +L0: + if a goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 +L3: + unreachable + +[case testFinalModuleStr] +from typing import Final + +x: Final = 'x' +y: Final = 'y' + +def f(a: bool) -> str: + if a: + return x + else: + return y +[out] +def f(a): + a :: bool + r0, r1 :: str +L0: + if a goto L1 else goto L2 :: bool +L1: + r0 = 'x' + return r0 +L2: + r1 = 'y' + return r1 +L3: + unreachable + +[case testFinalModuleBool] +from typing import Final + +x: Final = True +y: Final = False + +def f(a: bool) -> bool: + if a: + return x + else: + return y +[out] +def f(a): + a :: bool +L0: + if a goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +L3: + unreachable + +[case testFinalClass] +from typing import Final + +class C: + x: Final = 1 + y: Final = 2 + +def f(a: bool) -> int: + if a: + return C.x + else: + return C.y +[out] +def C.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.C +L0: + __mypyc_self__.x = 2 + __mypyc_self__.y = 4 + return 1 +def f(a): + a :: bool +L0: + if a goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 +L3: + unreachable + +[case testFinalStaticList] +from typing import Final + +x: Final = [1] + +def f() -> int: + return x[0] +[out] +def f(): + r0 :: list + r1 :: bool + r2 :: object + r3 :: int +L0: + r0 = __main__.x :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "x" was not set') + unreachable +L2: + r2 = CPyList_GetItemShort(r0, 0) + r3 = unbox(int, r2) + return r3 + +[case testFinalStaticTuple] +from typing import Final + +x: Final = (1, 2) + +def f() -> int: + return x[0] +[out] +def f(): + r0 :: tuple[int, int] + r1 :: bool + r2 :: int +L0: + r0 = __main__.x :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "x" was not set') + unreachable +L2: + r2 = r0[0] + return r2 + +[case testFinalStaticInt] +from typing import Final + +x: Final = 1 + int() + +def f() -> int: + return x - 1 +[out] +def f(): + r0 :: int + r1 :: bool + r2 :: int +L0: + r0 = __main__.x :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "x" was not set') + unreachable +L2: + r2 = CPyTagged_Subtract(r0, 2) + return r2 + +[case testFinalRestrictedTypeVar] +from typing import TypeVar +if False: + from typing import Final + +FOO = 10 # type: Final + +Targ = TypeVar('Targ', int, str) +def foo(z: Targ) -> None: + FOO +[out] +def foo(z): + z :: object +L0: + return 1 + +[case testFinalLocals] +from typing import Final + +def inlined() -> str: + # XXX: the final type must be declared explicitly for Var.final_value to be set. + const: Final[str] = "Oppenheimer" + return const + +def local() -> str: + const: Final[str] = inlined() + return const +[out] +def inlined(): + r0, const, r1 :: str +L0: + r0 = 'Oppenheimer' + const = r0 + r1 = 'Oppenheimer' + return r1 +def local(): + r0, const :: str +L0: + r0 = inlined() + const = r0 + return const + +[case testDirectlyCall__bool__] +class A: + def __bool__(self) -> bool: + return True +class B(A): + def __bool__(self) -> bool: + return False + +def lol(x: A) -> int: + if x: + return 1 + else: + return 0 + +[out] +def A.__bool__(self): + self :: __main__.A +L0: + return 1 +def B.__bool__(self): + self :: __main__.B +L0: + return 0 +def lol(x): + x :: __main__.A + r0 :: bool +L0: + r0 = x.__bool__() + if r0 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 0 +L3: + unreachable + +[case testRevealType] +def f(x: int) -> None: + reveal_type(x) # type: ignore +[out] +def f(x): + x :: int + r0 :: object + r1 :: str + r2, r3, r4 :: object +L0: + r0 = builtins :: module + r1 = 'reveal_type' + r2 = CPyObject_GetAttr(r0, r1) + r3 = box(int, x) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + return 1 + +[case testCallCWithStrJoinMethod] +from typing import List +def f(x: str, y: List[str]) -> str: + return x.join(y) +[out] +def f(x, y): + x :: str + y :: list + r0 :: str +L0: + r0 = PyUnicode_Join(x, y) + return r0 + +[case testCallCWithToListFunction] +from typing import List, Iterable, Tuple, Dict +# generic object +def f(x: Iterable[int]) -> List[int]: + return list(x) + +# need coercing +def g(x: Tuple[int, int, int]) -> List[int]: + return list(x) + +# non-list object +def h(x: Dict[int, str]) -> List[int]: + return list(x) + +[out] +def f(x): + x :: object + r0 :: list +L0: + r0 = PySequence_List(x) + return r0 +def g(x): + x :: tuple[int, int, int] + r0 :: object + r1 :: list +L0: + r0 = box(tuple[int, int, int], x) + r1 = PySequence_List(r0) + return r1 +def h(x): + x :: dict + r0 :: list +L0: + r0 = PySequence_List(x) + return r0 + +[case testBoolFunction] +def f(x: object) -> bool: + return bool(x) +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: bool +L0: + r0 = PyObject_IsTrue(x) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + return r2 + +[case testLocalImports] +def root() -> None: + import dataclasses + import enum + +def submodule() -> int: + import p.m + return p.x +[file p/__init__.py] +x = 1 +[file p/m.py] +[out] +def root(): + r0 :: dict + r1, r2 :: object + r3 :: bit + r4 :: str + r5 :: object + r6 :: str + r7 :: dict + r8 :: str + r9 :: object + r10 :: i32 + r11 :: bit + r12 :: dict + r13, r14 :: object + r15 :: bit + r16 :: str + r17 :: object + r18 :: str + r19 :: dict + r20 :: str + r21 :: object + r22 :: i32 + r23 :: bit +L0: + r0 = __main__.globals :: static + r1 = dataclasses :: module + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool +L1: + r4 = 'dataclasses' + r5 = PyImport_Import(r4) + dataclasses = r5 :: module +L2: + r6 = 'dataclasses' + r7 = PyImport_GetModuleDict() + r8 = 'dataclasses' + r9 = CPyDict_GetItem(r7, r8) + r10 = CPyDict_SetItem(r0, r6, r9) + r11 = r10 >= 0 :: signed + r12 = __main__.globals :: static + r13 = enum :: module + r14 = load_address _Py_NoneStruct + r15 = r13 != r14 + if r15 goto L4 else goto L3 :: bool +L3: + r16 = 'enum' + r17 = PyImport_Import(r16) + enum = r17 :: module +L4: + r18 = 'enum' + r19 = PyImport_GetModuleDict() + r20 = 'enum' + r21 = CPyDict_GetItem(r19, r20) + r22 = CPyDict_SetItem(r12, r18, r21) + r23 = r22 >= 0 :: signed + return 1 +def submodule(): + r0 :: dict + r1, r2 :: object + r3 :: bit + r4 :: str + r5 :: object + r6 :: str + r7 :: dict + r8 :: str + r9 :: object + r10 :: i32 + r11 :: bit + r12 :: dict + r13 :: str + r14 :: object + r15 :: str + r16 :: object + r17 :: int +L0: + r0 = __main__.globals :: static + r1 = p.m :: module + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool +L1: + r4 = 'p.m' + r5 = PyImport_Import(r4) + p.m = r5 :: module +L2: + r6 = 'p' + r7 = PyImport_GetModuleDict() + r8 = 'p' + r9 = CPyDict_GetItem(r7, r8) + r10 = CPyDict_SetItem(r0, r6, r9) + r11 = r10 >= 0 :: signed + r12 = PyImport_GetModuleDict() + r13 = 'p' + r14 = CPyDict_GetItem(r12, r13) + r15 = 'x' + r16 = CPyObject_GetAttr(r14, r15) + r17 = unbox(int, r16) + return r17 + +[case testIsinstanceBool] +def f(x: object) -> bool: + return isinstance(x, bool) +[out] +def f(x): + x, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool +L0: + r0 = load_address PyBool_Type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + return r3 + +[case testRangeObject] +def range_object() -> None: + r = range(4, 12, 2) + sum = 0 + for i in r: + sum += i + +def range_in_loop() -> None: + sum = 0 + for i in range(4, 12, 2): + sum += i +[out] +def range_object(): + r0, r1, r2, r3, r4 :: object + r5, r :: range + sum :: int + r6, r7 :: object + r8, i, r9 :: int + r10 :: bit +L0: + r0 = load_address PyRange_Type + r1 = object 4 + r2 = object 12 + r3 = object 2 + r4 = PyObject_CallFunctionObjArgs(r0, r1, r2, r3, 0) + r5 = cast(range, r4) + r = r5 + sum = 0 + r6 = PyObject_GetIter(r) +L1: + r7 = PyIter_Next(r6) + if is_error(r7) goto L4 else goto L2 +L2: + r8 = unbox(int, r7) + i = r8 + r9 = CPyTagged_Add(sum, i) + sum = r9 +L3: + goto L1 +L4: + r10 = CPy_NoErrOccured() +L5: + return 1 +def range_in_loop(): + sum :: int + r0 :: short_int + i :: int + r1 :: bit + r2 :: int + r3 :: short_int +L0: + sum = 0 + r0 = 8 + i = r0 +L1: + r1 = int_lt r0, 24 + if r1 goto L2 else goto L4 :: bool +L2: + r2 = CPyTagged_Add(sum, i) + sum = r2 +L3: + r3 = r0 + 4 + r0 = r3 + i = r3 + goto L1 +L4: + return 1 + +[case testLocalRedefinition] +# mypy: allow-redefinition +def f() -> None: + i = 0 + i += 1 + i = "foo" + i += i + i = 0.0 +[out] +def f(): + i, r0 :: int + r1, i__redef__, r2 :: str + i__redef____redef__ :: float +L0: + i = 0 + r0 = CPyTagged_Add(i, 2) + i = r0 + r1 = 'foo' + i__redef__ = r1 + r2 = CPyStr_Append(i__redef__, i__redef__) + i__redef__ = r2 + i__redef____redef__ = 0.0 + return 1 + +[case testNewType] +from typing import NewType + +class A: pass + +N = NewType("N", A) + +def f(arg: A) -> N: + return N(arg) +[out] +def f(arg): + arg :: __main__.A +L0: + return arg + +[case testTypeCheckingFlag] +from typing import TYPE_CHECKING, List + +def f(arg: List[int]) -> int: + if TYPE_CHECKING: + from collections.abc import Sized + s: Sized = arg + return len(s) + +[out] +def f(arg): + arg :: list + r0 :: bool + r1 :: int + r2 :: bit + s :: object + r3 :: int +L0: + r0 = 0 << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = r1 != 0 + if r2 goto L1 else goto L2 :: bool +L1: + goto L3 +L2: +L3: + s = arg + r3 = CPyObject_Size(s) + return r3 diff --git a/mypyc/test-data/irbuild-bool.test b/mypyc/test-data/irbuild-bool.test new file mode 100644 index 000000000000..128266e6b1d7 --- /dev/null +++ b/mypyc/test-data/irbuild-bool.test @@ -0,0 +1,424 @@ +[case testBoolToAndFromInt] +from mypy_extensions import i64 + +def bool_to_int(b: bool) -> int: + return b +def int_to_bool(n: int) -> bool: + return bool(n) +def bool_to_i64(b: bool) -> i64: + return b +def i64_to_bool(n: i64) -> bool: + return bool(n) +def bit_to_int(n1: i64, n2: i64) -> int: + return bool(n1 == n2) +def bit_to_i64(n1: i64, n2: i64) -> i64: + return bool(n1 == n2) +[out] +def bool_to_int(b): + b, r0 :: bool + r1 :: int +L0: + r0 = b << 1 + r1 = extend r0: builtins.bool to builtins.int + return r1 +def int_to_bool(n): + n :: int + r0 :: bit +L0: + r0 = n != 0 + return r0 +def bool_to_i64(b): + b :: bool + r0 :: i64 +L0: + r0 = extend b: builtins.bool to i64 + return r0 +def i64_to_bool(n): + n :: i64 + r0 :: bit +L0: + r0 = n != 0 + return r0 +def bit_to_int(n1, n2): + n1, n2 :: i64 + r0 :: bit + r1 :: bool + r2 :: int +L0: + r0 = n1 == n2 + r1 = r0 << 1 + r2 = extend r1: builtins.bool to builtins.int + return r2 +def bit_to_i64(n1, n2): + n1, n2 :: i64 + r0 :: bit + r1 :: i64 +L0: + r0 = n1 == n2 + r1 = extend r0: bit to i64 + return r1 + +[case testConversionToBool] +from typing import List, Optional + +class C: pass +class D: + def __bool__(self) -> bool: + return True + +def list_to_bool(l: List[str]) -> bool: + return bool(l) + +def always_truthy_instance_to_bool(o: C) -> bool: + return bool(o) + +def instance_to_bool(o: D) -> bool: + return bool(o) + +def optional_truthy_to_bool(o: Optional[C]) -> bool: + return bool(o) + +def optional_maybe_falsey_to_bool(o: Optional[D]) -> bool: + return bool(o) +[out] +def D.__bool__(self): + self :: __main__.D +L0: + return 1 +def list_to_bool(l): + l :: list + r0 :: native_int + r1 :: short_int + r2 :: bit +L0: + r0 = var_object_size l + r1 = r0 << 1 + r2 = int_ne r1, 0 + return r2 +def always_truthy_instance_to_bool(o): + o :: __main__.C + r0 :: i32 + r1 :: bit + r2 :: bool +L0: + r0 = PyObject_IsTrue(o) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + return r2 +def instance_to_bool(o): + o :: __main__.D + r0 :: bool +L0: + r0 = o.__bool__() + return r0 +def optional_truthy_to_bool(o): + o :: union[__main__.C, None] + r0 :: object + r1 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = o != r0 + return r1 +def optional_maybe_falsey_to_bool(o): + o :: union[__main__.D, None] + r0 :: object + r1 :: bit + r2 :: __main__.D + r3 :: bool + r4 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = o != r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = cast(__main__.D, o) + r3 = r2.__bool__() + r4 = r3 + goto L3 +L2: + r4 = 0 +L3: + return r4 + +[case testBoolComparisons] +def eq(x: bool, y: bool) -> bool: + return x == y + +def neq(x: bool, y: bool) -> bool: + return x != y + +def lt(x: bool, y: bool) -> bool: + return x < y + +def le(x: bool, y: bool) -> bool: + return x <= y + +def gt(x: bool, y: bool) -> bool: + return x > y + +def ge(x: bool, y: bool) -> bool: + return x >= y +[out] +def eq(x, y): + x, y :: bool + r0 :: bit +L0: + r0 = x == y + return r0 +def neq(x, y): + x, y :: bool + r0 :: bit +L0: + r0 = x != y + return r0 +def lt(x, y): + x, y :: bool + r0 :: bit +L0: + r0 = x < y :: signed + return r0 +def le(x, y): + x, y :: bool + r0 :: bit +L0: + r0 = x <= y :: signed + return r0 +def gt(x, y): + x, y :: bool + r0 :: bit +L0: + r0 = x > y :: signed + return r0 +def ge(x, y): + x, y :: bool + r0 :: bit +L0: + r0 = x >= y :: signed + return r0 + +[case testBoolMixedComparisons1] +from mypy_extensions import i64 + +def eq1(x: int, y: bool) -> bool: + return x == y + +def eq2(x: bool, y: int) -> bool: + return x == y + +def neq1(x: i64, y: bool) -> bool: + return x != y + +def neq2(x: bool, y: i64) -> bool: + return x != y +[out] +def eq1(x, y): + x :: int + y, r0 :: bool + r1 :: int + r2 :: bit +L0: + r0 = y << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = int_eq x, r1 + return r2 +def eq2(x, y): + x :: bool + y :: int + r0 :: bool + r1 :: int + r2 :: bit +L0: + r0 = x << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = int_eq r1, y + return r2 +def neq1(x, y): + x :: i64 + y :: bool + r0 :: i64 + r1 :: bit +L0: + r0 = extend y: builtins.bool to i64 + r1 = x != r0 + return r1 +def neq2(x, y): + x :: bool + y, r0 :: i64 + r1 :: bit +L0: + r0 = extend x: builtins.bool to i64 + r1 = r0 != y + return r1 + +[case testBoolMixedComparisons2] +from mypy_extensions import i64 + +def lt1(x: bool, y: int) -> bool: + return x < y + +def lt2(x: int, y: bool) -> bool: + return x < y + +def gt1(x: bool, y: i64) -> bool: + return x < y + +def gt2(x: i64, y: bool) -> bool: + return x < y +[out] +def lt1(x, y): + x :: bool + y :: int + r0 :: bool + r1 :: int + r2 :: bit +L0: + r0 = x << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = int_lt r1, y + return r2 +def lt2(x, y): + x :: int + y, r0 :: bool + r1 :: int + r2 :: bit +L0: + r0 = y << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = int_lt x, r1 + return r2 +def gt1(x, y): + x :: bool + y, r0 :: i64 + r1 :: bit +L0: + r0 = extend x: builtins.bool to i64 + r1 = r0 < y :: signed + return r1 +def gt2(x, y): + x :: i64 + y :: bool + r0 :: i64 + r1 :: bit +L0: + r0 = extend y: builtins.bool to i64 + r1 = x < r0 :: signed + return r1 + +[case testBoolBitwise] +from mypy_extensions import i64 +def bitand(x: bool, y: bool) -> bool: + b = x & y + return b +def bitor(x: bool, y: bool) -> bool: + b = x | y + return b +def bitxor(x: bool, y: bool) -> bool: + b = x ^ y + return b +def invert(x: bool) -> int: + return ~x +def mixed_bitand(x: i64, y: bool) -> i64: + return x & y +[out] +def bitand(x, y): + x, y, r0, b :: bool +L0: + r0 = x & y + b = r0 + return b +def bitor(x, y): + x, y, r0, b :: bool +L0: + r0 = x | y + b = r0 + return b +def bitxor(x, y): + x, y, r0, b :: bool +L0: + r0 = x ^ y + b = r0 + return b +def invert(x): + x, r0 :: bool + r1, r2 :: int +L0: + r0 = x << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = CPyTagged_Invert(r1) + return r2 +def mixed_bitand(x, y): + x :: i64 + y :: bool + r0, r1 :: i64 +L0: + r0 = extend y: builtins.bool to i64 + r1 = x & r0 + return r1 + +[case testBoolArithmetic] +def add(x: bool, y: bool) -> int: + z = x + y + return z +def mixed(b: bool, n: int) -> int: + z = b + n + z -= b + z = z * b + return z +def negate(b: bool) -> int: + return -b +def unary_plus(b: bool) -> int: + x = +b + return x +[out] +def add(x, y): + x, y, r0 :: bool + r1 :: int + r2 :: bool + r3, r4, z :: int +L0: + r0 = x << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = y << 1 + r3 = extend r2: builtins.bool to builtins.int + r4 = CPyTagged_Add(r1, r3) + z = r4 + return z +def mixed(b, n): + b :: bool + n :: int + r0 :: bool + r1, r2, z :: int + r3 :: bool + r4, r5 :: int + r6 :: bool + r7, r8 :: int +L0: + r0 = b << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = CPyTagged_Add(r1, n) + z = r2 + r3 = b << 1 + r4 = extend r3: builtins.bool to builtins.int + r5 = CPyTagged_Subtract(z, r4) + z = r5 + r6 = b << 1 + r7 = extend r6: builtins.bool to builtins.int + r8 = CPyTagged_Multiply(z, r7) + z = r8 + return z +def negate(b): + b, r0 :: bool + r1, r2 :: int +L0: + r0 = b << 1 + r1 = extend r0: builtins.bool to builtins.int + r2 = CPyTagged_Negate(r1) + return r2 +def unary_plus(b): + b, r0 :: bool + r1, x :: int +L0: + r0 = b << 1 + r1 = extend r0: builtins.bool to builtins.int + x = r1 + return x diff --git a/mypyc/test-data/irbuild-bytes.test b/mypyc/test-data/irbuild-bytes.test new file mode 100644 index 000000000000..b41836d8829f --- /dev/null +++ b/mypyc/test-data/irbuild-bytes.test @@ -0,0 +1,181 @@ +[case testBytesBasics] +def f(num: int, l: list, d: dict, s: str) -> None: + b1 = bytes() + b2 = bytes(num) + b3 = bytes(l) + b4 = bytes(d) + b5 = bytes(s) +[out] +def f(num, l, d, s): + num :: int + l :: list + d :: dict + s :: str + r0, r1 :: object + r2, b1 :: bytes + r3, r4, r5 :: object + r6, b2, r7, b3, r8, b4, r9, b5 :: bytes +L0: + r0 = load_address PyBytes_Type + r1 = PyObject_CallFunctionObjArgs(r0, 0) + r2 = cast(bytes, r1) + b1 = r2 + r3 = load_address PyBytes_Type + r4 = box(int, num) + r5 = PyObject_CallFunctionObjArgs(r3, r4, 0) + r6 = cast(bytes, r5) + b2 = r6 + r7 = PyBytes_FromObject(l) + b3 = r7 + r8 = PyBytes_FromObject(d) + b4 = r8 + r9 = PyBytes_FromObject(s) + b5 = r9 + return 1 + +[case testBytearrayBasics] +def f(s: str, num: int) -> None: + a = bytearray() + b = bytearray(s) + c = bytearray(num) +[out] +def f(s, num): + s :: str + num :: int + r0 :: object + r1 :: str + r2, r3, a :: object + r4 :: bytes + b, r5 :: object + r6 :: bytes + c :: object +L0: + r0 = builtins :: module + r1 = 'bytearray' + r2 = CPyObject_GetAttr(r0, r1) + r3 = PyObject_CallFunctionObjArgs(r2, 0) + a = r3 + r4 = PyByteArray_FromObject(s) + b = r4 + r5 = box(int, num) + r6 = PyByteArray_FromObject(r5) + c = r6 + return 1 + +[case testBytesEquality] +def eq(x: bytes, y: bytes) -> bool: + return x == y + +def neq(x: bytes, y: bytes) -> bool: + return x != y +[out] +def eq(x, y): + x, y :: bytes + r0 :: i32 + r1, r2 :: bit +L0: + r0 = CPyBytes_Compare(x, y) + r1 = r0 >= 0 :: signed + r2 = r0 == 1 + return r2 +def neq(x, y): + x, y :: bytes + r0 :: i32 + r1, r2 :: bit +L0: + r0 = CPyBytes_Compare(x, y) + r1 = r0 >= 0 :: signed + r2 = r0 != 1 + return r2 + +[case testBytesSlicing] +def f(a: bytes, start: int, end: int) -> bytes: + return a[start:end] +[out] +def f(a, start, end): + a :: bytes + start, end :: int + r0 :: bytes +L0: + r0 = CPyBytes_GetSlice(a, start, end) + return r0 + +[case testBytesIndex] +def f(a: bytes, i: int) -> int: + return a[i] +[out] +def f(a, i): + a :: bytes + i, r0 :: int +L0: + r0 = CPyBytes_GetItem(a, i) + return r0 + +[case testBytesConcat] +def f(a: bytes, b: bytes) -> bytes: + return a + b +[out] +def f(a, b): + a, b, r0 :: bytes +L0: + r0 = CPyBytes_Concat(a, b) + return r0 + +[case testBytesJoin] +from typing import List +def f(b: List[bytes]) -> bytes: + return b" ".join(b) +[out] +def f(b): + b :: list + r0, r1 :: bytes +L0: + r0 = b' ' + r1 = CPyBytes_Join(r0, b) + return r1 + +[case testBytesLen] +def f(b: bytes) -> int: + return len(b) +[out] +def f(b): + b :: bytes + r0 :: native_int + r1 :: short_int +L0: + r0 = var_object_size b + r1 = r0 << 1 + return r1 + +[case testBytesFormatting] +def f(var: bytes, num: int) -> None: + b1 = b'aaaa%bbbbb%s' % (var, var) + b2 = b'aaaa%bbbbb%s%d' % (var, var, num) + b3 = b'%b' % var + b4 = b'%ssss' % var +[typing fixtures/typing-full.pyi] +[out] +def f(var, num): + var :: bytes + num :: int + r0, r1, r2, b1, r3 :: bytes + r4 :: tuple[bytes, bytes, int] + r5, r6 :: object + r7, b2, r8, b3, r9, r10, b4 :: bytes +L0: + r0 = b'aaaa' + r1 = b'bbbb' + r2 = CPyBytes_Build(4, r0, var, r1, var) + b1 = r2 + r3 = b'aaaa%bbbbb%s%d' + r4 = (var, var, num) + r5 = box(tuple[bytes, bytes, int], r4) + r6 = PyNumber_Remainder(r3, r5) + r7 = cast(bytes, r6) + b2 = r7 + r8 = CPyBytes_Build(1, var) + b3 = r8 + r9 = b'sss' + r10 = CPyBytes_Build(2, var, r9) + b4 = r10 + return 1 diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test new file mode 100644 index 000000000000..2c15f09c9c34 --- /dev/null +++ b/mypyc/test-data/irbuild-classes.test @@ -0,0 +1,1302 @@ +[case testGetAttribute] +class A: + x: int + +def f(a: A) -> int: + return a.x +[out] +def f(a): + a :: __main__.A + r0 :: int +L0: + r0 = a.x + return r0 + +[case testSetAttribute] +class A: + x: int + +def f(a: A) -> None: + a.x = 1 +[out] +def f(a): + a :: __main__.A + r0 :: bool +L0: + a.x = 2; r0 = is_error + return 1 + +[case testUserClassInList] +class C: + x: int + +def f() -> int: + c = C() + c.x = 5 + a = [c] + d = a[0] + return d.x + 1 +[out] +def f(): + r0, c :: __main__.C + r1 :: bool + r2 :: list + r3 :: ptr + a :: list + r4 :: object + r5, d :: __main__.C + r6, r7 :: int +L0: + r0 = C() + c = r0 + c.x = 10; r1 = is_error + r2 = PyList_New(1) + r3 = list_items r2 + buf_init_item r3, 0, c + keep_alive r2 + a = r2 + r4 = CPyList_GetItemShort(a, 0) + r5 = cast(__main__.C, r4) + d = r5 + r6 = borrow d.x + r7 = CPyTagged_Add(r6, 2) + keep_alive d + return r7 + +[case testMethodCall] +class A: + def f(self, x: int, y: str) -> int: + return x + 10 + +def g(a: A) -> None: + a.f(1, 'hi') +[out] +def A.f(self, x, y): + self :: __main__.A + x :: int + y :: str + r0 :: int +L0: + r0 = CPyTagged_Add(x, 20) + return r0 +def g(a): + a :: __main__.A + r0 :: str + r1 :: int +L0: + r0 = 'hi' + r1 = a.f(2, r0) + return 1 + +[case testForwardUse] +def g(a: A) -> int: + return a.n + +class A: + n : int + +[out] +def g(a): + a :: __main__.A + r0 :: int +L0: + r0 = a.n + return r0 + +[case testOptionalMember] +from typing import Optional +class Node: + next: Optional[Node] + def length(self) -> int: + if self.next is not None: + return 1 + self.next.length() + return 1 +[out] +def Node.length(self): + self :: __main__.Node + r0 :: union[__main__.Node, None] + r1 :: object + r2 :: bit + r3 :: union[__main__.Node, None] + r4 :: __main__.Node + r5, r6 :: int +L0: + r0 = borrow self.next + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + keep_alive self + if r2 goto L1 else goto L2 :: bool +L1: + r3 = self.next + r4 = cast(__main__.Node, r3) + r5 = r4.length() + r6 = CPyTagged_Add(2, r5) + return r6 +L2: + return 2 + +[case testSubclass] +class A: + def __init__(self) -> None: + self.x = 10 +class B(A): + def __init__(self) -> None: + self.x = 20 + self.y = 30 +[out] +def A.__init__(self): + self :: __main__.A +L0: + self.x = 20 + return 1 +def B.__init__(self): + self :: __main__.B +L0: + self.x = 40 + self.y = 60 + return 1 + +[case testAttrLvalue] +class O(object): + def __init__(self) -> None: + self.x = 1 + +def increment(o: O) -> O: + o.x += 1 + return o +[out] +def O.__init__(self): + self :: __main__.O +L0: + self.x = 2 + return 1 +def increment(o): + o :: __main__.O + r0, r1 :: int + r2 :: bool +L0: + r0 = borrow o.x + r1 = CPyTagged_Add(r0, 2) + o.x = r1; r2 = is_error + return o + +[case testSubclass_toplevel] +from typing import TypeVar, Generic +from mypy_extensions import trait +T = TypeVar('T') +class C: + pass + +@trait +class S: + pass + +class D(C, S, Generic[T]): + pass + +[out] +def __top_level__(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4, r5 :: object + r6 :: str + r7 :: dict + r8, r9 :: object + r10 :: str + r11 :: dict + r12 :: object + r13 :: str + r14 :: dict + r15 :: str + r16, r17 :: object + r18 :: dict + r19 :: str + r20 :: i32 + r21 :: bit + r22 :: object + r23 :: str + r24, r25 :: object + r26 :: bool + r27 :: str + r28 :: tuple + r29 :: i32 + r30 :: bit + r31 :: dict + r32 :: str + r33 :: i32 + r34 :: bit + r35 :: object + r36 :: str + r37, r38 :: object + r39 :: str + r40 :: tuple + r41 :: i32 + r42 :: bit + r43 :: dict + r44 :: str + r45 :: i32 + r46 :: bit + r47, r48 :: object + r49 :: dict + r50 :: str + r51 :: object + r52 :: dict + r53 :: str + r54, r55 :: object + r56 :: tuple + r57 :: str + r58, r59 :: object + r60 :: bool + r61, r62 :: str + r63 :: tuple + r64 :: i32 + r65 :: bit + r66 :: dict + r67 :: str + r68 :: i32 + r69 :: bit +L0: + r0 = builtins :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = 'builtins' + r4 = PyImport_Import(r3) + builtins = r4 :: module +L2: + r5 = ('TypeVar', 'Generic') + r6 = 'typing' + r7 = __main__.globals :: static + r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) + typing = r8 :: module + r9 = ('trait',) + r10 = 'mypy_extensions' + r11 = __main__.globals :: static + r12 = CPyImport_ImportFromMany(r10, r9, r9, r11) + mypy_extensions = r12 :: module + r13 = 'T' + r14 = __main__.globals :: static + r15 = 'TypeVar' + r16 = CPyDict_GetItem(r14, r15) + r17 = PyObject_CallFunctionObjArgs(r16, r13, 0) + r18 = __main__.globals :: static + r19 = 'T' + r20 = CPyDict_SetItem(r18, r19, r17) + r21 = r20 >= 0 :: signed + r22 = :: object + r23 = '__main__' + r24 = __main__.C_template :: type + r25 = CPyType_FromTemplate(r24, r22, r23) + r26 = C_trait_vtable_setup() + r27 = '__mypyc_attrs__' + r28 = PyTuple_Pack(0) + r29 = PyObject_SetAttr(r25, r27, r28) + r30 = r29 >= 0 :: signed + __main__.C = r25 :: type + r31 = __main__.globals :: static + r32 = 'C' + r33 = CPyDict_SetItem(r31, r32, r25) + r34 = r33 >= 0 :: signed + r35 = :: object + r36 = '__main__' + r37 = __main__.S_template :: type + r38 = CPyType_FromTemplate(r37, r35, r36) + r39 = '__mypyc_attrs__' + r40 = PyTuple_Pack(0) + r41 = PyObject_SetAttr(r38, r39, r40) + r42 = r41 >= 0 :: signed + __main__.S = r38 :: type + r43 = __main__.globals :: static + r44 = 'S' + r45 = CPyDict_SetItem(r43, r44, r38) + r46 = r45 >= 0 :: signed + r47 = __main__.C :: type + r48 = __main__.S :: type + r49 = __main__.globals :: static + r50 = 'Generic' + r51 = CPyDict_GetItem(r49, r50) + r52 = __main__.globals :: static + r53 = 'T' + r54 = CPyDict_GetItem(r52, r53) + r55 = PyObject_GetItem(r51, r54) + r56 = PyTuple_Pack(3, r47, r48, r55) + r57 = '__main__' + r58 = __main__.D_template :: type + r59 = CPyType_FromTemplate(r58, r56, r57) + r60 = D_trait_vtable_setup() + r61 = '__mypyc_attrs__' + r62 = '__dict__' + r63 = PyTuple_Pack(1, r62) + r64 = PyObject_SetAttr(r59, r61, r63) + r65 = r64 >= 0 :: signed + __main__.D = r59 :: type + r66 = __main__.globals :: static + r67 = 'D' + r68 = CPyDict_SetItem(r66, r67, r59) + r69 = r68 >= 0 :: signed + return 1 + +[case testIsInstance] +class A: pass +class B(A): pass + +def f(x: A) -> B: + if isinstance(x, B): + return x + return B() +[out] +def f(x): + x :: __main__.A + r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4, r5 :: __main__.B +L0: + r0 = __main__.B :: type + r1 = get_element_ptr x ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive x + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = cast(__main__.B, x) + return r4 +L2: + r5 = B() + return r5 + +[case testIsInstanceTuple] +from typing import Union +class R: pass +class A(R): pass +class B(R): pass +class C(R): pass + +def f(x: R) -> Union[A, B]: + if isinstance(x, (A, B)): + return x + return A() +[out] +def f(x): + x :: __main__.R + r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4 :: bool + r5 :: object + r6 :: ptr + r7 :: object + r8 :: bit + r9 :: union[__main__.A, __main__.B] + r10 :: __main__.A +L0: + r0 = __main__.A :: type + r1 = get_element_ptr x ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive x + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = r3 + goto L3 +L2: + r5 = __main__.B :: type + r6 = get_element_ptr x ob_type :: PyObject + r7 = load_mem r6 :: builtins.object* + keep_alive x + r8 = r7 == r5 + r4 = r8 +L3: + if r4 goto L4 else goto L5 :: bool +L4: + r9 = cast(union[__main__.A, __main__.B], x) + return r9 +L5: + r10 = A() + return r10 + +[case testIsInstanceFewSubclasses] +class R: pass +class A(R): pass + +def f(x: object) -> R: + if isinstance(x, R): + return x + return A() +[out] +def f(x): + x, r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4 :: bool + r5 :: object + r6 :: ptr + r7 :: object + r8 :: bit + r9 :: __main__.R + r10 :: __main__.A +L0: + r0 = __main__.A :: type + r1 = get_element_ptr x ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive x + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = r3 + goto L3 +L2: + r5 = __main__.R :: type + r6 = get_element_ptr x ob_type :: PyObject + r7 = load_mem r6 :: builtins.object* + keep_alive x + r8 = r7 == r5 + r4 = r8 +L3: + if r4 goto L4 else goto L5 :: bool +L4: + r9 = cast(__main__.R, x) + return r9 +L5: + r10 = A() + return r10 + +[case testIsInstanceFewSubclassesTrait] +from mypy_extensions import trait +class B: pass +@trait +class R: pass +class A(B, R): pass +class C(B, R): pass + +def f(x: object) -> R: + if isinstance(x, R): + return x + return A() +[out] +def f(x): + x, r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4 :: bool + r5 :: object + r6 :: ptr + r7 :: object + r8 :: bit + r9 :: __main__.R + r10 :: __main__.A +L0: + r0 = __main__.A :: type + r1 = get_element_ptr x ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive x + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = r3 + goto L3 +L2: + r5 = __main__.C :: type + r6 = get_element_ptr x ob_type :: PyObject + r7 = load_mem r6 :: builtins.object* + keep_alive x + r8 = r7 == r5 + r4 = r8 +L3: + if r4 goto L4 else goto L5 :: bool +L4: + r9 = cast(__main__.R, x) + return r9 +L5: + r10 = A() + return r10 + +[case testIsInstanceManySubclasses] +class R: pass +class A(R): pass +class B(R): pass +class C(R): pass + +def f(x: object) -> R: + if isinstance(x, R): + return x + return B() +[out] +def f(x): + x, r0 :: object + r1 :: bool + r2 :: __main__.R + r3 :: __main__.B +L0: + r0 = __main__.R :: type + r1 = CPy_TypeCheck(x, r0) + if r1 goto L1 else goto L2 :: bool +L1: + r2 = cast(__main__.R, x) + return r2 +L2: + r3 = B() + return r3 + +[case testFakeSuper] +class A: + def __init__(self, x: int) -> None: + self.x = x +class B(A): + def __init__(self, x: int, y: int) -> None: + A.__init__(self, x) + self.y = y +[out] +def A.__init__(self, x): + self :: __main__.A + x :: int +L0: + self.x = x + return 1 +def B.__init__(self, x, y): + self :: __main__.B + x, y :: int + r0 :: None +L0: + r0 = A.__init__(self, x) + self.y = y + return 1 + +[case testClassMethod] +class C: + @staticmethod + def foo(x: int) -> int: return 10 + x + @classmethod + def bar(cls, x: int) -> int: return 10 + x + +def lol() -> int: + return C.foo(1) + C.bar(2) +[out] +def C.foo(x): + x, r0 :: int +L0: + r0 = CPyTagged_Add(20, x) + return r0 +def C.bar(cls, x): + cls :: object + x, r0 :: int +L0: + r0 = CPyTagged_Add(20, x) + return r0 +def lol(): + r0 :: int + r1 :: object + r2, r3 :: int +L0: + r0 = C.foo(2) + r1 = __main__.C :: type + r2 = C.bar(r1, 4) + r3 = CPyTagged_Add(r0, r2) + return r3 + +[case testCallClassMethodViaCls] +class C: + @classmethod + def f(cls, x: int) -> int: + return cls.g(x) + + @classmethod + def g(cls, x: int) -> int: + return x + +class D: + @classmethod + def f(cls, x: int) -> int: + # TODO: This could aso be optimized, since g is not ever overridden + return cls.g(x) + + @classmethod + def g(cls, x: int) -> int: + return x + +class DD(D): + pass +[out] +def C.f(cls, x): + cls :: object + x :: int + r0 :: object + r1 :: int +L0: + r0 = __main__.C :: type + r1 = C.g(r0, x) + return r1 +def C.g(cls, x): + cls :: object + x :: int +L0: + return x +def D.f(cls, x): + cls :: object + x :: int + r0 :: str + r1, r2 :: object + r3 :: int +L0: + r0 = 'g' + r1 = box(int, x) + r2 = CPyObject_CallMethodObjArgs(cls, r0, r1, 0) + r3 = unbox(int, r2) + return r3 +def D.g(cls, x): + cls :: object + x :: int +L0: + return x + +[case testCannotAssignToClsArgument] +from typing import Any, cast + +class C: + @classmethod + def m(cls) -> None: + cls = cast(Any, D) # E: Cannot assign to the first argument of classmethod + cls, x = cast(Any, D), 1 # E: Cannot assign to the first argument of classmethod + cls, x = cast(Any, [1, 2]) # E: Cannot assign to the first argument of classmethod + cls.m() + +class D: + pass + +[case testSuper1] +class A: + def __init__(self, x: int) -> None: + self.x = x +class B(A): + def __init__(self, x: int, y: int) -> None: + super().__init__(x) + self.y = y +[out] +def A.__init__(self, x): + self :: __main__.A + x :: int +L0: + self.x = x + return 1 +def B.__init__(self, x, y): + self :: __main__.B + x, y :: int + r0 :: None +L0: + r0 = A.__init__(self, x) + self.y = y + return 1 + +[case testSuper2] +from mypy_extensions import trait +@trait +class T: + def foo(self) -> None: pass + +class X(T): + def foo(self) -> None: + super().foo() +[out] +def T.foo(self): + self :: __main__.T +L0: + return 1 +def X.foo(self): + self :: __main__.X + r0 :: None +L0: + r0 = T.foo(self) + return 1 + +[case testSuperCallToObjectInitIsOmitted] +class C: + def __init__(self) -> None: + super().__init__() +class D: pass +class E(D): + def __init__(self) -> None: + super().__init__() +class F(C): + def __init__(self) -> None: + super().__init__() +class DictSubclass(dict): + def __init__(self) -> None: + super().__init__() +[out] +def C.__init__(self): + self :: __main__.C +L0: + return 1 +def E.__init__(self): + self :: __main__.E +L0: + return 1 +def F.__init__(self): + self :: __main__.F + r0 :: None +L0: + r0 = C.__init__(self) + return 1 +def DictSubclass.__init__(self): + self :: dict + r0 :: object + r1 :: str + r2, r3, r4 :: object + r5 :: str + r6, r7 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.DictSubclass :: type + r4 = PyObject_CallFunctionObjArgs(r2, r3, self, 0) + r5 = '__init__' + r6 = CPyObject_GetAttr(r4, r5) + r7 = PyObject_CallFunctionObjArgs(r6, 0) + return 1 + +[case testClassVariable] +from typing import ClassVar +class A: + x = 10 # type: ClassVar[int] + +def f() -> int: + return A.x +[out] +def f(): + r0 :: object + r1 :: str + r2 :: object + r3 :: int +L0: + r0 = __main__.A :: type + r1 = 'x' + r2 = CPyObject_GetAttr(r0, r1) + r3 = unbox(int, r2) + return r3 + +[case testNoEqDefined] +class A: + pass + +def f(a: A, b: A) -> bool: + return a == b + +def f2(a: A, b: A) -> bool: + return a != b + +[out] +def f(a, b): + a, b :: __main__.A + r0 :: bit +L0: + r0 = a == b + return r0 +def f2(a, b): + a, b :: __main__.A + r0 :: bit +L0: + r0 = a != b + return r0 + +[case testEqDefined] +class Base: + def __eq__(self, other: object) -> bool: + return False +class Derived(Base): + def __eq__(self, other: object) -> bool: + return True + +def f(a: Base, b: Base) -> bool: + return a == b + +def f2(a: Base, b: Base) -> bool: + return a != b + +def fOpt(a: Derived, b: Derived) -> bool: + return a == b + +def fOpt2(a: Derived, b: Derived) -> bool: + return a != b + +[out] +def Base.__eq__(self, other): + self :: __main__.Base + other, r0 :: object +L0: + r0 = box(bool, 0) + return r0 +def Base.__ne__(__mypyc_self__, rhs): + __mypyc_self__ :: __main__.Base + rhs, r0, r1 :: object + r2 :: bit + r3 :: i32 + r4 :: bit + r5 :: bool + r6 :: object +L0: + r0 = __mypyc_self__.__eq__(rhs) + r1 = load_address _Py_NotImplementedStruct + r2 = r0 == r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = PyObject_Not(r0) + r4 = r3 >= 0 :: signed + r5 = truncate r3: i32 to builtins.bool + r6 = box(bool, r5) + return r6 +L2: + return r1 +def Derived.__eq__(self, other): + self :: __main__.Derived + other, r0 :: object +L0: + r0 = box(bool, 1) + return r0 +def f(a, b): + a, b :: __main__.Base + r0 :: object + r1 :: bool +L0: + r0 = PyObject_RichCompare(a, b, 2) + r1 = unbox(bool, r0) + return r1 +def f2(a, b): + a, b :: __main__.Base + r0 :: object + r1 :: bool +L0: + r0 = PyObject_RichCompare(a, b, 3) + r1 = unbox(bool, r0) + return r1 +def fOpt(a, b): + a, b :: __main__.Derived + r0 :: object + r1 :: bool +L0: + r0 = a.__eq__(b) + r1 = unbox(bool, r0) + return r1 +def fOpt2(a, b): + a, b :: __main__.Derived + r0 :: object + r1 :: bool +L0: + r0 = a.__ne__(b) + r1 = unbox(bool, r0) + return r1 + +[case testEqDefinedLater] +def f(a: 'Base', b: 'Base') -> bool: + return a == b + +def f2(a: 'Base', b: 'Base') -> bool: + return a != b + +def fOpt(a: 'Derived', b: 'Derived') -> bool: + return a == b + +def fOpt2(a: 'Derived', b: 'Derived') -> bool: + return a != b + +class Base: + pass +class Derived(Base): + def __eq__(self, other: object) -> bool: + return True + +[out] +def f(a, b): + a, b :: __main__.Base + r0 :: object + r1 :: bool +L0: + r0 = PyObject_RichCompare(a, b, 2) + r1 = unbox(bool, r0) + return r1 +def f2(a, b): + a, b :: __main__.Base + r0 :: object + r1 :: bool +L0: + r0 = PyObject_RichCompare(a, b, 3) + r1 = unbox(bool, r0) + return r1 +def fOpt(a, b): + a, b :: __main__.Derived + r0 :: object + r1 :: bool +L0: + r0 = a.__eq__(b) + r1 = unbox(bool, r0) + return r1 +def fOpt2(a, b): + a, b :: __main__.Derived + r0 :: str + r1 :: object + r2 :: bool +L0: + r0 = '__ne__' + r1 = CPyObject_CallMethodObjArgs(a, r0, b, 0) + r2 = unbox(bool, r1) + return r2 +def Derived.__eq__(self, other): + self :: __main__.Derived + other, r0 :: object +L0: + r0 = box(bool, 1) + return r0 +def Derived.__ne__(__mypyc_self__, rhs): + __mypyc_self__ :: __main__.Derived + rhs, r0, r1 :: object + r2 :: bit + r3 :: i32 + r4 :: bit + r5 :: bool + r6 :: object +L0: + r0 = __mypyc_self__.__eq__(rhs) + r1 = load_address _Py_NotImplementedStruct + r2 = r0 == r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = PyObject_Not(r0) + r4 = r3 >= 0 :: signed + r5 = truncate r3: i32 to builtins.bool + r6 = box(bool, r5) + return r6 +L2: + return r1 + +[case testDefaultVars] +from typing import ClassVar, Optional +class A: + x = 10 + def lol(self) -> None: + self.x = 100 + +LOL = 'lol' +class B(A): + y = LOL + z: Optional[str] = None + b = True + bogus = None # type: int +[out] +def A.lol(self): + self :: __main__.A + r0 :: bool +L0: + self.x = 200; r0 = is_error + return 1 +def A.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.A +L0: + __mypyc_self__.x = 20 + return 1 +def B.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.B + r0 :: dict + r1 :: str + r2 :: object + r3 :: str + r4 :: object +L0: + __mypyc_self__.x = 20 + r0 = __main__.globals :: static + r1 = 'LOL' + r2 = CPyDict_GetItem(r0, r1) + r3 = cast(str, r2) + __mypyc_self__.y = r3 + r4 = box(None, 1) + __mypyc_self__.z = r4 + __mypyc_self__.b = 1 + return 1 + +[case testSubclassDictSpecalized] +from typing import Dict +class WelpDict(Dict[str, int]): + pass +def foo(x: WelpDict) -> None: + # we care that the specalized op gets used + x.update(x) +[out] +def foo(x): + x :: dict + r0 :: i32 + r1 :: bit +L0: + r0 = CPyDict_Update(x, x) + r1 = r0 >= 0 :: signed + return 1 + +[case testNoSpuriousLinearity] +# Make sure that the non-trait MRO linearity check isn't affected by processing order +class A(B): pass +class B(C): pass +class C: pass +[out] + +[case testDeletableSemanticAnalysis] +class Err1: + __deletable__ = 'x' # E: "__deletable__" must be initialized with a list or tuple expression +class Err2: + __deletable__ = [ + 1 # E: Invalid "__deletable__" item; string literal expected + ] +class Err3: + __deletable__ = ['x', ['y'], 'z'] # E: Invalid "__deletable__" item; string literal expected +class Err4: + __deletable__ = (1,) # E: Invalid "__deletable__" item; string literal expected +a = ['x'] +class Err5: + __deletable__ = a # E: "__deletable__" must be initialized with a list or tuple expression + +class Ok1: + __deletable__ = ('x',) + x: int +class Ok2: + __deletable__ = ['x'] + x: int + +[case testInvalidDeletableAttribute] +class NotDeletable: + __deletable__ = ['x'] + x: int + y: int + +def g(o: NotDeletable) -> None: + del o.x + del o.y # E: "y" cannot be deleted \ + # N: Using "__deletable__ = ['']" in the class body enables "del obj." + +class Base: + x: int + +class Deriv(Base): + __deletable__ = ['x'] # E: Attribute "x" not defined in "Deriv" (defined in "Base") + +class UndefinedDeletable: + __deletable__ = ['x'] # E: Attribute "x" not defined + +class DeletableProperty: + __deletable__ = ['prop'] # E: Cannot make property "prop" deletable + + @property + def prop(self) -> int: + return 5 + +[case testFinalDeletable] +from typing import Final + +class DeletableFinal1: + x: Final[int] # E: Deletable attribute cannot be final + + __deletable__ = ['x'] + + def __init__(self, x: int) -> None: + self.x = x + +class DeletableFinal2: + X: Final = 0 # E: Deletable attribute cannot be final + + __deletable__ = ['X'] + +[case testNeedAnnotateClassVar] +from typing import Final, ClassVar, Type + +class C: + a = 'A' + b: str = 'B' + f: Final = 'F' + c: ClassVar = 'C' + +class D(C): + pass + +def f() -> None: + C.a # E: Cannot access instance attribute "a" through class object \ + # N: (Hint: Use "x: Final = ..." or "x: ClassVar = ..." to define a class attribute) + C.b # E: Cannot access instance attribute "b" through class object \ + # N: (Hint: Use "x: Final = ..." or "x: ClassVar = ..." to define a class attribute) + C.f + C.c + + D.a # E: Cannot access instance attribute "a" through class object \ + # N: (Hint: Use "x: Final = ..." or "x: ClassVar = ..." to define a class attribute) + D.b # E: Cannot access instance attribute "b" through class object \ + # N: (Hint: Use "x: Final = ..." or "x: ClassVar = ..." to define a class attribute) + D.f + D.c + +def g(c: Type[C], d: Type[D]) -> None: + c.a # E: Cannot access instance attribute "a" through class object \ + # N: (Hint: Use "x: Final = ..." or "x: ClassVar = ..." to define a class attribute) + c.f + c.c + + d.a # E: Cannot access instance attribute "a" through class object \ + # N: (Hint: Use "x: Final = ..." or "x: ClassVar = ..." to define a class attribute) + d.f + d.c + +[case testSetAttributeWithDefaultInInit] +class C: + s = '' + + def __init__(self, s: str) -> None: + self.s = s +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str + r0 :: bool +L0: + self.s = s; r0 = is_error + return 1 +def C.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.C + r0 :: str +L0: + r0 = '' + __mypyc_self__.s = r0 + return 1 + +[case testBorrowAttribute] +def f(d: D) -> int: + return d.c.x + +class C: + x: int +class D: + c: C +[out] +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.x + keep_alive d + return r1 + +[case testNoBorrowOverPropertyAccess] +class C: + d: D +class D: + @property + def e(self) -> E: + return E() +class E: + x: int +def f(c: C) -> int: + return c.d.e.x +[out] +def D.e(self): + self :: __main__.D + r0 :: __main__.E +L0: + r0 = E() + return r0 +def f(c): + c :: __main__.C + r0 :: __main__.D + r1 :: __main__.E + r2 :: int +L0: + r0 = c.d + r1 = r0.e + r2 = r1.x + return r2 + +[case testBorrowResultOfCustomGetItemInIfStatement] +from typing import List + +class C: + def __getitem__(self, x: int) -> List[int]: + return [] + +def f(x: C) -> None: + # In this case the keep_alive must come before the branch, as otherwise + # reference count transform will get confused. + if x[1][0] == 2: + y = 1 + else: + y = 2 +[out] +def C.__getitem__(self, x): + self :: __main__.C + x :: int + r0 :: list +L0: + r0 = PyList_New(0) + return r0 +def f(x): + x :: __main__.C + r0 :: list + r1 :: object + r2 :: int + r3 :: bit + y :: int +L0: + r0 = x.__getitem__(2) + r1 = CPyList_GetItemShortBorrow(r0, 0) + r2 = unbox(int, r1) + r3 = int_eq r2, 4 + keep_alive r0 + if r3 goto L1 else goto L2 :: bool +L1: + y = 2 + goto L3 +L2: + y = 4 +L3: + return 1 + +[case testIncompatibleDefinitionOfAttributeInSubclass] +from mypy_extensions import trait + +class Base: + x: int + +class Bad1(Base): + x: bool # E: Type of "x" is incompatible with definition in class "Base" + +class Good1(Base): + x: int + +class Good2(Base): + x: int = 0 + +class Good3(Base): + x = 0 + +class Good4(Base): + def __init__(self) -> None: + self.x = 0 + +class Good5(Base): + def __init__(self) -> None: + self.x: int = 0 + +class Base2(Base): + pass + +class Bad2(Base2): + x: bool = False # E: Type of "x" is incompatible with definition in class "Base" + +class Bad3(Base): + x = False # E: Type of "x" is incompatible with definition in class "Base" + +@trait +class T: + y: object + +class E(T): + y: str # E: Type of "y" is incompatible with definition in trait "T" diff --git a/mypyc/test-data/irbuild-constant-fold.test b/mypyc/test-data/irbuild-constant-fold.test new file mode 100644 index 000000000000..97b13ab337c7 --- /dev/null +++ b/mypyc/test-data/irbuild-constant-fold.test @@ -0,0 +1,480 @@ +[case testIntConstantFolding] +def bin_ops() -> None: + add = 15 + 47 + add_mul = (2 + 3) * 5 + sub = 7 - 11 + div = 3 / 2 + bit_and = 6 & 10 + bit_or = 6 | 10 + bit_xor = 6 ^ 10 + lshift = 5 << 2 + rshift = 13 >> 2 + lshift0 = 5 << 0 + rshift0 = 13 >> 0 +def unary_ops() -> None: + neg1 = -5 + neg2 = --1 + neg3 = -0 + pos = +5 + inverted1 = ~0 + inverted2 = ~5 + inverted3 = ~3 +def pow() -> None: + p0 = 3**0 + p1 = 3**5 + p2 = (-5)**3 + p3 = 0**0 +[out] +def bin_ops(): + add, add_mul, sub :: int + div :: float + bit_and, bit_or, bit_xor, lshift, rshift, lshift0, rshift0 :: int +L0: + add = 124 + add_mul = 50 + sub = -8 + div = 1.5 + bit_and = 4 + bit_or = 28 + bit_xor = 24 + lshift = 40 + rshift = 6 + lshift0 = 10 + rshift0 = 26 + return 1 +def unary_ops(): + neg1, neg2, neg3, pos, inverted1, inverted2, inverted3 :: int +L0: + neg1 = -10 + neg2 = 2 + neg3 = 0 + pos = 10 + inverted1 = -2 + inverted2 = -12 + inverted3 = -8 + return 1 +def pow(): + p0, p1, p2, p3 :: int +L0: + p0 = 2 + p1 = 486 + p2 = -250 + p3 = 2 + return 1 + +[case testIntConstantFoldingDivMod] +def div() -> None: + div1 = 25 // 5 + div2 = 24 // 5 + div3 = 29 // 5 + div4 = 30 // 5 + div_zero = 0 // 5 + neg1 = -1 // 3 + neg2 = -2 // 3 + neg3 = -3 // 3 + neg4 = -4 // 3 + neg_neg = -765467 // -234 + pos_neg = 983745 // -7864 +def mod() -> None: + mod1 = 25 % 5 + mod2 = 24 % 5 + mod3 = 29 % 5 + mod4 = 30 % 5 + mod_zero = 0 % 5 + neg1 = -4 % 3 + neg2 = -5 % 3 + neg3 = -6 % 3 + neg4 = -7 % 3 + neg_neg = -765467 % -234 + pos_neg = 983745 % -7864 +[out] +def div(): + div1, div2, div3, div4, div_zero, neg1, neg2, neg3, neg4, neg_neg, pos_neg :: int +L0: + div1 = 10 + div2 = 8 + div3 = 10 + div4 = 12 + div_zero = 0 + neg1 = -2 + neg2 = -2 + neg3 = -2 + neg4 = -4 + neg_neg = 6542 + pos_neg = -252 + return 1 +def mod(): + mod1, mod2, mod3, mod4, mod_zero, neg1, neg2, neg3, neg4, neg_neg, pos_neg :: int +L0: + mod1 = 0 + mod2 = 8 + mod3 = 8 + mod4 = 0 + mod_zero = 0 + neg1 = 4 + neg2 = 2 + neg3 = 0 + neg4 = 4 + neg_neg = -106 + pos_neg = -14238 + return 1 + +[case testIntConstantFoldingUnsupportedCases] +def error_cases() -> None: + div_by_zero = 5 / 0 + floor_div_by_zero = 5 // 0 + mod_by_zero = 5 % 0 + lshift_neg = 6 << -1 + rshift_neg = 7 >> -1 +def unsupported_pow() -> None: + p = 3 ** (-1) +[out] +def error_cases(): + r0, div_by_zero :: float + r1, floor_div_by_zero, r2, mod_by_zero, r3, lshift_neg, r4, rshift_neg :: int +L0: + r0 = CPyTagged_TrueDivide(10, 0) + div_by_zero = r0 + r1 = CPyTagged_FloorDivide(10, 0) + floor_div_by_zero = r1 + r2 = CPyTagged_Remainder(10, 0) + mod_by_zero = r2 + r3 = CPyTagged_Lshift(12, -2) + lshift_neg = r3 + r4 = CPyTagged_Rshift(14, -2) + rshift_neg = r4 + return 1 +def unsupported_pow(): + r0, r1, r2 :: object + r3, p :: float +L0: + r0 = object 3 + r1 = object -1 + r2 = CPyNumber_Power(r0, r1) + r3 = unbox(float, r2) + p = r3 + return 1 + +[case testIntConstantFoldingBigIntResult_64bit] +def long_and_short() -> None: + # The smallest and largest representable short integers + short1 = 0x3ffffffffffffff0 + 0xf # (1 << 62) - 1 + short2 = -0x3fffffffffffffff - 1 # -(1 << 62) + short3 = -0x4000000000000000 + # Smallest big integers by absolute value + big1 = 1 << 62 + big2 = 0x4000000000000000 # 1 << 62 + big3 = -(1 << 62) - 1 + big4 = -0x4000000000000001 # -(1 << 62) - 1 + big5 = 123**41 +[out] +def long_and_short(): + short1, short2, short3, r0, big1, r1, big2, r2, big3, r3, big4, r4, big5 :: int +L0: + short1 = 9223372036854775806 + short2 = -9223372036854775808 + short3 = -9223372036854775808 + r0 = object 4611686018427387904 + big1 = r0 + r1 = object 4611686018427387904 + big2 = r1 + r2 = object -4611686018427387905 + big3 = r2 + r3 = object -4611686018427387905 + big4 = r3 + r4 = object 48541095000524544750127162673405880068636916264012200797813591925035550682238127143323 + big5 = r4 + return 1 + +[case testIntConstantFoldingFinal] +from typing_extensions import Final +X: Final = 5 +Y: Final = 2 + 4 + +def f() -> None: + a = X + 1 + a = Y + 1 +[out] +def f(): + a :: int +L0: + a = 12 + a = 14 + return 1 + +[case testIntConstantFoldingClassFinal] +from typing_extensions import Final +class C: + X: Final = 5 + +def f() -> None: + a = C.X + 1 +[out] +def C.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.C +L0: + __mypyc_self__.X = 10 + return 1 +def f(): + a :: int +L0: + a = 12 + return 1 + +[case testFloatConstantFolding] +from typing_extensions import Final + +N: Final = 1.5 +N2: Final = 1.5 * 2 + +def bin_ops() -> None: + add = 0.5 + 0.5 + add_mul = (1.5 + 3.5) * 5.0 + sub = 7.0 - 7.5 + div = 3.0 / 2.0 + floor_div = 3.0 // 2.0 +def bin_ops_neg() -> None: + add = 0.5 + -0.5 + add_mul = (-1.5 + 3.5) * -5.0 + add_mul2 = (1.5 + -3.5) * -5.0 + sub = 7.0 - -7.5 + div = 3.0 / -2.0 + floor_div = 3.0 // -2.0 +def unary_ops() -> None: + neg1 = -5.5 + neg2 = --1.5 + neg3 = -0.0 + pos = +5.5 +def pow() -> None: + p0 = 16.0**0 + p1 = 16.0**0.5 + p2 = (-5.0)**3 + p3 = 16.0**(-0) + p4 = 16.0**(-0.5) + p5 = (-2.0)**(-1) +def error_cases() -> None: + div = 2.0 / 0.0 + floor_div = 2.0 // 0.0 + power_imag = (-2.0)**0.5 + power_imag2 = (-2.0)**(-0.5) + power_overflow = 2.0**10000.0 +def final_floats() -> None: + add1 = N + 1.2 + add2 = N + N2 + add3 = -1.2 + N2 +[out] +def bin_ops(): + add, add_mul, sub, div, floor_div :: float +L0: + add = 1.0 + add_mul = 25.0 + sub = -0.5 + div = 1.5 + floor_div = 1.0 + return 1 +def bin_ops_neg(): + add, add_mul, add_mul2, sub, div, floor_div :: float +L0: + add = 0.0 + add_mul = -10.0 + add_mul2 = 10.0 + sub = 14.5 + div = -1.5 + floor_div = -2.0 + return 1 +def unary_ops(): + neg1, neg2, neg3, pos :: float +L0: + neg1 = -5.5 + neg2 = 1.5 + neg3 = -0.0 + pos = 5.5 + return 1 +def pow(): + p0, p1, p2, p3, p4, p5 :: float +L0: + p0 = 1.0 + p1 = 4.0 + p2 = -125.0 + p3 = 1.0 + p4 = 0.25 + p5 = -0.5 + return 1 +def error_cases(): + r0 :: bit + r1 :: bool + r2, div, r3, floor_div :: float + r4, r5, r6 :: object + r7, power_imag :: float + r8, r9, r10 :: object + r11, power_imag2 :: float + r12, r13, r14 :: object + r15, power_overflow :: float +L0: + r0 = 0.0 == 0.0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = raise ZeroDivisionError('float division by zero') + unreachable +L2: + r2 = 2.0 / 0.0 + div = r2 + r3 = CPyFloat_FloorDivide(2.0, 0.0) + floor_div = r3 + r4 = box(float, -2.0) + r5 = box(float, 0.5) + r6 = CPyNumber_Power(r4, r5) + r7 = unbox(float, r6) + power_imag = r7 + r8 = box(float, -2.0) + r9 = box(float, -0.5) + r10 = CPyNumber_Power(r8, r9) + r11 = unbox(float, r10) + power_imag2 = r11 + r12 = box(float, 2.0) + r13 = box(float, 10000.0) + r14 = CPyNumber_Power(r12, r13) + r15 = unbox(float, r14) + power_overflow = r15 + return 1 +def final_floats(): + add1, add2, add3 :: float +L0: + add1 = 2.7 + add2 = 4.5 + add3 = 1.8 + return 1 + +[case testMixedFloatIntConstantFolding] +def bin_ops() -> None: + add = 1 + 0.5 + sub = 1 - 0.5 + mul = 0.5 * 5 + div = 5 / 0.5 + floor_div = 9.5 // 5 +def error_cases() -> None: + div = 2.0 / 0 + floor_div = 2.0 // 0 + power_overflow = 2.0**10000 +[out] +def bin_ops(): + add, sub, mul, div, floor_div :: float +L0: + add = 1.5 + sub = 0.5 + mul = 2.5 + div = 10.0 + floor_div = 1.0 + return 1 +def error_cases(): + r0 :: bit + r1 :: bool + r2, div, r3, floor_div :: float + r4, r5, r6 :: object + r7, power_overflow :: float +L0: + r0 = 0.0 == 0.0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = raise ZeroDivisionError('float division by zero') + unreachable +L2: + r2 = 2.0 / 0.0 + div = r2 + r3 = CPyFloat_FloorDivide(2.0, 0.0) + floor_div = r3 + r4 = box(float, 2.0) + r5 = box(float, 10000.0) + r6 = CPyNumber_Power(r4, r5) + r7 = unbox(float, r6) + power_overflow = r7 + return 1 + +[case testStrConstantFolding] +from typing_extensions import Final + +S: Final = 'z' +N: Final = 2 + +def f() -> None: + x = 'foo' + 'bar' + y = 'x' + 'y' + S + mul = "foobar" * 2 + mul2 = N * "foobar" +[out] +def f(): + r0, x, r1, y, r2, mul, r3, mul2 :: str +L0: + r0 = 'foobar' + x = r0 + r1 = 'xyz' + y = r1 + r2 = 'foobarfoobar' + mul = r2 + r3 = 'foobarfoobar' + mul2 = r3 + return 1 + +[case testBytesConstantFolding] +from typing_extensions import Final + +N: Final = 2 + +def f() -> None: + # Unfortunately, mypy doesn't store the bytes value of final refs. + x = b'foo' + b'bar' + mul = b"foobar" * 2 + mul2 = N * b"foobar" +[out] +def f(): + r0, x, r1, mul, r2, mul2 :: bytes +L0: + r0 = b'foobar' + x = r0 + r1 = b'foobarfoobar' + mul = r1 + r2 = b'foobarfoobar' + mul2 = r2 + return 1 + +[case testComplexConstantFolding] +from typing_extensions import Final + +N: Final = 1 +FLOAT_N: Final = 1.5 + +def integral() -> None: + pos = 1+2j + pos_2 = 2j+N + neg = 1-2j + neg_2 = 2j-N +def floating() -> None: + pos = 1.5+2j + pos_2 = 2j+FLOAT_N + neg = 1.5-2j + neg_2 = 2j-FLOAT_N +[out] +def integral(): + r0, pos, r1, pos_2, r2, neg, r3, neg_2 :: object +L0: + r0 = (1+2j) + pos = r0 + r1 = (1+2j) + pos_2 = r1 + r2 = (1-2j) + neg = r2 + r3 = (-1+2j) + neg_2 = r3 + return 1 +def floating(): + r0, pos, r1, pos_2, r2, neg, r3, neg_2 :: object +L0: + r0 = (1.5+2j) + pos = r0 + r1 = (1.5+2j) + pos_2 = r1 + r2 = (1.5-2j) + neg = r2 + r3 = (-1.5+2j) + neg_2 = r3 + return 1 diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test new file mode 100644 index 000000000000..6139a02029b9 --- /dev/null +++ b/mypyc/test-data/irbuild-dict.test @@ -0,0 +1,584 @@ +[case testDictGet] +from typing import Dict +def f(d: Dict[int, bool]) -> bool: + return d[0] +[out] +def f(d): + d :: dict + r0, r1 :: object + r2 :: bool +L0: + r0 = object 0 + r1 = CPyDict_GetItem(d, r0) + r2 = unbox(bool, r1) + return r2 + +[case testDictSet] +from typing import Dict +def f(d: Dict[int, bool]) -> None: + d[0] = False +[out] +def f(d): + d :: dict + r0, r1 :: object + r2 :: i32 + r3 :: bit +L0: + r0 = object 0 + r1 = box(bool, 0) + r2 = CPyDict_SetItem(d, r0, r1) + r3 = r2 >= 0 :: signed + return 1 + +[case testNewEmptyDict] +from typing import Dict +def f() -> None: + d = {} # type: Dict[bool, int] +[out] +def f(): + r0, d :: dict +L0: + r0 = PyDict_New() + d = r0 + return 1 + +[case testNewEmptyDictViaFunc] +from typing import Dict +def f() -> None: + d: Dict[bool, int] = dict() + +[out] +def f(): + r0, d :: dict +L0: + r0 = PyDict_New() + d = r0 + return 1 + +[case testNewDictWithValues] +def f(x: object) -> None: + d = {1: 2, '': x} +[out] +def f(x): + x :: object + r0 :: str + r1, r2 :: object + r3, d :: dict +L0: + r0 = '' + r1 = object 1 + r2 = object 2 + r3 = CPyDict_Build(2, r1, r2, r0, x) + d = r3 + return 1 + +[case testInDict] +from typing import Dict +def f(d: Dict[int, int]) -> bool: + if 4 in d: + return True + else: + return False +[out] +def f(d): + d :: dict + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool +L0: + r0 = object 4 + r1 = PyDict_Contains(d, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +L3: + unreachable + +[case testNotInDict] +from typing import Dict +def f(d: Dict[int, int]) -> bool: + if 4 not in d: + return True + else: + return False +[out] +def f(d): + d :: dict + r0 :: object + r1 :: i32 + r2 :: bit + r3, r4 :: bool +L0: + r0 = object 4 + r1 = PyDict_Contains(d, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + r4 = r3 ^ 1 + if r4 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +L3: + unreachable + +[case testDictUpdate] +from typing import Dict +def f(a: Dict[int, int], b: Dict[int, int]) -> None: + a.update(b) +[out] +def f(a, b): + a, b :: dict + r0 :: i32 + r1 :: bit +L0: + r0 = CPyDict_Update(a, b) + r1 = r0 >= 0 :: signed + return 1 + +[case testDictKeyLvalue] +from typing import Dict +def increment(d: Dict[str, int]) -> Dict[str, int]: + for k in d: + d[k] += 1 + return d +[out] +def increment(d): + d :: dict + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, k :: str + r9, r10, r11 :: object + r12 :: i32 + r13, r14, r15 :: bit +L0: + r0 = 0 + r1 = PyDict_Size(d) + r2 = r1 << 1 + r3 = CPyDict_GetKeysIter(d) +L1: + r4 = CPyDict_NextKey(r3, r0) + r5 = r4[1] + r0 = r5 + r6 = r4[0] + if r6 goto L2 else goto L4 :: bool +L2: + r7 = r4[2] + r8 = cast(str, r7) + k = r8 + r9 = CPyDict_GetItem(d, k) + r10 = object 1 + r11 = PyNumber_InPlaceAdd(r9, r10) + r12 = CPyDict_SetItem(d, k, r11) + r13 = r12 >= 0 :: signed +L3: + r14 = CPyDict_CheckSize(d, r2) + goto L1 +L4: + r15 = CPy_NoErrOccured() +L5: + return d + +[case testDictDisplay] +from typing import Dict +def f(x: str, y: Dict[str, int]) -> Dict[str, int]: + return {x: 2, **y, 'z': 3} +[out] +def f(x, y): + x :: str + y :: dict + r0 :: str + r1 :: object + r2 :: dict + r3 :: i32 + r4 :: bit + r5 :: object + r6 :: i32 + r7 :: bit +L0: + r0 = 'z' + r1 = object 2 + r2 = CPyDict_Build(1, x, r1) + r3 = CPyDict_UpdateInDisplay(r2, y) + r4 = r3 >= 0 :: signed + r5 = object 3 + r6 = CPyDict_SetItem(r2, r0, r5) + r7 = r6 >= 0 :: signed + return r2 + +[case testDictIterationMethods] +from typing import Dict, Union +from typing_extensions import TypedDict + +class Person(TypedDict): + name: str + age: int + +def print_dict_methods(d1: Dict[int, int], d2: Dict[int, int]) -> None: + for v in d1.values(): + if v in d2: + return + for k, v in d2.items(): + d2[k] += v +def union_of_dicts(d: Union[Dict[str, int], Dict[str, str]]) -> None: + new = {} + for k, v in d.items(): + new[k] = int(v) +def typeddict(d: Person) -> None: + for k, v in d.items(): + if k == "name": + name = v +[out] +def print_dict_methods(d1, d2): + d1, d2 :: dict + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, v :: int + r9 :: object + r10 :: i32 + r11 :: bit + r12 :: bool + r13, r14 :: bit + r15 :: short_int + r16 :: native_int + r17 :: short_int + r18 :: object + r19 :: tuple[bool, short_int, object, object] + r20 :: short_int + r21 :: bool + r22, r23 :: object + r24, r25, k :: int + r26, r27, r28, r29, r30 :: object + r31 :: i32 + r32, r33, r34 :: bit +L0: + r0 = 0 + r1 = PyDict_Size(d1) + r2 = r1 << 1 + r3 = CPyDict_GetValuesIter(d1) +L1: + r4 = CPyDict_NextValue(r3, r0) + r5 = r4[1] + r0 = r5 + r6 = r4[0] + if r6 goto L2 else goto L6 :: bool +L2: + r7 = r4[2] + r8 = unbox(int, r7) + v = r8 + r9 = box(int, v) + r10 = PyDict_Contains(d2, r9) + r11 = r10 >= 0 :: signed + r12 = truncate r10: i32 to builtins.bool + if r12 goto L3 else goto L4 :: bool +L3: + return 1 +L4: +L5: + r13 = CPyDict_CheckSize(d1, r2) + goto L1 +L6: + r14 = CPy_NoErrOccured() +L7: + r15 = 0 + r16 = PyDict_Size(d2) + r17 = r16 << 1 + r18 = CPyDict_GetItemsIter(d2) +L8: + r19 = CPyDict_NextItem(r18, r15) + r20 = r19[1] + r15 = r20 + r21 = r19[0] + if r21 goto L9 else goto L11 :: bool +L9: + r22 = r19[2] + r23 = r19[3] + r24 = unbox(int, r22) + r25 = unbox(int, r23) + k = r24 + v = r25 + r26 = box(int, k) + r27 = CPyDict_GetItem(d2, r26) + r28 = box(int, v) + r29 = PyNumber_InPlaceAdd(r27, r28) + r30 = box(int, k) + r31 = CPyDict_SetItem(d2, r30, r29) + r32 = r31 >= 0 :: signed +L10: + r33 = CPyDict_CheckSize(d2, r17) + goto L8 +L11: + r34 = CPy_NoErrOccured() +L12: + return 1 +def union_of_dicts(d): + d, r0, new :: dict + r1 :: short_int + r2 :: native_int + r3 :: short_int + r4 :: object + r5 :: tuple[bool, short_int, object, object] + r6 :: short_int + r7 :: bool + r8, r9 :: object + r10 :: str + r11 :: union[int, str] + k :: str + v :: union[int, str] + r12, r13 :: object + r14 :: int + r15 :: object + r16 :: i32 + r17, r18, r19 :: bit +L0: + r0 = PyDict_New() + new = r0 + r1 = 0 + r2 = PyDict_Size(d) + r3 = r2 << 1 + r4 = CPyDict_GetItemsIter(d) +L1: + r5 = CPyDict_NextItem(r4, r1) + r6 = r5[1] + r1 = r6 + r7 = r5[0] + if r7 goto L2 else goto L4 :: bool +L2: + r8 = r5[2] + r9 = r5[3] + r10 = cast(str, r8) + r11 = cast(union[int, str], r9) + k = r10 + v = r11 + r12 = load_address PyLong_Type + r13 = PyObject_CallFunctionObjArgs(r12, v, 0) + r14 = unbox(int, r13) + r15 = box(int, r14) + r16 = CPyDict_SetItem(new, k, r15) + r17 = r16 >= 0 :: signed +L3: + r18 = CPyDict_CheckSize(d, r3) + goto L1 +L4: + r19 = CPy_NoErrOccured() +L5: + return 1 +def typeddict(d): + d :: dict + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: object + r4 :: tuple[bool, short_int, object, object] + r5 :: short_int + r6 :: bool + r7, r8 :: object + r9, k :: str + v :: object + r10 :: str + r11 :: i32 + r12 :: bit + r13 :: object + r14, r15, r16 :: bit + name :: object + r17, r18 :: bit +L0: + r0 = 0 + r1 = PyDict_Size(d) + r2 = r1 << 1 + r3 = CPyDict_GetItemsIter(d) +L1: + r4 = CPyDict_NextItem(r3, r0) + r5 = r4[1] + r0 = r5 + r6 = r4[0] + if r6 goto L2 else goto L9 :: bool +L2: + r7 = r4[2] + r8 = r4[3] + r9 = cast(str, r7) + k = r9 + v = r8 + r10 = 'name' + r11 = PyUnicode_Compare(k, r10) + r12 = r11 == -1 + if r12 goto L3 else goto L5 :: bool +L3: + r13 = PyErr_Occurred() + r14 = r13 != 0 + if r14 goto L4 else goto L5 :: bool +L4: + r15 = CPy_KeepPropagating() +L5: + r16 = r11 == 0 + if r16 goto L6 else goto L7 :: bool +L6: + name = v +L7: +L8: + r17 = CPyDict_CheckSize(d, r2) + goto L1 +L9: + r18 = CPy_NoErrOccured() +L10: + return 1 + +[case testDictLoadAddress] +def f() -> None: + x = dict +[out] +def f(): + r0, x :: object +L0: + r0 = load_address PyDict_Type + x = r0 + return 1 + +[case testDictClear] +from typing import Dict +def f(d: Dict[int, int]) -> None: + return d.clear() +[out] +def f(d): + d :: dict + r0 :: bit +L0: + r0 = CPyDict_Clear(d) + return 1 + +[case testDictCopy] +from typing import Dict +def f(d: Dict[int, int]) -> Dict[int, int]: + return d.copy() +[out] +def f(d): + d, r0 :: dict +L0: + r0 = CPyDict_Copy(d) + return r0 + +[case testDictSetdefault] +from typing import Dict +def f(d: Dict[object, object]) -> object: + return d.setdefault('a', 'b') + +def f2(d: Dict[object, object], flag: bool) -> object: + if flag: + return d.setdefault('a', set()) + else: + return d.setdefault('a', set('b')) + +def f3(d: Dict[object, object], flag: bool) -> object: + if flag: + return d.setdefault('a', []) + else: + return d.setdefault('a', [1]) + +def f4(d: Dict[object, object], flag: bool) -> object: + if flag: + return d.setdefault('a', {}) + else: + return d.setdefault('a', {'c': 1}) +[out] +def f(d): + d :: dict + r0, r1 :: str + r2 :: object +L0: + r0 = 'a' + r1 = 'b' + r2 = CPyDict_SetDefault(d, r0, r1) + return r2 +def f2(d, flag): + d :: dict + flag :: bool + r0 :: str + r1 :: object + r2, r3 :: str + r4 :: set + r5, r6 :: object +L0: + if flag goto L1 else goto L2 :: bool +L1: + r0 = 'a' + r1 = CPyDict_SetDefaultWithEmptyDatatype(d, r0, 3) + return r1 +L2: + r2 = 'a' + r3 = 'b' + r4 = PySet_New(r3) + r5 = CPyDict_SetDefault(d, r2, r4) + return r5 +L3: + r6 = box(None, 1) + return r6 +def f3(d, flag): + d :: dict + flag :: bool + r0 :: str + r1 :: object + r2 :: str + r3 :: list + r4 :: object + r5 :: ptr + r6, r7 :: object +L0: + if flag goto L1 else goto L2 :: bool +L1: + r0 = 'a' + r1 = CPyDict_SetDefaultWithEmptyDatatype(d, r0, 1) + return r1 +L2: + r2 = 'a' + r3 = PyList_New(1) + r4 = object 1 + r5 = list_items r3 + buf_init_item r5, 0, r4 + keep_alive r3 + r6 = CPyDict_SetDefault(d, r2, r3) + return r6 +L3: + r7 = box(None, 1) + return r7 +def f4(d, flag): + d :: dict + flag :: bool + r0 :: str + r1 :: object + r2, r3 :: str + r4 :: object + r5 :: dict + r6, r7 :: object +L0: + if flag goto L1 else goto L2 :: bool +L1: + r0 = 'a' + r1 = CPyDict_SetDefaultWithEmptyDatatype(d, r0, 2) + return r1 +L2: + r2 = 'a' + r3 = 'c' + r4 = object 1 + r5 = CPyDict_Build(1, r3, r4) + r6 = CPyDict_SetDefault(d, r2, r5) + return r6 +L3: + r7 = box(None, 1) + return r7 diff --git a/mypyc/test-data/irbuild-dunders.test b/mypyc/test-data/irbuild-dunders.test new file mode 100644 index 000000000000..1796a7e2160e --- /dev/null +++ b/mypyc/test-data/irbuild-dunders.test @@ -0,0 +1,215 @@ +# Test cases for (some) dunder methods + +[case testDundersLen] +class C: + def __len__(self) -> int: + return 2 + +def f(c: C) -> int: + return len(c) +[out] +def C.__len__(self): + self :: __main__.C +L0: + return 4 +def f(c): + c :: __main__.C + r0 :: int + r1 :: bit + r2 :: bool +L0: + r0 = c.__len__() + r1 = int_ge r0, 0 + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise ValueError('__len__() should return >= 0') + unreachable +L2: + return r0 + +[case testDundersSetItem] +class C: + def __setitem__(self, key: int, value: int) -> None: + pass + +def f(c: C) -> None: + c[3] = 4 +[out] +def C.__setitem__(self, key, value): + self :: __main__.C + key, value :: int +L0: + return 1 +def f(c): + c :: __main__.C + r0 :: None +L0: + r0 = c.__setitem__(6, 8) + return 1 + +[case testDundersContains] +from typing import Any + +class C: + def __contains__(self, x: int) -> bool: + return False + +def f(c: C) -> bool: + return 7 in c + +def g(c: C) -> bool: + return 7 not in c + +class D: + def __contains__(self, x: int) -> Any: + return 'x' + +def h(d: D) -> bool: + return 7 not in d +[out] +def C.__contains__(self, x): + self :: __main__.C + x :: int +L0: + return 0 +def f(c): + c :: __main__.C + r0 :: bool +L0: + r0 = c.__contains__(14) + return r0 +def g(c): + c :: __main__.C + r0, r1 :: bool +L0: + r0 = c.__contains__(14) + r1 = r0 ^ 1 + return r1 +def D.__contains__(self, x): + self :: __main__.D + x :: int + r0 :: str +L0: + r0 = 'x' + return r0 +def h(d): + d :: __main__.D + r0 :: object + r1 :: i32 + r2 :: bit + r3, r4 :: bool +L0: + r0 = d.__contains__(14) + r1 = PyObject_IsTrue(r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + r4 = r3 ^ 1 + return r4 + +[case testDundersDelItem] +class C: + def __delitem__(self, x: int) -> None: + pass + +def f(c: C) -> None: + del c[5] +[out] +def C.__delitem__(self, x): + self :: __main__.C + x :: int +L0: + return 1 +def f(c): + c :: __main__.C + r0 :: None +L0: + r0 = c.__delitem__(10) + return 1 + +[case testDundersUnary] +class C: + def __neg__(self) -> int: + return 1 + + def __invert__(self) -> int: + return 2 + + def __int__(self) -> int: + return 3 + + def __float__(self) -> float: + return 4.0 + + def __pos__(self) -> int: + return 5 + + def __abs__(self) -> int: + return 6 + + def __bool__(self) -> bool: + return False + + def __complex__(self) -> complex: + return 7j + +def f(c: C) -> None: + -c + ~c + int(c) + float(c) + +c + abs(c) + bool(c) + complex(c) +[out] +def C.__neg__(self): + self :: __main__.C +L0: + return 2 +def C.__invert__(self): + self :: __main__.C +L0: + return 4 +def C.__int__(self): + self :: __main__.C +L0: + return 6 +def C.__float__(self): + self :: __main__.C +L0: + return 4.0 +def C.__pos__(self): + self :: __main__.C +L0: + return 10 +def C.__abs__(self): + self :: __main__.C +L0: + return 12 +def C.__bool__(self): + self :: __main__.C +L0: + return 0 +def C.__complex__(self): + self :: __main__.C + r0 :: object +L0: + r0 = 7j + return r0 +def f(c): + c :: __main__.C + r0, r1, r2 :: int + r3 :: float + r4, r5 :: int + r6 :: bool + r7 :: object +L0: + r0 = c.__neg__() + r1 = c.__invert__() + r2 = c.__int__() + r3 = c.__float__() + r4 = c.__pos__() + r5 = c.__abs__() + r6 = c.__bool__() + r7 = c.__complex__() + return 1 diff --git a/mypyc/test-data/irbuild-float.test b/mypyc/test-data/irbuild-float.test new file mode 100644 index 000000000000..35e2eff62b86 --- /dev/null +++ b/mypyc/test-data/irbuild-float.test @@ -0,0 +1,497 @@ +[case testFloatAdd] +def f(x: float, y: float) -> float: + return x + y +def g(x: float) -> float: + z = x - 1.5 + return 2.5 * z +[out] +def f(x, y): + x, y, r0 :: float +L0: + r0 = x + y + return r0 +def g(x): + x, r0, z, r1 :: float +L0: + r0 = x - 1.5 + z = r0 + r1 = 2.5 * z + return r1 + +[case testFloatBoxAndUnbox] +from typing import Any +def f(x: float) -> object: + return x +def g(x: Any) -> float: + return x +[out] +def f(x): + x :: float + r0 :: object +L0: + r0 = box(float, x) + return r0 +def g(x): + x :: object + r0 :: float +L0: + r0 = unbox(float, x) + return r0 + +[case testFloatNegAndPos] +def f(x: float) -> float: + y = +x * -0.5 + return -y +[out] +def f(x): + x, r0, y, r1 :: float +L0: + r0 = x * -0.5 + y = r0 + r1 = -y + return r1 + +[case testFloatCoerceFromInt] +def from_int(x: int) -> float: + return x + +def from_literal() -> float: + return 5 + +def from_literal_neg() -> float: + return -2 +[out] +def from_int(x): + x :: int + r0 :: float +L0: + r0 = CPyFloat_FromTagged(x) + return r0 +def from_literal(): +L0: + return 5.0 +def from_literal_neg(): +L0: + return -2.0 + +[case testConvertBetweenFloatAndInt] +def to_int(x: float) -> int: + return int(x) +def from_int(x: int) -> float: + return float(x) +[out] +def to_int(x): + x :: float + r0 :: int +L0: + r0 = CPyTagged_FromFloat(x) + return r0 +def from_int(x): + x :: int + r0 :: float +L0: + r0 = CPyFloat_FromTagged(x) + return r0 + +[case testFloatOperatorAssignment] +def f(x: float, y: float) -> float: + x += y + x -= 5.0 + return x +[out] +def f(x, y): + x, y, r0, r1 :: float +L0: + r0 = x + y + x = r0 + r1 = x - 5.0 + x = r1 + return x + +[case testFloatOperatorAssignmentWithInt] +def f(x: float, y: int) -> None: + x += y + x -= 5 +[out] +def f(x, y): + x :: float + y :: int + r0, r1, r2 :: float +L0: + r0 = CPyFloat_FromTagged(y) + r1 = x + r0 + x = r1 + r2 = x - 5.0 + x = r2 + return 1 + +[case testFloatComparison] +def lt(x: float, y: float) -> bool: + return x < y +def eq(x: float, y: float) -> bool: + return x == y +[out] +def lt(x, y): + x, y :: float + r0 :: bit +L0: + r0 = x < y + return r0 +def eq(x, y): + x, y :: float + r0 :: bit +L0: + r0 = x == y + return r0 + +[case testFloatOpWithLiteralInt] +def f(x: float) -> None: + y = x * 2 + z = 1 - y + b = z < 3 + c = 0 == z +[out] +def f(x): + x, r0, y, r1, z :: float + r2 :: bit + b :: bool + r3 :: bit + c :: bool +L0: + r0 = x * 2.0 + y = r0 + r1 = 1.0 - y + z = r1 + r2 = z < 3.0 + b = r2 + r3 = 0.0 == z + c = r3 + return 1 + +[case testFloatCallFunctionWithLiteralInt] +def f(x: float) -> None: pass + +def g() -> None: + f(3) + f(-2) +[out] +def f(x): + x :: float +L0: + return 1 +def g(): + r0, r1 :: None +L0: + r0 = f(3.0) + r1 = f(-2.0) + return 1 + +[case testFloatAsBool] +def f(x: float) -> int: + if x: + return 2 + else: + return 5 +[out] +def f(x): + x :: float + r0 :: bit +L0: + r0 = x != 0.0 + if r0 goto L1 else goto L2 :: bool +L1: + return 4 +L2: + return 10 +L3: + unreachable + +[case testCallSqrtViaMathModule] +import math + +def f(x: float) -> float: + return math.sqrt(x) +[out] +def f(x): + x, r0 :: float +L0: + r0 = CPyFloat_Sqrt(x) + return r0 + +[case testFloatFinalConstant] +from typing_extensions import Final + +X: Final = 123.0 +Y: Final = -1.0 + +def f() -> float: + a = X + return a + Y +[out] +def f(): + a, r0 :: float +L0: + a = 123.0 + r0 = a + -1.0 + return r0 + +[case testFloatDefaultArg] +def f(x: float = 1.5) -> float: + return x +[out] +def f(x, __bitmap): + x :: float + __bitmap, r0 :: u32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 1.5 +L2: + return x + +[case testFloatMixedOperations] +def f(x: float, y: int) -> None: + if x < y: + z = x + y + x -= y + z = y + z + if y == x: + x -= 1 +[out] +def f(x, y): + x :: float + y :: int + r0 :: float + r1 :: bit + r2, r3, z, r4, r5, r6, r7, r8 :: float + r9 :: bit + r10 :: float +L0: + r0 = CPyFloat_FromTagged(y) + r1 = x < r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = CPyFloat_FromTagged(y) + r3 = x + r2 + z = r3 + r4 = CPyFloat_FromTagged(y) + r5 = x - r4 + x = r5 + r6 = CPyFloat_FromTagged(y) + r7 = r6 + z + z = r7 +L2: + r8 = CPyFloat_FromTagged(y) + r9 = r8 == x + if r9 goto L3 else goto L4 :: bool +L3: + r10 = x - 1.0 + x = r10 +L4: + return 1 + +[case testFloatDivideSimple] +def f(x: float, y: float) -> float: + z = x / y + z = z / 2.0 + return z / 3 +[out] +def f(x, y): + x, y :: float + r0 :: bit + r1 :: bool + r2, z, r3, r4 :: float +L0: + r0 = y == 0.0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = raise ZeroDivisionError('float division by zero') + unreachable +L2: + r2 = x / y + z = r2 + r3 = z / 2.0 + z = r3 + r4 = z / 3.0 + return r4 + +[case testFloatDivideIntOperand] +def f(n: int, m: int) -> float: + return n / m +[out] +def f(n, m): + n, m :: int + r0 :: float +L0: + r0 = CPyTagged_TrueDivide(n, m) + return r0 + +[case testFloatResultOfIntDivide] +def f(f: float, n: int) -> float: + x = f / n + return n / x +[out] +def f(f, n): + f :: float + n :: int + r0 :: float + r1 :: bit + r2 :: bool + r3, x, r4 :: float + r5 :: bit + r6 :: bool + r7 :: float +L0: + r0 = CPyFloat_FromTagged(n) + r1 = r0 == 0.0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = raise ZeroDivisionError('float division by zero') + unreachable +L2: + r3 = f / r0 + x = r3 + r4 = CPyFloat_FromTagged(n) + r5 = x == 0.0 + if r5 goto L3 else goto L4 :: bool +L3: + r6 = raise ZeroDivisionError('float division by zero') + unreachable +L4: + r7 = r4 / x + return r7 + +[case testFloatExplicitConversions] +def f(f: float, n: int) -> int: + x = float(n) + y = float(x) # no-op + return int(y) +[out] +def f(f, n): + f :: float + n :: int + r0, x, y :: float + r1 :: int +L0: + r0 = CPyFloat_FromTagged(n) + x = r0 + y = x + r1 = CPyTagged_FromFloat(y) + return r1 + +[case testFloatModulo] +def f(x: float, y: float) -> float: + return x % y +[out] +def f(x, y): + x, y :: float + r0 :: bit + r1 :: bool + r2, r3 :: float + r4, r5, r6, r7 :: bit + r8, r9 :: float +L0: + r0 = y == 0.0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = raise ZeroDivisionError('float modulo') + unreachable +L2: + r2 = x % y + r3 = r2 + r4 = r3 == 0.0 + if r4 goto L5 else goto L3 :: bool +L3: + r5 = x < 0.0 + r6 = y < 0.0 + r7 = r5 == r6 + if r7 goto L6 else goto L4 :: bool +L4: + r8 = r3 + y + r3 = r8 + goto L6 +L5: + r9 = copysign(0.0, y) + r3 = r9 +L6: + return r3 + +[case testFloatFloorDivide] +def f(x: float, y: float) -> float: + return x // y +def g(x: float, y: int) -> float: + return x // y +[out] +def f(x, y): + x, y, r0 :: float +L0: + r0 = CPyFloat_FloorDivide(x, y) + return r0 +def g(x, y): + x :: float + y :: int + r0, r1 :: float +L0: + r0 = CPyFloat_FromTagged(y) + r1 = CPyFloat_FloorDivide(x, r0) + return r1 + +[case testFloatNarrowToIntDisallowed] +class C: + x: float + +def narrow_local(x: float, n: int) -> int: + x = n # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + return x + +def narrow_tuple_lvalue(x: float, y: float, n: int) -> int: + x, y = 1.0, n # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + return y + +def narrow_multiple_lvalues(x: float, y: float, n: int) -> int: + x = a = n # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + a = y = n # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + return x + y + +def narrow_attribute(c: C, n: int) -> int: + c.x = n # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + return c.x + +def narrow_using_int_literal(x: float) -> int: + x = 1 # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + return x + +def narrow_using_declaration(n: int) -> int: + x: float + x = n # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + return x + +[case testFloatInitializeFromInt] +def init(n: int) -> None: + # These are strictly speaking safe, since these don't narrow, but for consistency with + # narrowing assignments, generate errors here + x: float = n # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + y: float = 5 # E: Incompatible value representations in assignment (expression has type "int", variable has type "float") + +[case testFloatCoerceTupleFromIntValues] +from __future__ import annotations + +def f(x: int) -> None: + t: tuple[float, float, float] = (x, 2.5, -7) +[out] +def f(x): + x :: int + r0 :: tuple[int, float, int] + r1 :: int + r2 :: float + r3, t :: tuple[float, float, float] +L0: + r0 = (x, 2.5, -14) + r1 = r0[0] + r2 = CPyFloat_FromTagged(r1) + r3 = (r2, 2.5, -7.0) + t = r3 + return 1 diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test new file mode 100644 index 000000000000..910148f80dda --- /dev/null +++ b/mypyc/test-data/irbuild-generics.test @@ -0,0 +1,191 @@ +[case testGenericFunction] +from typing import TypeVar, List +T = TypeVar('T') +def f(x: T) -> T: + return x +def g(x: List[T]) -> List[T]: + return [x[0]] +def h(x: int, y: List[int]) -> None: + x = f(x) + y = g(y) +[out] +def f(x): + x :: object +L0: + return x +def g(x): + x :: list + r0 :: object + r1 :: list + r2 :: ptr +L0: + r0 = CPyList_GetItemShort(x, 0) + r1 = PyList_New(1) + r2 = list_items r1 + buf_init_item r2, 0, r0 + keep_alive r1 + return r1 +def h(x, y): + x :: int + y :: list + r0, r1 :: object + r2 :: int + r3 :: list +L0: + r0 = box(int, x) + r1 = f(r0) + r2 = unbox(int, r1) + x = r2 + r3 = g(y) + y = r3 + return 1 + +[case testGenericAttrAndTypeApplication] +from typing import TypeVar, Generic +T = TypeVar('T') +class C(Generic[T]): + x: T +def f() -> None: + c = C[int]() + c.x = 1 + 2 + c.x +[out] +def f(): + r0, c :: __main__.C + r1 :: object + r2 :: bool + r3 :: object + r4, r5 :: int +L0: + r0 = C() + c = r0 + r1 = object 1 + c.x = r1; r2 = is_error + r3 = borrow c.x + r4 = unbox(int, r3) + r5 = CPyTagged_Add(4, r4) + keep_alive c + return 1 + +[case testGenericMethod] +from typing import TypeVar, Generic +T = TypeVar('T') +class C(Generic[T]): + x: T + def __init__(self, x: T) -> None: + self.x = x + def get(self) -> T: + return self.x + def set(self, y: T) -> None: + self.x = y +def f(x: C[int]) -> None: + y = x.get() + x.set(y + 1) + x = C(2) +[out] +def C.__init__(self, x): + self :: __main__.C + x :: object + r0 :: bool +L0: + self.x = x; r0 = is_error + return 1 +def C.get(self): + self :: __main__.C + r0 :: object +L0: + r0 = self.x + return r0 +def C.set(self, y): + self :: __main__.C + y :: object + r0 :: bool +L0: + self.x = y; r0 = is_error + return 1 +def f(x): + x :: __main__.C + r0 :: object + r1, y, r2 :: int + r3 :: object + r4 :: None + r5 :: object + r6 :: __main__.C +L0: + r0 = x.get() + r1 = unbox(int, r0) + y = r1 + r2 = CPyTagged_Add(y, 2) + r3 = box(int, r2) + r4 = x.set(r3) + r5 = object 2 + r6 = C(r5) + x = r6 + return 1 + +[case testMax] +from typing import TypeVar +T = TypeVar('T') +def f(x: T, y: T) -> T: + return max(x, y) +[out] +def f(x, y): + x, y, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: object +L0: + r0 = PyObject_RichCompare(y, x, 4) + r1 = PyObject_IsTrue(r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool +L1: + r4 = y + goto L3 +L2: + r4 = x +L3: + return r4 + + +[case testParamSpec] +from typing import Callable, ParamSpec, TypeVar + +P = ParamSpec("P") + +def execute(func: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> int: + return func(*args, **kwargs) + +def f(x: int) -> int: + return x + +execute(f, 1) +[out] +def execute(func, args, kwargs): + func :: object + args :: tuple + kwargs :: dict + r0 :: list + r1 :: object + r2 :: dict + r3 :: i32 + r4 :: bit + r5 :: tuple + r6 :: object + r7 :: int +L0: + r0 = PyList_New(0) + r1 = CPyList_Extend(r0, args) + r2 = PyDict_New() + r3 = CPyDict_UpdateInDisplay(r2, kwargs) + r4 = r3 >= 0 :: signed + r5 = PyList_AsTuple(r0) + r6 = PyObject_Call(func, r5, r2) + r7 = unbox(int, r6) + return r7 +def f(x): + x :: int +L0: + return x diff --git a/mypyc/test-data/irbuild-glue-methods.test b/mypyc/test-data/irbuild-glue-methods.test new file mode 100644 index 000000000000..3012c79586f2 --- /dev/null +++ b/mypyc/test-data/irbuild-glue-methods.test @@ -0,0 +1,437 @@ +# Test cases for glue methods. +# +# These are used when subclass method signature has a different representation +# compared to the base class. + +[case testSubclassSpecialize2] +class A: + def foo(self, x: int) -> object: + return str(x) +class B(A): + def foo(self, x: object) -> object: + return x +class C(B): + def foo(self, x: object) -> int: + return id(x) + +def use_a(x: A, y: int) -> object: + return x.foo(y) + +def use_b(x: B, y: object) -> object: + return x.foo(y) + +def use_c(x: C, y: object) -> int: + return x.foo(y) +[out] +def A.foo(self, x): + self :: __main__.A + x :: int + r0 :: str +L0: + r0 = CPyTagged_Str(x) + return r0 +def B.foo(self, x): + self :: __main__.B + x :: object +L0: + return x +def B.foo__A_glue(self, x): + self :: __main__.B + x :: int + r0, r1 :: object +L0: + r0 = box(int, x) + r1 = B.foo(self, r0) + return r1 +def C.foo(self, x): + self :: __main__.C + x :: object + r0 :: int +L0: + r0 = CPyTagged_Id(x) + return r0 +def C.foo__B_glue(self, x): + self :: __main__.C + x :: object + r0 :: int + r1 :: object +L0: + r0 = C.foo(self, x) + r1 = box(int, r0) + return r1 +def C.foo__A_glue(self, x): + self :: __main__.C + x :: int + r0 :: object + r1 :: int + r2 :: object +L0: + r0 = box(int, x) + r1 = C.foo(self, r0) + r2 = box(int, r1) + return r2 +def use_a(x, y): + x :: __main__.A + y :: int + r0 :: object +L0: + r0 = x.foo(y) + return r0 +def use_b(x, y): + x :: __main__.B + y, r0 :: object +L0: + r0 = x.foo(y) + return r0 +def use_c(x, y): + x :: __main__.C + y :: object + r0 :: int +L0: + r0 = x.foo(y) + return r0 + +[case testPropertyDerivedGen] +from typing import Callable +class BaseProperty: + @property + def value(self) -> object: + return self._incrementer + + @property + def bad_value(self) -> object: + return self._incrementer + + @property + def next(self) -> BaseProperty: + return BaseProperty(self._incrementer + 1) + + def __init__(self, value: int) -> None: + self._incrementer = value + +class DerivedProperty(BaseProperty): + @property + def value(self) -> int: + return self._incrementer + + @property + def bad_value(self) -> object: + return self._incrementer + + @property + def next(self) -> DerivedProperty: + return DerivedProperty(self._incr_func, self._incr_func(self.value)) + + def __init__(self, incr_func: Callable[[int], int], value: int) -> None: + BaseProperty.__init__(self, value) + self._incr_func = incr_func + + +class AgainProperty(DerivedProperty): + @property + def next(self) -> AgainProperty: + return AgainProperty(self._incr_func, self._incr_func(self._incr_func(self.value))) + + @property + def bad_value(self) -> int: + return self._incrementer +[out] +def BaseProperty.value(self): + self :: __main__.BaseProperty + r0 :: int + r1 :: object +L0: + r0 = self._incrementer + r1 = box(int, r0) + return r1 +def BaseProperty.bad_value(self): + self :: __main__.BaseProperty + r0 :: int + r1 :: object +L0: + r0 = self._incrementer + r1 = box(int, r0) + return r1 +def BaseProperty.next(self): + self :: __main__.BaseProperty + r0, r1 :: int + r2 :: __main__.BaseProperty +L0: + r0 = borrow self._incrementer + r1 = CPyTagged_Add(r0, 2) + keep_alive self + r2 = BaseProperty(r1) + return r2 +def BaseProperty.__init__(self, value): + self :: __main__.BaseProperty + value :: int +L0: + self._incrementer = value + return 1 +def DerivedProperty.value(self): + self :: __main__.DerivedProperty + r0 :: int +L0: + r0 = self._incrementer + return r0 +def DerivedProperty.value__BaseProperty_glue(__mypyc_self__): + __mypyc_self__ :: __main__.DerivedProperty + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.value + r1 = box(int, r0) + return r1 +def DerivedProperty.bad_value(self): + self :: __main__.DerivedProperty + r0 :: int + r1 :: object +L0: + r0 = self._incrementer + r1 = box(int, r0) + return r1 +def DerivedProperty.next(self): + self :: __main__.DerivedProperty + r0 :: object + r1 :: int + r2, r3, r4 :: object + r5 :: int + r6 :: __main__.DerivedProperty +L0: + r0 = self._incr_func + r1 = self.value + r2 = self._incr_func + r3 = box(int, r1) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = unbox(int, r4) + r6 = DerivedProperty(r0, r5) + return r6 +def DerivedProperty.next__BaseProperty_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.DerivedProperty +L0: + r0 = __mypyc_self__.next + return r0 +def DerivedProperty.__init__(self, incr_func, value): + self :: __main__.DerivedProperty + incr_func :: object + value :: int + r0 :: None +L0: + r0 = BaseProperty.__init__(self, value) + self._incr_func = incr_func + return 1 +def AgainProperty.next(self): + self :: __main__.AgainProperty + r0 :: object + r1 :: int + r2, r3, r4 :: object + r5 :: int + r6, r7, r8 :: object + r9 :: int + r10 :: __main__.AgainProperty +L0: + r0 = self._incr_func + r1 = self.value + r2 = self._incr_func + r3 = box(int, r1) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = unbox(int, r4) + r6 = self._incr_func + r7 = box(int, r5) + r8 = PyObject_CallFunctionObjArgs(r6, r7, 0) + r9 = unbox(int, r8) + r10 = AgainProperty(r0, r9) + return r10 +def AgainProperty.next__DerivedProperty_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.AgainProperty +L0: + r0 = __mypyc_self__.next + return r0 +def AgainProperty.next__BaseProperty_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.AgainProperty +L0: + r0 = __mypyc_self__.next + return r0 +def AgainProperty.bad_value(self): + self :: __main__.AgainProperty + r0 :: int +L0: + r0 = self._incrementer + return r0 +def AgainProperty.bad_value__DerivedProperty_glue(__mypyc_self__): + __mypyc_self__ :: __main__.AgainProperty + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.bad_value + r1 = box(int, r0) + return r1 +def AgainProperty.bad_value__BaseProperty_glue(__mypyc_self__): + __mypyc_self__ :: __main__.AgainProperty + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.bad_value + r1 = box(int, r0) + return r1 + +[case testPropertyTraitSubclassing] +from mypy_extensions import trait +@trait +class SubclassedTrait: + @property + def this(self) -> SubclassedTrait: + return self + + @property + def boxed(self) -> object: + return 3 + +class DerivingObject(SubclassedTrait): + @property + def this(self) -> DerivingObject: + return self + + @property + def boxed(self) -> int: + return 5 +[out] +def SubclassedTrait.this(self): + self :: __main__.SubclassedTrait +L0: + return self +def SubclassedTrait.boxed(self): + self :: __main__.SubclassedTrait + r0 :: object +L0: + r0 = object 3 + return r0 +def DerivingObject.this(self): + self :: __main__.DerivingObject +L0: + return self +def DerivingObject.this__SubclassedTrait_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.DerivingObject +L0: + r0 = __mypyc_self__.this + return r0 +def DerivingObject.boxed(self): + self :: __main__.DerivingObject +L0: + return 10 +def DerivingObject.boxed__SubclassedTrait_glue(__mypyc_self__): + __mypyc_self__ :: __main__.DerivingObject + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.boxed + r1 = box(int, r0) + return r1 + +[case testI64GlueWithExtraDefaultArg] +from mypy_extensions import i64 + +class C: + def f(self) -> None: pass + +class D(C): + def f(self, x: i64 = 44) -> None: pass +[out] +def C.f(self): + self :: __main__.C +L0: + return 1 +def D.f(self, x, __bitmap): + self :: __main__.D + x :: i64 + __bitmap, r0 :: u32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 44 +L2: + return 1 +def D.f__C_glue(self): + self :: __main__.D + r0 :: None +L0: + r0 = D.f(self, 0, 0) + return r0 + +[case testI64GlueWithSecondDefaultArg] +from mypy_extensions import i64 + +class C: + def f(self, x: i64 = 11) -> None: pass +class D(C): + def f(self, x: i64 = 12, y: i64 = 13) -> None: pass +[out] +def C.f(self, x, __bitmap): + self :: __main__.C + x :: i64 + __bitmap, r0 :: u32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 11 +L2: + return 1 +def D.f(self, x, y, __bitmap): + self :: __main__.D + x, y :: i64 + __bitmap, r0 :: u32 + r1 :: bit + r2 :: u32 + r3 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 12 +L2: + r2 = __bitmap & 2 + r3 = r2 == 0 + if r3 goto L3 else goto L4 :: bool +L3: + y = 13 +L4: + return 1 +def D.f__C_glue(self, x, __bitmap): + self :: __main__.D + x :: i64 + __bitmap :: u32 + r0 :: None +L0: + r0 = D.f(self, x, 0, __bitmap) + return r0 + +[case testI64GlueWithInvalidOverride] +from mypy_extensions import i64 + +class C: + def f(self, x: i64, y: i64 = 5) -> None: pass + def ff(self, x: int) -> None: pass +class CC(C): + def f(self, x: i64 = 12, y: i64 = 5) -> None: pass # Line 7 + def ff(self, x: int = 12) -> None: pass + +class D: + def f(self, x: int) -> None: pass +class DD(D): + def f(self, x: i64) -> None: pass # Line 13 + +class E: + def f(self, x: i64) -> None: pass +class EE(E): + def f(self, x: int) -> None: pass # Line 18 +[out] +main:7: error: An argument with type "i64" cannot be given a default value in a method override +main:13: error: Incompatible argument type "i64" (base class has type "int") +main:18: error: Incompatible argument type "int" (base class has type "i64") diff --git a/mypyc/test-data/irbuild-i16.test b/mypyc/test-data/irbuild-i16.test new file mode 100644 index 000000000000..a03c9df2c6ac --- /dev/null +++ b/mypyc/test-data/irbuild-i16.test @@ -0,0 +1,526 @@ +# Test cases for i16 native ints. Focus on things that are different from i64; no need to +# duplicate all i64 test cases here. + +[case testI16BinaryOp] +from mypy_extensions import i16 + +def add_op(x: i16, y: i16) -> i16: + x = y + x + y = x + 5 + y += x + y += 7 + x = 5 + y + return x +def compare(x: i16, y: i16) -> None: + a = x == y + b = x == -5 + c = x < y + d = x < -5 + e = -5 == x + f = -5 < x +[out] +def add_op(x, y): + x, y, r0, r1, r2, r3, r4 :: i16 +L0: + r0 = y + x + x = r0 + r1 = x + 5 + y = r1 + r2 = y + x + y = r2 + r3 = y + 7 + y = r3 + r4 = 5 + y + x = r4 + return x +def compare(x, y): + x, y :: i16 + r0 :: bit + a :: bool + r1 :: bit + b :: bool + r2 :: bit + c :: bool + r3 :: bit + d :: bool + r4 :: bit + e :: bool + r5 :: bit + f :: bool +L0: + r0 = x == y + a = r0 + r1 = x == -5 + b = r1 + r2 = x < y :: signed + c = r2 + r3 = x < -5 :: signed + d = r3 + r4 = -5 == x + e = r4 + r5 = -5 < x :: signed + f = r5 + return 1 + +[case testI16UnaryOp] +from mypy_extensions import i16 + +def unary(x: i16) -> i16: + y = -x + x = ~y + y = +x + return y +[out] +def unary(x): + x, r0, y, r1 :: i16 +L0: + r0 = 0 - x + y = r0 + r1 = y ^ -1 + x = r1 + y = x + return y + +[case testI16DivisionByConstant] +from mypy_extensions import i16 + +def div_by_constant(x: i16) -> i16: + x = x // 5 + x //= 17 + return x +[out] +def div_by_constant(x): + x, r0, r1 :: i16 + r2, r3, r4 :: bit + r5 :: i16 + r6 :: bit + r7, r8, r9 :: i16 + r10, r11, r12 :: bit + r13 :: i16 + r14 :: bit + r15 :: i16 +L0: + r0 = x / 5 + r1 = r0 + r2 = x < 0 :: signed + r3 = 5 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 * 5 + r6 = r5 == x + if r6 goto L3 else goto L2 :: bool +L2: + r7 = r1 - 1 + r1 = r7 +L3: + x = r1 + r8 = x / 17 + r9 = r8 + r10 = x < 0 :: signed + r11 = 17 < 0 :: signed + r12 = r10 == r11 + if r12 goto L6 else goto L4 :: bool +L4: + r13 = r9 * 17 + r14 = r13 == x + if r14 goto L6 else goto L5 :: bool +L5: + r15 = r9 - 1 + r9 = r15 +L6: + x = r9 + return x + +[case testI16ModByConstant] +from mypy_extensions import i16 + +def mod_by_constant(x: i16) -> i16: + x = x % 5 + x %= 17 + return x +[out] +def mod_by_constant(x): + x, r0, r1 :: i16 + r2, r3, r4, r5 :: bit + r6, r7, r8 :: i16 + r9, r10, r11, r12 :: bit + r13 :: i16 +L0: + r0 = x % 5 + r1 = r0 + r2 = x < 0 :: signed + r3 = 5 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 == 0 + if r5 goto L3 else goto L2 :: bool +L2: + r6 = r1 + 5 + r1 = r6 +L3: + x = r1 + r7 = x % 17 + r8 = r7 + r9 = x < 0 :: signed + r10 = 17 < 0 :: signed + r11 = r9 == r10 + if r11 goto L6 else goto L4 :: bool +L4: + r12 = r8 == 0 + if r12 goto L6 else goto L5 :: bool +L5: + r13 = r8 + 17 + r8 = r13 +L6: + x = r8 + return x + +[case testI16DivModByVariable] +from mypy_extensions import i16 + +def divmod(x: i16, y: i16) -> i16: + a = x // y + return a % y +[out] +def divmod(x, y): + x, y, r0, a, r1 :: i16 +L0: + r0 = CPyInt16_Divide(x, y) + a = r0 + r1 = CPyInt16_Remainder(a, y) + return r1 + +[case testI16BinaryOperationWithOutOfRangeOperand] +from mypy_extensions import i16 + +def out_of_range(x: i16) -> None: + x + (-32769) + (-32770) + x + x * 32768 + x + 32767 # OK + (-32768) + x # OK +[out] +main:4: error: Value -32769 is out of range for "i16" +main:5: error: Value -32770 is out of range for "i16" +main:6: error: Value 32768 is out of range for "i16" + +[case testI16BoxAndUnbox] +from typing import Any +from mypy_extensions import i16 + +def f(x: Any) -> Any: + y: i16 = x + return y +[out] +def f(x): + x :: object + r0, y :: i16 + r1 :: object +L0: + r0 = unbox(i16, x) + y = r0 + r1 = box(i16, y) + return r1 + +[case testI16MixedCompare1] +from mypy_extensions import i16 +def f(x: int, y: i16) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: i16 + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: i16 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 65536 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -65536 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to i16 + r6 = r5 + goto L5 +L4: + CPyInt16_Overflow() + unreachable +L5: + r7 = r6 == y + return r7 + +[case testI16MixedCompare2] +from mypy_extensions import i16 +def f(x: i16, y: int) -> bool: + return x == y +[out] +def f(x, y): + x :: i16 + y :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: i16 + r7 :: bit +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = y < 65536 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = y >= -65536 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = y >> 1 + r5 = truncate r4: native_int to i16 + r6 = r5 + goto L5 +L4: + CPyInt16_Overflow() + unreachable +L5: + r7 = x == r6 + return r7 + +[case testI16ConvertToInt] +from mypy_extensions import i16 + +def i16_to_int(a: i16) -> int: + return a +[out] +def i16_to_int(a): + a :: i16 + r0 :: native_int + r1 :: int +L0: + r0 = extend signed a: i16 to native_int + r1 = r0 << 1 + return r1 + +[case testI16OperatorAssignmentMixed] +from mypy_extensions import i16 + +def f(a: i16) -> None: + x = 0 + x += a +[out] +def f(a): + a :: i16 + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6, r7 :: i16 + r8 :: native_int + r9 :: int +L0: + x = 0 + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 65536 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -65536 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to i16 + r6 = r5 + goto L5 +L4: + CPyInt16_Overflow() + unreachable +L5: + r7 = r6 + a + r8 = extend signed r7: i16 to native_int + r9 = r8 << 1 + x = r9 + return 1 + +[case testI16InitializeFromLiteral] +from mypy_extensions import i16, i64 + +def f() -> None: + x: i16 = 0 + y: i16 = -127 + z: i16 = 5 + 7 +[out] +def f(): + x, y, z :: i16 +L0: + x = 0 + y = -127 + z = 12 + return 1 + +[case testI16ExplicitConversionFromNativeInt] +from mypy_extensions import i64, i32, i16 + +def from_i16(x: i16) -> i16: + return i16(x) + +def from_i32(x: i32) -> i16: + return i16(x) + +def from_i64(x: i64) -> i16: + return i16(x) +[out] +def from_i16(x): + x :: i16 +L0: + return x +def from_i32(x): + x :: i32 + r0 :: i16 +L0: + r0 = truncate x: i32 to i16 + return r0 +def from_i64(x): + x :: i64 + r0 :: i16 +L0: + r0 = truncate x: i64 to i16 + return r0 + +[case testI16ExplicitConversionFromInt] +from mypy_extensions import i16 + +def f(x: int) -> i16: + return i16(x) +[out] +def f(x): + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: i16 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 65536 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -65536 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to i16 + r6 = r5 + goto L5 +L4: + CPyInt16_Overflow() + unreachable +L5: + return r6 + +[case testI16ExplicitConversionFromLiteral] +from mypy_extensions import i16 + +def f() -> None: + x = i16(0) + y = i16(11) + z = i16(-3) + a = i16(32767) + b = i16(32768) # Truncate + c = i16(-32768) + d = i16(-32769) # Truncate +[out] +def f(): + x, y, z, a, b, c, d :: i16 +L0: + x = 0 + y = 11 + z = -3 + a = 32767 + b = -32768 + c = -32768 + d = 32767 + return 1 + +[case testI16ExplicitConversionFromVariousTypes] +from mypy_extensions import i16 + +def bool_to_i16(b: bool) -> i16: + return i16(b) + +def str_to_i16(s: str) -> i16: + return i16(s) + +class C: + def __int__(self) -> i16: + return 5 + +def instance_to_i16(c: C) -> i16: + return i16(c) + +def float_to_i16(x: float) -> i16: + return i16(x) +[out] +def bool_to_i16(b): + b :: bool + r0 :: i16 +L0: + r0 = extend b: builtins.bool to i16 + return r0 +def str_to_i16(s): + s :: str + r0 :: object + r1 :: i16 +L0: + r0 = CPyLong_FromStr(s) + r1 = unbox(i16, r0) + return r1 +def C.__int__(self): + self :: __main__.C +L0: + return 5 +def instance_to_i16(c): + c :: __main__.C + r0 :: i16 +L0: + r0 = c.__int__() + return r0 +def float_to_i16(x): + x :: float + r0 :: int + r1 :: native_int + r2, r3, r4 :: bit + r5 :: native_int + r6, r7 :: i16 +L0: + r0 = CPyTagged_FromFloat(x) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L4 :: bool +L1: + r3 = r0 < 65536 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = r0 >= -65536 :: signed + if r4 goto L3 else goto L4 :: bool +L3: + r5 = r0 >> 1 + r6 = truncate r5: native_int to i16 + r7 = r6 + goto L5 +L4: + CPyInt16_Overflow() + unreachable +L5: + return r7 diff --git a/mypyc/test-data/irbuild-i32.test b/mypyc/test-data/irbuild-i32.test new file mode 100644 index 000000000000..7dcb722ec906 --- /dev/null +++ b/mypyc/test-data/irbuild-i32.test @@ -0,0 +1,598 @@ +# Test cases for i32 native ints. Focus on things that are different from i64; no need to +# duplicate all i64 test cases here. + +[case testI32BinaryOp] +from mypy_extensions import i32 + +def add_op(x: i32, y: i32) -> i32: + x = y + x + y = x + 5 + y += x + y += 7 + x = 5 + y + return x +def compare(x: i32, y: i32) -> None: + a = x == y + b = x == -5 + c = x < y + d = x < -5 + e = -5 == x + f = -5 < x +[out] +def add_op(x, y): + x, y, r0, r1, r2, r3, r4 :: i32 +L0: + r0 = y + x + x = r0 + r1 = x + 5 + y = r1 + r2 = y + x + y = r2 + r3 = y + 7 + y = r3 + r4 = 5 + y + x = r4 + return x +def compare(x, y): + x, y :: i32 + r0 :: bit + a :: bool + r1 :: bit + b :: bool + r2 :: bit + c :: bool + r3 :: bit + d :: bool + r4 :: bit + e :: bool + r5 :: bit + f :: bool +L0: + r0 = x == y + a = r0 + r1 = x == -5 + b = r1 + r2 = x < y :: signed + c = r2 + r3 = x < -5 :: signed + d = r3 + r4 = -5 == x + e = r4 + r5 = -5 < x :: signed + f = r5 + return 1 + +[case testI32UnaryOp] +from mypy_extensions import i32 + +def unary(x: i32) -> i32: + y = -x + x = ~y + y = +x + return y +[out] +def unary(x): + x, r0, y, r1 :: i32 +L0: + r0 = 0 - x + y = r0 + r1 = y ^ -1 + x = r1 + y = x + return y + +[case testI32DivisionByConstant] +from mypy_extensions import i32 + +def div_by_constant(x: i32) -> i32: + x = x // 5 + x //= 17 + return x +[out] +def div_by_constant(x): + x, r0, r1 :: i32 + r2, r3, r4 :: bit + r5 :: i32 + r6 :: bit + r7, r8, r9 :: i32 + r10, r11, r12 :: bit + r13 :: i32 + r14 :: bit + r15 :: i32 +L0: + r0 = x / 5 + r1 = r0 + r2 = x < 0 :: signed + r3 = 5 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 * 5 + r6 = r5 == x + if r6 goto L3 else goto L2 :: bool +L2: + r7 = r1 - 1 + r1 = r7 +L3: + x = r1 + r8 = x / 17 + r9 = r8 + r10 = x < 0 :: signed + r11 = 17 < 0 :: signed + r12 = r10 == r11 + if r12 goto L6 else goto L4 :: bool +L4: + r13 = r9 * 17 + r14 = r13 == x + if r14 goto L6 else goto L5 :: bool +L5: + r15 = r9 - 1 + r9 = r15 +L6: + x = r9 + return x + +[case testI32ModByConstant] +from mypy_extensions import i32 + +def mod_by_constant(x: i32) -> i32: + x = x % 5 + x %= 17 + return x +[out] +def mod_by_constant(x): + x, r0, r1 :: i32 + r2, r3, r4, r5 :: bit + r6, r7, r8 :: i32 + r9, r10, r11, r12 :: bit + r13 :: i32 +L0: + r0 = x % 5 + r1 = r0 + r2 = x < 0 :: signed + r3 = 5 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 == 0 + if r5 goto L3 else goto L2 :: bool +L2: + r6 = r1 + 5 + r1 = r6 +L3: + x = r1 + r7 = x % 17 + r8 = r7 + r9 = x < 0 :: signed + r10 = 17 < 0 :: signed + r11 = r9 == r10 + if r11 goto L6 else goto L4 :: bool +L4: + r12 = r8 == 0 + if r12 goto L6 else goto L5 :: bool +L5: + r13 = r8 + 17 + r8 = r13 +L6: + x = r8 + return x + +[case testI32DivModByVariable] +from mypy_extensions import i32 + +def divmod(x: i32, y: i32) -> i32: + a = x // y + return a % y +[out] +def divmod(x, y): + x, y, r0, a, r1 :: i32 +L0: + r0 = CPyInt32_Divide(x, y) + a = r0 + r1 = CPyInt32_Remainder(a, y) + return r1 + +[case testI32BoxAndUnbox] +from typing import Any +from mypy_extensions import i32 + +def f(x: Any) -> Any: + y: i32 = x + return y +[out] +def f(x): + x :: object + r0, y :: i32 + r1 :: object +L0: + r0 = unbox(i32, x) + y = r0 + r1 = box(i32, y) + return r1 + +[case testI32MixedCompare1_64bit] +from mypy_extensions import i32 +def f(x: int, y: i32) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: i32 + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: i32 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to i32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + r7 = r6 == y + return r7 + +[case testI32MixedCompare2_64bit] +from mypy_extensions import i32 +def f(x: i32, y: int) -> bool: + return x == y +[out] +def f(x, y): + x :: i32 + y :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: i32 + r7 :: bit +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = y < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = y >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = y >> 1 + r5 = truncate r4: native_int to i32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + r7 = x == r6 + return r7 + +[case testI32MixedCompare_32bit] +from mypy_extensions import i32 +def f(x: int, y: i32) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: i32 + r0 :: native_int + r1 :: bit + r2, r3 :: i32 + r4 :: ptr + r5 :: c_ptr + r6 :: i32 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt32(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 == y + return r7 + +[case testI32ConvertToInt_64bit] +from mypy_extensions import i32 + +def i32_to_int(a: i32) -> int: + return a +[out] +def i32_to_int(a): + a :: i32 + r0 :: native_int + r1 :: int +L0: + r0 = extend signed a: i32 to native_int + r1 = r0 << 1 + return r1 + +[case testI32ConvertToInt_32bit] +from mypy_extensions import i32 + +def i32_to_int(a: i32) -> int: + return a +[out] +def i32_to_int(a): + a :: i32 + r0, r1 :: bit + r2, r3, r4 :: int +L0: + r0 = a <= 1073741823 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = a >= -1073741824 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromSsize_t(a) + r3 = r2 + goto L4 +L3: + r4 = a << 1 + r3 = r4 +L4: + return r3 + +[case testI32OperatorAssignmentMixed_64bit] +from mypy_extensions import i32 + +def f(a: i32) -> None: + x = 0 + x += a +[out] +def f(a): + a :: i32 + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6, r7 :: i32 + r8 :: native_int + r9 :: int +L0: + x = 0 + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to i32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + r7 = r6 + a + r8 = extend signed r7: i32 to native_int + r9 = r8 << 1 + x = r9 + return 1 + +[case testI32InitializeFromLiteral] +from mypy_extensions import i32, i64 + +def f() -> None: + x: i32 = 0 + y: i32 = -127 + z: i32 = 5 + 7 +[out] +def f(): + x, y, z :: i32 +L0: + x = 0 + y = -127 + z = 12 + return 1 + +[case testI32ExplicitConversionFromNativeInt] +from mypy_extensions import i64, i32, i16 + +def from_i16(x: i16) -> i32: + return i32(x) + +def from_i32(x: i32) -> i32: + return i32(x) + +def from_i64(x: i64) -> i32: + return i32(x) +[out] +def from_i16(x): + x :: i16 + r0 :: i32 +L0: + r0 = extend signed x: i16 to i32 + return r0 +def from_i32(x): + x :: i32 +L0: + return x +def from_i64(x): + x :: i64 + r0 :: i32 +L0: + r0 = truncate x: i64 to i32 + return r0 + +[case testI32ExplicitConversionFromInt_64bit] +from mypy_extensions import i32 + +def f(x: int) -> i32: + return i32(x) +[out] +def f(x): + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: i32 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to i32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + return r6 + +[case testI32ExplicitConversionFromLiteral_64bit] +from mypy_extensions import i32 + +def f() -> None: + x = i32(0) + y = i32(11) + z = i32(-3) + a = i32(2**31) +[out] +def f(): + x, y, z, a :: i32 +L0: + x = 0 + y = 11 + z = -3 + a = -2147483648 + return 1 + +[case testI32ExplicitConversionFromVariousTypes_64bit] +from mypy_extensions import i32 + +def bool_to_i32(b: bool) -> i32: + return i32(b) + +def str_to_i32(s: str) -> i32: + return i32(s) + +class C: + def __int__(self) -> i32: + return 5 + +def instance_to_i32(c: C) -> i32: + return i32(c) + +def float_to_i32(x: float) -> i32: + return i32(x) +[out] +def bool_to_i32(b): + b :: bool + r0 :: i32 +L0: + r0 = extend b: builtins.bool to i32 + return r0 +def str_to_i32(s): + s :: str + r0 :: object + r1 :: i32 +L0: + r0 = CPyLong_FromStr(s) + r1 = unbox(i32, r0) + return r1 +def C.__int__(self): + self :: __main__.C +L0: + return 5 +def instance_to_i32(c): + c :: __main__.C + r0 :: i32 +L0: + r0 = c.__int__() + return r0 +def float_to_i32(x): + x :: float + r0 :: int + r1 :: native_int + r2, r3, r4 :: bit + r5 :: native_int + r6, r7 :: i32 +L0: + r0 = CPyTagged_FromFloat(x) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L4 :: bool +L1: + r3 = r0 < 4294967296 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = r0 >= -4294967296 :: signed + if r4 goto L3 else goto L4 :: bool +L3: + r5 = r0 >> 1 + r6 = truncate r5: native_int to i32 + r7 = r6 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + return r7 + +[case testI32ExplicitConversionFromFloat_32bit] +from mypy_extensions import i32 + +def float_to_i32(x: float) -> i32: + return i32(x) +[out] +def float_to_i32(x): + x :: float + r0 :: int + r1 :: native_int + r2 :: bit + r3, r4 :: i32 + r5 :: ptr + r6 :: c_ptr + r7 :: i32 +L0: + r0 = CPyTagged_FromFloat(x) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = r0 >> 1 + r4 = r3 + goto L3 +L2: + r5 = r0 ^ 1 + r6 = r5 + r7 = CPyLong_AsInt32(r6) + r4 = r7 + keep_alive r0 +L3: + return r4 diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test new file mode 100644 index 000000000000..a52de16f3a6c --- /dev/null +++ b/mypyc/test-data/irbuild-i64.test @@ -0,0 +1,2144 @@ +[case testI64Basics] +from mypy_extensions import i64 + +def f() -> i64: + x: i64 = 5 + y = x + return y +[out] +def f(): + x, y :: i64 +L0: + x = 5 + y = x + return y + +[case testI64Compare] +from mypy_extensions import i64 + +def min(x: i64, y: i64) -> i64: + if x < y: + return x + else: + return y + +def all_comparisons(x: i64) -> int: + if x == 2: + y = 10 + elif 3 != x: + y = 11 + elif x > 4: + y = 12 + elif 6 >= x: + y = 13 + elif x < 5: + y = 14 + elif 6 <= x: + y = 15 + else: + y = 16 + return y +[out] +def min(x, y): + x, y :: i64 + r0 :: bit +L0: + r0 = x < y :: signed + if r0 goto L1 else goto L2 :: bool +L1: + return x +L2: + return y +L3: + unreachable +def all_comparisons(x): + x :: i64 + r0 :: bit + y :: int + r1, r2, r3, r4, r5 :: bit +L0: + r0 = x == 2 + if r0 goto L1 else goto L2 :: bool +L1: + y = 20 + goto L18 +L2: + r1 = 3 != x + if r1 goto L3 else goto L4 :: bool +L3: + y = 22 + goto L17 +L4: + r2 = x > 4 :: signed + if r2 goto L5 else goto L6 :: bool +L5: + y = 24 + goto L16 +L6: + r3 = 6 >= x :: signed + if r3 goto L7 else goto L8 :: bool +L7: + y = 26 + goto L15 +L8: + r4 = x < 5 :: signed + if r4 goto L9 else goto L10 :: bool +L9: + y = 28 + goto L14 +L10: + r5 = 6 <= x :: signed + if r5 goto L11 else goto L12 :: bool +L11: + y = 30 + goto L13 +L12: + y = 32 +L13: +L14: +L15: +L16: +L17: +L18: + return y + +[case testI64Arithmetic] +from mypy_extensions import i64 + +def f(x: i64, y: i64) -> i64: + z = x + y + return y - z +[out] +def f(x, y): + x, y, r0, z, r1 :: i64 +L0: + r0 = x + y + z = r0 + r1 = y - z + return r1 + +[case testI64Negation] +from mypy_extensions import i64 + +def f() -> i64: + i: i64 = -3 + return -i +[out] +def f(): + i, r0 :: i64 +L0: + i = -3 + r0 = 0 - i + return r0 + +[case testI64MoreUnaryOps] +from mypy_extensions import i64 + +def unary(x: i64) -> i64: + y = ~x + x = +y + return x +[out] +def unary(x): + x, r0, y :: i64 +L0: + r0 = x ^ -1 + y = r0 + x = y + return x + +[case testI64BoxingAndUnboxing] +from typing import Any +from mypy_extensions import i64 + +def f(a: Any) -> None: + b: i64 = a + a = b +[out] +def f(a): + a :: object + r0, b :: i64 + r1 :: object +L0: + r0 = unbox(i64, a) + b = r0 + r1 = box(i64, b) + a = r1 + return 1 + +[case testI64ListGetSetItem] +from typing import List +from mypy_extensions import i64 + +def get(a: List[i64], i: i64) -> i64: + return a[i] +def set(a: List[i64], i: i64, x: i64) -> None: + a[i] = x +[out] +def get(a, i): + a :: list + i :: i64 + r0 :: object + r1 :: i64 +L0: + r0 = CPyList_GetItemInt64(a, i) + r1 = unbox(i64, r0) + return r1 +def set(a, i, x): + a :: list + i, x :: i64 + r0 :: object + r1 :: bit +L0: + r0 = box(i64, x) + r1 = CPyList_SetItemInt64(a, i, r0) + return 1 + +[case testI64MixedArithmetic] +from mypy_extensions import i64 + +def f() -> i64: + a: i64 = 1 + b = a + 2 + return 3 - b +[out] +def f(): + a, r0, b, r1 :: i64 +L0: + a = 1 + r0 = a + 2 + b = r0 + r1 = 3 - b + return r1 + +[case testI64MixedComparison] +from mypy_extensions import i64 + +def f(a: i64) -> i64: + if a < 3: + return 1 + elif 3 < a: + return 2 + return 3 +[out] +def f(a): + a :: i64 + r0, r1 :: bit +L0: + r0 = a < 3 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + r1 = 3 < a :: signed + if r1 goto L3 else goto L4 :: bool +L3: + return 2 +L4: +L5: + return 3 + +[case testI64InplaceOperations] +from mypy_extensions import i64 + +def add(a: i64) -> i64: + b = a + b += 1 + a += b + return a +def others(a: i64, b: i64) -> i64: + a -= b + a *= b + a &= b + a |= b + a ^= b + a <<= b + a >>= b + return a +[out] +def add(a): + a, b, r0, r1 :: i64 +L0: + b = a + r0 = b + 1 + b = r0 + r1 = a + b + a = r1 + return a +def others(a, b): + a, b, r0, r1, r2, r3, r4, r5, r6 :: i64 +L0: + r0 = a - b + a = r0 + r1 = a * b + a = r1 + r2 = a & b + a = r2 + r3 = a | b + a = r3 + r4 = a ^ b + a = r4 + r5 = a << b + a = r5 + r6 = a >> b + a = r6 + return a + +[case testI64BitwiseOps] +from mypy_extensions import i64 + +def forward(a: i64, b: i64) -> i64: + b = a & 1 + a = b | 2 + b = a ^ 3 + a = b << 4 + b = a >> 5 + return b + +def reverse(a: i64, b: i64) -> i64: + b = 1 & a + a = 2 | b + b = 3 ^ a + a = 4 << b + b = 5 >> a + return b + +def unary(a: i64) -> i64: + return ~a +[out] +def forward(a, b): + a, b, r0, r1, r2, r3, r4 :: i64 +L0: + r0 = a & 1 + b = r0 + r1 = b | 2 + a = r1 + r2 = a ^ 3 + b = r2 + r3 = b << 4 + a = r3 + r4 = a >> 5 + b = r4 + return b +def reverse(a, b): + a, b, r0, r1, r2, r3, r4 :: i64 +L0: + r0 = 1 & a + b = r0 + r1 = 2 | b + a = r1 + r2 = 3 ^ a + b = r2 + r3 = 4 << b + a = r3 + r4 = 5 >> a + b = r4 + return b +def unary(a): + a, r0 :: i64 +L0: + r0 = a ^ -1 + return r0 + +[case testI64Division] +from mypy_extensions import i64 + +def constant_divisor(x: i64) -> i64: + return x // 7 +def variable_divisor(x: i64, y: i64) -> i64: + return x // y +def constant_lhs(x: i64) -> i64: + return 27 // x +def divide_by_neg_one(x: i64) -> i64: + return x // -1 +def divide_by_zero(x: i64) -> i64: + return x // 0 +[out] +def constant_divisor(x): + x, r0, r1 :: i64 + r2, r3, r4 :: bit + r5 :: i64 + r6 :: bit + r7 :: i64 +L0: + r0 = x / 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 * 7 + r6 = r5 == x + if r6 goto L3 else goto L2 :: bool +L2: + r7 = r1 - 1 + r1 = r7 +L3: + return r1 +def variable_divisor(x, y): + x, y, r0 :: i64 +L0: + r0 = CPyInt64_Divide(x, y) + return r0 +def constant_lhs(x): + x, r0 :: i64 +L0: + r0 = CPyInt64_Divide(27, x) + return r0 +def divide_by_neg_one(x): + x, r0 :: i64 +L0: + r0 = CPyInt64_Divide(x, -1) + return r0 +def divide_by_zero(x): + x, r0 :: i64 +L0: + r0 = CPyInt64_Divide(x, 0) + return r0 + +[case testI64Mod] +from mypy_extensions import i64 + +def constant_divisor(x: i64) -> i64: + return x % 7 +def variable_divisor(x: i64, y: i64) -> i64: + return x % y +def constant_lhs(x: i64) -> i64: + return 27 % x +def mod_by_zero(x: i64) -> i64: + return x % 0 +[out] +def constant_divisor(x): + x, r0, r1 :: i64 + r2, r3, r4, r5 :: bit + r6 :: i64 +L0: + r0 = x % 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 == 0 + if r5 goto L3 else goto L2 :: bool +L2: + r6 = r1 + 7 + r1 = r6 +L3: + return r1 +def variable_divisor(x, y): + x, y, r0 :: i64 +L0: + r0 = CPyInt64_Remainder(x, y) + return r0 +def constant_lhs(x): + x, r0 :: i64 +L0: + r0 = CPyInt64_Remainder(27, x) + return r0 +def mod_by_zero(x): + x, r0 :: i64 +L0: + r0 = CPyInt64_Remainder(x, 0) + return r0 + +[case testI64InPlaceDiv] +from mypy_extensions import i64 + +def by_constant(x: i64) -> i64: + x //= 7 + return x +def by_variable(x: i64, y: i64) -> i64: + x //= y + return x +[out] +def by_constant(x): + x, r0, r1 :: i64 + r2, r3, r4 :: bit + r5 :: i64 + r6 :: bit + r7 :: i64 +L0: + r0 = x / 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 * 7 + r6 = r5 == x + if r6 goto L3 else goto L2 :: bool +L2: + r7 = r1 - 1 + r1 = r7 +L3: + x = r1 + return x +def by_variable(x, y): + x, y, r0 :: i64 +L0: + r0 = CPyInt64_Divide(x, y) + x = r0 + return x + +[case testI64InPlaceMod] +from mypy_extensions import i64 + +def by_constant(x: i64) -> i64: + x %= 7 + return x +def by_variable(x: i64, y: i64) -> i64: + x %= y + return x +[out] +def by_constant(x): + x, r0, r1 :: i64 + r2, r3, r4, r5 :: bit + r6 :: i64 +L0: + r0 = x % 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 == 0 + if r5 goto L3 else goto L2 :: bool +L2: + r6 = r1 + 7 + r1 = r6 +L3: + x = r1 + return x +def by_variable(x, y): + x, y, r0 :: i64 +L0: + r0 = CPyInt64_Remainder(x, y) + x = r0 + return x + +[case testI64ForRange] +from mypy_extensions import i64 + +def g(a: i64) -> None: pass + +def f(x: i64) -> None: + n: i64 # TODO: Infer the type + for n in range(x): + g(n) +[out] +def g(a): + a :: i64 +L0: + return 1 +def f(x): + x, r0, n :: i64 + r1 :: bit + r2 :: None + r3 :: i64 +L0: + r0 = 0 + n = r0 +L1: + r1 = r0 < x :: signed + if r1 goto L2 else goto L4 :: bool +L2: + r2 = g(n) +L3: + r3 = r0 + 1 + r0 = r3 + n = r3 + goto L1 +L4: + return 1 + +[case testI64ConvertFromInt_64bit] +from mypy_extensions import i64 + +def int_to_i64(a: int) -> i64: + return a +[out] +def int_to_i64(a): + a :: int + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6 :: i64 +L0: + r0 = a & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = a >> 1 + r3 = r2 + goto L3 +L2: + r4 = a ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive a +L3: + return r3 + +[case testI64ConvertToInt_64bit] +from mypy_extensions import i64 + +def i64_to_int(a: i64) -> int: + return a +[out] +def i64_to_int(a): + a :: i64 + r0, r1 :: bit + r2, r3, r4 :: int +L0: + r0 = a <= 4611686018427387903 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = a >= -4611686018427387904 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(a) + r3 = r2 + goto L4 +L3: + r4 = a << 1 + r3 = r4 +L4: + return r3 + +[case testI64ConvertToInt_32bit] +from mypy_extensions import i64 + +def i64_to_int(a: i64) -> int: + return a +[out] +def i64_to_int(a): + a :: i64 + r0, r1 :: bit + r2, r3 :: int + r4 :: native_int + r5 :: int +L0: + r0 = a <= 1073741823 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = a >= -1073741824 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(a) + r3 = r2 + goto L4 +L3: + r4 = truncate a: i64 to native_int + r5 = r4 << 1 + r3 = r5 +L4: + return r3 + +[case testI64Tuple] +from typing import Tuple +from mypy_extensions import i64 + +def f(x: i64, y: i64) -> Tuple[i64, i64]: + return x, y + +def g() -> Tuple[i64, i64]: + return 1, 2 + +def h() -> i64: + x, y = g() + t = g() + return x + y + t[0] +[out] +def f(x, y): + x, y :: i64 + r0 :: tuple[i64, i64] +L0: + r0 = (x, y) + return r0 +def g(): + r0 :: tuple[int, int] + r1 :: tuple[i64, i64] +L0: + r0 = (2, 4) + r1 = (1, 2) + return r1 +def h(): + r0 :: tuple[i64, i64] + r1, x, r2, y :: i64 + r3, t :: tuple[i64, i64] + r4, r5, r6 :: i64 +L0: + r0 = g() + r1 = r0[0] + x = r1 + r2 = r0[1] + y = r2 + r3 = g() + t = r3 + r4 = x + y + r5 = t[0] + r6 = r4 + r5 + return r6 + +[case testI64MixWithTagged1_64bit] +from mypy_extensions import i64 +def f(x: i64, y: int) -> i64: + return x + y +[out] +def f(x, y): + x :: i64 + y :: int + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: i64 +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + r7 = x + r3 + return r7 + +[case testI64MixWithTagged2_64bit] +from mypy_extensions import i64 +def f(x: int, y: i64) -> i64: + return x + y +[out] +def f(x, y): + x :: int + y :: i64 + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: i64 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 + y + return r7 + +[case testI64MixWithTaggedInPlace1_64bit] +from mypy_extensions import i64 +def f(y: i64) -> int: + x = 0 + x += y + return x +[out] +def f(y): + y :: i64 + x :: int + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: i64 + r8, r9 :: bit + r10, r11, r12 :: int +L0: + x = 0 + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 + y + r8 = r7 <= 4611686018427387903 :: signed + if r8 goto L4 else goto L5 :: bool +L4: + r9 = r7 >= -4611686018427387904 :: signed + if r9 goto L6 else goto L5 :: bool +L5: + r10 = CPyTagged_FromInt64(r7) + r11 = r10 + goto L7 +L6: + r12 = r7 << 1 + r11 = r12 +L7: + x = r11 + return x + +[case testI64MixWithTaggedInPlace2_64bit] +from mypy_extensions import i64 +def f(y: int) -> i64: + x: i64 = 0 + x += y + return x +[out] +def f(y): + y :: int + x :: i64 + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: i64 +L0: + x = 0 + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + r7 = x + r3 + x = r7 + return x + +[case testI64MixedCompare1_64bit] +from mypy_extensions import i64 +def f(x: int, y: i64) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: i64 + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6 :: i64 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 == y + return r7 + +[case testI64MixedCompare2_64bit] +from mypy_extensions import i64 +def f(x: i64, y: int) -> bool: + return x == y +[out] +def f(x, y): + x :: i64 + y :: int + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6 :: i64 + r7 :: bit +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + r7 = x == r3 + return r7 + +[case testI64MixedCompare_32bit] +from mypy_extensions import i64 +def f(x: int, y: i64) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: i64 + r0 :: native_int + r1 :: bit + r2, r3, r4 :: i64 + r5 :: ptr + r6 :: c_ptr + r7 :: i64 + r8 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = extend signed x: builtins.int to i64 + r3 = r2 >> 1 + r4 = r3 + goto L3 +L2: + r5 = x ^ 1 + r6 = r5 + r7 = CPyLong_AsInt64(r6) + r4 = r7 + keep_alive x +L3: + r8 = r4 == y + return r8 + +[case testI64AsBool] +from mypy_extensions import i64 +def f(x: i64) -> i64: + if x: + return 5 + elif not x: + return 6 + return 3 +[out] +def f(x): + x :: i64 + r0, r1 :: bit +L0: + r0 = x != 0 + if r0 goto L1 else goto L2 :: bool +L1: + return 5 +L2: + r1 = x != 0 + if r1 goto L4 else goto L3 :: bool +L3: + return 6 +L4: +L5: + return 3 + +[case testI64AssignMixed_64bit] +from mypy_extensions import i64 +def f(x: i64, y: int) -> i64: + x = y + return x +def g(x: i64, y: int) -> int: + y = x + return y +[out] +def f(x, y): + x :: i64 + y :: int + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6 :: i64 +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + x = r3 + return x +def g(x, y): + x :: i64 + y :: int + r0, r1 :: bit + r2, r3, r4 :: int +L0: + r0 = x <= 4611686018427387903 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = x >= -4611686018427387904 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(x) + r3 = r2 + goto L4 +L3: + r4 = x << 1 + r3 = r4 +L4: + y = r3 + return y + +[case testBorrowOverI64Arithmetic] +from mypy_extensions import i64 + +def add_simple(c: C) -> i64: + return c.x + c.y + +def inplace_add_simple(c: C) -> None: + c.x += c.y + +def add_borrow(d: D) -> i64: + return d.c.x + d.c.y + +class D: + c: C + +class C: + x: i64 + y: i64 +[out] +def add_simple(c): + c :: __main__.C + r0, r1, r2 :: i64 +L0: + r0 = c.x + r1 = c.y + r2 = r0 + r1 + return r2 +def inplace_add_simple(c): + c :: __main__.C + r0, r1, r2 :: i64 + r3 :: bool +L0: + r0 = c.x + r1 = c.y + r2 = r0 + r1 + c.x = r2; r3 = is_error + return 1 +def add_borrow(d): + d :: __main__.D + r0 :: __main__.C + r1 :: i64 + r2 :: __main__.C + r3, r4 :: i64 +L0: + r0 = borrow d.c + r1 = r0.x + r2 = borrow d.c + r3 = r2.y + r4 = r1 + r3 + keep_alive d, d + return r4 + +[case testBorrowOverI64Bitwise] +from mypy_extensions import i64 + +def bitwise_simple(c: C) -> i64: + return c.x | c.y + +def inplace_bitwide_simple(c: C) -> None: + c.x &= c.y + +def bitwise_borrow(d: D) -> i64: + return d.c.x ^ d.c.y + +class D: + c: C + +class C: + x: i64 + y: i64 +[out] +def bitwise_simple(c): + c :: __main__.C + r0, r1, r2 :: i64 +L0: + r0 = c.x + r1 = c.y + r2 = r0 | r1 + return r2 +def inplace_bitwide_simple(c): + c :: __main__.C + r0, r1, r2 :: i64 + r3 :: bool +L0: + r0 = c.x + r1 = c.y + r2 = r0 & r1 + c.x = r2; r3 = is_error + return 1 +def bitwise_borrow(d): + d :: __main__.D + r0 :: __main__.C + r1 :: i64 + r2 :: __main__.C + r3, r4 :: i64 +L0: + r0 = borrow d.c + r1 = r0.x + r2 = borrow d.c + r3 = r2.y + r4 = r1 ^ r3 + keep_alive d, d + return r4 + +[case testBorrowOverI64ListGetItem1] +from mypy_extensions import i64 + +def f(n: i64) -> str: + a = [C()] + return a[n].s + +class C: + s: str +[out] +def f(n): + n :: i64 + r0 :: __main__.C + r1 :: list + r2 :: ptr + a :: list + r3 :: object + r4 :: __main__.C + r5 :: str +L0: + r0 = C() + r1 = PyList_New(1) + r2 = list_items r1 + buf_init_item r2, 0, r0 + keep_alive r1 + a = r1 + r3 = CPyList_GetItemInt64Borrow(a, n) + r4 = borrow cast(__main__.C, r3) + r5 = r4.s + keep_alive a, n, r3 + return r5 + +[case testBorrowOverI64ListGetItem2] +from typing import List +from mypy_extensions import i64 + +def f(a: List[i64], n: i64) -> bool: + if a[n] == 0: + return True + return False +[out] +def f(a, n): + a :: list + n :: i64 + r0 :: object + r1 :: i64 + r2 :: bit +L0: + r0 = CPyList_GetItemInt64Borrow(a, n) + r1 = unbox(i64, r0) + r2 = r1 == 0 + keep_alive a, n + if r2 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + +[case testCoerceShortIntToI64] +from mypy_extensions import i64 +from typing import List + +def f(a: List[i64], y: i64) -> bool: + if len(a) < y: + return True + return False + +def g(a: List[i64], y: i64) -> bool: + if y < len(a): + return True + return False +[out] +def f(a, y): + a :: list + y :: i64 + r0 :: native_int + r1 :: short_int + r2 :: i64 + r3 :: bit +L0: + r0 = var_object_size a + r1 = r0 << 1 + r2 = r1 >> 1 + r3 = r2 < y :: signed + if r3 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +def g(a, y): + a :: list + y :: i64 + r0 :: native_int + r1 :: short_int + r2 :: i64 + r3 :: bit +L0: + r0 = var_object_size a + r1 = r0 << 1 + r2 = r1 >> 1 + r3 = y < r2 :: signed + if r3 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + +[case testMultiplyListByI64_64bit] +from mypy_extensions import i64 +from typing import List + +def f(n: i64) -> List[i64]: + return [n] * n +[out] +def f(n): + n :: i64 + r0 :: list + r1 :: object + r2 :: ptr + r3, r4 :: bit + r5, r6, r7 :: int + r8 :: list +L0: + r0 = PyList_New(1) + r1 = box(i64, n) + r2 = list_items r0 + buf_init_item r2, 0, r1 + keep_alive r0 + r3 = n <= 4611686018427387903 :: signed + if r3 goto L1 else goto L2 :: bool +L1: + r4 = n >= -4611686018427387904 :: signed + if r4 goto L3 else goto L2 :: bool +L2: + r5 = CPyTagged_FromInt64(n) + r6 = r5 + goto L4 +L3: + r7 = n << 1 + r6 = r7 +L4: + r8 = CPySequence_Multiply(r0, r6) + return r8 + +[case testShortIntAndI64Op] +from mypy_extensions import i64 +from typing import List + +def add_i64(a: List[i64], n: i64) -> i64: + return len(a) + n +def add_i64_2(a: List[i64], n: i64) -> i64: + return n + len(a) +def eq_i64(a: List[i64], n: i64) -> bool: + if len(a) == n: + return True + return False +def lt_i64(a: List[i64], n: i64) -> bool: + if n < len(a): + return True + return False +[out] +def add_i64(a, n): + a :: list + n :: i64 + r0 :: native_int + r1 :: short_int + r2, r3 :: i64 +L0: + r0 = var_object_size a + r1 = r0 << 1 + r2 = r1 >> 1 + r3 = r2 + n + return r3 +def add_i64_2(a, n): + a :: list + n :: i64 + r0 :: native_int + r1 :: short_int + r2, r3 :: i64 +L0: + r0 = var_object_size a + r1 = r0 << 1 + r2 = r1 >> 1 + r3 = n + r2 + return r3 +def eq_i64(a, n): + a :: list + n :: i64 + r0 :: native_int + r1 :: short_int + r2 :: i64 + r3 :: bit +L0: + r0 = var_object_size a + r1 = r0 << 1 + r2 = r1 >> 1 + r3 = r2 == n + if r3 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +def lt_i64(a, n): + a :: list + n :: i64 + r0 :: native_int + r1 :: short_int + r2 :: i64 + r3 :: bit +L0: + r0 = var_object_size a + r1 = r0 << 1 + r2 = r1 >> 1 + r3 = n < r2 :: signed + if r3 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + +[case testOptionalI64_64bit] +from typing import Optional +from mypy_extensions import i64 + +def f(x: Optional[i64]) -> i64: + if x is None: + return 1 + return x +[out] +def f(x): + x :: union[i64, None] + r0 :: object + r1 :: bit + r2 :: i64 +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + r2 = unbox(i64, x) + return r2 + +[case testI64DefaultValueSingle] +from mypy_extensions import i64 + +def f(x: i64, y: i64 = 0) -> i64: + return x + y + +def g() -> i64: + return f(7) + f(8, 9) +[out] +def f(x, y, __bitmap): + x, y :: i64 + __bitmap, r0 :: u32 + r1 :: bit + r2 :: i64 +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + y = 0 +L2: + r2 = x + y + return r2 +def g(): + r0, r1, r2 :: i64 +L0: + r0 = f(7, 0, 0) + r1 = f(8, 9, 1) + r2 = r0 + r1 + return r2 + +[case testI64DefaultValueWithMultipleArgs] +from mypy_extensions import i64 + +def f(a: i64, b: i64 = 1, c: int = 2, d: i64 = 3) -> i64: + return 0 + +def g() -> i64: + return f(7) + f(8, 9) + f(1, 2, 3) + f(4, 5, 6, 7) +[out] +def f(a, b, c, d, __bitmap): + a, b :: i64 + c :: int + d :: i64 + __bitmap, r0 :: u32 + r1 :: bit + r2 :: u32 + r3 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + b = 1 +L2: + if is_error(c) goto L3 else goto L4 +L3: + c = 4 +L4: + r2 = __bitmap & 2 + r3 = r2 == 0 + if r3 goto L5 else goto L6 :: bool +L5: + d = 3 +L6: + return 0 +def g(): + r0 :: int + r1 :: i64 + r2 :: int + r3, r4, r5, r6, r7, r8 :: i64 +L0: + r0 = :: int + r1 = f(7, 0, r0, 0, 0) + r2 = :: int + r3 = f(8, 9, r2, 0, 1) + r4 = r1 + r3 + r5 = f(1, 2, 6, 0, 1) + r6 = r4 + r5 + r7 = f(4, 5, 12, 7, 3) + r8 = r6 + r7 + return r8 + +[case testI64MethodDefaultValue] +from mypy_extensions import i64 + +class C: + def m(self, x: i64 = 5) -> None: + pass + +def f(c: C) -> None: + c.m() + c.m(6) +[out] +def C.m(self, x, __bitmap): + self :: __main__.C + x :: i64 + __bitmap, r0 :: u32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 5 +L2: + return 1 +def f(c): + c :: __main__.C + r0, r1 :: None +L0: + r0 = c.m(0, 0) + r1 = c.m(6, 1) + return 1 + +[case testI64ExplicitConversionFromNativeInt] +from mypy_extensions import i64, i32, i16 + +def from_i16(x: i16) -> i64: + return i64(x) + +def from_i32(x: i32) -> i64: + return i64(x) + +def from_i64(x: i64) -> i64: + return i64(x) +[out] +def from_i16(x): + x :: i16 + r0 :: i64 +L0: + r0 = extend signed x: i16 to i64 + return r0 +def from_i32(x): + x :: i32 + r0 :: i64 +L0: + r0 = extend signed x: i32 to i64 + return r0 +def from_i64(x): + x :: i64 +L0: + return x + +[case testI64ExplicitConversionFromInt_64bit] +from mypy_extensions import i64 + +def f(x: int) -> i64: + return i64(x) +[out] +def f(x): + x :: int + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6 :: i64 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + return r3 + +[case testI64ExplicitConversionToInt_64bit] +from mypy_extensions import i64 + +def f(x: i64) -> int: + return int(x) +[out] +def f(x): + x :: i64 + r0, r1 :: bit + r2, r3, r4 :: int +L0: + r0 = x <= 4611686018427387903 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = x >= -4611686018427387904 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(x) + r3 = r2 + goto L4 +L3: + r4 = x << 1 + r3 = r4 +L4: + return r3 + +[case testI64ExplicitConversionFromLiteral] +from mypy_extensions import i64 + +def f() -> None: + x = i64(0) + y = i64(11) + z = i64(-3) +[out] +def f(): + x, y, z :: i64 +L0: + x = 0 + y = 11 + z = -3 + return 1 + +[case testI64ForLoopOverRange] +from mypy_extensions import i64 + +def f() -> None: + for x in range(i64(4)): + y = x +[out] +def f(): + r0, x :: i64 + r1 :: bit + y, r2 :: i64 +L0: + r0 = 0 + x = r0 +L1: + r1 = r0 < 4 :: signed + if r1 goto L2 else goto L4 :: bool +L2: + y = x +L3: + r2 = r0 + 1 + r0 = r2 + x = r2 + goto L1 +L4: + return 1 + +[case testI64ForLoopOverRange2] +from mypy_extensions import i64 + +def f() -> None: + for x in range(0, i64(4)): + y = x +[out] +def f(): + r0, x :: i64 + r1 :: bit + y, r2 :: i64 +L0: + r0 = 0 + x = r0 +L1: + r1 = r0 < 4 :: signed + if r1 goto L2 else goto L4 :: bool +L2: + y = x +L3: + r2 = r0 + 1 + r0 = r2 + x = r2 + goto L1 +L4: + return 1 + +[case testI64MethodDefaultValueOverride] +from mypy_extensions import i64 + +class C: + def f(self, x: i64 = 11) -> None: pass +class D(C): + def f(self, x: i64 = 12) -> None: pass +[out] +def C.f(self, x, __bitmap): + self :: __main__.C + x :: i64 + __bitmap, r0 :: u32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 11 +L2: + return 1 +def D.f(self, x, __bitmap): + self :: __main__.D + x :: i64 + __bitmap, r0 :: u32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 12 +L2: + return 1 + +[case testI64FinalConstants] +from typing_extensions import Final +from mypy_extensions import i64 + +A: Final = -1 +B: Final = -(1 + 3*2) +C: Final = 0 +D: Final = A - B +E: Final[i64] = 1 + 3 + +def f1() -> i64: + return A + +def f2() -> i64: + return A + B + +def f3() -> i64: + return C + +def f4() -> i64: + return D + +def f5() -> i64: + return E +[out] +def f1(): +L0: + return -1 +def f2(): +L0: + return -8 +def f3(): +L0: + return 0 +def f4(): +L0: + return 6 +def f5(): +L0: + return 4 + +[case testI64OperationsWithBools] +from mypy_extensions import i64 + +# TODO: Other mixed operations + +def add_bool_to_int(n: i64, b: bool) -> i64: + return n + b + +def compare_bool_to_i64(n: i64, b: bool) -> bool: + if n == b: + return b != n + return True +[out] +def add_bool_to_int(n, b): + n :: i64 + b :: bool + r0, r1 :: i64 +L0: + r0 = extend b: builtins.bool to i64 + r1 = n + r0 + return r1 +def compare_bool_to_i64(n, b): + n :: i64 + b :: bool + r0 :: i64 + r1 :: bit + r2 :: i64 + r3 :: bit +L0: + r0 = extend b: builtins.bool to i64 + r1 = n == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = extend b: builtins.bool to i64 + r3 = r2 != n + return r3 +L2: + return 1 + +[case testI64Cast_64bit] +from typing import cast +from mypy_extensions import i64 + +def cast_object(o: object) -> i64: + return cast(i64, o) + +def cast_int(x: int) -> i64: + return cast(i64, x) +[out] +def cast_object(o): + o :: object + r0 :: i64 +L0: + r0 = unbox(i64, o) + return r0 +def cast_int(x): + x :: int + r0 :: native_int + r1 :: bit + r2, r3 :: i64 + r4 :: ptr + r5 :: c_ptr + r6 :: i64 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + return r3 + +[case testI64Cast_32bit] +from typing import cast +from mypy_extensions import i64 + +def cast_int(x: int) -> i64: + return cast(i64, x) +[out] +def cast_int(x): + x :: int + r0 :: native_int + r1 :: bit + r2, r3, r4 :: i64 + r5 :: ptr + r6 :: c_ptr + r7 :: i64 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = extend signed x: builtins.int to i64 + r3 = r2 >> 1 + r4 = r3 + goto L3 +L2: + r5 = x ^ 1 + r6 = r5 + r7 = CPyLong_AsInt64(r6) + r4 = r7 + keep_alive x +L3: + return r4 + +[case testI64ExplicitConversionFromVariousTypes_64bit] +from mypy_extensions import i64 + +def bool_to_i64(b: bool) -> i64: + return i64(b) + +def str_to_i64(s: str) -> i64: + return i64(s) + +def str_to_i64_with_base(s: str) -> i64: + return i64(s, 2) + +class C: + def __int__(self) -> i64: + return 5 + +def instance_to_i64(c: C) -> i64: + return i64(c) + +def float_to_i64(x: float) -> i64: + return i64(x) +[out] +def bool_to_i64(b): + b :: bool + r0 :: i64 +L0: + r0 = extend b: builtins.bool to i64 + return r0 +def str_to_i64(s): + s :: str + r0 :: object + r1 :: i64 +L0: + r0 = CPyLong_FromStr(s) + r1 = unbox(i64, r0) + return r1 +def str_to_i64_with_base(s): + s :: str + r0 :: object + r1 :: i64 +L0: + r0 = CPyLong_FromStrWithBase(s, 4) + r1 = unbox(i64, r0) + return r1 +def C.__int__(self): + self :: __main__.C +L0: + return 5 +def instance_to_i64(c): + c :: __main__.C + r0 :: i64 +L0: + r0 = c.__int__() + return r0 +def float_to_i64(x): + x :: float + r0 :: int + r1 :: native_int + r2 :: bit + r3, r4 :: i64 + r5 :: ptr + r6 :: c_ptr + r7 :: i64 +L0: + r0 = CPyTagged_FromFloat(x) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = r0 >> 1 + r4 = r3 + goto L3 +L2: + r5 = r0 ^ 1 + r6 = r5 + r7 = CPyLong_AsInt64(r6) + r4 = r7 + keep_alive r0 +L3: + return r4 + +[case testI64ExplicitConversionFromFloat_32bit] +from mypy_extensions import i64 + +def float_to_i64(x: float) -> i64: + return i64(x) +[out] +def float_to_i64(x): + x :: float + r0 :: int + r1 :: native_int + r2 :: bit + r3, r4, r5 :: i64 + r6 :: ptr + r7 :: c_ptr + r8 :: i64 +L0: + r0 = CPyTagged_FromFloat(x) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = extend signed r0: builtins.int to i64 + r4 = r3 >> 1 + r5 = r4 + goto L3 +L2: + r6 = r0 ^ 1 + r7 = r6 + r8 = CPyLong_AsInt64(r7) + r5 = r8 + keep_alive r0 +L3: + return r5 + +[case testI64ConvertToFloat_64bit] +from mypy_extensions import i64 + +def i64_to_float(x: i64) -> float: + return float(x) +[out] +def i64_to_float(x): + x :: i64 + r0, r1 :: bit + r2, r3, r4 :: int + r5 :: float +L0: + r0 = x <= 4611686018427387903 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = x >= -4611686018427387904 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(x) + r3 = r2 + goto L4 +L3: + r4 = x << 1 + r3 = r4 +L4: + r5 = CPyFloat_FromTagged(r3) + return r5 + +[case testI64ConvertToFloat_32bit] +from mypy_extensions import i64 + +def i64_to_float(x: i64) -> float: + return float(x) +[out] +def i64_to_float(x): + x :: i64 + r0, r1 :: bit + r2, r3 :: int + r4 :: native_int + r5 :: int + r6 :: float +L0: + r0 = x <= 1073741823 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = x >= -1073741824 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(x) + r3 = r2 + goto L4 +L3: + r4 = truncate x: i64 to native_int + r5 = r4 << 1 + r3 = r5 +L4: + r6 = CPyFloat_FromTagged(r3) + return r6 + +[case testI64IsinstanceNarrowing] +from typing import Union +from mypy_extensions import i64 + +class C: + a: i64 + +def narrow1(x: Union[C, i64]) -> i64: + if isinstance(x, i64): + return x + return x.a + +def narrow2(x: Union[C, i64]) -> i64: + if isinstance(x, int): + return x + return x.a +[out] +def narrow1(x): + x :: union[__main__.C, i64] + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: i64 + r5 :: __main__.C + r6 :: i64 +L0: + r0 = load_address PyLong_Type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool +L1: + r4 = unbox(i64, x) + return r4 +L2: + r5 = borrow cast(__main__.C, x) + r6 = r5.a + keep_alive x + return r6 +def narrow2(x): + x :: union[__main__.C, i64] + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: i64 + r5 :: __main__.C + r6 :: i64 +L0: + r0 = load_address PyLong_Type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool +L1: + r4 = unbox(i64, x) + return r4 +L2: + r5 = borrow cast(__main__.C, x) + r6 = r5.a + keep_alive x + return r6 + +[case testI64ConvertBetweenTuples_64bit] +from __future__ import annotations +from mypy_extensions import i64 + +def f(t: tuple[int, i64, int]) -> None: + tt: tuple[int, i64, i64] = t + +def g(n: int) -> None: + t: tuple[i64, i64] = (1, n) +[out] +def f(t): + t :: tuple[int, i64, int] + r0 :: int + r1 :: i64 + r2 :: int + r3 :: native_int + r4 :: bit + r5, r6 :: i64 + r7 :: ptr + r8 :: c_ptr + r9 :: i64 + r10, tt :: tuple[int, i64, i64] +L0: + r0 = t[0] + r1 = t[1] + r2 = t[2] + r3 = r2 & 1 + r4 = r3 == 0 + if r4 goto L1 else goto L2 :: bool +L1: + r5 = r2 >> 1 + r6 = r5 + goto L3 +L2: + r7 = r2 ^ 1 + r8 = r7 + r9 = CPyLong_AsInt64(r8) + r6 = r9 + keep_alive r2 +L3: + r10 = (r0, r1, r6) + tt = r10 + return 1 +def g(n): + n :: int + r0 :: tuple[int, int] + r1 :: int + r2 :: native_int + r3 :: bit + r4, r5 :: i64 + r6 :: ptr + r7 :: c_ptr + r8 :: i64 + r9, t :: tuple[i64, i64] +L0: + r0 = (2, n) + r1 = r0[1] + r2 = r1 & 1 + r3 = r2 == 0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = r1 >> 1 + r5 = r4 + goto L3 +L2: + r6 = r1 ^ 1 + r7 = r6 + r8 = CPyLong_AsInt64(r7) + r5 = r8 + keep_alive r1 +L3: + r9 = (1, r5) + t = r9 + return 1 diff --git a/mypyc/test-data/irbuild-int.test b/mypyc/test-data/irbuild-int.test new file mode 100644 index 000000000000..9082cc0136d9 --- /dev/null +++ b/mypyc/test-data/irbuild-int.test @@ -0,0 +1,212 @@ +[case testIntNeq] +def f(x: int, y: int) -> bool: + return x != y +[out] +def f(x, y): + x, y :: int + r0 :: bit +L0: + r0 = int_ne x, y + return r0 + +[case testShortIntComparisons] +def f(x: int) -> int: + if x == 3: + return 1 + elif x != 4: + return 2 + elif 5 == x: + return 3 + elif 6 != x: + return 4 + elif x < 4: + return 5 + return 6 +[out] +def f(x): + x :: int + r0, r1, r2, r3, r4 :: bit +L0: + r0 = int_eq x, 6 + if r0 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + r1 = int_ne x, 8 + if r1 goto L3 else goto L4 :: bool +L3: + return 4 +L4: + r2 = int_eq 10, x + if r2 goto L5 else goto L6 :: bool +L5: + return 6 +L6: + r3 = int_ne 12, x + if r3 goto L7 else goto L8 :: bool +L7: + return 8 +L8: + r4 = int_lt x, 8 + if r4 goto L9 else goto L10 :: bool +L9: + return 10 +L10: +L11: +L12: +L13: +L14: + return 12 + +[case testIntMin] +def f(x: int, y: int) -> int: + return min(x, y) +[out] +def f(x, y): + x, y :: int + r0 :: bit + r1 :: int +L0: + r0 = int_lt y, x + if r0 goto L1 else goto L2 :: bool +L1: + r1 = y + goto L3 +L2: + r1 = x +L3: + return r1 + +[case testIntFloorDivideByPowerOfTwo] +def divby1(x: int) -> int: + return x // 1 +def divby2(x: int) -> int: + return x // 2 +def divby3(x: int) -> int: + return x // 3 +def divby4(x: int) -> int: + return x // 4 +def divby8(x: int) -> int: + return x // 8 +[out] +def divby1(x): + x, r0 :: int +L0: + r0 = CPyTagged_FloorDivide(x, 2) + return r0 +def divby2(x): + x, r0 :: int +L0: + r0 = CPyTagged_Rshift(x, 2) + return r0 +def divby3(x): + x, r0 :: int +L0: + r0 = CPyTagged_FloorDivide(x, 6) + return r0 +def divby4(x): + x, r0 :: int +L0: + r0 = CPyTagged_Rshift(x, 4) + return r0 +def divby8(x): + x, r0 :: int +L0: + r0 = CPyTagged_Rshift(x, 6) + return r0 + +[case testFinalConstantFolding] +from typing_extensions import Final + +X: Final = -1 +Y: Final = -(1 + 3*2) +Z: Final = Y + 1 + +class C: + A: Final = 1 + B: Final = -1 + +def f1() -> int: + return X + +def f2() -> int: + return X + Y + +def f3() -> int: + return Z + +def f4() -> int: + return C.A + +def f5() -> int: + return C.B +[out] +def C.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.C +L0: + __mypyc_self__.A = 2 + __mypyc_self__.B = -2 + return 1 +def f1(): +L0: + return -2 +def f2(): +L0: + return -16 +def f3(): +L0: + return -12 +def f4(): +L0: + return 2 +def f5(): +L0: + return -2 + +[case testConvertIntegralToInt] +def bool_to_int(b: bool) -> int: + return int(b) + +def int_to_int(n: int) -> int: + return int(n) +[out] +def bool_to_int(b): + b, r0 :: bool + r1 :: int +L0: + r0 = b << 1 + r1 = extend r0: builtins.bool to builtins.int + return r1 +def int_to_int(n): + n :: int +L0: + return n + +[case testIntUnaryOps] +def unary_minus(n: int) -> int: + x = -n + return x +def unary_plus(n: int) -> int: + x = +n + return x +def unary_invert(n: int) -> int: + x = ~n + return x +[out] +def unary_minus(n): + n, r0, x :: int +L0: + r0 = CPyTagged_Negate(n) + x = r0 + return x +def unary_plus(n): + n, x :: int +L0: + x = n + return x +def unary_invert(n): + n, r0, x :: int +L0: + r0 = CPyTagged_Invert(n) + x = r0 + return x diff --git a/mypyc/test-data/irbuild-isinstance.test b/mypyc/test-data/irbuild-isinstance.test new file mode 100644 index 000000000000..78da2e9c1e19 --- /dev/null +++ b/mypyc/test-data/irbuild-isinstance.test @@ -0,0 +1,109 @@ +[case testIsinstanceInt] +def is_int(value: object) -> bool: + return isinstance(value, int) + +[out] +def is_int(value): + value, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool +L0: + r0 = load_address PyLong_Type + r1 = PyObject_IsInstance(value, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + return r3 + +[case testIsinstanceNotBool1] +def is_not_bool(value: object) -> bool: + return not isinstance(value, bool) + +[out] +def is_not_bool(value): + value, r0 :: object + r1 :: i32 + r2 :: bit + r3, r4 :: bool +L0: + r0 = load_address PyBool_Type + r1 = PyObject_IsInstance(value, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + r4 = r3 ^ 1 + return r4 + +[case testIsinstanceIntAndNotBool] +# This test is to ensure that 'value' doesn't get coerced to int when we are +# checking if it's a bool, since an int can never be an instance of a bool +def is_not_bool_and_is_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + +[out] +def is_not_bool_and_is_int(value): + value, r0 :: object + r1 :: i32 + r2 :: bit + r3, r4 :: bool + r5 :: object + r6 :: i32 + r7 :: bit + r8, r9 :: bool +L0: + r0 = load_address PyLong_Type + r1 = PyObject_IsInstance(value, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L2 else goto L1 :: bool +L1: + r4 = r3 + goto L3 +L2: + r5 = load_address PyBool_Type + r6 = PyObject_IsInstance(value, r5) + r7 = r6 >= 0 :: signed + r8 = truncate r6: i32 to builtins.bool + r9 = r8 ^ 1 + r4 = r9 +L3: + return r4 + +[case testBorrowSpecialCaseWithIsinstance] +class C: + s: str + +def g() -> object: + pass + +def f() -> None: + x = g() + if isinstance(x, C): + x.s +[out] +def g(): + r0 :: object +L0: + r0 = box(None, 1) + return r0 +def f(): + r0, x, r1 :: object + r2 :: ptr + r3 :: object + r4 :: bit + r5 :: __main__.C + r6 :: str +L0: + r0 = g() + x = r0 + r1 = __main__.C :: type + r2 = get_element_ptr x ob_type :: PyObject + r3 = load_mem r2 :: builtins.object* + keep_alive x + r4 = r3 == r1 + if r4 goto L1 else goto L2 :: bool +L1: + r5 = borrow cast(__main__.C, x) + r6 = r5.s + keep_alive x +L2: + return 1 diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test new file mode 100644 index 000000000000..725f218b686a --- /dev/null +++ b/mypyc/test-data/irbuild-lists.test @@ -0,0 +1,513 @@ +[case testListGet] +from typing import List +def f(x: List[int]) -> int: + return x[0] +[out] +def f(x): + x :: list + r0 :: object + r1 :: int +L0: + r0 = CPyList_GetItemShort(x, 0) + r1 = unbox(int, r0) + return r1 + +[case testListOfListGet] +from typing import List +def f(x: List[List[int]]) -> List[int]: + return x[0] +[out] +def f(x): + x :: list + r0 :: object + r1 :: list +L0: + r0 = CPyList_GetItemShort(x, 0) + r1 = cast(list, r0) + return r1 + +[case testListOfListGet2] +from typing import List +def f(x: List[List[int]]) -> int: + return x[0][1] +[out] +def f(x): + x :: list + r0 :: object + r1 :: list + r2 :: object + r3 :: int +L0: + r0 = CPyList_GetItemShortBorrow(x, 0) + r1 = borrow cast(list, r0) + r2 = CPyList_GetItemShort(r1, 2) + r3 = unbox(int, r2) + keep_alive x, r0 + return r3 + +[case testListSet] +from typing import List +def f(x: List[int]) -> None: + x[0] = 1 +[out] +def f(x): + x :: list + r0 :: object + r1 :: bit +L0: + r0 = object 1 + r1 = CPyList_SetItem(x, 0, r0) + return 1 + +[case testNewListEmpty] +from typing import List +def f() -> None: + x = [] # type: List[int] +[out] +def f(): + r0, x :: list +L0: + r0 = PyList_New(0) + x = r0 + return 1 + +[case testNewListEmptyViaFunc] +from typing import List +def f() -> None: + x: List[int] = list() + +[out] +def f(): + r0, x :: list +L0: + r0 = PyList_New(0) + x = r0 + return 1 + +[case testNewListEmptyViaAlias] +from typing import List + +ListAlias = list + +def f() -> None: + x: List[int] = ListAlias() + +[out] +def f(): + r0, x :: list +L0: + r0 = PyList_New(0) + x = r0 + return 1 + +[case testNewListTwoItems] +from typing import List +def f() -> None: + x: List[int] = [1, 2] +[out] +def f(): + r0 :: list + r1, r2 :: object + r3 :: ptr + x :: list +L0: + r0 = PyList_New(2) + r1 = object 1 + r2 = object 2 + r3 = list_items r0 + buf_init_item r3, 0, r1 + buf_init_item r3, 1, r2 + keep_alive r0 + x = r0 + return 1 + +[case testNewListTenItems] +from typing import List +def f() -> None: + x: List[str] = ['a', 'b', 'c', 'd', 'e', + 'f', 'g', 'h', 'i', 'j'] +[out] +def f(): + r0, r1, r2, r3, r4, r5, r6, r7, r8, r9 :: str + r10, x :: list +L0: + r0 = 'a' + r1 = 'b' + r2 = 'c' + r3 = 'd' + r4 = 'e' + r5 = 'f' + r6 = 'g' + r7 = 'h' + r8 = 'i' + r9 = 'j' + r10 = CPyList_Build(10, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9) + x = r10 + return 1 + +[case testListMultiply] +from typing import List +def f(a: List[int]) -> None: + b = a * 2 + b = 3 * [4] +[out] +def f(a): + a, r0, b, r1 :: list + r2 :: object + r3 :: ptr + r4 :: list +L0: + r0 = CPySequence_Multiply(a, 4) + b = r0 + r1 = PyList_New(1) + r2 = object 4 + r3 = list_items r1 + buf_init_item r3, 0, r2 + keep_alive r1 + r4 = CPySequence_RMultiply(6, r1) + b = r4 + return 1 + +[case testListLen] +from typing import List +def f(a: List[int]) -> int: + return len(a) +[out] +def f(a): + a :: list + r0 :: native_int + r1 :: short_int +L0: + r0 = var_object_size a + r1 = r0 << 1 + return r1 + +[case testListAppend] +from typing import List +def f(a: List[int], x: int) -> None: + a.append(x) +[out] +def f(a, x): + a :: list + x :: int + r0 :: object + r1 :: i32 + r2 :: bit +L0: + r0 = box(int, x) + r1 = PyList_Append(a, r0) + r2 = r1 >= 0 :: signed + return 1 + +[case testIndexLvalue] +from typing import List +def increment(l: List[int]) -> List[int]: + for i in range(len(l)): + l[i] += 1 + return l +[out] +def increment(l): + l :: list + r0 :: native_int + r1, r2 :: short_int + i :: int + r3 :: bit + r4, r5, r6 :: object + r7 :: bit + r8 :: short_int +L0: + r0 = var_object_size l + r1 = r0 << 1 + r2 = 0 + i = r2 +L1: + r3 = int_lt r2, r1 + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyList_GetItem(l, i) + r5 = object 1 + r6 = PyNumber_InPlaceAdd(r4, r5) + r7 = CPyList_SetItem(l, i, r6) +L3: + r8 = r2 + 2 + r2 = r8 + i = r8 + goto L1 +L4: + return l + +[case testListDisplay] +from typing import List +def f(x: List[int], y: List[int]) -> List[int]: + return [1, 2, *x, *y, 3] +[out] +def f(x, y): + x, y, r0 :: list + r1, r2 :: object + r3 :: ptr + r4, r5, r6 :: object + r7 :: i32 + r8 :: bit +L0: + r0 = PyList_New(2) + r1 = object 1 + r2 = object 2 + r3 = list_items r0 + buf_init_item r3, 0, r1 + buf_init_item r3, 1, r2 + keep_alive r0 + r4 = CPyList_Extend(r0, x) + r5 = CPyList_Extend(r0, y) + r6 = object 3 + r7 = PyList_Append(r0, r6) + r8 = r7 >= 0 :: signed + return r0 + +[case testListIn] +from typing import List +def f(x: List[int], y: int) -> bool: + return y in x +[out] +def f(x, y): + x :: list + y :: int + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool +L0: + r0 = box(int, y) + r1 = PySequence_Contains(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + return r3 + +[case testListInsert] +from typing import List +def f(x: List[int], y: int) -> None: + x.insert(0, y) +[out] +def f(x, y): + x :: list + y :: int + r0 :: object + r1 :: i32 + r2 :: bit +L0: + r0 = box(int, y) + r1 = CPyList_Insert(x, 0, r0) + r2 = r1 >= 0 :: signed + return 1 + +[case testListBuiltFromGenerator] +from typing import List +def f(source: List[int]) -> None: + a = list(x + 1 for x in source) + b = [x + 1 for x in source] +[out] +def f(source): + source :: list + r0 :: native_int + r1 :: list + r2 :: short_int + r3 :: native_int + r4 :: short_int + r5 :: bit + r6 :: object + r7, x, r8 :: int + r9 :: object + r10 :: bit + r11 :: short_int + a :: list + r12 :: native_int + r13 :: list + r14 :: short_int + r15 :: native_int + r16 :: short_int + r17 :: bit + r18 :: object + r19, x_2, r20 :: int + r21 :: object + r22 :: bit + r23 :: short_int + b :: list +L0: + r0 = var_object_size source + r1 = PyList_New(r0) + r2 = 0 +L1: + r3 = var_object_size source + r4 = r3 << 1 + r5 = int_lt r2, r4 + if r5 goto L2 else goto L4 :: bool +L2: + r6 = CPyList_GetItemUnsafe(source, r2) + r7 = unbox(int, r6) + x = r7 + r8 = CPyTagged_Add(x, 2) + r9 = box(int, r8) + r10 = CPyList_SetItemUnsafe(r1, r2, r9) +L3: + r11 = r2 + 2 + r2 = r11 + goto L1 +L4: + a = r1 + r12 = var_object_size source + r13 = PyList_New(r12) + r14 = 0 +L5: + r15 = var_object_size source + r16 = r15 << 1 + r17 = int_lt r14, r16 + if r17 goto L6 else goto L8 :: bool +L6: + r18 = CPyList_GetItemUnsafe(source, r14) + r19 = unbox(int, r18) + x_2 = r19 + r20 = CPyTagged_Add(x_2, 2) + r21 = box(int, r20) + r22 = CPyList_SetItemUnsafe(r13, r14, r21) +L7: + r23 = r14 + 2 + r14 = r23 + goto L5 +L8: + b = r13 + return 1 + +[case testGeneratorNext] +from typing import List, Optional + +def test(x: List[int]) -> None: + res = next((i for i in x), None) +[out] +def test(x): + x :: list + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: bit + r4 :: object + r5, i :: int + r6 :: object + r7 :: union[int, None] + r8 :: short_int + r9 :: object + res :: union[int, None] +L0: + r0 = 0 +L1: + r1 = var_object_size x + r2 = r1 << 1 + r3 = int_lt r0, r2 + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyList_GetItemUnsafe(x, r0) + r5 = unbox(int, r4) + i = r5 + r6 = box(int, i) + r7 = r6 + goto L5 +L3: + r8 = r0 + 2 + r0 = r8 + goto L1 +L4: + r9 = box(None, 1) + r7 = r9 +L5: + res = r7 + return 1 + +[case testSimplifyListUnion] +from typing import List, Union, Optional + +def narrow(a: Union[List[str], List[bytes], int]) -> int: + if isinstance(a, list): + return len(a) + return a +def loop(a: Union[List[str], List[bytes]]) -> None: + for x in a: + pass +def nested_union(a: Union[List[str], List[Optional[str]]]) -> None: + for x in a: + pass +[out] +def narrow(a): + a :: union[list, int] + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: list + r5 :: native_int + r6 :: short_int + r7 :: int +L0: + r0 = load_address PyList_Type + r1 = PyObject_IsInstance(a, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool +L1: + r4 = borrow cast(list, a) + r5 = var_object_size r4 + r6 = r5 << 1 + keep_alive a + return r6 +L2: + r7 = unbox(int, a) + return r7 +def loop(a): + a :: list + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: bit + r4 :: object + r5, x :: union[str, bytes] + r6 :: short_int +L0: + r0 = 0 +L1: + r1 = var_object_size a + r2 = r1 << 1 + r3 = int_lt r0, r2 + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyList_GetItemUnsafe(a, r0) + r5 = cast(union[str, bytes], r4) + x = r5 +L3: + r6 = r0 + 2 + r0 = r6 + goto L1 +L4: + return 1 +def nested_union(a): + a :: list + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: bit + r4 :: object + r5, x :: union[str, None] + r6 :: short_int +L0: + r0 = 0 +L1: + r1 = var_object_size a + r2 = r1 << 1 + r3 = int_lt r0, r2 + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyList_GetItemUnsafe(a, r0) + r5 = cast(union[str, None], r4) + x = r5 +L3: + r6 = r0 + 2 + r0 = r6 + goto L1 +L4: + return 1 diff --git a/mypyc/test-data/irbuild-match.test b/mypyc/test-data/irbuild-match.test new file mode 100644 index 000000000000..ab5a19624ba6 --- /dev/null +++ b/mypyc/test-data/irbuild-match.test @@ -0,0 +1,1717 @@ +[case testMatchValuePattern_python3_10] +def f(): + match 123: + case 123: + print("matched") +[out] +def f(): + r0 :: bit + r1 :: str + r2 :: object + r3 :: str + r4 :: object + r5 :: object[1] + r6 :: object_ptr + r7, r8 :: object +L0: + r0 = int_eq 246, 246 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = 'matched' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = [r1] + r6 = load_address r5 + r7 = _PyObject_Vectorcall(r4, r6, 1, 0) + keep_alive r1 + goto L3 +L2: +L3: + r8 = box(None, 1) + return r8 + +[case testMatchOrPattern_python3_10] +def f(): + match 123: + case 123 | 456: + print("matched") +[out] +def f(): + r0, r1 :: bit + r2 :: str + r3 :: object + r4 :: str + r5 :: object + r6 :: object[1] + r7 :: object_ptr + r8, r9 :: object +L0: + r0 = int_eq 246, 246 + if r0 goto L3 else goto L1 :: bool +L1: + r1 = int_eq 246, 912 + if r1 goto L3 else goto L2 :: bool +L2: + goto L4 +L3: + r2 = 'matched' + r3 = builtins :: module + r4 = 'print' + r5 = CPyObject_GetAttr(r3, r4) + r6 = [r2] + r7 = load_address r6 + r8 = _PyObject_Vectorcall(r5, r7, 1, 0) + keep_alive r2 + goto L5 +L4: +L5: + r9 = box(None, 1) + return r9 + +[case testMatchOrPatternManyPatterns_python3_10] +def f(): + match 1: + case 1 | 2 | 3 | 4: + print("matched") +[out] +def f(): + r0, r1, r2, r3 :: bit + r4 :: str + r5 :: object + r6 :: str + r7 :: object + r8 :: object[1] + r9 :: object_ptr + r10, r11 :: object +L0: + r0 = int_eq 2, 2 + if r0 goto L5 else goto L1 :: bool +L1: + r1 = int_eq 2, 4 + if r1 goto L5 else goto L2 :: bool +L2: + r2 = int_eq 2, 6 + if r2 goto L5 else goto L3 :: bool +L3: + r3 = int_eq 2, 8 + if r3 goto L5 else goto L4 :: bool +L4: + goto L6 +L5: + r4 = 'matched' + r5 = builtins :: module + r6 = 'print' + r7 = CPyObject_GetAttr(r5, r6) + r8 = [r4] + r9 = load_address r8 + r10 = _PyObject_Vectorcall(r7, r9, 1, 0) + keep_alive r4 + goto L7 +L6: +L7: + r11 = box(None, 1) + return r11 + +[case testMatchClassPattern_python3_10] +def f(): + match 123: + case int(): + print("matched") +[out] +def f(): + r0, r1 :: object + r2 :: bool + r3 :: str + r4 :: object + r5 :: str + r6 :: object + r7 :: object[1] + r8 :: object_ptr + r9, r10 :: object +L0: + r0 = load_address PyLong_Type + r1 = object 123 + r2 = CPy_TypeCheck(r1, r0) + if r2 goto L1 else goto L2 :: bool +L1: + r3 = 'matched' + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = [r3] + r8 = load_address r7 + r9 = _PyObject_Vectorcall(r6, r8, 1, 0) + keep_alive r3 + goto L3 +L2: +L3: + r10 = box(None, 1) + return r10 +[case testMatchExaustivePattern_python3_10] +def f(): + match 123: + case _: + print("matched") +[out] +def f(): + r0 :: str + r1 :: object + r2 :: str + r3 :: object + r4 :: object[1] + r5 :: object_ptr + r6, r7 :: object +L0: +L1: + r0 = 'matched' + r1 = builtins :: module + r2 = 'print' + r3 = CPyObject_GetAttr(r1, r2) + r4 = [r0] + r5 = load_address r4 + r6 = _PyObject_Vectorcall(r3, r5, 1, 0) + keep_alive r0 + goto L3 +L2: +L3: + r7 = box(None, 1) + return r7 +[case testMatchMultipleBodies_python3_10] +def f(): + match 123: + case 123: + print("matched") + case 456: + print("no match") +[out] +def f(): + r0 :: bit + r1 :: str + r2 :: object + r3 :: str + r4 :: object + r5 :: object[1] + r6 :: object_ptr + r7 :: object + r8 :: bit + r9 :: str + r10 :: object + r11 :: str + r12 :: object + r13 :: object[1] + r14 :: object_ptr + r15, r16 :: object +L0: + r0 = int_eq 246, 246 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = 'matched' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = [r1] + r6 = load_address r5 + r7 = _PyObject_Vectorcall(r4, r6, 1, 0) + keep_alive r1 + goto L5 +L2: + r8 = int_eq 246, 912 + if r8 goto L3 else goto L4 :: bool +L3: + r9 = 'no match' + r10 = builtins :: module + r11 = 'print' + r12 = CPyObject_GetAttr(r10, r11) + r13 = [r9] + r14 = load_address r13 + r15 = _PyObject_Vectorcall(r12, r14, 1, 0) + keep_alive r9 + goto L5 +L4: +L5: + r16 = box(None, 1) + return r16 + +[case testMatchMultiBodyAndComplexOr_python3_10] +def f(): + match 123: + case 1: + print("here 1") + case 2 | 3: + print("here 2 | 3") + case 123: + print("here 123") +[out] +def f(): + r0 :: bit + r1 :: str + r2 :: object + r3 :: str + r4 :: object + r5 :: object[1] + r6 :: object_ptr + r7 :: object + r8, r9 :: bit + r10 :: str + r11 :: object + r12 :: str + r13 :: object + r14 :: object[1] + r15 :: object_ptr + r16 :: object + r17 :: bit + r18 :: str + r19 :: object + r20 :: str + r21 :: object + r22 :: object[1] + r23 :: object_ptr + r24, r25 :: object +L0: + r0 = int_eq 246, 2 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = 'here 1' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = [r1] + r6 = load_address r5 + r7 = _PyObject_Vectorcall(r4, r6, 1, 0) + keep_alive r1 + goto L9 +L2: + r8 = int_eq 246, 4 + if r8 goto L5 else goto L3 :: bool +L3: + r9 = int_eq 246, 6 + if r9 goto L5 else goto L4 :: bool +L4: + goto L6 +L5: + r10 = 'here 2 | 3' + r11 = builtins :: module + r12 = 'print' + r13 = CPyObject_GetAttr(r11, r12) + r14 = [r10] + r15 = load_address r14 + r16 = _PyObject_Vectorcall(r13, r15, 1, 0) + keep_alive r10 + goto L9 +L6: + r17 = int_eq 246, 246 + if r17 goto L7 else goto L8 :: bool +L7: + r18 = 'here 123' + r19 = builtins :: module + r20 = 'print' + r21 = CPyObject_GetAttr(r19, r20) + r22 = [r18] + r23 = load_address r22 + r24 = _PyObject_Vectorcall(r21, r23, 1, 0) + keep_alive r18 + goto L9 +L8: +L9: + r25 = box(None, 1) + return r25 + +[case testMatchWithGuard_python3_10] +def f(): + match 123: + case 123 if True: + print("matched") +[out] +def f(): + r0 :: bit + r1 :: str + r2 :: object + r3 :: str + r4 :: object + r5 :: object[1] + r6 :: object_ptr + r7, r8 :: object +L0: + r0 = int_eq 246, 246 + if r0 goto L1 else goto L3 :: bool +L1: + if 1 goto L2 else goto L3 :: bool +L2: + r1 = 'matched' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = [r1] + r6 = load_address r5 + r7 = _PyObject_Vectorcall(r4, r6, 1, 0) + keep_alive r1 + goto L4 +L3: +L4: + r8 = box(None, 1) + return r8 + +[case testMatchSingleton_python3_10] +def f(): + match 123: + case True: + print("value is True") + case False: + print("value is False") + case None: + print("value is None") +[out] +def f(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4 :: object + r5 :: str + r6 :: object + r7 :: object[1] + r8 :: object_ptr + r9, r10, r11 :: object + r12 :: bit + r13 :: str + r14 :: object + r15 :: str + r16 :: object + r17 :: object[1] + r18 :: object_ptr + r19, r20, r21 :: object + r22 :: bit + r23 :: str + r24 :: object + r25 :: str + r26 :: object + r27 :: object[1] + r28 :: object_ptr + r29, r30 :: object +L0: + r0 = object 123 + r1 = box(bool, 1) + r2 = r0 == r1 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = 'value is True' + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = [r3] + r8 = load_address r7 + r9 = _PyObject_Vectorcall(r6, r8, 1, 0) + keep_alive r3 + goto L7 +L2: + r10 = object 123 + r11 = box(bool, 0) + r12 = r10 == r11 + if r12 goto L3 else goto L4 :: bool +L3: + r13 = 'value is False' + r14 = builtins :: module + r15 = 'print' + r16 = CPyObject_GetAttr(r14, r15) + r17 = [r13] + r18 = load_address r17 + r19 = _PyObject_Vectorcall(r16, r18, 1, 0) + keep_alive r13 + goto L7 +L4: + r20 = load_address _Py_NoneStruct + r21 = object 123 + r22 = r21 == r20 + if r22 goto L5 else goto L6 :: bool +L5: + r23 = 'value is None' + r24 = builtins :: module + r25 = 'print' + r26 = CPyObject_GetAttr(r24, r25) + r27 = [r23] + r28 = load_address r27 + r29 = _PyObject_Vectorcall(r26, r28, 1, 0) + keep_alive r23 + goto L7 +L6: +L7: + r30 = box(None, 1) + return r30 +[case testMatchRecursiveOrPattern_python3_10] +def f(): + match 1: + case 1 | int(): + print("matched") +[out] +def f(): + r0 :: bit + r1, r2 :: object + r3 :: bool + r4 :: str + r5 :: object + r6 :: str + r7 :: object + r8 :: object[1] + r9 :: object_ptr + r10, r11 :: object +L0: + r0 = int_eq 2, 2 + if r0 goto L3 else goto L1 :: bool +L1: + r1 = load_address PyLong_Type + r2 = object 1 + r3 = CPy_TypeCheck(r2, r1) + if r3 goto L3 else goto L2 :: bool +L2: + goto L4 +L3: + r4 = 'matched' + r5 = builtins :: module + r6 = 'print' + r7 = CPyObject_GetAttr(r5, r6) + r8 = [r4] + r9 = load_address r8 + r10 = _PyObject_Vectorcall(r7, r9, 1, 0) + keep_alive r4 + goto L5 +L4: +L5: + r11 = box(None, 1) + return r11 + +[case testMatchAsPattern_python3_10] +def f(): + match 123: + case 123 as x: + print(x) +[out] +def f(): + r0 :: bit + r1, x, r2 :: object + r3 :: str + r4 :: object + r5 :: object[1] + r6 :: object_ptr + r7, r8 :: object +L0: + r0 = int_eq 246, 246 + r1 = object 123 + x = r1 + if r0 goto L1 else goto L2 :: bool +L1: + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = [x] + r6 = load_address r5 + r7 = _PyObject_Vectorcall(r4, r6, 1, 0) + keep_alive x + goto L3 +L2: +L3: + r8 = box(None, 1) + return r8 + +[case testMatchAsPatternOnOrPattern_python3_10] +def f(): + match 1: + case (1 | 2) as x: + print(x) +[out] +def f(): + r0 :: bit + r1, x :: object + r2 :: bit + r3, r4 :: object + r5 :: str + r6 :: object + r7 :: object[1] + r8 :: object_ptr + r9, r10 :: object +L0: + r0 = int_eq 2, 2 + r1 = object 1 + x = r1 + if r0 goto L3 else goto L1 :: bool +L1: + r2 = int_eq 2, 4 + r3 = object 2 + x = r3 + if r2 goto L3 else goto L2 :: bool +L2: + goto L4 +L3: + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = [x] + r8 = load_address r7 + r9 = _PyObject_Vectorcall(r6, r8, 1, 0) + keep_alive x + goto L5 +L4: +L5: + r10 = box(None, 1) + return r10 + +[case testMatchAsPatternOnClassPattern_python3_10] +def f(): + match 123: + case int() as i: + print(i) +[out] +def f(): + r0, r1 :: object + r2 :: bool + i :: int + r3 :: object + r4 :: str + r5, r6 :: object + r7 :: object[1] + r8 :: object_ptr + r9, r10 :: object +L0: + r0 = load_address PyLong_Type + r1 = object 123 + r2 = CPy_TypeCheck(r1, r0) + if r2 goto L1 else goto L3 :: bool +L1: + i = 246 +L2: + r3 = builtins :: module + r4 = 'print' + r5 = CPyObject_GetAttr(r3, r4) + r6 = box(int, i) + r7 = [r6] + r8 = load_address r7 + r9 = _PyObject_Vectorcall(r5, r8, 1, 0) + keep_alive r6 + goto L4 +L3: +L4: + r10 = box(None, 1) + return r10 +[case testMatchClassPatternWithPositionalArgs_python3_10] +class Position: + __match_args__ = ("x", "y", "z") + + x: int + y: int + z: int + +def f(x): + match x: + case Position(1, 2, 3): + print("matched") +[out] +def Position.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.Position + r0, r1, r2 :: str + r3 :: tuple[str, str, str] +L0: + r0 = 'x' + r1 = 'y' + r2 = 'z' + r3 = (r0, r1, r2) + __mypyc_self__.__match_args__ = r3 + return 1 +def f(x): + x, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: str + r5, r6, r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: str + r12, r13, r14 :: object + r15 :: i32 + r16 :: bit + r17 :: bool + r18 :: str + r19, r20, r21 :: object + r22 :: i32 + r23 :: bit + r24 :: bool + r25 :: str + r26 :: object + r27 :: str + r28 :: object + r29 :: object[1] + r30 :: object_ptr + r31, r32 :: object +L0: + r0 = __main__.Position :: type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L5 :: bool +L1: + r4 = 'x' + r5 = CPyObject_GetAttr(x, r4) + r6 = object 1 + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L2 else goto L5 :: bool +L2: + r11 = 'y' + r12 = CPyObject_GetAttr(x, r11) + r13 = object 2 + r14 = PyObject_RichCompare(r12, r13, 2) + r15 = PyObject_IsTrue(r14) + r16 = r15 >= 0 :: signed + r17 = truncate r15: i32 to builtins.bool + if r17 goto L3 else goto L5 :: bool +L3: + r18 = 'z' + r19 = CPyObject_GetAttr(x, r18) + r20 = object 3 + r21 = PyObject_RichCompare(r19, r20, 2) + r22 = PyObject_IsTrue(r21) + r23 = r22 >= 0 :: signed + r24 = truncate r22: i32 to builtins.bool + if r24 goto L4 else goto L5 :: bool +L4: + r25 = 'matched' + r26 = builtins :: module + r27 = 'print' + r28 = CPyObject_GetAttr(r26, r27) + r29 = [r25] + r30 = load_address r29 + r31 = _PyObject_Vectorcall(r28, r30, 1, 0) + keep_alive r25 + goto L6 +L5: +L6: + r32 = box(None, 1) + return r32 +[case testMatchClassPatternWithKeywordPatterns_python3_10] +class Position: + x: int + y: int + z: int + +def f(x): + match x: + case Position(z=1, y=2, x=3): + print("matched") +[out] +def f(x): + x, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: str + r5, r6, r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: str + r12, r13, r14 :: object + r15 :: i32 + r16 :: bit + r17 :: bool + r18 :: str + r19, r20, r21 :: object + r22 :: i32 + r23 :: bit + r24 :: bool + r25 :: str + r26 :: object + r27 :: str + r28 :: object + r29 :: object[1] + r30 :: object_ptr + r31, r32 :: object +L0: + r0 = __main__.Position :: type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L5 :: bool +L1: + r4 = 'z' + r5 = CPyObject_GetAttr(x, r4) + r6 = object 1 + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L2 else goto L5 :: bool +L2: + r11 = 'y' + r12 = CPyObject_GetAttr(x, r11) + r13 = object 2 + r14 = PyObject_RichCompare(r12, r13, 2) + r15 = PyObject_IsTrue(r14) + r16 = r15 >= 0 :: signed + r17 = truncate r15: i32 to builtins.bool + if r17 goto L3 else goto L5 :: bool +L3: + r18 = 'x' + r19 = CPyObject_GetAttr(x, r18) + r20 = object 3 + r21 = PyObject_RichCompare(r19, r20, 2) + r22 = PyObject_IsTrue(r21) + r23 = r22 >= 0 :: signed + r24 = truncate r22: i32 to builtins.bool + if r24 goto L4 else goto L5 :: bool +L4: + r25 = 'matched' + r26 = builtins :: module + r27 = 'print' + r28 = CPyObject_GetAttr(r26, r27) + r29 = [r25] + r30 = load_address r29 + r31 = _PyObject_Vectorcall(r28, r30, 1, 0) + keep_alive r25 + goto L6 +L5: +L6: + r32 = box(None, 1) + return r32 +[case testMatchClassPatternWithNestedPattern_python3_10] +class C: + num: int + +def f(x): + match x: + case C(num=1 | 2): + print("matched") +[out] +def f(x): + x, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: str + r5, r6, r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11, r12 :: object + r13 :: i32 + r14 :: bit + r15 :: bool + r16 :: str + r17 :: object + r18 :: str + r19 :: object + r20 :: object[1] + r21 :: object_ptr + r22, r23 :: object +L0: + r0 = __main__.C :: type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L5 :: bool +L1: + r4 = 'num' + r5 = CPyObject_GetAttr(x, r4) + r6 = object 1 + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L4 else goto L2 :: bool +L2: + r11 = object 2 + r12 = PyObject_RichCompare(r5, r11, 2) + r13 = PyObject_IsTrue(r12) + r14 = r13 >= 0 :: signed + r15 = truncate r13: i32 to builtins.bool + if r15 goto L4 else goto L3 :: bool +L3: + goto L5 +L4: + r16 = 'matched' + r17 = builtins :: module + r18 = 'print' + r19 = CPyObject_GetAttr(r17, r18) + r20 = [r16] + r21 = load_address r20 + r22 = _PyObject_Vectorcall(r19, r21, 1, 0) + keep_alive r16 + goto L6 +L5: +L6: + r23 = box(None, 1) + return r23 +[case testAsPatternDoesntBleedIntoSubPatterns_python3_10] +class C: + __match_args__ = ("a", "b") + a: int + b: int + +def f(x): + match x: + case C(1, 2) as y: + print("matched") +[out] +def C.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.C + r0, r1 :: str + r2 :: tuple[str, str] +L0: + r0 = 'a' + r1 = 'b' + r2 = (r0, r1) + __mypyc_self__.__match_args__ = r2 + return 1 +def f(x): + x, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4, y :: __main__.C + r5 :: str + r6, r7, r8 :: object + r9 :: i32 + r10 :: bit + r11 :: bool + r12 :: str + r13, r14, r15 :: object + r16 :: i32 + r17 :: bit + r18 :: bool + r19 :: str + r20 :: object + r21 :: str + r22 :: object + r23 :: object[1] + r24 :: object_ptr + r25, r26 :: object +L0: + r0 = __main__.C :: type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L5 :: bool +L1: + r4 = cast(__main__.C, x) + y = r4 +L2: + r5 = 'a' + r6 = CPyObject_GetAttr(x, r5) + r7 = object 1 + r8 = PyObject_RichCompare(r6, r7, 2) + r9 = PyObject_IsTrue(r8) + r10 = r9 >= 0 :: signed + r11 = truncate r9: i32 to builtins.bool + if r11 goto L3 else goto L5 :: bool +L3: + r12 = 'b' + r13 = CPyObject_GetAttr(x, r12) + r14 = object 2 + r15 = PyObject_RichCompare(r13, r14, 2) + r16 = PyObject_IsTrue(r15) + r17 = r16 >= 0 :: signed + r18 = truncate r16: i32 to builtins.bool + if r18 goto L4 else goto L5 :: bool +L4: + r19 = 'matched' + r20 = builtins :: module + r21 = 'print' + r22 = CPyObject_GetAttr(r20, r21) + r23 = [r19] + r24 = load_address r23 + r25 = _PyObject_Vectorcall(r22, r24, 1, 0) + keep_alive r19 + goto L6 +L5: +L6: + r26 = box(None, 1) + return r26 +[case testMatchClassPatternPositionalCapture_python3_10] +class C: + __match_args__ = ("x",) + + x: int + +def f(x): + match x: + case C(num): + print("matched") +[out] +def C.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.C + r0 :: str + r1 :: tuple[str] +L0: + r0 = 'x' + r1 = (r0) + __mypyc_self__.__match_args__ = r1 + return 1 +def f(x): + x, r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: str + r5 :: object + r6, num :: int + r7 :: str + r8 :: object + r9 :: str + r10 :: object + r11 :: object[1] + r12 :: object_ptr + r13, r14 :: object +L0: + r0 = __main__.C :: type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L3 :: bool +L1: + r4 = 'x' + r5 = CPyObject_GetAttr(x, r4) + r6 = unbox(int, r5) + num = r6 +L2: + r7 = 'matched' + r8 = builtins :: module + r9 = 'print' + r10 = CPyObject_GetAttr(r8, r9) + r11 = [r7] + r12 = load_address r11 + r13 = _PyObject_Vectorcall(r10, r12, 1, 0) + keep_alive r7 + goto L4 +L3: +L4: + r14 = box(None, 1) + return r14 +[case testMatchMappingEmpty_python3_10] +def f(x): + match x: + case {}: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: str + r3 :: object + r4 :: str + r5 :: object + r6 :: object[1] + r7 :: object_ptr + r8, r9 :: object +L0: + r0 = CPyMapping_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 'matched' + r3 = builtins :: module + r4 = 'print' + r5 = CPyObject_GetAttr(r3, r4) + r6 = [r2] + r7 = load_address r6 + r8 = _PyObject_Vectorcall(r5, r7, 1, 0) + keep_alive r2 + goto L3 +L2: +L3: + r9 = box(None, 1) + return r9 +[case testMatchMappingPatternWithKeys_python3_10] +def f(x): + match x: + case {"key": "value"}: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: str + r3 :: i32 + r4 :: bit + r5 :: object + r6 :: str + r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: str + r12 :: object + r13 :: str + r14 :: object + r15 :: object[1] + r16 :: object_ptr + r17, r18 :: object +L0: + r0 = CPyMapping_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = 'key' + r3 = PyMapping_HasKey(x, r2) + r4 = r3 != 0 + if r4 goto L2 else goto L4 :: bool +L2: + r5 = PyObject_GetItem(x, r2) + r6 = 'value' + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L3 else goto L4 :: bool +L3: + r11 = 'matched' + r12 = builtins :: module + r13 = 'print' + r14 = CPyObject_GetAttr(r12, r13) + r15 = [r11] + r16 = load_address r15 + r17 = _PyObject_Vectorcall(r14, r16, 1, 0) + keep_alive r11 + goto L5 +L4: +L5: + r18 = box(None, 1) + return r18 +[case testMatchMappingPatternWithRest_python3_10] +def f(x): + match x: + case {**rest}: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2, rest :: dict + r3 :: str + r4 :: object + r5 :: str + r6 :: object + r7 :: object[1] + r8 :: object_ptr + r9, r10 :: object +L0: + r0 = CPyMapping_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L3 :: bool +L1: + r2 = CPyDict_FromAny(x) + rest = r2 +L2: + r3 = 'matched' + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = [r3] + r8 = load_address r7 + r9 = _PyObject_Vectorcall(r6, r8, 1, 0) + keep_alive r3 + goto L4 +L3: +L4: + r10 = box(None, 1) + return r10 +[case testMatchMappingPatternWithRestPopKeys_python3_10] +def f(x): + match x: + case {"key": "value", **rest}: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: str + r3 :: i32 + r4 :: bit + r5 :: object + r6 :: str + r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11, rest :: dict + r12 :: i32 + r13 :: bit + r14 :: str + r15 :: object + r16 :: str + r17 :: object + r18 :: object[1] + r19 :: object_ptr + r20, r21 :: object +L0: + r0 = CPyMapping_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L5 :: bool +L1: + r2 = 'key' + r3 = PyMapping_HasKey(x, r2) + r4 = r3 != 0 + if r4 goto L2 else goto L5 :: bool +L2: + r5 = PyObject_GetItem(x, r2) + r6 = 'value' + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L3 else goto L5 :: bool +L3: + r11 = CPyDict_FromAny(x) + rest = r11 + r12 = PyDict_DelItem(r11, r2) + r13 = r12 >= 0 :: signed +L4: + r14 = 'matched' + r15 = builtins :: module + r16 = 'print' + r17 = CPyObject_GetAttr(r15, r16) + r18 = [r14] + r19 = load_address r18 + r20 = _PyObject_Vectorcall(r17, r19, 1, 0) + keep_alive r14 + goto L6 +L5: +L6: + r21 = box(None, 1) + return r21 +[case testMatchEmptySequencePattern_python3_10] +def f(x): + match x: + case []: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: native_int + r3, r4 :: bit + r5 :: str + r6 :: object + r7 :: str + r8 :: object + r9 :: object[1] + r10 :: object_ptr + r11, r12 :: object +L0: + r0 = CPySequence_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L3 :: bool +L1: + r2 = PyObject_Size(x) + r3 = r2 >= 0 :: signed + r4 = r2 == 0 + if r4 goto L2 else goto L3 :: bool +L2: + r5 = 'matched' + r6 = builtins :: module + r7 = 'print' + r8 = CPyObject_GetAttr(r6, r7) + r9 = [r5] + r10 = load_address r9 + r11 = _PyObject_Vectorcall(r8, r10, 1, 0) + keep_alive r5 + goto L4 +L3: +L4: + r12 = box(None, 1) + return r12 +[case testMatchFixedLengthSequencePattern_python3_10] +def f(x): + match x: + case [1, 2]: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: native_int + r3, r4 :: bit + r5, r6, r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11, r12, r13 :: object + r14 :: i32 + r15 :: bit + r16 :: bool + r17 :: str + r18 :: object + r19 :: str + r20 :: object + r21 :: object[1] + r22 :: object_ptr + r23, r24 :: object +L0: + r0 = CPySequence_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L5 :: bool +L1: + r2 = PyObject_Size(x) + r3 = r2 >= 0 :: signed + r4 = r2 == 2 + if r4 goto L2 else goto L5 :: bool +L2: + r5 = PySequence_GetItem(x, 0) + r6 = object 1 + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L3 else goto L5 :: bool +L3: + r11 = PySequence_GetItem(x, 1) + r12 = object 2 + r13 = PyObject_RichCompare(r11, r12, 2) + r14 = PyObject_IsTrue(r13) + r15 = r14 >= 0 :: signed + r16 = truncate r14: i32 to builtins.bool + if r16 goto L4 else goto L5 :: bool +L4: + r17 = 'matched' + r18 = builtins :: module + r19 = 'print' + r20 = CPyObject_GetAttr(r18, r19) + r21 = [r17] + r22 = load_address r21 + r23 = _PyObject_Vectorcall(r20, r22, 1, 0) + keep_alive r17 + goto L6 +L5: +L6: + r24 = box(None, 1) + return r24 +[case testMatchSequencePatternWithTrailingUnboundStar_python3_10] +def f(x): + match x: + case [1, 2, *_]: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: native_int + r3, r4 :: bit + r5, r6, r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11, r12, r13 :: object + r14 :: i32 + r15 :: bit + r16 :: bool + r17 :: str + r18 :: object + r19 :: str + r20 :: object + r21 :: object[1] + r22 :: object_ptr + r23, r24 :: object +L0: + r0 = CPySequence_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L5 :: bool +L1: + r2 = PyObject_Size(x) + r3 = r2 >= 0 :: signed + r4 = r2 >= 2 :: signed + if r4 goto L2 else goto L5 :: bool +L2: + r5 = PySequence_GetItem(x, 0) + r6 = object 1 + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L3 else goto L5 :: bool +L3: + r11 = PySequence_GetItem(x, 1) + r12 = object 2 + r13 = PyObject_RichCompare(r11, r12, 2) + r14 = PyObject_IsTrue(r13) + r15 = r14 >= 0 :: signed + r16 = truncate r14: i32 to builtins.bool + if r16 goto L4 else goto L5 :: bool +L4: + r17 = 'matched' + r18 = builtins :: module + r19 = 'print' + r20 = CPyObject_GetAttr(r18, r19) + r21 = [r17] + r22 = load_address r21 + r23 = _PyObject_Vectorcall(r20, r22, 1, 0) + keep_alive r17 + goto L6 +L5: +L6: + r24 = box(None, 1) + return r24 +[case testMatchSequencePatternWithTrailingBoundStar_python3_10] +def f(x): + match x: + case [1, 2, *rest]: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: native_int + r3, r4 :: bit + r5, r6, r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11, r12, r13 :: object + r14 :: i32 + r15 :: bit + r16 :: bool + r17 :: native_int + r18, rest :: object + r19 :: str + r20 :: object + r21 :: str + r22 :: object + r23 :: object[1] + r24 :: object_ptr + r25, r26 :: object +L0: + r0 = CPySequence_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L6 :: bool +L1: + r2 = PyObject_Size(x) + r3 = r2 >= 0 :: signed + r4 = r2 >= 2 :: signed + if r4 goto L2 else goto L6 :: bool +L2: + r5 = PySequence_GetItem(x, 0) + r6 = object 1 + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L3 else goto L6 :: bool +L3: + r11 = PySequence_GetItem(x, 1) + r12 = object 2 + r13 = PyObject_RichCompare(r11, r12, 2) + r14 = PyObject_IsTrue(r13) + r15 = r14 >= 0 :: signed + r16 = truncate r14: i32 to builtins.bool + if r16 goto L4 else goto L6 :: bool +L4: + r17 = r2 - 0 + r18 = PySequence_GetSlice(x, 2, r17) + rest = r18 +L5: + r19 = 'matched' + r20 = builtins :: module + r21 = 'print' + r22 = CPyObject_GetAttr(r20, r21) + r23 = [r19] + r24 = load_address r23 + r25 = _PyObject_Vectorcall(r22, r24, 1, 0) + keep_alive r19 + goto L7 +L6: +L7: + r26 = box(None, 1) + return r26 +[case testMatchSequenceWithStarPatternInTheMiddle_python3_10] +def f(x): + match x: + case ["start", *rest, "end"]: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: native_int + r3, r4 :: bit + r5 :: object + r6 :: str + r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: native_int + r12 :: object + r13 :: str + r14 :: object + r15 :: i32 + r16 :: bit + r17 :: bool + r18 :: native_int + r19, rest :: object + r20 :: str + r21 :: object + r22 :: str + r23 :: object + r24 :: object[1] + r25 :: object_ptr + r26, r27 :: object +L0: + r0 = CPySequence_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L6 :: bool +L1: + r2 = PyObject_Size(x) + r3 = r2 >= 0 :: signed + r4 = r2 >= 2 :: signed + if r4 goto L2 else goto L6 :: bool +L2: + r5 = PySequence_GetItem(x, 0) + r6 = 'start' + r7 = PyObject_RichCompare(r5, r6, 2) + r8 = PyObject_IsTrue(r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L3 else goto L6 :: bool +L3: + r11 = r2 - 1 + r12 = PySequence_GetItem(x, r11) + r13 = 'end' + r14 = PyObject_RichCompare(r12, r13, 2) + r15 = PyObject_IsTrue(r14) + r16 = r15 >= 0 :: signed + r17 = truncate r15: i32 to builtins.bool + if r17 goto L4 else goto L6 :: bool +L4: + r18 = r2 - 1 + r19 = PySequence_GetSlice(x, 1, r18) + rest = r19 +L5: + r20 = 'matched' + r21 = builtins :: module + r22 = 'print' + r23 = CPyObject_GetAttr(r21, r22) + r24 = [r20] + r25 = load_address r24 + r26 = _PyObject_Vectorcall(r23, r25, 1, 0) + keep_alive r20 + goto L7 +L6: +L7: + r27 = box(None, 1) + return r27 +[case testMatchSequenceWithStarPatternAtTheStart_python3_10] +def f(x): + match x: + case [*rest, 1, 2]: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: native_int + r3, r4 :: bit + r5 :: native_int + r6, r7, r8 :: object + r9 :: i32 + r10 :: bit + r11 :: bool + r12 :: native_int + r13, r14, r15 :: object + r16 :: i32 + r17 :: bit + r18 :: bool + r19 :: native_int + r20, rest :: object + r21 :: str + r22 :: object + r23 :: str + r24 :: object + r25 :: object[1] + r26 :: object_ptr + r27, r28 :: object +L0: + r0 = CPySequence_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L6 :: bool +L1: + r2 = PyObject_Size(x) + r3 = r2 >= 0 :: signed + r4 = r2 >= 2 :: signed + if r4 goto L2 else goto L6 :: bool +L2: + r5 = r2 - 2 + r6 = PySequence_GetItem(x, r5) + r7 = object 1 + r8 = PyObject_RichCompare(r6, r7, 2) + r9 = PyObject_IsTrue(r8) + r10 = r9 >= 0 :: signed + r11 = truncate r9: i32 to builtins.bool + if r11 goto L3 else goto L6 :: bool +L3: + r12 = r2 - 1 + r13 = PySequence_GetItem(x, r12) + r14 = object 2 + r15 = PyObject_RichCompare(r13, r14, 2) + r16 = PyObject_IsTrue(r15) + r17 = r16 >= 0 :: signed + r18 = truncate r16: i32 to builtins.bool + if r18 goto L4 else goto L6 :: bool +L4: + r19 = r2 - 2 + r20 = PySequence_GetSlice(x, 0, r19) + rest = r20 +L5: + r21 = 'matched' + r22 = builtins :: module + r23 = 'print' + r24 = CPyObject_GetAttr(r22, r23) + r25 = [r21] + r26 = load_address r25 + r27 = _PyObject_Vectorcall(r24, r26, 1, 0) + keep_alive r21 + goto L7 +L6: +L7: + r28 = box(None, 1) + return r28 +[case testMatchBuiltinClassPattern_python3_10] +def f(x): + match x: + case int(y): + print("matched") +[out] +def f(x): + x, r0 :: object + r1 :: bool + r2, y :: int + r3 :: str + r4 :: object + r5 :: str + r6 :: object + r7 :: object[1] + r8 :: object_ptr + r9, r10 :: object +L0: + r0 = load_address PyLong_Type + r1 = CPy_TypeCheck(x, r0) + if r1 goto L1 else goto L3 :: bool +L1: + r2 = unbox(int, x) + y = r2 +L2: + r3 = 'matched' + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = [r3] + r8 = load_address r7 + r9 = _PyObject_Vectorcall(r6, r8, 1, 0) + keep_alive r3 + goto L4 +L3: +L4: + r10 = box(None, 1) + return r10 +[case testMatchSequenceCaptureAll_python3_10] +def f(x): + match x: + case [*rest]: + print("matched") +[out] +def f(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: native_int + r3, r4 :: bit + r5 :: native_int + r6, rest :: object + r7 :: str + r8 :: object + r9 :: str + r10 :: object + r11 :: object[1] + r12 :: object_ptr + r13, r14 :: object +L0: + r0 = CPySequence_Check(x) + r1 = r0 != 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = PyObject_Size(x) + r3 = r2 >= 0 :: signed + r4 = r2 >= 0 :: signed + if r4 goto L2 else goto L4 :: bool +L2: + r5 = r2 - 0 + r6 = PySequence_GetSlice(x, 0, r5) + rest = r6 +L3: + r7 = 'matched' + r8 = builtins :: module + r9 = 'print' + r10 = CPyObject_GetAttr(r8, r9) + r11 = [r7] + r12 = load_address r11 + r13 = _PyObject_Vectorcall(r10, r12, 1, 0) + keep_alive r7 + goto L5 +L4: +L5: + r14 = box(None, 1) + return r14 +[case testMatchTypeAnnotatedNativeClass_python3_10] +class A: + a: int + +def f(x: A | int) -> int: + match x: + case A(a=a): + return a + case int(): + return x +[out] +def f(x): + x :: union[__main__.A, int] + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: str + r5 :: object + r6, a :: int + r7 :: object + r8 :: bool + r9 :: int +L0: + r0 = __main__.A :: type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L3 :: bool +L1: + r4 = 'a' + r5 = CPyObject_GetAttr(x, r4) + r6 = unbox(int, r5) + a = r6 +L2: + return a +L3: + r7 = load_address PyLong_Type + r8 = CPy_TypeCheck(x, r7) + if r8 goto L4 else goto L5 :: bool +L4: + r9 = unbox(int, x) + return r9 +L5: +L6: + unreachable diff --git a/mypyc/test-data/irbuild-math.test b/mypyc/test-data/irbuild-math.test new file mode 100644 index 000000000000..470e60c74f7d --- /dev/null +++ b/mypyc/test-data/irbuild-math.test @@ -0,0 +1,64 @@ +[case testMathLiteralsAreInlined] +import math +from math import pi, e, tau, inf, nan + +def f1() -> float: + return pi + +def f2() -> float: + return math.pi + +def f3() -> float: + return math.e + +def f4() -> float: + return math.e + +def f5() -> float: + return math.tau + +def f6() -> float: + return math.tau + +def f7() -> float: + return math.inf +def f8() -> float: + return math.inf + +def f9() -> float: + return math.nan + +def f10() -> float: + return math.nan + +[out] +def f1(): +L0: + return 3.141592653589793 +def f2(): +L0: + return 3.141592653589793 +def f3(): +L0: + return 2.718281828459045 +def f4(): +L0: + return 2.718281828459045 +def f5(): +L0: + return 6.283185307179586 +def f6(): +L0: + return 6.283185307179586 +def f7(): +L0: + return inf +def f8(): +L0: + return inf +def f9(): +L0: + return nan +def f10(): +L0: + return nan diff --git a/mypyc/test-data/irbuild-nested.test b/mypyc/test-data/irbuild-nested.test new file mode 100644 index 000000000000..62ae6eb9ee35 --- /dev/null +++ b/mypyc/test-data/irbuild-nested.test @@ -0,0 +1,807 @@ +[case testNestedFunctions] +from typing import Callable + +def a() -> Callable[[], object]: + def inner() -> object: + return None + return inner + +def b() -> Callable[[], Callable[[], str]]: + def first() -> Callable[[], str]: + def second() -> str: + return 'b.first.second: nested function' + return second + return first + +def c(num: float) -> Callable[[str], str]: + def inner(s: str) -> str: + return s + '!' + return inner + +def d(num: float) -> str: + def inner(s: str) -> str: + return s + '?' + a = inner('one') + b = inner('two') + return a + +def inner() -> str: + return 'inner: normal function' + +def first() -> str: + return 'first: normal function' + +def second() -> str: + return 'second: normal function' +[out] +def inner_a_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_a_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.inner_a_obj + r0 :: __main__.a_env + r1 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = box(None, 1) + return r1 +def a(): + r0 :: __main__.a_env + r1 :: __main__.inner_a_obj + r2 :: bool + inner :: object +L0: + r0 = a_env() + r1 = inner_a_obj() + r1.__mypyc_env__ = r0; r2 = is_error + inner = r1 + return inner +def second_b_first_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def second_b_first_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.second_b_first_obj + r0 :: __main__.first_b_env + r1 :: __main__.b_env + r2 :: str +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.__mypyc_env__ + r2 = 'b.first.second: nested function' + return r2 +def first_b_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def first_b_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.first_b_obj + r0 :: __main__.b_env + r1 :: __main__.first_b_env + r2 :: bool + r3 :: __main__.second_b_first_obj + r4 :: bool + second :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = first_b_env() + r1.__mypyc_env__ = r0; r2 = is_error + r3 = second_b_first_obj() + r3.__mypyc_env__ = r1; r4 = is_error + second = r3 + return second +def b(): + r0 :: __main__.b_env + r1 :: __main__.first_b_obj + r2 :: bool + first :: object +L0: + r0 = b_env() + r1 = first_b_obj() + r1.__mypyc_env__ = r0; r2 = is_error + first = r1 + return first +def inner_c_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_c_obj.__call__(__mypyc_self__, s): + __mypyc_self__ :: __main__.inner_c_obj + s :: str + r0 :: __main__.c_env + r1, r2 :: str +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = '!' + r2 = PyUnicode_Concat(s, r1) + return r2 +def c(num): + num :: float + r0 :: __main__.c_env + r1 :: __main__.inner_c_obj + r2 :: bool + inner :: object +L0: + r0 = c_env() + r1 = inner_c_obj() + r1.__mypyc_env__ = r0; r2 = is_error + inner = r1 + return inner +def inner_d_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_d_obj.__call__(__mypyc_self__, s): + __mypyc_self__ :: __main__.inner_d_obj + s :: str + r0 :: __main__.d_env + r1, r2 :: str +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = '?' + r2 = PyUnicode_Concat(s, r1) + return r2 +def d(num): + num :: float + r0 :: __main__.d_env + r1 :: __main__.inner_d_obj + r2 :: bool + inner :: object + r3 :: str + r4 :: object + r5, a, r6 :: str + r7 :: object + r8, b :: str +L0: + r0 = d_env() + r1 = inner_d_obj() + r1.__mypyc_env__ = r0; r2 = is_error + inner = r1 + r3 = 'one' + r4 = PyObject_CallFunctionObjArgs(inner, r3, 0) + r5 = cast(str, r4) + a = r5 + r6 = 'two' + r7 = PyObject_CallFunctionObjArgs(inner, r6, 0) + r8 = cast(str, r7) + b = r8 + return a +def inner(): + r0 :: str +L0: + r0 = 'inner: normal function' + return r0 +def first(): + r0 :: str +L0: + r0 = 'first: normal function' + return r0 +def second(): + r0 :: str +L0: + r0 = 'second: normal function' + return r0 + +[case testFreeVars] +from typing import Callable + +def a(num: int) -> int: + def inner() -> int: + return num + return inner() + +def b() -> int: + num = 3 + def inner() -> int: + nonlocal num + num = 4 + foo = 6 + return num + return inner() + num + +def c(flag: bool) -> str: + if flag: + def inner() -> str: + return 'f.inner: first definition' + else: + def inner() -> str: + return 'f.inner: second definition' + return inner() + +[out] +def inner_a_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_a_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.inner_a_obj + r0 :: __main__.a_env + r1 :: int +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.num + return r1 +def a(num): + num :: int + r0 :: __main__.a_env + r1 :: bool + r2 :: __main__.inner_a_obj + r3 :: bool + inner, r4 :: object + r5 :: int +L0: + r0 = a_env() + r0.num = num; r1 = is_error + r2 = inner_a_obj() + r2.__mypyc_env__ = r0; r3 = is_error + inner = r2 + r4 = PyObject_CallFunctionObjArgs(inner, 0) + r5 = unbox(int, r4) + return r5 +def inner_b_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_b_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.inner_b_obj + r0 :: __main__.b_env + r1 :: bool + foo, r2 :: int +L0: + r0 = __mypyc_self__.__mypyc_env__ + r0.num = 8; r1 = is_error + foo = 12 + r2 = r0.num + return r2 +def b(): + r0 :: __main__.b_env + r1 :: bool + r2 :: __main__.inner_b_obj + r3 :: bool + inner, r4 :: object + r5, r6, r7 :: int +L0: + r0 = b_env() + r0.num = 6; r1 = is_error + r2 = inner_b_obj() + r2.__mypyc_env__ = r0; r3 = is_error + inner = r2 + r4 = PyObject_CallFunctionObjArgs(inner, 0) + r5 = unbox(int, r4) + r6 = r0.num + r7 = CPyTagged_Add(r5, r6) + return r7 +def inner_c_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_c_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.inner_c_obj + r0 :: __main__.c_env + r1 :: str +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = 'f.inner: first definition' + return r1 +def inner_c_obj_0.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_c_obj_0.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.inner_c_obj_0 + r0 :: __main__.c_env + r1 :: str +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = 'f.inner: second definition' + return r1 +def c(flag): + flag :: bool + r0 :: __main__.c_env + r1 :: __main__.inner_c_obj + r2 :: bool + inner :: object + r3 :: __main__.inner_c_obj_0 + r4 :: bool + r5 :: object + r6 :: str +L0: + r0 = c_env() + if flag goto L1 else goto L2 :: bool +L1: + r1 = inner_c_obj() + r1.__mypyc_env__ = r0; r2 = is_error + inner = r1 + goto L3 +L2: + r3 = inner_c_obj_0() + r3.__mypyc_env__ = r0; r4 = is_error + inner = r3 +L3: + r5 = PyObject_CallFunctionObjArgs(inner, 0) + r6 = cast(str, r5) + return r6 + +[case testSpecialNested] +def a() -> int: + x = 1 + def b() -> int: + x += 1 + def c() -> int: + return x + return c() + return b() + +[out] +def c_a_b_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def c_a_b_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.c_a_b_obj + r0 :: __main__.b_a_env + r1 :: __main__.a_env + r2 :: int +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.__mypyc_env__ + r2 = r1.x + return r2 +def b_a_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def b_a_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.b_a_obj + r0 :: __main__.a_env + r1 :: __main__.b_a_env + r2 :: bool + r3, r4 :: int + r5 :: bool + r6 :: __main__.c_a_b_obj + r7 :: bool + c, r8 :: object + r9 :: int +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = b_a_env() + r1.__mypyc_env__ = r0; r2 = is_error + r3 = r0.x + r4 = CPyTagged_Add(r3, 2) + r0.x = r4; r5 = is_error + r6 = c_a_b_obj() + r6.__mypyc_env__ = r1; r7 = is_error + c = r6 + r8 = PyObject_CallFunctionObjArgs(c, 0) + r9 = unbox(int, r8) + return r9 +def a(): + r0 :: __main__.a_env + r1 :: bool + r2 :: __main__.b_a_obj + r3 :: bool + b, r4 :: object + r5 :: int +L0: + r0 = a_env() + r0.x = 2; r1 = is_error + r2 = b_a_obj() + r2.__mypyc_env__ = r0; r3 = is_error + b = r2 + r4 = PyObject_CallFunctionObjArgs(b, 0) + r5 = unbox(int, r4) + return r5 + +[case testNestedFunctionInsideStatements] +def f(flag: bool) -> str: + if flag: + def inner() -> str: + return 'f.inner: first definition' + else: + def inner() -> str: + return 'f.inner: second definition' + return inner() +[out] +def inner_f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_f_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.inner_f_obj + r0 :: __main__.f_env + r1 :: str +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = 'f.inner: first definition' + return r1 +def inner_f_obj_0.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def inner_f_obj_0.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.inner_f_obj_0 + r0 :: __main__.f_env + r1 :: str +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = 'f.inner: second definition' + return r1 +def f(flag): + flag :: bool + r0 :: __main__.f_env + r1 :: __main__.inner_f_obj + r2 :: bool + inner :: object + r3 :: __main__.inner_f_obj_0 + r4 :: bool + r5 :: object + r6 :: str +L0: + r0 = f_env() + if flag goto L1 else goto L2 :: bool +L1: + r1 = inner_f_obj() + r1.__mypyc_env__ = r0; r2 = is_error + inner = r1 + goto L3 +L2: + r3 = inner_f_obj_0() + r3.__mypyc_env__ = r0; r4 = is_error + inner = r3 +L3: + r5 = PyObject_CallFunctionObjArgs(inner, 0) + r6 = cast(str, r5) + return r6 + +[case testNestedFunctionsCallEachOther] +from typing import Callable, List + +def f(a: int) -> int: + def foo() -> int: + return a + 1 + + def bar() -> int: + return foo() + + def baz(n: int) -> int: + if n == 0: + return 0 + return n + baz(n - 1) + + return bar() + baz(a) + +[out] +def foo_f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def foo_f_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.foo_f_obj + r0 :: __main__.f_env + r1, r2 :: int +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.a + r2 = CPyTagged_Add(r1, 2) + return r2 +def bar_f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def bar_f_obj.__call__(__mypyc_self__): + __mypyc_self__ :: __main__.bar_f_obj + r0 :: __main__.f_env + r1, r2 :: object + r3 :: int +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.foo + r2 = PyObject_CallFunctionObjArgs(r1, 0) + r3 = unbox(int, r2) + return r3 +def baz_f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def baz_f_obj.__call__(__mypyc_self__, n): + __mypyc_self__ :: __main__.baz_f_obj + n :: int + r0 :: __main__.f_env + r1 :: bit + r2 :: int + r3, r4, r5 :: object + r6, r7 :: int +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = int_eq n, 0 + if r1 goto L1 else goto L2 :: bool +L1: + return 0 +L2: + r2 = CPyTagged_Subtract(n, 2) + r3 = r0.baz + r4 = box(int, r2) + r5 = PyObject_CallFunctionObjArgs(r3, r4, 0) + r6 = unbox(int, r5) + r7 = CPyTagged_Add(n, r6) + return r7 +def f(a): + a :: int + r0 :: __main__.f_env + r1 :: bool + r2 :: __main__.foo_f_obj + r3, r4 :: bool + r5 :: __main__.bar_f_obj + r6, r7 :: bool + r8 :: __main__.baz_f_obj + r9, r10 :: bool + r11, r12 :: object + r13, r14 :: int + r15, r16, r17 :: object + r18, r19 :: int +L0: + r0 = f_env() + r0.a = a; r1 = is_error + r2 = foo_f_obj() + r2.__mypyc_env__ = r0; r3 = is_error + r0.foo = r2; r4 = is_error + r5 = bar_f_obj() + r5.__mypyc_env__ = r0; r6 = is_error + r0.bar = r5; r7 = is_error + r8 = baz_f_obj() + r8.__mypyc_env__ = r0; r9 = is_error + r0.baz = r8; r10 = is_error + r11 = r0.bar + r12 = PyObject_CallFunctionObjArgs(r11, 0) + r13 = unbox(int, r12) + r14 = r0.a + r15 = r0.baz + r16 = box(int, r14) + r17 = PyObject_CallFunctionObjArgs(r15, r16, 0) + r18 = unbox(int, r17) + r19 = CPyTagged_Add(r13, r18) + return r19 + +[case testLambdas] +def f(x: int, y: int) -> None: + s = lambda a, b: a + b + t = lambda a, b: s(a, b) + return t(x, y) + +[out] +def __mypyc_lambda__0_f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def __mypyc_lambda__0_f_obj.__call__(__mypyc_self__, a, b): + __mypyc_self__ :: __main__.__mypyc_lambda__0_f_obj + a, b :: object + r0 :: __main__.f_env + r1 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = PyNumber_Add(a, b) + return r1 +def __mypyc_lambda__1_f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def __mypyc_lambda__1_f_obj.__call__(__mypyc_self__, a, b): + __mypyc_self__ :: __main__.__mypyc_lambda__1_f_obj + a, b :: object + r0 :: __main__.f_env + r1, r2 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.s + r2 = PyObject_CallFunctionObjArgs(r1, a, b, 0) + return r2 +def f(x, y): + x, y :: int + r0 :: __main__.f_env + r1 :: __main__.__mypyc_lambda__0_f_obj + r2, r3 :: bool + r4 :: __main__.__mypyc_lambda__1_f_obj + r5 :: bool + t, r6, r7, r8 :: object + r9 :: None +L0: + r0 = f_env() + r1 = __mypyc_lambda__0_f_obj() + r1.__mypyc_env__ = r0; r2 = is_error + r0.s = r1; r3 = is_error + r4 = __mypyc_lambda__1_f_obj() + r4.__mypyc_env__ = r0; r5 = is_error + t = r4 + r6 = box(int, x) + r7 = box(int, y) + r8 = PyObject_CallFunctionObjArgs(t, r6, r7, 0) + r9 = unbox(None, r8) + return r9 + +[case testRecursiveFunction] +from typing import Callable + +def baz(n: int) -> int: + if n == 0: + return 0 + return n + baz(n - 1) + +[out] +def baz(n): + n :: int + r0 :: bit + r1, r2, r3 :: int +L0: + r0 = int_eq n, 0 + if r0 goto L1 else goto L2 :: bool +L1: + return 0 +L2: + r1 = CPyTagged_Subtract(n, 2) + r2 = baz(r1) + r3 = CPyTagged_Add(n, r2) + return r3 diff --git a/mypyc/test-data/irbuild-optional.test b/mypyc/test-data/irbuild-optional.test new file mode 100644 index 000000000000..75c008586999 --- /dev/null +++ b/mypyc/test-data/irbuild-optional.test @@ -0,0 +1,536 @@ +[case testIsNone] +from typing import Optional + +class A: pass + +def f(x: Optional[A]) -> int: + if x is None: + return 1 + return 2 +[out] +def f(x): + x :: union[__main__.A, None] + r0 :: object + r1 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 + +[case testIsNotNone] +from typing import Optional + +class A: pass + +def f(x: Optional[A]) -> int: + if x is not None: + return 1 + return 2 +[out] +def f(x): + x :: union[__main__.A, None] + r0 :: object + r1 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 + +[case testIsTruthy] +from typing import Optional + +class A: pass + +def f(x: Optional[A]) -> int: + if x: + return 1 + return 2 +[out] +def f(x): + x :: union[__main__.A, None] + r0 :: object + r1 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 + +[case testIsTruthyOverride] +from typing import Optional + +class A: pass + +class B(A): + def __bool__(self) -> bool: + return False + + +def f(x: Optional[A]) -> int: + if x: + return 1 + return 2 +[out] +def B.__bool__(self): + self :: __main__.B +L0: + return 0 +def f(x): + x :: union[__main__.A, None] + r0 :: object + r1 :: bit + r2 :: __main__.A + r3 :: i32 + r4 :: bit + r5 :: bool +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L3 :: bool +L1: + r2 = cast(__main__.A, x) + r3 = PyObject_IsTrue(r2) + r4 = r3 >= 0 :: signed + r5 = truncate r3: i32 to builtins.bool + if r5 goto L2 else goto L3 :: bool +L2: + return 2 +L3: + return 4 + +[case testAssignToOptional] +from typing import Optional + +class A: + a: Optional[int] + +def f(x: Optional[A], y: Optional[A], z: Optional[int]) -> None: + x = None + x = A() + x = y + z = 1 + a = A() + a.a = 1 + a.a = None +[out] +def f(x, y, z): + x, y :: union[__main__.A, None] + z :: union[int, None] + r0 :: object + r1 :: __main__.A + r2 :: object + r3, a :: __main__.A + r4 :: object + r5 :: bool + r6 :: object + r7 :: bool +L0: + r0 = box(None, 1) + x = r0 + r1 = A() + x = r1 + x = y + r2 = object 1 + z = r2 + r3 = A() + a = r3 + r4 = object 1 + a.a = r4; r5 = is_error + r6 = box(None, 1) + a.a = r6; r7 = is_error + return 1 + +[case testBoxOptionalListItem] +from typing import List, Optional + +def f(x: List[Optional[int]]) -> None: + x[0] = 0 + x[1] = None +[out] +def f(x): + x :: list + r0 :: object + r1 :: bit + r2 :: object + r3 :: bit +L0: + r0 = object 0 + r1 = CPyList_SetItem(x, 0, r0) + r2 = box(None, 1) + r3 = CPyList_SetItem(x, 2, r2) + return 1 + +[case testNarrowDownFromOptional] +from typing import Optional + +class A: pass + +def f(x: Optional[A]) -> A: + y = A() + if x is not None: + y = x + return x + return y +[out] +def f(x): + x :: union[__main__.A, None] + r0, y :: __main__.A + r1 :: object + r2 :: bit + r3, r4 :: __main__.A +L0: + r0 = A() + y = r0 + r1 = load_address _Py_NoneStruct + r2 = x != r1 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = cast(__main__.A, x) + y = r3 + r4 = cast(__main__.A, x) + return r4 +L2: + return y + +[case testPartialOptionalType] +def f(y: int) -> None: + x = None + if y == 1: + x = y + if x is not None: + y = x +[out] +def f(y): + y :: int + r0 :: object + x :: union[int, None] + r1 :: bit + r2, r3 :: object + r4 :: bit + r5 :: int +L0: + r0 = box(None, 1) + x = r0 + r1 = int_eq y, 2 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = box(int, y) + x = r2 +L2: + r3 = load_address _Py_NoneStruct + r4 = x != r3 + if r4 goto L3 else goto L4 :: bool +L3: + r5 = unbox(int, x) + y = r5 +L4: + return 1 + +[case testUnionType] +from typing import Union + +class A: + a: int + +def f(x: Union[int, A]) -> int: + if isinstance(x, int): + return x + 1 + else: + return x.a +[out] +def f(x): + x :: union[int, __main__.A] + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4, r5 :: int + r6 :: __main__.A + r7 :: int +L0: + r0 = load_address PyLong_Type + r1 = PyObject_IsInstance(x, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool +L1: + r4 = unbox(int, x) + r5 = CPyTagged_Add(r4, 2) + return r5 +L2: + r6 = borrow cast(__main__.A, x) + r7 = r6.a + keep_alive x + return r7 +L3: + unreachable + +[case testUnionTypeInList] +from typing import List, Union + +def f(x: List[Union[int, str]]) -> object: + return x[0] +[out] +def f(x): + x :: list + r0 :: object + r1 :: union[int, str] +L0: + r0 = CPyList_GetItemShort(x, 0) + r1 = cast(union[int, str], r0) + return r1 + +[case testUnionAttributeAccess] +from typing import Union +class A: + a: int +class B: + a: object +def get(o: Union[A, B]) -> None: + z = o.a +def set(o: Union[A, B], s: str) -> None: + o.a = s + +[out] +def get(o): + o :: union[__main__.A, __main__.B] + r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4 :: __main__.A + r5 :: int + r6, r7 :: object + r8 :: __main__.B + r9, z :: object +L0: + r0 = __main__.A :: type + r1 = get_element_ptr o ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive o + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = cast(__main__.A, o) + r5 = r4.a + r6 = box(int, r5) + r7 = r6 + goto L3 +L2: + r8 = cast(__main__.B, o) + r9 = r8.a + r7 = r9 +L3: + z = r7 + return 1 +def set(o, s): + o :: union[__main__.A, __main__.B] + s, r0 :: str + r1 :: i32 + r2 :: bit +L0: + r0 = 'a' + r1 = PyObject_SetAttr(o, r0, s) + r2 = r1 >= 0 :: signed + return 1 + +[case testUnionMethodCall] +from typing import Union +class A: + def f(self, x: int) -> int: + return x +class B: + def f(self, x: object) -> object: + return x +class C: + def f(self, x: object) -> int: + return 0 +def g(o: Union[A, B, C]) -> None: + z = o.f(1) + +[out] +def A.f(self, x): + self :: __main__.A + x :: int +L0: + return x +def B.f(self, x): + self :: __main__.B + x :: object +L0: + return x +def C.f(self, x): + self :: __main__.C + x :: object +L0: + return 0 +def g(o): + o :: union[__main__.A, __main__.B, __main__.C] + r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4 :: __main__.A + r5 :: int + r6, r7, r8 :: object + r9 :: ptr + r10 :: object + r11 :: bit + r12 :: __main__.B + r13, r14 :: object + r15 :: __main__.C + r16 :: object + r17 :: int + r18, z :: object +L0: + r0 = __main__.A :: type + r1 = get_element_ptr o ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive o + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = cast(__main__.A, o) + r5 = r4.f(2) + r6 = box(int, r5) + r7 = r6 + goto L5 +L2: + r8 = __main__.B :: type + r9 = get_element_ptr o ob_type :: PyObject + r10 = load_mem r9 :: builtins.object* + keep_alive o + r11 = r10 == r8 + if r11 goto L3 else goto L4 :: bool +L3: + r12 = cast(__main__.B, o) + r13 = object 1 + r14 = r12.f(r13) + r7 = r14 + goto L5 +L4: + r15 = cast(__main__.C, o) + r16 = object 1 + r17 = r15.f(r16) + r18 = box(int, r17) + r7 = r18 +L5: + z = r7 + return 1 + +[case testUnionWithNonNativeItem] +from typing import Union +from m import B + +class A: + x: int + +def f(o: Union[A, B]) -> None: + o.x + +def g(o: Union[B, A]) -> None: + o.x + +[file m.py] +class B: + x: int + +[out] +def f(o): + o :: union[__main__.A, object] + r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4 :: __main__.A + r5, r6 :: int + r7 :: object + r8 :: str + r9 :: object + r10 :: int +L0: + r0 = __main__.A :: type + r1 = get_element_ptr o ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive o + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = cast(__main__.A, o) + r5 = r4.x + r6 = r5 + goto L3 +L2: + r7 = o + r8 = 'x' + r9 = CPyObject_GetAttr(r7, r8) + r10 = unbox(int, r9) + r6 = r10 +L3: + return 1 +def g(o): + o :: union[object, __main__.A] + r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit + r4 :: __main__.A + r5, r6 :: int + r7 :: object + r8 :: str + r9 :: object + r10 :: int +L0: + r0 = __main__.A :: type + r1 = get_element_ptr o ob_type :: PyObject + r2 = load_mem r1 :: builtins.object* + keep_alive o + r3 = r2 == r0 + if r3 goto L1 else goto L2 :: bool +L1: + r4 = cast(__main__.A, o) + r5 = r4.x + r6 = r5 + goto L3 +L2: + r7 = o + r8 = 'x' + r9 = CPyObject_GetAttr(r7, r8) + r10 = unbox(int, r9) + r6 = r10 +L3: + return 1 + +[case testUnionWithNoNativeItems] +from typing import Union +from m import A, B + +def f(o: Union[A, B]) -> None: + o.x + +[file m.py] +class A: + x: object +class B: + x: int + +[out] +def f(o): + o :: object + r0 :: str + r1 :: object +L0: + r0 = 'x' + r1 = CPyObject_GetAttr(o, r0) + return 1 diff --git a/mypyc/test-data/irbuild-set.test b/mypyc/test-data/irbuild-set.test new file mode 100644 index 000000000000..110801b78a66 --- /dev/null +++ b/mypyc/test-data/irbuild-set.test @@ -0,0 +1,809 @@ +[case testNewSet] +from typing import Set +def f() -> Set[int]: + return {1, 2, 3} +[out] +def f(): + r0 :: set + r1 :: object + r2 :: i32 + r3 :: bit + r4 :: object + r5 :: i32 + r6 :: bit + r7 :: object + r8 :: i32 + r9 :: bit +L0: + r0 = PySet_New(0) + r1 = object 1 + r2 = PySet_Add(r0, r1) + r3 = r2 >= 0 :: signed + r4 = object 2 + r5 = PySet_Add(r0, r4) + r6 = r5 >= 0 :: signed + r7 = object 3 + r8 = PySet_Add(r0, r7) + r9 = r8 >= 0 :: signed + return r0 + +[case testNewEmptySet] +from typing import Set +def f() -> Set[int]: + return set() +[out] +def f(): + r0 :: set +L0: + r0 = PySet_New(0) + return r0 + +[case testNewSetFromIterable] +from typing import Set, List, TypeVar + +T = TypeVar("T") + +def f(l: List[T]) -> Set[T]: + return set(l) +[out] +def f(l): + l :: list + r0 :: set +L0: + r0 = PySet_New(l) + return r0 + +[case testNewSetFromIterable2] +def f(x: int) -> int: + return x + +def test1() -> None: + tmp_list = [1, 3, 5] + a = set(f(x) for x in tmp_list) + +def test2() -> None: + tmp_tuple = (1, 3, 5) + b = set(f(x) for x in tmp_tuple) + +def test3() -> None: + tmp_dict = {1: '1', 3: '3', 5: '5'} + c = set(f(x) for x in tmp_dict) + +def test4() -> None: + d = set(f(x) for x in range(1, 6, 2)) + +def test5() -> None: + e = set((f(x) for x in range(1, 6, 2))) +[out] +def f(x): + x :: int +L0: + return x +def test1(): + r0 :: list + r1, r2, r3 :: object + r4 :: ptr + tmp_list :: list + r5 :: set + r6 :: short_int + r7 :: native_int + r8 :: short_int + r9 :: bit + r10 :: object + r11, x, r12 :: int + r13 :: object + r14 :: i32 + r15 :: bit + r16 :: short_int + a :: set +L0: + r0 = PyList_New(3) + r1 = object 1 + r2 = object 3 + r3 = object 5 + r4 = list_items r0 + buf_init_item r4, 0, r1 + buf_init_item r4, 1, r2 + buf_init_item r4, 2, r3 + keep_alive r0 + tmp_list = r0 + r5 = PySet_New(0) + r6 = 0 +L1: + r7 = var_object_size tmp_list + r8 = r7 << 1 + r9 = int_lt r6, r8 + if r9 goto L2 else goto L4 :: bool +L2: + r10 = CPyList_GetItemUnsafe(tmp_list, r6) + r11 = unbox(int, r10) + x = r11 + r12 = f(x) + r13 = box(int, r12) + r14 = PySet_Add(r5, r13) + r15 = r14 >= 0 :: signed +L3: + r16 = r6 + 2 + r6 = r16 + goto L1 +L4: + a = r5 + return 1 +def test2(): + r0, tmp_tuple :: tuple[int, int, int] + r1 :: set + r2, r3, r4 :: object + r5, x, r6 :: int + r7 :: object + r8 :: i32 + r9, r10 :: bit + b :: set +L0: + r0 = (2, 6, 10) + tmp_tuple = r0 + r1 = PySet_New(0) + r2 = box(tuple[int, int, int], tmp_tuple) + r3 = PyObject_GetIter(r2) +L1: + r4 = PyIter_Next(r3) + if is_error(r4) goto L4 else goto L2 +L2: + r5 = unbox(int, r4) + x = r5 + r6 = f(x) + r7 = box(int, r6) + r8 = PySet_Add(r1, r7) + r9 = r8 >= 0 :: signed +L3: + goto L1 +L4: + r10 = CPy_NoErrOccured() +L5: + b = r1 + return 1 +def test3(): + r0, r1, r2 :: str + r3, r4, r5 :: object + r6, tmp_dict :: dict + r7 :: set + r8 :: short_int + r9 :: native_int + r10 :: short_int + r11 :: object + r12 :: tuple[bool, short_int, object] + r13 :: short_int + r14 :: bool + r15 :: object + r16, x, r17 :: int + r18 :: object + r19 :: i32 + r20, r21, r22 :: bit + c :: set +L0: + r0 = '1' + r1 = '3' + r2 = '5' + r3 = object 1 + r4 = object 3 + r5 = object 5 + r6 = CPyDict_Build(3, r3, r0, r4, r1, r5, r2) + tmp_dict = r6 + r7 = PySet_New(0) + r8 = 0 + r9 = PyDict_Size(tmp_dict) + r10 = r9 << 1 + r11 = CPyDict_GetKeysIter(tmp_dict) +L1: + r12 = CPyDict_NextKey(r11, r8) + r13 = r12[1] + r8 = r13 + r14 = r12[0] + if r14 goto L2 else goto L4 :: bool +L2: + r15 = r12[2] + r16 = unbox(int, r15) + x = r16 + r17 = f(x) + r18 = box(int, r17) + r19 = PySet_Add(r7, r18) + r20 = r19 >= 0 :: signed +L3: + r21 = CPyDict_CheckSize(tmp_dict, r10) + goto L1 +L4: + r22 = CPy_NoErrOccured() +L5: + c = r7 + return 1 +def test4(): + r0 :: set + r1 :: short_int + x :: int + r2 :: bit + r3 :: int + r4 :: object + r5 :: i32 + r6 :: bit + r7 :: short_int + d :: set +L0: + r0 = PySet_New(0) + r1 = 2 + x = r1 +L1: + r2 = int_lt r1, 12 + if r2 goto L2 else goto L4 :: bool +L2: + r3 = f(x) + r4 = box(int, r3) + r5 = PySet_Add(r0, r4) + r6 = r5 >= 0 :: signed +L3: + r7 = r1 + 4 + r1 = r7 + x = r7 + goto L1 +L4: + d = r0 + return 1 +def test5(): + r0 :: set + r1 :: short_int + x :: int + r2 :: bit + r3 :: int + r4 :: object + r5 :: i32 + r6 :: bit + r7 :: short_int + e :: set +L0: + r0 = PySet_New(0) + r1 = 2 + x = r1 +L1: + r2 = int_lt r1, 12 + if r2 goto L2 else goto L4 :: bool +L2: + r3 = f(x) + r4 = box(int, r3) + r5 = PySet_Add(r0, r4) + r6 = r5 >= 0 :: signed +L3: + r7 = r1 + 4 + r1 = r7 + x = r7 + goto L1 +L4: + e = r0 + return 1 + +[case testNewSetFromIterable3] +def f1(x: int) -> int: + return x + +def f2(x: int) -> int: + return x * 10 + +def f3(x: int) -> int: + return x + 1 + +def test() -> None: + tmp_list = [1, 2, 3, 4, 5] + a = set(f3(x) for x in (f2(y) for y in (f1(z) for z in tmp_list if z < 4))) +[out] +def f1(x): + x :: int +L0: + return x +def f2(x): + x, r0 :: int +L0: + r0 = CPyTagged_Multiply(x, 20) + return r0 +def f3(x): + x, r0 :: int +L0: + r0 = CPyTagged_Add(x, 2) + return r0 +def test(): + r0 :: list + r1, r2, r3, r4, r5 :: object + r6 :: ptr + tmp_list :: list + r7 :: set + r8, r9 :: list + r10 :: short_int + r11 :: native_int + r12 :: short_int + r13 :: bit + r14 :: object + r15, z :: int + r16 :: bit + r17 :: int + r18 :: object + r19 :: i32 + r20 :: bit + r21 :: short_int + r22, r23, r24 :: object + r25, y, r26 :: int + r27 :: object + r28 :: i32 + r29, r30 :: bit + r31, r32, r33 :: object + r34, x, r35 :: int + r36 :: object + r37 :: i32 + r38, r39 :: bit + a :: set +L0: + r0 = PyList_New(5) + r1 = object 1 + r2 = object 2 + r3 = object 3 + r4 = object 4 + r5 = object 5 + r6 = list_items r0 + buf_init_item r6, 0, r1 + buf_init_item r6, 1, r2 + buf_init_item r6, 2, r3 + buf_init_item r6, 3, r4 + buf_init_item r6, 4, r5 + keep_alive r0 + tmp_list = r0 + r7 = PySet_New(0) + r8 = PyList_New(0) + r9 = PyList_New(0) + r10 = 0 +L1: + r11 = var_object_size tmp_list + r12 = r11 << 1 + r13 = int_lt r10, r12 + if r13 goto L2 else goto L6 :: bool +L2: + r14 = CPyList_GetItemUnsafe(tmp_list, r10) + r15 = unbox(int, r14) + z = r15 + r16 = int_lt z, 8 + if r16 goto L4 else goto L3 :: bool +L3: + goto L5 +L4: + r17 = f1(z) + r18 = box(int, r17) + r19 = PyList_Append(r9, r18) + r20 = r19 >= 0 :: signed +L5: + r21 = r10 + 2 + r10 = r21 + goto L1 +L6: + r22 = PyObject_GetIter(r9) + r23 = PyObject_GetIter(r22) +L7: + r24 = PyIter_Next(r23) + if is_error(r24) goto L10 else goto L8 +L8: + r25 = unbox(int, r24) + y = r25 + r26 = f2(y) + r27 = box(int, r26) + r28 = PyList_Append(r8, r27) + r29 = r28 >= 0 :: signed +L9: + goto L7 +L10: + r30 = CPy_NoErrOccured() +L11: + r31 = PyObject_GetIter(r8) + r32 = PyObject_GetIter(r31) +L12: + r33 = PyIter_Next(r32) + if is_error(r33) goto L15 else goto L13 +L13: + r34 = unbox(int, r33) + x = r34 + r35 = f3(x) + r36 = box(int, r35) + r37 = PySet_Add(r7, r36) + r38 = r37 >= 0 :: signed +L14: + goto L12 +L15: + r39 = CPy_NoErrOccured() +L16: + a = r7 + return 1 + +[case testSetSize] +from typing import Set +def f() -> int: + return len({1, 2, 3}) +[out] +def f(): + r0 :: set + r1 :: object + r2 :: i32 + r3 :: bit + r4 :: object + r5 :: i32 + r6 :: bit + r7 :: object + r8 :: i32 + r9 :: bit + r10 :: ptr + r11 :: native_int + r12 :: short_int +L0: + r0 = PySet_New(0) + r1 = object 1 + r2 = PySet_Add(r0, r1) + r3 = r2 >= 0 :: signed + r4 = object 2 + r5 = PySet_Add(r0, r4) + r6 = r5 >= 0 :: signed + r7 = object 3 + r8 = PySet_Add(r0, r7) + r9 = r8 >= 0 :: signed + r10 = get_element_ptr r0 used :: PySetObject + r11 = load_mem r10 :: native_int* + keep_alive r0 + r12 = r11 << 1 + return r12 + +[case testSetContains] +from typing import Set +def f() -> bool: + x = {3, 4} + return (5 in x) +[out] +def f(): + r0 :: set + r1 :: object + r2 :: i32 + r3 :: bit + r4 :: object + r5 :: i32 + r6 :: bit + x :: set + r7 :: object + r8 :: i32 + r9 :: bit + r10 :: bool +L0: + r0 = PySet_New(0) + r1 = object 3 + r2 = PySet_Add(r0, r1) + r3 = r2 >= 0 :: signed + r4 = object 4 + r5 = PySet_Add(r0, r4) + r6 = r5 >= 0 :: signed + x = r0 + r7 = object 5 + r8 = PySet_Contains(x, r7) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + return r10 + +[case testSetRemove] +from typing import Set +def f() -> Set[int]: + x = set() # type: Set[int] + x.remove(1) + return x +[out] +def f(): + r0, x :: set + r1 :: object + r2 :: bit +L0: + r0 = PySet_New(0) + x = r0 + r1 = object 1 + r2 = CPySet_Remove(x, r1) + return x + +[case testSetDiscard] +from typing import Set +def f() -> Set[int]: + x = set() # type: Set[int] + x.discard(1) + return x +[out] +def f(): + r0, x :: set + r1 :: object + r2 :: i32 + r3 :: bit +L0: + r0 = PySet_New(0) + x = r0 + r1 = object 1 + r2 = PySet_Discard(x, r1) + r3 = r2 >= 0 :: signed + return x + +[case testSetAdd] +from typing import Set +def f() -> Set[int]: + x = set() # type: Set[int] + x.add(1) + return x +[out] +def f(): + r0, x :: set + r1 :: object + r2 :: i32 + r3 :: bit +L0: + r0 = PySet_New(0) + x = r0 + r1 = object 1 + r2 = PySet_Add(x, r1) + r3 = r2 >= 0 :: signed + return x + +[case testSetClear] +from typing import Set +def f() -> Set[int]: + x = set() # type: Set[int] + x.clear() + return x +[out] +def f(): + r0, x :: set + r1 :: i32 + r2 :: bit +L0: + r0 = PySet_New(0) + x = r0 + r1 = PySet_Clear(x) + r2 = r1 >= 0 :: signed + return x + +[case testSetPop] +from typing import Set +def f(s : Set[int]) -> int: + return s.pop() +[out] +def f(s): + s :: set + r0 :: object + r1 :: int +L0: + r0 = PySet_Pop(s) + r1 = unbox(int, r0) + return r1 + +[case testSetUpdate] +from typing import Set, List +def update(s: Set[int], x: List[int]) -> None: + s.update(x) +[out] +def update(s, x): + s :: set + x :: list + r0 :: i32 + r1 :: bit +L0: + r0 = _PySet_Update(s, x) + r1 = r0 >= 0 :: signed + return 1 + +[case testSetDisplay] +from typing import Set +def f(x: Set[int], y: Set[int]) -> Set[int]: + return {1, 2, *x, *y, 3} +[out] +def f(x, y): + x, y, r0 :: set + r1 :: object + r2 :: i32 + r3 :: bit + r4 :: object + r5 :: i32 + r6 :: bit + r7 :: i32 + r8 :: bit + r9 :: i32 + r10 :: bit + r11 :: object + r12 :: i32 + r13 :: bit +L0: + r0 = PySet_New(0) + r1 = object 1 + r2 = PySet_Add(r0, r1) + r3 = r2 >= 0 :: signed + r4 = object 2 + r5 = PySet_Add(r0, r4) + r6 = r5 >= 0 :: signed + r7 = _PySet_Update(r0, x) + r8 = r7 >= 0 :: signed + r9 = _PySet_Update(r0, y) + r10 = r9 >= 0 :: signed + r11 = object 3 + r12 = PySet_Add(r0, r11) + r13 = r12 >= 0 :: signed + return r0 + +[case testOperatorInSetLiteral] +from typing_extensions import Final + +CONST: Final = "daylily" +non_const = 10 + +def precomputed(i: object) -> bool: + return i in {1, 2.0, 1 +2, 4j, "foo", b"bar", CONST, (None, (27,)), (), False} +def not_precomputed_non_final_name(i: int) -> bool: + return i in {non_const} +def not_precomputed_nested_set(i: int) -> bool: + return i in {frozenset({1}), 2} +[out] +def precomputed(i): + i :: object + r0 :: set + r1 :: i32 + r2 :: bit + r3 :: bool +L0: + r0 = frozenset({(), (None, (27,)), 1, 2.0, 3, 4j, False, b'bar', 'daylily', 'foo'}) + r1 = PySet_Contains(r0, i) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + return r3 +def not_precomputed_non_final_name(i): + i :: int + r0 :: dict + r1 :: str + r2 :: object + r3 :: int + r4 :: set + r5 :: object + r6 :: i32 + r7 :: bit + r8 :: object + r9 :: i32 + r10 :: bit + r11 :: bool +L0: + r0 = __main__.globals :: static + r1 = 'non_const' + r2 = CPyDict_GetItem(r0, r1) + r3 = unbox(int, r2) + r4 = PySet_New(0) + r5 = box(int, r3) + r6 = PySet_Add(r4, r5) + r7 = r6 >= 0 :: signed + r8 = box(int, i) + r9 = PySet_Contains(r4, r8) + r10 = r9 >= 0 :: signed + r11 = truncate r9: i32 to builtins.bool + return r11 +def not_precomputed_nested_set(i): + i :: int + r0 :: set + r1 :: object + r2 :: i32 + r3 :: bit + r4 :: object + r5 :: set + r6 :: i32 + r7 :: bit + r8 :: object + r9 :: i32 + r10 :: bit + r11 :: object + r12 :: i32 + r13 :: bit + r14 :: bool +L0: + r0 = PySet_New(0) + r1 = object 1 + r2 = PySet_Add(r0, r1) + r3 = r2 >= 0 :: signed + r4 = PyFrozenSet_New(r0) + r5 = PySet_New(0) + r6 = PySet_Add(r5, r4) + r7 = r6 >= 0 :: signed + r8 = object 2 + r9 = PySet_Add(r5, r8) + r10 = r9 >= 0 :: signed + r11 = box(int, i) + r12 = PySet_Contains(r5, r11) + r13 = r12 >= 0 :: signed + r14 = truncate r12: i32 to builtins.bool + return r14 + +[case testForSetLiteral] +from typing_extensions import Final + +CONST: Final = 10 +non_const = 20 + +def precomputed() -> None: + for _ in {"None", "True", "False"}: + pass + +def precomputed2() -> None: + for _ in {None, False, 1, 2.0, "4", b"5", (6,), 7j, CONST, CONST + 1}: + pass + +def not_precomputed() -> None: + for not_optimized in {non_const}: + pass + +[out] +def precomputed(): + r0 :: set + r1, r2 :: object + r3 :: str + _ :: object + r4 :: bit +L0: + r0 = frozenset({'False', 'None', 'True'}) + r1 = PyObject_GetIter(r0) +L1: + r2 = PyIter_Next(r1) + if is_error(r2) goto L4 else goto L2 +L2: + r3 = cast(str, r2) + _ = r3 +L3: + goto L1 +L4: + r4 = CPy_NoErrOccured() +L5: + return 1 +def precomputed2(): + r0 :: set + r1, r2, _ :: object + r3 :: bit +L0: + r0 = frozenset({(6,), 1, 10, 11, 2.0, '4', 7j, False, None, b'5'}) + r1 = PyObject_GetIter(r0) +L1: + r2 = PyIter_Next(r1) + if is_error(r2) goto L4 else goto L2 +L2: + _ = r2 +L3: + goto L1 +L4: + r3 = CPy_NoErrOccured() +L5: + return 1 +def not_precomputed(): + r0 :: dict + r1 :: str + r2 :: object + r3 :: int + r4 :: set + r5 :: object + r6 :: i32 + r7 :: bit + r8, r9 :: object + r10, not_optimized :: int + r11 :: bit +L0: + r0 = __main__.globals :: static + r1 = 'non_const' + r2 = CPyDict_GetItem(r0, r1) + r3 = unbox(int, r2) + r4 = PySet_New(0) + r5 = box(int, r3) + r6 = PySet_Add(r4, r5) + r7 = r6 >= 0 :: signed + r8 = PyObject_GetIter(r4) +L1: + r9 = PyIter_Next(r8) + if is_error(r9) goto L4 else goto L2 +L2: + r10 = unbox(int, r9) + not_optimized = r10 +L3: + goto L1 +L4: + r11 = CPy_NoErrOccured() +L5: + return 1 diff --git a/mypyc/test-data/irbuild-singledispatch.test b/mypyc/test-data/irbuild-singledispatch.test new file mode 100644 index 000000000000..e1053397546f --- /dev/null +++ b/mypyc/test-data/irbuild-singledispatch.test @@ -0,0 +1,257 @@ +[case testNativeCallsUsedInDispatchFunction] +from functools import singledispatch +@singledispatch +def f(arg) -> bool: + return False + +@f.register +def g(arg: int) -> bool: + return True +[out] +def __mypyc_singledispatch_main_function_f__(arg): + arg :: object +L0: + return 0 +def f_obj.__init__(__mypyc_self__): + __mypyc_self__ :: __main__.f_obj + r0, r1 :: dict + r2 :: str + r3 :: i32 + r4 :: bit +L0: + r0 = PyDict_New() + __mypyc_self__.registry = r0 + r1 = PyDict_New() + r2 = 'dispatch_cache' + r3 = PyObject_SetAttr(__mypyc_self__, r2, r1) + r4 = r3 >= 0 :: signed + return 1 +def f_obj.__call__(__mypyc_self__, arg): + __mypyc_self__ :: __main__.f_obj + arg :: object + r0 :: ptr + r1 :: object + r2 :: dict + r3, r4 :: object + r5 :: bit + r6, r7 :: object + r8 :: str + r9 :: object + r10 :: dict + r11 :: object + r12 :: i32 + r13 :: bit + r14 :: object + r15 :: ptr + r16 :: object + r17 :: bit + r18 :: int + r19 :: bit + r20 :: int + r21 :: bool + r22 :: object + r23 :: bool +L0: + r0 = get_element_ptr arg ob_type :: PyObject + r1 = load_mem r0 :: builtins.object* + keep_alive arg + r2 = __mypyc_self__.dispatch_cache + r3 = CPyDict_GetWithNone(r2, r1) + r4 = load_address _Py_NoneStruct + r5 = r3 != r4 + if r5 goto L1 else goto L2 :: bool +L1: + r6 = r3 + goto L3 +L2: + r7 = functools :: module + r8 = '_find_impl' + r9 = CPyObject_GetAttr(r7, r8) + r10 = __mypyc_self__.registry + r11 = PyObject_CallFunctionObjArgs(r9, r1, r10, 0) + r12 = CPyDict_SetItem(r2, r1, r11) + r13 = r12 >= 0 :: signed + r6 = r11 +L3: + r14 = load_address PyLong_Type + r15 = get_element_ptr r6 ob_type :: PyObject + r16 = load_mem r15 :: builtins.object* + keep_alive r6 + r17 = r16 == r14 + if r17 goto L4 else goto L7 :: bool +L4: + r18 = unbox(int, r6) + r19 = int_eq r18, 0 + if r19 goto L5 else goto L6 :: bool +L5: + r20 = unbox(int, arg) + r21 = g(r20) + return r21 +L6: + unreachable +L7: + r22 = PyObject_CallFunctionObjArgs(r6, arg, 0) + r23 = unbox(bool, r22) + return r23 +def f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def f_obj.register(__mypyc_self__, cls, func): + __mypyc_self__ :: __main__.f_obj + cls, func, r0 :: object +L0: + r0 = CPySingledispatch_RegisterFunction(__mypyc_self__, cls, func) + return r0 +def f(arg): + arg :: object + r0 :: dict + r1 :: str + r2 :: object + r3 :: bool +L0: + r0 = __main__.globals :: static + r1 = 'f' + r2 = CPyDict_GetItem(r0, r1) + r3 = f_obj.__call__(r2, arg) + return r3 +def g(arg): + arg :: int +L0: + return 1 + + +[case testCallsToSingledispatchFunctionsAreNative] +from functools import singledispatch + +@singledispatch +def f(x: object) -> None: + pass + +def test(): + f('a') +[out] +def __mypyc_singledispatch_main_function_f__(x): + x :: object +L0: + return 1 +def f_obj.__init__(__mypyc_self__): + __mypyc_self__ :: __main__.f_obj + r0, r1 :: dict + r2 :: str + r3 :: i32 + r4 :: bit +L0: + r0 = PyDict_New() + __mypyc_self__.registry = r0 + r1 = PyDict_New() + r2 = 'dispatch_cache' + r3 = PyObject_SetAttr(__mypyc_self__, r2, r1) + r4 = r3 >= 0 :: signed + return 1 +def f_obj.__call__(__mypyc_self__, x): + __mypyc_self__ :: __main__.f_obj + x :: object + r0 :: ptr + r1 :: object + r2 :: dict + r3, r4 :: object + r5 :: bit + r6, r7 :: object + r8 :: str + r9 :: object + r10 :: dict + r11 :: object + r12 :: i32 + r13 :: bit + r14 :: object + r15 :: ptr + r16 :: object + r17 :: bit + r18 :: int + r19 :: object + r20 :: None +L0: + r0 = get_element_ptr x ob_type :: PyObject + r1 = load_mem r0 :: builtins.object* + keep_alive x + r2 = __mypyc_self__.dispatch_cache + r3 = CPyDict_GetWithNone(r2, r1) + r4 = load_address _Py_NoneStruct + r5 = r3 != r4 + if r5 goto L1 else goto L2 :: bool +L1: + r6 = r3 + goto L3 +L2: + r7 = functools :: module + r8 = '_find_impl' + r9 = CPyObject_GetAttr(r7, r8) + r10 = __mypyc_self__.registry + r11 = PyObject_CallFunctionObjArgs(r9, r1, r10, 0) + r12 = CPyDict_SetItem(r2, r1, r11) + r13 = r12 >= 0 :: signed + r6 = r11 +L3: + r14 = load_address PyLong_Type + r15 = get_element_ptr r6 ob_type :: PyObject + r16 = load_mem r15 :: builtins.object* + keep_alive r6 + r17 = r16 == r14 + if r17 goto L4 else goto L5 :: bool +L4: + r18 = unbox(int, r6) + unreachable +L5: + r19 = PyObject_CallFunctionObjArgs(r6, x, 0) + r20 = unbox(None, r19) + return r20 +def f_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def f_obj.register(__mypyc_self__, cls, func): + __mypyc_self__ :: __main__.f_obj + cls, func, r0 :: object +L0: + r0 = CPySingledispatch_RegisterFunction(__mypyc_self__, cls, func) + return r0 +def f(x): + x :: object + r0 :: dict + r1 :: str + r2 :: object + r3 :: None +L0: + r0 = __main__.globals :: static + r1 = 'f' + r2 = CPyDict_GetItem(r0, r1) + r3 = f_obj.__call__(r2, x) + return r3 +def test(): + r0 :: str + r1 :: None + r2 :: object +L0: + r0 = 'a' + r1 = f(r0) + r2 = box(None, 1) + return r2 diff --git a/mypyc/test-data/irbuild-statements.test b/mypyc/test-data/irbuild-statements.test new file mode 100644 index 000000000000..825bc750f7a7 --- /dev/null +++ b/mypyc/test-data/irbuild-statements.test @@ -0,0 +1,1056 @@ +[case testForInRange] +def f() -> None: + x = 0 + for i in range(5): + x = x + i +[out] +def f(): + x :: int + r0 :: short_int + i :: int + r1 :: bit + r2 :: int + r3 :: short_int +L0: + x = 0 + r0 = 0 + i = r0 +L1: + r1 = int_lt r0, 10 + if r1 goto L2 else goto L4 :: bool +L2: + r2 = CPyTagged_Add(x, i) + x = r2 +L3: + r3 = r0 + 2 + r0 = r3 + i = r3 + goto L1 +L4: + return 1 + +[case testForInRangeVariableEndIndxe] +def f(a: int) -> None: + for i in range(a): + pass +[out] +def f(a): + a, r0, i :: int + r1 :: bit + r2 :: int +L0: + r0 = 0 + i = r0 +L1: + r1 = int_lt r0, a + if r1 goto L2 else goto L4 :: bool +L2: +L3: + r2 = CPyTagged_Add(r0, 2) + r0 = r2 + i = r2 + goto L1 +L4: + return 1 + +[case testForInNegativeRange] +def f() -> None: + for i in range(10, 0, -1): + pass +[out] +def f(): + r0 :: short_int + i :: int + r1 :: bit + r2 :: short_int +L0: + r0 = 20 + i = r0 +L1: + r1 = int_gt r0, 0 + if r1 goto L2 else goto L4 :: bool +L2: +L3: + r2 = r0 + -2 + r0 = r2 + i = r2 + goto L1 +L4: + return 1 + +[case testBreak] +def f() -> None: + n = 0 + while n < 5: + break +[out] +def f(): + n :: int + r0 :: bit +L0: + n = 0 +L1: + r0 = int_lt n, 10 + if r0 goto L2 else goto L3 :: bool +L2: +L3: + return 1 + +[case testBreakFor] +def f() -> None: + for n in range(5): + break +[out] +def f(): + r0 :: short_int + n :: int + r1 :: bit + r2 :: short_int +L0: + r0 = 0 + n = r0 +L1: + r1 = int_lt r0, 10 + if r1 goto L2 else goto L4 :: bool +L2: + goto L4 +L3: + r2 = r0 + 2 + r0 = r2 + n = r2 + goto L1 +L4: + return 1 + +[case testBreakNested] +def f() -> None: + n = 0 + while n < 5: + while n < 4: + break + break +[out] +def f(): + n :: int + r0, r1 :: bit +L0: + n = 0 +L1: + r0 = int_lt n, 10 + if r0 goto L2 else goto L6 :: bool +L2: +L3: + r1 = int_lt n, 8 + if r1 goto L4 else goto L5 :: bool +L4: +L5: +L6: + return 1 + +[case testContinue] +def f() -> None: + n = 0 + while n < 5: + continue +[out] +def f(): + n :: int + r0 :: bit +L0: + n = 0 +L1: + r0 = int_lt n, 10 + if r0 goto L2 else goto L3 :: bool +L2: + goto L1 +L3: + return 1 + +[case testContinueFor] +def f() -> None: + for n in range(5): + continue +[out] +def f(): + r0 :: short_int + n :: int + r1 :: bit + r2 :: short_int +L0: + r0 = 0 + n = r0 +L1: + r1 = int_lt r0, 10 + if r1 goto L2 else goto L4 :: bool +L2: +L3: + r2 = r0 + 2 + r0 = r2 + n = r2 + goto L1 +L4: + return 1 + +[case testContinueNested] +def f() -> None: + n = 0 + while n < 5: + while n < 4: + continue + continue +[out] +def f(): + n :: int + r0, r1 :: bit +L0: + n = 0 +L1: + r0 = int_lt n, 10 + if r0 goto L2 else goto L6 :: bool +L2: +L3: + r1 = int_lt n, 8 + if r1 goto L4 else goto L5 :: bool +L4: + goto L3 +L5: + goto L1 +L6: + return 1 + +[case testForList] +from typing import List + +def f(ls: List[int]) -> int: + y = 0 + for x in ls: + y = y + x + return y +[out] +def f(ls): + ls :: list + y :: int + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: bit + r4 :: object + r5, x, r6 :: int + r7 :: short_int +L0: + y = 0 + r0 = 0 +L1: + r1 = var_object_size ls + r2 = r1 << 1 + r3 = int_lt r0, r2 + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyList_GetItemUnsafe(ls, r0) + r5 = unbox(int, r4) + x = r5 + r6 = CPyTagged_Add(y, x) + y = r6 +L3: + r7 = r0 + 2 + r0 = r7 + goto L1 +L4: + return y + +[case testForDictBasic] +from typing import Dict + +def f(d: Dict[int, int]) -> None: + for key in d: + d[key] +[out] +def f(d): + d :: dict + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, key :: int + r9, r10 :: object + r11 :: int + r12, r13 :: bit +L0: + r0 = 0 + r1 = PyDict_Size(d) + r2 = r1 << 1 + r3 = CPyDict_GetKeysIter(d) +L1: + r4 = CPyDict_NextKey(r3, r0) + r5 = r4[1] + r0 = r5 + r6 = r4[0] + if r6 goto L2 else goto L4 :: bool +L2: + r7 = r4[2] + r8 = unbox(int, r7) + key = r8 + r9 = box(int, key) + r10 = CPyDict_GetItem(d, r9) + r11 = unbox(int, r10) +L3: + r12 = CPyDict_CheckSize(d, r2) + goto L1 +L4: + r13 = CPy_NoErrOccured() +L5: + return 1 + +[case testForDictContinue] +from typing import Dict + +def sum_over_even_values(d: Dict[int, int]) -> int: + s = 0 + for key in d: + if d[key] % 2: + continue + s = s + d[key] + return s +[out] +def sum_over_even_values(d): + d :: dict + s :: int + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, key :: int + r9, r10 :: object + r11, r12 :: int + r13 :: bit + r14, r15 :: object + r16, r17 :: int + r18, r19 :: bit +L0: + s = 0 + r0 = 0 + r1 = PyDict_Size(d) + r2 = r1 << 1 + r3 = CPyDict_GetKeysIter(d) +L1: + r4 = CPyDict_NextKey(r3, r0) + r5 = r4[1] + r0 = r5 + r6 = r4[0] + if r6 goto L2 else goto L6 :: bool +L2: + r7 = r4[2] + r8 = unbox(int, r7) + key = r8 + r9 = box(int, key) + r10 = CPyDict_GetItem(d, r9) + r11 = unbox(int, r10) + r12 = CPyTagged_Remainder(r11, 4) + r13 = r12 != 0 + if r13 goto L3 else goto L4 :: bool +L3: + goto L5 +L4: + r14 = box(int, key) + r15 = CPyDict_GetItem(d, r14) + r16 = unbox(int, r15) + r17 = CPyTagged_Add(s, r16) + s = r17 +L5: + r18 = CPyDict_CheckSize(d, r2) + goto L1 +L6: + r19 = CPy_NoErrOccured() +L7: + return s + +[case testMultipleAssignmentWithNoUnpacking] +from typing import Tuple + +def f(x: int, y: int) -> Tuple[int, int]: + x, y = y, x + return (x, y) + +def f2(x: int, y: str, z: float) -> Tuple[float, str, int]: + a, b, c = x, y, z + return (c, b, a) + +def f3(x: int, y: int) -> Tuple[int, int]: + [x, y] = [y, x] + return (x, y) +[out] +def f(x, y): + x, y, r0, r1 :: int + r2 :: tuple[int, int] +L0: + r0 = y + r1 = x + x = r0 + y = r1 + r2 = (x, y) + return r2 +def f2(x, y, z): + x :: int + y :: str + z :: float + r0 :: int + r1 :: str + r2 :: float + a :: int + b :: str + c :: float + r3 :: tuple[float, str, int] +L0: + r0 = x + r1 = y + r2 = z + a = r0 + b = r1 + c = r2 + r3 = (c, b, a) + return r3 +def f3(x, y): + x, y, r0, r1 :: int + r2 :: tuple[int, int] +L0: + r0 = y + r1 = x + x = r0 + y = r1 + r2 = (x, y) + return r2 + +[case testMultipleAssignmentBasicUnpacking] +from typing import Tuple, Any + +def from_tuple(t: Tuple[bool, None]) -> None: + x, y = t + +def from_any(a: Any) -> None: + x, y = a +[out] +def from_tuple(t): + t :: tuple[bool, None] + r0, x :: bool + r1, y :: None +L0: + r0 = t[0] + x = r0 + r1 = t[1] + y = r1 + return 1 +def from_any(a): + a, r0, r1 :: object + r2 :: bool + x, r3 :: object + r4 :: bool + y, r5 :: object + r6 :: bool +L0: + r0 = PyObject_GetIter(a) + r1 = PyIter_Next(r0) + if is_error(r1) goto L1 else goto L2 +L1: + r2 = raise ValueError('not enough values to unpack') + unreachable +L2: + x = r1 + r3 = PyIter_Next(r0) + if is_error(r3) goto L3 else goto L4 +L3: + r4 = raise ValueError('not enough values to unpack') + unreachable +L4: + y = r3 + r5 = PyIter_Next(r0) + if is_error(r5) goto L6 else goto L5 +L5: + r6 = raise ValueError('too many values to unpack') + unreachable +L6: + return 1 + +[case testMultiAssignmentCoercions] +from typing import Tuple, Any + +def from_tuple(t: Tuple[int, Any]) -> None: + x: object + y: int + x, y = t + +def from_any(a: Any) -> None: + x: int + x, y = a +[out] +def from_tuple(t): + t :: tuple[int, object] + r0, r1 :: int + r2, x, r3, r4 :: object + r5, y :: int +L0: + r0 = borrow t[0] + r1 = unborrow r0 + r2 = box(int, r1) + x = r2 + r3 = borrow t[1] + r4 = unborrow r3 + r5 = unbox(int, r4) + y = r5 + keep_alive steal t + return 1 +def from_any(a): + a, r0, r1 :: object + r2 :: bool + r3, x :: int + r4 :: object + r5 :: bool + y, r6 :: object + r7 :: bool +L0: + r0 = PyObject_GetIter(a) + r1 = PyIter_Next(r0) + if is_error(r1) goto L1 else goto L2 +L1: + r2 = raise ValueError('not enough values to unpack') + unreachable +L2: + r3 = unbox(int, r1) + x = r3 + r4 = PyIter_Next(r0) + if is_error(r4) goto L3 else goto L4 +L3: + r5 = raise ValueError('not enough values to unpack') + unreachable +L4: + y = r4 + r6 = PyIter_Next(r0) + if is_error(r6) goto L6 else goto L5 +L5: + r7 = raise ValueError('too many values to unpack') + unreachable +L6: + return 1 + +[case testMultiAssignmentNested] +from typing import Tuple, Any, List + +class A: + x: int + +def multi_assign(t: Tuple[int, Tuple[str, Any]], a: A, l: List[str]) -> None: + z: int + a.x, (l[0], z) = t +[out] +def multi_assign(t, a, l): + t :: tuple[int, tuple[str, object]] + a :: __main__.A + l :: list + r0 :: int + r1 :: bool + r2 :: tuple[str, object] + r3 :: str + r4 :: bit + r5 :: object + r6, z :: int +L0: + r0 = t[0] + a.x = r0; r1 = is_error + r2 = t[1] + r3 = r2[0] + r4 = CPyList_SetItem(l, 0, r3) + r5 = r2[1] + r6 = unbox(int, r5) + z = r6 + return 1 + +[case testMultipleAssignmentUnpackFromSequence] +from typing import List, Tuple + +def f(l: List[int], t: Tuple[int, ...]) -> None: + x: object + y: int + x, y = l + x, y = t +[out] +def f(l, t): + l :: list + t :: tuple + r0 :: i32 + r1 :: bit + r2, r3, x :: object + r4, y :: int + r5 :: i32 + r6 :: bit + r7, r8 :: object + r9 :: int +L0: + r0 = CPySequence_CheckUnpackCount(l, 2) + r1 = r0 >= 0 :: signed + r2 = CPyList_GetItemUnsafe(l, 0) + r3 = CPyList_GetItemUnsafe(l, 2) + x = r2 + r4 = unbox(int, r3) + y = r4 + r5 = CPySequence_CheckUnpackCount(t, 2) + r6 = r5 >= 0 :: signed + r7 = CPySequenceTuple_GetItem(t, 0) + r8 = CPySequenceTuple_GetItem(t, 2) + r9 = unbox(int, r8) + x = r7 + y = r9 + return 1 + +[case testAssert] +from typing import Optional + +def no_msg(x: bool) -> int: + assert x + return 1 + +def literal_msg(x: object) -> int: + assert x, 'message' + return 2 + +def complex_msg(x: Optional[str], s: str) -> None: + assert x, s +[out] +def no_msg(x): + x, r0 :: bool +L0: + if x goto L2 else goto L1 :: bool +L1: + r0 = raise AssertionError + unreachable +L2: + return 2 +def literal_msg(x): + x :: object + r0 :: i32 + r1 :: bit + r2, r3 :: bool +L0: + r0 = PyObject_IsTrue(x) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise AssertionError('message') + unreachable +L2: + return 4 +def complex_msg(x, s): + x :: union[str, None] + s :: str + r0 :: object + r1 :: bit + r2 :: str + r3 :: bit + r4 :: object + r5 :: str + r6, r7 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = cast(str, x) + r3 = CPyStr_IsTrue(r2) + if r3 goto L3 else goto L2 :: bool +L2: + r4 = builtins :: module + r5 = 'AssertionError' + r6 = CPyObject_GetAttr(r4, r5) + r7 = PyObject_CallFunctionObjArgs(r6, s, 0) + CPy_Raise(r7) + unreachable +L3: + return 1 + +[case testDelList] +def delList() -> None: + l = [1, 2] + del l[1] +def delListMultiple() -> None: + l = [1, 2, 3, 4, 5, 6, 7] + del l[1], l[2], l[3] +[out] +def delList(): + r0 :: list + r1, r2 :: object + r3 :: ptr + l :: list + r4 :: object + r5 :: i32 + r6 :: bit +L0: + r0 = PyList_New(2) + r1 = object 1 + r2 = object 2 + r3 = list_items r0 + buf_init_item r3, 0, r1 + buf_init_item r3, 1, r2 + keep_alive r0 + l = r0 + r4 = object 1 + r5 = PyObject_DelItem(l, r4) + r6 = r5 >= 0 :: signed + return 1 +def delListMultiple(): + r0 :: list + r1, r2, r3, r4, r5, r6, r7 :: object + r8 :: ptr + l :: list + r9 :: object + r10 :: i32 + r11 :: bit + r12 :: object + r13 :: i32 + r14 :: bit + r15 :: object + r16 :: i32 + r17 :: bit +L0: + r0 = PyList_New(7) + r1 = object 1 + r2 = object 2 + r3 = object 3 + r4 = object 4 + r5 = object 5 + r6 = object 6 + r7 = object 7 + r8 = list_items r0 + buf_init_item r8, 0, r1 + buf_init_item r8, 1, r2 + buf_init_item r8, 2, r3 + buf_init_item r8, 3, r4 + buf_init_item r8, 4, r5 + buf_init_item r8, 5, r6 + buf_init_item r8, 6, r7 + keep_alive r0 + l = r0 + r9 = object 1 + r10 = PyObject_DelItem(l, r9) + r11 = r10 >= 0 :: signed + r12 = object 2 + r13 = PyObject_DelItem(l, r12) + r14 = r13 >= 0 :: signed + r15 = object 3 + r16 = PyObject_DelItem(l, r15) + r17 = r16 >= 0 :: signed + return 1 + +[case testDelDict] +def delDict() -> None: + d = {"one":1, "two":2} + del d["one"] +def delDictMultiple() -> None: + d = {"one":1, "two":2, "three":3, "four":4} + del d["one"], d["four"] +[out] +def delDict(): + r0, r1 :: str + r2, r3 :: object + r4, d :: dict + r5 :: str + r6 :: i32 + r7 :: bit +L0: + r0 = 'one' + r1 = 'two' + r2 = object 1 + r3 = object 2 + r4 = CPyDict_Build(2, r0, r2, r1, r3) + d = r4 + r5 = 'one' + r6 = PyObject_DelItem(d, r5) + r7 = r6 >= 0 :: signed + return 1 +def delDictMultiple(): + r0, r1, r2, r3 :: str + r4, r5, r6, r7 :: object + r8, d :: dict + r9, r10 :: str + r11 :: i32 + r12 :: bit + r13 :: i32 + r14 :: bit +L0: + r0 = 'one' + r1 = 'two' + r2 = 'three' + r3 = 'four' + r4 = object 1 + r5 = object 2 + r6 = object 3 + r7 = object 4 + r8 = CPyDict_Build(4, r0, r4, r1, r5, r2, r6, r3, r7) + d = r8 + r9 = 'one' + r10 = 'four' + r11 = PyObject_DelItem(d, r9) + r12 = r11 >= 0 :: signed + r13 = PyObject_DelItem(d, r10) + r14 = r13 >= 0 :: signed + return 1 + +[case testDelAttribute] +class Dummy(): + __deletable__ = ('x', 'y') + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y +def delAttribute() -> None: + dummy = Dummy(1, 2) + del dummy.x +def delAttributeMultiple() -> None: + dummy = Dummy(1, 2) + del dummy.x, dummy.y +[out] +def Dummy.__init__(self, x, y): + self :: __main__.Dummy + x, y :: int +L0: + self.x = x + self.y = y + return 1 +def delAttribute(): + r0, dummy :: __main__.Dummy + r1 :: str + r2 :: i32 + r3 :: bit +L0: + r0 = Dummy(2, 4) + dummy = r0 + r1 = 'x' + r2 = PyObject_DelAttr(dummy, r1) + r3 = r2 >= 0 :: signed + return 1 +def delAttributeMultiple(): + r0, dummy :: __main__.Dummy + r1 :: str + r2 :: i32 + r3 :: bit + r4 :: str + r5 :: i32 + r6 :: bit +L0: + r0 = Dummy(2, 4) + dummy = r0 + r1 = 'x' + r2 = PyObject_DelAttr(dummy, r1) + r3 = r2 >= 0 :: signed + r4 = 'y' + r5 = PyObject_DelAttr(dummy, r4) + r6 = r5 >= 0 :: signed + return 1 + +[case testForEnumerate] +from typing import List, Iterable + +def f(a: List[int]) -> None: + for i, x in enumerate(a): + i + x +def g(x: Iterable[int]) -> None: + for i, n in enumerate(x): + pass +[out] +def f(a): + a :: list + r0, r1 :: short_int + r2 :: native_int + r3 :: short_int + r4 :: bit + i :: int + r5 :: object + r6, x, r7 :: int + r8, r9 :: short_int +L0: + r0 = 0 + r1 = 0 +L1: + r2 = var_object_size a + r3 = r2 << 1 + r4 = int_lt r1, r3 + if r4 goto L2 else goto L4 :: bool +L2: + i = r0 + r5 = CPyList_GetItemUnsafe(a, r1) + r6 = unbox(int, r5) + x = r6 + r7 = CPyTagged_Add(i, x) +L3: + r8 = r0 + 2 + r0 = r8 + r9 = r1 + 2 + r1 = r9 + goto L1 +L4: +L5: + return 1 +def g(x): + x :: object + r0 :: short_int + r1, r2 :: object + i, r3, n :: int + r4 :: short_int + r5 :: bit +L0: + r0 = 0 + r1 = PyObject_GetIter(x) +L1: + r2 = PyIter_Next(r1) + if is_error(r2) goto L4 else goto L2 +L2: + i = r0 + r3 = unbox(int, r2) + n = r3 +L3: + r4 = r0 + 2 + r0 = r4 + goto L1 +L4: + r5 = CPy_NoErrOccured() +L5: + return 1 + +[case testForZip] +from typing import List, Iterable, Sequence + +def f(a: List[int], b: Sequence[bool]) -> None: + for x, y in zip(a, b): + if b: + x = 1 + +def g(a: Iterable[bool], b: List[int]) -> None: + for x, y, z in zip(a, b, range(5)): + x = False +[out] +def f(a, b): + a :: list + b :: object + r0 :: short_int + r1 :: object + r2 :: native_int + r3 :: short_int + r4 :: bit + r5, r6 :: object + r7, x :: int + r8, y :: bool + r9 :: i32 + r10 :: bit + r11 :: bool + r12 :: short_int + r13 :: bit +L0: + r0 = 0 + r1 = PyObject_GetIter(b) +L1: + r2 = var_object_size a + r3 = r2 << 1 + r4 = int_lt r0, r3 + if r4 goto L2 else goto L7 :: bool +L2: + r5 = PyIter_Next(r1) + if is_error(r5) goto L7 else goto L3 +L3: + r6 = CPyList_GetItemUnsafe(a, r0) + r7 = unbox(int, r6) + x = r7 + r8 = unbox(bool, r5) + y = r8 + r9 = PyObject_IsTrue(b) + r10 = r9 >= 0 :: signed + r11 = truncate r9: i32 to builtins.bool + if r11 goto L4 else goto L5 :: bool +L4: + x = 2 +L5: +L6: + r12 = r0 + 2 + r0 = r12 + goto L1 +L7: + r13 = CPy_NoErrOccured() +L8: + return 1 +def g(a, b): + a :: object + b :: list + r0 :: object + r1, r2 :: short_int + z :: int + r3 :: object + r4 :: native_int + r5 :: short_int + r6, r7 :: bit + r8, x :: bool + r9 :: object + r10, y :: int + r11, r12 :: short_int + r13 :: bit +L0: + r0 = PyObject_GetIter(a) + r1 = 0 + r2 = 0 + z = r2 +L1: + r3 = PyIter_Next(r0) + if is_error(r3) goto L6 else goto L2 +L2: + r4 = var_object_size b + r5 = r4 << 1 + r6 = int_lt r1, r5 + if r6 goto L3 else goto L6 :: bool +L3: + r7 = int_lt r2, 10 + if r7 goto L4 else goto L6 :: bool +L4: + r8 = unbox(bool, r3) + x = r8 + r9 = CPyList_GetItemUnsafe(b, r1) + r10 = unbox(int, r9) + y = r10 + x = 0 +L5: + r11 = r1 + 2 + r1 = r11 + r12 = r2 + 2 + r2 = r12 + z = r12 + goto L1 +L6: + r13 = CPy_NoErrOccured() +L7: + return 1 + +[case testConditionalFunctionDefinition] +if int(): + def foo() -> int: + return 0 +else: + def foo() -> int: # E + return 1 + +def bar() -> int: + return 0 + +if int(): + def bar() -> int: # E + return 1 +[out] +main:5: error: Duplicate definition of "foo" not supported by mypyc +main:12: error: Duplicate definition of "bar" not supported by mypyc + +[case testRepeatedUnderscoreFunctions] +def _(arg): pass +def _(arg): pass +[out] +main:2: error: Duplicate definition of "_" not supported by mypyc diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test new file mode 100644 index 000000000000..d17c66bba22f --- /dev/null +++ b/mypyc/test-data/irbuild-str.test @@ -0,0 +1,428 @@ +[case testStrSplit] +from typing import Optional, List + +def do_split(s: str, sep: Optional[str] = None, max_split: Optional[int] = None) -> List[str]: + if sep is not None: + if max_split is not None: + return s.split(sep, max_split) + else: + return s.split(sep) + return s.split() +[out] +def do_split(s, sep, max_split): + s :: str + sep :: union[str, None] + max_split :: union[int, None] + r0, r1, r2 :: object + r3 :: bit + r4 :: object + r5 :: bit + r6 :: str + r7 :: int + r8 :: list + r9 :: str + r10, r11 :: list +L0: + if is_error(sep) goto L1 else goto L2 +L1: + r0 = box(None, 1) + sep = r0 +L2: + if is_error(max_split) goto L3 else goto L4 +L3: + r1 = box(None, 1) + max_split = r1 +L4: + r2 = load_address _Py_NoneStruct + r3 = sep != r2 + if r3 goto L5 else goto L9 :: bool +L5: + r4 = load_address _Py_NoneStruct + r5 = max_split != r4 + if r5 goto L6 else goto L7 :: bool +L6: + r6 = cast(str, sep) + r7 = unbox(int, max_split) + r8 = CPyStr_Split(s, r6, r7) + return r8 +L7: + r9 = cast(str, sep) + r10 = PyUnicode_Split(s, r9, -1) + return r10 +L8: +L9: + r11 = PyUnicode_Split(s, 0, -1) + return r11 + + +[case testStrEquality] +def eq(x: str, y: str) -> bool: + return x == y + +def neq(x: str, y: str) -> bool: + return x != y + +[out] +def eq(x, y): + x, y :: str + r0 :: i32 + r1 :: bit + r2 :: object + r3, r4, r5 :: bit +L0: + r0 = PyUnicode_Compare(x, y) + r1 = r0 == -1 + if r1 goto L1 else goto L3 :: bool +L1: + r2 = PyErr_Occurred() + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPy_KeepPropagating() +L3: + r5 = r0 == 0 + return r5 +def neq(x, y): + x, y :: str + r0 :: i32 + r1 :: bit + r2 :: object + r3, r4, r5 :: bit +L0: + r0 = PyUnicode_Compare(x, y) + r1 = r0 == -1 + if r1 goto L1 else goto L3 :: bool +L1: + r2 = PyErr_Occurred() + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPy_KeepPropagating() +L3: + r5 = r0 != 0 + return r5 + +[case testStrReplace] +from typing import Optional + +def do_replace(s: str, old_substr: str, new_substr: str, max_count: Optional[int] = None) -> str: + if max_count is not None: + return s.replace(old_substr, new_substr, max_count) + else: + return s.replace(old_substr, new_substr) +[out] +def do_replace(s, old_substr, new_substr, max_count): + s, old_substr, new_substr :: str + max_count :: union[int, None] + r0, r1 :: object + r2 :: bit + r3 :: int + r4, r5 :: str +L0: + if is_error(max_count) goto L1 else goto L2 +L1: + r0 = box(None, 1) + max_count = r0 +L2: + r1 = load_address _Py_NoneStruct + r2 = max_count != r1 + if r2 goto L3 else goto L4 :: bool +L3: + r3 = unbox(int, max_count) + r4 = CPyStr_Replace(s, old_substr, new_substr, r3) + return r4 +L4: + r5 = PyUnicode_Replace(s, old_substr, new_substr, -1) + return r5 +L5: + unreachable + +[case testStrToBool] +def is_true(x: str) -> bool: + if x: + return True + else: + return False +[out] +def is_true(x): + x :: str + r0 :: bit +L0: + r0 = CPyStr_IsTrue(x) + if r0 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +L3: + unreachable + +[case testStringFormatMethod] +def f(s: str, num: int) -> None: + s1 = "Hi! I'm {}, and I'm {} years old.".format(s, num) + s2 = ''.format() + s3 = 'abc'.format() + s4 = '}}{}{{{}}}{{{}'.format(num, num, num) +[out] +def f(s, num): + s :: str + num :: int + r0, r1, r2, r3, r4, s1, r5, s2, r6, s3, r7, r8, r9, r10, r11, r12, r13, s4 :: str +L0: + r0 = CPyTagged_Str(num) + r1 = "Hi! I'm " + r2 = ", and I'm " + r3 = ' years old.' + r4 = CPyStr_Build(5, r1, s, r2, r0, r3) + s1 = r4 + r5 = '' + s2 = r5 + r6 = 'abc' + s3 = r6 + r7 = CPyTagged_Str(num) + r8 = CPyTagged_Str(num) + r9 = CPyTagged_Str(num) + r10 = '}' + r11 = '{' + r12 = '}{' + r13 = CPyStr_Build(6, r10, r7, r11, r8, r12, r9) + s4 = r13 + return 1 + +[case testFStrings] +def f(var: str, num: int) -> None: + s1 = f"Hi! I'm {var}. I am {num} years old." + s2 = f'Hello {var:>{num}}' + s3 = f'' + s4 = f'abc' +[out] +def f(var, num): + var :: str + num :: int + r0, r1, r2, r3, r4, s1, r5, r6, r7, r8, r9, r10, r11 :: str + r12 :: object + r13 :: str + r14 :: list + r15 :: ptr + r16, s2, r17, s3, r18, s4 :: str +L0: + r0 = "Hi! I'm " + r1 = '. I am ' + r2 = CPyTagged_Str(num) + r3 = ' years old.' + r4 = CPyStr_Build(5, r0, var, r1, r2, r3) + s1 = r4 + r5 = '' + r6 = 'Hello ' + r7 = '{:{}}' + r8 = '>' + r9 = CPyTagged_Str(num) + r10 = CPyStr_Build(2, r8, r9) + r11 = 'format' + r12 = CPyObject_CallMethodObjArgs(r7, r11, var, r10, 0) + r13 = cast(str, r12) + r14 = PyList_New(2) + r15 = list_items r14 + buf_init_item r15, 0, r6 + buf_init_item r15, 1, r13 + keep_alive r14 + r16 = PyUnicode_Join(r5, r14) + s2 = r16 + r17 = '' + s3 = r17 + r18 = 'abc' + s4 = r18 + return 1 + +[case testStringFormattingCStyle] +def f(var: str, num: int) -> None: + s1 = "Hi! I'm %s." % var + s2 = "I am %d years old." % num + s3 = "Hi! I'm %s. I am %d years old." % (var, num) + s4 = "Float: %f" % num +[typing fixtures/typing-full.pyi] +[out] +def f(var, num): + var :: str + num :: int + r0, r1, r2, s1, r3, r4, r5, r6, s2, r7, r8, r9, r10, r11, s3, r12 :: str + r13, r14 :: object + r15, s4 :: str +L0: + r0 = "Hi! I'm " + r1 = '.' + r2 = CPyStr_Build(3, r0, var, r1) + s1 = r2 + r3 = CPyTagged_Str(num) + r4 = 'I am ' + r5 = ' years old.' + r6 = CPyStr_Build(3, r4, r3, r5) + s2 = r6 + r7 = CPyTagged_Str(num) + r8 = "Hi! I'm " + r9 = '. I am ' + r10 = ' years old.' + r11 = CPyStr_Build(5, r8, var, r9, r7, r10) + s3 = r11 + r12 = 'Float: %f' + r13 = box(int, num) + r14 = PyNumber_Remainder(r12, r13) + r15 = cast(str, r14) + s4 = r15 + return 1 + +[case testDecode] +def f(b: bytes) -> None: + b.decode() + b.decode('utf-8') + b.decode('utf-8', 'backslashreplace') +[out] +def f(b): + b :: bytes + r0, r1, r2, r3, r4, r5 :: str +L0: + r0 = CPy_Decode(b, 0, 0) + r1 = 'utf-8' + r2 = CPy_Decode(b, r1, 0) + r3 = 'utf-8' + r4 = 'backslashreplace' + r5 = CPy_Decode(b, r3, r4) + return 1 + +[case testEncode] +def f(s: str) -> None: + s.encode() + s.encode('utf-8') + s.encode('utf8', 'strict') + s.encode('latin1', errors='strict') + s.encode(encoding='ascii') + s.encode(errors='strict', encoding='latin-1') + s.encode('utf-8', 'backslashreplace') + s.encode('ascii', 'backslashreplace') + encoding = 'utf8' + s.encode(encoding) + errors = 'strict' + s.encode('utf8', errors) + s.encode('utf8', errors=errors) + s.encode(errors=errors) + s.encode(encoding=encoding, errors=errors) + s.encode('latin2') + +[out] +def f(s): + s :: str + r0, r1, r2, r3, r4, r5 :: bytes + r6, r7 :: str + r8 :: bytes + r9, r10 :: str + r11 :: bytes + r12, encoding :: str + r13 :: bytes + r14, errors, r15 :: str + r16 :: bytes + r17, r18 :: str + r19 :: object + r20 :: str + r21 :: tuple + r22 :: dict + r23 :: object + r24 :: str + r25 :: object + r26 :: str + r27 :: tuple + r28 :: dict + r29 :: object + r30 :: str + r31 :: object + r32, r33 :: str + r34 :: tuple + r35 :: dict + r36 :: object + r37 :: str + r38 :: bytes +L0: + r0 = PyUnicode_AsUTF8String(s) + r1 = PyUnicode_AsUTF8String(s) + r2 = PyUnicode_AsUTF8String(s) + r3 = PyUnicode_AsLatin1String(s) + r4 = PyUnicode_AsASCIIString(s) + r5 = PyUnicode_AsLatin1String(s) + r6 = 'utf-8' + r7 = 'backslashreplace' + r8 = CPy_Encode(s, r6, r7) + r9 = 'ascii' + r10 = 'backslashreplace' + r11 = CPy_Encode(s, r9, r10) + r12 = 'utf8' + encoding = r12 + r13 = CPy_Encode(s, encoding, 0) + r14 = 'strict' + errors = r14 + r15 = 'utf8' + r16 = CPy_Encode(s, r15, errors) + r17 = 'utf8' + r18 = 'encode' + r19 = CPyObject_GetAttr(s, r18) + r20 = 'errors' + r21 = PyTuple_Pack(1, r17) + r22 = CPyDict_Build(1, r20, errors) + r23 = PyObject_Call(r19, r21, r22) + r24 = 'encode' + r25 = CPyObject_GetAttr(s, r24) + r26 = 'errors' + r27 = PyTuple_Pack(0) + r28 = CPyDict_Build(1, r26, errors) + r29 = PyObject_Call(r25, r27, r28) + r30 = 'encode' + r31 = CPyObject_GetAttr(s, r30) + r32 = 'encoding' + r33 = 'errors' + r34 = PyTuple_Pack(0) + r35 = CPyDict_Build(2, r32, encoding, r33, errors) + r36 = PyObject_Call(r31, r34, r35) + r37 = 'latin2' + r38 = CPy_Encode(s, r37, 0) + return 1 + +[case testOrd] +def str_ord(x: str) -> int: + return ord(x) +def str_ord_literal() -> int: + return ord("a") +def bytes_ord(x: bytes) -> int: + return ord(x) +def bytes_ord_literal() -> int: + return ord(b"a") +def any_ord(x) -> int: + return ord(x) +[out] +def str_ord(x): + x :: str + r0 :: int +L0: + r0 = CPyStr_Ord(x) + return r0 +def str_ord_literal(): +L0: + return 194 +def bytes_ord(x): + x :: bytes + r0 :: int +L0: + r0 = CPyBytes_Ord(x) + return r0 +def bytes_ord_literal(): +L0: + return 194 +def any_ord(x): + x, r0 :: object + r1 :: str + r2, r3 :: object + r4 :: int +L0: + r0 = builtins :: module + r1 = 'ord' + r2 = CPyObject_GetAttr(r0, r1) + r3 = PyObject_CallFunctionObjArgs(r2, x, 0) + r4 = unbox(int, r3) + return r4 diff --git a/mypyc/test-data/irbuild-strip-asserts.test b/mypyc/test-data/irbuild-strip-asserts.test new file mode 100644 index 000000000000..25fd29818202 --- /dev/null +++ b/mypyc/test-data/irbuild-strip-asserts.test @@ -0,0 +1,12 @@ +[case testStripAssert1] +def g(): + x = 1 + 2 + assert x < 5 + return x +[out] +def g(): + r0, x :: object +L0: + r0 = object 3 + x = r0 + return x diff --git a/mypyc/test-data/irbuild-try.test b/mypyc/test-data/irbuild-try.test new file mode 100644 index 000000000000..a5b7b9a55b86 --- /dev/null +++ b/mypyc/test-data/irbuild-try.test @@ -0,0 +1,523 @@ +[case testTryExcept1] +def g() -> None: + try: + object() + except: + print("weeee") +[out] +def g(): + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: tuple[object, object, object] + r5 :: str + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: bit +L0: +L1: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = PyObject_CallFunctionObjArgs(r2, 0) + goto L5 +L2: (handler for L1) + r4 = CPy_CatchError() + r5 = 'weeee' + r6 = builtins :: module + r7 = 'print' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_CallFunctionObjArgs(r8, r5, 0) +L3: + CPy_RestoreExcInfo(r4) + goto L5 +L4: (handler for L2) + CPy_RestoreExcInfo(r4) + r10 = CPy_KeepPropagating() + unreachable +L5: + return 1 + +[case testTryExcept2] +def g(b: bool) -> None: + try: + if b: + object() + else: + str('hi') + except: + print("weeee") +[out] +def g(b): + b :: bool + r0 :: object + r1 :: str + r2, r3 :: object + r4, r5 :: str + r6 :: tuple[object, object, object] + r7 :: str + r8 :: object + r9 :: str + r10, r11 :: object + r12 :: bit +L0: +L1: + if b goto L2 else goto L3 :: bool +L2: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = PyObject_CallFunctionObjArgs(r2, 0) + goto L4 +L3: + r4 = 'hi' + r5 = PyObject_Str(r4) +L4: + goto L8 +L5: (handler for L1, L2, L3, L4) + r6 = CPy_CatchError() + r7 = 'weeee' + r8 = builtins :: module + r9 = 'print' + r10 = CPyObject_GetAttr(r8, r9) + r11 = PyObject_CallFunctionObjArgs(r10, r7, 0) +L6: + CPy_RestoreExcInfo(r6) + goto L8 +L7: (handler for L5) + CPy_RestoreExcInfo(r6) + r12 = CPy_KeepPropagating() + unreachable +L8: + return 1 + +[case testTryExcept3] +def g() -> None: + try: + print('a') + try: + object() + except AttributeError as e: + print('b', e) + except: + print("weeee") +[out] +def g(): + r0 :: str + r1 :: object + r2 :: str + r3, r4, r5 :: object + r6 :: str + r7, r8 :: object + r9 :: tuple[object, object, object] + r10 :: object + r11 :: str + r12 :: object + r13 :: bit + r14, e :: object + r15 :: str + r16 :: object + r17 :: str + r18, r19 :: object + r20 :: bit + r21 :: tuple[object, object, object] + r22 :: str + r23 :: object + r24 :: str + r25, r26 :: object + r27 :: bit +L0: +L1: + r0 = 'a' + r1 = builtins :: module + r2 = 'print' + r3 = CPyObject_GetAttr(r1, r2) + r4 = PyObject_CallFunctionObjArgs(r3, r0, 0) +L2: + r5 = builtins :: module + r6 = 'object' + r7 = CPyObject_GetAttr(r5, r6) + r8 = PyObject_CallFunctionObjArgs(r7, 0) + goto L8 +L3: (handler for L2) + r9 = CPy_CatchError() + r10 = builtins :: module + r11 = 'AttributeError' + r12 = CPyObject_GetAttr(r10, r11) + r13 = CPy_ExceptionMatches(r12) + if r13 goto L4 else goto L5 :: bool +L4: + r14 = CPy_GetExcValue() + e = r14 + r15 = 'b' + r16 = builtins :: module + r17 = 'print' + r18 = CPyObject_GetAttr(r16, r17) + r19 = PyObject_CallFunctionObjArgs(r18, r15, e, 0) + goto L6 +L5: + CPy_Reraise() + unreachable +L6: + CPy_RestoreExcInfo(r9) + goto L8 +L7: (handler for L3, L4, L5) + CPy_RestoreExcInfo(r9) + r20 = CPy_KeepPropagating() + unreachable +L8: + goto L12 +L9: (handler for L1, L6, L7, L8) + r21 = CPy_CatchError() + r22 = 'weeee' + r23 = builtins :: module + r24 = 'print' + r25 = CPyObject_GetAttr(r23, r24) + r26 = PyObject_CallFunctionObjArgs(r25, r22, 0) +L10: + CPy_RestoreExcInfo(r21) + goto L12 +L11: (handler for L9) + CPy_RestoreExcInfo(r21) + r27 = CPy_KeepPropagating() + unreachable +L12: + return 1 + +[case testTryExcept4] +def g() -> None: + try: + pass + except KeyError: + print("weeee") + except IndexError: + print("yo") +[out] +def g(): + r0 :: tuple[object, object, object] + r1 :: object + r2 :: str + r3 :: object + r4 :: bit + r5 :: str + r6 :: object + r7 :: str + r8, r9, r10 :: object + r11 :: str + r12 :: object + r13 :: bit + r14 :: str + r15 :: object + r16 :: str + r17, r18 :: object + r19 :: bit +L0: +L1: + goto L9 +L2: (handler for L1) + r0 = CPy_CatchError() + r1 = builtins :: module + r2 = 'KeyError' + r3 = CPyObject_GetAttr(r1, r2) + r4 = CPy_ExceptionMatches(r3) + if r4 goto L3 else goto L4 :: bool +L3: + r5 = 'weeee' + r6 = builtins :: module + r7 = 'print' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_CallFunctionObjArgs(r8, r5, 0) + goto L7 +L4: + r10 = builtins :: module + r11 = 'IndexError' + r12 = CPyObject_GetAttr(r10, r11) + r13 = CPy_ExceptionMatches(r12) + if r13 goto L5 else goto L6 :: bool +L5: + r14 = 'yo' + r15 = builtins :: module + r16 = 'print' + r17 = CPyObject_GetAttr(r15, r16) + r18 = PyObject_CallFunctionObjArgs(r17, r14, 0) + goto L7 +L6: + CPy_Reraise() + unreachable +L7: + CPy_RestoreExcInfo(r0) + goto L9 +L8: (handler for L2, L3, L4, L5, L6) + CPy_RestoreExcInfo(r0) + r19 = CPy_KeepPropagating() + unreachable +L9: + return 1 + +[case testTryFinally] +def a(b: bool) -> None: + try: + if b: + raise Exception('hi') + finally: + print('finally') +[out] +def a(b): + b :: bool + r0 :: str + r1 :: object + r2 :: str + r3, r4 :: object + r5, r6, r7 :: tuple[object, object, object] + r8 :: str + r9 :: object + r10 :: str + r11, r12 :: object + r13 :: bit +L0: +L1: + if b goto L2 else goto L3 :: bool +L2: + r0 = 'hi' + r1 = builtins :: module + r2 = 'Exception' + r3 = CPyObject_GetAttr(r1, r2) + r4 = PyObject_CallFunctionObjArgs(r3, r0, 0) + CPy_Raise(r4) + unreachable +L3: +L4: +L5: + r5 = :: tuple[object, object, object] + r6 = r5 + goto L7 +L6: (handler for L1, L2, L3) + r7 = CPy_CatchError() + r6 = r7 +L7: + r8 = 'finally' + r9 = builtins :: module + r10 = 'print' + r11 = CPyObject_GetAttr(r9, r10) + r12 = PyObject_CallFunctionObjArgs(r11, r8, 0) + if is_error(r6) goto L9 else goto L8 +L8: + CPy_Reraise() + unreachable +L9: + goto L13 +L10: (handler for L7, L8) + if is_error(r6) goto L12 else goto L11 +L11: + CPy_RestoreExcInfo(r6) +L12: + r13 = CPy_KeepPropagating() + unreachable +L13: + return 1 + +[case testWith] +from typing import Any +def foo(x: Any) -> None: + with x() as y: + print('hello') +[out] +def foo(x): + x, r0, r1 :: object + r2 :: str + r3 :: object + r4 :: str + r5, r6 :: object + r7 :: bool + y :: object + r8 :: str + r9 :: object + r10 :: str + r11, r12 :: object + r13, r14 :: tuple[object, object, object] + r15, r16, r17, r18 :: object + r19 :: i32 + r20 :: bit + r21 :: bool + r22 :: bit + r23, r24, r25 :: tuple[object, object, object] + r26, r27 :: object + r28 :: bit +L0: + r0 = PyObject_CallFunctionObjArgs(x, 0) + r1 = PyObject_Type(r0) + r2 = '__exit__' + r3 = CPyObject_GetAttr(r1, r2) + r4 = '__enter__' + r5 = CPyObject_GetAttr(r1, r4) + r6 = PyObject_CallFunctionObjArgs(r5, r0, 0) + r7 = 1 +L1: +L2: + y = r6 + r8 = 'hello' + r9 = builtins :: module + r10 = 'print' + r11 = CPyObject_GetAttr(r9, r10) + r12 = PyObject_CallFunctionObjArgs(r11, r8, 0) + goto L8 +L3: (handler for L2) + r13 = CPy_CatchError() + r7 = 0 + r14 = CPy_GetExcInfo() + r15 = r14[0] + r16 = r14[1] + r17 = r14[2] + r18 = PyObject_CallFunctionObjArgs(r3, r0, r15, r16, r17, 0) + r19 = PyObject_IsTrue(r18) + r20 = r19 >= 0 :: signed + r21 = truncate r19: i32 to builtins.bool + if r21 goto L5 else goto L4 :: bool +L4: + CPy_Reraise() + unreachable +L5: +L6: + CPy_RestoreExcInfo(r13) + goto L8 +L7: (handler for L3, L4, L5) + CPy_RestoreExcInfo(r13) + r22 = CPy_KeepPropagating() + unreachable +L8: +L9: +L10: + r23 = :: tuple[object, object, object] + r24 = r23 + goto L12 +L11: (handler for L1, L6, L7, L8) + r25 = CPy_CatchError() + r24 = r25 +L12: + if r7 goto L13 else goto L14 :: bool +L13: + r26 = load_address _Py_NoneStruct + r27 = PyObject_CallFunctionObjArgs(r3, r0, r26, r26, r26, 0) +L14: + if is_error(r24) goto L16 else goto L15 +L15: + CPy_Reraise() + unreachable +L16: + goto L20 +L17: (handler for L12, L13, L14, L15) + if is_error(r24) goto L19 else goto L18 +L18: + CPy_RestoreExcInfo(r24) +L19: + r28 = CPy_KeepPropagating() + unreachable +L20: + return 1 + +[case testWithNativeSimple] +class DummyContext: + def __enter__(self) -> None: + pass + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + pass + +def foo(x: DummyContext) -> None: + with x: + print('hello') +[out] +def DummyContext.__enter__(self): + self :: __main__.DummyContext +L0: + return 1 +def DummyContext.__exit__(self, exc_type, exc_val, exc_tb): + self :: __main__.DummyContext + exc_type, exc_val, exc_tb :: object +L0: + return 1 +def foo(x): + x :: __main__.DummyContext + r0 :: None + r1 :: bool + r2 :: str + r3 :: object + r4 :: str + r5, r6 :: object + r7, r8 :: tuple[object, object, object] + r9, r10, r11 :: object + r12 :: None + r13 :: object + r14 :: i32 + r15 :: bit + r16 :: bool + r17 :: bit + r18, r19, r20 :: tuple[object, object, object] + r21 :: object + r22 :: None + r23 :: bit +L0: + r0 = x.__enter__() + r1 = 1 +L1: +L2: + r2 = 'hello' + r3 = builtins :: module + r4 = 'print' + r5 = CPyObject_GetAttr(r3, r4) + r6 = PyObject_CallFunctionObjArgs(r5, r2, 0) + goto L8 +L3: (handler for L2) + r7 = CPy_CatchError() + r1 = 0 + r8 = CPy_GetExcInfo() + r9 = r8[0] + r10 = r8[1] + r11 = r8[2] + r12 = x.__exit__(r9, r10, r11) + r13 = box(None, r12) + r14 = PyObject_IsTrue(r13) + r15 = r14 >= 0 :: signed + r16 = truncate r14: i32 to builtins.bool + if r16 goto L5 else goto L4 :: bool +L4: + CPy_Reraise() + unreachable +L5: +L6: + CPy_RestoreExcInfo(r7) + goto L8 +L7: (handler for L3, L4, L5) + CPy_RestoreExcInfo(r7) + r17 = CPy_KeepPropagating() + unreachable +L8: +L9: +L10: + r18 = :: tuple[object, object, object] + r19 = r18 + goto L12 +L11: (handler for L1, L6, L7, L8) + r20 = CPy_CatchError() + r19 = r20 +L12: + if r1 goto L13 else goto L14 :: bool +L13: + r21 = load_address _Py_NoneStruct + r22 = x.__exit__(r21, r21, r21) +L14: + if is_error(r19) goto L16 else goto L15 +L15: + CPy_Reraise() + unreachable +L16: + goto L20 +L17: (handler for L12, L13, L14, L15) + if is_error(r19) goto L19 else goto L18 +L18: + CPy_RestoreExcInfo(r19) +L19: + r23 = CPy_KeepPropagating() + unreachable +L20: + return 1 diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test new file mode 100644 index 000000000000..a6813de4ee44 --- /dev/null +++ b/mypyc/test-data/irbuild-tuple.test @@ -0,0 +1,386 @@ +[case testTupleGet] +from typing import Tuple + +def f(x: Tuple[Tuple[int, bool], bool]) -> int: + return x[0][0] +[out] +def f(x): + x :: tuple[tuple[int, bool], bool] + r0 :: tuple[int, bool] + r1 :: int +L0: + r0 = x[0] + r1 = r0[0] + return r1 + +[case testTupleNew] +from typing import Tuple + +def f() -> int: + t = (True, 1) + return t[1] +[out] +def f(): + r0, t :: tuple[bool, int] + r1 :: int +L0: + r0 = (1, 2) + t = r0 + r1 = t[1] + return r1 + +[case testTupleLen] +from typing import Tuple +def f(x: Tuple[bool, bool, int]) -> int: + return len(x) +[out] +def f(x): + x :: tuple[bool, bool, int] +L0: + return 6 + +[case testSequenceTuple] +from typing import List +def f(x: List[bool]) -> bool: + return tuple(x)[1] +[out] +def f(x): + x :: list + r0 :: tuple + r1 :: object + r2 :: bool +L0: + r0 = PyList_AsTuple(x) + r1 = CPySequenceTuple_GetItem(r0, 2) + r2 = unbox(bool, r1) + return r2 + +[case testSequenceTupleLen] +from typing import Tuple +def f(x: Tuple[int, ...]) -> int: + return len(x) +[out] +def f(x): + x :: tuple + r0 :: native_int + r1 :: short_int +L0: + r0 = var_object_size x + r1 = r0 << 1 + return r1 + +[case testSequenceTupleForced] +from typing import Tuple +def f() -> int: + t = (1, 2) # type: Tuple[int, ...] + return t[1] +[out] +def f(): + r0 :: tuple[int, int] + r1 :: object + t :: tuple + r2 :: object + r3 :: int +L0: + r0 = (2, 4) + r1 = box(tuple[int, int], r0) + t = r1 + r2 = CPySequenceTuple_GetItem(t, 2) + r3 = unbox(int, r2) + return r3 + +[case testTupleDisplay] +from typing import Sequence, Tuple +def f(x: Sequence[int], y: Sequence[int]) -> Tuple[int, ...]: + return (1, 2, *x, *y, 3) +[out] +def f(x, y): + x, y :: object + r0 :: list + r1, r2 :: object + r3 :: ptr + r4, r5, r6 :: object + r7 :: i32 + r8 :: bit + r9 :: tuple +L0: + r0 = PyList_New(2) + r1 = object 1 + r2 = object 2 + r3 = list_items r0 + buf_init_item r3, 0, r1 + buf_init_item r3, 1, r2 + keep_alive r0 + r4 = CPyList_Extend(r0, x) + r5 = CPyList_Extend(r0, y) + r6 = object 3 + r7 = PyList_Append(r0, r6) + r8 = r7 >= 0 :: signed + r9 = PyList_AsTuple(r0) + return r9 + +[case testTupleFor] +from typing import Tuple, List +def f(xs: Tuple[str, ...]) -> None: + for x in xs: + pass +[out] +def f(xs): + xs :: tuple + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: bit + r4 :: object + r5, x :: str + r6 :: short_int +L0: + r0 = 0 +L1: + r1 = var_object_size xs + r2 = r1 << 1 + r3 = int_lt r0, r2 + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPySequenceTuple_GetItem(xs, r0) + r5 = cast(str, r4) + x = r5 +L3: + r6 = r0 + 2 + r0 = r6 + goto L1 +L4: + return 1 + +[case testNamedTupleAttribute] +from typing import NamedTuple + +NT = NamedTuple('NT', [('x', int), ('y', int)]) + +def f(nt: NT, b: bool) -> int: + if b: + return nt.x + return nt.y +[out] +def f(nt, b): + nt :: tuple + b :: bool + r0 :: object + r1 :: int + r2 :: object + r3 :: int +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = CPySequenceTuple_GetItem(nt, 0) + r1 = unbox(int, r0) + return r1 +L2: + r2 = CPySequenceTuple_GetItem(nt, 2) + r3 = unbox(int, r2) + return r3 + + +[case testTupleOperatorIn] +def f(i: int) -> bool: + return i in [1, 2, 3] +[out] +def f(i): + i :: int + r0 :: bit + r1 :: bool + r2 :: bit + r3 :: bool + r4 :: bit +L0: + r0 = int_eq i, 2 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = r0 + goto L3 +L2: + r2 = int_eq i, 4 + r1 = r2 +L3: + if r1 goto L4 else goto L5 :: bool +L4: + r3 = r1 + goto L6 +L5: + r4 = int_eq i, 6 + r3 = r4 +L6: + return r3 + +[case testTupleBuiltFromList] +def f(val: int) -> bool: + return val % 2 == 0 + +def test() -> None: + source = [1, 2, 3] + a = tuple(f(x) for x in source) +[out] +def f(val): + val, r0 :: int + r1 :: bit +L0: + r0 = CPyTagged_Remainder(val, 4) + r1 = int_eq r0, 0 + return r1 +def test(): + r0 :: list + r1, r2, r3 :: object + r4 :: ptr + source :: list + r5 :: native_int + r6 :: tuple + r7 :: short_int + r8 :: native_int + r9 :: short_int + r10 :: bit + r11 :: object + r12, x :: int + r13 :: bool + r14 :: object + r15 :: bit + r16 :: short_int + a :: tuple +L0: + r0 = PyList_New(3) + r1 = object 1 + r2 = object 2 + r3 = object 3 + r4 = list_items r0 + buf_init_item r4, 0, r1 + buf_init_item r4, 1, r2 + buf_init_item r4, 2, r3 + keep_alive r0 + source = r0 + r5 = var_object_size source + r6 = PyTuple_New(r5) + r7 = 0 +L1: + r8 = var_object_size source + r9 = r8 << 1 + r10 = int_lt r7, r9 + if r10 goto L2 else goto L4 :: bool +L2: + r11 = CPyList_GetItemUnsafe(source, r7) + r12 = unbox(int, r11) + x = r12 + r13 = f(x) + r14 = box(bool, r13) + r15 = CPySequenceTuple_SetItemUnsafe(r6, r7, r14) +L3: + r16 = r7 + 2 + r7 = r16 + goto L1 +L4: + a = r6 + return 1 + +[case testTupleBuiltFromStr] +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + source = "abc" + a = tuple(f2(x) for x in source) +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0, source :: str + r1 :: native_int + r2 :: bit + r3 :: tuple + r4 :: short_int + r5 :: native_int + r6 :: bit + r7 :: short_int + r8 :: bit + r9, x, r10 :: str + r11 :: bit + r12 :: short_int + a :: tuple +L0: + r0 = 'abc' + source = r0 + r1 = CPyStr_Size_size_t(source) + r2 = r1 >= 0 :: signed + r3 = PyTuple_New(r1) + r4 = 0 +L1: + r5 = CPyStr_Size_size_t(source) + r6 = r5 >= 0 :: signed + r7 = r5 << 1 + r8 = int_lt r4, r7 + if r8 goto L2 else goto L4 :: bool +L2: + r9 = CPyStr_GetItem(source, r4) + x = r9 + r10 = f2(x) + r11 = CPySequenceTuple_SetItemUnsafe(r3, r4, r10) +L3: + r12 = r4 + 2 + r4 = r12 + goto L1 +L4: + a = r3 + return 1 + +[case testTupleBuiltFromVariableLengthTuple] +from typing import Tuple + +def f(val: bool) -> bool: + return not val + +def test(source: Tuple[bool, ...]) -> None: + a = tuple(f(x) for x in source) +[out] +def f(val): + val, r0 :: bool +L0: + r0 = val ^ 1 + return r0 +def test(source): + source :: tuple + r0 :: native_int + r1 :: tuple + r2 :: short_int + r3 :: native_int + r4 :: short_int + r5 :: bit + r6 :: object + r7, x, r8 :: bool + r9 :: object + r10 :: bit + r11 :: short_int + a :: tuple +L0: + r0 = var_object_size source + r1 = PyTuple_New(r0) + r2 = 0 +L1: + r3 = var_object_size source + r4 = r3 << 1 + r5 = int_lt r2, r4 + if r5 goto L2 else goto L4 :: bool +L2: + r6 = CPySequenceTuple_GetItem(source, r2) + r7 = unbox(bool, r6) + x = r7 + r8 = f(x) + r9 = box(bool, r8) + r10 = CPySequenceTuple_SetItemUnsafe(r1, r2, r9) +L3: + r11 = r2 + 2 + r2 = r11 + goto L1 +L4: + a = r1 + return 1 diff --git a/mypyc/test-data/irbuild-u8.test b/mypyc/test-data/irbuild-u8.test new file mode 100644 index 000000000000..14f691c9451f --- /dev/null +++ b/mypyc/test-data/irbuild-u8.test @@ -0,0 +1,543 @@ +# Test cases for u8 native ints. Focus on things that are different from i64; no need to +# duplicate all i64 test cases here. + +[case testU8BinaryOp] +from mypy_extensions import u8 + +def add_op(x: u8, y: u8) -> u8: + x = y + x + y = x + 5 + y += x + y += 7 + x = 5 + y + return x +def compare(x: u8, y: u8) -> None: + a = x == y + b = x == 5 + c = x < y + d = x < 5 + e = 5 == x + f = 5 < x +[out] +def add_op(x, y): + x, y, r0, r1, r2, r3, r4 :: u8 +L0: + r0 = y + x + x = r0 + r1 = x + 5 + y = r1 + r2 = y + x + y = r2 + r3 = y + 7 + y = r3 + r4 = 5 + y + x = r4 + return x +def compare(x, y): + x, y :: u8 + r0 :: bit + a :: bool + r1 :: bit + b :: bool + r2 :: bit + c :: bool + r3 :: bit + d :: bool + r4 :: bit + e :: bool + r5 :: bit + f :: bool +L0: + r0 = x == y + a = r0 + r1 = x == 5 + b = r1 + r2 = x < y :: unsigned + c = r2 + r3 = x < 5 :: unsigned + d = r3 + r4 = 5 == x + e = r4 + r5 = 5 < x :: unsigned + f = r5 + return 1 + +[case testU8UnaryOp] +from mypy_extensions import u8 + +def unary(x: u8) -> u8: + y = -x + x = ~y + y = +x + return y +[out] +def unary(x): + x, r0, y, r1 :: u8 +L0: + r0 = 0 - x + y = r0 + r1 = y ^ 255 + x = r1 + y = x + return y + +[case testU8DivisionByConstant] +from mypy_extensions import u8 + +def div_by_constant(x: u8) -> u8: + x = x // 5 + x //= 17 + return x +[out] +def div_by_constant(x): + x, r0, r1 :: u8 +L0: + r0 = x / 5 + x = r0 + r1 = x / 17 + x = r1 + return x + +[case testU8ModByConstant] +from mypy_extensions import u8 + +def mod_by_constant(x: u8) -> u8: + x = x % 5 + x %= 17 + return x +[out] +def mod_by_constant(x): + x, r0, r1 :: u8 +L0: + r0 = x % 5 + x = r0 + r1 = x % 17 + x = r1 + return x + +[case testU8DivModByVariable] +from mypy_extensions import u8 + +def divmod(x: u8, y: u8) -> u8: + a = x // y + return a % y +[out] +def divmod(x, y): + x, y :: u8 + r0 :: bit + r1 :: bool + r2, a :: u8 + r3 :: bit + r4 :: bool + r5 :: u8 +L0: + r0 = y == 0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = raise ZeroDivisionError('integer division or modulo by zero') + unreachable +L2: + r2 = x / y + a = r2 + r3 = y == 0 + if r3 goto L3 else goto L4 :: bool +L3: + r4 = raise ZeroDivisionError('integer division or modulo by zero') + unreachable +L4: + r5 = a % y + return r5 + +[case testU8BinaryOperationWithOutOfRangeOperand] +from mypy_extensions import u8 + +def out_of_range(x: u8) -> None: + x + (-1) + (-2) + x + x * 256 + -1 < x + x > -5 + x == 1000 + x + 255 # OK + 255 + x # OK +[out] +main:4: error: Value -1 is out of range for "u8" +main:5: error: Value -2 is out of range for "u8" +main:6: error: Value 256 is out of range for "u8" +main:7: error: Value -1 is out of range for "u8" +main:8: error: Value -5 is out of range for "u8" +main:9: error: Value 1000 is out of range for "u8" + +[case testU8DetectMoreOutOfRangeLiterals] +from mypy_extensions import u8 + +def out_of_range() -> None: + a: u8 = 256 + b: u8 = -1 + f(256) + # The following are ok + c: u8 = 0 + d: u8 = 255 + f(0) + f(255) + +def f(x: u8) -> None: pass +[out] +main:4: error: Value 256 is out of range for "u8" +main:5: error: Value -1 is out of range for "u8" +main:6: error: Value 256 is out of range for "u8" + +[case testU8BoxAndUnbox] +from typing import Any +from mypy_extensions import u8 + +def f(x: Any) -> Any: + y: u8 = x + return y +[out] +def f(x): + x :: object + r0, y :: u8 + r1 :: object +L0: + r0 = unbox(u8, x) + y = r0 + r1 = box(u8, y) + return r1 + +[case testU8MixedCompare1] +from mypy_extensions import u8 +def f(x: int, y: u8) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: u8 + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: u8 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 512 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= 0 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to u8 + r6 = r5 + goto L5 +L4: + CPyUInt8_Overflow() + unreachable +L5: + r7 = r6 == y + return r7 + +[case testU8MixedCompare2] +from mypy_extensions import u8 +def f(x: u8, y: int) -> bool: + return x == y +[out] +def f(x, y): + x :: u8 + y :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: u8 + r7 :: bit +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = y < 512 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = y >= 0 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = y >> 1 + r5 = truncate r4: native_int to u8 + r6 = r5 + goto L5 +L4: + CPyUInt8_Overflow() + unreachable +L5: + r7 = x == r6 + return r7 + +[case testU8ConvertToInt] +from mypy_extensions import u8 + +def u8_to_int(a: u8) -> int: + return a +[out] +def u8_to_int(a): + a :: u8 + r0 :: native_int + r1 :: int +L0: + r0 = extend a: u8 to native_int + r1 = r0 << 1 + return r1 + +[case testU8OperatorAssignmentMixed] +from mypy_extensions import u8 + +def f(a: u8) -> None: + x = 0 + x += a +[out] +def f(a): + a :: u8 + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6, r7 :: u8 + r8 :: native_int + r9 :: int +L0: + x = 0 + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 512 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= 0 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to u8 + r6 = r5 + goto L5 +L4: + CPyUInt8_Overflow() + unreachable +L5: + r7 = r6 + a + r8 = extend r7: u8 to native_int + r9 = r8 << 1 + x = r9 + return 1 + +[case testU8InitializeFromLiteral] +from mypy_extensions import u8, i64 + +def f() -> None: + x: u8 = 0 + y: u8 = 255 + z: u8 = 5 + 7 +[out] +def f(): + x, y, z :: u8 +L0: + x = 0 + y = 255 + z = 12 + return 1 + +[case testU8ExplicitConversionFromNativeInt] +from mypy_extensions import i64, i32, i16, u8 + +def from_u8(x: u8) -> u8: + return u8(x) + +def from_i16(x: i16) -> u8: + return u8(x) + +def from_i32(x: i32) -> u8: + return u8(x) + +def from_i64(x: i64) -> u8: + return u8(x) +[out] +def from_u8(x): + x :: u8 +L0: + return x +def from_i16(x): + x :: i16 + r0 :: u8 +L0: + r0 = truncate x: i16 to u8 + return r0 +def from_i32(x): + x :: i32 + r0 :: u8 +L0: + r0 = truncate x: i32 to u8 + return r0 +def from_i64(x): + x :: i64 + r0 :: u8 +L0: + r0 = truncate x: i64 to u8 + return r0 + +[case testU8ExplicitConversionToNativeInt] +from mypy_extensions import i64, i32, i16, u8 + +def to_i16(x: u8) -> i16: + return i16(x) + +def to_i32(x: u8) -> i32: + return i32(x) + +def to_i64(x: u8) -> i64: + return i64(x) +[out] +def to_i16(x): + x :: u8 + r0 :: i16 +L0: + r0 = extend x: u8 to i16 + return r0 +def to_i32(x): + x :: u8 + r0 :: i32 +L0: + r0 = extend x: u8 to i32 + return r0 +def to_i64(x): + x :: u8 + r0 :: i64 +L0: + r0 = extend x: u8 to i64 + return r0 + +[case testU8ExplicitConversionFromInt] +from mypy_extensions import u8 + +def f(x: int) -> u8: + return u8(x) +[out] +def f(x): + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: u8 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 512 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= 0 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to u8 + r6 = r5 + goto L5 +L4: + CPyUInt8_Overflow() + unreachable +L5: + return r6 + +[case testU8ExplicitConversionFromLiteral] +from mypy_extensions import u8 + +def f() -> None: + x = u8(0) + y = u8(11) + z = u8(-3) # Truncate + zz = u8(258) # Truncate + a = u8(255) +[out] +def f(): + x, y, z, zz, a :: u8 +L0: + x = 0 + y = 11 + z = 253 + zz = 2 + a = 255 + return 1 + +[case testU8ExplicitConversionFromVariousTypes] +from mypy_extensions import u8 + +def bool_to_u8(b: bool) -> u8: + return u8(b) + +def str_to_u8(s: str) -> u8: + return u8(s) + +class C: + def __int__(self) -> u8: + return 5 + +def instance_to_u8(c: C) -> u8: + return u8(c) + +def float_to_u8(x: float) -> u8: + return u8(x) +[out] +def bool_to_u8(b): + b :: bool + r0 :: u8 +L0: + r0 = extend b: builtins.bool to u8 + return r0 +def str_to_u8(s): + s :: str + r0 :: object + r1 :: u8 +L0: + r0 = CPyLong_FromStr(s) + r1 = unbox(u8, r0) + return r1 +def C.__int__(self): + self :: __main__.C +L0: + return 5 +def instance_to_u8(c): + c :: __main__.C + r0 :: u8 +L0: + r0 = c.__int__() + return r0 +def float_to_u8(x): + x :: float + r0 :: int + r1 :: native_int + r2, r3, r4 :: bit + r5 :: native_int + r6, r7 :: u8 +L0: + r0 = CPyTagged_FromFloat(x) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L4 :: bool +L1: + r3 = r0 < 512 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = r0 >= 0 :: signed + if r4 goto L3 else goto L4 :: bool +L3: + r5 = r0 >> 1 + r6 = truncate r5: native_int to u8 + r7 = r6 + goto L5 +L4: + CPyUInt8_Overflow() + unreachable +L5: + return r7 diff --git a/mypyc/test-data/irbuild-unreachable.test b/mypyc/test-data/irbuild-unreachable.test new file mode 100644 index 000000000000..b5188c91ac58 --- /dev/null +++ b/mypyc/test-data/irbuild-unreachable.test @@ -0,0 +1,241 @@ +# Test cases for unreachable expressions and statements + +[case testUnreachableMemberExpr] +import sys + +def f() -> None: + y = sys.platform == "x" and sys.version_info > (3, 5) +[out] +def f(): + r0 :: object + r1 :: str + r2 :: object + r3, r4 :: str + r5 :: i32 + r6 :: bit + r7 :: object + r8, r9, r10 :: bit + r11, r12 :: bool + r13 :: object + r14 :: str + r15 :: object + r16 :: tuple[int, int] + r17, r18 :: object + r19, y :: bool +L0: + r0 = sys :: module + r1 = 'platform' + r2 = CPyObject_GetAttr(r0, r1) + r3 = cast(str, r2) + r4 = 'x' + r5 = PyUnicode_Compare(r3, r4) + r6 = r5 == -1 + if r6 goto L1 else goto L3 :: bool +L1: + r7 = PyErr_Occurred() + r8 = r7 != 0 + if r8 goto L2 else goto L3 :: bool +L2: + r9 = CPy_KeepPropagating() +L3: + r10 = r5 == 0 + if r10 goto L5 else goto L4 :: bool +L4: + r11 = r10 + goto L6 +L5: + r12 = raise RuntimeError('mypyc internal error: should be unreachable') + r13 = box(None, 1) + r14 = 'version_info' + r15 = CPyObject_GetAttr(r13, r14) + r16 = (6, 10) + r17 = box(tuple[int, int], r16) + r18 = PyObject_RichCompare(r15, r17, 4) + r19 = unbox(bool, r18) + r11 = r19 +L6: + y = r11 + return 1 + +[case testUnreachableNameExpr] +import sys + +def f() -> None: + y = sys.platform == 'x' and foobar +[out] +def f(): + r0 :: object + r1 :: str + r2 :: object + r3, r4 :: str + r5 :: i32 + r6 :: bit + r7 :: object + r8, r9, r10 :: bit + r11, r12 :: bool + r13 :: object + r14, y :: bool +L0: + r0 = sys :: module + r1 = 'platform' + r2 = CPyObject_GetAttr(r0, r1) + r3 = cast(str, r2) + r4 = 'x' + r5 = PyUnicode_Compare(r3, r4) + r6 = r5 == -1 + if r6 goto L1 else goto L3 :: bool +L1: + r7 = PyErr_Occurred() + r8 = r7 != 0 + if r8 goto L2 else goto L3 :: bool +L2: + r9 = CPy_KeepPropagating() +L3: + r10 = r5 == 0 + if r10 goto L5 else goto L4 :: bool +L4: + r11 = r10 + goto L6 +L5: + r12 = raise RuntimeError('mypyc internal error: should be unreachable') + r13 = box(None, 1) + r14 = unbox(bool, r13) + r11 = r14 +L6: + y = r11 + return 1 + +[case testUnreachableStatementAfterReturn] +def f(x: bool) -> int: + if x: + return 1 + f(False) + return 2 +[out] +def f(x): + x :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 + +[case testUnreachableStatementAfterContinue] +def c() -> bool: + return False + +def f() -> None: + n = True + while n: + if c(): + continue + if int(): + f() + n = False +[out] +def c(): +L0: + return 0 +def f(): + n, r0 :: bool +L0: + n = 1 +L1: + if n goto L2 else goto L5 :: bool +L2: + r0 = c() + if r0 goto L3 else goto L4 :: bool +L3: + goto L1 +L4: + n = 0 + goto L1 +L5: + return 1 + +[case testUnreachableStatementAfterBreak] +def c() -> bool: + return False + +def f() -> None: + n = True + while n: + if c(): + break + if int(): + f() + n = False +[out] +def c(): +L0: + return 0 +def f(): + n, r0 :: bool +L0: + n = 1 +L1: + if n goto L2 else goto L5 :: bool +L2: + r0 = c() + if r0 goto L3 else goto L4 :: bool +L3: + goto L5 +L4: + n = 0 + goto L1 +L5: + return 1 + +[case testUnreachableStatementAfterRaise] +def f(x: bool) -> int: + if x: + raise ValueError() + print('hello') + return 2 +[out] +def f(x): + x :: bool + r0 :: object + r1 :: str + r2, r3 :: object +L0: + if x goto L1 else goto L2 :: bool +L1: + r0 = builtins :: module + r1 = 'ValueError' + r2 = CPyObject_GetAttr(r0, r1) + r3 = PyObject_CallFunctionObjArgs(r2, 0) + CPy_Raise(r3) + unreachable +L2: + return 4 + +[case testUnreachableStatementAfterAssertFalse] +def f(x: bool) -> int: + if x: + assert False + print('hello') + return 2 +[out] +def f(x): + x, r0 :: bool + r1 :: str + r2 :: object + r3 :: str + r4, r5 :: object +L0: + if x goto L1 else goto L4 :: bool +L1: + if 0 goto L3 else goto L2 :: bool +L2: + r0 = raise AssertionError + unreachable +L3: + r1 = 'hello' + r2 = builtins :: module + r3 = 'print' + r4 = CPyObject_GetAttr(r2, r3) + r5 = PyObject_CallFunctionObjArgs(r4, r1, 0) +L4: + return 4 diff --git a/mypyc/test-data/irbuild-vectorcall.test b/mypyc/test-data/irbuild-vectorcall.test new file mode 100644 index 000000000000..1ba08efc2501 --- /dev/null +++ b/mypyc/test-data/irbuild-vectorcall.test @@ -0,0 +1,153 @@ +-- Test cases for calls using the vectorcall API (Python 3.8+) +-- +-- Vectorcalls are faster than the legacy API, especially with keyword arguments, +-- since there is no need to allocate a temporary dictionary for keyword args. + +[case testeVectorcallBasic_python3_8] +from typing import Any + +def f(c: Any) -> None: + c() + c('x', 'y') +[out] +def f(c): + c, r0 :: object + r1, r2 :: str + r3 :: object[2] + r4 :: object_ptr + r5 :: object +L0: + r0 = _PyObject_Vectorcall(c, 0, 0, 0) + r1 = 'x' + r2 = 'y' + r3 = [r1, r2] + r4 = load_address r3 + r5 = _PyObject_Vectorcall(c, r4, 2, 0) + keep_alive r1, r2 + return 1 + +[case testVectorcallKeywords_python3_8] +from typing import Any + +def f(c: Any) -> None: + c(x='a') + c('x', a='y', b='z') +[out] +def f(c): + c :: object + r0 :: str + r1 :: object[1] + r2 :: object_ptr + r3, r4 :: object + r5, r6, r7 :: str + r8 :: object[3] + r9 :: object_ptr + r10, r11 :: object +L0: + r0 = 'a' + r1 = [r0] + r2 = load_address r1 + r3 = ('x',) + r4 = _PyObject_Vectorcall(c, r2, 0, r3) + keep_alive r0 + r5 = 'x' + r6 = 'y' + r7 = 'z' + r8 = [r5, r6, r7] + r9 = load_address r8 + r10 = ('a', 'b') + r11 = _PyObject_Vectorcall(c, r9, 1, r10) + keep_alive r5, r6, r7 + return 1 + +[case testVectorcallMethod_python3_8] +from typing import Any + +def f(o: Any) -> None: + # On Python 3.8 vectorcalls are only faster with keyword args + o.m('x') + o.m('x', a='y') +[out] +def f(o): + o :: object + r0, r1 :: str + r2 :: object + r3, r4, r5 :: str + r6 :: object + r7 :: object[2] + r8 :: object_ptr + r9, r10 :: object +L0: + r0 = 'x' + r1 = 'm' + r2 = CPyObject_CallMethodObjArgs(o, r1, r0, 0) + r3 = 'x' + r4 = 'y' + r5 = 'm' + r6 = CPyObject_GetAttr(o, r5) + r7 = [r3, r4] + r8 = load_address r7 + r9 = ('a',) + r10 = _PyObject_Vectorcall(r6, r8, 1, r9) + keep_alive r3, r4 + return 1 + +[case testVectorcallMethod_python3_9_64bit] +from typing import Any + +def f(o: Any) -> None: + # Python 3.9 has a new API for calling methods + o.m('x') + o.m('x', 'y', a='z') +[out] +def f(o): + o :: object + r0, r1 :: str + r2 :: object[2] + r3 :: object_ptr + r4 :: object + r5, r6, r7, r8 :: str + r9 :: object[4] + r10 :: object_ptr + r11, r12 :: object +L0: + r0 = 'x' + r1 = 'm' + r2 = [o, r0] + r3 = load_address r2 + r4 = PyObject_VectorcallMethod(r1, r3, 9223372036854775810, 0) + keep_alive o, r0 + r5 = 'x' + r6 = 'y' + r7 = 'z' + r8 = 'm' + r9 = [o, r5, r6, r7] + r10 = load_address r9 + r11 = ('a',) + r12 = PyObject_VectorcallMethod(r8, r10, 9223372036854775811, r11) + keep_alive o, r5, r6, r7 + return 1 + +[case testVectorcallMethod_python3_9_32bit] +from typing import Any + +def f(o: Any) -> None: + # The IR is slightly different on 32-bit platforms + o.m('x', a='y') +[out] +def f(o): + o :: object + r0, r1, r2 :: str + r3 :: object[3] + r4 :: object_ptr + r5, r6 :: object +L0: + r0 = 'x' + r1 = 'y' + r2 = 'm' + r3 = [o, r0, r1] + r4 = load_address r3 + r5 = ('a',) + r6 = PyObject_VectorcallMethod(r2, r4, 2147483650, r5) + keep_alive o, r0, r1 + return 1 diff --git a/mypyc/test-data/lowering-int.test b/mypyc/test-data/lowering-int.test new file mode 100644 index 000000000000..e7df944c4458 --- /dev/null +++ b/mypyc/test-data/lowering-int.test @@ -0,0 +1,377 @@ +-- Test cases for converting high-level IR to lower-level IR (lowering). + +[case testLowerIntEq] +def f(x: int, y: int) -> int: + if x == y: + return 1 + else: + return 2 +[out] +def f(x, y): + x, y :: int + r0 :: native_int + r1, r2, r3 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = CPyTagged_IsEq_(x, y) + if r2 goto L3 else goto L4 :: bool +L2: + r3 = x == y + if r3 goto L3 else goto L4 :: bool +L3: + return 2 +L4: + return 4 + +[case testLowerIntNe] +def f(x: int, y: int) -> int: + if x != y: + return 1 + else: + return 2 +[out] +def f(x, y): + x, y :: int + r0 :: native_int + r1, r2, r3, r4 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = CPyTagged_IsEq_(x, y) + r3 = r2 ^ 1 + if r3 goto L3 else goto L4 :: bool +L2: + r4 = x != y + if r4 goto L3 else goto L4 :: bool +L3: + return 2 +L4: + return 4 + +[case testLowerIntEqWithConstant] +def f(x: int, y: int) -> int: + if x == 2: + return 1 + elif -1 == x: + return 2 + return 3 +[out] +def f(x, y): + x, y :: int + r0, r1 :: bit +L0: + r0 = x == 4 + if r0 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + r1 = -2 == x + if r1 goto L3 else goto L4 :: bool +L3: + return 4 +L4: + return 6 + +[case testLowerIntNeWithConstant] +def f(x: int, y: int) -> int: + if x != 2: + return 1 + elif -1 != x: + return 2 + return 3 +[out] +def f(x, y): + x, y :: int + r0, r1 :: bit +L0: + r0 = x != 4 + if r0 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + r1 = -2 != x + if r1 goto L3 else goto L4 :: bool +L3: + return 4 +L4: + return 6 + +[case testLowerIntEqValueContext] +def f(x: int, y: int) -> bool: + return x == y +[out] +def f(x, y): + x, y :: int + r0 :: native_int + r1, r2 :: bit + r3 :: bool + r4 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = CPyTagged_IsEq_(x, y) + r3 = r2 + goto L3 +L2: + r4 = x == y + r3 = r4 +L3: + return r3 + +[case testLowerIntLt] +def f(x: int, y: int) -> int: + if x < y: + return 1 + else: + return 2 +[out] +def f(x, y): + x, y :: int + r0 :: native_int + r1 :: bit + r2 :: native_int + r3, r4, r5 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L2 else goto L1 :: bool +L1: + r2 = y & 1 + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPyTagged_IsLt_(x, y) + if r4 goto L4 else goto L5 :: bool +L3: + r5 = x < y :: signed + if r5 goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 + +[case testLowerIntLe] +def f(x: int, y: int) -> int: + if x <= y: + return 1 + else: + return 2 +[out] +def f(x, y): + x, y :: int + r0 :: native_int + r1 :: bit + r2 :: native_int + r3, r4, r5, r6 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L2 else goto L1 :: bool +L1: + r2 = y & 1 + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPyTagged_IsLt_(y, x) + r5 = r4 ^ 1 + if r5 goto L4 else goto L5 :: bool +L3: + r6 = x <= y :: signed + if r6 goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 + +[case testLowerIntGt] +def f(x: int, y: int) -> int: + if x > y: + return 1 + else: + return 2 +[out] +def f(x, y): + x, y :: int + r0 :: native_int + r1 :: bit + r2 :: native_int + r3, r4, r5 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L2 else goto L1 :: bool +L1: + r2 = y & 1 + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPyTagged_IsLt_(y, x) + if r4 goto L4 else goto L5 :: bool +L3: + r5 = x > y :: signed + if r5 goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 + +[case testLowerIntGe] +def f(x: int, y: int) -> int: + if x >= y: + return 1 + else: + return 2 +[out] +def f(x, y): + x, y :: int + r0 :: native_int + r1 :: bit + r2 :: native_int + r3, r4, r5, r6 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L2 else goto L1 :: bool +L1: + r2 = y & 1 + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPyTagged_IsLt_(x, y) + r5 = r4 ^ 1 + if r5 goto L4 else goto L5 :: bool +L3: + r6 = x >= y :: signed + if r6 goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 + +[case testLowerIntLtShort] +def both() -> int: + if 3 < 5: + return 1 + else: + return 2 + +def rhs_only(x: int) -> int: + if x < 5: + return 1 + else: + return 2 + +def lhs_only(x: int) -> int: + if 5 < x: + return 1 + else: + return 2 +[out] +def both(): + r0 :: bit +L0: + r0 = 6 < 10 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 +def rhs_only(x): + x :: int + r0 :: native_int + r1 :: bit + r2 :: native_int + r3, r4, r5 :: bit +L0: + r0 = x & 1 + r1 = r0 != 0 + if r1 goto L2 else goto L1 :: bool +L1: + r2 = 10 & 1 + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPyTagged_IsLt_(x, 10) + if r4 goto L4 else goto L5 :: bool +L3: + r5 = x < 10 :: signed + if r5 goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 +def lhs_only(x): + x :: int + r0 :: native_int + r1 :: bit + r2 :: native_int + r3, r4, r5 :: bit +L0: + r0 = 10 & 1 + r1 = r0 != 0 + if r1 goto L2 else goto L1 :: bool +L1: + r2 = x & 1 + r3 = r2 != 0 + if r3 goto L2 else goto L3 :: bool +L2: + r4 = CPyTagged_IsLt_(10, x) + if r4 goto L4 else goto L5 :: bool +L3: + r5 = 10 < x :: signed + if r5 goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 + +[case testLowerIntForLoop] +from __future__ import annotations + +def f(l: list[int]) -> None: + for x in l: + pass +[out] +def f(l): + l :: list + r0 :: short_int + r1 :: ptr + r2 :: native_int + r3 :: short_int + r4 :: bit + r5 :: object + r6, x :: int + r7 :: short_int + r8 :: None +L0: + r0 = 0 +L1: + r1 = get_element_ptr l ob_size :: PyVarObject + r2 = load_mem r1 :: native_int* + r3 = r2 << 1 + r4 = r0 < r3 :: signed + if r4 goto L2 else goto L5 :: bool +L2: + r5 = CPyList_GetItemUnsafe(l, r0) + r6 = unbox(int, r5) + dec_ref r5 + if is_error(r6) goto L6 (error at f:4) else goto L3 +L3: + x = r6 + dec_ref x :: int +L4: + r7 = r0 + 2 + r0 = r7 + goto L1 +L5: + return 1 +L6: + r8 = :: None + return r8 diff --git a/mypyc/test-data/lowering-list.test b/mypyc/test-data/lowering-list.test new file mode 100644 index 000000000000..c8438d869970 --- /dev/null +++ b/mypyc/test-data/lowering-list.test @@ -0,0 +1,33 @@ +[case testLowerListDisplay] +def f() -> None: + a = [4, 6, 7] +[out] +def f(): + r0 :: list + r1, r2, r3 :: object + r4, r5, r6, r7 :: ptr + a :: list + r8 :: None +L0: + r0 = PyList_New(3) + if is_error(r0) goto L2 (error at f:2) else goto L1 +L1: + r1 = object 4 + r2 = object 6 + r3 = object 7 + r4 = get_element_ptr r0 ob_item :: PyListObject + r5 = load_mem r4 :: ptr* + inc_ref r1 + set_mem r5, r1 :: builtins.object* + inc_ref r2 + r6 = r5 + WORD_SIZE*1 + set_mem r6, r2 :: builtins.object* + inc_ref r3 + r7 = r5 + WORD_SIZE*2 + set_mem r7, r3 :: builtins.object* + a = r0 + dec_ref a + return 1 +L2: + r8 = :: None + return r8 diff --git a/mypyc/test-data/opt-copy-propagation.test b/mypyc/test-data/opt-copy-propagation.test new file mode 100644 index 000000000000..49b80f4385fc --- /dev/null +++ b/mypyc/test-data/opt-copy-propagation.test @@ -0,0 +1,400 @@ +-- Test cases for copy propagation optimization. This also tests IR transforms in general, +-- as copy propagation was the first IR transform that was implemented. + +[case testCopyPropagationSimple] +def g() -> int: + return 1 + +def f() -> int: + y = g() + return y +[out] +def g(): +L0: + return 2 +def f(): + r0 :: int +L0: + r0 = g() + return r0 + +[case testCopyPropagationChain] +def f(x: int) -> int: + y = x + z = y + return z +[out] +def f(x): + x :: int +L0: + return x + +[case testCopyPropagationChainPartial] +def f(x: int) -> int: + y = x + z = y + x = 2 + return z +[out] +def f(x): + x, y :: int +L0: + y = x + x = 4 + return y + +[case testCopyPropagationChainBad] +def f(x: int) -> int: + y = x + z = y + y = 2 + return z +[out] +def f(x): + x, y, z :: int +L0: + y = x + z = y + y = 4 + return z + +[case testCopyPropagationMutatedSource1] +def f(x: int) -> int: + y = x + x = 1 + return y +[out] +def f(x): + x, y :: int +L0: + y = x + x = 2 + return y + +[case testCopyPropagationMutatedSource2] +def f() -> int: + z = 1 + y = z + z = 2 + return y +[out] +def f(): + z, y :: int +L0: + z = 2 + y = z + z = 4 + return y + +[case testCopyPropagationTooComplex] +def f(b: bool, x: int) -> int: + if b: + y = x + return y + else: + y = 1 + return y +[out] +def f(b, x): + b :: bool + x, y :: int +L0: + if b goto L1 else goto L2 :: bool +L1: + y = x + return y +L2: + y = 2 + return y + +[case testCopyPropagationArg] +def f(x: int) -> int: + x = 2 + return x +[out] +def f(x): + x :: int +L0: + x = 4 + return x + +[case testCopyPropagationPartiallyDefined1] +def f(b: bool) -> int: + if b: + x = 1 + y = x + return y +[out] +def f(b): + b :: bool + r0, x :: int + r1 :: bool + y :: int +L0: + r0 = :: int + x = r0 + if b goto L1 else goto L2 :: bool +L1: + x = 2 +L2: + if is_error(x) goto L3 else goto L4 +L3: + r1 = raise UnboundLocalError('local variable "x" referenced before assignment') + unreachable +L4: + y = x + return y + +-- The remaining test cases test basic IRTransform functionality and are not +-- all needed for testing copy propagation as such. + +[case testIRTransformBranch] +from mypy_extensions import i64 + +def f(x: bool) -> int: + y = x + if y: + return 1 + else: + return 2 +[out] +def f(x): + x :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 + +[case testIRTransformAssignment] +def f(b: bool, x: int) -> int: + y = x + if b: + return y + else: + return 1 +[out] +def f(b, x): + b :: bool + x :: int +L0: + if b goto L1 else goto L2 :: bool +L1: + return x +L2: + return 2 + +[case testIRTransformRegisterOps1] +from __future__ import annotations +from typing import cast + +class C: + a: int + + def m(self, x: int) -> None: pass + +def get_attr(x: C) -> int: + y = x + return y.a + +def set_attr(x: C) -> None: + y = x + y.a = 1 + +def tuple_get(x: tuple[int, int]) -> int: + y = x + return y[0] + +def tuple_set(x: int, xx: int) -> tuple[int, int]: + y = x + z = xx + return y, z + +def call(x: int) -> int: + y = x + return call(y) + +def method_call(c: C, x: int) -> None: + y = x + c.m(y) + +def cast_op(x: object) -> str: + y = x + return cast(str, y) + +def box(x: int) -> object: + y = x + return y + +def unbox(x: object) -> int: + y = x + return cast(int, y) + +def call_c(x: list[str]) -> None: + y = x + y.append("x") + +def keep_alive(x: C) -> int: + y = x + return y.a + 1 +[out] +def C.m(self, x): + self :: __main__.C + x :: int +L0: + return 1 +def get_attr(x): + x :: __main__.C + r0 :: int +L0: + r0 = x.a + return r0 +def set_attr(x): + x :: __main__.C + r0 :: bool +L0: + x.a = 2; r0 = is_error + return 1 +def tuple_get(x): + x :: tuple[int, int] + r0 :: int +L0: + r0 = x[0] + return r0 +def tuple_set(x, xx): + x, xx :: int + r0 :: tuple[int, int] +L0: + r0 = (x, xx) + return r0 +def call(x): + x, r0 :: int +L0: + r0 = call(x) + return r0 +def method_call(c, x): + c :: __main__.C + x :: int + r0 :: None +L0: + r0 = c.m(x) + return 1 +def cast_op(x): + x :: object + r0 :: str +L0: + r0 = cast(str, x) + return r0 +def box(x): + x :: int + r0 :: object +L0: + r0 = box(int, x) + return r0 +def unbox(x): + x :: object + r0 :: int +L0: + r0 = unbox(int, x) + return r0 +def call_c(x): + x :: list + r0 :: str + r1 :: i32 + r2 :: bit +L0: + r0 = 'x' + r1 = PyList_Append(x, r0) + r2 = r1 >= 0 :: signed + return 1 +def keep_alive(x): + x :: __main__.C + r0, r1 :: int +L0: + r0 = borrow x.a + r1 = CPyTagged_Add(r0, 2) + keep_alive x + return r1 + +[case testIRTransformRegisterOps2] +from mypy_extensions import i32, i64 + +def truncate(x: i64) -> i32: + y = x + return i32(y) + +def extend(x: i32) -> i64: + y = x + return i64(y) + +def int_op(x: i64, xx: i64) -> i64: + y = x + z = xx + return y + z + +def comparison_op(x: i64, xx: i64) -> bool: + y = x + z = xx + return y == z + +def float_op(x: float, xx: float) -> float: + y = x + z = xx + return y + z + +def float_neg(x: float) -> float: + y = x + return -y + +def float_comparison_op(x: float, xx: float) -> bool: + y = x + z = xx + return y == z +[out] +def truncate(x): + x :: i64 + r0 :: i32 +L0: + r0 = truncate x: i64 to i32 + return r0 +def extend(x): + x :: i32 + r0 :: i64 +L0: + r0 = extend signed x: i32 to i64 + return r0 +def int_op(x, xx): + x, xx, r0 :: i64 +L0: + r0 = x + xx + return r0 +def comparison_op(x, xx): + x, xx :: i64 + r0 :: bit +L0: + r0 = x == xx + return r0 +def float_op(x, xx): + x, xx, r0 :: float +L0: + r0 = x + xx + return r0 +def float_neg(x): + x, r0 :: float +L0: + r0 = -x + return r0 +def float_comparison_op(x, xx): + x, xx :: float + r0 :: bit +L0: + r0 = x == xx + return r0 + +-- Note that transforms of these ops aren't tested here: +-- * LoadMem +-- * SetMem +-- * GetElementPtr +-- * LoadAddress +-- * Unborrow diff --git a/mypyc/test-data/opt-flag-elimination.test b/mypyc/test-data/opt-flag-elimination.test new file mode 100644 index 000000000000..337ced70a355 --- /dev/null +++ b/mypyc/test-data/opt-flag-elimination.test @@ -0,0 +1,296 @@ +-- Test cases for "flag elimination" optimization. Used to optimize away +-- registers that are always used immediately after assignment as branch conditions. + +[case testFlagEliminationSimple] +def c() -> bool: + return True +def d() -> bool: + return True + +def f(x: bool) -> int: + if x: + b = c() + else: + b = d() + if b: + return 1 + else: + return 2 +[out] +def c(): +L0: + return 1 +def d(): +L0: + return 1 +def f(x): + x, r0, r1 :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + r0 = c() + if r0 goto L3 else goto L4 :: bool +L2: + r1 = d() + if r1 goto L3 else goto L4 :: bool +L3: + return 2 +L4: + return 4 + +[case testFlagEliminationOneAssignment] +def c() -> bool: + return True + +def f(x: bool) -> int: + # Not applied here + b = c() + if b: + return 1 + else: + return 2 +[out] +def c(): +L0: + return 1 +def f(x): + x, r0, b :: bool +L0: + r0 = c() + b = r0 + if b goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 + +[case testFlagEliminationThreeCases] +def c(x: int) -> bool: + return True + +def f(x: bool, y: bool) -> int: + if x: + b = c(1) + elif y: + b = c(2) + else: + b = c(3) + if b: + return 1 + else: + return 2 +[out] +def c(x): + x :: int +L0: + return 1 +def f(x, y): + x, y, r0, r1, r2 :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + r0 = c(2) + if r0 goto L5 else goto L6 :: bool +L2: + if y goto L3 else goto L4 :: bool +L3: + r1 = c(4) + if r1 goto L5 else goto L6 :: bool +L4: + r2 = c(6) + if r2 goto L5 else goto L6 :: bool +L5: + return 2 +L6: + return 4 + +[case testFlagEliminationAssignmentNotLastOp] +def f(x: bool) -> int: + y = 0 + if x: + b = True + y = 1 + else: + b = False + if b: + return 1 + else: + return 2 +[out] +def f(x): + x :: bool + y :: int + b :: bool +L0: + y = 0 + if x goto L1 else goto L2 :: bool +L1: + b = 1 + y = 2 + goto L3 +L2: + b = 0 +L3: + if b goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 + +[case testFlagEliminationAssignmentNoDirectGoto] +def f(x: bool) -> int: + if x: + b = True + else: + b = False + if x: + if b: + return 1 + else: + return 2 + return 4 +[out] +def f(x): + x, b :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + b = 1 + goto L3 +L2: + b = 0 +L3: + if x goto L4 else goto L7 :: bool +L4: + if b goto L5 else goto L6 :: bool +L5: + return 2 +L6: + return 4 +L7: + return 8 + +[case testFlagEliminationBranchNotNextOpAfterGoto] +def f(x: bool) -> int: + if x: + b = True + else: + b = False + y = 1 # Prevents the optimization + if b: + return 1 + else: + return 2 +[out] +def f(x): + x, b :: bool + y :: int +L0: + if x goto L1 else goto L2 :: bool +L1: + b = 1 + goto L3 +L2: + b = 0 +L3: + y = 2 + if b goto L4 else goto L5 :: bool +L4: + return 2 +L5: + return 4 + +[case testFlagEliminationFlagReadTwice] +def f(x: bool) -> bool: + if x: + b = True + else: + b = False + if b: + return b # Prevents the optimization + else: + return False +[out] +def f(x): + x, b :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + b = 1 + goto L3 +L2: + b = 0 +L3: + if b goto L4 else goto L5 :: bool +L4: + return b +L5: + return 0 + +[case testFlagEliminationArgumentNotEligible] +def f(x: bool, b: bool) -> bool: + if x: + b = True + else: + b = False + if b: + return True + else: + return False +[out] +def f(x, b): + x, b :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + b = 1 + goto L3 +L2: + b = 0 +L3: + if b goto L4 else goto L5 :: bool +L4: + return 1 +L5: + return 0 + +[case testFlagEliminationFlagNotAlwaysDefined] +def f(x: bool, y: bool) -> bool: + if x: + b = True + elif y: + b = False + else: + bb = False # b not assigned here -> can't optimize + if b: + return True + else: + return False +[out] +def f(x, y): + x, y, r0, b, bb, r1 :: bool +L0: + r0 = :: bool + b = r0 + if x goto L1 else goto L2 :: bool +L1: + b = 1 + goto L5 +L2: + if y goto L3 else goto L4 :: bool +L3: + b = 0 + goto L5 +L4: + bb = 0 +L5: + if is_error(b) goto L6 else goto L7 +L6: + r1 = raise UnboundLocalError('local variable "b" referenced before assignment') + unreachable +L7: + if b goto L8 else goto L9 :: bool +L8: + return 1 +L9: + return 0 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test new file mode 100644 index 000000000000..e719ecb2afe1 --- /dev/null +++ b/mypyc/test-data/refcount.test @@ -0,0 +1,1482 @@ +-- Test cases for reference count insertion. + +[case testReturnLiteral] +def f() -> int: + return 1 +[out] +def f(): +L0: + return 2 + +[case testReturnLocal] +def f() -> int: + x = 1 + return x +[out] +def f(): + x :: int +L0: + x = 2 + return x + +[case testLocalVars] +def f() -> int: + x = 1 + y = x + x = y + return x +[out] +def f(): + x, y :: int +L0: + x = 2 + y = x + x = y + return x + +[case testLocalVars2] +def f() -> int: + x = 1 + y = x + z = x + return y + z +[out] +def f(): + x, y, z, r0 :: int +L0: + x = 2 + inc_ref x :: int + y = x + z = x + r0 = CPyTagged_Add(y, z) + dec_ref y :: int + dec_ref z :: int + return r0 + +[case testFreeAtReturn] +def f() -> int: + x = 1 + y = 2 + if x == 1: + return x + return y +[out] +def f(): + x, y :: int + r0 :: bit +L0: + x = 2 + y = 4 + r0 = int_eq x, 2 + if r0 goto L3 else goto L4 :: bool +L1: + return x +L2: + return y +L3: + dec_ref y :: int + goto L1 +L4: + dec_ref x :: int + goto L2 + +[case testArgumentsInOps] +def f(a: int, b: int) -> int: + x = a + 1 + y = x + a + return y +[out] +def f(a, b): + a, b, r0, x, r1, y :: int +L0: + r0 = CPyTagged_Add(a, 2) + x = r0 + r1 = CPyTagged_Add(x, a) + dec_ref x :: int + y = r1 + return y + +[case testArgumentsInAssign] +def f(a: int) -> int: + x = a + y = a + x = 1 + return x + y +[out] +def f(a): + a, x, y, r0 :: int +L0: + inc_ref a :: int + x = a + dec_ref x :: int + inc_ref a :: int + y = a + x = 2 + r0 = CPyTagged_Add(x, y) + dec_ref x :: int + dec_ref y :: int + return r0 + +[case testAssignToArgument1] +def f(a: int) -> int: + a = 1 + y = a + return y +[out] +def f(a): + a, y :: int +L0: + a = 2 + y = a + return y + +[case testAssignToArgument2] +def f(a: int) -> int: + a = 1 + a = 2 + a = 3 + return a +[out] +def f(a): + a :: int +L0: + a = 2 + dec_ref a :: int + a = 4 + dec_ref a :: int + a = 6 + return a + +[case testAssignToArgument3] +def f(a: int) -> int: + x = 1 + a = x + y = x + return a +[out] +def f(a): + a, x, y :: int +L0: + x = 2 + inc_ref x :: int + a = x + y = x + dec_ref y :: int + return a + +[case testReturnArgument] +def f(a: int) -> int: + return a +[out] +def f(a): + a :: int +L0: + inc_ref a :: int + return a + +[case testConditionalAssignToArgument1] +def f(a: int) -> int: + if a == a: + a = 1 + else: + x = 2 + y = a + 1 + return y +[out] +def f(a): + a :: int + r0 :: bit + x, r1, y :: int +L0: + r0 = int_eq a, a + if r0 goto L1 else goto L2 :: bool +L1: + a = 2 + goto L3 +L2: + x = 4 + dec_ref x :: int + goto L4 +L3: + r1 = CPyTagged_Add(a, 2) + dec_ref a :: int + y = r1 + return y +L4: + inc_ref a :: int + goto L3 + +[case testConditionalAssignToArgument2] +def f(a: int) -> int: + if a == a: + x = 2 + else: + a = 1 + y = a + 1 + return y +[out] +def f(a): + a :: int + r0 :: bit + x, r1, y :: int +L0: + r0 = int_eq a, a + if r0 goto L1 else goto L2 :: bool +L1: + x = 4 + dec_ref x :: int + goto L4 +L2: + a = 2 +L3: + r1 = CPyTagged_Add(a, 2) + dec_ref a :: int + y = r1 + return y +L4: + inc_ref a :: int + goto L3 + +[case testConditionalAssignToArgument3] +def f(a: int) -> int: + if a == a: + a = 1 + return a +[out] +def f(a): + a :: int + r0 :: bit +L0: + r0 = int_eq a, a + if r0 goto L1 else goto L3 :: bool +L1: + a = 2 +L2: + return a +L3: + inc_ref a :: int + goto L2 + +[case testAssignRegisterToItself] +def f(a: int) -> int: + a = a + x = 1 + x = x + return x + a +-- This is correct but bad code +[out] +def f(a): + a, x, r0 :: int +L0: + inc_ref a :: int + a = a + x = 2 + inc_ref x :: int + dec_ref x :: int + x = x + r0 = CPyTagged_Add(x, a) + dec_ref x :: int + dec_ref a :: int + return r0 + +[case testIncrement1] +def f(a: int) -> int: + a = a + 1 + x = 1 + x = x + 1 + return a + x +[out] +def f(a): + a, r0, x, r1, r2 :: int +L0: + r0 = CPyTagged_Add(a, 2) + a = r0 + x = 2 + r1 = CPyTagged_Add(x, 2) + dec_ref x :: int + x = r1 + r2 = CPyTagged_Add(a, x) + dec_ref a :: int + dec_ref x :: int + return r2 + +[case testIncrement2] +def f() -> None: + x = 1 + x = x + 1 +[out] +def f(): + x, r0 :: int +L0: + x = 2 + r0 = CPyTagged_Add(x, 2) + dec_ref x :: int + x = r0 + dec_ref x :: int + return 1 + +[case testAdd1] +def f() -> None: + y = 1 + x = y + 1 +[out] +def f(): + y, r0, x :: int +L0: + y = 2 + r0 = CPyTagged_Add(y, 2) + dec_ref y :: int + x = r0 + dec_ref x :: int + return 1 + +[case testAdd2] +def f(a: int) -> int: + a = a + a + x = a + x = x + x + return x +[out] +def f(a): + a, r0, x, r1 :: int +L0: + r0 = CPyTagged_Add(a, a) + a = r0 + x = a + r1 = CPyTagged_Add(x, x) + dec_ref x :: int + x = r1 + return x + +[case testAdd3] +def f(a: int) -> int: + x = a + a + y = x + x + return y +[out] +def f(a): + a, r0, x, r1, y :: int +L0: + r0 = CPyTagged_Add(a, a) + x = r0 + r1 = CPyTagged_Add(x, x) + dec_ref x :: int + y = r1 + return y + +[case testAdd4] +def f(a: int) -> None: + x = a + a + y = 1 + z = y + y +[out] +def f(a): + a, r0, x, y, r1, z :: int +L0: + r0 = CPyTagged_Add(a, a) + x = r0 + dec_ref x :: int + y = 2 + r1 = CPyTagged_Add(y, y) + dec_ref y :: int + z = r1 + dec_ref z :: int + return 1 + +[case testAdd5] +def f(a: int) -> None: + a = a + a + x = 1 + x = x + x +[out] +def f(a): + a, r0, x, r1 :: int +L0: + r0 = CPyTagged_Add(a, a) + a = r0 + dec_ref a :: int + x = 2 + r1 = CPyTagged_Add(x, x) + dec_ref x :: int + x = r1 + dec_ref x :: int + return 1 + +[case testReturnInMiddleOfFunction] +def f() -> int: + x = 1 + y = 2 + z = 3 + if z == z: + return z + a = 1 + return x + y - a +[out] +def f(): + x, y, z :: int + r0 :: bit + a, r1, r2 :: int +L0: + x = 2 + y = 4 + z = 6 + r0 = int_eq z, z + if r0 goto L3 else goto L4 :: bool +L1: + return z +L2: + a = 2 + r1 = CPyTagged_Add(x, y) + dec_ref x :: int + dec_ref y :: int + r2 = CPyTagged_Subtract(r1, a) + dec_ref r1 :: int + dec_ref a :: int + return r2 +L3: + dec_ref x :: int + dec_ref y :: int + goto L1 +L4: + dec_ref z :: int + goto L2 + +[case testLoop] +def f(a: int) -> int: + sum = 0 + i = 0 + while i <= a: + sum = sum + i + i = i + 1 + return sum +[out] +def f(a): + a, sum, i :: int + r0 :: bit + r1, r2 :: int +L0: + sum = 0 + i = 0 +L1: + r0 = int_le i, a + if r0 goto L2 else goto L4 :: bool +L2: + r1 = CPyTagged_Add(sum, i) + dec_ref sum :: int + sum = r1 + r2 = CPyTagged_Add(i, 2) + dec_ref i :: int + i = r2 + goto L1 +L3: + return sum +L4: + dec_ref i :: int + goto L3 + +[case testCall] +def f(a: int) -> int: + return f(a + 1) +[out] +def f(a): + a, r0, r1 :: int +L0: + r0 = CPyTagged_Add(a, 2) + r1 = f(r0) + dec_ref r0 :: int + return r1 + +[case testError] +def f(x: List[int]) -> None: pass # E: Name "List" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import List") + +[case testNewList] +def f() -> int: + a = [0, 1] + return 0 +[out] +def f(): + r0 :: list + r1, r2 :: object + r3 :: ptr + a :: list +L0: + r0 = PyList_New(2) + r1 = object 0 + r2 = object 1 + r3 = list_items r0 + inc_ref r1 + buf_init_item r3, 0, r1 + inc_ref r2 + buf_init_item r3, 1, r2 + a = r0 + dec_ref a + return 0 + +[case testListSet] +from typing import List +def f(a: List[int], b: List[int]) -> None: + a[0] = b[0] +[out] +def f(a, b): + a, b :: list + r0 :: object + r1 :: int + r2 :: object + r3 :: bit +L0: + r0 = CPyList_GetItemShort(b, 0) + r1 = unbox(int, r0) + dec_ref r0 + r2 = box(int, r1) + r3 = CPyList_SetItem(a, 0, r2) + return 1 + +[case testTupleRefcount] +from typing import Tuple +def f(x: Tuple[Tuple[int, bool], bool]) -> int: + return x[0][0] +[out] +def f(x): + x :: tuple[tuple[int, bool], bool] + r0 :: tuple[int, bool] + r1 :: int +L0: + r0 = x[0] + r1 = r0[0] + dec_ref r0 + return r1 + +[case testUserClassRefCount] +class C: + x: 'C' +def f() -> None: + c = C() + c.x = C() +[out] +def f(): + r0, c, r1 :: __main__.C + r2 :: bool +L0: + r0 = C() + c = r0 + r1 = C() + c.x = r1; r2 = is_error + dec_ref c + return 1 + +[case testCastRefCount] +class C: pass + +def f() -> None: + a = [C()] + d = a[0] +[out] +def f(): + r0 :: __main__.C + r1 :: list + r2 :: ptr + a :: list + r3 :: object + r4, d :: __main__.C +L0: + r0 = C() + r1 = PyList_New(1) + r2 = list_items r1 + buf_init_item r2, 0, r0 + a = r1 + r3 = CPyList_GetItemShort(a, 0) + dec_ref a + r4 = cast(__main__.C, r3) + d = r4 + dec_ref d + return 1 + +[case testUnaryBranchSpecialCase] +def f(x: bool) -> int: + if x: + return 1 + return 2 +[out] +def f(x): + x :: bool +L0: + if x goto L1 else goto L2 :: bool +L1: + return 2 +L2: + return 4 + +[case testReturnTuple] +from typing import Tuple + +class C: pass +def f() -> Tuple[C, C]: + a = C() + b = C() + return a, b +[out] +def f(): + r0, a, r1, b :: __main__.C + r2 :: tuple[__main__.C, __main__.C] +L0: + r0 = C() + a = r0 + r1 = C() + b = r1 + r2 = (a, b) + return r2 + +[case testDecomposeTuple] +from typing import Tuple + +class C: + a: int + +def f() -> int: + x, y = g() + return x.a + y.a + +def g() -> Tuple[C, C]: + return C(), C() +[out] +def f(): + r0 :: tuple[__main__.C, __main__.C] + r1, r2, x, r3, r4, y :: __main__.C + r5, r6, r7 :: int +L0: + r0 = g() + r1 = borrow r0[0] + r2 = unborrow r1 + x = r2 + r3 = borrow r0[1] + r4 = unborrow r3 + y = r4 + r5 = borrow x.a + r6 = borrow y.a + r7 = CPyTagged_Add(r5, r6) + dec_ref x + dec_ref y + return r7 +def g(): + r0, r1 :: __main__.C + r2 :: tuple[__main__.C, __main__.C] +L0: + r0 = C() + r1 = C() + r2 = (r0, r1) + return r2 + +[case testUnicodeLiteral] +def f() -> str: + return "some string" +[out] +def f(): + r0 :: str +L0: + r0 = 'some string' + inc_ref r0 + return r0 + +[case testPyMethodCall] +def g(x: str) -> int: + return int(x, base=2) +[out] +def g(x): + x :: str + r0 :: object + r1 :: str + r2 :: tuple + r3 :: object + r4 :: dict + r5 :: object + r6 :: int +L0: + r0 = load_address PyLong_Type + r1 = 'base' + r2 = PyTuple_Pack(1, x) + r3 = object 2 + r4 = CPyDict_Build(1, r1, r3) + r5 = PyObject_Call(r0, r2, r4) + dec_ref r2 + dec_ref r4 + r6 = unbox(int, r5) + dec_ref r5 + return r6 + +[case testListAppend] +from typing import List +def f(a: List[int], x: int) -> None: + a.append(x) +[out] +def f(a, x): + a :: list + x :: int + r0 :: object + r1 :: i32 + r2 :: bit +L0: + inc_ref x :: int + r0 = box(int, x) + r1 = PyList_Append(a, r0) + dec_ref r0 + r2 = r1 >= 0 :: signed + return 1 + +[case testForDict] +from typing import Dict + +def f(d: Dict[int, int]) -> None: + for key in d: + d[key] +[out] +def f(d): + d :: dict + r0 :: short_int + r1 :: native_int + r2 :: short_int + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, key :: int + r9, r10 :: object + r11 :: int + r12, r13 :: bit +L0: + r0 = 0 + r1 = PyDict_Size(d) + r2 = r1 << 1 + r3 = CPyDict_GetKeysIter(d) +L1: + r4 = CPyDict_NextKey(r3, r0) + r5 = r4[1] + r0 = r5 + r6 = r4[0] + if r6 goto L2 else goto L6 :: bool +L2: + r7 = r4[2] + dec_ref r4 + r8 = unbox(int, r7) + dec_ref r7 + key = r8 + r9 = box(int, key) + r10 = CPyDict_GetItem(d, r9) + dec_ref r9 + r11 = unbox(int, r10) + dec_ref r10 + dec_ref r11 :: int +L3: + r12 = CPyDict_CheckSize(d, r2) + goto L1 +L4: + r13 = CPy_NoErrOccured() +L5: + return 1 +L6: + dec_ref r3 + dec_ref r4 + goto L4 + +[case testBorrowRefs] +def make_garbage(arg: object) -> None: + b = True + while b: + arg = None + b = False +[out] +def make_garbage(arg): + arg :: object + b :: bool + r0 :: object +L0: + b = 1 +L1: + if b goto L2 else goto L3 :: bool +L2: + r0 = box(None, 1) + inc_ref r0 + arg = r0 + dec_ref arg + b = 0 + goto L1 +L3: + return 1 + +[case testGetElementPtrLifeTime] +from typing import List + +def f() -> int: + x: List[str] = [] + return len(x) +[out] +def f(): + r0, x :: list + r1 :: native_int + r2 :: short_int +L0: + r0 = PyList_New(0) + x = r0 + r1 = var_object_size x + dec_ref x + r2 = r1 << 1 + return r2 + +[case testSometimesUninitializedVariable] +def f(x: bool) -> int: + if x: + y = 1 + else: + z = 2 + return y + z +[out] +def f(x): + x :: bool + r0, y, r1, z :: int + r2, r3 :: bool + r4 :: int +L0: + r0 = :: int + y = r0 + r1 = :: int + z = r1 + if x goto L8 else goto L9 :: bool +L1: + y = 2 + goto L3 +L2: + z = 4 +L3: + if is_error(y) goto L10 else goto L5 +L4: + r2 = raise UnboundLocalError('local variable "y" referenced before assignment') + unreachable +L5: + if is_error(z) goto L11 else goto L7 +L6: + r3 = raise UnboundLocalError('local variable "z" referenced before assignment') + unreachable +L7: + r4 = CPyTagged_Add(y, z) + xdec_ref y :: int + xdec_ref z :: int + return r4 +L8: + xdec_ref y :: int + goto L1 +L9: + xdec_ref z :: int + goto L2 +L10: + xdec_ref z :: int + goto L4 +L11: + xdec_ref y :: int + goto L6 + +[case testVectorcall_python3_8] +from typing import Any + +def call(f: Any, x: int) -> int: + return f(x) +[out] +def call(f, x): + f :: object + x :: int + r0 :: object + r1 :: object[1] + r2 :: object_ptr + r3 :: object + r4 :: int +L0: + inc_ref x :: int + r0 = box(int, x) + r1 = [r0] + r2 = load_address r1 + r3 = _PyObject_Vectorcall(f, r2, 1, 0) + dec_ref r0 + r4 = unbox(int, r3) + dec_ref r3 + return r4 + +[case testVectorcallMethod_python3_9_64bit] +from typing import Any + +def call(o: Any, x: int) -> int: + return o.m(x) +[out] +def call(o, x): + o :: object + x :: int + r0 :: str + r1 :: object + r2 :: object[2] + r3 :: object_ptr + r4 :: object + r5 :: int +L0: + r0 = 'm' + inc_ref x :: int + r1 = box(int, x) + r2 = [o, r1] + r3 = load_address r2 + r4 = PyObject_VectorcallMethod(r0, r3, 9223372036854775810, 0) + dec_ref r1 + r5 = unbox(int, r4) + dec_ref r4 + return r5 + +[case testBorrowAttribute] +def g() -> int: + d = D() + return d.c.x + +def f(d: D) -> int: + return d.c.x + +class C: + x: int +class D: + c: C +[out] +def g(): + r0, d :: __main__.D + r1 :: __main__.C + r2 :: int +L0: + r0 = D() + d = r0 + r1 = borrow d.c + r2 = r1.x + dec_ref d + return r2 +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.x + return r1 + +[case testBorrowAttributeTwice] +def f(e: E) -> int: + return e.d.c.x + +class C: + x: int +class D: + c: C +class E: + d: D +[out] +def f(e): + e :: __main__.E + r0 :: __main__.D + r1 :: __main__.C + r2 :: int +L0: + r0 = borrow e.d + r1 = borrow r0.c + r2 = r1.x + return r2 + +[case testBorrowAttributeIsNone] +from typing import Optional + +def f(c: C) -> bool: + return c.x is not None + +def g(c: C) -> bool: + return c.x is None + +class C: + x: Optional[str] +[out] +def f(c): + c :: __main__.C + r0 :: union[str, None] + r1 :: object + r2 :: bit +L0: + r0 = borrow c.x + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + return r2 +def g(c): + c :: __main__.C + r0 :: union[str, None] + r1 :: object + r2 :: bit +L0: + r0 = borrow c.x + r1 = load_address _Py_NoneStruct + r2 = r0 == r1 + return r2 + +[case testBorrowAttributeNarrowOptional] +from typing import Optional + +def f(c: C) -> bool: + if c.x is not None: + return c.x.b + return False + +class C: + x: Optional[D] + +class D: + b: bool +[out] +def f(c): + c :: __main__.C + r0 :: union[__main__.D, None] + r1 :: object + r2 :: bit + r3 :: union[__main__.D, None] + r4 :: __main__.D + r5 :: bool +L0: + r0 = borrow c.x + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = borrow c.x + r4 = borrow cast(__main__.D, r3) + r5 = r4.b + return r5 +L2: + return 0 + +[case testBorrowLenArgument] +from typing import List + +def f(x: C) -> int: + return len(x.a) + +class C: + a: List[str] +[out] +def f(x): + x :: __main__.C + r0 :: list + r1 :: native_int + r2 :: short_int +L0: + r0 = borrow x.a + r1 = var_object_size r0 + r2 = r1 << 1 + return r2 + +[case testBorrowIsinstanceArgument] +from typing import List + +def f(x: C) -> bool: + if isinstance(x.a, D): + return x.a.b + else: + return True + +class C: + a: object + +class D: + b: bool +[out] +def f(x): + x :: __main__.C + r0, r1 :: object + r2 :: ptr + r3 :: object + r4 :: bit + r5 :: object + r6 :: __main__.D + r7 :: bool +L0: + r0 = borrow x.a + r1 = __main__.D :: type + r2 = get_element_ptr r0 ob_type :: PyObject + r3 = load_mem r2 :: builtins.object* + r4 = r3 == r1 + if r4 goto L1 else goto L2 :: bool +L1: + r5 = borrow x.a + r6 = borrow cast(__main__.D, r5) + r7 = r6.b + return r7 +L2: + return 1 + +[case testBorrowListGetItem1] +from typing import List + +def literal_index(x: C) -> str: + return x.a[0] + +def negative_index(x: C) -> str: + return x.a[-1] + +def lvar_index(x: C, n: int) -> str: + return x.a[n] + +class C: + a: List[str] + +[out] +def literal_index(x): + x :: __main__.C + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = borrow x.a + r1 = CPyList_GetItemShort(r0, 0) + r2 = cast(str, r1) + return r2 +def negative_index(x): + x :: __main__.C + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = borrow x.a + r1 = CPyList_GetItemShort(r0, -2) + r2 = cast(str, r1) + return r2 +def lvar_index(x, n): + x :: __main__.C + n :: int + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = borrow x.a + r1 = CPyList_GetItem(r0, n) + r2 = cast(str, r1) + return r2 + +[case testBorrowListGetItem2] +from typing import List + +def attr_before_index(x: C) -> str: + return x.a[x.n] + +def attr_after_index(a: List[C], i: int) -> int: + return a[i].n + +def attr_after_index_literal(a: List[C]) -> int: + return a[0].n + +class C: + a: List[str] + n: int +[out] +def attr_before_index(x): + x :: __main__.C + r0 :: list + r1 :: int + r2 :: object + r3 :: str +L0: + r0 = borrow x.a + r1 = borrow x.n + r2 = CPyList_GetItem(r0, r1) + r3 = cast(str, r2) + return r3 +def attr_after_index(a, i): + a :: list + i :: int + r0 :: object + r1 :: __main__.C + r2 :: int +L0: + r0 = CPyList_GetItemBorrow(a, i) + r1 = borrow cast(__main__.C, r0) + r2 = r1.n + return r2 +def attr_after_index_literal(a): + a :: list + r0 :: object + r1 :: __main__.C + r2 :: int +L0: + r0 = CPyList_GetItemShortBorrow(a, 0) + r1 = borrow cast(__main__.C, r0) + r2 = r1.n + return r2 + +[case testCannotBorrowListGetItem] +from typing import List + +def func_index(x: C) -> str: + return x.a[f()] + +def f() -> int: return 0 + +class C: + a: List[str] +[out] +def func_index(x): + x :: __main__.C + r0 :: list + r1 :: int + r2 :: object + r3 :: str +L0: + r0 = x.a + r1 = f() + r2 = CPyList_GetItem(r0, r1) + dec_ref r0 + dec_ref r1 :: int + r3 = cast(str, r2) + return r3 +def f(): +L0: + return 0 + +[case testBorrowListGetItemKeepAlive] +from typing import List + +def f() -> str: + a = [C()] + return a[0].s + +class C: + s: str +[out] +def f(): + r0 :: __main__.C + r1 :: list + r2 :: ptr + a :: list + r3 :: object + r4 :: __main__.C + r5 :: str +L0: + r0 = C() + r1 = PyList_New(1) + r2 = list_items r1 + buf_init_item r2, 0, r0 + a = r1 + r3 = CPyList_GetItemShortBorrow(a, 0) + r4 = borrow cast(__main__.C, r3) + r5 = r4.s + dec_ref a + return r5 + +[case testBorrowSetAttrObject] +from typing import Optional + +def f(x: Optional[C]) -> None: + if x is not None: + x.b = True + +def g(x: D) -> None: + x.c.b = False + +class C: + b: bool + +class D: + c: C +[out] +def f(x): + x :: union[__main__.C, None] + r0 :: object + r1 :: bit + r2 :: __main__.C + r3 :: bool +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = borrow cast(__main__.C, x) + r2.b = 1; r3 = is_error +L2: + return 1 +def g(x): + x :: __main__.D + r0 :: __main__.C + r1 :: bool +L0: + r0 = borrow x.c + r0.b = 0; r1 = is_error + return 1 + +[case testBorrowIntEquality] +def add(c: C) -> bool: + return c.x == c.y + +class C: + x: int + y: int +[out] +def add(c): + c :: __main__.C + r0, r1 :: int + r2 :: bit +L0: + r0 = borrow c.x + r1 = borrow c.y + r2 = int_eq r0, r1 + return r2 + +[case testBorrowIntLessThan] +def add(c: C) -> bool: + return c.x < c.y + +class C: + x: int + y: int +[out] +def add(c): + c :: __main__.C + r0, r1 :: int + r2 :: bit +L0: + r0 = borrow c.x + r1 = borrow c.y + r2 = int_lt r0, r1 + return r2 + +[case testBorrowIntCompareFinal] +from typing_extensions import Final + +X: Final = 10 + +def add(c: C) -> bool: + return c.x == X + +class C: + x: int +[out] +def add(c): + c :: __main__.C + r0 :: int + r1 :: bit +L0: + r0 = borrow c.x + r1 = int_eq r0, 20 + return r1 + +[case testBorrowIntArithmetic] +def add(c: C) -> int: + return c.x + c.y + +def sub(c: C) -> int: + return c.x - c.y + +class C: + x: int + y: int +[out] +def add(c): + c :: __main__.C + r0, r1, r2 :: int +L0: + r0 = borrow c.x + r1 = borrow c.y + r2 = CPyTagged_Add(r0, r1) + return r2 +def sub(c): + c :: __main__.C + r0, r1, r2 :: int +L0: + r0 = borrow c.x + r1 = borrow c.y + r2 = CPyTagged_Subtract(r0, r1) + return r2 + +[case testBorrowIntComparisonInIf] +def add(c: C, n: int) -> bool: + if c.x == c.y: + return True + return False + +class C: + x: int + y: int +[out] +def add(c, n): + c :: __main__.C + n, r0, r1 :: int + r2 :: bit +L0: + r0 = borrow c.x + r1 = borrow c.y + r2 = int_eq r0, r1 + if r2 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + +[case testBorrowIntInPlaceOp] +def add(c: C, n: int) -> None: + c.x += n + +def sub(c: C, n: int) -> None: + c.x -= c.y + +class C: + x: int + y: int +[out] +def add(c, n): + c :: __main__.C + n, r0, r1 :: int + r2 :: bool +L0: + r0 = borrow c.x + r1 = CPyTagged_Add(r0, n) + c.x = r1; r2 = is_error + return 1 +def sub(c, n): + c :: __main__.C + n, r0, r1, r2 :: int + r3 :: bool +L0: + r0 = borrow c.x + r1 = borrow c.y + r2 = CPyTagged_Subtract(r0, r1) + c.x = r2; r3 = is_error + return 1 + +[case testCoerceIntToI64_64bit] +from mypy_extensions import i64 + +def f(x: int) -> i64: + # TODO: On the fast path we shouldn't have a decref. Once we have high-level IR, + # coercion from int to i64 can be a single op, which makes it easier to + # generate optimal refcount handling for this case. + return x + 1 +[out] +def f(x): + x, r0 :: int + r1 :: native_int + r2 :: bit + r3, r4 :: i64 + r5 :: ptr + r6 :: c_ptr + r7 :: i64 +L0: + r0 = CPyTagged_Add(x, 2) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = r0 >> 1 + dec_ref r0 :: int + r4 = r3 + goto L3 +L2: + r5 = r0 ^ 1 + r6 = r5 + r7 = CPyLong_AsInt64(r6) + r4 = r7 + dec_ref r0 :: int +L3: + return r4 diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test new file mode 100644 index 000000000000..8488632e6574 --- /dev/null +++ b/mypyc/test-data/run-async.test @@ -0,0 +1,173 @@ +# async test cases (compile and run) + +[case testAsync] +import asyncio + +async def h() -> int: + return 1 + +async def g() -> int: + await asyncio.sleep(0) + return await h() + +async def f() -> int: + return await g() + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] + +[file driver.py] +from native import f +import asyncio + +result = asyncio.run(f()) +assert result == 1 + +[case testAsyncWith] +from testutil import async_val + +class async_ctx: + async def __aenter__(self) -> str: + await async_val("enter") + return "test" + + async def __aexit__(self, x, y, z) -> None: + await async_val("exit") + + +async def async_with() -> str: + async with async_ctx() as x: + return await async_val("body") + + +[file driver.py] +from native import async_with +from testutil import run_generator + +yields, val = run_generator(async_with(), [None, 'x', None]) +assert yields == ('enter', 'body', 'exit'), yields +assert val == 'x', val + + +[case testAsyncReturn] +from testutil import async_val + +async def async_return() -> str: + try: + return 'test' + finally: + await async_val('foo') + +[file driver.py] +from native import async_return +from testutil import run_generator + +yields, val = run_generator(async_return()) +assert yields == ('foo',) +assert val == 'test', val + + +[case testAsyncFor] +from typing import AsyncIterable, List, Set, Dict + +async def async_iter(xs: AsyncIterable[int]) -> List[int]: + ys = [] + async for x in xs: + ys.append(x) + return ys + +async def async_comp(xs: AsyncIterable[int]) -> List[int]: + ys = [x async for x in xs] + return ys + +async def async_comp_set(xs: AsyncIterable[int]) -> Set[int]: + return {x async for x in xs} + +async def async_comp_dict(xs: AsyncIterable[int]) -> Dict[int, str]: + return {x: str(x) async for x in xs} + +[typing fixtures/typing-full.pyi] + +[file driver.py] +from native import async_iter, async_comp, async_comp_set, async_comp_dict +from testutil import run_generator, async_val +from typing import AsyncIterable, List + +# defined here since we couldn't do it inside the test yet... +async def foo() -> AsyncIterable[int]: + for x in range(3): + await async_val(x) + yield x + +yields, val = run_generator(async_iter(foo())) +assert val == [0,1,2], val +assert yields == (0,1,2), yields + +yields, val = run_generator(async_comp(foo())) +assert val == [0,1,2], val +assert yields == (0,1,2), yields + +yields, val = run_generator(async_comp_set(foo())) +assert val == {0,1,2}, val +assert yields == (0,1,2), yields + +yields, val = run_generator(async_comp_dict(foo())) +assert val == {0: '0',1: '1', 2: '2'}, val +assert yields == (0,1,2), yields + +[case testAsyncFor2] +from typing import AsyncIterable, List + +async def async_iter(xs: AsyncIterable[int]) -> List[int]: + ys = [] + async for x in xs: + ys.append(x) + return ys + +[typing fixtures/typing-full.pyi] + +[file driver.py] +from native import async_iter +from testutil import run_generator, async_val +from typing import AsyncIterable, List + +# defined here since we couldn't do it inside the test yet... +async def foo() -> AsyncIterable[int]: + for x in range(3): + await async_val(x) + yield x + raise Exception('lol no') + +yields, val = run_generator(async_iter(foo())) +assert yields == (0,1,2), yields +assert val == 'lol no', val + +[case testAsyncWithVarReuse] +class ConMan: + async def __aenter__(self) -> int: + return 1 + async def __aexit__(self, *exc: object): + pass + +class ConManB: + async def __aenter__(self) -> int: + return 2 + async def __aexit__(self, *exc: object): + pass + +async def x() -> None: + value = 2 + async with ConMan() as f: + value += f + assert value == 3, value + async with ConManB() as f: + value += f + assert value == 5, value + +[typing fixtures/typing-full.pyi] +[file driver.py] +import asyncio +import native +asyncio.run(native.x()) diff --git a/mypyc/test-data/run-attrs.test b/mypyc/test-data/run-attrs.test new file mode 100644 index 000000000000..9c402a3eea7c --- /dev/null +++ b/mypyc/test-data/run-attrs.test @@ -0,0 +1,318 @@ +-- Test cases for dataclasses based on the attrs library, where auto_attribs=True + +[case testRunAttrsclass] +import attr +from typing import Set, List, Callable, Any + +@attr.s(auto_attribs=True) +class Person1: + age : int + name : str + + def __bool__(self) -> bool: + return self.name == 'robot' + +def testBool(p: Person1) -> bool: + if p: + return True + else: + return False + +@attr.s(auto_attribs=True) +class Person1b(Person1): + id: str = '000' + +@attr.s(auto_attribs=True) +class Person2: + age : int + name : str = attr.ib(default='robot') + +@attr.s(auto_attribs=True, order=True) +class Person3: + age : int = attr.ib(default = 6) + friendIDs : List[int] = attr.ib(factory = list) + + def get_age(self) -> int: + return (self.age) + + def set_age(self, new_age : int) -> None: + self.age = new_age + + def add_friendID(self, fid : int) -> None: + self.friendIDs.append(fid) + + def get_friendIDs(self) -> List[int]: + return self.friendIDs + +def get_next_age(g: Callable[[Any], int]) -> Callable[[Any], int]: + def f(a: Any) -> int: + return g(a) + 1 + return f + +@attr.s(auto_attribs=True) +class Person4: + age : int + _name : str = 'Bot' + + @get_next_age + def get_age(self) -> int: + return self.age + + @property + def name(self) -> str: + return self._name + +@attr.s(auto_attribs=True) +class Point: + x : int = attr.ib(converter=int) + y : int = attr.ib(init=False) + + def __attrs_post_init__(self): + self.y = self.x + 1 + + +[file other.py] +from native import Person1, Person1b, Person2, Person3, Person4, testBool, Point +i1 = Person1(age = 5, name = 'robot') +assert i1.age == 5 +assert i1.name == 'robot' +assert testBool(i1) == True +assert testBool(Person1(age = 5, name = 'robo')) == False +i1b = Person1b(age = 5, name = 'robot') +assert i1b.age == 5 +assert i1b.name == 'robot' +assert i1b.id == '000' +assert testBool(i1b) == True +assert testBool(Person1b(age = 5, name = 'robo')) == False +i1c = Person1b(age = 20, name = 'robot', id = 'test') +assert i1c.age == 20 +assert i1c.id == 'test' + +i2 = Person2(age = 5) +assert i2.age == 5 +assert i2.name == 'robot' +i3 = Person2(age = 5, name = 'new_robot') +assert i3.age == 5 +assert i3.name == 'new_robot' +i4 = Person3() +assert i4.age == 6 +assert i4.friendIDs == [] +i5 = Person3(age = 5) +assert i5.age == 5 +assert i5.friendIDs == [] +i6 = Person3(age = 5, friendIDs = [1,2,3]) +assert i6.age == 5 +assert i6.friendIDs == [1,2,3] +assert i6.get_age() == 5 +i6.set_age(10) +assert i6.get_age() == 10 +i6.add_friendID(4) +assert i6.get_friendIDs() == [1,2,3,4] +i7 = Person4(age = 5) +assert i7.get_age() == 6 +i7.age += 3 +assert i7.age == 8 +assert i7.name == 'Bot' +i8 = Person3(age = 1, friendIDs = [1,2]) +i9 = Person3(age = 1, friendIDs = [1,2]) +assert i8 == i9 +i8.age = 2 +assert i8 > i9 + +assert Person1.__annotations__ == {'age': int, 'name': str} +assert Person2.__annotations__ == {'age': int, 'name': str} + +p1 = Point(2) +assert p1.x == 2 +assert p1.y == 3 +p2 = Point('2') +assert p2.x == 2 +assert p2.y == 3 + +assert Point.__annotations__ == {'x': int, 'y': int} + +[file driver.py] +import sys + +# PEP 526 introduced in 3.6 +version = sys.version_info[:2] +if version[0] < 3 or version[1] < 6: + exit() + +# Run the tests in both interpreted and compiled mode +import other +import other_interpreted + +# Test for an exceptional cases +from testutil import assertRaises +from native import Person1, Person1b, Person3 +from types import BuiltinMethodType + +with assertRaises(TypeError, "missing 1 required positional argument"): + Person1(0) + +with assertRaises(TypeError, "missing 2 required positional arguments"): + Person1b() + +with assertRaises(TypeError, "int object expected; got str"): + Person1('nope', 'test') + +p = Person1(0, 'test') +with assertRaises(TypeError, "int object expected; got str"): + p.age = 'nope' + +assert isinstance(Person3().get_age, BuiltinMethodType) + + +[case testRunAttrsclassNonAuto] +import attr +from typing import Set, List, Callable, Any + +@attr.s +class Person1: + age = attr.ib(type=int) + name = attr.ib(type=str) + + def __bool__(self) -> bool: + return self.name == 'robot' + +def testBool(p: Person1) -> bool: + if p: + return True + else: + return False + +@attr.s +class Person1b(Person1): + id = attr.ib(type=str, default='000') + +@attr.s +class Person2: + age = attr.ib(type=int) + name = attr.ib(type=str, default='robot') + +@attr.s(order=True) +class Person3: + age = attr.ib(type=int, default=6) + friendIDs = attr.ib(factory=list, type=List[int]) + + def get_age(self) -> int: + return (self.age) + + def set_age(self, new_age : int) -> None: + self.age = new_age + + def add_friendID(self, fid : int) -> None: + self.friendIDs.append(fid) + + def get_friendIDs(self) -> List[int]: + return self.friendIDs + +def get_next_age(g: Callable[[Any], int]) -> Callable[[Any], int]: + def f(a: Any) -> int: + return g(a) + 1 + return f + +@attr.s +class Person4: + age = attr.ib(type=int) + _name = attr.ib(type=str, default='Bot') + + @get_next_age + def get_age(self) -> int: + return self.age + + @property + def name(self) -> str: + return self._name + +@attr.s +class Point: + x = attr.ib(type=int, converter=int) + y = attr.ib(type=int, init=False) + + def __attrs_post_init__(self): + self.y = self.x + 1 + + +[file other.py] +from native import Person1, Person1b, Person2, Person3, Person4, testBool, Point +i1 = Person1(age = 5, name = 'robot') +assert i1.age == 5 +assert i1.name == 'robot' +assert testBool(i1) == True +assert testBool(Person1(age = 5, name = 'robo')) == False +i1b = Person1b(age = 5, name = 'robot') +assert i1b.age == 5 +assert i1b.name == 'robot' +assert i1b.id == '000' +assert testBool(i1b) == True +assert testBool(Person1b(age = 5, name = 'robo')) == False +i1c = Person1b(age = 20, name = 'robot', id = 'test') +assert i1c.age == 20 +assert i1c.id == 'test' + +i2 = Person2(age = 5) +assert i2.age == 5 +assert i2.name == 'robot' +i3 = Person2(age = 5, name = 'new_robot') +assert i3.age == 5 +assert i3.name == 'new_robot' +i4 = Person3() +assert i4.age == 6 +assert i4.friendIDs == [] +i5 = Person3(age = 5) +assert i5.age == 5 +assert i5.friendIDs == [] +i6 = Person3(age = 5, friendIDs = [1,2,3]) +assert i6.age == 5 +assert i6.friendIDs == [1,2,3] +assert i6.get_age() == 5 +i6.set_age(10) +assert i6.get_age() == 10 +i6.add_friendID(4) +assert i6.get_friendIDs() == [1,2,3,4] +i7 = Person4(age = 5) +assert i7.get_age() == 6 +i7.age += 3 +assert i7.age == 8 +assert i7.name == 'Bot' +i8 = Person3(age = 1, friendIDs = [1,2]) +i9 = Person3(age = 1, friendIDs = [1,2]) +assert i8 == i9 +i8.age = 2 +assert i8 > i9 + +p1 = Point(2) +assert p1.x == 2 +assert p1.y == 3 +p2 = Point('2') +assert p2.x == 2 +assert p2.y == 3 + +[file driver.py] +import sys + +# Run the tests in both interpreted and compiled mode +import other +import other_interpreted + +# Test for an exceptional cases +from testutil import assertRaises +from native import Person1, Person1b, Person3 +from types import BuiltinMethodType + +with assertRaises(TypeError, "missing 1 required positional argument"): + Person1(0) + +with assertRaises(TypeError, "missing 2 required positional arguments"): + Person1b() + +with assertRaises(TypeError, "int object expected; got str"): + Person1('nope', 'test') + +p = Person1(0, 'test') +with assertRaises(TypeError, "int object expected; got str"): + p.age = 'nope' + +assert isinstance(Person3().get_age, BuiltinMethodType) diff --git a/mypyc/test-data/run-bench.test b/mypyc/test-data/run-bench.test new file mode 100644 index 000000000000..98dfd26ed3d1 --- /dev/null +++ b/mypyc/test-data/run-bench.test @@ -0,0 +1,196 @@ +-- TODO: build some generic benchmark harness +[case testBenchmarkTree] +from typing import Optional +class Node: + def __init__(self, value: int) -> None: + self.value = value + self.left = None # type: Optional[Node] + self.right = None # type: Optional[Node] + def sum(self) -> int: + left = 0 + if self.left is not None: + left = self.left.sum() + right = 0 + if self.right is not None: + right = self.right.sum() + return self.value + left + right +def sum_tree(x: Optional[Node]) -> int: + if x is None: + return 0 + return x.value + sum_tree(x.left) + sum_tree(x.right) +def build(n: int) -> Optional[Node]: + if n == 0: + return None + x = Node(n) + x.left = build(n - 1) + x.right = x.left + return x + +def bench_sum(x: Optional[Node]) -> None: + for i in range(1000000): + sum_tree(x) +def bench_sum_method(x: Node) -> None: + for i in range(1000000): + x.sum() +[file driver.py] +from typing import Optional +import native +import interpreted +from timeit import timeit +from time import time +import os + +def dumb_time(f): + t0 = time() + f() + t1 = time() + return t1 - t0 + +def basic_test(m): + tree = m.build(5) + assert(m.sum_tree(tree) == 57) + assert(tree.sum() == 57) + return tree + +def test(m): + tree = basic_test(m) + + g = {**globals(), **locals()} + sum = timeit('m.sum_tree(tree)', globals=g) + sum2 = timeit('tree.sum()', globals=g) + fsum = dumb_time(lambda: m.bench_sum(tree)) + fsum2 = dumb_time(lambda: m.bench_sum_method(tree)) + build = timeit('m.build(5)', globals=g) + return (sum, sum2, fsum, fsum2, build) + +# Basic functionality test +basic_test(native) + +# Benchmark if we are benchmarking +if os.environ.get('MYPYC_RUN_BENCH') == '1': + nsum, nsum2, nfsum, nfsum2, nbuild = test(native) + isum, isum2, ifsum, ifsum2, ibuild = test(interpreted) + print(nsum, nsum2, nfsum, nbuild) + print("Sum speedup:", isum/nsum) + print("Sum method speedup:", isum2/nsum2) + print("Sum (fast) speedup:", ifsum/nfsum) + print("Sum (fast) method speedup:", ifsum2/nfsum2) + print("Build speedup:", ibuild/nbuild) + +[case testBenchmarkVisitorTree] +from mypy_extensions import trait +from typing import cast, Generic, TypeVar, Any + +T = TypeVar('T') +class Tree: + def accept(self, v: 'TreeVisitor[T]') -> T: + pass +class Leaf(Tree): + def accept(self, v: 'TreeVisitor[T]') -> T: + return v.visit_leaf(self) +class Node(Tree): + def __init__(self, value: int, left: Tree, right: Tree) -> None: + self.value = value + self.left = left + self.right = right + def accept(self, v: 'TreeVisitor[T]') -> T: + return v.visit_node(self) + +@trait +class TreeVisitor(Generic[T]): + def visit_leaf(self, x: Leaf) -> T: return cast(T, None) + def visit_node(self, x: Node) -> T: return cast(T, None) + +class SumVisitor(TreeVisitor[int]): + def sum(self, x: Tree) -> int: + return x.accept(self) + def visit_leaf(self, x: Leaf) -> int: + return 0 + def visit_node(self, x: Node) -> int: + return x.value + self.sum(x.left) + self.sum(x.right) + +def equal(x: Tree, y: Tree) -> bool: + return EqualVisitor(x).equal(y) + +class EqualVisitor(TreeVisitor[bool]): + def __init__(self, left: Tree) -> None: + self.left = left + def equal(self, right: Tree) -> bool: + return right.accept(self) + def visit_leaf(self, right: Leaf) -> bool: + return isinstance(self.left, Leaf) + def visit_node(self, right: Node) -> bool: + if isinstance(self.left, Node): + # our boolean stuff is crap + if (self.left.value == right.value and equal(self.left.left, right.left) + and equal(self.left.right, right.right)): + return True + return False + +def sum_tree(x: Tree) -> int: + return SumVisitor().sum(x) + +def build(n: int) -> Tree: + if n == 0: + return Leaf() + return Node(n, build(n - 1), build(n - 1)) + +def bench_sum_tree(x: Tree) -> None: + for i in range(100000): + sum_tree(x) +def bench_equal_tree(x: Tree, y: Tree) -> None: + for i in range(100000): + equal(x, y) + +[file driver.py] +from typing import Optional +import interpreted +import native +from timeit import timeit +from time import time +import os +import sys + +# Side test: some stuff about MROs and generics +if sys.version_info[:3] > (3, 5, 2): + assert tuple(x.__name__ for x in interpreted.SumVisitor.mro()) == ('SumVisitor', 'TreeVisitor', 'Generic', 'object') + assert tuple(x.__name__ for x in native.SumVisitor.mro()) == ('SumVisitor', 'TreeVisitor', 'Generic', 'object') + assert str(native.TreeVisitor[native.T]) == "native.TreeVisitor[~T]" + +assert native.TreeVisitor.__name__ == "TreeVisitor" +assert native.SumVisitor.__name__ == "SumVisitor" + +def dumb_time(f): + t0 = time() + f() + t1 = time() + return t1 - t0 + +def basic_test(m): + tree = m.build(5) + tree2 = m.build(5) + tree2.right.right.right.value = 10 + assert m.sum_tree(tree) == 57 + assert m.equal(tree, tree) + assert not m.equal(tree, tree2) + + assert isinstance(native.SumVisitor(), native.TreeVisitor) + + return tree + +def test(m): + tree = basic_test(m) + + g = {**globals(), **locals()} + fsum = dumb_time(lambda: m.bench_sum_tree(tree)) + feq = dumb_time(lambda: m.bench_equal_tree(tree, tree)) + return fsum, feq + +basic_test(native) + +if os.environ.get('MYPYC_RUN_BENCH') == '1': + nfsum, nfeq = test(native) + ifsum, ifeq = test(interpreted) + print(nfsum) + print("Sum speedup:", ifsum/nfsum) + print("Equal speedup:", ifeq/nfeq) diff --git a/mypyc/test-data/run-bools.test b/mypyc/test-data/run-bools.test new file mode 100644 index 000000000000..d7a2aa37ade7 --- /dev/null +++ b/mypyc/test-data/run-bools.test @@ -0,0 +1,229 @@ +# Test cases for booleans (compile and run) + +[case testTrueAndFalse] +def t() -> bool: + return True + +def f() -> bool: + return False +[file driver.py] +from native import t, f +print(t()) +print(f()) +[out] +True +False + +[case testBoolOps] +from typing import Optional, Any +MYPY = False +if MYPY: + from mypy_extensions import i64 + +def f(x: bool) -> bool: + if x: + return False + else: + return True + +def test_if() -> None: + assert f(True) is False + assert f(False) is True + +def test_bitwise_and() -> None: + # Use eval() to avoid constand folding + t: bool = eval('True') + f: bool = eval('False') + assert t & t == True + assert t & f == False + assert f & t == False + assert f & f == False + t &= t + assert t == True + t &= f + assert t == False + +def test_bitwise_or() -> None: + # Use eval() to avoid constand folding + t: bool = eval('True') + f: bool = eval('False') + assert t | t == True + assert t | f == True + assert f | t == True + assert f | f == False + t |= f + assert t == True + f |= t + assert f == True + +def test_bitwise_xor() -> None: + # Use eval() to avoid constand folding + t: bool = eval('True') + f: bool = eval('False') + assert t ^ t == False + assert t ^ f == True + assert f ^ t == True + assert f ^ f == False + t ^= f + assert t == True + t ^= t + assert t == False + f ^= f + assert f == False + +def test_isinstance_bool() -> None: + a = True + b = 1.0 + c = 1 + d = False + assert isinstance(a, bool) == True + assert isinstance(b, bool) == False + assert isinstance(c, bool) == False + assert isinstance(d, bool) == True + +class C: pass +class D: + def __init__(self, b: bool) -> None: + self.b = b + + def __bool__(self) -> bool: + return self.b + +class E: pass +class F(E): + def __init__(self, b: bool) -> None: + self.b = b + + def __bool__(self) -> bool: + return self.b + +def optional_to_bool1(o: Optional[C]) -> bool: + return bool(o) + +def optional_to_bool2(o: Optional[D]) -> bool: + return bool(o) + +def optional_to_bool3(o: Optional[E]) -> bool: + return bool(o) + +def test_optional_to_bool() -> None: + assert not optional_to_bool1(None) + assert optional_to_bool1(C()) + assert not optional_to_bool2(None) + assert not optional_to_bool2(D(False)) + assert optional_to_bool2(D(True)) + assert not optional_to_bool3(None) + assert optional_to_bool3(E()) + assert not optional_to_bool3(F(False)) + assert optional_to_bool3(F(True)) + +def test_any_to_bool() -> None: + a: Any = int() + b: Any = a + 1 + assert not bool(a) + assert bool(b) + +def eq(x: bool, y: bool) -> bool: + return x == y + +def ne(x: bool, y: bool) -> bool: + return x != y + +def lt(x: bool, y: bool) -> bool: + return x < y + +def le(x: bool, y: bool) -> bool: + return x <= y + +def gt(x: bool, y: bool) -> bool: + return x > y + +def ge(x: bool, y: bool) -> bool: + return x >= y + +def test_comparisons() -> None: + for x in True, False: + for y in True, False: + x2: Any = x + y2: Any = y + assert eq(x, y) == (x2 == y2) + assert ne(x, y) == (x2 != y2) + assert lt(x, y) == (x2 < y2) + assert le(x, y) == (x2 <= y2) + assert gt(x, y) == (x2 > y2) + assert ge(x, y) == (x2 >= y2) + +def eq_mixed(x: bool, y: int) -> bool: + return x == y + +def neq_mixed(x: int, y: bool) -> bool: + return x != y + +def lt_mixed(x: bool, y: int) -> bool: + return x < y + +def gt_mixed(x: int, y: bool) -> bool: + return x > y + +def test_mixed_comparisons() -> None: + for x in True, False: + for n in -(1 << 70), -123, 0, 1, 1753, 1 << 70: + assert eq_mixed(x, n) == (int(x) == n) + assert neq_mixed(n, x) == (n != int(x)) + assert lt_mixed(x, n) == (int(x) < n) + assert gt_mixed(n, x) == (n > int(x)) + +def add(x: bool, y: bool) -> int: + return x + y + +def add_mixed(b: bool, n: int) -> int: + return b + n + +def sub_mixed(n: int, b: bool) -> int: + return n - b + +def test_arithmetic() -> None: + for x in True, False: + for y in True, False: + assert add(x, y) == int(x) + int(y) + for n in -(1 << 70), -123, 0, 1, 1753, 1 << 70: + assert add_mixed(x, n) == int(x) + n + assert sub_mixed(n, x) == n - int(x) + +def add_mixed_i64(b: bool, n: i64) -> i64: + return b + n + +def sub_mixed_i64(n: i64, b: bool) -> i64: + return n - b + +def test_arithmetic_i64() -> None: + for x in True, False: + for n in -(1 << 62), -123, 0, 1, 1753, 1 << 62: + assert add_mixed_i64(x, n) == int(x) + n + assert sub_mixed_i64(n, x) == n - int(x) + +def eq_mixed_i64(x: bool, y: i64) -> bool: + return x == y + +def neq_mixed_i64(x: i64, y: bool) -> bool: + return x != y + +def lt_mixed_i64(x: bool, y: i64) -> bool: + return x < y + +def gt_mixed_i64(x: i64, y: bool) -> bool: + return x > y + +def test_mixed_comparisons_i64() -> None: + for x in True, False: + for n in -(1 << 62), -123, 0, 1, 1753, 1 << 62: + assert eq_mixed_i64(x, n) == (int(x) == n) + assert neq_mixed_i64(n, x) == (n != int(x)) + assert lt_mixed_i64(x, n) == (int(x) < n) + assert gt_mixed_i64(n, x) == (n > int(x)) + +[case testBoolMixInt] +y = False +print((y or 0) and True) +[out] +0 diff --git a/mypyc/test-data/run-bytes.test b/mypyc/test-data/run-bytes.test new file mode 100644 index 000000000000..fa63c46a6798 --- /dev/null +++ b/mypyc/test-data/run-bytes.test @@ -0,0 +1,325 @@ +# Bytes test cases (compile and run) + +[case testBytesBasics] +# Note: Add tests for additional operations to testBytesOps or in a new test case + +def f(x: bytes) -> bytes: + return x + +def eq(a: bytes, b: bytes) -> bool: + return a == b + +def neq(a: bytes, b: bytes) -> bool: + return a != b +[file driver.py] +from native import f, eq, neq +assert f(b'123') == b'123' +assert f(b'\x07 \x0b " \t \x7f \xf0') == b'\x07 \x0b " \t \x7f \xf0' +assert eq(b'123', b'123') +assert not eq(b'123', b'1234') +assert not eq(b'123', b'124') +assert not eq(b'123', b'223') +assert neq(b'123', b'1234') +try: + f('x') + assert False +except TypeError: + pass + +[case testBytesInit] +def test_bytes_init() -> None: + b1 = bytes([5]) + assert b1 == b'\x05' + b2 = bytes([5, 10, 12]) + assert b2 == b'\x05\n\x0c' + b3 = bytes(bytearray(b'foo')) + assert b3 == b'foo' + b4 = bytes(b'aaa') + assert b4 == b'aaa' + b5 = bytes(5) + assert b5 == b'\x00\x00\x00\x00\x00' + try: + bytes('x') + assert False + except TypeError: + pass + +[case testBytesOps] +from testutil import assertRaises + +def test_indexing() -> None: + # Use bytes() to avoid constant folding + b = b'asdf' + bytes() + assert b[0] == 97 + assert b[1] == 115 + assert b[3] == 102 + assert b[-1] == 102 + b = b'\xae\x80\xfe\x15' + bytes() + assert b[0] == 174 + assert b[1] == 128 + assert b[2] == 254 + assert b[3] == 21 + assert b[-4] == 174 + with assertRaises(IndexError, "index out of range"): + b[4] + with assertRaises(IndexError, "index out of range"): + b[-5] + with assertRaises(IndexError, "index out of range"): + b[2**26] + +def test_concat() -> None: + b1 = b'123' + bytes() + b2 = b'456' + bytes() + assert b1 + b2 == b'123456' + b3 = b1 + b2 + b3 = b3 + b1 + assert b3 == b'123456123' + assert b1 == b'123' + assert b2 == b'456' + assert type(b1) == bytes + assert type(b2) == bytes + assert type(b3) == bytes + brr1: bytes = bytearray(3) + brr2: bytes = bytearray(range(5)) + b4 = b1 + brr1 + assert b4 == b'123\x00\x00\x00' + assert type(brr1) == bytearray + assert type(b4) == bytes + brr3 = brr1 + brr2 + assert brr3 == bytearray(b'\x00\x00\x00\x00\x01\x02\x03\x04') + assert len(brr3) == 8 + assert type(brr3) == bytearray + brr3 = brr3 + bytearray([10]) + assert brr3 == bytearray(b'\x00\x00\x00\x00\x01\x02\x03\x04\n') + b5 = brr2 + b2 + assert b5 == bytearray(b'\x00\x01\x02\x03\x04456') + assert type(b5) == bytearray + b5 = b2 + brr2 + assert b5 == b'456\x00\x01\x02\x03\x04' + assert type(b5) == bytes + +def test_join() -> None: + seq = (b'1', b'"', b'\xf0') + assert b'\x07'.join(seq) == b'1\x07"\x07\xf0' + assert b', '.join(()) == b'' + assert b', '.join([bytes() + b'ab']) == b'ab' + assert b', '.join([bytes() + b'ab', b'cd']) == b'ab, cd' + +def test_len() -> None: + # Use bytes() to avoid constant folding + b = b'foo' + bytes() + assert len(b) == 3 + assert len(bytes()) == 0 + +def test_ord() -> None: + assert ord(b'a') == ord('a') + assert ord(b'a' + bytes()) == ord('a') + assert ord(b'\x00') == 0 + assert ord(b'\x00' + bytes()) == 0 + assert ord(b'\xfe') == 254 + assert ord(b'\xfe' + bytes()) == 254 + + with assertRaises(TypeError): + ord(b'aa') + with assertRaises(TypeError): + ord(b'') + +def test_ord_bytesarray() -> None: + assert ord(bytearray(b'a')) == ord('a') + assert ord(bytearray(b'\x00')) == 0 + assert ord(bytearray(b'\xfe')) == 254 + + with assertRaises(TypeError): + ord(bytearray(b'aa')) + with assertRaises(TypeError): + ord(bytearray(b'')) + +[case testBytesSlicing] +def test_bytes_slicing() -> None: + b = b'abcdefg' + zero = int() + ten = 10 + zero + two = 2 + zero + five = 5 + zero + seven = 7 + zero + assert b[:ten] == b'abcdefg' + assert b[0:seven] == b'abcdefg' + assert b[0:(len(b)+1)] == b'abcdefg' + assert b[two:five] == b'cde' + assert b[two:two] == b'' + assert b[-two:-two] == b'' + assert b[-ten:(-ten+1)] == b'' + assert b[:-two] == b'abcde' + assert b[:two] == b'ab' + assert b[:] == b'abcdefg' + assert b[-two:] == b'fg' + assert b[zero:] == b'abcdefg' + assert b[:zero] == b'' + assert b[-ten:] == b'abcdefg' + assert b[-ten:ten] == b'abcdefg' + big_ints = [1000 * 1000 * 1000 * 1000 * 1000 * 1000 * 1000, 2**24, 2**63] + for big_int in big_ints: + assert b[1:big_int] == b'bcdefg' + assert b[big_int:] == b'' + assert b[-big_int:-1] == b'abcdef' + assert b[-big_int:big_int] == b'abcdefg' + assert type(b[-big_int:-1]) == bytes + assert type(b[-ten:]) == bytes + assert type(b[:]) == bytes + +[case testBytearrayBasics] +from typing import Any + +def test_basics() -> None: + brr1: bytes = bytearray(3) + assert brr1 == bytearray(b'\x00\x00\x00') + assert brr1 == b'\x00\x00\x00' + l = [10, 20, 30, 40] + brr2: bytes = bytearray(l) + assert brr2 == bytearray(b'\n\x14\x1e(') + assert brr2 == b'\n\x14\x1e(' + brr3: bytes = bytearray(range(5)) + assert brr3 == bytearray(b'\x00\x01\x02\x03\x04') + assert brr3 == b'\x00\x01\x02\x03\x04' + brr4: bytes = bytearray('string', 'utf-8') + assert brr4 == bytearray(b'string') + assert brr4 == b'string' + assert len(brr1) == 3 + assert len(brr2) == 4 + +def f(b: bytes) -> bool: + return True + +def test_bytearray_passed_into_bytes() -> None: + assert f(bytearray(3)) + brr1: Any = bytearray() + assert f(brr1) + +[case testBytearraySlicing] +def test_bytearray_slicing() -> None: + b: bytes = bytearray(b'abcdefg') + zero = int() + ten = 10 + zero + two = 2 + zero + five = 5 + zero + seven = 7 + zero + assert b[:ten] == b'abcdefg' + assert b[0:seven] == b'abcdefg' + assert b[two:five] == b'cde' + assert b[two:two] == b'' + assert b[-two:-two] == b'' + assert b[-ten:(-ten+1)] == b'' + assert b[:-two] == b'abcde' + assert b[:two] == b'ab' + assert b[:] == b'abcdefg' + assert b[-two:] == b'fg' + assert b[zero:] == b'abcdefg' + assert b[:zero] == b'' + assert b[-ten:] == b'abcdefg' + assert b[-ten:ten] == b'abcdefg' + big_ints = [1000 * 1000 * 1000 * 1000 * 1000 * 1000 * 1000, 2**24, 2**63] + for big_int in big_ints: + assert b[1:big_int] == b'bcdefg' + assert b[big_int:] == b'' + assert b[-big_int:-1] == b'abcdef' + assert b[-big_int:big_int] == b'abcdefg' + assert type(b[-big_int:-1]) == bytearray + assert type(b[-ten:]) == bytearray + assert type(b[:]) == bytearray + +[case testBytearrayIndexing] +from testutil import assertRaises + +def test_bytearray_indexing() -> None: + b: bytes = bytearray(b'\xae\x80\xfe\x15') + assert b[0] == 174 + assert b[1] == 128 + assert b[2] == 254 + assert b[3] == 21 + assert b[-4] == 174 + with assertRaises(IndexError, "index out of range"): + b[4] + with assertRaises(IndexError, "index out of range"): + b[-5] + b2 = bytearray([175, 255, 128, 22]) + assert b2[0] == 175 + assert b2[1] == 255 + assert b2[-1] == 22 + assert b2[2] == 128 + with assertRaises(ValueError, "byte must be in range(0, 256)"): + b2[0] = -1 + with assertRaises(ValueError, "byte must be in range(0, 256)"): + b2[0] = 256 + +[case testBytesJoin] +from typing import Any +from testutil import assertRaises +from a import bytes_subclass + +def test_bytes_join() -> None: + assert b' '.join([b'a', b'b']) == b'a b' + assert b' '.join([]) == b'' + + x: bytes = bytearray(b' ') + assert x.join([b'a', b'b']) == b'a b' + assert type(x.join([b'a', b'b'])) == bytearray + + y: bytes = bytes_subclass() + assert y.join([]) == b'spook' + + n: Any = 5 + with assertRaises(TypeError, "can only join an iterable"): + assert b' '.join(n) + +[file a.py] +class bytes_subclass(bytes): + def join(self, iter): + return b'spook' + +[case testBytesFormatting] +[typing fixtures/typing-full.pyi] +from testutil import assertRaises + +# https://www.python.org/dev/peps/pep-0461/ +def test_bytes_formatting() -> None: + val = 10 + assert b"%x" % val == b'a' + assert b'%4x' % val == b' a' + assert b'%#4x' % val == b' 0xa' + assert b'%04X' % val == b'000A' + + assert b'%c' % 48 == b'0' + assert b'%c' % b'a' == b'a' + assert b'%c%c' % (48, b'a') == b'0a' + + assert b'%b' % b'abc' == b'abc' + assert b'%b' % 'some string'.encode('utf8') == b'some string' + + assert b'%a' % 3.14 == b'3.14' + assert b'%a' % b'abc' == b"b'abc'" + assert b'%a' % 'def' == b"'def'" + +def test_bytes_formatting_2() -> None: + var = b'bb' + num = 10 + assert b'aaa%bbbb%s' % (var, var) == b'aaabbbbbbb' + assert b'aaa%dbbb%b' % (num, var) == b'aaa10bbbbb' + assert b'%s%b' % (var, var) == b'bbbb' + assert b'%b' % bytes() == b'' + assert b'%b' % b'' == b'' + + assert b'\xff%s' % b'\xff' == b'\xff\xff' + assert b'\xff%b' % '你好'.encode() == b'\xff\xe4\xbd\xa0\xe5\xa5\xbd' + + aa = b'\xe4\xbd\xa0\xe5\xa5\xbd%b' % b'\xe4\xbd\xa0\xe5\xa5\xbd' + assert aa == b'\xe4\xbd\xa0\xe5\xa5\xbd\xe4\xbd\xa0\xe5\xa5\xbd' + assert aa.decode() == '你好你好' + + +class A: + def __bytes__(self): + return b'aaa' + +def test_bytes_dunder() -> None: + assert b'%b' % A() == b'aaa' + assert b'%s' % A() == b'aaa' diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test new file mode 100644 index 000000000000..cf30bddbef64 --- /dev/null +++ b/mypyc/test-data/run-classes.test @@ -0,0 +1,2657 @@ +[case testEmptyClass] +class Empty: pass + +def f(e: Empty) -> Empty: + return e + +class EmptyEllipsis: ... + +def g(e: EmptyEllipsis) -> EmptyEllipsis: + return e +[file driver.py] +from native import Empty, EmptyEllipsis, f, g + +print(isinstance(Empty, type)) +print(Empty) +print(str(Empty())[:20]) + +e = Empty() +print(f(e) is e) + +print(isinstance(EmptyEllipsis, type)) +print(EmptyEllipsis) +print(str(EmptyEllipsis())[:28]) + +e2 = EmptyEllipsis() +print(g(e2) is e2) +[out] +True + + + None: + c = C() + c.x = 1 + c.y = 2 + c.z = 3 + del c.x + del c.y + assert c.z == 3 + with assertRaises(AttributeError, "attribute 'x' of 'C' undefined"): + c.x + with assertRaises(AttributeError, "attribute 'y' of 'C' undefined"): + c.y + +def test_delete_any() -> None: + c: Any = C() + c.x = 1 + c.y = 2 + c.z = 3 + del c.x + del c.y + with assertRaises(AttributeError, "'C' object attribute 'z' cannot be deleted"): + del c.z + assert c.z == 3 + with assertRaises(AttributeError): + c.x + with assertRaises(AttributeError): + c.y + +class Base: + __deletable__ = ['a'] + a: int + b: int + +class Deriv(Base): + __deletable__ = ('c',) + c: str + d: str + +def test_delete_with_inheritance() -> None: + d = Deriv() + d.a = 0 + d.b = 1 + d.c = 'X' + d.d = 'Y' + del d.a + with assertRaises(AttributeError): + d.a + del d.c + with assertRaises(AttributeError): + d.c + assert d.b == 1 + assert d.d == 'Y' + +def test_delete_with_inheritance_any() -> None: + d: Any = Deriv() + d.a = 0 + d.b = 1 + d.c = 'X' + d.d = 'Y' + del d.a + with assertRaises(AttributeError): + d.a + del d.c + with assertRaises(AttributeError): + d.c + with assertRaises(AttributeError): + del d.b + assert d.b == 1 + with assertRaises(AttributeError): + del d.d + assert d.d == 'Y' + +def decorator(cls): + return cls + +@decorator +class NonExt: + x: int + y: int + + # No effect in a non-native class + __deletable__ = ['x'] + +def test_non_ext() -> None: + n = NonExt() + n.x = 2 + n.y = 3 + del n.x + del n.y + with assertRaises(AttributeError): + n.x + with assertRaises(AttributeError): + n.y + +def test_non_ext_any() -> None: + n: Any = NonExt() + n.x = 2 + n.y = 3 + del n.x + del n.y + with assertRaises(AttributeError): + n.x + with assertRaises(AttributeError): + n.y + +[case testNonExtMisc] +from typing import Any, overload + +def decorator(cls) -> Any: + return cls + +@decorator +class C: + def __init__(self) -> None: + self.c = 3 + + def get_c(self) -> int: + return self.c + +@decorator +class B(C): + def __init__(self) -> None: + super().__init__() + self.b = 2 + + def get_b(self) -> int: + return self.b + +@decorator +class A(B): + def __init__(self) -> None: + super().__init__() + self.a = 1 + + @classmethod + def constant(cls) -> int: + return 4 + + def get_a(self) -> int: + return self.a + +@decorator +class Overload: + @overload + def get(self, index: int) -> int: ... + + @overload + def get(self, index: str) -> str: ... + + def get(self, index: Any) -> Any: + return index + +def get(c: Overload, s: str) -> str: + return c.get(s) + +@decorator +class Var: + x = 'xy' + +def get_class_var() -> str: + return Var.x + +[file driver.py] +from native import A, Overload, get, get_class_var +a = A() +assert a.a == 1 +assert a.b == 2 +assert a.c == 3 +assert a.get_a() == 1 +assert a.get_b() == 2 +assert a.get_c() == 3 +assert A.constant() == 4 + +o = Overload() +assert get(o, "test") == "test" +assert o.get(20) == 20 + +assert get_class_var() == 'xy' + +[case testEnum] +from enum import Enum + +class TestEnum(Enum): + _order_ = "a b" + a = 1 + b = 2 + + @classmethod + def test(cls) -> int: + return 3 + +assert TestEnum.test() == 3 + +import enum + +class Pokemon(enum.Enum): + magikarp = 1 + squirtle = 2 + slowbro = 3 + +assert Pokemon.magikarp.value == 1 +assert Pokemon.squirtle.name == 'squirtle' + +[file other.py] +# Force a multi-module test to make sure we can compile multi-file with +# non-extension classes + +[file driver.py] +import sys +# "_order_" isn't supported in 3.5 +if sys.version_info[:2] > (3, 5): + from native import TestEnum + assert TestEnum.a.name == 'a' + assert TestEnum.a.value == 1 + assert TestEnum.b.name == 'b' + assert TestEnum.b.value == 2 + +[case testGetAttribute] +class C: + x: int + y: int + +def getx(c: C) -> int: + return c.x + +def gety(c: C) -> int: + return c.y +[file driver.py] +from native import C, getx, gety +c = C() +c.x = 10**30 +c.y = 10**30 + 1 +print(getx(c)) +print(gety(c)) +[out] +1000000000000000000000000000000 +1000000000000000000000000000001 + +[case testSetAttribute] +class C: + x: int + y: int + +def setx(c: C, v: int) -> None: + c.x = v + +def sety(c: C, v: int) -> None: + c.y = v +[file driver.py] +from native import C, setx, sety +c = C() +setx(c, 10**30) +sety(c, 10**30 + 1) +print(c.x) +print(c.y) +setx(c, 4) +sety(c, 5) +print(c.x, c.y) +setx(c, 10**30 + 2) +sety(c, 10**30 + 3) +print(c.x) +print(c.y) +[out] +1000000000000000000000000000000 +1000000000000000000000000000001 +4 5 +1000000000000000000000000000002 +1000000000000000000000000000003 + +[case testAttributeTypes] +from typing import List, Tuple +class C: + a: List[int] + b: bool + c: C + d: object + e: int + +def setattrs(o: C, a: List[int], b: bool, c: C) -> None: + o.a = a + o.b = b + o.c = c + +def getattrs(o: C) -> Tuple[List[int], bool, C]: + return o.a, o.b, o.c +[file driver.py] +from native import C, setattrs, getattrs +from testutil import assertRaises + +c1 = C() +c2 = C() +aa = [2] +setattrs(c1, aa, True, c2) +a, b, c = getattrs(c1) +assert a is aa +assert b is True +assert c is c2 + +o = object() +c1.d = o +assert c1.d is o + +c3 = C() +with assertRaises(AttributeError, "attribute 'a' of 'C' undefined"): + c3.a +with assertRaises(AttributeError, "attribute 'b' of 'C' undefined"): + c3.b +with assertRaises(AttributeError, "attribute 'c' of 'C' undefined"): + c3.c +with assertRaises(AttributeError, "attribute 'd' of 'C' undefined"): + c3.d +with assertRaises(AttributeError, "attribute 'e' of 'C' undefined"): + c3.e + +[case testInitMethodWithMissingNoneReturnAnnotation] +class C: + def __init__(self): + self.x = 42 +[file driver.py] +from native import C +c = C() +assert c is not None +assert c.x == 42 + +[case testConstructClassWithDefaultConstructor] +class C: + a: int + b: int + +def f() -> C: + c = C() + c.a = 13 + return c +[file driver.py] +from native import f, C +c = f() +assert c.a == 13 +assert type(c) == C +assert not hasattr(c, 'b') + +[case testCastUserClass] +from typing import List + +class C: + x: int + +def f(a: List[C]) -> C: + return a[0] +[file driver.py] +from native import f, C +c = C() +assert f([c]) is c + +[case testClass1] +class A: + def __init__(self, x: int) -> None: + self.x = x + def foo(self) -> int: + return self.x+1 +def foo() -> int: + a = A(20) + return a.foo() +[file driver.py] +from native import A, foo +a = A(10) +assert a.foo() == 11 +assert foo() == 21 + +[case testGenericClass] +from typing import TypeVar, Generic, Sequence +T = TypeVar('T') +class C(Generic[T]): + x: T + def __init__(self, x: T) -> None: + self.x = x + def get(self) -> T: + return self.x + def set(self, y: T) -> None: + self.x = y + +# Test subclassing generic classes both with and without a generic param +class A(Sequence[int]): + pass +class B(Sequence[T]): + pass + +def f(c: C[int]) -> int: + y = c.get() + d = C[int](2) + c.set(c.get() + 1 + d.get()) + c.x = c.x + 2 + return c.x + +[file driver.py] +from native import C, f +c = C(6) +assert f(c) == 11 +c.x = 'x' +assert c.x == 'x' +c.set([1]) +assert c.x == [1] +assert c.get() == [1] + +[case testSubclass1] +from typing import Tuple + +class A: + def __init__(self) -> None: + self.x = 10 + def hi(self, suffix: str) -> str: + return str(self.x) + suffix +class B(A): + def __init__(self) -> None: + self.x = 20 + self.y = 'world' + def hi(self, suffix: str) -> str: + return 'hello ' + str(self.y) + suffix + +def use_a(x: A) -> Tuple[int, str]: + return (x.x, x.hi('')) +def use_b(x: B) -> str: + return x.hi('') + +[file driver.py] +from native import A, B, use_a, use_b +a = A() +b = B() +assert use_a(a) == (10, '10') +assert use_a(b) == (20, 'hello world') +assert a.x == 10 +assert b.x == 20 +assert b.y == 'world' +assert a.hi('!') == '10!' +assert b.hi('!') == 'hello world!' +assert use_b(b) == 'hello world' + +[case testSubclassSpecialize1] +class A: + def foo(self, x: int) -> object: + print('A') + return str(x) + def bar(self, x: int) -> None: + print(x + 1) +class B(A): + def foo(self, x: object) -> int: + print('B') + return id(x) + def bar(self, x: object) -> None: + print(x) + +def use_a(x: A, y: int) -> object: + x.bar(10) + return x.foo(y) + +def use_b(x: B, y: object) -> int: + return x.foo(y) + +[file driver.py] +from native import A, B, use_a, use_b +a = A() +b = B() +o = object() +i = 10 +assert a.foo(10) == '10' +assert b.foo(o) == id(o) +assert use_a(a, 10) == '10' +assert use_b(b, o) == id(o) +assert use_a(b, i) == id(i) +[out] +A +B +11 +A +B +10 +B + +[case testSubclassSpecialize2] +class A: + def foo(self, x: int) -> object: + print('A') + return str(x) +class B(A): + def foo(self, x: object) -> object: + print('B') + return x +class C(B): + def foo(self, x: object) -> int: + print('C') + return id(x) + +def use_a(x: A, y: int) -> object: + return x.foo(y) + +def use_b(x: B, y: object) -> object: + return x.foo(y) + +def use_c(x: C, y: object) -> int: + return x.foo(y) + +[file driver.py] +from native import A, B, C, use_a, use_b, use_c +a = A() +b = B() +c = C() +o = object() +i = 10 +assert a.foo(10) == '10' +assert b.foo(o) == o +assert c.foo(o) == id(o) +assert use_a(a, 10) == '10' +assert use_a(b, i) is i +assert use_a(c, i) == id(i) +assert use_b(b, o) == o +assert use_b(c, o) == id(o) +assert use_c(c, o) == id(o) +[out] +A +B +C +A +B +C +B +C +C + +[case testIsInstance] +from typing import Optional +class X: pass +class A(X): pass +class B(A): pass + +def isa(x: object) -> bool: + return isinstance(x, A) +def isint(x: object) -> bool: + return isinstance(x, int) +def isstr(x: object) -> bool: + return isinstance(x, str) +def islist(x: object) -> bool: + return isinstance(x, list) +def ist(x: object, t: object) -> bool: # TODO: Second argument should be 'type' + return isinstance(x, t) +def pointless(x: Optional[X]) -> str: + if isinstance(x, A): + return str(x) + return '' +[file driver.py] +from native import X, A, B, isa, isint, isstr, islist, ist +assert isa(1) == False +assert isa(A()) == True +assert isa(B()) == True +assert isa(X()) == False + +assert isint(1) == True +assert isint('') == False +assert isint(A()) == False + +assert isstr(1) == False +assert isstr('') == True + +assert islist(1) == False +assert islist([]) == True + +assert ist(1, int) == True +assert ist(1, str) == False +try: + ist(1, 2) +except TypeError: + pass +else: + assert False + +[case testSubclassUninitAttr] +class X: + x: int +class A(X): + pass +[file driver.py] +import traceback +from native import A +try: + A().x +except AttributeError: + traceback.print_exc() +[out] +Traceback (most recent call last): + File "driver.py", line 4, in + A().x +AttributeError: attribute 'x' of 'X' undefined + +[case testClassMethods] +from typing import ClassVar, Any +from typing_extensions import final +from mypy_extensions import mypyc_attr + +from interp import make_interpreted_subclass + +class C: + lurr: ClassVar[int] = 9 + @staticmethod + def foo(x: int) -> int: + return 10 + x + @classmethod + def bar(cls, x: int) -> int: + return cls.lurr + x + @staticmethod + def baz(x: int, y: int = 10) -> int: + return y - x + @classmethod + def quux(cls, x: int, y: int = 10) -> int: + return y - x + @classmethod + def call_other(cls, x: int) -> int: + return cls.quux(x, 3) + +class D(C): + def f(self) -> int: + return super().foo(1) + super().bar(2) + super().baz(10) + super().quux(10) + +def ctest1() -> int: + return C.foo(1) + C.bar(2) + C.baz(10) + C.quux(10) + C.quux(y=10, x=9) + +def ctest2() -> int: + c = C() + return c.foo(1) + c.bar(2) + c.baz(10) + +CAny: Any = C + +def test_classmethod_using_any() -> None: + assert CAny.foo(10) == 20 + assert CAny.bar(10) == 19 + +def test_classmethod_on_instance() -> None: + c = C() + assert c.foo(10) == 20 + assert c.bar(10) == 19 + assert c.call_other(1) == 2 + +def test_classmethod_misc() -> None: + assert ctest1() == 23 + assert ctest2() == 22 + assert C.call_other(2) == 1 + +def test_classmethod_using_super() -> None: + d = D() + assert d.f() == 22 + +@final +class F1: + @classmethod + def f(cls, x: int) -> int: + return cls.g(x) + + @classmethod + def g(cls, x: int) -> int: + return x + 1 + +class F2: # Implicitly final (no subclasses) + @classmethod + def f(cls, x: int) -> int: + return cls.g(x) + + @classmethod + def g(cls, x: int) -> int: + return x + 1 + +def test_classmethod_of_final_class() -> None: + assert F1.f(5) == 6 + assert F2.f(7) == 8 + +@mypyc_attr(allow_interpreted_subclasses=True) +class CI: + @classmethod + def f(cls, x: int) -> int: + return cls.g(x) + + @classmethod + def g(cls, x: int) -> int: + return x + 1 + +def test_classmethod_with_allow_interpreted() -> None: + assert CI.f(4) == 5 + sub = make_interpreted_subclass(CI) + assert sub.f(4) == 7 + +[file interp.py] +def make_interpreted_subclass(base): + class Sub(base): + @classmethod + def g(cls, x: int) -> int: + return x + 3 + return Sub + +[case testSuper] +from mypy_extensions import trait +from typing import List + +class A: + def __init__(self, x: int) -> None: + self.x = x + + def foo(self, x: int) -> int: + return x + +class B(A): + def __init__(self, x: int, y: int) -> None: + super().__init__(x) + self.y = y + + def foo(self, x: int) -> int: + return super().foo(x+1) + +class C(B): + def __init__(self, x: int, y: int) -> None: + super(C, self).__init__(x, y + 1) + + def foo(self, x: int) -> int: + # should go to A, not B + return super(B, self).foo(x+1) + +class X: + def __init__(self, x: int) -> None: + self.x = x +class Y(X): + pass +class Z(Y): + def __init__(self, x: int, y: int) -> None: + super().__init__(x) + self.y = y + +@trait +class T: + def v_int(self, x: int) -> None: pass + def v_list(self, x: List[int]) -> None: + if x: + self.v_int(x[0]) + self.v_list(x[1:]) + +class PrintList(T): + def v_int(self, x: int) -> None: + print(x) + def v_list(self, x: List[int]) -> None: + print('yo!') + super().v_list(x) + +[file driver.py] +import traceback +from native import * +b = B(10, 20) +assert b.x == 10 and b.y == 20 +c = C(10, 20) +assert c.x == 10 and c.y == 21 +z = Z(10, 20) +assert z.x == 10 and z.y == 20 + +assert c.foo(10) == 11 + +PrintList().v_list([1,2,3]) +[out] +yo! +1 +yo! +2 +yo! +3 +yo! + +[case testSubclassException] +class Failure(Exception): + def __init__(self, x: int) -> None: + self.x = x + +def foo() -> None: + raise Failure(10) + +def heyo() -> int: + try: + foo() + except Failure as e: + return e.x + return -1 + +[file driver.py] +from native import foo, heyo, Failure +try: + foo() +except Failure as e: + assert str(e) == '10' + assert e.x == 10 +heyo() + +[case testSubclassDict] +from typing import Dict +class WelpDict(Dict[str, int]): + def __init__(self) -> None: + self.emarhavil = 3 + def foo(self) -> int: + return self.emarhavil + +def welp() -> int: + x = WelpDict() + x['a'] = 10 + x['b'] = 15 + x.emarhavil = 5 + return x['a'] + x['b'] + x.emarhavil + x.foo() + +[file driver.py] +from native import welp +assert welp() == 35 + +[case testSubclassPy] +from b import B, V +class A(B): + def __init__(self, x: int, y: int) -> None: + super().__init__(y) + self.x = x + + def foo(self, x: int) -> int: + print("hi", x) + return x+1 + +class C(V[int]): + def f(self) -> int: return 10 + +assert isinstance(C(), V) + +def f(x: A) -> None: + print(x.x) + print(x.y) + print(x.foo(20)) + +[file b.py] +from typing import Generic, TypeVar +T = TypeVar('T') + +class B: + def __init__(self, y: int) -> None: + self.y = y + def foo(self, x: int) -> int: + print("parent!") + return x + self.y + def bar(self) -> None: + print("hello!", self.y) + +class V(Generic[T]): + def f(self) -> T: + raise Exception('unimplemented') +[file driver.py] +import native +a = native.A(10, 20) +a.foo(10) +a.bar() +native.f(a) +[out] +hi 10 +hello! 20 +10 +20 +hi 20 +21 + +[case testDisallowSubclassFromPy] +# We'll want to allow this at some point but right now we need to +# disallow it because it doesn't work. +class A: + pass +[file b.py] +from native import A + +# It would be better if we disallowed it at class decl time but it is +# really easy to do in __new__ + +class B(A): + pass + +[file driver.py] +from b import B +try: + B() +except TypeError: + pass +else: + assert False, "instantiating was supposed to fail" + +[case testClassVariable] +MYPY = False +if MYPY: + from typing import ClassVar +class A: + x = 10 # type: ClassVar[int] + +def g(x: int) -> None: + A.x = 10 +def f() -> int: + return A.x +[file driver.py] +from native import A, f +assert f() == 10 +A.x = 200 +assert f() == 200 + +[case testDefaultVars] +from typing import Optional +class A: + x = 10 + w: object = 10 + def lol(self) -> None: + self.x = 100 + +LOL = 'lol' +class B(A): + y = LOL + z = None # type: Optional[str] + b = True + bogus = None # type: int + +def g() -> None: + a = A() + assert a.x == 10 + a.x = 20 + assert a.x == 20 + b = B() + assert b.x == 10 + b.x = 20 + assert b.x == 20 + assert b.y == 'lol' + b.y = 'rofl' + assert b.y == 'rofl' + assert b.z is None +[file driver.py] +from native import * +g() + +a = A() +assert a.x == 10 +a.x = 20 +assert a.x == 20 +b = B() +assert b.x == 10 +b.x = 20 +assert b.x == 20 +assert b.y == 'lol' +b.y = 'rofl' +assert b.y == 'rofl' +assert b.z is None +# N.B: this doesn't match cpython +assert not hasattr(b, 'bogus') + +[case testProtocol] +from typing_extensions import Protocol + +class Proto(Protocol): + def foo(self, x: int) -> None: + pass + + def bar(self, x: int) -> None: + pass + +class A: + def foo(self, x: int) -> None: + print("A:", x) + + def bar(self, *args: int, **kwargs: int) -> None: + print("A:", args, kwargs) + +class B(A, Proto): + def foo(self, x: int) -> None: + print("B:", x) + + def bar(self, *args: int, **kwargs: int) -> None: + print("B:", args, kwargs) + +def f(x: Proto) -> None: + x.foo(20) + x.bar(x=20) + +[file driver.py] +from native import A, B, f + +f(A()) +f(B()) + +# ... this exploits a bug in glue methods to distinguish whether we +# are making a direct call or a pycall... +[out] +A: 20 +A: () {'x': 20} +B: 20 +B: (20,) {} + +[case testMethodOverrideDefault1] +class A: + def foo(self, x: int) -> None: + pass +class B(A): + def foo(self, x: int, y: int = 10) -> None: + print(x, y) + +def a(x: A) -> None: + x.foo(1) +def b(x: B) -> None: + x.foo(2) + x.foo(2, 3) + +[file driver.py] +from native import B, a, b +a(B()) +b(B()) +[out] +1 10 +2 10 +2 3 + +[case testMethodOverrideDefault2] +class A: + def foo(self, *, x: int = -1) -> None: + pass + def bar(self, *, x: int = -1, y: int = -1) -> None: + pass + def baz(self, x: int = -1) -> None: + pass +class B(A): + def foo(self, *, y: int = 0, x: int = 0) -> None: + print(x, y) + def bar(self, *, y: int = 0, x: int = 0) -> None: + print(x, y) + def baz(self, x: int = 0, *, y: int = 0) -> None: + print(x, y) + +def a(x: A) -> None: + x.foo(x=1) + x.bar(x=1, y=2) + x.bar(x=2, y=1) + x.baz() + x.baz(1) + x.baz(x=2) + +[file driver.py] +from native import B, a +a(B()) +[out] +1 0 +1 2 +2 1 +0 0 +1 0 +2 0 + +[case testMethodOverrideDefault3] +class A: + @classmethod + def foo(cls, *, x: int = 0) -> None: + pass + @staticmethod + def bar(*, x: int = 0) -> None: + pass + @staticmethod + def baz() -> object: + pass +class B(A): + @classmethod + def foo(cls, *, y: int = 0, x: int = 0) -> None: + print(x, y) + print(cls.__name__) # type: ignore + @staticmethod + def bar(*, y: int = 0, x: int = 0) -> None: + print(x, y) + @staticmethod + def baz() -> int: + return 10 + +# This is just to make sure that this stuff works even when the +# methods might be overridden. +class C(B): + @classmethod + def foo(cls, *, y: int = 0, x: int = 0) -> None: + pass + @staticmethod + def bar(*, y: int = 0, x: int = 0) -> None: + pass + @staticmethod + def baz() -> int: + return 10 + +def a(x: A) -> None: + x.foo(x=1) + x.bar(x=1) + print(x.baz()) + +[file driver.py] +from native import B, a +a(B()) +[out] +1 0 +B +1 0 +10 + +[case testMethodOverrideDefault4] +class Foo: + def f(self, x: int=20, *, z: int=10) -> None: + pass + +class Bar(Foo): + def f(self, *args: int, **kwargs: int) -> None: + print("stuff", args, kwargs) + +z: Foo = Bar() +z.f(1, z=50) +z.f() + +[out] +stuff (1,) {'z': 50} +stuff () {} + +[case testMethodOverrideDefault5] +from testutil import make_python_function +from mypy_extensions import mypyc_attr +from typing import TypeVar, Any + +@mypyc_attr(allow_interpreted_subclasses=True) +class Foo: + def f(self, x: int=20, *, z: int=10) -> None: + print("Foo", x, z) + +@make_python_function +def baz_f(self: Any, *args: int, **kwargs: int) -> None: + print("Baz", args, kwargs) + +# Make an "interpreted" subtype of Foo +type2: Any = type +Bar = type2('Bar', (Foo,), {}) +Baz = type2('Baz', (Foo,), {'f': baz_f}) + +y: Foo = Bar() +y.f(1, z=2) +y.f() + +z: Foo = Baz() +z.f(1, z=2) +z.f() + +[out] +Foo 1 2 +Foo 20 10 +Baz (1,) {'z': 2} +Baz () {} + +[case testMethodOverrideDefault6] +from typing import Optional + +class Foo: + def f(self, x: int=20) -> None: + pass + +class Bar(Foo): + def f(self, x: Optional[int]=None) -> None: + print(x) + +z: Foo = Bar() +z.f(1) +z.f() + +[out] +1 +None + +[case testMethodOverrideDefault7] +from typing import TypeVar, Any + +class Foo: + def f(self, x: int, *args: int, **kwargs: int) -> None: + print("Foo", x, args, kwargs) + +class Bar(Foo): + def f(self, *args: int, **kwargs: int) -> None: + print("Bar", args, kwargs) + +z: Foo = Bar() +z.f(1, z=2) +z.f(1, 2, 3) +# z.f(x=5) # Not tested because we (knowingly) do the wrong thing and pass it as positional + +[out] +Bar (1,) {'z': 2} +Bar (1, 2, 3) {} +--Bar () {'x': 5} + +[case testMethodOverrideDefault8] +from typing import TypeVar, Any + +class Foo: + def f(self, *args: int, **kwargs: int) -> None: + print("Foo", args, kwargs) + +class Bar(Foo): + def f(self, x: int = 10, *args: int, **kwargs: int) -> None: + print("Bar", x, args, kwargs) + +z: Foo = Bar() +z.f(1, z=2) +z.f(1, 2, 3) +z.f() + +[out] +Bar 1 () {'z': 2} +Bar 1 (2, 3) {} +Bar 10 () {} + +[case testMethodOverrideDefault9] +from testutil import make_python_function +from mypy_extensions import mypyc_attr +from typing import TypeVar, Any + +@mypyc_attr(allow_interpreted_subclasses=True) +class Foo: + def f(self, x: int=20, y: int=40) -> None: + print("Foo", x, y) + +# This sort of argument renaming is dodgy and not really sound but we +# shouldn't break it when they aren't actually used by name... +# (They *ought* to be positional only!) +@make_python_function +def baz_f(self, a: int=30, y: int=50) -> None: + print("Baz", a, y) + +# Make an "interpreted" subtype of Foo +type2: Any = type +Baz = type2('Baz', (Foo,), {'f': baz_f}) + +z: Foo = Baz() +z.f() +z.f(y=1) +z.f(1, 2) +# Not tested because we don't (and probably won't) match cpython here +# from testutil import assertRaises +# with assertRaises(TypeError): +# z.f(x=7) + +[out] +Baz 30 50 +Baz 30 1 +Baz 1 2 + +[case testOverride] +class A: + def f(self) -> int: + return 0 + def g(self) -> int: + return 1 + +class B(A): + def g(self) -> int: + return 2 + +class C(B): + def f(self) -> int: + return 3 + +def test() -> None: + ba: A = B() + ca: A = C() + assert ba.f() == 0 + assert ba.g() == 2 + assert ca.f() == 3 + assert ca.g() == 2 + cc = C() + assert cc.f() == 3 + assert cc.g() == 2 + print('ok') +[file driver.py] +import native +native.test() +[out] +ok + +[case testNoMetaclass] +from foo import Base + +class Nothing(Base): # type: ignore + pass + +[file foo.py] +from typing import Any +class Meta(type): + pass + +class _Base(metaclass=Meta): + pass +Base = _Base # type: Any + +[file driver.py] +try: + import native +except TypeError as e: + assert(str(e) == "mypyc classes can't have a metaclass") + +[case testMetaclass] +from meta import Meta + +class Nothing(metaclass=Meta): + pass + +def ident(x): return x + +@ident +class Test: + pass + +[file meta.py] +class Meta(type): + def __new__(mcs, name, bases, dct): + dct['X'] = 10 + return super().__new__(mcs, name, bases, dct) + + +[file driver.py] +from native import Nothing +assert Nothing.X == 10 + +[case testPickling] +from mypy_extensions import trait, mypyc_attr +from typing import Any, TypeVar, Generic + +def dec(x: Any) -> Any: + return x + +@mypyc_attr(allow_interpreted_subclasses=True) +class A: + x: int + y: str + +@mypyc_attr(allow_interpreted_subclasses=True) +class B(A): + z: bool + + def __init__(self, x: int, y: str, z: bool) -> None: + self.x = x + self.y = y + self.z = z + +@trait +class T: + a: str + +class C(B, T): + w: object + + # property shouldn't go in + @property + def foo(self) -> int: + return 0 + +@dec +class D: + x: int + +class E(D): + y: int + + +U = TypeVar('U') + +class F(Generic[U]): + y: int + +class G(F[int]): + pass + +[file driver.py] +from native import A, B, T, C, D, E, F, G + +import copy +import pickle + +assert A.__mypyc_attrs__ == ('x', 'y') +assert B.__mypyc_attrs__ == ('z', 'x', 'y') +assert T.__mypyc_attrs__ == ('a',) +assert C.__mypyc_attrs__ == ('w', 'z', 'x', 'y', 'a') +assert not hasattr(D, '__mypyc_attrs__') +assert E.__mypyc_attrs__ == ('y', '__dict__') +assert F.__mypyc_attrs__ == ('y', '__dict__') +assert G.__mypyc_attrs__ == ('y', '__dict__') + +b = B(10, '20', False) +assert b.__getstate__() == {'z': False, 'x': 10, 'y': '20'} +b2 = copy.copy(b) +assert b is not b2 and b.y == b2.y + +b3 = pickle.loads(pickle.dumps(b)) +assert b is not b3 and b.y == b3.y + +e = E() +e.x = 10 +e.y = 20 + +assert e.__getstate__() == {'y': 20, '__dict__': {'x': 10}} +e2 = pickle.loads(pickle.dumps(e)) +assert e is not e2 and e.x == e2.x and e.y == e2.y + + +[case testInterpretedParentInit] +from interp import C +from typing import TypeVar + +T = TypeVar('T') + +def dec(x: T) -> T: + return x + +@dec +class A: + def __init__(self, x: int) -> None: + self.x = x + +class B(A): + s = 'test' + +def b(x: int) -> B: return B(x) + +class D(C): + s = 'test' + +def d(x: int) -> D: return D(x) + + +[file interp.py] +class C: + def __init__(self, x: int) -> None: + self.x = x + +[file driver.py] +from native import b, d, B, D + +def test(f, v): + x = f(v) + assert x.x == v + assert x.s == 'test' + +test(b, 20) +test(d, 30) +test(B, -1) +test(D, -2) + +[case testInterpretedInherit] +from typing import TypeVar, Any, overload +from mypy_extensions import mypyc_attr, trait + +T = TypeVar('T') +def dec(x: T) -> T: return x + +@mypyc_attr(allow_interpreted_subclasses=True) +class Top: + def spam(self) -> str: + return "grandparent" + +@mypyc_attr(allow_interpreted_subclasses=True) +@trait +class Trait: + def trait_method(self) -> str: + return "trait" + +@mypyc_attr(allow_interpreted_subclasses=True) +class Foo(Top, Trait): + def __init__(self, x: int) -> None: + self.x = x + + def foo(self) -> str: + return "parent foo: " + self.bar(self.x) + + def bar(self, x: int) -> str: + return "parent bar: {}".format(x + self.x) + + @dec + def decorated(self) -> str: + return "decorated parent" + + @property + def read_property(self) -> str: + return "parent prop" + + @overload + def overloaded(self, index: int) -> int: ... + + @overload + def overloaded(self, index: str) -> str: ... + + def overloaded(self, index: Any) -> Any: + return index + +def foo(x: Foo) -> str: + return x.foo() + +def bar(x: Foo, y: int) -> str: + return x.bar(y) + +def spam(x: Top) -> str: + return x.spam() + +def decorated(x: Foo) -> str: + return x.decorated() + +def prop(x: Foo) -> str: + return x.read_property + +def trait_method(x: Trait) -> str: + return x.trait_method() + +def overloaded(x: Foo, s: str) -> str: + return x.overloaded(s) + +[file interp.py] +from typing import Any +from native import Foo + +class Bar(Foo): + def bar(self, x: int) -> str: + return "child bar: {}".format(x + self.x) + + def spam(self) -> str: + assert super().spam() == "grandparent" + return "child" + + @property + def read_property(self) -> str: + return "child prop" + + def decorated(self) -> str: + return "decorated child" + + def trait_method(self) -> str: + return "child" + + def overloaded(self, index: Any) -> Any: + return index + index + + +class InterpBase: + def eggs(self) -> str: + return "eggs" + +class Baz(InterpBase, Bar): + def __init__(self) -> None: + super().__init__(1000) + self.z = self.read_property + +[file driver.py] +from native import Foo, foo, bar, spam, decorated, overloaded, prop, trait_method +from interp import Bar, Baz +from unittest.mock import patch +from testutil import assertRaises + +x = Foo(10) +y = Bar(20) +z = Baz() + +assert isinstance(y, Bar) +assert y.x == 20 +assert y.bar(10) == "child bar: 30" +assert y.foo() == "parent foo: child bar: 40" +assert foo(y) == "parent foo: child bar: 40" +assert bar(y, 30) == "child bar: 50" +y.x = 30 +assert bar(y, 30) == "child bar: 60" + +assert spam(y) == "child" +assert y.read_property == "child prop" +assert prop(x) == "parent prop" +assert prop(y) == "child prop" +assert y.decorated() == "decorated child" +assert decorated(y) == "decorated child" +assert y.overloaded("test") == "testtest" +assert overloaded(y, "test") == "testtest" + +assert y.trait_method() == "child" +assert trait_method(y) == "child" + +assert z.bar(10) == "child bar: 1010" +assert bar(z, 10) == "child bar: 1010" +assert z.z == "child prop" +assert z.eggs() == "eggs" + +with patch("interp.Bar.spam", lambda self: "monkey patched"): + assert y.spam() == "monkey patched" + spam(y) == "monkey patched" + +with patch("interp.Bar.spam", lambda self: 20): + assert y.spam() == 20 + with assertRaises(TypeError, "str object expected; got int"): + spam(y) + +with assertRaises(TypeError, "int object expected; got str"): + y.x = "test" + +[case testProperty] +from typing import Callable +from mypy_extensions import trait +class Temperature: + @property + def celsius(self) -> float: + return 5.0 * (self.fahrenheit - 32.0) / 9.0 + + def __init__(self, fahrenheit: float) -> None: + self.fahrenheit = fahrenheit + + def print_temp(self) -> None: + print("F:", self.fahrenheit, "C:", self.celsius) + + @property + def rankine(self) -> float: + raise NotImplementedError + +class Access: + @property + def number_of_accesses(self) -> int: + self._count += 1 + return self._count + def __init__(self) -> None: + self._count = 0 + +from typing import Callable +class BaseProperty: + @property + def doc(self) -> str: + return "Represents a sequence of values. Updates itself by next, which is a new value." + + @property + def value(self) -> object: + return self._incrementer + + @property + def bad_value(self) -> object: + return self._incrementer + + @property + def next(self) -> BaseProperty: + return BaseProperty(self._incrementer + 1) + + def __init__(self, value: int) -> None: + self._incrementer = value + +class DerivedProperty(BaseProperty): + @property + def value(self) -> int: + return self._incrementer + + @property + def bad_value(self) -> object: + return self._incrementer + + def __init__(self, incr_func: Callable[[int], int], value: int) -> None: + BaseProperty.__init__(self, value) + self._incr_func = incr_func + + @property + def next(self) -> DerivedProperty: + return DerivedProperty(self._incr_func, self._incr_func(self.value)) + +class AgainProperty(DerivedProperty): + @property + def next(self) -> AgainProperty: + return AgainProperty(self._incr_func, self._incr_func(self._incr_func(self.value))) + + @property + def bad_value(self) -> int: + return self._incrementer + +def print_first_n(n: int, thing: BaseProperty) -> None: + vals = [] + cur_thing = thing + for _ in range(n): + vals.append(cur_thing.value) + cur_thing = cur_thing.next + print ('', vals) + +@trait +class Trait: + @property + def value(self) -> int: + return 3 + +class Printer(Trait): + def print_value(self) -> None: + print(self.value) + +[file driver.py] +from native import Temperature, Access +import traceback +x = Temperature(32.0) +try: + print (x.rankine) +except NotImplementedError as e: + traceback.print_exc() +print (x.celsius) +x.print_temp() + +y = Temperature(212.0) +print (y.celsius) +y.print_temp() + +z = Access() +print (z.number_of_accesses) +print (z.number_of_accesses) +print (z.number_of_accesses) +print (z.number_of_accesses) + +from native import BaseProperty, DerivedProperty, AgainProperty, print_first_n +a = BaseProperty(7) +b = DerivedProperty((lambda x: x // 2 if (x % 2 == 0) else 3 * x + 1), 7) +c = AgainProperty((lambda x: x // 2 if (x % 2 == 0) else 3 * x + 1), 7) + +def py_print_first_n(n: int, thing: BaseProperty) -> None: + vals = [] + cur_thing = thing + for _ in range(n): + vals.append(cur_thing.value) + cur_thing = cur_thing.next + print ('', vals) + +py_print_first_n(20, a) +py_print_first_n(20, b) +py_print_first_n(20, c) + +print(a.next.next.next.bad_value) +print(b.next.next.next.bad_value) +print(c.next.next.next.bad_value) + +print_first_n(20, a) +print_first_n(20, b) +print_first_n(20, c) + +print (a.doc) +print (b.doc) +print (c.doc) + +from native import Printer +Printer().print_value() +print (Printer().value) +[out] +Traceback (most recent call last): + File "driver.py", line 5, in + print (x.rankine) + File "native.py", line 16, in rankine + raise NotImplementedError +NotImplementedError +0.0 +F: 32.0 C: 0.0 +100.0 +F: 212.0 C: 100.0 +1 +2 +3 +4 + [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + [7, 22, 11, 34, 17, 52, 26, 13, 40, 20, 10, 5, 16, 8, 4, 2, 1, 4, 2, 1] + [7, 11, 17, 26, 40, 10, 16, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4] +10 +34 +26 + [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + [7, 22, 11, 34, 17, 52, 26, 13, 40, 20, 10, 5, 16, 8, 4, 2, 1, 4, 2, 1] + [7, 11, 17, 26, 40, 10, 16, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4] +Represents a sequence of values. Updates itself by next, which is a new value. +Represents a sequence of values. Updates itself by next, which is a new value. +Represents a sequence of values. Updates itself by next, which is a new value. +3 +3 +[out version>=3.11] +Traceback (most recent call last): + File "driver.py", line 5, in + print (x.rankine) + ^^^^^^^^^ + File "native.py", line 16, in rankine + raise NotImplementedError +NotImplementedError +0.0 +F: 32.0 C: 0.0 +100.0 +F: 212.0 C: 100.0 +1 +2 +3 +4 + [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + [7, 22, 11, 34, 17, 52, 26, 13, 40, 20, 10, 5, 16, 8, 4, 2, 1, 4, 2, 1] + [7, 11, 17, 26, 40, 10, 16, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4] +10 +34 +26 + [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + [7, 22, 11, 34, 17, 52, 26, 13, 40, 20, 10, 5, 16, 8, 4, 2, 1, 4, 2, 1] + [7, 11, 17, 26, 40, 10, 16, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4] +Represents a sequence of values. Updates itself by next, which is a new value. +Represents a sequence of values. Updates itself by next, which is a new value. +Represents a sequence of values. Updates itself by next, which is a new value. +3 +3 + +[case testPropertySetters] + +from mypy_extensions import trait + +class Foo(): + def __init__(self) -> None: + self.attr = "unmodified" + +class A: + def __init__(self) -> None: + self._x = 0 + self._foo = Foo() + + @property + def x(self) -> int: + return self._x + + @x.setter + def x(self, val : int) -> None: + self._x = val + + @property + def foo(self) -> Foo: + return self._foo + + @foo.setter + def foo(self, val : Foo) -> None: + self._foo = val + +# Overrides base property setters and getters +class B(A): + def __init__(self) -> None: + self._x = 10 + + @property + def x(self) -> int: + return self._x + 1 + + @x.setter + def x(self, val : int) -> None: + self._x = val + 1 + +# Inherits base property setters and getters +class C(A): + def __init__(self) -> None: + A.__init__(self) + +@trait +class D(): + def __init__(self) -> None: + self._x = 0 + + @property + def x(self) -> int: + return self._x + + @x.setter + def x(self, val : int) -> None: + self._x = val + +#Inherits trait property setters and getters +class E(D): + def __init__(self) -> None: + D.__init__(self) + +#Overrides trait property setters and getters +class F(D): + def __init__(self) -> None: + self._x = 10 + + @property + def x(self) -> int: + return self._x + 10 + + @x.setter + def x(self, val : int) -> None: + self._x = val + 10 + +# # Property setter and getter are subtypes of base property setters and getters +# # class G(A): +# # def __init__(self) -> None: +# # A.__init__(self) + +# # @property +# # def y(self) -> int: +# # return self._y + +# # @y.setter +# # def y(self, val : object) -> None: +# # self._y = val + +# No inheritance, just plain setter/getter +class G: + def __init__(self, x: int) -> None: + self._x = x + + @property + def x(self) -> int: + return self._x + + @x.setter + def x(self, x: int) -> None: + self._x = x + +class H: + def __init__(self, g: G) -> None: + self.g = g + self.g.x = 5 # Should not be treated as initialization + +[file other.py] +# Run in both interpreted and compiled mode + +from native import A, B, C, D, E, F, G + +a = A() +assert a.x == 0 +assert a._x == 0 +a.x = 1 +assert a.x == 1 +assert a._x == 1 +a._x = 0 +assert a.x == 0 +assert a._x == 0 +b = B() +assert b.x == 11 +assert b._x == 10 +b.x = 11 +assert b.x == 13 +b._x = 11 +assert b.x == 12 +c = C() +assert c.x == 0 +c.x = 1000 +assert c.x == 1000 +e = E() +assert e.x == 0 +e.x = 1000 +assert e.x == 1000 +f = F() +assert f.x == 20 +f.x = 30 +assert f.x == 50 +g = G(4) +g.x = 20 +assert g.x == 20 + +[file driver.py] +# Run the tests in both interpreted and compiled mode +import other +import other_interpreted + +[out] + +[case testAttributeOverridesProperty] +from typing import Any +from mypy_extensions import trait + +@trait +class T1: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + +class C1(T1): + x: int = 1 + y: int = 4 + +def test_read_only_property_in_trait_implemented_as_attribute() -> None: + c = C1() + c.x = 5 + assert c.x == 5 + assert c.y == 4 + c.y = 6 + assert c.y == 6 + t: T1 = C1() + assert t.y == 4 + t = c + assert t.x == 5 + assert t.y == 6 + a: Any = c + assert a.x == 5 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +class B2: + @property + def x(self) -> int: + return 11 + + @property + def y(self) -> int: + return 25 + +class C2(B2): + x: int = 1 + y: int = 4 + +def test_read_only_property_in_class_implemented_as_attribute() -> None: + c = C2() + c.x = 5 + assert c.x == 5 + assert c.y == 4 + c.y = 6 + assert c.y == 6 + b: B2 = C2() + assert b.y == 4 + b = c + assert b.x == 5 + assert b.y == 6 + a: Any = c + assert a.x == 5 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +@trait +class T3: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + +class B3: + x: int = 1 + y: int = 4 + +class C3(B3, T3): + pass + +def test_read_only_property_implemented_as_attribute_indirectly() -> None: + c = C3() + c.x = 5 + assert c.x == 5 + assert c.y == 4 + c.y = 6 + assert c.y == 6 + t: T3 = C3() + assert t.y == 4 + t = c + assert t.x == 5 + assert t.y == 6 + a: Any = c + assert a.x == 5 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +@trait +class T4: + @property + def x(self) -> int: ... + @x.setter + def x(self, v1: int) -> None: ... + + @property + def y(self) -> int: ... + @y.setter + def y(self, v2: int) -> None: ... + +class C4(T4): + x: int = 1 + y: int = 4 + +def test_read_write_property_implemented_as_attribute() -> None: + c = C4() + c.x = 5 + assert c.x == 5 + assert c.y == 4 + c.y = 6 + assert c.y == 6 + t: T4 = C4() + assert t.y == 4 + t.x = 5 + assert t.x == 5 + t.y = 6 + assert t.y == 6 + a: Any = c + assert a.x == 5 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +@trait +class T5: + @property + def x(self) -> int: ... + @x.setter + def x(self, v1: int) -> None: ... + + @property + def y(self) -> int: ... + @y.setter + def y(self, v2: int) -> None: ... + +class B5: + x: int = 1 + y: int = 4 + +class BB5(B5): + pass + +class C5(BB5, T5): + pass + +def test_read_write_property_indirectly_implemented_as_attribute() -> None: + c = C5() + c.x = 5 + assert c.x == 5 + assert c.y == 4 + c.y = 6 + assert c.y == 6 + t: T5 = C5() + assert t.y == 4 + t.x = 5 + assert t.x == 5 + t.y = 6 + assert t.y == 6 + a: Any = c + assert a.x == 5 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +[case testSubclassAttributeAccess] +from mypy_extensions import trait + +class A: + v = 0 + +class B(A): + v = 1 + +class C(B): + v = 2 + +[file driver.py] +from native import A, B, C + +a = A() +b = B() +c = C() + +[case testCopyAlwaysDefinedAttributes] +import copy +from typing import Union + +class A: pass + +class C: + def __init__(self, n: int = 0) -> None: + self.n = n + self.s = "" + self.t = ("", 0) + self.u: Union[str, bytes] = '' + self.a = A() + +def test_copy() -> None: + c1 = C() + c1.n = 1 + c1.s = "x" + c2 = copy.copy(c1) + assert c2.n == 1 + assert c2.s == "x" + assert c2.t == ("", 0) + assert c2.u == '' + assert c2.a is c1.a + +[case testNonNativeCallsToDunderNewAndInit] +from typing import Any +from testutil import assertRaises + +count_c = 0 + +class C: + def __init__(self) -> None: + self.x = 'a' # Always defined attribute + global count_c + count_c += 1 + + def get(self) -> str: + return self.x + +def test_no_init_args() -> None: + global count_c + count_c = 0 + + # Use Any to get non-native semantics + cls: Any = C + # __new__ implicitly calls __init__ for native classes + obj = cls.__new__(cls) + assert obj.get() == 'a' + assert count_c == 1 + # Make sure we don't call __init__ twice + obj2 = cls() + assert obj2.get() == 'a' + assert count_c == 2 + +count_d = 0 + +class D: + def __init__(self, x: str) -> None: + self.x = x # Always defined attribute + global count_d + count_d += 1 + + def get(self) -> str: + return self.x + +def test_init_arg() -> None: + global count_d + count_d = 0 + + # Use Any to get non-native semantics + cls: Any = D + # __new__ implicitly calls __init__ for native classes + obj = cls.__new__(cls, 'abc') + assert obj.get() == 'abc' + assert count_d == 1 + # Make sure we don't call __init__ twice + obj2 = cls('x') + assert obj2.get() == 'x' + assert count_d == 2 + # Keyword args should work + obj = cls.__new__(cls, x='abc') + assert obj.get() == 'abc' + assert count_d == 3 + +def test_invalid_init_args() -> None: + # Use Any to get non-native semantics + cls: Any = D + with assertRaises(TypeError): + cls() + with assertRaises(TypeError): + cls(y='x') + with assertRaises(TypeError): + cls(1) + +[case testTryDeletingAlwaysDefinedAttribute] +from typing import Any +from testutil import assertRaises + +class C: + def __init__(self) -> None: + self.x = 0 + +class D(C): + pass + +def test_try_deleting_always_defined_attr() -> None: + c: Any = C() + with assertRaises(AttributeError): + del c.x + d: Any = D() + with assertRaises(AttributeError): + del d.x + +[case testAlwaysDefinedAttributeAndAllowInterpretedSubclasses] +from mypy_extensions import mypyc_attr + +from m import define_interpreted_subclass + +@mypyc_attr(allow_interpreted_subclasses=True) +class Base: + x = 5 + y: int + def __init__(self, s: str) -> None: + self.s = s + +class DerivedNative(Base): + def __init__(self) -> None: + super().__init__('x') + self.z = 3 + +def test_native_subclass() -> None: + o = DerivedNative() + assert o.x == 5 + assert o.s == 'x' + assert o.z == 3 + +def test_interpreted_subclass() -> None: + define_interpreted_subclass(Base) + +[file m.py] +from testutil import assertRaises + +def define_interpreted_subclass(b): + class DerivedInterpreted1(b): + def __init__(self): + # Don't call base class __init__ + pass + d1 = DerivedInterpreted1() + assert d1.x == 5 + with assertRaises(AttributeError): + d1.y + with assertRaises(AttributeError): + d1.s + with assertRaises(AttributeError): + del d1.x + + class DerivedInterpreted1(b): + def __init__(self): + super().__init__('y') + d2 = DerivedInterpreted1() + assert d2.x == 5 + assert d2.s == 'y' + with assertRaises(AttributeError): + d2.y + with assertRaises(AttributeError): + del d2.x + +[case testBaseClassSometimesDefinesAttribute] +class C: + def __init__(self, b: bool) -> None: + if b: + self.x = [1] + +class D(C): + def __init__(self, b: bool) -> None: + super().__init__(b) + self.x = [2] + +def test_base_class() -> None: + c = C(True) + assert c.x == [1] + c = C(False) + try: + c.x + except AttributeError: + return + assert False + +def test_subclass() -> None: + d = D(True) + assert d.x == [2] + d = D(False) + assert d.x == [2] + +[case testSerializableClass] +from mypy_extensions import mypyc_attr +from typing import Any +import copy +from testutil import assertRaises + +@mypyc_attr(serializable=True) +class Base: + def __init__(self, s: str) -> None: + self.s = s + +class Derived(Base): + def __init__(self, s: str, n: int) -> None: + super().__init__(s) + self.n = n + +def test_copy_base() -> None: + o = Base('xyz') + o2 = copy.copy(o) + assert isinstance(o2, Base) + assert o2 is not o + assert o2.s == 'xyz' + +def test_copy_derived() -> None: + d = Derived('xyz', 5) + d2 = copy.copy(d) + assert isinstance(d2, Derived) + assert d2 is not d + assert d2.s == 'xyz' + assert d2.n == 5 + +class NonSerializable: + def __init__(self, s: str) -> None: + self.s = s + +@mypyc_attr(serializable=True) +class SerializableSub(NonSerializable): + def __init__(self, s: str, n: int) -> None: + super().__init__(s) + self.n = n + +def test_serializable_sub_class() -> None: + n = NonSerializable('xyz') + assert n.s == 'xyz' + + with assertRaises(TypeError): + copy.copy(n) + + s = SerializableSub('foo', 6) + s2 = copy.copy(s) + assert s2 is not s + assert s2.s == 'foo' + assert s2.n == 6 + +def test_serializable_sub_class_call_new() -> None: + t: Any = SerializableSub + sub: SerializableSub = t.__new__(t) + with assertRaises(AttributeError): + sub.s + with assertRaises(AttributeError): + sub.n + base: NonSerializable = sub + with assertRaises(AttributeError): + base.s + +[case testClassWithInherited__call__] +class Base: + def __call__(self) -> int: + return 1 + +class Derived(Base): + pass + +assert Derived()() == 1 + +[case testClassWithFinalAttribute] +from typing_extensions import Final + +class C: + A: Final = -1 + a: Final = [A] + +def test_final_attribute() -> None: + assert C.A == -1 + assert C.a == [-1] + +[case testClassWithFinalDecorator] +from typing import final + +@final +class C: + def a(self) -> int: + return 1 + +def test_class_final_attribute() -> None: + assert C().a() == 1 + + +[case testClassWithFinalDecoratorCtor] +from typing import final + +@final +class C: + def __init__(self) -> None: + self.a = 1 + + def b(self) -> int: + return 2 + + @property + def c(self) -> int: + return 3 + +def test_class_final_attribute() -> None: + assert C().a == 1 + assert C().b() == 2 + assert C().c == 3 + +[case testClassWithFinalDecoratorInheritedWithProperties] +from typing import final + +class B: + def a(self) -> int: + return 2 + + @property + def b(self) -> int: + return self.a() + 2 + + @property + def c(self) -> int: + return 3 + +def test_class_final_attribute_basic() -> None: + assert B().a() == 2 + assert B().b == 4 + assert B().c == 3 + +@final +class C(B): + def a(self) -> int: + return 1 + + @property + def b(self) -> int: + return self.a() + 1 + +def fn(cl: B) -> int: + return cl.a() + +def test_class_final_attribute_inherited() -> None: + assert C().a() == 1 + assert fn(C()) == 1 + assert B().a() == 2 + assert fn(B()) == 2 + + assert B().b == 4 + assert C().b == 2 + assert B().c == 3 + assert C().c == 3 + +[case testClassWithFinalAttributeAccess] +from typing import Final + +class C: + a: Final = {'x': 'y'} + b: Final = C.a + +def test_final_attribute() -> None: + assert C.a['x'] == 'y' + assert C.b['x'] == 'y' + assert C.a is C.b + +[case testClassDerivedFromIntEnum] +from enum import IntEnum, auto + +class Player(IntEnum): + MIN = auto() + +print(f'{Player.MIN = }') +[file driver.py] +from native import Player +[out] +Player.MIN = + +[case testStaticCallsWithUnpackingArgs] +from typing import Tuple + +class Foo: + @staticmethod + def static(a: int, b: int, c: int) -> Tuple[int, int, int]: + return (c+1, a+2, b+3) + + @classmethod + def clsmethod(cls, a: int, b: int, c: int) -> Tuple[int, int, int]: + return (c+1, a+2, b+3) + + +print(Foo.static(*[10, 20, 30])) +print(Foo.static(*(40, 50), *[60])) +assert Foo.static(70, 80, *[90]) == Foo.clsmethod(70, *(80, 90)) + +[file driver.py] +import native + +[out] +(31, 12, 23) +(61, 42, 53) diff --git a/mypyc/test-data/run-dicts.test b/mypyc/test-data/run-dicts.test new file mode 100644 index 000000000000..d4f5b945309e --- /dev/null +++ b/mypyc/test-data/run-dicts.test @@ -0,0 +1,338 @@ +# Test cases for dicts (compile and run) + +[case testDictStuff] +from typing import Dict, Any, List, Set, Tuple +from defaultdictwrap import make_dict + +def f(x: int) -> int: + dict1 = {} # type: Dict[int, int] + dict1[1] = 1 + dict2 = {} # type: Dict[int, int] + dict2[x] = 2 + dict1.update(dict2) + + l = [(5, 2)] # type: Any + dict1.update(l) + d2 = {6: 4} # type: Any + dict1.update(d2) + + return dict1[1] + +def g() -> int: + d = make_dict() + d['a'] = 10 + d['a'] += 10 + d['b'] += 10 + l = [('c', 2)] # type: Any + d.update(l) + d2 = {'d': 4} # type: Any + d.update(d2) + return d['a'] + d['b'] + +def h() -> None: + d = {} # type: Dict[Any, Any] + d[{}] + +def update_dict(x: Dict[Any, Any], y: Any): + x.update(y) + +def make_dict1(x: Any) -> Dict[Any, Any]: + return dict(x) + +def make_dict2(x: Dict[Any, Any]) -> Dict[Any, Any]: + return dict(x) + +def u(x: int) -> int: + d = {} # type: Dict[str, int] + d.update(x=x) + return d['x'] + +def get_content(d: Dict[int, int]) -> Tuple[List[int], List[int], List[Tuple[int, int]]]: + return list(d.keys()), list(d.values()), list(d.items()) + +def get_content_set(d: Dict[int, int]) -> Tuple[Set[int], Set[int], Set[Tuple[int, int]]]: + return set(d.keys()), set(d.values()), set(d.items()) +[file defaultdictwrap.py] +from typing import Dict +from collections import defaultdict # type: ignore +def make_dict() -> Dict[str, int]: + return defaultdict(int) + +[file driver.py] +from collections import OrderedDict +from native import ( + f, g, h, u, make_dict1, make_dict2, update_dict, get_content, get_content_set +) +assert f(1) == 2 +assert f(2) == 1 +assert g() == 30 +# Make sure we get a TypeError from indexing with unhashable and not KeyError +try: + h() +except TypeError: + pass +else: + assert False +d = {'a': 1, 'b': 2} +assert make_dict1(d) == d +assert make_dict1(d.items()) == d +assert make_dict2(d) == d +# object.__dict__ is a "mappingproxy" and not a dict +assert make_dict1(object.__dict__) == dict(object.__dict__) +d = {} +update_dict(d, object.__dict__) +assert d == dict(object.__dict__) + +assert u(10) == 10 +assert get_content({1: 2}) == ([1], [2], [(1, 2)]) +od = OrderedDict([(1, 2), (3, 4)]) +assert get_content(od) == ([1, 3], [2, 4], [(1, 2), (3, 4)]) +od.move_to_end(1) +assert get_content(od) == ([3, 1], [4, 2], [(3, 4), (1, 2)]) +assert get_content_set({1: 2}) == ({1}, {2}, {(1, 2)}) +assert get_content_set(od) == ({1, 3}, {2, 4}, {(1, 2), (3, 4)}) + +[typing fixtures/typing-full.pyi] + +[case testDictIterationMethodsRun] +from typing import Dict, Union +from typing_extensions import TypedDict + +class ExtensionDict(TypedDict): + python: str + c: str + +def print_dict_methods(d1: Dict[int, int], + d2: Dict[int, int], + d3: Dict[int, int]) -> None: + for k in d1.keys(): + print(k) + for k, v in d2.items(): + print(k) + print(v) + for v in d3.values(): + print(v) + +def print_dict_methods_special(d1: Union[Dict[int, int], Dict[str, str]], + d2: ExtensionDict) -> None: + for k in d1.keys(): + print(k) + for k, v in d1.items(): + print(k) + print(v) + for v2 in d2.values(): + print(v2) + for k2, v2 in d2.items(): + print(k2) + print(v2) + + +def clear_during_iter(d: Dict[int, int]) -> None: + for k in d: + d.clear() + +class Custom(Dict[int, int]): pass +[file driver.py] +from native import print_dict_methods, print_dict_methods_special, Custom, clear_during_iter +from collections import OrderedDict +print_dict_methods({}, {}, {}) +print_dict_methods({1: 2}, {3: 4, 5: 6}, {7: 8}) +print('==') +c = Custom({0: 1}) +print_dict_methods(c, c, c) +print('==') +d = OrderedDict([(1, 2), (3, 4)]) +print_dict_methods(d, d, d) +print('==') +print_dict_methods_special({1: 2}, {"python": ".py", "c": ".c"}) +d.move_to_end(1) +print_dict_methods(d, d, d) +clear_during_iter({}) # OK +try: + clear_during_iter({1: 2, 3: 4}) +except RuntimeError as e: + assert str(e) == "dictionary changed size during iteration" +else: + assert False +try: + clear_during_iter(d) +except RuntimeError as e: + assert str(e) in ( + "OrderedDict changed size during iteration", + # Error message changed in Python 3.13 and some 3.12 patch version + "OrderedDict mutated during iteration", + ) +else: + assert False + +class CustomMad(dict): + def __iter__(self): + return self + def __next__(self): + raise ValueError +m = CustomMad() +try: + clear_during_iter(m) +except ValueError: + pass +else: + assert False + +class CustomBad(dict): + def items(self): + return [(1, 2, 3)] # Oops +b = CustomBad() +try: + print_dict_methods(b, b, b) +except TypeError as e: + assert str(e) == "a tuple of length 2 expected" +else: + assert False +[out] +1 +3 +4 +5 +6 +8 +== +0 +0 +1 +1 +== +1 +3 +1 +2 +3 +4 +2 +4 +== +1 +1 +2 +.py +.c +python +.py +c +.c +3 +1 +3 +4 +1 +2 +4 +2 + +[case testDictMethods] +from collections import defaultdict +from typing import Dict, Optional, List, Set + +def test_dict_clear() -> None: + d = {'a': 1, 'b': 2} + d.clear() + assert d == {} + dd: Dict[str, int] = defaultdict(int) + dd['a'] = 1 + dd.clear() + assert dd == {} + +def test_dict_copy() -> None: + d: Dict[str, int] = {} + assert d.copy() == d + d = {'a': 1, 'b': 2} + assert d.copy() == d + assert d.copy() is not d + dd: Dict[str, int] = defaultdict(int) + dd['a'] = 1 + assert dd.copy() == dd + assert isinstance(dd.copy(), defaultdict) + +class MyDict(dict): + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + + def setdefault(self, k, v=None): + if v is None: + if k in self.keys(): + return self[k] + else: + return None + else: + return super().setdefault(k, v) + 10 + +def test_dict_setdefault() -> None: + d: Dict[str, Optional[int]] = {'a': 1, 'b': 2} + assert d.setdefault('a', 2) == 1 + assert d.setdefault('b', 2) == 2 + assert d.setdefault('c', 3) == 3 + assert d['a'] == 1 + assert d['c'] == 3 + assert d.setdefault('a') == 1 + assert d.setdefault('e') == None + assert d.setdefault('e', 100) == None + +def test_dict_subclass_setdefault() -> None: + d = MyDict() + d['a'] = 1 + assert d.setdefault('a', 2) == 11 + assert d.setdefault('b', 2) == 12 + assert d.setdefault('c', 3) == 13 + assert d['a'] == 1 + assert d['c'] == 3 + assert d.setdefault('a') == 1 + assert d.setdefault('e') == None + assert d.setdefault('e', 100) == 110 + +def test_dict_empty_collection_setdefault() -> None: + d1: Dict[str, List[int]] = {'a': [1, 2, 3]} + assert d1.setdefault('a', []) == [1, 2, 3] + assert d1.setdefault('b', []) == [] + assert 'b' in d1 + d1.setdefault('b', []).append(3) + assert d1['b'] == [3] + assert d1.setdefault('c', [1]) == [1] + + d2: Dict[str, Dict[str, int]] = {'a': {'a': 1}} + assert d2.setdefault('a', {}) == {'a': 1} + assert d2.setdefault('b', {}) == {} + assert 'b' in d2 + d2.setdefault('b', {})['aa'] = 2 + d2.setdefault('b', {})['bb'] = 3 + assert d2['b'] == {'aa': 2, 'bb': 3} + assert d2.setdefault('c', {'cc': 1}) == {'cc': 1} + + d3: Dict[str, Set[str]] = {'a': set('a')} + assert d3.setdefault('a', set()) == {'a'} + assert d3.setdefault('b', set()) == set() + d3.setdefault('b', set()).add('b') + d3.setdefault('b', set()).add('c') + assert d3['b'] == {'b', 'c'} + assert d3.setdefault('c', set('d')) == {'d'} + +[case testDictToBool] +from typing import Dict, List + +def is_true(x: dict) -> bool: + if x: + return True + else: + return False + +def is_false(x: dict) -> bool: + if not x: + return True + else: + return False + +def test_dict_to_bool() -> None: + assert is_false({}) + assert not is_true({}) + tmp_list: List[Dict] = [{2: bool}, {'a': 'b'}] + for x in tmp_list: + assert is_true(x) + assert not is_false(x) diff --git a/mypyc/test-data/run-dunders-special.test b/mypyc/test-data/run-dunders-special.test new file mode 100644 index 000000000000..cd02cca65eef --- /dev/null +++ b/mypyc/test-data/run-dunders-special.test @@ -0,0 +1,9 @@ +[case testDundersNotImplemented] +# This case is special because it tests the behavior of NotImplemented +# used in a typed function which return type is bool. +# This is a convention that can be overriden by the user. +class UsesNotImplemented: + def __eq__(self, b: object) -> bool: + return NotImplemented + +assert UsesNotImplemented() != object() diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test new file mode 100644 index 000000000000..b8fb13c9dcec --- /dev/null +++ b/mypyc/test-data/run-dunders.test @@ -0,0 +1,967 @@ +# Test cases for (some) dunder methods (compile and run) + +[case testDundersMisc] +# Legacy test case for dunders (don't add more here) + +from typing import Any +class Item: + def __init__(self, value: str) -> None: + self.value = value + + def __hash__(self) -> int: + return hash(self.value) + + def __eq__(self, rhs: object) -> bool: + return isinstance(rhs, Item) and self.value == rhs.value + + def __lt__(self, x: 'Item') -> bool: + return self.value < x.value + +class Subclass1(Item): + def __bool__(self) -> bool: + return bool(self.value) + +class NonBoxedThing: + def __getitem__(self, index: Item) -> Item: + return Item("2 * " + index.value + " + 1") + +class BoxedThing: + def __getitem__(self, index: int) -> int: + return 2 * index + 1 + +class Subclass2(BoxedThing): + pass + +def index_into(x : Any, y : Any) -> Any: + return x[y] + +def internal_index_into() -> None: + x = BoxedThing() + print (x[3]) + y = NonBoxedThing() + z = Item("3") + print(y[z].value) + +def is_truthy(x: Item) -> bool: + return True if x else False + +[file driver.py] +from native import * +x = BoxedThing() +y = 3 +print(x[y], index_into(x, y)) + +x = Subclass2() +y = 3 +print(x[y], index_into(x, y)) + +z = NonBoxedThing() +w = Item("3") +print(z[w].value, index_into(z, w).value) + +i1 = Item('lolol') +i2 = Item('lol' + 'ol') +i3 = Item('xyzzy') +assert hash(i1) == hash(i2) + +assert i1 == i2 +assert not i1 != i2 +assert not i1 == i3 +assert i1 != i3 +assert i2 < i3 +assert not i1 < i2 +assert i1 == Subclass1('lolol') + +assert is_truthy(Item('')) +assert is_truthy(Item('a')) +assert not is_truthy(Subclass1('')) +assert is_truthy(Subclass1('a')) + +internal_index_into() +[out] +7 7 +7 7 +2 * 3 + 1 2 * 3 + 1 +7 +2 * 3 + 1 + +[case testDundersContainer] +# Sequence/mapping dunder methods + +from typing import Any + +class Seq: + def __init__(self) -> None: + self.key = 0 + self.value = 0 + + def __len__(self) -> int: + return 5 + + def __setitem__(self, key: int, value: int) -> None: + self.key = key + self.value = value + + def __contains__(self, x: int) -> bool: + return x == 3 + + def __delitem__(self, key: int) -> None: + self.key = key + +class Plain: pass + +def any_seq() -> Any: + """Return Any-typed Seq.""" + return Seq() + +def any_plain() -> Any: + """Return Any-typed Seq.""" + return Plain() + +def test_len() -> None: + assert len(any_seq()) == 5 + assert len(Seq()) == 5 + +def test_len_error() -> None: + try: + len(any_plain()) + except TypeError: + pass + else: + assert False + +def test_set_item() -> None: + s = any_seq() + s[44] = 66 + assert s.key == 44 and s.value == 66 + ss = Seq() + ss[33] = 55 + assert ss.key == 33 and ss.value == 55 + +def test_contains() -> None: + assert 3 in any_seq() + assert 4 not in any_seq() + assert 2 not in any_seq() + assert 3 in Seq() + assert 4 not in Seq() + assert 2 not in Seq() + +def test_delitem() -> None: + s = any_seq() + del s[55] + assert s.key == 55 + +class SeqAny: + def __contains__(self, x: Any) -> Any: + return x == 3 + + def __setitem__(self, x: Any, y: Any) -> Any: + self.x = x + return 'x' + +def test_contains_any() -> None: + assert (3 in SeqAny()) is True + assert (2 in SeqAny()) is False + assert (3 not in SeqAny()) is False + assert (2 not in SeqAny()) is True + s = SeqAny() # type: Any + assert (3 in s) is True + assert (2 in s) is False + assert (3 not in s) is False + assert (2 not in s) is True + +def test_set_item_any() -> None: + s = SeqAny() + s[4] = 6 + assert s.x == 4 + ss = SeqAny() # type: Any + ss[5] = 7 + assert ss.x == 5 + +class SeqError: + def __setitem__(self, key: int, value: int) -> None: + raise RuntimeError() + + def __contains__(self, x: int) -> bool: + raise RuntimeError() + + def __len__(self): + return -5 + +def any_seq_error() -> Any: + return SeqError() + +def test_set_item_error_propagate() -> None: + s = any_seq_error() + try: + s[44] = 66 + except RuntimeError: + pass + else: + assert False + +def test_contains_error_propagate() -> None: + s = any_seq_error() + try: + 3 in s + except RuntimeError: + pass + else: + assert False + +def test_negative_len() -> None: + try: + len(SeqError()) + except ValueError: + pass + else: + assert False + +class DelItemNoSetItem: + def __delitem__(self, x: int) -> None: + self.key = x + +def test_del_item_with_no_set_item() -> None: + o = DelItemNoSetItem() + del o[22] + assert o.key == 22 + a = o # type: Any + del a[12] + assert a.key == 12 + try: + a[1] = 2 + except TypeError as e: + assert str(e) == "'DelItemNoSetItem' object does not support item assignment" + else: + assert False + +class SetItemOverride(dict): + # Only override __setitem__, __delitem__ comes from dict + + def __setitem__(self, x: int, y: int) -> None: + self.key = x + self.value = y + +def test_set_item_override() -> None: + o = SetItemOverride({'x': 12, 'y': 13}) + o[2] = 3 + assert o.key == 2 and o.value == 3 + a = o # type: Any + o[4] = 5 + assert o.key == 4 and o.value == 5 + assert o['x'] == 12 + assert o['y'] == 13 + del o['x'] + assert 'x' not in o and 'y' in o + del a['y'] + assert 'y' not in a and 'x' not in a + +class DelItemOverride(dict): + # Only override __delitem__, __setitem__ comes from dict + + def __delitem__(self, x: int) -> None: + self.key = x + +def test_del_item_override() -> None: + o = DelItemOverride() + del o[2] + assert o.key == 2 + a = o # type: Any + del o[5] + assert o.key == 5 + o['x'] = 12 + assert o['x'] == 12 + a['y'] = 13 + assert a['y'] == 13 + +class SetItemOverrideNative(Seq): + def __setitem__(self, key: int, value: int) -> None: + self.key = key + 1 + self.value = value + 1 + +def test_native_set_item_override() -> None: + o = SetItemOverrideNative() + o[1] = 4 + assert o.key == 2 and o.value == 5 + del o[6] + assert o.key == 6 + a = o # type: Any + a[10] = 12 + assert a.key == 11 and a.value == 13 + del a[16] + assert a.key == 16 + +class DelItemOverrideNative(Seq): + def __delitem__(self, key: int) -> None: + self.key = key + 2 + +def test_native_del_item_override() -> None: + o = DelItemOverrideNative() + o[1] = 4 + assert o.key == 1 and o.value == 4 + del o[6] + assert o.key == 8 + a = o # type: Any + a[10] = 12 + assert a.key == 10 and a.value == 12 + del a[16] + assert a.key == 18 + +[case testDundersNumber] +from typing import Any + +class C: + def __init__(self, x: int) -> None: + self.x = x + + def __neg__(self) -> int: + return self.x + 1 + + def __invert__(self) -> int: + return self.x + 2 + + def __int__(self) -> int: + return self.x + 3 + + def __float__(self) -> float: + return float(self.x + 4) + + def __pos__(self) -> int: + return self.x + 5 + + def __abs__(self) -> int: + return abs(self.x) + 6 + + +def test_unary_dunders_generic() -> None: + a: Any = C(10) + + assert -a == 11 + assert ~a == 12 + assert int(a) == 13 + assert float(a) == 14.0 + assert +a == 15 + assert abs(a) == 16 + +def test_unary_dunders_native() -> None: + c = C(10) + + assert -c == 11 + assert ~c == 12 + assert int(c) == 13 + assert float(c) == 14.0 + assert +c == 15 + assert abs(c) == 16 + +[case testDundersBinarySimple] +from typing import Any + +class C: + def __init__(self) -> None: + self.x = 5 + + def __add__(self, y: int) -> int: + return self.x + y + + def __sub__(self, y: int) -> int: + return self.x - y + + def __mul__(self, y: int) -> int: + return self.x * y + + def __mod__(self, y: int) -> int: + return self.x % y + + def __lshift__(self, y: int) -> int: + return self.x << y + + def __rshift__(self, y: int) -> int: + return self.x >> y + + def __and__(self, y: int) -> int: + return self.x & y + + def __or__(self, y: int) -> int: + return self.x | y + + def __xor__(self, y: int) -> int: + return self.x ^ y + + def __matmul__(self, y: int) -> int: + return self.x + y + 10 + + def __truediv__(self, y: int) -> int: + return self.x + y + 20 + + def __floordiv__(self, y: int) -> int: + return self.x + y + 30 + + def __divmod__(self, y: int) -> int: + return self.x + y + 40 + + def __pow__(self, y: int) -> int: + return self.x + y + 50 + +def test_generic() -> None: + a: Any = C() + assert a + 3 == 8 + assert a - 3 == 2 + assert a * 5 == 25 + assert a % 2 == 1 + assert a << 4 == 80 + assert a >> 0 == 5 + assert a >> 1 == 2 + assert a & 1 == 1 + assert a | 3 == 7 + assert a ^ 3 == 6 + assert a @ 3 == 18 + assert a / 2 == 27 + assert a // 2 == 37 + assert divmod(a, 2) == 47 + assert a ** 2 == 57 + +def test_native() -> None: + c = C() + assert c + 3 == 8 + assert c - 3 == 2 + assert divmod(c, 3) == 48 + assert c ** 3 == 58 + +def test_error() -> None: + a: Any = C() + try: + a + 'x' + except TypeError as e: + assert str(e) == "unsupported operand type(s) for +: 'C' and 'str'" + else: + assert False + try: + a - 'x' + except TypeError as e: + assert str(e) == "unsupported operand type(s) for -: 'C' and 'str'" + else: + assert False + try: + a ** 'x' + except TypeError as e: + assert str(e) == "unsupported operand type(s) for **: 'C' and 'str'" + else: + assert False + +[case testDundersBinaryReverse] +from typing import Any + +class C: + def __init__(self) -> None: + self.x = 5 + + def __add__(self, y: int) -> int: + return self.x + y + + def __radd__(self, y: int) -> int: + return self.x + y + 1 + + def __sub__(self, y: int) -> int: + return self.x - y + + def __rsub__(self, y: int) -> int: + return self.x - y - 1 + + def __pow__(self, y: int) -> int: + return self.x**y + + def __rpow__(self, y: int) -> int: + return self.x**y + 1 + +def test_generic() -> None: + a: Any = C() + assert a + 3 == 8 + assert 4 + a == 10 + assert a - 3 == 2 + assert 4 - a == 0 + assert a**3 == 125 + assert 4**a == 626 + +def test_native() -> None: + c = C() + assert c + 3 == 8 + assert 4 + c == 10 + assert c - 3 == 2 + assert 4 - c == 0 + assert c**3 == 125 + assert 4**c == 626 + +def test_errors() -> None: + a: Any = C() + try: + a + 'x' + except TypeError as e: + assert str(e) == "unsupported operand type(s) for +: 'C' and 'str'" + else: + assert False + try: + a - 'x' + except TypeError as e: + assert str(e) == "unsupported operand type(s) for -: 'C' and 'str'" + else: + assert False + try: + 'x' + a + except TypeError as e: + assert str(e) in ('can only concatenate str (not "C") to str', + 'must be str, not C') + else: + assert False + try: + 'x' ** a + except TypeError as e: + assert str(e) == "unsupported operand type(s) for ** or pow(): 'str' and 'C'" + else: + assert False + + +class F: + def __add__(self, x: int) -> int: + return 5 + + def __pow__(self, x: int) -> int: + return -5 + +class G: + def __add__(self, x: int) -> int: + return 33 + + def __pow__(self, x: int) -> int: + return -33 + + def __radd__(self, x: F) -> int: + return 6 + + def __rpow__(self, x: F) -> int: + return -6 + +def test_type_mismatch_fall_back_to_reverse() -> None: + assert F() + G() == 6 + assert F()**G() == -6 + +[case testDundersBinaryNotImplemented] +from typing import Any, Union +from testutil import assertRaises + +class C: + def __init__(self, v: int) -> None: + self.v = v + + def __add__(self, y: int) -> Union[int, Any]: + if y == 1: + return self.v + return NotImplemented + +def test_any_add() -> None: + a: Any = C(4) + assert a + 1 == 4 + try: + a + 2 + except TypeError: + pass + else: + assert False + +class D: + def __init__(self, x: int) -> None: + self.x = x + + def __add__(self, e: E) -> Union[int, Any]: + if e.x == 1: + return 2 + return NotImplemented + +class E: + def __init__(self, x: int) -> None: + self.x = x + + def __radd__(self, d: D) -> Union[int, Any]: + if d.x == 3: + return 4 + return NotImplemented + +def test_any_radd() -> None: + d1: Any = D(1) + d3: Any = D(3) + e1: Any = E(1) + e3: Any = E(3) + assert d1 + e1 == 2 + assert d3 + e1 == 2 + assert d3 + e3 == 4 + +class F: + def __init__(self, v): + self.v = v + + def __add__(self, x): + if isinstance(x, int): + return self.v + x + return NotImplemented + +class G: + def __radd__(self, x): + if isinstance(x, F): + return x.v + 1 + if isinstance(x, str): + return 'a' + return NotImplemented + +def test_unannotated_add() -> None: + o = F(4) + assert o + 5 == 9 + with assertRaises(TypeError, "unsupported operand type(s) for +: 'F' and 'str'"): + o + 'x' + +def test_unannotated_add_and_radd_1() -> None: + o = F(4) + assert o + G() == 5 + +def test_unannotated_radd() -> None: + assert 'x' + G() == 'a' + with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'G'"): + 1 + G() + +class H: + def __add__(self, x): + if isinstance(x, int): + return x + 1 + return NotImplemented + + def __radd__(self, x): + if isinstance(x, str): + return 22 + return NotImplemented + +def test_unannotated_add_and_radd_2() -> None: + h = H() + assert h + 5 == 6 + assert 'x' + h == 22 + with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'H'"): + 1 + h + +# TODO: Inheritance + +[case testDifferentReverseDunders] +class C: + # __radd__ and __rsub__ are tested elsewhere + + def __rmul__(self, x): + return 1 + + def __rtruediv__(self, x): + return 2 + + def __rmod__(self, x): + return 3 + + def __rfloordiv__(self, x): + return 4 + + def __rlshift__(self, x): + return 5 + + def __rrshift__(self, x): + return 6 + + def __rand__(self, x): + return 7 + + def __ror__(self, x): + return 8 + + def __rxor__(self, x): + return 9 + + def __rmatmul__(self, x): + return 10 + +def test_reverse_dunders() -> None: + x = 0 + c = C() + assert x * c == 1 + assert x / c == 2 + assert x % c == 3 + assert x // c == 4 + assert x << c == 5 + assert x >> c == 6 + assert x & c == 7 + assert x | c == 8 + assert x ^ c == 9 + assert x @ c == 10 + +[case testDundersInplace] +from typing import Any +from testutil import assertRaises + +class C: + def __init__(self) -> None: + self.x = 5 + + def __iadd__(self, y: int) -> C: + self.x += y + return self + + def __isub__(self, y: int) -> C: + self.x -= y + return self + + def __imul__(self, y: int) -> C: + self.x *= y + return self + + def __imod__(self, y: int) -> C: + self.x %= y + return self + + def __itruediv__(self, y: int) -> C: + self.x += y + 10 + return self + + def __ifloordiv__(self, y: int) -> C: + self.x += y + 20 + return self + + def __ilshift__(self, y: int) -> C: + self.x <<= y + return self + + def __irshift__(self, y: int) -> C: + self.x >>= y + return self + + def __iand__(self, y: int) -> C: + self.x &= y + return self + + def __ior__(self, y: int) -> C: + self.x |= y + return self + + def __ixor__(self, y: int) -> C: + self.x ^= y + return self + + def __imatmul__(self, y: int) -> C: + self.x += y + 5 + return self + + def __ipow__(self, y: int, __mod_throwaway: None = None) -> C: + self.x **= y + return self + +def test_generic_1() -> None: + c: Any = C() + c += 3 + assert c.x == 8 + c -= 5 + assert c.x == 3 + c *= 3 + assert c.x == 9 + c %= 4 + assert c.x == 1 + c /= 5 + assert c.x == 16 + c //= 4 + assert c.x == 40 + c **= 2 + assert c.x == 1600 + +def test_generic_2() -> None: + c: Any = C() + c <<= 4 + assert c.x == 80 + c >>= 3 + assert c.x == 10 + c &= 3 + assert c.x == 2 + c |= 6 + assert c.x == 6 + c ^= 12 + assert c.x == 10 + c @= 3 + assert c.x == 18 + +def test_native() -> None: + c = C() + c += 3 + assert c.x == 8 + c -= 5 + assert c.x == 3 + c *= 3 + assert c.x == 9 + c **= 2 + assert c.x == 81 + +def test_error() -> None: + c: Any = C() + with assertRaises(TypeError, "int object expected; got str"): + c += 'x' + +class BadInplaceAdd: + def __init__(self): + self.x = 0 + + def __iadd__(self, x): + self.x += x + +def test_in_place_operator_returns_none() -> None: + o = BadInplaceAdd() + with assertRaises(TypeError, "native.BadInplaceAdd object expected; got None"): + o += 5 + +[case testDunderMinMax] +class SomeItem: + def __init__(self, val: int) -> None: + self.val = val + + def __lt__(self, x: 'SomeItem') -> bool: + return self.val < x.val + + def __gt__(self, x: 'SomeItem') -> bool: + return self.val > x.val + +class AnotherItem: + def __init__(self, val: str) -> None: + self.val = val + + def __lt__(self, x: 'AnotherItem') -> bool: + return True + + def __gt__(self, x: 'AnotherItem') -> bool: + return True + +def test_dunder_min() -> None: + x = SomeItem(5) + y = SomeItem(10) + z = SomeItem(15) + assert min(x, y).val == 5 + assert min(y, z).val == 10 + assert max(x, y).val == 10 + assert max(y, z).val == 15 + x2 = AnotherItem('xxx') + y2 = AnotherItem('yyy') + z2 = AnotherItem('zzz') + assert min(x2, y2).val == 'yyy' + assert min(y2, x2).val == 'xxx' + assert max(x2, y2).val == 'yyy' + assert max(y2, x2).val == 'xxx' + assert min(y2, z2).val == 'zzz' + assert max(x2, z2).val == 'zzz' + + +[case testDundersPowerSpecial] +import sys +from typing import Any, Optional +from testutil import assertRaises + +class Forward: + def __pow__(self, exp: int, mod: Optional[int] = None) -> int: + if mod is None: + return 2**exp + else: + return 2**exp % mod + +class ForwardModRequired: + def __pow__(self, exp: int, mod: int) -> int: + return 2**exp % mod + +class ForwardNotImplemented: + def __pow__(self, exp: int, mod: Optional[object] = None) -> Any: + return NotImplemented + +class Reverse: + def __rpow__(self, exp: int) -> int: + return 2**exp + 1 + +class Both: + def __pow__(self, exp: int, mod: Optional[int] = None) -> int: + if mod is None: + return 2**exp + else: + return 2**exp % mod + + def __rpow__(self, exp: int) -> int: + return 2**exp + 1 + +class Child(ForwardNotImplemented): + def __rpow__(self, exp: object) -> int: + return 50 + +class Inplace: + value = 2 + + def __ipow__(self, exp: int, mod: Optional[int] = None) -> "Inplace": + self.value **= exp - (mod or 0) + return self + +def test_native() -> None: + f = Forward() + assert f**3 == 8 + assert pow(f, 3) == 8 + assert pow(f, 3, 3) == 2 + assert pow(ForwardModRequired(), 3, 3) == 2 + b = Both() + assert b**3 == 8 + assert 3**b == 9 + assert pow(b, 3) == 8 + assert pow(b, 3, 3) == 2 + i = Inplace() + i **= 2 + assert i.value == 4 + +def test_errors() -> None: + if sys.version_info[0] >= 3 and sys.version_info[1] >= 10: + op = "** or pow()" + else: + op = "pow()" + + f = Forward() + with assertRaises(TypeError, f"unsupported operand type(s) for {op}: 'Forward', 'int', 'str'"): + pow(f, 3, "x") # type: ignore + with assertRaises(TypeError, "unsupported operand type(s) for **: 'Forward' and 'str'"): + f**"x" # type: ignore + r = Reverse() + with assertRaises(TypeError, "unsupported operand type(s) for ** or pow(): 'str' and 'Reverse'"): + "x"**r # type: ignore + with assertRaises(TypeError, f"unsupported operand type(s) for {op}: 'int', 'Reverse', 'int'"): + # Ternary pow() does not fallback to __rpow__ if LHS's __pow__ returns NotImplemented. + pow(3, r, 3) # type: ignore + with assertRaises(TypeError, f"unsupported operand type(s) for {op}: 'ForwardNotImplemented', 'Child', 'int'"): + # Ternary pow() does not try RHS's __rpow__ first when it's a subclass and redefines + # __rpow__ unlike other ops. + pow(ForwardNotImplemented(), Child(), 3) # type: ignore + with assertRaises(TypeError, "unsupported operand type(s) for ** or pow(): 'ForwardModRequired' and 'int'"): + ForwardModRequired()**3 # type: ignore + +[case testDundersWithFinal] +from typing import final +class A: + def __init__(self, x: int) -> None: + self.x = x + + def __add__(self, y: int) -> int: + return self.x + y + + def __lt__(self, x: 'A') -> bool: + return self.x < x.x + +@final +class B(A): + def __add__(self, y: int) -> int: + return self.x + y + 1 + + def __lt__(self, x: 'A') -> bool: + return self.x < x.x + 1 + +def test_final() -> None: + a = A(5) + b = B(5) + assert a + 3 == 8 + assert b + 3 == 9 + assert (a < A(5)) is False + assert (b < A(5)) is True diff --git a/mypyc/test-data/run-exceptions.test b/mypyc/test-data/run-exceptions.test new file mode 100644 index 000000000000..1b180b933197 --- /dev/null +++ b/mypyc/test-data/run-exceptions.test @@ -0,0 +1,534 @@ +# Test cases for exceptions (compile and run) + +[case testException] +from typing import List +def f(x: List[int]) -> None: + g(x) + +def g(x: List[int]) -> bool: + x[5] = 2 + return True + +def r1() -> None: + q1() + +def q1() -> None: + raise Exception("test") + +def r2() -> None: + q2() + +def q2() -> None: + raise Exception + +class A: + def __init__(self) -> None: + raise Exception + +def hey() -> None: + A() + +[file driver.py] +from native import f, r1, r2, hey +import traceback +try: + f([]) +except IndexError: + traceback.print_exc() +try: + r1() +except Exception: + traceback.print_exc() +try: + r2() +except Exception: + traceback.print_exc() +try: + hey() +except Exception: + traceback.print_exc() +[out] +Traceback (most recent call last): + File "driver.py", line 4, in + f([]) + File "native.py", line 3, in f + g(x) + File "native.py", line 6, in g + x[5] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 8, in + r1() + File "native.py", line 10, in r1 + q1() + File "native.py", line 13, in q1 + raise Exception("test") +Exception: test +Traceback (most recent call last): + File "driver.py", line 12, in + r2() + File "native.py", line 16, in r2 + q2() + File "native.py", line 19, in q2 + raise Exception +Exception +Traceback (most recent call last): + File "driver.py", line 16, in + hey() + File "native.py", line 26, in hey + A() + File "native.py", line 23, in __init__ + raise Exception +Exception +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 4, in + f([]) + ~^^^^ + File "native.py", line 3, in f + g(x) + File "native.py", line 6, in g + x[5] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 8, in + r1() + ~~^^ + File "native.py", line 10, in r1 + q1() + File "native.py", line 13, in q1 + raise Exception("test") +Exception: test +Traceback (most recent call last): + File "driver.py", line 12, in + r2() + ~~^^ + File "native.py", line 16, in r2 + q2() + File "native.py", line 19, in q2 + raise Exception +Exception +Traceback (most recent call last): + File "driver.py", line 16, in + hey() + ~~~^^ + File "native.py", line 26, in hey + A() + File "native.py", line 23, in __init__ + raise Exception +Exception + +[case testTryExcept] +from typing import Any, Iterator +import wrapsys +def g(b: bool) -> None: + try: + if b: + x = [0] + x[1] + else: + raise Exception('hi') + except: + print("caught!") + +def r(x: int) -> None: + if x == 0: + [0][1] + elif x == 1: + raise Exception('hi') + elif x == 2: + {1: 1}[0] + elif x == 3: + a = object() # type: Any + a.lol + +def f(b: bool) -> None: + try: + r(int(b)) + except AttributeError: + print('no') + except: + print(str(wrapsys.exc_info()[1])) + print(str(wrapsys.exc_info()[1])) + +def h() -> None: + while True: + try: + raise Exception('gonna break') + except: + print(str(wrapsys.exc_info()[1])) + break + print(str(wrapsys.exc_info()[1])) + +def i() -> None: + try: + r(0) + except: + print(type(wrapsys.exc_info()[1])) + raise + +def j(n: int) -> None: + try: + r(n) + except (IndexError, KeyError): + print("lookup!") + except AttributeError as e: + print("attr! --", e) + +def k() -> None: + try: + r(1) + except: + r(0) + +def l() -> None: + try: + r(0) + except IndexError: + try: + r(2) + except KeyError as e: + print("key! --", e) + +def m(x: object) -> int: + try: + st = id(x) + except Exception: + return -1 + return st + 1 + +def iter_exception() -> Iterator[str]: + try: + r(0) + except KeyError as e: + yield 'lol' + +[file wrapsys.py] +# This is a gross hack around some limitations of the test system/mypyc. +from typing import Any +import sys +def exc_info() -> Any: + return sys.exc_info() # type: ignore + +[file driver.py] +import sys, traceback +from native import g, f, h, i, j, k, l, m, iter_exception +from testutil import assertRaises +print("== i ==") +try: + i() +except: + traceback.print_exc(file=sys.stdout) + +print("== k ==") +try: + k() +except: + traceback.print_exc(file=sys.stdout) + +print("== g ==") +g(True) +g(False) + +print("== f ==") +f(True) +f(False) + +print("== h ==") +h() + +print("== j ==") +j(0) +j(2) +j(3) +try: + j(1) +except: + print("out!") + +print("== l ==") +l() + +m('lol') + +with assertRaises(IndexError): + list(iter_exception()) + +[out] +== i == + +Traceback (most recent call last): + File "driver.py", line 6, in + i() + File "native.py", line 44, in i + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== k == +Traceback (most recent call last): + File "native.py", line 59, in k + r(1) + File "native.py", line 17, in r + raise Exception('hi') +Exception: hi + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "driver.py", line 12, in + k() + File "native.py", line 61, in k + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== g == +caught! +caught! +== f == +hi +None +list index out of range +None +== h == +gonna break +None +== j == +lookup! +lookup! +attr! -- 'object' object has no attribute 'lol' +out! +== l == +key! -- 0 +[out version>=3.13] +== i == + +Traceback (most recent call last): + File "driver.py", line 6, in + i() + ~^^ + File "native.py", line 44, in i + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== k == +Traceback (most recent call last): + File "native.py", line 59, in k + r(1) + File "native.py", line 17, in r + raise Exception('hi') +Exception: hi + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "driver.py", line 12, in + k() + ~^^ + File "native.py", line 61, in k + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== g == +caught! +caught! +== f == +hi +None +list index out of range +None +== h == +gonna break +None +== j == +lookup! +lookup! +attr! -- 'object' object has no attribute 'lol' +out! +== l == +key! -- 0 + +[case testTryFinally] +from typing import Any +import wrapsys + +def a(b1: bool, b2: int) -> None: + try: + if b1: + raise Exception('hi') + finally: + print('finally:', str(wrapsys.exc_info()[1])) + if b2 == 2: + return + if b2 == 1: + raise Exception('again!') + +def b(b1: int, b2: int) -> str: + try: + if b1 == 1: + raise Exception('hi') + elif b1 == 2: + [0][1] + elif b1 == 3: + return 'try' + except IndexError: + print('except') + finally: + print('finally:', str(wrapsys.exc_info()[1])) + if b2 == 2: + return 'finally' + if b2 == 1: + raise Exception('again!') + return 'outer' + +def c() -> str: + try: + try: + return 'wee' + finally: + print("out a") + finally: + print("out b") + + +[file wrapsys.py] +# This is a gross hack around some limitations of the test system/mypyc. +from typing import Any +import sys +def exc_info() -> Any: + return sys.exc_info() # type: ignore + +[file driver.py] +import traceback +import sys +from native import a, b, c + +def run(f): + try: + x = f() + if x: + print("returned:", x) + except Exception as e: + print("caught:", type(e).__name__ + ": " + str(e)) + +print("== a ==") +for i in range(3): + for b1 in [False, True]: + run(lambda: a(b1, i)) + +print("== b ==") +for i in range(4): + for j in range(3): + run(lambda: b(i, j)) + +print("== b ==") +print(c()) + +[out] +== a == +finally: None +finally: hi +caught: Exception: hi +finally: None +caught: Exception: again! +finally: hi +caught: Exception: again! +finally: None +finally: hi +== b == +finally: None +returned: outer +finally: None +caught: Exception: again! +finally: None +returned: finally +finally: hi +caught: Exception: hi +finally: hi +caught: Exception: again! +finally: hi +returned: finally +except +finally: None +returned: outer +except +finally: None +caught: Exception: again! +except +finally: None +returned: finally +finally: None +returned: try +finally: None +caught: Exception: again! +finally: None +returned: finally +== b == +out a +out b +wee + +[case testCustomException] +from typing import List + +class ListOutOfBounds(IndexError): + pass + +class UserListWarning(UserWarning): + pass + +def f(l: List[int], k: int) -> int: + try: + return l[k] + except IndexError: + raise ListOutOfBounds("Ruh-roh from f!") + +def g(l: List[int], k: int) -> int: + try: + return f([1,2,3], 3) + except ListOutOfBounds: + raise ListOutOfBounds("Ruh-roh from g!") + +def k(l: List[int], k: int) -> int: + try: + return g([1,2,3], 3) + except IndexError: + raise UserListWarning("Ruh-roh from k!") + +def h() -> int: + try: + return k([1,2,3], 3) + except UserWarning: + return -1 + +[file driver.py] +from native import h +assert h() == -1 + +[case testExceptionAtModuleTopLevel] +from typing import Any + +def f(x: int) -> None: pass + +y: Any = '' +f(y) + +[file driver.py] +import traceback +try: + import native +except TypeError: + traceback.print_exc() +else: + assert False + +[out] +Traceback (most recent call last): + File "driver.py", line 3, in + import native + File "native.py", line 6, in + f(y) +TypeError: int object expected; got str diff --git a/mypyc/test-data/run-floats.test b/mypyc/test-data/run-floats.test new file mode 100644 index 000000000000..2c101100549d --- /dev/null +++ b/mypyc/test-data/run-floats.test @@ -0,0 +1,516 @@ +# Test cases for floats (compile and run) + +[case testFloatOps] +from __future__ import annotations +from typing import Any, cast +from typing_extensions import Final +from testutil import assertRaises, float_vals, FLOAT_MAGIC +import math + +def test_arithmetic() -> None: + zero = float(0.0) + one = zero + 1.0 + x = one + one / 2.0 + assert x == 1.5 + assert x - one == 0.5 + assert x * x == 2.25 + assert x / 2.0 == 0.75 + assert x * (-0.5) == -0.75 + assert -x == -1.5 + for x in float_vals: + assert repr(-x) == repr(getattr(x, "__neg__")()) + + for y in float_vals: + assert repr(x + y) == repr(getattr(x, "__add__")(y)) + assert repr(x - y) == repr(getattr(x, "__sub__")(y)) + assert repr(x * y) == repr(getattr(x, "__mul__")(y)) + if y != 0: + assert repr(x / y) == repr(getattr(x, "__truediv__")(y)) + +def test_mod() -> None: + zero = float(0.0) + one = zero + 1.0 + x = one + one / 2.0 + assert x % 0.4 == 0.29999999999999993 + assert (-x) % 0.4 == 0.10000000000000009 + assert x % -0.4 == -0.10000000000000009 + assert (-x) % -0.4 == -0.29999999999999993 + for x in float_vals: + for y in float_vals: + if y != 0: + assert repr(x % y) == repr(getattr(x, "__mod__")(y)) + +def test_floor_div() -> None: + for x in float_vals: + for y in float_vals: + if y != 0: + assert repr(x // y) == repr(getattr(x, "__floordiv__")(y)) + else: + with assertRaises(ZeroDivisionError, "float floor division by zero"): + x // y + +def test_mixed_arithmetic() -> None: + zf = float(0.0) + zn = int() + assert (zf + 5.5) + (zn + 1) == 6.5 + assert (zn - 2) - (zf - 5.5) == 3.5 + x = zf + 3.4 + x += zn + 2 + assert x == 5.4 + +def test_arithmetic_errors() -> None: + zero = float(0.0) + one = zero + 1.0 + with assertRaises(ZeroDivisionError, "float division by zero"): + print(one / zero) + with assertRaises(ZeroDivisionError, "float modulo"): + print(one % zero) + +def test_comparisons() -> None: + zero = float(0.0) + one = zero + 1.0 + x = one + one / 2.0 + assert x < (1.51 + zero) + assert not (x < (1.49 + zero)) + assert x > (1.49 + zero) + assert not (x > (1.51 + zero)) + assert x <= (1.5 + zero) + assert not (x <= (1.49 + zero)) + assert x >= (1.5 + zero) + assert not (x >= (1.51 + zero)) + for x in float_vals: + for y in float_vals: + assert (x <= y) == getattr(x, "__le__")(y) + assert (x < y) == getattr(x, "__lt__")(y) + assert (x >= y) == getattr(x, "__ge__")(y) + assert (x > y) == getattr(x, "__gt__")(y) + assert (x == y) == getattr(x, "__eq__")(y) + assert (x != y) == getattr(x, "__ne__")(y) + +def test_mixed_comparisons() -> None: + zf = float(0.0) + zn = int() + if (zf + 1.0) == (zn + 1): + assert True + else: + assert False + if (zf + 1.1) == (zn + 1): + assert False + else: + assert True + assert (zf + 1.1) != (zn + 1) + assert (zf + 1.1) > (zn + 1) + assert not (zf + 0.9) > (zn + 1) + assert (zn + 1) < (zf + 1.1) + +def test_boxing_and_unboxing() -> None: + x = 1.5 + boxed: Any = x + assert repr(boxed) == "1.5" + assert type(boxed) is float + y: float = boxed + assert y == x + boxed_int: Any = 5 + assert [type(boxed_int)] == [int] # Avoid mypy type narrowing + z: float = boxed_int + assert z == 5.0 + for xx in float_vals: + bb: Any = xx + yy: float = bb + assert repr(xx) == repr(bb) + assert repr(xx) == repr(yy) + for b in True, False: + boxed_bool: Any = b + assert type(boxed_bool) is bool + zz: float = boxed_bool + assert zz == int(b) + +def test_unboxing_failure() -> None: + boxed: Any = '1.5' + with assertRaises(TypeError): + x: float = boxed + +def identity(x: float) -> float: + return x + +def test_coerce_from_int_literal() -> None: + assert identity(34) == 34.0 + assert identity(-1) == -1.0 + +def test_coerce_from_short_tagged_int() -> None: + n = int() - 17 + assert identity(n) == -17.0 + for i in range(-300, 300): + assert identity(i) == float(i) + +def test_coerce_from_long_tagged_int() -> None: + n = int() + 2**100 + x = identity(n) + assert repr(x) == '1.2676506002282294e+30' + n = int() - 2**100 + y = identity(n) + assert repr(y) == '-1.2676506002282294e+30' + +def test_coerce_from_very_long_tagged_int() -> None: + n = int() + 10**1000 + with assertRaises(OverflowError, "int too large to convert to float"): + identity(n) + with assertRaises(OverflowError, "int too large to convert to float"): + identity(int(n)) + n = int() - 10**1000 + with assertRaises(OverflowError, "int too large to convert to float"): + identity(n) + with assertRaises(OverflowError, "int too large to convert to float"): + identity(int(n)) + +def test_explicit_conversion_from_int() -> None: + float_any: Any = float + a = [0, 1, 2, 3, -1, -2, 13257, -928745] + for n in range(1, 100): + for delta in -1, 0, 1, 2342345: + a.append(2**n + delta) + a.append(-2**n + delta) + for x in a: + assert repr(float(x)) == repr(float_any(x)) + +def test_explicit_conversion_to_int() -> None: + int_any: Any = int + for x in float_vals: + if math.isinf(x): + with assertRaises(OverflowError, "cannot convert float infinity to integer"): + int(x) + elif math.isnan(x): + with assertRaises(ValueError, "cannot convert float NaN to integer"): + int(x) + else: + assert repr(int(x)) == repr(int_any(x)) + + # Test some edge cases + assert 2**30 == int(2.0**30 + int()) + assert 2**30 - 1 == int(1073741823.9999999 + int()) # math.nextafter(2.0**30, 0)) + assert -2**30 - 1 == int(-2.0**30 - 1 + int()) + assert -2**30 == int(-1073741824.9999998 + int()) # math.nextafter(-2.0**30 - 1, 0) + assert 2**62 == int(2.0**62 + int()) + assert 2**62 == int(2.0**62 - 1 + int()) + assert -2**62 == int(-2.0**62 + int()) + assert -2**62 == int(-2.0**62 - 1 + int()) + +def str_to_float(x: str) -> float: + return float(x) + +def test_str_to_float() -> None: + assert str_to_float("1") == 1.0 + assert str_to_float("1.234567") == 1.234567 + assert str_to_float("44324") == 44324.0 + assert str_to_float("23.4") == 23.4 + assert str_to_float("-43.44e-4") == -43.44e-4 + assert str_to_float("-43.44e-4") == -43.44e-4 + assert math.isinf(str_to_float("inf")) + assert math.isinf(str_to_float("-inf")) + assert str_to_float("inf") > 0.0 + assert str_to_float("-inf") < 0.0 + assert math.isnan(str_to_float("nan")) + assert math.isnan(str_to_float("NaN")) + assert repr(str_to_float("-0.0")) == "-0.0" + +def test_abs() -> None: + assert abs(0.0) == 0.0 + assert abs(-1.234567) == 1.234567 + assert abs(44324.732) == 44324.732 + assert abs(-23.4) == 23.4 + assert abs(-43.44e-4) == 43.44e-4 + abs_any: Any = abs + for x in float_vals: + assert repr(abs(x)) == repr(abs_any(x)) + +def test_float_min_max() -> None: + for x in float_vals: + for y in float_vals: + min_any: Any = min + assert repr(min(x, y)) == repr(min_any(x, y)) + max_any: Any = max + assert repr(max(x, y)) == repr(max_any(x, y)) + +def default(x: float = 2) -> float: + return x + 1 + +def test_float_default_value() -> None: + assert default(1.2) == 2.2 + for i in range(-200, 200): + assert default(float(i)) == i + 1 + assert default() == 3.0 + +def test_float_default_value_wrapper() -> None: + f: Any = default + assert f(1.2) == 2.2 + for i in range(-200, 200): + assert f(float(i)) == i + 1 + assert f() == 3.0 + +class C: + def __init__(self, x: float) -> None: + self.x = x + +def test_float_attr() -> None: + for i in range(-200, 200): + f = float(i) + c = C(f) + assert c.x == f + a: Any = c + assert a.x == f + c.x = FLOAT_MAGIC + assert c.x == FLOAT_MAGIC + assert a.x == FLOAT_MAGIC + a.x = 1.0 + assert a.x == 1.0 + a.x = FLOAT_MAGIC + assert a.x == FLOAT_MAGIC + +class D: + def __init__(self, x: float) -> None: + if x: + self.x = x + +def test_float_attr_maybe_undefned() -> None: + for i in range(-200, 200): + if i == 0: + d = D(0.0) + with assertRaises(AttributeError): + d.x + a: Any = d + with assertRaises(AttributeError): + a.x + d.x = FLOAT_MAGIC + assert d.x == FLOAT_MAGIC + assert a.x == FLOAT_MAGIC + d.x = 0.0 + assert d.x == 0.0 + assert a.x == 0.0 + a.x = FLOAT_MAGIC + assert a.x == FLOAT_MAGIC + d = D(0.0) + a = cast(Any, d) + a.x = FLOAT_MAGIC + assert d.x == FLOAT_MAGIC + else: + f = float(i) + d = D(f) + assert d.x == f + a2: Any = d + assert a2.x == f + +def f(x: float) -> float: + return x + 1 + +def test_return_values() -> None: + a: Any = f + for i in range(-200, 200): + x = float(i) + assert f(x) == x + 1 + assert a(x) == x + 1 + for x in float_vals: + if not math.isnan(x): + assert f(x) == x + 1 + else: + assert math.isnan(f(x)) + +def exc() -> float: + raise IndexError('x') + +def test_exception() -> None: + with assertRaises(IndexError): + exc() + a: Any = exc + with assertRaises(IndexError): + a() + +def test_undefined_local_var() -> None: + if not int(): + x = -113.0 + assert x == -113.0 + if int(): + y = -113.0 + with assertRaises(UnboundLocalError, 'local variable "y" referenced before assignment'): + print(y) + if not int(): + x2 = -1.0 + assert x2 == -1.0 + if int(): + y2 = -1.0 + with assertRaises(UnboundLocalError, 'local variable "y2" referenced before assignment'): + print(y2) + +def test_tuples() -> None: + t1: tuple[float, float] = (1.5, 2.5) + assert t1 == tuple([1.5, 2.5]) + n = int() + 5 + t2: tuple[float, float, float, float] = (n, 1.5, -7, -113) + assert t2 == tuple([5.0, 1.5, -7.0, -113.0]) + +[case testFloatGlueMethodsAndInheritance] +from typing import Any +from typing_extensions import Final + +from mypy_extensions import trait + +from testutil import assertRaises + +MAGIC: Final = -113.0 + +class Base: + def foo(self) -> float: + return 5.0 + + def bar(self, x: float = 2.0) -> float: + return x + 1 + + def hoho(self, x: float) -> float: + return x - 1 + +class Derived(Base): + def foo(self, x: float = 5.0) -> float: + return x + 10 + + def bar(self, x: float = 3, y: float = 20) -> float: + return x + y + 2 + + def hoho(self, x: float = 7) -> float: + return x - 2 + +def test_derived_adds_bitmap() -> None: + b: Base = Derived() + assert b.foo() == 15 + +def test_derived_adds_another_default_arg() -> None: + b: Base = Derived() + assert b.bar() == 25 + assert b.bar(1) == 23 + assert b.bar(MAGIC) == MAGIC + 22 + +def test_derived_switches_arg_to_have_default() -> None: + b: Base = Derived() + assert b.hoho(5) == 3 + assert b.hoho(MAGIC) == MAGIC - 2 + +@trait +class T: + @property + def x(self) -> float: ... + @property + def y(self) -> float: ... + +class C(T): + x: float = 1.0 + y: float = 4 + +def test_read_only_property_in_trait_implemented_as_attribute() -> None: + c = C() + c.x = 5.5 + assert c.x == 5.5 + c.x = MAGIC + assert c.x == MAGIC + assert c.y == 4 + c.y = 6.5 + assert c.y == 6.5 + t: T = C() + assert t.y == 4 + t = c + assert t.x == MAGIC + c.x = 55.5 + assert t.x == 55.5 + assert t.y == 6.5 + a: Any = c + assert a.x == 55.5 + assert a.y == 6.5 + a.x = 7.0 + a.y = 8.0 + assert a.x == 7 + assert a.y == 8 + +class D(T): + xx: float + + @property + def x(self) -> float: + return self.xx + + @property + def y(self) -> float: + raise TypeError + +def test_read_only_property_in_trait_implemented_as_property() -> None: + d = D() + d.xx = 5.0 + assert d.x == 5 + d.xx = MAGIC + assert d.x == MAGIC + with assertRaises(TypeError): + d.y + t: T = d + assert t.x == MAGIC + d.xx = 6.0 + assert t.x == 6 + with assertRaises(TypeError): + t.y + +@trait +class T2: + x: float + y: float + +class C2(T2): + pass + +def test_inherit_trait_attribute() -> None: + c = C2() + c.x = 5.0 + assert c.x == 5 + c.x = MAGIC + assert c.x == MAGIC + with assertRaises(AttributeError): + c.y + c.y = 6.0 + assert c.y == 6.0 + t: T2 = C2() + with assertRaises(AttributeError): + t.y + t = c + assert t.x == MAGIC + c.x = 55.0 + assert t.x == 55 + assert t.y == 6 + a: Any = c + assert a.x == 55 + assert a.y == 6 + a.x = 7.0 + a.y = 8.0 + assert a.x == 7 + assert a.y == 8 + +class D2(T2): + x: float + y: float = 4 + +def test_implement_trait_attribute() -> None: + d = D2() + d.x = 5.0 + assert d.x == 5 + d.x = MAGIC + assert d.x == MAGIC + assert d.y == 4 + d.y = 6.0 + assert d.y == 6 + t: T2 = D2() + assert t.y == 4 + t = d + assert t.x == MAGIC + d.x = 55.0 + assert t.x == 55 + assert t.y == 6 + a: Any = d + assert a.x == 55 + assert a.y == 6 + a.x = 7.0 + a.y = 8.0 + assert a.x == 7 + assert a.y == 8 diff --git a/mypyc/test-data/run-functions.test b/mypyc/test-data/run-functions.test new file mode 100644 index 000000000000..cf519f30dad8 --- /dev/null +++ b/mypyc/test-data/run-functions.test @@ -0,0 +1,1310 @@ +# Test cases for functions and calls (compile and run) + +[case testCallTrivialFunction] +def f(x: int) -> int: + return x +[file driver.py] +from native import f +print(f(3)) +print(f(-157)) +print(f(10**20)) +print(f(-10**20)) +[out] +3 +-157 +100000000000000000000 +-100000000000000000000 + +[case testRecursiveFibonacci] +def fib(n: int) -> int: + if n <= 1: + return 1 + else: + return fib(n - 1) + fib(n - 2) +[file driver.py] +from native import fib +print(fib(0)) +print(fib(1)) +print(fib(2)) +print(fib(6)) +[out] +1 +1 +2 +13 + +[case testNestedFunctions] +from typing import Callable, List + +def a() -> Callable[[], object]: + def inner() -> object: + return None + return inner + +def b() -> Callable[[], Callable[[], str]]: + def first() -> Callable[[], str]: + def second() -> str: + return 'b.first.second: nested function' + return second + return first + +def c(num: float) -> Callable[[str], str]: + def inner(s: str) -> str: + return s + '!' + return inner + +def d(num: float) -> str: + def inner(s: str) -> str: + return s + '?' + a = inner('one') + b = inner('two') + return a + +def e() -> int: + return 0 + +def f() -> int: + def inner() -> int: + return e() + return inner() + +def g() -> Callable[[], Callable[[], int]]: + def inner() -> Callable[[], int]: + return e + return inner + +def h(num: int) -> int: + def inner() -> int: + return num + return inner() + +def i() -> int: + num = 3 + def inner() -> int: + return num + return inner() + +def j(num: int) -> int: + x = 1 + y = 2 + def inner() -> int: + nonlocal x + x = 3 + return num + x + y + return inner() + +def k() -> int: + num = 3 + def inner() -> int: + nonlocal num + num = 5 + return num + return inner() + num + +def l() -> int: + num = 3 + def inner() -> int: + num = 5 + return num + return inner() + num + +def m() -> Callable[[], int]: + num = 1 + def inner() -> int: + num += 1 + return num + num += 1 + return inner + +def n() -> int: + x = 1 + def add_one() -> None: + x += 1 + def add_two() -> None: + x += 2 + add_one() + add_two() + return x + +def triple(a: int) -> Callable[[], Callable[[int], int]]: + x = 1 + def outer() -> Callable[[int], int]: + nonlocal x + x += 1 + x += a + a += 1 + def inner(b: int) -> int: + x += b + return x + return inner + return outer + +def if_else(flag: int) -> str: + def dummy_funtion() -> str: + return 'if_else.dummy_function' + + if flag < 0: + def inner() -> str: + return 'if_else.inner: first definition' + elif flag > 0: + def inner() -> str: + return 'if_else.inner: second definition' + else: + def inner() -> str: + return 'if_else.inner: third definition' + return inner() + +def for_loop() -> int: + def dummy_funtion() -> str: + return 'for_loop.dummy_function' + + for i in range(5): + def inner(i: int) -> int: + return i + if i == 3: + return inner(i) + return 0 + +def while_loop() -> int: + def dummy_funtion() -> str: + return 'while_loop.dummy_function' + + i = 0 + while i < 5: + def inner(i: int) -> int: + return i + if i == 3: + return inner(i) + i += 1 + return 0 + +def free_vars(foo: int, bar: int) -> int: + x = 1 + y = 2 + def g(): # type: ignore # missing type annotation for testing + nonlocal y + y = 3 + nonlocal bar + bar += y + z = 3 + g() + return bar + +def lambdas(x: int, y: int) -> int: + s = lambda a, b: a + b + x + y + return s(1, 2) + +def outer() -> str: + return 'outer: normal function' + +def inner() -> str: + return 'inner: normal function' + +class A: + def __init__(self, x: int) -> None: + self.x = x + + def outer(self, num: int) -> int: + y = 5 + def inner() -> int: + return self.x + y + num + return inner() + +def o() -> int: + a = [0, 0] + b = 0 + def b_incr() -> List[int]: + b += 10 + return a + c = 0 + def c_incr() -> int: + c += 1 + return c + + # x = 1, y = 1 + x = y = c_incr() + + # a = [2, 2], b = 20 + b_incr()[0] = b_incr()[1] = c_incr() + # Should return 26. + return x + y + a[0] + a[1] + b + +global_upvar = 20 + +toplevel_lambda = lambda x: 10 + global_upvar + x + +[file driver.py] +from native import ( + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, triple, if_else, for_loop, while_loop, + free_vars, lambdas, outer, inner, A, toplevel_lambda +) + +assert a()() == None +assert b()()() == 'b.first.second: nested function' +assert c(5.0)('c') == 'c!' +assert d(4.0) == 'one?' +assert e() == 0 +assert f() == 0 +assert g()()() == 0 +assert h(3) == 3 +assert i() == 3 +assert j(3) == 8 +assert k() == 10 +assert l() == 8 +assert m()() == 3 +assert n() == 4 +assert o() == 26 + +triple_outer = triple(2) +triple_inner = triple_outer() + +assert triple_inner(4) == 8 +assert triple_inner(4) == 12 +assert triple_outer()(4) == 20 + +assert if_else(-1) == 'if_else.inner: first definition' +assert if_else(1) == 'if_else.inner: second definition' +assert if_else(0) == 'if_else.inner: third definition' + +assert for_loop() == 3 +assert while_loop() == 3 + +assert free_vars(1, 2) == 5 +assert lambdas(3, 4) == 10 + +assert outer() == 'outer: normal function' +assert inner() == 'inner: normal function' + +assert A(3).outer(4) == 12 + +assert toplevel_lambda(5) == 35 + +[case testNestedFunctions2] +from typing import Callable + +def outer() -> Callable[[], object]: + def inner() -> object: + return None + return inner + +def first() -> Callable[[], Callable[[], str]]: + def second() -> Callable[[], str]: + def third() -> str: + return 'third: nested function' + return third + return second + +def f1() -> int: + x = 1 + def f2() -> int: + y = 2 + def f3() -> int: + z = 3 + return y + return f3() + return f2() + +def outer_func() -> int: + def inner_func() -> int: + return x + x = 1 + return inner_func() + +def mutual_recursion(start : int) -> int: + def f1(k : int) -> int: + if k <= 0: + return 0 + k -= 1 + return f2(k) + + def f2(k : int) -> int: + if k <= 0: + return 0 + k -= 1 + return f1(k) + return f1(start) + +def topLayer() -> int: + def middleLayer() -> int: + def bottomLayer() -> int: + return x + + return bottomLayer() + + x = 1 + return middleLayer() + +def nest1() -> str: + def nest2() -> str: + def nest3() -> str: + def mut1(val: int) -> str: + if val <= 0: + return "bottomed" + val -= 1 + return mut2(val) + def mut2(val: int) -> str: + if val <= 0: + return "bottomed" + val -= 1 + return mut1(val) + return mut1(start) + return nest3() + start = 3 + return nest2() + +def uno(num: float) -> Callable[[str], str]: + def dos(s: str) -> str: + return s + '!' + return dos + +def eins(num: float) -> str: + def zwei(s: str) -> str: + return s + '?' + a = zwei('eins') + b = zwei('zwei') + return a + +def call_other_inner_func(a: int) -> int: + def foo() -> int: + return a + 1 + + def bar() -> int: + return foo() + + def baz(n: int) -> int: + if n == 0: + return 0 + return n + baz(n - 1) + + return bar() + baz(a) + +def inner() -> str: + return 'inner: normal function' + +def second() -> str: + return 'second: normal function' + +def third() -> str: + return 'third: normal function' + +[file driver.py] +from native import (outer, inner, first, uno, eins, call_other_inner_func, +second, third, f1, outer_func, mutual_recursion, topLayer, nest1) + +assert outer()() == None +assert inner() == 'inner: normal function' +assert first()()() == 'third: nested function' +assert uno(5.0)('uno') == 'uno!' +assert eins(4.0) == 'eins?' +assert call_other_inner_func(5) == 21 +assert second() == 'second: normal function' +assert third() == 'third: normal function' +assert f1() == 2 +assert outer_func() == 1 +assert mutual_recursion(5) == 0 +assert topLayer() == 1 +assert nest1() == "bottomed" + +[case testFunctionCallWithDefaultArgs] +from typing import Tuple, List, Optional, Callable, Any +def f(x: int, y: int = 3, s: str = "test", z: object = 5) -> Tuple[int, str]: + def inner() -> int: + return x + y + return inner(), s +def g() -> None: + assert f(2) == (5, "test") + assert f(s = "123", x = -2) == (1, "123") +def h(a: Optional[object] = None, b: Optional[str] = None) -> Tuple[object, Optional[str]]: + return (a, b) + +def same(x: object = object()) -> object: + return x + +a_lambda: Callable[..., Any] = lambda n=20: n + +def nested_funcs(n: int) -> List[Callable[..., Any]]: + ls: List[Callable[..., Any]] = [] + for i in range(n): + def f(i: int = i) -> int: + return i + ls.append(f) + return ls + +def bool_default(x: bool = False, y: bool = True) -> str: + return str(x) + '-' + str(y) + +[file driver.py] +from native import f, g, h, same, nested_funcs, a_lambda, bool_default +g() +assert f(2) == (5, "test") +assert f(s = "123", x = -2) == (1, "123") +assert h() == (None, None) +assert h(10) == (10, None) +assert h(b='a') == (None, 'a') +assert h(10, 'a') == (10, 'a') +assert same() == same() + +assert [f() for f in nested_funcs(10)] == list(range(10)) + +assert a_lambda(10) == 10 +assert a_lambda() == 20 + +assert bool_default() == 'False-True' +assert bool_default(True) == 'True-True' +assert bool_default(True, False) == 'True-False' + +[case testMethodCallWithDefaultArgs] +from typing import Tuple, List +class A: + def f(self, x: int, y: int = 3, s: str = "test") -> Tuple[int, str]: + def inner() -> int: + return x + y + return inner(), s +def g() -> None: + a = A() + assert a.f(2) == (5, "test") + assert a.f(s = "123", x = -2) == (1, "123") +[file driver.py] +from native import A, g +g() +a = A() +assert a.f(2) == (5, "test") +assert a.f(s = "123", x = -2) == (1, "123") + +[case testMethodCallOrdering] +class A: + def __init__(self, s: str) -> None: + print(s) + def f(self, x: 'A', y: 'A') -> None: + pass + +def g() -> None: + A('A!').f(A('hello'), A('world')) +[file driver.py] +from native import g +g() +[out] +A! +hello +world + +[case testPyMethodCall] +from typing import List +def f(x: List[int]) -> int: + return x.pop() +def g(x: List[int], y: List[int]) -> None: + x.extend(y) +[file driver.py] +from native import f, g +l = [1, 2] +assert f(l) == 2 +g(l, [10]) +assert l == [1, 10] +assert f(l) == 10 +assert f(l) == 1 +g(l, [11, 12]) +assert l == [11, 12] + +[case testMethodCallWithKeywordArgs] +from typing import Tuple +import testmodule +class A: + def echo(self, a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c +def test_native_method_call_with_kwargs() -> None: + a = A() + assert a.echo(1, c=3, b=2) == (1, 2, 3) + assert a.echo(c = 3, a = 1, b = 2) == (1, 2, 3) +def test_module_method_call_with_kwargs() -> None: + a = testmodule.A() + assert a.echo(1, c=3, b=2) == (1, 2, 3) + assert a.echo(c = 3, a = 1, b = 2) == (1, 2, 3) +[file testmodule.py] +from typing import Tuple +class A: + def echo(self, a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c +[file driver.py] +import native +native.test_native_method_call_with_kwargs() +native.test_module_method_call_with_kwargs() + +[case testAnyCall] +from typing import Any +def call(f: Any) -> Any: + return f(1, 'x') +[file driver.py] +from native import call +def f(x, y): + return (x, y) +def g(x): pass + +assert call(f) == (1, 'x') +for bad in g, 1: + try: + call(bad) + except TypeError: + pass + else: + assert False, bad + +[case testCallableTypes] +from typing import Callable +def absolute_value(x: int) -> int: + return x if x > 0 else -x + +def call_native_function(x: int) -> int: + return absolute_value(x) + +def call_python_function(x: int) -> int: + return int(x) + +def return_float() -> float: + return 5.0 + +def return_callable_type() -> Callable[[], float]: + return return_float + +def call_callable_type() -> float: + f = return_callable_type() + return f() + +def return_passed_in_callable_type(f: Callable[[], float]) -> Callable[[], float]: + return f + +def call_passed_in_callable_type(f: Callable[[], float]) -> float: + return f() + +[file driver.py] +from native import call_native_function, call_python_function, return_float, return_callable_type, call_callable_type, return_passed_in_callable_type, call_passed_in_callable_type +a = call_native_function(1) +b = call_python_function(1) +c = return_callable_type() +d = call_callable_type() +e = return_passed_in_callable_type(return_float) +f = call_passed_in_callable_type(return_float) +assert a == 1 +assert b == 1 +assert c() == 5.0 +assert d == 5.0 +assert e() == 5.0 +assert f == 5.0 + +[case testKeywordArgs] +from typing import Tuple +import testmodule + +def g(a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c + +def test_call_native_function_with_keyword_args() -> None: + assert g(1, c = 3, b = 2) == (1, 2, 3) + assert g(c = 3, a = 1, b = 2) == (1, 2, 3) + +def test_call_module_function_with_keyword_args() -> None: + assert testmodule.g(1, c = 3, b = 2) == (1, 2, 3) + assert testmodule.g(c = 3, a = 1, b = 2) == (1, 2, 3) + +def test_call_python_function_with_keyword_args() -> None: + assert int("11", base=2) == 3 + +def test_call_lambda_function_with_keyword_args() -> None: + g = testmodule.get_lambda_function() + assert g(1, c = 3, b = 2) == (1, 2, 3) + assert g(c = 3, a = 1, b = 2) == (1, 2, 3) + +[file testmodule.py] +from typing import Tuple + +def g(a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c + +def get_lambda_function(): + return (lambda a, b, c: (a, b, c)) + +[file driver.py] +import native +native.test_call_native_function_with_keyword_args() +native.test_call_module_function_with_keyword_args() +native.test_call_python_function_with_keyword_args() +native.test_call_lambda_function_with_keyword_args() + +[case testStarArgs] +from typing import Tuple + +def g(a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c + +def test_star_args() -> None: + assert g(*[1, 2, 3]) == (1, 2, 3) + assert g(*(1, 2, 3)) == (1, 2, 3) + assert g(*(1,), *[2, 3]) == (1, 2, 3) + assert g(*(), *(1,), *(), *(2,), *(3,), *()) == (1, 2, 3) + assert g(*range(3)) == (0, 1, 2) + +[file driver.py] +import native +native.test_star_args() + +[case testStar2Args] +from typing import Tuple + +def g(a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c + +def test_star2_args() -> None: + assert g(**{'a': 1, 'b': 2, 'c': 3}) == (1, 2, 3) + assert g(**{'c': 3, 'a': 1, 'b': 2}) == (1, 2, 3) + assert g(b=2, **{'a': 1, 'c': 3}) == (1, 2, 3) + +def test_star2_args_bad(v: dict) -> bool: + return g(a=1, b=2, **v) == (1, 2, 3) +[file driver.py] +import native +native.test_star2_args() + +# this should raise TypeError due to duplicate kwarg, but currently it doesn't +assert native.test_star2_args_bad({'b': 2, 'c': 3}) + +[case testStarAndStar2Args] +from typing import Tuple +def g(a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c + +class C: + def g(self, a: int, b: int, c: int) -> Tuple[int, int, int]: + return a, b, c + +def test_star_and_star2_args() -> None: + assert g(1, *(2,), **{'c': 3}) == (1, 2, 3) + assert g(*[1], **{'b': 2, 'c': 3}) == (1, 2, 3) + c = C() + assert c.g(1, *(2,), **{'c': 3}) == (1, 2, 3) + assert c.g(*[1], **{'b': 2, 'c': 3}) == (1, 2, 3) + +[file driver.py] +import native +native.test_star_and_star2_args() + +[case testAllTheArgCombinations] +from typing import Tuple +def g(a: int, b: int, c: int, d: int = -1) -> Tuple[int, int, int, int]: + return a, b, c, d + +class C: + def g(self, a: int, b: int, c: int, d: int = -1) -> Tuple[int, int, int, int]: + return a, b, c, d + +def test_all_the_arg_combinations() -> None: + assert g(1, *(2,), **{'c': 3}) == (1, 2, 3, -1) + assert g(*[1], **{'b': 2, 'c': 3, 'd': 4}) == (1, 2, 3, 4) + c = C() + assert c.g(1, *(2,), **{'c': 3}) == (1, 2, 3, -1) + assert c.g(*[1], **{'b': 2, 'c': 3, 'd': 4}) == (1, 2, 3, 4) + +[file driver.py] +import native +native.test_all_the_arg_combinations() + +[case testOverloads] +from typing import overload, Union, Tuple + +@overload +def foo(x: int) -> int: ... + +@overload +def foo(x: str) -> str: ... + +def foo(x: Union[int, str]) -> Union[int, str]: + return x + +class A: + @overload + def foo(self, x: int) -> int: ... + + @overload + def foo(self, x: str) -> str: ... + + def foo(self, x: Union[int, str]) -> Union[int, str]: + return x + +def call1() -> Tuple[int, str]: + return (foo(10), foo('10')) +def call2() -> Tuple[int, str]: + x = A() + return (x.foo(10), x.foo('10')) + +[file driver.py] +from native import * +assert call1() == (10, '10') +assert call2() == (10, '10') + +[case testDecorators1] +from typing import Generator, Callable, Iterator +from contextlib import contextmanager + +def a(f: Callable[[], None]) -> Callable[[], None]: + def g() -> None: + print('Entering') + f() + print('Exited') + return g + +def b(f: Callable[[], None]) -> Callable[[], None]: + def g() -> None: + print('***') + f() + print('***') + return g + +@contextmanager +def foo() -> Iterator[int]: + try: + print('started') + yield 0 + finally: + print('finished') + +@contextmanager +def catch() -> Iterator[None]: + try: + print('started') + yield + except IndexError: + print('index') + raise + except Exception: + print('lol') + +def thing() -> None: + c() + +@a +@b +def c() -> None: + @a + @b + def d() -> None: + print('d') + print('c') + d() + +def hm() -> None: + x = [1] + with catch(): + x[2] + +[file driver.py] +from native import foo, c, thing, hm + +with foo() as f: + print('hello') + +c() +thing() +print('==') +try: + hm() +except IndexError: + pass +else: + assert False + +[out] +started +hello +finished +Entering +*** +c +Entering +*** +d +*** +Exited +*** +Exited +Entering +*** +c +Entering +*** +d +*** +Exited +*** +Exited +== +started +index + +[case testDecoratorsMethods] +from typing import Any, Callable, Iterator, TypeVar +from contextlib import contextmanager + +T = TypeVar('T') +def dec(f: T) -> T: + return f + +def a(f: Callable[[Any], None]) -> Callable[[Any], None]: + def g(a: Any) -> None: + print('Entering') + f(a) + print('Exited') + return g + +class A: + @a + def foo(self) -> None: + print('foo') + + @contextmanager + def generator(self) -> Iterator[int]: + try: + print('contextmanager: entering') + yield 0 + finally: + print('contextmanager: exited') + +class Lol: + @staticmethod + def foo() -> None: + Lol.bar() + Lol.baz() + + @staticmethod + @dec + def bar() -> None: + pass + + @classmethod + @dec + def baz(cls) -> None: + pass + +def inside() -> None: + with A().generator() as g: + print('hello!') + +with A().generator() as g: + print('hello!') + +def lol() -> None: + with A().generator() as g: + raise Exception + +[file driver.py] +from native import A, lol + +A.foo(A()) +A().foo() +with A().generator() as g: + print('hello!') +try: + lol() +except: + pass +else: + assert False + +[out] +contextmanager: entering +hello! +contextmanager: exited +Entering +foo +Exited +Entering +foo +Exited +contextmanager: entering +hello! +contextmanager: exited +contextmanager: entering +contextmanager: exited + +[case testUnannotatedFunction] +def g(x: int) -> int: + return x * 2 + +def f(x): + return g(x) +[file driver.py] +from native import f +assert f(3) == 6 + +[case testUnannotatedModuleLevelInitFunction] +# Ensure that adding an implicit `-> None` annotation only applies to `__init__` +# _methods_ specifically (not module-level `__init__` functions). +def __init__(): + return 42 +[file driver.py] +from native import __init__ +assert __init__() == 42 + +[case testDifferentArgCountsFromInterpreted] +# Test various signatures from interpreted code. +def noargs() -> int: + return 5 + +def onearg(x: int) -> int: + return x + 1 + +def twoargs(x: int, y: str) -> int: + return x + len(y) + +def one_or_two(x: int, y: str = 'a') -> int: + return x + len(y) + +[file driver.py] +from native import noargs, onearg, twoargs, one_or_two +from testutil import assertRaises + +assert noargs() == 5 +t = () +assert noargs(*t) == 5 +d = {} +assert noargs(**d) == 5 +assert noargs(*t, **d) == 5 + +assert onearg(12) == 13 +assert onearg(x=8) == 9 +t = (1,) +assert onearg(*t) == 2 +d = {'x': 5} +assert onearg(**d) == 6 + +# Test a bogus call to twoargs before any correct calls are made +with assertRaises(TypeError, "twoargs() missing required argument 'x' (pos 1)"): + twoargs() + +assert twoargs(5, 'foo') == 8 +assert twoargs(4, y='foo') == 7 +assert twoargs(y='foo', x=7) == 10 +t = (1, 'xy') +assert twoargs(*t) == 3 +d = {'y': 'xy'} +assert twoargs(2, **d) == 4 + +assert one_or_two(5) == 6 +assert one_or_two(x=3) == 4 +assert one_or_two(6, 'xy') == 8 +assert one_or_two(7, y='xy') == 9 +assert one_or_two(y='xy', x=2) == 4 +assert one_or_two(*t) == 3 +d = {'x': 5} +assert one_or_two(**d) == 6 +assert one_or_two(y='xx', **d) == 7 +d = {'y': 'abc'} +assert one_or_two(1, **d) == 4 + +with assertRaises(TypeError, 'noargs() takes at most 0 arguments (1 given)'): + noargs(1) +with assertRaises(TypeError, 'noargs() takes at most 0 keyword arguments (1 given)'): + noargs(x=1) + +with assertRaises(TypeError, "onearg() missing required argument 'x' (pos 1)"): + onearg() +with assertRaises(TypeError, 'onearg() takes at most 1 argument (2 given)'): + onearg(1, 2) +with assertRaises(TypeError, "onearg() missing required argument 'x' (pos 1)"): + onearg(y=1) +with assertRaises(TypeError, "onearg() takes at most 1 argument (2 given)"): + onearg(1, y=1) + +with assertRaises(TypeError, "twoargs() missing required argument 'x' (pos 1)"): + twoargs() +with assertRaises(TypeError, "twoargs() missing required argument 'y' (pos 2)"): + twoargs(1) +with assertRaises(TypeError, 'twoargs() takes at most 2 arguments (3 given)'): + twoargs(1, 'x', 2) +with assertRaises(TypeError, 'twoargs() takes at most 2 arguments (3 given)'): + twoargs(1, 'x', y=2) + +with assertRaises(TypeError, "one_or_two() missing required argument 'x' (pos 1)"): + one_or_two() +with assertRaises(TypeError, 'one_or_two() takes at most 2 arguments (3 given)'): + one_or_two(1, 'x', 2) +with assertRaises(TypeError, 'one_or_two() takes at most 2 arguments (3 given)'): + one_or_two(1, 'x', y=2) + +[case testComplicatedArgs] +from typing import Tuple, Dict + +def kwonly1(x: int = 0, *, y: int) -> Tuple[int, int]: + return x, y + +def kwonly2(*, x: int = 0, y: int) -> Tuple[int, int]: + return x, y + +def kwonly3(a: int, b: int = 0, *, y: int, x: int = 1) -> Tuple[int, int, int, int]: + return a, b, x, y + +def kwonly4(*, x: int, y: int) -> Tuple[int, int]: + return x, y + +def varargs1(*args: int) -> Tuple[int, ...]: + return args + +def varargs2(*args: int, **kwargs: int) -> Tuple[Tuple[int, ...], Dict[str, int]]: + return args, kwargs + +def varargs3(**kwargs: int) -> Dict[str, int]: + return kwargs + +def varargs4(a: int, b: int = 0, + *args: int, y: int, x: int = 1, + **kwargs: int) -> Tuple[Tuple[int, ...], Dict[str, int]]: + return (a, b, *args), {'x': x, 'y': y, **kwargs} + +class A: + def f(self, x: int) -> Tuple[int, ...]: + return (x,) + def g(self, x: int) -> Tuple[Tuple[int, ...], Dict[str, int]]: + return (x,), {} + +class B(A): + def f(self, *args: int) -> Tuple[int, ...]: + return args + def g(self, *args: int, **kwargs: int) -> Tuple[Tuple[int, ...], Dict[str, int]]: + return args, kwargs + +[file other.py] +# This file is imported in both compiled and interpreted mode in order to +# test both native calls and calls via the C API. + +from native import ( + kwonly1, kwonly2, kwonly3, kwonly4, + varargs1, varargs2, varargs3, varargs4, + A, B +) + +# kwonly arg tests +assert kwonly1(10, y=20) == (10, 20) +assert kwonly1(y=20) == (0, 20) + +assert kwonly2(x=10, y=20) == (10, 20) +assert kwonly2(y=20) == (0, 20) + +assert kwonly3(10, y=20) == (10, 0, 1, 20) +assert kwonly3(a=10, y=20) == (10, 0, 1, 20) +assert kwonly3(10, 30, y=20) == (10, 30, 1, 20) +assert kwonly3(10, b=30, y=20) == (10, 30, 1, 20) +assert kwonly3(a=10, b=30, y=20) == (10, 30, 1, 20) + +assert kwonly3(10, x=40, y=20) == (10, 0, 40, 20) +assert kwonly3(a=10, x=40, y=20) == (10, 0, 40, 20) +assert kwonly3(10, 30, x=40, y=20) == (10, 30, 40, 20) +assert kwonly3(10, b=30, x=40, y=20) == (10, 30, 40, 20) +assert kwonly3(a=10, b=30, x=40, y=20) == (10, 30, 40, 20) + +assert kwonly4(x=1, y=2) == (1, 2) +assert kwonly4(y=2, x=1) == (1, 2) + +# varargs tests +assert varargs1() == () +assert varargs1(1, 2, 3) == (1, 2, 3) +assert varargs1(1, *[2, 3, 4], 5, *[6, 7, 8], 9) == (1, 2, 3, 4, 5, 6, 7, 8, 9) +assert varargs2(1, 2, 3) == ((1, 2, 3), {}) +assert varargs2(1, 2, 3, x=4) == ((1, 2, 3), {'x': 4}) +assert varargs2(x=4) == ((), {'x': 4}) +assert varargs3() == {} +assert varargs3(x=4) == {'x': 4} +assert varargs3(x=4, y=5) == {'x': 4, 'y': 5} + +assert varargs4(-1, y=2) == ((-1, 0), {'x': 1, 'y': 2}) +assert varargs4(-1, 2, y=2) == ((-1, 2), {'x': 1, 'y': 2}) +assert varargs4(-1, 2, 3, y=2) == ((-1, 2, 3), {'x': 1, 'y': 2}) +assert varargs4(-1, 2, 3, x=10, y=2) == ((-1, 2, 3), {'x': 10, 'y': 2}) +assert varargs4(-1, x=10, y=2) == ((-1, 0), {'x': 10, 'y': 2}) +assert varargs4(-1, y=2, z=20) == ((-1, 0), {'x': 1, 'y': 2, 'z': 20}) +assert varargs4(-1, 2, y=2, z=20) == ((-1, 2), {'x': 1, 'y': 2, 'z': 20}) +assert varargs4(-1, 2, 3, y=2, z=20) == ((-1, 2, 3), {'x': 1, 'y': 2, 'z': 20}) +assert varargs4(-1, 2, 3, x=10, y=2, z=20) == ((-1, 2, 3), {'x': 10, 'y': 2, 'z': 20}) +assert varargs4(-1, x=10, y=2, z=20) == ((-1, 0), {'x': 10, 'y': 2, 'z': 20}) + +x = B() # type: A +assert x.f(1) == (1,) +assert x.g(1) == ((1,), {}) +# This one is really funny! When we make native calls we lose +# track of which arguments are positional or keyword, so the glue +# calls them all positional unless they are keyword only... +# It would be possible to fix this by dynamically tracking which +# arguments were passed by keyword (for example, by passing a bitmask +# to functions indicating this), but paying a speed, size, and complexity +# cost for something so deeply marginal seems like a bad choice. +# assert x.g(x=1) == ((), {'x': 1}) + +[file driver.py] +from testutil import assertRaises +from native import ( + kwonly1, kwonly2, kwonly3, kwonly4, + varargs1, varargs2, varargs3, varargs4, +) + +# Run the non-exceptional tests in both interpreted and compiled mode +import other +import other_interpreted + + +# And the tests for errors at the interfaces in interpreted only +with assertRaises(TypeError, "missing required keyword-only argument 'y'"): + kwonly1() +with assertRaises(TypeError, "takes at most 1 positional argument (2 given)"): + kwonly1(10, 20) + +with assertRaises(TypeError, "missing required keyword-only argument 'y'"): + kwonly2() +with assertRaises(TypeError, "takes no positional arguments"): + kwonly2(10, 20) + +with assertRaises(TypeError, "missing required argument 'a'"): + kwonly3(b=30, x=40, y=20) +with assertRaises(TypeError, "missing required keyword-only argument 'y'"): + kwonly3(10) + +with assertRaises(TypeError, "missing required keyword-only argument 'y'"): + kwonly4(x=1) +with assertRaises(TypeError, "missing required keyword-only argument 'x'"): + kwonly4(y=1) +with assertRaises(TypeError, "missing required keyword-only argument 'x'"): + kwonly4() + +with assertRaises(TypeError, "'x' is an invalid keyword argument for varargs1()"): + varargs1(x=10) +with assertRaises(TypeError, "'x' is an invalid keyword argument for varargs1()"): + varargs1(1, x=10) +with assertRaises(TypeError, "varargs3() takes no positional arguments"): + varargs3(10) +with assertRaises(TypeError, "varargs3() takes no positional arguments"): + varargs3(10, x=10) + +with assertRaises(TypeError, "varargs4() missing required argument 'a' (pos 1)"): + varargs4() +with assertRaises(TypeError, "varargs4() missing required keyword-only argument 'y'"): + varargs4(1, 2) +with assertRaises(TypeError, "varargs4() missing required keyword-only argument 'y'"): + varargs4(1, 2, x=1) +with assertRaises(TypeError, "varargs4() missing required keyword-only argument 'y'"): + varargs4(1, 2, 3) +with assertRaises(TypeError, "varargs4() missing required argument 'a' (pos 1)"): + varargs4(y=20) + +[case testDecoratorName] +def dec(f): return f + +@dec +def foo(): pass + +def test_decorator_name(): + assert foo.__name__ == "foo" + +[case testLambdaArgToOverloaded] +from lib import sub + +def test_str_overload() -> None: + assert sub('x', lambda m: m) == 'x' + +def test_bytes_overload() -> None: + assert sub(b'x', lambda m: m) == b'x' + +[file lib.py] +from typing import overload, Callable, TypeVar, Generic + +T = TypeVar("T") + +class Match(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + + def group(self, n: int) -> T: + return self.x + +@overload +def sub(s: str, f: Callable[[str], str]) -> str: ... +@overload +def sub(s: bytes, f: Callable[[bytes], bytes]) -> bytes: ... +def sub(s, f): + return f(s) + +[case testContextManagerSpecialCase] +from typing import Generator, Callable, Iterator +from contextlib import contextmanager + +@contextmanager +def f() -> Iterator[None]: + yield + +def g() -> None: + a = [''] + with f(): + a.pop() + +g() + +[case testUnpackKwargsCompiled] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]) -> None: + print(kwargs["name"]) + +# This is not really supported yet, just test that we behave reasonably. +foo(name='Jennifer', age=38) +[out] +Jennifer + +[case testNestedFunctionDunderDict312] +import sys + +def foo() -> None: + def inner() -> str: return "bar" + print(inner.__dict__) # type: ignore[attr-defined] + inner.__dict__.update({"x": 1}) # type: ignore[attr-defined] + print(inner.__dict__) # type: ignore[attr-defined] + print(inner.x) # type: ignore[attr-defined] + +if sys.version_info >= (3, 12): # type: ignore + foo() +[out] +[out version>=3.12] +{} +{'x': 1} +1 + +[case testFunctoolsUpdateWrapper] +import functools + +def bar() -> None: + def inner() -> str: return "bar" + functools.update_wrapper(inner, bar) # type: ignore + print(inner.__dict__) # type: ignore + +bar() +[out] +{'__module__': 'native', '__name__': 'bar', '__qualname__': 'bar', '__doc__': None, '__wrapped__': } + +[case testCallNestedFunctionWithNamed] +def f() -> None: + def a() -> None: + pass + def b() -> None: + a() + b() +[file driver.py] +from native import f +f() + +[case testCallNestedFunctionWithLambda] +def f(x: int) -> int: + def inc(x: int) -> int: + return x + 1 + return (lambda x: inc(x))(1) +[file driver.py] +from native import f +print(f(1)) +[out] +2 diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test new file mode 100644 index 000000000000..7e9804c49582 --- /dev/null +++ b/mypyc/test-data/run-generators.test @@ -0,0 +1,682 @@ +# Test cases for generators and yield (compile and run) + +[case testYield] +from typing import Generator, Iterable, Union, Tuple, Dict + +def yield_three_times() -> Iterable[int]: + yield 1 + yield 2 + yield 3 + +def yield_twice_and_return() -> Generator[int, None, int]: + yield 1 + yield 2 + return 4 + +def yield_while_loop() -> Generator[int, None, int]: + i = 0 + while i < 5: + if i == 3: + return i + yield i + i += 1 + return -1 + +def yield_for_loop() -> Iterable[int]: + l = [i for i in range(3)] + for i in l: + yield i + + d = {k: None for k in range(3)} + for k in d: + yield k + + for i in range(3): + yield i + + for i in range(three()): + yield i + +def yield_with_except() -> Generator[int, None, None]: + yield 10 + try: + return + except: + print('Caught exception inside generator function') + +def complex_yield(a: int, b: str, c: float) -> Generator[Union[str, int], None, float]: + x = 2 + while x < a: + if x % 2 == 0: + dummy_var = 1 + yield str(x) + ' ' + b + dummy_var = 1 + else: + dummy_var = 1 + yield x + dummy_var = 1 + x += 1 + return c + +def yield_with_default(x: bool = False) -> Iterable[int]: + if x: + yield 0 + +def yield_dict_methods(d1: Dict[int, int], + d2: Dict[int, int], + d3: Dict[int, int]) -> Iterable[int]: + for k in d1.keys(): + yield k + for k, v in d2.items(): + yield k + yield v + for v in d3.values(): + yield v + +def three() -> int: + return 3 + +class A(object): + def __init__(self, x: int) -> None: + self.x = x + + def generator(self) -> Iterable[int]: + yield self.x + +def return_tuple() -> Generator[int, None, Tuple[int, int]]: + yield 0 + return 1, 2 + +[file driver.py] +from native import ( + yield_three_times, + yield_twice_and_return, + yield_while_loop, + yield_for_loop, + yield_with_except, + complex_yield, + yield_with_default, + A, + return_tuple, + yield_dict_methods, +) +from testutil import run_generator +from collections import defaultdict + +assert run_generator(yield_three_times()) == ((1, 2, 3), None) +assert run_generator(yield_twice_and_return()) == ((1, 2), 4) +assert run_generator(yield_while_loop()) == ((0, 1, 2), 3) +assert run_generator(yield_for_loop()) == (tuple(4 * [i for i in range(3)]), None) +assert run_generator(yield_with_except()) == ((10,), None) +assert run_generator(complex_yield(5, 'foo', 1.0)) == (('2 foo', 3, '4 foo'), 1.0) +assert run_generator(yield_with_default()) == ((), None) +assert run_generator(A(0).generator()) == ((0,), None) +assert run_generator(return_tuple()) == ((0,), (1, 2)) +assert run_generator(yield_dict_methods({}, {}, {})) == ((), None) +assert run_generator(yield_dict_methods({1: 2}, {3: 4}, {5: 6})) == ((1, 3, 4, 6), None) +dd = defaultdict(int, {0: 1}) +assert run_generator(yield_dict_methods(dd, dd, dd)) == ((0, 0, 1, 1), None) + +for i in yield_twice_and_return(): + print(i) + +for i in yield_while_loop(): + print(i) + +[out] +1 +2 +0 +1 +2 + +[case testYieldTryFinallyWith] +from typing import Generator, Any + +class Thing: + def __init__(self, x: str) -> None: + self.x = x + def __enter__(self) -> str: + print('enter!', self.x) + if self.x == 'crash': + raise Exception('ohno') + return self.x + def __exit__(self, x: Any, y: Any, z: Any) -> None: + print('exit!', self.x, y) + +def yield_try_finally() -> Generator[int, None, str]: + try: + yield 1 + yield 2 + return 'lol' + except Exception: + raise + finally: + print('goodbye!') + +def yield_with(i: int) -> Generator[int, None, int]: + with Thing('a') as x: + yield 1 + print("yooo?", x) + if i == 0: + yield 2 + return 10 + elif i == 1: + raise Exception('exception!') + return -1 + +[file driver.py] +from native import yield_try_finally, yield_with +from testutil import run_generator + +print(run_generator(yield_try_finally(), p=True)) +print(run_generator(yield_with(0), p=True)) +print(run_generator(yield_with(1), p=True)) +[out] +1 +2 +goodbye! +((1, 2), 'lol') +enter! a +1 +yooo? a +2 +exit! a None +((1, 2), 10) +enter! a +1 +yooo? a +exit! a exception! +((1,), 'exception!') + +[case testYieldNested] +from typing import Callable, Generator + +def normal(a: int, b: float) -> Callable: + def generator(x: int, y: str) -> Generator: + yield a + yield b + yield x + yield y + return generator + +def generator(a: int) -> Generator: + def normal(x: int) -> int: + return a + x + for i in range(3): + yield normal(i) + +def triple() -> Callable: + def generator() -> Generator: + x = 0 + def inner() -> int: + x += 1 + return x + while x < 3: + yield inner() + return generator + +def another_triple() -> Callable: + def generator() -> Generator: + x = 0 + def inner_generator() -> Generator: + x += 1 + yield x + yield next(inner_generator()) + return generator + +def outer() -> Generator: + def recursive(n: int) -> Generator: + if n < 10: + for i in range(n): + yield i + return + for i in recursive(5): + yield i + return recursive(10) + +[file driver.py] +from native import normal, generator, triple, another_triple, outer +from testutil import run_generator + +assert run_generator(normal(1, 2.0)(3, '4.00')) == ((1, 2.0, 3, '4.00'), None) +assert run_generator(generator(1)) == ((1, 2, 3), None) +assert run_generator(triple()()) == ((1, 2, 3), None) +assert run_generator(another_triple()()) == ((1,), None) +assert run_generator(outer()) == ((0, 1, 2, 3, 4), None) + +[case testYieldThrow] +from typing import Generator, Iterable, Any, Union +from traceback import print_tb +from contextlib import contextmanager +import wrapsys + +def generator() -> Generator[int, None, Union[int, None]]: + try: + yield 1 + yield 2 + yield 3 + except Exception as e: + print_tb(wrapsys.exc_info()[2]) + s = str(e) + if s: + print('caught exception with value ' + s) + else: + print('caught exception without value') + return 0 + return None + +def no_except() -> Iterable[int]: + yield 1 + yield 2 + +def raise_something() -> Iterable[int]: + yield 1 + yield 2 + raise Exception('failure') + +def wrapper(x: Any) -> Any: + return (yield from x) + +def foo() -> Generator[int, None, None]: + try: + yield 1 + except Exception as e: + print(str(e)) + finally: + print('goodbye') + +ctx_manager = contextmanager(foo) + +[file wrapsys.py] +# This is a gross hack around some limitations of the test system/mypyc. +from typing import Any +import sys +def exc_info() -> Any: + return sys.exc_info() # type: ignore + +[file driver.py] +import sys +from typing import Generator, Tuple, TypeVar, Sequence +from native import generator, ctx_manager, wrapper, no_except, raise_something + +T = TypeVar('T') +U = TypeVar('U') + +def run_generator_and_throw(gen: Generator[T, None, U], + num_times: int, + value: object = None, + traceback: object = None) -> Tuple[Sequence[T], U]: + res = [] + try: + for i in range(num_times): + res.append(next(gen)) + if value is not None and traceback is not None: + gen.throw(Exception, value, traceback) + elif value is not None: + gen.throw(Exception, value) + else: + gen.throw(Exception) + except StopIteration as e: + return (tuple(res), e.value) + except Exception as e: + return (tuple(res), str(e)) + +assert run_generator_and_throw(generator(), 0, 'hello') == ((), 'hello') +assert run_generator_and_throw(generator(), 3) == ((1, 2, 3), 0) +assert run_generator_and_throw(generator(), 2, 'some string') == ((1, 2), 0) +try: + raise Exception +except Exception as e: + tb = sys.exc_info()[2] + assert run_generator_and_throw(generator(), 1, 'some other string', tb) == ((1,), 0) + +assert run_generator_and_throw(wrapper(generator()), 0, 'hello') == ((), 'hello') +assert run_generator_and_throw(wrapper(generator()), 3) == ((1, 2, 3), 0) +assert run_generator_and_throw(wrapper(generator()), 2, 'some string') == ((1, 2), 0) +# Make sure we aren't leaking exc_info +assert sys.exc_info()[0] is None + +assert run_generator_and_throw(wrapper([1, 2, 3]), 3, 'lol') == ((1, 2, 3), 'lol') +assert run_generator_and_throw(wrapper(no_except()), 2, 'lol') == ((1, 2), 'lol') + +assert run_generator_and_throw(wrapper(raise_something()), 3) == ((1, 2), 'failure') + +with ctx_manager() as c: + raise Exception('exception') + +[out] + File "native.py", line 10, in generator + yield 3 + File "native.py", line 9, in generator + yield 2 + File "native.py", line 8, in generator + yield 1 + File "driver.py", line 31, in + raise Exception + File "native.py", line 10, in generator + yield 3 + File "native.py", line 31, in wrapper + return (yield from x) + File "native.py", line 9, in generator + yield 2 + File "native.py", line 31, in wrapper + return (yield from x) +caught exception without value +caught exception with value some string +caught exception with value some other string +caught exception without value +caught exception with value some string +exception +goodbye + +[case testYieldSend] +from typing import Generator + +def basic() -> Generator[int, int, int]: + x = yield 1 + y = yield (x + 1) + return y + +def use_from() -> Generator[int, int, int]: + return (yield from basic()) + +[file driver.py] +from native import basic, use_from +from testutil import run_generator + +assert run_generator(basic(), [5, 50]) == ((1, 6), 50) +assert run_generator(use_from(), [5, 50]) == ((1, 6), 50) + +[case testYieldFrom] +from typing import Generator, Iterator, List + +def basic() -> Iterator[int]: + yield from [1, 2, 3] + +def call_next() -> int: + x = [] # type: List[int] + return next(iter(x)) + +def inner(b: bool) -> Generator[int, None, int]: + if b: + yield from [1, 2, 3] + return 10 + +def with_return(b: bool) -> Generator[int, None, int]: + x = yield from inner(b) + for a in [1, 2]: + pass + return x + +[file driver.py] +from native import basic, call_next, with_return +from testutil import run_generator, assertRaises + +assert run_generator(basic()) == ((1, 2, 3), None) + +with assertRaises(StopIteration): + call_next() + +assert run_generator(with_return(True)) == ((1, 2, 3), 10) +assert run_generator(with_return(False)) == ((), 10) + +[case testNextGenerator] +from typing import Iterable + +def f(x: int) -> int: + print(x) + return x + +def call_next_loud(l: Iterable[int], val: int) -> int: + return next(i for i in l if f(i) == val) + +def call_next_default(l: Iterable[int], val: int) -> int: + return next((i*2 for i in l if i == val), -1) + +def call_next_default_list(l: Iterable[int], val: int) -> int: + return next((i*2 for i in l if i == val), -1) +[file driver.py] +from native import call_next_loud, call_next_default, call_next_default_list +from testutil import assertRaises + +assert call_next_default([0, 1, 2], 0) == 0 +assert call_next_default([0, 1, 2], 1) == 2 +assert call_next_default([0, 1, 2], 2) == 4 +assert call_next_default([0, 1, 2], 3) == -1 +assert call_next_default([], 0) == -1 +assert call_next_default_list([0, 1, 2], 0) == 0 +assert call_next_default_list([0, 1, 2], 1) == 2 +assert call_next_default_list([0, 1, 2], 2) == 4 +assert call_next_default_list([0, 1, 2], 3) == -1 +assert call_next_default_list([], 0) == -1 + +assert call_next_loud([0, 1, 2], 0) == 0 +assert call_next_loud([0, 1, 2], 1) == 1 +assert call_next_loud([0, 1, 2], 2) == 2 +with assertRaises(StopIteration): + call_next_loud([42], 3) +with assertRaises(StopIteration): + call_next_loud([], 3) + +[out] +0 +0 +1 +0 +1 +2 +42 + +[case testGeneratorSuper] +from typing import Iterator, Callable, Any + +class A(): + def testA(self) -> int: + return 2 + +class B(A): + def testB(self) -> Iterator[int]: + x = super().testA() + while True: + yield x + +def testAsserts(): + b = B() + b_gen = b.testB() + assert next(b_gen) == 2 + +[file driver.py] +from native import testAsserts + +testAsserts() + +[case testNameClashIssues] +class A: + def foo(self) -> object: + yield +class B: + def foo(self) -> object: + yield + +class C: + def foo(self) -> None: + def bar(self) -> None: + pass + +def C___foo() -> None: pass + +class D: + def foo(self) -> None: + def bar(self) -> None: + pass + +class E: + default: int + switch: int + +[file driver.py] +# really I only care it builds + +[case testCloseStopIterationRaised] +def g() -> object: + try: + yield 1 + except GeneratorExit: + raise + +[file driver.py] +from native import g + +gen = g() +next(gen) +gen.close() + +[case testCloseGeneratorExitRaised] +def g() -> object: + yield 1 + +[file driver.py] +from native import g + +gen = g() +next(gen) +gen.close() + +[case testCloseGeneratorExitIgnored] +def g() -> object: + try: + yield 1 + except GeneratorExit: + pass + + yield 2 + +[file driver.py] +from native import g + +gen = g() +next(gen) +try: + gen.close() +except RuntimeError as e: + assert str(e) == 'generator ignored GeneratorExit' +else: + assert False + +[case testCloseGeneratorRaisesAnotherException] +def g() -> object: + try: + yield 1 + except GeneratorExit: + raise RuntimeError("error") + +[file driver.py] +from native import g + +gen = g() +next(gen) +try: + gen.close() +except RuntimeError as e: + assert str(e) == 'error' +else: + assert False + +[case testBorrowingInGeneratorNearYield] +from typing import Iterator + +class Foo: + flag: bool + +class C: + foo: Foo + + def genf(self) -> Iterator[None]: + self.foo.flag = True + yield + self.foo.flag = False + +[case testGeneratorEarlyReturnWithBorrows] +from typing import Iterator +class Bar: + bar = 0 +class Foo: + bar = Bar() + def f(self) -> Iterator[int]: + if self: + self.bar.bar += 1 + return + yield 0 + +[case testBorrowingInGeneratorInTupleAssignment] +from typing import Iterator + +class Foo: + flag1: bool + flag2: bool + +class C: + foo: Foo + + def genf(self) -> Iterator[None]: + self.foo.flag1, self.foo.flag2 = True, True + yield + self.foo.flag1, self.foo.flag2 = False, False + +def test_generator() -> None: + c = C() + c.foo = Foo() + gen = c.genf() + next(gen) + assert c.foo.flag1 == c.foo.flag2 == True + assert list(gen) == [] + assert c.foo.flag1 == c.foo.flag2 == False + + +[case testYieldInFinally] +from typing import Generator + +def finally_yield() -> Generator[str, None, str]: + try: + return 'test' + finally: + yield 'x' + + +[file driver.py] +from native import finally_yield +from testutil import run_generator + +yields, val = run_generator(finally_yield()) +assert yields == ('x',) +assert val == 'test', val + +[case testUnreachableComprehensionNoCrash] +from typing import List + +def list_comp() -> List[int]: + if True: + return [5] + return [i for i in [5]] + +[file driver.py] +from native import list_comp +assert list_comp() == [5] + +[case testWithNative] +class DummyContext: + def __init__(self) -> None: + self.x = 0 + + def __enter__(self) -> None: + self.x += 1 + + def __exit__(self, exc_type, exc_value, exc_tb) -> None: + self.x -= 1 + +def test_basic() -> None: + context = DummyContext() + with context: + assert context.x == 1 + assert context.x == 0 diff --git a/mypyc/test-data/run-i16.test b/mypyc/test-data/run-i16.test new file mode 100644 index 000000000000..fbb0c15220bc --- /dev/null +++ b/mypyc/test-data/run-i16.test @@ -0,0 +1,338 @@ +[case testI16BasicOps] +from typing import Any, Tuple + +from mypy_extensions import i16, i32, i64 + +from testutil import assertRaises + +def test_box_and_unbox() -> None: + values = (list(range(-2**15, -2**15 + 100)) + + list(range(-1000, 1000)) + + list(range(2**15 - 100, 2**15))) + for i in values: + o: Any = i + x: i16 = o + o2: Any = x + assert o == o2 + assert x == i + with assertRaises(OverflowError, "int too large to convert to i16"): + o = 2**15 + x2: i16 = o + with assertRaises(OverflowError, "int too large to convert to i16"): + o = -2**15 - 1 + x3: i16 = o + +def div_by_7(x: i16) -> i16: + return x // 7 +def div_by_neg_7(x: i16) -> i16: + return x // -7 + +def div(x: i16, y: i16) -> i16: + return x // y + +def test_divide_by_constant() -> None: + for i in range(-1000, 1000): + assert div_by_7(i) == i // 7 + for i in range(-2**15, -2**15 + 1000): + assert div_by_7(i) == i // 7 + for i in range(2**15 - 1000, 2**15): + assert div_by_7(i) == i // 7 + +def test_divide_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert div_by_neg_7(i) == i // -7 + for i in range(-2**15, -2**15 + 1000): + assert div_by_neg_7(i) == i // -7 + for i in range(2**15 - 1000, 2**15): + assert div_by_neg_7(i) == i // -7 + +def test_divide_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**15, -2**15 + 10)) + + list(range(2**15 - 10, 2**15))) + for x in values: + for y in values: + if y != 0: + if x // y == 2**15: + with assertRaises(OverflowError, "integer division overflow"): + div(x, y) + else: + assert div(x, y) == x // y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + div(x, y) + +def mod_by_7(x: i16) -> i16: + return x % 7 + +def mod_by_neg_7(x: i16) -> i16: + return x // -7 + +def mod(x: i16, y: i16) -> i16: + return x % y + +def test_mod_by_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_7(i) == i % 7 + for i in range(-2**15, -2**15 + 1000): + assert mod_by_7(i) == i % 7 + for i in range(2**15 - 1000, 2**15): + assert mod_by_7(i) == i % 7 + +def test_mod_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(-2**15, -2**15 + 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(2**15 - 1000, 2**15): + assert mod_by_neg_7(i) == i // -7 + +def test_mod_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**15, -2**15 + 10)) + + list(range(2**15 - 10, 2**15))) + for x in values: + for y in values: + if y != 0: + assert mod(x, y) == x % y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + mod(x, y) + +def test_simple_arithmetic_ops() -> None: + zero: i16 = int() + one: i16 = zero + 1 + two: i16 = one + 1 + neg_one: i16 = -one + assert one + one == 2 + assert one + two == 3 + assert one + neg_one == 0 + assert one - one == 0 + assert one - two == -1 + assert one * one == 1 + assert one * two == 2 + assert two * two == 4 + assert two * neg_one == -2 + assert neg_one * one == -1 + assert neg_one * neg_one == 1 + assert two * 0 == 0 + assert 0 * two == 0 + assert -one == -1 + assert -two == -2 + assert -neg_one == 1 + assert -zero == 0 + +def test_bitwise_ops() -> None: + x: i16 = 13855 + int() + y: i16 = 367 + int() + z: i16 = -11091 + int() + zero: i16 = int() + one: i16 = zero + 1 + two: i16 = zero + 2 + neg_one: i16 = -one + + assert x & y == 15 + assert x & z == 5133 + assert z & z == z + assert x & zero == 0 + + assert x | y == 14207 + assert x | z == -2369 + assert z | z == z + assert x | 0 == x + + assert x ^ y == 14192 + assert x ^ z == -7502 + assert z ^ z == 0 + assert z ^ 0 == z + + assert x << one == 27710 + assert x << two == -10116 + assert z << two == 21172 + assert z << 0 == z + + assert x >> one == 6927 + assert x >> two == 3463 + assert z >> two == -2773 + assert z >> 0 == z + + assert ~x == -13856 + assert ~z == 11090 + assert ~zero == -1 + assert ~neg_one == 0 + +def eq(x: i16, y: i16) -> bool: + return x == y + +def test_eq() -> None: + assert eq(int(), int()) + assert eq(5 + int(), 5 + int()) + assert eq(-5 + int(), -5 + int()) + assert not eq(int(), 1 + int()) + assert not eq(5 + int(), 6 + int()) + assert not eq(-5 + int(), -6 + int()) + assert not eq(-5 + int(), 5 + int()) + +def test_comparisons() -> None: + one: i16 = 1 + int() + one2: i16 = 1 + int() + two: i16 = 2 + int() + assert one < two + assert not (one < one2) + assert not (two < one) + assert two > one + assert not (one > one2) + assert not (one > two) + assert one <= two + assert one <= one2 + assert not (two <= one) + assert two >= one + assert one >= one2 + assert not (one >= two) + assert one == one2 + assert not (one == two) + assert one != two + assert not (one != one2) + +def test_mixed_comparisons() -> None: + i16_3: i16 = int() + 3 + int_5 = int() + 5 + assert i16_3 < int_5 + assert int_5 > i16_3 + b = i16_3 > int_5 + assert not b + + int_largest = int() + (1 << 15) - 1 + assert int_largest > i16_3 + int_smallest = int() - (1 << 15) + assert i16_3 > int_smallest + + int_too_big = int() + (1 << 15) + int_too_small = int() - (1 << 15) - 1 + with assertRaises(OverflowError): + assert i16_3 < int_too_big + with assertRaises(OverflowError): + assert int_too_big < i16_3 + with assertRaises(OverflowError): + assert i16_3 > int_too_small + with assertRaises(OverflowError): + assert int_too_small < i16_3 + +def test_mixed_arithmetic_and_bitwise_ops() -> None: + i16_3: i16 = int() + 3 + int_5 = int() + 5 + assert i16_3 + int_5 == 8 + assert int_5 - i16_3 == 2 + assert i16_3 << int_5 == 96 + assert int_5 << i16_3 == 40 + assert i16_3 ^ int_5 == 6 + assert int_5 | i16_3 == 7 + + int_largest = int() + (1 << 15) - 1 + assert int_largest - i16_3 == 32764 + int_smallest = int() - (1 << 15) + assert int_smallest + i16_3 == -32765 + + int_too_big = int() + (1 << 15) + int_too_small = int() - (1 << 15) - 1 + with assertRaises(OverflowError): + assert i16_3 & int_too_big + with assertRaises(OverflowError): + assert int_too_small & i16_3 + +def test_coerce_to_and_from_int() -> None: + for shift in range(0, 16): + for sign in 1, -1: + for delta in range(-5, 5): + n = sign * (1 << shift) + delta + if -(1 << 15) <= n < (1 << 15): + x: i16 = n + m: int = x + assert m == n + +def test_explicit_conversion_to_i16() -> None: + x = i16(5) + assert x == 5 + y = int() - 113 + x = i16(y) + assert x == -113 + n64: i64 = 1733 + x = i16(n64) + assert x == 1733 + n32: i32 = -1733 + x = i16(n32) + assert x == -1733 + z = i16(x) + assert z == -1733 + +def test_explicit_conversion_overflow() -> None: + max_i16 = int() + 2**15 - 1 + x = i16(max_i16) + assert x == 2**15 - 1 + assert int(x) == max_i16 + + min_i16 = int() - 2**15 + y = i16(min_i16) + assert y == -2**15 + assert int(y) == min_i16 + + too_big = int() + 2**15 + with assertRaises(OverflowError): + x = i16(too_big) + + too_small = int() - 2**15 - 1 + with assertRaises(OverflowError): + x = i16(too_small) + +def test_i16_from_large_small_literal() -> None: + x = i16(2**15 - 1) + assert x == 2**15 - 1 + x = i16(-2**15) + assert x == -2**15 + +def test_i16_truncate_from_i64() -> None: + large = i64(2**32 + 65536 + 157 + int()) + x = i16(large) + assert x == 157 + small = i64(-2**32 - 65536 - 157 + int()) + x = i16(small) + assert x == -157 + large2 = i64(2**15 + int()) + x = i16(large2) + assert x == -2**15 + small2 = i64(-2**15 - 1 - int()) + x = i16(small2) + assert x == 2**15 - 1 + +def test_i16_truncate_from_i32() -> None: + large = i32(2**16 + 2**30 + 5 + int()) + assert i16(large) == 5 + small = i32(-2**16 - 2**30 - 1 + int()) + assert i16(small) == -1 + +def from_float(x: float) -> i16: + return i16(x) + +def test_explicit_conversion_from_float() -> None: + assert from_float(0.0) == 0 + assert from_float(1.456) == 1 + assert from_float(-1234.567) == -1234 + assert from_float(2**15 - 1) == 2**15 - 1 + assert from_float(-2**15) == -2**15 + # The error message could be better, but this is acceptable + with assertRaises(OverflowError, "int too large to convert to i16"): + assert from_float(float(2**15)) + with assertRaises(OverflowError, "int too large to convert to i16"): + # One ulp below the lowest valid i64 value + from_float(float(-2**15 - 1)) + +def test_tuple_i16() -> None: + a: i16 = 1 + b: i16 = 2 + t = (a, b) + a, b = t + assert a == 1 + assert b == 2 + x: Any = t + tt: Tuple[i16, i16] = x + assert tt == (1, 2) diff --git a/mypyc/test-data/run-i32.test b/mypyc/test-data/run-i32.test new file mode 100644 index 000000000000..bb1fa43bb9fd --- /dev/null +++ b/mypyc/test-data/run-i32.test @@ -0,0 +1,336 @@ +[case testI32BasicOps] +from typing import Any, Tuple + +from mypy_extensions import i16, i32, i64 + +from testutil import assertRaises + +def test_box_and_unbox() -> None: + values = (list(range(-2**31, -2**31 + 100)) + + list(range(-1000, 1000)) + + list(range(2**31 - 100, 2**31))) + for i in values: + o: Any = i + x: i32 = o + o2: Any = x + assert o == o2 + assert x == i + with assertRaises(OverflowError, "int too large to convert to i32"): + o = 2**31 + x2: i32 = o + with assertRaises(OverflowError, "int too large to convert to i32"): + o = -2**32 - 1 + x3: i32 = o + +def div_by_7(x: i32) -> i32: + return x // 7 +def div_by_neg_7(x: i32) -> i32: + return x // -7 + +def div(x: i32, y: i32) -> i32: + return x // y + +def test_divide_by_constant() -> None: + for i in range(-1000, 1000): + assert div_by_7(i) == i // 7 + for i in range(-2**31, -2**31 + 1000): + assert div_by_7(i) == i // 7 + for i in range(2**31 - 1000, 2**31): + assert div_by_7(i) == i // 7 + +def test_divide_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert div_by_neg_7(i) == i // -7 + for i in range(-2**31, -2**31 + 1000): + assert div_by_neg_7(i) == i // -7 + for i in range(2**31 - 1000, 2**31): + assert div_by_neg_7(i) == i // -7 + +def test_divide_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**31, -2**31 + 10)) + + list(range(2**31 - 10, 2**31))) + for x in values: + for y in values: + if y != 0: + if x // y == 2**31: + with assertRaises(OverflowError, "integer division overflow"): + div(x, y) + else: + assert div(x, y) == x // y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + div(x, y) + +def mod_by_7(x: i32) -> i32: + return x % 7 + +def mod_by_neg_7(x: i32) -> i32: + return x // -7 + +def mod(x: i32, y: i32) -> i32: + return x % y + +def test_mod_by_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_7(i) == i % 7 + for i in range(-2**31, -2**31 + 1000): + assert mod_by_7(i) == i % 7 + for i in range(2**31 - 1000, 2**31): + assert mod_by_7(i) == i % 7 + +def test_mod_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(-2**31, -2**31 + 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(2**31 - 1000, 2**31): + assert mod_by_neg_7(i) == i // -7 + +def test_mod_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**31, -2**31 + 10)) + + list(range(2**31 - 10, 2**31))) + for x in values: + for y in values: + if y != 0: + assert mod(x, y) == x % y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + mod(x, y) + +def test_simple_arithmetic_ops() -> None: + zero: i32 = int() + one: i32 = zero + 1 + two: i32 = one + 1 + neg_one: i32 = -one + assert one + one == 2 + assert one + two == 3 + assert one + neg_one == 0 + assert one - one == 0 + assert one - two == -1 + assert one * one == 1 + assert one * two == 2 + assert two * two == 4 + assert two * neg_one == -2 + assert neg_one * one == -1 + assert neg_one * neg_one == 1 + assert two * 0 == 0 + assert 0 * two == 0 + assert -one == -1 + assert -two == -2 + assert -neg_one == 1 + assert -zero == 0 + +def test_bitwise_ops() -> None: + x: i32 = 1920687484 + int() + y: i32 = 383354614 + int() + z: i32 = -1879040563 + int() + zero: i32 = int() + one: i32 = zero + 1 + two: i32 = zero + 2 + neg_one: i32 = -one + + assert x & y == 307823732 + assert x & z == 268442956 + assert z & z == z + assert x & zero == 0 + + assert x | y == 1996218366 + assert x | z == -226796035 + assert z | z == z + assert x | 0 == x + + assert x ^ y == 1688394634 + assert x ^ z == -495238991 + assert z ^ z == 0 + assert z ^ 0 == z + + assert x << one == -453592328 + assert x << two == -907184656 + assert z << two == 1073772340 + assert z << 0 == z + + assert x >> one == 960343742 + assert x >> two == 480171871 + assert z >> two == -469760141 + assert z >> 0 == z + + assert ~x == -1920687485 + assert ~z == 1879040562 + assert ~zero == -1 + assert ~neg_one == 0 + +def eq(x: i32, y: i32) -> bool: + return x == y + +def test_eq() -> None: + assert eq(int(), int()) + assert eq(5 + int(), 5 + int()) + assert eq(-5 + int(), -5 + int()) + assert not eq(int(), 1 + int()) + assert not eq(5 + int(), 6 + int()) + assert not eq(-5 + int(), -6 + int()) + assert not eq(-5 + int(), 5 + int()) + +def test_comparisons() -> None: + one: i32 = 1 + int() + one2: i32 = 1 + int() + two: i32 = 2 + int() + assert one < two + assert not (one < one2) + assert not (two < one) + assert two > one + assert not (one > one2) + assert not (one > two) + assert one <= two + assert one <= one2 + assert not (two <= one) + assert two >= one + assert one >= one2 + assert not (one >= two) + assert one == one2 + assert not (one == two) + assert one != two + assert not (one != one2) + +def test_mixed_comparisons() -> None: + i32_3: i32 = int() + 3 + int_5 = int() + 5 + assert i32_3 < int_5 + assert int_5 > i32_3 + b = i32_3 > int_5 + assert not b + + int_largest = int() + (1 << 31) - 1 + assert int_largest > i32_3 + int_smallest = int() - (1 << 31) + assert i32_3 > int_smallest + + int_too_big = int() + (1 << 31) + int_too_small = int() - (1 << 31) - 1 + with assertRaises(OverflowError): + assert i32_3 < int_too_big + with assertRaises(OverflowError): + assert int_too_big < i32_3 + with assertRaises(OverflowError): + assert i32_3 > int_too_small + with assertRaises(OverflowError): + assert int_too_small < i32_3 + +def test_mixed_arithmetic_and_bitwise_ops() -> None: + i32_3: i32 = int() + 3 + int_5 = int() + 5 + assert i32_3 + int_5 == 8 + assert int_5 - i32_3 == 2 + assert i32_3 << int_5 == 96 + assert int_5 << i32_3 == 40 + assert i32_3 ^ int_5 == 6 + assert int_5 | i32_3 == 7 + + int_largest = int() + (1 << 31) - 1 + assert int_largest - i32_3 == 2147483644 + int_smallest = int() - (1 << 31) + assert int_smallest + i32_3 == -2147483645 + + int_too_big = int() + (1 << 31) + int_too_small = int() - (1 << 31) - 1 + with assertRaises(OverflowError): + assert i32_3 & int_too_big + with assertRaises(OverflowError): + assert int_too_small & i32_3 + +def test_coerce_to_and_from_int() -> None: + for shift in range(0, 32): + for sign in 1, -1: + for delta in range(-5, 5): + n = sign * (1 << shift) + delta + if -(1 << 31) <= n < (1 << 31): + x: i32 = n + m: int = x + assert m == n + +def test_explicit_conversion_to_i32() -> None: + x = i32(5) + assert x == 5 + y = int() - 113 + x = i32(y) + assert x == -113 + n64: i64 = 1733 + x = i32(n64) + assert x == 1733 + n32: i32 = -1733 + x = i32(n32) + assert x == -1733 + z = i32(x) + assert z == -1733 + a: i16 = int() + 19764 + assert i32(a) == 19764 + a = int() - 1 + assert i32(a) == -1 + +def test_explicit_conversion_overflow() -> None: + max_i32 = int() + 2**31 - 1 + x = i32(max_i32) + assert x == 2**31 - 1 + assert int(x) == max_i32 + + min_i32 = int() - 2**31 + y = i32(min_i32) + assert y == -2**31 + assert int(y) == min_i32 + + too_big = int() + 2**31 + with assertRaises(OverflowError): + x = i32(too_big) + + too_small = int() - 2**31 - 1 + with assertRaises(OverflowError): + x = i32(too_small) + +def test_i32_from_large_small_literal() -> None: + x = i32(2**31 - 1) + assert x == 2**31 - 1 + x = i32(-2**31) + assert x == -2**31 + +def test_i32_truncate_from_i64() -> None: + large = i64(2**32 + 157 + int()) + x = i32(large) + assert x == 157 + small = i64(-2**32 - 157 + int()) + x = i32(small) + assert x == -157 + large2 = i64(2**31 + int()) + x = i32(large2) + assert x == -2**31 + small2 = i64(-2**31 - 1 - int()) + x = i32(small2) + assert x == 2**31 - 1 + +def from_float(x: float) -> i32: + return i32(x) + +def test_explicit_conversion_from_float() -> None: + assert from_float(0.0) == 0 + assert from_float(1.456) == 1 + assert from_float(-1234.567) == -1234 + assert from_float(2**31 - 1) == 2**31 - 1 + assert from_float(-2**31) == -2**31 + # The error message could be better, but this is acceptable + with assertRaises(OverflowError, "int too large to convert to i32"): + assert from_float(float(2**31)) + with assertRaises(OverflowError, "int too large to convert to i32"): + # One ulp below the lowest valid i64 value + from_float(float(-2**31 - 2048)) + +def test_tuple_i32() -> None: + a: i32 = 1 + b: i32 = 2 + t = (a, b) + a, b = t + assert a == 1 + assert b == 2 + x: Any = t + tt: Tuple[i32, i32] = x + assert tt == (1, 2) diff --git a/mypyc/test-data/run-i64.test b/mypyc/test-data/run-i64.test new file mode 100644 index 000000000000..36567c949d79 --- /dev/null +++ b/mypyc/test-data/run-i64.test @@ -0,0 +1,1519 @@ +[case testI64BasicOps] +from typing import List, Any, Tuple, Union + +from mypy_extensions import i64, i32, i16 + +from testutil import assertRaises + +def inc(n: i64) -> i64: + return n + 1 + +def test_inc() -> None: + # Use int() to avoid constant folding + n = 1 + int() + m = 2 + int() + assert inc(n) == m + +def min_ll(x: i64, y: i64) -> i64: + if x < y: + return x + else: + return y + +def test_min() -> None: + assert min_ll(1 + int(), 2) == 1 + assert min_ll(2 + int(), 1) == 1 + assert min_ll(1 + int(), 1) == 1 + assert min_ll(-2 + int(), 1) == -2 + assert min_ll(1 + int(), -2) == -2 + +def eq(x: i64, y: i64) -> bool: + return x == y + +def test_eq() -> None: + assert eq(int(), int()) + assert eq(5 + int(), 5 + int()) + assert eq(-5 + int(), -5 + int()) + assert not eq(int(), 1 + int()) + assert not eq(5 + int(), 6 + int()) + assert not eq(-5 + int(), -6 + int()) + assert not eq(-5 + int(), 5 + int()) + +def test_comparisons() -> None: + one: i64 = 1 + int() + one2: i64 = 1 + int() + two: i64 = 2 + int() + assert one < two + assert not (one < one2) + assert not (two < one) + assert two > one + assert not (one > one2) + assert not (one > two) + assert one <= two + assert one <= one2 + assert not (two <= one) + assert two >= one + assert one >= one2 + assert not (one >= two) + assert one == one2 + assert not (one == two) + assert one != two + assert not (one != one2) + +def is_true(x: i64) -> bool: + if x: + return True + else: + return False + +def is_true2(x: i64) -> bool: + return bool(x) + +def is_false(x: i64) -> bool: + if not x: + return True + else: + return False + +def test_i64_as_bool() -> None: + assert not is_true(0) + assert not is_true2(0) + assert is_false(0) + for x in 1, 55, -1, -7, 1 << 40, -(1 << 50): + assert is_true(x) + assert is_true2(x) + assert not is_false(x) + +def bool_as_i64(b: bool) -> i64: + return b + +def test_bool_as_i64() -> None: + assert bool_as_i64(False) == 0 + assert bool_as_i64(True) == 1 + +def div_by_3(x: i64) -> i64: + return x // 3 + +def div_by_neg_3(x: i64) -> i64: + return x // -3 + +def div(x: i64, y: i64) -> i64: + return x // y + +def test_divide_by_constant() -> None: + for i in range(-1000, 1000): + assert div_by_3(i) == i // 3 + for i in range(-2**63, -2**63 + 1000): + assert div_by_3(i) == i // 3 + for i in range(2**63 - 1000, 2**63): + assert div_by_3(i) == i // 3 + +def test_divide_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert div_by_neg_3(i) == i // -3 + for i in range(-2**63, -2**63 + 1000): + assert div_by_neg_3(i) == i // -3 + for i in range(2**63 - 1000, 2**63): + assert div_by_neg_3(i) == i // -3 + +def test_divide_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**63, -2**63 + 10)) + + list(range(2**63 - 10, 2**63))) + for x in values: + for y in values: + if y != 0: + if x // y == 2**63: + with assertRaises(OverflowError, "integer division overflow"): + div(x, y) + else: + assert div(x, y) == x // y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + div(x, y) + +def mod_by_7(x: i64) -> i64: + return x % 7 + +def mod_by_neg_7(x: i64) -> i64: + return x // -7 + +def mod(x: i64, y: i64) -> i64: + return x % y + +def test_mod_by_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_7(i) == i % 7 + for i in range(-2**63, -2**63 + 1000): + assert mod_by_7(i) == i % 7 + for i in range(2**63 - 1000, 2**63): + assert mod_by_7(i) == i % 7 + +def test_mod_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(-2**63, -2**63 + 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(2**63 - 1000, 2**63): + assert mod_by_neg_7(i) == i // -7 + +def test_mod_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**63, -2**63 + 10)) + + list(range(2**63 - 10, 2**63))) + for x in values: + for y in values: + if y != 0: + assert mod(x, y) == x % y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + mod(x, y) + +def get_item(a: List[i64], n: i64) -> i64: + return a[n] + +def test_get_list_item() -> None: + a = [1, 6, -2] + assert get_item(a, 0) == 1 + assert get_item(a, 1) == 6 + assert get_item(a, 2) == -2 + assert get_item(a, -1) == -2 + assert get_item(a, -2) == 6 + assert get_item(a, -3) == 1 + with assertRaises(IndexError, "list index out of range"): + get_item(a, 3) + with assertRaises(IndexError, "list index out of range"): + get_item(a, -4) + # TODO: Very large/small values and indexes + +def test_simple_arithmetic_ops() -> None: + zero: i64 = int() + one: i64 = zero + 1 + two: i64 = one + 1 + neg_one: i64 = -one + assert one + one == 2 + assert one + two == 3 + assert one + neg_one == 0 + assert one - one == 0 + assert one - two == -1 + assert one * one == 1 + assert one * two == 2 + assert two * two == 4 + assert two * neg_one == -2 + assert neg_one * one == -1 + assert neg_one * neg_one == 1 + assert two * 0 == 0 + assert 0 * two == 0 + assert -one == -1 + assert -two == -2 + assert -neg_one == 1 + assert -zero == 0 + +def test_bitwise_ops() -> None: + x: i64 = 7997307308812232241 + int() + y: i64 = 4333433528471475340 + int() + z: i64 = -2462230749488444526 + int() + zero: i64 = int() + one: i64 = zero + 1 + two: i64 = zero + 2 + neg_one: i64 = -one + + assert x & y == 3179577071592752128 + assert x & z == 5536089561888850448 + assert z & z == z + assert x & zero == 0 + + assert x | y == 9151163765690955453 + assert x | z == -1013002565062733 + assert z | z == z + assert x | 0 == x + + assert x ^ y == 5971586694098203325 + assert x ^ z == -5537102564453913181 + assert z ^ z == 0 + assert z ^ 0 == z + + assert x << one == -2452129456085087134 + assert x << two == -4904258912170174268 + assert z << two == 8597821075755773512 + assert z << 0 == z + + assert x >> one == 3998653654406116120 + assert x >> two == 1999326827203058060 + assert z >> two == -615557687372111132 + assert z >> 0 == z + + assert ~x == -7997307308812232242 + assert ~z == 2462230749488444525 + assert ~zero == -1 + assert ~neg_one == 0 + +def test_coerce_to_and_from_int() -> None: + for shift in range(0, 64): + for sign in 1, -1: + for delta in range(-5, 5): + n = sign * (1 << shift) + delta + if -(1 << 63) <= n < (1 << 63): + x: i64 = n + m: int = x + assert m == n + +def test_coerce_to_and_from_int2() -> None: + for shift in range(0, 64): + for sign in 1, -1: + for delta in range(-5, 5): + n = sign * (1 << shift) + delta + if -(1 << 63) <= n < (1 << 63): + x: i64 = i64(n) + m: int = int(x) + assert m == n + +def test_explicit_conversion_to_i64() -> None: + x = i64(5) + assert x == 5 + y = int() - 113 + x = i64(y) + assert x == -113 + n32: i32 = 1733 + x = i64(n32) + assert x == 1733 + n32 = -1733 + x = i64(n32) + assert x == -1733 + z = i64(x) + assert z == -1733 + a: i16 = int() + 19764 + assert i64(a) == 19764 + a = int() - 1 + assert i64(a) == -1 + +def test_explicit_conversion_overflow() -> None: + max_i64 = int() + 2**63 - 1 + x = i64(max_i64) + assert x == 2**63 - 1 + assert int(x) == max_i64 + + min_i64 = int() - 2**63 + y = i64(min_i64) + assert y == -2**63 + assert int(y) == min_i64 + + too_big = int() + 2**63 + with assertRaises(OverflowError): + x = i64(too_big) + + too_small = int() - 2**63 - 1 + with assertRaises(OverflowError): + x = i64(too_small) + +def test_i64_from_large_small_literal() -> None: + x = i64(2**63 - 1) + assert x == 2**63 - 1 + x = i64(-2**63) + assert x == -2**63 + +def from_float(x: float) -> i64: + return i64(x) + +def test_explicit_conversion_from_float() -> None: + assert from_float(0.0) == 0 + assert from_float(1.456) == 1 + assert from_float(-1234.567) == -1234 + # Subtract 1024 due to limited precision of 64-bit floats + assert from_float(2**63 - 1024) == 2**63 - 1024 + assert from_float(-2**63) == -2**63 + # The error message could be better, but this is acceptable + with assertRaises(OverflowError, "int too large to convert to i64"): + assert from_float(float(2**63)) + with assertRaises(OverflowError, "int too large to convert to i64"): + # One ulp below the lowest valid i64 value + from_float(float(-2**63 - 2048)) + +def from_str(s: str) -> i64: + return i64(s) + +def test_explicit_conversion_from_str() -> None: + assert from_str("0") == 0 + assert from_str("1") == 1 + assert from_str("-1234") == -1234 + with assertRaises(ValueError): + from_str("1.2") + +def from_str_with_base(s: str, base: int) -> i64: + return i64(s, base) + +def test_explicit_conversion_from_str_with_base() -> None: + assert from_str_with_base("101", 2) == 5 + assert from_str_with_base("109", 10) == 109 + assert from_str_with_base("-f0A", 16) == -3850 + assert from_str_with_base("0x1a", 16) == 26 + assert from_str_with_base("0X1A", 16) == 26 + with assertRaises(ValueError): + from_str_with_base("1.2", 16) + +def from_bool(b: bool) -> i64: + return i64(b) + +def test_explicit_conversion_from_bool() -> None: + assert from_bool(True) == 1 + assert from_bool(False) == 0 + +class IntConv: + def __init__(self, x: i64) -> None: + self.x = x + + def __int__(self) -> i64: + return self.x + 1 + +def test_explicit_conversion_from_instance() -> None: + assert i64(IntConv(0)) == 1 + assert i64(IntConv(12345)) == 12346 + assert i64(IntConv(-23)) == -22 + +def test_explicit_conversion_from_any() -> None: + # This can't be specialized + a: Any = "101" + assert i64(a, base=2) == 5 + +def test_tuple_i64() -> None: + a: i64 = 1 + b: i64 = 2 + t = (a, b) + a, b = t + assert a == 1 + assert b == 2 + x: Any = t + tt: Tuple[i64, i64] = x + assert tt == (1, 2) + +def test_list_set_item() -> None: + a: List[i64] = [0, 2, 6] + z: i64 = int() + a[z] = 1 + assert a == [1, 2, 6] + a[z + 2] = 9 + assert a == [1, 2, 9] + a[-(z + 1)] = 10 + assert a == [1, 2, 10] + a[-(z + 3)] = 3 + assert a == [3, 2, 10] + with assertRaises(IndexError): + a[z + 3] = 0 + with assertRaises(IndexError): + a[-(z + 4)] = 0 + assert a == [3, 2, 10] + +class C: + def __init__(self, x: i64) -> None: + self.x = x + +def test_attributes() -> None: + i: i64 + for i in range(-1000, 1000): + c = C(i) + assert c.x == i + c.x = i + 1 + assert c.x == i + 1 + +def test_mixed_comparisons() -> None: + i64_3: i64 = int() + 3 + int_5 = int() + 5 + assert i64_3 < int_5 + assert int_5 > i64_3 + b = i64_3 > int_5 + assert not b + + int_largest = int() + (1 << 63) - 1 + assert int_largest > i64_3 + int_smallest = int() - (1 << 63) + assert i64_3 > int_smallest + + int_too_big = int() + (1 << 63) + int_too_small = int() - (1 << 63) - 1 + with assertRaises(OverflowError): + assert i64_3 < int_too_big + with assertRaises(OverflowError): + assert int_too_big < i64_3 + with assertRaises(OverflowError): + assert i64_3 > int_too_small + with assertRaises(OverflowError): + assert int_too_small < i64_3 + +def test_mixed_comparisons_32bit() -> None: + # Test edge cases on 32-bit platforms + i64_3: i64 = int() + 3 + int_5 = int() + 5 + + int_largest_short = int() + (1 << 30) - 1 + int_largest_short_i64: i64 = int_largest_short + assert int_largest_short > i64_3 + int_smallest_short = int() - (1 << 30) + int_smallest_short_i64: i64 = int_smallest_short + assert i64_3 > int_smallest_short + + int_big = int() + (1 << 30) + assert int_big > i64_3 + int_small = int() - (1 << 30) - 1 + assert i64_3 > int_small + + assert int_smallest_short_i64 > int_small + assert int_largest_short_i64 < int_big + +def test_mixed_arithmetic_and_bitwise_ops() -> None: + i64_3: i64 = int() + 3 + int_5 = int() + 5 + assert i64_3 + int_5 == 8 + assert int_5 - i64_3 == 2 + assert i64_3 << int_5 == 96 + assert int_5 << i64_3 == 40 + assert i64_3 ^ int_5 == 6 + assert int_5 | i64_3 == 7 + + int_largest = int() + (1 << 63) - 1 + assert int_largest - i64_3 == 9223372036854775804 + int_smallest = int() - (1 << 63) + assert int_smallest + i64_3 == -9223372036854775805 + + int_too_big = int() + (1 << 63) + int_too_small = int() - (1 << 63) - 1 + with assertRaises(OverflowError): + assert i64_3 & int_too_big + with assertRaises(OverflowError): + assert int_too_small & i64_3 + +def test_for_loop() -> None: + n: i64 = 0 + for i in range(i64(5 + int())): + n += i + assert n == 10 + n = 0 + for i in range(i64(5)): + n += i + assert n == 10 + n = 0 + for i in range(i64(2 + int()), 5 + int()): + n += i + assert n == 9 + n = 0 + for i in range(2, i64(5 + int())): + n += i + assert n == 9 + assert sum([x * x for x in range(i64(4 + int()))]) == 1 + 4 + 9 + +def narrow1(x: Union[str, i64]) -> i64: + if isinstance(x, i64): + return x + return len(x) + +def narrow2(x: Union[str, i64]) -> i64: + if isinstance(x, int): + return x + return len(x) + +def test_isinstance() -> None: + assert narrow1(123) == 123 + assert narrow1("foobar") == 6 + assert narrow2(123) == 123 + assert narrow2("foobar") == 6 + +[case testI64ErrorValuesAndUndefined] +from typing import Any, Tuple +import sys + +from mypy_extensions import mypyc_attr, i64 +from typing_extensions import Final + +from testutil import assertRaises + +def maybe_raise(n: i64, error: bool) -> i64: + if error: + raise ValueError() + return n + +def test_error_value() -> None: + for i in range(-1000, 1000): + assert maybe_raise(i, False) == i + with assertRaises(ValueError): + maybe_raise(0, True) + +class C: + def maybe_raise(self, n: i64, error: bool) -> i64: + if error: + raise ValueError() + return n + +def test_method_error_value() -> None: + for i in range(-1000, 1000): + assert C().maybe_raise(i, False) == i + with assertRaises(ValueError): + C().maybe_raise(0, True) + +def maybe_raise_tuple(n: i64, error: bool) -> Tuple[i64, i64]: + if error: + raise ValueError() + return n, n+ 1 + +def test_tuple_error_value() -> None: + for i in range(-1000, 1000): + assert maybe_raise_tuple(i, False) == (i, i + 1) + with assertRaises(ValueError): + maybe_raise_tuple(0, True) + f: Any = maybe_raise_tuple + for i in range(-1000, 1000): + assert f(i, False) == (i, i + 1) + with assertRaises(ValueError): + f(0, True) + +def maybe_raise_tuple2(n: i64, error: bool) -> Tuple[i64, int]: + if error: + raise ValueError() + return n, n+ 1 + +def test_tuple_error_value_2() -> None: + for i in range(-1000, 1000): + assert maybe_raise_tuple2(i, False) == (i, i + 1) + with assertRaises(ValueError): + maybe_raise_tuple(0, True) + +def test_unbox_int() -> None: + for i in list(range(-1000, 1000)) + [-(1 << 63), (1 << 63) - 1]: + o: Any = i + x: i64 = i + assert x == i + y: i64 = o + assert y == i + +def test_unbox_int_fails() -> None: + o: Any = 'x' + if sys.version_info[0] == 3 and sys.version_info[1] < 10: + msg = "an integer is required (got type str)" + else: + msg = "'str' object cannot be interpreted as an integer" + with assertRaises(TypeError, msg): + x: i64 = o + o2: Any = 1 << 63 + with assertRaises(OverflowError, "int too large to convert to i64"): + y: i64 = o2 + o3: Any = -(1 << 63 + 1) + with assertRaises(OverflowError, "int too large to convert to i64"): + z: i64 = o3 + +class Uninit: + x: i64 + y: i64 = 0 + z: i64 + +class Derived(Uninit): + a: i64 = 1 + b: i64 + c: i64 = 2 + +class Derived2(Derived): + h: i64 + +def test_uninitialized_attr() -> None: + o = Uninit() + assert o.y == 0 + with assertRaises(AttributeError): + o.x + with assertRaises(AttributeError): + o.z + o.x = 1 + assert o.x == 1 + with assertRaises(AttributeError): + o.z + o.z = 2 + assert o.z == 2 + +# This is the error value, but it's also a valid normal value +MAGIC: Final = -113 + +def test_magic_value() -> None: + o = Uninit() + o.x = MAGIC + assert o.x == MAGIC + with assertRaises(AttributeError): + o.z + o.z = MAGIC + assert o.x == MAGIC + assert o.z == MAGIC + +def test_magic_value_via_any() -> None: + o: Any = Uninit() + with assertRaises(AttributeError): + o.x + with assertRaises(AttributeError): + o.z + o.x = MAGIC + assert o.x == MAGIC + with assertRaises(AttributeError): + o.z + o.z = MAGIC + assert o.z == MAGIC + +def test_magic_value_and_inheritance() -> None: + o = Derived2() + o.x = MAGIC + assert o.x == MAGIC + with assertRaises(AttributeError): + o.z + with assertRaises(AttributeError): + o.b + with assertRaises(AttributeError): + o.h + o.z = MAGIC + assert o.z == MAGIC + with assertRaises(AttributeError): + o.b + with assertRaises(AttributeError): + o.h + o.h = MAGIC + assert o.h == MAGIC + with assertRaises(AttributeError): + o.b + o.b = MAGIC + assert o.b == MAGIC + +@mypyc_attr(allow_interpreted_subclasses=True) +class MagicInit: + x: i64 = MAGIC + +def test_magic_value_as_initializer() -> None: + o = MagicInit() + assert o.x == MAGIC + +class ManyUninit: + a1: i64 + a2: i64 + a3: i64 + a4: i64 + a5: i64 + a6: i64 + a7: i64 + a8: i64 + a9: i64 + a10: i64 + a11: i64 + a12: i64 + a13: i64 + a14: i64 + a15: i64 + a16: i64 + a17: i64 + a18: i64 + a19: i64 + a20: i64 + a21: i64 + a22: i64 + a23: i64 + a24: i64 + a25: i64 + a26: i64 + a27: i64 + a28: i64 + a29: i64 + a30: i64 + a31: i64 + a32: i64 + a33: i64 + a34: i64 + a35: i64 + a36: i64 + a37: i64 + a38: i64 + a39: i64 + a40: i64 + a41: i64 + a42: i64 + a43: i64 + a44: i64 + a45: i64 + a46: i64 + a47: i64 + a48: i64 + a49: i64 + a50: i64 + a51: i64 + a52: i64 + a53: i64 + a54: i64 + a55: i64 + a56: i64 + a57: i64 + a58: i64 + a59: i64 + a60: i64 + a61: i64 + a62: i64 + a63: i64 + a64: i64 + a65: i64 + a66: i64 + a67: i64 + a68: i64 + a69: i64 + a70: i64 + a71: i64 + a72: i64 + a73: i64 + a74: i64 + a75: i64 + a76: i64 + a77: i64 + a78: i64 + a79: i64 + a80: i64 + a81: i64 + a82: i64 + a83: i64 + a84: i64 + a85: i64 + a86: i64 + a87: i64 + a88: i64 + a89: i64 + a90: i64 + a91: i64 + a92: i64 + a93: i64 + a94: i64 + a95: i64 + a96: i64 + a97: i64 + a98: i64 + a99: i64 + a100: i64 + +def test_many_uninitialized_attributes() -> None: + o = ManyUninit() + with assertRaises(AttributeError): + o.a1 + with assertRaises(AttributeError): + o.a10 + with assertRaises(AttributeError): + o.a20 + with assertRaises(AttributeError): + o.a30 + with assertRaises(AttributeError): + o.a31 + with assertRaises(AttributeError): + o.a32 + with assertRaises(AttributeError): + o.a33 + with assertRaises(AttributeError): + o.a40 + with assertRaises(AttributeError): + o.a50 + with assertRaises(AttributeError): + o.a60 + with assertRaises(AttributeError): + o.a62 + with assertRaises(AttributeError): + o.a63 + with assertRaises(AttributeError): + o.a64 + with assertRaises(AttributeError): + o.a65 + with assertRaises(AttributeError): + o.a80 + with assertRaises(AttributeError): + o.a100 + o.a30 = MAGIC + assert o.a30 == MAGIC + o.a31 = MAGIC + assert o.a31 == MAGIC + o.a32 = MAGIC + assert o.a32 == MAGIC + o.a33 = MAGIC + assert o.a33 == MAGIC + with assertRaises(AttributeError): + o.a34 + o.a62 = MAGIC + assert o.a62 == MAGIC + o.a63 = MAGIC + assert o.a63 == MAGIC + o.a64 = MAGIC + assert o.a64 == MAGIC + o.a65 = MAGIC + assert o.a65 == MAGIC + with assertRaises(AttributeError): + o.a66 + +class BaseNoBitmap: + x: int = 5 + +class DerivedBitmap(BaseNoBitmap): + # Subclass needs a bitmap, but base class doesn't have it. + y: i64 + +def test_derived_adds_bitmap() -> None: + d = DerivedBitmap() + d.x = 643 + b: BaseNoBitmap = d + assert b.x == 643 + +class Delete: + __deletable__ = ['x', 'y'] + x: i64 + y: i64 + +def test_del() -> None: + o = Delete() + o.x = MAGIC + o.y = -1 + assert o.x == MAGIC + assert o.y == -1 + del o.x + with assertRaises(AttributeError): + o.x + assert o.y == -1 + del o.y + with assertRaises(AttributeError): + o.y + o.x = 5 + assert o.x == 5 + with assertRaises(AttributeError): + o.y + del o.x + with assertRaises(AttributeError): + o.x + +class UndefinedTuple: + def __init__(self, x: i64, y: i64) -> None: + if x != 0: + self.t = (x, y) + +def test_undefined_native_int_tuple() -> None: + o = UndefinedTuple(MAGIC, MAGIC) + assert o.t[0] == MAGIC + assert o.t[1] == MAGIC + o = UndefinedTuple(0, 0) + with assertRaises(AttributeError): + o.t + o = UndefinedTuple(-13, 45) + assert o.t == (-13, 45) + +def test_undefined_native_int_tuple_via_any() -> None: + cls: Any = UndefinedTuple + o: Any = cls(MAGIC, MAGIC) + assert o.t[0] == MAGIC + assert o.t[1] == MAGIC + o = cls(0, 0) + with assertRaises(AttributeError): + o.t + o = UndefinedTuple(-13, 45) + assert o.t == (-13, 45) + +[case testI64DefaultArgValues] +from typing import Any, Iterator, Tuple +from typing_extensions import Final + +MAGIC: Final = -113 + +from mypy_extensions import i64 + +def f(x: i64, y: i64 = 5) -> i64: + return x + y + +def test_simple_default_arg() -> None: + assert f(3) == 8 + assert f(4, 9) == 13 + assert f(5, MAGIC) == -108 + for i in range(-1000, 1000): + assert f(1, i) == 1 + i + f2: Any = f + assert f2(3) == 8 + assert f2(4, 9) == 13 + assert f2(5, MAGIC) == -108 + +def g(a: i64, b: i64 = 1, c: int = 2, d: i64 = 3) -> i64: + return a + b + c + d + +def test_two_default_args() -> None: + assert g(10) == 16 + assert g(10, 2) == 17 + assert g(10, 2, 3) == 18 + assert g(10, 2, 3, 4) == 19 + g2: Any = g + assert g2(10) == 16 + assert g2(10, 2) == 17 + assert g2(10, 2, 3) == 18 + assert g2(10, 2, 3, 4) == 19 + +class C: + def __init__(self) -> None: + self.i: i64 = 1 + + def m(self, a: i64, b: i64 = 1, c: int = 2, d: i64 = 3) -> i64: + return self.i + a + b + c + d + +class D(C): + def m(self, a: i64, b: i64 = 2, c: int = 3, d: i64 = 4) -> i64: + return self.i + a + b + c + d + + def mm(self, a: i64 = 2, b: i64 = 1) -> i64: + return self.i + a + b + + @staticmethod + def s(a: i64 = 2, b: i64 = 1) -> i64: + return a + b + + @classmethod + def c(cls, a: i64 = 2, b: i64 = 3) -> i64: + assert cls is D + return a + b + +def test_method_default_args() -> None: + a = [C(), D()] + assert a[0].m(4) == 11 + d = D() + assert d.mm() == 4 + assert d.mm(5) == 7 + assert d.mm(MAGIC) == MAGIC + 2 + assert d.mm(b=5) == 8 + assert D.mm(d) == 4 + assert D.mm(d, 6) == 8 + assert D.mm(d, MAGIC) == MAGIC + 2 + assert D.mm(d, b=6) == 9 + dd: Any = d + assert dd.mm() == 4 + assert dd.mm(5) == 7 + assert dd.mm(MAGIC) == MAGIC + 2 + assert dd.mm(b=5) == 8 + +def test_static_method_default_args() -> None: + d = D() + assert d.s() == 3 + assert d.s(5) == 6 + assert d.s(MAGIC) == MAGIC + 1 + assert d.s(5, 6) == 11 + assert D.s() == 3 + assert D.s(5) == 6 + assert D.s(MAGIC) == MAGIC + 1 + assert D.s(5, 6) == 11 + dd: Any = d + assert dd.s() == 3 + assert dd.s(5) == 6 + assert dd.s(MAGIC) == MAGIC + 1 + assert dd.s(5, 6) == 11 + +def test_class_method_default_args() -> None: + d = D() + assert d.c() == 5 + assert d.c(5) == 8 + assert d.c(MAGIC) == MAGIC + 3 + assert d.c(b=5) == 7 + assert D.c() == 5 + assert D.c(5) == 8 + assert D.c(MAGIC) == MAGIC + 3 + assert D.c(b=5) == 7 + dd: Any = d + assert dd.c() == 5 + assert dd.c(5) == 8 + assert dd.c(MAGIC) == MAGIC + 3 + assert dd.c(b=5) == 7 + +class Init: + def __init__(self, x: i64 = 2, y: i64 = 5) -> None: + self.x = x + self.y = y + +def test_init_default_args() -> None: + o = Init() + assert o.x == 2 + assert o.y == 5 + o = Init(7, 8) + assert o.x == 7 + assert o.y == 8 + o = Init(4) + assert o.x == 4 + assert o.y == 5 + o = Init(MAGIC, MAGIC) + assert o.x == MAGIC + assert o.y == MAGIC + o = Init(3, MAGIC) + assert o.x == 3 + assert o.y == MAGIC + o = Init(MAGIC, 11) + assert o.x == MAGIC + assert o.y == 11 + o = Init(MAGIC) + assert o.x == MAGIC + assert o.y == 5 + o = Init(y=MAGIC) + assert o.x == 2 + assert o.y == MAGIC + +def kw_only(*, a: i64 = 1, b: int = 2, c: i64 = 3) -> i64: + return a + b + c * 2 + +def test_kw_only_default_args() -> None: + assert kw_only() == 9 + assert kw_only(a=2) == 10 + assert kw_only(b=4) == 11 + assert kw_only(c=11) == 25 + assert kw_only(a=2, c=4) == 12 + assert kw_only(c=4, a=2) == 12 + kw_only2: Any = kw_only + assert kw_only2() == 9 + assert kw_only2(a=2) == 10 + assert kw_only2(b=4) == 11 + assert kw_only2(c=11) == 25 + assert kw_only2(a=2, c=4) == 12 + assert kw_only2(c=4, a=2) == 12 + +def tuples(t: Tuple[i64, i64] = (MAGIC, MAGIC)) -> i64: + return t[0] + t[1] + +def test_tuple_arg_defaults() -> None: + assert tuples() == 2 * MAGIC + assert tuples((1, 2)) == 3 + assert tuples((MAGIC, MAGIC)) == 2 * MAGIC + tuples2: Any = tuples + assert tuples2() == 2 * MAGIC + assert tuples2((1, 2)) == 3 + assert tuples2((MAGIC, MAGIC)) == 2 * MAGIC + +class TupleInit: + def __init__(self, t: Tuple[i64, i64] = (MAGIC, MAGIC)) -> None: + self.t = t[0] + t[1] + +def test_tuple_init_arg_defaults() -> None: + assert TupleInit().t == 2 * MAGIC + assert TupleInit((1, 2)).t == 3 + assert TupleInit((MAGIC, MAGIC)).t == 2 * MAGIC + o: Any = TupleInit + assert o().t == 2 * MAGIC + assert o((1, 2)).t == 3 + assert o((MAGIC, MAGIC)).t == 2 * MAGIC + +def many_args( + a1: i64 = 0, + a2: i64 = 1, + a3: i64 = 2, + a4: i64 = 3, + a5: i64 = 4, + a6: i64 = 5, + a7: i64 = 6, + a8: i64 = 7, + a9: i64 = 8, + a10: i64 = 9, + a11: i64 = 10, + a12: i64 = 11, + a13: i64 = 12, + a14: i64 = 13, + a15: i64 = 14, + a16: i64 = 15, + a17: i64 = 16, + a18: i64 = 17, + a19: i64 = 18, + a20: i64 = 19, + a21: i64 = 20, + a22: i64 = 21, + a23: i64 = 22, + a24: i64 = 23, + a25: i64 = 24, + a26: i64 = 25, + a27: i64 = 26, + a28: i64 = 27, + a29: i64 = 28, + a30: i64 = 29, + a31: i64 = 30, + a32: i64 = 31, + a33: i64 = 32, + a34: i64 = 33, +) -> i64: + return a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8 + a9 + a10 + a11 + a12 + a13 + a14 + a15 + a16 + a17 + a18 + a19 + a20 + a21 + a22 + a23 + a24 + a25 + a26 + a27 + a28 + a29 + a30 + a31 + a32 + a33 + a34 + +def test_many_args() -> None: + assert many_args() == 561 + assert many_args(a1=100) == 661 + assert many_args(a2=101) == 661 + assert many_args(a15=114) == 661 + assert many_args(a31=130) == 661 + assert many_args(a32=131) == 661 + assert many_args(a33=232) == 761 + assert many_args(a34=333) == 861 + assert many_args(a1=100, a33=232) == 861 + f: Any = many_args + assert f() == 561 + assert f(a1=100) == 661 + assert f(a2=101) == 661 + assert f(a15=114) == 661 + assert f(a31=130) == 661 + assert f(a32=131) == 661 + assert f(a33=232) == 761 + assert f(a34=333) == 861 + assert f(a1=100, a33=232) == 861 + +def test_nested_function_defaults() -> None: + a: i64 = 1 + + def nested(x: i64 = 2, y: i64 = 3) -> i64: + return a + x + y + + assert nested() == 6 + assert nested(3) == 7 + assert nested(y=5) == 8 + assert nested(MAGIC) == MAGIC + 4 + a = 11 + assert nested() == 16 + + +def test_nested_function_defaults_via_any() -> None: + a: i64 = 1 + + def nested_native(x: i64 = 2, y: i64 = 3) -> i64: + return a + x + y + + nested: Any = nested_native + + assert nested() == 6 + assert nested(3) == 7 + assert nested(y=5) == 8 + assert nested(MAGIC) == MAGIC + 4 + a = 11 + assert nested() == 16 + +def gen(x: i64 = 1, y: i64 = 2) -> Iterator[i64]: + yield x + y + +def test_generator() -> None: + g = gen() + assert next(g) == 3 + g = gen(2) + assert next(g) == 4 + g = gen(2, 3) + assert next(g) == 5 + a: Any = gen + g = a() + assert next(g) == 3 + g = a(2) + assert next(g) == 4 + g = a(2, 3) + assert next(g) == 5 + +def magic_default(x: i64 = MAGIC) -> i64: + return x + +def test_magic_default() -> None: + assert magic_default() == MAGIC + assert magic_default(1) == 1 + assert magic_default(MAGIC) == MAGIC + a: Any = magic_default + assert a() == MAGIC + assert a(1) == 1 + assert a(MAGIC) == MAGIC + +[case testI64UndefinedLocal] +from typing_extensions import Final + +from mypy_extensions import i64, i32 + +from testutil import assertRaises + +MAGIC: Final = -113 + + +def test_conditionally_defined_local() -> None: + x = not int() + if x: + y: i64 = 5 + z: i32 = 6 + assert y == 5 + assert z == 6 + +def test_conditionally_undefined_local() -> None: + x = int() + if x: + y: i64 = 5 + z: i32 = 6 + else: + ok: i64 = 7 + assert ok == 7 + try: + print(y) + except NameError as e: + assert str(e) == 'local variable "y" referenced before assignment' + else: + assert False + try: + print(z) + except NameError as e: + assert str(e) == 'local variable "z" referenced before assignment' + else: + assert False + +def test_assign_error_value_conditionally() -> None: + x = int() + if not x: + y: i64 = MAGIC + z: i32 = MAGIC + assert y == MAGIC + assert z == MAGIC + +def tuple_case(x: i64, y: i64) -> None: + if not int(): + t = (x, y) + assert t == (x, y) + if int(): + t2 = (x, y) + try: + print(t2) + except NameError as e: + assert str(e) == 'local variable "t2" referenced before assignment' + else: + assert False + +def test_conditionally_undefined_tuple() -> None: + tuple_case(2, 3) + tuple_case(-2, -3) + tuple_case(MAGIC, MAGIC) + +def test_many_locals() -> None: + x = int() + if x: + a0: i64 = 0 + a1: i64 = 1 + a2: i64 = 2 + a3: i64 = 3 + a4: i64 = 4 + a5: i64 = 5 + a6: i64 = 6 + a7: i64 = 7 + a8: i64 = 8 + a9: i64 = 9 + a10: i64 = 10 + a11: i64 = 11 + a12: i64 = 12 + a13: i64 = 13 + a14: i64 = 14 + a15: i64 = 15 + a16: i64 = 16 + a17: i64 = 17 + a18: i64 = 18 + a19: i64 = 19 + a20: i64 = 20 + a21: i64 = 21 + a22: i64 = 22 + a23: i64 = 23 + a24: i64 = 24 + a25: i64 = 25 + a26: i64 = 26 + a27: i64 = 27 + a28: i64 = 28 + a29: i64 = 29 + a30: i64 = 30 + a31: i64 = 31 + a32: i64 = 32 + a33: i64 = 33 + with assertRaises(UnboundLocalError): + print(a0) + with assertRaises(UnboundLocalError): + print(a31) + with assertRaises(UnboundLocalError): + print(a32) + with assertRaises(UnboundLocalError): + print(a33) + a0 = 5 + assert a0 == 5 + with assertRaises(UnboundLocalError): + print(a31) + with assertRaises(UnboundLocalError): + print(a32) + with assertRaises(UnboundLocalError): + print(a33) + a32 = 55 + assert a0 == 5 + assert a32 == 55 + with assertRaises(UnboundLocalError): + print(a31) + with assertRaises(UnboundLocalError): + print(a33) + a31 = 10 + a33 = 20 + assert a0 == 5 + assert a31 == 10 + assert a32 == 55 + assert a33 == 20 + +[case testI64GlueMethodsAndInheritance] +from typing import Any +from typing_extensions import Final + +from mypy_extensions import i64, trait + +from testutil import assertRaises + +MAGIC: Final = -113 + +class Base: + def foo(self) -> i64: + return 5 + + def bar(self, x: i64 = 2) -> i64: + return x + 1 + + def hoho(self, x: i64) -> i64: + return x - 1 + +class Derived(Base): + def foo(self, x: i64 = 5) -> i64: + return x + 10 + + def bar(self, x: i64 = 3, y: i64 = 20) -> i64: + return x + y + 2 + + def hoho(self, x: i64 = 7) -> i64: + return x - 2 + +def test_derived_adds_bitmap() -> None: + b: Base = Derived() + assert b.foo() == 15 + +def test_derived_adds_another_default_arg() -> None: + b: Base = Derived() + assert b.bar() == 25 + assert b.bar(1) == 23 + assert b.bar(MAGIC) == MAGIC + 22 + +def test_derived_switches_arg_to_have_default() -> None: + b: Base = Derived() + assert b.hoho(5) == 3 + assert b.hoho(MAGIC) == MAGIC - 2 + +@trait +class T: + @property + def x(self) -> i64: ... + @property + def y(self) -> i64: ... + +class C(T): + x: i64 = 1 + y: i64 = 4 + +def test_read_only_property_in_trait_implemented_as_attribute() -> None: + c = C() + c.x = 5 + assert c.x == 5 + c.x = MAGIC + assert c.x == MAGIC + assert c.y == 4 + c.y = 6 + assert c.y == 6 + t: T = C() + assert t.y == 4 + t = c + assert t.x == MAGIC + c.x = 55 + assert t.x == 55 + assert t.y == 6 + a: Any = c + assert a.x == 55 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +class D(T): + xx: i64 + + @property + def x(self) -> i64: + return self.xx + + @property + def y(self) -> i64: + raise TypeError + +def test_read_only_property_in_trait_implemented_as_property() -> None: + d = D() + d.xx = 5 + assert d.x == 5 + d.xx = MAGIC + assert d.x == MAGIC + with assertRaises(TypeError): + d.y + t: T = d + assert t.x == MAGIC + d.xx = 6 + assert t.x == 6 + with assertRaises(TypeError): + t.y + +@trait +class T2: + x: i64 + y: i64 + +class C2(T2): + pass + +def test_inherit_trait_attribute() -> None: + c = C2() + c.x = 5 + assert c.x == 5 + c.x = MAGIC + assert c.x == MAGIC + with assertRaises(AttributeError): + c.y + c.y = 6 + assert c.y == 6 + t: T2 = C2() + with assertRaises(AttributeError): + t.y + t = c + assert t.x == MAGIC + c.x = 55 + assert t.x == 55 + assert t.y == 6 + a: Any = c + assert a.x == 55 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +class D2(T2): + x: i64 + y: i64 = 4 + +def test_implement_trait_attribute() -> None: + d = D2() + d.x = 5 + assert d.x == 5 + d.x = MAGIC + assert d.x == MAGIC + assert d.y == 4 + d.y = 6 + assert d.y == 6 + t: T2 = D2() + assert t.y == 4 + t = d + assert t.x == MAGIC + d.x = 55 + assert t.x == 55 + assert t.y == 6 + a: Any = d + assert a.x == 55 + assert a.y == 6 + a.x = 7 + a.y = 8 + assert a.x == 7 + assert a.y == 8 + +class DunderErr: + def __contains__(self, i: i64) -> bool: + raise IndexError() + +def test_dunder_arg_check() -> None: + o: Any = DunderErr() + with assertRaises(TypeError): + 'x' in o + with assertRaises(TypeError): + 2**63 in o + with assertRaises(IndexError): + 1 in o diff --git a/mypyc/test-data/run-imports.test b/mypyc/test-data/run-imports.test new file mode 100644 index 000000000000..c5839d57820e --- /dev/null +++ b/mypyc/test-data/run-imports.test @@ -0,0 +1,265 @@ +# Test cases for imports and related features (compile and run) + +[case testImports] +import testmodule +import pkg2.mod +import pkg2.mod2 as mm2 + +def f(x: int) -> int: + return testmodule.factorial(5) + +def g(x: int) -> int: + from welp import foo + return foo(x) + +def test_import_basics() -> None: + assert f(5) == 120 + assert g(5) == 5 + assert "pkg2.mod" not in globals(), "the root module should be in globals!" + assert pkg2.mod.x == 1 + assert "mod2" not in globals(), "pkg2.mod2 is aliased to mm2!" + assert mm2.y == 2 + +def test_import_submodule_within_function() -> None: + import pkg.mod + assert pkg.x == 1 + assert pkg.mod.y == 2 + assert "pkg.mod" not in globals(), "the root module should be in globals!" + +def test_import_as_submodule_within_function() -> None: + import pkg.mod as mm + assert mm.y == 2 + assert "pkg.mod" not in globals(), "the root module should be in globals!" + +# TODO: Don't add local imports to globals() +# +# def test_local_import_not_in_globals() -> None: +# import nob +# assert 'nob' not in globals() + +def test_import_module_without_stub_in_function() -> None: + # 'psutil' must not have a stub in typeshed for this test case + import psutil # type: ignore + # TODO: We shouldn't add local imports to globals() + # assert 'psutil' not in globals() + assert isinstance(psutil.__name__, str) + +def test_import_as_module_without_stub_in_function() -> None: + # 'psutil' must not have a stub in typeshed for this test case + import psutil as pp # type: ignore + assert 'psutil' not in globals() + # TODO: We shouldn't add local imports to globals() + # assert 'pp' not in globals() + assert isinstance(pp.__name__, str) + +[file testmodule.py] +def factorial(x: int) -> int: + if x == 0: + return 1 + else: + return x * factorial(x-1) +[file welp.py] +def foo(x: int) -> int: + return x +[file pkg/__init__.py] +x = 1 +[file pkg/mod.py] +y = 2 +[file pkg2/__init__.py] +[file pkg2/mod.py] +x = 1 +[file pkg2/mod2.py] +y = 2 +[file nob.py] +z = 3 + +[case testImportMissing] +# The unchecked module is configured by the test harness to not be +# picked up by mypy, so we can test that we do that right thing when +# calling library modules without stubs. +import unchecked # type: ignore +import unchecked as lol # type: ignore +assert unchecked.x == 10 +assert lol.x == 10 +[file unchecked.py] +x = 10 + +[file driver.py] +import native + +[case testFromImport] +from testmodule import g + +def f(x: int) -> int: + return g(x) +[file testmodule.py] +def g(x: int) -> int: + return x + 1 +[file driver.py] +from native import f +assert f(1) == 2 + +[case testFromImportWithUntypedModule] + +# avoid including an __init__.py and use type: ignore to test what happens +# if mypy can't tell if mod isn't a module +from pkg import mod # type: ignore + +def test_import() -> None: + assert mod.h(8) == 24 + +[file pkg/mod.py] +def h(x): + return x * 3 + +[case testFromImportWithKnownModule] +from pkg import mod1 +from pkg import mod2 as modmod +from pkg.mod2 import g as gg +from pkg.mod3 import h as h2, g as g2 + +def test_import() -> None: + assert mod1.h(8) == 24 + assert modmod.g(1) == 1 + assert gg(2) == 2 + assert h2(10) == 12 + assert g2(10) == 13 + +[file pkg/__init__.py] +[file pkg/mod1.py] +def h(x: int) -> int: + return x * 3 + +[file pkg/mod2.py] +def g(x: int) -> int: + return x + +[file pkg/mod3.py] +def h(x: int) -> int: + return x + 2 + +def g(x: int) -> int: + return x + 3 + +[case testFromImportWithUnKnownModule] +def test_import() -> None: + try: + from pkg import a # type: ignore + except ImportError: + pass + +[file pkg/__init__.py] + +[case testMultipleFromImportsWithSamePackageButDifferentModules] +from pkg import a +from pkg import b + +def test_import() -> None: + assert a.g() == 4 + assert b.h() == 39 + +[file pkg/__init__.py] +[file pkg/a.py] + +def g() -> int: + return 4 + +[file pkg/b.py] + +def h() -> int: + return 39 + +[case testReexport] +# Test that we properly handle accessing values that have been reexported +import a +def f(x: int) -> int: + return a.g(x) + a.foo + a.b.foo + +whatever = a.A() + +[file a.py] +from b import g as g, A as A, foo as foo +import b + +[file b.py] +def g(x: int) -> int: + return x + 1 + +class A: + pass + +foo = 20 + +[file driver.py] +from native import f, whatever +import b + +assert f(20) == 61 +assert isinstance(whatever, b.A) + +[case testAssignModule] +import a +assert a.x == 20 +a.x = 10 +[file a.py] +x = 20 +[file driver.py] +import native + +[case testLazyImport] +import shared + +def do_import() -> None: + import a + +assert shared.counter == 0 +do_import() +assert shared.counter == 1 + +[file a.py] +import shared +shared.counter += 1 + +[file shared.py] +counter = 0 + +[case testDelayedImport] +import a +print("inbetween") +import b + +[file a.py] +print("first") + +[file b.py] +print("last") + +[out] +first +inbetween +last + +[case testImportErrorLineNumber] +try: + import enum + import dataclasses, missing # type: ignore[import] +except ImportError as e: + line = e.__traceback__.tb_lineno # type: ignore[attr-defined] + assert line == 3, f"traceback's line number is {line}, expected 3" + +[case testImportGroupIsolation] +def func() -> None: + import second + +import first +func() + +[file first.py] +print("first") + +[file second.py] +print("second") + +[out] +first +second diff --git a/mypyc/test-data/run-integers.test b/mypyc/test-data/run-integers.test new file mode 100644 index 000000000000..d575e141b567 --- /dev/null +++ b/mypyc/test-data/run-integers.test @@ -0,0 +1,540 @@ +# Test cases for integers (compile and run) + +[case testInc] +def inc(x: int) -> int: + return x + 1 +[file driver.py] +from native import inc +print(inc(3)) +print(inc(-5)) +print(inc(10**20)) +[out] +4 +-4 +100000000000000000001 + +[case testCount] +def count(n: int) -> int: + i = 1 + while i <= n: + i = i + 1 + return i +[file driver.py] +from native import count +print(count(0)) +print(count(1)) +print(count(5)) +[out] +1 +2 +6 + +[case testIntMathOps] +# This tests integer math things that are either easier to test in Python than +# in our C tests or are tested here because (for annoying reasons) we don't run +# the C unit tests in our 32-bit CI. +def multiply(x: int, y: int) -> int: + return x * y + +# these stringify their outputs because that will catch if exceptions are mishandled +def floor_div(x: int, y: int) -> str: + return str(x // y) +def remainder(x: int, y: int) -> str: + return str(x % y) + +[file driver.py] +from native import multiply, floor_div, remainder + +def test_multiply(x, y): + assert multiply(x, y) == x * y +def test_floor_div(x, y): + assert floor_div(x, y) == str(x // y) +def test_remainder(x, y): + assert remainder(x, y) == str(x % y) + +test_multiply(10**6, 10**6) +test_multiply(2**15, 2**15-1) +test_multiply(2**14, 2**14) + +test_multiply(10**12, 10**12) +test_multiply(2**30, 2**30-1) +test_multiply(2**29, 2**29) + +test_floor_div(-2**62, -1) +test_floor_div(-2**30, -1) +try: + floor_div(10, 0) +except ZeroDivisionError: + pass +else: + assert False, "Expected ZeroDivisionError" + +test_remainder(-2**62, -1) +test_remainder(-2**30, -1) +try: + remainder(10, 0) +except ZeroDivisionError: + pass +else: + assert False, "Expected ZeroDivisionError" + +[case testBigIntLiteral] +def big_int() -> None: + a_62_bit = 4611686018427387902 + max_62_bit = 4611686018427387903 + b_63_bit = 4611686018427387904 + c_63_bit = 9223372036854775806 + max_63_bit = 9223372036854775807 + d_64_bit = 9223372036854775808 + max_32_bit = 2147483647 + max_32_bit_plus1 = 2147483648 + max_31_bit = 1073741823 + max_31_bit_plus1 = 1073741824 + neg = -1234567 + min_signed_63_bit = -4611686018427387904 + underflow = -4611686018427387905 + min_signed_64_bit = -9223372036854775808 + min_signed_31_bit = -1073741824 + min_signed_31_bit_plus1 = -1073741823 + min_signed_31_bit_minus1 = -1073741825 + min_signed_32_bit = -2147483648 + print(a_62_bit) + print(max_62_bit) + print(b_63_bit) + print(c_63_bit) + print(max_63_bit) + print(d_64_bit) + print('==') + print(max_32_bit) + print(max_32_bit_plus1) + print(max_31_bit) + print(max_31_bit_plus1) + print(neg) + print(min_signed_63_bit) + print(underflow) + print(min_signed_64_bit) + print(min_signed_31_bit) + print(min_signed_31_bit_plus1) + print(min_signed_31_bit_minus1) + print(min_signed_32_bit) +[file driver.py] +from native import big_int +big_int() +[out] +4611686018427387902 +4611686018427387903 +4611686018427387904 +9223372036854775806 +9223372036854775807 +9223372036854775808 +== +2147483647 +2147483648 +1073741823 +1073741824 +-1234567 +-4611686018427387904 +-4611686018427387905 +-9223372036854775808 +-1073741824 +-1073741823 +-1073741825 +-2147483648 + +[case testNeg] +def neg(x: int) -> int: + return -x +[file driver.py] +from native import neg +assert neg(5) == -5 +assert neg(-5) == 5 +assert neg(1073741823) == -1073741823 +assert neg(-1073741823) == 1073741823 +assert neg(1073741824) == -1073741824 +assert neg(-1073741824) == 1073741824 +assert neg(2147483647) == -2147483647 +assert neg(-2147483647) == 2147483647 +assert neg(2147483648) == -2147483648 +assert neg(-2147483648) == 2147483648 +assert neg(4611686018427387904) == -4611686018427387904 +assert neg(-4611686018427387904) == 4611686018427387904 +assert neg(9223372036854775807) == -9223372036854775807 +assert neg(-9223372036854775807) == 9223372036854775807 +assert neg(9223372036854775808) == -9223372036854775808 +assert neg(-9223372036854775808) == 9223372036854775808 + +[case testIsinstanceIntAndNotBool] +def test_isinstance_int_and_not_bool(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) +[file driver.py] +from native import test_isinstance_int_and_not_bool +assert test_isinstance_int_and_not_bool(True) == False +assert test_isinstance_int_and_not_bool(1) == True + +[case testIntOps] +from typing import Any +from testutil import assertRaises + +def check_and(x: int, y: int) -> None: + # eval() can be trusted to calculate expected result + expected = eval('{} & {}'.format(x, y)) + actual = x & y + assert actual == expected, '{} & {}: got {}, expected {}'.format(x, y, actual, expected) + +def check_or(x: int, y: int) -> None: + # eval() can be trusted to calculate expected result + expected = eval('{} | {}'.format(x, y)) + actual = x | y + assert actual == expected, '{} | {}: got {}, expected {}'.format(x, y, actual, expected) + +def check_xor(x: int, y: int) -> None: + # eval() can be trusted to calculate expected result + expected = eval('{} ^ {}'.format(x, y)) + actual = x ^ y + assert actual == expected, '{} ^ {}: got {}, expected {}'.format(x, y, actual, expected) + +def check_bitwise(x: int, y: int) -> None: + for l, r in (x, y), (y, x): + for ll, rr in (l, r), (-l, r), (l, -r), (-l, -r): + check_and(ll, rr) + check_or(ll, rr) + check_xor(ll, rr) + +SHIFT = 30 +DIGIT0a = 615729753 +DIGIT0b = 832796681 +DIGIT1a = 744342356 << SHIFT +DIGIT1b = 321006080 << SHIFT +DIGIT2a = 643582106 << (SHIFT * 2) +DIGIT2b = 656420725 << (SHIFT * 2) +DIGIT50 = 315723472 << (SHIFT * 50) +DIGIT100a = 1020652627 << (SHIFT * 100) +DIGIT100b = 923752451 << (SHIFT * 100) +BIG_SHORT = 3491190729721336556 +MAX_SHORT = (1 << 62) - 1 +MIN_SHORT = -(1 << 62) +MAX_SHORT_32 = (1 << 30) - 1 +MIN_SHORT_32 = -(1 << 30) + +def test_and_or_xor() -> None: + check_bitwise(0, 0) + check_bitwise(0, 1) + check_bitwise(1, 1) + check_bitwise(DIGIT0a, DIGIT0b) + check_bitwise(DIGIT1a, DIGIT1b) + check_bitwise(DIGIT2a, DIGIT2b) + check_bitwise(DIGIT100a, DIGIT100b) + check_bitwise(DIGIT0a, DIGIT0b + DIGIT2a) + check_bitwise(DIGIT0a, DIGIT0b + DIGIT50) + check_bitwise(DIGIT50 + DIGIT1a, DIGIT100a + DIGIT2b) + check_bitwise(BIG_SHORT, DIGIT0a) + check_bitwise(BIG_SHORT, DIGIT0a + DIGIT1a) + check_bitwise(BIG_SHORT, DIGIT0a + DIGIT1a + DIGIT2a) + check_bitwise(BIG_SHORT, DIGIT0a + DIGIT1a + DIGIT2a + DIGIT50) + + for x in range(-25, 25): + for y in range(-25, 25): + check_bitwise(x, y) + +def test_bitwise_inplace() -> None: + # Basic sanity checks; these should use the same code as the non-in-place variants + for x, y in (DIGIT0a, DIGIT1a), (DIGIT2a, DIGIT0a + DIGIT2b): + n = x + n &= y + assert n == x & y + n = x + n |= y + assert n == x | y + n = x + n ^= y + assert n == x ^ y + +def check_invert(x: int) -> None: + # Use eval() as the source of truth + assert ~x == eval('~{}'.format(x)) + assert ~(-x) == eval('~({})'.format(-x)) + +def test_invert() -> None: + check_invert(0) + check_invert(1) + check_invert(DIGIT0a) + check_invert(DIGIT0a + DIGIT1a) + check_invert(DIGIT0a + DIGIT1a + DIGIT2a) + check_invert(DIGIT0a + DIGIT1a + DIGIT2a + DIGIT50) + check_invert(BIG_SHORT) + for delta in -1, 0, 1: + check_invert(MAX_SHORT + delta) + check_invert(MIN_SHORT + delta) + check_invert(MAX_SHORT_32 + delta) + check_invert(MIN_SHORT_32 + delta) + +def check_right_shift(x: int, n: int) -> None: + if n < 0: + try: + x >> n + except ValueError: + return + assert False, "no exception raised" + # Use eval() as the source of truth + expected = eval('{} >> {}'.format(x, n)) + actual = x >> n + assert actual == expected, "{} >> {}: got {}, expected {}".format(x, n, actual, expected) + +def test_right_shift() -> None: + for x in 0, 1, 1235, DIGIT0a, DIGIT0a + DIGIT1a, DIGIT0a + DIGIT50: + for n in 0, 1, 2, 3, 4, 10, 40, 10000, DIGIT1a, -1, -1334444, -DIGIT1a: + check_right_shift(x, n) + check_right_shift(-x, n) + x = DIGIT0a + x >>= 1 + assert x == DIGIT0a >> 1 + x = DIGIT50 + x >>= 5 + assert x == DIGIT50 >> 5 + for i in range(256): + check_right_shift(1, i) + check_right_shift(137, i) + check_right_shift(MAX_SHORT, i) + check_right_shift(MAX_SHORT_32, i) + check_right_shift(MAX_SHORT + 1, i) + check_right_shift(MAX_SHORT_32 + 1, i) + for x in 1, DIGIT50: + try: + # It's okay if this raises an exception + assert x >> DIGIT2a == 0 + except Exception: + pass + try: + x >> -DIGIT2a + assert False + except Exception: + pass + +def check_left_shift(x: int, n: int) -> None: + if n < 0: + try: + x << n + except ValueError: + return + assert False, "no exception raised" + # Use eval() as the source of truth + expected = eval('{} << {}'.format(x, n)) + actual = x << n + assert actual == expected, "{} << {}: got {}, expected {}".format(x, n, actual, expected) + +def test_left_shift() -> None: + for x in 0, 1, 1235, DIGIT0a, DIGIT0a + DIGIT1a, DIGIT0a + DIGIT50: + for n in 0, 1, 2, 10, 40, 10000, -1, -1334444: + check_left_shift(x, n) + check_left_shift(-x, n) + x = DIGIT0a + x <<= 1 + assert x == DIGIT0a << 1 + x = DIGIT50 + x <<= 5 + assert x == DIGIT50 << 5 + for shift in range(256): + check_left_shift(1, shift) + check_left_shift(137, shift) + for x in 1, DIGIT50: + try: + x << DIGIT50 + assert False + except Exception: + pass + try: + x << -DIGIT50 + assert False + except Exception: + pass + +def is_true(x: int) -> bool: + if x: + return True + else: + return False + +def is_true2(x: int) -> bool: + return bool(x) + +def is_false(x: int) -> bool: + if not x: + return True + else: + return False + +def test_int_as_bool() -> None: + assert not is_true(0) + assert not is_true2(0) + assert is_false(0) + for x in 1, 55, -1, -7, 1 << 50, 1 << 101, -(1 << 50), -(1 << 101): + assert is_true(x) + assert is_true2(x) + assert not is_false(x) + +def bool_as_int(b: bool) -> int: + return b + +def bool_as_int2(b: bool) -> int: + return int(b) + +def test_bool_as_int() -> None: + assert bool_as_int(False) == 0 + assert bool_as_int(True) == 1 + assert bool_as_int2(False) == 0 + assert bool_as_int2(True) == 1 + +def no_op_conversion(n: int) -> int: + return int(n) + +def test_no_op_conversion() -> None: + for x in 1, 55, -1, -7, 1 << 50, 1 << 101, -(1 << 50), -(1 << 101): + assert no_op_conversion(x) == x + +def test_floor_divide() -> None: + for x in range(-100, 100): + for y in range(-100, 100): + if y != 0: + assert x // y == getattr(x, "__floordiv__")(y) + +def test_mod() -> None: + for x in range(-100, 100): + for y in range(-100, 100): + if y != 0: + assert x % y == getattr(x, "__mod__")(y) + +def test_constant_fold() -> None: + assert str(-5 + 3) == "-2" + assert str(15 - 3) == "12" + assert str(1000 * 1000) == "1000000" + assert str(12325 // 12 ) == "1027" + assert str(87645 % 321) == "12" + assert str(674253 | 76544) == "748493" + assert str(765 ^ 82) == "687" + assert str(6546 << 3) == "52368" + assert str(6546 >> 7) == "51" + assert str(3**5) == "243" + assert str(~76) == "-77" + try: + 2 / 0 + except ZeroDivisionError: + pass + else: + assert False, "no exception raised" + + x = int() + y = int() - 1 + assert x == -1 or y != -3 + assert -1 <= x + assert -1 == y + + # Use int() to avoid constant propagation + i30 = (1 << 30) + int() + assert i30 == 1 << 30 + i31 = (1 << 31) + int() + assert i31 == 1 << 31 + i32 = (1 << 32) + int() + assert i32 == 1 << 32 + i62 = (1 << 62) + int() + assert i62 == 1 << 62 + i63 = (1 << 63) + int() + assert i63 == 1 << 63 + i64 = (1 << 64) + int() + assert i64 == 1 << 64 + + n30 = -(1 << 30) + int() + assert n30 == -(1 << 30) + n31 = -(1 << 31) + int() + assert n31 == -(1 << 31) + n32 = -(1 << 32) + int() + assert n32 == -(1 << 32) + n62 = -(1 << 62) + int() + assert n62 == -(1 << 62) + n63 = -(1 << 63) + int() + assert n63 == -(1 << 63) + n64 = -(1 << 64) + int() + assert n64 == -(1 << 64) + +def div_by_2(x: int) -> int: + return x // 2 + +def div_by_3(x: int) -> int: + return x // 3 + +def div_by_4(x: int) -> int: + return x // 4 + +def test_floor_divide_by_literal() -> None: + for i in range(-100, 100): + i_boxed: Any = i + assert div_by_2(i) == i_boxed // int('2') + assert div_by_3(i) == i_boxed // int('3') + assert div_by_4(i) == i_boxed // int('4') + +def test_true_divide() -> None: + for x in range(-150, 100): + for y in range(-150, 100): + if y != 0: + assert x / y == getattr(x, "__truediv__")(y) + large1 = (123 + int())**123 + large2 = (121 + int())**121 + assert large1 / large2 == getattr(large1, "__truediv__")(large2) + assert large1 / 135 == getattr(large1, "__truediv__")(135) + assert large1 / -2 == getattr(large1, "__truediv__")(-2) + assert 17 / large2 == getattr(17, "__truediv__")(large2) + + huge = 10**1000 + int() + with assertRaises(OverflowError, "integer division result too large for a float"): + huge / 2 + with assertRaises(OverflowError, "integer division result too large for a float"): + huge / -2 + assert 1 / huge == 0.0 + +[case testIntMinMax] +def test_int_min_max() -> None: + x: int = 200 + y: int = 30 + assert min(x, y) == 30 + assert max(x, y) == 200 + assert min(y, x) == 30 + assert max(y, x) == 200 + +def test_int_hybrid_min_max() -> None: + from typing import Any + + x: object = 30 + y: Any = 20.0 + assert min(x, y) == 20.0 + assert max(x, y) == 30 + + u: object = 20 + v: float = 30.0 + assert min(u, v) == 20 + assert max(u, v) == 30.0 + +def test_int_incompatible_min_max() -> None: + x: int = 2 + y: str = 'aaa' + try: + print(min(x, y)) + except TypeError as e: + assert str(e) == "'<' not supported between instances of 'str' and 'int'" + try: + print(max(x, y)) + except TypeError as e: + assert str(e) == "'>' not supported between instances of 'str' and 'int'" + +def test_int_bool_min_max() -> None: + x: int = 2 + y: bool = False + z: bool = True + assert min(x, y) == False + assert min(x, z) == True + assert max(x, y) == 2 + assert max(x, z) == 2 + + u: int = -10 + assert min(u, y) == -10 + assert min(u, z) == -10 + assert max(u, y) == False + assert max(u, z) == True diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test new file mode 100644 index 000000000000..84d5ee121a20 --- /dev/null +++ b/mypyc/test-data/run-lists.test @@ -0,0 +1,411 @@ +# Test cases for lists (compile and run) + +[case testListPlusEquals] +from typing import Any +def append(x: Any) -> None: + x += [1] + +[file driver.py] +from native import append +x = [] +append(x) +assert x == [1] + +[case testListSum] +from typing import List +def sum(a: List[int], l: int) -> int: + sum = 0 + i = 0 + while i < l: + sum = sum + a[i] + i = i + 1 + return sum +[file driver.py] +from native import sum +print(sum([], 0)) +print(sum([3], 1)) +print(sum([5, 6, -4], 3)) +print(sum([2**128 + 5, -2**127 - 8], 2)) +[out] +0 +3 +7 +170141183460469231731687303715884105725 + +[case testListSet] +from typing import List +def copy(a: List[int], b: List[int], l: int) -> int: + i = 0 + while i < l: + a[i] = b[i] + i = i + 1 + return 0 +[file driver.py] +from native import copy +a = [0, ''] +copy(a, [-1, 5], 2) +print(1, a) +copy(a, [2**128 + 5, -2**127 - 8], 2) +print(2, a) +[out] +1 [-1, 5] +2 [340282366920938463463374607431768211461, -170141183460469231731687303715884105736] + +[case testSieve] +from typing import List + +def primes(n: int) -> List[int]: + a = [1] * (n + 1) + a[0] = 0 + a[1] = 0 + i = 0 + while i < n: + if a[i] == 1: + j = i * i + while j < n: + a[j] = 0 + j = j + i + i = i + 1 + return a +[file driver.py] +from native import primes +print(primes(3)) +print(primes(13)) +[out] +\[0, 0, 1, 1] +\[0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1] + +[case testListBuild] +def test_list_build() -> None: + # Currently LIST_BUILDING_EXPANSION_THRESHOLD equals to 10 + # long list built by list_build_op + l1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + l1.pop() + l1.append(100) + assert l1 == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100] + # short list built by Setmem + l2 = [1, 2] + l2.append(3) + l2.pop() + l2.pop() + assert l2 == [1] + # empty list + l3 = [] + l3.append('a') + assert l3 == ['a'] + +[case testListPrims] +from typing import List + +def test_append() -> None: + l = [1, 2] + l.append(10) + assert l == [1, 2, 10] + l.append(3) + l.append(4) + l.append(5) + assert l == [1, 2, 10, 3, 4, 5] + +def test_pop_last() -> None: + l = [1, 2, 10, 3, 4, 5] + l.pop() + l.pop() + assert l == [1, 2, 10, 3] + +def test_pop_index() -> None: + l = [1, 2, 10, 3] + l.pop(2) + assert l == [1, 2, 3] + l.pop(-2) + assert l == [1, 3] + +def test_count() -> None: + l = [1, 3] + assert l.count(1) == 1 + assert l.count(2) == 0 + +def test_insert() -> None: + l = [1, 3] + l.insert(0, 0) + assert l == [0, 1, 3] + l.insert(2, 2) + assert l == [0, 1, 2, 3] + l.insert(4, 4) + assert l == [0, 1, 2, 3, 4] + l.insert(-1, 5) + assert l == [0, 1, 2, 3, 5, 4] + l = [1, 3] + l.insert(100, 5) + assert l == [1, 3, 5] + l.insert(-100, 6) + assert l == [6, 1, 3, 5] + for long_int in 1 << 100, -(1 << 100): + try: + l.insert(long_int, 5) + except Exception as e: + # The error message is used by CPython + assert type(e).__name__ == 'OverflowError' + assert str(e) == 'Python int too large to convert to C ssize_t' + else: + assert False + +def test_sort() -> None: + l = [1, 4, 3, 6, -1] + l.sort() + assert l == [-1, 1, 3, 4, 6] + l.sort() + assert l == [-1, 1, 3, 4, 6] + l = [] + l.sort() + assert l == [] + +def test_reverse() -> None: + l = [1, 4, 3, 6, -1] + l.reverse() + assert l == [-1, 6, 3, 4, 1] + l.reverse() + assert l == [1, 4, 3, 6, -1] + l = [] + l.reverse() + assert l == [] + +def test_remove() -> None: + l = [1, 3, 4, 3] + l.remove(3) + assert l == [1, 4, 3] + l.remove(3) + assert l == [1, 4] + try: + l.remove(3) + except ValueError: + pass + else: + assert False + +def test_index() -> None: + l = [1, 3, 4, 3] + assert l.index(1) == 0 + assert l.index(3) == 1 + assert l.index(4) == 2 + try: + l.index(0) + except ValueError: + pass + else: + assert False + +[case testListOfUserDefinedClass] +class C: + x: int + +def f() -> int: + c = C() + c.x = 5 + a = [c] + d = a[0] + return d.x + 1 + +def g() -> int: + a = [C()] + a[0].x = 3 + return a[0].x + 4 +[file driver.py] +from native import f, g +print(f()) +print(g()) +[out] +6 +7 + +[case testListOps] +def test_slicing() -> None: + # Use dummy adds to avoid constant folding + zero = int() + two = zero + 2 + s = ["f", "o", "o", "b", "a", "r"] + assert s[two:] == ["o", "b", "a", "r"] + assert s[:two] == ["f", "o"] + assert s[two:-two] == ["o", "b"] + assert s[two:two] == [] + assert s[two:two + 1] == ["o"] + assert s[-two:] == ["a", "r"] + assert s[:-two] == ["f", "o", "o", "b"] + assert s[:] == ["f", "o", "o", "b", "a", "r"] + assert s[two:333] == ["o", "b", "a", "r"] + assert s[333:two] == [] + assert s[two:-333] == [] + assert s[-333:two] == ["f", "o"] + long_int: int = 1000 * 1000 * 1000 * 1000 * 1000 * 1000 * 1000 + assert s[1:long_int] == ["o", "o", "b", "a", "r"] + assert s[long_int:] == [] + assert s[-long_int:-1] == ["f", "o", "o", "b", "a"] + +[case testOperatorInExpression] + +def tuple_in_int0(i: int) -> bool: + return i in [] + +def tuple_in_int1(i: int) -> bool: + return i in (1,) + +def tuple_in_int3(i: int) -> bool: + return i in (1, 2, 3) + +def tuple_not_in_int0(i: int) -> bool: + return i not in [] + +def tuple_not_in_int1(i: int) -> bool: + return i not in (1,) + +def tuple_not_in_int3(i: int) -> bool: + return i not in (1, 2, 3) + +def tuple_in_str(s: "str") -> bool: + return s in ("foo", "bar", "baz") + +def tuple_not_in_str(s: "str") -> bool: + return s not in ("foo", "bar", "baz") + +def list_in_int0(i: int) -> bool: + return i in [] + +def list_in_int1(i: int) -> bool: + return i in (1,) + +def list_in_int3(i: int) -> bool: + return i in (1, 2, 3) + +def list_not_in_int0(i: int) -> bool: + return i not in [] + +def list_not_in_int1(i: int) -> bool: + return i not in (1,) + +def list_not_in_int3(i: int) -> bool: + return i not in (1, 2, 3) + +def list_in_str(s: "str") -> bool: + return s in ("foo", "bar", "baz") + +def list_not_in_str(s: "str") -> bool: + return s not in ("foo", "bar", "baz") + +def list_in_mixed(i: object): + return i in [[], (), "", 0, 0.0, False, 0j, {}, set(), type] + +[file driver.py] + +from native import * + +assert not tuple_in_int0(0) +assert not tuple_in_int1(0) +assert tuple_in_int1(1) +assert not tuple_in_int3(0) +assert tuple_in_int3(1) +assert tuple_in_int3(2) +assert tuple_in_int3(3) +assert not tuple_in_int3(4) + +assert tuple_not_in_int0(0) +assert tuple_not_in_int1(0) +assert not tuple_not_in_int1(1) +assert tuple_not_in_int3(0) +assert not tuple_not_in_int3(1) +assert not tuple_not_in_int3(2) +assert not tuple_not_in_int3(3) +assert tuple_not_in_int3(4) + +assert tuple_in_str("foo") +assert tuple_in_str("bar") +assert tuple_in_str("baz") +assert not tuple_in_str("apple") +assert not tuple_in_str("pie") +assert not tuple_in_str("\0") +assert not tuple_in_str("") + +assert not list_in_int0(0) +assert not list_in_int1(0) +assert list_in_int1(1) +assert not list_in_int3(0) +assert list_in_int3(1) +assert list_in_int3(2) +assert list_in_int3(3) +assert not list_in_int3(4) + +assert list_not_in_int0(0) +assert list_not_in_int1(0) +assert not list_not_in_int1(1) +assert list_not_in_int3(0) +assert not list_not_in_int3(1) +assert not list_not_in_int3(2) +assert not list_not_in_int3(3) +assert list_not_in_int3(4) + +assert list_in_str("foo") +assert list_in_str("bar") +assert list_in_str("baz") +assert not list_in_str("apple") +assert not list_in_str("pie") +assert not list_in_str("\0") +assert not list_in_str("") + +assert list_in_mixed(0) +assert list_in_mixed([]) +assert list_in_mixed({}) +assert list_in_mixed(()) +assert list_in_mixed(False) +assert list_in_mixed(0.0) +assert not list_in_mixed([1]) +assert not list_in_mixed(object) +assert list_in_mixed(type) + +[case testListBuiltFromGenerator] +def test() -> None: + source_a = ["a", "b", "c"] + a = list(x + "f2" for x in source_a) + assert a == ["af2", "bf2", "cf2"] + source_b = [1, 2, 3, 4, 5] + b = [x * 2 for x in source_b] + assert b == [2, 4, 6, 8, 10] + source_c = [10, 20, 30] + c = [x + "f4" for x in (str(y) + "yy" for y in source_c)] + assert c == ["10yyf4", "20yyf4", "30yyf4"] + source_d = [True, False] + d = [not x for x in source_d] + assert d == [False, True] + source_e = [0, 1, 2] + e = list((x ** 2) for x in (y + 2 for y in source_e)) + assert e == [4, 9, 16] + source_str = "abcd" + f = list("str:" + x for x in source_str) + assert f == ["str:a", "str:b", "str:c", "str:d"] + +[case testNextBug] +from typing import List, Optional + +def test(x: List[int]) -> None: + res = next((i for i in x), None) + +[case testListGetItemWithBorrow] +from typing import List + +class D: + def __init__(self, n: int) -> None: + self.n = n + +class C: + def __init__(self, d: D) -> None: + self.d = d + +def test_index_with_literal() -> None: + d1 = D(1) + d2 = D(2) + a = [C(d1), C(d2)] + d = a[0].d + assert d is d1 + d = a[1].d + assert d is d2 + d = a[-1].d + assert d is d2 + d = a[-2].d + assert d is d1 diff --git a/mypyc/test-data/run-loops.test b/mypyc/test-data/run-loops.test new file mode 100644 index 000000000000..76fbb06200a3 --- /dev/null +++ b/mypyc/test-data/run-loops.test @@ -0,0 +1,547 @@ +# Test cases for "range" objects, "for" and "while" loops (compile and run) + +[case testFor] +from typing import List, Tuple +def count(n: int) -> None: + for i in range(n): + print(i) +def count_between(n: int, k: int) -> None: + for i in range(n, k): + print(i) + print('n=', n) +def count_down(n: int, k: int) -> None: + for i in range(n, k, -1): + print(i) +def count_double(n: int, k: int) -> None: + for i in range(n, k, 2): + print(i) +def list_iter(l: List[int]) -> None: + for i in l: + print(i) +def tuple_iter(l: Tuple[int, ...]) -> None: + for i in l: + print(i) +def str_iter(l: str) -> None: + for i in l: + print(i) +def list_rev_iter(l: List[int]) -> None: + for i in reversed(l): + print(i) +def list_rev_iter_lol(l: List[int]) -> None: + for i in reversed(l): + print(i) + if i == 3: + while l: + l.pop() +def count_down_short() -> None: + for i in range(10, 0, -1): + print(i) +[file driver.py] +from native import ( + count, list_iter, list_rev_iter, list_rev_iter_lol, count_between, count_down, count_double, + count_down_short, tuple_iter, str_iter, +) +count(5) +list_iter(list(reversed(range(5)))) +list_rev_iter(list(reversed(range(5)))) +count_between(11, 15) +count_between(10**20, 10**20+3) +count_down(20, 10) +count_double(10, 15) +count_down_short() +print('==') +list_rev_iter_lol(list(reversed(range(5)))) +tuple_iter((1, 2, 3)) +str_iter("abc") +[out] +0 +1 +2 +3 +4 +4 +3 +2 +1 +0 +0 +1 +2 +3 +4 +11 +12 +13 +14 +n= 11 +100000000000000000000 +100000000000000000001 +100000000000000000002 +n= 100000000000000000000 +20 +19 +18 +17 +16 +15 +14 +13 +12 +11 +10 +12 +14 +10 +9 +8 +7 +6 +5 +4 +3 +2 +1 +== +0 +1 +2 +3 +1 +2 +3 +a +b +c + +[case testLoopElse] +from typing import Iterator +def run_for_range(n: int) -> None: + for i in range(n): + if i == 3: + break + print(i) + else: + print(n+1) + +def run_for_list(n: int) -> None: + for i in list(range(n)): + if i == 3: + break + print(i) + else: + print(n+1) + +def run_for_iter(n: int) -> None: + def identity(x: Iterator[int]) -> Iterator[int]: + return x + for i in identity(range(n)): + if i == 3: + break + print(i) + else: + print(n+1) + +def count(n: int) -> int: + i = 1 + while i <= n: + i = i + 1 + if i == 5: + break + else: + i *= -1 + return i + +def nested_while() -> int: + while True: + while False: + pass + else: + break + else: + return -1 + return 0 + +def nested_for() -> int: + for x in range(1000): + for y in [1,2,3]: + pass + else: + break + else: + return -1 + return 0 + +[file driver.py] +from native import run_for_range, run_for_list, run_for_iter, count, nested_while, nested_for +assert nested_while() == 0 +assert nested_for() == 0 +assert count(0) == -1 +assert count(1) == -2 +assert count(5) == 5 +assert count(6) == 5 +run_for_range(3) +run_for_range(5) +print('==') +run_for_list(3) +run_for_list(5) +print('==') +run_for_iter(3) +run_for_iter(5) +[out] +0 +1 +2 +4 +0 +1 +2 +== +0 +1 +2 +4 +0 +1 +2 +== +0 +1 +2 +4 +0 +1 +2 + +[case testNestedLoopSameIdx] +from typing import List, Generator + +def nested_enumerate() -> None: + l1 = [0,1,2] + l2 = [0,1,2] + outer_seen = [] + outer = 0 + for i, j in enumerate(l1): + assert i == outer + outer_seen.append(i) + inner = 0 + for i, k in enumerate(l2): + assert i == inner + inner += 1 + outer += 1 + assert i == 2 + assert outer_seen == l1 + +def nested_range() -> None: + outer = 0 + outer_seen = [] + for i in range(3): + assert i == outer + outer_seen.append(i) + inner = 0 + for i in range(3): + assert i == inner + inner += 1 + outer += 1 + assert outer_seen == [0,1,2] + +def nested_list() -> None: + l1 = [0,1,2] + l2 = [0,1,2] + outer_seen = [] + outer = 0 + for i in l1: + assert i == outer + outer_seen.append(i) + inner = 0 + for i in l2: + assert i == inner + inner += 1 + outer += 1 + assert outer_seen == l1 + +def nested_yield() -> Generator: + for i in range(3): + for i in range(3): + yield i + yield i + + +[file driver.py] +from native import nested_enumerate, nested_range, nested_list, nested_yield +nested_enumerate() +nested_range() +nested_list() +gen = nested_yield() +for k in range(12): + assert next(gen) == k % 4 +[out] + +[case testForIterable] +from typing import Iterable, Dict, Any, Tuple, TypeVar + +T = TypeVar("T") + +def iterate_over_any(a: Any) -> None: + for element in a: + print(element) + +def iterate_over_iterable(iterable: Iterable[T]) -> None: + for element in iterable: + print(element) + +def iterate_and_delete(d: Dict[int, int]) -> None: + for key in d: + d.pop(key) + +def sum_over_values(d: Dict[int, int]) -> int: + s = 0 + for key in d: + s = s + d[key] + return s + +def sum_over_even_values(d: Dict[int, int]) -> int: + s = 0 + for key in d: + if d[key] % 2: + continue + s = s + d[key] + return s + +def sum_over_two_values(d: Dict[int, int]) -> int: + s = 0 + i = 0 + for key in d: + if i == 2: + break + s = s + d[key] + i = i + 1 + return s + +def iterate_over_tuple(iterable: Tuple[int, int, int]) -> None: + for element in iterable: + print(element) + +[file driver.py] +from native import iterate_over_any, iterate_over_iterable, iterate_and_delete, sum_over_values, sum_over_even_values, sum_over_two_values, iterate_over_tuple +import traceback +def broken_generator(n): + num = 0 + while num < n: + yield num + num += 1 + raise Exception('Exception Manually Raised') + +d = {1:1, 2:2, 3:3, 4:4, 5:5} +print(sum_over_values(d)) +print(sum_over_even_values(d)) +print(sum_over_two_values(d)) + +try: + iterate_over_any(5) +except TypeError: + traceback.print_exc() +try: + iterate_over_iterable(broken_generator(5)) +except Exception: + traceback.print_exc() +try: + iterate_and_delete(d) +except RuntimeError: + traceback.print_exc() + +iterate_over_tuple((1, 2, 3)) +[out] +Traceback (most recent call last): + File "driver.py", line 16, in + iterate_over_any(5) + File "native.py", line 6, in iterate_over_any + for element in a: +TypeError: 'int' object is not iterable +Traceback (most recent call last): + File "driver.py", line 20, in + iterate_over_iterable(broken_generator(5)) + File "native.py", line 10, in iterate_over_iterable + for element in iterable: + File "driver.py", line 8, in broken_generator + raise Exception('Exception Manually Raised') +Exception: Exception Manually Raised +Traceback (most recent call last): + File "driver.py", line 24, in + iterate_and_delete(d) + File "native.py", line 14, in iterate_and_delete + for key in d: +RuntimeError: dictionary changed size during iteration +15 +6 +3 +0 +1 +2 +3 +4 +1 +2 +3 +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 16, in + iterate_over_any(5) + ~~~~~~~~~~~~~~~~^^^ + File "native.py", line 6, in iterate_over_any + for element in a: +TypeError: 'int' object is not iterable +Traceback (most recent call last): + File "driver.py", line 20, in + iterate_over_iterable(broken_generator(5)) + ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ + File "native.py", line 10, in iterate_over_iterable + for element in iterable: + File "driver.py", line 8, in broken_generator + raise Exception('Exception Manually Raised') +Exception: Exception Manually Raised +Traceback (most recent call last): + File "driver.py", line 24, in + iterate_and_delete(d) + ~~~~~~~~~~~~~~~~~~^^^ + File "native.py", line 14, in iterate_and_delete + for key in d: +RuntimeError: dictionary changed size during iteration +15 +6 +3 +0 +1 +2 +3 +4 +1 +2 +3 + +[case testContinueFor] +def f() -> None: + for n in range(5): + continue +[file driver.py] +from native import f +f() + +[case testMultipleVarsWithLoops] +# Test comprehensions and for loops with multiple index variables +l = [(1, 2, 'a'), (3, 4, 'b'), (5, 6, 'c')] +l2 = [str(a*100+b)+' '+c for a, b, c in l] +l3 = [] +for a, b, c in l: + l3.append(str(a*1000+b)+' '+c) +[file driver.py] +from native import l, l2, l3 +for a in l2 + l3: + print(a) +[out] +102 a +304 b +506 c +1002 a +3004 b +5006 c + +[case testForZipAndEnumerate] +from typing import Iterable, List, Any +def f(a: Iterable[int], b: List[int]) -> List[Any]: + res = [] + for (x, y), z in zip(enumerate(a), b): + res.append((x, y, z)) + return res +def g(a: Iterable[int], b: Iterable[str]) -> List[Any]: + res = [] + for x, (y, z) in enumerate(zip(a, b)): + res.append((x, y, z)) + return res + +[file driver.py] +from native import f, g + +assert f([6, 7], [8, 9]) == [(0, 6, 8), (1, 7, 9)] +assert g([6, 7], ['a', 'b']) == [(0, 6, 'a'), (1, 7, 'b')] +assert f([6, 7], [8]) == [(0, 6, 8)] +assert f([6], [8, 9]) == [(0, 6, 8)] + +[case testEnumerateEmptyList] +from typing import List + +def get_enumerate_locals(iterable: List[int]) -> int: + for i, j in enumerate(iterable): + pass + try: + return i + except NameError: + return -100 + +[file driver.py] +from native import get_enumerate_locals + +print(get_enumerate_locals([])) +print(get_enumerate_locals([55])) +print(get_enumerate_locals([551, 552])) + +[out] +-100 +0 +1 + +[case testIterTypeTrickiness] +# Test inferring the type of a for loop body doesn't cause us grief +# Extracted from somethings that broke in mypy + +from typing import Optional + +# really I only care that this one build +def foo(x: object) -> None: + if isinstance(x, dict): + for a in x: + pass + +def bar(x: Optional[str]) -> None: + vars = ( + ("a", 'lol'), + ("b", 'asdf'), + ("lol", x), + ("an int", 10), + ) + for name, value in vars: + pass + +[file driver.py] +from native import bar +bar(None) + +[case testRangeObject] +from typing import Any + +def f(x: range) -> int: + sum = 0 + for i in x: + sum += i + return sum + +def test_range_object() -> None: + r1 = range(4, 12, 2) + tmp_list = [x for x in r1] + assert tmp_list == [4, 6, 8, 10] + assert f(r1) == 28 + r2: Any = range(10) + assert f(r2) == 45 + r3: Any = 'x' + try: + f(r3) + except TypeError as e: + assert "range object expected; got str" in str(e) + try: + ff: Any = f + ff(r3) + except TypeError as e: + assert "range object expected; got str" in str(e) + try: + r4 = range(4, 12, 0) + except ValueError as e: + assert "range() arg 3 must not be zero" in str(e) diff --git a/mypyc/test-data/run-match.test b/mypyc/test-data/run-match.test new file mode 100644 index 000000000000..7b7ad9a4342c --- /dev/null +++ b/mypyc/test-data/run-match.test @@ -0,0 +1,283 @@ +[case testTheBigMatch_python3_10] +class Person: + __match_args__ = ("name", "age") + + name: str + age: int + + def __init__(self, name: str, age: int) -> None: + self.name = name + self.age = age + + def __str__(self) -> str: + return f"Person(name={self.name!r}, age={self.age})" + + +def f(x: object) -> None: + match x: + case 123: + print("test 1") + + case 456 | 789: + print("test 2") + + case True | False | None: + print("test 3") + + case Person("bob" as name, age): + print(f"test 4 ({name=}, {age=})") + + case num if num == 5: + print("test 5") + + case 6 as num: + print(f"test 6 ({num=})") + + case (7 | "7") as value: + print(f"test 7 ({value=})") + + case Person("alice", age=123): + print("test 8") + + case Person("charlie", age=123 | 456): + print("test 9") + + case Person("dave", 123) as dave: + print(f"test 10 {dave}") + + case {"test": 11}: + print("test 11") + + case {"test": 12, **rest}: + print(f"test 12 (rest={rest})") + + case {}: + print("test map final") + + case ["test", 13]: + print("test 13") + + case ["test", 13, _]: + print("test 13b") + + case ["test", 14, *_]: + print("test 14") + + # TODO: Fix "rest" being used here coliding with above "rest" + case ["test", 15, *rest2]: + print(f"test 15 ({rest2})") + + case ["test", *rest3, 16]: + print(f"test 16 ({rest3})") + + case [*rest4, "test", 17]: + print(f"test 17 ({rest4})") + + case [*rest4, "test", 18, "some", "fluff"]: + print(f"test 18 ({rest4})") + + case str("test 19"): + print("test 19") + + case str(test_20) if test_20.startswith("test 20"): + print(f"test 20 ({test_20[7:]!r})") + + case ("test 21" as value) | ("test 21 as well" as value): + print(f"test 21 ({value[7:]!r})") + + case []: + print("test sequence final") + + case _: + print("test final") +[file driver.py] +from native import f, Person + +# test 1 +f(123) + +# test 2 +f(456) +f(789) + +# test 3 +f(True) +f(False) +f(None) + +# test 4 +f(Person("bob", 123)) + +# test 5 +f(5) + +# test 6 +f(6) + +# test 7 +f(7) +f("7") + +# test 8 +f(Person("alice", 123)) + +# test 9 +f(Person("charlie", 123)) +f(Person("charlie", 456)) + +# test 10 +f(Person("dave", 123)) + +# test 11 +f({"test": 11}) +f({"test": 11, "some": "key"}) + +# test 12 +f({"test": 12}) +f({"test": 12, "key": "value"}) +f({"test": 12, "key": "value", "abc": "123"}) + +# test map final +f({}) + +# test 13 +f(["test", 13]) + +# test 13b +f(["test", 13, "fail"]) + +# test 14 +f(["test", 14]) +f(["test", 14, "something"]) + +# test 15 +f(["test", 15]) +f(["test", 15, "something"]) + +# test 16 +f(["test", 16]) +f(["test", "filler", 16]) +f(["test", "more", "filler", 16]) + +# test 17 +f(["test", 17]) +f(["stuff", "test", 17]) +f(["more", "stuff", "test", 17]) + +# test 18 +f(["test", 18, "some", "fluff"]) +f(["stuff", "test", 18, "some", "fluff"]) +f(["more", "stuff", "test", 18, "some", "fluff"]) + +# test 19 +f("test 19") + +# test 20 +f("test 20") +f("test 20 something else") + +# test 21 +f("test 21") +f("test 21 as well") + +# test sequence final +f([]) + +# test final +f("") + +[out] +test 1 +test 2 +test 2 +test 3 +test 3 +test 3 +test 4 (name='bob', age=123) +test 5 +test 6 (num=6) +test 7 (value=7) +test 7 (value='7') +test 8 +test 9 +test 9 +test 10 Person(name='dave', age=123) +test 11 +test 11 +test 12 (rest={}) +test 12 (rest={'key': 'value'}) +test 12 (rest={'key': 'value', 'abc': '123'}) +test map final +test 13 +test 13b +test 14 +test 14 +test 15 ([]) +test 15 (['something']) +test 16 ([]) +test 16 (['filler']) +test 16 (['more', 'filler']) +test 17 ([]) +test 17 (['stuff']) +test 17 (['more', 'stuff']) +test 18 ([]) +test 18 (['stuff']) +test 18 (['more', 'stuff']) +test 19 +test 20 ('') +test 20 (' something else') +test 21 ('') +test 21 (' as well') +test sequence final +test final +[case testCustomMappingAndSequenceObjects_python3_10] +def f(x: object) -> None: + match x: + case {"key": "value", **rest}: + print(rest, type(rest)) + + case [1, 2, *rest2]: + print(rest2, type(rest2)) + +[file driver.py] +from collections.abc import Mapping, Sequence + +from native import f + +class CustomMapping(Mapping): + inner: dict + + def __init__(self, inner: dict) -> None: + self.inner = inner + + def __getitem__(self, key): + return self.inner[key] + + def __iter__(self): + return iter(self.inner) + + def __len__(self) -> int: + return len(self.inner) + + +class CustomSequence(Sequence): + inner: list + + def __init__(self, inner: list) -> None: + self.inner = inner + + def __getitem__(self, index: int) -> None: + return self.inner[index] + + def __len__(self) -> int: + return len(self.inner) + +mapping = CustomMapping({"key": "value", "some": "data"}) +sequence = CustomSequence([1, 2, 3]) + +f(mapping) +f(sequence) + +[out] +{'some': 'data'} +[3] diff --git a/mypyc/test-data/run-math.test b/mypyc/test-data/run-math.test new file mode 100644 index 000000000000..266b4851575f --- /dev/null +++ b/mypyc/test-data/run-math.test @@ -0,0 +1,106 @@ +# Test cases for the math module (compile and run) + +[case testMathOps] +from typing import Any, Callable +from typing_extensions import Final +import math +from math import pi, e, tau, inf, nan +from testutil import assertRaises, float_vals, assertDomainError, assertMathRangeError + +pymath: Any = math + +def validate_one_arg(test: Callable[[float], float], validate: Callable[[float], float]) -> None: + """Ensure that test and validate behave the same for various float args.""" + for x in float_vals: + try: + expected = validate(x) + except Exception as e: + try: + test(x) + assert False, f"no exception raised for {x!r}, expected {e!r}" + except Exception as e2: + assert repr(e) == repr(e2), f"actual for {x!r}: {e2!r}, expected: {e!r}" + continue + actual = test(x) + assert repr(actual) == repr(expected), ( + f"actual for {x!r}: {actual!r}, expected {expected!r}") + +def validate_two_arg(test: Callable[[float, float], float], + validate: Callable[[float, float], float]) -> None: + """Ensure that test and validate behave the same for various float args.""" + for x in float_vals: + for y in float_vals: + args = f"({x!r}, {y!r})" + try: + expected = validate(x, y) + except Exception as e: + try: + test(x, y) + assert False, f"no exception raised for {args}, expected {e!r}" + except Exception as e2: + assert repr(e) == repr(e2), f"actual for {args}: {e2!r}, expected: {e!r}" + continue + try: + actual = test(x, y) + except Exception as e: + assert False, f"no exception expected for {args}, got {e!r}" + assert repr(actual) == repr(expected), ( + f"actual for {args}: {actual!r}, expected {expected!r}") + +def test_sqrt() -> None: + validate_one_arg(lambda x: math.sqrt(x), pymath.sqrt) + +def test_sin() -> None: + validate_one_arg(lambda x: math.sin(x), pymath.sin) + +def test_cos() -> None: + validate_one_arg(lambda x: math.cos(x), pymath.cos) + +def test_tan() -> None: + validate_one_arg(lambda x: math.tan(x), pymath.tan) + +def test_exp() -> None: + validate_one_arg(lambda x: math.exp(x), pymath.exp) + +def test_log() -> None: + validate_one_arg(lambda x: math.log(x), pymath.log) + +def test_floor() -> None: + validate_one_arg(lambda x: math.floor(x), pymath.floor) + +def test_ceil() -> None: + validate_one_arg(lambda x: math.ceil(x), pymath.ceil) + +def test_fabs() -> None: + validate_one_arg(lambda x: math.fabs(x), pymath.fabs) + +def test_pow() -> None: + validate_two_arg(lambda x, y: math.pow(x, y), pymath.pow) + +def test_copysign() -> None: + validate_two_arg(lambda x, y: math.copysign(x, y), pymath.copysign) + +def test_isinf() -> None: + for x in float_vals: + assert repr(math.isinf(x)) == repr(pymath.isinf(x)) + +def test_isnan() -> None: + for x in float_vals: + assert repr(math.isnan(x)) == repr(pymath.isnan(x)) + + +def test_pi_is_inlined_correctly() -> None: + assert math.pi == pi == 3.141592653589793 + +def test_e_is_inlined_correctly() -> None: + assert math.e == e == 2.718281828459045 + +def test_tau_is_inlined_correctly() -> None: + assert math.tau == tau == 6.283185307179586 + +def test_inf_is_inlined_correctly() -> None: + assert math.inf == inf == float("inf") + +def test_nan_is_inlined_correctly() -> None: + assert math.isnan(math.nan) + assert math.isnan(nan) diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test new file mode 100644 index 000000000000..2252f3aa104a --- /dev/null +++ b/mypyc/test-data/run-misc.test @@ -0,0 +1,1173 @@ +[case testMaybeUninitVar] +class C: + def __init__(self, x: int) -> None: + self.x = x + +def f(b: bool) -> None: + u = C(1) + while b: + v = C(2) + if v is not u: + break + print(v.x) +[file driver.py] +from native import f +f(True) +[out] +2 + +[case testUninitBoom] +def f(a: bool, b: bool) -> None: + if a: + x = 'lol' + if b: + print(x) + +def g() -> None: + try: + [0][1] + y = 1 + except Exception: + pass + print(y) + +[file driver.py] +from native import f, g +from testutil import assertRaises + +f(True, True) +f(False, False) +with assertRaises(UnboundLocalError): + f(False, True) +with assertRaises(UnboundLocalError): + g() +[out] +lol + +[case testBuiltins] +y = 10 +def f(x: int) -> None: + print(5) + d = globals() + assert d['y'] == 10 + d['y'] = 20 + assert y == 20 +[file driver.py] +from native import f +f(5) +[out] +5 + +[case testOptional] +from typing import Optional + +class A: pass + +def f(x: Optional[A]) -> Optional[A]: + return x + +def g(x: Optional[A]) -> int: + if x is None: + return 1 + if x is not None: + return 2 + return 3 + +def h(x: Optional[int], y: Optional[bool]) -> None: + pass + +[file driver.py] +from native import f, g, A +a = A() +assert f(None) is None +assert f(a) is a +assert g(None) == 1 +assert g(a) == 2 + +[case testInferredOptionalAssignment] +from typing import Any, Generator + +def f(b: bool) -> Any: + if b: + x = None + else: + x = 1 + + if b: + y = 1 + else: + y = None + + m = 1 if b else None + n = None if b else 1 + return ((x, y), (m, n)) + +def gen(b: bool) -> Generator[Any, None, None]: + if b: + y = 1 + else: + y = None + yield y + +assert f(False) == ((1, None), (None, 1)) +assert f(True) == ((None, 1), (1, None)) +assert next(gen(False)) is None +assert next(gen(True)) == 1 + +[case testWith] +from typing import Any +class Thing: + def __init__(self, x: str) -> None: + self.x = x + def __enter__(self) -> str: + print('enter!', self.x) + if self.x == 'crash': + raise Exception('ohno') + return self.x + def __exit__(self, x: Any, y: Any, z: Any) -> None: + print('exit!', self.x, y) + +def foo(i: int) -> int: + with Thing('a') as x: + print("yooo?", x) + if i == 0: + return 10 + elif i == 1: + raise Exception('exception!') + return -1 + +def bar() -> None: + with Thing('a') as x, Thing('b') as y: + print("yooo?", x, y) + +def baz() -> None: + with Thing('a') as x, Thing('crash') as y: + print("yooo?", x, y) + +[file driver.py] +from native import foo, bar, baz +assert foo(0) == 10 +print('== foo ==') +try: + foo(1) +except Exception: + print('caught') +assert foo(2) == -1 + +print('== bar ==') +bar() + +print('== baz ==') +try: + baz() +except Exception: + print('caught') + +[out] +enter! a +yooo? a +exit! a None +== foo == +enter! a +yooo? a +exit! a exception! +caught +enter! a +yooo? a +exit! a None +== bar == +enter! a +enter! b +yooo? a b +exit! b None +exit! a None +== baz == +enter! a +enter! crash +exit! a ohno +caught + +[case testDisplays] +from typing import List, Set, Tuple, Sequence, Dict, Any, Mapping + +def listDisplay(x: List[int], y: List[int]) -> List[int]: + return [1, 2, *x, *y, 3] + +def setDisplay(x: Set[int], y: Set[int]) -> Set[int]: + return {1, 2, *x, *y, 3} + +def tupleDisplay(x: Sequence[str], y: Sequence[str]) -> Tuple[str, ...]: + return ('1', '2', *x, *y, '3') + +def dictDisplay(x: str, y1: Dict[str, int], y2: Dict[str, int]) -> Dict[str, int]: + return {x: 2, **y1, 'z': 3, **y2} + +def dictDisplayUnpackMapping(obj: Mapping[str, str]) -> Dict[str, str]: + return {**obj, "env": "value"} + +[file driver.py] +import os +from native import listDisplay, setDisplay, tupleDisplay, dictDisplay, dictDisplayUnpackMapping +assert listDisplay([4], [5, 6]) == [1, 2, 4, 5, 6, 3] +assert setDisplay({4}, {5}) == {1, 2, 3, 4, 5} +assert tupleDisplay(['4', '5'], ['6']) == ('1', '2', '4', '5', '6', '3') +assert dictDisplay('x', {'y1': 1}, {'y2': 2, 'z': 5}) == {'x': 2, 'y1': 1, 'y2': 2, 'z': 5} +assert dictDisplayUnpackMapping(os.environ) == {**os.environ, "env": "value"} + +[case testArbitraryLvalues] +from typing import List, Dict, Any + +class O(object): + def __init__(self) -> None: + self.x = 1 + +def increment_attr(a: Any) -> Any: + a.x += 1 + return a + +def increment_attr_o(o: O) -> O: + o.x += 1 + return o + +def increment_all_indices(l: List[int]) -> List[int]: + for i in range(len(l)): + l[i] += 1 + return l + +def increment_all_keys(d: Dict[str, int]) -> Dict[str, int]: + for k in d: + d[k] += 1 + return d + +[file driver.py] +from native import O, increment_attr, increment_attr_o, increment_all_indices, increment_all_keys + +class P(object): + def __init__(self) -> None: + self.x = 0 + +assert increment_attr(P()).x == 1 +assert increment_attr_o(O()).x == 2 +assert increment_all_indices([1, 2, 3]) == [2, 3, 4] +assert increment_all_keys({'a':1, 'b':2, 'c':3}) == {'a':2, 'b':3, 'c':4} + +[case testControlFlowExprs] +from typing import Tuple +def foo() -> object: + print('foo') + return 'foo' +def bar() -> object: + print('bar') + return 'bar' +def t(x: int) -> int: + print(x) + return x + +def f(b: bool) -> Tuple[object, object, object]: + x = foo() if b else bar() + y = b or foo() + z = b and foo() + return (x, y, z) +def g() -> Tuple[object, object]: + return (foo() or bar(), foo() and bar()) + +def nand(p: bool, q: bool) -> bool: + if not (p and q): + return True + return False + +def chained(x: int, y: int, z: int) -> bool: + return t(x) < t(y) > t(z) + +def chained2(x: int, y: int, z: int, w: int) -> bool: + return t(x) < t(y) < t(z) < t(w) +[file driver.py] +from native import f, g, nand, chained, chained2 +assert f(True) == ('foo', True, 'foo') +print() +assert f(False) == ('bar', 'foo', False) +print() +assert g() == ('foo', 'bar') + +assert nand(True, True) == False +assert nand(True, False) == True +assert nand(False, True) == True +assert nand(False, False) == True + +print() +assert chained(10, 20, 15) == True +print() +assert chained(10, 20, 30) == False +print() +assert chained(21, 20, 30) == False +print() +assert chained2(1, 2, 3, 4) == True +print() +assert chained2(1, 0, 3, 4) == False +print() +assert chained2(1, 2, 0, 4) == False +[out] +foo +foo + +bar +foo + +foo +foo +bar + +10 +20 +15 + +10 +20 +30 + +21 +20 + +1 +2 +3 +4 + +1 +0 + +1 +2 +0 + +[case testMultipleAssignment] +from typing import Tuple, List, Any + +def from_tuple(t: Tuple[int, str]) -> List[Any]: + x, y = t + return [y, x] + +def from_tuple_sequence(t: Tuple[int, ...]) -> List[int]: + x, y, z = t + return [z, y, x] + +def from_list(l: List[int]) -> List[int]: + x, y = l + return [y, x] + +def from_list_complex(l: List[int]) -> List[int]: + ll = l[:] + ll[1], ll[0] = l + return ll + +def from_any(o: Any) -> List[Any]: + x, y = o + return [y, x] + +def multiple_assignments(t: Tuple[int, str]) -> List[Any]: + a, b = c, d = t + e, f = g, h = 1, 2 + return [a, b, c, d, e, f, g, h] +[file driver.py] +from native import ( + from_tuple, from_tuple_sequence, from_list, from_list_complex, from_any, multiple_assignments +) + +assert from_tuple((1, 'x')) == ['x', 1] + +assert from_tuple_sequence((1, 5, 4)) == [4, 5, 1] +try: + from_tuple_sequence((1, 5)) +except ValueError as e: + assert 'not enough values to unpack (expected 3, got 2)' in str(e) +else: + assert False + +assert from_list([3, 4]) == [4, 3] +try: + from_list([5, 4, 3]) +except ValueError as e: + assert 'too many values to unpack (expected 2)' in str(e) +else: + assert False + +assert from_list_complex([7, 6]) == [6, 7] +try: + from_list_complex([5, 4, 3]) +except ValueError as e: + assert 'too many values to unpack (expected 2)' in str(e) +else: + assert False + +assert from_any('xy') == ['y', 'x'] + +assert multiple_assignments((4, 'x')) == [4, 'x', 4, 'x', 1, 2, 1, 2] + +[case testUnpack] +from typing import List + +a, *b = [1, 2, 3, 4, 5] + +*c, d = [1, 2, 3, 4, 5] + +e, *f = [1,2] + +j, *k, l = [1, 2, 3] + +m, *n, o = [1, 2, 3, 4, 5, 6] + +p, q, r, *s, t = [1,2,3,4,5,6,7,8,9,10] + +tup = (1,2,3) +y, *z = tup + +def unpack1(l : List[int]) -> None: + *v1, v2, v3 = l + +def unpack2(l : List[int]) -> None: + v1, *v2, v3 = l + +def unpack3(l : List[int]) -> None: + v1, v2, *v3 = l + +[file driver.py] +from native import a, b, c, d, e, f, j, k, l, m, n, o, p, q, r, s, t, y, z +from native import unpack1, unpack2, unpack3 +from testutil import assertRaises + +assert a == 1 +assert b == [2,3,4,5] +assert c == [1,2,3,4] +assert d == 5 +assert e == 1 +assert f == [2] +assert j == 1 +assert k == [2] +assert l == 3 +assert m == 1 +assert n == [2,3,4,5] +assert o == 6 +assert p == 1 +assert q == 2 +assert r == 3 +assert s == [4,5,6,7,8,9] +assert t == 10 +assert y == 1 +assert z == [2,3] + +with assertRaises(ValueError, "not enough values to unpack"): + unpack1([1]) + +with assertRaises(ValueError, "not enough values to unpack"): + unpack2([1]) + +with assertRaises(ValueError, "not enough values to unpack"): + unpack3([1]) + +[out] + +[case testModuleTopLevel] +x = 1 +print(x) + +def f() -> None: + print(x + 1) + +def g() -> None: + global x + x = 77 + +[file driver.py] +import native +native.f() +native.x = 5 +native.f() +native.g() +print(native.x) + +[out] +1 +2 +6 +77 + +[case testComprehensions] +from typing import List + +# A list comprehension +l = [str(x) + " " + str(y) + " " + str(x*y) for x in range(10) + if x != 6 if x != 5 for y in range(x) if y*x != 8] + +# Test short-circuiting as well +def pred(x: int) -> bool: + if x > 6: + raise Exception() + return x > 3 +# If we fail to short-circuit, pred(x) will be called with x=7 +# eventually and will raise an exception. +l2 = [x for x in range(10) if x <= 6 if pred(x)] + +src = ['x'] + +def f() -> List[str]: + global src + res = src + src = [] + return res + +l3 = [s for s in f()] +l4 = [s for s in f()] + +# A dictionary comprehension +d = {k: k*k for k in range(10) if k != 5 if k != 6} + +# A set comprehension +s = {str(x) + " " + str(y) + " " + str(x*y) for x in range(10) + if x != 6 if x != 5 for y in range(x) if y*x != 8} + +[file driver.py] +from native import l, l2, l3, l4, d, s +for a in l: + print(a) +print(tuple(l2)) +assert l3 == ['x'] +assert l4 == [] +for k in sorted(d): + print(k, d[k]) +for a in sorted(s): + print(a) +[out] +1 0 0 +2 0 0 +2 1 2 +3 0 0 +3 1 3 +3 2 6 +4 0 0 +4 1 4 +4 3 12 +7 0 0 +7 1 7 +7 2 14 +7 3 21 +7 4 28 +7 5 35 +7 6 42 +8 0 0 +8 2 16 +8 3 24 +8 4 32 +8 5 40 +8 6 48 +8 7 56 +9 0 0 +9 1 9 +9 2 18 +9 3 27 +9 4 36 +9 5 45 +9 6 54 +9 7 63 +9 8 72 +(4, 5, 6) +0 0 +1 1 +2 4 +3 9 +4 16 +7 49 +8 64 +9 81 +1 0 0 +2 0 0 +2 1 2 +3 0 0 +3 1 3 +3 2 6 +4 0 0 +4 1 4 +4 3 12 +7 0 0 +7 1 7 +7 2 14 +7 3 21 +7 4 28 +7 5 35 +7 6 42 +8 0 0 +8 2 16 +8 3 24 +8 4 32 +8 5 40 +8 6 48 +8 7 56 +9 0 0 +9 1 9 +9 2 18 +9 3 27 +9 4 36 +9 5 45 +9 6 54 +9 7 63 +9 8 72 + +[case testDummyTypes] +from typing import Tuple, List, Dict, NamedTuple +from typing_extensions import Literal, TypedDict, NewType + +class A: + pass + +T = List[A] +U = List[Tuple[int, str]] +Z = List[List[int]] +D = Dict[int, List[int]] +N = NewType('N', int) +G = Tuple[int, str] +def foo(x: N) -> int: + return x +foo(N(10)) +z = N(10) +Lol = NamedTuple('Lol', (('a', int), ('b', T))) +x = Lol(1, []) +def take_lol(x: Lol) -> int: + return x.a + +TD = TypedDict('TD', {'a': int}) +def take_typed_dict(x: TD) -> int: + return x['a'] + +def take_literal(x: Literal[1, 2, 3]) -> None: + print(x) + +[file driver.py] +import sys +from native import * + +if sys.version_info[:3] > (3, 5, 2): + assert "%s %s %s %s" % (T, U, Z, D) == "typing.List[native.A] typing.List[typing.Tuple[int, str]] typing.List[typing.List[int]] typing.Dict[int, typing.List[int]]" +print(x) +print(z) +print(take_lol(x)) +print(take_typed_dict({'a': 20})) +try: + take_typed_dict(None) +except Exception as e: + print(type(e).__name__) + + +take_literal(1) +# We check that the type is the real underlying type +try: + take_literal(None) +except Exception as e: + print(type(e).__name__) +# ... but not that it is a valid literal value +take_literal(10) +[out] +Lol(a=1, b=[]) +10 +1 +20 +TypeError +1 +TypeError +10 + +[case testClassBasedTypedDict] +from typing_extensions import TypedDict + +class TD(TypedDict): + a: int + +class TD2(TD): + b: int + +class TD3(TypedDict, total=False): + c: int + +class TD4(TD3, TD2, total=False): + d: int + +def test_typed_dict() -> None: + d = TD(a=5) + assert d['a'] == 5 + assert type(d) == dict + # TODO: This doesn't work yet + # assert TD.__annotations__ == {'a': int} + +def test_inherited_typed_dict() -> None: + d = TD2(a=5, b=3) + assert d['a'] == 5 + assert d['b'] == 3 + assert type(d) == dict + +def test_non_total_typed_dict() -> None: + d3 = TD3(c=3) + d4 = TD4(a=1, b=2, c=3, d=4) + assert d3['c'] == 3 + assert d4['d'] == 4 + +[case testClassBasedNamedTuple] +from typing import NamedTuple +import sys + +# Class-based NamedTuple requires Python 3.6+ +version = sys.version_info[:2] +if version[0] == 3 and version[1] < 6: + exit() + +class NT(NamedTuple): + a: int + +def test_named_tuple() -> None: + t = NT(a=1) + assert t.a == 1 + assert type(t) is NT + assert isinstance(t, tuple) + assert not isinstance(tuple([1]), NT) + +[case testUnion] +from typing import Union + +class A: + def __init__(self, x: int) -> None: + self.x = x + def f(self, y: int) -> int: + return y + self.x + +class B: + def __init__(self, x: object) -> None: + self.x = x + def f(self, y: object) -> object: + return y + +def f(x: Union[A, str]) -> object: + if isinstance(x, A): + return x.x + else: + return x + 'x' + +def g(x: int) -> Union[A, int]: + if x == 0: + return A(1) + else: + return x + 1 + +def get(x: Union[A, B]) -> object: + return x.x + +def call(x: Union[A, B]) -> object: + return x.f(5) + +[file driver.py] +from native import A, B, f, g, get, call +assert f('a') == 'ax' +assert f(A(4)) == 4 +assert isinstance(g(0), A) +assert g(2) == 3 +assert get(A(5)) == 5 +assert get(B('x')) == 'x' +assert call(A(4)) == 9 +assert call(B('x')) == 5 +try: + f(1) +except TypeError: + pass +else: + assert False + +[case testAnyAll] +from typing import Iterable + +def call_any_nested(l: Iterable[Iterable[int]], val: int = 0) -> int: + res = any(i == val for l2 in l for i in l2) + return 0 if res else 1 + +def call_any(l: Iterable[int], val: int = 0) -> int: + res = any(i == val for i in l) + return 0 if res else 1 + +def call_all(l: Iterable[int], val: int = 0) -> int: + res = all(i == val for i in l) + return 0 if res else 1 + +[file driver.py] +from native import call_any, call_all, call_any_nested + +zeros = [0, 0, 0] +ones = [1, 1, 1] +mixed_001 = [0, 0, 1] +mixed_010 = [0, 1, 0] +mixed_100 = [1, 0, 0] +mixed_011 = [0, 1, 1] +mixed_101 = [1, 0, 1] +mixed_110 = [1, 1, 0] + +assert call_any([]) == 1 +assert call_any(zeros) == 0 +assert call_any(ones) == 1 +assert call_any(mixed_001) == 0 +assert call_any(mixed_010) == 0 +assert call_any(mixed_100) == 0 +assert call_any(mixed_011) == 0 +assert call_any(mixed_101) == 0 +assert call_any(mixed_110) == 0 + +assert call_all([]) == 0 +assert call_all(zeros) == 0 +assert call_all(ones) == 1 +assert call_all(mixed_001) == 1 +assert call_all(mixed_010) == 1 +assert call_all(mixed_100) == 1 +assert call_all(mixed_011) == 1 +assert call_all(mixed_101) == 1 +assert call_all(mixed_110) == 1 + +assert call_any_nested([[1, 1, 1], [1, 1], []]) == 1 +assert call_any_nested([[1, 1, 1], [0, 1], []]) == 0 + +[case testSum] +[typing fixtures/typing-full.pyi] +from typing import Any, List + +def test_sum_of_numbers() -> None: + assert sum(x for x in [1, 2, 3]) == 6 + assert sum(x for x in [0.0, 1.2, 2]) == 6.2 + assert sum(x for x in [1, 1j]) == 1 + 1j + +def test_sum_callables() -> None: + assert sum((lambda x: x == 0)(x) for x in []) == 0 + assert sum((lambda x: x == 0)(x) for x in [0]) == 1 + assert sum((lambda x: x == 0)(x) for x in [0, 0, 0]) == 3 + assert sum((lambda x: x == 0)(x) for x in [0, 1, 0]) == 2 + assert sum((lambda x: x % 2 == 0)(x) for x in range(2**10)) == 2**9 + +def test_sum_comparisons() -> None: + assert sum(x == 0 for x in []) == 0 + assert sum(x == 0 for x in [0]) == 1 + assert sum(x == 0 for x in [0, 0, 0]) == 3 + assert sum(x == 0 for x in [0, 1, 0]) == 2 + assert sum(x % 2 == 0 for x in range(2**10)) == 2**9 + +def test_sum_multi() -> None: + assert sum(i + j == 0 for i, j in zip([0, 0, 0], [0, 1, 0])) == 2 + +def test_sum_misc() -> None: + # misc cases we do optimize (note, according to sum's helptext, we don't need to support + # non-numeric cases, but CPython and mypyc both do anyway) + assert sum(c == 'd' for c in 'abcdd') == 2 + # misc cases we do not optimize + assert sum([0, 1]) == 1 + assert sum([0, 1], 1) == 2 + +def test_sum_start_given() -> None: + a = 1 + assert sum((x == 0 for x in [0, 1]), a) == 2 + assert sum(((lambda x: x == 0)(x) for x in []), 1) == 1 + assert sum(((lambda x: x == 0)(x) for x in [0]), 1) == 2 + assert sum(((lambda x: x == 0)(x) for x in [0, 0, 0]), 1) == 4 + assert sum(((lambda x: x == 0)(x) for x in [0, 1, 0]), 1) == 3 + assert sum(((lambda x: x % 2 == 0)(x) for x in range(2**10)), 1) == 2**9 + 1 + assert sum((x for x in [1, 1j]), 2j) == 1 + 3j + assert sum((c == 'd' for c in 'abcdd'), 1) == 3 + +[case testNoneStuff] +from typing import Optional +class A: + x: int + +def lol(x: A) -> None: + setattr(x, 'x', 5) + +def none() -> None: + return + +def arg(x: Optional[A]) -> bool: + return x is None + +[file driver.py] +import native +native.lol(native.A()) + +# Catch refcounting failures +for i in range(10000): + native.none() + native.arg(None) + +[case testBorrowRefs] +def make_garbage(arg: object) -> None: + b = True + while b: + arg = None + b = False + +[file driver.py] +from native import make_garbage +import sys + +def test(): + x = object() + r0 = sys.getrefcount(x) + make_garbage(x) + r1 = sys.getrefcount(x) + assert r0 == r1 + +test() + +[case testFinalStaticRunFail] +if False: + from typing import Final + +if bool(): + x: 'Final' = [1] + +def f() -> int: + return x[0] + +[file driver.py] +from native import f +try: + print(f()) +except NameError as e: + print(e.args[0]) +[out] +value for final name "x" was not set + +[case testFinalStaticRunListTupleInt] +if False: + from typing import Final + +x: 'Final' = [1] +y: 'Final' = (1, 2) +z: 'Final' = 1 + 1 + +def f() -> int: + return x[0] +def g() -> int: + return y[0] +def h() -> int: + return z - 1 + +[file driver.py] +from native import f, g, h, x, y, z +print(f()) +print(x[0]) +print(g()) +print(y) +print(h()) +print(z) +[out] +1 +1 +1 +(1, 2) +1 +2 + +[case testCheckVersion] +import sys + +if sys.version_info[:2] == (3, 13): + def version() -> int: + return 13 +elif sys.version_info[:2] == (3, 12): + def version() -> int: + return 12 +elif sys.version_info[:2] == (3, 11): + def version() -> int: + return 11 +elif sys.version_info[:2] == (3, 10): + def version() -> int: + return 10 +elif sys.version_info[:2] == (3, 9): + def version() -> int: + return 9 +elif sys.version_info[:2] == (3, 8): + def version() -> int: + return 8 +elif sys.version_info[:2] == (3, 7): + def version() -> int: + return 7 +elif sys.version_info[:2] == (3, 6): + def version() -> int: + return 6 +else: + raise Exception("we don't support this version yet!") + + +[file driver.py] +import sys +version = sys.version_info[:2] + +import native +assert native.version() == sys.version_info[1] + +[case testTypeErrorMessages] +from typing import Tuple +class A: + pass +class B: + pass + +def f(x: B) -> None: + pass +def g(x: Tuple[int, A]) -> None: + pass +[file driver.py] +from testutil import assertRaises +from native import A, f, g + +class Busted: + pass +Busted.__module__ = None + +with assertRaises(TypeError, "int"): + f(0) +with assertRaises(TypeError, "native.A"): + f(A()) +with assertRaises(TypeError, "tuple[None, native.A]"): + f((None, A())) +with assertRaises(TypeError, "tuple[tuple[int, str], native.A]"): + f(((1, "ha"), A())) +with assertRaises(TypeError, "tuple[<50 items>]"): + f(tuple(range(50))) + +with assertRaises(TypeError, "errored formatting real type!"): + f(Busted()) + +with assertRaises(TypeError, "tuple[int, native.A] object expected; got tuple[int, int]"): + g((20, 30)) + +[case testComprehensionShadowBinder] +def foo(x: object) -> object: + if isinstance(x, list): + return tuple(x for x in x), x + return None + +[file driver.py] +from native import foo + +assert foo(None) == None +assert foo([1, 2, 3]) == ((1, 2, 3), [1, 2, 3]) + +[case testAllLiterals] +# Test having all sorts of literals in a single file + +def test_str() -> None: + assert '' == eval("''") + assert len('foo bar' + str()) == 7 + assert 'foo bar' == eval("'foo bar'") + assert 'foo\u1245\0bar' == eval("'foo' + chr(0x1245) + chr(0) + 'bar'") + assert 'foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345' == eval("'foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345'") + assert 'Zoobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar123' == eval("'Zoobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar123'") + +def test_bytes() -> None: + assert b'' == eval("b''") + assert b'foo bar' == eval("b'foo bar'") + assert b'\xafde' == eval(r"b'\xafde'") + assert b'foo\xde\0bar' == eval("b'foo' + bytes([0xde, 0]) + b'bar'") + assert b'foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345' == eval("b'foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345foobar12345'") + +def test_int() -> None: + assert 2875872359823758923758923759 == eval('2875872359823758923758923759') + assert -552875872359823758923758923759 == eval('-552875872359823758923758923759') + +def test_float() -> None: + assert 1.5 == eval('1.5') + assert -3.75 == eval('-3.75') + assert 2.5e10 == eval('2.5e10') + assert 2.5e50 == eval('2.5e50') + assert 2.5e1000 == eval('2.5e1000') + assert -2.5e1000 == eval('-2.5e1000') + +def test_complex() -> None: + assert 1.5j == eval('1.5j') + assert 1.5j + 2.5 == eval('2.5 + 1.5j') + assert -3.75j == eval('-3.75j') + assert 2.5e10j == eval('2.5e10j') + assert 2.5e50j == eval('2.5e50j') + assert 2.5e1000j == eval('2.5e1000j') + assert 2.5e1000j + 3.5e2000 == eval('3.5e2000 + 2.5e1000j') + assert -2.5e1000j == eval('-2.5e1000j') + +[case testUnreachableExpressions] +from typing import cast +import sys + +A = sys.platform == 'x' and foobar +B = sys.platform == 'x' and sys.foobar +C = sys.platform == 'x' and f(a, -b, 'y') > [c + e, g(y=2)] +C = sys.platform == 'x' and cast(a, b[c]) +C = sys.platform == 'x' and (lambda x: y + x) +C = sys.platform == 'x' and (x for y in z) +C = sys.platform == 'x' and [x for y in z] +C = sys.platform == 'x' and {x: x for y in z} +C = sys.platform == 'x' and {x for y in z} + +assert not A +assert not B +assert not C + +[case testDoesntSegfaultWhenTopLevelFails] +# make the initial import fail +assert False + +[file driver.py] +# load native, cause PyInit to be run, create the module but don't finish initializing the globals +for _ in range(2): + try: + import native + raise RuntimeError('exception expected') + except AssertionError: + pass + +[case testUnderscoreFunctionsInMethods] + +class A: + def _(arg): pass + def _(arg): pass +class B(A): + def _(arg): pass + def _(arg): pass + +[case testGlobalRedefinition_toplevel] +# mypy: allow-redefinition +i = 0 +i += 1 +i = "foo" +i += i +i = b"foo" + +def test_redefinition() -> None: + assert i == b"foo" + +[case testWithNative] +class DummyContext: + def __init__(self): + self.c = 0 + def __enter__(self) -> None: + self.c += 1 + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.c -= 1 + +def test_dummy_context() -> None: + c = DummyContext() + with c: + assert c.c == 1 + assert c.c == 0 + +[case testWithNativeVarArgs] +class DummyContext: + def __init__(self): + self.c = 0 + def __enter__(self) -> None: + self.c += 1 + def __exit__(self, *args: object) -> None: + self.c -= 1 + +def test_dummy_context() -> None: + c = DummyContext() + with c: + assert c.c == 1 + assert c.c == 0 diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test new file mode 100644 index 000000000000..5edd5688140e --- /dev/null +++ b/mypyc/test-data/run-multimodule.test @@ -0,0 +1,904 @@ +-- These test cases compile two or more modules at a time. +-- Any file prefixed with "other" is compiled. +-- +-- Note that these are run in three compilation modes: regular, +-- multi-file and separate. See the docstrings of +-- mypyc.test.test_run.TestRunMultiFile and +-- mypyc.test.test_run.TestRunSeparate for more information. +-- +-- Some of these files perform multiple incremental runs. See +-- test-data/unit/check-incremental.test for more information +-- about how this is specified (e.g. .2 file name suffixes). + +[case testMultiModulePackage] +from p.other import g, _i as i +def f(x: int) -> int: + from p.other import h + return i(h(g(x + 1))) +[file p/__init__.py] +[file p/other.py] +def g(x: int) -> int: + return x + 2 +def h(x: int) -> int: + return x + 1 +def _i(x: int) -> int: + return x + 3 +[file driver.py] +import native +from native import f +from p.other import g +assert f(3) == 10 +assert g(2) == 4 +try: + f(1.1) +except TypeError: + pass +else: + assert False +try: + g(1.1) +except TypeError: + pass +else: + assert False + +[case testMultiModuleFastpaths] +[file other_main.py] + +[file other_main.py.2] +from other_b import A, func + +class B(A): + pass + +def test() -> None: + a = A() + assert func() == 12 + assert a.method() == "test" + +test() + +[file other_b.py] +class A: + def method(self) -> str: + return "test" + +def func() -> int: + return 12 + +# Remove all the methods and functions from globals to ensure that +# they get called via the fastpaths even when doing incremental +# compilation. +setattr(A, 'method', None) +setattr(A, '__init__', None) +globals()['func'] = None +globals()['A'] = None + +[file driver.py] +import other_main + +[case testMultiModuleSameNames] +# Use same names in both modules +import other +def f() -> int: + return 0 +class C: + x: int + def __init__(self) -> None: + self.x = 1 + def f(self, x: int) -> int: + return self.x + x +class D(C): pass + +def g(x: 'other.C') -> None: + pass + +[file other.py] +def f(x: int) -> int: + return x + 1 +class C: + x: int + def __init__(self) -> None: + self.x = 2 + def f(self, x: int) -> int: + return self.x + x + 1 +class D(C): pass +[file driver.py] +import native, other +assert native.f() == 0 +assert other.f(3) == 4 +c1 = native.C() +c1.x += 3 +c2 = other.C() +c2.x += 6 +assert c1.f(9) == 1 + 3 + 9 +assert c2.f(7) == 2 + 6 + 7 + 1 +assert isinstance(native.D(), native.C) +assert isinstance(other.D(), other.C) +assert not isinstance(native.D(), other.C) +assert not isinstance(other.D(), native.C) + +[case testMultiModuleInitializeImportedModules] +from other import f + +def g() -> int: + return f(1) +[file other.py] +def f(x: int) -> int: + return x + 4 +[file driver.py] +import sys +assert 'other' not in sys.modules +from native import g +assert 'other' in sys.modules +assert g() == 5 +f = sys.modules['other'].f +assert f(1) == 5 +try: + f(1.1) +except TypeError: + pass +else: + assert False + +[case testMultiModuleImportClass] +from typing import cast +from other import C, a_global + +class D(C): + def __init__(self, x: int) -> None: + self.x = x + +def f(c: C) -> int: + d = D(3) + o: object = c + c = cast(C, o) + return a_global + c.x + c.f() + d.x + d.f() + 1 +[file other.py] +from typing_extensions import Final +a_global: Final = int('5') + +class C: + x: int + + def __init__(self, x: int) -> None: + self.x = x + + def __hash__(self) -> int: + return self.x + + def __str__(self) -> str: + return str(self.x) + + def f(self) -> int: + return 2 + + def check(self) -> None: + assert isinstance(self, C) + +[file driver.py] +from native import f, D +from other import C +c = C(4) +assert f(c) == 5 + 4 + 2 + 3 + 2 + 1 +assert str(D(10)) == '10' +assert hash(10) == 10 +try: + f(1) +except TypeError: + pass +else: + assert False + +assert isinstance(D(10), C) + +c.check() +D(10).check() + +[case testMultiModuleSpecialize] +from other import A + +class B(A): + def foo(self, x: object) -> int: + print(2) + return id(x) +[file other.py] +class A: + def foo(self, x: int) -> object: + print(1) + return str(x) + +def use_a(x: A, y: int) -> object: + return x.foo(y) + +[file driver.py] +from native import B +from other import A, use_a +a = A() +b = B() +o = object() +i = 10 +assert a.foo(10) == '10' +assert b.foo(o) == id(o) +assert use_a(a, 10) == '10' +assert use_a(b, i) == id(i) +[out] +1 +2 +1 +2 + +[case testMultiModuleLiterals] +from other import gs, gi, gf + +def fs() -> str: + return 'f' + gs() +def fi() -> int: + return 10001000100010001000 + gi() +def ff() -> float: + return 2.0 + gf() +[file other.py] +def gi() -> int: + return 20001000100010001000 +def gs() -> str: + return 'g' +def gf() -> float: + return 3.0 +[file driver.py] +from native import fs, fi, ff +assert fs() == 'fg' +assert fi() == 30002000200020002000 +assert ff() == 5.0 + +[case testMultiModuleTraceback] +from other import fail2 + +def fail() -> None: + fail2() +[file other.py] +def fail2() -> None: + x = [1] + x[2] = 2 +[file driver.py] +import traceback +import sys +import native +import other +try: + other.fail2() +except IndexError: + tb = sys.exc_info()[2] + assert tb.tb_next.tb_frame.f_globals is other.__dict__ + traceback.print_exc() +try: + native.fail() +except IndexError: + tb = sys.exc_info()[2] + assert tb.tb_next.tb_frame.f_globals is native.__dict__ + traceback.print_exc() +[out] +Traceback (most recent call last): + File "driver.py", line 6, in + other.fail2() + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 12, in + native.fail() + File "native.py", line 4, in fail + fail2() + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 6, in + other.fail2() + ~~~~~~~~~~~^^ + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 12, in + native.fail() + ~~~~~~~~~~~^^ + File "native.py", line 4, in fail + fail2() + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range + +[case testMultiModuleCycle] +if False: + from typing import Final +import other + +x = int('0') # type: Final + +def f1() -> int: + return other.f2() + other.x + +def f3() -> int: + return 5 +[file other.py] +if False: + from typing import Final +import native + +x = int('0') # type: Final + +def f2() -> int: + return native.f3() + native.x +[file driver.py] +from native import f1 +assert f1() == 5 + +[case testMultiModuleCycleWithClasses] +import other + +class D: pass + +def f() -> other.C: + return other.C() + +def g(c: other.C) -> D: + return c.d + +[file other.py] +import native + +class C: + def __init__(self) -> None: + self.d = native.D() + +def h(d: native.D) -> None: + pass + +[file driver.py] +from native import f, g +from other import C, h + +c = f() +assert isinstance(c, C) +assert g(c) is c.d +h(c.d) + +try: + g(1) +except TypeError: + pass +else: + assert False + +try: + h(1) +except TypeError: + pass +else: + assert False + +[case testMultiModuleCycleWithInheritance] +import other + +class Deriv1(other.Base1): + def __init__(self) -> None: + super().__init__() + +class Base2: + y: int + def __init__(self) -> None: + self.y = 2 + +[file other.py] +from typing import Tuple +import native + +class Base1: + a: Tuple[int, int] + x: int + def __init__(self) -> None: + self.x = 1 + +def make_2() -> native.Base2: + return native.Base2() + +[file driver.py] +from native import Deriv1 +from other import make_2 +a = Deriv1() +assert a.x == 1 +b = make_2() +assert b.y == 2 + +[case testMultiModuleTraitInheritance] +from other import Base1, Base2 + +class Deriv1(Base1, Base2): + pass + +[file other.py] +from mypy_extensions import trait + +@trait +class Base1: + def foo(self) -> int: return 10 +@trait +class Base2: + def bar(self) -> int: return 12 + +[file driver.py] +from native import Deriv1 +a = Deriv1() +assert a.foo() == 10 and a.bar() == 12 + + +[case testImportCycleWithNonCompiledModule] +import m + +class C: pass + +def f1() -> int: + m.D() + return m.f2() + +def f3() -> int: + return 2 + +[file m.py] +# This module is NOT compiled +import native + +class D: pass + +def f2() -> int: + native.C() + return native.f3() + +[file driver.py] +from native import f1 + +assert f1() == 2 + +[case testImportCycleWithTopLevelStatements] +import other +x = 1 +print(x) + +[file other.py] +import native +x = 2 +print(x) + +[file driver.py] +import other +print('-') +import native +print('>', native.x) +print('>', other.x) + +[out] +1 +2 +- +> 1 +> 2 + +[case testMultiModuleCycleIfMypy1] +from other import foo, bar + +class Foo: + def foo(self) -> None: + foo(self) +class Bar: + def bar(self) -> None: + bar(self) + +[file other.py] +from typing_extensions import TYPE_CHECKING +MYPY = False +if MYPY: + from native import Foo +if TYPE_CHECKING: + from native import Bar + +def foo(x: 'Foo') -> None: + pass +def bar(x: 'Bar') -> None: + pass + +[file driver.py] +from native import Foo, Bar +Foo().foo() +Bar().bar() + +[case testMultiModuleCycleIfMypy2] +MYPY = False +if MYPY: + from other import C + +class D: + def __init__(self) -> None: + self.y = 1 + +def f(c: 'C') -> int: + return c.x + +[file other.py] +from typing_extensions import TYPE_CHECKING +if TYPE_CHECKING: + from native import D + +class C: + def __init__(self) -> None: + self.x = 2 + +def g(d: 'D') -> int: + return d.y + +[file driver.py] +from native import f, D +from other import g, C + +assert f(C()) == 2 +assert g(D()) == 1 + +try: + f(D()) +except TypeError: + pass +else: + assert False + +try: + g(C()) +except TypeError: + pass +else: + assert False + +[case testMultiModuleRelative] +from package.a import f +[file package/__init__.py] +[file package/a.py] +from . import b +from .c import c3 +def f() -> None: + print("Hello " + b.b2()) + print("Hello " + c3()) +[file package/b.py] +def b2() -> str: + return "moon!" +[file package/c.py] +def c3() -> str: + return "sun!" + +[file driver.py] +from native import f +f() +[out] +Hello moon! +Hello sun! + +[case testMultiModuleCrash] +b = False +if b: + import other + +def foo() -> None: + try: + other.x + except: + pass + else: + assert False + +[file other.py] +x = 10 + +[file driver.py] +from native import foo +foo() + +[case testTrivialIncremental] +# separate: [(["other.py", "other_b.py"], "stuff")] +from other import x +from other_b import z +y = x + z +[file other.py] +x = 1 +[file other.py.2] +x = 2 +[file other_b.py] +z = 1 + +[file driver.py] +from native import y +print(y) +[out] +2 +[out2] +3 +[rechecked other, other_b] + +[case testIncrementalCompilation1] +import non_native +from other_a import A +from other_b import z + +a = A() +assert a.y == z + +assert non_native.foo() == 0 + +[file other_a.py] +from other_b import z +from typing import Iterable + +class A: + def __init__(self) -> None: + self.y = z +[file other_a.py.2] +from other_b import z +from typing import Iterable + +class A: + def __init__(self) -> None: + self.x = 'test' + self.y = z +[file other_b.py] +import other_a + +z = 10 + +def foo() -> 'other_a.A': + return other_a.A() +[file other_b.py.3] +import other_a + +z = 20 + +def foo() -> 'other_a.A': + return other_a.A() + +[file non_native.py] +import other_a + +def foo() -> int: + return 0 + +[file non_native.py.4] +import other_a + +def foo() -> float: + return 0 + +[file driver.py] +from native import a +print(a.y, getattr(a, 'x', None)) + +[out] +10 None +[out2] +10 test +[out3] +20 test +[out4] +20 test + +[rechecked other_a, other_b, native, non_native] +[rechecked2 other_a, other_b] +[rechecked3 native, non_native] + + +-- This one tests a group that is not an SCC. +[case testIncrementalCompilation2] +# separate: [(["other_a.py", "other_b.py"], "stuff")] +from other_a import A +from other_b import z + +a = A() +assert a.y == z + +[file other_a.py] +from other_b import z + +class A: + def __init__(self) -> None: + self.y = z +[file other_a.py.2] +from other_b import z + +class A: + def __init__(self) -> None: + self.x = 'test' + self.y = z + +[file other_b.py] +z = 10 + +[file driver.py] +from native import a +print(a.y, getattr(a, 'x', None)) + +[out] +10 None +[out2] +10 test + +[rechecked other_a, other_b, native] + +[case testIncrementalCompilation3] +from other import X +Y = X +def foo() -> int: + return X + +[file other.py] +from typing_extensions import Final +X: Final = 10 + +[file other.py.2] +from typing_extensions import Final +X: Final = 20 + +[file driver.py] +import native +import other +assert native.Y == other.X +assert native.foo() == other.X + +[rechecked native, other] + +-- This one tests a group changing +[case testIncrementalCompilation4] +# separate: [(["other_a.py", "other_b.py"], "stuff")] +# separate2: [] +from other_a import A +from other_b import z + +a = A() +assert a.y == z + +[file other_a.py] +from other_b import z + +class A: + def __init__(self) -> None: + self.y = z + +[file other_b.py] +z = 10 + +[file wtvr.py.2] + +[file driver.py] +from native import a +print(a.y, getattr(a, 'x', None)) + +[out] +10 None +[out2] +10 None + +[rechecked other_a, other_b, native] + +-- This one tests cases where other modules *do not* need rechecked +[case testIncrementalCompilation5] +import other_a +[file other_a.py] +from other_b import f +assert f(10) == 20 +[file other_a.py.2] +from other_b import f +assert f(20) == 40 + +[file other_b.py] +def f(x: int) -> int: + return x * 2 + +[file driver.py] +import native + +[rechecked other_a] + +-- Delete one of the C files and make sure this forces recompilation +[case testIncrementalCompilation6] +import other_a +assert other_a.foo() == 10 +[file other_a.py] +def foo() -> int: return 10 + +[file build/__native_other_a.c] + +[delete build/__native_other_a.c.2] + +[file driver.py] +import native + +[rechecked native, other_a] + +[case testSeparateCompilationWithUndefinedAttribute] +from other_a import A + +def f() -> None: + a = A() + if a.x == 5: + print(a.y) + print(a.m()) + else: + assert a.x == 6 + try: + print(a.y) + except AttributeError: + print('y undefined') + else: + assert False + + try: + print(a.m()) + except AttributeError: + print('y undefined') + else: + assert False + +[file other_a.py] +from other_b import B + +class A(B): + def __init__(self) -> None: + self.y = 9 + +[file other_a.py.2] +from other_b import B + +class A(B): + x = 6 + + def __init__(self) -> None: + pass + +[file other_b.py] +class B: + x = 5 + + def __init__(self) -> None: + self.y = 7 + + def m(self) -> int: + return self.y + +[file driver.py] +from native import f +f() + +[rechecked native, other_a] + +[out] +9 +9 +[out2] +y undefined +y undefined + +[case testIncrementalCompilationWithDeletable] +import other_a +[file other_a.py] +from other_b import C +[file other_a.py.2] +from other_b import C +c = C() +print(getattr(c, 'x', None)) +del c.x +print(getattr(c, 'x', None)) +[file other_b.py] +class C: + __deletable__ = ['x'] + def __init__(self) -> None: + self.x = 0 +[file driver.py] +import native +[out] +[out2] +0 +None diff --git a/mypyc/test-data/run-mypy-sim.test b/mypyc/test-data/run-mypy-sim.test new file mode 100644 index 000000000000..b48665c27855 --- /dev/null +++ b/mypyc/test-data/run-mypy-sim.test @@ -0,0 +1,138 @@ +-- Some test code that tries to simulate important/interesting parts of mypy itself! +[case testSimulateMypy] +from mypy_extensions import trait +from typing import List, TypeVar, cast, Generic +from abc import abstractmethod +import other_strconv as strconv +#from other_visitor import ExpressionVisitor, StatementVisitor, NodeVisitor +import other_visitor as visitor + +T = TypeVar('T') + +############ nodes.py + +class Context: + def __init__(self) -> None: + self.line = -1 + def set_line(self, line: int) -> None: + self.line = line + +class Node(Context): + def accept(self, visitor: visitor.NodeVisitor[T]) -> T: return cast(T, None) + def to_str(self) -> str: + return self.accept(strconv.StrConv()) + +@trait +class Statement(Node): + def accept(self, visitor: visitor.StatementVisitor[T]) -> T: return cast(T, None) + +@trait +class Expression(Node): + def accept(self, visitor: visitor.ExpressionVisitor[T]) -> T: return cast(T, None) + +@trait +class SymbolNode(Node): + """Nodes that can be stored in a symbol table.""" + @abstractmethod + def name(self) -> str: return cast(str, None) + +class FuncBase(Node): + def __init__(self) -> None: + super().__init__() + self.is_static = False + +class Block(Statement): + def __init__(self, stmts: List[Statement]) -> None: + self.stmts = stmts + + def accept(self, visitor: visitor.StatementVisitor[T]) -> T: + return visitor.visit_block(self) + +class ExprStmt(Statement): + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: visitor.StatementVisitor[T]) -> T: + return visitor.visit_expr_stmt(self) + +class FuncItem(FuncBase): + def __init__(self, body: Block) -> None: + self.body = body + +class FuncDef(FuncItem, SymbolNode, Statement): + def __init__(self, name: str, body: Block) -> None: + super().__init__(body) + self._name = name + def accept(self, visitor: visitor.StatementVisitor[T]) -> T: + return visitor.visit_func_def(self) + def name(self) -> str: + return self._name + +class LambdaExpr(FuncItem, Expression): + def accept(self, visitor: visitor.ExpressionVisitor[T]) -> T: + return visitor.visit_lambda_expr(self) + +def lol(x: Statement) -> int: + return x.line + +[file other_visitor.py] +from mypy_extensions import trait +from typing import TypeVar, cast, Generic +from abc import abstractmethod +import native as nodes + +T = TypeVar('T') + +@trait +class ExpressionVisitor(Generic[T]): + @abstractmethod + def visit_lambda_expr(self, o: 'nodes.LambdaExpr') -> T: + return cast(T, None) + +@trait +class StatementVisitor(Generic[T]): + @abstractmethod + def visit_block(self, o: 'nodes.Block') -> T: + return cast(T, None) + + @abstractmethod + def visit_func_def(self, o: 'nodes.FuncDef') -> T: + return cast(T, None) + + @abstractmethod + def visit_expr_stmt(self, o: 'nodes.ExprStmt') -> T: + return cast(T, None) + +@trait +class NodeVisitor(Generic[T], ExpressionVisitor[T], StatementVisitor[T]): + pass + +[file other_strconv.py] +from typing import List +import native as nodes +from other_visitor import NodeVisitor + +class StrConv(NodeVisitor[str]): + def visit_block(self, b: nodes.Block) -> str: + # we really need comprehensions! + # TODO: PartialType unsupported + things = [] # type: List[str] + for s in b.stmts: + things.append(s.accept(self)) + return "{" + "; ".join(things) + "}" + + def visit_func_def(self, f: nodes.FuncDef) -> str: + return "def " + f.name() + "(): " + f.body.accept(self) + + def visit_expr_stmt(self, e: nodes.ExprStmt) -> str: + return e.expr.accept(self) + + def visit_lambda_expr(self, b: nodes.LambdaExpr) -> str: + return "(fn: " + b.body.accept(self) + ")" + +[file driver.py] +from native import * + +block = Block([Block([]), ExprStmt(LambdaExpr(Block([])))]) +fn = FuncDef('test', block) +assert fn.to_str() == "def test(): {{}; (fn: {})}" diff --git a/mypyc/test-data/run-primitives.test b/mypyc/test-data/run-primitives.test new file mode 100644 index 000000000000..694700d4738c --- /dev/null +++ b/mypyc/test-data/run-primitives.test @@ -0,0 +1,375 @@ +# Test cases for misc primitives (compile and run) +# +# Please only add tests that don't have an obvious place in type-specific test +# files such as run-strings.test, run-lists.test, etc. + +[case testGenericEquality] +def eq(a: object, b: object) -> bool: + if a == b: + return True + else: + return False +def ne(a: object, b: object) -> bool: + if a != b: + return True + else: + return False +def f(o: object) -> bool: + if [1, 2] == o: + return True + else: + return False +[file driver.py] +from native import eq, ne, f +assert eq('xz', 'x' + 'z') +assert not eq('x', 'y') +assert not ne('xz', 'x' + 'z') +assert ne('x', 'y') +assert f([1, 2]) +assert not f([2, 2]) +assert not f(1) + +[case testGenericBinaryOps] +from typing import Any +def add(x: Any, y: Any) -> Any: + return x + y +def subtract(x: Any, y: Any) -> Any: + return x - y +def multiply(x: Any, y: Any) -> Any: + return x * y +def floor_div(x: Any, y: Any) -> Any: + return x // y +def true_div(x: Any, y: Any) -> Any: + return x / y +def remainder(x: Any, y: Any) -> Any: + return x % y +def power(x: Any, y: Any) -> Any: + return x ** y +def lshift(x: Any, y: Any) -> Any: + return x << y +def rshift(x: Any, y: Any) -> Any: + return x >> y +def num_and(x: Any, y: Any) -> Any: + return x & y +def num_xor(x: Any, y: Any) -> Any: + return x ^ y +def num_or(x: Any, y: Any) -> Any: + return x | y +def lt(x: Any, y: Any) -> Any: + if x < y: + return True + else: + return False +def le(x: Any, y: Any) -> Any: + if x <= y: + return True + else: + return False +def gt(x: Any, y: Any) -> Any: + if x > y: + return True + else: + return False +def ge(x: Any, y: Any) -> Any: + if x >= y: + return True + else: + return False +def contains(x: Any, y: Any) -> Any: + if x in y: + return True + else: + return False +def identity(x: Any, y: Any) -> Any: + if x is y: + return True + else: + return False +def disidentity(x: Any, y: Any) -> Any: + if x is not y: + return True + else: + return False +def not_eq_cond(a: Any, b: Any) -> bool: + if not (a == b): + return True + else: + return False +def eq2(a: Any, b: Any) -> bool: + return a == b +def slice1(x: Any) -> Any: + return x[:] +def slice2(x: Any, y: Any) -> Any: + return x[y:] +def slice3(x: Any, y: Any) -> Any: + return x[:y] +def slice4(x: Any, y: Any, z: Any) -> Any: + return x[y:z] +def slice5(x: Any, y: Any, z: Any, zz: Any) -> Any: + return x[y:z:zz] +[file driver.py] +from native import * +assert add(5, 6) == 11 +assert add('x', 'y') == 'xy' +assert subtract(8, 3) == 5 +assert multiply(8, 3) == 24 +assert floor_div(8, 3) == 2 +assert true_div(7, 2) == 3.5 +assert remainder(11, 4) == 3 +assert remainder('%.3d', 5) == '005' +assert remainder('%d-%s', (5, 'xy')) == '5-xy' +assert power(3, 4) == 81 +assert lshift(5, 3) == 40 +assert rshift(41, 3) == 5 +assert num_and(99, 56) == 32 +assert num_xor(99, 56) == 91 +assert num_or(99, 56) == 123 +assert lt('a', 'b') +assert not lt('a', 'a') +assert not lt('b', 'a') +assert not gt('a', 'b') +assert not gt('a', 'a') +assert gt('b', 'a') +assert le('a', 'b') +assert le('a', 'a') +assert not le('b', 'a') +assert not ge('a', 'b') +assert ge('a', 'a') +assert ge('b', 'a') +assert contains('x', 'axb') +assert not contains('X', 'axb') +assert contains('x', {'x', 'y'}) +a = [1, 3, 5] +assert slice1(a) == a +assert slice1(a) is not a +assert slice2(a, 1) == [3, 5] +assert slice3(a, -1) == [1, 3] +assert slice4(a, 1, -1) == [3] +assert slice5(a, 2, 0, -1) == [5, 3] +o1, o2 = object(), object() +assert identity(o1, o1) +assert not identity(o1, o2) +assert not disidentity(o1, o1) +assert disidentity(o1, o2) +assert eq2('xz', 'x' + 'z') +assert not eq2('x', 'y') +assert not not_eq_cond('xz', 'x' + 'z') +assert not_eq_cond('x', 'y') + +[case testGenericMiscOps] +from typing import Any +def neg(x: Any) -> Any: + return -x +def pos(x: Any) -> Any: + return +x +def invert(x: Any) -> Any: + return ~x +def get_item(o: Any, k: Any) -> Any: + return o[k] +def set_item(o: Any, k: Any, v: Any) -> Any: + o[k] = v +[file driver.py] +from native import * +assert neg(6) == -6 +assert pos(6) == 6 +assert invert(6) == -7 +d = {'x': 5} +assert get_item(d, 'x') == 5 +set_item(d, 'y', 6) +assert d['y'] == 6 + +[case testAnyAttributeAndMethodAccess] +from typing import Any, List +class C: + a: int + def m(self, x: int, a: List[int]) -> int: + return self.a + x + a[0] +def get_a(x: Any) -> Any: + return x.a +def set_a(x: Any, y: Any) -> None: + x.a = y +def call_m(x: Any) -> Any: + return x.m(1, [3]) +[file driver.py] +from native import C, get_a, set_a, call_m +class D: + def m(self, x, a): + return self.a + x + a[0] + +c = C() +c.a = 6 +d = D() +d.a = 2 +assert get_a(c) == 6 +assert get_a(d) == 2 +assert call_m(c) == 10 +assert call_m(d) == 6 +set_a(c, 5) +assert c.a == 5 +set_a(d, 4) +assert d.a == 4 +try: + get_a(object()) +except AttributeError: + pass +else: + assert False +try: + call_m(object()) +except AttributeError: + pass +else: + assert False +try: + set_a(object(), 5) +except AttributeError: + pass +else: + assert False + +[case testFloat] +def assign_and_return_float_sum() -> float: + f1 = 1.0 + f2 = 2.0 + f3 = 3.0 + return f1 * f2 + f3 + +def from_int(i: int) -> float: + return float(i) + +def to_int(x: float) -> int: + return int(x) + +def get_complex() -> complex: + return 5.2j + 3.5 + 1j + +[file driver.py] +from native import assign_and_return_float_sum, from_int, to_int, get_complex +sum = 0.0 +for i in range(10): + sum += assign_and_return_float_sum() +assert sum == 50.0 + +assert str(from_int(10)) == '10.0' +assert str(to_int(3.14)) == '3' +assert str(to_int(3)) == '3' +assert get_complex() == 3.5 + 6.2j + +[case testDel] +from typing import List +from testutil import assertRaises + +def printDict(dict) -> None: + l = list(dict.keys()) # type: List[str] + l.sort() + for key in l: + print(key, dict[key]) + print("#########") + +def delList() -> None: + l = [1, 2, 3] + print(tuple(l)) + del l[1] + print(tuple(l)) + +def delDict() -> None: + d = {"one":1, "two":2} + printDict(d) + del d["one"] + printDict(d) + +def delListMultiple() -> None: + l = [1, 2, 3, 4, 5, 6, 7] + print(tuple(l)) + del l[1], l[2], l[3] + print(tuple(l)) + +def delDictMultiple() -> None: + d = {"one":1, "two":2, "three":3, "four":4} + printDict(d) + del d["two"], d["four"] + printDict(d) + +class Dummy(): + __deletable__ = ('x', 'y') + + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + +def delAttribute() -> None: + dummy = Dummy(1, 2) + del dummy.x + with assertRaises(AttributeError): + dummy.x + +def delAttributeMultiple() -> None: + dummy = Dummy(1, 2) + del dummy.x, dummy.y + with assertRaises(AttributeError): + dummy.x + with assertRaises(AttributeError): + dummy.y + +def delLocal(b: bool) -> int: + dummy = 10 + if b: + del dummy + return dummy + +def delLocalLoop() -> None: + # Try deleting a local in a loop to make sure the control flow analysis works + dummy = 1 + for i in range(10): + print(dummy) + dummy *= 2 + if i == 4: + del dummy + +global_var = 10 +del global_var + +[file driver.py] +from native import ( + delList, delDict, delListMultiple, delDictMultiple, delAttribute, + delAttributeMultiple, delLocal, delLocalLoop, +) +import native +from testutil import assertRaises + +delList() +delDict() +delListMultiple() +delDictMultiple() +delAttribute() +delAttributeMultiple() +with assertRaises(AttributeError): + native.global_var +with assertRaises(UnboundLocalError, 'local variable "dummy" referenced before assignment'): + delLocal(True) +assert delLocal(False) == 10 +with assertRaises(UnboundLocalError, 'local variable "dummy" referenced before assignment'): + delLocalLoop() +[out] +(1, 2, 3) +(1, 3) +one 1 +two 2 +######### +two 2 +######### +(1, 2, 3, 4, 5, 6, 7) +(1, 3, 5, 7) +four 4 +one 1 +three 3 +two 2 +######### +one 1 +three 3 +######### +1 +2 +4 +8 +16 diff --git a/mypyc/test-data/run-python312.test b/mypyc/test-data/run-python312.test new file mode 100644 index 000000000000..a5a3f058d1e2 --- /dev/null +++ b/mypyc/test-data/run-python312.test @@ -0,0 +1,223 @@ +[case testPEP695Basics] +from typing import Any, TypeAliasType, cast + +from testutil import assertRaises + +def id[T](x: T) -> T: + return x + +def test_call_generic_function() -> None: + assert id(2) == 2 + assert id('x') == 'x' + +class C[T]: + x: T + + def __init__(self, x: T) -> None: + self.x = x + +class D[T, S]: + x: T + y: S + + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + + def set(self, x: object, y: object) -> None: + self.x = cast(T, x) + self.y = cast(S, y) + +def test_generic_class() -> None: + c = C(5) + assert c.x == 5 + c2 = C[str]('x') + assert c2.x == 'x' + d = D[str, int]('a', 5) + assert d.x == 'a' + assert d.y == 5 + d.set('b', 6) + assert d.x == 'b' + assert d.y == 6 + +def test_generic_class_via_any() -> None: + c_any: Any = C + c = c_any(2) + assert c.x == 2 + c2 = c_any[str]('y') + assert c2.x == 'y' + assert str(c_any[str]) == 'native.C[str]' + + d_any: Any = D + d = d_any(1, 'x') + assert d.x == 1 + assert d.y == 'x' + d2 = d_any[int, str](2, 'y') + assert d2.x == 2 + assert d2.y == 'y' + + with assertRaises(TypeError): + c_any[int, str] + with assertRaises(TypeError): + d_any[int] + +class E[*Ts]: pass + +def test_type_var_tuple() -> None: + e: E[int, str] = E() + e_any: Any = E + assert isinstance(e_any(), E) + assert isinstance(e_any[int](), E) + assert isinstance(e_any[int, str](), E) + +class F[**P]: pass + +def test_param_spec() -> None: + f: F[[int, str]] = F() + f_any: Any = F + assert isinstance(f_any(), F) + assert isinstance(f_any[[int, str]](), F) + +class SubC[S](C[S]): + def __init__(self, x: S) -> None: + super().__init__(x) + +def test_generic_subclass() -> None: + s = SubC(1) + assert s.x == 1 + s2 = SubC[str]('y') + assert s2.x == 'y' + sub_any: Any = SubC + assert sub_any(1).x == 1 + assert sub_any[str]('x').x == 'x' + assert isinstance(s, SubC) + assert isinstance(s, C) + +class SubD[ + T, # Put everything on separate lines + S]( + D[T, + S]): pass + +def test_generic_subclass_two_params() -> None: + s = SubD(3, 'y') + assert s.x == 3 + assert s.y == 'y' + s2 = SubD[str, int]('z', 4) + assert s2.x == 'z' + assert s2.y == 4 + sub_any: Any = SubD + assert sub_any(3, 'y').y == 'y' + assert sub_any[int, str](3, 'y').y == 'y' + assert isinstance(s, SubD) + assert isinstance(s, D) + +class SubE[*Ts](E[*Ts]): pass + +def test_type_var_tuple_subclass() -> None: + sub_any: Any = SubE + assert isinstance(sub_any(), SubE) + assert isinstance(sub_any(), E) + assert isinstance(sub_any[int](), SubE) + assert isinstance(sub_any[int, str](), SubE) + + +class SubF[**P](F[P]): pass + +def test_param_spec_subclass() -> None: + sub_any: Any = SubF + assert isinstance(sub_any(), SubF) + assert isinstance(sub_any(), F) + assert isinstance(sub_any[[int]](), SubF) + assert isinstance(sub_any[[int, str]](), SubF) + +# We test that upper bounds and restricted values can be used, but not that +# they are introspectable + +def bound[T: C](x: T) -> T: + return x + +def test_function_with_upper_bound() -> None: + c = C(1) + assert bound(c) is c + +def restriction[T: (int, str)](x: T) -> T: + return x + +def test_function_with_value_restriction() -> None: + assert restriction(1) == 1 + assert restriction('x') == 'x' + +class Bound[T: C]: + def __init__(self, x: T) -> None: + self.x = x + +def test_class_with_upper_bound() -> None: + c = C(1) + b = Bound(c) + assert b.x is c + b2 = Bound[C](c) + assert b2.x is c + +class Restriction[T: (int, str)]: + def __init__(self, x: T) -> None: + self.x = x + +def test_class_with_value_restriction() -> None: + r = Restriction(1) + assert r.x == 1 + r2 = Restriction[str]('a') + assert r2.x == 'a' + +type A = int + +def test_simple_type_alias() -> None: + assert isinstance(A, TypeAliasType) + assert getattr(A, "__value__") is int + assert str(A) == "A" + +type B = Fwd[int] +Fwd = list + +def test_forward_reference_in_alias() -> None: + assert isinstance(B, TypeAliasType) + assert getattr(B, "__value__") == list[int] + +type R = int | list[R] + +def test_recursive_type_alias() -> None: + assert isinstance(R, TypeAliasType) + assert getattr(R, "__value__") == (int | list[R]) +[typing fixtures/typing-full.pyi] + +[case testPEP695GenericTypeAlias] +from typing import Callable +from types import GenericAlias + +from testutil import assertRaises + +type A[T] = list[T] + +def test_generic_alias() -> None: + assert type(A[str]) is GenericAlias + assert str(A[str]) == "A[str]" + assert str(getattr(A, "__value__")) == "list[T]" + +type B[T, S] = dict[S, T] + +def test_generic_alias_with_two_args() -> None: + assert str(B[str, int]) == "B[str, int]" + assert str(getattr(B, "__value__")) == "dict[S, T]" + +type C[*Ts] = tuple[*Ts] + +def test_type_var_tuple_type_alias() -> None: + assert str(C[int, str]) == "C[int, str]" + assert str(getattr(C, "__value__")) == "tuple[typing.Unpack[Ts]]" + +type D[**P] = Callable[P, int] + +def test_param_spec_type_alias() -> None: + assert str(D[[int, str]]) == "D[[int, str]]" + assert str(getattr(D, "__value__")) == "typing.Callable[P, int]" +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test-data/run-python37.test b/mypyc/test-data/run-python37.test new file mode 100644 index 000000000000..61d428c17a44 --- /dev/null +++ b/mypyc/test-data/run-python37.test @@ -0,0 +1,159 @@ +-- Test cases for Python 3.7 features + +[case testRunDataclass] +import dataclasses +from dataclasses import dataclass, field +from typing import Set, FrozenSet, List, Callable, Any + +@dataclass +class Person1: + age : int + name : str + + def __bool__(self) -> bool: + return self.name == 'robot' + +def testBool(p: Person1) -> bool: + if p: + return True + else: + return False + +@dataclass +class Person1b(Person1): + id: str = '000' + +@dataclass +class Person2: + age : int + name : str = field(default='robot') + +@dataclasses.dataclass +class Person2b: + age : int + name : str = dataclasses.field(default='robot') + +@dataclass(order = True) +class Person3: + age : int = field(default = 6) + friendIDs : List[int] = field(default_factory = list) + + def get_age(self) -> int: + return (self.age) + + def set_age(self, new_age : int) -> None: + self.age = new_age + + def add_friendID(self, fid : int) -> None: + self.friendIDs.append(fid) + + def get_friendIDs(self) -> List[int]: + return self.friendIDs + +def get_next_age(g: Callable[[Any], int]) -> Callable[[Any], int]: + def f(a: Any) -> int: + return g(a) + 1 + return f + +@dataclass +class Person4: + age : int + _name : str = 'Bot' + + @get_next_age + def get_age(self) -> int: + return self.age + + @property + def name(self) -> str: + return self._name + +@dataclass +class Person5: + weight: float + friends: Set[str] = field(default_factory=set) + parents: FrozenSet[str] = frozenset() + +[file other.py] +from native import Person1, Person1b, Person2, Person3, Person4, Person5, testBool +i1 = Person1(age = 5, name = 'robot') +assert i1.age == 5 +assert i1.name == 'robot' +assert testBool(i1) == True +assert testBool(Person1(age = 5, name = 'robo')) == False +i1b = Person1b(age = 5, name = 'robot') +assert i1b.age == 5 +assert i1b.name == 'robot' +assert testBool(i1b) == True +assert testBool(Person1b(age = 5, name = 'robo')) == False +i1c = Person1b(age = 20, name = 'robot', id = 'test') +assert i1c.age == 20 +assert i1c.id == 'test' + +i2 = Person2(age = 5) +assert i2.age == 5 +assert i2.name == 'robot' +i3 = Person2(age = 5, name = 'new_robot') +assert i3.age == 5 +assert i3.name == 'new_robot' +i4 = Person3() +assert i4.age == 6 +assert i4.friendIDs == [] +i5 = Person3(age = 5) +assert i5.age == 5 +assert i5.friendIDs == [] +i6 = Person3(age = 5, friendIDs = [1,2,3]) +assert i6.age == 5 +assert i6.friendIDs == [1,2,3] +assert i6.get_age() == 5 +i6.set_age(10) +assert i6.get_age() == 10 +i6.add_friendID(4) +assert i6.get_friendIDs() == [1,2,3,4] +i7 = Person4(age = 5) +assert i7.get_age() == 6 +i7.age += 3 +assert i7.age == 8 +assert i7.name == 'Bot' +i8 = Person3(age = 1, friendIDs = [1,2]) +i9 = Person3(age = 1, friendIDs = [1,2]) +assert i8 == i9 +i8.age = 2 +assert i8 > i9 + +assert Person1.__annotations__ == {'age': int, 'name': str} +assert Person2.__annotations__ == {'age': int, 'name': str} +assert Person5.__annotations__ == {'weight': float, 'friends': set, + 'parents': frozenset} + +[file driver.py] +import sys + +# Dataclasses introduced in 3.7 +version = sys.version_info[:2] +if version[0] < 3 or version[1] < 7: + exit() + +# Run the tests in both interpreted and compiled mode +import other +import other_interpreted + +# Test for an exceptional cases +from testutil import assertRaises +from native import Person1, Person1b, Person3 +from types import BuiltinMethodType + +with assertRaises(TypeError, "missing 1 required positional argument"): + Person1(0) + +with assertRaises(TypeError, "missing 2 required positional arguments"): + Person1b() + +with assertRaises(TypeError, "int object expected; got str"): + Person1('nope', 'test') + +p = Person1(0, 'test') +with assertRaises(TypeError, "int object expected; got str"): + p.age = 'nope' + +assert isinstance(Person3().get_age, BuiltinMethodType) diff --git a/mypyc/test-data/run-python38.test b/mypyc/test-data/run-python38.test new file mode 100644 index 000000000000..7de43907cb86 --- /dev/null +++ b/mypyc/test-data/run-python38.test @@ -0,0 +1,88 @@ +-- Test cases for Python 3.8 features + +[case testWalrus1] +from typing import Optional + +def foo(x: int) -> Optional[int]: + if x < 0: + return None + return x + +def test(x: int) -> str: + if (n := foo(x)) is not None: + return str(x) + else: + return "" + +[file driver.py] +from native import test + +assert test(10) == "10" +assert test(-1) == "" + + +[case testWalrus2] +from typing import Optional, Tuple, List + +class Node: + def __init__(self, val: int, next: Optional['Node']) -> None: + self.val = val + self.next = next + +def pairs(nobe: Optional[Node]) -> List[Tuple[int, int]]: + if nobe is None: + return [] + l = [] + while next := nobe.next: + l.append((nobe.val, next.val)) + nobe = next + return l + +def make(l: List[int]) -> Optional[Node]: + cur: Optional[Node] = None + for x in reversed(l): + cur = Node(x, cur) + return cur + +[file driver.py] +from native import Node, make, pairs + +assert pairs(make([1,2,3])) == [(1,2), (2,3)] +assert pairs(make([1])) == [] +assert pairs(make([])) == [] + +[case testFStrings] +from datetime import datetime + +def test_fstring_equal_sign() -> None: + today = datetime(year=2017, month=1, day=27) + assert f"{today=:%B %d, %Y}" == 'today=January 27, 2017' # using date format specifier and debugging + + foo = "bar" + assert f"{ foo = }" == " foo = 'bar'" # preserves whitespace + + line = "The mill's closed" + assert f"{line = }" == 'line = "The mill\'s closed"' + assert f"{line = :20}" == "line = The mill's closed " + assert f"{line = !r:20}" == 'line = "The mill\'s closed" ' + +[case testMethodOverrideDefaultPosOnly1] +class Foo: + def f(self, x: int=20, /, *, z: int=10) -> None: + pass + +class Bar(Foo): + def f(self, *args: int, **kwargs: int) -> None: + print("stuff", args, kwargs) + +z: Foo = Bar() +z.f(1, z=50) +z.f() +z.f(1) +z.f(z=50) + +[out] +stuff (1,) {'z': 50} +stuff () {} +stuff (1,) {} +stuff () {'z': 50} diff --git a/mypyc/test-data/run-sets.test b/mypyc/test-data/run-sets.test new file mode 100644 index 000000000000..8d178d03a75b --- /dev/null +++ b/mypyc/test-data/run-sets.test @@ -0,0 +1,150 @@ +# Test cases for sets (compile and run) + +[case testSets] +from typing import Set, List +def instantiateLiteral() -> Set[int]: + return {1, 2, 3, 5, 8} + +def fromIterator() -> List[Set[int]]: + a = set([1, 3, 5]) + b = set((1, 3, 5)) + c = set({1: '1', 3: '3', 5: '5'}) + d = set(x for x in range(1, 6, 2)) + e = set((x for x in range(1, 6, 2))) + return [a, b, c, d, e] + +def fromIterator2() -> Set[int]: + tmp_list = [1, 2, 3, 4, 5] + return set((x + 1) for x in ((y * 10) for y in (z for z in tmp_list if z < 4))) + +def addIncrementing(s : Set[int]) -> None: + for a in [1, 2, 3]: + if a not in s: + s.add(a) + return + +def replaceWith1(s : Set[int]) -> None: + s.clear() + s.add(1) + +def remove1(s : Set[int]) -> None: + s.remove(1) + +def discard1(s: Set[int]) -> None: + s.discard(1) + +def pop(s : Set[int]) -> int: + return s.pop() + +def update(s: Set[int], x: List[int]) -> None: + s.update(x) + +[file driver.py] +from native import instantiateLiteral +from testutil import assertRaises + +val = instantiateLiteral() +assert 1 in val +assert 2 in val +assert 3 in val +assert 5 in val +assert 8 in val +assert len(val) == 5 +assert val == {1, 2, 3, 5, 8} +s = 0 +for i in val: + s += i +assert s == 19 + +from native import fromIterator +sets = fromIterator() +for s in sets: + assert s == {1, 3, 5} + +from native import fromIterator2 +s = fromIterator2() +assert s == {11, 21, 31} + +from native import addIncrementing +s = set() +addIncrementing(s) +assert s == {1} +addIncrementing(s) +assert s == {1, 2} +addIncrementing(s) +assert s == {1, 2, 3} + +from native import replaceWith1 +s = {3, 7, 12} +replaceWith1(s) +assert s == {1} + +from native import remove1 +import traceback +s = {1, 4, 6} +remove1(s) +assert s == {4, 6} +with assertRaises(KeyError, '1'): + remove1(s) + +from native import discard1 +s = {1, 4, 6} +discard1(s) +assert s == {4, 6} +discard1(s) +assert s == {4, 6} + +from native import pop +s = {1, 2, 3} +x = pop(s) +assert len(s) == 2 +assert x in [1, 2, 3] +y = pop(s) +assert len(s) == 1 +assert y in [1, 2, 3] +assert x != y +z = pop(s) +assert len(s) == 0 +assert z in [1, 2, 3] +assert x != z +assert y != z +with assertRaises(KeyError, 'pop from an empty set'): + pop(s) + +from native import update +s = {1, 2, 3} +update(s, [5, 4, 3]) +assert s == {1, 2, 3, 4, 5} + +[case testPrecomputedFrozenSets] +from typing import Any +from typing_extensions import Final + +CONST: Final = "CONST" +non_const = "non_const" + +def main_set(item: Any) -> bool: + return item in {None, False, 1, 2.0, "3", b"4", 5j, (6,), ((7,),), (), CONST} + +def main_negated_set(item: Any) -> bool: + return item not in {None, False, 1, 2.0, "3", b"4", 5j, (6,), ((7,),), (), CONST} + +def non_final_name_set(item: Any) -> bool: + return item in {non_const} + +s = set() +for i in {None, False, 1, 2.0, "3", b"4", 5j, (6,), CONST}: + s.add(i) + +def test_in_set() -> None: + for item in (None, False, 1, 2.0, "3", b"4", 5j, (6,), ((7,),), (), CONST): + assert main_set(item), f"{item!r} should be in set_main" + assert not main_negated_set(item), item + + global non_const + assert non_final_name_set(non_const) + non_const = "updated" + assert non_final_name_set("updated") + +def test_for_set() -> None: + assert not s ^ {None, False, 1, 2.0, "3", b"4", 5j, (6,), CONST}, s diff --git a/mypyc/test-data/run-singledispatch.test b/mypyc/test-data/run-singledispatch.test new file mode 100644 index 000000000000..61e4897c96d6 --- /dev/null +++ b/mypyc/test-data/run-singledispatch.test @@ -0,0 +1,698 @@ +# Test cases related to the functools.singledispatch decorator +# Most of these tests are marked as xfails because mypyc doesn't support singledispatch yet +# (These tests will be re-enabled when mypyc supports singledispatch) + +[case testSpecializedImplementationUsed] +from functools import singledispatch + +@singledispatch +def fun(arg) -> bool: + return False + +@fun.register +def fun_specialized(arg: str) -> bool: + return True + +def test_specialize() -> None: + assert fun('a') + assert not fun(3) + +[case testSubclassesOfExpectedTypeUseSpecialized] +from functools import singledispatch +class A: pass +class B(A): pass + +@singledispatch +def fun(arg) -> bool: + return False + +@fun.register +def fun_specialized(arg: A) -> bool: + return True + +def test_specialize() -> None: + assert fun(B()) + assert fun(A()) + +[case testSuperclassImplementationNotUsedWhenSubclassHasImplementation] +from functools import singledispatch +class A: pass +class B(A): pass + +@singledispatch +def fun(arg) -> bool: + # shouldn't be using this + assert False + +@fun.register +def fun_specialized(arg: A) -> bool: + return False + +@fun.register +def fun_specialized2(arg: B) -> bool: + return True + +def test_specialize() -> None: + assert fun(B()) + assert not fun(A()) + +[case testMultipleUnderscoreFunctionsIsntError] +from functools import singledispatch + +@singledispatch +def fun(arg) -> str: + return 'default' + +@fun.register +def _(arg: str) -> str: + return 'str' + +@fun.register +def _(arg: int) -> str: + return 'int' + +# extra function to make sure all 3 underscore functions aren't treated as one OverloadedFuncDef +def a(b): pass + +@fun.register +def _(arg: list) -> str: + return 'list' + +def test_singledispatch() -> None: + assert fun(0) == 'int' + assert fun('a') == 'str' + assert fun([1, 2]) == 'list' + assert fun({'a': 'b'}) == 'default' + +[case testCanRegisterCompiledClasses] +from functools import singledispatch +class A: pass + +@singledispatch +def fun(arg) -> bool: + return False +@fun.register +def fun_specialized(arg: A) -> bool: + return True + +def test_singledispatch() -> None: + assert fun(A()) + assert not fun(1) + +[case testTypeUsedAsArgumentToRegister] +from functools import singledispatch + +@singledispatch +def fun(arg) -> bool: + return False + +@fun.register(int) +def fun_specialized(arg) -> bool: + return True + +def test_singledispatch() -> None: + assert fun(1) + assert not fun('a') + +[case testUseRegisterAsAFunction] +from functools import singledispatch + +@singledispatch +def fun(arg) -> bool: + return False + +def fun_specialized_impl(arg) -> bool: + return True + +fun.register(int, fun_specialized_impl) + +def test_singledispatch() -> None: + assert fun(0) + assert not fun('a') + +[case testRegisterDoesntChangeFunction] +from functools import singledispatch + +@singledispatch +def fun(arg) -> bool: + return False + +@fun.register(int) +def fun_specialized(arg) -> bool: + return True + +def test_singledispatch() -> None: + assert fun_specialized('a') + +# TODO: turn this into a mypy error +[case testNoneIsntATypeWhenUsedAsArgumentToRegister] +from functools import singledispatch + +@singledispatch +def fun(arg) -> bool: + return False + +try: + @fun.register + def fun_specialized(arg: None) -> bool: + return True +except TypeError: + pass + +[case testRegisteringTheSameFunctionSeveralTimes] +from functools import singledispatch + +@singledispatch +def fun(arg) -> bool: + return False + +@fun.register(int) +@fun.register(str) +def fun_specialized(arg) -> bool: + return True + +def test_singledispatch() -> None: + assert fun(0) + assert fun('a') + assert not fun([1, 2]) + +[case testTypeIsAnABC] +from functools import singledispatch +from collections.abc import Mapping + +@singledispatch +def fun(arg) -> bool: + return False + +@fun.register +def fun_specialized(arg: Mapping) -> bool: + return True + +def test_singledispatch() -> None: + assert not fun(1) + assert fun({'a': 'b'}) + +[case testSingleDispatchMethod-xfail] +from functools import singledispatchmethod +class A: + @singledispatchmethod + def fun(self, arg) -> str: + return 'default' + + @fun.register + def fun_int(self, arg: int) -> str: + return 'int' + + @fun.register + def fun_str(self, arg: str) -> str: + return 'str' + +def test_singledispatchmethod() -> None: + x = A() + assert x.fun(5) == 'int' + assert x.fun('a') == 'str' + assert x.fun([1, 2]) == 'default' + +[case testSingleDispatchMethodWithOtherDecorator-xfail] +from functools import singledispatchmethod +class A: + @singledispatchmethod + @staticmethod + def fun(arg) -> str: + return 'default' + + @fun.register + @staticmethod + def fun_int(arg: int) -> str: + return 'int' + + @fun.register + @staticmethod + def fun_str(arg: str) -> str: + return 'str' + +def test_singledispatchmethod() -> None: + x = A() + assert x.fun(5) == 'int' + assert x.fun('a') == 'str' + assert x.fun([1, 2]) == 'default' + +[case testSingledispatchTreeSumAndEqual] +from functools import singledispatch + +class Tree: + pass +class Leaf(Tree): + pass +class Node(Tree): + def __init__(self, value: int, left: Tree, right: Tree) -> None: + self.value = value + self.left = left + self.right = right + +@singledispatch +def calc_sum(x: Tree) -> int: + raise TypeError('invalid type for x') + +@calc_sum.register +def _(x: Leaf) -> int: + return 0 + +@calc_sum.register +def _(x: Node) -> int: + return x.value + calc_sum(x.left) + calc_sum(x.right) + +@singledispatch +def equal(to_compare: Tree, known: Tree) -> bool: + raise TypeError('invalid type for x') + +@equal.register +def _(to_compare: Leaf, known: Tree) -> bool: + return isinstance(known, Leaf) + +@equal.register +def _(to_compare: Node, known: Tree) -> bool: + if isinstance(known, Node): + if to_compare.value != known.value: + return False + else: + return equal(to_compare.left, known.left) and equal(to_compare.right, known.right) + return False + +def build(n: int) -> Tree: + if n == 0: + return Leaf() + return Node(n, build(n - 1), build(n - 1)) + +def test_sum_and_equal(): + tree = build(5) + tree2 = build(5) + tree2.right.right.right.value = 10 + assert calc_sum(tree) == 57 + assert calc_sum(tree2) == 65 + assert equal(tree, tree) + assert not equal(tree, tree2) + tree3 = build(4) + assert not equal(tree, tree3) + +[case testSimulateMypySingledispatch] +from functools import singledispatch +from mypy_extensions import trait +from typing import Iterator, Union, TypeVar, Any, List, Type +# based on use of singledispatch in stubtest.py +class Error: + def __init__(self, msg: str) -> None: + self.msg = msg + +@trait +class Node: pass + +class MypyFile(Node): pass +class TypeInfo(Node): pass + + +@trait +class SymbolNode(Node): pass +@trait +class Expression(Node): pass +class TypeVarLikeExpr(SymbolNode, Expression): pass +class TypeVarExpr(TypeVarLikeExpr): pass +class TypeAlias(SymbolNode): pass + +class Missing: pass +MISSING = Missing() + +T = TypeVar("T") + +MaybeMissing = Union[T, Missing] + +@singledispatch +def verify(stub: Node, a: MaybeMissing[Any], b: List[str]) -> Iterator[Error]: + yield Error('unknown node type') + +@verify.register(MypyFile) +def verify_mypyfile(stub: MypyFile, a: MaybeMissing[int], b: List[str]) -> Iterator[Error]: + if isinstance(a, Missing): + yield Error("shouldn't be missing") + return + if not isinstance(a, int): + # this check should be unnecessary because of the type signature and the previous check, + # but stubtest.py has this check + yield Error("should be an int") + return + yield from verify(TypeInfo(), str, ['abc', 'def']) + +@verify.register(TypeInfo) +def verify_typeinfo(stub: TypeInfo, a: MaybeMissing[Type[Any]], b: List[str]) -> Iterator[Error]: + yield Error('in TypeInfo') + yield Error('hello') + +@verify.register(TypeVarExpr) +def verify_typevarexpr(stub: TypeVarExpr, a: MaybeMissing[Any], b: List[str]) -> Iterator[Error]: + if False: + yield None + +def verify_list(stub, a, b) -> List[str]: + """Helper function that converts iterator of errors to list of messages""" + return list(err.msg for err in verify(stub, a, b)) + +def test_verify() -> None: + assert verify_list(TypeAlias(), 'a', ['a', 'b']) == ['unknown node type'] + assert verify_list(MypyFile(), MISSING, ['a', 'b']) == ["shouldn't be missing"] + assert verify_list(MypyFile(), 5, ['a', 'b']) == ['in TypeInfo', 'hello'] + assert verify_list(TypeInfo(), str, ['a', 'b']) == ['in TypeInfo', 'hello'] + assert verify_list(TypeVarExpr(), 'a', ['x', 'y']) == [] + + +[case testArgsInRegisteredImplNamedDifferentlyFromMainFunction] +from functools import singledispatch + +@singledispatch +def f(a) -> bool: + return False + +@f.register +def g(b: int) -> bool: + return True + +def test_singledispatch(): + assert f(5) + assert not f('a') + +[case testKeywordArguments] +from functools import singledispatch + +@singledispatch +def f(arg, *, kwarg: int = 0) -> int: + return kwarg + 10 + +@f.register +def g(arg: int, *, kwarg: int = 5) -> int: + return kwarg - 10 + +def test_keywords(): + assert f('a') == 10 + assert f('a', kwarg=3) == 13 + assert f('a', kwarg=7) == 17 + + assert f(1) == -5 + assert f(1, kwarg=4) == -6 + assert f(1, kwarg=6) == -4 + +[case testGeneratorAndMultipleTypesOfIterable] +from functools import singledispatch +from typing import * + +@singledispatch +def f(arg: Any) -> Iterable[int]: + yield 1 + +@f.register +def g(arg: str) -> Iterable[int]: + return [0] + +def test_iterables(): + assert f(1) != [1] + assert list(f(1)) == [1] + assert f('a') == [0] + +[case testRegisterUsedAtSameTimeAsOtherDecorators] +from functools import singledispatch +from typing import TypeVar + +class A: pass +class B: pass + +T = TypeVar('T') + +def decorator(f: T) -> T: + return f + +@singledispatch +def f(arg) -> int: + return 0 + +@f.register +@decorator +def h(arg: str) -> int: + return 2 + +def test_singledispatch(): + assert f(1) == 0 + assert f('a') == 2 + +[case testDecoratorModifiesFunction] +from functools import singledispatch +from typing import Callable, Any + +class A: pass + +def decorator(f: Callable[[Any], int]) -> Callable[[Any], int]: + def wrapper(x) -> int: + return f(x) * 7 + return wrapper + +@singledispatch +def f(arg) -> int: + return 10 + +@f.register +@decorator +def h(arg: str) -> int: + return 5 + + +def test_singledispatch(): + assert f('a') == 35 + assert f(A()) == 10 + +[case testMoreSpecificTypeBeforeLessSpecificType] +from functools import singledispatch +class A: pass +class B(A): pass + +@singledispatch +def f(arg) -> str: + return 'default' + +@f.register +def g(arg: B) -> str: + return 'b' + +@f.register +def h(arg: A) -> str: + return 'a' + +def test_singledispatch(): + assert f(B()) == 'b' + assert f(A()) == 'a' + assert f(5) == 'default' + +[case testMultipleRelatedClassesBeingRegistered] +from functools import singledispatch + +class A: pass +class B(A): pass +class C(B): pass + +@singledispatch +def f(arg) -> str: return 'default' + +@f.register +def _(arg: A) -> str: return 'a' + +@f.register +def _(arg: C) -> str: return 'c' + +@f.register +def _(arg: B) -> str: return 'b' + +def test_singledispatch(): + assert f(A()) == 'a' + assert f(B()) == 'b' + assert f(C()) == 'c' + assert f(1) == 'default' + +[case testRegisteredImplementationsInDifferentFiles] +from other_a import f, A, B, C +@f.register +def a(arg: A) -> int: + return 2 + +@f.register +def _(arg: C) -> int: + return 3 + +def test_singledispatch(): + assert f(B()) == 1 + assert f(A()) == 2 + assert f(C()) == 3 + assert f(1) == 0 + +[file other_a.py] +from functools import singledispatch + +class A: pass +class B(A): pass +class C(B): pass + +@singledispatch +def f(arg) -> int: + return 0 + +@f.register +def g(arg: B) -> int: + return 1 + +[case testOrderCanOnlyBeDeterminedFromMRONotIsinstanceChecks] +from mypy_extensions import trait +from functools import singledispatch + +@trait +class A: pass +@trait +class B: pass +class AB(A, B): pass +class BA(B, A): pass + +@singledispatch +def f(arg) -> str: + return "default" + pass + +@f.register +def fa(arg: A) -> str: + return "a" + +@f.register +def fb(arg: B) -> str: + return "b" + +def test_singledispatch(): + assert f(AB()) == "a" + assert f(BA()) == "b" + +[case testCallingFunctionBeforeAllImplementationsRegistered] +from functools import singledispatch + +class A: pass +class B(A): pass + +@singledispatch +def f(arg) -> str: + return 'default' + +assert f(A()) == 'default' +assert f(B()) == 'default' +assert f(1) == 'default' + +@f.register +def g(arg: A) -> str: + return 'a' + +assert f(A()) == 'a' +assert f(B()) == 'a' +assert f(1) == 'default' + +@f.register +def _(arg: B) -> str: + return 'b' + +assert f(A()) == 'a' +assert f(B()) == 'b' +assert f(1) == 'default' + + +[case testDynamicallyRegisteringFunctionFromInterpretedCode] +from functools import singledispatch + +class A: pass +class B(A): pass +class C(B): pass +class D(C): pass + +@singledispatch +def f(arg) -> str: + return "default" + +@f.register +def _(arg: B) -> str: + return 'b' + +[file register_impl.py] +from native import f, A, B, C + +@f.register(A) +def a(arg) -> str: + return 'a' + +@f.register +def c(arg: C) -> str: + return 'c' + +[file driver.py] +from native import f, A, B, C +from register_impl import a, c +# We need a custom driver here because register_impl has to be run before we test this (so that the +# additional implementations are registered) +assert f(C()) == 'c' +assert f(A()) == 'a' +assert f(B()) == 'b' +assert a(C()) == 'a' +assert c(A()) == 'c' + +[case testMalformedDynamicRegisterCall] +from functools import singledispatch + +@singledispatch +def f(arg) -> None: + pass +[file register.py] +from native import f +from testutil import assertRaises + +with assertRaises(TypeError, 'Invalid first argument to `register()`'): + @f.register + def _(): + pass + +[file driver.py] +import register + +[case testCacheClearedWhenNewFunctionRegistered] +from functools import singledispatch + +@singledispatch +def f(arg) -> str: + return 'default' + +[file register.py] +from native import f +class A: pass +class B: pass +class C: pass + +# annotated function +assert f(A()) == 'default' +@f.register +def _(arg: A) -> str: + return 'a' +assert f(A()) == 'a' + +# type passed as argument +assert f(B()) == 'default' +@f.register(B) +def _(arg: B) -> str: + return 'b' +assert f(B()) == 'b' + +# 2 argument form +assert f(C()) == 'default' +def c(arg) -> str: + return 'c' +f.register(C, c) +assert f(C()) == 'c' + + +[file driver.py] +import register diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test new file mode 100644 index 000000000000..1caddce9848d --- /dev/null +++ b/mypyc/test-data/run-strings.test @@ -0,0 +1,657 @@ +# Test cases for strings (compile and run) + +[case testStrBasics] +from typing import Tuple +def f() -> str: + return 'some string' +def g() -> str: + return 'some\a \v \t \x7f " \n \0string 🐍' +def tostr(x: int) -> str: + return str(x) +def booltostr(x: bool) -> str: + return str(x) +def concat(x: str, y: str) -> str: + return x + y +def eq(x: str) -> int: + if x == 'foo': + return 0 + elif x != 'bar': + return 1 + return 2 +def match(x: str, y: str) -> Tuple[bool, bool]: + return (x.startswith(y), x.endswith(y)) + +[file driver.py] +from native import f, g, tostr, booltostr, concat, eq, match +import sys + +assert f() == 'some string' +assert f() is sys.intern('some string') +assert g() == 'some\a \v \t \x7f " \n \0string 🐍' +assert tostr(57) == '57' +assert concat('foo', 'bar') == 'foobar' +assert booltostr(True) == 'True' +assert booltostr(False) == 'False' +assert eq('foo') == 0 +assert eq('zar') == 1 +assert eq('bar') == 2 + +assert int(tostr(0)) == 0 +assert int(tostr(20)) == 20 +assert match('', '') == (True, True) +assert match('abc', '') == (True, True) +assert match('abc', 'a') == (True, False) +assert match('abc', 'c') == (False, True) +assert match('', 'abc') == (False, False) + +[case testStringOps] +from typing import List, Optional + +def do_split(s: str, sep: Optional[str] = None, max_split: Optional[int] = None) -> List[str]: + if sep is not None: + if max_split is not None: + return s.split(sep, max_split) + else: + return s.split(sep) + return s.split() + +ss = "abc abcd abcde abcdef" + +def test_split() -> None: + assert do_split(ss) == ["abc", "abcd", "abcde", "abcdef"] + assert do_split(ss, " ") == ["abc", "abcd", "abcde", "abcdef"] + assert do_split(ss, "-") == ["abc abcd abcde abcdef"] + assert do_split(ss, " ", -1) == ["abc", "abcd", "abcde", "abcdef"] + assert do_split(ss, " ", 0) == ["abc abcd abcde abcdef"] + assert do_split(ss, " ", 1) == ["abc", "abcd abcde abcdef"] + assert do_split(ss, " ", 2) == ["abc", "abcd", "abcde abcdef"] + +def getitem(s: str, index: int) -> str: + return s[index] + +from testutil import assertRaises + +s = "abc" + +def test_getitem() -> None: + assert getitem(s, 0) == "a" + assert getitem(s, 1) == "b" + assert getitem(s, 2) == "c" + assert getitem(s, -3) == "a" + assert getitem(s, -2) == "b" + assert getitem(s, -1) == "c" + with assertRaises(IndexError, "string index out of range"): + getitem(s, 4) + with assertRaises(IndexError, "string index out of range"): + getitem(s, -4) + +def str_to_int(s: str, base: Optional[int] = None) -> int: + if base: + return int(s, base) + else: + return int(s) + +def test_str_to_int() -> None: + assert str_to_int("1") == 1 + assert str_to_int("10") == 10 + assert str_to_int("a", 16) == 10 + assert str_to_int("1a", 16) == 26 + with assertRaises(ValueError, "invalid literal for int() with base 10: 'xyz'"): + str_to_int("xyz") + +def test_slicing() -> None: + # Use dummy adds to avoid constant folding + zero = int() + two = zero + 2 + s = "foobar" + str() + assert s[two:] == "obar" + assert s[:two] == "fo" + assert s[two:-two] == "ob" + assert s[two:two] == "" + assert s[two:two + 1] == "o" + assert s[-two:] == "ar" + assert s[:-two] == "foob" + assert s[:] == "foobar" + assert s[two:333] == "obar" + assert s[333:two] == "" + assert s[two:-333] == "" + assert s[-333:two] == "fo" + big_int: int = 1000 * 1000 * 1000 * 1000 * 1000 * 1000 * 1000 + assert s[1:big_int] == "oobar" + assert s[big_int:] == "" + assert s[-big_int:-1] == "fooba" + +def test_str_replace() -> None: + a = "foofoofoo" + assert a.replace("foo", "bar") == "barbarbar" + assert a.replace("foo", "bar", -1) == "barbarbar" + assert a.replace("foo", "bar", 1) == "barfoofoo" + assert a.replace("foo", "bar", 4) == "barbarbar" + assert a.replace("aaa", "bar") == "foofoofoo" + assert a.replace("ofo", "xyzw") == "foxyzwxyzwo" + +def is_true(x: str) -> bool: + if x: + return True + else: + return False + +def is_true2(x: str) -> bool: + return bool(x) + +def is_false(x: str) -> bool: + if not x: + return True + else: + return False + +def test_str_to_bool() -> None: + assert is_false('') + assert not is_true('') + assert not is_true2('') + for x in 'a', 'foo', 'bar', 'some string': + assert is_true(x) + assert is_true2(x) + assert not is_false(x) + +def test_str_min_max() -> None: + x: str = 'aaa' + y: str = 'bbb' + z: str = 'aa' + assert min(x, y) == 'aaa' + assert min(x, z) == 'aa' + assert max(x, y) == 'bbb' + assert max(x, z) == 'aaa' + +[case testStringFormattingCStyle] +[typing fixtures/typing-full.pyi] +from typing import Tuple + +var = 'mypyc' +num = 20 + +def test_basics() -> None: + assert 'Hello %s, this is a test' % var == "Hello mypyc, this is a test" + assert 'Hello %s %d, this is a test' % (var, num) == "Hello mypyc 20, this is a test" + t: Tuple[str, int] = (var, num) + assert 'Hello %s %d, this is a test' % t == "Hello mypyc 20, this is a test" + + large_num = 2**65 + assert 'number: %d' % large_num == 'number: 36893488147419103232' + neg_num = -3 + assert 'negative integer: %d' % neg_num == 'negative integer: -3' + assert 'negative integer: %d' % (-large_num) == 'negative integer: -36893488147419103232' + + bool_var1 = True + bool_var2 = False + assert 'bool: %s, %s' % (bool_var1, bool_var2) == 'bool: True, False' + + float_num = 123.4 + assert '%f' % float_num == '123.400000' + assert '%.2f' % float_num == '123.40' + assert '%.5f' % float_num == '123.40000' + assert '%10.2f' % float_num == ' 123.40' + assert '%10.5f' % float_num == ' 123.40000' + assert '%010.5f' % float_num == '0123.40000' + assert '%015.5f' % float_num == '000000123.40000' + assert '%e' % float_num == '1.234000e+02' + large_float = 1.23e30 + large_float2 = 1234123412341234123400000000000000000 + small_float = 1.23e-20 + assert '%f, %f, %f' % (small_float, large_float, large_float2) == \ + '0.000000, 1229999999999999959718843908096.000000, 1234123412341234169005079998930878464.000000' + assert '%s, %s, %s' % (small_float, large_float, large_float2) == \ + '1.23e-20, 1.23e+30, 1234123412341234123400000000000000000' + assert '%d, %d, %d' % (small_float, large_float, large_float2) == \ + '0, 1229999999999999959718843908096, 1234123412341234123400000000000000000' + + nan_num = float('nan') + inf_num = float('inf') + assert '%s, %s' % (nan_num, inf_num) == 'nan, inf' + assert '%f, %f' % (nan_num, inf_num) == 'nan, inf' + +[case testFStrings] +import decimal +from datetime import datetime + +var = 'mypyc' +num = 20 + +def test_fstring_basics() -> None: + assert f'Hello {var}, this is a test' == "Hello mypyc, this is a test" + + large_num = 2**65 + assert f'number: {large_num}' == 'number: 36893488147419103232' + neg_num = -3 + assert f'negative integer: {neg_num}' == 'negative integer: -3' + assert f'negative integer: {-large_num}' == 'negative integer: -36893488147419103232' + + bool_var1 = True + bool_var2 = False + assert f'bool: {bool_var1}, {bool_var2}' == 'bool: True, False' + + x = bytes([1, 2, 3, 4]) + # assert f'bytes: {x}' == "bytes: b'\\x01\\x02\\x03\\x04'" + # error: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes + + float_num = 123.4 + assert f'{float_num}' == '123.4' + assert f'{float_num:.2f}' == '123.40' + assert f'{float_num:.5f}' == '123.40000' + assert f'{float_num:>10.2f}' == ' 123.40' + assert f'{float_num:>10.5f}' == ' 123.40000' + assert f'{float_num:>010.5f}' == '0123.40000' + assert f'{float_num:>015.5f}' == '000000123.40000' + assert f'{float_num:e}' == '1.234000e+02' + + large_float = 1.23e30 + large_float2 = 1234123412341234123400000000000000000 + small_float = 1.23e-20 + assert f'{small_float}, {large_float}, {large_float2}' == '1.23e-20, 1.23e+30, 1234123412341234123400000000000000000' + nan_num = float('nan') + inf_num = float('inf') + assert f'{nan_num}, {inf_num}' == 'nan, inf' + +# F-strings would be translated into ''.join[string literals, format method call, ...] in mypy AST. +# Currently we are using a str.join specializer for f-string speed up. We might not cover all cases +# and the rest ones should fall back to a normal str.join method call. +# TODO: Once we have a new pipeline for f-strings, this test case can be moved to testStringOps. +def test_str_join() -> None: + var = 'mypyc' + num = 10 + assert ''.join(['a', 'b', '{}'.format(var), 'c']) == 'abmypycc' + assert ''.join(['a', 'b', '{:{}}'.format(var, ''), 'c']) == 'abmypycc' + assert ''.join(['a', 'b', '{:{}}'.format(var, '>10'), 'c']) == 'ab mypycc' + assert ''.join(['a', 'b', '{:{}}'.format(var, '>{}'.format(num)), 'c']) == 'ab mypycc' + assert var.join(['a', '{:{}}'.format(var, ''), 'b']) == 'amypycmypycmypycb' + assert ','.join(['a', '{:{}}'.format(var, ''), 'b']) == 'a,mypyc,b' + assert ''.join(['x', var]) == 'xmypyc' + +class A: + def __init__(self, name, age): + self.name = name + self.age = age + + def __repr__(self): + return f'{self.name} is {self.age} years old.' + +def test_fstring_datatype() -> None: + u = A('John Doe', 14) + assert f'{u}' == 'John Doe is 14 years old.' + d = {'name': 'John Doe', 'age': 14} + assert f'{d}' == "{'name': 'John Doe', 'age': 14}" + +def test_fstring_escape() -> None: + assert f"{'inside'}" == 'inside' + assert f'{"inside"}' == 'inside' + assert f"""inside""" == 'inside' + assert f'''inside''' == 'inside' + assert f"\"{'inside'}\"" == '"inside"' + assert f'\'{"inside"}\'' == "'inside'" + + assert f'{{10}}' == '{10}' + assert f'{{10 + 10}}' == '{10 + 10}' + assert f'{{{10 + 10}}}' == '{20}' + assert f'{{{{10 + 10}}}}' == '{{10 + 10}}' + +def test_fstring_conversion() -> None: + assert f'Hello {var!r}' == "Hello 'mypyc'" + # repr() is equivalent to !r + assert f'Hello {repr(var)}' == "Hello 'mypyc'" + + assert f'Hello {var!a}' == "Hello 'mypyc'" + # ascii() is equivalent to !a + assert f'Hello {ascii(var)}' == "Hello 'mypyc'" + + tmp_str = """this + is a new line.""" + assert f'Test: {tmp_str!a}' == "Test: 'this\\n is a new line.'" + + s = 'test: āĀēĒčČ..šŠūŪžŽ' + assert f'{s}' == 'test: āĀēĒčČ..šŠūŪžŽ' + assert f'{s!a}' == "'test: \\u0101\\u0100\\u0113\\u0112\\u010d\\u010c..\\u0161\\u0160\\u016b\\u016a\\u017e\\u017d'" + + assert f'Hello {var!s}' == 'Hello mypyc' + assert f'Hello {num!s}' == 'Hello 20' + +def test_fstring_align() -> None: + assert f'Hello {var:>20}' == "Hello mypyc" + assert f'Hello {var!r:>20}' == "Hello 'mypyc'" + assert f'Hello {var:>{num}}' == "Hello mypyc" + assert f'Hello {var!r:>{num}}' == "Hello 'mypyc'" + +def test_fstring_multi() -> None: + assert f'Hello {var}, hello again {var}' == "Hello mypyc, hello again mypyc" + a = 'py' + s = f'my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}my{a}' + assert s == 'mypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypymypy' + +def test_fstring_python_doc() -> None: + name = 'Fred' + assert f"He said his name is {name!r}." == "He said his name is 'Fred'." + assert f"He said his name is {repr(name)}." == "He said his name is 'Fred'." + + width = 10 + precision = 4 + value = decimal.Decimal('12.34567') + assert f'result: {value:{width}.{precision}}' == 'result: 12.35' # nested field + + today = datetime(year=2017, month=1, day=27) + assert f'{today:%B %d, %Y}' == 'January 27, 2017' # using date format specifier + + number = 1024 + assert f'{number:#0x}' == '0x400' # using integer format specifier + +[case testStringFormatMethod] +from typing import Tuple + +def test_format_method_basics() -> None: + x = str() + assert 'x{}'.format(x) == 'x' + assert 'ā{}'.format(x) == 'ā' + assert '😀{}'.format(x) == '😀' + assert ''.format() == '' + assert 'abc'.format() == 'abc' + assert '{}{}'.format(1, 2) == '12' + + name = 'Eric' + age = 14 + assert "My name is {name}, I'm {age}.".format(name=name, age=age) == "My name is Eric, I'm 14." + assert "My name is {A}, I'm {B}.".format(A=name, B=age) == "My name is Eric, I'm 14." + assert "My name is {}, I'm {B}.".format(name, B=age) == "My name is Eric, I'm 14." + + bool_var1 = True + bool_var2 = False + assert 'bool: {}, {}'.format(bool_var1, bool_var2) == 'bool: True, False' + +def test_format_method_empty_braces() -> None: + name = 'Eric' + age = 14 + + assert 'Hello, {}!'.format(name) == 'Hello, Eric!' + assert '{}'.format(name) == 'Eric' + assert '{}! Hi!'.format(name) == 'Eric! Hi!' + assert '{}, Hi, {}'.format(name, name) == 'Eric, Hi, Eric' + assert 'Hi! {}'.format(name) == 'Hi! Eric' + assert "Hi, I'm {}. I'm {}.".format(name, age) == "Hi, I'm Eric. I'm 14." + + assert '{{}}'.format() == '{}' + assert '{{{{}}}}'.format() == '{{}}' + assert '{{}}{}'.format(name) == '{}Eric' + assert 'Hi! {{{}}}'.format(name) == 'Hi! {Eric}' + assert 'Hi! {{ {}'.format(name) == 'Hi! { Eric' + assert 'Hi! {{ {} }}}}'.format(name) == 'Hi! { Eric }}' + +def test_format_method_numbers() -> None: + s = 'int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}'.format(-233) + assert s == 'int: -233; hex: -e9; oct: -351; bin: -11101001' + num = 2**65 + s = 'int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}'.format(num) + assert s == 'int: 36893488147419103232; hex: 20000000000000000; oct: 4000000000000000000000; bin: 100000000000000000000000000000000000000000000000000000000000000000' + s = 'int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}'.format(-num) + assert s == 'int: -36893488147419103232; hex: -20000000000000000; oct: -4000000000000000000000; bin: -100000000000000000000000000000000000000000000000000000000000000000' + + large_num = 2**65 + assert 'number: {}'.format(large_num) == 'number: 36893488147419103232' + neg_num = -3 + assert 'negative integer: {}'.format(neg_num) == 'negative integer: -3' + assert 'negative integer: {}'.format(-large_num) == 'negative integer: -36893488147419103232' + + large_float = 1.23e30 + large_float2 = 1234123412341234123400000000000000000 + small_float = 1.23e-20 + assert '{}, {}, {}'.format(small_float, large_float, large_float2) == '1.23e-20, 1.23e+30, 1234123412341234123400000000000000000' + nan_num = float('nan') + inf_num = float('inf') + assert '{}, {}'.format(nan_num, inf_num) == 'nan, inf' + +def format_args(*args: int) -> str: + return 'x{}y{}'.format(*args) +def format_kwargs(**kwargs: int) -> str: + return 'c{x}d{y}'.format(**kwargs) +def format_args_self(*args: int) -> str: + return '{}'.format(args) +def format_kwargs_self(**kwargs: int) -> str: + return '{}'.format(kwargs) + +def test_format_method_args() -> None: + assert format_args(10, 2) == 'x10y2' + assert format_args_self(10, 2) == '(10, 2)' + assert format_kwargs(x=10, y=2) == 'c10d2' + assert format_kwargs(x=10, y=2, z=1) == 'c10d2' + assert format_kwargs_self(x=10, y=2, z=1) == "{'x': 10, 'y': 2, 'z': 1}" + +def test_format_method_different_kind() -> None: + s1 = "Literal['😀']" + assert 'Revealed type is {}'.format(s1) == "Revealed type is Literal['😀']" + s2 = "Revealed type is" + assert "{} Literal['😀']".format(s2) == "Revealed type is Literal['😀']" + s3 = "测试:" + assert "{}{} {}".format(s3, s2, s1) == "测试:Revealed type is Literal['😀']" + assert "Test: {}{}".format(s3, s1) == "Test: 测试:Literal['😀']" + assert "Test: {}{}".format(s3, s2) == "Test: 测试:Revealed type is" + +def test_format_method_nested() -> None: + var = 'mypyc' + num = 10 + assert '{:{}}'.format(var, '') == 'mypyc' + assert '{:{}}'.format(var, '>10') == ' mypyc' + assert '{:{}}'.format(var, '>{}'.format(num)) == ' mypyc' + +class Point: + def __init__(self, x, y): + self.x, self.y = x, y + def __str__(self): + return 'Point({self.x}, {self.y})'.format(self=self) + +# Format examples from Python doc +# https://docs.python.org/3/library/string.html#formatexamples +def test_format_method_python_doc() -> None: + # Accessing arguments by position: + assert '{0}, {1}, {2}'.format('a', 'b', 'c') == 'a, b, c' + assert '{}, {}, {}'.format('a', 'b', 'c') == 'a, b, c' + assert '{2}, {1}, {0}'.format('a', 'b', 'c') == 'c, b, a' + assert '{2}, {1}, {0}'.format(*'abc') == 'c, b, a' # unpacking argument sequence + # assert '{0}{1}{0}'.format('abra', 'cad') = 'abracadabra' # arguments' indices can be repeated + + # Accessing arguments by name: + s = 'Coordinates: {latitude}, {longitude}'.format(latitude='37.24N', longitude='-115.81W') + assert s == 'Coordinates: 37.24N, -115.81W' + coord = {'latitude': '37.24N', 'longitude': '-115.81W'} + assert 'Coordinates: {latitude}, {longitude}'.format(**coord) == 'Coordinates: 37.24N, -115.81W' + + # Accessing arguments’ attributes: + assert str(Point(4, 2)) == 'Point(4, 2)' + + # Accessing arguments’ items: + coord2 = (3, 5) + assert 'X: {0[0]}; Y: {0[1]}'.format(coord2) == 'X: 3; Y: 5' + + # Replacing %s and %r: + s = "repr() shows quotes: {!r}; str() doesn't: {!s}".format('test1', 'test2') + assert s == "repr() shows quotes: 'test1'; str() doesn't: test2" + + # Aligning the text and specifying a width: + assert '{:<30}'.format('left aligned') == 'left aligned ' + assert '{:>30}'.format('right aligned') == ' right aligned' + assert '{:^30}'.format('centered') == ' centered ' + assert '{:*^30}'.format('centered') == '***********centered***********' # use '*' as a fill char + + # Replacing %+f, %-f, and % f and specifying a sign: + assert '{:+f}; {:+f}'.format(3.14, -3.14) == '+3.140000; -3.140000' # show it always + assert '{: f}; {: f}'.format(3.14, -3.14) == ' 3.140000; -3.140000' # show a space for positive numbers + assert '{:-f}; {:-f}'.format(3.14, -3.14) == '3.140000; -3.140000' # show only the minus -- same as '{:f}; {:f}' + + # Replacing %x and %o and converting the value to different bases: + s = 'int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}'.format(42) # format also supports binary numbers + assert s == 'int: 42; hex: 2a; oct: 52; bin: 101010' + s = 'int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}'.format(42) # with 0x, 0o, or 0b as prefix: + assert s == 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010' + + # Using the comma as a thousands separator: + assert '{:,}'.format(1234567890) == '1,234,567,890' + + # Expressing a percentage: + points = 19.0 + total = 22.0 + assert 'Correct answers: {:.2%}'.format(points/total) == 'Correct answers: 86.36%' + + # Using type-specific formatting: + import datetime + d = datetime.datetime(2010, 7, 4, 12, 15, 58) + assert '{:%Y-%m-%d %H:%M:%S}'.format(d) == '2010-07-04 12:15:58' + + # Nesting arguments and more complex examples: + tmp_strs = [] + for align, text in zip('<^>', ['left', 'center', 'right']): + tmp_strs.append('{0:{fill}{align}16}'.format(text, fill=align, align=align)) + assert tmp_strs == ['left<<<<<<<<<<<<', '^^^^^center^^^^^', '>>>>>>>>>>>right'] + + octets = [192, 168, 0, 1] + assert '{:02X}{:02X}{:02X}{:02X}'.format(*octets) == 'C0A80001' + + width = 5 + tmp_strs = [] + for num in range(5,12): + tmp_str = '' + for base in 'dXob': + tmp_str += ('{0:{width}{base}}'.format(num, base=base, width=width)) + tmp_strs.append(tmp_str) + assert tmp_strs == [' 5 5 5 101',\ + ' 6 6 6 110',\ + ' 7 7 7 111',\ + ' 8 8 10 1000',\ + ' 9 9 11 1001',\ + ' 10 A 12 1010',\ + ' 11 B 13 1011'] + +[case testChr] +# Some test cases are from https://docs.python.org/3/howto/unicode.html + +def try_invalid(x: int) -> bool: + try: + chr(x + int()) + return False + except ValueError: + return True + +def test_chr() -> None: + assert chr(57344) == '\ue000' + assert chr(0) == '\x00' + assert chr(65) == 'A' + assert chr(150) == '\x96' + try: + chr(-1) + assert False + except ValueError: + pass + try: + chr(1114112) + assert False + except ValueError: + pass + assert chr(1114111) == '\U0010ffff' + x = 0 + assert chr(x + int()) == '\x00' + x = 100 + assert chr(x + int()) == 'd' + x = 150 + assert chr(x + int()) == '\x96' + x = 257 + assert chr(x + int()) == 'ā' + x = 65537 + assert chr(x + int()) == '𐀁' + assert try_invalid(-1) + assert try_invalid(1114112) + +[case testOrd] +from testutil import assertRaises + +def test_ord() -> None: + assert ord(' ') == 32 + assert ord(' ' + str()) == 32 + assert ord('\x00') == 0 + assert ord('\x00' + str()) == 0 + assert ord('\ue000') == 57344 + assert ord('\ue000' + str()) == 57344 + s = "a\xac\u1234\u20ac\U00010000" + # ^^^^ two-digit hex escape + # ^^^^^^ four-digit Unicode escape + # ^^^^^^^^^^ eight-digit Unicode escape + l1 = [ord(c) for c in s] + assert l1 == [97, 172, 4660, 8364, 65536] + u = 'abcdé' + assert ord(u[-1]) == 233 + assert ord(b'a') == 97 + assert ord(b'a' + bytes()) == 97 + u2 = '\U0010ffff' + str() + assert ord(u2) == 1114111 + assert ord('\U0010ffff') == 1114111 + with assertRaises(TypeError, "ord() expected a character, but a string of length 2 found"): + ord('aa') + with assertRaises(TypeError): + ord('') + +[case testDecode] +def test_decode() -> None: + assert "\N{GREEK CAPITAL LETTER DELTA}" == '\u0394' + assert "\u0394" == "\u0394" + assert "\U00000394" == '\u0394' + assert b'\x80abc'.decode('utf-8', 'replace') == '\ufffdabc' + assert b'\x80abc'.decode('utf-8', 'backslashreplace') == '\\x80abc' + assert b'abc'.decode() == 'abc' + assert b'abc'.decode('utf-8') == 'abc' + assert b'\x80abc'.decode('utf-8', 'ignore') == 'abc' + assert b'\x80abc'.decode('UTF-8', 'ignore') == 'abc' + assert b'\x80abc'.decode('Utf-8', 'ignore') == 'abc' + assert b'\x80abc'.decode('utf_8', 'ignore') == 'abc' + assert b'\x80abc'.decode('latin1', 'ignore') == '\x80abc' + assert b'\xd2\xbb\xb6\xfe\xc8\xfd'.decode('gbk', 'ignore') == '一二三' + assert b'\xd2\xbb\xb6\xfe\xc8\xfd'.decode('latin1', 'ignore') == 'Ò»¶þÈý' + assert b'Z\xc3\xbcrich'.decode("utf-8") == 'Zürich' + try: + b'Z\xc3\xbcrich'.decode('ascii') + assert False + except UnicodeDecodeError: + pass + assert bytearray(range(5)).decode() == '\x00\x01\x02\x03\x04' + b = bytearray(b'\xe4\xbd\xa0\xe5\xa5\xbd') + assert b.decode() == '你好' + assert b.decode('gbk') == '浣犲ソ' + assert b.decode('latin1') == 'ä½\xa0好' + +[case testEncode] +from testutil import assertRaises + +def test_encode() -> None: + u = chr(40960) + 'abcd' + chr(1972) + assert u.encode() == b'\xea\x80\x80abcd\xde\xb4' + assert u.encode('utf-8') == b'\xea\x80\x80abcd\xde\xb4' + with assertRaises(UnicodeEncodeError): + u.encode('ascii') + with assertRaises(LookupError): + u.encode('aaa') + assert u.encode('utf-8', 'aaaaaa') == b'\xea\x80\x80abcd\xde\xb4' + assert u.encode('ascii', 'ignore') == b'abcd' + assert u.encode('ASCII', 'ignore') == b'abcd' + assert u.encode('ascii', 'replace') == b'?abcd?' + assert u.encode('ascii', 'xmlcharrefreplace') == b'ꀀabcd޴' + assert u.encode('ascii', 'backslashreplace') == b'\\ua000abcd\\u07b4' + assert u.encode('ascii', 'namereplace') == b'\\N{YI SYLLABLE IT}abcd\\u07b4' + assert 'pythön!'.encode() == b'pyth\xc3\xb6n!' + assert '一二三'.encode('gbk') == b'\xd2\xbb\xb6\xfe\xc8\xfd' + assert u.encode('UTF-8', 'ignore') == b'\xea\x80\x80abcd\xde\xb4' + assert u.encode('Utf_8') == b'\xea\x80\x80abcd\xde\xb4' + assert u.encode('UTF_8') == b'\xea\x80\x80abcd\xde\xb4' + assert u'\u00E1'.encode('latin1') == b'\xe1' + with assertRaises(UnicodeEncodeError): + u.encode('latin1') + +[case testUnicodeSurrogate] +def f() -> str: + return "\ud800" + +def test_surrogate() -> None: + assert ord(f()) == 0xd800 + assert ord("\udfff") == 0xdfff + assert repr("foobar\x00\xab\ud912\U00012345") == r"'foobar\x00«\ud912𒍅'" diff --git a/mypyc/test-data/run-traits.test b/mypyc/test-data/run-traits.test new file mode 100644 index 000000000000..d1410a3d1e86 --- /dev/null +++ b/mypyc/test-data/run-traits.test @@ -0,0 +1,411 @@ +[case testTraitBasic1] +from mypy_extensions import trait + +class A: + line: int + def foo(self) -> None: + print("foo") + +@trait +class T: + def bar(self) -> None: + print("bar") + def baz(self) -> object: + return None + +class C(A, T): + def baz(self) -> int: + return 10 + +def use_t(t: T) -> object: + t.bar() + return t.baz() + +def use_c(c: C) -> int: + use_t(c) + c.foo() + c.bar() + return c.baz() + +use_t(C()) + +# This trait is dead code but there's no reason it shouldn't compile +@trait +class ChildlessTrait: + def __init__(self) -> None: + pass + +[file driver.py] +from native import A, T, C, use_c, use_t +c = C() +c.foo() +c.bar() +assert c.baz() == 10 +assert use_c(c) == 10 +assert use_t(c) == 10 +[out] +bar +foo +bar +bar +foo +bar +bar + +[case testTraitBasic2] +from mypy_extensions import trait + +class A: + line: int + def foo(self) -> None: + print("foo") + +@trait +class T: + def bar(self) -> None: + print("bar", self.baz()) + def baz(self) -> int: + return -1 + +@trait +class T2: + line: int + def baz(self) -> int: + return -2 + +class C(A, T): + def __init__(self) -> None: + self.line = 1337 + self.x = 12 + def baz(self) -> int: + return self.x + +class D(C, T2): + def __init__(self) -> None: + self.line = 1337 + self.x = 13 + +@trait +class T3: + def baz(self) -> int: + return -2 + +class E(T3): + def __init__(self) -> None: + pass + + +def use_t(t: T) -> None: + t.bar() +def use_t2(t: T2) -> int: + t.line = t.line + return t.line + +def use_c(c: C) -> int: + use_t(c) + c.foo() + c.bar() + return c.line + +def use_d(d: D) -> int: + return d.baz() + +[file driver.py] +from native import A, T, C, D, use_c, use_t, use_d, use_t2 +c = C() +d = D() +c.foo() +c.bar() +print("baz", c.baz()) +print("baz", d.baz()) +use_c(c) +use_t(c) +use_c(d) +use_t(d) +assert use_d(d) == 13 +print(d.line) +assert d.line == 1337 +assert use_t2(d) == 1337 +[out] +foo +bar 12 +baz 12 +baz 13 +bar 12 +foo +bar 12 +bar 12 +bar 13 +foo +bar 13 +bar 13 +1337 + +[case testTrait3] +from mypy_extensions import trait +from typing import Generic, TypeVar + +@trait +class T1: pass +@trait +class T2: pass + +T = TypeVar('T') + +class C(Generic[T], T1, T2): + pass + +@trait +class S1(Generic[T]): + def foo(self) -> None: pass + def bar(self, x: T) -> T: raise Exception + +@trait +class S2(S1[T]): + def bar(self, x: T) -> T: return x + +@trait +class S3(S2[T]): + def bar(self, x: T) -> T: return x + +class D(S3[bool]): + def bar(self, x: bool) -> bool: return x + + +[file driver.py] +import native + +[case testTrait4] +from mypy_extensions import trait +from typing import Generic, TypeVar + +T = TypeVar('T') + + +@trait +class S1(Generic[T]): + def bar(self) -> T: raise Exception + +class S2(S1[bool]): + def bar(self) -> bool: return False + +class D(S2): + pass + +def lol(x: S1) -> None: + x.bar() + +[file driver.py] +import native +native.lol(native.D()) + +[case testTraitOrdering] +import other_b +# Regression test for a bug where inheriting from a class that +# inherited from a trait that got processed later on the command line +# filed to compile. +[file other_b.py] +from other_c import Plugin + +class Whatever(Plugin): + pass + +[file other_c.py] +from mypy_extensions import trait + +@trait +class Base: + x = None # type: int + +class Plugin(Base): + def __init__(self) -> None: + self.x = 10 + +[file driver.py] +from native import * + +[case testDiamond] +from mypy_extensions import trait + +@trait +class Base: + def get_value(self) -> str: + return "Base" + +@trait +class Trait(Base): + def get_value(self) -> str: + return "Trait" + +class Message(Base): + def show_message(self) -> None: + print("I am a " + self.get_value()) + +class DerivedMessage(Message, Trait): + pass + +[file driver.py] +from native import * +a = Message() +a.show_message() +b = DerivedMessage() +b.show_message() + +[out] +I am a Base +I am a Trait + +[case testTraitAttrsSubTrait] +from mypy_extensions import trait + +class A: + a: int + +@trait +class T1: + x: int + +@trait +class T2(T1): + y: int + +class C(A, T2): + c: int + +def f(t1: T1, t2: T2) -> None: + t1.x, t2.x = t2.x, t1.x + +def g(t1: T1, t2: T2) -> None: + t2.y = t1.x + +def get_x(c: C) -> int: + return c.x + +def get_y(c: C) -> int: + return c.y + +[file driver.py] +from native import C, f, g, get_x, get_y + +c1 = C() +c2 = C() + +c1.x = 1 +c1.y = 0 +c2.x = 2 +c2.y = 0 + +f(c1, c2) +assert c1.x == 2 +assert c2.x == 1 +assert get_x(c1) == 2 +assert get_x(c2) == 1 + +assert get_y(c2) == 0 +g(c1, c2) +assert get_y(c2) == 2 +[out] + +[case testTraitAttrsTriangle] +from mypy_extensions import trait + +class A: + a: int + +@trait +class T(A): + x: int + def meth(self) -> int: + return self.a + +class B(A): + b: int + +class C(B, T): + pass + +def take_t(t: T) -> int: + return t.x + t.meth() + +def take_c(c: C) -> int: + return c.x + c.meth() + +[file driver.py] +from native import C, take_t, take_c + +c = C() +c.a = 1 +c.x = 10 + +assert take_t(c) == take_c(c) == 11 +[out] + +[case testTraitAttrsTree] +from mypy_extensions import trait + +class A: + a: int + +@trait +class T1: + x: int + +class B(A, T1): + b: int + +@trait +class T2: + x: int + +class C(B, T2): + pass + +def f(t1: T1, t2: T2) -> None: + t1.x, t2.x = t2.x, t1.x + +def g(c1: C, c2: C) -> None: + c1.x, c2.x = c2.x, c1.x + +[file driver.py] +from native import C, f, g + +c1 = C() +c2 = C() + +c1.x = 1 +c2.x = 2 + +f(c1, c2) +assert c1.x == 2 +assert c2.x == 1 + +g(c1, c2) +assert c1.x == 1 +assert c2.x == 2 +[out] + +[case testTraitErrorMessages] +from mypy_extensions import trait + +@trait +class Trait: + pass + +def create() -> Trait: + return Trait() + +[file driver.py] +from native import Trait, create +from testutil import assertRaises + +with assertRaises(TypeError, "traits may not be directly created"): + Trait() + +with assertRaises(TypeError, "traits may not be directly created"): + create() + +class Sub(Trait): + pass + +with assertRaises(TypeError, "interpreted classes cannot inherit from compiled traits"): + Sub() diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test new file mode 100644 index 000000000000..0851c15e57fd --- /dev/null +++ b/mypyc/test-data/run-tuples.test @@ -0,0 +1,258 @@ +# Test cases for tuples (compile and run) + +[case testTuple] +from typing import List, Optional, Tuple +from typing import Tuple +def f(x: Tuple[int, int]) -> Tuple[int,int]: + return x + +def lurr(x: List[Optional[Tuple[int, str]]]) -> object: + return x[0] + +def asdf(x: Tuple[int, str]) -> None: + pass +[file driver.py] +from testutil import assertRaises +from native import f, lurr, asdf + +assert f((1,2)) == (1, 2) +assert lurr([(1, '2')]) == (1, '2') + +with assertRaises(TypeError): + print(lurr([(1, 2)])) + +with assertRaises(TypeError): + asdf((1, 2)) + +[case testTupleGet] +from typing import Tuple +def f(x: Tuple[Tuple[int, bool], int]) -> int: + return x[0][0] +[file driver.py] +from native import f +print(f(((1,True),2))) +big_number = pow(2, 80) +print(f(((big_number,True),2))) +[out] +1 +1208925819614629174706176 + +[case testSequenceTupleArg] +from typing import Tuple +def f(x: Tuple[int, ...]) -> int: + return x[1] +[file driver.py] +from native import f +print(f((1,2,3,4))) +[out] +2 + +[case testTupleAttr] +from typing import Tuple +class C: + b: Tuple[Tuple[Tuple[int, int], int], int, str, object] + c: Tuple[()] +def f() -> None: + c = C() + c.b = (((1, 2), 2), 1, 'hi', 'hi2') + print(c.b) + +def g() -> None: + try: + h() + except Exception: + print('caught the exception') + +def h() -> Tuple[Tuple[Tuple[int, int], int], int, str, object]: + raise Exception('Intentional exception') +[file driver.py] +from native import f, g, C +f() +g() +assert not hasattr(C(), 'c') +[out] +(((1, 2), 2), 1, 'hi', 'hi2') +caught the exception + +[case testNamedTupleAttributeRun] +from typing import NamedTuple + +NT = NamedTuple('NT', [('x', int), ('y', int)]) + +def f(nt: NT) -> int: + if nt.x > nt.y: + return nt.x + return nt.y + +nt = NT(1, 2) +[file driver.py] +from native import NT, nt, f + +assert f(nt) == 2 +assert f(NT(3, 2)) == 3 + +class Sub(NT): + pass +assert f(Sub(3, 2)) == 3 + +-- Ref: https://github.com/mypyc/mypyc/issues/924 +[case testNamedTupleClassSyntax] +from typing import Dict, List, NamedTuple, Optional, Tuple, Union +from typing_extensions import final + +class FuncIR: pass + +StealsDescription = Union[bool, List[bool]] + +class Record(NamedTuple): + st_mtime: float + st_size: int + is_borrowed: bool + hash: str + python_path: Tuple[str, ...] + type: 'ClassIR' + method: FuncIR + shadow_method: Optional[FuncIR] + classes: Dict[str, 'ClassIR'] + steals: StealsDescription + ordering: Optional[List[int]] + extra_int_constants: List[Tuple[int]] + +# Make sure mypyc loads the annotation string for this forward reference. +# Ref: https://github.com/mypyc/mypyc/issues/938 +class ClassIR: pass + +# Ref: https://github.com/mypyc/mypyc/issues/927 +@final +class Inextensible(NamedTuple): + x: int + +[file driver.py] +from typing import ForwardRef, Optional +from native import ClassIR, FuncIR, Record + +assert Record.__annotations__ == { + 'st_mtime': float, + 'st_size': int, + 'is_borrowed': bool, + 'hash': str, + 'python_path': tuple, + 'type': ForwardRef('ClassIR'), + 'method': FuncIR, + 'shadow_method': type, + 'classes': dict, + 'steals': type, + 'ordering': type, + 'extra_int_constants': list, +}, Record.__annotations__ + +[case testTupleOps] +from typing import Tuple, List, Any, Optional +from typing_extensions import Final + +def f() -> Tuple[()]: + return () + +def test_empty_tuple() -> None: + assert f() == () + +def f2() -> Any: + return () + +def test_empty_tuple_with_any_type(): + assert f2() == () + +def f3() -> int: + x = (False, 1) + return x[1] + +def test_new_tuple() -> None: + assert f3() == 1 + +def f4(y: int) -> int: + x = (False, y) + return x[1] + +def test_new_tuple_boxed_int() -> None: + big_number = 1208925819614629174706176 + assert f4(big_number) == big_number + +def f5(x: List[int]) -> int: + return tuple(x)[1] + +def test_sequence_tuple() -> None: + assert f5([1,2,3,4]) == 2 + +def f6(x: List[int]) -> int: + return len(tuple(x)) + +def test_sequence_tuple_len() -> None: + assert f6([1,2,3,4]) == 4 + +def f7(x: List[Tuple[int, int]]) -> int: + a, b = x[0] + return a + b + +def test_unbox_tuple() -> None: + assert f7([(5, 6)]) == 11 + +# Test that order is irrelevant to unions. Really I only care that this builds. + +class A: + pass + +def lol() -> A: + return A() + +def foo(x: bool, y: bool) -> Tuple[Optional[A], bool]: + z = lol() + + return None if y else z, x + +def test_slicing() -> None: + # Use dummy adds to avoid constant folding + zero = int() + two = zero + 2 + s: Tuple[str, ...] = ("f", "o", "o", "b", "a", "r") + assert s[two:] == ("o", "b", "a", "r") + assert s[:two] == ("f", "o") + assert s[two:-two] == ("o", "b") + assert s[two:two] == () + assert s[two:two + 1] == ("o",) + assert s[-two:] == ("a", "r") + assert s[:-two] == ("f", "o", "o", "b") + assert s[:] == ("f", "o", "o", "b", "a", "r") + assert s[two:333] == ("o", "b", "a", "r") + assert s[333:two] == () + assert s[two:-333] == () + assert s[-333:two] == ("f", "o") + long_int: int = 1000 * 1000 * 1000 * 1000 * 1000 * 1000 * 1000 + assert s[1:long_int] == ("o", "o", "b", "a", "r") + assert s[long_int:] == () + assert s[-long_int:-1] == ("f", "o", "o", "b", "a") + +def f8(val: int) -> bool: + return val % 2 == 0 + +def test_sequence_generator() -> None: + source_list = [1, 2, 3] + a = tuple(f8(x) for x in source_list) + assert a == (False, True, False) + + source_tuple: Tuple[int, ...] = (1, 2, 3) + a = tuple(f8(x) for x in source_tuple) + assert a == (False, True, False) + + source_fixed_length_tuple = (1, 2, 3, 4) + a = tuple(f8(x) for x in source_fixed_length_tuple) + assert a == (False, True, False, True) + + source_str = 'abbc' + b = tuple('s:' + x for x in source_str) + assert b == ('s:a', 's:b', 's:b', 's:c') + +TUPLE: Final[Tuple[str, ...]] = ('x', 'y') + +def test_final_boxed_tuple() -> None: + t = TUPLE + assert t == ('x', 'y') diff --git a/mypyc/test-data/run-u8.test b/mypyc/test-data/run-u8.test new file mode 100644 index 000000000000..cddb031e3352 --- /dev/null +++ b/mypyc/test-data/run-u8.test @@ -0,0 +1,303 @@ +[case testU8BasicOps] +from typing import Any, Tuple + +from mypy_extensions import u8, i16, i32, i64 +from typing_extensions import Final + +from testutil import assertRaises + +ERROR: Final = 239 + +def test_box_and_unbox() -> None: + for i in range(0, 256): + o: Any = i + x: u8 = o + o2: Any = x + assert o == o2 + assert x == i + with assertRaises(OverflowError, "int too large or small to convert to u8"): + o = 256 + x2: u8 = o + with assertRaises(OverflowError, "int too large or small to convert to u8"): + o = -1 + x3: u8 = o + +def div_by_7(x: u8) -> u8: + return x // 7 + +def div(x: u8, y: u8) -> u8: + return x // y + +def test_divide_by_constant() -> None: + for i in range(0, 256): + assert div_by_7(i) == i // 7 + +def test_divide_by_variable() -> None: + for x in range(0, 256): + for y in range(0, 256): + if y != 0: + assert div(x, y) == x // y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + div(x, y) + +def mod_by_7(x: u8) -> u8: + return x % 7 + +def mod(x: u8, y: u8) -> u8: + return x % y + +def test_mod_by_constant() -> None: + for i in range(0, 256): + assert mod_by_7(i) == i % 7 + +def test_mod_by_variable() -> None: + for x in range(0, 256): + for y in range(0, 256): + if y != 0: + assert mod(x, y) == x % y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + mod(x, y) + +def test_simple_arithmetic_ops() -> None: + zero: u8 = int() + one: u8 = zero + 1 + two: u8 = one + 1 + neg_one: u8 = -one + assert neg_one == 255 + assert one + one == 2 + assert one + two == 3 + assert one + neg_one == 0 + assert one - one == 0 + assert one - two == 255 + assert one * one == 1 + assert one * two == 2 + assert two * two == 4 + assert two * neg_one == 254 + assert neg_one * one == 255 + assert neg_one * neg_one == 1 + assert two * 0 == 0 + assert 0 * two == 0 + assert -one == 255 + assert -two == 254 + assert -neg_one == 1 + assert -zero == 0 + +def test_bitwise_ops() -> None: + x: u8 = 184 + int() + y: u8 = 79 + int() + z: u8 = 113 + int() + zero: u8 = int() + one: u8 = zero + 1 + two: u8 = zero + 2 + neg_one: u8 = -one + + assert x & y == 8 + assert x & z == 48 + assert z & z == z + assert x & zero == 0 + + assert x | y == 255 + assert x | z == 249 + assert z | z == z + assert x | 0 == x + + assert x ^ y == 247 + assert x ^ z == 201 + assert z ^ z == 0 + assert z ^ 0 == z + + assert x << one == 112 + assert x << two == 224 + assert z << two == 196 + assert z << 0 == z + + assert x >> one == 92 + assert x >> two == 46 + assert z >> two == 28 + assert z >> 0 == z + + for i in range(256): + t: u8 = i + assert ~t == (~(i + int()) & 0xff) + +def eq(x: u8, y: u8) -> bool: + return x == y + +def test_eq() -> None: + assert eq(int(), int()) + assert eq(5 + int(), 5 + int()) + assert not eq(int(), 1 + int()) + assert not eq(5 + int(), 6 + int()) + +def test_comparisons() -> None: + one: u8 = 1 + int() + one2: u8 = 1 + int() + two: u8 = 2 + int() + assert one < two + assert not (one < one2) + assert not (two < one) + assert two > one + assert not (one > one2) + assert not (one > two) + assert one <= two + assert one <= one2 + assert not (two <= one) + assert two >= one + assert one >= one2 + assert not (one >= two) + assert one == one2 + assert not (one == two) + assert one != two + assert not (one != one2) + +def test_mixed_comparisons() -> None: + u8_3: u8 = int() + 3 + int_5 = int() + 5 + assert u8_3 < int_5 + assert int_5 > u8_3 + b = u8_3 > int_5 + assert not b + + int_largest = int() + 255 + assert int_largest > u8_3 + int_smallest = int() + assert u8_3 > int_smallest + + int_too_big = int() + 256 + int_too_small = int() -1 + with assertRaises(OverflowError): + assert u8_3 < int_too_big + with assertRaises(OverflowError): + assert int_too_big < u8_3 + with assertRaises(OverflowError): + assert u8_3 > int_too_small + with assertRaises(OverflowError): + assert int_too_small < u8_3 + +def test_mixed_arithmetic_and_bitwise_ops() -> None: + u8_3: u8 = int() + 3 + int_5 = int() + 5 + assert u8_3 + int_5 == 8 + assert int_5 - u8_3 == 2 + assert u8_3 << int_5 == 96 + assert int_5 << u8_3 == 40 + assert u8_3 ^ int_5 == 6 + assert int_5 | u8_3 == 7 + + int_largest = int() + 255 + assert int_largest - u8_3 == 252 + int_smallest = int() + assert int_smallest + u8_3 == 3 + + int_too_big = int() + 256 + int_too_small = int() - 1 + with assertRaises(OverflowError): + assert u8_3 & int_too_big + with assertRaises(OverflowError): + assert int_too_small & u8_3 + +def test_coerce_to_and_from_int() -> None: + for n in range(0, 256): + x: u8 = n + m: int = x + assert m == n + +def test_explicit_conversion_to_u8() -> None: + x = u8(5) + assert x == 5 + y = int() + ERROR + x = u8(y) + assert x == ERROR + n64: i64 = 233 + x = u8(n64) + assert x == 233 + n32: i32 = 234 + x = u8(n32) + assert x == 234 + z = u8(x) + assert z == 234 + n16: i16 = 231 + x = u8(n16) + assert x == 231 + +def test_explicit_conversion_overflow() -> None: + max_u8 = int() + 255 + x = u8(max_u8) + assert x == 255 + assert int(x) == max_u8 + + min_u8 = int() + y = u8(min_u8) + assert y == 0 + assert int(y) == min_u8 + + too_big = int() + 256 + with assertRaises(OverflowError): + x = u8(too_big) + + too_small = int() - 1 + with assertRaises(OverflowError): + x = u8(too_small) + +def test_u8_from_large_small_literal() -> None: + x = u8(255) # XXX u8(2**15 - 1) + assert x == 255 + x = u8(0) + assert x == 0 + +def test_u8_truncate_from_i64() -> None: + large = i64(2**32 + 256 + 157 + int()) + x = u8(large) + assert x == 157 + small = i64(-2**32 - 256 - 157 + int()) + x = u8(small) + assert x == 256 - 157 + large2 = i64(2**8 + int()) + x = u8(large2) + assert x == 0 + small2 = i64(-2**8 - 1 - int()) + x = u8(small2) + assert x == 255 + +def test_u8_truncate_from_i32() -> None: + large = i32(2**16 + 2**8 + 5 + int()) + assert u8(large) == 5 + small = i32(-2**16 - 2**8 - 1 + int()) + assert u8(small) == 255 + +def from_float(x: float) -> u8: + return u8(x) + +def test_explicit_conversion_from_float() -> None: + assert from_float(0.0) == 0 + assert from_float(1.456) == 1 + assert from_float(234.567) == 234 + assert from_float(255) == 255 + assert from_float(0) == 0 + assert from_float(-0.999) == 0 + # The error message could be better, but this is acceptable + with assertRaises(OverflowError, "int too large or small to convert to u8"): + assert from_float(float(256)) + with assertRaises(OverflowError, "int too large or small to convert to u8"): + # One ulp below the lowest valid i64 value + from_float(float(-1.0)) + +def test_tuple_u8() -> None: + a: u8 = 1 + b: u8 = 2 + t = (a, b) + a, b = t + assert a == 1 + assert b == 2 + x: Any = t + tt: Tuple[u8, u8] = x + assert tt == (1, 2) + +def test_convert_u8_to_native_int() -> None: + for i in range(256): + x: u8 = i + assert i16(x) == i + assert i32(x) == i + assert i64(x) == i diff --git a/mypyc/test/__init__.py b/mypyc/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/test/config.py b/mypyc/test/config.py new file mode 100644 index 000000000000..8345cd954b5f --- /dev/null +++ b/mypyc/test/config.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import os + +provided_prefix = os.getenv("MYPY_TEST_PREFIX", None) +if provided_prefix: + PREFIX = provided_prefix +else: + this_file_dir = os.path.dirname(os.path.realpath(__file__)) + PREFIX = os.path.dirname(os.path.dirname(this_file_dir)) + +# Location of test data files such as test case descriptions. +test_data_prefix = os.path.join(PREFIX, "mypyc", "test-data") diff --git a/mypyc/test/test_alwaysdefined.py b/mypyc/test/test_alwaysdefined.py new file mode 100644 index 000000000000..d6c4214ba6a2 --- /dev/null +++ b/mypyc/test/test_alwaysdefined.py @@ -0,0 +1,46 @@ +"""Test cases for inferring always defined attributes in classes.""" + +from __future__ import annotations + +import os.path + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file2, + infer_ir_build_options_from_test_name, + use_custom_builtins, +) + +files = ["alwaysdefined.test"] + + +class TestAlwaysDefined(MypycDataSuite): + files = files + base_path = test_temp_dir + + def run_case(self, testcase: DataDrivenTestCase) -> None: + """Perform a runtime checking transformation test case.""" + options = infer_ir_build_options_from_test_name(testcase.name) + if options is None: + # Skipped test case + return + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + try: + ir = build_ir_for_single_file2(testcase.input, options) + except CompileError as e: + actual = e.messages + else: + actual = [] + for cl in ir.classes: + if cl.name.startswith("_"): + continue + actual.append( + "{}: [{}]".format(cl.name, ", ".join(sorted(cl._always_initialized_attrs))) + ) + + assert_test_output(testcase, actual, "Invalid test output", testcase.output) diff --git a/mypyc/test/test_analysis.py b/mypyc/test/test_analysis.py new file mode 100644 index 000000000000..7d297ea575b7 --- /dev/null +++ b/mypyc/test/test_analysis.py @@ -0,0 +1,77 @@ +"""Test runner for data-flow analysis test cases.""" + +from __future__ import annotations + +import os.path + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.analysis import dataflow +from mypyc.common import TOP_LEVEL_NAME +from mypyc.ir.func_ir import all_values +from mypyc.ir.ops import Value +from mypyc.ir.pprint import format_func, generate_names_for_ir +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file, + use_custom_builtins, +) +from mypyc.transform import exceptions + +files = ["analysis.test"] + + +class TestAnalysis(MypycDataSuite): + files = files + base_path = test_temp_dir + optional_out = True + + def run_case(self, testcase: DataDrivenTestCase) -> None: + """Perform a data-flow analysis test case.""" + + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + try: + ir = build_ir_for_single_file(testcase.input) + except CompileError as e: + actual = e.messages + else: + actual = [] + for fn in ir: + if fn.name == TOP_LEVEL_NAME and not testcase.name.endswith("_toplevel"): + continue + exceptions.insert_exception_handling(fn) + actual.extend(format_func(fn)) + cfg = dataflow.get_cfg(fn.blocks) + args: set[Value] = set(fn.arg_regs) + name = testcase.name + if name.endswith("_MaybeDefined"): + # Forward, maybe + analysis_result = dataflow.analyze_maybe_defined_regs(fn.blocks, cfg, args) + elif name.endswith("_Liveness"): + # Backward, maybe + analysis_result = dataflow.analyze_live_regs(fn.blocks, cfg) + elif name.endswith("_MustDefined"): + # Forward, must + analysis_result = dataflow.analyze_must_defined_regs( + fn.blocks, cfg, args, regs=all_values(fn.arg_regs, fn.blocks) + ) + elif name.endswith("_BorrowedArgument"): + # Forward, must + analysis_result = dataflow.analyze_borrowed_arguments(fn.blocks, cfg, args) + else: + assert False, "No recognized _AnalysisName suffix in test case" + + names = generate_names_for_ir(fn.arg_regs, fn.blocks) + + for key in sorted( + analysis_result.before.keys(), key=lambda x: (x[0].label, x[1]) + ): + pre = ", ".join(sorted(names[reg] for reg in analysis_result.before[key])) + post = ", ".join(sorted(names[reg] for reg in analysis_result.after[key])) + actual.append( + "%-8s %-23s %s" % ((key[0].label, key[1]), "{%s}" % pre, "{%s}" % post) + ) + assert_test_output(testcase, actual, "Invalid source code output") diff --git a/mypyc/test/test_cheader.py b/mypyc/test/test_cheader.py new file mode 100644 index 000000000000..7ab055c735ad --- /dev/null +++ b/mypyc/test/test_cheader.py @@ -0,0 +1,46 @@ +"""Test that C functions used in primitives are declared in a header such as CPy.h.""" + +from __future__ import annotations + +import glob +import os +import re +import unittest + +from mypyc.ir.ops import PrimitiveDescription +from mypyc.primitives import registry + + +class TestHeaderInclusion(unittest.TestCase): + def test_primitives_included_in_header(self) -> None: + base_dir = os.path.join(os.path.dirname(__file__), "..", "lib-rt") + with open(os.path.join(base_dir, "CPy.h")) as f: + header = f.read() + with open(os.path.join(base_dir, "pythonsupport.h")) as f: + header += f.read() + + def check_name(name: str) -> None: + if name.startswith("CPy"): + assert re.search( + rf"\b{name}\b", header + ), f'"{name}" is used in mypyc.primitives but not declared in CPy.h' + + for values in [ + registry.method_call_ops.values(), + registry.binary_ops.values(), + registry.unary_ops.values(), + registry.function_ops.values(), + ]: + for ops in values: + if isinstance(ops, PrimitiveDescription): + ops = [ops] + for op in ops: + if op.c_function_name is not None: + check_name(op.c_function_name) + + primitives_path = os.path.join(os.path.dirname(__file__), "..", "primitives") + for fnam in glob.glob(f"{primitives_path}/*.py"): + with open(fnam) as f: + content = f.read() + for name in re.findall(r'c_function_name=["\'](CPy[A-Z_a-z0-9]+)', content): + check_name(name) diff --git a/mypyc/test/test_commandline.py b/mypyc/test/test_commandline.py new file mode 100644 index 000000000000..f66ca2ec8ff0 --- /dev/null +++ b/mypyc/test/test_commandline.py @@ -0,0 +1,82 @@ +"""Test cases for invoking mypyc on the command line. + +These are slow -- do not add test cases unless you have a very good reason to do so. +""" + +from __future__ import annotations + +import glob +import os +import os.path +import re +import subprocess +import sys + +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypy.test.helpers import normalize_error_messages +from mypyc.test.testutil import MypycDataSuite, assert_test_output + +files = ["commandline.test"] + + +base_path = os.path.join(os.path.dirname(__file__), "..", "..") + +python3_path = sys.executable + + +class TestCommandLine(MypycDataSuite): + files = files + base_path = test_temp_dir + optional_out = True + + def run_case(self, testcase: DataDrivenTestCase) -> None: + # Parse options from test case description (arguments must not have spaces) + text = "\n".join(testcase.input) + m = re.search(r"# *cmd: *(.*)", text) + assert m is not None, 'Test case missing "# cmd: " section' + args = m.group(1).split() + + # Write main program to run (not compiled) + program = "_%s.py" % testcase.name + program_path = os.path.join(test_temp_dir, program) + with open(program_path, "w") as f: + f.write(text) + + env = os.environ.copy() + env["PYTHONPATH"] = base_path + + out = b"" + try: + # Compile program + cmd = subprocess.run( + [sys.executable, "-m", "mypyc", *args], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd="tmp", + env=env, + ) + if "ErrorOutput" in testcase.name or cmd.returncode != 0: + out += cmd.stdout + elif "WarningOutput" in testcase.name: + # Strip out setuptools build related output since we're only + # interested in the messages emitted during compilation. + messages, _, _ = cmd.stdout.partition(b"running build_ext") + out += messages + + if cmd.returncode == 0: + # Run main program + out += subprocess.check_output([python3_path, program], cwd="tmp") + finally: + suffix = "pyd" if sys.platform == "win32" else "so" + so_paths = glob.glob(f"tmp/**/*.{suffix}", recursive=True) + for path in so_paths: + os.remove(path) + + # Strip out 'tmp/' from error message paths in the testcase output, + # due to a mismatch between this test and mypy's test suite. + expected = [x.replace("tmp/", "") for x in testcase.output] + + # Verify output + actual = normalize_error_messages(out.decode().splitlines()) + assert_test_output(testcase, actual, "Invalid output", expected=expected) diff --git a/mypyc/test/test_emit.py b/mypyc/test/test_emit.py new file mode 100644 index 000000000000..e4ace3ec01f0 --- /dev/null +++ b/mypyc/test/test_emit.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import unittest + +from mypyc.codegen.emit import Emitter, EmitterContext +from mypyc.ir.ops import BasicBlock, Register, Value +from mypyc.ir.rtypes import RTuple, bool_rprimitive, int_rprimitive, str_rprimitive +from mypyc.namegen import NameGenerator + + +class TestEmitter(unittest.TestCase): + def setUp(self) -> None: + self.n = Register(int_rprimitive, "n") + self.context = EmitterContext(NameGenerator([["mod"]])) + + def test_label(self) -> None: + emitter = Emitter(self.context, {}) + assert emitter.label(BasicBlock(4)) == "CPyL4" + + def test_reg(self) -> None: + names: dict[Value, str] = {self.n: "n"} + emitter = Emitter(self.context, names) + assert emitter.reg(self.n) == "cpy_r_n" + + def test_object_annotation(self) -> None: + emitter = Emitter(self.context, {}) + assert emitter.object_annotation("hello, world", "line;") == " /* 'hello, world' */" + assert ( + emitter.object_annotation(list(range(30)), "line;") + == """\ + /* [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29] */""" + ) + + def test_emit_line(self) -> None: + emitter = Emitter(self.context, {}) + emitter.emit_line("line;") + emitter.emit_line("a {") + emitter.emit_line("f();") + emitter.emit_line("}") + assert emitter.fragments == ["line;\n", "a {\n", " f();\n", "}\n"] + emitter = Emitter(self.context, {}) + emitter.emit_line("CPyStatics[0];", ann="hello, world") + emitter.emit_line("CPyStatics[1];", ann=list(range(30))) + assert emitter.fragments[0] == "CPyStatics[0]; /* 'hello, world' */\n" + assert ( + emitter.fragments[1] + == """\ +CPyStatics[1]; /* [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29] */\n""" + ) + + def test_emit_undefined_value_for_simple_type(self) -> None: + emitter = Emitter(self.context, {}) + assert emitter.c_undefined_value(int_rprimitive) == "CPY_INT_TAG" + assert emitter.c_undefined_value(str_rprimitive) == "NULL" + assert emitter.c_undefined_value(bool_rprimitive) == "2" + + def test_emit_undefined_value_for_tuple(self) -> None: + emitter = Emitter(self.context, {}) + assert ( + emitter.c_undefined_value(RTuple([str_rprimitive, int_rprimitive, bool_rprimitive])) + == "(tuple_T3OIC) { NULL, CPY_INT_TAG, 2 }" + ) + assert emitter.c_undefined_value(RTuple([str_rprimitive])) == "(tuple_T1O) { NULL }" + assert ( + emitter.c_undefined_value(RTuple([RTuple([str_rprimitive]), bool_rprimitive])) + == "(tuple_T2T1OC) { { NULL }, 2 }" + ) diff --git a/mypyc/test/test_emitclass.py b/mypyc/test/test_emitclass.py new file mode 100644 index 000000000000..eb04b22495de --- /dev/null +++ b/mypyc/test/test_emitclass.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import unittest + +from mypyc.codegen.emitclass import getter_name, setter_name, slot_key +from mypyc.ir.class_ir import ClassIR +from mypyc.namegen import NameGenerator + + +class TestEmitClass(unittest.TestCase): + def test_slot_key(self) -> None: + attrs = ["__add__", "__radd__", "__rshift__", "__rrshift__", "__setitem__", "__delitem__"] + s = sorted(attrs, key=lambda x: slot_key(x)) + # __delitem__ and reverse methods should come last. + assert s == [ + "__add__", + "__rshift__", + "__setitem__", + "__delitem__", + "__radd__", + "__rrshift__", + ] + + def test_setter_name(self) -> None: + cls = ClassIR(module_name="testing", name="SomeClass") + generator = NameGenerator([["mod"]]) + + # This should never be `setup`, as it will conflict with the class `setup` + assert setter_name(cls, "up", generator) == "testing___SomeClass_set_up" + + def test_getter_name(self) -> None: + cls = ClassIR(module_name="testing", name="SomeClass") + generator = NameGenerator([["mod"]]) + + assert getter_name(cls, "down", generator) == "testing___SomeClass_get_down" diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py new file mode 100644 index 000000000000..90df131288f9 --- /dev/null +++ b/mypyc/test/test_emitfunc.py @@ -0,0 +1,931 @@ +from __future__ import annotations + +import unittest + +from mypy.test.helpers import assert_string_arrays_equal +from mypyc.codegen.emit import Emitter, EmitterContext +from mypyc.codegen.emitfunc import FunctionEmitterVisitor, generate_native_function +from mypyc.common import PLATFORM_SIZE +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg +from mypyc.ir.ops import ( + ERR_NEVER, + Assign, + AssignMulti, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + DecRef, + Extend, + GetAttr, + GetElementPtr, + Goto, + IncRef, + Integer, + IntOp, + LoadAddress, + LoadMem, + Op, + Register, + Return, + SetAttr, + SetMem, + TupleGet, + Unbox, + Unreachable, + Value, +) +from mypyc.ir.pprint import generate_names_for_ir +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RStruct, + RTuple, + RType, + bool_rprimitive, + c_int_rprimitive, + dict_rprimitive, + int32_rprimitive, + int64_rprimitive, + int_rprimitive, + list_rprimitive, + object_rprimitive, + pointer_rprimitive, + short_int_rprimitive, +) +from mypyc.irbuild.vtable import compute_vtable +from mypyc.namegen import NameGenerator +from mypyc.primitives.dict_ops import ( + dict_get_item_op, + dict_new_op, + dict_set_item_op, + dict_update_op, +) +from mypyc.primitives.int_ops import int_neg_op +from mypyc.primitives.list_ops import list_append_op, list_get_item_op, list_set_item_op +from mypyc.primitives.misc_ops import none_object_op +from mypyc.primitives.registry import binary_ops +from mypyc.subtype import is_subtype + + +class TestFunctionEmitterVisitor(unittest.TestCase): + """Test generation of fragments of C from individual IR ops.""" + + def setUp(self) -> None: + self.registers: list[Register] = [] + + def add_local(name: str, rtype: RType) -> Register: + reg = Register(rtype, name) + self.registers.append(reg) + return reg + + self.n = add_local("n", int_rprimitive) + self.m = add_local("m", int_rprimitive) + self.k = add_local("k", int_rprimitive) + self.l = add_local("l", list_rprimitive) + self.ll = add_local("ll", list_rprimitive) + self.o = add_local("o", object_rprimitive) + self.o2 = add_local("o2", object_rprimitive) + self.d = add_local("d", dict_rprimitive) + self.b = add_local("b", bool_rprimitive) + self.s1 = add_local("s1", short_int_rprimitive) + self.s2 = add_local("s2", short_int_rprimitive) + self.i32 = add_local("i32", int32_rprimitive) + self.i32_1 = add_local("i32_1", int32_rprimitive) + self.i64 = add_local("i64", int64_rprimitive) + self.i64_1 = add_local("i64_1", int64_rprimitive) + self.ptr = add_local("ptr", pointer_rprimitive) + self.t = add_local("t", RTuple([int_rprimitive, bool_rprimitive])) + self.tt = add_local( + "tt", RTuple([RTuple([int_rprimitive, bool_rprimitive]), bool_rprimitive]) + ) + ir = ClassIR("A", "mod") + ir.attributes = { + "x": bool_rprimitive, + "y": int_rprimitive, + "i1": int64_rprimitive, + "i2": int32_rprimitive, + } + ir.bitmap_attrs = ["i1", "i2"] + compute_vtable(ir) + ir.mro = [ir] + self.r = add_local("r", RInstance(ir)) + + self.context = EmitterContext(NameGenerator([["mod"]])) + + def test_goto(self) -> None: + self.assert_emit(Goto(BasicBlock(2)), "goto CPyL2;") + + def test_goto_next_block(self) -> None: + next_block = BasicBlock(2) + self.assert_emit(Goto(next_block), "", next_block=next_block) + + def test_return(self) -> None: + self.assert_emit(Return(self.m), "return cpy_r_m;") + + def test_integer(self) -> None: + self.assert_emit(Assign(self.n, Integer(5)), "cpy_r_n = 10;") + self.assert_emit(Assign(self.i32, Integer(5, c_int_rprimitive)), "cpy_r_i32 = 5;") + + def test_tuple_get(self) -> None: + self.assert_emit(TupleGet(self.t, 1, 0), "cpy_r_r0 = cpy_r_t.f1;") + + def test_load_None(self) -> None: # noqa: N802 + self.assert_emit( + LoadAddress(none_object_op.type, none_object_op.src, 0), + "cpy_r_r0 = (PyObject *)&_Py_NoneStruct;", + ) + + def test_assign_int(self) -> None: + self.assert_emit(Assign(self.m, self.n), "cpy_r_m = cpy_r_n;") + + def test_int_add(self) -> None: + self.assert_emit_binary_op( + "+", self.n, self.m, self.k, "cpy_r_r0 = CPyTagged_Add(cpy_r_m, cpy_r_k);" + ) + + def test_int_sub(self) -> None: + self.assert_emit_binary_op( + "-", self.n, self.m, self.k, "cpy_r_r0 = CPyTagged_Subtract(cpy_r_m, cpy_r_k);" + ) + + def test_int_neg(self) -> None: + assert int_neg_op.c_function_name is not None + self.assert_emit( + CallC( + int_neg_op.c_function_name, + [self.m], + int_neg_op.return_type, + int_neg_op.steals, + int_neg_op.is_borrowed, + int_neg_op.is_borrowed, + int_neg_op.error_kind, + 55, + ), + "cpy_r_r0 = CPyTagged_Negate(cpy_r_m);", + ) + + def test_branch(self) -> None: + self.assert_emit( + Branch(self.b, BasicBlock(8), BasicBlock(9), Branch.BOOL), + """if (cpy_r_b) { + goto CPyL8; + } else + goto CPyL9; + """, + ) + b = Branch(self.b, BasicBlock(8), BasicBlock(9), Branch.BOOL) + b.negated = True + self.assert_emit( + b, + """if (!cpy_r_b) { + goto CPyL8; + } else + goto CPyL9; + """, + ) + + def test_branch_no_else(self) -> None: + next_block = BasicBlock(9) + b = Branch(self.b, BasicBlock(8), next_block, Branch.BOOL) + self.assert_emit(b, """if (cpy_r_b) goto CPyL8;""", next_block=next_block) + next_block = BasicBlock(9) + b = Branch(self.b, BasicBlock(8), next_block, Branch.BOOL) + b.negated = True + self.assert_emit(b, """if (!cpy_r_b) goto CPyL8;""", next_block=next_block) + + def test_branch_no_else_negated(self) -> None: + next_block = BasicBlock(1) + b = Branch(self.b, next_block, BasicBlock(2), Branch.BOOL) + self.assert_emit(b, """if (!cpy_r_b) goto CPyL2;""", next_block=next_block) + next_block = BasicBlock(1) + b = Branch(self.b, next_block, BasicBlock(2), Branch.BOOL) + b.negated = True + self.assert_emit(b, """if (cpy_r_b) goto CPyL2;""", next_block=next_block) + + def test_branch_is_error(self) -> None: + b = Branch(self.b, BasicBlock(8), BasicBlock(9), Branch.IS_ERROR) + self.assert_emit( + b, + """if (cpy_r_b == 2) { + goto CPyL8; + } else + goto CPyL9; + """, + ) + b = Branch(self.b, BasicBlock(8), BasicBlock(9), Branch.IS_ERROR) + b.negated = True + self.assert_emit( + b, + """if (cpy_r_b != 2) { + goto CPyL8; + } else + goto CPyL9; + """, + ) + + def test_branch_is_error_next_block(self) -> None: + next_block = BasicBlock(8) + b = Branch(self.b, next_block, BasicBlock(9), Branch.IS_ERROR) + self.assert_emit(b, """if (cpy_r_b != 2) goto CPyL9;""", next_block=next_block) + b = Branch(self.b, next_block, BasicBlock(9), Branch.IS_ERROR) + b.negated = True + self.assert_emit(b, """if (cpy_r_b == 2) goto CPyL9;""", next_block=next_block) + + def test_branch_rare(self) -> None: + self.assert_emit( + Branch(self.b, BasicBlock(8), BasicBlock(9), Branch.BOOL, rare=True), + """if (unlikely(cpy_r_b)) { + goto CPyL8; + } else + goto CPyL9; + """, + ) + next_block = BasicBlock(9) + self.assert_emit( + Branch(self.b, BasicBlock(8), next_block, Branch.BOOL, rare=True), + """if (unlikely(cpy_r_b)) goto CPyL8;""", + next_block=next_block, + ) + next_block = BasicBlock(8) + b = Branch(self.b, next_block, BasicBlock(9), Branch.BOOL, rare=True) + self.assert_emit(b, """if (likely(!cpy_r_b)) goto CPyL9;""", next_block=next_block) + next_block = BasicBlock(8) + b = Branch(self.b, next_block, BasicBlock(9), Branch.BOOL, rare=True) + b.negated = True + self.assert_emit(b, """if (likely(cpy_r_b)) goto CPyL9;""", next_block=next_block) + + def test_call(self) -> None: + decl = FuncDecl( + "myfn", None, "mod", FuncSignature([RuntimeArg("m", int_rprimitive)], int_rprimitive) + ) + self.assert_emit(Call(decl, [self.m], 55), "cpy_r_r0 = CPyDef_myfn(cpy_r_m);") + + def test_call_two_args(self) -> None: + decl = FuncDecl( + "myfn", + None, + "mod", + FuncSignature( + [RuntimeArg("m", int_rprimitive), RuntimeArg("n", int_rprimitive)], int_rprimitive + ), + ) + self.assert_emit( + Call(decl, [self.m, self.k], 55), "cpy_r_r0 = CPyDef_myfn(cpy_r_m, cpy_r_k);" + ) + + def test_inc_ref(self) -> None: + self.assert_emit(IncRef(self.o), "CPy_INCREF(cpy_r_o);") + self.assert_emit(IncRef(self.o), "CPy_INCREF(cpy_r_o);", rare=True) + + def test_dec_ref(self) -> None: + self.assert_emit(DecRef(self.o), "CPy_DECREF(cpy_r_o);") + self.assert_emit(DecRef(self.o), "CPy_DecRef(cpy_r_o);", rare=True) + + def test_inc_ref_int(self) -> None: + self.assert_emit(IncRef(self.m), "CPyTagged_INCREF(cpy_r_m);") + self.assert_emit(IncRef(self.m), "CPyTagged_IncRef(cpy_r_m);", rare=True) + + def test_dec_ref_int(self) -> None: + self.assert_emit(DecRef(self.m), "CPyTagged_DECREF(cpy_r_m);") + self.assert_emit(DecRef(self.m), "CPyTagged_DecRef(cpy_r_m);", rare=True) + + def test_dec_ref_tuple(self) -> None: + self.assert_emit(DecRef(self.t), "CPyTagged_DECREF(cpy_r_t.f0);") + + def test_dec_ref_tuple_nested(self) -> None: + self.assert_emit(DecRef(self.tt), "CPyTagged_DECREF(cpy_r_tt.f0.f0);") + + def test_list_get_item(self) -> None: + self.assert_emit( + CallC( + str(list_get_item_op.c_function_name), + [self.m, self.k], + list_get_item_op.return_type, + list_get_item_op.steals, + list_get_item_op.is_borrowed, + list_get_item_op.error_kind, + 55, + ), + """cpy_r_r0 = CPyList_GetItem(cpy_r_m, cpy_r_k);""", + ) + + def test_list_set_item(self) -> None: + self.assert_emit( + CallC( + str(list_set_item_op.c_function_name), + [self.l, self.n, self.o], + list_set_item_op.return_type, + list_set_item_op.steals, + list_set_item_op.is_borrowed, + list_set_item_op.error_kind, + 55, + ), + """cpy_r_r0 = CPyList_SetItem(cpy_r_l, cpy_r_n, cpy_r_o);""", + ) + + def test_box_int(self) -> None: + self.assert_emit(Box(self.n), """cpy_r_r0 = CPyTagged_StealAsObject(cpy_r_n);""") + + def test_unbox_int(self) -> None: + self.assert_emit( + Unbox(self.m, int_rprimitive, 55), + """if (likely(PyLong_Check(cpy_r_m))) + cpy_r_r0 = CPyTagged_FromObject(cpy_r_m); + else { + CPy_TypeError("int", cpy_r_m); cpy_r_r0 = CPY_INT_TAG; + } + """, + ) + + def test_box_i64(self) -> None: + self.assert_emit(Box(self.i64), """cpy_r_r0 = PyLong_FromLongLong(cpy_r_i64);""") + + def test_unbox_i64(self) -> None: + self.assert_emit( + Unbox(self.o, int64_rprimitive, 55), """cpy_r_r0 = CPyLong_AsInt64(cpy_r_o);""" + ) + + def test_list_append(self) -> None: + self.assert_emit( + CallC( + str(list_append_op.c_function_name), + [self.l, self.o], + list_append_op.return_type, + list_append_op.steals, + list_append_op.is_borrowed, + list_append_op.error_kind, + 1, + ), + """cpy_r_r0 = PyList_Append(cpy_r_l, cpy_r_o);""", + ) + + def test_get_attr(self) -> None: + self.assert_emit( + GetAttr(self.r, "y", 1), + """cpy_r_r0 = ((mod___AObject *)cpy_r_r)->_y; + if (unlikely(cpy_r_r0 == CPY_INT_TAG)) { + PyErr_SetString(PyExc_AttributeError, "attribute 'y' of 'A' undefined"); + } else { + CPyTagged_INCREF(cpy_r_r0); + } + """, + ) + + def test_get_attr_non_refcounted(self) -> None: + self.assert_emit( + GetAttr(self.r, "x", 1), + """cpy_r_r0 = ((mod___AObject *)cpy_r_r)->_x; + if (unlikely(cpy_r_r0 == 2)) { + PyErr_SetString(PyExc_AttributeError, "attribute 'x' of 'A' undefined"); + } + """, + ) + + def test_get_attr_merged(self) -> None: + op = GetAttr(self.r, "y", 1) + branch = Branch(op, BasicBlock(8), BasicBlock(9), Branch.IS_ERROR) + branch.traceback_entry = ("foobar", 123) + self.assert_emit( + op, + """\ + cpy_r_r0 = ((mod___AObject *)cpy_r_r)->_y; + if (unlikely(cpy_r_r0 == CPY_INT_TAG)) { + CPy_AttributeError("prog.py", "foobar", "A", "y", 123, CPyStatic_prog___globals); + goto CPyL8; + } + CPyTagged_INCREF(cpy_r_r0); + goto CPyL9; + """, + next_branch=branch, + skip_next=True, + ) + + def test_get_attr_with_bitmap(self) -> None: + self.assert_emit( + GetAttr(self.r, "i1", 1), + """cpy_r_r0 = ((mod___AObject *)cpy_r_r)->_i1; + if (unlikely(cpy_r_r0 == -113) && !(((mod___AObject *)cpy_r_r)->bitmap & 1)) { + PyErr_SetString(PyExc_AttributeError, "attribute 'i1' of 'A' undefined"); + } + """, + ) + + def test_set_attr(self) -> None: + self.assert_emit( + SetAttr(self.r, "y", self.m, 1), + """if (((mod___AObject *)cpy_r_r)->_y != CPY_INT_TAG) { + CPyTagged_DECREF(((mod___AObject *)cpy_r_r)->_y); + } + ((mod___AObject *)cpy_r_r)->_y = cpy_r_m; + cpy_r_r0 = 1; + """, + ) + + def test_set_attr_non_refcounted(self) -> None: + self.assert_emit( + SetAttr(self.r, "x", self.b, 1), + """((mod___AObject *)cpy_r_r)->_x = cpy_r_b; + cpy_r_r0 = 1; + """, + ) + + def test_set_attr_no_error(self) -> None: + op = SetAttr(self.r, "y", self.m, 1) + op.error_kind = ERR_NEVER + self.assert_emit( + op, + """if (((mod___AObject *)cpy_r_r)->_y != CPY_INT_TAG) { + CPyTagged_DECREF(((mod___AObject *)cpy_r_r)->_y); + } + ((mod___AObject *)cpy_r_r)->_y = cpy_r_m; + """, + ) + + def test_set_attr_non_refcounted_no_error(self) -> None: + op = SetAttr(self.r, "x", self.b, 1) + op.error_kind = ERR_NEVER + self.assert_emit( + op, + """((mod___AObject *)cpy_r_r)->_x = cpy_r_b; + """, + ) + + def test_set_attr_with_bitmap(self) -> None: + # For some rtypes the error value overlaps a valid value, so we need + # to use a separate bitmap to track defined attributes. + self.assert_emit( + SetAttr(self.r, "i1", self.i64, 1), + """if (unlikely(cpy_r_i64 == -113)) { + ((mod___AObject *)cpy_r_r)->bitmap |= 1; + } + ((mod___AObject *)cpy_r_r)->_i1 = cpy_r_i64; + cpy_r_r0 = 1; + """, + ) + self.assert_emit( + SetAttr(self.r, "i2", self.i32, 1), + """if (unlikely(cpy_r_i32 == -113)) { + ((mod___AObject *)cpy_r_r)->bitmap |= 2; + } + ((mod___AObject *)cpy_r_r)->_i2 = cpy_r_i32; + cpy_r_r0 = 1; + """, + ) + + def test_set_attr_init_with_bitmap(self) -> None: + op = SetAttr(self.r, "i1", self.i64, 1) + op.is_init = True + self.assert_emit( + op, + """if (unlikely(cpy_r_i64 == -113)) { + ((mod___AObject *)cpy_r_r)->bitmap |= 1; + } + ((mod___AObject *)cpy_r_r)->_i1 = cpy_r_i64; + cpy_r_r0 = 1; + """, + ) + + def test_dict_get_item(self) -> None: + self.assert_emit( + CallC( + str(dict_get_item_op.c_function_name), + [self.d, self.o2], + dict_get_item_op.return_type, + dict_get_item_op.steals, + dict_get_item_op.is_borrowed, + dict_get_item_op.error_kind, + 1, + ), + """cpy_r_r0 = CPyDict_GetItem(cpy_r_d, cpy_r_o2);""", + ) + + def test_dict_set_item(self) -> None: + self.assert_emit( + CallC( + str(dict_set_item_op.c_function_name), + [self.d, self.o, self.o2], + dict_set_item_op.return_type, + dict_set_item_op.steals, + dict_set_item_op.is_borrowed, + dict_set_item_op.error_kind, + 1, + ), + """cpy_r_r0 = CPyDict_SetItem(cpy_r_d, cpy_r_o, cpy_r_o2);""", + ) + + def test_dict_update(self) -> None: + self.assert_emit( + CallC( + str(dict_update_op.c_function_name), + [self.d, self.o], + dict_update_op.return_type, + dict_update_op.steals, + dict_update_op.is_borrowed, + dict_update_op.error_kind, + 1, + ), + """cpy_r_r0 = CPyDict_Update(cpy_r_d, cpy_r_o);""", + ) + + def test_new_dict(self) -> None: + self.assert_emit( + CallC( + dict_new_op.c_function_name, + [], + dict_new_op.return_type, + dict_new_op.steals, + dict_new_op.is_borrowed, + dict_new_op.error_kind, + 1, + ), + """cpy_r_r0 = PyDict_New();""", + ) + + def test_dict_contains(self) -> None: + self.assert_emit_binary_op( + "in", self.b, self.o, self.d, """cpy_r_r0 = PyDict_Contains(cpy_r_d, cpy_r_o);""" + ) + + def test_int_op(self) -> None: + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.ADD, 1), + """cpy_r_r0 = cpy_r_s1 + cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.SUB, 1), + """cpy_r_r0 = cpy_r_s1 - cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.MUL, 1), + """cpy_r_r0 = cpy_r_s1 * cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.DIV, 1), + """cpy_r_r0 = cpy_r_s1 / cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.MOD, 1), + """cpy_r_r0 = cpy_r_s1 % cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.AND, 1), + """cpy_r_r0 = cpy_r_s1 & cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.OR, 1), + """cpy_r_r0 = cpy_r_s1 | cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.XOR, 1), + """cpy_r_r0 = cpy_r_s1 ^ cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.LEFT_SHIFT, 1), + """cpy_r_r0 = cpy_r_s1 << cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.s1, self.s2, IntOp.RIGHT_SHIFT, 1), + """cpy_r_r0 = (Py_ssize_t)cpy_r_s1 >> (Py_ssize_t)cpy_r_s2;""", + ) + self.assert_emit( + IntOp(short_int_rprimitive, self.i64, self.i64_1, IntOp.RIGHT_SHIFT, 1), + """cpy_r_r0 = cpy_r_i64 >> cpy_r_i64_1;""", + ) + + def test_comparison_op(self) -> None: + # signed + self.assert_emit( + ComparisonOp(self.s1, self.s2, ComparisonOp.SLT, 1), + """cpy_r_r0 = (Py_ssize_t)cpy_r_s1 < (Py_ssize_t)cpy_r_s2;""", + ) + self.assert_emit( + ComparisonOp(self.i32, self.i32_1, ComparisonOp.SLT, 1), + """cpy_r_r0 = cpy_r_i32 < cpy_r_i32_1;""", + ) + self.assert_emit( + ComparisonOp(self.i64, self.i64_1, ComparisonOp.SLT, 1), + """cpy_r_r0 = cpy_r_i64 < cpy_r_i64_1;""", + ) + # unsigned + self.assert_emit( + ComparisonOp(self.s1, self.s2, ComparisonOp.ULT, 1), + """cpy_r_r0 = cpy_r_s1 < cpy_r_s2;""", + ) + self.assert_emit( + ComparisonOp(self.i32, self.i32_1, ComparisonOp.ULT, 1), + """cpy_r_r0 = (uint32_t)cpy_r_i32 < (uint32_t)cpy_r_i32_1;""", + ) + self.assert_emit( + ComparisonOp(self.i64, self.i64_1, ComparisonOp.ULT, 1), + """cpy_r_r0 = (uint64_t)cpy_r_i64 < (uint64_t)cpy_r_i64_1;""", + ) + + # object type + self.assert_emit( + ComparisonOp(self.o, self.o2, ComparisonOp.EQ, 1), + """cpy_r_r0 = cpy_r_o == cpy_r_o2;""", + ) + self.assert_emit( + ComparisonOp(self.o, self.o2, ComparisonOp.NEQ, 1), + """cpy_r_r0 = cpy_r_o != cpy_r_o2;""", + ) + + def test_load_mem(self) -> None: + self.assert_emit(LoadMem(bool_rprimitive, self.ptr), """cpy_r_r0 = *(char *)cpy_r_ptr;""") + + def test_set_mem(self) -> None: + self.assert_emit( + SetMem(bool_rprimitive, self.ptr, self.b), """*(char *)cpy_r_ptr = cpy_r_b;""" + ) + + def test_get_element_ptr(self) -> None: + r = RStruct( + "Foo", ["b", "i32", "i64"], [bool_rprimitive, int32_rprimitive, int64_rprimitive] + ) + self.assert_emit( + GetElementPtr(self.o, r, "b"), """cpy_r_r0 = (CPyPtr)&((Foo *)cpy_r_o)->b;""" + ) + self.assert_emit( + GetElementPtr(self.o, r, "i32"), """cpy_r_r0 = (CPyPtr)&((Foo *)cpy_r_o)->i32;""" + ) + self.assert_emit( + GetElementPtr(self.o, r, "i64"), """cpy_r_r0 = (CPyPtr)&((Foo *)cpy_r_o)->i64;""" + ) + + def test_load_address(self) -> None: + self.assert_emit( + LoadAddress(object_rprimitive, "PyDict_Type"), + """cpy_r_r0 = (PyObject *)&PyDict_Type;""", + ) + + def test_assign_multi(self) -> None: + t = RArray(object_rprimitive, 2) + a = Register(t, "a") + self.registers.append(a) + self.assert_emit( + AssignMulti(a, [self.o, self.o2]), """PyObject *cpy_r_a[2] = {cpy_r_o, cpy_r_o2};""" + ) + + def test_long_unsigned(self) -> None: + a = Register(int64_rprimitive, "a") + self.assert_emit( + Assign(a, Integer(1 << 31, int64_rprimitive)), """cpy_r_a = 2147483648LL;""" + ) + self.assert_emit( + Assign(a, Integer((1 << 31) - 1, int64_rprimitive)), """cpy_r_a = 2147483647;""" + ) + + def test_long_signed(self) -> None: + a = Register(int64_rprimitive, "a") + self.assert_emit( + Assign(a, Integer(-(1 << 31) + 1, int64_rprimitive)), """cpy_r_a = -2147483647;""" + ) + self.assert_emit( + Assign(a, Integer(-(1 << 31), int64_rprimitive)), """cpy_r_a = -2147483648LL;""" + ) + + def test_cast_and_branch_merge(self) -> None: + op = Cast(self.r, dict_rprimitive, 1) + next_block = BasicBlock(9) + branch = Branch(op, BasicBlock(8), next_block, Branch.IS_ERROR) + branch.traceback_entry = ("foobar", 123) + self.assert_emit( + op, + """\ +if (likely(PyDict_Check(cpy_r_r))) + cpy_r_r0 = cpy_r_r; +else { + CPy_TypeErrorTraceback("prog.py", "foobar", 123, CPyStatic_prog___globals, "dict", cpy_r_r); + goto CPyL8; +} +""", + next_block=next_block, + next_branch=branch, + skip_next=True, + ) + + def test_cast_and_branch_no_merge_1(self) -> None: + op = Cast(self.r, dict_rprimitive, 1) + branch = Branch(op, BasicBlock(8), BasicBlock(9), Branch.IS_ERROR) + branch.traceback_entry = ("foobar", 123) + self.assert_emit( + op, + """\ + if (likely(PyDict_Check(cpy_r_r))) + cpy_r_r0 = cpy_r_r; + else { + CPy_TypeError("dict", cpy_r_r); + cpy_r_r0 = NULL; + } + """, + next_block=BasicBlock(10), + next_branch=branch, + skip_next=False, + ) + + def test_cast_and_branch_no_merge_2(self) -> None: + op = Cast(self.r, dict_rprimitive, 1) + next_block = BasicBlock(9) + branch = Branch(op, BasicBlock(8), next_block, Branch.IS_ERROR) + branch.negated = True + branch.traceback_entry = ("foobar", 123) + self.assert_emit( + op, + """\ + if (likely(PyDict_Check(cpy_r_r))) + cpy_r_r0 = cpy_r_r; + else { + CPy_TypeError("dict", cpy_r_r); + cpy_r_r0 = NULL; + } + """, + next_block=next_block, + next_branch=branch, + ) + + def test_cast_and_branch_no_merge_3(self) -> None: + op = Cast(self.r, dict_rprimitive, 1) + next_block = BasicBlock(9) + branch = Branch(op, BasicBlock(8), next_block, Branch.BOOL) + branch.traceback_entry = ("foobar", 123) + self.assert_emit( + op, + """\ + if (likely(PyDict_Check(cpy_r_r))) + cpy_r_r0 = cpy_r_r; + else { + CPy_TypeError("dict", cpy_r_r); + cpy_r_r0 = NULL; + } + """, + next_block=next_block, + next_branch=branch, + ) + + def test_cast_and_branch_no_merge_4(self) -> None: + op = Cast(self.r, dict_rprimitive, 1) + next_block = BasicBlock(9) + branch = Branch(op, BasicBlock(8), next_block, Branch.IS_ERROR) + self.assert_emit( + op, + """\ + if (likely(PyDict_Check(cpy_r_r))) + cpy_r_r0 = cpy_r_r; + else { + CPy_TypeError("dict", cpy_r_r); + cpy_r_r0 = NULL; + } + """, + next_block=next_block, + next_branch=branch, + ) + + def test_extend(self) -> None: + a = Register(int32_rprimitive, "a") + self.assert_emit(Extend(a, int64_rprimitive, signed=True), """cpy_r_r0 = cpy_r_a;""") + self.assert_emit( + Extend(a, int64_rprimitive, signed=False), """cpy_r_r0 = (uint32_t)cpy_r_a;""" + ) + if PLATFORM_SIZE == 4: + self.assert_emit( + Extend(self.n, int64_rprimitive, signed=True), + """cpy_r_r0 = (Py_ssize_t)cpy_r_n;""", + ) + self.assert_emit( + Extend(self.n, int64_rprimitive, signed=False), """cpy_r_r0 = cpy_r_n;""" + ) + if PLATFORM_SIZE == 8: + self.assert_emit(Extend(a, int_rprimitive, signed=True), """cpy_r_r0 = cpy_r_a;""") + self.assert_emit( + Extend(a, int_rprimitive, signed=False), """cpy_r_r0 = (uint32_t)cpy_r_a;""" + ) + + def assert_emit( + self, + op: Op, + expected: str, + next_block: BasicBlock | None = None, + *, + rare: bool = False, + next_branch: Branch | None = None, + skip_next: bool = False, + ) -> None: + block = BasicBlock(0) + block.ops.append(op) + value_names = generate_names_for_ir(self.registers, [block]) + emitter = Emitter(self.context, value_names) + declarations = Emitter(self.context, value_names) + emitter.fragments = [] + declarations.fragments = [] + + visitor = FunctionEmitterVisitor(emitter, declarations, "prog.py", "prog") + visitor.next_block = next_block + visitor.rare = rare + if next_branch: + visitor.ops = [op, next_branch] + else: + visitor.ops = [op] + visitor.op_index = 0 + + op.accept(visitor) + frags = declarations.fragments + emitter.fragments + actual_lines = [line.strip(" ") for line in frags] + assert all(line.endswith("\n") for line in actual_lines) + actual_lines = [line.rstrip("\n") for line in actual_lines] + if not expected.strip(): + expected_lines = [] + else: + expected_lines = expected.rstrip().split("\n") + expected_lines = [line.strip(" ") for line in expected_lines] + assert_string_arrays_equal( + expected_lines, actual_lines, msg="Generated code unexpected", traceback=True + ) + if skip_next: + assert visitor.op_index == 1 + else: + assert visitor.op_index == 0 + + def assert_emit_binary_op( + self, op: str, dest: Value, left: Value, right: Value, expected: str + ) -> None: + if op in binary_ops: + ops = binary_ops[op] + for desc in ops: + if is_subtype(left.type, desc.arg_types[0]) and is_subtype( + right.type, desc.arg_types[1] + ): + args = [left, right] + if desc.ordering is not None: + args = [args[i] for i in desc.ordering] + # This only supports primitives that map to C calls + assert desc.c_function_name is not None + self.assert_emit( + CallC( + desc.c_function_name, + args, + desc.return_type, + desc.steals, + desc.is_borrowed, + desc.error_kind, + 55, + ), + expected, + ) + return + else: + assert False, "Could not find matching op" + + +class TestGenerateFunction(unittest.TestCase): + def setUp(self) -> None: + self.arg = RuntimeArg("arg", int_rprimitive) + self.reg = Register(int_rprimitive, "arg") + self.block = BasicBlock(0) + + def test_simple(self) -> None: + self.block.ops.append(Return(self.reg)) + fn = FuncIR( + FuncDecl("myfunc", None, "mod", FuncSignature([self.arg], int_rprimitive)), + [self.reg], + [self.block], + ) + value_names = generate_names_for_ir(fn.arg_regs, fn.blocks) + emitter = Emitter(EmitterContext(NameGenerator([["mod"]])), value_names) + generate_native_function(fn, emitter, "prog.py", "prog") + result = emitter.fragments + assert_string_arrays_equal( + ["CPyTagged CPyDef_myfunc(CPyTagged cpy_r_arg) {\n", " return cpy_r_arg;\n", "}\n"], + result, + msg="Generated code invalid", + ) + + def test_register(self) -> None: + reg = Register(int_rprimitive) + op = Assign(reg, Integer(5)) + self.block.ops.append(op) + self.block.ops.append(Unreachable()) + fn = FuncIR( + FuncDecl("myfunc", None, "mod", FuncSignature([self.arg], list_rprimitive)), + [self.reg], + [self.block], + ) + value_names = generate_names_for_ir(fn.arg_regs, fn.blocks) + emitter = Emitter(EmitterContext(NameGenerator([["mod"]])), value_names) + generate_native_function(fn, emitter, "prog.py", "prog") + result = emitter.fragments + assert_string_arrays_equal( + [ + "PyObject *CPyDef_myfunc(CPyTagged cpy_r_arg) {\n", + " CPyTagged cpy_r_r0;\n", + " cpy_r_r0 = 10;\n", + " CPy_Unreachable();\n", + "}\n", + ], + result, + msg="Generated code invalid", + ) diff --git a/mypyc/test/test_emitwrapper.py b/mypyc/test/test_emitwrapper.py new file mode 100644 index 000000000000..c4465656444c --- /dev/null +++ b/mypyc/test/test_emitwrapper.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import unittest + +from mypy.test.helpers import assert_string_arrays_equal +from mypyc.codegen.emit import Emitter, EmitterContext, ReturnHandler +from mypyc.codegen.emitwrapper import generate_arg_check +from mypyc.ir.rtypes import int_rprimitive, list_rprimitive +from mypyc.namegen import NameGenerator + + +class TestArgCheck(unittest.TestCase): + def setUp(self) -> None: + self.context = EmitterContext(NameGenerator([["mod"]])) + + def test_check_list(self) -> None: + emitter = Emitter(self.context) + generate_arg_check("x", list_rprimitive, emitter, ReturnHandler("NULL")) + lines = emitter.fragments + self.assert_lines( + [ + "PyObject *arg_x;", + "if (likely(PyList_Check(obj_x)))", + " arg_x = obj_x;", + "else {", + ' CPy_TypeError("list", obj_x);', + " return NULL;", + "}", + ], + lines, + ) + + def test_check_int(self) -> None: + emitter = Emitter(self.context) + generate_arg_check("x", int_rprimitive, emitter, ReturnHandler("NULL")) + generate_arg_check("y", int_rprimitive, emitter, ReturnHandler("NULL"), optional=True) + lines = emitter.fragments + self.assert_lines( + [ + "CPyTagged arg_x;", + "if (likely(PyLong_Check(obj_x)))", + " arg_x = CPyTagged_BorrowFromObject(obj_x);", + "else {", + ' CPy_TypeError("int", obj_x); return NULL;', + "}", + "CPyTagged arg_y;", + "if (obj_y == NULL) {", + " arg_y = CPY_INT_TAG;", + "} else if (likely(PyLong_Check(obj_y)))", + " arg_y = CPyTagged_BorrowFromObject(obj_y);", + "else {", + ' CPy_TypeError("int", obj_y); return NULL;', + "}", + ], + lines, + ) + + def assert_lines(self, expected: list[str], actual: list[str]) -> None: + actual = [line.rstrip("\n") for line in actual] + assert_string_arrays_equal(expected, actual, "Invalid output") diff --git a/mypyc/test/test_exceptions.py b/mypyc/test/test_exceptions.py new file mode 100644 index 000000000000..71587e616d1a --- /dev/null +++ b/mypyc/test/test_exceptions.py @@ -0,0 +1,56 @@ +"""Test runner for exception handling transform test cases. + +The transform inserts exception handling branch operations to IR. +""" + +from __future__ import annotations + +import os.path + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.analysis.blockfreq import frequently_executed_blocks +from mypyc.common import TOP_LEVEL_NAME +from mypyc.ir.pprint import format_func +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file, + remove_comment_lines, + use_custom_builtins, +) +from mypyc.transform.exceptions import insert_exception_handling +from mypyc.transform.refcount import insert_ref_count_opcodes +from mypyc.transform.uninit import insert_uninit_checks + +files = ["exceptions.test", "exceptions-freq.test"] + + +class TestExceptionTransform(MypycDataSuite): + files = files + base_path = test_temp_dir + + def run_case(self, testcase: DataDrivenTestCase) -> None: + """Perform a runtime checking transformation test case.""" + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + expected_output = remove_comment_lines(testcase.output) + try: + ir = build_ir_for_single_file(testcase.input) + except CompileError as e: + actual = e.messages + else: + actual = [] + for fn in ir: + if fn.name == TOP_LEVEL_NAME and not testcase.name.endswith("_toplevel"): + continue + insert_uninit_checks(fn) + insert_exception_handling(fn) + insert_ref_count_opcodes(fn) + actual.extend(format_func(fn)) + if testcase.name.endswith("_freq"): + common = frequently_executed_blocks(fn.blocks[0]) + actual.append("hot blocks: %s" % sorted(b.label for b in common)) + + assert_test_output(testcase, actual, "Invalid source code output", expected_output) diff --git a/mypyc/test/test_external.py b/mypyc/test/test_external.py new file mode 100644 index 000000000000..010c74dee42e --- /dev/null +++ b/mypyc/test/test_external.py @@ -0,0 +1,51 @@ +"""Test cases that run tests as subprocesses.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest + +base_dir = os.path.join(os.path.dirname(__file__), "..", "..") + + +class TestExternal(unittest.TestCase): + # TODO: Get this to work on Windows. + # (Or don't. It is probably not a good use of time.) + @unittest.skipIf(sys.platform.startswith("win"), "rt tests don't work on windows") + def test_c_unit_test(self) -> None: + """Run C unit tests in a subprocess.""" + cppflags: list[str] = [] + env = os.environ.copy() + if sys.platform == "darwin": + cppflags += ["-O0", "-mmacosx-version-min=10.10", "-stdlib=libc++"] + elif sys.platform == "linux": + cppflags += ["-O0"] + env["CPPFLAGS"] = " ".join(cppflags) + # Build Python wrapper for C unit tests. + + with tempfile.TemporaryDirectory() as tmpdir: + status = subprocess.check_call( + [ + sys.executable, + "setup.py", + "build_ext", + f"--build-lib={tmpdir}", + f"--build-temp={tmpdir}", + ], + env=env, + cwd=os.path.join(base_dir, "mypyc", "lib-rt"), + ) + # Run C unit tests. + env = os.environ.copy() + if "GTEST_COLOR" not in os.environ: + env["GTEST_COLOR"] = "yes" # Use fancy colors + status = subprocess.call( + [sys.executable, "-c", "import sys, test_capi; sys.exit(test_capi.run_tests())"], + env=env, + cwd=tmpdir, + ) + if status != 0: + raise AssertionError("make test: C unit test failure") diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py new file mode 100644 index 000000000000..5b3f678d8f17 --- /dev/null +++ b/mypyc/test/test_irbuild.py @@ -0,0 +1,87 @@ +"""Test cases for IR generation.""" + +from __future__ import annotations + +import os.path +import sys + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.common import TOP_LEVEL_NAME +from mypyc.ir.pprint import format_func +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file, + infer_ir_build_options_from_test_name, + remove_comment_lines, + replace_word_size, + use_custom_builtins, +) + +files = [ + "irbuild-basic.test", + "irbuild-int.test", + "irbuild-bool.test", + "irbuild-lists.test", + "irbuild-tuple.test", + "irbuild-dict.test", + "irbuild-set.test", + "irbuild-str.test", + "irbuild-bytes.test", + "irbuild-float.test", + "irbuild-statements.test", + "irbuild-nested.test", + "irbuild-classes.test", + "irbuild-optional.test", + "irbuild-any.test", + "irbuild-generics.test", + "irbuild-try.test", + "irbuild-strip-asserts.test", + "irbuild-i64.test", + "irbuild-i32.test", + "irbuild-i16.test", + "irbuild-u8.test", + "irbuild-vectorcall.test", + "irbuild-unreachable.test", + "irbuild-isinstance.test", + "irbuild-dunders.test", + "irbuild-singledispatch.test", + "irbuild-constant-fold.test", + "irbuild-glue-methods.test", + "irbuild-math.test", +] + +if sys.version_info >= (3, 10): + files.append("irbuild-match.test") + + +class TestGenOps(MypycDataSuite): + files = files + base_path = test_temp_dir + optional_out = True + + def run_case(self, testcase: DataDrivenTestCase) -> None: + """Perform a runtime checking transformation test case.""" + options = infer_ir_build_options_from_test_name(testcase.name) + if options is None: + # Skipped test case + return + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + expected_output = remove_comment_lines(testcase.output) + expected_output = replace_word_size(expected_output) + name = testcase.name + try: + ir = build_ir_for_single_file(testcase.input, options) + except CompileError as e: + actual = e.messages + else: + actual = [] + for fn in ir: + if fn.name == TOP_LEVEL_NAME and not name.endswith("_toplevel"): + continue + actual.extend(format_func(fn)) + + assert_test_output(testcase, actual, "Invalid source code output", expected_output) diff --git a/mypyc/test/test_ircheck.py b/mypyc/test/test_ircheck.py new file mode 100644 index 000000000000..7f7063cdc5e6 --- /dev/null +++ b/mypyc/test/test_ircheck.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +import unittest + +from mypyc.analysis.ircheck import FnError, can_coerce_to, check_func_ir +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature +from mypyc.ir.ops import ( + Assign, + BasicBlock, + Goto, + Integer, + LoadAddress, + LoadLiteral, + Op, + Register, + Return, +) +from mypyc.ir.pprint import format_func +from mypyc.ir.rtypes import ( + RInstance, + RType, + RUnion, + bytes_rprimitive, + int32_rprimitive, + int64_rprimitive, + none_rprimitive, + object_rprimitive, + pointer_rprimitive, + str_rprimitive, +) + + +def assert_has_error(fn: FuncIR, error: FnError) -> None: + errors = check_func_ir(fn) + assert errors == [error] + + +def assert_no_errors(fn: FuncIR) -> None: + assert not check_func_ir(fn) + + +NONE_VALUE = Integer(0, rtype=none_rprimitive) + + +class TestIrcheck(unittest.TestCase): + def setUp(self) -> None: + self.label = 0 + + def basic_block(self, ops: list[Op]) -> BasicBlock: + self.label += 1 + block = BasicBlock(self.label) + block.ops = ops + return block + + def func_decl(self, name: str, ret_type: RType | None = None) -> FuncDecl: + if ret_type is None: + ret_type = none_rprimitive + return FuncDecl( + name=name, + class_name=None, + module_name="module", + sig=FuncSignature(args=[], ret_type=ret_type), + ) + + def test_valid_fn(self) -> None: + assert_no_errors( + FuncIR( + decl=self.func_decl(name="func_1"), + arg_regs=[], + blocks=[self.basic_block(ops=[Return(value=NONE_VALUE)])], + ) + ) + + def test_block_not_terminated_empty_block(self) -> None: + block = self.basic_block([]) + fn = FuncIR(decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block]) + assert_has_error(fn, FnError(source=block, desc="Block not terminated")) + + def test_valid_goto(self) -> None: + block_1 = self.basic_block([Return(value=NONE_VALUE)]) + block_2 = self.basic_block([Goto(label=block_1)]) + fn = FuncIR(decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block_1, block_2]) + assert_no_errors(fn) + + def test_invalid_goto(self) -> None: + block_1 = self.basic_block([Return(value=NONE_VALUE)]) + goto = Goto(label=block_1) + block_2 = self.basic_block([goto]) + fn = FuncIR( + decl=self.func_decl(name="func_1"), + arg_regs=[], + # block_1 omitted + blocks=[block_2], + ) + assert_has_error(fn, FnError(source=goto, desc="Invalid control operation target: 1")) + + def test_invalid_register_source(self) -> None: + ret = Return(value=Register(type=none_rprimitive, name="r1")) + block = self.basic_block([ret]) + fn = FuncIR(decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block]) + assert_has_error(fn, FnError(source=ret, desc="Invalid op reference to register 'r1'")) + + def test_invalid_op_source(self) -> None: + ret = Return(value=LoadLiteral(value="foo", rtype=str_rprimitive)) + block = self.basic_block([ret]) + fn = FuncIR(decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block]) + assert_has_error( + fn, FnError(source=ret, desc="Invalid op reference to op of type LoadLiteral") + ) + + def test_invalid_return_type(self) -> None: + ret = Return(value=Integer(value=5, rtype=int32_rprimitive)) + fn = FuncIR( + decl=self.func_decl(name="func_1", ret_type=int64_rprimitive), + arg_regs=[], + blocks=[self.basic_block([ret])], + ) + assert_has_error( + fn, FnError(source=ret, desc="Cannot coerce source type i32 to dest type i64") + ) + + def test_invalid_assign(self) -> None: + arg_reg = Register(type=int64_rprimitive, name="r1") + assign = Assign(dest=arg_reg, src=Integer(value=5, rtype=int32_rprimitive)) + ret = Return(value=NONE_VALUE) + fn = FuncIR( + decl=self.func_decl(name="func_1"), + arg_regs=[arg_reg], + blocks=[self.basic_block([assign, ret])], + ) + assert_has_error( + fn, FnError(source=assign, desc="Cannot coerce source type i32 to dest type i64") + ) + + def test_can_coerce_to(self) -> None: + cls = ClassIR(name="Cls", module_name="cls") + valid_cases = [ + (int64_rprimitive, int64_rprimitive), + (str_rprimitive, str_rprimitive), + (str_rprimitive, object_rprimitive), + (object_rprimitive, str_rprimitive), + (RUnion([bytes_rprimitive, str_rprimitive]), str_rprimitive), + (str_rprimitive, RUnion([bytes_rprimitive, str_rprimitive])), + (RInstance(cls), object_rprimitive), + ] + + invalid_cases = [ + (int64_rprimitive, int32_rprimitive), + (RInstance(cls), str_rprimitive), + (str_rprimitive, bytes_rprimitive), + ] + + for src, dest in valid_cases: + assert can_coerce_to(src, dest) + for src, dest in invalid_cases: + assert not can_coerce_to(src, dest) + + def test_duplicate_op(self) -> None: + arg_reg = Register(type=int32_rprimitive, name="r1") + assign = Assign(dest=arg_reg, src=Integer(value=5, rtype=int32_rprimitive)) + block = self.basic_block([assign, assign, Return(value=NONE_VALUE)]) + fn = FuncIR(decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block]) + assert_has_error(fn, FnError(source=assign, desc="Func has a duplicate op")) + + def test_pprint(self) -> None: + block_1 = self.basic_block([Return(value=NONE_VALUE)]) + goto = Goto(label=block_1) + block_2 = self.basic_block([goto]) + fn = FuncIR( + decl=self.func_decl(name="func_1"), + arg_regs=[], + # block_1 omitted + blocks=[block_2], + ) + errors = [(goto, "Invalid control operation target: 1")] + formatted = format_func(fn, errors) + assert formatted == [ + "def func_1():", + "L0:", + " goto L1", + " ERR: Invalid control operation target: 1", + ] + + def test_load_address_declares_register(self) -> None: + rx = Register(str_rprimitive, "x") + ry = Register(pointer_rprimitive, "y") + load_addr = LoadAddress(pointer_rprimitive, rx) + assert_no_errors( + FuncIR( + decl=self.func_decl(name="func_1"), + arg_regs=[], + blocks=[ + self.basic_block( + ops=[load_addr, Assign(ry, load_addr), Return(value=NONE_VALUE)] + ) + ], + ) + ) diff --git a/mypyc/test/test_literals.py b/mypyc/test/test_literals.py new file mode 100644 index 000000000000..a8c17d10d30d --- /dev/null +++ b/mypyc/test/test_literals.py @@ -0,0 +1,90 @@ +"""Test code geneneration for literals.""" + +from __future__ import annotations + +import unittest + +from mypyc.codegen.literals import ( + Literals, + _encode_bytes_values, + _encode_int_values, + _encode_str_values, + format_str_literal, +) + + +class TestLiterals(unittest.TestCase): + def test_format_str_literal(self) -> None: + assert format_str_literal("") == b"\x00" + assert format_str_literal("xyz") == b"\x03xyz" + assert format_str_literal("x" * 127) == b"\x7f" + b"x" * 127 + assert format_str_literal("x" * 128) == b"\x81\x00" + b"x" * 128 + assert format_str_literal("x" * 131) == b"\x81\x03" + b"x" * 131 + + def test_encode_str_values(self) -> None: + assert _encode_str_values({}) == [b""] + assert _encode_str_values({"foo": 0}) == [b"\x01\x03foo", b""] + assert _encode_str_values({"foo": 0, "b": 1}) == [b"\x02\x03foo\x01b", b""] + assert _encode_str_values({"foo": 0, "x" * 70: 1}) == [ + b"\x01\x03foo", + bytes([1, 70]) + b"x" * 70, + b"", + ] + assert _encode_str_values({"y" * 100: 0}) == [bytes([1, 100]) + b"y" * 100, b""] + + def test_encode_bytes_values(self) -> None: + assert _encode_bytes_values({}) == [b""] + assert _encode_bytes_values({b"foo": 0}) == [b"\x01\x03foo", b""] + assert _encode_bytes_values({b"foo": 0, b"b": 1}) == [b"\x02\x03foo\x01b", b""] + assert _encode_bytes_values({b"foo": 0, b"x" * 70: 1}) == [ + b"\x01\x03foo", + bytes([1, 70]) + b"x" * 70, + b"", + ] + assert _encode_bytes_values({b"y" * 100: 0}) == [bytes([1, 100]) + b"y" * 100, b""] + + def test_encode_int_values(self) -> None: + assert _encode_int_values({}) == [b""] + assert _encode_int_values({123: 0}) == [b"\x01123", b""] + assert _encode_int_values({123: 0, 9: 1}) == [b"\x02123\x009", b""] + assert _encode_int_values({123: 0, 45: 1, 5 * 10**70: 2}) == [ + b"\x02123\x0045", + b"\x015" + b"0" * 70, + b"", + ] + assert _encode_int_values({6 * 10**100: 0}) == [b"\x016" + b"0" * 100, b""] + + def test_simple_literal_index(self) -> None: + lit = Literals() + lit.record_literal(1) + lit.record_literal("y") + lit.record_literal(True) + lit.record_literal(None) + lit.record_literal(False) + assert lit.literal_index(None) == 0 + assert lit.literal_index(False) == 1 + assert lit.literal_index(True) == 2 + assert lit.literal_index("y") == 3 + assert lit.literal_index(1) == 4 + + def test_tuple_literal(self) -> None: + lit = Literals() + lit.record_literal((1, "y", None, (b"a", "b"))) + lit.record_literal((b"a", "b")) + lit.record_literal(()) + assert lit.literal_index((b"a", "b")) == 7 + assert lit.literal_index((1, "y", None, (b"a", "b"))) == 8 + assert lit.literal_index(()) == 9 + print(lit.encoded_tuple_values()) + assert lit.encoded_tuple_values() == [ + "3", # Number of tuples + "2", + "5", + "4", # First tuple (length=2) + "4", + "6", + "3", + "0", + "7", # Second tuple (length=4) + "0", # Third tuple (length=0) + ] diff --git a/mypyc/test/test_lowering.py b/mypyc/test/test_lowering.py new file mode 100644 index 000000000000..50a9a7390855 --- /dev/null +++ b/mypyc/test/test_lowering.py @@ -0,0 +1,56 @@ +"""Runner for lowering transform tests.""" + +from __future__ import annotations + +import os.path + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.common import TOP_LEVEL_NAME +from mypyc.ir.pprint import format_func +from mypyc.options import CompilerOptions +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file, + remove_comment_lines, + replace_word_size, + use_custom_builtins, +) +from mypyc.transform.exceptions import insert_exception_handling +from mypyc.transform.flag_elimination import do_flag_elimination +from mypyc.transform.lower import lower_ir +from mypyc.transform.refcount import insert_ref_count_opcodes +from mypyc.transform.uninit import insert_uninit_checks + + +class TestLowering(MypycDataSuite): + files = ["lowering-int.test", "lowering-list.test"] + base_path = test_temp_dir + + def run_case(self, testcase: DataDrivenTestCase) -> None: + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + expected_output = remove_comment_lines(testcase.output) + expected_output = replace_word_size(expected_output) + try: + ir = build_ir_for_single_file(testcase.input) + except CompileError as e: + actual = e.messages + else: + actual = [] + for fn in ir: + if fn.name == TOP_LEVEL_NAME and not testcase.name.endswith("_toplevel"): + continue + options = CompilerOptions() + # Lowering happens after exception handling and ref count opcodes have + # been added. Any changes must maintain reference counting semantics. + insert_uninit_checks(fn) + insert_exception_handling(fn) + insert_ref_count_opcodes(fn) + lower_ir(fn, options) + do_flag_elimination(fn, options) + actual.extend(format_func(fn)) + + assert_test_output(testcase, actual, "Invalid source code output", expected_output) diff --git a/mypyc/test/test_namegen.py b/mypyc/test/test_namegen.py new file mode 100644 index 000000000000..f88edbd00dce --- /dev/null +++ b/mypyc/test/test_namegen.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import unittest + +from mypyc.namegen import ( + NameGenerator, + candidate_suffixes, + exported_name, + make_module_translation_map, +) + + +class TestNameGen(unittest.TestCase): + def test_candidate_suffixes(self) -> None: + assert candidate_suffixes("foo") == ["", "foo."] + assert candidate_suffixes("foo.bar") == ["", "bar.", "foo.bar."] + + def test_exported_name(self) -> None: + assert exported_name("foo") == "foo" + assert exported_name("foo.bar") == "foo___bar" + + def test_make_module_translation_map(self) -> None: + assert make_module_translation_map(["foo", "bar"]) == {"foo": "foo.", "bar": "bar."} + assert make_module_translation_map(["foo.bar", "foo.baz"]) == { + "foo.bar": "bar.", + "foo.baz": "baz.", + } + assert make_module_translation_map(["zar", "foo.bar", "foo.baz"]) == { + "foo.bar": "bar.", + "foo.baz": "baz.", + "zar": "zar.", + } + assert make_module_translation_map(["foo.bar", "fu.bar", "foo.baz"]) == { + "foo.bar": "foo.bar.", + "fu.bar": "fu.bar.", + "foo.baz": "baz.", + } + assert make_module_translation_map(["foo", "foo.foo", "bar.foo", "bar.foo.bar.foo"]) == { + "foo": "foo.", + "foo.foo": "foo.foo.", + "bar.foo": "bar.foo.", + "bar.foo.bar.foo": "foo.bar.foo.", + } + + def test_name_generator(self) -> None: + g = NameGenerator([["foo", "foo.zar"]]) + assert g.private_name("foo", "f") == "foo___f" + assert g.private_name("foo", "C.x.y") == "foo___C___x___y" + assert g.private_name("foo", "C.x.y") == "foo___C___x___y" + assert g.private_name("foo.zar", "C.x.y") == "zar___C___x___y" + assert g.private_name("foo", "C.x_y") == "foo___C___x_y" + assert g.private_name("foo", "C_x_y") == "foo___C_x_y" + assert g.private_name("foo", "C_x_y") == "foo___C_x_y" + assert g.private_name("foo", "___") == "foo______3_" diff --git a/mypyc/test/test_optimizations.py b/mypyc/test/test_optimizations.py new file mode 100644 index 000000000000..3f1f46ac1dd7 --- /dev/null +++ b/mypyc/test/test_optimizations.py @@ -0,0 +1,68 @@ +"""Runner for IR optimization tests.""" + +from __future__ import annotations + +import os.path + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.common import TOP_LEVEL_NAME +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.pprint import format_func +from mypyc.options import CompilerOptions +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file, + remove_comment_lines, + use_custom_builtins, +) +from mypyc.transform.copy_propagation import do_copy_propagation +from mypyc.transform.flag_elimination import do_flag_elimination +from mypyc.transform.uninit import insert_uninit_checks + + +class OptimizationSuite(MypycDataSuite): + """Base class for IR optimization test suites. + + To use this, add a base class and define "files" and "do_optimizations". + """ + + base_path = test_temp_dir + + def run_case(self, testcase: DataDrivenTestCase) -> None: + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + expected_output = remove_comment_lines(testcase.output) + try: + ir = build_ir_for_single_file(testcase.input) + except CompileError as e: + actual = e.messages + else: + actual = [] + for fn in ir: + if fn.name == TOP_LEVEL_NAME and not testcase.name.endswith("_toplevel"): + continue + insert_uninit_checks(fn) + self.do_optimizations(fn) + actual.extend(format_func(fn)) + + assert_test_output(testcase, actual, "Invalid source code output", expected_output) + + def do_optimizations(self, fn: FuncIR) -> None: + raise NotImplementedError + + +class TestCopyPropagation(OptimizationSuite): + files = ["opt-copy-propagation.test"] + + def do_optimizations(self, fn: FuncIR) -> None: + do_copy_propagation(fn, CompilerOptions()) + + +class TestFlagElimination(OptimizationSuite): + files = ["opt-flag-elimination.test"] + + def do_optimizations(self, fn: FuncIR) -> None: + do_flag_elimination(fn, CompilerOptions()) diff --git a/mypyc/test/test_pprint.py b/mypyc/test/test_pprint.py new file mode 100644 index 000000000000..d9e2bdb7fc92 --- /dev/null +++ b/mypyc/test/test_pprint.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import unittest + +from mypyc.ir.ops import Assign, BasicBlock, Integer, IntOp, Op, Register, Unreachable +from mypyc.ir.pprint import generate_names_for_ir +from mypyc.ir.rtypes import int_rprimitive + + +def register(name: str) -> Register: + return Register(int_rprimitive, "foo", is_arg=True) + + +def make_block(ops: list[Op]) -> BasicBlock: + block = BasicBlock() + block.ops.extend(ops) + return block + + +class TestGenerateNames(unittest.TestCase): + def test_empty(self) -> None: + assert generate_names_for_ir([], []) == {} + + def test_arg(self) -> None: + reg = register("foo") + assert generate_names_for_ir([reg], []) == {reg: "foo"} + + def test_int_op(self) -> None: + n1 = Integer(2) + n2 = Integer(4) + op1 = IntOp(int_rprimitive, n1, n2, IntOp.ADD) + op2 = IntOp(int_rprimitive, op1, n2, IntOp.ADD) + block = make_block([op1, op2, Unreachable()]) + assert generate_names_for_ir([], [block]) == {op1: "r0", op2: "r1"} + + def test_assign(self) -> None: + reg = register("foo") + n = Integer(2) + op1 = Assign(reg, n) + op2 = Assign(reg, n) + block = make_block([op1, op2]) + assert generate_names_for_ir([reg], [block]) == {reg: "foo"} diff --git a/mypyc/test/test_rarray.py b/mypyc/test/test_rarray.py new file mode 100644 index 000000000000..b8d788b4f336 --- /dev/null +++ b/mypyc/test/test_rarray.py @@ -0,0 +1,48 @@ +"""Unit tests for RArray types.""" + +from __future__ import annotations + +import unittest + +from mypyc.common import PLATFORM_SIZE +from mypyc.ir.rtypes import ( + RArray, + bool_rprimitive, + compute_rtype_alignment, + compute_rtype_size, + int_rprimitive, +) + + +class TestRArray(unittest.TestCase): + def test_basics(self) -> None: + a = RArray(int_rprimitive, 10) + assert a.item_type == int_rprimitive + assert a.length == 10 + + def test_str_conversion(self) -> None: + a = RArray(int_rprimitive, 10) + assert str(a) == "int[10]" + assert repr(a) == "[10]>" + + def test_eq(self) -> None: + a = RArray(int_rprimitive, 10) + assert a == RArray(int_rprimitive, 10) + assert a != RArray(bool_rprimitive, 10) + assert a != RArray(int_rprimitive, 9) + + def test_hash(self) -> None: + assert hash(RArray(int_rprimitive, 10)) == hash(RArray(int_rprimitive, 10)) + assert hash(RArray(bool_rprimitive, 5)) == hash(RArray(bool_rprimitive, 5)) + + def test_alignment(self) -> None: + a = RArray(int_rprimitive, 10) + assert compute_rtype_alignment(a) == PLATFORM_SIZE + b = RArray(bool_rprimitive, 55) + assert compute_rtype_alignment(b) == 1 + + def test_size(self) -> None: + a = RArray(int_rprimitive, 9) + assert compute_rtype_size(a) == 9 * PLATFORM_SIZE + b = RArray(bool_rprimitive, 3) + assert compute_rtype_size(b) == 3 diff --git a/mypyc/test/test_refcount.py b/mypyc/test/test_refcount.py new file mode 100644 index 000000000000..afeda89682ce --- /dev/null +++ b/mypyc/test/test_refcount.py @@ -0,0 +1,59 @@ +"""Test runner for reference count opcode insertion transform test cases. + +The transform inserts needed reference count increment/decrement +operations to IR. +""" + +from __future__ import annotations + +import os.path + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.common import TOP_LEVEL_NAME +from mypyc.ir.pprint import format_func +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file, + infer_ir_build_options_from_test_name, + remove_comment_lines, + replace_word_size, + use_custom_builtins, +) +from mypyc.transform.refcount import insert_ref_count_opcodes +from mypyc.transform.uninit import insert_uninit_checks + +files = ["refcount.test"] + + +class TestRefCountTransform(MypycDataSuite): + files = files + base_path = test_temp_dir + optional_out = True + + def run_case(self, testcase: DataDrivenTestCase) -> None: + """Perform a runtime checking transformation test case.""" + options = infer_ir_build_options_from_test_name(testcase.name) + if options is None: + # Skipped test case + return + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + expected_output = remove_comment_lines(testcase.output) + expected_output = replace_word_size(expected_output) + try: + ir = build_ir_for_single_file(testcase.input, options) + except CompileError as e: + actual = e.messages + else: + actual = [] + for fn in ir: + if fn.name == TOP_LEVEL_NAME and not testcase.name.endswith("_toplevel"): + continue + insert_uninit_checks(fn) + insert_ref_count_opcodes(fn) + actual.extend(format_func(fn)) + + assert_test_output(testcase, actual, "Invalid source code output", expected_output) diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py new file mode 100644 index 000000000000..dd3c79da7b9b --- /dev/null +++ b/mypyc/test/test_run.py @@ -0,0 +1,442 @@ +"""Test cases for building an C extension and running it.""" + +from __future__ import annotations + +import ast +import contextlib +import glob +import os.path +import re +import shutil +import subprocess +import sys +import time +from typing import Any, Iterator + +from mypy import build +from mypy.errors import CompileError +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypy.test.helpers import assert_module_equivalence, perform_file_operations +from mypyc.build import construct_groups +from mypyc.codegen import emitmodule +from mypyc.errors import Errors +from mypyc.options import CompilerOptions +from mypyc.test.test_serialization import check_serialization_roundtrip +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + TESTUTIL_PATH, + MypycDataSuite, + assert_test_output, + fudge_dir_mtimes, + show_c, + use_custom_builtins, +) + +files = [ + "run-async.test", + "run-misc.test", + "run-functions.test", + "run-integers.test", + "run-i64.test", + "run-i32.test", + "run-i16.test", + "run-u8.test", + "run-floats.test", + "run-math.test", + "run-bools.test", + "run-strings.test", + "run-bytes.test", + "run-tuples.test", + "run-lists.test", + "run-dicts.test", + "run-sets.test", + "run-primitives.test", + "run-loops.test", + "run-exceptions.test", + "run-imports.test", + "run-classes.test", + "run-traits.test", + "run-generators.test", + "run-multimodule.test", + "run-bench.test", + "run-mypy-sim.test", + "run-dunders.test", + "run-dunders-special.test", + "run-singledispatch.test", + "run-attrs.test", + "run-python37.test", + "run-python38.test", +] + +if sys.version_info >= (3, 10): + files.append("run-match.test") +if sys.version_info >= (3, 12): + files.append("run-python312.test") + +setup_format = """\ +from setuptools import setup +from mypyc.build import mypycify + +setup(name='test_run_output', + ext_modules=mypycify({}, separate={}, skip_cgen_input={!r}, strip_asserts=False, + multi_file={}, opt_level='{}'), +) +""" + +WORKDIR = "build" + + +def run_setup(script_name: str, script_args: list[str]) -> bool: + """Run a setup script in a somewhat controlled environment. + + This is adapted from code in distutils and our goal here is that is + faster to not need to spin up a python interpreter to run it. + + We had to fork it because the real run_setup swallows errors + and KeyboardInterrupt with no way to recover them (!). + The real version has some extra features that we removed since + we weren't using them. + + Returns whether the setup succeeded. + """ + save_argv = sys.argv.copy() + g = {"__file__": script_name} + try: + try: + sys.argv[0] = script_name + sys.argv[1:] = script_args + with open(script_name, "rb") as f: + exec(f.read(), g) + finally: + sys.argv = save_argv + except SystemExit as e: + # distutils converts KeyboardInterrupt into a SystemExit with + # "interrupted" as the argument. Convert it back so that + # pytest will exit instead of just failing the test. + if e.code == "interrupted": + raise KeyboardInterrupt from e + + return e.code == 0 or e.code is None + + return True + + +@contextlib.contextmanager +def chdir_manager(target: str) -> Iterator[None]: + dir = os.getcwd() + os.chdir(target) + try: + yield + finally: + os.chdir(dir) + + +class TestRun(MypycDataSuite): + """Test cases that build a C extension and run code.""" + + files = files + base_path = test_temp_dir + optional_out = True + multi_file = False + separate = False # If True, using separate (incremental) compilation + strict_dunder_typing = False + + def run_case(self, testcase: DataDrivenTestCase) -> None: + # setup.py wants to be run from the root directory of the package, which we accommodate + # by chdiring into tmp/ + with use_custom_builtins( + os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase + ), chdir_manager("tmp"): + self.run_case_inner(testcase) + + def run_case_inner(self, testcase: DataDrivenTestCase) -> None: + if not os.path.isdir(WORKDIR): # (one test puts something in build...) + os.mkdir(WORKDIR) + + text = "\n".join(testcase.input) + + with open("native.py", "w", encoding="utf-8") as f: + f.write(text) + with open("interpreted.py", "w", encoding="utf-8") as f: + f.write(text) + + shutil.copyfile(TESTUTIL_PATH, "testutil.py") + + step = 1 + self.run_case_step(testcase, step) + + steps = testcase.find_steps() + if steps == [[]]: + steps = [] + + for operations in steps: + # To make sure that any new changes get picked up as being + # new by distutils, shift the mtime of all of the + # generated artifacts back by a second. + fudge_dir_mtimes(WORKDIR, -1) + # On some OS, changing the mtime doesn't work reliably. As + # a workaround, sleep. + # TODO: Figure out a better approach, since this slows down tests. + time.sleep(1.0) + + step += 1 + with chdir_manager(".."): + perform_file_operations(operations) + self.run_case_step(testcase, step) + + def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> None: + bench = testcase.config.getoption("--bench", False) and "Benchmark" in testcase.name + + options = Options() + options.use_builtins_fixtures = True + options.show_traceback = True + options.strict_optional = True + options.python_version = sys.version_info[:2] + options.export_types = True + options.preserve_asts = True + options.allow_empty_bodies = True + options.incremental = self.separate + + # Avoid checking modules/packages named 'unchecked', to provide a way + # to test interacting with code we don't have types for. + options.per_module_options["unchecked.*"] = {"follow_imports": "error"} + + source = build.BuildSource("native.py", "native", None) + sources = [source] + module_names = ["native"] + module_paths = ["native.py"] + + # Hard code another module name to compile in the same compilation unit. + to_delete = [] + for fn, text in testcase.files: + fn = os.path.relpath(fn, test_temp_dir) + + if os.path.basename(fn).startswith("other") and fn.endswith(".py"): + name = fn.split(".")[0].replace(os.sep, ".") + module_names.append(name) + sources.append(build.BuildSource(fn, name, None)) + to_delete.append(fn) + module_paths.append(fn) + + shutil.copyfile(fn, os.path.join(os.path.dirname(fn), name + "_interpreted.py")) + + for source in sources: + options.per_module_options.setdefault(source.module, {})["mypyc"] = True + + separate = ( + self.get_separate("\n".join(testcase.input), incremental_step) + if self.separate + else False + ) + + groups = construct_groups(sources, separate, len(module_names) > 1) + + try: + compiler_options = CompilerOptions( + multi_file=self.multi_file, + separate=self.separate, + strict_dunder_typing=self.strict_dunder_typing, + ) + result = emitmodule.parse_and_typecheck( + sources=sources, + options=options, + compiler_options=compiler_options, + groups=groups, + alt_lib_path=".", + ) + errors = Errors(options) + ir, cfiles = emitmodule.compile_modules_to_c( + result, compiler_options=compiler_options, errors=errors, groups=groups + ) + if errors.num_errors: + errors.flush_errors() + assert False, "Compile error" + except CompileError as e: + for line in e.messages: + print(fix_native_line_number(line, testcase.file, testcase.line)) + assert False, "Compile error" + + # Check that serialization works on this IR. (Only on the first + # step because the returned ir only includes updated code.) + if incremental_step == 1: + check_serialization_roundtrip(ir) + + opt_level = int(os.environ.get("MYPYC_OPT_LEVEL", 0)) + debug_level = int(os.environ.get("MYPYC_DEBUG_LEVEL", 0)) + + setup_file = os.path.abspath(os.path.join(WORKDIR, "setup.py")) + # We pass the C file information to the build script via setup.py unfortunately + with open(setup_file, "w", encoding="utf-8") as f: + f.write( + setup_format.format( + module_paths, separate, cfiles, self.multi_file, opt_level, debug_level + ) + ) + + if not run_setup(setup_file, ["build_ext", "--inplace"]): + if testcase.config.getoption("--mypyc-showc"): + show_c(cfiles) + assert False, "Compilation failed" + + # Assert that an output file got created + suffix = "pyd" if sys.platform == "win32" else "so" + assert glob.glob(f"native.*.{suffix}") or glob.glob(f"native.{suffix}") + + driver_path = "driver.py" + if not os.path.isfile(driver_path): + # No driver.py provided by test case. Use the default one + # (mypyc/test-data/driver/driver.py) that calls each + # function named test_*. + default_driver = os.path.join( + os.path.dirname(__file__), "..", "test-data", "driver", "driver.py" + ) + shutil.copy(default_driver, driver_path) + env = os.environ.copy() + env["MYPYC_RUN_BENCH"] = "1" if bench else "0" + + debugger = testcase.config.getoption("debugger") + if debugger: + if debugger == "lldb": + subprocess.check_call(["lldb", "--", sys.executable, driver_path], env=env) + elif debugger == "gdb": + subprocess.check_call(["gdb", "--args", sys.executable, driver_path], env=env) + else: + assert False, "Unsupported debugger" + # TODO: find a way to automatically disable capturing + # stdin/stdout when in debugging mode + assert False, ( + "Test can't pass in debugging mode. " + "(Make sure to pass -s to pytest to interact with the debugger)" + ) + proc = subprocess.Popen( + [sys.executable, driver_path], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + ) + if sys.version_info >= (3, 12): + # TODO: testDecorators1 hangs on 3.12, remove this once fixed + proc.wait(timeout=30) + output = proc.communicate()[0].decode("utf8") + outlines = output.splitlines() + + if testcase.config.getoption("--mypyc-showc"): + show_c(cfiles) + if proc.returncode != 0: + print() + print("*** Exit status: %d" % proc.returncode) + + # Verify output. + if bench: + print("Test output:") + print(output) + else: + if incremental_step == 1: + msg = "Invalid output" + expected = testcase.output + else: + msg = f"Invalid output (step {incremental_step})" + expected = testcase.output2.get(incremental_step, []) + + if not expected: + # Tweak some line numbers, but only if the expected output is empty, + # as tweaked output might not match expected output. + outlines = [ + fix_native_line_number(line, testcase.file, testcase.line) for line in outlines + ] + assert_test_output(testcase, outlines, msg, expected) + + if incremental_step > 1 and options.incremental: + suffix = "" if incremental_step == 2 else str(incremental_step - 1) + expected_rechecked = testcase.expected_rechecked_modules.get(incremental_step - 1) + if expected_rechecked is not None: + assert_module_equivalence( + "rechecked" + suffix, expected_rechecked, result.manager.rechecked_modules + ) + expected_stale = testcase.expected_stale_modules.get(incremental_step - 1) + if expected_stale is not None: + assert_module_equivalence( + "stale" + suffix, expected_stale, result.manager.stale_modules + ) + + assert proc.returncode == 0 + + def get_separate(self, program_text: str, incremental_step: int) -> Any: + template = r"# separate{}: (\[.*\])$" + m = re.search(template.format(incremental_step), program_text, flags=re.MULTILINE) + if not m: + m = re.search(template.format(""), program_text, flags=re.MULTILINE) + if m: + return ast.literal_eval(m.group(1)) + else: + return True + + +class TestRunMultiFile(TestRun): + """Run the main multi-module tests in multi-file compilation mode. + + In multi-file mode each module gets compiled into a separate C file, + but all modules (C files) are compiled together. + """ + + multi_file = True + test_name_suffix = "_multi" + files = ["run-multimodule.test", "run-mypy-sim.test"] + + +class TestRunSeparate(TestRun): + """Run the main multi-module tests in separate compilation mode. + + In this mode there are multiple compilation groups, which are compiled + incrementally. Each group is compiled to a separate C file, and these C + files are compiled separately. + + Each compiled module is placed into a separate compilation group, unless + overridden by a special comment. Consider this example: + + # separate: [(["other.py", "other_b.py"], "stuff")] + + This puts other.py and other_b.py into a compilation group named "stuff". + Any files not mentioned in the comment will get single-file groups. + """ + + separate = True + test_name_suffix = "_separate" + files = ["run-multimodule.test", "run-mypy-sim.test"] + + +class TestRunStrictDunderTyping(TestRun): + """Run the tests with strict dunder typing.""" + + strict_dunder_typing = True + test_name_suffix = "_dunder_typing" + files = ["run-dunders.test", "run-floats.test"] + + +def fix_native_line_number(message: str, fnam: str, delta: int) -> str: + """Update code locations in test case output to point to the .test file. + + The description of the test case is written to native.py, and line numbers + in test case output often are relative to native.py. This translates the + line numbers to be relative to the .test file that contains the test case + description, and also updates the file name to the .test file name. + + Args: + message: message to update + fnam: path of the .test file + delta: line number of the beginning of the test case in the .test file + + Returns updated message (or original message if we couldn't find anything). + """ + fnam = os.path.basename(fnam) + message = re.sub( + r"native\.py:([0-9]+):", lambda m: "%s:%d:" % (fnam, int(m.group(1)) + delta), message + ) + message = re.sub( + r'"native.py", line ([0-9]+),', + lambda m: '"%s", line %d,' % (fnam, int(m.group(1)) + delta), + message, + ) + return message diff --git a/mypyc/test/test_serialization.py b/mypyc/test/test_serialization.py new file mode 100644 index 000000000000..19de05d32cf1 --- /dev/null +++ b/mypyc/test/test_serialization.py @@ -0,0 +1,108 @@ +"""Functions to check that serialization round-tripped properly.""" + +# This file is named test_serialization.py even though it doesn't +# contain its own tests so that pytest will rewrite the asserts... + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature +from mypyc.ir.module_ir import ModuleIR, deserialize_modules +from mypyc.ir.ops import DeserMaps +from mypyc.ir.rtypes import RType +from mypyc.sametype import is_same_signature, is_same_type + + +def get_dict(x: Any) -> dict[str, Any]: + if hasattr(x, "__mypyc_attrs__"): + return {k: getattr(x, k) for k in x.__mypyc_attrs__ if hasattr(x, k)} + else: + return dict(x.__dict__) + + +def get_function_dict(x: FuncIR) -> dict[str, Any]: + """Get a dict of function attributes safe to compare across serialization""" + d = get_dict(x) + d.pop("blocks", None) + d.pop("env", None) + return d + + +def assert_blobs_same(x: Any, y: Any, trail: tuple[Any, ...]) -> None: + """Compare two blobs of IR as best we can. + + FuncDecls, FuncIRs, and ClassIRs are compared by fullname to avoid + infinite recursion. + (More detailed comparisons should be done manually.) + + Types and signatures are compared using mypyc.sametype. + + Containers are compared recursively. + + Anything else is compared with ==. + + The `trail` argument is used in error messages. + """ + + assert type(x) is type(y), (f"Type mismatch at {trail}", type(x), type(y)) + if isinstance(x, (FuncDecl, FuncIR, ClassIR)): + assert x.fullname == y.fullname, f"Name mismatch at {trail}" + elif isinstance(x, dict): + assert len(x.keys()) == len(y.keys()), f"Keys mismatch at {trail}" + for (xk, xv), (yk, yv) in zip(x.items(), y.items()): + assert_blobs_same(xk, yk, trail + ("keys",)) + assert_blobs_same(xv, yv, trail + (xk,)) + elif isinstance(x, dict): + assert x.keys() == y.keys(), f"Keys mismatch at {trail}" + for k in x.keys(): + assert_blobs_same(x[k], y[k], trail + (k,)) + elif isinstance(x, Iterable) and not isinstance(x, (str, set)): + # Special case iterables to generate better assert error messages. + # We can't use this for sets since the ordering is unpredictable, + # and strings should be treated as atomic values. + for i, (xv, yv) in enumerate(zip(x, y)): + assert_blobs_same(xv, yv, trail + (i,)) + elif isinstance(x, RType): + assert is_same_type(x, y), f"RType mismatch at {trail}" + elif isinstance(x, FuncSignature): + assert is_same_signature(x, y), f"Signature mismatch at {trail}" + else: + assert x == y, f"Value mismatch at {trail}" + + +def assert_modules_same(ir1: ModuleIR, ir2: ModuleIR) -> None: + """Assert that two module IRs are the same (*). + + * Or rather, as much as we care about preserving across + serialization. We drop the actual IR bodies of functions but try + to preserve everything else. + """ + assert ir1.fullname == ir2.fullname + + assert ir1.imports == ir2.imports + + for cls1, cls2 in zip(ir1.classes, ir2.classes): + assert_blobs_same(get_dict(cls1), get_dict(cls2), (ir1.fullname, cls1.fullname)) + + for fn1, fn2 in zip(ir1.functions, ir2.functions): + assert_blobs_same( + get_function_dict(fn1), get_function_dict(fn2), (ir1.fullname, fn1.fullname) + ) + assert_blobs_same(get_dict(fn1.decl), get_dict(fn2.decl), (ir1.fullname, fn1.fullname)) + + assert_blobs_same(ir1.final_names, ir2.final_names, (ir1.fullname, "final_names")) + + +def check_serialization_roundtrip(irs: dict[str, ModuleIR]) -> None: + """Check that we can serialize modules out and deserialize them to the same thing.""" + serialized = {k: ir.serialize() for k, ir in irs.items()} + + ctx = DeserMaps({}, {}) + irs2 = deserialize_modules(serialized, ctx) + assert irs.keys() == irs2.keys() + + for k in irs: + assert_modules_same(irs[k], irs2[k]) diff --git a/mypyc/test/test_struct.py b/mypyc/test/test_struct.py new file mode 100644 index 000000000000..82990e6afd82 --- /dev/null +++ b/mypyc/test/test_struct.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import unittest + +from mypyc.ir.rtypes import ( + RStruct, + bool_rprimitive, + int32_rprimitive, + int64_rprimitive, + int_rprimitive, + object_rprimitive, +) +from mypyc.rt_subtype import is_runtime_subtype + + +class TestStruct(unittest.TestCase): + def test_struct_offsets(self) -> None: + # test per-member alignment + r = RStruct("", [], [bool_rprimitive, int32_rprimitive, int64_rprimitive]) + assert r.size == 16 + assert r.offsets == [0, 4, 8] + + # test final alignment + r1 = RStruct("", [], [bool_rprimitive, bool_rprimitive]) + assert r1.size == 2 + assert r1.offsets == [0, 1] + r2 = RStruct("", [], [int32_rprimitive, bool_rprimitive]) + r3 = RStruct("", [], [int64_rprimitive, bool_rprimitive]) + assert r2.offsets == [0, 4] + assert r3.offsets == [0, 8] + assert r2.size == 8 + assert r3.size == 16 + + r4 = RStruct("", [], [bool_rprimitive, bool_rprimitive, bool_rprimitive, int32_rprimitive]) + assert r4.size == 8 + assert r4.offsets == [0, 1, 2, 4] + + # test nested struct + r5 = RStruct("", [], [bool_rprimitive, r]) + assert r5.offsets == [0, 8] + assert r5.size == 24 + r6 = RStruct("", [], [int32_rprimitive, r5]) + assert r6.offsets == [0, 8] + assert r6.size == 32 + # test nested struct with alignment less than 8 + r7 = RStruct("", [], [bool_rprimitive, r4]) + assert r7.offsets == [0, 4] + assert r7.size == 12 + + def test_struct_str(self) -> None: + r = RStruct("Foo", ["a", "b"], [bool_rprimitive, object_rprimitive]) + assert str(r) == "Foo{a:bool, b:object}" + assert ( + repr(r) == ", " + "b:}>" + ) + r1 = RStruct("Bar", ["c"], [int32_rprimitive]) + assert str(r1) == "Bar{c:i32}" + assert repr(r1) == "}>" + r2 = RStruct("Baz", [], []) + assert str(r2) == "Baz{}" + assert repr(r2) == "" + + def test_runtime_subtype(self) -> None: + # right type to check with + r = RStruct("Foo", ["a", "b"], [bool_rprimitive, int_rprimitive]) + + # using the exact same fields + r1 = RStruct("Foo", ["a", "b"], [bool_rprimitive, int_rprimitive]) + + # names different + r2 = RStruct("Bar", ["c", "b"], [bool_rprimitive, int_rprimitive]) + + # name different + r3 = RStruct("Baz", ["a", "b"], [bool_rprimitive, int_rprimitive]) + + # type different + r4 = RStruct("FooBar", ["a", "b"], [bool_rprimitive, int32_rprimitive]) + + # number of types different + r5 = RStruct( + "FooBarBaz", ["a", "b", "c"], [bool_rprimitive, int_rprimitive, bool_rprimitive] + ) + + assert is_runtime_subtype(r1, r) is True + assert is_runtime_subtype(r2, r) is False + assert is_runtime_subtype(r3, r) is False + assert is_runtime_subtype(r4, r) is False + assert is_runtime_subtype(r5, r) is False + + def test_eq_and_hash(self) -> None: + r = RStruct("Foo", ["a", "b"], [bool_rprimitive, int_rprimitive]) + + # using the exact same fields + r1 = RStruct("Foo", ["a", "b"], [bool_rprimitive, int_rprimitive]) + assert hash(r) == hash(r1) + assert r == r1 + + # different name + r2 = RStruct("Foq", ["a", "b"], [bool_rprimitive, int_rprimitive]) + assert hash(r) != hash(r2) + assert r != r2 + + # different names + r3 = RStruct("Foo", ["a", "c"], [bool_rprimitive, int_rprimitive]) + assert hash(r) != hash(r3) + assert r != r3 + + # different type + r4 = RStruct("Foo", ["a", "b"], [bool_rprimitive, int_rprimitive, bool_rprimitive]) + assert hash(r) != hash(r4) + assert r != r4 diff --git a/mypyc/test/test_tuplename.py b/mypyc/test/test_tuplename.py new file mode 100644 index 000000000000..5dd51d45c16f --- /dev/null +++ b/mypyc/test/test_tuplename.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import unittest + +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.rtypes import ( + RInstance, + RTuple, + RUnion, + bool_rprimitive, + int_rprimitive, + list_rprimitive, + object_rprimitive, +) + + +class TestTupleNames(unittest.TestCase): + def setUp(self) -> None: + self.inst_a = RInstance(ClassIR("A", "__main__")) + self.inst_b = RInstance(ClassIR("B", "__main__")) + + def test_names(self) -> None: + assert RTuple([int_rprimitive, int_rprimitive]).unique_id == "T2II" + assert RTuple([list_rprimitive, object_rprimitive, self.inst_a]).unique_id == "T3OOO" + assert RTuple([list_rprimitive, object_rprimitive, self.inst_b]).unique_id == "T3OOO" + assert RTuple([]).unique_id == "T0" + assert ( + RTuple([RTuple([]), RTuple([int_rprimitive, int_rprimitive])]).unique_id == "T2T0T2II" + ) + assert ( + RTuple([bool_rprimitive, RUnion([bool_rprimitive, int_rprimitive])]).unique_id + == "T2CO" + ) diff --git a/mypyc/test/test_typeops.py b/mypyc/test/test_typeops.py new file mode 100644 index 000000000000..ff2c05ad983e --- /dev/null +++ b/mypyc/test/test_typeops.py @@ -0,0 +1,97 @@ +"""Test cases for various RType operations.""" + +from __future__ import annotations + +import unittest + +from mypyc.ir.rtypes import ( + RUnion, + bit_rprimitive, + bool_rprimitive, + int16_rprimitive, + int32_rprimitive, + int64_rprimitive, + int_rprimitive, + object_rprimitive, + short_int_rprimitive, + str_rprimitive, +) +from mypyc.rt_subtype import is_runtime_subtype +from mypyc.subtype import is_subtype + +native_int_types = [int64_rprimitive, int32_rprimitive, int16_rprimitive] + + +class TestSubtype(unittest.TestCase): + def test_bit(self) -> None: + assert is_subtype(bit_rprimitive, bool_rprimitive) + assert is_subtype(bit_rprimitive, int_rprimitive) + assert is_subtype(bit_rprimitive, short_int_rprimitive) + for rt in native_int_types: + assert is_subtype(bit_rprimitive, rt) + + def test_bool(self) -> None: + assert not is_subtype(bool_rprimitive, bit_rprimitive) + assert is_subtype(bool_rprimitive, int_rprimitive) + assert is_subtype(bool_rprimitive, short_int_rprimitive) + for rt in native_int_types: + assert is_subtype(bool_rprimitive, rt) + + def test_int64(self) -> None: + assert is_subtype(int64_rprimitive, int64_rprimitive) + assert is_subtype(int64_rprimitive, int_rprimitive) + assert not is_subtype(int64_rprimitive, short_int_rprimitive) + assert not is_subtype(int64_rprimitive, int32_rprimitive) + assert not is_subtype(int64_rprimitive, int16_rprimitive) + + def test_int32(self) -> None: + assert is_subtype(int32_rprimitive, int32_rprimitive) + assert is_subtype(int32_rprimitive, int_rprimitive) + assert not is_subtype(int32_rprimitive, short_int_rprimitive) + assert not is_subtype(int32_rprimitive, int64_rprimitive) + assert not is_subtype(int32_rprimitive, int16_rprimitive) + + def test_int16(self) -> None: + assert is_subtype(int16_rprimitive, int16_rprimitive) + assert is_subtype(int16_rprimitive, int_rprimitive) + assert not is_subtype(int16_rprimitive, short_int_rprimitive) + assert not is_subtype(int16_rprimitive, int64_rprimitive) + assert not is_subtype(int16_rprimitive, int32_rprimitive) + + +class TestRuntimeSubtype(unittest.TestCase): + def test_bit(self) -> None: + assert is_runtime_subtype(bit_rprimitive, bool_rprimitive) + assert not is_runtime_subtype(bit_rprimitive, int_rprimitive) + + def test_bool(self) -> None: + assert not is_runtime_subtype(bool_rprimitive, bit_rprimitive) + assert not is_runtime_subtype(bool_rprimitive, int_rprimitive) + + def test_union(self) -> None: + bool_int_mix = RUnion([bool_rprimitive, int_rprimitive]) + assert not is_runtime_subtype(bool_int_mix, short_int_rprimitive) + assert not is_runtime_subtype(bool_int_mix, int_rprimitive) + assert not is_runtime_subtype(short_int_rprimitive, bool_int_mix) + assert not is_runtime_subtype(int_rprimitive, bool_int_mix) + + +class TestUnionSimplification(unittest.TestCase): + def test_simple_type_result(self) -> None: + assert RUnion.make_simplified_union([int_rprimitive]) == int_rprimitive + + def test_remove_duplicate(self) -> None: + assert RUnion.make_simplified_union([int_rprimitive, int_rprimitive]) == int_rprimitive + + def test_cannot_simplify(self) -> None: + assert RUnion.make_simplified_union( + [int_rprimitive, str_rprimitive, object_rprimitive] + ) == RUnion([int_rprimitive, str_rprimitive, object_rprimitive]) + + def test_nested(self) -> None: + assert RUnion.make_simplified_union( + [int_rprimitive, RUnion([str_rprimitive, int_rprimitive])] + ) == RUnion([int_rprimitive, str_rprimitive]) + assert RUnion.make_simplified_union( + [int_rprimitive, RUnion([str_rprimitive, RUnion([int_rprimitive])])] + ) == RUnion([int_rprimitive, str_rprimitive]) diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py new file mode 100644 index 000000000000..6446af3427af --- /dev/null +++ b/mypyc/test/testutil.py @@ -0,0 +1,283 @@ +"""Helpers for writing tests""" + +from __future__ import annotations + +import contextlib +import os +import os.path +import re +import shutil +from typing import Callable, Iterator + +from mypy import build +from mypy.errors import CompileError +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal +from mypyc.analysis.ircheck import assert_func_ir_valid +from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE +from mypyc.errors import Errors +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.module_ir import ModuleIR +from mypyc.irbuild.main import build_ir +from mypyc.irbuild.mapper import Mapper +from mypyc.options import CompilerOptions +from mypyc.test.config import test_data_prefix + +# The builtins stub used during icode generation test cases. +ICODE_GEN_BUILTINS = os.path.join(test_data_prefix, "fixtures/ir.py") +# The testutil support library +TESTUTIL_PATH = os.path.join(test_data_prefix, "fixtures/testutil.py") + + +class MypycDataSuite(DataSuite): + # Need to list no files, since this will be picked up as a suite of tests + files: list[str] = [] + data_prefix = test_data_prefix + + +def builtins_wrapper( + func: Callable[[DataDrivenTestCase], None], path: str +) -> Callable[[DataDrivenTestCase], None]: + """Decorate a function that implements a data-driven test case to copy an + alternative builtins module implementation in place before performing the + test case. Clean up after executing the test case. + """ + return lambda testcase: perform_test(func, path, testcase) + + +@contextlib.contextmanager +def use_custom_builtins(builtins_path: str, testcase: DataDrivenTestCase) -> Iterator[None]: + for path, _ in testcase.files: + if os.path.basename(path) == "builtins.pyi": + default_builtins = False + break + else: + # Use default builtins. + builtins = os.path.abspath(os.path.join(test_temp_dir, "builtins.pyi")) + shutil.copyfile(builtins_path, builtins) + default_builtins = True + + # Actually perform the test case. + try: + yield None + finally: + if default_builtins: + # Clean up. + os.remove(builtins) + + +def perform_test( + func: Callable[[DataDrivenTestCase], None], builtins_path: str, testcase: DataDrivenTestCase +) -> None: + for path, _ in testcase.files: + if os.path.basename(path) == "builtins.py": + default_builtins = False + break + else: + # Use default builtins. + builtins = os.path.join(test_temp_dir, "builtins.py") + shutil.copyfile(builtins_path, builtins) + default_builtins = True + + # Actually perform the test case. + func(testcase) + + if default_builtins: + # Clean up. + os.remove(builtins) + + +def build_ir_for_single_file( + input_lines: list[str], compiler_options: CompilerOptions | None = None +) -> list[FuncIR]: + return build_ir_for_single_file2(input_lines, compiler_options).functions + + +def build_ir_for_single_file2( + input_lines: list[str], compiler_options: CompilerOptions | None = None +) -> ModuleIR: + program_text = "\n".join(input_lines) + + # By default generate IR compatible with the earliest supported Python C API. + # If a test needs more recent API features, this should be overridden. + compiler_options = compiler_options or CompilerOptions(capi_version=(3, 7)) + options = Options() + options.show_traceback = True + options.hide_error_codes = True + options.use_builtins_fixtures = True + options.strict_optional = True + options.python_version = compiler_options.python_version or (3, 6) + options.export_types = True + options.preserve_asts = True + options.allow_empty_bodies = True + options.per_module_options["__main__"] = {"mypyc": True} + + source = build.BuildSource("main", "__main__", program_text) + # Construct input as a single single. + # Parse and type check the input program. + result = build.build(sources=[source], options=options, alt_lib_path=test_temp_dir) + if result.errors: + raise CompileError(result.errors) + + errors = Errors(options) + modules = build_ir( + [result.files["__main__"]], + result.graph, + result.types, + Mapper({"__main__": None}), + compiler_options, + errors, + ) + if errors.num_errors: + raise CompileError(errors.new_messages()) + + module = list(modules.values())[0] + for fn in module.functions: + assert_func_ir_valid(fn) + return module + + +def update_testcase_output(testcase: DataDrivenTestCase, output: list[str]) -> None: + # TODO: backport this to mypy + assert testcase.old_cwd is not None, "test was not properly set up" + testcase_path = os.path.join(testcase.old_cwd, testcase.file) + with open(testcase_path) as f: + data_lines = f.read().splitlines() + + # We can't rely on the test line numbers to *find* the test, since + # we might fix multiple tests in a run. So find it by the case + # header. Give up if there are multiple tests with the same name. + test_slug = f"[case {testcase.name}]" + if data_lines.count(test_slug) != 1: + return + start_idx = data_lines.index(test_slug) + stop_idx = start_idx + 11 + while stop_idx < len(data_lines) and not data_lines[stop_idx].startswith("[case "): + stop_idx += 1 + + test = data_lines[start_idx:stop_idx] + out_start = test.index("[out]") + test[out_start + 1 :] = output + data_lines[start_idx:stop_idx] = test + [""] + data = "\n".join(data_lines) + + with open(testcase_path, "w") as f: + print(data, file=f) + + +def assert_test_output( + testcase: DataDrivenTestCase, + actual: list[str], + message: str, + expected: list[str] | None = None, + formatted: list[str] | None = None, +) -> None: + __tracebackhide__ = True + + expected_output = expected if expected is not None else testcase.output + if expected_output != actual and testcase.config.getoption("--update-data", False): + update_testcase_output(testcase, actual) + + assert_string_arrays_equal( + expected_output, actual, f"{message} ({testcase.file}, line {testcase.line})" + ) + + +def get_func_names(expected: list[str]) -> list[str]: + res = [] + for s in expected: + m = re.match(r"def ([_a-zA-Z0-9.*$]+)\(", s) + if m: + res.append(m.group(1)) + return res + + +def remove_comment_lines(a: list[str]) -> list[str]: + """Return a copy of array with comments removed. + + Lines starting with '--' (but not with '---') are removed. + """ + r = [] + for s in a: + if s.strip().startswith("--") and not s.strip().startswith("---"): + pass + else: + r.append(s) + return r + + +def print_with_line_numbers(s: str) -> None: + lines = s.splitlines() + for i, line in enumerate(lines): + print("%-4d %s" % (i + 1, line)) + + +def heading(text: str) -> None: + print("=" * 20 + " " + text + " " + "=" * 20) + + +def show_c(cfiles: list[list[tuple[str, str]]]) -> None: + heading("Generated C") + for group in cfiles: + for cfile, ctext in group: + print(f"== {cfile} ==") + print_with_line_numbers(ctext) + heading("End C") + + +def fudge_dir_mtimes(dir: str, delta: int) -> None: + for dirpath, _, filenames in os.walk(dir): + for name in filenames: + path = os.path.join(dirpath, name) + new_mtime = os.stat(path).st_mtime + delta + os.utime(path, times=(new_mtime, new_mtime)) + + +def replace_word_size(text: list[str]) -> list[str]: + """Replace WORDSIZE with platform specific word sizes""" + result = [] + for line in text: + index = line.find("WORD_SIZE") + if index != -1: + # get 'WORDSIZE*n' token + word_size_token = line[index:].split()[0] + n = int(word_size_token[10:]) + replace_str = str(PLATFORM_SIZE * n) + result.append(line.replace(word_size_token, replace_str)) + else: + result.append(line) + return result + + +def infer_ir_build_options_from_test_name(name: str) -> CompilerOptions | None: + """Look for magic substrings in test case name to set compiler options. + + Return None if the test case should be skipped (always pass). + + Supported naming conventions: + + *_64bit*: + Run test case only on 64-bit platforms + *_32bit*: + Run test caseonly on 32-bit platforms + *_python3_8* (or for any Python version): + Use Python 3.8+ C API features (default: lowest supported version) + *StripAssert*: + Don't generate code for assert statements + """ + # If this is specific to some bit width, always pass if platform doesn't match. + if "_64bit" in name and IS_32_BIT_PLATFORM: + return None + if "_32bit" in name and not IS_32_BIT_PLATFORM: + return None + options = CompilerOptions(strip_asserts="StripAssert" in name, capi_version=(3, 7)) + # A suffix like _python3.8 is used to set the target C API version. + m = re.search(r"_python([3-9]+)_([0-9]+)(_|\b)", name) + if m: + options.capi_version = (int(m.group(1)), int(m.group(2))) + options.python_version = options.capi_version + elif "_py" in name or "_Python" in name: + assert False, f"Invalid _py* suffix (should be _pythonX_Y): {name}" + return options diff --git a/mypyc/transform/__init__.py b/mypyc/transform/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypyc/transform/copy_propagation.py b/mypyc/transform/copy_propagation.py new file mode 100644 index 000000000000..49de616f85a3 --- /dev/null +++ b/mypyc/transform/copy_propagation.py @@ -0,0 +1,94 @@ +"""Simple copy propagation optimization. + +Example input: + + x = f() + y = x + +The register x is redundant and we can directly assign its value to y: + + y = f() + +This can optimize away registers that are assigned to once. +""" + +from __future__ import annotations + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.ops import Assign, AssignMulti, LoadAddress, LoadErrorValue, Register, Value +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.options import CompilerOptions +from mypyc.sametype import is_same_type +from mypyc.transform.ir_transform import IRTransform + + +def do_copy_propagation(fn: FuncIR, options: CompilerOptions) -> None: + """Perform copy propagation optimization for fn.""" + + # Anything with an assignment count >1 will not be optimized + # here, as it would be require data flow analysis and we want to + # keep this simple and fast, at least until we've made data flow + # analysis much faster. + counts: dict[Value, int] = {} + replacements: dict[Value, Value] = {} + for arg in fn.arg_regs: + # Arguments are always assigned to initially + counts[arg] = 1 + + for block in fn.blocks: + for op in block.ops: + if isinstance(op, Assign): + c = counts.get(op.dest, 0) + counts[op.dest] = c + 1 + # Does this look like a supported assignment? + # TODO: Something needs LoadErrorValue assignments to be preserved? + if ( + c == 0 + and is_same_type(op.dest.type, op.src.type) + and not isinstance(op.src, LoadErrorValue) + ): + replacements[op.dest] = op.src + elif c == 1: + # Too many assignments -- don't replace this one + replacements.pop(op.dest, 0) + elif isinstance(op, AssignMulti): + # Copy propagation not supported for AssignMulti destinations + counts[op.dest] = 2 + replacements.pop(op.dest, 0) + elif isinstance(op, LoadAddress): + # We don't support taking the address of an arbitrary Value, + # so we'll need to preserve the operands of LoadAddress. + if isinstance(op.src, Register): + counts[op.src] = 2 + replacements.pop(op.src, 0) + + # Follow chains of propagation with more than one assignment. + for src, dst in list(replacements.items()): + if counts.get(dst, 0) > 1: + # Not supported + del replacements[src] + else: + while dst in replacements: + dst = replacements[dst] + if counts.get(dst, 0) > 1: + # Not supported + del replacements[src] + if src in replacements: + replacements[src] = dst + + builder = LowLevelIRBuilder(None, options) + transform = CopyPropagationTransform(builder, replacements) + transform.transform_blocks(fn.blocks) + fn.blocks = builder.blocks + + +class CopyPropagationTransform(IRTransform): + def __init__(self, builder: LowLevelIRBuilder, map: dict[Value, Value]) -> None: + super().__init__(builder) + self.op_map.update(map) + self.removed = set(map) + + def visit_assign(self, op: Assign) -> Value | None: + if op.dest in self.removed: + return None + return self.add(op) diff --git a/mypyc/transform/exceptions.py b/mypyc/transform/exceptions.py new file mode 100644 index 000000000000..33dfeb693cf7 --- /dev/null +++ b/mypyc/transform/exceptions.py @@ -0,0 +1,182 @@ +"""Transform that inserts error checks after opcodes. + +When initially building the IR, the code doesn't perform error checks +for exceptions. This module is used to insert all required error checks +afterwards. Each Op describes how it indicates an error condition (if +at all). + +We need to split basic blocks on each error check since branches can +only be placed at the end of a basic block. +""" + +from __future__ import annotations + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.ops import ( + ERR_ALWAYS, + ERR_FALSE, + ERR_MAGIC, + ERR_MAGIC_OVERLAPPING, + ERR_NEVER, + NO_TRACEBACK_LINE_NO, + BasicBlock, + Branch, + CallC, + ComparisonOp, + Float, + GetAttr, + Integer, + LoadErrorValue, + Op, + RegisterOp, + Return, + SetAttr, + TupleGet, + Value, +) +from mypyc.ir.rtypes import RTuple, bool_rprimitive, is_float_rprimitive +from mypyc.primitives.exc_ops import err_occurred_op +from mypyc.primitives.registry import CFunctionDescription + + +def insert_exception_handling(ir: FuncIR) -> None: + # Generate error block if any ops may raise an exception. If an op + # fails without its own error handler, we'll branch to this + # block. The block just returns an error value. + error_label: BasicBlock | None = None + for block in ir.blocks: + adjust_error_kinds(block) + if error_label is None and any(op.can_raise() for op in block.ops): + error_label = add_default_handler_block(ir) + if error_label: + ir.blocks = split_blocks_at_errors(ir.blocks, error_label, ir.traceback_name) + + +def add_default_handler_block(ir: FuncIR) -> BasicBlock: + block = BasicBlock() + ir.blocks.append(block) + op = LoadErrorValue(ir.ret_type) + block.ops.append(op) + block.ops.append(Return(op)) + return block + + +def split_blocks_at_errors( + blocks: list[BasicBlock], default_error_handler: BasicBlock, func_name: str | None +) -> list[BasicBlock]: + new_blocks: list[BasicBlock] = [] + + # First split blocks on ops that may raise. + for block in blocks: + ops = block.ops + block.ops = [] + cur_block = block + new_blocks.append(cur_block) + + # If the block has an error handler specified, use it. Otherwise + # fall back to the default. + error_label = block.error_handler or default_error_handler + block.error_handler = None + + for op in ops: + target: Value = op + cur_block.ops.append(op) + if isinstance(op, RegisterOp) and op.error_kind != ERR_NEVER: + # Split + new_block = BasicBlock() + new_blocks.append(new_block) + + if op.error_kind == ERR_MAGIC: + # Op returns an error value on error that depends on result RType. + variant = Branch.IS_ERROR + negated = False + elif op.error_kind == ERR_FALSE: + # Op returns a C false value on error. + variant = Branch.BOOL + negated = True + elif op.error_kind == ERR_ALWAYS: + variant = Branch.BOOL + negated = True + # this is a hack to represent the always fail + # semantics, using a temporary bool with value false + target = Integer(0, bool_rprimitive) + elif op.error_kind == ERR_MAGIC_OVERLAPPING: + comp = insert_overlapping_error_value_check(cur_block.ops, target) + new_block2 = BasicBlock() + new_blocks.append(new_block2) + branch = Branch( + comp, + true_label=new_block2, + false_label=new_block, + op=Branch.BOOL, + rare=True, + ) + cur_block.ops.append(branch) + cur_block = new_block2 + target = primitive_call(err_occurred_op, [], target.line) + cur_block.ops.append(target) + variant = Branch.IS_ERROR + negated = True + else: + assert False, "unknown error kind %d" % op.error_kind + + # Void ops can't generate errors since error is always + # indicated by a special value stored in a register. + if op.error_kind != ERR_ALWAYS: + assert not op.is_void, "void op generating errors?" + + branch = Branch( + target, true_label=error_label, false_label=new_block, op=variant, line=op.line + ) + branch.negated = negated + if op.line != NO_TRACEBACK_LINE_NO and func_name is not None: + branch.traceback_entry = (func_name, op.line) + cur_block.ops.append(branch) + cur_block = new_block + + return new_blocks + + +def primitive_call(desc: CFunctionDescription, args: list[Value], line: int) -> CallC: + return CallC( + desc.c_function_name, + [], + desc.return_type, + desc.steals, + desc.is_borrowed, + desc.error_kind, + line, + ) + + +def adjust_error_kinds(block: BasicBlock) -> None: + """Infer more precise error_kind attributes for ops. + + We have access here to more information than what was available + when the IR was initially built. + """ + for op in block.ops: + if isinstance(op, GetAttr): + if op.class_type.class_ir.is_always_defined(op.attr): + op.error_kind = ERR_NEVER + if isinstance(op, SetAttr): + if op.class_type.class_ir.is_always_defined(op.attr): + op.error_kind = ERR_NEVER + + +def insert_overlapping_error_value_check(ops: list[Op], target: Value) -> ComparisonOp: + """Append to ops to check for an overlapping error value.""" + typ = target.type + if isinstance(typ, RTuple): + item = TupleGet(target, 0) + ops.append(item) + return insert_overlapping_error_value_check(ops, item) + else: + errvalue: Value + if is_float_rprimitive(target.type): + errvalue = Float(float(typ.c_undefined)) + else: + errvalue = Integer(int(typ.c_undefined), rtype=typ) + op = ComparisonOp(target, errvalue, ComparisonOp.EQ) + ops.append(op) + return op diff --git a/mypyc/transform/flag_elimination.py b/mypyc/transform/flag_elimination.py new file mode 100644 index 000000000000..605e5bc46ae4 --- /dev/null +++ b/mypyc/transform/flag_elimination.py @@ -0,0 +1,108 @@ +"""Bool register elimination optimization. + +Example input: + + L1: + r0 = f() + b = r0 + goto L3 + L2: + r1 = g() + b = r1 + goto L3 + L3: + if b goto L4 else goto L5 + +The register b is redundant and we replace the assignments with two copies of +the branch in L3: + + L1: + r0 = f() + if r0 goto L4 else goto L5 + L2: + r1 = g() + if r1 goto L4 else goto L5 + +This helps generate simpler IR for tagged integers comparisons, for example. +""" + +from __future__ import annotations + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.ops import Assign, BasicBlock, Branch, Goto, Register, Unreachable +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.options import CompilerOptions +from mypyc.transform.ir_transform import IRTransform + + +def do_flag_elimination(fn: FuncIR, options: CompilerOptions) -> None: + # Find registers that are used exactly once as source, and in a branch. + counts: dict[Register, int] = {} + branches: dict[Register, Branch] = {} + labels: dict[Register, BasicBlock] = {} + for block in fn.blocks: + for i, op in enumerate(block.ops): + for src in op.sources(): + if isinstance(src, Register): + counts[src] = counts.get(src, 0) + 1 + if i == 0 and isinstance(op, Branch) and isinstance(op.value, Register): + branches[op.value] = op + labels[op.value] = block + + # Based on these we can find the candidate registers. + candidates: set[Register] = { + r for r in branches if counts.get(r, 0) == 1 and r not in fn.arg_regs + } + + # Remove candidates with invalid assignments. + for block in fn.blocks: + for i, op in enumerate(block.ops): + if isinstance(op, Assign) and op.dest in candidates: + next_op = block.ops[i + 1] + if not (isinstance(next_op, Goto) and next_op.label is labels[op.dest]): + # Not right + candidates.remove(op.dest) + + builder = LowLevelIRBuilder(None, options) + transform = FlagEliminationTransform( + builder, {x: y for x, y in branches.items() if x in candidates} + ) + transform.transform_blocks(fn.blocks) + fn.blocks = builder.blocks + + +class FlagEliminationTransform(IRTransform): + def __init__(self, builder: LowLevelIRBuilder, branch_map: dict[Register, Branch]) -> None: + super().__init__(builder) + self.branch_map = branch_map + self.branches = set(branch_map.values()) + + def visit_assign(self, op: Assign) -> None: + old_branch = self.branch_map.get(op.dest) + if old_branch: + # Replace assignment with a copy of the old branch, which is in a + # separate basic block. The old branch will be deletecd in visit_branch. + new_branch = Branch( + op.src, + old_branch.true, + old_branch.false, + old_branch.op, + old_branch.line, + rare=old_branch.rare, + ) + new_branch.negated = old_branch.negated + new_branch.traceback_entry = old_branch.traceback_entry + self.add(new_branch) + else: + self.add(op) + + def visit_goto(self, op: Goto) -> None: + # This is a no-op if basic block already terminated + self.builder.goto(op.label) + + def visit_branch(self, op: Branch) -> None: + if op in self.branches: + # This branch is optimized away + self.add(Unreachable()) + else: + self.add(op) diff --git a/mypyc/transform/ir_transform.py b/mypyc/transform/ir_transform.py new file mode 100644 index 000000000000..a631bd7352b5 --- /dev/null +++ b/mypyc/transform/ir_transform.py @@ -0,0 +1,368 @@ +"""Helpers for implementing generic IR to IR transforms.""" + +from __future__ import annotations + +from typing import Final, Optional + +from mypyc.ir.ops import ( + Assign, + AssignMulti, + BasicBlock, + Box, + Branch, + Call, + CallC, + Cast, + ComparisonOp, + DecRef, + Extend, + FloatComparisonOp, + FloatNeg, + FloatOp, + GetAttr, + GetElementPtr, + Goto, + IncRef, + InitStatic, + IntOp, + KeepAlive, + LoadAddress, + LoadErrorValue, + LoadGlobal, + LoadLiteral, + LoadMem, + LoadStatic, + MethodCall, + Op, + OpVisitor, + PrimitiveOp, + RaiseStandardError, + Return, + SetAttr, + SetMem, + Truncate, + TupleGet, + TupleSet, + Unborrow, + Unbox, + Unreachable, + Value, +) +from mypyc.irbuild.ll_builder import LowLevelIRBuilder + + +class IRTransform(OpVisitor[Optional[Value]]): + """Identity transform. + + Subclass and override to perform changes to IR. + + Subclass IRTransform and override any OpVisitor visit_* methods + that perform any IR changes. The default implementations implement + an identity transform. + + A visit method can return None to remove ops. In this case the + transform must ensure that no op uses the original removed op + as a source after the transform. + + You can retain old BasicBlock and op references in ops. The transform + will automatically patch these for you as needed. + """ + + def __init__(self, builder: LowLevelIRBuilder) -> None: + self.builder = builder + # Subclasses add additional op mappings here. A None value indicates + # that the op/register is deleted. + self.op_map: dict[Value, Value | None] = {} + + def transform_blocks(self, blocks: list[BasicBlock]) -> None: + """Transform basic blocks that represent a single function. + + The result of the transform will be collected at self.builder.blocks. + """ + block_map: dict[BasicBlock, BasicBlock] = {} + op_map = self.op_map + empties = set() + for block in blocks: + new_block = BasicBlock() + block_map[block] = new_block + self.builder.activate_block(new_block) + new_block.error_handler = block.error_handler + for op in block.ops: + new_op = op.accept(self) + if new_op is not op: + op_map[op] = new_op + # A transform can produce empty blocks which can be removed. + if is_empty_block(new_block) and not is_empty_block(block): + empties.add(new_block) + self.builder.blocks = [block for block in self.builder.blocks if block not in empties] + # Update all op/block references to point to the transformed ones. + patcher = PatchVisitor(op_map, block_map) + for block in self.builder.blocks: + for op in block.ops: + op.accept(patcher) + if block.error_handler is not None: + block.error_handler = block_map.get(block.error_handler, block.error_handler) + + def add(self, op: Op) -> Value: + return self.builder.add(op) + + def visit_goto(self, op: Goto) -> None: + self.add(op) + + def visit_branch(self, op: Branch) -> None: + self.add(op) + + def visit_return(self, op: Return) -> None: + self.add(op) + + def visit_unreachable(self, op: Unreachable) -> None: + self.add(op) + + def visit_assign(self, op: Assign) -> Value | None: + return self.add(op) + + def visit_assign_multi(self, op: AssignMulti) -> Value | None: + return self.add(op) + + def visit_load_error_value(self, op: LoadErrorValue) -> Value | None: + return self.add(op) + + def visit_load_literal(self, op: LoadLiteral) -> Value | None: + return self.add(op) + + def visit_get_attr(self, op: GetAttr) -> Value | None: + return self.add(op) + + def visit_set_attr(self, op: SetAttr) -> Value | None: + return self.add(op) + + def visit_load_static(self, op: LoadStatic) -> Value | None: + return self.add(op) + + def visit_init_static(self, op: InitStatic) -> Value | None: + return self.add(op) + + def visit_tuple_get(self, op: TupleGet) -> Value | None: + return self.add(op) + + def visit_tuple_set(self, op: TupleSet) -> Value | None: + return self.add(op) + + def visit_inc_ref(self, op: IncRef) -> Value | None: + return self.add(op) + + def visit_dec_ref(self, op: DecRef) -> Value | None: + return self.add(op) + + def visit_call(self, op: Call) -> Value | None: + return self.add(op) + + def visit_method_call(self, op: MethodCall) -> Value | None: + return self.add(op) + + def visit_cast(self, op: Cast) -> Value | None: + return self.add(op) + + def visit_box(self, op: Box) -> Value | None: + return self.add(op) + + def visit_unbox(self, op: Unbox) -> Value | None: + return self.add(op) + + def visit_raise_standard_error(self, op: RaiseStandardError) -> Value | None: + return self.add(op) + + def visit_call_c(self, op: CallC) -> Value | None: + return self.add(op) + + def visit_primitive_op(self, op: PrimitiveOp) -> Value | None: + return self.add(op) + + def visit_truncate(self, op: Truncate) -> Value | None: + return self.add(op) + + def visit_extend(self, op: Extend) -> Value | None: + return self.add(op) + + def visit_load_global(self, op: LoadGlobal) -> Value | None: + return self.add(op) + + def visit_int_op(self, op: IntOp) -> Value | None: + return self.add(op) + + def visit_comparison_op(self, op: ComparisonOp) -> Value | None: + return self.add(op) + + def visit_float_op(self, op: FloatOp) -> Value | None: + return self.add(op) + + def visit_float_neg(self, op: FloatNeg) -> Value | None: + return self.add(op) + + def visit_float_comparison_op(self, op: FloatComparisonOp) -> Value | None: + return self.add(op) + + def visit_load_mem(self, op: LoadMem) -> Value | None: + return self.add(op) + + def visit_set_mem(self, op: SetMem) -> Value | None: + return self.add(op) + + def visit_get_element_ptr(self, op: GetElementPtr) -> Value | None: + return self.add(op) + + def visit_load_address(self, op: LoadAddress) -> Value | None: + return self.add(op) + + def visit_keep_alive(self, op: KeepAlive) -> Value | None: + return self.add(op) + + def visit_unborrow(self, op: Unborrow) -> Value | None: + return self.add(op) + + +class PatchVisitor(OpVisitor[None]): + def __init__( + self, op_map: dict[Value, Value | None], block_map: dict[BasicBlock, BasicBlock] + ) -> None: + self.op_map: Final = op_map + self.block_map: Final = block_map + + def fix_op(self, op: Value) -> Value: + new = self.op_map.get(op, op) + assert new is not None, "use of removed op" + return new + + def fix_block(self, block: BasicBlock) -> BasicBlock: + return self.block_map.get(block, block) + + def visit_goto(self, op: Goto) -> None: + op.label = self.fix_block(op.label) + + def visit_branch(self, op: Branch) -> None: + op.value = self.fix_op(op.value) + op.true = self.fix_block(op.true) + op.false = self.fix_block(op.false) + + def visit_return(self, op: Return) -> None: + op.value = self.fix_op(op.value) + + def visit_unreachable(self, op: Unreachable) -> None: + pass + + def visit_assign(self, op: Assign) -> None: + op.src = self.fix_op(op.src) + + def visit_assign_multi(self, op: AssignMulti) -> None: + op.src = [self.fix_op(s) for s in op.src] + + def visit_load_error_value(self, op: LoadErrorValue) -> None: + pass + + def visit_load_literal(self, op: LoadLiteral) -> None: + pass + + def visit_get_attr(self, op: GetAttr) -> None: + op.obj = self.fix_op(op.obj) + + def visit_set_attr(self, op: SetAttr) -> None: + op.obj = self.fix_op(op.obj) + op.src = self.fix_op(op.src) + + def visit_load_static(self, op: LoadStatic) -> None: + pass + + def visit_init_static(self, op: InitStatic) -> None: + op.value = self.fix_op(op.value) + + def visit_tuple_get(self, op: TupleGet) -> None: + op.src = self.fix_op(op.src) + + def visit_tuple_set(self, op: TupleSet) -> None: + op.items = [self.fix_op(item) for item in op.items] + + def visit_inc_ref(self, op: IncRef) -> None: + op.src = self.fix_op(op.src) + + def visit_dec_ref(self, op: DecRef) -> None: + op.src = self.fix_op(op.src) + + def visit_call(self, op: Call) -> None: + op.args = [self.fix_op(arg) for arg in op.args] + + def visit_method_call(self, op: MethodCall) -> None: + op.obj = self.fix_op(op.obj) + op.args = [self.fix_op(arg) for arg in op.args] + + def visit_cast(self, op: Cast) -> None: + op.src = self.fix_op(op.src) + + def visit_box(self, op: Box) -> None: + op.src = self.fix_op(op.src) + + def visit_unbox(self, op: Unbox) -> None: + op.src = self.fix_op(op.src) + + def visit_raise_standard_error(self, op: RaiseStandardError) -> None: + if isinstance(op.value, Value): + op.value = self.fix_op(op.value) + + def visit_call_c(self, op: CallC) -> None: + op.args = [self.fix_op(arg) for arg in op.args] + + def visit_primitive_op(self, op: PrimitiveOp) -> None: + op.args = [self.fix_op(arg) for arg in op.args] + + def visit_truncate(self, op: Truncate) -> None: + op.src = self.fix_op(op.src) + + def visit_extend(self, op: Extend) -> None: + op.src = self.fix_op(op.src) + + def visit_load_global(self, op: LoadGlobal) -> None: + pass + + def visit_int_op(self, op: IntOp) -> None: + op.lhs = self.fix_op(op.lhs) + op.rhs = self.fix_op(op.rhs) + + def visit_comparison_op(self, op: ComparisonOp) -> None: + op.lhs = self.fix_op(op.lhs) + op.rhs = self.fix_op(op.rhs) + + def visit_float_op(self, op: FloatOp) -> None: + op.lhs = self.fix_op(op.lhs) + op.rhs = self.fix_op(op.rhs) + + def visit_float_neg(self, op: FloatNeg) -> None: + op.src = self.fix_op(op.src) + + def visit_float_comparison_op(self, op: FloatComparisonOp) -> None: + op.lhs = self.fix_op(op.lhs) + op.rhs = self.fix_op(op.rhs) + + def visit_load_mem(self, op: LoadMem) -> None: + op.src = self.fix_op(op.src) + + def visit_set_mem(self, op: SetMem) -> None: + op.dest = self.fix_op(op.dest) + op.src = self.fix_op(op.src) + + def visit_get_element_ptr(self, op: GetElementPtr) -> None: + op.src = self.fix_op(op.src) + + def visit_load_address(self, op: LoadAddress) -> None: + if isinstance(op.src, LoadStatic): + new = self.fix_op(op.src) + assert isinstance(new, LoadStatic) + op.src = new + + def visit_keep_alive(self, op: KeepAlive) -> None: + op.src = [self.fix_op(s) for s in op.src] + + def visit_unborrow(self, op: Unborrow) -> None: + op.src = self.fix_op(op.src) + + +def is_empty_block(block: BasicBlock) -> bool: + return len(block.ops) == 1 and isinstance(block.ops[0], Unreachable) diff --git a/mypyc/transform/lower.py b/mypyc/transform/lower.py new file mode 100644 index 000000000000..b717657095f9 --- /dev/null +++ b/mypyc/transform/lower.py @@ -0,0 +1,33 @@ +"""Transform IR to lower-level ops. + +Higher-level ops are used in earlier compiler passes, as they make +various analyses, optimizations and transforms easier to implement. +Later passes use lower-level ops, as they are easier to generate code +from, and they help with lower-level optimizations. + +Lowering of various primitive ops is implemented in the mypyc.lower +package. +""" + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.ops import PrimitiveOp, Value +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.lower.registry import lowering_registry +from mypyc.options import CompilerOptions +from mypyc.transform.ir_transform import IRTransform + + +def lower_ir(ir: FuncIR, options: CompilerOptions) -> None: + builder = LowLevelIRBuilder(None, options) + visitor = LoweringVisitor(builder) + visitor.transform_blocks(ir.blocks) + ir.blocks = builder.blocks + + +class LoweringVisitor(IRTransform): + def visit_primitive_op(self, op: PrimitiveOp) -> Value: + # The lowering implementation functions of various primitive ops are stored + # in a registry, which is populated using function decorators. The name + # of op (such as "int_eq") is used as the key. + lower_fn = lowering_registry[op.desc.name] + return lower_fn(self.builder, op.args, op.line) diff --git a/mypyc/transform/refcount.py b/mypyc/transform/refcount.py new file mode 100644 index 000000000000..f2ab438f6576 --- /dev/null +++ b/mypyc/transform/refcount.py @@ -0,0 +1,294 @@ +"""Transformation for inserting refrecence count inc/dec opcodes. + +This transformation happens towards the end of compilation. Before this +transformation, reference count management is not explicitly handled at all. +By postponing this pass, the previous passes are simpler as they don't have +to update reference count opcodes. + +The approach is to decrement reference counts soon after a value is no +longer live, to quickly free memory (and call __del__ methods), though +there are no strict guarantees -- other than that local variables are +freed before return from a function. + +Function arguments are a little special. They are initially considered +'borrowed' from the caller and their reference counts don't need to be +decremented before returning. An assignment to a borrowed value turns it +into a regular, owned reference that needs to freed before return. +""" + +from __future__ import annotations + +from typing import Dict, Iterable, Tuple + +from mypyc.analysis.dataflow import ( + AnalysisDict, + analyze_borrowed_arguments, + analyze_live_regs, + analyze_must_defined_regs, + cleanup_cfg, + get_cfg, +) +from mypyc.ir.func_ir import FuncIR, all_values +from mypyc.ir.ops import ( + Assign, + BasicBlock, + Branch, + ControlOp, + DecRef, + Goto, + IncRef, + Integer, + KeepAlive, + LoadAddress, + Op, + Register, + RegisterOp, + Value, +) + +Decs = Tuple[Tuple[Value, bool], ...] +Incs = Tuple[Value, ...] + +# A cache of basic blocks that decrement and increment specific values +# and then jump to some target block. This lets us cut down on how +# much code we generate in some circumstances. +BlockCache = Dict[Tuple[BasicBlock, Decs, Incs], BasicBlock] + + +def insert_ref_count_opcodes(ir: FuncIR) -> None: + """Insert reference count inc/dec opcodes to a function. + + This is the entry point to this module. + """ + cfg = get_cfg(ir.blocks) + values = all_values(ir.arg_regs, ir.blocks) + + borrowed = {value for value in values if value.is_borrowed} + args: set[Value] = set(ir.arg_regs) + live = analyze_live_regs(ir.blocks, cfg) + borrow = analyze_borrowed_arguments(ir.blocks, cfg, borrowed) + defined = analyze_must_defined_regs(ir.blocks, cfg, args, values, strict_errors=True) + ordering = make_value_ordering(ir) + cache: BlockCache = {} + for block in ir.blocks.copy(): + if isinstance(block.ops[-1], (Branch, Goto)): + insert_branch_inc_and_decrefs( + block, + cache, + ir.blocks, + live.before, + borrow.before, + borrow.after, + defined.after, + ordering, + ) + transform_block(block, live.before, live.after, borrow.before, defined.after) + + cleanup_cfg(ir.blocks) + + +def is_maybe_undefined(post_must_defined: set[Value], src: Value) -> bool: + return isinstance(src, Register) and src not in post_must_defined + + +def maybe_append_dec_ref( + ops: list[Op], dest: Value, defined: AnalysisDict[Value], key: tuple[BasicBlock, int] +) -> None: + if dest.type.is_refcounted and not isinstance(dest, Integer): + ops.append(DecRef(dest, is_xdec=is_maybe_undefined(defined[key], dest))) + + +def maybe_append_inc_ref(ops: list[Op], dest: Value) -> None: + if dest.type.is_refcounted: + ops.append(IncRef(dest)) + + +def transform_block( + block: BasicBlock, + pre_live: AnalysisDict[Value], + post_live: AnalysisDict[Value], + pre_borrow: AnalysisDict[Value], + post_must_defined: AnalysisDict[Value], +) -> None: + old_ops = block.ops + ops: list[Op] = [] + for i, op in enumerate(old_ops): + key = (block, i) + + assert op not in pre_live[key] + dest = op.dest if isinstance(op, Assign) else op + stolen = op.stolen() + + # Incref any references that are being stolen that stay live, were borrowed, + # or are stolen more than once by this operation. + for j, src in enumerate(stolen): + if src in post_live[key] or src in pre_borrow[key] or src in stolen[:j]: + maybe_append_inc_ref(ops, src) + # For assignments to registers that were already live, + # decref the old value. + if dest not in pre_borrow[key] and dest in pre_live[key]: + assert isinstance(op, Assign) + maybe_append_dec_ref(ops, dest, post_must_defined, key) + + # Strip KeepAlive. Its only purpose is to help with this transform. + if not isinstance(op, KeepAlive): + ops.append(op) + + # Control ops don't have any space to insert ops after them, so + # their inc/decrefs get inserted by insert_branch_inc_and_decrefs. + if isinstance(op, ControlOp): + continue + + for src in op.unique_sources(): + # Decrement source that won't be live afterwards. + if src not in post_live[key] and src not in pre_borrow[key] and src not in stolen: + maybe_append_dec_ref(ops, src, post_must_defined, key) + # Decrement the destination if it is dead after the op and + # wasn't a borrowed RegisterOp + if ( + not dest.is_void + and dest not in post_live[key] + and not (isinstance(op, RegisterOp) and dest.is_borrowed) + ): + maybe_append_dec_ref(ops, dest, post_must_defined, key) + block.ops = ops + + +def insert_branch_inc_and_decrefs( + block: BasicBlock, + cache: BlockCache, + blocks: list[BasicBlock], + pre_live: AnalysisDict[Value], + pre_borrow: AnalysisDict[Value], + post_borrow: AnalysisDict[Value], + post_must_defined: AnalysisDict[Value], + ordering: dict[Value, int], +) -> None: + """Insert inc_refs and/or dec_refs after a branch/goto. + + Add dec_refs for registers that become dead after a branch. + Add inc_refs for registers that become unborrowed after a branch or goto. + + Branches are special as the true and false targets may have a different + live and borrowed register sets. Add new blocks before the true/false target + blocks that tweak reference counts. + + Example where we need to add an inc_ref: + + def f(a: int) -> None + if a: + a = 1 + return a # a is borrowed if condition is false and unborrowed if true + """ + prev_key = (block, len(block.ops) - 1) + source_live_regs = pre_live[prev_key] + source_borrowed = post_borrow[prev_key] + source_defined = post_must_defined[prev_key] + + term = block.terminator + for i, target in enumerate(term.targets()): + # HAX: After we've checked against an error value the value we must not touch the + # refcount since it will be a null pointer. The correct way to do this would be + # to perform data flow analysis on whether a value can be null (or is always + # null). + omitted: Iterable[Value] + if isinstance(term, Branch) and term.op == Branch.IS_ERROR and i == 0: + omitted = (term.value,) + else: + omitted = () + + decs = after_branch_decrefs( + target, pre_live, source_defined, source_borrowed, source_live_regs, ordering, omitted + ) + incs = after_branch_increfs(target, pre_live, pre_borrow, source_borrowed, ordering) + term.set_target(i, add_block(decs, incs, cache, blocks, target)) + + +def after_branch_decrefs( + label: BasicBlock, + pre_live: AnalysisDict[Value], + source_defined: set[Value], + source_borrowed: set[Value], + source_live_regs: set[Value], + ordering: dict[Value, int], + omitted: Iterable[Value], +) -> tuple[tuple[Value, bool], ...]: + target_pre_live = pre_live[label, 0] + decref = source_live_regs - target_pre_live - source_borrowed + if decref: + return tuple( + (reg, is_maybe_undefined(source_defined, reg)) + for reg in sorted(decref, key=lambda r: ordering[r]) + if reg.type.is_refcounted and reg not in omitted + ) + return () + + +def after_branch_increfs( + label: BasicBlock, + pre_live: AnalysisDict[Value], + pre_borrow: AnalysisDict[Value], + source_borrowed: set[Value], + ordering: dict[Value, int], +) -> tuple[Value, ...]: + target_pre_live = pre_live[label, 0] + target_borrowed = pre_borrow[label, 0] + incref = (source_borrowed - target_borrowed) & target_pre_live + if incref: + return tuple( + reg for reg in sorted(incref, key=lambda r: ordering[r]) if reg.type.is_refcounted + ) + return () + + +def add_block( + decs: Decs, incs: Incs, cache: BlockCache, blocks: list[BasicBlock], label: BasicBlock +) -> BasicBlock: + if not decs and not incs: + return label + + # TODO: be able to share *partial* results + if (label, decs, incs) in cache: + return cache[label, decs, incs] + + block = BasicBlock() + blocks.append(block) + block.ops.extend(DecRef(reg, is_xdec=xdec) for reg, xdec in decs) + block.ops.extend(IncRef(reg) for reg in incs) + block.ops.append(Goto(label)) + cache[label, decs, incs] = block + return block + + +def make_value_ordering(ir: FuncIR) -> dict[Value, int]: + """Create a ordering of values that allows them to be sorted. + + This omits registers that are only ever read. + """ + # TODO: Never initialized values?? + result: dict[Value, int] = {} + n = 0 + + for arg in ir.arg_regs: + result[arg] = n + n += 1 + + for block in ir.blocks: + for op in block.ops: + if ( + isinstance(op, LoadAddress) + and isinstance(op.src, Register) + and op.src not in result + ): + # Taking the address of a register allows initialization. + result[op.src] = n + n += 1 + if isinstance(op, Assign): + if op.dest not in result: + result[op.dest] = n + n += 1 + elif op not in result: + result[op] = n + n += 1 + + return result diff --git a/mypyc/transform/uninit.py b/mypyc/transform/uninit.py new file mode 100644 index 000000000000..6bf71ac4a8bc --- /dev/null +++ b/mypyc/transform/uninit.py @@ -0,0 +1,190 @@ +"""Insert checks for uninitialized values.""" + +from __future__ import annotations + +from mypyc.analysis.dataflow import AnalysisDict, analyze_must_defined_regs, cleanup_cfg, get_cfg +from mypyc.common import BITMAP_BITS +from mypyc.ir.func_ir import FuncIR, all_values +from mypyc.ir.ops import ( + Assign, + BasicBlock, + Branch, + ComparisonOp, + Integer, + IntOp, + LoadAddress, + LoadErrorValue, + Op, + RaiseStandardError, + Register, + Unreachable, + Value, +) +from mypyc.ir.rtypes import bitmap_rprimitive + + +def insert_uninit_checks(ir: FuncIR) -> None: + # Remove dead blocks from the CFG, which helps avoid spurious + # checks due to unused error handling blocks. + cleanup_cfg(ir.blocks) + + cfg = get_cfg(ir.blocks) + must_defined = analyze_must_defined_regs( + ir.blocks, cfg, set(ir.arg_regs), all_values(ir.arg_regs, ir.blocks) + ) + + ir.blocks = split_blocks_at_uninits(ir.blocks, must_defined.before) + + +def split_blocks_at_uninits( + blocks: list[BasicBlock], pre_must_defined: AnalysisDict[Value] +) -> list[BasicBlock]: + new_blocks: list[BasicBlock] = [] + + init_registers = [] + init_registers_set = set() + bitmap_registers: list[Register] = [] # Init status bitmaps + bitmap_backed: list[Register] = [] # These use bitmaps to track init status + + # First split blocks on ops that may raise. + for block in blocks: + ops = block.ops + block.ops = [] + cur_block = block + new_blocks.append(cur_block) + + for i, op in enumerate(ops): + defined = pre_must_defined[block, i] + for src in op.unique_sources(): + # If a register operand is not guaranteed to be + # initialized is an operand to something other than a + # check that it is defined, insert a check. + + # Note that for register operand in a LoadAddress op, + # we should be able to use it without initialization + # as we may need to use its address to update itself + if ( + isinstance(src, Register) + and src not in defined + and not (isinstance(op, Branch) and op.op == Branch.IS_ERROR) + and not isinstance(op, LoadAddress) + ): + new_block, error_block = BasicBlock(), BasicBlock() + new_block.error_handler = error_block.error_handler = cur_block.error_handler + new_blocks += [error_block, new_block] + + if src not in init_registers_set: + init_registers.append(src) + init_registers_set.add(src) + + if not src.type.error_overlap: + cur_block.ops.append( + Branch( + src, + true_label=error_block, + false_label=new_block, + op=Branch.IS_ERROR, + line=op.line, + ) + ) + else: + # We need to use bitmap for this one. + check_for_uninit_using_bitmap( + cur_block.ops, + src, + bitmap_registers, + bitmap_backed, + error_block, + new_block, + op.line, + ) + + raise_std = RaiseStandardError( + RaiseStandardError.UNBOUND_LOCAL_ERROR, + f'local variable "{src.name}" referenced before assignment', + op.line, + ) + error_block.ops.append(raise_std) + error_block.ops.append(Unreachable()) + cur_block = new_block + cur_block.ops.append(op) + + if bitmap_backed: + update_register_assignments_to_set_bitmap(new_blocks, bitmap_registers, bitmap_backed) + + if init_registers: + new_ops: list[Op] = [] + for reg in init_registers: + err = LoadErrorValue(reg.type, undefines=True) + new_ops.append(err) + new_ops.append(Assign(reg, err)) + for reg in bitmap_registers: + new_ops.append(Assign(reg, Integer(0, bitmap_rprimitive))) + new_blocks[0].ops[0:0] = new_ops + + return new_blocks + + +def check_for_uninit_using_bitmap( + ops: list[Op], + src: Register, + bitmap_registers: list[Register], + bitmap_backed: list[Register], + error_block: BasicBlock, + ok_block: BasicBlock, + line: int, +) -> None: + """Check if src is defined using a bitmap. + + Modifies ops, bitmap_registers and bitmap_backed. + """ + if src not in bitmap_backed: + # Set up a new bitmap backed register. + bitmap_backed.append(src) + n = (len(bitmap_backed) - 1) // BITMAP_BITS + if len(bitmap_registers) <= n: + bitmap_registers.append(Register(bitmap_rprimitive, f"__locals_bitmap{n}")) + + index = bitmap_backed.index(src) + masked = IntOp( + bitmap_rprimitive, + bitmap_registers[index // BITMAP_BITS], + Integer(1 << (index & (BITMAP_BITS - 1)), bitmap_rprimitive), + IntOp.AND, + line, + ) + ops.append(masked) + chk = ComparisonOp(masked, Integer(0, bitmap_rprimitive), ComparisonOp.EQ) + ops.append(chk) + ops.append(Branch(chk, error_block, ok_block, Branch.BOOL)) + + +def update_register_assignments_to_set_bitmap( + blocks: list[BasicBlock], bitmap_registers: list[Register], bitmap_backed: list[Register] +) -> None: + """Update some assignments to registers to also set a bit in a bitmap. + + The bitmaps are used to track if a local variable has been assigned to. + + Modifies blocks. + """ + for block in blocks: + if any(isinstance(op, Assign) and op.dest in bitmap_backed for op in block.ops): + new_ops: list[Op] = [] + for op in block.ops: + if isinstance(op, Assign) and op.dest in bitmap_backed: + index = bitmap_backed.index(op.dest) + new_ops.append(op) + reg = bitmap_registers[index // BITMAP_BITS] + new = IntOp( + bitmap_rprimitive, + reg, + Integer(1 << (index & (BITMAP_BITS - 1)), bitmap_rprimitive), + IntOp.OR, + op.line, + ) + new_ops.append(new) + new_ops.append(Assign(reg, new)) + else: + new_ops.append(op) + block.ops = new_ops diff --git a/pinfer/.gitignore b/pinfer/.gitignore deleted file mode 100644 index e1dace58f791..000000000000 --- a/pinfer/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -__pycache__ -*~ -*.pyc diff --git a/pinfer/LICENSE b/pinfer/LICENSE deleted file mode 100644 index ecdce98229ee..000000000000 --- a/pinfer/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -pinfer is licensed under the terms of the MIT license, reproduced below. - -= = = = = - -The MIT License - -Copyright (c) 2013, 2014 Jukka Lehtosalo - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -= = = = = diff --git a/pinfer/README b/pinfer/README deleted file mode 100644 index 06005692795b..000000000000 --- a/pinfer/README +++ /dev/null @@ -1,47 +0,0 @@ -ReadMe for pinfer -================= - -Introduction ------------- - -Pinfer is tool for runtime type inference of variable types and -function signatures in Python programs. The inferred types are mostly -compatible with mypy types. It is intended for coming up with draft -types when migrating Python code to static tryping, but it can also be -used as a code understanding or analysis tool. - -Pinfer is very experimental! - -Requirements ------------- - - * Python 3.2 or later - -Basic usage ------------ - -To infer types of all functions and methods in a module: - - import foo # target module - import pinfer - - # set up type inference and dumping - pinfer.infer_module(foo) - pinfer.dump_at_exit() - - # now do something with the module, e.g. run tests - -For inferring a Python module, add the above lines to the test suite. - -Handy wrapper -------------- - -The p.py script provides a handy wrapper for the above. - - -Copyright -------------- - -This project includes files from the open source CPython project. Those files are Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved. The license can be found at https://github.com/python/cpython/blob/master/LICENSE. - diff --git a/pinfer/inspect3.py b/pinfer/inspect3.py deleted file mode 100644 index 4d74be19b66d..000000000000 --- a/pinfer/inspect3.py +++ /dev/null @@ -1,122 +0,0 @@ -# from Python 3's inspect.py -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved -''' -provide getfullargspec() and getcallargs() for Python 2 -''' - -import sys -import inspect - -if sys.version_info.major == 2: - - def getfullargspec(func): - (args, varargs, keywords, defaults) = inspect.getargspec(func) - return (args, varargs, keywords, defaults, [], [], {}) - - - def getcallargs(*func_and_positional, **named): - """Get the mapping of arguments to values. - - A dict is returned, with keys the function argument names (including the - names of the * and ** arguments, if any), and values the respective bound - values from 'positional' and 'named'.""" - func = func_and_positional[0] - positional = func_and_positional[1:] - spec = getfullargspec(func) - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = spec - f_name = func.__name__ - arg2value = {} - - - if inspect.ismethod(func) and func.__self__ is not None: - # implicit 'self' (or 'cls' for classmethods) argument - positional = (func.__self__,) + positional - num_pos = len(positional) - num_args = len(args) - num_defaults = len(defaults) if defaults else 0 - - n = min(num_pos, num_args) - for i in range(n): - arg2value[args[i]] = positional[i] - if varargs: - arg2value[varargs] = tuple(positional[n:]) - possible_kwargs = set(args + kwonlyargs) - if varkw: - arg2value[varkw] = {} - for kw, value in named.items(): - if kw not in possible_kwargs: - if not varkw: - raise TypeError("%s() got an unexpected keyword argument %r" % - (f_name, kw)) - arg2value[varkw][kw] = value - continue - if kw in arg2value: - raise TypeError("%s() got multiple values for argument %r" % - (f_name, kw)) - arg2value[kw] = value - if num_pos > num_args and not varargs: - _too_many(f_name, args, kwonlyargs, varargs, num_defaults, - num_pos, arg2value) - if num_pos < num_args: - req = args[:num_args - num_defaults] - for arg in req: - if arg not in arg2value: - _missing_arguments(f_name, req, True, arg2value) - for i, arg in enumerate(args[num_args - num_defaults:]): - if arg not in arg2value: - arg2value[arg] = defaults[i] - missing = 0 - for kwarg in kwonlyargs: - if kwarg not in arg2value: - if kwonlydefaults and kwarg in kwonlydefaults: - arg2value[kwarg] = kwonlydefaults[kwarg] - else: - missing += 1 - if missing: - _missing_arguments(f_name, kwonlyargs, False, arg2value) - return arg2value - - - def _too_many(f_name, args, kwonly, varargs, defcount, given, values): - atleast = len(args) - defcount - kwonly_given = len([arg for arg in kwonly if arg in values]) - if varargs: - plural = atleast != 1 - sig = "at least %d" % (atleast,) - elif defcount: - plural = True - sig = "from %d to %d" % (atleast, len(args)) - else: - plural = len(args) != 1 - sig = str(len(args)) - kwonly_sig = "" - if kwonly_given: - msg = " positional argument%s (and %d keyword-only argument%s)" - kwonly_sig = (msg % ("s" if given != 1 else "", kwonly_given, - "s" if kwonly_given != 1 else "")) - raise TypeError("%s() takes %s positional argument%s but %d%s %s given" % - (f_name, sig, "s" if plural else "", given, kwonly_sig, - "was" if given == 1 and not kwonly_given else "were")) - - - def _missing_arguments(f_name, argnames, pos, values): - names = [repr(name) for name in argnames if name not in values] - missing = len(names) - if missing == 1: - s = names[0] - elif missing == 2: - s = "{} and {}".format(*names) - else: - tail = ", {} and {}".format(*names[-2:]) - del names[-2:] - s = ", ".join(names) + tail - raise TypeError("%s() missing %i required %s argument%s: %s" % - (f_name, missing, - "positional" if pos else "keyword-only", - "" if missing == 1 else "s", s)) - - -else: - getfullargspec = inspect.getfullargspec - getcallargs = inspect.getcallargs diff --git a/pinfer/p.py b/pinfer/p.py deleted file mode 100644 index 451038dd13cd..000000000000 --- a/pinfer/p.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -"""Stub to run pinfer on a module. - -Usage: - - p.py targetmod testfile [outfile] [ -- testargs] - -Where: - - targetmod: the full target module (e.g. textwrap) - testfile: the full test module file (e.g. test/test_textwrap.py) - outfile: where to write the annotated module. If unspecified, will - write stubs at end of stdout. - -Example invocation: - - python3 p.py textwrap test/test_textwrap.py -""" - - -import sys -import imp -import pinfer -import os -import atexit -import inspect - -iport = __builtins__.__import__ -watched = set() - - -def inferring_import(*args, **kwargs): - module = iport(*args, **kwargs) - if module not in watched: - watched.add(module) - pinfer.infer_module(module) - return module - - -def main(): - if '--' in sys.argv: - argslen = sys.argv.index('--') - else: - argslen = len(sys.argv) - args = sys.argv[1:argslen] - del sys.argv[1:argslen + 1] - - if len(args) == 2: - targetpackage, testfile = args - outfile = None - elif len(args) == 3: - targetpackage, testfile, outfile = args - else: - sys.stderr.write('Usage: %s targetmodule testfile [outfile] [ -- testargs]\n' % - sys.argv[0]) - sys.exit(2) - - # help us with local imports - filemodule = os.path.dirname(os.path.abspath(testfile)) - sys.path.append(filemodule) - - targetmod = __import__(targetpackage) - targetfile = inspect.getfile(targetmod) - pinfer.infer_module(targetmod) - - if outfile: - @atexit.register - def rewrite_file(targetfile=targetfile, outfile=outfile, pinfer=pinfer): - if targetfile.endswith(".pyc"): - targetfile = targetfile[0:-1] - annotated = pinfer.annotate_file(targetfile) - open(outfile, "w").write(annotated) - else: - pinfer.dump_at_exit() - - pinfer.ignore_files.add(os.path.abspath(testfile)) - - # run testfile as main - del sys.modules['__main__'] - imp.load_source('__main__', testfile) - -if __name__ == '__main__': - main() diff --git a/pinfer/pinfer.py b/pinfer/pinfer.py deleted file mode 100644 index 5366e7720842..000000000000 --- a/pinfer/pinfer.py +++ /dev/null @@ -1,686 +0,0 @@ -"""Tools for runtime type inference""" - -import inspect -from inspect3 import getfullargspec, getcallargs -import types -import codecs -import os -import tokenize -try: - from StringIO import StringIO - from unparse import Unparser -except: - from io import StringIO - from unparse3 import Unparser -import ast - - -MAX_INFERRED_TUPLE_LENGTH = 10 -PREFERRED_LINE_LENGTH = 79 - - -var_db = {} # (location, variable) -> type -func_argid_db = {} # funcid -> argspec -func_arg_db = {} # (funcid, name) -> type -func_return_db = {} # funcname -> type -func_source_db = {} # funcid -> source string -#func_info_db = {} # funcid -> (class, name, argspec, file, line, source) -ignore_files = set() - -# The type inferencing wrapper should not be reentrant. It's not, in theory, calling -# out to any external code which we would want to infer the types of. However, -# sometimes we do something like infer_type(arg.keys()) or infer_type(arg.values()) if -# the arg is a collection, and we want to know about the types of its elements. .keys(), -# .values(), etc. can be overloaded, possibly to a method we've wrapped. This can become -# infinitely recursive, particuarly because on something like arg.keys(), keys() gets passed -# arg as the first parameter, so if we've wrapped keys() we'll try to infer_type(arg), -# which will detect it's a dictionary, call infer_type(arg.keys()), recurse and so on. -# We ran in to this problem with collections.OrderedDict. -# To prevent reentrancy, we set is_performing_inference = True iff we're in the middle of -# inferring the types of a function. If we try to run another function we've wrapped, -# we skip type inferencing so we can't accidentally infinitely recurse. -is_performing_inference = False - - -def reset(): - global var_db, func_argid_db, func_arg_db, func_return_db, func_source_db - global ignore_files, is_performing_inference - var_db = {} - func_arg_db = {} - func_return_db = {} - # we don't actually want to clear these on reset(), or we'll - # lose the functions we've already wrapped forever. - #func_source_db = {} - #func_argid_db = {} - is_performing_inference = False - ignore_files = set() - - -def format_state(pretty=False): - lines = [] - for loc, var in sorted(var_db.keys()): - lines.append('%s: %s' % (var, var_db[(loc, var)])) - funcnames = sorted(set(func_return_db.keys())) - prevclass = '' - indent = '' - for funcid in funcnames: - curclass, name, sourcefile, sourceline = funcid - if curclass != prevclass: - if curclass: - lines.append('class %s(...):' % curclass) - indent = ' ' * 4 - else: - indent = '' - prevclass = curclass - - lines.append(format_sig(funcid, name, indent, pretty)) - return '\n'.join(lines) - - -def unparse_ast(node): - buf = StringIO() - Unparser(node, buf) - return buf.getvalue().strip() - - -def format_sig(funcid, fname, indent, pretty, defaults=[]): - (argnames, varargs, varkw, _, kwonlyargs, _, _) = func_argid_db[funcid] - - # to get defaults, parse the function, get the nodes for the - # defaults, then unparse them - try: - fn_ast = ast.parse(func_source_db[funcid].strip()).body[0] - - # override fname if we parsed a different one - fname = fn_ast.name - - defaults = [unparse_ast(dn) for dn in fn_ast.args.defaults] - - if hasattr(fn_ast.args, 'kw_defaults'): - kwonly_defaults = [unparse_ast(dn) for dn in fn_ast.args.kw_defaults] - else: - kwonly_defaults = [] - except: - defaults, kwonly_defaults = [], [] - finally: - # pad defaults to match the length of args - defaults = ([None] * (len(argnames) - len(defaults))) + defaults - kwonly_defaults = ([None] * (len(kwonlyargs) - len(kwonly_defaults))) + kwonly_defaults - - args = [('', arg, default) for (arg, default) in zip(argnames, defaults)] - - if varargs: - args += [('*', varargs, None)] - elif len(kwonlyargs) > 0: - args += [('*', '', None)] - if len(kwonlyargs) > 0: - args += [('', arg, default) for (arg, default) in zip(kwonlyargs, kwonly_defaults)] - if varkw: - args += [('**', varkw, None)] - - argstrs = [] - for i, (prefix, arg, default) in enumerate(args): - argstr = prefix + arg - - # Omit type of self argument. - if (funcid, arg) in func_arg_db and not (i == 0 and arg == 'self'): - argstr += ': %s' % func_arg_db[(funcid, arg)] - - if default: - argstr += ' = %s' % default - - argstrs.append(argstr) - - ret = str(func_return_db.get(funcid, Unknown())) - - sig = 'def %s(%s) -> %s' % (fname, ', '.join(argstrs), ret) - if not pretty or len(sig) <= PREFERRED_LINE_LENGTH or not args: - return indent + sig - - else: - # Format into multiple lines to conserve horizontal space. - first = indent + 'def %s(' % fname - extra_indent = first.index('(') + 1 - - decl = indent + first - decl += (',\n' + indent + ' ' * extra_indent).join(argstrs) - decl += ')\n%s -> %s' % (indent + ' ' * (extra_indent - 4), ret) - return decl - - -def annotate_file(path): - # this should be documented somewhere... - INDENT_TOKEN = 5 - - with open(path, 'r') as targetfile: - source = targetfile.read() - - line_offsets = [] - source_length = 0 - for line in source.split('\n'): - line_offsets.append(source_length) - source_length = source_length + len(line) + 1 - - funcids = set(funcid for funcid, arg in func_arg_db) - - # list of (oldstart, oldend, replacement) - replacements = [] # type: List[Tuple[Int, Int, String]] - - for funcid in funcids: - class_name, name, sourcefile, def_start_line = funcid - if sourcefile != path: - continue - - func_source = func_source_db[funcid] - tokens = list(tokenize.generate_tokens(StringIO(func_source).readline)) - assert len(tokens) > 0 - - # we're making the assumption that the def at least gets to start on - # it's own line, which is fine for non-lambdas - - if tokens[0][0] == INDENT_TOKEN: - indent = tokens[0][1] - del tokens[0] - else: - indent = '' - - # Find the first indent, which should be between the end of the def - # and before the start of the body. Then find the preceding colon, - # which should be at the end of the def. - - for indent_loc in range(len(tokens)): - if tokens[indent_loc][0] == INDENT_TOKEN: - function_is_one_line = False - break - else: - function_is_one_line = True - - if function_is_one_line: - # we're also making the assumption that the def has an indent on the - # line following the signature, which is true almost all of the time. - # If this is not the case, we should just leave a comment above the - # function, although I might not have time to do that now. - continue - - for def_end_loc in range(indent_loc, -1, -1): - if tokens[def_end_loc][1] == ':': - break - - assert def_end_loc > 0 - - def_end_line, def_end_col = tokens[def_end_loc][2] - def_end_line -= 1 # the tokenizer apparently 1-indexes lines - def_end_line += def_start_line - - def_start_offset = line_offsets[def_start_line] - def_end_offset = line_offsets[def_end_line] + def_end_col - - annotated_def = format_sig(funcid, name, indent, True) - - replacements.append((def_start_offset, def_end_offset, annotated_def)) - - # ideally, we'd put this after the docstring - replacements.append((0, 0, "from typing import List, Dict, Set, Tuple, Function, Pattern, Match, Union, Optional\n")) - - # absurdly inefficient algorithm: replace with O(n) writer - - for (start, end, replacement) in sorted(replacements, key=lambda r: r[0], reverse=True): - source = source[0:start] + replacement + source[end:] - - return source - - -def dump(): - s = format_state(pretty=True) - if s: - print() - print('INFERRED TYPES:') - print(s) - reset() - - -def dump_at_exit(): - import atexit - atexit.register(dump) - - -def get_defining_file(obj): - try: - path = os.path.abspath(inspect.getfile(obj)) - if path.endswith('.pyc'): - path = path[0:-1] - return path - except: - return None - - -def infer_var(name, value): - key = (None, name) - update_var_db(key, value) - - -def infer_attrs(x): - if hasattr(x, '__class__'): - t = x.__class__ - else: - t = type(x) - cls = t.__name__ - typedict = t.__dict__ - for dict in x.__dict__, typedict: - for attr, value in dict.items(): - if attr in ('__dict__', '__doc__', '__module__', '__weakref__'): - continue - if type(value) is type(infer_attrs) and dict is typedict: - # Skip methods. - continue - key = (None, '%s.%s' % (cls, attr)) - update_var_db(key, value) - - -def infer_method_signature(class_name): - def decorator(func): - return infer_signature(func, class_name) - return decorator - - -def infer_signature(func, class_name=''): - """Decorator that infers the signature of a function.""" - - # infer_method_signature should be idempotent - if hasattr(func, '__is_inferring_sig'): - return func - - assert func.__module__ != infer_method_signature.__module__ - - try: - funcfile = get_defining_file(func) - funcsource, sourceline = inspect.getsourcelines(func) - sourceline -= 1 # getsourcelines is apparently 1-indexed - except: - return func - - funcid = (class_name, func.__name__, funcfile, sourceline) - func_source_db[funcid] = ''.join(funcsource) - - try: - func_argid_db[funcid] = getfullargspec(func) - vargs_name, kwargs_name = func_argid_db[funcid][1], func_argid_db[funcid][2] - except TypeError: - # Not supported. - return func - - def wrapper(*args, **kwargs): - global is_performing_inference - # If we're already doing inference, we should be in our own code, not code we're checking. - # Not doing this check sometimes results in infinite recursion. - - if is_performing_inference: - return func(*args, **kwargs) - - expecting_type_error, got_type_error, got_exception = False, False, False - - is_performing_inference = True - try: - callargs = getcallargs(func, *args, **kwargs) - - # we have to handle *args and **kwargs separately - if vargs_name: - va = callargs.pop(vargs_name) - if kwargs_name: - kw = callargs.pop(kwargs_name) - - arg_db = {arg: infer_value_type(value) for arg, value in callargs.items()} - - # *args and **kwargs need to merge the types of all their values - if vargs_name: - arg_db[vargs_name] = union_many_types(*[infer_value_type(v) for v in va]) - if kwargs_name: - arg_db[kwargs_name] = union_many_types(*[infer_value_type(v) for v in kw.values()]) - - except TypeError: - got_exception = expecting_type_error = True - except: - got_exception = True - finally: - is_performing_inference = False - - try: - ret = func(*args, **kwargs) - except TypeError: - got_type_error = got_exception = True - raise - except: - got_exception = True - raise - finally: - if not got_exception: - assert not expecting_type_error - - # if we didn't get a TypeError, update the actual database - for arg, t in arg_db.items(): - update_db(func_arg_db, (funcid, arg), t) - - # if we got an exception, we don't have a ret - if not got_exception: - is_performing_inference = True - try: - type = infer_value_type(ret) - update_db(func_return_db, funcid, type) - except: - pass - finally: - is_performing_inference = False - - return ret - - if hasattr(func, '__name__'): - wrapper.__name__ = func.__name__ - wrapper.__is_inferring_sig = True - return wrapper - - -def infer_class(cls): - """Class decorator for inferring signatures of all methods of the class.""" - for attr, value in cls.__dict__.items(): - if type(value) is type(infer_class): - setattr(cls, attr, infer_method_signature(cls.__name__)(value)) - return cls - - -def infer_module(namespace): - if hasattr(namespace, '__dict__'): - namespace = namespace.__dict__ - for name, value in list(namespace.items()): - if inspect.isfunction(value): - namespace[name] = infer_signature(value) - elif inspect.isclass(value): - namespace[name] = infer_class(value) - - -def update_var_db(key, value): - type = infer_value_type(value) - update_db(var_db, key, type) - - -def update_db(db, key, type): - if key not in db: - db[key] = type - else: - db[key] = combine_types(db[key], type) - - -def merge_db(db, other): - assert id(db) != id(other) - for key in other.keys(): - if key not in db: - db[key] = other[key] - else: - db[key] = combine_types(db[key], other[key]) - - -def infer_value_type(value, depth=0): - # Prevent infinite recursion - if depth > 5: - return Unknown() - depth += 1 - - if value is None: - return None - elif isinstance(value, list): - return Generic('List', [infer_value_types(value, depth)]) - elif isinstance(value, dict): - keytype = infer_value_types(value.keys(), depth) - valuetype = infer_value_types(value.values(), depth) - return Generic('Dict', (keytype, valuetype)) - elif isinstance(value, tuple): - if len(value) <= MAX_INFERRED_TUPLE_LENGTH: - return Tuple(infer_value_type(item, depth) - for item in value) - else: - return Generic('TupleSequence', [infer_value_types(value, depth)]) - elif isinstance(value, set): - return Generic('Set', [infer_value_types(value, depth)]) - elif isinstance(value, types.MethodType) or isinstance(value, types.FunctionType): - return Instance(Function) - else: - for t in type(value).mro(): - if get_defining_file(t) in ignore_files: - continue - elif t is object: - return Any() - elif hasattr(types, 'InstanceType') and t is types.InstanceType: - return Any() - else: - return Instance(t) - else: - return Any() - - -def infer_value_types(values, depth=0): - """Infer a single type for an iterable of values. - - >>> infer_value_types((1, 'x')) - Union(int, str) - >>> infer_value_types([]) - Unknown - """ - inferred = Unknown() - for value in sample(values): - type = infer_value_type(value, depth) - inferred = combine_types(inferred, type) - return inferred - - -def sample(values): - # TODO only return a sample of values - return list(values) - - -def union_many_types(*types): - union = Unknown() - for t in types: - union = combine_types(union, t) - return union - - -def combine_types(x, y): - """Perform a union of two types. - - >>> combine_types(Instance(int), None) - Optional[int] - """ - if isinstance(x, Unknown): - return y - if isinstance(y, Unknown): - return x - if isinstance(x, Any): - return x - if isinstance(y, Any): - return y - if isinstance(x, Union): - return combine_either(x, y) - if isinstance(y, Union): - return combine_either(y, x) - if x == y: - return x - return simplify_either([x], [y]) - - -def combine_either(either, x): - if isinstance(x, Union): - xtypes = x.types - else: - xtypes = [x] - return simplify_either(either.types, xtypes) - - -def simplify_either(x, y): - numerics = [Instance(int), Instance(float), Instance(complex)] - - # TODO this is O(n**2); use an O(n) algorithm instead - result = list(x) - for type in y: - if isinstance(type, Generic): - for i, rt in enumerate(result): - if isinstance(rt, Generic) and type.typename == rt.typename: - result[i] = Generic(rt.typename, - (combine_types(t, s) - for t, s in zip(type.args, rt.args))) - break - else: - result.append(type) - elif isinstance(type, Tuple): - for i, rt in enumerate(result): - if isinstance(rt, Tuple) and len(type) == len(rt): - result[i] = Tuple(combine_types(t, s) - for t, s in zip(type.itemtypes, - rt.itemtypes)) - break - else: - result.append(type) - elif type in numerics: - for i, rt in enumerate(result): - if rt in numerics: - result[i] = numerics[max(numerics.index(rt), numerics.index(type))] - break - else: - result.append(type) - elif isinstance(type, Instance): - for i, rt in enumerate(result): - if isinstance(rt, Instance): - # Union[A, SubclassOfA] -> A - # Union[A, A] -> A, because issubclass(A, A) == True, - if issubclass(type.typeobj, rt.typeobj): - break - elif issubclass(rt.typeobj, type.typeobj): - result[i] = type - break - else: - result.append(type) - elif type not in result: - result.append(type) - - if len(result) > 1: - return Union(result) - else: - return result[0] - - -class TypeBase(object): - """Abstract base class of all type objects. - - Type objects use isinstance tests librarally -- they don't support duck - typing well. - """ - - def __eq__(self, other): - if type(other) is not type(self): - return False - for attr in self.__dict__: - if getattr(other, attr) != getattr(self, attr): - return False - return True - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return str(self) - - -class Instance(TypeBase): - def __init__(self, typeobj): - assert not inspect.isclass(typeobj) or not issubclass(typeobj, TypeBase) - self.typeobj = typeobj - - def __str__(self): - # cheat on regular expression objects which have weird class names - # to be consistent with typing.py - if self.typeobj == Pattern: - return "Pattern" - elif self.typeobj == Match: - return "Match" - else: - return self.typeobj.__name__ - - def __repr__(self): - return 'Instance(%s)' % self - - -class Generic(TypeBase): - def __init__(self, typename, args): - self.typename = typename - self.args = tuple(args) - - def __str__(self): - return '%s[%s]' % (self.typename, ', '.join(str(t) - for t in self.args)) - - -class Tuple(TypeBase): - def __init__(self, itemtypes): - self.itemtypes = tuple(itemtypes) - - def __len__(self): - return len(self.itemtypes) - - def __str__(self): - return 'Tuple[%s]' % (', '.join(str(t) for t in self.itemtypes)) - - -class Union(TypeBase): - def __init__(self, types): - assert len(types) > 1 - self.types = tuple(types) - - def __eq__(self, other): - if type(other) is not Union: - return False - # TODO this is O(n**2); use an O(n) algorithm instead - for t in self.types: - if t not in other.types: - return False - for t in other.types: - if t not in self.types: - return False - return True - - def __str__(self): - types = list(self.types) - if str != bytes: # on Python 2 str == bytes - if Instance(bytes) in types and Instance(str) in types: - # we Union[bytes, str] -> AnyStr as late as possible so we avoid - # corner cases like subclasses of bytes or str - types.remove(Instance(bytes)) - types.remove(Instance(str)) - types.append(Instance(AnyStr)) - if len(types) == 1: - return str(types[0]) - elif len(types) == 2 and None in types: - type = [t for t in types if t is not None][0] - return 'Optional[%s]' % type - else: - return 'Union[%s]' % (', '.join(sorted(str(t) for t in types))) - - -class Unknown(TypeBase): - def __str__(self): - return 'Unknown' - - def __repr__(self): - return 'Unknown()' - - -class Any(TypeBase): - def __str__(self): - return 'Any' - - def __repr__(self): - return 'Any()' - - -class AnyStr(object): pass -class Function(object): pass -import re -Pattern = type(re.compile(u'')) -Match = type(re.match(u'', u'')) diff --git a/pinfer/test_pinfer.py b/pinfer/test_pinfer.py deleted file mode 100644 index b50d60e263f5..000000000000 --- a/pinfer/test_pinfer.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Test cases for the infer module""" - -import unittest - -from pinfer import Instance, Generic, Tuple, Union, Unknown -import pinfer - - -class TestInfer(unittest.TestCase): - def setUp(self): - self.int = Instance(int) - self.float = Instance(float) - - def tearDown(self): - pinfer.reset() - - def test_instance(self): - i = self.int - self.assertEqual(i.typeobj, int) - self.assertEqual(str(i), 'int') - self.assertEqual(repr(i), 'Instance(int)') - - self.assertTrue(i == Instance(int)) - self.assertFalse(i != Instance(int)) - self.assertTrue(i != self.float) - self.assertFalse(i == self.float) - self.assertNotEqual(i, None) - - def test_generic_with_one_arg(self): - g = Generic('List', [self.int]) - self.assertEqual(g.typename, 'List') - self.assertEqual(str(g.args), '(Instance(int),)') - self.assertEqual(str(g), 'List[int]') - self.assertEqual(repr(g), 'List[int]') - - self.assertEqual(g, Generic('List', [self.int])) - self.assertNotEqual(g, Generic('Set', [self.int])) - self.assertNotEqual(g, Generic('List', [self.float])) - self.assertNotEqual(g, self.int) - - def test_generic_with_two_args(self): - g = Generic('Dict', (self.int, self.float)) - self.assertEqual(g.typename, 'Dict') - self.assertEqual(str(g), 'Dict[int, float]') - - def test_tuple(self): - t0 = Tuple(()) - t1 = Tuple([self.int]) - t2 = Tuple((self.float, self.int)) - self.assertEqual(t0.itemtypes, ()) - self.assertEqual(str(t1.itemtypes[0]), 'int') - self.assertEqual(str(t2.itemtypes[0]), 'float') - self.assertEqual(str(t2.itemtypes[1]), 'int') - self.assertEqual(str(t0), 'Tuple[]') - self.assertEqual(str(t1), 'Tuple[int]') - self.assertEqual(str(t2), 'Tuple[float, int]') - - self.assertEqual(t1, Tuple([self.int])) - self.assertNotEqual(t1, Tuple([self.float])) - self.assertNotEqual(t1, Tuple([self.int, self.int])) - self.assertNotEqual(t1, self.int) - - def test_either(self): - i = self.int - f = self.float - s = Instance(str) - - e2 = Union((i, f)) - self.assertEqual(len(e2.types), 2) - self.assertEqual(str(e2), 'Union[float, int]') - - self.assertEqual(e2, Union((i, f))) - self.assertEqual(e2, Union((f, i))) - self.assertNotEqual(e2, Union((i, s))) - self.assertNotEqual(e2, Union((i, f, s))) - self.assertNotEqual(Union((i, f, s)), e2) - self.assertNotEqual(e2, i) - - def test_either_as_optional(self): - optint = Union((self.int, None)) - self.assertEqual(str(optint), 'Optional[int]') - optfloat = Union((None, self.float)) - self.assertEqual(str(optfloat), 'Optional[float]') - eithernone = Union((self.int, self.float, None)) - self.assertEqual(str(eithernone), 'Union[None, float, int]') - - def test_unknown(self): - unknown = Unknown() - self.assertEqual(str(unknown), 'Unknown') - self.assertEqual(repr(unknown), 'Unknown()') - - self.assertEqual(unknown, Unknown()) - self.assertNotEqual(unknown, self.int) - - def test_combine_types(self): - i = self.int - f = self.float - s = Instance(str) - c = Instance(complex) - class Foo: pass - o = Instance(Foo) - - # Simple types - self.assert_combine(i, i, i) - self.assert_combine(s, s, s) - self.assert_combine(i, s, Union((i, s))) - self.assert_combine(i, None, Union((i, None))) - # Unknowns - self.assert_combine(i, Unknown(), i) - self.assert_combine(Unknown(), Unknown(), Unknown()) - # Union types - self.assert_combine(o, Union((f, s)), Union((o, f, s))) - self.assert_combine(i, Union((i, s)), Union((i, s))) - self.assert_combine(Union((o, f)), Union((o, s)), Union((o, f, s))) - # Tuple types - self.assert_combine(Tuple([i, i]), Tuple([i, i]), Tuple([i, i])) - self.assert_combine(Tuple([i, i]), Tuple([o, s]), - Tuple([Union([o, i]), Union([s, i])])) - # Numeric types - self.assert_combine(i, f, f) - self.assert_combine(i, c, c) - self.assert_combine(c, f, c) - # Unions with numerics - self.assert_combine(i, Union((o, f)), Union((o, f))) - self.assert_combine(Union((o, f)), i, Union((o, f))) - self.assert_combine(Union((o, i)), f, Union((o, f))) - # Tuples with numerics - self.assert_combine(Tuple([i, i]), Tuple([f, i]), Tuple([f, i])) - self.assert_combine(Tuple([i, i]), Tuple([f, o]), Tuple([f, Union((i, o))])) - self.assert_combine(Tuple([f, i]), Tuple([i, o]), Tuple([f, Union((i, o))])) - - def test_combine_special_cases(self): - i = self.int - f = self.float - u = Unknown() - def list_(x): - return Generic('List', [x]) - # Simplify generic types. - self.assert_combine(list_(i), list_(u), list_(i)) - - def assert_combine(self, t, s, combined): - self.assertEqual(pinfer.combine_types(t, s), combined) - self.assertEqual(pinfer.combine_types(s, t), combined) - - def test_sample(self): - sample = pinfer.sample - self.assertEqual(sample(()), []) - self.assertEqual(sample((1, 2)), [1, 2]) - self.assertEqual(sample([]), []) - self.assertEqual(sample([1]), [1]) - self.assertEqual(sample([1, 2]), [1, 2]) - # TODO larger collections - - def test_infer_simple_value_type(self): - self.assert_infer_type(1, 'int') - self.assert_infer_type('', 'str') - self.assert_infer_type(None, 'None') - - def test_infer_collection_type(self): - # List - self.assert_infer_type([], 'List[Unknown]') - self.assert_infer_type([1], 'List[int]') - self.assert_infer_type([1, None], 'List[Optional[int]]') - # Dict - self.assert_infer_type({1: 'x', 2: None}, - 'Dict[int, Optional[str]]') - # Set - self.assert_infer_type({1, None}, 'Set[Optional[int]]') - # Tuple - self.assert_infer_type((1, 'x'), 'Tuple[int, str]') - self.assert_infer_type((1, None) * 100, 'TupleSequence[Optional[int]]') - - def assert_infer_type(self, value, type): - self.assertEqual(str(pinfer.infer_value_type(value)), type) - - def test_infer_variables(self): - pinfer.infer_var('x', 1) - self.assert_infer_state('x: int') - pinfer.infer_var('x', 1) - pinfer.infer_var('x', None) - pinfer.infer_var('y', 1.1) - self.assert_infer_state('x: Optional[int]\n' - 'y: float') - - def test_infer_instance_var(self): - class A: pass - a = A() - a.x = 1 - a.y = 'x' - pinfer.infer_attrs(a) - self.assert_infer_state('A.x: int\n' - 'A.y: str') - - def test_infer_class_var(self): - class A: - x = 1.1 - pinfer.infer_attrs(A()) - self.assert_infer_state('A.x: float') - - def test_infer_function_attr(self): - class A: - def f(self): pass - a = A() - a.g = lambda x: 1 - pinfer.infer_attrs(a) - self.assert_infer_state('A.g: Function') - - def test_infer_simple_function_signature(self): - @pinfer.infer_signature - def f(a): - return 'x' - f(1) - f(None) - self.assertEqual(f.__name__, 'f') - self.assert_infer_state('def f(a: Optional[int]) -> str') - - def test_infer_function_with_two_args(self): - @pinfer.infer_signature - def f(x, y): - return x * y - f(1, 2) - f(1, 'x') - self.assert_infer_state( - 'def f(x: int, y: Union[int, str]) -> Union[int, str]') - - def test_infer_method(self): - class A: - @pinfer.infer_signature - def f(self, x): pass - A().f('x') - self.assert_infer_state('def f(self, x: str) -> None') - - def test_infer_default_arg_values(self): - @pinfer.infer_signature - def f(x=1, y=None): pass - f() - self.assert_infer_state('def f(x: int, y: None) -> None') - f('x') - f('x', 1.1) - f() - self.assert_infer_state( - 'def f(x: Union[int, str], y: Optional[float]) -> None') - - def test_infer_varargs(self): - @pinfer.infer_signature - def f(x, *y): pass - f(1) - f(1, 'x', None) - self.assert_infer_state('def f(x: int, *y: Optional[str]) -> None') - f(1) - self.assert_infer_state('def f(x: int, *y: Unknown) -> None') - - def test_infer_keyword_args(self): - @pinfer.infer_signature - def f(x): pass - f(x=1) - self.assert_infer_state('def f(x: int) -> None') - - @pinfer.infer_signature - def f(x='x'): pass - f(x=1) - self.assert_infer_state('def f(x: int) -> None') - - def test_infer_keyword_varargs(self): - @pinfer.infer_signature - def f(a, **kwargs): pass - f(None, x=1, y='x') - self.assert_infer_state( - 'def f(a: None, **kwargs: Union[int, str]) -> None') - - def test_infer_class(self): - @pinfer.infer_class - class A: - def f(self, x): return 0 - A().f('x') - self.assert_infer_state('class A(...):\n' - ' def f(self, x: str) -> int') - - @pinfer.infer_class - class A: - def f(self, x): return 0 - @pinfer.infer_class - class B: - def f(self): pass - def g(self): pass - A().f('') - B().f() - B().g() - self.assert_infer_state('class A(...):\n' - ' def f(self, x: str) -> int\n' - 'class B(...):\n' - ' def f(self) -> None\n' - ' def g(self) -> None') - - def assert_infer_state(self, expected): - state = pinfer.format_state() - self.assertEqual(state, expected) - pinfer.reset() - - -if __name__ == '__main__': - unittest.main() diff --git a/pinfer/test_pinfer3.py b/pinfer/test_pinfer3.py deleted file mode 100644 index 688e8c09f6af..000000000000 --- a/pinfer/test_pinfer3.py +++ /dev/null @@ -1,31 +0,0 @@ -""" tests cases that require python3 syntax """ - -import unittest -import pinfer - -# Include all of the shared unit tests -from test_pinfer import TestInfer - - -class TestInfer3(unittest.TestCase): - def test_infer_keyword_only_args(self): - # decorators break the parsing - def f(x, *, y=0): pass - f = pinfer.infer_signature(f) - f(1, y='x') - self.assert_infer_state( - 'def f(x: int, *, y: str = 0) -> None') - - def f(*, x=None, y=None): pass - f = pinfer.infer_signature(f) - f(y='x') - self.assert_infer_state( - 'def f(*, x: None = None, y: str = None) -> None') - - def assert_infer_state(self, expected): - state = pinfer.format_state() - self.assertEqual(state, expected) - pinfer.reset() - -if __name__ == '__main__': - unittest.main() diff --git a/pinfer/unparse.py b/pinfer/unparse.py deleted file mode 100644 index 6e1e49336ead..000000000000 --- a/pinfer/unparse.py +++ /dev/null @@ -1,610 +0,0 @@ -# From Python 2's Demo/parser/unparse.py -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved - -"Usage: unparse.py " -import sys -import ast -import cStringIO -import os - -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -def interleave(inter, f, seq): - """Call f on each item in seq, calling inter() in between. - """ - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - -class Unparser: - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded. """ - - def __init__(self, tree, file = sys.stdout): - """Unparser(tree, file=sys.stdout) -> None. - Print the source for tree to file.""" - self.f = file - self.future_imports = [] - self._indent = 0 - self.dispatch(tree) - self.f.write("") - self.f.flush() - - def fill(self, text = ""): - "Indent a piece of text, according to the current indentation level" - self.f.write("\n"+" "*self._indent + text) - - def write(self, text): - "Append a piece of text to the current line." - self.f.write(text) - - def enter(self): - "Print ':', and increase the indentation." - self.write(":") - self._indent += 1 - - def leave(self): - "Decrease the indentation level." - self._indent -= 1 - - def dispatch(self, tree): - "Dispatcher function, dispatching tree type T to method _T." - if isinstance(tree, list): - for t in tree: - self.dispatch(t) - return - meth = getattr(self, "_"+tree.__class__.__name__) - meth(tree) - - - ############### Unparsing methods ###################### - # There should be one method per concrete grammar type # - # Constructors should be grouped by sum type. Ideally, # - # this would follow the order in the grammar, but # - # currently doesn't. # - ######################################################## - - def _Module(self, tree): - for stmt in tree.body: - self.dispatch(stmt) - - # stmt - def _Expr(self, tree): - self.fill() - self.dispatch(tree.value) - - def _Import(self, t): - self.fill("import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _ImportFrom(self, t): - # A from __future__ import may affect unparsing, so record it. - if t.module and t.module == '__future__': - self.future_imports.extend(n.name for n in t.names) - - self.fill("from ") - self.write("." * t.level) - if t.module: - self.write(t.module) - self.write(" import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _Assign(self, t): - self.fill() - for target in t.targets: - self.dispatch(target) - self.write(" = ") - self.dispatch(t.value) - - def _AugAssign(self, t): - self.fill() - self.dispatch(t.target) - self.write(" "+self.binop[t.op.__class__.__name__]+"= ") - self.dispatch(t.value) - - def _Return(self, t): - self.fill("return") - if t.value: - self.write(" ") - self.dispatch(t.value) - - def _Pass(self, t): - self.fill("pass") - - def _Break(self, t): - self.fill("break") - - def _Continue(self, t): - self.fill("continue") - - def _Delete(self, t): - self.fill("del ") - interleave(lambda: self.write(", "), self.dispatch, t.targets) - - def _Assert(self, t): - self.fill("assert ") - self.dispatch(t.test) - if t.msg: - self.write(", ") - self.dispatch(t.msg) - - def _Exec(self, t): - self.fill("exec ") - self.dispatch(t.body) - if t.globals: - self.write(" in ") - self.dispatch(t.globals) - if t.locals: - self.write(", ") - self.dispatch(t.locals) - - def _Print(self, t): - self.fill("print ") - do_comma = False - if t.dest: - self.write(">>") - self.dispatch(t.dest) - do_comma = True - for e in t.values: - if do_comma:self.write(", ") - else:do_comma=True - self.dispatch(e) - if not t.nl: - self.write(",") - - def _Global(self, t): - self.fill("global ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Yield(self, t): - self.write("(") - self.write("yield") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Raise(self, t): - self.fill('raise ') - if t.type: - self.dispatch(t.type) - if t.inst: - self.write(", ") - self.dispatch(t.inst) - if t.tback: - self.write(", ") - self.dispatch(t.tback) - - def _TryExcept(self, t): - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - - for ex in t.handlers: - self.dispatch(ex) - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _TryFinally(self, t): - if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept): - # try-except-finally - self.dispatch(t.body) - else: - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - - self.fill("finally") - self.enter() - self.dispatch(t.finalbody) - self.leave() - - def _ExceptHandler(self, t): - self.fill("except") - if t.type: - self.write(" ") - self.dispatch(t.type) - if t.name: - self.write(" as ") - self.dispatch(t.name) - self.enter() - self.dispatch(t.body) - self.leave() - - def _ClassDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("class "+t.name) - if t.bases: - self.write("(") - for a in t.bases: - self.dispatch(a) - self.write(", ") - self.write(")") - self.enter() - self.dispatch(t.body) - self.leave() - - def _FunctionDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("def "+t.name + "(") - self.dispatch(t.args) - self.write(")") - self.enter() - self.dispatch(t.body) - self.leave() - - def _For(self, t): - self.fill("for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _If(self, t): - self.fill("if ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # collapse nested ifs into equivalent elifs. - while (t.orelse and len(t.orelse) == 1 and - isinstance(t.orelse[0], ast.If)): - t = t.orelse[0] - self.fill("elif ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # final else - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _While(self, t): - self.fill("while ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _With(self, t): - self.fill("with ") - self.dispatch(t.context_expr) - if t.optional_vars: - self.write(" as ") - self.dispatch(t.optional_vars) - self.enter() - self.dispatch(t.body) - self.leave() - - # expr - def _Str(self, tree): - # if from __future__ import unicode_literals is in effect, - # then we want to output string literals using a 'b' prefix - # and unicode literals with no prefix. - if "unicode_literals" not in self.future_imports: - self.write(repr(tree.s)) - elif isinstance(tree.s, str): - self.write("b" + repr(tree.s)) - elif isinstance(tree.s, unicode): - self.write(repr(tree.s).lstrip("u")) - else: - assert False, "shouldn't get here" - - def _Name(self, t): - self.write(t.id) - - def _Repr(self, t): - self.write("`") - self.dispatch(t.value) - self.write("`") - - def _Num(self, t): - repr_n = repr(t.n) - # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2. - if repr_n.startswith("-"): - self.write("(") - # Substitute overflowing decimal literal for AST infinities. - self.write(repr_n.replace("inf", INFSTR)) - if repr_n.startswith("-"): - self.write(")") - - def _List(self, t): - self.write("[") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("]") - - def _ListComp(self, t): - self.write("[") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("]") - - def _GeneratorExp(self, t): - self.write("(") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write(")") - - def _SetComp(self, t): - self.write("{") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _DictComp(self, t): - self.write("{") - self.dispatch(t.key) - self.write(": ") - self.dispatch(t.value) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _comprehension(self, t): - self.write(" for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - for if_clause in t.ifs: - self.write(" if ") - self.dispatch(if_clause) - - def _IfExp(self, t): - self.write("(") - self.dispatch(t.body) - self.write(" if ") - self.dispatch(t.test) - self.write(" else ") - self.dispatch(t.orelse) - self.write(")") - - def _Set(self, t): - assert(t.elts) # should be at least one element - self.write("{") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("}") - - def _Dict(self, t): - self.write("{") - def write_pair(pair): - (k, v) = pair - self.dispatch(k) - self.write(": ") - self.dispatch(v) - interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) - self.write("}") - - def _Tuple(self, t): - self.write("(") - if len(t.elts) == 1: - (elt,) = t.elts - self.dispatch(elt) - self.write(",") - else: - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write(")") - - unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} - def _UnaryOp(self, t): - self.write("(") - self.write(self.unop[t.op.__class__.__name__]) - self.write(" ") - # If we're applying unary minus to a number, parenthesize the number. - # This is necessary: -2147483648 is different from -(2147483648) on - # a 32-bit machine (the first is an int, the second a long), and - # -7j is different from -(7j). (The first has real part 0.0, the second - # has real part -0.0.) - if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num): - self.write("(") - self.dispatch(t.operand) - self.write(")") - else: - self.dispatch(t.operand) - self.write(")") - - binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", - "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", - "FloorDiv":"//", "Pow": "**"} - def _BinOp(self, t): - self.write("(") - self.dispatch(t.left) - self.write(" " + self.binop[t.op.__class__.__name__] + " ") - self.dispatch(t.right) - self.write(")") - - cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", - "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} - def _Compare(self, t): - self.write("(") - self.dispatch(t.left) - for o, e in zip(t.ops, t.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.dispatch(e) - self.write(")") - - boolops = {ast.And: 'and', ast.Or: 'or'} - def _BoolOp(self, t): - self.write("(") - s = " %s " % self.boolops[t.op.__class__] - interleave(lambda: self.write(s), self.dispatch, t.values) - self.write(")") - - def _Attribute(self,t): - self.dispatch(t.value) - # Special case: 3.__abs__() is a syntax error, so if t.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): - self.write(" ") - self.write(".") - self.write(t.attr) - - def _Call(self, t): - self.dispatch(t.func) - self.write("(") - comma = False - for e in t.args: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - if t.starargs: - if comma: self.write(", ") - else: comma = True - self.write("*") - self.dispatch(t.starargs) - if t.kwargs: - if comma: self.write(", ") - else: comma = True - self.write("**") - self.dispatch(t.kwargs) - self.write(")") - - def _Subscript(self, t): - self.dispatch(t.value) - self.write("[") - self.dispatch(t.slice) - self.write("]") - - # slice - def _Ellipsis(self, t): - self.write("...") - - def _Index(self, t): - self.dispatch(t.value) - - def _Slice(self, t): - if t.lower: - self.dispatch(t.lower) - self.write(":") - if t.upper: - self.dispatch(t.upper) - if t.step: - self.write(":") - self.dispatch(t.step) - - def _ExtSlice(self, t): - interleave(lambda: self.write(', '), self.dispatch, t.dims) - - # others - def _arguments(self, t): - first = True - # normal arguments - defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults - for a,d in zip(t.args, defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a), - if d: - self.write("=") - self.dispatch(d) - - # varargs - if t.vararg: - if first:first = False - else: self.write(", ") - self.write("*") - self.write(t.vararg) - - # kwargs - if t.kwarg: - if first:first = False - else: self.write(", ") - self.write("**"+t.kwarg) - - def _keyword(self, t): - self.write(t.arg) - self.write("=") - self.dispatch(t.value) - - def _Lambda(self, t): - self.write("(") - self.write("lambda ") - self.dispatch(t.args) - self.write(": ") - self.dispatch(t.body) - self.write(")") - - def _alias(self, t): - self.write(t.name) - if t.asname: - self.write(" as "+t.asname) - -def roundtrip(filename, output=sys.stdout): - with open(filename, "r") as pyfile: - source = pyfile.read() - tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) - Unparser(tree, output) - - - -def testdir(a): - try: - names = [n for n in os.listdir(a) if n.endswith('.py')] - except OSError: - sys.stderr.write("Directory not readable: %s" % a) - else: - for n in names: - fullname = os.path.join(a, n) - if os.path.isfile(fullname): - output = cStringIO.StringIO() - print 'Testing %s' % fullname - try: - roundtrip(fullname, output) - except Exception as e: - print ' Failed to compile, exception is %s' % repr(e) - elif os.path.isdir(fullname): - testdir(fullname) - -def main(args): - if args[0] == '--testdir': - for a in args[1:]: - testdir(a) - else: - for a in args: - roundtrip(a) - -if __name__=='__main__': - main(sys.argv[1:]) diff --git a/pinfer/unparse3.py b/pinfer/unparse3.py deleted file mode 100644 index 0936cb2995cf..000000000000 --- a/pinfer/unparse3.py +++ /dev/null @@ -1,610 +0,0 @@ -# From Python 3's Tools/parser/unparse.py -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved - -"Usage: unparse.py " -import sys -import ast -import tokenize -import io -import os - -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -def interleave(inter, f, seq): - """Call f on each item in seq, calling inter() in between. - """ - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - -class Unparser: - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded. """ - - def __init__(self, tree, file = sys.stdout): - """Unparser(tree, file=sys.stdout) -> None. - Print the source for tree to file.""" - self.f = file - self._indent = 0 - self.dispatch(tree) - print("", file=self.f) - self.f.flush() - - def fill(self, text = ""): - "Indent a piece of text, according to the current indentation level" - self.f.write("\n"+" "*self._indent + text) - - def write(self, text): - "Append a piece of text to the current line." - self.f.write(text) - - def enter(self): - "Print ':', and increase the indentation." - self.write(":") - self._indent += 1 - - def leave(self): - "Decrease the indentation level." - self._indent -= 1 - - def dispatch(self, tree): - "Dispatcher function, dispatching tree type T to method _T." - if isinstance(tree, list): - for t in tree: - self.dispatch(t) - return - meth = getattr(self, "_"+tree.__class__.__name__) - meth(tree) - - - ############### Unparsing methods ###################### - # There should be one method per concrete grammar type # - # Constructors should be grouped by sum type. Ideally, # - # this would follow the order in the grammar, but # - # currently doesn't. # - ######################################################## - - def _Module(self, tree): - for stmt in tree.body: - self.dispatch(stmt) - - # stmt - def _Expr(self, tree): - self.fill() - self.dispatch(tree.value) - - def _Import(self, t): - self.fill("import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _ImportFrom(self, t): - self.fill("from ") - self.write("." * t.level) - if t.module: - self.write(t.module) - self.write(" import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _Assign(self, t): - self.fill() - for target in t.targets: - self.dispatch(target) - self.write(" = ") - self.dispatch(t.value) - - def _AugAssign(self, t): - self.fill() - self.dispatch(t.target) - self.write(" "+self.binop[t.op.__class__.__name__]+"= ") - self.dispatch(t.value) - - def _Return(self, t): - self.fill("return") - if t.value: - self.write(" ") - self.dispatch(t.value) - - def _Pass(self, t): - self.fill("pass") - - def _Break(self, t): - self.fill("break") - - def _Continue(self, t): - self.fill("continue") - - def _Delete(self, t): - self.fill("del ") - interleave(lambda: self.write(", "), self.dispatch, t.targets) - - def _Assert(self, t): - self.fill("assert ") - self.dispatch(t.test) - if t.msg: - self.write(", ") - self.dispatch(t.msg) - - def _Global(self, t): - self.fill("global ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Nonlocal(self, t): - self.fill("nonlocal ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Yield(self, t): - self.write("(") - self.write("yield") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _YieldFrom(self, t): - self.write("(") - self.write("yield from") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Raise(self, t): - self.fill("raise") - if not t.exc: - assert not t.cause - return - self.write(" ") - self.dispatch(t.exc) - if t.cause: - self.write(" from ") - self.dispatch(t.cause) - - def _Try(self, t): - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - for ex in t.handlers: - self.dispatch(ex) - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - if t.finalbody: - self.fill("finally") - self.enter() - self.dispatch(t.finalbody) - self.leave() - - def _ExceptHandler(self, t): - self.fill("except") - if t.type: - self.write(" ") - self.dispatch(t.type) - if t.name: - self.write(" as ") - self.write(t.name) - self.enter() - self.dispatch(t.body) - self.leave() - - def _ClassDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("class "+t.name) - self.write("(") - comma = False - for e in t.bases: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - if t.starargs: - if comma: self.write(", ") - else: comma = True - self.write("*") - self.dispatch(t.starargs) - if t.kwargs: - if comma: self.write(", ") - else: comma = True - self.write("**") - self.dispatch(t.kwargs) - self.write(")") - - self.enter() - self.dispatch(t.body) - self.leave() - - def _FunctionDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("def "+t.name + "(") - self.dispatch(t.args) - self.write(")") - if t.returns: - self.write(" -> ") - self.dispatch(t.returns) - self.enter() - self.dispatch(t.body) - self.leave() - - def _For(self, t): - self.fill("for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _If(self, t): - self.fill("if ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # collapse nested ifs into equivalent elifs. - while (t.orelse and len(t.orelse) == 1 and - isinstance(t.orelse[0], ast.If)): - t = t.orelse[0] - self.fill("elif ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # final else - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _While(self, t): - self.fill("while ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _With(self, t): - self.fill("with ") - interleave(lambda: self.write(", "), self.dispatch, t.items) - self.enter() - self.dispatch(t.body) - self.leave() - - # expr - def _Bytes(self, t): - self.write(repr(t.s)) - - def _Str(self, tree): - self.write(repr(tree.s)) - - def _Name(self, t): - self.write(t.id) - - def _NameConstant(self, t): - self.write(repr(t.value)) - - def _Num(self, t): - # Substitute overflowing decimal literal for AST infinities. - self.write(repr(t.n).replace("inf", INFSTR)) - - def _List(self, t): - self.write("[") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("]") - - def _ListComp(self, t): - self.write("[") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("]") - - def _GeneratorExp(self, t): - self.write("(") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write(")") - - def _SetComp(self, t): - self.write("{") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _DictComp(self, t): - self.write("{") - self.dispatch(t.key) - self.write(": ") - self.dispatch(t.value) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _comprehension(self, t): - self.write(" for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - for if_clause in t.ifs: - self.write(" if ") - self.dispatch(if_clause) - - def _IfExp(self, t): - self.write("(") - self.dispatch(t.body) - self.write(" if ") - self.dispatch(t.test) - self.write(" else ") - self.dispatch(t.orelse) - self.write(")") - - def _Set(self, t): - assert(t.elts) # should be at least one element - self.write("{") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("}") - - def _Dict(self, t): - self.write("{") - def write_pair(pair): - (k, v) = pair - self.dispatch(k) - self.write(": ") - self.dispatch(v) - interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) - self.write("}") - - def _Tuple(self, t): - self.write("(") - if len(t.elts) == 1: - (elt,) = t.elts - self.dispatch(elt) - self.write(",") - else: - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write(")") - - unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} - def _UnaryOp(self, t): - self.write("(") - self.write(self.unop[t.op.__class__.__name__]) - self.write(" ") - self.dispatch(t.operand) - self.write(")") - - binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", - "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", - "FloorDiv":"//", "Pow": "**"} - def _BinOp(self, t): - self.write("(") - self.dispatch(t.left) - self.write(" " + self.binop[t.op.__class__.__name__] + " ") - self.dispatch(t.right) - self.write(")") - - cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", - "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} - def _Compare(self, t): - self.write("(") - self.dispatch(t.left) - for o, e in zip(t.ops, t.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.dispatch(e) - self.write(")") - - boolops = {ast.And: 'and', ast.Or: 'or'} - def _BoolOp(self, t): - self.write("(") - s = " %s " % self.boolops[t.op.__class__] - interleave(lambda: self.write(s), self.dispatch, t.values) - self.write(")") - - def _Attribute(self,t): - self.dispatch(t.value) - # Special case: 3.__abs__() is a syntax error, so if t.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): - self.write(" ") - self.write(".") - self.write(t.attr) - - def _Call(self, t): - self.dispatch(t.func) - self.write("(") - comma = False - for e in t.args: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - if t.starargs: - if comma: self.write(", ") - else: comma = True - self.write("*") - self.dispatch(t.starargs) - if t.kwargs: - if comma: self.write(", ") - else: comma = True - self.write("**") - self.dispatch(t.kwargs) - self.write(")") - - def _Subscript(self, t): - self.dispatch(t.value) - self.write("[") - self.dispatch(t.slice) - self.write("]") - - def _Starred(self, t): - self.write("*") - self.dispatch(t.value) - - # slice - def _Ellipsis(self, t): - self.write("...") - - def _Index(self, t): - self.dispatch(t.value) - - def _Slice(self, t): - if t.lower: - self.dispatch(t.lower) - self.write(":") - if t.upper: - self.dispatch(t.upper) - if t.step: - self.write(":") - self.dispatch(t.step) - - def _ExtSlice(self, t): - interleave(lambda: self.write(', '), self.dispatch, t.dims) - - # argument - def _arg(self, t): - self.write(t.arg) - if t.annotation: - self.write(": ") - self.dispatch(t.annotation) - - # others - def _arguments(self, t): - first = True - # normal arguments - defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults - for a, d in zip(t.args, defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a) - if d: - self.write("=") - self.dispatch(d) - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if t.vararg or t.kwonlyargs: - if first:first = False - else: self.write(", ") - self.write("*") - if t.vararg: - self.write(t.vararg.arg) - if t.vararg.annotation: - self.write(": ") - self.dispatch(t.vararg.annotation) - - # keyword-only arguments - if t.kwonlyargs: - for a, d in zip(t.kwonlyargs, t.kw_defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a), - if d: - self.write("=") - self.dispatch(d) - - # kwargs - if t.kwarg: - if first:first = False - else: self.write(", ") - self.write("**"+t.kwarg.arg) - if t.kwarg.annotation: - self.write(": ") - self.dispatch(t.kwarg.annotation) - - def _keyword(self, t): - self.write(t.arg) - self.write("=") - self.dispatch(t.value) - - def _Lambda(self, t): - self.write("(") - self.write("lambda ") - self.dispatch(t.args) - self.write(": ") - self.dispatch(t.body) - self.write(")") - - def _alias(self, t): - self.write(t.name) - if t.asname: - self.write(" as "+t.asname) - - def _withitem(self, t): - self.dispatch(t.context_expr) - if t.optional_vars: - self.write(" as ") - self.dispatch(t.optional_vars) - -def roundtrip(filename, output=sys.stdout): - with open(filename, "rb") as pyfile: - encoding = tokenize.detect_encoding(pyfile.readline)[0] - with open(filename, "r", encoding=encoding) as pyfile: - source = pyfile.read() - tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) - Unparser(tree, output) - - - -def testdir(a): - try: - names = [n for n in os.listdir(a) if n.endswith('.py')] - except OSError: - print("Directory not readable: %s" % a, file=sys.stderr) - else: - for n in names: - fullname = os.path.join(a, n) - if os.path.isfile(fullname): - output = io.StringIO() - print('Testing %s' % fullname) - try: - roundtrip(fullname, output) - except Exception as e: - print(' Failed to compile, exception is %s' % repr(e)) - elif os.path.isdir(fullname): - testdir(fullname) - -def main(args): - if args[0] == '--testdir': - for a in args[1:]: - testdir(a) - else: - for a in args: - roundtrip(a) - -if __name__=='__main__': - main(sys.argv[1:]) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000000..1a7adf21c0a6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,218 @@ +[build-system] +requires = [ + # NOTE: this needs to be kept in sync with mypy-requirements.txt + # and build-requirements.txt, because those are both needed for + # self-typechecking :/ + "setuptools >= 75.1.0", + # the following is from mypy-requirements.txt/setup.py + "typing_extensions>=4.6.0", + "mypy_extensions>=1.0.0", + "tomli>=1.1.0; python_version<'3.11'", + # the following is from build-requirements.txt + "types-psutil", + "types-setuptools", +] +build-backend = "setuptools.build_meta" + +[project] +name = "mypy" +description = "Optional static typing for Python" +readme = {text = """ +Mypy -- Optional Static Typing for Python +========================================= + +Add type annotations to your Python programs, and use mypy to type +check them. Mypy is essentially a Python linter on steroids, and it +can catch many programming errors by analyzing your program, without +actually having to run it. Mypy has a powerful type system with +features such as type inference, gradual typing, generics and union +types. +""", content-type = "text/x-rst"} +authors = [{name = "Jukka Lehtosalo", email = "jukka.lehtosalo@iki.fi"}] +license = {text = "MIT"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development", + "Typing :: Typed", +] +requires-python = ">=3.8" +dependencies = [ + # When changing this, also update build-system.requires and mypy-requirements.txt + "typing_extensions>=4.6.0", + "mypy_extensions>=1.0.0", + "tomli>=1.1.0; python_version<'3.11'", +] +dynamic = ["version"] + +[project.optional-dependencies] +dmypy = ["psutil>=4.0"] +mypyc = ["setuptools>=50"] +python2 = [] +reports = ["lxml"] +install-types = ["pip"] +faster-cache = ["orjson"] + +[project.urls] +Homepage = "https://www.mypy-lang.org/" +Documentation = "https://mypy.readthedocs.io/en/stable/index.html" +Repository = "https://github.com/python/mypy" +Changelog = "https://github.com/python/mypy/blob/master/CHANGELOG.md" +Issues = "https://github.com/python/mypy/issues" + +[project.scripts] +mypy = "mypy.__main__:console_entry" +stubgen = "mypy.stubgen:main" +stubtest = "mypy.stubtest:main" +dmypy = "mypy.dmypy.client:console_entry" +mypyc = "mypyc.__main__:main" + +[tool.setuptools.packages.find] +include = ["mypy*", "mypyc*", "*__mypyc*"] +namespaces = false + +[tool.setuptools.package-data] +mypy = [ + "py.typed", + "typeshed/**/*.py", + "typeshed/**/*.pyi", + "typeshed/stdlib/VERSIONS", + "xml/*.xsd", + "xml/*.xslt", + "xml/*.css", +] + +[tool.black] +line-length = 99 +target-version = ["py38", "py39", "py310", "py311", "py312"] +skip-magic-trailing-comma = true +force-exclude = ''' +^/mypy/typeshed| +^/mypyc/test-data| +^/test-data +''' + +[tool.ruff] +line-length = 99 +target-version = "py38" +fix = true + +extend-exclude = [ + "@*", + # Sphinx configuration is irrelevant + "docs/source/conf.py", + "mypyc/doc/conf.py", + # tests have more relaxed styling requirements + # fixtures have their own .pyi-specific configuration + "test-data/*", + "mypyc/test-data/*", + # typeshed has its own .pyi-specific configuration + "mypy/typeshed/*", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle (error) + "F", # pyflakes + "W", # pycodestyle (warning) + "B", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "RUF100", # Unused noqa comments + "PGH004", # blanket noqa comments + "UP", # pyupgrade + "C4", # flake8-comprehensions + "SIM201", "SIM202", # simplify comparisons involving not + "ISC001", # implicitly concatenated string + "RET501", "RET502", # better return None handling +] + +ignore = [ + "B007", # Loop control variable not used within the loop body. + "B011", # Don't use assert False + "B023", # Function definition does not bind loop variable + "E2", # conflicts with black + "E402", # module level import not at top of file + "E501", # conflicts with black + "E721", # Use `is` and `is not` for type comparisons, or `isinstance()` for isinstance checks + "E731", # Do not assign a `lambda` expression, use a `def` + "E741", # Ambiguous variable name + "N818", # Exception should be named with an Error suffix + "N806", # UPPER_CASE used for constant local variables + "UP031", # Use format specifiers instead of percent format + "UP032", # 'f-string always preferable to format' is controversial + "C416", # There are a few cases where it's nice to have names for the dict items +] + +unfixable = [ + "F841", # unused variable. ruff keeps the call, but mostly we want to get rid of it all + "F601", # automatic fix might obscure issue + "F602", # automatic fix might obscure issue + "B018", # automatic fix might obscure issue + "UP036", # sometimes it's better to just noqa this +] + +[tool.ruff.lint.per-file-ignores] +# Mixed case variable and function names. +"mypy/fastparse.py" = ["N802", "N816"] + +[tool.ruff.lint.isort] +combine-as-imports = true +extra-standard-library = ["typing_extensions"] + +[tool.check-manifest] +ignore = ["**/.readthedocs.yaml"] + +[tool.pytest.ini_options] +minversion = "7.0.0" +testpaths = ["mypy/test", "mypyc/test"] +python_files = 'test*.py' + +# Where do the test cases come from? We provide our own collection +# logic by implementing `pytest_pycollect_makeitem` in mypy.test.data; +# the test files import that module, and pytest sees the magic name +# and invokes it at the relevant moment. See +# https://doc.pytest.org/en/latest/how-to/writing_plugins.html#collection-hooks + +# Both our plugin and unittest provide their own collection logic, +# So we can disable the default python collector by giving it empty +# patterns to search for. +# Note that unittest requires that no "Test*" classes exist. +python_classes = [] +python_functions = [] + +# always run in parallel (requires pytest-xdist, see test-requirements.txt) +# and enable strict mode: require all markers +# to be defined and raise on invalid config values +addopts = "-nauto --strict-markers --strict-config" + +# treat xpasses as test failures so they get converted to regular tests as soon as possible +xfail_strict = true + +[tool.coverage.run] +branch = true +source = ["mypy"] +parallel = true + +[tool.coverage.report] +show_missing = true +skip_covered = true +omit = ['mypy/test/*'] +exclude_lines = [ + '\#\s*pragma: no cover', + '^\s*raise AssertionError\b', + '^\s*raise NotImplementedError\b', + '^\s*return NotImplemented\b', + '^\s*raise$', + '^assert False\b', + '''^if __name__ == ['"]__main__['"]:$''', +] diff --git a/runtests.py b/runtests.py new file mode 100755 index 000000000000..9863e8491500 --- /dev/null +++ b/runtests.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import subprocess +from subprocess import Popen +from sys import argv, executable, exit + +# Slow test suites +CMDLINE = "PythonCmdline" +PEP561 = "PEP561Suite" +EVALUATION = "PythonEvaluation" +DAEMON = "testdaemon" +STUBGEN_CMD = "StubgenCmdLine" +STUBGEN_PY = "StubgenPythonSuite" +MYPYC_RUN = "TestRun" +MYPYC_RUN_MULTI = "TestRunMultiFile" +MYPYC_EXTERNAL = "TestExternal" +MYPYC_COMMAND_LINE = "TestCommandLine" +MYPYC_SEPARATE = "TestRunSeparate" +MYPYC_MULTIMODULE = "multimodule" # Subset of mypyc run tests that are slow +ERROR_STREAM = "ErrorStreamSuite" + + +ALL_NON_FAST = [ + CMDLINE, + PEP561, + EVALUATION, + DAEMON, + STUBGEN_CMD, + STUBGEN_PY, + MYPYC_RUN, + MYPYC_RUN_MULTI, + MYPYC_EXTERNAL, + MYPYC_COMMAND_LINE, + MYPYC_SEPARATE, + ERROR_STREAM, +] + + +# This must be enabled by explicitly including 'pytest-extra' on the command line +PYTEST_OPT_IN = [PEP561] + + +# These must be enabled by explicitly including 'mypyc-extra' on the command line. +MYPYC_OPT_IN = [MYPYC_RUN, MYPYC_RUN_MULTI, MYPYC_SEPARATE] + +# These mypyc test filters cover most slow test cases +MYPYC_SLOW = [MYPYC_RUN_MULTI, MYPYC_COMMAND_LINE, MYPYC_SEPARATE, MYPYC_MULTIMODULE] + + +# We split the pytest run into three parts to improve test +# parallelization. Each run should have tests that each take a roughly similar +# time to run. +cmds = { + # Self type check + "self": [ + executable, + "-m", + "mypy", + "--config-file", + "mypy_self_check.ini", + "-p", + "mypy", + "-p", + "mypyc", + ], + # Lint + "lint": ["pre-commit", "run", "--all-files"], + # Fast test cases only (this is the bulk of the test suite) + "pytest-fast": ["pytest", "-q", "-k", f"not ({' or '.join(ALL_NON_FAST)})"], + # Test cases that invoke mypy (with small inputs) + "pytest-cmdline": [ + "pytest", + "-q", + "-k", + " or ".join([CMDLINE, EVALUATION, STUBGEN_CMD, STUBGEN_PY]), + ], + # Test cases that may take seconds to run each + "pytest-slow": [ + "pytest", + "-q", + "-k", + " or ".join([DAEMON, MYPYC_EXTERNAL, MYPYC_COMMAND_LINE, ERROR_STREAM]), + ], + "mypyc-fast": ["pytest", "-q", "mypyc", "-k", f"not ({' or '.join(MYPYC_SLOW)})"], + # Test cases that might take minutes to run + "pytest-extra": ["pytest", "-q", "-k", " or ".join(PYTEST_OPT_IN)], + # Mypyc tests that aren't run by default, since they are slow and rarely + # fail for commits that don't touch mypyc + "mypyc-extra": ["pytest", "-q", "-k", " or ".join(MYPYC_OPT_IN)], +} + +# Stop run immediately if these commands fail +FAST_FAIL = ["self", "lint"] + +EXTRA_COMMANDS = ("pytest-extra", "mypyc-fast", "mypyc-extra") +DEFAULT_COMMANDS = [cmd for cmd in cmds if cmd not in EXTRA_COMMANDS] + +assert all(cmd in cmds for cmd in FAST_FAIL) + + +def run_cmd(name: str) -> int: + status = 0 + cmd = cmds[name] + print(f"run {name}: {cmd}") + proc = subprocess.run(cmd, stderr=subprocess.STDOUT) + if proc.returncode: + print("\nFAILED: %s" % name) + status = proc.returncode + if name in FAST_FAIL: + exit(status) + return status + + +def start_background_cmd(name: str) -> Popen: + cmd = cmds[name] + proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return proc + + +def wait_background_cmd(name: str, proc: Popen) -> int: + output = proc.communicate()[0] + status = proc.returncode + print(f"run {name}: {cmds[name]}") + if status: + print(output.decode().rstrip()) + print("\nFAILED:", name) + if name in FAST_FAIL: + exit(status) + return status + + +def main() -> None: + prog, *args = argv + + if not set(args).issubset(cmds): + print("usage:", prog, " ".join(f"[{k}]" for k in cmds)) + print() + print( + "Run the given tests. If given no arguments, run everything except" + + " pytest-extra and mypyc-extra." + ) + exit(1) + + if not args: + args = DEFAULT_COMMANDS.copy() + + status = 0 + + if "self" in args and "lint" in args: + # Perform lint and self check in parallel as it's faster. + proc = start_background_cmd("lint") + cmd_status = run_cmd("self") + if cmd_status: + status = cmd_status + cmd_status = wait_background_cmd("lint", proc) + if cmd_status: + status = cmd_status + args = [arg for arg in args if arg not in ("self", "lint")] + + for arg in args: + cmd_status = run_cmd(arg) + if cmd_status: + status = cmd_status + + exit(status) + + +if __name__ == "__main__": + main() diff --git a/samples/bottles.py b/samples/bottles.py deleted file mode 100644 index ddf77f59eaa0..000000000000 --- a/samples/bottles.py +++ /dev/null @@ -1,13 +0,0 @@ -import typing - -REFRAIN = ''' -%d bottles of beer on the wall, -%d bottles of beer, -take one down, pass it around, -%d bottles of beer on the wall! -''' -bottles_of_beer = 99 -while bottles_of_beer > 1: - print(REFRAIN % (bottles_of_beer, bottles_of_beer, - bottles_of_beer - 1)) - bottles_of_beer -= 1 diff --git a/samples/class.py b/samples/class.py deleted file mode 100644 index d2eb4ac0516f..000000000000 --- a/samples/class.py +++ /dev/null @@ -1,18 +0,0 @@ -import typing - - -class BankAccount(object): - def __init__(self, initial_balance: int = 0) -> None: - self.balance = initial_balance - - def deposit(self, amount: int) -> None: - self.balance += amount - - def withdraw(self, amount: int) -> None: - self.balance -= amount - - def overdrawn(self) -> bool: - return self.balance < 0 -my_account = BankAccount(15) -my_account.withdraw(5) -print(my_account.balance) diff --git a/samples/cmdline.py b/samples/cmdline.py deleted file mode 100644 index 105c27a305b9..000000000000 --- a/samples/cmdline.py +++ /dev/null @@ -1,8 +0,0 @@ -# This program adds up integers in the command line -import sys -import typing -try: - total = sum(int(arg) for arg in sys.argv[1:]) - print('sum =', total) -except ValueError: - print('Please supply integer arguments') diff --git a/samples/codec/example.py b/samples/codec/example.py deleted file mode 100644 index 2a0591e86e4e..000000000000 --- a/samples/codec/example.py +++ /dev/null @@ -1,13 +0,0 @@ -# coding: mypy - -# very simple script to test type annotations - - -def f(x: int, y: str='abc') -> [int, - str]: - return x, y - -print(f(123)) # abc123 - -x = 1 + \ - 2 diff --git a/samples/dict.py b/samples/dict.py deleted file mode 100644 index d74a5b5ea01a..000000000000 --- a/samples/dict.py +++ /dev/null @@ -1,8 +0,0 @@ -import typing -prices = {'apple': 0.40, 'banana': 0.50} -my_purchase = { - 'apple': 1, - 'banana': 6} -grocery_bill = sum(prices[fruit] * my_purchase[fruit] - for fruit in my_purchase) -print('I owe the grocer $%.2f' % grocery_bill) diff --git a/samples/fib.py b/samples/fib.py deleted file mode 100644 index 26248c866b1f..000000000000 --- a/samples/fib.py +++ /dev/null @@ -1,5 +0,0 @@ -import typing -parents, babies = (1, 1) -while babies < 100: - print('This generation has {0} babies'.format(babies)) - parents, babies = (babies, parents + babies) diff --git a/samples/files.py b/samples/files.py deleted file mode 100644 index f540c7c2b665..000000000000 --- a/samples/files.py +++ /dev/null @@ -1,14 +0,0 @@ -# indent your Python code to put into an email -import glob -import typing -# glob supports Unix style pathname extensions -python_files = glob.glob('*.py') -for file_name in sorted(python_files): - print(' ------' + file_name) - - f = open(file_name) - for line in f: - print(' ' + line.rstrip()) - f.close() - - print() diff --git a/samples/for.py b/samples/for.py deleted file mode 100644 index f7eeed4efbe6..000000000000 --- a/samples/for.py +++ /dev/null @@ -1,4 +0,0 @@ -import typing -friends = ['john', 'pat', 'gary', 'michael'] -for i, name in enumerate(friends): - print("iteration {iteration} is {name}".format(iteration=i, name=name)) diff --git a/samples/generators.py b/samples/generators.py deleted file mode 100644 index 9150c96c8276..000000000000 --- a/samples/generators.py +++ /dev/null @@ -1,24 +0,0 @@ -# Prime number sieve with generators - -import itertools -from typing import Iterator - - -def iter_primes() -> Iterator[int]: - # an iterator of all numbers between 2 and +infinity - numbers = itertools.count(2) - - # generate primes forever - while True: - # get the first number from the iterator (always a prime) - prime = next(numbers) - yield prime - - # this code iteratively builds up a chain of - # filters...slightly tricky, but ponder it a bit - numbers = filter(prime.__rmod__, numbers) - -for p in iter_primes(): - if p > 1000: - break - print(p) diff --git a/samples/greet.py b/samples/greet.py deleted file mode 100644 index 47e7626410c3..000000000000 --- a/samples/greet.py +++ /dev/null @@ -1,8 +0,0 @@ -import typing - - -def greet(name: str) -> None: - print('Hello', name) -greet('Jack') -greet('Jill') -greet('Bob') diff --git a/samples/guess.py b/samples/guess.py deleted file mode 100644 index d3f1cee4edc7..000000000000 --- a/samples/guess.py +++ /dev/null @@ -1,32 +0,0 @@ -# "Guess the Number" Game (edited) from http://inventwithpython.com - -import random -import typing - -guesses_made = 0 - -name = input('Hello! What is your name?\n') - -number = random.randint(1, 20) -print('Well, {0}, I am thinking of a number between 1 and 20.'.format(name)) - -while guesses_made < 6: - - guess = int(input('Take a guess: ')) - - guesses_made += 1 - - if guess < number: - print('Your guess is too low.') - - if guess > number: - print('Your guess is too high.') - - if guess == number: - break - -if guess == number: - print('Good job, {0}! You guessed my number in {1} guesses!'.format( - name, guesses_made)) -else: - print('Nope. The number I was thinking of was {0}'.format(number)) diff --git a/samples/hello.py b/samples/hello.py deleted file mode 100644 index 6c0b2caa7a60..000000000000 --- a/samples/hello.py +++ /dev/null @@ -1,2 +0,0 @@ -import typing -print('Hello, world') diff --git a/samples/input.py b/samples/input.py deleted file mode 100644 index cca92336f06b..000000000000 --- a/samples/input.py +++ /dev/null @@ -1,3 +0,0 @@ -import typing -name = input('What is your name?\n') -print('Hi, %s.' % name) diff --git a/samples/itertool.py b/samples/itertool.py deleted file mode 100644 index 9ee2475e01fb..000000000000 --- a/samples/itertool.py +++ /dev/null @@ -1,16 +0,0 @@ -from itertools import groupby -import typing -lines = ''' -This is the -first paragraph. - -This is the second. -'''.splitlines() -# Use itertools.groupby and bool to return groups of -# consecutive lines that either have content or don't. -for has_chars, frags in groupby(lines, bool): - if has_chars: - print(' '.join(frags)) -# PRINTS: -# This is the first paragraph. -# This is the second. diff --git a/samples/readme.txt b/samples/readme.txt deleted file mode 100644 index 5889a8ee00ca..000000000000 --- a/samples/readme.txt +++ /dev/null @@ -1,25 +0,0 @@ -Mypy Sample Programs --------------------- - -The sample programs use static typing unless otherwise noted in comments. - -Original credits for sample programs: - - fib.py - Python Wiki [1] - for.py - Python Wiki [1] - greet.py - Python Wiki [1] - hello.py - Python Wiki [1] - input.py - Python Wiki [1] - regexp.py - Python Wiki [1] - dict.py - Python Wiki [1] - cmdline.py - Python Wiki [1] - files.py - Python Wiki [1] - bottles.py - Python Wiki [1] - class.py - Python Wiki [1] - guess.py - Python Wiki [1] - generators.py - Python Wiki [1] - itertool.py - Python Wiki [1] - -The sample programs were ported to mypy by Jukka Lehtosalo. - -[1] http://wiki.python.org/moin/SimplePrograms diff --git a/samples/regexp.py b/samples/regexp.py deleted file mode 100644 index 6d8d7992d0ae..000000000000 --- a/samples/regexp.py +++ /dev/null @@ -1,7 +0,0 @@ -import typing -import re -for test_string in ['555-1212', 'ILL-EGAL']: - if re.match(r'^\d{3}-\d{4}$', test_string): - print(test_string, 'is a valid US local phone number') - else: - print(test_string, 'rejected') diff --git a/scripts/mypy b/scripts/mypy deleted file mode 100644 index 0c91bcca4fa7..000000000000 --- a/scripts/mypy +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python -"""Mypy type checker command line tool.""" - -import os -import os.path -import shutil -import subprocess -import sys -import tempfile - -import typing -from typing import List, Tuple - -from mypy import build -from mypy.errors import CompileError - - -class Options: - def __init__(self) -> None: - # Set default options. - self.target = build.TYPE_CHECK - self.build_flags = List[str]() - self.pyversion = 3 - self.custom_typing_module = None # type: str - self.html_report_dir = None # type: str - self.python_path = False - - -def main() -> None: - bin_dir = find_bin_directory() - path, module, options = process_options(sys.argv[1:]) - try: - if options.target == build.TYPE_CHECK: - type_check_only(path, module, bin_dir, options) - else: - raise RuntimeError('unsupported target %d' % options.target) - except CompileError as e: - for m in e.messages: - sys.stderr.write(m + '\n') - sys.exit(1) - - -def find_bin_directory() -> str: - """Find the directory that contains this script. - - This is used by build to find stubs and other data files. - """ - script = __file__ - # Follow up to 5 symbolic links (cap to avoid cycles). - for i in range(5): - if os.path.islink(script): - script = readlinkabs(script) - else: - break - return os.path.dirname(script) - - -def readlinkabs(link: str) -> str: - """Return an absolute path to symbolic link destination.""" - # Adapted from code by Greg Smith. - assert os.path.islink(link) - path = os.readlink(link) - if os.path.isabs(path): - return path - return os.path.join(os.path.dirname(link), path) - - -def type_check_only(path: str, module: str, bin_dir: str, options: Options) -> None: - # Type check the program and dependencies and translate to Python. - build.build(path, - module=module, - bin_dir=bin_dir, - target=build.TYPE_CHECK, - pyversion=options.pyversion, - custom_typing_module=options.custom_typing_module, - html_report_dir=options.html_report_dir, - flags=options.build_flags, - python_path=options.python_path) - - -def process_options(args: List[str]) -> Tuple[str, str, Options]: - """Process command line arguments. - - Return (mypy program path (or None), - module to run as script (or None), - parsed flags) - """ - options = Options() - help = False - while args and args[0].startswith('-'): - if args[0] == '--verbose': - options.build_flags.append(build.VERBOSE) - args = args[1:] - elif args[0] == '--py2': - # Use Python 2 mode. - options.pyversion = 2 - args = args[1:] - elif args[0] == '-m' and args[1:]: - options.build_flags.append(build.MODULE) - return None, args[1], options - elif args[0] in ('-h', '--help'): - help = True - args = args[1:] - elif args[0] == '--stats': - options.build_flags.append('dump-type-stats') - args = args[1:] - elif args[0] == '--inferstats': - options.build_flags.append('dump-infer-stats') - args = args[1:] - elif args[0] == '--custom-typing' and args[1:]: - options.custom_typing_module = args[1] - args = args[2:] - elif args[0] == '--html-report' and args[1:]: - options.html_report_dir = args[1] - options.build_flags.append('html-report') - args = args[2:] - elif args[0] == '--use-python-path': - options.python_path = True - args = args[1:] - else: - usage('Unknown option: {}'.format(args[0])) - - if help: - usage() - - if not args: - usage('Missing target file or module') - - if args[1:]: - usage('Extra argument: {}'.format(args[1])) - - if options.python_path and options.pyversion == 2: - usage('--py2 specified, ' - 'but --use-python-path will search in sys.path of Python 3') - - return args[0], None, options - - -def usage(msg: str = None) -> None: - if msg: - sys.stderr.write('%s\n' % msg) - sys.stderr.write( -"""usage: mypy [option ...] [-m mod | file] -Try 'mypy -h' for more information. -""") - else: - sys.stderr.write( -"""usage: mypy [option ...] [-m mod | file] - -Optional arguments: - -h, --help print this help message and exit - --html-report dir generate a HTML report of type precision under dir/ - -m mod type check module - --verbose more verbose messages - --use-python-path search for modules in sys.path of running Python - -Environment variables: - MYPYPATH additional module search path -""") - sys.exit(2) - - -def fail(msg: str) -> None: - sys.stderr.write('%s\n' % msg) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index f74cf3a93bbe..180faf6d8ded 100644 --- a/setup.py +++ b/setup.py @@ -1,67 +1,162 @@ #!/usr/bin/env python +from __future__ import annotations + import glob import os import os.path import sys +from typing import TYPE_CHECKING, Any -from distutils.core import setup - -if sys.version_info < (3, 2, 0): - sys.stderr.write("ERROR: You need Python 3.2 or later to use mypy.\n") +if sys.version_info < (3, 8, 0): # noqa: UP036 + sys.stderr.write("ERROR: You need Python 3.8 or later to use mypy.\n") exit(1) -version = '0.1.0' -description = 'Optional static type checker for Python' -long_description = ''' -Mypy -- Optional Static Type Checker for Python -=============================================== - -Mypy lets you add type annotations to Python programs, type check them -without running them, and run the programs using a standard Python 3 -interpreter. -'''.lstrip() - -stubs = [] - -for py_version in ['3.4', '3.3', '3.2', '2.7']: - base = os.path.join('stubs', py_version) - if not os.path.exists(base): - os.mkdir(base) - - stub_dirs = [''] + [name for name in os.listdir(base) - if os.path.isdir(os.path.join(base, name))] - for stub_dir in stub_dirs: - target = os.path.join('lib', 'mypy', 'stubs', py_version, stub_dir) - files = glob.glob(os.path.join(base, stub_dir, '*.py')) - files += glob.glob(os.path.join(base, stub_dir, '*.pyi')) - stubs.append((target, files)) - -classifiers = [ - 'Development Status :: 2 - Pre-Alpha', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: POSIX', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Software Development', -] - -setup(name='mypy-lang', - version=version, - description=description, - long_description=long_description, - author='Jukka Lehtosalo', - author_email='jukka.lehtosalo@iki.fi', - url='http://www.mypy-lang.org/', - license='MIT License', - platforms=['POSIX'], - package_dir={'': 'lib-typing/3.2', 'mypy': 'mypy'}, - py_modules=['typing'], - packages=['mypy'], - scripts=['scripts/mypy'], - data_files=stubs, - classifiers=classifiers, - ) +# we'll import stuff from the source tree, let's ensure is on the sys path +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) + +# This requires setuptools when building; setuptools is not needed +# when installing from a wheel file (though it is still needed for +# alternative forms of installing, as suggested by README.md). +from setuptools import Extension, setup +from setuptools.command.build_py import build_py + +from mypy.version import __version__ as version + +if TYPE_CHECKING: + from typing_extensions import TypeGuard + + +def is_list_of_setuptools_extension(items: list[Any]) -> TypeGuard[list[Extension]]: + return all(isinstance(item, Extension) for item in items) + + +def find_package_data(base, globs, root="mypy"): + """Find all interesting data files, for setup(package_data=) + + Arguments: + root: The directory to search in. + globs: A list of glob patterns to accept files. + """ + + rv_dirs = [root for root, dirs, files in os.walk(base)] + rv = [] + for rv_dir in rv_dirs: + files = [] + for pat in globs: + files += glob.glob(os.path.join(rv_dir, pat)) + if not files: + continue + rv.extend([os.path.relpath(f, root) for f in files]) + return rv + + +class CustomPythonBuild(build_py): + def pin_version(self): + path = os.path.join(self.build_lib, "mypy") + self.mkpath(path) + with open(os.path.join(path, "version.py"), "w") as stream: + stream.write(f'__version__ = "{version}"\n') + + def run(self): + self.execute(self.pin_version, ()) + build_py.run(self) + + +cmdclass = {"build_py": CustomPythonBuild} + +USE_MYPYC = False +# To compile with mypyc, a mypyc checkout must be present on the PYTHONPATH +if len(sys.argv) > 1 and "--use-mypyc" in sys.argv: + sys.argv.remove("--use-mypyc") + USE_MYPYC = True +if os.getenv("MYPY_USE_MYPYC", None) == "1": + USE_MYPYC = True + +if USE_MYPYC: + MYPYC_BLACKLIST = tuple( + os.path.join("mypy", x) + for x in ( + # Need to be runnable as scripts + "__main__.py", + "pyinfo.py", + os.path.join("dmypy", "__main__.py"), + # Uses __getattr__/__setattr__ + "split_namespace.py", + # Lies to mypy about code reachability + "bogus_type.py", + # We don't populate __file__ properly at the top level or something? + # Also I think there would be problems with how we generate version.py. + "version.py", + # Skip these to reduce the size of the build + "stubtest.py", + "stubgenc.py", + "stubdoc.py", + ) + ) + ( + # Don't want to grab this accidentally + os.path.join("mypyc", "lib-rt", "setup.py"), + # Uses __file__ at top level https://github.com/mypyc/mypyc/issues/700 + os.path.join("mypyc", "__main__.py"), + ) + + everything = [os.path.join("mypy", x) for x in find_package_data("mypy", ["*.py"])] + [ + os.path.join("mypyc", x) for x in find_package_data("mypyc", ["*.py"], root="mypyc") + ] + # Start with all the .py files + all_real_pys = [ + x for x in everything if not x.startswith(os.path.join("mypy", "typeshed") + os.sep) + ] + # Strip out anything in our blacklist + mypyc_targets = [x for x in all_real_pys if x not in MYPYC_BLACKLIST] + # Strip out any test code + mypyc_targets = [ + x + for x in mypyc_targets + if not x.startswith( + ( + os.path.join("mypy", "test") + os.sep, + os.path.join("mypyc", "test") + os.sep, + os.path.join("mypyc", "doc") + os.sep, + os.path.join("mypyc", "test-data") + os.sep, + ) + ) + ] + # ... and add back in the one test module we need + mypyc_targets.append(os.path.join("mypy", "test", "visitors.py")) + + # The targets come out of file system apis in an unspecified + # order. Sort them so that the mypyc output is deterministic. + mypyc_targets.sort() + + use_other_mypyc = os.getenv("ALTERNATE_MYPYC_PATH", None) + if use_other_mypyc: + # This bit is super unfortunate: we want to use a different + # mypy/mypyc version, but we've already imported parts, so we + # remove the modules that we've imported already, which will + # let the right versions be imported by mypyc. + del sys.modules["mypy"] + del sys.modules["mypy.version"] + del sys.modules["mypy.git"] + sys.path.insert(0, use_other_mypyc) + + from mypyc.build import mypycify + + opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") + debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") + force_multifile = os.getenv("MYPYC_MULTI_FILE", "") == "1" + ext_modules = mypycify( + mypyc_targets + ["--config-file=mypy_bootstrap.ini"], + opt_level=opt_level, + debug_level=debug_level, + # Use multi-file compilation mode on windows because without it + # our Appveyor builds run out of memory sometimes. + multi_file=sys.platform == "win32" or force_multifile, + ) + assert is_list_of_setuptools_extension(ext_modules), "Expected mypycify to use setuptools" + +else: + ext_modules = [] + + +setup(version=version, ext_modules=ext_modules, cmdclass=cmdclass) diff --git a/stubs/2.7/__future__.py b/stubs/2.7/__future__.py deleted file mode 100644 index 4c20971f6bf7..000000000000 --- a/stubs/2.7/__future__.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Undefined - -class _Feature: pass - -absolute_import = Undefined(_Feature) -division = Undefined(_Feature) -generators = Undefined(_Feature) -nested_scopes = Undefined(_Feature) -print_function = Undefined(_Feature) -unicode_literals = Undefined(_Feature) -with_statement = Undefined(_Feature) diff --git a/stubs/2.7/abc.py b/stubs/2.7/abc.py deleted file mode 100644 index b59a6fe3ef6a..000000000000 --- a/stubs/2.7/abc.py +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for abc. - -# Thesee definitions have special processing in type checker. -class ABCMeta: pass -abstractmethod = object() diff --git a/stubs/2.7/builtins.py b/stubs/2.7/builtins.py deleted file mode 100644 index e517e5c7f630..000000000000 --- a/stubs/2.7/builtins.py +++ /dev/null @@ -1,912 +0,0 @@ -# Stubs for builtins (Python 2.7) - -from typing import ( - Undefined, typevar, AbstractGeneric, Iterator, Iterable, overload, - Sequence, Mapping, Tuple, List, Any, Dict, Function, Generic, Set, - AbstractSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsAbs, - SupportsRound, IO, BinaryIO, builtinclass, ducktype, Union, AnyStr -) -from abc import abstractmethod, ABCMeta - -_T = typevar('_T') -_KT = typevar('_KT') -_VT = typevar('_VT') -_S = typevar('_S') -_T1 = typevar('_T1') -_T2 = typevar('_T2') -_T3 = typevar('_T3') -_T4 = typevar('_T4') - -staticmethod = object() # Only valid as a decorator. -classmethod = object() # Only valid as a decorator. -property = object() - -@builtinclass -class object: - __doc__ = '' - __class__ = Undefined # type: type - - def __init__(self) -> None: pass - def __eq__(self, o: object) -> bool: pass - def __ne__(self, o: object) -> bool: pass - def __str__(self) -> str: pass - def __repr__(self) -> str: pass - def __hash__(self) -> int: pass - -@builtinclass -class type: - __name__ = '' - __module__ = '' - __dict__ = Undefined # type: Dict[unicode, Any] - - def __init__(self, o: object) -> None: pass - -@builtinclass -@ducktype(float) -class int(SupportsInt, SupportsFloat, SupportsAbs[int]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, x: SupportsInt) -> None: pass - @overload - def __init__(self, x: unicode, base: int = 10) -> None: pass - @overload - def __init__(self, x: bytearray, base: int = 10) -> None: pass - def bit_length(self) -> int: pass - - def __add__(self, x: int) -> int: pass - def __sub__(self, x: int) -> int: pass - def __mul__(self, x: int) -> int: pass - def __floordiv__(self, x: int) -> int: pass - def __div__(self, x: int) -> int: pass - def __truediv__(self, x: int) -> float: pass - def __mod__(self, x: int) -> int: pass - def __radd__(self, x: int) -> int: pass - def __rsub__(self, x: int) -> int: pass - def __rmul__(self, x: int) -> int: pass - def __rfloordiv__(self, x: int) -> int: pass - def __rdiv__(self, x: int) -> int: pass - def __rtruediv__(self, x: int) -> float: pass - def __rmod__(self, x: int) -> int: pass - def __pow__(self, x: int) -> Any: pass # Return type can be int or float, depending on x. - def __rpow__(self, x: int) -> Any: pass - def __and__(self, n: int) -> int: pass - def __or__(self, n: int) -> int: pass - def __xor__(self, n: int) -> int: pass - def __lshift__(self, n: int) -> int: pass - def __rshift__(self, n: int) -> int: pass - def __rand__(self, n: int) -> int: pass - def __ror__(self, n: int) -> int: pass - def __rxor__(self, n: int) -> int: pass - def __rlshift__(self, n: int) -> int: pass - def __rrshift__(self, n: int) -> int: pass - def __neg__(self) -> int: pass - def __pos__(self) -> int: pass - def __invert__(self) -> int: pass - - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: int) -> bool: pass - def __le__(self, x: int) -> bool: pass - def __gt__(self, x: int) -> bool: pass - def __ge__(self, x: int) -> bool: pass - - def __str__(self) -> str: pass - def __float__(self) -> float: pass - def __int__(self) -> int: return self - def __abs__(self) -> int: pass - def __hash__(self) -> int: pass - -@builtinclass -class float(SupportsFloat, SupportsInt, SupportsAbs[float]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, x: SupportsFloat) -> None: pass - @overload - def __init__(self, x: unicode) -> None: pass - @overload - def __init__(self, x: bytearray) -> None: pass - - def __add__(self, x: float) -> float: pass - def __sub__(self, x: float) -> float: pass - def __mul__(self, x: float) -> float: pass - def __floordiv__(self, x: float) -> float: pass - def __div__(self, x: float) -> float: pass - def __truediv__(self, x: float) -> float: pass - def __mod__(self, x: float) -> float: pass - def __pow__(self, x: float) -> float: pass - def __radd__(self, x: float) -> float: pass - def __rsub__(self, x: float) -> float: pass - def __rmul__(self, x: float) -> float: pass - def __rfloordiv__(self, x: float) -> float: pass - def __rdiv__(self, x: float) -> float: pass - def __rtruediv__(self, x: float) -> float: pass - def __rmod__(self, x: float) -> float: pass - def __rpow__(self, x: float) -> float: pass - - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: float) -> bool: pass - def __le__(self, x: float) -> bool: pass - def __gt__(self, x: float) -> bool: pass - def __ge__(self, x: float) -> bool: pass - def __neg__(self) -> float: pass - def __pos__(self) -> float: pass - - def __str__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: return self - def __abs__(self) -> float: pass - def __hash__(self) -> int: pass - -@builtinclass -class complex(SupportsAbs[float]): - @overload - def __init__(self, re: float = 0.0, im: float = 0.0) -> None: pass - @overload - def __init__(self, s: str) -> None: pass - - @property - def real(self) -> float: pass - @property - def imag(self) -> float: pass - - def conjugate(self) -> complex: pass - - def __add__(self, x: complex) -> complex: pass - def __sub__(self, x: complex) -> complex: pass - def __mul__(self, x: complex) -> complex: pass - def __pow__(self, x: complex) -> complex: pass - def __truediv__(self, x: complex) -> complex: pass - def __radd__(self, x: complex) -> complex: pass - def __rsub__(self, x: complex) -> complex: pass - def __rmul__(self, x: complex) -> complex: pass - def __rpow__(self, x: complex) -> complex: pass - def __rtruediv__(self, x: complex) -> complex: pass - - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __neg__(self) -> complex: pass - def __pos__(self) -> complex: pass - - def __str__(self) -> str: pass - def __abs__(self) -> float: pass - def __hash__(self) -> int: pass - -@builtinclass -class unicode(Sequence[unicode]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, o: object) -> None: pass - @overload - def __init__(self, o: str, encoding: str = None, errors: str = 'strict') -> None: pass - @overload - def __init__(self, o: str, encoding: unicode = None, errors: unicode = 'strict') -> None: pass - @overload - def __init__(self, o: bytearray, encoding: str = None, errors: str = 'strict') -> None: pass - @overload - def __init__(self, o: bytearray, encoding: unicode = None, - errors: unicode = 'strict') -> None: pass - def capitalize(self) -> unicode: pass - def center(self, width: int, fillchar: unicode = u' ') -> unicode: pass - def count(self, x: unicode) -> int: pass - def decode(self, encoding: unicode = 'utf-8', errors: unicode = 'strict') -> unicode: pass - def encode(self, encoding: unicode = 'utf-8', errors: unicode = 'strict') -> str: pass - def endswith(self, suffix: unicode, start: int = 0, - end: int = None) -> bool: pass # TODO tuple suffix; None value for int - def expandtabs(self, tabsize: int = 8) -> unicode: pass - def find(self, sub: unicode, start: int = 0, end: int = 0) -> int: pass - def format(self, *args: Any, **kwargs: Any) -> unicode: pass - def format_map(self, map: Mapping[unicode, Any]) -> unicode: pass - def index(self, sub: unicode, start: int = 0, end: int = 0) -> int: pass - def isalnum(self) -> bool: pass - def isalpha(self) -> bool: pass - def isdecimal(self) -> bool: pass - def isdigit(self) -> bool: pass - def isidentifier(self) -> bool: pass - def islower(self) -> bool: pass - def isnumeric(self) -> bool: pass - def isprintable(self) -> bool: pass - def isspace(self) -> bool: pass - def istitle(self) -> bool: pass - def isupper(self) -> bool: pass - def join(self, iterable: Iterable[unicode]) -> unicode: pass - def ljust(self, width: int, fillchar: unicode = u' ') -> unicode: pass - def lower(self) -> unicode: pass - def lstrip(self, chars: unicode = None) -> unicode: pass - def partition(self, sep: unicode) -> Tuple[unicode, unicode, unicode]: pass - def replace(self, old: unicode, new: unicode, count: int = -1) -> unicode: pass - def rfind(self, sub: unicode, start: int = 0, end: int = 0) -> int: pass - def rindex(self, sub: unicode, start: int = 0, end: int = 0) -> int: pass - def rjust(self, width: int, fillchar: unicode = u' ') -> unicode: pass - def rpartition(self, sep: unicode) -> Tuple[unicode, unicode, unicode]: pass - def rsplit(self, sep: unicode = None, maxsplit: int = -1) -> List[unicode]: pass - def rstrip(self, chars: unicode = None) -> unicode: pass - def split(self, sep: unicode = None, maxsplit: int = -1) -> List[unicode]: pass - def splitlines(self, keepends: bool = False) -> List[unicode]: pass - def startswith(self, prefix: unicode, start: int = 0, - end: int = None) -> bool: pass # TODO tuple prefix; None value for int - def strip(self, chars: unicode = None) -> unicode: pass - def swapcase(self) -> unicode: pass - def title(self) -> unicode: pass - def translate(self, table: Dict[int, Any]) -> unicode: pass - def upper(self) -> unicode: pass - def zfill(self, width: int) -> unicode: pass - # TODO maketrans - - @overload - def __getitem__(self, i: int) -> unicode: pass - @overload - def __getitem__(self, s: slice) -> unicode: pass - def __getslice__(self, start: int, stop: int) -> unicode: pass - def __add__(self, s: unicode) -> unicode: pass - def __mul__(self, n: int) -> unicode: pass - def __mod__(self, *args: Any) -> unicode: pass - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: unicode) -> bool: pass - def __le__(self, x: unicode) -> bool: pass - def __gt__(self, x: unicode) -> bool: pass - def __ge__(self, x: unicode) -> bool: pass - - def __len__(self) -> int: pass - def __contains__(self, s: object) -> bool: pass - def __iter__(self) -> Iterator[unicode]: pass - def __str__(self) -> str: pass - def __repr__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: pass - def __hash__(self) -> int: pass - -@builtinclass -@ducktype(unicode) -class str(Sequence[str]): - def __init__(self, object: object) -> None: pass - def capitalize(self) -> str: pass - def center(self, width: int, fillchar: Union[str, bytearray] = None) -> str: pass - def count(self, x: Union[unicode, bytearray]) -> int: pass - def decode(self, encoding: unicode = 'utf-8', errors: unicode = 'strict') -> unicode: pass - def encode(self, encoding: unicode = 'utf-8', errors: unicode = 'strict') -> str: pass - def endswith(self, suffix: Union[unicode, bytearray]) -> bool: pass - def expandtabs(self, tabsize: int = 8) -> str: pass - def find(self, sub: Union[unicode, bytearray], start: int = 0, end: int = 0) -> int: pass - def index(self, sub: Union[unicode, bytearray], start: int = 0, end: int = 0) -> int: pass - def isalnum(self) -> bool: pass - def isalpha(self) -> bool: pass - def isdigit(self) -> bool: pass - def islower(self) -> bool: pass - def isspace(self) -> bool: pass - def istitle(self) -> bool: pass - def isupper(self) -> bool: pass - def join(self, iterable: Iterable[AnyStr]) -> AnyStr: pass - def ljust(self, width: int, fillchar: Union[str, bytearray] = None) -> str: pass - def lower(self) -> str: pass - @overload - def lstrip(self, chars: Union[str, bytearray] = None) -> str: pass - @overload - def lstrip(self, chars: unicode) -> unicode: pass - @overload - def partition(self, sep: str) -> Tuple[str, str, str]: pass - @overload - def partition(self, sep: bytearray) -> Tuple[str, bytearray, str]: pass - @overload - def partition(self, sep: unicode) -> Tuple[unicode, unicode, unicode]: pass - @overload - def replace(self, old: Union[str, bytearray], new: Union[str, bytearray], - count: int = -1) -> str: pass - @overload - def replace(self, old: unicode, new: unicode, count: int = -1) -> unicode: pass - def rfind(self, sub: Union[unicode, bytearray], start: int = 0, end: int = 0) -> int: pass - def rindex(self, sub: Union[unicode, bytearray], start: int = 0, end: int = 0) -> int: pass - def rjust(self, width: int, fillchar: Union[str, bytearray] = None) -> str: pass - @overload - def rpartition(self, sep: str) -> Tuple[str, str, str]: pass - @overload - def rpartition(self, sep: bytearray) -> Tuple[str, bytearray, str]: pass - @overload - def rpartition(self, sep: unicode) -> Tuple[unicode, unicode, unicode]: pass - @overload - def rsplit(self, sep: Union[str, bytearray] = None, maxsplit: int = -1) -> List[str]: pass - @overload - def rsplit(self, sep: unicode, maxsplit: int = -1) -> List[unicode]: pass - @overload - def rstrip(self, chars: Union[str, bytearray] = None) -> str: pass - @overload - def rstrip(self, chars: unicode) -> unicode: pass - @overload - def split(self, sep: Union[str, bytearray] = None, maxsplit: int = -1) -> List[str]: pass - @overload - def split(self, sep: unicode, maxsplit: int = -1) -> List[unicode]: pass - def splitlines(self, keepends: bool = False) -> List[str]: pass - def startswith(self, prefix: Union[unicode, bytearray]) -> bool: pass - @overload - def strip(self, chars: Union[str, bytearray] = None) -> str: pass - @overload - def strip(self, chars: unicode) -> unicode: pass - def swapcase(self) -> str: pass - def title(self) -> str: pass - @overload - def translate(self, table: Union[str, bytearray]) -> str: pass - @overload - def translate(self, table: unicode) -> unicode: pass - def upper(self) -> str: pass - def zfill(self, width: int) -> str: pass - # TODO fromhex - # TODO maketrans - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[str]: pass - def __str__(self) -> str: pass - def __repr__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: pass - def __hash__(self) -> int: pass - @overload - def __getitem__(self, i: int) -> str: pass - @overload - def __getitem__(self, s: slice) -> str: pass - def __getslice__(self, start: int, stop: int) -> str: pass - @overload - def __add__(self, s: str) -> str: pass - @overload - def __add__(self, s: bytearray) -> str: pass - @overload - def __add__(self, s: unicode) -> unicode: pass - def __mul__(self, n: int) -> str: pass - def __rmul__(self, n: int) -> str: pass - def __contains__(self, o: object) -> bool: pass - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: unicode) -> bool: pass - def __le__(self, x: unicode) -> bool: pass - def __gt__(self, x: unicode) -> bool: pass - def __ge__(self, x: unicode) -> bool: pass - - -@builtinclass -class bytearray(Sequence[int]): - @overload - def __init__(self, ints: Iterable[int]) -> None: pass - @overload - def __init__(self, string: unicode, encoding: unicode, - errors: unicode = 'strict') -> None: pass - @overload - def __init__(self, length: int) -> None: pass - @overload - def __init__(self) -> None: pass - def capitalize(self) -> bytearray: pass - @overload - def center(self, width: int, fillchar: str = None) -> bytearray: pass - @overload - def center(self, width: int, fillchar: bytearray = None) -> bytearray: pass - @overload - def count(self, x: str) -> int: pass - @overload - def count(self, x: bytearray) -> int: pass - def decode(self, encoding: unicode = 'utf-8', errors: unicode = 'strict') -> str: pass - @overload - def endswith(self, suffix: str) -> bool: pass - @overload - def endswith(self, suffix: bytearray) -> bool: pass - def expandtabs(self, tabsize: int = 8) -> bytearray: pass - @overload - def find(self, sub: str, start: int = 0) -> int: pass - @overload - def find(self, sub: str, start: int, end: int) -> int: pass - @overload - def find(self, sub: bytearray, start: int = 0) -> int: pass - @overload - def find(self, sub: bytearray, start: int, end: int) -> int: pass - @overload - def index(self, sub: str, start: int = 0) -> int: pass - @overload - def index(self, sub: str, start: int, end: int) -> int: pass - @overload - def index(self, sub: bytearray, start: int = 0) -> int: pass - @overload - def index(self, sub: bytearray, start: int, end: int) -> int: pass - def isalnum(self) -> bool: pass - def isalpha(self) -> bool: pass - def isdigit(self) -> bool: pass - def islower(self) -> bool: pass - def isspace(self) -> bool: pass - def istitle(self) -> bool: pass - def isupper(self) -> bool: pass - @overload - def join(self, iterable: Iterable[str]) -> bytearray: pass - @overload - def join(self, iterable: Iterable[bytearray]) -> bytearray: pass - @overload - def ljust(self, width: int, fillchar: str = None) -> bytearray: pass - @overload - def ljust(self, width: int, fillchar: bytearray = None) -> bytearray: pass - def lower(self) -> bytearray: pass - @overload - def lstrip(self, chars: str = None) -> bytearray: pass - @overload - def lstrip(self, chars: bytearray = None) -> bytearray: pass - @overload - def partition(self, sep: str) -> Tuple[bytearray, bytearray, bytearray]: pass - @overload - def partition(self, sep: bytearray) -> Tuple[bytearray, bytearray, bytearray]: pass - @overload - def replace(self, old: str, new: str, count: int = -1) -> bytearray: pass - @overload - def replace(self, old: bytearray, new: bytearray, count: int = -1) -> bytearray: pass - @overload - def rfind(self, sub: str, start: int = 0) -> int: pass - @overload - def rfind(self, sub: str, start: int, end: int) -> int: pass - @overload - def rfind(self, sub: bytearray, start: int = 0) -> int: pass - @overload - def rfind(self, sub: bytearray, start: int, end: int) -> int: pass - @overload - def rindex(self, sub: str, start: int = 0) -> int: pass - @overload - def rindex(self, sub: str, start: int, end: int) -> int: pass - @overload - def rindex(self, sub: bytearray, start: int = 0) -> int: pass - @overload - def rindex(self, sub: bytearray, start: int, end: int) -> int: pass - @overload - def rjust(self, width: int, fillchar: str = None) -> bytearray: pass - @overload - def rjust(self, width: int, fillchar: bytearray = None) -> bytearray: pass - @overload - def rpartition(self, sep: str) -> Tuple[bytearray, bytearray, bytearray]: pass - @overload - def rpartition(self, sep: bytearray) -> Tuple[bytearray, bytearray, bytearray]:pass - @overload - def rsplit(self, sep: str = None, maxsplit: int = -1) -> List[bytearray]: pass - @overload - def rsplit(self, sep: bytearray = None, - maxsplit: int = -1) -> List[bytearray]: pass - @overload - def rstrip(self, chars: str = None) -> bytearray: pass - @overload - def rstrip(self, chars: bytearray = None) -> bytearray: pass - @overload - def split(self, sep: str = None, - maxsplit: int = -1) -> List[bytearray]: pass - @overload - def split(self, sep: bytearray = None, - maxsplit: int = -1) -> List[bytearray]: pass - def splitlines(self, keepends: bool = False) -> List[bytearray]: pass - @overload - def startswith(self, prefix: str) -> bool: pass - @overload - def startswith(self, prefix: bytearray) -> bool: pass - @overload - def strip(self, chars: str = None) -> bytearray: pass - @overload - def strip(self, chars: bytearray = None) -> bytearray: pass - def swapcase(self) -> bytearray: pass - def title(self) -> bytearray: pass - @overload - def translate(self, table: str) -> bytearray: pass - @overload - def translate(self, table: bytearray) -> bytearray: pass - def upper(self) -> bytearray: pass - def zfill(self, width: int) -> bytearray: pass - # TODO fromhex - # TODO maketrans - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[int]: pass - def __str__(self) -> str: pass - def __repr__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: pass - def __hash__(self) -> int: pass - @overload - def __getitem__(self, i: int) -> int: pass - @overload - def __getitem__(self, s: slice) -> bytearray: pass - def __getslice__(self, start: int, stop: int) -> bytearray: pass - @overload - def __setitem__(self, i: int, x: int) -> None: pass - @overload - def __setitem__(self, s: slice, x: Sequence[int]) -> None: pass - def __setslice__(self, start: int, stop: int, x: Sequence[int]) -> None: pass - @overload - def __delitem__(self, i: int) -> None: pass - @overload - def __delitem__(self, s: slice) -> None: pass - def __delslice__(self, start: int, stop: int) -> None: pass - @overload - def __add__(self, s: str) -> bytearray: pass - @overload - def __add__(self, s: bytearray) -> bytearray: pass - def __mul__(self, n: int) -> bytearray: pass - def __contains__(self, o: object) -> bool: pass - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - @overload - def __lt__(self, x: bytearray) -> bool: pass - @overload - def __lt__(self, x: str) -> bool: pass - @overload - def __le__(self, x: bytearray) -> bool: pass - @overload - def __le__(self, x: str) -> bool: pass - @overload - def __gt__(self, x: bytearray) -> bool: pass - @overload - def __gt__(self, x: str) -> bool: pass - @overload - def __ge__(self, x: bytearray) -> bool: pass - @overload - def __ge__(self, x: str) -> bool: pass - -@builtinclass -class bool(int, SupportsInt, SupportsFloat): - def __init__(self, o: object = False) -> None: pass - -@builtinclass -class slice: - start = 0 - step = 0 - stop = 0 - def __init__(self, start: int, stop: int, step: int) -> None: pass - -@builtinclass -class tuple(Sequence[Any]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, iterable: Iterable[Any]) -> None: pass - def __len__(self) -> int: pass - def __contains__(self, x: object) -> bool: pass - @overload - def __getitem__(self, x: int) -> Any: pass - @overload - def __getitem__(self, x: slice) -> tuple: pass - def __iter__(self) -> Iterator[Any]: pass - def __lt__(self, x: tuple) -> bool: pass - def __le__(self, x: tuple) -> bool: pass - def __gt__(self, x: tuple) -> bool: pass - def __ge__(self, x: tuple) -> bool: pass - def __add__(self, x: tuple) -> tuple: pass - def count(self, x: Any) -> int: pass - def index(self, x: Any) -> int: pass - -@builtinclass -class function: - # TODO name of the class (corresponds to Python 'function' class) - __name__ = '' - __module__ = '' - -@builtinclass -class list(Sequence[_T], Reversible[_T], AbstractGeneric[_T]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, iterable: Iterable[_T]) -> None: pass - def append(self, object: _T) -> None: pass - def extend(self, iterable: Iterable[_T]) -> None: pass - def pop(self, index: int = -1) -> _T: pass - def index(self, object: _T, start: int = 0, stop: int = Undefined(int)) -> int: pass - def count(self, object: _T) -> int: pass - def insert(self, index: int, object: _T) -> None: pass - def remove(self, object: _T) -> None: pass - def reverse(self) -> None: pass - def sort(self, *, key: Function[[_T], Any] = None, reverse: bool = False) -> None: pass - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[_T]: pass - def __str__(self) -> str: pass - def __hash__(self) -> int: pass - @overload - def __getitem__(self, i: int) -> _T: pass - @overload - def __getitem__(self, s: slice) -> List[_T]: pass - def __getslice__(self, start: int, stop: int) -> List[_T]: pass - @overload - def __setitem__(self, i: int, o: _T) -> None: pass - @overload - def __setitem__(self, s: slice, o: Sequence[_T]) -> None: pass - def __setslice__(self, start: int, stop: int, o: Sequence[_T]) -> None: pass - @overload - def __delitem__(self, i: int) -> None: pass - @overload - def __delitem__(self, s: slice) -> None: pass - def __delslice(self, start: int, stop: int) -> None: pass - def __add__(self, x: List[_T]) -> List[_T]: pass - def __mul__(self, n: int) -> List[_T]: pass - def __rmul__(self, n: int) -> List[_T]: pass - def __contains__(self, o: object) -> bool: pass - def __reversed__(self) -> Iterator[_T]: pass - def __gt__(self, x: List[_T]) -> bool: pass - def __ge__(self, x: List[_T]) -> bool: pass - def __lt__(self, x: List[_T]) -> bool: pass - def __le__(self, x: List[_T]) -> bool: pass - -@builtinclass -class dict(Mapping[_KT, _VT], Generic[_KT, _VT]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, map: Mapping[_KT, _VT]) -> None: pass - @overload - def __init__(self, iterable: Iterable[Tuple[_KT, _VT]]) -> None: pass # TODO keyword args - def has_key(self, k: _KT) -> bool: pass - def clear(self) -> None: pass - def copy(self) -> Dict[_KT, _VT]: pass - @overload - def get(self, k: _KT) -> _VT: pass - @overload - def get(self, k: _KT, default: _VT) -> _VT: pass - @overload - def pop(self, k: _KT) -> _VT: pass - @overload - def pop(self, k: _KT, default: _VT) -> _VT: pass - def popitem(self) -> Tuple[_KT, _VT]: pass - @overload - def setdefault(self, k: _KT) -> _VT: pass - @overload - def setdefault(self, k: _KT, default: _VT) -> _VT: pass - @overload - def update(self, m: Mapping[_KT, _VT]) -> None: pass - @overload - def update(self, m: Iterable[Tuple[_KT, _VT]]) -> None: pass - def keys(self) -> List[_KT]: pass - def values(self) -> List[_VT]: pass - def items(self) -> List[Tuple[_KT, _VT]]: pass - def iterkeys(self) -> Iterator[_KT]: pass - def itervalues(self) -> Iterator[_VT]: pass - def iteritems(self) -> Iterator[Tuple[_KT, _VT]]: pass - def __len__(self) -> int: pass - def __getitem__(self, k: _KT) -> _VT: pass - def __setitem__(self, k: _KT, v: _VT) -> None: pass - def __delitem__(self, v: _KT) -> None: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[_KT]: pass - def __str__(self) -> str: pass - -@builtinclass -class set(AbstractSet[_T], Generic[_T]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, iterable: Iterable[_T]) -> None: pass - def add(self, element: _T) -> None: pass - def remove(self, element: _T) -> None: pass - def copy(self) -> AbstractSet[_T]: pass - def isdisjoint(self, s: AbstractSet[_T]) -> bool: pass - def __len__(self) -> int: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[_T]: pass - def __str__(self) -> str: pass - def __and__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - def __or__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - def __sub__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - def __xor__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - # TODO more set operations - -@builtinclass -class frozenset(AbstractSet[_T], Generic[_T]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, iterable: Iterable[_T]) -> None: pass - def isdisjoint(self, s: AbstractSet[_T]) -> bool: pass - def __len__(self) -> int: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[_T]: pass - def __str__(self) -> str: pass - def __and__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __or__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __sub__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __xor__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - # TODO more set operations - -@builtinclass -class enumerate(Iterator[Tuple[int, _T]], Generic[_T]): - def __init__(self, iterable: Iterable[_T], start: int = 0) -> None: pass - def __iter__(self) -> Iterator[Tuple[int, _T]]: pass - def next(self) -> Tuple[int, _T]: pass - # TODO __getattribute__ - -@builtinclass -class xrange(Sized, Iterable[int], Reversible[int]): - @overload - def __init__(self, stop: int) -> None: pass - @overload - def __init__(self, start: int, stop: int, step: int = 1) -> None: pass - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[int]: pass - def __getitem__(self, i: int) -> int: pass - def __reversed__(self) -> Iterator[int]: pass - -@builtinclass -class module: - __name__ = '' - __file__ = '' - __dict__ = Undefined # type: Dict[unicode, Any] - -True = Undefined # type: bool -False = Undefined # type: bool -__debug__ = False - -long = int -bytes = str - -NotImplemented = Undefined # type: Any - -def abs(n: SupportsAbs[_T]) -> _T: pass -def all(i: Iterable) -> bool: pass -def any(i: Iterable) -> bool: pass -def callable(o: object) -> bool: pass -def chr(code: int) -> str: pass -def delattr(o: Any, name: unicode) -> None: pass -def dir(o: object = None) -> List[str]: pass -@overload -def divmod(a: int, b: int) -> Tuple[int, int]: pass -@overload -def divmod(a: float, b: float) -> Tuple[float, float]: pass -def filter(function: Function[[_T], Any], - iterable: Iterable[_T]) -> List[_T]: pass -def format(o: object, format_spec: str = '') -> str: pass # TODO unicode -def getattr(o: Any, name: unicode, default: Any = None) -> Any: pass -def hasattr(o: Any, name: unicode) -> bool: pass -def hash(o: object) -> int: pass -def hex(i: int) -> str: pass # TODO __index__ -def id(o: object) -> int: pass -def input(prompt: unicode = None) -> Any: pass -@overload -def iter(iterable: Iterable[_T]) -> Iterator[_T]: pass -@overload -def iter(function: Function[[], _T], sentinel: _T) -> Iterator[_T]: pass -@overload -def isinstance(o: object, t: type) -> bool: pass -@overload -def isinstance(o: object, t: tuple) -> bool: pass -def issubclass(cls: type, classinfo: type) -> bool: pass -# TODO support this -#def issubclass(type cld, classinfo: Sequence[type]) -> bool: pass -def len(o: Sized) -> int: pass -@overload -def map(func: Function[[_T1], _S], iter1: Iterable[_T1]) -> List[_S]: pass -@overload -def map(func: Function[[_T1, _T2], _S], - iter1: Iterable[_T1], - iter2: Iterable[_T2]) -> List[_S]: pass # TODO more than two iterables -@overload -def max(iterable: Iterable[_T]) -> _T: pass # TODO keyword argument key -@overload -def max(arg1: _T, arg2: _T, *args: _T) -> _T: pass -# TODO memoryview -@overload -def min(iterable: Iterable[_T]) -> _T: pass -@overload -def min(arg1: _T, arg2: _T, *args: _T) -> _T: pass -@overload -def next(i: Iterator[_T]) -> _T: pass -@overload -def next(i: Iterator[_T], default: _T) -> _T: pass -def oct(i: int) -> str: pass # TODO __index__ -@overload -def open(file: str, mode: str = 'r', buffering: int = -1) -> BinaryIO: pass -@overload -def open(file: unicode, mode: str = 'r', buffering: int = -1) -> BinaryIO: pass -@overload -def open(file: int, mode: str = 'r', buffering: int = -1) -> BinaryIO: pass -@overload -def ord(c: unicode) -> int: pass -@overload -def ord(c: bytearray) -> int: pass -# This is only available after from __future__ import print_function. -def print(*values: Any, sep: unicode = u' ', end: unicode = u'\n', - file: IO[Any] = None) -> None: pass -@overload -def pow(x: int, y: int) -> Any: pass # The return type can be int or float, depending on y. -@overload -def pow(x: int, y: int, z: int) -> Any: pass -@overload -def pow(x: float, y: float) -> float: pass -@overload -def pow(x: float, y: float, z: float) -> float: pass -def range(x: int, y: int = 0, step: int = 1) -> List[int]: pass -def raw_input(prompt: unicode = None) -> str: pass -@overload -def reversed(object: Reversible[_T]) -> Iterator[_T]: pass -@overload -def reversed(object: Sequence[_T]) -> Iterator[_T]: pass -def repr(o: object) -> str: pass -@overload -def round(number: float) -> int: pass -@overload -def round(number: float, ndigits: int) -> float: pass # Always return a float if given ndigits. -@overload -def round(number: SupportsRound[_T]) -> _T: pass -@overload -def round(number: SupportsRound[_T], ndigits: int) -> _T: pass -def setattr(object: Any, name: unicode, value: Any) -> None: pass -def sorted(iterable: Iterable[_T], *, - cmp: Function[[_T, _T], int] = None, - key: Function[[_T], Any] = None, - reverse: bool = False) -> List[_T]: pass -def sum(iterable: Iterable[_T], start: _T = None) -> _T: pass -@overload -def zip(iter1: Iterable[_T1]) -> List[Tuple[_T1]]: pass -@overload -def zip(iter1: Iterable[_T1], - iter2: Iterable[_T2]) -> List[Tuple[_T1, _T2]]: pass -@overload -def zip(iter1: Iterable[_T1], iter2: Iterable[_T2], - iter3: Iterable[_T3]) -> List[Tuple[_T1, _T2, _T3]]: pass -@overload -def zip(iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], - iter4: Iterable[_T4]) -> List[Tuple[_T1, _T2, - _T3, _T4]]: pass # TODO more than four iterables -def __import__(name: unicode, - globals: Dict[str, Any] = {}, - locals: Dict[str, Any] = {}, - fromlist: List[str] = [], level: int = -1) -> Any: pass - -# Exceptions - -@builtinclass -class BaseException: - args = Undefined # type: Any - def __init__(self, *args: Any) -> None: pass - def with_traceback(self, tb: Any) -> BaseException: pass - -class GeneratorExit(BaseException): pass -class KeyboardInterrupt(BaseException): pass -class SystemExit(BaseException): - code = 0 -class Exception(BaseException): pass -class StopIteration(Exception): pass -class StandardError(Exception): pass -class ArithmeticError(StandardError): pass -@builtinclass -class EnvironmentError(StandardError): - errno = 0 - strerror = '' - filename = '' # TODO can this be unicode? -class LookupError(StandardError): pass -class RuntimeError(StandardError): pass -class ValueError(StandardError): pass -class AssertionError(StandardError): pass -class AttributeError(StandardError): pass -class EOFError(StandardError): pass -class FloatingPointError(ArithmeticError): pass -class IOError(EnvironmentError): pass -class ImportError(StandardError): pass -class IndexError(LookupError): pass -class KeyError(LookupError): pass -class MemoryError(StandardError): pass -class NameError(StandardError): pass -class NotImplementedError(RuntimeError): pass -class OSError(EnvironmentError): pass -class WindowsError(OSError): pass # TODO Windows-only -# TODO: VMSError? -class OverflowError(ArithmeticError): pass -class ReferenceError(StandardError): pass -class SyntaxError(StandardError): pass -class IndentationError(SyntaxError): pass -class TabError(IndentationError): pass -class SystemError(StandardError): pass -class TypeError(StandardError): pass -class UnboundLocalError(NameError): pass -class UnicodeError(ValueError): pass -class UnicodeDecodeError(UnicodeError): pass -class UnicodeEncodeError(UnicodeError): pass -class UnicodeTranslateError(UnicodeError): pass -class ZeroDivisionError(ArithmeticError): pass - -class Warning(Exception): pass -class UserWarning(Warning): pass -class DeprecationWarning(Warning): pass -class SyntaxWarning(Warning): pass -class RuntimeWarning(Warning): pass -class FutureWarning(Warning): pass -class PendingDeprecationWarning(Warning): pass -class ImportWarning(Warning): pass -class UnicodeWarning(Warning): pass -class BytesWarning(Warning): pass -class ResourceWarning(Warning): pass diff --git a/stubs/2.7/collections.py b/stubs/2.7/collections.py deleted file mode 100644 index 8076d1b39e8e..000000000000 --- a/stubs/2.7/collections.py +++ /dev/null @@ -1,4 +0,0 @@ -# NOTE: These are incomplete! - -# namedtuple is special-cased in the type checker; the initializer is ignored. -namedtuple = object() diff --git a/stubs/2.7/io.py b/stubs/2.7/io.py deleted file mode 100644 index d9c9e1c9d169..000000000000 --- a/stubs/2.7/io.py +++ /dev/null @@ -1,98 +0,0 @@ -# Stubs for io - -# Based on https://docs.python.org/2/library/io.html - -# Only a subset of functionality is included. - -DEFAULT_BUFFER_SIZE = 0 - -from typing import List, BinaryIO, TextIO, IO, overload, Iterator, Iterable, Any, Union - -def open(file: Union[str, unicode, int], - mode: unicode = 'r', buffering: int = -1, encoding: unicode = None, - errors: unicode = None, newline: unicode = None, - closefd: bool = True) -> IO[Any]: pass - -class BytesIO(BinaryIO): - def __init__(self, initial_bytes: str = b'') -> None: pass - # TODO getbuffer - # TODO see comments in BinaryIO for missing functionality - def close(self) -> None: pass - def closed(self) -> bool: pass - def fileno(self) -> int: pass - def flush(self) -> None: pass - def isatty(self) -> bool: pass - def read(self, n: int = -1) -> str: pass - def readable(self) -> bool: pass - def readline(self, limit: int = -1) -> str: pass - def readlines(self, hint: int = -1) -> List[str]: pass - def seek(self, offset: int, whence: int = 0) -> int: pass - def seekable(self) -> bool: pass - def tell(self) -> int: pass - def truncate(self, size: int = None) -> int: pass - def writable(self) -> bool: pass - @overload - def write(self, s: str) -> int: pass - @overload - def write(self, s: bytearray) -> int: pass - def writelines(self, lines: Iterable[str]) -> None: pass - def getvalue(self) -> str: pass - def read1(self) -> str: pass - - def __iter__(self) -> Iterator[str]: pass - def __enter__(self) -> 'BytesIO': pass - def __exit__(self, type, value, traceback) -> bool: pass - -class StringIO(TextIO): - def __init__(self, initial_value: unicode = '', - newline: unicode = None) -> None: pass - # TODO see comments in BinaryIO for missing functionality - def close(self) -> None: pass - def closed(self) -> bool: pass - def fileno(self) -> int: pass - def flush(self) -> None: pass - def isatty(self) -> bool: pass - def read(self, n: int = -1) -> unicode: pass - def readable(self) -> bool: pass - def readline(self, limit: int = -1) -> unicode: pass - def readlines(self, hint: int = -1) -> List[unicode]: pass - def seek(self, offset: int, whence: int = 0) -> int: pass - def seekable(self) -> bool: pass - def tell(self) -> int: pass - def truncate(self, size: int = None) -> int: pass - def writable(self) -> bool: pass - def write(self, s: unicode) -> int: pass - def writelines(self, lines: Iterable[unicode]) -> None: pass - def getvalue(self) -> unicode: pass - - def __iter__(self) -> Iterator[unicode]: pass - def __enter__(self) -> 'StringIO': pass - def __exit__(self, type, value, traceback) -> bool: pass - -class TextIOWrapper(TextIO): - # write_through is undocumented but used by subprocess - def __init__(self, buffer: IO[str], encoding: unicode = None, - errors: unicode = None, newline: unicode = None, - line_buffering: bool = False, - write_through: bool = True) -> None: pass - # TODO see comments in BinaryIO for missing functionality - def close(self) -> None: pass - def closed(self) -> bool: pass - def fileno(self) -> int: pass - def flush(self) -> None: pass - def isatty(self) -> bool: pass - def read(self, n: int = -1) -> unicode: pass - def readable(self) -> bool: pass - def readline(self, limit: int = -1) -> unicode: pass - def readlines(self, hint: int = -1) -> List[unicode]: pass - def seek(self, offset: int, whence: int = 0) -> int: pass - def seekable(self) -> bool: pass - def tell(self) -> int: pass - def truncate(self, size: int = None) -> int: pass - def writable(self) -> bool: pass - def write(self, s: unicode) -> int: pass - def writelines(self, lines: Iterable[unicode]) -> None: pass - - def __iter__(self) -> Iterator[unicode]: pass - def __enter__(self) -> StringIO: pass - def __exit__(self, type, value, traceback) -> bool: pass diff --git a/stubs/2.7/json.py b/stubs/2.7/json.py deleted file mode 100644 index 5b8987ec61f3..000000000000 --- a/stubs/2.7/json.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Any, IO - -class JSONDecodeError(object): - def dumps(self, obj: Any) -> str: pass - def dump(self, obj: Any, fp: IO[str], *args: Any, **kwds: Any) -> None: pass - def loads(self, s: str) -> Any: pass - def load(self, fp: IO[str]) -> Any: pass - -def dumps(obj: Any) -> str: pass -def dump(obj: Any, fp: IO[str], *args: Any, **kwds: Any) -> None: pass -def loads(s: str) -> Any: pass -def load(fp: IO[str]) -> Any: pass diff --git a/stubs/2.7/re.py b/stubs/2.7/re.py deleted file mode 100644 index dd7484a7ad37..000000000000 --- a/stubs/2.7/re.py +++ /dev/null @@ -1,63 +0,0 @@ -# Stubs for re -# Ron Murawski -# 'bytes' support added by Jukka Lehtosalo - -# based on: http://docs.python.org/2.7/library/re.html - -from typing import ( - Undefined, List, Iterator, overload, Function, Tuple, Sequence, Dict, - Generic, AnyStr, Match, Pattern -) - -# ----- re variables and constants ----- -A = 0 -ASCII = 0 -DEBUG = 0 -I = 0 -IGNORECASE = 0 -L = 0 -LOCALE = 0 -M = 0 -MULTILINE = 0 -S = 0 -DOTALL = 0 -X = 0 -VERBOSE = 0 - -class error(Exception): pass - -def compile(pattern: AnyStr, flags: int = 0) -> Pattern[AnyStr]: pass -def search(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> Match[AnyStr]: pass -def match(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> Match[AnyStr]: pass -def split(pattern: AnyStr, string: AnyStr, maxsplit: int = 0, - flags: int = 0) -> List[AnyStr]: pass -def findall(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> List[AnyStr]: pass - -# Return an iterator yielding match objects over all non-overlapping matches -# for the RE pattern in string. The string is scanned left-to-right, and -# matches are returned in the order found. Empty matches are included in the -# result unless they touch the beginning of another match. -def finditer(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> Iterator[Match[AnyStr]]: pass - -@overload -def sub(pattern: AnyStr, repl: AnyStr, string: AnyStr, count: int = 0, - flags: int = 0) -> AnyStr: pass -@overload -def sub(pattern: AnyStr, repl: Function[[Match[AnyStr]], AnyStr], - string: AnyStr, count: int = 0, flags: int = 0) -> AnyStr: pass - -@overload -def subn(pattern: AnyStr, repl: AnyStr, string: AnyStr, count: int = 0, - flags: int = 0) -> Tuple[AnyStr, int]: pass -@overload -def subn(pattern: AnyStr, repl: Function[[Match[AnyStr]], AnyStr], - string: AnyStr, count: int = 0, - flags: int = 0) -> Tuple[AnyStr, int]: pass - -def escape(string: AnyStr) -> AnyStr: pass - -def purge() -> None: pass diff --git a/stubs/2.7/simplejson.py b/stubs/2.7/simplejson.py deleted file mode 100644 index 5b8987ec61f3..000000000000 --- a/stubs/2.7/simplejson.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Any, IO - -class JSONDecodeError(object): - def dumps(self, obj: Any) -> str: pass - def dump(self, obj: Any, fp: IO[str], *args: Any, **kwds: Any) -> None: pass - def loads(self, s: str) -> Any: pass - def load(self, fp: IO[str]) -> Any: pass - -def dumps(obj: Any) -> str: pass -def dump(obj: Any, fp: IO[str], *args: Any, **kwds: Any) -> None: pass -def loads(s: str) -> Any: pass -def load(fp: IO[str]) -> Any: pass diff --git a/stubs/2.7/sys.py b/stubs/2.7/sys.py deleted file mode 100644 index 5ea0b16e260a..000000000000 --- a/stubs/2.7/sys.py +++ /dev/null @@ -1,153 +0,0 @@ -# Stubs for sys -# Ron Murawski - -# based on http://docs.python.org/2.7/library/sys.html - -# Partially adapted to Python 2.7 by Jukka Lehtosalo. - -from typing import ( - Undefined, List, Sequence, Any, Dict, Tuple, BinaryIO, overload -) - -# ----- sys variables ----- -abiflags = '' -argv = Undefined(List[str]) -byteorder = '' -builtin_module_names = Undefined(Sequence[str]) # actually a tuple of strings -copyright = '' -#dllhandle = 0 # Windows only -dont_write_bytecode = False -__displayhook__ = Undefined(Any) # contains the original value of displayhook -__excepthook__ = Undefined(Any) # contains the original value of excepthook -exec_prefix = '' -executable = '' -float_repr_style = '' -hexversion = 0 # this is a 32-bit int -last_type = Undefined(Any) -last_value = Undefined(Any) -last_traceback = Undefined(Any) -maxsize = 0 -maxunicode = 0 -meta_path = Undefined(List[Any]) -modules = Undefined(Dict[str, Any]) -path = Undefined(List[str]) -path_hooks = Undefined(List[Any]) # TODO precise type; function, path to finder -path_importer_cache = Undefined(Dict[str, Any]) # TODO precise type -platform = '' -prefix = '' -ps1 = '' -ps2 = '' -stdin = Undefined(BinaryIO) -stdout = Undefined(BinaryIO) -stderr = Undefined(BinaryIO) -__stdin__ = Undefined(BinaryIO) -__stdout__ = Undefined(BinaryIO) -__stderr__ = Undefined(BinaryIO) -subversion = Undefined(Tuple[str, str, str]) -tracebacklimit = 0 -version = '' -api_version = 0 -warnoptions = Undefined(Any) -# Each entry is a tuple of the form (action, message, category, module, -# lineno) -#winver = '' # Windows only -_xoptions = Undefined(Dict[Any, Any]) - -flags = Undefined(_flags) -class _flags: - debug = 0 - division_warning = 0 - inspect = 0 - interactive = 0 - optimize = 0 - dont_write_bytecode = 0 - no_user_site = 0 - no_site = 0 - ignore_environment = 0 - verbose = 0 - bytes_warning = 0 - quiet = 0 - hash_randomization = 0 - -float_info = Undefined(_float_info) -class _float_info: - epsilon = 0.0 # DBL_EPSILON - dig = 0 # DBL_DIG - mant_dig = 0 # DBL_MANT_DIG - max = 0.0 # DBL_MAX - max_exp = 0 # DBL_MAX_EXP - max_10_exp = 0 # DBL_MAX_10_EXP - min = 0.0 # DBL_MIN - min_exp = 0 # DBL_MIN_EXP - min_10_exp = 0 # DBL_MIN_10_EXP - radix = 0 # FLT_RADIX - rounds = 0 # FLT_ROUNDS - -hash_info = Undefined(_hash_info) -class _hash_info: - width = 0 # width in bits used for hash values - modulus = 0 # prime modulus P used for numeric hash scheme - inf = 0 # hash value returned for a positive infinity - nan = 0 # hash value returned for a nan - imag = 0 # multiplier used for the imaginary part of a complex number - -int_info = Undefined(_int_info) -class _int_info: - bits_per_digit = 0 # number of bits held in each digit. Python integers - # are stored internally in - # base 2**int_info.bits_per_digit - sizeof_digit = 0 # size in bytes of C type used to represent a digit - -version_info = Undefined(_version_info) -class _version_info: - major = 0 - minor = 0 - micro = 0 - releaselevel = '' - serial = 0 - - -# ----- sys function stubs ----- -def call_tracing(fn: Any, args: Any) -> object: pass -def _clear_type_cache() -> None: pass -def _current_frames() -> Dict[int, Any]: pass -def displayhook(value: int) -> None: pass # value might be None -def excepthook(type_: type, value: BaseException, traceback: Any) -> None: - # TODO traceback type - pass -def exc_info() -> Tuple[type, Any, Any]: pass # see above -def exit(arg: int = 0) -> None: pass # arg might be None -def getcheckinterval() -> int: pass # deprecated -def getdefaultencoding() -> str: pass -#def getdlopenflags() -> int: pass # Unix only -def getfilesystemencoding() -> str: pass # cannot return None -#def getrefcount(object) -> int: pass # no ref counts in MyPy! -def getrecursionlimit() -> int: pass - -@overload -def getsizeof(obj: object) -> int: pass -@overload -def getsizeof(obj: object, default: int) -> int: pass - -def getswitchinterval() -> float: pass - -@overload -def _getframe() -> Any: pass -@overload -def _getframe(depth: int) -> Any: pass - -def getprofile() -> Any: pass # TODO return type -def gettrace() -> Any: pass # TODO return -def getwindowsversion() -> Any: pass # Windows only, TODO return type -def intern(string: str) -> str: pass -def setcheckinterval(interval: int) -> None: pass # deprecated -#def setdlopenflags(n: int) -> None: pass # Linux only -def setprofile(profilefunc: Any) -> None: pass # TODO type -def setrecursionlimit(limit: int) -> None: pass -def setswitchinterval(interval: float) -> None: pass -def settrace(tracefunc: Any) -> None: pass # TODO type -# Trace functions should have three arguments: frame, event, and arg. frame -# is the current stack frame. event is a string: 'call', 'line', 'return', -# 'exception', 'c_call', 'c_return', or 'c_exception'. arg depends on the -# event type. -def settscdump(on_flag: bool) -> None: pass diff --git a/stubs/2.7/typing.py b/stubs/2.7/typing.py deleted file mode 100644 index 17e3382d9766..000000000000 --- a/stubs/2.7/typing.py +++ /dev/null @@ -1,301 +0,0 @@ -# Stubs for typing (Python 2.7) - -from abc import abstractmethod, ABCMeta - -# Definitions of special type checking related constructs. Their definition -# are not used, so their value does not matter. - -cast = object() -overload = object() -Undefined = object() -Any = object() -typevar = object() -Generic = object() -AbstractGeneric = object() -Tuple = object() -Function = object() -builtinclass = object() -ducktype = object() -disjointclass = object() - -# Type aliases - -class TypeAlias: - """Class for defining generic aliases for library types.""" - - def __init__(self, target_type): - self.target_type = target_type - - def __getitem__(self, typeargs): - return self.target_type - -Union = TypeAlias(object) -List = TypeAlias(object) -Dict = TypeAlias(object) -Set = TypeAlias(object) - -# Predefined type variables. -AnyStr = typevar('AnyStr', values=(str, unicode)) - -# Abstract base classes. - -_T = typevar('_T') -_KT = typevar('_KT') -_VT = typevar('_VT') - -# TODO Container etc. - -class SupportsInt(metaclass=ABCMeta): - @abstractmethod - def __int__(self) -> int: pass - -class SupportsFloat(metaclass=ABCMeta): - @abstractmethod - def __float__(self) -> float: pass - -class SupportsAbs(AbstractGeneric[_T]): - @abstractmethod - def __abs__(self) -> _T: pass - -@disjointclass(int) -@disjointclass(float) -class SupportsRound(AbstractGeneric[_T]): - @abstractmethod - def __round__(self, ndigits: int = 0) -> _T: pass - -class Reversible(AbstractGeneric[_T]): - @abstractmethod - def __reversed__(self) -> Iterator[_T]: pass - -class Sized(metaclass=ABCMeta): - @abstractmethod - def __len__(self) -> int: pass - -class Iterable(AbstractGeneric[_T]): - @abstractmethod - def __iter__(self) -> Iterator[_T]: pass - -class Iterator(Iterable[_T], AbstractGeneric[_T]): - @abstractmethod - def next(self) -> _T: pass - -class Sequence(Sized, Iterable[_T], AbstractGeneric[_T]): - @abstractmethod - def __contains__(self, x: object) -> bool: pass - @overload - @abstractmethod - def __getitem__(self, i: int) -> _T: pass - @overload - @abstractmethod - def __getitem__(self, s: slice) -> Sequence[_T]: pass - -class AbstractSet(Sized, Iterable[_T], AbstractGeneric[_T]): - @abstractmethod - def __contains__(self, x: object) -> bool: pass - # TODO __le__, __lt__, __gt__, __ge__ - @abstractmethod - def __and__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - @abstractmethod - def __or__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - @abstractmethod - def __sub__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - @abstractmethod - def __xor__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - # TODO argument can be any container? - @abstractmethod - def isdisjoint(self, s: AbstractSet[_T]) -> bool: pass - -class Mapping(Sized, Iterable[_KT], AbstractGeneric[_KT, _VT]): - @abstractmethod - def __getitem__(self, k: _KT) -> _VT: pass - @abstractmethod - def __setitem__(self, k: _KT, v: _VT) -> None: pass - @abstractmethod - def __delitem__(self, v: _KT) -> None: pass - @abstractmethod - def __contains__(self, o: object) -> bool: pass - - @abstractmethod - def clear(self) -> None: pass - @abstractmethod - def copy(self) -> Mapping[_KT, _VT]: pass - @overload - @abstractmethod - def get(self, k: _KT) -> _VT: pass - @overload - @abstractmethod - def get(self, k: _KT, default: _VT) -> _VT: pass - @overload - @abstractmethod - def pop(self, k: _KT) -> _VT: pass - @overload - @abstractmethod - def pop(self, k: _KT, default: _VT) -> _VT: pass - @abstractmethod - def popitem(self) -> Tuple[_KT, _VT]: pass - @overload - @abstractmethod - def setdefault(self, k: _KT) -> _VT: pass - @overload - @abstractmethod - def setdefault(self, k: _KT, default: _VT) -> _VT: pass - - # TODO keyword arguments - @overload - @abstractmethod - def update(self, m: Mapping[_KT, _VT]) -> None: pass - @overload - @abstractmethod - def update(self, m: Iterable[Tuple[_KT, _VT]]) -> None: pass - - @abstractmethod - def keys(self) -> list[_KT]: pass - @abstractmethod - def values(self) -> list[_VT]: pass - @abstractmethod - def items(self) -> list[Tuple[_KT, _VT]]: pass - @abstractmethod - def iterkeys(self) -> Iterator[_KT]: pass - @abstractmethod - def itervalues(self) -> Iterator[_VT]: pass - @abstractmethod - def iteritems(self) -> Iterator[Tuple[_KT, _VT]]: pass - -class IO(Iterable[AnyStr], AbstractGeneric[AnyStr]): - # TODO detach - # TODO use abstract properties - @property - def mode(self) -> str: pass - @property - def name(self) -> str: pass - @abstractmethod - def close(self) -> None: pass - @property - def closed(self) -> bool: pass - @abstractmethod - def fileno(self) -> int: pass - @abstractmethod - def flush(self) -> None: pass - @abstractmethod - def isatty(self) -> bool: pass - # TODO what if n is None? - @abstractmethod - def read(self, n: int = -1) -> AnyStr: pass - @abstractmethod - def readable(self) -> bool: pass - @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: pass - @abstractmethod - def readlines(self, hint: int = -1) -> list[AnyStr]: pass - @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: pass - @abstractmethod - def seekable(self) -> bool: pass - @abstractmethod - def tell(self) -> int: pass - # TODO None should not be compatible with int - @abstractmethod - def truncate(self, size: int = None) -> int: pass - @abstractmethod - def writable(self) -> bool: pass - # TODO buffer objects - @abstractmethod - def write(self, s: AnyStr) -> int: pass - @abstractmethod - def writelines(self, lines: Iterable[AnyStr]) -> None: pass - - @abstractmethod - def __iter__(self) -> Iterator[AnyStr]: pass - @abstractmethod - def __enter__(self) -> 'IO[AnyStr]': pass - @abstractmethod - def __exit__(self, type, value, traceback) -> bool: pass - -class BinaryIO(IO[str]): - # TODO readinto - # TODO read1? - # TODO peek? - @overload - @abstractmethod - def write(self, s: str) -> int: pass - @overload - @abstractmethod - def write(self, s: bytearray) -> int: pass - - @abstractmethod - def __enter__(self) -> BinaryIO: pass - -class TextIO(IO[unicode]): - # TODO use abstractproperty - @property - def buffer(self) -> BinaryIO: pass - @property - def encoding(self) -> str: pass - @property - def errors(self) -> str: pass - @property - def line_buffering(self) -> bool: pass - @property - def newlines(self) -> Any: pass # None, str or tuple - @abstractmethod - def __enter__(self) -> TextIO: pass - -class Match(Generic[AnyStr]): - pos = 0 - endpos = 0 - lastindex = 0 - lastgroup = Undefined(AnyStr) - string = Undefined(AnyStr) - - # The regular expression object whose match() or search() method produced - # this match instance. - re = Undefined('Pattern[AnyStr]') - - def expand(self, template: AnyStr) -> AnyStr: pass - - @overload - def group(self, group1: int = 0) -> AnyStr: pass - @overload - def group(self, group1: str) -> AnyStr: pass - @overload - def group(self, group1: int, group2: int, - *groups: int) -> Sequence[AnyStr]: pass - @overload - def group(self, group1: str, group2: str, - *groups: str) -> Sequence[AnyStr]: pass - - def groups(self, default: AnyStr = None) -> Sequence[AnyStr]: pass - def groupdict(self, default: AnyStr = None) -> dict[str, AnyStr]: pass - def start(self, group: int = 0) -> int: pass - def end(self, group: int = 0) -> int: pass - def span(self, group: int = 0) -> Tuple[int, int]: pass - -class Pattern(Generic[AnyStr]): - flags = 0 - groupindex = 0 - groups = 0 - pattern = Undefined(AnyStr) - - def search(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> Match[AnyStr]: pass - def match(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> Match[AnyStr]: pass - def split(self, string: AnyStr, maxsplit: int = 0) -> list[AnyStr]: pass - def findall(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> list[AnyStr]: pass - def finditer(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> Iterator[Match[AnyStr]]: pass - - @overload - def sub(self, repl: AnyStr, string: AnyStr, - count: int = 0) -> AnyStr: pass - @overload - def sub(self, repl: Function[[Match[AnyStr]], AnyStr], string: AnyStr, - count: int = 0) -> AnyStr: pass - - @overload - def subn(self, repl: AnyStr, string: AnyStr, - count: int = 0) -> Tuple[AnyStr, int]: pass - @overload - def subn(self, repl: Function[[Match[AnyStr]], AnyStr], string: AnyStr, - count: int = 0) -> Tuple[AnyStr, int]: pass diff --git a/stubs/3.2/__future__.py b/stubs/3.2/__future__.py deleted file mode 100644 index 4c20971f6bf7..000000000000 --- a/stubs/3.2/__future__.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Undefined - -class _Feature: pass - -absolute_import = Undefined(_Feature) -division = Undefined(_Feature) -generators = Undefined(_Feature) -nested_scopes = Undefined(_Feature) -print_function = Undefined(_Feature) -unicode_literals = Undefined(_Feature) -with_statement = Undefined(_Feature) diff --git a/stubs/3.2/_dummy_thread.py b/stubs/3.2/_dummy_thread.py deleted file mode 100644 index b53256224f8b..000000000000 --- a/stubs/3.2/_dummy_thread.py +++ /dev/null @@ -1,11 +0,0 @@ -# Stubs for _dummy_thread - -# NOTE: These are incomplete! - -from typing import Undefined, Any - -class LockType: - def acquire(self) -> None: pass - def release(self) -> None: pass - -def allocate_lock() -> LockType: pass diff --git a/stubs/3.2/_posixsubprocess.py b/stubs/3.2/_posixsubprocess.py deleted file mode 100644 index 52c49446d373..000000000000 --- a/stubs/3.2/_posixsubprocess.py +++ /dev/null @@ -1,13 +0,0 @@ -# Stubs for _posixsubprocess - -# NOTE: These are incomplete! - -from typing import Tuple, Sequence - -def cloexec_pipe() -> Tuple[int, int]: pass -def fork_exec(args: Sequence[str], - executable_list, close_fds, fds_to_keep, cwd: str, env_list, - p2cread: int, p2cwrite: int, c2pred: int, c2pwrite: int, - errread: int, errwrite: int, errpipe_read: int, - errpipe_write: int, restore_signals, start_new_session, - preexec_fn) -> int: pass diff --git a/stubs/3.2/_random.py b/stubs/3.2/_random.py deleted file mode 100644 index db1fda01bbac..000000000000 --- a/stubs/3.2/_random.py +++ /dev/null @@ -1,12 +0,0 @@ -# Stubs for _random - -# NOTE: These are incomplete! - -from typing import Any - -class Random: - def seed(self, x: Any = None) -> None: pass - def getstate(self) -> tuple: pass - def setstate(self, state: tuple) -> None: pass - def random(self) -> float: pass - def getrandbits(self, k: int) -> int: pass diff --git a/stubs/3.2/_subprocess.py b/stubs/3.2/_subprocess.py deleted file mode 100644 index 56437a724b83..000000000000 --- a/stubs/3.2/_subprocess.py +++ /dev/null @@ -1,38 +0,0 @@ -# Stubs for _subprocess - -# NOTE: These are incomplete! - -from typing import Mapping, Any, Tuple - -CREATE_NEW_CONSOLE = 0 -CREATE_NEW_PROCESS_GROUP = 0 -STD_INPUT_HANDLE = 0 -STD_OUTPUT_HANDLE = 0 -STD_ERROR_HANDLE = 0 -SW_HIDE = 0 -STARTF_USESTDHANDLES = 0 -STARTF_USESHOWWINDOW = 0 -INFINITE = 0 -DUPLICATE_SAME_ACCESS = 0 -WAIT_OBJECT_0 = 0 - -# TODO not exported by the Python module -class Handle: - def Close(self) -> None: pass - -def GetVersion() -> int: pass -def GetExitCodeProcess(handle: Handle) -> int: pass -def WaitForSingleObject(handle: Handle, timeout: int) -> int: pass -def CreateProcess(executable: str, cmd_line: str, - proc_attrs, thread_attrs, - inherit: int, flags: int, - env_mapping: Mapping[str, str], - curdir: str, - startupinfo: Any) -> Tuple[Any, Handle, int, int]: pass -def GetModuleFileName(module: int) -> str: pass -def GetCurrentProcess() -> Handle: pass -def DuplicateHandle(source_proc: Handle, source: Handle, target_proc: Handle, - target: Any, access: int, inherit: int) -> int: pass -def CreatePipe(pipe_attrs, size: int) -> Tuple[Handle, Handle]: pass -def GetStdHandle(arg: int) -> int: pass -def TerminateProcess(handle: Handle, exit_code: int) -> None: pass diff --git a/stubs/3.2/_thread.py b/stubs/3.2/_thread.py deleted file mode 100644 index e70c209631aa..000000000000 --- a/stubs/3.2/_thread.py +++ /dev/null @@ -1,14 +0,0 @@ -# Stubs for _thread - -# NOTE: These are incomplete! - -from typing import Undefined, Any - -def _count() -> int: pass -_dangling = Undefined(Any) - -class LockType: - def acquire(self) -> None: pass - def release(self) -> None: pass - -def allocate_lock() -> LockType: pass diff --git a/stubs/3.2/abc.py b/stubs/3.2/abc.py deleted file mode 100644 index b59a6fe3ef6a..000000000000 --- a/stubs/3.2/abc.py +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for abc. - -# Thesee definitions have special processing in type checker. -class ABCMeta: pass -abstractmethod = object() diff --git a/stubs/3.2/array.py b/stubs/3.2/array.py deleted file mode 100644 index 3bbc660abe71..000000000000 --- a/stubs/3.2/array.py +++ /dev/null @@ -1,50 +0,0 @@ -# Stubs for array - -# Based on http://docs.python.org/3.2/library/array.html - -from typing import Any, Iterable, Tuple, List, Iterator, BinaryIO, overload - -typecodes = '' - -class array: - def __init__(self, typecode: str, - initializer: Iterable[Any] = None) -> None: - typecode = '' - itemsize = 0 - - def append(self, x: Any) -> None: pass - def buffer_info(self) -> Tuple[int, int]: pass - def byteswap(self) -> None: pass - def count(self, x: Any) -> int: pass - def extend(self, iterable: Iterable[Any]) -> None: pass - def frombytes(self, s: bytes) -> None: pass - def fromfile(self, f: BinaryIO, n: int) -> None: pass - def fromlist(self, list: List[Any]) -> None: pass - def fromstring(self, s: bytes) -> None: pass - def fromunicode(self, s: str) -> None: pass - def index(self, x: Any) -> int: pass - def insert(self, i: int, x: Any) -> None: pass - def pop(self, i: int = -1) -> Any: pass - def remove(self, x: Any) -> None: pass - def reverse(self) -> None: pass - def tobytes(self) -> bytes: pass - def tofile(self, f: BinaryIO) -> None: pass - def tolist(self) -> List[Any]: pass - def tostring(self) -> bytes: pass - def tounicode(self) -> str: pass - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[Any]: pass - def __str__(self) -> str: pass - def __hash__(self) -> int: pass - - @overload - def __getitem__(self, i: int) -> Any: pass - @overload - def __getitem__(self, s: slice) -> 'array': pass - - def __setitem__(self, i: int, o: Any) -> None: pass - def __delitem__(self, i: int) -> None: pass - def __add__(self, x: 'array') -> 'array': pass - def __mul__(self, n: int) -> 'array': pass - def __contains__(self, o: object) -> bool: pass diff --git a/stubs/3.2/atexit.py b/stubs/3.2/atexit.py deleted file mode 100644 index 780965cd34ba..000000000000 --- a/stubs/3.2/atexit.py +++ /dev/null @@ -1,5 +0,0 @@ -from typing import typevar, Any - -_FT = typevar('_FT') - -def register(func: _FT, *args: Any, **kargs: Any) -> _FT: pass diff --git a/stubs/3.2/base64.py b/stubs/3.2/base64.py deleted file mode 100644 index f665f0224b95..000000000000 --- a/stubs/3.2/base64.py +++ /dev/null @@ -1,25 +0,0 @@ -# Stubs for base64 - -# Based on http://docs.python.org/3.2/library/base64.html - -from typing import IO - -def b64encode(s: bytes, altchars: bytes = None) -> bytes: pass -def b64decode(s: bytes, altchars: bytes = None, - validate: bool = False) -> bytes: pass -def standard_b64encode(s: bytes) -> bytes: pass -def standard_b64decode(s: bytes) -> bytes: pass -def urlsafe_b64encode(s: bytes) -> bytes: pass -def urlsafe_b64decode(s: bytes) -> bytes: pass -def b32encode(s: bytes) -> bytes: pass -def b32decode(s: bytes, casefold: bool = False, - map01: bytes = None) -> bytes: pass -def b16encode(s: bytes) -> bytes: pass -def b16decode(s: bytes, casefold: bool = False) -> bytes: pass - -def decode(input: IO[bytes], output: IO[bytes]) -> None: pass -def decodebytes(s: bytes) -> bytes: pass -def decodestring(s: bytes) -> bytes: pass -def encode(input: IO[bytes], output: IO[bytes]) -> None: pass -def encodebytes(s: bytes) -> bytes: pass -def encodestring(s: bytes) -> bytes: pass diff --git a/stubs/3.2/binascii.py b/stubs/3.2/binascii.py deleted file mode 100644 index 629e8fd65a57..000000000000 --- a/stubs/3.2/binascii.py +++ /dev/null @@ -1,26 +0,0 @@ -# Stubs for binascii - -# Based on http://docs.python.org/3.2/library/binascii.html - -import typing - -def a2b_uu(string: bytes) -> bytes: pass -def b2a_uu(data: bytes) -> bytes: pass -def a2b_base64(string: bytes) -> bytes: pass -def b2a_base64(data: bytes) -> bytes: pass -def a2b_qp(string: bytes, header: bool = False) -> bytes: pass -def b2a_qp(data: bytes, quotetabs: bool = False, istext: bool = True, - header: bool = False) -> bytes: pass -def a2b_hqx(string: bytes) -> bytes: pass -def rledecode_hqx(data: bytes) -> bytes: pass -def rlecode_hqx(data: bytes) -> bytes: pass -def b2a_hqx(data: bytes) -> bytes: pass -def crc_hqx(data: bytes, crc: int) -> int: pass -def crc32(data: bytes, crc: int = None) -> int: pass -def b2a_hex(data: bytes) -> bytes: pass -def hexlify(data: bytes) -> bytes: pass -def a2b_hex(hexstr: bytes) -> bytes: pass -def unhexlify(hexlify: bytes) -> bytes: pass - -class Error(Exception): pass -class Incomplete(Exception): pass diff --git a/stubs/3.2/builtins.py b/stubs/3.2/builtins.py deleted file mode 100644 index 3a814c915bc8..000000000000 --- a/stubs/3.2/builtins.py +++ /dev/null @@ -1,782 +0,0 @@ -# Stubs for builtins - -from typing import ( - Undefined, typevar, AbstractGeneric, Iterator, Iterable, overload, - Sequence, Mapping, Tuple, List, Any, Dict, Function, Generic, Set, - AbstractSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsAbs, - SupportsRound, IO, builtinclass, ducktype, Union -) -from abc import abstractmethod, ABCMeta - -# Note that names imported above are not automatically made visible via the -# implicit builtins import. - -_T = typevar('_T') -_KT = typevar('_KT') -_VT = typevar('_VT') -_S = typevar('_S') -_T1 = typevar('_T1') -_T2 = typevar('_T2') -_T3 = typevar('_T3') -_T4 = typevar('_T4') - -staticmethod = object() # Only valid as a decorator. -classmethod = object() # Only valid as a decorator. -property = object() - -_byte_types = Union[bytes, bytearray] - -@builtinclass -class object: - __doc__ = '' - __class__ = Undefined # type: type - - def __init__(self) -> None: pass - def __eq__(self, o: object) -> bool: pass - def __ne__(self, o: object) -> bool: pass - def __str__(self) -> str: pass - def __repr__(self) -> str: pass - def __hash__(self) -> int: pass - -@builtinclass -class type: - __name__ = '' - __module__ = '' - __dict__ = Undefined # type: Dict[str, Any] - - def __init__(self, o: object) -> None: pass - -@builtinclass -@ducktype(float) -class int(SupportsInt, SupportsFloat, SupportsAbs[int]): - def __init__(self, x: Union[SupportsInt, str, _byte_types]=None, base: int=None) -> None: pass - def bit_length(self) -> int: pass - def to_bytes(self, length: int, byteorder: str, *, signed: bool = False) -> bytes: pass - @classmethod - def from_bytes(cls, bytes: Sequence[int], byteorder: str, *, - signed: bool = False) -> int: pass # TODO buffer object argument - - def __add__(self, x: int) -> int: pass - def __sub__(self, x: int) -> int: pass - def __mul__(self, x: int) -> int: pass - def __floordiv__(self, x: int) -> int: pass - def __truediv__(self, x: int) -> float: pass - def __mod__(self, x: int) -> int: pass - def __radd__(self, x: int) -> int: pass - def __rsub__(self, x: int) -> int: pass - def __rmul__(self, x: int) -> int: pass - def __rfloordiv__(self, x: int) -> int: pass - def __rtruediv__(self, x: int) -> float: pass - def __rmod__(self, x: int) -> int: pass - def __pow__(self, x: int) -> Any: pass # Return type can be int or float, depending on x. - def __rpow__(self, x: int) -> Any: pass - def __and__(self, n: int) -> int: pass - def __or__(self, n: int) -> int: pass - def __xor__(self, n: int) -> int: pass - def __lshift__(self, n: int) -> int: pass - def __rshift__(self, n: int) -> int: pass - def __rand__(self, n: int) -> int: pass - def __ror__(self, n: int) -> int: pass - def __rxor__(self, n: int) -> int: pass - def __rlshift__(self, n: int) -> int: pass - def __rrshift__(self, n: int) -> int: pass - def __neg__(self) -> int: pass - def __pos__(self) -> int: pass - def __invert__(self) -> int: pass - - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: int) -> bool: pass - def __le__(self, x: int) -> bool: pass - def __gt__(self, x: int) -> bool: pass - def __ge__(self, x: int) -> bool: pass - - def __str__(self) -> str: pass - def __float__(self) -> float: pass - def __int__(self) -> int: return self - def __abs__(self) -> int: pass - def __hash__(self) -> int: pass - -@builtinclass -@ducktype(complex) -class float(SupportsFloat, SupportsInt, SupportsAbs[float]): - def __init__(self, x: Union[SupportsFloat, str, _byte_types]=None) -> None: pass - def as_integer_ratio(self) -> Tuple[int, int]: pass - def hex(self) -> str: pass - def is_integer(self) -> bool: pass - @classmethod - def fromhex(cls, s: str) -> float: pass - - def __add__(self, x: float) -> float: pass - def __sub__(self, x: float) -> float: pass - def __mul__(self, x: float) -> float: pass - def __floordiv__(self, x: float) -> float: pass - def __truediv__(self, x: float) -> float: pass - def __mod__(self, x: float) -> float: pass - def __pow__(self, x: float) -> float: pass - def __radd__(self, x: float) -> float: pass - def __rsub__(self, x: float) -> float: pass - def __rmul__(self, x: float) -> float: pass - def __rfloordiv__(self, x: float) -> float: pass - def __rtruediv__(self, x: float) -> float: pass - def __rmod__(self, x: float) -> float: pass - def __rpow__(self, x: float) -> float: pass - - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: float) -> bool: pass - def __le__(self, x: float) -> bool: pass - def __gt__(self, x: float) -> bool: pass - def __ge__(self, x: float) -> bool: pass - def __neg__(self) -> float: pass - def __pos__(self) -> float: pass - - def __str__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: pass - def __abs__(self) -> float: pass - def __hash__(self) -> int: pass - -@builtinclass -class complex(SupportsAbs[float]): - @overload - def __init__(self, re: float = 0.0, im: float = 0.0) -> None: pass - @overload - def __init__(self, s: str) -> None: pass - - @property - def real(self) -> float: pass - @property - def imag(self) -> float: pass - - def conjugate(self) -> complex: pass - - def __add__(self, x: complex) -> complex: pass - def __sub__(self, x: complex) -> complex: pass - def __mul__(self, x: complex) -> complex: pass - def __pow__(self, x: complex) -> complex: pass - def __truediv__(self, x: complex) -> complex: pass - def __radd__(self, x: complex) -> complex: pass - def __rsub__(self, x: complex) -> complex: pass - def __rmul__(self, x: complex) -> complex: pass - def __rpow__(self, x: complex) -> complex: pass - def __rtruediv__(self, x: complex) -> complex: pass - - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __neg__(self) -> complex: pass - def __pos__(self) -> complex: pass - - def __str__(self) -> str: pass - def __abs__(self) -> float: pass - def __hash__(self) -> int: pass - -@builtinclass -class str(Sequence[str]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, o: object) -> None: pass - @overload - def __init__(self, o: _byte_types, encoding: str = None, errors: str = 'strict') -> None: pass - def capitalize(self) -> str: pass - def center(self, width: int, fillchar: str = ' ') -> str: pass - def count(self, x: str) -> int: pass - def encode(self, encoding: str = 'utf-8', errors: str = 'strict') -> bytes: pass - def endswith(self, suffix: str, start: int = 0, - end: int = None) -> bool: pass # TODO tuple suffix; None value for int - def expandtabs(self, tabsize: int = 8) -> str: pass - def find(self, sub: str, start: int = 0, end: int = 0) -> int: pass - def format(self, *args: Any, **kwargs: Any) -> str: pass - def format_map(self, map: Mapping[str, Any]) -> str: pass - def index(self, sub: str, start: int = 0, end: int = 0) -> int: pass - def isalnum(self) -> bool: pass - def isalpha(self) -> bool: pass - def isdecimal(self) -> bool: pass - def isdigit(self) -> bool: pass - def isidentifier(self) -> bool: pass - def islower(self) -> bool: pass - def isnumeric(self) -> bool: pass - def isprintable(self) -> bool: pass - def isspace(self) -> bool: pass - def istitle(self) -> bool: pass - def isupper(self) -> bool: pass - def join(self, iterable: Iterable[str]) -> str: pass - def ljust(self, width: int, fillchar: str = ' ') -> str: pass - def lower(self) -> str: pass - def lstrip(self, chars: str = None) -> str: pass - def partition(self, sep: str) -> Tuple[str, str, str]: pass - def replace(self, old: str, new: str, count: int = -1) -> str: pass - def rfind(self, sub: str, start: int = 0, end: int = 0) -> int: pass - def rindex(self, sub: str, start: int = 0, end: int = 0) -> int: pass - def rjust(self, width: int, fillchar: str = ' ') -> str: pass - def rpartition(self, sep: str) -> Tuple[str, str, str]: pass - def rsplit(self, sep: str = None, maxsplit: int = -1) -> List[str]: pass - def rstrip(self, chars: str = None) -> str: pass - def split(self, sep: str = None, maxsplit: int = -1) -> List[str]: pass - def splitlines(self, keepends: bool = False) -> List[str]: pass - def startswith(self, prefix: str, start: int = 0, - end: int = None) -> bool: pass # TODO tuple prefix; None value for int - def strip(self, chars: str = None) -> str: pass - def swapcase(self) -> str: pass - def title(self) -> str: pass - def translate(self, table: Dict[int, Any]) -> str: pass - def upper(self) -> str: pass - def zfill(self, width: int) -> str: pass - # TODO maketrans - - def __getitem__(self, i: Union[int, slice]) -> str: pass - def __add__(self, s: str) -> str: pass - def __mul__(self, n: int) -> str: pass - def __rmul__(self, n: int) -> str: pass - def __mod__(self, *args: Any) -> str: pass - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: str) -> bool: pass - def __le__(self, x: str) -> bool: pass - def __gt__(self, x: str) -> bool: pass - def __ge__(self, x: str) -> bool: pass - - def __len__(self) -> int: pass - def __contains__(self, s: object) -> bool: pass - def __iter__(self) -> Iterator[str]: pass - def __str__(self) -> str: return self - def __repr__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: pass - def __hash__(self) -> int: pass - -@builtinclass -class bytes(Sequence[int]): - @overload - def __init__(self, ints: Iterable[int]) -> None: pass - @overload - def __init__(self, string: str, encoding: str, - errors: str = 'strict') -> None: pass - @overload - def __init__(self, length: int) -> None: pass - @overload - def __init__(self) -> None: pass - def capitalize(self) -> bytes: pass - def center(self, width: int, fillchar: _byte_types = None) -> bytes: pass - def count(self, x: _byte_types) -> int: pass - def decode(self, encoding: str = 'utf-8', errors: str = 'strict') -> str: pass - def endswith(self, suffix: _byte_types) -> bool: pass - def expandtabs(self, tabsize: int = 8) -> bytes: pass - def find(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def index(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def isalnum(self) -> bool: pass - def isalpha(self) -> bool: pass - def isdigit(self) -> bool: pass - def islower(self) -> bool: pass - def isspace(self) -> bool: pass - def istitle(self) -> bool: pass - def isupper(self) -> bool: pass - def join(self, iterable: Iterable[_byte_types]) -> bytes: pass - def ljust(self, width: int, fillchar: _byte_types = None) -> bytes: pass - def lower(self) -> bytes: pass - def lstrip(self, chars: _byte_types = None) -> bytes: pass - def partition(self, sep: _byte_types) -> Tuple[bytes, bytes, bytes]: pass - def replace(self, old: _byte_types, new: _byte_types, count: int = -1) -> bytes: pass - def rfind(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def rindex(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def rjust(self, width: int, fillchar: _byte_types = None) -> bytes: pass - def rpartition(self, sep: _byte_types) -> Tuple[bytes, bytes, bytes]: pass - def rsplit(self, sep: _byte_types = None, maxsplit: int = -1) -> List[bytes]: pass - def rstrip(self, chars: _byte_types = None) -> bytes: pass - def split(self, sep: _byte_types = None, maxsplit: int = -1) -> List[bytes]: pass - def splitlines(self, keepends: bool = False) -> List[bytes]: pass - def startswith(self, prefix: _byte_types) -> bool: pass - def strip(self, chars: _byte_types = None) -> bytes: pass - def swapcase(self) -> bytes: pass - def title(self) -> bytes: pass - def translate(self, table: _byte_types) -> bytes: pass - def upper(self) -> bytes: pass - def zfill(self, width: int) -> bytes: pass - # TODO fromhex - # TODO maketrans - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[int]: pass - def __str__(self) -> str: pass - def __repr__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: pass - def __hash__(self) -> int: pass - @overload - def __getitem__(self, i: int) -> int: pass - @overload - def __getitem__(self, s: slice) -> bytes: pass - def __add__(self, s: _byte_types) -> bytes: pass - def __mul__(self, n: int) -> bytes: pass - def __rmul__(self, n: int) -> bytes: pass - def __contains__(self, o: object) -> bool: pass - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: bytes) -> bool: pass - def __le__(self, x: bytes) -> bool: pass - def __gt__(self, x: bytes) -> bool: pass - def __ge__(self, x: bytes) -> bool: pass - -@builtinclass -class bytearray(Sequence[int]): - @overload - def __init__(self, ints: Iterable[int]) -> None: pass - @overload - def __init__(self, string: str, encoding: str, errors: str = 'strict') -> None: pass - @overload - def __init__(self, length: int) -> None: pass - @overload - def __init__(self) -> None: pass - def capitalize(self) -> bytearray: pass - def center(self, width: int, fillchar: _byte_types = None) -> bytearray: pass - def count(self, x: _byte_types) -> int: pass - def decode(self, encoding: str = 'utf-8', errors: str = 'strict') -> str: pass - def endswith(self, suffix: _byte_types) -> bool: pass - def expandtabs(self, tabsize: int = 8) -> bytearray: pass - def find(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def index(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def isalnum(self) -> bool: pass - def isalpha(self) -> bool: pass - def isdigit(self) -> bool: pass - def islower(self) -> bool: pass - def isspace(self) -> bool: pass - def istitle(self) -> bool: pass - def isupper(self) -> bool: pass - def join(self, iterable: Iterable[_byte_types]) -> bytearray: pass - def ljust(self, width: int, fillchar: _byte_types = None) -> bytearray: pass - def lower(self) -> bytearray: pass - def lstrip(self, chars: _byte_types = None) -> bytearray: pass - def partition(self, sep: _byte_types) -> Tuple[bytearray, bytearray, bytearray]: pass - def replace(self, old: _byte_types, new: bytes, count: int = -1) -> bytearray: pass - def rfind(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def rindex(self, sub: _byte_types, start: int = 0, end: int = 0) -> int: pass - def rjust(self, width: int, fillchar: _byte_types = None) -> bytearray: pass - def rpartition(self, sep: _byte_types) -> Tuple[bytearray, bytearray, bytearray]: pass - def rsplit(self, sep: _byte_types = None, maxsplit: int = -1) -> List[bytearray]: pass - def rstrip(self, chars: _byte_types = None) -> bytearray: pass - def split(self, sep: _byte_types = None, maxsplit: int = -1) -> List[bytearray]: pass - def splitlines(self, keepends: bool = False) -> List[bytearray]: pass - def startswith(self, prefix: _byte_types) -> bool: pass - def strip(self, chars: _byte_types = None) -> bytearray: pass - def swapcase(self) -> bytearray: pass - def title(self) -> bytearray: pass - def translate(self, table: _byte_types) -> bytearray: pass - def upper(self) -> bytearray: pass - def zfill(self, width: int) -> bytearray: pass - # TODO fromhex - # TODO maketrans - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[int]: pass - def __str__(self) -> str: pass - def __repr__(self) -> str: pass - def __int__(self) -> int: pass - def __float__(self) -> float: pass - def __hash__(self) -> int: pass - @overload - def __getitem__(self, i: int) -> int: pass - @overload - def __getitem__(self, s: slice) -> bytearray: pass - @overload - def __setitem__(self, i: int, x: int) -> None: pass - @overload - def __setitem__(self, s: slice, x: Sequence[int]) -> None: pass - @overload - def __delitem__(self, i: int) -> None: pass - @overload - def __delitem__(self, s: slice) -> None: pass - def __add__(self, s: _byte_types) -> bytearray: pass - def __iadd__(self, s: _byte_types) -> bytearray: pass - def __mul__(self, n: int) -> bytearray: pass - def __rmul__(self, n: int) -> bytearray: pass - def __imul__(self, n: int) -> bytearray: pass - def __contains__(self, o: object) -> bool: pass - def __eq__(self, x: object) -> bool: pass - def __ne__(self, x: object) -> bool: pass - def __lt__(self, x: _byte_types) -> bool: pass - def __le__(self, x: _byte_types) -> bool: pass - def __gt__(self, x: _byte_types) -> bool: pass - def __ge__(self, x: _byte_types) -> bool: pass - -@builtinclass -class bool(int, SupportsInt, SupportsFloat): - def __init__(self, o: object = False) -> None: pass - -@builtinclass -class slice: - start = 0 - step = 0 - stop = 0 - def __init__(self, start: int, stop: int, step: int) -> None: pass - -@builtinclass -class tuple(Sequence[Any]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, iterable: Iterable[Any]) -> None: pass - @overload - def __init__(self, iterable: tuple) -> None: pass - def __len__(self) -> int: pass - def __contains__(self, x: object) -> bool: pass - @overload - def __getitem__(self, x: int) -> Any: pass - @overload - def __getitem__(self, x: slice) -> tuple: pass - def __iter__(self) -> Iterator[Any]: pass - def __lt__(self, x: tuple) -> bool: pass - def __le__(self, x: tuple) -> bool: pass - def __gt__(self, x: tuple) -> bool: pass - def __ge__(self, x: tuple) -> bool: pass - def __add__(self, x: tuple) -> tuple: pass - def count(self, x: Any) -> int: pass - def index(self, x: Any) -> int: pass - -@builtinclass -class function: - # TODO not defined in builtins! - __name__ = '' - __module__ = '' - __code__ = Undefined(Any) - -@builtinclass -class list(Sequence[_T], Reversible[_T], AbstractGeneric[_T]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, iterable: Iterable[_T]) -> None: pass - def clear(self) -> None: pass - def copy(self) -> List[_T]: pass - def append(self, object: _T) -> None: pass - def extend(self, iterable: Iterable[_T]) -> None: pass - def pop(self, index: int = -1) -> _T: pass - def index(self, object: _T, start: int = 0, stop: int = Undefined(int)) -> int: pass - def count(self, object: _T) -> int: pass - def insert(self, index: int, object: _T) -> None: pass - def remove(self, object: _T) -> None: pass - def reverse(self) -> None: pass - def sort(self, *, key: Function[[_T], Any] = None, reverse: bool = False) -> None: pass - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[_T]: pass - def __str__(self) -> str: pass - def __hash__(self) -> int: pass - @overload - def __getitem__(self, i: int) -> _T: pass - @overload - def __getitem__(self, s: slice) -> List[_T]: pass - @overload - def __setitem__(self, i: int, o: _T) -> None: pass - @overload - def __setitem__(self, s: slice, o: Sequence[_T]) -> None: pass - @overload - def __delitem__(self, i: int) -> None: pass - @overload - def __delitem__(self, s: slice) -> None: pass - def __add__(self, x: List[_T]) -> List[_T]: pass - def __iadd__(self, x: Iterable[_T]) -> List[_T]: pass - def __mul__(self, n: int) -> List[_T]: pass - def __rmul__(self, n: int) -> List[_T]: pass - def __imul__(self, n: int) -> List[_T]: pass - def __contains__(self, o: object) -> bool: pass - def __reversed__(self) -> Iterator[_T]: pass - def __gt__(self, x: List[_T]) -> bool: pass - def __ge__(self, x: List[_T]) -> bool: pass - def __lt__(self, x: List[_T]) -> bool: pass - def __le__(self, x: List[_T]) -> bool: pass - -@builtinclass -class dict(Mapping[_KT, _VT], Generic[_KT, _VT]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, map: Mapping[_KT, _VT]) -> None: pass - @overload - def __init__(self, iterable: Iterable[Tuple[_KT, _VT]]) -> None: pass # TODO keyword args - def clear(self) -> None: pass - def copy(self) -> Dict[_KT, _VT]: pass - def get(self, k: _KT, default: _VT=None) -> _VT: pass - def pop(self, k: _KT, default: _VT=None) -> _VT: pass - def popitem(self) -> Tuple[_KT, _VT]: pass - def setdefault(self, k: _KT, default: _VT=None) -> _VT: pass - @overload - def update(self, m: Mapping[_KT, _VT]) -> None: pass - @overload - def update(self, m: Iterable[Tuple[_KT, _VT]]) -> None: pass - def keys(self) -> Set[_KT]: pass - def values(self) -> Set[_VT]: pass - def items(self) -> Set[Tuple[_KT, _VT]]: pass - @staticmethod - @overload - def fromkeys(seq: Sequence[_T]) -> Dict[_T, Any]: pass # TODO: Actually a class method - @staticmethod - @overload - def fromkeys(seq: Sequence[_T], value: _S) -> Dict[_T, _S]: pass - def __len__(self) -> int: pass - def __getitem__(self, k: _KT) -> _VT: pass - def __setitem__(self, k: _KT, v: _VT) -> None: pass - def __delitem__(self, v: _KT) -> None: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[_KT]: pass - def __str__(self) -> str: pass - -@builtinclass -class set(AbstractSet[_T], Generic[_T]): - def __init__(self, iterable: Iterable[_T]=None) -> None: pass - def add(self, element: _T) -> None: pass - def clear(self) -> None: pass - def copy(self) -> set[_T]: pass - def difference(self, s: Iterable[Any]) -> set[_T]: pass - def difference_update(self, s: Iterable[Any]) -> None: pass - def discard(self, element: _T) -> None: pass - def intersection(self, s: Iterable[Any]) -> set[_T]: pass - def intersection_update(self, s: Iterable[Any]) -> None: pass - def isdisjoint(self, s: AbstractSet[Any]) -> bool: pass - def issubset(self, s: AbstractSet[Any]) -> bool: pass - def issuperset(self, s: AbstractSet[Any]) -> bool: pass - def pop(self) -> _T: pass - def remove(self, element: _T) -> None: pass - def symmetric_difference(self, s: Iterable[_T]) -> set[_T]: pass - def symmetric_difference_update(self, s: Iterable[_T]) -> None: pass - def union(self, s: Iterable[_T]) -> set[_T]: pass - def update(self, s: Iterable[_T]) -> None: pass - def __len__(self) -> int: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[_T]: pass - def __str__(self) -> str: pass - def __and__(self, s: AbstractSet[Any]) -> set[_T]: pass - def __iand__(self, s: AbstractSet[Any]) -> set[_T]: pass - def __or__(self, s: AbstractSet[_T]) -> set[_T]: pass - def __ior__(self, s: AbstractSet[_T]) -> set[_T]: pass - def __sub__(self, s: AbstractSet[Any]) -> set[_T]: pass - def __isub__(self, s: AbstractSet[Any]) -> set[_T]: pass - def __xor__(self, s: AbstractSet[_T]) -> set[_T]: pass - def __ixor__(self, s: AbstractSet[_T]) -> set[_T]: pass - def __le__(self, s: AbstractSet[Any]) -> bool: pass - def __lt__(self, s: AbstractSet[Any]) -> bool: pass - def __ge__(self, s: AbstractSet[Any]) -> bool: pass - def __gt__(self, s: AbstractSet[Any]) -> bool: pass - # TODO more set operations - -@builtinclass -class frozenset(AbstractSet[_T], Generic[_T]): - def __init__(self, iterable: Iterable[_T]=None) -> None: pass - def copy(self) -> frozenset[_T]: pass - def difference(self, s: AbstractSet[Any]) -> frozenset[_T]: pass - def intersection(self, s: AbstractSet[Any]) -> frozenset[_T]: pass - def isdisjoint(self, s: AbstractSet[_T]) -> bool: pass - def issubset(self, s: AbstractSet[Any]) -> bool: pass - def issuperset(self, s: AbstractSet[Any]) -> bool: pass - def symmetric_difference(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def union(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __len__(self) -> int: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[_T]: pass - def __str__(self) -> str: pass - def __and__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __or__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __sub__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __xor__(self, s: AbstractSet[_T]) -> frozenset[_T]: pass - def __le__(self, s: AbstractSet[Any]) -> bool: pass - def __lt__(self, s: AbstractSet[Any]) -> bool: pass - def __ge__(self, s: AbstractSet[Any]) -> bool: pass - def __gt__(self, s: AbstractSet[Any]) -> bool: pass - -@builtinclass -class enumerate(Iterator[Tuple[int, _T]], Generic[_T]): - def __init__(self, iterable: Iterable[_T], start: int = 0) -> None: pass - def __iter__(self) -> Iterator[Tuple[int, _T]]: pass - def __next__(self) -> Tuple[int, _T]: pass - # TODO __getattribute__ - -@builtinclass -class range(Sequence[int], Reversible[int]): - @overload - def __init__(self, stop: int) -> None: pass - @overload - def __init__(self, start: int, stop: int, step: int = 1) -> None: pass - def count(self, value: int) -> int: pass - def index(self, value: int, start: int = 0, stop: int = None) -> int: pass - def __len__(self) -> int: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[int]: pass - @overload - def __getitem__(self, i: int) -> int: pass - @overload - def __getitem__(self, s: slice) -> range: pass - def __repr__(self) -> str: pass - def __reversed__(self) -> Iterator[int]: pass - -@builtinclass -class module: - # TODO not defined in builtins! - __name__ = '' - __file__ = '' - __dict__ = Undefined # type: Dict[str, Any] - -True = Undefined # type: bool -False = Undefined # type: bool -__debug__ = False - -NotImplemented = Undefined # type: Any - -def abs(n: SupportsAbs[_T]) -> _T: pass -def all(i: Iterable) -> bool: pass -def any(i: Iterable) -> bool: pass -def ascii(o: object) -> str: pass -def callable(o: object) -> bool: pass -def chr(code: int) -> str: pass -def delattr(o: Any, name: str) -> None: pass -def dir(o: object = None) -> List[str]: pass -_N = typevar('_N', values=(int, float)) -def divmod(a: _N, b: _N) -> Tuple[_N, _N]: pass -def eval(source: str, globals: Dict[str, Any] = None, - locals: Mapping[str, Any] = None) -> Any: pass # TODO code object as source -def filter(function: Function[[_T], Any], iterable: Iterable[_T]) -> Iterator[_T]: pass -def format(o: object, format_spec: str = '') -> str: pass -def getattr(o: Any, name: str, default: Any = None) -> Any: pass -def globals() -> Dict[str, Any]: pass -def hasattr(o: Any, name: str) -> bool: pass -def hash(o: object) -> int: pass -def hex(i: int) -> str: pass # TODO __index__ -def id(o: object) -> int: pass -def input(prompt: str = None) -> str: pass -@overload -def iter(iterable: Iterable[_T]) -> Iterator[_T]: pass -@overload -def iter(function: Function[[], _T], sentinel: _T) -> Iterator[_T]: pass -def isinstance(o: object, t: Union[type, tuple]) -> bool: pass -def issubclass(cls: type, classinfo: type) -> bool: pass -# TODO support this -#def issubclass(type cld, classinfo: Sequence[type]) -> bool: pass -@overload -def len(o: Sized) -> int: pass -@overload -def len(o: tuple) -> int: pass -def locals() -> Dict[str, Any]: pass -@overload -def map(func: Function[[_T1], _S], iter1: Iterable[_T1]) -> Iterator[_S]: pass -@overload -def map(func: Function[[_T1, _T2], _S], iter1: Iterable[_T1], - iter2: Iterable[_T2]) -> Iterator[_S]: pass # TODO more than two iterables -@overload -def max(iterable: Iterable[_T]) -> _T: pass # TODO keyword argument key -@overload -def max(arg1: _T, arg2: _T, *args: _T) -> _T: pass -# TODO memoryview -@overload -def min(iterable: Iterable[_T]) -> _T: pass -@overload -def min(arg1: _T, arg2: _T, *args: _T) -> _T: pass -@overload -def next(i: Iterator[_T]) -> _T: pass -@overload -def next(i: Iterator[_T], default: _T) -> _T: pass -def oct(i: int) -> str: pass # TODO __index__ -def open(file: Union[str, bytes, int], mode: str = 'r', buffering: int = -1, encoding: str = None, - errors: str = None, newline: str = None, closefd: bool = True) -> IO[Any]: pass -def ord(c: Union[str, bytes, bytearray]) -> int: pass -def print(*values: Any, sep: str = ' ', end: str = '\n', file: IO[str] = None) -> None: pass -@overload -def pow(x: int, y: int) -> Any: pass # The return type can be int or float, depending on y -@overload -def pow(x: int, y: int, z: int) -> Any: pass -@overload -def pow(x: float, y: float) -> float: pass -@overload -def pow(x: float, y: float, z: float) -> float: pass -@overload -def reversed(object: Reversible[_T]) -> Iterator[_T]: pass -@overload -def reversed(object: Sequence[_T]) -> Iterator[_T]: pass -def repr(o: object) -> str: pass -@overload -def round(number: float) -> int: pass -@overload -def round(number: float, ndigits: int) -> float: pass # Always return a float if given ndigits. -@overload -def round(number: SupportsRound[_T]) -> _T: pass -@overload -def round(number: SupportsRound[_T], ndigits: int) -> _T: pass -def setattr(object: Any, name: str, value: Any) -> None: pass -def sorted(iterable: Iterable[_T], *, key: Function[[_T], Any] = None, - reverse: bool = False) -> List[_T]: pass -def sum(iterable: Iterable[_T], start: _T = None) -> _T: pass -@overload -def zip(iter1: Iterable[_T1]) -> Iterator[Tuple[_T1]]: pass -@overload -def zip(iter1: Iterable[_T1], iter2: Iterable[_T2]) -> Iterator[Tuple[_T1, _T2]]: pass -@overload -def zip(iter1: Iterable[_T1], iter2: Iterable[_T2], - iter3: Iterable[_T3]) -> Iterator[Tuple[_T1, _T2, _T3]]: pass -@overload -def zip(iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], - iter4: Iterable[_T4]) -> Iterator[Tuple[_T1, _T2, - _T3, _T4]]: pass # TODO more than four iterables -def __import__(name: str, globals: Dict[str, Any] = {}, locals: Dict[str, Any] = {}, - fromlist: List[str] = [], level: int = -1) -> Any: pass - -# Exceptions - -@builtinclass -class BaseException: - args = Undefined # type: Any - def __init__(self, *args: Any) -> None: pass - def with_traceback(self, tb: Any) -> BaseException: pass - -class GeneratorExit(BaseException): pass -class KeyboardInterrupt(BaseException): pass -@builtinclass -class SystemExit(BaseException): - code = 0 -class Exception(BaseException): pass -class ArithmeticError(Exception): pass -@builtinclass -class EnvironmentError(Exception): - errno = 0 - strerror = '' - filename = '' # TODO can this be bytes? -class LookupError(Exception): pass -class RuntimeError(Exception): pass -class ValueError(Exception): pass -class AssertionError(Exception): pass -class AttributeError(Exception): pass -class EOFError(Exception): pass -class FloatingPointError(ArithmeticError): pass -class IOError(EnvironmentError): pass -class ImportError(Exception): pass -class IndexError(LookupError): pass -class KeyError(LookupError): pass -class MemoryError(Exception): pass -class NameError(Exception): pass -class NotImplementedError(RuntimeError): pass -class OSError(EnvironmentError): pass -class WindowsError(OSError): pass # TODO Windows-only -# TODO: VMSError? -class OverflowError(ArithmeticError): pass -class ReferenceError(Exception): pass -class StopIteration(Exception): pass -class SyntaxError(Exception): pass -class IndentationError(SyntaxError): pass -class TabError(IndentationError): pass -class SystemError(Exception): pass -class TypeError(Exception): pass -class UnboundLocalError(NameError): pass -class UnicodeError(ValueError): pass -class UnicodeDecodeError(UnicodeError): pass -class UnicodeEncodeError(UnicodeError): pass -class UnicodeTranslateError(UnicodeError): pass -class ZeroDivisionError(ArithmeticError): pass - -class Warning(Exception): pass -class UserWarning(Warning): pass -class DeprecationWarning(Warning): pass -class SyntaxWarning(Warning): pass -class RuntimeWarning(Warning): pass -class FutureWarning(Warning): pass -class PendingDeprecationWarning(Warning): pass -class ImportWarning(Warning): pass -class UnicodeWarning(Warning): pass -class BytesWarning(Warning): pass -class ResourceWarning(Warning): pass diff --git a/stubs/3.2/calendar.py b/stubs/3.2/calendar.py deleted file mode 100644 index b9d5f9210069..000000000000 --- a/stubs/3.2/calendar.py +++ /dev/null @@ -1,15 +0,0 @@ -# Stubs for calendar - -# NOTE: These are incomplete! - -from typing import overload, Tuple - -# TODO actually, any number of items larger than 5 is fine -@overload -def timegm(t: Tuple[int, int, int, int, int, int]) -> int: pass -@overload -def timegm(t: Tuple[int, int, int, int, int, int, int]) -> int: pass -@overload -def timegm(t: Tuple[int, int, int, int, int, int, int, int]) -> int: pass -@overload -def timegm(t: Tuple[int, int, int, int, int, int, int, int, int]) -> int: pass diff --git a/stubs/3.2/cgi.py b/stubs/3.2/cgi.py deleted file mode 100644 index 715d9d6a5946..000000000000 --- a/stubs/3.2/cgi.py +++ /dev/null @@ -1 +0,0 @@ -def escape(s: str) -> str: pass diff --git a/stubs/3.2/codecs.py b/stubs/3.2/codecs.py deleted file mode 100644 index 13f7c36effc5..000000000000 --- a/stubs/3.2/codecs.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Function - -BOM_UTF8 = b'' - -class CodecInfo(tuple): - def __init__(self, *args) -> None: pass - -def register(search_function: Function[[str], CodecInfo]) -> None: - pass - -def lookup(encoding: str) -> CodecInfo: - pass diff --git a/stubs/3.2/collections.py b/stubs/3.2/collections.py deleted file mode 100644 index cae8109d215c..000000000000 --- a/stubs/3.2/collections.py +++ /dev/null @@ -1,102 +0,0 @@ -# Stubs for collections - -# Based on http://docs.python.org/3.2/library/collections.html - -# TODO UserDict -# TODO UserList -# TODO UserString -# TODO more abstract base classes (interfaces in mypy) - -from typing import ( - typevar, Iterable, AbstractGeneric, Iterator, Dict, Generic, overload, - Mapping, List, Tuple, Undefined, Function, Set, Sequence, Sized -) - -_T = typevar('_T') -_KT = typevar('_KT') -_VT = typevar('_VT') - - -# namedtuple is special-cased in the type checker; the initializer is ignored. -namedtuple = object() - - -class deque(Sized, Iterable[_T], AbstractGeneric[_T]): - # TODO int with None default - maxlen = 0 # TODO readonly - def __init__(self, iterable: Iterable[_T] = None, - maxlen: int = None) -> None: pass - def append(self, x: _T) -> None: pass - def appendleft(self, x: _T) -> None: pass - def clear(self) -> None: pass - def count(self, x: _T) -> int: pass - def extend(self, iterable: Iterable[_T]) -> None: pass - def extendleft(self, iterable: Iterable[_T]) -> None: pass - def pop(self) -> _T: pass - def popleft(self) -> _T: pass - def remove(self, value: _T) -> None: pass - def reverse(self) -> None: pass - def rotate(self, n: int) -> None: pass - - def __len__(self) -> int: pass - def __iter__(self) -> Iterator[_T]: pass - def __str__(self) -> str: pass - def __hash__(self) -> int: pass - - def __getitem__(self, i: int) -> _T: pass - def __setitem__(self, i: int, x: _T) -> None: pass - def __contains__(self, o: _T) -> bool: pass - - # TODO __reversed__ - - -class Counter(Dict[_T, int], Generic[_T]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, Mapping: Mapping[_T, int]) -> None: pass - @overload - def __init__(self, iterable: Iterable[_T]) -> None: pass - # TODO keyword arguments - - def elements(self) -> Iterator[_T]: pass - - @overload - def most_common(self) -> List[_T]: pass - @overload - def most_common(self, n: int) -> List[_T]: pass - - @overload - def subtract(self, Mapping: Mapping[_T, int]) -> None: pass - @overload - def subtract(self, iterable: Iterable[_T]) -> None: pass - - # TODO update - - -class OrderedDict(Dict[_KT, _VT], Generic[_KT, _VT]): - def popitem(self, last: bool = True) -> Tuple[_KT, _VT]: pass - def move_to_end(self, key: _KT, last: bool = True) -> None: pass - - -class defaultdict(Dict[_KT, _VT], Generic[_KT, _VT]): - default_factory = Undefined(Function[[], _VT]) - - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, map: Mapping[_KT, _VT]) -> None: pass - @overload - def __init__(self, iterable: Iterable[Tuple[_KT, _VT]]) -> None: pass - @overload - def __init__(self, default_factory: Function[[], _VT]) -> None: pass - @overload - def __init__(self, default_factory: Function[[], _VT], - map: Mapping[_KT, _VT]) -> None: pass - @overload - def __init__(self, default_factory: Function[[], _VT], - iterable: Iterable[Tuple[_KT, _VT]]) -> None: pass - # TODO __init__ keyword args - - def __missing__(self, key: _KT) -> _VT: pass - # TODO __reversed__ diff --git a/stubs/3.2/contextlib.py b/stubs/3.2/contextlib.py deleted file mode 100644 index 6261f682b19b..000000000000 --- a/stubs/3.2/contextlib.py +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for contextlib - -# NOTE: These are incomplete! - -from typing import Any - -# TODO more precise type? -def contextmanager(func: Any) -> Any: pass diff --git a/stubs/3.2/copy.py b/stubs/3.2/copy.py deleted file mode 100644 index 2208a8428190..000000000000 --- a/stubs/3.2/copy.py +++ /dev/null @@ -1,9 +0,0 @@ -# Stubs for copy - -# NOTE: These are incomplete! - -from typing import typevar - -_T = typevar('_T') - -def deepcopy(x: _T) -> _T: pass diff --git a/stubs/3.2/distutils/errors.py b/stubs/3.2/distutils/errors.py deleted file mode 100644 index fbb4e4ac4f7b..000000000000 --- a/stubs/3.2/distutils/errors.py +++ /dev/null @@ -1,4 +0,0 @@ -import typing - -class DistutilsError(Exception): pass -class DistutilsExecError(DistutilsError): pass diff --git a/stubs/3.2/distutils/spawn.py b/stubs/3.2/distutils/spawn.py deleted file mode 100644 index 6dc7fc30b31c..000000000000 --- a/stubs/3.2/distutils/spawn.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import List - -# In Python, arguments have integer default values -def spawn(cmd: List[str], search_path: bool = True, verbose: bool = False, - dry_run: bool = False) -> None: pass -def find_executable(executable: str, path: str = None) -> str: pass diff --git a/stubs/3.2/doctest.py b/stubs/3.2/doctest.py deleted file mode 100644 index 797232b78d60..000000000000 --- a/stubs/3.2/doctest.py +++ /dev/null @@ -1,9 +0,0 @@ -# Stubs for doctest - -# NOTE: These are incomplete! - -from typing import Any, Tuple - -# TODO arguments missing -def testmod(module: Any = None, *, name: str = None, globs: Any = None, - verbose: bool = None) -> Tuple[int, int]: pass diff --git a/stubs/3.2/docutils/__init__.py b/stubs/3.2/docutils/__init__.py deleted file mode 100644 index 2ae28399f5fd..000000000000 --- a/stubs/3.2/docutils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -pass diff --git a/stubs/3.2/docutils/examples.py b/stubs/3.2/docutils/examples.py deleted file mode 100644 index 9f960daf08d1..000000000000 --- a/stubs/3.2/docutils/examples.py +++ /dev/null @@ -1,3 +0,0 @@ -from typing import Undefined, Any - -html_parts = Undefined(Any) diff --git a/stubs/3.2/docutils/nodes.py b/stubs/3.2/docutils/nodes.py deleted file mode 100644 index 8be72e562294..000000000000 --- a/stubs/3.2/docutils/nodes.py +++ /dev/null @@ -1,8 +0,0 @@ -from typing import Any, List - -class reference: - def __init__(self, - rawsource: str = '', - text: str = '', - *children: List[Any], - **attributes) -> None: pass diff --git a/stubs/3.2/docutils/parsers/__init__.py b/stubs/3.2/docutils/parsers/__init__.py deleted file mode 100644 index 2ae28399f5fd..000000000000 --- a/stubs/3.2/docutils/parsers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -pass diff --git a/stubs/3.2/docutils/parsers/rst/nodes.py b/stubs/3.2/docutils/parsers/rst/nodes.py deleted file mode 100644 index 2ae28399f5fd..000000000000 --- a/stubs/3.2/docutils/parsers/rst/nodes.py +++ /dev/null @@ -1 +0,0 @@ -pass diff --git a/stubs/3.2/docutils/parsers/rst/roles.py b/stubs/3.2/docutils/parsers/rst/roles.py deleted file mode 100644 index b1f97c3cb8aa..000000000000 --- a/stubs/3.2/docutils/parsers/rst/roles.py +++ /dev/null @@ -1,10 +0,0 @@ -import docutils.nodes -import docutils.parsers.rst.states - -from typing import Function, Any, List, Dict, Tuple - -def register_local_role(name: str, - role_fn: Function[[str, str, str, int, docutils.parsers.rst.states.Inliner, Dict, List], - Tuple[List[docutils.nodes.reference], List[docutils.nodes.reference]]] - ) -> None: - pass diff --git a/stubs/3.2/docutils/parsers/rst/states.py b/stubs/3.2/docutils/parsers/rst/states.py deleted file mode 100644 index 803f9e3f10d5..000000000000 --- a/stubs/3.2/docutils/parsers/rst/states.py +++ /dev/null @@ -1,5 +0,0 @@ -import typing - -class Inliner: - def __init__(self) -> None: - pass diff --git a/stubs/3.2/encodings.py b/stubs/3.2/encodings.py deleted file mode 100644 index c0a277ae4b71..000000000000 --- a/stubs/3.2/encodings.py +++ /dev/null @@ -1,6 +0,0 @@ -import codecs - -import typing - -def search_function(encoding: str) -> codecs.CodecInfo: - pass diff --git a/stubs/3.2/errno.py b/stubs/3.2/errno.py deleted file mode 100644 index e94b7092bda8..000000000000 --- a/stubs/3.2/errno.py +++ /dev/null @@ -1,132 +0,0 @@ -# Stubs for errno - -# Based on http://docs.python.org/3.2/library/errno.html - -from typing import Undefined, Dict - -errorcode = Undefined(Dict[int, str]) - -# TODO some of the names below are platform specific - -EPERM = 0 -ENOENT = 0 -ESRCH = 0 -EINTR = 0 -EIO = 0 -ENXIO = 0 -E2BIG = 0 -ENOEXEC = 0 -EBADF = 0 -ECHILD = 0 -EAGAIN = 0 -ENOMEM = 0 -EACCES = 0 -EFAULT = 0 -ENOTBLK = 0 -EBUSY = 0 -EEXIST = 0 -EXDEV = 0 -ENODEV = 0 -ENOTDIR = 0 -EISDIR = 0 -EINVAL = 0 -ENFILE = 0 -EMFILE = 0 -ENOTTY = 0 -ETXTBSY = 0 -EFBIG = 0 -ENOSPC = 0 -ESPIPE = 0 -EROFS = 0 -EMLINK = 0 -EPIPE = 0 -EDOM = 0 -ERANGE = 0 -EDEADLK = 0 -ENAMETOOLONG = 0 -ENOLCK = 0 -ENOSYS = 0 -ENOTEMPTY = 0 -ELOOP = 0 -EWOULDBLOCK = 0 -ENOMSG = 0 -EIDRM = 0 -ECHRNG = 0 -EL2NSYNC = 0 -EL3HLT = 0 -EL3RST = 0 -ELNRNG = 0 -EUNATCH = 0 -ENOCSI = 0 -EL2HLT = 0 -EBADE = 0 -EBADR = 0 -EXFULL = 0 -ENOANO = 0 -EBADRQC = 0 -EBADSLT = 0 -EDEADLOCK = 0 -EBFONT = 0 -ENOSTR = 0 -ENODATA = 0 -ETIME = 0 -ENOSR = 0 -ENONET = 0 -ENOPKG = 0 -EREMOTE = 0 -ENOLINK = 0 -EADV = 0 -ESRMNT = 0 -ECOMM = 0 -EPROTO = 0 -EMULTIHOP = 0 -EDOTDOT = 0 -EBADMSG = 0 -EOVERFLOW = 0 -ENOTUNIQ = 0 -EBADFD = 0 -EREMCHG = 0 -ELIBACC = 0 -ELIBBAD = 0 -ELIBSCN = 0 -ELIBMAX = 0 -ELIBEXEC = 0 -EILSEQ = 0 -ERESTART = 0 -ESTRPIPE = 0 -EUSERS = 0 -ENOTSOCK = 0 -EDESTADDRREQ = 0 -EMSGSIZE = 0 -EPROTOTYPE = 0 -ENOPROTOOPT = 0 -EPROTONOSUPPORT = 0 -ESOCKTNOSUPPORT = 0 -EOPNOTSUPP = 0 -EPFNOSUPPORT = 0 -EAFNOSUPPORT = 0 -EADDRINUSE = 0 -EADDRNOTAVAIL = 0 -ENETDOWN = 0 -ENETUNREACH = 0 -ENETRESET = 0 -ECONNABORTED = 0 -ECONNRESET = 0 -ENOBUFS = 0 -EISCONN = 0 -ENOTCONN = 0 -ESHUTDOWN = 0 -ETOOMANYREFS = 0 -ETIMEDOUT = 0 -ECONNREFUSED = 0 -EHOSTDOWN = 0 -EHOSTUNREACH = 0 -EALREADY = 0 -EINPROGRESS = 0 -ESTALE = 0 -EUCLEAN = 0 -ENOTNAM = 0 -ENAVAIL = 0 -EISNAM = 0 -EREMOTEIO = 0 -EDQUOT = 0 diff --git a/stubs/3.2/fcntl.py b/stubs/3.2/fcntl.py deleted file mode 100644 index e1c8b125a6d5..000000000000 --- a/stubs/3.2/fcntl.py +++ /dev/null @@ -1,11 +0,0 @@ -# Stubs for fcntl - -# NOTE: These are incomplete! - -import typing - -FD_CLOEXEC = 0 -F_GETFD = 0 -F_SETFD = 0 - -def fcntl(fd: int, op: int, arg: int = 0) -> int: pass diff --git a/stubs/3.2/fnmatch.py b/stubs/3.2/fnmatch.py deleted file mode 100644 index 137fd9d29b02..000000000000 --- a/stubs/3.2/fnmatch.py +++ /dev/null @@ -1,22 +0,0 @@ -# Stubs for fnmatch - -# Based on http://docs.python.org/3.2/library/fnmatch.html and -# python-lib/fnmatch.py - -from typing import overload, Iterable, List - -@overload -def fnmatch(name: str, pat: str) -> bool: pass -@overload -def fnmatch(name: bytes, pat: bytes) -> bool: pass - -@overload -def fnmatchcase(name: str, pat: str) -> bool: pass -@overload -def fnmatchcase(name: bytes, pat: bytes) -> bool: pass - -@overload -def filter(names: Iterable[str], pat: str) -> List[str]: pass -@overload -def filter(names: Iterable[bytes], pat: bytes) -> List[bytes]: pass -def translate(pat: str) -> str: pass diff --git a/stubs/3.2/functools.py b/stubs/3.2/functools.py deleted file mode 100644 index 1297cb10607b..000000000000 --- a/stubs/3.2/functools.py +++ /dev/null @@ -1,13 +0,0 @@ -# Stubs for functools - -# NOTE: These are incomplete! - -from typing import Function, Any - -# TODO implement as class; more precise typing -# TODO cache_info and __wrapped__ attributes -# TODO None valid as value for maxsize -def lru_cache(maxsize: int = 100) -> Function[[Any], Any]: pass - -# TODO more precise typing? -def wraps(func: Any) -> Any: pass diff --git a/stubs/3.2/gc.py b/stubs/3.2/gc.py deleted file mode 100644 index 6cfbdd69d870..000000000000 --- a/stubs/3.2/gc.py +++ /dev/null @@ -1,10 +0,0 @@ -# Stubs for gc - -# NOTE: These are incomplete! - -import typing - -def collect(generation: int = -1) -> int: pass -def disable() -> None: pass -def enable() -> None: pass -def isenabled() -> bool: pass diff --git a/stubs/3.2/getopt.py b/stubs/3.2/getopt.py deleted file mode 100644 index 99d573e3aef4..000000000000 --- a/stubs/3.2/getopt.py +++ /dev/null @@ -1,19 +0,0 @@ -# Stubs for getopt - -# Based on http://docs.python.org/3.2/library/getopt.html - -from typing import List, Tuple - -def getopt(args: List[str], shortopts: str, - longopts: List[str]) -> Tuple[List[Tuple[str, str]], - List[str]]: pass - -def gnu_getopt(args: List[str], shortopts: str, - longopts: List[str]) -> Tuple[List[Tuple[str, str]], - List[str]]: pass - -class GetoptError(Exception): - msg = '' - opt = '' - -error = GetoptError diff --git a/stubs/3.2/glob.py b/stubs/3.2/glob.py deleted file mode 100644 index ccf0fae18a14..000000000000 --- a/stubs/3.2/glob.py +++ /dev/null @@ -1,14 +0,0 @@ -# Stubs for glob - -# Based on http://docs.python.org/3.2/library/glob.html - -from typing import overload, List, Iterator - -@overload -def glob(pathname: str) -> List[str]: pass -@overload -def glob(pathname: bytes) -> List[bytes]: pass -@overload -def iglob(pathname: str) -> Iterator[str]: pass -@overload -def iglob(pathname: bytes) -> Iterator[bytes]: pass diff --git a/stubs/3.2/grp.py b/stubs/3.2/grp.py deleted file mode 100644 index c67618acd04b..000000000000 --- a/stubs/3.2/grp.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import List, Undefined - -# TODO group database entry object type - -class struct_group: - gr_name = '' - gr_passwd = '' - gr_gid = 0 - gr_mem = Undefined(List[str]) - -def getgrgid(gid: int) -> struct_group: pass -def getgrnam(name: str) -> struct_group: pass -def getgrall() -> List[struct_group]: pass diff --git a/stubs/3.2/hashlib.py b/stubs/3.2/hashlib.py deleted file mode 100644 index b1e2cd57cfa6..000000000000 --- a/stubs/3.2/hashlib.py +++ /dev/null @@ -1,25 +0,0 @@ -# Stubs for hashlib - -# NOTE: These are incomplete! - -from abc import abstractmethod, ABCMeta -import typing - -class Hash(metaclass=ABCMeta): - @abstractmethod - def update(self, arg: bytes) -> None: pass - @abstractmethod - def digest(self) -> bytes: pass - @abstractmethod - def hexdigest(self) -> str: pass - @abstractmethod - def copy(self) -> 'Hash': pass - -def md5(arg: bytes = None) -> Hash: pass -def sha1(arg: bytes = None) -> Hash: pass -def sha224(arg: bytes = None) -> Hash: pass -def sha256(arg: bytes = None) -> Hash: pass -def sha384(arg: bytes = None) -> Hash: pass -def sha512(arg: bytes = None) -> Hash: pass - -def new(name: str, data: bytes = None) -> Hash: pass diff --git a/stubs/3.2/heapq.py b/stubs/3.2/heapq.py deleted file mode 100644 index fb365bacb693..000000000000 --- a/stubs/3.2/heapq.py +++ /dev/null @@ -1,18 +0,0 @@ -# Stubs for heapq - -# Based on http://docs.python.org/3.2/library/heapq.html - -from typing import typevar, List, Iterable, Any, Function - -_T = typevar('_T') - -def heappush(heap: List[_T], item: _T) -> None: pass -def heappop(heap: List[_T]) -> _T: pass -def heappushpop(heap: List[_T], item: _T) -> _T: pass -def heapify(x: List[_T]) -> None: pass -def heapreplace(heap: List[_T], item: _T) -> _T: pass -def merge(*iterables: Iterable[_T]) -> Iterable[_T]: pass -def nlargest(n: int, iterable: Iterable[_T], - key: Function[[_T], Any] = None) -> List[_T]: pass -def nsmallest(n: int, iterable: Iterable[_T], - key: Function[[_T], Any] = None) -> List[_T]: pass diff --git a/stubs/3.2/imp.py b/stubs/3.2/imp.py deleted file mode 100644 index 8344134c1a01..000000000000 --- a/stubs/3.2/imp.py +++ /dev/null @@ -1,10 +0,0 @@ -# Stubs for imp - -# NOTE: These are incomplete! - -from typing import typevar - -_T = typevar('_T') - -def cache_from_source(path: str, debug_override: bool = None) -> str: pass -def reload(module: _T) -> _T: pass # TODO imprecise signature diff --git a/stubs/3.2/importlib.py b/stubs/3.2/importlib.py deleted file mode 100644 index d2d920dfba79..000000000000 --- a/stubs/3.2/importlib.py +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for importlib - -# NOTE: These are incomplete! - -from typing import Any - -# TODO more precise type? -def import_module(name: str, package: str = None) -> Any: pass diff --git a/stubs/3.2/inspect.py b/stubs/3.2/inspect.py deleted file mode 100644 index 152fb50dd1ce..000000000000 --- a/stubs/3.2/inspect.py +++ /dev/null @@ -1,31 +0,0 @@ -# Stubs for inspect - -from typing import Undefined, Any, Tuple, List, Function - -_object = object - -def getmembers(obj: object, predicate: Function[[Any], bool]) -> List[Tuple[str, object]]: pass - -def isclass(obj: object) -> bool: pass - -# namedtuple('Attribute', 'name kind defining_class object') -class Attribute(tuple): - name = Undefined(str) - kind = Undefined(str) - defining_class = Undefined(type) - object = Undefined(_object) - -def classify_class_attrs(cls: type) -> List[Attribute]: pass - -def cleandoc(doc: str) -> str: pass - -def getsourcelines(obj: object) -> Tuple[List[str], int]: pass - -# namedtuple('ArgSpec', 'args varargs keywords defaults') -class ArgSpec(tuple): - args = Undefined(List[str]) - varargs = Undefined(str) - keywords = Undefined(str) - defaults = Undefined(tuple) - -def getargspec(func: object) -> ArgSpec: pass diff --git a/stubs/3.2/io.py b/stubs/3.2/io.py deleted file mode 100644 index 45fced7ce294..000000000000 --- a/stubs/3.2/io.py +++ /dev/null @@ -1,104 +0,0 @@ -# Stubs for io - -# Based on http://docs.python.org/3.2/library/io.html - -# Only a subset of functionality is included (see below). -# TODO IOBase -# TODO RawIOBase -# TODO BufferedIOBase -# TODO FileIO -# TODO BufferedReader -# TODO BufferedWriter -# TODO BufferedRandom -# TODO BufferedRWPair -# TODO TextIOBase -# TODO IncrementalNewlineDecoder - -DEFAULT_BUFFER_SIZE = 0 - -from builtins import open -from typing import List, BinaryIO, TextIO, IO, overload, Iterator, Iterable - -class BytesIO(BinaryIO): - def __init__(self, initial_bytes: bytes = b'') -> None: pass - # TODO getbuffer - # TODO see comments in BinaryIO for missing functionality - def close(self) -> None: pass - def closed(self) -> bool: pass - def fileno(self) -> int: pass - def flush(self) -> None: pass - def isatty(self) -> bool: pass - def read(self, n: int = -1) -> bytes: pass - def readable(self) -> bool: pass - def readline(self, limit: int = -1) -> bytes: pass - def readlines(self, hint: int = -1) -> List[bytes]: pass - def seek(self, offset: int, whence: int = 0) -> int: pass - def seekable(self) -> bool: pass - def tell(self) -> int: pass - def truncate(self, size: int = None) -> int: pass - def writable(self) -> bool: pass - @overload - def write(self, s: bytes) -> int: pass - @overload - def write(self, s: bytearray) -> int: pass - def writelines(self, lines: Iterable[bytes]) -> None: pass - def getvalue(self) -> bytes: pass - def read1(self) -> str: pass - - def __iter__(self) -> Iterator[bytes]: pass - def __enter__(self) -> 'BytesIO': pass - def __exit__(self, type, value, traceback) -> bool: pass - -class StringIO(TextIO): - def __init__(self, initial_value: str = '', - newline: str = None) -> None: pass - # TODO see comments in BinaryIO for missing functionality - def close(self) -> None: pass - def closed(self) -> bool: pass - def fileno(self) -> int: pass - def flush(self) -> None: pass - def isatty(self) -> bool: pass - def read(self, n: int = -1) -> str: pass - def readable(self) -> bool: pass - def readline(self, limit: int = -1) -> str: pass - def readlines(self, hint: int = -1) -> List[str]: pass - def seek(self, offset: int, whence: int = 0) -> int: pass - def seekable(self) -> bool: pass - def tell(self) -> int: pass - def truncate(self, size: int = None) -> int: pass - def writable(self) -> bool: pass - def write(self, s: str) -> int: pass - def writelines(self, lines: Iterable[str]) -> None: pass - def getvalue(self) -> str: pass - - def __iter__(self) -> Iterator[str]: pass - def __enter__(self) -> 'StringIO': pass - def __exit__(self, type, value, traceback) -> bool: pass - -class TextIOWrapper(TextIO): - # write_through is undocumented but used by subprocess - def __init__(self, buffer: IO[bytes], encoding: str = None, - errors: str = None, newline: str = None, - line_buffering: bool = False, - write_through: bool = True) -> None: pass - # TODO see comments in BinaryIO for missing functionality - def close(self) -> None: pass - def closed(self) -> bool: pass - def fileno(self) -> int: pass - def flush(self) -> None: pass - def isatty(self) -> bool: pass - def read(self, n: int = -1) -> str: pass - def readable(self) -> bool: pass - def readline(self, limit: int = -1) -> str: pass - def readlines(self, hint: int = -1) -> List[str]: pass - def seek(self, offset: int, whence: int = 0) -> int: pass - def seekable(self) -> bool: pass - def tell(self) -> int: pass - def truncate(self, size: int = None) -> int: pass - def writable(self) -> bool: pass - def write(self, s: str) -> int: pass - def writelines(self, lines: Iterable[str]) -> None: pass - - def __iter__(self) -> Iterator[str]: pass - def __enter__(self) -> StringIO: pass - def __exit__(self, type, value, traceback) -> bool: pass diff --git a/stubs/3.2/itertools.py b/stubs/3.2/itertools.py deleted file mode 100644 index a54bfcb2444c..000000000000 --- a/stubs/3.2/itertools.py +++ /dev/null @@ -1,57 +0,0 @@ -# Stubs for itertools - -# Based on http://docs.python.org/3.2/library/itertools.html - -from typing import (Iterator, typevar, Iterable, overload, Any, Function, Tuple, - Union, Sequence) - -_T = typevar('_T') -_S = typevar('_S') - -def count(start: int = 0, - step: int = 1) -> Iterator[int]: pass # more general types? -def cycle(iterable: Iterable[_T]) -> Iterator[_T]: pass - -@overload -def repeat(object: _T) -> Iterator[_T]: pass -@overload -def repeat(object: _T, times: int) -> Iterator[_T]: pass - -def accumulate(iterable: Iterable[_T]) -> Iterator[_T]: pass -def chain(*iterables: Iterable[_T]) -> Iterator[_T]: pass -# TODO chain.from_Iterable -def compress(data: Iterable[_T], selectors: Iterable[Any]) -> Iterator[_T]: pass -def dropwhile(predicate: Function[[_T], Any], - iterable: Iterable[_T]) -> Iterator[_T]: pass -def filterfalse(predicate: Function[[_T], Any], - iterable: Iterable[_T]) -> Iterator[_T]: pass - -@overload -def groupby(iterable: Iterable[_T]) -> Iterator[Tuple[_T, Iterator[_T]]]: pass -@overload -def groupby(iterable: Iterable[_T], - key: Function[[_T], _S]) -> Iterator[Tuple[_S, Iterator[_T]]]: pass - -@overload -def islice(iterable: Iterable[_T], stop: int) -> Iterator[_T]: pass -@overload -def islice(iterable: Iterable[_T], start: int, stop: int, - step: int = 1) -> Iterator[_T]: pass - -def starmap(func: Any, iterable: Iterable[Any]) -> Iterator[Any]: pass -def takewhile(predicate: Function[[_T], Any], - iterable: Iterable[_T]) -> Iterator[_T]: pass -def tee(iterable: Iterable[Any], n: int = 2) -> Iterator[Any]: pass -def zip_longest(*p: Iterable[Any], - fillvalue: Any = None) -> Iterator[Any]: pass - -# TODO: Return type should be Iterator[Tuple[..]], but unknown tuple shape. -# Iterator[Sequence[_T]] loses this type information. -def product(*p: Iterable[_T], repeat: int = 1) -> Iterator[Sequence[_T]]: pass - -def permutations(iterable: Iterable[_T], - r: Union[int, None] = None) -> Iterator[Sequence[_T]]: pass -def combinations(iterable: Iterable[_T], - r: int) -> Iterable[Sequence[_T]]: pass -def combinations_with_replacement(iterable: Iterable[_T], - r: int) -> Iterable[Sequence[_T]]: pass diff --git a/stubs/3.2/json.py b/stubs/3.2/json.py deleted file mode 100644 index 5b8987ec61f3..000000000000 --- a/stubs/3.2/json.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Any, IO - -class JSONDecodeError(object): - def dumps(self, obj: Any) -> str: pass - def dump(self, obj: Any, fp: IO[str], *args: Any, **kwds: Any) -> None: pass - def loads(self, s: str) -> Any: pass - def load(self, fp: IO[str]) -> Any: pass - -def dumps(obj: Any) -> str: pass -def dump(obj: Any, fp: IO[str], *args: Any, **kwds: Any) -> None: pass -def loads(s: str) -> Any: pass -def load(fp: IO[str]) -> Any: pass diff --git a/stubs/3.2/locale.py b/stubs/3.2/locale.py deleted file mode 100644 index 410b6c32c9b0..000000000000 --- a/stubs/3.2/locale.py +++ /dev/null @@ -1,10 +0,0 @@ -# Stubs for locale - -# NOTE: These are incomplete! - -from typing import overload, Iterable - -@overload -def setlocale(category: int, locale: str = None) -> str: pass -@overload -def setlocale(category: int, locale: Iterable[str]) -> str: pass diff --git a/stubs/3.2/logging/__init__.py b/stubs/3.2/logging/__init__.py deleted file mode 100644 index 4fb6ac9088ec..000000000000 --- a/stubs/3.2/logging/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for logging - -import typing - -# TODO signature -def getLogger(name=None): pass - -# TODO add more diff --git a/stubs/3.2/logging/handlers.py b/stubs/3.2/logging/handlers.py deleted file mode 100644 index 13ae6713715a..000000000000 --- a/stubs/3.2/logging/handlers.py +++ /dev/null @@ -1,11 +0,0 @@ -# Stubs for logging.handlers - -# NOTE: These are incomplete! - -from typing import Any - -class BufferingHandler: - def __init__(self, capacity: int) -> None: pass - def emit(self, record: Any) -> None: pass - def flush(self) -> None: pass - def shouldFlush(self, record: Any) -> bool: pass diff --git a/stubs/3.2/math.py b/stubs/3.2/math.py deleted file mode 100644 index 652a270a18b7..000000000000 --- a/stubs/3.2/math.py +++ /dev/null @@ -1,53 +0,0 @@ -# Stubs for math -# Ron Murawski - -# based on: http://docs.python.org/3.2/library/math.html - -from typing import overload, Tuple, Iterable - -# ----- variables and constants ----- -e = 0.0 -pi = 0.0 - -# ----- functions ----- -def ceil(x: float) -> int: pass -def copysign(x: float, y: float) -> float: pass -def fabs(x: float) -> float: pass -def factorial(x: int) -> int: pass -def floor(x: float) -> int: pass -def fmod(x: float, y: float) -> float: pass -def frexp(x: float) -> Tuple[float, int]: pass -def fsum(iterable: Iterable) -> float: pass -def isfinite(x: float) -> bool: pass -def isinf(x: float) -> bool: pass -def isnan(x: float) -> bool: pass -def ldexp(x: float, i: int) -> float: pass -def modf(x: float) -> Tuple[float, float]: pass -def trunc(x: float) -> float: pass -def exp(x: float) -> float: pass -def expm1(x: float) -> float: pass -def log(x: float, base: float = e) -> float: pass -def log1p(x: float) -> float: pass -def log10(x: float) -> float: pass -def pow(x: float, y: float) -> float: pass -def sqrt(x: float) -> float: pass -def acos(x: float) -> float: pass -def asin(x: float) -> float: pass -def atan(x: float) -> float: pass -def atan2(y: float, x: float) -> float: pass -def cos(x: float) -> float: pass -def hypot(x: float, y: float) -> float: pass -def sin(x: float) -> float: pass -def tan(x: float) -> float: pass -def degrees(x: float) -> float: pass -def radians(x: float) -> float: pass -def acosh(x: float) -> float: pass -def asinh(x: float) -> float: pass -def atanh(x: float) -> float: pass -def cosh(x: float) -> float: pass -def sinh(x: float) -> float: pass -def tanh(x: float) -> float: pass -def erf(x: object) -> float: pass -def erfc(x: object) -> float: pass -def gamma(x: object) -> float: pass -def lgamma(x: object) -> float: pass diff --git a/stubs/3.2/msvcrt.py b/stubs/3.2/msvcrt.py deleted file mode 100644 index f13173983d1b..000000000000 --- a/stubs/3.2/msvcrt.py +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for msvcrt - -# NOTE: These are incomplete! - -from typing import overload, BinaryIO, TextIO - -def get_osfhandle(file: int) -> int: pass -def open_osfhandle(handle: int, flags: int) -> int: pass diff --git a/stubs/3.2/operator.py b/stubs/3.2/operator.py deleted file mode 100644 index 5f40e2db2c8c..000000000000 --- a/stubs/3.2/operator.py +++ /dev/null @@ -1,7 +0,0 @@ -# Stubs for operator - -# NOTE: These are incomplete! - -from typing import Any - -def add(a: Any, b: Any) -> Any: pass diff --git a/stubs/3.2/os/__init__.py b/stubs/3.2/os/__init__.py deleted file mode 100644 index 3101915bee15..000000000000 --- a/stubs/3.2/os/__init__.py +++ /dev/null @@ -1,332 +0,0 @@ -# Stubs for os -# Ron Murawski - -# based on http://docs.python.org/3.2/library/os.html - -from typing import Undefined, Mapping, Dict, List, Any, Tuple, Iterator, overload, Union, AnyStr -from builtins import OSError as error -import os.path as path - -# ----- os variables ----- - -supports_bytes_environ = False # TODO: True when bytes implemented? - -SEEK_SET = 0 # type: int -SEEK_CUR = 1 # type: int -SEEK_END = 2 # type: int - -O_RDONLY = 0 -O_WRONLY = 0 -O_RDWR = 0 -O_APPEND = 0 -O_CREAT = 0 -O_EXCL = 0 -O_TRUNC = 0 -O_DSYNC = 0 # Unix only -O_RSYNC = 0 # Unix only -O_SYNC = 0 # Unix only -O_NDELAY = 0 # Unix only -O_NONBLOCK = 0 # Unix only -O_NOCTTY = 0 # Unix only -O_SHLOCK = 0 # Unix only -O_EXLOCK = 0 # Unix only -O_BINARY = 0 # Windows only -O_NOINHERIT = 0 # Windows only -O_SHORT_LIVED = 0# Windows only -O_TEMPORARY = 0 # Windows only -O_RANDOM = 0 # Windows only -O_SEQUENTIAL = 0 # Windows only -O_TEXT = 0 # Windows only -O_ASYNC = 0 # Gnu extension if in C library -O_DIRECT = 0 # Gnu extension if in C library -O_DIRECTORY = 0 # Gnu extension if in C library -O_NOFOLLOW = 0 # Gnu extension if in C library -O_NOATIME = 0 # Gnu extension if in C library - -curdir = '' -pardir = '' -sep = '' -altsep = '' -extsep = '' -pathsep = '' -defpath = '' -linesep = '' -devnull = '' - -F_OK = 0 -R_OK = 0 -W_OK = 0 -X_OK = 0 - -environ = Undefined(Mapping[str, str]) -environb = Undefined(Mapping[bytes, bytes]) - -confstr_names = Undefined(Dict[str, int]) # Unix only -pathconf_names = Undefined(Dict[str, int]) # Unix only -sysconf_names = Undefined(Dict[str, int]) # Unix only - -EX_OK = 0 # Unix only -EX_USAGE = 0 # Unix only -EX_DATAERR = 0 # Unix only -EX_NOINPUT = 0 # Unix only -EX_NOUSER = 0 # Unix only -EX_NOHOST = 0 # Unix only -EX_UNAVAILABLE = 0 # Unix only -EX_SOFTWARE = 0 # Unix only -EX_OSERR = 0 # Unix only -EX_OSFILE = 0 # Unix only -EX_CANTCREAT = 0 # Unix only -EX_IOERR = 0 # Unix only -EX_TEMPFAIL = 0 # Unix only -EX_PROTOCOL = 0 # Unix only -EX_NOPERM = 0 # Unix only -EX_CONFIG = 0 # Unix only -EX_NOTFOUND = 0 # Unix only - -P_NOWAIT = 0 -P_NOWAITO = 0 -P_WAIT = 0 -#P_DETACH = 0 # Windows only -#P_OVERLAY = 0 # Windows only - -# wait()/waitpid() options -WNOHANG = 0 # Unix only -#WCONTINUED = 0 # some Unix systems -#WUNTRACED = 0 # Unix only - -TMP_MAX = 0 # Undocumented, but used by tempfile - -# ----- os classes (structures) ----- -class stat_result: - # For backward compatibility, the return value of stat() is also - # accessible as a tuple of at least 10 integers giving the most important - # (and portable) members of the stat structure, in the order st_mode, - # st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, - # st_ctime. More items may be added at the end by some implementations. - - st_mode = 0 # protection bits, - st_ino = 0 # inode number, - st_dev = 0 # device, - st_nlink = 0 # number of hard links, - st_uid = 0 # user id of owner, - st_gid = 0 # group id of owner, - st_size = 0 # size of file, in bytes, - st_atime = 0.0 # time of most recent access, - st_mtime = 0.0 # time of most recent content modification, - st_ctime = 0.0 # platform dependent (time of most recent metadata change - # on Unix, or the time of creation on Windows) - - def __init__(self, tuple) -> None: pass - - # On some Unix systems (such as Linux), the following attributes may also - # be available: - st_blocks = 0 # number of blocks allocated for file - st_blksize = 0 # filesystem blocksize - st_rdev = 0 # type of device if an inode device - st_flags = 0 # user defined flags for file - - # On other Unix systems (such as FreeBSD), the following attributes may be - # available (but may be only filled out if root tries to use them): - st_gen = 0 # file generation number - st_birthtime = 0 # time of file creation - - # On Mac OS systems, the following attributes may also be available: - st_rsize = 0 - st_creator = 0 - st_type = 0 - -class statvfs_result: # Unix only - f_bsize = 0 - f_frsize = 0 - f_blocks = 0 - f_bfree = 0 - f_bavail = 0 - f_files = 0 - f_ffree = 0 - f_favail = 0 - f_flag = 0 - f_namemax = 0 - -# ----- os function stubs ----- -def name() -> str: pass -def fsencode(filename: str) -> bytes: pass -def fsdecode(filename: bytes) -> str: pass -def get_exec_path(env=None) -> List[str] : pass -# NOTE: get_exec_path(): returns List[bytes] when env not None -def ctermid() -> str: pass # Unix only -def getegid() -> int: pass # Unix only -def geteuid() -> int: pass # Unix only -def getgid() -> int: pass # Unix only -def getgroups() -> List[int]: pass # Unix only, behaves differently on Mac -def initgroups(username: str, gid: int) -> None: pass # Unix only -def getlogin() -> str: pass -def getpgid(pid: int) -> int: pass # Unix only -def getpgrp() -> int: pass # Unix only -def getpid() -> int: pass -def getppid() -> int: pass -def getresuid() -> Tuple[int, int, int]: pass # Unix only -def getresgid() -> Tuple[int, int, int]: pass # Unix only -def getuid() -> int: pass # Unix only -def getenv(key: str, default: str = None) -> str: pass -def getenvb(key: bytes, default: bytes = None) -> bytes: pass -# TODO mixed str/bytes putenv arguments -def putenv(key: AnyStr, value: AnyStr) -> None: pass -def setegid(egid: int) -> None: pass # Unix only -def seteuid(euid: int) -> None: pass # Unix only -def setgid(gid: int) -> None: pass # Unix only -def setgroups(groups: List[int]) -> None: pass # Unix only -def setpgrp() -> int: pass # Unix only -def setpgid(pid: int, pgrp: int) -> int: pass # Unix only -def setregid(rgid: int, egid: int) -> None: pass # Unix only -def setresgid(rgid: int, egid: int, sgid: int) -> None: pass # Unix only -def setresuid(ruid: int, euid: int, suid: int) -> None: pass # Unix only -def setreuid(ruid: int, euid: int) -> None: pass # Unix only -def getsid(pid: int) -> int: pass # Unix only -def setsid() -> int: pass # Unix only -def setuid(uid) -> None: pass # Unix only -def strerror(code: int) -> str: pass -def umask(mask: int) -> int: pass -def uname() -> Tuple[str, str, str, str, str]: pass # Unix only -def unsetenv(key: AnyStr) -> None: pass -# Return IO or TextIO -def fdopen(fd: int, mode: str = 'r', encoding: str = None, errors: str = None, - newline: str = None, closefd: bool = True) -> Any: pass -def close(fd: int) -> None: pass -def closerange(fd_low: int, fd_high: int) -> None: pass -def device_encoding(fd: int) -> str: pass # May return None -def dup(fd: int) -> int: pass -def dup2(fd: int, fd2: int) -> None: pass -def fchmod(fd: int, intmode) -> None: pass # Unix only -def fchown(fd: int, uid: int, gid: int) -> None: pass # Unix only -def fdatasync(fd: int) -> None: pass # Unix only, not Mac -def fpathconf(fd: int, name: str) -> int: pass # Unix only -def fstat(fd: int) -> stat_result: pass -def fstatvfs(fd: int) -> statvfs_result: pass # Unix only -def fsync(fd: int) -> None: pass -def ftruncate(fd: int, length: int) -> None: pass # Unix only -def isatty(fd: int) -> bool: pass # Unix only -def lseek(fd: int, pos: int, how: int) -> int: pass -def open(file: AnyStr, flags: int, mode: int = 0o777) -> int: pass -def openpty() -> Tuple[int, int]: pass # some flavors of Unix -def pipe() -> Tuple[int, int]: pass -def read(fd: int, n: int) -> bytes: pass -def tcgetpgrp(fd: int) -> int: pass # Unix only -def tcsetpgrp(fd: int, pg: int) -> None: pass # Unix only -def ttyname(fd: int) -> str: pass # Unix only -def write(fd: int, string: bytes) -> int: pass -def access(path: AnyStr, mode: int) -> bool: pass -def chdir(path: AnyStr) -> None: pass -def fchdir(fd: int) -> None: pass -def getcwd() -> str: pass -def getcwdb() -> bytes: pass -def chflags(path: str, flags: int) -> None: pass # Unix only -def chroot(path: str) -> None: pass # Unix only -def chmod(path: AnyStr, mode: int) -> None: pass -def chown(path: AnyStr, uid: int, gid: int) -> None: pass # Unix only -def lchflags(path: str, flags: int) -> None: pass # Unix only -def lchmod(path: str, mode: int) -> None: pass # Unix only -def lchown(path: str, uid: int, gid: int) -> None: pass # Unix only -def link(src: AnyStr, link_name: AnyStr) -> None: pass - -@overload -def listdir(path: str = '.') -> List[str]: pass -@overload -def listdir(path: bytes) -> List[bytes]: pass - -def lstat(path: AnyStr) -> stat_result: pass -def mkfifo(path, mode: int=0o666) -> None: pass # Unix only -def mknod(filename: AnyStr, mode: int = 0o600, device: int = 0) -> None: pass -def major(device: int) -> int: pass -def minor(device: int) -> int: pass -def makedev(major: int, minor: int) -> int: pass -def mkdir(path: AnyStr, mode: int = 0o777) -> None: pass -def makedirs(path: AnyStr, mode: int = 0o777, - exist_ok: bool = False) -> None: pass -def pathconf(path: str, name: str) -> int: pass # Unix only -def readlink(path: AnyStr) -> AnyStr: pass -def remove(path: AnyStr) -> None: pass -def removedirs(path: AnyStr) -> None: pass -def rename(src: AnyStr, dst: AnyStr) -> None: pass -def renames(old: AnyStr, new: AnyStr) -> None: pass -def rmdir(path: AnyStr) -> None: pass -def stat(path: AnyStr) -> stat_result: pass -def stat_float_times(newvalue: Union[bool, None] = None) -> bool: pass -def statvfs(path: str) -> statvfs_result: pass # Unix only -def symlink(source: AnyStr, link_name: AnyStr, - target_is_directory: bool = False) -> None: - pass # final argument in Windows only -def unlink(path: AnyStr) -> None: pass -def utime(path: AnyStr, times: Union[Tuple[int, int], Tuple[float, float]] = None) -> None: pass - -# TODO onerror: function from OSError to void -def walk(top: AnyStr, topdown: bool = True, onerror: Any = None, - followlinks: bool = False) -> Iterator[Tuple[AnyStr, List[AnyStr], - List[AnyStr]]]: pass -# walk(): "By default errors from the os.listdir() call are ignored. If -# optional arg 'onerror' is specified, it should be a function; it -# will be called with one argument, an os.error instance. It can -# report the error to continue with the walk, or raise the exception -# to abort the walk. Note that the filename is available as the -# filename attribute of the exception object." - -def abort() -> 'None': pass -def execl(path: AnyStr, arg0: AnyStr, *args: AnyStr) -> None: pass -def execle(path: AnyStr, arg0: AnyStr, - *args: Any) -> None: pass # Imprecise signature -def execlp(path: AnyStr, arg0: AnyStr, *args: AnyStr) -> None: pass -def execlpe(path: AnyStr, arg0: AnyStr, - *args: Any) -> None: pass # Imprecise signature -def execv(path: AnyStr, args: List[AnyStr]) -> None: pass -def execve(path: AnyStr, args: List[AnyStr], env: Mapping[AnyStr, AnyStr]) -> None: pass -def execvp(file: AnyStr, args: List[AnyStr]) -> None: pass -def execvpe(file: AnyStr, args: List[AnyStr], - env: Mapping[str, str]) -> None: pass -def _exit(n: int) -> None: pass -def fork() -> int: pass # Unix only -def forkpty() -> Tuple[int, int]: pass # some flavors of Unix -def kill(pid: int, sig: int) -> None: pass -def killpg(pgid: int, sig: int) -> None: pass # Unix only -def nice(increment: int) -> int: pass # Unix only -def plock(op: int) -> None: pass # Unix only ???op is int? - -from io import TextIOWrapper as _TextIOWrapper -class popen(_TextIOWrapper): - # TODO 'b' modes or bytes command not accepted? - def __init__(self, command: str, mode: str = 'r', - bufsize: int = -1) -> None: pass - def close(self) -> Any: pass # may return int - -def spawnl(mode: int, path: AnyStr, arg0: AnyStr, *args: AnyStr) -> int: pass -def spawnle(mode: int, path: AnyStr, arg0: AnyStr, - *args: Any) -> int: pass # Imprecise sig -def spawnlp(mode: int, file: AnyStr, arg0: AnyStr, - *args: AnyStr) -> int: pass # Unix only TODO -def spawnlpe(mode: int, file: AnyStr, arg0: AnyStr, *args: Any) -> int: - pass # Imprecise signature; Unix only TODO -def spawnv(mode: int, path: AnyStr, args: List[AnyStr]) -> int: pass -def spawnve(mode: int, path: AnyStr, args: List[AnyStr], - env: Mapping[str, str]) -> int: pass -def spawnvp(mode: int, file: AnyStr, args: List[AnyStr]) -> int: pass # Unix only -def spawnvpe(mode: int, file: AnyStr, args: List[AnyStr], - env: Mapping[str, str]) -> int: - pass # Unix only -def startfile(path: str, operation: Union[str, None] = None) -> None: pass # Windows only -def system(command: AnyStr) -> int: pass -def times() -> Tuple[float, float, float, float, float]: pass -def wait() -> Tuple[int, int]: pass # Unix only -def waitpid(pid: int, options: int) -> Tuple[int, int]: pass -def wait3(options: Union[int, None] = None) -> Tuple[int, int, Any]: pass # Unix only -def wait4(pid: int, options: int) -> Tuple[int, int, Any]: - pass # Unix only -def WCOREDUMP(status: int) -> bool: pass # Unix only -def WIFCONTINUED(status: int) -> bool: pass # Unix only -def WIFSTOPPED(status: int) -> bool: pass # Unix only -def WIFSIGNALED(status: int) -> bool: pass # Unix only -def WIFEXITED(status: int) -> bool: pass # Unix only -def WEXITSTATUS(status: int) -> bool: pass # Unix only -def WSTOPSIG(status: int) -> bool: pass # Unix only -def WTERMSIG(status: int) -> bool: pass # Unix only -def confstr(name: str) -> str: pass # Unix only -def getloadavg() -> Tuple[float, float, float]: pass # Unix only -def sysconf(name: str) -> int: pass # Unix only -def urandom(n: int) -> bytes: pass diff --git a/stubs/3.2/os/path.py b/stubs/3.2/os/path.py deleted file mode 100644 index a9cda3497a80..000000000000 --- a/stubs/3.2/os/path.py +++ /dev/null @@ -1,61 +0,0 @@ -# Stubs for os.path -# Ron Murawski - -# based on http://docs.python.org/3.2/library/os.path.html - -from typing import overload, List, Any, AnyStr, Tuple, BinaryIO, TextIO - -# ----- os.path variables ----- -supports_unicode_filenames = False -# aliases (also in os) -curdir = '' -pardir = '' -sep = '' -altsep = '' -extsep = '' -pathsep = '' -defpath = '' -devnull = '' - -# ----- os.path function stubs ----- -def abspath(path: AnyStr) -> AnyStr: pass -def basename(path: AnyStr) -> AnyStr: pass - -# NOTE: Empty List[bytes] results in '' (str) => fall back to Any return type. -def commonprefix(list: List[AnyStr]) -> Any: pass -def dirname(path: AnyStr) -> AnyStr: pass -def exists(path: AnyStr) -> bool: pass -def lexists(path: AnyStr) -> bool: pass -def expanduser(path: AnyStr) -> AnyStr: pass -def expandvars(path: AnyStr) -> AnyStr: pass - - -# These return float if os.stat_float_times() == True -def getatime(path: AnyStr) -> Any: pass -def getmtime(path: AnyStr) -> Any: pass -def getctime(path: AnyStr) -> Any: pass - -def getsize(path: AnyStr) -> int: pass -def isabs(path: AnyStr) -> bool: pass -def isfile(path: AnyStr) -> bool: pass -def isdir(path: AnyStr) -> bool: pass -def islink(path: AnyStr) -> bool: pass -def ismount(path: AnyStr) -> bool: pass - -def join(path: AnyStr, *paths: AnyStr) -> AnyStr: pass - -def normcase(path: AnyStr) -> AnyStr: pass -def normpath(path: AnyStr) -> AnyStr: pass -def realpath(path: AnyStr) -> AnyStr: pass -def relpath(path: AnyStr, start: AnyStr = None) -> AnyStr: pass - -def samefile(path1: AnyStr, path2: AnyStr) -> bool: pass -def sameopenfile(fp1: int, fp2: int) -> bool: pass -#def samestat(stat1: stat_result, -# stat2: stat_result) -> bool: pass # Unix only - -def split(path: AnyStr) -> Tuple[AnyStr, AnyStr]: pass -def splitdrive(path: AnyStr) -> Tuple[AnyStr, AnyStr]: pass -def splitext(path: AnyStr) -> Tuple[AnyStr, AnyStr]: pass - -#def splitunc(path: str) -> Tuple[str, str]: pass # Windows only, deprecated diff --git a/stubs/3.2/pickle.py b/stubs/3.2/pickle.py deleted file mode 100644 index da466d6c77f6..000000000000 --- a/stubs/3.2/pickle.py +++ /dev/null @@ -1,12 +0,0 @@ -# Stubs for pickle - -# NOTE: These are incomplete! - -from typing import Any, IO - -def dumps(obj: Any, protocol: int = None, *, - fix_imports: bool = True) -> bytes: pass -def loads(p: bytes, *, fix_imports: bool = True, - encoding: str = 'ASCII', errors: str = 'strict') -> Any: pass -def load(file: IO[bytes], *, fix_imports: bool = True, encoding: str = 'ASCII', - errors: str = 'strict') -> Any: pass diff --git a/stubs/3.2/platform.py b/stubs/3.2/platform.py deleted file mode 100644 index e60953d85372..000000000000 --- a/stubs/3.2/platform.py +++ /dev/null @@ -1,9 +0,0 @@ -# Stubs for platform - -# NOTE: These are incomplete! - -from typing import Tuple - -def mac_ver(release: str = '', - version_info: Tuple[str, str, str] = ('', '', ''), - machine: str = '') -> Tuple[str, Tuple[str, str, str], str]: pass diff --git a/stubs/3.2/posix.py b/stubs/3.2/posix.py deleted file mode 100644 index 3debbbfa522e..000000000000 --- a/stubs/3.2/posix.py +++ /dev/null @@ -1,7 +0,0 @@ -# Stubs for posix - -# NOTE: These are incomplete! - -import typing -from os import stat_result - diff --git a/stubs/3.2/posixpath.py b/stubs/3.2/posixpath.py deleted file mode 100644 index f751c1f6c5fb..000000000000 --- a/stubs/3.2/posixpath.py +++ /dev/null @@ -1,46 +0,0 @@ -# Stubs for os.path -# Ron Murawski - -# based on http://docs.python.org/3.2/library/os.path.html - -from typing import Any, List, Tuple, IO, overload - -# ----- os.path variables ----- -supports_unicode_filenames = False - -# ----- os.path function stubs ----- -def abspath(path: str) -> str: pass -def basename(path) -> str: pass -def commonprefix(list: List[str]) -> str: pass -def dirname(path: str) -> str: pass -def exists(path: str) -> bool: pass -def lexists(path: str) -> bool: pass -def expanduser(path: str) -> str: pass -def expandvars(path: str) -> str: pass -def getatime(path: str) -> int: - pass # return float if os.stat_float_times() returns True -def getmtime(path: str) -> int: - pass # return float if os.stat_float_times() returns True -def getctime(path: str) -> int: - pass # return float if os.stat_float_times() returns True -def getsize(path: str) -> int: pass -def isabs(path: str) -> bool: pass -def isfile(path: str) -> bool: pass -def isdir(path: str) -> bool: pass -def islink(path: str) -> bool: pass -def ismount(path: str) -> bool: pass -def join(path: str, *paths: str) -> str: pass -def normcase(path: str) -> str: pass -def normpath(path: str) -> str: pass -def realpath(path: str) -> str: pass -def relpath(path: str, start: str = None) -> str: pass -def samefile(path1: str, path2: str) -> bool: pass - -def sameopenfile(fp1: IO[Any], fp2: IO[Any]) -> bool: pass - -#def samestat(stat1: stat_result, stat2: stat_result) -> bool: -# pass # Unix only -def split(path: str) -> Tuple[str, str]: pass -def splitdrive(path: str) -> Tuple[str, str]: pass -def splitext(path: str) -> Tuple[str, str]: pass -#def splitunc(path: str) -> Tuple[str, str] : pass # Windows only, deprecated diff --git a/stubs/3.2/pprint.py b/stubs/3.2/pprint.py deleted file mode 100644 index c0791a39f6c7..000000000000 --- a/stubs/3.2/pprint.py +++ /dev/null @@ -1,23 +0,0 @@ -# Stubs for pprint - -# Based on http://docs.python.org/3.2/library/pprint.html - -from typing import Any, Dict, Tuple, TextIO - -def pformat(o: object, indent: int = 1, width: int = 80, - depth: int = None) -> str: pass -def pprint(o: object, stream: TextIO = None, indent: int = 1, width: int = 80, - depth: int = None) -> None: pass -def isreadable(o: object) -> bool: pass -def isrecursive(o: object) -> bool: pass -def saferepr(o: object) -> str: pass - -class PrettyPrinter: - def __init__(self, indent: int = 1, width: int = 80, depth: int = None, - stream: TextIO = None) -> None: pass - def pformat(self, o: object) -> str: pass - def pprint(self, o: object) -> None: pass - def isreadable(self, o: object) -> bool: pass - def isrecursive(self, o: object) -> bool: pass - def format(self, o: object, context: Dict[int, Any], maxlevels: int, - level: int) -> Tuple[str, bool, bool]: pass diff --git a/stubs/3.2/pwd.py b/stubs/3.2/pwd.py deleted file mode 100644 index b51b3f8a2a88..000000000000 --- a/stubs/3.2/pwd.py +++ /dev/null @@ -1,18 +0,0 @@ -# Stubs for pwd - -# NOTE: These are incomplete! - -import typing - -class struct_passwd: - # TODO use namedtuple - pw_name = '' - pw_passwd = '' - pw_uid = 0 - pw_gid = 0 - pw_gecos = '' - pw_dir = '' - pw_shell = '' - -def getpwuid(uid: int) -> struct_passwd: pass -def getpwnam(name: str) -> struct_passwd: pass diff --git a/stubs/3.2/random.py b/stubs/3.2/random.py deleted file mode 100644 index 2c534c251e7e..000000000000 --- a/stubs/3.2/random.py +++ /dev/null @@ -1,70 +0,0 @@ -# Stubs for random -# Ron Murawski -# Updated by Jukka Lehtosalo - -# based on http://docs.python.org/3.2/library/random.html - -# ----- random classes ----- - -import _random -from typing import ( - Any, typevar, Sequence, List, Function, AbstractSet, Union -) - -_T = typevar('_T') - -class Random(_random.Random): - def __init__(self, x: Any = None) -> None: pass - def seed(self, a: Any = None, version: int = 2) -> None: pass - def getstate(self) -> tuple: pass - def setstate(self, state: tuple) -> None: pass - def getrandbits(self, k: int) -> int: pass - def randrange(self, start: int, stop: Union[int, None] = None, step: int = 1) -> int: pass - def randint(self, a: int, b: int) -> int: pass - def choice(self, seq: Sequence[_T]) -> _T: pass - def shuffle(self, x: List[Any], random: Union[Function[[], float], None] = None) -> None: pass - def sample(self, population: Union[Sequence[_T], AbstractSet[_T]], k: int) -> List[_T]: pass - def random(self) -> float: pass - def uniform(self, a: float, b: float) -> float: pass - def triangular(self, low: float = 0.0, high: float = 1.0, - mode: float = None) -> float: pass - def betavariate(self, alpha: float, beta: float) -> float: pass - def expovariate(self, lambd: float) -> float: pass - def gammavariate(self, alpha: float, beta: float) -> float: pass - def gauss(self, mu: float, sigma: float) -> float: pass - def lognormvariate(self, mu: float, sigma: float) -> float: pass - def normalvariate(self, mu: float, sigma: float) -> float: pass - def vonmisesvariate(self, mu: float, kappa: float) -> float: pass - def paretovariate(self, alpha: float) -> float: pass - def weibullvariate(self, alpha: float, beta: float) -> float: pass - -# SystemRandom is not implemented for all OS's; good on Windows & Linux -class SystemRandom: - def __init__(self, randseed: object = None) -> None: pass - def random(self) -> float: pass - def getrandbits(self, k: int) -> int: pass - def seed(self, arg: object) -> None: pass - -# ----- random function stubs ----- -def seed(a: Any = None, version: int = 2) -> None: pass -def getstate() -> object: pass -def setstate(state: object) -> None: pass -def getrandbits(k: int) -> int: pass -def randrange(start: int, stop: Union[None, int] = None, step: int = 1) -> int: pass -def randint(a: int, b: int) -> int: pass -def choice(seq: Sequence[_T]) -> _T: pass -def shuffle(x: List[Any], random: Union[Function[[], float], None] = None) -> None: pass -def sample(population: Union[Sequence[_T], AbstractSet[_T]], k: int) -> List[_T]: pass -def random() -> float: pass -def uniform(a: float, b: float) -> float: pass -def triangular(low: float = 0.0, high: float = 1.0, - mode: float = None) -> float: pass -def betavariate(alpha: float, beta: float) -> float: pass -def expovariate(lambd: float) -> float: pass -def gammavariate(alpha: float, beta: float) -> float: pass -def gauss(mu: float, sigma: float) -> float: pass -def lognormvariate(mu: float, sigma: float) -> float: pass -def normalvariate(mu: float, sigma: float) -> float: pass -def vonmisesvariate(mu: float, kappa: float) -> float: pass -def paretovariate(alpha: float) -> float: pass -def weibullvariate(alpha: float, beta: float) -> float: pass diff --git a/stubs/3.2/re.py b/stubs/3.2/re.py deleted file mode 100644 index 8002850cbaf0..000000000000 --- a/stubs/3.2/re.py +++ /dev/null @@ -1,66 +0,0 @@ -# Stubs for re -# Ron Murawski -# 'bytes' support added by Jukka Lehtosalo - -# based on: http://docs.python.org/3.2/library/re.html -# and http://hg.python.org/cpython/file/618ea5612e83/Lib/re.py - -from typing import ( - Undefined, List, Iterator, overload, Function, Tuple, Sequence, Dict, - Generic, AnyStr, Match, Pattern -) - -# ----- re variables and constants ----- -A = 0 -ASCII = 0 -DEBUG = 0 -I = 0 -IGNORECASE = 0 -L = 0 -LOCALE = 0 -M = 0 -MULTILINE = 0 -S = 0 -DOTALL = 0 -X = 0 -VERBOSE = 0 -U = 0 -UNICODE = 0 - -class error(Exception): pass - -def compile(pattern: AnyStr, flags: int = 0) -> Pattern[AnyStr]: pass -def search(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> Match[AnyStr]: pass -def match(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> Match[AnyStr]: pass -def split(pattern: AnyStr, string: AnyStr, maxsplit: int = 0, - flags: int = 0) -> List[AnyStr]: pass -def findall(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> List[AnyStr]: pass - -# Return an iterator yielding match objects over all non-overlapping matches -# for the RE pattern in string. The string is scanned left-to-right, and -# matches are returned in the order found. Empty matches are included in the -# result unless they touch the beginning of another match. -def finditer(pattern: AnyStr, string: AnyStr, - flags: int = 0) -> Iterator[Match[AnyStr]]: pass - -@overload -def sub(pattern: AnyStr, repl: AnyStr, string: AnyStr, count: int = 0, - flags: int = 0) -> AnyStr: pass -@overload -def sub(pattern: AnyStr, repl: Function[[Match[AnyStr]], AnyStr], - string: AnyStr, count: int = 0, flags: int = 0) -> AnyStr: pass - -@overload -def subn(pattern: AnyStr, repl: AnyStr, string: AnyStr, count: int = 0, - flags: int = 0) -> Tuple[AnyStr, int]: pass -@overload -def subn(pattern: AnyStr, repl: Function[[Match[AnyStr]], AnyStr], - string: AnyStr, count: int = 0, - flags: int = 0) -> Tuple[AnyStr, int]: pass - -def escape(string: AnyStr) -> AnyStr: pass - -def purge() -> None: pass diff --git a/stubs/3.2/resource.py b/stubs/3.2/resource.py deleted file mode 100644 index 70c087cce25f..000000000000 --- a/stubs/3.2/resource.py +++ /dev/null @@ -1,13 +0,0 @@ -# Stubs for resource - -# NOTE: These are incomplete! - -from typing import Tuple - -RLIMIT_CORE = 0 - -def getrlimit(resource: int) -> Tuple[int, int]: pass -def setrlimit(resource: int, limits: Tuple[int, int]) -> None: pass - -# NOTE: This is an alias of OSError in Python 3.3. -class error(Exception): pass diff --git a/stubs/3.2/select.py b/stubs/3.2/select.py deleted file mode 100644 index 1308162e0003..000000000000 --- a/stubs/3.2/select.py +++ /dev/null @@ -1,27 +0,0 @@ -# Stubs for select - -# NOTE: These are incomplete! - -from typing import Any, Tuple, List, Sequence - -class error(Exception): pass - -POLLIN = 0 -POLLPRI = 0 -POLLOUT = 0 -POLLERR = 0 -POLLHUP = 0 -POLLNVAL = 0 - -class poll: - def __init__(self) -> None: pass - def register(self, fd: Any, - eventmask: int = POLLIN|POLLPRI|POLLOUT) -> None: pass - def modify(self, fd: Any, eventmask: int) -> None: pass - def unregister(self, fd: Any) -> None: pass - def poll(self, timeout: int = None) -> List[Tuple[int, int]]: pass - -def select(rlist: Sequence, wlist: Sequence, xlist: Sequence, - timeout: float = None) -> Tuple[List[int], - List[int], - List[int]]: pass diff --git a/stubs/3.2/shlex.py b/stubs/3.2/shlex.py deleted file mode 100644 index 975823e47850..000000000000 --- a/stubs/3.2/shlex.py +++ /dev/null @@ -1,36 +0,0 @@ -# Stubs for shlex - -# Based on http://docs.python.org/3.2/library/shlex.html - -from typing import List, Undefined, Tuple, Any, TextIO - -def split(s: str, comments: bool = False, - posix: bool = True) -> List[str]: pass - -class shlex: - commenters = '' - wordchars = '' - whitespace = '' - escape = '' - quotes = '' - escapedquotes = '' - whitespace_split = '' - infile = '' - instream = Undefined(TextIO) - source = '' - debug = 0 - lineno = 0 - token = '' - eof = '' - - def __init__(self, instream=None, infile=None, - posix: bool = False) -> None: pass - def get_token(self) -> str: pass - def push_token(self, tok: str) -> None: pass - def read_token(self) -> str: pass - def sourcehook(self, filename: str) -> Tuple[str, TextIO]: pass - # TODO argument types - def push_source(self, newstream: Any, newfile: Any = None) -> None: pass - def pop_source(self) -> None: pass - def error_leader(self, infile: str = None, - lineno: int = None) -> None: pass diff --git a/stubs/3.2/shutil.py b/stubs/3.2/shutil.py deleted file mode 100644 index 4fd087702a01..000000000000 --- a/stubs/3.2/shutil.py +++ /dev/null @@ -1,48 +0,0 @@ -# Stubs for shutil - -# Based on http://docs.python.org/3.2/library/shutil.html - -# 'bytes' paths are not properly supported: they don't work with all functions, -# sometimes they only work partially (broken exception messages), and the test -# cases don't use them. - -from typing import ( - overload, List, Iterable, Function, Any, Tuple, Sequence, IO, AnyStr -) - -def copyfileobj(fsrc: IO[AnyStr], fdst: IO[AnyStr], - length: int = None) -> None: pass - -def copyfile(src: str, dst: str) -> None: pass -def copymode(src: str, dst: str) -> None: pass -def copystat(src: str, dst: str) -> None: pass -def copy(src: str, dst: str) -> None: pass -def copy2(src: str, dst: str) -> None: pass -def ignore_patterns(*patterns: str) -> Function[[str, List[str]], - Iterable[str]]: pass -def copytree(src: str, dst: str, symlinks: bool = False, - ignore: Function[[str, List[str]], Iterable[str]] = None, - copy_function: Function[[str, str], None] = copy2, - ignore_dangling_symlinks: bool = False) -> None: pass -def rmtree(path: str, ignore_errors: bool = False, - onerror: Function[[Any, str, Any], None] = None) -> None: pass -def move(src: str, dst: str) -> None: pass - -class Error(Exception): pass - -def make_archive(base_name: str, format: str, root_dir: str = None, - base_dir: str = None, verbose: bool = False, - dry_run: bool = False, owner: str = None, group: str = None, - logger: Any = None) -> str: pass -def get_archive_formats() -> List[Tuple[str, str]]: pass -def register_archive_format(name: str, function: Any, - extra_args: Sequence[Tuple[str, Any]] = None, - description: str = None) -> None: pass -def unregister_archive_format(name: str) -> None: pass -def unpack_archive(filename: str, extract_dir: str = None, - format: str = None) -> None: pass -def register_unpack_format(name: str, extensions: List[str], function: Any, - extra_args: Sequence[Tuple[str, Any]] = None, - description: str = None) -> None: pass -def unregister_unpack_format(name: str) -> None: pass -def get_unpack_formats() -> List[Tuple[str, List[str], str]]: pass diff --git a/stubs/3.2/signal.py b/stubs/3.2/signal.py deleted file mode 100644 index 7c95b3d446e6..000000000000 --- a/stubs/3.2/signal.py +++ /dev/null @@ -1,51 +0,0 @@ -# Stubs for signal - -# Based on http://docs.python.org/3.2/library/signal.html - -from typing import Any, overload, Function - -SIG_DFL = 0 -SIG_IGN = 0 - -# TODO more SIG* constants (these should be platform specific?) -SIGHUP = 0 -SIGINT = 0 -SIGQUIT = 0 -SIGABRT = 0 -SIGKILL = 0 -SIGALRM = 0 -SIGTERM = 0 - -SIGUSR1 = 0 -SIGUSR2 = 0 -SIGCONT = 0 -SIGSTOP = 0 - -SIGPOLL = 0 -SIGVTALRM = 0 - -CTRL_C_EVENT = 0 # Windows -CTRL_BREAK_EVENT = 0 # Windows - -NSIG = 0 -ITIMER_REAL = 0 -ITIMER_VIRTUAL = 0 -ITIMER_PROF = 0 - -class ItimerError(IOError): pass - -def alarm(time: int) -> int: pass # Unix -def getsignal(signalnum: int) -> Any: pass -def pause() -> None: pass # Unix -#def setitimer(which: int, seconds: float, -# internval: float = None) -> Tuple[float, float]: pass # Unix -#def getitimer(int which): pass # Unix -def set_wakeup_fd(fd: int) -> None: pass -def siginterrupt(signalnum: int, flag: bool) -> None: pass - -@overload -def signal(signalnum: int, handler: int) -> Any: pass -@overload -def signal(signalnum: int, - handler: Function[[int, Any], None]) -> Any: - pass # TODO frame object type diff --git a/stubs/3.2/socket.py b/stubs/3.2/socket.py deleted file mode 100644 index a1beb4c861d4..000000000000 --- a/stubs/3.2/socket.py +++ /dev/null @@ -1,387 +0,0 @@ -# Stubs for socket -# Ron Murawski - -# based on: http://docs.python.org/3.2/library/socket.html -# see: http://hg.python.org/cpython/file/3d0686d90f55/Lib/socket.py -# see: http://nullege.com/codes/search/socket - -from typing import Undefined, Any, Tuple, overload, List - -# ----- variables and constants ----- - -AF_UNIX = 0 -AF_INET = 0 -AF_INET6 = 0 -SOCK_STREAM = 0 -SOCK_DGRAM = 0 -SOCK_RAW = 0 -SOCK_RDM = 0 -SOCK_SEQPACKET = 0 -SOCK_CLOEXEC = 0 -SOCK_NONBLOCK = 0 -SOMAXCONN = 0 -has_ipv6 = False -_GLOBAL_DEFAULT_TIMEOUT = 0.0 -SocketType = Undefined(Any) -SocketIO = Undefined(Any) - - -# the following constants are included with Python 3.2.3 (Ubuntu) -# some of the constants may be Linux-only -# all Windows/Mac-specific constants are absent -AF_APPLETALK = 0 -AF_ASH = 0 -AF_ATMPVC = 0 -AF_ATMSVC = 0 -AF_AX25 = 0 -AF_BLUETOOTH = 0 -AF_BRIDGE = 0 -AF_DECnet = 0 -AF_ECONET = 0 -AF_IPX = 0 -AF_IRDA = 0 -AF_KEY = 0 -AF_LLC = 0 -AF_NETBEUI = 0 -AF_NETLINK = 0 -AF_NETROM = 0 -AF_PACKET = 0 -AF_PPPOX = 0 -AF_ROSE = 0 -AF_ROUTE = 0 -AF_SECURITY = 0 -AF_SNA = 0 -AF_TIPC = 0 -AF_UNSPEC = 0 -AF_WANPIPE = 0 -AF_X25 = 0 -AI_ADDRCONFIG = 0 -AI_ALL = 0 -AI_CANONNAME = 0 -AI_NUMERICHOST = 0 -AI_NUMERICSERV = 0 -AI_PASSIVE = 0 -AI_V4MAPPED = 0 -BDADDR_ANY = 0 -BDADDR_LOCAL = 0 -BTPROTO_HCI = 0 -BTPROTO_L2CAP = 0 -BTPROTO_RFCOMM = 0 -BTPROTO_SCO = 0 -CAPI = 0 -EAGAIN = 0 -EAI_ADDRFAMILY = 0 -EAI_AGAIN = 0 -EAI_BADFLAGS = 0 -EAI_FAIL = 0 -EAI_FAMILY = 0 -EAI_MEMORY = 0 -EAI_NODATA = 0 -EAI_NONAME = 0 -EAI_OVERFLOW = 0 -EAI_SERVICE = 0 -EAI_SOCKTYPE = 0 -EAI_SYSTEM = 0 -EBADF = 0 -EINTR = 0 -EWOULDBLOCK = 0 -HCI_DATA_DIR = 0 -HCI_FILTER = 0 -HCI_TIME_STAMP = 0 -INADDR_ALLHOSTS_GROUP = 0 -INADDR_ANY = 0 -INADDR_BROADCAST = 0 -INADDR_LOOPBACK = 0 -INADDR_MAX_LOCAL_GROUP = 0 -INADDR_NONE = 0 -INADDR_UNSPEC_GROUP = 0 -IPPORT_RESERVED = 0 -IPPORT_USERRESERVED = 0 -IPPROTO_AH = 0 -IPPROTO_DSTOPTS = 0 -IPPROTO_EGP = 0 -IPPROTO_ESP = 0 -IPPROTO_FRAGMENT = 0 -IPPROTO_GRE = 0 -IPPROTO_HOPOPTS = 0 -IPPROTO_ICMP = 0 -IPPROTO_ICMPV6 = 0 -IPPROTO_IDP = 0 -IPPROTO_IGMP = 0 -IPPROTO_IP = 0 -IPPROTO_IPIP = 0 -IPPROTO_IPV6 = 0 -IPPROTO_NONE = 0 -IPPROTO_PIM = 0 -IPPROTO_PUP = 0 -IPPROTO_RAW = 0 -IPPROTO_ROUTING = 0 -IPPROTO_RSVP = 0 -IPPROTO_TCP = 0 -IPPROTO_TP = 0 -IPPROTO_UDP = 0 -IPV6_CHECKSUM = 0 -IPV6_DSTOPTS = 0 -IPV6_HOPLIMIT = 0 -IPV6_HOPOPTS = 0 -IPV6_JOIN_GROUP = 0 -IPV6_LEAVE_GROUP = 0 -IPV6_MULTICAST_HOPS = 0 -IPV6_MULTICAST_IF = 0 -IPV6_MULTICAST_LOOP = 0 -IPV6_NEXTHOP = 0 -IPV6_PKTINFO = 0 -IPV6_RECVDSTOPTS = 0 -IPV6_RECVHOPLIMIT = 0 -IPV6_RECVHOPOPTS = 0 -IPV6_RECVPKTINFO = 0 -IPV6_RECVRTHDR = 0 -IPV6_RECVTCLASS = 0 -IPV6_RTHDR = 0 -IPV6_RTHDRDSTOPTS = 0 -IPV6_RTHDR_TYPE_0 = 0 -IPV6_TCLASS = 0 -IPV6_UNICAST_HOPS = 0 -IPV6_V6ONLY = 0 -IP_ADD_MEMBERSHIP = 0 -IP_DEFAULT_MULTICAST_LOOP = 0 -IP_DEFAULT_MULTICAST_TTL = 0 -IP_DROP_MEMBERSHIP = 0 -IP_HDRINCL = 0 -IP_MAX_MEMBERSHIPS = 0 -IP_MULTICAST_IF = 0 -IP_MULTICAST_LOOP = 0 -IP_MULTICAST_TTL = 0 -IP_OPTIONS = 0 -IP_RECVOPTS = 0 -IP_RECVRETOPTS = 0 -IP_RETOPTS = 0 -IP_TOS = 0 -IP_TTL = 0 -MSG_CTRUNC = 0 -MSG_DONTROUTE = 0 -MSG_DONTWAIT = 0 -MSG_EOR = 0 -MSG_OOB = 0 -MSG_PEEK = 0 -MSG_TRUNC = 0 -MSG_WAITALL = 0 -NETLINK_DNRTMSG = 0 -NETLINK_FIREWALL = 0 -NETLINK_IP6_FW = 0 -NETLINK_NFLOG = 0 -NETLINK_ROUTE = 0 -NETLINK_USERSOCK = 0 -NETLINK_XFRM = 0 -NI_DGRAM = 0 -NI_MAXHOST = 0 -NI_MAXSERV = 0 -NI_NAMEREQD = 0 -NI_NOFQDN = 0 -NI_NUMERICHOST = 0 -NI_NUMERICSERV = 0 -PACKET_BROADCAST = 0 -PACKET_FASTROUTE = 0 -PACKET_HOST = 0 -PACKET_LOOPBACK = 0 -PACKET_MULTICAST = 0 -PACKET_OTHERHOST = 0 -PACKET_OUTGOING = 0 -PF_PACKET = 0 -SHUT_RD = 0 -SHUT_RDWR = 0 -SHUT_WR = 0 -SOL_HCI = 0 -SOL_IP = 0 -SOL_SOCKET = 0 -SOL_TCP = 0 -SOL_TIPC = 0 -SOL_UDP = 0 -SO_ACCEPTCONN = 0 -SO_BROADCAST = 0 -SO_DEBUG = 0 -SO_DONTROUTE = 0 -SO_ERROR = 0 -SO_KEEPALIVE = 0 -SO_LINGER = 0 -SO_OOBINLINE = 0 -SO_RCVBUF = 0 -SO_RCVLOWAT = 0 -SO_RCVTIMEO = 0 -SO_REUSEADDR = 0 -SO_SNDBUF = 0 -SO_SNDLOWAT = 0 -SO_SNDTIMEO = 0 -SO_TYPE = 0 -TCP_CORK = 0 -TCP_DEFER_ACCEPT = 0 -TCP_INFO = 0 -TCP_KEEPCNT = 0 -TCP_KEEPIDLE = 0 -TCP_KEEPINTVL = 0 -TCP_LINGER2 = 0 -TCP_MAXSEG = 0 -TCP_NODELAY = 0 -TCP_QUICKACK = 0 -TCP_SYNCNT = 0 -TCP_WINDOW_CLAMP = 0 -TIPC_ADDR_ID = 0 -TIPC_ADDR_NAME = 0 -TIPC_ADDR_NAMESEQ = 0 -TIPC_CFG_SRV = 0 -TIPC_CLUSTER_SCOPE = 0 -TIPC_CONN_TIMEOUT = 0 -TIPC_CRITICAL_IMPORTANCE = 0 -TIPC_DEST_DROPPABLE = 0 -TIPC_HIGH_IMPORTANCE = 0 -TIPC_IMPORTANCE = 0 -TIPC_LOW_IMPORTANCE = 0 -TIPC_MEDIUM_IMPORTANCE = 0 -TIPC_NODE_SCOPE = 0 -TIPC_PUBLISHED = 0 -TIPC_SRC_DROPPABLE = 0 -TIPC_SUBSCR_TIMEOUT = 0 -TIPC_SUB_CANCEL = 0 -TIPC_SUB_PORTS = 0 -TIPC_SUB_SERVICE = 0 -TIPC_TOP_SRV = 0 -TIPC_WAIT_FOREVER = 0 -TIPC_WITHDRAWN = 0 -TIPC_ZONE_SCOPE = 0 - - -# ----- exceptions ----- -class error(IOError): - pass - -class herror(error): - def __init__(self, herror: int, string: str) -> None: pass - -class gaierror(error): - def __init__(self, error: int, string: str) -> None: pass - -class timeout(error): - pass - - -# Addresses can be either tuples of varying lengths (AF_INET, AF_INET6, -# AF_NETLINK, AF_TIPC) or strings (AF_UNIX). - -# TODO AF_PACKET and AF_BLUETOOTH address objects - - -# ----- classes ----- -class socket: - family = 0 - type = 0 - proto = 0 - - def __init__(self, family: int = AF_INET, type: int = SOCK_STREAM, - proto: int = 0, fileno: int = None) -> None: pass - - # --- methods --- - # second tuple item is an address - def accept(self) -> Tuple['socket', Any]: pass - - @overload - def bind(self, address: tuple) -> None: pass - @overload - def bind(self, address: str) -> None: pass - - def close(self) -> None: pass - - @overload - def connect(self, address: tuple) -> None: pass - @overload - def connect(self, address: str) -> None: pass - - @overload - def connect_ex(self, address: tuple) -> int: pass - @overload - def connect_ex(self, address: str) -> int: pass - - def detach(self) -> int: pass - def fileno(self) -> int: pass - - # return value is an address - def getpeername(self) -> Any: pass - def getsockname(self) -> Any: pass - - @overload - def getsockopt(self, level: int, optname: str) -> bytes: pass - @overload - def getsockopt(self, level: int, optname: str, buflen: int) -> bytes: pass - - def gettimeout(self) -> float: pass - def ioctl(self, control: object, - option: Tuple[int, int, int]) -> None: pass - def listen(self, backlog: int) -> None: pass - # TODO the return value may be BinaryIO or TextIO, depending on mode - def makefile(self, mode: str = 'r', buffering: int = None, - encoding: str = None, errors: str = None, - newline: str = None) -> Any: - pass - def recv(self, bufsize: int, flags: int = 0) -> bytes: pass - - # return type is an address - def recvfrom(self, bufsize: int, flags: int = 0) -> Any: pass - def recvfrom_into(self, buffer: bytes, nbytes: int, - flags: int = 0) -> Any: pass - def recv_into(self, buffer: bytes, nbytes: int, - flags: int = 0) -> Any: pass - def send(self, data: bytes, flags=0) -> int: pass - def sendall(self, data: bytes, flags=0) -> Any: - pass # return type: None on success - - @overload - def sendto(self, data: bytes, address: tuple, flags: int = 0) -> int: pass - @overload - def sendto(self, data: bytes, address: str, flags: int = 0) -> int: pass - - def setblocking(self, flag: bool) -> None: pass - # TODO None valid for the value argument - def settimeout(self, value: float) -> None: pass - - @overload - def setsockopt(self, level: int, optname: str, value: int) -> None: pass - @overload - def setsockopt(self, level: int, optname: str, value: bytes) -> None: pass - - def shutdown(self, how: int) -> None: pass - - -# ----- functions ----- -def create_connection(address: Tuple[str, int], - timeout: float = _GLOBAL_DEFAULT_TIMEOUT, - source_address: Tuple[str, int] = None) -> socket: pass - -# the 5th tuple item is an address -def getaddrinfo( - host: str, port: int, family: int = 0, type: int = 0, proto: int = 0, - flags: int = 0) -> List[Tuple[int, int, int, str, tuple]]: - pass - -def getfqdn(name: str = '') -> str: pass -def gethostbyname(hostname: str) -> str: pass -def gethostbyname_ex(hostname: str) -> Tuple[str, List[str], List[str]]: pass -def gethostname() -> str: pass -def gethostbyaddr(ip_address: str) -> Tuple[str, List[str], List[str]]: pass -def getnameinfo(sockaddr: tuple, flags: int) -> Tuple[str, int]: pass -def getprotobyname(protocolname: str) -> int: pass -def getservbyname(servicename: str, protocolname: str = None) -> int: pass -def getservbyport(port: int, protocolname: str = None) -> str: pass -def socketpair(family: int = AF_INET, - type: int = SOCK_STREAM, - proto: int = 0) -> Tuple[socket, socket]: pass -def fromfd(fd: int, family: int, type: int, proto: int = 0) -> socket: pass -def ntohl(x: int) -> int: pass # param & ret val are 32-bit ints -def ntohs(x: int) -> int: pass # param & ret val are 16-bit ints -def htonl(x: int) -> int: pass # param & ret val are 32-bit ints -def htons(x: int) -> int: pass # param & ret val are 16-bit ints -def inet_aton(ip_string: str) -> bytes: pass # ret val 4 bytes in length -def inet_ntoa(packed_ip: bytes) -> str: pass -def inet_pton(address_family: int, ip_string: str) -> bytes: pass -def inet_ntop(address_family: int, packed_ip: bytes) -> str: pass -# TODO the timeout may be None -def getdefaulttimeout() -> float: pass -def setdefaulttimeout(timeout: float) -> None: pass diff --git a/stubs/3.2/stat.py b/stubs/3.2/stat.py deleted file mode 100644 index a7a7c0384eae..000000000000 --- a/stubs/3.2/stat.py +++ /dev/null @@ -1,71 +0,0 @@ -# Stubs for stat - -# Based on http://docs.python.org/3.2/library/stat.html - -import typing - -def S_ISDIR(mode: int) -> bool: pass -def S_ISCHR(mode: int) -> bool: pass -def S_ISBLK(mode: int) -> bool: pass -def S_ISREG(mode: int) -> bool: pass -def S_ISFIFO(mode: int) -> bool: pass -def S_ISLNK(mode: int) -> bool: pass -def S_ISSOCK(mode: int) -> bool: pass - -def S_IMODE(mode: int) -> int: pass -def S_IFMT(mode) -> int: pass - -ST_MODE = 0 -ST_INO = 0 -ST_DEV = 0 -ST_NLINK = 0 -ST_UID = 0 -ST_GID = 0 -ST_SIZE = 0 -ST_ATIME = 0 -ST_MTIME = 0 -ST_CTIME = 0 - -S_IFSOCK = 0 -S_IFLNK = 0 -S_IFREG = 0 -S_IFBLK = 0 -S_IFDIR = 0 -S_IFCHR = 0 -S_IFIFO = 0 -S_ISUID = 0 -S_ISGID = 0 -S_ISVTX = 0 - -S_IRWXU = 0 -S_IRUSR = 0 -S_IWUSR = 0 -S_IXUSR = 0 - -S_IRWXG = 0 -S_IRGRP = 0 -S_IWGRP = 0 -S_IXGRP = 0 - -S_IRWXO = 0 -S_IROTH = 0 -S_IWOTH = 0 -S_IXOTH = 0 - -S_ENFMT = 0 -S_IREAD = 0 -S_IWRITE = 0 -S_IEXEC = 0 - -UF_NODUMP = 0 -UF_IMMUTABLE = 0 -UF_APPEND = 0 -UF_OPAQUE = 0 -UF_NOUNLINK = 0 -#int UF_COMPRESSED # OS X 10.6+ only -#int UF_HIDDEN # OX X 10.5+ only -SF_ARCHIVED = 0 -SF_IMMUTABLE = 0 -SF_APPEND = 0 -SF_NOUNLINK = 0 -SF_SNAPSHOT = 0 diff --git a/stubs/3.2/string.py b/stubs/3.2/string.py deleted file mode 100644 index 6233fec1df17..000000000000 --- a/stubs/3.2/string.py +++ /dev/null @@ -1,27 +0,0 @@ -# Stubs for string - -# Based on http://docs.python.org/3.2/library/string.html - -from typing import Mapping - -ascii_letters = '' -ascii_lowercase = '' -ascii_uppercase = '' -digits = '' -hexdigits = '' -octdigits = '' -punctuation = '' -printable = '' -whitespace = '' - -def capwords(s: str, sep: str = None) -> str: pass - -class Template: - template = '' - - def __init__(self, template: str) -> None: pass - def substitute(self, mapping: Mapping[str, str], **kwds: str) -> str: pass - def safe_substitute(self, mapping: Mapping[str, str], - **kwds: str) -> str: pass - -# TODO Formatter diff --git a/stubs/3.2/struct.py b/stubs/3.2/struct.py deleted file mode 100644 index 21bc7ca2c5f0..000000000000 --- a/stubs/3.2/struct.py +++ /dev/null @@ -1,52 +0,0 @@ -# Stubs for struct - -# Based on http://docs.python.org/3.2/library/struct.html - -from typing import overload, Any, Undefined - -class error(Exception): pass - -@overload -def pack(fmt: str, *v: Any) -> bytes: pass -@overload -def pack(fmt: bytes, *v: Any) -> bytes: pass - -@overload -def pack_into(fmt: str, buffer: Any, offset: int, *v: Any) -> None: pass -# TODO buffer type -@overload -def pack_into(fmt: bytes, buffer: Any, offset: int, *v: Any) -> None: pass - -# TODO return type should be tuple -# TODO buffer type -@overload -def unpack(fmt: str, buffer: Any) -> Any: pass -@overload -def unpack(fmt: bytes, buffer: Any) -> Any: pass - -@overload -def unpack_from(fmt: str, buffer: Any) -> Any: pass -@overload -def unpack_from(fmt: bytes, buffer: Any, offset: int = 0) -> Any: pass - -@overload -def calcsize(fmt: str) -> int: pass -@overload -def calcsize(fmt: bytes) -> int: pass - -class Struct: - format = b'' - size = 0 - - @overload - def __init__(self, format: str) -> None: pass - @overload - def __init__(self, format: bytes) -> None: pass - - def pack(self, *v: Any) -> bytes: pass - # TODO buffer type - def pack_into(self, buffer: Any, offset: int, *v: Any) -> None: pass - # TOTO return type should be tuple - # TODO buffer type - def unpack(self, buffer: Any) -> Any: pass - def unpack_from(self, buffer: Any, offset: int = 0) -> Any: pass diff --git a/stubs/3.2/subprocess.py b/stubs/3.2/subprocess.py deleted file mode 100644 index ff96273a9bdc..000000000000 --- a/stubs/3.2/subprocess.py +++ /dev/null @@ -1,68 +0,0 @@ -# Stubs for subprocess - -# Based on http://docs.python.org/3.2/library/subprocess.html - -from typing import Sequence, Any, Mapping, Undefined, Function, Tuple, IO - -# TODO force keyword arguments -# TODO more keyword arguments -def call(args: Sequence[str], *, stdin: Any = None, stdout: Any = None, - stderr: Any = None, shell: bool = False, - env: Mapping[str, str] = None) -> int: pass -def check_call(args: Sequence[str], *, stdin: Any = None, stdout: Any = None, - stderr: Any = None, shell: bool = False, - env: Mapping[str, str] = None) -> int: pass -# Return str/bytes -def check_output(args: Sequence[str], *, stdin: Any = None, stderr: Any = None, - shell: bool = False, universal_newlines: bool = False, - env: Mapping[str, str] = None) -> Any: pass - -# TODO types -PIPE = Undefined(Any) -STDOUT = Undefined(Any) - -class CalledProcessError(Exception): - returncode = 0 - cmd = '' - output = b'' # May be None - -class Popen: - stdin = Undefined(IO[Any]) - stdout = Undefined(IO[Any]) - stderr = Undefined(IO[Any]) - pid = 0 - returncode = 0 - - def __init__(self, - args: Sequence[str], - bufsize: int = 0, - executable: str = None, - stdin: Any = None, - stdout: Any = None, - stderr: Any = None, - preexec_fn: Function[[], Any] = None, - close_fds: bool = True, - shell: bool = False, - cwd: str = None, - env: Mapping[str, str] = None, - universal_newlines: bool = False, - startupinfo: Any = None, - creationflags: int = 0, - restore_signals: bool = True, - start_new_session: bool = False, - pass_fds: Any = ()) -> None: pass - - def poll(self) -> int: pass - def wait(self) -> int: pass - # Return str/bytes - def communicate(self, input=None) -> Tuple[Any, Any]: pass - def send_signal(self, signal: int) -> None: pass - def terminatate(self) -> None: pass - def kill(self) -> None: pass - def __enter__(self) -> 'Popen': pass - def __exit__(self, type, value, traceback) -> bool: pass - -def getstatusoutput(cmd: str) -> Tuple[int, str]: pass -def getoutput(cmd: str) -> str: pass - -# Windows-only: STARTUPINFO etc. diff --git a/stubs/3.2/sys.py b/stubs/3.2/sys.py deleted file mode 100644 index 74eca070d061..000000000000 --- a/stubs/3.2/sys.py +++ /dev/null @@ -1,153 +0,0 @@ -# Stubs for sys -# Ron Murawski - -# based on http://docs.python.org/3.2/library/sys.html - -from typing import ( - Undefined, List, Sequence, Any, Dict, Tuple, TextIO, overload, Traceback -) - -# ----- sys variables ----- -abiflags = '' -argv = Undefined(List[str]) -byteorder = '' -builtin_module_names = Undefined(Sequence[str]) # actually a tuple of strings -copyright = '' -#dllhandle = 0 # Windows only -dont_write_bytecode = False -__displayhook__ = Undefined(Any) # contains the original value of displayhook -__excepthook__ = Undefined(Any) # contains the original value of excepthook -exec_prefix = '' -executable = '' -float_repr_style = '' -hexversion = 0 # this is a 32-bit int -last_type = Undefined(Any) -last_value = Undefined(Any) -last_traceback = Undefined(Any) -maxsize = 0 -maxunicode = 0 -meta_path = Undefined(List[Any]) -modules = Undefined(Dict[str, Any]) -path = Undefined(List[str]) -path_hooks = Undefined(List[Any]) # TODO precise type; function, path to finder -path_importer_cache = Undefined(Dict[str, Any]) # TODO precise type -platform = '' -prefix = '' -ps1 = '' -ps2 = '' -stdin = Undefined(TextIO) -stdout = Undefined(TextIO) -stderr = Undefined(TextIO) -__stdin__ = Undefined(TextIO) -__stdout__ = Undefined(TextIO) -__stderr__ = Undefined(TextIO) -# deprecated and removed in Python 3.3: -subversion = Undefined(Tuple[str, str, str]) -tracebacklimit = 0 -version = '' -api_version = 0 -warnoptions = Undefined(Any) -# Each entry is a tuple of the form (action, message, category, module, -# lineno) -#winver = '' # Windows only -_xoptions = Undefined(Dict[Any, Any]) - -flags = Undefined(_flags) -class _flags: - debug = 0 - division_warning = 0 - inspect = 0 - interactive = 0 - optimize = 0 - dont_write_bytecode = 0 - no_user_site = 0 - no_site = 0 - ignore_environment = 0 - verbose = 0 - bytes_warning = 0 - quiet = 0 - hash_randomization = 0 - -float_info = Undefined(_float_info) -class _float_info: - epsilon = 0.0 # DBL_EPSILON - dig = 0 # DBL_DIG - mant_dig = 0 # DBL_MANT_DIG - max = 0.0 # DBL_MAX - max_exp = 0 # DBL_MAX_EXP - max_10_exp = 0 # DBL_MAX_10_EXP - min = 0.0 # DBL_MIN - min_exp = 0 # DBL_MIN_EXP - min_10_exp = 0 # DBL_MIN_10_EXP - radix = 0 # FLT_RADIX - rounds = 0 # FLT_ROUNDS - -hash_info = Undefined(_hash_info) -class _hash_info: - width = 0 # width in bits used for hash values - modulus = 0 # prime modulus P used for numeric hash scheme - inf = 0 # hash value returned for a positive infinity - nan = 0 # hash value returned for a nan - imag = 0 # multiplier used for the imaginary part of a complex number - -int_info = Undefined(_int_info) -class _int_info: - bits_per_digit = 0 # number of bits held in each digit. Python integers - # are stored internally in - # base 2**int_info.bits_per_digit - sizeof_digit = 0 # size in bytes of C type used to represent a digit - -version_info = Undefined(_version_info) -class _version_info: - major = 0 - minor = 0 - micro = 0 - releaselevel = '' - serial = 0 - - -# ----- sys function stubs ----- -def call_tracing(fn: Any, args: Any) -> object: pass -def _clear_type_cache() -> None: pass -def _current_frames() -> Dict[int, Any]: pass -def displayhook(value: int) -> None: pass # value might be None -def excepthook(type_: type, value: BaseException, - traceback: Traceback) -> None: pass -def exc_info() -> Tuple[type, BaseException, Traceback]: pass -def exit(arg: int = 0) -> None: pass # arg might be None -def getcheckinterval() -> int: pass # deprecated -def getdefaultencoding() -> str: pass -def getdlopenflags() -> int: pass # Unix only -def getfilesystemencoding() -> str: pass # cannot return None -def getrefcount(object) -> int: pass -def getrecursionlimit() -> int: pass - -@overload -def getsizeof(obj: object) -> int: pass -@overload -def getsizeof(obj: object, default: int) -> int: pass - -def getswitchinterval() -> float: pass - -@overload -def _getframe() -> Any: pass -@overload -def _getframe(depth: int) -> Any: pass - -def getprofile() -> Any: pass # TODO return type -def gettrace() -> Any: pass # TODO return -def getwindowsversion() -> Any: pass # Windows only, TODO return type -def intern(string: str) -> str: pass -def setcheckinterval(interval: int) -> None: pass # deprecated -#def setdlopenflags(n: int) -> None: pass # Linux only -def setprofile(profilefunc: Any) -> None: pass # TODO type -def setrecursionlimit(limit: int) -> None: pass -def setswitchinterval(interval: float) -> None: pass -def settrace(tracefunc: Any) -> None: pass # TODO type -# Trace functions should have three arguments: frame, event, and arg. frame -# is the current stack frame. event is a string: 'call', 'line', 'return', -# 'exception', 'c_call', 'c_return', or 'c_exception'. arg depends on the -# event type. -def settscdump(on_flag: bool) -> None: pass - -def gettotalrefcount() -> int: pass # Debug builds only diff --git a/stubs/3.2/sysconfig.py b/stubs/3.2/sysconfig.py deleted file mode 100644 index e0d7398faed0..000000000000 --- a/stubs/3.2/sysconfig.py +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for sysconfig - -# NOTE: These are incomplete! - -import typing - -def get_config_var(name: str) -> str: pass -def is_python_build() -> bool: pass diff --git a/stubs/3.2/tarfile.py b/stubs/3.2/tarfile.py deleted file mode 100644 index e7f0c960555f..000000000000 --- a/stubs/3.2/tarfile.py +++ /dev/null @@ -1,33 +0,0 @@ -# TODO these are incomplete - -from typing import Any, List, overload, Function - -class TarError(Exception): pass - -class TarInfo: - name = '' - size = 0 - uid = 0 - gid = 0 - -class TarFile: - def getmember(self, name: str) -> TarInfo: pass - def getmembers(self) -> List[TarInfo]: pass - def getnames(self) -> List[str]: pass - def extractall(self, path: str = ".", - members: List[TarInfo] = None) -> None: pass - - @overload - def extract(self, member: str, path: str = "", - set_attrs: bool = True) -> None: pass - @overload - def extract(self, member: TarInfo, path: str = "", - set_attrs: bool = True) -> None: pass - - def add(self, name: str, arcname: str = None, recursive: bool = True, - exclude: Function[[str], bool] = None, *, - filter: 'Function[[TarFile], TarFile]' = None) -> None: pass - def close(self) -> None: pass - -def open(name: str, mode: str = 'r', fileobj: Any = None, bufsize: int = 10240, - **kwargs) -> TarFile: pass diff --git a/stubs/3.2/tempfile.py b/stubs/3.2/tempfile.py deleted file mode 100644 index 20e18f8d8780..000000000000 --- a/stubs/3.2/tempfile.py +++ /dev/null @@ -1,45 +0,0 @@ -# Stubs for tempfile -# Ron Murawski - -# based on http://docs.python.org/3.3/library/tempfile.html - -from typing import Tuple, BinaryIO - -# global variables -tempdir = '' -template = '' - -# TODO text files - -# function stubs -def TemporaryFile( - mode: str = 'w+b', buffering: int = None, encoding: str = None, - newline: str = None, suffix: str = '', prefix: str = 'tmp', - dir: str = None) -> BinaryIO: - pass -def NamedTemporaryFile( - mode: str = 'w+b', buffering: int = None, encoding: str = None, - newline: str = None, suffix: str = '', prefix: str = 'tmp', - dir: str = None, delete=True) -> BinaryIO: - pass -def SpooledTemporaryFile( - max_size: int = 0, mode: str = 'w+b', buffering: int = None, - encoding: str = None, newline: str = None, suffix: str = '', - prefix: str = 'tmp', dir: str = None) -> BinaryIO: - pass - -class TemporaryDirectory: - name = '' - def __init__(self, suffix: str = '', prefix: str = 'tmp', - dir: str = None) -> None: pass - def cleanup(self) -> None: pass - def __enter__(self) -> str: pass - def __exit__(self, type, value, traceback) -> bool: pass - -def mkstemp(suffix: str = '', prefix: str = 'tmp', dir: str = None, - text: bool = False) -> Tuple[int, str]: pass -def mkdtemp(suffix: str = '', prefix: str = 'tmp', - dir: str = None) -> str: pass -def mktemp(suffix: str = '', prefix: str = 'tmp', dir: str = None) -> str: pass -def gettempdir() -> str: pass -def gettempprefix() -> str: pass diff --git a/stubs/3.2/threading.py b/stubs/3.2/threading.py deleted file mode 100644 index 60f919ab9b48..000000000000 --- a/stubs/3.2/threading.py +++ /dev/null @@ -1,41 +0,0 @@ -# Stubs for threading - -# NOTE: These are incomplete! - -from typing import Any, Dict - -class Thread: - name = '' - ident = 0 - daemon = False - - def __init__(self, group: Any = None, target: Any = None, args: Any = (), - kwargs: Dict[Any, Any] = None, - verbose: Any = None) -> None: pass - def start(self) -> None: pass - def run(self) -> None: pass - # TODO None value for float - def join(self, timeout: float = None) -> None: pass - def is_alive(self) -> bool: pass - - # Legacy methods - def getName(self) -> str: pass - def setName(self, name: str) -> None: pass - def isDaemon(self) -> bool: pass - def setDaemon(self, daemon: bool) -> None: pass - -class Event: - def is_set(self) -> bool: pass - def set(self) -> None: pass - def clear(self) -> None: pass - # TODO can it return None? - # TOOD None value for float - def wait(self, timeout: float = None) -> bool: pass - -class RLock: - # TODO may return None - def acquire(self, blocking: bool = True, - timeout: float = -1.0) -> bool: pass - def release(self) -> None: pass - def __enter__(self) -> bool: pass - def __exit__(self, type, value, traceback) -> bool: pass diff --git a/stubs/3.2/time.py b/stubs/3.2/time.py deleted file mode 100644 index 0f81666211d7..000000000000 --- a/stubs/3.2/time.py +++ /dev/null @@ -1,64 +0,0 @@ -# Stubs for time -# Ron Murawski - -# based on: http://docs.python.org/3.2/library/time.html#module-time -# see: http://nullege.com/codes/search?cq=time - -from typing import Undefined, Tuple, Union - -# ----- variables and constants ----- -accept2dyear = False -altzone = 0 -daylight = 0 -timezone = 0 -tzname = Undefined(Tuple[str, str]) - - -# ----- classes/methods ----- -class struct_time: - # this is supposed to be a namedtuple object - # namedtuple is not yet implemented (see file: mypy/stubs/collections.py) - # see: http://docs.python.org/3.2/library/time.html#time.struct_time - # see: http://nullege.com/codes/search/time.struct_time - # TODO: namedtuple() object problem - #namedtuple __init__(self, int, int, int, int, int, int, int, int, int): - # pass - tm_year = 0 - tm_mon = 0 - tm_mday = 0 - tm_hour = 0 - tm_min = 0 - tm_sec = 0 - tm_wday = 0 - tm_yday = 0 - tm_isdst = 0 - - -# ----- functions ----- -def asctime(t: Union[Tuple[int, int, int, int, int, int, int, int, int], - struct_time, - None] = None) -> str: pass # return current time - -def clock() -> float: pass - -def ctime(secs: Union[float, None] = None) -> str: pass # return current time - -def gmtime(secs: Union[float, None] = None) -> struct_time: pass # return current time - -def localtime(secs: Union[float, None] = None) -> struct_time: pass # return current time - -def mktime(t: Union[Tuple[int, int, int, int, int, - int, int, int, int], - struct_time]) -> float: pass - -def sleep(secs: Union[int, float]) -> None: pass - -def strftime(format: str, t: Union[Tuple[int, int, int, int, int, - int, int, int, int], - struct_time, - None] = None) -> str: pass # return current time - -def strptime(string: str, - format: str = "%a %b %d %H:%M:%S %Y") -> struct_time: pass -def time() -> float: pass -def tzset() -> None: pass # Unix only diff --git a/stubs/3.2/traceback.py b/stubs/3.2/traceback.py deleted file mode 100644 index 31c9ce756fd9..000000000000 --- a/stubs/3.2/traceback.py +++ /dev/null @@ -1,9 +0,0 @@ -# Stubs for traceback - -import typing - -# TODO signatures -def format_tb(traceback): pass -def print_ecx(limit=None, file=None, chain=True): pass - -# TODO add more diff --git a/stubs/3.2/types.py b/stubs/3.2/types.py deleted file mode 100644 index d375aa47c503..000000000000 --- a/stubs/3.2/types.py +++ /dev/null @@ -1,13 +0,0 @@ -# Stubs for types - -# TODO this is work in progress - -from typing import Any, Undefined - -class ModuleType: - __name__ = Undefined(str) - __file__ = Undefined(str) - def __init__(self, name: str, doc: Any) -> None: pass - -class MethodType: pass -class BuiltinMethodType: pass diff --git a/stubs/3.2/typing.py b/stubs/3.2/typing.py deleted file mode 100644 index ad5c0e1140fd..000000000000 --- a/stubs/3.2/typing.py +++ /dev/null @@ -1,308 +0,0 @@ -# Stubs for typing - -from abc import abstractmethod, ABCMeta - -# Definitions of special type checking related constructs. Their definition -# are not used, so their value does not matter. - -cast = object() -overload = object() -Undefined = object() -Any = object() -typevar = object() -Generic = object() -AbstractGeneric = object() -Tuple = object() -Function = object() -builtinclass = object() -ducktype = object() -disjointclass = object() -NamedTuple = object() - -# Type aliases - -class TypeAlias: - """Class for defining generic aliases for library types.""" - - def __init__(self, target_type): - self.target_type = target_type - - def __getitem__(self, typeargs): - return self.target_type - -Union = TypeAlias(object) -List = TypeAlias(object) -Dict = TypeAlias(object) -Set = TypeAlias(object) - -# Predefined type variables. -AnyStr = typevar('AnyStr', values=(str, bytes)) - -# Abstract base classes. - -_T = typevar('_T') -_KT = typevar('_KT') -_VT = typevar('_VT') - -# TODO Container etc. - -class SupportsInt(metaclass=ABCMeta): - @abstractmethod - def __int__(self) -> int: pass - -class SupportsFloat(metaclass=ABCMeta): - @abstractmethod - def __float__(self) -> float: pass - -class SupportsAbs(AbstractGeneric[_T]): - @abstractmethod - def __abs__(self) -> _T: pass - -@disjointclass(int) -@disjointclass(float) -class SupportsRound(AbstractGeneric[_T]): - @abstractmethod - def __round__(self, ndigits: int = 0) -> _T: pass - -class Reversible(AbstractGeneric[_T]): - @abstractmethod - def __reversed__(self) -> Iterator[_T]: pass - -class Sized(metaclass=ABCMeta): - @abstractmethod - def __len__(self) -> int: pass - -class Iterable(AbstractGeneric[_T]): - @abstractmethod - def __iter__(self) -> Iterator[_T]: pass - -class Iterator(Iterable[_T], AbstractGeneric[_T]): - @abstractmethod - def __next__(self) -> _T: pass - -class Sequence(Iterable[_T], Sized, AbstractGeneric[_T]): - @abstractmethod - def __contains__(self, x: object) -> bool: pass - @overload - @abstractmethod - def __getitem__(self, i: int) -> _T: pass - @overload - @abstractmethod - def __getitem__(self, s: slice) -> Sequence[_T]: pass - -class AbstractSet(Iterable[_T], Sized, AbstractGeneric[_T]): - @abstractmethod - def __contains__(self, x: object) -> bool: pass - @abstractmethod - def __le__(self, s: AbstractSet[Any]) -> bool: pass - @abstractmethod - def __lt__(self, s: AbstractSet[Any]) -> bool: pass - @abstractmethod - def __gt__(self, s: AbstractSet[Any]) -> bool: pass - @abstractmethod - def __ge__(self, s: AbstractSet[Any]) -> bool: pass - @abstractmethod - def __and__(self, s: AbstractSet[Any]) -> AbstractSet[_T]: pass - # In order to support covariance,_T should not be used within an argument - # type. We need union types to properly model this. - @abstractmethod - def __or__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - @abstractmethod - def __sub__(self, s: AbstractSet[Any]) -> AbstractSet[_T]: pass - @abstractmethod - def __xor__(self, s: AbstractSet[_T]) -> AbstractSet[_T]: pass - # TODO argument can be any container? - @abstractmethod - def isdisjoint(self, s: AbstractSet[Any]) -> bool: pass - -class Mapping(Iterable[_KT], Sized, AbstractGeneric[_KT, _VT]): - @abstractmethod - def __getitem__(self, k: _KT) -> _VT: pass - @abstractmethod - def __setitem__(self, k: _KT, v: _VT) -> None: pass - @abstractmethod - def __delitem__(self, v: _KT) -> None: pass - @abstractmethod - def __contains__(self, o: object) -> bool: pass - - @abstractmethod - def clear(self) -> None: pass - @abstractmethod - def copy(self) -> Mapping[_KT, _VT]: pass - @overload - @abstractmethod - def get(self, k: _KT) -> _VT: pass - @overload - @abstractmethod - def get(self, k: _KT, default: _VT) -> _VT: pass - @overload - @abstractmethod - def pop(self, k: _KT) -> _VT: pass - @overload - @abstractmethod - def pop(self, k: _KT, default: _VT) -> _VT: pass - @abstractmethod - def popitem(self) -> Tuple[_KT, _VT]: pass - @overload - @abstractmethod - def setdefault(self, k: _KT) -> _VT: pass - @overload - @abstractmethod - def setdefault(self, k: _KT, default: _VT) -> _VT: pass - - # TODO keyword arguments - @overload - @abstractmethod - def update(self, m: Mapping[_KT, _VT]) -> None: pass - @overload - @abstractmethod - def update(self, m: Iterable[Tuple[_KT, _VT]]) -> None: pass - - # TODO use views for the return values instead - @abstractmethod - def keys(self) -> AbstractSet[_KT]: pass - @abstractmethod - def values(self) -> AbstractSet[_VT]: pass - @abstractmethod - def items(self) -> AbstractSet[Tuple[_KT, _VT]]: pass - -class IO(Iterable[AnyStr], AbstractGeneric[AnyStr]): - # TODO detach - # TODO use abstract properties - @property - def mode(self) -> str: pass - @property - def name(self) -> str: pass - @abstractmethod - def close(self) -> None: pass - @property - def closed(self) -> bool: pass - @abstractmethod - def fileno(self) -> int: pass - @abstractmethod - def flush(self) -> None: pass - @abstractmethod - def isatty(self) -> bool: pass - # TODO what if n is None? - @abstractmethod - def read(self, n: int = -1) -> AnyStr: pass - @abstractmethod - def readable(self) -> bool: pass - @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: pass - @abstractmethod - def readlines(self, hint: int = -1) -> list[AnyStr]: pass - @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: pass - @abstractmethod - def seekable(self) -> bool: pass - @abstractmethod - def tell(self) -> int: pass - # TODO None should not be compatible with int - @abstractmethod - def truncate(self, size: int = None) -> int: pass - @abstractmethod - def writable(self) -> bool: pass - # TODO buffer objects - @abstractmethod - def write(self, s: AnyStr) -> int: pass - @abstractmethod - def writelines(self, lines: Iterable[AnyStr]) -> None: pass - - @abstractmethod - def __iter__(self) -> Iterator[AnyStr]: pass - @abstractmethod - def __enter__(self) -> 'IO[AnyStr]': pass - @abstractmethod - def __exit__(self, type, value, traceback) -> bool: pass - -class BinaryIO(IO[bytes]): - # TODO readinto - # TODO read1? - # TODO peek? - @overload - @abstractmethod - def write(self, s: bytes) -> int: pass - @overload - @abstractmethod - def write(self, s: bytearray) -> int: pass - - @abstractmethod - def __enter__(self) -> BinaryIO: pass - -class TextIO(IO[str]): - # TODO use abstractproperty - @property - def buffer(self) -> BinaryIO: pass - @property - def encoding(self) -> str: pass - @property - def errors(self) -> str: pass - @property - def line_buffering(self) -> int: pass # int on PyPy, bool on CPython - @property - def newlines(self) -> Any: pass # None, str or tuple - @abstractmethod - def __enter__(self) -> TextIO: pass - -class Match(Generic[AnyStr]): - pos = 0 - endpos = 0 - lastindex = 0 - lastgroup = Undefined(AnyStr) - string = Undefined(AnyStr) - - # The regular expression object whose match() or search() method produced - # this match instance. - re = Undefined('Pattern[AnyStr]') - - def expand(self, template: AnyStr) -> AnyStr: pass - - @overload - def group(self, group1: int = 0) -> AnyStr: pass - @overload - def group(self, group1: str) -> AnyStr: pass - @overload - def group(self, group1: int, group2: int, - *groups: int) -> Sequence[AnyStr]: pass - @overload - def group(self, group1: str, group2: str, - *groups: str) -> Sequence[AnyStr]: pass - - def groups(self, default: AnyStr = None) -> Sequence[AnyStr]: pass - def groupdict(self, default: AnyStr = None) -> dict[str, AnyStr]: pass - def start(self, group: int = 0) -> int: pass - def end(self, group: int = 0) -> int: pass - def span(self, group: int = 0) -> Tuple[int, int]: pass - -class Pattern(Generic[AnyStr]): - flags = 0 - groupindex = 0 - groups = 0 - pattern = Undefined(AnyStr) - - def search(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> Match[AnyStr]: pass - def match(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> Match[AnyStr]: pass - def split(self, string: AnyStr, maxsplit: int = 0) -> list[AnyStr]: pass - def findall(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> list[AnyStr]: pass - def finditer(self, string: AnyStr, pos: int = 0, - endpos: int = -1) -> Iterator[Match[AnyStr]]: pass - - @overload - def sub(self, repl: AnyStr, string: AnyStr, - count: int = 0) -> AnyStr: pass - @overload - def sub(self, repl: Function[[Match[AnyStr]], AnyStr], string: AnyStr, - count: int = 0) -> AnyStr: pass - - @overload - def subn(self, repl: AnyStr, string: AnyStr, - count: int = 0) -> Tuple[AnyStr, int]: pass - @overload - def subn(self, repl: Function[[Match[AnyStr]], AnyStr], string: AnyStr, - count: int = 0) -> Tuple[AnyStr, int]: pass - -class Traceback: pass diff --git a/stubs/3.2/unicodedata.py b/stubs/3.2/unicodedata.py deleted file mode 100644 index 9cd14f079086..000000000000 --- a/stubs/3.2/unicodedata.py +++ /dev/null @@ -1,7 +0,0 @@ -# Stubs for unicodedata - -# NOTE: These are incomplete! - -import typing - -def normalize(form: str, unistr: str) -> str: pass diff --git a/stubs/3.2/unittest.py b/stubs/3.2/unittest.py deleted file mode 100644 index 809ba35c06f1..000000000000 --- a/stubs/3.2/unittest.py +++ /dev/null @@ -1,167 +0,0 @@ -# Stubs for unittest - -# Based on http://docs.python.org/3.0/library/unittest.html - -# NOTE: These stubs are based on the 3.0 version API, since later versions -# would require featurs not supported currently by mypy. - -# Only a subset of functionality is included. - -from typing import ( - Any, Function, Iterable, Undefined, Tuple, List, TextIO, Sequence, - overload, typevar, Pattern -) -from abc import abstractmethod, ABCMeta - -_T = typevar('_T') -_FT = typevar('_FT') - -class Testable(metaclass=ABCMeta): - @abstractmethod - def run(self, result: 'TestResult') -> None: pass - @abstractmethod - def debug(self) -> None: pass - @abstractmethod - def countTestCases(self) -> int: pass - -# TODO ABC for test runners? - -class TestResult: - errors = Undefined(List[Tuple[Testable, str]]) - failures = Undefined(List[Tuple[Testable, str]]) - testsRun = 0 - shouldStop = False - - def wasSuccessful(self) -> bool: pass - def stop(self) -> None: pass - def startTest(self, test: Testable) -> None: pass - def stopTest(self, test: Testable) -> None: pass - def addError(self, test: Testable, - err: Tuple[type, Any, Any]) -> None: pass # TODO - def addFailure(self, test: Testable, - err: Tuple[type, Any, Any]) -> None: pass # TODO - def addSuccess(self, test: Testable) -> None: pass - -class _AssertRaisesBaseContext: - expected = Undefined(Any) - failureException = Undefined(type) - obj_name = '' - expected_regex = Undefined(Pattern[str]) - -class _AssertRaisesContext(_AssertRaisesBaseContext): - exception = Undefined(Any) # TODO precise type - def __enter__(self) -> _AssertRaisesContext: pass - def __exit__(self, exc_type, exc_value, tb) -> bool: pass - -class _AssertWarnsContext(_AssertRaisesBaseContext): - warning = Undefined(Any) # TODO precise type - filename = '' - lineno = 0 - def __enter__(self) -> _AssertWarnsContext: pass - def __exit__(self, exc_type, exc_value, tb) -> bool: pass - -class TestCase(Testable): - def __init__(self, methodName: str = 'runTest') -> None: pass - # TODO failureException - def setUp(self) -> None: pass - def tearDown(self) -> None: pass - def run(self, result: TestResult = None) -> None: pass - def debug(self) -> None: pass - def assert_(self, expr: Any, msg: object = None) -> None: pass - def failUnless(self, expr: Any, msg: object = None) -> None: pass - def assertTrue(self, expr: Any, msg: object = None) -> None: pass - def assertEqual(self, first: Any, second: Any, - msg: object = None) -> None: pass - def failUnlessEqual(self, first: Any, second: Any, - msg: object = None) -> None: pass - def assertNotEqual(self, first: Any, second: Any, - msg: object = None) -> None: pass - def assertSequenceEqual(self, first: Sequence[Any], second: Sequence[Any], - msg: object = None, - seq_type: type = None) -> None: pass - def failIfEqual(self, first: Any, second: Any, - msg: object = None) -> None: pass - def assertAlmostEqual(self, first: float, second: float, places: int = 7, - msg: object = None, - delta: float = None) -> None: pass - def failUnlessAlmostEqual(self, first: float, second: float, - places: int = 7, - msg: object = None) -> None: pass - def assertNotAlmostEqual(self, first: float, second: float, - places: int = 7, msg: object = None, - delta: float = None) -> None: pass - def failIfAlmostEqual(self, first: float, second: float, places: int = 7, - msg: object = None) -> None: pass - def assertGreater(self, first: Any, second: Any, - msg: object = None) -> None: pass - def assertGreaterEqual(self, first: Any, second: Any, - msg: object = None) -> None: pass - def assertLess(self, first: Any, second: Any, - msg: object = None) -> None: pass - def assertLessEqual(self, first: Any, second: Any, - msg: object = None) -> None: pass - # TODO: If callableObj is None, the return value is None. - def assertRaises(self, excClass: type, callableObj: Any = None, - *args: Any, **kwargs: Any) -> _AssertRaisesContext: pass - def failIf(self, expr: Any, msg: object = None) -> None: pass - def assertFalse(self, expr: Any, msg: object = None) -> None: pass - def assertIs(self, first: object, second: object, - msg: object = None) -> None: pass - def assertIsNot(self, first: object, second: object, - msg: object = None) -> None: pass - def assertIsNone(self, expr: Any, msg: object = None) -> None: pass - def assertIsNotNone(self, expr: Any, msg: object = None) -> None: pass - def assertIn(self, first: _T, second: Iterable[_T], - msg: object = None) -> None: pass - def assertNotIn(self, first: _T, second: Iterable[_T], - msg: object = None) -> None: pass - def assertIsInstance(self, obj: Any, cls: type, - msg: object = None) -> None: pass - def assertNotIsInstance(self, obj: Any, cls: type, - msg: object = None) -> None: pass - def assertWarns(self, expected_warning: type, callable_obj: Any = None, - *args: Any, **kwargs: Any) -> _AssertWarnsContext: pass - def fail(self, msg: object = None) -> None: pass - def countTestCases(self) -> int: pass - def defaultTestResult(self) -> TestResult: pass - def id(self) -> str: pass - def shortDescription(self) -> str: pass # May return None - def addCleanup(function: Any, *args: Any, **kwargs: Any) -> None: pass - def skipTest(self, reason: Any) -> None: pass - -class FunctionTestCase(Testable): - def __init__(self, testFunc: Function[[], None], - setUp: Function[[], None] = None, - tearDown: Function[[], None] = None, - description: str = None) -> None: pass - def run(self, result: TestResult) -> None: pass - def debug(self) -> None: pass - def countTestCases(self) -> int: pass - -class TestSuite(Testable): - def __init__(self, tests: Iterable[Testable] = None) -> None: pass - def addTest(self, test: Testable) -> None: pass - def addTests(self, tests: Iterable[Testable]) -> None: pass - def run(self, result: TestResult) -> None: pass - def debug(self) -> None: pass - def countTestCases(self) -> int: pass - -# TODO TestLoader -# TODO defaultTestLoader - -class TextTestRunner: - def __init__(self, stream: TextIO = None, descriptions: bool = True, - verbosity: int = 1, failfast: bool = False) -> None: pass - -class SkipTest(Exception): - pass - -# TODO precise types -def skipUnless(condition: Any, reason: str) -> Any: pass -def skipIf(condition: Any, reason: str) -> Any: pass -def expectedFailure(func: _FT) -> _FT: pass -def skip(reason: str) -> Any: pass - -def main(module: str = '__main__', defaultTest: str = None, - argv: List[str] = None, testRunner: Any = None, - testLoader: Any = None) -> None: pass # TODO types diff --git a/stubs/3.2/urllib/parse.py b/stubs/3.2/urllib/parse.py deleted file mode 100644 index 41279b9b1227..000000000000 --- a/stubs/3.2/urllib/parse.py +++ /dev/null @@ -1,123 +0,0 @@ -# Stubs for urllib.parse -from typing import List, Dict, Tuple, AnyStr, Undefined, Generic, overload, Sequence, Mapping - -__all__ = ( - 'urlparse', - 'urlunparse', - 'urljoin', - 'urldefrag', - 'urlsplit', - 'urlunsplit', - 'urlencode', - 'parse_qs', - 'parse_qsl', - 'quote', - 'quote_plus', - 'quote_from_bytes', - 'unquote', - 'unquote_plus', - 'unquote_to_bytes' -) - - -class _ResultMixinBase(Generic[AnyStr]): - def geturl(self) -> AnyStr: pass - -class _ResultMixinStr(_ResultMixinBase[str]): - def encode(self, encoding: str = 'ascii', errors: str = 'strict') -> '_ResultMixinBytes': pass - - -class _ResultMixinBytes(_ResultMixinBase[str]): - def decode(self, encoding: str = 'ascii', errors: str = 'strict') -> '_ResultMixinStr': pass - - -class _NetlocResultMixinBase(Generic[AnyStr]): - username = Undefined(AnyStr) - password = Undefined(AnyStr) - hostname = Undefined(AnyStr) - port = Undefined(int) - -class _NetlocResultMixinStr(_NetlocResultMixinBase[str], _ResultMixinStr): pass - - -class _NetlocResultMixinBytes(_NetlocResultMixinBase[str], _ResultMixinBytes): pass - -class _DefragResultBase(tuple, Generic[AnyStr]): - url = Undefined(AnyStr) - fragment = Undefined(AnyStr) - -class _SplitResultBase(tuple, Generic[AnyStr]): - scheme = Undefined(AnyStr) - netloc = Undefined(AnyStr) - path = Undefined(AnyStr) - query = Undefined(AnyStr) - fragment = Undefined(AnyStr) - -class _ParseResultBase(tuple, Generic[AnyStr]): - scheme = Undefined(AnyStr) - netloc = Undefined(AnyStr) - path = Undefined(AnyStr) - params = Undefined(AnyStr) - query = Undefined(AnyStr) - fragment = Undefined(AnyStr) - -# Structured result objects for string data -class DefragResult(_DefragResultBase[str], _ResultMixinStr): pass - -class SplitResult(_SplitResultBase[str], _NetlocResultMixinStr): pass - -class ParseResult(_ParseResultBase[str], _NetlocResultMixinStr): pass - -# Structured result objects for bytes data -class DefragResultBytes(_DefragResultBase[bytes], _ResultMixinBytes): pass - -class SplitResultBytes(_SplitResultBase[bytes], _NetlocResultMixinBytes): pass - -class ParseResultBytes(_ParseResultBase[bytes], _NetlocResultMixinBytes): pass - - -def parse_qs(qs: str, keep_blank_values : bool = False, strict_parsing : bool = False, encoding : str = 'utf-8', errors: str = 'replace') -> Dict[str, List[str]]: pass - -def parse_qsl(qs: str, keep_blank_values: bool = False, strict_parsing: bool = False, encoding: str = 'utf-8', errors: str = 'replace') -> List[Tuple[str,str]]: pass - -def quote(string: AnyStr, safe: AnyStr = None, encoding: str = None, errors: str = None) -> str: pass - -def quote_from_bytes(bs: bytes, safe: AnyStr = None) -> bytes: pass - -def quote_plus(string: AnyStr, safe: AnyStr = None, encoding: str = None, errors: str = None) -> str: pass - -def unquote(string: str, encoding: str = 'utf-8', errors: str = 'replace') -> str: pass - -def unquote_to_bytes(string: AnyStr) -> bytes: pass - -@overload -def urldefrag(url: str) -> DefragResult: pass -@overload -def urldefrag(url: bytes) -> DefragResultBytes: pass - -@overload -def urlencode(query: Mapping[AnyStr, AnyStr], doseq: bool = False, safe: AnyStr = None, encoding: str = None, errors: str = None) -> str: pass -@overload -def urlencode(query: Sequence[Tuple[AnyStr, AnyStr]], doseq: bool = False, safe: AnyStr = None, encoding: str = None, errors: str = None) -> str: pass - -def urljoin(base: AnyStr, url: AnyStr, allow_fragments: bool = True) -> AnyStr: pass - -@overload -def urlparse(url: str, scheme: str = None, allow_framgents: bool = True) -> ParseResult: pass -@overload -def urlparse(url: bytes, scheme: bytes = None, allow_framgents: bool = True) -> ParseResultBytes: pass - -@overload -def urlsplit(url: str, scheme: str = None, allow_fragments: bool = True) -> SplitResult: pass -@overload -def urlsplit(url: bytes, scheme: bytes = None, allow_fragments: bool = True) -> SplitResultBytes: pass - -@overload -def urlunparse(components: Sequence[AnyStr]) -> AnyStr: pass -@overload -def urlunparse(components: Tuple[AnyStr, AnyStr, AnyStr, AnyStr, AnyStr, AnyStr]) -> AnyStr: pass - -@overload -def urlunsplit(components: Sequence[AnyStr]) -> AnyStr: pass -@overload -def urlunsplit(components: Tuple[AnyStr, AnyStr, AnyStr, AnyStr, AnyStr]) -> AnyStr: pass diff --git a/stubs/3.2/urllib/request.py b/stubs/3.2/urllib/request.py deleted file mode 100644 index 6db895031fcd..000000000000 --- a/stubs/3.2/urllib/request.py +++ /dev/null @@ -1,3 +0,0 @@ -# Stubs for urllib.request - -# NOTE: These are incomplete! diff --git a/stubs/3.2/warnings.py b/stubs/3.2/warnings.py deleted file mode 100644 index 492db75ecbcf..000000000000 --- a/stubs/3.2/warnings.py +++ /dev/null @@ -1,42 +0,0 @@ -# Stubs for warnings - -# Based on http://docs.python.org/3.2/library/warnings.html - -from typing import overload, Any, List, TextIO - -@overload -def warn(message: str, category: type = None, - stacklevel: int = 1) -> None: pass -@overload -def warn(message: Warning, category: type = None, - stacklevel: int = 1) -> None: pass - -@overload -def warn_explicit(message: str, category: type, filename: str, lineno: int, - module: str = None, registry: Any = None, - module_globals: Any = None) -> None: pass -@overload -def warn_explicit(message: Warning, category: type, filename: str, lineno: int, - module: str = None, registry: Any = None, - module_globals: Any = None) -> None: pass - -# logging modifies showwarning => make it a variable. -def _showwarning(message: str, category: type, filename: str, lineno: int, - file: TextIO = None, line: str = None) -> None: pass -showwarning = _showwarning - -def formatwarning(message: str, category: type, filename: str, lineno: int, - line: str = None) -> None: pass -def filterwarnings(action: str, message: str = '', category: type = Warning, - module: str = '', lineno: int = 0, - append: bool = False) -> None: pass -def simplefilter(action: str, category: type = Warning, lineno: int = 0, - append: bool = False) -> None: pass -def resetwarnings() -> None: pass - -class catch_warnings: - # TODO record and module must be keyword arguments! - # TODO type of module? - def __init__(self, record: bool = False, module: Any = None) -> None: pass - def __enter__(self) -> List[Any]: pass - def __exit__(self, type, value, traceback) -> bool: pass diff --git a/stubs/3.2/weakref.py b/stubs/3.2/weakref.py deleted file mode 100644 index 67f681e8096a..000000000000 --- a/stubs/3.2/weakref.py +++ /dev/null @@ -1,70 +0,0 @@ -# Stubs for weakref - -# NOTE: These are incomplete! - -from typing import ( - typevar, Generic, Any, Function, overload, Mapping, Iterator, Dict, Tuple, - Iterable -) - -_T = typevar('_T') -_KT = typevar('_KT') -_VT = typevar('_VT') - -class ReferenceType(Generic[_T]): - # TODO members - pass - -def ref(o: _T, callback: Function[[ReferenceType[_T]], - Any] = None) -> ReferenceType[_T]: pass - -# TODO callback -def proxy(object: _T) -> _T: pass - -class WeakValueDictionary(Generic[_KT, _VT]): - # TODO tuple iterable argument? - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, map: Mapping[_KT, _VT]) -> None: pass - - def __len__(self) -> int: pass - def __getitem__(self, k: _KT) -> _VT: pass - def __setitem__(self, k: _KT, v: _VT) -> None: pass - def __delitem__(self, v: _KT) -> None: pass - def __contains__(self, o: object) -> bool: pass - def __iter__(self) -> Iterator[_KT]: pass - def __str__(self) -> str: pass - - def clear(self) -> None: pass - def copy(self) -> Dict[_KT, _VT]: pass - - @overload - def get(self, k: _KT) -> _VT: pass - @overload - def get(self, k: _KT, default: _VT) -> _VT: pass - - @overload - def pop(self, k: _KT) -> _VT: pass - @overload - def pop(self, k: _KT, default: _VT) -> _VT: pass - - def popitem(self) -> Tuple[_KT, _VT]: pass - - @overload - def setdefault(self, k: _KT) -> _VT: pass - @overload - def setdefault(self, k: _KT, default: _VT) -> _VT: pass - - @overload - def update(self, m: Mapping[_KT, _VT]) -> None: pass - @overload - def update(self, m: Iterable[Tuple[_KT, _VT]]) -> None: pass - - # NOTE: incompatible with Mapping - def keys(self) -> Iterator[_KT]: pass - def values(self) -> Iterator[_VT]: pass - def items(self) -> Iterator[Tuple[_KT, _VT]]: pass - - # TODO return type - def valuerefs(self) -> Iterable[Any]: pass diff --git a/stubs/3.2/zipfile.py b/stubs/3.2/zipfile.py deleted file mode 100644 index 04e2f06d9e6a..000000000000 --- a/stubs/3.2/zipfile.py +++ /dev/null @@ -1,29 +0,0 @@ -# TODO these are incomplete - -from typing import List, Undefined, Tuple, BinaryIO, Union - -ZIP_STORED = 0 -ZIP_DEFLATED = 0 - -def is_zipfile(filename: Union[str, BinaryIO]) -> bool: pass - -class ZipInfo: - filename = '' - date_time = Undefined(Tuple[int, int, int, int, int, int]) - compressed_size = 0 - file_size = 0 - -class ZipFile: - def __init__(self, file: Union[str, BinaryIO], mode: str = 'r', - compression: int = ZIP_STORED, - allowZip64: bool = False) -> None: pass - def close(self) -> None: pass - def getinfo(name: str) -> ZipInfo: pass - def infolist(self) -> List[ZipInfo]: pass - def namelist(self) -> List[str]: pass - def read(self, name: Union[str, ZipInfo], pwd: str = None) -> bytes: pass - def write(self, filename: str, arcname: str = None, - compress_type: int = None) -> None: pass - - def __enter__(self) -> 'ZipFile': pass - def __exit__(self, type, value, traceback) -> bool: pass diff --git a/stubs/3.2/zlib.py b/stubs/3.2/zlib.py deleted file mode 100644 index 92c3fb997c6c..000000000000 --- a/stubs/3.2/zlib.py +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for zlib - -# Based on http://docs.python.org/3.2/library/zlib.html - -# TODO this stub file is an empty placeholder diff --git a/stubs/3.4/asyncio/__init__.py b/stubs/3.4/asyncio/__init__.py deleted file mode 100644 index 98d3092224a9..000000000000 --- a/stubs/3.4/asyncio/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""The asyncio package, tracking PEP 3156.""" -from asyncio.futures import Future -from asyncio.tasks import (coroutine, sleep, Task, FIRST_COMPLETED, - FIRST_EXCEPTION, ALL_COMPLETED, wait, wait_for) -from asyncio.events import (AbstractEventLoopPolicy, AbstractEventLoop, - Handle, get_event_loop) - -__all__ = (futures.__all__, - tasks.__all__, - events.__all__) \ No newline at end of file diff --git a/stubs/3.4/asyncio/events.py b/stubs/3.4/asyncio/events.py deleted file mode 100644 index 8a1be6cfdfac..000000000000 --- a/stubs/3.4/asyncio/events.py +++ /dev/null @@ -1,172 +0,0 @@ -from typing import Any, typevar, List, Function, Tuple, Union, Dict, Undefined -from abc import ABCMeta, abstractmethod -from asyncio.futures import Future - -# __all__ = ['AbstractServer', -# 'TimerHandle', -# 'get_event_loop_policy', 'set_event_loop_policy', -# 'set_event_loop', 'new_event_loop', -# 'get_child_watcher', 'set_child_watcher', -# ] - - -__all__ = ['AbstractEventLoopPolicy', 'AbstractEventLoop', 'Handle', 'get_event_loop'] - -_T = typevar('_T') - -PIPE = Undefined(Any) # from subprocess.PIPE - -AF_UNSPEC = 0 # from socket -AI_PASSIVE = 0 - -class Handle: - __slots__ = [] # type: List[str] - _cancelled = False - _args = [] # type: List[Any] - def __init__(self, callback: Function[[],Any], args: List[Any], - loop: AbstractEventLoop) -> None: pass - def __repr__(self) -> str: pass - def cancel(self) -> None: pass - def _run(self) -> None: pass - - -class AbstractEventLoop(metaclass=ABCMeta): - @abstractmethod - def run_forever(self) -> None: pass - @abstractmethod - def run_until_complete(self, future: Future[_T]) -> _T: pass - @abstractmethod - def stop(self) -> None: pass - @abstractmethod - def is_running(self) -> bool: pass - @abstractmethod - def close(self) -> None: pass - # Methods scheduling callbacks. All these return Handles. - @abstractmethod - def call_soon(self, callback: Function[[],Any], *args: Any) -> Handle: pass - @abstractmethod - def call_later(self, delay: Union[int, float], callback: Function[[],Any], *args: Any) -> Handle: pass - @abstractmethod - def call_at(self, when: float, callback: Function[[],Any], *args: Any) -> Handle: pass - @abstractmethod - def time(self) -> float: pass - # Methods for interacting with threads - @abstractmethod - def call_soon_threadsafe(self, callback: Function[[],Any], *args: Any) -> Handle: pass - @abstractmethod - def run_in_executor(self, executor: Any, - callback: Function[[],Any], *args: Any) -> Future[Any]: pass - @abstractmethod - def set_default_executor(self, executor: Any) -> None: pass - # Network I/O methods returning Futures. - @abstractmethod - def getaddrinfo(self, host: str, port: int, *, - family: int = 0, type: int = 0, proto: int = 0, flags: int = 0) -> List[Tuple[int, int, int, str, tuple]]: pass - @abstractmethod - def getnameinfo(self, sockaddr: tuple, flags: int = 0) -> Tuple[str, int]: pass - @abstractmethod - def create_connection(self, protocol_factory: Any, host: str = None, port: int = None, *, - ssl: Any = None, family: int = 0, proto: int = 0, flags: int = 0, sock: Any = None, - local_addr: str = None, server_hostname: str = None) -> tuple: pass - # ?? check Any - # return (Transport, Protocol) - @abstractmethod - def create_server(self, protocol_factory: Any, host: str = None, port: int = None, *, - family: int = AF_UNSPEC, flags: int = AI_PASSIVE, - sock: Any = None, backlog: int = 100, ssl: Any = None, reuse_address: Any = None) -> Any: pass - # ?? check Any - # return Server - @abstractmethod - def create_unix_connection(self, protocol_factory: Any, path: str, *, - ssl: Any = None, sock: Any = None, - server_hostname: str = None) -> tuple: pass - # ?? check Any - # return tuple(Transport, Protocol) - @abstractmethod - def create_unix_server(self, protocol_factory: Any, path: str, *, - sock: Any = None, backlog: int = 100, ssl: Any = None) -> Any: pass - # ?? check Any - # return Server - @abstractmethod - def create_datagram_endpoint(self, protocol_factory: Any, - local_addr: str = None, remote_addr: str = None, *, - family: int = 0, proto: int = 0, flags: int = 0) -> tuple: pass - #?? check Any - # return (Transport, Protocol) - # Pipes and subprocesses. - @abstractmethod - def connect_read_pipe(self, protocol_factory: Any, pipe: Any) -> tuple: pass - #?? check Any - # return (Transport, Protocol) - @abstractmethod - def connect_write_pipe(self, protocol_factory: Any, pipe: Any) -> tuple: pass - #?? check Any - # return (Transport, Protocol) - @abstractmethod - def subprocess_shell(self, protocol_factory: Any, cmd: Union[bytes, str], *, stdin: Any = PIPE, - stdout: Any = PIPE, stderr: Any = PIPE, - **kwargs: Dict[str, Any]) -> tuple: pass - #?? check Any - # return (Transport, Protocol) - @abstractmethod - def subprocess_exec(self, protocol_factory: Any, *args: List[Any], stdin: Any = PIPE, - stdout: Any = PIPE, stderr: Any = PIPE, - **kwargs: Dict[str, Any]) -> tuple: pass - #?? check Any - # return (Transport, Protocol) - @abstractmethod - def add_reader(self, fd: int, callback: Function[[],Any], *args: List[Any]) -> None: pass - @abstractmethod - def remove_reader(self, fd: int) -> None: pass - @abstractmethod - def add_writer(self, fd: int, callback: Function[[],Any], *args: List[Any]) -> None: pass - @abstractmethod - def remove_writer(self, fd: int) -> None: pass - # Completion based I/O methods returning Futures. - @abstractmethod - def sock_recv(self, sock: Any, nbytes: int) -> Any: pass #TODO - @abstractmethod - def sock_sendall(self, sock: Any, data: bytes) -> None: pass #TODO - @abstractmethod - def sock_connect(self, sock: Any, address: str) -> Any: pass #TODO - @abstractmethod - def sock_accept(self, sock: Any) -> Any: pass - # Signal handling. - @abstractmethod - def add_signal_handler(self, sig: int, callback: Function[[],Any], *args: List[Any]) -> None: pass - @abstractmethod - def remove_signal_handler(self, sig: int) -> None: pass - # Error handlers. - @abstractmethod - def set_exception_handler(self, handler: Function[[], Any]) -> None: pass - @abstractmethod - def default_exception_handler(self, context: Any) -> None: pass - @abstractmethod - def call_exception_handler(self, context: Any) -> None: pass - # Debug flag management. - @abstractmethod - def get_debug(self) -> bool: pass - @abstractmethod - def set_debug(self, enabled: bool) -> None: pass - -class AbstractEventLoopPolicy(metaclass=ABCMeta): - @abstractmethod - def get_event_loop(self) -> AbstractEventLoop: pass - @abstractmethod - def set_event_loop(self, loop: AbstractEventLoop): pass - @abstractmethod - def new_event_loop(self) -> Any: pass # return selector_events.BaseSelectorEventLoop - # Child processes handling (Unix only). - @abstractmethod - def get_child_watcher(self) -> Any: pass # return unix_events.AbstractChildWatcher - @abstractmethod - def set_child_watcher(self, watcher: Any) -> None: pass # gen unix_events.AbstractChildWatcher - -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): - def __init__(self) -> None: pass - def get_event_loop(self) -> AbstractEventLoop: pass - def set_event_loop(self, loop: AbstractEventLoop): pass - def new_event_loop(self) -> Any: pass # Same return than AbstractEventLoop - - -def get_event_loop() -> AbstractEventLoop: pass diff --git a/stubs/3.4/asyncio/futures.py b/stubs/3.4/asyncio/futures.py deleted file mode 100644 index b08e3430bfd8..000000000000 --- a/stubs/3.4/asyncio/futures.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Any, Function, typevar, List, Generic, Iterable, Iterator -from asyncio.events import AbstractEventLoop -# __all__ = ['CancelledError', 'TimeoutError', -# 'InvalidStateError', -# 'wrap_future', -# ] -__all__ = ['Future'] - -_T = typevar('_T') - -class _TracebackLogger: - __slots__ = [] # type: List[str] - exc = Any # Exception - tb = [] # type: List[str] - def __init__(self, exc: Any, loop: AbstractEventLoop) -> None: pass - def activate(self) -> None: pass - def clear(self) -> None: pass - def __del__(self) -> None: pass - -class Future(Iterator[_T], Generic[_T]): - _state = '' - _exception = Any #Exception - _blocking = False - _log_traceback = False - _tb_logger = _TracebackLogger - def __init__(self, *, loop: AbstractEventLoop = None) -> None: pass - def __repr__(self) -> str: pass - def __del__(self) -> None: pass - def cancel(self) -> bool: pass - def _schedule_callbacks(self) -> None: pass - def cancelled(self) -> bool: pass - def done(self) -> bool: pass - def result(self) -> _T: pass - def exception(self) -> Any: pass - def add_done_callback(self, fn: Function[[Future[_T]], Any]) -> None: pass - def remove_done_callback(self, fn: Function[[Future[_T]], Any]) -> int: pass - def set_result(self, result: _T) -> None: pass - def set_exception(self, exception: Any) -> None: pass - def _copy_state(self, other: Any) -> None: pass - def __iter__(self) -> Iterator[_T]: pass - def __next__(self) -> _T: pass diff --git a/stubs/3.4/asyncio/tasks.py b/stubs/3.4/asyncio/tasks.py deleted file mode 100644 index 6574454c8667..000000000000 --- a/stubs/3.4/asyncio/tasks.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Any, Iterable, typevar, Set, Dict, List, TextIO, Union, Tuple, Generic, Function -from asyncio.events import AbstractEventLoop -from asyncio.futures import Future -# __all__ = ['iscoroutinefunction', 'iscoroutine', -# 'as_completed', 'async', -# 'gather', 'shield', -# ] - -__all__ = ['coroutine', 'Task', 'sleep', - 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', - 'wait', 'wait_for'] - -FIRST_EXCEPTION = 'FIRST_EXCEPTION' -FIRST_COMPLETED = 'FIRST_COMPLETED' -ALL_COMPLETED = 'ALL_COMPLETED' -_T = typevar('_T') -def coroutine(f: _T) -> _T: pass # Here comes and go a function -def sleep(delay: float, result: _T = None, loop: AbstractEventLoop = None) -> Future[_T]: pass -def wait(fs: List[Task[_T]], *, loop: AbstractEventLoop = None, - timeout: float = None, return_when: str = ALL_COMPLETED) -> Future[Tuple[Set[Future[_T]], Set[Future[_T]]]]: pass -def wait_for(fut: Future[_T], timeout: float, *, loop: AbstractEventLoop = None) -> Future[_T]: pass - - -class Task(Future[_T], Generic[_T]): - _all_tasks = None # type: Set[Task] - _current_tasks = {} # type: Dict[AbstractEventLoop, Task] - @classmethod - def current_task(cls, loop: AbstractEventLoop = None) -> Task: pass - @classmethod - def all_tasks(cls, loop: AbstractEventLoop = None) -> Set[Task]: pass - def __init__(self, coro: Future[_T], *, loop: AbstractEventLoop = None) -> None: pass - def __repr__(self) -> str: pass - def get_stack(self, *, limit: int = None) -> List[Any]: pass # return List[stackframe] - def print_stack(self, *, limit: int = None, file: TextIO = None) -> None: pass - def cancel(self) -> bool: pass - def _step(self, value: Any = None, exc: Exception = None) -> None: pass - def _wakeup(self, future: Future[Any]) -> None: pass - diff --git a/test-data/packages/modulefinder-site-packages/baz/baz_pkg/__init__.py b/test-data/packages/modulefinder-site-packages/baz/baz_pkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/baz/baz_pkg/py.typed b/test-data/packages/modulefinder-site-packages/baz/baz_pkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/baz/ns_baz_pkg/a.py b/test-data/packages/modulefinder-site-packages/baz/ns_baz_pkg/a.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/baz/ns_baz_pkg/py.typed b/test-data/packages/modulefinder-site-packages/baz/ns_baz_pkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/foo-stubs/__init__.pyi b/test-data/packages/modulefinder-site-packages/foo-stubs/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/foo-stubs/bar.pyi b/test-data/packages/modulefinder-site-packages/foo-stubs/bar.pyi new file mode 100644 index 000000000000..833a52007f57 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/foo-stubs/bar.pyi @@ -0,0 +1 @@ +bar_var: str diff --git a/test-data/packages/modulefinder-site-packages/foo/__init__.py b/test-data/packages/modulefinder-site-packages/foo/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/foo/bar.py b/test-data/packages/modulefinder-site-packages/foo/bar.py new file mode 100644 index 000000000000..7782aba46492 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/foo/bar.py @@ -0,0 +1 @@ +bar_var = "bar" diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_typed/a.py b/test-data/packages/modulefinder-site-packages/ns_pkg_typed/a.py new file mode 100644 index 000000000000..c0cca79b8552 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/ns_pkg_typed/a.py @@ -0,0 +1 @@ +a_var = "a" diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_typed/b/c.py b/test-data/packages/modulefinder-site-packages/ns_pkg_typed/b/c.py new file mode 100644 index 000000000000..0ed729e24e43 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/ns_pkg_typed/b/c.py @@ -0,0 +1 @@ +c_var = "c" diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_typed/py.typed b/test-data/packages/modulefinder-site-packages/ns_pkg_typed/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_untyped/a.py b/test-data/packages/modulefinder-site-packages/ns_pkg_untyped/a.py new file mode 100644 index 000000000000..c0cca79b8552 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/ns_pkg_untyped/a.py @@ -0,0 +1 @@ +a_var = "a" diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_untyped/b/c.py b/test-data/packages/modulefinder-site-packages/ns_pkg_untyped/b/c.py new file mode 100644 index 000000000000..0ed729e24e43 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/ns_pkg_untyped/b/c.py @@ -0,0 +1 @@ +c_var = "c" diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs-stubs/typed/__init__.pyi b/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs-stubs/typed/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/typed/__init__.py b/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/typed/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/typed_inline/__init__.py b/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/typed_inline/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/typed_inline/py.typed b/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/typed_inline/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/untyped/__init__.py b/test-data/packages/modulefinder-site-packages/ns_pkg_w_stubs/untyped/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/pkg_typed/__init__.py b/test-data/packages/modulefinder-site-packages/pkg_typed/__init__.py new file mode 100644 index 000000000000..f49ab244c27e --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_typed/__init__.py @@ -0,0 +1 @@ +pkg_typed_var = "pkg_typed" diff --git a/test-data/packages/modulefinder-site-packages/pkg_typed/a.py b/test-data/packages/modulefinder-site-packages/pkg_typed/a.py new file mode 100644 index 000000000000..c0cca79b8552 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_typed/a.py @@ -0,0 +1 @@ +a_var = "a" diff --git a/test-data/packages/modulefinder-site-packages/pkg_typed/b/__init__.py b/test-data/packages/modulefinder-site-packages/pkg_typed/b/__init__.py new file mode 100644 index 000000000000..6cea6ed4292a --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_typed/b/__init__.py @@ -0,0 +1 @@ +b_var = "b" diff --git a/test-data/packages/modulefinder-site-packages/pkg_typed/b/c.py b/test-data/packages/modulefinder-site-packages/pkg_typed/b/c.py new file mode 100644 index 000000000000..0ed729e24e43 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_typed/b/c.py @@ -0,0 +1 @@ +c_var = "c" diff --git a/test-data/packages/modulefinder-site-packages/pkg_typed/py.typed b/test-data/packages/modulefinder-site-packages/pkg_typed/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-site-packages/pkg_untyped/__init__.py b/test-data/packages/modulefinder-site-packages/pkg_untyped/__init__.py new file mode 100644 index 000000000000..4960ea0a9555 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_untyped/__init__.py @@ -0,0 +1 @@ +pkg_untyped_var = "pkg_untyped" diff --git a/test-data/packages/modulefinder-site-packages/pkg_untyped/a.py b/test-data/packages/modulefinder-site-packages/pkg_untyped/a.py new file mode 100644 index 000000000000..c0cca79b8552 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_untyped/a.py @@ -0,0 +1 @@ +a_var = "a" diff --git a/test-data/packages/modulefinder-site-packages/pkg_untyped/b/__init__.py b/test-data/packages/modulefinder-site-packages/pkg_untyped/b/__init__.py new file mode 100644 index 000000000000..6cea6ed4292a --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_untyped/b/__init__.py @@ -0,0 +1 @@ +b_var = "b" diff --git a/test-data/packages/modulefinder-site-packages/pkg_untyped/b/c.py b/test-data/packages/modulefinder-site-packages/pkg_untyped/b/c.py new file mode 100644 index 000000000000..0ed729e24e43 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/pkg_untyped/b/c.py @@ -0,0 +1 @@ +c_var = "c" diff --git a/test-data/packages/modulefinder-site-packages/standalone.py b/test-data/packages/modulefinder-site-packages/standalone.py new file mode 100644 index 000000000000..ce436beefe85 --- /dev/null +++ b/test-data/packages/modulefinder-site-packages/standalone.py @@ -0,0 +1 @@ +standalone_var = "standalone" diff --git a/test-data/packages/modulefinder-src/neighbor_pkg/__init__.py b/test-data/packages/modulefinder-src/neighbor_pkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-src/neighbor_pkg/py.typed b/test-data/packages/modulefinder-src/neighbor_pkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-src/ns_neighbor_pkg/a.py b/test-data/packages/modulefinder-src/ns_neighbor_pkg/a.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder-src/ns_neighbor_pkg/py.typed b/test-data/packages/modulefinder-src/ns_neighbor_pkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsx-pkg1/nsx/a/__init__.py b/test-data/packages/modulefinder/nsx-pkg1/nsx/a/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsx-pkg2/nsx/b/__init__.py b/test-data/packages/modulefinder/nsx-pkg2/nsx/b/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsx-pkg3/nsx/c/c b/test-data/packages/modulefinder/nsx-pkg3/nsx/c/c new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsx-pkg3/nsx/c/c.py b/test-data/packages/modulefinder/nsx-pkg3/nsx/c/c.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.py b/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.pyi b/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/b.pyi b/test-data/packages/modulefinder/nsy-pkg2/nsy/b.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/b/__init__.py b/test-data/packages/modulefinder/nsy-pkg2/nsy/b/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/c.py b/test-data/packages/modulefinder/nsy-pkg2/nsy/c.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/c.pyi b/test-data/packages/modulefinder/nsy-pkg2/nsy/c.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/pkg1/a b/test-data/packages/modulefinder/pkg1/a new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/pkg1/a.py b/test-data/packages/modulefinder/pkg1/a.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/pkg2/b/__init__.py b/test-data/packages/modulefinder/pkg2/b/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/modulefinder/readme.txt b/test-data/packages/modulefinder/readme.txt new file mode 100644 index 000000000000..669bcfcd2663 --- /dev/null +++ b/test-data/packages/modulefinder/readme.txt @@ -0,0 +1,4 @@ +Samples for testing modulefinder.FindModuleCache. + +Contains three packages for the `nsx` namespace, and two packages +providing `a` and `b`. diff --git a/test-data/packages/typedpkg-stubs/pyproject.toml b/test-data/packages/typedpkg-stubs/pyproject.toml new file mode 100644 index 000000000000..c984c5d91e0a --- /dev/null +++ b/test-data/packages/typedpkg-stubs/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = 'typedpkg-stubs' +version = '0.1' +description = 'test' + +[tool.hatch.build] +include = ["**/*.pyi"] + +[build-system] +requires = ["hatchling==1.18"] +build-backend = "hatchling.build" diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed b/test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed new file mode 100644 index 000000000000..b648ac923333 --- /dev/null +++ b/test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed @@ -0,0 +1 @@ +partial diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi new file mode 100644 index 000000000000..355deefd6a2d --- /dev/null +++ b/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi @@ -0,0 +1,2 @@ +from typing import Iterable, List +def ex(a: Iterable[str]) -> List[str]: ... diff --git a/test-data/packages/typedpkg/pyproject.toml b/test-data/packages/typedpkg/pyproject.toml new file mode 100644 index 000000000000..6b55d4b3df60 --- /dev/null +++ b/test-data/packages/typedpkg/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = 'typedpkg' +version = '0.1' +description = 'test' + +[build-system] +requires = ["hatchling==1.18"] +build-backend = "hatchling.build" diff --git a/test-data/packages/typedpkg/typedpkg/__init__.py b/test-data/packages/typedpkg/typedpkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/dne.py b/test-data/packages/typedpkg/typedpkg/dne.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/pkg/__init__.py b/test-data/packages/typedpkg/typedpkg/pkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/pkg/aaa.py b/test-data/packages/typedpkg/typedpkg/pkg/aaa.py new file mode 100644 index 000000000000..4e9edd13c859 --- /dev/null +++ b/test-data/packages/typedpkg/typedpkg/pkg/aaa.py @@ -0,0 +1,2 @@ +def af(a: str) -> str: + return a + " nested" diff --git a/test-data/packages/typedpkg/typedpkg/pkg/py.typed b/test-data/packages/typedpkg/typedpkg/pkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/py.typed b/test-data/packages/typedpkg/typedpkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py new file mode 100644 index 000000000000..7e88816e824a --- /dev/null +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -0,0 +1,7 @@ +from typing import Iterable, Tuple + + +def ex(a): + # type: (Iterable[str]) -> Tuple[str, ...] + """Example typed package.""" + return list(a) diff --git a/test-data/packages/typedpkg_ns_a/pyproject.toml b/test-data/packages/typedpkg_ns_a/pyproject.toml new file mode 100644 index 000000000000..f41ad16b5bc2 --- /dev/null +++ b/test-data/packages/typedpkg_ns_a/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = 'typedpkg_namespace.alpha' +version = '0.1' +description = 'test' + +[tool.hatch.build] +include = ["**/*.py", "**/*.pyi", "**/py.typed"] + +[build-system] +requires = ["hatchling==1.18"] +build-backend = "hatchling.build" diff --git a/test-data/packages/typedpkg_ns_a/typedpkg_ns/__init__.py b/test-data/packages/typedpkg_ns_a/typedpkg_ns/__init__.py new file mode 100644 index 000000000000..3ac255b8a577 --- /dev/null +++ b/test-data/packages/typedpkg_ns_a/typedpkg_ns/__init__.py @@ -0,0 +1,2 @@ +# namespace pkg +__import__("pkg_resources").declare_namespace(__name__) diff --git a/test-data/packages/typedpkg_ns_a/typedpkg_ns/a/__init__.py b/test-data/packages/typedpkg_ns_a/typedpkg_ns/a/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg_ns_a/typedpkg_ns/a/bbb.py b/test-data/packages/typedpkg_ns_a/typedpkg_ns/a/bbb.py new file mode 100644 index 000000000000..33c9a9106584 --- /dev/null +++ b/test-data/packages/typedpkg_ns_a/typedpkg_ns/a/bbb.py @@ -0,0 +1,2 @@ +def bf(a: bool) -> bool: + return not a diff --git a/test-data/packages/typedpkg_ns_a/typedpkg_ns/a/py.typed b/test-data/packages/typedpkg_ns_a/typedpkg_ns/a/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg_ns_b-stubs/pyproject.toml b/test-data/packages/typedpkg_ns_b-stubs/pyproject.toml new file mode 100644 index 000000000000..2c1c206c361d --- /dev/null +++ b/test-data/packages/typedpkg_ns_b-stubs/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = 'typedpkg_ns-stubs' +version = '0.1' +description = 'test' + +[tool.hatch.build] +include = ["**/*.pyi"] + +[build-system] +requires = ["hatchling==1.18"] +build-backend = "hatchling.build" diff --git a/test-data/packages/typedpkg_ns_b-stubs/typedpkg_ns-stubs/b/__init__.pyi b/test-data/packages/typedpkg_ns_b-stubs/typedpkg_ns-stubs/b/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg_ns_b-stubs/typedpkg_ns-stubs/b/bbb.pyi b/test-data/packages/typedpkg_ns_b-stubs/typedpkg_ns-stubs/b/bbb.pyi new file mode 100644 index 000000000000..e00e9e52c05f --- /dev/null +++ b/test-data/packages/typedpkg_ns_b-stubs/typedpkg_ns-stubs/b/bbb.pyi @@ -0,0 +1 @@ +def bf(a: bool) -> bool: ... diff --git a/test-data/packages/typedpkg_ns_b/pyproject.toml b/test-data/packages/typedpkg_ns_b/pyproject.toml new file mode 100644 index 000000000000..b8ae0d59072e --- /dev/null +++ b/test-data/packages/typedpkg_ns_b/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = 'typedpkg_namespace.beta' +version = '0.1' +description = 'test' + +[build-system] +requires = ["hatchling==1.18"] +build-backend = "hatchling.build" diff --git a/test-data/packages/typedpkg_ns_b/typedpkg_ns/__init__.py b/test-data/packages/typedpkg_ns_b/typedpkg_ns/__init__.py new file mode 100644 index 000000000000..3ac255b8a577 --- /dev/null +++ b/test-data/packages/typedpkg_ns_b/typedpkg_ns/__init__.py @@ -0,0 +1,2 @@ +# namespace pkg +__import__("pkg_resources").declare_namespace(__name__) diff --git a/test-data/packages/typedpkg_ns_b/typedpkg_ns/b/__init__.py b/test-data/packages/typedpkg_ns_b/typedpkg_ns/b/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg_ns_b/typedpkg_ns/b/bbb.py b/test-data/packages/typedpkg_ns_b/typedpkg_ns/b/bbb.py new file mode 100644 index 000000000000..f10802daace9 --- /dev/null +++ b/test-data/packages/typedpkg_ns_b/typedpkg_ns/b/bbb.py @@ -0,0 +1,2 @@ +def bf(a): + return not a diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi new file mode 100644 index 000000000000..90afb46d6d94 --- /dev/null +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi @@ -0,0 +1,27 @@ +import os +from . import demo as demo +from typing import overload + +class StaticMethods: + def __init__(self, *args, **kwargs) -> None: ... + @overload + @staticmethod + def overloaded_static_method(value: int) -> int: ... + @overload + @staticmethod + def overloaded_static_method(value: float) -> float: ... + @staticmethod + def some_static_method(a: int, b: int) -> int: ... + +class TestStruct: + field_readwrite: int + field_readwrite_docstring: int + def __init__(self, *args, **kwargs) -> None: ... + @property + def field_readonly(self) -> int: ... + +def func_incomplete_signature(*args, **kwargs): ... +def func_returning_optional() -> int | None: ... +def func_returning_pair() -> tuple[int, float]: ... +def func_returning_path() -> os.PathLike: ... +def func_returning_vector() -> list[float]: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi new file mode 100644 index 000000000000..87b8ec0e4ad6 --- /dev/null +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi @@ -0,0 +1,61 @@ +from typing import ClassVar, overload + +PI: float +__version__: str + +class Point: + class AngleUnit: + __members__: ClassVar[dict] = ... # read-only + __entries: ClassVar[dict] = ... + degree: ClassVar[Point.AngleUnit] = ... + radian: ClassVar[Point.AngleUnit] = ... + def __init__(self, value: int) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __int__(self) -> int: ... + def __ne__(self, other: object) -> bool: ... + @property + def name(self) -> str: ... + @property + def value(self) -> int: ... + + class LengthUnit: + __members__: ClassVar[dict] = ... # read-only + __entries: ClassVar[dict] = ... + inch: ClassVar[Point.LengthUnit] = ... + mm: ClassVar[Point.LengthUnit] = ... + pixel: ClassVar[Point.LengthUnit] = ... + def __init__(self, value: int) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __int__(self) -> int: ... + def __ne__(self, other: object) -> bool: ... + @property + def name(self) -> str: ... + @property + def value(self) -> int: ... + angle_unit: ClassVar[Point.AngleUnit] = ... + length_unit: ClassVar[Point.LengthUnit] = ... + x_axis: ClassVar[Point] = ... # read-only + y_axis: ClassVar[Point] = ... # read-only + origin: ClassVar[Point] = ... + x: float + y: float + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, x: float, y: float) -> None: ... + def as_list(self) -> list[float]: ... + @overload + def distance_to(self, x: float, y: float) -> float: ... + @overload + def distance_to(self, other: Point) -> float: ... + @property + def length(self) -> float: ... + +def answer() -> int: ... +def midpoint(left: float, right: float) -> float: ... +def sum(arg0: int, arg1: int) -> int: ... +def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi new file mode 100644 index 000000000000..db04bccab028 --- /dev/null +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi @@ -0,0 +1,52 @@ +import os +from . import demo as demo +from typing import overload + +class StaticMethods: + def __init__(self, *args, **kwargs) -> None: + """Initialize self. See help(type(self)) for accurate signature.""" + @overload + @staticmethod + def overloaded_static_method(value: int) -> int: + """overloaded_static_method(*args, **kwargs) + Overloaded function. + + 1. overloaded_static_method(value: int) -> int + + 2. overloaded_static_method(value: float) -> float + """ + @overload + @staticmethod + def overloaded_static_method(value: float) -> float: + """overloaded_static_method(*args, **kwargs) + Overloaded function. + + 1. overloaded_static_method(value: int) -> int + + 2. overloaded_static_method(value: float) -> float + """ + @staticmethod + def some_static_method(a: int, b: int) -> int: + """some_static_method(a: int, b: int) -> int + + None + """ + +class TestStruct: + field_readwrite: int + field_readwrite_docstring: int + def __init__(self, *args, **kwargs) -> None: + """Initialize self. See help(type(self)) for accurate signature.""" + @property + def field_readonly(self) -> int: ... + +def func_incomplete_signature(*args, **kwargs): + """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding""" +def func_returning_optional() -> int | None: + """func_returning_optional() -> Optional[int]""" +def func_returning_pair() -> tuple[int, float]: + """func_returning_pair() -> Tuple[int, float]""" +def func_returning_path() -> os.PathLike: + """func_returning_path() -> os.PathLike""" +def func_returning_vector() -> list[float]: + """func_returning_vector() -> List[float]""" diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi new file mode 100644 index 000000000000..1be0bc905a43 --- /dev/null +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi @@ -0,0 +1,112 @@ +from typing import ClassVar, overload + +PI: float +__version__: str + +class Point: + class AngleUnit: + __members__: ClassVar[dict] = ... # read-only + __entries: ClassVar[dict] = ... + degree: ClassVar[Point.AngleUnit] = ... + radian: ClassVar[Point.AngleUnit] = ... + def __init__(self, value: int) -> None: + """__init__(self: pybind11_fixtures.demo.Point.AngleUnit, value: int) -> None""" + def __eq__(self, other: object) -> bool: + """__eq__(self: object, other: object) -> bool""" + def __hash__(self) -> int: + """__hash__(self: object) -> int""" + def __index__(self) -> int: + """__index__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" + def __int__(self) -> int: + """__int__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" + def __ne__(self, other: object) -> bool: + """__ne__(self: object, other: object) -> bool""" + @property + def name(self) -> str: ... + @property + def value(self) -> int: ... + + class LengthUnit: + __members__: ClassVar[dict] = ... # read-only + __entries: ClassVar[dict] = ... + inch: ClassVar[Point.LengthUnit] = ... + mm: ClassVar[Point.LengthUnit] = ... + pixel: ClassVar[Point.LengthUnit] = ... + def __init__(self, value: int) -> None: + """__init__(self: pybind11_fixtures.demo.Point.LengthUnit, value: int) -> None""" + def __eq__(self, other: object) -> bool: + """__eq__(self: object, other: object) -> bool""" + def __hash__(self) -> int: + """__hash__(self: object) -> int""" + def __index__(self) -> int: + """__index__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" + def __int__(self) -> int: + """__int__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" + def __ne__(self, other: object) -> bool: + """__ne__(self: object, other: object) -> bool""" + @property + def name(self) -> str: ... + @property + def value(self) -> int: ... + angle_unit: ClassVar[Point.AngleUnit] = ... + length_unit: ClassVar[Point.LengthUnit] = ... + x_axis: ClassVar[Point] = ... # read-only + y_axis: ClassVar[Point] = ... # read-only + origin: ClassVar[Point] = ... + x: float + y: float + @overload + def __init__(self) -> None: + """__init__(*args, **kwargs) + Overloaded function. + + 1. __init__(self: pybind11_fixtures.demo.Point) -> None + + 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None + """ + @overload + def __init__(self, x: float, y: float) -> None: + """__init__(*args, **kwargs) + Overloaded function. + + 1. __init__(self: pybind11_fixtures.demo.Point) -> None + + 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None + """ + def as_list(self) -> list[float]: + """as_list(self: pybind11_fixtures.demo.Point) -> List[float]""" + @overload + def distance_to(self, x: float, y: float) -> float: + """distance_to(*args, **kwargs) + Overloaded function. + + 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float + + 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float + """ + @overload + def distance_to(self, other: Point) -> float: + """distance_to(*args, **kwargs) + Overloaded function. + + 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float + + 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float + """ + @property + def length(self) -> float: ... + +def answer() -> int: + '''answer() -> int + + answer docstring, with end quote" + ''' +def midpoint(left: float, right: float) -> float: + """midpoint(left: float, right: float) -> float""" +def sum(arg0: int, arg1: int) -> int: + '''sum(arg0: int, arg1: int) -> int + + multiline docstring test, edge case quotes """\'\'\' + ''' +def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float: + """weighted_midpoint(left: float, right: float, alpha: float = 0.5) -> float""" diff --git a/test-data/pybind11_fixtures/pyproject.toml b/test-data/pybind11_fixtures/pyproject.toml new file mode 100644 index 000000000000..773d036e62f5 --- /dev/null +++ b/test-data/pybind11_fixtures/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + # Officially supported pybind11 version. This is pinned to guarantee 100% reproducible CI. + # As a result, the version needs to be bumped manually at will. + "pybind11==2.9.2", +] + +build-backend = "setuptools.build_meta" diff --git a/test-data/pybind11_fixtures/setup.py b/test-data/pybind11_fixtures/setup.py new file mode 100644 index 000000000000..e227b49935ea --- /dev/null +++ b/test-data/pybind11_fixtures/setup.py @@ -0,0 +1,18 @@ +# pybind11 is available at setup time due to pyproject.toml +from pybind11.setup_helpers import Pybind11Extension +from setuptools import setup + +# Documentation: https://pybind11.readthedocs.io/en/stable/compiling.html +ext_modules = [ + Pybind11Extension( + "pybind11_fixtures", + ["src/main.cpp"], + cxx_std=17, + ), +] + +setup( + name="pybind11_fixtures", + version="0.0.1", + ext_modules=ext_modules, +) diff --git a/test-data/pybind11_fixtures/src/main.cpp b/test-data/pybind11_fixtures/src/main.cpp new file mode 100644 index 000000000000..4d275ab1fd70 --- /dev/null +++ b/test-data/pybind11_fixtures/src/main.cpp @@ -0,0 +1,279 @@ +/** + * This file contains the pybind11 reference implementation for the stugen tests, + * and was originally inspired by: + * + * https://github.com/sizmailov/pybind11-mypy-demo + * + * Copyright (c) 2016 The Pybind Development Team, All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You are under no obligation whatsoever to provide any bug fixes, patches, or + * upgrades to the features, functionality or performance of the source code + * ("Enhancements") to anyone; however, if you choose to make your Enhancements + * available either publicly, or directly to the author of this software, without + * imposing a separate written license agreement for such Enhancements, then you + * hereby grant the following license: a non-exclusive, royalty-free perpetual + * license to install, use, modify, prepare derivative works, incorporate into + * other computer software, distribute, and sublicense such enhancements or + * derivative works thereof, in binary and source code form. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; + +// ---------------------------------------------------------------------------- +// Dedicated test cases +// ---------------------------------------------------------------------------- + +std::vector funcReturningVector() +{ + return std::vector{1.0, 2.0, 3.0}; +} + +std::pair funcReturningPair() +{ + return std::pair{42, 1.0}; +} + +std::optional funcReturningOptional() +{ + return std::nullopt; +} + +std::filesystem::path funcReturningPath() +{ + return std::filesystem::path{"foobar"}; +} + +namespace dummy_sub_namespace { + struct HasNoBinding{}; +} + +// We can enforce the case of an incomplete signature by referring to a type in +// some namespace that doesn't have a pybind11 binding. +dummy_sub_namespace::HasNoBinding funcIncompleteSignature() +{ + return dummy_sub_namespace::HasNoBinding{}; +} + +struct TestStruct +{ + int field_readwrite; + int field_readwrite_docstring; + int field_readonly; +}; + +struct StaticMethods +{ + static int some_static_method(int a, int b) { return 42; } + static int overloaded_static_method(int value) { return 42; } + static double overloaded_static_method(double value) { return 1.0; } +}; + +// Bindings + +void bind_test_cases(py::module& m) { + m.def("func_returning_vector", &funcReturningVector); + m.def("func_returning_pair", &funcReturningPair); + m.def("func_returning_optional", &funcReturningOptional); + m.def("func_returning_path", &funcReturningPath); + + m.def("func_incomplete_signature", &funcIncompleteSignature); + + py::class_(m, "TestStruct") + .def_readwrite("field_readwrite", &TestStruct::field_readwrite) + .def_readwrite("field_readwrite_docstring", &TestStruct::field_readwrite_docstring, "some docstring") + .def_property_readonly( + "field_readonly", + [](const TestStruct& x) { + return x.field_readonly; + }, + "some docstring"); + + // Static methods + py::class_ pyStaticMethods(m, "StaticMethods"); + + pyStaticMethods + .def_static( + "some_static_method", + &StaticMethods::some_static_method, R"#(None)#", py::arg("a"), py::arg("b")) + .def_static( + "overloaded_static_method", + py::overload_cast(&StaticMethods::overloaded_static_method), py::arg("value")) + .def_static( + "overloaded_static_method", + py::overload_cast(&StaticMethods::overloaded_static_method), py::arg("value")); +} + +// ---------------------------------------------------------------------------- +// Original demo +// ---------------------------------------------------------------------------- + +namespace demo { + +int answer() { + return 42; +} + +int sum(int a, int b) { + return a + b; +} + +double midpoint(double left, double right){ + return left + (right - left)/2; +} + +double weighted_midpoint(double left, double right, double alpha=0.5) { + return left + (right - left) * alpha; +} + +struct Point { + + enum class LengthUnit { + mm=0, + pixel, + inch + }; + + enum class AngleUnit { + radian=0, + degree + }; + + Point() : Point(0, 0) {} + Point(double x, double y) : x(x), y(y) {} + + static const Point origin; + static const Point x_axis; + static const Point y_axis; + + static LengthUnit length_unit; + static AngleUnit angle_unit; + + double length() const { + return std::sqrt(x * x + y * y); + } + + double distance_to(double other_x, double other_y) const { + double dx = x - other_x; + double dy = y - other_y; + return std::sqrt(dx*dx + dy*dy); + } + + double distance_to(const Point& other) const { + return distance_to(other.x, other.y); + } + + std::vector as_vector() + { + return std::vector{x, y}; + } + + double x, y; +}; + +const Point Point::origin = Point(0, 0); +const Point Point::x_axis = Point(1, 0); +const Point Point::y_axis = Point(0, 1); + +Point::LengthUnit Point::length_unit = Point::LengthUnit::mm; +Point::AngleUnit Point::angle_unit = Point::AngleUnit::radian; + +} // namespace: demo + +// Bindings + +void bind_demo(py::module& m) { + + using namespace demo; + + // Functions + m.def("answer", &answer, "answer docstring, with end quote\""); // tests explicit docstrings + m.def("sum", &sum, "multiline docstring test, edge case quotes \"\"\"'''"); + m.def("midpoint", &midpoint, py::arg("left"), py::arg("right")); + m.def("weighted_midpoint", weighted_midpoint, py::arg("left"), py::arg("right"), py::arg("alpha")=0.5); + + // Classes + py::class_ pyPoint(m, "Point"); + py::enum_ pyLengthUnit(pyPoint, "LengthUnit"); + py::enum_ pyAngleUnit(pyPoint, "AngleUnit"); + + pyPoint + .def(py::init<>()) + .def(py::init(), py::arg("x"), py::arg("y")) + .def("distance_to", py::overload_cast(&Point::distance_to, py::const_), py::arg("x"), py::arg("y")) + .def("distance_to", py::overload_cast(&Point::distance_to, py::const_), py::arg("other")) + .def("as_list", &Point::as_vector) + .def_readwrite("x", &Point::x, "some docstring") + .def_property("y", + [](Point& self){ return self.y; }, + [](Point& self, double value){ self.y = value; } + ) + .def_property_readonly("length", &Point::length) + .def_property_readonly_static("x_axis", [](py::object cls){return Point::x_axis;}) + .def_property_readonly_static("y_axis", [](py::object cls){return Point::y_axis;}, "another docstring") + .def_readwrite_static("length_unit", &Point::length_unit) + .def_property_static("angle_unit", + [](py::object& /*cls*/){ return Point::angle_unit; }, + [](py::object& /*cls*/, Point::AngleUnit value){ Point::angle_unit = value; } + ); + + pyPoint.attr("origin") = Point::origin; + + pyLengthUnit + .value("mm", Point::LengthUnit::mm) + .value("pixel", Point::LengthUnit::pixel) + .value("inch", Point::LengthUnit::inch); + + pyAngleUnit + .value("radian", Point::AngleUnit::radian) + .value("degree", Point::AngleUnit::degree); + + // Module-level attributes + m.attr("PI") = std::acos(-1); + m.attr("__version__") = "0.0.1"; +} + +// ---------------------------------------------------------------------------- +// Module entry point +// ---------------------------------------------------------------------------- + +PYBIND11_MODULE(pybind11_fixtures, m) { + bind_test_cases(m); + + auto demo = m.def_submodule("demo"); + bind_demo(demo); +} diff --git a/test-data/unit/README.md b/test-data/unit/README.md new file mode 100644 index 000000000000..aaf774d1b62f --- /dev/null +++ b/test-data/unit/README.md @@ -0,0 +1,231 @@ +Tests +===== + + +Quick Start +----------- + +To add a simple unit test for a new feature you developed, open or create a +`test-data/unit/check-*.test` file with a name that roughly relates to the +feature you added. If you added a new `check-*.test` file, it will be autodiscovered during unittests run. + +Add the test in this format anywhere in the file: + + [case testNewSyntaxBasics] + # flags: --python-version 3.10 + x: int + x = 5 + y: int = 5 + + a: str + a = 5 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + b: str = 5 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + zzz: int + zzz: str # E: Name "zzz" already defined + +- no code here is executed, just type checked +- optional `# flags: ` indicates which flags to use for this unit test +- `# E: abc...` indicates that this line should result in type check error +with text "abc..." +- note a space after `E:` and `flags:` +- `# E:12` adds column number to the expected error +- use `\` to escape the `#` character and indicate that the rest of the line is part of +the error message +- repeating `# E: ` several times in one line indicates multiple expected errors in one line +- `W: ...` and `N: ...` works exactly like `E: ...`, but report a warning and a note respectively +- lines that don't contain the above should cause no type check errors +- optional `[builtins fixtures/...]` tells the type checker to use +`builtins` stubs from the indicated file (see Fixtures section below) +- optional `[out]` is an alternative to the `# E: ` notation: it indicates that +any text after it contains the expected type checking error messages. +Usually, `# E: ` is preferred because it makes it easier to associate the +errors with the code generating them at a glance, and to change the code of +the test without having to change line numbers in `[out]` +- an empty `[out]` section has no effect +- to add tests for a feature that hasn't been implemented yet, append `-xfail` + to the end of the test name +- to run just this test, use `pytest -n0 -k testNewSyntaxBasics` + + +Fixtures +-------- + +The unit tests use minimal stubs for builtins, so a lot of operations are not +possible. You should generally define any needed classes within the test case +instead of relying on builtins, though clearly this is not always an option +(see below for more about stubs in test cases). This way tests run much +faster and don't break if the stubs change. If your test crashes mysteriously +even though the code works when run manually, you should make sure you have +all the stubs you need for your test case, including built-in classes such as +`list` or `dict`, as these are not included by default. + +Where the stubs for builtins come from for a given test: + +- The builtins used by default in unit tests live in + `test-data/unit/lib-stub`. + +- Individual test cases can override the `builtins` stubs by using + `[builtins fixtures/foo.pyi]`; this targets files in `test-data/unit/fixtures`. + Feel free to modify existing files there or create new ones as you deem fit. + +- Test cases can also use `[typing fixtures/typing-full.pyi]` to use a more + complete stub for `typing` that contains the async types, among other things. + +- Feel free to add additional stubs to that `fixtures` directory, but + generally don't expand files in `lib-stub` without first discussing the + addition with other mypy developers, as additions could slow down the test + suite. + +- Some tests choose to customize the standard library in a way that's local to the test: + ``` + [case testFoo] + ... + [file builtins.py] + class int: + def next_fibonacci() -> int: pass + ``` + Another possible syntax is: + ``` + [fixture builtins.py] + ``` + Whether you use `[file ...]` or `[fixture ...]` depends on whether you want + the file to be part of the tested corpus (e.g. contribute to `[out]` section) + or only support the test. + +Running tests and linting +------------------------- + +First install any additional dependencies needed for testing: + + python3 -m pip install -U -r test-requirements.txt + +Configure `pre-commit` to run the linters automatically when you commit: + + pre-commit install + +The unit test suites are driven by the `pytest` framework. To run all mypy tests, +run `pytest` in the mypy repository: + + pytest -q mypy + +This will run all tests, including integration and regression tests, +and will verify that all stubs are valid. This may take several +minutes to run, so you don't want to use this all the time while doing +development. (The `-q` option activates less verbose output that looks +better when running tests using many CPU cores.) + +Test suites for individual components are in the files `mypy/test/test*.py`. + +Note that some tests will be disabled for older python versions. + +If you work on mypyc, you will want to also run mypyc tests: + + pytest -q mypyc + +You can run tests from a specific module directly, a specific suite within a +module, or a test in a suite (even if it's data-driven): + + pytest -q mypy/test/testdiff.py + + pytest -q mypy/test/testsemanal.py::SemAnalTypeInfoSuite + + pytest -n0 mypy/test/testargs.py::ArgSuite::test_coherence + + pytest -n0 mypy/test/testcheck.py::TypeCheckSuite::testCallingVariableWithFunctionType + +To control which tests are run and how, you can use the `-k` switch: + + pytest -q -k "MethodCall" + +You can also run the type checker for manual testing without +installing it by setting up the Python module search path suitably: + + export PYTHONPATH=$PWD + python3 -m mypy PROGRAM.py + +You will have to manually install the `typing` module if you're running Python +3.4 or earlier. + +You can also execute mypy as a module + + python3 -m mypy PROGRAM.py + +You can check a module or string instead of a file: + + python3 -m mypy PROGRAM.py + python3 -m mypy -m MODULE + python3 -m mypy -c 'import MODULE' + +To run mypy on itself: + + python3 -m mypy --config-file mypy_self_check.ini -p mypy + +To run the linter (this commands just wraps `pre-commit`, so you can also +invoke it directly like `pre-commit run -a`, and this will also run when you +`git commit` if enabled): + + python3 runtests.py lint + +You can also run all of the above tests using `runtests.py` (this includes +type checking mypy and linting): + + python3 runtests.py + +By default, this runs everything except some mypyc tests. You can give it +arguments to control what gets run, such as `self` to run mypy on itself: + + python3 runtests.py self + +Run `python3 runtests.py mypyc-extra` to run mypyc tests that are not +enabled by default. This is typically only needed if you work on mypyc. + +Many test suites store test case descriptions in text files +(`test-data/unit/*.test`). The module `mypy.test.data` parses these +descriptions. + +Python evaluation test cases are a little different from unit tests +(`mypy/test/testpythoneval.py`, `test-data/unit/pythoneval.test`). These +type check programs and run them. Unlike the unit tests, these use the +full builtins and library stubs instead of minimal ones. Run them using +`pytest -k testpythoneval`. + +`pytest` determines the number of processes to use. The default (set in +`./pytest.ini`) is the number of logical cores; this can be overridden using +`-n` option. To run a single process, use `pytest -n0`. + +Note that running more processes than logical cores is likely to +significantly decrease performance. + +To run tests with coverage: + + python3 -m pytest --cov mypy --cov-config setup.cfg --cov-report=term-missing:skip-covered --cov-report=html + + +Debugging +--------- + +You can use interactive debuggers like `pdb` to debug failing tests. You +need to pass the `-n0` option to disable parallelization: + + pytest -n0 --pdb -k MethodCall + +You can also write `import pdb; pdb.set_trace()` in code to enter the +debugger. + +The `--mypy-verbose` flag can be used to enable additional debug output from +most tests (as if `--verbose` had been passed to mypy): + + pytest -n0 --mypy-verbose -k MethodCall + +Coverage reports +---------------- + +There is an experimental feature to generate coverage reports. To use +this feature, you need to `pip install -U lxml`. This is an extension +module and requires various library headers to install; on a +Debian-derived system the command + `apt-get install python3-dev libxml2-dev libxslt1-dev` +may provide the necessary dependencies. + +To use the feature, pass e.g. `--txt-report "$(mktemp -d)"`. diff --git a/test-data/unit/check-abstract.test b/test-data/unit/check-abstract.test new file mode 100644 index 000000000000..3b0b9c520b75 --- /dev/null +++ b/test-data/unit/check-abstract.test @@ -0,0 +1,1689 @@ +-- Type checker test cases for abstract classes. + + +-- Subtyping with abstract classes +-- ------------------------------- + + +[case testAbstractClassSubclasses] + +from abc import abstractmethod, ABCMeta + +i: I +j: J +a: A +b: B +c: C + +def f(): i, j, a, b, c # Prevent redefinition + +j = c # E: Incompatible types in assignment (expression has type "C", variable has type "J") +a = i # E: Incompatible types in assignment (expression has type "I", variable has type "A") +a = j # E: Incompatible types in assignment (expression has type "J", variable has type "A") +b = i # E: Incompatible types in assignment (expression has type "I", variable has type "B") + +i = a +i = b +i = c +j = a +j = b +a = b + +class I(metaclass=ABCMeta): + @abstractmethod + def f(self): pass +class J(metaclass=ABCMeta): + @abstractmethod + def g(self): pass +class A(I, J): pass +class B(A): pass +class C(I): pass +[builtins fixtures/tuple.pyi] + +[case testAbstractClassSubtypingViaExtension] + +from abc import abstractmethod, ABCMeta + +i: I +j: J +a: A +o: object + +def f(): i, j, a, o # Prevent redefinition + +j = i # E: Incompatible types in assignment (expression has type "I", variable has type "J") +a = i # E: Incompatible types in assignment (expression has type "I", variable has type "A") +a = j # E: Incompatible types in assignment (expression has type "J", variable has type "A") +i = o # E: Incompatible types in assignment (expression has type "object", variable has type "I") +j = o # E: Incompatible types in assignment (expression has type "object", variable has type "J") + +i = a +j = a +i = j +o = i +o = j + +class I(metaclass=ABCMeta): + @abstractmethod + def f(self): pass +class J(I): pass +class A(J): pass +[builtins fixtures/tuple.pyi] + +[case testInheritingAbstractClassInSubclass] +from abc import abstractmethod, ABCMeta + +i: I +a: A +b: B + +if int(): + i = a # E: Incompatible types in assignment (expression has type "A", variable has type "I") +if int(): + b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a = b +if int(): + i = b + +class I(metaclass=ABCMeta): + @abstractmethod + def f(self): pass +class A: pass +class B(A, I): pass + + +-- Abstract class objects +-- ---------------------- + + +[case testAbstractClassAsTypeObject] + +from abc import abstractmethod, ABCMeta + +class I(metaclass=ABCMeta): + @abstractmethod + def f(self): pass + +o: object +t: type + +o = I +t = I + +[case testAbstractClassInCasts] +from typing import cast +from abc import abstractmethod, ABCMeta + +class I(metaclass=ABCMeta): + @abstractmethod + def f(self): pass +class A(I): pass +class B: pass + +i: I +a: A +b: B +o: object + +if int(): + a = cast(I, o) # E: Incompatible types in assignment (expression has type "I", variable has type "A") +if int(): + b = cast(B, i) # Ok; a subclass of B might inherit I +if int(): + i = cast(I, b) # Ok; a subclass of B might inherit I + +if int(): + i = cast(I, o) +if int(): + i = cast(I, a) +[builtins fixtures/tuple.pyi] + +[case testInstantiatingClassThatImplementsAbstractMethod] +from abc import abstractmethod, ABCMeta +import typing +class A(metaclass=ABCMeta): + @abstractmethod + def f(self): pass +class B(A): + def f(self): pass +B() +[out] + +[case testInstantiatingAbstractClass] +from abc import abstractmethod, ABCMeta +import typing +class A(metaclass=ABCMeta): pass +class B(metaclass=ABCMeta): + @abstractmethod + def f(self): pass +A() # OK +B() # E: Cannot instantiate abstract class "B" with abstract attribute "f" +[out] + +[case testInstantiatingClassWithInheritedAbstractMethod] +from abc import abstractmethod, ABCMeta +import typing +class A(metaclass=ABCMeta): + @abstractmethod + def f(self): pass + @abstractmethod + def g(self): pass +class B(A): pass +B() # E: Cannot instantiate abstract class "B" with abstract attributes "f" and "g" +[out] + +[case testInstantiationAbstractsInTypeForFunctions] +from typing import Type +from abc import abstractmethod + +class A: + @abstractmethod + def m(self) -> None: pass +class B(A): pass +class C(B): + def m(self) -> None: + pass + +def f(cls: Type[A]) -> A: + return cls() # OK +def g() -> A: + return A() # E: Cannot instantiate abstract class "A" with abstract attribute "m" + +f(A) # E: Only concrete class can be given where "Type[A]" is expected +f(B) # E: Only concrete class can be given where "Type[A]" is expected +f(C) # OK +x: Type[B] +f(x) # OK +[out] + +[case testAbstractTypeInADict] +from typing import Dict, Type +from abc import abstractmethod + +class Class: + @abstractmethod + def method(self) -> None: + pass + +my_dict_init: Dict[int, Type[Class]] = {0: Class} # E: Only concrete class can be given where "Tuple[int, Type[Class]]" is expected + +class Child(Class): + def method(self) -> None: ... + +other_dict_init: Dict[int, Type[Class]] = {0: Child} # ok +[builtins fixtures/dict.pyi] +[out] + +[case testInstantiationAbstractsInTypeForAliases] +from typing import Type +from abc import abstractmethod + +class A: + @abstractmethod + def m(self) -> None: pass +class B(A): pass +class C(B): + def m(self) -> None: + pass + +def f(cls: Type[A]) -> A: + return cls() # OK + +Alias = A +GoodAlias = C +Alias() # E: Cannot instantiate abstract class "A" with abstract attribute "m" +GoodAlias() +f(Alias) # E: Only concrete class can be given where "Type[A]" is expected +f(GoodAlias) +[out] + +[case testInstantiationAbstractsInTypeForVariables] +# flags: --no-strict-optional +from typing import Type, overload +from abc import abstractmethod + +class A: + @abstractmethod + def m(self) -> None: pass +class B(A): pass +class C(B): + def m(self) -> None: + pass + +var: Type[A] +var() +if int(): + var = A # E: Can only assign concrete classes to a variable of type "Type[A]" +if int(): + var = B # E: Can only assign concrete classes to a variable of type "Type[A]" +if int(): + var = C # OK + +var_old = None # type: Type[A] # Old syntax for variable annotations +var_old() +if int(): + var_old = A # E: Can only assign concrete classes to a variable of type "Type[A]" +if int(): + var_old = B # E: Can only assign concrete classes to a variable of type "Type[A]" +if int(): + var_old = C # OK + +class D(A): + @overload + def __new__(cls, a) -> "D": ... + @overload + def __new__(cls) -> "D": ... + def __new__(cls, a=None) -> "D": ... +if int(): + var = D # E: Can only assign concrete classes to a variable of type "Type[A]" +[out] + +[case testInstantiationAbstractsInTypeForClassMethods] +from typing import Type +from abc import abstractmethod + +class Logger: + @staticmethod + def log(a: Type[C]): + pass +class C: + @classmethod + def action(cls) -> None: + cls() #OK for classmethods + Logger.log(cls) #OK for classmethods + @abstractmethod + def m(self) -> None: + pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testInstantiatingClassWithInheritedAbstractMethodAndSuppression] +from abc import abstractmethod, ABCMeta +import typing +class A(metaclass=ABCMeta): + @abstractmethod + def a(self): pass + @abstractmethod + def b(self): pass + @abstractmethod + def c(self): pass + @abstractmethod + def d(self): pass + @abstractmethod + def e(self): pass + @abstractmethod + def f(self): pass + @abstractmethod + def g(self): pass + @abstractmethod + def h(self): pass + @abstractmethod + def i(self): pass + @abstractmethod + def j(self): pass +a = A() # E: Cannot instantiate abstract class "A" with abstract attributes "a", "b", ... and "j" (7 methods suppressed) +[out] + + +-- Implementing abstract methods +-- ----------------------------- + + +[case testImplementingAbstractMethod] +from abc import abstractmethod, ABCMeta +import typing +class A(metaclass=ABCMeta): + @abstractmethod + def f(self, x: int) -> int: pass + @abstractmethod + def g(self, x: int) -> int: pass +class B(A): + def f(self, x: str) -> int: \ + # E: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + return 0 + def g(self, x: int) -> int: return 0 +[out] + +[case testImplementingAbstractMethodWithMultipleBaseClasses] +from abc import abstractmethod, ABCMeta +import typing +class I(metaclass=ABCMeta): + @abstractmethod + def f(self, x: int) -> int: pass +class J(metaclass=ABCMeta): + @abstractmethod + def g(self, x: str) -> str: pass +class A(I, J): + def f(self, x: str) -> int: return 0 \ + # E: Argument 1 of "f" is incompatible with supertype "I"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def g(self, x: str) -> int: return 0 \ + # E: Return type "int" of "g" incompatible with return type "str" in supertype "J" + def h(self) -> int: return 0 # Not related to any base class +[out] + +[case testImplementingAbstractMethodWithExtension] +from abc import abstractmethod, ABCMeta +import typing +class J(metaclass=ABCMeta): + @abstractmethod + def f(self, x: int) -> int: pass +class I(J): pass +class A(I): + def f(self, x: str) -> int: return 0 \ + # E: Argument 1 of "f" is incompatible with supertype "J"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[out] + +[case testInvalidOverridingAbstractMethod] +from abc import abstractmethod, ABCMeta +import typing +class J(metaclass=ABCMeta): + @abstractmethod + def f(self, x: 'J') -> None: pass +class I(J): + @abstractmethod + def f(self, x: 'I') -> None: pass # E: Argument 1 of "f" is incompatible with supertype "J"; supertype defines the argument type as "J" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[out] + +[case testAbstractClassCoAndContraVariance] +from abc import abstractmethod, ABCMeta +import typing +class I(metaclass=ABCMeta): + @abstractmethod + def f(self, a: A) -> 'I': pass + @abstractmethod + def g(self, a: A) -> 'I': pass + @abstractmethod + def h(self, a: 'I') -> A: pass +class A(I): + def h(self, a: 'A') -> 'I': # Fail + return A() + def f(self, a: 'I') -> 'I': + return A() + def g(self, a: 'A') -> 'A': + return A() +[out] +main:11: error: Return type "I" of "h" incompatible with return type "A" in supertype "I" +main:11: error: Argument 1 of "h" is incompatible with supertype "I"; supertype defines the argument type as "I" +main:11: note: This violates the Liskov substitution principle +main:11: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + + +-- Accessing abstract members +-- -------------------------- + + +[case testAccessingAbstractMethod] +from abc import abstractmethod, ABCMeta + +class I(metaclass=ABCMeta): + @abstractmethod + def f(self, a: int) -> str: pass + +i: I +a: int +b: str + +if int(): + a = i.f(a) # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + b = i.f(b) # E: Argument 1 to "f" of "I" has incompatible type "str"; expected "int" +i.g() # E: "I" has no attribute "g" + +if int(): + b = i.f(a) +[builtins fixtures/tuple.pyi] + +[case testAccessingInheritedAbstractMethod] +from abc import abstractmethod, ABCMeta + +class J(metaclass=ABCMeta): + @abstractmethod + def f(self, a: int) -> str: pass +class I(J): pass + +i: I +a: int +b: str + +if int(): + a = i.f(1) # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + b = i.f(1) + + +-- Any (dynamic) types +-- ------------------- +[builtins fixtures/tuple.pyi] + + +[case testAbstractClassWithAllDynamicTypes] +from abc import abstractmethod, ABCMeta +import typing +class I(metaclass=ABCMeta): + @abstractmethod + def f(self, x): pass + @abstractmethod + def g(self, x): pass +class A(I): + def f(self, x): pass + def g(self, x, y) -> None: pass # Fail +[out] +main:10: error: Signature of "g" incompatible with supertype "I" +main:10: note: Superclass: +main:10: note: def g(self, x: Any) -> Any +main:10: note: Subclass: +main:10: note: def g(self, x: Any, y: Any) -> None + +[case testAbstractClassWithAllDynamicTypes2] +from abc import abstractmethod, ABCMeta +import typing +class I(metaclass=ABCMeta): + @abstractmethod + def f(self, x): pass + @abstractmethod + def g(self, x): pass +class A(I): + def f(self, x): pass + def g(self, x, y): pass +[out] + +[case testAbstractClassWithImplementationUsingDynamicTypes] +from abc import abstractmethod, ABCMeta +import typing +class I(metaclass=ABCMeta): + @abstractmethod + def f(self, x: int) -> None: pass + @abstractmethod + def g(self, x: int) -> None: pass +class A(I): + def f(self, x): pass + def g(self, x, y): pass +[out] + + +-- Special cases +-- ------------- + + +[case testMultipleAbstractBases] +from abc import abstractmethod, ABCMeta +import typing +class A(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass +class B(metaclass=ABCMeta): + @abstractmethod + def g(self) -> None: pass +class C(A, B): + @abstractmethod + def h(self) -> None: pass + +[case testMemberAccessWithMultipleAbstractBaseClasses] + +from abc import abstractmethod, ABCMeta + +class A(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass +class B(metaclass=ABCMeta): + @abstractmethod + def g(self) -> None: pass +class C(A, B): pass +x: C +x.f() +x.g() +x.f(x) # E: Too many arguments for "f" of "A" +x.g(x) # E: Too many arguments for "g" of "B" + +[case testInstantiatingAbstractClassWithMultipleBaseClasses] + +from abc import abstractmethod, ABCMeta + +class A(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass +class B(metaclass=ABCMeta): + @abstractmethod + def g(self) -> None: pass +class C(A, B): + def f(self) -> None: pass +class D(A, B): + def g(self) -> None: pass +class E(A, B): + def f(self) -> None: pass + def g(self) -> None: pass +C() # E: Cannot instantiate abstract class "C" with abstract attribute "g" +D() # E: Cannot instantiate abstract class "D" with abstract attribute "f" +E() + +[case testInconsistentMro] +from abc import abstractmethod, ABCMeta +import typing + +class A(metaclass=ABCMeta): pass +class B(object, A): pass \ + # E: Cannot determine consistent method resolution order (MRO) for "B" + +[case testOverloadedAbstractMethod] +from foo import * +[file foo.pyi] +from abc import abstractmethod, ABCMeta +from typing import overload + +class A(metaclass=ABCMeta): + @abstractmethod + @overload + def f(self, x: int) -> int: pass + @abstractmethod + @overload + def f(self, x: str) -> str: pass + +class B(A): + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass +A() # E: Cannot instantiate abstract class "A" with abstract attribute "f" +B() +B().f(1) +a = B() # type: A +a.f(1) +a.f('') +a.f(B()) # E: No overload variant of "f" of "A" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f(self, x: int) -> int \ + # N: def f(self, x: str) -> str + +[case testOverloadedAbstractMethodWithAlternativeDecoratorOrder] +from foo import * +[file foo.pyi] +from abc import abstractmethod, ABCMeta +from typing import overload + +class A(metaclass=ABCMeta): + @overload + @abstractmethod + def f(self, x: int) -> int: pass + @overload + @abstractmethod + def f(self, x: str) -> str: pass + +class B(A): + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass +A() # E: Cannot instantiate abstract class "A" with abstract attribute "f" +B() +B().f(1) +a = B() # type: A +a.f(1) +a.f('') +a.f(B()) # E: No overload variant of "f" of "A" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f(self, x: int) -> int \ + # N: def f(self, x: str) -> str + +[case testOverloadedAbstractMethodVariantMissingDecorator0] +from foo import * +[file foo.pyi] +from abc import abstractmethod, ABCMeta +from typing import overload + +class A(metaclass=ABCMeta): + @abstractmethod \ + # E: Overloaded method has both abstract and non-abstract variants + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass +[out] + +[case testOverloadedAbstractMethodVariantMissingDecorator1] +from foo import * +[file foo.pyi] +from abc import abstractmethod, ABCMeta +from typing import overload + +class A(metaclass=ABCMeta): + @overload \ + # E: Overloaded method has both abstract and non-abstract variants + def f(self, x: int) -> int: pass + @abstractmethod + @overload + def f(self, x: str) -> str: pass +[out] + +[case testMultipleInheritanceAndAbstractMethod] +import typing +from abc import abstractmethod, ABCMeta +class A: + def f(self, x: str) -> None: pass +class B(metaclass=ABCMeta): + @abstractmethod + def f(self, x: str) -> None: pass +class C(A, B): pass + +[case testMultipleInheritanceAndAbstractMethod2] +import typing +from abc import abstractmethod, ABCMeta +class A: + def f(self, x: str) -> None: pass +class B(metaclass=ABCMeta): + @abstractmethod + def f(self, x: int) -> None: pass +class C(A, B): pass +[out] +main:8: error: Definition of "f" in base class "A" is incompatible with definition in base class "B" + +[case testCallAbstractMethodBeforeDefinition] +import typing +from abc import abstractmethod, ABCMeta +class A(metaclass=ABCMeta): + def f(self) -> None: + self.g(1) # E: Argument 1 to "g" of "A" has incompatible type "int"; expected "str" + @abstractmethod + def g(self, x: str) -> None: pass +[out] + +[case testAbstractOperatorMethods1] +import typing +from abc import abstractmethod, ABCMeta +class A(metaclass=ABCMeta): + @abstractmethod + def __lt__(self, other: 'A') -> int: pass + @abstractmethod + def __gt__(self, other: 'A') -> int: pass + +[case testAbstractOperatorMethods2] +from typing import cast, Any +from abc import abstractmethod, ABCMeta +class A(metaclass=ABCMeta): + @abstractmethod + def __radd__(self, other: 'C') -> str: pass # Error +class B: + @abstractmethod + def __add__(self, other: 'A') -> int: pass +class C: + def __add__(self, other: int) -> B: + return cast(Any, None) +[out] + +[case testAbstractClassWithAnyBase] +from typing import Any +from abc import abstractmethod, ABCMeta + +A: Any + +class D(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + +class C(A, D): + pass + +C() # A might implement 'f' + + +-- Abstract properties +-- ------------------- + + +[case testReadOnlyAbstractProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +def f(a: A) -> None: + a.x() # E: "int" not callable + a.x = 1 # E: Property "x" defined in "A" is read-only +[out] + +[case testReadOnlyAbstractPropertyForwardRef] +from abc import abstractproperty, ABCMeta +def f(a: A) -> None: + a.x() # E: "int" not callable + a.x = 1 # E: Property "x" defined in "A" is read-only +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +[out] + +[case testReadWriteAbstractProperty] +from abc import abstractproperty, ABCMeta +def f(a: A) -> None: + a.x.y # E: "int" has no attribute "y" + a.x = 1 +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass + @x.setter + def x(self, x: int) -> None: pass + +[case testReadWriteDeleteAbstractProperty] +# flags: --no-strict-optional +from abc import ABC, abstractmethod +class Abstract(ABC): + @property + @abstractmethod + def prop(self) -> str: ... + + @prop.setter + @abstractmethod + def prop(self, code: str) -> None: ... + + @prop.deleter + @abstractmethod + def prop(self) -> None: ... + +class Good(Abstract): + @property + def prop(self) -> str: ... + @prop.setter + def prop(self, code: str) -> None: ... + @prop.deleter + def prop(self) -> None: ... + +class Bad1(Abstract): + @property # E: Read-only property cannot override read-write property + def prop(self) -> str: ... + +class ThisShouldProbablyError(Abstract): + @property + def prop(self) -> str: ... + @prop.setter + def prop(self, code: str) -> None: ... + +a = Good() +reveal_type(a.prop) # N: Revealed type is "builtins.str" +a.prop = 123 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/property.pyi] + +[case testInstantiateClassWithReadOnlyAbstractProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +class B(A): pass +b = B() # E: Cannot instantiate abstract class "B" with abstract attribute "x" + +[case testInstantiateClassWithReadWriteAbstractProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass + @x.setter + def x(self, x: int) -> None: pass +class B(A): pass +b = B() # E: Cannot instantiate abstract class "B" with abstract attribute "x" + +[case testImplementAbstractPropertyViaProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +class B(A): + @property + def x(self) -> int: return 0 +b = B() +b.x() # E: "int" not callable +[builtins fixtures/property.pyi] + +[case testImplementReadWriteAbstractPropertyViaProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass + @x.setter + def x(self, v: int) -> None: pass +class B(A): + @property + def x(self) -> int: return 0 + @x.setter + def x(self, v: int) -> None: pass +b = B() +b.x.y # E: "int" has no attribute "y" +[builtins fixtures/property.pyi] + +[case testImplementAbstractPropertyViaPropertyInvalidType] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +class B(A): + @property + def x(self) -> str: return "no" # E: Signature of "x" incompatible with supertype "A" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: str +b = B() +b.x() # E: "str" not callable +[builtins fixtures/property.pyi] + +[case testCantImplementAbstractPropertyViaInstanceVariable] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +class B(A): + def __init__(self) -> None: + self.x = 1 # E +b = B() # E +b.x.y # E +[builtins fixtures/property.pyi] +[out] +main:7: error: Property "x" defined in "A" is read-only +main:8: error: Cannot instantiate abstract class "B" with abstract attribute "x" +main:9: error: "int" has no attribute "y" + +[case testSuperWithAbstractProperty] +# flags: --no-strict-optional +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +class B(A): + @property + def x(self) -> int: + return super().x.y # E: Call to abstract method "x" of "A" with trivial body via super() is unsafe \ + # E: "int" has no attribute "y" +[builtins fixtures/property.pyi] + +[case testSuperWithReadWriteAbstractProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass + @x.setter + def x(self, v: int) -> None: pass +class B(A): + @property + def x(self) -> int: + return super().x.y # E + @x.setter + def x(self, v: int) -> None: + super().x = '' # E +[builtins fixtures/property.pyi] +[out] +main:10: error: "int" has no attribute "y" +main:13: error: Invalid assignment target + +[case testOnlyImplementGetterOfReadWriteAbstractProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass + @x.setter + def x(self, v: int) -> None: pass +class B(A): + @property # E + def x(self) -> int: return 0 +b = B() +b.x.y # E +[builtins fixtures/property.pyi] +[out] +main:8: error: Read-only property cannot override read-write property +main:11: error: "int" has no attribute "y" + +[case testDynamicallyTypedReadOnlyAbstractProperty] +from abc import abstractproperty, ABCMeta +class A(metaclass=ABCMeta): + @abstractproperty + def x(self): pass +def f(a: A) -> None: + a.x.y + a.x = 1 # E: Property "x" defined in "A" is read-only +[out] + +[case testDynamicallyTypedReadOnlyAbstractPropertyForwardRef] +from abc import abstractproperty, ABCMeta +def f(a: A) -> None: + a.x.y + a.x = 1 # E: Property "x" defined in "A" is read-only +class A(metaclass=ABCMeta): + @abstractproperty + def x(self): pass +[out] + +[case testDynamicallyTypedReadWriteAbstractProperty] +from abc import abstractproperty, ABCMeta +def f(a: A) -> None: + a.x.y + a.x = 1 +class A(metaclass=ABCMeta): + @abstractproperty + def x(self): pass + @x.setter + def x(self, x): pass +[out] + +[case testMixinTypedAbstractProperty] +from abc import ABCMeta, abstractproperty +class A(metaclass=ABCMeta): + @abstractproperty + def foo(cls) -> str: + pass +class Mixin: + foo = "foo" +class C(Mixin, A): + pass +[out] + +[case testMixinTypedProperty] +class A: + @property + def foo(cls) -> str: + return "yes" +class Mixin: + foo = "foo" +class C(Mixin, A): + pass +[builtins fixtures/property.pyi] +[out] + +[case testMixinSubtypedProperty] +class X: + pass +class Y(X): + pass +class A: + @property + def foo(cls) -> X: + return X() +class Mixin: + foo = Y() +class C(Mixin, A): + pass +[builtins fixtures/property.pyi] +[out] + +[case testMixinTypedPropertyReversed] +class A: + @property + def foo(cls) -> str: + return "no" +class Mixin: + foo = "foo" +class C(A, Mixin): # E: Definition of "foo" in base class "A" is incompatible with definition in base class "Mixin" + pass +[builtins fixtures/property.pyi] +[out] + +-- Special cases +-- ------------- + + +[case testNestedAbstractClass] +from abc import abstractmethod, ABCMeta + +class A: + class B(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + + class C(B): pass + +A.B() # E: Cannot instantiate abstract class "B" with abstract attribute "f" +A.C() # E: Cannot instantiate abstract class "C" with abstract attribute "f" + +[case testAbstractNewTypeAllowed] +from typing import NewType, Mapping + +Config = NewType('Config', Mapping[str, str]) + +bad = Mapping[str, str]() # E: Cannot instantiate abstract class "Mapping" with abstract attribute "__iter__" +default = Config({'cannot': 'modify'}) # OK + +default[1] = 2 # E: Unsupported target for indexed assignment ("Config") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testSubclassOfABCFromDictionary] +from abc import abstractmethod, ABCMeta + +class MyAbstractType(metaclass=ABCMeta): + @abstractmethod + def do(self): pass + +class MyConcreteA(MyAbstractType): + def do(self): + print('A') + +class MyConcreteB(MyAbstractType): + def do(self): + print('B') + +class MyAbstractA(MyAbstractType): + @abstractmethod + def do(self): pass + +class MyAbstractB(MyAbstractType): + @abstractmethod + def do(self): pass + +my_concrete_types = { + 'A': MyConcreteA, + 'B': MyConcreteB, +} + +my_abstract_types = { + 'A': MyAbstractA, + 'B': MyAbstractB, +} + +reveal_type(my_concrete_types) # N: Revealed type is "builtins.dict[builtins.str, def () -> __main__.MyAbstractType]" +reveal_type(my_abstract_types) # N: Revealed type is "builtins.dict[builtins.str, def () -> __main__.MyAbstractType]" + +a = my_concrete_types['A']() +a.do() +b = my_concrete_types['B']() +b.do() + +c = my_abstract_types['A']() # E: Cannot instantiate abstract class "MyAbstractType" with abstract attribute "do" +c.do() +d = my_abstract_types['B']() # E: Cannot instantiate abstract class "MyAbstractType" with abstract attribute "do" +d.do() + +[builtins fixtures/dict.pyi] + +[case testAbstractClassesWorkWithGenericDecorators] +from abc import abstractmethod, ABCMeta +from typing import Type, TypeVar + +T = TypeVar("T") +def deco(cls: Type[T]) -> Type[T]: return cls + +@deco +class A(metaclass=ABCMeta): + @abstractmethod + def foo(self, x: int) -> None: ... + +[case testAbstractPropertiesAllowed] +from abc import abstractmethod + +class B: + @property + @abstractmethod + def x(self) -> int: ... + @property + @abstractmethod + def y(self) -> int: ... + @y.setter + @abstractmethod + def y(self, value: int) -> None: ... + +B() # E: Cannot instantiate abstract class "B" with abstract attributes "x" and "y" +b: B +b.x = 1 # E: Property "x" defined in "B" is read-only +b.y = 1 +[builtins fixtures/property.pyi] + + +-- Treatment of empty bodies in ABCs and protocols +-- ----------------------------------------------- + +[case testEmptyBodyProhibitedFunction] +from typing import overload, Union + +def func1(x: str) -> int: pass # E: Missing return statement +def func2(x: str) -> int: ... # E: Missing return statement +def func3(x: str) -> int: # E: Missing return statement + """Some function.""" + +@overload +def func4(x: int) -> int: ... +@overload +def func4(x: str) -> str: ... +def func4(x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + pass + +@overload +def func5(x: int) -> int: ... +@overload +def func5(x: str) -> str: ... +def func5(x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + """Some function.""" + +[case testEmptyBodyProhibitedMethodNonAbstract] +from typing import overload, Union + +class A: + def func1(self, x: str) -> int: pass # E: Missing return statement + def func2(self, x: str) -> int: ... # E: Missing return statement + def func3(self, x: str) -> int: # E: Missing return statement + """Some function.""" + +class B: + @classmethod + def func1(cls, x: str) -> int: pass # E: Missing return statement + @classmethod + def func2(cls, x: str) -> int: ... # E: Missing return statement + @classmethod + def func3(cls, x: str) -> int: # E: Missing return statement + """Some function.""" + +class C: + @overload + def func4(self, x: int) -> int: ... + @overload + def func4(self, x: str) -> str: ... + def func4(self, x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + pass + + @overload + def func5(self, x: int) -> int: ... + @overload + def func5(self, x: str) -> str: ... + def func5(self, x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + """Some function.""" +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyProhibitedPropertyNonAbstract] +class A: + @property + def x(self) -> int: ... # E: Missing return statement + @property + def y(self) -> int: ... # E: Missing return statement + @y.setter + def y(self, value: int) -> None: ... + +class B: + @property + def x(self) -> int: pass # E: Missing return statement + @property + def y(self) -> int: pass # E: Missing return statement + @y.setter + def y(self, value: int) -> None: pass + +class C: + @property + def x(self) -> int: # E: Missing return statement + """Some property.""" + @property + def y(self) -> int: # E: Missing return statement + """Some property.""" + @y.setter + def y(self, value: int) -> None: pass +[builtins fixtures/property.pyi] + +[case testEmptyBodyNoteABCMeta] +from abc import ABC + +class A(ABC): + def foo(self) -> int: # E: Missing return statement \ + # N: If the method is meant to be abstract, use @abc.abstractmethod + ... + +[case testEmptyBodyAllowedFunctionStub] +import stub +[file stub.pyi] +from typing import overload, Union + +def func1(x: str) -> int: pass +def func2(x: str) -> int: ... +def func3(x: str) -> int: + """Some function.""" + +[case testEmptyBodyAllowedMethodNonAbstractStub] +import stub +[file stub.pyi] +from typing import overload, Union + +class A: + def func1(self, x: str) -> int: pass + def func2(self, x: str) -> int: ... + def func3(self, x: str) -> int: + """Some function.""" + +class B: + @classmethod + def func1(cls, x: str) -> int: pass + @classmethod + def func2(cls, x: str) -> int: ... + @classmethod + def func3(cls, x: str) -> int: + """Some function.""" +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyAllowedPropertyNonAbstractStub] +import stub +[file stub.pyi] +class A: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + @y.setter + def y(self, value: int) -> None: ... + +class B: + @property + def x(self) -> int: pass + @property + def y(self) -> int: pass + @y.setter + def y(self, value: int) -> None: pass + +class C: + @property + def x(self) -> int: + """Some property.""" + @property + def y(self) -> int: + """Some property.""" + @y.setter + def y(self, value: int) -> None: pass +[builtins fixtures/property.pyi] + +[case testEmptyBodyAllowedMethodAbstract] +from typing import overload, Union +from abc import abstractmethod + +class A: + @abstractmethod + def func1(self, x: str) -> int: pass + @abstractmethod + def func2(self, x: str) -> int: ... + @abstractmethod + def func3(self, x: str) -> int: + """Some function.""" + +class B: + @classmethod + @abstractmethod + def func1(cls, x: str) -> int: pass + @classmethod + @abstractmethod + def func2(cls, x: str) -> int: ... + @classmethod + @abstractmethod + def func3(cls, x: str) -> int: + """Some function.""" + +class C: + @overload + @abstractmethod + def func4(self, x: int) -> int: ... + @overload + @abstractmethod + def func4(self, x: str) -> str: ... + @abstractmethod + def func4(self, x: Union[int, str]) -> Union[int, str]: + pass + + @overload + @abstractmethod + def func5(self, x: int) -> int: ... + @overload + @abstractmethod + def func5(self, x: str) -> str: ... + @abstractmethod + def func5(self, x: Union[int, str]) -> Union[int, str]: + """Some function.""" +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyAllowedPropertyAbstract] +from abc import abstractmethod +class A: + @property + @abstractmethod + def x(self) -> int: ... + @property + @abstractmethod + def y(self) -> int: ... + @y.setter + @abstractmethod + def y(self, value: int) -> None: ... + +class B: + @property + @abstractmethod + def x(self) -> int: pass + @property + @abstractmethod + def y(self) -> int: pass + @y.setter + @abstractmethod + def y(self, value: int) -> None: pass + +class C: + @property + @abstractmethod + def x(self) -> int: + """Some property.""" + @property + @abstractmethod + def y(self) -> int: + """Some property.""" + @y.setter + @abstractmethod + def y(self, value: int) -> None: pass +[builtins fixtures/property.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocol] +from typing import Protocol, overload, Union + +class P1(Protocol): + def meth(self) -> int: ... +class B1(P1): ... +class C1(P1): + def meth(self) -> int: + return 0 +B1() # E: Cannot instantiate abstract class "B1" with abstract attribute "meth" +C1() + +class P2(Protocol): + @classmethod + def meth(cls) -> int: ... +class B2(P2): ... +class C2(P2): + @classmethod + def meth(cls) -> int: + return 0 +B2() # E: Cannot instantiate abstract class "B2" with abstract attribute "meth" +C2() + +class P3(Protocol): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +class B3(P3): ... +class C3(P3): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return 0 +B3() # E: Cannot instantiate abstract class "B3" with abstract attribute "meth" +C3() +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocolProperty] +from typing import Protocol + +class P1(Protocol): + @property + def attr(self) -> int: ... +class B1(P1): ... +class C1(P1): + @property + def attr(self) -> int: + return 0 +B1() # E: Cannot instantiate abstract class "B1" with abstract attribute "attr" +C1() + +class P2(Protocol): + @property + def attr(self) -> int: ... + @attr.setter + def attr(self, value: int) -> None: ... +class B2(P2): ... +class C2(P2): + @property + def attr(self) -> int: return 0 + @attr.setter + def attr(self, value: int) -> None: pass +B2() # E: Cannot instantiate abstract class "B2" with abstract attribute "attr" +C2() +[builtins fixtures/property.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocolStub] +from stub import P1, P2, P3, P4 + +class B1(P1): ... +class B2(P2): ... +class B3(P3): ... +class B4(P4): ... + +B1() +B2() +B3() +B4() # E: Cannot instantiate abstract class "B4" with abstract attribute "meth" + +[file stub.pyi] +from typing import Protocol, overload, Union +from abc import abstractmethod + +class P1(Protocol): + def meth(self) -> int: ... + +class P2(Protocol): + @classmethod + def meth(cls) -> int: ... + +class P3(Protocol): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + +class P4(Protocol): + @abstractmethod + def meth(self) -> int: ... +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyUnsafeAbstractSuper] +from stub import StubProto, StubAbstract +from typing import Protocol +from abc import abstractmethod + +class Proto(Protocol): + def meth(self) -> int: ... +class ProtoDef(Protocol): + def meth(self) -> int: return 0 + +class Abstract: + @abstractmethod + def meth(self) -> int: ... +class AbstractDef: + @abstractmethod + def meth(self) -> int: return 0 + +class SubProto(Proto): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Proto" with trivial body via super() is unsafe +class SubProtoDef(ProtoDef): + def meth(self) -> int: + return super().meth() + +class SubAbstract(Abstract): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Abstract" with trivial body via super() is unsafe +class SubAbstractDef(AbstractDef): + def meth(self) -> int: + return super().meth() + +class SubStubProto(StubProto): + def meth(self) -> int: + return super().meth() +class SubStubAbstract(StubAbstract): + def meth(self) -> int: + return super().meth() + +[file stub.pyi] +from typing import Protocol +from abc import abstractmethod + +class StubProto(Protocol): + def meth(self) -> int: ... +class StubAbstract: + @abstractmethod + def meth(self) -> int: ... + +[case testEmptyBodyUnsafeAbstractSuperProperty] +from stub import StubProto, StubAbstract +from typing import Protocol +from abc import abstractmethod + +class Proto(Protocol): + @property + def attr(self) -> int: ... +class SubProto(Proto): + @property + def attr(self) -> int: return super().attr # E: Call to abstract method "attr" of "Proto" with trivial body via super() is unsafe + +class ProtoDef(Protocol): + @property + def attr(self) -> int: return 0 +class SubProtoDef(ProtoDef): + @property + def attr(self) -> int: return super().attr + +class Abstract: + @property + @abstractmethod + def attr(self) -> int: ... +class SubAbstract(Abstract): + @property + @abstractmethod + def attr(self) -> int: return super().attr # E: Call to abstract method "attr" of "Abstract" with trivial body via super() is unsafe + +class AbstractDef: + @property + @abstractmethod + def attr(self) -> int: return 0 +class SubAbstractDef(AbstractDef): + @property + @abstractmethod + def attr(self) -> int: return super().attr + +class SubStubProto(StubProto): + @property + def attr(self) -> int: return super().attr +class SubStubAbstract(StubAbstract): + @property + def attr(self) -> int: return super().attr + +[file stub.pyi] +from typing import Protocol +from abc import abstractmethod + +class StubProto(Protocol): + @property + def attr(self) -> int: ... +class StubAbstract: + @property + @abstractmethod + def attr(self) -> int: ... +[builtins fixtures/property.pyi] + +[case testEmptyBodyUnsafeAbstractSuperOverloads] +from stub import StubProto +from typing import Protocol, overload, Union + +class ProtoEmptyImpl(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + raise NotImplementedError +class ProtoDefImpl(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return 0 +class ProtoNoImpl(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + +class SubProtoEmptyImpl(ProtoEmptyImpl): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) # E: Call to abstract method "meth" of "ProtoEmptyImpl" with trivial body via super() is unsafe +class SubProtoDefImpl(ProtoDefImpl): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) +class SubStubProto(StubProto): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) + +# TODO: it would be good to also give an error in this case. +class SubProtoNoImpl(ProtoNoImpl): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) + +[file stub.pyi] +from typing import Protocol, overload + +class StubProto(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + +[builtins fixtures/exception.pyi] + +[case testEmptyBodyNoSuperWarningWithoutStrict] +# flags: --no-strict-optional +from typing import Protocol +from abc import abstractmethod + +class Proto(Protocol): + def meth(self) -> int: ... +class Abstract: + @abstractmethod + def meth(self) -> int: ... + +class SubProto(Proto): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Proto" with trivial body via super() is unsafe +class SubAbstract(Abstract): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Abstract" with trivial body via super() is unsafe + +[case testEmptyBodyNoSuperWarningOptionalReturn] +from typing import Protocol, Optional +from abc import abstractmethod + +class Proto(Protocol): + def meth(self) -> Optional[int]: pass +class Abstract: + @abstractmethod + def meth(self) -> Optional[int]: pass + +class SubProto(Proto): + def meth(self) -> Optional[int]: + return super().meth() # E: Call to abstract method "meth" of "Proto" with trivial body via super() is unsafe +class SubAbstract(Abstract): + def meth(self) -> Optional[int]: + return super().meth() # E: Call to abstract method "meth" of "Abstract" with trivial body via super() is unsafe + +[case testEmptyBodyTypeCheckingOnly] +from typing import TYPE_CHECKING + +class C: + if TYPE_CHECKING: + def dynamic(self) -> int: ... # OK diff --git a/test-data/unit/check-annotated.test b/test-data/unit/check-annotated.test new file mode 100644 index 000000000000..d4309b8ad213 --- /dev/null +++ b/test-data/unit/check-annotated.test @@ -0,0 +1,159 @@ +[case testAnnotated0] +from typing_extensions import Annotated +x: Annotated[int, ...] +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testAnnotated1] +from typing import Union +from typing_extensions import Annotated +x: Annotated[Union[int, str], ...] +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAnnotated2] +from typing_extensions import Annotated +x: Annotated[int, THESE, ARE, IGNORED, FOR, NOW] +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testAnnotated3] +from typing_extensions import Annotated +x: Annotated[int, -+~12.3, "som"[e], more(anno+a+ions, that=[are]), (b"ignored",), 4, N.O.W, ...] +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedBadType] +from typing_extensions import Annotated +x: Annotated[XXX, ...] # E: Name "XXX" is not defined +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedBadNoArgs] +from typing_extensions import Annotated +x: Annotated # E: Annotated[...] must have exactly one type argument and at least one annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedBadOneArg] +from typing_extensions import Annotated +x: Annotated[int] # E: Annotated[...] must have exactly one type argument and at least one annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedNested0] +from typing_extensions import Annotated +x: Annotated[Annotated[int, ...], ...] +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedNested1] +from typing import Union +from typing_extensions import Annotated +x: Annotated[Annotated[Union[int, str], ...], ...] +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedNestedBadType] +from typing_extensions import Annotated +x: Annotated[Annotated[XXX, ...], ...] # E: Name "XXX" is not defined +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedNestedBadNoArgs] +from typing_extensions import Annotated +x: Annotated[Annotated, ...] # E: Annotated[...] must have exactly one type argument and at least one annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedNestedBadOneArg] +from typing_extensions import Annotated +x: Annotated[Annotated[int], ...] # E: Annotated[...] must have exactly one type argument and at least one annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedNoImport] +x: Annotated[int, ...] # E: Name "Annotated" is not defined +reveal_type(x) # N: Revealed type is "Any" + +[case testAnnotatedDifferentName] +from typing_extensions import Annotated as An +x: An[int, ...] +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedAliasSimple] +from typing import Tuple +from typing_extensions import Annotated +Alias = Annotated[Tuple[int, ...], ...] +x: Alias +reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedAliasTypeVar] +from typing import TypeVar +from typing_extensions import Annotated +T = TypeVar('T') +Alias = Annotated[T, ...] +x: Alias[int] +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedAliasGenericTuple] +from typing import TypeVar, Tuple +from typing_extensions import Annotated +T = TypeVar('T') +Alias = Annotated[Tuple[T, T], ...] +x: Alias[int] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedAliasGenericUnion] +from typing import TypeVar, Union +from typing_extensions import Annotated +T = TypeVar('T') +Alias = Annotated[Union[T, str], ...] +x: Alias[int] +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedSecondParamNonType] +from typing_extensions import Annotated + +class Meta: + ... + +x = Annotated[int, Meta()] +reveal_type(x) # N: Revealed type is "def () -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedStringLiteralInFunc] +from typing import TypeVar +from typing_extensions import Annotated +def f1(a: Annotated[str, "metadata"]): + pass +reveal_type(f1) # N: Revealed type is "def (a: builtins.str) -> Any" +def f2(a: Annotated["str", "metadata"]): + pass +reveal_type(f2) # N: Revealed type is "def (a: builtins.str) -> Any" +def f3(a: Annotated["notdefined", "metadata"]): # E: Name "notdefined" is not defined + pass +T = TypeVar('T') +def f4(a: Annotated[T, "metatdata"]): + pass +reveal_type(f4) # N: Revealed type is "def [T] (a: T`-1) -> Any" +[builtins fixtures/tuple.pyi] + +[case testSliceAnnotated39] +# flags: --python-version 3.9 +from typing_extensions import Annotated +a: Annotated[int, 1:2] +reveal_type(a) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testSliceAnnotated38] +# flags: --python-version 3.8 +from typing_extensions import Annotated +a: Annotated[int, 1:2] +reveal_type(a) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-assert-type-fail.test b/test-data/unit/check-assert-type-fail.test new file mode 100644 index 000000000000..89b3a863f8c7 --- /dev/null +++ b/test-data/unit/check-assert-type-fail.test @@ -0,0 +1,33 @@ +[case testAssertTypeFail1] +import typing +import array as arr +class array: + pass +def f(si: arr.array[int]): + typing.assert_type(si, array) # E: Expression is of type "array.array[int]", not "__main__.array" +[builtins fixtures/tuple.pyi] + +[case testAssertTypeFail2] +import typing +import array as arr +class array: + class array: + i = 1 +def f(si: arr.array[int]): + typing.assert_type(si, array.array) # E: Expression is of type "array.array[int]", not "__main__.array.array" +[builtins fixtures/tuple.pyi] + +[case testAssertTypeFail3] +import typing +import array as arr +class array: + class array: + i = 1 +def f(si: arr.array[int]): + typing.assert_type(si, int) # E: Expression is of type "array[int]", not "int" +[builtins fixtures/tuple.pyi] + +[case testAssertTypeFailCallableArgKind] +from typing import assert_type, Callable +def myfunc(arg: int) -> None: pass +assert_type(myfunc, Callable[[int], None]) # E: Expression is of type "Callable[[Arg(int, 'arg')], None]", not "Callable[[int], None]" diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test new file mode 100644 index 000000000000..0ef08e5a0775 --- /dev/null +++ b/test-data/unit/check-async-await.test @@ -0,0 +1,1082 @@ +-- Tests for async def and await (PEP 492) +-- --------------------------------------- + +[case testAsyncDefPass] + +async def f() -> int: + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncDefReturn] + +async def f() -> int: + return 0 +_ = reveal_type(f()) # N: Revealed type is "typing.Coroutine[Any, Any, builtins.int]" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncDefMissingReturn] +# flags: --warn-no-return +async def f() -> int: + make_this_not_trivial = 1 +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:2: error: Missing return statement + +[case testAsyncDefReturnWithoutValue] + +async def f() -> int: + make_this_not_trivial = 1 + return +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:4: error: Return value expected + +[case testAwaitCoroutine] + +async def f() -> int: + x = await f() + reveal_type(x) # N: Revealed type is "builtins.int" + return x +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] + +[case testAwaitDefaultContext] + +from typing import TypeVar +T = TypeVar('T') +async def f(x: T) -> T: + y = await f(x) + reveal_type(y) + return y +[typing fixtures/typing-async.pyi] +[out] +main:6: note: Revealed type is "T`-1" + +[case testAwaitAnyContext] + +from typing import Any, TypeVar +T = TypeVar('T') +async def f(x: T) -> T: + y = await f(x) # type: Any + reveal_type(y) + return y +[typing fixtures/typing-async.pyi] +[out] +main:6: note: Revealed type is "Any" + +[case testAwaitExplicitContext] + +from typing import TypeVar +T = TypeVar('T') +async def f(x: T) -> T: + y = await f(x) # type: int + reveal_type(y) + return x +[typing fixtures/typing-async.pyi] +[out] +main:5: error: Argument 1 to "f" has incompatible type "T"; expected "int" +main:6: note: Revealed type is "builtins.int" + +[case testAwaitGeneratorError] + +from typing import Any, Generator +def g() -> Generator[int, None, str]: + yield 0 + return '' +async def f() -> int: + x = await g() + return x +[typing fixtures/typing-async.pyi] +[out] +main:7: error: Incompatible types in "await" (actual type "Generator[int, None, str]", expected type "Awaitable[Any]") + +[case testAwaitIteratorError] + +from typing import Any, Iterator +def g() -> Iterator[Any]: + yield +async def f() -> int: + x = await g() + return x +[typing fixtures/typing-async.pyi] +[out] +main:6: error: Incompatible types in "await" (actual type "Iterator[Any]", expected type "Awaitable[Any]") + +[case testAwaitArgumentError] + +def g() -> int: + return 0 +async def f() -> int: + x = await g() + return x +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:5: error: Incompatible types in "await" (actual type "int", expected type "Awaitable[Any]") + +[case testAwaitResultError] + +async def g() -> int: + return 0 +async def f() -> str: + x = await g() # type: str + return x +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:5: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAwaitReturnError] + +async def g() -> int: + return 0 +async def f() -> str: + x = await g() + return x +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:6: error: Incompatible return value type (got "int", expected "str") + +[case testAsyncFor] + +from typing import AsyncIterator +class C(AsyncIterator[int]): + async def __anext__(self) -> int: return 0 +async def f() -> None: + async for x in C(): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncForError] + +from typing import AsyncIterator +async def f() -> None: + async for x in [1]: + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:4: error: "List[int]" has no attribute "__aiter__" (not async iterable) + +[case testAsyncForErrorNote] + +from typing import AsyncIterator, AsyncGenerator +async def g() -> AsyncGenerator[str, None]: + pass + +async def f() -> None: + async for x in g(): + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:7: error: "Coroutine[Any, Any, AsyncGenerator[str, None]]" has no attribute "__aiter__" (not async iterable) +main:7: note: Maybe you forgot to use "await"? + +[case testAsyncForErrorCanBeIgnored] + +from typing import AsyncIterator, AsyncGenerator +async def g() -> AsyncGenerator[str, None]: + pass + +async def f() -> None: + async for x in g(): # type: ignore[attr-defined] + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncForTypeComments] + +from typing import AsyncIterator, Union +class C(AsyncIterator[int]): + async def __anext__(self) -> int: return 0 +async def f() -> None: + async for x in C(): # type: str # E: Incompatible types in assignment (expression has type "int", variable has type "str") + pass + + async for y in C(): # type: int + pass + + async for z in C(): # type: Union[int, str] + reveal_type(z) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncForComprehension] +from typing import Generic, Iterable, TypeVar, AsyncIterator, Tuple + +T = TypeVar('T') + +class asyncify(Generic[T], AsyncIterator[T]): + def __init__(self, iterable: Iterable[T]) -> None: + self.iterable = iter(iterable) + def __aiter__(self) -> AsyncIterator[T]: + return self + async def __anext__(self) -> T: + try: + return next(self.iterable) + except StopIteration: + raise StopAsyncIteration + +async def listcomp(obj: Iterable[int]): + lst = [i async for i in asyncify(obj)] + reveal_type(lst) # N: Revealed type is "builtins.list[builtins.int]" + lst2 = [i async for i in asyncify(obj) for j in obj] + reveal_type(lst2) # N: Revealed type is "builtins.list[builtins.int]" + +async def setcomp(obj: Iterable[int]): + lst = {i async for i in asyncify(obj)} + reveal_type(lst) # N: Revealed type is "builtins.set[builtins.int]" + +async def dictcomp(obj: Iterable[Tuple[int, str]]): + lst = {i: j async for i, j in asyncify(obj)} + reveal_type(lst) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +async def generatorexp(obj: Iterable[int]): + lst = (i async for i in asyncify(obj)) + reveal_type(lst) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + lst2 = (i async for i in asyncify(obj) for i in obj) + reveal_type(lst2) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncForComprehensionErrors] +from typing import Generic, Iterable, TypeVar, AsyncIterator, Tuple + +T = TypeVar('T') + +class asyncify(Generic[T], AsyncIterator[T]): + def __init__(self, iterable: Iterable[T]) -> None: + self.iterable = iter(iterable) + def __aiter__(self) -> AsyncIterator[T]: + return self + async def __anext__(self) -> T: + try: + return next(self.iterable) + except StopIteration: + raise StopAsyncIteration + +async def wrong_iterable(obj: Iterable[int]): + [i async for i in obj] # E: "Iterable[int]" has no attribute "__aiter__" (not async iterable) + [i for i in asyncify(obj)] # E: "asyncify[int]" has no attribute "__iter__"; maybe "__aiter__"? (not iterable) + {i: i async for i in obj} # E: "Iterable[int]" has no attribute "__aiter__" (not async iterable) + {i: i for i in asyncify(obj)} # E: "asyncify[int]" has no attribute "__iter__"; maybe "__aiter__"? (not iterable) +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncWith] + +class C: + async def __aenter__(self) -> int: pass + async def __aexit__(self, x, y, z) -> None: pass +async def f() -> None: + async with C() as x: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + + +[case testAsyncWithError] + +class C: + def __enter__(self) -> int: pass + def __exit__(self, x, y, z) -> None: pass +async def f() -> None: + async with C() as x: + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:6: error: "C" has no attribute "__aenter__"; maybe "__enter__"? +main:6: error: "C" has no attribute "__aexit__"; maybe "__exit__"? + +[case testAsyncWithErrorBadAenter] + +class C: + def __aenter__(self) -> int: pass + async def __aexit__(self, x, y, z) -> None: pass +async def f() -> None: + async with C() as x: # E: Incompatible types in "async with" for "__aenter__" (actual type "int", expected type "Awaitable[Any]") + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncWithErrorBadAenter2] +# flags: --no-strict-optional +class C: + def __aenter__(self) -> None: pass + async def __aexit__(self, x, y, z) -> None: pass +async def f() -> None: + async with C() as x: # E: "None" has no attribute "__await__" + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncWithErrorBadAexit] + +class C: + async def __aenter__(self) -> int: pass + def __aexit__(self, x, y, z) -> int: pass +async def f() -> None: + async with C() as x: # E: Incompatible types in "async with" for "__aexit__" (actual type "int", expected type "Awaitable[Any]") + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncWithErrorBadAexit2] +# flags: --no-strict-optional +class C: + async def __aenter__(self) -> int: pass + def __aexit__(self, x, y, z) -> None: pass +async def f() -> None: + async with C() as x: # E: "None" has no attribute "__await__" + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncWithTypeComments] + +class C: + async def __aenter__(self) -> int: pass + async def __aexit__(self, x, y, z) -> None: pass +async def f() -> None: + async with C() as x: # type: int + pass + + async with C() as y, C() as z: # type: str, int # E: Incompatible types in assignment (expression has type "int", variable has type "str") + pass + + async with C() as a: # type: int, int # E: Syntax error in type annotation # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testNoYieldFromInAsyncDef] + +async def f(): + yield from [] +async def g(): + x = yield from [] +[builtins fixtures/async_await.pyi] +[out] +main:3: error: "yield from" in async function +main:5: error: "yield from" in async function + +[case testYieldFromNoAwaitable] + +from typing import Any, Generator +async def f() -> str: + return '' +def g() -> Generator[Any, None, str]: + x = yield from f() + return x +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +main:6: error: "yield from" can't be applied to "Coroutine[Any, Any, str]" + +[case testAwaitableSubclass] + +from typing import Any, AsyncIterator, Awaitable, Generator +class A(Awaitable[int]): + def __await__(self) -> Generator[Any, None, int]: + yield + return 0 +class C: + def __aenter__(self) -> A: + return A() + def __aexit__(self, *a) -> A: + return A() +class I(AsyncIterator[int]): + def __aiter__(self) -> 'I': + return self + def __anext__(self) -> A: + return A() +async def main() -> None: + x = await A() + reveal_type(x) # N: Revealed type is "builtins.int" + async with C() as y: + reveal_type(y) # N: Revealed type is "builtins.int" + async for z in I(): + reveal_type(z) # N: Revealed type is "builtins.int" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testYieldTypeCheckInDecoratedCoroutine] + +from typing import Generator +from types import coroutine +@coroutine +def f() -> Generator[int, str, int]: + x = yield 0 + x = yield '' # E: Incompatible types in "yield" (actual type "str", expected type "int") + reveal_type(x) # N: Revealed type is "builtins.str" + if x: + return 0 + else: + return '' # E: Incompatible return value type (got "str", expected "int") +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +-- Async generators (PEP 525), some test cases adapted from the PEP text +-- --------------------------------------------------------------------- + +[case testAsyncGenerator] +from typing import AsyncGenerator, Generator + +async def f() -> int: + return 42 + +async def g() -> AsyncGenerator[int, None]: + value = await f() + reveal_type(value) # N: Revealed type is "builtins.int" + yield value + + yield 'not an int' # E: Incompatible types in "yield" (actual type "str", expected type "int") + # return without a value is fine + return +reveal_type(g) # N: Revealed type is "def () -> typing.AsyncGenerator[builtins.int, None]" +reveal_type(g()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + +async def h() -> None: + async for item in g(): + reveal_type(item) # N: Revealed type is "builtins.int" + +async def wrong_return() -> Generator[int, None, None]: # E: The return type of an async generator function should be "AsyncGenerator" or one of its supertypes + yield 3 + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorReturnIterator] +from typing import AsyncIterator + +async def gen() -> AsyncIterator[int]: + yield 3 + + yield 'not an int' # E: Incompatible types in "yield" (actual type "str", expected type "int") + +async def use_gen() -> None: + async for item in gen(): + reveal_type(item) # N: Revealed type is "builtins.int" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorManualIter] +from typing import AsyncGenerator + +async def genfunc() -> AsyncGenerator[int, None]: + yield 1 + yield 2 + +async def user() -> None: + gen = genfunc() + + reveal_type(gen.__aiter__()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + + reveal_type(await gen.__anext__()) # N: Revealed type is "builtins.int" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorAsend] +from typing import AsyncGenerator + +async def f() -> None: + pass + +async def gen() -> AsyncGenerator[int, str]: + await f() + v = yield 42 + reveal_type(v) # N: Revealed type is "builtins.str" + await f() + +async def h() -> None: + g = gen() + await g.asend(()) # E: Argument 1 to "asend" of "AsyncGenerator" has incompatible type "Tuple[()]"; expected "str" + reveal_type(await g.asend('hello')) # N: Revealed type is "builtins.int" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorAthrow] +from typing import AsyncGenerator + +async def gen() -> AsyncGenerator[str, int]: + try: + yield 'hello' + except BaseException: + yield 'world' + +async def h() -> None: + g = gen() + v = await g.asend(1) + reveal_type(v) # N: Revealed type is "builtins.str" + reveal_type(await g.athrow(BaseException)) # N: Revealed type is "builtins.str" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorNoSyncIteration] +from typing import AsyncGenerator + +async def gen() -> AsyncGenerator[int, None]: + for i in [1, 2, 3]: + yield i + +def h() -> None: + for i in gen(): # E: "AsyncGenerator[int, None]" has no attribute "__iter__"; maybe "__aiter__"? (not iterable) + pass + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorNoYieldFrom] +from typing import AsyncGenerator + +async def f() -> AsyncGenerator[int, None]: + pass + +async def gen() -> AsyncGenerator[int, None]: + yield from f() # E: "yield from" in async function + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorNoReturnWithValue] +from typing import AsyncGenerator + +async def return_int() -> AsyncGenerator[int, None]: + yield 1 + return 42 # E: "return" with value in async generator is not allowed + +async def return_none() -> AsyncGenerator[int, None]: + yield 1 + return None # E: "return" with value in async generator is not allowed + +def f() -> None: + return + +async def return_f() -> AsyncGenerator[int, None]: + yield 1 + return f() # E: "return" with value in async generator is not allowed + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testImplicitAsyncGenerator] +from typing import List + +async def get_list() -> List[int]: + return [1] + +async def predicate() -> bool: + return True + +async def test_implicit_generators() -> None: + reveal_type(await predicate() for _ in [1]) # N: Revealed type is "typing.AsyncGenerator[builtins.bool, None]" + reveal_type(x for x in [1] if await predicate()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + reveal_type(x for x in await get_list()) # N: Revealed type is "typing.Generator[builtins.int, None, None]" + reveal_type(x for _ in [1] for x in await get_list()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + + +-- The full matrix of coroutine compatibility +-- ------------------------------------------ + +[case testFullCoroutineMatrix] + +from typing import Any, AsyncIterator, Awaitable, Generator, Iterator +from types import coroutine + +# The various things you might try to use in `await` or `yield from`. + +def plain_generator() -> Generator[str, None, int]: + yield 'a' + return 1 + +async def plain_coroutine() -> int: + return 1 + +@coroutine +def decorated_generator() -> Generator[str, None, int]: + yield 'a' + return 1 + +@coroutine +async def decorated_coroutine() -> int: + return 1 + +class It(Iterator[str]): + def __iter__(self) -> 'It': + return self + def __next__(self) -> str: + return 'a' + +def other_iterator() -> It: + return It() + +class Aw(Awaitable[int]): + def __await__(self) -> Generator[str, Any, int]: + yield 'a' + return 1 + +def other_coroutine() -> Aw: + return Aw() + +# The various contexts in which `await` or `yield from` might occur. + +def plain_host_generator() -> Generator[str, None, None]: + yield 'a' + x = 0 + x = yield from plain_generator() + x = yield from plain_coroutine() # E: "yield from" can't be applied to "Coroutine[Any, Any, int]" + x = yield from decorated_generator() + x = yield from decorated_coroutine() # E: "yield from" can't be applied to "AwaitableGenerator[Any, Any, int, Coroutine[Any, Any, int]]" + x = yield from other_iterator() + x = yield from other_coroutine() # E: "yield from" can't be applied to "Aw" + +async def plain_host_coroutine() -> None: + x = 0 + x = await plain_generator() # E: Incompatible types in "await" (actual type "Generator[str, None, int]", expected type "Awaitable[Any]") + x = await plain_coroutine() + x = await decorated_generator() + x = await decorated_coroutine() + x = await other_iterator() # E: Incompatible types in "await" (actual type "It", expected type "Awaitable[Any]") + x = await other_coroutine() + +@coroutine +def decorated_host_generator() -> Generator[str, None, None]: + yield 'a' + x = 0 + x = yield from plain_generator() + x = yield from plain_coroutine() + x = yield from decorated_generator() + x = yield from decorated_coroutine() + x = yield from other_iterator() + x = yield from other_coroutine() # E: "yield from" can't be applied to "Aw" + +@coroutine +async def decorated_host_coroutine() -> None: + x = 0 + x = await plain_generator() # E: Incompatible types in "await" (actual type "Generator[str, None, int]", expected type "Awaitable[Any]") + x = await plain_coroutine() + x = await decorated_generator() + x = await decorated_coroutine() + x = await other_iterator() # E: Incompatible types in "await" (actual type "It", expected type "Awaitable[Any]") + x = await other_coroutine() + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] + +[case testAsyncGenDisallowUntyped] +# flags: --disallow-untyped-defs +# These should not crash +from typing import AsyncGenerator, Any + +async def f() -> AsyncGenerator[int, None]: + yield 0 + +async def g() -> AsyncGenerator[Any, None]: + yield 0 +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] + +[case testAsyncGenDisallowUntypedTriggers] +# flags: --disallow-untyped-defs +from typing import AsyncGenerator, Any + +async def f() -> AsyncGenerator[Any, Any]: + yield None + +async def h() -> Any: + yield 0 + +async def g(): # E: Function is missing a return type annotation + yield 0 +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] + +[case testAsyncOverloadedFunction] +from typing import overload + +@overload +async def f(x: int) -> int: ... +@overload +async def f(x: str) -> str: ... +async def f(x): + pass + +reveal_type(f) # N: Revealed type is "Overload(def (x: builtins.int) -> typing.Coroutine[Any, Any, builtins.int], def (x: builtins.str) -> typing.Coroutine[Any, Any, builtins.str])" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncForwardRefInBody] + +async def f() -> None: + forwardref: C + class C: pass + +def dec(x): pass + +@dec +async def g() -> None: + forwardref: C + class C: pass + +reveal_type(f) # N: Revealed type is "def () -> typing.Coroutine[Any, Any, None]" +reveal_type(g) # N: Revealed type is "Any" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncDeferredAnalysis] + +def t() -> None: + async def f() -> int: + return 1 + + def g() -> int: + return next(iter(x)) + + x = [1] + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncWithInGenericClass] +from typing import Generic, AsyncContextManager, TypeVar + +T = TypeVar('T', str, int) + +class Foo(Generic[T]): + async def foo(self, manager: AsyncContextManager): + async with manager: + pass + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAwaitOverloadSpecialCase] +from typing import Any, Awaitable, Iterable, overload, Tuple, List, TypeVar, Generic + +T = TypeVar("T") +FT = TypeVar("FT", bound='Future[Any]') + +class Future(Awaitable[T], Iterable[T]): + pass + +class Task(Future[T]): + pass + +@overload +def wait(fs: Iterable[FT]) -> Future[Tuple[List[FT], List[FT]]]: ... \ + # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error +@overload +def wait(fs: Iterable[Awaitable[T]]) -> Future[Tuple[List[Task[T]], List[Task[T]]]]: ... +def wait(fs: Any) -> Any: + pass + +async def imprecise1(futures: Iterable[Task[Any]]) -> None: + done: Any + pending: Any + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "Any" + +async def imprecise2(futures: Iterable[Awaitable[Any]]) -> None: + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "builtins.list[__main__.Task[Any]]" + +async def precise1(futures: Iterable[Future[int]]) -> None: + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "builtins.list[__main__.Future[builtins.int]]" + +async def precise2(futures: Iterable[Awaitable[int]]) -> None: + done, pending = await wait(futures) + reveal_type(done) # N: Revealed type is "builtins.list[__main__.Task[builtins.int]]" + + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testUnusedAwaitable] +# flags: --show-error-codes --enable-error-code unused-awaitable +from typing import Iterable + +async def foo() -> None: + pass + +class A: + def __await__(self) -> Iterable[int]: + yield 5 + +# Things with __getattr__ should not simply be considered awaitable. +class B: + def __getattr__(self, attr) -> object: + return 0 + +def bar() -> None: + A() # E: Value of type "A" must be used [unused-awaitable] \ + # N: Are you missing an await? + foo() # E: Value of type "Coroutine[Any, Any, None]" must be used [unused-coroutine] \ + # N: Are you missing an await? + B() + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncForOutsideCoroutine] +async def g(): + yield 0 + +def f() -> None: + [x async for x in g()] # E: "async for" outside async function + {x async for x in g()} # E: "async for" outside async function + {x: True async for x in g()} # E: "async for" outside async function + (x async for x in g()) + async for x in g(): ... # E: "async for" outside async function + +[x async for x in g()] # E: "async for" outside async function +{x async for x in g()} # E: "async for" outside async function +{x: True async for x in g()} # E: "async for" outside async function +(x async for x in g()) +async for x in g(): ... # E: "async for" outside async function + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncWithOutsideCoroutine] +class C: + async def __aenter__(self): pass + async def __aexit__(self, x, y, z): pass + +def f() -> None: + async with C() as x: # E: "async with" outside async function + pass + +async with C() as x: # E: "async with" outside async function + pass + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAwaitMissingNote] +from typing import Generic, TypeVar, Generator, Any, Awaitable, Type + +class C: + x: int +class D(C): ... + +async def foo() -> D: ... +def g(x: C) -> None: ... + +T = TypeVar("T") +class Custom(Generic[T]): + def __await__(self) -> Generator[Any, Any, T]: ... + +class Sub(Custom[T]): ... + +async def test(x: Sub[D], tx: Type[Sub[D]]) -> None: + foo().x # E: "Coroutine[Any, Any, D]" has no attribute "x" \ + # N: Maybe you forgot to use "await"? + (await foo()).x + foo().bad # E: "Coroutine[Any, Any, D]" has no attribute "bad" + + g(foo()) # E: Argument 1 to "g" has incompatible type "Coroutine[Any, Any, D]"; expected "C" \ + # N: Maybe you forgot to use "await"? + g(await foo()) + unknown: Awaitable[Any] + g(unknown) # E: Argument 1 to "g" has incompatible type "Awaitable[Any]"; expected "C" + + x.x # E: "Sub[D]" has no attribute "x" \ + # N: Maybe you forgot to use "await"? + (await x).x + x.bad # E: "Sub[D]" has no attribute "bad" + + a: C = x # E: Incompatible types in assignment (expression has type "Sub[D]", variable has type "C") \ + # N: Maybe you forgot to use "await"? + b: C = await x + unknown2: Awaitable[Any] + d: C = unknown2 # E: Incompatible types in assignment (expression has type "Awaitable[Any]", variable has type "C") + + # The notes are not show for Type[...] (because awaiting them will not work) + tx.x # E: "Type[Sub[D]]" has no attribute "x" + a2: C = tx # E: Incompatible types in assignment (expression has type "Type[Sub[D]]", variable has type "C") + +class F: + def __await__(self: T) -> Generator[Any, Any, T]: ... +class G(F): ... + +# This should not crash. +x: int = G() # E: Incompatible types in assignment (expression has type "G", variable has type "int") + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncGeneratorExpressionAwait] +from typing import AsyncGenerator + +async def f() -> AsyncGenerator[int, None]: + async def g(x: int) -> int: + return x + + return (await g(x) for x in [1, 2, 3]) + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAwaitUnion] +from typing import overload, Union + +class A: ... +class B: ... + +@overload +async def foo(x: A) -> B: ... +@overload +async def foo(x: B) -> A: ... +async def foo(x): ... + +async def bar(x: Union[A, B]) -> None: + reveal_type(await foo(x)) # N: Revealed type is "Union[__main__.B, __main__.A]" + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncIteratorWithIgnoredErrors] +import m + +async def func(l: m.L) -> None: + reveal_type(l.get_iterator) # N: Revealed type is "def () -> typing.AsyncIterator[builtins.str]" + reveal_type(l.get_iterator2) # N: Revealed type is "def () -> typing.AsyncIterator[builtins.str]" + async for i in l.get_iterator(): + reveal_type(i) # N: Revealed type is "builtins.str" + + reveal_type(m.get_generator) # N: Revealed type is "def () -> typing.AsyncGenerator[builtins.int, None]" + async for i2 in m.get_generator(): + reveal_type(i2) # N: Revealed type is "builtins.int" + +[file m.py] +# mypy: ignore-errors=True +from typing import AsyncIterator, AsyncGenerator + +class L: + async def some_func(self, i: int) -> str: + return 'x' + + async def get_iterator(self) -> AsyncIterator[str]: + yield await self.some_func(0) + + async def get_iterator2(self) -> AsyncIterator[str]: + if self: + a = (yield 'x') + +async def get_generator() -> AsyncGenerator[int, None]: + yield 1 + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncIteratorWithIgnoredErrorsAndYieldFrom] +from m import L + +async def func(l: L) -> None: + reveal_type(l.get_iterator) + +[file m.py] +# mypy: ignore-errors=True +from typing import AsyncIterator + +class L: + async def get_iterator(self) -> AsyncIterator[str]: + yield from ['x'] # E: "yield from" in async function +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testInvalidComprehensionNoCrash] +# flags: --show-error-codes +async def foo(x: int) -> int: ... + +# These are allowed in some cases: +top_level = await foo(1) # E: "await" outside function [top-level-await] +crasher = [await foo(x) for x in [1, 2, 3]] # E: "await" outside function [top-level-await] + +def bad() -> None: + # These are always critical / syntax issues: + y = [await foo(x) for x in [1, 2, 3]] # E: "await" outside coroutine ("async def") [await-not-async] +async def good() -> None: + y = [await foo(x) for x in [1, 2, 3]] # OK +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testNestedAsyncFunctionAndTypeVarAvalues] +from typing import TypeVar + +T = TypeVar('T', int, str) + +def f(x: T) -> None: + async def g() -> T: + return x +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testNestedAsyncGeneratorAndTypeVarAvalues] +from typing import AsyncGenerator, TypeVar + +T = TypeVar('T', int, str) + +def f(x: T) -> None: + async def g() -> AsyncGenerator[T, None]: + yield x +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testNestedDecoratedCoroutineAndTypeVarValues] +from typing import Generator, TypeVar +from types import coroutine + +T = TypeVar('T', int, str) + +def f(x: T) -> None: + @coroutine + def inner() -> Generator[T, None, None]: + yield x + reveal_type(inner) # N: Revealed type is "def () -> typing.AwaitableGenerator[builtins.int, None, None, typing.Generator[builtins.int, None, None]]" \ + # N: Revealed type is "def () -> typing.AwaitableGenerator[builtins.str, None, None, typing.Generator[builtins.str, None, None]]" + +@coroutine +def coro() -> Generator[int, None, None]: + yield 1 +reveal_type(coro) # N: Revealed type is "def () -> typing.AwaitableGenerator[builtins.int, None, None, typing.Generator[builtins.int, None, None]]" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case asyncIteratorInProtocol] +from typing import AsyncIterator, Protocol + +class P(Protocol): + async def launch(self) -> AsyncIterator[int]: + raise BaseException + +class Launcher(P): + def launch(self) -> AsyncIterator[int]: # E: Return type "AsyncIterator[int]" of "launch" incompatible with return type "Coroutine[Any, Any, AsyncIterator[int]]" in supertype "P" \ + # N: Consider declaring "launch" in supertype "P" without "async" \ + # N: See https://mypy.readthedocs.io/en/stable/more_types.html#asynchronous-iterators + raise BaseException + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] diff --git a/test-data/unit/check-basic.test b/test-data/unit/check-basic.test new file mode 100644 index 000000000000..959d80cb2104 --- /dev/null +++ b/test-data/unit/check-basic.test @@ -0,0 +1,528 @@ +[case testEmptyFile] +[out] + +[case testAssignmentAndVarDef] +a: A +b: B +if int(): + a = a +if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") +class A: pass +class B: pass + +[case testConstructionAndAssignment] +class A: + def __init__(self): pass +class B: + def __init__(self): pass + +x: A +x = A() +if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[case testInheritInitFromObject] +class A(object): pass +class B(object): pass +x: A +if int(): + x = A() +if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[case testImplicitInheritInitFromObject] +class A: pass +class B: pass +x: A +o: object +if int(): + x = o # E: Incompatible types in assignment (expression has type "object", variable has type "A") +if int(): + x = A() +if int(): + o = x +[case testTooManyConstructorArgs] +import typing +object(object()) +[out] +main:2: error: Too many arguments for "object" + +[case testVarDefWithInit] +import typing +class A: pass +a = A() # type: A +b = object() # type: A # E: Incompatible types in assignment (expression has type "object", variable has type "A") +[case testInheritanceBasedSubtyping] +import typing +class A: pass +class B(A): pass +x = B() # type: A +y = A() # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") +[case testDeclaredVariableInParentheses] + +(x) = 2 # type: int +if int(): + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + x = 1 + + +[case testIncompatibleAssignmentAmbiguousShortnames] + +class Any: pass +class List: pass +class Dict: pass +class Iterator: pass + +x = Any() +x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "__main__.Any") + +y = List() +y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "__main__.List") + +z = Dict() +z = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "__main__.Dict") + +w = Iterator() +w = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "__main__.Iterator") + + +-- Simple functions and calling +-- ---------------------------- + + +[case testFunction] +import typing +class A: pass +class B: pass +def f(x: 'A') -> None: pass +f(A()) +f(B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +[case testNotCallable] +import typing +class A: pass +A()() # E: "A" not callable +[case testSubtypeArgument] +import typing +class A: pass +class B(A): pass +def f(x: 'A', y: 'B') -> None: pass +f(B(), A()) # E: Argument 2 to "f" has incompatible type "A"; expected "B" +f(B(), B()) +[case testInvalidArgumentCount] +import typing +def f(x, y) -> None: pass +f(object()) +f(object(), object(), object()) +[out] +main:3: error: Missing positional argument "y" in call to "f" +main:4: error: Too many arguments for "f" + +[case testMissingPositionalArguments] +class Foo: + def __init__(self, bar: int): + pass +c = Foo() +def foo(baz: int, bas: int):pass +foo() +[out] +main:4: error: Missing positional argument "bar" in call to "Foo" +main:6: error: Missing positional arguments "baz", "bas" in call to "foo" + + +-- Locals +-- ------ + + +[case testLocalVariables] +def f() -> None: + x: A + y: B + if int(): + x = x + x = y # E: Incompatible types in assignment (expression has type "B", variable has type "A") +class A: pass +class B: pass + +[case testLocalVariableScope] +def f() -> None: + x: A + x = A() +def g() -> None: + x: B + x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") +class A: pass +class B: pass + +[case testFunctionArguments] +import typing +def f(x: 'A', y: 'B') -> None: + if int(): + x = y # E: Incompatible types in assignment (expression has type "B", variable has type "A") + x = x + y = B() +class A: pass +class B: pass + +[case testLocalVariableInitialization] +import typing +def f() -> None: + a = A() # type: A + b = B() # type: A # Fail +class A: pass +class B: pass +[out] +main:4: error: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testVariableInitializationWithSubtype] +import typing +class A: pass +class B(A): pass +x = B() # type: A +y = A() # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") + + +-- Misc +-- ---- + + +[case testInvalidReturn] +import typing +def f() -> 'A': + return B() +class A: pass +class B: pass +[out] +main:3: error: Incompatible return value type (got "B", expected "A") + +[case testTopLevelContextAndInvalidReturn] +import typing +class A: pass +class B: pass +def f() -> 'A': + return B() # E: Incompatible return value type (got "B", expected "A") +a = B() # type: A # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[case testEmptyReturnInAnyTypedFunction] +from typing import Any +def f() -> Any: + return + +[case testEmptyYieldInAnyTypedFunction] +from typing import Any +def f() -> Any: + yield + +[case testModuleImplicitAttributes] +import typing +class A: pass +reveal_type(__name__) # N: Revealed type is "builtins.str" +reveal_type(__doc__) # N: Revealed type is "builtins.str" +reveal_type(__file__) # N: Revealed type is "builtins.str" +reveal_type(__package__) # N: Revealed type is "builtins.str" +reveal_type(__annotations__) # N: Revealed type is "builtins.dict[builtins.str, Any]" +# This will actually reveal Union[importlib.machinery.ModuleSpec, None] +reveal_type(__spec__) # N: Revealed type is "Union[builtins.object, None]" + +import module +reveal_type(module.__name__) # N: Revealed type is "builtins.str" +# This will actually reveal importlib.machinery.ModuleSpec +reveal_type(module.__spec__) # N: Revealed type is "builtins.object" +[file module.py] +[builtins fixtures/primitives.pyi] + + +-- Scoping and shadowing +-- --------------------- + + +[case testLocalVariableShadowing] +class A: pass +class B: pass +a: A +if int(): + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = A() +def f() -> None: + a: B + if int(): + a = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = B() +a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a = A() +[case testGlobalDefinedInBlockWithType] +class A: pass +while 1: + a: A + if int(): + a = A() + a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") + + +-- # type: signatures +-- ------------------ + + +[case testFunctionSignatureAsComment] +def f(x): # type: (int) -> str + return 1 +f('') +[out] +main:2: error: Incompatible return value type (got "int", expected "str") +main:3: error: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testMethodSignatureAsComment] +class A: + def f(self, x): + # type: (int) -> str + self.f('') # Fail + return 1 +A().f('') # Fail +[out] +main:4: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +main:5: error: Incompatible return value type (got "int", expected "str") +main:6: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testTrailingCommaParsing] +x = 1 +x in 1, # E: Unsupported right operand type for in ("int") +[builtins fixtures/tuple.pyi] + +[case testTrailingCommaInIfParsing] +if x in 1, : pass +[out] +main:1: error: invalid syntax + +[case testInitReturnTypeError] +class C: + def __init__(self): + # type: () -> int + pass +[out] +main:2: error: The return type of "__init__" must be None + +-- WritesCache signals to testcheck to do the cache validation +[case testWritesCache] +import a +import d +[file a.py] +import b +import c +[file b.py] +[file c.py] +[file d.py] + +[case testWritesCacheErrors] +import a +import d +[file a.py] +import b +import c +[file b.py] +[file c.py] +[file d.py] +import e +[file e.py] +1+'no' # E: Unsupported operand types for + ("int" and "str") + +[case testModuleAsTypeNoCrash] + +import mock +from typing import Union + +class A: ... +class B: ... + +x: Union[mock, A] # E: Module "mock" is not valid as a type \ + # N: Perhaps you meant to use a protocol matching the module structure? + +if isinstance(x, B): + pass +[file mock.py] +[builtins fixtures/isinstance.pyi] +[out] + +[case testModuleAsTypeNoCrash2] + +import mock +from typing import overload, Any, Union + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> Union[mock, str]: ... # E: Module "mock" is not valid as a type \ + # N: Perhaps you meant to use a protocol matching the module structure? +def f(x): + pass + +x: Any +f(x) +[file mock.py] +[builtins fixtures/isinstance.pyi] +[out] + +[case testPartialTypeComments] +def foo( + a, # type: str + b, + args=None, +): + # type: (...) -> None + pass + +[case testNoneHasBool] +none = None +b = none.__bool__() +reveal_type(b) # N: Revealed type is "Literal[False]" +[builtins fixtures/bool.pyi] + +[case testAssignmentInvariantNoteForList] +from typing import List +x: List[int] +y: List[float] +y = x # E: Incompatible types in assignment (expression has type "List[int]", variable has type "List[float]") \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant +[builtins fixtures/list.pyi] + +[case testAssignmentInvariantNoteForDict] +from typing import Dict +x: Dict[str, int] +y: Dict[str, float] +y = x # E: Incompatible types in assignment (expression has type "Dict[str, int]", variable has type "Dict[str, float]") \ + # N: "Dict" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Mapping" instead, which is covariant in the value type +[builtins fixtures/dict.pyi] + +[case testDistinctTypes] +import b + +[file a.py] +from typing import NamedTuple +from typing_extensions import TypedDict +from enum import Enum +class A: pass +N = NamedTuple('N', [('x', int)]) +D = TypedDict('D', {'x': int}) +class B(Enum): + b = 10 + +[file b.py] +from typing import List, Optional, Union, Sequence, NamedTuple, Tuple, Type +from typing_extensions import Literal, Final, TypedDict +from enum import Enum +import a +class A: pass +N = NamedTuple('N', [('x', int)]) +class B(Enum): + b = 10 +D = TypedDict('D', {'y': int}) + +def foo() -> Optional[A]: + b = True + return a.A() if b else None # E: Incompatible return value type (got "Optional[a.A]", expected "Optional[b.A]") + +def bar() -> List[A]: + l = [a.A()] + return l # E: Incompatible return value type (got "List[a.A]", expected "List[b.A]") + +def baz() -> Union[A, int]: + b = True + return a.A() if b else 10 # E: Incompatible return value type (got "Union[a.A, int]", expected "Union[b.A, int]") + +def spam() -> Optional[A]: + return a.A() # E: Incompatible return value type (got "a.A", expected "Optional[b.A]") + +def eggs() -> Sequence[A]: + x = [a.A()] + return x # E: Incompatible return value type (got "List[a.A]", expected "Sequence[b.A]") + +def eggs2() -> Sequence[N]: + x = [a.N(0)] + return x # E: Incompatible return value type (got "List[a.N]", expected "Sequence[b.N]") + +def asdf1() -> Sequence[Tuple[a.A, A]]: + x = [(a.A(), a.A())] + return x # E: Incompatible return value type (got "List[Tuple[a.A, a.A]]", expected "Sequence[Tuple[a.A, b.A]]") + +def asdf2() -> Sequence[Tuple[A, a.A]]: + x = [(a.A(), a.A())] + return x # E: Incompatible return value type (got "List[Tuple[a.A, a.A]]", expected "Sequence[Tuple[b.A, a.A]]") + +def arg() -> Tuple[A, A]: + return A() # E: Incompatible return value type (got "A", expected "Tuple[A, A]") + +def types() -> Sequence[Type[A]]: + x = [a.A] + return x # E: Incompatible return value type (got "List[Type[a.A]]", expected "Sequence[Type[b.A]]") + +def literal() -> Sequence[Literal[B.b]]: + x = [a.B.b] # type: List[Literal[a.B.b]] + return x # E: Incompatible return value type (got "List[Literal[a.B.b]]", expected "Sequence[Literal[b.B.b]]") + +def typeddict() -> Sequence[D]: + x = [{'x': 0}] # type: List[a.D] + return x # E: Incompatible return value type (got "List[a.D]", expected "Sequence[b.D]") + +a = (a.A(), A()) +a.x # E: "Tuple[a.A, b.A]" has no attribute "x" + +[builtins fixtures/dict.pyi] + +[case testReturnAnyFromFunctionDeclaredToReturnObject] +# flags: --warn-return-any +from typing import Any + +def f() -> object: + x: Any = 1 + return x + +[case testImportModuleAsClassMember] +import test + +class A: + def __init__(self) -> None: + self.test = test + + def __call__(self) -> None: + self.test.foo("Message") + +[file test.py] +def foo(s: str) -> None: ... + +[case testLocalImportModuleAsClassMember] +class A: + def __init__(self) -> None: + import test + + self.test = test + + def __call__(self) -> None: + self.test.foo("Message") + +[file test.py] +def foo(s: str) -> None: ... + +[case testInlineAssertions] +import a, b +s1: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[file a.py] +s2: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[file b.py] +s3: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[file c.py] +s3: str = 'foo' + +[case testMultilineQuotedAnnotation] +x: """ + + int | + str + +""" +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +y: """( + int | + str +) +""" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" diff --git a/test-data/unit/check-bound.test b/test-data/unit/check-bound.test new file mode 100644 index 000000000000..1c713fd77c38 --- /dev/null +++ b/test-data/unit/check-bound.test @@ -0,0 +1,229 @@ +-- Enforcement of upper bounds +-- --------------------------- + + +[case testBoundOnGenericFunction] +from typing import TypeVar + +class A: pass +class B(A): pass +class C(A): pass +class D: pass + +T = TypeVar('T', bound=A) +U = TypeVar('U') +def f(x: T) -> T: pass +def g(x: U) -> U: + return f(x) # E: Value of type variable "T" of "f" cannot be "U" + +f(A()) +f(B()) +f(D()) # E: Value of type variable "T" of "f" cannot be "D" + +b = B() +if int(): + b = f(b) +if int(): + b = f(C()) # E: Incompatible types in assignment (expression has type "C", variable has type "B") + + +[case testBoundOnGenericClass] +from typing import TypeVar, Generic + +class A: pass +class B(A): pass +T = TypeVar('T', bound=A) + +class G(Generic[T]): + def __init__(self, x: T) -> None: pass + +v: G[A] +w: G[B] +x: G[str] # E: Type argument "str" of "G" must be a subtype of "A" +y = G('a') # E: Value of type variable "T" of "G" cannot be "str" +z = G(A()) +z = G(B()) + + +[case testBoundVoid] +# flags: --no-strict-optional +from typing import TypeVar, Generic +T = TypeVar('T', bound=int) +class C(Generic[T]): + t = None # type: T + def get(self) -> T: + return self.t +c1 = None # type: C[None] +c1.get() +d = c1.get() +reveal_type(d) # N: Revealed type is "None" + + +[case testBoundAny] +from typing import TypeVar, Generic +T = TypeVar('T', bound=int) +class C(Generic[T]): + def __init__(self, x: T) -> None: pass +def f(x: T) -> T: + return x + +def g(): pass + +f(g()) +C(g()) +z: C + + +[case testBoundHigherOrderWithVoid] +# flags: --no-strict-optional +from typing import TypeVar, Callable +class A: pass +T = TypeVar('T', bound=A) +def f(g: Callable[[], T]) -> T: + return g() +def h() -> None: pass +f(h) +a = f(h) +reveal_type(a) # N: Revealed type is "None" + + +[case testBoundInheritance] +from typing import TypeVar, Generic +class A: pass +T = TypeVar('T') +TA = TypeVar('TA', bound=A) + +class C(Generic[TA]): pass +class D0(C[TA], Generic[TA]): pass +class D1(C[T], Generic[T]): pass # E: Type argument "T" of "C" must be a subtype of "A" +class D2(C[A]): pass +class D3(C[str]): pass # E: Type argument "str" of "C" must be a subtype of "A" + + +-- Using information from upper bounds +-- ----------------------------------- + + +[case testBoundGenericFunctions] +from typing import TypeVar +class A: pass +class B(A): pass + +T = TypeVar('T') +TA = TypeVar('TA', bound=A) +TB = TypeVar('TB', bound=B) + +def f(x: T) -> T: + return x +def g(x: TA) -> TA: + return f(x) +def h(x: TB) -> TB: + return g(x) +def g2(x: TA) -> TA: + return h(x) # Fail + +def j(x: TA) -> A: + return x +def k(x: TA) -> B: + return x # Fail +[out] +main:16: error: Value of type variable "TB" of "h" cannot be "TA" +main:21: error: Incompatible return value type (got "TA", expected "B") + + +[case testBoundMethodUsage] +from typing import TypeVar +class A0: + def foo(self) -> None: pass +class A(A0): + def bar(self) -> None: pass + a = 1 + @property + def b(self) -> int: + return self.a +class B(A): + def baz(self) -> None: pass + +T = TypeVar('T', bound=A) + +def f(x: T) -> T: + x.foo() + x.bar() + x.baz() # E: "T" has no attribute "baz" + x.a + x.b + return x + +b = f(B()) +[builtins fixtures/property.pyi] +[out] + +[case testBoundClassMethod] +from typing import TypeVar +class A0: + @classmethod + def foo(cls, x: int) -> int: pass +class A(A0): pass + +T = TypeVar('T', bound=A) +def f(x: T) -> int: + return x.foo(22) +[builtins fixtures/classmethod.pyi] + + +[case testBoundClassMethodWithNamedTupleBase] +from typing import NamedTuple, Type, TypeVar +class A(NamedTuple): + @classmethod + def foo(cls) -> None: ... + +T = TypeVar('T', bound=A) +def f(x: Type[T]) -> None: + reveal_type(x.foo) # N: Revealed type is "def ()" + x.foo() +[builtins fixtures/classmethod.pyi] + + +[case testBoundStaticMethod] +from typing import TypeVar +class A0: + @staticmethod + def foo(x: int) -> int: pass +class A(A0): pass + +T = TypeVar('T', bound=A) +def f(x: T) -> int: + return x.foo(22) +[builtins fixtures/staticmethod.pyi] + + +[case testBoundOnDecorator] +from typing import TypeVar, Callable, Any, cast +T = TypeVar('T', bound=Callable[..., Any]) + +def twice(f: T) -> T: + def result(*args, **kwargs) -> Any: + f(*args, **kwargs) + return f(*args, **kwargs) + return cast(T, result) + +@twice +def foo(x: int) -> int: + return x + +a = 1 +b = foo(a) +if int(): + b = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +twice(a) # E: Value of type variable "T" of "twice" cannot be "int" +[builtins fixtures/args.pyi] + + +[case testIterableBoundUnpacking] +from typing import Tuple, TypeVar +TupleT = TypeVar("TupleT", bound=Tuple[int, ...]) +def f(t: TupleT) -> None: + a, *b = t + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-callable.test b/test-data/unit/check-callable.test new file mode 100644 index 000000000000..39e6c4fa3ff1 --- /dev/null +++ b/test-data/unit/check-callable.test @@ -0,0 +1,630 @@ +[case testCallableDef] +def f() -> None: pass + +if callable(f): + f() +else: + f += 5 + +[builtins fixtures/callable.pyi] + +[case testCallableLambda] +f = lambda: None + +if callable(f): + f() +else: + f += 5 + +[builtins fixtures/callable.pyi] + +[case testCallableNotCallable] +x = 5 + +if callable(x): + x() +else: + x += 5 + +[builtins fixtures/callable.pyi] + +[case testUnion] +from typing import Callable, Union + +x = 5 # type: Union[int, Callable[[], str]] + +if callable(x): + y = x() + 'test' +else: + z = x + 6 + +[builtins fixtures/callable.pyi] + +[case testUnionMultipleReturnTypes] +from typing import Callable, Union + +x = 5 # type: Union[int, Callable[[], str], Callable[[], int]] + +if callable(x): + y = x() + 2 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[str, int]" +else: + z = x + 6 + +[builtins fixtures/callable.pyi] + +[case testUnionMultipleNonCallableTypes] +from typing import Callable, Union + +x = 5 # type: Union[int, str, Callable[[], str]] + +if callable(x): + y = x() + 'test' +else: + z = x + 6 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + +[builtins fixtures/callable.pyi] + +[case testCallableThenIsinstance] +from typing import Callable, Union + +x = 5 # type: Union[int, str, Callable[[], str], Callable[[], int]] + +if callable(x): + y = x() + if isinstance(y, int): + b1 = y + 2 + else: + b2 = y + 'test' +else: + if isinstance(x, int): + b3 = x + 3 + else: + b4 = x + 'test2' + +[builtins fixtures/callable.pyi] + +[case testIsinstanceThenCallable] +from typing import Callable, Union + +x = 5 # type: Union[int, str, Callable[[], str], Callable[[], int]] + +if isinstance(x, int): + b1 = x + 1 +else: + if callable(x): + y = x() + if isinstance(y, int): + b2 = y + 1 + else: + b3 = y + 'test' + else: + b4 = x + 'test2' + +[builtins fixtures/callable.pyi] + +[case testCallableWithDifferentArgTypes] +from typing import Callable, Union + +x = 5 # type: Union[int, Callable[[], None], Callable[[int], None]] + +if callable(x): + x() # E: Too few arguments + +[builtins fixtures/callable.pyi] + +[case testClassInitializer] +from typing import Callable, Union + +class A: + x = 5 + +a = A # type: Union[A, Callable[[], A]] + +if callable(a): + a = a() + +a.x + 6 + +[builtins fixtures/callable.pyi] + +[case testCallableVariables] +from typing import Union + +class A: + x = 5 + +class B: + x = int + +x = A() # type: Union[A, B] + +if callable(x.x): + y = x.x() +else: + y = x.x + 5 + +[builtins fixtures/callable.pyi] + +[case testCallableAnd] +from typing import Union, Callable + +x = 5 # type: Union[int, Callable[[], str]] + +if callable(x) and x() == 'test': + x() +else: + x + 5 # E: Unsupported left operand type for + ("Callable[[], str]") \ + # N: Left operand is of type "Union[int, Callable[[], str]]" + +[builtins fixtures/callable.pyi] + +[case testCallableOr] +from typing import Union, Callable + +x = 5 # type: Union[int, Callable[[], str]] + +if callable(x) or x() == 'test': # E: "int" not callable + x() # E: "int" not callable +else: + x + 5 +[builtins fixtures/callable.pyi] + +[case testCallableOrOtherType] +from typing import Union, Callable + +x = 5 # type: Union[int, Callable[[], str]] + +if callable(x) or x == 2: + pass +else: + pass +[builtins fixtures/callable.pyi] + +[case testAnyCallable] +from typing import Any + +x = 5 # type: Any + +if callable(x): + reveal_type(x) # N: Revealed type is "Any" +else: + reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/callable.pyi] + +[case testCallableCallableClasses] +from typing import Union + + +class A: + pass + + +class B: + def __call__(self) -> None: + pass + + +a = A() # type: A +b = B() # type: B +c = A() # type: Union[A, B] + +if callable(a): + 5 + 'test' # E: Unsupported operand types for + ("int" and "str") + +if not callable(b): + 5 + 'test' + +if callable(c): + reveal_type(c) # N: Revealed type is "__main__.B" +else: + reveal_type(c) # N: Revealed type is "__main__.A" + +[builtins fixtures/callable.pyi] + +[case testDecoratedCallMethods] +from typing import Any, Callable, Union, TypeVar + +F = TypeVar('F', bound=Callable) + +def decorator(f: F) -> F: + pass +def change(f: Callable) -> Callable[[Any], str]: + pass +def untyped(f): + pass + +class Some1: + @decorator + def __call__(self) -> int: + pass +class Some2: + @change + def __call__(self) -> int: + pass +class Some3: + @untyped + def __call__(self) -> int: + pass +class Some4: + __call__: Any + +s1: Some1 +s2: Some2 +s3: Some3 +s4: Some4 + +if callable(s1): + 1 + 'a' # E: Unsupported operand types for + ("int" and "str") +else: + 2 + 'b' +if callable(s2): + 1 + 'a' # E: Unsupported operand types for + ("int" and "str") +else: + 2 + 'b' +if callable(s3): + 1 + 'a' # E: Unsupported operand types for + ("int" and "str") +else: + 2 + 'b' # E: Unsupported operand types for + ("int" and "str") +if callable(s4): + 1 + 'a' # E: Unsupported operand types for + ("int" and "str") +else: + 2 + 'b' # E: Unsupported operand types for + ("int" and "str") +[builtins fixtures/callable.pyi] + +[case testCallableNestedUnions] +from typing import Callable, Union + +T = Union[Union[int, Callable[[], int]], Union[str, Callable[[], str]]] + +def f(t: T) -> None: + if callable(t): + reveal_type(t()) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(t) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[builtins fixtures/callable.pyi] + +[case testCallableTypeVarEmpty] +from typing import TypeVar + +T = TypeVar('T') + +def f(t: T) -> T: + if callable(t): + return 5 # E: Incompatible return value type (got "int", expected "T") + else: + return t + +[builtins fixtures/callable.pyi] + +[case testCallableTypeVarUnion] +from typing import Callable, TypeVar, Union + +T = TypeVar('T', int, Callable[[], int], Union[str, Callable[[], str]]) + +def f(t: T) -> None: + if callable(t): + reveal_type(t()) # N: Revealed type is "Any" \ + # N: Revealed type is "builtins.int" \ + # N: Revealed type is "builtins.str" + else: + reveal_type(t) # N: Revealed type is "builtins.int" # N: Revealed type is "builtins.str" + +[builtins fixtures/callable.pyi] + +[case testCallableTypeVarBound] +from typing import TypeVar + + +class A: + def __call__(self) -> str: + return 'hi' + + +T = TypeVar('T', bound=A) + +def f(t: T) -> str: + if callable(t): + return t() + else: + return 5 + +[builtins fixtures/callable.pyi] + +[case testCallableTypeType] +from typing import Type + + +class A: + pass + + +T = Type[A] + +def f(t: T) -> A: + if callable(t): + return t() + else: + return 5 + +[builtins fixtures/callable.pyi] + +[case testCallableTypeUnion] +from abc import ABCMeta, abstractmethod +from typing import Type, Union + + +class A(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: + pass + + +class B: + pass + + +x = B # type: Union[Type[A], Type[B]] +if callable(x): + # Abstract classes raise an error when called, but are indeed `callable` + pass +else: + 'test' + 5 + +[builtins fixtures/callable.pyi] + +[case testCallableUnionOfTypes] +from abc import ABCMeta, abstractmethod +from typing import Type, Union + + +class A(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: + pass + + +class B: + pass + + +x = B # type: Type[Union[A, B]] +if callable(x): + # Abstract classes raise an error when called, but are indeed `callable` + pass +else: + 'test' + 5 + +[builtins fixtures/callable.pyi] + +[case testCallableObject] + +def f(o: object) -> None: + if callable(o): + o(1,2,3) + 1 + 'boom' # E: Unsupported operand types for + ("int" and "str") + o('hi') + 12 + reveal_type(o) # N: Revealed type is "__main__." + +[builtins fixtures/callable.pyi] + +[case testCallableObject2] + +class Foo(object): + def bar(self) -> None: + pass + +def g(o: Foo) -> None: + o.bar() + if callable(o): + o.foo() # E: "Foo" has no attribute "foo" + o.bar() + o(1,2,3) + else: + o.bar() + +[builtins fixtures/callable.pyi] + +[case testCallableObjectAny] + +from typing import Any + +class Foo(Any): + def bar(self) -> None: + pass + +def g(o: Foo) -> None: + o.bar() + o.baz() + if callable(o): + o('test') + o.lurr(1,2,3) + +[builtins fixtures/callable.pyi] + +[case testCallableObjectGeneric] + +from typing import TypeVar, Generic + +T = TypeVar('T') +class Test(Generic[T]): + def __self__(self, x: T) -> None: + self.x = x + +def g(o: Test[T], x: T) -> T: + if callable(o): + o.foo() # E: "Test[T]" has no attribute "foo" + o(1,2,3) + o.x = x + o.x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "T") + 1 + o.x # E: Unsupported operand types for + ("int" and "T") + return o.x + + return x + +[builtins fixtures/callable.pyi] + +[case testCallablePromote] + +def take_float(f: float) -> None: + pass + +def g(o: int) -> None: + if callable(o): + take_float(o) + o(1,2,3) + +[builtins fixtures/callable.pyi] + +[case testCallableTuple] + +from typing import NamedTuple + +Thing = NamedTuple('Thing', [('s', str), ('n', int)]) + +def g(o: Thing) -> None: + if callable(o): + o.s + o.n # E: Unsupported operand types for + ("str" and "int") + i, s = o + i + s # E: Unsupported operand types for + ("str" and "int") + o(1,2,3) + +[builtins fixtures/callable.pyi] + +[case testCallableNoArgs] + +if callable(): # E: Missing positional argument "x" in call to "callable" + pass + +[builtins fixtures/callable.pyi] + +[case testCallableWithNoneArgs] + +fn = None +if callable(fn): + fn() + +[builtins fixtures/callable.pyi] + +[case testCallableUnionOfNoneAndCallable] + +from typing import Union, Callable + +def f() -> int: + return 42 + +fn = f # type: Union[None, Callable[[], int]] + +if callable(fn): + reveal_type(fn) # N: Revealed type is "def () -> builtins.int" +else: + reveal_type(fn) # N: Revealed type is "None" + +[builtins fixtures/callable.pyi] + +[case testBuiltinsTypeAsCallable] +from __future__ import annotations + +reveal_type(type) # N: Revealed type is "def (x: Any) -> builtins.type" +_TYPE = type +reveal_type(_TYPE) # N: Revealed type is "def (x: Any) -> builtins.type" +_TYPE('bar') + +[builtins fixtures/callable.pyi] + +[case testErrorMessageAboutSelf] +# https://github.com/python/mypy/issues/11309 +class Some: + def method(self, a) -> None: pass + @classmethod + def cls_method(cls, a) -> None: pass + @staticmethod + def st_method(a) -> None: pass + + def bad_method(a) -> None: pass + @classmethod + def bad_cls_method(a) -> None: pass + @staticmethod + def bad_st_method() -> None: pass + +s: Some + +s.method(1) +s.cls_method(1) +Some.cls_method(1) +s.st_method(1) +Some.st_method(1) + +s.method(1, 2) # E: Too many arguments for "method" of "Some" +s.cls_method(1, 2) # E: Too many arguments for "cls_method" of "Some" +Some.cls_method(1, 2) # E: Too many arguments for "cls_method" of "Some" +s.st_method(1, 2) # E: Too many arguments for "st_method" of "Some" +Some.st_method(1, 2) # E: Too many arguments for "st_method" of "Some" + +s.bad_method(1) # E: Too many arguments for "bad_method" of "Some" \ + # N: Looks like the first special argument in a method is not named "self", "cls", or "mcs", maybe it is missing? +s.bad_cls_method(1) # E: Too many arguments for "bad_cls_method" of "Some" \ + # N: Looks like the first special argument in a method is not named "self", "cls", or "mcs", maybe it is missing? +Some.bad_cls_method(1) # E: Too many arguments for "bad_cls_method" of "Some" \ + # N: Looks like the first special argument in a method is not named "self", "cls", or "mcs", maybe it is missing? +s.bad_st_method(1) # E: Too many arguments for "bad_st_method" of "Some" +Some.bad_st_method(1) # E: Too many arguments for "bad_st_method" of "Some" +[builtins fixtures/callable.pyi] + +[case testClassMethodAliasStub] +from a import f +f("no") # E: Argument 1 has incompatible type "str"; expected "int" +[file a.pyi] +from b import C +f = C.f +[file b.pyi] +import a +class C(B): + @classmethod + def f(self, x: int) -> C: ... +class B: ... +[builtins fixtures/classmethod.pyi] + +[case testCallableUnionCallback] +from typing import Union, Callable, TypeVar + +TA = TypeVar("TA", bound="A") +class A: + def __call__(self: TA, other: Union[Callable, TA]) -> TA: ... +a: A +a() # E: Missing positional argument "other" in call to "__call__" of "A" +a(a) +a(lambda: None) + +[case testCallableSubtypingTrivialSuffix] +from typing import Any, Protocol + +class Call(Protocol): + def __call__(self, x: int, *args: Any, **kwargs: Any) -> None: ... + +def f1() -> None: ... +a1: Call = f1 # E: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "Call") \ + # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" +def f2(x: str) -> None: ... +a2: Call = f2 # E: Incompatible types in assignment (expression has type "Callable[[str], None]", variable has type "Call") \ + # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" +def f3(y: int) -> None: ... +a3: Call = f3 # E: Incompatible types in assignment (expression has type "Callable[[int], None]", variable has type "Call") \ + # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" +def f4(x: int) -> None: ... +a4: Call = f4 + +def f5(x: int, y: int) -> None: ... +a5: Call = f5 + +def f6(x: int, y: int = 0) -> None: ... +a6: Call = f6 + +def f7(x: int, *, y: int) -> None: ... +a7: Call = f7 + +def f8(x: int, *args: int, **kwargs: str) -> None: ... +a8: Call = f8 +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test new file mode 100644 index 000000000000..fd564c7e96cb --- /dev/null +++ b/test-data/unit/check-class-namedtuple.test @@ -0,0 +1,685 @@ +[case testNewNamedTupleNoUnderscoreFields] +from typing import NamedTuple + +class X(NamedTuple): + x: int + _y: int # E: NamedTuple field name cannot start with an underscore: _y + _z: int # E: NamedTuple field name cannot start with an underscore: _z +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleAccessingAttributes] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str + +x: X +x.x +x.y +x.z # E: "X" has no attribute "z" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleAttributesAreReadOnly] +from typing import NamedTuple + +class X(NamedTuple): + x: int + +x: X +x.x = 5 # E: Property "x" defined in "X" is read-only +x.y = 5 # E: "X" has no attribute "y" + +class A(X): pass +a: A +a.x = 5 # E: Property "x" defined in "X" is read-only +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleCreateWithPositionalArguments] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str + +x = X(1, '2') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Missing positional argument "y" in call to "X" +x = X(1, '2', 3) # E: Too many arguments for "X" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleShouldBeSingleBase] +from typing import NamedTuple + +class A: ... +class X(NamedTuple, A): # E: NamedTuple should be a single base + pass +[builtins fixtures/tuple.pyi] + +[case testCreateNewNamedTupleWithKeywordArguments] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str + +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y='x') # E: Missing positional argument "x" in call to "X" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleCreateAndUseAsTuple] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str + +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleWithItemTypes] +from typing import NamedTuple + +class N(NamedTuple): + a: int + b: str + +n = N(1, 'x') +s: str = n.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i: int = n.b # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +if int(): + x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleConstructorArgumentTypes] +from typing import NamedTuple + +class N(NamedTuple): + a: int + b: str + +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument "b" to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleAsBaseClass] +from typing import NamedTuple + +class N(NamedTuple): + a: int + b: str + +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +if int(): + s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + i, s = x +if int(): + s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple + +class A(NamedTuple): + a: int + b: str + +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + if int(): + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[builtins fixtures/tuple.pyi] +[out] + +[case testNewNamedTupleTypeReferenceToClassDerivedFrom] +from typing import NamedTuple + +class A(NamedTuple): + a: int + b: str + +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + if int(): + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleSubtyping] +from typing import NamedTuple, Tuple + +class A(NamedTuple): + a: int + b: str + +class B(A): pass +a = A(1, '') +b = B(1, '') +t: Tuple[int, str] +if int(): + b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +if int(): + b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +if int(): + t = a +if int(): + t = (1, '') +if int(): + t = b +if int(): + a = b +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple + +class A(NamedTuple): + a: int + +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.pyi] + +[case testNewNamedTupleMissingClassAttribute] +from typing import NamedTuple + +class MyNamedTuple(NamedTuple): + a: int + b: str + +MyNamedTuple.x # E: "Type[MyNamedTuple]" has no attribute "x" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleEmptyItems] +from typing import NamedTuple + +class A(NamedTuple): + ... +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleForwardRef] +from typing import NamedTuple + +class A(NamedTuple): + b: 'B' + +class B: ... + +a = A(B()) +a = A(1) # E: Argument 1 to "A" has incompatible type "int"; expected "B" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleProperty36] +from typing import NamedTuple + +class A(NamedTuple): + a: int + +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b + +[builtins fixtures/property.pyi] + +[case testNewNamedTupleAsDict] +from typing import NamedTuple, Any + +class X(NamedTuple): + x: Any + y: Any + +x: X +reveal_type(x._asdict()) # N: Revealed type is "builtins.dict[builtins.str, Any]" + +[builtins fixtures/dict.pyi] + +[case testNewNamedTupleReplaceTyped] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str + +x: X +reveal_type(x._replace()) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.X]" +x._replace(x=5) +x._replace(y=5) # E: Argument "y" to "_replace" of "X" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleFields] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str + +reveal_type(X._fields) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +reveal_type(X._field_types) # N: Revealed type is "builtins.dict[builtins.str, Any]" +reveal_type(X._field_defaults) # N: Revealed type is "builtins.dict[builtins.str, Any]" + +# In typeshed's stub for builtins.pyi, __annotations__ is `dict[str, Any]`, +# but it's inferred as `Mapping[str, object]` here due to the fixture we're using +reveal_type(X.__annotations__) # N: Revealed type is "typing.Mapping[builtins.str, builtins.object]" + +[builtins fixtures/dict-full.pyi] + +[case testNewNamedTupleUnit] +from typing import NamedTuple + +class X(NamedTuple): + pass + +x: X = X() +x._replace() +x._fields[0] # E: Tuple index out of range +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleJoinNamedTuple] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str +class Y(NamedTuple): + x: int + y: str + +reveal_type([X(3, 'b'), Y(1, 'a')]) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" + +[builtins fixtures/list.pyi] + +[case testNewNamedTupleJoinTuple] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: str + +reveal_type([(3, 'b'), X(1, 'a')]) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" +reveal_type([X(1, 'a'), (3, 'b')]) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" + +[builtins fixtures/list.pyi] + +[case testNewNamedTupleWithTooManyArguments] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y = z = 2 # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" + def f(self): pass +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleWithInvalidItems2] +import typing + +class X(typing.NamedTuple): + x: int + y = 1 # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" + x.x: int # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" + z: str = 'z' + aa: int # E: Non-default NamedTuple fields cannot follow default fields +[builtins fixtures/list.pyi] + +[case testNewNamedTupleWithoutTypesSpecified] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y = 2 # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" +[builtins fixtures/tuple.pyi] + +[case testTypeUsingTypeCNamedTuple] +from typing import NamedTuple, Type + +class N(NamedTuple): + x: int + y: str + +def f(a: Type[N]): + a() # E: Missing positional arguments "x", "y" in call to "N" +[builtins fixtures/list.pyi] + +[case testNewNamedTupleWithDefaults] +from typing import List, NamedTuple, Optional + +class X(NamedTuple): + x: int + y: int = 2 + +reveal_type(X(1)) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.X]" +reveal_type(X(1, 2)) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.X]" + +X(1, 'a') # E: Argument 2 to "X" has incompatible type "str"; expected "int" +X(1, z=3) # E: Unexpected keyword argument "z" for "X" + +class HasNone(NamedTuple): + x: int + y: Optional[int] = None + +reveal_type(HasNone(1)) # N: Revealed type is "Tuple[builtins.int, Union[builtins.int, None], fallback=__main__.HasNone]" + +class Parameterized(NamedTuple): + x: int + y: List[int] = [1] + [2] + z: List[int] = [] + +reveal_type(Parameterized(1)) # N: Revealed type is "Tuple[builtins.int, builtins.list[builtins.int], builtins.list[builtins.int], fallback=__main__.Parameterized]" +Parameterized(1, ['not an int']) # E: List item 0 has incompatible type "str"; expected "int" + +class Default: + pass + +class UserDefined(NamedTuple): + x: Default = Default() + +reveal_type(UserDefined()) # N: Revealed type is "Tuple[__main__.Default, fallback=__main__.UserDefined]" +reveal_type(UserDefined(Default())) # N: Revealed type is "Tuple[__main__.Default, fallback=__main__.UserDefined]" +UserDefined(1) # E: Argument 1 to "UserDefined" has incompatible type "int"; expected "Default" + +[builtins fixtures/list.pyi] + +[case testNewNamedTupleWithDefaultsStrictOptional] +from typing import List, NamedTuple, Optional + +class HasNone(NamedTuple): + x: int + y: Optional[int] = None + +reveal_type(HasNone(1)) # N: Revealed type is "Tuple[builtins.int, Union[builtins.int, None], fallback=__main__.HasNone]" +HasNone(None) # E: Argument 1 to "HasNone" has incompatible type "None"; expected "int" +HasNone(1, y=None) +HasNone(1, y=2) + +class CannotBeNone(NamedTuple): + x: int + y: int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +[builtins fixtures/list.pyi] + +[case testNewNamedTupleWrongType] +from typing import NamedTuple + +class X(NamedTuple): + x: int + y: int = 'not an int' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleErrorInDefault] +from typing import NamedTuple + +class X(NamedTuple): + x: int = 1 + '1' # E: Unsupported left operand type for + ("int") +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleInheritance] +from typing import NamedTuple + +class X(NamedTuple): + x: str + y: int = 3 + +class Y(X): + def method(self) -> str: + self.y + return self.x + +reveal_type(Y('a')) # N: Revealed type is "Tuple[builtins.str, builtins.int, fallback=__main__.Y]" +Y(y=1, x='1').method() + +class CallsBaseInit(X): + def __init__(self, x: str) -> None: + super().__init__(x) # E: Too many arguments for "__init__" of "object" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleWithMethods] +from typing import NamedTuple + +class XMeth(NamedTuple): + x: int + def double(self) -> int: + return self.x + async def asyncdouble(self) -> int: + return self.x + +class XRepr(NamedTuple): + x: int + y: int = 1 + def __str__(self) -> str: + return 'string' + def __sub__(self, other: XRepr) -> int: + return 0 + +reveal_type(XMeth(1).double()) # N: Revealed type is "builtins.int" +_ = reveal_type(XMeth(1).asyncdouble()) # N: Revealed type is "typing.Coroutine[Any, Any, builtins.int]" +reveal_type(XMeth(42).x) # N: Revealed type is "builtins.int" +reveal_type(XRepr(42).__str__()) # N: Revealed type is "builtins.str" +reveal_type(XRepr(1, 2).__sub__(XRepr(3))) # N: Revealed type is "builtins.int" +[typing fixtures/typing-async.pyi] +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleOverloading] +from typing import NamedTuple, overload + +class Overloader(NamedTuple): + x: int + @overload + def method(self, y: str) -> str: pass + @overload + def method(self, y: int) -> int: pass + def method(self, y): + return y + +reveal_type(Overloader(1).method('string')) # N: Revealed type is "builtins.str" +reveal_type(Overloader(1).method(1)) # N: Revealed type is "builtins.int" +Overloader(1).method(('tuple',)) # E: No overload variant of "method" of "Overloader" matches argument type "Tuple[str]" \ + # N: Possible overload variants: \ + # N: def method(self, y: str) -> str \ + # N: def method(self, y: int) -> int +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleMethodInheritance] +from typing import NamedTuple, TypeVar + +T = TypeVar('T') + +class Base(NamedTuple): + x: int + def copy(self: T) -> T: + reveal_type(self) # N: Revealed type is "T`-1" + return self + def good_override(self) -> int: + reveal_type(self) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.Base]" + reveal_type(self[0]) # N: Revealed type is "builtins.int" + self[0] = 3 # E: Unsupported target for indexed assignment ("Base") + reveal_type(self.x) # N: Revealed type is "builtins.int" + self.x = 3 # E: Property "x" defined in "Base" is read-only + self[1] # E: Tuple index out of range + reveal_type(self[T]) # N: Revealed type is "builtins.int" \ + # E: No overload variant of "__getitem__" of "tuple" matches argument type "TypeVar" \ + # N: Possible overload variants: \ + # N: def __getitem__(self, int, /) -> int \ + # N: def __getitem__(self, slice, /) -> Tuple[int, ...] + return self.x + def bad_override(self) -> int: + return self.x + +class Child(Base): + def new_method(self) -> int: + reveal_type(self) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.Child]" + reveal_type(self[0]) # N: Revealed type is "builtins.int" + self[0] = 3 # E: Unsupported target for indexed assignment ("Child") + reveal_type(self.x) # N: Revealed type is "builtins.int" + self.x = 3 # E: Property "x" defined in "Base" is read-only + self[1] # E: Tuple index out of range + return self.x + def good_override(self) -> int: + return 0 + def bad_override(self) -> str: # E: Return type "str" of "bad_override" incompatible with return type "int" in supertype "Base" + return 'incompatible' + +def takes_base(base: Base) -> int: + return base.x + +reveal_type(Base(1).copy()) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.Base]" +reveal_type(Child(1).copy()) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.Child]" +reveal_type(Base(1).good_override()) # N: Revealed type is "builtins.int" +reveal_type(Child(1).good_override()) # N: Revealed type is "builtins.int" +reveal_type(Base(1).bad_override()) # N: Revealed type is "builtins.int" +reveal_type(takes_base(Base(1))) # N: Revealed type is "builtins.int" +reveal_type(takes_base(Child(1))) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testNewNamedTupleIllegalNames] +from typing import Callable, NamedTuple + +class XMethBad(NamedTuple): + x: int + def _fields(self): # E: Cannot overwrite NamedTuple attribute "_fields" + return 'no chance for this' + +class MagicalFields(NamedTuple): + x: int + def __slots__(self) -> None: pass # E: Cannot overwrite NamedTuple attribute "__slots__" + def __new__(cls) -> MagicalFields: pass # E: Cannot overwrite NamedTuple attribute "__new__" + def _source(self) -> int: pass # E: Cannot overwrite NamedTuple attribute "_source" + __annotations__ = {'x': float} # E: NamedTuple field name cannot start with an underscore: __annotations__ \ + # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" \ + # E: Cannot overwrite NamedTuple attribute "__annotations__" + +class AnnotationsAsAMethod(NamedTuple): + x: int + # This fails at runtime because typing.py assumes that __annotations__ is a dictionary. + def __annotations__(self) -> float: # E: Cannot overwrite NamedTuple attribute "__annotations__" + return 1.0 + +class ReuseNames(NamedTuple): + x: int + def x(self) -> str: # E: Name "x" already defined on line 22 + return '' + + def y(self) -> int: + return 0 + y: str # E: Name "y" already defined on line 26 + +class ReuseCallableNamed(NamedTuple): + z: Callable[[ReuseNames], int] + def z(self) -> int: # E: Name "z" already defined on line 31 + return 0 + +[builtins fixtures/dict.pyi] + +[case testNewNamedTupleDocString] +from typing import NamedTuple + +class Documented(NamedTuple): + """This is a docstring.""" + x: int + +reveal_type(Documented.__doc__) # N: Revealed type is "builtins.str" +reveal_type(Documented(1).x) # N: Revealed type is "builtins.int" + +class BadDoc(NamedTuple): + x: int + def __doc__(self) -> str: + return '' + +reveal_type(BadDoc(1).__doc__()) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testNewNamedTupleClassMethod] +from typing import NamedTuple + +class HasClassMethod(NamedTuple): + x: str + + @classmethod + def new(cls, f: str) -> 'HasClassMethod': + reveal_type(cls) # N: Revealed type is "Type[Tuple[builtins.str, fallback=__main__.HasClassMethod]]" + reveal_type(HasClassMethod) # N: Revealed type is "def (x: builtins.str) -> Tuple[builtins.str, fallback=__main__.HasClassMethod]" + return cls(x=f) + +[builtins fixtures/classmethod.pyi] + +[case testNewNamedTupleStaticMethod] +from typing import NamedTuple + +class HasStaticMethod(NamedTuple): + x: str + + @staticmethod + def new(f: str) -> 'HasStaticMethod': + return HasStaticMethod(x=f) + +[builtins fixtures/classmethod.pyi] + +[case testNewNamedTupleProperty] +from typing import NamedTuple + +class HasStaticMethod(NamedTuple): + x: str + + @property + def size(self) -> int: + reveal_type(self) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.HasStaticMethod]" + return 4 + +[builtins fixtures/property.pyi] + +[case testTypingExtensionsNamedTuple] +from typing_extensions import NamedTuple + +class Point(NamedTuple): + x: int + y: int + +bad_point = Point('foo') # E: Missing positional argument "y" in call to "Point" \ + # E: Argument 1 to "Point" has incompatible type "str"; expected "int" +point = Point(1, 2) +x, y = point +x = point.x +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.int" +point.y = 6 # E: Property "y" defined in "Point" is read-only + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test new file mode 100644 index 000000000000..5ce80faaee18 --- /dev/null +++ b/test-data/unit/check-classes.test @@ -0,0 +1,8172 @@ +-- Methods +-- ------- + + +[case testMethodCall] +class A: + def foo(self, x: 'A') -> None: pass +class B: + def bar(self, x: 'B', y: A) -> None: pass + +a: A +b: B + +a.foo(B()) # E: Argument 1 to "foo" of "A" has incompatible type "B"; expected "A" +a.bar(B(), A()) # E: "A" has no attribute "bar" + +a.foo(A()) +b.bar(B(), A()) + +[case testMethodCallWithSubtype] +class A: + def foo(self, x: 'A') -> None: pass + def bar(self, x: 'B') -> None: pass +class B(A): pass + +a: A +a.foo(A()) +a.foo(B()) +a.bar(A()) # E: Argument 1 to "bar" of "A" has incompatible type "A"; expected "B" +a.bar(B()) + +[case testInheritingMethod] +class A: + def foo(self, x: 'B') -> None: pass +class B(A): pass + +a: B +a.foo(A()) # Fail +a.foo(B()) + +[targets __main__, __main__.A.foo] +[out] +main:6: error: Argument 1 to "foo" of "A" has incompatible type "A"; expected "B" + +[case testMethodCallWithInvalidNumberOfArguments] +class A: + def foo(self, x: 'A') -> None: pass + +a: A +a.foo() # Fail +a.foo(object(), A()) # Fail +[out] +main:5: error: Missing positional argument "x" in call to "foo" of "A" +main:6: error: Too many arguments for "foo" of "A" +main:6: error: Argument 1 to "foo" of "A" has incompatible type "object"; expected "A" + +[case testMethodBody] +import typing +class A: + def f(self) -> None: + a = object() # type: A # Fail +[out] +main:4: error: Incompatible types in assignment (expression has type "object", variable has type "A") + +[case testMethodArguments] +import typing +class A: + def f(self, a: 'A', b: 'B') -> None: + if int(): + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = A() + b = B() + a = a + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") +class B: pass +[out] + +[case testReturnFromMethod] +import typing +class A: + def f(self) -> 'A': + return B() # Fail + return A() +class B: pass +[out] +main:4: error: Incompatible return value type (got "B", expected "A") + +[case testSelfArgument] +import typing +class A: + def f(self) -> None: + o = self # type: B # Fail + self.g() # Fail + a = self # type: A + self.f() +class B: pass +[out] +main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") +main:5: error: "A" has no attribute "g" + +[case testAssignToMethodViaInstance] +import typing +class A: + def f(self): pass +A().f = None # E: Cannot assign to a method \ + # E: Incompatible types in assignment (expression has type "None", variable has type "Callable[[], Any]") + + +[case testOverrideAttributeWithMethod] +# This was crashing: +# https://github.com/python/mypy/issues/10134 +from typing import Protocol + +class Base: + __hash__ = None + +class Derived(Base): + def __hash__(self) -> int: # E: Signature of "__hash__" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: None \ + # N: Subclass: \ + # N: def __hash__(self) -> int + pass + +# Correct: + +class CallableProtocol(Protocol): + def __call__(self, arg: int) -> int: + pass + +class CorrectBase: + attr: CallableProtocol + +class CorrectDerived(CorrectBase): + def attr(self, arg: int) -> int: + pass + +[case testOverrideMethodWithAttribute] +# The reverse should not crash as well: +from typing import Callable + +class Base: + def __hash__(self) -> int: + pass + +class Derived(Base): + __hash__ = 1 # E: Incompatible types in assignment (expression has type "int", base class "Base" defined the type as "Callable[[Base], int]") + +[case testOverridePartialAttributeWithMethod] +# This was crashing: https://github.com/python/mypy/issues/11686. +class Base: + def __init__(self, arg: int): + self.partial_type = [] # E: Need type annotation for "partial_type" (hint: "partial_type: List[] = ...") + self.force_deferral = [] + + # Force inference of the `force_deferral` attribute in `__init__` to be + # deferred to a later pass by providing a definition in another context, + # which means `partial_type` remains only partially inferred. + force_deferral = [] # E: Need type annotation for "force_deferral" (hint: "force_deferral: List[] = ...") + + +class Derived(Base): + def partial_type(self) -> int: # E: Signature of "partial_type" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: List[Any] \ + # N: Subclass: \ + # N: def partial_type(self) -> int + ... + + +-- Attributes +-- ---------- + + +[case testReferToInvalidAttribute] + +class A: + def __init__(self) -> None: + self.x = object() +a: A +a.y # E: "A" has no attribute "y" +a.y = object() # E: "A" has no attribute "y" +a.x +a.x = object() + +[case testReferToInvalidAttributeUnannotatedInit] +class A: + def __init__(self): + self.x = object() + +a: A +a.y # E: "A" has no attribute "y" +a.y = object() # E: "A" has no attribute "y" +a.x +a.x = object() + +[case testArgumentTypeInference] + +class A: + def __init__(self, aa: 'A', bb: 'B') -> None: + self.a = aa + self.b = bb +class B: pass +a: A +b: B +a.a = b # Fail +a.b = a # Fail +b.a # Fail +a.a = a +a.b = b +[out] +main:9: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:10: error: Incompatible types in assignment (expression has type "A", variable has type "B") +main:11: error: "B" has no attribute "a" + +[case testExplicitAttributeInBody] + +class A: + x: A +a: A +a.x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +a.x = A() + +[case testAttributeDefinedInNonInitMethod] +import typing +class A: + def f(self) -> None: + self.x = 1 + self.y = '' + self.x = 1 +a = A() +a.x = 1 +a.y = '' +a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +a.z = 0 # E: "A" has no attribute "z" + +[case testInheritanceAndAttributeAssignment] +import typing +class A: + def f(self) -> None: + self.x = 0 +class B(A): + def f(self) -> None: + self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[targets __main__, __main__.A.f, __main__.B.f] + +[case testAssignmentToAttributeInMultipleMethods] +import typing +class A: + def f(self) -> None: + self.x = 0 + def g(self) -> None: + self.x = '' # Fail + def __init__(self) -> None: + self.x = '' # Fail +[out] +main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:8: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testClassNamesDefinedOnSelUsedInClassBody] +class A(object): + def f(self): + self.attr = 1 + attr = 0 + +class B(object): + attr = 0 + def f(self): + self.attr = 1 + +class C(object): + attr = 0 + def f(self): + self.attr = 1 + attr = 0 + +class D(object): + def g(self): + self.attr = 1 + attr = 0 + def f(self): + self.attr = 1 +[out] + +[case testClassNamesDefinedOnSelUsedInClassBodyReveal] + +class A(object): + def f(self) -> None: + self.attr = 1 + attr # E: Name "attr" is not defined + +class B(object): + attr = 0 + def f(self) -> None: + reveal_type(self.attr) # N: Revealed type is "builtins.int" +[out] + + +-- Method overriding +-- ----------------- + + +[case testMethodOverridingWithIdenticalSignature] +import typing +class A: + def f(self, x: 'A') -> None: pass + def g(self, x: 'B' , y: object) -> 'A': pass + def h(self) -> None: pass +class B(A): + def f(self, x: A) -> None: pass + def g(self, x: 'B' , y: object) -> A: pass + def h(self) -> None: pass +[out] + +[case testMethodOverridingWithCovariantType] +import typing +class A: + def f(self, x: 'A', y: 'B') -> 'A': pass + def g(self, x: 'A', y: 'B') -> 'A': pass +class B(A): + def f(self, x: A, y: 'B') -> 'B': pass + def g(self, x: A, y: A) -> 'A': pass +[out] + +[case testMethodOverridingWithIncompatibleTypes] +import typing +class A: + def f(self, x: 'A', y: 'B') -> 'A': pass + def g(self, x: 'A', y: 'B') -> 'A': pass + def h(self, x: 'A', y: 'B') -> 'A': pass +class B(A): + def f(self, x: 'B', y: 'B') -> A: pass # Fail + def g(self, x: A, y: A) -> A: pass + def h(self, x: A, y: 'B') -> object: pass # Fail +[out] +main:7: error: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "A" +main:7: note: This violates the Liskov substitution principle +main:7: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:9: error: Return type "object" of "h" incompatible with return type "A" in supertype "A" + +[case testMethodOverridingWithIncompatibleTypesOnMultipleLines] +class A: + def f(self, x: int, y: str) -> None: pass +class B(A): + def f( + self, + x: int, + y: bool, + ) -> None: + pass +[out] +main:7: error: Argument 2 of "f" is incompatible with supertype "A"; supertype defines the argument type as "str" +main:7: note: This violates the Liskov substitution principle +main:7: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + +[case testMultiLineMethodOverridingWithIncompatibleTypesIgnorableAtArgument] +class A: + def f(self, x: int, y: str) -> None: pass + +class B(A): + def f( + self, + x: int, + y: bool, # type: ignore[override] + ) -> None: + pass + +[case testMultiLineMethodOverridingWithIncompatibleTypesIgnorableAtDefinition] +class A: + def f(self, x: int, y: str) -> None: pass +class B(A): + def f( # type: ignore[override] + self, + x: int, + y: bool, + ) -> None: + pass + +[case testMultiLineMethodOverridingWithIncompatibleTypesWrongIgnore] +class A: + def f(self, x: int, y: str) -> None: pass +class B(A): + def f( # type: ignore[return-type] + self, + x: int, + y: bool, + ) -> None: + pass +[out] +main:7: error: Argument 2 of "f" is incompatible with supertype "A"; supertype defines the argument type as "str" +main:7: note: This violates the Liskov substitution principle +main:7: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + +[case testEqMethodsOverridingWithNonObjects] +class A: + def __eq__(self, other: A) -> bool: pass # Fail +[builtins fixtures/plugin_attrs.pyi] +[out] +main:2: error: Argument 1 of "__eq__" is incompatible with supertype "object"; supertype defines the argument type as "object" +main:2: note: This violates the Liskov substitution principle +main:2: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:2: note: It is recommended for "__eq__" to work with arbitrary objects, for example: +main:2: note: def __eq__(self, other: object) -> bool: +main:2: note: if not isinstance(other, A): +main:2: note: return NotImplemented +main:2: note: return + +[case testMethodOverridingWithIncompatibleArgumentCount] +import typing +class A: + def f(self, x: 'A') -> None: pass + def g(self, x: 'A', y: 'B') -> 'A': pass +class B(A): + def f(self, x: A, y: A) -> None: pass # Fail + def g(self, x: A) -> A: pass # Fail +[out] +main:6: error: Signature of "f" incompatible with supertype "A" +main:6: note: Superclass: +main:6: note: def f(self, x: A) -> None +main:6: note: Subclass: +main:6: note: def f(self, x: A, y: A) -> None +main:7: error: Signature of "g" incompatible with supertype "A" +main:7: note: Superclass: +main:7: note: def g(self, x: A, y: B) -> A +main:7: note: Subclass: +main:7: note: def g(self, x: A) -> A + +[case testMethodOverridingAcrossDeepInheritanceHierarchy1] +import typing +class A: + def f(self, x: 'B') -> None: pass +class B(A): pass +class C(B): # with gap in implementations + def f(self, x: 'C') -> None: # Fail + pass +[out] +main:6: error: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "B" +main:6: note: This violates the Liskov substitution principle +main:6: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + +[case testMethodOverridingAcrossDeepInheritanceHierarchy2] +import typing +class A: + def f(self) -> 'B': pass +class B(A): + def f(self) -> 'C': pass +class C(B): # with multiple implementations + def f(self) -> B: # Fail + pass +[out] +main:7: error: Return type "B" of "f" incompatible with return type "C" in supertype "B" + +[case testMethodOverridingWithVoidReturnValue] +import typing +class A: + def f(self) -> None: pass + def g(self) -> 'A': pass +class B(A): + def f(self) -> A: pass # Fail + def g(self) -> None: pass # Fail +[out] +main:6: error: Return type "A" of "f" incompatible with return type "None" in supertype "A" +main:7: error: Return type "None" of "g" incompatible with return type "A" in supertype "A" + +[case testOverride__new__WithDifferentSignature] +class A: + def __new__(cls, x: int) -> A: + pass + +class B(A): + def __new__(cls) -> B: + pass + +[case testOverride__new__AndCallObject] +from typing import TypeVar, Generic + +class A: + def __new__(cls, x: int) -> 'A': + return object.__new__(cls) + +T = TypeVar('T') +class B(Generic[T]): + def __new__(cls, foo: T) -> 'B[T]': + x = object.__new__(cls) + # object.__new__ doesn't have a great type :( + reveal_type(x) # N: Revealed type is "Any" + return x + +[builtins fixtures/__new__.pyi] + +[case testInnerFunctionNotOverriding] +class A: + def f(self) -> int: pass + +class B(A): + def g(self) -> None: + def f(self) -> str: pass + +[case testOverride__init_subclass__WithDifferentSignature] +class A: + def __init_subclass__(cls, x: int) -> None: pass +class B(A): # E: Missing positional argument "x" in call to "__init_subclass__" of "A" + def __init_subclass__(cls) -> None: pass + +[case testOverrideWithDecorator] +from typing import Callable + +def int_to_none(f: Callable[..., int]) -> Callable[..., None]: ... +def str_to_int(f: Callable[..., str]) -> Callable[..., int]: ... + +class A: + def f(self) -> None: pass + def g(self) -> str: pass + def h(self) -> None: pass + +class B(A): + @int_to_none + def f(self) -> int: pass + @str_to_int + def g(self) -> str: pass # Fail + @int_to_none + @str_to_int + def h(self) -> str: pass +[out] +main:15: error: Signature of "g" incompatible with supertype "A" +main:15: note: Superclass: +main:15: note: def g(self) -> str +main:15: note: Subclass: +main:15: note: def g(*Any, **Any) -> int + +[case testOverrideDecorated] +from typing import Callable + +def str_to_int(f: Callable[..., str]) -> Callable[..., int]: ... + +class A: + @str_to_int + def f(self) -> str: pass + @str_to_int + def g(self) -> str: pass + @str_to_int + def h(self) -> str: pass + +class B(A): + def f(self) -> int: pass + def g(self) -> str: pass # Fail + @str_to_int + def h(self) -> str: pass +[out] +main:15: error: Signature of "g" incompatible with supertype "A" +main:15: note: Superclass: +main:15: note: def g(*Any, **Any) -> int +main:15: note: Subclass: +main:15: note: def g(self) -> str + +[case testOverrideWithDecoratorReturningAny] +def dec(f): pass + +class A: + def f(self) -> str: pass + +class B(A): + @dec + def f(self) -> int: pass + +[case testOverrideWithDecoratorReturningInstance] +def dec(f) -> str: pass + +class A: + def f(self) -> str: pass + @dec + def g(self) -> int: pass + @dec + def h(self) -> int: pass + +class B(A): + @dec + def f(self) -> int: pass # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: def f(self) -> str \ + # N: Subclass: \ + # N: str + def g(self) -> int: pass # E: Signature of "g" incompatible with supertype "A" \ + # N: Superclass: \ + # N: str \ + # N: Subclass: \ + # N: def g(self) -> int + @dec + def h(self) -> str: pass + +[case testOverrideIncompatibleWithMultipleSupertypes] +class A: + def f(self, *, a: int) -> None: + return + +class B(A): + def f(self, *, b: int) -> None: # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: def f(self, *, a: int) -> None \ + # N: Subclass: \ + # N: def f(self, *, b: int) -> None + return + +class C(B): + def f(self, *, c: int) -> None: # E: Signature of "f" incompatible with supertype "B" \ + # N: Superclass: \ + # N: def f(self, *, b: int) -> None \ + # N: Subclass: \ + # N: def f(self, *, c: int) -> None \ + # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: def f(self, *, a: int) -> None \ + # N: Subclass: \ + # N: def f(self, *, c: int) -> None + return + +[case testOverrideStaticMethodWithStaticMethod] +class A: + @staticmethod + def f(x: int, y: str) -> None: pass + @staticmethod + def g(x: int, y: str) -> None: pass + +class B(A): + @staticmethod + def f(x: int, y: str) -> None: pass + @staticmethod + def g(x: str, y: str) -> None: pass # E: Argument 1 of "g" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[builtins fixtures/classmethod.pyi] + +[case testOverrideClassMethodWithClassMethod] +class A: + @classmethod + def f(cls, x: int, y: str) -> None: pass + @classmethod + def g(cls, x: int, y: str) -> None: pass + +class B(A): + @classmethod + def f(cls, x: int, y: str) -> None: pass + @classmethod + def g(cls, x: str, y: str) -> None: pass # E: Argument 1 of "g" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[builtins fixtures/classmethod.pyi] + +[case testOverrideClassMethodWithStaticMethod] +class A: + @classmethod + def f(cls, x: int) -> None: pass + @classmethod + def g(cls, x: int) -> int: pass + @classmethod + def h(cls) -> int: pass + +class B(A): + @staticmethod + def f(x: int) -> None: pass + @staticmethod + def g(x: str) -> int: pass # E: Argument 1 of "g" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + @staticmethod + def h() -> int: pass +[builtins fixtures/classmethod.pyi] + +[case testOverrideStaticMethodWithClassMethod] +class A: + @staticmethod + def f(x: int) -> None: pass + @staticmethod + def g(x: str) -> int: pass + @staticmethod + def h() -> int: pass + +class B(A): + @classmethod + def f(cls, x: int) -> None: pass + @classmethod + def g(cls, x: int) -> int: pass # E: Argument 1 of "g" is incompatible with supertype "A"; supertype defines the argument type as "str" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + @classmethod + def h(cls) -> int: pass +[builtins fixtures/classmethod.pyi] + +[case testAllowCovarianceInReadOnlyAttributes] +from typing import Callable, TypeVar + +T = TypeVar('T') + +class X: + pass + + +class Y(X): + pass + +def dec(f: Callable[..., T]) -> T: pass + +class A: + @dec + def f(self) -> X: pass + +class B(A): + @dec + def f(self) -> Y: pass + +[case testOverrideCallableAttributeWithMethod] +from typing import Callable + +class A: + f1: Callable[[str], None] + f2: Callable[[str], None] + f3: Callable[[str], None] + +class B(A): + def f1(self, x: object) -> None: + pass + + @classmethod + def f2(cls, x: object) -> None: + pass + + @staticmethod + def f3(x: object) -> None: + pass +[builtins fixtures/classmethod.pyi] + +[case testOverrideCallableAttributeWithMethodMutableOverride] +# flags: --enable-error-code=mutable-override +from typing import Callable + +class A: + f1: Callable[[str], None] + f2: Callable[[str], None] + f3: Callable[[str], None] + +class B(A): + def f1(self, x: object) -> None: pass # E: Covariant override of a mutable attribute (base class "A" defined the type as "Callable[[str], None]", override has type "Callable[[object], None]") + + @classmethod + def f2(cls, x: object) -> None: pass # E: Covariant override of a mutable attribute (base class "A" defined the type as "Callable[[str], None]", override has type "Callable[[object], None]") + + @staticmethod + def f3(x: object) -> None: pass # E: Covariant override of a mutable attribute (base class "A" defined the type as "Callable[[str], None]", override has type "Callable[[object], None]") +[builtins fixtures/classmethod.pyi] + +[case testOverrideCallableAttributeWithSettableProperty] +from typing import Callable + +class A: + f: Callable[[str], None] + +class B(A): + @property + def f(self) -> Callable[[object], None]: pass + @func.setter + def f(self, x: object) -> None: pass +[builtins fixtures/property.pyi] + +[case testOverrideCallableAttributeWithSettablePropertyMutableOverride] +# flags: --enable-error-code=mutable-override +from typing import Callable + +class A: + f: Callable[[str], None] + +class B(A): + @property # E: Covariant override of a mutable attribute (base class "A" defined the type as "Callable[[str], None]", override has type "Callable[[object], None]") + def f(self) -> Callable[[object], None]: pass + @func.setter + def f(self, x: object) -> None: pass +[builtins fixtures/property.pyi] + +[case testOverrideCallableUnionAttributeWithMethod] +from typing import Callable, Union + +class A: + f1: Union[Callable[[str], str], str] + f2: Union[Callable[[str], str], str] + f3: Union[Callable[[str], str], str] + f4: Union[Callable[[str], str], str] + +class B(A): + def f1(self, x: str) -> str: + pass + + def f2(self, x: object) -> str: + pass + + @classmethod + def f3(cls, x: str) -> str: + pass + + @staticmethod + def f4(x: str) -> str: + pass +[builtins fixtures/classmethod.pyi] + +[case testOverrideCallableUnionAttributeWithMethodMutableOverride] +# flags: --enable-error-code=mutable-override +from typing import Callable, Union + +class A: + f1: Union[Callable[[str], str], str] + f2: Union[Callable[[str], str], str] + f3: Union[Callable[[str], str], str] + f4: Union[Callable[[str], str], str] + +class B(A): + def f1(self, x: str) -> str: # E: Covariant override of a mutable attribute (base class "A" defined the type as "Union[Callable[[str], str], str]", override has type "Callable[[str], str]") + pass + + def f2(self, x: object) -> str: # E: Covariant override of a mutable attribute (base class "A" defined the type as "Union[Callable[[str], str], str]", override has type "Callable[[object], str]") + pass + + @classmethod + def f3(cls, x: str) -> str: # E: Covariant override of a mutable attribute (base class "A" defined the type as "Union[Callable[[str], str], str]", override has type "Callable[[str], str]") + pass + + @staticmethod + def f4(x: str) -> str: # E: Covariant override of a mutable attribute (base class "A" defined the type as "Union[Callable[[str], str], str]", override has type "Callable[[str], str]") + pass +[builtins fixtures/classmethod.pyi] + +-- Constructors +-- ------------ + + +[case testTrivialConstructor] +class A: + def __init__(self) -> None: pass + +a = A() # type: A +b = A() # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") +class B: pass +[case testConstructor] +class A: + def __init__(self, x: 'B') -> None: pass +class B: pass + +a = A(B()) # type: A +aa = A(object()) # type: A # E: Argument 1 to "A" has incompatible type "object"; expected "B" +b = A(B()) # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testConstructorWithTwoArguments] +class A: + def __init__(self, x: 'B', y: 'C') -> None: pass +class B: pass +class C(B): pass + +a = A(C(), B()) # type: A # E: Argument 2 to "A" has incompatible type "B"; expected "C" + +[case testInheritedConstructor] +class B(A): pass +class C: pass +class D: pass + +b = B(C()) # type: B +a = B(D()) # type: A # E: Argument 1 to "B" has incompatible type "D"; expected "C" +class A: + def __init__(self, x: 'C') -> None: pass + +[case testOverridingWithIncompatibleConstructor] +class A: + def __init__(self, x: 'C') -> None: pass +class B(A): + def __init__(self) -> None: pass +class C: pass + +A() # E: Missing positional argument "x" in call to "A" +B(C()) # E: Too many arguments for "B" +A(C()) +B() + +[case testConstructorWithReturnValueType] +import typing +class A: + def __init__(self) -> 'A': pass +[out] +main:3: error: The return type of "__init__" must be None + +[case testConstructorWithImplicitReturnValueType] +import typing +class A: + def __init__(self, x: int): pass +[out] + +[case testDecoratedConstructorWithImplicitReturnValueType] +import typing +from typing import Callable + +def deco(fn: Callable) -> Callable: + return fn + +class A: + @deco + def __init__(self, x: int): pass +[out] + +[case testOverloadedConstructorWithImplicitReturnValueType] +from foo import * +[file foo.pyi] +from typing import overload +class Foo: + @overload + def __init__(self, a: int): + pass + + @overload + def __init__(self, a: str): + pass + +[case testConstructorWithAnyReturnValueType] +import typing +from typing import Any +class A: + def __init__(self) -> Any: pass # E: The return type of "__init__" must be None + +[case testDecoratedConstructorWithAnyReturnValueType] +import typing +from typing import Callable, Any + +def deco(fn: Callable) -> Callable: + return fn + +class A: + @deco + def __init__(self) -> Any: pass # E: The return type of "__init__" must be None + +[case testOverloadedConstructorWithAnyReturnValueType] +from foo import * +[file foo.pyi] +from typing import overload, Any +class Foo: + @overload + def __init__(self, a: int) -> Any: # E: The return type of "__init__" must be None + pass + + @overload + def __init__(self, a: str) -> Any: # E: The return type of "__init__" must be None + pass + +[case testInitSubclassWithReturnValueType] +import typing +class A: + def __init_subclass__(cls) -> 'A': pass +[out] +main:3: error: The return type of "__init_subclass__" must be None + +[case testInitSubclassWithImplicitReturnValueType] +import typing +class A: + def __init_subclass__(cls, x: int=1): pass +[out] + +[case testDecoratedInitSubclassWithImplicitReturnValueType] +import typing +from typing import Callable + +def deco(fn: Callable) -> Callable: + return fn + +class A: + @deco + def __init_subclass__(cls, x: int=1): pass +[out] + +[case testOverloadedInitSubclassWithImplicitReturnValueType] +from foo import * +[file foo.pyi] +from typing import overload +class Foo: + @overload + def __init_subclass__(cls, a: int): + pass + + @overload + def __init_subclass__(cls, a: str): + pass + +[case testInitSubclassWithAnyReturnValueType] +import typing +from typing import Any +class A: + def __init_subclass__(cls) -> Any: pass # E: The return type of "__init_subclass__" must be None + +[case testDecoratedInitSubclassWithAnyReturnValueType] +import typing +from typing import Callable, Any + +def deco(fn: Callable) -> Callable: + return fn + +class A: + @deco + def __init_subclass__(cls) -> Any: pass # E: The return type of "__init_subclass__" must be None +[out] + +[case testOverloadedInitSubclassWithAnyReturnValueType] +from foo import * +[file foo.pyi] +from typing import overload, Any +class Foo: + @overload + def __init_subclass__(cls, a: int) -> Any: # E: The return type of "__init_subclass__" must be None + pass + + @overload + def __init_subclass__(cls, a: str) -> Any: # E: The return type of "__init_subclass__" must be None + pass + +[case testGlobalFunctionInitWithReturnType] +class A: pass +class B: pass +def __init__() -> 'A': pass + +a = __init__() # type: A +b = __init__() # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") +[case testAccessingInit] +from typing import Any, cast +class A: + def __init__(self, a: 'A') -> None: pass +a: A +a.__init__(a) # E: Accessing "__init__" on an instance is unsound, since instance.__init__ could be from an incompatible subclass +(cast(Any, a)).__init__(a) + +[case testDeepInheritanceHierarchy] +class A: pass +class B(A): pass +class C(B): pass +class D(C): pass +class D2(C): pass + +d = C() # type: D # E: Incompatible types in assignment (expression has type "C", variable has type "D") +if int(): + d = B() # E: Incompatible types in assignment (expression has type "B", variable has type "D") +if int(): + d = A() # E: Incompatible types in assignment (expression has type "A", variable has type "D") +if int(): + d = D2() # E: Incompatible types in assignment (expression has type "D2", variable has type "D") +a = D() # type: A +if int(): + a = D2() +b = D() # type: B +if int(): + b = D2() + +[case testConstructorJoinsWithCustomMetaclass] +from typing import TypeVar +import abc + +def func() -> None: pass +class NormalClass: pass +class WithMetaclass(metaclass=abc.ABCMeta): pass + +T = TypeVar('T') +def join(x: T, y: T) -> T: pass + +f1 = join(func, WithMetaclass) +reveal_type(f1()) # N: Revealed type is "Union[__main__.WithMetaclass, None]" + +f2 = join(WithMetaclass, func) +reveal_type(f2()) # N: Revealed type is "Union[__main__.WithMetaclass, None]" + +-- Attribute access in class body +-- ------------------------------ + + +[case testDataAttributeRefInClassBody] +import typing +class B: pass +class A: + x = B() + y = x + b = x # type: B + if int(): + b = x + c = x # type: A # E: Incompatible types in assignment (expression has type "B", variable has type "A") + if int(): + c = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[out] + +[case testMethodRefInClassBody] +from typing import Callable +class B: pass +class A: + def f(self) -> None: pass + g = f + h = f # type: Callable[[A], None] + if int(): + h = f + g = h + ff = f # type: Callable[[B], None] # E: Incompatible types in assignment (expression has type "Callable[[A], None]", variable has type "Callable[[B], None]") + if int(): + g = ff # E: Incompatible types in assignment (expression has type "Callable[[B], None]", variable has type "Callable[[A], None]") +[out] + + +-- Arbitrary statements in class body +-- ---------------------------------- + + +[case testStatementsInClassBody] +import typing +class B: pass +class A: + for x in [A()]: + y = x + if int(): + y = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + if int(): + x = A() + if int(): + y = A() + if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[builtins fixtures/for.pyi] +[out] + + +-- Class attributes +-- ---------------- + + +[case testAccessMethodViaClass] +import typing +class A: + def f(self) -> None: pass +A.f(A()) +A.f(object()) # E: Argument 1 to "f" of "A" has incompatible type "object"; expected "A" +A.f() # E: Missing positional argument "self" in call to "f" of "A" +A.f(None, None) # E: Too many arguments for "f" of "A" \ + # E: Argument 1 to "f" of "A" has incompatible type "None"; expected "A" + +[case testAccessAttributeViaClass] +import typing +class B: pass +class A: + x: A +a = A.x # type: A +b = A.x # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testAccessingUndefinedAttributeViaClass] +import typing +class A: pass +A.x # E: "Type[A]" has no attribute "x" + +[case testAccessingUndefinedAttributeViaClassWithOverloadedInit] +from foo import * +[file foo.pyi] +from typing import overload +class A: + @overload + def __init__(self): pass + @overload + def __init__(self, x): pass +A.x # E: "Type[A]" has no attribute "x" + +[case testAccessMethodOfClassWithOverloadedInit] +from foo import * +[file foo.pyi] +from typing import overload, Any +class A: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: Any) -> None: pass + def f(self) -> None: pass +A.f(A()) +A.f() # E: Missing positional argument "self" in call to "f" of "A" + +[case testAssignmentToClassDataAttribute] +import typing +class B: pass +class A: + x: B +A.x = B() +A.x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "B") + +[case testAssignmentToInferredClassDataAttribute] +import typing +class B: pass +class A: + x = B() +A.x = B() +A.x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testInitMethodUnbound] + +class B: pass +class A: + def __init__(self, b: B) -> None: pass +a: A +b: B +A.__init__(a, b) +A.__init__(b, b) # E: Argument 1 to "__init__" of "A" has incompatible type "B"; expected "A" +A.__init__(a, a) # E: Argument 2 to "__init__" of "A" has incompatible type "A"; expected "B" + +[case testAssignToMethodViaClass] +import typing +class A: + def f(self): pass +A.f = None # E: Cannot assign to a method \ + # E: Incompatible types in assignment (expression has type "None", variable has type "Callable[[A], Any]") + +[case testAssignToNestedClassViaClass] +import typing +class A: + class B: pass +A.B = None # E: Cannot assign to a type \ + # E: Incompatible types in assignment (expression has type "None", variable has type "Type[B]") +[targets __main__] + +[case testAccessingClassAttributeWithTypeInferenceIssue] +x = C.x # E: Cannot determine type of "x" # E: Name "C" is used before definition +def f() -> int: return 1 +class C: + x = f() +[builtins fixtures/list.pyi] + +[case testAccessingClassAttributeWithTypeInferenceIssue2] +class C: + x = [] +x = C.x +[builtins fixtures/list.pyi] +[out] +main:2: error: Need type annotation for "x" (hint: "x: List[] = ...") + +[case testAccessingGenericClassAttribute] +from typing import Generic, TypeVar +T = TypeVar('T') +class A(Generic[T]): + x = None # type: T +A.x # E: Access to generic instance variables via class is ambiguous +A[int].x # E: Access to generic instance variables via class is ambiguous +[targets __main__] + +[case testAccessingNestedGenericClassAttribute] +from typing import Generic, List, TypeVar, Union +T = TypeVar('T') +U = TypeVar('U') +class A(Generic[T, U]): + x = None # type: Union[T, List[U]] +A.x # E: Access to generic instance variables via class is ambiguous +A[int, int].x # E: Access to generic instance variables via class is ambiguous +[builtins fixtures/list.pyi] + + +-- Nested classes +-- -------------- + + +[case testClassWithinFunction] +def f() -> None: + class A: + def g(self) -> None: pass + a: A + a.g() + a.g(a) # E: Too many arguments for "g" of "A" +[targets __main__, __main__.f] + +[case testGenericClassWithinFunction] +from typing import TypeVar + +def test() -> None: + T = TypeVar('T', bound='Foo') + class Foo: + def returns_int(self) -> int: + return 0 + + def bar(self, foo: T) -> T: + x: T = foo + reveal_type(x) # N: Revealed type is "T`-1" + reveal_type(x.returns_int()) # N: Revealed type is "builtins.int" + return foo + reveal_type(Foo.bar) # N: Revealed type is "def [T <: __main__.Foo@5] (self: __main__.Foo@5, foo: T`1) -> T`1" + +[case testGenericClassWithInvalidTypevarUseWithinFunction] +from typing import TypeVar + +def test() -> None: + T = TypeVar('T', bound='Foo') + class Foo: + invalid: T # E: Type variable "T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) + + def bar(self, foo: T) -> T: + pass + +[case testConstructNestedClass] +import typing +class A: + class B: pass + b = B() + if int(): + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + if int(): + b = B(b) # E: Too many arguments for "B" +[out] + +[case testConstructNestedClassWithCustomInit] +import typing +class A: + def f(self) -> None: + class B: + def __init__(self, a: 'A') -> None: pass + b = B(A()) + if int(): + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + b = B() # E: Missing positional argument "a" in call to "B" +[out] + +[case testDeclareVariableWithNestedClassType] + +def f() -> None: + class A: pass + a: A + if int(): + a = A() + a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +[out] + +[case testExternalReferenceToClassWithinClass] +class A: + class B: pass +b: A.B +if int(): + b = A.B() +if int(): + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = A.B(b) # E: Too many arguments for "B" + +[case testAliasNestedClass] +class Outer: + class Inner: + def make_int(self) -> int: return 1 + reveal_type(Inner().make_int) # N: Revealed type is "def () -> builtins.int" + some_int = Inner().make_int() + +reveal_type(Outer.Inner.make_int) # N: Revealed type is "def (self: __main__.Outer.Inner) -> builtins.int" +reveal_type(Outer().some_int) # N: Revealed type is "builtins.int" +Bar = Outer.Inner +reveal_type(Bar.make_int) # N: Revealed type is "def (self: __main__.Outer.Inner) -> builtins.int" +x = Bar() # type: Bar +def produce() -> Bar: + reveal_type(Bar().make_int) # N: Revealed type is "def () -> builtins.int" + return Bar() + +[case testInnerClassPropertyAccess] +class Foo: + class Meta: + name = 'Bar' + meta = Meta + +reveal_type(Foo.Meta) # N: Revealed type is "def () -> __main__.Foo.Meta" +reveal_type(Foo.meta) # N: Revealed type is "def () -> __main__.Foo.Meta" +reveal_type(Foo.Meta.name) # N: Revealed type is "builtins.str" +reveal_type(Foo.meta.name) # N: Revealed type is "builtins.str" +reveal_type(Foo().Meta) # N: Revealed type is "def () -> __main__.Foo.Meta" +reveal_type(Foo().meta) # N: Revealed type is "def () -> __main__.Foo.Meta" +reveal_type(Foo().meta.name) # N: Revealed type is "builtins.str" +reveal_type(Foo().Meta.name) # N: Revealed type is "builtins.str" + +-- Declaring attribute type in method +-- ---------------------------------- + + +[case testDeclareAttributeTypeInInit] + +class A: + def __init__(self): + self.x: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs +a: A +a.x = 1 +a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testAccessAttributeDeclaredInInitBeforeDeclaration] + +a: A +a.x = 1 +a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +class A: + def __init__(self): + self.x: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs + + +-- Special cases +-- ------------- + + +[case testMultipleClassDefinition] +class A: pass +class A: pass # E: Name "A" already defined on line 1 +A() +[case testDocstringInClass] +import typing +class A: + """Foo""" +class B: + 'x' + y = B() +[builtins fixtures/primitives.pyi] + +[case testErrorMessageInFunctionNestedWithinMethod] +import typing +class A: + def f(self) -> None: + def g() -> None: + "" + 1 # E: Unsupported operand types for + ("str" and "int") + "" + 1 # E: Unsupported operand types for + ("str" and "int") +[builtins fixtures/primitives.pyi] + +-- Static methods +-- -------------- + + +[case testSimpleStaticMethod] +import typing +class A: + @staticmethod + def f(x: int) -> None: pass +A.f(1) +A().f(1) +A.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +[builtins fixtures/staticmethod.pyi] + +[case testBuiltinStaticMethod] +import typing +int.from_bytes(b'', '') +int.from_bytes('', '') # E: Argument 1 to "from_bytes" of "int" has incompatible type "str"; expected "bytes" +[builtins fixtures/staticmethod.pyi] + +[case testAssignStaticMethodOnInstance] +import typing +class A: + @staticmethod + def f(x: int) -> None: pass +A().f = A.f # E: Cannot assign to a method +[builtins fixtures/staticmethod.pyi] + + +-- Class methods +-- ------------- + + +[case testSimpleClassMethod] +import typing +class A: + @classmethod + def f(cls, x: int) -> None: pass +A.f(1) +A().f(1) +A.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +[builtins fixtures/classmethod.pyi] +[targets __main__, __main__.A.f] + +[case testBuiltinClassMethod] +import typing +int.from_bytes(b'', '') +int.from_bytes('', '') # E: Argument 1 to "from_bytes" of "int" has incompatible type "str"; expected "bytes" +[builtins fixtures/classmethod.pyi] + +[case testAssignClassMethodOnClass] +import typing +class A: + @classmethod + def f(cls, x: int) -> None: pass +A.f = A.f # E: Cannot assign to a method +[builtins fixtures/classmethod.pyi] + +[case testAssignClassMethodOnInstance] +import typing +class A: + @classmethod + def f(cls, x: int) -> None: pass +A().f = A.f # E: Cannot assign to a method +[builtins fixtures/classmethod.pyi] + +[case testClassMethodCalledInClassMethod] +import typing +class C: + @classmethod + def foo(cls) -> None: pass + @classmethod + def bar(cls) -> None: + cls() + cls(1) # E: Too many arguments for "C" + cls.bar() + cls.bar(1) # E: Too many arguments for "bar" of "C" + cls.bozo() # E: "Type[C]" has no attribute "bozo" +[builtins fixtures/classmethod.pyi] +[out] + +[case testClassMethodCalledOnClass] +import typing +class C: + @classmethod + def foo(cls) -> None: pass +C.foo() +C.foo(1) # E: Too many arguments for "foo" of "C" +C.bozo() # E: "Type[C]" has no attribute "bozo" +[builtins fixtures/classmethod.pyi] + +[case testClassMethodCalledOnInstance] +import typing +class C: + @classmethod + def foo(cls) -> None: pass +C().foo() +C().foo(1) # E: Too many arguments for "foo" of "C" +C.bozo() # E: "Type[C]" has no attribute "bozo" +[builtins fixtures/classmethod.pyi] + +[case testClassMethodMayCallAbstractMethod] +from abc import abstractmethod +import typing +class C: + @classmethod + def foo(cls) -> None: + cls().bar() + @abstractmethod + def bar(self) -> None: + pass +[builtins fixtures/classmethod.pyi] + +[case testClassMethodSubclassing] +class A: + @classmethod + def f(cls) -> None: pass + + def g(self) -> None: pass + +class B(A): + def f(self) -> None: pass # Fail + + @classmethod + def g(cls) -> None: pass + +class C(A): + @staticmethod + def f() -> None: pass +[builtins fixtures/classmethod.pyi] +[out] +main:8: error: Signature of "f" incompatible with supertype "A" +main:8: note: Superclass: +main:8: note: @classmethod +main:8: note: def f(cls) -> None +main:8: note: Subclass: +main:8: note: def f(self) -> None + +[case testClassMethodAndStaticMethod] +class C: + @classmethod # E: Cannot have both classmethod and staticmethod + @staticmethod + def foo(cls) -> None: pass +[builtins fixtures/classmethod.pyi] + +-- Properties +-- ---------- + + +[case testAccessingReadOnlyProperty] +import typing +class A: + @property + def f(self) -> str: pass +a = A() +reveal_type(a.f) # N: Revealed type is "builtins.str" +[builtins fixtures/property.pyi] + +[case testAssigningToReadOnlyProperty] +import typing +class A: + @property + def f(self) -> str: pass +A().f = '' # E: Property "f" defined in "A" is read-only +[builtins fixtures/property.pyi] + +[case testAssigningToInheritedReadOnlyProperty] +class A: + @property + def f(self) -> str: pass +class B(A): pass +class C(A): + @property + def f(self) -> str: pass + +A().f = '' # E: Property "f" defined in "A" is read-only +B().f = '' # E: Property "f" defined in "A" is read-only +C().f = '' # E: Property "f" defined in "C" is read-only +[builtins fixtures/property.pyi] + +[case testPropertyGetterBody] +import typing +class A: + @property + def f(self) -> str: + self.x = 1 + self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + return '' +[builtins fixtures/property.pyi] +[out] + +[case testDynamicallyTypedProperty] +import typing +class A: + @property + def f(self): pass +a = A() +a.f.xx +a.f = '' # E: Property "f" defined in "A" is read-only +[builtins fixtures/property.pyi] + +[case testPropertyWithSetter] +import typing +class A: + @property + def f(self) -> int: + return 1 + @f.setter + def f(self, x: int) -> None: + pass +a = A() +a.f = a.f +a.f.x # E: "int" has no attribute "x" +a.f = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +a.f = 1 +reveal_type(a.f) # N: Revealed type is "builtins.int" +[builtins fixtures/property.pyi] + +[case testPropertyWithDeleterButNoSetter] +import typing +class A: + @property + def f(self) -> int: + return 1 + @f.deleter + def f(self, x) -> None: + pass +a = A() +a.f = a.f # E: Property "f" defined in "A" is read-only +a.f.x # E: "int" has no attribute "x" +[builtins fixtures/property.pyi] + +-- Descriptors +-- ----------- + + +[case testAccessingNonDataDescriptor] +from typing import Any +class D: + def __get__(self, inst: Any, own: Any) -> str: return 's' +class A: + f = D() +a = A() +reveal_type(a.f) # N: Revealed type is "builtins.str" + +[case testSettingNonDataDescriptor] +from typing import Any +class D: + def __get__(self, inst: Any, own: Any) -> str: return 's' +class A: + f = D() +a = A() +a.f = 'foo' +a.f = D() # E: Incompatible types in assignment (expression has type "D", variable has type "str") + +[case testSettingDataDescriptor] +from typing import Any +class D: + def __get__(self, inst: Any, own: Any) -> str: return 's' + def __set__(self, inst: Any, value: str) -> None: pass +class A: + f = D() +a = A() +a.f = '' +a.f = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testSettingDescriptorWithOverloadedDunderSet1] +from typing import Any, overload, Union +class D: + @overload + def __set__(self, inst: Any, value: str) -> None: pass + @overload + def __set__(self, inst: Any, value: int) -> None: pass + def __set__(self, inst: Any, value: Union[str, int]) -> None: pass +class A: + f = D() +a = A() +a.f = '' +a.f = 1 +a.f = 1.5 # E +[out] +main:13: error: No overload variant of "__set__" of "D" matches argument types "A", "float" +main:13: note: Possible overload variants: +main:13: note: def __set__(self, inst: Any, value: str) -> None +main:13: note: def __set__(self, inst: Any, value: int) -> None + +[case testSettingDescriptorWithOverloadedDunderSet2] +from typing import overload, Union +class D: + @overload + def __set__(self, inst: A, value: str) -> None: pass + @overload + def __set__(self, inst: B, value: int) -> None: pass + def __set__(self, inst: Union[A, B], value: Union[str, int]) -> None: pass +class A: + f = D() +class B: + f = D() +a = A() +b = B() +a.f = '' +b.f = 1 +a.f = 1 # E +b.f = '' # E +[out] +main:16: error: No overload variant of "__set__" of "D" matches argument types "A", "int" +main:16: note: Possible overload variants: +main:16: note: def __set__(self, inst: A, value: str) -> None +main:16: note: def __set__(self, inst: B, value: int) -> None +main:17: error: No overload variant of "__set__" of "D" matches argument types "B", "str" +main:17: note: Possible overload variants: +main:17: note: def __set__(self, inst: A, value: str) -> None +main:17: note: def __set__(self, inst: B, value: int) -> None + +[case testReadingDescriptorWithoutDunderGet] +from typing import Union, Any +class D: + def __set__(self, inst: Any, value: str) -> None: pass +class A: + f = D() + def __init__(self): self.f = 's' +a = A() +reveal_type(a.f) # N: Revealed type is "__main__.D" + +[case testAccessingDescriptorFromClass] +from d import D, Base +class A(Base): + f = D() +reveal_type(A.f) # N: Revealed type is "d.D" +reveal_type(A().f) # N: Revealed type is "builtins.str" +[file d.pyi] +from typing import TypeVar, Type, Generic, overload +class Base: pass +class D: + def __init__(self) -> None: pass + @overload + def __get__(self, inst: None, own: Type[Base]) -> D: pass + @overload + def __get__(self, inst: Base, own: Type[Base]) -> str: pass +[builtins fixtures/bool.pyi] + +[case testAccessingDescriptorFromClassWrongBase] +from d import D, Base +class A: + f = D() +reveal_type(A.f) +reveal_type(A().f) +[file d.pyi] +from typing import TypeVar, Type, Generic, overload +class Base: pass +class D: + def __init__(self) -> None: pass + @overload + def __get__(self, inst: None, own: Type[Base]) -> D: pass + @overload + def __get__(self, inst: Base, own: Type[Base]) -> str: pass +[builtins fixtures/bool.pyi] +[out] +main:4: error: Argument 2 to "__get__" of "D" has incompatible type "Type[A]"; expected "Type[Base]" +main:4: note: Revealed type is "d.D" +main:5: error: No overload variant of "__get__" of "D" matches argument types "A", "Type[A]" +main:5: note: Possible overload variants: +main:5: note: def __get__(self, inst: None, own: Type[Base]) -> D +main:5: note: def __get__(self, inst: Base, own: Type[Base]) -> str +main:5: note: Revealed type is "Any" + +[case testAccessingGenericNonDataDescriptor] +from typing import TypeVar, Type, Generic, Any +V = TypeVar('V') +class D(Generic[V]): + def __init__(self, v: V) -> None: self.v = v + def __get__(self, inst: Any, own: Type) -> V: return self.v +class A: + f = D(10) + g = D('10') +a = A() +reveal_type(a.f) # N: Revealed type is "builtins.int" +reveal_type(a.g) # N: Revealed type is "builtins.str" + +[case testSettingGenericDataDescriptor] +from typing import TypeVar, Type, Generic, Any +V = TypeVar('V') +class D(Generic[V]): + def __init__(self, v: V) -> None: self.v = v + def __get__(self, inst: Any, own: Type) -> V: return self.v + def __set__(self, inst: Any, v: V) -> None: pass +class A: + f = D(10) + g = D('10') +a = A() +a.f = 1 +a.f = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +a.g = '' +a.g = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAccessingGenericDescriptorFromClass] +from d import D +class A: + f = D(10) # type: D[A, int] + g = D('10') # type: D[A, str] +reveal_type(A.f) # N: Revealed type is "d.D[__main__.A, builtins.int]" +reveal_type(A.g) # N: Revealed type is "d.D[__main__.A, builtins.str]" +reveal_type(A().f) # N: Revealed type is "builtins.int" +reveal_type(A().g) # N: Revealed type is "builtins.str" +[file d.pyi] +from typing import TypeVar, Type, Generic, overload +T = TypeVar('T') +V = TypeVar('V') +class D(Generic[T, V]): + def __init__(self, v: V) -> None: pass + @overload + def __get__(self, inst: None, own: Type[T]) -> 'D[T, V]': pass + @overload + def __get__(self, inst: T, own: Type[T]) -> V: pass +[builtins fixtures/bool.pyi] + +[case testAccessingGenericDescriptorFromInferredClass] +from typing import Type +from d import D +class A: + f = D(10) # type: D[A, int] + g = D('10') # type: D[A, str] +def f(some_class: Type[A]): + reveal_type(some_class.f) + reveal_type(some_class.g) +[file d.pyi] +from typing import TypeVar, Type, Generic, overload +T = TypeVar('T') +V = TypeVar('V') +class D(Generic[T, V]): + def __init__(self, v: V) -> None: pass + @overload + def __get__(self, inst: None, own: Type[T]) -> 'D[T, V]': pass + @overload + def __get__(self, inst: T, own: Type[T]) -> V: pass +[builtins fixtures/bool.pyi] +[out] +main:7: note: Revealed type is "d.D[__main__.A, builtins.int]" +main:8: note: Revealed type is "d.D[__main__.A, builtins.str]" + +[case testAccessingGenericDescriptorFromClassBadOverload] +from d import D +class A: + f = D(10) # type: D[A, int] +reveal_type(A.f) +[file d.pyi] +from typing import TypeVar, Type, Generic, overload +T = TypeVar('T') +V = TypeVar('V') +class D(Generic[T, V]): + def __init__(self, v: V) -> None: pass + @overload + def __get__(self, inst: None, own: None) -> 'D[T, V]': pass + @overload + def __get__(self, inst: T, own: Type[T]) -> V: pass +[builtins fixtures/bool.pyi] +[out] +main:4: error: No overload variant of "__get__" of "D" matches argument types "None", "Type[A]" +main:4: note: Possible overload variants: +main:4: note: def __get__(self, inst: None, own: None) -> D[A, int] +main:4: note: def __get__(self, inst: A, own: Type[A]) -> int +main:4: note: Revealed type is "Any" + +[case testAccessingNonDataDescriptorSubclass] +from typing import Any +class C: + def __get__(self, inst: Any, own: Any) -> str: return 's' +class D(C): pass +class A: + f = D() +a = A() +reveal_type(a.f) # N: Revealed type is "builtins.str" + +[case testSettingDataDescriptorSubclass] +from typing import Any +class C: + def __get__(self, inst: Any, own: Any) -> str: return 's' + def __set__(self, inst: Any, v: str) -> None: pass +class D(C): pass +class A: + f = D() +a = A() +a.f = '' +a.f = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testReadingDescriptorSubclassWithoutDunderGet] +from typing import Union, Any +class C: + def __set__(self, inst: Any, v: str) -> None: pass +class D(C): pass +class A: + f = D() + def __init__(self): self.f = 's' +a = A() +reveal_type(a.f) # N: Revealed type is "__main__.D" + +[case testAccessingGenericNonDataDescriptorSubclass] +from typing import TypeVar, Type, Generic, Any +V = TypeVar('V') +class C(Generic[V]): + def __init__(self, v: V) -> None: self.v = v + def __get__(self, inst: Any, own: Type) -> V: return self.v +class D(C[V], Generic[V]): pass +class A: + f = D(10) + g = D('10') +a = A() +reveal_type(a.f) # N: Revealed type is "builtins.int" +reveal_type(a.g) # N: Revealed type is "builtins.str" + +[case testSettingGenericDataDescriptorSubclass] +from typing import TypeVar, Type, Generic +T = TypeVar('T') +V = TypeVar('V') +class C(Generic[T, V]): + def __init__(self, v: V) -> None: self.v = v + def __get__(self, inst: T, own: Type[T]) -> V: return self.v + def __set__(self, inst: T, v: V) -> None: pass +class D(C[T, V], Generic[T, V]): pass +class A: + f = D(10) # type: D[A, int] + g = D('10') # type: D[A, str] +a = A() +a.f = 1 +a.f = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +a.g = '' +a.g = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testSetDescriptorOnClass] +from typing import TypeVar, Type, Generic +T = TypeVar('T') +V = TypeVar('V') +class D(Generic[T, V]): + def __init__(self, v: V) -> None: self.v = v + def __get__(self, inst: T, own: Type[T]) -> V: return self.v + def __set__(self, inst: T, v: V) -> None: pass +class A: + f = D(10) # type: D[A, int] +A.f = D(20) +A.f = D('some string') # E: Argument 1 to "D" has incompatible type "str"; expected "int" + +[case testSetDescriptorOnInferredClass] +from typing import TypeVar, Type, Generic, Any +V = TypeVar('V') +class D(Generic[V]): + def __init__(self, v: V) -> None: self.v = v + def __get__(self, inst: Any, own: Type) -> V: return self.v + def __set__(self, inst: Any, v: V) -> None: pass +class A: + f = D(10) +def f(some_class: Type[A]): + A.f = D(20) + A.f = D('some string') +[out] +main:11: error: Argument 1 to "D" has incompatible type "str"; expected "int" + +[case testDescriptorUncallableDunderSet] +class D: + __set__ = 's' +class A: + f = D() +A().f = 'x' # E: __main__.D.__set__ is not callable + +[case testDescriptorDunderSetTooFewArgs] +class D: + def __set__(self, inst): pass +class A: + f = D() +A().f = 'x' # E: Too many arguments for "__set__" + +[case testDescriptorDunderSetTooManyArgs] +class D: + def __set__(self, inst, v, other): pass +class A: + f = D() +A().f = 'x' # E: Too few arguments for "__set__" + +[case testDescriptorDunderSetWrongArgTypes] +class D: + def __set__(self, inst: str, v:str) -> None: pass +class A: + f = D() +A().f = 'x' # E: Argument 1 to "__set__" of "D" has incompatible type "A"; expected "str" + +[case testDescriptorUncallableDunderGet] +class D: + __get__ = 's' +class A: + f = D() +A().f # E: __main__.D.__get__ is not callable + +[case testDescriptorDunderGetTooFewArgs] +class D: + def __get__(self, inst): pass +class A: + f = D() +A().f # E: Too many arguments for "__get__" + +[case testDescriptorDunderGetTooManyArgs] +class D: + def __get__(self, inst, own, other): pass +class A: + f = D() +A().f = 'x' # E: Too few arguments for "__get__" + +[case testDescriptorDunderGetWrongArgTypeForInstance] +from typing import Any +class D: + def __get__(self, inst: str, own: Any) -> Any: pass +class A: + f = D() +A().f # E: Argument 1 to "__get__" of "D" has incompatible type "A"; expected "str" + +[case testDescriptorDunderGetWrongArgTypeForOwner] +from typing import Any +class D: + def __get__(self, inst: Any, own: str) -> Any: pass +class A: + f = D() +A().f # E: Argument 2 to "__get__" of "D" has incompatible type "Type[A]"; expected "str" + +[case testDescriptorGetSetDifferentTypes] +from typing import Any +class D: + def __get__(self, inst: Any, own: Any) -> str: return 's' + def __set__(self, inst: Any, v: int) -> None: pass +class A: + f = D() +a = A() +a.f = 1 +reveal_type(a.f) # N: Revealed type is "builtins.str" + +[case testDescriptorGetUnion] +from typing import Any, Union +class String: + def __get__(self, inst: Any, owner: Any) -> str: + return '' + +class A: + attr: str + +class B: + attr = String() + +def foo(x: Union[A, B]) -> None: + reveal_type(x.attr) # N: Revealed type is "builtins.str" + +[case testDescriptorGetUnionRestricted] +from typing import Any, Union + +class getter: + def __get__(self, instance: X1, owner: Any) -> str: ... + +class X1: + prop = getter() + +class X2: + prop: str + +def foo(x: Union[X1, X2]) -> None: + reveal_type(x.prop) # N: Revealed type is "builtins.str" + +[case testDescriptorGetUnionType] +from typing import Any, Union, Type, overload + +class getter: + @overload + def __get__(self, instance: None, owner: Any) -> getter: ... + @overload + def __get__(self, instance: object, owner: Any) -> str: ... + def __get__(self, instance, owner): + ... + +class X1: + prop = getter() +class X2: + prop = getter() + +def foo(x: Type[Union[X1, X2]]) -> None: + reveal_type(x.prop) # N: Revealed type is "__main__.getter" + + +-- _promote decorators +-- ------------------- + + +[case testSimpleDucktypeDecorator] +from typing import _promote +class A: pass +@_promote(A) +class B: pass +a: A +b: B +if int(): + b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = b +[typing fixtures/typing-medium.pyi] + +[case testDucktypeTransitivityDecorator] +from typing import _promote +class A: pass +@_promote(A) +class B: pass +@_promote(B) +class C: pass +a: A +c: C +if int(): + c = a # E: Incompatible types in assignment (expression has type "A", variable has type "C") + a = c +[typing fixtures/typing-medium.pyi] + + +-- Hard coded type promotions +-- -------------------------- + +[case testHardCodedTypePromotions] +import typing +def f(x: float) -> None: pass +def g(x: complex) -> None: pass +f(1) +g(1) +g(1.1) +[builtins fixtures/complex.pyi] + + +-- Operator methods +-- ---------------- + + +[case testOperatorMethodOverrideIntroducingOverloading] +from foo import * +[file foo.pyi] +from typing import overload +class A: + def __add__(self, x: int) -> int: pass +class B(A): + @overload # Fail + def __add__(self, x: int) -> int: pass + @overload + def __add__(self, x: str) -> str: pass +[out] +tmp/foo.pyi:5: error: Signature of "__add__" incompatible with supertype "A" +tmp/foo.pyi:5: note: Superclass: +tmp/foo.pyi:5: note: def __add__(self, int, /) -> int +tmp/foo.pyi:5: note: Subclass: +tmp/foo.pyi:5: note: @overload +tmp/foo.pyi:5: note: def __add__(self, int, /) -> int +tmp/foo.pyi:5: note: @overload +tmp/foo.pyi:5: note: def __add__(self, str, /) -> str +tmp/foo.pyi:5: note: Overloaded operator methods can't have wider argument types in overrides + +[case testOperatorMethodOverrideWideningArgumentType] +import typing +class A: + def __add__(self, x: int) -> int: pass +class B(A): + def __add__(self, x: object) -> int: pass +[out] + +[case testOperatorMethodOverrideNarrowingReturnType] +import typing +class A: + def __add__(self, x: int) -> 'A': pass +class B(A): + def __add__(self, x: int) -> 'B': pass + +[case testOperatorMethodOverrideWithDynamicallyTyped] +import typing +class A: + def __add__(self, x: int) -> 'A': pass +class B(A): + def __add__(self, x): pass + +[case testOperatorMethodAgainstSameType] +class A: + def __add__(self, x: int) -> 'A': + if isinstance(x, int): + return A() + else: + return NotImplemented + + def __radd__(self, x: 'A') -> 'A': + if isinstance(x, A): + return A() + else: + return NotImplemented + +class B(A): pass + +# Note: This is a runtime error. If we run x.__add__(y) +# where x and y are *not* the same type, Python will not try +# calling __radd__. +A() + A() # E: Unsupported operand types for + ("A" and "A") + +# Here, Python *will* call __radd__(...) +reveal_type(B() + A()) # N: Revealed type is "__main__.A" +reveal_type(A() + B()) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testBinaryOpeartorMethodPositionalArgumentsOnly] +class A: + def __add__(self, other: int) -> int: pass + def __iadd__(self, other: int) -> int: pass + def __radd__(self, other: int) -> int: pass + +reveal_type(A.__add__) # N: Revealed type is "def (__main__.A, builtins.int) -> builtins.int" +reveal_type(A.__iadd__) # N: Revealed type is "def (__main__.A, builtins.int) -> builtins.int" +reveal_type(A.__radd__) # N: Revealed type is "def (__main__.A, builtins.int) -> builtins.int" + +[case testOperatorMethodOverrideWithIdenticalOverloadedType] +from foo import * +[file foo.pyi] +from typing import overload +class A: + @overload + def __add__(self, x: int) -> 'A': pass + @overload + def __add__(self, x: str) -> 'A': pass +class B(A): + @overload + def __add__(self, x: int) -> 'A': pass + @overload + def __add__(self, x: str) -> 'A': pass + +[case testOverloadedOperatorMethodOverrideWithDynamicallyTypedMethod] +from foo import * +[file foo.pyi] +from typing import overload, Any +class A: + @overload + def __add__(self, x: int) -> 'A': pass + @overload + def __add__(self, x: str) -> 'A': pass +class B(A): + def __add__(self, x): pass +class C(A): + def __add__(self, x: Any) -> A: pass + +[case testOverloadedOperatorMethodOverrideWithNewItem] +from foo import * +[file foo.pyi] +from typing import overload, Any +class A: + @overload + def __add__(self, x: int) -> 'A': pass + @overload + def __add__(self, x: str) -> 'A': pass +class B(A): + @overload # Fail + def __add__(self, x: int) -> A: pass + @overload + def __add__(self, x: str) -> A: pass + @overload + def __add__(self, x: type) -> A: pass +[out] +tmp/foo.pyi:8: error: Signature of "__add__" incompatible with supertype "A" +tmp/foo.pyi:8: note: Superclass: +tmp/foo.pyi:8: note: @overload +tmp/foo.pyi:8: note: def __add__(self, int, /) -> A +tmp/foo.pyi:8: note: @overload +tmp/foo.pyi:8: note: def __add__(self, str, /) -> A +tmp/foo.pyi:8: note: Subclass: +tmp/foo.pyi:8: note: @overload +tmp/foo.pyi:8: note: def __add__(self, int, /) -> A +tmp/foo.pyi:8: note: @overload +tmp/foo.pyi:8: note: def __add__(self, str, /) -> A +tmp/foo.pyi:8: note: @overload +tmp/foo.pyi:8: note: def __add__(self, type, /) -> A +tmp/foo.pyi:8: note: Overloaded operator methods can't have wider argument types in overrides + +[case testOverloadedOperatorMethodOverrideWithSwitchedItemOrder] +from foo import * +[file foo.pyi] +from typing import overload, Any +class A: + @overload + def __add__(self, x: 'B') -> 'B': pass + @overload + def __add__(self, x: 'A') -> 'A': pass +class B(A): + @overload + def __add__(self, x: 'A') -> 'A': pass + @overload + def __add__(self, x: 'B') -> 'B': pass +class C(A): + @overload + def __add__(self, x: 'B') -> 'B': pass + @overload + def __add__(self, x: 'A') -> 'A': pass +[out] +tmp/foo.pyi:8: error: Signature of "__add__" incompatible with supertype "A" +tmp/foo.pyi:8: note: Overload variants must be defined in the same order as they are in "A" +tmp/foo.pyi:11: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +[case testReverseOperatorMethodArgumentType] +from typing import Any +class A: pass +class B: + def __radd__(self, x: A) -> int: pass # Error +class C: + def __radd__(self, x: A) -> Any: pass +class D: + def __radd__(self, x: A) -> object: pass +[out] + +[case testReverseOperatorMethodArgumentType2] +from typing import Any, Tuple, Callable +class A: + def __radd__(self, x: Tuple[int, str]) -> int: pass +class B: + def __radd__(self, x: Callable[[], int]) -> int: pass +class C: + def __radd__(self, x: Any) -> int: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testReverseOperatorMethodInvalid] +from foo import * +[file foo.pyi] +class A: ... +class B: + def __rmul__(self) -> A: ... +class C: + def __radd__(self, other, oops) -> int: ... +[out] +tmp/foo.pyi:3: error: Invalid signature "Callable[[B], A]" +tmp/foo.pyi:5: error: Invalid signature "Callable[[C, Any, Any], int]" + +[case testReverseOperatorOrderingCase1] +class A: + def __radd__(self, other: 'A') -> int: ... + +# Note: Python only tries calling __add__ and never __radd__, even though it's present +A() + A() # E: Unsupported left operand type for + ("A") + +[case testReverseOperatorOrderingCase2] +class A: + def __lt__(self, other: object) -> bool: ... + +# Not all operators have the above shortcut though. +reveal_type(A() > A()) # N: Revealed type is "builtins.bool" +reveal_type(A() < A()) # N: Revealed type is "builtins.bool" +[builtins fixtures/bool.pyi] + +[case testReverseOperatorOrderingCase3] +class A: + def __add__(self, other: B) -> int: ... + +class B: + def __radd__(self, other: A) -> str: ... # E: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping + +# Normally, we try calling __add__ before __radd__ +reveal_type(A() + B()) # N: Revealed type is "builtins.int" + +[case testReverseOperatorOrderingCase4] +class A: + def __add__(self, other: B) -> int: ... + +class B(A): + def __radd__(self, other: A) -> str: ... # E: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping + +# However, if B is a subtype of A, we try calling __radd__ first. +reveal_type(A() + B()) # N: Revealed type is "builtins.str" + +[case testReverseOperatorOrderingCase5] +# Note: these two methods are not unsafely overlapping because __radd__ is +# never called -- see case 1. +class A: + def __add__(self, other: B) -> int: ... + def __radd__(self, other: A) -> str: ... + +class B(A): pass + +# ...but only if B specifically defines a new __radd__. +reveal_type(A() + B()) # N: Revealed type is "builtins.int" + +[case testReverseOperatorOrderingCase6] +class A: + def __add__(self, other: B) -> int: ... + def __radd__(self, other: A) -> str: ... + +class B(A): + # Although A.__radd__ can never be called, B.__radd__ *can* be -- so the + # unsafe overlap check kicks in here. + def __radd__(self, other: A) -> str: ... # E: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping + +reveal_type(A() + B()) # N: Revealed type is "builtins.str" + +[case testReverseOperatorOrderingCase7] +class A: + def __add__(self, other: B) -> int: ... + def __radd__(self, other: A) -> str: ... + +class B(A): + def __radd__(self, other: A) -> str: ... # E: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping + +class C(B): pass + +# A refinement made by a parent also counts +reveal_type(A() + C()) # N: Revealed type is "builtins.str" + +[case testReverseOperatorWithOverloads1] +from typing import overload + +class A: + def __add__(self, other: C) -> int: ... + +class B: + def __add__(self, other: C) -> int: ... + +class C: + @overload + def __radd__(self, other: A) -> str: ... # E: Signatures of "__radd__" of "C" and "__add__" of "A" are unsafely overlapping + @overload + def __radd__(self, other: B) -> str: ... # E: Signatures of "__radd__" of "C" and "__add__" of "B" are unsafely overlapping + def __radd__(self, other): pass + +reveal_type(A() + C()) # N: Revealed type is "builtins.int" +reveal_type(B() + C()) # N: Revealed type is "builtins.int" + +[case testReverseOperatorWithOverloads2] +from typing import overload, Union + +class Num1: + def __add__(self, other: Num1) -> Num1: ... + def __radd__(self, other: Num1) -> Num1: ... + +class Num2(Num1): + @overload + def __add__(self, other: Num2) -> Num2: ... + @overload + def __add__(self, other: Num1) -> Num2: ... + def __add__(self, other): pass + + @overload + def __radd__(self, other: Num2) -> Num2: ... + @overload + def __radd__(self, other: Num1) -> Num2: ... + def __radd__(self, other): pass + +class Num3(Num1): + def __add__(self, other: Union[Num1, Num3]) -> Num3: ... + def __radd__(self, other: Union[Num1, Num3]) -> Num3: ... + +reveal_type(Num1() + Num2()) # N: Revealed type is "__main__.Num2" +reveal_type(Num2() + Num1()) # N: Revealed type is "__main__.Num2" + +reveal_type(Num1() + Num3()) # N: Revealed type is "__main__.Num3" +reveal_type(Num3() + Num1()) # N: Revealed type is "__main__.Num3" + +reveal_type(Num2() + Num3()) # N: Revealed type is "__main__.Num2" +reveal_type(Num3() + Num2()) # N: Revealed type is "__main__.Num3" + +[case testDivReverseOperator] +# No error: __div__ has no special meaning in Python 3 +class A1: + def __div__(self, x: B1) -> int: ... +class B1: + def __rdiv__(self, x: A1) -> str: ... + +class A2: + def __truediv__(self, x: B2) -> int: ... +class B2: + def __rtruediv__(self, x: A2) -> str: ... # E: Signatures of "__rtruediv__" of "B2" and "__truediv__" of "A2" are unsafely overlapping + +A1() / B1() # E: Unsupported left operand type for / ("A1") +reveal_type(A2() / B2()) # N: Revealed type is "builtins.int" + +[case testReverseOperatorMethodForwardIsAny] +from typing import Any +def deco(f: Any) -> Any: return f +class C: + @deco + def __add__(self, other: C) -> C: return C() + def __radd__(self, other: C) -> C: return C() +[out] + +[case testReverseOperatorMethodForwardIsAny2] +from typing import Any +def deco(f: Any) -> Any: return f +class C: + __add__ = None # type: Any + def __radd__(self, other: C) -> C: return C() +[out] + +[case testReverseOperatorMethodForwardIsAny3] +from typing import Any +def deco(f: Any) -> Any: return f +class C: + __add__ = 42 + def __radd__(self, other: C) -> C: return C() +[out] +main:5: error: Forward operator "__add__" is not callable + +[case testOverloadedReverseOperatorMethodArgumentType] +from foo import * +[file foo.pyi] +from typing import overload, Any +class A: + @overload + def __radd__(self, x: 'A') -> str: pass + @overload + def __radd__(self, x: 'A') -> Any: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +[out] + +[case testReverseOperatorMethodArgumentTypeAndOverloadedMethod] +from foo import * +[file foo.pyi] +from typing import overload +class A: + @overload + def __add__(self, x: int) -> int: pass + @overload + def __add__(self, x: str) -> int: pass + def __radd__(self, x: 'A') -> str: pass + +[case testReverseOperatorStar] +class B: + def __radd__(*self) -> int: pass + def __rsub__(*self: 'B') -> int: pass +[builtins fixtures/tuple.pyi] + +[case testReverseOperatorTypeVar1] +from typing import TypeVar +T = TypeVar("T", bound='Real') +class Real: + def __add__(self, other: object) -> str: ... +class Fraction(Real): + def __radd__(self, other: T) -> T: ... # E: Signatures of "__radd__" of "Fraction" and "__add__" of "T" are unsafely overlapping + +# Note: When doing A + B and if B is a subtype of A, we will always call B.__radd__(A) first +# and only try A.__add__(B) second if necessary. +reveal_type(Real() + Fraction()) # N: Revealed type is "__main__.Real" + +# Note: When doing A + A, we only ever call A.__add__(A), never A.__radd__(A). +reveal_type(Fraction() + Fraction()) # N: Revealed type is "builtins.str" + +[case testReverseOperatorTypeVar2a] +from typing import TypeVar +T = TypeVar("T", bound='Real') +class Real: + def __add__(self, other: Fraction) -> str: ... +class Fraction(Real): + def __radd__(self, other: T) -> T: ... # E: Signatures of "__radd__" of "Fraction" and "__add__" of "T" are unsafely overlapping + +reveal_type(Real() + Fraction()) # N: Revealed type is "__main__.Real" +reveal_type(Fraction() + Fraction()) # N: Revealed type is "builtins.str" + + +[case testReverseOperatorTypeVar2b] +from typing import TypeVar +T = TypeVar("T", "Real", "Fraction") +class Real: + def __add__(self, other: Fraction) -> str: ... +class Fraction(Real): + def __radd__(self, other: T) -> T: ... # E: Signatures of "__radd__" of "Fraction" and "__add__" of "Real" are unsafely overlapping + +reveal_type(Real() + Fraction()) # N: Revealed type is "__main__.Real" +reveal_type(Fraction() + Fraction()) # N: Revealed type is "builtins.str" + +[case testReverseOperatorTypeVar3] +from typing import TypeVar +T = TypeVar("T", bound='Real') +class Real: + def __add__(self, other: FractionChild) -> str: ... +class Fraction(Real): + def __radd__(self, other: T) -> T: ... # E: Signatures of "__radd__" of "Fraction" and "__add__" of "T" are unsafely overlapping +class FractionChild(Fraction): pass + +reveal_type(Real() + Fraction()) # N: Revealed type is "__main__.Real" +reveal_type(FractionChild() + Fraction()) # N: Revealed type is "__main__.FractionChild" +reveal_type(FractionChild() + FractionChild()) # N: Revealed type is "builtins.str" + +# Runtime error: we try calling __add__, it doesn't match, and we don't try __radd__ since +# the LHS and the RHS are not the same. +Fraction() + Fraction() # E: Unsupported operand types for + ("Fraction" and "Fraction") + +[case testReverseOperatorTypeType] +from typing import TypeVar, Type +class Real(type): + def __add__(self, other: FractionChild) -> str: ... +class Fraction(Real): + def __radd__(self, other: Type['A']) -> Real: ... # E: Signatures of "__radd__" of "Fraction" and "__add__" of "Type[A]" are unsafely overlapping +class FractionChild(Fraction): pass + +class A(metaclass=Real): pass + +[case testOperatorDoubleUnionIntFloat] +from typing import Union + +a: Union[int, float] +b: int +c: float + +reveal_type(a + a) # N: Revealed type is "Union[builtins.int, builtins.float]" +reveal_type(a + b) # N: Revealed type is "Union[builtins.int, builtins.float]" +reveal_type(b + a) # N: Revealed type is "Union[builtins.int, builtins.float]" +reveal_type(a + c) # N: Revealed type is "builtins.float" +reveal_type(c + a) # N: Revealed type is "builtins.float" +[builtins fixtures/ops.pyi] + +[case testOperatorDoubleUnionStandardSubtyping] +from typing import Union + +class Parent: + def __add__(self, x: Parent) -> Parent: pass + def __radd__(self, x: Parent) -> Parent: pass + +class Child(Parent): + def __add__(self, x: Parent) -> Child: pass + def __radd__(self, x: Parent) -> Child: pass + +a: Union[Parent, Child] +b: Parent +c: Child + +reveal_type(a + a) # N: Revealed type is "__main__.Parent" +reveal_type(a + b) # N: Revealed type is "__main__.Parent" +reveal_type(b + a) # N: Revealed type is "__main__.Parent" +reveal_type(a + c) # N: Revealed type is "__main__.Child" +reveal_type(c + a) # N: Revealed type is "__main__.Child" + +[case testOperatorDoubleUnionNoRelationship1] +from typing import Union + +class Foo: + def __add__(self, x: Foo) -> Foo: pass + def __radd__(self, x: Foo) -> Foo: pass + +class Bar: + def __add__(self, x: Bar) -> Bar: pass + def __radd__(self, x: Bar) -> Bar: pass + +a: Union[Foo, Bar] +b: Foo +c: Bar + +a + a # E: Unsupported operand types for + ("Foo" and "Bar") \ + # E: Unsupported operand types for + ("Bar" and "Foo") \ + # N: Both left and right operands are unions + +a + b # E: Unsupported operand types for + ("Bar" and "Foo") \ + # N: Left operand is of type "Union[Foo, Bar]" + +b + a # E: Unsupported operand types for + ("Foo" and "Bar") \ + # N: Right operand is of type "Union[Foo, Bar]" + +a + c # E: Unsupported operand types for + ("Foo" and "Bar") \ + # N: Left operand is of type "Union[Foo, Bar]" + +c + a # E: Unsupported operand types for + ("Bar" and "Foo") \ + # N: Right operand is of type "Union[Foo, Bar]" + +[case testOperatorDoubleUnionNoRelationship2] +from typing import Union + +class Foo: + def __add__(self, x: Foo) -> Foo: pass + def __radd__(self, x: Foo) -> Foo: pass + +class Bar: + def __add__(self, x: Union[Foo, Bar]) -> Bar: pass + def __radd__(self, x: Union[Foo, Bar]) -> Bar: pass + +a: Union[Foo, Bar] +b: Foo +c: Bar + +reveal_type(a + a) # N: Revealed type is "Union[__main__.Foo, __main__.Bar]" +reveal_type(a + b) # N: Revealed type is "Union[__main__.Foo, __main__.Bar]" +reveal_type(b + a) # N: Revealed type is "Union[__main__.Foo, __main__.Bar]" +reveal_type(a + c) # N: Revealed type is "__main__.Bar" +reveal_type(c + a) # N: Revealed type is "__main__.Bar" + +[case testOperatorDoubleUnionNaiveAdd] +from typing import Union + +class A: pass +class B: pass +class C: + def __radd__(self, x: A) -> int: pass +class D: + def __radd__(self, x: B) -> str: pass + +x: Union[A, B] +y: Union[C, D] + +x + y # E: Unsupported operand types for + ("A" and "D") \ + # E: Unsupported operand types for + ("B" and "C") \ + # N: Both left and right operands are unions + +[case testOperatorDoubleUnionInterwovenUnionAdd] +from typing import Union + +class Out1: pass +class Out2: pass +class Out3: pass +class Out4: pass + +class A: + def __add__(self, x: D) -> Out1: pass +class B: + def __add__(self, x: C) -> Out2: pass +class C: + def __radd__(self, x: A) -> Out3: pass +class D: + def __radd__(self, x: B) -> Out4: pass + +x: Union[A, B] +y: Union[C, D] + +reveal_type(x + y) # N: Revealed type is "Union[__main__.Out3, __main__.Out1, __main__.Out2, __main__.Out4]" +reveal_type(A() + y) # N: Revealed type is "Union[__main__.Out3, __main__.Out1]" +reveal_type(B() + y) # N: Revealed type is "Union[__main__.Out2, __main__.Out4]" +reveal_type(x + C()) # N: Revealed type is "Union[__main__.Out3, __main__.Out2]" +reveal_type(x + D()) # N: Revealed type is "Union[__main__.Out1, __main__.Out4]" + +[case testOperatorDoubleUnionDivision] +from typing import Union +def f(a): + # type: (Union[int, float]) -> None + a /= 1.1 + b = a / 1.1 + reveal_type(b) # N: Revealed type is "builtins.float" +[builtins fixtures/ops.pyi] + +[case testOperatorWithInference] +from typing import TypeVar, Iterable, Union + +T = TypeVar('T') +def sum(x: Iterable[T]) -> Union[T, int]: ... + +def len(x: Iterable[T]) -> int: ... + +x = [1.1, 2.2, 3.3] +reveal_type(sum(x)) # N: Revealed type is "Union[builtins.float, builtins.int]" +reveal_type(sum(x) / len(x)) # N: Revealed type is "Union[builtins.float, builtins.int]" +[builtins fixtures/floatdict.pyi] + +[case testOperatorWithEmptyListAndSum] +from typing import TypeVar, Iterable, Union, overload + +T = TypeVar('T') +S = TypeVar('S') +@overload +def sum(x: Iterable[T]) -> Union[T, int]: ... +@overload +def sum(x: Iterable[T], default: S) -> Union[T, S]: ... +def sum(*args): pass + +x = ["a", "b", "c"] +reveal_type(x + sum([x, x, x], [])) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/floatdict.pyi] + +[case testAbstractReverseOperatorMethod] +import typing +from abc import abstractmethod +class A: + @abstractmethod + def __lt__(self, x: 'A') -> int: pass +class B: + @abstractmethod + def __lt__(self, x: 'B') -> int: pass + @abstractmethod + def __gt__(self, x: 'B') -> int: pass +[out] + +[case testOperatorMethodsAndOverloadingSpecialCase] +from foo import * +[file foo.pyi] +from typing import overload +class A: + @overload + def __add__(self, x: 'A') -> int: pass + @overload + def __add__(self, x: str) -> int: pass +class B: + def __radd__(self, x: 'A') -> str: pass +[out] + +[case testUnsafeOverlappingWithOperatorMethodsAndOverloading2] +from foo import A, B +from foo import * +[file foo.pyi] +from typing import overload +class A: + def __add__(self, x: 'A') -> int: pass +class B: + @overload + def __radd__(self, x: 'X') -> str: pass # Error + @overload + def __radd__(self, x: A) -> str: pass # Error +class X: + def __add__(self, x: B) -> int: pass +[out] +tmp/foo.pyi:6: error: Signatures of "__radd__" of "B" and "__add__" of "X" are unsafely overlapping + +[case testUnsafeOverlappingNotWithAny] +from typing import TypeVar +class Real: + def __add__(self, other) -> str: ... +class Fraction(Real): + def __radd__(self, other: Real) -> Real: ... + +[case testOverlappingNormalAndInplaceOperatorMethod] +import typing +class A: + # Incompatible (potential trouble with __radd__) + def __add__(self, x: 'A') -> int: pass + def __iadd__(self, x: 'B') -> int: pass +class B: + # Safe + def __add__(self, x: 'C') -> int: pass + def __iadd__(self, x: A) -> int: pass +class C(A): pass +[out] +main:5: error: Signatures of "__iadd__" and "__add__" are incompatible + +[case testOverloadedNormalAndInplaceOperatorMethod] +from foo import * +[file foo.pyi] +from typing import overload +class A: + @overload + def __add__(self, x: int) -> int: pass + @overload + def __add__(self, x: str) -> int: pass + @overload # Error + def __iadd__(self, x: int) -> int: pass + @overload + def __iadd__(self, x: object) -> int: pass +class B: + @overload + def __add__(self, x: int) -> int: pass + @overload + def __add__(self, x: str) -> str: pass + @overload + def __iadd__(self, x: int) -> int: pass + @overload + def __iadd__(self, x: str) -> str: pass +[out] +tmp/foo.pyi:7: error: Signatures of "__iadd__" and "__add__" are incompatible + +[case testIntroducingInplaceOperatorInSubclass] +import typing +class A: + def __add__(self, x: 'A') -> 'B': pass +class B(A): + # __iadd__ effectively partially overrides __add__ + def __iadd__(self, x: 'A') -> 'A': pass # Error +class C(A): + def __iadd__(self, x: int) -> 'B': pass # Error +class D(A): + def __iadd__(self, x: 'A') -> 'B': pass +[out] +main:6: error: Return type "A" of "__iadd__" incompatible with return type "B" in "__add__" of supertype "A" +main:8: error: Signatures of "__iadd__" and "__add__" are incompatible +main:8: error: Argument 1 of "__iadd__" is incompatible with "__add__" of supertype "A"; supertype defines the argument type as "A" +main:8: note: This violates the Liskov substitution principle +main:8: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + +[case testGetattribute] +a: A +b: B +class A: + def __getattribute__(self, x: str) -> A: + return A() +class B: pass + +a = a.foo +b = a.bar +[builtins fixtures/tuple.pyi] +[out] +main:9: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testDecoratedGetAttribute] +from typing import Callable, TypeVar + +T = TypeVar('T', bound=Callable) + +def decorator(f: T) -> T: + return f + +def bad(f: Callable) -> Callable[..., int]: + return f + +class A: + @decorator + def __getattribute__(self, x: str) -> A: + return A() +class B: + @bad # We test that type will be taken from decorated type, not node itself + def __getattribute__(self, x: str) -> A: + return A() + +a: A +b: B + +a1: A = a.foo +b1: B = a.bar # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a2: A = b.baz # E: Incompatible types in assignment (expression has type "int", variable has type "A") +b2: B = b.roo # E: Incompatible types in assignment (expression has type "int", variable has type "B") +[builtins fixtures/tuple.pyi] + +[case testGetattributeSignature] +class A: + def __getattribute__(self, x: str) -> A: pass +class B: + def __getattribute__(self, x: A) -> B: pass +class C: + def __getattribute__(self, x: str, y: str) -> C: pass +class D: + def __getattribute__(self, x: str) -> None: pass +[out] +main:4: error: Invalid signature "Callable[[B, A], B]" for "__getattribute__" +main:6: error: Invalid signature "Callable[[C, str, str], C]" for "__getattribute__" + +[case testGetattr] +a: A +b: B +class A: + def __getattr__(self, x: str) -> A: + return A() +class B: pass + +a = a.foo +b = a.bar +[builtins fixtures/tuple.pyi] +[out] +main:9: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testDecoratedGetattr] +from typing import Callable, TypeVar + +T = TypeVar('T', bound=Callable) + +def decorator(f: T) -> T: + return f + +def bad(f: Callable) -> Callable[..., int]: + return f + +class A: + @decorator + def __getattr__(self, x: str) -> A: + return A() +class B: + @bad # We test that type will be taken from decorated type, not node itself + def __getattr__(self, x: str) -> A: + return A() + +a: A +b: B + +a1: A = a.foo +b1: B = a.bar # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a2: A = b.baz # E: Incompatible types in assignment (expression has type "int", variable has type "A") +b2: B = b.roo # E: Incompatible types in assignment (expression has type "int", variable has type "B") +[builtins fixtures/tuple.pyi] + +[case testGetattrWithGetitem] +class A: + def __getattr__(self, x: str) -> 'A': + return A() + +a = A() +a[0] # E: Value of type "A" is not indexable + +[case testGetattrWithCall] +class A: + def __getattr__(self, x: str) -> 'A': + return A() + +a = A() +a.y() # E: "A" not callable + +[case testGetattrWithCallable] +from typing import Callable, Any + +class C: + def __getattr__(self, attr: str) -> C: ... + +def do(cd: Callable[..., Any]) -> None: ... + +do(C()) # E: Argument 1 to "do" has incompatible type "C"; expected "Callable[..., Any]" + +[case testGetattrWithCallableTypeVar] +from typing import Callable, Any, TypeVar + +class C: + def __getattr__(self, attr: str) -> C: ... + +T = TypeVar('T', bound=Callable[..., Any]) + +def do(cd: T) -> T: ... + +do(C()) # E: Value of type variable "T" of "do" cannot be "C" + +[case testNestedGetattr] +def foo() -> object: + def __getattr__() -> None: # no error because not in a class + pass + return __getattr__ + +class X: + def foo(self) -> object: + def __getattr__() -> None: # no error because not directly inside a class + pass + return __getattr__ + +[case testGetattrSignature] +class A: + def __getattr__(self, x: str) -> A: pass +class B: + def __getattr__(self, x: A) -> B: pass +class C: + def __getattr__(self, x: str, y: str) -> C: pass +class D: + def __getattr__(self, x: str) -> None: pass +[out] +main:4: error: Invalid signature "Callable[[B, A], B]" for "__getattr__" +main:6: error: Invalid signature "Callable[[C, str, str], C]" for "__getattr__" + +[case testSetattr] +from typing import Union, Any +class A: + def __setattr__(self, name: str, value: Any) -> None: ... + +a = A() +a.test = 'hello' + +class B: + def __setattr__(self, name: str, value: Union[int, str]) -> None: ... + +b = B() +b.both = 1 +b.work = '2' + +class C: + def __setattr__(self, name: str, value: str) -> None: ... + +c = C() +c.fail = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +class D: + __setattr__ = 'hello' # E: Invalid signature "str" for "__setattr__" + +d = D() +d.crash = 4 # E: "D" has no attribute "crash" + +class Ex: + def __setattr__(self, name: str, value: int) -> None:... + test = '42' # type: str +e = Ex() +e.test = 'hello' +e.t = 4 + +class Super: + def __setattr__(self, name: str, value: int) -> None: ... + +class Sub(Super): + ... +s = Sub() +s.success = 4 +s.fail = 'fail' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testDecoratedSetattr] +from typing import Any, Callable, TypeVar + +T = TypeVar('T', bound=Callable) + +def decorator(f: T) -> T: + return f + +def bad(f: Callable) -> Callable[[Any, str, int], None]: + return f + +class A: + @decorator + def __setattr__(self, k: str, v: str) -> None: + pass +class B: + @bad # We test that type will be taken from decorated type, not node itself + def __setattr__(self, k: str, v: str) -> None: + pass + +a: A +a.foo = 'a' +a.bar = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +b: B +b.good = 1 +b.bad = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] + +[case testSetattrSignature] +from typing import Any + +class Test: + def __setattr__() -> None: ... # E: Method must have at least one argument. Did you forget the "self" argument? # E: Invalid signature "Callable[[], None]" for "__setattr__" +t = Test() +t.crash = 'test' # E: "Test" has no attribute "crash" + +class A: + def __setattr__(self): ... # E: Invalid signature "Callable[[A], Any]" for "__setattr__" +a = A() +a.test = 4 # E: "A" has no attribute "test" + +class B: + def __setattr__(self, name, value: int): ... +b = B() +b.integer = 5 + +class C: + def __setattr__(self, name: int, value: int) -> None: ... # E: Invalid signature "Callable[[C, int, int], None]" for "__setattr__" +c = C() +c.check = 13 + +class X: + __setattr__ = ... # type: Any + +[case testGetattrAndSetattr] +from typing import Any +class A: + def __setattr__(self, name: str, value: Any) -> None: ... + def __getattr__(self, name: str) -> Any: ... +a = A() +a.test = 4 +t = a.test + +class B: + def __setattr__(self, name: str, value: int) -> None: ... + def __getattr__(self, name: str) -> str: ... +integer = 0 +b = B() +b.at = '3' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + integer = b.at # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testSetattrKeywordArg] +from typing import Any + +class C: + def __setattr__(self, key: str, value: Any, p: bool = False) -> None: ... + +c: C +c.__setattr__("x", 42, p=True) + +-- CallableType objects +-- ---------------- + + +[case testCallableObject] +class A: + def __call__(self, x: 'A') -> 'A': + pass +class B: pass + +a = A() +b = B() + +a() # E: Missing positional argument "x" in call to "__call__" of "A" +a(a, a) # E: Too many arguments for "__call__" of "A" +if int(): + a = a(a) +if int(): + a = a(b) # E: Argument 1 to "__call__" of "A" has incompatible type "B"; expected "A" +if int(): + b = a(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") + + +-- __new__ +-- -------- + + +[case testConstructInstanceWith__new__] +from typing import Optional +class C: + def __new__(cls, foo: Optional[int] = None) -> 'C': + obj = object.__new__(cls) + return obj + +x = C(foo=12) +x.a # E: "C" has no attribute "a" +C(foo='') # E: Argument "foo" to "C" has incompatible type "str"; expected "Optional[int]" +[builtins fixtures/__new__.pyi] + +[case testConstructInstanceWithDynamicallyTyped__new__] +class C: + def __new__(cls, foo): # N: "C" defined here + obj = object.__new__(cls) + return obj + +x = C(foo=12) +x = C(foo='x') +x.a # E: "C" has no attribute "a" +C(bar='') # E: Unexpected keyword argument "bar" for "C" +[builtins fixtures/__new__.pyi] + +[case testClassWith__new__AndCompatibilityWithType] +from typing import Optional +class C: + def __new__(cls, foo: Optional[int] = None) -> 'C': + obj = object.__new__(cls) + return obj +def f(x: type) -> None: pass +def g(x: int) -> None: pass +f(C) +g(C) # E: Argument 1 to "g" has incompatible type "Type[C]"; expected "int" +[builtins fixtures/__new__.pyi] + +[case testClassWith__new__AndCompatibilityWithType2] +class C: + def __new__(cls, foo): + obj = object.__new__(cls) + return obj +def f(x: type) -> None: pass +def g(x: int) -> None: pass +f(C) +g(C) # E: Argument 1 to "g" has incompatible type "Type[C]"; expected "int" +[builtins fixtures/__new__.pyi] + +[case testGenericClassWith__new__] +from typing import TypeVar, Generic +T = TypeVar('T') +class C(Generic[T]): + def __new__(cls, foo: T) -> 'C[T]': + obj = object.__new__(cls) + return obj + def set(self, x: T) -> None: pass +c = C('') +c.set('') +c.set(1) # E: Argument 1 to "set" of "C" has incompatible type "int"; expected "str" +[builtins fixtures/__new__.pyi] + +[case testOverloaded__new__] +from foo import * +[file foo.pyi] +from typing import overload +class C: + @overload + def __new__(cls, foo: int) -> 'C': + obj = object.__new__(cls) + return obj + @overload + def __new__(cls, x: str, y: str) -> 'C': + obj = object.__new__(cls) + return obj +c = C(1) +c.a # E: "C" has no attribute "a" +C('', '') +C('') # E: No overload variant of "C" matches argument type "str" \ + # N: Possible overload variants: \ + # N: def __new__(cls, foo: int) -> C \ + # N: def __new__(cls, x: str, y: str) -> C +[builtins fixtures/__new__.pyi] + + +-- Special cases +-- ------------- + + +[case testSubclassInt] +import typing +class A(int): pass +n = 0 +if int(): + n = A() +a = A() +if int(): + a = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "A") + +[case testForwardReferenceToNestedClass] +def f(o: 'B.C') -> None: + o.f('') # E: Argument 1 to "f" of "C" has incompatible type "str"; expected "int" + +class B: + class C: + def f(self, x: int) -> None: pass +[out] + +[case testForwardReferenceToNestedClassDeep] +def f(o: 'B.C.D') -> None: + o.f('') # E: Argument 1 to "f" of "D" has incompatible type "str"; expected "int" + +class B: + class C: + class D: + def f(self, x: int) -> None: pass +[out] + +[case testForwardReferenceToNestedClassWithinClass] +class B: + def f(self, o: 'C.D') -> None: + o.f('') # E: Argument 1 to "f" of "D" has incompatible type "str"; expected "int" + + class C: + class D: + def f(self, x: int) -> None: pass +[out] + +[case testClassVsInstanceDisambiguation] +class A: pass +def f(x: A) -> None: pass +f(A) # E: Argument 1 to "f" has incompatible type "Type[A]"; expected "A" +[out] + +-- TODO +-- attribute inherited from superclass; assign in __init__ +-- refer to attribute before type has been inferred (the initialization in +-- __init__ has not been analyzed) + +[case testAnyBaseClassUnconstrainedConstructor] +from typing import Any +B = None # type: Any +class C(B): pass +C(0) +C(arg=0) +[out] + +[case testErrorMapToSupertype] +import typing +class X(Nope): pass # E: Name "Nope" is not defined +a, b = X() # Used to crash here (#2244) + + +-- Class-valued attributes +-- ----------------------- + +[case testClassValuedAttributesBasics] +class A: ... +class B: + a = A + bad = lambda: 42 + +B().bad() # E: Attribute function "bad" with type "Callable[[], int]" does not accept self argument +reveal_type(B.a) # N: Revealed type is "def () -> __main__.A" +reveal_type(B().a) # N: Revealed type is "def () -> __main__.A" +reveal_type(B().a()) # N: Revealed type is "__main__.A" + +class C: + a = A + def __init__(self) -> None: + self.aa = self.a() + +reveal_type(C().aa) # N: Revealed type is "__main__.A" +[out] + +[case testClassValuedAttributesGeneric] +from typing import Generic, TypeVar, Type +T = TypeVar('T') + +class A(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x +class B(Generic[T]): + a: Type[A[T]] = A + +reveal_type(B[int]().a) # N: Revealed type is "Type[__main__.A[builtins.int]]" +B[int]().a('hi') # E: Argument 1 to "A" has incompatible type "str"; expected "int" + +class C(Generic[T]): + a = A + def __init__(self) -> None: + self.aa = self.a(42) + +reveal_type(C().aa) # N: Revealed type is "__main__.A[builtins.int]" +[out] + +[case testClassValuedAttributesAlias] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') + +class A(Generic[T, S]): ... + +SameA = A[T, T] + +class B: + a_any = SameA + a_int = SameA[int] + +reveal_type(B().a_any) # N: Revealed type is "def () -> __main__.A[Any, Any]" +reveal_type(B().a_int()) # N: Revealed type is "__main__.A[builtins.int, builtins.int]" + +class C: + a_int = SameA[int] + def __init__(self) -> None: + self.aa = self.a_int() + +reveal_type(C().aa) # N: Revealed type is "__main__.A[builtins.int, builtins.int]" +[out] + + +-- Type[C] +-- ------- + + +[case testTypeUsingTypeCBasic] +from typing import Type +class User: pass +class ProUser(User): pass +def new_user(user_class: Type[User]) -> User: + return user_class() +reveal_type(new_user(User)) # N: Revealed type is "__main__.User" +reveal_type(new_user(ProUser)) # N: Revealed type is "__main__.User" +[out] + +[case testTypeUsingTypeCDefaultInit] +from typing import Type +class B: + pass +def f(A: Type[B]) -> None: + A(0) # E: Too many arguments for "B" + A() +[out] + +[case testTypeUsingTypeCInitWithArg] +from typing import Type +class B: + def __init__(self, a: int) -> None: pass +def f(A: Type[B]) -> None: + A(0) + A() # E: Missing positional argument "a" in call to "B" +[out] + +[case testTypeUsingTypeCTypeVar] +from typing import Type, TypeVar +class User: pass +class ProUser(User): pass +U = TypeVar('U', bound=User) +def new_user(user_class: Type[U]) -> U: + user = user_class() + reveal_type(user) + return user +pro_user = new_user(ProUser) +reveal_type(pro_user) +[out] +main:7: note: Revealed type is "U`-1" +main:10: note: Revealed type is "__main__.ProUser" + +[case testTypeUsingTypeCTypeVarDefaultInit] +from typing import Type, TypeVar +class B: + pass +T = TypeVar('T', bound=B) +def f(A: Type[T]) -> None: + A() + A(0) # E: Too many arguments for "B" +[out] + +[case testTypeUsingTypeCTypeVarWithInit] +from typing import Type, TypeVar +class B: + def __init__(self, a: int) -> None: pass +T = TypeVar('T', bound=B) +def f(A: Type[T]) -> None: + A() # E: Missing positional argument "a" in call to "B" + A(0) +[out] + +[case testTypeUsingTypeCTwoTypeVars] +from typing import Type, TypeVar +class User: pass +class ProUser(User): pass +class WizUser(ProUser): pass +U = TypeVar('U', bound=User) +def new_user(u_c: Type[U]) -> U: pass +P = TypeVar('P', bound=ProUser) +def new_pro(pro_c: Type[P]) -> P: + return new_user(pro_c) +wiz = new_pro(WizUser) +reveal_type(wiz) +def error(u_c: Type[U]) -> P: # Error here, see below + return new_pro(u_c) # Error here, see below +[out] +main:11: note: Revealed type is "__main__.WizUser" +main:12: error: A function returning TypeVar should receive at least one argument containing the same TypeVar +main:12: note: Consider using the upper bound "ProUser" instead +main:13: error: Value of type variable "P" of "new_pro" cannot be "U" +main:13: error: Incompatible return value type (got "U", expected "P") + +[case testTypeUsingTypeCCovariance] +from typing import Type, TypeVar +class User: pass +class ProUser(User): pass +def new_user(user_class: Type[User]) -> User: + return user_class() +def new_pro_user(user_class: Type[ProUser]): + new_user(user_class) +[out] + +[case testAllowCovariantArgsInConstructor] +from typing import Generic, TypeVar + +T_co = TypeVar('T_co', covariant=True) + +class C(Generic[T_co]): + def __init__(self, x: T_co) -> None: # This should be allowed + self.x = x + def meth(self) -> None: + reveal_type(self.x) # N: Revealed type is "T_co`1" + +reveal_type(C(1).x) # N: Revealed type is "builtins.int" +[builtins fixtures/property.pyi] +[out] + +[case testTypeUsingTypeCErrorCovariance] +from typing import Type, TypeVar +class User: pass +def new_user(user_class: Type[User]): + return user_class() +def foo(arg: Type[int]): + new_user(arg) # E: Argument 1 to "new_user" has incompatible type "Type[int]"; expected "Type[User]" +[out] + +[case testTypeUsingTypeCUnionOverload] +from foo import * +[file foo.pyi] +from typing import Type, Union, overload +class X: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, a: int) -> None: pass +class Y: + def __init__(self) -> None: pass +def bar(o: Type[Union[X, Y]]): pass +bar(X) +bar(Y) +[out] + +[case testTypeUsingTypeCTypeAny] +from typing import Type, Any +def foo(arg: Type[Any]): + x = arg() + x = arg(0) + x = arg('', ()) + reveal_type(x) # N: Revealed type is "Any" + x.foo +class X: pass +foo(X) +[builtins fixtures/tuple.pyi] +[out] + +[case testTypeUsingTypeCTypeAnyMember] +from typing import Type, Any +def foo(arg: Type[Any]): + x = arg.member_name + arg.new_member_name = 42 + # Member access is ok and types as Any + reveal_type(x) # N: Revealed type is "Any" + # But Type[Any] is distinct from Any + y: int = arg # E: Incompatible types in assignment (expression has type "Type[Any]", variable has type "int") +[out] + +[case testTypeUsingTypeCTypeAnyMemberFallback] +from typing import Type, Any +def foo(arg: Type[Any]): + reveal_type(arg.__str__) # N: Revealed type is "def () -> builtins.str" + reveal_type(arg.mro()) # N: Revealed type is "builtins.list[builtins.type]" +[builtins fixtures/type.pyi] +[out] + +[case testTypeUsingTypeCTypeNoArg] +from typing import Type +def foo(arg: Type): + x = arg() + reveal_type(x) # N: Revealed type is "Any" +class X: pass +foo(X) +[out] + +[case testTypeUsingTypeCBuiltinType] +from typing import Type +def foo(arg: type): pass +class X: pass +def bar(arg: Type[X]): + foo(arg) +foo(X) +[builtins fixtures/tuple.pyi] +[out] + +[case testTypeUsingTypeCClassMethod] +from typing import Type +class User: + @classmethod + def foo(cls) -> int: pass + def bar(self) -> int: pass +def process(cls: Type[User]): + reveal_type(cls.foo()) # N: Revealed type is "builtins.int" + obj = cls() + reveal_type(cls.bar(obj)) # N: Revealed type is "builtins.int" + cls.mro() # Defined in class type + cls.error # E: "Type[User]" has no attribute "error" +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeUsingTypeCClassMethodUnion] +from typing import Type, Union +class User: + @classmethod + def foo(cls) -> int: pass + def bar(self) -> int: pass +class ProUser(User): pass +class BasicUser(User): pass +def process(cls: Type[Union[BasicUser, ProUser]]): + cls.foo() + obj = cls() + cls.bar(obj) + cls.mro() # Defined in class type + cls.error # E: Item "type" of "Union[Type[BasicUser], Type[ProUser]]" has no attribute "error" +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeUsingTypeCClassMethodFromTypeVar] +from typing import Type, TypeVar +class User: + @classmethod + def foo(cls) -> int: pass + def bar(self) -> int: pass +U = TypeVar('U', bound=User) +def process(cls: Type[U]): + reveal_type(cls.foo()) # N: Revealed type is "builtins.int" + obj = cls() + reveal_type(cls.bar(obj)) # N: Revealed type is "builtins.int" + cls.mro() # Defined in class type + cls.error # E: "Type[U]" has no attribute "error" +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeUsingTypeCClassMethodFromTypeVarUnionBound] +# Ideally this would work, but not worth the effort; just don't crash +from typing import Type, TypeVar, Union +class User: + @classmethod + def foo(cls) -> int: pass + def bar(self) -> int: pass +class ProUser(User): pass +class BasicUser(User): pass +U = TypeVar('U', bound=Union[ProUser, BasicUser]) +def process(cls: Type[U]): + cls.foo() + obj = cls() + cls.bar(obj) + cls.mro() # Defined in class type + cls.error # E: "Type[U]" has no attribute "error" +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeUsingTypeCErrorUnsupportedType] +from typing import Type, Tuple +def foo(arg: Type[Tuple[int]]): + arg() # E: Cannot instantiate type "Type[Tuple[int]]" +[builtins fixtures/tuple.pyi] + +[case testTypeUsingTypeCOverloadedClass] +from foo import * +[file foo.pyi] +from typing import Type, TypeVar, overload +class User: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, arg: int) -> None: pass + @classmethod + def foo(cls) -> None: pass +U = TypeVar('U', bound=User) +def new(uc: Type[U]) -> U: + uc.foo() + u = uc() + u.foo() + if 1: + u = uc(0) + u.foo() + u = uc('') # Error + u.foo(0) # Error + return uc() +u = new(User) +[builtins fixtures/classmethod.pyi] +[out] +tmp/foo.pyi:17: error: No overload variant of "User" matches argument type "str" +tmp/foo.pyi:17: note: Possible overload variants: +tmp/foo.pyi:17: note: def __init__(self) -> U +tmp/foo.pyi:17: note: def __init__(self, arg: int) -> U +tmp/foo.pyi:18: error: Too many arguments for "foo" of "User" + +[case testTypeUsingTypeCInUpperBound] +from typing import TypeVar, Type +class B: pass +T = TypeVar('T', bound=Type[B]) +def f(a: T): pass +[out] + +[case testTypeUsingTypeCTuple] +from typing import Type, Tuple +def f(a: Type[Tuple[int, int]]): + a() # E: Cannot instantiate type "Type[Tuple[int, int]]" +[builtins fixtures/tuple.pyi] + +[case testTypeUsingTypeCNamedTuple] +from typing import Type, NamedTuple +N = NamedTuple('N', [('x', int), ('y', int)]) +def f(a: Type[N]): + a() +[builtins fixtures/list.pyi] +[out] +main:4: error: Missing positional arguments "x", "y" in call to "N" + +[case testTypeUsingTypeCJoin] +from typing import Type +class B: pass +class C(B): pass +class D(B): pass +def foo(c: Type[C], d: Type[D]) -> None: + x = [c, d] + reveal_type(x) + +[builtins fixtures/list.pyi] +[out] +main:7: note: Revealed type is "builtins.list[Type[__main__.B]]" + +[case testTypeEquivalentTypeAny] +from typing import Type, Any + +a: Type[Any] +b = a # type: type + +x: type +y = x # type: Type[Any] + +class C: ... + +p: type +q = p # type: Type[C] + +[builtins fixtures/list.pyi] +[out] + +[case testTypeEquivalentTypeAny2] +from typing import Type, Any, TypeVar, Generic + +class C: ... +x: type +y: Type[Any] +z: Type[C] + +lst = [x, y, z] +reveal_type(lst) # N: Revealed type is "builtins.list[builtins.type]" + +T1 = TypeVar('T1', bound=type) +T2 = TypeVar('T2', bound=Type[Any]) +class C1(Generic[T1]): ... +class C2(Generic[T2]): ... + +C1[Type[Any]], C2[type] # both these should not fail +[builtins fixtures/list.pyi] +[out] + +[case testTypeEquivalentTypeAnyEdgeCase] +class C: + pass + +class M(type): + def __init__(cls, x) -> None: + type.__init__(cls, x) + +class Mbad(type): + def __init__(cls, x) -> None: + type.__init__(C(), x) # E: Argument 1 to "__init__" of "type" has incompatible type "C"; expected "type" +[builtins fixtures/primitives.pyi] +[out] + +[case testTypeMatchesOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, overload, Any + +class User: pass +UserType = User # type: Type[User] + +@overload +def f(a: int) -> Any: pass +@overload +def f(a: object) -> int: pass + +reveal_type(f(User)) # N: Revealed type is "builtins.int" +reveal_type(f(UserType)) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeMatchesGeneralTypeInOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class User: pass +UserType = User # type: Type[User] + +@overload +def f(a: type) -> int: + return 1 +@overload +def f(a: int) -> str: + return "a" + +reveal_type(f(User)) # N: Revealed type is "builtins.int" +reveal_type(f(UserType)) # N: Revealed type is "builtins.int" +reveal_type(f(1)) # N: Revealed type is "builtins.str" +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeMatchesSpecificTypeInOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class User: pass +UserType = User # type: Type[User] + +@overload +def f(a: User) -> User: + return User() +@overload +def f(a: Type[User]) -> int: + return 1 +@overload +def f(a: int) -> str: + return "a" + +reveal_type(f(User)) # N: Revealed type is "builtins.int" +reveal_type(f(UserType)) # N: Revealed type is "builtins.int" +reveal_type(f(User())) # N: Revealed type is "foo.User" +reveal_type(f(1)) # N: Revealed type is "builtins.str" +[builtins fixtures/classmethod.pyi] +[out] + +[case testMixingTypeTypeInOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class User: pass + +@overload +def f(a: User) -> Type[User]: + return User +@overload +def f(a: Type[User]) -> User: + return a() +@overload +def f(a: int) -> Type[User]: + return User +@overload +def f(a: str) -> User: + return User() + +reveal_type(f(User())) # N: Revealed type is "Type[foo.User]" +reveal_type(f(User)) # N: Revealed type is "foo.User" +reveal_type(f(3)) # N: Revealed type is "Type[foo.User]" +reveal_type(f("hi")) # N: Revealed type is "foo.User" +[builtins fixtures/classmethod.pyi] +[out] + +[case testGeneralTypeMatchesSpecificTypeInOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, Any, overload + +class User: pass + +@overload +def f(a: Type[User]) -> None: pass +@overload +def f(a: int) -> None: pass + +def mock_1() -> type: return User +def mock_2() -> Type[Any]: return User + +f(User) +f(mock_1()) +f(mock_2()) +[builtins fixtures/classmethod.pyi] +[out] + +[case testNonTypeDoesNotMatchOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class User: pass + +@overload +def f(a: Type[User]) -> None: pass +@overload +def f(a: type) -> None: pass + +f(3) # E: No overload variant of "f" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def f(a: Type[User]) -> None \ + # N: def f(a: type) -> None +[builtins fixtures/classmethod.pyi] +[out] + +[case testInstancesDoNotMatchTypeInOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class User: pass + +@overload +def f(a: Type[User]) -> None: pass +@overload +def f(a: int) -> None: pass + +f(User) +f(User()) # E: No overload variant of "f" matches argument type "User" \ + # N: Possible overload variants: \ + # N: def f(a: Type[User]) -> None \ + # N: def f(a: int) -> None +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeCovarianceWithOverloadedFunctions] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class A: pass +class B(A): pass +class C(B): pass +AType = A # type: Type[A] +BType = B # type: Type[B] +CType = C # type: Type[C] + +@overload +def f(a: Type[B]) -> None: pass +@overload +def f(a: int) -> None: pass + +f(A) # E: Argument 1 to "f" has incompatible type "Type[A]"; expected "Type[B]" +f(B) +f(C) +f(AType) # E: Argument 1 to "f" has incompatible type "Type[A]"; expected "Type[B]" +f(BType) +f(CType) +[builtins fixtures/classmethod.pyi] +[out] + + +[case testOverloadedCovariantTypesFail] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class A: pass +class B(A): pass + +@overload +def f(a: Type[B]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(a: Type[A]) -> str: pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testDistinctOverloadedCovariantTypesSucceed] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class A: pass +class AChild(A): pass +class B: pass +class BChild(B): pass + +@overload +def f(a: Type[A]) -> int: pass +@overload +def f(a: Type[B]) -> str: pass +@overload +def f(a: A) -> A: pass +@overload +def f(a: B) -> B: pass + +reveal_type(f(A)) # N: Revealed type is "builtins.int" +reveal_type(f(AChild)) # N: Revealed type is "builtins.int" +reveal_type(f(B)) # N: Revealed type is "builtins.str" +reveal_type(f(BChild)) # N: Revealed type is "builtins.str" + +reveal_type(f(A())) # N: Revealed type is "foo.A" +reveal_type(f(AChild())) # N: Revealed type is "foo.A" +reveal_type(f(B())) # N: Revealed type is "foo.B" +reveal_type(f(BChild())) # N: Revealed type is "foo.B" +[builtins fixtures/classmethod.pyi] +[out] + +[case testSubtypeWithMoreOverloadsThanSupertypeSucceeds] +from foo import * +[file foo.pyi] +from typing import overload + + +class X: pass +class Y: pass +class Z: pass + + +class A: + @overload + def f(self, x: X) -> X: pass + @overload + def f(self, y: Y) -> Y: pass + +class B(A): + @overload + def f(self, x: X) -> X: pass + @overload + def f(self, y: Y) -> Y: pass + @overload + def f(self, z: Z) -> Z: pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testSubtypeOverloadCoveringMultipleSupertypeOverloadsSucceeds] +from foo import * +[file foo.pyi] +from typing import overload + + +class A: pass +class B(A): pass +class C(A): pass +class D: pass + + +class Super: + @overload + def foo(self, a: B) -> C: pass + @overload + def foo(self, a: C) -> A: pass + @overload + def foo(self, a: D) -> D: pass + +class Sub(Super): + @overload + def foo(self, a: A) -> C: pass + @overload + def foo(self, a: D) -> D: pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testSubtypeOverloadWithOverlappingArgumentsButWrongReturnType] +from foo import * +[file foo.pyi] +from typing import overload + + +class A: pass +class B(A): pass +class C: pass + + +class Super: + @overload + def foo(self, a: A) -> A: pass + @overload + def foo(self, a: C) -> C: pass + +class Sub(Super): + @overload + def foo(self, a: A) -> A: pass + @overload + def foo(self, a: B) -> C: pass # Fail + @overload + def foo(self, a: C) -> C: pass + +class Sub2(Super): + @overload + def foo(self, a: B) -> C: pass # Fail + @overload + def foo(self, a: A) -> A: pass + @overload + def foo(self, a: C) -> C: pass + +class Sub3(Super): + @overload + def foo(self, a: A) -> int: pass + @overload + def foo(self, a: A) -> A: pass + @overload + def foo(self, a: C) -> C: pass +[builtins fixtures/classmethod.pyi] +[out] +tmp/foo.pyi:19: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +tmp/foo.pyi:24: error: Signature of "foo" incompatible with supertype "Super" +tmp/foo.pyi:24: note: Superclass: +tmp/foo.pyi:24: note: @overload +tmp/foo.pyi:24: note: def foo(self, a: A) -> A +tmp/foo.pyi:24: note: @overload +tmp/foo.pyi:24: note: def foo(self, a: C) -> C +tmp/foo.pyi:24: note: Subclass: +tmp/foo.pyi:24: note: @overload +tmp/foo.pyi:24: note: def foo(self, a: B) -> C +tmp/foo.pyi:24: note: @overload +tmp/foo.pyi:24: note: def foo(self, a: A) -> A +tmp/foo.pyi:24: note: @overload +tmp/foo.pyi:24: note: def foo(self, a: C) -> C +tmp/foo.pyi:25: error: Overloaded function signatures 1 and 2 overlap with incompatible return types +tmp/foo.pyi:32: error: Signature of "foo" incompatible with supertype "Super" +tmp/foo.pyi:32: note: Superclass: +tmp/foo.pyi:32: note: @overload +tmp/foo.pyi:32: note: def foo(self, a: A) -> A +tmp/foo.pyi:32: note: @overload +tmp/foo.pyi:32: note: def foo(self, a: C) -> C +tmp/foo.pyi:32: note: Subclass: +tmp/foo.pyi:32: note: @overload +tmp/foo.pyi:32: note: def foo(self, a: A) -> int +tmp/foo.pyi:32: note: @overload +tmp/foo.pyi:32: note: def foo(self, a: A) -> A +tmp/foo.pyi:32: note: @overload +tmp/foo.pyi:32: note: def foo(self, a: C) -> C +tmp/foo.pyi:35: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +[case testTypeTypeOverlapsWithObjectAndType] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class User: pass + +@overload +def f(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(a: object) -> str: pass + +# Note: plain type is equivalent to Type[Any] so no error here +@overload +def g(a: Type[User]) -> int: pass +@overload +def g(a: type) -> str: pass + +@overload +def h(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def h(a: Type[object]) -> str: pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeOverlapsWithObject] +from foo import * +[file foo.pyi] +from typing import Type, overload + +class User: pass + +@overload +def f(a: type) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(a: object) -> str: pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeConstructorReturnsTypeType] +class User: + @classmethod + def test_class_method(cls) -> int: pass + @staticmethod + def test_static_method() -> str: pass + def test_instance_method(self) -> None: pass + +u = User() + +reveal_type(type(u)) # N: Revealed type is "Type[__main__.User]" +reveal_type(type(u).test_class_method()) # N: Revealed type is "builtins.int" +reveal_type(type(u).test_static_method()) # N: Revealed type is "builtins.str" +type(u).test_instance_method() # E: Missing positional argument "self" in call to "test_instance_method" of "User" +[builtins fixtures/classmethod.pyi] +[out] + +[case testObfuscatedTypeConstructorReturnsTypeType] +from typing import TypeVar +class User: pass + +f1 = type + +A = TypeVar('A') +def f2(func: A) -> A: + return func + +u = User() + +reveal_type(f1(u)) # N: Revealed type is "Type[__main__.User]" +reveal_type(f2(type)(u)) # N: Revealed type is "Type[__main__.User]" +[builtins fixtures/classmethod.pyi] +[out] + +[case testTypeConstructorLookalikeFails] +class User: pass + +def fake1(a: object) -> type: + return User +def fake2(a: int) -> type: + return User + +reveal_type(type(User())) # N: Revealed type is "Type[__main__.User]" +reveal_type(fake1(User())) # N: Revealed type is "builtins.type" +reveal_type(fake2(3)) # N: Revealed type is "builtins.type" +[builtins fixtures/classmethod.pyi] +[out] + +[case testOtherTypeConstructorsSucceed] +def foo(self) -> int: return self.attr + +User = type('User', (object,), {'foo': foo, 'attr': 3}) +reveal_type(User) # N: Revealed type is "builtins.type" +[builtins fixtures/args.pyi] +[out] + +[case testTypeTypeComparisonWorks] +class User: pass + +User == User +User == type(User()) +type(User()) == User +type(User()) == type(User()) + +User != User +User != type(User()) +type(User()) != User +type(User()) != type(User()) + +int == int +int == type(3) +type(3) == int +type(3) == type(3) + +int != int +int != type(3) +type(3) != int +type(3) != type(3) + +User is User +User is type(User) +type(User) is User +type(User) is type(User) + +int is int +int is type(3) +type(3) is int +type(3) is type(3) + +int.__eq__(int) +int.__eq__(3, 4) +[builtins fixtures/args.pyi] +[out] +main:33: error: Too few arguments for "__eq__" of "int" +main:33: error: Unsupported operand types for == ("int" and "Type[int]") + +[case testDupBaseClasses] +class A: + def method(self) -> str: ... + +class B(A, A): # E: Duplicate base class "A" + attr: int + +b: B + +reveal_type(b.method()) # N: Revealed type is "Any" +reveal_type(b.missing()) # N: Revealed type is "Any" +reveal_type(b.attr) # N: Revealed type is "builtins.int" + +[case testDupBaseClassesGeneric] +from typing import Generic, TypeVar + +T = TypeVar('T') +class A(Generic[T]): + def method(self) -> T: ... + +class B(A[int], A[str]): # E: Duplicate base class "A" + attr: int + +reveal_type(B().method()) # N: Revealed type is "Any" +reveal_type(B().attr) # N: Revealed type is "builtins.int" + +[case testCannotDetermineMro] +class A: pass +class B(A): pass +class C(B): pass +class D(A, B): pass # E: Cannot determine consistent method resolution order (MRO) for "D" +class E(C, D): pass + +[case testInconsistentMroLocalRef] +class A: pass +class B(object, A): # E: Cannot determine consistent method resolution order (MRO) for "B" + def readlines(self): pass + __iter__ = readlines + +[case testDynamicMetaclass] +class C(metaclass=int()): # E: Dynamic metaclass not supported for "C" + pass + +[case testDynamicMetaclassCrash] +class C(metaclass=int().x): # E: Dynamic metaclass not supported for "C" + pass + +[case testVariableSubclass] +class A: + a = 1 # type: int +class B(A): + a = 1 +[out] + +[case testVariableSubclassAssignMismatch] +class A: + a = 1 # type: int +class B(A): + a = "a" +[out] +main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") + +[case testVariableSubclassAssignment] +class A: + a = None # type: int +class B(A): + def __init__(self) -> None: + self.a = "a" +[out] +main:5: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testVariableSubclassTypeOverwrite] +class A: + a = None # type: int +class B(A): + a = None # type: str +class C(B): + a = "a" +[out] +main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") + +[case testVariableSubclassTypeOverwriteImplicit] +class A: + a = 1 +class B(A): + a = None # type: str +[out] +main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") + +[case testVariableSuperUsage] +class A: + a = [] # type: list +class B(A): + a = [1, 2] +class C(B): + a = B.a + [3] +[builtins fixtures/list.pyi] +[out] + +[case testClassAllBases] +from typing import Union +class A: + a = None # type: Union[int, str] +class B(A): + a = 1 +class C(B): + a = "str" +class D(A): + a = "str" +[out] +main:7: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testVariableTypeVar] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + a = None # type: T +class B(A[int]): + a = 1 + +[case testVariableTypeVarInvalid] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + a = None # type: T +class B(A[int]): + a = "abc" +[out] +main:6: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") + +[case testVariableTypeVarIndirectly] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + a = None # type: T +class B(A[int]): + pass +class C(B): + a = "a" +[out] +main:8: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") + +[case testVariableTypeVarList] +from typing import List, TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + a = None # type: List[T] + b = None # type: List[T] +class B(A[int]): + a = [1] + b = [''] +[builtins fixtures/list.pyi] +[out] +main:8: error: List item 0 has incompatible type "str"; expected "int" + +[case testVariableMethod] +class A: + def a(self) -> None: pass + b = 1 +class B(A): + a = 1 # E: Incompatible types in assignment (expression has type "int", base class "A" defined the type as "Callable[[A], None]") + def b(self) -> None: pass # E: Signature of "b" incompatible with supertype "A" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: def b(self) -> None + +[case testVariableProperty] +class A: + @property + def a(self) -> bool: pass +class B(A): + a = None # type: bool +class C(A): + a = True +class D(A): + a = 1 +[builtins fixtures/property.pyi] +[out] +main:9: error: Incompatible types in assignment (expression has type "int", base class "A" defined the type as "bool") + +[case testVariableOverwriteAny] +from typing import Any +class A: + a = 1 +class B(A): + a = 'x' # type: Any +[out] + +[case testInstanceMethodOverwrite] +class B(): + def n(self, a: int) -> None: pass +class C(B): + def m(self, a: int) -> None: pass + n = m +[out] + +[case testInstanceMethodOverwriteError] +class B(): + def n(self, a: int) -> None: pass +class C(B): + def m(self, a: str) -> None: pass + n = m +[out] +main:5: error: Incompatible types in assignment (expression has type "Callable[[str], None]", base class "B" defined the type as "Callable[[int], None]") + +[case testInstanceMethodOverwriteTypevar] +from typing import Generic, TypeVar +T = TypeVar("T") +class B(Generic[T]): + def n(self, a: T) -> None: pass +class C(B[int]): + def m(self, a: int) -> None: pass + n = m + +[case testInstanceMethodOverwriteTwice] +class I: + def foo(self) -> None: pass +class A(I): + def foo(self) -> None: pass +class B(A): + def bar(self) -> None: pass + foo = bar +class C(B): + def bar(self) -> None: pass + foo = bar + +[case testClassMethodOverwrite] +class B(): + @classmethod + def n(self, a: int) -> None: pass +class C(B): + @classmethod + def m(self, a: int) -> None: pass + n = m +[builtins fixtures/classmethod.pyi] +[out] + +[case testClassMethodOverwriteError] +class B(): + @classmethod + def n(self, a: int) -> None: pass +class C(B): + @classmethod + def m(self, a: str) -> None: pass + n = m +[builtins fixtures/classmethod.pyi] +[out] +main:7: error: Incompatible types in assignment (expression has type "Callable[[str], None]", base class "B" defined the type as "Callable[[int], None]") + +[case testClassSpec] +from typing import Callable +class A(): + b = None # type: Callable[[A, int], int] +class B(A): + def c(self, a: int) -> int: pass + b = c + +[case testClassSpecError] +from typing import Callable +class A(): + b = None # type: Callable[[A, int], int] +class B(A): + def c(self, a: str) -> int: pass + b = c +[out] +main:6: error: Incompatible types in assignment (expression has type "Callable[[str], int]", base class "A" defined the type as "Callable[[int], int]") + +[case testClassStaticMethod] +class A(): + @staticmethod + def a(a: int) -> None: pass +class B(A): + @staticmethod + def b(a: str) -> None: pass + a = b +[builtins fixtures/staticmethod.pyi] +[out] +main:7: error: Incompatible types in assignment (expression has type "Callable[[str], None]", base class "A" defined the type as "Callable[[int], None]") + +[case testClassStaticMethodIndirect] +class A(): + @staticmethod + def a(a: int) -> None: pass + c = a +class B(A): + @staticmethod + def b(a: str) -> None: pass + c = b +[builtins fixtures/staticmethod.pyi] +[out] +main:8: error: Incompatible types in assignment (expression has type "Callable[[str], None]", base class "A" defined the type as "Callable[[int], None]") + +[case testClassStaticMethodSubclassing] +class A: + @staticmethod + def a() -> None: pass + + def b(self) -> None: pass + + @staticmethod + def c() -> None: pass + +class B(A): + def a(self) -> None: pass # Fail + + @classmethod + def b(cls) -> None: pass + + @staticmethod + def c() -> None: pass +[builtins fixtures/classmethod.pyi] +[out] +main:11: error: Signature of "a" incompatible with supertype "A" +main:11: note: Superclass: +main:11: note: @staticmethod +main:11: note: def a() -> None +main:11: note: Subclass: +main:11: note: def a(self) -> None + +[case testTempNode] +class A(): + def a(self) -> None: pass +class B(A): + def b(self) -> None: pass + a = c = b + +[case testListObject] +from typing import List +class A: + x = [] # type: List[object] +class B(A): + x = [1] +[builtins fixtures/list.pyi] + +[case testClassMemberObject] +class A: + x = object() +class B(A): + x = 1 +class C(B): + x = '' +[out] +main:6: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testSlots] +class A: + __slots__ = ("a") +class B(A): + __slots__ = ("a", "b") +[builtins fixtures/tuple.pyi] + +[case testClassOrderOfError] +class A: + x = 1 +class B(A): + x = "a" +class C(B): + x = object() +[out] +main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") +main:6: error: Incompatible types in assignment (expression has type "object", base class "A" defined the type as "int") + +[case testClassOneErrorPerLine] +class A: + x = 1 +class B(A): + x = "" + x = 1.0 +[out] +main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") +main:5: error: Incompatible types in assignment (expression has type "float", base class "A" defined the type as "int") + +[case testClassIgnoreType_RedefinedAttributeAndGrandparentAttributeTypesNotIgnored] +class A: + x = 0 +class B(A): + x = '' # type: ignore +class C(B): + x = '' # E: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") +[out] + +[case testClassIgnoreType_RedefinedAttributeTypeIgnoredInChildren] +class A: + x = 0 +class B(A): + x = '' # type: ignore +class C(B): + x = '' # type: ignore +[out] + +[case testInvalidMetaclassStructure] +class X(type): pass +class Y(type): pass +class A(metaclass=X): pass +class B(A, metaclass=Y): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +[case testMetaclassNoTypeReveal] +class M: + x = 0 # type: int + +class A(metaclass=M): pass # E: Metaclasses not inheriting from "type" are not supported + +A.x # E: "Type[A]" has no attribute "x" + +[case testMetaclassTypeReveal] +from typing import Type +class M(type): + x = 0 # type: int + +class A(metaclass=M): pass + +def f(TA: Type[A]): + reveal_type(TA) # N: Revealed type is "Type[__main__.A]" + reveal_type(TA.x) # N: Revealed type is "builtins.int" + +[case testMetaclassConflictingInstanceVars] +from typing import ClassVar + +class Meta(type): + foo: int + bar: int + eggs: ClassVar[int] = 42 + spam: ClassVar[int] = 42 + +class Foo(metaclass=Meta): + foo: str + bar: ClassVar[str] = 'bar' + eggs: str + spam: ClassVar[str] = 'spam' + +reveal_type(Foo.foo) # N: Revealed type is "builtins.int" +reveal_type(Foo.bar) # N: Revealed type is "builtins.str" +reveal_type(Foo.eggs) # N: Revealed type is "builtins.int" +reveal_type(Foo.spam) # N: Revealed type is "builtins.str" + +class MetaSub(Meta): ... + +class Bar(metaclass=MetaSub): + foo: str + bar: ClassVar[str] = 'bar' + eggs: str + spam: ClassVar[str] = 'spam' + +reveal_type(Bar.foo) # N: Revealed type is "builtins.int" +reveal_type(Bar.bar) # N: Revealed type is "builtins.str" +reveal_type(Bar.eggs) # N: Revealed type is "builtins.int" +reveal_type(Bar.spam) # N: Revealed type is "builtins.str" + +[case testSubclassMetaclass] +class M1(type): + x = 0 +class M2(M1): pass +class C(metaclass=M2): + pass +reveal_type(C.x) # N: Revealed type is "builtins.int" + +[case testMetaclassSubclass] +from typing import Type +class M(type): + x = 0 # type: int + +class A(metaclass=M): pass +class B(A): pass + +def f(TB: Type[B]): + reveal_type(TB) # N: Revealed type is "Type[__main__.B]" + reveal_type(TB.x) # N: Revealed type is "builtins.int" + +[case testMetaclassAsAny] +from typing import Any, ClassVar, Type + +MyAny: Any +class WithMeta(metaclass=MyAny): + x: ClassVar[int] + +reveal_type(WithMeta.a) # N: Revealed type is "Any" +reveal_type(WithMeta.m) # N: Revealed type is "Any" +reveal_type(WithMeta.x) # N: Revealed type is "builtins.int" +reveal_type(WithMeta().x) # N: Revealed type is "builtins.int" +WithMeta().m # E: "WithMeta" has no attribute "m" +WithMeta().a # E: "WithMeta" has no attribute "a" +t: Type[WithMeta] +t.unknown # OK + +[case testMetaclassAsAnyWithAFlag] +# flags: --disallow-subclassing-any +from typing import Any, ClassVar, Type + +MyAny: Any +class WithMeta(metaclass=MyAny): # E: Class cannot use "MyAny" as a metaclass (has type "Any") + x: ClassVar[int] + +reveal_type(WithMeta.a) # N: Revealed type is "Any" +reveal_type(WithMeta.m) # N: Revealed type is "Any" +reveal_type(WithMeta.x) # N: Revealed type is "builtins.int" +reveal_type(WithMeta().x) # N: Revealed type is "builtins.int" +WithMeta().m # E: "WithMeta" has no attribute "m" +WithMeta().a # E: "WithMeta" has no attribute "a" +t: Type[WithMeta] +t.unknown # OK + +[case testUnpackIterableClassWithOverloadedIter] +from typing import Generic, overload, Iterator, TypeVar, Union + +AnyNum = TypeVar('AnyNum', int, float) + +class Foo(Generic[AnyNum]): + @overload + def __iter__(self: Foo[int]) -> Iterator[float]: ... + @overload + def __iter__(self: Foo[float]) -> Iterator[int]: ... + def __iter__(self) -> Iterator[Union[float, int]]: + ... + +a, b, c = Foo[int]() +reveal_type(a) # N: Revealed type is "builtins.float" +reveal_type(b) # N: Revealed type is "builtins.float" +reveal_type(c) # N: Revealed type is "builtins.float" + +x, y = Foo[float]() +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testUnpackIterableClassWithOverloadedIter2] +from typing import Union, TypeVar, Generic, overload, Iterator + +X = TypeVar('X') + +class Foo(Generic[X]): + @overload + def __iter__(self: Foo[str]) -> Iterator[int]: ... # type: ignore + @overload + def __iter__(self: Foo[X]) -> Iterator[str]: ... + def __iter__(self) -> Iterator[Union[int, str]]: + ... + +a, b, c = Foo[str]() +reveal_type(a) # N: Revealed type is "builtins.int" +reveal_type(b) # N: Revealed type is "builtins.int" +reveal_type(c) # N: Revealed type is "builtins.int" + +x, y = Foo[float]() +reveal_type(x) # N: Revealed type is "builtins.str" +reveal_type(y) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + +[case testUnpackIterableRegular] +from typing import TypeVar, Generic, Iterator + +X = TypeVar('X') + +class Foo(Generic[X]): + def __iter__(self) -> Iterator[X]: + ... + +a, b = Foo[int]() +reveal_type(a) # N: Revealed type is "builtins.int" +reveal_type(b) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testUnpackNotIterableClass] +class Foo: ... + +a, b, c = Foo() # E: "Foo" object is not iterable +[builtins fixtures/list.pyi] + +[case testMetaclassIterable] +from typing import Iterable, Iterator + +class ImplicitMeta(type): + def __iter__(self) -> Iterator[int]: yield 1 + +class Implicit(metaclass=ImplicitMeta): pass + +for _ in Implicit: pass +reveal_type(list(Implicit)) # N: Revealed type is "builtins.list[builtins.int]" + +class ExplicitMeta(type, Iterable[int]): + def __iter__(self) -> Iterator[int]: yield 1 + +class Explicit(metaclass=ExplicitMeta): pass +for _ in Explicit: pass +reveal_type(list(Explicit)) # N: Revealed type is "builtins.list[builtins.int]" + +[builtins fixtures/list.pyi] + +[case testMetaclassTuple] +from typing import Tuple + +class M(Tuple[int]): pass +class C(metaclass=M): pass # E: Invalid metaclass "M" + +[builtins fixtures/tuple.pyi] + +[case testMetaclassOperatorBeforeReversed] +class X: + def __radd__(self, x: int) -> int: ... + +class Meta(type): + def __add__(cls, x: X) -> str: ... + +class Concrete(metaclass=Meta): + pass + +reveal_type(Concrete + X()) # N: Revealed type is "builtins.str" +Concrete + "hello" # E: Unsupported operand types for + ("Type[Concrete]" and "str") + +[case testMetaclassOperatorTypeVar] +from typing import Type, TypeVar + +class MetaClass(type): + def __mul__(cls, other: int) -> str: + return "" + +class Test(metaclass=MetaClass): + pass + +S = TypeVar("S", bound=Test) + +def f(x: Type[Test]) -> str: + return x * 0 +def g(x: Type[S]) -> str: + return reveal_type(x * 0) # N: Revealed type is "builtins.str" + +[case testMetaclassGetitem] +import types + +class M(type): + def __getitem__(self, key) -> int: return 1 + +class A(metaclass=M): pass + +reveal_type(A[M]) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMetaclassSelfType] +from typing import TypeVar, Type + +class M(type): pass +T = TypeVar('T') + +class M1(M): + def foo(cls: Type[T]) -> T: ... + +class A(metaclass=M1): pass +reveal_type(A.foo()) # N: Revealed type is "__main__.A" + +[case testMetaclassAndSkippedImport] +# flags: --ignore-missing-imports +from missing import M +class A(metaclass=M): + y = 0 +reveal_type(A.y) # N: Revealed type is "builtins.int" +reveal_type(A.x) # N: Revealed type is "Any" + +[case testValidTypeAliasAsMetaclass] +from typing_extensions import TypeAlias + +Explicit: TypeAlias = type +Implicit = type + +class E(metaclass=Explicit): ... +class I(metaclass=Implicit): ... +[builtins fixtures/classmethod.pyi] + +[case testValidTypeAliasOfTypeAliasAsMetaclass] +from typing_extensions import TypeAlias + +Explicit: TypeAlias = type +Implicit = type + +A1: TypeAlias = Explicit +A2 = Explicit +A3: TypeAlias = Implicit +A4 = Implicit + +class C1(metaclass=A1): ... +class C2(metaclass=A2): ... +class C3(metaclass=A3): ... +class C4(metaclass=A4): ... +[builtins fixtures/classmethod.pyi] + +[case testTypeAliasWithArgsAsMetaclass] +from typing import Generic, TypeVar +from typing_extensions import TypeAlias + +T = TypeVar('T') +class Meta(Generic[T]): ... + +Explicit: TypeAlias = Meta[T] +Implicit = Meta[T] + +class E(metaclass=Explicit): ... # E: Invalid metaclass "Explicit" +class I(metaclass=Implicit): ... # E: Invalid metaclass "Implicit" +[builtins fixtures/classmethod.pyi] + +[case testTypeAliasNonTypeAsMetaclass] +from typing_extensions import TypeAlias + +Explicit: TypeAlias = int +Implicit = int + +class E(metaclass=Explicit): ... # E: Metaclasses not inheriting from "type" are not supported +class I(metaclass=Implicit): ... # E: Metaclasses not inheriting from "type" are not supported +[builtins fixtures/classmethod.pyi] + +[case testInvalidVariableAsMetaclass] +from typing import Any +M = 0 # type: int +MM = 0 +class A(metaclass=M): # E: Invalid metaclass "M" + y = 0 +class B(metaclass=MM): # E: Invalid metaclass "MM" + y = 0 +reveal_type(A.y) # N: Revealed type is "builtins.int" +A.x # E: "Type[A]" has no attribute "x" + +[case testAnyAsBaseOfMetaclass] +from typing import Any, Type +M = None # type: Any +class MM(M): pass + +class A(metaclass=MM): + y = 0 + @classmethod + def f(cls) -> None: pass + def g(self) -> None: pass + +def h(a: Type[A], b: Type[object]) -> None: + h(a, a) + h(b, a) # E: Argument 1 to "h" has incompatible type "Type[object]"; expected "Type[A]" + a.f(1) # E: Too many arguments for "f" of "A" + reveal_type(a.y) # N: Revealed type is "builtins.int" + +x = A # type: MM +reveal_type(A.y) # N: Revealed type is "builtins.int" +reveal_type(A.x) # N: Revealed type is "Any" +A.f(1) # E: Too many arguments for "f" of "A" +A().g(1) # E: Too many arguments for "g" of "A" +[builtins fixtures/classmethod.pyi] + +[case testMetaclassTypeCallable] +class M(type): + x = 5 + +class A(metaclass=M): pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" + +[case testMetaclassStrictSupertypeOfTypeWithClassmethods] +from typing import Type, TypeVar +TA = TypeVar('TA', bound='A') +TTA = TypeVar('TTA', bound='Type[A]') +TM = TypeVar('TM', bound='M') + +class M(type): + def g1(cls: 'Type[A]') -> A: pass # E: The erased type of self "Type[__main__.A]" is not a supertype of its class "__main__.M" + def g2(cls: Type[TA]) -> TA: pass # E: The erased type of self "Type[__main__.A]" is not a supertype of its class "__main__.M" + def g3(cls: TTA) -> TTA: pass # E: The erased type of self "Type[__main__.A]" is not a supertype of its class "__main__.M" + def g4(cls: TM) -> TM: pass +m: M + +class A(metaclass=M): + def foo(self): pass + +reveal_type(A.g1) # N: Revealed type is "def () -> __main__.A" +reveal_type(A.g2) # N: Revealed type is "def () -> __main__.A" +reveal_type(A.g3) # N: Revealed type is "def () -> def () -> __main__.A" +reveal_type(A.g4) # N: Revealed type is "def () -> def () -> __main__.A" + +class B(metaclass=M): + def foo(self): pass + +B.g1 # E: Invalid self argument "Type[B]" to attribute function "g1" with type "Callable[[Type[A]], A]" +B.g2 # E: Invalid self argument "Type[B]" to attribute function "g2" with type "Callable[[Type[TA]], TA]" +B.g3 # E: Invalid self argument "Type[B]" to attribute function "g3" with type "Callable[[TTA], TTA]" +reveal_type(B.g4) # N: Revealed type is "def () -> def () -> __main__.B" + +# 4 examples of unsoundness - instantiation, classmethod, staticmethod and ClassVar: + +ta: Type[A] = m # E: Incompatible types in assignment (expression has type "M", variable has type "Type[A]") +a: A = ta() +reveal_type(ta.g1) # N: Revealed type is "def () -> __main__.A" +reveal_type(ta.g2) # N: Revealed type is "def () -> __main__.A" +reveal_type(ta.g3) # N: Revealed type is "def () -> Type[__main__.A]" +reveal_type(ta.g4) # N: Revealed type is "def () -> Type[__main__.A]" + +x: M = ta +x.g1 # E: Invalid self argument "M" to attribute function "g1" with type "Callable[[Type[A]], A]" +x.g2 # E: Invalid self argument "M" to attribute function "g2" with type "Callable[[Type[TA]], TA]" +x.g3 # E: Invalid self argument "M" to attribute function "g3" with type "Callable[[TTA], TTA]" +reveal_type(x.g4) # N: Revealed type is "def () -> __main__.M" + +def r(ta: Type[TA], tta: TTA) -> None: + x: M = ta + y: M = tta + +class Class(metaclass=M): + @classmethod + def f1(cls: Type[Class]) -> None: pass + @classmethod + def f2(cls: M) -> None: pass +cl: Type[Class] = m # E: Incompatible types in assignment (expression has type "M", variable has type "Type[Class]") +reveal_type(cl.f1) # N: Revealed type is "def ()" +reveal_type(cl.f2) # N: Revealed type is "def ()" +x1: M = cl + +class Static(metaclass=M): + @staticmethod + def f() -> None: pass +s: Type[Static] = m # E: Incompatible types in assignment (expression has type "M", variable has type "Type[Static]") +reveal_type(s.f) # N: Revealed type is "def ()" +x2: M = s + +from typing import ClassVar +class Cvar(metaclass=M): + x = 1 # type: ClassVar[int] +cv: Type[Cvar] = m # E: Incompatible types in assignment (expression has type "M", variable has type "Type[Cvar]") +cv.x +x3: M = cv + +[builtins fixtures/classmethod.pyi] + +[case testMetaclassOverloadResolution] +from typing import Type, overload +class A: pass + +class EM(type): pass +class E(metaclass=EM): pass + +class EM1(type): pass +class E1(metaclass=EM1): pass + +@overload +def f(x: EM) -> int: ... +@overload +def f(x: EM1) -> A: ... +@overload +def f(x: str) -> str: ... +def f(x: object) -> object: return '' + +e: EM +reveal_type(f(e)) # N: Revealed type is "builtins.int" + +et: Type[E] +reveal_type(f(et)) # N: Revealed type is "builtins.int" + +e1: EM1 +reveal_type(f(e1)) # N: Revealed type is "__main__.A" + +e1t: Type[E1] +reveal_type(f(e1t)) # N: Revealed type is "__main__.A" + +reveal_type(f('')) # N: Revealed type is "builtins.str" + +[case testTypeCErasesGenericsFromC] +from typing import Generic, Type, TypeVar + +K = TypeVar('K') +V = TypeVar('V') +class ExampleDict(Generic[K, V]): ... + +D = TypeVar('D') +def mkdict(dict_type: Type[D]) -> D: ... +reveal_type(mkdict(ExampleDict)) # N: Revealed type is "__main__.ExampleDict[Any, Any]" + +[case testTupleForwardBase] +from m import a +a[0]() # E: "int" not callable + +[file m.py] +from typing import Tuple +a: A +class A(Tuple[int, str]): pass +[builtins fixtures/tuple.pyi] + +-- Synthetic types crashes +-- ----------------------- + +[case testCrashOnSelfRecursiveNamedTupleVar] +from typing import NamedTuple + +def test() -> None: + N = NamedTuple('N', [('x', N)]) # E: Cannot resolve name "N" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + n: N + reveal_type(n) # N: Revealed type is "Tuple[Any, fallback=__main__.N@4]" +[builtins fixtures/tuple.pyi] + +[case testCrashOnSelfRecursiveTypedDictVar] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'a': 'A'}) # type: ignore +a: A +[builtins fixtures/isinstancelist.pyi] + +[case testCrashInJoinOfSelfRecursiveNamedTuples] + +from typing import NamedTuple + +class N(NamedTuple): + x: N # type: ignore +class M(NamedTuple): + x: M # type: ignore + +n: N +m: M +lst = [n, m] +[builtins fixtures/isinstancelist.pyi] + +[case testCorrectJoinOfSelfRecursiveTypedDicts] +from mypy_extensions import TypedDict + +def test() -> None: + class N(TypedDict): + x: N # E: Cannot resolve name "N" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + class M(TypedDict): + x: M # E: Cannot resolve name "M" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + + n: N + m: M + lst = [n, m] + reveal_type(lst[0]['x']) # N: Revealed type is "Any" +[builtins fixtures/isinstancelist.pyi] + +[case testCrashInForwardRefToNamedTupleWithIsinstance] +from typing import Dict, NamedTuple + +NameDict = Dict[str, 'NameInfo'] +class NameInfo(NamedTuple): + ast: bool + +def parse_ast(name_dict: NameDict) -> None: + if isinstance(name_dict[''], int): + pass + reveal_type(name_dict['test']) # N: Revealed type is "Tuple[builtins.bool, fallback=__main__.NameInfo]" +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCrashInForwardRefToTypedDictWithIsinstance] +from mypy_extensions import TypedDict +from typing import Dict + +NameDict = Dict[str, 'NameInfo'] +class NameInfo(TypedDict): + ast: bool + +def parse_ast(name_dict: NameDict) -> None: + if isinstance(name_dict[''], int): + pass + reveal_type(name_dict['']['ast']) # N: Revealed type is "builtins.bool" +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCorrectIsinstanceInForwardRefToNewType] +from typing import Dict, NewType + +NameDict = Dict[str, 'NameInfo'] +class Base: + ast: bool +NameInfo = NewType('NameInfo', Base) + +def parse_ast(name_dict: NameDict) -> None: + if isinstance(name_dict[''], int): + pass + x = name_dict[''] + reveal_type(x) # N: Revealed type is "__main__.NameInfo" + if int(): + x = NameInfo(Base()) # OK + x = Base() # E: Incompatible types in assignment (expression has type "Base", variable has type "NameInfo") +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +[case testNoCrashForwardRefToBrokenDoubleNewType] +from typing import Any, Dict, List, NewType + +Foo = NewType('NotFoo', int) # E: String argument 1 "NotFoo" to NewType(...) does not match variable name "Foo" +Foos = NewType('Foos', List[Foo]) # type: ignore + +def frob(foos: Dict[Any, Foos]) -> None: + foo = foos.get(1) + assert foo + dict(foo) +[builtins fixtures/dict.pyi] +[out] + +[case testNoCrashForwardRefToBrokenDoubleNewTypeClass] +from typing import Any, Dict, List, NewType + +Foo = NewType('NotFoo', int) # type: ignore +Foos = NewType('Foos', List[Foo]) # type: ignore + +x: C +class C: + def frob(self, foos: Dict[Any, Foos]) -> None: + foo = foos.get(1) + assert foo + dict(foo) + +reveal_type(x.frob) # N: Revealed type is "def (foos: builtins.dict[Any, __main__.Foos])" +[builtins fixtures/dict.pyi] +[out] + +[case testNewTypeFromForwardNamedTuple] +from typing import NewType, NamedTuple, Tuple + +NT = NewType('NT', 'N') +class N(NamedTuple): + x: int + +x: NT = N(1) # E: Incompatible types in assignment (expression has type "N", variable has type "NT") +x = NT(N(1)) +[builtins fixtures/tuple.pyi] +[out] + +[case testNewTypeFromForwardTypedDict] + +from typing import NewType, Tuple +from mypy_extensions import TypedDict + +NT = NewType('NT', 'N') # E: Argument 2 to NewType(...) must be subclassable (got "N") +class N(TypedDict): + x: int +[builtins fixtures/dict.pyi] +[out] + +[case testCorrectAttributeInForwardRefToNamedTuple] +from typing import NamedTuple +proc: Process +reveal_type(proc.state) # N: Revealed type is "builtins.int" + +def get_state(proc: 'Process') -> int: + return proc.state +class Process(NamedTuple): + state: int +[builtins fixtures/tuple.pyi] +[out] + +[case testCorrectItemTypeInForwardRefToTypedDict] +from mypy_extensions import TypedDict +proc: Process +reveal_type(proc['state']) # N: Revealed type is "builtins.int" + +def get_state(proc: 'Process') -> int: + return proc['state'] +class Process(TypedDict): + state: int +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCorrectDoubleForwardNamedTuple] +from typing import NamedTuple + +x: A +class A(NamedTuple): + one: 'B' + other: int +class B(NamedTuple): + attr: str +y: A +y = x +reveal_type(x.one.attr) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[out] + +[case testCrashOnDoubleForwardTypedDict] +from mypy_extensions import TypedDict + +x: A +class A(TypedDict): + one: 'B' + other: int +class B(TypedDict): + attr: str + +reveal_type(x['one']['attr']) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCrashOnForwardUnionOfNamedTuples] +from typing import Union, NamedTuple + +Node = Union['Foo', 'Bar'] +class Foo(NamedTuple): + x: int +class Bar(NamedTuple): + x: int + +def foo(node: Node) -> int: + x = node + reveal_type(node) # N: Revealed type is "Union[Tuple[builtins.int, fallback=__main__.Foo], Tuple[builtins.int, fallback=__main__.Bar]]" + return x.x +[builtins fixtures/tuple.pyi] +[out] + +[case testCrashOnForwardUnionOfTypedDicts] +from mypy_extensions import TypedDict +from typing import Union + +NodeType = Union['Foo', 'Bar'] +class Foo(TypedDict): + x: int +class Bar(TypedDict): + x: int + +def foo(node: NodeType) -> int: + x = node + return x['x'] +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testSupportForwardUnionOfNewTypes] +from typing import Union, NewType +x: Node +reveal_type(x.x) # N: Revealed type is "builtins.int" + +class A: + x: int +class B: + x: int + +Node = Union['Foo', 'Bar'] +Foo = NewType('Foo', A) +Bar = NewType('Bar', B) + +def foo(node: Node) -> Node: + x = node + return Foo(A()) +[out] + +[case testForwardReferencesInNewTypeMRORecomputed] +from typing import NewType +x: Foo +Foo = NewType('Foo', 'B') +class A: + x: int +class B(A): + pass + +reveal_type(x.x) # N: Revealed type is "builtins.int" +[out] + +[case testCrashOnComplexNamedTupleUnionProperty] +from typing import NamedTuple, Union + +x: AOrB +AOrB = Union['A', 'B'] +class A(NamedTuple): + x: int + +class B(object): + def __init__(self, a: AOrB) -> None: + self.a = a + @property + def x(self) -> int: + return self.a.x + +reveal_type(x.x) # N: Revealed type is "builtins.int" +[builtins fixtures/property.pyi] +[out] + +[case testCorrectIsinstanceWithForwardUnion] +from typing import Union, NamedTuple + +ForwardUnion = Union['TP', int] +class TP(NamedTuple('TP', [('x', int)])): pass + +def f(x: ForwardUnion) -> None: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, fallback=__main__.TP], builtins.int]" + if isinstance(x, TP): + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.TP]" +[builtins fixtures/isinstance.pyi] +[out] + +[case testCrashInvalidArgsSyntheticClassSyntax] +from typing import List, NamedTuple +from mypy_extensions import TypedDict +class TD(TypedDict): + x: List[int, str] # E: "list" expects 1 type argument, but 2 given +class NM(NamedTuple): + x: List[int, str] # E: "list" expects 1 type argument, but 2 given + +# These two should never crash, reveals are in the next test +TD({'x': []}) +NM(x=[]) +[builtins fixtures/dict.pyi] +[out] + +[case testCrashInvalidArgsSyntheticClassSyntaxReveals] +from typing import List, NamedTuple +from mypy_extensions import TypedDict +class TD(TypedDict): + x: List[int, str] # E: "list" expects 1 type argument, but 2 given +class NM(NamedTuple): + x: List[int, str] # E: "list" expects 1 type argument, but 2 given + +x: TD +x1 = TD({'x': []}) +y: NM +y1 = NM(x=[]) +reveal_type(x) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.list[Any]})" +reveal_type(x1) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.list[Any]})" +reveal_type(y) # N: Revealed type is "Tuple[builtins.list[Any], fallback=__main__.NM]" +reveal_type(y1) # N: Revealed type is "Tuple[builtins.list[Any], fallback=__main__.NM]" +[builtins fixtures/dict.pyi] +[out] + +[case testCrashInvalidArgsSyntheticFunctionSyntax] +from typing import List, NewType, NamedTuple +from mypy_extensions import TypedDict +TD = TypedDict('TD', {'x': List[int, str]}) # E: "list" expects 1 type argument, but 2 given +NM = NamedTuple('NM', [('x', List[int, str])]) # E: "list" expects 1 type argument, but 2 given +NT = NewType('NT', List[int, str]) # E: "list" expects 1 type argument, but 2 given + +# These three should not crash +TD({'x': []}) +NM(x=[]) +NT([]) +[builtins fixtures/dict.pyi] +[out] + +[case testCrashForwardSyntheticClassSyntax] +from typing import NamedTuple +from mypy_extensions import TypedDict +class A1(NamedTuple): + b: 'B' + x: int +class A2(TypedDict): + b: 'B' + x: int +class B: + pass +x: A1 +y: A2 +reveal_type(x.b) # N: Revealed type is "__main__.B" +reveal_type(y['b']) # N: Revealed type is "__main__.B" +[builtins fixtures/dict.pyi] +[out] + +[case testCrashForwardSyntheticFunctionSyntax] +from typing import NamedTuple +from mypy_extensions import TypedDict +A1 = NamedTuple('A1', [('b', 'B'), ('x', int)]) +A2 = TypedDict('A2', {'b': 'B', 'x': int}) +class B: + pass +x: A1 +y: A2 +reveal_type(x.b) # N: Revealed type is "__main__.B" +reveal_type(y['b']) # N: Revealed type is "__main__.B" +[builtins fixtures/dict.pyi] +[out] + +-- Special support for six +-- ----------------------- + +[case testSixMetaclass] +import six +class M(type): + x = 5 +class A(six.with_metaclass(M)): pass +@six.add_metaclass(M) +class B: pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +reveal_type(type(B).x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testFromSixMetaclass] +from six import with_metaclass, add_metaclass +class M(type): + x = 5 +class A(with_metaclass(M)): pass +@add_metaclass(M) +class B: pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +reveal_type(type(B).x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testSixMetaclassImportFrom] +import six +from metadefs import M +class A(six.with_metaclass(M)): pass +@six.add_metaclass(M) +class B: pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +reveal_type(type(B).x) # N: Revealed type is "builtins.int" +[file metadefs.py] +class M(type): + x = 5 +[builtins fixtures/tuple.pyi] + +[case testSixMetaclassImport] +import six +import metadefs +class A(six.with_metaclass(metadefs.M)): pass +@six.add_metaclass(metadefs.M) +class B: pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +reveal_type(type(B).x) # N: Revealed type is "builtins.int" +[file metadefs.py] +class M(type): + x = 5 +[builtins fixtures/tuple.pyi] + +[case testSixMetaclassAndBase] +from typing import Iterable, Iterator +import six +class M(type, Iterable[int]): + x = 5 + def __iter__(self) -> Iterator[int]: ... +class A: + def foo(self): pass +class B: + def bar(self): pass +class C1(six.with_metaclass(M, A)): pass +@six.add_metaclass(M) +class D1(A): pass +class C2(six.with_metaclass(M, A, B)): pass +@six.add_metaclass(M) +class D2(A, B): pass +reveal_type(type(C1).x) # N: Revealed type is "builtins.int" +reveal_type(type(D1).x) # N: Revealed type is "builtins.int" +reveal_type(type(C2).x) # N: Revealed type is "builtins.int" +reveal_type(type(D2).x) # N: Revealed type is "builtins.int" +C1().foo() +D1().foo() +C1().bar() # E: "C1" has no attribute "bar" +D1().bar() # E: "D1" has no attribute "bar" +for x in C1: reveal_type(x) # N: Revealed type is "builtins.int" +for x in C2: reveal_type(x) # N: Revealed type is "builtins.int" +C2().foo() +D2().foo() +C2().bar() +D2().bar() +C2().baz() # E: "C2" has no attribute "baz" +D2().baz() # E: "D2" has no attribute "baz" +[builtins fixtures/tuple.pyi] + +[case testSixMetaclassGenerics] +from typing import Generic, GenericMeta, TypeVar +import six +class DestroyableMeta(type): + pass +class Destroyable(six.with_metaclass(DestroyableMeta)): + pass +T_co = TypeVar('T_co', bound='Destroyable', covariant=True) +class ArcMeta(GenericMeta, DestroyableMeta): + pass +class Arc(six.with_metaclass(ArcMeta, Generic[T_co], Destroyable)): + pass +@six.add_metaclass(ArcMeta) +class Arc1(Generic[T_co], Destroyable): + pass +class MyDestr(Destroyable): + pass +reveal_type(Arc[MyDestr]()) # N: Revealed type is "__main__.Arc[__main__.MyDestr]" +reveal_type(Arc1[MyDestr]()) # N: Revealed type is "__main__.Arc1[__main__.MyDestr]" +[builtins fixtures/bool.pyi] +[typing fixtures/typing-full.pyi] + +[case testSixMetaclassErrors] + +import six +class M(type): pass +class A(object): pass +def f() -> type: return M +class C1(six.with_metaclass(M), object): pass # E: Unsupported dynamic base class "six.with_metaclass" +class C2(C1, six.with_metaclass(M)): pass # E: Unsupported dynamic base class "six.with_metaclass" +class C3(six.with_metaclass(A)): pass # E: Metaclasses not inheriting from "type" are not supported +@six.add_metaclass(A) # E: Metaclasses not inheriting from "type" are not supported \ + # E: Argument 1 to "add_metaclass" has incompatible type "Type[A]"; expected "Type[type]" + +class D3(A): pass +class C4(six.with_metaclass(M), metaclass=M): pass # E: Multiple metaclass definitions +@six.add_metaclass(M) +class D4(metaclass=M): pass # E: Multiple metaclass definitions +class C5(six.with_metaclass(f())): pass # E: Dynamic metaclass not supported for "C5" +@six.add_metaclass(f()) # E: Dynamic metaclass not supported for "D5" +class D5: pass + +@six.add_metaclass(M) +class CD(six.with_metaclass(M)): pass # E: Multiple metaclass definitions + +class M1(type): pass +class Q1(metaclass=M1): pass +@six.add_metaclass(M) +class CQA(Q1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class CQW(six.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +[builtins fixtures/tuple.pyi] + +[case testSixMetaclassAny] +import t # type: ignore +import six +class E(metaclass=t.M): pass +class F(six.with_metaclass(t.M)): pass +@six.add_metaclass(t.M) +class G: pass +[builtins fixtures/tuple.pyi] + +[case testSixMetaclassGenericBase] +import six +import abc +from typing import TypeVar, Generic + +T = TypeVar("T") + +class C(six.with_metaclass(abc.ABCMeta, Generic[T])): + pass +class D(six.with_metaclass(abc.ABCMeta, C[T])): + pass +[builtins fixtures/tuple.pyi] + +-- Special support for future.utils +-- -------------------------------- + +[case testFutureMetaclass] +import future.utils +class M(type): + x = 5 +class A(future.utils.with_metaclass(M)): pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testFromFutureMetaclass] +from future.utils import with_metaclass +class M(type): + x = 5 +class A(with_metaclass(M)): pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testFutureMetaclassImportFrom] +import future.utils +from metadefs import M +class A(future.utils.with_metaclass(M)): pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +[file metadefs.py] +class M(type): + x = 5 +[builtins fixtures/tuple.pyi] + +[case testFutureMetaclassImport] +import future.utils +import metadefs +class A(future.utils.with_metaclass(metadefs.M)): pass +reveal_type(type(A).x) # N: Revealed type is "builtins.int" +[file metadefs.py] +class M(type): + x = 5 +[builtins fixtures/tuple.pyi] + +[case testFutureMetaclassAndBase] +from typing import Iterable, Iterator +import future.utils +class M(type, Iterable[int]): + x = 5 + def __iter__(self) -> Iterator[int]: ... +class A: + def foo(self): pass +class B: + def bar(self): pass +class C1(future.utils.with_metaclass(M, A)): pass +class C2(future.utils.with_metaclass(M, A, B)): pass +reveal_type(type(C1).x) # N: Revealed type is "builtins.int" +reveal_type(type(C2).x) # N: Revealed type is "builtins.int" +C1().foo() +C1().bar() # E: "C1" has no attribute "bar" +for x in C1: reveal_type(x) # N: Revealed type is "builtins.int" +for x in C2: reveal_type(x) # N: Revealed type is "builtins.int" +C2().foo() +C2().bar() +C2().baz() # E: "C2" has no attribute "baz" +[builtins fixtures/tuple.pyi] + +[case testFutureMetaclassGenerics] +from typing import Generic, GenericMeta, TypeVar +import future.utils +class DestroyableMeta(type): + pass +class Destroyable(future.utils.with_metaclass(DestroyableMeta)): + pass +T_co = TypeVar('T_co', bound='Destroyable', covariant=True) +class ArcMeta(GenericMeta, DestroyableMeta): + pass +class Arc(future.utils.with_metaclass(ArcMeta, Generic[T_co], Destroyable)): + pass +class MyDestr(Destroyable): + pass +reveal_type(Arc[MyDestr]()) # N: Revealed type is "__main__.Arc[__main__.MyDestr]" +[builtins fixtures/bool.pyi] +[typing fixtures/typing-full.pyi] + +[case testFutureMetaclassErrors] +import future.utils +class M(type): pass +class A(object): pass +def f() -> type: return M +class C1(future.utils.with_metaclass(M), object): pass # E: Unsupported dynamic base class "future.utils.with_metaclass" +class C2(C1, future.utils.with_metaclass(M)): pass # E: Unsupported dynamic base class "future.utils.with_metaclass" +class C3(future.utils.with_metaclass(A)): pass # E: Metaclasses not inheriting from "type" are not supported +class C4(future.utils.with_metaclass(M), metaclass=M): pass # E: Multiple metaclass definitions +class C5(future.utils.with_metaclass(f())): pass # E: Dynamic metaclass not supported for "C5" + +class M1(type): pass +class Q1(metaclass=M1): pass +class CQW(future.utils.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +[builtins fixtures/tuple.pyi] + +[case testFutureMetaclassAny] +import t # type: ignore +import future.utils +class E(metaclass=t.M): pass +class F(future.utils.with_metaclass(t.M)): pass + +-- Misc +-- ---- +[builtins fixtures/tuple.pyi] + +[case testCorrectEnclosingClassPushedInDeferred] +class C: + def __getattr__(self, attr: str) -> int: + x: F + return x.f + +class F: + def __init__(self, f: int) -> None: + self.f = f +[out] + +[case testCorrectEnclosingClassPushedInDeferred2] +from typing import TypeVar +T = TypeVar('T', bound='C') +class C: + def m(self: T) -> T: + class Inner: + x: F + f = x.f + return self + +class F: + def __init__(self, f: int) -> None: + self.f = f +[out] + +[case testCorrectEnclosingClassPushedInDeferred3] +class A: + def f(self) -> None: + def g(x: int) -> int: + return y + +y = int() +[out] + +[case testMetaclassMemberAccessViaType] +from typing import Type +class M(type): + def m(cls, x: int) -> int: + pass + +class C(metaclass=M): + pass +x = C +y: Type[C] = C + +reveal_type(type(C).m) # N: Revealed type is "def (cls: __main__.M, x: builtins.int) -> builtins.int" +reveal_type(type(x).m) # N: Revealed type is "def (cls: __main__.M, x: builtins.int) -> builtins.int" +reveal_type(type(y).m) # N: Revealed type is "def (cls: __main__.M, x: builtins.int) -> builtins.int" +[out] + +[case testMetaclassMemberAccessViaType2] +from typing import Any, Type +class M(type): + def m(cls, x: int) -> int: + pass +B: Any +class C(B, metaclass=M): + pass + +x: Type[C] +reveal_type(x.m) # N: Revealed type is "def (x: builtins.int) -> builtins.int" +reveal_type(x.whatever) # N: Revealed type is "Any" +[out] + +[case testMetaclassMemberAccessViaType3] +from typing import Any, Type, TypeVar +T = TypeVar('T') +class C(Any): + def bar(self: T) -> Type[T]: pass + def foo(self) -> None: + reveal_type(self.bar()) # N: Revealed type is "Type[__main__.C]" + reveal_type(self.bar().__name__) # N: Revealed type is "builtins.str" +[builtins fixtures/type.pyi] +[out] + +[case testClassDecoratorIsTypeChecked] +from typing import Callable, Type +def decorate(x: int) -> Callable[[type], type]: # N: "decorate" defined here + ... +def decorate_forward_ref() -> Callable[[Type[A]], Type[A]]: + ... +@decorate(y=17) # E: Unexpected keyword argument "y" for "decorate" +@decorate() # E: Missing positional argument "x" in call to "decorate" +@decorate(22, 25) # E: Too many arguments for "decorate" +@decorate_forward_ref() +@decorate(11) +class A: pass + +@decorate # E: Argument 1 to "decorate" has incompatible type "Type[A2]"; expected "int" +class A2: pass + +[case testClassDecoratorIncorrect] +def not_a_class_decorator(x: int) -> int: ... +@not_a_class_decorator(7) +class A3: pass # E: "int" not callable + +not_a_function = 17 +@not_a_function() # E: "int" not callable +class B: pass + +@not_a_function +class B2: pass # E: "int" not callable + +b = object() +@b.nothing # E: "object" has no attribute "nothing" +class C: pass + +@undefined # E: Name "undefined" is not defined +class D: pass + +[case testSlotsCompatibility] +class A: + __slots__ = () +class B(A): + __slots__ = ('a', 'b') +class C: + __slots__ = ('x',) +class D(B, C): + __slots__ = ('aa', 'bb', 'cc') +[builtins fixtures/tuple.pyi] + +[case testRevealLocalsOnClassVars] +class C1(object): + t = 'a' + y = 3.0 + class Inner(object): pass + reveal_locals() + +[out] +main:5: note: Revealed local types are: +main:5: note: t: builtins.str +main:5: note: y: builtins.float + +[case testAbstractClasses] +import a +import b + +[file a.pyi] +from abc import ABCMeta, abstractmethod +from typing import Protocol + +class A: # OK, has @abstractmethod + @abstractmethod + def f(self) -> None: + pass + +class B(A): # E: Class a.B has abstract attributes "f" # N: If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass + pass + +class C(A, metaclass=ABCMeta): # OK, has ABCMeta as a metaclass + pass + +class D(A): # OK, implements the abstract method + def f(self) -> None: + pass + +class E(Protocol): # OK, is a protocol + @abstractmethod + def f(self) -> None: + pass + +class F(E, Protocol): # OK, is a protocol + pass + +# Custom metaclass subclassing `ABCMeta`, see #13561 +class CustomMeta(ABCMeta): + pass + +class G(A, metaclass=CustomMeta): # Ok, has CustomMeta as a metaclass + pass + +[file b.py] +# All of these are OK because this is not a stub file. +from abc import ABCMeta, abstractmethod +from typing import Protocol + +class A: + @abstractmethod + def f(self) -> None: + pass + +class B(A): + pass + +class C(A, metaclass=ABCMeta): + pass + +class D(A): + def f(self) -> None: + pass + +class E(Protocol): + @abstractmethod + def f(self) -> None: + pass + +class F(E, Protocol): + pass + +class CustomMeta(ABCMeta): + pass + +class G(A, metaclass=CustomMeta): + pass + +[case testClassMethodOverride] +from typing import Callable, Any + +def deco(f: Callable[..., Any]) -> Callable[..., Any]: ... + +class B: + @classmethod + def meth(cls, x: int) -> int: ... + +class C(B): + @classmethod + @deco + def meth(cls, x: int) -> int: ... +[builtins fixtures/classmethod.pyi] +[out] + +[case testGetAttrImportAnnotation] +import a +x: a.A +y: a.A.B.C +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "Any" +[file a.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testGetAttrImportBaseClass] +import a +class B(a.A): ... +[file a.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testGetAttrDescriptor] +from typing import TypeVar, Generic, Any + +T = TypeVar('T') +class C(Generic[T]): + normal: T + def __getattr__(self, attr: str) -> T: ... + +class Descr: + def __get__(self, inst: Any, owner: Any) -> int: ... + +class D(C[Descr]): + other: Descr + +d: D +reveal_type(d.normal) # N: Revealed type is "builtins.int" +reveal_type(d.dynamic) # N: Revealed type is "__main__.Descr" +reveal_type(D.other) # N: Revealed type is "builtins.int" +D.dynamic # E: "Type[D]" has no attribute "dynamic" +[out] + +[case testSelfDescriptorAssign] +from typing import Any + +class Descr: + def __get__(self, inst: Any, owner: Any) -> int: ... + +class C: + def __init__(self, x: Descr) -> None: + self.x = x + +c = C(Descr()) +reveal_type(c.x) # N: Revealed type is "__main__.Descr" +[out] + +[case testForwardInstanceWithWrongArgCount] +from typing import TypeVar, Generic + +T = TypeVar('T') +class G(Generic[T]): ... + +A = G +x: A[B[int, int]] # E: "G" expects 1 type argument, but 2 given +B = G +[out] + +[case testForwardInstanceWithNoArgs] +from typing import TypeVar, Generic + +T = TypeVar('T') +class G(Generic[T]): ... + +A = G +x: A[B] +reveal_type(x) # N: Revealed type is "__main__.G[__main__.G[Any]]" +B = G +[out] + +[case testForwardInstanceWithBound] +# flags: --show-column-numbers +from typing import TypeVar, Generic + +T = TypeVar('T', bound=str) +class G(Generic[T]): ... + +A = G +x: A[B[int]] # E +B = G +[out] +main:8:4: error: Type argument "G[int]" of "G" must be a subtype of "str" +main:8:6: error: Type argument "int" of "G" must be a subtype of "str" + +[case testExtremeForwardReferencing] +from typing import TypeVar, Generic + +T = TypeVar('T', covariant=True) +class B(Generic[T]): ... + +y: A +z: A[int] +x = [y, z] +reveal_type(x) # N: Revealed type is "builtins.list[__main__.B[Any]]" + +A = B +[builtins fixtures/list.pyi] +[out] + +[case testNoneAnyFallback] +from typing import Any +dynamic: Any +class C(dynamic): pass +x: None = C() # E: Incompatible types in assignment (expression has type "C", variable has type "None") +[out] + +[case testNoneAnyFallbackDescriptor] +from typing import Any +from d import Descr + +dynamic: Any +class C(dynamic): + id = Descr(int) + name = Descr(str) + +c: C +reveal_type(c.id) # N: Revealed type is "builtins.int" +reveal_type(C.name) # N: Revealed type is "d.Descr[builtins.str]" + +[file d.pyi] +from typing import Any, overload, Generic, TypeVar, Type + +T = TypeVar('T') +class Descr(Generic[T]): + def __init__(self, tp: Type[T]) -> None: ... + @overload + def __get__(self, inst: None, owner: Any) -> Descr[T]: ... + @overload + def __get__(self, inst: object, owner: Any) -> T: ... +[out] + +[case testClassCustomPropertyWorks] +from typing import TypeVar, Generic, Callable, Any + +V = TypeVar('V') + +class classproperty(Generic[V]): + def __init__(self, getter: Callable[[Any], V]) -> None: + self.getter = getter + def __get__(self, instance: Any, owner: Any) -> V: + return self.getter(owner) + +class C: + @classproperty + def foo(cls) -> int: + return 42 + +reveal_type(C.foo) # N: Revealed type is "builtins.int" +reveal_type(C().foo) # N: Revealed type is "builtins.int" +[out] + +[case testMultipleInheritanceCycle] +import b +[file a.py] +from b import B +class A: ... +class C(A, B): ... +class D(C): ... +class Other: ... +[file b.py] +from a import Other +class B: ... +[out] + +[case testMultipleInheritanceCycle2] +import b +[file a.py] +from b import B +class A: ... +class C(A, B): ... +class D(C): ... +class Other: ... +a: A +b: B +c: C +d: D +d = A() # E: Incompatible types in assignment (expression has type "A", variable has type "D") +if int(): + d = B() # E: Incompatible types in assignment (expression has type "B", variable has type "D") +if int(): + d = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") +a = D() +b = D() +c = D() +[file b.py] +from a import Other +class B: ... +[out] + +[case testAllowPropertyAndInit1] +class C: + def __init__(self, x: int) -> None: + self.x = x + @property + def x(self) -> int: pass + @x.setter + def x(self, x: int) -> None: pass +[builtins fixtures/property.pyi] +[out] + +[case testAllowPropertyAndInit2] +class C: + @property + def x(self) -> int: pass + @x.setter + def x(self, x: int) -> None: pass + def __init__(self, x: int) -> None: + self.x = x +[builtins fixtures/property.pyi] + +[case testAllowPropertyAndInit3] +class C: + def __init__(self, x: int) -> None: + self.x = x # type: ignore + @property # Should be no error here + def x(self) -> int: pass +[builtins fixtures/property.pyi] +[out] + +[case testClassMethodBeforeInit1] +class Foo: + @classmethod + def bar(cls) -> Foo: + return cls("bar") + + def __init__(self, baz: str) -> None: + self.baz = baz +[builtins fixtures/classmethod.pyi] + +[case testClassMethodBeforeInit2] +class Foo: + @classmethod + def bar(cls) -> Foo: + return cls(Bar()) + + def __init__(self, baz: 'Bar') -> None: + self.baz = baz + +class Bar: pass +[builtins fixtures/classmethod.pyi] + +[case testClassMethodBeforeInit3] +from typing import overload +class Foo: + @classmethod + @overload + def bar(cls, x: int) -> Foo: ... + @classmethod + @overload + def bar(cls, x: str) -> Foo: ... + @classmethod + def bar(cls, x: object) -> Foo: + return cls(x) + + def __init__(self, baz: object) -> None: + self.baz = baz + +[builtins fixtures/classmethod.pyi] + +[case testNewAndInit1] +class A: + def __init__(self, x: int) -> None: + pass + +class B(A): + def __new__(cls) -> B: + pass + +B() + +[case testNewAndInit2] +from typing import Any + +class A: + def __new__(cls, *args: Any) -> 'A': + ... + +class B(A): + def __init__(self, x: int) -> None: + pass + +reveal_type(B) # N: Revealed type is "def (x: builtins.int) -> __main__.B" +[builtins fixtures/tuple.pyi] + +[case testNewAndInit3] +from typing import Any + +class A: + def __new__(cls, *args: Any) -> 'A': + ... + def __init__(self, x: int) -> None: + pass + +reveal_type(A) # N: Revealed type is "def (x: builtins.int) -> __main__.A" +[builtins fixtures/tuple.pyi] + +[case testCyclicDecorator] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: ... +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[out] + +[case testCyclicOverload] +import b +[file a.pyi] +import b +from typing import overload + +class A(b.B): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +[file b.pyi] +import a +from typing import overload + +class B: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +[out] + +[case testCyclicOverloadDeferred] +import b +[file a.py] +import b +from typing import overload, Union + +class A(b.B): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x) -> Union[int, str]: + reveal_type(other.x) # N: Revealed type is "builtins.int" + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +import a +from typing import overload + +class B: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x): + pass +[out] + +[case testCyclicOverrideAny] +import a +[file b.py] +import a +class Sub(a.Base): + def x(self) -> int: pass + +[file a.py] +import b +class Base: + def __init__(self): + self.x = 1 +[out] + +[case testCyclicOverrideChecked] +import a +[file b.py] +import a +class Sub(a.Base): + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: def x(self) -> int + +[file a.py] +import b +class Base: + def __init__(self) -> None: + self.x = 1 +[out] + +[case testCyclicOverrideCheckedDecorator] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: def x(*Any, **Any) -> Tuple[int, int] + +[file a.py] +import b +import c +class Base: + def __init__(self) -> None: + self.x = 1 +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[out] + +[case testCyclicOverrideCheckedDecoratorDeferred] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: def x(*Any, **Any) -> Tuple[int, int] + +[file a.py] +import b +import c +class Base: + def __init__(self) -> None: + self.x = f() + +def f() -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[out] + +[case testCyclicOverrideAnyDecoratorDeferred] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass + +[file a.py] +from b import Sub +import c +class Base: + def __init__(self) -> None: + self.x = f() + +def f() -> int: ... +[file c.py] +from typing import Any, Callable +def deco(f: Callable[..., Any]) -> Any: ... +[out] + +[case testCyclicDecoratorDoubleDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + reveal_type(other.x) # N: Revealed type is "builtins.int" + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[out] + +[case testCyclicDecoratorSuper] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + y = super().meth() + reveal_type(y) # N: Revealed type is "Tuple[builtins.int, builtins.int]" + return 0 +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[out] + +[case testCyclicDecoratorBothDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + pass +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + reveal_type(other.x) # N: Revealed type is "builtins.int" + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[out] + +[case testCyclicDecoratorSuperDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + y = super().meth() + reveal_type(y) # N: Revealed type is "Tuple[builtins.int, builtins.int]" + reveal_type(other.x) # N: Revealed type is "builtins.int" + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[out] + +[case testOptionalDescriptorsBinder] +from typing import Type, TypeVar, Optional +T = TypeVar('T') + +class IntDescr: + def __get__(self, obj: T, typ: Type[T]) -> Optional[int]: ... + def __set__(self, obj: T, value: Optional[int]) -> None: ... + +class C: + spec = IntDescr() + + def meth_spec(self) -> None: + if self.spec is None: + self.spec = 0 + reveal_type(self.spec) # N: Revealed type is "builtins.int" +[builtins fixtures/bool.pyi] + +[case testUnionDescriptorsBinder] +from typing import Type, TypeVar, Union +T = TypeVar('T') + +class A: ... +class B: ... + +class UnionDescr: + def __get__(self, obj: T, typ: Type[T]) -> Union[A, B]: ... + def __set__(self, obj: T, value: Union[A, B]) -> None: ... + +class C: + spec = UnionDescr() + + def meth_spec(self) -> None: + self.spec = A() + reveal_type(self.spec) # N: Revealed type is "__main__.A" +[builtins fixtures/bool.pyi] + +[case testSubclassDescriptorsBinder] +from typing import Type, TypeVar, Optional +T = TypeVar('T') + +class A: ... +class B(A): ... + +class SubDescr: + def __get__(self, obj: T, typ: Type[T]) -> A: ... + def __set__(self, obj: T, value: A) -> None: ... + +class C: + spec = SubDescr() + + def meth_spec(self) -> None: + self.spec = B() + reveal_type(self.spec) # N: Revealed type is "__main__.B" +[builtins fixtures/bool.pyi] + +[case testDecoratedDunderGet] +from typing import Any, Callable, TypeVar, Type + +F = TypeVar('F', bound=Callable) +T = TypeVar('T') + +def decorator(f: F) -> F: + return f + +def change(f: Callable) -> Callable[..., int]: + pass + +def untyped(f): + return f + +class A: ... + +class Descr1: + @decorator + def __get__(self, obj: T, typ: Type[T]) -> A: ... +class Descr2: + @change + def __get__(self, obj: T, typ: Type[T]) -> A: ... +class Descr3: + @untyped + def __get__(self, obj: T, typ: Type[T]) -> A: ... + +class C: + spec1 = Descr1() + spec2 = Descr2() + spec3 = Descr3() + +c: C +reveal_type(c.spec1) # N: Revealed type is "__main__.A" +reveal_type(c.spec2) # N: Revealed type is "builtins.int" +reveal_type(c.spec3) # N: Revealed type is "Any" +[builtins fixtures/bool.pyi] + +[case testDecoratedDunderSet] +from typing import Any, Callable, TypeVar, Type + +F = TypeVar('F', bound=Callable) +T = TypeVar('T') + +def decorator(f: F) -> F: + return f + +def change(f: Callable) -> Callable[[Any, Any, int], None]: + pass + +def untyped(f): + return f + +class A: ... + +class Descr1: + @decorator + def __set__(self, obj: T, value: A) -> None: ... +class Descr2: + @change + def __set__(self, obj: T, value: A) -> None: ... +class Descr3: + @untyped + def __set__(self, obj: T, value: A) -> None: ... + +class C: + spec1 = Descr1() + spec2 = Descr2() + spec3 = Descr3() + +c: C +c.spec1 = A() +c.spec1 = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "A") +c.spec2 = A() # E: Incompatible types in assignment (expression has type "A", variable has type "int") +c.spec2 = 1 +c.spec3 = A() +c.spec3 = 1 +[builtins fixtures/bool.pyi] + +[case testClassLevelImport] +# flags: --ignore-missing-imports +class Test: + import a + def __init__(self) -> None: + some_module = self.a +[out] + +[case testIsInstanceTypeVsMetaclass] +from typing import Type +class Meta(type): + pass +class Thing(metaclass=Meta): + pass + +def foo(x: Type[Thing]) -> Type[Thing]: + assert isinstance(x, Meta) + return x +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceTypeVsUnionOfType] +from typing import Type, Union + +class AA: pass +class AB: pass + +class M: pass + +class A(M, AA): pass +class B(M, AB): pass + +AOrB = Union[A, B] + +class T(object): + def __init__(self, typ: Type[AOrB] = A) -> None: + assert isinstance(typ, type(M)) + self.typ: Type[AOrB] = typ +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceTypeIsSubclass] +from typing import Union, Type + +class C: ... + +x: Union[C, Type[C]] + +if isinstance(x, type) and issubclass(x, C): + reveal_type(x) # N: Revealed type is "Type[__main__.C]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceTypeByAssert] +class A: + x = 42 + +i: type = A +assert issubclass(i, A) +reveal_type(i.x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceTypeTypeVar] +from typing import Type, TypeVar, Generic + +class Base: ... +class Sub(Base): ... + +T = TypeVar('T', bound=Base) + +class C(Generic[T]): + def meth(self, cls: Type[T]) -> None: + if not issubclass(cls, Sub): + return + reveal_type(cls) # N: Revealed type is "Type[__main__.Sub]" + def other(self, cls: Type[T]) -> None: + if not issubclass(cls, Sub): + return + reveal_type(cls) # N: Revealed type is "Type[__main__.Sub]" + +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceTypeSubclass] +from typing import Type, Optional +class Base: ... +class One(Base): + x: int +class Other(Base): + x: int + +def test() -> None: + x: Optional[Type[Base]] + if int(): + x = One + elif int(): + x = Other + else: + return + reveal_type(x) # N: Revealed type is "Union[def () -> __main__.One, def () -> __main__.Other]" + reveal_type(x.x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] + +[case testMemberRedefinition] +class C: + def __init__(self) -> None: + self.foo = 12 + self.foo: int = 12 # E: Attribute "foo" already defined on line 3 + +[case testMemberRedefinitionDefinedInClass] +class C: + foo = 12 + def __init__(self) -> None: + self.foo: int = 12 # E: Attribute "foo" already defined on line 2 + +[case testAbstractInit] +from abc import abstractmethod, ABCMeta +class A(metaclass=ABCMeta): + @abstractmethod + def __init__(self, a: int) -> None: + pass +class B(A): + pass +class C(B): + def __init__(self, a: int) -> None: + self.c = a +a = A(1) # E: Cannot instantiate abstract class "A" with abstract attribute "__init__" +A.c # E: "Type[A]" has no attribute "c" +b = B(2) # E: Cannot instantiate abstract class "B" with abstract attribute "__init__" +B.c # E: "Type[B]" has no attribute "c" +c = C(3) +c.c +C.c + +[case testDecoratedConstructors] +from typing import TypeVar, Callable, Any + +F = TypeVar('F', bound=Callable[..., Any]) + +def dec(f: F) -> F: ... + +class A: + @dec + def __init__(self, x: int) -> None: ... + +class B: + @dec + def __new__(cls, x: int) -> B: ... + +reveal_type(A) # N: Revealed type is "def (x: builtins.int) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (x: builtins.int) -> __main__.B" + +[case testDecoratedConstructorsBad] +from typing import Callable, Any + +def dec(f: Callable[[Any, int], Any]) -> int: ... + +class A: + @dec # E: Unsupported decorated constructor type + def __init__(self, x: int) -> None: ... + +class B: + @dec # E: Unsupported decorated constructor type + def __new__(cls, x: int) -> B: ... + +[case testIgnorePrivateAttributesTypeCheck] +class B: + __foo_: int +class C(B): + __foo_: str +[out] + +[case testIgnorePrivateMethodsTypeCheck] +class B: + def __foo_(self) -> int: ... +class C(B): + def __foo_(self) -> str: ... +[out] + +[case testCheckForPrivateMethodsWhenPublicCheck] +class B: + __foo__: int +class C(B): + __foo__: str +[out] +main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testIgnorePrivateMethodsTypeCheck2] +class A: + def __foo_(self) -> int: ... +class B: + def __foo_(self) -> str: ... + +class C(A, B): pass +[out] + +[case testAttributeDefOrder1] +import a + +[file a.py] +from b import C + +class D(C): + def g(self) -> None: + self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + + def f(self) -> None: + reveal_type(self.x) # N: Revealed type is "builtins.int" + + +[file b.py] +import a + +class C: + def __init__(self) -> None: + self.x = 0 + +[targets b, a, b.C.__init__, a.D.g, a.D.f, __main__] + +[case testAttributeDefOrder2] +class D(C): + def g(self) -> None: + self.x = '' + + def f(self) -> None: + # https://github.com/python/mypy/issues/7162 + reveal_type(self.x) # N: Revealed type is "builtins.str" + + +class C: + def __init__(self) -> None: + self.x = 0 + +class E(C): + def g(self) -> None: + self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + + def f(self) -> None: + reveal_type(self.x) # N: Revealed type is "builtins.int" + +[targets __main__, __main__, __main__.D.g, __main__.D.f, __main__.C.__init__, __main__.E.g, __main__.E.f] + +[case testNewReturnType1] +class A: + def __new__(cls) -> B: + pass + +class B(A): pass + +reveal_type(A()) # N: Revealed type is "__main__.B" +reveal_type(B()) # N: Revealed type is "__main__.B" + +[case testNewReturnType2] +from typing import Any + +# make sure that __new__ method that return Any are ignored when +# determining the return type +class A: + def __new__(cls): + pass + +class B: + def __new__(cls) -> Any: + pass + +reveal_type(A()) # N: Revealed type is "__main__.A" +reveal_type(B()) # N: Revealed type is "__main__.B" + +[case testNewReturnType3] + +# Check for invalid __new__ typing + +class A: + def __new__(cls) -> int: # E: Incompatible return type for "__new__" (returns "int", but must return a subtype of "A") + pass + +reveal_type(A()) # N: Revealed type is "__main__.A" + +[case testNewReturnType4] +from typing import TypeVar, Type + +# Check for __new__ using type vars + +TX = TypeVar('TX', bound='X') +class X: + def __new__(lol: Type[TX], x: int) -> TX: + pass +class Y(X): pass + +reveal_type(X(20)) # N: Revealed type is "__main__.X" +reveal_type(Y(20)) # N: Revealed type is "__main__.Y" + +[case testNewReturnType5] +from typing import Any, TypeVar, Generic, overload + +T = TypeVar('T') +class O(Generic[T]): + @overload + def __new__(cls) -> O[int]: + pass + @overload + def __new__(cls, x: int) -> O[str]: + pass + def __new__(cls, x: int = 0) -> O[Any]: + pass + +reveal_type(O()) # N: Revealed type is "__main__.O[builtins.int]" +reveal_type(O(10)) # N: Revealed type is "__main__.O[builtins.str]" + +[case testNewReturnType6] +from typing import Tuple, Optional + +# Check for some cases that aren't allowed + +class X: + def __new__(cls) -> Optional[Y]: # E: "__new__" must return a class instance (got "Optional[Y]") + pass +class Y: + def __new__(cls) -> Optional[int]: # E: "__new__" must return a class instance (got "Optional[int]") + pass + + +[case testNewReturnType7] +from typing import NamedTuple + +# ... test __new__ returning tuple type +class A: + def __new__(cls) -> 'B': + pass + +N = NamedTuple('N', [('x', int)]) +class B(A, N): pass + +reveal_type(A()) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.B]" +[builtins fixtures/tuple.pyi] + +[case testNewReturnType8] +from typing import TypeVar, Any + +# test type var from a different argument +TX = TypeVar('TX', bound='X') +class X: + def __new__(cls, x: TX) -> TX: # E: "__new__" must return a class instance (got "TX") + pass + +[case testNewReturnType9] +class A: + def __new__(cls) -> A: + pass + +class B(A): + pass + +reveal_type(B()) # N: Revealed type is "__main__.B" + +[case testNewReturnType10] +# https://github.com/python/mypy/issues/11398 +from typing import Type + +class MyMetaClass(type): + def __new__(cls, name, bases, attrs) -> Type['MyClass']: + pass + +class MyClass(metaclass=MyMetaClass): + pass + +[case testNewReturnType11] +# https://github.com/python/mypy/issues/11398 +class MyMetaClass(type): + def __new__(cls, name, bases, attrs) -> type: + pass + +class MyClass(metaclass=MyMetaClass): + pass + +[case testNewReturnType12] +# https://github.com/python/mypy/issues/11398 +from typing import Type + +class MyMetaClass(type): + def __new__(cls, name, bases, attrs) -> int: # E: Incompatible return type for "__new__" (returns "int", but must return a subtype of "type") + pass + +class MyClass(metaclass=MyMetaClass): + pass + + +[case testMetaclassPlaceholderNode] +from sympy.assumptions import ManagedProperties +from sympy.ops import AssocOp +reveal_type(AssocOp.x) # N: Revealed type is "sympy.basic.Basic" +reveal_type(AssocOp.y) # N: Revealed type is "builtins.int" + +[file sympy/__init__.py] + +[file sympy/assumptions.py] +from .basic import Basic +class ManagedProperties(type): + x: Basic + y: int +# The problem is with the next line, +# it creates the following order (classname, metaclass): +# 1. Basic NameExpr(ManagedProperties) +# 2. AssocOp None +# 3. ManagedProperties None +# 4. Basic NameExpr(ManagedProperties [sympy.assumptions.ManagedProperties]) +# So, `AssocOp` will still have `metaclass_type` as `None` +# and all its `mro` types will have `declared_metaclass` as `None`. +from sympy.ops import AssocOp + +[file sympy/basic.py] +from .assumptions import ManagedProperties +class Basic(metaclass=ManagedProperties): ... + +[file sympy/ops.py] +from sympy.basic import Basic +class AssocOp(Basic): ... + +[case testMetaclassSubclassSelf] +# This does not make much sense, but we must not crash: +import a +[file m.py] +from a import A # E: Module "a" has no attribute "A" +class Meta(A): pass +[file a.py] +from m import Meta +class A(metaclass=Meta): pass + +[case testMetaclassConflict] +class MyMeta1(type): ... +class MyMeta2(type): ... +class MyMeta3(type): ... +class A(metaclass=MyMeta1): ... +class B(metaclass=MyMeta2): ... +class C(metaclass=type): ... +class A1(A): ... +class E: ... + +class CorrectMeta(MyMeta1, MyMeta2): ... +class CorrectSubclass1(A1, B, E, metaclass=CorrectMeta): ... +class CorrectSubclass2(A, B, E, metaclass=CorrectMeta): ... +class CorrectSubclass3(B, A, metaclass=CorrectMeta): ... + +class ChildOfCorrectSubclass1(CorrectSubclass1): ... + +class CorrectWithType1(C, A1): ... +class CorrectWithType2(B, C): ... + +class Conflict1(A1, B, E): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class Conflict2(A, B): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class Conflict3(B, A): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +class ChildOfConflict1(Conflict3): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class ChildOfConflict2(Conflict3, metaclass=CorrectMeta): ... + +class ConflictingMeta(MyMeta1, MyMeta3): ... +class Conflict4(A1, B, E, metaclass=ConflictingMeta): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +class ChildOfCorrectButWrongMeta(CorrectSubclass1, metaclass=ConflictingMeta): # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + ... + +[case testGenericOverride] +from typing import Generic, TypeVar, Any + +T = TypeVar('T') + +class B(Generic[T]): + x: T + +class C(B): + def __init__(self) -> None: + self.x: Any + +[case testGenericOverridePreciseInvalid] +from typing import Generic, TypeVar, Any + +T = TypeVar('T') + +class B(Generic[T]): + x: T + +class C(B[str]): + def __init__(self) -> None: + self.x: int # E: Incompatible types in assignment (expression has type "int", base class "B" defined the type as "str") + +[case testGenericOverridePreciseValid] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class B(Generic[T]): + x: T + +class C(B[float]): + def __init__(self) -> None: + self.x: int # We currently allow covariant overriding. + +[case testGenericOverrideGeneric] +from typing import Generic, TypeVar, List + +T = TypeVar('T') + +class B(Generic[T]): + x: T + +class C(B[T]): + def __init__(self) -> None: + self.x: List[T] # E: Incompatible types in assignment (expression has type "List[T]", base class "B" defined the type as "T") +[builtins fixtures/list.pyi] + +[case testGenericOverrideGenericChained] +from typing import Generic, TypeVar, Tuple + +T = TypeVar('T') +S = TypeVar('S') + +class A(Generic[T]): + x: T + +class B(A[Tuple[T, S]]): ... + +class C(B[int, T]): + def __init__(self) -> None: + # TODO: error message could be better. + self.x: Tuple[str, T] # E: Incompatible types in assignment (expression has type "Tuple[str, T]", base class "A" defined the type as "Tuple[int, T]") +[builtins fixtures/tuple.pyi] + +[case testInitSubclassWrongType] +class Base: + default_name: str + + def __init_subclass__(cls, default_name: str): + super().__init_subclass__() + cls.default_name = default_name + return + +class Child(Base, default_name=5): # E: Argument "default_name" to "__init_subclass__" of "Base" has incompatible type "int"; expected "str" + pass +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassTooFewArgs] +class Base: + default_name: str + + def __init_subclass__(cls, default_name: str, **kwargs): + super().__init_subclass__() + cls.default_name = default_name + return + +class Child(Base): # E: Missing positional argument "default_name" in call to "__init_subclass__" of "Base" + pass +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassTooFewArgs2] +class Base: + default_name: str + + def __init_subclass__(cls, default_name: str, thing: int): + super().__init_subclass__() + cls.default_name = default_name + return +# TODO implement this, so that no error is raised? +d = {"default_name": "abc", "thing": 0} +class Child(Base, **d): # E: Missing positional arguments "default_name", "thing" in call to "__init_subclass__" of "Base" + pass +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassOK] +class Base: + default_name: str + thing: int + + def __init_subclass__(cls, default_name: str, thing:int, **kwargs): + super().__init_subclass__() + cls.default_name = default_name + return + +class Child(Base, thing=5, default_name=""): + pass +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassWithMetaclassOK] +class Base: + thing: int + + def __init_subclass__(cls, thing: int): + cls.thing = thing + +class Child(Base, metaclass=type, thing=0): + pass +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassWithCustomMetaclassOK] +class M(type): ... +class Child(metaclass=M, thing=0): + pass +[builtins fixtures/object_with_init_subclass.pyi] + +[case testTooManyArgsForObject] +class A(thing=5): + pass +[out] +main:1: error: Unexpected keyword argument "thing" for "__init_subclass__" of "object" +tmp/builtins.pyi:5: note: "__init_subclass__" of "object" defined here +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassWithImports] +from init_subclass.a import Base +class Child(Base, thing=5): # E: Missing positional argument "default_name" in call to "__init_subclass__" of "Base" + pass +[file init_subclass/a.py] +class Base: + default_name: str + thing: int + + def __init_subclass__(cls, default_name: str, thing:int, **kwargs): + pass +[file init_subclass/__init__.py] +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassWithImportsOK] +from init_subclass.a import MidBase +class Main(MidBase, test=True): pass +[file init_subclass/a.py] +class Base: + def __init_subclass__(cls, **kwargs) -> None: pass +class MidBase(Base): pass +[file init_subclass/__init__.py] +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassUnannotated] +class A: + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + +class B(A): + pass + +reveal_type(A.__init_subclass__) # N: Revealed type is "def (*args: Any, **kwargs: Any) -> Any" +[builtins fixtures/object_with_init_subclass.pyi] + +[case testInitSubclassUnannotatedMulti] +from typing import ClassVar, List, Type + +class A: + registered_classes: ClassVar[List[Type[A]]] = [] + def __init_subclass__(cls, *args, register=True, **kwargs): + if register: + cls.registered_classes.append(cls) + super().__init_subclass__(*args, **kwargs) + +class B(A): ... +class C(A, register=False): ... +class D(C): ... +[builtins fixtures/object_with_init_subclass.pyi] + +[case testClassMethodUnannotated] +class C: + def __new__(cls): ... + @classmethod + def meth(cls): ... + +reveal_type(C.meth) # N: Revealed type is "def () -> Any" +reveal_type(C.__new__) # N: Revealed type is "def (cls: Type[__main__.C]) -> Any" +[builtins fixtures/classmethod.pyi] + +[case testOverrideGenericSelfClassMethod] +from typing import Generic, TypeVar, Type, List + +T = TypeVar('T', bound='A') + +class A: + @classmethod + def meth(cls: Type[T]) -> List[T]: ... + +class B(A): + @classmethod + def meth(cls: Type[T]) -> List[T]: ... + +[builtins fixtures/isinstancelist.pyi] + +[case testCheckUntypedDefsSelf1] +# flags: --check-untyped-defs + +from typing import Generic, TypeVar +T = TypeVar('T') + +class Desc: + def __get__(self, x, y): + # type: (...) -> bool + pass + +class Foo: + y = Desc() + + def __init__(self): + self.x = 0 + + def foo(self): + reveal_type(self.x) # N: Revealed type is "builtins.int" + reveal_type(self.y) # N: Revealed type is "builtins.bool" + self.bar() + self.baz() # E: "Foo" has no attribute "baz" + + @classmethod + def bar(cls): + cls.baz() # E: "Type[Foo]" has no attribute "baz" + +class C(Generic[T]): + x: T + def meth(self): + self.x + 1 # E: Unsupported left operand type for + ("T") +[builtins fixtures/classmethod.pyi] + +[case testCheckUntypedDefsSelf2] +# flags: --check-untyped-defs + +class Foo: + def __init__(self): + self.x = None + self.y = [] + +reveal_type(Foo().x) # N: Revealed type is "Union[Any, None]" +reveal_type(Foo().y) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testCheckUntypedDefsSelf3] +# flags: --check-untyped-defs + +class Foo: + def bad(): # E: Method must have at least one argument. Did you forget the "self" argument? + self.x = 0 # E: Name "self" is not defined + +[case testTypeAfterAttributeAccessWithDisallowAnyExpr] +# flags: --disallow-any-expr + +def access_before_declaration(self) -> None: + obj = Foo('bar') + obj.value + x = 1 + + reveal_type(x) # N: Revealed type is "builtins.int" + x = x + 1 + +class Foo: + def __init__(self, value: str) -> None: + self.value = value + +def access_after_declaration(self) -> None: + obj = Foo('bar') + obj.value + x = 1 + + reveal_type(x) # N: Revealed type is "builtins.int" + x = x + 1 + +[case testIsSubClassNarrowDownTypesOfTypeVariables] +from typing import Type, TypeVar, Generic + +class Base: + field: int = 42 + +TypeT = TypeVar("TypeT", bound=type) + +TypeT1 = TypeVar("TypeT1", bound=Type[Base]) + +class C1: + def method(self, other: type) -> int: + if issubclass(other, Base): + reveal_type(other) # N: Revealed type is "Type[__main__.Base]" + return other.field + return 0 + +class C2(Generic[TypeT]): + def method(self, other: TypeT) -> int: + if issubclass(other, Base): + reveal_type(other) # N: Revealed type is "Type[__main__.Base]" + return other.field + return 0 + +class C3(Generic[TypeT1]): + def method(self, other: TypeT1) -> int: + if issubclass(other, Base): + reveal_type(other) # N: Revealed type is "TypeT1`1" + return other.field + return 0 + +[builtins fixtures/isinstancelist.pyi] + +[case testPropertyWithExtraMethod] +def dec(f): + return f + +class A: + @property + def x(self): ... + @x.setter + def x(self, value) -> None: ... + def x(self) -> None: ... # E: Unexpected definition for property "x" + + @property + def y(self) -> int: ... + @y.setter + def y(self, value: int) -> None: ... + @dec # E: Only supported top decorator is @y.setter + def y(self) -> None: ... + +reveal_type(A().y) # N: Revealed type is "builtins.int" +[builtins fixtures/property.pyi] + +[case testEnclosingScopeLambdaNoCrash] +class C: + x = lambda x: x.y.g() + +[case testEnclosingScopeLambdaNoCrashExplicit] +from typing import Callable +class C: + x: Callable[[C], int] = lambda x: x.y.g() # E: "C" has no attribute "y" + +[case testOpWithInheritedFromAny-xfail] +from typing import Any +C: Any +class D(C): + pass + +class D1(C): + def __add__(self, rhs: float) -> D1: + return self + +reveal_type(0.5 + C) # N: Revealed type is "Any" + +reveal_type(0.5 + D()) # N: Revealed type is "Any" +reveal_type(D() + 0.5) # N: Revealed type is "Any" +reveal_type("str" + D()) # N: Revealed type is "builtins.str" +reveal_type(D() + "str") # N: Revealed type is "Any" + + +reveal_type(0.5 + D1()) # N: Revealed type is "Any" +reveal_type(D1() + 0.5) # N: Revealed type is "__main__.D1" +[builtins fixtures/primitives.pyi] + +[case testRefMethodWithDecorator] +from typing import Type, final + +class A: + pass + +class B: + @staticmethod + def A() -> Type[A]: ... + @staticmethod + def B() -> Type[A]: # E: Function "__main__.B.A" is not valid as a type \ + # N: Perhaps you need "Callable[...]" or a callback protocol? + return A + +class C: + @final + @staticmethod + def A() -> Type[A]: + return A + +[builtins fixtures/staticmethod.pyi] + +[case testRefMethodWithOverloadDecorator] +from typing import Type, overload + +class A: + pass + +class B: + @classmethod + @overload + def A(cls, x: int) -> Type[A]: ... + @classmethod + @overload + def A(cls, x: str) -> Type[A]: ... + @classmethod + def A(cls, x: object) -> Type[A]: ... + def B(cls, x: int) -> Type[A]: ... # E: Function "__main__.B.A" is not valid as a type \ + # N: Perhaps you need "Callable[...]" or a callback protocol? + +[builtins fixtures/classmethod.pyi] + +[case testFinalClassWithAbstractAttributes] +from abc import abstractmethod, ABCMeta +from typing import final + +@final +class A(metaclass=ABCMeta): # E: Final class __main__.A has abstract attributes "bar", "foo" + @abstractmethod + def foo(self): + pass + + @property + @abstractmethod + def bar(self): + pass + +[builtins fixtures/property.pyi] + +[case testFinalClassWithoutABCMeta] +from abc import abstractmethod +from typing import final + +@final +class A(): # E: Final class __main__.A has abstract attributes "bar", "foo" + @abstractmethod + def foo(self): + pass + + @property + @abstractmethod + def bar(self): + pass + +[builtins fixtures/property.pyi] + +[case testFinalClassInheritedAbstractAttributes] +from abc import abstractmethod, ABCMeta +from typing import final + +class A(metaclass=ABCMeta): + @abstractmethod + def foo(self): + pass + +@final +class B(A): # E: Final class __main__.B has abstract attributes "foo" + pass + +[case testUndefinedBaseclassInNestedClass] +class C: + class C1(XX): pass # E: Name "XX" is not defined + +[case testArgsKwargsInheritance] +from typing import Any + +class A(object): + def f(self, *args: Any, **kwargs: Any) -> int: ... + +class B(A): + def f(self, x: int) -> int: ... +[builtins fixtures/dict.pyi] + +[case testClassScopeImports] +class Foo: + from mod import plain_function # E: Unsupported class scoped import + from mod import plain_var + +reveal_type(Foo.plain_function) # N: Revealed type is "Any" +reveal_type(Foo().plain_function) # N: Revealed type is "Any" + +reveal_type(Foo.plain_var) # N: Revealed type is "builtins.int" +reveal_type(Foo().plain_var) # N: Revealed type is "builtins.int" + +[file mod.py] +def plain_function(x: int, y: int) -> int: ... +plain_var: int + +[case testClassScopeImportModule] +class Foo: + import mod + +reveal_type(Foo.mod) # N: Revealed type is "builtins.object" +reveal_type(Foo.mod.foo) # N: Revealed type is "builtins.int" +[file mod.py] +foo: int + +[case testClassScopeImportAlias] +class Foo: + from mod import function # E: Unsupported class scoped import + foo = function + + from mod import var1 + bar = var1 + + from mod import var2 + baz = var2 + + from mod import var3 + qux = var3 + +reveal_type(Foo.foo) # N: Revealed type is "Any" +reveal_type(Foo.function) # N: Revealed type is "Any" + +reveal_type(Foo.bar) # N: Revealed type is "builtins.int" +reveal_type(Foo.var1) # N: Revealed type is "builtins.int" + +reveal_type(Foo.baz) # N: Revealed type is "mod.C" +reveal_type(Foo.var2) # N: Revealed type is "mod.C" + +reveal_type(Foo.qux) # N: Revealed type is "builtins.int" +reveal_type(Foo.var3) # N: Revealed type is "builtins.int" + +[file mod.py] +def function(x: int, y: int) -> int: ... +var1: int + +class C: ... +var2: C + +A = int +var3: A + + +[case testClassScopeImportModuleStar] +class Foo: + from mod import * # E: Unsupported class scoped import + +reveal_type(Foo.foo) # N: Revealed type is "builtins.int" +reveal_type(Foo.bar) # N: Revealed type is "Any" +reveal_type(Foo.baz) # E: "Type[Foo]" has no attribute "baz" \ + # N: Revealed type is "Any" + +[file mod.py] +foo: int +def bar(x: int) -> int: ... + +[case testClassScopeImportFunctionNested] +class Foo: + class Bar: + from mod import baz # E: Unsupported class scoped import + +reveal_type(Foo.Bar.baz) # N: Revealed type is "Any" +reveal_type(Foo.Bar().baz) # N: Revealed type is "Any" + +[file mod.py] +def baz(x: int) -> int: ... + +[case testClassScopeImportUndefined] +class Foo: + from unknown import foo # E: Cannot find implementation or library stub for module named "unknown" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +reveal_type(Foo.foo) # N: Revealed type is "Any" +reveal_type(Foo().foo) # N: Revealed type is "Any" + +[case testClassScopeImportWithFollowImports] +# flags: --follow-imports=skip +class Foo: + from mod import foo + +reveal_type(Foo().foo) # N: Revealed type is "Any" +[file mod.py] +def foo(x: int, y: int) -> int: ... + +[case testClassScopeImportVarious] +class Foo: + from mod1 import foo # E: Unsupported class scoped import + from mod2 import foo + + from mod1 import meth1 # E: Unsupported class scoped import + def meth1(self, a: str) -> str: ... # E: Name "meth1" already defined on line 5 + + def meth2(self, a: str) -> str: ... + from mod1 import meth2 # E: Incompatible import of "meth2" (imported name has type "Callable[[int], int]", local name has type "Callable[[Foo, str], str]") + +class Bar: + from mod1 import foo # E: Unsupported class scoped import + +import mod1 +reveal_type(Foo.foo) # N: Revealed type is "Any" +reveal_type(Bar.foo) # N: Revealed type is "Any" +reveal_type(mod1.foo) # N: Revealed type is "def (x: builtins.int, y: builtins.int) -> builtins.int" + +[file mod1.py] +def foo(x: int, y: int) -> int: ... +def meth1(x: int) -> int: ... +def meth2(x: int) -> int: ... +[file mod2.py] +def foo(z: str) -> int: ... + + +[case testClassScopeImportWithError] +class Foo: + from mod import meth1 # E: Unsupported class scoped import + from mod import meth2 # E: Unsupported class scoped import + from mod import T + +reveal_type(Foo.T) # E: Type variable "Foo.T" cannot be used as an expression \ + # N: Revealed type is "Any" + +[file mod.pyi] +from typing import Any, TypeVar, overload + +@overload +def meth1(self: Any, y: int) -> int: ... +@overload +def meth1(self: Any, y: str) -> str: ... + +T = TypeVar("T") +def meth2(self: Any, y: T) -> T: ... + +[case testNewAndInitNoReturn] +from typing import NoReturn + +class A: + def __new__(cls) -> NoReturn: ... + +class B: + def __init__(self) -> NoReturn: ... + +class C: + def __new__(cls) -> "C": ... + def __init__(self) -> NoReturn: ... + +class D: + def __new__(cls) -> NoReturn: ... + def __init__(self) -> NoReturn: ... + +if object(): + reveal_type(A()) # N: Revealed type is "Never" +if object(): + reveal_type(B()) # N: Revealed type is "Never" +if object(): + reveal_type(C()) # N: Revealed type is "Never" +if object(): + reveal_type(D()) # N: Revealed type is "Never" + +[case testOverloadedNewAndInitNoReturn] +from typing import NoReturn, overload + +class A: + @overload + def __new__(cls) -> NoReturn: ... + @overload + def __new__(cls, a: int) -> "A": ... + def __new__(cls, a: int = ...) -> "A": ... + +class B: + @overload + def __init__(self) -> NoReturn: ... + @overload + def __init__(self, a: int) -> None: ... + def __init__(self, a: int = ...) -> None: ... + +class C: + def __new__(cls, a: int = ...) -> "C": ... + @overload + def __init__(self) -> NoReturn: ... + @overload + def __init__(self, a: int) -> None: ... + def __init__(self, a: int = ...) -> None: ... + +class D: + @overload + def __new__(cls) -> NoReturn: ... + @overload + def __new__(cls, a: int) -> "D": ... + def __new__(cls, a: int = ...) -> "D": ... + @overload + def __init__(self) -> NoReturn: ... + @overload + def __init__(self, a: int) -> None: ... + def __init__(self, a: int = ...) -> None: ... + +if object(): + reveal_type(A()) # N: Revealed type is "Never" +reveal_type(A(1)) # N: Revealed type is "__main__.A" + +if object(): + reveal_type(B()) # N: Revealed type is "Never" +reveal_type(B(1)) # N: Revealed type is "__main__.B" + +if object(): + reveal_type(C()) # N: Revealed type is "Never" +reveal_type(C(1)) # N: Revealed type is "__main__.C" + +if object(): + reveal_type(D()) # N: Revealed type is "Never" +reveal_type(D(1)) # N: Revealed type is "__main__.D" + +[case testClassScopeImportWithWrapperAndError] +class Foo: + from mod import foo # E: Unsupported class scoped import + +[file mod.py] +from typing import Any, Callable, TypeVar + +FuncT = TypeVar("FuncT", bound=Callable[..., Any]) +def identity_wrapper(func: FuncT) -> FuncT: + return func + +@identity_wrapper +def foo(self: Any) -> str: + return "" + +[case testParentClassWithTypeAliasAndSubclassWithMethod] +from typing import Any, Callable, TypeVar + +class Parent: + foo = Callable[..., int] + class bar: + pass + import typing as baz + foobar = TypeVar("foobar") + +class Child(Parent): + def foo(self, val: int) -> int: # E: Signature of "foo" incompatible with supertype "Parent" \ + # N: Superclass: \ + # N: None \ + # N: Subclass: \ + # N: def foo(self, val: int) -> int + return val + def bar(self, val: str) -> str: # E: Signature of "bar" incompatible with supertype "Parent" \ + # N: Superclass: \ + # N: None \ + # N: Subclass: \ + # N: def bar(self, val: str) -> str + return val + def baz(self, val: float) -> float: # E: Signature of "baz" incompatible with supertype "Parent" \ + # N: Superclass: \ + # N: None \ + # N: Subclass: \ + # N: def baz(self, val: float) -> float + return val + def foobar(self) -> bool: # E: Signature of "foobar" incompatible with supertype "Parent" \ + # N: Superclass: \ + # N: None \ + # N: Subclass: \ + # N: def foobar(self) -> bool + return False + +x: Parent.foo = lambda: 5 +y: Parent.bar = Parent.bar() +z: Parent.baz.Any = 1 +child = Child() +a: int = child.foo(1) +b: str = child.bar("abc") +c: float = child.baz(3.4) +d: bool = child.foobar() + +[case testGenericTupleTypeCreation] +from typing import Generic, Tuple, TypeVar + +T = TypeVar("T") +S = TypeVar("S") +class C(Tuple[T, S]): + def __init__(self, x: T, y: S) -> None: ... + def foo(self, arg: T) -> S: ... + +cis: C[int, str] +reveal_type(cis) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C[builtins.int, builtins.str]]" +cii = C(0, 1) +reveal_type(cii) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.C[builtins.int, builtins.int]]" +reveal_type(cis.foo) # N: Revealed type is "def (arg: builtins.int) -> builtins.str" +[builtins fixtures/tuple.pyi] + +[case testGenericTupleTypeSubclassing] +from typing import Generic, Tuple, TypeVar, List + +T = TypeVar("T") +class C(Tuple[T, T]): ... +class D(C[List[T]]): ... + +di: D[int] +reveal_type(di) # N: Revealed type is "Tuple[builtins.list[builtins.int], builtins.list[builtins.int], fallback=__main__.D[builtins.int]]" +[builtins fixtures/tuple.pyi] + +[case testOverrideAttrWithSettableProperty] +class Foo: + def __init__(self) -> None: + self.x = 42 + +class Bar(Foo): + @property + def x(self) -> int: ... + @x.setter + def x(self, value: int) -> None: ... +[builtins fixtures/property.pyi] + +[case testOverrideAttrWithSettablePropertyAnnotation] +class Foo: + x: int + +class Bar(Foo): + @property + def x(self) -> int: ... + @x.setter + def x(self, value: int) -> None: ... +[builtins fixtures/property.pyi] + +[case testOverrideMethodProperty] +class B: + def foo(self) -> int: + ... +class C(B): + @property + def foo(self) -> int: # E: Signature of "foo" incompatible with supertype "B" \ + # N: Superclass: \ + # N: def foo(self) -> int \ + # N: Subclass: \ + # N: int + ... +[builtins fixtures/property.pyi] + +[case testOverridePropertyMethod] +class B: + @property + def foo(self) -> int: + ... +class C(B): + def foo(self) -> int: # E: Signature of "foo" incompatible with supertype "B" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: def foo(self) -> int + ... +[builtins fixtures/property.pyi] + +[case testAllowArgumentAsBaseClass] +from typing import Any, Type + +def e(b) -> None: + class D(b): ... + +def f(b: Any) -> None: + class D(b): ... + +def g(b: Type[Any]) -> None: + class D(b): ... + +def h(b: type) -> None: + class D(b): ... + +[case testNoCrashOnSelfWithForwardRefGenericClass] +from typing import Generic, Sequence, TypeVar, Self + +_T = TypeVar('_T', bound="Foo") + +class Foo: + foo: int + +class Element(Generic[_T]): + elements: Sequence[Self] + +class Bar(Foo): ... +e: Element[Bar] +reveal_type(e.elements) # N: Revealed type is "typing.Sequence[__main__.Element[__main__.Bar]]" + +[case testIterableUnpackingWithGetAttr] +from typing import Union, Tuple + +class C: + def __getattr__(self, name): + pass + +class D: + def f(self) -> C: + return C() + + def g(self) -> None: + # iter(x) looks up `__iter__` on the type of x rather than x itself, + # so this is correct behaviour. + # Instances of C should not be treated as being iterable, + # despite having a __getattr__ method + # that could allow for arbitrary attributes to be accessed on instances, + # since `type(C()).__iter__` still raises AttributeError at runtime, + # and that's what matters. + a, b = self.f() # E: "C" has no attribute "__iter__" (not iterable) +[builtins fixtures/tuple.pyi] + +[case testUsingNumbersType] +from numbers import Number, Complex, Real, Rational, Integral + +def f1(x: Number) -> None: pass +f1(1) # E: Argument 1 to "f1" has incompatible type "int"; expected "Number" \ + # N: Types from "numbers" aren't supported for static type checking \ + # N: See https://peps.python.org/pep-0484/#the-numeric-tower \ + # N: Consider using a protocol instead, such as typing.SupportsFloat + +def f2(x: Complex) -> None: pass +f2(1) # E: Argument 1 to "f2" has incompatible type "int"; expected "Complex" \ + # N: Types from "numbers" aren't supported for static type checking \ + # N: See https://peps.python.org/pep-0484/#the-numeric-tower \ + # N: Consider using a protocol instead, such as typing.SupportsFloat + +def f3(x: Real) -> None: pass +f3(1) # E: Argument 1 to "f3" has incompatible type "int"; expected "Real" \ + # N: Types from "numbers" aren't supported for static type checking \ + # N: See https://peps.python.org/pep-0484/#the-numeric-tower \ + # N: Consider using a protocol instead, such as typing.SupportsFloat + +def f4(x: Rational) -> None: pass +f4(1) # E: Argument 1 to "f4" has incompatible type "int"; expected "Rational" \ + # N: Types from "numbers" aren't supported for static type checking \ + # N: See https://peps.python.org/pep-0484/#the-numeric-tower \ + # N: Consider using a protocol instead, such as typing.SupportsFloat + +def f5(x: Integral) -> None: pass +f5(1) # E: Argument 1 to "f5" has incompatible type "int"; expected "Integral" \ + # N: Types from "numbers" aren't supported for static type checking \ + # N: See https://peps.python.org/pep-0484/#the-numeric-tower \ + # N: Consider using a protocol instead, such as typing.SupportsFloat + +[case testImplicitClassScopedNames] +class C: + reveal_type(__module__) # N: Revealed type is "builtins.str" + reveal_type(__qualname__) # N: Revealed type is "builtins.str" + def f(self) -> None: + __module__ # E: Name "__module__" is not defined + __qualname__ # E: Name "__qualname__" is not defined diff --git a/test-data/unit/check-classvar.test b/test-data/unit/check-classvar.test new file mode 100644 index 000000000000..1e87e441dea2 --- /dev/null +++ b/test-data/unit/check-classvar.test @@ -0,0 +1,336 @@ +[case testAssignmentOnClass] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +A.x = 2 + +[case testAssignmentOnInstance] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +A().x = 2 +[out] +main:4: error: Cannot assign to class variable "x" via instance + +[case testAssignmentOnSubclassInstance] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +class B(A): + pass +B().x = 2 +[out] +main:6: error: Cannot assign to class variable "x" via instance + +[case testOverrideOnSelf] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] + def __init__(self) -> None: + self.x = 0 +[out] +main:5: error: Cannot assign to class variable "x" via instance + +[case testOverrideOnSelfInSubclass] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] +class B(A): + def __init__(self) -> None: + self.x = 0 +[out] +main:6: error: Cannot assign to class variable "x" via instance + +[case testReadingFromInstance] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +A().x +reveal_type(A().x) +[out] +main:5: note: Revealed type is "builtins.int" + +[case testReadingFromSelf] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] + def __init__(self) -> None: + reveal_type(self.x) +[out] +main:5: note: Revealed type is "builtins.int" + +[case testTypecheckSimple] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +y = A.x # type: int + +[case testTypecheckWithUserType] +from typing import ClassVar +class A: + pass +class B: + x = A() # type: ClassVar[A] + +[case testTypeCheckOnAssignment] +from typing import ClassVar +class A: + pass +class B: + pass +class C: + x = None # type: ClassVar[A] +C.x = B() +[out] +main:8: error: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testTypeCheckWithOverridden] +from typing import ClassVar +class A: + pass +class B(A): + pass +class C: + x = A() # type: ClassVar[A] +C.x = B() + +[case testRevealType] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] +reveal_type(A.x) +[out] +main:4: note: Revealed type is "builtins.int" + +[case testInfer] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +y = A.x +reveal_type(y) +[out] +main:5: note: Revealed type is "builtins.int" + +[case testAssignmentOnUnion] +from typing import ClassVar, Union +class A: + x = None # type: int +class B: + x = None # type: ClassVar[int] +c = A() # type: Union[A, B] +c.x = 1 +[out] +main:7: error: Cannot assign to class variable "x" via instance + +[case testAssignmentOnInstanceFromType] +from typing import ClassVar, Type +class A: + x = None # type: ClassVar[int] +def f(a: Type[A]) -> None: + a().x = 0 +[out] +main:5: error: Cannot assign to class variable "x" via instance + +[case testAssignmentOnInstanceFromSubclassType] +from typing import ClassVar, Type +class A: + x = None # type: ClassVar[int] +class B(A): + pass +def f(b: Type[B]) -> None: + b().x = 0 +[out] +main:7: error: Cannot assign to class variable "x" via instance + +[case testClassVarWithList] +from typing import ClassVar, List +class A: + x = None # type: ClassVar[List[int]] +A.x = ['a'] +A().x.append(1) +A().x.append('') +[builtins fixtures/list.pyi] +[out] +main:4: error: List item 0 has incompatible type "str"; expected "int" +main:6: error: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" + +[case testClassVarWithUnion] +from typing import ClassVar, Union +class A: + x = None # type: ClassVar[Union[int, str]] +class B: + pass +A.x = 0 +A.x = 'a' +A.x = B() +reveal_type(A().x) +[out] +main:8: error: Incompatible types in assignment (expression has type "B", variable has type "Union[int, str]") +main:9: note: Revealed type is "Union[builtins.int, builtins.str]" + +[case testOverrideWithNarrowedUnion] +from typing import ClassVar, Union +class A: pass +class B: pass +class C: pass +class D: + x = None # type: ClassVar[Union[A, B, C]] +class E(D): + x = None # type: ClassVar[Union[A, B]] + +[case testOverrideWithExtendedUnion] +from typing import ClassVar, Union +class A: pass +class B: pass +class C: pass +class D: + x = None # type: ClassVar[Union[A, B]] +class E(D): + x = None # type: ClassVar[Union[A, B, C]] +[out] +main:8: error: Incompatible types in assignment (expression has type "Union[A, B, C]", base class "D" defined the type as "Union[A, B]") + +[case testAssignmentToCallableRet] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] +def f() -> A: + return A() +f().x = 0 +[out] +main:6: error: Cannot assign to class variable "x" via instance + +[case testOverrideWithIncomatibleType] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] +class B(A): + x = None # type: ClassVar[str] +[out] +main:5: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") + +[case testOverrideWithNormalAttribute] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +class B(A): + x = 2 # type: int +[out] +main:5: error: Cannot override class variable (previously declared on base class "A") with instance variable + +[case testOverrideWithAttributeWithClassVar] +from typing import ClassVar +class A: + x = 1 # type: int +class B(A): + x = 2 # type: ClassVar[int] +[out] +main:5: error: Cannot override instance variable (previously declared on base class "A") with class variable + +[case testOverrideClassVarManyBases] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +class B: + x = 2 # type: int +class C(A, B): + x = 3 # type: ClassVar[int] +[out] +main:7: error: Cannot override instance variable (previously declared on base class "B") with class variable + +[case testOverrideClassVarWithClassVar] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +class B(A): + x = 2 # type: ClassVar[int] + +[case testOverrideClassVarWithImplicitClassVar] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +class B(A): + x = 2 + +[case testOverrideClassVarWithImplicitThenExplicit] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +class B(A): + x = 2 +class C(B): + x = 3 # type: ClassVar[int] + +[case testOverrideOnABCSubclass] +from abc import ABCMeta +from typing import ClassVar +class A(metaclass=ABCMeta): + x = None # type: ClassVar[int] +class B(A): + x = 0 # type: ClassVar[int] + +[case testAcrossModules] +import m +reveal_type(m.A().x) +m.A().x = 0 +[file m.py] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] +[out] +main:2: note: Revealed type is "builtins.int" +main:3: error: Cannot assign to class variable "x" via instance + +[case testClassVarWithGeneric] +from typing import ClassVar, Generic, TypeVar +T = TypeVar('T') +class A(Generic[T]): + x: ClassVar[T] # E: ClassVar cannot contain type variables + @classmethod + def foo(cls) -> T: + return cls.x # OK + +A.x # E: Access to generic class variables is ambiguous +A.x = 1 # E: Access to generic class variables is ambiguous +A[int].x # E: Access to generic class variables is ambiguous + +class Bad(A[int]): + pass +Bad.x # E: Access to generic class variables is ambiguous + +class Good(A[int]): + x = 42 +reveal_type(Good.x) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testClassVarWithNestedGeneric] +from typing import ClassVar, Generic, Tuple, TypeVar, Union, Type +T = TypeVar('T') +U = TypeVar('U') +class A(Generic[T, U]): + x: ClassVar[Union[T, Tuple[U, Type[U]]]] # E: ClassVar cannot contain type variables + @classmethod + def foo(cls) -> Union[T, Tuple[U, Type[U]]]: + return cls.x # OK + +A.x # E: Access to generic class variables is ambiguous +A.x = 1 # E: Access to generic class variables is ambiguous +A[int, str].x # E: Access to generic class variables is ambiguous + +class Bad(A[int, str]): + pass +Bad.x # E: Access to generic class variables is ambiguous + +class Good(A[int, str]): + x = 42 +reveal_type(Good.x) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testSuggestClassVarOnTooFewArgumentsMethod] +from typing import Callable + +class C: + foo: Callable[[C], int] +c:C +c.foo() # E: Too few arguments \ + # N: "foo" is considered instance variable, to make it class variable use ClassVar[...] diff --git a/test-data/unit/check-columns.test b/test-data/unit/check-columns.test new file mode 100644 index 000000000000..44524b9df943 --- /dev/null +++ b/test-data/unit/check-columns.test @@ -0,0 +1,390 @@ +# Test column numbers in messages. --show-column-numbers is enabled implicitly by test runner. + +[case testColumnsSyntaxError] +f() +1 + +[out] +main:2:5: error: invalid syntax + +[case testColumnsNestedFunctions] +import typing +def f() -> 'A': + def g() -> 'B': + return A() # E:16: Incompatible return value type (got "A", expected "B") + return B() # E:12: Incompatible return value type (got "B", expected "A") +class A: pass +class B: pass + +[case testColumnsMethodDefaultArgumentsAndSignatureAsComment] +import typing +class A: + def f(self, x = 1, y = 'hello'): # type: (int, str) -> str + pass +A().f() +A().f(1) +A().f('') # E:7: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +A().f(1, 1) # E:10: Argument 2 to "f" of "A" has incompatible type "int"; expected "str" +(A().f(1, 'hello', 'hi')) # E:2: Too many arguments for "f" of "A" + +[case testColumnsInvalidArgumentType] +def f(x: int, y: str) -> None: ... +def g(*x: int) -> None: pass +def h(**x: int) -> None: pass +def ff(x: int) -> None: pass + +class A: + x: str + def __neg__(self) -> str: pass + def __add__(self, other: int) -> str: pass + def __lt__(self, other: int) -> str: pass + +f( + y=0, x=0) # E:4: Argument "y" to "f" has incompatible type "int"; expected "str" +f(x=0, + y=None) # E:6: Argument "y" to "f" has incompatible type "None"; expected "str" +g(1, '', 2) # E:6: Argument 2 to "g" has incompatible type "str"; expected "int" +aaa: str +h(x=1, y=aaa, z=2) # E:10: Argument "y" to "h" has incompatible type "str"; expected "int" +a: A +ff(a.x) # E:4: Argument 1 to "ff" has incompatible type "str"; expected "int" +ff([1]) # E:4: Argument 1 to "ff" has incompatible type "List[int]"; expected "int" +# TODO: Different column in Python 3.8+ +#ff([1 for x in [1]]) # Argument 1 to "ff" has incompatible type "List[int]"; expected "int" +ff({1: 2}) # E:4: Argument 1 to "ff" has incompatible type "Dict[int, int]"; expected "int" +ff(1.1) # E:4: Argument 1 to "ff" has incompatible type "float"; expected "int" +# TODO: Different column in Python 3.8+ +#ff( ( 1, 1)) # Argument 1 to "ff" has incompatible type "Tuple[int, int]"; expected "int" +ff(-a) # E:4: Argument 1 to "ff" has incompatible type "str"; expected "int" +ff(a + 1) # E:4: Argument 1 to "ff" has incompatible type "str"; expected "int" +ff(a < 1) # E:4: Argument 1 to "ff" has incompatible type "str"; expected "int" +ff([''][0]) # E:4: Argument 1 to "ff" has incompatible type "str"; expected "int" + +class B(A): + def f(self) -> None: + ff(super().__neg__()) # E:12: Argument 1 to "ff" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testColumnsInvalidArgumentTypeVarArgs] +def f(*x: int) -> None: pass +def g(**x: int) -> None: pass + +a = [''] +f(*a) # E:4: Argument 1 to "f" has incompatible type "*List[str]"; expected "int" +b = {'x': 'y'} +g(**b) # E:5: Argument 1 to "g" has incompatible type "**Dict[str, str]"; expected "int" +[builtins fixtures/dict.pyi] + +[case testColumnsMultipleStatementsPerLine] +x = 15 +y = 'hello' +if int(): + x = 2; y = x; y += 1 +[builtins fixtures/primitives.pyi] +[out] +main:4:16: error: Incompatible types in assignment (expression has type "int", variable has type "str") +main:4:24: error: Unsupported operand types for + ("str" and "int") + +[case testColumnsAssignment] +class A: + x = 0 + +A().x = '' # E:9: Incompatible types in assignment (expression has type "str", variable has type "int") +a = [0] +a[0] = '' # E:8: Incompatible types in assignment (expression has type "str", target has type "int") +b = 0 +c = 0 +b, c = 0, '' # E:11: Incompatible types in assignment (expression has type "str", variable has type "int") +b, c = '', 0 # E:8: Incompatible types in assignment (expression has type "str", variable has type "int") + +t = 0, '' +b, c = t # E:8: Incompatible types in assignment (expression has type "str", variable has type "int") + +class B(A): + x = '' # E:9: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") +[builtins fixtures/list.pyi] + +[case testColumnsAttributeIncompatibleWithBaseClassUsingAnnotation] +class A: + x: str + +class B(A): + x: int # E:5: Incompatible types in assignment (expression has type "int", base class "A" defined the type as "str") + +[case testColumnsSimpleIsinstance] +import typing +def f(x: object, n: int, s: str) -> None: + if int(): + n = x # E:13: Incompatible types in assignment (expression has type "object", variable has type "int") + if isinstance(x, int): + n = x + s = x # E:17: Incompatible types in assignment (expression has type "int", variable has type "str") + n = x # E:13: Incompatible types in assignment (expression has type "object", variable has type "int") +[builtins fixtures/isinstance.pyi] + +[case testColumnHasNoAttribute] +import m +if int(): + from m import foobaz # E:5: Module "m" has no attribute "foobaz"; maybe "foobar"? +1 .x # E:1: "int" has no attribute "x" +(m.foobaz()) # E:2: Module has no attribute "foobaz"; maybe "foobar"? + +[file m.py] +def foobar(): pass + +[builtins fixtures/module.pyi] + +[case testColumnUnexpectedOrMissingKeywordArg] +def f(): pass # N:1: "f" defined here +# TODO: Point to "x" instead +(f(x=1)) # E:2: Unexpected keyword argument "x" for "f" +def g(*, x: int) -> None: pass +(g()) # E:2: Missing named argument "x" for "g" + +[case testColumnDefinedHere] +class A: pass +if int(): + def f(a: 'A') -> None: pass # N:5: "f" defined here + (f(b=object())) # E:6: Unexpected keyword argument "b" for "f" + +[case testColumnInvalidType] + +from typing import Iterable + +bad = 0 + +def f(x: bad): # E:10: Variable "__main__.bad" is not valid as a type \ + # N:10: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + y: bad # E:8: Variable "__main__.bad" is not valid as a type \ + # N:8: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +if int(): + def g(x): # E:5: Variable "__main__.bad" is not valid as a type \ + # N:5: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + # type: (bad) -> None + y = 0 # type: bad # E:9: Variable "__main__.bad" is not valid as a type \ + # N:9: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +z: Iterable[bad] # E:13: Variable "__main__.bad" is not valid as a type \ + # N:13: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +h: bad[int] # E:4: Variable "__main__.bad" is not valid as a type \ + # N:4: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testColumnFunctionMissingTypeAnnotation] +# flags: --disallow-untyped-defs +if int(): + def f(x: int): # E:5: Function is missing a return type annotation + pass + + def g(x): # E:5: Function is missing a type annotation + pass + +[case testColumnNameIsNotDefined] +((x)) # E:3: Name "x" is not defined + +[case testColumnNeedTypeAnnotation] +if 1: + x = [] # E:5: Need type annotation for "x" (hint: "x: List[] = ...") +[builtins fixtures/list.pyi] + +[case testColumnCallToUntypedFunction] +# flags: --disallow-untyped-calls +def f() -> None: + (g(1)) # E:6: Call to untyped function "g" in typed context + +def g(x): + pass + +[case testColumnInvalidArguments] +def f(x, y): pass +(f()) # E:2: Missing positional arguments "x", "y" in call to "f" +(f(y=1)) # E:2: Missing positional argument "x" in call to "f" + +[case testColumnListOrDictItemHasIncompatibleType] +from typing import List, Dict +x: List[int] = [ + 'x', # E:5: List item 0 has incompatible type "str"; expected "int" + 1.1] # E:7: List item 1 has incompatible type "float"; expected "int" +y: Dict[int, int] = { + 'x': 1 # E:5: Dict entry 0 has incompatible type "str": "int"; expected "int": "int" +} +[builtins fixtures/dict.pyi] + +[case testColumnCannotDetermineType] +(x) # E:2: Cannot determine type of "x" # E:2: Name "x" is used before definition +x = None + +[case testColumnInvalidIndexing] +from typing import List +([1]['']) # E:6: Invalid index type "str" for "List[int]"; expected type "int" +(1[1]) # E:2: Value of type "int" is not indexable +def f() -> None: + 1[1] = 1 # E:5: Unsupported target for indexed assignment ("int") +[builtins fixtures/list.pyi] + +[case testColumnTypedDict] +from typing import TypedDict +class D(TypedDict): + x: int +t: D = {'x': + 'y'} # E:5: Incompatible types (expression has type "str", TypedDict item "x" has type "int") + +if int(): + del t['y'] # E:5: TypedDict "D" has no key "y" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testColumnSignatureIncompatibleWithSuperType] +class A: + def f(self, x: int) -> None: pass +class B(A): + def f(self, x: str) -> None: pass # E:17: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N:17: This violates the Liskov substitution principle \ + # N:17: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +class C(A): + def f(self, x: int) -> int: pass # E:5: Return type "int" of "f" incompatible with return type "None" in supertype "A" +class D(A): + def f(self) -> None: pass # E:5: Signature of "f" incompatible with supertype "A" \ + # N:5: Superclass: \ + # N:5: def f(self, x: int) -> None \ + # N:5: Subclass: \ + # N:5: def f(self) -> None + +[case testColumnMissingTypeParameters] +# flags: --disallow-any-generics +from typing import List, Callable +def f(x: List) -> None: pass # E:10: Missing type parameters for generic type "List" +def g(x: list) -> None: pass # E:10: Implicit generic "Any". Use "typing.List" and specify generic parameters +if int(): + c: Callable # E:8: Missing type parameters for generic type "Callable" +[builtins fixtures/list.pyi] + +[case testColumnIncompatibleDefault] +if int(): + def f(x: int = '') -> None: # E:20: Incompatible default for argument "x" (default has type "str", argument has type "int") + pass + +[case testColumnMissingProtocolMember] +from typing import Protocol + +class P(Protocol): + x: int + y: int + +class C: + x: int + +p: P +if int(): + p = C() # E:9: Incompatible types in assignment (expression has type "C", variable has type "P") \ + # N:9: "C" is missing following "P" protocol member: \ + # N:9: y + +[case testColumnRedundantCast] +# flags: --warn-redundant-casts +from typing import cast +y = 1 +x = cast(int, y) # E:5: Redundant cast to "int" + +[case testColumnTypeSignatureHasTooFewArguments] +if int(): + def f(x, y): # E:5: Type signature has too few arguments + # type: (int) -> None + pass + +[case testColumnRevealedType] +if int(): + reveal_type(1) # N:17: Revealed type is "Literal[1]?" + +[case testColumnNonOverlappingEqualityCheck] +# flags: --strict-equality +if 1 == '': # E:4: Non-overlapping equality check (left operand type: "Literal[1]", right operand type: "Literal['']") + pass +[builtins fixtures/bool.pyi] + +[case testColumnValueOfTypeVariableCannotBe] + +from typing import TypeVar, Generic + +T = TypeVar('T', int, str) + +class C(Generic[T]): + pass + +def f(c: C[object]) -> None: pass # E:10: Value of type variable "T" of "C" cannot be "object" +(C[object]()) # E:2: Value of type variable "T" of "C" cannot be "object" + +[case testColumnSyntaxErrorInTypeAnnotation] +if int(): + def f(x # type: int, + ): + pass +[out] +main:2:11: error: Syntax error in type annotation +main:2:11: note: Suggestion: Is there a spurious trailing comma? + +[case testColumnSyntaxErrorInTypeAnnotation2] +if int(): + # TODO: It would be better to point to the type comment + xyz = 0 # type: blurbnard blarb +[out] +main:3:5: error: Syntax error in type comment "blurbnard blarb" + +[case testColumnProperty] +class A: + @property + def x(self) -> int: pass + + @x.setter + def x(self, x: int) -> None: pass + +class B(A): + @property # E:6: Read-only property cannot override read-write property + def x(self) -> int: pass +[builtins fixtures/property.pyi] + +[case testColumnOverloaded] +from typing import overload, Any +class A: + @overload # E:6: An overloaded function outside a stub file must have an implementation + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass + +[case testColumnFunctionWithTypeVarValues] +from typing import TypeVar, List + +T = TypeVar('T', int, str) + +def g(x): pass # N:1: "g" defined here + +def f(x: T) -> T: + (x.bad) # E:6: "int" has no attribute "bad" \ + # E:6: "str" has no attribute "bad" + g(y=x) # E:5: Unexpected keyword argument "y" for "g" + y: List[int, str] # E:8: "list" expects 1 type argument, but 2 given + del 1[0] # E:5: "int" has no attribute "__delitem__" + bb: List[int] = [''] # E:22: List item 0 has incompatible type "str"; expected "int" + # XXX: Disabled because the column differs in 3.8 + # aa: List[int] = ['' for x in [1]] # :22: List comprehension has incompatible type List[str]; expected List[int] + cc = 1 .bad # E:10: "int" has no attribute "bad" + n: int = '' # E:14: Incompatible types in assignment (expression has type "str", variable has type "int") + return x +[builtins fixtures/list.pyi] + +[case testColumnReturnValueExpected] +def f() -> int: + return # E:5: Return value expected + +[case testCheckEndColumnPositions] +# flags: --show-error-end +x: int = "no way" + +def g() -> int: ... +def f(x: str) -> None: ... +f(g( +)) +x[0] +[out] +main:2:10:2:17: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:6:3:7:1: error: Argument 1 to "f" has incompatible type "int"; expected "str" +main:8:1:8:4: error: Value of type "int" is not indexable diff --git a/test-data/unit/check-ctypes.test b/test-data/unit/check-ctypes.test new file mode 100644 index 000000000000..1eefdd3c66c1 --- /dev/null +++ b/test-data/unit/check-ctypes.test @@ -0,0 +1,182 @@ +[case testCtypesArrayStandardElementType] +import ctypes + +class MyCInt(ctypes.c_int): + pass + +intarr4 = ctypes.c_int * 4 +a = intarr4(1, ctypes.c_int(2), MyCInt(3), 4) +intarr4(1, 2, 3, "invalid") # E: Array constructor argument 4 of type "str" is not convertible to the array element type "c_int" +reveal_type(a) # N: Revealed type is "_ctypes.Array[ctypes.c_int]" +reveal_type(a[0]) # N: Revealed type is "builtins.int" +reveal_type(a[1:3]) # N: Revealed type is "builtins.list[builtins.int]" +a[0] = 42 +a[1] = ctypes.c_int(42) +a[2] = MyCInt(42) +a[3] = b"bytes" # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "bytes" \ + # N: Possible overload variants: \ + # N: def __setitem__(self, int, Union[c_int, int], /) -> None \ + # N: def __setitem__(self, slice, List[Union[c_int, int]], /) -> None +for x in a: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesArrayCustomElementType] +import ctypes +from typing import Union, List + +class MyCInt(ctypes.c_int): + pass + +myintarr4 = MyCInt * 4 +mya = myintarr4(1, 2, MyCInt(3), 4) +myintarr4(1, ctypes.c_int(2), MyCInt(3), "invalid") # E: Array constructor argument 2 of type "c_int" is not convertible to the array element type "MyCInt" \ + # E: Array constructor argument 4 of type "str" is not convertible to the array element type "MyCInt" +reveal_type(mya) # N: Revealed type is "_ctypes.Array[__main__.MyCInt]" +reveal_type(mya[0]) # N: Revealed type is "__main__.MyCInt" +reveal_type(mya[1:3]) # N: Revealed type is "builtins.list[__main__.MyCInt]" +mya[0] = 42 +mya[1] = ctypes.c_int(42) # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "c_int" \ + # N: Possible overload variants: \ + # N: def __setitem__(self, int, Union[MyCInt, int], /) -> None \ + # N: def __setitem__(self, slice, List[Union[MyCInt, int]], /) -> None +mya[2] = MyCInt(42) +mya[3] = b"bytes" # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "bytes" \ + # N: Possible overload variants: \ + # N: def __setitem__(self, int, Union[MyCInt, int], /) -> None \ + # N: def __setitem__(self, slice, List[Union[MyCInt, int]], /) -> None +for myx in mya: + reveal_type(myx) # N: Revealed type is "__main__.MyCInt" + +myu: Union[ctypes.Array[ctypes.c_int], List[str]] +for myi in myu: + reveal_type(myi) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesArrayUnionElementType] +import ctypes +from typing import Union + +class MyCInt(ctypes.c_int): + pass + +mya: ctypes.Array[Union[MyCInt, ctypes.c_uint]] +reveal_type(mya) # N: Revealed type is "_ctypes.Array[Union[__main__.MyCInt, ctypes.c_uint]]" +reveal_type(mya[0]) # N: Revealed type is "Union[__main__.MyCInt, builtins.int]" +reveal_type(mya[1:3]) # N: Revealed type is "builtins.list[Union[__main__.MyCInt, builtins.int]]" +# The behavior here is not strictly correct, but intentional. +# See the comment in mypy.plugins.ctypes._autoconvertible_to_cdata for details. +mya[0] = 42 +mya[1] = ctypes.c_uint(42) +mya[2] = MyCInt(42) +mya[3] = b"bytes" # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "bytes" \ + # N: Possible overload variants: \ + # N: def __setitem__(self, int, Union[MyCInt, int, c_uint], /) -> None \ + # N: def __setitem__(self, slice, List[Union[MyCInt, int, c_uint]], /) -> None +for myx in mya: + reveal_type(myx) # N: Revealed type is "Union[__main__.MyCInt, builtins.int]" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesCharArrayAttrs] +import ctypes + +ca = (ctypes.c_char * 4)(b'a', b'b', b'c', b'\x00') +reveal_type(ca.value) # N: Revealed type is "builtins.bytes" +reveal_type(ca.raw) # N: Revealed type is "builtins.bytes" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesCharPArrayDoesNotCrash] +import ctypes + +# The following line used to crash with "Could not find builtin symbol 'NoneType'" +ca = (ctypes.c_char_p * 0)() +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesWcharArrayAttrs] +import ctypes + +wca = (ctypes.c_wchar * 4)('a', 'b', 'c', '\x00') +reveal_type(wca.value) # N: Revealed type is "builtins.str" +wca.raw # E: Array attribute "raw" is only available with element type "c_char", not "c_wchar" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesCharUnionArrayAttrs] +import ctypes +from typing import Union + +cua: ctypes.Array[Union[ctypes.c_char, ctypes.c_wchar]] +reveal_type(cua.value) # N: Revealed type is "Union[builtins.bytes, builtins.str]" +cua.raw # E: Array attribute "raw" is only available with element type "c_char", not "Union[c_char, c_wchar]" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesAnyUnionArrayAttrs] +import ctypes +from typing import Any, Union + +caa: ctypes.Array[Union[ctypes.c_char, Any]] +reveal_type(caa.value) # N: Revealed type is "Union[builtins.bytes, Any]" +reveal_type(caa.raw) # N: Revealed type is "builtins.bytes" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesOtherUnionArrayAttrs] +import ctypes +from typing import Union + +cua: ctypes.Array[Union[ctypes.c_char, ctypes.c_int]] +cua.value # E: Array attribute "value" is only available with element type "c_char" or "c_wchar", not "Union[c_char, c_int]" +cua.raw # E: Array attribute "raw" is only available with element type "c_char", not "Union[c_char, c_int]" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesAnyArrayAttrs] +import ctypes + +aa: ctypes.Array[Any] +reveal_type(aa.value) # N: Revealed type is "Any" +reveal_type(aa.raw) # N: Revealed type is "builtins.bytes" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesOtherArrayAttrs] +import ctypes + +oa = (ctypes.c_int * 4)(1, 2, 3, 4) +oa.value # E: Array attribute "value" is only available with element type "c_char" or "c_wchar", not "c_int" +oa.raw # E: Array attribute "raw" is only available with element type "c_char", not "c_int" +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testCtypesArrayConstructorStarargs] +import ctypes + +intarr4 = ctypes.c_int * 4 +intarr6 = ctypes.c_int * 6 +int_values = [1, 2, 3, 4] +c_int_values = [ctypes.c_int(1), ctypes.c_int(2), ctypes.c_int(3), ctypes.c_int(4)] +reveal_type(intarr4(*int_values)) # N: Revealed type is "_ctypes.Array[ctypes.c_int]" +reveal_type(intarr4(*c_int_values)) # N: Revealed type is "_ctypes.Array[ctypes.c_int]" +reveal_type(intarr6(1, ctypes.c_int(2), *int_values)) # N: Revealed type is "_ctypes.Array[ctypes.c_int]" +reveal_type(intarr6(1, ctypes.c_int(2), *c_int_values)) # N: Revealed type is "_ctypes.Array[ctypes.c_int]" +[typing fixtures/typing-medium.pyi] + +float_values = [1.0, 2.0, 3.0, 4.0] +intarr4(*float_values) # E: Array constructor argument 1 of type "List[float]" is not convertible to the array element type "Iterable[c_int]" +[builtins fixtures/floatdict.pyi] + +[case testCtypesArrayConstructorKwargs] +import ctypes +intarr4 = ctypes.c_int * 4 + +x = {"a": 1, "b": 2} +intarr4(**x) + +[builtins fixtures/floatdict.pyi] +[typing fixtures/typing-medium.pyi] diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test new file mode 100644 index 000000000000..1e06f300570e --- /dev/null +++ b/test-data/unit/check-custom-plugin.test @@ -0,0 +1,1100 @@ +-- Test cases for user-defined plugins +-- +-- Note: Plugins used by tests live under test-data/unit/plugins. Defining +-- plugin files in test cases does not work reliably. + +[case testFunctionPluginFile] +# flags: --config-file tmp/mypy.ini +def f() -> str: ... +reveal_type(f()) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/fnplugin.py + +[case testFunctionPluginFilePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +def f() -> str: ... +reveal_type(f()) # N: Revealed type is "builtins.int" +[file pyproject.toml] +\[tool.mypy] +plugins='/test-data/unit/plugins/fnplugin.py' + +[case testFunctionPlugin] +# flags: --config-file tmp/mypy.ini +def f() -> str: ... +reveal_type(f()) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=fnplugin + +[case testFunctionPluginPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +def f() -> str: ... +reveal_type(f()) # N: Revealed type is "builtins.int" +[file pyproject.toml] +\[tool.mypy] +plugins = 'fnplugin' + +[case testFunctionPluginFullnameIsNotNone] +# flags: --config-file tmp/mypy.ini +from typing import Callable, TypeVar +f: Callable[[], None] +T = TypeVar('T') +def g(x: T) -> T: return x # This strips out the name of a callable +g(f)() +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/fnplugin.py + +[case testFunctionPluginFullnameIsNotNonePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +from typing import Callable, TypeVar +f: Callable[[], None] +T = TypeVar('T') +def g(x: T) -> T: return x # This strips out the name of a callable +g(f)() +[file pyproject.toml] +\[tool.mypy] +plugins="/test-data/unit/plugins/fnplugin.py" + +[case testTwoPlugins] +# flags: --config-file tmp/mypy.ini +def f(): ... +def g(): ... +def h(): ... +reveal_type(f()) # N: Revealed type is "builtins.int" +reveal_type(g()) # N: Revealed type is "builtins.str" +reveal_type(h()) # N: Revealed type is "Any" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/fnplugin.py, + /test-data/unit/plugins/plugin2.py + +[case testTwoPluginsPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +def f(): ... +def g(): ... +def h(): ... +reveal_type(f()) # N: Revealed type is "builtins.int" +reveal_type(g()) # N: Revealed type is "builtins.str" +reveal_type(h()) # N: Revealed type is "Any" +[file pyproject.toml] +\[tool.mypy] +plugins=['/test-data/unit/plugins/fnplugin.py', + '/test-data/unit/plugins/plugin2.py' +] + +[case testTwoPluginsMixedType] +# flags: --config-file tmp/mypy.ini +def f(): ... +def g(): ... +def h(): ... +reveal_type(f()) # N: Revealed type is "builtins.int" +reveal_type(g()) # N: Revealed type is "builtins.str" +reveal_type(h()) # N: Revealed type is "Any" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/fnplugin.py, plugin2 + +[case testTwoPluginsMixedTypePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +def f(): ... +def g(): ... +def h(): ... +reveal_type(f()) # N: Revealed type is "builtins.int" +reveal_type(g()) # N: Revealed type is "builtins.str" +reveal_type(h()) # N: Revealed type is "Any" +[file pyproject.toml] +\[tool.mypy] +plugins=['/test-data/unit/plugins/fnplugin.py', 'plugin2'] + +[case testMissingPluginFile] +# flags: --config-file tmp/mypy.ini +[file mypy.ini] +\[mypy] +plugins=missing.py +[out] +tmp/mypy.ini:2: error: Can't find plugin "tmp/missing.py" +--' (work around syntax highlighting) + +[case testMissingPlugin] +# flags: --config-file tmp/mypy.ini +[file mypy.ini] +\[mypy] +plugins=missing +[out] +tmp/mypy.ini:2: error: Error importing plugin "missing": No module named 'missing' + +[case testMultipleSectionsDefinePlugin] +# flags: --config-file tmp/mypy.ini +[file mypy.ini] +\[acme] +plugins=acmeplugin +\[mypy] +plugins=missing.py +\[another] +plugins=another_plugin +[out] +tmp/mypy.ini:4: error: Can't find plugin "tmp/missing.py" +--' (work around syntax highlighting) + +[case testInvalidPluginExtension] +# flags: --config-file tmp/mypy.ini +[file mypy.ini] +\[mypy] +plugins=dir/badext.pyi +[file dir/badext.pyi] +[out] +tmp/mypy.ini:2: error: Plugin "badext.pyi" does not have a .py extension + +[case testMissingPluginEntryPoint] +# flags: --config-file tmp/mypy.ini +[file mypy.ini] +\[mypy] + plugins = /test-data/unit/plugins/noentry.py +[out] +tmp/mypy.ini:2: error: Plugin "/test-data/unit/plugins/noentry.py" does not define entry point function "plugin" + +[case testCustomPluginEntryPointFile] +# flags: --config-file tmp/mypy.ini +def f() -> str: ... +reveal_type(f()) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/customentry.py:register + +[case testCustomPluginEntryPointFileTrailingComma] +# flags: --config-file tmp/mypy.ini +def f() -> str: ... +reveal_type(f()) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins = + /test-data/unit/plugins/customentry.py:register, + +[case testCustomPluginEntryPoint] +# flags: --config-file tmp/mypy.ini +def f() -> str: ... +reveal_type(f()) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=customentry:register + +[case testInvalidPluginEntryPointReturnValue] +# flags: --config-file tmp/mypy.ini +def f(): pass +f() +[file mypy.ini] +\[mypy] + +plugins=/test-data/unit/plugins/badreturn.py +[out] +tmp/mypy.ini:3: error: Type object expected as the return value of "plugin"; got None (in /test-data/unit/plugins/badreturn.py) + +[case testInvalidPluginEntryPointReturnValue2] +# flags: --config-file tmp/mypy.ini +def f(): pass +f() +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/badreturn2.py +[out] +tmp/mypy.ini:2: error: Return value of "plugin" must be a subclass of "mypy.plugin.Plugin" (in /test-data/unit/plugins/badreturn2.py) + +[case testAttributeTypeHookPlugin] +# flags: --config-file tmp/mypy.ini +from typing import Callable +from m import Signal, DerivedSignal +s: Signal[Callable[[int], None]] = Signal() +s(1) +s('') # E: Argument 1 has incompatible type "str"; expected "int" + +ds: DerivedSignal[Callable[[int], None]] = DerivedSignal() +ds('') # E: Argument 1 has incompatible type "str"; expected "int" +[file m.py] +from typing import TypeVar, Generic, Callable +T = TypeVar('T', bound=Callable[..., None]) +class Signal(Generic[T]): + __call__: Callable[..., None] # This type is replaced by the plugin + +class DerivedSignal(Signal[T]): ... +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/attrhook.py + +[case testAttributeTypeHookPluginUntypedDecoratedGetattr] +# flags: --config-file tmp/mypy.ini +from m import Magic, DerivedMagic + +magic = Magic() +reveal_type(magic.magic_field) # N: Revealed type is "builtins.str" +reveal_type(magic.non_magic_method()) # N: Revealed type is "builtins.int" +reveal_type(magic.non_magic_field) # N: Revealed type is "builtins.int" +magic.nonexistent_field # E: Field does not exist +reveal_type(magic.fallback_example) # N: Revealed type is "Any" +reveal_type(magic.no_assignment_field) # N: Revealed type is "builtins.float" +magic.no_assignment_field = "bad" # E: Cannot assign to field + +derived = DerivedMagic() +reveal_type(derived.magic_field) # N: Revealed type is "builtins.str" +derived.nonexistent_field # E: Field does not exist +reveal_type(derived.fallback_example) # N: Revealed type is "Any" + +[file m.py] +from typing import Any, Callable + +def decorator(f): + pass + +class Magic: + # Triggers plugin infrastructure: + @decorator + def __getattr__(self, x: Any) -> Any: ... + def non_magic_method(self) -> int: ... + non_magic_field: int + no_assignment_field: float + +class DerivedMagic(Magic): ... +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/attrhook2.py + +[case testAttributeTypeHookPluginDecoratedGetattr] +# flags: --config-file tmp/mypy.ini +from m import Magic, DerivedMagic + +magic = Magic() +reveal_type(magic.magic_field) # N: Revealed type is "builtins.str" +reveal_type(magic.non_magic_method()) # N: Revealed type is "builtins.int" +reveal_type(magic.non_magic_field) # N: Revealed type is "builtins.int" +magic.nonexistent_field # E: Field does not exist +reveal_type(magic.fallback_example) # N: Revealed type is "builtins.bool" + +derived = DerivedMagic() +reveal_type(derived.magic_field) # N: Revealed type is "builtins.str" +derived.nonexistent_field # E: Field does not exist +reveal_type(derived.fallback_example) # N: Revealed type is "builtins.bool" + +[file m.py] +from typing import Any, Callable + +def decorator(f: Callable) -> Callable[[Any, str], bool]: + pass + +class Magic: + # Triggers plugin infrastructure: + @decorator + def __getattr__(self, x: Any) -> Any: ... + def non_magic_method(self) -> int: ... + non_magic_field: int + +class DerivedMagic(Magic): ... +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/attrhook2.py + +[case testAttributeHookPluginForDynamicClass] +# flags: --config-file tmp/mypy.ini +from m import Magic, DerivedMagic + +magic = Magic() +reveal_type(magic.magic_field) # N: Revealed type is "builtins.str" +reveal_type(magic.non_magic_method()) # N: Revealed type is "builtins.int" +reveal_type(magic.non_magic_field) # N: Revealed type is "builtins.int" +magic.nonexistent_field # E: Field does not exist +reveal_type(magic.fallback_example) # N: Revealed type is "Any" + +derived = DerivedMagic() +reveal_type(derived.magic_field) # N: Revealed type is "builtins.str" +derived.nonexistent_field # E: Field does not exist +reveal_type(magic.fallback_example) # N: Revealed type is "Any" + +[file m.py] +from typing import Any +class Magic: + # Triggers plugin infrastructure: + def __getattr__(self, x: Any) -> Any: ... + def non_magic_method(self) -> int: ... + non_magic_field: int + +class DerivedMagic(Magic): ... + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/attrhook2.py + +[case testTypeAnalyzeHookPlugin] +# flags: --config-file tmp/mypy.ini +from typing import Callable +from mypy_extensions import DefaultArg +from m import Signal +s: Signal[[int, DefaultArg(str, 'x')]] = Signal() +reveal_type(s) # N: Revealed type is "m.Signal[def (builtins.int, x: builtins.str =)]" +s.x # E: "Signal[Callable[[int, str], None]]" has no attribute "x" +ss: Signal[int, str] # E: Invalid "Signal" type (expected "Signal[[t, ...]]") +[file m.py] +from typing import TypeVar, Generic, Callable +T = TypeVar('T', bound=Callable[..., None]) +class Signal(Generic[T]): + __call__: Callable[..., None] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/type_anal_hook.py +[builtins fixtures/dict.pyi] + +[case testFunctionPluginHookForClass] +# flags: --config-file tmp/mypy.ini +import mod +from mod import AttrInt + +Alias = AttrInt +AnotherAlias = mod.Attr + +class C: + x = Alias() + y = mod.AttrInt(required=True) + z = AnotherAlias(int, required=False) + +c = C() +reveal_type(c.x) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(c.y) # N: Revealed type is "builtins.int" +reveal_type(c.z) # N: Revealed type is "Union[builtins.int, None]" + +[file mod.py] +from typing import Generic, TypeVar, Type +T = TypeVar('T') + +class Attr(Generic[T]): + def __init__(self, tp: Type[T], required: bool = False) -> None: + pass + def __get__(self, instance: object, owner: type) -> T: + pass + +class AttrInt(Attr[int]): + def __init__(self, required: bool = False) -> None: + pass + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/class_callable.py +[builtins fixtures/bool.pyi] +[out] + +[case testFunctionPluginHookForReturnedCallable] +# flags: --config-file tmp/mypy.ini +from m import decorator1, decorator2 +@decorator1() +def f() -> None: pass +@decorator2() +def g() -> None: pass +reveal_type(f) # N: Revealed type is "def (*Any, **Any) -> builtins.str" +reveal_type(g) # N: Revealed type is "def (*Any, **Any) -> builtins.int" +[file m.py] +from typing import Callable +def decorator1() -> Callable[..., Callable[..., int]]: pass +def decorator2() -> Callable[..., Callable[..., int]]: pass +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/named_callable.py + +[case testFunctionMethodContextsHasArgNames] +# flags: --config-file tmp/mypy.ini +from mod import Class, func + +reveal_type(Class().method(arg1=1, arg2=2, classname='builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(Class.myclassmethod(arg1=1, arg2=2, classname='builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(Class.mystaticmethod(arg1=1, arg2=2, classname='builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(Class.method(self=Class(), arg1=1, arg2=2, classname='builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(func(arg1=1, arg2=2, classname='builtins.str')) # N: Revealed type is "builtins.str" + +[file mod.py] +from typing import Any +class Class: + def method(self, classname: str, arg1: Any, arg2: Any) -> Any: + pass + @classmethod + def myclassmethod(cls, classname: str, arg1: Any, arg2: Any): + pass + @staticmethod + def mystaticmethod(classname: str, arg1: Any, arg2: Any): + pass +def func(classname: str, arg1: Any, arg2: Any) -> Any: + pass + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_names.py +[builtins fixtures/classmethod.pyi] + +[case testFunctionMethodContextsHasArgNamesPositionals] +# flags: --config-file tmp/mypy.ini +from mod import Class, func + +reveal_type(Class().method('builtins.str', arg1=1, arg2=2)) # N: Revealed type is "builtins.str" +reveal_type(Class.myclassmethod('builtins.str', arg1=1, arg2=2)) # N: Revealed type is "builtins.str" +reveal_type(Class.mystaticmethod('builtins.str', arg1=1, arg2=2)) # N: Revealed type is "builtins.str" +reveal_type(Class.method(Class(), 'builtins.str', arg1=1, arg2=2)) # N: Revealed type is "builtins.str" +reveal_type(func('builtins.str', arg1=1, arg2=2)) # N: Revealed type is "builtins.str" + +[file mod.py] +from typing import Any +class Class: + def method(self, classname: str, arg1: Any, arg2: Any) -> Any: + pass + @classmethod + def myclassmethod(cls, classname: str, arg1: Any, arg2: Any): + pass + @staticmethod + def mystaticmethod(classname: str, arg1: Any, arg2: Any): + pass +def func(classname: str, arg1: Any, arg2: Any) -> Any: + pass + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_names.py +[builtins fixtures/classmethod.pyi] + +[case testFunctionMethodContextsHasArgNamesInitMethod] +# flags: --config-file tmp/mypy.ini +from mod import ClassInit, Outer + +reveal_type(ClassInit('builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(ClassInit(classname='builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(Outer.NestedClassInit(classname='builtins.str')) # N: Revealed type is "builtins.str" +[file mod.py] +from typing import Any +class ClassInit: + def __init__(self, classname: str): + pass +class Outer: + class NestedClassInit: + def __init__(self, classname: str): + pass + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_names.py + +[case testFunctionMethodContextsHasArgNamesUnfilledArguments] +# flags: --config-file tmp/mypy.ini +from mod import ClassUnfilled, func_unfilled + +reveal_type(ClassUnfilled().method(classname='builtins.str', arg1=1)) # N: Revealed type is "builtins.str" +reveal_type(ClassUnfilled().method(arg2=1, classname='builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(ClassUnfilled().method('builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(func_unfilled(classname='builtins.str', arg1=1)) # N: Revealed type is "builtins.str" +reveal_type(func_unfilled(arg2=1, classname='builtins.str')) # N: Revealed type is "builtins.str" +reveal_type(func_unfilled('builtins.str')) # N: Revealed type is "builtins.str" + +[file mod.py] +from typing import Any +class ClassUnfilled: + def method(self, classname: str, arg1: Any = None, arg2: Any = None) -> Any: + pass +def func_unfilled(classname: str, arg1: Any = None, arg2: Any = None) -> Any: + pass + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_names.py + +[case testFunctionMethodContextsHasArgNamesStarExpressions] +# flags: --config-file tmp/mypy.ini +from mod import ClassStarExpr, func_star_expr + +reveal_type(ClassStarExpr().method(classname='builtins.str', arg1=1)) # N: Revealed type is "builtins.str" +reveal_type(ClassStarExpr().method('builtins.str', arg1=1)) # N: Revealed type is "builtins.str" +reveal_type(ClassStarExpr().method('builtins.str', arg1=1, arg2=1)) # N: Revealed type is "builtins.str" +reveal_type(ClassStarExpr().method('builtins.str', 2, 3, 4, arg1=1, arg2=1)) # N: Revealed type is "builtins.str" +reveal_type(func_star_expr(classname='builtins.str', arg1=1)) # N: Revealed type is "builtins.str" +reveal_type(func_star_expr('builtins.str', arg1=1)) # N: Revealed type is "builtins.str" +reveal_type(func_star_expr('builtins.str', 2, 3, 4, arg1=1, arg2=2)) # N: Revealed type is "builtins.str" + +[file mod.py] +from typing import Any +class ClassStarExpr: + def method(self, classname: str, *args, **kwargs) -> Any: + pass +def func_star_expr(classname: str, *args, **kwargs) -> Any: + pass + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_names.py +[builtins fixtures/dict.pyi] + +[case testFunctionMethodContextArgNamesForInheritedMethods] +# flags: --config-file tmp/mypy.ini +from mod import ClassChild + +reveal_type(ClassChild().method(classname='builtins.str', arg1=1, arg2=1)) # N: Revealed type is "builtins.str" +reveal_type(ClassChild().method(arg1=1, classname='builtins.str', arg2=1)) # N: Revealed type is "builtins.str" +reveal_type(ClassChild().method('builtins.str', arg1=1, arg2=1)) # N: Revealed type is "builtins.str" +reveal_type(ClassChild.myclassmethod('builtins.str')) # N: Revealed type is "builtins.str" +[file mod.py] +from typing import Any +class Base: + def method(self, classname: str, arg1: Any, arg2: Any) -> Any: + pass + @classmethod + def myclassmethod(cls, classname: str) -> Any: + pass +class ClassChild(Base): + pass + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_names.py +[builtins fixtures/classmethod.pyi] + +[case testMethodSignatureHook] +# flags: --config-file tmp/mypy.ini +from typing import Iterator + +class Foo: + # Test that method signature hooks are applied in various cases: explicit method calls, and + # implicit dunder method calls through language syntax. + # The plugin's method signature hook should turn all str occurrences into int. + def __init__(self) -> None: ... + def __getitem__(self, index: str) -> str: ... + def __setitem__(self, index: str, value: str) -> None: ... + def __iter__(self) -> Iterator[str]: ... + def __next__(self) -> str: ... + def __call__(self, *args: str) -> str: ... + def m(self, arg: str) -> str: ... + +foo = Foo() +reveal_type(foo.m(2)) # N: Revealed type is "builtins.int" +reveal_type(foo[3]) # N: Revealed type is "builtins.int" +reveal_type(foo(4, 5, 6)) # N: Revealed type is "builtins.int" +foo[4] = 5 +for x in foo: + reveal_type(x) # N: Revealed type is "builtins.int" + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/method_sig_hook.py +[builtins fixtures/tuple.pyi] + +[case testMethodSignatureHookNamesFullyQualified] +# flags: --config-file tmp/mypy.ini +from mypy_extensions import TypedDict +from typing import NamedTuple + +class FullyQualifiedTestClass: + @classmethod + def class_method(self) -> str: ... + def instance_method(self) -> str: ... + +class FullyQualifiedTestTypedDict(TypedDict): + foo: str + +FullyQualifiedTestNamedTuple = NamedTuple('FullyQualifiedTestNamedTuple', [('foo', str)]) + +# Check the return types to ensure that the method signature hook is called in each case +reveal_type(FullyQualifiedTestClass.class_method()) # N: Revealed type is "builtins.int" +reveal_type(FullyQualifiedTestClass().instance_method()) # N: Revealed type is "builtins.int" +reveal_type(FullyQualifiedTestNamedTuple('')._asdict()) # N: Revealed type is "builtins.int" + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/fully_qualified_test_hook.py +[builtins fixtures/classmethod.pyi] + +[case testDynamicClassPlugin] +# flags: --config-file tmp/mypy.ini +from mod import declarative_base, Column, Instr + +Base = declarative_base() + +class Model(Base): + x: Column[int] +class Other: + x: Column[int] + +reveal_type(Model().x) # N: Revealed type is "mod.Instr[builtins.int]" +reveal_type(Other().x) # N: Revealed type is "mod.Column[builtins.int]" +[file mod.py] +from typing import Generic, TypeVar +def declarative_base(): ... + +T = TypeVar('T') + +class Column(Generic[T]): ... +class Instr(Generic[T]): ... + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/dyn_class.py + +[case testDynamicClassPluginChainCall] +# flags: --config-file tmp/mypy.ini +from mod import declarative_base, Column, Instr + +Base = declarative_base().with_optional_xxx() + +class Model(Base): + x: Column[int] + +reveal_type(Model().x) # N: Revealed type is "mod.Instr[builtins.int]" +[file mod.py] +from typing import Generic, TypeVar + +class Base: + def with_optional_xxx(self) -> Base: ... + +def declarative_base() -> Base: ... + +T = TypeVar('T') + +class Column(Generic[T]): ... +class Instr(Generic[T]): ... + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/dyn_class.py + +[case testDynamicClassPluginChainedAssignment] +# flags: --config-file tmp/mypy.ini +from mod import declarative_base + +Base1 = Base2 = declarative_base() + +class C1(Base1): ... +class C2(Base2): ... +[file mod.py] +def declarative_base(): ... +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/dyn_class.py + +[case testDynamicClassPluginNegatives] +# flags: --config-file tmp/mypy.ini +from mod import non_declarative_base + +Bad1 = non_declarative_base() + +class C1(Bad1): ... # E: Variable "__main__.Bad1" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "Bad1" +[file mod.py] +def non_declarative_base(): ... +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/dyn_class.py + +[case testDynamicClassHookFromClassMethod] +# flags: --config-file tmp/mypy.ini + +from mod import QuerySet, Manager, GenericQuerySet + +MyManager = Manager.from_queryset(QuerySet) +ManagerFromGenericQuerySet = GenericQuerySet[int].as_manager() + +reveal_type(MyManager()) # N: Revealed type is "__main__.MyManager" +reveal_type(MyManager().attr) # N: Revealed type is "builtins.str" +reveal_type(ManagerFromGenericQuerySet()) # N: Revealed type is "__main__.ManagerFromGenericQuerySet" +reveal_type(ManagerFromGenericQuerySet().attr) # N: Revealed type is "builtins.int" +queryset: GenericQuerySet[int] = ManagerFromGenericQuerySet() + +def func(manager: MyManager) -> None: + reveal_type(manager) # N: Revealed type is "__main__.MyManager" + reveal_type(manager.attr) # N: Revealed type is "builtins.str" + +func(MyManager()) + +[file mod.py] +from typing import Generic, TypeVar, Type +class QuerySet: + attr: str +class Manager: + @classmethod + def from_queryset(cls, queryset_cls: Type[QuerySet]): ... +T = TypeVar("T") +class GenericQuerySet(Generic[T]): + attr: T + + @classmethod + def as_manager(cls): ... + +[builtins fixtures/classmethod.pyi] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/dyn_class_from_method.py + +[case testBaseClassPluginHookWorksIncremental] +# flags: --config-file tmp/mypy.ini +import a + +[file a.py] +from base import Base +class C(Base): ... + +[file a.py.2] +from base import Base +class C(Base): ... +reveal_type(C().__magic__) +Base.__magic__ + +[file base.py] +from lib import declarative_base +Base = declarative_base() + +[file lib.py] +from typing import Any +def declarative_base() -> Any: ... + +[file mypy.ini] +\[mypy] +python_version=3.6 +plugins=/test-data/unit/plugins/common_api_incremental.py +[out] +[out2] +tmp/a.py:3: note: Revealed type is "builtins.str" +tmp/a.py:4: error: "Type[Base]" has no attribute "__magic__" + +[case testArgKindsMethod] +# flags: --config-file tmp/mypy.ini +class Class: + def method(self, *args, **kwargs): + pass + +Class().method(1, *[2], **{'a': 1}) # E: [[0, 2], [4]] +[builtins fixtures/dict.pyi] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_kinds.py + +[case testArgKindsFunction] +# flags: --config-file tmp/mypy.ini +def func(*args, **kwargs): + pass + +func(1, 2, [3, 4], *[5, 6, 7], **{'a': 1}) # E: [[0, 0, 0, 2], [4]] +[builtins fixtures/dict.pyi] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/arg_kinds.py + +[case testHookCallableInstance] +# flags: --config-file tmp/mypy.ini +from typing import Generic, TypeVar +T = TypeVar("T") +class Class(Generic[T]): + def __init__(self, one: T): ... + def __call__(self, two: T) -> int: ... +reveal_type(Class("hi")("there")) # N: Revealed type is "builtins.str" +instance = Class(3.14) +reveal_type(instance(2)) # N: Revealed type is "builtins.float" + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/callable_instance.py + +[case testGetMethodHooksOnUnions] +# flags: --config-file tmp/mypy.ini --no-strict-optional +from typing import Union + +class Foo: + def meth(self, x: str) -> str: ... +class Bar: + def meth(self, x: int) -> float: ... +class Other: + meth: int + +x: Union[Foo, Bar, Other] +if isinstance(x.meth, int): + reveal_type(x.meth) # N: Revealed type is "builtins.int" +else: + reveal_type(x.meth(int())) # N: Revealed type is "builtins.int" + +[builtins fixtures/isinstancelist.pyi] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/union_method.py + +[case testGetMethodHooksOnUnionsStrictOptional] +# flags: --config-file tmp/mypy.ini +from typing import Union + +class Foo: + def meth(self, x: str) -> str: ... +class Bar: + def meth(self, x: int) -> float: ... +class Other: + meth: int + +x: Union[Foo, Bar, Other] +if isinstance(x.meth, int): + reveal_type(x.meth) # N: Revealed type is "builtins.int" +else: + reveal_type(x.meth(int())) # N: Revealed type is "builtins.int" + +[builtins fixtures/isinstancelist.pyi] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/union_method.py + +[case testGetMethodHooksOnUnionsSpecial] +# flags: --config-file tmp/mypy.ini +from typing import Union + +class Foo: + def __getitem__(self, x: str) -> str: ... +class Bar: + def __getitem__(self, x: int) -> float: ... + +x: Union[Foo, Bar] +reveal_type(x[int()]) # N: Revealed type is "builtins.int" + +[builtins fixtures/isinstancelist.pyi] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/union_method.py + +[case testPluginDependencies] +# flags: --config-file tmp/mypy.ini + +# The top level file here doesn't do anything, but the plugin should add +# a dependency on err that will cause err to be processed and an error reported. + +[file err.py] +1 + 'lol' # E: Unsupported operand types for + ("int" and "str") + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/depshook.py + +[case testCustomizeMroTrivial] +# flags: --config-file tmp/mypy.ini +class A: pass +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/customize_mro.py + +[case testDescriptorMethods] +# flags: --config-file tmp/mypy.ini + +class Desc: + def __get__(self, obj, cls): + pass + + def __set__(self, obj, val): + pass + +class Cls: + attr = Desc() + +reveal_type(Cls().attr) # N: Revealed type is "builtins.int" +reveal_type(Cls.attr) # N: Revealed type is "builtins.str" + +Cls().attr = 3 +Cls().attr = "foo" # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/descriptor.py + +[case testFunctionSigPluginFile] +# flags: --config-file tmp/mypy.ini + +def dynamic_signature(arg1: str) -> str: ... +a: int = 1 +reveal_type(dynamic_signature(a)) # N: Revealed type is "builtins.int" +b: bytes = b'foo' +reveal_type(dynamic_signature(b)) # N: Revealed type is "builtins.bytes" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/function_sig_hook.py + +[case testPluginCalledCorrectlyWhenMethodInDecorator] +# flags: --config-file tmp/mypy.ini +from typing import TypeVar, Callable + +T = TypeVar('T') +class Foo: + def a(self, x: Callable[[], T]) -> Callable[[], T]: ... + +b = Foo() + +@b.a +def f() -> None: + pass + +reveal_type(f()) # N: Revealed type is "builtins.str" + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/method_in_decorator.py + +[case testClassAttrPluginClassVar] +# flags: --config-file tmp/mypy.ini + +from typing import Type + +class Cls: + attr = 'test' + unchanged = 'test' + +reveal_type(Cls().attr) # N: Revealed type is "builtins.str" +reveal_type(Cls.attr) # N: Revealed type is "builtins.int" +reveal_type(Cls.unchanged) # N: Revealed type is "builtins.str" +x: Type[Cls] +reveal_type(x.attr) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/class_attr_hook.py + +[case testClassAttrPluginMethod] +# flags: --config-file tmp/mypy.ini + +class Cls: + def attr(self) -> None: + pass + +reveal_type(Cls.attr) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/class_attr_hook.py + +[case testClassAttrPluginEnum] +# flags: --config-file tmp/mypy.ini + +import enum + +class Cls(enum.Enum): + attr = 'test' + +reveal_type(Cls.attr) # N: Revealed type is "builtins.int" +[builtins fixtures/enum.pyi] +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/class_attr_hook.py + +[case testClassAttrPluginMetaclassAnyBase] +# flags: --config-file tmp/mypy.ini + +from typing import Any, Type +class M(type): + attr = 'test' + +B: Any +class Cls(B, metaclass=M): + pass + +reveal_type(Cls.attr) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/class_attr_hook.py + +[case testClassAttrPluginMetaclassRegularBase] +# flags: --config-file tmp/mypy.ini + +from typing import Any, Type +class M(type): + attr = 'test' + +class B: + attr = None + +class Cls(B, metaclass=M): + pass + +reveal_type(Cls.attr) # N: Revealed type is "builtins.int" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/class_attr_hook.py + +[case testClassAttrPluginPartialType] +# flags: --config-file tmp/mypy.ini + +class Cls: + attr = None + def f(self) -> int: + return Cls.attr + 1 + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/class_attr_hook.py + +[case testAddClassMethodPlugin] +# flags: --config-file tmp/mypy.ini +class BaseAddMethod: pass + +class MyClass(BaseAddMethod): + pass + +reveal_type(MyClass.foo_classmethod) # N: Revealed type is "def ()" +reveal_type(MyClass.foo_staticmethod) # N: Revealed type is "def (builtins.int) -> builtins.str" + +my_class = MyClass() +reveal_type(my_class.foo_classmethod) # N: Revealed type is "def ()" +reveal_type(my_class.foo_staticmethod) # N: Revealed type is "def (builtins.int) -> builtins.str" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_classmethod.py + +[case testAddOverloadedMethodPlugin] +# flags: --config-file tmp/mypy.ini +class AddOverloadedMethod: pass + +class MyClass(AddOverloadedMethod): + pass + +reveal_type(MyClass.method) # N: Revealed type is "Overload(def (self: __main__.MyClass, arg: builtins.int) -> builtins.str, def (self: __main__.MyClass, arg: builtins.str) -> builtins.int)" +reveal_type(MyClass.clsmethod) # N: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +reveal_type(MyClass.stmethod) # N: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" + +my_class = MyClass() +reveal_type(my_class.method) # N: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +reveal_type(my_class.clsmethod) # N: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +reveal_type(my_class.stmethod) # N: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_overloaded_method.py + +[case testAddMethodPluginExplicitOverride] +# flags: --python-version 3.12 --config-file tmp/mypy.ini +from typing import override, TypeVar + +T = TypeVar('T', bound=type) + +def inject_foo(t: T) -> T: + # Imitates: + # t.foo_implicit = some_method + return t + +class BaseWithoutFoo: pass + +@inject_foo +class ChildWithFoo(BaseWithoutFoo): pass +reveal_type(ChildWithFoo.foo_implicit) # N: Revealed type is "def (self: __main__.ChildWithFoo)" + +@inject_foo +class SomeWithFoo(ChildWithFoo): pass +reveal_type(SomeWithFoo.foo_implicit) # N: Revealed type is "def (self: __main__.SomeWithFoo)" + +class ExplicitOverride(SomeWithFoo): + @override + def foo_implicit(self) -> None: pass + +class ImplicitOverride(SomeWithFoo): + def foo_implicit(self) -> None: pass # E: Method "foo_implicit" is not using @override but is overriding a method in class "__main__.SomeWithFoo" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_method.py +enable_error_code = explicit-override +[typing fixtures/typing-override.pyi] + +[case testCustomErrorCodePlugin] +# flags: --config-file tmp/mypy.ini --show-error-codes +def main() -> int: + return 2 + +main() # E: Custom error [custom] +reveal_type(1) # N: Revealed type is "Literal[1]?" + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/custom_errorcode.py diff --git a/test-data/unit/check-dataclass-transform.test b/test-data/unit/check-dataclass-transform.test new file mode 100644 index 000000000000..51b2e186214f --- /dev/null +++ b/test-data/unit/check-dataclass-transform.test @@ -0,0 +1,1074 @@ +[case testDataclassTransformReusesDataclassLogic] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type + +@dataclass_transform() +def my_dataclass(cls: Type) -> Type: + return cls + +@my_dataclass +class Person: + name: str + age: int + + def summary(self): + return "%s is %d years old." % (self.name, self.age) + +reveal_type(Person) # N: Revealed type is "def (name: builtins.str, age: builtins.int) -> __main__.Person" +Person('John', 32) +Person('Jonh', 21, None) # E: Too many arguments for "Person" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformIsFoundInTypingExtensions] +from typing import Type +from typing_extensions import dataclass_transform + +@dataclass_transform() +def my_dataclass(cls: Type) -> Type: + return cls + +@my_dataclass +class Person: + name: str + age: int + + def summary(self): + return "%s is %d years old." % (self.name, self.age) + +reveal_type(Person) # N: Revealed type is "def (name: builtins.str, age: builtins.int) -> __main__.Person" +Person('John', 32) +Person('Jonh', 21, None) # E: Too many arguments for "Person" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformParametersAreApplied] +# flags: --python-version 3.11 +from typing import dataclass_transform, Callable, Type + +@dataclass_transform() +def my_dataclass(*, eq: bool, order: bool) -> Callable[[Type], Type]: + def transform(cls: Type) -> Type: + return cls + return transform + +@my_dataclass(eq=False, order=True) # E: "eq" must be True if "order" is True +class Person: + name: str + age: int + +reveal_type(Person) # N: Revealed type is "def (name: builtins.str, age: builtins.int) -> __main__.Person" +Person('John', 32) +Person('John', 21, None) # E: Too many arguments for "Person" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformParametersMustBeBoolLiterals] +# flags: --python-version 3.11 +from typing import dataclass_transform, Callable, Type + +@dataclass_transform() +def my_dataclass(*, eq: bool = True, order: bool = False) -> Callable[[Type], Type]: + def transform(cls: Type) -> Type: + return cls + return transform +@dataclass_transform() +class BaseClass: + def __init_subclass__(cls, *, eq: bool): ... +@dataclass_transform() +class Metaclass(type): ... + +BOOL_CONSTANT = True +@my_dataclass(eq=BOOL_CONSTANT) # E: "eq" argument must be a True or False literal +class A: ... +@my_dataclass(order=not False) # E: "order" argument must be a True or False literal +class B: ... +class C(BaseClass, eq=BOOL_CONSTANT): ... # E: "eq" argument must be a True or False literal +class D(metaclass=Metaclass, order=not False): ... # E: "order" argument must be a True or False literal + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformDefaultParamsMustBeLiterals] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type, Final + +BOOLEAN_CONSTANT = True +FINAL_BOOLEAN: Final = True + +@dataclass_transform(eq_default=BOOLEAN_CONSTANT) # E: "eq_default" argument must be a True or False literal +def foo(cls: Type) -> Type: + return cls +@dataclass_transform(eq_default=(not True)) # E: "eq_default" argument must be a True or False literal +def bar(cls: Type) -> Type: + return cls +@dataclass_transform(eq_default=FINAL_BOOLEAN) # E: "eq_default" argument must be a True or False literal +def baz(cls: Type) -> Type: + return cls + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformUnrecognizedParamsAreErrors] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type + +BOOLEAN_CONSTANT = True + +@dataclass_transform(nonexistant=True) # E: Unrecognized dataclass_transform parameter "nonexistant" +def foo(cls: Type) -> Type: + return cls + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + + +[case testDataclassTransformDefaultParams] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type, Callable + +@dataclass_transform(eq_default=False) +def no_eq(*, order: bool = False) -> Callable[[Type], Type]: + return lambda cls: cls +@no_eq() +class Foo: ... +@no_eq(order=True) # E: "eq" must be True if "order" is True +class Bar: ... + + +@dataclass_transform(kw_only_default=True) +def always_use_kw(cls: Type) -> Type: + return cls +@always_use_kw +class Baz: + x: int +Baz(x=5) +Baz(5) # E: Too many positional arguments for "Baz" + +@dataclass_transform(order_default=True) +def ordered(*, eq: bool = True) -> Callable[[Type], Type]: + return lambda cls: cls +@ordered() +class A: + x: int +A(1) > A(2) + +@dataclass_transform(frozen_default=True) +def frozen(cls: Type) -> Type: + return cls +@frozen +class B: + x: int +b = B(x=1) +b.x = 2 # E: Property "x" defined in "B" is read-only + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformDefaultsCanBeOverridden] +# flags: --python-version 3.11 +from typing import dataclass_transform, Callable, Type + +@dataclass_transform(kw_only_default=True) +def my_dataclass(*, kw_only: bool = True) -> Callable[[Type], Type]: + return lambda cls: cls + +@my_dataclass() +class KwOnly: + x: int +@my_dataclass(kw_only=False) +class KwOptional: + x: int + +KwOnly(5) # E: Too many positional arguments for "KwOnly" +KwOptional(5) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformFieldSpecifiersDefaultsToEmpty] +# flags: --python-version 3.11 +from dataclasses import field, dataclass +from typing import dataclass_transform, Type + +@dataclass_transform() +def my_dataclass(cls: Type) -> Type: + return cls + +@my_dataclass +class Foo: + foo: int = field(kw_only=True) + +# Does not cause a type error because `dataclasses.field` is not a recognized field specifier by +# default +Foo(5) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformFieldSpecifierRejectMalformed] +# flags: --python-version 3.11 +from typing import dataclass_transform, Any, Callable, Final, Type + +def some_type() -> Type: ... +def some_function() -> Callable[[], None]: ... + +def field(*args, **kwargs): ... +def fields_tuple() -> tuple[type | Callable[..., Any], ...]: return (field,) +CONSTANT: Final = (field,) + +@dataclass_transform(field_specifiers=(some_type(),)) # E: "field_specifiers" must only contain identifiers +def bad_dataclass1() -> None: ... +@dataclass_transform(field_specifiers=(some_function(),)) # E: "field_specifiers" must only contain identifiers +def bad_dataclass2() -> None: ... +@dataclass_transform(field_specifiers=CONSTANT) # E: "field_specifiers" argument must be a tuple literal +def bad_dataclass3() -> None: ... +@dataclass_transform(field_specifiers=fields_tuple()) # E: "field_specifiers" argument must be a tuple literal +def bad_dataclass4() -> None: ... + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformFieldSpecifierParams] +# flags: --python-version 3.11 +from typing import dataclass_transform, Any, Callable, Type, Final + +def field( + *, + init: bool = True, + kw_only: bool = False, + alias: str | None = None, + default: Any | None = None, + default_factory: Callable[[], Any] | None = None, + factory: Callable[[], Any] | None = None, +): ... +@dataclass_transform(field_specifiers=(field,)) +def my_dataclass(cls: Type) -> Type: + return cls + +B: Final = 'b_' +@my_dataclass +class Foo: + a: int = field(alias='a_') + b: int = field(alias=B) + # cannot be passed as a positional + kwonly: int = field(kw_only=True, default=0) + # Safe to omit from constructor, error to pass + noinit: int = field(init=False, default=1) + # It should be safe to call the constructor without passing any of these + unused1: int = field(default=0) + unused2: int = field(factory=lambda: 0) + unused3: int = field(default_factory=lambda: 0) + +Foo(a=5, b_=1) # E: Unexpected keyword argument "a" for "Foo" +Foo(a_=1, b_=1, noinit=1) # E: Unexpected keyword argument "noinit" for "Foo" +Foo(1, 2, 3) # E: Too many positional arguments for "Foo" +foo = Foo(1, 2, kwonly=3) +reveal_type(foo.noinit) # N: Revealed type is "builtins.int" +reveal_type(foo.unused1) # N: Revealed type is "builtins.int" +Foo(a_=5, b_=1, unused1=2, unused2=3, unused3=4) + +def some_str() -> str: ... +def some_bool() -> bool: ... +@my_dataclass +class Bad: + bad1: int = field(alias=some_str()) # E: "alias" argument to dataclass field must be a string literal + bad2: int = field(kw_only=some_bool()) # E: "kw_only" argument must be a boolean literal + +reveal_type(Foo.__dataclass_fields__) # N: Revealed type is "builtins.dict[builtins.str, Any]" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformFieldSpecifierExtraArgs] +# flags: --python-version 3.11 +from typing import dataclass_transform + +def field(extra1, *, kw_only=False, extra2=0): ... +@dataclass_transform(field_specifiers=(field,)) +def my_dataclass(cls): + return cls + +@my_dataclass +class Good: + a: int = field(5) + b: int = field(5, extra2=1) + c: int = field(5, kw_only=True) + +@my_dataclass +class Bad: + a: int = field(kw_only=True) # E: Missing positional argument "extra1" in call to "field" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformMultipleFieldSpecifiers] +# flags: --python-version 3.11 +from typing import dataclass_transform + +def field1(*, default: int) -> int: ... +def field2(*, default: str) -> str: ... + +@dataclass_transform(field_specifiers=(field1, field2)) +def my_dataclass(cls): return cls + +@my_dataclass +class Foo: + a: int = field1(default=0) + b: str = field2(default='hello') + +reveal_type(Foo) # N: Revealed type is "def (a: builtins.int =, b: builtins.str =) -> __main__.Foo" +Foo() +Foo(a=1, b='bye') + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformFieldSpecifierImplicitInit] +# flags: --python-version 3.11 +from typing import dataclass_transform, Literal, overload + +def init(*, init: Literal[True] = True): ... +def no_init(*, init: Literal[False] = False): ... + +@overload +def field_overload(*, custom: None, init: Literal[True] = True): ... +@overload +def field_overload(*, custom: str, init: Literal[False] = False): ... +def field_overload(*, custom, init): ... + +@dataclass_transform(field_specifiers=(init, no_init, field_overload)) +def my_dataclass(cls): return cls + +@my_dataclass +class Foo: + a: int = init() + b: int = field_overload(custom=None) + + bad1: int = no_init() + bad2: int = field_overload(custom="bad2") + +reveal_type(Foo) # N: Revealed type is "def (a: builtins.int, b: builtins.int) -> __main__.Foo" +Foo(a=1, b=2) +Foo(a=1, b=2, bad1=0) # E: Unexpected keyword argument "bad1" for "Foo" +Foo(a=1, b=2, bad2=0) # E: Unexpected keyword argument "bad2" for "Foo" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformOverloadsDecoratorOnOverload] +# flags: --python-version 3.11 +from typing import dataclass_transform, overload, Any, Callable, Type, Literal + +@overload +def my_dataclass(*, foo: str) -> Callable[[Type], Type]: ... +@overload +@dataclass_transform(frozen_default=True) +def my_dataclass(*, foo: int) -> Callable[[Type], Type]: ... +def my_dataclass(*, foo: Any) -> Callable[[Type], Type]: + return lambda cls: cls +@my_dataclass(foo="hello") +class A: + a: int +@my_dataclass(foo=5) +class B: + b: int + +reveal_type(A) # N: Revealed type is "def (a: builtins.int) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (b: builtins.int) -> __main__.B" +A(1, "hello") # E: Too many arguments for "A" +a = A(1) +a.a = 2 # E: Property "a" defined in "A" is read-only + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformOverloadsDecoratorOnImpl] +# flags: --python-version 3.11 +from typing import dataclass_transform, overload, Any, Callable, Type, Literal + +@overload +def my_dataclass(*, foo: str) -> Callable[[Type], Type]: ... +@overload +def my_dataclass(*, foo: int) -> Callable[[Type], Type]: ... +@dataclass_transform(frozen_default=True) +def my_dataclass(*, foo: Any) -> Callable[[Type], Type]: + return lambda cls: cls +@my_dataclass(foo="hello") +class A: + a: int +@my_dataclass(foo=5) +class B: + b: int + +reveal_type(A) # N: Revealed type is "def (a: builtins.int) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (b: builtins.int) -> __main__.B" +A(1, "hello") # E: Too many arguments for "A" +a = A(1) +a.a = 2 # E: Property "a" defined in "A" is read-only + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformViaBaseClass] +# flags: --python-version 3.11 +from typing import dataclass_transform + +@dataclass_transform(frozen_default=True) +class Dataclass: + def __init_subclass__(cls, *, kw_only: bool = False): ... + +class Person(Dataclass, kw_only=True): + name: str + age: int + +reveal_type(Person) # N: Revealed type is "def (*, name: builtins.str, age: builtins.int) -> __main__.Person" +Person('Jonh', 21) # E: Too many positional arguments for "Person" +person = Person(name='John', age=32) +person.name = "John Smith" # E: Property "name" defined in "Person" is read-only + +class Contact(Person): + email: str + +reveal_type(Contact) # N: Revealed type is "def (email: builtins.str, *, name: builtins.str, age: builtins.int) -> __main__.Contact" +Contact('john@john.com', name='John', age=32) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformViaMetaclass] +# flags: --python-version 3.11 +from typing import dataclass_transform + +@dataclass_transform(frozen_default=True) +class Dataclass(type): ... + +# Note that PEP 681 states that a class that directly specifies a dataclass_transform-decorated +# metaclass should be treated as neither frozen nor unfrozen. For Person to have frozen semantics, +# it may not directly specify the metaclass. +class BaseDataclass(metaclass=Dataclass): ... +class Person(BaseDataclass, kw_only=True): + name: str + age: int + +reveal_type(Person) # N: Revealed type is "def (*, name: builtins.str, age: builtins.int) -> __main__.Person" +Person('Jonh', 21) # E: Too many positional arguments for "Person" +person = Person(name='John', age=32) +person.name = "John Smith" # E: Property "name" defined in "Person" is read-only + +class Contact(Person): + email: str + +reveal_type(Contact) # N: Revealed type is "def (email: builtins.str, *, name: builtins.str, age: builtins.int) -> __main__.Contact" +Contact('john@john.com', name='John', age=32) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformViaSubclassOfMetaclass] +# flags: --python-version 3.11 +from typing import dataclass_transform + +@dataclass_transform(frozen_default=True) +class BaseMeta(type): ... +class SubMeta(BaseMeta): ... + +# MyPy does *not* recognize this as a dataclass because the metaclass is not directly decorated with +# dataclass_transform +class Foo(metaclass=SubMeta): + foo: int + +reveal_type(Foo) # N: Revealed type is "def () -> __main__.Foo" +Foo(1) # E: Too many arguments for "Foo" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformTypeCheckingInFunction] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type, TYPE_CHECKING + +@dataclass_transform() +def model(cls: Type) -> Type: + return cls + +@model +class FunctionModel: + if TYPE_CHECKING: + string_: str + integer_: int + else: + string_: tuple + integer_: tuple + +FunctionModel(string_="abc", integer_=1) +FunctionModel(string_="abc", integer_=tuple()) # E: Argument "integer_" to "FunctionModel" has incompatible type "Tuple[Never, ...]"; expected "int" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformNegatedTypeCheckingInFunction] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type, TYPE_CHECKING + +@dataclass_transform() +def model(cls: Type) -> Type: + return cls + +@model +class FunctionModel: + if not TYPE_CHECKING: + string_: tuple + integer_: tuple + else: + string_: str + integer_: int + +FunctionModel(string_="abc", integer_=1) +FunctionModel(string_="abc", integer_=tuple()) # E: Argument "integer_" to "FunctionModel" has incompatible type "Tuple[Never, ...]"; expected "int" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + + +[case testDataclassTransformTypeCheckingInBaseClass] +# flags: --python-version 3.11 +from typing import dataclass_transform, TYPE_CHECKING + +@dataclass_transform() +class ModelBase: + ... + +class BaseClassModel(ModelBase): + if TYPE_CHECKING: + string_: str + integer_: int + else: + string_: tuple + integer_: tuple + +BaseClassModel(string_="abc", integer_=1) +BaseClassModel(string_="abc", integer_=tuple()) # E: Argument "integer_" to "BaseClassModel" has incompatible type "Tuple[Never, ...]"; expected "int" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformNegatedTypeCheckingInBaseClass] +# flags: --python-version 3.11 +from typing import dataclass_transform, TYPE_CHECKING + +@dataclass_transform() +class ModelBase: + ... + +class BaseClassModel(ModelBase): + if not TYPE_CHECKING: + string_: tuple + integer_: tuple + else: + string_: str + integer_: int + +BaseClassModel(string_="abc", integer_=1) +BaseClassModel(string_="abc", integer_=tuple()) # E: Argument "integer_" to "BaseClassModel" has incompatible type "Tuple[Never, ...]"; expected "int" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformTypeCheckingInMetaClass] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type, TYPE_CHECKING + +@dataclass_transform() +class ModelMeta(type): + ... + +class ModelBaseWithMeta(metaclass=ModelMeta): + ... + +class MetaClassModel(ModelBaseWithMeta): + if TYPE_CHECKING: + string_: str + integer_: int + else: + string_: tuple + integer_: tuple + +MetaClassModel(string_="abc", integer_=1) +MetaClassModel(string_="abc", integer_=tuple()) # E: Argument "integer_" to "MetaClassModel" has incompatible type "Tuple[Never, ...]"; expected "int" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformNegatedTypeCheckingInMetaClass] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type, TYPE_CHECKING + +@dataclass_transform() +class ModelMeta(type): + ... + +class ModelBaseWithMeta(metaclass=ModelMeta): + ... + +class MetaClassModel(ModelBaseWithMeta): + if not TYPE_CHECKING: + string_: tuple + integer_: tuple + else: + string_: str + integer_: int + +MetaClassModel(string_="abc", integer_=1) +MetaClassModel(string_="abc", integer_=tuple()) # E: Argument "integer_" to "MetaClassModel" has incompatible type "Tuple[Never, ...]"; expected "int" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformStaticConditionalAttributes] +# flags: --python-version 3.11 --always-true TRUTH +from typing import dataclass_transform, Type, TYPE_CHECKING + +TRUTH = False # Is set to --always-true + +@dataclass_transform() +def model(cls: Type) -> Type: + return cls + +@model +class FunctionModel: + if TYPE_CHECKING: + present_1: int + else: + skipped_1: int + if True: # Mypy does not know if it is True or False, so the block is used + present_2: int + if False: # Mypy does not know if it is True or False, so the block is used + present_3: int + if not TRUTH: + skipped_2: int + else: + present_4: int + +FunctionModel( + present_1=1, + present_2=2, + present_3=3, + present_4=4, +) +FunctionModel() # E: Missing positional arguments "present_1", "present_2", "present_3", "present_4" in call to "FunctionModel" +FunctionModel( # E: Unexpected keyword argument "skipped_1" for "FunctionModel" + present_1=1, + present_2=2, + present_3=3, + present_4=4, + skipped_1=5, +) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + + +[case testDataclassTransformStaticDeterministicConditionalElifAttributes] +# flags: --python-version 3.11 --always-true TRUTH --always-false LIE +from typing import dataclass_transform, Type, TYPE_CHECKING + +TRUTH = False # Is set to --always-true +LIE = True # Is set to --always-false + +@dataclass_transform() +def model(cls: Type) -> Type: + return cls + +@model +class FunctionModel: + if TYPE_CHECKING: + present_1: int + elif TRUTH: + skipped_1: int + else: + skipped_2: int + if LIE: + skipped_3: int + elif TRUTH: + present_2: int + else: + skipped_4: int + if LIE: + skipped_5: int + elif LIE: + skipped_6: int + else: + present_3: int + +FunctionModel( + present_1=1, + present_2=2, + present_3=3, +) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformStaticNotDeterministicConditionalElifAttributes] +# flags: --python-version 3.11 --always-true TRUTH --always-false LIE +from typing import dataclass_transform, Type, TYPE_CHECKING + +TRUTH = False # Is set to --always-true +LIE = True # Is set to --always-false + +@dataclass_transform() +def model(cls: Type) -> Type: + return cls + +@model +class FunctionModel: + if 123: # Mypy does not know if it is True or False, so this block is used + present_1: int + elif TRUTH: # Mypy does not know if previous condition is True or False, so it uses also this block + present_2: int + else: # Previous block is for sure True, so this block is skipped + skipped_1: int + if 123: + present_3: int + elif 123: + present_4: int + else: + present_5: int + if 123: # Mypy does not know if it is True or False, so this block is used + present_6: int + elif LIE: # This is for sure False, so the block is skipped used + skipped_2: int + else: # None of the conditions above for sure True, so this block is used + present_7: int + +FunctionModel( + present_1=1, + present_2=2, + present_3=3, + present_4=4, + present_5=5, + present_6=6, + present_7=7, +) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformFunctionConditionalAttributes] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type + +@dataclass_transform() +def model(cls: Type) -> Type: + return cls + +def condition() -> bool: + return True + +@model +class FunctionModel: + if condition(): + x: int + y: int + z1: int + else: + x: str # E: Name "x" already defined on line 14 + y: int # E: Name "y" already defined on line 15 + z2: int + +FunctionModel(x=1, y=2, z1=3, z2=4) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + + +[case testDataclassTransformNegatedFunctionConditionalAttributes] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type + +@dataclass_transform() +def model(cls: Type) -> Type: + return cls + +def condition() -> bool: + return True + +@model +class FunctionModel: + if not condition(): + x: int + y: int + z1: int + else: + x: str # E: Name "x" already defined on line 14 + y: int # E: Name "y" already defined on line 15 + z2: int + +FunctionModel(x=1, y=2, z1=3, z2=4) + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformDirectMetaclassNeitherFrozenNorNotFrozen] +# flags: --python-version 3.11 +from typing import dataclass_transform, Type + +@dataclass_transform() +class Meta(type): ... +class Base(metaclass=Meta): + base: int +class Foo(Base, frozen=True): + foo: int +class Bar(Base, frozen=False): + bar: int + + +foo = Foo(0, 1) +foo.foo = 5 # E: Property "foo" defined in "Foo" is read-only +foo.base = 6 +reveal_type(foo.base) # N: Revealed type is "builtins.int" +bar = Bar(0, 1) +bar.bar = 5 +bar.base = 6 +reveal_type(bar.base) # N: Revealed type is "builtins.int" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformReplace] +from dataclasses import replace +from typing import dataclass_transform, Type + +@dataclass_transform() +def my_dataclass(cls: Type) -> Type: + return cls + +@my_dataclass +class Person: + name: str + +p = Person('John') +y = replace(p, name='Bob') + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformSimpleDescriptor] +# flags: --python-version 3.11 + +from typing import dataclass_transform, overload, Any + +@dataclass_transform() +def my_dataclass(cls): ... + +class Desc: + @overload + def __get__(self, instance: None, owner: Any) -> Desc: ... + @overload + def __get__(self, instance: object, owner: Any) -> str: ... + def __get__(self, instance: object | None, owner: Any) -> Desc | str: ... + + def __set__(self, instance: Any, value: str) -> None: ... + +@my_dataclass +class C: + x: Desc + y: int + +C(x='x', y=1) +C(x=1, y=1) # E: Argument "x" to "C" has incompatible type "int"; expected "str" +reveal_type(C(x='x', y=1).x) # N: Revealed type is "builtins.str" +reveal_type(C(x='x', y=1).y) # N: Revealed type is "builtins.int" +reveal_type(C.x) # N: Revealed type is "__main__.Desc" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformUnannotatedDescriptor] +# flags: --python-version 3.11 + +from typing import dataclass_transform, overload, Any + +@dataclass_transform() +def my_dataclass(cls): ... + +class Desc: + @overload + def __get__(self, instance: None, owner: Any) -> Desc: ... + @overload + def __get__(self, instance: object, owner: Any) -> str: ... + def __get__(self, instance: object | None, owner: Any) -> Desc | str: ... + + def __set__(*args, **kwargs): ... + +@my_dataclass +class C: + x: Desc + y: int + +C(x='x', y=1) +C(x=1, y=1) +reveal_type(C(x='x', y=1).x) # N: Revealed type is "builtins.str" +reveal_type(C(x='x', y=1).y) # N: Revealed type is "builtins.int" +reveal_type(C.x) # N: Revealed type is "__main__.Desc" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformGenericDescriptor] +# flags: --python-version 3.11 + +from typing import dataclass_transform, overload, Any, TypeVar, Generic + +@dataclass_transform() +def my_dataclass(frozen: bool = False): ... + +T = TypeVar("T") + +class Desc(Generic[T]): + @overload + def __get__(self, instance: None, owner: Any) -> Desc[T]: ... + @overload + def __get__(self, instance: object, owner: Any) -> T: ... + def __get__(self, instance: object | None, owner: Any) -> Desc | T: ... + + def __set__(self, instance: Any, value: T) -> None: ... + +@my_dataclass() +class C: + x: Desc[str] + +C(x='x') +C(x=1) # E: Argument "x" to "C" has incompatible type "int"; expected "str" +reveal_type(C(x='x').x) # N: Revealed type is "builtins.str" +reveal_type(C.x) # N: Revealed type is "__main__.Desc[builtins.str]" + +@my_dataclass() +class D(C): + y: Desc[int] + +d = D(x='x', y=1) +reveal_type(d.x) # N: Revealed type is "builtins.str" +reveal_type(d.y) # N: Revealed type is "builtins.int" +reveal_type(D.x) # N: Revealed type is "__main__.Desc[builtins.str]" +reveal_type(D.y) # N: Revealed type is "__main__.Desc[builtins.int]" + +@my_dataclass(frozen=True) +class F: + x: Desc[str] = Desc() + +F(x='x') +F(x=1) # E: Argument "x" to "F" has incompatible type "int"; expected "str" +reveal_type(F(x='x').x) # N: Revealed type is "builtins.str" +reveal_type(F.x) # N: Revealed type is "__main__.Desc[builtins.str]" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformGenericDescriptorWithInheritance] +# flags: --python-version 3.11 + +from typing import dataclass_transform, overload, Any, TypeVar, Generic + +@dataclass_transform() +def my_dataclass(cls): ... + +T = TypeVar("T") + +class Desc(Generic[T]): + @overload + def __get__(self, instance: None, owner: Any) -> Desc[T]: ... + @overload + def __get__(self, instance: object, owner: Any) -> T: ... + def __get__(self, instance: object | None, owner: Any) -> Desc | T: ... + + def __set__(self, instance: Any, value: T) -> None: ... + +class Desc2(Desc[str]): + pass + +@my_dataclass +class C: + x: Desc2 + +C(x='x') +C(x=1) # E: Argument "x" to "C" has incompatible type "int"; expected "str" +reveal_type(C(x='x').x) # N: Revealed type is "builtins.str" +reveal_type(C.x) # N: Revealed type is "__main__.Desc[builtins.str]" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformDescriptorWithDifferentGetSetTypes] +# flags: --python-version 3.11 + +from typing import dataclass_transform, overload, Any + +@dataclass_transform() +def my_dataclass(cls): ... + +class Desc: + @overload + def __get__(self, instance: None, owner: Any) -> int: ... + @overload + def __get__(self, instance: object, owner: Any) -> str: ... + def __get__(self, instance, owner): ... + + def __set__(self, instance: Any, value: bytes | None) -> None: ... + +@my_dataclass +class C: + x: Desc + +c = C(x=b'x') +c = C(x=None) +C(x=1) # E: Argument "x" to "C" has incompatible type "int"; expected "Optional[bytes]" +reveal_type(c.x) # N: Revealed type is "builtins.str" +reveal_type(C.x) # N: Revealed type is "builtins.int" +c.x = b'x' +c.x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[bytes]") + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTransformUnsupportedDescriptors] +# flags: --python-version 3.11 + +from typing import dataclass_transform, overload, Any + +@dataclass_transform() +def my_dataclass(cls): ... + +class Desc: + @overload + def __get__(self, instance: None, owner: Any) -> int: ... + @overload + def __get__(self, instance: object, owner: Any) -> str: ... + def __get__(self, instance, owner): ... + + def __set__(*args, **kwargs) -> None: ... + +class Desc2: + @overload + def __get__(self, instance: None, owner: Any) -> int: ... + @overload + def __get__(self, instance: object, owner: Any) -> str: ... + def __get__(self, instance, owner): ... + + @overload + def __set__(self, instance: Any, value: bytes) -> None: ... + @overload + def __set__(self) -> None: ... + def __set__(self, *args, **kawrga) -> None: ... + +@my_dataclass +class C: + x: Desc # E: Unsupported signature for "__set__" in "Desc" + y: Desc2 # E: Unsupported "__set__" in "Desc2" + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test new file mode 100644 index 000000000000..6de428109c72 --- /dev/null +++ b/test-data/unit/check-dataclasses.test @@ -0,0 +1,2542 @@ +[case testDataclassesBasic] +from dataclasses import dataclass + +@dataclass +class Person: + name: str + age: int + + def summary(self): + return "%s is %d years old." % (self.name, self.age) + +reveal_type(Person) # N: Revealed type is "def (name: builtins.str, age: builtins.int) -> __main__.Person" +Person('John', 32) +Person('Jonh', 21, None) # E: Too many arguments for "Person" + +[builtins fixtures/dataclasses.pyi] +[typing fixtures/typing-medium.pyi] + +[case testDataclassesCustomInit] +from dataclasses import dataclass + +@dataclass +class A: + a: int + + def __init__(self, a: str) -> None: + pass + +A('1') + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesBasicInheritance] +from dataclasses import dataclass + +@dataclass +class Mammal: + age: int + +@dataclass +class Person(Mammal): + name: str + + def summary(self): + return "%s is %d years old." % (self.name, self.age) + +reveal_type(Person) # N: Revealed type is "def (age: builtins.int, name: builtins.str) -> __main__.Person" +Mammal(10) +Person(32, 'John') +Person(21, 'Jonh', None) # E: Too many arguments for "Person" + +[builtins fixtures/dataclasses.pyi] +[typing fixtures/typing-medium.pyi] + +[case testDataclassesDeepInheritance] +from dataclasses import dataclass + +@dataclass +class A: + a: int + +@dataclass +class B(A): + b: int + +@dataclass +class C(B): + c: int + +@dataclass +class D(C): + d: int + +reveal_type(A) # N: Revealed type is "def (a: builtins.int) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (a: builtins.int, b: builtins.int) -> __main__.B" +reveal_type(C) # N: Revealed type is "def (a: builtins.int, b: builtins.int, c: builtins.int) -> __main__.C" +reveal_type(D) # N: Revealed type is "def (a: builtins.int, b: builtins.int, c: builtins.int, d: builtins.int) -> __main__.D" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesMultipleInheritance] +from dataclasses import dataclass, field, InitVar +@dataclass +class A: + a: bool + +@dataclass +class B: + b: InitVar[bool] + _b: bool = field(init=False) + + def __post_init__(self, b: bool): + self._b = b + +@dataclass +class C(A, B): + pass + +reveal_type(C) # N: Revealed type is "def (b: builtins.bool, a: builtins.bool) -> __main__.C" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesDeepInitVarInheritance] +from dataclasses import dataclass, field, InitVar +@dataclass +class A: + a: bool + +@dataclass +class B: + b: InitVar[bool] + _b: bool = field(init=False) + + def __post_init__(self, b: bool): + self._b = b + +@dataclass(init=False) +class C(B): + def __init__(self): + super().__init__(True) + +@dataclass +class D(C): + pass + +reveal_type(C) # N: Revealed type is "def () -> __main__.C" +reveal_type(D) # N: Revealed type is "def (b: builtins.bool) -> __main__.D" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOverriding] +from dataclasses import dataclass + +@dataclass +class Mammal: + age: int + +@dataclass +class Person(Mammal): + name: str + age: int + +@dataclass +class SpecialPerson(Person): + special_factor: float + +@dataclass +class ExtraSpecialPerson(SpecialPerson): + age: int + special_factor: float + name: str + +reveal_type(Person) # N: Revealed type is "def (age: builtins.int, name: builtins.str) -> __main__.Person" +reveal_type(SpecialPerson) # N: Revealed type is "def (age: builtins.int, name: builtins.str, special_factor: builtins.float) -> __main__.SpecialPerson" +reveal_type(ExtraSpecialPerson) # N: Revealed type is "def (age: builtins.int, name: builtins.str, special_factor: builtins.float) -> __main__.ExtraSpecialPerson" +Person(32, 'John') +Person(21, 'John', None) # E: Too many arguments for "Person" +SpecialPerson(21, 'John', 0.5) +ExtraSpecialPerson(21, 'John', 0.5) + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOverridingWithDefaults] +# Issue #5681 https://github.com/python/mypy/issues/5681 +from dataclasses import dataclass +from typing import Any + +@dataclass +class Base: + some_int: Any + some_str: str = 'foo' + + +@dataclass +class C(Base): + some_int: int + +reveal_type(C) # N: Revealed type is "def (some_int: builtins.int, some_str: builtins.str =) -> __main__.C" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassIncompatibleOverrides] +from dataclasses import dataclass + +@dataclass +class Base: + foo: int + +@dataclass +class BadDerived1(Base): + def foo(self) -> int: # E: Dataclass attribute may only be overridden by another attribute \ + # E: Signature of "foo" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: def foo(self) -> int + return 1 + +@dataclass +class BadDerived2(Base): + @property # E: Dataclass attribute may only be overridden by another attribute + def foo(self) -> int: # E: Cannot override writeable attribute with read-only property + return 2 + +@dataclass +class BadDerived3(Base): + class foo: pass # E: Dataclass attribute may only be overridden by another attribute +[builtins fixtures/dataclasses.pyi] + +[case testDataclassMultipleInheritance] +from dataclasses import dataclass + +class Unrelated: + foo: str + +@dataclass +class Base: + bar: int + +@dataclass +class Derived(Base, Unrelated): + pass + +d = Derived(3) +reveal_type(d.foo) # N: Revealed type is "builtins.str" +reveal_type(d.bar) # N: Revealed type is "builtins.int" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassIncompatibleFrozenOverride] +from dataclasses import dataclass + +@dataclass(frozen=True) +class Base: + foo: int + +@dataclass(frozen=True) +class BadDerived(Base): + @property # E: Dataclass attribute may only be overridden by another attribute + def foo(self) -> int: + return 3 +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesFreezing] +from dataclasses import dataclass + +@dataclass(frozen=True) +class Person: + name: str + +john = Person('John') +john.name = 'Ben' # E: Property "name" defined in "Person" is read-only + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesInconsistentFreezing] +from dataclasses import dataclass + +@dataclass(frozen=True) +class FrozenBase: + pass + +@dataclass +class BadNormalDerived(FrozenBase): # E: Cannot inherit non-frozen dataclass from a frozen one + pass + +@dataclass +class NormalBase: + pass + +@dataclass(frozen=True) +class BadFrozenDerived(NormalBase): # E: Cannot inherit frozen dataclass from a non-frozen one + pass + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesFields] +from dataclasses import dataclass, field + +@dataclass +class Person: + name: str + age: int = field(default=0, init=False) + +reveal_type(Person) # N: Revealed type is "def (name: builtins.str) -> __main__.Person" +john = Person('John') +john.age = 'invalid' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +john.age = 24 + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesBadInit] +from dataclasses import dataclass, field + +@dataclass +class Person: + name: str + age: int = field(init=None) # E: No overload variant of "field" matches argument type "None" \ + # N: Possible overload variants: \ + # N: def [_T] field(*, default: _T, init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> _T \ + # N: def [_T] field(*, default_factory: Callable[[], _T], init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> _T \ + # N: def field(*, init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> Any + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesMultiInit] +from dataclasses import dataclass, field +from typing import List + +@dataclass +class Person: + name: str + age: int = field(init=False) + friend_names: List[str] = field(init=True) + enemy_names: List[str] + +reveal_type(Person) # N: Revealed type is "def (name: builtins.str, friend_names: builtins.list[builtins.str], enemy_names: builtins.list[builtins.str]) -> __main__.Person" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesMultiInitDefaults] +from dataclasses import dataclass, field +from typing import List, Optional + +@dataclass +class Person: + name: str + age: int = field(init=False) + friend_names: List[str] = field(init=True) + enemy_names: List[str] + nickname: Optional[str] = None + +reveal_type(Person) # N: Revealed type is "def (name: builtins.str, friend_names: builtins.list[builtins.str], enemy_names: builtins.list[builtins.str], nickname: Union[builtins.str, None] =) -> __main__.Person" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesDefaults] +from dataclasses import dataclass + +@dataclass +class Application: + name: str = 'Unnamed' + rating: int = 0 + +reveal_type(Application) # N: Revealed type is "def (name: builtins.str =, rating: builtins.int =) -> __main__.Application" +app = Application() + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesDefaultFactories] +from dataclasses import dataclass, field + +@dataclass +class Application: + name: str = 'Unnamed' + rating: int = field(default_factory=int) + rating_count: int = field() # E: Attributes without a default cannot follow attributes with one + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesDefaultFactoryTypeChecking] +from dataclasses import dataclass, field + +@dataclass +class Application: + name: str = 'Unnamed' + rating: int = field(default_factory=str) # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesDefaultOrdering] +from dataclasses import dataclass + +@dataclass +class Application: + name: str = 'Unnamed' + rating: int # E: Attributes without a default cannot follow attributes with one + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOrderingKwOnly] +# flags: --python-version 3.10 +from dataclasses import dataclass + +@dataclass(kw_only=True) +class Application: + name: str = 'Unnamed' + rating: int + +Application(rating=5) +Application(name='name', rating=5) +Application() # E: Missing named argument "rating" for "Application" +Application('name') # E: Too many positional arguments for "Application" # E: Missing named argument "rating" for "Application" +Application('name', 123) # E: Too many positional arguments for "Application" +Application('name', rating=123) # E: Too many positional arguments for "Application" +Application(name=123, rating='name') # E: Argument "name" to "Application" has incompatible type "int"; expected "str" # E: Argument "rating" to "Application" has incompatible type "str"; expected "int" +Application(rating='name', name=123) # E: Argument "rating" to "Application" has incompatible type "str"; expected "int" # E: Argument "name" to "Application" has incompatible type "int"; expected "str" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOrderingKwOnlyOnField] +# flags: --python-version 3.10 +from dataclasses import dataclass, field + +@dataclass +class Application: + name: str = 'Unnamed' + rating: int = field(kw_only=True) + +Application(rating=5) +Application('name', rating=123) +Application(name='name', rating=5) +Application() # E: Missing named argument "rating" for "Application" +Application('name') # E: Missing named argument "rating" for "Application" +Application('name', 123) # E: Too many positional arguments for "Application" +Application(123, rating='name') # E: Argument 1 to "Application" has incompatible type "int"; expected "str" # E: Argument "rating" to "Application" has incompatible type "str"; expected "int" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOrderingKwOnlyOnFieldFalse] +# flags: --python-version 3.10 +from dataclasses import dataclass, field + +@dataclass +class Application: + name: str = 'Unnamed' + rating: int = field(kw_only=False) # E: Attributes without a default cannot follow attributes with one + +Application(name='name', rating=5) +Application('name', 123) +Application('name', rating=123) +Application() # E: Missing positional argument "name" in call to "Application" +Application('name') # E: Too few arguments for "Application" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOrderingKwOnlyWithSentinel] +# flags: --python-version 3.10 +from dataclasses import dataclass, KW_ONLY + +@dataclass +class Application: + _: KW_ONLY + name: str = 'Unnamed' + rating: int + +Application(rating=5) +Application(name='name', rating=5) +Application() # E: Missing named argument "rating" for "Application" +Application('name') # E: Too many positional arguments for "Application" # E: Missing named argument "rating" for "Application" +Application('name', 123) # E: Too many positional arguments for "Application" +Application('name', rating=123) # E: Too many positional arguments for "Application" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOrderingKwOnlyWithSentinelAndFieldOverride] +# flags: --python-version 3.10 +from dataclasses import dataclass, field, KW_ONLY + +@dataclass +class Application: + _: KW_ONLY + name: str = 'Unnamed' + rating: int = field(kw_only=False) # E: Attributes without a default cannot follow attributes with one + +Application(name='name', rating=5) +Application() # E: Missing positional argument "name" in call to "Application" +Application('name') # E: Too many positional arguments for "Application" # E: Too few arguments for "Application" +Application('name', 123) # E: Too many positional arguments for "Application" +Application('name', rating=123) # E: Too many positional arguments for "Application" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOrderingKwOnlyWithSentinelAndSubclass] +# flags: --python-version 3.10 +from dataclasses import dataclass, field, KW_ONLY + +@dataclass +class Base: + x: str + _: KW_ONLY + y: int = 0 + w: int = 1 + +@dataclass +class D(Base): + z: str + a: str = "a" + +D("Hello", "World") +D(x="Hello", z="World") +D("Hello", "World", y=1, w=2, a="b") +D("Hello") # E: Missing positional argument "z" in call to "D" +D() # E: Missing positional arguments "x", "z" in call to "D" +D(123, "World") # E: Argument 1 to "D" has incompatible type "int"; expected "str" +D("Hello", False) # E: Argument 2 to "D" has incompatible type "bool"; expected "str" +D(123, False) # E: Argument 1 to "D" has incompatible type "int"; expected "str" # E: Argument 2 to "D" has incompatible type "bool"; expected "str" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOrderingKwOnlyWithMultipleSentinel] +# flags: --python-version 3.10 +from dataclasses import dataclass, field, KW_ONLY + +@dataclass +class Base: + x: str + _: KW_ONLY + y: int = 0 + __: KW_ONLY # E: There may not be more than one field with the KW_ONLY type + w: int = 1 + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesClassmethods] +from dataclasses import dataclass + +@dataclass +class Application: + name: str + + @classmethod + def parse(cls, request: str) -> "Application": + return cls(name='...') + +app = Application.parse('') + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesOverloadsAndClassmethods] +from dataclasses import dataclass +from typing import overload, Union + +@dataclass +class A: + a: int + b: str + + @classmethod + def other(cls) -> str: + return "..." + + @overload + @classmethod + def foo(cls, x: int) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x: Union[int, str]) -> Union[int, str]: + reveal_type(cls) # N: Revealed type is "Type[__main__.A]" + reveal_type(cls.other()) # N: Revealed type is "builtins.str" + return x + +reveal_type(A.foo(3)) # N: Revealed type is "builtins.int" +reveal_type(A.foo("foo")) # N: Revealed type is "builtins.str" + +[builtins fixtures/dataclasses.pyi] + +[case testClassmethodShadowingFieldDoesNotCrash] +from dataclasses import dataclass + +# This used to crash -- see #6217 +@dataclass +class Foo: + bar: str + @classmethod # E: Name "bar" already defined on line 6 + def bar(cls) -> "Foo": + return cls('asdf') +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesClassVars] +from dataclasses import dataclass +from typing import ClassVar + +@dataclass +class Application: + name: str + + COUNTER: ClassVar[int] = 0 + +reveal_type(Application) # N: Revealed type is "def (name: builtins.str) -> __main__.Application" +application = Application("example") +application.COUNTER = 1 # E: Cannot assign to class variable "COUNTER" via instance +Application.COUNTER = 1 + +[builtins fixtures/dataclasses.pyi] + +[case testTypeAliasInDataclassDoesNotCrash] +from dataclasses import dataclass +from typing import Callable +from typing_extensions import TypeAlias + +@dataclass +class Foo: + x: int + +@dataclass +class One: + S: TypeAlias = Foo # E: Type aliases inside dataclass definitions are not supported at runtime + +a = One() +reveal_type(a.S) # N: Revealed type is "def (x: builtins.int) -> __main__.Foo" +a.S() # E: Missing positional argument "x" in call to "Foo" +reveal_type(a.S(5)) # N: Revealed type is "__main__.Foo" + +@dataclass +class Two: + S: TypeAlias = Callable[[int], str] # E: Type aliases inside dataclass definitions are not supported at runtime + +c = Two() +x = c.S +reveal_type(x) # N: Revealed type is "typing._SpecialForm" +[builtins fixtures/dataclasses.pyi] +[typing fixtures/typing-medium.pyi] + +[case testDataclassOrdering] +from dataclasses import dataclass + +@dataclass(order=True) +class Application: + name: str + rating: int + +app1 = Application('example-1', 5) +app2 = Application('example-2', 5) +app1 < app2 +app1 > app2 +app1 <= app2 +app1 >= app2 +app1 < 5 # E: Unsupported operand types for < ("Application" and "int") +app1 > 5 # E: Unsupported operand types for > ("Application" and "int") +app1 <= 5 # E: Unsupported operand types for <= ("Application" and "int") +app1 >= 5 # E: Unsupported operand types for >= ("Application" and "int") + +class SpecializedApplication(Application): + ... + +app3 = SpecializedApplication('example-3', 5) +app1 < app3 +app1 > app3 +app1 <= app3 +app1 >= app3 + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassOrderingWithoutEquality] +from dataclasses import dataclass + +@dataclass(eq=False, order=True) # E: "eq" must be True if "order" is True +class Application: + ... + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassOrderingWithCustomMethods] +from dataclasses import dataclass + +@dataclass(order=True) +class Application: + def __lt__(self, other: 'Application') -> bool: # E: You may not have a custom "__lt__" method when "order" is True + ... + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassDefaultsInheritance] +from dataclasses import dataclass +from typing import Optional + +@dataclass(order=True) +class Application: + id: Optional[int] + name: str + +@dataclass +class SpecializedApplication(Application): + rating: int = 0 + +reveal_type(SpecializedApplication) # N: Revealed type is "def (id: Union[builtins.int, None], name: builtins.str, rating: builtins.int =) -> __main__.SpecializedApplication" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenerics] +from dataclasses import dataclass +from typing import Generic, List, Optional, TypeVar + +T = TypeVar('T') + +@dataclass +class A(Generic[T]): + x: T + y: T + z: List[T] + + def foo(self) -> List[T]: + return [self.x, self.y] + + def bar(self) -> T: + return self.z[0] + + def problem(self) -> T: + return self.z # E: Incompatible return value type (got "List[T]", expected "T") + +reveal_type(A) # N: Revealed type is "def [T] (x: T`1, y: T`1, z: builtins.list[T`1]) -> __main__.A[T`1]" +A(1, 2, ["a", "b"]) # E: Cannot infer type argument 1 of "A" +a = A(1, 2, [1, 2]) +reveal_type(a) # N: Revealed type is "__main__.A[builtins.int]" +reveal_type(a.x) # N: Revealed type is "builtins.int" +reveal_type(a.y) # N: Revealed type is "builtins.int" +reveal_type(a.z) # N: Revealed type is "builtins.list[builtins.int]" +s: str = a.bar() # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericCovariant] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T_co = TypeVar("T_co", covariant=True) + +@dataclass +class MyDataclass(Generic[T_co]): + a: T_co + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassUntypedGenericInheritance] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar("T") + +@dataclass +class Base(Generic[T]): + attr: T + +@dataclass +class Sub(Base): + pass + +sub = Sub(attr=1) +reveal_type(sub) # N: Revealed type is "__main__.Sub" +reveal_type(sub.attr) # N: Revealed type is "Any" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericSubtype] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar("T") + +@dataclass +class Base(Generic[T]): + attr: T + +S = TypeVar("S") + +@dataclass +class Sub(Base[S]): + pass + +sub_int = Sub[int](attr=1) +reveal_type(sub_int) # N: Revealed type is "__main__.Sub[builtins.int]" +reveal_type(sub_int.attr) # N: Revealed type is "builtins.int" + +sub_str = Sub[str](attr='ok') +reveal_type(sub_str) # N: Revealed type is "__main__.Sub[builtins.str]" +reveal_type(sub_str.attr) # N: Revealed type is "builtins.str" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericInheritance] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") + +@dataclass +class Base(Generic[T1, T2, T3]): + one: T1 + two: T2 + three: T3 + +@dataclass +class Sub(Base[int, str, float]): + pass + +sub = Sub(one=1, two='ok', three=3.14) +reveal_type(sub) # N: Revealed type is "__main__.Sub" +reveal_type(sub.one) # N: Revealed type is "builtins.int" +reveal_type(sub.two) # N: Revealed type is "builtins.str" +reveal_type(sub.three) # N: Revealed type is "builtins.float" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassMultiGenericInheritance] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar("T") + +@dataclass +class Base(Generic[T]): + base_attr: T + +S = TypeVar("S") + +@dataclass +class Middle(Base[int], Generic[S]): + middle_attr: S + +@dataclass +class Sub(Middle[str]): + pass + +sub = Sub(base_attr=1, middle_attr='ok') +reveal_type(sub) # N: Revealed type is "__main__.Sub" +reveal_type(sub.base_attr) # N: Revealed type is "builtins.int" +reveal_type(sub.middle_attr) # N: Revealed type is "builtins.str" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericsClassmethod] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar('T') + +@dataclass +class A(Generic[T]): + x: T + + @classmethod + def foo(cls) -> None: + reveal_type(cls) # N: Revealed type is "Type[__main__.A[T`1]]" + cls.x # E: Access to generic instance variables via class is ambiguous + + @classmethod + def other(cls, x: T) -> A[T]: ... + +reveal_type(A(0).other) # N: Revealed type is "def (x: builtins.int) -> __main__.A[builtins.int]" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesForwardRefs] +from dataclasses import dataclass + +@dataclass +class A: + b: 'B' + +@dataclass +class B: + x: int + +reveal_type(A) # N: Revealed type is "def (b: __main__.B) -> __main__.A" +A(b=B(42)) +A(b=42) # E: Argument "b" to "A" has incompatible type "int"; expected "B" + +[builtins fixtures/dataclasses.pyi] + + +[case testDataclassesInitVars] +from dataclasses import InitVar, dataclass + +@dataclass +class Application: + name: str + database_name: InitVar[str] + +reveal_type(Application) # N: Revealed type is "def (name: builtins.str, database_name: builtins.str) -> __main__.Application" +app = Application("example", 42) # E: Argument 2 to "Application" has incompatible type "int"; expected "str" +app = Application("example", "apps") +app.name +app.database_name # E: "Application" has no attribute "database_name" + + +@dataclass +class SpecializedApplication(Application): + rating: int + +reveal_type(SpecializedApplication) # N: Revealed type is "def (name: builtins.str, database_name: builtins.str, rating: builtins.int) -> __main__.SpecializedApplication" +app = SpecializedApplication("example", "apps", "five") # E: Argument 3 to "SpecializedApplication" has incompatible type "str"; expected "int" +app = SpecializedApplication("example", "apps", 5) +app.name +app.rating +app.database_name # E: "SpecializedApplication" has no attribute "database_name" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesInitVarsAndDefer] +from dataclasses import InitVar, dataclass + +defer: Yes + +@dataclass +class Application: + name: str + database_name: InitVar[str] + +reveal_type(Application) # N: Revealed type is "def (name: builtins.str, database_name: builtins.str) -> __main__.Application" +app = Application("example", 42) # E: Argument 2 to "Application" has incompatible type "int"; expected "str" +app = Application("example", "apps") +app.name +app.database_name # E: "Application" has no attribute "database_name" + +class Yes: ... +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesNoInitInitVarInheritance] +from dataclasses import dataclass, field, InitVar + +@dataclass +class Super: + foo: InitVar = field(init=False) + +@dataclass +class Sub(Super): + bar: int + +sub = Sub(5) +sub.foo # E: "Sub" has no attribute "foo" +sub.bar +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFactory] +from typing import Type, TypeVar +from dataclasses import dataclass + +T = TypeVar('T', bound='A') + +@dataclass +class A: + @classmethod + def make(cls: Type[T]) -> T: + reveal_type(cls) # N: Revealed type is "Type[T`-1]" + reveal_type(cls()) # N: Revealed type is "T`-1" + return cls() +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesInitVarOverride] +import dataclasses + +@dataclasses.dataclass +class A: + a: dataclasses.InitVar[int] + _a: int = dataclasses.field(init=False) + + def __post_init__(self, a): + self._a = a + +@dataclasses.dataclass(init=False) +class B(A): + b: dataclasses.InitVar[int] + _b: int = dataclasses.field(init=False) + + def __init__(self, b): + super().__init__(b+1) + self._b = b + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesInitVarNoOverride] +import dataclasses + +@dataclasses.dataclass +class A: + a: dataclasses.InitVar[int] + _a: int = dataclasses.field(init=False) + + def __post_init__(self, a): + self._a = a + +@dataclasses.dataclass(init=True) +class B(A): + b: dataclasses.InitVar[int] + _b: int = dataclasses.field(init=False) + + def __post_init__(self, a, b): + self._a = a + self._b = b + +B(1, 2) +B(1, 'a') # E: Argument 2 to "B" has incompatible type "str"; expected "int" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesInitVarPostInitOverride] +import dataclasses + +@dataclasses.dataclass +class A: + a: dataclasses.InitVar[int] + _a: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + self._a = a + +@dataclasses.dataclass +class B(A): + b: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + super().__post_init__(a) + self.b = a + 1 + +@dataclasses.dataclass(init=False) +class C(B): + c: int + + def __init__(self, a: int, c: int) -> None: + super().__init__(a) + self.c = c + self.b + +A(1) +B(1) +B(1, 2) # E: Too many arguments for "B" +C(1, 2) +C(1, 'a') # E: Argument 2 to "C" has incompatible type "str"; expected "int" + +[builtins fixtures/primitives.pyi] + +[case testDataclassesInitVarIncremental] +import a + +[file a.py] +import dataclasses +from b import A + +@dataclasses.dataclass +class B(A): + b: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + super().__post_init__(a) + self.b = a + 1 + +[file a.py.2] +import dataclasses +from b import A + +@dataclasses.dataclass +class B(A): + b: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + super().__post_init__(a) + self.b = a + 2 + +reveal_type(B) + +[file b.py] +import dataclasses + +@dataclasses.dataclass +class A: + a: dataclasses.InitVar[int] + _a: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + self._a = a +[out2] +tmp/a.py:12: note: Revealed type is "def (a: builtins.int) -> a.B" + +[builtins fixtures/primitives.pyi] + + +[case testNoComplainFieldNone] +# flags: --no-strict-optional +from dataclasses import dataclass, field +from typing import Optional + +@dataclass +class Foo: + bar: Optional[int] = field(default=None) +[builtins fixtures/dataclasses.pyi] +[out] + +[case testNoComplainFieldNoneStrict] +from dataclasses import dataclass, field +from typing import Optional + +@dataclass +class Foo: + bar: Optional[int] = field(default=None) +[builtins fixtures/dataclasses.pyi] +[out] + +[case testDisallowUntypedWorksForward] +# flags: --disallow-untyped-defs +from dataclasses import dataclass +from typing import List + +@dataclass +class B: + x: C + +class C(List[C]): + pass + +reveal_type(B) # N: Revealed type is "def (x: __main__.C) -> __main__.B" +[builtins fixtures/dataclasses.pyi] + +[case testDisallowUntypedWorksForwardBad] +# flags: --disallow-untyped-defs +from dataclasses import dataclass + +@dataclass +class B: + x: Undefined # E: Name "Undefined" is not defined + y = undefined() # E: Name "undefined" is not defined + +reveal_type(B) # N: Revealed type is "def (x: Any) -> __main__.B" +[builtins fixtures/dataclasses.pyi] + +[case testMemberExprWorksAsField] +import dataclasses + +@dataclasses.dataclass +class A: + x: int = dataclasses.field(metadata={"doc": "foo"}) + y: str + +@dataclasses.dataclass +class B: + x: int = dataclasses.field(init=False, default=1) + y: str + +@dataclasses.dataclass +class C: + x: int = dataclasses.field(default=1) + y: str = dataclasses.field(metadata={"doc": "foo"}) # E: Attributes without a default cannot follow attributes with one +[builtins fixtures/dict.pyi] + +[case testDataclassOrderingDeferred] +from dataclasses import dataclass + +defer: Yes + +@dataclass(order=True) +class Application: + name: str + rating: int + +a = Application('', 0) +b = Application('', 0) +a < b + +class Yes: ... +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFieldDeferred] +from dataclasses import field, dataclass + +@dataclass +class C: + x: int = field(default=func()) + +def func() -> int: ... +C('no') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFieldDeferredFrozen] +from dataclasses import field, dataclass + +@dataclass(frozen=True) +class C: + x: int = field(default=func()) + +def func() -> int: ... +c: C +c.x = 1 # E: Property "x" defined in "C" is read-only +[builtins fixtures/dataclasses.pyi] + +[case testTypeInDataclassDeferredStar] +import lib +[file lib.py] +from dataclasses import dataclass +MYPY = False +if MYPY: # Force deferral + from other import * + +@dataclass +class C: + total: int + +C() # E: Missing positional argument "total" in call to "C" +C('no') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +[file other.py] +import lib +[builtins fixtures/dataclasses.pyi] + +[case testDeferredDataclassInitSignature] +# flags: --no-strict-optional +from dataclasses import dataclass +from typing import Optional, Type + +@dataclass +class C: + x: Optional[int] = None + y: Type[Deferred] = Deferred + + @classmethod + def default(cls) -> C: + return cls(x=None, y=None) + +class Deferred: pass +[builtins fixtures/dataclasses.pyi] + +[case testDeferredDataclassInitSignatureSubclass] +from dataclasses import dataclass +from typing import Optional + +@dataclass +class B: + x: Optional[C] + +@dataclass +class C(B): + y: str + +a = C(None, 'abc') +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesDefaultsIncremental] +import a + +[file a.py] +from dataclasses import dataclass +from b import Person + +@dataclass +class Asdf(Person): + c: str = 'test' + +[file a.py.2] +from dataclasses import dataclass +from b import Person + +@dataclass +class Asdf(Person): + c: str = 'test' + +# asdf + +[file b.py] +from dataclasses import dataclass + +@dataclass +class Person: + b: int + a: str = 'test' + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesDefaultsMroOtherFile] +import a + +[file a.py] +from dataclasses import dataclass +from b import A1, A2 + +@dataclass +class Asdf(A1, A2): # E: Attributes without a default cannot follow attributes with one + pass + +[file b.py] +from dataclasses import dataclass + +# a bunch of blank lines to make sure the error doesn't accidentally line up... + + + +@dataclass +class A1: + a: int + +@dataclass +class A2: + b: str = 'test' + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesInheritingDuplicateField] +# see mypy issue #7792 +from dataclasses import dataclass + +@dataclass +class A: + x: int = 0 + x: int = 0 # E: Name "x" already defined on line 6 + +@dataclass +class B(A): + pass + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassInheritanceNoAnnotation] +from dataclasses import dataclass + +@dataclass +class A: + foo: int + +x = 0 +@dataclass +class B(A): + foo = x + +reveal_type(B) # N: Revealed type is "def (foo: builtins.int) -> __main__.B" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassInheritanceNoAnnotation2] +from dataclasses import dataclass + +@dataclass(frozen=True) +class A: + foo: int + +@dataclass(frozen=True) +class B(A): + @property # E: Dataclass attribute may only be overridden by another attribute + def foo(self) -> int: pass + +reveal_type(B) # N: Revealed type is "def (foo: builtins.int) -> __main__.B" + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassHasAttributeWithFields] +from dataclasses import dataclass + +@dataclass +class A: + pass + +reveal_type(A.__dataclass_fields__) # N: Revealed type is "builtins.dict[builtins.str, dataclasses.Field[Any]]" + +[builtins fixtures/dict.pyi] + +[case testDataclassCallableFieldAccess] +from dataclasses import dataclass +from typing import Callable + +@dataclass +class A: + x: Callable[[int], int] + y: Callable[[int], int] = lambda i: i + +a = A(lambda i:i) +x: int = a.x(0) +y: str = a.y(0) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +reveal_type(a.x) # N: Revealed type is "def (builtins.int) -> builtins.int" +reveal_type(a.y) # N: Revealed type is "def (builtins.int) -> builtins.int" +reveal_type(A.y) # N: Revealed type is "def (builtins.int) -> builtins.int" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassCallableFieldAssignment] +from dataclasses import dataclass +from typing import Callable + +@dataclass +class A: + x: Callable[[int], int] + +def x(i: int) -> int: + return i +def x2(s: str) -> str: + return s + +a = A(lambda i:i) +a.x = x +a.x = x2 # E: Incompatible types in assignment (expression has type "Callable[[str], str]", variable has type "Callable[[int], int]") +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFieldDoesNotFailOnKwargsUnpacking] +# https://github.com/python/mypy/issues/10879 +from dataclasses import dataclass, field + +@dataclass +class Foo: + bar: float = field(**{"repr": False}) +[out] +main:6: error: Unpacking **kwargs in "field()" is not supported +main:6: error: No overload variant of "field" matches argument type "Dict[str, bool]" +main:6: note: Possible overload variants: +main:6: note: def [_T] field(*, default: _T, init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> _T +main:6: note: def [_T] field(*, default_factory: Callable[[], _T], init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> _T +main:6: note: def field(*, init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> Any +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFieldWithPositionalArguments] +from dataclasses import dataclass, field + +@dataclass +class C: + x: int = field(0) # E: "field()" does not accept positional arguments \ + # E: No overload variant of "field" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def [_T] field(*, default: _T, init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> _T \ + # N: def [_T] field(*, default_factory: Callable[[], _T], init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> _T \ + # N: def field(*, init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...) -> Any +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFieldWithTypedDictUnpacking] +from dataclasses import dataclass, field +from typing_extensions import TypedDict + +class FieldKwargs(TypedDict): + repr: bool + +field_kwargs: FieldKwargs = {"repr": False} + +@dataclass +class Foo: + bar: float = field(**field_kwargs) # E: Unpacking **kwargs in "field()" is not supported + +reveal_type(Foo(bar=1.5)) # N: Revealed type is "__main__.Foo" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithSlotsArg] +# flags: --python-version 3.10 +from dataclasses import dataclass + +@dataclass(slots=True) +class Some: + x: int + + def __init__(self, x: int) -> None: + self.x = x + self.y = 0 # E: Trying to assign name "y" that is not in "__slots__" of type "__main__.Some" + + def __post_init__(self) -> None: + self.y = 1 # E: Trying to assign name "y" that is not in "__slots__" of type "__main__.Some" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithSlotsDef] +# flags: --python-version 3.10 +from dataclasses import dataclass + +@dataclass(slots=False) +class Some: + __slots__ = ('x',) + x: int + + def __init__(self, x: int) -> None: + self.x = x + self.y = 0 # E: Trying to assign name "y" that is not in "__slots__" of type "__main__.Some" + + def __post_init__(self) -> None: + self.y = 1 # E: Trying to assign name "y" that is not in "__slots__" of type "__main__.Some" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithSlotsDerivedFromNonSlot] +# flags: --python-version 3.10 +from dataclasses import dataclass + +class A: + pass + +@dataclass(slots=True) +class B(A): + x: int + + def __post_init__(self) -> None: + self.y = 42 + +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithSlotsConflict] +# flags: --python-version 3.10 +from dataclasses import dataclass + +@dataclass(slots=True) +class Some: # E: "Some" both defines "__slots__" and is used with "slots=True" + __slots__ = ('x',) + x: int + +@dataclass(slots=True) +class EmptyDef: # E: "EmptyDef" both defines "__slots__" and is used with "slots=True" + __slots__ = () + x: int + +slots = ('x',) + +@dataclass(slots=True) +class DynamicDef: # E: "DynamicDef" both defines "__slots__" and is used with "slots=True" + __slots__ = slots + x: int +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithSlotsArgBefore310] +# flags: --python-version 3.9 +from dataclasses import dataclass + +@dataclass(slots=True) # E: Keyword argument "slots" for "dataclass" is only valid in Python 3.10 and higher +class Some: + x: int + +# Possible conflict: +@dataclass(slots=True) # E: Keyword argument "slots" for "dataclass" is only valid in Python 3.10 and higher +class Other: + __slots__ = ('x',) + x: int +[builtins fixtures/dataclasses.pyi] + + +[case testDataclassWithSlotsRuntimeAttr] +# flags: --python-version 3.10 +from dataclasses import dataclass + +@dataclass(slots=True) +class Some: + x: int + y: str + z: bool + +reveal_type(Some.__slots__) # N: Revealed type is "Tuple[builtins.str, builtins.str, builtins.str]" + +@dataclass(slots=True) +class Other: + x: int + y: str + +reveal_type(Other.__slots__) # N: Revealed type is "Tuple[builtins.str, builtins.str]" + + +@dataclass +class NoSlots: + x: int + y: str + +NoSlots.__slots__ # E: "Type[NoSlots]" has no attribute "__slots__" +[builtins fixtures/dataclasses.pyi] + + +[case testSlotsDefinitionWithTwoPasses1] +# flags: --python-version 3.10 +# https://github.com/python/mypy/issues/11821 +from typing import TypeVar, Protocol, Generic +from dataclasses import dataclass + +C = TypeVar("C", bound="Comparable") + +class Comparable(Protocol): + pass + +V = TypeVar("V", bound=Comparable) + +@dataclass(slots=True) +class Node(Generic[V]): # Error was here + data: V +[builtins fixtures/dataclasses.pyi] + +[case testSlotsDefinitionWithTwoPasses2] +# flags: --python-version 3.10 +from typing import TypeVar, Protocol, Generic +from dataclasses import dataclass + +C = TypeVar("C", bound="Comparable") + +class Comparable(Protocol): + pass + +V = TypeVar("V", bound=Comparable) + +@dataclass(slots=True) # Explicit slots are still not ok: +class Node(Generic[V]): # E: "Node" both defines "__slots__" and is used with "slots=True" + __slots__ = ('data',) + data: V +[builtins fixtures/dataclasses.pyi] + +[case testSlotsDefinitionWithTwoPasses3] +# flags: --python-version 3.10 +from typing import TypeVar, Protocol, Generic +from dataclasses import dataclass + +C = TypeVar("C", bound="Comparable") + +class Comparable(Protocol): + pass + +V = TypeVar("V", bound=Comparable) + +@dataclass(slots=True) # Explicit slots are still not ok, even empty ones: +class Node(Generic[V]): # E: "Node" both defines "__slots__" and is used with "slots=True" + __slots__ = () + data: V +[builtins fixtures/dataclasses.pyi] + +[case testSlotsDefinitionWithTwoPasses4] +# flags: --python-version 3.10 +import dataclasses as dtc + +PublishedMessagesVar = dict[int, 'PublishedMessages'] + +@dtc.dataclass(frozen=True, slots=True) +class PublishedMessages: + left: int +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesAnyInherit] +from dataclasses import dataclass +from typing import Any +B: Any +@dataclass +class A(B): + a: int +@dataclass +class C(B): + generated_args: int + generated_kwargs: int + +A(a=1, b=2) +A(1) +A(a="foo") # E: Argument "a" to "A" has incompatible type "str"; expected "int" +C(generated_args="foo", generated_kwargs="bar") # E: Argument "generated_args" to "C" has incompatible type "str"; expected "int" \ + # E: Argument "generated_kwargs" to "C" has incompatible type "str"; expected "int" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesCallableFrozen] +from dataclasses import dataclass +from typing import Any, Callable +@dataclass(frozen=True) +class A: + a: Callable[..., None] + +def func() -> None: + pass + +reveal_type(A.a) # N: Revealed type is "def (*Any, **Any)" +A(a=func).a() +A(a=func).a = func # E: Property "a" defined in "A" is read-only +[builtins fixtures/dataclasses.pyi] + +[case testDataclassInFunctionDoesNotCrash] +from dataclasses import dataclass + +def foo() -> None: + @dataclass + class Foo: + foo: int + # This used to crash (see #8703) + # The return type of __call__ here needs to be something undefined + # In order to trigger the crash that existed prior to #12762 + def __call__(self) -> asdf: ... # E: Name "asdf" is not defined +[builtins fixtures/dataclasses.pyi] + +[case testDataclassesMultipleInheritanceWithNonDataclass] +# flags: --python-version 3.10 +from dataclasses import dataclass + +@dataclass +class A: + prop_a: str + +@dataclass +class B: + prop_b: bool + +class Derived(A, B): + pass +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericInheritance2] +from dataclasses import dataclass +from typing import Any, Callable, Generic, TypeVar, List + +T = TypeVar("T") +S = TypeVar("S") + +@dataclass +class Parent(Generic[T]): + f: Callable[[T], Any] + +@dataclass +class Child(Parent[T]): ... + +class A: ... +def func(obj: A) -> bool: ... + +reveal_type(Child[A](func).f) # N: Revealed type is "def (__main__.A) -> Any" + +@dataclass +class Parent2(Generic[T]): + a: List[T] + +@dataclass +class Child2(Generic[T, S], Parent2[S]): + b: List[T] + +reveal_type(Child2([A()], [1]).a) # N: Revealed type is "builtins.list[__main__.A]" +reveal_type(Child2[int, A]([A()], [1]).b) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassInheritOptionalType] +from dataclasses import dataclass +from typing import Any, Callable, Generic, TypeVar, List, Optional + +T = TypeVar("T") + +@dataclass +class Parent(Generic[T]): + x: Optional[str] +@dataclass +class Child(Parent): + y: Optional[int] +Child(x=1, y=1) # E: Argument "x" to "Child" has incompatible type "int"; expected "Optional[str]" +Child(x='', y='') # E: Argument "y" to "Child" has incompatible type "str"; expected "Optional[int]" +Child(x='', y=1) +Child(x=None, y=None) +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericInheritanceSpecialCase1] +from dataclasses import dataclass +from typing import Generic, TypeVar, List + +T = TypeVar("T") + +@dataclass +class Parent(Generic[T]): + x: List[T] + +@dataclass +class Child1(Parent["Child2"]): ... + +@dataclass +class Child2(Parent["Child1"]): ... + +def f(c: Child2) -> None: + reveal_type(Child1([c]).x) # N: Revealed type is "builtins.list[__main__.Child2]" + +def g(c: Child1) -> None: + reveal_type(Child2([c]).x) # N: Revealed type is "builtins.list[__main__.Child1]" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericInheritanceSpecialCase2] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar("T") + +# A subclass might be analyzed before base in import cycles. They are +# defined here in reversed order to simulate this. + +@dataclass +class Child1(Parent["Child2"]): + x: int + +@dataclass +class Child2(Parent["Child1"]): + y: int + +@dataclass +class Parent(Generic[T]): + key: str + +Child1(x=1, key='') +Child2(y=1, key='') +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericWithBound] +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar("T", bound="C") + +@dataclass +class C(Generic[T]): + x: int + +c: C[C] +d: C[str] # E: Type argument "str" of "C" must be a subtype of "C[Any]" +C(x=2) +[builtins fixtures/dataclasses.pyi] + +[case testDataclassGenericBoundToInvalidTypeVarDoesNotCrash] +import dataclasses +from typing import Generic, TypeVar + +T = TypeVar("T", bound="NotDefined") # E: Name "NotDefined" is not defined + +@dataclasses.dataclass +class C(Generic[T]): + x: float +[builtins fixtures/dataclasses.pyi] + +[case testDataclassInitVarCannotBeSet] +from dataclasses import dataclass, InitVar + +@dataclass +class C: + x: InitVar[int] = 0 + y: InitVar[str] = '' + + def f(self) -> None: + # This works at runtime, but it seems like an abuse of the InitVar + # feature and thus we don't support it + self.x = 1 # E: "C" has no attribute "x" + self.y: str = 'x' # E: "C" has no attribute "y" + +c = C() +c2 = C(x=1) +c.x # E: "C" has no attribute "x" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassCheckTypeVarBounds] +from dataclasses import dataclass +from typing import ClassVar, Protocol, Dict, TypeVar, Generic + +class DataclassProtocol(Protocol): + __dataclass_fields__: ClassVar[Dict] + +T = TypeVar("T", bound=DataclassProtocol) + +@dataclass +class MyDataclass: + x: int = 1 + +class MyGeneric(Generic[T]): ... +class MyClass(MyGeneric[MyDataclass]): ... +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithMatchArgs] +# flags: --python-version 3.10 +from dataclasses import dataclass +@dataclass +class One: + bar: int + baz: str +o: One +reveal_type(o.__match_args__) # N: Revealed type is "Tuple[Literal['bar'], Literal['baz']]" +@dataclass(match_args=True) +class Two: + bar: int +t: Two +reveal_type(t.__match_args__) # N: Revealed type is "Tuple[Literal['bar']]" +@dataclass +class Empty: + ... +e: Empty +reveal_type(e.__match_args__) # N: Revealed type is "Tuple[()]" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithoutMatchArgs] +# flags: --python-version 3.10 +from dataclasses import dataclass +@dataclass(match_args=False) +class One: + bar: int + baz: str +o: One +reveal_type(o.__match_args__) # E: "One" has no attribute "__match_args__" \ + # N: Revealed type is "Any" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassWithMatchArgsOldVersion] +# flags: --python-version 3.9 +from dataclasses import dataclass +@dataclass(match_args=True) +class One: + bar: int +o: One +reveal_type(o.__match_args__) # E: "One" has no attribute "__match_args__" \ + # N: Revealed type is "Any" +@dataclass +class Two: + bar: int +t: Two +reveal_type(t.__match_args__) # E: "Two" has no attribute "__match_args__" \ + # N: Revealed type is "Any" +[builtins fixtures/dataclasses.pyi] + +[case testFinalInDataclass] +from dataclasses import dataclass +from typing import Final + +@dataclass +class FirstClass: + FIRST_CONST: Final = 3 # OK + +@dataclass +class SecondClass: + SECOND_CONST: Final = FirstClass.FIRST_CONST # E: Need type argument for Final[...] with non-literal default in dataclass + +reveal_type(FirstClass().FIRST_CONST) # N: Revealed type is "Literal[3]?" +FirstClass().FIRST_CONST = 42 # E: Cannot assign to final attribute "FIRST_CONST" +reveal_type(SecondClass().SECOND_CONST) # N: Revealed type is "Literal[3]?" +SecondClass().SECOND_CONST = 42 # E: Cannot assign to final attribute "SECOND_CONST" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFieldsProtocol] +# flags: --python-version 3.9 +from dataclasses import dataclass +from typing import Any, Protocol + +class ConfigProtocol(Protocol): + __dataclass_fields__: dict[str, Any] + +def takes_cp(cp: ConfigProtocol): ... + +@dataclass +class MyDataclass: + x: int = 3 + +takes_cp(MyDataclass) +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTypeAnnotationAliasUpdated] +import a +[file a.py] +from dataclasses import dataclass +from b import B + +@dataclass +class D: + x: B + +reveal_type(D) # N: Revealed type is "def (x: builtins.list[b.C]) -> a.D" +[file b.py] +from typing import List +import a +class CC: ... +class C(CC): ... +B = List[C] +[builtins fixtures/dataclasses.pyi] + +[case testDataclassSelfType] +from dataclasses import dataclass +from typing import Self, TypeVar, Generic, Optional + +T = TypeVar("T") + +@dataclass +class LinkedList(Generic[T]): + value: T + next: Optional[Self] = None + + def meth(self) -> None: + reveal_type(self.next) # N: Revealed type is "Union[Self`0, None]" + +l_int: LinkedList[int] = LinkedList(1, LinkedList("no", None)) # E: Argument 1 to "LinkedList" has incompatible type "str"; expected "int" + +@dataclass +class SubLinkedList(LinkedList[int]): ... + +lst = SubLinkedList(1, LinkedList(2)) # E: Argument 2 to "SubLinkedList" has incompatible type "LinkedList[int]"; expected "Optional[SubLinkedList]" +reveal_type(lst.next) # N: Revealed type is "Union[__main__.SubLinkedList, None]" +reveal_type(SubLinkedList) # N: Revealed type is "def (value: builtins.int, next: Union[__main__.SubLinkedList, None] =) -> __main__.SubLinkedList" +[builtins fixtures/dataclasses.pyi] + +[case testNoCrashOnNestedGenericCallable] +from dataclasses import dataclass +from typing import Generic, TypeVar, Callable + +T = TypeVar('T') +R = TypeVar('R') +X = TypeVar('X') + +@dataclass +class Box(Generic[T]): + inner: T + +@dataclass +class Cont(Generic[R]): + run: Box[Callable[[X], R]] + +def const_two(x: T) -> str: + return "two" + +c = Cont(Box(const_two)) +reveal_type(c) # N: Revealed type is "__main__.Cont[builtins.str]" +[builtins fixtures/dataclasses.pyi] + +[case testNoCrashOnSelfWithForwardRefGenericDataclass] +from typing import Generic, Sequence, TypeVar, Self +from dataclasses import dataclass + +_T = TypeVar('_T', bound="Foo") + +@dataclass +class Foo: + foo: int + +@dataclass +class Element(Generic[_T]): + elements: Sequence[Self] + +@dataclass +class Bar(Foo): ... +e: Element[Bar] +reveal_type(e.elements) # N: Revealed type is "typing.Sequence[__main__.Element[__main__.Bar]]" +[builtins fixtures/dataclasses.pyi] + +[case testIfConditionsInDefinition] +# flags: --python-version 3.11 --always-true TRUTH +from dataclasses import dataclass +from typing import TYPE_CHECKING + +TRUTH = False # Is set to --always-true + +@dataclass +class Foo: + if TYPE_CHECKING: + present_1: int + else: + skipped_1: int + if True: # Mypy does not know if it is True or False, so the block is used + present_2: int + if False: # Mypy does not know if it is True or False, so the block is used + present_3: int + if not TRUTH: + skipped_2: int + elif 123: + present_4: int + elif TRUTH: + present_5: int + else: + skipped_3: int + +Foo( + present_1=1, + present_2=2, + present_3=3, + present_4=4, + present_5=5, +) + +[builtins fixtures/dataclasses.pyi] + +[case testReplace] +from dataclasses import dataclass, replace, InitVar +from typing import ClassVar + +@dataclass +class A: + x: int + q: InitVar[int] + q2: InitVar[int] = 0 + c: ClassVar[int] + + +a = A(x=42, q=7) +a2 = replace(a) # E: Missing named argument "q" for "replace" of "A" +a2 = replace(a, q=42) +a2 = replace(a, x=42, q=42) +a2 = replace(a, x=42, q=42, c=7) # E: Unexpected keyword argument "c" for "replace" of "A" +a2 = replace(a, x='42', q=42) # E: Argument "x" to "replace" of "A" has incompatible type "str"; expected "int" +a2 = replace(a, q='42') # E: Argument "q" to "replace" of "A" has incompatible type "str"; expected "int" +reveal_type(a2) # N: Revealed type is "__main__.A" + +[builtins fixtures/tuple.pyi] + +[case testReplaceUnion] +from typing import Generic, Union, TypeVar +from dataclasses import dataclass, replace, InitVar + +T = TypeVar('T') + +@dataclass +class A(Generic[T]): + x: T # exercises meet(T=int, int) = int + y: bool # exercises meet(bool, int) = bool + z: str # exercises meet(str, bytes) = Never + w: dict # exercises meet(dict, Never) = Never + init_var: InitVar[int] # exercises (non-optional, optional) = non-optional + +@dataclass +class B: + x: int + y: int + z: bytes + init_var: int + + +a_or_b: Union[A[int], B] +_ = replace(a_or_b, x=42, y=True, init_var=42) +_ = replace(a_or_b, x=42, y=True) # E: Missing named argument "init_var" for "replace" of "Union[A[int], B]" +_ = replace(a_or_b, x=42, y=True, z='42', init_var=42) # E: Argument "z" to "replace" of "Union[A[int], B]" has incompatible type "str"; expected "Never" +_ = replace(a_or_b, x=42, y=True, w={}, init_var=42) # E: Argument "w" to "replace" of "Union[A[int], B]" has incompatible type "Dict[Never, Never]"; expected "Never" +_ = replace(a_or_b, y=42, init_var=42) # E: Argument "y" to "replace" of "Union[A[int], B]" has incompatible type "int"; expected "bool" + +[builtins fixtures/tuple.pyi] + +[case testReplaceUnionOfTypeVar] +from typing import Generic, Union, TypeVar +from dataclasses import dataclass, replace + +@dataclass +class A: + x: int + y: int + z: str + w: dict + +class B: + pass + +TA = TypeVar('TA', bound=A) +TB = TypeVar('TB', bound=B) + +def f(b_or_t: Union[TA, TB, int]) -> None: + a2 = replace(b_or_t) # E: Value of type variable "_DataclassT" of "replace" cannot be "Union[TA, TB, int]" + +[builtins fixtures/tuple.pyi] + +[case testReplaceTypeVarBoundNotDataclass] +from dataclasses import dataclass, replace +from typing import Union, TypeVar + +TInt = TypeVar('TInt', bound=int) +TAny = TypeVar('TAny') +TNone = TypeVar('TNone', bound=None) +TUnion = TypeVar('TUnion', bound=Union[str, int]) + +def f1(t: TInt) -> None: + _ = replace(t, x=42) # E: Value of type variable "_DataclassT" of "replace" cannot be "TInt" + +def f2(t: TAny) -> TAny: + return replace(t, x='spam') # E: Value of type variable "_DataclassT" of "replace" cannot be "TAny" + +def f3(t: TNone) -> TNone: + return replace(t, x='spam') # E: Value of type variable "_DataclassT" of "replace" cannot be "TNone" + +def f4(t: TUnion) -> TUnion: + return replace(t, x='spam') # E: Value of type variable "_DataclassT" of "replace" cannot be "TUnion" + +[builtins fixtures/tuple.pyi] + +[case testReplaceTypeVarBound] +from dataclasses import dataclass, replace +from typing import TypeVar + +@dataclass +class A: + x: int + +@dataclass +class B(A): + pass + +TA = TypeVar('TA', bound=A) + +def f(t: TA) -> TA: + t2 = replace(t, x=42) + reveal_type(t2) # N: Revealed type is "TA`-1" + _ = replace(t, x='42') # E: Argument "x" to "replace" of "TA" has incompatible type "str"; expected "int" + return t2 + +f(A(x=42)) +f(B(x=42)) + +[builtins fixtures/tuple.pyi] + +[case testReplaceAny] +from dataclasses import replace +from typing import Any + +a: Any +a2 = replace(a) +reveal_type(a2) # N: Revealed type is "Any" + +[builtins fixtures/tuple.pyi] + +[case testReplaceNotDataclass] +from dataclasses import replace + +replace(5) # E: Value of type variable "_DataclassT" of "replace" cannot be "int" + +class C: + pass + +replace(C()) # E: Value of type variable "_DataclassT" of "replace" cannot be "C" + +replace(None) # E: Value of type variable "_DataclassT" of "replace" cannot be "None" + +[builtins fixtures/tuple.pyi] + +[case testReplaceIsDataclass] +from dataclasses import is_dataclass, replace + +def f(x: object) -> None: + _ = replace(x) # E: Value of type variable "_DataclassT" of "replace" cannot be "object" + if is_dataclass(x): + _ = replace(x) # E: Value of type variable "_DataclassT" of "replace" cannot be "Union[DataclassInstance, Type[DataclassInstance]]" + if not isinstance(x, type): + _ = replace(x) + +[builtins fixtures/tuple.pyi] + +[case testReplaceGeneric] +from dataclasses import dataclass, replace, InitVar +from typing import ClassVar, Generic, TypeVar + +T = TypeVar('T') + +@dataclass +class A(Generic[T]): + x: T + +a = A(x=42) +reveal_type(a) # N: Revealed type is "__main__.A[builtins.int]" +a2 = replace(a, x=42) +reveal_type(a2) # N: Revealed type is "__main__.A[builtins.int]" +a2 = replace(a, x='42') # E: Argument "x" to "replace" of "A[int]" has incompatible type "str"; expected "int" +reveal_type(a2) # N: Revealed type is "__main__.A[builtins.int]" + +[builtins fixtures/tuple.pyi] + +[case testPostInitNotMethod] +def __post_init__() -> None: + pass + +[case testPostInitCorrectSignature] +from typing import Any, Generic, TypeVar, Callable, Self +from dataclasses import dataclass, InitVar + +@dataclass +class Test1: + x: int + def __post_init__(self) -> None: ... + +@dataclass +class Test2: + x: int + y: InitVar[int] + z: str + def __post_init__(self, y: int) -> None: ... + +@dataclass +class Test3: + x: InitVar[int] + y: InitVar[str] + def __post_init__(self, x: int, y: str) -> None: ... + +@dataclass +class Test4: + x: int + y: InitVar[str] + z: InitVar[bool] = True + def __post_init__(self, y: str, z: bool) -> None: ... + +@dataclass +class Test5: + y: InitVar[str] = 'a' + z: InitVar[bool] = True + def __post_init__(self, y: str = 'a', z: bool = True) -> None: ... + +F = TypeVar('F', bound=Callable[..., Any]) +def identity(f: F) -> F: return f + +@dataclass +class Test6: + y: InitVar[str] + @identity # decorated method works + def __post_init__(self, y: str) -> None: ... + +T = TypeVar('T') + +@dataclass +class Test7(Generic[T]): + t: InitVar[T] + def __post_init__(self, t: T) -> None: ... + +@dataclass +class Test8: + s: InitVar[Self] + def __post_init__(self, s: Self) -> None: ... +[builtins fixtures/dataclasses.pyi] + +[case testPostInitSubclassing] +from dataclasses import dataclass, InitVar + +@dataclass +class Base: + a: str + x: InitVar[int] + def __post_init__(self, x: int) -> None: ... + +@dataclass +class Child(Base): + b: str + y: InitVar[str] + def __post_init__(self, x: int, y: str) -> None: ... + +@dataclass +class GrandChild(Child): + c: int + z: InitVar[str] = "a" + def __post_init__(self, x: int, y: str, z: str) -> None: ... +[builtins fixtures/dataclasses.pyi] + +[case testPostInitNotADataclassCheck] +from dataclasses import dataclass, InitVar + +class Regular: + __post_init__ = 1 # can be whatever + +class Base: + x: InitVar[int] + def __post_init__(self) -> None: ... # can be whatever + +@dataclass +class Child(Base): + y: InitVar[str] + def __post_init__(self, y: str) -> None: ... +[builtins fixtures/dataclasses.pyi] + +[case testPostInitMissingParam] +from dataclasses import dataclass, InitVar + +@dataclass +class Child: + y: InitVar[str] + def __post_init__(self) -> None: ... +[builtins fixtures/dataclasses.pyi] +[out] +main:6: error: Signature of "__post_init__" incompatible with supertype "dataclass" +main:6: note: Superclass: +main:6: note: def __post_init__(self: Child, y: str) -> None +main:6: note: Subclass: +main:6: note: def __post_init__(self: Child) -> None + +[case testPostInitWrongTypeAndName] +from dataclasses import dataclass, InitVar + +@dataclass +class Test1: + y: InitVar[str] + def __post_init__(self, x: int) -> None: ... # E: Argument 2 of "__post_init__" is incompatible with supertype "dataclass"; supertype defines the argument type as "str" + +@dataclass +class Test2: + y: InitVar[str] = 'a' + def __post_init__(self, x: int) -> None: ... # E: Argument 2 of "__post_init__" is incompatible with supertype "dataclass"; supertype defines the argument type as "str" +[builtins fixtures/dataclasses.pyi] + +[case testPostInitExtraParam] +from dataclasses import dataclass, InitVar + +@dataclass +class Child: + y: InitVar[str] + def __post_init__(self, y: str, z: int) -> None: ... +[builtins fixtures/dataclasses.pyi] +[out] +main:6: error: Signature of "__post_init__" incompatible with supertype "dataclass" +main:6: note: Superclass: +main:6: note: def __post_init__(self: Child, y: str) -> None +main:6: note: Subclass: +main:6: note: def __post_init__(self: Child, y: str, z: int) -> None + +[case testPostInitReturnType] +from dataclasses import dataclass, InitVar + +@dataclass +class Child: + y: InitVar[str] + def __post_init__(self, y: str) -> int: ... # E: Return type "int" of "__post_init__" incompatible with return type "None" in supertype "dataclass" +[builtins fixtures/dataclasses.pyi] + +[case testPostInitDecoratedMethodError] +from dataclasses import dataclass, InitVar +from typing import Any, Callable, TypeVar + +F = TypeVar('F', bound=Callable[..., Any]) +def identity(f: F) -> F: return f + +@dataclass +class Klass: + y: InitVar[str] + @identity + def __post_init__(self) -> None: ... +[builtins fixtures/dataclasses.pyi] +[out] +main:11: error: Signature of "__post_init__" incompatible with supertype "dataclass" +main:11: note: Superclass: +main:11: note: def __post_init__(self: Klass, y: str) -> None +main:11: note: Subclass: +main:11: note: def __post_init__(self: Klass) -> None + +[case testPostInitIsNotAFunction] +from dataclasses import dataclass, InitVar + +@dataclass +class Test: + y: InitVar[str] + __post_init__ = 1 # E: "__post_init__" method must be an instance method +[builtins fixtures/dataclasses.pyi] + +[case testPostInitClassMethod] +from dataclasses import dataclass, InitVar + +@dataclass +class Test: + y: InitVar[str] + @classmethod + def __post_init__(cls) -> None: ... +[builtins fixtures/dataclasses.pyi] +[out] +main:7: error: Signature of "__post_init__" incompatible with supertype "dataclass" +main:7: note: Superclass: +main:7: note: def __post_init__(self: Test, y: str) -> None +main:7: note: Subclass: +main:7: note: @classmethod +main:7: note: def __post_init__(cls: Type[Test]) -> None + +[case testPostInitStaticMethod] +from dataclasses import dataclass, InitVar + +@dataclass +class Test: + y: InitVar[str] + @staticmethod + def __post_init__() -> None: ... +[builtins fixtures/dataclasses.pyi] +[out] +main:7: error: Signature of "__post_init__" incompatible with supertype "dataclass" +main:7: note: Superclass: +main:7: note: def __post_init__(self: Test, y: str) -> None +main:7: note: Subclass: +main:7: note: @staticmethod +main:7: note: def __post_init__() -> None + +[case testProtocolNoCrash] +from typing import Protocol, Union, ClassVar +from dataclasses import dataclass, field + +DEFAULT = 0 + +@dataclass +class Test(Protocol): + x: int + def reset(self) -> None: + self.x = DEFAULT +[builtins fixtures/dataclasses.pyi] + +[case testProtocolNoCrashOnJoining] +from dataclasses import dataclass +from typing import Protocol + +@dataclass +class MyDataclass(Protocol): ... + +a: MyDataclass +b = [a, a] # trigger joining the types + +[builtins fixtures/dataclasses.pyi] + +[case testPropertyAndFieldRedefinitionNoCrash] +from dataclasses import dataclass + +@dataclass +class Foo: + @property + def c(self) -> int: + return 0 + + c: int # E: Name "c" already defined on line 5 +[builtins fixtures/dataclasses.pyi] + +[case testDataclassInheritanceWorksWithExplicitOverrides] +# flags: --enable-error-code explicit-override +from dataclasses import dataclass + +@dataclass +class Base: + x: int + +@dataclass +class Child(Base): + y: int +[builtins fixtures/dataclasses.pyi] + + +[case testDataclassInheritanceWorksWithExplicitOverridesAndOrdering] +# flags: --enable-error-code explicit-override +from dataclasses import dataclass + +@dataclass(order=True) +class Base: + x: int + +@dataclass(order=True) +class Child(Base): + y: int +[builtins fixtures/dataclasses.pyi] + +[case testDunderReplacePresent] +# flags: --python-version 3.13 +from dataclasses import dataclass, field + +@dataclass +class Coords: + x: int + y: int + # non-init fields are not allowed with replace: + z: int = field(init=False) + + +replaced = Coords(2, 4).__replace__(x=2, y=5) +reveal_type(replaced) # N: Revealed type is "__main__.Coords" + +replaced = Coords(2, 4).__replace__(x=2) +reveal_type(replaced) # N: Revealed type is "__main__.Coords" + +Coords(2, 4).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "Coords" has incompatible type "str"; expected "int" +Coords(2, 4).__replace__(23) # E: Too many positional arguments for "__replace__" of "Coords" +Coords(2, 4).__replace__(23, 25) # E: Too many positional arguments for "__replace__" of "Coords" +Coords(2, 4).__replace__(x=23, y=25, z=42) # E: Unexpected keyword argument "z" for "__replace__" of "Coords" + +from typing import Generic, TypeVar +T = TypeVar('T') + +@dataclass +class Gen(Generic[T]): + x: T + +replaced_2 = Gen(2).__replace__(x=2) +reveal_type(replaced_2) # N: Revealed type is "__main__.Gen[builtins.int]" +Gen(2).__replace__(x="not an int") # E: Argument "x" to "__replace__" of "Gen" has incompatible type "str"; expected "int" + +[builtins fixtures/tuple.pyi] + +[case testDunderReplaceCovariantOverride] +# flags: --python-version 3.13 +from dataclasses import dataclass + +@dataclass +class Base: + a: object + +@dataclass +class Child(Base): # E: Argument 1 of "__replace__" is incompatible with supertype "Base"; supertype defines the argument type as "object" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + a: int +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-deprecated.test b/test-data/unit/check-deprecated.test new file mode 100644 index 000000000000..362d8725f183 --- /dev/null +++ b/test-data/unit/check-deprecated.test @@ -0,0 +1,636 @@ +-- Type checker test cases for reporting deprecations. + + +[case testDeprecatedDisabled] + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f() + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedAsNoteWithErrorCode] +# flags: --enable-error-code=deprecated --show-error-codes --report-deprecated-as-note + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f() # type: ignore[deprecated] +f() # N: function __main__.f is deprecated: use f2 instead [deprecated] + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedAsErrorWithErrorCode] +# flags: --enable-error-code=deprecated --show-error-codes + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f() # type: ignore[deprecated] +f() # E: function __main__.f is deprecated: use f2 instead [deprecated] + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedFunction] +# flags: --enable-error-code=deprecated + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f # E: function __main__.f is deprecated: use f2 instead # type: ignore[deprecated] +f(1) # E: function __main__.f is deprecated: use f2 instead \ + # E: Too many arguments for "f" +f[1] # E: function __main__.f is deprecated: use f2 instead \ + # E: Value of type "Callable[[], None]" is not indexable +g = f # E: function __main__.f is deprecated: use f2 instead +g() +t = (f, f, g) # E: function __main__.f is deprecated: use f2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedFunctionDifferentModule] +# flags: --enable-error-code=deprecated + +import m +import p.s +import m as n +import p.s as ps +from m import f # E: function m.f is deprecated: use f2 instead +from p.s import g # E: function p.s.g is deprecated: use g2 instead +from k import * + +m.f() # E: function m.f is deprecated: use f2 instead +p.s.g() # E: function p.s.g is deprecated: use g2 instead +n.f() # E: function m.f is deprecated: use f2 instead +ps.g() # E: function p.s.g is deprecated: use g2 instead +f() +g() +h() # E: function k.h is deprecated: use h2 instead + +[file m.py] +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +[file p/s.py] +from typing_extensions import deprecated + +@deprecated("use g2 instead") +def g() -> None: ... + +[file k.py] +from typing_extensions import deprecated + +@deprecated("use h2 instead") +def h() -> None: ... + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClass] +# flags: --enable-error-code=deprecated + +from typing import Callable, List, Optional, Tuple, Union +from typing_extensions import deprecated, TypeAlias, TypeVar + +@deprecated("use C2 instead") +class C: ... + +c: C # E: class __main__.C is deprecated: use C2 instead +C() # E: class __main__.C is deprecated: use C2 instead +C.missing() # E: class __main__.C is deprecated: use C2 instead \ + # E: "Type[C]" has no attribute "missing" +C.__init__(c) # E: class __main__.C is deprecated: use C2 instead +C(1) # E: class __main__.C is deprecated: use C2 instead \ + # E: Too many arguments for "C" + +D = C # E: class __main__.C is deprecated: use C2 instead +D() +t = (C, C, D) # E: class __main__.C is deprecated: use C2 instead + +u1: Union[C, int] = 1 # E: class __main__.C is deprecated: use C2 instead +u1 = 1 +u2 = 1 # type: Union[C, int] # E: class __main__.C is deprecated: use C2 instead +u2 = 1 + +c1 = c2 = C() # E: class __main__.C is deprecated: use C2 instead +i, c3 = 1, C() # E: class __main__.C is deprecated: use C2 instead + +class E: ... + +x1: Optional[C] # E: class __main__.C is deprecated: use C2 instead +x2: Union[D, C, E] # E: class __main__.C is deprecated: use C2 instead +x3: Union[D, Optional[C], E] # E: class __main__.C is deprecated: use C2 instead +x4: Tuple[D, C, E] # E: class __main__.C is deprecated: use C2 instead +x5: Tuple[Tuple[D, C], E] # E: class __main__.C is deprecated: use C2 instead +x6: List[C] # E: class __main__.C is deprecated: use C2 instead +x7: List[List[C]] # E: class __main__.C is deprecated: use C2 instead +x8: List[Optional[Tuple[Union[List[C], int]]]] # E: class __main__.C is deprecated: use C2 instead +x9: Callable[[int], C] # E: class __main__.C is deprecated: use C2 instead +x10: Callable[[int, C, int], int] # E: class __main__.C is deprecated: use C2 instead + +T = TypeVar("T") +A1: TypeAlias = Optional[C] # E: class __main__.C is deprecated: use C2 instead +x11: A1 +A2: TypeAlias = List[Union[A2, C]] # E: class __main__.C is deprecated: use C2 instead +x12: A2 +A3: TypeAlias = List[Optional[T]] +x13: A3[C] # E: class __main__.C is deprecated: use C2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedBaseClass] +# flags: --enable-error-code=deprecated + +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +class D(C): ... # E: class __main__.C is deprecated: use C2 instead +class E(D): ... +class F(D, C): ... # E: class __main__.C is deprecated: use C2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClassInTypeVar] +# flags: --enable-error-code=deprecated + +from typing import Generic, TypeVar +from typing_extensions import deprecated + +class B: ... +@deprecated("use C2 instead") +class C: ... + +T = TypeVar("T", bound=C) # E: class __main__.C is deprecated: use C2 instead +def f(x: T) -> T: ... +class D(Generic[T]): ... + +V = TypeVar("V", B, C) # E: class __main__.C is deprecated: use C2 instead +def g(x: V) -> V: ... +class E(Generic[V]): ... + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClassInCast] +# flags: --enable-error-code=deprecated + +from typing import cast, Generic +from typing_extensions import deprecated + +class B: ... +@deprecated("use C2 instead") +class C: ... + +c = C() # E: class __main__.C is deprecated: use C2 instead +b = cast(B, c) + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedInstanceInFunctionDefinition] +# flags: --enable-error-code=deprecated + +from typing import Generic, List, Optional, TypeVar +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +def f1(c: C) -> None: # E: class __main__.C is deprecated: use C2 instead + def g1() -> None: ... + +def f2(c: List[Optional[C]]) -> None: # E: class __main__.C is deprecated: use C2 instead + def g2() -> None: ... + +def f3() -> C: # E: class __main__.C is deprecated: use C2 instead + def g3() -> None: ... + return C() # E: class __main__.C is deprecated: use C2 instead + +def f4() -> List[Optional[C]]: # E: class __main__.C is deprecated: use C2 instead + def g4() -> None: ... + return [] + +def f5() -> None: + def g5(c: C) -> None: ... # E: class __main__.C is deprecated: use C2 instead + +def f6() -> None: + def g6() -> C: ... # E: class __main__.C is deprecated: use C2 instead + + +@deprecated("use D2 instead") +class D: + + def f1(self, c: C) -> None: # E: class __main__.C is deprecated: use C2 instead + def g1() -> None: ... + + def f2(self, c: List[Optional[C]]) -> None: # E: class __main__.C is deprecated: use C2 instead + def g2() -> None: ... + + def f3(self) -> None: + def g3(c: C) -> None: ... # E: class __main__.C is deprecated: use C2 instead + + def f4(self) -> None: + def g4() -> C: ... # E: class __main__.C is deprecated: use C2 instead + +T = TypeVar("T") + +@deprecated("use E2 instead") +class E(Generic[T]): + + def f1(self: E[C]) -> None: ... # E: class __main__.C is deprecated: use C2 instead + def f2(self, e: E[C]) -> None: ... # E: class __main__.C is deprecated: use C2 instead + def f3(self) -> E[C]: ... # E: class __main__.C is deprecated: use C2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClassDifferentModule] +# flags: --enable-error-code=deprecated + +import m +import p.s +import m as n +import p.s as ps +from m import B, C # E: class m.B is deprecated: use B2 instead \ + # E: class m.C is deprecated: use C2 instead +from p.s import D # E: class p.s.D is deprecated: use D2 instead +from k import * + +m.C() # E: class m.C is deprecated: use C2 instead +p.s.D() # E: class p.s.D is deprecated: use D2 instead +n.C() # E: class m.C is deprecated: use C2 instead +ps.D() # E: class p.s.D is deprecated: use D2 instead +C() +D() +E() # E: class k.E is deprecated: use E2 instead + +x1: m.A # E: class m.A is deprecated: use A2 instead +x2: m.A = m.A() # E: class m.A is deprecated: use A2 instead +y1: B +y2: B = B() + +[file m.py] +from typing_extensions import deprecated + +@deprecated("use A2 instead") +class A: ... + +@deprecated("use B2 instead") +class B: ... + +@deprecated("use C2 instead") +class C: ... + +[file p/s.py] +from typing_extensions import deprecated + +@deprecated("use D2 instead") +class D: ... + +[file k.py] +from typing_extensions import deprecated + +@deprecated("use E2 instead") +class E: ... + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClassInitMethod] +# flags: --enable-error-code=deprecated + +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: + def __init__(self) -> None: ... + +c: C # E: class __main__.C is deprecated: use C2 instead +C() # E: class __main__.C is deprecated: use C2 instead +C.__init__(c) # E: class __main__.C is deprecated: use C2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedSpecialMethods] +# flags: --enable-error-code=deprecated + +from typing import Iterator +from typing_extensions import deprecated + +class A: + @deprecated("no A + int") + def __add__(self, v: int) -> None: ... + + @deprecated("no int + A") + def __radd__(self, v: int) -> None: ... + + @deprecated("no A = A + int") + def __iadd__(self, v: int) -> A: ... + + @deprecated("no iteration") + def __iter__(self) -> Iterator[int]: ... + + @deprecated("no in") + def __contains__(self, v: int) -> int: ... + + @deprecated("no integer") + def __int__(self) -> int: ... + + @deprecated("no inversion") + def __invert__(self) -> A: ... + +class B: + @deprecated("still no in") + def __contains__(self, v: int) -> int: ... + +a = A() +b = B() +a + 1 # E: function __main__.A.__add__ is deprecated: no A + int +1 + a # E: function __main__.A.__radd__ is deprecated: no int + A +a += 1 # E: function __main__.A.__iadd__ is deprecated: no A = A + int +for i in a: # E: function __main__.A.__iter__ is deprecated: no iteration + reveal_type(i) # N: Revealed type is "builtins.int" +1 in a # E: function __main__.A.__contains__ is deprecated: no in +1 in b # E: function __main__.B.__contains__ is deprecated: still no in +~a # E: function __main__.A.__invert__ is deprecated: no inversion + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedOverloadedSpecialMethods] +# flags: --enable-error-code=deprecated + +from typing import Iterator, Union +from typing_extensions import deprecated, overload + +class A: + @overload + @deprecated("no A + int") + def __add__(self, v: int) -> None: ... + @overload + def __add__(self, v: str) -> None: ... + def __add__(self, v: Union[int, str]) -> None: ... + + @overload + def __radd__(self, v: int) -> None: ... + @overload + @deprecated("no str + A") + def __radd__(self, v: str) -> None: ... + def __radd__(self, v: Union[int, str]) -> None: ... + + @overload + def __iadd__(self, v: int) -> A: ... + @overload + def __iadd__(self, v: str) -> A: ... + @deprecated("no A += Any") + def __iadd__(self, v: Union[int, str]) -> A: ... + +a = A() +a + 1 # E: overload def (__main__.A, builtins.int) of function __main__.A.__add__ is deprecated: no A + int +a + "x" +1 + a +"x" + a # E: overload def (__main__.A, builtins.str) of function __main__.A.__radd__ is deprecated: no str + A +a += 1 # E: function __main__.A.__iadd__ is deprecated: no A += Any +a += "x" # E: function __main__.A.__iadd__ is deprecated: no A += Any + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedMethod] +# flags: --enable-error-code=deprecated + +from typing_extensions import deprecated + +class C: + @deprecated("use g instead") + def f(self) -> None: ... + + def g(self) -> None: ... + + @staticmethod + @deprecated("use g instead") + def h() -> None: ... + + @deprecated("use g instead") + @staticmethod + def k() -> None: ... + +C.f # E: function __main__.C.f is deprecated: use g instead +C().f # E: function __main__.C.f is deprecated: use g instead +C().f() # E: function __main__.C.f is deprecated: use g instead +C().f(1) # E: function __main__.C.f is deprecated: use g instead \ + # E: Too many arguments for "f" of "C" +f = C().f # E: function __main__.C.f is deprecated: use g instead +f() +t = (C.f, C.f, C.g) # E: function __main__.C.f is deprecated: use g instead + +C().g() +C().h() # E: function __main__.C.h is deprecated: use g instead +C().k() # E: function __main__.C.k is deprecated: use g instead + +[builtins fixtures/callable.pyi] + + +[case testDeprecatedClassWithDeprecatedMethod] +# flags: --enable-error-code=deprecated + +from typing_extensions import deprecated + +@deprecated("use D instead") +class C: + @deprecated("use g instead") + def f(self) -> None: ... + def g(self) -> None: ... + +C().f() # E: class __main__.C is deprecated: use D instead \ + # E: function __main__.C.f is deprecated: use g instead +C().g() # E: class __main__.C is deprecated: use D instead + +[builtins fixtures/callable.pyi] + + +[case testDeprecatedProperty] +# flags: --enable-error-code=deprecated + +from typing_extensions import deprecated + +class C: + @property + @deprecated("use f2 instead") + def f(self) -> int: ... + + @property + def g(self) -> int: ... + @g.setter + @deprecated("use g2 instead") + def g(self, v: int) -> None: ... + + +C.f # E: function __main__.C.f is deprecated: use f2 instead +C().f # E: function __main__.C.f is deprecated: use f2 instead +C().f() # E: function __main__.C.f is deprecated: use f2 instead \ + # E: "int" not callable +C().f = 1 # E: function __main__.C.f is deprecated: use f2 instead \ + # E: Property "f" defined in "C" is read-only + + +C.g +C().g +C().g = 1 # E: function __main__.C.g is deprecated: use g2 instead +C().g = "x" # E: function __main__.C.g is deprecated: use g2 instead \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[builtins fixtures/property.pyi] + + +[case testDeprecatedDescriptor] +# flags: --enable-error-code=deprecated + +from typing import Any, Optional, Union +from typing_extensions import deprecated, overload + +@deprecated("use E1 instead") +class D1: + def __get__(self, obj: Optional[C], objtype: Any) -> Union[D1, int]: ... + +class D2: + @deprecated("use E2.__get__ instead") + def __get__(self, obj: Optional[C], objtype: Any) -> Union[D2, int]: ... + + @deprecated("use E2.__set__ instead") + def __set__(self, obj: C, value: int) -> None: ... + +class D3: + @overload + @deprecated("use E3.__get__ instead") + def __get__(self, obj: None, objtype: Any) -> D3: ... + @overload + @deprecated("use E3.__get__ instead") + def __get__(self, obj: C, objtype: Any) -> int: ... + def __get__(self, obj: Optional[C], objtype: Any) -> Union[D3, int]: ... + + @overload + def __set__(self, obj: C, value: int) -> None: ... + @overload + @deprecated("use E3.__set__ instead") + def __set__(self, obj: C, value: str) -> None: ... + def __set__(self, obj: C, value: Union[int, str]) -> None: ... + +class C: + d1 = D1() # E: class __main__.D1 is deprecated: use E1 instead + d2 = D2() + d3 = D3() + +c: C +C.d1 +c.d1 +c.d1 = 1 + +C.d2 # E: function __main__.D2.__get__ is deprecated: use E2.__get__ instead +c.d2 # E: function __main__.D2.__get__ is deprecated: use E2.__get__ instead +c.d2 = 1 # E: function __main__.D2.__set__ is deprecated: use E2.__set__ instead + +C.d3 # E: overload def (self: __main__.D3, obj: None, objtype: Any) -> __main__.D3 of function __main__.D3.__get__ is deprecated: use E3.__get__ instead +c.d3 # E: overload def (self: __main__.D3, obj: __main__.C, objtype: Any) -> builtins.int of function __main__.D3.__get__ is deprecated: use E3.__get__ instead +c.d3 = 1 +c.d3 = "x" # E: overload def (self: __main__.D3, obj: __main__.C, value: builtins.str) of function __main__.D3.__set__ is deprecated: use E3.__set__ instead +[builtins fixtures/property.pyi] + + +[case testDeprecatedOverloadedFunction] +# flags: --enable-error-code=deprecated + +from typing import Union +from typing_extensions import deprecated, overload + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +@deprecated("use f2 instead") +def f(x: Union[int, str]) -> Union[int, str]: ... + +f # E: function __main__.f is deprecated: use f2 instead +f(1) # E: function __main__.f is deprecated: use f2 instead +f("x") # E: function __main__.f is deprecated: use f2 instead +f(1.0) # E: function __main__.f is deprecated: use f2 instead \ + # E: No overload variant of "f" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def f(x: int) -> int \ + # N: def f(x: str) -> str + +@overload +@deprecated("work with str instead") +def g(x: int) -> int: ... +@overload +def g(x: str) -> str: ... +def g(x: Union[int, str]) -> Union[int, str]: ... + +g +g(1) # E: overload def (x: builtins.int) -> builtins.int of function __main__.g is deprecated: work with str instead +g("x") +g(1.0) # E: No overload variant of "g" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def g(x: int) -> int \ + # N: def g(x: str) -> str + +@overload +def h(x: int) -> int: ... +@deprecated("work with int instead") +@overload # N: @overload should be placed before @deprecated +def h(x: str) -> str: ... +def h(x: Union[int, str]) -> Union[int, str]: ... + +h +h(1) +h("x") # E: overload def (x: builtins.str) -> builtins.str of function __main__.h is deprecated: work with int instead +h(1.0) # E: No overload variant of "h" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def h(x: int) -> int \ + # N: def h(x: str) -> str + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedImportedOverloadedFunction] +# flags: --enable-error-code=deprecated + +import m + +m.g +m.g(1) # E: overload def (x: builtins.int) -> builtins.int of function m.g is deprecated: work with str instead +m.g("x") + +[file m.py] + +from typing import Union +from typing_extensions import deprecated, overload + +@overload +@deprecated("work with str instead") +def g(x: int) -> int: ... +@overload +def g(x: str) -> str: ... +def g(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-dynamic-typing.test b/test-data/unit/check-dynamic-typing.test new file mode 100644 index 000000000000..21fd52169ff5 --- /dev/null +++ b/test-data/unit/check-dynamic-typing.test @@ -0,0 +1,830 @@ +-- Assignment +-- ---------- + + +[case testAssignmentWithDynamic] +from typing import Any +d: Any +a: A + +if int(): + a = d # Everything ok +if int(): + d = a +if int(): + d = d +d.x = a +d.x = d + +class A: pass + +[case testMultipleAssignmentWithDynamic] +from typing import Any +d: Any +a: A +b: B + +if int(): + d, a = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + d, d = d, d, d # E: Too many values to unpack (2 expected, 3 provided) + +if int(): + a, b = d, d +if int(): + d, d = a, b +if int(): + a, b = d +s, t = d + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + + +-- Expressions +-- ----------- + + +[case testCallingFunctionWithDynamicArgumentTypes] +from typing import Any + +def f(x: Any) -> 'A': + pass + +a: A +b: B + +if int(): + b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a = f(a) +if int(): + a = f(b) +if int(): + a = f(None) +if int(): + a = f(f) + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testCallingWithDynamicReturnType] +from typing import Any + +def f(x: 'A') -> Any: + pass + +a: A +b: B + +a = f(b) # E: Argument 1 to "f" has incompatible type "B"; expected "A" + +a = f(a) +b = f(a) + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testBinaryOperationsWithDynamicLeftOperand] +from typing import Any +d: Any +a: A +c: C +b: bool +n = 0 + +d in a # E: Unsupported right operand type for in ("A") +d and a +d or a +if int(): + c = d and b # E: Incompatible types in assignment (expression has type "Union[Any, bool]", variable has type "C") +if int(): + c = d or b # E: Incompatible types in assignment (expression has type "Union[Any, bool]", variable has type "C") + +if int(): + c = d + a +if int(): + c = d - a +if int(): + c = d * a +if int(): + c = d / a +if int(): + c = d // a +if int(): + c = d % a +if int(): + c = d ** a +if int(): + b = d == a +if int(): + b = d != a +if int(): + b = d < a +if int(): + b = d <= a +if int(): + b = d > a +if int(): + b = d >= a +if int(): + b = d in c +if int(): + b = d and b +if int(): + b = d or b + +class A: pass +class C: + def __contains__(self, a: A) -> bool: + pass +[file builtins.py] +class object: + def __init__(self): pass +class bool: pass +class int: pass +class type: pass +class function: pass +class str: pass +class dict: pass + +[case testBinaryOperationsWithDynamicAsRightOperand] +from typing import Any +d: Any +a: A +c: C +b: bool +n = 0 + +a and d +a or d +if int(): + c = a in d # E: Incompatible types in assignment (expression has type "bool", variable has type "C") +if int(): + c = b and d # E: Incompatible types in assignment (expression has type "Union[Literal[False], Any]", variable has type "C") +if int(): + c = b or d # E: Incompatible types in assignment (expression has type "Union[Literal[True], Any]", variable has type "C") +if int(): + b = a + d +if int(): + b = a / d + +if int(): + c = a + d +if int(): + c = a - d +if int(): + c = a * d +if int(): + c = a / d +if int(): + c = a // d +if int(): + c = a % d +if int(): + c = a ** d +if int(): + b = a in d +if int(): + b = b and d +if int(): + b = b or d + +class A: + def __add__(self, a: 'A') -> 'C': + pass + def __sub__(self, a: 'A') -> 'C': + pass + def __mul__(self, a: 'A') -> 'C': + pass + def __truediv__(self, a: 'A') -> 'C': + pass + def __floordiv__(self, a: 'A') -> 'C': + pass + def __mod__(self, a: 'A') -> 'C': + pass + def __pow__(self, a: 'A') -> 'C': + pass + def _lt(self, a: 'A') -> bool: + pass + def _gt(self, a: 'A') -> bool: + pass + +class C: pass +[file builtins.py] +class object: + def __init__(self): pass +class bool: pass +class int: pass +class type: pass +class function: pass +class str: pass +class dict: pass + +[case testDynamicWithUnaryExpressions] +from typing import Any +d: Any +a: A +b: bool +if int(): + a = not d # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + b = not d + a = -d +class A: pass +[builtins fixtures/bool.pyi] +[out] + +[case testDynamicWithMemberAccess] +from typing import Any +d: Any +a: A + +if int(): + a = d.foo(a()) # E: "A" not callable + +if int(): + a = d.x +if int(): + a = d.foo(a, a) +d.x = a +d.x.y.z # E: "A" has no attribute "y" + +class A: pass +[out] + +[case testIndexingWithDynamic] +from typing import Any +d: Any +a: A + +if int(): + a = d[a()] # E: "A" not callable +d[a()] = a # E: "A" not callable + +if int(): + a = d[a] +d[a] = a +d[a], d[a] = a, a + +class A: pass + +[case testTupleExpressionsWithDynamic] +from typing import Tuple, Any +t2: Tuple[A, A] +d: Any + +if int(): + t2 = (d, d, d) # E: Incompatible types in assignment (expression has type "Tuple[Any, Any, Any]", variable has type "Tuple[A, A]") +if int(): + t2 = (d, d) + +class A: pass +[builtins fixtures/tuple.pyi] + +[case testCastsWithDynamicType] +from typing import Any, cast +class A: pass +class B: pass +def f() -> None: pass + +d: Any +a: A +b: B +if int(): + b = cast(A, d) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = cast(A, d) +if int(): + b = cast(Any, d) +if int(): + a = cast(Any, f()) +[case testCompatibilityOfDynamicWithOtherTypes] +from typing import Any, Tuple + +def g(a: 'A') -> None: + pass + +class A: pass +class B: pass + +d: Any +t: Tuple[A, A] +# TODO: callable types, overloaded functions + +d = None # All ok +d = t +d = g +d = A +t = d +f = d +[builtins fixtures/tuple.pyi] + + +-- Statements +-- ---------- + + +[case testDynamicCondition] +from typing import Any +d = None # type: Any +while d: + pass +if d: + pass +elif d: + pass +[builtins fixtures/bool.pyi] + +[case testRaiseWithDynamic] +from typing import Any +d = None # type: Any +raise d +[builtins fixtures/exception.pyi] + +[case testReturnWithDynamic] +from typing import Any +d = None # type: Any + +def f() -> None: + return d # Ok + +def g() -> 'A': + return d # Ok + +class A: pass + + +-- Implicit dynamic types for functions +-- ------------------------------------ + + +[case testImplicitGlobalFunctionSignature] +from typing import Any, Callable +x: Any +a: A +g: Callable[[], None] +h: Callable[[A], None] + +def f(x): pass + +f() # E: Missing positional argument "x" in call to "f" +f(x, x) # E: Too many arguments for "f" +if int(): + g = f # E: Incompatible types in assignment (expression has type "Callable[[Any], Any]", variable has type "Callable[[], None]") +f(a) +f(x) +if int(): + a = f(a) +if int(): + h = f + +class A: pass + +[case testImplicitGlobalFunctionSignatureWithDifferentArgCounts] +from typing import Callable +g0: Callable[[], None] +g1: Callable[[A], None] +g2: Callable[[A, A], None] +a: A + +def f0(): pass +def f2(x, y): pass + +if int(): + g1 = f0 # E: Incompatible types in assignment (expression has type "Callable[[], Any]", variable has type "Callable[[A], None]") +if int(): + g2 = f0 # E: Incompatible types in assignment (expression has type "Callable[[], Any]", variable has type "Callable[[A, A], None]") +if int(): + g0 = f2 # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], Any]", variable has type "Callable[[], None]") +if int(): + g1 = f2 # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], Any]", variable has type "Callable[[A], None]") + +if int(): + g0 = g0 +if int(): + g2 = f2 +f0() +f2(a, a) + +class A: pass + +[case testImplicitGlobalFunctionSignatureWithDefaultArgs] +from typing import Callable +class A: pass +class B: pass + +a: A +b: B + +def f01(x = b): pass +def f13(x, y = b, z = b): pass + +g0: Callable[[], None] +g1: Callable[[A], None] +g2: Callable[[A, A], None] +g3: Callable[[A, A, A], None] +g4: Callable[[A, A, A, A], None] + +f01(a, a) # E: Too many arguments for "f01" +f13() # E: Missing positional argument "x" in call to "f13" +f13(a, a, a, a) # E: Too many arguments for "f13" +if int(): + g2 = f01 # E: Incompatible types in assignment (expression has type "Callable[[Any], Any]", variable has type "Callable[[A, A], None]") +if int(): + g0 = f13 # E: Incompatible types in assignment (expression has type "Callable[[Any, Any, Any], Any]", variable has type "Callable[[], None]") +if int(): + g4 = f13 # E: Incompatible types in assignment (expression has type "Callable[[Any, Any, Any], Any]", variable has type "Callable[[A, A, A, A], None]") + +f01() +f01(a) +f13(a) +f13(a, a) +f13(a, a, a) + +if int(): + g0 = f01 +if int(): + g1 = f01 +if int(): + g1 = f13 +if int(): + g2 = f13 +if int(): + g3 = f13 + +[builtins fixtures/tuple.pyi] + +[case testSkipTypeCheckingWithImplicitSignature] +a: A +def f(): + a() +def g(x): + a() + a.x + a + a + if a(): + a() +class A: pass +[builtins fixtures/bool.pyi] + +[case testSkipTypeCheckingWithImplicitSignatureAndDefaultArgs] +a: A +def f(x=a()): + a() +def g(x, y=a, z=a()): + a() +class A: pass + +[case testImplicitMethodSignature] +from typing import Callable +g0: Callable[[], None] +g1: Callable[[A], None] +g2: Callable[[A, A], None] +a: A + +if int(): + g0 = a.f # E: Incompatible types in assignment (expression has type "Callable[[Any], Any]", variable has type "Callable[[], None]") +if int(): + g2 = a.f # E: Incompatible types in assignment (expression has type "Callable[[Any], Any]", variable has type "Callable[[A, A], None]") +if int(): + a = a.f # E: Incompatible types in assignment (expression has type "Callable[[Any], Any]", variable has type "A") + +class A: + def g(self) -> None: + a = self.f(a) + def f(self, x): pass + +if int(): + g1 = a.f +if int(): + a = a.f(a) + +[case testSkipTypeCheckingImplicitMethod] + +a: A +class A: + def f(self): + a() + def g(self, x, y=a()): + a() + +[case testImplicitInheritedMethod] +from typing import Callable +g0: Callable[[], None] +g1: Callable[[A], None] +a: A + +if int(): + g0 = a.f # E: Incompatible types in assignment (expression has type "Callable[[Any], Any]", variable has type "Callable[[], None]") + +if int(): + g1 = a.f +if int(): + a = a.f(a) + +class B: + def f(self, x): + pass +class A(B): + def g(self) -> None: + a = self.f(a) + +[case testEmptyReturnWithImplicitSignature] +import typing +def f(): + return +class A: + def g(self): + return + +[case testVarArgsWithImplicitSignature] +from typing import Any +o = None # type: Any +def f(x, *a): pass +f() # E: Missing positional argument "x" in call to "f" +f(o) +f(o, o) +f(o, o, o) +[builtins fixtures/list.pyi] + + +-- Implicit types for constructors +-- ------------------------------- + + +[case testInitMethodWithImplicitSignature] +from typing import Callable + +class A: + def __init__(self, a, b): pass + +f1: Callable[[A], A] +f2: Callable[[A, A], A] +a: A + +A(a) # E: Missing positional argument "b" in call to "A" +if int(): + f1 = A # E: Incompatible types in assignment (expression has type "Type[A]", variable has type "Callable[[A], A]") + +A(a, a) +if int(): + f2 = A + +[case testUsingImplicitTypeObjectWithIs] +class A: pass +class B: + def __init__(self): pass + +t: type +t = A +t = B +-- Type compatibility +-- ------------------ + + +[case testTupleTypeCompatibility] +from typing import Any, Tuple +t1: Tuple[Any, A] +t2: Tuple[A, Any] +t3: Tuple[Any, Any] +t4: Tuple[A, A] +t5: Tuple[Any, Any, Any] + +def f(): t1, t2, t3, t4, t5 # Prevent redefinition + +t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[Any, Any, Any]", variable has type "Tuple[Any, Any]") +t5 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[Any, Any, Any]") + +t1 = t1 +t1 = t2 +t1 = t3 +t1 = t4 +t2 = t1 +t2 = t3 +t2 = t4 +t3 = t1 +t3 = t2 +t3 = t4 +t4 = t1 +t4 = t2 +t4 = t3 + +class A: pass +[builtins fixtures/tuple.pyi] + +[case testFunctionTypeCompatibilityAndReturnTypes] +from typing import Any, Callable, Optional +f1: Callable[[], Any] +f11: Callable[[], Any] +f2: Callable[[], Optional[A]] +f3: Callable[[], None] + +f2 = f3 + +f1 = f2 +f1 = f3 +f2 = f11 +f3 = f11 + +class A: pass + +[case testFunctionTypeCompatibilityAndArgumentTypes] +from typing import Any, Callable +f1: Callable[[A, Any], None] +f2: Callable[[Any, A], None] +f3: Callable[[A, A], None] + +f1 = f1 +f1 = f2 +f1 = f3 + +f2 = f1 +f2 = f2 +f2 = f3 + +f3 = f1 +f3 = f2 +f3 = f3 + +class A: pass + +[case testFunctionTypeCompatibilityAndArgumentCounts] +from typing import Any, Callable +f1: Callable[[Any], None] +f2: Callable[[Any, Any], None] + +if int(): + f1 = f2 # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], None]", variable has type "Callable[[Any], None]") + + +-- Overriding +-- ---------- + + +[case testOverridingMethodWithDynamicTypes] +from typing import Any +a: A +b: B + +b.f(b) # E: Argument 1 to "f" of "B" has incompatible type "B"; expected "A" +a = a.f(b) + +class B: + def f(self, x: 'A') -> 'B': + pass + def g(self, x: 'B') -> None: + pass +class A(B): + def f(self, x: Any) -> Any: + pass + def g(self, x: Any) -> None: + pass +[builtins fixtures/tuple.pyi] + +[case testOverridingMethodWithImplicitDynamicTypes] +a: A +b: B + +b.f(b) # E: Argument 1 to "f" of "B" has incompatible type "B"; expected "A" +a = a.f(b) + +class B: + def f(self, x: 'A') -> 'B': + pass + def g(self, x: 'B') -> None: + pass +class A(B): + def f(self, x): + pass + def g(self, x): + pass +[builtins fixtures/tuple.pyi] + +[case testOverridingMethodAcrossHierarchy] +import typing +class C: + def f(self, a: 'A') -> None: pass +class B(C): + def f(self, a): pass +class A(B): + def f(self, a: 'D') -> None: # E: Argument 1 of "f" is incompatible with supertype "C"; supertype defines the argument type as "A" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + pass +class D: pass +[out] + +[case testInvalidOverrideArgumentCountWithImplicitSignature1] +import typing +class B: + def f(self, x: A) -> None: pass +class A(B): + def f(self, x, y): # dynamic function not type checked + x() +[out] + +[case testInvalidOverrideArgumentCountWithImplicitSignature2] +import typing +class B: + def f(self, x, y): pass +class A(B): + def f(self, x: 'A') -> None: # Fail + pass +[out] +main:5: error: Signature of "f" incompatible with supertype "B" +main:5: note: Superclass: +main:5: note: def f(self, x: Any, y: Any) -> Any +main:5: note: Subclass: +main:5: note: def f(self, x: A) -> None + +[case testInvalidOverrideArgumentCountWithImplicitSignature3] +import typing +class B: + def f(self, x: A) -> None: pass +class A(B): + def f(self, x, y) -> None: # Fail + x() +[out] +main:5: error: Signature of "f" incompatible with supertype "B" +main:5: note: Superclass: +main:5: note: def f(self, x: A) -> None +main:5: note: Subclass: +main:5: note: def f(self, x: Any, y: Any) -> None + +[case testInvalidOverrideArgumentCountWithImplicitSignature4] +# flags: --check-untyped-defs +import typing +class B: + def f(self, x: A) -> None: pass +class A(B): + def f(self, x, y): + x() +[out] +main:6: error: Signature of "f" incompatible with supertype "B" +main:6: note: Superclass: +main:6: note: def f(self, x: A) -> None +main:6: note: Subclass: +main:6: note: def f(self, x: Any, y: Any) -> Any + +[case testInvalidOverrideWithImplicitSignatureAndClassMethod1] +class B: + @classmethod + def f(cls, x, y): pass +class A(B): + @classmethod + def f(cls, x, y, z): pass # No error since no annotations +[builtins fixtures/classmethod.pyi] + +[case testInvalidOverrideWithImplicitSignatureAndClassMethod2] +class B: + @classmethod + def f(cls, x: int, y): pass +class A(B): + @classmethod + def f(cls, x, y, z): pass # No error since no annotations +[builtins fixtures/classmethod.pyi] + +[case testInvalidOverrideWithImplicitSignatureAndStaticMethod1] +class B: + @staticmethod + def f(x, y): pass +class A(B): + @staticmethod + def f(x, y, z): pass # No error since no annotations +[builtins fixtures/classmethod.pyi] + +[case testInvalidOverrideWithImplicitSignatureAndStaticMethod2] +class B: + @staticmethod + def f(self, x: int, y): pass +class A(B): + @staticmethod + def f(self, x, y, z): pass # No error since no annotations +[builtins fixtures/classmethod.pyi] + + +-- Don't complain about too few/many arguments in dynamic functions +-- ---------------------------------------------------------------- + +[case testTooManyArgsInDynamic] +def f() -> None: pass +def g(): + f(1) # Silent +[out] + +[case testTooFewArgsInDynamic] +def f(a: int) -> None: pass +def g(): + f() # Silent +[out] + +[case testJustRightInDynamic] +def f(a: int) -> None: pass +def g(): + f('') # Silent +[out] diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test new file mode 100644 index 000000000000..e6e42d805052 --- /dev/null +++ b/test-data/unit/check-enum.test @@ -0,0 +1,2323 @@ +-- This test file checks Enum + +[case testEnumBasics] +from enum import Enum +class Medal(Enum): + gold = 1 + silver = 2 + bronze = 3 +reveal_type(Medal.bronze) # N: Revealed type is "Literal[__main__.Medal.bronze]?" +m = Medal.gold +if int(): + m = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Medal") + +[builtins fixtures/enum.pyi] + +-- Creation from Enum call +-- ----------------------- + +[case testEnumCreatedFromStringLiteral] +from enum import Enum +from typing_extensions import Literal + +x: Literal['ANT BEE CAT DOG'] = 'ANT BEE CAT DOG' +Animal = Enum('Animal', x) +reveal_type(Animal.ANT) # N: Revealed type is "Literal[__main__.Animal.ANT]?" +reveal_type(Animal.BEE) # N: Revealed type is "Literal[__main__.Animal.BEE]?" +reveal_type(Animal.CAT) # N: Revealed type is "Literal[__main__.Animal.CAT]?" +reveal_type(Animal.DOG) # N: Revealed type is "Literal[__main__.Animal.DOG]?" + +[builtins fixtures/tuple.pyi] + +[case testEnumCreatedFromFinalValue] +from enum import Enum +from typing_extensions import Final + +x: Final['str'] = 'ANT BEE CAT DOG' +Animal = Enum('Animal', x) +reveal_type(Animal.ANT) # N: Revealed type is "Literal[__main__.Animal.ANT]?" +reveal_type(Animal.BEE) # N: Revealed type is "Literal[__main__.Animal.BEE]?" +reveal_type(Animal.CAT) # N: Revealed type is "Literal[__main__.Animal.CAT]?" +reveal_type(Animal.DOG) # N: Revealed type is "Literal[__main__.Animal.DOG]?" + +[builtins fixtures/tuple.pyi] + +-- Creation from EnumMeta +-- ---------------------- + +[case testEnumFromEnumMetaBasics] +from enum import EnumMeta +class Medal(metaclass=EnumMeta): + gold = 1 + silver = "hello" + bronze = None + # Without __init__ the definition fails at runtime, but we want to verify that mypy + # uses `enum.EnumMeta` and not `enum.Enum` as the definition of what is enum. + def __init__(self, *args): pass +reveal_type(Medal.bronze) # N: Revealed type is "Literal[__main__.Medal.bronze]?" +m = Medal.gold +if int(): + m = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Medal") +[builtins fixtures/tuple.pyi] + +[case testEnumFromEnumMetaSubclass] +from enum import EnumMeta +class Achievement(metaclass=EnumMeta): pass +class Medal(Achievement): + gold = 1 + silver = "hello" + bronze = None + # See comment in testEnumFromEnumMetaBasics + def __init__(self, *args): pass +reveal_type(Medal.bronze) # N: Revealed type is "Literal[__main__.Medal.bronze]?" +m = Medal.gold +if int(): + m = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Medal") +[builtins fixtures/tuple.pyi] + +[case testEnumFromEnumMetaGeneric] +from enum import EnumMeta +from typing import Generic, TypeVar +T = TypeVar("T") +class Medal(Generic[T], metaclass=EnumMeta): # E: Enum class cannot be generic + q = None +[builtins fixtures/enum.pyi] + +[case testEnumNameAndValue] +from enum import Enum +class Truth(Enum): + true = True + false = False +x = '' +x = Truth.true.name +reveal_type(Truth.true.name) # N: Revealed type is "Literal['true']?" +reveal_type(Truth.false.value) # N: Revealed type is "Literal[False]?" +[builtins fixtures/bool.pyi] + +[case testEnumValueExtended] +from enum import Enum +class Truth(Enum): + true = True + false = False + +def infer_truth(truth: Truth) -> None: + reveal_type(truth.value) # N: Revealed type is "Union[Literal[True]?, Literal[False]?]" +[builtins fixtures/bool.pyi] + +[case testEnumValueAllAuto] +from enum import Enum, auto +class Truth(Enum): + true = auto() + false = auto() + +def infer_truth(truth: Truth) -> None: + reveal_type(truth.value) # N: Revealed type is "builtins.int" +[builtins fixtures/primitives.pyi] + +[case testEnumValueSomeAuto] +from enum import Enum, auto +class Truth(Enum): + true = 8675309 + false = auto() + +def infer_truth(truth: Truth) -> None: + reveal_type(truth.value) # N: Revealed type is "builtins.int" +[builtins fixtures/primitives.pyi] + +[case testEnumValueExtraMethods] +from enum import Enum +class Truth(Enum): + true = True + false = False + + def foo(self) -> str: + return 'bar' + +def infer_truth(truth: Truth) -> None: + reveal_type(truth.value) # N: Revealed type is "Union[Literal[True]?, Literal[False]?]" +[builtins fixtures/bool.pyi] + +[case testEnumValueCustomAuto] +from enum import Enum, auto +class AutoName(Enum): + + # In `typeshed`, this is a staticmethod and has more arguments, + # but I have lied a bit to keep the test stubs lean. + def _generate_next_value_(self) -> str: + return "name" + +class Truth(AutoName): + true = auto() + false = auto() + +def infer_truth(truth: Truth) -> None: + reveal_type(truth.value) # N: Revealed type is "builtins.str" +[builtins fixtures/primitives.pyi] + +[case testEnumValueInhomogenous] +from enum import Enum +class Truth(Enum): + true = 'True' + false = 0 + +def cannot_infer_truth(truth: Truth) -> None: + reveal_type(truth.value) # N: Revealed type is "Any" +[builtins fixtures/bool.pyi] + +[case testEnumValueSameType] +from enum import Enum + +def newbool() -> bool: + ... + +class Truth(Enum): + true = newbool() + false = newbool() + +def infer_truth(truth: Truth) -> None: + reveal_type(truth.value) # N: Revealed type is "builtins.bool" +[builtins fixtures/bool.pyi] + +[case testEnumTruthyness] +# mypy: warn-unreachable +import enum +class E(enum.Enum): + x = 0 +if not E.x: + "noop" +[builtins fixtures/tuple.pyi] +[out] +main:6: error: Statement is unreachable + +[case testEnumTruthynessCustomDunderBool] +# mypy: warn-unreachable +import enum +from typing_extensions import Literal +class E(enum.Enum): + x = 0 + def __bool__(self) -> Literal[False]: + return False +if E.x: + "noop" +[builtins fixtures/enum.pyi] +[out] +main:9: error: Statement is unreachable + +[case testEnumUnique] +import enum +@enum.unique +class E(enum.Enum): + x = 1 + y = 1 # NOTE: This duplicate value is not detected by mypy at the moment +x = 1 +x = E.x +[builtins fixtures/enum.pyi] +[out] +main:7: error: Incompatible types in assignment (expression has type "E", variable has type "int") + +[case testIntEnum_assignToIntVariable] +from enum import IntEnum +class N(IntEnum): + x = 1 + y = 1 +n = 1 +if int(): + n = N.x # Subclass of int, so it's okay +s = '' +if int(): + s = N.y # E: Incompatible types in assignment (expression has type "N", variable has type "str") +[builtins fixtures/enum.pyi] + +[case testIntEnum_functionTakingIntEnum] +from enum import IntEnum +class SomeIntEnum(IntEnum): + x = 1 +def takes_some_int_enum(n: SomeIntEnum): + pass +takes_some_int_enum(SomeIntEnum.x) +takes_some_int_enum(1) # Error +takes_some_int_enum(SomeIntEnum(1)) # How to deal with the above +[builtins fixtures/enum.pyi] +[out] +main:7: error: Argument 1 to "takes_some_int_enum" has incompatible type "int"; expected "SomeIntEnum" + +[case testIntEnum_functionTakingInt] +from enum import IntEnum +class SomeIntEnum(IntEnum): + x = 1 +def takes_int(i: int): + pass +takes_int(SomeIntEnum.x) +takes_int(2) +[builtins fixtures/enum.pyi] + +[case testIntEnum_functionReturningIntEnum] +from enum import IntEnum +class SomeIntEnum(IntEnum): + x = 1 +def returns_some_int_enum() -> SomeIntEnum: + return SomeIntEnum.x +an_int = 1 +an_int = returns_some_int_enum() + +an_enum = SomeIntEnum.x +an_enum = returns_some_int_enum() +[builtins fixtures/enum.pyi] +[out] + +[case testStrEnumCreation] +# flags: --python-version 3.11 +from enum import StrEnum + +class MyStrEnum(StrEnum): + x = 'x' + y = 'y' + +reveal_type(MyStrEnum.x) # N: Revealed type is "Literal[__main__.MyStrEnum.x]?" +reveal_type(MyStrEnum.x.value) # N: Revealed type is "Literal['x']?" +reveal_type(MyStrEnum.y) # N: Revealed type is "Literal[__main__.MyStrEnum.y]?" +reveal_type(MyStrEnum.y.value) # N: Revealed type is "Literal['y']?" +[builtins fixtures/enum.pyi] +[out] + +[case testEnumMethods] +from enum import Enum + +class Color(Enum): + red = 1 + green = 2 + + def m(self, x: int): pass + @staticmethod + def m2(x: int): pass + +Color.red.m('') +Color.m2('') +[builtins fixtures/staticmethod.pyi] +[out] +main:11: error: Argument 1 to "m" of "Color" has incompatible type "str"; expected "int" +main:12: error: Argument 1 to "m2" of "Color" has incompatible type "str"; expected "int" + +[case testIntEnum_ExtendedIntEnum_functionTakingExtendedIntEnum] +from enum import IntEnum +class ExtendedIntEnum(IntEnum): + pass +class SomeExtIntEnum(ExtendedIntEnum): + x = 1 + +def takes_int(i: int): + pass +takes_int(SomeExtIntEnum.x) + +def takes_some_ext_int_enum(s: SomeExtIntEnum): + pass +takes_some_ext_int_enum(SomeExtIntEnum.x) +[builtins fixtures/enum.pyi] + +[case testNamedTupleEnum] +from typing import NamedTuple +from enum import Enum + +N = NamedTuple('N', [('bar', int)]) + +class E(N, Enum): + X = N(1) + +def f(x: E) -> None: pass + +f(E.X) +[builtins fixtures/tuple.pyi] + +[case testEnumCall] +from enum import IntEnum +class E(IntEnum): + a = 1 +x: int +reveal_type(E(x)) +[builtins fixtures/tuple.pyi] +[out] +main:5: note: Revealed type is "__main__.E" + +[case testEnumIndex] +from enum import IntEnum +class E(IntEnum): + a = 1 +s: str +reveal_type(E[s]) +[builtins fixtures/enum.pyi] +[out] +main:5: note: Revealed type is "__main__.E" + +[case testEnumIndexError] +from enum import IntEnum +class E(IntEnum): + a = 1 +E[1] # E: Enum index should be a string (actual index type "int") +x = E[1] # E: Enum index should be a string (actual index type "int") +[builtins fixtures/enum.pyi] + +[case testEnumIndexIsNotAnAlias] +from enum import Enum + +class E(Enum): + a = 1 + b = 2 +reveal_type(E['a']) # N: Revealed type is "__main__.E" +E['a'] +x = E['a'] +reveal_type(x) # N: Revealed type is "__main__.E" + +def get_member(name: str) -> E: + val = E[name] + return val + +reveal_type(get_member('a')) # N: Revealed type is "__main__.E" +[builtins fixtures/enum.pyi] + +[case testGenericEnum] +from enum import Enum +from typing import Generic, TypeVar + +T = TypeVar('T') + +class F(Generic[T], Enum): # E: Enum class cannot be generic + x: T + y: T + +reveal_type(F[int].x) # N: Revealed type is "__main__.F[builtins.int]" +[builtins fixtures/enum.pyi] + +[case testEnumFlag] +from enum import Flag +class C(Flag): + a = 1 + b = 2 +x = C.a +if int(): + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "C") +if int(): + x = x | C.b +[builtins fixtures/enum.pyi] + +[case testEnumIntFlag] +from enum import IntFlag +class C(IntFlag): + a = 1 + b = 2 +x = C.a +if int(): + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "C") +if int(): + x = x | C.b +[builtins fixtures/enum.pyi] + +[case testAnonymousEnum] +from enum import Enum +class A: + def f(self) -> None: + class E(Enum): + a = 1 + self.x = E.a +a = A() +reveal_type(a.x) +[builtins fixtures/enum.pyi] +[out] +main:8: note: Revealed type is "__main__.E@4" + +[case testEnumInClassBody] +from enum import Enum +class A: + class E(Enum): + a = 1 +class B: + class E(Enum): + a = 1 +x = A.E.a +y = B.E.a +if int(): + x = y # E: Incompatible types in assignment (expression has type "__main__.B.E", variable has type "__main__.A.E") +[builtins fixtures/enum.pyi] + +[case testFunctionalEnumString] +from enum import Enum, IntEnum +E = Enum('E', 'foo bar') +I = IntEnum('I', ' bar, baz ') +reveal_type(E.foo) +reveal_type(E.bar.value) +reveal_type(I.bar) +reveal_type(I.baz.value) +[builtins fixtures/enum.pyi] +[out] +main:4: note: Revealed type is "Literal[__main__.E.foo]?" +main:5: note: Revealed type is "Any" +main:6: note: Revealed type is "Literal[__main__.I.bar]?" +main:7: note: Revealed type is "builtins.int" + +[case testFunctionalEnumListOfStrings] +from enum import Enum, IntEnum +E = Enum('E', ('foo', 'bar')) +F = IntEnum('F', ['bar', 'baz']) +reveal_type(E.foo) +reveal_type(F.baz) +[builtins fixtures/enum.pyi] +[out] +main:4: note: Revealed type is "Literal[__main__.E.foo]?" +main:5: note: Revealed type is "Literal[__main__.F.baz]?" + +[case testFunctionalEnumListOfPairs] +from enum import Enum, IntEnum +E = Enum('E', [('foo', 1), ['bar', 2]]) +F = IntEnum('F', (['bar', 1], ('baz', 2))) +reveal_type(E.foo) +reveal_type(F.baz) +reveal_type(E.foo.value) +reveal_type(F.bar.name) +[builtins fixtures/enum.pyi] +[out] +main:4: note: Revealed type is "Literal[__main__.E.foo]?" +main:5: note: Revealed type is "Literal[__main__.F.baz]?" +main:6: note: Revealed type is "Literal[1]?" +main:7: note: Revealed type is "Literal['bar']?" + +[case testFunctionalEnumDict] +from enum import Enum, IntEnum +E = Enum('E', {'foo': 1, 'bar': 2}) +F = IntEnum('F', {'bar': 1, 'baz': 2}) +reveal_type(E.foo) +reveal_type(F.baz) +reveal_type(E.foo.value) +reveal_type(F.bar.name) +[builtins fixtures/enum.pyi] +[out] +main:4: note: Revealed type is "Literal[__main__.E.foo]?" +main:5: note: Revealed type is "Literal[__main__.F.baz]?" +main:6: note: Revealed type is "Literal[1]?" +main:7: note: Revealed type is "Literal['bar']?" + + +[case testEnumKeywordsArgs] +from enum import Enum, IntEnum + +PictureSize = Enum('PictureSize', 'P0 P1 P2 P3 P4 P5 P6 P7 P8', type=str, module=__name__) +fake_enum1 = Enum('fake_enum1', ['a', 'b']) +fake_enum2 = Enum('fake_enum2', names=['a', 'b']) +fake_enum3 = Enum(value='fake_enum3', names=['a', 'b']) +fake_enum4 = Enum(value='fake_enum4', names=['a', 'b'] , module=__name__) +[builtins fixtures/enum.pyi] + +[case testFunctionalEnumErrors] +from enum import Enum, IntEnum +A = Enum('A') # E: Too few arguments for Enum() +B = Enum('B', 42) # E: Second argument of Enum() must be string, tuple, list or dict literal for mypy to determine Enum members +C = Enum('C', 'a b', 'x', 'y', 'z', 'p', 'q') # E: Too many arguments for Enum() +D = Enum('D', foo) # E: Second argument of Enum() must be string, tuple, list or dict literal for mypy to determine Enum members \ + # E: Name "foo" is not defined +bar = 'x y z' +E = Enum('E', bar) # E: Second argument of Enum() must be string, tuple, list or dict literal for mypy to determine Enum members +I = IntEnum('I') # E: Too few arguments for IntEnum() +J = IntEnum('I', 42) # E: Second argument of IntEnum() must be string, tuple, list or dict literal for mypy to determine Enum members +K = IntEnum('I', 'p q', 'x', 'y', 'z', 'p', 'q') # E: Too many arguments for IntEnum() +L = Enum('L', ' ') # E: Enum() needs at least one item +M = Enum('M', ()) # E: Enum() needs at least one item +N = IntEnum('M', []) # E: IntEnum() needs at least one item +P = Enum('P', [42]) # E: Enum() with tuple or list expects strings or (name, value) pairs +Q = Enum('Q', [('a', 42, 0)]) # E: Enum() with tuple or list expects strings or (name, value) pairs +R = IntEnum('R', [[0, 42]]) # E: IntEnum() with tuple or list expects strings or (name, value) pairs +S = Enum('S', {1: 1}) # E: Enum() with dict literal requires string literals +T = Enum('T', keyword='a b') # E: Unexpected keyword argument "keyword" +U = Enum('U', *['a']) # E: Unexpected arguments to Enum() +V = Enum('U', **{'a': 1}) # E: Unexpected arguments to Enum() +W = Enum('W', 'a b') +W.c # E: "Type[W]" has no attribute "c" +X = Enum('Something', 'a b') # E: String argument 1 "Something" to enum.Enum(...) does not match variable name "X" +reveal_type(X.a) # N: Revealed type is "Literal[__main__.Something@23.a]?" +X.asdf # E: "Type[Something@23]" has no attribute "asdf" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testFunctionalEnumFlag] +from enum import Flag, IntFlag +A = Flag('A', 'x y') +B = IntFlag('B', 'a b') +reveal_type(A.x) # N: Revealed type is "Literal[__main__.A.x]?" +reveal_type(B.a) # N: Revealed type is "Literal[__main__.B.a]?" +reveal_type(A.x.name) # N: Revealed type is "Literal['x']?" +reveal_type(B.a.name) # N: Revealed type is "Literal['a']?" + +# TODO: The revealed type should be 'int' here +reveal_type(A.x.value) # N: Revealed type is "Any" +reveal_type(B.a.value) # N: Revealed type is "Any" +[builtins fixtures/enum.pyi] + +[case testAnonymousFunctionalEnum] +from enum import Enum +class A: + def f(self) -> None: + E = Enum('E', 'a b') + self.x = E.a +a = A() +reveal_type(a.x) +[builtins fixtures/enum.pyi] +[out] +main:7: note: Revealed type is "__main__.A.E@4" + +[case testFunctionalEnumInClassBody] +from enum import Enum +class A: + E = Enum('E', 'a b') +class B: + E = Enum('E', 'a b') +x = A.E.a +y = B.E.a +if int(): + x = y # E: Incompatible types in assignment (expression has type "__main__.B.E", variable has type "__main__.A.E") +[builtins fixtures/enum.pyi] + +[case testFunctionalEnumProtocols] +from enum import IntEnum +Color = IntEnum('Color', 'red green blue') +reveal_type(Color['green']) # N: Revealed type is "__main__.Color" +for c in Color: + reveal_type(c) # N: Revealed type is "__main__.Color" +reveal_type(list(Color)) # N: Revealed type is "builtins.list[__main__.Color]" + +[builtins fixtures/list.pyi] + +[case testEnumWorkWithForward] +from enum import Enum +a: E = E.x # type: ignore[used-before-def] +class E(Enum): + x = 1 + y = 2 +[builtins fixtures/enum.pyi] +[out] + +[case testEnumWorkWithForward2] +from enum import Enum +b: F +F = Enum('F', {'x': 1, 'y': 2}) + +def fn(x: F) -> None: + pass +fn(b) +[builtins fixtures/enum.pyi] +[out] + +[case testFunctionalEnum] +# TODO: Needs to have enum34 stubs somehow +from enum import Enum +Eu = Enum(u'Eu', u'a b') +Eb = Enum(b'Eb', b'a b') # E: Enum() expects a string literal as the first argument +Gu = Enum(u'Gu', {u'a': 1}) +Gb = Enum(b'Gb', {b'a': 1}) # E: Enum() expects a string literal as the first argument +Hu = Enum(u'Hu', [u'a']) +Hb = Enum(b'Hb', [b'a']) # E: Enum() expects a string literal as the first argument +Eu.a +Eb.a +Gu.a +Gb.a +Hu.a +Hb.a +[builtins fixtures/enum.pyi] +[out] + +[case testEnumIncremental] +import m +reveal_type(m.E.a) +reveal_type(m.F.b) +[file m.py] +from enum import Enum +class E(Enum): + a = 1 + b = 2 +F = Enum('F', 'a b') +[builtins fixtures/enum.pyi] +[rechecked] +[stale] +[out1] +main:2: note: Revealed type is "Literal[m.E.a]?" +main:3: note: Revealed type is "Literal[m.F.b]?" +[out2] +main:2: note: Revealed type is "Literal[m.E.a]?" +main:3: note: Revealed type is "Literal[m.F.b]?" + +[case testEnumAuto] +from enum import Enum, auto +class Test(Enum): + a = auto() + b = auto() + +reveal_type(Test.a) # N: Revealed type is "Literal[__main__.Test.a]?" +[builtins fixtures/primitives.pyi] + +[case testEnumAttributeAccessMatrix] +from enum import Enum, IntEnum, IntFlag, Flag, EnumMeta, auto +from typing_extensions import Literal + +def is_x(val: Literal['x']) -> None: pass + +A1 = Enum('A1', 'x') +class A2(Enum): + x = auto() +class A3(Enum): + x = 1 + +is_x(reveal_type(A1.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(A1.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(A1.x.value) # N: Revealed type is "Any" +reveal_type(A1.x._value_) # N: Revealed type is "Any" +is_x(reveal_type(A2.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(A2.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(A2.x.value) # N: Revealed type is "builtins.int" +reveal_type(A2.x._value_) # N: Revealed type is "builtins.int" +is_x(reveal_type(A3.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(A3.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(A3.x.value) # N: Revealed type is "Literal[1]?" +reveal_type(A3.x._value_) # N: Revealed type is "Literal[1]?" + +B1 = IntEnum('B1', 'x') +class B2(IntEnum): + x = auto() +class B3(IntEnum): + x = 1 + +# TODO: getting B1.x._value_ and B2.x._value_ to have type 'int' requires a typeshed change + +is_x(reveal_type(B1.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(B1.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(B1.x.value) # N: Revealed type is "builtins.int" +reveal_type(B1.x._value_) # N: Revealed type is "Any" +is_x(reveal_type(B2.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(B2.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(B2.x.value) # N: Revealed type is "builtins.int" +reveal_type(B2.x._value_) # N: Revealed type is "builtins.int" +is_x(reveal_type(B3.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(B3.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(B3.x.value) # N: Revealed type is "Literal[1]?" +reveal_type(B3.x._value_) # N: Revealed type is "Literal[1]?" + +# TODO: C1.x.value and C2.x.value should also be of type 'int' +# This requires either a typeshed change or a plugin refinement + +C1 = IntFlag('C1', 'x') +class C2(IntFlag): + x = auto() +class C3(IntFlag): + x = 1 + +is_x(reveal_type(C1.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(C1.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(C1.x.value) # N: Revealed type is "Any" +reveal_type(C1.x._value_) # N: Revealed type is "Any" +is_x(reveal_type(C2.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(C2.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(C2.x.value) # N: Revealed type is "builtins.int" +reveal_type(C2.x._value_) # N: Revealed type is "builtins.int" +is_x(reveal_type(C3.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(C3.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(C3.x.value) # N: Revealed type is "Literal[1]?" +reveal_type(C3.x._value_) # N: Revealed type is "Literal[1]?" + +D1 = Flag('D1', 'x') +class D2(Flag): + x = auto() +class D3(Flag): + x = 1 + +is_x(reveal_type(D1.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(D1.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(D1.x.value) # N: Revealed type is "Any" +reveal_type(D1.x._value_) # N: Revealed type is "Any" +is_x(reveal_type(D2.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(D2.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(D2.x.value) # N: Revealed type is "builtins.int" +reveal_type(D2.x._value_) # N: Revealed type is "builtins.int" +is_x(reveal_type(D3.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(D3.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(D3.x.value) # N: Revealed type is "Literal[1]?" +reveal_type(D3.x._value_) # N: Revealed type is "Literal[1]?" + +# TODO: Generalize our enum functional API logic to work with subclasses of Enum +# See https://github.com/python/mypy/issues/6037 + +class Parent(Enum): pass +# E1 = Parent('E1', 'x') # See above TODO +class E2(Parent): + x = auto() +class E3(Parent): + x = 1 + +is_x(reveal_type(E2.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(E2.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(E2.x.value) # N: Revealed type is "builtins.int" +reveal_type(E2.x._value_) # N: Revealed type is "builtins.int" +is_x(reveal_type(E3.x.name)) # N: Revealed type is "Literal['x']" +is_x(reveal_type(E3.x._name_)) # N: Revealed type is "Literal['x']" +reveal_type(E3.x.value) # N: Revealed type is "Literal[1]?" +reveal_type(E3.x._value_) # N: Revealed type is "Literal[1]?" + + +# TODO: Figure out if we can construct enums using EnumMetas using the functional API. +# Also figure out if we even care about supporting that use case. +class F2(metaclass=EnumMeta): + x = auto() +class F3(metaclass=EnumMeta): + x = 1 + +F2.x.name # E: "F2" has no attribute "name" +F2.x._name_ # E: "F2" has no attribute "_name_" +F2.x.value # E: "F2" has no attribute "value" +F2.x._value_ # E: "F2" has no attribute "_value_" +F3.x.name # E: "F3" has no attribute "name" +F3.x._name_ # E: "F3" has no attribute "_name_" +F3.x.value # E: "F3" has no attribute "value" +F3.x._value_ # E: "F3" has no attribute "_value_" +[builtins fixtures/primitives.pyi] + +[case testEnumAttributeChangeIncremental] +from a import SomeEnum +reveal_type(SomeEnum.a.value) + +[file a.py] +from b import SomeEnum + +[file b.py] +from enum import Enum +class SomeEnum(Enum): + a = 1 + +[file b.py.2] +from enum import Enum +class SomeEnum(Enum): + a = "foo" +[builtins fixtures/enum.pyi] +[out] +main:2: note: Revealed type is "Literal[1]?" +[out2] +main:2: note: Revealed type is "Literal['foo']?" + +[case testEnumReachabilityChecksBasic] +from enum import Enum +from typing_extensions import Literal + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +x: Literal[Foo.A, Foo.B, Foo.C] +if x is Foo.A: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" +elif x is Foo.B: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.B]" +elif x is Foo.C: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.C]" +else: + reveal_type(x) # No output here: this branch is unreachable +reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.A], Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" + +if Foo.A is x: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" +elif Foo.B is x: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.B]" +elif Foo.C is x: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.C]" +else: + reveal_type(x) # No output here: this branch is unreachable +reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.A], Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" + +y: Foo +if y is Foo.A: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +elif y is Foo.B: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.B]" +elif y is Foo.C: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.C]" +else: + reveal_type(y) # No output here: this branch is unreachable +reveal_type(y) # N: Revealed type is "__main__.Foo" + +if Foo.A is y: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +elif Foo.B is y: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.B]" +elif Foo.C is y: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.C]" +else: + reveal_type(y) # No output here: this branch is unreachable +reveal_type(y) # N: Revealed type is "__main__.Foo" +[builtins fixtures/bool.pyi] + +[case testEnumReachabilityChecksWithOrdering] +from enum import Enum +from typing_extensions import Literal + +class Foo(Enum): + _order_ = "A B" + A = 1 + B = 2 + +Foo._order_ # E: "Type[Foo]" has no attribute "_order_" + +x: Literal[Foo.A, Foo.B] +if x is Foo.A: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" +elif x is Foo.B: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.B]" +else: + reveal_type(x) # No output here: this branch is unreachable + +class Bar(Enum): + __order__ = "A B" + A = 1 + B = 2 + +Bar.__order__ # E: "Type[Bar]" has no attribute "__order__" + +y: Literal[Bar.A, Bar.B] +if y is Bar.A: + reveal_type(y) # N: Revealed type is "Literal[__main__.Bar.A]" +elif y is Bar.B: + reveal_type(y) # N: Revealed type is "Literal[__main__.Bar.B]" +else: + reveal_type(y) # No output here: this branch is unreachable + +x2: Foo +if x2 is Foo.A: + reveal_type(x2) # N: Revealed type is "Literal[__main__.Foo.A]" +elif x2 is Foo.B: + reveal_type(x2) # N: Revealed type is "Literal[__main__.Foo.B]" +else: + reveal_type(x2) # No output here: this branch is unreachable + +y2: Bar +if y2 is Bar.A: + reveal_type(y2) # N: Revealed type is "Literal[__main__.Bar.A]" +elif y2 is Bar.B: + reveal_type(y2) # N: Revealed type is "Literal[__main__.Bar.B]" +else: + reveal_type(y2) # No output here: this branch is unreachable +[builtins fixtures/tuple.pyi] + +[case testEnumReachabilityChecksIndirect] +from enum import Enum +from typing_extensions import Literal, Final + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +def accepts_foo_a(x: Literal[Foo.A]) -> None: ... + +x: Foo +y: Literal[Foo.A] +z: Final = Foo.A + +if x is y: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +reveal_type(x) # N: Revealed type is "__main__.Foo" +if y is x: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +reveal_type(x) # N: Revealed type is "__main__.Foo" + +if x is z: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(z) # N: Revealed type is "Literal[__main__.Foo.A]?" + accepts_foo_a(z) +else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" + reveal_type(z) # N: Revealed type is "Literal[__main__.Foo.A]?" + accepts_foo_a(z) +reveal_type(x) # N: Revealed type is "__main__.Foo" +if z is x: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(z) # N: Revealed type is "Literal[__main__.Foo.A]?" + accepts_foo_a(z) +else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" + reveal_type(z) # N: Revealed type is "Literal[__main__.Foo.A]?" + accepts_foo_a(z) +reveal_type(x) # N: Revealed type is "__main__.Foo" + +if y is z: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(z) # N: Revealed type is "Literal[__main__.Foo.A]?" + accepts_foo_a(z) +else: + reveal_type(y) # No output: this branch is unreachable + reveal_type(z) # No output: this branch is unreachable +if z is y: + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(z) # N: Revealed type is "Literal[__main__.Foo.A]?" + accepts_foo_a(z) +else: + reveal_type(y) # No output: this branch is unreachable + reveal_type(z) # No output: this branch is unreachable +[builtins fixtures/bool.pyi] + +[case testEnumReachabilityNoNarrowingForUnionMessiness] +from enum import Enum +from typing_extensions import Literal + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +x: Foo +y: Literal[Foo.A, Foo.B] +z: Literal[Foo.B, Foo.C] + +# For the sake of simplicity, no narrowing is done when the narrower type is a Union. +if x is y: + reveal_type(x) # N: Revealed type is "__main__.Foo" + reveal_type(y) # N: Revealed type is "Union[Literal[__main__.Foo.A], Literal[__main__.Foo.B]]" +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" + reveal_type(y) # N: Revealed type is "Union[Literal[__main__.Foo.A], Literal[__main__.Foo.B]]" + +if y is z: + reveal_type(y) # N: Revealed type is "Union[Literal[__main__.Foo.A], Literal[__main__.Foo.B]]" + reveal_type(z) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" +else: + reveal_type(y) # N: Revealed type is "Union[Literal[__main__.Foo.A], Literal[__main__.Foo.B]]" + reveal_type(z) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" +[builtins fixtures/bool.pyi] + +[case testEnumReachabilityWithNone] +from enum import Enum +from typing import Optional + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +x: Optional[Foo] +if x: + reveal_type(x) # N: Revealed type is "__main__.Foo" +else: + reveal_type(x) # N: Revealed type is "None" + +if x is not None: + reveal_type(x) # N: Revealed type is "__main__.Foo" +else: + reveal_type(x) # N: Revealed type is "None" + +if x is Foo.A: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" +else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C], None]" +reveal_type(x) # N: Revealed type is "Union[__main__.Foo, None]" +[builtins fixtures/enum.pyi] + +[case testEnumReachabilityWithMultipleEnums] +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Foo(Enum): + A = 1 + B = 2 +class Bar(Enum): + A = 1 + B = 2 + +x1: Union[Foo, Bar] +if x1 is Foo.A: + reveal_type(x1) # N: Revealed type is "Literal[__main__.Foo.A]" +else: + reveal_type(x1) # N: Revealed type is "Union[Literal[__main__.Foo.B], __main__.Bar]" +reveal_type(x1) # N: Revealed type is "Union[__main__.Foo, __main__.Bar]" + +x2: Union[Foo, Bar] +if x2 is Bar.A: + reveal_type(x2) # N: Revealed type is "Literal[__main__.Bar.A]" +else: + reveal_type(x2) # N: Revealed type is "Union[__main__.Foo, Literal[__main__.Bar.B]]" +reveal_type(x2) # N: Revealed type is "Union[__main__.Foo, __main__.Bar]" + +x3: Union[Foo, Bar] +if x3 is Foo.A or x3 is Bar.A: + reveal_type(x3) # N: Revealed type is "Union[Literal[__main__.Foo.A], Literal[__main__.Bar.A]]" +else: + reveal_type(x3) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Bar.B]]" +reveal_type(x3) # N: Revealed type is "Union[__main__.Foo, __main__.Bar]" + +[builtins fixtures/bool.pyi] + +[case testEnumReachabilityPEP484ExampleWithFinal] +from typing import Union +from typing_extensions import Final +from enum import Enum + +class Empty(Enum): + token = 0 +_empty: Final = Empty.token + +def func(x: Union[int, None, Empty] = _empty) -> int: + boom = x + 42 # E: Unsupported left operand type for + ("None") \ + # E: Unsupported left operand type for + ("Empty") \ + # N: Left operand is of type "Union[int, Empty, None]" + if x is _empty: + reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" + return 0 + elif x is None: + reveal_type(x) # N: Revealed type is "None" + return 1 + else: # At this point typechecker knows that x can only have type int + reveal_type(x) # N: Revealed type is "builtins.int" + return x + 2 +[builtins fixtures/primitives.pyi] + +[case testEnumReachabilityPEP484ExampleWithMultipleValues] +from typing import Union +from enum import Enum + +class Reason(Enum): + timeout = 1 + error = 2 + +def process(response: Union[str, Reason] = '') -> str: + if response is Reason.timeout: + reveal_type(response) # N: Revealed type is "Literal[__main__.Reason.timeout]" + return 'TIMEOUT' + elif response is Reason.error: + reveal_type(response) # N: Revealed type is "Literal[__main__.Reason.error]" + return 'ERROR' + else: + # response can be only str, all other possible values exhausted + reveal_type(response) # N: Revealed type is "builtins.str" + return 'PROCESSED: ' + response +[builtins fixtures/primitives.pyi] + + +[case testEnumReachabilityPEP484ExampleSingleton] +from typing import Union +from typing_extensions import Final +from enum import Enum + +class Empty(Enum): + token = 0 +_empty = Empty.token + +def func(x: Union[int, None, Empty] = _empty) -> int: + boom = x + 42 # E: Unsupported left operand type for + ("None") \ + # E: Unsupported left operand type for + ("Empty") \ + # N: Left operand is of type "Union[int, Empty, None]" + if x is _empty: + reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" + return 0 + elif x is None: + reveal_type(x) # N: Revealed type is "None" + return 1 + else: # At this point typechecker knows that x can only have type int + reveal_type(x) # N: Revealed type is "builtins.int" + return x + 2 +[builtins fixtures/primitives.pyi] + +[case testEnumReachabilityPEP484ExampleSingletonWithMethod] +from typing import Union +from typing_extensions import Final +from enum import Enum + +class Empty(Enum): + token = lambda x: x + + def f(self) -> int: + return 1 + +_empty = Empty.token + +def func(x: Union[int, None, Empty] = _empty) -> int: + boom = x + 42 # E: Unsupported left operand type for + ("None") \ + # E: Unsupported left operand type for + ("Empty") \ + # N: Left operand is of type "Union[int, Empty, None]" + if x is _empty: + reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" + return 0 + elif x is None: + reveal_type(x) # N: Revealed type is "None" + return 1 + else: # At this point typechecker knows that x can only have type int + reveal_type(x) # N: Revealed type is "builtins.int" + return x + 2 +[builtins fixtures/primitives.pyi] + +[case testAssignEnumAsAttribute] +from enum import Enum + +class A: + def __init__(self) -> None: + self.b = Enum("b", [("foo", "bar")]) # E: Enum type as attribute is not supported + +reveal_type(A().b) # N: Revealed type is "Any" +[builtins fixtures/enum.pyi] + +[case testEnumReachabilityWithChaining] +from enum import Enum + +class Foo(Enum): + A = 1 + B = 2 + +x: Foo +y: Foo + +# We can't narrow anything in the else cases -- what if +# x is Foo.A and y is Foo.B or vice versa, for example? +if x is y is Foo.A: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +elif x is y is Foo.B: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.B]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.B]" +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" + reveal_type(y) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(y) # N: Revealed type is "__main__.Foo" + +if x is Foo.A is y: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +elif x is Foo.B is y: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.B]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.B]" +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" + reveal_type(y) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(y) # N: Revealed type is "__main__.Foo" + +if Foo.A is x is y: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +elif Foo.B is x is y: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.B]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.B]" +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" + reveal_type(y) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(y) # N: Revealed type is "__main__.Foo" + +[builtins fixtures/primitives.pyi] + +[case testEnumReachabilityWithChainingDisjoint] +# flags: --warn-unreachable +from enum import Enum + +class Foo(Enum): + A = 1 + B = 2 + + # Used to divide up a chained comparison into multiple identity groups + def __lt__(self, other: object) -> bool: return True + +x: Foo +y: Foo + +# No conflict +if x is Foo.A < y is Foo.B: + reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.B]" +else: + # Note: we can't narrow in this case. What if both x and y + # are Foo.A, for example? + reveal_type(x) # N: Revealed type is "__main__.Foo" + reveal_type(y) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(y) # N: Revealed type is "__main__.Foo" + +# The standard output when we end up inferring two disjoint facts about the same expr +if x is Foo.A and x is Foo.B: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" + +# ..and we get the same result if we have two disjoint groups within the same comp expr +if x is Foo.A < x is Foo.B: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" +[builtins fixtures/primitives.pyi] + +[case testEnumReachabilityWithChainingDirectConflict] +# flags: --warn-unreachable +from enum import Enum +from typing_extensions import Literal, Final + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +x: Foo +if x is Foo.A is Foo.B: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" + +literal_a: Literal[Foo.A] +literal_b: Literal[Foo.B] +if x is literal_a is literal_b: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" + +final_a: Final = Foo.A +final_b: Final = Foo.B +if x is final_a is final_b: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "__main__.Foo" +reveal_type(x) # N: Revealed type is "__main__.Foo" + +[builtins fixtures/primitives.pyi] + +[case testEnumReachabilityWithChainingBigDisjoints] +# flags: --warn-unreachable +from enum import Enum +from typing_extensions import Literal, Final + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + + def __lt__(self, other: object) -> bool: return True + +x0: Foo +x1: Foo +x2: Foo +x3: Foo +x4: Foo +x5: Foo + +if x0 is x1 is Foo.A is x2 < x3 is Foo.B is x4 is x5: + reveal_type(x0) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(x1) # N: Revealed type is "Literal[__main__.Foo.A]" + reveal_type(x2) # N: Revealed type is "Literal[__main__.Foo.A]" + + reveal_type(x3) # N: Revealed type is "Literal[__main__.Foo.B]" + reveal_type(x4) # N: Revealed type is "Literal[__main__.Foo.B]" + reveal_type(x5) # N: Revealed type is "Literal[__main__.Foo.B]" +else: + # We unfortunately can't narrow away anything. For example, + # what if x0 == Foo.A and x1 == Foo.B or vice versa? + reveal_type(x0) # N: Revealed type is "__main__.Foo" + reveal_type(x1) # N: Revealed type is "__main__.Foo" + reveal_type(x2) # N: Revealed type is "__main__.Foo" + + reveal_type(x3) # N: Revealed type is "__main__.Foo" + reveal_type(x4) # N: Revealed type is "__main__.Foo" + reveal_type(x5) # N: Revealed type is "__main__.Foo" +[builtins fixtures/primitives.pyi] + +[case testPrivateAttributeNotAsEnumMembers] +import enum + +class Comparator(enum.Enum): + LessThan = "<" + LessThanOrEqualTo = "<=" + EqualTo = "==" + NotEqualTo = "!=" + GreaterThanOrEqualTo = ">=" + GreaterThan = ">" + + __foo__ = { + LessThan: 1, + LessThanOrEqualTo: 2, + EqualTo: 3, + NotEqualTo: 4, + GreaterThanOrEqualTo: 5, + GreaterThan: 6, + } + + def foo(self) -> int: + return Comparator.__foo__[self.value] + +reveal_type(Comparator.__foo__) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +[builtins fixtures/dict.pyi] + +[case testEnumWithInstanceAttributes] +from enum import Enum +class Foo(Enum): + def __init__(self, value: int) -> None: + self.foo = "bar" + A = 1 + B = 2 + +a = Foo.A +reveal_type(a.value) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" +reveal_type(a._value_) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" +[builtins fixtures/enum.pyi] + +[case testNewSetsUnexpectedValueType] +from enum import Enum + +class bytes: + def __new__(cls): pass + +class Foo(bytes, Enum): + def __new__(cls, value: int) -> 'Foo': + obj = bytes.__new__(cls) + obj._value_ = "Number %d" % value + return obj + A = 1 + B = 2 + +a = Foo.A +reveal_type(a.value) # N: Revealed type is "Any" +reveal_type(a._value_) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testValueTypeWithNewInParentClass] +from enum import Enum + +class bytes: + def __new__(cls): pass + +class Foo(bytes, Enum): + def __new__(cls, value: int) -> 'Foo': + obj = bytes.__new__(cls) + obj._value_ = "Number %d" % value + return obj + +class Bar(Foo): + A = 1 + B = 2 + +a = Bar.A +reveal_type(a.value) # N: Revealed type is "Any" +reveal_type(a._value_) # N: Revealed type is "Any" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testEnumNarrowedToTwoLiterals] +# Regression test: two literals of an enum would be joined +# as the full type, regardless of the amount of elements +# the enum contains. +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +def f(x: Foo): + if x is Foo.A: + return x + if x is Foo.B: + pass + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.Foo.B], Literal[__main__.Foo.C]]" + +[builtins fixtures/bool.pyi] + +[case testEnumTypeCompatibleWithLiteralUnion] +from enum import Enum +from typing_extensions import Literal + +class E(Enum): + A = 1 + B = 2 + C = 3 + +e: E +a: Literal[E.A, E.B, E.C] = e +b: Literal[E.A, E.B] = e # E: Incompatible types in assignment (expression has type "E", variable has type "Literal[E.A, E.B]") +c: Literal[E.A, E.C] = e # E: Incompatible types in assignment (expression has type "E", variable has type "Literal[E.A, E.C]") +b = a # E: Incompatible types in assignment (expression has type "Literal[E.A, E.B, E.C]", variable has type "Literal[E.A, E.B]") +[builtins fixtures/bool.pyi] + +[case testIntEnumWithNewTypeValue] +from typing import NewType +from enum import IntEnum + +N = NewType("N", int) + +class E(IntEnum): + A = N(0) + +reveal_type(E.A.value) # N: Revealed type is "__main__.N" +[builtins fixtures/enum.pyi] + + +[case testEnumFinalValues] +from enum import Enum +class Medal(Enum): + gold = 1 + silver = 2 + +# Another value: +Medal.gold = 0 # E: Cannot assign to final attribute "gold" +# Same value: +Medal.silver = 2 # E: Cannot assign to final attribute "silver" +[builtins fixtures/enum.pyi] + + +[case testEnumFinalValuesCannotRedefineValueProp] +from enum import Enum +class Types(Enum): + key = 0 + value = 1 +[builtins fixtures/enum.pyi] + + +[case testEnumReusedKeys] +# https://github.com/python/mypy/issues/11248 +from enum import Enum +class Correct(Enum): + x = 'y' + y = 'x' +class Correct2(Enum): + x = 'y' + __z = 'y' + __z = 'x' +class Foo(Enum): + A = 1 + A = 'a' # E: Attempted to reuse member name "A" in Enum definition "Foo" \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") +reveal_type(Foo.A.value) # N: Revealed type is "Literal[1]?" + +class Bar(Enum): + A = 1 + B = A = 2 # E: Attempted to reuse member name "A" in Enum definition "Bar" +class Baz(Enum): + A = 1 + B, A = (1, 2) # E: Attempted to reuse member name "A" in Enum definition "Baz" +[builtins fixtures/tuple.pyi] + +[case testEnumReusedKeysOverlapWithLocalVar] +from enum import Enum +x = 1 +class Foo(Enum): + x = 2 + def method(self) -> None: + x = 3 +x = 4 +[builtins fixtures/bool.pyi] + +[case testEnumImplicitlyFinalForSubclassing] +from enum import Enum, IntEnum, Flag, IntFlag + +class NonEmptyEnum(Enum): + x = 1 +class NonEmptyIntEnum(IntEnum): + x = 1 +class NonEmptyFlag(Flag): + x = 1 +class NonEmptyIntFlag(IntFlag): + x = 1 + +class ErrorEnumWithValue(NonEmptyEnum): # E: Cannot extend enum with existing members: "NonEmptyEnum" + x = 1 # E: Cannot override final attribute "x" (previously declared in base class "NonEmptyEnum") +class ErrorIntEnumWithValue(NonEmptyIntEnum): # E: Cannot extend enum with existing members: "NonEmptyIntEnum" + x = 1 # E: Cannot override final attribute "x" (previously declared in base class "NonEmptyIntEnum") +class ErrorFlagWithValue(NonEmptyFlag): # E: Cannot extend enum with existing members: "NonEmptyFlag" + x = 1 # E: Cannot override final attribute "x" (previously declared in base class "NonEmptyFlag") +class ErrorIntFlagWithValue(NonEmptyIntFlag): # E: Cannot extend enum with existing members: "NonEmptyIntFlag" + x = 1 # E: Cannot override final attribute "x" (previously declared in base class "NonEmptyIntFlag") + +class ErrorEnumWithoutValue(NonEmptyEnum): # E: Cannot extend enum with existing members: "NonEmptyEnum" + pass +class ErrorIntEnumWithoutValue(NonEmptyIntEnum): # E: Cannot extend enum with existing members: "NonEmptyIntEnum" + pass +class ErrorFlagWithoutValue(NonEmptyFlag): # E: Cannot extend enum with existing members: "NonEmptyFlag" + pass +class ErrorIntFlagWithoutValue(NonEmptyIntFlag): # E: Cannot extend enum with existing members: "NonEmptyIntFlag" + pass +[builtins fixtures/bool.pyi] + +[case testSubclassingNonFinalEnums] +from enum import Enum, IntEnum, Flag, IntFlag, EnumMeta + +def decorator(func): + return func + +class EmptyEnum(Enum): + pass +class EmptyIntEnum(IntEnum): + pass +class EmptyFlag(Flag): + pass +class EmptyIntFlag(IntFlag): + pass +class EmptyEnumMeta(EnumMeta): + pass + +class NonEmptyEnumSub(EmptyEnum): + x = 1 +class NonEmptyIntEnumSub(EmptyIntEnum): + x = 1 +class NonEmptyFlagSub(EmptyFlag): + x = 1 +class NonEmptyIntFlagSub(EmptyIntFlag): + x = 1 +class NonEmptyEnumMetaSub(EmptyEnumMeta): + x = 1 + +class EmptyEnumSub(EmptyEnum): + def method(self) -> None: pass + @decorator + def other(self) -> None: pass +class EmptyIntEnumSub(EmptyIntEnum): + def method(self) -> None: pass +class EmptyFlagSub(EmptyFlag): + def method(self) -> None: pass +class EmptyIntFlagSub(EmptyIntFlag): + def method(self) -> None: pass +class EmptyEnumMetaSub(EmptyEnumMeta): + def method(self) -> None: pass + +class NestedEmptyEnumSub(EmptyEnumSub): + x = 1 +class NestedEmptyIntEnumSub(EmptyIntEnumSub): + x = 1 +class NestedEmptyFlagSub(EmptyFlagSub): + x = 1 +class NestedEmptyIntFlagSub(EmptyIntFlagSub): + x = 1 +class NestedEmptyEnumMetaSub(EmptyEnumMetaSub): + x = 1 +[builtins fixtures/bool.pyi] + +[case testEnumExplicitlyAndImplicitlyFinal] +from typing import final +from enum import Enum, IntEnum, Flag, IntFlag, EnumMeta + +@final +class EmptyEnum(Enum): + pass +@final +class EmptyIntEnum(IntEnum): + pass +@final +class EmptyFlag(Flag): + pass +@final +class EmptyIntFlag(IntFlag): + pass +@final +class EmptyEnumMeta(EnumMeta): + pass + +class EmptyEnumSub(EmptyEnum): # E: Cannot inherit from final class "EmptyEnum" + pass +class EmptyIntEnumSub(EmptyIntEnum): # E: Cannot inherit from final class "EmptyIntEnum" + pass +class EmptyFlagSub(EmptyFlag): # E: Cannot inherit from final class "EmptyFlag" + pass +class EmptyIntFlagSub(EmptyIntFlag): # E: Cannot inherit from final class "EmptyIntFlag" + pass +class EmptyEnumMetaSub(EmptyEnumMeta): # E: Cannot inherit from final class "EmptyEnumMeta" + pass + +@final +class NonEmptyEnum(Enum): + x = 1 +@final +class NonEmptyIntEnum(IntEnum): + x = 1 +@final +class NonEmptyFlag(Flag): + x = 1 +@final +class NonEmptyIntFlag(IntFlag): + x = 1 +@final +class NonEmptyEnumMeta(EnumMeta): + x = 1 + +class ErrorEnumWithoutValue(NonEmptyEnum): # E: Cannot inherit from final class "NonEmptyEnum" \ + # E: Cannot extend enum with existing members: "NonEmptyEnum" + pass +class ErrorIntEnumWithoutValue(NonEmptyIntEnum): # E: Cannot inherit from final class "NonEmptyIntEnum" \ + # E: Cannot extend enum with existing members: "NonEmptyIntEnum" + pass +class ErrorFlagWithoutValue(NonEmptyFlag): # E: Cannot inherit from final class "NonEmptyFlag" \ + # E: Cannot extend enum with existing members: "NonEmptyFlag" + pass +class ErrorIntFlagWithoutValue(NonEmptyIntFlag): # E: Cannot inherit from final class "NonEmptyIntFlag" \ + # E: Cannot extend enum with existing members: "NonEmptyIntFlag" + pass +class ErrorEnumMetaWithoutValue(NonEmptyEnumMeta): # E: Cannot inherit from final class "NonEmptyEnumMeta" + pass +[builtins fixtures/bool.pyi] + +[case testEnumFinalSubtypingEnumMetaSpecialCase] +from enum import EnumMeta +# `EnumMeta` types are not `Enum`s +class SubMeta(EnumMeta): + x = 1 +class SubSubMeta(SubMeta): + x = 2 +[builtins fixtures/bool.pyi] + +[case testEnumFinalSubtypingOverloadedSpecialCase] +from typing import overload +from enum import Enum, IntEnum, Flag, IntFlag, EnumMeta + +class EmptyEnum(Enum): + @overload + def method(self, arg: int) -> int: + pass + @overload + def method(self, arg: str) -> str: + pass + def method(self, arg): + pass +class EmptyIntEnum(IntEnum): + @overload + def method(self, arg: int) -> int: + pass + @overload + def method(self, arg: str) -> str: + pass + def method(self, arg): + pass +class EmptyFlag(Flag): + @overload + def method(self, arg: int) -> int: + pass + @overload + def method(self, arg: str) -> str: + pass + def method(self, arg): + pass +class EmptyIntFlag(IntFlag): + @overload + def method(self, arg: int) -> int: + pass + @overload + def method(self, arg: str) -> str: + pass + def method(self, arg): + pass +class EmptyEnumMeta(EnumMeta): + @overload + def method(self, arg: int) -> int: + pass + @overload + def method(self, arg: str) -> str: + pass + def method(self, arg): + pass + +class NonEmptyEnumSub(EmptyEnum): + x = 1 +class NonEmptyIntEnumSub(EmptyIntEnum): + x = 1 +class NonEmptyFlagSub(EmptyFlag): + x = 1 +class NonEmptyIntFlagSub(EmptyIntFlag): + x = 1 +class NonEmptyEnumMetaSub(EmptyEnumMeta): + x = 1 +[builtins fixtures/bool.pyi] + +[case testEnumFinalSubtypingMethodAndValueSpecialCase] +from enum import Enum, IntEnum, Flag, IntFlag, EnumMeta + +def decorator(func): + return func + +class NonEmptyEnum(Enum): + x = 1 + def method(self) -> None: pass + @decorator + def other(self) -> None: pass +class NonEmptyIntEnum(IntEnum): + x = 1 + def method(self) -> None: pass +class NonEmptyFlag(Flag): + x = 1 + def method(self) -> None: pass +class NonEmptyIntFlag(IntFlag): + x = 1 + def method(self) -> None: pass + +class ErrorEnumWithoutValue(NonEmptyEnum): # E: Cannot extend enum with existing members: "NonEmptyEnum" + pass +class ErrorIntEnumWithoutValue(NonEmptyIntEnum): # E: Cannot extend enum with existing members: "NonEmptyIntEnum" + pass +class ErrorFlagWithoutValue(NonEmptyFlag): # E: Cannot extend enum with existing members: "NonEmptyFlag" + pass +class ErrorIntFlagWithoutValue(NonEmptyIntFlag): # E: Cannot extend enum with existing members: "NonEmptyIntFlag" + pass +[builtins fixtures/bool.pyi] + +[case testFinalEnumWithClassDef] +from enum import Enum + +class A(Enum): + class Inner: pass +class B(A): pass # E: Cannot extend enum with existing members: "A" +[builtins fixtures/bool.pyi] + +[case testEnumFinalSpecialProps] +# https://github.com/python/mypy/issues/11699 +# https://github.com/python/mypy/issues/11820 +from enum import Enum, IntEnum + +class BaseWithSpecials: + __slots__ = () + __doc__ = 'doc' + __module__ = 'module' + __annotations__ = {'a': int} + __dict__ = {'a': 1} + +class E(BaseWithSpecials, Enum): + name = 'a' + value = 'b' + _name_ = 'a1' + _value_ = 'b2' + _order_ = 'X Y' + __order__ = 'X Y' + __slots__ = () + __doc__ = 'doc' + __module__ = 'module' + __annotations__ = {'a': int} + __dict__ = {'a': 1} + +class EI(IntEnum): + name = 'a' + value = 1 + _name_ = 'a1' + _value_ = 2 + _order_ = 'X Y' + __order__ = 'X Y' + __slots__ = () + __doc__ = 'doc' + __module__ = 'module' + __annotations__ = {'a': int} + __dict__ = {'a': 1} + +E._order_ = 'a' # E: Cannot assign to final attribute "_order_" +EI.value = 2 # E: Cannot assign to final attribute "value" +[builtins fixtures/dict.pyi] + +[case testEnumNotFinalWithMethodsAndUninitializedValues] +# https://github.com/python/mypy/issues/11578 +from enum import Enum +from typing import Final + +class A(Enum): + x: int + def method(self) -> int: pass +class B(A): + x = 1 # E: Cannot override writable attribute "x" with a final one + +class A1(Enum): + x: int = 1 # E: Enum members must be left unannotated \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members +class B1(A1): # E: Cannot extend enum with existing members: "A1" + pass + +class A2(Enum): + x = 2 +class B2(A2): # E: Cannot extend enum with existing members: "A2" + pass + +# We leave this `Final` without a value, +# because we need to test annotation only mode: +class A3(Enum): + x: Final[int] # type: ignore +class B3(A3): + x = 1 # E: Cannot override final attribute "x" (previously declared in base class "A3") + +[builtins fixtures/bool.pyi] + +[case testEnumNotFinalWithMethodsAndUninitializedValuesStub] +import lib + +[file lib.pyi] +from enum import Enum +class A(Enum): # E: Detected enum "lib.A" in a type stub with zero members. There is a chance this is due to a recent change in the semantics of enum membership. If so, use `member = value` to mark an enum member, instead of `member: type` \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members + x: int +class B(A): # E: Cannot extend enum with existing members: "A" + x = 1 # E: Cannot override writable attribute "x" with a final one + +class C(Enum): + x = 1 +class D(C): # E: Cannot extend enum with existing members: "C" \ + # E: Detected enum "lib.D" in a type stub with zero members. There is a chance this is due to a recent change in the semantics of enum membership. If so, use `member = value` to mark an enum member, instead of `member: type` \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members + x: int # E: Cannot assign to final name "x" +[builtins fixtures/bool.pyi] + +[case testEnumLiteralValues] +from enum import Enum + +class A(Enum): + str = "foo" + int = 1 + bool = False + tuple = (1,) + +reveal_type(A.str.value) # N: Revealed type is "Literal['foo']?" +reveal_type(A.int.value) # N: Revealed type is "Literal[1]?" +reveal_type(A.bool.value) # N: Revealed type is "Literal[False]?" +reveal_type(A.tuple.value) # N: Revealed type is "Tuple[Literal[1]?]" +[builtins fixtures/tuple.pyi] + +[case testFinalWithPrivateAssignment] +import enum +class Some(enum.Enum): + __priv = 1 + +class Other(Some): # Should pass + pass +[builtins fixtures/tuple.pyi] + +[case testFinalWithDunderAssignment] +import enum +class Some(enum.Enum): + __some__ = 1 + +class Other(Some): # Should pass + pass +[builtins fixtures/tuple.pyi] + +[case testFinalWithSunderAssignment] +import enum +class Some(enum.Enum): + _some_ = 1 + +class Other(Some): # Should pass + pass +[builtins fixtures/tuple.pyi] + +[case testFinalWithMethodAssignment] +import enum +from typing import overload +class Some(enum.Enum): + def lor(self, other) -> bool: + pass + + ror = lor + +class Other(Some): # Should pass + pass + + +class WithOverload(enum.IntEnum): + @overload + def meth(self, arg: int) -> int: pass + @overload + def meth(self, arg: str) -> str: pass + def meth(self, arg): pass + + alias = meth + +class SubWithOverload(WithOverload): # Should pass + pass +[builtins fixtures/tuple.pyi] + +[case testEnumBaseClassesOrder] +import enum + +# Base types: + +class First: + def __new__(cls, val): + pass + +class Second: + def __new__(cls, val): + pass + +class Third: + def __new__(cls, val): + pass + +class Mixin: + pass + +class EnumWithCustomNew(enum.Enum): + def __new__(cls, val): + pass + +class SecondEnumWithCustomNew(enum.Enum): + def __new__(cls, val): + pass + +# Correct Enums: + +class Correct0(enum.Enum): + pass + +class Correct1(Mixin, First, enum.Enum): + pass + +class Correct2(First, enum.Enum): + pass + +class Correct3(Mixin, enum.Enum): + pass + +class RegularClass(Mixin, First, Second): + pass + +class Correct5(enum.Enum): + pass + +# Correct inheritance: + +class _InheritingDataAndMixin(Correct1): + pass + +class _CorrectWithData(First, Correct0): + pass + +class _CorrectWithDataAndMixin(Mixin, First, Correct0): + pass + +class _CorrectWithMixin(Mixin, Correct2): + pass + +class _CorrectMultipleEnumBases(Correct0, Correct5): + pass + +class _MultipleEnumBasesAndMixin(int, Correct0, enum.Flag): + pass + +class _MultipleEnumBasesWithCustomNew(int, EnumWithCustomNew, SecondEnumWithCustomNew): + pass + +# Wrong Enums: + +class TwoDataTypesViaInheritance(Second, Correct2): # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Correct2" + pass + +class TwoDataTypesViaInheritanceAndMixin(Second, Correct2, Mixin): # E: No non-enum mixin classes are allowed after "__main__.Correct2" \ + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Correct2" + pass + +class MixinAfterEnum1(enum.Enum, Mixin): # E: No non-enum mixin classes are allowed after "enum.Enum" + pass + +class MixinAfterEnum2(First, enum.Enum, Mixin): # E: No non-enum mixin classes are allowed after "enum.Enum" + pass + +class TwoDataTypes(First, Second, enum.Enum): # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Second" + pass + +class TwoDataTypesAndIntEnumMixin(First, Second, enum.IntEnum, Mixin): # E: No non-enum mixin classes are allowed after "enum.IntEnum" \ + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Second" + pass + +class ThreeDataTypes(First, Second, Third, enum.Enum): # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Second" \ + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Third" + pass + +class ThreeDataTypesAndMixin(First, Second, Third, enum.Enum, Mixin): # E: No non-enum mixin classes are allowed after "enum.Enum" \ + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Second" \ + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Third" + pass + +class FromEnumAndOther1(Correct2, Second, enum.Enum): # E: No non-enum mixin classes are allowed after "__main__.Correct2" \ + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Second" + pass + +class FromEnumAndOther2(Correct2, Second): # E: No non-enum mixin classes are allowed after "__main__.Correct2" \ + # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.Second" + pass +[builtins fixtures/tuple.pyi] + +[case testRegression12258] +from enum import Enum + +class MyEnum(Enum): ... + +class BytesEnum(bytes, MyEnum): ... # Should be ok +[builtins fixtures/tuple.pyi] + +[case testEnumWithNewHierarchy] +import enum + +class A: + def __new__(cls, val): ... +class B(A): + def __new__(cls, val): ... +class C: + def __new__(cls, val): ... + +class E1(A, enum.Enum): ... +class E2(B, enum.Enum): ... + +# Errors: + +class W1(C, E1): ... # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.E1" +class W2(C, E2): ... # E: Only a single data type mixin is allowed for Enum subtypes, found extra "__main__.E2" +[builtins fixtures/tuple.pyi] + +[case testEnumValueUnionSimplification] +from enum import IntEnum +from typing import Any + +class C(IntEnum): + X = 0 + Y = 1 + Z = 2 + +def f1(c: C) -> None: + x = {'x': c.value} + reveal_type(x) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + +def f2(c: C, a: Any) -> None: + x = {'x': c.value, 'y': a} + reveal_type(x) # N: Revealed type is "builtins.dict[builtins.str, Any]" + y = {'y': a, 'x': c.value} + reveal_type(y) # N: Revealed type is "builtins.dict[builtins.str, Any]" +[builtins fixtures/dict.pyi] + +[case testEnumIgnoreIsDeleted] +from enum import Enum + +class C(Enum): + _ignore_ = 'X' + +C._ignore_ # E: "Type[C]" has no attribute "_ignore_" +[builtins fixtures/enum.pyi] + +[case testCanOverrideDunderAttributes] +import typing +from enum import Enum, Flag + +class BaseEnum(Enum): + __dunder__ = 1 + __labels__: typing.Dict[int, str] + +class Override(BaseEnum): + __dunder__ = 2 + __labels__ = {1: "1"} + +Override.__dunder__ = 3 +BaseEnum.__dunder__ = 3 +Override.__labels__ = {2: "2"} + +class FlagBase(Flag): + __dunder__ = 1 + __labels__: typing.Dict[int, str] + +class FlagOverride(FlagBase): + __dunder__ = 2 + __labels = {1: "1"} + +FlagOverride.__dunder__ = 3 +FlagBase.__dunder__ = 3 +FlagOverride.__labels__ = {2: "2"} +[builtins fixtures/dict.pyi] + +[case testCanNotInitialize__members__] +import typing +from enum import Enum + +class WritingMembers(Enum): + __members__: typing.Dict[Enum, Enum] = {} # E: Assigned "__members__" will be overridden by "Enum" internally + +class OnlyAnnotatedMembers(Enum): + __members__: typing.Dict[Enum, Enum] +[builtins fixtures/dict.pyi] + +[case testCanOverrideDunderOnNonFirstBaseEnum] +import typing +from enum import Enum + +class Some: + __labels__: typing.Dict[int, str] + +class A(Some, Enum): + __labels__ = {1: "1"} +[builtins fixtures/dict.pyi] + +[case testEnumWithPartialTypes] +from enum import Enum + +class Mixed(Enum): + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + b = None + + def check(self) -> None: + reveal_type(Mixed.a.value) # N: Revealed type is "builtins.list[Any]" + reveal_type(Mixed.b.value) # N: Revealed type is "None" + + # Inferring Any here instead of a union seems to be a deliberate + # choice; see the testEnumValueInhomogenous case above. + reveal_type(self.value) # N: Revealed type is "Any" + + for field in Mixed: + reveal_type(field.value) # N: Revealed type is "Any" + if field.value is None: + pass + +class AllPartialList(Enum): + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + b = [] # E: Need type annotation for "b" (hint: "b: List[] = ...") + + def check(self) -> None: + reveal_type(self.value) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/tuple.pyi] + +[case testEnumPrivateAttributeNotMember] +from enum import Enum + +class MyEnum(Enum): + A = 1 + B = 2 + __my_dict = {A: "ham", B: "spam"} + +# TODO: change the next line to use MyEnum._MyEnum__my_dict when mypy implements name mangling +x: MyEnum = MyEnum.__my_dict # E: Incompatible types in assignment (expression has type "Dict[int, str]", variable has type "MyEnum") +[builtins fixtures/enum.pyi] + +[case testEnumWithPrivateAttributeReachability] +# flags: --warn-unreachable +from enum import Enum + +class MyEnum(Enum): + A = 1 + B = 2 + __my_dict = {A: "ham", B: "spam"} + +e: MyEnum +if e == MyEnum.A: + reveal_type(e) # N: Revealed type is "Literal[__main__.MyEnum.A]" +elif e == MyEnum.B: + reveal_type(e) # N: Revealed type is "Literal[__main__.MyEnum.B]" +else: + reveal_type(e) # E: Statement is unreachable +[builtins fixtures/dict.pyi] + + +[case testEnumNonMemberSupport] +# flags: --python-version 3.11 +# This was added in 3.11 +from enum import Enum, nonmember + +class My(Enum): + a = 1 + b = 2 + c = nonmember(3) + +reveal_type(My.a) # N: Revealed type is "Literal[__main__.My.a]?" +reveal_type(My.b) # N: Revealed type is "Literal[__main__.My.b]?" +reveal_type(My.c) # N: Revealed type is "builtins.int" + +def accepts_my(my: My): + reveal_type(my.value) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" + +class Other(Enum): + a = 1 + @nonmember + class Support: + b = 2 + +reveal_type(Other.a) # N: Revealed type is "Literal[__main__.Other.a]?" +reveal_type(Other.Support.b) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + + +[case testEnumMemberSupport] +# flags: --python-version 3.11 +# This was added in 3.11 +from enum import Enum, member + +class A(Enum): + x = member(1) + y = 2 + +reveal_type(A.x) # N: Revealed type is "Literal[__main__.A.x]?" +reveal_type(A.x.value) # N: Revealed type is "Literal[1]?" +reveal_type(A.y) # N: Revealed type is "Literal[__main__.A.y]?" +reveal_type(A.y.value) # N: Revealed type is "Literal[2]?" + +def some_a(a: A): + reveal_type(a.value) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" +[builtins fixtures/dict.pyi] + + +[case testErrorOnAnnotatedMember] +from enum import Enum + +class Medal(Enum): + gold: int = 1 # E: Enum members must be left unannotated \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members + silver: str = 2 # E: Enum members must be left unannotated \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members \ + # E: Incompatible types in assignment (expression has type "int", variable has type "str") + bronze = 3 +[builtins fixtures/enum.pyi] + +[case testEnumMemberWithPlaceholder] +from enum import Enum + +class Pet(Enum): + CAT = ... + DOG: str = ... # E: Enum members must be left unannotated \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members \ + # E: Incompatible types in assignment (expression has type "ellipsis", variable has type "str") +[builtins fixtures/enum.pyi] + +[case testEnumValueWithPlaceholderNodeType] +# https://github.com/python/mypy/issues/11971 +from enum import Enum +from typing import Any, Callable, Dict +class Foo(Enum): + Bar: Foo = Callable[[str], None] # E: Enum members must be left unannotated \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members \ + # E: Incompatible types in assignment (expression has type "", variable has type "Foo") + Baz: Any = Callable[[Dict[str, "Missing"]], None] # E: Enum members must be left unannotated \ + # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members \ + # E: Type application targets a non-generic function or class \ + # E: Name "Missing" is not defined + +reveal_type(Foo.Bar) # N: Revealed type is "Literal[__main__.Foo.Bar]?" +reveal_type(Foo.Bar.value) # N: Revealed type is "__main__.Foo" +reveal_type(Foo.Baz) # N: Revealed type is "Literal[__main__.Foo.Baz]?" +reveal_type(Foo.Baz.value) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + + +[case testEnumWithOnlyImplicitMembersUsingAnnotationOnly] +# flags: --warn-unreachable +import enum + + +class E(enum.IntEnum): + A: int + B: int + + +def do_check(value: E) -> None: + reveal_type(value) # N: Revealed type is "__main__.E" + # this is a nonmember check, not an emum member check, and it should not narrow the value + if value is E.A: + return + + reveal_type(value) # N: Revealed type is "__main__.E" + "should be reachable" + +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test new file mode 100644 index 000000000000..10cc145d0c70 --- /dev/null +++ b/test-data/unit/check-errorcodes.test @@ -0,0 +1,1238 @@ +-- Tests for error codes and ignoring errors using error codes +-- +-- These implicitly use --show-error-codes. + +[case testErrorCodeNoAttribute] +import m +m.x # E: Module has no attribute "x" [attr-defined] +'x'.foobar # E: "str" has no attribute "foobar" [attr-defined] +from m import xx # E: Module "m" has no attribute "xx" [attr-defined] +from m import think # E: Module "m" has no attribute "think"; maybe "thing"? [attr-defined] +for x in 1: # E: "int" has no attribute "__iter__" (not iterable) [attr-defined] + pass +[file m.py] +thing = 0 +[builtins fixtures/module.pyi] + +[case testErrorCodeUndefinedName] +x # E: Name "x" is not defined [name-defined] +def f() -> None: + y # E: Name "y" is not defined [name-defined] +[file m.py] +[builtins fixtures/module.pyi] + +[case testErrorCodeUnclassifiedError] +class A: + def __init__(self) -> int: \ + # E: The return type of "__init__" must be None [misc] + pass + +[case testErrorCodeNoteHasNoCode] +reveal_type(1) # N: Revealed type is "Literal[1]?" + +[case testErrorCodeSyntaxError] +1 '' +[out] +main:1: error: invalid syntax [syntax] +[out version==3.10.0] +main:1: error: invalid syntax. Perhaps you forgot a comma? [syntax] + +[case testErrorCodeSyntaxError2] +def f(): # E: Type signature has too many arguments [syntax] + # type: (int) -> None + 1 + +x = 0 # type: x y # E: Syntax error in type comment "x y" [syntax] + +[case testErrorCodeSyntaxError3] +# This is a bit inconsistent -- syntax error would be more logical? +x: 'a b' # E: Invalid type comment or annotation [valid-type] +for v in x: # type: int, int # E: Syntax error in type annotation [syntax] \ + # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + pass + +[case testErrorCodeSyntaxErrorIgnoreNote] +# This is a bit inconsistent -- syntax error would be more logical? +x: 'a b' # type: ignore[valid-type] +for v in x: # type: int, int # type: ignore[syntax] + pass + +[case testErrorCodeIgnore1] +'x'.foobar # type: ignore[attr-defined] +'x'.foobar # type: ignore[xyz] # E: "str" has no attribute "foobar" [attr-defined] \ + # N: Error code "attr-defined" not covered by "type: ignore" comment +'x'.foobar # type: ignore + +[case testErrorCodeIgnore2] +a = 'x'.foobar # type: int # type: ignore[attr-defined] +b = 'x'.foobar # type: int # type: ignore[xyz] # E: "str" has no attribute "foobar" [attr-defined] \ + # N: Error code "attr-defined" not covered by "type: ignore" comment +c = 'x'.foobar # type: int # type: ignore + +[case testErrorCodeIgnoreMultiple1] +a = 'x'.foobar(b) # type: ignore[name-defined, attr-defined] +a = 'x'.foobar(b) # type: ignore[name-defined, xyz] # E: "str" has no attribute "foobar" [attr-defined] \ + # N: Error code "attr-defined" not covered by "type: ignore" comment +a = 'x'.foobar(b) # type: ignore[xyz, w, attr-defined] # E: Name "b" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment + +[case testErrorCodeIgnoreMultiple2] +a = 'x'.foobar(c) # type: int # type: ignore[name-defined, attr-defined] +b = 'x'.foobar(c) # type: int # type: ignore[name-defined, xyz] # E: "str" has no attribute "foobar" [attr-defined] \ + # N: Error code "attr-defined" not covered by "type: ignore" comment + +[case testErrorCodeWarnUnusedIgnores1] +# flags: --warn-unused-ignores +x # type: ignore[name-defined, attr-defined] # E: Unused "type: ignore[attr-defined]" comment [unused-ignore] + +[case testErrorCodeWarnUnusedIgnores2] +# flags: --warn-unused-ignores +"x".foobar(y) # type: ignore[name-defined, attr-defined] + +[case testErrorCodeWarnUnusedIgnores3] +# flags: --warn-unused-ignores +"x".foobar(y) # type: ignore[name-defined, attr-defined, xyz] # E: Unused "type: ignore[xyz]" comment [unused-ignore] + +[case testErrorCodeWarnUnusedIgnores4] +# flags: --warn-unused-ignores +"x".foobar(y) # type: ignore[name-defined, attr-defined, valid-type] # E: Unused "type: ignore[valid-type]" comment [unused-ignore] + +[case testErrorCodeWarnUnusedIgnores5] +# flags: --warn-unused-ignores +"x".foobar(y) # type: ignore[name-defined, attr-defined, valid-type, xyz] # E: Unused "type: ignore[valid-type, xyz]" comment [unused-ignore] + +[case testErrorCodeWarnUnusedIgnores6_NoDetailWhenSingleErrorCode] +# flags: --warn-unused-ignores +"x" # type: ignore[name-defined] # E: Unused "type: ignore" comment [unused-ignore] + +[case testErrorCodeMissingWhenRequired] +# flags: --enable-error-code ignore-without-code +"x" # type: ignore # E: "type: ignore" comment without error code [ignore-without-code] +y # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[name-defined]" instead) [ignore-without-code] +z # type: ignore[name-defined] +"a" # type: ignore[ignore-without-code] + +[case testErrorCodeMissingDoesntTrampleUnusedIgnoresWarning] +# flags: --enable-error-code ignore-without-code --warn-unused-ignores +"x" # type: ignore # E: Unused "type: ignore" comment [unused-ignore] +"y" # type: ignore[ignore-without-code] # E: Unused "type: ignore" comment [unused-ignore] +z # type: ignore[ignore-without-code] # E: Unused "type: ignore" comment [unused-ignore] \ + # E: Name "z" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment + +[case testErrorCodeMissingWholeFileIgnores] +# flags: --enable-error-code ignore-without-code +# type: ignore # whole file ignore +x +y # type: ignore # ignore the lack of error code since we ignore the whole file + +[case testErrorCodeMissingMultiple] +# flags: --enable-error-code ignore-without-code +from __future__ import annotations +class A: + attr: int + def func(self, var: int) -> A | None: ... + +a: A | None +# 'union-attr' should only be listed once (instead of twice) and list should be sorted +a.func("invalid string").attr # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[arg-type, union-attr]" instead) [ignore-without-code] +[builtins fixtures/tuple.pyi] + +[case testErrorCodeIgnoreWithExtraSpace] +x # type: ignore [name-defined] +x2 # type: ignore [ name-defined ] +x3 # type: ignore [ xyz , name-defined ] +x4 # type: ignore[xyz,name-defined] +y # type: ignore [xyz] # E: Name "y" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment +y # type: ignore[ xyz ] # E: Name "y" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment +y # type: ignore[ xyz , foo ] # E: Name "y" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment + +a = z # type: int # type: ignore [name-defined] +b = z2 # type: int # type: ignore [ name-defined ] +c = z2 # type: int # type: ignore [ name-defined , xyz ] +d = zz # type: int # type: ignore [xyz] # E: Name "zz" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment +e = zz # type: int # type: ignore [ xyz ] # E: Name "zz" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment +f = zz # type: int # type: ignore [ xyz,foo ] # E: Name "zz" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment + +[case testErrorCodeIgnoreAfterArgComment] +def f(x # type: xyz # type: ignore[name-defined] # Comment + ): + # type () -> None + pass + +def g(x # type: xyz # type: ignore # Comment + ): + # type () -> None + pass + +def h(x # type: xyz # type: ignore[foo] # E: Name "xyz" is not defined [name-defined] \ + # N: Error code "name-defined" not covered by "type: ignore" comment + ): + # type () -> None + pass + +[case testErrorCodeIgnoreWithNote] +import nostub # type: ignore[import] +from defusedxml import xyz # type: ignore[import] + +[case testErrorCodeBadIgnore] +import nostub # type: ignore xyz # E: Invalid "type: ignore" comment [syntax] \ + # E: Cannot find implementation or library stub for module named "nostub" [import-not-found] \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +import nostub # type: ignore[ # E: Invalid "type: ignore" comment [syntax] +import nostub # type: ignore[foo # E: Invalid "type: ignore" comment [syntax] +import nostub # type: ignore[foo, # E: Invalid "type: ignore" comment [syntax] +import nostub # type: ignore[foo]] # E: Invalid "type: ignore" comment [syntax] +import nostub # type: ignore[foo][bar] # E: Invalid "type: ignore" comment [syntax] +import nostub # type: ignore[foo] [bar] # E: Invalid "type: ignore" comment [syntax] + +x = 0 # type: ignore[ # E: Invalid "type: ignore" comment [syntax] + +def f(x, # type: int # type: ignore[ # E: Invalid "type: ignore" comment [syntax] + ): + # type: (...) -> None + pass + +[case testErrorCodeBadIgnoreNoExtraComment] +# Omit the E: ... comments, as they affect parsing +import nostub # type: ignore xyz +import nostub # type: ignore[xyz +import nostub # type: ignore[xyz][xyz] +x = 0 # type: ignore[ +def f(x, # type: int # type: ignore[ + ): + # type: (...) -> None + pass +[out] +main:2: error: Invalid "type: ignore" comment [syntax] +main:2: error: Cannot find implementation or library stub for module named "nostub" [import-not-found] +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:3: error: Invalid "type: ignore" comment [syntax] +main:4: error: Invalid "type: ignore" comment [syntax] +main:5: error: Invalid "type: ignore" comment [syntax] +main:6: error: Invalid "type: ignore" comment [syntax] + +[case testErrorCodeArgKindAndCount] +def f(x: int) -> None: pass # N: "f" defined here +f() # E: Missing positional argument "x" in call to "f" [call-arg] +f(1, 2) # E: Too many arguments for "f" [call-arg] +f(y=1) # E: Unexpected keyword argument "y" for "f" [call-arg] + +def g(*, x: int) -> None: pass +g() # E: Missing named argument "x" for "g" [call-arg] + +def h(x: int, y: int, z: int) -> None: pass +h(y=1, z=1) # E: Missing positional argument "x" in call to "h" [call-arg] +h(y=1) # E: Missing positional arguments "x", "z" in call to "h" [call-arg] + +[case testErrorCodeArgType] +def f(x: int) -> None: pass +f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" [arg-type] + +class A: + def g(self, *, x: int) -> None: pass + +A().g(x='') # E: Argument "x" to "g" of "A" has incompatible type "str"; expected "int" [arg-type] + +[case testErrorCodeInvalidType] +def f(): pass + +x: f # E: Function "__main__.f" is not valid as a type [valid-type] \ + # N: Perhaps you need "Callable[...]" or a callback protocol? + +import sys +y: sys # E: Module "sys" is not valid as a type [valid-type] \ + # N: Perhaps you meant to use a protocol matching the module structure? +z: y # E: Variable "__main__.y" is not valid as a type [valid-type] \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/tuple.pyi] + +[case testErrorCodeNeedTypeAnnotation] +from typing import TypeVar + +T = TypeVar('T') +def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same TypeVar [type-var] +x = f() # E: Need type annotation for "x" [var-annotated] +y = [] # E: Need type annotation for "y" (hint: "y: List[] = ...") [var-annotated] +[builtins fixtures/list.pyi] + +[case testErrorCodeBadOverride] +from typing import overload + +class A: + def f(self) -> int: + return 0 +class B(A): + def f(self) -> str: # E: Return type "str" of "f" incompatible with return type "int" in supertype "A" [override] + return '' +class C(A): + def f(self, x: int) -> int: # E: Signature of "f" incompatible with supertype "A" [override] \ + # N: Superclass: \ + # N: def f(self) -> int \ + # N: Subclass: \ + # N: def f(self, x: int) -> int + return 0 +class D: + def f(self, x: int) -> int: + return 0 +class E(D): + def f(self, x: str) -> int: # E: Argument 1 of "f" is incompatible with supertype "D"; supertype defines the argument type as "int" [override] \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + return 0 + +class O: + @overload + def f(self, x: int) -> None: pass + @overload + def f(self, x: str) -> None: pass + def f(self, x): + pass + +class OO(O): + @overload # E: Signature of "f" incompatible with supertype "O" [override] \ + # N: Overload variants must be defined in the same order as they are in "O" + def f(self, x: str) -> None: pass + @overload + def f(self, x: int) -> None: pass + def f(self, x): + pass + +[case testErrorCodeReturnValue] +def f() -> int: + return '' # E: Incompatible return value type (got "str", expected "int") [return-value] + +[case testErrorCodeMissingReturnValueInReturnStatement] +def f() -> int: + return # E: Return value expected [return-value] + +[case testErrorCodeAssignment] +x: str = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] + +def f(x: str = 0) -> None: # E: Incompatible default for argument "x" (default has type "int", argument has type "str") [assignment] + pass + +class A: + x = 0 + +class B(A): + x = '' # E: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") [assignment] + +a: A +a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + +[case testErrorCodeMissingTypeArg] +# flags: --disallow-any-generics +from typing import List, TypeVar +x: List # E: Missing type parameters for generic type "List" [type-arg] +y: list # E: Implicit generic "Any". Use "typing.List" and specify generic parameters [type-arg] +T = TypeVar('T') +L = List[List[T]] +z: L # E: Missing type parameters for generic type "L" [type-arg] +[builtins fixtures/list.pyi] + +[case testErrorCodeUnionAttribute] +from typing import Union +class A: + x: int +class B: + y: str +a: Union[A, B] +a.x # E: Item "B" of "Union[A, B]" has no attribute "x" [union-attr] + +[case testErrorCodeFunctionHasNoAnnotation] +# flags: --disallow-untyped-defs + +def f(x): # E: Function is missing a type annotation [no-untyped-def] + pass + +def g(x: int): # E: Function is missing a return type annotation [no-untyped-def] + pass + +def h(x) -> None: # E: Function is missing a type annotation for one or more arguments [no-untyped-def] + pass + +def gen(): # E: Function is missing a return type annotation [no-untyped-def] + yield 1 + +def gen2(x: int): # E: Function is missing a return type annotation [no-untyped-def] + yield 1 + +async def asyncf(): # E: Function is missing a return type annotation [no-untyped-def] + return 0 + +async def asyncf2(x: int): # E: Function is missing a return type annotation [no-untyped-def] + return 0 +[typing fixtures/typing-async.pyi] +[builtins fixtures/tuple.pyi] + +[case testErrorCodeCallUntypedFunction] +# flags: --disallow-untyped-calls + +def f() -> None: + g() # E: Call to untyped function "g" in typed context [no-untyped-call] + +def g(): + pass + +[case testErrorCodeIndexing] +from typing import Dict +x: Dict[int, int] +x[''] # E: Invalid index type "str" for "Dict[int, int]"; expected type "int" [index] +1[''] # E: Value of type "int" is not indexable [index] +1[''] = 1 # E: Unsupported target for indexed assignment ("int") [index] +[builtins fixtures/dict.pyi] + +[case testErrorCodeInvalidTypeArg] +from typing import TypeVar, Generic + +T = TypeVar('T', int, str) +TT = TypeVar('TT', int, None) +S = TypeVar('S', bound=str) + +def f(x: T) -> T: + return x + +f(object()) # E: Value of type variable "T" of "f" cannot be "object" [type-var] + +def g(x: S) -> S: + return x + +g(1) # E: Value of type variable "S" of "g" cannot be "int" [type-var] + +class C(Generic[T]): pass +class D(Generic[S]): pass +class E(Generic[S, T]): pass + +x: C[object] # E: Value of type variable "T" of "C" cannot be "object" [type-var] +y: D[int] # E: Type argument "int" of "D" must be a subtype of "str" [type-var] +z: D[int, int] # E: "D" expects 1 type argument, but 2 given [type-arg] + +def h(a: TT, s: S) -> None: + b: C[TT] # E: Invalid type argument value for "C" [type-var] + c: C[S] # E: Type variable "S" not valid as type argument value for "C" [type-var] + +[case testErrorCodeOperators] +class A: pass +A() + 1 # E: Unsupported left operand type for + ("A") [operator] +1 in A() # E: Unsupported right operand type for in ("A") [operator] +A() < 1 # E: Unsupported left operand type for < ("A") [operator] +-A() # E: Unsupported operand type for unary - ("A") [operator] ++A() # E: Unsupported operand type for unary + ("A") [operator] +~A() # E: Unsupported operand type for ~ ("A") [operator] + +class B: + def __add__(self, other: int) -> 'B': + return self + def __radd__(self, other: int) -> 'B': + return self + def __contains__(self, other: int) -> int: + return 0 + +B() + '' # E: Unsupported operand types for + ("B" and "str") [operator] +'' + B() # E: Unsupported operand types for + ("str" and "B") [operator] +'' in B() # E: Unsupported operand types for in ("str" and "B") [operator] + +1() # E: "int" not callable [operator] +[builtins fixtures/tuple.pyi] + +[case testErrorCodeListOrDictItem] +from typing import List, Dict +x: List[int] = [''] # E: List item 0 has incompatible type "str"; expected "int" [list-item] +y: Dict[int, int] = {1: ''} # E: Dict entry 0 has incompatible type "int": "str"; expected "int": "int" [dict-item] +[builtins fixtures/dict.pyi] + +[case testErrorCodeTypedDict] +from typing_extensions import TypedDict +class D(TypedDict): + x: int +class E(TypedDict): + x: int + y: int + +a: D = {'x': ''} # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") [typeddict-item] +b: D = {'y': ''} # E: Missing key "x" for TypedDict "D" [typeddict-item] \ + # E: Extra key "y" for TypedDict "D" [typeddict-unknown-key] +c = D(x=0) if int() else E(x=0, y=0) +c = {} # E: Missing key "x" for TypedDict "D" [typeddict-item] +d: D = {'x': '', 'y': 1} # E: Extra key "y" for TypedDict "D" [typeddict-unknown-key] \ + # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") [typeddict-item] + + +a['y'] = 1 # E: TypedDict "D" has no key "y" [typeddict-unknown-key] +a['x'] = 'x' # E: Value of "x" has incompatible type "str"; expected "int" [typeddict-item] +a['y'] # E: TypedDict "D" has no key "y" [typeddict-item] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testErrorCodeTypedDictNoteIgnore] +from typing_extensions import TypedDict +class A(TypedDict): + one_commonpart: int + two_commonparts: int + +a: A = {'one_commonpart': 1, 'two_commonparts': 2} +a['other_commonpart'] = 3 # type: ignore[typeddict-unknown-key] +not_exist = a['not_exist'] # type: ignore[typeddict-item] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testErrorCodeTypedDictSubCodeIgnore] +from typing_extensions import TypedDict +class D(TypedDict): + x: int +d: D = {'x': 1, 'y': 2} # type: ignore[typeddict-item] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testErrorCodeCannotDetermineType] +y = x # E: Cannot determine type of "x" [has-type] # E: Name "x" is used before definition [used-before-def] +reveal_type(y) # N: Revealed type is "Any" +x = None + +[case testErrorCodeRedundantCast] +# flags: --warn-redundant-casts +from typing import cast + +x = cast(int, int()) # E: Redundant cast to "int" [redundant-cast] + +[case testErrorCodeInvalidCommentSignature] +def f(x): # E: Type signature has too few arguments [syntax] + # type: () -> None + pass + +def g(x): # E: Type signature has too many arguments [syntax] + # type: (int, int) -> None + pass + +[case testErrorCodeNonOverlappingEquality] +# flags: --strict-equality +if int() == str(): # E: Non-overlapping equality check (left operand type: "int", right operand type: "str") [comparison-overlap] + pass +if int() != str(): # E: Non-overlapping equality check (left operand type: "int", right operand type: "str") [comparison-overlap] + pass +if int() is str(): # E: Non-overlapping identity check (left operand type: "int", right operand type: "str") [comparison-overlap] + pass +[builtins fixtures/primitives.pyi] + +[case testErrorCodeMissingModule] +from defusedxml import xyz # E: Cannot find implementation or library stub for module named "defusedxml" [import-not-found] +from nonexistent import foobar # E: Cannot find implementation or library stub for module named "nonexistent" [import-not-found] +import nonexistent2 # E: Cannot find implementation or library stub for module named "nonexistent2" [import-not-found] +from nonexistent3 import * # E: Cannot find implementation or library stub for module named "nonexistent3" [import-not-found] +from pkg import bad # E: Module "pkg" has no attribute "bad" [attr-defined] +from pkg.bad2 import bad3 # E: Cannot find implementation or library stub for module named "pkg.bad2" [import-not-found] \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +[file pkg/__init__.py] + +[case testErrorCodeAlreadyDefined] +x: int +x: str # E: Name "x" already defined on line 1 [no-redef] + +def f(): + pass +def f(): # E: Name "f" already defined on line 4 [no-redef] + pass + +[case testErrorCodeMissingReturn] +def f() -> int: # E: Missing return statement [return] + x = 0 + +[case testErrorCodeReturnValueNotExpected] +def f() -> None: + return 1 # E: No return value expected [return-value] + +[case testErrorCodeFunctionDoesNotReturnValue] +from typing import Callable + +def f() -> None: pass + +x = f() # E: "f" does not return a value (it only ever returns None) [func-returns-value] + +class A: + def g(self) -> None: pass + +y = A().g() # E: "g" of "A" does not return a value (it only ever returns None) [func-returns-value] + +c: Callable[[], None] +z = c() # E: Function does not return a value (it only ever returns None) [func-returns-value] + +[case testErrorCodeInstantiateAbstract] +from abc import abstractmethod + +class A: + @abstractmethod + def f(self): pass + +class B(A): + pass + +B() # E: Cannot instantiate abstract class "B" with abstract attribute "f" [abstract] + +[case testErrorCodeNewTypeNotSubclassable] +from typing import Union, NewType + +X = NewType('X', Union[int, str]) # E: Argument 2 to NewType(...) must be subclassable (got "Union[int, str]") [valid-newtype] + +[case testErrorCodeOverloadVariant] +from typing import overload + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x): + return x + +f(object()) # E: No overload variant of "f" matches argument type "object" [call-overload] \ + # N: Possible overload variants: \ + # N: def f(x: int) -> int \ + # N: def f(x: str) -> str + +f() # E: All overload variants of "f" require at least one argument [call-overload] \ + # N: Possible overload variants: \ + # N: def f(x: int) -> int \ + # N: def f(x: str) -> str + +f(1, 1) # E: No overload variant of "f" matches argument types "int", "int" [call-overload] \ + # N: Possible overload variants: \ + # N: def f(x: int) -> int \ + # N: def f(x: str) -> str + +[case testErrorCodeOverloadVariantIgnore] +from typing import overload + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x): + return x + +f(object()) # type: ignore[call-overload] + +[case testErrorCodeAnyFromUnfollowedImport] +# flags: --disallow-any-unimported +from m import C # type: ignore +def f(x: C) -> None: # E: Argument 1 to "f" becomes "Any" due to an unfollowed import [no-any-unimported] + pass + +def g() -> C: ... # E: Return type becomes "Any" due to an unfollowed import [no-any-unimported] + +[case testErrorCodeReturnAny] +# flags: --warn-return-any +def f(): pass + +def g() -> int: + return f() # E: Returning Any from function declared to return "int" [no-any-return] + +[case testErrorCodeFormatCall] +'{:d}'.format('no') # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") [str-format] +'{!x}'.format('Hm...') # E: Invalid conversion type "x", must be one of "r", "s" or "a" [str-format] +'}{'.format() # E: Invalid conversion specifier in format string: unexpected } [str-format] + +'%d' % 'no' # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float, SupportsInt]") [str-format] +'%d + %d' % (1, 2, 3) # E: Not all arguments converted during string formatting [str-format] + +'{}'.format(b'abc') # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes [str-bytes-safe] +'%s' % b'abc' # E: If x = b'abc' then "%s" % x produces "b'abc'", not "abc". If this is desired behavior use "%r" % x. Otherwise, decode the bytes [str-bytes-safe] +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testErrorCodeIgnoreNamedDefinedNote] +x: List[int] # type: ignore[name-defined] + +[case testErrorCodeProtocolProblemsIgnore] +from typing_extensions import Protocol + +class P(Protocol): + def f(self, x: str) -> None: ... + +class A: + def f(self, x: int) -> None: ... + +def g(p: P) -> None: pass + +p: A +g(p) # type: ignore[arg-type] +[builtins fixtures/tuple.pyi] + +[case testErrorCodeNoneReturnNoteIgnore] +# flags: --disallow-untyped-defs + +def f(): # type: ignore[no-untyped-def] + pass + +[case testErrorCodeVarianceNoteIgnore] +from typing import List +def f(x: List[object]) -> None: pass +a = [1] +f(a) # type: ignore[arg-type] +[builtins fixtures/list.pyi] + +[case testErrorCodeAssignToMethod] +class A: + def f(self) -> None: pass + +def g(self: A) -> None: pass + +A.f = g # E: Cannot assign to a method [method-assign] + +[case testErrorCodeDefinedHereNoteIgnore] +import m +m.f(kw=1) # type: ignore[call-arg] +[file m.py] +def f() -> None: pass + +[case testErrorCodeUnionNoteIgnore] +from typing import Union + +class Foo: + def __add__(self, x: Foo) -> Foo: pass + def __radd__(self, x: Foo) -> Foo: pass + +class Bar: + def __add__(self, x: Bar) -> Bar: pass + def __radd__(self, x: Bar) -> Bar: pass + +a: Union[Foo, Bar] + +a + a # type: ignore[operator] +a + Foo() # type: ignore[operator] +Foo() + a # type: ignore[operator] + +[case testErrorCodeTypeIgnoreMisspelled1] +x = y # type: ignored[foo] +xx = y # type: ignored [foo] +[out] +main:1: error: Name "ignored" is not defined [name-defined] +main:1: error: Name "y" is not defined [name-defined] +main:2: error: Name "ignored" is not defined [name-defined] +main:2: error: Name "y" is not defined [name-defined] + +[case testErrorCodeTypeIgnoreMisspelled2] +x = y # type: int # type: ignored[foo] +x = y # type: int # type: ignored [foo] +[out] +main:1: error: Syntax error in type comment "int" [syntax] +main:2: error: Syntax error in type comment "int" [syntax] + +[case testErrorCode__exit__Return] +class InvalidReturn: + def __exit__(self, x, y, z) -> bool: # E: "bool" is invalid as return type for "__exit__" that always returns False [exit-return] \ +# N: Use "typing_extensions.Literal[False]" as the return type or change it to "None" \ +# N: If return type of "__exit__" implies that it may return True, the context manager may swallow exceptions + return False +[builtins fixtures/bool.pyi] + +[case testErrorCodeOverloadedOperatorMethod] +from typing import Optional, overload + +class A: + @overload + def __add__(self, x: int) -> A: ... + @overload + def __add__(self, x: str) -> str: ... + def __add__(self, x): pass + +class B: + pass + +x: Optional[B] +A() + x # type: ignore[operator] + +class C: + @overload + def __rsub__(self, x: int) -> A: ... + @overload + def __rsub__(self, x: str) -> str: ... + def __rsub__(self, x): pass + +x - C() # type: ignore[operator] + +[case testErrorCodeMultiLineBinaryOperatorOperand] +from typing import Optional + +class C: pass + +def f() -> Optional[C]: + return None + +f( # type: ignore[operator] +) + C() + +[case testErrorCodeSpecialArgTypeErrors] +from typing import TypedDict + +class C(TypedDict): + x: int + +c: C +c.setdefault('x', '1') # type: ignore[typeddict-item] + +class A: + pass + +class B(A): + def f(self) -> None: + super(1, self).foo() # type: ignore[arg-type] + +def f(**x: int) -> None: + pass + +f(**1) # type: ignore[arg-type] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRedundantExpressions] +# flags: --enable-error-code redundant-expr +def foo() -> bool: ... + +lst = [1, 2, 3, 4] + +b = False or foo() # E: Left operand of "or" is always false [redundant-expr] +c = True and foo() # E: Left operand of "and" is always true [redundant-expr] +g = 3 if True else 4 # E: If condition is always true [redundant-expr] +h = 3 if False else 4 # E: If condition is always false [redundant-expr] +i = [x for x in lst if True] # E: If condition in comprehension is always true [redundant-expr] +j = [x for x in lst if False] # E: If condition in comprehension is always false [redundant-expr] +k = [x for x in lst if isinstance(x, int) or foo()] # E: If condition in comprehension is always true [redundant-expr] +[builtins fixtures/isinstancelist.pyi] + +[case testRedundantExprTruthiness] +# flags: --enable-error-code redundant-expr +from typing import List + +def maybe() -> bool: ... + +class Foo: + def __init__(self, x: List[int]) -> None: + self.x = x or [] + + def method(self) -> int: + if not self.x or maybe(): + return 1 + return 2 +[builtins fixtures/list.pyi] + +[case testNamedTupleNameMismatch] +from typing import NamedTuple + +Foo = NamedTuple("Bar", []) # E: First argument to namedtuple() should be "Foo", not "Bar" [name-match] +[builtins fixtures/tuple.pyi] + +[case testTypedDictNameMismatch] +from typing_extensions import TypedDict + +Foo = TypedDict("Bar", {}) # E: First argument "Bar" to TypedDict() does not match variable name "Foo" [name-match] +[builtins fixtures/dict.pyi] + +[case testTruthyBool] +# flags: --enable-error-code truthy-bool +from typing import List, Union, Any + +class Foo: + pass +class Bar: + pass + +foo = Foo() +if foo: # E: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + pass + +not foo # E: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + +zero = 0 +if zero: + pass + +not zero + +false = False +if false: + pass + +not false + +null = None +if null: + pass + +not null + +s = '' +if s: + pass + +not s + +good_union: Union[str, int] = 5 +if good_union: + pass +if not good_union: + pass + +not good_union + +bad_union: Union[Foo, Bar] = Foo() +if bad_union: # E: "__main__.bad_union" has type "Union[Foo, Bar]" of which no members implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + pass +if not bad_union: # E: "__main__.bad_union" has type "Union[Foo, Bar]" of which no members implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + pass + +not bad_union # E: "__main__.bad_union" has type "Union[Foo, Bar]" of which no members implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + +# 'object' is special and is treated as potentially falsy +obj: object = Foo() +if obj: + pass +if not obj: + pass + +not obj + +lst: List[int] = [] +if lst: + pass + +not lst + +a: Any +if a: + pass + +not a + +any_or_object: Union[object, Any] +if any_or_object: + pass + +not any_or_object + +if (my_foo := Foo()): # E: "__main__.my_foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + pass + +if my_a := (a or Foo()): # E: "__main__.Foo" returns "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + pass +[builtins fixtures/list.pyi] + +[case testTruthyFunctions] +def f(): + pass +if f: # E: Function "f" could always be true in boolean context [truthy-function] + pass +if not f: # E: Function "f" could always be true in boolean context [truthy-function] + pass +conditional_result = 'foo' if f else 'bar' # E: Function "f" could always be true in boolean context [truthy-function] + +not f # E: Function "f" could always be true in boolean context [truthy-function] + +[case testTruthyIterable] +# flags: --enable-error-code truthy-iterable +from typing import Iterable +def func(var: Iterable[str]) -> None: + if var: # E: "var" has type "Iterable[str]" which can always be true in boolean context. Consider using "Collection[str]" instead. [truthy-iterable] + ... + + not var # E: "var" has type "Iterable[str]" which can always be true in boolean context. Consider using "Collection[str]" instead. [truthy-iterable] + +[case testNoOverloadImplementation] +from typing import overload + +@overload # E: An overloaded function outside a stub file must have an implementation [no-overload-impl] +def f(arg: int) -> int: + ... + +@overload +def f(arg: str) -> str: + ... + +[case testSliceInDict39] +# flags: --python-version 3.9 --show-column-numbers +from typing import Dict +b: Dict[int, x:y] +c: Dict[x:y] + +[builtins fixtures/dict.pyi] +[out] +main:3:14: error: Invalid type comment or annotation [valid-type] +main:3:14: note: did you mean to use ',' instead of ':' ? +main:4:4: error: "dict" expects 2 type arguments, but 1 given [type-arg] +main:4:9: error: Invalid type comment or annotation [valid-type] +main:4:9: note: did you mean to use ',' instead of ':' ? + +[case testSliceInDict38] +# flags: --python-version 3.8 --show-column-numbers +from typing import Dict +b: Dict[int, x:y] +c: Dict[x:y] + +[builtins fixtures/dict.pyi] +[out] +main:3:14: error: Invalid type comment or annotation [valid-type] +main:3:14: note: did you mean to use ',' instead of ':' ? +main:4:4: error: "dict" expects 2 type arguments, but 1 given [type-arg] +main:4:9: error: Invalid type comment or annotation [valid-type] +main:4:9: note: did you mean to use ',' instead of ':' ? + + +[case testSliceInCustomTensorType] +# syntactically mimics torchtyping.TensorType +class TensorType: ... +t: TensorType["batch":..., float] # type: ignore +reveal_type(t) # N: Revealed type is "__main__.TensorType" +[builtins fixtures/tuple.pyi] + +[case testNoteAboutChangedTypedDictErrorCode] +from typing_extensions import TypedDict +class D(TypedDict): + x: int + +def f(d: D, s: str) -> None: + d[s] # type: ignore[xyz] \ + # E: TypedDict key must be a string literal; expected one of ("x") [literal-required] \ + # N: Error code "literal-required" not covered by "type: ignore" comment + d[s] # E: TypedDict key must be a string literal; expected one of ("x") [literal-required] + d[s] # type: ignore[misc] \ + # E: TypedDict key must be a string literal; expected one of ("x") [literal-required] \ + # N: Error code changed to literal-required; "type: ignore" comment may be out of date + d[s] # type: ignore[literal-required] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecommendErrorCode] +# type: ignore[whatever] # E: type ignore with error code is not supported for modules; use `# mypy: disable-error-code="whatever"` [syntax] \ + # N: Error code "syntax" not covered by "type: ignore" comment +1 + "asdf" + +[case testRecommendErrorCode2] +# type: ignore[whatever, other] # E: type ignore with error code is not supported for modules; use `# mypy: disable-error-code="whatever, other"` [syntax] \ + # N: Error code "syntax" not covered by "type: ignore" comment +1 + "asdf" + +[case testShowErrorCodesInConfig] +# flags: --config-file tmp/mypy.ini +# Test 'show_error_codes = True' in config doesn't raise an exception +var: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + +[file mypy.ini] +\[mypy] +show_error_codes = True + +[case testErrorCodeUnsafeSuper_no_empty] +from abc import abstractmethod + +class Base: + @abstractmethod + def meth(self) -> int: + raise NotImplementedError() +class Sub(Base): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe [safe-super] +[builtins fixtures/exception.pyi] + +[case testDedicatedErrorCodeForEmpty_no_empty] +from typing import Optional +def foo() -> int: ... # E: Missing return statement [empty-body] +def bar() -> None: ... +# This is inconsistent with how --warn-no-return behaves in general +# but we want to minimize fallout of finally handling empty bodies. +def baz() -> Optional[int]: ... # OK + +[case testDedicatedErrorCodeTypeAbstract] +import abc +from typing import TypeVar, Type + +class C(abc.ABC): + @abc.abstractmethod + def foo(self) -> None: ... + +T = TypeVar("T") +def test(tp: Type[T]) -> T: ... +test(C) # E: Only concrete class can be given where "Type[C]" is expected [type-abstract] + +class D(C): + @abc.abstractmethod + def bar(self) -> None: ... +cls: Type[C] = D # E: Can only assign concrete classes to a variable of type "Type[C]" [type-abstract] + +[case testUncheckedAnnotationCodeShown] +def f(): + x: int = "no" # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs [annotation-unchecked] + +[case testUncheckedAnnotationSuppressed] +# flags: --disable-error-code=annotation-unchecked +def f(): + x: int = "no" # No warning here + +[case testMethodAssignmentSuppressed] +# flags: --disable-error-code=method-assign +class A: + def f(self) -> None: pass + def g(self) -> None: pass + +def h(self: A) -> None: pass + +A.f = h +# This actually works at runtime, but there is no way to express this in current type system +A.f = A().g # E: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "Callable[[A], None]") [assignment] + +[case testMethodAssignCoveredByAssignmentIgnore] +class A: + def f(self) -> None: pass +def h(self: A) -> None: pass +A.f = h # type: ignore[assignment] + +[case testMethodAssignCoveredByAssignmentFlag] +# flags: --disable-error-code=assignment +class A: + def f(self) -> None: pass +def h(self: A) -> None: pass +A.f = h # OK + +[case testMethodAssignCoveredByAssignmentUnused] +# flags: --warn-unused-ignores +class A: + def f(self) -> None: pass +def h(self: A) -> None: pass +A.f = h # type: ignore[assignment] # E: Unused "type: ignore" comment, use narrower [method-assign] instead of [assignment] code [unused-ignore] + +[case testUnusedIgnoreEnableCode] +# flags: --enable-error-code=unused-ignore +x = 1 # type: ignore # E: Unused "type: ignore" comment [unused-ignore] + +[case testErrorCodeUnsafeOverloadError] +from typing import overload, Union + +@overload +def unsafe_func(x: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types [overload-overlap] +@overload +def unsafe_func(x: object) -> str: ... +def unsafe_func(x: object) -> Union[int, str]: + if isinstance(x, int): + return 42 + else: + return "some string" +[builtins fixtures/isinstancelist.pyi] + + +### +# unimported-reveal +### + +[case testUnimportedRevealType] +# flags: --enable-error-code=unimported-reveal +x = 1 +reveal_type(x) +[out] +main:3: error: Name "reveal_type" is not defined [unimported-reveal] +main:3: note: Did you forget to import it from "typing_extensions"? (Suggestion: "from typing_extensions import reveal_type") +main:3: note: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] + +[case testUnimportedRevealTypePy311] +# flags: --enable-error-code=unimported-reveal --python-version=3.11 +x = 1 +reveal_type(x) +[out] +main:3: error: Name "reveal_type" is not defined [unimported-reveal] +main:3: note: Did you forget to import it from "typing"? (Suggestion: "from typing import reveal_type") +main:3: note: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] + +[case testUnimportedRevealTypeInUncheckedFunc] +# flags: --enable-error-code=unimported-reveal +def unchecked(): + x = 1 + reveal_type(x) +[out] +main:4: error: Name "reveal_type" is not defined [unimported-reveal] +main:4: note: Did you forget to import it from "typing_extensions"? (Suggestion: "from typing_extensions import reveal_type") +main:4: note: Revealed type is "Any" +main:4: note: 'reveal_type' always outputs 'Any' in unchecked functions +[builtins fixtures/isinstancelist.pyi] + +[case testUnimportedRevealTypeImportedTypingExtensions] +# flags: --enable-error-code=unimported-reveal +from typing_extensions import reveal_type +x = 1 +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] + +[case testUnimportedRevealTypeImportedTyping311] +# flags: --enable-error-code=unimported-reveal --python-version=3.11 +from typing import reveal_type +x = 1 +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + +[case testUnimportedRevealLocals] +# flags: --enable-error-code=unimported-reveal +x = 1 +reveal_locals() +[out] +main:3: note: Revealed local types are: +main:3: note: x: builtins.int +main:3: error: Name "reveal_locals" is not defined [unimported-reveal] +[builtins fixtures/isinstancelist.pyi] + +[case testCovariantMutableOverride] +# flags: --enable-error-code=mutable-override +from typing import Any + +class C: + x: float + y: float + z: float + w: Any + @property + def foo(self) -> float: ... + @property + def bar(self) -> float: ... + @bar.setter + def bar(self, val: float) -> None: ... + baz: float + bad1: float + bad2: float +class D(C): + x: int # E: Covariant override of a mutable attribute (base class "C" defined the type as "float", expression has type "int") [mutable-override] + y: float + z: Any + w: float + foo: int + bar: int # E: Covariant override of a mutable attribute (base class "C" defined the type as "float", expression has type "int") [mutable-override] + def one(self) -> None: + self.baz = 5 + bad1 = 5 # E: Covariant override of a mutable attribute (base class "C" defined the type as "float", expression has type "int") [mutable-override] + def other(self) -> None: + self.bad2: int = 5 # E: Covariant override of a mutable attribute (base class "C" defined the type as "float", expression has type "int") [mutable-override] +[builtins fixtures/property.pyi] + +[case testNarrowedTypeNotSubtype] +from typing_extensions import TypeIs + +def f(x: str) -> TypeIs[int]: # E: Narrowed type "int" is not a subtype of input type "str" [narrowed-type-not-subtype] + pass + +[builtins fixtures/tuple.pyi] + + +[case testOverloadedFunctionSignature] +from typing import overload, Union + +@overload +def process(response1: float,response2: float) -> float: + ... +@overload +def process(response1: int,response2: int) -> int: # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [overload-cannot-match] + ... + +def process(response1,response2)-> Union[float,int]: + return response1 + response2 diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test new file mode 100644 index 000000000000..cd26c9bb408a --- /dev/null +++ b/test-data/unit/check-expressions.test @@ -0,0 +1,2454 @@ +-- Test cases for simple expressions. +-- +-- See also: +-- * check-functions.test contains test cases for calls. +-- * check-varargs.test contains test cases for *args. +-- * check-dynamic.test contains test cases related to 'Any' type. +-- * check-generics.test contains test cases for generic values. + + +-- None expression +-- --------------- + + +[case testNoneAsRvalue] +import typing +a: A +class A: pass +[out] + +[case testNoneAsArgument] +# flags: --no-strict-optional +import typing +def f(x: 'A', y: 'B') -> None: pass +f(None, None) +class A: pass +class B(A): pass +[out] + + +-- Simple expressions +-- ------------------ + + +[case testIntLiteral] +a = 0 +b: A +if int(): + b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "A") +if int(): + a = 1 +class A: + pass + +[case testStrLiteral] +a = '' +b: A +if int(): + b = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +if int(): + a = 'x' +if int(): + a = r"x" +if int(): + a = """foo""" +class A: + pass + +[case testFloatLiteral] +a = 0.0 +b: A +if str(): + b = 1.1 # E: Incompatible types in assignment (expression has type "float", variable has type "A") +if str(): + a = 1.1 +class A: + pass +[builtins fixtures/dict.pyi] + +[case testComplexLiteral] +a = 0.0j +b: A +if str(): + b = 1.1j # E: Incompatible types in assignment (expression has type "complex", variable has type "A") +if str(): + a = 1.1j +class A: + pass +[builtins fixtures/dict.pyi] + +[case testBytesLiteral] +b: bytes +a: A +if str(): + b = b'foo' +if str(): + b = br"foo" +if str(): + b = b'''foo''' +if str(): + a = b'foo' # E: Incompatible types in assignment (expression has type "bytes", variable has type "A") +class A: pass +[builtins fixtures/dict.pyi] + +[case testUnicodeLiteralInPython3] +s: str +if int(): + s = u'foo' +b: bytes +if int(): + b = u'foo' # E: Incompatible types in assignment (expression has type "str", variable has type "bytes") +[builtins fixtures/primitives.pyi] + + +-- Binary operators +-- ---------------- + + +[case testAdd] +a: A +b: B +c: C +if int(): + c = a + c # E: Unsupported operand types for + ("A" and "C") +if int(): + a = a + b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b + a # E: Unsupported left operand type for + ("B") +if int(): + c = a + b + +class A: + def __add__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testSub] +a: A +b: B +c: C +if int(): + c = a - c # E: Unsupported operand types for - ("A" and "C") +if int(): + a = a - b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b - a # E: Unsupported left operand type for - ("B") +if int(): + c = a - b + +class A: + def __sub__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testMul] +a: A +b: B +c: C +if int(): + c = a * c # E: Unsupported operand types for * ("A" and "C") +if int(): + a = a * b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b * a # E: Unsupported left operand type for * ("B") +if int(): + c = a * b + +class A: + def __mul__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testMatMul] +a: A +b: B +c: C +if int(): + c = a @ c # E: Unsupported operand types for @ ("A" and "C") +if int(): + a = a @ b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b @ a # E: Unsupported left operand type for @ ("B") +if int(): + c = a @ b + +class A: + def __matmul__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testDiv] +a: A +b: B +c: C +if int(): + c = a / c # E: Unsupported operand types for / ("A" and "C") + a = a / b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b / a # E: Unsupported left operand type for / ("B") +if int(): + c = a / b + +class A: + def __truediv__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testIntDiv] +a: A +b: B +c: C +if int(): + c = a // c # E: Unsupported operand types for // ("A" and "C") + a = a // b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b // a # E: Unsupported left operand type for // ("B") +if int(): + c = a // b + +class A: + def __floordiv__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testMod] +a: A +b: B +c: C +if int(): + c = a % c # E: Unsupported operand types for % ("A" and "C") +if int(): + a = a % b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b % a # E: Unsupported left operand type for % ("B") +if int(): + c = a % b + +class A: + def __mod__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testPow] +a: A +b: B +c: C +if int(): + c = a ** c # E: Unsupported operand types for ** ("A" and "C") +if int(): + a = a ** b # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b ** a # E: Unsupported left operand type for ** ("B") +if int(): + c = a ** b + +class A: + def __pow__(self, x: 'B') -> 'C': + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] + +[case testMiscBinaryOperators] +a: A +b: B +b = a & a # Fail +b = a | b # Fail +b = a ^ a # Fail +b = a << b # Fail +b = a >> a # Fail + +b = a & b +b = a | a +b = a ^ b +b = a << a +b = a >> b +class A: + def __and__(self, x: 'B') -> 'B': pass + def __or__(self, x: 'A') -> 'B': pass + def __xor__(self, x: 'B') -> 'B': pass + def __lshift__(self, x: 'A') -> 'B': pass + def __rshift__(self, x: 'B') -> 'B': pass +class B: pass +[builtins fixtures/tuple.pyi] +[out] +main:3: error: Unsupported operand types for & ("A" and "A") +main:4: error: Unsupported operand types for | ("A" and "B") +main:5: error: Unsupported operand types for ^ ("A" and "A") +main:6: error: Unsupported operand types for << ("A" and "B") +main:7: error: Unsupported operand types for >> ("A" and "A") + +[case testBooleanAndOr] +a: A +b: bool +if int(): + b = b and b +if int(): + b = b or b +if int(): + b = b and a # E: Incompatible types in assignment (expression has type "Union[Literal[False], A]", variable has type "bool") +if int(): + b = a and b # E: Incompatible types in assignment (expression has type "Union[A, bool]", variable has type "bool") +if int(): + b = b or a # E: Incompatible types in assignment (expression has type "Union[Literal[True], A]", variable has type "bool") +if int(): + b = a or b # E: Incompatible types in assignment (expression has type "Union[A, bool]", variable has type "bool") +class A: pass + +[builtins fixtures/bool.pyi] + +[case testRestrictedTypeAnd] + +b: bool +i: str +j = not b and i +if j: + reveal_type(j) # N: Revealed type is "builtins.str" +[builtins fixtures/bool.pyi] + +[case testRestrictedTypeOr] + +b: bool +i: str +j = b or i +if not j: + reveal_type(j) # N: Revealed type is "Literal['']" +[builtins fixtures/bool.pyi] + +[case testAndOr] + +s = "" +b = bool() +reveal_type(s and b or b) # N: Revealed type is "builtins.bool" +[builtins fixtures/bool.pyi] + +[case testRestrictedBoolAndOrWithGenerics] +from typing import List + +def f(a: List[str], b: bool) -> bool: + x = a and b + y: bool + return reveal_type(x or y) # N: Revealed type is "builtins.bool" +[builtins fixtures/list.pyi] + +[case testNonBooleanOr] +c: C +d: D +b: bool +if int(): + c = c or c +if int(): + c = c or d +if int(): + c = d or c +if int(): + b = c or c # E: Incompatible types in assignment (expression has type "C", variable has type "bool") +if int(): + d = c or d # E: Incompatible types in assignment (expression has type "C", variable has type "D") +if int(): + d = d or c # E: Incompatible types in assignment (expression has type "C", variable has type "D") +class C: pass +class D(C): pass +[builtins fixtures/bool.pyi] + +[case testInOperator] +from typing import Iterator, Iterable, Any +a: A +b: B +c: bool +d: D +e: Any +if int(): + c = c in a # E: Unsupported operand types for in ("bool" and "A") +if int(): + a = b in a # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + c = a in b # E: Unsupported right operand type for in ("B") +if int(): + c = b in d # E: Unsupported operand types for in ("B" and "D") +if int(): + c = b in a +if int(): + c = a in d +if int(): + c = e in d +if int(): + c = a in e + +class A: + def __contains__(self, x: 'B') -> bool: pass +class B: pass +class D(Iterable[A]): + def __iter__(self) -> Iterator[A]: pass +[builtins fixtures/bool.pyi] + +[case testNotInOperator] +from typing import Iterator, Iterable, Any +a: A +b: B +c: bool +d: D +e: Any +if int(): + c = c not in a # E: Unsupported operand types for in ("bool" and "A") +if int(): + a = b not in a # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + c = a not in b # E: Unsupported right operand type for in ("B") +if int(): + c = b not in d # E: Unsupported operand types for in ("B" and "D") +if int(): + c = b not in a +if int(): + c = a not in d +if int(): + c = e in d +if int(): + c = a in e + +class A: + def __contains__(self, x: 'B') -> bool: pass +class B: pass +class D(Iterable[A]): + def __iter__(self) -> Iterator[A]: pass +[builtins fixtures/bool.pyi] + +[case testNonBooleanContainsReturnValue] +a: A +b: bool +c: str +if int(): + b = a not in a +if int(): + b = a in a +if int(): + c = a not in a # E: Incompatible types in assignment (expression has type "bool", variable has type "str") +if int(): + c = a in a # E: Incompatible types in assignment (expression has type "bool", variable has type "str") + +class A: + def __contains__(self, x: 'A') -> str: pass +[builtins fixtures/bool.pyi] + +[case testInWithInvalidArgs] +a = 1 in ([1] + ['x']) # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testEq] +a: A +b: bool +if int(): + a = a == b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + a = a != b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + b = a == b +if int(): + b = a != b + +class A: + def __eq__(self, o: object) -> bool: pass + def __ne__(self, o: object) -> bool: pass +[builtins fixtures/bool.pyi] + +[case testLtAndGt] +a: A +b: B +bo: bool +if int(): + a = a < b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + a = a > b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + bo = a < b +if int(): + bo = a > b + +class A: + def __lt__(self, o: 'B') -> bool: pass + def __gt__(self, o: 'B') -> bool: pass +class B: + def __lt__(self, o: 'B') -> bool: pass + def __gt__(self, o: 'B') -> bool: pass +[builtins fixtures/bool.pyi] + +[case cmpIgnoredPy3] +a: A +b: B +bo: bool +bo = a <= b # E: Unsupported left operand type for <= ("A") + +class A: + def __cmp__(self, o: 'B') -> bool: pass +class B: + pass +[builtins fixtures/bool.pyi] + +[case testLeAndGe] +a: A +b: B +bo: bool +if int(): + a = a <= b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + a = a >= b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + bo = a <= b +if int(): + bo = a >= b + +class A: + def __le__(self, o: 'B') -> bool: pass + def __ge__(self, o: 'B') -> bool: pass +class B: + def __le__(self, o: 'B') -> bool: pass + def __ge__(self, o: 'B') -> bool: pass +[builtins fixtures/bool.pyi] + +[case testChainedComp] +a: A +b: B +bo: bool +a < a < b < b # Fail +a < b < b < b +a < a > a < b # Fail + +class A: + def __lt__(self, o: 'B') -> bool: pass + def __gt__(self, o: 'B') -> bool: pass +class B: + def __lt__(self, o: 'B') -> bool: pass + def __gt__(self, o: 'B') -> bool: pass +[builtins fixtures/bool.pyi] +[out] +main:4: error: Unsupported operand types for < ("A" and "A") +main:6: error: Unsupported operand types for < ("A" and "A") +main:6: error: Unsupported operand types for > ("A" and "A") + + +[case testChainedCompBoolRes] +a: A +b: B +bo: bool +if int(): + bo = a < b < b +if int(): + a = a < b < b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") + +class A: + def __lt__(self, o: 'B') -> bool: pass + def __gt__(self, o: 'B') -> bool: pass +class B: + def __lt__(self, o: 'B') -> bool: pass + def __gt__(self, o: 'B') -> bool: pass +[builtins fixtures/bool.pyi] + + +[case testChainedCompResTyp] +x: X +y: Y +a: A +b: B +p: P +bo: bool +if int(): + b = y == y == y +if int(): + bo = y == y == y # E: Incompatible types in assignment (expression has type "B", variable has type "bool") +if int(): + a = x < y +if int(): + a = x < y == y # E: Incompatible types in assignment (expression has type "P", variable has type "A") +if int(): + p = x < y == y + +class P: + pass +class A(P): + pass +class B(P): + pass + +class X: + def __lt__(self, o: 'Y') -> A: pass + def __gt__(self, o: 'Y') -> A: pass +class Y: + def __lt__(self, o: 'Y') -> A: pass + def __gt__(self, o: 'Y') -> A: pass + def __eq__(self, o: 'Y') -> B: pass # type: ignore +[builtins fixtures/bool.pyi] + + +[case testIs] +a: A +b: bool +if int(): + a = a is b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + b = a is b +if int(): + b = b is a +if int(): + b = a is None +class A: pass +[builtins fixtures/bool.pyi] + +[case testIsNot] +a: A +b: bool +if int(): + a = a is not b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + b = a is not b +if int(): + b = b is not a +if int(): + b = a is not None +class A: pass +[builtins fixtures/bool.pyi] + +[case testIsRightOperand] + +1 is 1() +[builtins fixtures/bool.pyi] +[out] +main:2: error: "int" not callable + +[case testReverseBinaryOperator] + +class A: + def __add__(self, x: int) -> int: pass +class B: + def __radd__(self, x: A) -> str: pass +s: str +n: int +if int(): + n = A() + 1 +if int(): + s = A() + B() +if int(): + n = A() + B() # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReverseBinaryOperator2] +class A: + def __add__(self, x: 'A') -> object: pass +class B: + def __radd__(self, x: A) -> str: pass +s: str +n: int +if int(): + s = A() + B() + n = A() + B() # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReverseBinaryOperator3] + +class N: + def __add__(self, x: 'N') -> object: pass +class A: + def __add__(self, x: N) -> int: pass +class B: + def __radd__(self, x: N) -> str: pass +s: str +s = A() + B() # E: Unsupported operand types for + ("A" and "B") + +[case testBinaryOperatorWithAnyRightOperand] +from typing import Any, cast +class A: pass +A() + cast(Any, 1) + +[case testReverseComparisonOperator] +class C: + def __gt__(self, x: 'A') -> object: pass +class A: + def __lt__(self, x: C) -> int: pass # E: Signatures of "__lt__" of "A" and "__gt__" of "C" are unsafely overlapping +class B: + def __gt__(self, x: A) -> str: pass +s: str +n: int +if int(): + n = A() < C() + s = A() < B() +if int(): + n = A() < B() # E: Incompatible types in assignment (expression has type "str", variable has type "int") + s = object() < B() # E: Unsupported operand types for > ("B" and "object") + +[case testReversibleComparisonWithExtraArgument] +class C: + def __lt__(self, o: object, x: str = "") -> int: ... + +[case testErrorContextAndBinaryOperators] +import typing +class A: + def __getitem__(self, i: str) -> int: pass +def f() -> None: + A()[1] # Error +class B: + A()[1] # Error +A()[1] # Error +[out] +main:5: error: Invalid index type "int" for "A"; expected type "str" +main:7: error: Invalid index type "int" for "A"; expected type "str" +main:8: error: Invalid index type "int" for "A"; expected type "str" + +[case testErrorContextAndBinaryOperators2] +import m +[file m.py] +import typing +class A: + def __getitem__(self, i: str) -> int: pass +def f() -> None: + A()[1] # Error +class B: + A()[1] # Error +A()[1] # Error +[out] +tmp/m.py:5: error: Invalid index type "int" for "A"; expected type "str" +tmp/m.py:7: error: Invalid index type "int" for "A"; expected type "str" +tmp/m.py:8: error: Invalid index type "int" for "A"; expected type "str" + + +[case testDivmod] +# flags: --disable-error-code=used-before-def +from typing import Tuple, Union, SupportsInt +_Decimal = Union[Decimal, int] +class Decimal(SupportsInt): + def __init__(self, int) -> None: ... + def __divmod__(self, other: _Decimal) -> Tuple[Decimal, Decimal]: ... + def __rdivmod__(self, other: _Decimal) -> Tuple[Decimal, Decimal]: ... + +i = 8 +f = 8.0 +d = Decimal(8) + +reveal_type(divmod(i, i)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +reveal_type(divmod(f, i)) # N: Revealed type is "Tuple[builtins.float, builtins.float]" +reveal_type(divmod(d, i)) # N: Revealed type is "Tuple[__main__.Decimal, __main__.Decimal]" + +reveal_type(divmod(i, f)) # N: Revealed type is "Tuple[builtins.float, builtins.float]" +reveal_type(divmod(f, f)) # N: Revealed type is "Tuple[builtins.float, builtins.float]" +divmod(d, f) # E: Unsupported operand types for divmod ("Decimal" and "float") + +reveal_type(divmod(i, d)) # N: Revealed type is "Tuple[__main__.Decimal, __main__.Decimal]" +divmod(f, d) # E: Unsupported operand types for divmod ("float" and "Decimal") +reveal_type(divmod(d, d)) # N: Revealed type is "Tuple[__main__.Decimal, __main__.Decimal]" + +# Now some bad calls +divmod() # E: "divmod" expects 2 arguments \ + # E: Missing positional arguments "_x", "_y" in call to "divmod" +divmod(7) # E: "divmod" expects 2 arguments \ + # E: Missing positional argument "_y" in call to "divmod" +divmod(7, 8, 9) # E: "divmod" expects 2 arguments \ + # E: Too many arguments for "divmod" +divmod(_x=7, _y=9) # E: "divmod" must be called with 2 positional arguments + +divmod('foo', 'foo') # E: Unsupported left operand type for divmod ("str") +divmod(i, 'foo') # E: Unsupported operand types for divmod ("int" and "str") +divmod(f, 'foo') # E: Unsupported operand types for divmod ("float" and "str") +divmod(d, 'foo') # E: Unsupported operand types for divmod ("Decimal" and "str") + +divmod('foo', i) # E: Unsupported operand types for divmod ("str" and "int") +divmod('foo', f) # E: Unsupported operand types for divmod ("str" and "float") +divmod('foo', d) # E: Unsupported operand types for divmod ("str" and "Decimal") + +[builtins fixtures/divmod.pyi] +[typing fixtures/typing-medium.pyi] + + +-- Unary operators +-- --------------- + + +[case testUnaryMinus] +a: A +b: B +if int(): + a = -a # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = -b # E: Unsupported operand type for unary - ("B") +if int(): + b = -a + +class A: + def __neg__(self) -> 'B': + pass +class B: + pass +[builtins fixtures/tuple.pyi] + +[case testUnaryPlus] +a: A +b: B +if int(): + a = +a # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = +b # E: Unsupported operand type for unary + ("B") +if int(): + b = +a + +class A: + def __pos__(self) -> 'B': + pass +class B: + pass +[builtins fixtures/tuple.pyi] + +[case testUnaryNot] +a: A +b: bool +if int(): + a = not b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if int(): + b = not a +if int(): + b = not b +class A: + pass +[builtins fixtures/bool.pyi] + +[case testUnaryBitwiseNeg] +a: A +b: B +if int(): + a = ~a # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = ~b # E: Unsupported operand type for ~ ("B") +if int(): + b = ~a + +class A: + def __invert__(self) -> 'B': + pass +class B: + pass + + +-- Indexing +-- -------- +[builtins fixtures/tuple.pyi] + + +[case testIndexing] +a: A +b: B +c: C +if int(): + c = a[c] # E: Invalid index type "C" for "A"; expected type "B" +if int(): + a = a[b] # E: Incompatible types in assignment (expression has type "C", variable has type "A") +if int(): + c = b[a] # E: Value of type "B" is not indexable +if int(): + c = a[b] + +class A: + def __getitem__(self, x: 'B') -> 'C': + pass +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] + +[case testIndexingAsLvalue] +a: A +b: B +c: C +a[c] = c # Fail +a[b] = a # Fail +b[a] = c # Fail +a[b] = c + +class A: + def __setitem__(self, x: 'B', y: 'C') -> None: + pass +class B: + pass +class C: + pass +[builtins fixtures/tuple.pyi] +[out] +main:4: error: Invalid index type "C" for "A"; expected type "B" +main:5: error: Incompatible types in assignment (expression has type "A", target has type "C") +main:6: error: Unsupported target for indexed assignment ("B") + +[case testOverloadedIndexing] +from foo import * +[file foo.pyi] +from typing import overload +a: A +b: B +c: C +a[b] +a[c] +a[1] # E: No overload variant of "__getitem__" of "A" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def __getitem__(self, B, /) -> int \ + # N: def __getitem__(self, C, /) -> str + +i: int +s: str +if int(): + i = a[b] +if int(): + s = a[b] # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + i = a[c] # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + s = a[c] + +class A: + @overload + def __getitem__(self, x: 'B') -> int: + pass + @overload + def __getitem__(self, x: 'C') -> str: + pass +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] +[out] + + +-- Cast expression +-- --------------- + + +[case testCastExpressions] +from typing import cast, Any +class A: pass +class B: pass +class C(A): pass +a: A +b: B +c: C + +if int(): + a = cast(A, a()) # E: "A" not callable +if int(): + a = cast(Any, a()) # E: "A" not callable + b = cast(A, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a = cast(A, b) +if int(): + a = cast(A, a) + c = cast(C, a) +if int(): + a = cast(A, c) +if int(): + a = cast(Any, b) + b = cast(Any, a) +[builtins fixtures/tuple.pyi] +[out] + +[case testAnyCast] +from typing import cast, Any +a: A +b: B +a = cast(Any, a()) # Fail +a = cast(Any, b) +b = cast(Any, a) +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] +[out] +main:4: error: "A" not callable + +-- assert_type() + +[case testAssertType] +from typing import assert_type, Any +from typing_extensions import Literal +a: int = 1 +returned = assert_type(a, int) +reveal_type(returned) # N: Revealed type is "builtins.int" +assert_type(a, str) # E: Expression is of type "int", not "str" +assert_type(a, Any) # E: Expression is of type "int", not "Any" +assert_type(a, Literal[1]) # E: Expression is of type "int", not "Literal[1]" +assert_type(42, Literal[42]) +assert_type(42, int) # E: Expression is of type "Literal[42]", not "int" +[builtins fixtures/tuple.pyi] + +[case testAssertTypeGeneric] +from typing import assert_type, TypeVar, Generic +from typing_extensions import Literal +T = TypeVar("T") +def f(x: T) -> T: return x +assert_type(f(1), int) +class Gen(Generic[T]): + def __new__(cls, obj: T) -> Gen[T]: ... +assert_type(Gen(1), Gen[int]) +# With type context, it infers Gen[Literal[1]] instead. +y: Gen[Literal[1]] = assert_type(Gen(1), Gen[Literal[1]]) + +[builtins fixtures/tuple.pyi] + +[case testAssertTypeUncheckedFunction] +from typing import assert_type +from typing_extensions import Literal +def f(): + x = 42 + assert_type(x, Literal[42]) +[out] +main:5: error: Expression is of type "Any", not "Literal[42]" +main:5: note: "assert_type" expects everything to be "Any" in unchecked functions +[builtins fixtures/tuple.pyi] + +[case testAssertTypeUncheckedFunctionWithUntypedCheck] +# flags: --check-untyped-defs +from typing import assert_type +from typing_extensions import Literal +def f(): + x = 42 + assert_type(x, Literal[42]) +[out] +main:6: error: Expression is of type "int", not "Literal[42]" +[builtins fixtures/tuple.pyi] + +[case testAssertTypeNoPromoteUnion] +from typing import Union, assert_type + +Scalar = Union[int, bool, bytes, bytearray] + + +def reduce_it(s: Scalar) -> Scalar: + return s + +assert_type(reduce_it(True), Scalar) +[builtins fixtures/tuple.pyi] + +[case testAssertTypeWithDeferredNodes] +from typing import Callable, TypeVar, assert_type + +T = TypeVar("T") + +def dec(f: Callable[[], T]) -> Callable[[], T]: + return f + +def func() -> None: + some = _inner_func() + assert_type(some, int) + +@dec +def _inner_func() -> int: + return 1 +[builtins fixtures/tuple.pyi] + +-- None return type +-- ---------------- + + +[case testNoneReturnTypeBasics] +def f() -> None: + pass + +class A: + def g(self, x: object) -> None: + pass + def __call__(self) -> None: + pass + +a: A +o: object +if int(): + a = f() # E: "f" does not return a value (it only ever returns None) +if int(): + o = a() # E: Function does not return a value (it only ever returns None) +if int(): + o = A().g(a) # E: "g" of "A" does not return a value (it only ever returns None) +if int(): + o = A.g(a, a) # E: "g" of "A" does not return a value (it only ever returns None) +A().g(f()) # E: "f" does not return a value (it only ever returns None) +x: A = f() # E: "f" does not return a value (it only ever returns None) +f() +A().g(a) +[builtins fixtures/tuple.pyi] + +[case testNoneReturnTypeWithStatements] +import typing +def f() -> None: pass + +if f(): # E: "f" does not return a value (it only ever returns None) + pass +elif f(): # E: "f" does not return a value (it only ever returns None) + pass +while f(): # E: "f" does not return a value (it only ever returns None) + pass +def g() -> object: + return f() # E: "f" does not return a value (it only ever returns None) +raise f() # E: "f" does not return a value (it only ever returns None) +[builtins fixtures/exception.pyi] + +[case testNoneReturnTypeWithExpressions] +from typing import cast + +def f() -> None: pass +class A: + def __add__(self, x: 'A') -> 'A': pass + +a: A +[f()] # E: "f" does not return a value (it only ever returns None) +f() + a # E: "f" does not return a value (it only ever returns None) +a + f() # E: "f" does not return a value (it only ever returns None) +f() == a # E: "f" does not return a value (it only ever returns None) +a != f() # E: "f" does not return a value (it only ever returns None) +cast(A, f()) +f().foo # E: "f" does not return a value (it only ever returns None) +[builtins fixtures/list.pyi] + +[case testNoneReturnTypeWithExpressions2] +import typing + +def f() -> None: pass +class A: + def __add__(self, x: 'A') -> 'A': + pass + +a: A +b: bool +f() in a # E: "f" does not return a value (it only ever returns None) # E: Unsupported right operand type for in ("A") +a < f() # E: "f" does not return a value (it only ever returns None) +f() <= a # E: "f" does not return a value (it only ever returns None) +a in f() # E: "f" does not return a value (it only ever returns None) +-f() # E: "f" does not return a value (it only ever returns None) +not f() # E: "f" does not return a value (it only ever returns None) +f() and b # E: "f" does not return a value (it only ever returns None) +b or f() # E: "f" does not return a value (it only ever returns None) +[builtins fixtures/bool.pyi] + + +-- Slicing +-- ------- + + +[case testGetSlice] +a: A +b: B +if int(): + a = a[1:2] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = a[1:] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = a[:2] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = a[:] # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +if int(): + b = a[1:2] +if int(): + b = a[1:] +if int(): + b = a[:2] +if int(): + b = a[:] + +class A: + def __getitem__(self, s: slice) -> 'B': pass +class B: pass +[builtins fixtures/slice.pyi] + +[case testSlicingWithInvalidBase] + +a: A +a[1:2] # E: Invalid index type "slice[int, int, None]" for "A"; expected type "int" +a[:] # E: Invalid index type "slice[None, None, None]" for "A"; expected type "int" +class A: + def __getitem__(self, n: int) -> 'A': pass +[builtins fixtures/slice.pyi] + +[case testSlicingWithNonindexable] + +o: object +o[1:2] # E: Value of type "object" is not indexable +o[:] # E: Value of type "object" is not indexable +[builtins fixtures/slice.pyi] + +[case testNonIntSliceBounds] +from typing import Any +a: Any +o: object +a[o:1] # E: Slice index must be an integer, SupportsIndex or None +a[1:o] # E: Slice index must be an integer, SupportsIndex or None +a[o:] # E: Slice index must be an integer, SupportsIndex or None +a[:o] # E: Slice index must be an integer, SupportsIndex or None +[builtins fixtures/slice.pyi] + +[case testSliceSupportsIndex] +import typing_extensions +class Index: + def __init__(self, value: int) -> None: + self.value = value + def __index__(self) -> int: + return self.value + +c = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +reveal_type(c[Index(0):Index(5)]) # N: Revealed type is "builtins.list[builtins.int]" +[file typing_extensions.pyi] +from typing import Protocol +class SupportsIndex(Protocol): + def __index__(self) -> int: ... +[builtins fixtures/slice.pyi] + +[case testNoneSliceBounds] +from typing import Any +a: Any +a[None:1] +a[1:None] +a[None:] +a[:None] +[builtins fixtures/slice.pyi] + +[case testNoneSliceBoundsWithStrictOptional] +from typing import Any +a: Any +a[None:1] +a[1:None] +a[None:] +a[:None] +[builtins fixtures/slice.pyi] + + +-- Lambdas +-- ------- + + +[case testTrivialLambda] +from typing import Callable +f = lambda: 1 # type: Callable[[], int] +if int(): + f = lambda: ''.x # E: "str" has no attribute "x" +if int(): + f = lambda: '' \ + # E: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "Callable[[], int]") \ + # E: Incompatible return value type (got "str", expected "int") + +[case testVoidLambda] +import typing +def void() -> None: + pass +x = lambda: void() # type: typing.Callable[[], None] + +[case testNoCrashOnLambdaGenerator] +# flags: --no-strict-optional +from typing import Iterator, Callable + +# These should not crash +lambda: (yield) + +gen: Callable[[], Iterator[str]] +gen = (lambda: (yield 1)) # E: Incompatible types in "yield" (actual type "int", expected type "str") + +def fun(cb: Callable[[], Iterator[str]]) -> None: + pass +fun(lambda: (yield from [1])) # E: Incompatible types in "yield from" (actual type "int", expected type "str") +[builtins fixtures/list.pyi] +[out] + +[case testLambdaAndReachability] +def f() -> None: + aa = [] + y = lambda x: 1 + aa.append(1) + 1() # E: "int" not callable +[builtins fixtures/list.pyi] + + +-- List comprehensions +-- ------------------- + + +[case testSimpleListComprehension] +from typing import List +a: List[A] +a = [x for x in a] +b = [x for x in a] # type: List[B] # E: List comprehension has incompatible type List[A]; expected List[B] +class A: pass +class B: pass +[builtins fixtures/for.pyi] + +[case testSimpleListComprehensionNestedTuples] +from typing import List, Tuple +l: List[Tuple[A, Tuple[A, B]]] +a = [a2 for a1, (a2, b1) in l] # type: List[A] +b = [a2 for a1, (a2, b1) in l] # type: List[B] # E: List comprehension has incompatible type List[A]; expected List[B] +class A: pass +class B: pass +[builtins fixtures/for.pyi] + +[case testSimpleListComprehensionNestedTuples2] +from typing import List, Tuple +l: List[Tuple[int, Tuple[int, str]]] +a = [f(d) for d, (i, s) in l] +b = [f(s) for d, (i, s) in l] # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +def f(x: int): pass +[builtins fixtures/for.pyi] + +[case testListComprehensionWithNonDirectMapping] +from typing import List +a: List[A] +b: List[B] +if int(): + b = [f(x) for x in a] +if int(): + a = [f(x) for x in a] # E: List comprehension has incompatible type List[B]; expected List[A] +([f(x) for x in b]) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +class A: pass +class B: pass +def f(a: A) -> B: pass +[builtins fixtures/for.pyi] + +[case testErrorInListComprehensionCondition] +from typing import List +a: List[A] +a = [x for x in a if x()] # E: "A" not callable +class A: pass +[builtins fixtures/for.pyi] + +[case testTypeInferenceOfListComprehension] +from typing import List +a: List[A] +o = [x for x in a] # type: List[object] +class A: pass +[builtins fixtures/for.pyi] + +[case testSimpleListComprehensionInClassBody] +from typing import List +class A: + a: List[A] + a = [x for x in a] + b = [x for x in a] # type: List[B] # E: List comprehension has incompatible type List[A]; expected List[B] +class B: pass +[builtins fixtures/for.pyi] +[out] + + +-- Set comprehension +-- ----------------- + + +[case testSimpleSetComprehension] +from typing import Set +a: Set[A] +a = {x for x in a} +b = {x for x in a} # type: Set[B] # E: Set comprehension has incompatible type Set[A]; expected Set[B] +class A: pass +class B: pass +[builtins fixtures/set.pyi] + + +-- Dictionary comprehension +-- ------------------------ + + +[case testSimpleDictionaryComprehension] +from typing import Dict, List, Tuple +abd: Dict[A, B] +abl: List[Tuple[A, B]] +abd = {a: b for a, b in abl} +x = {a: b for a, b in abl} # type: Dict[B, A] +y = {a: b for a, b in abl} # type: A +class A: pass +class B: pass +[builtins fixtures/dict.pyi] +[out] +main:5: error: Key expression in dictionary comprehension has incompatible type "A"; expected type "B" +main:5: error: Value expression in dictionary comprehension has incompatible type "B"; expected type "A" +main:6: error: Incompatible types in assignment (expression has type "Dict[A, B]", variable has type "A") + + +[case testDictionaryComprehensionWithNonDirectMapping] +from typing import Dict, List, Tuple +abd: Dict[A, B] +abl: List[Tuple[A, B]] +abd = {a: f(b) for a, b in abl} +class A: pass +class B: pass +class C: pass +def f(b: A) -> C: pass +[builtins fixtures/dict.pyi] +[out] +main:4: error: Value expression in dictionary comprehension has incompatible type "C"; expected type "B" +main:4: error: Argument 1 to "f" has incompatible type "B"; expected "A" + + +-- Generator expressions +-- --------------------- + + +[case testSimpleGeneratorExpression] +from typing import Iterator +# The implementation is mostly identical to list comprehensions, so only a few +# test cases is ok. +a: Iterator[int] +if int(): + a = (x for x in a) +b: Iterator[str] +if int(): + b = (x for x in a) # E: Generator has incompatible item type "int"; expected "str" +[builtins fixtures/for.pyi] + +[case testGeneratorIncompatibleErrorMessage] +from typing import Callable, Iterator, List + +a = [] # type: List[Callable[[], str]] +b: Iterator[Callable[[], int]] +if int(): + b = (x for x in a) # E: Generator has incompatible item type "Callable[[], str]"; expected "Callable[[], int]" +[builtins fixtures/list.pyi] + +-- Conditional expressions +-- ----------------------- + + +[case testSimpleConditionalExpression] +import typing +y = '' +x = 1 if y else 2 +if int(): + x = 3 +if int(): + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testConditionalExpressionWithEmptyCondition] +import typing +def f() -> None: pass +x = 1 if f() else 2 # E: "f" does not return a value (it only ever returns None) + +[case testConditionalExpressionWithSubtyping] +import typing +class A: pass +class B(A): pass +x = B() if bool() else A() +if int(): + x = A() +if int(): + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +y = A() if bool() else B() +if int(): + y = A() +if int(): + y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +[builtins fixtures/bool.pyi] + +[case testConditionalExpressionAndTypeContext] +import typing +x = [1] if bool() else [] +if int(): + x = [1] +if int(): + x = ['x'] # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testConditionalExpressionUnion] +from typing import Union +reveal_type(1 if bool() else 2) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" +reveal_type(1 if bool() else '') # N: Revealed type is "Union[Literal[1]?, Literal['']?]" +x: Union[int, str] = reveal_type(1 if bool() else '') # N: Revealed type is "Union[Literal[1]?, Literal['']?]" +class A: + pass +class B(A): + pass +class C: + pass +class D(A): + pass +a = A() +b = B() +c = C() +d = D() +reveal_type(a if bool() else b) # N: Revealed type is "__main__.A" +reveal_type(b if bool() else c) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(c if bool() else b) # N: Revealed type is "Union[__main__.C, __main__.B]" +reveal_type(c if bool() else a) # N: Revealed type is "Union[__main__.C, __main__.A]" +reveal_type(d if bool() else b) # N: Revealed type is "Union[__main__.D, __main__.B]" +[builtins fixtures/bool.pyi] + +[case testConditionalExpressionUnionWithAny] +from typing import Union, Any +a: Any +x: Union[int, str] = reveal_type(a if int() else 1) # N: Revealed type is "Union[Any, Literal[1]?]" +reveal_type(a if int() else 1) # N: Revealed type is "Union[Any, Literal[1]?]" + +[case testConditionalExpressionStatementNoReturn] +from typing import List, Union +x = [] +y = "" +x.append(y) if bool() else x.append(y) +z = x.append(y) if bool() else x.append(y) # E: "append" of "list" does not return a value (it only ever returns None) +[builtins fixtures/list.pyi] + +-- Special cases +-- ------------- + + +[case testOperationsWithNonInstanceTypes] +from typing import cast +class A: + def __add__(self, a: 'A') -> 'A': pass +def f() -> None: + pass +a: A +None + a # E: Unsupported left operand type for + ("None") +f + a # E: Unsupported left operand type for + ("Callable[[], None]") +a + f # E: Unsupported operand types for + ("A" and "Callable[[], None]") +cast(A, f) +[case testOperatorMethodWithInvalidArgCount] + +a: A +a + a # Fail + +class A: + def __add__(self) -> 'A': + pass +[out] +main:3: error: Too many arguments for "__add__" of "A" + +[case testOperatorMethodAsVar] +from typing import Any +class A: + def __init__(self, _add: Any) -> None: + self.__add__ = _add +a: A +a + a +[out] + +[case testOperatorMethodAsVar2] + +class A: + def f(self, x: int) -> str: pass + __add__ = f +s: str +s = A() + 1 +A() + (A() + 1) +[out] +main:7: error: Argument 1 has incompatible type "str"; expected "int" + +[case testIndexedLvalueWithSubtypes] +a: A +b: B +c: C +a[c] = c +a[b] = c +a[c] = b + +class A: + def __setitem__(self, x: 'B', y: 'B') -> None: + pass +class B: + pass +class C(B): + pass +[builtins fixtures/tuple.pyi] +[out] + + +-- Ellipsis +-- -------- + + +[case testEllipsis] + +a: A +if str(): + a = ... # E: Incompatible types in assignment (expression has type "ellipsis", variable has type "A") +b = ... +c = ... +if str(): + b = c +....__class__ +....a # E: "ellipsis" has no attribute "a" + +class A: pass +[builtins fixtures/dict-full.pyi] + + +-- Yield expression +-- ---------------- + + +[case testYieldExpression] +def f(x: int) -> None: + x = yield f('') + x = 1 +[builtins fixtures/for.pyi] +[out] +main:1: error: The return type of a generator function should be "Generator" or one of its supertypes +main:2: error: "f" does not return a value (it only ever returns None) +main:2: error: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testYieldExpressionWithNone] +from typing import Iterator +def f(x: int) -> Iterator[None]: + (yield) +[builtins fixtures/for.pyi] +[out] + + +-- Yield from expression +-- ---------------- + + +[case testYieldFromIteratorHasNoValue] +from typing import Iterator +def f() -> Iterator[int]: + yield 5 +def g() -> Iterator[int]: + a = yield from f() # E: Function does not return a value (it only ever returns None) + +[case testYieldFromGeneratorHasValue] +from typing import Iterator, Generator +def f() -> Generator[int, None, str]: + yield 5 + return "ham" +def g() -> Iterator[int]: + a = "string" + a = yield from f() +[out] + +[case testYieldFromTupleExpression] +from typing import Generator +def g() -> Generator[int, None, None]: + x = yield from () # E: Function does not return a value (it only ever returns None) + x = yield from (0, 1, 2) # E: Function does not return a value (it only ever returns None) + x = yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int") \ + # E: Function does not return a value (it only ever returns None) + x = yield from ("ERROR",) # E: Incompatible types in "yield from" (actual type "str", expected type "int") \ + # E: Function does not return a value (it only ever returns None) +[builtins fixtures/tuple.pyi] + +-- dict(...) +-- --------- + + +-- Note that the stub used in unit tests does not have all overload +-- variants, but it should not matter. + +[case testDictWithKeywordArgsOnly] +from typing import Dict, Any +d1 = dict(a=1, b=2) # type: Dict[str, int] +d2 = dict(a=1, b='') # type: Dict[str, int] # E: Dict entry 1 has incompatible type "str": "str"; expected "str": "int" +d3 = dict(a=1) # type: Dict[int, int] # E: Dict entry 0 has incompatible type "str": "int"; expected "int": "int" +d4 = dict(a=1, b=1) +d4.xyz # E: "Dict[str, int]" has no attribute "xyz" +d5 = dict(a=1, b='') # type: Dict[str, Any] +[builtins fixtures/dict.pyi] + +[case testDictWithoutKeywordArgs] +from typing import Dict +d = dict() # E: Need type annotation for "d" (hint: "d: Dict[, ] = ...") +d2 = dict() # type: Dict[int, str] +dict(undefined) # E: Name "undefined" is not defined +[builtins fixtures/dict.pyi] + +[case testDictFromList] +from typing import Dict +d = dict([(1, 'x'), (2, 'y')]) +d() # E: "Dict[int, str]" not callable +d2 = dict([(1, 'x')]) # type: Dict[str, str] # E: List item 0 has incompatible type "Tuple[int, str]"; expected "Tuple[str, str]" +[builtins fixtures/dict.pyi] + +[case testDictFromIterableAndKeywordArg] +from typing import Dict +it = [('x', 1)] + +d = dict(it, x=1) +d() # E: "Dict[str, int]" not callable + +d2 = dict(it, x='') +d2() # E: "Dict[str, object]" not callable + +d3 = dict(it, x='') # type: Dict[str, int] # E: Argument "x" to "dict" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testDictFromIterableAndKeywordArg2] +it = [(1, 'x')] +dict(it, x='y') # E: Keyword argument only valid with "str" key type in call to "dict" +[builtins fixtures/dict.pyi] + +[case testDictFromIterableAndKeywordArg3] +d = dict([], x=1) +d() # E: "Dict[str, int]" not callable +[builtins fixtures/dict.pyi] + +[case testDictFromIterableAndStarStarArgs] +from typing import Dict +it = [('x', 1)] + +kw = {'x': 1} +d = dict(it, **kw) +d() # E: "Dict[str, int]" not callable + +kw2 = {'x': ''} +d2 = dict(it, **kw2) +d2() # E: "Dict[str, object]" not callable + +d3 = dict(it, **kw2) # type: Dict[str, int] # E: Argument 2 to "dict" has incompatible type "**Dict[str, str]"; expected "int" +[builtins fixtures/dict.pyi] + +[case testDictFromIterableAndStarStarArgs2] +it = [(1, 'x')] +kw = {'x': 'y'} +d = dict(it, **kw) # E: Keyword argument only valid with "str" key type in call to "dict" +d() # E: "Dict[int, str]" not callable +[builtins fixtures/dict.pyi] + +[case testUserDefinedClassNamedDict] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') +class dict(Generic[T, S]): + def __init__(self, x: T, **kwargs: T) -> None: pass +dict(1, y=1) +[builtins fixtures/dict.pyi] + +[case testSpecialSignatureForSubclassOfDict] +from typing import TypeVar, Dict, Generic +T = TypeVar('T') +S = TypeVar('S') +class D1(dict): pass # Implicit base class Dict[Any, Any] +D1([(1, 2)], x=1) +class D2(Dict[T, S], Generic[T, S]): pass +da = D2([('x', 2)], x=1) +da() # E: "D2[str, int]" not callable +D2([(1, 2)], x=1) # E: Keyword argument only valid with "str" key type in call to "dict" +db = D2(x=1) +db() # E: "D2[str, int]" not callable +[builtins fixtures/dict.pyi] + +[case testSpecialSignatureForSubclassOfDict2] +from typing import TypeVar, Dict, Generic +T = TypeVar('T') +class D(Dict[str, T], Generic[T]): pass +D([('x', 1)], x=1) +[builtins fixtures/dict.pyi] + +[case testOverridingSpecialSignatureInSubclassOfDict] +from typing import TypeVar, Dict, Generic +T = TypeVar('T') +S = TypeVar('S') +class D(Dict[T, S], Generic[T, S]): + def __init__(self, x: S, y: T) -> None: pass +d = D(1, y='') +d() # E: "D[str, int]" not callable +[builtins fixtures/dict.pyi] + +[case testRevealType] +reveal_type(1) # N: Revealed type is "Literal[1]?" + +[case testRevealLocals] +x = 1 +y = 2 +z = x + y +reveal_locals() +[out] +main:4: note: Revealed local types are: +main:4: note: x: builtins.int +main:4: note: y: builtins.int +main:4: note: z: builtins.int + +[case testUndefinedRevealType] +reveal_type(x) +[out] +main:1: error: Name "x" is not defined +main:1: note: Revealed type is "Any" + +[case testUserDefinedRevealType] +def reveal_type(x: int) -> None: pass +reveal_type("foo") # E: Argument 1 to "reveal_type" has incompatible type "str"; expected "int" + +[case testTypingRevealType] +from typing import reveal_type +from typing import reveal_type as show_me_the_type + +reveal_type(1) # N: Revealed type is "Literal[1]?" +show_me_the_type(1) # N: Revealed type is "Literal[1]?" + +[case testTypingExtensionsRevealType] +from typing_extensions import reveal_type +from typing_extensions import reveal_type as show_me_the_type + +reveal_type(1) # N: Revealed type is "Literal[1]?" +show_me_the_type(1) # N: Revealed type is "Literal[1]?" + +[builtins fixtures/tuple.pyi] + +[case testRevealTypeVar] +reveal_type = 1 +1 + "foo" # E: Unsupported operand types for + ("int" and "str") + +[case testRevealForward] +def f() -> None: + reveal_type(x) +x = 1 + int() +[out] +main:2: note: Revealed type is "builtins.int" + +[case testRevealUncheckedFunction] +def f(): + x = 42 + reveal_type(x) +[out] +main:3: note: Revealed type is "Any" +main:3: note: 'reveal_type' always outputs 'Any' in unchecked functions + +[case testRevealCheckUntypedDefs] +# flags: --check-untyped-defs +def f(): + x = 42 + reveal_type(x) +[out] +main:4: note: Revealed type is "builtins.int" + +[case testRevealTypedDef] +def f() -> None: + x = 42 + reveal_type(x) +[out] +main:3: note: Revealed type is "builtins.int" + +[case testLambdaTypedContext] +def f() -> None: + lambda: 'a'.missing() # E: "str" has no attribute "missing" + +[case testLambdaUnypedContext] +def f(): + lambda: 'a'.missing() + +[case testLambdaCheckUnypedContext] +# flags: --check-untyped-defs +def f(): + lambda: 'a'.missing() # E: "str" has no attribute "missing" + +[case testEqNone] +None == None +[builtins fixtures/ops.pyi] + +[case testLtNone] +None < None # E: Unsupported left operand type for < ("None") +[builtins fixtures/ops.pyi] + +[case testDictWithStarExpr] + +b = {'z': 26, *a} # E: invalid syntax +[builtins fixtures/dict.pyi] + +[case testDictWithStarStarExpr] + +from typing import Dict, Iterable + +class Thing: + def keys(self) -> Iterable[str]: + ... + def __getitem__(self, key: str) -> int: + ... + +a = {'a': 1} +b = {'z': 26, **a} +c = {**b} +d = {**a, **b, 'c': 3} +e = {1: 'a', **a} # E: Cannot infer type argument 1 of \ + # N: Try assigning the literal to a variable annotated as dict[, ] +f = {**b} # type: Dict[int, int] # E: Unpacked dict entry 0 has incompatible type "Dict[str, int]"; expected "SupportsKeysAndGetItem[int, int]" +g = {**Thing()} +h = {**a, **Thing()} +i = {**Thing()} # type: Dict[int, int] # E: Unpacked dict entry 0 has incompatible type "Thing"; expected "SupportsKeysAndGetItem[int, int]" \ + # N: Following member(s) of "Thing" have conflicts: \ + # N: Expected: \ + # N: def __getitem__(self, int, /) -> int \ + # N: Got: \ + # N: def __getitem__(self, str, /) -> int \ + # N: Expected: \ + # N: def keys(self) -> Iterable[int] \ + # N: Got: \ + # N: def keys(self) -> Iterable[str] +j = {1: 'a', **Thing()} # E: Cannot infer type argument 1 of \ + # N: Try assigning the literal to a variable annotated as dict[, ] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testDictIncompatibleTypeErrorMessage] +from typing import Dict, Callable + +def things() -> int: + return 42 + +stuff: Dict[int, Callable[[], str]] = { + 1: things # E: Dict entry 0 has incompatible type "int": "Callable[[], int]"; expected "int": "Callable[[], str]" +} +[builtins fixtures/dict.pyi] + +[case testDictIncompatibleKeyVerbosity] +from typing import Dict +import mod + +class A: ... +class B(A): ... + +d: Dict[A, B] = {A(): mod.B()} # E: Dict entry 0 has incompatible type "A": "mod.B"; expected "A": "__main__.B" + +[file mod.py] +class B: ... + +[builtins fixtures/dict.pyi] + +[case testDictIncompatibleValueVerbosity] +from typing import Dict +import mod + +class A: ... +class B(A): ... + +d: Dict[B, A] = {mod.B(): A()} # E: Dict entry 0 has incompatible type "mod.B": "A"; expected "__main__.B": "A" + +[file mod.py] +class B: ... + +[builtins fixtures/dict.pyi] + +[case testTypeAnnotationNeededMultipleAssignment] +x, y = [], [] # E: Need type annotation for "x" (hint: "x: List[] = ...") \ + # E: Need type annotation for "y" (hint: "y: List[] = ...") +[builtins fixtures/list.pyi] + +[case testStrictEqualityEq] +# flags: --strict-equality +class A: ... +class B: ... +class C(B): ... + +A() == B() # E: Non-overlapping equality check (left operand type: "A", right operand type: "B") +B() == C() +C() == B() +A() != B() # E: Non-overlapping equality check (left operand type: "A", right operand type: "B") +B() != C() +C() != B() +[builtins fixtures/bool.pyi] + +[case testStrictEqualityIs] +# flags: --strict-equality +class A: ... +class B: ... +class C(B): ... + +A() is B() # E: Non-overlapping identity check (left operand type: "A", right operand type: "B") +B() is C() +C() is B() +A() is not B() # E: Non-overlapping identity check (left operand type: "A", right operand type: "B") +B() is not C() +C() is not B() +[builtins fixtures/bool.pyi] + +[case testStrictEqualityContains] +# flags: --strict-equality +class A: ... +class B: ... +class C(B): ... + +A() in [B()] # E: Non-overlapping container check (element type: "A", container item type: "B") +B() in [C()] +C() in [B()] +A() not in [B()] # E: Non-overlapping container check (element type: "A", container item type: "B") +B() not in [C()] +C() not in [B()] +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityUnions] +# flags: --strict-equality +from typing import Container, Union + +class A: ... +class B: ... + +a: Union[int, str] +b: Union[A, B] + +a == int() +b == int() # E: Non-overlapping equality check (left operand type: "Union[A, B]", right operand type: "int") + +a is int() +b is int() # E: Non-overlapping identity check (left operand type: "Union[A, B]", right operand type: "int") + +ca: Union[Container[int], Container[str]] +cb: Union[Container[A], Container[B]] + +42 in ca +42 in cb # E: Non-overlapping container check (element type: "int", container item type: "Union[A, B]") +[builtins fixtures/bool.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityBytesSpecial] +# flags: --strict-equality +b'abc' in b'abcde' +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStrictEqualityBytesSpecialUnion] +# flags: --strict-equality +from typing import Union +x: Union[bytes, str] + +b'abc' in x +x in b'abc' +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStrictEqualityByteArraySpecial] +# flags: --strict-equality +b'abc' in bytearray(b'abcde') +bytearray(b'abc') in b'abcde' # OK on Python 3 +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStrictEqualityNoPromotePy3] +# flags: --strict-equality +'a' == b'a' # E: Non-overlapping equality check (left operand type: "Literal['a']", right operand type: "Literal[b'a']") +b'a' in 'abc' # E: Non-overlapping container check (element type: "bytes", container item type: "str") + +x: str +y: bytes +x != y # E: Non-overlapping equality check (left operand type: "str", right operand type: "bytes") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityOkPromote] +# flags: --strict-equality +from typing import Container +c: Container[int] + +1 == 1.0 # OK +1.0 in c # OK +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityAny] +# flags: --strict-equality +from typing import Any, Container + +x: Any +c: Container[str] +x in c +x == 42 +x is 42 +[builtins fixtures/bool.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityStrictOptional] +# flags: --strict-equality + +x: str +if x is not None: # OK even with strict-optional + pass +[builtins fixtures/bool.pyi] + +[case testStrictEqualityNoStrictOptional] +# flags: --strict-equality --no-strict-optional + +x: str +if x is not None: # OK without strict-optional + pass +[builtins fixtures/bool.pyi] + +[case testStrictEqualityEqNoOptionalOverlap] +# flags: --strict-equality +from typing import Optional + +x: Optional[str] +y: Optional[int] +if x == y: # E: Non-overlapping equality check (left operand type: "Optional[str]", right operand type: "Optional[int]") + ... +[builtins fixtures/bool.pyi] + +[case testCustomEqCheckStrictEquality] +# flags: --strict-equality +class A: + def __eq__(self, other: A) -> bool: # type: ignore + ... +class B: + def __eq__(self, other: B) -> bool: # type: ignore + ... + +# Don't report non-overlapping check if there is already and error. +A() == B() # E: Unsupported operand types for == ("A" and "B") +[builtins fixtures/bool.pyi] + +[case testStrictEqualitySequenceAndCustomEq] +# flags: --strict-equality +from typing import Tuple + +class C: pass +class D: + def __eq__(self, other): return True + +a = [C()] +b = [D()] +a == b +b == a +t1: Tuple[C, ...] +t2: Tuple[D, ...] +t1 == t2 +t2 == t1 +[builtins fixtures/bool.pyi] + +[case testCustomEqCheckStrictEqualityOKInstance] +# flags: --strict-equality +class A: + def __eq__(self, other: object) -> bool: + ... +class B: + def __eq__(self, other: object) -> bool: + ... + +A() == int() # OK +int() != B() # OK +[builtins fixtures/bool.pyi] + +[case testCustomEqCheckStrictEqualityOKUnion] +# flags: --strict-equality +from typing import Union +class A: + def __eq__(self, other: object) -> bool: + ... + +x: Union[A, str] +x == int() +[builtins fixtures/bool.pyi] + +[case testCustomEqCheckStrictEqualityTuple] +# flags: --strict-equality +from typing import NamedTuple + +class Base(NamedTuple): + attr: int + +class Custom(Base): + def __eq__(self, other: object) -> bool: ... + +Base(int()) == int() # E: Non-overlapping equality check (left operand type: "Base", right operand type: "int") +Base(int()) == tuple() +Custom(int()) == int() +[builtins fixtures/bool.pyi] + +[case testCustomEqCheckStrictEqualityMeta] +# flags: --strict-equality +class CustomMeta(type): + def __eq__(self, other: object) -> bool: ... + +class Normal: ... +class Custom(metaclass=CustomMeta): ... + +Normal == int() # E: Non-overlapping equality check (left operand type: "Type[Normal]", right operand type: "int") +Normal == Normal +Custom == int() +[builtins fixtures/bool.pyi] + +[case testCustomContainsCheckStrictEquality] +# flags: --strict-equality +class A: + def __contains__(self, other: A) -> bool: + ... + +# Don't report non-overlapping check if there is already and error. +42 in A() # E: Unsupported operand types for in ("int" and "A") +[builtins fixtures/bool.pyi] + +[case testStrictEqualityTypeVsCallable] +# flags: --strict-equality +from typing import Type, List +class C: ... +class D(C): ... +class Bad: ... + +subclasses: List[Type[C]] +object in subclasses +D in subclasses +Bad in subclasses # E: Non-overlapping container check (element type: "Type[Bad]", container item type: "Type[C]") +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityMetaclass] +# flags: --strict-equality +from typing import List, Type, Any + +class Meta(type): ... +class OtherMeta(type): ... + +class A(metaclass=Meta): ... +class B(metaclass=Meta): ... +class C(metaclass=OtherMeta): ... + +o: Type[object] +a: Type[Any] +aa: type +exp: List[Meta] + +A in exp +B in exp +C in exp # E: Non-overlapping container check (element type: "Type[C]", container item type: "Meta") + +o in exp +a in exp +aa in exp + +a in [A, B] +aa in [A, B] + +class AA: ... +class BB: ... +a in [AA, BB] +aa in [AA, BB] +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + +[case testEmptyListOverlap] +# mypy: strict-equality +from typing import List + +x: List[int] +x == [] +[builtins fixtures/isinstancelist.pyi] + +[case testCustomEqDecoratedStrictEquality] +# flags: --strict-equality +from typing import TypeVar, Callable, Any + +F = TypeVar('F', bound=Callable[..., Any]) + +def deco(f: F) -> F: ... + +class Custom: + @deco + def __eq__(self, other: object) -> bool: ... + +Custom() == int() +[builtins fixtures/bool.pyi] + +[case testCustomEqVarStrictEquality] +# flags: --strict-equality + +class Custom: + def compare(self, other: object) -> bool: ... + __eq__ = compare + +Custom() == int() +[builtins fixtures/bool.pyi] + +[case testStrictEqualityDisabledWithTypeVarRestrictions] +# flags: --strict-equality +from typing import TypeVar + +T = TypeVar('T', str, int) + +def f(x: T) -> T: + if x == int(): # OK + ... + return x +[builtins fixtures/bool.pyi] + +[case testStrictEqualityWithALiteral] +# flags: --strict-equality +from typing_extensions import Literal, Final + +def returns_a_or_b() -> Literal['a', 'b']: + ... +def returns_1_or_2() -> Literal[1, 2]: + ... +THREE: Final = 3 + +if returns_a_or_b() == 'c': # E: Non-overlapping equality check (left operand type: "Literal['a', 'b']", right operand type: "Literal['c']") + ... +if returns_1_or_2() is THREE: # E: Non-overlapping identity check (left operand type: "Literal[1, 2]", right operand type: "Literal[3]") + ... +[builtins fixtures/bool.pyi] + +[case testStrictEqualityWithALiteralNewType] +# flags: --strict-equality +from typing import NewType + +UserId = NewType('UserId', int) +FileId = NewType('FileId', str) + +u: UserId +f: FileId + +if u == 0: # OK + ... +if f == 0: # E: Non-overlapping equality check (left operand type: "FileId", right operand type: "Literal[0]") + ... +[builtins fixtures/bool.pyi] + +[case testStrictEqualityWithFixedLengthTupleInCheck] +# flags: --strict-equality +if 1 in ('x', 'y'): # E: Non-overlapping container check (element type: "int", container item type: "str") + pass +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverlappingAnyTypeWithoutStrictOptional] +# flags: --no-strict-optional --strict-equality +from typing import Any, Optional + +x: Optional[Any] + +if x in (1, 2): + pass +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + + +[case testOverlappingClassCallables] +# flags: --strict-equality +from typing import Any, Callable, Type + +x: Type[int] +y: Callable[[], Any] +x == y +y == x +int == y +y == int +[builtins fixtures/bool.pyi] + +[case testStrictEqualityAndEnumWithCustomEq] +# flags: --strict-equality +from enum import Enum + +class E1(Enum): + X = 0 + Y = 1 + +class E2(Enum): + X = 0 + Y = 1 + + def __eq__(self, other: object) -> bool: + return bool() + +E1.X == E1.Y # E: Non-overlapping equality check (left operand type: "Literal[E1.X]", right operand type: "Literal[E1.Y]") +E2.X == E2.Y +[builtins fixtures/bool.pyi] + +[case testStrictEqualityWithBytesContains] +# flags: --strict-equality +data = b"xy" +b"x" in data +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityWithDifferentMapTypes] +# flags: --strict-equality +from typing import Mapping + +class A(Mapping[int, str]): ... +class B(Mapping[int, str]): ... + +a: A +b: B +assert a == b +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityWithRecursiveMapTypes] +# flags: --strict-equality +from typing import Dict + +R = Dict[str, R] + +a: R +b: R +assert a == b + +R2 = Dict[int, R2] +c: R2 +assert a == c # E: Non-overlapping equality check (left operand type: "Dict[str, R]", right operand type: "Dict[int, R2]") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testStrictEqualityWithRecursiveListTypes] +# flags: --strict-equality +from typing import List, Union + +R = List[Union[str, R]] + +a: R +b: R +assert a == b + +R2 = List[Union[int, R2]] +c: R2 +assert a == c +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + +[case testUnimportedHintAny] +def f(x: Any) -> None: # E: Name "Any" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Any") + pass + + +[case testUnimportedHintAnyLower] +def f(x: any) -> None: # E: Name "any" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Any") + pass + + +[case testUnimportedHintOptional] +def f(x: Optional[str]) -> None: # E: Name "Optional" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Optional") + pass + +[case testAssertionLazilyWithIsNone] +from typing import Optional, List +li: Optional[List] = [] +assert li is None, li[0] +[builtins fixtures/list.pyi] + + +[case testAssertionLazilyWithIsInstance] +from typing import Optional, List +li: Optional[List] = [] +assert not isinstance(li,list), li[0] +[builtins fixtures/isinstancelist.pyi] + +[case testAssertCurrentFrameIsNotUnreachable] +def f() -> int: # E: Missing return statement + x: int + assert isinstance(x, int), '...' +[builtins fixtures/isinstance.pyi] + +[case testTypeVarAsValue] +from typing import TypeVar +T = TypeVar("T") +x: int +x + T # E: Unsupported left operand type for + ("int") +T() # E: "TypeVar" not callable +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test new file mode 100644 index 000000000000..534967b1edbf --- /dev/null +++ b/test-data/unit/check-fastparse.test @@ -0,0 +1,326 @@ +[case testFastParseSyntaxError] + +1 + # E: invalid syntax + +[case testFastParseTypeCommentSyntaxError] + +x = None # type: a : b # E: Syntax error in type comment "a : b" + +[case testFastParseInvalidTypeComment] + +x = None # type: a + b # E: Invalid type comment or annotation + +-- Function type comments are attributed to the function def line. +-- This happens in both parsers. +[case testFastParseFunctionAnnotationSyntaxError] + +def f(): # E: Syntax error in type comment "None -> None" # N: Suggestion: wrap argument types in parentheses + # type: None -> None + pass + +[case testFastParseFunctionAnnotationSyntaxErrorSpaces] + +def f(): # E: Syntax error in type comment "None -> None" # N: Suggestion: wrap argument types in parentheses + # type: None -> None + pass + +[case testFastParseInvalidFunctionAnnotation] + +def f(x): # E: Invalid type comment or annotation + # type: (a + b) -> None + pass + +[case testFastParseInvalidTypes3] +# All of these should not crash +from typing import Callable, Tuple, Iterable + +x: Tuple[int, str].x # E: Invalid type comment or annotation +a: Iterable[x].x # E: Invalid type comment or annotation +b: Tuple[x][x] # E: Invalid type comment or annotation +c: Iterable[x][x] # E: Invalid type comment or annotation +d: Callable[..., int][x] # E: Invalid type comment or annotation +e: Callable[..., int].x # E: Invalid type comment or annotation + +f = None # type: Tuple[int, str].x # E: Invalid type comment or annotation +g = None # type: Iterable[x].x # E: Invalid type comment or annotation +h = None # type: Tuple[x][x] # E: Invalid type comment or annotation +i = None # type: Iterable[x][x] # E: Invalid type comment or annotation +j = None # type: Callable[..., int][x] # E: Invalid type comment or annotation +k = None # type: Callable[..., int].x # E: Invalid type comment or annotation + +def f1(x: Tuple[int, str].x) -> None: pass # E: Invalid type comment or annotation +def f2(x: Iterable[x].x) -> None: pass # E: Invalid type comment or annotation +def f3(x: Tuple[x][x]) -> None: pass # E: Invalid type comment or annotation +def f4(x: Iterable[x][x]) -> None: pass # E: Invalid type comment or annotation +def f5(x: Callable[..., int][x]) -> None: pass # E: Invalid type comment or annotation +def f6(x: Callable[..., int].x) -> None: pass # E: Invalid type comment or annotation + +[case testFastParseTypeWithIgnore] +def f(x, # type: x # type: ignore + ): + # type: (...) -> None + pass + +[case testFastParseVariableTypeWithIgnore] + +x = 1 # type: str # type: ignore + +[case testFastParseVariableTypeWithIgnoreNoSpace] + +x = 1 # type: str #type:ignore + +[case testFastParseVariableTypeWithIgnoreAndComment] + +x = 1 # type: str # type: ignore # comment + +[case testFastParseTypeWithIgnoreWithStmt] +with open('test', 'r') as f: # type: int # type: ignore + pass + +[case testFastParseTypeWithIgnoreForStmt] +for i in (1, 2, 3, 100): # type: str # type: ignore + pass +[builtins fixtures/tuple.pyi] + +[case testFastParseVariableCommentThenIgnore] +a="test" # type: int #comment # type: ignore # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testFastParseProperty] + +class C: + @property + def x(self) -> str: pass + @x.setter + def x(self, value: str) -> None: pass +[builtins fixtures/property.pyi] + +[case testFastParseConditionalProperty] + +class C: + if bool(): + @property + def x(self) -> str: pass + @x.setter + def x(self, value: str) -> None: pass +[builtins fixtures/property.pyi] + +[case testFastParsePerArgumentAnnotations] +# flags: --implicit-optional + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass +class F: pass +def f(a, # type: A + b = None, # type: B + *args, # type: C + d = None, # type: D + e, # type: E + **kwargs # type: F + ): + reveal_type(a) # N: Revealed type is "__main__.A" + reveal_type(b) # N: Revealed type is "Union[__main__.B, None]" + reveal_type(args) # N: Revealed type is "builtins.tuple[__main__.C, ...]" + reveal_type(d) # N: Revealed type is "Union[__main__.D, None]" + reveal_type(e) # N: Revealed type is "__main__.E" + reveal_type(kwargs) # N: Revealed type is "builtins.dict[builtins.str, __main__.F]" +[builtins fixtures/dict.pyi] +[out] + +[case testFastParsePerArgumentAnnotationsWithReturn] +# flags: --implicit-optional + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass +class F: pass +def f(a, # type: A + b = None, # type: B + *args, # type: C + d = None, # type: D + e, # type: E + **kwargs # type: F + ): + # type: (...) -> int + reveal_type(a) # N: Revealed type is "__main__.A" + reveal_type(b) # N: Revealed type is "Union[__main__.B, None]" + reveal_type(args) # N: Revealed type is "builtins.tuple[__main__.C, ...]" + reveal_type(d) # N: Revealed type is "Union[__main__.D, None]" + reveal_type(e) # N: Revealed type is "__main__.E" + reveal_type(kwargs) # N: Revealed type is "builtins.dict[builtins.str, __main__.F]" + return "not an int" # E: Incompatible return value type (got "str", expected "int") +[builtins fixtures/dict.pyi] +[out] + +[case testFastParsePerArgumentAnnotationsWithAnnotatedBareStar] + +def f(*, # type: int # E: bare * has associated type comment + x # type: str + ): + # type: (...) -> int + pass +[builtins fixtures/dict.pyi] +[out] + +[case testFastParsePerArgumentAnnotationsWithReturnAndBareStar] + +def f(*, + x # type: str + ): + # type: (...) -> int + reveal_type(x) # N: Revealed type is "builtins.str" + return "not an int" # E: Incompatible return value type (got "str", expected "int") +[builtins fixtures/dict.pyi] +[out] + +[case testFasterParseTooManyArgumentsAnnotation] +def f(): # E: Type signature has too many arguments + # type: (int) -> None + pass + +f() +f(1) # E: Too many arguments for "f" + +[case testFasterParseTooFewArgumentsAnnotation] +def f(x, y): # E: Type signature has too few arguments + # type: (int) -> None + x() + y() + +f(1, 2) +f(1) # E: Missing positional argument "y" in call to "f" + +[case testFasterParseTypeErrorCustom] + +from typing import TypeVar, Generic +T = TypeVar('T') +class Foo(Generic[T]): + pass + +def f(a: Foo(int)) -> int: + pass +[out] +main:7: error: Invalid type comment or annotation +main:7: note: Suggestion: use Foo[...] instead of Foo(...) + +[case testFastParseMatMul] + +from typing import Any +x = None # type: Any +x @ 1 +x @= 1 + +[case testFastParserShowsMultipleErrors] +def f(x): # E: Type signature has too few arguments + # type: () -> None + pass +def g(): # E: Type signature has too many arguments + # type: (int) -> None + pass + +[case testFastParseMalformedAssert] + +assert 1, 2 +assert (1, 2) # E: Assertion is always true, perhaps remove parentheses? +assert (1, 2), 3 # E: Assertion is always true, perhaps remove parentheses? +assert (1,) # E: Assertion is always true, perhaps remove parentheses? +assert () +[builtins fixtures/tuple.pyi] + +[case testFastParseAssertMessage] + +assert 1 +assert 1, 2 +assert 1, 1+2 +assert 1, 1+'test' # E: Unsupported operand types for + ("int" and "str") +assert 1, f() # E: Name "f" is not defined + +[case testFastParserConsistentFunctionTypes] + +def f(x, y, z): + # type: (int, int, int) -> int + pass + +def f(x, # type: int # E: Function has duplicate type signatures + y, # type: int + z # type: int + ): + # type: (int, int, int) -> int + pass + +def f(x, # type: int + y, # type: int + z # type: int + ): + # type: (...) -> int + pass + +def f(x, y, z): + # type: (int, int, int) -> int + pass + +def f(x) -> int: # E: Function has duplicate type signatures + # type: (int) -> int + pass + +def f(x: int, y: int, z: int): + # type: (...) -> int + pass + +def f(x: int): # E: Function has duplicate type signatures + # type: (int) -> int + pass + +[case testFastParserDuplicateNames] + +def f(x, y, z): + pass + +def g(x, y, x): # E: Duplicate argument "x" in function definition + pass + +def h(x, y, *x): # E: Duplicate argument "x" in function definition + pass + +def i(x, y, *z, **z): # E: Duplicate argument "z" in function definition + pass + +def j(x: int, y: int, *, x: int = 3): # E: Duplicate argument "x" in function definition + pass + +def k(*, y, z, y): # E: Duplicate argument "y" in function definition + pass + +lambda x, y, x: ... # E: Duplicate argument "x" in function definition + +[case testNoCrashOnImportFromStar] +from pack import * +[file pack/__init__.py] +from . import * + +[case testNoCrashOnImportFromStarNested] + +import blamodule +[file blamodule/__init__.py] +from . import command +from . import backends + +[file blamodule/backends/__init__.py] +from .Bla import Bla +Bla().method() + +[file blamodule/backends/Bla.py] +from .. import * + +class Bla: + def method(self) -> str: + return command.call() + +[file blamodule/command.py] +def call() -> str: pass + +[builtins fixtures/module.pyi] diff --git a/test-data/unit/check-final.test b/test-data/unit/check-final.test new file mode 100644 index 000000000000..763183159e94 --- /dev/null +++ b/test-data/unit/check-final.test @@ -0,0 +1,1220 @@ +-- Test cases for final qualifier +-- + +-- Definitions + +[case testFinalDefiningModuleVar] +from typing import Final + +x: Final = int() +y: Final[float] = int() +z: Final[int] = int() +bad: Final[str] = int() # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.float" +reveal_type(z) # N: Revealed type is "builtins.int" +[out] + +[case testFinalDefiningInstanceVar] +from typing import Final + +class C: + x: Final = int() + y: Final[float] = int() + z: Final[int] = int() + bad: Final[str] = int() # E: Incompatible types in assignment (expression has type "int", variable has type "str") +class D(C): pass + +reveal_type(D.x) # N: Revealed type is "builtins.int" +reveal_type(D.y) # N: Revealed type is "builtins.float" +reveal_type(D.z) # N: Revealed type is "builtins.int" +reveal_type(D().x) # N: Revealed type is "builtins.int" +reveal_type(D().y) # N: Revealed type is "builtins.float" +reveal_type(D().z) # N: Revealed type is "builtins.int" +[out] + +[case testFinalDefiningInstanceVarImplicit] +from typing import Final, Tuple, Any + +class C: + def __init__(self, x: Tuple[int, Any]) -> None: + self.x: Final = x + self.y: Final[float] = 1 +reveal_type(C((1, 2)).x) # N: Revealed type is "Tuple[builtins.int, Any]" +reveal_type(C((1, 2)).y) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalBadDefinitionTooManyArgs] +from typing import Final + +x: Final[int, str] # E: Final name must be initialized with a value \ + # E: Final[...] takes at most one type argument +reveal_type(x) # N: Revealed type is "builtins.int" + +class C: + def __init__(self) -> None: + self.x: Final[float, float] = 1 # E: Final[...] takes at most one type argument +reveal_type(C().x) # N: Revealed type is "builtins.float" +[out] + +[case testFinalInvalidDefinitions] + +# Errors are shown in a different order with the new analyzer. +from typing import Final, Any + +x = y = 1 # type: Final[float] # E: Invalid final declaration +z: Any +z[0]: Final[int] # E: Invalid final declaration \ + # E: Unexpected type declaration +[out] + +[case testFinalDefiningInstanceVarStubs] +# Allow skipping r.h.s. +import mod +[file mod.pyi] +from typing import Final + +x: Final # E: Type in Final[...] can only be omitted if there is an initializer +y: Final[int] +class C: + x: Final # E: Type in Final[...] can only be omitted if there is an initializer + y: Final[int] + def __init__(self) -> None: + self.z: Final # E: Type in Final[...] can only be omitted if there is an initializer + +reveal_type(x) # N: Revealed type is "Any" +reveal_type(C.x) # N: Revealed type is "Any" +v: C +reveal_type(v.z) # N: Revealed type is "Any" +[out] + +[case testFinalDefiningFunc] +from typing import final + +@final # E: @final cannot be used with non-method functions +def f(x: int) -> None: ... +[out] + +[case testFinalDefiningFuncOverloaded] +from typing import final, overload + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +@final # E: @final cannot be used with non-method functions +def f(x): + pass +[out] + +[case testFinalDefiningMeth] +from typing import final + +class C: + @final + def f(self, x: int) -> None: ... +reveal_type(C().f) # N: Revealed type is "def (x: builtins.int)" +[out] + +[case testFinalDefiningMethOverloaded] +from typing import final, overload + +class C: + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... + @final + def f(self, x): + pass + + @overload + def bad(self, x: int) -> int: ... + @final # E: @final should be applied only to overload implementation + @overload + def bad(self, x: str) -> str: ... + def bad(self, x): + pass + +reveal_type(C().f) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str)" +[out] + +[case testFinalDefiningMethOverloadedStubs] +from mod import C + +reveal_type(C().f) +[file mod.pyi] +from typing import final, overload + +class C: + @final + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... + + @overload + def bad(self, x: int) -> int: ... + @final # Error! + @overload + def bad(self, x: str) -> str: ... +[out] +tmp/mod.pyi:12: error: In a stub file @final must be applied only to the first overload +main:3: note: Revealed type is "Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str)" + +[case testFinalDefiningProperty] +from typing import final + +class C: + @final + @property + def f(self) -> int: pass + @property + @final + def g(self) -> int: pass +reveal_type(C().f) # N: Revealed type is "builtins.int" +reveal_type(C().g) # N: Revealed type is "builtins.int" +[builtins fixtures/property.pyi] +[out] + +[case testFinalDefiningOuterOnly] +from typing import Final, Callable, Tuple, Any +x: Tuple[Final] # E: Final can be only used as an outermost qualifier in a variable annotation +y: Callable[[], Tuple[Final[int]]] # E: Final can be only used as an outermost qualifier in a variable annotation +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalDefiningNotInMethod] +from typing import Final + +def f(x: Final[int]) -> int: ... # E: Final can be only used as an outermost qualifier in a variable annotation +def g(x: int) -> Final[int]: ... # E: Final can be only used as an outermost qualifier in a variable annotation +[out] + +[case testFinalDefiningNotInMethodExtensions] +from typing_extensions import Final + +def f(x: Final[int]) -> int: ... # E: Final can be only used as an outermost qualifier in a variable annotation +def g(x: int) -> Final[int]: ... # E: Final can be only used as an outermost qualifier in a variable annotation +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalDefiningNoRhs] +from typing import Final +x: Final # E: Type in Final[...] can only be omitted if there is an initializer +y: Final[int] # E: Final name must be initialized with a value +class C: + x: Final # E: Type in Final[...] can only be omitted if there is an initializer + y: Final[int] # E: Final name must be initialized with a value + def __init__(self) -> None: + self.z: Final # E: Type in Final[...] can only be omitted if there is an initializer +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "builtins.int" +reveal_type(C().x) # N: Revealed type is "Any" +reveal_type(C().y) # N: Revealed type is "builtins.int" +reveal_type(C().z) # N: Revealed type is "Any" +[out] + +[case testFinalDefiningNoRhsSubclass] +from typing import Final + +class A: + x: Final[int] # E: Final name must be initialized with a value + +class B(A): + x = 1 # E: Cannot assign to final name "x" + def __init__(self) -> None: + self.x = 1 # E: Cannot assign to final attribute "x" +[out] + +[case testFinalDefiningNoTypevarsExplicit] +from typing import Final, TypeVar, Generic, Tuple, Any + +T = TypeVar('T') +d: Any + +class C(Generic[T]): + x: Final[Tuple[T, T]] = d # E: Final name declared in class body cannot depend on type variables +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalDefiningTypevarsImplicit] +from typing import Final, TypeVar, Generic, Tuple, Any + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, x: Tuple[T, T]) -> None: + self.x: Final = x + self.y: Final = 1 + +reveal_type(C((1, 2)).x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +C.x # E: Cannot access final instance attribute "x" on class object \ + # E: Access to generic instance variables via class is ambiguous +C.y # E: Cannot access final instance attribute "y" on class object +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalDefiningNotInOtherMethod] +from typing import Final, Any, Tuple + +class C: + def meth(self, x: Tuple[int, Any]) -> None: + self.x: Final = x # E: Can only declare a final attribute in class body or __init__ + self.y: Final[float] = 1 # E: Can only declare a final attribute in class body or __init__ +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalDefiningOnlyOnSelf] +from typing import Final, Any, Tuple + +class U: + x: Any + y: Any +class C: + def __init__(self, x: Tuple[int, Any]) -> None: + slf = U() + slf.x: Final = x # E: Final can be only applied to a name or an attribute on self + slf.y: Final[float] = 1 # E: Type cannot be declared in assignment to non-self attribute \ + # E: Final can be only applied to a name or an attribute on self +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalNotInProtocol] +from typing import Final, final, Protocol, overload + +class P(Protocol): + x: Final[float] = 1 # E: Protocol member cannot be final + @final # E: Protocol member cannot be final + def meth(self, x) -> int: + pass + @overload + def other(self, x: int) -> int: ... + @overload + def other(self, x: str) -> str: ... + @final # E: Protocol member cannot be final + def other(self, x): + pass +[out] + +[case testFinalInProtocol] +from typing import Final, Protocol, final + +class P(Protocol): + var1 : Final[int] = 0 # E: Protocol member cannot be final + + @final # E: Protocol member cannot be final + def meth1(self) -> None: + var2: Final = 0 + + def meth2(self) -> None: + var3: Final = 0 + + def meth3(self) -> None: + class Inner: + var3: Final = 0 # OK + + @final + def inner(self) -> None: ... + + class Inner: + var3: Final = 0 # OK + + @final + def inner(self) -> None: ... + +[out] + +[case testFinalWithClassVarInProtocol] +from typing import Protocol, Final, final, ClassVar + +class P(Protocol): + var1 : Final[ClassVar[int]] = 0 # E: Variable should not be annotated with both ClassVar and Final + var2: ClassVar[int] = 1 + + @final # E: Protocol member cannot be final + def meth1(self) -> None: + ... + + def meth2(self) -> None: + var3: Final[ClassVar[int]] = 0 # E: Variable should not be annotated with both ClassVar and Final # E: ClassVar can only be used for assignments in class body + +[out] + +[case testFinalNotInLoops] +from typing import Final + +for i in [1, 2, 3]: + x: Final = i # E: Cannot use Final inside a loop + +while True: + y: Final = True # E: Cannot use Final inside a loop +[builtins fixtures/list.pyi] +[out] + +[case testFinalDelayedDefinition] +from typing import Final + +class C: + x: Final[int] # OK, defined in __init__ + bad: Final[int] # E: Final name must be initialized with a value + + def __init__(self, x: int) -> None: + self.x = x # OK, deferred definition + self.x = 2 # E: Cannot assign to final attribute "x" + + def meth(self) -> None: + self.x = 2 # E: Cannot assign to final attribute "x" + +c: C +c.x = 3 # E: Cannot assign to final attribute "x" +class D(C): + x = 4 # E: Cannot assign to final name "x" +d: D +d.x = 5 # E: Cannot assign to final attribute "x" +[out] + +[case testFinalDelayedDefinitionOtherMethod] +from typing import Final + +class C: + x: Final[int] # E: Final name must be initialized with a value + + def meth(self) -> None: + self.x = 2 # E: Cannot assign to final attribute "x" +[out] + +-- Reassignments + +[case testFinalReassignModuleVar] +# flags: --allow-redefinition +from typing import Final + +x: Final = 1 +x +x = 2 # E: Cannot assign to final name "x" +def f() -> int: + global x + x = 3 # No error here is okay since we reported an error above + return x + +x2: Final = 1 +x2 +def f2() -> None: + global x2 + x2 = 1 # E: Cannot assign to final name "x2" + +y = 1 +y +y: Final = 2 # E: Cannot redefine an existing name as final +y = 3 # E: Cannot assign to final name "y" + +z: Final = 1 +z: Final = 2 # E: Cannot redefine an existing name as final +z = 3 # E: Cannot assign to final name "z" + +[case testFinalReassignModuleVar2] +# flags: --allow-redefinition +from typing import Final + +x: Final = 1 +x +def f() -> int: + global x + x = 3 # E: Cannot assign to final name "x" + return x + +y = 1 +y +y = 2 +y +y: Final = 3 # E: Cannot redefine an existing name as final + +[case testFinalReassignModuleVar3] +# flags: --disallow-redefinition +# Error formatting is subtly different with new analyzer. +from typing import Final + +x: Final = 1 +x +x = 2 # E: Cannot assign to final name "x" +def f() -> int: + global x + x = 3 # E: Cannot assign to final name "x" + return x + +x2: Final = 1 +x2 +def f2() -> None: + global x2 + x2 = 1 # E: Cannot assign to final name "x2" + +y = 1 # E: Cannot assign to final name "y" +y +y: Final = 2 # E: Cannot redefine an existing name as final +y = 3 # E: Cannot assign to final name "y" + +z: Final = 1 +z: Final = 2 # E: Cannot redefine an existing name as final +z = 3 # E: Cannot assign to final name "z" + +[case testFinalReassignModuleReexport] + +# Error formatting is subtly different with the new analyzer. +from typing import Final + +from lib import X +from lib.mod import ID + +X = 1 # Error! +ID: Final = 1 # Two errors! +ID = 1 # Error! +[file lib/__init__.pyi] +from lib.const import X as X + +[file lib/mod.pyi] +from lib.const import * + +[file lib/const.pyi] +from typing import Final + +ID: Final # Error! +X: Final[int] +[out] +tmp/lib/const.pyi:3: error: Type in Final[...] can only be omitted if there is an initializer +main:8: error: Cannot assign to final name "X" +main:9: error: Cannot redefine an existing name as final +main:10: error: Cannot assign to final name "ID" + +[case testFinalReassignFuncScope] +from typing import Final + +def f() -> None: + nl: Final = 0 + x: Final = 1 + x = 1 # E: Cannot assign to final name "x" + + y: Final = 1 + y: Final = 2 # E: Cannot redefine an existing name as final + def nested() -> None: + nonlocal nl + nl = 1 # E: Cannot assign to final name "nl" +[out] + +[case testFinalReassignModuleVarExternal] +import mod +mod.x = 2 # E: Cannot assign to final name "x" +[file mod.pyi] +from typing import Final +x: Final[int] +[out] + +[case testFinalReassignInstanceVarClassBody] +from typing import Final + +class C: + x: Final = 1 + x = 2 # E: Cannot assign to final name "x" + + y = 1 # E: Cannot assign to final name "y" + y: Final = 2 # E: Cannot redefine an existing name as final +[out] + +[case testFinalReassignInstanceVarInit] +from typing import Final + +class C: + def __init__(self) -> None: + self.x: Final = 1 + self.y = 1 + self.y: Final = 2 # E: Cannot redefine an existing name as final + def meth(self) -> None: + self.x = 2 # E: Cannot assign to final attribute "x" +[out] + +[case testFinalReassignInstanceVarClassVsInit] + +from typing import Final + +class C: + y: Final = 1 + def __init__(self) -> None: + # Methods are processed after top-level in new analyzer. + self.x: Final = 1 # E: Cannot redefine an existing name as final + self.y = 2 # E: Cannot assign to final attribute "y" + x = 2 +[out] + +[case testFinalReassignInstanceVarMethod] +from typing import Final + +class C: + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 + def meth(self) -> None: + self.x = 2 # E: Cannot assign to final attribute "x" + self.y = 2 # E: Cannot assign to final attribute "y" + def other(self) -> None: + self.x = 2 # E: Cannot assign to final attribute "x" + self.y = 2 # E: Cannot assign to final attribute "y" + @classmethod + def cm(cls) -> None: + cls.x = 2 # E: Cannot assign to final attribute "x" + cls.y # E: Cannot access final instance attribute "y" on class object +[builtins fixtures/classmethod.pyi] +[out] + +[case testFinalReassignInstanceVarExternalClass] +from typing import Final + +class C: + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 + +class D(C): pass + +C.x = 2 # E: Cannot assign to final attribute "x" +D.x = 2 # E: Cannot assign to final attribute "x" +D.y = 2 # E: Cannot access final instance attribute "y" on class object \ + # E: Cannot assign to final attribute "y" +[out] + +[case testFinalReassignInstanceVarExternalInstance] +from typing import Final + +class C: + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 + +class D(C): pass + +C().x = 2 # E: Cannot assign to final attribute "x" +D().x = 2 # E: Cannot assign to final attribute "x" +D().y = 2 # E: Cannot assign to final attribute "y" +[out] + +[case testFinalWorksWithComplexTargets] +from typing import Final, Any + +y: Final[Any] = 1 +x = a, (b, y), c = 2, (2, 2), 2 # E: Cannot assign to final name "y" +t, *y, s = u = [2, 2, 2] # E: Cannot assign to final name "y" +[builtins fixtures/list.pyi] +[out] + +[case testFinalInplaceAssign] +from typing import Final + +class A: # no such things in fixtures + def __add__(self, other: A) -> A: ... +class B: + def __add__(self, other: B) -> B: ... + def __iadd__(self, other: B) -> B: ... + +a: Final = A() +b: Final = B() +class C: + a: Final = A() + b: Final = B() +class D(C): + pass + +a += A() # E: Cannot assign to final name "a" +b += B() # E: Cannot assign to final name "b" +D().a += A() # E: Cannot assign to final attribute "a" +D().b += B() # E: Cannot assign to final attribute "b" +[out] + +-- Overriding + +[case testFinalOverridingVarClassBody] +from typing import Final + +# We use properties in this tests and below because we want to check +# that any existing variable before final doesn't affect logic of +# subsequent overrides but writable attributes cannot be overridden by final. +class A: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + +class B(A): + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 +class C(B): + x: int = 2 # E: Cannot assign to final name "x" + y: int = 2 # E: Cannot assign to final name "y" + x = 3 # E: Cannot assign to final name "x" + y = 3 # E: Cannot assign to final name "y" +class D(C): + pass +D.x = 4 # E: Cannot assign to final attribute "x" +D.y = 4 # E: Cannot assign to final attribute "y" +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarClassBodyExplicit] +from typing import Final + +class A: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... +class B(A): + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 +class C(B): + x: Final = 2 # E: Cannot override final attribute "x" (previously declared in base class "B") + y: Final = 2 # E: Cannot override final attribute "y" (previously declared in base class "B") +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarInit] +from typing import Final + +class A: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... +class B(A): + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 +class C(B): + def __init__(self) -> None: + self.x = 2 # E: Cannot assign to final attribute "x" + self.y = 2 # E: Cannot assign to final attribute "y" + def meth(self) -> None: + self.x = 3 # E: Cannot assign to final attribute "x" + self.y = 3 # E: Cannot assign to final attribute "y" +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarInit2] +from typing import Final + +class A: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... +class B(A): + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 +class C(B): + def __init__(self) -> None: + self.x: Final = 2 # E: Cannot override final attribute "x" (previously declared in base class "B") + self.y: Final = 2 # E: Cannot override final attribute "y" (previously declared in base class "B") +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarOtherMethod] +from typing import Final + +class A: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... +class B(A): + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 +class C(B): + def meth(self) -> None: + self.x: int = 2 # E: Cannot assign to final attribute "x" + self.y: int = 2 # E: Cannot assign to final attribute "y" + + self.x = 3 # E: Cannot assign to final attribute "x" + self.y = 3 # E: Cannot assign to final attribute "y" +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarMultipleInheritanceClass] +from typing import Final, Any + +class A: + x: Final[Any] = 1 +class B: + @property + def x(self) -> int: ... +class C(A, B): ... +class D(B, A): ... # E: Cannot override final attribute "x" (previously declared in base class "A") +C.x = 3 # E: Cannot assign to final attribute "x" +C().x = 4 # E: Cannot assign to final attribute "x" +D().x = 4 # E: Cannot assign to final attribute "x" \ + # E: Property "x" defined in "B" is read-only +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarMultipleInheritanceInit] +from typing import Final, Any + +class A: + def __init__(self) -> None: + self.x: Final[Any] = 1 +class B: + @property + def x(self) -> int: ... +class C(A, B): ... +class D(B, A): ... # E: Cannot override final attribute "x" (previously declared in base class "A") +C.x = 3 # E: Cannot access final instance attribute "x" on class object \ + # E: Cannot assign to final attribute "x" +C().x = 4 # E: Cannot assign to final attribute "x" +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarMultipleInheritanceMixed] +from typing import Final + +class A: + x: Final = 1 +class B: + def __init__(self) -> None: + self.x = 2 +class C(A, B): ... # E: Cannot override writable attribute "x" with a final one +class D(B, A): ... # E: Cannot override final attribute "x" (previously declared in base class "A") +C.x = 3 # E: Cannot assign to final attribute "x" +D.x = 3 # E: Cannot assign to final attribute "x" +C().x = 4 # E: Cannot assign to final attribute "x" +D().x = 4 # E: Cannot assign to final attribute "x" +[out] + +[case testFinalOverridingVarWithMethod] +from typing import Final, Any + +class A: + x: Final[Any] = 1 + def __init__(self) -> None: + self.y: Final[Any] = 1 + +class B(A): + def x(self) -> None: pass # E: Cannot override final attribute "x" (previously declared in base class "A") + def y(self) -> None: pass # E: Cannot override final attribute "y" (previously declared in base class "A") + +class C(A): + @property # E: Cannot override final attribute "x" (previously declared in base class "A") + def x(self) -> None: pass + @property # E: Cannot override final attribute "y" (previously declared in base class "A") + def y(self) -> None: pass +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingVarWithMethodClass] +from typing import Final, Any + +class A: + x: Final[Any] = 1 + def __init__(self) -> None: + self.y: Final[Any] = 1 + +class B(A): + @classmethod # E: Cannot override final attribute "x" (previously declared in base class "A") + def x(self) -> None: pass + @classmethod # E: Cannot override final attribute "y" (previously declared in base class "A") + def y(self) -> None: pass + +[builtins fixtures/classmethod.pyi] +[out] + +[case testFinalOverridingMethodRegular] +from typing import final + +class B: + @final + def meth(self) -> None: ... +class C(B): + def meth(self) -> None: ... # E: Cannot override final attribute "meth" (previously declared in base class "B") +[out] + +[case testFinalOverridingMethodInitNew] +from typing import final + +class B: + @final + def __init__(self) -> None: ... + @final + def __new__(cls) -> B: ... +class C(B): + def __init__(self) -> None: ... # E: Cannot override final attribute "__init__" (previously declared in base class "B") + def __new__(cls) -> C: ... # E: Cannot override final attribute "__new__" (previously declared in base class "B") +[out] + +[case testFinalOverridingMethodWithVar] +from typing import final, Final, Any + +a: Any + +class A: + @final + def f(self) -> None: pass + @final + @property + def p(self) -> int: pass + +class B(A): + f = a # E: Cannot override final attribute "f" (previously declared in base class "A") + p = a # E: Cannot override final attribute "p" (previously declared in base class "A") +class C(A): + f: Any # E: Cannot override final attribute "f" (previously declared in base class "A") + p: Any # E: Cannot override final attribute "p" (previously declared in base class "A") +class D(A): + f: Final = a # E: Cannot override final attribute "f" (previously declared in base class "A") + p: Final = a # E: Cannot override final attribute "p" (previously declared in base class "A") +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingMethodWithVarImplicit] +from typing import final, Any, Final + +a: Any + +class A: + @final + def f(self) -> None: pass + @final + @classmethod + def c(cls) -> int: pass + +class B(A): + def __init__(self) -> None: + self.f: Any # E: Cannot assign to final attribute "f" \ + # E: Cannot override final attribute "f" (previously declared in base class "A") + self.c: Any # E: Cannot assign to final attribute "c" \ + # E: Cannot override final attribute "c" (previously declared in base class "A") + +B().f = a # E: Cannot assign to final attribute "f" +B().c = a # E: Cannot assign to final attribute "c" + +class C(A): + def __init__(self) -> None: + self.f: Final = a # E: Cannot override final attribute "f" (previously declared in base class "A") + self.c: Final = a # E: Cannot override final attribute "c" (previously declared in base class "A") +[builtins fixtures/classmethod.pyi] +[out] + +[case testFinalCanOverrideMethodWithFinal] +from typing import final + +class B: + def meth(self) -> None: ... +class C(B): + @final # OK + def meth(self) -> None: ... +[out] + +[case testFinalOverridingMethodMultipleInheritance] +from typing import final + +class A: + def m(self) -> int: pass +class B: + @final + def m(self) -> int: pass + +class C(A, B): pass # E: Cannot override final attribute "m" (previously declared in base class "B") +class D(B, A): pass +[out] + +[case testFinalOverridingMethodMultipleInheritanceVar] +from typing import final, Any + +class A: + m: Any +class B: + @final + def m(self) -> int: pass + +class C(A, B): pass # E: Cannot override final attribute "m" (previously declared in base class "B") +class D(B, A): pass # E: Cannot override writable attribute "m" with a final one +[out] + +[case testFinalOverridingClassMethod] +from typing import final + +class B: + @classmethod + @final + def f(cls) -> int: pass + +class C(B): + @classmethod # E: Cannot override final attribute "f" (previously declared in base class "B") + def f(cls) -> int: pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testFinalOverridingStaticMethod] +from typing import final + +class B: + @staticmethod + @final + def f() -> int: pass + @final + @staticmethod + def g() -> int: pass + +class C(B): + @staticmethod # E: Cannot override final attribute "f" (previously declared in base class "B") + def f() -> int: pass + @staticmethod # E: Cannot override final attribute "g" (previously declared in base class "B") + def g() -> int: pass +[builtins fixtures/staticmethod.pyi] +[out] + +[case testFinalOverridingProperty] +from typing import final + +class B: + @final + @property + def f(self) -> int: pass + @property + @final + def g(self) -> int: pass + +class C(B): + @property # E: Cannot override final attribute "f" (previously declared in base class "B") + def f(self) -> int: pass + @property # E: Cannot override final attribute "g" (previously declared in base class "B") + def g(self) -> int: pass +[builtins fixtures/property.pyi] +[out] + +[case testFinalOverridingMethodOverloads] +from typing import final, overload + +class B: + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... + @final + def f(self, x): + pass + +class C(B): + @overload # E: Cannot override final attribute "f" (previously declared in base class "B") + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... + def f(self, x): + pass +[out] + +[case testFinalClassNoInheritance] +from typing import final + +@final +class B: ... +class C(B): # E: Cannot inherit from final class "B" + pass +class D(C): # E: Cannot inherit from final class "B" + pass +[out] + +[case testFinalClassNoInheritanceMulti] +from typing import final + +class A: ... +@final +class B: ... +class C(B, A): # E: Cannot inherit from final class "B" + pass +class D(A, B): # E: Cannot inherit from final class "B" + pass +[out] + +[case testFinalCantOverrideWriteable] +from typing import Any, Final, final + +class B: + x: Any + @property + def y(self) -> Any: ... + @y.setter + def y(self, x: Any) -> None: ... + +class C(B): + x: Final = 1 # E: Cannot override writable attribute "x" with a final one + y: Final = 1 # E: Cannot override writable attribute "y" with a final one + +class D(B): + @final # E: Cannot override writable attribute "x" with a final one + def x(self) -> int: ... + @final # E: Cannot override writable attribute "y" with a final one + def y(self) -> int: ... +[builtins fixtures/property.pyi] +[out] + +[case testFinalCanUseTypingExtensions] +from typing_extensions import final, Final + +x: Final = 1 +x = 2 # E: Cannot assign to final name "x" + +class S: + x: Final = 1 +S.x = 2 # E: Cannot assign to final attribute "x" + +class B: + @final + def meth(self) -> None: ... +class C(B): + def meth(self) -> None: ... # E: Cannot override final attribute "meth" (previously declared in base class "B") + +@final +class F: ... +class E(F): ... # E: Cannot inherit from final class "F" +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalCanUseTypingExtensionsAliased] +from typing_extensions import final as f, Final as F + +x: F = 1 +x = 2 # E: Cannot assign to final name "x" + +class S: + x: F = 1 +S.x = 2 # E: Cannot assign to final attribute "x" + +class B: + @f + def meth(self) -> None: ... +class C(B): + def meth(self) -> None: ... # E: Cannot override final attribute "meth" (previously declared in base class "B") + +@f +class D(C): ... +class E(D): ... # E: Cannot inherit from final class "D" +[builtins fixtures/tuple.pyi] +[out] + +[case testFinalMultiassignAllowed] +from typing import Final + +class A: + x: Final[int] + y: Final[int] + def __init__(self) -> None: + self.x, self.y = 1, 2 + +class B: + x: Final[int] + y: Final[int] + def __init__(self) -> None: + self.x = self.y = 1 +[out] + +[case testFinalInDeferredMethod] +from typing_extensions import Final + +class A: + def __init__(self) -> None: + self.x = 10 # type: Final + undefined # type: ignore +[builtins fixtures/tuple.pyi] + +[case testFinalUsedWithClassVar] +from typing import Final, ClassVar + +class A: + a: Final[ClassVar[int]] # E: Variable should not be annotated with both ClassVar and Final + b: ClassVar[Final[int]] # E: Final can be only used as an outermost qualifier in a variable annotation + c: ClassVar[Final] = 1 # E: Final can be only used as an outermost qualifier in a variable annotation +[out] + +[case testFinalClassWithAbstractMethod] +from typing import final +from abc import ABC, abstractmethod + +@final +class A(ABC): # E: Final class __main__.A has abstract attributes "B" + @abstractmethod + def B(self) -> None: ... + +[case testFinalDefiningFuncWithAbstractMethod] +from typing import final +from abc import ABC, abstractmethod + +class A(ABC): + @final # E: Method B is both abstract and final + @abstractmethod + def B(self) -> None: ... + +[case testFinalClassVariableRedefinitionDoesNotCrash] +# This used to crash -- see #12950 +from typing import Final + +class MyClass: + a: None + a: Final[int] = 1 # E: Cannot redefine an existing name as final # E: Name "a" already defined on line 5 + +[case testFinalOverrideAllowedForPrivate] +from typing import Final, final + +class Parent: + __foo: Final[int] = 0 + @final + def __bar(self) -> None: ... + +class Child(Parent): + __foo: Final[int] = 1 + @final + def __bar(self) -> None: ... + +[case testFinalWithoutBool] +from typing_extensions import final, Literal + +class A: + pass + +@final +class B: + pass + +@final +class C: + def __len__(self) -> Literal[1]: return 1 + +reveal_type(A() and 42) # N: Revealed type is "Union[__main__.A, Literal[42]?]" +reveal_type(B() and 42) # N: Revealed type is "Literal[42]?" +reveal_type(C() and 42) # N: Revealed type is "Literal[42]?" + +[builtins fixtures/bool.pyi] + +[case testFinalWithoutBoolButWithLen] +from typing_extensions import final, Literal + +# Per Python data model, __len__ is called if __bool__ does not exist. +# In a @final class, __bool__ would not exist. + +@final +class A: + def __len__(self) -> int: ... + +@final +class B: + def __len__(self) -> Literal[1]: return 1 + +@final +class C: + def __len__(self) -> Literal[0]: return 0 + +reveal_type(A() and 42) # N: Revealed type is "Union[__main__.A, Literal[42]?]" +reveal_type(B() and 42) # N: Revealed type is "Literal[42]?" +reveal_type(C() and 42) # N: Revealed type is "__main__.C" + +[builtins fixtures/bool.pyi] diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test new file mode 100644 index 000000000000..c6419923ebc6 --- /dev/null +++ b/test-data/unit/check-flags.test @@ -0,0 +1,2445 @@ +[case testUnannotatedFunction] +# flags: --disallow-untyped-defs +def f(x): pass +[out] +main:2: error: Function is missing a type annotation + +[case testUnannotatedArgument] +# flags: --disallow-untyped-defs +def f(x) -> int: pass +[out] +main:2: error: Function is missing a type annotation for one or more arguments + +[case testNoArgumentFunction] +# flags: --disallow-untyped-defs +def f() -> int: pass +[out] + +[case testUnannotatedReturn] +# flags: --disallow-untyped-defs +def f(x: int): pass +[out] +main:2: error: Function is missing a return type annotation + +[case testUnannotatedReturnWithFastParser] +# flags: --disallow-untyped-defs +def f(x: int): pass +[out] +main:2: error: Function is missing a return type annotation + +[case testLambda] +# flags: --disallow-untyped-defs +lambda x: x +[out] + +[case testUntypedDef] +# flags: --disallow-untyped-defs +def f(): + 1 + "str" +[out] +main:2: error: Function is missing a return type annotation +main:2: note: Use "-> None" if function does not return a value + +[case testUnannotatedReturnWithOnlySelfArgument] +# flags: --disallow-untyped-defs +def f(self): pass +[out] +main:2: error: Function is missing a return type annotation +main:2: note: Use "-> None" if function does not return a value + +[case testUnannotatedReturnWithNontrivialReturn] +# flags: --disallow-untyped-defs +def f(): return 1 +[out] +main:2: error: Function is missing a return type annotation + +[case testUntypedAsyncDef] +# flags: --disallow-untyped-defs +async def f(): # E: Function is missing a return type annotation \ + # N: Use "-> None" if function does not return a value + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-medium.pyi] + +[case testAsyncUnannotatedArgument] +# flags: --disallow-untyped-defs +async def f(x) -> None: # E: Function is missing a type annotation for one or more arguments + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncUnannotatedReturn] +# flags: --disallow-untyped-defs +from typing import Any +async def f(x: int): # E: Function is missing a return type annotation + pass +# Make sure explicit Any is allowed. +async def g(x: int) -> Any: + pass +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testDisallowUntypedDefsAndGeneric] +# flags: --disallow-untyped-defs --disallow-any-generics +def get_tasks(self): + return 'whatever' +[out] +main:2: error: Function is missing a return type annotation + +[case testDisallowUntypedDefsUntypedDecorator] +# flags: --disallow-untyped-decorators +def d(p): + return p + +@d # E: Untyped decorator makes function "f" untyped +def f(i: int) -> int: + return i + +[case testDisallowUntypedDecoratorsUnresolvedDecorator] +# flags: --disallow-untyped-decorators --ignore-missing-imports +from nonexistent import d + +@d # E: Untyped decorator makes function "f" untyped +def f(i: int) -> int: + return i + +[case testDisallowUntypedDecoratorUntypedDef] +# flags: --disallow-untyped-decorators + +def d(p): + return p + +@d # no error +def f(): pass + +[case testDisallowUntypedDecoratorsPartialFunction] +# flags: --disallow-untyped-decorators + +def d(p): + return p + +@d # E: Untyped decorator makes function "f" untyped +def f(x) -> None: pass + +@d # E: Untyped decorator makes function "g" untyped +def g(x, y: int): pass + +@d # E: Untyped decorator makes function "h" untyped +def h(x: int): pass + +[case testDisallowUntypedDecoratorsImpreciseDecorator] +# flags: --disallow-untyped-decorators +from typing import Any + +def d(p) -> Any: + return p + +@d # no error +def f() -> None: pass + +[case testDisallowUntypedDecoratorsMultipleDecorators] +# flags: --disallow-untyped-decorators +from typing import Any + +def d1(p): + return p +def d2(p): + return p +def d3(p) -> Any: + return p + +@d1 # E: Untyped decorator makes function "f" untyped +@d2 # E: Untyped decorator makes function "f" untyped +@d3 # no error +@d1 # E: Untyped decorator makes function "f" untyped +def f() -> None: pass + +[case testDisallowUntypedDecoratorsCallableInstance] +# flags: --disallow-untyped-decorators +from typing import Callable + +class TypedDecorator: + def __call__(self, c: Callable) -> Callable: + return function + +class UntypedDecorator: + def __call__(self, c): + return function + +@TypedDecorator() +def f() -> None: pass + +@UntypedDecorator() # E: Untyped decorator makes function "g" untyped +def g() -> None: pass + +@TypedDecorator() +@UntypedDecorator() # E: Untyped decorator makes function "h" untyped +def h() -> None: pass + +@UntypedDecorator() # E: Untyped decorator makes function "i" untyped +@TypedDecorator() +def i() -> None: pass + +reveal_type(f) # N: Revealed type is "def (*Any, **Any) -> Any" +reveal_type(g) # N: Revealed type is "Any" +reveal_type(h) # N: Revealed type is "def (*Any, **Any) -> Any" +reveal_type(i) # N: Revealed type is "Any" + +[case testDisallowUntypedDecoratorsCallableInstanceDecoratedCall] +# flags: --disallow-untyped-decorators +from typing import Callable, TypeVar + +C = TypeVar('C', bound=Callable) + +def typed_decorator(c: C) -> C: + return c + +def untyped_decorator(c): + return c + +class TypedDecorator: + @typed_decorator + def __call__(self, c: Callable) -> Callable: + return function + +class UntypedDecorator1: + @untyped_decorator + def __call__(self, c): + return function + +class UntypedDecorator2: + @untyped_decorator # E: Untyped decorator makes function "__call__" untyped + def __call__(self, c: Callable) -> Callable: + return function + +class UntypedDecorator3: + @typed_decorator + @untyped_decorator # E: Untyped decorator makes function "__call__" untyped + def __call__(self, c: Callable) -> Callable: + return function + +class UntypedDecorator4: + @untyped_decorator # E: Untyped decorator makes function "__call__" untyped + @typed_decorator + def __call__(self, c: Callable) -> Callable: + return function + +@TypedDecorator() +def f() -> None: pass + +@UntypedDecorator1() # E: Untyped decorator makes function "g1" untyped +def g1() -> None: pass + +@UntypedDecorator2() # E: Untyped decorator makes function "g2" untyped +def g2() -> None: pass + +@UntypedDecorator3() # E: Untyped decorator makes function "g3" untyped +def g3() -> None: pass + +@UntypedDecorator4() # E: Untyped decorator makes function "g4" untyped +def g4() -> None: pass + +reveal_type(f) # N: Revealed type is "def (*Any, **Any) -> Any" +reveal_type(g1) # N: Revealed type is "Any" +reveal_type(g2) # N: Revealed type is "Any" +reveal_type(g3) # N: Revealed type is "Any" +reveal_type(g4) # N: Revealed type is "Any" +[builtins fixtures/bool.pyi] + +[case testDisallowUntypedDecoratorsNonCallableInstance] +# flags: --disallow-untyped-decorators +class Decorator: + pass + +@Decorator() # E: "Decorator" not callable +def f() -> None: pass + +[case testSubclassingAny] +# flags: --disallow-subclassing-any +from typing import Any +FakeClass = None # type: Any +class Foo(FakeClass): pass # E: Class cannot subclass "FakeClass" (has type "Any") +[out] + +[case testSubclassingAnyMultipleBaseClasses] +# flags: --disallow-subclassing-any +from typing import Any +FakeClass = None # type: Any +class ActualClass: pass +class Foo(ActualClass, FakeClass): pass # E: Class cannot subclass "FakeClass" (has type "Any") +[out] + +[case testSubclassingAnySilentImports] +# flags: --disallow-subclassing-any --follow-imports=skip +# cmd: mypy -m main + +[file main.py] +from ignored_module import BaseClass +class Foo(BaseClass): pass + +[file ignored_module.py] +class BaseClass: pass + +[out] +tmp/main.py:2: error: Class cannot subclass "BaseClass" (has type "Any") + +[case testSubclassingAnySilentImports2] +# flags: --disallow-subclassing-any --follow-imports=skip +# cmd: mypy -m main + +[file main.py] +import ignored_module +class Foo(ignored_module.BaseClass): pass + +[file ignored_module.py] +class BaseClass: pass + +[out] +tmp/main.py:2: error: Class cannot subclass "BaseClass" (has type "Any") + +[case testWarnNoReturnIgnoresTrivialFunctions] +# flags: --warn-no-return +def f() -> int: + pass +def g() -> int: + ... +def h() -> int: + """with docstring""" + pass +def i() -> int: + """with docstring""" + ... +def j() -> int: + u"""with unicode docstring""" + pass +def k() -> int: + """docstring only""" + +[case testWarnNoReturnWorksWithAlwaysTrue] +# flags: --warn-no-return +PY3 = True +def f() -> int: + if PY3: + return 0 + else: + return 0 +[builtins fixtures/bool.pyi] + +[case testWarnNoReturnWorksWithAlwaysFalse] +# flags: --warn-no-return +PY2 = False +def f() -> int: + if PY2: + return 0 + else: + return 0 +[builtins fixtures/bool.pyi] + +[case testWarnNoReturnWorksWithMypyTrue] +# flags: --warn-no-return +MYPY = False +def f() -> int: + if MYPY: + return 0 + else: + return 0 +[builtins fixtures/bool.pyi] + +[case testNoReturnDisallowsReturn] +# flags: --warn-no-return +from mypy_extensions import NoReturn + +def f() -> NoReturn: + if bool(): + return 5 # E: Return statement in function which does not return + else: + return # E: Return statement in function which does not return +[builtins fixtures/dict.pyi] + +[case testNoReturnWithoutImplicitReturn] +# flags: --warn-no-return +from mypy_extensions import NoReturn + +def no_return() -> NoReturn: pass +def f() -> NoReturn: + no_return() +[builtins fixtures/dict.pyi] + +[case testNoReturnDisallowsImplicitReturn] +# flags: --warn-no-return +from mypy_extensions import NoReturn + +def f() -> NoReturn: # E: Implicit return in function which does not return + non_trivial_function = 1 +[builtins fixtures/dict.pyi] + +[case testNoReturnImplicitReturnCheckInDeferredNode] +# flags: --warn-no-return +from typing import NoReturn + +def exit() -> NoReturn: ... + +def force_forward_reference() -> int: + return 4 + +def f() -> NoReturn: + x + exit() + +x = force_forward_reference() +[builtins fixtures/exception.pyi] + +[case testNoReturnNoWarnNoReturn] +# flags: --warn-no-return +from mypy_extensions import NoReturn + +def no_return() -> NoReturn: pass +def f() -> int: + if bool(): + return 0 + else: + no_return() +[builtins fixtures/dict.pyi] + +[case testNoReturnInExpr] +# flags: --warn-no-return +from mypy_extensions import NoReturn + +def no_return() -> NoReturn: pass +def f() -> int: + return 0 +reveal_type(f() or no_return()) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testNoReturnVariable] +# flags: --warn-no-return +from mypy_extensions import NoReturn + +x = 0 # type: NoReturn # E: Incompatible types in assignment (expression has type "int", variable has type "Never") +[builtins fixtures/dict.pyi] + +[case testNoReturnAsync] +# flags: --warn-no-return +from mypy_extensions import NoReturn + +async def f() -> NoReturn: ... + +async def g() -> NoReturn: + await f() + +async def h() -> NoReturn: # E: Implicit return in function which does not return + # Purposely not evaluating coroutine + _ = f() +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + +[case testNoWarnNoReturn] +# flags: --no-warn-no-return +import typing + +def implicit_optional_return(arg) -> typing.Optional[str]: + if arg: + return "false" + +def unsound_implicit_return(arg) -> str: # E: Incompatible return value type (implicitly returns "None", expected "str") + if arg: + return "false" + +def implicit_return_gen(arg) -> typing.Generator[int, None, typing.Optional[str]]: + yield 1 + +def unsound_implicit_return_gen(arg) -> typing.Generator[int, None, str]: # E: Incompatible return value type (implicitly returns "None", expected "str") + yield 1 +[builtins fixtures/dict.pyi] + +[case testNoWarnNoReturnNoStrictOptional] +# flags: --no-warn-no-return --no-strict-optional +import typing + +def implicit_optional_return(arg) -> typing.Optional[str]: + if arg: + return "false" + +def unsound_implicit_return(arg) -> str: + if arg: + return "false" + +def implicit_return_gen(arg) -> typing.Generator[int, None, typing.Optional[str]]: + yield 1 + +def unsound_implicit_return_gen(arg) -> typing.Generator[int, None, str]: + yield 1 +[builtins fixtures/dict.pyi] + +[case testNoReturnImportFromTyping] +from typing import NoReturn + +def h() -> NoReturn: + if bool(): + return 5 # E: Return statement in function which does not return + else: + return # E: Return statement in function which does not return + +def no_return() -> NoReturn: pass +def f() -> NoReturn: + no_return() + +x: NoReturn = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "Never") +[builtins fixtures/dict.pyi] + +[case testShowErrorContextFunction] +# flags: --show-error-context +def f() -> None: + 0 + "" +[out] +main: note: In function "f": +main:3: error: Unsupported operand types for + ("int" and "str") + +[case testShowErrorContextClass] +# flags: --show-error-context +class A: + 0 + "" +[out] +main: note: In class "A": +main:3: error: Unsupported operand types for + ("int" and "str") + +[case testShowErrorContextMember] +# flags: --show-error-context +class A: + def f(self, x: int) -> None: + self.f("") +[out] +main: note: In member "f" of class "A": +main:4: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testShowErrorContextModule] +# flags: --show-error-context +import m +[file m.py] +0 + "" +[out] +main:2: note: In module imported here: +tmp/m.py:1: error: Unsupported operand types for + ("int" and "str") + +[case testShowErrorContextTopLevel] +# flags: --show-error-context +def f() -> None: + 0 + "" +0 + "" +[out] +main: note: In function "f": +main:3: error: Unsupported operand types for + ("int" and "str") +main: note: At top level: +main:4: error: Unsupported operand types for + ("int" and "str") + +[case testShowErrorContextFromHere] +# flags: --show-error-context +import a +[file a.py] +import b +[file b.py] +0 + "" +[out] +tmp/a.py:1: note: In module imported here, +main:2: note: ... from here: +tmp/b.py:1: error: Unsupported operand types for + ("int" and "str") + +[case testFollowImportsNormal] +# flags: --follow-imports=normal +from mod import x +x + 0 +x + "" # E: Unsupported operand types for + ("int" and "str") +import mod +mod.x + 0 +mod.x + "" # E: Unsupported operand types for + ("int" and "str") +mod.y # E: "object" has no attribute "y" +mod + 0 # E: Unsupported left operand type for + ("object") +[file mod.py] +1 + "" # E: Unsupported operand types for + ("int" and "str") +x = 0 +x += "" # E: Unsupported operand types for + ("int" and "str") + +[case testFollowImportsSilent] +# flags: --follow-imports=silent +from mod import x +x + "" # E: Unsupported operand types for + ("int" and "str") +import mod +mod.x + "" # E: Unsupported operand types for + ("int" and "str") +mod.y # E: "object" has no attribute "y" +mod + 0 # E: Unsupported left operand type for + ("object") +[file mod.py] +1 + "" +x = 0 +x += "" + +[case testFollowImportsSilentTypeIgnore] +# flags: --warn-unused-ignores --follow-imports=silent +import mod +[file mod.py] +x = 3 # type: ignore + +[case testFollowImportsSkip] +# flags: --follow-imports=skip +from mod import x +reveal_type(x) # N: Revealed type is "Any" +x + "" +import mod +reveal_type(mod.x) # N: Revealed type is "Any" +[file mod.py] +this deliberate syntax error will not be reported + +[case testFollowImportsError] +# flags: --follow-imports=error +from mod import x # E: Import of "mod" ignored \ + # N: (Using --follow-imports=error, module not passed on command line) +x + "" +reveal_type(x) # N: Revealed type is "Any" +import mod +reveal_type(mod.x) # N: Revealed type is "Any" +[file mod.py] +deliberate syntax error + +[case testFollowImportsSelective] +# flags: --config-file tmp/mypy.ini +import normal +import silent +import skip +import error # E: Import of "error" ignored \ + # N: (Using --follow-imports=error, module not passed on command line) +reveal_type(normal.x) # N: Revealed type is "builtins.int" +reveal_type(silent.x) # N: Revealed type is "builtins.int" +reveal_type(skip) # N: Revealed type is "Any" +reveal_type(error) # N: Revealed type is "Any" +[file mypy.ini] +\[mypy] +\[mypy-normal] +follow_imports = normal +\[mypy-silent] +follow_imports = silent +\[mypy-skip] +follow_imports = skip +\[mypy-error] +follow_imports = error +[file normal.py] +x = 0 +x += '' # E: Unsupported operand types for + ("int" and "str") +[file silent.py] +x = 0 +x += '' +[file skip.py] +bla bla +[file error.py] +bla bla + +[case testIgnoreMissingImportsFalse] +from mod import x +[out] +main:1: error: Cannot find implementation or library stub for module named "mod" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testIgnoreMissingImportsTrue] +# flags: --ignore-missing-imports +from mod import x +[out] + +[case testNoConfigFile] +# flags: --config-file= +# type: ignore + +[file mypy.ini] +\[mypy] +warn_unused_ignores = True +[out] + +[case testPerFileIncompleteDefsBasic] +# flags: --config-file tmp/mypy.ini +import standard, incomplete + +[file standard.py] +def incomplete(x) -> int: + return 0 +[file incomplete.py] +def incomplete(x) -> int: # E: Function is missing a type annotation for one or more arguments + return 0 +[file mypy.ini] +\[mypy] +disallow_incomplete_defs = False +\[mypy-incomplete] +disallow_incomplete_defs = True + + +[case testPerFileIncompleteDefsBasicPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import standard, incomplete + +[file standard.py] +def incomplete(x) -> int: + return 0 +[file incomplete.py] +def incomplete(x) -> int: # E: Function is missing a type annotation for one or more arguments + return 0 +[file pyproject.toml] +\[tool.mypy] +disallow_incomplete_defs = false +\[[tool.mypy.overrides]] +module = 'incomplete' +disallow_incomplete_defs = true + + +[case testPerFileStrictOptionalBasic] +# flags: --config-file tmp/mypy.ini +import standard, optional + +[file standard.py] +x = 0 +if int(): + x = None +[file optional.py] +x = 0 +if int(): + x = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-optional] +strict_optional = True + + +[case testPerFileStrictOptionalBasicPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import standard, optional + +[file standard.py] +x = 0 +if int(): + x = None +[file optional.py] +x = 0 +if int(): + x = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +[file pyproject.toml] +\[tool.mypy] +strict_optional = false +\[[tool.mypy.overrides]] +module = 'optional' +strict_optional = true + + +[case testPerFileStrictOptionalBasicImportStandard] +# flags: --config-file tmp/mypy.ini +import standard, optional + +[file standard.py] +from typing import Optional +def f(x: int) -> None: pass +an_int = 0 # type: int +optional_int = None # type: Optional[int] +f(an_int) # ints can be used as ints +f(optional_int) # optional ints can be used as ints in this file + +[file optional.py] +import standard +def f(x: int) -> None: pass +standard.an_int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") +standard.optional_int = None # OK -- explicitly declared as optional +f(standard.an_int) # ints can be used as ints +f(standard.optional_int) # E: Argument 1 to "f" has incompatible type "None"; expected "int" + +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-optional] +strict_optional = True + + +[case testPerFileStrictOptionalBasicImportStandardPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import standard, optional + +[file standard.py] +from typing import Optional +def f(x: int) -> None: pass +an_int = 0 # type: int +optional_int = None # type: Optional[int] +f(an_int) # ints can be used as ints +f(optional_int) # optional ints can be used as ints in this file + +[file optional.py] +import standard +def f(x: int) -> None: pass +standard.an_int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") +standard.optional_int = None # OK -- explicitly declared as optional +f(standard.an_int) # ints can be used as ints +f(standard.optional_int) # E: Argument 1 to "f" has incompatible type "None"; expected "int" + +[file pyproject.toml] +\[tool.mypy] +strict_optional = false +\[[tool.mypy.overrides]] +module = 'optional' +strict_optional = true + + +[case testPerFileStrictOptionalBasicImportOptional] +# flags: --config-file tmp/mypy.ini +import standard, optional + +[file standard.py] +import optional +def f(x: int) -> None: pass +f(optional.x) # OK -- in non-strict Optional context +f(optional.y) # OK -- in non-strict Optional context + +[file optional.py] +from typing import Optional +def f(x: int) -> None: pass +x = 0 # type: Optional[int] +y = None # type: None + +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-optional] +strict_optional = True + + +[case testPerFileStrictOptionalBasicImportOptionalPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import standard, optional + +[file standard.py] +import optional +def f(x: int) -> None: pass +f(optional.x) # OK -- in non-strict Optional context +f(optional.y) # OK -- in non-strict Optional context + +[file optional.py] +from typing import Optional +def f(x: int) -> None: pass +x = 0 # type: Optional[int] +y = None # type: None + +[file pyproject.toml] +\[tool.mypy] +strict_optional = false +\[[tool.mypy.overrides]] +module = 'optional' +strict_optional = true + + +[case testPerFileStrictOptionalListItemImportOptional] +# flags: --config-file tmp/mypy.ini +import standard, optional + +[file standard.py] +import optional +from typing import List +def f(x: List[int]) -> None: pass +f(optional.x) # OK -- in non-strict Optional context +f(optional.y) # OK -- in non-strict Optional context + +[file optional.py] +from typing import Optional, List +def f(x: List[int]) -> None: pass +x = [] # type: List[Optional[int]] +y = [] # type: List[int] + +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-optional] +strict_optional = True +[builtins fixtures/list.pyi] + + +[case testPerFileStrictOptionalListItemImportOptionalPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import standard, optional + +[file standard.py] +import optional +from typing import List +def f(x: List[int]) -> None: pass +f(optional.x) # OK -- in non-strict Optional context +f(optional.y) # OK -- in non-strict Optional context + +[file optional.py] +from typing import Optional, List +def f(x: List[int]) -> None: pass +x = [] # type: List[Optional[int]] +y = [] # type: List[int] + +[file pyproject.toml] +\[tool.mypy] +strict_optional = false +\[[tool.mypy.overrides]] +module = 'optional' +strict_optional = true + +[builtins fixtures/list.pyi] + + +[case testPerFileStrictOptionalComplicatedList] +from typing import Union, Optional, List + +def f() -> None: + x = [] # type: Union[List[Optional[str]], str] +[builtins fixtures/list.pyi] + +[case testPerFileStrictOptionalNoneArguments] +# flags: --config-file tmp/mypy.ini +import standard, optional + +[file standard.py] +def f(x: int = None) -> None: pass + +[file optional.py] +import standard +def f(x: int = None) -> None: pass +standard.f(None) + +[file mypy.ini] +\[mypy] +strict_optional = False +implicit_optional = true +\[mypy-optional] +strict_optional = True + + +[case testPerFileStrictOptionalNoneArgumentsPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import standard, optional + +[file standard.py] +def f(x: int = None) -> None: pass + +[file optional.py] +import standard +def f(x: int = None) -> None: pass +standard.f(None) + +[file pyproject.toml] +\[tool.mypy] +strict_optional = false +implicit_optional = true +\[[tool.mypy.overrides]] +module = 'optional' +strict_optional = true + +[case testSilentMissingImportsOff] +-- ignore_missing_imports is False by default. +import missing # E: Cannot find implementation or library stub for module named "missing" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +reveal_type(missing.x) # N: Revealed type is "Any" + +[case testSilentMissingImportsOn] +# flags: --ignore-missing-imports +import missing +reveal_type(missing.x) # N: Revealed type is "Any" + +[case testDisallowImplicitTypesIgnoreMissingTypes] +# flags: --ignore-missing-imports --disallow-any-unimported +from missing import MyType + +def f(x: MyType) -> None: # E: Argument 1 to "f" becomes "Any" due to an unfollowed import + pass + +[case testDisallowImplicitTypes] +# flags: --disallow-any-unimported +from missing import MyType + +def f(x: MyType) -> None: + pass +[out] +main:2: error: Cannot find implementation or library stub for module named "missing" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:4: error: Argument 1 to "f" becomes "Any" due to an unfollowed import + +[case testDisallowImplicitAnyVariableDefinition] +# flags: --ignore-missing-imports --disallow-any-unimported +from missing import Unchecked + +t: Unchecked = 12 # E: Type of variable becomes "Any" due to an unfollowed import + +[case testAllowImplicitAnyVariableDefinition] +# flags: --ignore-missing-imports --allow-any-unimported +from missing import Unchecked + +t: Unchecked = 12 + +[case testDisallowImplicitAnyGeneric] +# flags: --ignore-missing-imports --disallow-any-unimported +from missing import Unchecked +from typing import List + +def foo(l: List[Unchecked]) -> List[Unchecked]: + t = [] # type: List[Unchecked] + return l +[builtins fixtures/list.pyi] +[out] +main:5: error: Return type becomes "List[Any]" due to an unfollowed import +main:5: error: Argument 1 to "foo" becomes "List[Any]" due to an unfollowed import +main:6: error: Type of variable becomes "List[Any]" due to an unfollowed import + +[case testDisallowImplicitAnyInherit] +# flags: --ignore-missing-imports --disallow-any-unimported +from missing import Unchecked +from typing import List + +class C(Unchecked): # E: Base type Unchecked becomes "Any" due to an unfollowed import + pass + +class A(List[Unchecked]): # E: Base type becomes "List[Any]" due to an unfollowed import + pass +[builtins fixtures/list.pyi] + +[case testDisallowImplicitAnyAlias] +# flags: --ignore-missing-imports --disallow-any-unimported +from missing import Unchecked +from typing import List + +X = List[Unchecked] # E: Type alias target becomes "List[Any]" due to an unfollowed import + +def f(x: X) -> None: + pass +[builtins fixtures/list.pyi] + +[case testDisallowImplicitAnyCast] +# flags: --ignore-missing-imports --disallow-any-unimported +from missing import Unchecked +from typing import List, cast + + +foo = [1, 2, 3] +cast(List[Unchecked], foo) # E: Target type of cast becomes "List[Any]" due to an unfollowed import +cast(Unchecked, foo) # E: Target type of cast becomes "Any" due to an unfollowed import +[builtins fixtures/list.pyi] + +[case testDisallowImplicitAnyNamedTuple] +# flags: --ignore-missing-imports --disallow-any-unimported +from typing import List, NamedTuple +from missing import Unchecked + +Point = NamedTuple('Point', [('x', List[Unchecked]), + ('y', Unchecked)]) +[builtins fixtures/list.pyi] +[out] +main:5: error: NamedTuple type becomes "Tuple[List[Any], Any]" due to an unfollowed import + +[case testDisallowImplicitAnyTypeVarConstraints] +# flags: --ignore-missing-imports --disallow-any-unimported +from typing import List, NamedTuple, TypeVar, Any +from missing import Unchecked + +T = TypeVar('T', Unchecked, List[Unchecked], str) +[builtins fixtures/list.pyi] +[out] +main:5: error: Constraint 1 becomes "Any" due to an unfollowed import +main:5: error: Constraint 2 becomes "List[Any]" due to an unfollowed import + +[case testDisallowImplicitAnyNewType] +# flags: --ignore-missing-imports --disallow-any-unimported +from typing import NewType, List +from missing import Unchecked + +Baz = NewType('Baz', Unchecked) # E: Argument 2 to NewType(...) must be subclassable (got "Any") +Bar = NewType('Bar', List[Unchecked]) # E: Argument 2 to NewType(...) becomes "List[Any]" due to an unfollowed import + +[builtins fixtures/list.pyi] + +[case testDisallowImplicitAnyCallableAndTuple] +# flags: --ignore-missing-imports --disallow-any-unimported +from typing import Callable, Tuple +from missing import Unchecked + +def foo(f: Callable[[], Unchecked]) -> Tuple[Unchecked]: + return f() +[builtins fixtures/list.pyi] +[out] +main:5: error: Return type becomes "Tuple[Any]" due to an unfollowed import +main:5: error: Argument 1 to "foo" becomes "Callable[[], Any]" due to an unfollowed import + +[case testDisallowImplicitAnySubclassingExplicitAny] +# flags: --ignore-missing-imports --disallow-any-unimported --disallow-subclassing-any +from typing import Any + +class C(Any): # E: Class cannot subclass "Any" (has type "Any") + pass + +[case testDisallowImplicitAnyVarDeclaration] +# flags: --ignore-missing-imports --disallow-any-unimported +from missing import Unchecked + +foo: Unchecked = "" +foo = "" +x, y = 1, 2 # type: Unchecked, Unchecked +[builtins fixtures/tuple.pyi] +[out] +main:4: error: Type of variable becomes "Any" due to an unfollowed import +main:6: error: A type on this line becomes "Any" due to an unfollowed import + +[case testDisallowUnimportedAnyTypedDictSimple] +# flags: --ignore-missing-imports --disallow-any-unimported +from mypy_extensions import TypedDict +from x import Unchecked + +M = TypedDict('M', {'x': str, 'y': Unchecked}) # E: Type of a TypedDict key becomes "Any" due to an unfollowed import + +def f(m: M) -> M: pass # no error +[builtins fixtures/dict.pyi] + +[case testDisallowUnimportedAnyTypedDictGeneric] +# flags: --ignore-missing-imports --disallow-any-unimported + +from mypy_extensions import TypedDict +from typing import List +from x import Unchecked + +M = TypedDict('M', {'x': str, 'y': List[Unchecked]}) # E: Type of a TypedDict key becomes "List[Any]" due to an unfollowed import + +def f(m: M) -> M: pass # no error +[builtins fixtures/dict.pyi] + +[case testDisallowAnyDecoratedUnannotatedDecorator] +# flags: --disallow-any-decorated +from typing import Any + +def d(f): + return f + +@d +def f(x: Any) -> Any: # E: Function is untyped after decorator transformation + pass +@d +def h(x): # E: Function is untyped after decorator transformation + pass +[builtins fixtures/list.pyi] +[case testDisallowAnyDecoratedErrorIsReportedOnlyOnce] +# flags: --disallow-any-decorated + +def d(f): + return f + +def d2(f): + return f + +@d +@d2 +@d +def f(x: int) -> None: pass # E: Function is untyped after decorator transformation +[case testDisallowAnyDecoratedReturnAny] +# flags: --disallow-any-decorated +from typing import Any + +def d(f) -> Any: + return f + +@d +def f() -> None: pass # E: Function is untyped after decorator transformation +[builtins fixtures/list.pyi] +[case testDisallowAnyDecoratedReturnCallable] +# flags: --disallow-any-decorated +from typing import Any, Callable + +def d(f) -> Callable[..., None]: + return f + +@d +def g(i: int, s: str) -> None: pass # E: Type of decorated function contains type "Any" ("Callable[..., None]") + +[builtins fixtures/list.pyi] +[case testDisallowAnyDecoratedNonexistentDecorator] +# flags: --disallow-any-decorated --ignore-missing-imports +from nonexistent import d + +@d +def f() -> None: pass # E: Function is untyped after decorator transformation +[builtins fixtures/list.pyi] + +[case testDisallowAnyDecoratedPartlyTypedCallable] +# flags: --disallow-any-decorated --ignore-missing-imports +from typing import Callable, Any, List + +def d(f) -> Callable[[int, Any], Any]: pass +def d2(f) -> Callable[[int], List[Any]]: pass +def d3(f) -> Callable[[Any], List[str]]: pass + +@d +def f(i: int, s: str) -> None: # E: Type of decorated function contains type "Any" ("Callable[[int, Any], Any]") + pass +@d2 +def g(i: int) -> None: # E: Type of decorated function contains type "Any" ("Callable[[int], List[Any]]") + pass +@d3 +def h(i: int) -> None: # E: Type of decorated function contains type "Any" ("Callable[[Any], List[str]]") + pass +[builtins fixtures/list.pyi] + +[case testDisallowAnyDecoratedReturnsCallableNoParams] +# flags: --disallow-any-decorated +from typing import Callable + +def d(p) -> Callable[[], int]: + return p + +@d +def f(i): + return i +[builtins fixtures/list.pyi] + +[case testDisallowAnyDecoratedDecoratorReturnsNonCallable] +# flags: --disallow-any-decorated +def d(p) -> int: + return p(0) + +@d +def f(i): + return i + +[case testDisallowAnyDecoratedUntypedUndecoratedFunction] +# flags: --disallow-any-decorated +from typing import Callable + +def f(i): # no error + return i + +[case testDisallowAnyDecoratedTwoDecorators] +# flags: --disallow-any-decorated +from typing import Callable + +def typed_dec(f) -> Callable[[], int]: pass +def untyped_dec(f): pass + +@typed_dec +@untyped_dec +def f(): # no error + return i + +@untyped_dec +@typed_dec +def g(): # E: Function is untyped after decorator transformation + return i + +[case testDisallowAnyExprSimple] +# flags: --disallow-any-expr +from typing import Any +def f(s): + yield s + +def g(x) -> Any: + yield x # E: Expression has type "Any" + +x = f(0) # E: Expression has type "Any" +for x in f(0): # E: Expression has type "Any" + g(x) # E: Expression has type "Any" + +l = [1, 2, 3] +l[f(0)] # E: Expression has type "Any" +f(l) +f(f(0)) # E: Expression has type "Any" +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprUnannotatedFunction] +# flags: --disallow-any-expr +def g(s): + return s + +g(0) +w: int = g(1) + +[case testDisallowAnyExprExplicitAnyParam] +# flags: --disallow-any-expr +from typing import Any, List +def f(s: Any) -> None: + pass + +def g(s: List[Any]) -> None: + pass + +f(0) + +# type of list below is inferred with expected type of "List[Any]", so that becomes it's type +# instead of List[str] +g(['']) # E: Expression type contains "Any" (has type "List[Any]") +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprAllowsAnyInCast] +# flags: --disallow-any-expr +from typing import Any, cast +class Foo: + g: Any = 2 + +z = cast(int, Foo().g) +m = cast(Any, Foo().g) # E: Expression has type "Any" +k = Foo.g # E: Expression has type "Any" +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprAllowsAnyInVariableAssignmentWithExplicitTypeAnnotation] +# flags: --disallow-any-expr +from typing import Any +class Foo: + g: Any = 2 + +z: int = Foo().g +x = Foo().g # type: int +m: Any = Foo().g # E: Expression has type "Any" +n = Foo().g # type: Any # E: Expression has type "Any" +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprGeneric] +# flags: --disallow-any-expr +from typing import List + +l: List = [] +l.append(1) # E: Expression type contains "Any" (has type "List[Any]") +k = l[0] # E: Expression type contains "Any" (has type "List[Any]") # E: Expression has type "Any" +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprTypeVar] +# flags: --disallow-any-expr +from typing import TypeVar + +T = TypeVar('T') # no error + +def f(t: T) -> T: + return t +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprNamedTuple] +# flags: --disallow-any-expr +from typing import NamedTuple + +Point = NamedTuple('Point', [('x', int), ('y', int)]) # no error + +def origin() -> Point: + return Point(x=0, y=0) +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprNewType] +# flags: --disallow-any-expr +from typing import NewType + +NT = NewType('NT', int) # no error + +def nt() -> NT: + return NT(1) +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprEnum] +# flags: --disallow-any-expr +from enum import Enum +E = Enum('E', '1, 2, 3') # no error + +def k(s: E) -> None: pass +[builtins fixtures/list.pyi] + +[case testDisallowAnyExprTypedDict] +# flags: --disallow-any-expr +from mypy_extensions import TypedDict + +Movie = TypedDict('Movie', {'name': str, 'year': int}) + +def g(m: Movie) -> Movie: + return m +[builtins fixtures/dict.pyi] + +[case testDisallowIncompleteDefs] +# flags: --disallow-incomplete-defs + +def f(i: int): # E: Function is missing a return type annotation + pass +def g(i) -> None: # E: Function is missing a type annotation for one or more arguments + pass +def h(i: int) -> int: # no error + return i +def i() -> None: # no error + pass + +[case testDisallowIncompleteDefsNoReturn] +# flags: --disallow-incomplete-defs --disallow-untyped-defs + +def f(i: int): # E: Function is missing a return type annotation + pass + +[case testDisallowIncompleteDefsSelf] +# flags: --disallow-incomplete-defs +class C: + def foo(self) -> None: # no error + pass + +[case testDisallowIncompleteDefsPartiallyAnnotatedParams] +# flags: --disallow-incomplete-defs + +def f(i: int, s): + pass + +[out] +main:3: error: Function is missing a return type annotation +main:3: error: Function is missing a type annotation for one or more arguments + +[case testDisallowIncompleteDefsAttrsNoAnnotations] +# flags: --disallow-incomplete-defs +import attrs + +@attrs.define +class Unannotated: + foo = attrs.field() + +[builtins fixtures/plugin_attrs.pyi] + +[case testDisallowIncompleteDefsAttrsWithAnnotations] +# flags: --disallow-incomplete-defs +import attrs + +@attrs.define +class Annotated: + bar: int = attrs.field() + +[builtins fixtures/plugin_attrs.pyi] + +[case testDisallowIncompleteDefsAttrsPartialAnnotations] +# flags: --disallow-incomplete-defs +import attrs + +@attrs.define +class PartiallyAnnotated: # E: Function is missing a type annotation for one or more arguments + bar: int = attrs.field() + baz = attrs.field() + +[builtins fixtures/plugin_attrs.pyi] + +[case testAlwaysTrueAlwaysFalseFlags] +# flags: --always-true=YOLO --always-true=YOLO1 --always-false=BLAH1 --always-false BLAH --ignore-missing-imports +from somewhere import YOLO, BLAH +if not YOLO: + 1+() +if BLAH: + 1+() +[builtins fixtures/bool.pyi] + +[case testAlwaysTrueAlwaysFalseConfigFile] +# flags: --config-file tmp/mypy.ini +from somewhere import YOLO, BLAH +if not YOLO: + 1+() +if BLAH: + 1+() +[file mypy.ini] +\[mypy] +ignore_missing_imports = True +always_true = YOLO1, YOLO +always_false = BLAH, BLAH1 +[builtins fixtures/bool.pyi] + + +[case testAlwaysTrueAlwaysFalseConfigFilePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +from somewhere import YOLO, BLAH +if not YOLO: + 1+() +if BLAH: + 1+() + +[file pyproject.toml] +\[tool.mypy] +ignore_missing_imports = true +always_true = ['YOLO1', 'YOLO'] +always_false = ['BLAH', 'BLAH1'] + +[builtins fixtures/bool.pyi] + + +[case testDisableErrorCodeConfigFile] +# flags: --config-file tmp/mypy.ini --disallow-untyped-defs +import foo +def bar(): + pass +[file mypy.ini] +\[mypy] +disable_error_code = import, no-untyped-def + + +[case testDisableErrorCodeConfigFilePyProjectTOML] +# flags: --config-file tmp/pyproject.toml --disallow-untyped-defs +import foo +def bar(): + pass + +[file pyproject.toml] +\[tool.mypy] +disable_error_code = ['import', 'no-untyped-def'] + + +[case testCheckDisallowAnyGenericsNamedTuple] +# flags: --disallow-any-generics +from typing import NamedTuple + +N = NamedTuple('N', [('x', N)]) # type: ignore +n: N +[builtins fixtures/tuple.pyi] +[out] + +[case testCheckDisallowAnyGenericsTypedDict] +# flags: --disallow-any-generics +from typing import Dict, Any, Optional +from mypy_extensions import TypedDict + +VarsDict = Dict[str, Any] +HostsDict = Dict[str, Optional[VarsDict]] + +GroupDataDict = TypedDict( + "GroupDataDict", {"children": "GroupsDict", # type: ignore + "vars": VarsDict, + "hosts": HostsDict}, total=False +) + +GroupsDict = Dict[str, GroupDataDict] # type: ignore +[builtins fixtures/dict.pyi] + + +[case testCheckDisallowAnyGenericsStubOnly] +# flags: --disallow-any-generics --python-version 3.8 +from asyncio import Future +from queue import Queue +x: Future[str] +y: Queue[int] + +p: Future # E: Missing type parameters for generic type "Future" \ + # N: Subscripting classes that are not generic at runtime may require escaping, see https://mypy.readthedocs.io/en/stable/runtime_troubles.html#not-generic-runtime +q: Queue # E: Missing type parameters for generic type "Queue" \ + # N: Subscripting classes that are not generic at runtime may require escaping, see https://mypy.readthedocs.io/en/stable/runtime_troubles.html#not-generic-runtime +[file asyncio/__init__.pyi] +from asyncio.futures import Future as Future +[file asyncio/futures.pyi] +from typing import TypeVar, Generic +_T = TypeVar('_T') +class Future(Generic[_T]): ... +[file queue.pyi] +from typing import TypeVar, Generic +_T = TypeVar('_T') +class Queue(Generic[_T]): ... +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-full.pyi] + +[case testDisallowAnyGenericsBuiltinTuplePre39] +# flags: --disallow-any-generics --python-version 3.8 +s = tuple([1, 2, 3]) +def f(t: tuple) -> None: pass # E: Implicit generic "Any". Use "typing.Tuple" and specify generic parameters +[builtins fixtures/tuple.pyi] + +[case testDisallowAnyGenericsBuiltinListPre39] +# flags: --disallow-any-generics --python-version 3.8 +l = list([1, 2, 3]) +def f(t: list) -> None: pass # E: Implicit generic "Any". Use "typing.List" and specify generic parameters +[builtins fixtures/list.pyi] + +[case testDisallowAnyGenericsBuiltinSetPre39] +# flags: --disallow-any-generics --python-version 3.8 +l = set({1, 2, 3}) +def f(s: set) -> None: pass # E: Implicit generic "Any". Use "typing.Set" and specify generic parameters +[builtins fixtures/set.pyi] + +[case testDisallowAnyGenericsBuiltinDictPre39] +# flags: --disallow-any-generics --python-version 3.8 +l = dict([('a', 1)]) +def f(d: dict) -> None: pass # E: Implicit generic "Any". Use "typing.Dict" and specify generic parameters +[builtins fixtures/dict.pyi] + +[case testCheckDefaultAllowAnyGeneric] +from typing import TypeVar, Callable + +T = TypeVar('T') +C = Callable[[], T] + +def f(c: C): + pass +[out] + +[case testCheckAllowAnyGenericAnyGeneric] +# flags: --strict --allow-any-generics +from typing import TypeVar, Callable + +T = TypeVar('T') +C = Callable[[], T] + +def f(c: C) -> None: + pass +[out] + +[case testCheckDisallowAnyGenericsAnyGeneric] +# flags: --disallow-any-generics +from typing import TypeVar, Callable + +T = TypeVar('T') +C = Callable[[], T] + +def f(c: C): # E: Missing type parameters for generic type "C" + pass +[out] + +[case testStrictAnyGeneric] +# flags: --strict +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): + pass + +def f(c: A) -> None: # E: Missing type parameters for generic type "A" + pass +[out] + +[case testStrictInConfigAnyGeneric] +# flags: --config-file tmp/mypy.ini +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): + pass + +def f(c: A) -> None: # E: Missing type parameters for generic type "A" + pass +[file mypy.ini] +\[mypy] +strict = True +[out] + + +[case testStrictInConfigAnyGenericPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): + pass + +def f(c: A) -> None: # E: Missing type parameters for generic type "A" + pass + +[file pyproject.toml] +\[tool.mypy] +strict = true + +[out] + + +[case testStrictFalseInConfigAnyGeneric] +# flags: --config-file tmp/mypy.ini +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): + pass + +def f(c: A) -> None: + pass +[file mypy.ini] +\[mypy] +strict = False +[out] + + +[case testStrictFalseInConfigAnyGenericPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): + pass + +def f(c: A) -> None: + pass + +[file pyproject.toml] +\[tool.mypy] +strict = false + +[out] + + +[case testStrictAndStrictEquality] +# flags: --strict +x = 0 +y = '' +if x == y: # E: Non-overlapping equality check (left operand type: "int", right operand type: "str") + int() +[builtins fixtures/ops.pyi] + +[case testStrictEqualityPerFile] +# flags: --config-file tmp/mypy.ini +import b +42 == 'no' # E: Non-overlapping equality check (left operand type: "Literal[42]", right operand type: "Literal['no']") +[file b.py] +42 == 'no' +[file mypy.ini] +\[mypy] +strict_equality = True +\[mypy-b] +strict_equality = False +[builtins fixtures/bool.pyi] + + +[case testStrictEqualityPerFilePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import b +42 == 'no' # E: Non-overlapping equality check (left operand type: "Literal[42]", right operand type: "Literal['no']") + +[file b.py] +42 == 'no' + +[file pyproject.toml] +\[tool.mypy] +strict_equality = true +\[[tool.mypy.overrides]] +module = 'b' +strict_equality = false + +[builtins fixtures/bool.pyi] + + +[case testNoImplicitReexport] +# flags: --no-implicit-reexport --show-error-codes +from other_module_2 import a # E: Module "other_module_2" does not explicitly export attribute "a" [attr-defined] +reveal_type(a) # N: Revealed type is "builtins.int" + +import other_module_2 +reveal_type(other_module_2.a) # E: Module "other_module_2" does not explicitly export attribute "a" [attr-defined] \ + # N: Revealed type is "builtins.int" + +from other_module_2 import b # E: Module "other_module_2" does not explicitly export attribute "b" [attr-defined] +reveal_type(b) # N: Revealed type is "def (a: builtins.int) -> builtins.str" + +import other_module_2 +reveal_type(other_module_2.b) # E: Module "other_module_2" does not explicitly export attribute "b" [attr-defined] \ + # N: Revealed type is "def (a: builtins.int) -> builtins.str" + +[file other_module_1.py] +a = 5 +def b(a: int) -> str: ... +[file other_module_2.py] +from other_module_1 import a, b +[builtins fixtures/module.pyi] + +[case testNoImplicitReexportRespectsAll] +# flags: --no-implicit-reexport +from other_module_2 import a +from other_module_2 import b +[file other_module_1.py] +a = 5 +b = 6 +[file other_module_2.py] +from other_module_1 import a, b +__all__ = ('b',) +[builtins fixtures/tuple.pyi] +[out] +main:2: error: Module "other_module_2" does not explicitly export attribute "a" + +[case testNoImplicitReexportStarConsideredExplicit] +# flags: --no-implicit-reexport +from other_module_2 import a +from other_module_2 import b +[file other_module_1.py] +a = 5 +b = 6 +[file other_module_2.py] +from other_module_1 import * +__all__ = ('b',) +[builtins fixtures/tuple.pyi] + +[case testNoImplicitReexportGetAttr] +# flags: --no-implicit-reexport +from other_module_2 import a # E: Module "other_module_2" does not explicitly export attribute "a" +reveal_type(a) # N: Revealed type is "builtins.int" +from other_module_2 import b # E: Module "other_module_2" does not explicitly export attribute "b" +reveal_type(b) # N: Revealed type is "builtins.str" +[file other_module_1.py] +b: str = "asdf" +def __getattr__(name: str) -> int: ... +[file other_module_2.py] +from other_module_1 import a, b +def __getattr__(name: str) -> bytes: ... +[builtins fixtures/tuple.pyi] + +[case textNoImplicitReexportSuggestions] +# flags: --no-implicit-reexport +from other_module_2 import attr_1 + +[file other_module_1.py] +attr_1 = 5 +attr_2 = 6 +[file other_module_2.py] +from other_module_1 import attr_1, attr_2 +[out] +main:2: error: Module "other_module_2" does not explicitly export attribute "attr_1" + +[case testNoImplicitReexportMypyIni] +# flags: --config-file tmp/mypy.ini +from other_module_2 import a + +[file other_module_1.py] +a = 5 + +[file other_module_2.py] +from other_module_1 import a + +[file mypy.ini] +\[mypy] +implicit_reexport = True +\[mypy-other_module_2] +implicit_reexport = False +[out] +main:2: error: Module "other_module_2" does not explicitly export attribute "a" + + +[case testNoImplicitReexportPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +from other_module_2 import a + +[file other_module_1.py] +a = 5 + +[file other_module_2.py] +from other_module_1 import a + +[file pyproject.toml] +\[tool.mypy] +implicit_reexport = true +\[[tool.mypy.overrides]] +module = 'other_module_2' +implicit_reexport = false + +[out] +main:2: error: Module "other_module_2" does not explicitly export attribute "a" + + +[case testImplicitAnyOKForNoArgs] +# flags: --disallow-any-generics --show-column-numbers +from typing import List + +A = List # OK +B = List[A] # E:10: Missing type parameters for generic type "A" +x: A # E:4: Missing type parameters for generic type "A" +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitDefSignature] +# flags: --disallow-any-explicit + +from typing import Any, List + +def f(x: Any) -> None: # E: Explicit "Any" is not allowed + pass + +def g() -> Any: # E: Explicit "Any" is not allowed + pass + +def h() -> List[Any]: # E: Explicit "Any" is not allowed + pass +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitVarDeclaration] +# flags: --disallow-any-explicit +from typing import Any +v: Any = '' # E: Explicit "Any" is not allowed +w = '' # type: Any # E: Explicit "Any" is not allowed +class X: + y = '' # type: Any # E: Explicit "Any" is not allowed + +[case testDisallowAnyExplicitGenericVarDeclaration] +# flags: --disallow-any-explicit +from typing import Any, List +v: List[Any] = [] # E: Explicit "Any" is not allowed +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitInheritance] +# flags: --disallow-any-explicit +from typing import Any, List + +class C(Any): # E: Explicit "Any" is not allowed + pass + +class D(List[Any]): # E: Explicit "Any" is not allowed + pass +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitAlias] +# flags: --disallow-any-explicit +from typing import Any, List + +X = Any # E: Explicit "Any" is not allowed +Y = List[Any] # E: Explicit "Any" is not allowed + +def foo(x: X) -> Y: # no error + x.nonexistent() # no error + return x +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitGenericAlias] +# flags: --disallow-any-explicit +from typing import Any, TypeVar, Tuple + +T = TypeVar('T') + +TupleAny = Tuple[Any, T] # E: Explicit "Any" is not allowed + +def foo(x: TupleAny[str]) -> None: # no error + pass + +def goo(x: TupleAny[Any]) -> None: # E: Explicit "Any" is not allowed + pass +[builtins fixtures/tuple.pyi] + +[case testDisallowAnyExplicitCast] +# flags: --disallow-any-explicit +from typing import Any, List, cast + +x = 1 +y = cast(Any, x) # E: Explicit "Any" is not allowed +z = cast(List[Any], x) # E: Explicit "Any" is not allowed +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitNamedTuple] +# flags: --disallow-any-explicit +from typing import Any, List, NamedTuple + +Point = NamedTuple('Point', [('x', List[Any]), ('y', Any)]) # E: Explicit "Any" is not allowed +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitTypeVarConstraint] +# flags: --disallow-any-explicit +from typing import Any, List, TypeVar + +T = TypeVar('T', Any, List[Any]) # E: Explicit "Any" is not allowed +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitNewType] +# flags: --disallow-any-explicit +from typing import Any, List, NewType + +# this error does not come from `--disallow-any-explicit` flag +Baz = NewType('Baz', Any) # E: Argument 2 to NewType(...) must be subclassable (got "Any") +Bar = NewType('Bar', List[Any]) # E: Explicit "Any" is not allowed +[builtins fixtures/list.pyi] + +[case testDisallowAnyExplicitTypedDictSimple] +# flags: --disallow-any-explicit +from mypy_extensions import TypedDict +from typing import Any + +M = TypedDict('M', {'x': str, 'y': Any}) # E: Explicit "Any" is not allowed +M(x='x', y=2) # no error +def f(m: M) -> None: pass # no error +[builtins fixtures/dict.pyi] + +[case testDisallowAnyExplicitTypedDictGeneric] +# flags: --disallow-any-explicit +from mypy_extensions import TypedDict +from typing import Any, List + +M = TypedDict('M', {'x': str, 'y': List[Any]}) # E: Explicit "Any" is not allowed +N = TypedDict('N', {'x': str, 'y': List}) # no error +[builtins fixtures/dict.pyi] + +[case testDisallowAnyGenericsTupleNoTypeParams] +# flags: --disallow-any-generics +from typing import Tuple + +def f(s: Tuple) -> None: pass # E: Missing type parameters for generic type "Tuple" +def g(s) -> Tuple: # E: Missing type parameters for generic type "Tuple" + return 'a', 'b' +def h(s) -> Tuple[str, str]: # no error + return 'a', 'b' +x: Tuple = () # E: Missing type parameters for generic type "Tuple" +[builtins fixtures/tuple.pyi] + +[case testDisallowAnyGenericsTupleWithNoTypeParamsGeneric] +# flags: --disallow-any-generics +from typing import Tuple, List + +def f(s: Tuple) -> None: pass # E: Missing type parameters for generic type "Tuple" +def g(s: List[Tuple]) -> None: pass # E: Missing type parameters for generic type "Tuple" +def h(s: List[Tuple[str, str]]) -> None: pass # no error +[builtins fixtures/list.pyi] + +[case testDisallowAnyGenericsTypeType] +# flags: --disallow-any-generics +from typing import Type, Any + +def f(s: Type[Any]) -> None: pass # no error +def g(s) -> Type: # E: Missing type parameters for generic type "Type" + return s +def h(s) -> Type[str]: # no error + return s +x: Type = g(0) # E: Missing type parameters for generic type "Type" + +[case testDisallowAnyGenericsAliasGenericType] +# flags: --disallow-any-generics +from typing import List + +L = List # no error + +def f(l: L) -> None: pass # E: Missing type parameters for generic type "L" +def g(l: L[str]) -> None: pass # no error +[builtins fixtures/list.pyi] + +[case testDisallowAnyGenericsGenericAlias] +# flags: --disallow-any-generics +from typing import TypeVar, Tuple + +T = TypeVar('T') +A = Tuple[T, str, T] + +def f(s: A) -> None: pass # E: Missing type parameters for generic type "A" +def g(s) -> A: # E: Missing type parameters for generic type "A" + return 'a', 'b', 1 +def h(s) -> A[str]: # no error + return 'a', 'b', 'c' +x: A = ('a', 'b', 1) # E: Missing type parameters for generic type "A" +[builtins fixtures/tuple.pyi] + +[case testDisallowAnyGenericsPlainList] +# flags: --disallow-any-generics +from typing import List + +def f(l: List) -> None: pass # E: Missing type parameters for generic type "List" +def g(l: List[str]) -> None: pass +def h(l: List[List]) -> None: pass # E: Missing type parameters for generic type "List" +def i(l: List[List[List[List]]]) -> None: pass # E: Missing type parameters for generic type "List" +def j() -> List: pass # E: Missing type parameters for generic type "List" + +x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +y: List = [] # E: Missing type parameters for generic type "List" +[builtins fixtures/list.pyi] + +[case testDisallowAnyGenericsPlainDict] +# flags: --disallow-any-generics +from typing import List, Dict + +def f(d: Dict) -> None: pass # E: Missing type parameters for generic type "Dict" +def g(d: Dict[str, Dict]) -> None: pass # E: Missing type parameters for generic type "Dict" +def h(d: List[Dict]) -> None: pass # E: Missing type parameters for generic type "Dict" + +d: Dict = {} # E: Missing type parameters for generic type "Dict" +[builtins fixtures/dict.pyi] + +[case testDisallowAnyGenericsPlainSet] +# flags: --disallow-any-generics +from typing import Set + +def f(s: Set) -> None: pass # E: Missing type parameters for generic type "Set" +def g(s: Set[Set]) -> None: pass # E: Missing type parameters for generic type "Set" + +s: Set = set() # E: Missing type parameters for generic type "Set" +[builtins fixtures/set.pyi] + +[case testDisallowAnyGenericsCustomGenericClass] +# flags: --disallow-any-generics +from typing import Generic, TypeVar, Any + +T = TypeVar('T') +class G(Generic[T]): pass + +def f() -> G: # E: Missing type parameters for generic type "G" + return G() + +x: G[Any] = G() # no error +y: G = x # E: Missing type parameters for generic type "G" + +[case testDisallowAnyGenericsForAliasesInRuntimeContext] +# flags: --disallow-any-generics +from typing import Any, TypeVar, Generic, Tuple + +T = TypeVar("T") +class G(Generic[T]): + @classmethod + def foo(cls) -> T: ... + +A = G[Tuple[T, T]] +A() # E: Missing type parameters for generic type "A" +A.foo() # E: Missing type parameters for generic type "A" + +B = G +B() +B.foo() + +def foo(x: Any) -> None: ... +foo(A) +foo(A.foo) +[builtins fixtures/classmethod.pyi] + +[case testDisallowSubclassingAny] +# flags: --config-file tmp/mypy.ini +import m +import y + +[file m.py] +from typing import Any + +x = None # type: Any + +class ShouldBeFine(x): ... + +[file y.py] +from typing import Any + +x = None # type: Any + +class ShouldNotBeFine(x): ... # E: Class cannot subclass "x" (has type "Any") + +[file mypy.ini] +\[mypy] +disallow_subclassing_any = True +\[mypy-m] +disallow_subclassing_any = False + + +[case testDisallowSubclassingAnyPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import m +import y + +[file m.py] +from typing import Any + +x = None # type: Any + +class ShouldBeFine(x): ... + +[file y.py] +from typing import Any + +x = None # type: Any + +class ShouldNotBeFine(x): ... # E: Class cannot subclass "x" (has type "Any") + +[file pyproject.toml] +\[tool.mypy] +disallow_subclassing_any = true +\[[tool.mypy.overrides]] +module = 'm' +disallow_subclassing_any = false + + +[case testNoImplicitOptionalPerModule] +# flags: --config-file tmp/mypy.ini +import m + +[file m.py] +def f(a: str = None) -> int: + return 0 + +[file mypy.ini] +\[mypy] +no_implicit_optional = True +\[mypy-m] +no_implicit_optional = False + + +[case testNoImplicitOptionalPerModulePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import m + +[file m.py] +def f(a: str = None) -> int: + return 0 + +[file pyproject.toml] +\[tool.mypy] +no_implicit_optional = true +\[[tool.mypy.overrides]] +module = 'm' +no_implicit_optional = false + +[case testDisableErrorCode] +# flags: --disable-error-code attr-defined +x = 'should be fine' +x.trim() + +[case testDisableDifferentErrorCode] +# flags: --disable-error-code name-defined --show-error-codes +x = 'should not be fine' +x.trim() # E: "str" has no attribute "trim" [attr-defined] + +[case testDisableMultipleErrorCode] +# flags: --disable-error-code attr-defined --disable-error-code return-value --show-error-codes +x = 'should be fine' +x.trim() + +def bad_return_type() -> str: + return None + +bad_return_type('no args taken!') # E: Too many arguments for "bad_return_type" [call-arg] + +[case testEnableErrorCode] +# flags: --disable-error-code attr-defined --enable-error-code attr-defined --show-error-codes +x = 'should be fine' +x.trim() # E: "str" has no attribute "trim" [attr-defined] + +[case testEnableDifferentErrorCode] +# flags: --disable-error-code attr-defined --enable-error-code name-defined --show-error-codes +x = 'should not be fine' +x.trim() +y.trim() # E: Name "y" is not defined [name-defined] + +[case testEnableMultipleErrorCode] +# flags: \ + --disable-error-code attr-defined \ + --disable-error-code return-value \ + --disable-error-code call-arg \ + --enable-error-code attr-defined \ + --enable-error-code return-value --show-error-codes +x = 'should be fine' +x.trim() # E: "str" has no attribute "trim" [attr-defined] + +def bad_return_type() -> str: + return None # E: Incompatible return value type (got "None", expected "str") [return-value] + +bad_return_type('no args taken!') + +[case testDisallowUntypedCallsArgType] +# flags: --disallow-untyped-calls +def f(x): + pass + +y = 1 +f(reveal_type(y)) # E: Call to untyped function "f" in typed context \ + # N: Revealed type is "builtins.int" + +[case testDisallowUntypedCallsAllowListFlags] +# flags: --disallow-untyped-calls --untyped-calls-exclude=foo --untyped-calls-exclude=bar.A +from foo import test_foo +from bar import A, B +from baz import test_baz +from foobar import bad + +test_foo(42) # OK +test_baz(42) # E: Call to untyped function "test_baz" in typed context +bad(42) # E: Call to untyped function "bad" in typed context + +a: A +b: B +a.meth() # OK +b.meth() # E: Call to untyped function "meth" in typed context +[file foo.py] +def test_foo(x): pass +[file foobar.py] +def bad(x): pass +[file bar.py] +class A: + def meth(self): pass +class B: + def meth(self): pass +[file baz.py] +def test_baz(x): pass + +[case testDisallowUntypedCallsAllowListConfig] +# flags: --config-file tmp/mypy.ini +from foo import test_foo +from bar import A, B +from baz import test_baz + +test_foo(42) # OK +test_baz(42) # E: Call to untyped function "test_baz" in typed context + +a: A +b: B +a.meth() # OK +b.meth() # E: Call to untyped function "meth" in typed context +[file foo.py] +def test_foo(x): pass +[file bar.py] +class A: + def meth(self): pass +class B: + def meth(self): pass +[file baz.py] +def test_baz(x): pass + +[file mypy.ini] +\[mypy] +disallow_untyped_calls = True +untyped_calls_exclude = foo, bar.A + +[case testPerModuleErrorCodes] +# flags: --config-file tmp/mypy.ini +import tests.foo +import bar +[file bar.py] +x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +[file tests/__init__.py] +[file tests/foo.py] +x = [] # OK +[file mypy.ini] +\[mypy] +strict = True + +\[mypy-tests.*] +allow_untyped_defs = True +allow_untyped_calls = True +disable_error_code = var-annotated + +[case testPerFileIgnoreErrors] +# flags: --config-file tmp/mypy.ini +import foo, bar +[file foo.py] +x: str = 5 +[file bar.py] +x: str = 5 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[file mypy.ini] +\[mypy] +\[mypy-foo] +ignore_errors = True + +[case testPerFileUntypedDefs] +# flags: --config-file tmp/mypy.ini +import x, y, z +[file x.py] +def f(a): ... # E: Function is missing a type annotation +def g(a: int) -> int: return f(a) +[file y.py] +def f(a): pass +def g(a: int) -> int: return f(a) +[file z.py] +def f(a): pass # E: Function is missing a type annotation +def g(a: int) -> int: return f(a) # E: Call to untyped function "f" in typed context +[file mypy.ini] +\[mypy] +disallow_untyped_defs = True +\[mypy-y] +disallow_untyped_defs = False +\[mypy-z] +disallow_untyped_calls = True + +[case testPerModuleErrorCodesOverride] +# flags: --config-file tmp/mypy.ini +import tests.foo +import bar +[file bar.py] +def foo() -> int: ... +if foo: ... # E: Function "foo" could always be true in boolean context +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) +[file tests/__init__.py] +[file tests/foo.py] +def foo() -> int: ... +if foo: ... # E: Function "foo" could always be true in boolean context +42 + "no" # type: ignore +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code, truthy-bool, used-before-def + +\[mypy-tests.*] +disable_error_code = ignore-without-code + +[case testShowErrorCodes] +# flags: --show-error-codes +x: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + +[case testHideErrorCodes] +# flags: --hide-error-codes +x: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testDisableBytearrayPromotion] +# flags: --disable-bytearray-promotion +def f(x: bytes) -> None: ... +f(bytearray(b"asdf")) # E: Argument 1 to "f" has incompatible type "bytearray"; expected "bytes" +f(memoryview(b"asdf")) +[builtins fixtures/primitives.pyi] + +[case testDisableMemoryviewPromotion] +# flags: --disable-memoryview-promotion +def f(x: bytes) -> None: ... +f(bytearray(b"asdf")) +f(memoryview(b"asdf")) # E: Argument 1 to "f" has incompatible type "memoryview"; expected "bytes" +[builtins fixtures/primitives.pyi] + +[case testDisableBytearrayMemoryviewPromotionStrictEquality] +# flags: --disable-bytearray-promotion --disable-memoryview-promotion --strict-equality +def f(x: bytes, y: bytearray, z: memoryview) -> None: + x == y + y == z + x == z + 97 in x + 97 in y + 97 in z + x in y + x in z +[builtins fixtures/primitives.pyi] + +[case testEnableBytearrayMemoryviewPromotionStrictEquality] +# flags: --strict-equality +def f(x: bytes, y: bytearray, z: memoryview) -> None: + x == y + y == z + x == z + 97 in x + 97 in y + 97 in z + x in y + x in z +[builtins fixtures/primitives.pyi] + +[case testNoCrashFollowImportsForStubs] +# flags: --config-file tmp/mypy.ini +{**{"x": "y"}} + +[file mypy.ini] +\[mypy] +follow_imports = skip +follow_imports_for_stubs = true +[builtins fixtures/dict.pyi] + +[case testReturnAnyLambda] +# flags: --warn-return-any +from typing import Any, Callable + +def cb(f: Callable[[int], int]) -> None: ... +a: Any +cb(lambda x: a) # OK + +fn = lambda x: a +cb(fn) + +[case testShowErrorCodeLinks] +# flags: --show-error-codes --show-error-code-links + +x: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] +list(1) # E: No overload variant of "list" matches argument type "int" [call-overload] \ + # N: Possible overload variants: \ + # N: def [T] __init__(self) -> List[T] \ + # N: def [T] __init__(self, x: Iterable[T]) -> List[T] \ + # N: See https://mypy.rtfd.io/en/stable/_refs.html#code-call-overload for more info +list(2) # E: No overload variant of "list" matches argument type "int" [call-overload] \ + # N: Possible overload variants: \ + # N: def [T] __init__(self) -> List[T] \ + # N: def [T] __init__(self, x: Iterable[T]) -> List[T] +[builtins fixtures/list.pyi] + +[case testNestedGenericInAliasDisallow] +# flags: --disallow-any-generics +from typing import TypeVar, Generic, List, Union + +class C(Generic[T]): ... + +A = Union[C, List] # E: Missing type parameters for generic type "C" \ + # E: Missing type parameters for generic type "List" +[builtins fixtures/list.pyi] + +[case testNestedGenericInAliasAllow] +# flags: --allow-any-generics +from typing import TypeVar, Generic, List, Union + +class C(Generic[T]): ... + +A = Union[C, List] # OK +[builtins fixtures/list.pyi] + +[case testNotesOnlyResultInExitSuccess] +-- check_untyped_defs is False by default. +def f(): + x: int = "no" # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs diff --git a/test-data/unit/check-formatting.test b/test-data/unit/check-formatting.test new file mode 100644 index 000000000000..83ae9b526f22 --- /dev/null +++ b/test-data/unit/check-formatting.test @@ -0,0 +1,632 @@ + +-- String interpolation +-- -------------------- + +[case testStringInterpolationType] +from typing import Tuple +i: int +f: float +s: str +t: Tuple[int] +'%d' % i +'%f' % f +'%s' % s +'%d' % (f,) +'%d' % (s,) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float, SupportsInt]") +'%d' % t +'%d' % s # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float, SupportsInt]") +'%f' % s # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float, SupportsFloat]") +'%x' % f # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "int") +'%i' % f +'%o' % f # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "int") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationSAcceptsAnyType] +from typing import Any +i: int +o: object +s: str +'%s %s %s' % (i, o, s) +[builtins fixtures/primitives.pyi] + +[case testStringInterpolationSBytesVsStrErrorPy3] +xb: bytes +xs: str + +'%s' % xs # OK +'%s' % xb # E: If x = b'abc' then "%s" % x produces "b'abc'", not "abc". If this is desired behavior use "%r" % x. Otherwise, decode the bytes +'%(name)s' % {'name': b'value'} # E: If x = b'abc' then "%s" % x produces "b'abc'", not "abc". If this is desired behavior use "%r" % x. Otherwise, decode the bytes +[builtins fixtures/primitives.pyi] + +[case testStringInterpolationCount] +'%d %d' % 1 # E: Not enough arguments for format string +'%d %d' % (1, 2) +'%d %d' % (1, 2, 3) # E: Not all arguments converted during string formatting +t = 1, 's' +'%d %s' % t +'%s %d' % t # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float, SupportsInt]") +'%d' % t # E: Not all arguments converted during string formatting +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationWithAnyType] +from typing import Any +a = None # type: Any +'%d %d' % a +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationInvalidPlaceholder] +'%W' % 1 # E: Unsupported format character "W" +'%b' % 1 # E: Format character "b" is only supported on bytes patterns + +[case testStringInterpolationWidth] +'%2f' % 3.14 +'%*f' % 3.14 # E: Not enough arguments for format string +'%*f' % (4, 3.14) +'%*f' % (1.1, 3.14) # E: * wants int +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationPrecision] +'%.2f' % 3.14 +'%.*f' % 3.14 # E: Not enough arguments for format string +'%.*f' % (4, 3.14) +'%.*f' % (1.1, 3.14) # E: * wants int +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationWidthAndPrecision] +'%4.2f' % 3.14 +'%4.*f' % 3.14 # E: Not enough arguments for format string +'%*.2f' % 3.14 # E: Not enough arguments for format string +'%*.*f' % 3.14 # E: Not enough arguments for format string +'%*.*f' % (4, 2, 3.14) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationFlagsAndLengthModifiers] +'%04hd' % 1 +'%-.4ld' % 1 +'%+*Ld' % (1, 1) +'% .*ld' % (1, 1) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationDoublePercentage] +'%% %d' % 1 +'%3% %d' % 1 +'%*%' % 1 +'%*% %d' % 1 # E: Not enough arguments for format string +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationC] +'%c' % 1 +'%c' % 1.0 # E: "%c" requires int or char (expression has type "float") +'%c' % 's' +'%c' % '' # E: "%c" requires int or char +'%c' % 'ab' # E: "%c" requires int or char +'%c' % b'a' # E: "%c" requires int or char (expression has type "bytes") +'%c' % b'' # E: "%c" requires int or char (expression has type "bytes") +'%c' % b'ab' # E: "%c" requires int or char (expression has type "bytes") +[builtins fixtures/primitives.pyi] + +[case testStringInterpolationMappingTypes] +'%(a)d %(b)s' % {'a': 1, 'b': 's'} +'%(a)d %(b)s' % {'a': 's', 'b': 1} # E: Incompatible types in string interpolation (expression has type "str", placeholder with key 'a' has type "Union[int, float, SupportsInt]") +b'%(x)s' % {b'x': b'data'} +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationMappingKeys] +'%()d' % {'': 2} +'%(a)d' % {'a': 1, 'b': 2, 'c': 3} +'%(q)d' % {'a': 1, 'b': 2, 'c': 3} # E: Key "q" not found in mapping +'%(a)d %%' % {'a': 1} +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationMappingDictTypes] +from typing import Any, Dict, Iterable + +class StringThing: + def keys(self) -> Iterable[str]: + ... + def __getitem__(self, __key: str) -> str: + ... + +class BytesThing: + def keys(self) -> Iterable[bytes]: + ... + def __getitem__(self, __key: bytes) -> str: + ... + +a: Any +ds: Dict[str, int] +do: Dict[object, int] +di: Dict[int, int] +'%(a)' % 1 # E: Format requires a mapping (expression has type "int", expected type for mapping is "SupportsKeysAndGetItem[str, Any]") +'%()d' % a +'%()d' % ds +'%()d' % do # E: Format requires a mapping (expression has type "Dict[object, int]", expected type for mapping is "SupportsKeysAndGetItem[str, Any]") +b'%()d' % ds # E: Format requires a mapping (expression has type "Dict[str, int]", expected type for mapping is "SupportsKeysAndGetItem[bytes, Any]") +'%()s' % StringThing() +b'%()s' % BytesThing() +[builtins fixtures/primitives.pyi] + +[case testStringInterpolationMappingInvalidSpecifiers] +'%(a)d %d' % 1 # E: String interpolation mixes specifier with and without mapping keys +'%(b)*d' % 1 # E: String interpolation contains both stars and mapping keys +'%(b).*d' % 1 # E: String interpolation contains both stars and mapping keys + +[case testStringInterpolationMappingFlagsAndLengthModifiers] +'%(a)1d' % {'a': 1} +'%(a).1d' % {'a': 1} +'%(a)#1.1ld' % {'a': 1} +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationFloatPrecision] +'%.f' % 1.2 +'%.3f' % 1.2 +'%.f' % 'x' +'%.3f' % 'x' +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] +[out] +main:3: error: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float, SupportsFloat]") +main:4: error: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float, SupportsFloat]") + +[case testStringInterpolationSpaceKey] +'%( )s' % {' ': 'foo'} + +[case testStringInterpolationStarArgs] +x = (1, 2) +"%d%d" % (*x,) +[typing fixtures/typing-medium.pyi] +[builtins fixtures/tuple.pyi] + +[case testStringInterpolationVariableLengthTuple] +from typing import Tuple +def f(t: Tuple[int, ...]) -> None: + '%d %d' % t + '%d %d %d' % t +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testStringInterpolationUnionType] +from typing import Tuple, Union +a: Union[Tuple[int, str], Tuple[str, int]] = ('A', 1) +'%s %s' % a +'%s' % a # E: Not all arguments converted during string formatting + +b: Union[Tuple[int, str], Tuple[int, int], Tuple[str, int]] = ('A', 1) +'%s %s' % b +'%s %s %s' % b # E: Not enough arguments for format string + +c: Union[Tuple[str, int], Tuple[str, int, str]] = ('A', 1) +'%s %s' % c # E: Not all arguments converted during string formatting +[builtins fixtures/tuple.pyi] + +[case testStringInterpolationIterableType] +from typing import Sequence, List, Tuple, Iterable + +t1: Sequence[str] = ('A', 'B') +t2: List[str] = ['A', 'B'] +t3: Tuple[str, ...] = ('A', 'B') +t4: Tuple[str, str] = ('A', 'B') +t5: Iterable[str] = ('A', 'B') +'%s %s' % t1 +'%s %s' % t2 +'%s %s' % t3 +'%s %s %s' % t3 +'%s %s' % t4 +'%s %s %s' % t4 # E: Not enough arguments for format string +'%s %s' % t5 +[builtins fixtures/tuple.pyi] + + +-- Bytes interpolation +-- -------------------- + +[case testBytesInterpolation] +b'%b' % 1 # E: Incompatible types in string interpolation (expression has type "int", placeholder has type "bytes") +b'%b' % b'1' +b'%a' % 3 + +[case testBytesInterpolationC] +b'%c' % 1 +b'%c' % 1.0 # E: "%c" requires an integer in range(256) or a single byte (expression has type "float") +b'%c' % 's' # E: "%c" requires an integer in range(256) or a single byte (expression has type "str") +b'%c' % '' # E: "%c" requires an integer in range(256) or a single byte (expression has type "str") +b'%c' % 'ab' # E: "%c" requires an integer in range(256) or a single byte (expression has type "str") +b'%c' % b'a' +b'%c' % b'' # E: "%c" requires an integer in range(256) or a single byte +b'%c' % b'aa' # E: "%c" requires an integer in range(256) or a single byte +[builtins fixtures/primitives.pyi] + +[case testByteByteInterpolation] +def foo(a: bytes, b: bytes): + b'%s:%s' % (a, b) +foo(b'a', b'b') == b'a:b' +[builtins fixtures/tuple.pyi] + +[case testBytePercentInterpolationSupported] +b'%s' % (b'xyz',) +b'%(name)s' % {'name': b'jane'} # E: Dictionary keys in bytes formatting must be bytes, not strings +b'%(name)s' % {b'name': 'jane'} # E: On Python 3 b'%s' requires bytes, not string +b'%c' % (123) +[builtins fixtures/tuple.pyi] + + +-- str.format() calls +-- ------------------ + + +[case testFormatCallParseErrors] +'}'.format() # E: Invalid conversion specifier in format string: unexpected } +'{'.format() # E: Invalid conversion specifier in format string: unmatched { + +'}}'.format() # OK +'{{'.format() # OK + +'{{}}}'.format() # E: Invalid conversion specifier in format string: unexpected } +'{{{}}'.format() # E: Invalid conversion specifier in format string: unexpected } + +'{}}{{}'.format() # E: Invalid conversion specifier in format string: unexpected } +'{{{}:{}}}'.format(0) # E: Cannot find replacement for positional format specifier 1 +[builtins fixtures/primitives.pyi] + +[case testFormatCallValidationErrors] +'{!}}'.format(0) # E: Invalid conversion specifier in format string: unexpected } +'{!x}'.format(0) # E: Invalid conversion type "x", must be one of "r", "s" or "a" +'{!:}'.format(0) # E: Invalid conversion specifier in format string + +'{{}:s}'.format(0) # E: Invalid conversion specifier in format string: unexpected } +'{{}.attr}'.format(0) # E: Invalid conversion specifier in format string: unexpected } +'{{}[key]}'.format(0) # E: Invalid conversion specifier in format string: unexpected } + +'{ {}:s}'.format() # E: Conversion value must not contain { or } +'{ {}.attr}'.format() # E: Conversion value must not contain { or } +'{ {}[key]}'.format() # E: Conversion value must not contain { or } +[builtins fixtures/primitives.pyi] + +[case testFormatCallEscaping] +'{}'.format() # E: Cannot find replacement for positional format specifier 0 +'{}'.format(0) # OK + +'{{}}'.format() # OK +'{{}}'.format(0) # E: Not all arguments converted during string formatting + +'{{{}}}'.format() # E: Cannot find replacement for positional format specifier 0 +'{{{}}}'.format(0) # OK + +'{{}} {} {{}}'.format(0) # OK +'{{}} {:d} {{}} {:d}'.format('a', 'b') # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") + +'foo({}, {}) == {{}} ({{}} expected)'.format(0) # E: Cannot find replacement for positional format specifier 1 +'foo({}, {}) == {{}} ({{}} expected)'.format(0, 1) # OK +'foo({}, {}) == {{}} ({{}} expected)'.format(0, 1, 2) # E: Not all arguments converted during string formatting +[builtins fixtures/primitives.pyi] + +[case testFormatCallNestedFormats] +'{:{}{}}'.format(42, '*') # E: Cannot find replacement for positional format specifier 2 +'{:{}{}}'.format(42, '*', '^') # OK +'{:{}{}}'.format(42, '*', '^', 0) # E: Not all arguments converted during string formatting + +# NOTE: we don't check format specifiers that contain { or } at all +'{:{{}}}'.format() # E: Cannot find replacement for positional format specifier 0 + +'{:{:{}}}'.format() # E: Formatting nesting must be at most two levels deep +'{:{{}:{}}}'.format() # E: Invalid conversion specifier in format string: unexpected } + +'{!s:{fill:d}{align}}'.format(42, fill='*', align='^') # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +[builtins fixtures/primitives.pyi] + +[case testFormatCallAutoNumbering] +'{}, {{}}, {0}'.format() # E: Cannot combine automatic field numbering and manual field specification +'{0}, {1}, {}'.format() # E: Cannot combine automatic field numbering and manual field specification + +'{0}, {1}, {0}'.format(1, 2, 3) # E: Not all arguments converted during string formatting +'{}, {other:+d}, {}'.format(1, 2, other='no') # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +'{0}, {other}, {}'.format() # E: Cannot combine automatic field numbering and manual field specification + +'{:{}}, {:{:.5d}{}}'.format(1, 2, 3, 'a', 5) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +[builtins fixtures/primitives.pyi] + +[case testFormatCallMatchingPositional] +'{}'.format(positional='no') # E: Cannot find replacement for positional format specifier 0 \ + # E: Not all arguments converted during string formatting +'{.x}, {}, {}'.format(1, 'two', 'three') # E: "int" has no attribute "x" +'Reverse {2.x}, {1}, {0}'.format(1, 2, 'three') # E: "str" has no attribute "x" +''.format(1, 2) # E: Not all arguments converted during string formatting +[builtins fixtures/primitives.pyi] + +[case testFormatCallMatchingNamed] +'{named}'.format(0) # E: Cannot find replacement for named format specifier "named" \ + # E: Not all arguments converted during string formatting +'{one.x}, {two}'.format(one=1, two='two') # E: "int" has no attribute "x" +'{one}, {two}, {.x}'.format(1, one='two', two='three') # E: "int" has no attribute "x" +''.format(stuff='yes') # E: Not all arguments converted during string formatting +[builtins fixtures/primitives.pyi] + +[case testFormatCallMatchingVarArg] +from typing import List +args: List[int] = [] +'{}, {}'.format(1, 2, *args) # Don't flag this because args may be empty + +strings: List[str] +'{:d}, {[0].x}'.format(*strings) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") \ + # E: "str" has no attribute "x" +# TODO: this is a runtime error, but error message is confusing +'{[0][:]:d}'.format(*strings) # E: Syntax error in format specifier "0[0][" +[builtins fixtures/primitives.pyi] + +[case testFormatCallMatchingKwArg] +from typing import Dict +kwargs: Dict[str, str] = {} +'{one}, {two}'.format(one=1, two=2, **kwargs) # Don't flag this because args may be empty + +'{stuff:.3d}'.format(**kwargs) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +'{stuff[0]:f}, {other}'.format(**kwargs) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float]") +'{stuff[0]:c}'.format(**kwargs) +[builtins fixtures/primitives.pyi] + +[case testFormatCallCustomFormatSpec] +from typing import Union +class Bad: + ... +class Good: + def __format__(self, spec: str) -> str: ... + +'{:OMG}'.format(Good()) +'{:OMG}'.format(Bad()) # E: Unrecognized format specification "OMG" +'{!s:OMG}'.format(Good()) # E: Unrecognized format specification "OMG" +'{:{}OMG{}}'.format(Bad(), 'too', 'dynamic') + +x: Union[Good, Bad] +'{:OMG}'.format(x) # E: Unrecognized format specification "OMG" +[builtins fixtures/primitives.pyi] + +[case testFormatCallFormatTypes] +'{:x}'.format(42) +'{:E}'.format(42) +'{:g}'.format(42) +'{:x}'.format('no') # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +'{:E}'.format('no') # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float]") +'{:g}'.format('no') # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float]") +'{:n}'.format(3.14) +'{:d}'.format(3.14) # E: Incompatible types in string interpolation (expression has type "float", placeholder has type "int") + +'{:s}'.format(42) +'{:s}'.format('yes') + +'{:z}'.format('what') # E: Unsupported format character "z" +'{:Z}'.format('what') # E: Unsupported format character "Z" +[builtins fixtures/primitives.pyi] + +[case testFormatCallFormatTypesChar] +'{:c}'.format(42) +'{:c}'.format('no') # E: ":c" requires int or char +'{:c}'.format('c') + +class C: + ... +'{:c}'.format(C()) # E: Incompatible types in string interpolation (expression has type "C", placeholder has type "Union[int, str]") +x: str +'{:c}'.format(x) +[builtins fixtures/primitives.pyi] + +[case testFormatCallFormatTypesCustomFormat] +from typing import Union +class Bad: + ... +class Good: + def __format__(self, spec: str) -> str: ... + +x: Union[Good, Bad] +y: Union[Good, int] +z: Union[Bad, int] +t: Union[Good, str] +'{:d}'.format(x) # E: Incompatible types in string interpolation (expression has type "Bad", placeholder has type "int") +'{:d}'.format(y) +'{:d}'.format(z) # E: Incompatible types in string interpolation (expression has type "Bad", placeholder has type "int") +'{:d}'.format(t) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +[builtins fixtures/primitives.pyi] + +[case testFormatCallFormatTypesBytes] +from typing import Union, TypeVar, NewType, Generic + +A = TypeVar('A', str, bytes) +B = TypeVar('B', bound=bytes) + +x: Union[str, bytes] +a: str +b: bytes + +N = NewType('N', bytes) +n: N + +'{}'.format(a) +'{}'.format(b) # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes +'{}'.format(x) # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes +'{}'.format(n) # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes + +f'{b}' # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes +f'{x}' # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes +f'{n}' # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes + +class C(Generic[B]): + x: B + def meth(self) -> None: + '{}'.format(self.x) # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes + +def func(x: A) -> A: + '{}'.format(x) # E: If x = b'abc' then f"{x}" or "{}".format(x) produces "b'abc'", not "abc". If this is desired behavior, use f"{x!r}" or "{!r}".format(x). Otherwise, decode the bytes + return x + +'{!r}'.format(a) +'{!r}'.format(b) +'{!r}'.format(x) +'{!r}'.format(n) +f'{a}' +f'{a!r}' +f'{b!r}' +f'{x!r}' +f'{n!r}' + +class D(bytes): + def __str__(self) -> str: + return "overrides __str__ of bytes" + +'{}'.format(D()) +[builtins fixtures/primitives.pyi] + +[case testNoSpuriousFormattingErrorsDuringFailedOverlodMatch] +from typing import overload, Callable + +@overload +def sub(pattern: str, repl: Callable[[str], str]) -> str: ... +@overload +def sub(pattern: bytes, repl: Callable[[bytes], bytes]) -> bytes: ... +def sub(pattern: object, repl: object) -> object: + pass + +def better_snakecase(text: str) -> str: + # Mypy used to emit a spurious error here + # warning about interpolating bytes into an f-string: + text = sub(r"([A-Z])([A-Z]+)([A-Z](?:[^A-Z]|$))", lambda match: f"{match}") + return text +[builtins fixtures/primitives.pyi] + +[case testFormatCallFinal] +from typing_extensions import Final + +FMT: Final = '{.x}, {:{:d}}' + +FMT.format(1, 2, 'no') # E: "int" has no attribute "x" \ + # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +[builtins fixtures/primitives.pyi] + +[case testFormatCallFinalChar] +from typing_extensions import Final + +GOOD: Final = 'c' +BAD: Final = 'no' +OK: Final[str] = '...' + +'{:c}'.format(GOOD) +'{:c}'.format(BAD) # E: ":c" requires int or char +'{:c}'.format(OK) +[builtins fixtures/primitives.pyi] + +[case testFormatCallForcedConversions] +'{!r}'.format(42) +'{!s}'.format(42) +'{!s:d}'.format(42) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "int") +'{!s:s}'.format('OK') +'{} and {!x}'.format(0, 1) # E: Invalid conversion type "x", must be one of "r", "s" or "a" +[builtins fixtures/primitives.pyi] + +[case testFormatCallAccessorsBasic] +from typing import Any +x: Any + +'{.x:{[0]}}'.format('yes', 42) # E: "str" has no attribute "x" \ + # E: Value of type "int" is not indexable + +'{.1+}'.format(x) # E: Syntax error in format specifier "0.1+" +'{name.x[x]()[x]:.2f}'.format(name=x) # E: Only index and member expressions are allowed in format field accessors; got "name.x[x]()[x]" +[builtins fixtures/primitives.pyi] + +[case testFormatCallAccessorsIndices] +from typing_extensions import TypedDict + +class User(TypedDict): + id: int + name: str + +u: User +'{user[name]:.3f}'.format(user=u) # E: Incompatible types in string interpolation (expression has type "str", placeholder has type "Union[int, float]") + +def f() -> str: ... +'{[f()]}'.format(u) # E: Invalid index expression in format field accessor "[f()]" +[builtins fixtures/primitives.pyi] + +[case testFormatCallFlags] +from typing import Union + +class Good: + def __format__(self, spec: str) -> str: ... + +'{:#}'.format(42) + +'{:#}'.format('no') # E: Numeric flags are only allowed for numeric types +'{!s:#}'.format(42) # E: Numeric flags are only allowed for numeric types + +'{:#s}'.format(42) # E: Numeric flags are only allowed for numeric types +'{:+s}'.format(42) # E: Numeric flags are only allowed for numeric types + +'{:+d}'.format(42) +'{:#d}'.format(42) + +x: Union[float, Good] +'{:+f}'.format(x) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testFormatCallSpecialCases] +'{:08b}'.format(int('3')) + +class S: + def __int__(self) -> int: ... + +'{:+d}'.format(S()) # E: Incompatible types in string interpolation (expression has type "S", placeholder has type "int") +'%d' % S() # This is OK however +'{:%}'.format(0.001) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testEnumWithStringToFormatValue] +from enum import Enum + +class Responses(str, Enum): + TEMPLATED = 'insert {} here' + TEMPLATED_WITH_KW = 'insert {value} here' + NORMAL = 'something' + +Responses.TEMPLATED.format(42) +Responses.TEMPLATED_WITH_KW.format(value=42) +Responses.TEMPLATED.format() # E: Cannot find replacement for positional format specifier 0 +Responses.TEMPLATED_WITH_KW.format() # E: Cannot find replacement for named format specifier "value" +Responses.NORMAL.format(42) # E: Not all arguments converted during string formatting +Responses.NORMAL.format(value=42) # E: Not all arguments converted during string formatting +[builtins fixtures/primitives.pyi] + +[case testNonStringEnumToFormatValue] +from enum import Enum + +class Responses(Enum): + TEMPLATED = 'insert {value} here' + +Responses.TEMPLATED.format(value=42) # E: "Responses" has no attribute "format" +[builtins fixtures/primitives.pyi] + +[case testStrEnumWithStringToFormatValue] +# flags: --python-version 3.11 +from enum import StrEnum + +class Responses(StrEnum): + TEMPLATED = 'insert {} here' + TEMPLATED_WITH_KW = 'insert {value} here' + NORMAL = 'something' + +Responses.TEMPLATED.format(42) +Responses.TEMPLATED_WITH_KW.format(value=42) +Responses.TEMPLATED.format() # E: Cannot find replacement for positional format specifier 0 +Responses.TEMPLATED_WITH_KW.format() # E: Cannot find replacement for named format specifier "value" +Responses.NORMAL.format(42) # E: Not all arguments converted during string formatting +Responses.NORMAL.format(value=42) # E: Not all arguments converted during string formatting +[builtins fixtures/primitives.pyi] diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test new file mode 100644 index 000000000000..b8a02a1ec7d4 --- /dev/null +++ b/test-data/unit/check-functions.test @@ -0,0 +1,3465 @@ +-- Test cases for the type checker related to functions, function types and +-- calls. + +-- See also check-varargs.test. + + +-- Callable type basics +-- -------------------- + + +[case testCallingVariableWithFunctionType] +from typing import Callable +f: Callable[[A], B] +a: A +b: B +if int(): + a = f(a) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = f(b) # E: Argument 1 has incompatible type "B"; expected "A" +if int(): + b = f() # E: Too few arguments +if int(): + b = f(a, a) # E: Too many arguments +if int(): + b = f(a) + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testKeywordOnlyArgumentOrderInsensitivity] +import typing + +class A(object): + def f(self, *, a: int, b: str) -> None: pass + +class B(A): + def f(self, *, b: str, a: int) -> None: pass + +class C(A): + def f(self, *, b: int, a: str) -> None: pass # Fail +[out] +main:10: error: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "str" +main:10: note: This violates the Liskov substitution principle +main:10: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:10: error: Argument 2 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" + +[case testPositionalOverridingArgumentNameInsensitivity] +import typing + +class A(object): + def f(self, a: int, b: str) -> None: pass + +class B(A): + def f(self, b: str, a: int) -> None: pass # E: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides \ + # E: Argument 2 of "f" is incompatible with supertype "A"; supertype defines the argument type as "str" + +class C(A): + def f(self, foo: int, bar: str) -> None: pass + + +[case testPositionalOverridingArgumentNamesCheckedWhenMismatchingPos] +import typing + +class A(object): + def f(self, a: int, b: str) -> None: pass + +class B(A): + def f(self, b: int, a: str) -> None: pass # Fail +[out] +main:7: error: Signature of "f" incompatible with supertype "A" +main:7: note: Superclass: +main:7: note: def f(self, a: int, b: str) -> None +main:7: note: Subclass: +main:7: note: def f(self, b: int, a: str) -> None + +[case testSubtypingFunctionTypes] +from typing import Callable + +class A: pass +class B(A): pass + +f: Callable[[B], A] +g: Callable[[A], A] # subtype of f +h: Callable[[B], B] # subtype of f +if int(): + g = h # E: Incompatible types in assignment (expression has type "Callable[[B], B]", variable has type "Callable[[A], A]") +if int(): + h = f # E: Incompatible types in assignment (expression has type "Callable[[B], A]", variable has type "Callable[[B], B]") +if int(): + h = g # E: Incompatible types in assignment (expression has type "Callable[[A], A]", variable has type "Callable[[B], B]") +if int(): + g = f # E: Incompatible types in assignment (expression has type "Callable[[B], A]", variable has type "Callable[[A], A]") +if int(): + f = g +if int(): + f = h +if int(): + f = f +if int(): + g = g +if int(): + h = h + +[case testSubtypingFunctionsDoubleCorrespondence] + +def l(x) -> None: ... +def r(__, *, x) -> None: ... +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Any, NamedArg(Any, 'x')], None]") + +[case testSubtypingFunctionsRequiredLeftArgNotPresent] + +def l(x, y) -> None: ... +def r(x) -> None: ... +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], None]", variable has type "Callable[[Any], None]") + +[case testSubtypingFunctionsImplicitNames] +from typing import Any + +def f(a, b): pass +def g(c: Any, d: Any) -> Any: pass + +ff = f +gg = g + +gg = f +ff = g + +[case testSubtypingFunctionsDefaultsNames] +from typing import Callable + +def f(a: int, b: str) -> None: pass +f_nonames: Callable[[int, str], None] +def g(a: int, b: str = "") -> None: pass +def h(aa: int, b: str = "") -> None: pass + +ff_nonames = f_nonames +ff = f +gg = g +hh = h + +if int(): + ff = gg +if int(): + ff_nonames = ff +if int(): + ff_nonames = f_nonames # reset +if int(): + ff = ff_nonames # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") +if int(): + ff = f # reset +if int(): + gg = ff # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]", variable has type "Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]") +if int(): + gg = hh # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'aa'), DefaultArg(str, 'b')], None]", variable has type "Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]") + +[case testSubtypingFunctionsArgsKwargs] +from typing import Any, Callable + +def everything(*args: Any, **kwargs: Any) -> None: pass +everywhere: Callable[..., None] + +def specific_1(a: int, b: str) -> None: pass +def specific_2(a: int, *, b: str) -> None: pass + +ss_1 = specific_1 +ss_2 = specific_2 +ee_def = everything +ee_var = everywhere + +if int(): + ss_1 = ee_def +if int(): + ss_1 = specific_1 +if int(): + ss_2 = ee_def +if int(): + ss_2 = specific_2 +if int(): + ee_def = everywhere +if int(): + ee_def = everything +if int(): + ee_var = everything +if int(): + ee_var = everywhere + +if int(): + ee_var = specific_1 +if int(): + ee_def = specific_1 + +[builtins fixtures/dict.pyi] + +[case testSubtypingFunctionsDecorated] +from typing import Any + +# untyped decorator +def deco(f): pass + +class A: + @deco + def f(self) -> Any: + pass + +class B(A): + @deco + def f(self) -> Any: + pass + +[builtins fixtures/list.pyi] + + +[case testLackOfNames] +def f(__a: int, __b: str) -> None: pass +def g(a: int, b: str) -> None: pass + +ff = f +gg = g + +if int(): + ff = g +if int(): + gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + +[case testLackOfNamesFastparse] +def f(__a: int, __b: str) -> None: pass +def g(a: int, b: str) -> None: pass + +ff = f +gg = g + +if int(): + ff = g +if int(): + gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + +[case testFunctionTypeCompatibilityWithOtherTypes] +# flags: --no-strict-optional +from typing import Callable +f = None # type: Callable[[], None] +a, o = None, None # type: (A, object) +if int(): + a = f # E: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "A") +if int(): + f = a # E: Incompatible types in assignment (expression has type "A", variable has type "Callable[[], None]") +if int(): + f = o # E: Incompatible types in assignment (expression has type "object", variable has type "Callable[[], None]") +if int(): + f = f() # E: Function does not return a value (it only ever returns None) + +if int(): + f = f +if int(): + f = None +if int(): + o = f + +class A: pass +[builtins fixtures/tuple.pyi] + +[case testReturnEmptyTuple] +from typing import Tuple +def f(x): # type: (int) -> () # E: Syntax error in type annotation \ + # N: Suggestion: Use Tuple[()] instead of () for an empty tuple, or None for a function without a return value + pass + +def g(x: int) -> Tuple[()]: + pass +[builtins fixtures/tuple.pyi] + +[case testFunctionSubtypingWithVoid] +from typing import Callable +f: Callable[[], None] +g: Callable[[], object] +if int(): + f = g # E: Incompatible types in assignment (expression has type "Callable[[], object]", variable has type "Callable[[], None]") +if int(): + g = f # OK + +if int(): + f = f +if int(): + g = g + +[case testFunctionSubtypingWithMultipleArgs] +from typing import Callable +f: Callable[[A, A], None] +g: Callable[[A, B], None] +h: Callable[[B, B], None] +if int(): + f = g # E: Incompatible types in assignment (expression has type "Callable[[A, B], None]", variable has type "Callable[[A, A], None]") +if int(): + f = h # E: Incompatible types in assignment (expression has type "Callable[[B, B], None]", variable has type "Callable[[A, A], None]") +if int(): + g = h # E: Incompatible types in assignment (expression has type "Callable[[B, B], None]", variable has type "Callable[[A, B], None]") +if int(): + g = f +if int(): + h = f +if int(): + h = g +if int(): + f = f +if int(): + g = g +if int(): + h = h + +class A: pass +class B(A): pass + +[case testFunctionTypesWithDifferentArgumentCounts] +from typing import Callable +f: Callable[[], None] +g: Callable[[A], None] +h: Callable[[A, A], None] + +if int(): + f = g # E: Incompatible types in assignment (expression has type "Callable[[A], None]", variable has type "Callable[[], None]") +if int(): + f = h # E: Incompatible types in assignment (expression has type "Callable[[A, A], None]", variable has type "Callable[[], None]") +if int(): + h = f # E: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "Callable[[A, A], None]") +if int(): + h = g # E: Incompatible types in assignment (expression has type "Callable[[A], None]", variable has type "Callable[[A, A], None]") + +if int(): + f = f +if int(): + g = g +if int(): + h = h + +class A: pass +[out] + +[case testCompatibilityOfSimpleTypeObjectWithStdType] +class A: + def __init__(self, a: 'A') -> None: pass + +def f() -> None: pass + +t: type +a: A + +if int(): + a = A # E: Incompatible types in assignment (expression has type "Type[A]", variable has type "A") +if int(): + t = f # E: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "type") +if int(): + t = A + +[case testFunctionTypesWithOverloads] +from foo import * +[file foo.pyi] +from typing import Callable, overload +f: Callable[[AA], A] +g: Callable[[B], B] +h: Callable[[A], AA] + +if int(): + h = i # E: Incompatible types in assignment (expression has type overloaded function, variable has type "Callable[[A], AA]") +if int(): + f = j + +if int(): + f = i +if int(): + g = i +if int(): + g = j + +class A: pass +class AA(A): pass + +class B: pass + +@overload +def i(x: AA) -> A: + pass +@overload +def i(x: B) -> B: + pass + +@overload +def j(x: B) -> B: + pass +@overload +def j(x: A) -> AA: + pass + +[case testOverloadWithThreeItems] +from foo import * +[file foo.pyi] +from typing import Callable, overload +g1: Callable[[A], A] +g2: Callable[[B], B] +g3: Callable[[C], C] +g4: Callable[[A], B] +a: A +b: B +c: C + +if int(): + b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = f(c) # E: Incompatible types in assignment (expression has type "C", variable has type "B") +if int(): + g4 = f # E: Incompatible types in assignment (expression has type overloaded function, variable has type "Callable[[A], B]") + +if int(): + g1 = f +if int(): + g2 = f +if int(): + g3 = f +if int(): + a = f(a) +if int(): + b = f(b) +if int(): + c = f(c) + +class A: pass +class B: pass +class C: pass + +@overload +def f(x: A) -> A: pass +@overload +def f(x: B) -> B: pass +@overload +def f(x: C) -> C: pass +[builtins fixtures/tuple.pyi] + +[case testInferConstraintsUnequalLengths] +from typing import Any, Callable, List +def f(fields: List[Callable[[Any], Any]]): pass +class C: pass +f([C]) # E: List item 0 has incompatible type "Type[C]"; expected "Callable[[Any], Any]" +class D: + def __init__(self, a, b): pass +f([D]) # E: List item 0 has incompatible type "Type[D]"; expected "Callable[[Any], Any]" +[builtins fixtures/list.pyi] + +[case testSubtypingTypeTypeAsCallable] +from typing import Callable, Type +class A: pass +x: Callable[..., A] +y: Type[A] +x = y + +[case testSubtypingCallableAsTypeType] +from typing import Callable, Type +class A: pass +x: Callable[..., A] +y: Type[A] +if int(): + y = x # E: Incompatible types in assignment (expression has type "Callable[..., A]", variable has type "Type[A]") + +-- Default argument values +-- ----------------------- + + +[case testCallingFunctionsWithDefaultArgumentValues] +# flags: --implicit-optional --no-strict-optional +class A: pass +class AA(A): pass +class B: pass + +def f(x: 'A' = None) -> 'B': pass + +a, b = None, None # type: (A, B) +if int(): + a = f() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = f(b) # E: Argument 1 to "f" has incompatible type "B"; expected "Optional[A]" +if int(): + b = f(a, a) # E: Too many arguments for "f" + +if int(): + b = f() +if int(): + b = f(a) +if int(): + b = f(AA()) + +[builtins fixtures/tuple.pyi] + +[case testDefaultArgumentExpressions] +import typing +class B: pass +class A: pass + +def f(x: 'A' = A()) -> None: + b = x # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = x # type: A +[out] + +[case testDefaultArgumentExpressions2] +import typing +class B: pass +class A: pass + +def f(x: 'A' = B()) -> None: # E: Incompatible default for argument "x" (default has type "B", argument has type "A") + b = x # type: B # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = x # type: A +[case testDefaultArgumentExpressionsGeneric] +from typing import TypeVar +T = TypeVar('T', bound='A') + +class B: pass +class A: pass + +def f(x: T = B()) -> None: # E: Incompatible default for argument "x" (default has type "B", argument has type "T") + b = x # type: B # E: Incompatible types in assignment (expression has type "T", variable has type "B") + a = x # type: A +[case testDefaultArgumentsWithSubtypes] +import typing +class A: pass +class B(A): pass + +def f(x: 'B' = A()) -> None: # E: Incompatible default for argument "x" (default has type "A", argument has type "B") + pass +def g(x: 'A' = B()) -> None: + pass +[out] + +[case testMultipleDefaultArgumentExpressions] +import typing +class A: pass +class B: pass + +def f(x: 'A' = B(), y: 'B' = B()) -> None: # E: Incompatible default for argument "x" (default has type "B", argument has type "A") + pass +def h(x: 'A' = A(), y: 'B' = B()) -> None: + pass +[out] + +[case testMultipleDefaultArgumentExpressions2] +import typing +class A: pass +class B: pass + +def g(x: 'A' = A(), y: 'B' = A()) -> None: # E: Incompatible default for argument "y" (default has type "A", argument has type "B") + pass +[out] + +[case testDefaultArgumentsAndSignatureAsComment] +import typing +def f(x = 1): # type: (int) -> str + pass +f() +f(1) +f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testMethodDefaultArgumentsAndSignatureAsComment] +import typing +class A: + def f(self, x = 1): # type: (int) -> str + pass +A().f() +A().f(1) +A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + + +-- Access to method defined as a data attribute +-- -------------------------------------------- + + +[case testMethodAsDataAttribute] +from typing import Any, Callable, ClassVar +class B: pass +x: Any +class A: + f = x # type: ClassVar[Callable[[A], None]] + g = x # type: ClassVar[Callable[[A, B], None]] +a: A +a.f() +a.g(B()) +a.f(a) # E: Too many arguments +a.g() # E: Too few arguments + +[case testMethodWithInvalidMethodAsDataAttribute] +from typing import Any, Callable, ClassVar +class B: pass +x: Any +class A: + f = x # type: ClassVar[Callable[[], None]] + g = x # type: ClassVar[Callable[[B], None]] +a: A +a.f() # E: Attribute function "f" with type "Callable[[], None]" does not accept self argument +a.g() # E: Invalid self argument "A" to attribute function "g" with type "Callable[[B], None]" + +[case testMethodWithDynamicallyTypedMethodAsDataAttribute] +from typing import Any, Callable, ClassVar +class B: pass +x: Any +class A: + f = x # type: ClassVar[Callable[[Any], Any]] +a: A +a.f() +a.f(a) # E: Too many arguments + +[case testMethodWithInferredMethodAsDataAttribute] +from typing import Any +def m(self: "A") -> int: ... + +class A: + n = m + +a = A() +reveal_type(a.n()) # N: Revealed type is "builtins.int" +reveal_type(A.n(a)) # N: Revealed type is "builtins.int" +A.n() # E: Too few arguments + +[case testOverloadedMethodAsDataAttribute] +from foo import * +[file foo.pyi] +from typing import overload +class B: pass +class A: + @overload + def f(self) -> None: pass + @overload + def f(self, b: B) -> None: pass + g = f +a: A +a.g() +a.g(B()) +a.g(a) # E: No overload variant matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f(self) -> None \ + # N: def f(self, b: B) -> None + +[case testMethodAsDataAttributeInferredFromDynamicallyTypedMethod] + +class A: + def f(self, x): pass + g = f +a: A +a.g(object()) +a.g(a, a) # E: Too many arguments +a.g() # E: Too few arguments + +[case testMethodAsDataAttributeInGenericClass] +from typing import TypeVar, Generic +t = TypeVar('t') +class B: pass +class A(Generic[t]): + def f(self, x: t) -> None: pass + g = f +a: A[B] +a.g(B()) +a.g(a) # E: Argument 1 has incompatible type "A[B]"; expected "B" + +[case testInvalidMethodAsDataAttributeInGenericClass] +from typing import Any, TypeVar, Generic, Callable, ClassVar +t = TypeVar('t') +class B: pass +class C: pass +x: Any +class A(Generic[t]): + f = x # type: ClassVar[Callable[[A[B]], None]] +ab: A[B] +ac: A[C] +ab.f() +ac.f() # E: Invalid self argument "A[C]" to attribute function "f" with type "Callable[[A[B]], None]" + +[case testPartiallyTypedSelfInMethodDataAttribute] +from typing import Any, TypeVar, Generic, Callable, ClassVar +t = TypeVar('t') +class B: pass +class C: pass +x: Any +class A(Generic[t]): + f = x # type: ClassVar[Callable[[A], None]] +ab: A[B] +ac: A[C] +ab.f() +ac.f() + +[case testCallableDataAttribute] +from typing import Callable, ClassVar +class A: + g: ClassVar[Callable[[A], None]] + def __init__(self, f: Callable[[], None]) -> None: + self.f = f +a = A(lambda: None) +a.f() +a.g() +a.f(a) # E: Too many arguments +a.g(a) # E: Too many arguments + + +-- Nested functions +-- ---------------- + + +[case testSimpleNestedFunction] +import typing +def f(a: 'A') -> None: + def g(b: 'B') -> None: + if int(): + b = a \ + # E: Incompatible types in assignment (expression has type "A", variable has type "B") + aa = a # type: A # ok + b = B() + g(a) # E: Argument 1 to "g" has incompatible type "A"; expected "B" + g(B()) +class A: pass +class B: pass + +[case testReturnAndNestedFunction] +import typing +def f() -> 'A': + def g() -> 'B': + return A() # fail + return B() + return B() # fail + return A() +class A: pass +class B: pass +[out] +main:4: error: Incompatible return value type (got "A", expected "B") +main:6: error: Incompatible return value type (got "B", expected "A") + +[case testDynamicallyTypedNestedFunction] +import typing +def f(x: object) -> None: + def g(y): + pass + g() # E: Missing positional argument "y" in call to "g" + g(x) +[out] + +[case testNestedFunctionInMethod] +import typing +class A: + def f(self) -> None: + def g(x: int) -> None: + y = x # type: int + a = x # type: A # fail + g(2) + g(A()) # fail +[out] +main:6: error: Incompatible types in assignment (expression has type "int", variable has type "A") +main:8: error: Argument 1 to "g" has incompatible type "A"; expected "int" + +[case testNestedFunctionInMethodWithTooFewArgumentsInTypeComment] +class A: + def f(self): + # type: () -> None + def g(x): # E: Type signature has too few arguments + # type: () -> None + pass + +[case testDeepNestedFunctionWithTooFewArgumentsInTypeComment] +class A: + def f(self): + # type: () -> None + class B: + def g(self): + # type: () -> None + def h(x): # E: Type signature has too few arguments + # type: () -> None + pass + +[case testDeepNestedMethodInTypeComment] +class A: + def f(self): + # type: () -> None + class B: + class C: + def g(self): + # type: () -> None + pass + +[case testMutuallyRecursiveNestedFunctions] +def f() -> None: + def g() -> None: + h(1) + h('') # E + def h(x: int) -> None: + g() + g(1) # E +[out] +main:4: error: Argument 1 to "h" has incompatible type "str"; expected "int" +main:7: error: Too many arguments for "g" + +[case testMutuallyRecursiveDecoratedFunctions] +from typing import Callable, Any +def dec(f) -> Callable[..., Any]: pass +def f() -> None: + @dec + def g() -> None: + h() + h.x # E + @dec + def h(x: int) -> None: + g(1) + g.x # E +[out] +main:7: error: "Callable[..., Any]" has no attribute "x" +main:11: error: "Callable[..., Any]" has no attribute "x" + +[case testNestedGenericFunctions] +from typing import TypeVar +T = TypeVar('T') +U = TypeVar('U') + +def outer(x: T) -> T: + def inner(y: U) -> T: ... + return inner(1) + + +-- Casts +-- ----- + + +[case testCastsToAndFromFunctionTypes] +from typing import TypeVar, Callable, Any, cast +t = TypeVar('t') +def f(x: t, + f1: Callable[[], None], + f2: Callable[[Any], None], o: object) -> None: + x = cast(t, f1) + f1 = cast(Callable[[], None], x) + f1 = cast(Callable[[], None], f2) + f1 = cast(Callable[[], None], o) + + +-- Function decorators +-- ------------------- + + +[case testTrivialStaticallyTypedFunctionDecorator] +from typing import TypeVar +t = TypeVar('t') +def dec(f: t) -> t: + return f +@dec +def f(x: int) -> None: pass +f(1) +f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testTrivialStaticallyTypedMethodDecorator] +from typing import TypeVar +t = TypeVar('t') +def dec(f: t) -> t: + return f +class A: + @dec + def f(self, x: int) -> None: pass +A().f(1) +A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +class B: pass + +[case testTrivialDecoratedNestedFunction] +from typing import TypeVar +t = TypeVar('t') +def dec(f: t) -> t: + return f +def g() -> None: + @dec + def f(x: int) -> None: pass + f(1) + f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +[out] + +[case testCheckingDecoratedFunction] +import typing +def dec(f): pass +@dec +def f(x: 'A') -> None: + a = x # type: A + if int(): + x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +class A: pass +[out] + +[case testDecoratorThatSwitchesType] +from typing import Callable +def dec(x) -> Callable[[], None]: pass +@dec +def f(y): pass +f() +f(None) # E: Too many arguments for "f" + +[case testDecoratorThatSwitchesTypeWithMethod] +from typing import Any, Callable +def dec(x) -> Callable[[Any], None]: pass +class A: + @dec + def f(self, a, b, c): pass +a: A +a.f() +a.f(None) # E: Too many arguments for "f" of "A" + +[case testNestedDecorators] +from typing import Any, Callable +def dec1(f: Callable[[Any], None]) -> Callable[[], None]: pass +def dec2(f: Callable[[Any, Any], None]) -> Callable[[Any], None]: pass +@dec1 +@dec2 +def f(x, y): pass +f() +f(None) # E: Too many arguments for "f" + +[case testInvalidDecorator1] +from typing import Any, Callable +def dec1(f: Callable[[Any], None]) -> Callable[[], None]: pass +def dec2(f: Callable[[Any, Any], None]) -> Callable[[Any], None]: pass +@dec1 # E: Argument 1 to "dec2" has incompatible type "Callable[[Any], Any]"; expected "Callable[[Any, Any], None]" +@dec2 +def f(x): pass + +[case testInvalidDecorator2] +from typing import Any, Callable +def dec1(f: Callable[[Any, Any], None]) -> Callable[[], None]: pass +def dec2(f: Callable[[Any, Any], None]) -> Callable[[Any], None]: pass +@dec1 # E: Argument 1 to "dec1" has incompatible type "Callable[[Any], None]"; expected "Callable[[Any, Any], None]" +@dec2 +def f(x, y): pass + +[case testNoTypeCheckDecoratorOnMethod1] +from typing import no_type_check + +@no_type_check +def foo(x: 'bar', y: {'x': 4}) -> 42: + 1 + 'x' +[typing fixtures/typing-medium.pyi] + +[case testNoTypeCheckDecoratorOnMethod2] +import typing + +@typing.no_type_check +def foo(x: 's', y: {'x': 4}) -> 42: + 1 + 'x' + +@typing.no_type_check +def bar() -> None: + 1 + 'x' +[typing fixtures/typing-medium.pyi] + +[case testCallingNoTypeCheckFunction] +import typing + +@typing.no_type_check +def foo(x: {1:2}) -> [1]: + 1 + 'x' + +foo() +foo(1, 'b') +[typing fixtures/typing-medium.pyi] + +[case testCallingNoTypeCheckFunction2] +import typing + +def f() -> None: + foo() + +@typing.no_type_check +def foo(x: {1:2}) -> [1]: + 1 + 'x' +[typing fixtures/typing-medium.pyi] + +[case testNoTypeCheckDecoratorSemanticError] +import typing + +@typing.no_type_check +def foo(x: {1:2}) -> [1]: + x = y +[typing fixtures/typing-medium.pyi] + + +-- Forward references to decorated functions +-- ----------------------------------------- + + +[case testForwardReferenceToDynamicallyTypedDecorator] +def f(self) -> None: + g() + g(1) + +def dec(f): + return f + +@dec +def g(): + pass + +[case testForwardReferenceToDecoratorWithAnyReturn] +from typing import Any + +def f(self) -> None: + g() + g(1) + +def dec(f) -> Any: + return f + +@dec +def g(): + pass + +[case testForwardReferenceToDecoratorWithIdentityMapping] +from typing import TypeVar + +def f(self) -> None: + g() + g(1) # E: Too many arguments for "g" + h(1).x # E: "str" has no attribute "x" + h('') # E: Argument 1 to "h" has incompatible type "str"; expected "int" + +T = TypeVar('T') +def dec(f: T) -> T: + return f + +@dec +def g(): pass +@dec +def h(x: int) -> str: pass +[out] + +[case testForwardReferenceToDynamicallyTypedDecoratedMethod] +def f(self) -> None: + A().f(1).y + A().f() + +class A: + @dec + def f(self, x): pass + +def dec(f): return f +[builtins fixtures/staticmethod.pyi] + +[case testForwardReferenceToStaticallyTypedDecoratedMethod] +from typing import TypeVar + +def f(self) -> None: + A().f(1).y # E: "str" has no attribute "y" + A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +class A: + @dec + def f(self, a: int) -> str: return '' + +T = TypeVar('T') +def dec(f: T) -> T: return f +[builtins fixtures/staticmethod.pyi] +[out] + +[case testForwardReferenceToDynamicallyTypedProperty] +def f(self) -> None: + A().x.y + +class A: + @property + def x(self): pass +[builtins fixtures/property.pyi] + +[case testForwardReferenceToStaticallyTypedProperty] +def f(self) -> None: + A().x.y # E: "int" has no attribute "y" + +class A: + @property + def x(self) -> int: return 1 +[builtins fixtures/property.pyi] +[out] + +[case testForwardReferenceToDynamicallyTypedStaticMethod] +def f(self) -> None: + A.x(1).y + A.x() # E: Missing positional argument "x" in call to "x" + +class A: + @staticmethod + def x(x): pass +[builtins fixtures/staticmethod.pyi] +[out] + +[case testForwardReferenceToStaticallyTypedStaticMethod] +def f(self) -> None: + A.x(1).y # E: "str" has no attribute "y" + A.x('') # E: Argument 1 to "x" of "A" has incompatible type "str"; expected "int" + +class A: + @staticmethod + def x(a: int) -> str: return '' +[builtins fixtures/staticmethod.pyi] +[out] + +[case testForwardReferenceToDynamicallyTypedClassMethod] +def f(self) -> None: + A.x(1).y + A.x() # E: Missing positional argument "a" in call to "x" + +class A: + @classmethod + def x(cls, a): pass +[builtins fixtures/classmethod.pyi] +[out] + +[case testForwardReferenceToStaticallyTypedClassMethod] +def f(self) -> None: + A.x(1).y # E: "str" has no attribute "y" + A.x('') # E: Argument 1 to "x" of "A" has incompatible type "str"; expected "int" + +class A: + @classmethod + def x(cls, x: int) -> str: return '' +[builtins fixtures/classmethod.pyi] +[out] + +[case testForwardReferenceToDecoratedFunctionUsingMemberExpr] +import m + +def f(self) -> None: + g(1).x # E: "str" has no attribute "x" + +@m.dec +def g(x: int) -> str: pass +[file m.py] +from typing import TypeVar +T = TypeVar('T') +def dec(f: T) -> T: + return f +[out] + +[case testForwardReferenceToFunctionWithMultipleDecorators] +# flags: --disable-error-code=used-before-def +def f(self) -> None: + g() + g(1) + +def dec(f): + return f + +@dec +@dec2 +def g(): + pass + +def dec2(f): + return f + +[case testForwardReferenceToDynamicallyTypedDecoratedStaticMethod] +def f(self) -> None: + A().f(1).y + A().f() + A().g(1).y + A().g() + +class A: + @dec + @staticmethod + def f(self, x): pass + @staticmethod + @dec + def g(self, x): pass + +def dec(f): return f +[builtins fixtures/staticmethod.pyi] + +[case testForwardRefereceToDecoratedFunctionWithCallExpressionDecorator] +# flags: --disable-error-code=used-before-def +def f(self) -> None: + g() + g(1) + +@dec(1) +def g(): pass + +def dec(f): pass + + +-- Decorator functions in import cycles +-- ------------------------------------ + + +[case testDecoratorWithIdentityTypeInImportCycle] +import a + +[file a.py] +import b +from d import dec +@dec +def f(x: int) -> None: pass +b.g(1) # E + +[file b.py] +import a +from d import dec +@dec +def g(x: str) -> None: pass +a.f('') + +[file d.py] +from typing import TypeVar +T = TypeVar('T') +def dec(f: T) -> T: return f + +[out] +tmp/b.py:5: error: Argument 1 to "f" has incompatible type "str"; expected "int" +tmp/a.py:5: error: Argument 1 to "g" has incompatible type "int"; expected "str" + +[case testDecoratorWithNoAnnotationInImportCycle] +import a + +[file a.py] +import b +from d import dec +@dec +def f(x: int) -> None: pass +b.g(1, z=4) + +[file b.py] +import a +from d import dec +@dec +def g(x: str) -> None: pass +a.f('', y=2) + +[file d.py] +def dec(f): return f + +[case testDecoratorWithFixedReturnTypeInImportCycle] +import a + +[file a.py] +import b +from d import dec +@dec +def f(x: int) -> str: pass +b.g(1)() + +[file b.py] +import a +from d import dec +@dec +def g(x: int) -> str: pass +a.f(1)() + +[file d.py] +from typing import Callable +def dec(f: Callable[[int], str]) -> Callable[[int], str]: return f + +[out] +tmp/b.py:5: error: "str" not callable +tmp/a.py:5: error: "str" not callable + +[case testDecoratorWithCallAndFixedReturnTypeInImportCycle] +import a + +[file a.py] +import b +from d import dec +@dec() +def f(x: int) -> str: pass +b.g(1)() + +[file b.py] +import a +from d import dec +@dec() +def g(x: int) -> str: pass +a.f(1)() + +[file d.py] +from typing import Callable +def dec() -> Callable[[Callable[[int], str]], Callable[[int], str]]: pass + +[out] +tmp/b.py:5: error: "str" not callable +tmp/a.py:5: error: "str" not callable + +[case testDecoratorWithCallAndFixedReturnTypeInImportCycleAndDecoratorArgs] +import a + +[file a.py] +import b +from d import dec +@dec(1) +def f(x: int) -> str: pass +b.g(1)() + +[file b.py] +import a +from d import dec +@dec(1) +def g(x: int) -> str: pass +a.f(1)() + +[file d.py] +from typing import Callable +def dec(x: str) -> Callable[[Callable[[int], str]], Callable[[int], str]]: pass + +[out] +tmp/b.py:3: error: Argument 1 to "dec" has incompatible type "int"; expected "str" +tmp/b.py:5: error: "str" not callable +tmp/a.py:3: error: Argument 1 to "dec" has incompatible type "int"; expected "str" +tmp/a.py:5: error: "str" not callable + +[case testUndefinedDecoratorInImportCycle] +# cmd: mypy -m foo.base +[file foo/__init__.py] +import foo.base +class Derived(foo.base.Base): + def method(self) -> None: pass +[file foo/base.py] +import foo +class Base: + @decorator + def method(self) -> None: pass +[out] +tmp/foo/base.py:3: error: Name "decorator" is not defined + + +-- Conditional function definition +-- ------------------------------- + + +[case testTypeCheckBodyOfConditionalFunction] +from typing import Any +x = None # type: Any +if x: + def f(x: int) -> None: + if int(): + x = 1 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[out] + +[case testCallConditionalFunction] +from typing import Any +x = None # type: Any +if x: + def f(x: int) -> None: pass + f(1) + f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +f(1) +f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testConditionalFunctionDefinitionWithIfElse] +from typing import Any +x = None # type: Any +if x: + def f(x: int) -> None: + 'x' + x # fail + if int(): + x = 1 +else: + def f(x: int) -> None: + x + 'x' # fail + if int(): + x = 1 +f(1) +f('x') # fail +[builtins fixtures/primitives.pyi] +[out] +main:5: error: Unsupported operand types for + ("str" and "int") +main:10: error: Unsupported operand types for + ("int" and "str") +main:14: error: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testNestedConditionalFunctionDefinitionWithIfElse] +from typing import Any +x = None # type: Any +def top() -> None: + if x: + def f(x: int) -> None: + if int(): + x = 'x' # E: Incompatible types in assignment \ + (expression has type "str", variable has type "int") + x = 1 + else: + def f(x: int) -> None: + x + 'x' # E: Unsupported operand types for + ("int" and "str") + x = 1 + f(1) + f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testUnconditionalRedefinitionOfConditionalFunction] +from typing import Any +x = None # type: Any +if x: + def f(): pass +def f(): pass # E: Name "f" already defined on line 4 + +[case testIncompatibleConditionalFunctionDefinition] +from typing import Any +x = None # type: Any +if x: + def f(x: int) -> None: pass +else: + def f(x): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(x: Any) -> Any + +[case testIncompatibleConditionalFunctionDefinition2] +from typing import Any +x = None # type: Any +if x: + def f(x: int) -> None: pass +else: + def f(y: int) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(y: int) -> None + +[case testIncompatibleConditionalFunctionDefinition3] +from typing import Any +x = None # type: Any +if x: + def f(x: int) -> None: pass +else: + def f(x: int = 0) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(x: int = ...) -> None + +[case testIncompatibleConditionalFunctionDefinition4] +from typing import Any, Union, TypeVar +T1 = TypeVar('T1') +T2 = TypeVar('T2', bound=Union[int, str]) +x = None # type: Any +if x: + def f(x: T1) -> T1: pass +else: + def f(x: T2) -> T2: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def [T1] f(x: T1) -> T1 \ + # N: Redefinition: \ + # N: def [T2: Union[int, str]] f(x: T2) -> T2 + +[case testConditionalFunctionDefinitionUsingDecorator1] +from typing import Callable + +def dec(f) -> Callable[[int], None]: pass + +x = int() +if x: + @dec + def f(): pass +else: + def f(x: int) -> None: pass + +[case testConditionalFunctionDefinitionUsingDecorator2] +from typing import Callable + +def dec(f) -> Callable[[int], None]: pass + +x = int() +if x: + @dec + def f(): pass +else: + def f(x: str) -> None: pass # E: Incompatible redefinition (redefinition with type "Callable[[str], None]", original type "Callable[[int], None]") + +[case testConditionalFunctionDefinitionUsingDecorator3] + +from typing import Callable + +def dec(f) -> Callable[[int], None]: pass + +x = int() +if x: + def f(x: int, /) -> None: pass +else: + @dec + def f(): pass + +[case testConditionalFunctionDefinitionUsingDecorator4] + +from typing import Callable + +def dec(f) -> Callable[[int], None]: pass + +x = int() +if x: + def f(x: str) -> None: pass +else: + @dec + def f(): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: str) -> None \ + # N: Redefinition: \ + # N: def f(int, /) -> None + +[case testConditionalFunctionDefinitionUnreachable] +def bar() -> None: + if False: + foo = 1 + else: + def foo(obj): ... + +def baz() -> None: + if False: + foo: int = 1 + else: + def foo(obj): ... # E: Incompatible redefinition (redefinition with type "Callable[[Any], Any]", original type "int") +[builtins fixtures/tuple.pyi] + +[case testConditionalRedefinitionOfAnUnconditionalFunctionDefinition1] +from typing import Any +def f(x: str) -> None: pass +x = None # type: Any +if x: + def f(x: int) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: str) -> None \ + # N: Redefinition: \ + # N: def f(x: int) -> None + +[case testConditionalRedefinitionOfAnUnconditionalFunctionDefinition2] +from typing import Any +def f(x: int) -> None: pass # N: "f" defined here +x = None # type: Any +if x: + def f(y: int) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(y: int) -> None +f(x=1) # The first definition takes precedence. +f(y=1) # E: Unexpected keyword argument "y" for "f" + +[case testRedefineFunctionDefinedAsVariable] +def g(): pass +f = g +if g(): + def f(): pass +f() +f(1) # E: Too many arguments + +[case testRedefineFunctionDefinedAsVariableInitializedToNone] +def g(): pass +f = None +if g(): + def f(): pass +f() +f(1) # E: Too many arguments for "f" + +[case testRedefineNestedFunctionDefinedAsVariableInitializedToNone] +def g() -> None: + f = None + if object(): + def f(x: int) -> None: pass + f() # E: Missing positional argument "x" in call to "f" + f(1) + f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +[out] + +[case testRedefineFunctionDefinedAsVariableWithInvalidSignature] +def g(): pass +f = g +if g(): + def f(x): pass # E: Incompatible redefinition (redefinition with type "Callable[[Any], Any]", original type "Callable[[], Any]") + +[case testRedefineFunctionDefinedAsVariableWithVariance1] +class B: pass +class C(B): pass +def g(x: C) -> B: pass +f = g +if g(C()): + def f(x: C) -> C: pass + +[case testRedefineFunctionDefinedAsVariableWithVariance2] +class B: pass +class C(B): pass +def g(x: C) -> B: pass +f = g +if g(C()): + def f(x: B) -> B: pass + +[case testRedefineFunctionDefinedAsVariableInitializedToEmptyList] +f = [] # E: Need type annotation for "f" (hint: "f: List[] = ...") +if object(): + def f(): pass # E: Incompatible redefinition +f() # E: "List[Any]" not callable +f(1) # E: "List[Any]" not callable +[builtins fixtures/list.pyi] + +[case testDefineConditionallyAsImportedAndDecorated] +from typing import Callable + +def dec(f: Callable[[], None]) -> Callable[[], None]: ... + +if int(): + from m import f +else: + @dec + def f(): + yield +[file m.py] +def f() -> None: pass + +[case testDefineConditionallyAsImportedAndDecoratedWithInference] +if int(): + from m import f +else: + from contextlib import contextmanager + + @contextmanager + def f(): + yield +[file m.py] +from contextlib import contextmanager + +@contextmanager +def f(): + yield +[typing fixtures/typing-medium.pyi] +[builtins fixtures/tuple.pyi] + + +-- Conditional method definition +-- ----------------------------- + + +[case testTypeCheckBodyOfConditionalMethod] +from typing import Any +x = None # type: Any +class A: + if x: + def f(self, x: int) -> None: + if int(): + x = 1 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[out] + +[case testCallConditionalMethodInClassBody] +from typing import Any +x = None # type: Any +class A: + if x: + def f(self, x: int) -> None: pass + f(x, 1) + f(x, 'x') # E: Argument 2 to "f" of "A" has incompatible type "str"; expected "int" + f(x, 1) + f(x, 'x') # E: Argument 2 to "f" of "A" has incompatible type "str"; expected "int" +[out] + +[case testCallConditionalMethodViaInstance] +from typing import Any +x = None # type: Any +class A: + if x: + def f(self, x: int) -> None: pass +A().f(1) +A().f('x') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testConditionalMethodDefinitionWithIfElse] +from typing import Any +x = None # type: Any +class A: + if x: + def f(self, x: int) -> None: + 'x' + x # fail + if int(): + x = 1 + else: + def f(self, x: int) -> None: + x + 'x' # fail + if int(): + x = 1 +A().f(1) +A().f('x') # fail +[builtins fixtures/primitives.pyi] +[out] +main:6: error: Unsupported operand types for + ("str" and "int") +main:11: error: Unsupported operand types for + ("int" and "str") +main:15: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testUnconditionalRedefinitionOfConditionalMethod] +from typing import Any +x = None # type: Any +class A: + if x: + def f(self): pass + def f(self): pass # E: Name "f" already defined on line 5 + +[case testIncompatibleConditionalMethodDefinition] +from typing import Any +x = None # type: Any +class A: + if x: + def f(self, x: int) -> None: pass + else: + def f(self, x): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(self: A, x: int) -> None \ + # N: Redefinition: \ + # N: def f(self: A, x: Any) -> Any +[out] + +[case testConditionalFunctionDefinitionInTry] +import typing +try: + def f(x: int) -> None: pass +except: + def g(x: str) -> None: pass +f(1) +f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +g('x') +g(1) # E: Argument 1 to "g" has incompatible type "int"; expected "str" + +[case testConditionalMethodDefinitionUsingDecorator] +from typing import Callable + +def dec(f) -> Callable[['A', int], None]: pass + +class A: + x = int() + if x: + @dec + def f(self): pass + else: + def f(self, x: int) -> None: pass + + +-- Callable with specific arg list +-- ------------------------------- + + +[case testCallableWithNamedArg] +from typing import Callable +from mypy_extensions import Arg + +def a(f: Callable[[Arg(int, 'x')], int]): + f(x=4) + f(5) + f(y=3) # E: Unexpected keyword argument "y" + +[builtins fixtures/dict.pyi] + +[case testCallableWithOptionalArg] +from typing import Callable +from mypy_extensions import DefaultArg + +def a(f: Callable[[DefaultArg(int, 'x')], int]): + f(x=4) + f(2) + f() + f(y=3) # E: Unexpected keyword argument "y" + f("foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableWithNamedArgFromExpr] +from typing import Callable +from mypy_extensions import Arg + +F = Callable[[Arg(int, 'x')], int] + +def a(f: F): + f(x=4) + f(5) + f(y=3) # E: Unexpected keyword argument "y" + +[builtins fixtures/dict.pyi] + +[case testCallableWithOptionalArgFromExpr] +from typing import Callable +from mypy_extensions import DefaultArg + +F = Callable[[DefaultArg(int, 'x')], int] +def a(f: F): + f(x=4) + f(2) + f() + f(y=3) # E: Unexpected keyword argument "y" + f("foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableParsingInInheritence] + +from collections import namedtuple +class C(namedtuple('t', 'x')): + pass +[builtins fixtures/tuple.pyi] + +[case testCallableParsingSameName] +from typing import Callable + +def Arg(x, y): pass + +F = Callable[[Arg(int, 'x')], int] # E: Invalid argument constructor "__main__.Arg" + +[case testCallableParsingFromExpr] +# flags: --python-version 3.9 +from typing import Callable, List +from mypy_extensions import Arg, VarArg, KwArg +import mypy_extensions +import types # Needed for type checking + +def WrongArg(x, y): return y +# Note that for this test, the 'Value of type "int" is not indexable' errors are silly, +# and a consequence of Callable being set to an int in the test stub. We can't set it to +# something else sensible, because other tests require the stub not have anything +# that looks like a function call. +F = Callable[[WrongArg(int, 'x')], int] # E: Invalid argument constructor "__main__.WrongArg" +G = Callable[[Arg(1, 'x')], int] # E: Invalid type: try using Literal[1] instead? +H = Callable[[VarArg(int, 'x')], int] # E: VarArg arguments should not have names +I = Callable[[VarArg(int)], int] # ok +J = Callable[[VarArg(), KwArg()], int] # ok +K = Callable[[VarArg(), int], int] # E: Required positional args may not appear after default, named or var args +L = Callable[[Arg(name='x', type=int)], int] # ok +# I have commented out the following test because I don't know how to expect the "defined here" note part of the error. +# M = Callable[[Arg(gnome='x', type=int)], int] E: Invalid type alias: expression is not a valid type E: Unexpected keyword argument "gnome" for "Arg" +N = Callable[[Arg(name=None, type=int)], int] # ok +O = Callable[[List[Arg(int)]], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: Type expected within [...] +P = Callable[[mypy_extensions.VarArg(int)], int] # ok +Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: "Arg" gets multiple values for keyword argument "type" +R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: "Arg" gets multiple values for keyword argument "name" + + + + + + + +[builtins fixtures/dict.pyi] + +[case testCallableParsing] +from typing import Callable +from mypy_extensions import Arg, VarArg, KwArg + +def WrongArg(x, y): return y + +def b(f: Callable[[Arg(1, 'x')], int]): pass # Invalid type. Try using Literal[1] instead? +def d(f: Callable[[VarArg(int)], int]): pass # ok +def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok +def g(f: Callable[[Arg(name='x', type=int)], int]): pass # ok +def h(f: Callable[[Arg(gnome='x', type=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok +def j(f: Callable[[Arg(int, 'x', name='y')], int]): pass # E: "Arg" gets multiple values for keyword argument "name" +def k(f: Callable[[Arg(int, type=int)], int]): pass # E: "Arg" gets multiple values for keyword argument "type" + +[builtins fixtures/dict.pyi] + +[case testCallableTypeAnalysis] +from typing import Callable +from mypy_extensions import Arg, VarArg as VARG, KwArg +import mypy_extensions as ext + +def WrongArg(x, y): return y +def a(f: Callable[[WrongArg(int, 'x')], int]): pass # E: Invalid argument constructor "__main__.WrongArg" +def b(f: Callable[[BadArg(int, 'x')], int]): pass # E: Name "BadArg" is not defined +def d(f: Callable[[ext.VarArg(int)], int]): pass # ok +def e(f: Callable[[VARG(), ext.KwArg()], int]): pass # ok +def g(f: Callable[[ext.Arg(name='x', type=int)], int]): pass # ok +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok + +def f1(*args) -> int: pass +def f2(*args, **kwargs) -> int: pass + +d(f1) +e(f2) +d(f2) +e(f1) + +[builtins fixtures/dict.pyi] + +[case testCallableWrongTypeType] +from typing import Callable +from mypy_extensions import Arg +def b(f: Callable[[Arg(1, 'x')], int]): pass # E: Invalid type: try using Literal[1] instead? +[builtins fixtures/dict.pyi] + +[case testCallableTooManyVarArg] +from typing import Callable +from mypy_extensions import VarArg +def c(f: Callable[[VarArg(int, 'x')], int]): pass # E: VarArg arguments should not have names +[builtins fixtures/dict.pyi] + +[case testCallableFastParseGood] +from typing import Callable +from mypy_extensions import VarArg, Arg, KwArg +def d(f: Callable[[VarArg(int)], int]): pass # ok +def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok +def g(f: Callable[[Arg(name='x', type=int)], int]): pass # ok +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok +[builtins fixtures/dict.pyi] + +[case testCallableFastParseBadArgArgName] +from typing import Callable +from mypy_extensions import Arg +def h(f: Callable[[Arg(gnome='x', type=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor +[builtins fixtures/dict.pyi] + +[case testCallableKindsOrdering] +from typing import Callable, Any +from mypy_extensions import Arg, VarArg, KwArg, DefaultArg, NamedArg + +def f(f: Callable[[VarArg(), int], int]): pass # E: Required positional args may not appear after default, named or var args +def g(f: Callable[[VarArg(), VarArg()], int]): pass # E: Var args may not appear after named or var args +def h(f: Callable[[KwArg(), KwArg()], int]): pass # E: You may only have one **kwargs argument +def i(f: Callable[[DefaultArg(), int], int]): pass # E: Required positional args may not appear after default, named or var args +def j(f: Callable[[NamedArg(Any, 'x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or var args +def k(f: Callable[[KwArg(), NamedArg(Any, 'x')], int]): pass # E: A **kwargs argument must be the last argument +[builtins fixtures/dict.pyi] + +[case testCallableDuplicateNames] +from typing import Callable +from mypy_extensions import Arg, VarArg, KwArg, DefaultArg + +def f(f: Callable[[Arg(int, 'x'), int, Arg(int, 'x')], int]): pass # E: Duplicate argument "x" in Callable + +[builtins fixtures/dict.pyi] + + +[case testCallableWithKeywordOnlyArg] +from typing import Callable +from mypy_extensions import NamedArg + +def a(f: Callable[[NamedArg(int, 'x')], int]): + f(x=4) + f(2) # E: Too many positional arguments + f() # E: Missing named argument "x" + f(y=3) # E: Unexpected keyword argument "y" + f(x="foo") # E: Argument "x" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableWithKeywordOnlyOptionalArg] +from typing import Callable +from mypy_extensions import DefaultNamedArg + +def a(f: Callable[[DefaultNamedArg(int, 'x')], int]): + f(x=4) + f(2) # E: Too many positional arguments + f() + f(y=3) # E: Unexpected keyword argument "y" + f(x="foo") # E: Argument "x" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableWithKwargs] +from typing import Callable +from mypy_extensions import KwArg + +def a(f: Callable[[KwArg(int)], int]): + f(x=4) + f(2) # E: Too many arguments + f() + f(y=3) + f(x=4, y=3, z=10) + f(x="foo") # E: Argument "x" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + + +[case testCallableWithVarArg] +from typing import Callable +from mypy_extensions import VarArg + +def a(f: Callable[[VarArg(int)], int]): + f(x=4) # E: Unexpected keyword argument "x" + f(2) + f() + f(3, 4, 5) + f("a") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableArgKindSubtyping] +from typing import Callable +from mypy_extensions import Arg, DefaultArg + +int_str_fun: Callable[[int, str], str] +int_opt_str_fun: Callable[[int, DefaultArg(str, None)], str] +int_named_str_fun: Callable[[int, Arg(str, 's')], str] + +def isf(ii: int, ss: str) -> str: + return ss + +def iosf(i: int, s: str = "bar") -> str: + return s + +def isf_unnamed(__i: int, __s: str) -> str: + return __s + +int_str_fun = isf +int_str_fun = isf_unnamed +int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "Callable[[int, Arg(str, 's')], str]") +int_opt_str_fun = iosf +int_str_fun = iosf +int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str]", variable has type "Callable[[int, DefaultArg(str)], str]") + +int_named_str_fun = isf # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str]", variable has type "Callable[[int, Arg(str, 's')], str]") +int_named_str_fun = iosf + +[builtins fixtures/dict.pyi] + +-- Callable[..., T] +-- ---------------- + + +[case testCallableWithArbitraryArgs] +from typing import Callable +def f(x: Callable[..., int]) -> None: + x() + x(1) + x(z=1) + x() + '' # E: Unsupported operand types for + ("int" and "str") +[out] + +[case testCallableWithArbitraryArgs2] +from typing import Callable +def f(x: Callable[..., int]) -> None: + x(*[1], **{'x': 2}) +[builtins fixtures/dict.pyi] + +[case testCastWithCallableAndArbitraryArgs] +from typing import Callable, cast +f = cast(Callable[..., int], None) +f(x=4) + '' # E: Unsupported operand types for + ("int" and "str") + +[case testCallableWithArbitraryArgsInErrorMessage] +from typing import Callable +def f(x: Callable[..., int]) -> None: + if int(): + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[..., int]") +[out] + +[case testCallableWithArbitraryArgsInGenericFunction] +from typing import Callable, TypeVar +T = TypeVar('T') +def f(x: Callable[..., T]) -> T: pass +def g(*x: int) -> str: pass +x = f(g) +x + 1 # E: Unsupported left operand type for + ("str") +[builtins fixtures/list.pyi] + +[case testCallableWithArbitraryArgsSubtyping] +from typing import Callable +def f(x: Callable[..., int]) -> None: pass +def g1(): pass +def g2(x, y) -> int: pass +def g3(*, y: str) -> int: pass +def g4(*, y: int) -> str: pass +f(g1) +f(g2) +f(g3) +f(g4) # E: Argument 1 to "f" has incompatible type "Callable[[NamedArg(int, 'y')], str]"; expected "Callable[..., int]" + +[case testCallableWithArbitraryArgsSubtypingWithGenericFunc] +from typing import Callable, TypeVar +T = TypeVar('T') +def f(x: Callable[..., int]) -> None: pass +def g1(x: T) -> int: pass +def g2(*x: T) -> int: pass +def g3(*x: T) -> T: pass +f(g1) +f(g2) +f(g3) +[builtins fixtures/tuple.pyi] + +-- (...) -> T +-- ---------------- + +[case testEllipsisWithArbitraryArgsOnBareFunction] +def f(x, y, z): # type: (...) -> None + pass +f(1, "hello", []) +f(x=1, y="hello", z=[]) +[builtins fixtures/dict.pyi] + +[case testEllipsisWithArbitraryArgsOnBareFunctionWithDefaults] +def f(x, y=1, z="hey"): # type: (...) -> None + pass +f(1, "hello", []) +f(x=1, y="hello", z=[]) +[builtins fixtures/dict.pyi] + +[case testEllipsisWithArbitraryArgsOnBareFunctionWithKwargs] +from typing import Dict +def f(x, **kwargs): # type: (...) -> None + success_dict_type = kwargs # type: Dict[str, str] + failure_dict_type = kwargs # type: Dict[int, str] # E: Incompatible types in assignment (expression has type "Dict[str, Any]", variable has type "Dict[int, str]") +f(1, thing_in_kwargs=["hey"]) +[builtins fixtures/dict.pyi] +[out] + +[case testEllipsisWithArbitraryArgsOnBareFunctionWithVarargs] +from typing import Tuple, Any +def f(x, *args): # type: (...) -> None + success_tuple_type = args # type: Tuple[Any, ...] + fail_tuple_type = args # type: None # E: Incompatible types in assignment (expression has type "Tuple[Any, ...]", variable has type "None") +f(1, "hello") +[builtins fixtures/tuple.pyi] +[out] + +[case testEllipsisWithArbitraryArgsOnInstanceMethod] +class A: + def f(self, x, y, z): # type: (...) -> None + pass + +[case testEllipsisWithArbitraryArgsOnClassMethod] +class A: + @classmethod + def f(cls, x, y, z): # type: (...) -> None + pass +[builtins fixtures/classmethod.pyi] + +[case testEllipsisWithArbitraryArgsOnStaticMethod] +class A: + @staticmethod + def f(x, y, z): # type: (...) -> None + pass +[builtins fixtures/staticmethod.pyi] + +[case testEllipsisWithSomethingAfterItFails] +def f(x, y, z): # type: (..., int) -> None + pass +[out] +main:1: error: Ellipses cannot accompany other argument types in function type signature + +[case testEllipsisWithSomethingBeforeItFails] +def f(x, y, z): # type: (int, ...) -> None + pass +[out] +main:1: error: Ellipses cannot accompany other argument types in function type signature + +[case testRejectCovariantArgument] +from typing import TypeVar, Generic + +t = TypeVar('t', covariant=True) +class A(Generic[t]): + def foo(self, x: t) -> None: + return None +[builtins fixtures/bool.pyi] +[out] +main:5: error: Cannot use a covariant type variable as a parameter + +[case testRejectCovariantArgumentSplitLine] +from typing import TypeVar, Generic + +t = TypeVar('t', covariant=True) +class A(Generic[t]): + def foo(self, + x: t) -> None: + return None +[builtins fixtures/bool.pyi] +[out] +main:6: error: Cannot use a covariant type variable as a parameter + +[case testRejectCovariantArgumentInLambda] +from typing import TypeVar, Generic, Callable + +t = TypeVar('t', covariant=True) +class Thing(Generic[t]): + def chain(self, func: Callable[[t], None]) -> None: pass + def end(self) -> None: + return self.chain( # Note that lambda args have no line numbers + lambda _: None) +[builtins fixtures/bool.pyi] +[out] +main:8: error: Cannot use a covariant type variable as a parameter + +[case testRejectCovariantArgumentInLambdaSplitLine] +from typing import TypeVar, Generic, Callable + +[case testRejectContravariantReturnType] +# flags: --no-strict-optional +from typing import TypeVar, Generic + +t = TypeVar('t', contravariant=True) +class A(Generic[t]): + def foo(self) -> t: + return None +[builtins fixtures/bool.pyi] +[out] +main:6: error: Cannot use a contravariant type variable as return type + +[case testAcceptCovariantReturnType] +# flags: --no-strict-optional +from typing import TypeVar, Generic + +t = TypeVar('t', covariant=True) +class A(Generic[t]): + def foo(self) -> t: + return None +[builtins fixtures/bool.pyi] + +[case testAcceptContravariantArgument] +from typing import TypeVar, Generic + +t = TypeVar('t', contravariant=True) +class A(Generic[t]): + def foo(self, x: t) -> None: + return None +[builtins fixtures/bool.pyi] + + +-- Redefining functions +-- -------------------- + + +[case testRedefineFunction] +from typing import Any +def f(x) -> Any: pass +def g(x, y): pass +def h(x): pass +def j(y) -> Any: pass +f = h +f = j # E: Incompatible types in assignment (expression has type "Callable[[Arg(Any, 'y')], Any]", variable has type "Callable[[Arg(Any, 'x')], Any]") +f = g # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], Any]", variable has type "Callable[[Any], Any]") + +[case testRedefineFunction2] +def f() -> None: pass +def f() -> None: pass # E: Name "f" already defined on line 1 + + +-- Special cases +-- ------------- + + +[case testFunctionDefinitionWithForStatement] +for _ in [1]: + def f(): pass +else: + def g(): pass +f() +g() +[builtins fixtures/list.pyi] + +[case testFunctionDefinitionWithWhileStatement] +while bool(): + def f(): pass +else: + def g(): pass +f() +g() +[builtins fixtures/bool.pyi] + +[case testBareCallable] +from typing import Callable, Any + +def foo(f: Callable) -> bool: + return f() + +def f1() -> bool: + return False + +foo(f1) +[builtins fixtures/bool.pyi] + +[case testFunctionNestedWithinWith] +from typing import Any +a = 1 # type: Any +with a: + def f() -> None: + pass + f(1) # E: Too many arguments for "f" + + +[case testNameForDecoratorMethod] +from typing import Callable + +class A: + def f(self) -> None: + # In particular, test that the error message contains "g" of "A". + self.g() # E: Too few arguments for "g" of "A" + self.g(1) + @dec + def g(self, x: str) -> None: pass + +def dec(f: Callable[[A, str], None]) -> Callable[[A, int], None]: pass +[out] + +[case testUnknownFunctionNotCallable] +from typing import TypeVar + +def f() -> None: + pass +def g(x: int) -> None: + pass +h = f if bool() else g +reveal_type(h) # N: Revealed type is "Union[def (), def (x: builtins.int)]" +h(7) # E: Too many arguments for "f" + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +h2 = join(f, g) +reveal_type(h2) # N: Revealed type is "builtins.function" +h2(7) # E: Cannot call function of unknown type + +h3 = join(g, f) +reveal_type(h3) # N: Revealed type is "builtins.function" +h3(7) # E: Cannot call function of unknown type +[builtins fixtures/bool.pyi] + +[case testFunctionWithNameUnderscore] +def _(x: int) -> None: pass + +_(1) +_('x') # E: Argument 1 to "_" has incompatible type "str"; expected "int" + +-- Positional-only arguments +-- ------------------------- + +[case testPositionalOnlyArg] +def f(__a: int) -> None: pass +def g(__a__: int) -> None: pass + +f(1) +f(__a=1) # E: Unexpected keyword argument "__a" for "f" + +g(1) +# Argument names that also end with __ are not positional-only. +g(__a__=1) + +[builtins fixtures/bool.pyi] +[out] +main:1: note: "f" defined here + +[case testMagicMethodPositionalOnlyArg] +class A(object): + def __eq__(self, other) -> bool: return True # We are all equal. # N: "__eq__" of "A" defined here + +a = A() +a.__eq__(a) +a.__eq__(other=a) # E: Unexpected keyword argument "other" for "__eq__" of "A" + +[builtins fixtures/bool.pyi] + +[case testMagicMethodPositionalOnlyArgFastparse] + + +class A(object): + def __eq__(self, other) -> bool: return True # We are all equal. # N: "__eq__" of "A" defined here + +a = A() +a.__eq__(a) +a.__eq__(other=a) # E: Unexpected keyword argument "other" for "__eq__" of "A" + +[builtins fixtures/bool.pyi] + +-- Type variable shenanagins +-- ------------------------- + +[case testGenericFunctionTypeDecl] +from typing import Callable, TypeVar + +T = TypeVar('T') + +f: Callable[[T], T] +reveal_type(f) # N: Revealed type is "def [T] (T`-1) -> T`-1" +def g(__x: T) -> T: pass +f = g +reveal_type(f) # N: Revealed type is "def [T] (T`-1) -> T`-1" +i = f(3) +reveal_type(i) # N: Revealed type is "builtins.int" + +[case testFunctionReturningGenericFunction] +from typing import Callable, TypeVar + +T = TypeVar('T') +def deco() -> Callable[[T], T]: pass +reveal_type(deco) # N: Revealed type is "def () -> def [T] (T`-1) -> T`-1" +f = deco() +reveal_type(f) # N: Revealed type is "def [T] (T`1) -> T`1" +i = f(3) +reveal_type(i) # N: Revealed type is "builtins.int" + +[case testFunctionReturningGenericFunctionPartialBinding] +from typing import Callable, TypeVar + +T = TypeVar('T') +U = TypeVar('U') + +def deco(x: U) -> Callable[[T, U], T]: pass +reveal_type(deco) # N: Revealed type is "def [U] (x: U`-1) -> def [T] (T`-2, U`-1) -> T`-2" +f = deco("foo") +reveal_type(f) # N: Revealed type is "def [T] (T`1, builtins.str) -> T`1" +i = f(3, "eggs") +reveal_type(i) # N: Revealed type is "builtins.int" + +[case testFunctionReturningGenericFunctionTwoLevelBinding] +from typing import Callable, TypeVar + +T = TypeVar('T') +R = TypeVar('R') +def deco() -> Callable[[T], Callable[[T, R], R]]: pass +f = deco() +reveal_type(f) # N: Revealed type is "def [T] (T`2) -> def [R] (T`2, R`1) -> R`1" +g = f(3) +reveal_type(g) # N: Revealed type is "def [R] (builtins.int, R`3) -> R`3" +s = g(4, "foo") +reveal_type(s) # N: Revealed type is "builtins.str" + +[case testGenericFunctionReturnAsDecorator] +from typing import Callable, TypeVar + +T = TypeVar('T') +def deco(__i: int) -> Callable[[T], T]: pass + +@deco(3) +def lol(x: int) -> str: ... + +reveal_type(lol) # N: Revealed type is "def (x: builtins.int) -> builtins.str" +s = lol(4) +reveal_type(s) # N: Revealed type is "builtins.str" + +[case testGenericFunctionOnReturnTypeOnly] +from typing import TypeVar, List + +T = TypeVar('T') + +def make_list() -> List[T]: pass + +l: List[int] = make_list() + +bad = make_list() # E: Need type annotation for "bad" (hint: "bad: List[] = ...") +[builtins fixtures/list.pyi] + +[case testAnonymousArgumentError] +def foo(__b: int, x: int, y: int) -> int: pass +foo(x=2, y=2) # E: Too few arguments for "foo" +foo(y=2) # E: Too few arguments for "foo" + +[case testMissingArgumentError] +def f(a, b, c, d=None) -> None: pass +f(1, 2, d=3) # E: Missing positional argument "c" in call to "f" + +[case testMissingArgumentErrorMoreThanOneOptional] +def f(a: int, b=None, c=None) -> None: pass +f(b=4) # E: Missing positional argument "a" in call to "f" + +[case testMissingArgumentsError] +def f(a, b, c, d=None) -> None: pass +f(1, d=3) # E: Missing positional arguments "b", "c" in call to "f" + +[case testReturnTypeLineNumberWithDecorator] +def dec(f): pass + +@dec +def test(a: str) -> (str,): # E: Syntax error in type annotation # N: Suggestion: Is there a spurious trailing comma? + return None + +[case testReturnTypeLineNumberNewLine] +def fn(a: str + ) -> badtype: # E: Name "badtype" is not defined + pass + +[case testArgumentTypeLineNumberWithDecorator] +def dec(f): pass + +@dec +def some_method(self: badtype): pass # E: Name "badtype" is not defined + +[case TestArgumentTypeLineNumberNewline] +def fn( + a: badtype) -> None: # E: Name "badtype" is not defined + pass + +[case testInferredTypeSubTypeOfReturnType] +from typing import Union, Dict, List +def f() -> List[Union[str, int]]: + x = ['a'] + return x # E: Incompatible return value type (got "List[str]", expected "List[Union[str, int]]") \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant \ + # N: Perhaps you need a type annotation for "x"? Suggestion: "List[Union[str, int]]" + +def g() -> Dict[str, Union[str, int]]: + x = {'a': 'a'} + return x # E: Incompatible return value type (got "Dict[str, str]", expected "Dict[str, Union[str, int]]") \ + # N: "Dict" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Mapping" instead, which is covariant in the value type \ + # N: Perhaps you need a type annotation for "x"? Suggestion: "Dict[str, Union[str, int]]" + +def h() -> Dict[Union[str, int], str]: + x = {'a': 'a'} + return x # E: Incompatible return value type (got "Dict[str, str]", expected "Dict[Union[str, int], str]") \ +# N: Perhaps you need a type annotation for "x"? Suggestion: "Dict[Union[str, int], str]" + +def i() -> List[Union[int, float]]: + x: List[int] = [1] + return x # E: Incompatible return value type (got "List[int]", expected "List[Union[int, float]]") \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant + +[builtins fixtures/dict.pyi] + +[case testInferredTypeNotSubTypeOfReturnType] +from typing import Union, List +def f() -> List[Union[int, float]]: + x = ['a'] + return x # E: Incompatible return value type (got "List[str]", expected "List[Union[int, float]]") + +def g() -> List[Union[str, int]]: + x = ('a', 2) + return x # E: Incompatible return value type (got "Tuple[str, int]", expected "List[Union[str, int]]") + +[builtins fixtures/list.pyi] + +[case testInferredTypeIsObjectMismatch] +from typing import Union, Dict, List +def f() -> Dict[str, Union[str, int]]: + x = {'a': 'a', 'b': 2} + return x # E: Incompatible return value type (got "Dict[str, object]", expected "Dict[str, Union[str, int]]") + +def g() -> Dict[str, Union[str, int]]: + x: Dict[str, Union[str, int]] = {'a': 'a', 'b': 2} + return x + +def h() -> List[Union[str, int]]: + x = ['a', 2] + return x # E: Incompatible return value type (got "List[object]", expected "List[Union[str, int]]") + +def i() -> List[Union[str, int]]: + x: List[Union[str, int]] = ['a', 2] + return x + +[builtins fixtures/dict.pyi] + +[case testLambdaSemanal] +f = lambda: xyz +[out] +main:1: error: Name "xyz" is not defined + +[case testLambdaTypeCheck] +f = lambda: 1 + '1' +[out] +main:1: error: Unsupported operand types for + ("int" and "str") + +[case testLambdaTypeInference] +f = lambda: 5 +reveal_type(f) +[out] +main:2: note: Revealed type is "def () -> builtins.int" + +[case testRevealLocalsFunction] +a = 1.0 + +class A: pass + +def f(a: int, b: int) -> int: + reveal_locals() + c = a + b + class C: pass + reveal_locals() + return c + +reveal_locals() +[out] +main:6: note: Revealed local types are: +main:6: note: a: builtins.int +main:6: note: b: builtins.int +main:9: note: Revealed local types are: +main:9: note: a: builtins.int +main:9: note: b: builtins.int +main:9: note: c: builtins.int +main:12: note: Revealed local types are: +main:12: note: a: builtins.float + +[case testNoComplainOverloadNone] +# flags: --no-strict-optional +from typing import overload, Optional +@overload +def bar(x: None) -> None: + ... +@overload +def bar(x: int) -> str: + ... +def bar(x: Optional[int]) -> Optional[str]: + if x is None: + return None + return "number" + +reveal_type(bar(None)) # N: Revealed type is "None" +[builtins fixtures/isinstance.pyi] +[out] + +[case testNoComplainOverloadNoneStrict] +from typing import overload, Optional +@overload +def bar(x: None) -> None: + ... +@overload +def bar(x: int) -> str: + ... +def bar(x: Optional[int]) -> Optional[str]: + if x is None: + return None + return "number" + +reveal_type(bar(None)) # N: Revealed type is "None" +[builtins fixtures/isinstance.pyi] +[out] + +[case testNoComplainInferredNone] +# flags: --no-strict-optional +from typing import TypeVar, Optional +T = TypeVar('T') +def X(val: T) -> T: ... +x_in = None +def Y(x: Optional[str] = X(x_in)): ... + +xx: Optional[int] = X(x_in) +[out] + +[case testNoComplainInferredNoneStrict] +from typing import TypeVar, Optional +T = TypeVar('T') +def X(val: T) -> T: ... +x_in = None +def Y(x: Optional[str] = X(x_in)): ... + +xx: Optional[int] = X(x_in) +[out] + +[case testNoComplainNoneReturnFromUntyped] +def foo() -> None: + pass + +def lol(): + x = foo() + +[case testConditionalImportFunction] +import p +[file p/__init__.py] + +if int(): + from p.a import f +elif int(): + from p.b import f +else: + from p.c import f + +[file p/a.py] +def f() -> int: ... + +[file p/b.py] +from p.d import f + +[file p/c.py] +def f() -> int: ... + +[file p/d.py] +import p +def f() -> int: ... + +[case testLambdaDefaultTypeErrors] +lambda a=(1 + 'asdf'): a # E: Unsupported operand types for + ("int" and "str") +lambda a=nonsense: a # E: Name "nonsense" is not defined +def f(x: int = i): # E: Name "i" is not defined + i = 42 + +[case testRevealTypeOfCallExpressionReturningNoneWorks] +def foo() -> None: + pass + +reveal_type(foo()) # N: Revealed type is "None" + +[case testAnyArgument] +def a(b: any): pass # E: Function "builtins.any" is not valid as a type \ + # N: Perhaps you meant "typing.Any" instead of "any"? +[builtins fixtures/any.pyi] + +[case testCallableArgument] +def a(b: callable): pass # E: Function "builtins.callable" is not valid as a type \ + # N: Perhaps you meant "typing.Callable" instead of "callable"? +[builtins fixtures/callable.pyi] + +[case testDecoratedProperty] +from typing import TypeVar, Callable, final + +T = TypeVar("T") + +def dec(f: Callable[[T], int]) -> Callable[[T], str]: ... +def dec2(f: T) -> T: ... + +class A: + @property + @dec + def f(self) -> int: pass + @property + @dec2 + def g(self) -> int: pass +reveal_type(A().f) # N: Revealed type is "builtins.str" +reveal_type(A().g) # N: Revealed type is "builtins.int" + +class B: + @final + @property + @dec + def f(self) -> int: pass +reveal_type(B().f) # N: Revealed type is "builtins.str" + +class C: + @property # E: Only instance methods can be decorated with @property + @classmethod + def f(cls) -> int: pass +reveal_type(C().f) # N: Revealed type is "builtins.int" +[builtins fixtures/property.pyi] +[out] + +[case testDecoratedPropertySetter] +from typing import TypeVar, Callable, final + +T = TypeVar("T") +def dec(f: T) -> T: ... + +class A: + @property + @dec + def f(self) -> int: pass + @f.setter + @dec + def f(self, v: int) -> None: pass +reveal_type(A().f) # N: Revealed type is "builtins.int" + +class B: + @property + @dec + def f(self) -> int: pass + @dec # E: Only supported top decorator is @f.setter + @f.setter + def f(self, v: int) -> None: pass + +class C: + @dec # E: Decorators on top of @property are not supported + @property + def f(self) -> int: pass + @f.setter + @dec + def f(self, v: int) -> None: pass +[builtins fixtures/property.pyi] +[out] + +[case testInvalidArgCountForProperty] +from typing import Callable, TypeVar + +T = TypeVar("T") +def dec(f: Callable[[T], int]) -> Callable[[T, int], int]: ... + +class A: + @property # E: Too many arguments for property + def f(self, x) -> int: pass + @property # E: Too many arguments for property + @dec + def e(self) -> int: pass + @property + def g() -> int: pass # E: Method must have at least one argument. Did you forget the "self" argument? + @property + def h(self, *args, **kwargs) -> int: pass # OK +[builtins fixtures/property.pyi] +[out] + +[case testSubtypingUnionGenericBounds] +from typing import Callable, TypeVar, Union, Sequence + +TI = TypeVar("TI", bound=int) +TS = TypeVar("TS", bound=str) + +f: Callable[[Sequence[TI]], None] +g: Callable[[Union[Sequence[TI], Sequence[TS]]], None] +f = g + +[case testOverrideDecoratedProperty] +class Base: + @property + def foo(self) -> int: ... + +class decorator: + def __init__(self, fn): + self.fn = fn + def __call__(self, decorated_self) -> int: + return self.fn(decorated_self) + +class Child(Base): + @property + @decorator + def foo(self) -> int: + return 42 + +reveal_type(Child().foo) # N: Revealed type is "builtins.int" + +class BadChild1(Base): + @decorator + def foo(self) -> int: # E: Signature of "foo" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: decorator + return 42 + +class not_a_decorator: + def __init__(self, fn): ... + +class BadChild2(Base): + @property + @not_a_decorator + def foo(self) -> int: # E: "not_a_decorator" not callable \ + # E: Signature of "foo" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: int \ + # N: Subclass: \ + # N: not_a_decorator + return 42 +[builtins fixtures/property.pyi] + +[case explicitOverride] +# flags: --python-version 3.12 +from typing import override + +class A: + def f(self, x: int) -> str: pass + @override + def g(self, x: int) -> str: pass # E: Method "g" is marked as an override, but no base method was found with this name + +class B(A): + @override + def f(self, x: int) -> str: pass + @override + def g(self, x: int) -> str: pass + +class C(A): + @override + def f(self, x: str) -> str: pass # E: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def g(self, x: int) -> str: pass + +class D(A): pass +class E(D): pass +class F(E): + @override + def f(self, x: int) -> str: pass +[typing fixtures/typing-override.pyi] + +[case explicitOverrideStaticmethod] +# flags: --python-version 3.12 +from typing import override + +class A: + @staticmethod + def f(x: int) -> str: pass + +class B(A): + @staticmethod + @override + def f(x: int) -> str: pass + @override + @staticmethod + def g(x: int) -> str: pass # E: Method "g" is marked as an override, but no base method was found with this name + +class C(A): # inverted order of decorators + @override + @staticmethod + def f(x: int) -> str: pass + @override + @staticmethod + def g(x: int) -> str: pass # E: Method "g" is marked as an override, but no base method was found with this name + +class D(A): + @staticmethod + @override + def f(x: str) -> str: pass # E: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[typing fixtures/typing-override.pyi] +[builtins fixtures/staticmethod.pyi] + +[case explicitOverrideClassmethod] +# flags: --python-version 3.12 +from typing import override + +class A: + @classmethod + def f(cls, x: int) -> str: pass + +class B(A): + @classmethod + @override + def f(cls, x: int) -> str: pass + @override + @classmethod + def g(cls, x: int) -> str: pass # E: Method "g" is marked as an override, but no base method was found with this name + +class C(A): # inverted order of decorators + @override + @classmethod + def f(cls, x: int) -> str: pass + @override + @classmethod + def g(cls, x: int) -> str: pass # E: Method "g" is marked as an override, but no base method was found with this name + +class D(A): + @classmethod + @override + def f(cls, x: str) -> str: pass # E: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[typing fixtures/typing-override.pyi] +[builtins fixtures/classmethod.pyi] + +[case explicitOverrideProperty] +# flags: --python-version 3.12 +from typing import override + +class A: + @property + def f(self) -> str: pass + +class B(A): + @property + @override + def f(self) -> str: pass + @override + @property + def g(self) -> str: pass # E: Method "g" is marked as an override, but no base method was found with this name + +class C(A): # inverted order of decorators + @override + @property + def f(self) -> str: pass + @override + @property + def g(self) -> str: pass # E: Method "g" is marked as an override, but no base method was found with this name + +class D(A): + @property + @override + def f(self) -> int: pass # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: str \ + # N: Subclass: \ + # N: int +[typing fixtures/typing-override.pyi] +[builtins fixtures/property.pyi] + +[case explicitOverrideSettableProperty] +# flags: --python-version 3.12 +from typing import override + +class A: + @property + def f(self) -> str: pass + + @f.setter + def f(self, value: str) -> None: pass + +class B(A): + @property # E: Read-only property cannot override read-write property + @override + def f(self) -> str: pass + +class C(A): + @override + @property + def f(self) -> str: pass + + @f.setter + def f(self, value: str) -> None: pass + +class D(A): + @override # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: str \ + # N: Subclass: \ + # N: int + @property + def f(self) -> int: pass + + @f.setter + def f(self, value: int) -> None: pass +[typing fixtures/typing-override.pyi] +[builtins fixtures/property.pyi] + +[case invalidExplicitOverride] +# flags: --python-version 3.12 +from typing import override + +@override # E: "override" used with a non-method +def f(x: int) -> str: pass + +@override # this should probably throw an error but the signature from typeshed should ensure this already +class A: pass + +def g() -> None: + @override # E: "override" used with a non-method + def h(b: bool) -> int: pass +[typing fixtures/typing-override.pyi] + +[case explicitOverrideSpecialMethods] +# flags: --python-version 3.12 +from typing import override + +class A: + def __init__(self, a: int) -> None: pass + +class B(A): + @override + def __init__(self, b: str) -> None: pass + +class C: + @override + def __init__(self, a: int) -> None: pass +[typing fixtures/typing-override.pyi] + +[case explicitOverrideFromExtensions] +from typing_extensions import override + +class A: + def f(self, x: int) -> str: pass + +class B(A): + @override + def f2(self, x: int) -> str: pass # E: Method "f2" is marked as an override, but no base method was found with this name +[builtins fixtures/tuple.pyi] + +[case explicitOverrideOverloads] +# flags: --python-version 3.12 +from typing import overload, override + +class A: + def f(self, x: int) -> str: pass + +class B(A): + @overload # E: Method "f2" is marked as an override, but no base method was found with this name + def f2(self, x: int) -> str: pass + @overload + def f2(self, x: str) -> str: pass + @override + def f2(self, x: int | str) -> str: pass +[typing fixtures/typing-override.pyi] + +[case explicitOverrideNotOnOverloadsImplementation] +# flags: --python-version 3.12 +from typing import overload, override + +class A: + def f(self, x: int) -> str: pass + +class B(A): + @overload # E: Method "f2" is marked as an override, but no base method was found with this name + def f2(self, x: int) -> str: pass + @override + @overload + def f2(self, x: str) -> str: pass + def f2(self, x: int | str) -> str: pass + +class C(A): + @overload + def f(self, y: int) -> str: pass + @override + @overload + def f(self, y: str) -> str: pass + def f(self, y: int | str) -> str: pass +[typing fixtures/typing-override.pyi] + +[case explicitOverrideOnMultipleOverloads] +# flags: --python-version 3.12 +from typing import overload, override + +class A: + def f(self, x: int) -> str: pass + +class B(A): + @override # E: Method "f2" is marked as an override, but no base method was found with this name + @overload + def f2(self, x: int) -> str: pass + @override + @overload + def f2(self, x: str) -> str: pass + def f2(self, x: int | str) -> str: pass + +class C(A): + @overload + def f(self, y: int) -> str: pass + @override + @overload + def f(self, y: str) -> str: pass + @override + def f(self, y: int | str) -> str: pass +[typing fixtures/typing-override.pyi] + +[case explicitOverrideCyclicDependency] +# flags: --python-version 3.12 +import b +[file a.py] +from typing import override +import b +import c + +class A(b.B): + @override # This is fine + @c.deco + def meth(self) -> int: ... +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-override.pyi] + +[case requireExplicitOverrideMethod] +# flags: --enable-error-code explicit-override --python-version 3.12 +from typing import override + +class A: + def f(self, x: int) -> str: pass + +class B(A): + @override + def f(self, y: int) -> str: pass + +class C(A): + def f(self, y: int) -> str: pass # E: Method "f" is not using @override but is overriding a method in class "__main__.A" + +class D(B): + def f(self, y: int) -> str: pass # E: Method "f" is not using @override but is overriding a method in class "__main__.B" +[typing fixtures/typing-override.pyi] + +[case requireExplicitOverrideSpecialMethod] +# flags: --enable-error-code explicit-override --python-version 3.12 +from typing import Callable, Self, TypeVar, override, overload + +T = TypeVar('T') +def some_decorator(f: Callable[..., T]) -> Callable[..., T]: ... + +# Don't require override decorator for __init__ and __new__ +# See: https://github.com/python/typing/issues/1376 +class A: + def __init__(self) -> None: pass + def __new__(cls) -> Self: pass + +class B(A): + def __init__(self) -> None: pass + def __new__(cls) -> Self: pass + +class C(A): + @some_decorator + def __init__(self) -> None: pass + + @some_decorator + def __new__(cls) -> Self: pass + +class D(A): + @overload + def __init__(self, x: int) -> None: ... + @overload + def __init__(self, x: str) -> None: ... + def __init__(self, x): pass + + @overload + def __new__(cls, x: int) -> Self: pass + @overload + def __new__(cls, x: str) -> Self: pass + def __new__(cls, x): pass +[typing fixtures/typing-override.pyi] + +[case requireExplicitOverrideProperty] +# flags: --enable-error-code explicit-override --python-version 3.12 +from typing import override + +class A: + @property + def prop(self) -> int: pass + +class B(A): + @override + @property + def prop(self) -> int: pass + +class C(A): + @property + def prop(self) -> int: pass # E: Method "prop" is not using @override but is overriding a method in class "__main__.A" +[typing fixtures/typing-override.pyi] +[builtins fixtures/property.pyi] + +[case requireExplicitOverrideOverload] +# flags: --enable-error-code explicit-override --python-version 3.12 +from typing import overload, override + +class A: + @overload + def f(self, x: int) -> str: ... + @overload + def f(self, x: str) -> str: ... + def f(self, x): pass + +class B(A): + @overload + def f(self, y: int) -> str: ... + @overload + def f(self, y: str) -> str: ... + @override + def f(self, y): pass + +class C(A): + @overload + @override + def f(self, y: int) -> str: ... + @overload + def f(self, y: str) -> str: ... + def f(self, y): pass + +class D(A): + @overload + def f(self, y: int) -> str: ... + @overload + def f(self, y: str) -> str: ... + def f(self, y): pass # E: Method "f" is not using @override but is overriding a method in class "__main__.A" +[typing fixtures/typing-override.pyi] + +[case requireExplicitOverrideMultipleInheritance] +# flags: --enable-error-code explicit-override --python-version 3.12 +from typing import override + +class A: + def f(self, x: int) -> str: pass +class B: + def f(self, y: int) -> str: pass + +class C(A, B): + @override + def f(self, z: int) -> str: pass + +class D(A, B): + def f(self, z: int) -> str: pass # E: Method "f" is not using @override but is overriding a method in class "__main__.A" +[typing fixtures/typing-override.pyi] + +[case testExplicitOverrideAllowedForPrivate] +# flags: --enable-error-code explicit-override --python-version 3.12 + +class B: + def __f(self, y: int) -> str: pass + +class C(B): + def __f(self, y: int) -> str: pass # OK +[typing fixtures/typing-override.pyi] + +[case testCallableProperty] +from typing import Callable + +class something_callable: + def __call__(self, fn) -> str: ... + +def decorator(fn: Callable[..., int]) -> something_callable: ... + +class A: + @property + @decorator + def f(self) -> int: ... + +reveal_type(A.f) # N: Revealed type is "__main__.something_callable" +reveal_type(A().f) # N: Revealed type is "builtins.str" +[builtins fixtures/property.pyi] + +[case testFinalOverrideOnUntypedDef] +from typing import final + +class Base: + @final + def foo(self): + pass + +class Derived(Base): + def foo(self): # E: Cannot override final attribute "foo" (previously declared in base class "Base") + pass + +[case testTypeVarIdClashPolymorphic] +from typing import Callable, Generic, TypeVar + +A = TypeVar("A") +B = TypeVar("B") + +class Gen(Generic[A]): ... + +def id_(x: A) -> A: ... +def f(x: Gen[A], y: A) -> Gen[Gen[A]]: ... +def g(x: Gen[A], id_: Callable[[B], B], f: Callable[[A, B], Gen[A]]) -> A: ... + +def test(x: Gen[Gen[A]]) -> Gen[A]: + return g(x, id_, f) # Technically OK + +x: Gen[Gen[int]] +reveal_type(g(x, id_, f)) # N: Revealed type is "__main__.Gen[builtins.int]" + +def h(x: A, y: A) -> A: ... +def gn(id_: Callable[[B], B], step: Callable[[A, B], A]) -> A: ... + +def fn(x: A) -> A: + return gn(id_, h) # Technically OK + +[case testTypeVarIdsNested] +from typing import Callable, TypeVar + +A = TypeVar("A") +B = TypeVar("B") + +def f(x: Callable[[A], A]) -> Callable[[B], B]: + def g(x: B) -> B: ... + return g + +reveal_type(f(f)) # N: Revealed type is "def [B] (B`1) -> B`1" +reveal_type(f(f)(f)) # N: Revealed type is "def [A] (x: def (A`-1) -> A`-1) -> def [B] (B`-2) -> B`-2" + +[case testGenericUnionFunctionJoin] +from typing import TypeVar, Union + +T = TypeVar("T") +S = TypeVar("S") + +def f(x: T, y: S) -> Union[T, S]: ... +def g(x: T, y: S) -> Union[T, S]: ... + +x = [f, g] +reveal_type(x) # N: Revealed type is "builtins.list[def [T, S] (x: T`4, y: S`5) -> Union[T`4, S`5]]" +[builtins fixtures/list.pyi] + +[case testTypeVariableClashErrorMessage] +from typing import TypeVar + +T = TypeVar("T") + +class C: # Note: Generic[T] missing + def bad_idea(self, x: T) -> None: + self.x = x + + def nope(self, x: T) -> None: + self.x = x # E: Incompatible types in assignment (expression has type "T@nope", variable has type "T@bad_idea") + +[case testNoCrashOnBadCallablePropertyOverride] +from typing import Callable, Union + +class C: ... +class D: ... + +A = Callable[[C], None] +B = Callable[[D], None] + +class Foo: + @property + def method(self) -> Callable[[int, Union[A, B]], None]: + ... + +class Bar(Foo): + @property + def method(self) -> Callable[[int, A], None]: # E: Argument 2 of "method" is incompatible with supertype "Foo"; supertype defines the argument type as "Union[Callable[[C], None], Callable[[D], None]]" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + ... +[builtins fixtures/property.pyi] + +[case testNoCrashOnUnpackOverride] +from typing import Unpack +from typing_extensions import TypedDict + +class Params(TypedDict): + x: int + y: str + +class Other(TypedDict): + x: int + y: int + +class B: + def meth(self, **kwargs: Unpack[Params]) -> None: + ... +class C(B): + def meth(self, **kwargs: Unpack[Other]) -> None: # E: Signature of "meth" incompatible with supertype "B" \ + # N: Superclass: \ + # N: def meth(*, x: int, y: str) -> None \ + # N: Subclass: \ + # N: def meth(*, x: int, y: int) -> None + + ... +[builtins fixtures/tuple.pyi] + +[case testOverrideErrorLocationNamed] +class B: + def meth( + self, *, + x: int, + y: str, + ) -> None: + ... +class C(B): + def meth( + self, *, + y: int, # E: Argument 1 of "meth" is incompatible with supertype "B"; supertype defines the argument type as "str" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + x: int, + ) -> None: + ... +[builtins fixtures/tuple.pyi] + +[case testLambdaAlwaysAllowed] +# flags: --disallow-untyped-calls +from typing import Callable, Optional + +def func() -> Optional[str]: ... +var: Optional[str] + +factory: Callable[[], Optional[str]] +for factory in ( + lambda: var, + func, +): + reveal_type(factory) # N: Revealed type is "def () -> Union[builtins.str, None]" + var = factory() +[builtins fixtures/tuple.pyi] + +[case testLambdaInDeferredDecoratorNoCrash] +def foo(x): + pass + +class Bar: + def baz(self, x): + pass + +class Qux(Bar): + @foo(lambda x: None) + def baz(self, x) -> None: + pass +[builtins fixtures/tuple.pyi] + +[case testGeneratorInDeferredDecoratorNoCrash] +from typing import Protocol, TypeVar + +T = TypeVar("T", covariant=True) + +class SupportsNext(Protocol[T]): + def __next__(self) -> T: ... + +def next(i: SupportsNext[T]) -> T: ... + +def foo(x): + pass + +class Bar: + def baz(self, x): + pass + +class Qux(Bar): + @next(f for f in [foo]) + def baz(self, x) -> None: + pass +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test new file mode 100644 index 000000000000..c1868b3e3d72 --- /dev/null +++ b/test-data/unit/check-functools.test @@ -0,0 +1,641 @@ +[case testTotalOrderingEqLt] +from functools import total_ordering + +@total_ordering +class Ord: + def __eq__(self, other: object) -> bool: + return False + + def __lt__(self, other: "Ord") -> bool: + return False + +reveal_type(Ord() < Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() <= Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() == Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() > Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() >= Ord()) # N: Revealed type is "builtins.bool" + +Ord() < 1 # E: Unsupported operand types for < ("Ord" and "int") +Ord() <= 1 # E: Unsupported operand types for <= ("Ord" and "int") +Ord() == 1 +Ord() > 1 # E: Unsupported operand types for > ("Ord" and "int") +Ord() >= 1 # E: Unsupported operand types for >= ("Ord" and "int") +[builtins fixtures/dict.pyi] + +[case testTotalOrderingLambda] +from functools import total_ordering +from typing import Any, Callable, ClassVar + +@total_ordering +class Ord: + __eq__: Callable[[Any, object], bool] = lambda self, other: False + __lt__: Callable[[Any, "Ord"], bool] = lambda self, other: False + +reveal_type(Ord() < Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() <= Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() == Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() > Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() >= Ord()) # N: Revealed type is "builtins.bool" + +Ord() < 1 # E: Argument 1 has incompatible type "int"; expected "Ord" +Ord() <= 1 # E: Unsupported operand types for <= ("Ord" and "int") +Ord() == 1 +Ord() > 1 # E: Unsupported operand types for > ("Ord" and "int") +Ord() >= 1 # E: Unsupported operand types for >= ("Ord" and "int") +[builtins fixtures/dict.pyi] + +[case testTotalOrderingNonCallable] +from functools import total_ordering + +@total_ordering +class Ord(object): + def __eq__(self, other: object) -> bool: + return False + + __lt__ = 5 + +Ord() <= Ord() # E: Unsupported left operand type for <= ("Ord") +Ord() > Ord() # E: "int" not callable +Ord() >= Ord() # E: Unsupported left operand type for >= ("Ord") +[builtins fixtures/dict.pyi] + +[case testTotalOrderingReturnNotBool] +from functools import total_ordering + +@total_ordering +class Ord: + def __eq__(self, other: object) -> bool: + return False + + def __lt__(self, other: "Ord") -> str: + return "blah" + +reveal_type(Ord() < Ord()) # N: Revealed type is "builtins.str" +reveal_type(Ord() <= Ord()) # N: Revealed type is "Any" +reveal_type(Ord() == Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() > Ord()) # N: Revealed type is "Any" +reveal_type(Ord() >= Ord()) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] + +[case testTotalOrderingAllowsAny] +from functools import total_ordering + +@total_ordering +class Ord: + def __eq__(self, other): + return False + + def __gt__(self, other): + return False + +reveal_type(Ord() < Ord()) # N: Revealed type is "Any" +Ord() <= Ord() # E: Unsupported left operand type for <= ("Ord") +reveal_type(Ord() == Ord()) # N: Revealed type is "Any" +reveal_type(Ord() > Ord()) # N: Revealed type is "Any" +Ord() >= Ord() # E: Unsupported left operand type for >= ("Ord") + +Ord() < 1 # E: Unsupported left operand type for < ("Ord") +Ord() <= 1 # E: Unsupported left operand type for <= ("Ord") +Ord() == 1 +Ord() > 1 +Ord() >= 1 # E: Unsupported left operand type for >= ("Ord") +[builtins fixtures/dict.pyi] + +[case testCachedProperty] +from functools import cached_property +class Parent: + @property + def f(self) -> str: pass +class Child(Parent): + @cached_property + def f(self) -> str: pass + @cached_property + def g(self) -> int: pass + @cached_property # E: Too many arguments for property + def h(self, arg) -> int: pass +reveal_type(Parent().f) # N: Revealed type is "builtins.str" +reveal_type(Child().f) # N: Revealed type is "builtins.str" +reveal_type(Child().g) # N: Revealed type is "builtins.int" +Child().f = "Hello World" +Child().g = "invalid" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[file functools.pyi] +import sys +from typing import TypeVar, Generic +_T = TypeVar('_T') +class cached_property(Generic[_T]): ... +[builtins fixtures/property.pyi] + +[case testTotalOrderingWithForwardReference] +from typing import Generic, Any, TypeVar +import functools + +T = TypeVar("T", bound="C") + +@functools.total_ordering +class D(Generic[T]): + def __lt__(self, other: Any) -> bool: + ... + +class C: + pass + +def f(d: D[C]) -> None: + reveal_type(d.__gt__) # N: Revealed type is "def (other: Any) -> builtins.bool" + +d: D[int] # E: Type argument "int" of "D" must be a subtype of "C" +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialBasic] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... # N: "foo" defined here + +p1 = functools.partial(foo) +p1(1, "a", 3) # OK +p1(1, "a", c=3) # OK +p1(1, b="a", c=3) # OK + +reveal_type(p1) # N: Revealed type is "functools.partial[builtins.int]" + +def takes_callable_int(f: Callable[..., int]) -> None: ... +def takes_callable_str(f: Callable[..., str]) -> None: ... +takes_callable_int(p1) +takes_callable_str(p1) # E: Argument 1 to "takes_callable_str" has incompatible type "partial[int]"; expected "Callable[..., str]" \ + # N: "partial[int].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], int]" + +p2 = functools.partial(foo, 1) +p2("a") # OK +p2("a", 3) # OK +p2("a", c=3) # OK +p2(1, 3) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" +p2(1, "a", 3) # E: Too many arguments for "foo" \ + # E: Argument 1 to "foo" has incompatible type "int"; expected "str" \ + # E: Argument 2 to "foo" has incompatible type "str"; expected "int" +p2(a=1, b="a", c=3) # E: Unexpected keyword argument "a" for "foo" + +p3 = functools.partial(foo, b="a") +p3(1) # OK +p3(1, c=3) # OK +p3(a=1) # OK +p3(1, b="a", c=3) # OK, keywords can be clobbered +p3(1, 3) # E: Too many positional arguments for "foo" \ + # E: Argument 2 to "foo" has incompatible type "int"; expected "str" + +functools.partial(foo, "a") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" +functools.partial(foo, b=1) # E: Argument "b" to "foo" has incompatible type "int"; expected "str" +functools.partial(foo, a=1, b=2, c=3) # E: Argument "b" to "foo" has incompatible type "int"; expected "str" +functools.partial(1) # E: "int" not callable \ + # E: Argument 1 to "partial" has incompatible type "int"; expected "Callable[..., Never]" +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialStar] +import functools +from typing import List + +def foo(a: int, b: str, *args: int, d: str, **kwargs: int) -> int: ... + +p1 = functools.partial(foo, 1, d="a", x=9) +p1("a", 2, 3, 4) # OK +p1("a", 2, 3, 4, d="a") # OK +p1("a", 2, 3, 4, "a") # E: Argument 5 to "foo" has incompatible type "str"; expected "int" +p1("a", 2, 3, 4, x="a") # E: Argument "x" to "foo" has incompatible type "str"; expected "int" + +p2 = functools.partial(foo, 1, "a") +p2(2, 3, 4, d="a") # OK +p2("a") # E: Missing named argument "d" for "foo" \ + # E: Argument 1 to "foo" has incompatible type "str"; expected "int" +p2(2, 3, 4) # E: Missing named argument "d" for "foo" + +functools.partial(foo, 1, "a", "b", "c", d="a") # E: Argument 3 to "foo" has incompatible type "str"; expected "int" \ + # E: Argument 4 to "foo" has incompatible type "str"; expected "int" + +def bar(*a: bytes, **k: int): + p1("a", 2, 3, 4, d="a", **k) + p1("a", d="a", **k) + p1("a", **k) # E: Argument 2 to "foo" has incompatible type "**Dict[str, int]"; expected "str" + p1(**k) # E: Argument 1 to "foo" has incompatible type "**Dict[str, int]"; expected "str" + p1(*a) # E: Expected iterable as variadic argument + + +def baz(a: int, b: int) -> int: ... +def test_baz(xs: List[int]): + p3 = functools.partial(baz, *xs) + p3() + p3(1) # E: Too many arguments for "baz" + + +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialGeneric] +from typing import TypeVar +import functools + +T = TypeVar("T") +U = TypeVar("U") + +def foo(a: T, b: T) -> T: ... + +p1 = functools.partial(foo, 1) +reveal_type(p1(2)) # N: Revealed type is "builtins.int" +p1("a") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" + +p2 = functools.partial(foo, "a") +p2(1) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" +reveal_type(p2("a")) # N: Revealed type is "builtins.str" + +def bar(a: T, b: U) -> U: ... + +p3 = functools.partial(bar, 1) +reveal_type(p3(2)) # N: Revealed type is "builtins.int" +reveal_type(p3("a")) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialCallable] +from typing import Callable +import functools + +def main1(f: Callable[[int, str], int]) -> None: + p = functools.partial(f, 1) + p("a") # OK + p(1) # E: Argument 1 has incompatible type "int"; expected "str" + + functools.partial(f, a=1) # E: Unexpected keyword argument "a" + +class CallbackProto: + def __call__(self, a: int, b: str) -> int: ... + +def main2(f: CallbackProto) -> None: + p = functools.partial(f, b="a") + p(1) # OK + p("a") # E: Argument 1 to "__call__" of "CallbackProto" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialOverload] +from typing import overload +import functools + +@overload +def foo(a: int, b: str) -> int: ... +@overload +def foo(a: str, b: int) -> str: ... +def foo(*a, **k): ... + +p1 = functools.partial(foo) +reveal_type(p1(1, "a")) # N: Revealed type is "builtins.int" +reveal_type(p1("a", 1)) # N: Revealed type is "builtins.int" +p1(1, 2) # TODO: false negative +p1("a", "b") # TODO: false negative +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialTypeGuard] +import functools +from typing_extensions import TypeGuard + +def is_str_list(val: list[object]) -> TypeGuard[list[str]]: ... # E: "list" is not subscriptable, use "typing.List" instead + +reveal_type(functools.partial(is_str_list, [1, 2, 3])) # N: Revealed type is "functools.partial[builtins.bool]" +reveal_type(functools.partial(is_str_list, [1, 2, 3])()) # N: Revealed type is "builtins.bool" +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialType] +import functools +from typing import Type + +class A: + def __init__(self, a: int, b: str) -> None: ... # N: "A" defined here + +p = functools.partial(A, 1) +reveal_type(p) # N: Revealed type is "functools.partial[__main__.A]" + +p("a") # OK +p(1) # E: Argument 1 to "A" has incompatible type "int"; expected "str" +p(z=1) # E: Unexpected keyword argument "z" for "A" + +def main(t: Type[A]) -> None: + p = functools.partial(t, 1) + reveal_type(p) # N: Revealed type is "functools.partial[__main__.A]" + + p("a") # OK + p(1) # E: Argument 1 to "A" has incompatible type "int"; expected "str" + p(z=1) # E: Unexpected keyword argument "z" for "A" + +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialTypeVarTuple] +import functools +import typing +Ts = typing.TypeVarTuple("Ts") +def foo(fn: typing.Callable[[typing.Unpack[Ts]], None], /, *arg: typing.Unpack[Ts], kwarg: str) -> None: ... +p = functools.partial(foo, kwarg="asdf") + +def bar(a: int, b: str, c: float) -> None: ... +p(bar, 1, "a", 3.0) # OK +p(bar, 1, "a", 3.0, kwarg="asdf") # OK +p(bar, 1, "a", "b") # E: Argument 1 to "foo" has incompatible type "Callable[[int, str, float], None]"; expected "Callable[[int, str, str], None]" +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialUnion] +import functools +from typing import Any, Callable, Union + +cls1: Any +cls2: Union[Any, Any] +reveal_type(functools.partial(cls1, 2)()) # N: Revealed type is "Any" +reveal_type(functools.partial(cls2, 2)()) # N: Revealed type is "Any" + +fn1: Union[Callable[[int], int], Callable[[int], int]] +reveal_type(functools.partial(fn1, 2)()) # N: Revealed type is "builtins.int" + +fn2: Union[Callable[[int], int], Callable[[int], str]] +reveal_type(functools.partial(fn2, 2)()) # N: Revealed type is "Union[builtins.int, builtins.str]" + +fn3: Union[Callable[[int], int], str] +reveal_type(functools.partial(fn3, 2)()) # E: "str" not callable \ + # N: Revealed type is "builtins.int" \ + # E: Argument 1 to "partial" has incompatible type "Union[Callable[[int], int], str]"; expected "Callable[..., int]" +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialUnionOfTypeAndCallable] +import functools +from typing import Callable, Union, Type +from typing_extensions import TypeAlias + +class FooBar: + def __init__(self, arg1: str) -> None: + pass + +def f1(t: Union[Type[FooBar], Callable[..., 'FooBar']]) -> None: + val = functools.partial(t) + +FooBarFunc: TypeAlias = Callable[..., 'FooBar'] + +def f2(t: Union[Type[FooBar], FooBarFunc]) -> None: + val = functools.partial(t) +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialExplicitType] +from functools import partial +from typing import Type, TypeVar, Callable + +T = TypeVar("T") +def generic(string: str, integer: int, resulting_type: Type[T]) -> T: ... + +p: partial[str] = partial(generic, resulting_type=str) +q: partial[bool] = partial(generic, resulting_type=str) # E: Argument "resulting_type" to "generic" has incompatible type "Type[str]"; expected "Type[bool]" + +pc: Callable[..., str] = partial(generic, resulting_type=str) +qc: Callable[..., bool] = partial(generic, resulting_type=str) # E: Incompatible types in assignment (expression has type "partial[str]", variable has type "Callable[..., bool]") \ + # N: "partial[str].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], str]" +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialNestedPartial] +from functools import partial +from typing import Any + +def foo(x: int) -> int: ... +p = partial(partial, foo) +reveal_type(p()(1)) # N: Revealed type is "builtins.int" +p()("no") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" + +q = partial(partial, partial, foo) +q()()("no") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" + +r = partial(partial, foo, 1) +reveal_type(r()()) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialTypeObject] +import functools +from typing import Type, Generic, TypeVar + +class A: + def __init__(self, val: int) -> None: ... + +cls1: Type[A] +reveal_type(functools.partial(cls1, 2)()) # N: Revealed type is "__main__.A" +functools.partial(cls1, "asdf") # E: Argument 1 to "A" has incompatible type "str"; expected "int" + +T = TypeVar("T") +class B(Generic[T]): + def __init__(self, val: T) -> None: ... + +cls2: Type[B[int]] +reveal_type(functools.partial(cls2, 2)()) # N: Revealed type is "__main__.B[builtins.int]" +functools.partial(cls2, "asdf") # E: Argument 1 to "B" has incompatible type "str"; expected "int" + +def foo(cls3: Type[B[T]]): + reveal_type(functools.partial(cls3, "asdf")) # N: Revealed type is "functools.partial[__main__.B[T`-1]]" \ + # E: Argument 1 to "B" has incompatible type "str"; expected "T" + reveal_type(functools.partial(cls3, 2)()) # N: Revealed type is "__main__.B[T`-1]" \ + # E: Argument 1 to "B" has incompatible type "int"; expected "T" +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialTypedDictUnpack] +from typing_extensions import TypedDict, Unpack +from functools import partial + +class D1(TypedDict, total=False): + a1: int + +def fn1(a1: int) -> None: ... # N: "fn1" defined here +def main1(**d1: Unpack[D1]) -> None: + partial(fn1, **d1)() + partial(fn1, **d1)(**d1) + partial(fn1, **d1)(a1=1) + partial(fn1, **d1)(a1="asdf") # E: Argument "a1" to "fn1" has incompatible type "str"; expected "int" + partial(fn1, **d1)(oops=1) # E: Unexpected keyword argument "oops" for "fn1" + +def fn2(**kwargs: Unpack[D1]) -> None: ... # N: "fn2" defined here +def main2(**d1: Unpack[D1]) -> None: + partial(fn2, **d1)() + partial(fn2, **d1)(**d1) + partial(fn2, **d1)(a1=1) + partial(fn2, **d1)(a1="asdf") # E: Argument "a1" to "fn2" has incompatible type "str"; expected "int" + partial(fn2, **d1)(oops=1) # E: Unexpected keyword argument "oops" for "fn2" + +class D2(TypedDict, total=False): + a1: int + a2: str + +class A2Good(TypedDict, total=False): + a2: str +class A2Bad(TypedDict, total=False): + a2: int + +def fn3(a1: int, a2: str) -> None: ... # N: "fn3" defined here +def main3(a2good: A2Good, a2bad: A2Bad, **d2: Unpack[D2]) -> None: + partial(fn3, **d2)() + partial(fn3, **d2)(a1=1, a2="asdf") + + partial(fn3, **d2)(**d2) + + partial(fn3, **d2)(a1="asdf") # E: Argument "a1" to "fn3" has incompatible type "str"; expected "int" + partial(fn3, **d2)(a1=1, a2="asdf", oops=1) # E: Unexpected keyword argument "oops" for "fn3" + + partial(fn3, **d2)(**a2good) + partial(fn3, **d2)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + +def fn4(**kwargs: Unpack[D2]) -> None: ... # N: "fn4" defined here +def main4(a2good: A2Good, a2bad: A2Bad, **d2: Unpack[D2]) -> None: + partial(fn4, **d2)() + partial(fn4, **d2)(a1=1, a2="asdf") + + partial(fn4, **d2)(**d2) + + partial(fn4, **d2)(a1="asdf") # E: Argument "a1" to "fn4" has incompatible type "str"; expected "int" + partial(fn4, **d2)(a1=1, a2="asdf", oops=1) # E: Unexpected keyword argument "oops" for "fn4" + + partial(fn3, **d2)(**a2good) + partial(fn3, **d2)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + +def main5(**d2: Unpack[D2]) -> None: + partial(fn1, **d2)() # E: Extra argument "a2" from **args for "fn1" + partial(fn2, **d2)() # E: Extra argument "a2" from **args for "fn2" + +def main6(a2good: A2Good, a2bad: A2Bad, **d1: Unpack[D1]) -> None: + partial(fn3, **d1)() # E: Missing positional argument "a1" in call to "fn3" + partial(fn3, **d1)("asdf") # E: Too many positional arguments for "fn3" \ + # E: Too few arguments for "fn3" \ + # E: Argument 1 to "fn3" has incompatible type "str"; expected "int" + partial(fn3, **d1)(a2="asdf") + partial(fn3, **d1)(**a2good) + partial(fn3, **d1)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + + partial(fn4, **d1)() + partial(fn4, **d1)("asdf") # E: Too many positional arguments for "fn4" \ + # E: Argument 1 to "fn4" has incompatible type "str"; expected "int" + partial(fn4, **d1)(a2="asdf") + partial(fn4, **d1)(**a2good) + partial(fn4, **d1)(**a2bad) # E: Argument "a2" to "fn4" has incompatible type "int"; expected "str" + +[builtins fixtures/dict.pyi] + + +[case testFunctoolsPartialNestedGeneric] +from functools import partial +from typing import Generic, TypeVar, List + +T = TypeVar("T") +def get(n: int, args: List[T]) -> T: ... +first = partial(get, 0) + +x: List[str] +reveal_type(first(x)) # N: Revealed type is "builtins.str" +reveal_type(first([1])) # N: Revealed type is "builtins.int" + +first_kw = partial(get, n=0) +reveal_type(first_kw(args=[1])) # N: Revealed type is "builtins.int" + +# TODO: this is indeed invalid, but the error is incomprehensible. +first_kw([1]) # E: Too many positional arguments for "get" \ + # E: Too few arguments for "get" \ + # E: Argument 1 to "get" has incompatible type "List[int]"; expected "int" +[builtins fixtures/list.pyi] + +[case testFunctoolsPartialHigherOrder] +from functools import partial +from typing import Callable + +def fn(a: int, b: str, c: bytes) -> int: ... + +def callback1(fn: Callable[[str, bytes], int]) -> None: ... +def callback2(fn: Callable[[str, int], int]) -> None: ... + +callback1(partial(fn, 1)) +# TODO: false negative +# https://github.com/python/mypy/issues/17461 +callback2(partial(fn, 1)) +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialClassObjectMatchingPartial] +from functools import partial + +class A: + def __init__(self, var: int, b: int, c: int) -> None: ... + +p = partial(A, 1) +reveal_type(p) # N: Revealed type is "functools.partial[__main__.A]" +p(1, "no") # E: Argument 2 to "A" has incompatible type "str"; expected "int" + +q: partial[A] = partial(A, 1) # OK +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialTypeVarBound] +from typing import Callable, TypeVar, Type +import functools + +T = TypeVar("T", bound=Callable[[str, int], str]) +S = TypeVar("S", bound=Type[int]) + +def foo(f: T) -> T: + g = functools.partial(f, "foo") + return f + +def bar(f: S) -> S: + g = functools.partial(f, "foo") + return f +[builtins fixtures/primitives.pyi] + +[case testFunctoolsPartialAbstractType] +# flags: --python-version 3.9 +from abc import ABC, abstractmethod +from functools import partial + +class A(ABC): + def __init__(self) -> None: ... + @abstractmethod + def method(self) -> None: ... + +def f1(cls: type[A]) -> None: + cls() + partial_cls = partial(cls) + partial_cls() + +def f2() -> None: + A() # E: Cannot instantiate abstract class "A" with abstract attribute "method" + partial_cls = partial(A) # E: Cannot instantiate abstract class "A" with abstract attribute "method" + partial_cls() # E: Cannot instantiate abstract class "A" with abstract attribute "method" +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialSelfType] +from functools import partial +from typing_extensions import Self + +class A: + def __init__(self, ts: float, msg: str) -> None: ... + + @classmethod + def from_msg(cls, msg: str) -> Self: + factory = partial(cls, ts=0) + return factory(msg=msg) +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialTypeVarValues] +from functools import partial +from typing import TypeVar + +T = TypeVar("T", int, str) + +def f(x: int, y: T) -> T: + return y + +def g(x: T, y: int) -> T: + return x + +def h(x: T, y: T) -> T: + return x + +fp = partial(f, 1) +reveal_type(fp(1)) # N: Revealed type is "builtins.int" +reveal_type(fp("a")) # N: Revealed type is "builtins.str" +fp(object()) # E: Value of type variable "T" of "f" cannot be "object" + +gp = partial(g, 1) +reveal_type(gp(1)) # N: Revealed type is "builtins.int" +gp("a") # E: Argument 1 to "g" has incompatible type "str"; expected "int" + +hp = partial(h, 1) +reveal_type(hp(1)) # N: Revealed type is "builtins.int" +hp("a") # E: Argument 1 to "h" has incompatible type "str"; expected "int" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-generic-alias.test b/test-data/unit/check-generic-alias.test new file mode 100644 index 000000000000..3ae815a5cd48 --- /dev/null +++ b/test-data/unit/check-generic-alias.test @@ -0,0 +1,284 @@ +-- Test cases for generic aliases + +[case testGenericBuiltinWarning] +# flags: --python-version 3.8 +t1: list +t2: list[int] # E: "list" is not subscriptable, use "typing.List" instead +t3: list[str] # E: "list" is not subscriptable, use "typing.List" instead + +t4: tuple +t5: tuple[int] # E: "tuple" is not subscriptable, use "typing.Tuple" instead +t6: tuple[int, str] # E: "tuple" is not subscriptable, use "typing.Tuple" instead +t7: tuple[int, ...] # E: Unexpected "..." \ + # E: "tuple" is not subscriptable, use "typing.Tuple" instead + +t8: dict = {} +t9: dict[int, str] # E: "dict" is not subscriptable, use "typing.Dict" instead + +t10: type +t11: type[int] # E: "type" expects no type arguments, but 1 given +[builtins fixtures/dict.pyi] + + +[case testGenericBuiltinSetWarning] +# flags: --python-version 3.8 +t1: set +t2: set[int] # E: "set" is not subscriptable, use "typing.Set" instead +[builtins fixtures/set.pyi] + + +[case testGenericCollectionsWarning] +# flags: --python-version 3.8 +import collections + +t01: collections.deque +t02: collections.deque[int] # E: "deque" is not subscriptable, use "typing.Deque" instead +t03: collections.defaultdict +t04: collections.defaultdict[int, str] # E: "defaultdict" is not subscriptable, use "typing.DefaultDict" instead +t05: collections.OrderedDict +t06: collections.OrderedDict[int, str] +t07: collections.Counter +t08: collections.Counter[int] # E: "Counter" is not subscriptable, use "typing.Counter" instead +t09: collections.ChainMap +t10: collections.ChainMap[int, str] # E: "ChainMap" is not subscriptable, use "typing.ChainMap" instead + + +[case testGenericBuiltinFutureAnnotations] +from __future__ import annotations +t1: list +t2: list[int] +t3: list[str] + +t4: tuple +t5: tuple[int] +t6: tuple[int, str] +t7: tuple[int, ...] + +t8: dict = {} +t9: dict[int, str] + +t10: type +t11: type[int] +[builtins fixtures/dict.pyi] + + +[case testGenericCollectionsFutureAnnotations] +from __future__ import annotations +import collections + +t01: collections.deque +t02: collections.deque[int] +t03: collections.defaultdict +t04: collections.defaultdict[int, str] +t05: collections.OrderedDict +t06: collections.OrderedDict[int, str] +t07: collections.Counter +t08: collections.Counter[int] +t09: collections.ChainMap +t10: collections.ChainMap[int, str] +[builtins fixtures/tuple.pyi] + + +[case testGenericAliasBuiltinsReveal] +# flags: --python-version 3.9 +t1: list +t2: list[int] +t3: list[str] + +t4: tuple +t5: tuple[int] +t6: tuple[int, str] +t7: tuple[int, ...] + +t8: dict = {} +t9: dict[int, str] + +t10: type +t11: type[int] + +reveal_type(t1) # N: Revealed type is "builtins.list[Any]" +reveal_type(t2) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(t3) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type(t4) # N: Revealed type is "builtins.tuple[Any, ...]" +# TODO: ideally these would reveal builtins.tuple +reveal_type(t5) # N: Revealed type is "Tuple[builtins.int]" +reveal_type(t6) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +# TODO: this is incorrect, see #9522 +reveal_type(t7) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(t8) # N: Revealed type is "builtins.dict[Any, Any]" +reveal_type(t9) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" +reveal_type(t10) # N: Revealed type is "builtins.type" +reveal_type(t11) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/dict.pyi] + + +[case testGenericAliasBuiltinsSetReveal] +# flags: --python-version 3.9 +t1: set +t2: set[int] +t3: set[str] + +reveal_type(t1) # N: Revealed type is "builtins.set[Any]" +reveal_type(t2) # N: Revealed type is "builtins.set[builtins.int]" +reveal_type(t3) # N: Revealed type is "builtins.set[builtins.str]" +[builtins fixtures/set.pyi] + + +[case testGenericAliasCollectionsReveal] +# flags: --python-version 3.9 +import collections + +t1: collections.deque[int] +t2: collections.defaultdict[int, str] +t3: collections.OrderedDict[int, str] +t4: collections.Counter[int] +t5: collections.ChainMap[int, str] + +reveal_type(t1) # N: Revealed type is "collections.deque[builtins.int]" +reveal_type(t2) # N: Revealed type is "collections.defaultdict[builtins.int, builtins.str]" +reveal_type(t3) # N: Revealed type is "collections.OrderedDict[builtins.int, builtins.str]" +reveal_type(t4) # N: Revealed type is "collections.Counter[builtins.int]" +reveal_type(t5) # N: Revealed type is "collections.ChainMap[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + + +[case testGenericAliasCollectionsABCReveal] +# flags: --python-version 3.9 +import collections.abc + +t01: collections.abc.Awaitable[int] +t02: collections.abc.Coroutine[str, int, float] +t03: collections.abc.AsyncIterable[int] +t04: collections.abc.AsyncIterator[int] +t05: collections.abc.AsyncGenerator[int, float] +t06: collections.abc.Iterable[int] +t07: collections.abc.Iterator[int] +t08: collections.abc.Generator[int, float, str] +t09: collections.abc.Reversible[int] +t10: collections.abc.Container[int] +t11: collections.abc.Collection[int] +t12: collections.abc.Callable[[int], float] +t13: collections.abc.Set[int] +t14: collections.abc.MutableSet[int] +t15: collections.abc.Mapping[int, str] +t16: collections.abc.MutableMapping[int, str] +t17: collections.abc.Sequence[int] +t18: collections.abc.MutableSequence[int] +t19: collections.abc.ByteString +t20: collections.abc.MappingView[int, int] +t21: collections.abc.KeysView[int] +t22: collections.abc.ItemsView[int, str] +t23: collections.abc.ValuesView[str] + +# TODO: these currently reveal the classes from typing, see #7907 +# reveal_type(t01) # Nx Revealed type is "collections.abc.Awaitable[builtins.int]" +# reveal_type(t02) # Nx Revealed type is "collections.abc.Coroutine[builtins.str, builtins.int, builtins.float]" +# reveal_type(t03) # Nx Revealed type is "collections.abc.AsyncIterable[builtins.int]" +# reveal_type(t04) # Nx Revealed type is "collections.abc.AsyncIterator[builtins.int]" +# reveal_type(t05) # Nx Revealed type is "collections.abc.AsyncGenerator[builtins.int, builtins.float]" +# reveal_type(t06) # Nx Revealed type is "collections.abc.Iterable[builtins.int]" +# reveal_type(t07) # Nx Revealed type is "collections.abc.Iterator[builtins.int]" +# reveal_type(t08) # Nx Revealed type is "collections.abc.Generator[builtins.int, builtins.float, builtins.str]" +# reveal_type(t09) # Nx Revealed type is "collections.abc.Reversible[builtins.int]" +# reveal_type(t10) # Nx Revealed type is "collections.abc.Container[builtins.int]" +# reveal_type(t11) # Nx Revealed type is "collections.abc.Collection[builtins.int]" +# reveal_type(t12) # Nx Revealed type is "collections.abc.Callable[[builtins.int], builtins.float]" +# reveal_type(t13) # Nx Revealed type is "collections.abc.Set[builtins.int]" +# reveal_type(t14) # Nx Revealed type is "collections.abc.MutableSet[builtins.int]" +# reveal_type(t15) # Nx Revealed type is "collections.abc.Mapping[builtins.int, builtins.str]" +# reveal_type(t16) # Nx Revealed type is "collections.abc.MutableMapping[builtins.int, builtins.str]" +# reveal_type(t17) # Nx Revealed type is "collections.abc.Sequence[builtins.int]" +# reveal_type(t18) # Nx Revealed type is "collections.abc.MutableSequence[builtins.int]" +# reveal_type(t19) # Nx Revealed type is "collections.abc.ByteString" +# reveal_type(t20) # Nx Revealed type is "collections.abc.MappingView[builtins.int, builtins.int]" +# reveal_type(t21) # Nx Revealed type is "collections.abc.KeysView[builtins.int]" +# reveal_type(t22) # Nx Revealed type is "collections.abc.ItemsView[builtins.int, builtins.str]" +# reveal_type(t23) # Nx Revealed type is "collections.abc.ValuesView[builtins.str]" +[builtins fixtures/tuple.pyi] + + +[case testGenericBuiltinTupleTyping] +from typing import Tuple + +t01: Tuple = () +t02: Tuple[int] = (1, ) +t03: Tuple[int, str] = (1, 'a') +t04: Tuple[int, int] = (1, 2) +t05: Tuple[int, int, int] = (1, 2, 3) +t06: Tuple[int, ...] +t07: Tuple[int, ...] = (1,) +t08: Tuple[int, ...] = (1, 2) +t09: Tuple[int, ...] = (1, 2, 3) +[builtins fixtures/tuple.pyi] + + +[case testGenericBuiltinTuple] +# flags: --python-version 3.9 + +t01: tuple = () +t02: tuple[int] = (1, ) +t03: tuple[int, str] = (1, 'a') +t04: tuple[int, int] = (1, 2) +t05: tuple[int, int, int] = (1, 2, 3) +t06: tuple[int, ...] +t07: tuple[int, ...] = (1,) +t08: tuple[int, ...] = (1, 2) +t09: tuple[int, ...] = (1, 2, 3) + +from typing import Tuple +t10: Tuple[int, ...] = t09 +[builtins fixtures/tuple.pyi] + +[case testTypeAliasWithBuiltinTuple] +# flags: --python-version 3.9 + +A = tuple[int, ...] +a: A = () +b: A = (1, 2, 3) +c: A = ('x', 'y') # E: Incompatible types in assignment (expression has type "Tuple[str, str]", variable has type "Tuple[int, ...]") + +B = tuple[int, str] +x: B = (1, 'x') +y: B = ('x', 1) # E: Incompatible types in assignment (expression has type "Tuple[str, int]", variable has type "Tuple[int, str]") + +reveal_type(tuple[int, ...]()) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testTypeAliasWithBuiltinTupleInStub] +import m +reveal_type(m.a) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(m.b) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +[file m.pyi] +A = tuple[int, ...] +a: A +B = tuple[int, str] +b: B +[builtins fixtures/tuple.pyi] + +[case testTypeAliasWithBuiltinListInStub] +import m +reveal_type(m.a) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(m.b) # N: Revealed type is "builtins.list[builtins.list[builtins.int]]" +m.C # has complex representation, ignored +reveal_type(m.d) # N: Revealed type is "Type[builtins.str]" + +[file m.pyi] +A = list[int] +a: A +B = list[list[int]] +b: B +class C(list[int]): + pass +d: type[str] +[builtins fixtures/list.pyi] + + +[case testTypeAliasWithBuiltinListAliasInStub] +import m +reveal_type(m.a()[0]) # N: Revealed type is "builtins.int" + +[file m.pyi] +List = list +a = List[int] +[builtins fixtures/list.pyi] diff --git a/test-data/unit/check-generic-subtyping.test b/test-data/unit/check-generic-subtyping.test new file mode 100644 index 000000000000..90180e0f83f6 --- /dev/null +++ b/test-data/unit/check-generic-subtyping.test @@ -0,0 +1,1067 @@ +-- Test cases for the type checker related to subtyping and inheritance with +-- generics. + + +-- Subtyping + inheritance +-- ----------------------- + + +[case testSubtypingAndInheritingNonGenericTypeFromGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +ac: A[C] +ad: A[D] +b: B + +if int(): + b = ad # E: Incompatible types in assignment (expression has type "A[D]", variable has type "B") + ad = b # E: Incompatible types in assignment (expression has type "B", variable has type "A[D]") +if int(): + b = ac # E: Incompatible types in assignment (expression has type "A[C]", variable has type "B") + +if int(): + b = b + ac = b + +class C: pass +class A(Generic[T]): pass +class B(A[C]): pass +class D: pass + +[case testSubtypingAndInheritingGenericTypeFromNonGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +a: A +bc: B[C] +bd: B[D] + +if int(): + bc = bd # E: Incompatible types in assignment (expression has type "B[D]", variable has type "B[C]") + bd = bc # E: Incompatible types in assignment (expression has type "B[C]", variable has type "B[D]") +if int(): + bc = a # E: Incompatible types in assignment (expression has type "A", variable has type "B[C]") + bd = a # E: Incompatible types in assignment (expression has type "A", variable has type "B[D]") + +if int(): + a = bc +if int(): + a = bd + +class A: pass +class B(A, Generic[T]): pass +class C: pass +class D: pass + +[case testSubtypingAndInheritingGenericTypeFromGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +ac: A[C] +ad: A[D] +bcc: B[C, C] +bdc: B[D, C] + +if int(): + ad = bcc # E: Incompatible types in assignment (expression has type "B[C, C]", variable has type "A[D]") +if int(): + ad = bdc # E: Incompatible types in assignment (expression has type "B[D, C]", variable has type "A[D]") + bcc = ac # E: Incompatible types in assignment (expression has type "A[C]", variable has type "B[C, C]") + bdc = ac # E: Incompatible types in assignment (expression has type "A[C]", variable has type "B[D, C]") + +if int(): + bcc = bcc + bdc = bdc + ac = bcc +if int(): + ac = bdc + +class A(Generic[T]): pass +class B(A[S], Generic[T, S]): pass +class C: pass +class D: pass + +[case testSubtypingAndInheritingGenericTypeFromGenericTypeAcrossHierarchy] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +X = TypeVar('X') +Y = TypeVar('Y') +ae: A[A[E]] +af: A[A[F]] + +cef: C[E, F] +cff: C[F, F] +cfe: C[F, E] + +if int(): + ae = cef # E: Incompatible types in assignment (expression has type "C[E, F]", variable has type "A[A[E]]") + af = cfe # E: Incompatible types in assignment (expression has type "C[F, E]", variable has type "A[A[F]]") + +if int(): + ae = cfe + af = cef +if int(): + af = cff + +class A(Generic[T]): pass +class B(A[S], Generic[T, S]): pass +class C(B[A[X], A[Y]], Generic[X, Y]): pass +class E: pass +class F: pass + +[case testIncludingBaseClassTwice] +from typing import TypeVar, Generic +t = TypeVar('t') +class I(Generic[t]): pass +class A(I[C], I[object]): pass # E: Duplicate base class "I" +class C: pass + + +-- Accessing inherited generic members +-- ----------------------------------- + + +[case testAccessingMethodInheritedFromGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +b: B[C, D] +c: C +d: D + +b.f(c) # E: Argument 1 to "f" of "A" has incompatible type "C"; expected "D" +b.f(d) + +class A(Generic[T]): + def f(self, a: T) -> None: + pass +class B(A[S], Generic[T, S]): pass +class C: pass +class D: pass +[builtins fixtures/tuple.pyi] + +[case testAccessingMethodInheritedFromGenericTypeInNonGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +b: B +c: C +d: D + +b.f(c) # E: Argument 1 to "f" of "A" has incompatible type "C"; expected "D" +b.f(d) + +class C: pass +class D: pass +class A(Generic[T]): + def f(self, a: T) -> None: + pass +class B(A[D]): pass +[builtins fixtures/tuple.pyi] + +[case testAccessingMemberVarInheritedFromGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T]): + def __init__(self, a: T) -> None: + self.a = a + +b: B[C, D] +c: C +d: D + +b.a = c # E: Incompatible types in assignment (expression has type "C", variable has type "D") +b.a = d + +class B(A[S], Generic[T, S]): pass +class C: pass +class D: pass +[builtins fixtures/tuple.pyi] + + +-- Overriding with generic types +-- ----------------------------- + + +[case testOverridingMethodInSimpleTypeInheritingGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +class B(Generic[T]): + def f(self, a: T) -> None: pass + def g(self, a: T) -> None: pass +class C: pass +class D: pass +class A(B[C]): + def f(self, a: D) -> None: pass \ + # E: Argument 1 of "f" is incompatible with supertype "B"; supertype defines the argument type as "C" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def g(self, a: C) -> None: pass +[out] + +[case testOverridingMethodInGenericTypeInheritingSimpleType] +from typing import TypeVar, Generic +T = TypeVar('T') +class C: pass +class B: + def f(self, a: C) -> None: pass + def g(self, a: C) -> None: pass +class A(B, Generic[T]): + def f(self, a: T) -> None: pass \ + # E: Argument 1 of "f" is incompatible with supertype "B"; supertype defines the argument type as "C" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def g(self, a: 'C') -> None: pass +[out] + +[case testOverridingMethodInGenericTypeInheritingGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class B(Generic[T]): + def f(self, a: T) -> None: pass + def g(self, a: T) -> None: pass +class A(B[S], Generic[T, S]): + def f(self, a: T) -> None: pass \ + # E: Argument 1 of "f" is incompatible with supertype "B"; supertype defines the argument type as "S" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def g(self, a: S) -> None: pass +[out] + +[case testOverridingMethodInMultilevelHierarchyOfGenericTypes] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +U = TypeVar('U') +V = TypeVar('V') + +class D: pass +class C(Generic[T, U, V]): + def f(self, a: V) -> None: pass + def g(self, a: V) -> None: pass +class B(C[D, D, T], Generic[T]): pass +class A(B[S], Generic[T, S]): + def f(self, a: T) -> None: pass \ + # E: Argument 1 of "f" is incompatible with supertype "C"; supertype defines the argument type as "S" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def g(self, a: S) -> None: pass +[out] + +[case testOverrideGenericMethodInNonGenericClass] +from typing import TypeVar + +T = TypeVar('T') +S = TypeVar('S') + +class A: + def f(self, x: T, y: S) -> None: pass +class B(A): + def f(self, x: S, y: T) -> None: pass +class C(A): + # Okay, because T = object allows any type for the arguments. + def f(self, x: T, y: T) -> None: pass + +[case testOverrideGenericMethodInNonGenericClassLists] +from typing import TypeVar, List + +T = TypeVar('T') +S = TypeVar('S') + +class A: + def f(self, x: List[T], y: List[S]) -> None: pass +class B(A): + def f(self, x: List[S], y: List[T]) -> None: pass +class C(A): + def f(self, x: List[T], y: List[T]) -> None: pass # Fail +[builtins fixtures/list.pyi] +[out] +main:11: error: Signature of "f" incompatible with supertype "A" +main:11: note: Superclass: +main:11: note: def [T, S] f(self, x: List[T], y: List[S]) -> None +main:11: note: Subclass: +main:11: note: def [T] f(self, x: List[T], y: List[T]) -> None + +[case testOverrideGenericMethodInNonGenericClassGeneralize] +from typing import TypeVar + +T = TypeVar('T') +T1 = TypeVar('T1', bound=str) +S = TypeVar('S') + +class A: + def f(self, x: int, y: S) -> None: pass +class B(A): + def f(self, x: T, y: S) -> None: pass +class C(A): + def f(self, x: T, y: str) -> None: pass +class D(A): + def f(self, x: T1, y: S) -> None: pass # TODO: This error could be more specific. +[out] +main:12: error: Argument 2 of "f" is incompatible with supertype "A"; supertype defines the argument type as "S" +main:12: note: This violates the Liskov substitution principle +main:12: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:14: error: Signature of "f" incompatible with supertype "A" +main:14: note: Superclass: +main:14: note: def [S] f(self, x: int, y: S) -> None +main:14: note: Subclass: +main:14: note: def [T1: str, S] f(self, x: T1, y: S) -> None + +-- Inheritance from generic types with implicit dynamic supertype +-- -------------------------------------------------------------- + + +[case testInheritanceFromGenericWithImplicitDynamicAndSubtyping] +from typing import TypeVar, Generic +T = TypeVar('T') +a: A +bc: B[C] +bd: B[D] + +if int(): + a = bc # E: Incompatible types in assignment (expression has type "B[C]", variable has type "A") + bc = a + bd = a + +class B(Generic[T]): pass +class A(B): pass +class C: pass +class D: pass +[out] + +[case testInheritanceFromGenericWithImplicitDynamicAndExternalAccess] +from typing import TypeVar, Generic +T = TypeVar('T') + +class B(Generic[T]): + def f(self, a: 'B[T]') -> None: pass + def __init__(self, x: 'B[T]') -> None: + self.x = x +class A(B): pass +class C: pass + +a: A +c: C +bc: B[C] + +a.x = c # E: Incompatible types in assignment (expression has type "C", variable has type "B[Any]") +a.f(c) # E: Argument 1 to "f" of "B" has incompatible type "C"; expected "B[Any]" +a.x = bc +a.f(bc) +[out] + +[case testInheritanceFromGenericWithImplicitDynamic] +from typing import TypeVar, Generic +T = TypeVar('T') +a: A +c: C +bc: B[C] + +class B(Generic[T]): + def f(self, a: 'B[T]') -> None: pass + def __init__(self, x: 'B[T]') -> None: + self.x = x + +class A(B): + def g(self) -> None: + self.x = c # E: Incompatible types in assignment (expression has type "C", variable has type "B[Any]") + self.f(c) # E: Argument 1 to "f" of "B" has incompatible type "C"; expected "B[Any]" + self.x = bc + self.f(bc) + +class C: pass +[out] + +[case testInheritanceFromGenericWithImplicitDynamicAndOverriding] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +class B(Generic[T]): + def f(self, a: T, b: 'Tuple[T, B[T]]') -> None: + pass +class A(B): + def f(self, a, b): pass +[builtins fixtures/tuple.pyi] +[out] + + +-- Inheritance from generic types and super expressions +-- ---------------------------------------------------- + + +[case testSuperExpressionsWhenInheritingFromGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class B(Generic[T]): + def f(self, a: T) -> None: pass +class A(B[S], Generic[T, S]): + def g(self, t: T, s: S) -> None: + super().f(t) # E: Argument 1 to "f" of "B" has incompatible type "T"; expected "S" + super().f(s) +[out] + +[case testSuperExpressionsWhenInheritingFromGenericTypeAndDeepHierarchy] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +U = TypeVar('U') +V = TypeVar('V') +class C(Generic[T, U, V]): + def f(self, a: V) -> None: pass +class D: pass +class B(C[D, D, T], Generic[T]): pass +class A(B[S], Generic[T, S]): + def g(self, t: T, s: S) -> None: + super().f(t) # E: Argument 1 to "f" of "C" has incompatible type "T"; expected "S" + super().f(s) +[out] + + +-- Type of inherited constructor +-- ----------------------------- + + +[case testInheritedConstructor] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def __init__(self, x: T) -> None: pass +class B(A[T], Generic[T]): pass +class C(A[int]): pass +class D(A[A[T]], Generic[T]): pass +B(1) +C(1) +C('a') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +D(A(1)) +D(1) # E: Argument 1 to "D" has incompatible type "int"; expected "A[Never]" + + +[case testInheritedConstructor2] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +Z = TypeVar('Z') +class A(Generic[T, U]): + def __init__(self, x: T, y: U, z: Z) -> None: pass +class B(A[int, T], Generic[T]): pass +class C(B[A[T, str]], Generic[T, U]): pass +# C[T, U] <: B[A[T, str]] <: A[int, A[T, str]] +C(1, A(1, 'a', 0), 'z') +C(1, A('1', 'a', 0), 'z') +C('1', A(1, 'a', 0), 'z') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +C(1, A(1, 1, 0), 'z') # E: Argument 2 to "A" has incompatible type "int"; expected "str" + + +-- Subtyping with a generic abstract base class +-- -------------------------------------------- + + +[case testSubtypingWithGenericTypeSubclassingGenericAbstractClass] +from typing import TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +S = TypeVar('S') +acd: A[C, D] +adc: A[D, C] +ic: I[C] +id: I[D] + +if int(): + ic = acd # E: Incompatible types in assignment (expression has type "A[C, D]", variable has type "I[C]") + id = adc # E: Incompatible types in assignment (expression has type "A[D, C]", variable has type "I[D]") + adc = ic # E: Incompatible types in assignment (expression has type "I[C]", variable has type "A[D, C]") + +if int(): + ic = adc + id = acd + +class I(Generic[T]): + @abstractmethod + def f(self): pass +class A(I[S], Generic[T, S]): pass +class C: pass +class D: pass + +[case testSubtypingWithTypeImplementingGenericABCViaInheritance] +from typing import TypeVar, Generic +S = TypeVar('S') +a: A +b: B +ic: I[C] +id: I[D] +ie: I[E] + +class I(Generic[S]): pass +class B(I[C]): pass +class A(B): pass + +if int(): + ie = a # E: Incompatible types in assignment (expression has type "A", variable has type "I[E]") + a = ic # E: Incompatible types in assignment (expression has type "I[C]", variable has type "A") +if int(): + a = id # E: Incompatible types in assignment (expression has type "I[D]", variable has type "A") +if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + id = a # E: Incompatible types in assignment (expression has type "A", variable has type "I[D]") + + ic = a + b = a + +class C: pass +class D: pass +class E: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testSubtypingWithTypeImplementingGenericABCViaInheritance2-skip] +from typing import TypeVar, Generic +T = TypeVar('T') +class I(Generic[T]): pass +class A(I[C]): pass +class B(A, I[D]): pass # Fail + +class C: pass +class D: pass +[out] +main:5: error: Class "B" has base "I" duplicated inconsistently + +[case testSubtypingAndABCExtension] +from typing import TypeVar, Generic +from abc import abstractmethod, ABCMeta +t = TypeVar('t') +a: A[object] +i: I[object] +j: J[object] +(ii, jj) = (i, j) +if int(): + ii = a + jj = a +if int(): + jj = i + a = i # E: Incompatible types in assignment (expression has type "I[object]", variable has type "A[object]") +if int(): + a = j # E: Incompatible types in assignment (expression has type "J[object]", variable has type "A[object]") + +class J(Generic[t]): pass +class X(metaclass=ABCMeta): pass +class I(X, J[t], Generic[t]): pass +class A(I[t], Generic[t]): pass +[builtins fixtures/tuple.pyi] + + +-- Subclassing a generic ABC +-- ------------------------- + + +[case testSubclassingGenericABC1] +from typing import TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +class I(Generic[T]): + @abstractmethod + def f(self, a: T) -> None: pass + @abstractmethod + def g(self, a: T) -> None: pass +class A(I[C]): + def f(self, a: 'D') -> None: pass \ + # E: Argument 1 of "f" is incompatible with supertype "I"; supertype defines the argument type as "C" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def g(self, a: 'C') -> None: pass +class C: pass +class D: pass +[out] + + +-- Extending a generic ABC with deep type hierarchy +-- ------------------------------------------------ + + +[case testSubclassingGenericABCWithDeepHierarchy] +from typing import Any, TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +a: A +ic: I[C] +id: I[D] + +if int(): + id = a # E: Incompatible types in assignment (expression has type "A", variable has type "I[D]") + ic = a + +class I(Generic[T]): + @abstractmethod + def f(self, a: T, b: T) -> None: pass + @abstractmethod + def g(self, a: T, b: 'D') -> None: pass +class B(I[C]): + def f(self, a: 'C', b: 'C') -> None: pass + def g(self, a: 'C', b: Any) -> None: pass +class A(B): + def g(self, a: 'C', b: 'C') -> None: pass \ + # E: Argument 2 of "g" is incompatible with supertype "I"; supertype defines the argument type as "D" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + def f(self, a: 'C', b: 'C') -> None: pass +class C: pass +class D: pass +[builtins fixtures/tuple.pyi] + +[case testSubclassingGenericABCWithDeepHierarchy2] +from typing import Any, TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +class I(Generic[T]): + @abstractmethod + def f(self, a: T, b: T) -> None: pass +class B(I[C]): + def f(self, a: 'C', b: Any) -> None: pass +class A(B): + def f(self, a: 'C', b: 'D') -> None: pass \ + # E: Argument 2 of "f" is incompatible with supertype "I"; supertype defines the argument type as "C" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +class C: pass +class D: pass +[out] + + +-- Implicit Any types and subclassing generic ABC +-- ---------------------------------------------- + + +[case testSubclassingGenericABCWithImplicitAny] +from typing import Any, TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +a: Any +ic: I[C] +id: I[D] + +ic = a +id = a + +class I(Generic[T]): + @abstractmethod + def f(self, a: T) -> None: pass +class A(I): + def f(self, a): pass + +class C: pass +class D: pass + +[case testSubclassingGenericABCWithImplicitAnyAndDeepHierarchy] +from typing import Any, TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +a: Any +ic: I[C] +id: I[D] + +ic = a +id = a + +class I(Generic[T]): + @abstractmethod + def f(self, a: T, b: T) -> None: pass +class B(I): + def f(self, a, b): pass +class A(B): + def f(self, a: 'C', b: 'D') -> None: pass +class C: pass +class D: pass + +[case testImplementingGenericABCWithImplicitAnyAndDeepHierarchy2] +from typing import Any, TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +a: Any +jc: J[C] +jd: J[D] + +jc = a +jd = a + +class J(Generic[T]): + @abstractmethod + def f(self, a: T, b: T) -> None: pass +class I(J): + @abstractmethod + def f(self, a, b): pass +class A(I): + def f(self, a: 'C', b: 'D') -> None: pass + +class C: pass +class D: pass + + +-- Accessing generic ABC members +-- ----------------------------- + + +[case testAccessingGenericABCMembers] +from typing import TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +class I(Generic[T]): + @abstractmethod + def f(self, a: T) -> None: pass +class A: pass +class B: pass + +a: A +b: B +ia: I[A] + +ia.f(b) # E: Argument 1 to "f" of "I" has incompatible type "B"; expected "A" +ia.f(a) +[builtins fixtures/tuple.pyi] + +[case testAccessingInheritedGenericABCMembers] +from typing import TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T') +class J(Generic[T]): + @abstractmethod + def f(self, a: T) -> None: pass +class I(J[T], Generic[T]): pass +class A: pass +class B: pass +a: A +b: B +ia: I[A] + +ia.f(b) # E: Argument 1 to "f" of "J" has incompatible type "B"; expected "A" +ia.f(a) +[builtins fixtures/tuple.pyi] + + +-- Misc +-- ---- + + +[case testMultipleAssignmentAndGenericSubtyping] +from typing import Iterable +n: int +s: str +class Nums(Iterable[int]): + def __iter__(self): pass + def __next__(self): pass +n, n = Nums() +s, s = Nums() # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/for.pyi] +[out] + + +-- Variance +-- -------- + + +[case testCovariant] +from typing import TypeVar, Generic +T = TypeVar('T', covariant=True) + +class G(Generic[T]): pass +class A: pass +class B(A): pass +class C(B): pass + +a: G[A] +b: G[B] +c: G[C] + +if int(): + b = a # E: Incompatible types in assignment (expression has type "G[A]", variable has type "G[B]") + b = c +[builtins fixtures/bool.pyi] +[out] + +[case testContravariant] +from typing import TypeVar, Generic +T = TypeVar('T', contravariant=True) + +class G(Generic[T]): pass +class A: pass +class B(A): pass +class C(B): pass + +a: G[A] +b: G[B] +c: G[C] + +if int(): + b = a + b = c # E: Incompatible types in assignment (expression has type "G[C]", variable has type "G[B]") +[builtins fixtures/bool.pyi] +[out] + +[case testInvariant] +from typing import TypeVar, Generic +T = TypeVar('T') # invariant (default) + +class G(Generic[T]): pass +class A: pass +class B(A): pass +class C(B): pass + +a: G[A] +b: G[B] +c: G[C] + +if int(): + b = a # E: Incompatible types in assignment (expression has type "G[A]", variable has type "G[B]") + b = c # E: Incompatible types in assignment (expression has type "G[C]", variable has type "G[B]") +[builtins fixtures/bool.pyi] +[out] + + +[case testTypeVarSubtypeUnion] +from typing import Union, TypeVar, Generic + +class U: pass +class W: pass + +T = TypeVar('T', bound=Union[U, W]) + +class Y(Generic[T]): + def __init__(self) -> None: + pass + def f(self) -> T: + return U() # E: Incompatible return value type (got "U", expected "T") + + +[case testTypeVarBoundToOldUnionAttributeAccess] +from typing import Union, TypeVar + +class U: + a: float +class V: + b: float +class W: + c: float + +T = TypeVar("T", bound=Union[U, V, W]) + +def f(x: T) -> None: + x.a # E + x.b = 1.0 # E + del x.c # E + +[out] +main:13: error: Item "V" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "a" +main:13: error: Item "W" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "a" +main:14: error: Item "U" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "b" +main:14: error: Item "W" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "b" +main:15: error: Item "U" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "c" +main:15: error: Item "V" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "c" + + +[case testTypeVarBoundToNewUnionAttributeAccess] +# flags: --python-version 3.10 +from typing import TypeVar + +class U: + a: int +class V: + b: int +class W: + c: int + +T = TypeVar("T", bound=U | V | W) + +def f(x: T) -> None: + x.a # E + x.b = 1 # E + del x.c # E + +[builtins fixtures/tuple.pyi] +[out] +main:14: error: Item "V" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "a" +main:14: error: Item "W" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "a" +main:15: error: Item "U" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "b" +main:15: error: Item "W" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "b" +main:16: error: Item "U" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "c" +main:16: error: Item "V" of the upper bound "Union[U, V, W]" of type variable "T" has no attribute "c" + + +[case testSubtypingIterableUnpacking1] +# https://github.com/python/mypy/issues/11138 +from typing import Generic, Iterator, TypeVar +T = TypeVar("T") +U = TypeVar("U") + +class X1(Iterator[U], Generic[T, U]): + pass + +x1: X1[str, int] +reveal_type(list(x1)) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type([*x1]) # N: Revealed type is "builtins.list[builtins.int]" + +class X2(Iterator[T], Generic[T, U]): + pass + +x2: X2[str, int] +reveal_type(list(x2)) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type([*x2]) # N: Revealed type is "builtins.list[builtins.str]" + +class X3(Generic[T, U], Iterator[U]): + pass + +x3: X3[str, int] +reveal_type(list(x3)) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type([*x3]) # N: Revealed type is "builtins.list[builtins.int]" + +class X4(Generic[T, U], Iterator[T]): + pass + +x4: X4[str, int] +reveal_type(list(x4)) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type([*x4]) # N: Revealed type is "builtins.list[builtins.str]" + +class X5(Iterator[T]): + pass + +x5: X5[str] +reveal_type(list(x5)) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type([*x5]) # N: Revealed type is "builtins.list[builtins.str]" + +class X6(Generic[T, U], Iterator[bool]): + pass + +x6: X6[str, int] +reveal_type(list(x6)) # N: Revealed type is "builtins.list[builtins.bool]" +reveal_type([*x6]) # N: Revealed type is "builtins.list[builtins.bool]" +[builtins fixtures/list.pyi] + +[case testSubtypingIterableUnpacking2] +from typing import Generic, Iterator, TypeVar, Mapping +T = TypeVar("T") +U = TypeVar("U") + +class X1(Generic[T, U], Iterator[U], Mapping[U, T]): + pass + +x1: X1[str, int] +reveal_type(list(x1)) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type([*x1]) # N: Revealed type is "builtins.list[builtins.int]" + +class X2(Generic[T, U], Iterator[U], Mapping[T, U]): + pass + +x2: X2[str, int] +reveal_type(list(x2)) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type([*x2]) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testSubtypingMappingUnpacking1] +# https://github.com/python/mypy/issues/11138 +from typing import Generic, TypeVar, Mapping +T = TypeVar("T") +U = TypeVar("U") + +class X1(Generic[T, U], Mapping[U, T]): + pass + +x1: X1[str, int] +reveal_type(iter(x1)) # N: Revealed type is "typing.Iterator[builtins.int]" +reveal_type({**x1}) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +class X2(Generic[T, U], Mapping[T, U]): + pass + +x2: X2[str, int] +reveal_type(iter(x2)) # N: Revealed type is "typing.Iterator[builtins.str]" +reveal_type({**x2}) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + +class X3(Generic[T, U], Mapping[bool, float]): + pass + +x3: X3[str, int] +reveal_type(iter(x3)) # N: Revealed type is "typing.Iterator[builtins.bool]" +reveal_type({**x3}) # N: Revealed type is "builtins.dict[builtins.bool, builtins.float]" +[builtins fixtures/dict.pyi] + +[case testSubtypingMappingUnpacking2] +from typing import Generic, TypeVar, Mapping +T = TypeVar("T") +U = TypeVar("U") + +class X1(Generic[T, U], Mapping[U, T]): + pass + +def func_with_kwargs(**kwargs: int): + pass + +x1: X1[str, int] +reveal_type(iter(x1)) +reveal_type({**x1}) +func_with_kwargs(**x1) +[out] +main:12: note: Revealed type is "typing.Iterator[builtins.int]" +main:13: note: Revealed type is "builtins.dict[builtins.int, builtins.str]" +main:14: error: Keywords must be strings +main:14: error: Argument 1 to "func_with_kwargs" has incompatible type "**X1[str, int]"; expected "int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testSubtypingMappingUnpacking3] +from typing import Generic, TypeVar, Mapping, Iterable +T = TypeVar("T") +U = TypeVar("U") + +class X1(Generic[T, U], Mapping[U, T], Iterable[U]): + pass + +x1: X1[str, int] +reveal_type(iter(x1)) # N: Revealed type is "typing.Iterator[builtins.int]" +reveal_type({**x1}) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +# Some people would expect this to raise an error, but this currently does not: +# `Mapping` has `Iterable[U]` base class, `X2` has direct `Iterable[T]` base class. +# It would be impossible to define correct `__iter__` method for incompatible `T` and `U`. +class X2(Generic[T, U], Mapping[U, T], Iterable[T]): + pass + +x2: X2[str, int] +reveal_type(iter(x2)) # N: Revealed type is "typing.Iterator[builtins.int]" +reveal_type({**x2}) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" +[builtins fixtures/dict.pyi] + +[case testNotDirectIterableAndMappingSubtyping] +from typing import Generic, TypeVar, Dict, Iterable, Iterator, List +T = TypeVar("T") +U = TypeVar("U") + +class X1(Generic[T, U], Dict[U, T], Iterable[U]): + def __iter__(self) -> Iterator[U]: pass + +x1: X1[str, int] +reveal_type(iter(x1)) # N: Revealed type is "typing.Iterator[builtins.int]" +reveal_type({**x1}) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +class X2(Generic[T, U], List[U]): + def __iter__(self) -> Iterator[U]: pass + +x2: X2[str, int] +reveal_type(iter(x2)) # N: Revealed type is "typing.Iterator[builtins.int]" +reveal_type([*x2]) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/dict.pyi] + +[case testIncompatibleVariance] +from typing import TypeVar, Generic +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class A(Generic[T_co]): ... +class B(A[T_contra], Generic[T_contra]): ... # E: Variance of TypeVar "T_contra" incompatible with variance in parent type + +class C(Generic[T_contra]): ... +class D(C[T_co], Generic[T_co]): ... # E: Variance of TypeVar "T_co" incompatible with variance in parent type + +class E(Generic[T]): ... +class F(E[T_co], Generic[T_co]): ... # E: Variance of TypeVar "T_co" incompatible with variance in parent type + +class G(Generic[T]): ... +class H(G[T_contra], Generic[T_contra]): ... # E: Variance of TypeVar "T_contra" incompatible with variance in parent type diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test new file mode 100644 index 000000000000..b8cc0422b749 --- /dev/null +++ b/test-data/unit/check-generics.test @@ -0,0 +1,3549 @@ +-- Simple generic types +-- -------------------- + + +[case testGenericMethodReturnType] +from typing import TypeVar, Generic +T = TypeVar('T') +a: A[B] +b: B +c: C +if int(): + c = a.f() # E: Incompatible types in assignment (expression has type "B", variable has type "C") + b = a.f() + +class A(Generic[T]): + def f(self) -> T: pass + +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] + +[case testGenericMethodArgument] +from typing import TypeVar, Generic +T = TypeVar('T') + +class A(Generic[T]): + def f(self, a: T) -> None: pass + +a: A[B] +b: B +c: C + +a.f(c) # E: Argument 1 to "f" of "A" has incompatible type "C"; expected "B" +a.f(b) + +class B: pass +class C: pass +[case testGenericMemberVariable] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def __init__(self, v: T) -> None: + self.v = v + +a: A[B] +b: B +c: C +a.v = c # Fail +a.v = b + +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:10: error: Incompatible types in assignment (expression has type "C", variable has type "B") + +[case testGenericMemberVariable2] +from typing import TypeVar, Generic +T = TypeVar('T') +a: A[B] +b: B +c: C +a.v = c # Fail +a.v = b + +class A(Generic[T]): + v: T +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:6: error: Incompatible types in assignment (expression has type "C", variable has type "B") + +[case testSimpleGenericSubtyping] +from typing import TypeVar, Generic +T = TypeVar('T') +b: A[B] +bb: A[B] +c: A[C] +if int(): + c = b # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[C]") + b = c # E: Incompatible types in assignment (expression has type "A[C]", variable has type "A[B]") + +if int(): + b = b +if int(): + b = bb + +class A(Generic[T]): pass +class B: pass +class C(B): pass +[builtins fixtures/tuple.pyi] + +[case testGenericTypeCompatibilityWithAny] +from typing import Any, TypeVar, Generic +T = TypeVar('T') +b: A[B] +c: A[C] +d: A[Any] + +b = d +c = d +d = b +d = c + +class A(Generic[T]): pass +class B: pass +class C(B): pass +[builtins fixtures/tuple.pyi] +[out] + +[case testTypeVariableAsTypeArgument] +from typing import TypeVar, Generic +T = TypeVar('T') +a: A[B] +b: A[B] +c: A[C] + +a.v = c # E: Incompatible types in assignment (expression has type "A[C]", variable has type "A[B]") +if int(): + c = a.v # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[C]") +a.v = b +if int(): + b = a.v + +class A(Generic[T]): + v = None # type: A[T] + +class B: pass +class C: pass + +[case testMultipleGenericTypeParametersWithMemberVars] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +a: A[B, C] +s: B +t: C + +if int(): + t = a.s # E: Incompatible types in assignment (expression has type "B", variable has type "C") + s = a.t # E: Incompatible types in assignment (expression has type "C", variable has type "B") + +if int(): + s = a.s + t = a.t + +class A(Generic[S, T]): + s: S + t: T +class B: pass +class C: pass + +[case testMultipleGenericTypeParametersWithMethods] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +a: A[B, C] +s: B +t: C + +a.f(s, s) # Fail +a.f(t, t) # Fail +a.f(s, t) + +class A(Generic[S, T]): + def f(self, s: S, t: T) -> None: pass +class B: pass +class C: pass +[out] +main:8: error: Argument 2 to "f" of "A" has incompatible type "B"; expected "C" +main:9: error: Argument 1 to "f" of "A" has incompatible type "C"; expected "B" + +[case testMultipleGenericTypeParametersAndSubtyping] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +bc: A[B, C] +bb: A[B, B] +cb: A[C, B] + +if int(): + bb = bc # E: Incompatible types in assignment (expression has type "A[B, C]", variable has type "A[B, B]") +if int(): + bb = cb # E: Incompatible types in assignment (expression has type "A[C, B]", variable has type "A[B, B]") + bc = bb # E: Incompatible types in assignment (expression has type "A[B, B]", variable has type "A[B, C]") + +if int(): + bb = bb + bc = bc + +class A(Generic[S, T]): + s: S + t: T + +class B: pass +class C(B):pass + + +-- Simple generic type bodies +-- -------------------------- + + +[case testGenericTypeBody1] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + a: T + + def f(self, b: T) -> T: + self.f(x) # Fail + d = self # type: A[B] # Fail + self.a = self.f(self.a) + return self.a + c = self # type: A[T] +x: B +class B: pass +[out] +main:7: error: Argument 1 to "f" of "A" has incompatible type "B"; expected "T" +main:8: error: Incompatible types in assignment (expression has type "A[T]", variable has type "A[B]") + +[case testGenericTypeBodyWithMultipleVariables] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +class A(Generic[S, T]): + def f(self) -> None: + s: S + t: T + if int(): + s = t # E: Incompatible types in assignment (expression has type "T", variable has type "S") + t = s # E: Incompatible types in assignment (expression has type "S", variable has type "T") + a = self # type: A[S, B] # E: Incompatible types in assignment (expression has type "A[S, T]", variable has type "A[S, B]") + b = self # type: A[T, T] # E: Incompatible types in assignment (expression has type "A[S, T]", variable has type "A[T, T]") + c = self # type: A[S, T] + if int(): + t = t + +class B: pass +[out] + +[case testCompatibilityOfNoneWithTypeVar] +# flags: --no-strict-optional +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def f(self) -> None: + a = None # type: T + a = None +[out] + +[case testCompatibilityOfTypeVarWithObject] +# flags: --no-strict-optional +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def f(self) -> T: + a = object() # type: T # E: Incompatible types in assignment (expression has type "object", variable has type "T") + if int(): + a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "T") + b = self.f() # type: object + if int(): + b = self.f() + return None +[out] + + +-- Operations with generic types +-- ----------------------------- + + +[case testGenericOperations] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +a: A[B, C] +b: B +c: C + +if int(): + b = a + b # E: Incompatible types in assignment (expression has type "C", variable has type "B") + c = a + c # E: Unsupported operand types for + ("A[B, C]" and "C") +if int(): + c = a[c] # E: Incompatible types in assignment (expression has type "B", variable has type "C") + b = a[b] # E: Invalid index type "B" for "A[B, C]"; expected type "C" + +if int(): + c = a + b + b = a[c] + +class A(Generic[S, T]): + def __add__(self, a: S) -> T: pass + def __getitem__(self, i: T) -> S: pass + +class B: pass +class C: pass + +[case testOperatorAssignmentWithIndexLvalue1] +from typing import TypeVar, Generic +T = TypeVar('T') +b: B +c: C +ac: A[C] + +ac[b] += b # Fail +ac[c] += c # Fail +ac[b] += c +ac[b] = ac[b] + c + +class A(Generic[T]): + def __getitem__(self, i: 'B') -> T: pass + def __setitem__(self, i: 'B', v: T) -> None: pass + +class B: pass +class C: + def __add__(self, o: 'C') -> 'C': pass +[out] +main:7: error: Unsupported operand types for + ("C" and "B") +main:8: error: Invalid index type "C" for "A[C]"; expected type "B" + +[case testOperatorAssignmentWithIndexLvalue2] +from typing import TypeVar, Generic +T = TypeVar('T') +b: B +c: C +ac: A[C] + +ac[b] += c # Fail +ac[c] += c # Fail +ac[b] = ac[b] + c # Fail + +class A(Generic[T]): + def __getitem__(self, i: 'B') -> T: pass + def __setitem__(self, i: 'C', v: T) -> None: pass + +class B: pass +class C: + def __add__(self, o: 'C') -> 'C': pass +[out] +main:7: error: Invalid index type "B" for "A[C]"; expected type "C" +main:8: error: Invalid index type "C" for "A[C]"; expected type "B" +main:9: error: Invalid index type "B" for "A[C]"; expected type "C" + + +-- Nested generic types +-- -------------------- + + +[case testNestedGenericTypes] +from typing import TypeVar, Generic +T = TypeVar('T') +aab: A[A[B]] +aac: A[A[C]] +ab: A[B] +ac: A[C] + +if int(): + ac = aab.x # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[C]") +ac.y = aab # E: Incompatible types in assignment (expression has type "A[A[B]]", variable has type "A[A[C]]") + +if int(): + ab = aab.x + ac = aac.x +ab.y = aab +ac.y = aac + +class A(Generic[T]): + x: T + y: A[A[T]] + +class B: + pass +class C: + pass + + +-- Generic functions +-- ----------------- + + +[case testTypeCheckingGenericFunctionBody] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +class A: pass +class p(Generic[T, S]): + def __init__(self, t: T, a: S) -> None: pass +def f(s: S, t: T) -> p[T, A]: + a = t # type: S # E: Incompatible types in assignment (expression has type "T", variable has type "S") + if int(): + s = t # E: Incompatible types in assignment (expression has type "T", variable has type "S") + p_s_a: p[S, A] + if s: + return p_s_a # E: Incompatible return value type (got "p[S, A]", expected "p[T, A]") + b = t # type: T + c = s # type: S + p_t_a: p[T, A] + return p_t_a +[out] + +[case testTypeCheckingGenericMethodBody] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class p(Generic[T, S]): + def __init__(self, t: T, a: S) -> None: pass +class A(Generic[T]): + def f(self, s: S, t: T) -> p[S, T]: + if int(): + s = t # E: Incompatible types in assignment (expression has type "T", variable has type "S") + p_s_s: p[S, S] + if s: + return p_s_s # E: Incompatible return value type (got "p[S, S]", expected "p[S, T]") + p_t_t: p[T, T] + if t: + return p_t_t # E: Incompatible return value type (got "p[T, T]", expected "p[S, T]") + if 1: + t = t + s = s + p_s_t: p[S, T] + return p_s_t +[out] + +[case testProhibitTypeApplicationToGenericFunctions] +from typing import TypeVar +T = TypeVar('T') +def f(x: T) -> T: pass + +y = f[int] # E: Type application is only supported for generic classes +[out] + + +-- Generic types in expressions +-- ---------------------------- + + +[case testTypeApplicationArgs] +from typing import TypeVar, Generic +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + ... +Node[int]() # E: Missing positional argument "x" in call to "Node" +Node[int](1, 1, 1) # E: Too many arguments for "Node" +[out] + +[case testTypeApplicationTvars] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T, S]): pass +A[int]() # E: Type application has too few types (2 expected) +A[int, str, int]() # E: Type application has too many types (2 expected) +[out] + +[case testInvalidTypeApplicationType] +import types +a: A +class A: pass +a[A]() # E: Value of type "A" is not indexable +A[A]() # E: The type "Type[A]" is not generic and not indexable +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeApplicationArgTypes] +from typing import TypeVar, Generic +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + ... + +Node[int](1) +Node[int]('a') # E: Argument 1 to "Node" has incompatible type "str"; expected "int" + +class Dummy(Generic[T]): + def meth(self, x: T) -> None: + ... + def methout(self) -> T: + ... + +Dummy[int]().meth(1) +Dummy[int]().meth('a') # E: Argument 1 to "meth" of "Dummy" has incompatible type "str"; expected "int" +reveal_type(Dummy[int]()) # N: Revealed type is "__main__.Dummy[builtins.int]" +reveal_type(Dummy[int]().methout()) # N: Revealed type is "builtins.int" +[out] + +[case testTypeApplicationArgTypesSubclasses] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class C(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + ... + +class D(C[int, T]): ... + +D[str](1, 'a') +D[str](1, 1) # E: Argument 2 to "D" has incompatible type "int"; expected "str" + +class E(D[str]): ... +E(1, 'a') +E(1, 1) # E: Argument 2 to "E" has incompatible type "int"; expected "str" +[out] + +[case testTypeApplicationAlias] +from typing import TypeVar, Generic +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + ... + +Alias = Node +Alias[int](1) +Alias[int]("a") # E: Argument 1 to "Node" has incompatible type "str"; expected "int" +[out] + +[case testTypeApplicationCrash] +import types +type[int] # this was crashing, see #2302 (comment) # E: The type "Type[type]" is not generic and not indexable +[builtins fixtures/tuple.pyi] + + +-- Generic type aliases +-- -------------------- + +[case testGenericTypeAliasesBasic] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + ... + +IntNode = Node[int, S] +IntIntNode = Node[int, int] +SameNode = Node[T, T] + +n = Node(1, 1) # type: IntIntNode +n1 = Node(1, 'a') # type: IntIntNode # E: Argument 2 to "Node" has incompatible type "str"; expected "int" + +m = Node(1, 1) # type: IntNode +m1 = Node('x', 1) # type: IntNode # E: Argument 1 to "Node" has incompatible type "str"; expected "int" +m2 = Node(1, 1) # type: IntNode[str] # E: Argument 2 to "Node" has incompatible type "int"; expected "str" + +s = Node(1, 1) # type: SameNode[int] +reveal_type(s) # N: Revealed type is "__main__.Node[builtins.int, builtins.int]" +s1 = Node(1, 'x') # type: SameNode[int] # E: Argument 2 to "Node" has incompatible type "str"; expected "int" + +[out] + +[case testGenericTypeAliasesBasic2] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + ... + +IntNode = Node[int, S] +IntIntNode = Node[int, int] +SameNode = Node[T, T] + +def output_bad() -> IntNode[str]: + return Node(1, 1) # Error - bad return type, see out + +def input(x: IntNode[str]) -> None: + pass +input(Node(1, 's')) +input(Node(1, 1)) # E: Argument 2 to "Node" has incompatible type "int"; expected "str" + +def output() -> IntNode[str]: + return Node(1, 'x') +reveal_type(output()) # N: Revealed type is "__main__.Node[builtins.int, builtins.str]" + +def func(x: IntNode[T]) -> IntNode[T]: + return x +reveal_type(func) # N: Revealed type is "def [T] (x: __main__.Node[builtins.int, T`-1]) -> __main__.Node[builtins.int, T`-1]" + +func(1) # E: Argument 1 to "func" has incompatible type "int"; expected "Node[int, Never]" +func(Node('x', 1)) # E: Argument 1 to "Node" has incompatible type "str"; expected "int" +reveal_type(func(Node(1, 'x'))) # N: Revealed type is "__main__.Node[builtins.int, builtins.str]" + +def func2(x: SameNode[T]) -> SameNode[T]: + return x +reveal_type(func2) # N: Revealed type is "def [T] (x: __main__.Node[T`-1, T`-1]) -> __main__.Node[T`-1, T`-1]" + +func2(Node(1, 'x')) # E: Cannot infer type argument 1 of "func2" +y = func2(Node('x', 'x')) +reveal_type(y) # N: Revealed type is "__main__.Node[builtins.str, builtins.str]" + +def wrap(x: T) -> IntNode[T]: + return Node(1, x) + +z: str +reveal_type(wrap(z)) # N: Revealed type is "__main__.Node[builtins.int, builtins.str]" + +[out] +main:13: error: Argument 2 to "Node" has incompatible type "int"; expected "str" + +-- Error formatting is a bit different (and probably better) with new analyzer +[case testGenericTypeAliasesWrongAliases] +# flags: --show-column-numbers --no-strict-optional +from typing import TypeVar, Generic, List, Callable, Tuple, Union +T = TypeVar('T') +S = TypeVar('S') +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + ... + +A = Node[T] # Error +B = Node[T, T] +C = Node[T, T, T] # Error +D = Node[T, S] +E = Node[Node[T, T], List[T]] + +F = Node[List[T, T], S] # Error +G = Callable[..., List[T, T]] # Error +H = Union[int, Tuple[T, Node[T]]] # Error +h: H # This was reported on previous line +h1: H[int, str] # Error + +x = None # type: D[int, str] +reveal_type(x) +y = None # type: E[int] +reveal_type(y) + +X = T # Error + +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] +[out] +main:9:5: error: "Node" expects 2 type arguments, but 1 given +main:11:5: error: "Node" expects 2 type arguments, but 3 given +main:15:10: error: "list" expects 1 type argument, but 2 given +main:16:19: error: "list" expects 1 type argument, but 2 given +main:17:25: error: "Node" expects 2 type arguments, but 1 given +main:19:5: error: Bad number of arguments for type alias, expected 1, given 2 +main:22:13: note: Revealed type is "__main__.Node[builtins.int, builtins.str]" +main:24:13: note: Revealed type is "__main__.Node[__main__.Node[builtins.int, builtins.int], builtins.list[builtins.int]]" +main:26:5: error: Type variable "__main__.T" is invalid as target for type alias + +[case testGenericTypeAliasesForAliases] +from typing import TypeVar, Generic, List, Union +T = TypeVar('T') +S = TypeVar('S') + +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + pass + +ListedNode = Node[List[T], List[S]] +Second = ListedNode[int, T] +Third = Union[int, Second[str]] + +def f2(x: T) -> Second[T]: + return Node([1], [x]) +reveal_type(f2('a')) # N: Revealed type is "__main__.Node[builtins.list[builtins.int], builtins.list[builtins.str]]" + +def f3() -> Third: + return Node([1], ['x']) +reveal_type(f3()) # N: Revealed type is "Union[builtins.int, __main__.Node[builtins.list[builtins.int], builtins.list[builtins.str]]]" + +[builtins fixtures/list.pyi] + +[case testGenericTypeAliasesWithNestedArgs] +# flags: --pretty --show-error-codes +import other +a: other.Array[float] +reveal_type(a) # N: Revealed type is "other.array[Any, other.dtype[builtins.float]]" + +[out] +main:3: error: Type argument "float" of "Array" must be a subtype of "generic" [type-var] + a: other.Array[float] + ^ +[file other.py] +from typing import Any, Generic, TypeVar + +DT = TypeVar("DT", covariant=True, bound='dtype[Any]') +DTS = TypeVar("DTS", covariant=True, bound='generic') +S = TypeVar("S", bound=Any) +ST = TypeVar("ST", bound='generic', covariant=True) + +class common: pass +class generic(common): pass +class dtype(Generic[DTS]): pass +class array(common, Generic[S, DT]): pass +Array = array[Any, dtype[ST]] + +[case testGenericTypeAliasesAny] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + +IntNode = Node[int, S] +AnyNode = Node[S, T] + +def output() -> IntNode[str]: + return Node(1, 'x') +x = output() # type: IntNode # This is OK (implicit Any) + +y: IntNode +y.x = 1 +y.x = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +y.y = 1 # Both are OK (implicit Any) +y.y = 'x' + +z = Node(1, 'x') # type: AnyNode +reveal_type(z) # N: Revealed type is "__main__.Node[Any, Any]" + +[out] + +[case testGenericTypeAliasesAcessingMethods] +from typing import TypeVar, Generic, List +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + def meth(self) -> T: + return self.x + +ListedNode = Node[List[T]] +l: ListedNode[int] +l.x.append(1) +l.meth().append(1) +reveal_type(l.meth()) # N: Revealed type is "builtins.list[builtins.int]" +l.meth().append('x') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" + +ListedNode[str]([]).x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "List[str]") + +[builtins fixtures/list.pyi] + +[case testGenericTypeAliasesSubclassing] +from typing import TypeVar, Generic, Tuple, List +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + ... + +TupledNode = Node[Tuple[T, T]] + +class D(TupledNode[T]): + ... +class L(List[TupledNode[T]]): + ... + +def f_bad(x: T) -> D[T]: + return D(1) # Error, see out + +L[int]().append(Node((1, 1))) +L[int]().append(5) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "Node[Tuple[int, int]]" + +x = D((1, 1)) # type: D[int] +y = D(5) # type: D[int] # E: Argument 1 to "D" has incompatible type "int"; expected "Tuple[int, int]" + +def f(x: T) -> D[T]: + return D((x, x)) +reveal_type(f('a')) # N: Revealed type is "__main__.D[builtins.str]" + +[builtins fixtures/list.pyi] +[out] +main:15: error: Argument 1 to "D" has incompatible type "int"; expected "Tuple[T, T]" + +[case testGenericTypeAliasesSubclassingBad] + +from typing import TypeVar, Generic, Tuple, Union +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + ... + +TupledNode = Node[Tuple[T, T]] +UNode = Union[int, Node[T]] + +class C(TupledNode): ... # Same as TupledNode[Any] +class D(TupledNode[T]): ... +class E(Generic[T], UNode[T]): ... # E: Invalid base class "UNode" + +reveal_type(D((1, 1))) # N: Revealed type is "__main__.D[builtins.int]" +[builtins fixtures/list.pyi] + +[case testGenericTypeAliasesUnion] +from typing import TypeVar, Generic, Union, Any +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + +UNode = Union[int, Node[T]] +x = 1 # type: UNode[int] + +x + 1 # E: Unsupported left operand type for + ("Node[int]") \ + # N: Left operand is of type "Union[int, Node[int]]" +if not isinstance(x, Node): + x + 1 + +if not isinstance(x, int): + x.x = 1 + x.x = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +def f(x: T) -> UNode[T]: + if int(): + return Node(x) + else: + return 1 + +reveal_type(f(1)) # N: Revealed type is "Union[builtins.int, __main__.Node[builtins.int]]" + +TNode = Union[T, Node[int]] +s = 1 # type: TNode[str] # E: Incompatible types in assignment (expression has type "int", variable has type "Union[str, Node[int]]") + +if not isinstance(s, str): + s.x = 1 + +z = None # type: TNode # Same as TNode[Any] +z.x +z.foo() # E: Item "Node[int]" of "Union[Any, Node[int]]" has no attribute "foo" + +[builtins fixtures/isinstance.pyi] + +[case testGenericTypeAliasesTuple] +from typing import TypeVar, Tuple +T = TypeVar('T') + +SameTP = Tuple[T, T] +IntTP = Tuple[int, T] + +def f1(x: T) -> SameTP[T]: + return x, x + +a, b, c = f1(1) # E: Need more than 2 values to unpack (3 expected) +x, y = f1(1) +reveal_type(x) # N: Revealed type is "builtins.int" + +def f2(x: IntTP[T]) -> IntTP[T]: + return x + +f2((1, 2, 3)) # E: Argument 1 to "f2" has incompatible type "Tuple[int, int, int]"; expected "Tuple[int, Never]" +reveal_type(f2((1, 'x'))) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +[builtins fixtures/for.pyi] + +[case testGenericTypeAliasesCallable] +from typing import TypeVar, Generic, Callable +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + ... + +BadC = Callable[T] # E: Please use "Callable[[], ]" or "Callable" + +C = Callable[..., T] +C2 = Callable[[T, T], Node[T]] + +def make_cb(x: T) -> C[T]: + return lambda *args: x + +reveal_type(make_cb(1)) # N: Revealed type is "def (*Any, **Any) -> builtins.int" + +def use_cb(arg: T, cb: C2[T]) -> Node[T]: + return cb(arg, arg) + +use_cb(1, 1) # E: Argument 2 to "use_cb" has incompatible type "int"; expected "Callable[[int, int], Node[int]]" +my_cb: C2[int] +use_cb('x', my_cb) # E: Argument 2 to "use_cb" has incompatible type "Callable[[int, int], Node[int]]"; expected "Callable[[str, str], Node[str]]" +reveal_type(use_cb(1, my_cb)) # N: Revealed type is "__main__.Node[builtins.int]" +[builtins fixtures/tuple.pyi] + +[out] + +[case testGenericTypeAliasesPEPBasedExample] +from typing import TypeVar, List, Tuple +T = TypeVar('T', int, bool) + +Vec = List[Tuple[T, T]] + +vec = [] # type: Vec[bool] +vec.append('x') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "Tuple[bool, bool]" +reveal_type(vec[0]) # N: Revealed type is "Tuple[builtins.bool, builtins.bool]" + +def fun1(v: Vec[T]) -> T: + return v[0][0] +def fun2(v: Vec[T], scale: T) -> Vec[T]: + return v + +reveal_type(fun1([(1, 1)])) # N: Revealed type is "builtins.int" +fun1(1) # E: Argument 1 to "fun1" has incompatible type "int"; expected "List[Tuple[bool, bool]]" +fun1([(1, 'x')]) # E: Cannot infer type argument 1 of "fun1" + +reveal_type(fun2([(1, 1)], 1)) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.int]]" +fun2([('x', 'x')], 'x') # E: Value of type variable "T" of "fun2" cannot be "str" + +[builtins fixtures/list.pyi] + +[case testGenericTypeAliasesImporting] +from typing import TypeVar +from a import Node, TupledNode +T = TypeVar('T') + +n: TupledNode[int] +n.x = 1 +n.y = (1, 1) +n.y = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "Tuple[int, int]") + +def f(x: Node[T, T]) -> TupledNode[T]: + return Node(x.x, (x.x, x.x)) + +f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "Node[Never, Never]" +f(Node(1, 'x')) # E: Cannot infer type argument 1 of "f" +reveal_type(Node('x', 'x')) # N: Revealed type is "a.Node[builtins.str, builtins.str]" + +[file a.py] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +S = TypeVar('S') +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + +TupledNode = Node[T, Tuple[T, T]] + +[builtins fixtures/list.pyi] + +[case testGenericTypeAliasesImportingWithoutTypeVar] +from typing import Tuple +from lib import Transform + +def int_tf(m: int) -> Transform[int, str]: + def transform(i: int, pos: int) -> Tuple[int, str]: + pass + return transform + +var: Transform[int, str] +reveal_type(var) # N: Revealed type is "def (builtins.int, builtins.int) -> Tuple[builtins.int, builtins.str]" +[file lib.py] +from typing import Callable, TypeVar, Tuple + +T = TypeVar('T') +R = TypeVar('R') + +Transform = Callable[[T, int], Tuple[T, R]] +[builtins fixtures/tuple.pyi] +[out] + +[case testGenericTypeAliasesImportingWithoutTypeVarError] +from a import Alias +x: Alias[int, str] # E: Bad number of arguments for type alias, expected 1, given 2 +reveal_type(x) # N: Revealed type is "builtins.list[builtins.list[Any]]" + +[file a.py] +from typing import TypeVar, List +T = TypeVar('T') + +Alias = List[List[T]] +[builtins fixtures/list.pyi] + +[case testGenericAliasWithTypeVarsFromDifferentModules] +from mod import Alias, TypeVar + +S = TypeVar('S') +NewAlias = Alias[int, int, S, S] +class C: pass + +x: NewAlias[str] +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.int, builtins.str, builtins.str]]" +y: Alias[int, str, C, C] +reveal_type(y) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str, __main__.C, __main__.C]]" + +[file mod.py] +from typing import TypeVar, List, Tuple +import a +import b +T = TypeVar('T') +Alias = List[Tuple[T, a.T, b.T, b.B.T]] # alias_tvars here will be ['T', 'a.T', 'b.T', 'b.B.T'] + +[file a.py] +from typing import TypeVar +T = TypeVar('T') + +[file b.py] +from typing import TypeVar + +T = TypeVar('T') +class B: + T = TypeVar('T') +[builtins fixtures/list.pyi] +[out] + +[case testTypeAliasesResultingInPlainInstance] +from typing import Optional, Union + +O = Optional[int] +U = Union[int] + +x: O +y: U + +reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(y) # N: Revealed type is "builtins.int" + +U[int] # E: Type application targets a non-generic function or class +O[int] # E: Bad number of arguments for type alias, expected 0, given 1 \ + # E: Type application is only supported for generic classes + +[case testAliasesInClassBodyNormalVsSubscripted] + +from typing import Union, Type, Iterable + +class A: pass +class B(A): pass +class C: + a = A # This is a variable + b = Union[int, str] # This is an alias + c: Type[object] = Iterable[int] # This is however also a variable + if int(): + a = B + if int(): + b = int # E: Cannot assign multiple types to name "b" without an explicit "Type[...]" annotation + if int(): + c = int + def f(self, x: a) -> None: pass # E: Variable "__main__.C.a" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + def g(self, x: b) -> None: pass + def h(self, x: c) -> None: pass # E: Variable "__main__.C.c" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + x: b + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[out] + +[case testGenericTypeAliasesRuntimeExpressionsInstance] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + ... + +IntNode = Node[int, T] +IntNode[int](1, 1) +IntNode[int](1, 'a') # E: Argument 2 to "Node" has incompatible type "str"; expected "int" + +SameNode = Node[T, T] +ff = SameNode[T](1, 1) # E: Type variable "__main__.T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) +a = SameNode(1, 'x') +reveal_type(a) # N: Revealed type is "__main__.Node[Any, Any]" +b = SameNode[int](1, 1) +reveal_type(b) # N: Revealed type is "__main__.Node[builtins.int, builtins.int]" +SameNode[int](1, 'x') # E: Argument 2 to "Node" has incompatible type "str"; expected "int" + +[out] + +[case testGenericTypeAliasesRuntimeExpressionsOther] +from typing import TypeVar, Union, Tuple, Callable, Any +T = TypeVar('T') + +CA = Callable[[T], int] +TA = Tuple[T, int] +UA = Union[T, int] + +cs = CA + 1 # E: Unsupported left operand type for + ("") +reveal_type(cs) # N: Revealed type is "Any" + +ts = TA() # E: "" not callable +reveal_type(ts) # N: Revealed type is "Any" + +us = UA.x # E: "" has no attribute "x" +reveal_type(us) # N: Revealed type is "Any" + +xx = CA[str] + 1 # E: Type application is only supported for generic classes +yy = TA[str]() # E: Type application is only supported for generic classes +zz = UA[str].x # E: Type application is only supported for generic classes +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] +[out] + +[case testGenericTypeAliasesTypeVarBinding] +from typing import TypeVar, Generic, List +T = TypeVar('T') +S = TypeVar('S') + +class A(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: ... + +class B(Generic[T, S]): + def __init__(self, x: List[T], y: List[S]) -> None: ... + +SameA = A[T, T] +SameB = B[T, T] + +class C(Generic[T]): + a = None # type: SameA[T] + b = SameB[T]([], []) + +reveal_type(C[int]().a) # N: Revealed type is "__main__.A[builtins.int, builtins.int]" +reveal_type(C[str]().b) # N: Revealed type is "__main__.B[builtins.str, builtins.str]" + +[builtins fixtures/list.pyi] + +[case testGenericTypeAliasesTypeVarConstraints] +# flags: --show-column-numbers --no-strict-optional +from typing import TypeVar, Generic +T = TypeVar('T', int, list) +S = TypeVar('S', int, list) + +class A(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: ... + +BadA = A[str, T] # One error here +SameA = A[T, T] + +x = None # type: SameA[int] +y = None # type: SameA[str] # Another error here + +[builtins fixtures/list.pyi] +[out] +main:9:8: error: Value of type variable "T" of "A" cannot be "str" +main:13:1: error: Value of type variable "T" of "SameA" cannot be "str" + +[case testGenericTypeAliasesIgnoredPotentialAlias] +class A: ... +Bad = A[int] # type: ignore + +reveal_type(Bad) # N: Revealed type is "Any" +[out] + +[case testNoSubscriptionOfBuiltinAliases] +from typing import List, TypeVar + +list[int]() # E: "list" is not subscriptable + +ListAlias = List +def fun() -> ListAlias[int]: + pass + +reveal_type(fun()) # N: Revealed type is "builtins.list[builtins.int]" + +BuiltinAlias = list +BuiltinAlias[int]() # E: "list" is not subscriptable + +#check that error is reported only once, and type is still stored +T = TypeVar('T') +BadGenList = list[T] # E: "list" is not subscriptable + +reveal_type(BadGenList[int]()) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(BadGenList()) # N: Revealed type is "builtins.list[Any]" + +[builtins fixtures/list.pyi] +[out] + +[case testImportedTypeAliasInRuntimeContext] +from m import Alias + +n = Alias[int]([1]) +reveal_type(n) # N: Revealed type is "m.Node[builtins.list[builtins.int]]" +bad = Alias[str]([1]) # E: List item 0 has incompatible type "int"; expected "str" + +n2 = Alias([1]) # Same as Node[List[Any]] +reveal_type(n2) # N: Revealed type is "m.Node[builtins.list[Any]]" +[file m.py] +from typing import TypeVar, Generic, List +T = TypeVar('T') + +class Node(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + +Alias = Node[List[T]] +[builtins fixtures/list.pyi] +[out] + +-- Simplified declaration of generics +-- ---------------------------------- + +[case testSimplifiedGenericSimple] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class B(Generic[T]): + def b(self) -> T: ... + +class C(Generic[T]): + def c(self) -> T: ... + +class D(B[T], C[S]): ... + +reveal_type(D[str, int]().b()) # N: Revealed type is "builtins.str" +reveal_type(D[str, int]().c()) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] +[out] + +[case testSimplifiedGenericCallable] +from typing import TypeVar, Generic, Callable +T = TypeVar('T') +S = TypeVar('S') +class B(Generic[T]): + def b(self) -> T: ... + +class D(B[Callable[[T], S]]): ... + +reveal_type(D[str, int]().b()) # N: Revealed type is "def (builtins.str) -> builtins.int" +[builtins fixtures/list.pyi] +[out] + +[case testSimplifiedGenericComplex] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +S = TypeVar('S') +U = TypeVar('U') + +class A(Generic[T, S]): + pass + +class B(Generic[T, S]): + def m(self) -> Tuple[T, S]: + pass + +class C(A[S, B[T, int]], B[U, A[int, T]]): + pass + +c = C[object, int, str]() +reveal_type(c.m()) # N: Revealed type is "Tuple[builtins.str, __main__.A[builtins.int, builtins.int]]" +[builtins fixtures/tuple.pyi] +[out] + + +[case testSimplifiedGenericOrder] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') + +class B(Generic[T]): + def b(self) -> T: ... + +class C(Generic[T]): + def c(self) -> T: ... + +class D(B[T], C[S], Generic[S, T]): ... + +reveal_type(D[str, int]().b()) # N: Revealed type is "builtins.int" +reveal_type(D[str, int]().c()) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] +[out] + +[case testSimplifiedGenericDuplicate] +from typing import TypeVar, Generic +T = TypeVar('T') + +class A(Generic[T, T]): # E: Duplicate type variables in Generic[...] or Protocol[...] + pass + +a = A[int]() +[builtins fixtures/list.pyi] +[out] + +[case testSimplifiedGenericNotAll] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') + +class A(Generic[T]): + pass +class B(Generic[T]): + pass + +class C(A[T], B[S], Generic[T]): # E: If Generic[...] or Protocol[...] is present it should list all type variables + pass + +c = C[int, str]() +[builtins fixtures/list.pyi] +[out] + +[case testSimplifiedGenericInvalid] +from typing import TypeVar, Generic +T = TypeVar('T') + +class A(Generic[T]): + pass + +class B(A[S]): # E: Name "S" is not defined + pass +[builtins fixtures/list.pyi] +[out] + + +-- Multiple assignment with lists +-- ------------------------------ + + +[case testMultipleAssignmentWithLists] +from typing import List +class A: pass +class B: pass +class B2(B): pass +a: A +b: B +b2: B2 + +list_a = [a] +list_b = [b] +list_b2 = [b2] + +if int(): + a, b = list_a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b, a = list_a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b2, b2 = list_b # E: Incompatible types in assignment (expression has type "B", variable has type "B2") + +a, a = list_a +b, b2, b = list_b2 +[builtins fixtures/for.pyi] + +[case testMultipleAssignmentWithListsInInitialization] +from typing import List +class A: pass +list_object = [object()] +list_a = [A()] +a, b = list_object # type: (A, object) # E: Incompatible types in assignment (expression has type "object", variable has type "A") +c, d = list_object # type: (object, A) # E: Incompatible types in assignment (expression has type "object", variable has type "A") +e, f = list_a # type: (A, object) +[builtins fixtures/for.pyi] + +[case testMultipleAssignmentWithListAndIndexing] +from typing import List +a: List[A] +b: List[int] + +a[1], b[1] = a # E: Incompatible types in assignment (expression has type "A", target has type "int") +a[1], a[2] = a + +class A: pass +[file builtins.py] +from typing import TypeVar, Generic, Iterable +T = TypeVar('T') +class object: pass +class list(Iterable[T]): + def __setitem__(self, x: int, v: T) -> None: pass +class int: pass +class type: pass +class tuple: pass +class function: pass +class str: pass +class dict: pass + +[case testMultipleAssignmentWithIterable] +from typing import Iterable, TypeVar +a: int +b: str +T = TypeVar('T') + +def f(x: T) -> Iterable[T]: pass + +a, b = f(a) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +b, b = f(a) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +a, a = f(a) +b, b = f(b) +[builtins fixtures/for.pyi] + + +-- Error messages +-- -------------- + + +[case testErrorWithLongGenericTypeName] +from typing import TypeVar, Generic +B = TypeVar('B') +C = TypeVar('C') +D = TypeVar('D') +E = TypeVar('E') +F = TypeVar('F') +G = TypeVar('G') +H = TypeVar('H') +I = TypeVar('I') +J = TypeVar('J') +K = TypeVar('K') +L = TypeVar('L') +M = TypeVar('M') +N = TypeVar('N') +O = TypeVar('O') +P = TypeVar('P') +Q = TypeVar('Q') +R = TypeVar('R') +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') +V = TypeVar('V') +W = TypeVar('W') +X = TypeVar('X') +Y = TypeVar('Y') +Z = TypeVar('Z') +class OO: pass +a: A[object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object] + +def f(a: OO) -> None: + pass + +f(a) # E: Argument 1 to "f" has incompatible type "A[object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object]"; expected "OO" + +class A(Generic[B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z]): pass + +[case testErrorWithShorterGenericTypeName] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +a: A[object, B] +def f(a: 'B') -> None: pass + +f(a) # E: Argument 1 to "f" has incompatible type "A[object, B]"; expected "B" + +class A(Generic[S, T]): pass +class B: pass + +[case testErrorWithShorterGenericTypeName2] +from typing import Callable, TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +a: A[object, Callable[[], None]] +def f(a: 'B') -> None: pass + +f(a) # E: Argument 1 to "f" has incompatible type "A[object, Callable[[], None]]"; expected "B" + +class A(Generic[S, T]): pass +class B: pass + + +-- Overloads + generics +-- -------------------- + + +[case testGenericArgumentInOverload] +from foo import * +[file foo.pyi] +from typing import overload, List +class A: pass +class B: pass +a: A +b: B + +@overload +def f(a: List[A]) -> A: pass +@overload +def f(a: B) -> B: pass + +b = f([a]) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = f([b]) # E: List item 0 has incompatible type "B"; expected "A" +a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +a = f([a]) +b = f(b) +[builtins fixtures/list.pyi] + +[case testGenericFunctionAsOverloadItem] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar, List +T = TypeVar('T') +class A: pass +class B: pass + +@overload +def f(a: B) -> B: pass +@overload +def f(a: List[T]) -> T: pass + +a: A +b: B + +if int(): + b = f([a]) # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = f([b]) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +if int(): + a = f([a]) + b = f([b]) +if int(): + b = f(b) +[builtins fixtures/list.pyi] + +[case testGenericDictWithOverload] +from typing import Dict, Generic, TypeVar, Any, overload +T = TypeVar("T") + +class Key(Generic[T]): ... +class CustomDict(dict): + @overload # type: ignore[override] + def __setitem__(self, key: Key[T], value: T) -> None: ... + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + def __setitem__(self, key, value): + return super().__setitem__(key, value) + +def a1(d: Dict[str, Any]) -> None: + if (var := d.get("arg")) is None: + var = d["arg"] = {} + reveal_type(var) # N: Revealed type is "builtins.dict[Any, Any]" + +def a2(d: CustomDict) -> None: + if (var := d.get("arg")) is None: + var = d["arg"] = {} + reveal_type(var) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + + +-- Type variable scoping +-- --------------------- + + +[case testLocalTypeVariable] +from typing import TypeVar +def f() -> None: + T = TypeVar('T') + def g(x: T) -> T: pass + a = g(1) + if int(): + a = 1 + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[out] + +[case testClassLevelTypeVariable] +from typing import TypeVar +class A: + T = TypeVar('T') + def g(self, x: T) -> T: pass +a = A().g(1) +if int(): + a = 1 +if int(): + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testGenericInnerClass] +from typing import TypeVar, Generic +T = TypeVar('T') +class A: + class B(Generic[T]): + def meth(self) -> T: ... + B[int]() + reveal_type(B[int]().meth) # N: Revealed type is "def () -> builtins.int" + +A.B[int]() +reveal_type(A.B[int]().meth) # N: Revealed type is "def () -> builtins.int" + +[case testGenericClassInnerFunctionTypeVariable] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def __init__(self, a: T) -> None: + self.a = a + def f(self, n: int) -> None: + def g(a: T): + self.a = a + g(self.a) + g(n) # E: Argument 1 to "g" has incompatible type "int"; expected "T" + +-- This is non-trivial with new analyzer (and also in fine grained incremental): +-- We need to store whole tvar_scope, not only active class. +[case testFunctionInGenericInnerClassTypeVariable-skip] + +from typing import TypeVar, Generic + +T = TypeVar('T') +class Outer(Generic[T]): + class Inner: + x: T # E: Invalid type "__main__.T" + def f(self, x: T) -> T: ... # E: Type variable "T" is bound by an outer class + def g(self) -> None: + y: T # E: Invalid type "__main__.T" + +[case testGenericClassInsideOtherGenericClass] +from typing import TypeVar, Generic +T = TypeVar("T") +K = TypeVar("K") + +class C(Generic[T]): + def __init__(self, t: T) -> None: ... + class F(Generic[K]): + def __init__(self, k: K) -> None: ... + def foo(self) -> K: ... + +reveal_type(C.F(17).foo()) # N: Revealed type is "builtins.int" +reveal_type(C("").F(17).foo()) # N: Revealed type is "builtins.int" +reveal_type(C.F) # N: Revealed type is "def [K] (k: K`1) -> __main__.C.F[K`1]" +reveal_type(C("").F) # N: Revealed type is "def [K] (k: K`6) -> __main__.C.F[K`6]" + + +-- Callable subtyping with generic functions +-- ----------------------------------------- + + +[case testSubtypingWithGenericFunctions] +from typing import TypeVar +A = TypeVar('A') +B = TypeVar('B') + +def f1(x: A) -> A: ... +def f2(x: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar +def f3(x: B) -> B: ... +def f4(x: int) -> A: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + +y1 = f1 +if int(): + y1 = f1 +if int(): + y1 = f2 +if int(): + y1 = f3 +if int(): + y1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[int], A@f4]", variable has type "Callable[[A@f1], A@f1]") + +y2 = f2 +if int(): + y2 = f2 +if int(): + y2 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[A@f2], B]") +if int(): + y2 = f3 # E: Incompatible types in assignment (expression has type "Callable[[B@f3], B@f3]", variable has type "Callable[[A], B@f2]") +if int(): + y2 = f4 # E: Incompatible types in assignment (expression has type "Callable[[int], A@f4]", variable has type "Callable[[A@f2], B]") + +y3 = f3 +if int(): + y3 = f3 +if int(): + y3 = f1 +if int(): + y3 = f2 +if int(): + y3 = f4 # E: Incompatible types in assignment (expression has type "Callable[[int], A]", variable has type "Callable[[B], B]") + +y4 = f4 +if int(): + y4 = f4 +if int(): + y4 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[int], A@f4]") +if int(): + y4 = f2 +if int(): + y4 = f3 # E: Incompatible types in assignment (expression has type "Callable[[B], B]", variable has type "Callable[[int], A]") + +[case testSubtypingWithGenericInnerFunctions] +from typing import TypeVar +A = TypeVar('A') +B = TypeVar('B') +T = TypeVar('T') +def outer(t: T) -> None: + def f1(x: A) -> A: ... + def f2(x: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + def f3(x: T) -> A: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + def f4(x: A) -> T: ... + def f5(x: T) -> T: ... + + y1 = f1 + if int(): + y1 = f2 + y1 = f3 # E: Incompatible types in assignment (expression has type "Callable[[T], A@f3]", variable has type "Callable[[A@f1], A@f1]") + y1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[A@f4], T]", variable has type "Callable[[A@f1], A@f1]") + y1 = f5 # E: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[A], A]") + + y2 = f2 + if int(): + y2 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[A@f2], B]") + + y3 = f3 + if int(): + y3 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[T], A@f3]") + y3 = f2 + y3 = f4 # E: Incompatible types in assignment (expression has type "Callable[[A@f4], T]", variable has type "Callable[[T], A@f3]") + y3 = f5 # E: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[T], A]") + + y4 = f4 + if int(): + y4 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[A@f4], T]") + y4 = f2 + y4 = f3 # E: Incompatible types in assignment (expression has type "Callable[[T], A@f3]", variable has type "Callable[[A@f4], T]") + y4 = f5 # E: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[A], T]") + + y5 = f5 + if int(): + y5 = f1 + y5 = f2 + y5 = f3 + y5 = f4 + +[case testSubtypingWithGenericFunctionUsingTypevarWithValues] +from typing import TypeVar, Callable +T = TypeVar('T', int, str) +def f(x: T) -> T: pass +def g1(f: Callable[[str], str]) -> None: pass +g1(f) +def g2(f: Callable[[int], int]) -> None: pass +g2(f) +def g3(f: Callable[[object], object]) -> None: pass +g3(f) # E: Argument 1 to "g3" has incompatible type "Callable[[T], T]"; \ + expected "Callable[[object], object]" + +[case testSubtypingWithGenericFunctionUsingTypevarWithValues2] +from typing import TypeVar, Callable +T = TypeVar('T', int, str) +def f(x: T) -> T: pass +g = f +g = f + + +--Operations on type variable types +-- --------------------------------- + + +[case testTypeVariableTypeEquality] +from typing import TypeVar +T = TypeVar('T') +def f(a: T, b: T) -> T: + a.__ne__(b) + if a == b: + return a + else: + return b +[builtins fixtures/ops.pyi] + +[case testTypeVariableTypeIs] +from typing import TypeVar +T = TypeVar('T') +def f(a: T, b: T) -> T: + if a is b or a is 1: + return a + else: + return b +[builtins fixtures/ops.pyi] + +[case testTypeVarLessThan] +from typing import TypeVar +T = TypeVar('T') +def f(a: T, b: T) -> T: + if a < b: # E: Unsupported left operand type for < ("T") + return a + else: + return b +[builtins fixtures/ops.pyi] + +[case testTypeVarReversibleOperator] +from typing import TypeVar +class A: + def __mul__(cls, other: int) -> str: return "" +T = TypeVar("T", bound=A) +def f(x: T) -> str: + return reveal_type(x * 0) # N: Revealed type is "builtins.str" + +[case testTypeVarReversibleOperatorTuple] +from typing import TypeVar, Tuple +class A(Tuple[int, int]): + def __mul__(cls, other: Tuple[int, int]) -> str: return "" # type: ignore # overriding default __mul__ +T = TypeVar("T", bound=A) +def f(x: T) -> str: + return reveal_type(x * (1, 2) ) # N: Revealed type is "builtins.str" + +[builtins fixtures/tuple.pyi] + + +-- Subtyping generic callables +-- --------------------------- + +[case testSubtypingGenericTypeObject] +from typing import Callable, Generic, TypeVar +T = TypeVar('T') +class C(Generic[T]): + def __init__(self) -> None: pass +x = C # type: Callable[[], C[int]] +y = C # type: Callable[[], int] # E: Incompatible types in assignment (expression has type "Type[C[Any]]", variable has type "Callable[[], int]") + + +-- Special cases +-- ------------- + + +[case testIdentityHigherOrderFunction] +from typing import Callable, TypeVar +A = TypeVar('A') +B = TypeVar('B') +def square(n: int) -> int: + return n +def id(f: Callable[[A], B]) -> Callable[[A], B]: + return f +g = id(square) +g(1) +g('x') # E: Argument 1 has incompatible type "str"; expected "int" + + +[case testIdentityHigherOrderFunction2] +from typing import Callable, TypeVar +A = TypeVar('A') +def voidify(n: int) -> None: pass +def identity(f: Callable[[A], None]) -> Callable[[A], None]: + return f +identity(voidify)(3) + +[case testIdentityHigherOrderFunction3] +from typing import Callable, TypeVar +A = TypeVar('A') +B = TypeVar('B') +def fn(n: B) -> None: pass +def identity(f: A) -> A: + return f +identity(fn) +identity(fn)('x') + +[case testTypeVariableUnionAndCallableInTypeInference] +from typing import Union, Callable, TypeVar +T = TypeVar('T') +def f(x: T, y: Union[T, Callable[[T], None]]) -> None: pass +f('', '') + +[case testGenericFunctionsWithUnalignedIds] +from typing import TypeVar +A = TypeVar('A') +B = TypeVar('B') +def f1(x: int, y: A) -> A: ... +def f2(x: int, y: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar +def f3(x: A, y: B) -> B: ... +g = f1 +g = f2 +g = f3 + +[case testTypeVariableWithContainerAndTuple] +from typing import TypeVar, Container +T = TypeVar('T') +def f(x: Container[T]) -> T: ... +reveal_type(f((1, 2))) # N: Revealed type is "builtins.int" +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testClassMethodInGenericClassWithGenericConstructorArg] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def __init__(self, a: T) -> None: pass + @classmethod + def f(cls) -> None: pass +[builtins fixtures/classmethod.pyi] + +[case testClassMethodInClassWithGenericConstructor] +from typing import TypeVar, Generic +T = TypeVar('T') +class A: + def __init__(self, a: T) -> None: pass + @classmethod + def f(cls) -> None: pass +[builtins fixtures/classmethod.pyi] + +[case testGenericOperatorMethodOverlapping] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +T2 = TypeVar('T2') +S = TypeVar('S', bound=str) +S2 = TypeVar('S2', bound=str) +class G(Generic[T]): + pass +class A: + def __or__(self, x: G[T]) -> G[T]: pass + def __ior__(self, x: G[T2]) -> G[T2]: pass +class B: + def __or__(self, x: G[T]) -> G[T]: pass + def __ior__(self, x: G[S]) -> G[S]: pass \ + # E: Signatures of "__ior__" and "__or__" are incompatible +class C: + def __or__(self, x: G[S]) -> G[S]: pass + def __ior__(self, x: G[S2]) -> G[S2]: pass + +[case testGenericOperatorMethodOverlapping2] +from typing import TypeVar, Generic, Tuple +X = TypeVar('X') +T = TypeVar('T', int, str) +T2 = TypeVar('T2', int, str) +S = TypeVar('S', float, str) +S2 = TypeVar('S2', float, str) +class G(Generic[X]): + pass +class A: + def __or__(self, x: G[T]) -> G[T]: pass + def __ior__(self, x: G[T2]) -> G[T2]: pass +class B: + def __or__(self, x: G[T]) -> G[T]: pass + def __ior__(self, x: G[S]) -> G[S]: pass \ + # E: Signatures of "__ior__" and "__or__" are incompatible +class C: + def __or__(self, x: G[S]) -> G[S]: pass + def __ior__(self, x: G[S2]) -> G[S2]: pass +class D: + def __or__(self, x: G[X]) -> G[X]: pass + def __ior__(self, x: G[S2]) -> G[S2]: pass \ + # E: Signatures of "__ior__" and "__or__" are incompatible + +[case testConstraintInferenceForAnyAgainstTypeT] +from typing import Type, Any, TypeVar + +T = TypeVar('T') + +def f(c: Type[T]) -> T: ... + +x: Any +reveal_type(f(x)) # N: Revealed type is "Any" + +[case testCallTypeTWithGenericBound] +from typing import Generic, TypeVar, Type +T = TypeVar('T') +S = TypeVar('S', bound='A') + +class A(Generic[T]): pass + +def f(cls: Type[S]) -> None: + cls() + +[case testQualifiedTypeVariableName] +import b +def f(x: b.T) -> b.T: return x +reveal_type(f) +reveal_type(b.g) +[file b.py] +from typing import TypeVar +T = TypeVar('T') +def g(x: T) -> T: return x +[out] +main:3: note: Revealed type is "def [b.T] (x: b.T`-1) -> b.T`-1" +main:4: note: Revealed type is "def [T] (x: T`-1) -> T`-1" + +[case testPartiallyQualifiedTypeVariableName] +from p import b +def f(x: b.T) -> b.T: return x +reveal_type(f) +reveal_type(b.g) +[file p/__init__.py] +[file p/b.py] +from typing import TypeVar +T = TypeVar('T') +def g(x: T) -> T: return x +[out] +main:3: note: Revealed type is "def [b.T] (x: b.T`-1) -> b.T`-1" +main:4: note: Revealed type is "def [T] (x: T`-1) -> T`-1" + +[case testGenericClassMethodSimple] +from typing import Generic, TypeVar +T = TypeVar('T') + +class C(Generic[T]): + @classmethod + def get(cls) -> T: ... + +class D(C[str]): ... + +reveal_type(D.get()) # N: Revealed type is "builtins.str" +reveal_type(D().get()) # N: Revealed type is "builtins.str" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodExpansion] +from typing import Generic, TypeVar, Tuple +T = TypeVar('T') + +class C(Generic[T]): + @classmethod + def get(cls) -> T: ... +class D(C[Tuple[T, T]]): ... +class E(D[str]): ... + +reveal_type(E.get()) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +reveal_type(E().get()) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodExpansionReplacingTypeVar] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') + +class C(Generic[T]): + @classmethod + def get(cls) -> T: ... + +class D(C[S]): ... +class E(D[int]): ... + +reveal_type(E.get()) # N: Revealed type is "builtins.int" +reveal_type(E().get()) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodUnboundOnClass] +from typing import Generic, TypeVar +T = TypeVar('T') + +class C(Generic[T]): + @classmethod + def get(cls) -> T: ... + @classmethod + def make_one(cls, x: T) -> C[T]: ... + +reveal_type(C.get) # N: Revealed type is "def [T] () -> T`1" +reveal_type(C[int].get) # N: Revealed type is "def () -> builtins.int" +reveal_type(C.make_one) # N: Revealed type is "def [T] (x: T`1) -> __main__.C[T`1]" +reveal_type(C[int].make_one) # N: Revealed type is "def (x: builtins.int) -> __main__.C[builtins.int]" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodUnboundOnSubClass] +from typing import Generic, TypeVar, Tuple +T = TypeVar('T') +S = TypeVar('S') + +class C(Generic[T]): + @classmethod + def get(cls) -> T: ... + @classmethod + def make_one(cls, x: T) -> C[T]: ... +class D(C[Tuple[T, S]]): ... +class E(D[S, str]): ... + +reveal_type(D.make_one) # N: Revealed type is "def [T, S] (x: Tuple[T`1, S`2]) -> __main__.C[Tuple[T`1, S`2]]" +reveal_type(D[int, str].make_one) # N: Revealed type is "def (x: Tuple[builtins.int, builtins.str]) -> __main__.C[Tuple[builtins.int, builtins.str]]" +reveal_type(E.make_one) # N: Revealed type is "def [S] (x: Tuple[S`1, builtins.str]) -> __main__.C[Tuple[S`1, builtins.str]]" +reveal_type(E[int].make_one) # N: Revealed type is "def (x: Tuple[builtins.int, builtins.str]) -> __main__.C[Tuple[builtins.int, builtins.str]]" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassClsNonGeneric] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + @classmethod + def f(cls, x: T) -> T: + return x + + @classmethod + def other(cls) -> None: + reveal_type(C) # N: Revealed type is "def [T] () -> __main__.C[T`1]" + reveal_type(C[T]) # N: Revealed type is "def () -> __main__.C[T`1]" + reveal_type(C.f) # N: Revealed type is "def [T] (x: T`1) -> T`1" + reveal_type(C[T].f) # N: Revealed type is "def (x: T`1) -> T`1" + reveal_type(cls.f) # N: Revealed type is "def (x: T`1) -> T`1" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassUnrelatedVars] +from typing import TypeVar, Generic + +T = TypeVar('T') +T2 = TypeVar('T2') + +class C(Generic[T]): + @classmethod + def f(cls, x: T) -> T: + return x + + @classmethod + def g(cls, x: T2) -> T2: + cls.f(x) # E: Argument 1 to "f" of "C" has incompatible type "T2"; expected "T" + return x +[builtins fixtures/classmethod.pyi] + +[case testGenericClassInGenericFunction] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, item: T) -> None: ... + @classmethod + def f(cls, x: T) -> T: + return x + +def foo(x: T, y: int) -> T: + C(y) # OK + C[T](y) # E: Argument 1 to "C" has incompatible type "int"; expected "T" + C[T].f(y) # E: Argument 1 to "f" of "C" has incompatible type "int"; expected "T" + C[T].f(x) # OK + return x +[builtins fixtures/classmethod.pyi] + +# TODO: enable this when #7935 is fixed. +[case testGenericClassInGenericFunctionOverloadedConstructor-skip] +from typing import TypeVar, Generic, overload + +T = TypeVar('T') + +class C(Generic[T]): + @overload + def __new__(cls) -> C[None]: ... + @overload + def __new__(cls, item: T) -> C[T]: ... + def __new__(cls, item=None): + ... + @classmethod + def f(cls, x: T) -> T: + return x + +def foo(x: T, y: int) -> T: + C.f(y) + C(y) # OK + C[T](y) # E: Argument 1 to "C" has incompatible type "int"; expected "T" + C[T].f(y) # E: Argument 1 to "f" of "C" has incompatible type "int"; expected "T" + C[T].f(x) # OK + return x +[builtins fixtures/classmethod.pyi] + +[case testGenericClassDirectCall] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, item: T) -> None: ... + @classmethod + def f(cls) -> None: + cls(1) # E: Argument 1 to "C" has incompatible type "int"; expected "T" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassAlternativeConstructorPrecise] +from typing import Generic, TypeVar, Type, Tuple + +T = TypeVar('T') + +class Base(Generic[T]): + Q = TypeVar('Q', bound=Base[T]) + + def __init__(self, item: T) -> None: ... + + @classmethod + def make_pair(cls: Type[Q], item: T) -> Tuple[Q, Q]: + if bool(): + return cls(0), cls(0) # E: Argument 1 to "Base" has incompatible type "int"; expected "T" + return cls(item), cls(item) +[builtins fixtures/classmethod.pyi] + +[case testGenericClassAlternativeConstructorPreciseOverloaded] +from typing import Generic, TypeVar, Type, Tuple, overload, Union + +T = TypeVar('T') + +class Base(Generic[T]): + Q = TypeVar('Q', bound=Base[T]) + + def __init__(self, item: T) -> None: ... + + @overload + @classmethod + def make_some(cls: Type[Q], item: T) -> Q: ... + + @overload + @classmethod + def make_some(cls: Type[Q], item: T, n: int) -> Tuple[Q, ...]: ... + + @classmethod + def make_some(cls: Type[Q], item: T, n: int = 0) -> Union[Q, Tuple[Q, ...]]: + if n: + return (cls(item),) + return cls(item) + +reveal_type(Base.make_some) # N: Revealed type is "Overload(def [T] (item: T`1) -> __main__.Base[T`1], def [T] (item: T`1, n: builtins.int) -> builtins.tuple[__main__.Base[T`1], ...])" +reveal_type(Base.make_some(1)) # N: Revealed type is "__main__.Base[builtins.int]" +reveal_type(Base.make_some(1, 1)) # N: Revealed type is "builtins.tuple[__main__.Base[builtins.int], ...]" + +class Sub(Base[str]): ... +Sub.make_some(1) # E: No overload variant of "make_some" of "Base" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def make_some(cls, item: str) -> Sub \ + # N: def make_some(cls, item: str, n: int) -> Tuple[Sub, ...] +[builtins fixtures/classmethod.pyi] + +[case testNoGenericAccessOnImplicitAttributes] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + + @classmethod + def meth(cls) -> None: + cls.x # E: Access to generic instance variables via class is ambiguous +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodUnboundOnClassNonMatchingIdNonGeneric] +from typing import Generic, TypeVar, Any, Tuple, Type + +T = TypeVar('T') +S = TypeVar('S') +Q = TypeVar('Q', bound='A[Any]') + +class A(Generic[T]): + @classmethod + def foo(cls: Type[Q]) -> Tuple[T, Q]: ... + +class B(A[T], Generic[T, S]): + def meth(self) -> None: + reveal_type(A[T].foo) # N: Revealed type is "def () -> Tuple[T`1, __main__.A[T`1]]" + @classmethod + def other(cls) -> None: + reveal_type(cls.foo) # N: Revealed type is "def () -> Tuple[T`1, __main__.B[T`1, S`2]]" +reveal_type(B.foo) # N: Revealed type is "def [T, S] () -> Tuple[T`1, __main__.B[T`1, S`2]]" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassAlternativeConstructorPrecise2] +from typing import Generic, TypeVar, Type, Tuple, Any + +T = TypeVar('T') +Q = TypeVar('Q') + +class Base(Generic[T]): + def __init__(self, item: T) -> None: ... + @classmethod + def make_pair(cls: Type[Q], item: T) -> Tuple[Q, Q]: ... +class Sub(Base[T]): + ... + +reveal_type(Sub.make_pair('yes')) # N: Revealed type is "Tuple[__main__.Sub[builtins.str], __main__.Sub[builtins.str]]" +Sub[int].make_pair('no') # E: Argument 1 to "make_pair" of "Base" has incompatible type "str"; expected "int" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassAttrUnboundOnClass] +from typing import Generic, TypeVar +T = TypeVar('T') + +class C(Generic[T]): + x: T + @classmethod + def get(cls) -> T: + return cls.x # OK + +x = C.x # E: Access to generic instance variables via class is ambiguous +reveal_type(x) # N: Revealed type is "Any" +xi = C[int].x # E: Access to generic instance variables via class is ambiguous +reveal_type(xi) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassAttrUnboundOnSubClass] +from typing import Generic, TypeVar, Tuple +T = TypeVar('T') + +class C(Generic[T]): + x: T +class D(C[int]): ... +class E(C[int]): + x = 42 + +x = D.x # E: Access to generic instance variables via class is ambiguous +reveal_type(x) # N: Revealed type is "builtins.int" +E.x # OK + +[case testGenericClassMethodOverloaded] +from typing import Generic, TypeVar, overload, Tuple +T = TypeVar('T') + +class C(Generic[T]): + @overload + @classmethod + def get(cls) -> T: ... + @overload + @classmethod + def get(cls, n: int) -> Tuple[T, ...]: ... + @classmethod + def get(cls, n: int = 0): + pass + +class D(C[str]): ... + +reveal_type(D.get()) # N: Revealed type is "builtins.str" +reveal_type(D.get(42)) # N: Revealed type is "builtins.tuple[builtins.str, ...]" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodAnnotation] +from typing import Generic, TypeVar, Type +T = TypeVar('T') + +class Maker(Generic[T]): + x: T + @classmethod + def get(cls) -> T: ... + +class B(Maker[B]): ... + +def f(o: Maker[T]) -> T: + if bool(): + return o.x + return o.get() +b = f(B()) +reveal_type(b) # N: Revealed type is "__main__.B" + +def g(t: Type[Maker[T]]) -> T: + if bool(): + return t.x + return t.get() +bb = g(B) +reveal_type(bb) # N: Revealed type is "__main__.B" +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodAnnotationDecorator] +from typing import Generic, Callable, TypeVar, Iterator + +T = TypeVar('T') + +class Box(Generic[T]): + @classmethod + def wrap(cls, generator: Callable[[], T]) -> Box[T]: ... + +class IteratorBox(Box[Iterator[T]]): ... + +@IteratorBox.wrap # E: Argument 1 to "wrap" of "Box" has incompatible type "Callable[[], int]"; expected "Callable[[], Iterator[Never]]" +def g() -> int: + ... +[builtins fixtures/classmethod.pyi] + +[case testGenericClassMethodInGenericFunction] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') + +class C(Generic[T]): + @classmethod + def get(cls) -> T: ... + +def func(x: S) -> S: + return C[S].get() +[builtins fixtures/classmethod.pyi] + +[case testGenericStaticMethodInGenericFunction] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') + +class C(Generic[T]): + @staticmethod + def get() -> T: ... + +def func(x: S) -> S: + return C[S].get() +[builtins fixtures/staticmethod.pyi] + +[case testMultipleAssignmentFromAnyIterable] +from typing import Any +class A: + def __iter__(self) -> Any: ... + +x, y = A() +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "Any" + +[case testSubclassingGenericSelfClassMethod] +from typing import TypeVar, Type + +AT = TypeVar('AT', bound='A') + +class A: + @classmethod + def from_config(cls: Type[AT]) -> AT: + ... + +class B(A): + @classmethod + def from_config(cls: Type[B]) -> B: + return B() +[builtins fixtures/classmethod.pyi] + +[case testSubclassingGenericSelfClassMethodOptional] +from typing import TypeVar, Type, Optional + +AT = TypeVar('AT', bound='A') + +class A: + @classmethod + def from_config(cls: Type[AT]) -> Optional[AT]: + return None + +class B(A): + @classmethod + def from_config(cls: Type[B]) -> Optional[B]: + return B() +[builtins fixtures/classmethod.pyi] + +[case testSubclassingGenericSelfClassMethodNonAnnotated] +from typing import TypeVar, Type + +AT = TypeVar('AT', bound='A') + +class A: + @classmethod + def from_config(cls: Type[AT]) -> AT: + ... + +class B(A): + @classmethod + def from_config(cls) -> B: + return B() +[builtins fixtures/classmethod.pyi] + +[case testAbstractGenericMethodInference] +from abc import ABC, abstractmethod +from typing import Callable, Generic, TypeVar + +A = TypeVar('A') +B = TypeVar('B') +C = TypeVar('C') + +class TwoTypes(Generic[A, B]): + + def __call__(self) -> B: pass + +class MakeTwoAbstract(ABC, Generic[A]): + + def __init__(self) -> None: pass + + @abstractmethod + def __call__(self, b: B) -> TwoTypes[A, B]: pass + +class MakeTwoConcrete(Generic[A]): + + def __call__(self, b: B) -> TwoTypes[A, B]: pass + + +class MakeTwoGenericSubAbstract(Generic[C], MakeTwoAbstract[C]): + + def __call__(self, b: B) -> TwoTypes[C, B]: pass + +class MakeTwoAppliedSubAbstract(MakeTwoAbstract[str]): + + def __call__(self, b: B) -> TwoTypes[str, B]: pass + +class Test(): + + def make_two(self, + mts: MakeTwoAbstract[A], + mte: MakeTwoConcrete[A], + mtgsa: MakeTwoGenericSubAbstract[A], + mtasa: MakeTwoAppliedSubAbstract) -> None: + reveal_type(mts(2)) # N: Revealed type is "__main__.TwoTypes[A`-1, builtins.int]" + reveal_type(mte(2)) # N: Revealed type is "__main__.TwoTypes[A`-1, builtins.int]" + reveal_type(mtgsa(2)) # N: Revealed type is "__main__.TwoTypes[A`-1, builtins.int]" + reveal_type(mtasa(2)) # N: Revealed type is "__main__.TwoTypes[builtins.str, builtins.int]" + reveal_type(MakeTwoConcrete[int]()('foo')) # N: Revealed type is "__main__.TwoTypes[builtins.int, builtins.str]" + reveal_type(MakeTwoConcrete[str]()(2)) # N: Revealed type is "__main__.TwoTypes[builtins.str, builtins.int]" + reveal_type(MakeTwoAppliedSubAbstract()('foo')) # N: Revealed type is "__main__.TwoTypes[builtins.str, builtins.str]" + reveal_type(MakeTwoAppliedSubAbstract()(2)) # N: Revealed type is "__main__.TwoTypes[builtins.str, builtins.int]" + reveal_type(MakeTwoGenericSubAbstract[str]()('foo')) # N: Revealed type is "__main__.TwoTypes[builtins.str, builtins.str]" + reveal_type(MakeTwoGenericSubAbstract[str]()(2)) # N: Revealed type is "__main__.TwoTypes[builtins.str, builtins.int]" + +[case testGenericClassPropertyBound] +from typing import Generic, TypeVar, Callable, Type, List, Dict + +T = TypeVar('T') +U = TypeVar('U') + +def classproperty(f: Callable[..., U]) -> U: ... + +class C(Generic[T]): + @classproperty + def test(self) -> T: ... + +class D(C[str]): ... +class E1(C[T], Generic[T, U]): ... +class E2(C[U], Generic[T, U]): ... +class G(C[List[T]]): ... + +x: C[int] +y: Type[C[int]] +reveal_type(x.test) # N: Revealed type is "builtins.int" +reveal_type(y.test) # N: Revealed type is "builtins.int" + +xd: D +yd: Type[D] +reveal_type(xd.test) # N: Revealed type is "builtins.str" +reveal_type(yd.test) # N: Revealed type is "builtins.str" + +ye1: Type[E1[int, str]] +ye2: Type[E2[int, str]] +reveal_type(ye1.test) # N: Revealed type is "builtins.int" +reveal_type(ye2.test) # N: Revealed type is "builtins.str" + +xg: G[int] +yg: Type[G[int]] +reveal_type(xg.test) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(yg.test) # N: Revealed type is "builtins.list[builtins.int]" + +class Sup: + attr: int +S = TypeVar('S', bound=Sup) + +def func(tp: Type[C[S]]) -> S: + reveal_type(tp.test.attr) # N: Revealed type is "builtins.int" + + reg: Dict[S, G[S]] + reveal_type(reg[tp.test]) # N: Revealed type is "__main__.G[S`-1]" + reveal_type(reg[tp.test].test) # N: Revealed type is "builtins.list[S`-1]" + + if bool(): + return tp.test + else: + return reg[tp.test].test[0] +[builtins fixtures/dict.pyi] + +[case testGenericFunctionAliasExpand] +from typing import Optional, TypeVar + +T = TypeVar("T") +def gen(x: T) -> T: ... +gen_a = gen + +S = TypeVar("S", int, str) +class C: ... +def test() -> Optional[S]: + reveal_type(gen_a(C())) # N: Revealed type is "__main__.C" + return None + +[case testGenericFunctionMemberExpand] +from typing import Optional, TypeVar, Callable + +T = TypeVar("T") + +class A: + def __init__(self) -> None: + self.gen: Callable[[T], T] + +S = TypeVar("S", int, str) +class C: ... +def test() -> Optional[S]: + reveal_type(A().gen(C())) # N: Revealed type is "__main__.C" + return None + +[case testGenericJoinCovariant] +from typing import Generic, TypeVar, List + +T = TypeVar("T", covariant=True) + +class Container(Generic[T]): ... +class Base: ... +class A(Base): ... +class B(Base): ... + +a: A +b: B + +a_c: Container[A] +b_c: Container[B] + +reveal_type([a, b]) # N: Revealed type is "builtins.list[__main__.Base]" +reveal_type([a_c, b_c]) # N: Revealed type is "builtins.list[__main__.Container[__main__.Base]]" +[builtins fixtures/list.pyi] + +[case testGenericJoinContravariant] +from typing import Generic, TypeVar, List + +T = TypeVar("T", contravariant=True) + +class Container(Generic[T]): ... +class A: ... +class B(A): ... + +a_c: Container[A] +b_c: Container[B] + +# TODO: this can be more precise than "object", see a comment in mypy/join.py +reveal_type([a_c, b_c]) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/list.pyi] + +[case testGenericJoinRecursiveTypes] +from typing import Sequence, TypeVar + +class A(Sequence[A]): ... +class B(Sequence[B]): ... + +a: A +b: B + +reveal_type([a, b]) # N: Revealed type is "builtins.list[typing.Sequence[builtins.object]]" +[builtins fixtures/list.pyi] + +[case testGenericJoinRecursiveInvariant] +from typing import Generic, TypeVar + +T = TypeVar("T") +class I(Generic[T]): ... + +class A(I[A]): ... +class B(I[B]): ... + +a: A +b: B +reveal_type([a, b]) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/list.pyi] + +[case testGenericJoinNestedInvariantAny] +from typing import Any, Generic, TypeVar + +T = TypeVar("T") +class I(Generic[T]): ... + +a: I[I[int]] +b: I[I[Any]] +reveal_type([a, b]) # N: Revealed type is "builtins.list[__main__.I[__main__.I[Any]]]" +reveal_type([b, a]) # N: Revealed type is "builtins.list[__main__.I[__main__.I[Any]]]" +[builtins fixtures/list.pyi] + +[case testOverlappingTypeVarIds] +from typing import TypeVar, Generic + +class A: ... +class B: ... + +T = TypeVar("T", bound=A) +V = TypeVar("V", bound=B) +S = TypeVar("S") + +class Whatever(Generic[T]): + def something(self: S) -> S: + return self + +# the "V" here had the same id as "T" and so mypy used to think it could expand one into another. +# this test is here to make sure that doesn't happen! +class WhateverPartTwo(Whatever[A], Generic[V]): + def something(self: S) -> S: + return self + + +[case testConstrainedGenericSuper] +from typing import Generic, TypeVar + +AnyStr = TypeVar("AnyStr", str, bytes) + +class Foo(Generic[AnyStr]): + def method1(self, s: AnyStr, t: AnyStr) -> None: ... + +class Bar(Foo[AnyStr]): + def method1(self, s: AnyStr, t: AnyStr) -> None: + super().method1('x', b'y') # Should be an error +[out] +main:10: error: Argument 1 to "method1" of "Foo" has incompatible type "str"; expected "AnyStr" +main:10: error: Argument 2 to "method1" of "Foo" has incompatible type "bytes"; expected "AnyStr" + +[case testTypeVariableClashVar] +from typing import Generic, TypeVar, Callable + +T = TypeVar("T") +R = TypeVar("R") +class C(Generic[R]): + x: Callable[[T], R] + +def func(x: C[R]) -> R: + return x.x(42) # OK + +[case testTypeVariableClashVarTuple] +from typing import Generic, TypeVar, Callable, Tuple + +T = TypeVar("T") +R = TypeVar("R") +class C(Generic[R]): + x: Callable[[T], Tuple[R, T]] + +def func(x: C[R]) -> R: + if bool(): + return x.x(42)[0] # OK + else: + return x.x(42)[1] # E: Incompatible return value type (got "int", expected "R") +[builtins fixtures/tuple.pyi] + +[case testTypeVariableClashMethod] +from typing import Generic, TypeVar, Callable + +T = TypeVar("T") +R = TypeVar("R") +class C(Generic[R]): + def x(self) -> Callable[[T], R]: ... + +def func(x: C[R]) -> R: + return x.x()(42) # OK + +[case testTypeVariableClashMethodTuple] +from typing import Generic, TypeVar, Callable, Tuple + +T = TypeVar("T") +R = TypeVar("R") +class C(Generic[R]): + def x(self) -> Callable[[T], Tuple[R, T]]: ... + +def func(x: C[R]) -> R: + if bool(): + return x.x()(42)[0] # OK + else: + return x.x()(42)[1] # E: Incompatible return value type (got "int", expected "R") +[builtins fixtures/tuple.pyi] + +[case testTypeVariableClashVarSelf] +from typing import Self, TypeVar, Generic, Callable + +T = TypeVar("T") +S = TypeVar("S") + +class C(Generic[T]): + x: Callable[[S], Self] + y: T + +def foo(x: C[T]) -> T: + return x.x(42).y # OK + +[case testNestedGenericFunctionTypeApplication] +from typing import TypeVar, Generic, List + +A = TypeVar("A") +B = TypeVar("B") + +class C(Generic[A]): + x: A + +def foo(x: A) -> A: + def bar() -> List[A]: + y = C[List[A]]() + z = C[List[B]]() # E: Type variable "__main__.B" is unbound \ + # N: (Hint: Use "Generic[B]" or "Protocol[B]" base class to bind "B" inside a class) \ + # N: (Hint: Use "B" in function signature to bind "B" inside a function) + return y.x + return bar()[0] + + +-- TypeVar imported from typing_extensions +-- --------------------------------------- + +[case testTypeVarTypingExtensionsSimpleGeneric] +from typing import Generic +from typing_extensions import TypeVar + +T = TypeVar("T") + +class A(Generic[T]): + def __init__(self, value: T) -> None: + self.value = value + +a: A = A(8) +b: A[str] = A("") + +reveal_type(A(1.23)) # N: Revealed type is "__main__.A[builtins.float]" + +[builtins fixtures/tuple.pyi] + +[case testTypeVarTypingExtensionsSimpleBound] +from typing_extensions import TypeVar + +T= TypeVar("T") + +def func(var: T) -> T: + return var + +reveal_type(func(1)) # N: Revealed type is "builtins.int" + +[builtins fixtures/tuple.pyi] + +[case testGenericLambdaGenericMethodNoCrash] +# flags: --new-type-inference +from typing import TypeVar, Union, Callable, Generic + +S = TypeVar("S") +T = TypeVar("T") + +def f(x: Callable[[G[T]], int]) -> T: ... + +class G(Generic[T]): + def g(self, x: S) -> Union[S, T]: ... + +reveal_type(f(lambda x: x.g(0))) # N: Revealed type is "builtins.int" + +[case testDictStarInference] +class B: ... +class C1(B): ... +class C2(B): ... + +dict1 = {"a": C1()} +dict2 = {"a": C2(), **dict1} +reveal_type(dict2) # N: Revealed type is "builtins.dict[builtins.str, __main__.B]" +[builtins fixtures/dict.pyi] + +[case testDictStarAnyKeyJoinValue] +from typing import Any + +class B: ... +class C1(B): ... +class C2(B): ... + +dict1: Any +dict2 = {"a": C1(), **{x: C2() for x in dict1}} +reveal_type(dict2) # N: Revealed type is "builtins.dict[Any, __main__.B]" +[builtins fixtures/dict.pyi] + +-- Type inference for generic decorators applied to generic callables +-- ------------------------------------------------------------------ + +[case testInferenceAgainstGenericCallable] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +X = TypeVar('X') +T = TypeVar('T') + +def foo(x: Callable[[int], X]) -> List[X]: + ... +def bar(x: Callable[[X], int]) -> List[X]: + ... + +def id(x: T) -> T: + ... +reveal_type(foo(id)) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(bar(id)) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableNoLeak] +# flags: --new-type-inference +from typing import TypeVar, Callable + +T = TypeVar('T') + +def f(x: Callable[..., T]) -> T: + return x() + +def tpl(x: T) -> T: + return x + +# This is valid because of "..." +reveal_type(f(tpl)) # N: Revealed type is "Any" +[out] + +[case testInferenceAgainstGenericCallableChain] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +X = TypeVar('X') +T = TypeVar('T') + +def chain(f: Callable[[X], T], g: Callable[[T], int]) -> Callable[[X], int]: ... +def id(x: T) -> T: + ... +reveal_type(chain(id, id)) # N: Revealed type is "def (builtins.int) -> builtins.int" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableGeneric] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +def dec(f: Callable[[S], T]) -> Callable[[S], List[T]]: + ... +def id(x: U) -> U: + ... +reveal_type(dec(id)) # N: Revealed type is "def [S] (S`1) -> builtins.list[S`1]" + +@dec +def same(x: U) -> U: + ... +reveal_type(same) # N: Revealed type is "def [S] (S`3) -> builtins.list[S`3]" +reveal_type(same(42)) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableGenericReverse] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +def dec(f: Callable[[S], List[T]]) -> Callable[[S], T]: + ... +def id(x: U) -> U: + ... +reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`2]) -> T`2" + +@dec +def same(x: U) -> U: + ... +reveal_type(same) # N: Revealed type is "def [T] (builtins.list[T`4]) -> T`4" +reveal_type(same([42])) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableGenericArg] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +def dec(f: Callable[[S], T]) -> Callable[[S], T]: + ... +def test(x: U) -> List[U]: + ... +reveal_type(dec(test)) # N: Revealed type is "def [S] (S`1) -> builtins.list[S`1]" + +@dec +def single(x: U) -> List[U]: + ... +reveal_type(single) # N: Revealed type is "def [S] (S`3) -> builtins.list[S`3]" +reveal_type(single(42)) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableGenericChain] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +def comb(f: Callable[[T], S], g: Callable[[S], U]) -> Callable[[T], U]: ... +def id(x: U) -> U: + ... +reveal_type(comb(id, id)) # N: Revealed type is "def [T] (T`1) -> T`1" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableGenericNonLinear] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +def mix(fs: List[Callable[[S], T]]) -> Callable[[S], List[T]]: + def inner(x: S) -> List[T]: + return [f(x) for f in fs] + return inner + +# Errors caused by arg *name* mismatch are truly cryptic, but this is a known issue :/ +def id(__x: U) -> U: + ... +fs = [id, id, id] +reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`11) -> builtins.list[S`11]" +reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`13) -> builtins.list[S`13]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCurry] +# flags: --new-type-inference +from typing import Callable, List, TypeVar + +S = TypeVar("S") +T = TypeVar("T") +U = TypeVar("U") +V = TypeVar("V") + +def dec1(f: Callable[[T], S]) -> Callable[[], Callable[[T], S]]: ... +def dec2(f: Callable[[T, U], S]) -> Callable[[U], Callable[[T], S]]: ... + +def test1(x: V) -> V: ... +def test2(x: V, y: V) -> V: ... + +reveal_type(dec1(test1)) # N: Revealed type is "def () -> def [T] (T`1) -> T`1" +reveal_type(dec2(test2)) # N: Revealed type is "def [T] (T`3) -> def (T`3) -> T`3" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableNewVariable] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +def dec(f: Callable[[S], T]) -> Callable[[S], T]: + ... +def test(x: List[U]) -> List[U]: + ... +reveal_type(dec(test)) # N: Revealed type is "def [U] (builtins.list[U`-1]) -> builtins.list[U`-1]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableGenericAlias] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +A = Callable[[S], T] +B = Callable[[S], List[T]] + +def dec(f: A[S, T]) -> B[S, T]: + ... +def id(x: U) -> U: + ... +reveal_type(dec(id)) # N: Revealed type is "def [S] (S`1) -> builtins.list[S`1]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableGenericProtocol] +# flags: --new-type-inference +from typing import TypeVar, Protocol, Generic, Optional + +T = TypeVar('T') + +class F(Protocol[T]): + def __call__(self, __x: T) -> T: ... + +def lift(f: F[T]) -> F[Optional[T]]: ... +def g(x: T) -> T: + return x + +reveal_type(lift(g)) # N: Revealed type is "def [T] (Union[T`1, None]) -> Union[T`1, None]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericSplitOrder] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') + +def dec(f: Callable[[T], S], g: Callable[[T], int]) -> Callable[[T], List[S]]: ... +def id(x: U) -> U: + ... + +reveal_type(dec(id, id)) # N: Revealed type is "def (builtins.int) -> builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericSplitOrderGeneric] +# flags: --new-type-inference +from typing import TypeVar, Callable, Tuple + +S = TypeVar('S') +T = TypeVar('T') +U = TypeVar('U') +V = TypeVar('V') + +def dec(f: Callable[[T], S], g: Callable[[T], U]) -> Callable[[T], Tuple[S, U]]: ... +def id(x: V) -> V: + ... + +reveal_type(dec(id, id)) # N: Revealed type is "def [T] (T`1) -> Tuple[T`1, T`1]" +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericEllipsisSelfSpecialCase] +# flags: --new-type-inference +from typing import Self, Callable, TypeVar + +T = TypeVar("T") +def dec(f: Callable[..., T]) -> Callable[..., T]: ... + +class C: + @dec + def test(self) -> Self: ... + +c: C +reveal_type(c.test()) # N: Revealed type is "__main__.C" + +[case testInferenceAgainstGenericBoundsAndValues] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +class B: ... +class C(B): ... + +S = TypeVar('S') +T = TypeVar('T') +UB = TypeVar('UB', bound=B) +UC = TypeVar('UC', bound=C) +V = TypeVar('V', int, str) + +def dec1(f: Callable[[S], T]) -> Callable[[S], List[T]]: + ... +def dec2(f: Callable[[UC], T]) -> Callable[[UC], List[T]]: + ... +def id1(x: UB) -> UB: + ... +def id2(x: V) -> V: + ... + +reveal_type(dec1(id1)) # N: Revealed type is "def [S <: __main__.B] (S`1) -> builtins.list[S`1]" +reveal_type(dec1(id2)) # N: Revealed type is "def [S in (builtins.int, builtins.str)] (S`3) -> builtins.list[S`3]" +reveal_type(dec2(id1)) # N: Revealed type is "def [UC <: __main__.C] (UC`5) -> builtins.list[UC`5]" +reveal_type(dec2(id2)) # N: Revealed type is "def (Never) -> builtins.list[Never]" \ + # E: Argument 1 to "dec2" has incompatible type "Callable[[V], V]"; expected "Callable[[Never], Never]" + +[case testInferenceAgainstGenericLambdas] +# flags: --new-type-inference +from typing import TypeVar, Callable, List + +S = TypeVar('S') +T = TypeVar('T') + +def dec1(f: Callable[[T], T]) -> Callable[[T], List[T]]: + ... +def dec2(f: Callable[[S], T]) -> Callable[[S], List[T]]: + ... +def dec3(f: Callable[[List[S]], T]) -> Callable[[S], T]: + def g(x: S) -> T: + return f([x]) + return g +def dec4(f: Callable[[S], List[T]]) -> Callable[[S], T]: + ... +def dec5(f: Callable[[int], T]) -> Callable[[int], List[T]]: + def g(x: int) -> List[T]: + return [f(x)] * x + return g + +I = TypeVar("I", bound=int) +def dec4_bound(f: Callable[[I], List[T]]) -> Callable[[I], T]: + ... + +reveal_type(dec1(lambda x: x)) # N: Revealed type is "def [T] (T`3) -> builtins.list[T`3]" +reveal_type(dec2(lambda x: x)) # N: Revealed type is "def [S] (S`5) -> builtins.list[S`5]" +reveal_type(dec3(lambda x: x[0])) # N: Revealed type is "def [S] (S`8) -> S`8" +reveal_type(dec4(lambda x: [x])) # N: Revealed type is "def [S] (S`12) -> S`12" +reveal_type(dec1(lambda x: 1)) # N: Revealed type is "def (builtins.int) -> builtins.list[builtins.int]" +reveal_type(dec5(lambda x: x)) # N: Revealed type is "def (builtins.int) -> builtins.list[builtins.int]" +reveal_type(dec3(lambda x: x)) # N: Revealed type is "def [S] (S`20) -> builtins.list[S`20]" +reveal_type(dec4(lambda x: x)) # N: Revealed type is "def [T] (builtins.list[T`24]) -> T`24" +dec4_bound(lambda x: x) # E: Value of type variable "I" of "dec4_bound" cannot be "List[T]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecBasicInList] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import ParamSpec + +T = TypeVar('T') +P = ParamSpec('P') +U = TypeVar('U') +V = TypeVar('V') + +def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... +def pair(x: U, y: V) -> Tuple[U, V]: ... +reveal_type(dec(id)) # N: Revealed type is "def [T] (x: T`3) -> builtins.list[T`3]" +reveal_type(dec(either)) # N: Revealed type is "def [T] (x: T`5, y: T`5) -> builtins.list[T`5]" +reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (x: U`-1, y: V`-2) -> builtins.list[Tuple[U`-1, V`-2]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecBasicDeList] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import ParamSpec + +T = TypeVar('T') +P = ParamSpec('P') +U = TypeVar('U') +V = TypeVar('V') + +def dec(f: Callable[P, List[T]]) -> Callable[P, T]: ... +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... +reveal_type(dec(id)) # N: Revealed type is "def [T] (x: builtins.list[T`3]) -> T`3" +reveal_type(dec(either)) # N: Revealed type is "def [T] (x: builtins.list[T`5], y: builtins.list[T`5]) -> T`5" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecPopOff] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import ParamSpec, Concatenate + +T = TypeVar('T') +S = TypeVar('S') +P = ParamSpec('P') +U = TypeVar('U') +V = TypeVar('V') + +def dec(f: Callable[Concatenate[T, P], S]) -> Callable[P, Callable[[T], S]]: ... +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... +def pair(x: U, y: V) -> Tuple[U, V]: ... +reveal_type(dec(id)) # N: Revealed type is "def () -> def [T] (T`2) -> T`2" +reveal_type(dec(either)) # N: Revealed type is "def [T] (y: T`5) -> def (T`5) -> T`5" +reveal_type(dec(pair)) # N: Revealed type is "def [V] (y: V`-2) -> def [T] (T`8) -> Tuple[T`8, V`-2]" +reveal_type(dec(dec)) # N: Revealed type is "def () -> def [T, P, S] (def (T`-1, *P.args, **P.kwargs) -> S`-3) -> def (*P.args, **P.kwargs) -> def (T`-1) -> S`-3" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecPopOn] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import ParamSpec, Concatenate + +T = TypeVar('T') +S = TypeVar('S') +P = ParamSpec('P') +U = TypeVar('U') +V = TypeVar('V') + +def dec(f: Callable[P, Callable[[T], S]]) -> Callable[Concatenate[T, P], S]: ... +def id() -> Callable[[U], U]: ... +def either(x: U) -> Callable[[U], U]: ... +def pair(x: U) -> Callable[[V], Tuple[V, U]]: ... +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> T`3" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`6, x: T`6) -> T`6" +reveal_type(dec(pair)) # N: Revealed type is "def [T, U] (T`9, x: U`-1) -> Tuple[T`9, U`-1]" +# This is counter-intuitive but looks correct, dec matches itself only if P can be empty +reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`13, f: def () -> def (T`13) -> S`14) -> S`14" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecVsParamSpec] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple, Generic +from typing_extensions import ParamSpec, Concatenate + +T = TypeVar('T') +P = ParamSpec('P') +Q = ParamSpec('Q') + +class Foo(Generic[P]): ... +class Bar(Generic[P, T]): ... + +def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... +def f(*args: Q.args, **kwargs: Q.kwargs) -> Foo[Q]: ... +reveal_type(dec(f)) # N: Revealed type is "def [P] (*P.args, **P.kwargs) -> builtins.list[__main__.Foo[P`2]]" +g: Callable[Concatenate[int, Q], Foo[Q]] +reveal_type(dec(g)) # N: Revealed type is "def [Q] (builtins.int, *Q.args, **Q.kwargs) -> builtins.list[__main__.Foo[Q`-1]]" +h: Callable[Concatenate[T, Q], Bar[Q, T]] +reveal_type(dec(h)) # N: Revealed type is "def [T, Q] (T`-1, *Q.args, **Q.kwargs) -> builtins.list[__main__.Bar[Q`-2, T`-1]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecVsParamSpecConcatenate] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple, Generic +from typing_extensions import ParamSpec, Concatenate + +T = TypeVar('T') +P = ParamSpec('P') +Q = ParamSpec('Q') + +class Foo(Generic[P]): ... + +def dec(f: Callable[P, int]) -> Callable[P, Foo[P]]: ... +h: Callable[Concatenate[T, Q], int] +g: Callable[Concatenate[T, Q], int] +h = g +reveal_type(dec(h)) # N: Revealed type is "def [T, Q] (T`-1, *Q.args, **Q.kwargs) -> __main__.Foo[[T`-1, **Q`-2]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecSecondary] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple, Generic +from typing_extensions import ParamSpec, Concatenate + +T = TypeVar('T') +P = ParamSpec('P') +Q = ParamSpec('Q') + +class Foo(Generic[P]): ... + +def dec(f: Callable[P, Foo[P]]) -> Callable[P, Foo[P]]: ... +g: Callable[[T], Foo[[int]]] +reveal_type(dec(g)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[[builtins.int]]" +h: Callable[Q, Foo[[int]]] +reveal_type(dec(g)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[[builtins.int]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericParamSpecSecondOrder] +# flags: --new-type-inference +from typing import TypeVar, Callable +from typing_extensions import ParamSpec, Concatenate + +T = TypeVar('T') +S = TypeVar('S') +P = ParamSpec('P') +Q = ParamSpec('Q') +U = TypeVar('U') +W = ParamSpec('W') + +def transform( + dec: Callable[[Callable[P, T]], Callable[Q, S]] +) -> Callable[[Callable[Concatenate[int, P], T]], Callable[Concatenate[int, Q], S]]: ... + +def dec(f: Callable[W, U]) -> Callable[W, U]: ... +def dec2(f: Callable[Concatenate[str, W], U]) -> Callable[Concatenate[bytes, W], U]: ... +reveal_type(transform(dec)) # N: Revealed type is "def [P, T] (def (builtins.int, *P.args, **P.kwargs) -> T`3) -> def (builtins.int, *P.args, **P.kwargs) -> T`3" +reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.int, builtins.str, *W.args, **W.kwargs) -> T`7) -> def (builtins.int, builtins.bytes, *W.args, **W.kwargs) -> T`7" +[builtins fixtures/tuple.pyi] + +[case testNoAccidentalVariableClashInNestedGeneric] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic, Tuple + +T = TypeVar('T') +S = TypeVar('S') +U = TypeVar('U') + +def pipe(x: T, f1: Callable[[T], S], f2: Callable[[S], U]) -> U: ... +def and_then(a: T) -> Callable[[S], Tuple[S, T]]: ... + +def apply(a: S, b: T) -> None: + v1 = and_then(b) + v2: Callable[[Tuple[S, T]], None] + return pipe(a, v1, v2) +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericParamSpecSpuriousBoundsNotUsed] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic +from typing_extensions import ParamSpec, Concatenate + +Q = ParamSpec("Q") +class Foo(Generic[Q]): ... + +T1 = TypeVar("T1", bound=Foo[...]) +T2 = TypeVar("T2", bound=Foo[...]) +P = ParamSpec("P") +def pop_off(fn: Callable[Concatenate[T1, P], T2]) -> Callable[P, Callable[[T1], T2]]: + ... + +@pop_off +def test(command: Foo[Q]) -> Foo[Q]: ... +reveal_type(test) # N: Revealed type is "def () -> def [Q] (__main__.Foo[Q`-1]) -> __main__.Foo[Q`-1]" +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicBasicInList] +# flags: --new-type-inference +from typing import Tuple, TypeVar, List, Callable +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], List[T]]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... +def pair(x: U, y: V) -> Tuple[U, V]: ... + +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> builtins.list[T`3]" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`5, T`5) -> builtins.list[T`5]" +reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (U`-1, V`-2) -> builtins.list[Tuple[U`-1, V`-2]]" +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicBasicDeList] +# flags: --new-type-inference +from typing import Tuple, TypeVar, List, Callable +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[Unpack[Ts]], List[T]]) -> Callable[[Unpack[Ts]], T]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... + +reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`3]) -> T`3" +reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`5], builtins.list[T`5]) -> T`5" +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicPopOff] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[T, Unpack[Ts]], S]) -> Callable[[Unpack[Ts]], Callable[[T], S]]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... +def pair(x: U, y: V) -> Tuple[U, V]: ... + +reveal_type(dec(id)) # N: Revealed type is "def () -> def [T] (T`2) -> T`2" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`5) -> def (T`5) -> T`5" +reveal_type(dec(pair)) # N: Revealed type is "def [V] (V`-2) -> def [T] (T`8) -> Tuple[T`8, V`-2]" +reveal_type(dec(dec)) # N: Revealed type is "def () -> def [T, Ts, S] (def (T`-1, *Unpack[Ts`-2]) -> S`-3) -> def (*Unpack[Ts`-2]) -> def (T`-1) -> S`-3" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicPopOn] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[Unpack[Ts]], Callable[[T], S]]) -> Callable[[T, Unpack[Ts]], S]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id() -> Callable[[U], U]: ... +def either(x: U) -> Callable[[U], U]: ... +def pair(x: U) -> Callable[[V], Tuple[V, U]]: ... + +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> T`3" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`6, T`6) -> T`6" +reveal_type(dec(pair)) # N: Revealed type is "def [T, U] (T`9, U`-1) -> Tuple[T`9, U`-1]" +# This is counter-intuitive but looks correct, dec matches itself only if Ts is empty +reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`13, def () -> def (T`13) -> S`14) -> S`14" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicVsVariadic] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... +class Bar(Generic[Unpack[Ts], T]): ... + +def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], List[T]]: ... +def f(*args: Unpack[Us]) -> Foo[Unpack[Us]]: ... +reveal_type(dec(f)) # N: Revealed type is "def [Ts] (*Unpack[Ts`2]) -> builtins.list[__main__.Foo[Unpack[Ts`2]]]" +g: Callable[[Unpack[Us]], Foo[Unpack[Us]]] +reveal_type(dec(g)) # N: Revealed type is "def [Ts] (*Unpack[Ts`4]) -> builtins.list[__main__.Foo[Unpack[Ts`4]]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicVsVariadicConcatenate] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... + +def dec(f: Callable[[Unpack[Ts]], int]) -> Callable[[Unpack[Ts]], Foo[Unpack[Ts]]]: ... +h: Callable[[T, Unpack[Us]], int] +g: Callable[[T, Unpack[Us]], int] +h = g +reveal_type(dec(h)) # N: Revealed type is "def [T, Us] (T`-1, *Unpack[Us`-2]) -> __main__.Foo[T`-1, Unpack[Us`-2]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicSecondary] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... + +def dec(f: Callable[[Unpack[Ts]], Foo[Unpack[Ts]]]) -> Callable[[Unpack[Ts]], Foo[Unpack[Ts]]]: ... +g: Callable[[T], Foo[int]] +reveal_type(dec(g)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[builtins.int]" +h: Callable[[Unpack[Us]], Foo[int]] +reveal_type(dec(h)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[builtins.int]" +[builtins fixtures/list.pyi] + +[case testTypeApplicationGenericConstructor] +from typing import Generic, TypeVar, Callable + +T = TypeVar("T") +S = TypeVar("S") +class C(Generic[T]): + def __init__(self, f: Callable[[S], T], x: S) -> None: + self.x = f(x) + +reveal_type(C[int]) # N: Revealed type is "def [S] (f: def (S`-1) -> builtins.int, x: S`-1) -> __main__.C[builtins.int]" +Alias = C[int] +C[int, str] # E: Type application has too many types (1 expected) + +[case testHigherOrderGenericPartial] +from typing import TypeVar, Callable + +T = TypeVar("T") +S = TypeVar("S") +U = TypeVar("U") +def apply(f: Callable[[T], S], x: T) -> S: ... +def id(x: U) -> U: ... + +A1 = TypeVar("A1") +A2 = TypeVar("A2") +R = TypeVar("R") +def fake_partial(fun: Callable[[A1, A2], R], arg: A1) -> Callable[[A2], R]: ... + +f_pid = fake_partial(apply, id) +reveal_type(f_pid) # N: Revealed type is "def [A2] (A2`2) -> A2`2" +reveal_type(f_pid(1)) # N: Revealed type is "builtins.int" + +[case testInvalidTypeVarParametersConcrete] +from typing import Callable, Generic, ParamSpec, Protocol, TypeVar, overload + +P = ParamSpec('P') +P2 = ParamSpec('P2') +R = TypeVar('R') +R2 = TypeVar('R2') + +class C(Generic[P, R, P2, R2]): ... + +class Proto(Protocol[P, R]): + @overload + def __call__(self, f: Callable[P2, R2]) -> C[P2, R2, ..., R]: ... + @overload + def __call__(self, **kwargs) -> C[P, R, ..., [int, str]]: ... # E: Cannot use "[int, str]" for regular type variable, only for ParamSpec +[builtins fixtures/tuple.pyi] + +[case testInvalidTypeVarParametersArbitrary] +from typing import Callable, Generic, ParamSpec, Protocol, TypeVar, overload + +P = ParamSpec('P') +P2 = ParamSpec('P2') +R = TypeVar('R') +R2 = TypeVar('R2') + +class C(Generic[P, R, P2, R2]): ... + +class Proto(Protocol[P, R]): + @overload + def __call__(self, f: Callable[P2, R2]) -> C[P2, R2, ..., R]: ... + @overload + def __call__(self, **kwargs) -> C[P, R, ..., ...]: ... # E: Cannot use "[VarArg(Any), KwArg(Any)]" for regular type variable, only for ParamSpec +[builtins fixtures/tuple.pyi] + +[case testGenericOverloadOverlapUnion] +from typing import TypeVar, overload, Union, Generic + +K = TypeVar("K") +V = TypeVar("V") +T = TypeVar("T") + +class C(Generic[K, V]): + @overload + def pop(self, key: K) -> V: ... + @overload + def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]: ... + def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]: + ... + +[case testOverloadedGenericInit] +from typing import TypeVar, overload, Union, Generic + +T = TypeVar("T") +S = TypeVar("S") + +class Int(Generic[T]): ... +class Str(Generic[T]): ... + +class C(Generic[T]): + @overload + def __init__(self: C[Int[S]], x: int, y: S) -> None: ... + @overload + def __init__(self: C[Str[S]], x: str, y: S) -> None: ... + def __init__(self, x, y) -> None: ... + +def foo(x: T): + reveal_type(C) # N: Revealed type is "Overload(def [T, S] (x: builtins.int, y: S`-1) -> __main__.C[__main__.Int[S`-1]], def [T, S] (x: builtins.str, y: S`-1) -> __main__.C[__main__.Str[S`-1]])" + reveal_type(C(0, x)) # N: Revealed type is "__main__.C[__main__.Int[T`-1]]" + reveal_type(C("yes", x)) # N: Revealed type is "__main__.C[__main__.Str[T`-1]]" diff --git a/test-data/unit/check-ignore.test b/test-data/unit/check-ignore.test new file mode 100644 index 000000000000..fa451f373e70 --- /dev/null +++ b/test-data/unit/check-ignore.test @@ -0,0 +1,292 @@ +[case testIgnoreTypeError] +x = 1 +x() # type: ignore +x() # E: "int" not callable + +[case testIgnoreUndefinedName] +x = 1 +y # type: ignore +z # E: Name "z" is not defined + +[case testIgnoreImportError] +import xyz_m # type: ignore +xyz_m.foo +1() # E: "int" not callable + +[case testIgnoreImportFromError] +from xyz_m import a, b # type: ignore +a.foo +b() +1() # E: "int" not callable + +[case testIgnoreImportFromErrorMultiline] +from xyz_m import ( # type: ignore + a, b +) +a.foo +b() +1() # E: "int" not callable + +[case testIgnoreImportAllError] +from xyz_m import * # type: ignore +x # E: Name "x" is not defined +1() # E: "int" not callable + +[case testIgnoreImportBadModule] +import m # type: ignore +from m import a # type: ignore +[file m.py] ++ +[out] +tmp/m.py:1: error: invalid syntax + +[case testIgnoreAppliesOnlyToMissing] +import a # type: ignore +import b # type: ignore +reveal_type(a.foo) # N: Revealed type is "Any" +reveal_type(b.foo) # N: Revealed type is "builtins.int" +a.bar() +b.bar() # E: Module has no attribute "bar" + +[file b.py] +foo = 3 + +[builtins fixtures/module_all.pyi] +[out] + +[case testIgnoreImportStarFromBadModule] +from m import * # type: ignore +[file m.py] ++ +[out] +tmp/m.py:1: error: invalid syntax + +[case testIgnoreAssignmentTypeError] +x = 1 +if int(): + x = '' # type: ignore +if int(): + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testIgnoreInvalidOverride] +class A: + def f(self) -> int: pass +class B(A): + def f(self) -> str: pass # type: ignore + +[case testIgnoreMissingModuleAttribute] +import m +m.x = object # type: ignore +m.f() # type: ignore +m.y # E: Module has no attribute "y" +[file m.py] +[builtins fixtures/module.pyi] + +[case testIgnoreTypeInferenceError] +x = [] # type: ignore +y = x +x.append(1) +[builtins fixtures/list.pyi] + +[case testIgnoreTypeInferenceError2] +def f() -> None: pass +x = f() # type: ignore +y = x +x = 1 +[builtins fixtures/list.pyi] + +[case testIgnoreTypeInferenceErrorAndMultipleAssignment] +x, y = [], [] # type: ignore +z = x +z = y +[builtins fixtures/list.pyi] + +[case testIgnoreSomeStarImportErrors] +from m1 import * +from m2 import * # type: ignore +# We should still import things that don't conflict. +y() # E: "str" not callable +z() # E: "int" not callable +x() # E: "int" not callable +[file m1.py] +x = 1 +y = '' +[file m2.py] +x = '' +z = 1 + +[case testIgnoredModuleDefinesBaseClass1] +from m import B # type: ignore + +class C(B): + def f(self) -> None: + self.f(1) # E: Too many arguments for "f" of "C" + self.g(1) +[out] + +[case testIgnoredModuleDefinesBaseClass2] +import m # type: ignore + +class C(m.B): + def f(self) -> None: ... + +c = C() +c.f(1) # E: Too many arguments for "f" of "C" +c.g(1) +c.x = 1 +[out] + +[case testIgnoredModuleDefinesBaseClassAndClassAttribute] +import m # type: ignore + +class C(m.B): + @staticmethod + def f() -> None: pass + +C.f(1) # E: Too many arguments for "f" of "C" +C.g(1) +C.x = 1 +[builtins fixtures/staticmethod.pyi] +[out] + +[case testIgnoredModuleDefinesBaseClassWithInheritance1] +from m import B # type: ignore + +class C: pass +class D(C, B): + def f(self) -> None: + self.f(1) # E: Too many arguments for "f" of "D" + self.g(1) +[out] + +[case testIgnoredModuleDefinesBaseClassWithInheritance2] +from m import B # type: ignore + +class C(B): pass +class D(C): + def f(self) -> None: + self.f(1) # E: Too many arguments for "f" of "D" + self.g(1) +[out] + +[case testIgnoreWithFollowingIndentedComment] +if 1: # type: ignore + # blah + pass +[out] + +[case testIgnoreTooManyTypeArguments] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') + +class Base(Generic[T, U]): + pass + +class PartialBase(Base[T, int], Generic[T]): + pass + +class Child(PartialBase[str, int]): # type: ignore + pass + + +def foo(x: Base[str, int]) -> None: pass +foo(Child()) + +def bar(x: Base[str, str]) -> None: pass +bar(Child()) +[out] +main:19: error: Argument 1 to "bar" has incompatible type "Child"; expected "Base[str, str]" + +[case testTypeIgnoreLineNumberWithinFile] +import m +pass # type: ignore +m.f(kw=1) +[file m.py] +pass +def f() -> None: pass +[out] +main:3: error: Unexpected keyword argument "kw" for "f" +tmp/m.py:2: note: "f" defined here + +[case testIgnoreUnexpectedKeywordArgument] +import m +m.f(kw=1) # type: ignore +[file m.py] +def f() -> None: pass +[out] + +[case testCannotIgnoreBlockingError] +yield # type: ignore # E: "yield" outside function + +[case testIgnoreWholeModule1] +# type: ignore +if True: + IGNORE + +[case testIgnoreWholeModule2] +# type: ignore +@d +class C: ... +IGNORE + +[case testIgnoreWholeModule3] +# type: ignore +@d + +def f(): ... +IGNORE + +[case testIgnoreWholeModule4] +# type: ignore +import MISSING + +[case testDontIgnoreWholeModule1] +if True: + # type: ignore + ERROR # E: Name "ERROR" is not defined +ERROR # E: Name "ERROR" is not defined + +[case testDontIgnoreWholeModule2] +@d # type: ignore +class C: ... +ERROR # E: Name "ERROR" is not defined + +[case testDontIgnoreWholeModule3] +@d # type: ignore + +def f(): ... +ERROR # E: Name "ERROR" is not defined + +[case testIgnoreInsideFunctionDoesntAffectWhole] +# flags: --disallow-untyped-defs + +def f(): # E: Function is missing a return type annotation + 42 + 'no way' # type: ignore + return 0 + +[case testIgnoreInsideClassDoesntAffectWhole] +import six +class M(type): pass + +@six.add_metaclass(M) +class CD(six.with_metaclass(M)): # E: Multiple metaclass definitions + 42 + 'no way' # type: ignore + +[builtins fixtures/tuple.pyi] + +[case testUnusedIgnoreTryExcept] +# flags: --warn-unused-ignores +try: + import foo # type: ignore # E: Unused "type: ignore" comment + import bar # type: ignore[import] # E: Unused "type: ignore" comment + import foobar # type: ignore[unused-ignore] + import barfoo # type: ignore[import,unused-ignore] + import missing # type: ignore[import,unused-ignore] +except Exception: + pass +[file foo.py] +[file bar.py] +[file foobar.py] +[file barfoo.py] +[builtins fixtures/exception.pyi] diff --git a/test-data/unit/check-incomplete-fixture.test b/test-data/unit/check-incomplete-fixture.test new file mode 100644 index 000000000000..146494df1bd6 --- /dev/null +++ b/test-data/unit/check-incomplete-fixture.test @@ -0,0 +1,80 @@ +-- Test cases for reporting errors when a test case uses a fixture with +-- missing definitions. At least in the most common cases this should not +-- result in an uncaught exception. These tests make sure that this behavior +-- does not regress. +-- +-- NOTE: These tests do NOT test behavior of mypy outside tests. + +[case testVariableUndefinedUsingDefaultFixture] +import m +# This used to cause a crash since types.ModuleType is not available +# by default. We fall back to 'object' now. +m.x # E: "object" has no attribute "x" +[file m.py] + +[case testSetMissingFromStubs] +from typing import Set +def f(x: Set[int]) -> None: pass +[out] +main:1: error: Module "typing" has no attribute "Set" +main:1: note: Maybe your test fixture does not define "builtins.set"? +main:1: note: Consider adding [builtins fixtures/set.pyi] to your test description + +[case testBaseExceptionMissingFromStubs] +e: BaseException +[out] +main:1: error: Name "BaseException" is not defined +main:1: note: Maybe your test fixture does not define "builtins.BaseException"? +main:1: note: Consider adding [builtins fixtures/exception.pyi] to your test description + +[case testExceptionMissingFromStubs] +e: Exception +[out] +main:1: error: Name "Exception" is not defined +main:1: note: Maybe your test fixture does not define "builtins.Exception"? +main:1: note: Consider adding [builtins fixtures/exception.pyi] to your test description + +[case testIsinstanceMissingFromStubs] +if isinstance(1, int): + pass +[out] +main:1: error: Name "isinstance" is not defined +main:1: note: Maybe your test fixture does not define "builtins.isinstance"? +main:1: note: Consider adding [builtins fixtures/isinstancelist.pyi] to your test description + +[case testTupleMissingFromStubs1] +tuple() +[out] +main:1: error: Name "tuple" is not defined +main:1: note: Maybe your test fixture does not define "builtins.tuple"? +main:1: note: Consider adding [builtins fixtures/tuple.pyi] to your test description +main:1: note: Did you forget to import it from "typing"? (Suggestion: "from typing import Tuple") + +[case testTupleMissingFromStubs2] +tuple() +from typing import Tuple +x: Tuple[int, str] +[out] +main:1: error: Name "tuple" is not defined +main:1: note: Maybe your test fixture does not define "builtins.tuple"? +main:1: note: Consider adding [builtins fixtures/tuple.pyi] to your test description +main:1: note: Did you forget to import it from "typing"? (Suggestion: "from typing import Tuple") +main:3: error: Name "tuple" is not defined + +[case testClassmethodMissingFromStubs] +class A: + @classmethod + def f(cls): pass +[out] +main:2: error: Name "classmethod" is not defined +main:2: note: Maybe your test fixture does not define "builtins.classmethod"? +main:2: note: Consider adding [builtins fixtures/classmethod.pyi] to your test description + +[case testPropertyMissingFromStubs] +class A: + @property + def f(self): pass +[out] +main:2: error: Name "property" is not defined +main:2: note: Maybe your test fixture does not define "builtins.property"? +main:2: note: Consider adding [builtins fixtures/property.pyi] to your test description diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test new file mode 100644 index 000000000000..888b4c26a7c7 --- /dev/null +++ b/test-data/unit/check-incremental.test @@ -0,0 +1,6746 @@ +-- Checks for incremental mode (see testcheck.py). +-- Each test is run at least twice, once with a cold cache, once with a warm cache. +-- Before the tests are run again, in step N any *.py.N files are copied to +-- *.py. There are at least two runs; more as long as there are *.py.N files. +-- +-- You can add an empty section like `[delete mod.py.2]` to delete `mod.py` +-- before the second run. +-- +-- Errors expected in the first run should be in the `[out1]` section, and +-- errors expected in the second run should be in the `[out2]` section, and so on. +-- If a section is omitted, it is expected there are no errors on that run. +-- The number of runs is determined by the highest N in all [outN] sections, but +-- there are always at least two runs. (Note that [out] is equivalent to [out1].) +-- +-- The list of modules to be checked can be specified using +-- # cmd: mypy -m mod1 mod2 mod3 +-- To check a different list on the second run, use +-- # cmd2: mypy -m mod1 mod3 +-- (and cmd3 for the third run, and so on). +-- +-- Extra command line flags may be specified using +-- # flags: --some-flag +-- If the second run requires different flags, those can be specified using +-- # flags2: --another-flag +-- (and flags3 for the third run, and so on). +-- +-- Incremental tests involving plugins that get updated are also supported. +-- All plugin files that are updated *must* end in '_plugin', so they will +-- be unloaded from 'sys.modules' between incremental steps. +-- +-- Any files that we expect to be rechecked should be annotated in the [rechecked] +-- annotation, and any files expect to be stale (aka have a modified interface) +-- should be annotated in the [stale] annotation. Note that a file that ends up +-- producing an error has its caches deleted and is marked stale automatically. +-- Such files do not need to be included in [stale ...] list. +-- +-- The test suite will automatically assume that __main__ is stale and rechecked in +-- all cases so we can avoid constantly having to annotate it. The list of +-- rechecked/stale files can be in any arbitrary order, or can be left empty +-- if no files should be rechecked/stale. +-- +-- There are additional incremental mode test cases in check-serialize.test. + +[case testIncrementalEmpty] +[rechecked] +[stale] + +[case testIncrementalBasics] +import m +[file m.py] +def foo(): + pass +[file m.py.2] +def foo() -> None: + pass +[rechecked m] +[stale m] + +[case testIncrementalError] +import m +[file m.py] +def foo() -> None: + pass +[file m.py.2] +def foo() -> None: + bar() +[rechecked m] +[stale] +[out2] +tmp/m.py:2: error: Name "bar" is not defined + +[case testIncrementalSimpleImportSequence] +import mod1 +mod1.func1() + +[file mod1.py] +import mod2 +def func1() -> None: mod2.func2() + +[file mod2.py] +import mod3 +def func2() -> None: mod3.func3() + +[file mod3.py] +def func3() -> None: pass + +[rechecked] +[stale] + + +[case testIncrementalInternalChangeOnly] +import mod1 +mod1.func1() + +[file mod1.py] +import mod2 +def func1() -> None: mod2.func2() + +[file mod2.py] +import mod3 +def func2() -> None: mod3.func3() + +[file mod3.py] +def func3() -> None: pass + +[file mod3.py.2] +def func3() -> None: 3 + 2 + +[rechecked mod3] +[stale] + + +[case testIncrementalImportGone] +import mod1 + +[file mod1.py] +from mod2 import A +def func1() -> A: pass + +[file mod2.py] +class A: pass + +[file mod1.py.2] +def func1() -> A: pass + +[rechecked mod1] +[stale] +[out2] +tmp/mod1.py:1: error: Name "A" is not defined + +[case testIncrementalCallable] +import mod1 + +[file mod1.py] +from typing import Callable +from mypy_extensions import Arg +def func1() -> Callable[[Arg(int, 'x')], int]: pass + +[file mod1.py.2] +from typing import Callable +from mypy_extensions import Arg +def func1() -> Callable[[Arg(int, 'x')], int]: ... + + +[rechecked mod1] +[stale] + +[builtins fixtures/dict.pyi] + +[case testIncrementalSameNameChange] +import mod1 + +[file mod1.py] +from mod2 import A +def func1() -> A: pass + +[file mod2.py] +class A: pass + +[file mod2.py.2] +class Parent: pass +class A(Parent): pass + +[rechecked mod1, mod2] +[stale mod2] + +[case testIncrementalPartialInterfaceChange] +import mod1 +mod1.func1() + +[file mod1.py] +import mod2 +def func1() -> None: mod2.func2() + +[file mod2.py] +import mod3 +def func2() -> None: mod3.func3() + +[file mod3.py] +def func3() -> None: pass + +[file mod3.py.2] +def func3() -> int: return 2 + +[rechecked mod2, mod3] +[stale mod3] + +[case testIncrementalInternalFunctionDefinitionChange] +import mod1 + +[file mod1.py] +import mod2 +def accepts_int(a: int) -> int: return a +accepts_int(mod2.foo()) + +[file mod2.py] +def foo() -> int: + def inner() -> int: + return 42 + return inner() + +[file mod2.py.2] +def foo() -> int: + def inner2() -> str: + return "foo" + return inner2() + +[rechecked mod1, mod2] +[stale] +[out2] +tmp/mod2.py:4: error: Incompatible return value type (got "str", expected "int") + +[case testIncrementalInternalScramble] +import mod1 + +[file mod1.py] +import mod2 +mod2.foo() + +[file mod2.py] +def baz() -> int: + return 3 + +def bar() -> int: + return baz() + +def foo() -> int: + return bar() + +[file mod2.py.2] +def foo() -> int: + return baz() + +def bar() -> int: + return bar() + +def baz() -> int: + return 42 +[rechecked mod2] +[stale] + +[case testIncrementalMethodInterfaceChange] +import mod1 + +[file mod1.py] +import mod2 + +[file mod2.py] +class Foo: + def bar(self, a: str) -> str: + return "a" + +[file mod2.py.2] +class Foo: + def bar(self, a: float) -> str: + return "a" + +[rechecked mod1, mod2] +[stale mod2] + +[case testIncrementalBaseClassChange] +import mod1 + +[file mod1.py] +from mod2 import Child +Child().good_method() + +[file mod2.py] +class Good: + def good_method(self) -> int: return 1 +class Bad: pass +class Child(Good): pass + +[file mod2.py.2] +class Good: + def good_method(self) -> int: return 1 +class Bad: pass +class Child(Bad): pass + +[rechecked mod1, mod2] +[stale mod2] +[out2] +tmp/mod1.py:2: error: "Child" has no attribute "good_method" + +[case testIncrementalCascadingChange] +import mod1 + +[file mod1.py] +from mod2 import A +def accepts_int(a: int) -> None: pass +accepts_int(A) + +[file mod2.py] +from mod3 import B +A = B + +[file mod3.py] +from mod4 import C +B = C + +[file mod4.py] +C = 3 + +[file mod4.py.2] +C = "A" + +[rechecked mod1, mod2, mod3, mod4] +[stale mod2, mod3, mod4] +[out2] +tmp/mod1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" + +[case testIncrementalBrokenCascade] +import mod1 + +[file mod1.py] +import mod2 +def accept_int(a: int) -> int: return a +accept_int(mod2.mod3.mod4.const) + +[file mod2.py] +import mod3 + +[file mod3.py] +import mod4 + +[file mod4.py] +const = 3 + +[file mod3.py.2] +# Import to mod4 is gone! + +[rechecked mod1, mod2, mod3] +[stale mod3] +[builtins fixtures/module.pyi] +[out2] +tmp/mod1.py:3: error: Module has no attribute "mod4" + +[case testIncrementalLongBrokenCascade] +import mod1 + +[file mod1.py] +import mod2 +def accept_int(a: int) -> int: return a +accept_int(mod2.mod3.mod4.mod5.mod6.mod7.const) + +[file mod2.py] +import mod3 + +[file mod3.py] +import mod4 + +[file mod4.py] +import mod5 + +[file mod5.py] +import mod6 + +[file mod6.py] +import mod7 + +[file mod7.py] +const = 3 + +[file mod6.py.2] +# Import to mod7 is gone! + +[rechecked mod1, mod5, mod6] +[stale mod6] +[builtins fixtures/module.pyi] +[out2] +tmp/mod1.py:3: error: Module has no attribute "mod7" + +[case testIncrementalNestedBrokenCascade] +import mod1 + +[file mod1.py] +import mod2 +def accept_int(a: int) -> int: return a +accept_int(mod2.mod3.mod4.const) + +[file mod2/__init__.py] +import mod2.mod3 as mod3 + +[file mod2/mod3/__init__.py] +import mod2.mod3.mod4 as mod4 + +[file mod2/mod3/__init__.py.2] +# Import is gone! + +[file mod2/mod3/mod4.py] +const = 3 + +[rechecked mod1, mod2, mod2.mod3] +[stale mod2.mod3] +[builtins fixtures/module.pyi] +[out2] +tmp/mod1.py:3: error: Module has no attribute "mod4" + +[case testIncrementalNestedBrokenCascadeWithType1] +import mod1, mod2.mod3.mod5 + +[file mod1.py] +import mod2 +def accept_int(x: int) -> None: pass +def produce() -> mod2.CustomType: + return mod2.CustomType() +a = produce() +accept_int(a.foo()) + +[file mod2/__init__.py] +from mod2.mod3 import CustomType + +[file mod2/mod3/__init__.py] +from mod2.mod3.mod4 import CustomType + +[file mod2/mod3/__init__.py.2] +# Import a different class that also happens to be called 'CustomType' +from mod2.mod3.mod5 import CustomType +def produce() -> CustomType: + return CustomType() + +[file mod2/mod3/mod4.py] +class CustomType: + def foo(self) -> int: return 1 + +[file mod2/mod3/mod5.py] +class CustomType: + def foo(self) -> str: return "a" + +[rechecked mod1, mod2, mod2.mod3] +[stale mod2, mod2.mod3] +[builtins fixtures/module.pyi] +[out1] +[out2] +tmp/mod1.py:6: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalNestedBrokenCascadeWithType2] +import mod1, mod2.mod3.mod5 + +[file mod1.py] +from mod2 import produce +def accept_int(x: int) -> None: pass +a = produce() +accept_int(a.foo()) + +[file mod2/__init__.py] +from mod2.mod3 import produce + +[file mod2/mod3/__init__.py] +from mod2.mod3.mod4 import CustomType +def produce() -> CustomType: + return CustomType() + +[file mod2/mod3/__init__.py.2] +# Import a different class that also happens to be called 'CustomType' +from mod2.mod3.mod5 import CustomType +def produce() -> CustomType: + return CustomType() + +[file mod2/mod3/mod4.py] +class CustomType: + def foo(self) -> int: return 1 + +[file mod2/mod3/mod5.py] +class CustomType: + def foo(self) -> str: return "a" + +[rechecked mod1, mod2, mod2.mod3] +[stale mod2.mod3] +[builtins fixtures/module.pyi] +[out1] +[out2] +tmp/mod1.py:4: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalRemoteChange] +import mod1 + +[file mod1.py] +import mod2 +def accepts_int(a: int) -> None: pass +accepts_int(mod2.mod3.mod4.const) + +[file mod2.py] +import mod3 + +[file mod3.py] +import mod4 + +[file mod4.py] +const = 3 + +[file mod4.py.2] +const = "foo" + +[rechecked mod1, mod3, mod4] +[stale mod4] +[out2] +tmp/mod1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" + +[case testIncrementalBadChange] +import mod1 + +[file mod1.py] +from mod2 import func2 + +def func1() -> int: + return func2() + +[file mod2.py] +def func2() -> int: + return 1 + +[file mod2.py.2] +def func2() -> str: + return "foo" + +[rechecked mod1, mod2] +[stale mod2] +[out2] +tmp/mod1.py:4: error: Incompatible return value type (got "str", expected "int") + +[case testIncrementalBadChangeWithSave] +import mod0 + +[file mod0.py] +import mod1 +A = mod1.func2() + +[file mod1.py] +from mod2 import func2 + +def func1() -> int: + return func2() + +[file mod2.py] +def func2() -> int: + return 1 + +[file mod2.py.2] +def func2() -> str: + return "foo" + +[rechecked mod0, mod1, mod2] +[stale mod2] +[out2] +tmp/mod1.py:4: error: Incompatible return value type (got "str", expected "int") + +[case testIncrementalOkChangeWithSave] +import mod0 + +[file mod0.py] +import mod1 +A = mod1.func2() + +[file mod1.py] +from mod2 import func2 + +def func1() -> int: + func2() + return 1 + +[file mod2.py] +def func2() -> int: + return 1 + +[file mod2.py.2] +def func2() -> str: + return "foo" + +[rechecked mod0, mod1, mod2] +[stale mod0, mod2] +[out2] + +[case testIncrementalWithComplexDictExpression] +import mod1 + +[file mod1.py] +import mod1_private + +[file mod1_private.py] +my_dict = { + 'a': [1, 2, 3], + 'b': [4, 5, 6] +} + +[file mod1_private.py.2] +my_dict = { + 'a': [1, 2, 3], + 'b': [4, 5, 'a'] +} + +[rechecked mod1, mod1_private] +[stale mod1_private] +[builtins fixtures/dict.pyi] + +[case testIncrementalWithComplexConstantExpressionNoAnnotation] +import mod1 + +[file mod1.py] +import mod1_private + +[file mod1_private.py] +def foobar() -> int: return 1 +def baz() -> int: return 2 +const = 1 + foobar() + +[file mod1_private.py.2] +def foobar() -> int: return 1 +def baz() -> int: return 2 +const = 1 + baz() + +[rechecked mod1_private] +[stale] + +[case testIncrementalWithComplexConstantExpressionWithAnnotation] +import mod1 + +[file mod1.py] +import mod1_private + +[file mod1_private.py] +def foobar() -> int: return 1 +def baz() -> int: return 2 +const = 1 + foobar() # type: int + +[file mod1_private.py.2] +def foobar() -> int: return 1 +def baz() -> int: return 2 +const = 1 + baz() # type: int + +[rechecked mod1_private] +[stale] + +[case testIncrementalSmall] +import mod1 + +[file mod1.py] +import mod1_private +def accepts_int(a: int) -> None: pass +accepts_int(mod1_private.some_func(12)) + +[file mod1_private.py] +def some_func(a: int) -> int: + return 1 + +[file mod1_private.py.2] +def some_func(a: int) -> str: + return "a" + +[rechecked mod1, mod1_private] +[stale mod1_private] +[builtins fixtures/ops.pyi] +[out2] +tmp/mod1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" + +[case testIncrementalWithDecorators] +import mod1 + +[file mod1.py] +import mod1_private +def accepts_int(a: int) -> None: pass +accepts_int(mod1_private.some_func(12)) + +[file mod1_private.py] +from typing import Callable +def multiply(f: Callable[[int], int]) -> Callable[[int], int]: + return lambda a: f(a) * 10 + +def stringify(f: Callable[[int], int]) -> Callable[[int], str]: + return lambda a: str(f(a)) + +@multiply +def some_func(a: int) -> int: + return a + 2 + +[file mod1_private.py.2] +from typing import Callable +def multiply(f: Callable[[int], int]) -> Callable[[int], int]: + return lambda a: f(a) * 10 + +def stringify(f: Callable[[int], int]) -> Callable[[int], str]: + return lambda a: str(f(a)) + +@stringify +def some_func(a: int) -> int: + return a + 2 +[rechecked mod1, mod1_private] +[stale mod1_private] +[builtins fixtures/ops.pyi] +[out2] +tmp/mod1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" + +[case testIncrementalChangingClassAttributes] +import mod1 + +[file mod1.py] +import mod2 +mod2.Foo.A + +[file mod2.py] +class Foo: + A = 3 + +[file mod2.py.2] +class Foo: + A = "hello" + +[rechecked mod1, mod2] +[stale mod2] + +[case testIncrementalChangingFields] +import mod1 + +[file mod1.py] +import mod2 +f = mod2.Foo() +f.A + +[file mod2.py] +class Foo: + def __init__(self) -> None: + self.A = 3 + +[file mod2.py.2] +class Foo: + def __init__(self) -> None: + self.A = "hello" + +[rechecked mod1, mod2] +[stale mod2] +[out2] + +[case testIncrementalChangingFieldsWithAssignment] +import mod1 + +[file mod1.py] +import mod2 +f = mod2.Foo() +B = f.A + +[file mod2.py] +class Foo: + def __init__(self) -> None: + self.A = 3 + +[file mod2.py.2] +class Foo: + def __init__(self) -> None: + self.A = "hello" + +[rechecked mod1, mod2] +[stale mod1, mod2] + +[case testIncrementalCheckingChangingFields] +import mod1 + +[file mod1.py] +import mod2 +def accept_int(a: int) -> int: return a +f = mod2.Foo() +accept_int(f.A) + +[file mod2.py] +class Foo: + def __init__(self) -> None: + self.A = 3 + +[file mod2.py.2] +class Foo: + def __init__(self) -> None: + self.A = "hello" + +[rechecked mod1, mod2] +[stale mod2] +[out2] +tmp/mod1.py:4: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalNestedClassDefinition] +import mod1 + +[file mod1.py] +import mod2 +b = mod2.Foo.Bar() +b.attr + +[file mod2.py] +class Foo: + class Bar: + attr = 3 + +[file mod2.py.2] +class Foo: + class Bar: + attr = "foo" + +[rechecked mod1, mod2] +[stale mod2] + +[case testIncrementalSimpleBranchingModules] +import mod1 +import mod2 + +[file mod1.py] +def func() -> None: pass + +[file mod2.py] +def func() -> None: pass + +[file mod1.py.2] +def func() -> int: return 1 + +[rechecked mod1] +[stale mod1] + +[case testIncrementalSubmoduleImport] +from parent.childA import Foo + +def func1() -> Foo: + return Foo() + +[file parent/__init__.py] +from parent.childA import Foo +from parent.childB import Bar + +__all__ = ['Foo', 'Bar'] + +[file parent/childA.py] +import parent + +class Foo: + def test(self) -> int: + return parent.Bar().test() + +[file parent/childB.py] +class Bar: + def test(self) -> int: return 3 + +[builtins fixtures/module_all.pyi] +[rechecked] +[stale] + +[case testIncrementalSubmoduleWithAttr] +import mod.child +x = mod.child.Foo() +x.bar() + +[file mod/__init__.py] + +[file mod/child.py] +class Foo: + def bar(self) -> None: pass +[builtins fixtures/module.pyi] +[rechecked] +[stale] + +[case testIncrementalNestedSubmoduleImportFromWithAttr] +from mod1.mod2 import mod3 +def accept_int(a: int) -> None: pass + +accept_int(mod3.val3) + +[file mod1/__init__.py] +val1 = 1 + +[file mod1/mod2/__init__.py] +val2 = 1 + +[file mod1/mod2/mod3.py] +val3 = 1 + +[builtins fixtures/module.pyi] +[rechecked] +[stale] + +[case testIncrementalNestedSubmoduleWithAttr] +import mod1.mod2.mod3 +def accept_int(a: int) -> None: pass + +accept_int(mod1.mod2.mod3.val3) +accept_int(mod1.mod2.val2) +accept_int(mod1.val1) + +[file mod1/__init__.py] +val1 = 1 + +[file mod1/mod2/__init__.py] +val2 = 1 + +[file mod1/mod2/mod3.py] +val3 = 1 + +[builtins fixtures/module.pyi] +[rechecked] +[stale] + +[case testIncrementalSubmoduleParentWithImportFrom] +import parent + +[file parent/__init__.py] +from parent import a + +[file parent/a.py] +val = 3 + +[builtins fixtures/args.pyi] +[stale] + +[case testIncrementalSubmoduleParentBackreference] +import parent + +[file parent/__init__.py] +from parent import a + +[file parent/a.py] +import parent.b + +[file parent/b.py] + +[builtins fixtures/args.pyi] +[stale] + +[case testIncrementalSubmoduleParentBackreferenceComplex] +import parent + +[file parent/__init__.py] +import parent.a + +[file parent/a.py] +import parent.b +import parent.c + +[file parent/b.py] +import parent.a + +[file parent/c.py] +import parent.a + +[builtins fixtures/args.pyi] +[stale] + +[case testIncrementalReferenceNewFileWithImportFrom] +from parent import a + +[file parent/__init__.py] + +[file parent/a.py] + +[file parent/a.py.2] +from parent import b +reveal_type(b.x) + +[file parent/b.py.2] +x = 10 + +[stale parent.b] +[rechecked parent.a, parent.b] +[out2] +tmp/parent/a.py:2: note: Revealed type is "builtins.int" + +[case testIncrementalReferenceExistingFileWithImportFrom] +from parent import a, b + +[file parent/__init__.py] + +[file parent/a.py] + +[file parent/b.py] + +[file parent/a.py.2] +from parent import b + +[stale parent.a] + +[case testIncrementalWithTypeIgnoreOnDirectImport] +import a, b + +[file a.py] +import b # type: ignore + +[file b.py] +import c + +[file c.py] + +[stale] + +[case testIncrementalWithTypeIgnoreOnImportFrom] +import a, b + +[file a.py] +from b import something # type: ignore + +[file b.py] +import c +something = 3 + +[file c.py] + +[stale] + +[case testIncrementalWithPartialTypeIgnore] +import a # type: ignore +import a.b + +[file a/__init__.py] + +[file a/b.py] + +[stale] + +[case testIncrementalAnyIsDifferentFromIgnore] +import b + +[file b.py] +from typing import Any +import a.b + +[file b.py.2] +from typing import Any + +a = 3 # type: Any +import a.b + +[file a/__init__.py] + +[file a/b.py] + +[stale b] + +[case testIncrementalSilentImportsAndImportsInClass] +# flags: --ignore-missing-imports +class MyObject(object): + from bar import FooBar +[stale] + +[case testIncrementalSameFileSize] +import m + +[file m.py] +def foo(a: int) -> None: pass +def bar(a: str) -> None: pass + +foo(3) + +[file m.py.2] +def foo(a: int) -> None: pass +def bar(a: str) -> None: pass + +bar(3) + +[rechecked m] +[stale] +[out2] +tmp/m.py:4: error: Argument 1 to "bar" has incompatible type "int"; expected "str" + +[case testIncrementalUnsilencingModule] +# cmd: mypy -m main package.subpackage.mod2 +# cmd2: mypy -m main package.subpackage.mod1 +# flags: --follow-imports=skip + +[file main.py] +from package.subpackage.mod1 import Class + +def handle(c: Class) -> None: + c.some_attribute + +[file package/__init__.py] +# empty + +[file package/subpackage/__init__.py] +# empty + +[file package/subpackage/mod1.py] +import collections # Any previously unloaded package works here + +class Class: pass + +[file package/subpackage/mod2.py] +# empty + +[builtins fixtures/args.pyi] +[rechecked collections, main, package.subpackage.mod1] +[stale collections, package.subpackage.mod1] +[out2] +tmp/main.py:4: error: "Class" has no attribute "some_attribute" + +[case testIncrementalWithIgnores] +import foo # type: ignore + +[builtins fixtures/module.pyi] +[stale] + +[case testIncrementalWithSilentImportsAndIgnore] +# cmd: mypy -m main b +# cmd2: mypy -m main c c.submodule +# flags: --follow-imports=skip + +[file main.py] +import a # type: ignore +import b +import c + +a.A().foo() +b.B().foo() +c.C().foo() + +[file b.py] +class B: + def foo(self) -> None: pass + +[file b.py.2] + +[file c/__init__.py] +class C: pass + +[file c/submodule.py] +val = 3 # type: int +if int(): + val = "foo" + +[builtins fixtures/module_all.pyi] +[rechecked main, c, c.submodule] +[stale c] +[out2] +tmp/c/submodule.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/main.py:7: error: "C" has no attribute "foo" + +[case testIncrementalRemoteError] +import m +m.C().foo().bar() +[file m.py] +import n +class C: + def foo(self) -> n.A: pass +[file n.py] +class A: + def bar(self): pass +[file n.py.2] +class A: + pass +[rechecked m, n] +[stale n] +[out2] +main:2: error: "A" has no attribute "bar" + +[case testIncrementalRemoteErrorFixed] +import m +m.C().foo().bar() +[file m.py] +import n +class C: + def foo(self) -> n.A: pass +[file n.py] +class A: + pass +[file n.py.2] +class A: + def bar(self): pass +[rechecked m, n] +[stale n] +[out1] +main:2: error: "A" has no attribute "bar" + +[case testIncrementalChangedError] +import m +[file m.py] +import n +def accept_int(x: int) -> None: pass +accept_int(n.foo) +[file n.py] +foo = "hello" +reveal_type(foo) +[file n.py.2] +foo = 3.14 +reveal_type(foo) +[rechecked m, n] +[stale] +[out1] +tmp/n.py:2: note: Revealed type is "builtins.str" +tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" +[out2] +tmp/n.py:2: note: Revealed type is "builtins.float" +tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "float"; expected "int" + +[case testIncrementalReplacingImports] +import good, bad, client + +[file good.py] +def foo(a: int) -> None: pass + +[file bad.py] +def foo(a: str) -> None: pass + +[file client.py] +import good +import bad +from good import foo +foo(3) + +[file client.py.2] +import good +import bad +from bad import foo +foo(3) + +[rechecked client] +[stale] +[out2] +tmp/client.py:4: error: Argument 1 to "foo" has incompatible type "int"; expected "str" + +[case testIncrementalChangingAlias] +import m1, m2, m3, m4, m5 + +[file m1.py] +from m2 import A +def accepts_int(x: int) -> None: pass +accepts_int(A()) + +[file m2.py] +from m3 import A + +[file m3.py] +from m4 import B +A = B + +[file m3.py.2] +from m5 import C +A = C + +[file m4.py] +def B() -> int: + return 42 + +[file m5.py] +def C() -> str: + return "hello" + +[rechecked m1, m2, m3] +[stale m3] +[out2] +tmp/m1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" + +[case testIncrementalStoresAliasTypeVars] +import a + +[file mod.py] +from typing import TypeVar, Union +T = TypeVar('T') +Alias = Union[int, T] +x: Alias[str] + +[file a.py] +from mod import Alias, x + +[file a.py.2] +from mod import Alias, x + +reveal_type(x) +y: Alias[int] +reveal_type(y) +[out2] +tmp/a.py:3: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/a.py:5: note: Revealed type is "Union[builtins.int, builtins.int]" + +[case testIncrementalSilentImportsWithBlatantError] +# cmd: mypy -m main +# flags: --follow-imports=skip + +[file main.py] +from evil import Hello + +[file main.py.2] +from evil import Hello +reveal_type(Hello()) + +[file evil.py] +def accept_int(x: int) -> None: pass +accept_int("not an int") + +[rechecked main] +[stale] +[out2] +tmp/main.py:2: note: Revealed type is "Any" + +[case testIncrementalImportIsNewlySilenced] +# cmd: mypy -m main foo +# cmd2: mypy -m main +# flags: --follow-imports=skip + +[file main.py] +from foo import bar +def accept_int(x: int) -> None: pass +accept_int(bar) + +[file foo.py] +bar = 3 + +[file foo.py.2] +# Empty! + +[rechecked main] +[stale main] + +[case testIncrementalSilencedModuleNoLongerCausesError] +# cmd: mypy -m main evil +# cmd2: mypy -m main +# flags: --follow-imports=skip + +[file main.py] +from evil import bar +def accept_int(x: int) -> None: pass +accept_int(bar) +reveal_type(bar) + +[file evil.py] +bar = "str" + +[rechecked main] +[stale] +[out1] +tmp/main.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" +tmp/main.py:4: note: Revealed type is "builtins.str" +[out2] +tmp/main.py:4: note: Revealed type is "Any" + +[case testIncrementalFixedBugCausesPropagation] +import mod1 + +[file mod1.py] +from mod2 import A +val = A().makeB().makeC().foo() +reveal_type(val) + +[file mod2.py] +from mod3 import B +class A: + def makeB(self) -> B: return B() + +[file mod3.py] +from mod4 import C +class B: + def makeC(self) -> C: + val = 3 # type: int + if 1: + val = "str" # deliberately triggering error + return C() + +[file mod3.py.2] +from mod4 import C +class B: + def makeC(self) -> C: return C() + +[file mod4.py] +class C: + def foo(self) -> int: return 1 + +[rechecked mod3, mod2, mod1] +[stale mod3, mod2] +[out1] +tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/mod1.py:3: note: Revealed type is "builtins.int" + +[out2] +tmp/mod1.py:3: note: Revealed type is "builtins.int" + +[case testIncrementalIncidentalChangeWithBugCausesPropagation] +import mod1 + +[file mod1.py] +from mod2 import A +val = A().makeB().makeC().foo() +reveal_type(val) + +[file mod2.py] +from mod3 import B +class A: + def makeB(self) -> B: return B() + +[file mod3.py] +from mod4 import C +class B: + def makeC(self) -> C: + val = 3 # type: int + if 1: + val = "str" # deliberately triggering error + return C() + +[file mod4.py] +class C: + def foo(self) -> int: return 1 + +[file mod4.py.2] +class C: + def foo(self) -> str: return 'a' + +[rechecked mod4, mod3, mod2, mod1] +[stale mod4] +[out1] +tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/mod1.py:3: note: Revealed type is "builtins.int" + +[out2] +tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/mod1.py:3: note: Revealed type is "builtins.str" + +[case testIncrementalIncidentalChangeWithBugFixCausesPropagation] +import mod1 + +[file mod1.py] +from mod2 import A +val = A().makeB().makeC().foo() +reveal_type(val) + +[file mod2.py] +from mod3 import B +class A: + def makeB(self) -> B: return B() + +[file mod3.py] +from mod4 import C +class B: + def makeC(self) -> C: + val = 3 # type: int + if 1: + val = "str" # deliberately triggering error + return C() + +[file mod3.py.2] +from mod4 import C +class B: + def makeC(self) -> C: return C() + +[file mod4.py] +class C: + def foo(self) -> int: return 1 + +[file mod4.py.2] +class C: + def foo(self) -> str: return 'a' + +[rechecked mod4, mod3, mod2, mod1] +[stale mod4, mod3, mod2] +[out1] +tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/mod1.py:3: note: Revealed type is "builtins.int" + +[out2] +tmp/mod1.py:3: note: Revealed type is "builtins.str" + +[case testIncrementalSilentImportsWithInnerImports] +# cmd: mypy -m main foo +# flags: --ignore-missing-imports + +[file main.py] +from foo import MyClass +m = MyClass() + +[file main.py.2] +from foo import MyClass +m = MyClass() +reveal_type(m.val) + +[file foo.py] +class MyClass: + def __init__(self) -> None: + import unrelated + self.val = unrelated.test() + +[rechecked main] +[stale] +[out2] +tmp/main.py:3: note: Revealed type is "Any" + +[case testIncrementalSilentImportsWithInnerImportsAndNewFile] +# cmd: mypy -m main foo +# cmd2: mypy -m main foo unrelated +# flags: --follow-imports=skip + +[file main.py] +from foo import MyClass +m = MyClass() + +[file main.py.2] +from foo import MyClass +m = MyClass() +reveal_type(m.val) + +[file foo.py] +class MyClass: + def __init__(self) -> None: + import unrelated + self.val = unrelated.test() + +[file unrelated.py] +def test() -> str: return "foo" + +[rechecked main, foo, unrelated] +[stale foo, unrelated] +[out2] +tmp/main.py:3: note: Revealed type is "builtins.str" + +[case testIncrementalWorksWithNestedClasses] +import foo + +[file foo.py] +class MyClass: + class NestedClass: + pass + + class_attr = NestedClass() + +[rechecked] +[stale] + +[case testIncrementalWorksWithBasicProtocols] +import a +[file a.py] +from b import P + +x: int +y: P[int] +x = y.meth() + +class C: + def meth(self) -> int: + pass +y = C() + +[file a.py.2] +from b import P + +x: str +y: P[str] +x = y.meth() + +class C: + def meth(self) -> str: + pass +y = C() +[file b.py] +from typing import Protocol, TypeVar + +T = TypeVar('T', covariant=True) +class P(Protocol[T]): + def meth(self) -> T: + pass + +[case testIncrementalSwitchFromNominalToStructural] +import a +[file a.py] +from b import B, fun +class C(B): + def x(self) -> int: pass + def y(self) -> int: pass +fun(C()) + +[file b.py] +from typing import Protocol +class B: + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[file b.py.2] +from typing import Protocol +class B(Protocol): + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[file a.py.3] +from b import fun +class C: + def x(self) -> int: pass + def y(self) -> int: pass +fun(C()) +[out1] +[out2] +[out3] + +[case testIncrementalSwitchFromStructuralToNominal] +import a +[file a.py] +from b import fun +class C: + def x(self) -> int: pass + def y(self) -> int: pass +fun(C()) + +[file b.py] +from typing import Protocol +class B(Protocol): + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[file b.py.2] +from typing import Protocol +class B: + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[out1] +[out2] +tmp/a.py:5: error: Argument 1 to "fun" has incompatible type "C"; expected "B" + +[case testIncrementalWorksWithNamedTuple] +import foo + +[file foo.py] +from mid import MyTuple +def accept_int(x: int) -> None: pass +accept_int(MyTuple(1, "b", "c").a) + +[file mid.py] +from bar import MyTuple + +[file bar.py] +from typing import NamedTuple +MyTuple = NamedTuple('MyTuple', [ + ('a', int), + ('b', str), + ('c', str) +]) + +[file bar.py.2] +from typing import NamedTuple +MyTuple = NamedTuple('MyTuple', [ + ('b', int), # a and b are swapped + ('a', str), + ('c', str) +]) + +[rechecked bar, mid, foo] +[stale bar] +[builtins fixtures/tuple.pyi] +[out2] +tmp/foo.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalWorksWithNestedNamedTuple] +import foo + +[file foo.py] +from mid import Outer +def accept_int(x: int) -> None: pass +accept_int(Outer.MyTuple(1, "b", "c").a) + +[file mid.py] +from bar import Outer + +[file bar.py] +from typing import NamedTuple +class Outer: + MyTuple = NamedTuple('MyTuple', [ + ('a', int), + ('b', str), + ('c', str) + ]) + +[file bar.py.2] +from typing import NamedTuple +class Outer: + MyTuple = NamedTuple('MyTuple', [ + ('b', int), # a and b are swapped + ('a', str), + ('c', str) + ]) + +[rechecked bar, mid, foo] +[stale bar] +[builtins fixtures/tuple.pyi] +[out2] +tmp/foo.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalPartialSubmoduleUpdate] +# cmd: mypy -m a +# cmd2: mypy -m a a.c +# flags: --follow-imports=skip + +[file a/__init__.py] +from .b import B +from .c import C + +[file a/b.py] +class B: pass + +[file a/c.py] +class C: pass + +[file a/c.py.2] +class C: pass +pass + +[rechecked a, a.c] +[stale a, a.c] +[out] + +[case testIncrementalNestedClassRef] +import top + +[file top.py] +from funcs import callee +from classes import Outer +def caller(a: Outer.Inner) -> None: + callee(a) + +[file funcs.py] +from classes import Outer +def callee(a: Outer.Inner) -> None: + pass + +[file classes.py] +class Outer: + class Inner: + pass + +[file top.py.2] +from funcs import callee +from classes import Outer +def caller(a: Outer.Inner) -> int: + callee(a) + return 0 + +[case testIncrementalLoadsParentAfterChild] +# cmd: mypy -m r.s + +[file r/__init__.py] +from . import s + +[file r/m.py] +class R: pass + +[file r/s.py] +from . import m +R = m.R +a: R + +[file r/s.py.2] +from . import m +R = m.R +a: R + +[case testIncrementalBaseClassAttributeConflict] +class A: pass +class B: pass + +class X: + attr = None # type: A +class Y: + attr = None # type: B +class Z(X, Y): pass +[stale] +[out] +main:8: error: Definition of "attr" in base class "X" is incompatible with definition in base class "Y" +[out2] +main:8: error: Definition of "attr" in base class "X" is incompatible with definition in base class "Y" + +[case testIncrementalFollowImportsSilent] +# flags: --follow-imports=silent +import a +[file a.py] +x = 0 +[file a.py.2] +x = 0 +x + '' + +[case testIncrementalFollowImportsSkip] +# flags: --follow-imports=skip +import a +reveal_type(a.x) +[file a.py] +/ +[file a.py.2] +// +[out] +main:3: note: Revealed type is "Any" +[out2] +main:3: note: Revealed type is "Any" + +[case testIncrementalFollowImportsError] +# flags: --follow-imports=error +import a +[file a.py] +/ +[file a.py.2] +// +[out1] +main:2: error: Import of "a" ignored +main:2: note: (Using --follow-imports=error, module not passed on command line) +[out2] +main:2: error: Import of "a" ignored +main:2: note: (Using --follow-imports=error, module not passed on command line) + +[case testIncrementalFollowImportsVariable] +# flags: --config-file tmp/mypy.ini +import a +reveal_type(a.x) +[file a.py] +x = 0 +[file mypy.ini] +\[mypy] +follow_imports = normal +[file mypy.ini.2] +\[mypy] +follow_imports = skip +[out1] +main:3: note: Revealed type is "builtins.int" +[out2] +main:3: note: Revealed type is "Any" + + +[case testIncrementalFollowImportsVariablePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import a +reveal_type(a.x) + +[file a.py] +x = 0 + +[file pyproject.toml] +\[tool.mypy] +follow_imports = 'normal' + +[file pyproject.toml.2] +\[tool.mypy] +follow_imports = 'skip' + +[out1] +main:3: note: Revealed type is "builtins.int" + +[out2] +main:3: note: Revealed type is "Any" + + +[case testIncrementalIgnoreErrors] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +import module_that_will_be_deleted +[file module_that_will_be_deleted.py] + +[file mypy.ini] +\[mypy] +\[mypy-a] +ignore_errors = True +[delete module_that_will_be_deleted.py.2] +[out1] +[out2] + +[case testIncrementalNamedTupleInMethod] +from ntcrash import nope +[file ntcrash.py] +from typing import NamedTuple +class C: + def f(self) -> None: + A = NamedTuple('A', [('x', int), ('y', int)]) +[builtins fixtures/tuple.pyi] +[out1] +main:1: error: Module "ntcrash" has no attribute "nope" +[out2] +main:1: error: Module "ntcrash" has no attribute "nope" + +[case testIncrementalNamedTupleInMethod2] +from ntcrash import nope +[file ntcrash.py] +from typing import NamedTuple +class C: + class D: + def f(self) -> None: + A = NamedTuple('A', [('x', int), ('y', int)]) +[builtins fixtures/tuple.pyi] +[out1] +main:1: error: Module "ntcrash" has no attribute "nope" +[out2] +main:1: error: Module "ntcrash" has no attribute "nope" + +[case testIncrementalNamedTupleInMethod3] +from ntcrash import nope +[file ntcrash.py] +from typing import NamedTuple +class C: + def a(self): + class D: + def f(self) -> None: + A = NamedTuple('A', [('x', int), ('y', int)]) +[builtins fixtures/tuple.pyi] +[out1] +main:1: error: Module "ntcrash" has no attribute "nope" +[out2] +main:1: error: Module "ntcrash" has no attribute "nope" + +[case testIncrementalTypedDictInMethod] +from tdcrash import nope +[file tdcrash.py] +from mypy_extensions import TypedDict +class C: + def f(self) -> None: + A = TypedDict('A', {'x': int, 'y': int}) +[builtins fixtures/dict.pyi] +[out1] +main:1: error: Module "tdcrash" has no attribute "nope" +[out2] +main:1: error: Module "tdcrash" has no attribute "nope" + +[case testIncrementalTypedDictInMethod2] +from tdcrash import nope +[file tdcrash.py] +from mypy_extensions import TypedDict +class C: + class D: + def f(self) -> None: + A = TypedDict('A', {'x': int, 'y': int}) +[builtins fixtures/dict.pyi] +[out1] +main:1: error: Module "tdcrash" has no attribute "nope" +[out2] +main:1: error: Module "tdcrash" has no attribute "nope" + +[case testIncrementalTypedDictInMethod3] +from tdcrash import nope +[file tdcrash.py] +from mypy_extensions import TypedDict +class C: + def a(self): + class D: + def f(self) -> None: + A = TypedDict('A', {'x': int, 'y': int}) +[builtins fixtures/dict.pyi] +[out1] +main:1: error: Module "tdcrash" has no attribute "nope" +[out2] +main:1: error: Module "tdcrash" has no attribute "nope" + +[case testIncrementalNewTypeInMethod] +from ntcrash import nope +[file ntcrash.py] +from mypy_extensions import TypedDict +from typing import NewType, NamedTuple +class C: + def f(self) -> None: + X = NewType('X', int) + A = TypedDict('A', {'x': X, 'y': int}) + B = NamedTuple('B', [('x', X)]) + +def f() -> None: + X = NewType('X', int) + A = TypedDict('A', {'x': X, 'y': int}) + B = NamedTuple('B', [('x', X)]) + +[builtins fixtures/dict.pyi] +[out1] +main:1: error: Module "ntcrash" has no attribute "nope" +[out2] +main:1: error: Module "ntcrash" has no attribute "nope" + +[case testIncrementalInnerClassAttrInMethod] +import crash +nonexisting +[file crash.py] +class C: + def f(self) -> None: + class A: + pass + self.a = A() +[out1] +main:2: error: Name "nonexisting" is not defined +[out2] +main:2: error: Name "nonexisting" is not defined + +[case testIncrementalInnerClassAttrInMethodReveal] +import crash +reveal_type(crash.C().a) +reveal_type(crash.D().a) +[file crash.py] +from typing import TypeVar, Generic +T = TypeVar('T') +class C: + def f(self) -> None: + class A: + pass + self.a = A() +reveal_type(C().a) +class D: + def f(self) -> None: + class A: + def g(self) -> None: + class B(Generic[T]): + pass + self.b = B[int]() + self.a = A().b +reveal_type(D().a) +[out1] +tmp/crash.py:8: note: Revealed type is "crash.A@5" +tmp/crash.py:17: note: Revealed type is "crash.B@13[builtins.int]" +main:2: note: Revealed type is "crash.A@5" +main:3: note: Revealed type is "crash.B@13[builtins.int]" +[out2] +tmp/crash.py:8: note: Revealed type is "crash.A@5" +tmp/crash.py:17: note: Revealed type is "crash.B@13[builtins.int]" +main:2: note: Revealed type is "crash.A@5" +main:3: note: Revealed type is "crash.B@13[builtins.int]" + +[case testGenericMethodRestoreMetaLevel] +from typing import Dict + +d = {} # type: Dict[str, int] +g = d.get # This should not crash: see https://github.com/python/mypy/issues/2804 +[builtins fixtures/dict.pyi] + +[case testGenericMethodRestoreMetaLevel2] +from typing import TypeVar + +T = TypeVar('T') + +class D: + def m(self, x: T) -> T: + return x + +g = D().m # This should not crash: see https://github.com/python/mypy/issues/2804 +[builtins fixtures/dict.pyi] + +[case testGenericMethodRestoreMetaLevel3] +from typing import TypeVar +T = TypeVar('T') + +class C: + def m(self, x: T) -> T: + return x + +class D(C): + def __init__(self) -> None: + self.d = super().m # This should not crash: see https://github.com/python/mypy/issues/2804 +[builtins fixtures/dict.pyi] + +[case testIncrementalPerFileFlags] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +pass +[file mypy.ini] +\[mypy] +warn_no_return = False +\[mypy-a] +warn_no_return = True +[rechecked] + +[case testIncrementalClassVar] +from typing import ClassVar +class A: + x = None # type: ClassVar +A().x = 0 +[out1] +main:4: error: Cannot assign to class variable "x" via instance +[out2] +main:4: error: Cannot assign to class variable "x" via instance + +[case testIncrementalClassVarGone] +import m +m.A().x = 0 +[file m.py] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] +[file m.py.2] +class A: + x = None # type: int +[out1] +main:2: error: Cannot assign to class variable "x" via instance + +[case testCachingClassVar] +import b +[file a.py] +from typing import ClassVar +class A: + x = None # type: ClassVar[int] +[file b.py] +import a +[file b.py.2] +import a +a.A().x = 0 +[out2] +tmp/b.py:2: error: Cannot assign to class variable "x" via instance + +[case testSerializeTypedDict] +import b +reveal_type(b.x) +y: b.A +reveal_type(y) +[file b.py] +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': int, 'y': str}) +x: A +[builtins fixtures/dict.pyi] +[out1] +main:2: note: Revealed type is "TypedDict('b.A', {'x': builtins.int, 'y': builtins.str})" +main:4: note: Revealed type is "TypedDict('b.A', {'x': builtins.int, 'y': builtins.str})" +[out2] +main:2: note: Revealed type is "TypedDict('b.A', {'x': builtins.int, 'y': builtins.str})" +main:4: note: Revealed type is "TypedDict('b.A', {'x': builtins.int, 'y': builtins.str})" + +[case testSerializeMetaclass] +import b +reveal_type(b.A.f()) +m: b.M = b.A +reveal_type(b.a.f()) +[file b.py] +from typing import Type + +class M(type): + def f(cls) -> int: return 0 +class A(metaclass=M): pass +a: Type[A] +[out] +main:2: note: Revealed type is "builtins.int" +main:4: note: Revealed type is "builtins.int" +[out2] +main:2: note: Revealed type is "builtins.int" +main:4: note: Revealed type is "builtins.int" + +[case testSerializeMetaclassInImportCycle1] +import b +import c +reveal_type(b.A.f()) +m: c.M = b.A +reveal_type(b.a.f()) +[file b.py] +from typing import Type +from c import M +class A(metaclass=M): pass +a: Type[A] +[file c.py] +class M(type): + def f(cls) -> int: return 0 +[out] +main:3: note: Revealed type is "builtins.int" +main:5: note: Revealed type is "builtins.int" +[out2] +main:3: note: Revealed type is "builtins.int" +main:5: note: Revealed type is "builtins.int" + +[case testSerializeMetaclassInImportCycle2] +import b +import c +reveal_type(c.A.f()) +m: b.M = c.A +reveal_type(c.a.f()) +[file b.py] +from c import a +class M(type): + def f(cls) -> int: return 0 +[file c.py] +from typing import Type +import b +class A(metaclass=b.M): pass +a: Type[A] +[out] +main:3: note: Revealed type is "builtins.int" +main:5: note: Revealed type is "builtins.int" +[out2] +main:3: note: Revealed type is "builtins.int" +main:5: note: Revealed type is "builtins.int" + +[case testDeleteFile] +import n +[file n.py] +import m +[file m.py] +x = 1 +[delete m.py.2] +[rechecked n] +[stale] +[out2] +tmp/n.py:1: error: Cannot find implementation or library stub for module named "m" +tmp/n.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testDeleteFileWithinCycle] +import a +[file a.py] +import b +[file b.py] +import c +[file c.py] +import a +[file a.py.2] +import c +[delete b.py.2] +[rechecked a, c] +[stale a] +[out2] + +[case testThreePassesBasic] +import m +[file m.py] +def foo(): + pass +[file m.py.2] +def foo() -> None: + pass +[file m.py.3] +def foo(): + pass +[rechecked m] +[stale m] +[rechecked2 m] +[stale2 m] +[out3] + +[case testThreePassesErrorInThirdPass] +import m +[file m.py] +def foo(): + pass +[file m.py.2] +def foo() -> None: + pass +[file m.py.3] +def foo() -> int: + return '' +[rechecked m] +[stale m] +[rechecked2 m] +[stale2] +[out3] +tmp/m.py:2: error: Incompatible return value type (got "str", expected "int") + +[case testThreePassesThirdPassFixesError] +import n +[file n.py] +import m +x = m.foo(1) +[file m.py] +def foo(x): + pass +[file m.py.2] +def foo() -> str: + pass +[file m.py.3] +def foo(x) -> int: + pass +[rechecked m, n] +[stale m] +[rechecked2 m, n] +[stale2 m, n] +[out2] +tmp/n.py:2: error: Too many arguments for "foo" +[out3] + +[case testCacheDeletedAfterErrorsFound] +import a +[file a.py] +from b import x +[file b.py] +from c import x +[file c.py] +x = 1 +[file c.py.2] +1 + 1 +[file a.py.3] +from b import x +1 + 1 +[out] +[out2] +tmp/b.py:1: error: Module "c" has no attribute "x" +[out3] +tmp/b.py:1: error: Module "c" has no attribute "x" + +[case testCacheDeletedAfterErrorsFound2] + +import a +[file a.py] +from b import x +[file b.py] +from c import C +x: C +[file c.py] +class C: pass +[file c.py.2] +def C(): pass +[file a.py.3] +from b import x +1 + 1 +[out] +[out2] +tmp/b.py:2: error: Function "c.C" is not valid as a type +tmp/b.py:2: note: Perhaps you need "Callable[...]" or a callback protocol? +[out3] +tmp/b.py:2: error: Function "c.C" is not valid as a type +tmp/b.py:2: note: Perhaps you need "Callable[...]" or a callback protocol? + +[case testCacheDeletedAfterErrorsFound3] +import a +[file a.py] +import b +b.f() +[file b.py] +def f() -> None: pass +[file b.py.2] +def f(x) -> None: pass +[out] +[out2] +tmp/a.py:2: error: Missing positional argument "x" in call to "f" +[out3] +tmp/a.py:2: error: Missing positional argument "x" in call to "f" + +[case testCacheDeletedAfterErrorsFound4] +import a +[file a.py] +from b import x +[file b.py] +from c import x +[file c.py] +from d import x +[file d.py] +x = 1 +[file d.py.2] +1 + 1 +[file a.py.3] +from b import x +1 + 1 +[out] +[out2] +tmp/c.py:1: error: Module "d" has no attribute "x" +[out3] +tmp/c.py:1: error: Module "d" has no attribute "x" + +[case testNoCrashOnDeletedWithCacheOnCmdline] +# cmd: mypy -m nonexistent +# cmd2: mypy -m nonexistent +[file nonexistent.py] +[delete nonexistent.py.2] +[out] +[out2] +mypy: can't read file 'tmp/nonexistent.py': No such file or directory +-- ' + +[case testSerializeAbstractPropertyIncremental] +from abc import abstractmethod +import typing +class A: + @property + def f(self) -> int: + return 1 + @f.setter # type: ignore + @abstractmethod + def f(self, x: int) -> None: + pass +a = A() +[builtins fixtures/property.pyi] + +[case testSerializeAbstractPropertyDisallowUntypedIncremental] +# flags: --disallow-untyped-defs +from abc import abstractmethod +import typing +class A: + @property + def f(self) -> int: + return 1 + @f.setter # type: ignore + @abstractmethod + def f(self, x: int) -> None: + pass +a = A() +[builtins fixtures/property.pyi] + +[case testClassNamesResolutionCrashAccess] +import mod + +[file mod.py] +class C: + def __init__(self) -> None: + self.int = '' + + def f(self, f: int) -> None: + pass + +[file mod.py.2] +class C: + def __init__(self) -> None: + self.int = '' + + def f(self, f: int) -> None: + f.x + +[out] +[out2] +tmp/mod.py:6: error: "int" has no attribute "x" + +[case testClassNamesResolutionCrashReadCache] +import mod + +[file mod.py] +import submod + +[file mod.py.2] +from submod import C + +c = C() +reveal_type(c.int) +reveal_type(c.y) + +[file submod.py] +from typing import List + +class C: + def __init__(self) -> None: + self.int = [] # type: List[int] + + def f(self, f: int) -> None: + self.y = f + +[builtins fixtures/list.pyi] +[out] +[out2] +tmp/mod.py:4: note: Revealed type is "builtins.list[builtins.int]" +tmp/mod.py:5: note: Revealed type is "builtins.int" + +[case testClassNamesResolutionCrashReveal] +import mod + +[file mod.py] +class Foo(object): + + def __init__(self) -> None: + self.bytes = b"foo" + + def bar(self, f: bytes): + pass + +foo = Foo() +foo.bar(b"test") + +[file mod.py.2] +class Foo(object): + + def __init__(self) -> None: + self.bytes = b"foo" + + def bar(self, f: bytes): + reveal_type(f) + +foo = Foo() +foo.bar(b"test") +[out] +[out2] +tmp/mod.py:7: note: Revealed type is "builtins.bytes" + +[case testIncrementalWithSilentImports] +# cmd: mypy -m a +# cmd2: mypy -m b +# flags: --follow-imports=silent +[file a.py] +import b + +b.foo(1, 2) + +[file b.py] +def foo(a: int, b: int) -> str: + return a + b + +[out1] +[out2] +tmp/b.py:2: error: Incompatible return value type (got "int", expected "str") + +[case testForwardNamedTupleToUnionWithOtherNamedTUple] +from typing import NamedTuple, Union + +class Person(NamedTuple): + name: Union[str, "Pair"] + +class Pair(NamedTuple): + first: str + last: str + +Person(name=Pair(first="John", last="Doe")) +[builtins fixtures/tuple.pyi] +[out] + +[case testNoCrashForwardRefToBrokenDoubleNewTypeIncremental] +from typing import Any, List, NewType + +Foo = NewType('NotFoo', int) # type: ignore +Foos = NewType('Foos', List[Foo]) # type: ignore + +def frob(foos: List[Foos]) -> None: + pass +[builtins fixtures/list.pyi] +[out] + +[case testNoCrashForwardRefOverloadIncremental] +from typing import overload, List + +@overload +def f(x: int) -> int: ... +@overload +def f(x: F) -> F: ... +def f(x): + pass + +F = List[int] +[builtins fixtures/list.pyi] +[out] + +[case testNoCrashForwardRefOverloadIncrementalClass] +from typing import overload, Tuple, NamedTuple + +x: C +class C: + @overload + def f(self, x: str) -> N: pass + @overload + def f(self, x: int) -> int: pass + def f(self, x): + pass + +class N(NamedTuple): + x: A +A = Tuple[int] +[builtins fixtures/tuple.pyi] +[out] + +[case testNewTypeFromForwardNamedTupleIncremental] +from typing import NewType, NamedTuple, Tuple + +NT = NewType('NT', 'N') +class N(NamedTuple): + x: int + +x: NT = N(1) # type: ignore +x = NT(N(1)) +[builtins fixtures/tuple.pyi] +[out] + +[case testNewTypeFromForwardTypedDictIncremental] +from typing import NewType, Tuple, Dict +from mypy_extensions import TypedDict + +NT = NewType('NT', N) # type: ignore +class N(TypedDict): + x: A +A = Dict[str, int] +[builtins fixtures/dict.pyi] +[out] + +-- Some crazy self-referential named tuples, types dicts, and aliases +-- to be sure that everything can be _serialized_ (i.e. ForwardRefs are removed). +-- For this reason errors are silenced (tests with # type: ignore have equivalents in other files) + +[case testForwardTypeAliasInBase1] +from typing import List +class C(List['A']): + pass + +A = List[int] +x: int = C()[0][0] +[builtins fixtures/list.pyi] +[out] + +[case testForwardTypeAliasInBase2] + +from typing import List, Generic, TypeVar, NamedTuple +T = TypeVar('T') + +class C(A, B): # type: ignore + pass +class G(Generic[T]): pass +A = G[C] # type: ignore +class B(NamedTuple): + x: int + +C(1).x +C(1)[0] +[builtins fixtures/list.pyi] +[out] + +[case testSerializeRecursiveAliases1] + +from typing import Type, Callable, Union + +A = Union[A, int] # type: ignore +B = Callable[[B], int] # type: ignore +C = Type[C] # type: ignore +[out] + +[case testSerializeRecursiveAliases2] + +from typing import Type, Callable, Union + +A = Union[B, int] # type: ignore +B = Callable[[C], int] # type: ignore +C = Type[A] # type: ignore +[out] + +[case testSerializeRecursiveAliases3] + +from typing import Type, Callable, Union, NamedTuple + +A = Union[B, int] # type: ignore +B = Callable[[C], int] # type: ignore +class C(NamedTuple): # type: ignore + x: A +[builtins fixtures/tuple.pyi] +[out] + +[case testGenericTypeAliasesForwardAnyIncremental1] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +IntNode = Node[int, S] # type: ignore[used-before-def] +AnyNode = Node[S, T] # type: ignore[used-before-def] + +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + +def output() -> IntNode[str]: + return Node(1, 'x') +x = output() # type: IntNode + +y: IntNode +y.x = 1 +y.y = 1 +y.y = 'x' + +z = Node(1, 'x') # type: AnyNode +[out] + +[case testGenericTypeAliasesForwardAnyIncremental2] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') + +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + +def output() -> IntNode[str]: + return Node(1, 'x') +x = output() # type: IntNode + +y: IntNode +y.x = 1 +y.y = 1 +y.y = 'x' + +z = Node(1, 'x') # type: AnyNode +IntNode = Node[int, S] +AnyNode = Node[S, T] +[out] + +[case testNamedTupleForwardAsUpperBoundSerialization] +from typing import NamedTuple, TypeVar, Generic +T = TypeVar('T', bound='M') +class G(Generic[T]): + x: T + +yg: G[M] +z: int = G[M]().x.x # type: ignore[used-before-def] +z = G[M]().x[0] # type: ignore[used-before-def] +M = NamedTuple('M', [('x', int)]) +[builtins fixtures/tuple.pyi] +[out] + +[case testSelfRefNTIncremental1] + +from typing import Tuple, NamedTuple + +Node = NamedTuple('Node', [ + ('name', str), + ('children', Tuple['Node', ...]), # type: ignore + ]) +n: Node +[builtins fixtures/tuple.pyi] + +[case testSelfRefNTIncremental2] + +from typing import Tuple, NamedTuple + +A = NamedTuple('A', [ + ('x', str), + ('y', Tuple['B', ...]), # type: ignore + ]) +class B(NamedTuple): + x: A + y: int + +n: A +[builtins fixtures/tuple.pyi] + +[case testSelfRefNTIncremental3] + +from typing import NamedTuple, Tuple + +class B(NamedTuple): + x: Tuple[A, int] # type: ignore + y: int +A = NamedTuple('A', [ + ('x', str), + ('y', 'B'), + ]) +n: B +m: A +lst = [m, n] +[builtins fixtures/tuple.pyi] + +[case testSelfRefNTIncremental4] + +from typing import NamedTuple + +class B(NamedTuple): + x: A # type: ignore + y: int +class A(NamedTuple): + x: str + y: B + +n: A +[builtins fixtures/tuple.pyi] + +[case testSelfRefNTIncremental5] + +from typing import NamedTuple + +B = NamedTuple('B', [ + ('x', A), # type: ignore + ('y', int), + ]) +A = NamedTuple('A', [ + ('x', str), + ('y', 'B'), + ]) +n: A +def f(m: B) -> None: pass +[builtins fixtures/tuple.pyi] + +[case testCrashWithPartialGlobalAndCycle] +import bar + +[file foo.py] +import bar +my_global_dict = {} # type: ignore +def external_func_0() -> None: + global my_global_dict + bar.external_list + my_global_dict[12] = 0 + +[file bar.py] +import foo + +external_list = [0] + +[builtins fixtures/dict.pyi] + +[case testIncrementalCrashOnTypeWithFunction] +import a +[file a.py] +import b +[file a.py.2] +from b import x + +[file b.py] +from typing import TypeVar, Type +T = TypeVar('T') + +def tp(arg: T) -> Type[T]: + pass +def func(x: int) -> int: + pass + +x = tp(func) +[out] +[out2] + +[case testReprocessModuleEvenIfInterfaceHashDoesNotChange] +import a +import d + +[file a.py] +import b +x: b.c.A +x = b.c.A() + +[file b.py] +import c + +[file c.py] +class A: + x = 1 + +[file d.py] +import a +def f() -> None: pass + +[file a.py.2] +import b +x: b.c.A + +[file c.py.3] +class A: + x = 2 + +[file d.py.4] +import a +def f() -> None: + from c import A + a.x = [A(), a.x][0] + +[builtins fixtures/list.pyi] +[stale] +[rechecked a] +[stale2] +[rechecked2 c] +[stale3] +[rechecked3 d] +[out1] +[out2] +[out3] +[out4] + +[case testTreeShadowingViaParentPackage] +import m.semanal + +[file m/__init__.py] +pass + +[file m/nodes.py] +if False: + import m.types + import m.semanal +class Node: + line: int +class FuncBase(Node): + type: m.types.Type +class OverloadedFuncDef(FuncBase): pass + +[file m/types.py] +from m.nodes import Node +class Type(Node): pass +class Overloaded(Type): pass + +[file m/semanal.py] +from m.nodes import OverloadedFuncDef +from m.types import Overloaded + +class C: + def func(self, defn: OverloadedFuncDef): + defn.type = Overloaded() + defn.type.line = 0 + +[file m/nodes.py.2] +if False: + import m.types + import m.semanal +class Node: + line: int +class FuncBase(Node): + type: m.types.Type +class OverloadedFuncDef(FuncBase): pass +extra = 1 + +[file m/types.py.2] +from m.nodes import Node +class Type(Node): pass +class Overloaded(Type): pass +extra = 1 +[builtins fixtures/list.pyi] + +[file m/semanal.py.2] +from m.nodes import OverloadedFuncDef +from m.types import Overloaded + +class C: + def func(self, defn: OverloadedFuncDef): + defn.type = Overloaded() + defn.type.line = 0 + +extra = 1 + +[out1] +[out2] + +[case testErrorsAffectDependentsOnly] +# cmd: mypy -m m.a m.b m.c +[file m/__init__.py] +[file m/a.py] +1 + '' # Deliberate error +[file m/b.py] +import m.a # Depends on module with error +[file m/c.py] +import m # No error here +[rechecked m.a, m.b] +[out1] +tmp/m/a.py:1: error: Unsupported operand types for + ("int" and "str") +[out2] +tmp/m/a.py:1: error: Unsupported operand types for + ("int" and "str") + +[case testDisallowAnyExprIncremental] +# cmd: mypy -m main +# flags: --disallow-any-expr + +[file ns.py] +class Namespace: + def __init__(self): + self.user = 0 + +[file main.py] +import ns +user = ns.Namespace.user + +[out1] +tmp/main.py:2: error: Expression has type "Any" + +[out2] +tmp/main.py:2: error: Expression has type "Any" + +[case testIncrementalStrictOptional] +import a +1 + a.foo() +[file a.py] +def foo() -> int: return 0 +[file a.py.2] +from typing import Optional +def foo() -> Optional[int]: return 0 +[out1] +[out2] +main:2: error: Unsupported operand types for + ("int" and "None") +main:2: note: Right operand is of type "Optional[int]" + +[case testAttrsIncrementalSubclassingCached] +from a import A +import attrs +@attrs.define +class B(A): + e: str = 'e' +a = B(5, [5], 'foo') +a.a = 6 +a._b = [2] +a.c = 'yo' +a._d = 22 +a.e = 'hi' + +[file a.py] +import attrs +from typing import List, ClassVar +@attrs.define +class A: + a: int + _b: List[int] + c: str = '18' + _d: int = attrs.field(validator=None, default=18) + E = 7 + F: ClassVar[int] = 22 + +[builtins fixtures/list.pyi] +[out1] +[out2] + +[case testAttrsIncrementalSubclassingCachedConverter] +from a import A +import attrs +@attrs.define +class B(A): + pass +reveal_type(B) + +[file a.py] +def converter(s:int) -> str: + return 'hello' + +import attrs +@attrs.define +class A: + x: str = attrs.field(converter=converter) + +[builtins fixtures/list.pyi] +[out1] +main:6: note: Revealed type is "def (x: builtins.int) -> __main__.B" + +[out2] +main:6: note: Revealed type is "def (x: builtins.int) -> __main__.B" + +[case testAttrsIncrementalSubclassingCachedType] +from a import A +import attrs +@attrs.define +class B(A): + pass +reveal_type(B) + +[file a.py] +import attrs +@attrs.define +class A: + x: int + +[builtins fixtures/list.pyi] +[out1] +main:6: note: Revealed type is "def (x: builtins.int) -> __main__.B" +[out2] +main:6: note: Revealed type is "def (x: builtins.int) -> __main__.B" + +[case testAttrsIncrementalArguments] +from a import Frozen, NoInit, NoCmp +f = Frozen(5) +f.x = 6 + +g = NoInit() + +Frozen(1) < Frozen(2) +Frozen(1) <= Frozen(2) +Frozen(1) > Frozen(2) +Frozen(1) >= Frozen(2) + +NoCmp(1) < NoCmp(2) +NoCmp(1) <= NoCmp(2) +NoCmp(1) > NoCmp(2) +NoCmp(1) >= NoCmp(2) + +[file a.py] +import attrs +@attrs.frozen +class Frozen: + x: int +@attrs.define(init=False) +class NoInit: + x: int +@attrs.define(eq=False) +class NoCmp: + x: int + +[builtins fixtures/plugin_attrs.pyi] +[rechecked] +[stale] +[out1] +main:3: error: Property "x" defined in "Frozen" is read-only +main:12: error: Unsupported left operand type for < ("NoCmp") +main:13: error: Unsupported left operand type for <= ("NoCmp") +main:14: error: Unsupported left operand type for > ("NoCmp") +main:15: error: Unsupported left operand type for >= ("NoCmp") + +[out2] +main:3: error: Property "x" defined in "Frozen" is read-only +main:12: error: Unsupported left operand type for < ("NoCmp") +main:13: error: Unsupported left operand type for <= ("NoCmp") +main:14: error: Unsupported left operand type for > ("NoCmp") +main:15: error: Unsupported left operand type for >= ("NoCmp") + +[case testAttrsIncrementalDunder] +from a import A +reveal_type(A) # N: Revealed type is "def (a: builtins.int) -> a.A" +reveal_type(A.__lt__) # N: Revealed type is "def [_AT] (self: _AT`3, other: _AT`3) -> builtins.bool" +reveal_type(A.__le__) # N: Revealed type is "def [_AT] (self: _AT`4, other: _AT`4) -> builtins.bool" +reveal_type(A.__gt__) # N: Revealed type is "def [_AT] (self: _AT`5, other: _AT`5) -> builtins.bool" +reveal_type(A.__ge__) # N: Revealed type is "def [_AT] (self: _AT`6, other: _AT`6) -> builtins.bool" + +A(1) < A(2) +A(1) <= A(2) +A(1) > A(2) +A(1) >= A(2) +A(1) == A(2) +A(1) != A(2) + +A(1) < 1 # E: Unsupported operand types for < ("A" and "int") +A(1) <= 1 # E: Unsupported operand types for <= ("A" and "int") +A(1) > 1 # E: Unsupported operand types for > ("A" and "int") +A(1) >= 1 # E: Unsupported operand types for >= ("A" and "int") +A(1) == 1 +A(1) != 1 + +1 < A(1) # E: Unsupported operand types for > ("A" and "int") +1 <= A(1) # E: Unsupported operand types for >= ("A" and "int") +1 > A(1) # E: Unsupported operand types for < ("A" and "int") +1 >= A(1) # E: Unsupported operand types for <= ("A" and "int") +1 == A(1) +1 != A(1) + +[file a.py] +from attr import attrib, attrs +@attrs(auto_attribs=True) +class A: + a: int + +[builtins fixtures/plugin_attrs.pyi] +[rechecked] +[stale] +[out2] +main:2: note: Revealed type is "def (a: builtins.int) -> a.A" +main:3: note: Revealed type is "def [_AT] (self: _AT`1, other: _AT`1) -> builtins.bool" +main:4: note: Revealed type is "def [_AT] (self: _AT`2, other: _AT`2) -> builtins.bool" +main:5: note: Revealed type is "def [_AT] (self: _AT`3, other: _AT`3) -> builtins.bool" +main:6: note: Revealed type is "def [_AT] (self: _AT`4, other: _AT`4) -> builtins.bool" +main:15: error: Unsupported operand types for < ("A" and "int") +main:16: error: Unsupported operand types for <= ("A" and "int") +main:17: error: Unsupported operand types for > ("A" and "int") +main:18: error: Unsupported operand types for >= ("A" and "int") +main:22: error: Unsupported operand types for > ("A" and "int") +main:23: error: Unsupported operand types for >= ("A" and "int") +main:24: error: Unsupported operand types for < ("A" and "int") +main:25: error: Unsupported operand types for <= ("A" and "int") + +[case testAttrsIncrementalSubclassModified] +from b import B +B(5, 'foo') + +[file a.py] +import attrs +@attrs.define +class A: + x: int + +[file b.py] +import attrs +from a import A +@attrs.define +class B(A): + y: str + +[file b.py.2] +import attrs +from a import A +@attrs.define +class B(A): + y: int + +[builtins fixtures/list.pyi] +[out1] +[out2] +main:2: error: Argument 2 to "B" has incompatible type "str"; expected "int" +[rechecked b] + +[case testAttrsIncrementalSubclassModifiedErrorFirst] +from b import B +B(5, 'foo') +[file a.py] +import attrs +@attrs.define +class A: + x: int + +[file b.py] +import attrs +from a import A +@attrs.define +class B(A): + y: int + +[file b.py.2] +import attrs +from a import A +@attrs.define +class B(A): + y: str + +[builtins fixtures/list.pyi] +[out1] +main:2: error: Argument 2 to "B" has incompatible type "str"; expected "int" + +[out2] +[rechecked b] + +[case testAttrsIncrementalThreeFiles] +from c import C +C(5, 'foo', True) + +[file a.py] +import attrs +@attrs.define +class A: + a: int + +[file b.py] +import attrs +@attrs.define +class B: + b: str + +[file c.py] +from a import A +from b import B +import attrs +@attrs.define +class C(A, B): + c: bool + +[builtins fixtures/list.pyi] +[out1] +[out2] + +[case testAttrsIncrementalConverterInSubmodule] +from a.a import A +reveal_type(A) +[file a/__init__.py] +[file a/a.py] +from typing import Optional +def converter(s:Optional[int]) -> int: + ... + +import attrs +@attrs.define +class A: + x: int = attrs.field(converter=converter) + +[builtins fixtures/list.pyi] +[out1] +main:2: note: Revealed type is "def (x: Union[builtins.int, None]) -> a.a.A" +[out2] +main:2: note: Revealed type is "def (x: Union[builtins.int, None]) -> a.a.A" + +[case testAttrsIncrementalConverterManyStyles] +import a +[file a.py] +from base import Base +Base(1, 'str', True) +Base(None, None, None) + +from subclass import A, B +A(1, 'str', True) +A(None, None, None) +B(1, 'str', True, 1, 'str', True) +B(None, None, None, None, None, None) + +from submodule.base import SubBase +SubBase(1, 'str', True) +SubBase(None, None, None) + +from submodule.subclass import AA, BB +AA(1, 'str', True) +AA(None, None, None) +BB(1, 'str', True, 1, 'str', True) +BB(None, None, None, None, None, None) + +from submodule.subsubclass import SubAA, SubBB +SubAA(1, 'str', True) +SubAA(None, None, None) +SubBB(1, 'str', True, 1, 'str', True) +SubBB(None, None, None, None, None, None) + +[file a.py.2] +# Now with errors. +from base import Base +Base(1, 1, True) + +from subclass import A, B +A(1, 1, True) +B(1, 'str', True, 1, 1, True) + +from submodule.base import SubBase +SubBase(1, 1, True) + +from submodule.subclass import AA, BB +AA(1, 1, True) +BB(1, 'str', True, 1, 1, True) + +from submodule.subsubclass import SubAA, SubBB +SubAA(1, 1, True) +SubBB(1, 'str', True, 1, 1, True) + +[file foo.py] +from typing import Optional +def maybe_int(x: Optional[int]) -> int: + ... +[file bar.py] +from typing import Optional +def maybe_bool(x: Optional[bool]) -> bool: + ... +[file base.py] +from typing import Optional +import attrs +import bar +from foo import maybe_int +def maybe_str(x: Optional[str]) -> str: + ... +@attrs.define +class Base: + x: int = attrs.field(converter=maybe_int) + y: str = attrs.field(converter=maybe_str) + z: bool = attrs.field(converter=bar.maybe_bool) +[file subclass.py] +from typing import Optional +import attrs +from base import Base +@attrs.define +class A(Base): pass + +import bar +from foo import maybe_int +def maybe_str(x: Optional[str]) -> str: + ... +@attrs.define +class B(Base): + xx: int = attrs.field(converter=maybe_int) + yy: str = attrs.field(converter=maybe_str) + zz: bool = attrs.field(converter=bar.maybe_bool) + +[file submodule/__init__.py] +[file submodule/base.py] +from typing import Optional +import attrs +import bar +from foo import maybe_int +def maybe_str(x: Optional[str]) -> str: + ... +@attrs.define +class SubBase: + x: int = attrs.field(converter=maybe_int) + y: str = attrs.field(converter=maybe_str) + z: bool = attrs.field(converter=bar.maybe_bool) + +[file submodule/subclass.py] +from typing import Optional +import attrs +from base import Base +@attrs.define +class AA(Base): pass + +import bar +from foo import maybe_int +def maybe_str(x: Optional[str]) -> str: + ... +@attrs.define +class BB(Base): + xx: int = attrs.field(converter=maybe_int) + yy: str = attrs.field(converter=maybe_str) + zz: bool = attrs.field(converter=bar.maybe_bool) + +[file submodule/subsubclass.py] +from typing import Optional +import attrs +from .base import SubBase +@attrs.define +class SubAA(SubBase): pass + +import bar +from foo import maybe_int +def maybe_str(x: Optional[str]) -> str: + ... +@attrs.define +class SubBB(SubBase): + xx: int = attrs.field(converter=maybe_int) + yy: str = attrs.field(converter=maybe_str) + zz: bool = attrs.field(converter=bar.maybe_bool) +[builtins fixtures/list.pyi] +[out1] +[out2] +tmp/a.py:3: error: Argument 2 to "Base" has incompatible type "int"; expected "Optional[str]" +tmp/a.py:6: error: Argument 2 to "A" has incompatible type "int"; expected "Optional[str]" +tmp/a.py:7: error: Argument 5 to "B" has incompatible type "int"; expected "Optional[str]" +tmp/a.py:10: error: Argument 2 to "SubBase" has incompatible type "int"; expected "Optional[str]" +tmp/a.py:13: error: Argument 2 to "AA" has incompatible type "int"; expected "Optional[str]" +tmp/a.py:14: error: Argument 5 to "BB" has incompatible type "int"; expected "Optional[str]" +tmp/a.py:17: error: Argument 2 to "SubAA" has incompatible type "int"; expected "Optional[str]" +tmp/a.py:18: error: Argument 5 to "SubBB" has incompatible type "int"; expected "Optional[str]" + +[case testAttrsIncrementalConverterInFunction] +import attrs +def foo() -> None: + def foo(x: str) -> int: + ... + @attrs.define + class A: + x: int = attrs.field(converter=foo) + reveal_type(A) +[builtins fixtures/list.pyi] +[out1] +main:8: note: Revealed type is "def (x: builtins.str) -> __main__.A@6" +[out2] +main:8: note: Revealed type is "def (x: builtins.str) -> __main__.A@6" + +-- FIXME: new analyzer busted +[case testAttrsIncrementalConverterInSubmoduleForwardRef-skip] + +from a.a import A +reveal_type(A) +[file a/__init__.py] +[file a/a.py] +from typing import List +def converter(s:F) -> int: + ... + +import attrs +@attrs.define +class A: + x: int = attrs.field(converter=converter) + +F = List[int] + +[builtins fixtures/list.pyi] +[out1] +main:3: note: Revealed type is "def (x: builtins.list[builtins.int]) -> a.a.A" +[out2] +main:3: note: Revealed type is "def (x: builtins.list[builtins.int]) -> a.a.A" + +-- FIXME: new analyzer busted +[case testAttrsIncrementalConverterType-skip] + +from a import C +import attrs +o = C("1", "2", "3", "4") +o = C(1, 2, "3", 4) +reveal_type(C) +@attrs.define +class D(C): + x: str +reveal_type(D) +[file a.py] +from typing import overload +import attrs +@attrs.define +class A: + x: str +@overload +def parse(x: int) -> int: + ... +@overload +def parse(x: str, y: str = '') -> int: + ... +def parse(x, y): ... +@attrs.define +class C: + a: complex = attrs.field(converter=complex) + b: int = attrs.field(converter=int) + c: A = attrs.field(converter=A) + d: int = attrs.field(converter=parse) +[builtins fixtures/plugin_attrs.pyi] +[out1] +main:6: note: Revealed type is "def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str]) -> a.C" +main:10: note: Revealed type is "def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str], x: builtins.str) -> __main__.D" +[out2] +main:6: note: Revealed type is "def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str]) -> a.C" +main:10: note: Revealed type is "def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str], x: builtins.str) -> __main__.D" + +[case testAttrsIncrementalThreeRuns] +from a import A +A(5) + +[file a.py] +import attrs +@attrs.define +class A: + a: int + +[file a.py.2] +import attrs +@attrs.define +class A: + a: str + +[file a.py.3] +import attrs +@attrs.define +class A: + a: int = 6 + +[builtins fixtures/list.pyi] +[out1] +[out2] +main:2: error: Argument 1 to "A" has incompatible type "int"; expected "str" +[out3] + +[case testDeletedDepLineNumber] +# The import is not on line 1 and that data should be preserved +import a +[file a.py] +[delete a.py.2] +[out1] + +[out2] +main:2: error: Cannot find implementation or library stub for module named "a" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testIncrementalInheritanceAddAnnotation] +import a +[file a.py] +import b +def foo() -> None: + 1 + b.Bar().get() +[file b.py] +from c import Baz +class Bar(Baz): pass + +[file c.py] +class Baz: + def get(self): + return 1 +[file c.py.2] +from typing import Optional +class Baz: + def get(self) -> Optional[int]: + return 1 +[out] +[out2] +tmp/a.py:3: error: Unsupported operand types for + ("int" and "None") +tmp/a.py:3: note: Right operand is of type "Optional[int]" + +[case testIncrementalMetaclassUpdate] +import a +[file a.py] +from b import B +B.x + +[file b.py] +import c +class B(metaclass=c.M): pass + +[file c.py] +class M(type): + x: int + +[file c.py.2] +class M(type): + y: int +[out] +[out2] +tmp/a.py:2: error: "Type[B]" has no attribute "x" + +[case testIncrementalLotsOfInheritance] +import a +[file a.py] +from b import B +from d import D +def take(d: D) -> None: pass +def foo() -> None: + take(B()) +[file b.py] +from c import C +class B(C): pass + +[file c.py] +from d import D +class C(D): pass + +[file c.py.2] +from d import D +class C: pass + +[file d.py] +class D: pass +[out] +[out2] +tmp/a.py:5: error: Argument 1 to "take" has incompatible type "B"; expected "D" + +[case testIncrementalInheritanceProperty] +import a +[file a.py] +import b +def foo() -> None: + 1 + b.Bar().x +[file b.py] +from c import Baz +class Bar(Baz): pass + +[file c.py] +class Baz: + def __init__(self) -> None: + self.x = 12 # type: int +[file c.py.2] +class Baz: + def __init__(self) -> None: + self.x = 'lol' # type: str +[out] +[out2] +tmp/a.py:3: error: Unsupported operand types for + ("int" and "str") + +[case testIncrementalWithIgnoresTwice] +import a +[file a.py] +import b +import foo # type: ignore +[file b.py] +x = 1 +[file b.py.2] +x = 'hi' +[file b.py.3] +x = 1 +[builtins fixtures/module.pyi] +[out] +[out2] +[out3] + +[case testIgnoredImport2] +import x +[file y.py] +import xyz # type: ignore +B = 0 +from x import A +[file x.py] +A = 0 +from y import B +[file x.py.2] +A = 1 +from y import B +[file x.py.3] +A = 2 +from y import B +[out] +[out2] +[out3] + +[case testDeletionOfSubmoduleTriggersImportFrom2] +from p.q import f +f() +[file p/__init__.py] +[file p/q.py] +def f() -> None: pass +[delete p/q.py.2] +[file p/q.py.3] +def f(x: int) -> None: pass +[out] +[out2] +main:1: error: Cannot find implementation or library stub for module named "p.q" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +[out3] +main:2: error: Missing positional argument "x" in call to "f" + +[case testDeleteIndirectDependency] +import b +b.x.foo() +[file b.py] +import c +x = c.Foo() +[file c.py] +class Foo: + def foo(self) -> None: pass +[delete c.py.2] +[file b.py.2] +class Foo: + def foo(self) -> None: pass +x = Foo() +[out] +[out2] + +[case testImportReExportInCycle] +from m import One +[file m/__init__.py] +from .one import One +from .two import Two +[file m/one.py] +class One: + pass +[file m/two.py] +import m +class Two: + pass +[file m/one.py.2] +class One: + name: str +[file m/two.py.2] +import m +reveal_type(m.One.name) +class Two: + pass +[out2] +tmp/m/two.py:2: note: Revealed type is "builtins.str" + +[case testImportUnusedIgnore1] +# flags: --warn-unused-ignores +import a +[file a.py] +import b +import foo # type: ignore +[file b.py] +x = 1 +[file b.py.2] +x = '2' + +[case testImportUnusedIgnore2] +# flags: --warn-unused-ignores +import a +[file a.py] +import b +import c # type: ignore +[file b.py] +x = 1 +[file b.py.2] +x = 'hi' +[file c.py.3] +pass +[out] +[out2] +[out3] +tmp/a.py:2: error: Unused "type: ignore" comment + +-- Test that a non cache_fine_grained run can use a fine-grained cache +[case testRegularUsesFgCache] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +x = 0 +[file mypy.ini] +\[mypy] +cache_fine_grained = True +local_partial_types = True +[file mypy.ini.2] +\[mypy] +cache_fine_grained = False +local_partial_types = True +-- Nothing should get rechecked +[rechecked] +[stale] + +[case testFgCacheNeedsFgCache] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +x = 0 +[file mypy.ini] +\[mypy] +cache_fine_grained = False +[file mypy.ini.2] +\[mypy] +cache_fine_grained = True +[rechecked _typeshed, a, builtins, typing] +[stale _typeshed, a, builtins, typing] +[builtins fixtures/tuple.pyi] + +[case testIncrementalPackageNameOverload] +# cmd: mypy -m main a +# flags: --follow-imports=skip +[file main.py] +from a import x +x.foo() +[file a/__init__.py] +pass +[file a/__init__.py.2] +x = 10 +[file a/x.py] +def foo() -> None: + pass +[out] +[out2] +tmp/main.py:2: error: "int" has no attribute "foo" + +[case testIncrementalFineGrainedCacheError1] +# flags: --cache-fine-grained --no-sqlite-cache +import a +[file a.py] +[file b.py] +x = 0 +[file a.py.2] +from b import x +1 + 'lol' +[out] +[out2] +tmp/a.py:2: error: Unsupported operand types for + ("int" and "str") + +[case testIncrementalBustedFineGrainedCache1] +# flags: --cache-fine-grained --no-sqlite-cache +import a +import b +[file a.py] +[file b.py] +-- This is a heinous hack, but we simulate having a invalid cache by clobbering +-- the proto deps file with something with mtime mismatches. +[file ../.mypy_cache/3.8/@deps.meta.json.2] +{"snapshot": {"__main__": "a7c958b001a45bd6a2a320f4e53c4c16", "a": "d41d8cd98f00b204e9800998ecf8427e", "b": "d41d8cd98f00b204e9800998ecf8427e", "builtins": "c532c89da517a4b779bcf7a964478d67"}, "deps_meta": {"@root": {"path": "@root.deps.json", "mtime": 0}, "__main__": {"path": "__main__.deps.json", "mtime": 0}, "a": {"path": "a.deps.json", "mtime": 0}, "b": {"path": "b.deps.json", "mtime": 0}, "builtins": {"path": "builtins.deps.json", "mtime": 0}}} +[file ../.mypy_cache/.gitignore] +# Another hack to not trigger a .gitignore creation failure "false positive" +[file ../.mypy_cache/CACHEDIR.TAG] +Signature: 8a477f597d28d172789f06886806bc55 +# Another another hack to not trigger a CACHEDIR.TAG creation failure "false positive" +[file b.py.2] +# uh +-- Every file should get reloaded, since the cache was invalidated +[stale _typeshed, a, b, builtins, typing] +[rechecked _typeshed, a, b, builtins, typing] +[builtins fixtures/tuple.pyi] + +[case testIncrementalBustedFineGrainedCache2] +# flags2: --cache-fine-grained +import a +import b +[file a.py] +[file b.py] +[file b.py.2] +# uh +-- Every file should get reloaded, since the settings changed +[stale _typeshed, a, b, builtins, typing] +[rechecked _typeshed, a, b, builtins, typing] +[builtins fixtures/tuple.pyi] + +[case testIncrementalBustedFineGrainedCache3] +# flags: --cache-fine-grained --no-sqlite-cache +import a +import b +[file a.py] +[file b.py] +-- This is a heinous hack, but we simulate having a invalid cache by deleting +-- the proto deps file. +[delete ../.mypy_cache/3.8/@deps.meta.json.2] +[file b.py.2] +# uh +-- Every file should get reloaded, since the cache was invalidated +[stale _typeshed, a, b, builtins, typing] +[rechecked _typeshed, a, b, builtins, typing] +[builtins fixtures/tuple.pyi] + +[case testIncrementalWorkingFineGrainedCache] +# flags: --cache-fine-grained +# flags2: --cache-fine-grained +import a +import b +[file a.py] +[file b.py] +[file b.py.2] +# uh +-- b gets rechecked because it changed, but nothing is stale +-- since the interface did not change +[stale] +[rechecked b] + +[case testIncrementalDataclassesSubclassingCached] +from a import A +from dataclasses import dataclass + +@dataclass +class B(A): + e: str = 'e' + +a = B(5, [5], 'foo') +a.a = 6 +a._b = [2] +a.c = 'yo' +a._d = 22 +a.e = 'hi' + +[file a.py] +from dataclasses import dataclass, field +from typing import ClassVar, List + +@dataclass +class A: + a: int + _b: List[int] + c: str = '18' + _d: int = field(default=False) + E = 7 + F: ClassVar[int] = 22 + +[builtins fixtures/dataclasses.pyi] +[out1] +[out2] + +[case testIncrementalDataclassesSubclassingCachedType] +import b + +[file b.py] +from a import A +from dataclasses import dataclass + +@dataclass +class B(A): + pass + +[file b.py.2] +from a import A +from dataclasses import dataclass + +@dataclass +class B(A): + pass + +reveal_type(B) + +[file a.py] +from dataclasses import dataclass + +@dataclass +class A: + x: int + +[builtins fixtures/dataclasses.pyi] +[out1] +[out2] +tmp/b.py:8: note: Revealed type is "def (x: builtins.int) -> b.B" + +[case testIncrementalDataclassesArguments] +import b + +[file b.py] +from a import Frozen, NoInit, NoCmp + +[file b.py.2] +from a import Frozen, NoInit, NoCmp + +f = Frozen(5) +f.x = 6 + +g = NoInit() + +Frozen(1) < Frozen(2) +Frozen(1) <= Frozen(2) +Frozen(1) > Frozen(2) +Frozen(1) >= Frozen(2) + +NoCmp(1) < NoCmp(2) +NoCmp(1) <= NoCmp(2) +NoCmp(1) > NoCmp(2) +NoCmp(1) >= NoCmp(2) + +[file a.py] +from dataclasses import dataclass + +@dataclass(frozen=True, order=True) +class Frozen: + x: int + +@dataclass(init=False) +class NoInit: + x: int + +@dataclass(order=False) +class NoCmp: + x: int + +[builtins fixtures/dataclasses.pyi] +[out1] +[out2] +tmp/b.py:4: error: Property "x" defined in "Frozen" is read-only +tmp/b.py:13: error: Unsupported left operand type for < ("NoCmp") +tmp/b.py:14: error: Unsupported left operand type for <= ("NoCmp") +tmp/b.py:15: error: Unsupported left operand type for > ("NoCmp") +tmp/b.py:16: error: Unsupported left operand type for >= ("NoCmp") + +[case testIncrementalDataclassesDunder] +import b + +[file b.py] +from a import A + +[file b.py.2] +from a import A + +reveal_type(A) +reveal_type(A.__eq__) +reveal_type(A.__ne__) +reveal_type(A.__lt__) +reveal_type(A.__le__) +reveal_type(A.__gt__) +reveal_type(A.__ge__) + +A(1) < A(2) +A(1) <= A(2) +A(1) > A(2) +A(1) >= A(2) +A(1) == A(2) +A(1) != A(2) + +A(1) < 1 +A(1) <= 1 +A(1) > 1 +A(1) >= 1 +A(1) == 1 +A(1) != 1 + +1 < A(1) +1 <= A(1) +1 > A(1) +1 >= A(1) +1 == A(1) +1 != A(1) + +[file a.py] +from dataclasses import dataclass + +@dataclass(order=True) +class A: + a: int + +[builtins fixtures/dataclasses.pyi] +[out1] +[out2] +tmp/b.py:3: note: Revealed type is "def (a: builtins.int) -> a.A" +tmp/b.py:4: note: Revealed type is "def (builtins.object, builtins.object) -> builtins.bool" +tmp/b.py:5: note: Revealed type is "def (builtins.object, builtins.object) -> builtins.bool" +tmp/b.py:6: note: Revealed type is "def [_DT] (self: _DT`1, other: _DT`1) -> builtins.bool" +tmp/b.py:7: note: Revealed type is "def [_DT] (self: _DT`2, other: _DT`2) -> builtins.bool" +tmp/b.py:8: note: Revealed type is "def [_DT] (self: _DT`3, other: _DT`3) -> builtins.bool" +tmp/b.py:9: note: Revealed type is "def [_DT] (self: _DT`4, other: _DT`4) -> builtins.bool" +tmp/b.py:18: error: Unsupported operand types for < ("A" and "int") +tmp/b.py:19: error: Unsupported operand types for <= ("A" and "int") +tmp/b.py:20: error: Unsupported operand types for > ("A" and "int") +tmp/b.py:21: error: Unsupported operand types for >= ("A" and "int") +tmp/b.py:25: error: Unsupported operand types for > ("A" and "int") +tmp/b.py:26: error: Unsupported operand types for >= ("A" and "int") +tmp/b.py:27: error: Unsupported operand types for < ("A" and "int") +tmp/b.py:28: error: Unsupported operand types for <= ("A" and "int") + +[case testIncrementalDataclassesSubclassModified] +from b import B +B(5, 'foo') + +[file a.py] +from dataclasses import dataclass + +@dataclass +class A: + x: int + +[file b.py] +from a import A +from dataclasses import dataclass + +@dataclass +class B(A): + y: str + +[file b.py.2] +from a import A +from dataclasses import dataclass + +@dataclass +class B(A): + y: int + +[builtins fixtures/dataclasses.pyi] +[out1] +[out2] +main:2: error: Argument 2 to "B" has incompatible type "str"; expected "int" +[rechecked b] + +[case testIncrementalDataclassesSubclassModifiedErrorFirst] +from b import B +B(5, 'foo') + +[file a.py] +from dataclasses import dataclass + +@dataclass +class A: + x: int + +[file b.py] +from a import A +from dataclasses import dataclass + +@dataclass +class B(A): + y: int + +[file b.py.2] +from a import A +from dataclasses import dataclass + +@dataclass +class B(A): + y: str + +[builtins fixtures/dataclasses.pyi] +[out1] +main:2: error: Argument 2 to "B" has incompatible type "str"; expected "int" + +[out2] +[rechecked b] + +[case testIncrementalDataclassesThreeFiles] +from c import C +C('foo', 5, True) + +[file a.py] +from dataclasses import dataclass + +@dataclass +class A: + a: int + +[file b.py] +from dataclasses import dataclass + +@dataclass +class B: + b: str + +[file b.py.2] +from dataclasses import dataclass + +@dataclass +class B: + b: str + c: str + +[file c.py] +from a import A +from b import B +from dataclasses import dataclass + +@dataclass +class C(A, B): + c: bool + +[builtins fixtures/dataclasses.pyi] +[out1] +[out2] +tmp/c.py:7: error: Incompatible types in assignment (expression has type "bool", base class "B" defined the type as "str") +main:2: error: Argument 2 to "C" has incompatible type "int"; expected "bool" + +[case testIncrementalDataclassesThreeRuns] +from a import A +A(5) + +[file a.py] +from dataclasses import dataclass + +@dataclass +class A: + a: int + +[file a.py.2] +from dataclasses import dataclass + +@dataclass +class A: + a: str + +[file a.py.3] +from dataclasses import dataclass + +@dataclass +class A: + a: int = 6 + +[builtins fixtures/dataclasses.pyi] +[out1] +[out2] +main:2: error: Argument 1 to "A" has incompatible type "int"; expected "str" +[out3] + +[case testParentPatchingMess] +# flags: --ignore-missing-imports --follow-imports=skip +# cmd: mypy -m d d.k d.k.a d.k.v t +[file d/__init__.py] +[file d/k/__init__.py] +from d.k.a import x +[file d/k/a.py] +x = 10 +[file d/k/v.py] +from d.k.e import x + +[file t.py] +from d import k +[file t.py.2] +from d import k +# dummy change + +[case testCachedBadProtocolNote] +import b +[file a.py] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +[file b.py] +from typing import Iterable +from a import Point +p: Point +it: Iterable[int] = p +[file b.py.2] +from typing import Iterable +from a import Point +p: Point +it: Iterable[int] = p # change +[typing fixtures/typing-medium.pyi] +[builtins fixtures/dict.pyi] +[out] +tmp/b.py:4: error: Incompatible types in assignment (expression has type "Point", variable has type "Iterable[int]") +tmp/b.py:4: note: Following member(s) of "Point" have conflicts: +tmp/b.py:4: note: Expected: +tmp/b.py:4: note: def __iter__(self) -> Iterator[int] +tmp/b.py:4: note: Got: +tmp/b.py:4: note: def __iter__(self) -> Iterator[str] +[out2] +tmp/b.py:4: error: Incompatible types in assignment (expression has type "Point", variable has type "Iterable[int]") +tmp/b.py:4: note: Following member(s) of "Point" have conflicts: +tmp/b.py:4: note: Expected: +tmp/b.py:4: note: def __iter__(self) -> Iterator[int] +tmp/b.py:4: note: Got: +tmp/b.py:4: note: def __iter__(self) -> Iterator[str] + +[case testIndirectDepsAlwaysPatched-writescache] +# flags: --no-incremental +# flags2: --incremental +from b import C +def f() -> None: + x: int = C().x +[file b.py] +from c import C +[file c.pyi] +class C: + x: int +[file c.pyi.2] +class C: + x: str +[out] +[out2] +main:5: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testBazelFlagIgnoresFileChanges-skip] +-- This test fails on windows, when the mypy source in on a different drive than +-- the run-directory. In this case os.path.relpath(...) fails with an exception +-- Since the initial run wrote a cache file, the second run ignores the source +# flags: --bazel +from a import f +f() +[file a.py] +def f(): pass +[file a.py.2] +[out] +[out2] + +[case testModuleGetattrInitIncremental] +import c +[file c.py] +import a.b +x = a.b.f() +[file c.py.2] +import a.b +x = a.b.f() +# touch +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] +[out2] + +[case testModuleGetattrInitIncremental2] +import c +[file c.py] +import a.b.c +[file c.py.2] +import a.b.c +# touch +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[file a/b.pyi] +# empty +[builtins fixtures/module.pyi] +[out] +tmp/c.py:1: error: Cannot find implementation or library stub for module named "a.b.c" +tmp/c.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +[out2] +tmp/c.py:1: error: Cannot find implementation or library stub for module named "a.b.c" +tmp/c.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testModuleGetattrIncrementalSerializeVarFlag] +import main + +[file main.py] +from b import A, f +f() + +[file main.py.3] +from b import A, f # foo +f() + +[file b.py] +from c import A +def f() -> A: ... + +[file b.py.2] +from c import A # foo +def f() -> A: ... + +[file c.py] +from d import A + +[file d.pyi] +def __getattr__(n): ... +[out1] +[out2] +[out3] + +[case testAddedMissingStubs] +# flags: --ignore-missing-imports +from missing import f +f(int()) +[file missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsPackage] +# flags: --ignore-missing-imports +import package.missing +package.missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsPackageFrom] +# flags: --ignore-missing-imports +from package import missing +missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsPackagePartial] +# flags: --ignore-missing-imports +import package.missing +package.missing.f(int()) +[file package/__init__.pyi] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsPackagePartialGetAttr] +import package.missing +package.missing.f(int()) +[file package/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsIgnore] +from missing import f # type: ignore +f(int()) +[file missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsIgnorePackage] +import package.missing # type: ignore +package.missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsIgnorePackageFrom] +from package import missing # type: ignore +missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddedMissingStubsIgnorePackagePartial] +import package.missing # type: ignore +package.missing.f(int()) +[file package/__init__.pyi] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +[out2] +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +-- Test cases for final qualifier + +[case testFinalAddFinalVarAssign] +import mod +from a import D +from mod import x + +mod.x = 2 # This an all below are errors. +x = 2 +d: D +d.y = 2 +d.z = 2 +D.y = 2 +[file a.py] +import mod + +class D(mod.C): + pass +[file mod.py] +x = 1 +class C: + y = 1 + def __init__(self) -> None: + self.z = 1 + +[file mod.py.2] +from typing import Final + +x: Final = 1 +class C: + y: Final = 1 + def __init__(self) -> None: + self.z: Final = 1 +[out] +[out2] +main:5: error: Cannot assign to final name "x" +main:6: error: Cannot assign to final name "x" +main:8: error: Cannot assign to final attribute "y" +main:9: error: Cannot assign to final attribute "z" +main:10: error: Cannot assign to final attribute "y" + +[case testFinalAddFinalVarOverride] +from mod import C + +class D(C): + x = 2 + def __init__(self) -> None: + self.y = 2 +class E(C): + y = 2 + def __init__(self) -> None: + self.x = 2 + +[file mod.py] +class C: + x = 1 + def __init__(self) -> None: + self.y = 1 + +[file mod.py.2] +from typing import Final + +class C: + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 +[out] +[out2] +main:4: error: Cannot assign to final name "x" +main:6: error: Cannot assign to final attribute "y" +main:8: error: Cannot assign to final name "y" +main:10: error: Cannot assign to final attribute "x" + +[case testFinalAddFinalMethodOverride] +from mod import C + +class D(C): + def meth(self) -> int: ... + +[file mod.py] +class C: + def meth(self) -> int: ... + +[file mod.py.2] +from typing import final + +class C: + @final + def meth(self) -> int: ... +[out] +[out2] +main:4: error: Cannot override final attribute "meth" (previously declared in base class "C") + +-- These tests should just not crash +[case testOverrideByBadVar] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class Slow: + pass + +s: Slow +from cext import Slow # type: ignore +[out] +[out2] + +[case testOverrideByBadVarAlias] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class Slow: + pass + +A = Slow +from cext import Slow # type: ignore +[out] +[out2] + +[case testOverrideByBadVarClass] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class C: + class Slow: + pass + s: Slow + from cext import Slow # type: ignore +[out] +[out2] + +[case testOverrideByBadVarClassAlias] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class C: + class Slow: + pass + A = Slow + from cext import Slow # type: ignore +[out] +[out2] + +[case testOverrideByBadVarExisting] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class Slow: + pass + +s: Slow +from cext import Slow # type: ignore +[file cext.py] +Slow = 1 +[out] +[out2] + +[case testOverrideByBadVarAliasExisting] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class Slow: + pass + +A = Slow +from cext import Slow # type: ignore +[file cext.py] +Slow = 1 +[out] +[out2] + +[case testOverrideByBadFunction] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class C: + class Slow: + pass + + s: Slow + def Slow() -> None: ... # type: ignore +[out] +[out2] + +[case testOverrideByBadVarLocal] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +def outer() -> None: + class Slow: + pass + + s: Slow + from cext import Slow # type: ignore +[out] +[out2] + +[case testRecursiveAliasImported] +import a + +[file a.py] +import lib +x: int + +[file a.py.2] +import lib +x: lib.A +reveal_type(x) + +[file lib.pyi] +from typing import List +MYPY = False +if MYPY: # Force processing order + from other import B +A = List[B] # type: ignore + +[file other.pyi] +from typing import List +from lib import A +B = List[A] + +[builtins fixtures/list.pyi] +[out] +[out2] +tmp/a.py:3: note: Revealed type is "builtins.list[builtins.list[...]]" + +[case testRecursiveNamedTupleTypedDict] +import a +[file a.py] +import lib +x: int +[file a.py.2] +import lib +x: lib.A +reveal_type(x.x['x']) +[file lib.pyi] +from typing import NamedTuple +from other import B +A = NamedTuple('A', [('x', B)]) +[file other.pyi] +from mypy_extensions import TypedDict +from lib import A +B = TypedDict('B', {'x': A}) +[builtins fixtures/dict.pyi] +[out] +[out2] +tmp/a.py:3: note: Revealed type is "Tuple[TypedDict('other.B', {'x': Tuple[..., fallback=lib.A]}), fallback=lib.A]" + +[case testFollowImportSkipNotInvalidatedOnPresent] +# flags: --follow-imports=skip +# cmd: mypy -m main +[file main.py] +import other +[file other.py] +x = 1 +[file other.py.2] +x = 'hi' +[stale] +[rechecked] + +[case testFollowImportSkipNotInvalidatedOnPresentPackage] +# flags: --follow-imports=skip +# cmd: mypy -m main +[file main.py] +import other +[file other/__init__.py] +x = 1 +[file other/__init__.py.2] +x = 'hi' +[stale] +[rechecked] + +[case testFollowImportSkipNotInvalidatedOnAdded] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy -m main +[file main.py] +import other +[file other.py.2] +x = 1 +[stale] +[rechecked] + +[case testFollowImportSkipInvalidatedOnAddedStub] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy -m main +[file main.py] +import other +[file other.pyi.2] +x = 1 +[stale main, other] +[rechecked main, other] + +[case testFollowImportSkipNotInvalidatedOnAddedStubOnFollowForStubs] +# flags: --follow-imports=skip --ignore-missing-imports --config-file=tmp/mypy.ini +# cmd: mypy -m main +[file main.py] +import other +[file other.pyi.2] +x = 1 +[file mypy.ini] +\[mypy] +follow_imports_for_stubs = True +[stale] +[rechecked] + +[case testAddedSkippedStubsPackageFrom] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy -m main +# cmd2: mypy -m main package package.missing +[file main.py] +from package import missing +missing.f(int()) +[file package/__init__.py] +[file package/missing.py] +def f(x: str) -> None: pass +[out] +[out2] +tmp/main.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testOverrideByIdemAlias] +# https://github.com/python/mypy/issues/6404 + +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +C = C # type: ignore +class C: # type: ignore + pass +[out] +[out2] + +[case testOverrideByIdemAliasReversed] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +class C: + pass +C = C # type: ignore +x: C +[out] +[out2] + +[case testOverrideByIdemAliasGeneric] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class C(Generic[T]): + pass +C = C[int] # type: ignore +x: C +[out] +[out2] + +[case testOverrideByIdemAliasImported] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +from other import C +C = C # type: ignore +x: C +[file other.py] +class C: + pass +[out] +[out2] + +[case testOverrideByIdemAliasImportedReversed] +import a +[file a.py] +import lib +x = 1 +[file a.py.2] +import lib +x = 2 +[file lib.py] +C = C # type: ignore +from other import C +[file other.py] +class C: + pass +[out] +[out2] + +[case testConditionalExceptionAliasOverride] +import a +[file a.py] +import lib +try: + x = 1 +except lib.Exception as e: + pass +[file a.py.2] +import lib +try: + x = 2 +except lib.Exception as e: + pass +[file lib.py] +try: + Exception = Exception +except BaseException: + class Exception(BaseException): pass # type: ignore + +try: + pass +except Exception as e: + pass +[builtins fixtures/exception.pyi] +[out] +[out2] + +[case testBadEnumLoading] +import a +[file a.py] +from b import E +x: E +y = 1 + +[file a.py.2] +from b import E +x: E +y = 2 + +[file b.py] +from typing import List +from enum import Enum + +def f() -> List[str]: ... + +E = Enum('E', f()) # type: ignore +[builtins fixtures/list.pyi] +[out] +[out2] + +[case testChangedPluginsInvalidateCache] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +from b import x +y: int = x + +[file a.py.2] +from b import x +y: int = x +touch = 1 + +[file b.py] +class C: ... +def f() -> C: ... +x = f() + +[file basic_plugin.py] +from mypy.plugin import Plugin + +class MyPlugin(Plugin): + def get_function_hook(self, fullname): + if fullname.endswith('.f'): + return my_hook + assert fullname is not None + return None + +def my_hook(ctx): + return ctx.api.named_generic_type('builtins.int', []) + +def plugin(version): + return MyPlugin + +[file basic_plugin.py.2] +from mypy.plugin import Plugin + +class MyPlugin(Plugin): + def get_function_hook(self, fullname): + if fullname.endswith('.f'): + return my_hook + assert fullname is not None + return None + +def my_hook(ctx): + return ctx.api.named_generic_type('builtins.str', []) + +def plugin(version): + return MyPlugin +[file mypy.ini] +\[mypy] +plugins=basic_plugin.py +[out] +[out2] +tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testChangedPluginsInvalidateCache2] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +from b import x +y: int = x + +[file a.py.2] +from b import x +y: int = x +touch = 1 + +[file b.py] +class C: ... +def f() -> C: ... +x = f() + +[file basic_plugin.py] +from mypy.plugin import Plugin +from version_plugin import __version__, choice + +class MyPlugin(Plugin): + def get_function_hook(self, fullname): + if fullname.endswith('.f'): + return my_hook + assert fullname is not None + return None + +def my_hook(ctx): + if choice: + return ctx.api.named_generic_type('builtins.int', []) + else: + return ctx.api.named_generic_type('builtins.str', []) + +def plugin(version): + return MyPlugin + +[file version_plugin.py] +__version__ = 0.1 +choice = True + +[file version_plugin.py.2] +__version__ = 0.2 +choice = False +[file mypy.ini] +\[mypy] +plugins=basic_plugin.py +[out] +[out2] +tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testAddedPluginsInvalidateCache] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +from b import x +y: int = x + +[file a.py.2] +from b import x +y: int = x +touch = 1 + +[file b.py] +def f() -> int: ... +x = f() + +[file basic_plugin.py] +from mypy.plugin import Plugin + +class MyPlugin(Plugin): + def get_function_hook(self, fullname): + if fullname.endswith('.f'): + return my_hook + assert fullname is not None + return None + +def my_hook(ctx): + return ctx.api.named_generic_type('builtins.str', []) + +def plugin(version): + return MyPlugin + +[file mypy.ini] +\[mypy] +python_version=3.6 +[file mypy.ini.2] +\[mypy] +python_version=3.6 +plugins=basic_plugin.py +[out] +[out2] +tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testRemovedPluginsInvalidateCache] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +from b import x +y: str = x + +[file a.py.2] +from b import x +y: str = x +touch = 1 + +[file b.py] +def f() -> int: ... +x = f() + +[file basic_plugin.py] +from mypy.plugin import Plugin + +class MyPlugin(Plugin): + def get_function_hook(self, fullname): + if fullname.endswith('.f'): + return my_hook + assert fullname is not None + return None + +def my_hook(ctx): + return ctx.api.named_generic_type('builtins.str', []) + +def plugin(version): + return MyPlugin + +[file mypy.ini] +\[mypy] +python_version=3.6 +plugins=basic_plugin.py +[file mypy.ini.2] +\[mypy] +python_version=3.6 +[out] +[out2] +tmp/a.py:2: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testPluginConfigData] +# flags: --config-file tmp/mypy.ini +import a +import b +[file a.py] +[file b.py] +[file test.json] +{"a": false, "b": false} +[file test.json.2] +{"a": true, "b": false} + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/config_data.py + +# The config change will force a to be rechecked but not b. +[rechecked a] + +[case testLiteralIncrementalTurningIntoLiteral] +import mod +reveal_type(mod.a) +[file mod.py] +from typing_extensions import Literal +a = 1 +[file mod.py.2] +from typing_extensions import Literal +a: Literal[2] = 2 +[builtins fixtures/tuple.pyi] +[out] +main:2: note: Revealed type is "builtins.int" +[out2] +main:2: note: Revealed type is "Literal[2]" + +[case testAddedSubStarImport] +# cmd: mypy -m a pack pack.mod b +# cmd2: mypy -m other +[file a.py] +from pack import * +[file pack/__init__.py] +[file pack/mod.py] +[file b.py] +import pack.mod +[file other.py] +import a +[out] +[out2] + +[case testNewAnalyzerIncrementalBrokenNamedTuple] + +import a +[file a.py] +from b import NT +x: NT +[file a.py.2] +from b import NT +x: NT +reveal_type(x) +[file b.py] +from typing import NamedTuple +NT = NamedTuple('BadName', [('x', int)]) +[builtins fixtures/tuple.pyi] +[out] +tmp/b.py:2: error: First argument to namedtuple() should be "NT", not "BadName" +[out2] +tmp/b.py:2: error: First argument to namedtuple() should be "NT", not "BadName" +tmp/a.py:3: note: Revealed type is "Tuple[builtins.int, fallback=b.NT]" + +[case testNewAnalyzerIncrementalBrokenNamedTupleNested] + +import a +[file a.py] +from b import C +x: C +[file a.py.2] +from b import C +x: C +# touch +[file b.py] +class C: ... +from collections import namedtuple +def test() -> None: + NT = namedtuple('BadName', ['x', 'y']) +[builtins fixtures/list.pyi] +[out] +tmp/b.py:4: error: First argument to namedtuple() should be "NT", not "BadName" +[out2] +tmp/b.py:4: error: First argument to namedtuple() should be "NT", not "BadName" + +[case testNewAnalyzerIncrementalMethodNamedTuple] +import a +[file a.py] +from b import C +x: C +[file a.py.2] +from b import C +x: C +reveal_type(x.h) +[file b.py] +from typing import NamedTuple +class C: + def __init__(self) -> None: + self.h: Hidden + Hidden = NamedTuple('Hidden', [('x', int)]) +[builtins fixtures/tuple.pyi] +[out] +[out2] +tmp/a.py:3: note: Revealed type is "Tuple[builtins.int, fallback=b.C.Hidden@5]" + +[case testIncrementalNodeCreatedFromGetattr] +import a +[file a.py] +from b import C +c: C +[file b.py] +from c import C +[file c.pyi] +def __getattr__(s): ... +[file a.py.2] +from b import C +c: C +reveal_type(c) +[out] +[out2] +tmp/a.py:3: note: Revealed type is "Any" + +[case testNewAnalyzerIncrementalNestedEnum] + +import a +[file a.py] +from b import C +x: C +[file a.py.2] +from b import C +x: C +# touch +[file b.py] +class C: ... +from enum import Enum + +def test() -> None: + Color = Enum('Color', 'RED BLACK') +[builtins fixtures/list.pyi] +[out] +[out2] + +[case testCannotDetermineTypeFromOtherModule] + +import aa + +[file aa.py] +import a + +[file aa.py.2] +import a # dummy + +[file a.py] +from b import Sub + +Sub().foo +Sub().foo + +[file b.py] +from typing import Any + +class desc: + def __get__(self, _: Any, __: Any = None) -> int: + return 42 + +class Base: + @property + def foo(self) -> int: ... + +class Sub(Base): + foo = desc(42) # type: ignore + +[builtins fixtures/property.pyi] +[out] +tmp/a.py:3: error: Cannot determine type of "foo" +tmp/a.py:4: error: Cannot determine type of "foo" +[out2] +tmp/a.py:3: error: Cannot determine type of "foo" +tmp/a.py:4: error: Cannot determine type of "foo" + +[case testRedefinitionClass] +import b +[file a.py] +from whatever import Foo # type: ignore + +class Foo: # type: ignore + def f(self) -> None: + pass +[file b.py] +import a +[file b.py.2] +import a # a change + +[case testIsInstanceAdHocIntersectionIncrementalNoChange] +import b +[file a.py] +class A: pass +class B: pass + +class Foo: + def __init__(self) -> None: + x: A + assert isinstance(x, B) + self.x = x +[file b.py] +from a import Foo +[file b.py.2] +from a import Foo +reveal_type(Foo().x) +[builtins fixtures/isinstance.pyi] +[out] +[out2] +tmp/b.py:2: note: Revealed type is "a." + +[case testIsInstanceAdHocIntersectionIncrementalNoChangeSameName] +import b +[file c.py] +class B: pass +[file a.py] +import c +class B: pass + +class Foo: + def __init__(self) -> None: + x: c.B + assert isinstance(x, B) + self.x = x +[file b.py] +from a import Foo +[file b.py.2] +from a import Foo +reveal_type(Foo().x) +[builtins fixtures/isinstance.pyi] +[out] +[out2] +tmp/b.py:2: note: Revealed type is "a." + + +[case testIsInstanceAdHocIntersectionIncrementalNoChangeTuple] +import b +[file a.py] +from typing import Tuple +class B: pass + +class Foo: + def __init__(self) -> None: + x: Tuple[int, ...] + assert isinstance(x, B) + self.x = x +[file b.py] +from a import Foo +[file b.py.2] +from a import Foo +reveal_type(Foo().x) +[builtins fixtures/isinstance.pyi] +[out] +[out2] +tmp/b.py:2: note: Revealed type is "a." + +[case testIsInstanceAdHocIntersectionIncrementalIsInstanceChange] +import c +[file a.py] +class A: pass +class B: pass +class C: pass + +class Foo: + def __init__(self) -> None: + x: A + assert isinstance(x, B) + self.x = x +[file a.py.2] +class A: pass +class B: pass +class C: pass + +class Foo: + def __init__(self) -> None: + x: A + assert isinstance(x, C) + self.x = x + +[file b.py] +from a import Foo +y = Foo().x + +[file c.py] +from b import y +reveal_type(y) +[builtins fixtures/isinstance.pyi] +[out] +tmp/c.py:2: note: Revealed type is "a." +[out2] +tmp/c.py:2: note: Revealed type is "a." + +[case testIsInstanceAdHocIntersectionIncrementalUnderlyingObjChang] +import c +[file a.py] +class A: pass +class B: pass +class C: pass +Extra = B +[file a.py.2] +class A: pass +class B: pass +class C: pass +Extra = C + +[file b.py] +from a import A, Extra +x: A +if isinstance(x, Extra): + y = x + +[file c.py] +from b import y +reveal_type(y) +[builtins fixtures/isinstance.pyi] +[out] +tmp/c.py:2: note: Revealed type is "b." +[out2] +tmp/c.py:2: note: Revealed type is "b." + +[case testIsInstanceAdHocIntersectionIncrementalIntersectionToUnreachable] +import c +[file a.py] +class A: + x: int +class B: + x: int +x: A +assert isinstance(x, B) +y = x + +[file a.py.2] +class A: + x: int +class B: + x: str +x: A +assert isinstance(x, B) +y = x + +[file b.py] +from a import y +z = y + +[file c.py] +from b import z +reveal_type(z) +[builtins fixtures/isinstance.pyi] +[out] +tmp/c.py:2: note: Revealed type is "a." +[out2] +tmp/b.py:2: error: Cannot determine type of "y" +tmp/c.py:2: note: Revealed type is "Any" + +[case testIsInstanceAdHocIntersectionIncrementalUnreachaableToIntersection] +import c +[file a.py] +class A: + x: int +class B: + x: str +x: A +assert isinstance(x, B) +y = x + +[file a.py.2] +class A: + x: int +class B: + x: int +x: A +assert isinstance(x, B) +y = x + +[file b.py] +from a import y +z = y + +[file c.py] +from b import z +reveal_type(z) +[builtins fixtures/isinstance.pyi] +[out] +tmp/b.py:2: error: Cannot determine type of "y" +tmp/c.py:2: note: Revealed type is "Any" +[out2] +tmp/c.py:2: note: Revealed type is "a." + +[case testStubFixupIssues] +import a +[file a.py] +import p +[file a.py.2] +import p +p.N + +[file p/__init__.pyi] +from p.util import * + +[file p/util.pyi] +from p.params import N +class Test: ... +x: N + +[file p/params.pyi] +import p.util +class N(p.util.Test): + ... +[out2] +tmp/a.py:2: error: "object" has no attribute "N" + +[case testIncrementalIndirectSkipWarnUnused] +# flags: --follow-imports=skip --warn-unused-ignores +# cmd: mypy -m main a b c1 +# cmd2: mypy -m main a b c2 + +[file main.py] +import a +a.foo.bar() + +[file a.py] +import b +foo = b.Foo() + +[file b.py] +from c1 import C +class Foo: + def bar(self) -> C: + return C() + +[file c1.py] +class C: pass + +[file b.py.2] +from c2 import C +class Foo: + def bar(self) -> C: + return C() + +[file c2.py] + +[delete c1.py.2] +[file c2.py.2] +class C: pass + +[case testIncrementalNestedNamedTuple] +import a + +[file a.py] +import b + +[file a.py.2] +import b # foo + +[file b.py] +from typing import NamedTuple + +def f() -> None: + class NT(NamedTuple): + x: int + + n: NT = NT(x=2) + +def g() -> None: + NT = NamedTuple('NT', [('y', str)]) + + n: NT = NT(y='x') + +[builtins fixtures/tuple.pyi] + +[case testIncrementalNestedTypeAlias] +import a + +[file a.py] +import b + +[file a.py.2] +import b +reveal_type(b.C().x) +reveal_type(b.D().x) + +[file b.py] +from typing import List + +class C: + def __init__(self) -> None: + Alias = List[int] + self.x = [] # type: Alias + +class D: + def __init__(self) -> None: + Alias = List[str] + self.x = [] # type: Alias + +[builtins fixtures/list.pyi] +[out2] +tmp/a.py:2: note: Revealed type is "builtins.list[builtins.int]" +tmp/a.py:3: note: Revealed type is "builtins.list[builtins.str]" + +[case testIncrementalNamespacePackage1] +# flags: --namespace-packages +import m +[file m.py] +from foo.bar import x +x + 0 +[file foo/bar.py] +x = 0 +[rechecked] +[stale] + +[case testIncrementalNamespacePackage2] +# flags: --namespace-packages +import m +[file m.py] +from foo import bar +bar.x + 0 +[file foo/bar.py] +x = 0 +[rechecked] +[stale] + +[case testExplicitReexportImportCycleWildcard] +# flags: --no-implicit-reexport +import pkg.a +[file pkg/__init__.pyi] + +[file pkg/a.pyi] +MYPY = False +if MYPY: + from pkg.b import B + +[file pkg/b.pyi] +import pkg.a +MYPY = False +if MYPY: + from pkg.c import C +class B: + pass + +[file pkg/c.pyi] +from pkg.a import * +class C: + pass +[rechecked] +[stale] + + +[case testEnumAreStillFinalAfterCache] +import a +class Ok(a.RegularEnum): + x = 1 +class NotOk(a.FinalEnum): + x = 1 +[file a.py] +from enum import Enum +class RegularEnum(Enum): + x: int +class FinalEnum(Enum): + x = 1 +[builtins fixtures/isinstance.pyi] +[out] +main:3: error: Cannot override writable attribute "x" with a final one +main:4: error: Cannot extend enum with existing members: "FinalEnum" +main:5: error: Cannot override final attribute "x" (previously declared in base class "FinalEnum") +[out2] +main:3: error: Cannot override writable attribute "x" with a final one +main:4: error: Cannot extend enum with existing members: "FinalEnum" +main:5: error: Cannot override final attribute "x" (previously declared in base class "FinalEnum") + +[case testSlotsSerialization] +import a +[file a.py] +from b import C + +class D(C): + pass +[file b.py] +class C: + __slots__ = ('x',) +[file a.py.2] +from b import C + +class D(C): + __slots__ = ('y',) + + def __init__(self) -> None: + self.x = 1 + self.y = 2 + self.z = 3 +[builtins fixtures/tuple.pyi] +[out] +[out2] +tmp/a.py:9: error: Trying to assign name "z" that is not in "__slots__" of type "a.D" + +[case testMethodAliasIncremental] +import b +[file a.py] +class A: + def f(self) -> None: pass + g = f + +[file b.py] +from a import A +A().g() +[file b.py.2] +# trivial change +from a import A +A().g() +[out] +[out2] + +[case testIncrementalWithDifferentKindsOfNestedTypesWithinMethod] + +import a + +[file a.py] +import b + +[file a.py.2] +import b +b.xyz + +[file b.py] +from typing import NamedTuple, NewType +from typing_extensions import TypedDict, TypeAlias +from enum import Enum +from dataclasses import dataclass + +class C: + def f(self) -> None: + class C: + c: int + class NT1(NamedTuple): + c: int + NT2 = NamedTuple("NT2", [("c", int)]) + class NT3(NT1): + pass + class TD(TypedDict): + c: int + TD2 = TypedDict("TD2", {"c": int}) + class E(Enum): + X = 1 + @dataclass + class DC: + c: int + Alias: TypeAlias = NT1 + N = NewType("N", NT1) + + c: C = C() + nt1: NT1 = NT1(c=1) + nt2: NT2 = NT2(c=1) + nt3: NT3 = NT3(c=1) + td: TD = TD(c=1) + td2: TD2 = TD2(c=1) + e: E = E.X + dc: DC = DC(c=1) + al: Alias = Alias(c=1) + n: N = N(NT1(c=1)) + +[builtins fixtures/dict.pyi] +[out2] +tmp/a.py:2: error: "object" has no attribute "xyz" + +[case testIncrementalInvalidNamedTupleInUnannotatedFunction] +# flags: --disable-error-code=annotation-unchecked +import a + +[file a.py] +import b + +[file a.py.2] +import b # f + +[file b.py] +from typing import NamedTuple + +def toplevel(fields): + TupleType = NamedTuple("TupleType", fields) + class InheritFromTuple(TupleType): + pass + NT2 = NamedTuple("bad", [('x', int)]) + nt2: NT2 = NT2(x=1) + +class C: + def method(self, fields): + TupleType = NamedTuple("TupleType", fields) + class InheritFromTuple(TupleType): + pass + NT2 = NamedTuple("bad", [('x', int)]) + nt2: NT2 = NT2(x=1) + +[builtins fixtures/tuple.pyi] + +[case testNamedTupleUpdateNonRecursiveToRecursiveCoarse] +import c +[file a.py] +from b import M +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[M] + x: int +n: N +[file b.py] +from a import N +from typing import NamedTuple +class M(NamedTuple): + r: None + x: int +[file b.py.2] +from a import N +from typing import NamedTuple, Optional +class M(NamedTuple): + r: Optional[N] + x: int +[file c.py] +import a +def f(x: a.N) -> None: + if x.r is not None: + s: int = x.r.x +[file c.py.3] +import a +def f(x: a.N) -> None: + if x.r is not None and x.r.r is not None and x.r.r.r is not None: + reveal_type(x) + s: int = x.r.r.r.r +f(a.n) +reveal_type(a.n) +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") +tmp/c.py:7: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" + +[case testTupleTypeUpdateNonRecursiveToRecursiveCoarse] +import c +[file a.py] +from b import M +from typing import Tuple, Optional +class N(Tuple[Optional[M], int]): ... +[file b.py] +from a import N +from typing import Tuple +class M(Tuple[None, int]): ... +[file b.py.2] +from a import N +from typing import Tuple, Optional +class M(Tuple[Optional[N], int]): ... +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTypeAliasUpdateNonRecursiveToRecursiveCoarse] +import c +[file a.py] +from b import M +from typing import Tuple, Optional +N = Tuple[Optional[M], int] +[file b.py] +from a import N +from typing import Tuple +M = Tuple[None, int] +[file b.py.2] +from a import N +from typing import Tuple, Optional +M = Tuple[Optional[N], int] +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTypedDictUpdateNonRecursiveToRecursiveCoarse] +import c +[file a.py] +from b import M +from typing import TypedDict, Optional +class N(TypedDict): + r: Optional[M] + x: int +n: N +[file b.py] +from a import N +from typing import TypedDict +class M(TypedDict): + r: None + x: int +[file b.py.2] +from a import N +from typing import TypedDict, Optional +class M(TypedDict): + r: Optional[N] + x: int +[file c.py] +import a +def f(x: a.N) -> None: + if x["r"] is not None: + s: int = x["r"]["x"] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x["r"] is not None and x["r"]["r"] is not None and x["r"]["r"]["r"] is not None: + reveal_type(x) + s: int = x["r"]["r"]["r"]["r"] +f(a.n) +reveal_type(a.n) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M', {'r': Union[..., None], 'x': builtins.int}), None], 'x': builtins.int})" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") +tmp/c.py:7: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M', {'r': Union[..., None], 'x': builtins.int}), None], 'x': builtins.int})" + +[case testIncrementalAddClassMethodPlugin] +# flags: --config-file tmp/mypy.ini +import b + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_classmethod.py + +[file a.py] +class BaseAddMethod: pass + +class MyClass(BaseAddMethod): + pass + +[file b.py] +import a + +[file b.py.2] +import a + +my_class = a.MyClass() +reveal_type(a.MyClass.foo_classmethod) +reveal_type(a.MyClass.foo_staticmethod) +reveal_type(my_class.foo_classmethod) +reveal_type(my_class.foo_staticmethod) + +[rechecked b] +[out2] +tmp/b.py:4: note: Revealed type is "def ()" +tmp/b.py:5: note: Revealed type is "def (builtins.int) -> builtins.str" +tmp/b.py:6: note: Revealed type is "def ()" +tmp/b.py:7: note: Revealed type is "def (builtins.int) -> builtins.str" + +[case testIncrementalAddOverloadedMethodPlugin] +# flags: --config-file tmp/mypy.ini +import b + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_overloaded_method.py + +[file a.py] +class AddOverloadedMethod: pass + +class MyClass(AddOverloadedMethod): + pass + +[file b.py] +import a + +[file b.py.2] +import a + +reveal_type(a.MyClass.method) +reveal_type(a.MyClass.clsmethod) +reveal_type(a.MyClass.stmethod) + +my_class = a.MyClass() +reveal_type(my_class.method) +reveal_type(my_class.clsmethod) +reveal_type(my_class.stmethod) +[rechecked b] +[out2] +tmp/b.py:3: note: Revealed type is "Overload(def (self: a.MyClass, arg: builtins.int) -> builtins.str, def (self: a.MyClass, arg: builtins.str) -> builtins.int)" +tmp/b.py:4: note: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +tmp/b.py:5: note: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +tmp/b.py:8: note: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +tmp/b.py:9: note: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" +tmp/b.py:10: note: Revealed type is "Overload(def (arg: builtins.int) -> builtins.str, def (arg: builtins.str) -> builtins.int)" + +[case testGenericNamedTupleSerialization] +import b +[file a.py] +from typing import NamedTuple, Generic, TypeVar + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +[file b.py] +from a import NT +nt = NT(key=0, value="yes") +s: str = nt.value +[file b.py.2] +from a import NT +nt = NT(key=0, value=42) +s: str = nt.value +[builtins fixtures/tuple.pyi] +[out] +[out2] +tmp/b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testGenericTypedDictSerialization] +import b +[file a.py] +from typing import TypedDict, Generic, TypeVar + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + key: int + value: T + +[file b.py] +from a import TD +td = TD(key=0, value="yes") +s: str = td["value"] +[file b.py.2] +from a import TD +td = TD(key=0, value=42) +s: str = td["value"] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +tmp/b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testUnpackKwargsSerialize] +import m +[file lib.py] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]): + ... + +[file m.py] +from lib import foo +foo(name='Jennifer', age=38) +[file m.py.2] +from lib import foo +foo(name='Jennifer', age="38") +[builtins fixtures/dict.pyi] +[out] +[out2] +tmp/m.py:2: error: Argument "age" to "foo" has incompatible type "str"; expected "int" + +[case testDisableEnableErrorCodesIncremental] +# flags: --disable-error-code truthy-bool +# flags2: --enable-error-code truthy-bool +class Foo: + pass + +foo = Foo() +if foo: + ... +[out] +[out2] +main:7: error: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context + +[case testModuleAsProtocolImplementationSerialize] +import m +[file m.py] +from typing import Protocol +from lib import C + +class Options(Protocol): + timeout: int + def update(self) -> bool: ... + +def setup(options: Options) -> None: ... +setup(C().config) + +[file lib.py] +import default_config + +class C: + config = default_config + +[file default_config.py] +timeout = 100 +def update() -> bool: ... + +[file default_config.py.2] +timeout = 100 +def update() -> str: ... +[builtins fixtures/module.pyi] +[out] +[out2] +tmp/m.py:9: error: Argument 1 to "setup" has incompatible type Module; expected "Options" +tmp/m.py:9: note: Following member(s) of Module "default_config" have conflicts: +tmp/m.py:9: note: Expected: +tmp/m.py:9: note: def update() -> bool +tmp/m.py:9: note: Got: +tmp/m.py:9: note: def update() -> str + +[case testAbstractBodyTurnsEmptyCoarse] +from b import Base + +class Sub(Base): + def meth(self) -> int: + return super().meth() + +[file b.py] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: return 0 + +[file b.py.2] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: ... +[out] +[out2] +main:5: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe + +[case testNoCrashDoubleReexportFunctionEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +def foo(arg: c.C) -> None: pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportBaseEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportMetaEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(metaclass=c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C(type): ... +[file pb2.py.2] +class C(type): ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportTypedDictEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +from typing_extensions import TypedDict +import c +class D(TypedDict): + x: c.C + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[builtins fixtures/dict.pyi] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportTupleEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +from typing import Tuple +import c +class D(Tuple[c.C, int]): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportOverloadEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +from typing import Any, overload +import c + +@overload +def foo(arg: int) -> None: ... +@overload +def foo(arg: c.C) -> None: ... +def foo(arg: Any) -> None: + pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testNoCrashOnPartialLambdaInference] +import m +[file m.py] +from typing import TypeVar, Callable + +V = TypeVar("V") +def apply(val: V, func: Callable[[V], None]) -> None: + return func(val) + +xs = [] +apply(0, lambda a: xs.append(a)) +[file m.py.2] +from typing import TypeVar, Callable + +V = TypeVar("V") +def apply(val: V, func: Callable[[V], None]) -> None: + return func(val) + +xs = [] +apply(0, lambda a: xs.append(a)) +reveal_type(xs) +[builtins fixtures/list.pyi] +[out] +[out2] +tmp/m.py:9: note: Revealed type is "builtins.list[builtins.int]" + +[case testTypingSelfCoarse] +import m +[file lib.py] +from typing import Self + +class C: + def meth(self, other: Self) -> Self: ... + +[file m.py] +import lib +class D: ... +[file m.py.2] +import lib +class D(lib.C): ... + +reveal_type(D.meth) +reveal_type(D().meth) +[out] +[out2] +tmp/m.py:4: note: Revealed type is "def [Self <: lib.C] (self: Self`1, other: Self`1) -> Self`1" +tmp/m.py:5: note: Revealed type is "def (other: m.D) -> m.D" + +[case testIncrementalNestedGenericCallableCrash] +from typing import TypeVar, Callable + +T = TypeVar("T") + +class B: + def foo(self) -> Callable[[T], T]: ... + +class C(B): + def __init__(self) -> None: + self.x = self.foo() +[out] +[out2] + +[case testNoCrashIncrementalMetaAny] +import a +[file a.py] +from m import Foo +[file a.py.2] +from m import Foo +# touch +[file m.py] +from missing_module import Meta # type: ignore[import] +class Foo(metaclass=Meta): ... + +[case testIncrementalNativeInt] +import a +[file a.py] +from mypy_extensions import i64 +x: i64 = 0 +[file a.py.2] +from mypy_extensions import i64 +x: i64 = 0 +y: int = x +[builtins fixtures/tuple.pyi] +[out] +[out2] + +[case testGenericTypedDictWithError] +import b +[file a.py] +from typing import Generic, TypeVar +from typing_extensions import TypedDict + +TValue = TypeVar("TValue") +class Dict(TypedDict, Generic[TValue]): + value: TValue + +[file b.py] +from a import Dict, TValue + +def f(d: Dict[TValue]) -> TValue: + return d["value"] +def g(d: Dict[TValue]) -> TValue: + return d["x"] + +[file b.py.2] +from a import Dict, TValue + +def f(d: Dict[TValue]) -> TValue: + return d["value"] +def g(d: Dict[TValue]) -> TValue: + return d["y"] +[builtins fixtures/dict.pyi] +[out] +tmp/b.py:6: error: TypedDict "a.Dict[TValue]" has no key "x" +[out2] +tmp/b.py:6: error: TypedDict "a.Dict[TValue]" has no key "y" + +[case testParamSpecNoCrash] +import m +[file m.py] +from typing import Callable, TypeVar +from lib import C + +T = TypeVar("T") +def test(x: Callable[..., T]) -> T: ... +test(C) # type: ignore + +[file m.py.2] +from typing import Callable, TypeVar +from lib import C + +T = TypeVar("T") +def test(x: Callable[..., T]) -> T: ... +test(C) # type: ignore +# touch +[file lib.py] +from typing import ParamSpec, Generic, Callable + +P = ParamSpec("P") +class C(Generic[P]): + def __init__(self, fn: Callable[P, int]) -> None: ... +[builtins fixtures/dict.pyi] + +[case testVariadicClassIncrementalUpdateRegularToVariadic] +from typing import Any +from lib import C + +x: C[int, str] + +[file lib.py] +from typing import Generic, TypeVar + +T = TypeVar("T") +S = TypeVar("S") +class C(Generic[T, S]): ... + +[file lib.py.2] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): ... +[builtins fixtures/tuple.pyi] + +[case testVariadicClassIncrementalUpdateVariadicToRegular] +from typing import Any +from lib import C + +x: C[int, str, int] + +[file lib.py] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): ... +[file lib.py.2] +from typing import Generic, TypeVar + +T = TypeVar("T") +S = TypeVar("S") +class C(Generic[T, S]): ... +[builtins fixtures/tuple.pyi] +[out2] +main:4: error: "C" expects 2 type arguments, but 3 given + +[case testVariadicTupleIncrementalUpdateNoCrash] +import m +[file m.py] +from typing import Any +from lib import C + +x: C[Any] +[file m.py.2] +from lib import C + +x: C[int] +[file lib.py] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Tuple[Unpack[Ts]]): ... +[builtins fixtures/tuple.pyi] + +[case testNoIncrementalCrashOnInvalidTypedDict] +import m +[file m.py] +import counts +[file m.py.2] +import counts +# touch +[file counts.py] +from typing_extensions import TypedDict +Counts = TypedDict("Counts", {k: int for k in "abc"}) # type: ignore +[builtins fixtures/dict.pyi] + +[case testNoIncrementalCrashOnInvalidTypedDictFunc] +import m +[file m.py] +import counts +[file m.py.2] +import counts +# touch +[file counts.py] +from typing_extensions import TypedDict +def test() -> None: + Counts = TypedDict("Counts", {k: int for k in "abc"}) # type: ignore +[builtins fixtures/dict.pyi] + +[case testNoIncrementalCrashOnTypedDictMethod] +import a +[file a.py] +from b import C +x: C +[file a.py.2] +from b import C +x: C +reveal_type(x.h) +[file b.py] +from typing_extensions import TypedDict +class C: + def __init__(self) -> None: + self.h: Hidden + class Hidden(TypedDict): + x: int +[builtins fixtures/dict.pyi] +[out] +[out2] +tmp/a.py:3: note: Revealed type is "TypedDict('b.C.Hidden@5', {'x': builtins.int})" + +[case testNoIncrementalCrashOnInvalidEnumMethod] +import a +[file a.py] +from lib import TheClass +[file a.py.2] +from lib import TheClass +x: TheClass +reveal_type(x.enum_type) +[file lib.py] +import enum + +class TheClass: + def __init__(self) -> None: + names = ["foo"] + pyenum = enum.Enum('Blah', { # type: ignore[misc] + x.upper(): x + for x in names + }) + self.enum_type = pyenum +[builtins fixtures/tuple.pyi] +[out] +[out2] +tmp/a.py:3: note: Revealed type is "def (value: builtins.object) -> lib.TheClass.pyenum@6" + + +[case testIncrementalFunctoolsPartial] +import a + +[file a.py] +from typing import Callable +from partial import p1, p2 + +p1(1, "a", 3) # OK +p1(1, "a", c=3) # OK +p1(1, b="a", c=3) # OK + +reveal_type(p1) + +def takes_callable_int(f: Callable[..., int]) -> None: ... +def takes_callable_str(f: Callable[..., str]) -> None: ... +takes_callable_int(p1) +takes_callable_str(p1) + +p2("a") # OK +p2("a", 3) # OK +p2("a", c=3) # OK +p2(1, 3) +p2(1, "a", 3) +p2(a=1, b="a", c=3) + +[file a.py.2] +from typing import Callable +from partial import p3 + +p3(1) # OK +p3(1, c=3) # OK +p3(a=1) # OK +p3(1, b="a", c=3) # OK, keywords can be clobbered +p3(1, 3) + +[file partial.py] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... + +p1 = functools.partial(foo) +p2 = functools.partial(foo, 1) +p3 = functools.partial(foo, b="a") +[builtins fixtures/dict.pyi] +[out] +tmp/a.py:8: note: Revealed type is "functools.partial[builtins.int]" +tmp/a.py:13: error: Argument 1 to "takes_callable_str" has incompatible type "partial[int]"; expected "Callable[..., str]" +tmp/a.py:13: note: "partial[int].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], int]" +tmp/a.py:18: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +tmp/a.py:19: error: Too many arguments for "foo" +tmp/a.py:19: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +tmp/a.py:19: error: Argument 2 to "foo" has incompatible type "str"; expected "int" +tmp/a.py:20: error: Unexpected keyword argument "a" for "foo" +tmp/partial.py:4: note: "foo" defined here +[out2] +tmp/a.py:8: error: Too many positional arguments for "foo" +tmp/a.py:8: error: Argument 2 to "foo" has incompatible type "int"; expected "str" + + +[case testStartUsingTypeGuard] +import a +[file a.py] +from lib import guard +from typing import Union +from typing_extensions import assert_type +x: Union[int, str] + +[file a.py.2] +from lib import guard +from typing import Union +from typing_extensions import assert_type +x: Union[int, str] +if guard(x): + assert_type(x, int) +else: + assert_type(x, Union[int, str]) +[file lib.py] +from typing_extensions import TypeGuard +def guard(x: object) -> TypeGuard[int]: + pass +[builtins fixtures/tuple.pyi] + +[case testStartUsingTypeIs] +import a +[file a.py] +from lib import guard +from typing import Union +from typing_extensions import assert_type +x: Union[int, str] + +[file a.py.2] +from lib import guard +from typing import Union +from typing_extensions import assert_type +x: Union[int, str] +if guard(x): + assert_type(x, int) +else: + assert_type(x, str) +[file lib.py] +from typing_extensions import TypeIs +def guard(x: object) -> TypeIs[int]: + pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardToTypeIs] +import a +[file a.py] +from lib import guard +from typing import Union +from typing_extensions import assert_type +x: Union[int, str] +if guard(x): + assert_type(x, int) +else: + assert_type(x, Union[int, str]) +[file a.py.2] +from lib import guard +from typing import Union +from typing_extensions import assert_type +x: Union[int, str] +if guard(x): + assert_type(x, int) +else: + assert_type(x, str) +[file lib.py] +from typing_extensions import TypeGuard +def guard(x: object) -> TypeGuard[int]: + pass +[file lib.py.2] +from typing_extensions import TypeIs +def guard(x: object) -> TypeIs[int]: + pass +[builtins fixtures/tuple.pyi] + +[case testStartUsingPEP604Union] +# flags: --python-version 3.10 +import a +[file a.py] +import lib + +[file a.py.2] +from lib import IntOrStr +assert isinstance(1, IntOrStr) + +[file lib.py] +from typing_extensions import TypeAlias + +IntOrStr: TypeAlias = int | str +assert isinstance(1, IntOrStr) +[builtins fixtures/type.pyi] diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test new file mode 100644 index 000000000000..17ae6d9934b7 --- /dev/null +++ b/test-data/unit/check-inference-context.test @@ -0,0 +1,1497 @@ + + +-- Basic test cases +-- ---------------- + + +[case testBasicContextInference] +from typing import TypeVar, Generic +T = TypeVar('T') + +def f() -> 'A[T]': pass + +class A(Generic[T]): pass +class B: pass + +ab: A[B] +ao: A[object] +b: B + +if int(): + ao = f() +if int(): + ab = f() +if int(): + b = f() # E: Incompatible types in assignment (expression has type "A[Never]", variable has type "B") +[case testBasicContextInferenceForConstructor] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): pass +class B: pass +ab: A[B] +ao: A[object] +b: B + +if int(): + ao = A() +if int(): + ab = A() +if int(): + b = A() # E: Incompatible types in assignment (expression has type "A[Never]", variable has type "B") +[case testIncompatibleContextInference] +from typing import TypeVar, Generic +T = TypeVar('T') +def f(a: T) -> 'A[T]': + pass + +class A(Generic[T]): pass + +class B: pass +class C: pass +b: B +c: C +ab: A[B] +ao: A[object] +ac: A[C] + +if int(): + ac = f(b) # E: Argument 1 to "f" has incompatible type "B"; expected "C" +if int(): + ab = f(c) # E: Argument 1 to "f" has incompatible type "C"; expected "B" + +if int(): + ao = f(b) +if int(): + ab = f(b) +if int(): + ao = f(c) +if int(): + ac = f(c) + + +-- Local variables +-- --------------- + + +[case testInferGenericLocalVariableTypeWithEmptyContext] +from typing import TypeVar, Generic +T = TypeVar('T') +def g() -> None: + ao: A[object] + ab: A[B] + o: object + b: B + + x = f(o) + if int(): + ab = x # E: Incompatible types in assignment (expression has type "A[object]", variable has type "A[B]") + ao = x + y = f(b) + if int(): + ao = y # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") + ab = y + +def f(a: T) -> 'A[T]': pass + +class A(Generic[T]): pass +class B: pass +[out] + +[case testInferLocalVariableTypeWithUnderspecifiedGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +def g() -> None: + x = f() # E: Need type annotation for "x" + +def f() -> 'A[T]': pass +class A(Generic[T]): pass +[out] + +[case testInferMultipleLocalVariableTypesWithTupleRvalue] +from typing import TypeVar, Generic +T = TypeVar('T') +def g() -> None: + ao: A[object] + ab: A[B] + b: B + x, y = f(b), f(b) + if int(): + ao = x # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") + ao = y # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") + ab = x + ab = y + +def f(a: T) -> 'A[T]': pass +class A(Generic[T]): pass +class B: pass +[out] + +[case testInferMultipleLocalVariableTypesWithArrayRvalueAndNesting] +from typing import TypeVar, List, Generic +T = TypeVar('T') +def h() -> None: + ao: A[object] + ab: A[B] + b: B + x, y = g(f(b)) + if int(): + ao = x # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") + ao = y # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") + ab = x + ab = y + +def f(a: T) -> 'A[T]': pass +def g(a: T) -> List[T]: pass + +class A(Generic[T]): pass +class B: pass +[builtins fixtures/for.pyi] +[out] + + +-- Return types with multiple tvar instances +-- ----------------------------------------- + + +[case testInferenceWithTypeVariableTwiceInReturnType] +from typing import TypeVar, Tuple, Generic +T = TypeVar('T') + +def f(a: T) -> 'Tuple[A[T], A[T]]': pass + +class A(Generic[T]): pass +class B: pass + +b: B +o: object +ab: A[B] +ao: A[object] + +if int(): + ab, ao = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") +if int(): + ao, ab = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") + +if int(): + ao, ao = f(b) +if int(): + ab, ab = f(b) +if int(): + ao, ao = f(o) +[builtins fixtures/tuple.pyi] + +[case testInferenceWithTypeVariableTwiceInReturnTypeAndMultipleVariables] +from typing import TypeVar, Tuple, Generic +S = TypeVar('S') +T = TypeVar('T') + +def f(a: S, b: T) -> 'Tuple[A[S], A[T], A[T]]': pass +def g(a: S, b: T) -> 'Tuple[A[S], A[S], A[T]]': pass +def h(a: S, b: T) -> 'Tuple[A[S], A[S], A[T], A[T]]': pass + +class A(Generic[T]): pass +class B: pass + +b: B +o: object +ab: A[B] +ao: A[object] + +if int(): + ao, ao, ab = f(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") +if int(): + ao, ab, ao = g(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") +if int(): + ao, ab, ab, ab = h(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") +if int(): + ab, ab, ao, ab = h(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") + +if int(): + ao, ab, ab = f(b, b) +if int(): + ab, ab, ao = g(b, b) +if int(): + ab, ab, ab, ab = h(b, b) +[builtins fixtures/tuple.pyi] + + +-- Multiple tvar instances in arguments +-- ------------------------------------ + + +[case testMultipleTvatInstancesInArgs] +from typing import TypeVar, Generic +T = TypeVar('T') + +def f(a: T, b: T) -> 'A[T]': pass + +class A(Generic[T]): pass +class B: pass +class C(B): pass + +ac: A[C] +ab: A[B] +ao: A[object] +b: B +c: C +o: object + +if int(): + ab = f(b, o) # E: Argument 2 to "f" has incompatible type "object"; expected "B" +if int(): + ab = f(o, b) # E: Argument 1 to "f" has incompatible type "object"; expected "B" +if int(): + ac = f(b, c) # E: Argument 1 to "f" has incompatible type "B"; expected "C" +if int(): + ac = f(c, b) # E: Argument 2 to "f" has incompatible type "B"; expected "C" + +if int(): + ao = f(b, c) +if int(): + ao = f(c, b) +if int(): + ab = f(c, b) + + +-- Nested generic function calls +-- ----------------------------- + + +[case testNestedGenericFunctionCall1] +from typing import TypeVar, Generic +T = TypeVar('T') + +def f(a: T) -> 'A[T]': pass + +class A(Generic[T]): pass +class B: pass + +aab: A[A[B]] +aao: A[A[object]] +ao: A[object] +b: B +o: object + +if int(): + aab = f(f(o)) # E: Argument 1 to "f" has incompatible type "object"; expected "B" + +if int(): + aab = f(f(b)) + aao = f(f(b)) + ao = f(f(b)) + +[case testNestedGenericFunctionCall2] +from typing import TypeVar, Generic +T = TypeVar('T') + +def f(a: T) -> T: pass +def g(a: T) -> 'A[T]': pass + +class A(Generic[T]): pass +class B: pass + +ab: A[B] +ao: A[object] +b: B +o: object + +if int(): + ab = f(g(o)) # E: Argument 1 to "g" has incompatible type "object"; expected "B" + +if int(): + ab = f(g(b)) + ao = f(g(b)) + +[case testNestedGenericFunctionCall3] +from typing import TypeVar, Generic +T = TypeVar('T') +def f(a: T, b: T) -> T: + pass + +def g(a: T) -> 'A[T]': pass + +class A(Generic[T]): pass +class B: pass +ab: A[B] +ao: A[object] +b: B +o: object + +if int(): + ab = f(g(o), g(b)) # E: Argument 1 to "g" has incompatible type "object"; expected "B" +if int(): + ab = f(g(b), g(o)) # E: Argument 1 to "g" has incompatible type "object"; expected "B" + +if int(): + ab = f(g(b), g(b)) + ao = f(g(b), g(o)) +if int(): + ao = f(g(o), g(b)) + + +-- Method calls +-- ------------ + + +[case testMethodCallWithContextInference] +from typing import TypeVar, Generic +T = TypeVar('T') +o: object +b: B +c: C +def f(a: T) -> 'A[T]': pass + +class A(Generic[T]): + def g(self, a: 'A[T]') -> 'A[T]': pass + +class B: pass +class C(B): pass +ao: A[object] +ab: A[B] +ac: A[C] + +ab.g(f(o)) # E: Argument 1 to "f" has incompatible type "object"; expected "B" +if int(): + ac = f(b).g(f(c)) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[C]") +if int(): + ac = f(c).g(f(b)) # E: Argument 1 to "f" has incompatible type "B"; expected "C" + +if int(): + ab = f(b).g(f(c)) +ab.g(f(c)) + + +-- List expressions +-- ---------------- + + +[case testEmptyListExpression] +from typing import List +aa: List[A] +ao: List[object] +a: A +def f(): a, aa, ao # Prevent redefinition + +a = [] # E: Incompatible types in assignment (expression has type "List[Never]", variable has type "A") + +aa = [] +ao = [] + +class A: pass +[builtins fixtures/list.pyi] + +[case testSingleItemListExpressions] +from typing import List, Optional +aa: List[Optional[A]] +ab: List[B] +ao: List[object] +a: A +b: B +def f(): aa, ab, ao # Prevent redefinition + +aa = [b] # E: List item 0 has incompatible type "B"; expected "Optional[A]" +ab = [a] # E: List item 0 has incompatible type "A"; expected "B" + +aa = [a] +ab = [b] +ao = [a] +aa = [None] +ao = [None] + +class A: pass +class B: pass +[builtins fixtures/list.pyi] + +[case testMultiItemListExpressions] +from typing import List +aa: List[A] +ab: List[B] +ao: List[object] +a: A +b: B +def f(): ab, aa, ao # Prevent redefinition + +ab = [b, a] # E: List item 1 has incompatible type "A"; expected "B" +ab = [a, b] # E: List item 0 has incompatible type "A"; expected "B" + +aa = [a, b, a] +ao = [a, b] + +class A: pass +class B(A): pass +[builtins fixtures/list.pyi] + +[case testLocalVariableInferenceFromEmptyList] +import typing +def f() -> None: + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + b = [None] + c = [B()] + if int(): + c = [object()] # E: List item 0 has incompatible type "object"; expected "B" + c = [B()] +class B: pass +[builtins fixtures/list.pyi] +[out] + +[case testNestedListExpressions] +# flags: --no-strict-optional +from typing import List +aao = None # type: List[List[object]] +aab = None # type: List[List[B]] +ab = None # type: List[B] +b = None # type: B +o = None # type: object +def f(): aao, aab # Prevent redefinition + +aao = [[o], ab] # E: List item 1 has incompatible type "List[B]"; expected "List[object]" +aab = [[], [o]] # E: List item 0 has incompatible type "object"; expected "B" + +aao = [[None], [b], [], [o]] +aab = [[None], [b], []] +aab = [ab, []] + +class B: pass +[builtins fixtures/list.pyi] + + +-- Complex context +-- --------------- + + +[case testParenthesesAndContext] +from typing import List +class A: pass +l = ([A()]) # type: List[object] +[builtins fixtures/list.pyi] + +[case testComplexTypeInferenceWithTuple] +from typing import TypeVar, Tuple, Generic +k = TypeVar('k') +t = TypeVar('t') +v = TypeVar('v') + +class A(Generic[t]): pass +class B: pass +class C: pass +class D(Generic[k, v]): pass + +def f(x: Tuple[k]) -> 'A[k]': pass + +d = f((A(),)) # type: A[A[B]] +[builtins fixtures/list.pyi] + + +-- Dictionary literals +-- ------------------- + + +[case testDictionaryLiteralInContext] +from typing import Dict, TypeVar, Generic +t = TypeVar('t') +class A(Generic[t]): pass +class B: pass +class C: pass +a_b = A() # type: A[B] +a_c = A() # type: A[C] +d = {A() : a_c, + a_b : A()} # type: Dict[A[B], A[C]] +[builtins fixtures/dict.pyi] + + +-- Special cases (regression tests etc.) +-- ------------------------------------- + + +[case testInitializationWithInferredGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') + +def f(x: T) -> T: pass +class C(Generic[T]): pass +class A: pass + +c = f(A()) # type: C[A] # E: Argument 1 to "f" has incompatible type "A"; expected "C[A]" +[case testInferredGenericTypeAsReturnValue] +from typing import TypeVar, Generic +T = TypeVar('T') +def t() -> 'A[B]': + return f(D()) # E: Argument 1 to "f" has incompatible type "D"; expected "B" + return A() + return f(C()) + +def f(a: T) -> 'A[T]': pass +class A(Generic[T]): pass +class B: pass +class C(B): pass +class D: pass +[out] + +[case testIntersectionWithInferredGenericArgument] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar, Generic +T = TypeVar('T') +f(A()) + +@overload +def f(x: 'A[B]') -> None: pass +@overload +def f(x: 'B') -> None: pass +class A(Generic[T]): pass +class B: pass + +[case testInferenceWithAbstractClassContext] +from typing import TypeVar, Generic +from abc import abstractmethod, ABCMeta +t = TypeVar('t') + +class I(Generic[t]): + @abstractmethod + def f(self): pass +class A(I[t], Generic[t]): + def f(self): pass + +x = A() # type: I[int] +a_object = A() # type: A[object] +y = a_object # type: I[int] # E: Incompatible types in assignment (expression has type "A[object]", variable has type "I[int]") + +[case testInferenceWithAbstractClassContext2] +from typing import TypeVar, Generic +from abc import abstractmethod, ABCMeta +t = TypeVar('t') +class I(Generic[t]): pass +class A(I[t], Generic[t]): pass +def f(i: I[t]) -> A[t]: pass +a = f(A()) # type: A[int] +a_int = A() # type: A[int] +aa = f(a_int) + +[case testInferenceWithAbstractClassContext3] +from typing import TypeVar, Generic, Iterable +t = TypeVar('t') +class set(Generic[t]): + def __init__(self, iterable: Iterable[t]) -> None: pass +b = bool() +l = set([b]) +if int(): + l = set([object()]) # E: List item 0 has incompatible type "object"; expected "bool" +[builtins fixtures/for.pyi] + + +-- Infer generic type in 'Any' context +-- ----------------------------------- + + +[case testInferGenericTypeInAnyContext] +from typing import Any, TypeVar, Generic +s = TypeVar('s') +t = TypeVar('t') +class C(Generic[s, t]): pass +x = [] # type: Any +y = C() # type: Any +[builtins fixtures/list.pyi] + + +-- Lambdas +-- ------- + + +[case testInferLambdaArgumentTypeUsingContext] +from typing import Callable +f: Callable[[B], A] +if int(): + f = lambda x: x.o + f = lambda x: x.x # E: "B" has no attribute "x" +class A: pass +class B: + o: A + +[case testInferLambdaReturnTypeUsingContext] +from typing import List, Callable +f: Callable[[], List[A]] +if int(): + f = lambda: [] + f = lambda: [B()] # E: List item 0 has incompatible type "B"; expected "A" +class A: pass +class B: pass +[builtins fixtures/list.pyi] + +[case testInferLambdaTypeUsingContext] +x : str = (lambda x: x + 1)(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +reveal_type((lambda x, y: x + y)(1, 2)) # N: Revealed type is "builtins.int" +(lambda x, y: x + y)(1, "") # E: Unsupported operand types for + ("int" and "str") +(lambda *, x, y: x + y)(x=1, y="") # E: Unsupported operand types for + ("int" and "str") +reveal_type((lambda s, i: s)(i=0, s='x')) # N: Revealed type is "Literal['x']?" +reveal_type((lambda s, i: i)(i=0, s='x')) # N: Revealed type is "Literal[0]?" +reveal_type((lambda x, s, i: x)(1.0, i=0, s='x')) # N: Revealed type is "builtins.float" +if object(): + (lambda x, s, i: x)() # E: Too few arguments +if object(): + (lambda: 0)(1) # E: Too many arguments +-- varargs are not handled, but it should not crash +reveal_type((lambda *k, s, i: i)(type, i=0, s='x')) # N: Revealed type is "Any" +reveal_type((lambda s, *k, i: i)(i=0, s='x')) # N: Revealed type is "Any" +reveal_type((lambda s, i, **k: i)(i=0, s='x')) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] + +[case testInferLambdaAsGenericFunctionArgument] +from typing import TypeVar, List, Any, Callable +t = TypeVar('t') +class A: + x = None # type: A +def f(a: List[t], fn: Callable[[t], Any]) -> None: pass +list_a = [] # type: List[A] +f(list_a, lambda a: a.x) +[builtins fixtures/list.pyi] + +[case testLambdaWithoutContext] +reveal_type(lambda x: x) # N: Revealed type is "def (x: Any) -> Any" +reveal_type(lambda x: 1) # N: Revealed type is "def (x: Any) -> Literal[1]?" + +[case testLambdaContextVararg] +from typing import Callable +def f(t: Callable[[str], str]) -> str: '' +f(lambda *_: '') +[builtins fixtures/tuple.pyi] + +[case testInvalidContextForLambda] +from typing import Callable +f = lambda x: A() # type: Callable[[], A] +f2 = lambda: A() # type: Callable[[A], A] +class A: pass +[out] +main:2: error: Cannot infer type of lambda +main:2: error: Incompatible types in assignment (expression has type "Callable[[Any], A]", variable has type "Callable[[], A]") +main:3: error: Cannot infer type of lambda +main:3: error: Incompatible types in assignment (expression has type "Callable[[], A]", variable has type "Callable[[A], A]") + +[case testEllipsisContextForLambda] +from typing import Callable +f1 = lambda x: 1 # type: Callable[..., int] +f2 = lambda: 1 # type: Callable[..., int] +f3 = lambda *args, **kwargs: 1 # type: Callable[..., int] +f4 = lambda x: x # type: Callable[..., int] +g = lambda x: 1 # type: Callable[..., str] +[builtins fixtures/dict.pyi] +[out] +main:6: error: Incompatible types in assignment (expression has type "Callable[[Any], int]", variable has type "Callable[..., str]") +main:6: error: Incompatible return value type (got "int", expected "str") + +[case testEllipsisContextForLambda2] +from typing import TypeVar, Callable +T = TypeVar('T') +def foo(arg: Callable[..., T]) -> None: pass +foo(lambda: 1) + +[case testLambdaNoneInContext] +# flags: --no-strict-optional +from typing import Callable +def f(x: Callable[[], None]) -> None: pass +def g(x: Callable[[], int]) -> None: pass +f(lambda: None) +g(lambda: None) + +[case testIsinstanceInInferredLambda] +# flags: --new-type-inference +from typing import TypeVar, Callable, Optional +T = TypeVar('T') +S = TypeVar('S') +class A: pass +class B(A): pass +class C(A): pass +def f(func: Callable[[T], S], *z: T, r: Optional[S] = None) -> S: pass +reveal_type(f(lambda x: 0 if isinstance(x, B) else 1)) # N: Revealed type is "Union[Literal[0]?, Literal[1]?]" +f(lambda x: 0 if isinstance(x, B) else 1, A())() # E: "int" not callable +f(lambda x: x if isinstance(x, B) else B(), A(), r=B())() # E: "B" not callable +f( + lambda x: # E: Argument 1 to "f" has incompatible type "Callable[[A], A]"; expected "Callable[[A], B]" + B() if isinstance(x, B) else x, # E: Incompatible return value type (got "A", expected "B") + A(), r=B()) +[builtins fixtures/isinstance.pyi] + + +-- Overloads + generic functions +-- ----------------------------- + + +[case testMapWithOverloadedFunc] +from foo import * +[file foo.pyi] +from typing import TypeVar, Callable, List, overload, Any +t = TypeVar('t') +s = TypeVar('s') +def map(f: Callable[[t], s], seq: List[t]) -> List[s]: pass + +@overload +def g(o: object) -> 'B': pass +@overload +def g(o: 'A', x: Any = None) -> 'B': pass + +class A: pass +class B: pass + +m = map(g, [A()]) +b = m # type: List[B] +a = m # type: List[A] # E: Incompatible types in assignment (expression has type "List[B]", variable has type "List[A]") +[builtins fixtures/list.pyi] + + +-- Boolean operators +-- ----------------- + + +[case testOrOperationInferredFromContext] +from typing import List +class A: pass +class B: pass +class C(B): pass +a: List[A] +b: List[B] +c: List[C] +if int(): + a = a or [] +if int(): + a = [] or a +if int(): + b = b or [C()] +if int(): + a = a or b # E: Incompatible types in assignment (expression has type "Union[List[A], List[B]]", variable has type "List[A]") +if int(): + b = b or c # E: Incompatible types in assignment (expression has type "Union[List[B], List[C]]", variable has type "List[B]") +[builtins fixtures/list.pyi] + + +-- Special cases +-- ------------- + + +[case testSomeTypeVarsInferredFromContext] +from typing import List, TypeVar +t = TypeVar('t') +s = TypeVar('s') +# Some type variables can be inferred using context, but not all of them. +a: List[A] +def f(a: s, b: t) -> List[s]: pass +class A: pass +class B: pass +if int(): + a = f(A(), B()) +if int(): + a = f(B(), B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +[builtins fixtures/list.pyi] + +[case testSomeTypeVarsInferredFromContext2] +from typing import List, TypeVar +s = TypeVar('s') +t = TypeVar('t') +def f(a: s, b: t) -> List[s]: pass +class A: pass +class B: pass +# Like testSomeTypeVarsInferredFromContext, but tvars in different order. +a: List[A] +if int(): + a = f(A(), B()) +if int(): + a = f(B(), B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +[builtins fixtures/list.pyi] + +[case testLambdaInListAndHigherOrderFunction] +from typing import TypeVar, Callable, List +t = TypeVar('t') +s = TypeVar('s') +def map(f: List[Callable[[t], s]], a: List[t]) -> List[s]: pass +class A: pass +map( + [lambda x: x], []) +[builtins fixtures/list.pyi] +[out] + +[case testChainedAssignmentInferenceContexts] +from typing import List +i: List[int] +s: List[str] +if int(): + i = i = [] +if int(): + i = s = [] # E: Incompatible types in assignment (expression has type "List[str]", variable has type "List[int]") +[builtins fixtures/list.pyi] + +[case testContextForAttributeDeclaredInInit] +from typing import List +class A: + def __init__(self): + self.x = [] # type: List[int] # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs +a = A() +a.x = [] +a.x = [1] +a.x = [''] # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testListMultiplyInContext] +from typing import List +a: List[int] +if int(): + a = [None] * 3 # E: List item 0 has incompatible type "None"; expected "int" + a = [''] * 3 # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testUnionTypeContext] +from typing import Union, List, TypeVar +T = TypeVar('T') +def f(x: Union[List[T], str]) -> None: pass +f([1]) +f('') +f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "Union[List[Never], str]" +[builtins fixtures/isinstancelist.pyi] + +[case testIgnoringInferenceContext] +from typing import TypeVar, List +T = TypeVar('T') +def f(x: List[T]) -> T: pass +def g(y: object) -> None: pass +a = [1] +g(f(a)) +[builtins fixtures/list.pyi] + +[case testStar2Context] +from typing import Any, Dict, Tuple, Iterable +def f1(iterable: Iterable[Tuple[str, Any]] = ()) -> None: + f2(**dict(iterable)) +def f2(iterable: Iterable[Tuple[str, Any]], **kw: Any) -> None: + pass +[builtins fixtures/dict.pyi] +[out] + +[case testInferenceInGenericFunction] +from typing import TypeVar, List +T = TypeVar('T') +def f(a: T) -> None: + l = [] # type: List[T] + l.append(a) + l.append(1) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "T" +[builtins fixtures/list.pyi] +[out] + +[case testInferenceInGenericClass] +from typing import TypeVar, Generic, List +S = TypeVar('S') +T = TypeVar('T') +class A(Generic[S]): + def f(self, a: T, b: S) -> None: + l = [] # type: List[T] + l.append(a) + l.append(b) # E: Argument 1 to "append" of "list" has incompatible type "S"; expected "T" +[builtins fixtures/list.pyi] +[out] + +[case testLambdaInGenericFunction] +from typing import TypeVar, Callable +T = TypeVar('T') +S = TypeVar('S') +def f(a: T, b: S) -> None: + c = lambda x: x # type: Callable[[T], S] +[out] +main:5: error: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[T], S]") +main:5: error: Incompatible return value type (got "T", expected "S") + +[case testLambdaInGenericClass] +from typing import TypeVar, Callable, Generic +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T]): + def f(self, b: S) -> None: + c = lambda x: x # type: Callable[[T], S] +[out] +main:6: error: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[T], S]") +main:6: error: Incompatible return value type (got "T", expected "S") + +[case testRevealTypeContext] +from typing import TypeVar, Callable, Generic +T = TypeVar('T') +class A(Generic[T]): + pass +reveal_type(A()) # N: Revealed type is "__main__.A[Never]" +b = reveal_type(A()) # type: A[int] # N: Revealed type is "__main__.A[builtins.int]" + +[case testUnionWithGenericTypeItemContext] +from typing import TypeVar, Union, List + +T = TypeVar('T') + +def f(x: Union[T, List[int]]) -> Union[T, List[int]]: pass +reveal_type(f(1)) # N: Revealed type is "Union[builtins.int, builtins.list[builtins.int]]" +reveal_type(f([])) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(f(None)) # N: Revealed type is "Union[None, builtins.list[builtins.int]]" +[builtins fixtures/list.pyi] + +[case testUnionWithGenericTypeItemContextAndStrictOptional] +from typing import TypeVar, Union, List + +T = TypeVar('T') + +def f(x: Union[T, List[int]]) -> Union[T, List[int]]: pass +reveal_type(f(1)) # N: Revealed type is "Union[builtins.int, builtins.list[builtins.int]]" +reveal_type(f([])) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(f(None)) # N: Revealed type is "Union[None, builtins.list[builtins.int]]" +[builtins fixtures/list.pyi] + +[case testUnionWithGenericTypeItemContextInMethod] +from typing import TypeVar, Union, List, Generic + +T = TypeVar('T') +S = TypeVar('S') + +class C(Generic[T]): + def f(self, x: Union[T, S]) -> Union[T, S]: pass + +c = C[List[int]]() +reveal_type(c.f('')) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.str]" +reveal_type(c.f([1])) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(c.f([])) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(c.f(None)) # N: Revealed type is "Union[builtins.list[builtins.int], None]" +[builtins fixtures/list.pyi] + +[case testGenericMethodCalledInGenericContext] +from typing import TypeVar, Generic + +_KT = TypeVar('_KT') +_VT = TypeVar('_VT') +_T = TypeVar('_T') + +class M(Generic[_KT, _VT]): + def get(self, k: _KT, default: _T) -> _T: ... + +def f(d: M[_KT, _VT], k: _KT) -> _VT: + return d.get(k, None) # E: Incompatible return value type (got "None", expected "_VT") + +[case testGenericMethodCalledInGenericContext2] +from typing import TypeVar, Generic, Union + +_KT = TypeVar('_KT') +_VT = TypeVar('_VT') +_T = TypeVar('_T') + +class M(Generic[_KT, _VT]): + def get(self, k: _KT, default: _T) -> Union[_VT, _T]: ... + +def f(d: M[_KT, _VT], k: _KT) -> Union[_VT, None]: + return d.get(k, None) + +[case testLambdaDeferredCrash] +from typing import Callable + +class C: + def f(self) -> None: + g: Callable[[], int] = lambda: 1 or self.x + self.x = int() + +[case testInferTypeVariableFromTwoGenericTypes1] +from typing import TypeVar, List, Sequence + +T = TypeVar('T') + +class C: ... +class D(C): ... + +def f(x: Sequence[T], y: Sequence[T]) -> List[T]: ... + +reveal_type(f([C()], [D()])) # N: Revealed type is "builtins.list[__main__.C]" +[builtins fixtures/list.pyi] + +[case testInferTypeVariableFromTwoGenericTypes2] +from typing import TypeVar, List + +T = TypeVar('T') + +class C: ... +class D(C): ... + +def f(x: List[T], y: List[T]) -> List[T]: ... + +f([C()], [D()]) # E: Cannot infer type argument 1 of "f" +[builtins fixtures/list.pyi] + +[case testInferTypeVariableFromTwoGenericTypes3] +from typing import Generic, TypeVar + +T = TypeVar('T') +T_contra = TypeVar('T_contra', contravariant=True) + +class A(Generic[T_contra]): pass +class B(A[T]): pass + +class C: ... +class D(C): ... + +def f(x: A[T], y: A[T]) -> B[T]: ... + +c: B[C] +d: B[D] +reveal_type(f(c, d)) # N: Revealed type is "__main__.B[__main__.D]" + +[case testInferTypeVariableFromTwoGenericTypes4] +from typing import Generic, TypeVar, Callable, List + +T = TypeVar('T') +T_contra = TypeVar('T_contra', contravariant=True) + +class A(Generic[T_contra]): pass +class B(A[T_contra]): pass + +class C: ... +class D(C): ... + +def f(x: Callable[[B[T]], None], + y: Callable[[B[T]], None]) -> List[T]: ... + +def gc(x: A[C]) -> None: pass # B[C] +def gd(x: A[D]) -> None: pass # B[C] + +reveal_type(f(gc, gd)) # N: Revealed type is "builtins.list[__main__.C]" +[builtins fixtures/list.pyi] + +[case testWideOuterContextSubClassBound] +from typing import TypeVar + +class A: ... +class B(A): ... + +T = TypeVar('T', bound=B) +def f(x: T) -> T: ... +def outer(x: A) -> None: ... + +outer(f(B())) +x: A = f(B()) + +[case testWideOuterContextSubClassBoundGenericReturn] +from typing import TypeVar, Iterable, List + +class A: ... +class B(A): ... + +T = TypeVar('T', bound=B) +def f(x: T) -> List[T]: ... +def outer(x: Iterable[A]) -> None: ... + +outer(f(B())) +x: Iterable[A] = f(B()) +[builtins fixtures/list.pyi] + +[case testWideOuterContextSubClassValues] +from typing import TypeVar + +class A: ... +class B(A): ... + +T = TypeVar('T', B, int) +def f(x: T) -> T: ... +def outer(x: A) -> None: ... + +outer(f(B())) +x: A = f(B()) + +[case testWideOuterContextSubClassValuesGenericReturn] +from typing import TypeVar, Iterable, List + +class A: ... +class B(A): ... + +T = TypeVar('T', B, int) +def f(x: T) -> List[T]: ... +def outer(x: Iterable[A]) -> None: ... + +outer(f(B())) +x: Iterable[A] = f(B()) +[builtins fixtures/list.pyi] + +[case testWideOuterContextSubclassBoundGeneric] +from typing import TypeVar, Generic + +S = TypeVar('S') +class A(Generic[S]): ... +class B(A[S]): ... + +T = TypeVar('T', bound=B[int]) +def f(x: T) -> T: ... +def outer(x: A[int]) -> None: ... + +y: B[int] +outer(f(y)) +x: A[int] = f(y) + +[case testWideOuterContextSubclassBoundGenericCovariant] +from typing import TypeVar, Generic + +S_co = TypeVar('S_co', covariant=True) +class A(Generic[S_co]): ... +class B(A[S_co]): ... + +T = TypeVar('T', bound=B[int]) +def f(x: T) -> T: ... +def outer(x: A[int]) -> None: ... + +y: B[int] +outer(f(y)) +x: A[int] = f(y) + +[case testWideOuterContextSubclassValuesGeneric] +from typing import TypeVar, Generic + +S = TypeVar('S') +class A(Generic[S]): ... +class B(A[S]): ... + +T = TypeVar('T', B[int], int) +def f(x: T) -> T: ... +def outer(x: A[int]) -> None: ... + +y: B[int] +outer(f(y)) +x: A[int] = f(y) + +[case testWideOuterContextSubclassValuesGenericCovariant] +from typing import TypeVar, Generic + +S_co = TypeVar('S_co', covariant=True) +class A(Generic[S_co]): ... +class B(A[S_co]): ... + +T = TypeVar('T', B[int], int) +def f(x: T) -> T: ... +def outer(x: A[int]) -> None: ... + +y: B[int] +outer(f(y)) +x: A[int] = f(y) + +[case testWideOuterContextUnionBound] +from typing import TypeVar, Union + +class A: ... +class B: ... + +T = TypeVar('T', bound=B) +def f(x: T) -> T: ... +def outer(x: Union[A, B]) -> None: ... + +outer(f(B())) +x: Union[A, B] = f(B()) + +[case testWideOuterContextUnionBoundGenericReturn] +from typing import TypeVar, Union, Iterable, List + +class A: ... +class B: ... + +T = TypeVar('T', bound=B) +def f(x: T) -> List[T]: ... +def outer(x: Iterable[Union[A, B]]) -> None: ... + +outer(f(B())) +x: Iterable[Union[A, B]] = f(B()) +[builtins fixtures/list.pyi] + +[case testWideOuterContextUnionValues] +from typing import TypeVar, Union + +class A: ... +class B: ... + +T = TypeVar('T', B, int) +def f(x: T) -> T: ... +def outer(x: Union[A, B]) -> None: ... + +outer(f(B())) +x: Union[A, B] = f(B()) + +[case testWideOuterContextUnionValuesGenericReturn] +from typing import TypeVar, Union, Iterable, List + +class A: ... +class B: ... + +T = TypeVar('T', B, int) +def f(x: T) -> List[T]: ... +def outer(x: Iterable[Union[A, B]]) -> None: ... + +outer(f(B())) +x: Iterable[Union[A, B]] = f(B()) +[builtins fixtures/list.pyi] + +[case testWideOuterContextOptional] +from typing import Optional, Type, TypeVar + +class Custom: + pass + +T = TypeVar('T', bound=Custom) + +def a(x: T) -> Optional[T]: ... + +def b(x: T) -> Optional[T]: + return a(x) + +[case testWideOuterContextOptionalGenericReturn] +from typing import Optional, Type, TypeVar, Iterable + +class Custom: + pass + +T = TypeVar('T', bound=Custom) + +def a(x: T) -> Iterable[Optional[T]]: ... + +def b(x: T) -> Iterable[Optional[T]]: + return a(x) + +[case testWideOuterContextOptionalMethod] +from typing import Optional, Type, TypeVar + +class A: pass +class B: pass + +T = TypeVar('T', A, B) +class C: + def meth_a(self) -> Optional[A]: + return self.meth(A) + + def meth(self, cls: Type[T]) -> Optional[T]: ... + +[case testWideOuterContextValuesOverlapping] +from typing import TypeVar, List + +class A: + pass +class B(A): + pass +class C: + pass + +T = TypeVar('T', A, B, C) +def foo(xs: List[T]) -> T: ... + +S = TypeVar('S', B, C) +def bar(xs: List[S]) -> S: + foo(xs) + return xs[0] +[builtins fixtures/list.pyi] + +[case testWideOuterContextOptionalTypeVarReturn] +from typing import Callable, Iterable, List, Optional, TypeVar + +class C: + x: str + +T = TypeVar('T') +def f(i: Iterable[T], c: Callable[[T], str]) -> Optional[T]: ... + +def g(l: List[C], x: str) -> Optional[C]: + def pred(c: C) -> str: + return c.x + return f(l, pred) +[builtins fixtures/list.pyi] + +[case testWideOuterContextOptionalTypeVarReturnLambda] +from typing import Callable, Iterable, List, Optional, TypeVar + +class C: + x: str + +T = TypeVar('T') +def f(i: Iterable[T], c: Callable[[T], str]) -> Optional[T]: ... + +def g(l: List[C], x: str) -> Optional[C]: + return f(l, lambda c: reveal_type(c).x) # N: Revealed type is "__main__.C" +[builtins fixtures/list.pyi] + +[case testPartialTypeContextWithTwoLambdas] +from typing import Any, Generic, TypeVar, Callable + +def int_to_any(x: int) -> Any: ... +def any_to_int(x: Any) -> int: ... +def any_to_str(x: Any) -> str: ... + +T = TypeVar("T") +class W(Generic[T]): + def __init__( + self, serialize: Callable[[T], Any], deserialize: Callable[[Any], T] + ) -> None: + ... +reveal_type(W(lambda x: int_to_any(x), lambda x: any_to_int(x))) # N: Revealed type is "__main__.W[builtins.int]" +W( + lambda x: int_to_any(x), # E: Argument 1 to "int_to_any" has incompatible type "str"; expected "int" + lambda x: any_to_str(x) +) + +[case testWideOuterContextEmpty] +from typing import List, TypeVar + +T = TypeVar('T', bound=int) +def f(x: List[T]) -> T: ... + +# mypy infers List[Never] here, and Never is a subtype of str +y: str = f([]) +[builtins fixtures/list.pyi] + +[case testWideOuterContextEmptyError] +from typing import List, TypeVar + +T = TypeVar('T', bound=int) +def f(x: List[T]) -> List[T]: ... + +y: List[str] = f([]) +[builtins fixtures/list.pyi] + +[case testWideOuterContextNoArgs] +from typing import TypeVar, Optional + +T = TypeVar('T', bound=int) +def f(x: Optional[T] = None) -> T: ... + +y: str = f() + +[case testWideOuterContextNoArgsError] +from typing import TypeVar, Optional, List + +T = TypeVar('T', bound=int) +def f(x: Optional[T] = None) -> List[T]: ... + +y: List[str] = f() +[builtins fixtures/list.pyi] + +[case testUseCovariantGenericOuterContext] +from typing import TypeVar, Callable, Tuple + +T = TypeVar('T') + +def f(x: Callable[..., T]) -> T: + return x() + +x: Tuple[str, ...] = f(tuple) +[builtins fixtures/tuple.pyi] +[out] + +[case testUseCovariantGenericOuterContextUserDefined] +from typing import TypeVar, Callable, Generic + +T_co = TypeVar('T_co', covariant=True) +T = TypeVar('T') + +class G(Generic[T_co]): ... + +def f(x: Callable[..., T]) -> T: + return x() + +x: G[str] = f(G) +[out] + +[case testConditionalExpressionWithEmptyListAndUnionWithAny] +from typing import Union, List, Any + +def f(x: Union[List[str], Any]) -> None: + a = x if x else [] + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.str], Any, builtins.list[Union[builtins.str, Any]]]" +[builtins fixtures/list.pyi] + +[case testConditionalExpressionWithEmptyIteableAndUnionWithAny] +from typing import Union, Iterable, Any + +def f(x: Union[Iterable[str], Any]) -> None: + a = x if x else [] + reveal_type(a) # N: Revealed type is "Union[typing.Iterable[builtins.str], Any, builtins.list[Union[builtins.str, Any]]]" +[builtins fixtures/list.pyi] + +[case testInferMultipleAnyUnionCovariant] +from typing import Any, Mapping, Sequence, Union + +def foo(x: Union[Mapping[Any, Any], Mapping[Any, Sequence[Any]]]) -> None: + ... +foo({1: 2}) +[builtins fixtures/dict.pyi] + +[case testInferMultipleAnyUnionInvariant] +from typing import Any, Dict, Sequence, Union + +def foo(x: Union[Dict[Any, Any], Dict[Any, Sequence[Any]]]) -> None: + ... +foo({1: 2}) +[builtins fixtures/dict.pyi] + +[case testInferMultipleAnyUnionDifferentVariance] +from typing import Any, Dict, Mapping, Sequence, Union + +def foo(x: Union[Dict[Any, Any], Mapping[Any, Sequence[Any]]]) -> None: + ... +foo({1: 2}) + +def bar(x: Union[Mapping[Any, Any], Dict[Any, Sequence[Any]]]) -> None: + ... +bar({1: 2}) +[builtins fixtures/dict.pyi] + +[case testOptionalTypeNarrowedByGenericCall] +from typing import Dict, Optional + +d: Dict[str, str] = {} + +def foo(arg: Optional[str] = None) -> None: + if arg is None: + arg = d.get("a", "b") + reveal_type(arg) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] + +[case testOptionalTypeNarrowedByGenericCall2] +from typing import Dict, Optional + +d: Dict[str, str] = {} +x: Optional[str] +if x: + reveal_type(x) # N: Revealed type is "builtins.str" + x = d.get(x, x) + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] + +[case testOptionalTypeNarrowedByGenericCall3] +from typing import Generic, TypeVar, Union + +T = TypeVar("T") +def bar(arg: Union[str, T]) -> Union[str, T]: ... + +def foo(arg: Union[str, int]) -> None: + if isinstance(arg, int): + arg = bar("default") + reveal_type(arg) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstance.pyi] + +[case testOptionalTypeNarrowedByGenericCall4] +from typing import Optional, List, Generic, TypeVar + +T = TypeVar("T", covariant=True) +class C(Generic[T]): ... + +x: Optional[C[int]] = None +y = x = C() +reveal_type(y) # N: Revealed type is "__main__.C[builtins.int]" + +[case testOptionalTypeNarrowedByGenericCall5] +from typing import Any, Tuple, Union + +i: Union[Tuple[Any, ...], int] +b: Any +i = i if isinstance(i, int) else b +reveal_type(i) # N: Revealed type is "Union[Any, builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testLambdaInferenceUsesNarrowedTypes] +from typing import Optional, Callable + +def f1(key: Callable[[], str]) -> None: ... +def f2(key: object) -> None: ... + +def g(b: Optional[str]) -> None: + if b: + f1(lambda: reveal_type(b)) # N: Revealed type is "builtins.str" + z: Callable[[], str] = lambda: reveal_type(b) # N: Revealed type is "builtins.str" + f2(lambda: reveal_type(b)) # N: Revealed type is "builtins.str" + lambda: reveal_type(b) # N: Revealed type is "builtins.str" diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test new file mode 100644 index 000000000000..5a99e65c9c90 --- /dev/null +++ b/test-data/unit/check-inference.test @@ -0,0 +1,3886 @@ +-- Inferring locals/globals with simple types +-- ------------------------------------------ + + +[case testInferSimpleGvarType] +class A: pass +class B: pass + +x = A() +y = B() +if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + x = A() +if int(): + x = y # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + x = x +[case testInferSimpleLvarType] +import typing +def f() -> None: + x = A() + y = B() + if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + x = A() + x = y # E: Incompatible types in assignment (expression has type "B", variable has type "A") + x = x +class A: pass +class B: pass +[out] + +[case testLvarInitializedToVoid] +import typing +def f() -> None: + a = g() # E: "g" does not return a value (it only ever returns None) + #b, c = g() # "g" does not return a value (it only ever returns None) TODO + +def g() -> None: pass +[out] + +[case testInferringLvarTypeFromArgument] +import typing +def f(a: 'A') -> None: + b = a + if int(): + b = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + b = a + a = b + +class A: pass +class B: pass +[out] + +[case testInferringLvarTypeFromGvar] + +g: B + +def f() -> None: + a = g + if int(): + a = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = B() + +class A: pass +class B: pass +[out] + +[case testInferringImplicitDynamicTypeForLvar] +import typing +def f() -> None: + a = g() + None(a) # E: "None" not callable + a.x() + +def g(): pass +[out] + +[case testInferringExplicitDynamicTypeForLvar] +from typing import Any +g: Any + +def f(a: Any) -> None: + b = g + None(b) # E: "None" not callable + a.x() +[out] + + +-- Inferring types of local variables with complex types +-- ----------------------------------------------------- + + +[case testInferringTupleTypeForLvar] + +def f() -> None: + a = A(), B() + aa: A + bb: B + if int(): + bb = a[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + aa = a[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") + aa = a[0] + bb = a[1] + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testInferringTupleTypeForLvarWithNones] +import typing +def f() -> None: + a = A(), None + b = None, A() + +class A: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testInferringGenericTypeForLvar] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): pass +a_i: A[int] +a_s: A[str] + +def f() -> None: + a_int = A() # type: A[int] + a = a_int + if int(): + a = a_s # E: Incompatible types in assignment (expression has type "A[str]", variable has type "A[int]") + a = a_i +[builtins fixtures/tuple.pyi] +[out] + +[case testInferringFunctionTypeForLvar] +import typing +def f() -> None: + a = g + a(B()) # E: Argument 1 has incompatible type "B"; expected "A" + a(A()) + +def g(a: 'A') -> None: pass + +class A: pass +class B: pass +[out] + +[case testInferringFunctionTypeForLvarFromTypeObject] +import typing +def f() -> None: + a = A + a(A()) # E: Too many arguments + a() + t = a # type: type + +class A: pass +[out] + + +-- Inferring variable types in multiple definition +-- ----------------------------------------------- + + +[case testInferringLvarTypesInMultiDef] +import typing +def f() -> None: + a, b = A(), B() + if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + + a = A() + b = B() + +class A: pass +class B: pass +[out] + +[case testInferringLvarTypesInTupleAssignment] +from typing import Tuple +def f() -> None: + t: Tuple[A, B] + a, b = t + if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + + a = A() + b = B() + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testInferringLvarTypesInNestedTupleAssignment1] +from typing import Tuple +def f() -> None: + t: Tuple[A, B] + a1, (a, b) = A(), t + if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + + a = A() + b = B() + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testInferringLvarTypesInNestedTupleAssignment2] +import typing +def f() -> None: + a, (b, c) = A(), (B(), C()) + if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + c = A() # E: Incompatible types in assignment (expression has type "A", variable has type "C") + + a = A() + b = B() + c = C() + +class A: pass +class B: pass +class C: pass +[out] + +[case testInferringLvarTypesInNestedListAssignment] +import typing +def f() -> None: + a, (b, c) = A(), [B(), C()] + if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + c = A() # E: Incompatible types in assignment (expression has type "A", variable has type "C") + + a = A() + b = B() + c = C() + +class A: pass +class B: pass +class C: pass +[out] + +[case testInferringLvarTypesInMultiDefWithNoneTypes] +import typing +def f() -> None: + a, b = A(), None + c, d = None, A() + +class A: pass +[out] + +[case testInferringLvarTypesInNestedTupleAssignmentWithNoneTypes] +import typing +def f() -> None: + a1, (a2, b) = A(), (A(), None) + +class A: pass +[out] + +[case testClassObjectsNotUnpackableWithoutIterableMetaclass] +from typing import Type + +class Foo: ... +A: Type[Foo] = Foo +a, b = Foo # E: "Type[Foo]" object is not iterable +c, d = A # E: "Type[Foo]" object is not iterable + +class Meta(type): ... +class Bar(metaclass=Meta): ... +B: Type[Bar] = Bar +e, f = Bar # E: "Type[Bar]" object is not iterable +g, h = B # E: "Type[Bar]" object is not iterable + +reveal_type(a) # E: Cannot determine type of "a" # N: Revealed type is "Any" +reveal_type(b) # E: Cannot determine type of "b" # N: Revealed type is "Any" +reveal_type(c) # E: Cannot determine type of "c" # N: Revealed type is "Any" +reveal_type(d) # E: Cannot determine type of "d" # N: Revealed type is "Any" +reveal_type(e) # E: Cannot determine type of "e" # N: Revealed type is "Any" +reveal_type(f) # E: Cannot determine type of "f" # N: Revealed type is "Any" +reveal_type(g) # E: Cannot determine type of "g" # N: Revealed type is "Any" +reveal_type(h) # E: Cannot determine type of "h" # N: Revealed type is "Any" +[out] + +[case testInferringLvarTypesUnpackedFromIterableClassObject] +from typing import Iterator, Type, TypeVar, Union, overload +class Meta(type): + def __iter__(cls) -> Iterator[int]: + yield from [1, 2, 3] + +class Meta2(type): + def __iter__(cls) -> Iterator[str]: + yield from ["foo", "bar", "baz"] + +class Meta3(type): ... + +class Foo(metaclass=Meta): ... +class Bar(metaclass=Meta2): ... +class Baz(metaclass=Meta3): ... +class Spam: ... + +class Eggs(metaclass=Meta): + @overload + def __init__(self, x: int) -> None: ... + @overload + def __init__(self, x: int, y: int, z: int) -> None: ... + def __init__(self, x: int, y: int = ..., z: int = ...) -> None: ... + +A: Type[Foo] = Foo +B: Type[Union[Foo, Bar]] = Foo +C: Union[Type[Foo], Type[Bar]] = Foo +D: Type[Union[Foo, Baz]] = Foo +E: Type[Union[Foo, Spam]] = Foo +F: Type[Eggs] = Eggs +G: Type[Union[Foo, Eggs]] = Foo + +a, b, c = Foo +d, e, f = A +g, h, i = B +j, k, l = C +m, n, o = D # E: "Type[Baz]" object is not iterable +p, q, r = E # E: "Type[Spam]" object is not iterable +s, t, u = Eggs +v, w, x = F +y, z, aa = G + +for var in [a, b, c, d, e, f, s, t, u, v, w, x, y, z, aa]: + reveal_type(var) # N: Revealed type is "builtins.int" + +for var2 in [g, h, i, j, k, l]: + reveal_type(var2) # N: Revealed type is "Union[builtins.int, builtins.str]" + +for var3 in [m, n, o, p, q, r]: + reveal_type(var3) # N: Revealed type is "Union[builtins.int, Any]" + +T = TypeVar("T", bound=Type[Foo]) + +def check(x: T) -> T: + a, b, c = x + for var in [a, b, c]: + reveal_type(var) # N: Revealed type is "builtins.int" + return x + +T2 = TypeVar("T2", bound=Type[Union[Foo, Bar]]) + +def check2(x: T2) -> T2: + a, b, c = x + for var in [a, b, c]: + reveal_type(var) # N: Revealed type is "Union[builtins.int, builtins.str]" + return x + +T3 = TypeVar("T3", bound=Union[Type[Foo], Type[Bar]]) + +def check3(x: T3) -> T3: + a, b, c = x + for var in [a, b, c]: + reveal_type(var) # N: Revealed type is "Union[builtins.int, builtins.str]" + return x +[out] + +[case testInferringLvarTypesUnpackedFromIterableClassObjectWithGenericIter] +from typing import Iterator, Type, TypeVar + +T = TypeVar("T") +class Meta(type): + def __iter__(self: Type[T]) -> Iterator[T]: ... +class Foo(metaclass=Meta): ... + +A, B, C = Foo +reveal_type(A) # N: Revealed type is "__main__.Foo" +reveal_type(B) # N: Revealed type is "__main__.Foo" +reveal_type(C) # N: Revealed type is "__main__.Foo" +[out] + +[case testInferringLvarTypesInMultiDefWithInvalidTuple] +from typing import Tuple +t: Tuple[object, object, object] + +def f() -> None: + a, b = t # Fail + c, d, e, f = t # Fail + g, h, i = t +[builtins fixtures/tuple.pyi] +[out] +main:5: error: Too many values to unpack (2 expected, 3 provided) +main:6: error: Need more than 3 values to unpack (4 expected) + +[case testInvalidRvalueTypeInInferredMultipleLvarDefinition] +import typing +def f() -> None: + a, b = f # E: "Callable[[], None]" object is not iterable + c, d = A() # E: "A" object is not iterable +class A: pass +[builtins fixtures/for.pyi] +[out] + +[case testInvalidRvalueTypeInInferredNestedTupleAssignment] +import typing +def f() -> None: + a1, (a2, b) = A(), f # E: "Callable[[], None]" object is not iterable + a3, (c, d) = A(), A() # E: "A" object is not iterable +class A: pass +[builtins fixtures/for.pyi] +[out] + +[case testInferringMultipleLvarDefinitionWithListRvalue] +from typing import List + +class C: pass +class D: pass + +def f() -> None: + list_c = [C()] + list_d = [D()] + a, b = list_c + c, d, e = list_d + if int(): + a = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") + b = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") + c = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") + b = c # E: Incompatible types in assignment (expression has type "D", variable has type "C") + + a = C() + b = C() + c = D() + d = D() + e = D() + + a = b + c = d + d = e +[builtins fixtures/for.pyi] +[out] + +[case testInferringNestedTupleAssignmentWithListRvalue] +from typing import List + +class C: pass +class D: pass + +def f() -> None: + list_c = [C()] + list_d = [D()] + c1, (a, b) = C(), list_c + c2, (c, d, e) = C(), list_d + if int(): + a = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") + b = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") + c = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") + b = c # E: Incompatible types in assignment (expression has type "D", variable has type "C") + + a = C() + b = C() + c = D() + d = D() + e = D() + + a = b + c = d + d = e +[builtins fixtures/for.pyi] +[out] + +[case testInferringMultipleLvarDefinitionWithImplicitDynamicRvalue] +import typing +def f() -> None: + a, b = g() + a.x + b.x +def g(): pass + +[case testInferringMultipleLvarDefinitionWithExplicitDynamicRvalue] +from typing import Any +def f(d: Any) -> None: + a, b = d + a.x + b.x + +[case testInferringTypesFromIterable] +from typing import Iterable +class Nums(Iterable[int]): + def __iter__(self): pass + def __next__(self): pass +a, b = Nums() +reveal_type(a) # N: Revealed type is "builtins.int" +reveal_type(b) # N: Revealed type is "builtins.int" +if int(): + a = b = 1 +if int(): + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + b = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/for.pyi] + +[case testInferringTypesFromIterableStructuralSubtyping1] +from typing import Iterator +class Nums: + def __iter__(self) -> Iterator[int]: pass +a, b = Nums() +reveal_type(a) # N: Revealed type is "builtins.int" +reveal_type(b) # N: Revealed type is "builtins.int" +if int(): + a = b = 1 +if int(): + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + b = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/for.pyi] + +[case testInferringTypesFromIterableStructuralSubtyping2] +from typing import Self +class Nums: + def __iter__(self) -> Self: pass + def __next__(self) -> int: pass +a, b = Nums() +reveal_type(a) # N: Revealed type is "builtins.int" +reveal_type(b) # N: Revealed type is "builtins.int" +if int(): + a = b = 1 +if int(): + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + b = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] + + +-- Type variable inference for generic functions +-- --------------------------------------------- + + +[case testInferSimpleGenericFunction] +from typing import Tuple, TypeVar +T = TypeVar('T') +a: A +b: B +c: Tuple[A, object] + +def id(a: T) -> T: pass + +if int(): + b = id(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = id(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = id(c) # E: Incompatible types in assignment (expression has type "Tuple[A, object]", variable has type "A") + +if int(): + a = id(a) + b = id(b) + c = id(c) + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testInferringGenericFunctionTypeForLvar] +from typing import TypeVar +T = TypeVar('T') +def f() -> None: + a = id + b: int + c: str + if int(): + b = a(c) # E: Incompatible types in assignment (expression has type "str", variable has type "int") + b = a(b) + c = a(c) +def id(x: T) -> T: + return x +[out] + +[case testUnderspecifiedInferenceResult] +# flags: --no-strict-optional +from typing import TypeVar +T = TypeVar('T') +class A: pass +a: A + +def ff() -> None: + x = f() # E: Need type annotation for "x" + reveal_type(x) # N: Revealed type is "Any" + +def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same TypeVar +def g(a: T) -> None: pass + +g(None) # Ok +f() # Ok because not used to infer local variable type +g(a) +[out] + +[case testInferenceWithMultipleConstraints] +from typing import TypeVar + +class A: pass +class B(A): pass + +T = TypeVar('T') +a: A +b: B + +def f(a: T, b: T) -> T: pass + +if int(): + b = f(a, b) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = f(b, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = f(a, b) +if int(): + a = f(b, a) + +[case testInferenceWithMultipleVariables] +from typing import Tuple, TypeVar +T = TypeVar('T') +S = TypeVar('S') + +def f(a: T, b: S) -> Tuple[T, S]: pass + +class A: pass +class B: pass + +a: A +b: B +taa: Tuple[A, A] +tab: Tuple[A, B] +tba: Tuple[B, A] + +if int(): + taa = f(a, b) # E: Argument 2 to "f" has incompatible type "B"; expected "A" +if int(): + taa = f(b, a) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +if int(): + tba = f(a, b) # E: Argument 1 to "f" has incompatible type "A"; expected "B" \ + # E: Argument 2 to "f" has incompatible type "B"; expected "A" + +if int(): + tab = f(a, b) +if int(): + tba = f(b, a) +[builtins fixtures/tuple.pyi] + +[case testConstraintSolvingWithSimpleGenerics] +from typing import TypeVar, Generic +T = TypeVar('T') +ao: A[object] +ab: A[B] +ac: A[C] + +def f(a: 'A[T]') -> 'A[T]': pass + +def g(a: T) -> T: pass + +class A(Generic[T]): pass +class B: pass +class C: pass + +if int(): + ab = f(ao) # E: Argument 1 to "f" has incompatible type "A[object]"; expected "A[B]" + ao = f(ab) # E: Argument 1 to "f" has incompatible type "A[B]"; expected "A[object]" +if int(): + ab = f(ac) # E: Argument 1 to "f" has incompatible type "A[C]"; expected "A[B]" +if int(): + ab = g(ao) # E: Argument 1 to "g" has incompatible type "A[object]"; expected "A[B]" + ao = g(ab) # E: Argument 1 to "g" has incompatible type "A[B]"; expected "A[object]" + +if int(): + ab = f(ab) + ac = f(ac) + ao = f(ao) + +if int(): + ab = g(ab) + ao = g(ao) +[case testConstraintSolvingFailureWithSimpleGenerics] +from typing import TypeVar, Generic +T = TypeVar('T') +ao: A[object] +ab: A[B] + +def f(a: 'A[T]', b: 'A[T]') -> None: pass + +class A(Generic[T]): pass +class B: pass + + +f(ao, ab) # E: Cannot infer type argument 1 of "f" +f(ab, ao) # E: Cannot infer type argument 1 of "f" +f(ao, ao) +f(ab, ab) + +[case testTypeInferenceWithCalleeDefaultArgs] +# flags: --no-strict-optional +from typing import TypeVar +T = TypeVar('T') +a = None # type: A +o = None # type: object + +def f(a: T = None) -> T: pass +def g(a: T, b: T = None) -> T: pass + +class A: pass + +if int(): + a = f(o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") +if int(): + a = g(a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") + +if int(): + o = f() +if int(): + o = f(o) +if int(): + a = f(a) +if int(): + a = g(a) + + +-- Generic function inference with multiple inheritance +-- ---------------------------------------------------- + + +[case testGenericFunctionInferenceWithMultipleInheritance] +from typing import TypeVar + +class I: pass +class J: pass + +class A(I, J): pass +class B(I, J): pass +class C(I): pass +class D(J): pass + +T = TypeVar('T') +def f(a: T, b: T) -> T: pass +def g(x: I) -> None: pass + +a = f(A(), C()) +g(a) +b = f(A(), B()) +g(b) +c = f(A(), D()) +g(c) # E: Argument 1 to "g" has incompatible type "J"; expected "I" +d = f(D(), A()) +g(d) # E: Argument 1 to "g" has incompatible type "J"; expected "I" +e = f(D(), C()) +g(e) # E: Argument 1 to "g" has incompatible type "object"; expected "I" + +[case testGenericFunctionInferenceWithMultipleInheritance2] +from typing import TypeVar + +class I: pass +class J: pass + +class A(I): pass +class B(A, J): pass +class C(I, J): pass + +T = TypeVar('T') +def f(a: T, b: T) -> T: pass +def g(x: I) -> None: pass +def h(x: J) -> None: pass + +a = f(B(), C()) +g(a) +h(a) # E: Argument 1 to "h" has incompatible type "I"; expected "J" +b = f(C(), B()) +g(b) +h(b) # E: Argument 1 to "h" has incompatible type "I"; expected "J" +c = f(A(), B()) +g(a) +h(b) # E: Argument 1 to "h" has incompatible type "I"; expected "J" + +[case testGenericFunctionInferenceWithMultipleInheritance3] +from typing import TypeVar + +class I: pass +class J: pass +class K(J): pass + +class A(K): pass +class B(A, I): pass +class C(I, J): pass + +T = TypeVar('T') +def f(a: T, b: T) -> T: pass +def g(x: K) -> None: pass + +a = f(B(), C()) +g(a) # E: Argument 1 to "g" has incompatible type "J"; expected "K" +b = f(A(), C()) +g(b) # E: Argument 1 to "g" has incompatible type "J"; expected "K" +c = f(A(), B()) +g(c) + +[case testPrecedenceOfFirstBaseAsInferenceResult] +from typing import TypeVar +from abc import abstractmethod, ABCMeta +class A: pass +class B(A, I, J): pass +class C(A, I, J): pass + +def f(a: T, b: T) -> T: pass + +T = TypeVar('T') +a: A +i: I +j: J + +a = f(B(), C()) + +class I(metaclass=ABCMeta): pass +class J(metaclass=ABCMeta): pass + + +[builtins fixtures/tuple.pyi] + + +-- Generic function inference with function arguments +-- -------------------------------------------------- + + +[case testNonOverloadedMapInference] +from typing import TypeVar, Callable, List +t = TypeVar('t') +s = TypeVar('s') +class A: pass +b = bool() +def f(x: bool) -> A: pass +def mymap(f: Callable[[t], s], a: List[t]) -> List[s]: pass + +l = mymap(f, [b]) +if int(): + l = [A()] +lb = [b] +if int(): + l = lb # E: Incompatible types in assignment (expression has type "List[bool]", variable has type "List[A]") +[builtins fixtures/for.pyi] + +[case testGenericFunctionWithTypeTypeAsCallable] +from typing import Callable, Type, TypeVar +T = TypeVar('T') +def f(x: Callable[..., T]) -> T: return x() +class A: pass +x: Type[A] +y = f(x) +reveal_type(y) # N: Revealed type is "__main__.A" + +-- Generic function inference with unions +-- -------------------------------------- + + +[case testUnionInference] +from typing import TypeVar, Union, List +T = TypeVar('T') +U = TypeVar('U') +def f(x: Union[T, int], y: T) -> T: pass +f(1, 'a')() # E: "str" not callable +f('a', 1)() # E: "object" not callable +f('a', 'a')() # E: "str" not callable +f(1, 1)() # E: "int" not callable + +def g(x: Union[T, List[T]]) -> List[T]: pass +def h(x: List[str]) -> None: pass +g('a')() # E: "List[str]" not callable + +# The next line is a case where there are multiple ways to satisfy a constraint +# involving a Union. Either T = List[str] or T = str would turn out to be valid, +# but mypy doesn't know how to branch on these two options (and potentially have +# to backtrack later) and defaults to T = Never. The result is an +# awkward error message. Either a better error message, or simply accepting the +# call, would be preferable here. +g(['a']) # E: Argument 1 to "g" has incompatible type "List[str]"; expected "List[Never]" + +h(g(['a'])) + +def i(x: Union[List[T], List[U]], y: List[T], z: List[U]) -> None: pass +a = [1] +b = ['b'] +i(a, a, b) +i(b, a, b) +i(a, b, b) # E: Argument 1 to "i" has incompatible type "List[int]"; expected "List[str]" +[builtins fixtures/list.pyi] + +[case testCallableListJoinInference] +from typing import Any, Callable + +def fun() -> None: + callbacks = [ + callback1, + callback2, + ] + + for c in callbacks: + call(c, 1234) # this must not fail + +def callback1(i: int) -> int: + return i +def callback2(i: int) -> str: + return 'hello' +def call(c: Callable[[int], Any], i: int) -> None: + c(i) +[builtins fixtures/list.pyi] +[out] + +[case testCallableMeetAndJoin] +from typing import Callable, Any, TypeVar + +class A: ... +class B(A): ... + +def f(c: Callable[[B], int]) -> None: ... + +c: Callable[[A], int] +d: Callable[[B], int] + +lst = [c, d] +reveal_type(lst) # N: Revealed type is "builtins.list[def (__main__.B) -> builtins.int]" + +T = TypeVar('T') +def meet_test(x: Callable[[T], int], y: Callable[[T], int]) -> T: ... + +CA = Callable[[A], A] +CB = Callable[[B], B] + +ca: Callable[[CA], int] +cb: Callable[[CB], int] +reveal_type(meet_test(ca, cb)) # N: Revealed type is "def (__main__.A) -> __main__.B" +[builtins fixtures/list.pyi] +[out] + +[case testUnionInferenceWithTypeVarValues] +from typing import TypeVar, Union +AnyStr = TypeVar('AnyStr', bytes, str) +def f(x: Union[AnyStr, int], *a: AnyStr) -> None: pass +f('foo') +f('foo', 'bar') +f('foo', b'bar') # E: Value of type variable "AnyStr" of "f" cannot be "Sequence[object]" +f(1) +f(1, 'foo') +f(1, 'foo', b'bar') # E: Value of type variable "AnyStr" of "f" cannot be "Sequence[object]" +[builtins fixtures/primitives.pyi] + + +[case testUnionTwoPassInference-skip] +from typing import TypeVar, Union, List +T = TypeVar('T') +U = TypeVar('U') +def j(x: Union[List[T], List[U]], y: List[T]) -> List[U]: pass + +a = [1] +b = ['b'] +# We could infer: Since List[str] <: List[T], we must have T = str. +# Then since List[int] <: Union[List[str], List[U]], and List[int] is +# not a subtype of List[str], we must have U = int. +# This is not currently implemented. +j(a, b) +[builtins fixtures/list.pyi] + + +[case testUnionContext] +from typing import TypeVar, Union, List +T = TypeVar('T') +def f() -> List[T]: pass +d1 = f() # type: Union[List[int], str] +d2 = f() # type: Union[int, str] # E: Incompatible types in assignment (expression has type "List[Never]", variable has type "Union[int, str]") +def g(x: T) -> List[T]: pass +d3 = g(1) # type: Union[List[int], List[str]] +[builtins fixtures/list.pyi] + + +[case testGenericFunctionSubtypingWithUnions] +from typing import TypeVar, Union, List +T = TypeVar('T') +S = TypeVar('S') +def k1(x: int, y: List[T]) -> List[Union[T, int]]: pass +def k2(x: S, y: List[T]) -> List[Union[T, int]]: pass +a = k2 +if int(): + a = k2 +if int(): + a = k1 # E: Incompatible types in assignment (expression has type "Callable[[int, List[T@k1]], List[Union[T@k1, int]]]", variable has type "Callable[[S, List[T@k2]], List[Union[T@k2, int]]]") +b = k1 +if int(): + b = k1 +if int(): + b = k2 +[builtins fixtures/list.pyi] + +[case testAmbiguousUnionContextAndMultipleInheritance] +from typing import TypeVar, Union, Generic + +_T = TypeVar('_T') + +class T(Generic[_T]): pass +class U(Generic[_T]): pass +class V(T[_T], U[_T]): pass + +def wait_for(fut: Union[T[_T], U[_T]]) -> _T: ... + +reveal_type(wait_for(V[str]())) # N: Revealed type is "builtins.str" + +[case testAmbiguousUnionContextAndMultipleInheritance2] +from typing import TypeVar, Union, Generic + +_T = TypeVar('_T') +_S = TypeVar('_S') + +class T(Generic[_T, _S]): pass +class U(Generic[_T, _S]): pass +class V(T[_T, _S], U[_T, _S]): pass + +def wait_for(fut: Union[T[_T, _S], U[_T, _S]]) -> T[_T, _S]: ... + +reveal_type(wait_for(V[int, str]())) \ + # N: Revealed type is "__main__.T[builtins.int, builtins.str]" + + +-- Literal expressions +-- ------------------- + + +[case testDictLiteral] +from typing import Dict +class A: pass +class B: pass +def d_ab() -> Dict[A, B]: return {} +def d_aa() -> Dict[A, A]: return {} +a: A +b: B +d = {a:b} +if int(): + d = d_ab() +if int(): + d = d_aa() # E: Incompatible types in assignment (expression has type "Dict[A, A]", variable has type "Dict[A, B]") +[builtins fixtures/dict.pyi] + +[case testSetLiteral] +from typing import Any, Set +a: int +x: Any +def s_i() -> Set[int]: return set() +def s_s() -> Set[str]: return set() +s = {a} +if int(): + s = {x} +if int(): + s = s_i() +if int(): + s = s_s() # E: Incompatible types in assignment (expression has type "Set[str]", variable has type "Set[int]") +[builtins fixtures/set.pyi] + +[case testSetWithStarExpr] +s = {1, 2, *(3, 4)} +t = {1, 2, *s} +reveal_type(s) # N: Revealed type is "builtins.set[builtins.int]" +reveal_type(t) # N: Revealed type is "builtins.set[builtins.int]" +[builtins fixtures/set.pyi] + +[case testListLiteralWithFunctionsErasesNames] +def f1(x: int) -> int: ... +def g1(y: int) -> int: ... +def h1(x: int) -> int: ... + +list_1 = [f1, g1] +list_2 = [f1, h1] +reveal_type(list_1) # N: Revealed type is "builtins.list[def (builtins.int) -> builtins.int]" +reveal_type(list_2) # N: Revealed type is "builtins.list[def (x: builtins.int) -> builtins.int]" + +def f2(x: int, z: str) -> int: ... +def g2(y: int, z: str) -> int: ... +def h2(x: int, z: str) -> int: ... + +list_3 = [f2, g2] +list_4 = [f2, h2] +reveal_type(list_3) # N: Revealed type is "builtins.list[def (builtins.int, z: builtins.str) -> builtins.int]" +reveal_type(list_4) # N: Revealed type is "builtins.list[def (x: builtins.int, z: builtins.str) -> builtins.int]" +[builtins fixtures/list.pyi] + +[case testListLiteralWithSimilarFunctionsErasesName] +from typing import Union + +class A: ... +class B(A): ... +class C: ... +class D: ... + +def f(x: Union[A, C], y: B) -> A: ... +def g(z: Union[B, D], y: A) -> B: ... +def h(x: Union[B, D], y: A) -> B: ... + +list_1 = [f, g] +list_2 = [f, h] +reveal_type(list_1) # N: Revealed type is "builtins.list[def (__main__.B, y: __main__.B) -> __main__.A]" +reveal_type(list_2) # N: Revealed type is "builtins.list[def (x: __main__.B, y: __main__.B) -> __main__.A]" +[builtins fixtures/list.pyi] + +[case testListLiteralWithNameOnlyArgsDoesNotEraseNames] +def f(*, x: int) -> int: ... +def g(*, y: int) -> int: ... +def h(*, x: int) -> int: ... + +list_1 = [f, g] # E: List item 0 has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[NamedArg(int, 'y')], int]" +list_2 = [f, h] +[builtins fixtures/list.pyi] + + +-- For statements +-- -------------- + + +[case testInferenceOfFor1] +a: A +b: B + +class A: pass +class B: pass + +for x in [A()]: + b = x # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = x + +for y in []: # E: Need type annotation for "y" + a = y + reveal_type(y) # N: Revealed type is "Any" +[builtins fixtures/for.pyi] + +[case testInferenceOfFor2] +class A: pass +class B: pass +class C: pass + +a: A +b: B +c: C +for x, (y, z) in [(A(), (B(), C()))]: + b = x # E: Incompatible types in assignment (expression has type "A", variable has type "B") + c = y # E: Incompatible types in assignment (expression has type "B", variable has type "C") + a = z # E: Incompatible types in assignment (expression has type "C", variable has type "A") + a = x + b = y + c = z +for xx, yy, zz in [(A(), B())]: # E: Need more than 2 values to unpack (3 expected) + pass +for xx, (yy, zz) in [(A(), B())]: # E: "B" object is not iterable + pass +for xxx, yyy in [(None, None)]: + pass +[builtins fixtures/for.pyi] + +[case testInferenceOfFor3] +class A: pass +class B: pass + +a: A +b: B + +for x, y in [[A()]]: + b = x # E: Incompatible types in assignment (expression has type "A", variable has type "B") + b = y # E: Incompatible types in assignment (expression has type "A", variable has type "B") + a = x + a = y + +for e, f in [[]]: # E: Need type annotation for "e" \ + # E: Need type annotation for "f" + reveal_type(e) # N: Revealed type is "Any" + reveal_type(f) # N: Revealed type is "Any" + +[builtins fixtures/for.pyi] + +[case testForStatementInferenceWithVoid] +def f() -> None: pass + +for x in f(): # E: "f" does not return a value (it only ever returns None) + pass +[builtins fixtures/for.pyi] + +[case testReusingInferredForIndex] +import typing + +class A: pass +class B: pass + +for a in [A()]: pass +a = A() +if int(): + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + for a in []: pass + a = A() + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[builtins fixtures/for.pyi] + +[case testReusingInferredForIndex2] +# flags: --allow-redefinition + +def f() -> None: + for a in [A()]: pass + a = A() + a + if int(): + a = B() \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + for a in []: pass # E: Need type annotation for "a" + a = A() + if int(): + a = B() \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") +class A: pass +class B: pass +[builtins fixtures/for.pyi] +[out] + +[case testReusingInferredForIndex3] +# flags: --disallow-redefinition +def f() -> None: + for a in [A()]: pass + a = A() + a + if int(): + a = B() \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + for a in []: pass + a = A() + if int(): + a = B() \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") +class A: pass +class B: pass +[builtins fixtures/for.pyi] +[out] + +[case testForStatementIndexNarrowing] +from typing_extensions import TypedDict + +class X(TypedDict): + hourly: int + daily: int + +x: X +for a in ("hourly", "daily"): + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" + reveal_type(x[a]) # N: Revealed type is "builtins.int" + reveal_type(a.upper()) # N: Revealed type is "builtins.str" + c = a + reveal_type(c) # N: Revealed type is "builtins.str" + a = "monthly" + reveal_type(a) # N: Revealed type is "builtins.str" + a = "yearly" + reveal_type(a) # N: Revealed type is "builtins.str" + a = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(a) # N: Revealed type is "builtins.str" + d = a + reveal_type(d) # N: Revealed type is "builtins.str" + +b: str +for b in ("hourly", "daily"): + reveal_type(b) # N: Revealed type is "builtins.str" + reveal_type(b.upper()) # N: Revealed type is "builtins.str" +[builtins fixtures/for.pyi] + + +-- Regression tests +-- ---------------- + + +[case testMultipleAssignmentWithPartialDefinition] +a: A +if int(): + x, a = a, a + if int(): + x = a + a = x + if int(): + x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") + a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +class A: pass + +[case testMultipleAssignmentWithPartialDefinition2] +a: A +if int(): + a, x = [a, a] + if int(): + x = a + a = x + if int(): + x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") + a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +class A: pass +[builtins fixtures/for.pyi] + +[case testMultipleAssignmentWithPartialDefinition3] +from typing import Any, cast +a: A +if int(): + x, a = cast(Any, a) + if int(): + x = a + a = x + if int(): + x = object() + a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +class A: pass + +[case testInferGlobalDefinedInBlock] +class A: pass +class B: pass + +if int(): + a = A() + if int(): + a = A() + if int(): + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testAssigningAnyStrToNone] +from typing import Tuple, TypeVar +AnyStr = TypeVar('AnyStr', str, bytes) + +def f(x: AnyStr) -> Tuple[AnyStr]: pass +x = None +(x,) = f('') +reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + + +-- Inferring attribute types +-- ------------------------- + + +[case testInferAttributeType] +import typing +class A: + a = B() +class B: pass + +A().a = B() +A().a = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testInferAttributeTypeAndAssignInInit] +import typing +class A: + a = B() + def __init__(self) -> None: + self.a = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + self.a = B() +class B: pass +[out] + +[case testInferAttributeInInit] +import typing +class B: pass +class A: + def __init__(self) -> None: + self.a = A() + self.b = B() +a = A() +a.a = A() +a.b = B() +a.a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a.b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testInferAttributeInInitUsingChainedAssignment] +import typing +class B: pass +class A: + def __init__(self) -> None: + self.a = self.b = A() +a = A() +a.a = A() +a.b = A() +a.a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a.b = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + + +-- Lambdas +-- ------- + + +[case testInferLambdaType] +from typing import List, Callable +li = [1] +l = lambda: li +f1 = l # type: Callable[[], List[int]] +f2 = l # type: Callable[[], List[str]] # E: Incompatible types in assignment (expression has type "Callable[[], List[int]]", variable has type "Callable[[], List[str]]") +[builtins fixtures/list.pyi] + +[case testInferLambdaType2] +from typing import List, Callable +l = lambda: [B()] +f1 = l # type: Callable[[], List[B]] +f2 = l # type: Callable[[], List[A]] # E: Incompatible types in assignment (expression has type "Callable[[], List[B]]", variable has type "Callable[[], List[A]]") + +class A: pass +class B: pass +[builtins fixtures/list.pyi] + +[case testUninferableLambda] +# flags: --new-type-inference +from typing import TypeVar, Callable +X = TypeVar('X') +def f(x: Callable[[X], X]) -> X: pass +y = f(lambda x: x) # E: Need type annotation for "y" + +[case testUninferableLambdaWithTypeError] +# flags: --new-type-inference +from typing import TypeVar, Callable +X = TypeVar('X') +def f(x: Callable[[X], X], y: str) -> X: pass +y = f(lambda x: x, 1) # E: Need type annotation for "y" \ + # E: Argument 2 to "f" has incompatible type "int"; expected "str" + +[case testInferLambdaNone] +# flags: --no-strict-optional +from typing import Callable +def f(x: Callable[[], None]) -> None: pass +def g(x: Callable[[], int]) -> None: pass +a = lambda: None +f(a) +g(a) +b = lambda: None # type: Callable[[], None] +f(b) +g(b) + +[case testLambdaDefaultContext] +from typing import Callable +def f(a: Callable[..., None] = lambda *a, **k: None): + pass + +def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible default for argument "a" (default has type "Callable[[VarArg(Any), KwArg(Any)], int]", argument has type "Callable[..., None]") + pass +[builtins fixtures/dict.pyi] + +[case testLambdaVarargContext] +# Should not crash +from typing import Callable +def f(a: Callable[[int, int, int], int] = lambda *a, **k: 1): + pass +[builtins fixtures/dict.pyi] + +[case testLambdaDeferredSpecialCase] +from typing import Callable + +class A: + def f(self) -> None: + h(lambda: self.x) + + def g(self) -> None: + self.x = 1 + +def h(x: Callable[[], int]) -> None: + pass + +[case testLambdaJoinWithDynamicConstructor] +from typing import Any, Union + +class Wrapper: + def __init__(self, x: Any) -> None: ... + +def f(cond: bool) -> Any: + f = Wrapper if cond else lambda x: x + reveal_type(f) # N: Revealed type is "Union[def (x: Any) -> __main__.Wrapper, def (x: Any) -> Any]" + return f(3) + +def g(cond: bool) -> Any: + f = lambda x: x if cond else Wrapper + reveal_type(f) # N: Revealed type is "def (x: Any) -> Union[Any, def (x: Any) -> __main__.Wrapper]" + return f(3) + +def h(cond: bool) -> Any: + f = (lambda x: x) if cond else Wrapper + reveal_type(f) # N: Revealed type is "Union[def (x: Any) -> Any, def (x: Any) -> __main__.Wrapper]" + return f(3) + +-- Boolean operators +-- ----------------- + +[case testOrOperationWithGenericOperands] +from typing import List +a: List[A] +o: List[object] +a2 = a or [] +if int(): + a = a2 + a2 = o # E: Incompatible types in assignment (expression has type "List[object]", variable has type "List[A]") +class A: pass +[builtins fixtures/list.pyi] + + +-- Accessing variable before its type has been inferred +-- ---------------------------------------------------- + + +[case testAccessGlobalVarBeforeItsTypeIsAvailable] +import typing +x.y # E: Cannot determine type of "x" # E: Name "x" is used before definition +x = object() +x.y # E: "object" has no attribute "y" + +[case testAccessDataAttributeBeforeItsTypeIsAvailable] + +a: A +a.x.y # E: Cannot determine type of "x" +class A: + def __init__(self) -> None: + self.x = object() +a.x.y # E: "object" has no attribute "y" + + +-- Ducktype declarations +-- --------------------- + + +[case testListWithDucktypeCompatibility] +from typing import List, _promote +class A: pass +@_promote(A) +class B: pass +a: List[A] +x1 = [A(), B()] +x2 = [B(), A()] +x3 = [B(), B()] +if int(): + a = x1 +if int(): + a = x2 +if int(): + a = x3 \ + # E: Incompatible types in assignment (expression has type "List[B]", variable has type "List[A]") \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] + +[case testListWithDucktypeCompatibilityAndTransitivity] +from typing import List, _promote +class A: pass +@_promote(A) +class B: pass +@_promote(B) +class C: pass +a: List[A] +x1 = [A(), C()] +x2 = [C(), A()] +x3 = [B(), C()] +if int(): + a = x1 +if int(): + a = x2 +if int(): + a = x3 \ + # E: Incompatible types in assignment (expression has type "List[B]", variable has type "List[A]") \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] + + +-- Inferring type of variable when initialized to an empty collection +-- ------------------------------------------------------------------ + + +[case testInferListInitializedToEmpty] +a = [] +a.append(1) +a.append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyUsingUpdate] +a = [] +a.extend(['']) +a.append(0) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "str" +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndNotAnnotated] +a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndReadBeforeAppend] +a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") +if a: pass +a.xyz # E: "List[Any]" has no attribute "xyz" +a.append('') +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndIncompleteTypeInAppend] +a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") +a.append([]) +a() # E: "List[Any]" not callable +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndMultipleAssignment] +a, b = [], [] +a.append(1) +b.append('') +a() # E: "List[int]" not callable +b() # E: "List[str]" not callable +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyInFunction] +def f() -> None: + a = [] + a.append(1) + a.append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndNotAnnotatedInFunction] +def f() -> None: + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + +def g() -> None: pass + +a = [] +a.append(1) +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndReadBeforeAppendInFunction] +def f() -> None: + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + if a: pass + a.xyz # E: "List[Any]" has no attribute "xyz" + a.append('') +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyInClassBody] +class A: + a = [] + a.append(1) + a.append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndNotAnnotatedInClassBody] +class A: + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + +class B: + a = [] + a.append(1) +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyInMethod] +class A: + def f(self) -> None: + a = [] + a.append(1) + a.append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndNotAnnotatedInMethod] +class A: + def f(self) -> None: + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyInMethodViaAttribute] +class A: + def f(self) -> None: + # Attributes aren't supported right now. + self.a = [] + self.a.append(1) + self.a.append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyInClassBodyAndOverriden] +from typing import List + +class A: + def __init__(self) -> None: + self.x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") + +class B(A): + @property + def x(self) -> List[int]: # E: Cannot override writeable attribute with read-only property + return [123] +[builtins fixtures/list.pyi] + +[case testInferSetInitializedToEmpty] +a = set() +a.add(1) +a.add('') # E: Argument 1 to "add" of "set" has incompatible type "str"; expected "int" +[builtins fixtures/set.pyi] + +[case testInferSetInitializedToEmptyUsingDiscard] +a = set() +a.discard('') +a.add(0) # E: Argument 1 to "add" of "set" has incompatible type "int"; expected "str" +[builtins fixtures/set.pyi] + +[case testInferSetInitializedToEmptyUsingUpdate] +a = set() +a.update({0}) +a.add('') # E: Argument 1 to "add" of "set" has incompatible type "str"; expected "int" +[builtins fixtures/set.pyi] + +[case testInferDictInitializedToEmpty] +a = {} +a[1] = '' +a() # E: "Dict[int, str]" not callable +[builtins fixtures/dict.pyi] + +[case testInferDictInitializedToEmptyUsingUpdate] +a = {} +a.update({'': 42}) +a() # E: "Dict[str, int]" not callable +[builtins fixtures/dict.pyi] + +[case testInferDictInitializedToEmptyUsingUpdateError] +a = {} # E: Need type annotation for "a" (hint: "a: Dict[, ] = ...") +a.update([1, 2]) # E: Argument 1 to "update" of "dict" has incompatible type "List[int]"; expected "SupportsKeysAndGetItem[Any, Any]" \ + # N: "list" is missing following "SupportsKeysAndGetItem" protocol member: \ + # N: keys +a() # E: "Dict[Any, Any]" not callable +[builtins fixtures/dict.pyi] + +[case testInferDictInitializedToEmptyAndIncompleteTypeInUpdate] +a = {} # E: Need type annotation for "a" (hint: "a: Dict[, ] = ...") +a[1] = {} +b = {} # E: Need type annotation for "b" (hint: "b: Dict[, ] = ...") +b[{}] = 1 +[builtins fixtures/dict.pyi] + +[case testInferDictInitializedToEmptyAndUpdatedFromMethod] +map = {} +def add() -> None: + map[1] = 2 +[builtins fixtures/dict.pyi] + +[case testInferDictInitializedToEmptyAndUpdatedFromMethodUnannotated] +map = {} +def add(): + map[1] = 2 +[builtins fixtures/dict.pyi] + +[case testSpecialCaseEmptyListInitialization] +def f(blocks: Any): # E: Name "Any" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Any") + to_process = [] + to_process = list(blocks) +[builtins fixtures/list.pyi] + +[case testSpecialCaseEmptyListInitialization2] +def f(blocks: object): + to_process = [] + to_process = list(blocks) # E: No overload variant of "list" matches argument type "object" \ + # N: Possible overload variants: \ + # N: def [T] __init__(self) -> List[T] \ + # N: def [T] __init__(self, x: Iterable[T]) -> List[T] +[builtins fixtures/list.pyi] + +[case testInferListInitializedToEmptyAndAssigned] +a = [] +if bool(): + a = [1] +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +def f(): + return [1] +b = [] +if bool(): + b = f() +reveal_type(b) # N: Revealed type is "builtins.list[Any]" + +d = {} +if bool(): + d = {1: 'x'} +reveal_type(d) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +dd = {} # E: Need type annotation for "dd" (hint: "dd: Dict[, ] = ...") +if bool(): + dd = [1] # E: Incompatible types in assignment (expression has type "List[int]", variable has type "Dict[Any, Any]") +reveal_type(dd) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testInferOrderedDictInitializedToEmpty] +from collections import OrderedDict + +o = OrderedDict() +o[1] = 'x' +reveal_type(o) # N: Revealed type is "collections.OrderedDict[builtins.int, builtins.str]" + +d = {1: 'x'} +oo = OrderedDict() +oo.update(d) +reveal_type(oo) # N: Revealed type is "collections.OrderedDict[builtins.int, builtins.str]" +[builtins fixtures/dict.pyi] + +[case testEmptyCollectionAssignedToVariableTwiceIncremental] +x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +y = x +x = [] +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +d = {} # E: Need type annotation for "d" (hint: "d: Dict[, ] = ...") +z = d +d = {} +reveal_type(d) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] +[out2] +main:1: error: Need type annotation for "x" (hint: "x: List[] = ...") +main:4: note: Revealed type is "builtins.list[Any]" +main:5: error: Need type annotation for "d" (hint: "d: Dict[, ] = ...") +main:8: note: Revealed type is "builtins.dict[Any, Any]" + +[case testEmptyCollectionAssignedToVariableTwiceNoReadIncremental] +x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +x = [] +[builtins fixtures/list.pyi] +[out2] +main:1: error: Need type annotation for "x" (hint: "x: List[] = ...") + +[case testInferAttributeInitializedToEmptyAndAssigned] +class C: + def __init__(self) -> None: + self.a = [] + if bool(): + self.a = [1] +reveal_type(C().a) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testInferAttributeInitializedToEmptyAndAppended] +class C: + def __init__(self) -> None: + self.a = [] + if bool(): + self.a.append(1) +reveal_type(C().a) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testInferAttributeInitializedToEmptyAndAssignedItem] +class C: + def __init__(self) -> None: + self.a = {} + if bool(): + self.a[0] = 'yes' +reveal_type(C().a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" +[builtins fixtures/dict.pyi] + +[case testInferAttributeInitializedToNoneAndAssigned] +class C: + def __init__(self) -> None: + self.a = None + if bool(): + self.a = 1 +reveal_type(C().a) # N: Revealed type is "Union[builtins.int, None]" + +[case testInferAttributeInitializedToEmptyNonSelf] +class C: + def __init__(self) -> None: + self.a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + if bool(): + a = self + a.a = [1] + a.a.append(1) +reveal_type(C().a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferAttributeInitializedToEmptyAndAssignedOtherMethod] +class C: + def __init__(self) -> None: + self.a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + def meth(self) -> None: + self.a = [1] +reveal_type(C().a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferAttributeInitializedToEmptyAndAppendedOtherMethod] +class C: + def __init__(self) -> None: + self.a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + def meth(self) -> None: + self.a.append(1) +reveal_type(C().a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferAttributeInitializedToEmptyAndAssignedItemOtherMethod] +class C: + def __init__(self) -> None: + self.a = {} # E: Need type annotation for "a" (hint: "a: Dict[, ] = ...") + def meth(self) -> None: + self.a[0] = 'yes' +reveal_type(C().a) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testInferAttributeInitializedToNoneAndAssignedOtherMethod] +class C: + def __init__(self) -> None: + self.a = None + def meth(self) -> None: + self.a = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "None") +reveal_type(C().a) # N: Revealed type is "None" + +[case testInferAttributeInitializedToEmptyAndAssignedClassBody] +class C: + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + def __init__(self) -> None: + self.a = [1] +reveal_type(C().a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferAttributeInitializedToEmptyAndAppendedClassBody] +class C: + a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + def __init__(self) -> None: + self.a.append(1) +reveal_type(C().a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferAttributeInitializedToEmptyAndAssignedItemClassBody] +class C: + a = {} # E: Need type annotation for "a" (hint: "a: Dict[, ] = ...") + def __init__(self) -> None: + self.a[0] = 'yes' +reveal_type(C().a) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testInferAttributeInitializedToNoneAndAssignedClassBody] +class C: + a = None + def __init__(self) -> None: + self.a = 1 +reveal_type(C().a) # N: Revealed type is "Union[builtins.int, None]" + +[case testInferListTypeFromEmptyListAndAny] +def f(): + return [] + +def g() -> None: + x = [] + if bool(): + x = f() + reveal_type(x) # N: Revealed type is "builtins.list[Any]" + + y = [] + y.extend(f()) + reveal_type(y) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferFromEmptyDictWhenUsingIn] +d = {} +if 'x' in d: + d['x'] = 1 +reveal_type(d) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + +dd = {} +if 'x' not in dd: + dd['x'] = 1 +reveal_type(dd) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +[builtins fixtures/dict.pyi] + +[case testInferFromEmptyDictWhenUsingInSpecialCase] +# flags: --no-strict-optional +d = None +if 'x' in d: # E: "None" has no attribute "__iter__" (not iterable) + pass +reveal_type(d) # N: Revealed type is "None" +[builtins fixtures/dict.pyi] + +[case testInferFromEmptyListWhenUsingInWithStrictEquality] +# flags: --strict-equality +def f() -> None: + a = [] + if 1 in a: # TODO: This should be an error + a.append('x') +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + +[case testInferListTypeFromInplaceAdd] +a = [] +a += [1] +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testInferSetTypeFromInplaceOr] +# flags: --no-strict-optional +a = set() +a |= {'x'} +reveal_type(a) # N: Revealed type is "builtins.set[builtins.str]" +[builtins fixtures/set.pyi] + + +-- Inferring types of variables first initialized to None (partial types) +-- ---------------------------------------------------------------------- + + +[case testLocalVariablePartiallyInitializedToNone] +def f() -> None: + if object(): + x = None + else: + x = 1 + x() # E: "int" not callable \ + # E: "None" not callable +[out] + +[case testLocalVariablePartiallyTwiceInitializedToNone] +def f() -> None: + if object(): + x = None + elif object(): + x = None + else: + x = 1 + x() # E: "int" not callable \ + # E: "None" not callable +[out] + +[case testLvarInitializedToNoneWithoutType] +import typing +def f() -> None: + a = None + a.x() # E: "None" has no attribute "x" +[out] + +[case testGvarPartiallyInitializedToNone] +x = None +if object(): + x = 1 +x() # E: "int" not callable \ + # E: "None" not callable + +[case testPartiallyInitializedToNoneAndThenToPartialList] +x = None +if object(): + # Promote from partial None to partial list. + x = [] + x.append(1) +x.append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testPartiallyInitializedToNoneAndThenReadPartialList] +x = None +if object(): + # Promote from partial None to partial list. + x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") + x +[builtins fixtures/list.pyi] + +[case testPartiallyInitializedToNoneAndPartialListAndLeftPartial] +def f() -> None: + x = None + if object(): + # Promote from partial None to partial list. + x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +[builtins fixtures/list.pyi] +[out] + +[case testPartiallyInitializedToNoneAndThenToIncompleteType-skip] +# TODO(ddfisher): fix partial type bug and re-enable +from typing import TypeVar, Dict +T = TypeVar('T') +def f(*x: T) -> Dict[int, T]: pass +x = None # E: Need type annotation for "x" +if object(): + x = f() +[builtins fixtures/dict.pyi] + +[case testPartiallyInitializedVariableDoesNotEscapeScope1] +def f() -> None: + x = None + reveal_type(x) # N: Revealed type is "None" +x = 1 +[out] + +[case testPartiallyInitializedVariableDoesNotEscapeScope2] +x = None +def f() -> None: + x = None + x = 1 +x() # E: "None" not callable + +[case testAttributePartiallyInitializedToNone] +class A: + def f(self) -> None: + self.x = None + self.x = 1 + self.x() # E: "int" not callable +[out] + +[case testAttributePartiallyInitializedToNoneWithMissingAnnotation] +class A: + def f(self) -> None: + self.x = None + + def g(self) -> None: + self.x = 1 + self.x() +[out] +main:6: error: Incompatible types in assignment (expression has type "int", variable has type "None") +main:7: error: "None" not callable + +[case testGlobalInitializedToNoneSetFromFunction] +a = None +def f(): + global a + a = 42 +[out] + +[case testGlobalInitializedToNoneSetFromMethod] +a = None +class C: + def m(self): + global a + a = 42 +[out] + +-- More partial type errors +-- ------------------------ + +[case testPartialTypeErrorSpecialCase1] +# This used to crash. +class A: + x = None + def f(self) -> None: + for a in self.x: + pass +[builtins fixtures/for.pyi] +[out] +main:5: error: "None" has no attribute "__iter__" (not iterable) + +[case testPartialTypeErrorSpecialCase2] +# This used to crash. +class A: + x = [] + def f(self) -> None: + for a in self.x: + pass +[builtins fixtures/for.pyi] +[out] +main:3: error: Need type annotation for "x" (hint: "x: List[] = ...") + +[case testPartialTypeErrorSpecialCase3] +class A: + x = None + def f(self) -> None: + for a in A.x: + pass +[builtins fixtures/for.pyi] +[out] +main:4: error: "None" has no attribute "__iter__" (not iterable) + +[case testPartialTypeErrorSpecialCase4] +# This used to crash. +arr = [] +arr.append(arr.append(1)) +[builtins fixtures/list.pyi] +[out] +main:3: error: "append" of "list" does not return a value (it only ever returns None) + +-- Multipass +-- --------- + + +[case testMultipassAndAccessVariableBeforeDefinition] +def f() -> None: + y = x + y() # E: "int" not callable +x = 1 +[out] + +[case testMultipassAndAccessInstanceVariableBeforeDefinition] +class A: + def f(self) -> None: + y = self.x + y() # E: "int" not callable + + def g(self) -> None: + self.x = 1 +[out] + +[case testMultipassAndTopLevelVariable] +y = x # E: Cannot determine type of "x" # E: Name "x" is used before definition +y() +x = 1+int() +[out] + +[case testMultipassAndDecoratedMethod] +from typing import Callable, TypeVar + +T = TypeVar('T') + +class A: + def f(self) -> None: + self.g() # E: Too few arguments for "g" of "A" + self.g(1) + @dec + def g(self, x: str) -> None: pass + +def dec(f: Callable[[A, str], T]) -> Callable[[A, int], T]: pass +[out] + +[case testMultipassAndDefineAttributeBasedOnNotReadyAttribute] +class A: + def f(self) -> None: + self.y = self.x + + def g(self) -> None: + self.x = 1 + + def h(self) -> None: + self.y() # E: "int" not callable +[out] + +[case testMultipassAndDefineAttributeBasedOnNotReadyAttribute2] +class A: + def f(self) -> None: + self.y = self.x + self.z = self.y + self.z() # E + self.y() # E + + def g(self) -> None: + self.x = 1 + + def h(self) -> None: + self.y() # E +[out] +main:5: error: "int" not callable +main:6: error: "int" not callable +main:12: error: "int" not callable + +[case testMultipassAndPartialTypes] +def f() -> None: + x = [] + y + x.append(1) + x.append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" + x.append(y) # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +y = '' +[builtins fixtures/list.pyi] +[out] + +[case testMultipassAndPartialTypes2] +s = '' +n = 0 +def f() -> None: + global s, n + x = [] + x.append(y) + s = x[0] + n = x[0] # E: Incompatible types in assignment (expression has type "str", variable has type "int") + x.append(1) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "str" +y = '' +[builtins fixtures/list.pyi] +[out] + +[case testMultipassAndPartialTypes3] +from typing import Dict +def g(d: Dict[str, int]) -> None: pass +def f() -> None: + x = {} + x[1] = y + g(x) # E: Argument 1 to "g" has incompatible type "Dict[int, str]"; expected "Dict[str, int]" + x[1] = 1 # E: Incompatible types in assignment (expression has type "int", target has type "str") + x[1] = '' +y = '' +[builtins fixtures/dict.pyi] +[out] + +[case testMultipassAndPartialTypes4] +from typing import Dict +def g(d: Dict[str, int]) -> None: pass +def f() -> None: + x = {} + y + x[1] = 1 + g(x) # E: Argument 1 to "g" has incompatible type "Dict[int, int]"; expected "Dict[str, int]" +y = '' +[builtins fixtures/dict.pyi] +[out] + +[case testMultipassAndCircularDependency] +class A: + def f(self) -> None: + self.x = self.y # E: Cannot determine type of "y" + + def g(self) -> None: + self.y = self.x +[out] + +[case testMultipassAndPartialTypesSpecialCase1] +def f() -> None: + y = o + x = [] + x.append(y) + x() # E: "List[int]" not callable +o = 1 +[builtins fixtures/list.pyi] +[out] + +[case testMultipassAndPartialTypesSpecialCase2] +def f() -> None: + y = o + x = {} + x[''] = y + x() # E: "Dict[str, int]" not callable +o = 1 +[builtins fixtures/dict.pyi] +[out] + +[case testMultipassAndPartialTypesSpecialCase3] +def f() -> None: + x = {} # E: Need type annotation for "x" (hint: "x: Dict[, ] = ...") + y = o + z = {} # E: Need type annotation for "z" (hint: "z: Dict[, ] = ...") +o = 1 +[builtins fixtures/dict.pyi] +[out] + +[case testMultipassAndPartialTypesSpecialCase4] +def f() -> None: + y = o + x = None + x = y + x() # E: "int" not callable +o = 1 +[out] + +[case testMultipassAndPartialTypesSpecialCase5] +def f() -> None: + x = None + y = o + x = y + x() # E: "int" not callable +o = 1 +[out] + +[case testMultipassAndClassAttribute] +class S: + def foo(self) -> int: + return R.X + +class R: + X = 2 + +[case testMultipassAndMultipleFiles] +import m +def f() -> None: + x() +x = 0 +[file m.py] +def g() -> None: + y() +y = 0 +[out] +tmp/m.py:2: error: "int" not callable +main:3: error: "int" not callable + +[case testForwardReferenceToDecoratedClassMethod] +from typing import TypeVar, Callable + +T = TypeVar('T') +def dec() -> Callable[[T], T]: pass + +A.g # E: Cannot determine type of "g" # E: Name "A" is used before definition + +class A: + @classmethod + def f(cls) -> None: + reveal_type(cls.g) # N: Revealed type is "def (x: builtins.str)" + + @classmethod + @dec() + def g(cls, x: str) -> None: + pass + + @classmethod + def h(cls) -> None: + reveal_type(cls.g) # N: Revealed type is "def (x: builtins.str)" + +reveal_type(A.g) # N: Revealed type is "def (x: builtins.str)" +[builtins fixtures/classmethod.pyi] + + +-- Tests for special cases of unification +-- -------------------------------------- + + +[case testUnificationRedundantUnion] +from typing import Union +a: Union[int, str] +b: Union[str, tuple] +def f(): pass +def g(x: Union[int, str]): pass +c = a if f() else b +g(c) # E: Argument 1 to "g" has incompatible type "Union[int, str, Tuple[Any, ...]]"; expected "Union[int, str]" +[builtins fixtures/tuple.pyi] + +[case testUnificationMultipleInheritance] +class A: pass +class B: + def foo(self): pass +class C(A, B): pass +def f(): pass +a1 = B() if f() else C() +a1.foo() +a2 = C() if f() else B() +a2.foo() + +[case testUnificationMultipleInheritanceAmbiguous] +# Show that join_instances_via_supertype() breaks ties using the first base class. +class A1: pass +class B1: + def foo1(self): pass +class C1(A1, B1): pass + +class A2: pass +class B2: + def foo2(self): pass +class C2(A2, B2): pass + +class D1(C1, C2): pass +class D2(C2, C1): pass + +def f(): pass + +a1 = D1() if f() else D2() +a1.foo1() +a2 = D2() if f() else D1() +a2.foo2() + +[case testUnificationEmptyListLeft] +def f(): pass +a = [] if f() else [0] +a() # E: "List[int]" not callable +[builtins fixtures/list.pyi] + +[case testUnificationEmptyListRight] +def f(): pass +a = [0] if f() else [] +a() # E: "List[int]" not callable +[builtins fixtures/list.pyi] + +[case testUnificationEmptyListLeftInContext] +from typing import List +def f(): pass +a = [] if f() else [0] # type: List[int] +a() # E: "List[int]" not callable +[builtins fixtures/list.pyi] + +[case testUnificationEmptyListRightInContext] +# TODO Find an example that really needs the context +from typing import List +def f(): pass +a = [0] if f() else [] # type: List[int] +a() # E: "List[int]" not callable +[builtins fixtures/list.pyi] + +[case testUnificationEmptySetLeft] +def f(): pass +a = set() if f() else {0} +a() # E: "Set[int]" not callable +[builtins fixtures/set.pyi] + +[case testUnificationEmptyDictLeft] +def f(): pass +a = {} if f() else {0: 0} +a() # E: "Dict[int, int]" not callable +[builtins fixtures/dict.pyi] + +[case testUnificationEmptyDictRight] +def f(): pass +a = {0: 0} if f() else {} +a() # E: "Dict[int, int]" not callable +[builtins fixtures/dict.pyi] + +[case testUnificationDictWithEmptyListLeft] +def f(): pass +a = {0: []} if f() else {0: [0]} +a() # E: "Dict[int, List[int]]" not callable +[builtins fixtures/dict.pyi] + +[case testUnificationDictWithEmptyListRight] +def f(): pass +a = {0: [0]} if f() else {0: []} +a() # E: "Dict[int, List[int]]" not callable +[builtins fixtures/dict.pyi] + +[case testMisguidedSetItem] +from typing import Generic, Sequence, TypeVar +T = TypeVar('T') +class C(Sequence[T], Generic[T]): pass +C[0] = 0 +[out] +main:4: error: Unsupported target for indexed assignment ("Type[C[Any]]") +main:4: error: Invalid type: try using Literal[0] instead? + +[case testNoCrashOnPartialMember] +class C: + x = None + def __init__(self) -> None: + self.x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +[builtins fixtures/list.pyi] +[out] + +[case testNoCrashOnPartialVariable] +from typing import Tuple, TypeVar +T = TypeVar('T', bound=str) + +def f(x: T) -> Tuple[T]: + ... +x = None +(x,) = f('') +reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[out] + +[case testNoCrashOnPartialVariable2] +from typing import Tuple, TypeVar +T = TypeVar('T', bound=str) + +def f() -> Tuple[T]: + ... +x = None +if int(): + (x,) = f() +[builtins fixtures/tuple.pyi] +[out] + +[case testNoCrashOnPartialVariable3] +from typing import Tuple, TypeVar +T = TypeVar('T') + +def f(x: T) -> Tuple[T, T]: + ... +x = None +(x, x) = f('') +reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[out] + +[case testInferenceNestedTuplesFromGenericIterable] +from typing import Tuple, TypeVar + +T = TypeVar('T') + +def make_tuple(elem: T) -> Tuple[T]: + return (elem,) + +def main() -> None: + ((a, b),) = make_tuple((1, 2)) + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[out] + +[case testDontMarkUnreachableAfterInferenceUninhabited] +from typing import TypeVar +T = TypeVar('T') +def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + +class C: + x = f() # E: Need type annotation for "x" + def m(self) -> str: + return 42 # E: Incompatible return value type (got "int", expected "str") + +if bool(): + f() + 1() # E: "int" not callable +[builtins fixtures/list.pyi] +[out] + +[case testDontMarkUnreachableAfterInferenceUninhabited2] +from typing import TypeVar, Optional +T = TypeVar('T') +def f(x: Optional[T] = None) -> T: pass + +class C: + x = f() # E: Need type annotation for "x" + def m(self) -> str: + return 42 # E: Incompatible return value type (got "int", expected "str") + +if bool(): + f() + 1() # E: "int" not callable +[builtins fixtures/list.pyi] +[out] + +[case testDontMarkUnreachableAfterInferenceUninhabited3] +from typing import TypeVar, List +T = TypeVar('T') +def f(x: List[T]) -> T: pass + +class C: + x = f([]) # E: Need type annotation for "x" + def m(self) -> str: + return 42 # E: Incompatible return value type (got "int", expected "str") + +if bool(): + f([]) + 1() # E: "int" not callable +[builtins fixtures/list.pyi] +[out] + + +-- --local-partial-types +-- --------------------- + + +[case testLocalPartialTypesWithGlobalInitializedToNone] +# flags: --local-partial-types +x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") + +def f() -> None: + global x + x = 1 + +# TODO: "Any" could be a better type here to avoid multiple error messages +reveal_type(x) # N: Revealed type is "None" + +[case testLocalPartialTypesWithGlobalInitializedToNone2] +# flags: --local-partial-types +x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") + +def f(): + global x + x = 1 + +# TODO: "Any" could be a better type here to avoid multiple error messages +reveal_type(x) # N: Revealed type is "None" + +[case testLocalPartialTypesWithGlobalInitializedToNone3] +# flags: --local-partial-types --no-strict-optional +x = None + +def f() -> None: + global x + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +x = '' +reveal_type(x) # N: Revealed type is "builtins.str" + +[case testLocalPartialTypesWithGlobalInitializedToNoneStrictOptional] +# flags: --local-partial-types +x = None + +def f() -> None: + global x + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[str]") + +x = '' +def g() -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" + +[case testLocalPartialTypesWithGlobalInitializedToNone4] +# flags: --local-partial-types --no-strict-optional +a = None + +def f() -> None: + reveal_type(a) # N: Revealed type is "builtins.str" + +# TODO: This should probably be 'builtins.str', since there could be a +# call that causes a non-None value to be assigned +reveal_type(a) # N: Revealed type is "None" +a = '' +reveal_type(a) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + +[case testLocalPartialTypesWithClassAttributeInitializedToNone] +# flags: --local-partial-types +class A: + x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") + + def f(self) -> None: + self.x = 1 + +[case testLocalPartialTypesWithClassAttributeInitializedToEmptyDict] +# flags: --local-partial-types +class A: + x = {} # E: Need type annotation for "x" (hint: "x: Dict[, ] = ...") + + def f(self) -> None: + self.x[0] = '' + +reveal_type(A().x) # N: Revealed type is "builtins.dict[Any, Any]" +reveal_type(A.x) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testLocalPartialTypesWithGlobalInitializedToEmptyList] +# flags: --local-partial-types +a = [] + +def f() -> None: + a[0] + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +a.append(1) +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testLocalPartialTypesWithGlobalInitializedToEmptyList2] +# flags: --local-partial-types +a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + +def f() -> None: + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[Any]" + +reveal_type(a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testLocalPartialTypesWithGlobalInitializedToEmptyList3] +# flags: --local-partial-types +a = [] # E: Need type annotation for "a" (hint: "a: List[] = ...") + +def f(): + a.append(1) + +reveal_type(a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testLocalPartialTypesWithGlobalInitializedToEmptyDict] +# flags: --local-partial-types +a = {} + +def f() -> None: + a[0] + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +a[0] = '' +reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" +[builtins fixtures/dict.pyi] + +[case testLocalPartialTypesWithGlobalInitializedToEmptyDict2] +# flags: --local-partial-types +a = {} # E: Need type annotation for "a" (hint: "a: Dict[, ] = ...") + +def f() -> None: + a[0] = '' + reveal_type(a) # N: Revealed type is "builtins.dict[Any, Any]" + +reveal_type(a) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testLocalPartialTypesWithGlobalInitializedToEmptyDict3] +# flags: --local-partial-types +a = {} # E: Need type annotation for "a" (hint: "a: Dict[, ] = ...") + +def f(): + a[0] = '' + +reveal_type(a) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testLocalPartialTypesWithNestedFunction] +# flags: --local-partial-types +def f() -> None: + a = {} + def g() -> None: + a[0] = '' + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" +[builtins fixtures/dict.pyi] + +[case testLocalPartialTypesWithNestedFunction2] +# flags: --local-partial-types +def f() -> None: + a = [] + def g() -> None: + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testLocalPartialTypesWithNestedFunction3] +# flags: --local-partial-types --no-strict-optional +def f() -> None: + a = None + def g() -> None: + nonlocal a + a = '' + reveal_type(a) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] + +[case testLocalPartialTypesWithInheritance] +# flags: --local-partial-types +from typing import Optional + +class A: + x: Optional[str] + +class B(A): + x = None + +reveal_type(B.x) # N: Revealed type is "None" + +[case testLocalPartialTypesWithInheritance2] +# flags: --local-partial-types +class A: + x: str + +class B(A): + x = None # E: Incompatible types in assignment (expression has type "None", base class "A" defined the type as "str") + +[case testLocalPartialTypesWithAnyBaseClass] +# flags: --local-partial-types +from typing import Any + +A: Any + +class B(A): + x = None + +class C(B): + y = None + +[case testLocalPartialTypesInMultipleMroItems] +# flags: --local-partial-types +from typing import Optional + +class A: + x: Optional[str] + +class B(A): + x = None + +class C(B): + x = None + +# TODO: Inferring None below is unsafe (https://github.com/python/mypy/issues/3208) +reveal_type(B.x) # N: Revealed type is "None" +reveal_type(C.x) # N: Revealed type is "None" + +[case testLocalPartialTypesWithInheritance3] +# flags: --local-partial-types +from typing import Optional + +class X: pass +class Y(X): pass + +class A: + x: Optional[X] + +class B(A): + x = None + x = Y() + +reveal_type(B.x) # N: Revealed type is "Union[__main__.Y, None]" + +[case testLocalPartialTypesBinderSpecialCase] +# flags: --local-partial-types +from typing import List + +def f(x): pass + +class A: + x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") + + def f(self, p: List[str]) -> None: + self.x = f(p) + f(z for z in p) +[builtins fixtures/list.pyi] + +[case testLocalPartialTypesAccessPartialNoneAttribute] +# flags: --local-partial-types +class C: + a = None # E: Need type annotation for "a" (hint: "a: Optional[] = ...") + + def f(self, x) -> None: + C.a.y # E: Item "None" of "Optional[Any]" has no attribute "y" + +[case testLocalPartialTypesAccessPartialNoneAttribute2] +# flags: --local-partial-types +class C: + a = None # E: Need type annotation for "a" (hint: "a: Optional[] = ...") + + def f(self, x) -> None: + self.a.y # E: Item "None" of "Optional[Any]" has no attribute "y" + +-- Special case for assignment to '_' +-- ---------------------------------- + +[case testUnusedTargetLocal] +def foo() -> None: + _ = 0 + _ = '' + +[case testUnusedTargetNotGlobal] +_ = 0 +_ = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testUnusedTargetNotClass] +# flags: --allow-redefinition +class C: + _, _ = 0, 0 + _ = '' +reveal_type(C._) # N: Revealed type is "builtins.str" + +[case testUnusedTargetNotClass2] +# flags: --disallow-redefinition +class C: + _, _ = 0, 0 + _ = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +reveal_type(C._) # N: Revealed type is "builtins.int" + +[case testUnusedTargetTupleUnpacking] +def foo() -> None: + _, _ = (0, '') + _ = 0 + _ = '' +def bar() -> None: + t = (0, '') + _, _ = t + _ = 0 + _ = '' +[builtins fixtures/tuple.pyi] + +[case testUnusedTargetMultipleTargets] +def foo() -> None: + _ = x = 0 + _ = y = '' + _ = 0 + _ = '' +def bar() -> None: + x = _ = 0 + y = _ = '' + _ = 0 + _ = '' + x + 0 + y + '' + x + '' # E: Unsupported operand types for + ("int" and "str") + y + 0 # E: Unsupported operand types for + ("str" and "int") +[builtins fixtures/primitives.pyi] + +[case testUnusedTargetNotImport] +import d, c, b, a +[file _.py] +def f(): pass +[file m.py] +def f(): pass +_ = f +_ = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[[], Any]") +[file a.py] +def foo() -> None: + import _ + _.f() + _ = 0 # E: Incompatible types in assignment (expression has type "int", variable has type Module) +[file b.py] +def foo() -> None: + import m as _ + _.f() + _ = 0 # E: Incompatible types in assignment (expression has type "int", variable has type Module) +[file c.py] +def foo() -> None: + from m import _ + _() + _ = '' # E: Incompatible types in assignment (expression has type "str", variable has type "Callable[[], Any]") +[file d.py] +def foo() -> None: + from m import f as _ + _() + _ = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[[], Any]") +[builtins fixtures/module.pyi] + +[case testUnderscoreClass] +def foo() -> None: + class _: + pass + _().method() # E: "_" has no attribute "method" + +[case testUnusedTargetForLoop] +def f() -> None: + a = [(0, '', 0)] + for _, _, x in a: + x = 0 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + _ = 0 + _ = '' +[builtins fixtures/list.pyi] + +[case testUnusedTargetWithClause] +class C: + def __enter__(self) -> int: pass + def __exit__(self, *args): pass +def f() -> None: + with C() as _: pass + _ = 0 + _ = '' +[builtins fixtures/tuple.pyi] + +[case testUnusedTargetNotExceptClause] +# Things don't work for except clauses. +# This is due to the implementation, but it's just as well. +def f() -> None: + try: pass + except BaseException as _: + _ = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "BaseException") + _ = '' # E: Incompatible types in assignment (expression has type "str", variable has type "BaseException") +[builtins fixtures/exception.pyi] + +-- Tests for permissive toplevel checking +-- -------------- + +[case testPermissiveAttributeOverride1] +# flags: --allow-untyped-globals + +class A: + x = None + +class B(A): + x = 12 + +class C(A): + x = '12' + +reveal_type(A.x) # N: Revealed type is "Union[Any, None]" +reveal_type(B.x) # N: Revealed type is "builtins.int" +reveal_type(C.x) # N: Revealed type is "builtins.str" + +[case testPermissiveAttributeOverride2] +# flags: --allow-untyped-globals + +class A: + x = [] + +class B(A): + x = [12] + +class C(A): + x = ['12'] + +reveal_type(A.x) # N: Revealed type is "builtins.list[Any]" +reveal_type(B.x) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(C.x) # N: Revealed type is "builtins.list[builtins.str]" + +[builtins fixtures/list.pyi] + +[case testPermissiveAttribute] +# flags: --allow-untyped-globals + +class A: + x = [] + def f(self) -> None: + reveal_type(self.x) # N: Revealed type is "builtins.list[Any]" + +[builtins fixtures/list.pyi] + +[case testPermissiveGlobalContainer1] +# flags: --allow-untyped-globals --local-partial-types + +import a + +[file b.py] +x = [] +y = {} + +def foo() -> None: + reveal_type(x) # N: Revealed type is "builtins.list[Any]" + reveal_type(y) # N: Revealed type is "builtins.dict[Any, Any]" + +[file a.py] +from b import x, y +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +reveal_type(y) # N: Revealed type is "builtins.dict[Any, Any]" + +[builtins fixtures/dict.pyi] + +[case testPermissiveGlobalContainer2] +# flags: --allow-untyped-globals + +import a + +[file b.py] +x = [] +y = {} + +def foo() -> None: + reveal_type(x) # N: Revealed type is "builtins.list[Any]" + reveal_type(y) # N: Revealed type is "builtins.dict[Any, Any]" + +[file a.py] +from b import x, y +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +reveal_type(y) # N: Revealed type is "builtins.dict[Any, Any]" + +[builtins fixtures/dict.pyi] + +[case testPermissiveGlobalContainer3] +# flags: --allow-untyped-globals --local-partial-types + +import a + +[file b.py] +x = [] +y = {} +z = y + + +[file a.py] +from b import x, y +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +reveal_type(y) # N: Revealed type is "builtins.dict[Any, Any]" + +[builtins fixtures/dict.pyi] +[case testPermissiveGlobalContainer4] +# flags: --allow-untyped-globals + +import a + +[file b.py] +x = [] +y = {} +z = y + + +[file a.py] +from b import x, y +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +reveal_type(y) # N: Revealed type is "builtins.dict[Any, Any]" + +[builtins fixtures/dict.pyi] + +[case testInheritedAttributeNoStrictOptional] +# flags: --no-strict-optional +class A: + x: str + +class B(A): + x = None + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" + +[case testIncompatibleInheritedAttributeNoStrictOptional] +# flags: --no-strict-optional +class A: + x: str + +class B(A): + x = None + x = 2 # E: Incompatible types in assignment (expression has type "int", base class "A" defined the type as "str") + +[case testInheritedAttributeStrictOptional] +class A: + x: str + +class B(A): + x = None # E: Incompatible types in assignment (expression has type "None", base class "A" defined the type as "str") + x = '' + +[case testNeedAnnotationForCallable] +from typing import TypeVar, Optional, Callable + +T = TypeVar('T') + +def f(x: Optional[T] = None) -> Callable[..., T]: ... + +x = f() # E: Need type annotation for "x" +y = x + +[case testDontNeedAnnotationForCallable] +from typing import TypeVar, Optional, Callable, NoReturn + +T = TypeVar('T') + +def f() -> Callable[..., NoReturn]: ... + +x = f() +reveal_type(x) # N: Revealed type is "def (*Any, **Any) -> Never" + +[case testDeferralInNestedScopes] + + +def g() -> None: + def f() -> None: + x + 'no way' # E: Unsupported operand types for + ("int" and "str") + x = int() + f() + +[case testDeferralOfMemberNested] +from typing import Tuple + +def f() -> None: + c: C + t: Tuple[str, Tuple[str, str]] + x, (y, c.a) = t # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +class C: + def __init__(self, a: int) -> None: + self.a = a +[builtins fixtures/tuple.pyi] + +[case testUnionGenericWithBoundedVariable] +from typing import Generic, TypeVar, Union + +class A: ... +class B(A): ... + +T = TypeVar('T', bound=A) +class Z(Generic[T]): + def __init__(self, y: T) -> None: + self.y = y + +F = TypeVar('F', bound=A) + +def q1(x: Union[F, Z[F]]) -> F: + if isinstance(x, Z): + return x.y + else: + return x + +def q2(x: Union[Z[F], F]) -> F: + if isinstance(x, Z): + return x.y + else: + return x + +b: B +reveal_type(q1(b)) # N: Revealed type is "__main__.B" +reveal_type(q2(b)) # N: Revealed type is "__main__.B" + +z: Z[B] +reveal_type(q1(z)) # N: Revealed type is "__main__.B" +reveal_type(q2(z)) # N: Revealed type is "__main__.B" + +reveal_type(q1(Z(b))) # N: Revealed type is "__main__.B" +reveal_type(q2(Z(b))) # N: Revealed type is "__main__.B" +[builtins fixtures/isinstancelist.pyi] + +[case testUnionInvariantSubClassAndCovariantBase] +from typing import Union, Generic, TypeVar + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) + +class Cov(Generic[T_co]): ... +class Inv(Cov[T]): ... + +X = Union[Cov[T], Inv[T]] + +def f(x: X[T]) -> T: ... +x: Inv[int] +reveal_type(f(x)) # N: Revealed type is "builtins.int" + +[case testOptionalTypeVarAgainstOptional] +from typing import Optional, TypeVar, Iterable, Iterator, List + +_T = TypeVar('_T') + +def filter(__function: None, __iterable: Iterable[Optional[_T]]) -> List[_T]: ... + +x: Optional[str] + +y = filter(None, [x]) +reveal_type(y) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/list.pyi] + +[case testPartialDefaultDict] +from collections import defaultdict +x = defaultdict(int) +x[''] = 1 +reveal_type(x) # N: Revealed type is "collections.defaultdict[builtins.str, builtins.int]" + +y = defaultdict(int) # E: Need type annotation for "y" + +z = defaultdict(int) # E: Need type annotation for "z" +z[''] = '' +reveal_type(z) # N: Revealed type is "collections.defaultdict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testPartialDefaultDictInconsistentValueTypes] +from collections import defaultdict +a = defaultdict(int) # E: Need type annotation for "a" +a[''] = '' +a[''] = 1 +reveal_type(a) # N: Revealed type is "collections.defaultdict[builtins.str, builtins.int]" +[builtins fixtures/dict.pyi] + +[case testPartialDefaultDictListValue] +# flags: --no-strict-optional +from collections import defaultdict +a = defaultdict(list) +a['x'].append(1) +reveal_type(a) # N: Revealed type is "collections.defaultdict[builtins.str, builtins.list[builtins.int]]" + +b = defaultdict(lambda: []) +b[1].append('x') +reveal_type(b) # N: Revealed type is "collections.defaultdict[builtins.int, builtins.list[builtins.str]]" +[builtins fixtures/dict.pyi] + +[case testPartialDefaultDictListValueStrictOptional] +from collections import defaultdict +a = defaultdict(list) +a['x'].append(1) +reveal_type(a) # N: Revealed type is "collections.defaultdict[builtins.str, builtins.list[builtins.int]]" + +b = defaultdict(lambda: []) +b[1].append('x') +reveal_type(b) # N: Revealed type is "collections.defaultdict[builtins.int, builtins.list[builtins.str]]" +[builtins fixtures/dict.pyi] + +[case testPartialDefaultDictSpecialCases] +from collections import defaultdict +class A: + def f(self) -> None: + self.x = defaultdict(list) + self.x['x'].append(1) + reveal_type(self.x) # N: Revealed type is "collections.defaultdict[builtins.str, builtins.list[builtins.int]]" + self.y = defaultdict(list) # E: Need type annotation for "y" + s = self + s.y['x'].append(1) + +x = {} # E: Need type annotation for "x" (hint: "x: Dict[, ] = ...") +x['x'].append(1) + +y = defaultdict(list) # E: Need type annotation for "y" +y[[]].append(1) +[builtins fixtures/dict.pyi] + +[case testPartialDefaultDictSpecialCases2] +from collections import defaultdict + +x = defaultdict(lambda: [1]) # E: Need type annotation for "x" +x[1].append('') # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +reveal_type(x) # N: Revealed type is "collections.defaultdict[Any, builtins.list[builtins.int]]" + +xx = defaultdict(lambda: {'x': 1}) # E: Need type annotation for "xx" +xx[1]['z'] = 3 +reveal_type(xx) # N: Revealed type is "collections.defaultdict[Any, builtins.dict[builtins.str, builtins.int]]" + +y = defaultdict(dict) # E: Need type annotation for "y" +y['x'][1] = [3] + +z = defaultdict(int) # E: Need type annotation for "z" +z[1].append('') +reveal_type(z) # N: Revealed type is "collections.defaultdict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testPartialDefaultDictSpecialCase3] +from collections import defaultdict + +x = defaultdict(list) +x['a'] = [1, 2, 3] +reveal_type(x) # N: Revealed type is "collections.defaultdict[builtins.str, builtins.list[builtins.int]]" + +y = defaultdict(list) # E: Need type annotation for "y" +y['a'] = [] +reveal_type(y) # N: Revealed type is "collections.defaultdict[Any, Any]" +[builtins fixtures/dict.pyi] + +[case testInferCallableReturningNone1] +# flags: --no-strict-optional +from typing import Callable, TypeVar + +T = TypeVar("T") + +def f(x: Callable[[], T]) -> T: + return x() + +reveal_type(f(lambda: None)) # N: Revealed type is "None" +reveal_type(f(lambda: 1)) # N: Revealed type is "builtins.int" + +def g() -> None: pass + +reveal_type(f(g)) # N: Revealed type is "None" + +[case testInferCallableReturningNone2] +from typing import Callable, TypeVar + +T = TypeVar("T") + +def f(x: Callable[[], T]) -> T: + return x() + +reveal_type(f(lambda: None)) # N: Revealed type is "None" +reveal_type(f(lambda: 1)) # N: Revealed type is "builtins.int" + +def g() -> None: pass + +reveal_type(f(g)) # N: Revealed type is "None" + +[case testInferredTypeIsSimpleNestedList] +from typing import Any, Union, List + +y: Union[List[Any], Any] +x: Union[List[Any], Any] +x = [y] +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferredTypeIsSimpleNestedIterable] +from typing import Any, Union, Iterable + +y: Union[Iterable[Any], Any] +x: Union[Iterable[Any], Any] +x = [y] +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferredTypeIsSimpleNestedListLoop] +from typing import Any, Union, List + +def test(seq: List[Union[List, Any]]) -> None: + k: Union[List, Any] + for k in seq: + if bool(): + k = [k] + reveal_type(k) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testInferredTypeIsSimpleNestedIterableLoop] +from typing import Any, Union, List, Iterable + +def test(seq: List[Union[Iterable, Any]]) -> None: + k: Union[Iterable, Any] + for k in seq: + if bool(): + k = [k] + reveal_type(k) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testErasedTypeRuntimeCoverage] +# https://github.com/python/mypy/issues/11913 +from typing import TypeVar, Type, Generic, Callable, Iterable + +class DataType: ... + +T1 = TypeVar('T1') +T2 = TypeVar("T2", bound=DataType) + +def map(__func: T1) -> None: ... + +def collection_from_dict_value(model: Type[T2]) -> None: + map(lambda i: i if isinstance(i, model) else i) +[builtins fixtures/isinstancelist.pyi] + +[case testRegression11705_Strict] +# See: https://github.com/python/mypy/issues/11705 +from typing import Dict, Optional, NamedTuple +class C(NamedTuple): + x: int + +t: Optional[C] +d: Dict[C, bytes] +x = t and d[t] +reveal_type(x) # N: Revealed type is "Union[None, builtins.bytes]" +if x: + reveal_type(x) # N: Revealed type is "builtins.bytes" +[builtins fixtures/dict.pyi] + +[case testRegression11705_NoStrict] +# flags: --no-strict-optional +# See: https://github.com/python/mypy/issues/11705 +from typing import Dict, Optional, NamedTuple +class C(NamedTuple): + x: int + +t: Optional[C] +d: Dict[C, bytes] +x = t and d[t] +reveal_type(x) # N: Revealed type is "builtins.bytes" +if x: + reveal_type(x) # N: Revealed type is "builtins.bytes" +[builtins fixtures/dict.pyi] + +[case testSuggestPep604AnnotationForPartialNone] +# flags: --local-partial-types --python-version 3.10 +x = None # E: Need type annotation for "x" (hint: "x: | None = ...") + +[case testTupleContextFromIterable] +from typing import TypeVar, Iterable, List, Union + +T = TypeVar("T") + +def foo(x: List[T]) -> List[T]: ... +x: Iterable[List[Union[int, str]]] = (foo([1]), foo(["a"])) +[builtins fixtures/tuple.pyi] + +[case testTupleContextFromIterable2] +from typing import Dict, Iterable, Tuple, Union + +def foo(x: Union[Tuple[str, Dict[str, int], str], Iterable[object]]) -> None: ... +foo(("a", {"a": "b"}, "b")) +[builtins fixtures/dict.pyi] + +[case testUseSupertypeAsInferenceContext] +from typing import List, Optional + +class B: + x: List[Optional[int]] + +class C(B): + x = [1] + +reveal_type(C().x) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextInvalidType] +from typing import List +class P: + x: List[int] +class C(P): + x = ['a'] # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartial] +from typing import List + +class A: + x: List[str] + +class B(A): + x = [] + +reveal_type(B().x) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartialError] +class A: + x = ['a', 'b'] + +class B(A): + x = [] + x.append(2) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "str" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartialErrorProperty] +from typing import List + +class P: + @property + def x(self) -> List[int]: ... +class C(P): + x = [] + +C.x.append("no") # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextConflict] +from typing import List +class P: + x: List[int] +class M: + x: List[str] +class C(P, M): + x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +reveal_type(C.x) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testNoPartialInSupertypeAsContext] +class A: + args = {} # E: Need type annotation for "args" (hint: "args: Dict[, ] = ...") + def f(self) -> None: + value = {1: "Hello"} + class B(A): + args = value +[builtins fixtures/dict.pyi] + +[case testInferSimpleLiteralInClassBodyCycle] +import a +[file a.py] +import b +reveal_type(b.B.x) +class A: + x = 42 +[file b.py] +import a +reveal_type(a.A.x) +class B: + x = 42 +[out] +tmp/b.py:2: note: Revealed type is "builtins.int" +tmp/a.py:2: note: Revealed type is "builtins.int" + +[case testUnionTypeCallableInference] +from typing import Callable, Type, TypeVar, Union + +class A: + def __init__(self, x: str) -> None: ... + +T = TypeVar("T") +def type_or_callable(value: T, tp: Union[Type[T], Callable[[int], T]]) -> T: ... +reveal_type(type_or_callable(A("test"), A)) # N: Revealed type is "__main__.A" + +[case testUpperBoundAsInferenceFallback] +from typing import Callable, TypeVar, Any, Mapping, Optional +T = TypeVar("T", bound=Mapping[str, Any]) +def raises(opts: Optional[T]) -> T: pass +def assertRaises(cb: Callable[..., object]) -> None: pass +assertRaises(raises) # OK +[builtins fixtures/dict.pyi] + +[case testJoinWithAnyFallback] +from unknown import X # type: ignore[import] + +class A: ... +class B(X, A): ... +class C(B): ... +class D(C): ... +class E(D): ... + +reveal_type([E(), D()]) # N: Revealed type is "builtins.list[__main__.D]" +reveal_type([D(), E()]) # N: Revealed type is "builtins.list[__main__.D]" + +[case testCallableInferenceAgainstCallablePosVsStar] +from typing import TypeVar, Callable, Tuple + +T = TypeVar('T') +S = TypeVar('S') + +def f(x: Callable[[T, S], None]) -> Tuple[T, S]: ... +def g(*x: int) -> None: ... +reveal_type(f(g)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallableStarVsPos] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T, S]): + def __call__(self, __x: T, *args: S) -> None: ... + +def f(x: Call[T, S]) -> Tuple[T, S]: ... +def g(*x: int) -> None: ... +reveal_type(f(g)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallableNamedVsStar] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T, S]): + def __call__(self, *, x: T, y: S) -> None: ... + +def f(x: Call[T, S]) -> Tuple[T, S]: ... +def g(**kwargs: int) -> None: ... +reveal_type(f(g)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallableStarVsNamed] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T, S]): + def __call__(self, *, x: T, **kwargs: S) -> None: ... + +def f(x: Call[T, S]) -> Tuple[T, S]: ... +def g(**kwargs: int) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallableNamedVsNamed] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T, S]): + def __call__(self, *, x: T, y: S) -> None: ... + +def f(x: Call[T, S]) -> Tuple[T, S]: ... + +# Note: order of names is different w.r.t. protocol +def g(*, y: int, x: str) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Tuple[builtins.str, builtins.int]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallablePosOnlyVsNamed] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T]): + def __call__(self, *, x: T) -> None: ... + +def f(x: Call[T]) -> Tuple[T, T]: ... + +def g(__x: str) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Tuple[Never, Never]" \ + # E: Argument 1 to "f" has incompatible type "Callable[[str], None]"; expected "Call[Never]" \ + # N: "Call[Never].__call__" has type "Callable[[NamedArg(Never, 'x')], None]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallableNamedVsPosOnly] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T]): + def __call__(self, __x: T) -> None: ... + +def f(x: Call[T]) -> Tuple[T, T]: ... + +def g(*, x: str) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Tuple[Never, Never]" \ + # E: Argument 1 to "f" has incompatible type "Callable[[NamedArg(str, 'x')], None]"; expected "Call[Never]" \ + # N: "Call[Never].__call__" has type "Callable[[Never], None]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallablePosOnlyVsKwargs] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T]): + def __call__(self, __x: T) -> None: ... + +def f(x: Call[T]) -> Tuple[T, T]: ... + +def g(**x: str) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Tuple[Never, Never]" \ + # E: Argument 1 to "f" has incompatible type "Callable[[KwArg(str)], None]"; expected "Call[Never]" \ + # N: "Call[Never].__call__" has type "Callable[[Never], None]" +[builtins fixtures/list.pyi] + +[case testCallableInferenceAgainstCallableNamedVsArgs] +from typing import TypeVar, Callable, Tuple, Protocol + +T = TypeVar('T', contravariant=True) +S = TypeVar('S', contravariant=True) + +class Call(Protocol[T]): + def __call__(self, *, x: T) -> None: ... + +def f(x: Call[T]) -> Tuple[T, T]: ... + +def g(*args: str) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Tuple[Never, Never]" \ + # E: Argument 1 to "f" has incompatible type "Callable[[VarArg(str)], None]"; expected "Call[Never]" \ + # N: "Call[Never].__call__" has type "Callable[[NamedArg(Never, 'x')], None]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstTypeVarActualBound] +from typing import Callable, TypeVar + +T = TypeVar("T") +S = TypeVar("S") +def test(f: Callable[[T], S]) -> Callable[[T], S]: ... + +F = TypeVar("F", bound=Callable[..., object]) +def dec(f: F) -> F: + reveal_type(test(f)) # N: Revealed type is "def (Any) -> builtins.object" + return f + +[case testInferenceAgainstTypeVarActualUnionBound] +from typing import Protocol, TypeVar, Union + +T_co = TypeVar("T_co", covariant=True) +class SupportsFoo(Protocol[T_co]): + def foo(self) -> T_co: ... + +class A: + def foo(self) -> A: ... +class B: + def foo(self) -> B: ... + +def foo(f: SupportsFoo[T_co]) -> T_co: ... + +ABT = TypeVar("ABT", bound=Union[A, B]) +def simpler(k: ABT): + foo(k) + +[case testInferenceWorksWithEmptyCollectionsNested] +from typing import List, TypeVar, NoReturn +T = TypeVar('T') +def f(a: List[T], b: List[T]) -> T: pass +x = ["yes"] +reveal_type(f(x, [])) # N: Revealed type is "builtins.str" +reveal_type(f(["yes"], [])) # N: Revealed type is "builtins.str" + +empty: List[NoReturn] +f(x, empty) # E: Cannot infer type argument 1 of "f" +f(["no"], empty) # E: Cannot infer type argument 1 of "f" +[builtins fixtures/list.pyi] + +[case testInferenceWorksWithEmptyCollectionsUnion] +from typing import Any, Dict, NoReturn, NoReturn, Union + +def foo() -> Union[Dict[str, Any], Dict[int, Any]]: + return {} + +empty: Dict[NoReturn, NoReturn] +def bar() -> Union[Dict[str, Any], Dict[int, Any]]: + return empty +[builtins fixtures/dict.pyi] + +[case testUpperBoundInferenceFallbackNotOverused] +from typing import TypeVar, Protocol, List + +S = TypeVar("S", covariant=True) +class Foo(Protocol[S]): + def foo(self) -> S: ... +def foo(x: Foo[S]) -> S: ... + +T = TypeVar("T", bound="Base") +class Base: + def foo(self: T) -> T: ... +class C(Base): + pass + +def f(values: List[T]) -> T: ... +x = foo(f([C()])) +reveal_type(x) # N: Revealed type is "__main__.C" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallableUnion] +from typing import Callable, TypeVar, List, Union + +T = TypeVar("T") +S = TypeVar("S") + +def dec(f: Callable[[S], T]) -> Callable[[S], List[T]]: ... +@dec +def func(arg: T) -> Union[T, str]: + ... +reveal_type(func) # N: Revealed type is "def [S] (S`1) -> builtins.list[Union[S`1, builtins.str]]" +reveal_type(func(42)) # N: Revealed type is "builtins.list[Union[builtins.int, builtins.str]]" + +def dec2(f: Callable[[S], List[T]]) -> Callable[[S], T]: ... +@dec2 +def func2(arg: T) -> List[Union[T, str]]: + ... +reveal_type(func2) # N: Revealed type is "def [S] (S`4) -> Union[S`4, builtins.str]" +reveal_type(func2(42)) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericCallbackProtoMultiple] +from typing import Callable, Protocol, TypeVar +from typing_extensions import Concatenate, ParamSpec + +V_co = TypeVar("V_co", covariant=True) +class Metric(Protocol[V_co]): + def __call__(self) -> V_co: ... + +T = TypeVar("T") +P = ParamSpec("P") +def simple_metric(func: Callable[Concatenate[int, P], T]) -> Callable[P, T]: ... + +@simple_metric +def Negate(count: int, /, metric: Metric[float]) -> float: ... +@simple_metric +def Combine(count: int, m1: Metric[T], m2: Metric[T], /, *more: Metric[T]) -> T: ... + +reveal_type(Negate) # N: Revealed type is "def (metric: __main__.Metric[builtins.float]) -> builtins.float" +reveal_type(Combine) # N: Revealed type is "def [T] (def () -> T`5, def () -> T`5, *more: def () -> T`5) -> T`5" + +def m1() -> float: ... +def m2() -> float: ... +reveal_type(Combine(m1, m2)) # N: Revealed type is "builtins.float" +[builtins fixtures/list.pyi] + +[case testInferenceWithUninhabitedType] +from typing import Dict, Generic, List, Never, TypeVar + +T = TypeVar("T") + +class A(Generic[T]): ... +class B(Dict[T, T]): ... + +def func1(a: A[T], b: T) -> T: ... +def func2(a: T, b: A[T]) -> T: ... + +def a1(a: A[Dict[str, int]]) -> None: + reveal_type(func1(a, {})) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + reveal_type(func2({}, a)) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + +def a2(check: bool, a: B[str]) -> None: + reveal_type(a if check else {}) # N: Revealed type is "builtins.dict[builtins.str, builtins.str]" + +def a3() -> None: + a = {} # E: Need type annotation for "a" (hint: "a: Dict[, ] = ...") + b = {1: {}} # E: Need type annotation for "b" + c = {1: {}, 2: {"key": {}}} # E: Need type annotation for "c" + reveal_type(a) # N: Revealed type is "builtins.dict[Any, Any]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, builtins.dict[Any, Any]]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.dict[builtins.str, builtins.dict[Any, Any]]]" + +def a4(x: List[str], y: List[Never]) -> None: + z1 = [x, y] + z2 = [y, x] + reveal_type(z1) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z2) # N: Revealed type is "builtins.list[builtins.object]" + z1[1].append("asdf") # E: "object" has no attribute "append" +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test new file mode 100644 index 000000000000..bedba811d95b --- /dev/null +++ b/test-data/unit/check-inline-config.test @@ -0,0 +1,325 @@ +-- Checks for 'mypy: option' directives inside files + +[case testInlineSimple1] + +# mypy: disallow-any-generics, no-warn-no-return + +from typing import List, Optional +def foo() -> Optional[List]: # E: Missing type parameters for generic type "List" + 20 + +[builtins fixtures/list.pyi] + +[case testInlineSimple2] + +# mypy: disallow-any-generics +# mypy: no-warn-no-return + +from typing import List, Optional +def foo() -> Optional[List]: # E: Missing type parameters for generic type "List" + 20 + +[builtins fixtures/list.pyi] + +[case testInlineSimple3] + +# mypy: disallow-any-generics=true, warn-no-return=0 + +from typing import List, Optional +def foo() -> Optional[List]: # E: Missing type parameters for generic type "List" + 20 + +[builtins fixtures/list.pyi] + + +[case testInlineSimple4] + +# mypy: disallow-any-generics = true, warn-no-return = 0 + +from typing import List, Optional +def foo() -> Optional[List]: # E: Missing type parameters for generic type "List" + 20 + +[builtins fixtures/list.pyi] + +[case testInlineList] + +# mypy: disallow-any-generics,always-false="FOO,BAR" + +from typing import List + +def foo(FOO: bool, BAR: bool) -> List: # E: Missing type parameters for generic type "List" + if FOO or BAR: + 1+'lol' + return [] + +[builtins fixtures/list.pyi] + +[case testInlineInvert1] +# flags: --disallow-any-generics --allow-untyped-globals +import a +[file a.py] +# mypy: allow-any-generics, disallow-untyped-globals + +x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") + +from typing import List +def foo() -> List: + ... + +[builtins fixtures/list.pyi] + +[case testInlineInvert2] + +import a +[file a.py] +# mypy: no-always-true + +[out] +tmp/a.py:1: error: Can not invert non-boolean key always_true + +[case testInlineIncremental1] + +import a +[file a.py] +# mypy: disallow-any-generics, no-warn-no-return + +from typing import List, Optional +def foo() -> Optional[List]: + 20 + +[file a.py.2] +# mypy: no-warn-no-return + +from typing import List, Optional +def foo() -> Optional[List]: + 20 + +[file a.py.3] +from typing import List, Optional +def foo() -> Optional[List]: + 20 +[out] +tmp/a.py:4: error: Missing type parameters for generic type "List" +[out2] +[out3] +tmp/a.py:2: error: Missing return statement + +[builtins fixtures/list.pyi] + +[case testInlineIncremental2] + +# flags2: --disallow-any-generics +import a +[file a.py] +# mypy: no-warn-no-return + +from typing import Optional, List +def foo() -> Optional[List]: + 20 + +[file b.py.2] +# no changes to a.py, but flag change should cause recheck + +[out] +[out2] +tmp/a.py:4: error: Missing type parameters for generic type "List" + +[builtins fixtures/list.pyi] + +[case testInlineIncremental3] +import a, b +[file a.py] +# mypy: no-warn-no-return +from typing import Optional + +def foo() -> Optional[int]: + 20 + +[file b.py] +[file b.py.2] +# no changes to a.py and we want to make sure it isn't rechecked +[out] +[out2] +[rechecked b] + +[case testInlineError1] +# mypy: invalid-whatever +# mypy: no-warn-no-return; no-strict-optional +# mypy: always-true=FOO,BAR +# mypy: always-true="FOO,BAR +[out] +main:1: error: Unrecognized option: invalid_whatever = True +main:2: error: Unrecognized option: no_warn_no_return; no_strict_optional = True +main:3: error: Unrecognized option: bar = True +main:4: error: Unterminated quote in configuration comment + +[case testInlineError2] +# mypy: skip-file +[out] +main:1: error: Unrecognized option: skip_file = True + +[case testInlineStrict] +# mypy: strict +[out] +main:1: error: Setting "strict" not supported in inline configuration: specify it in a configuration file instead, or set individual inline flags (see "mypy -h" for the list of flags enabled in strict mode) + +[case testInlineErrorCodes] +# mypy: enable-error-code="ignore-without-code,truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... # E: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) + +[case testInlineErrorCodesOverrideConfig] +# flags: --config-file tmp/mypy.ini +import foo +import tests.bar +import tests.baz +[file foo.py] +# mypy: disable-error-code="truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) + +[file tests/__init__.py] +[file tests/bar.py] +# mypy: enable-error-code="ignore-without-code" + +def foo() -> int: ... +if foo: ... # E: Function "foo" could always be true in boolean context +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) + +[file tests/baz.py] +# mypy: disable-error-code="truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... +42 + "no" # type: ignore + +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code, truthy-bool + +\[mypy-tests.*] +disable_error_code = ignore-without-code + +[case testIgnoreErrorsSimple] +# mypy: ignore-errors=True + +def f() -> None: + while 1(): + pass + +[case testIgnoreErrorsInImportedModule] +from m import C +c = C() +reveal_type(c.x) # N: Revealed type is "builtins.int" + +[file m.py] +# mypy: ignore-errors=True + +class C: + def f(self) -> None: + self.x = 1 + +[case testIgnoreErrorsWithLambda] +# mypy: ignore-errors=True + +def f(self, x=lambda: 1) -> None: + pass + +class C: + def f(self) -> None: + l = lambda: 1 + self.x = 1 + +[case testIgnoreErrorsWithUnsafeSuperCall_no_empty] + +from m import C + +class D(C): + def m(self) -> None: + super().m1() + super().m2() \ + # E: Call to abstract method "m2" of "C" with trivial body via super() is unsafe + super().m3() \ + # E: Call to abstract method "m3" of "C" with trivial body via super() is unsafe + super().m4() \ + # E: Call to abstract method "m4" of "C" with trivial body via super() is unsafe + super().m5() \ + # E: Call to abstract method "m5" of "C" with trivial body via super() is unsafe + super().m6() \ + # E: Call to abstract method "m6" of "C" with trivial body via super() is unsafe + super().m7() + + def m1(self) -> int: + return 0 + + def m2(self) -> int: + return 0 + + def m3(self) -> int: + return 0 + + def m4(self) -> int: + return 0 + + def m5(self) -> int: + return 0 + + def m6(self) -> int: + return 0 + +[file m.py] +# mypy: ignore-errors=True +import abc + +class C: + @abc.abstractmethod + def m1(self) -> int: + """x""" + return 0 + + @abc.abstractmethod + def m2(self) -> int: + """doc""" + + @abc.abstractmethod + def m3(self) -> int: + pass + + @abc.abstractmethod + def m4(self) -> int: ... + + @abc.abstractmethod + def m5(self) -> int: + """doc""" + ... + + @abc.abstractmethod + def m6(self) -> int: + raise NotImplementedError() + + @abc.abstractmethod + def m7(self) -> int: + raise NotImplementedError() + pass + +[builtins fixtures/exception.pyi] + +[case testInlineErrorCodesMultipleCodes] +# mypy: disable-error-code="truthy-bool, ignore-without-code" +class Foo: + pass + +foo = Foo() +if foo: ... +42 + "no" # type: ignore diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test new file mode 100644 index 000000000000..99bd62765b11 --- /dev/null +++ b/test-data/unit/check-isinstance.test @@ -0,0 +1,2919 @@ +[case testForcedAssignment] +x = 1 # type: object +y = 1 +def f(): x, y # Prevent independent redefinition +y = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") +x = 2 +y = x +[builtins fixtures/tuple.pyi] + +[case testJoinAny] +from typing import List, Any + +x: List[Any] + +def foo() -> List[int]: pass +def bar() -> List[str]: pass + +if bool(): + x = foo() +else: + x = bar() + +x * 2 +[builtins fixtures/list.pyi] + +[case testGeneratorExpressionTypes] +class A: y = 1 +x = [A()] +y = [x] + +z = [1,2] +z = [a.y for b in y for a in b] +[builtins fixtures/list.pyi] + +[case testIsinstanceNestedTuple] +from typing import Union, List, Tuple, Dict + +def f(x: Union[int, str, List]) -> None: + if isinstance(x, (str, (int,))): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + x[1] # E: Value of type "Union[int, str]" is not indexable + else: + reveal_type(x) # N: Revealed type is "builtins.list[Any]" + x[1] + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" + if isinstance(x, (str, (list,))): + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.list[Any]]" + x[1] + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" +[builtins fixtures/isinstancelist.pyi] + +[case testClassAttributeInitialization] +class A: + x: int + def __init__(self) -> None: + self.y: int + z = self.x + w = self.y + +[case testAssignmentSubtypes] +from typing import Union + +def foo(x: Union[str, int]): + if isinstance(x, int): + x = 'a' + x + 'a' + z = x + y = [x] + y[0] + 'a' + # TODO: should we allow these two lines? + y + [1] # E: List item 0 has incompatible type "int"; expected "str" + z = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +x: int +y = [x] +[builtins fixtures/isinstancelist.pyi] + +[case testFunctionDefaultArgs] +class A: pass +class B(A): y = 1 + +x = A() + +def foo(x: A = B()): + x.y # E: "A" has no attribute "y" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceFancyConditionals] +class A: pass + +class B(A): + y = 1 + +x = A() +if isinstance(x, B): + x.y +while isinstance(x, B): + x.y +while isinstance(x, B): + x.y + x = B() +[builtins fixtures/isinstance.pyi] + +[case testSubtypingWithAny] +class A: + y = 1 + +class B(A): + z = 1 + +def foo(): pass + +x = A() +if int(): + x = B() + x.z + x = foo() + x.z # E: "A" has no attribute "z" + x.y + +[case testSingleMultiAssignment] +x = 'a' +(x,) = ('a',) + +[case testUnionMultiAssignment] +from typing import Union +x: Union[int, str] +if int(): + x = 1 + x = 'a' + x + 1 # E: Unsupported operand types for + ("str" and "int") + x = 1 + (x, y) = ('a', 1) + x + 1 # E: Unsupported operand types for + ("str" and "int") +[builtins fixtures/isinstancelist.pyi] + +[case testUnionIfZigzag] +from typing import Union + +def f(x: Union[int, str]) -> None: + if 1: # Without this, the assignment below could create a new variable "x" of type "int" + x = 1 + if x: + x = 'a' + x = 1 + x + 1 + x + 1 +[builtins fixtures/isinstancelist.pyi] + +[case testTwoLoopsUnion] +from typing import Union + +def foo() -> Union[int, str]: pass + +def bar() -> None: + x = foo() + if isinstance(x, int): + return + while bool(): + x + 'a' + while bool(): + x = foo() + if bool(): + return + x = 'a' + x + 'a' +[builtins fixtures/isinstancelist.pyi] + +[case testComplicatedBlocks] +from typing import Union + +def foo() -> Union[int, str]: pass + +def bar() -> None: + x = foo() + if isinstance(x, int): + return + while bool(): + x + 'a' + while bool(): + x = foo() + if bool(): + return + x = 'a' + x + 'a' + + x = foo() + if isinstance(x, int): + return + while bool(): + x + 'a' + while bool(): + x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + x = foo() + if bool(): + continue + x = 'a' + x = 'a' + x + 'a' +[builtins fixtures/isinstancelist.pyi] + +[case testUnionTryExcept] +class A: + y = A() + +class B(A): + z = 1 + +x = A() +def f(): x # Prevent redefinition of x +x = B() +x.z +try: + x.z + x = A() + x = B() + x.z +except: + pass +x.z # E: "A" has no attribute "z" + +[case testUnionTryExcept2] +class A: + y = A() + +class B(A): + z = 1 + +x = A() +try: + x.z # E: "A" has no attribute "z" + x = A() + x = B() + x.z +except: + x.z # E: "A" has no attribute "z" + x = B() + x.z +else: + x = B() +x.z + +[case testUnionTryExcept3] +class A: + y = A() + +class B(A): + z = 1 + +x = A() +def f(): x # Prevent redefinition of x +x = B() +try: + raise BaseException() + x = A() +except: + pass +x.z +x = B() +try: + x = A() + raise BaseException() +except: + pass +x.z # E: "A" has no attribute "z" +x = B() +try: + pass +except: + x = A() + raise BaseException() +x.z +try: + x = A() +except: + pass +x.z # E: "A" has no attribute "z" +x = B() +try: + pass +except: + x = A() +x.z # E: "A" has no attribute "z" +[builtins fixtures/exception.pyi] + +[case testUnionTryExcept4] +class A: pass + +class B(A): + z = 1 + +x = A() +while bool(): + try: + x.z # E: "A" has no attribute "z" + x = A() + except: + x = B() + else: + x = B() + x.z +[builtins fixtures/exception.pyi] + +[case testUnionTryFinally] +class A: pass + +class B(A): + b = 1 + +x = A() +def f(): x # Prevent redefinition +x = B() +try: + x = A() + x.b # E: "A" has no attribute "b" + x = B() +finally: + x.b # E: "A" has no attribute "b" +x.b + +[case testUnionTryFinally2] +class A: pass + +class B(A): + b = 1 + +x = A() +def f(): x # Prevent redefinition +x = B() +try: + x = A() + x = B() +except: + pass +finally: + pass +x.b # E: "A" has no attribute "b" + +[case testUnionTryFinally3] +class A: pass + +class B(A): + b = 1 + +x = A() +def f(): x # Prevent redefinition +x = B() +try: + x = A() + x = B() +except: + pass +finally: + x = B() +x.b + +[case testUnionTryFinally4] +class A: pass + +class B(A): + b = 1 + +while 2: + x = A() + def f(): x # Prevents redefinition + x = B() + try: + x = A() + x = B() + except: + pass + finally: + x.b # E: "A" has no attribute "b" + if not isinstance(x, B): + break + x.b +[builtins fixtures/isinstancelist.pyi] + +[case testUnionTryFinally5] +class A: pass + +class B(A): + b = 1 + +while 2: + x = A() + try: + x = A() + x = B() + finally: + x.b # E: "A" has no attribute "b" + break + x.b + x.b + +[case testUnionTryFinally6] +class A: pass + +class B(A): + b = 1 + +def f() -> int: + x = B() # type: A + try: + x = B() + except: + x = A() + # An exception could occur here + x = B() + finally: + return x.b # E: "A" has no attribute "b" + +[case testUnionListIsinstance] +from typing import Union, List + +def f(x: Union[List[int], List[str], int]) -> None: + if isinstance(x, list): + a = x[0] + if isinstance(a, int): + a + 1 + a + 'x' # E: Unsupported operand types for + ("int" and "str") + + # type of a? + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" + x + 1 # E: Unsupported operand types for + ("List[int]" and "int") \ + # E: Unsupported operand types for + ("List[str]" and "int") \ + # N: Left operand is of type "Union[List[int], List[str]]" + else: + x[0] # E: Value of type "int" is not indexable + x + 1 + x[0] # E: Value of type "Union[List[int], List[str], int]" is not indexable + x + 1 # E: Unsupported operand types for + ("List[int]" and "int") \ + # E: Unsupported operand types for + ("List[str]" and "int") \ + # N: Left operand is of type "Union[List[int], List[str], int]" +[builtins fixtures/isinstancelist.pyi] + +[case testUnionListIsinstance2] +from typing import Union, List + +class A: + a = 1 + +class B: pass +class C: pass + +def g(x: Union[A, B]) -> A: pass +def h(x: C) -> A: pass +def f(x: Union[A, B, C]) -> None: + if isinstance(x, C): + x = h(x) + else: + x = g(x) + x.a +[builtins fixtures/isinstancelist.pyi] + +[case testUnionStrictDefnBasic] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +if int(): + x = 1 + x = x + 1 + x = foo() + x = x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +if isinstance(x, str): + x = x + 1 # E: Unsupported operand types for + ("str" and "int") + x = 1 + x = x + 1 +[builtins fixtures/isinstancelist.pyi] + +[case testSubtypeRedefinitionBasic] +from typing import Union + +class A: pass + +class B(A): + y = 1 + +x = A() +x.y # E: "A" has no attribute "y" +x = B() +x.y # OK: x is known to be a B +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceBasic] +from typing import Union +x: Union[int, str] +if isinstance(x, str): + x = x + 1 # E: Unsupported operand types for + ("str" and "int") + x = x + 'a' +else: + x = x + 'a' # E: Unsupported operand types for + ("int" and "str") + x = x + 1 +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceIndexing] +from typing import Union +x: Union[int, str] +j = [x] +if isinstance(j[0], str): + j[0] = j[0] + 'a' + j[0] = j[0] + 1 # E: Unsupported operand types for + ("str" and "int") +else: + j[0] = j[0] + 'a' # E: Unsupported operand types for + ("int" and "str") + j[0] = j[0] + 1 +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceSubClassMember] +from typing import Union + +class Animal: pass + +class Dog(Animal): + paws = 4 # type: Union[int, str] + + def bark(self): pass + +class House: + pet = None # type: Animal + +h = House() +h.pet = Dog() + +while bool(): + if isinstance(h.pet, Dog): + if isinstance(h.pet.paws, str): + x = h.pet.paws + 'a' + y = h.pet.paws + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + z = h.pet.paws + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + if isinstance(h.pet.paws, str): + x = h.pet.paws + 'a' + break + y = h.pet.paws + 1 + z = h.pet.paws + 'a' # E: Unsupported operand types for + ("int" and "str") +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceSubClassReset] +class A: + pass + +class B(A): + b = 1 + +class C: + a = A() + +x = C() +x.a.b # E: "A" has no attribute "b" +if isinstance(x.a, B): + x.a.b + x = C() + x.a.b # E: "A" has no attribute "b" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceTuple] +from typing import Union + +class A: pass + +class B: + def method2(self, arg: int): + return 123 + +class C: + def method2(self, arg: int): + return 456 + + def method3(self, arg: str): + return 'abc' + +v = A() # type: Union[A, B, C] + +if isinstance(v, (B, C)): + v.method2(123) + v.method3('xyz') # E: Item "B" of "Union[B, C]" has no attribute "method3" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceNeverWidens] +from typing import Union + +class A: pass +class B: pass +class C: pass + +a = A() # type: A +assert isinstance(a, (A, B)) +reveal_type(a) # N: Revealed type is "__main__.A" + +b = A() # type: Union[A, B] +assert isinstance(b, (A, B, C)) +reveal_type(b) # N: Revealed type is "Union[__main__.A, __main__.B]" +[builtins fixtures/isinstance.pyi] + +[case testMemberAssignmentChanges] +from typing import Union + +class Dog: + paws = 1 # type: Union[int, str] + +pet = Dog() + +pet.paws + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" +pet.paws = 'a' +pet.paws + 'a' +pet.paws = 1 +pet.paws + 1 +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceSubClassMemberHard] +from typing import Union + +class Animal: + pass + +class Dog(Animal): + paws = 4 # type: Union[int, str] + + def bark(self): pass + +class House: + pet = None # type: Animal + +h = House() +h.pet = Dog() + +if isinstance(h.pet, Dog): + if isinstance(h.pet.paws, str): + for i in [1]: + # TODO: should we allow this if iterable is of length one or zero? + h.pet.paws + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + if bool(): + break + h.pet.paws = 1 + h.pet.paws + 1 + + if isinstance(h.pet.paws, str): + h.pet.paws + 'a' + else: + h.pet.paws + 1 +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceReturn] +from typing import Union + +def foo() -> None: + x = 1 # type: Union[int, str] + if isinstance(x, int): + return + y = x + 'asdad' + +def bar() -> None: + x = 1 # type: Union[int, str] + if isinstance(x, int): + return + else: + pass + y = x + 'asdad' + +foo() +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceBadBreak] +from typing import Union + +def foo() -> None: + x: Union[int, str] + if isinstance(x, int): + for z in [1,2]: + break + else: + pass + y = x + 'asdad' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + +foo() +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceThreeUnion] +from typing import Union, List + +x: Union[int, str, List[int]] + +while bool(): + if isinstance(x, int): + x + 1 + elif isinstance(x, str): + x + 'a' + else: + x + [1] + x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # E: Unsupported operand types for + ("List[int]" and "str") \ + # N: Left operand is of type "Union[int, str, List[int]]" + +x + [1] # E: Unsupported operand types for + ("int" and "List[int]") \ + # E: Unsupported operand types for + ("str" and "List[int]") \ + # N: Left operand is of type "Union[int, str, List[int]]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceThreeUnion2] +from typing import Union, List +x: Union[int, str, List[int]] +while bool(): + if isinstance(x, int): + x + 1 + break + elif isinstance(x, str): + x + 'a' + break + x + [1] + x + 'a' # E: Unsupported operand types for + ("List[int]" and "str") +x + [1] # E: Unsupported operand types for + ("int" and "List[int]") \ + # E: Unsupported operand types for + ("str" and "List[int]") \ + # N: Left operand is of type "Union[int, str, List[int]]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceThreeUnion3] +from typing import Union, List + +while bool(): + x: Union[int, str, List[int]] + def f(): x # Prevent redefinition + x = 1 + if isinstance(x, int): + x + 1 + break + elif isinstance(x, str): + x + 'a' + break + x + [1] # These lines aren't reached because x was an int + x + 'a' +x + [1] # E: Unsupported operand types for + ("int" and "List[int]") \ + # E: Unsupported operand types for + ("str" and "List[int]") \ + # N: Left operand is of type "Union[int, str, List[int]]" +[builtins fixtures/isinstancelist.pyi] + +[case testRemovingTypeRepeatedly] +from typing import Union + +def foo() -> Union[int, str]: pass + +for i in [1, 2]: + x = foo() + x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + if isinstance(x, int): + break + x + 'a' + + x = foo() + x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + if isinstance(x, int): + break + x + 'a' + + x = foo() + x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + if isinstance(x, int): + break + x + 'a' + +x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" +[builtins fixtures/isinstancelist.pyi] + +[case testModifyRepeatedly] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +def f(): x # Prevent redefinition +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" + +x = 1 +x + 1 +x + 'a' # E: Unsupported operand types for + ("int" and "str") + +x = 'a' +x + 1 # E: Unsupported operand types for + ("str" and "int") +x + 'a' + +x = foo() +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +x + 'a' # E: Unsupported operand types for + ("int" and "str") \ + # N: Left operand is of type "Union[int, str]" +[builtins fixtures/isinstancelist.pyi] + +[case testModifyLoop] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +def f(): x # Prevent redefinition +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +x = 'a' +x + 1 # E: Unsupported operand types for + ("str" and "int") +x = 1 +x + 1 + +while bool(): + x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + x = 'a' +[builtins fixtures/isinstancelist.pyi] + +[case testModifyLoop2] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +def f(): x # Prevent redefinition +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +x = 'a' +x + 1 # E: Unsupported operand types for + ("str" and "int") +x = 1 +x + 1 + +for i in [1]: + x = 'a' + +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +[builtins fixtures/isinstancelist.pyi] + +[case testModifyLoop3] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +def f(): x # Prevent redefinition +x = 1 + +while bool(): + x + 1 + x = 'a' + break +else: + x + 1 +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +x = 1 +for y in [1]: + x + 1 + x = 'a' + break +else: + x + 1 +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +[builtins fixtures/isinstancelist.pyi] + +[case testModifyLoopWhile4] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +def f(): x # Prevent redefinition +x = 1 + +while bool(): + x + 1 + if bool(): + x = 'a' + break +else: + x + 1 + x = 'a' +x + 'a' +x = 1 +while bool(): + x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + if bool(): + x = 'a' + continue +else: + x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + x = 'a' +x + 'a' +[builtins fixtures/isinstancelist.pyi] + +[case testModifyLoopFor4] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +def f(): x # Prevent redefinition +x = 1 + +for y in [1]: + x + 1 + if bool(): + x = 'a' + break +else: + x + 1 + x = 'a' +x + 'a' +x = 1 +for y in [1]: + x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + if bool(): + x = 'a' + continue +else: + x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + x = 'a' +x + 'a' +[builtins fixtures/isinstancelist.pyi] + +[case testModifyNestedLoop] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +def f(): x # Prevent redefinition +x = 1 + +for y in [1]: + for z in [1]: + break + else: + x = 'a' + break +else: + x + 1 +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +x = 1 +while bool(): + while bool(): + break + else: + x = 'a' + break +else: + x + 1 +x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +[builtins fixtures/isinstancelist.pyi] + +[case testModifyLoopLong] +from typing import Union + +class A: a = 1 + +def foo() -> Union[int, str, A]: pass + +def bar() -> None: + x = foo() + x + 1 # E: Unsupported left operand type for + ("A") \ + # N: Left operand is of type "Union[int, str, A]" \ + # E: Unsupported operand types for + ("str" and "int") + if isinstance(x, A): + x.a + else: + if isinstance(x, int): + x + 1 + x + 'a' # E: Unsupported operand types for + ("int" and "str") + else: + x + 'a' + x.a # E: "str" has no attribute "a" + x = A() + + if isinstance(x, str): + x + 'a' + else: + while bool(): + if isinstance(x, int): + x + 1 + else: + x.a + break + while bool(): + if isinstance(x, int): + x + 1 + else: + x.a + continue + + while bool(): + if isinstance(x, int): + x + 1 + else: + x.a # E: Item "str" of "Union[str, A]" has no attribute "a" + x = 'a' +[builtins fixtures/isinstancelist.pyi] + +[case testWhileExitCondition1] +from typing import Union +x = 1 # type: Union[int, str] +while isinstance(x, int): + if bool(): + continue + x = 'a' +else: + reveal_type(x) # N: Revealed type is "builtins.str" +reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstance.pyi] + +[case testWhileExitCondition2] +from typing import Union +x = 1 # type: Union[int, str] +while isinstance(x, int): + if bool(): + break + x = 'a' +else: + reveal_type(x) # N: Revealed type is "builtins.str" +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/isinstance.pyi] + +[case testWhileLinkedList] +from typing import Union +LinkedList = Union['Cons', 'Nil'] +class Nil: pass +class Cons: + tail = None # type: LinkedList +def last(x: LinkedList) -> Nil: + while isinstance(x, Cons): + x = x.tail + return x +[builtins fixtures/isinstance.pyi] + +[case testReturnAndFlow] +def foo() -> int: + return 1 and 2 + return 'a' +[case testCastIsinstance] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +y = 1 # type: int + +if isinstance(x, str): + x = y +x + 1 +x + 'a' # E: Unsupported operand types for + ("int" and "str") +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableCode] +x = 1 # type: int + +while bool(): + x = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + break + x = 'a' # Note: no error because unreachable code +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableCode2] +x = 1 +while bool(): + try: + pass + except: + continue + else: + continue + x + 'a' +[builtins fixtures/isinstance.pyi] + +[case testUnreachableWhileTrue] +def f(x: int) -> None: + while True: + if x: + return + 1() +[builtins fixtures/bool.pyi] + +[case testUnreachableAssertFalse] +def f() -> None: + assert False + 1() +[builtins fixtures/bool.pyi] + +[case testUnreachableAssertFalse2] +def f() -> None: + # The old parser doesn't understand the syntax below + assert False, "hi" + 1() +[builtins fixtures/bool.pyi] + +[case testUnreachableReturnOrAssertFalse] +def f(x: int) -> int: + if x: + return x + else: + assert False + 1() +[builtins fixtures/bool.pyi] + +[case testUnreachableTryExcept] +def f() -> None: + try: + f() + return + except BaseException: + return + 1() +[builtins fixtures/exception.pyi] + +[case testUnreachableTryExceptElse] +def f() -> None: + try: + f() + except BaseException: + return + else: + return + 1() +[builtins fixtures/exception.pyi] + +[case testUnreachableTryReturnFinally1] +def f() -> None: + try: + return + finally: + pass + 1() + +[case testUnreachableTryReturnFinally2] +def f() -> None: + try: + pass + finally: + return + 1() + +[case testUnreachableTryReturnExceptRaise] +def f() -> None: + try: + return + except: + raise + 1() + +[case testUnreachableReturnLambda] +from typing import Callable +def g(t: Callable[[int], int]) -> int: pass +def f() -> int: + return g(lambda x: x) + 1() + +[case testIsinstanceAnd] +class A: pass + +class B(A): + flag = 1 + +x = B() # type: A + +if isinstance(x, B) and 1: + x.flag +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceShortcircuit] +class A: + pass + +class B(A): + flag = 1 + +x = B() # type: A + +if isinstance(x, B) and x.flag: + pass +if isinstance(x, B) or x.flag: # E: "A" has no attribute "flag" + pass +if not isinstance(x, B) or x.flag: + pass +if not isinstance(x, B) and x.flag: # E: "A" has no attribute "flag" + pass +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceExpression] +class A: + pass + +class B(A): + flag = 1 + +x = B() # type: A + +x.flag if isinstance(x, B) else 0 +0 if not isinstance(x, B) else x.flag +0 if isinstance(x, B) else x.flag # E: "A" has no attribute "flag" +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceMultiAnd] +class A: pass + +class B(A): + flag = 1 + +class C(A): + glaf = 1 + +x = B() # type: A +y = C() # type: A + +if isinstance(x, B) and isinstance(y, C): + x.flag += 1 + y.glaf += 1 + x() # E: "B" not callable + y() # E: "C" not callable +else: + x() # E: "A" not callable + y() # E: "A" not callable +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceMultiAndSpecialCase] +class A: + # Ensure A.__add__ and int.__add__ are different to + # force 'isinstance(y, int)' checks below to never succeed. + def __add__(self, other: A) -> A: pass + +class B(A): + flag = 1 + +class C(A): + glaf = 1 + +x = B() # type: A +y = C() # type: A + +if isinstance(x, B) and isinstance(y, int): + 1() # type checking skipped +if isinstance(y, int) and isinstance(x, B): + 1() # type checking skipped +if isinstance(y, int) and y > 42: + 1() # type checking skipped +[builtins fixtures/isinstancelist.pyi] + +[case testReturnWithCallExprAndIsinstance] +from typing import Union + +def f(x: Union[int, str]) -> None: + if not isinstance(x, int): + return foo() + x() # E: "int" not callable + +def foo(): pass +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceOr1] +from typing import Optional + +def f(a: bool, x: object) -> Optional[int]: + if a or not isinstance(x, int): + return None + reveal_type(x) # N: Revealed type is "builtins.int" + return x +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceOr2] +from typing import Optional + +def g(a: bool, x: object) -> Optional[int]: + if not isinstance(x, int) or a: + return None + reveal_type(x) # N: Revealed type is "builtins.int" + return x +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceOr3] +from typing import Optional + +def h(a: bool, x: object) -> Optional[int]: + if a or isinstance(x, int): + return None + return x # E: Incompatible return value type (got "object", expected "Optional[int]") +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceWithOverlappingUnionType] +from typing import Union + +def f(x: Union[float, int]) -> None: + if isinstance(x, float): + pass + if not isinstance(x, int): + f(x) +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceWithOverlappingUnionType2] +from typing import Union + +class A: pass +class B(A): pass + +def f(x: Union[A, B]) -> None: + if isinstance(x, A): + pass + if not isinstance(x, B): + f(x) +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceWithOverlappingPromotionTypes] +from typing import Union + +class FloatLike: pass +class IntLike(FloatLike): pass + +def f1(x: Union[float, int]) -> None: + # We ignore promotions in isinstance checks + if isinstance(x, float): + reveal_type(x) # N: Revealed type is "builtins.float" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + +def f2(x: Union[FloatLike, IntLike]) -> None: + # ...but not regular subtyping relationships + if isinstance(x, FloatLike): + reveal_type(x) # N: Revealed type is "Union[__main__.FloatLike, __main__.IntLike]" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceOfSuperclass] +class A: pass +class B(A): pass + +x = B() +if isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__.B" +if not isinstance(x, A): + reveal_type(x) # unreachable + x = A() +reveal_type(x) # N: Revealed type is "__main__.B" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceOfNonoverlapping] +class A: pass +class B: pass + +x = B() +if isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__." +else: + reveal_type(x) # N: Revealed type is "__main__.B" +reveal_type(x) # N: Revealed type is "__main__.B" +[builtins fixtures/isinstance.pyi] + +[case testAssertIsinstance] +def f(x: object): + assert isinstance(x, int) + y = 0 # type: int + y = x +[builtins fixtures/isinstance.pyi] + +[case testUnionAssertIsinstance] +from typing import Union + +def f(x: Union[str, int]): + assert isinstance(x, int) + y = 0 # type: int + y = x +[builtins fixtures/isinstance.pyi] + +[case testAnyAssertIsinstance] +from typing import Any + +def f(x: Any): + assert isinstance(x, int) # this should narrow x to type int + x + "foo" # E: Unsupported operand types for + ("int" and "str") +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceOfGenericClassRetainsParameters] +from typing import List, Union + +def f(x: Union[List[int], str]) -> None: + if isinstance(x, list): + x[0]() # E: "int" not callable + else: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.str]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceOrIsinstance] +class A: pass + +class B(A): + flag = 1 + +class C(A): + flag = 2 + +x1 = A() +if isinstance(x1, B) or isinstance(x1, C): + reveal_type(x1) # N: Revealed type is "Union[__main__.B, __main__.C]" + f = x1.flag # type: int +else: + reveal_type(x1) # N: Revealed type is "__main__.A" + f = 0 +reveal_type(x1) # N: Revealed type is "__main__.A" +x2 = A() +if isinstance(x2, A) or isinstance(x2, C): + reveal_type(x2) # N: Revealed type is "__main__.A" + f = x2.flag # E: "A" has no attribute "flag" +else: + # unreachable + 1() +reveal_type(x2) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testComprehensionIsInstance] +from typing import List, Union +a = [] # type: List[Union[int, str]] +l = [x for x in a if isinstance(x, int)] +g = (x for x in a if isinstance(x, int)) +d = {0: x for x in a if isinstance(x, int)} +reveal_type(l) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(g) # N: Revealed type is "typing.Generator[builtins.int, None, None]" +reveal_type(d) # N: Revealed type is "builtins.dict[builtins.int, builtins.int]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceInWrongOrderInBooleanOp] +class A: + m = 1 + +def f(x: object) -> None: + if x.m and isinstance(x, A) or False: # E: "object" has no attribute "m" + pass +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceAndOr] +class A: + a = None # type: A + +def f(x: object) -> None: + b = isinstance(x, A) and x.a or A() + reveal_type(b) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceWithUnknownType] +from typing import Union + +def f(x: Union[int, str], typ: type) -> None: + if isinstance(x, (typ, int)): + x + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceWithBoundedType] +from typing import Union, Type + +class A: pass + +def f(x: Union[int, A], a: Type[A]) -> None: + if isinstance(x, (a, int)): + reveal_type(x) # N: Revealed type is "Union[builtins.int, __main__.A]" + else: + reveal_type(x) # N: Revealed type is "__main__.A" + reveal_type(x) # N: Revealed type is "Union[builtins.int, __main__.A]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceWithEmtpy2ndArg] +from typing import Union + +def f(x: Union[int, str]) -> None: + if isinstance(x, ()): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceWithTypeObject] +from typing import Union, Type + +class A: pass + +def f(x: Union[int, A], a: Type[A]) -> None: + if isinstance(x, a): + reveal_type(x) # N: Revealed type is "__main__.A" + elif isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "__main__.A" + reveal_type(x) # N: Revealed type is "Union[builtins.int, __main__.A]" +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclassUnreachable] +from typing import Type, Sequence, Union + +x: Type[str] +if issubclass(x, int): + reveal_type(x) # unreachable block + +class X: pass +class Y(X): pass +class Z(X): pass + +a: Union[Type[Y], Type[Z]] +if issubclass(a, X): + reveal_type(a) # N: Revealed type is "Union[Type[__main__.Y], Type[__main__.Z]]" +else: + reveal_type(a) # unreachable block +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclasDestructuringUnions1] +from typing import Union, List, Tuple, Dict, Type +def f(x: Union[Type[int], Type[str], Type[List]]) -> None: + if issubclass(x, (str, (int,))): + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str]" + x()[1] # E: Value of type "Union[int, str]" is not indexable + else: + reveal_type(x) # N: Revealed type is "Type[builtins.list[Any]]" + reveal_type(x()) # N: Revealed type is "builtins.list[Any]" + x()[1] + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" + if issubclass(x, (str, (list,))): + reveal_type(x) # N: Revealed type is "Union[Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.str, builtins.list[Any]]" + x()[1] + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclasDestructuringUnions2] +from typing import Union, List, Tuple, Dict, Type + +def f(x: Type[Union[int, str, List]]) -> None: + if issubclass(x, (str, (int,))): + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str]" + x()[1] # E: Value of type "Union[int, str]" is not indexable + else: + reveal_type(x) # N: Revealed type is "Type[builtins.list[Any]]" + reveal_type(x()) # N: Revealed type is "builtins.list[Any]" + x()[1] + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" + if issubclass(x, (str, (list,))): + reveal_type(x) # N: Revealed type is "Union[Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.str, builtins.list[Any]]" + x()[1] + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclasDestructuringUnions3] +from typing import Union, List, Tuple, Dict, Type + +def f(x: Type[Union[int, str, List]]) -> None: + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" + if issubclass(x, (str, (int,))): + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str]" + x()[1] # E: Value of type "Union[int, str]" is not indexable + else: + reveal_type(x) # N: Revealed type is "Type[builtins.list[Any]]" + reveal_type(x()) # N: Revealed type is "builtins.list[Any]" + x()[1] + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" + if issubclass(x, (str, (list,))): + reveal_type(x) # N: Revealed type is "Union[Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.str, builtins.list[Any]]" + x()[1] + reveal_type(x) # N: Revealed type is "Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]" + reveal_type(x()) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.list[Any]]" +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclass] +from typing import Type, ClassVar + +class Goblin: + level: int + +class GoblinAmbusher(Goblin): + job: ClassVar[str] = 'Ranger' + +def test_issubclass(cls: Type[Goblin]) -> None: + if issubclass(cls, GoblinAmbusher): + reveal_type(cls) # N: Revealed type is "Type[__main__.GoblinAmbusher]" + cls.level + cls.job + ga = cls() + ga.level = 15 + ga.job + ga.job = "Warrior" # E: Cannot assign to class variable "job" via instance + else: + reveal_type(cls) # N: Revealed type is "Type[__main__.Goblin]" + cls.level + cls.job # E: "Type[Goblin]" has no attribute "job" + g = cls() + g.level = 15 + g.job # E: "Goblin" has no attribute "job" +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclassDeepHierarchy] +from typing import Type, ClassVar + +class Mob: pass + +class Goblin(Mob): + level: int + +class GoblinAmbusher(Goblin): + job: ClassVar[str] = 'Ranger' + +def test_issubclass(cls: Type[Mob]) -> None: + if issubclass(cls, Goblin): + reveal_type(cls) # N: Revealed type is "Type[__main__.Goblin]" + cls.level + cls.job # E: "Type[Goblin]" has no attribute "job" + g = cls() + g.level = 15 + g.job # E: "Goblin" has no attribute "job" + if issubclass(cls, GoblinAmbusher): + reveal_type(cls) # N: Revealed type is "Type[__main__.GoblinAmbusher]" + cls.level + cls.job + g = cls() + g.level = 15 + g.job + g.job = 'Warrior' # E: Cannot assign to class variable "job" via instance + else: + reveal_type(cls) # N: Revealed type is "Type[__main__.Mob]" + cls.job # E: "Type[Mob]" has no attribute "job" + cls.level # E: "Type[Mob]" has no attribute "level" + m = cls() + m.level = 15 # E: "Mob" has no attribute "level" + m.job # E: "Mob" has no attribute "job" + if issubclass(cls, GoblinAmbusher): + reveal_type(cls) # N: Revealed type is "Type[__main__.GoblinAmbusher]" + cls.job + cls.level + ga = cls() + ga.level = 15 + ga.job + ga.job = 'Warrior' # E: Cannot assign to class variable "job" via instance + + if issubclass(cls, GoblinAmbusher): + reveal_type(cls) # N: Revealed type is "Type[__main__.GoblinAmbusher]" + cls.level + cls.job + ga = cls() + ga.level = 15 + ga.job + ga.job = "Warrior" # E: Cannot assign to class variable "job" via instance +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclassTuple] +from typing import Type, ClassVar + +class Mob: pass + +class Goblin(Mob): + level: int + +class GoblinAmbusher(Goblin): + job: ClassVar[str] = 'Ranger' + +class GoblinDigger(Goblin): + job: ClassVar[str] = 'Thief' + +def test_issubclass(cls: Type[Mob]) -> None: + if issubclass(cls, (Goblin, GoblinAmbusher)): + reveal_type(cls) # N: Revealed type is "Type[__main__.Goblin]" + cls.level + cls.job # E: "Type[Goblin]" has no attribute "job" + g = cls() + g.level = 15 + g.job # E: "Goblin" has no attribute "job" + if issubclass(cls, GoblinAmbusher): + cls.level + reveal_type(cls) # N: Revealed type is "Type[__main__.GoblinAmbusher]" + cls.job + ga = cls() + ga.level = 15 + ga.job + ga.job = "Warrior" # E: Cannot assign to class variable "job" via instance + else: + reveal_type(cls) # N: Revealed type is "Type[__main__.Mob]" + cls.job # E: "Type[Mob]" has no attribute "job" + cls.level # E: "Type[Mob]" has no attribute "level" + m = cls() + m.level = 15 # E: "Mob" has no attribute "level" + m.job # E: "Mob" has no attribute "job" + if issubclass(cls, GoblinAmbusher): + reveal_type(cls) # N: Revealed type is "Type[__main__.GoblinAmbusher]" + cls.job + cls.level + ga = cls() + ga.level = 15 + ga.job + ga.job = "Warrior" # E: Cannot assign to class variable "job" via instance + + if issubclass(cls, (GoblinDigger, GoblinAmbusher)): + reveal_type(cls) # N: Revealed type is "Union[Type[__main__.GoblinDigger], Type[__main__.GoblinAmbusher]]" + cls.level + cls.job + g = cls() + g.level = 15 + g.job + g.job = "Warrior" # E: Cannot assign to class variable "job" via instance +[builtins fixtures/isinstancelist.pyi] + +[case testIssubclassBuiltins] +from typing import List, Type + +class MyList(List): pass +class MyIntList(List[int]): pass + +def f(cls: Type[object]) -> None: + if issubclass(cls, MyList): + reveal_type(cls) # N: Revealed type is "Type[__main__.MyList]" + cls()[0] + else: + reveal_type(cls) # N: Revealed type is "Type[builtins.object]" + cls()[0] # E: Value of type "object" is not indexable + + if issubclass(cls, MyIntList): + reveal_type(cls) # N: Revealed type is "Type[__main__.MyIntList]" + cls()[0] + 1 +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceTypeArgs] +from typing import Iterable, TypeVar +x = 1 +isinstance(x, Iterable) +isinstance(x, Iterable[int]) # E: Parameterized generics cannot be used with class or instance checks +isinstance(x, (int, Iterable[int])) # E: Parameterized generics cannot be used with class or instance checks +isinstance(x, (int, (str, Iterable[int]))) # E: Parameterized generics cannot be used with class or instance checks +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + +[case testIsinstanceAnyAlias] +from typing import Any +A = Any +isinstance(object(), A) # E: Cannot use isinstance() with Any type +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceTypeArgsAliases] +from typing import Iterable, TypeVar +x = 1 +T = TypeVar('T') +It = Iterable +It2 = Iterable[T] + +isinstance(x, It[int]) # E: Parameterized generics cannot be used with class or instance checks +isinstance(x, It) +isinstance(x, It2[int]) # E: Parameterized generics cannot be used with class or instance checks +isinstance(x, It2) # E: Parameterized generics cannot be used with class or instance checks +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testIssubclassTypeArgs] +from typing import Iterable, TypeVar +x = int +issubclass(x, Iterable) +issubclass(x, Iterable[int]) # E: Parameterized generics cannot be used with class or instance checks +issubclass(x, (int, Iterable[int])) # E: Parameterized generics cannot be used with class or instance checks +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testIssubclassWithMetaclasses] +# flags: --no-strict-optional +class FooMetaclass(type): ... +class Foo(metaclass=FooMetaclass): ... +class Bar: ... + +fm: FooMetaclass +reveal_type(fm) # N: Revealed type is "__main__.FooMetaclass" +if issubclass(fm, Foo): + reveal_type(fm) # N: Revealed type is "Type[__main__.Foo]" +if issubclass(fm, Bar): + reveal_type(fm) # N: Revealed type is "None" +[builtins fixtures/isinstance.pyi] + +[case testIssubclassWithMetaclassesStrictOptional] +class FooMetaclass(type): ... +class BarMetaclass(type): ... +class Foo(metaclass=FooMetaclass): ... +class Bar(metaclass=BarMetaclass): ... +class Baz: ... + +fm: FooMetaclass +reveal_type(fm) # N: Revealed type is "__main__.FooMetaclass" +if issubclass(fm, Foo): + reveal_type(fm) # N: Revealed type is "Type[__main__.Foo]" +if issubclass(fm, Bar): + reveal_type(fm) # N: Revealed type is "Never" +if issubclass(fm, Baz): + reveal_type(fm) # N: Revealed type is "Never" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceAndNarrowTypeVariable] +from typing import TypeVar + +class A: pass +class B(A): pass + +T = TypeVar('T', bound=A) + +def f(x: T) -> None: + if isinstance(x, B): + reveal_type(x) # N: Revealed type is "__main__.B" + else: + reveal_type(x) # N: Revealed type is "T`-1" + reveal_type(x) # N: Revealed type is "T`-1" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceAndNegativeNarrowTypeVariableWithUnionBound] +from typing import Union, TypeVar + +class A: + a: int +class B: + b: int + +T = TypeVar("T", bound=Union[A, B]) + +def f(x: T) -> T: + if isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__.A" + x.a + x.b # E: "A" has no attribute "b" + else: + reveal_type(x) # N: Revealed type is "T`-1" + x.a # E: "T" has no attribute "a" + x.b + x.a # E: Item "B" of the upper bound "Union[A, B]" of type variable "T" has no attribute "a" + x.b # E: Item "A" of the upper bound "Union[A, B]" of type variable "T" has no attribute "b" + return x +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceAndTypeType] +from typing import Type +def f(x: Type[int]) -> None: + if isinstance(x, type): + reveal_type(x) # N: Revealed type is "Type[builtins.int]" + else: + reveal_type(x) # Unreachable + reveal_type(x) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceVariableSubstitution] +T = (int, str) +U = (list, T) +x: object = None + +if isinstance(x, T): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +if isinstance(x, U): + reveal_type(x) # N: Revealed type is "Union[builtins.list[Any], builtins.int, builtins.str]" + +if isinstance(x, (set, (list, T))): + reveal_type(x) # N: Revealed type is "Union[builtins.set[Any], builtins.list[Any], builtins.int, builtins.str]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceTooFewArgs] +isinstance() # E: Missing positional arguments "x", "t" in call to "isinstance" +x: object +if isinstance(): # E: Missing positional arguments "x", "t" in call to "isinstance" + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" +if isinstance(x): # E: Missing positional argument "t" in call to "isinstance" + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] + +[case testIsSubclassTooFewArgs] +from typing import Type + +issubclass() # E: Missing positional arguments "x", "t" in call to "issubclass" +y: Type[object] +if issubclass(): # E: Missing positional arguments "x", "t" in call to "issubclass" + reveal_type(y) # N: Revealed type is "Type[builtins.object]" +if issubclass(y): # E: Missing positional argument "t" in call to "issubclass" + reveal_type(y) # N: Revealed type is "Type[builtins.object]" + +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceTooManyArgs] +isinstance(1, 1, 1) # E: Too many arguments for "isinstance" \ + # E: Argument 2 to "isinstance" has incompatible type "int"; expected "Union[type, Tuple[Any, ...]]" +x: object +if isinstance(x, str, 1): # E: Too many arguments for "isinstance" + reveal_type(x) # N: Revealed type is "builtins.object" + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] + +[case testIsinstanceNarrowAny] +from typing import Any + +def narrow_any_to_str_then_reassign_to_int() -> None: + v = 1 # type: Any + + if isinstance(v, str): + reveal_type(v) # N: Revealed type is "builtins.str" + v = 2 + reveal_type(v) # N: Revealed type is "Any" + +[builtins fixtures/isinstance.pyi] + +[case testNarrowTypeAfterInList] +from typing import List, Optional + +x: List[int] +y: Optional[int] + +if y in x: + reveal_type(y) # N: Revealed type is "builtins.int" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +if y not in x: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] +[out] + +[case testNarrowTypeAfterInListOfOptional] +from typing import List, Optional + +x: List[Optional[int]] +y: Optional[int] + +if y not in x: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/list.pyi] +[out] + +[case testNarrowTypeAfterInListNonOverlapping] +from typing import List, Optional + +x: List[str] +y: Optional[int] + +if y in x: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/list.pyi] +[out] + +[case testNarrowTypeAfterInListNested] +from typing import List, Optional, Any + +x: Optional[int] +lst: Optional[List[int]] +nested_any: List[List[Any]] + +if lst in nested_any: + reveal_type(lst) # N: Revealed type is "builtins.list[builtins.int]" +if x in nested_any: + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/list.pyi] +[out] + +[case testNarrowTypeAfterInTuple] +from typing import Optional +class A: pass +class B(A): pass +class C(A): pass + +y: Optional[B] +if y in (B(), C()): + reveal_type(y) # N: Revealed type is "__main__.B" +else: + reveal_type(y) # N: Revealed type is "Union[__main__.B, None]" +[builtins fixtures/tuple.pyi] +[out] + +[case testNarrowTypeAfterInNamedTuple] +from typing import NamedTuple, Optional +class NT(NamedTuple): + x: int + y: int +nt: NT + +y: Optional[int] +if y not in nt: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[out] + +[case testNarrowTypeAfterInDict] +from typing import Dict, Optional +x: Dict[str, int] +y: Optional[str] + +if y in x: + reveal_type(y) # N: Revealed type is "builtins.str" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.str, None]" +if y not in x: + reveal_type(y) # N: Revealed type is "Union[builtins.str, None]" +else: + reveal_type(y) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] +[out] + +[case testNarrowTypeAfterInNoAnyOrObject] +from typing import Any, List, Optional +x: List[Any] +z: List[object] + +y: Optional[int] +if y in x: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" + +if y not in z: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +[typing fixtures/typing-medium.pyi] +[builtins fixtures/list.pyi] +[out] + +[case testNarrowTypeAfterInUserDefined] +from typing import Container, Optional + +class C(Container[int]): + def __contains__(self, item: object) -> bool: + return item is 'surprise' + +y: Optional[int] +# We never trust user defined types +if y in C(): + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +if y not in C(): + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +[typing fixtures/typing-full.pyi] +[builtins fixtures/list.pyi] +[out] + +[case testNarrowTypeAfterInSet] +from typing import Optional, Set +s: Set[str] + +y: Optional[str] +if y in {'a', 'b', 'c'}: + reveal_type(y) # N: Revealed type is "builtins.str" +else: + reveal_type(y) # N: Revealed type is "Union[builtins.str, None]" +if y not in s: + reveal_type(y) # N: Revealed type is "Union[builtins.str, None]" +else: + reveal_type(y) # N: Revealed type is "builtins.str" +[builtins fixtures/set.pyi] +[out] + +[case testNarrowTypeAfterInTypedDict] +from typing import Optional +from mypy_extensions import TypedDict +class TD(TypedDict): + a: int + b: str +td: TD + +def f() -> None: + x: Optional[str] + if x not in td: + return + reveal_type(x) # N: Revealed type is "builtins.str" +[typing fixtures/typing-typeddict.pyi] +[builtins fixtures/dict.pyi] +[out] + +[case testIsinstanceWidensWithAnyArg] +from typing import Any +class A: ... +B: Any +x: A +x.foo() # E: "A" has no attribute "foo" +assert isinstance(x, B) +x.foo() +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceWidensUnionWithAnyArg] +from typing import Any, Union +class A: ... +B: Any +x: Union[A, B] +reveal_type(x) # N: Revealed type is "Union[__main__.A, Any]" +assert isinstance(x, B) +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceIgnoredImport] +from typing import Union +from foo import A # type: ignore +def f(x: Union[A, str]) -> None: + x.method_only_in_a() # E: Item "str" of "Union[Any, str]" has no attribute "method_only_in_a" + if isinstance(x, A): + x.method_only_in_a() +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceIgnoredImportDualAny] +from typing import Any +from foo import Bad, OtherBad # type: ignore +x: Any +if isinstance(x, Bad): + reveal_type(x) # N: Revealed type is "Any" +else: + reveal_type(x) # N: Revealed type is "Any" + +if isinstance(x, (Bad, OtherBad)): + reveal_type(x) # N: Revealed type is "Any" +else: + reveal_type(x) # N: Revealed type is "Any" + +y: object +if isinstance(y, Bad): + reveal_type(y) # N: Revealed type is "Any" +else: + reveal_type(y) # N: Revealed type is "builtins.object" + +class Ok: pass +z: Any +if isinstance(z, Ok): + reveal_type(z) # N: Revealed type is "__main__.Ok" +else: + reveal_type(z) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceInitialNoneCheckSkipsImpossibleCasesNoStrictOptional] +from typing import Optional, Union + +class A: pass + +def foo1(x: Union[A, str, None]) -> None: + if x is None: + reveal_type(x) # N: Revealed type is "None" + elif isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__.A" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + +def foo2(x: Optional[str]) -> None: + if x is None: + reveal_type(x) # N: Revealed type is "None" + elif isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__." + else: + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceInitialNoneCheckSkipsImpossibleCasesInNoStrictOptional] +# flags: --no-strict-optional +from typing import Optional, Union + +class A: pass + +def foo1(x: Union[A, str, None]) -> None: + if x is None: + reveal_type(x) # N: Revealed type is "None" + elif isinstance(x, A): + # Note that Union[None, A] == A in no-strict-optional + reveal_type(x) # N: Revealed type is "__main__.A" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + +def foo2(x: Optional[str]) -> None: + if x is None: + reveal_type(x) # N: Revealed type is "None" + elif isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__." + else: + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstance.pyi] + +[case testNoneCheckDoesNotMakeTypeVarOptional] +from typing import TypeVar + +T = TypeVar('T') + +def foo_if(x: T) -> T: + out = None + out = x + if out is None: + pass + return out + +def foo_while(x: T) -> T: + out = None + out = x + while out is None: + pass + return out +[builtins fixtures/isinstance.pyi] + +[case testNoneCheckDoesNotNarrowWhenUsingTypeVarsNoStrictOptional] +# flags: --no-strict-optional +from typing import TypeVar + +T = TypeVar('T') + +def foo(x: T) -> T: + out = None + out = x + if out is None: + pass + return out +[builtins fixtures/isinstance.pyi] + +[case testNoneAndGenericTypesOverlapNoStrictOptional] +# flags: --no-strict-optional +from typing import Union, Optional, List + +# Note: this test is indirectly making sure meet.is_overlapping_types +# correctly ignores 'None' in unions. + +def foo(x: Optional[List[str]]) -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.str], None]" + assert isinstance(x, list) + reveal_type(x) # N: Revealed type is "builtins.list[builtins.str]" + +def bar(x: Union[List[str], List[int], None]) -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.str], builtins.list[builtins.int], None]" + assert isinstance(x, list) + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.str], builtins.list[builtins.int]]" +[builtins fixtures/isinstancelist.pyi] + +[case testNoneAndGenericTypesOverlapStrictOptional] +from typing import Union, Optional, List + +# This test is the same as the one above, except for strict-optional. +# It isn't testing anything explicitly and mostly exists for the sake +# of completeness. + +def foo(x: Optional[List[str]]) -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.str], None]" + assert isinstance(x, list) + reveal_type(x) # N: Revealed type is "builtins.list[builtins.str]" + +def bar(x: Union[List[str], List[int], None]) -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.str], builtins.list[builtins.int], None]" + assert isinstance(x, list) + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.str], builtins.list[builtins.int]]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceWithStarExpression] +from typing import Union, List, Tuple + +def f(var: Union[List[str], Tuple[str, str], str]) -> None: + reveal_type(var) # N: Revealed type is "Union[builtins.list[builtins.str], Tuple[builtins.str, builtins.str], builtins.str]" + if isinstance(var, (list, *(str, int))): + reveal_type(var) # N: Revealed type is "Union[builtins.list[builtins.str], builtins.str]" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceWithStarExpressionAndVariable] +from typing import Union + +def f(var: Union[int, str]) -> None: + reveal_type(var) # N: Revealed type is "Union[builtins.int, builtins.str]" + some_types = (str, tuple) + another_type = list + if isinstance(var, (*some_types, another_type)): + reveal_type(var) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceWithWrongStarExpression] +var = 'some string' +if isinstance(var, *(str, int)): # E: Too many arguments for "isinstance" + pass +[builtins fixtures/isinstancelist.pyi] + +[case testIsInstanceAdHocIntersectionBasic] +class A: + def f1(self) -> int: ... +class B: + def f2(self) -> int: ... +class C: + def f3(self) -> int: ... + +x: A +if isinstance(x, B): + reveal_type(x) # N: Revealed type is "__main__." + if isinstance(x, C): + reveal_type(x) # N: Revealed type is "__main__." + reveal_type(x.f1()) # N: Revealed type is "builtins.int" + reveal_type(x.f2()) # N: Revealed type is "builtins.int" + reveal_type(x.f3()) # N: Revealed type is "builtins.int" + x.bad() # E: "" has no attribute "bad" + else: + reveal_type(x) # N: Revealed type is "__main__." +else: + reveal_type(x) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionRepeatedChecks] +# flags: --warn-unreachable + +class A: pass +class B: pass + +x: A +if isinstance(x, B): + reveal_type(x) # N: Revealed type is "__main__." + if isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__." + if isinstance(x, B): + reveal_type(x) # N: Revealed type is "__main__." +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionIncompatibleClasses] +# flags: --warn-unreachable +class A: + def f(self) -> int: ... +class B: + def f(self) -> str: ... +class C: + def f(self) -> str: ... + +class Example(A, B): pass # E: Definition of "f" in base class "A" is incompatible with definition in base class "B" +x: A +if isinstance(x, B): # E: Subclass of "A" and "B" cannot exist: would have incompatible method signatures + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "__main__.A" + +y: C +if isinstance(y, B): + reveal_type(y) # N: Revealed type is "__main__." + if isinstance(y, A): # E: Subclass of "C", "B", and "A" cannot exist: would have incompatible method signatures + reveal_type(y) # E: Statement is unreachable +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionReversed] +# flags: --warn-unreachable + +from abc import abstractmethod +from typing_extensions import Literal + +class A0: + def f(self) -> Literal[0]: + ... + +class A1: + def f(self) -> Literal[1]: + ... + +class A2: + def f(self) -> Literal[2]: + ... + +class B: + @abstractmethod + def f(self) -> Literal[1, 2]: + ... + + def t0(self) -> None: + if isinstance(self, A0): # E: Subclass of "B" and "A0" cannot exist: would have incompatible method signatures + x0: Literal[0] = self.f() # E: Statement is unreachable + + def t1(self) -> None: + if isinstance(self, A1): + reveal_type(self) # N: Revealed type is "__main__." + x0: Literal[0] = self.f() # E: Incompatible types in assignment (expression has type "Literal[1]", variable has type "Literal[0]") + x1: Literal[1] = self.f() + + def t2(self) -> None: + if isinstance(self, (A0, A1)): + reveal_type(self) # N: Revealed type is "__main__.1" + x0: Literal[0] = self.f() # E: Incompatible types in assignment (expression has type "Literal[1]", variable has type "Literal[0]") + x1: Literal[1] = self.f() + + def t3(self) -> None: + if isinstance(self, (A1, A2)): + reveal_type(self) # N: Revealed type is "Union[__main__.2, __main__.]" + x0: Literal[0] = self.f() # E: Incompatible types in assignment (expression has type "Literal[1, 2]", variable has type "Literal[0]") + x1: Literal[1] = self.f() # E: Incompatible types in assignment (expression has type "Literal[1, 2]", variable has type "Literal[1]") + +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionGenerics] +# flags: --warn-unreachable +from typing import Generic, TypeVar + +class Parent: pass +class Child(Parent): pass + +T = TypeVar('T') +class A(Generic[T]): + def f(self) -> T: ... +class B: + def f(self) -> Parent: ... + +x: A[int] +if isinstance(x, B): # E: Subclass of "A[int]" and "B" cannot exist: would have incompatible method signatures + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "__main__.A[builtins.int]" + +y: A[Parent] +if isinstance(y, B): + reveal_type(y) # N: Revealed type is "__main__." + reveal_type(y.f()) # N: Revealed type is "__main__.Parent" +else: + reveal_type(y) # N: Revealed type is "__main__.A[__main__.Parent]" + +z: A[Child] +if isinstance(z, B): + reveal_type(z) # N: Revealed type is "__main__.1" + reveal_type(z.f()) # N: Revealed type is "__main__.Child" +else: + reveal_type(z) # N: Revealed type is "__main__.A[__main__.Child]" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionGenericsWithValues] +# flags: --warn-unreachable +from typing import TypeVar + +class A: + attr: int +class B: + attr: int +class C: + attr: str + +T1 = TypeVar('T1', A, B) +def f1(x: T1) -> T1: + if isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__.A" \ + # N: Revealed type is "__main__." + if isinstance(x, B): + reveal_type(x) # N: Revealed type is "__main__." \ + # N: Revealed type is "__main__." + else: + reveal_type(x) # N: Revealed type is "__main__.A" + else: + reveal_type(x) # N: Revealed type is "__main__.B" + return x + +T2 = TypeVar('T2', B, C) +def f2(x: T2) -> T2: + if isinstance(x, B): + reveal_type(x) # N: Revealed type is "__main__.B" + # Note: even though --warn-unreachable is set, we don't report + # errors for the below: we don't yet have a way of filtering out + # reachability errors that occur for only one variation of the + # TypeVar yet. + if isinstance(x, C): + reveal_type(x) + else: + reveal_type(x) # N: Revealed type is "__main__.B" + else: + reveal_type(x) # N: Revealed type is "__main__.C" + return x +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionGenericsWithValuesDirectReturn] +# flags: --warn-unreachable +from typing import TypeVar + +class A: + attr: int +class B: + attr: int +class C: + attr: str + +T1 = TypeVar('T1', A, B) +def f1(x: T1) -> T1: + if isinstance(x, A): + # The error message is confusing, but we indeed do run into problems if + # 'x' is a subclass of A and B + return A() # E: Incompatible return value type (got "A", expected "B") + else: + return B() + +T2 = TypeVar('T2', B, C) +def f2(x: T2) -> T2: + if isinstance(x, B): + # In contrast, it's impossible for a subclass of "B" and "C" to + # exist, so this is fine + return B() + else: + return C() +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionUsage] +# flags: --warn-unreachable +class A: pass +class B: pass +class Concrete(A, B): pass + +def accept_a(a: A) -> None: pass +def accept_b(a: B) -> None: pass +def accept_concrete(c: Concrete) -> None: pass + +x: A +if isinstance(x, B): + var = x + reveal_type(var) # N: Revealed type is "__main__." + accept_a(var) + accept_b(var) + accept_concrete(var) # E: Argument 1 to "accept_concrete" has incompatible type ""; expected "Concrete" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionReinfer] +# flags: --warn-unreachable +class A: pass +class B: pass + +x: A +assert isinstance(x, B) +reveal_type(x) # N: Revealed type is "__main__." + +y: A +assert isinstance(y, B) +reveal_type(y) # N: Revealed type is "__main__.1" + +x = y +reveal_type(x) # N: Revealed type is "__main__.1" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionWithUnions] +# flags: --warn-unreachable +from typing import Type, Union +class A: pass +class B: pass +class C: pass +class D: pass + +v1: A +if isinstance(v1, (B, C)): + reveal_type(v1) # N: Revealed type is "Union[__main__., __main__.]" + +v2: Union[A, B] +if isinstance(v2, C): + reveal_type(v2) # N: Revealed type is "Union[__main__.1, __main__.]" + +v3: Union[A, B] +if isinstance(v3, (C, D)): + reveal_type(v3) # N: Revealed type is "Union[__main__.2, __main__., __main__.1, __main__.]" +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionSameNames] +# flags: --warn-unreachable +from foo import A as A2 +class A: pass + +x: A +if isinstance(x, A2): + reveal_type(x) # N: Revealed type is "__main__." + +[file foo.py] +class A: pass +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionBadMro] +# flags: --warn-unreachable +class X: pass +class Y: pass +class A(X, Y): pass +class B(Y, X): pass + +foo: A +if isinstance(foo, B): # E: Subclass of "A" and "B" cannot exist: would have inconsistent method resolution order + reveal_type(foo) # E: Statement is unreachable +[builtins fixtures/isinstance.pyi] + +[case testIsInstanceAdHocIntersectionAmbiguousClass] +# flags: --warn-unreachable +from typing import Any + +class Concrete: + x: int +class Ambiguous: + x: Any + +# We bias towards assuming these two classes could be overlapping +foo: Concrete +if isinstance(foo, Ambiguous): + reveal_type(foo) # N: Revealed type is "__main__." + reveal_type(foo.x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] + +[case testIsSubclassAdHocIntersection] +# flags: --warn-unreachable +from typing import Type + +class A: + x: int +class B: + x: int +class C: + x: str + +x: Type[A] +if issubclass(x, B): + reveal_type(x) # N: Revealed type is "Type[__main__.]" + if issubclass(x, C): # E: Subclass of "A", "B", and "C" cannot exist: would have incompatible method signatures + reveal_type(x) # E: Statement is unreachable + else: + reveal_type(x) # N: Revealed type is "Type[__main__.]" +else: + reveal_type(x) # N: Revealed type is "Type[__main__.A]" +[builtins fixtures/isinstance.pyi] + +[case testTypeEqualsCheck] +from typing import Any + +y: Any +if type(y) == int: + reveal_type(y) # N: Revealed type is "builtins.int" + + +[case testMultipleTypeEqualsCheck] +from typing import Any + +x: Any +y: Any +if type(x) == type(y) == int: + reveal_type(y) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "builtins.int" + +[case testTypeEqualsCheckUsingIs] +from typing import Any + +y: Any +if type(y) is int: + reveal_type(y) # N: Revealed type is "builtins.int" + +[case testTypeEqualsNarrowingUnionWithElse] +from typing import Union + +x: Union[int, str] +if type(x) is int: + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testTypeEqualsMultipleTypesShouldntNarrow] +# make sure we don't do any narrowing if there are multiple types being compared + +from typing import Union + +x: Union[int, str] +if type(x) == int == str: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +# mypy thinks int isn't defined unless we include this +[builtins fixtures/primitives.pyi] + +[case testTypeNotEqualsCheck] +from typing import Union + +x: Union[int, str] +if type(x) != int: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +else: + reveal_type(x) # N: Revealed type is "builtins.int" + +# mypy thinks int isn't defined unless we include this +[builtins fixtures/primitives.pyi] + +[case testTypeNotEqualsCheckUsingIsNot] +from typing import Union + +x: Union[int, str] +if type(x) is not int: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +else: + reveal_type(x) # N: Revealed type is "builtins.int" + +[case testNarrowInElseCaseIfFinal] +from typing import final, Union +@final +class C: + pass +class D: + pass + +x: Union[C, D] +if type(x) is C: + reveal_type(x) # N: Revealed type is "__main__.C" +else: + reveal_type(x) # N: Revealed type is "__main__.D" +[case testNarrowInIfCaseIfFinalUsingIsNot] +from typing import final, Union +@final +class C: + pass +class D: + pass + +x: Union[C, D] +if type(x) is not C: + reveal_type(x) # N: Revealed type is "__main__.D" +else: + reveal_type(x) # N: Revealed type is "__main__.C" + +[case testHasAttrExistingAttribute] +class C: + x: int +c: C +if hasattr(c, "x"): + reveal_type(c.x) # N: Revealed type is "builtins.int" +else: + # We don't mark this unreachable since people may check for deleted attributes + reveal_type(c.x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeInstance] +class B: ... +b: B +if hasattr(b, "x"): + reveal_type(b.x) # N: Revealed type is "Any" +else: + b.x # E: "B" has no attribute "x" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeFunction] +def foo(x: int) -> None: ... +if hasattr(foo, "x"): + reveal_type(foo.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeClassObject] +class C: ... +if hasattr(C, "x"): + reveal_type(C.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeTypeType] +from typing import Type +class C: ... +c: Type[C] +if hasattr(c, "x"): + reveal_type(c.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeTypeVar] +from typing import TypeVar + +T = TypeVar("T") +def foo(x: T) -> T: + if hasattr(x, "x"): + reveal_type(x.x) # N: Revealed type is "Any" + return x + else: + return x +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeChained] +class B: ... +b: B +if hasattr(b, "x"): + reveal_type(b.x) # N: Revealed type is "Any" +elif hasattr(b, "y"): + reveal_type(b.y) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeNested] +class A: ... +class B: ... + +x: A +if hasattr(x, "x"): + if isinstance(x, B): + reveal_type(x.x) # N: Revealed type is "Any" + +if hasattr(x, "x") and hasattr(x, "y"): + reveal_type(x.x) # N: Revealed type is "Any" + reveal_type(x.y) # N: Revealed type is "Any" + +if hasattr(x, "x"): + if hasattr(x, "y"): + reveal_type(x.x) # N: Revealed type is "Any" + reveal_type(x.y) # N: Revealed type is "Any" + +if hasattr(x, "x") or hasattr(x, "y"): + x.x # E: "A" has no attribute "x" + x.y # E: "A" has no attribute "y" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrPreciseType] +class A: ... + +x: A +if hasattr(x, "a") and isinstance(x.a, int): + reveal_type(x.a) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeUnion] +from typing import Union + +class A: ... +class B: + x: int + +xu: Union[A, B] +if hasattr(xu, "x"): + reveal_type(xu) # N: Revealed type is "Union[__main__.A, __main__.B]" + reveal_type(xu.x) # N: Revealed type is "Union[Any, builtins.int]" +else: + reveal_type(xu) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeOuterUnion] +from typing import Union + +class A: ... +class B: ... +xu: Union[A, B] +if isinstance(xu, B): + if hasattr(xu, "x"): + reveal_type(xu.x) # N: Revealed type is "Any" + +if isinstance(xu, B) and hasattr(xu, "x"): + reveal_type(xu.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrDoesntInterfereGetAttr] +class C: + def __getattr__(self, attr: str) -> str: ... + +c: C +if hasattr(c, "foo"): + reveal_type(c.foo) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeLiteral] +from typing import Final +class B: ... +b: B +ATTR: Final = "x" +if hasattr(b, ATTR): + reveal_type(b.x) # N: Revealed type is "Any" +else: + b.x # E: "B" has no attribute "x" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrDeferred] +def foo() -> str: ... + +class Test: + def stream(self) -> None: + if hasattr(self, "_body"): + reveal_type(self._body) # N: Revealed type is "builtins.str" + + def body(self) -> str: + if not hasattr(self, "_body"): + self._body = foo() + return self._body +[builtins fixtures/isinstance.pyi] + +[case testHasAttrModule] +import mod + +if hasattr(mod, "y"): + reveal_type(mod.y) # N: Revealed type is "Any" + reveal_type(mod.x) # N: Revealed type is "builtins.int" +else: + mod.y # E: Module has no attribute "y" + reveal_type(mod.x) # N: Revealed type is "builtins.int" + +if hasattr(mod, "x"): + mod.y # E: Module has no attribute "y" + reveal_type(mod.x) # N: Revealed type is "builtins.int" +else: + mod.y # E: Module has no attribute "y" + reveal_type(mod.x) # N: Revealed type is "builtins.int" + +[file mod.py] +x: int +[builtins fixtures/module.pyi] + +[case testHasAttrDoesntInterfereModuleGetAttr] +import mod + +if hasattr(mod, "y"): + reveal_type(mod.y) # N: Revealed type is "builtins.str" + +[file mod.py] +def __getattr__(attr: str) -> str: ... +[builtins fixtures/module.pyi] diff --git a/test-data/unit/check-kwargs.test b/test-data/unit/check-kwargs.test new file mode 100644 index 000000000000..3a8c7f5ba454 --- /dev/null +++ b/test-data/unit/check-kwargs.test @@ -0,0 +1,569 @@ +-- Test cases for keyword arguments. + + +[case testTypeErrorInKeywordArgument] +import typing +def f(o: object) -> None: pass +f(o=None()) # E: "None" not callable + +[case testSimpleKeywordArgument] +import typing +class A: pass +def f(a: 'A') -> None: pass +f(a=A()) +f(a=object()) # E: Argument "a" to "f" has incompatible type "object"; expected "A" + +[case testTwoKeywordArgumentsNotInOrder] +import typing +class A: pass +class B: pass +def f(a: 'A', b: 'B') -> None: pass +f(b=A(), a=A()) # E: Argument "b" to "f" has incompatible type "A"; expected "B" +f(b=B(), a=B()) # E: Argument "a" to "f" has incompatible type "B"; expected "A" +f(a=A(), b=B()) +f(b=B(), a=A()) + +[case testOneOfSeveralOptionalKeywordArguments] +# flags: --implicit-optional +import typing +class A: pass +class B: pass +class C: pass +def f(a: 'A' = None, b: 'B' = None, c: 'C' = None) -> None: pass +f(a=A()) +f(b=B()) +f(c=C()) +f(b=B(), c=C()) +f(a=B()) # E: Argument "a" to "f" has incompatible type "B"; expected "Optional[A]" +f(b=A()) # E: Argument "b" to "f" has incompatible type "A"; expected "Optional[B]" +f(c=B()) # E: Argument "c" to "f" has incompatible type "B"; expected "Optional[C]" +f(b=B(), c=A()) # E: Argument "c" to "f" has incompatible type "A"; expected "Optional[C]" +[case testBothPositionalAndKeywordArguments] +import typing +class A: pass +class B: pass +def f(a: 'A', b: 'B') -> None: pass +f(A(), b=A()) # E: Argument "b" to "f" has incompatible type "A"; expected "B" +f(A(), b=B()) + +[case testContextSensitiveTypeInferenceForKeywordArg] +from typing import List +class A: pass +def f(a: 'A', b: 'List[A]') -> None: pass +f(b=[], a=A()) +[builtins fixtures/list.pyi] + +[case testGivingArgumentAsPositionalAndKeywordArg] +# flags: --no-strict-optional +import typing +class A: pass +class B: pass +def f(a: 'A', b: 'B' = None) -> None: pass +f(A(), a=A()) # E: "f" gets multiple values for keyword argument "a" + +[case testGivingArgumentAsPositionalAndKeywordArg2] +# flags: --no-strict-optional +import typing +class A: pass +class B: pass +def f(a: 'A' = None, b: 'B' = None) -> None: pass +f(A(), a=A()) # E: "f" gets multiple values for keyword argument "a" + +[case testPositionalAndKeywordForSameArg] +# This used to crash in check_argument_count(). See #1095. +def f(a: int): pass +def g(): f(0, a=1) +[out] + +[case testInvalidKeywordArgument] +import typing +def f(a: 'A') -> None: pass # N: "f" defined here +f(b=object()) # E: Unexpected keyword argument "b" for "f" +class A: pass + +[case testKeywordMisspelling] +class A: pass +def f(other: 'A') -> None: pass # N: "f" defined here +f(otter=A()) # E: Unexpected keyword argument "otter" for "f"; did you mean "other"? + +[case testMultipleKeywordsForMisspelling] +class A: pass +class B: pass +def f(thing : 'A', other: 'A', atter: 'A', btter: 'B') -> None: pass # N: "f" defined here +f(otter=A()) # E: Unexpected keyword argument "otter" for "f"; did you mean "atter" or "other"? + +[case testKeywordMisspellingDifferentType] +class A: pass +class B: pass +def f(other: 'A') -> None: pass # N: "f" defined here +f(otter=B()) # E: Unexpected keyword argument "otter" for "f"; did you mean "other"? + +[case testKeywordMisspellingInheritance] +class A: pass +class B(A): pass +class C: pass +def f(atter: 'A', btter: 'B', ctter: 'C') -> None: pass # N: "f" defined here +f(otter=B()) # E: Unexpected keyword argument "otter" for "f"; did you mean "atter" or "btter"? + +[case testKeywordMisspellingFloatInt] +def f(atter: float, btter: int) -> None: pass # N: "f" defined here +x: int = 5 +f(otter=x) # E: Unexpected keyword argument "otter" for "f"; did you mean "atter" or "btter"? + +[case testKeywordMisspellingVarArgs] +class A: pass +def f(other: 'A', *atter: 'A') -> None: pass # N: "f" defined here +f(otter=A()) # E: Unexpected keyword argument "otter" for "f"; did you mean "other"? +[builtins fixtures/tuple.pyi] + +[case testKeywordMisspellingOnlyVarArgs] +class A: pass +def f(*other: 'A') -> None: pass # N: "f" defined here +f(otter=A()) # E: Unexpected keyword argument "otter" for "f" +[builtins fixtures/tuple.pyi] + +[case testKeywordMisspellingVarArgsDifferentTypes] +class A: pass +class B: pass +def f(other: 'B', *atter: 'A') -> None: pass # N: "f" defined here +f(otter=A()) # E: Unexpected keyword argument "otter" for "f"; did you mean "other"? +[builtins fixtures/tuple.pyi] + +[case testKeywordMisspellingVarKwargs] +class A: pass +def f(other: 'A', **atter: 'A') -> None: pass +f(otter=A()) # E: Missing positional argument "other" in call to "f" +[builtins fixtures/dict.pyi] + +[case testKeywordArgumentsWithDynamicallyTypedCallable] +from typing import Any +f: Any +f(x=f(), z=None()) # E: "None" not callable +f(f, zz=None()) # E: "None" not callable +f(x=None) + +[case testKeywordArgumentWithFunctionObject] +from typing import Callable +class A: pass +class B: pass +f: Callable[[A, B], None] +f(a=A(), b=B()) # E: Unexpected keyword argument "a" # E: Unexpected keyword argument "b" +f(A(), b=B()) # E: Unexpected keyword argument "b" + +[case testKeywordOnlyArguments] +# flags: --no-strict-optional +import typing +class A: pass +class B: pass +def f(a: 'A', *, b: 'B' = None) -> None: pass +def g(a: 'A', *, b: 'B') -> None: pass +def h(a: 'A', *, b: 'B', aa: 'A') -> None: pass +def i(a: 'A', *, b: 'B', aa: 'A' = None) -> None: pass +f(A(), b=B()) +f(b=B(), a=A()) +f(A()) +f(A(), B()) # E: Too many positional arguments for "f" +g(A(), b=B()) +g(b=B(), a=A()) +g(A()) # E: Missing named argument "b" for "g" +g(A(), B()) # E: Too many positional arguments for "g" +h(A()) # E: Missing named argument "b" for "h" # E: Missing named argument "aa" for "h" +h(A(), b=B()) # E: Missing named argument "aa" for "h" +h(A(), aa=A()) # E: Missing named argument "b" for "h" +h(A(), b=B(), aa=A()) +h(A(), aa=A(), b=B()) +i(A()) # E: Missing named argument "b" for "i" +i(A(), b=B()) +i(A(), aa=A()) # E: Missing named argument "b" for "i" +i(A(), b=B(), aa=A()) +i(A(), aa=A(), b=B()) + +[case testKeywordOnlyArgumentsFastparse] +# flags: --no-strict-optional +import typing + +class A: pass +class B: pass + +def f(a: 'A', *, b: 'B' = None) -> None: pass +def g(a: 'A', *, b: 'B') -> None: pass +def h(a: 'A', *, b: 'B', aa: 'A') -> None: pass +def i(a: 'A', *, b: 'B', aa: 'A' = None) -> None: pass +f(A(), b=B()) +f(b=B(), a=A()) +f(A()) +f(A(), B()) # E: Too many positional arguments for "f" +g(A(), b=B()) +g(b=B(), a=A()) +g(A()) # E: Missing named argument "b" for "g" +g(A(), B()) # E: Too many positional arguments for "g" +h(A()) # E: Missing named argument "b" for "h" # E: Missing named argument "aa" for "h" +h(A(), b=B()) # E: Missing named argument "aa" for "h" +h(A(), aa=A()) # E: Missing named argument "b" for "h" +h(A(), b=B(), aa=A()) +h(A(), aa=A(), b=B()) +i(A()) # E: Missing named argument "b" for "i" +i(A(), b=B()) +i(A(), aa=A()) # E: Missing named argument "b" for "i" +i(A(), b=B(), aa=A()) +i(A(), aa=A(), b=B()) +[case testKwargsAfterBareArgs] +from typing import Tuple, Any +def f(a, *, b=None) -> None: pass +a = None # type: Any +b = None # type: Any +f(a, **b) + +[builtins fixtures/dict.pyi] + +[case testKeywordArgAfterVarArgs] +# flags: --implicit-optional +import typing +class A: pass +class B: pass +def f(*a: 'A', b: 'B' = None) -> None: pass +f() +f(A()) +f(A(), A()) +f(b=B()) +f(A(), b=B()) +f(A(), A(), b=B()) +f(B()) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +f(A(), B()) # E: Argument 2 to "f" has incompatible type "B"; expected "A" +f(b=A()) # E: Argument "b" to "f" has incompatible type "A"; expected "Optional[B]" +[builtins fixtures/list.pyi] + +[case testKeywordArgAfterVarArgsWithBothCallerAndCalleeVarArgs] +# flags: --implicit-optional --no-strict-optional +from typing import List +class A: pass +class B: pass +def f(*a: 'A', b: 'B' = None) -> None: pass +a = None # type: List[A] +f(*a) +f(A(), *a) +f(b=B()) +f(*a, b=B()) +f(A(), *a, b=B()) +f(A(), B()) # E: Argument 2 to "f" has incompatible type "B"; expected "A" +f(A(), b=A()) # E: Argument "b" to "f" has incompatible type "A"; expected "Optional[B]" +f(*a, b=A()) # E: Argument "b" to "f" has incompatible type "A"; expected "Optional[B]" +[builtins fixtures/list.pyi] + +[case testCallingDynamicallyTypedFunctionWithKeywordArgs] +import typing +class A: pass +def f(x, y=A()): pass # N: "f" defined here +f(x=A(), y=A()) +f(y=A(), x=A()) +f(y=A()) # E: Missing positional argument "x" in call to "f" +f(A(), z=A()) # E: Unexpected keyword argument "z" for "f" + +[case testKwargsArgumentInFunctionBody] +from typing import Dict, Any +def f( **kwargs: 'A') -> None: + d1 = kwargs # type: Dict[str, A] + d2 = kwargs # type: Dict[A, Any] # E: Incompatible types in assignment (expression has type "Dict[str, A]", variable has type "Dict[A, Any]") + d3 = kwargs # type: Dict[Any, str] # E: Incompatible types in assignment (expression has type "Dict[str, A]", variable has type "Dict[Any, str]") +class A: pass +[builtins fixtures/dict.pyi] +[out] + +[case testKwargsArgumentInFunctionBodyWithImplicitAny] +from typing import Dict, Any +def f(**kwargs) -> None: + d1 = kwargs # type: Dict[str, A] + d2 = kwargs # type: Dict[str, str] + d3 = kwargs # type: Dict[A, Any] # E: Incompatible types in assignment (expression has type "Dict[str, Any]", variable has type "Dict[A, Any]") +class A: pass +[builtins fixtures/dict.pyi] +[out] + +[case testCallingFunctionThatAcceptsVarKwargs] +import typing +class A: pass +class B: pass +def f( **kwargs: 'A') -> None: pass +f() +f(x=A()) +f(y=A(), z=A()) +f(x=B()) # E: Argument "x" to "f" has incompatible type "B"; expected "A" +f(A()) # E: Too many arguments for "f" +# Perhaps a better message would be "Too many *positional* arguments..." +[builtins fixtures/dict.pyi] + +[case testCallingFunctionWithKeywordVarArgs] +from typing import Dict +class A: pass +class B: pass +def f( **kwargs: 'A') -> None: pass +d: Dict[str, A] +f(**d) +f(x=A(), **d) +d2: Dict[str, B] +f(**d2) # E: Argument 1 to "f" has incompatible type "**Dict[str, B]"; expected "A" +f(x=A(), **d2) # E: Argument 2 to "f" has incompatible type "**Dict[str, B]"; expected "A" +f(**{'x': B()}) # E: Argument 1 to "f" has incompatible type "**Dict[str, B]"; expected "A" +[builtins fixtures/dict.pyi] + +[case testKwargsAllowedInDunderCall] +class Formatter: + def __call__(self, message: str, bold: bool = False) -> str: + pass + +formatter = Formatter() +formatter("test", bold=True) +reveal_type(formatter.__call__) # N: Revealed type is "def (message: builtins.str, bold: builtins.bool =) -> builtins.str" +[builtins fixtures/bool.pyi] +[out] + +[case testKwargsAllowedInDunderCallKwOnly] +class Formatter: + def __call__(self, message: str, *, bold: bool = False) -> str: + pass + +formatter = Formatter() +formatter("test", bold=True) +reveal_type(formatter.__call__) # N: Revealed type is "def (message: builtins.str, *, bold: builtins.bool =) -> builtins.str" +[builtins fixtures/bool.pyi] +[out] + +[case testPassingMappingForKeywordVarArg] +from typing import Mapping +def f(**kwargs: 'A') -> None: pass +b: Mapping +d: Mapping[A, A] +m: Mapping[str, A] +f(**d) # E: Keywords must be strings +f(**m) +f(**b) +class A: pass +[builtins fixtures/dict.pyi] + +[case testPassingMappingSubclassForKeywordVarArg] +from typing import Mapping +class MappingSubclass(Mapping[str, str]): pass +def f(**kwargs: 'A') -> None: pass +d: MappingSubclass +f(**d) +class A: pass +[builtins fixtures/dict.pyi] + +[case testInvalidTypeForKeywordVarArg] +from typing import Dict, Any, Optional +class A: pass +def f(**kwargs: 'A') -> None: pass +d = {} # type: Dict[A, A] +f(**d) # E: Keywords must be strings +f(**A()) # E: Argument after ** must be a mapping, not "A" +kwargs: Optional[Any] +f(**kwargs) # E: Argument after ** must be a mapping, not "Optional[Any]" + +def g(a: int) -> None: pass +g(a=1, **4) # E: Argument after ** must be a mapping, not "int" +[builtins fixtures/dict.pyi] + +[case testPassingKeywordVarArgsToNonVarArgsFunction] +from typing import Any, Dict +def f(a: 'A', b: 'B') -> None: pass +d: Dict[str, Any] +f(**d) +d2: Dict[str, A] +f(**d2) # E: Argument 1 to "f" has incompatible type "**Dict[str, A]"; expected "B" +class A: pass +class B: pass +[builtins fixtures/dict.pyi] + +[case testBothKindsOfVarArgs] +from typing import Any, List, Dict +def f(a: 'A', b: 'A') -> None: pass +l: List[Any] +d: Dict[Any, Any] +f(*l, **d) +class A: pass +[builtins fixtures/dict.pyi] + +[case testPassingMultipleKeywordVarArgs] +from typing import Any, Dict +def f1(a: 'A', b: 'A') -> None: pass +def f2(a: 'A') -> None: pass +def f3(a: 'A', **kwargs: 'A') -> None: pass +def f4(**kwargs: 'A') -> None: pass +d: Dict[Any, Any] +d2: Dict[Any, Any] +f1(**d, **d2) +f2(**d, **d2) +f3(**d, **d2) +f4(**d, **d2) +class A: pass +[builtins fixtures/dict.pyi] + +[case testPassingKeywordVarArgsToVarArgsOnlyFunction] +from typing import Any, Dict +def f(*args: 'A') -> None: pass +d: Dict[Any, Any] +f(**d) +class A: pass +[builtins fixtures/dict.pyi] + +[case testKeywordArgumentAndCommentSignature] +import typing +def f(x): # type: (int) -> str # N: "f" defined here + pass +f(x='') # E: Argument "x" to "f" has incompatible type "str"; expected "int" +f(x=0) +f(y=0) # E: Unexpected keyword argument "y" for "f" + +[case testKeywordArgumentAndCommentSignature2] +import typing +class A: + def f(self, x): # type: (int) -> str # N: "f" of "A" defined here + pass +A().f(x='') # E: Argument "x" to "f" of "A" has incompatible type "str"; expected "int" +A().f(x=0) +A().f(y=0) # E: Unexpected keyword argument "y" for "f" of "A" + +[case testKeywordVarArgsAndCommentSignature] +import typing +def f(**kwargs): # type: (**int) -> None + pass +f(z=1) +f(x=1, y=1) +f(x='', y=1) # E: Argument "x" to "f" has incompatible type "str"; expected "int" +f(x=1, y='') # E: Argument "y" to "f" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallsWithStars] +def f(a: int) -> None: + pass + +s = ('',) +f(*s) # E: Argument 1 to "f" has incompatible type "*Tuple[str]"; expected "int" + +a = {'': 0} +f(a) # E: Argument 1 to "f" has incompatible type "Dict[str, int]"; expected "int" +f(**a) # okay + +b = {'': ''} +f(b) # E: Argument 1 to "f" has incompatible type "Dict[str, str]"; expected "int" +f(**b) # E: Argument 1 to "f" has incompatible type "**Dict[str, str]"; expected "int" + +c = {0: 0} +f(**c) # E: Keywords must be strings +[builtins fixtures/dict.pyi] + +[case testCallStar2WithStar] +def f(**k): pass +f(*(1, 2)) # E: Too many arguments for "f" +[builtins fixtures/dict.pyi] + +[case testUnexpectedMethodKwargInNestedClass] +class A: + class B: + def __init__(self) -> None: # N: "B" defined here + pass +A.B(x=1) # E: Unexpected keyword argument "x" for "B" + +[case testUnexpectedMethodKwargFromOtherModule] +import m +m.A(x=1) +[file m.py] +1+'asdf' +class A: + def __init__(self) -> None: + pass +[out] +-- Note that the messages appear "out of order" because the m.py:3 +-- message is really an attachment to the main:2 error and should be +-- reported with it. +tmp/m.py:1: error: Unsupported operand types for + ("int" and "str") +main:2: error: Unexpected keyword argument "x" for "A" +tmp/m.py:3: note: "A" defined here + +[case testStarArgsAndKwArgsSpecialCase] +from typing import Dict, Mapping + +def f(*vargs: int, **kwargs: object) -> None: + pass + +def g(arg: int = 0, **kwargs: object) -> None: + pass + +d = {} # type: Dict[str, object] +f(**d) +g(**d) # E: Argument 1 to "g" has incompatible type "**Dict[str, object]"; expected "int" + +m = {} # type: Mapping[str, object] +f(**m) +g(**m) # E: Argument 1 to "g" has incompatible type "**Mapping[str, object]"; expected "int" +[builtins fixtures/dict.pyi] + +[case testPassingEmptyDictWithStars] +def f(): pass +def g(x=1): pass + +f(**{}) +g(**{}) +[builtins fixtures/dict.pyi] + +[case testKeywordUnpackWithDifferentTypes] +# https://github.com/python/mypy/issues/11144 +from typing import Dict, Generic, TypeVar, Mapping, Iterable + +T = TypeVar("T") +T2 = TypeVar("T2") + +class A(Dict[T, T2]): + ... + +class B(Mapping[T, T2]): + ... + +class C(Generic[T, T2]): + ... + +class D: + ... + +class E: + def keys(self) -> Iterable[str]: + ... + def __getitem__(self, key: str) -> float: + ... + +def foo(**i: float) -> float: + ... + +a: A[str, str] +b: B[str, str] +c: C[str, float] +d: D +e: E +f = {"a": "b"} + +foo(k=1.5) +foo(**a) +foo(**b) +foo(**c) +foo(**d) +foo(**e) +foo(**f) + +# Correct: + +class Good(Mapping[str, float]): + ... + +good1: Good +good2: A[str, float] +good3: B[str, float] +foo(**good1) +foo(**good2) +foo(**good3) +[out] +main:36: error: Argument 1 to "foo" has incompatible type "**A[str, str]"; expected "float" +main:37: error: Argument 1 to "foo" has incompatible type "**B[str, str]"; expected "float" +main:38: error: Argument after ** must be a mapping, not "C[str, float]" +main:39: error: Argument after ** must be a mapping, not "D" +main:41: error: Argument 1 to "foo" has incompatible type "**Dict[str, str]"; expected "float" +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test new file mode 100644 index 000000000000..77acdafd3319 --- /dev/null +++ b/test-data/unit/check-lists.test @@ -0,0 +1,96 @@ +-- Nested list assignment +-- ----------------------------- + +[case testNestedListAssignment] +from typing import List +a1: A +a2: A +b1: B +b2: B +c1: C +c2: C + +if int(): + a1, [b1, c1] = a2, [b2, c2] +if int(): + a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] +if int(): + a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.pyi] +[out] + +[case testNestedListAssignmentToTuple] +from typing import List +a: A +b: B +c: C + +a, b = [a, b] +a, b = [a] # E: Need more than 1 value to unpack (2 expected) +a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.pyi] +[out] + +[case testListAssignmentFromTuple] +from typing import List +a: A +b: B +c: C +t = a, b + +if int(): + [a, b], c = t, c +if int(): + [a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") +if int(): + [a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) +if int(): + [a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.pyi] +[out] + +[case testListAssignmentUnequalAmountToUnpack] +from typing import List +a: A +b: B +c: C + +def f() -> None: # needed because test parser tries to parse [a, b] as section header + [a, b] = [a, b] + [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) + [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.pyi] +[out] + +[case testListWithStarExpr] +(x, *a) = [1, 2, 3] +a = [1, *[2, 3]] +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" +b = [0, *a] +reveal_type(b) # N: Revealed type is "builtins.list[builtins.int]" +c = [*a, 0] +reveal_type(c) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testComprehensionShadowBinder] +def foo(x: object) -> None: + if isinstance(x, str): + [reveal_type(x) for x in [1, 2, 3]] # N: Revealed type is "builtins.int" + +[builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test new file mode 100644 index 000000000000..2f94b5df0f83 --- /dev/null +++ b/test-data/unit/check-literal.test @@ -0,0 +1,2986 @@ +-- +-- Check to see how we handle raw types, error handling, and other +-- semantic analysis shenanigans +-- + +[case testLiteralInvalidString] +from typing_extensions import Literal +def f1(x: 'A[') -> None: pass # E: Invalid type comment or annotation +def g1(x: Literal['A[']) -> None: pass +reveal_type(f1) # N: Revealed type is "def (x: Any)" +reveal_type(g1) # N: Revealed type is "def (x: Literal['A['])" + +def f2(x: 'A B') -> None: pass # E: Invalid type comment or annotation +def g2(x: Literal['A B']) -> None: pass +def h2(x: 'A|int') -> None: pass # E: Name "A" is not defined +def i2(x: Literal['A|B']) -> None: pass +reveal_type(f2) # N: Revealed type is "def (x: Any)" +reveal_type(g2) # N: Revealed type is "def (x: Literal['A B'])" +reveal_type(h2) # N: Revealed type is "def (x: Union[Any, builtins.int])" +reveal_type(i2) # N: Revealed type is "def (x: Literal['A|B'])" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInvalidTypeComment] +from typing_extensions import Literal +def f(x): # E: Syntax error in type comment "(A[) -> None" + # type: (A[) -> None + pass + +[case testLiteralInvalidTypeComment2] +from typing_extensions import Literal +def f(x): # E: Invalid type comment or annotation + # type: ("A[") -> None + pass + +def g(x): + # type: (Literal["A["]) -> None + pass + +reveal_type(f) # N: Revealed type is "def (x: Any)" +reveal_type(g) # N: Revealed type is "def (x: Literal['A['])" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralFromTypingWorks] +from typing import Literal + +x: Literal[42] +x = 43 # E: Incompatible types in assignment (expression has type "Literal[43]", variable has type "Literal[42]") + +y: Literal[43] +y = 43 +[typing fixtures/typing-medium.pyi] + +[case testLiteralInsideOtherTypes] +from typing import Tuple +from typing_extensions import Literal + +x: Tuple[1] # E: Invalid type: try using Literal[1] instead? +def foo(x: Tuple[1]) -> None: ... # E: Invalid type: try using Literal[1] instead? + +y: Tuple[Literal[2]] +def bar(x: Tuple[Literal[2]]) -> None: ... +reveal_type(x) # N: Revealed type is "Tuple[Any]" +reveal_type(y) # N: Revealed type is "Tuple[Literal[2]]" +reveal_type(bar) # N: Revealed type is "def (x: Tuple[Literal[2]])" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInsideOtherTypesTypeCommentsPython3] +from typing import Tuple, Optional +from typing_extensions import Literal + +x = None # type: Optional[Tuple[1]] # E: Invalid type: try using Literal[1] instead? +def foo(x): # E: Invalid type: try using Literal[1] instead? + # type: (Tuple[1]) -> None + pass + +y = None # type: Optional[Tuple[Literal[2]]] +def bar(x): + # type: (Tuple[Literal[2]]) -> None + pass +reveal_type(x) # N: Revealed type is "Union[Tuple[Any], None]" +reveal_type(y) # N: Revealed type is "Union[Tuple[Literal[2]], None]" +reveal_type(bar) # N: Revealed type is "def (x: Tuple[Literal[2]])" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralValidExpressionsInStringsPython3] +from wrapper import * + +[file wrapper.pyi] +from typing_extensions import Literal + +alias_1 = Literal['a+b'] +alias_2 = Literal['1+2'] +alias_3 = Literal['3'] +alias_4 = Literal['True'] +alias_5 = Literal['None'] +alias_6 = Literal['"foo"'] +expr_of_alias_1: alias_1 +expr_of_alias_2: alias_2 +expr_of_alias_3: alias_3 +expr_of_alias_4: alias_4 +expr_of_alias_5: alias_5 +expr_of_alias_6: alias_6 +reveal_type(expr_of_alias_1) # N: Revealed type is "Literal['a+b']" +reveal_type(expr_of_alias_2) # N: Revealed type is "Literal['1+2']" +reveal_type(expr_of_alias_3) # N: Revealed type is "Literal['3']" +reveal_type(expr_of_alias_4) # N: Revealed type is "Literal['True']" +reveal_type(expr_of_alias_5) # N: Revealed type is "Literal['None']" +reveal_type(expr_of_alias_6) # N: Revealed type is "Literal['"foo"']" + +expr_ann_1: Literal['a+b'] +expr_ann_2: Literal['1+2'] +expr_ann_3: Literal['3'] +expr_ann_4: Literal['True'] +expr_ann_5: Literal['None'] +expr_ann_6: Literal['"foo"'] +reveal_type(expr_ann_1) # N: Revealed type is "Literal['a+b']" +reveal_type(expr_ann_2) # N: Revealed type is "Literal['1+2']" +reveal_type(expr_ann_3) # N: Revealed type is "Literal['3']" +reveal_type(expr_ann_4) # N: Revealed type is "Literal['True']" +reveal_type(expr_ann_5) # N: Revealed type is "Literal['None']" +reveal_type(expr_ann_6) # N: Revealed type is "Literal['"foo"']" + +expr_str_1: "Literal['a+b']" +expr_str_2: "Literal['1+2']" +expr_str_3: "Literal['3']" +expr_str_4: "Literal['True']" +expr_str_5: "Literal['None']" +expr_str_6: "Literal['\"foo\"']" +reveal_type(expr_str_1) # N: Revealed type is "Literal['a+b']" +reveal_type(expr_str_2) # N: Revealed type is "Literal['1+2']" +reveal_type(expr_str_3) # N: Revealed type is "Literal['3']" +reveal_type(expr_str_4) # N: Revealed type is "Literal['True']" +reveal_type(expr_str_5) # N: Revealed type is "Literal['None']" +reveal_type(expr_str_6) # N: Revealed type is "Literal['"foo"']" + +expr_com_1 = ... # type: Literal['a+b'] +expr_com_2 = ... # type: Literal['1+2'] +expr_com_3 = ... # type: Literal['3'] +expr_com_4 = ... # type: Literal['True'] +expr_com_5 = ... # type: Literal['None'] +expr_com_6 = ... # type: Literal['"foo"'] +reveal_type(expr_com_1) # N: Revealed type is "Literal['a+b']" +reveal_type(expr_com_2) # N: Revealed type is "Literal['1+2']" +reveal_type(expr_com_3) # N: Revealed type is "Literal['3']" +reveal_type(expr_com_4) # N: Revealed type is "Literal['True']" +reveal_type(expr_com_5) # N: Revealed type is "Literal['None']" +reveal_type(expr_com_6) # N: Revealed type is "Literal['"foo"']" +[builtins fixtures/bool.pyi] +[out] + +[case testLiteralMixingUnicodeAndBytesPython3] +from typing_extensions import Literal + +a_ann: Literal[u"foo"] +b_ann: Literal["foo"] +c_ann: Literal[b"foo"] + +a_hint = u"foo" # type: Literal[u"foo"] +b_hint = "foo" # type: Literal["foo"] +c_hint = b"foo" # type: Literal[b"foo"] + +AAlias = Literal[u"foo"] +BAlias = Literal["foo"] +CAlias = Literal[b"foo"] +a_alias: AAlias +b_alias: BAlias +c_alias: CAlias + +def accepts_str_1(x: Literal[u"foo"]) -> None: pass +def accepts_str_2(x: Literal["foo"]) -> None: pass +def accepts_bytes(x: Literal[b"foo"]) -> None: pass + +reveal_type(a_ann) # N: Revealed type is "Literal['foo']" +reveal_type(b_ann) # N: Revealed type is "Literal['foo']" +reveal_type(c_ann) # N: Revealed type is "Literal[b'foo']" +reveal_type(a_hint) # N: Revealed type is "Literal['foo']" +reveal_type(b_hint) # N: Revealed type is "Literal['foo']" +reveal_type(c_hint) # N: Revealed type is "Literal[b'foo']" +reveal_type(a_alias) # N: Revealed type is "Literal['foo']" +reveal_type(b_alias) # N: Revealed type is "Literal['foo']" +reveal_type(c_alias) # N: Revealed type is "Literal[b'foo']" + +accepts_str_1(a_ann) +accepts_str_1(b_ann) +accepts_str_1(c_ann) # E: Argument 1 to "accepts_str_1" has incompatible type "Literal[b'foo']"; expected "Literal['foo']" +accepts_str_1(a_hint) +accepts_str_1(b_hint) +accepts_str_1(c_hint) # E: Argument 1 to "accepts_str_1" has incompatible type "Literal[b'foo']"; expected "Literal['foo']" +accepts_str_1(a_alias) +accepts_str_1(b_alias) +accepts_str_1(c_alias) # E: Argument 1 to "accepts_str_1" has incompatible type "Literal[b'foo']"; expected "Literal['foo']" + +accepts_str_2(a_ann) +accepts_str_2(b_ann) +accepts_str_2(c_ann) # E: Argument 1 to "accepts_str_2" has incompatible type "Literal[b'foo']"; expected "Literal['foo']" +accepts_str_2(a_hint) +accepts_str_2(b_hint) +accepts_str_2(c_hint) # E: Argument 1 to "accepts_str_2" has incompatible type "Literal[b'foo']"; expected "Literal['foo']" +accepts_str_2(a_alias) +accepts_str_2(b_alias) +accepts_str_2(c_alias) # E: Argument 1 to "accepts_str_2" has incompatible type "Literal[b'foo']"; expected "Literal['foo']" + +accepts_bytes(a_ann) # E: Argument 1 to "accepts_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" +accepts_bytes(b_ann) # E: Argument 1 to "accepts_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" +accepts_bytes(c_ann) +accepts_bytes(a_hint) # E: Argument 1 to "accepts_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" +accepts_bytes(b_hint) # E: Argument 1 to "accepts_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" +accepts_bytes(c_hint) +accepts_bytes(a_alias) # E: Argument 1 to "accepts_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" +accepts_bytes(b_alias) # E: Argument 1 to "accepts_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" +accepts_bytes(c_alias) +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralMixingUnicodeAndBytesPython3ForwardStrings] +from typing import TypeVar, Generic +from typing_extensions import Literal + +a_unicode_wrapper: u"Literal[u'foo']" +b_unicode_wrapper: u"Literal['foo']" +c_unicode_wrapper: u"Literal[b'foo']" + +a_str_wrapper: "Literal[u'foo']" +b_str_wrapper: "Literal['foo']" +c_str_wrapper: "Literal[b'foo']" + +# In Python 3, forward references MUST be str, not bytes +a_bytes_wrapper: b"Literal[u'foo']" # E: Invalid type comment or annotation +b_bytes_wrapper: b"Literal['foo']" # E: Invalid type comment or annotation +c_bytes_wrapper: b"Literal[b'foo']" # E: Invalid type comment or annotation + +reveal_type(a_unicode_wrapper) # N: Revealed type is "Literal['foo']" +reveal_type(b_unicode_wrapper) # N: Revealed type is "Literal['foo']" +reveal_type(c_unicode_wrapper) # N: Revealed type is "Literal[b'foo']" + +reveal_type(a_str_wrapper) # N: Revealed type is "Literal['foo']" +reveal_type(b_str_wrapper) # N: Revealed type is "Literal['foo']" +reveal_type(c_str_wrapper) # N: Revealed type is "Literal[b'foo']" + +T = TypeVar('T') +class Wrap(Generic[T]): pass + +AUnicodeWrapperAlias = Wrap[u"Literal[u'foo']"] +BUnicodeWrapperAlias = Wrap[u"Literal['foo']"] +CUnicodeWrapperAlias = Wrap[u"Literal[b'foo']"] +a_unicode_wrapper_alias: AUnicodeWrapperAlias +b_unicode_wrapper_alias: BUnicodeWrapperAlias +c_unicode_wrapper_alias: CUnicodeWrapperAlias + +AStrWrapperAlias = Wrap["Literal[u'foo']"] +BStrWrapperAlias = Wrap["Literal['foo']"] +CStrWrapperAlias = Wrap["Literal[b'foo']"] +a_str_wrapper_alias: AStrWrapperAlias +b_str_wrapper_alias: BStrWrapperAlias +c_str_wrapper_alias: CStrWrapperAlias + +ABytesWrapperAlias = Wrap[b"Literal[u'foo']"] +BBytesWrapperAlias = Wrap[b"Literal['foo']"] +CBytesWrapperAlias = Wrap[b"Literal[b'foo']"] +a_bytes_wrapper_alias: ABytesWrapperAlias +b_bytes_wrapper_alias: BBytesWrapperAlias +c_bytes_wrapper_alias: CBytesWrapperAlias + +# In Python 3, we assume that Literal['foo'] and Literal[u'foo'] are always +# equivalent, no matter what. +reveal_type(a_unicode_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal['foo']]" +reveal_type(b_unicode_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal['foo']]" +reveal_type(c_unicode_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal[b'foo']]" + +reveal_type(a_str_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal['foo']]" +reveal_type(b_str_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal['foo']]" +reveal_type(c_str_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal[b'foo']]" + +reveal_type(a_bytes_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal['foo']]" +reveal_type(b_bytes_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal['foo']]" +reveal_type(c_bytes_wrapper_alias) # N: Revealed type is "__main__.Wrap[Literal[b'foo']]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralUnicodeWeirdCharacters-skip_path_normalization] +from typing import Any +from typing_extensions import Literal + +a1: Literal["\x00\xAC\x62 \u2227 \u03bb(p)"] +b1: Literal["\x00¬b ∧ λ(p)"] +c1: Literal["¬b ∧ λ(p)"] +d1: Literal["\U0001F600"] +e1: Literal["😀"] + +Alias1 = Literal["\x00\xAC\x62 \u2227 \u03bb(p)"] +Alias2 = Literal["\x00¬b ∧ λ(p)"] +Alias3 = Literal["¬b ∧ λ(p)"] +Alias4 = Literal["\U0001F600"] +Alias5 = Literal["😀"] +a2: Alias1 +b2: Alias2 +c2: Alias3 +d2: Alias4 +e2: Alias5 + +blah: Any +a3 = blah # type: Literal["\x00\xAC\x62 \u2227 \u03bb(p)"] +b3 = blah # type: Literal["\x00¬b ∧ λ(p)"] +c3 = blah # type: Literal["¬b ∧ λ(p)"] +d3 = blah # type: Literal["\U0001F600"] +e3 = blah # type: Literal["😀"] + +reveal_type(a1) # N: Revealed type is "Literal['\x00¬b ∧ λ(p)']" +reveal_type(b1) # N: Revealed type is "Literal['\x00¬b ∧ λ(p)']" +reveal_type(c1) # N: Revealed type is "Literal['¬b ∧ λ(p)']" +reveal_type(d1) # N: Revealed type is "Literal['😀']" +reveal_type(e1) # N: Revealed type is "Literal['😀']" + +reveal_type(a2) # N: Revealed type is "Literal['\x00¬b ∧ λ(p)']" +reveal_type(b2) # N: Revealed type is "Literal['\x00¬b ∧ λ(p)']" +reveal_type(c2) # N: Revealed type is "Literal['¬b ∧ λ(p)']" +reveal_type(d2) # N: Revealed type is "Literal['😀']" +reveal_type(e2) # N: Revealed type is "Literal['😀']" + +reveal_type(a3) # N: Revealed type is "Literal['\x00¬b ∧ λ(p)']" +reveal_type(b3) # N: Revealed type is "Literal['\x00¬b ∧ λ(p)']" +reveal_type(c3) # N: Revealed type is "Literal['¬b ∧ λ(p)']" +reveal_type(d3) # N: Revealed type is "Literal['😀']" +reveal_type(e3) # N: Revealed type is "Literal['😀']" + +a1 = b1 +a1 = c1 # E: Incompatible types in assignment (expression has type "Literal['¬b ∧ λ(p)']", variable has type "Literal['\x00¬b ∧ λ(p)']") +a1 = a2 +a1 = b2 +a1 = c2 # E: Incompatible types in assignment (expression has type "Literal['¬b ∧ λ(p)']", variable has type "Literal['\x00¬b ∧ λ(p)']") +a1 = a3 +a1 = b3 +a1 = c3 # E: Incompatible types in assignment (expression has type "Literal['¬b ∧ λ(p)']", variable has type "Literal['\x00¬b ∧ λ(p)']") +[builtins fixtures/tuple.pyi] + +[out] + +[case testLiteralRenamingImportWorks] +from typing_extensions import Literal as Foo + +x: Foo[3] +reveal_type(x) # N: Revealed type is "Literal[3]" + +y: Foo["hello"] +reveal_type(y) # N: Revealed type is "Literal['hello']" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralRenamingImportViaAnotherImportWorks] +from other_module import Foo, Bar + +x: Foo[3] +y: Bar + +reveal_type(x) # N: Revealed type is "Literal[3]" +reveal_type(y) # N: Revealed type is "Literal[4]" + +[file other_module.py] +from typing_extensions import Literal as Foo +Bar = Foo[4] +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralRenamingImportNameConfusion] +from typing_extensions import Literal as Foo + +x: Foo["Foo"] +reveal_type(x) # N: Revealed type is "Literal['Foo']" + +y: Foo[Foo] # E: Literal[...] must have at least one parameter +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralBadRawExpressionWithBadType] + +NotAType = 3 +def f() -> NotAType['also' + 'not' + 'a' + 'type']: ... # E: Variable "__main__.NotAType" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid type comment or annotation + +# Note: this makes us re-inspect the type (e.g. via '_patch_indirect_dependencies' +# in build.py) so we can confirm the RawExpressionType did not leak out. +indirect = f() +[out] + +-- +-- Check to make sure we can construct the correct range of literal +-- types (and correctly reject invalid literal types) +-- +-- Note: the assignment tests exercise the logic in 'fastparse.py'; +-- the type alias tests exercise the logic in 'exprtotype.py'. +-- + +[case testLiteralBasicIntUsage] +from typing_extensions import Literal + +a1: Literal[4] +b1: Literal[0x2a] +c1: Literal[-300] +d1: Literal[+8] + +reveal_type(a1) # N: Revealed type is "Literal[4]" +reveal_type(b1) # N: Revealed type is "Literal[42]" +reveal_type(c1) # N: Revealed type is "Literal[-300]" +reveal_type(d1) # N: Revealed type is "Literal[8]" + +a2t = Literal[4] +b2t = Literal[0x2a] +c2t = Literal[-300] +d2t = Literal[+8] +a2: a2t +b2: b2t +c2: c2t +d2: d2t + +reveal_type(a2) # N: Revealed type is "Literal[4]" +reveal_type(b2) # N: Revealed type is "Literal[42]" +reveal_type(c2) # N: Revealed type is "Literal[-300]" +reveal_type(d2) # N: Revealed type is "Literal[8]" + +def f1(x: Literal[4]) -> Literal[4]: pass +def f2(x: Literal[0x2a]) -> Literal[0x2a]: pass +def f3(x: Literal[-300]) -> Literal[-300]: pass +def f4(x: Literal[+8]) -> Literal[+8]: pass + +reveal_type(f1) # N: Revealed type is "def (x: Literal[4]) -> Literal[4]" +reveal_type(f2) # N: Revealed type is "def (x: Literal[42]) -> Literal[42]" +reveal_type(f3) # N: Revealed type is "def (x: Literal[-300]) -> Literal[-300]" +reveal_type(f4) # N: Revealed type is "def (x: Literal[8]) -> Literal[8]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralBasicBoolUsage] +from typing_extensions import Literal + +a1: Literal[True] +b1: Literal[False] + +reveal_type(a1) # N: Revealed type is "Literal[True]" +reveal_type(b1) # N: Revealed type is "Literal[False]" + +a2t = Literal[True] +b2t = Literal[False] +a2: a2t +b2: b2t + +reveal_type(a2) # N: Revealed type is "Literal[True]" +reveal_type(b2) # N: Revealed type is "Literal[False]" + +def f1(x: Literal[True]) -> Literal[True]: pass +def f2(x: Literal[False]) -> Literal[False]: pass + +reveal_type(f1) # N: Revealed type is "def (x: Literal[True]) -> Literal[True]" +reveal_type(f2) # N: Revealed type is "def (x: Literal[False]) -> Literal[False]" +[builtins fixtures/bool.pyi] +[out] + +[case testLiteralBasicStrUsage] +from typing_extensions import Literal + +a: Literal[""] +b: Literal[" foo bar "] +c: Literal[' foo bar '] +d: Literal["foo"] +e: Literal['foo'] + +reveal_type(a) # N: Revealed type is "Literal['']" +reveal_type(b) # N: Revealed type is "Literal[' foo bar ']" +reveal_type(c) # N: Revealed type is "Literal[' foo bar ']" +reveal_type(d) # N: Revealed type is "Literal['foo']" +reveal_type(e) # N: Revealed type is "Literal['foo']" + +def f1(x: Literal[""]) -> Literal[""]: pass +def f2(x: Literal[" foo bar "]) -> Literal[" foo bar "]: pass +def f3(x: Literal[' foo bar ']) -> Literal[' foo bar ']: pass +def f4(x: Literal["foo"]) -> Literal["foo"]: pass +def f5(x: Literal['foo']) -> Literal['foo']: pass + +reveal_type(f1) # N: Revealed type is "def (x: Literal['']) -> Literal['']" +reveal_type(f2) # N: Revealed type is "def (x: Literal[' foo bar ']) -> Literal[' foo bar ']" +reveal_type(f3) # N: Revealed type is "def (x: Literal[' foo bar ']) -> Literal[' foo bar ']" +reveal_type(f4) # N: Revealed type is "def (x: Literal['foo']) -> Literal['foo']" +reveal_type(f5) # N: Revealed type is "def (x: Literal['foo']) -> Literal['foo']" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralBasicStrUsageSlashes-skip_path_normalization] +from typing_extensions import Literal + +a: Literal[r"foo\nbar"] +b: Literal["foo\nbar"] + +reveal_type(a) +reveal_type(b) +[builtins fixtures/tuple.pyi] +[out] +main:6: note: Revealed type is "Literal['foo\\nbar']" +main:7: note: Revealed type is "Literal['foo\nbar']" + +[case testLiteralBasicNoneUsage] +# Note: Literal[None] and None are equivalent +from typing_extensions import Literal +a: Literal[None] +reveal_type(a) # N: Revealed type is "None" + +def f1(x: Literal[None]) -> None: pass +def f2(x: None) -> Literal[None]: pass +def f3(x: Literal[None]) -> Literal[None]: pass + +reveal_type(f1) # N: Revealed type is "def (x: None)" +reveal_type(f2) # N: Revealed type is "def (x: None)" +reveal_type(f3) # N: Revealed type is "def (x: None)" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralCallingUnionFunction] +from typing_extensions import Literal + +def func(x: Literal['foo', 'bar', ' foo ']) -> None: ... + +func('foo') +func('bar') +func(' foo ') +func('baz') # E: Argument 1 to "func" has incompatible type "Literal['baz']"; expected "Literal['foo', 'bar', ' foo ']" + +a: Literal['foo'] +b: Literal['bar'] +c: Literal[' foo '] +d: Literal['foo', 'bar'] +e: Literal['foo', 'bar', ' foo '] +f: Literal['foo', 'bar', 'baz'] + +func(a) +func(b) +func(c) +func(d) +func(e) +func(f) # E: Argument 1 to "func" has incompatible type "Literal['foo', 'bar', 'baz']"; expected "Literal['foo', 'bar', ' foo ']" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralDisallowAny] +from typing import Any +from typing_extensions import Literal +from missing_module import BadAlias # E: Cannot find implementation or library stub for module named "missing_module" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +a: Literal[Any] # E: Parameter 1 of Literal[...] cannot be of type "Any" +b: Literal[BadAlias] # E: Parameter 1 of Literal[...] cannot be of type "Any" + +reveal_type(a) # N: Revealed type is "Any" +reveal_type(b) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralDisallowActualTypes] +from typing_extensions import Literal + +a: Literal[int] # E: Parameter 1 of Literal[...] is invalid +b: Literal[float] # E: Parameter 1 of Literal[...] is invalid +c: Literal[bool] # E: Parameter 1 of Literal[...] is invalid +d: Literal[str] # E: Parameter 1 of Literal[...] is invalid + +reveal_type(a) # N: Revealed type is "Any" +reveal_type(b) # N: Revealed type is "Any" +reveal_type(c) # N: Revealed type is "Any" +reveal_type(d) # N: Revealed type is "Any" +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralDisallowFloatsAndComplex] + +from typing_extensions import Literal +a1: Literal[3.14] # E: Parameter 1 of Literal[...] cannot be of type "float" +b1: 3.14 # E: Invalid type: float literals cannot be used as a type +c1: Literal[3j] # E: Parameter 1 of Literal[...] cannot be of type "complex" +d1: 3j # E: Invalid type: complex literals cannot be used as a type + +a2t = Literal[3.14] # E: Parameter 1 of Literal[...] cannot be of type "float" +b2t = 3.14 +c2t = Literal[3j] # E: Parameter 1 of Literal[...] cannot be of type "complex" +d2t = 3j + +a2: a2t +reveal_type(a2) # N: Revealed type is "Any" +b2: b2t # E: Variable "__main__.b2t" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +c2: c2t +reveal_type(c2) # N: Revealed type is "Any" +d2: d2t # E: Variable "__main__.d2t" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/complex_tuple.pyi] +[out] + +[case testLiteralDisallowComplexExpressions] +from typing_extensions import Literal +def dummy() -> int: return 3 +a: Literal[3 + 4] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +b: Literal[" foo ".trim()] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +d: Literal[~12] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +e: Literal[dummy()] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralDisallowCollections] +from typing_extensions import Literal +a: Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid +b: Literal[{1, 2, 3}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +c: {"a": 1, "b": 2} # E: Inline TypedDict is experimental, must be enabled with --enable-incomplete-feature=InlineTypedDict \ + # E: Invalid type: try using Literal[1] instead? \ + # E: Invalid type: try using Literal[2] instead? +d: {1, 2, 3} # E: Invalid type comment or annotation +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testLiteralDisallowCollections2] +from typing_extensions import Literal +a: (1, 2, 3) # E: Syntax error in type annotation \ + # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) +b: Literal[[1, 2, 3]] # E: Parameter 1 of Literal[...] is invalid +c: [1, 2, 3] # E: Bracketed expression "[...]" is not valid as a type +[builtins fixtures/tuple.pyi] + +[case testLiteralDisallowCollectionsTypeAlias] +from typing_extensions import Literal +at = Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid +bt = {"a": 1, "b": 2} +a: at +reveal_type(a) # N: Revealed type is "Any" +b: bt # E: Variable "__main__.bt" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testLiteralDisallowCollectionsTypeAlias2] +from typing_extensions import Literal +at = Literal[{1, 2, 3}] # E: Invalid type alias: expression is not a valid type +bt = {1, 2, 3} +a: at # E: Variable "__main__.at" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +b: bt # E: Variable "__main__.bt" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/set.pyi] +[out] + +[case testLiteralDisallowTypeVar] +from typing import TypeVar, Tuple +from typing_extensions import Literal + +T = TypeVar('T') + +at = Literal[T] # E: Parameter 1 of Literal[...] is invalid +a: at + +def foo(b: Literal[T]) -> Tuple[T]: pass # E: Parameter 1 of Literal[...] is invalid +[builtins fixtures/tuple.pyi] +[out] + + +-- +-- Test mixing and matching literals with other types +-- + +[case testLiteralMultipleValues] +from typing_extensions import Literal +a: Literal[1, 2, 3] +b: Literal["a", "b", "c"] +c: Literal[1, "b", True, None] +d: Literal[1, 1, 1] +e: Literal[None, None, None] +reveal_type(a) # N: Revealed type is "Union[Literal[1], Literal[2], Literal[3]]" +reveal_type(b) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" +reveal_type(c) # N: Revealed type is "Union[Literal[1], Literal['b'], Literal[True], None]" + +# Note: I was thinking these should be simplified, but it seems like +# mypy doesn't simplify unions with duplicate values with other types. +reveal_type(d) # N: Revealed type is "Union[Literal[1], Literal[1], Literal[1]]" +reveal_type(e) # N: Revealed type is "Union[None, None, None]" +[builtins fixtures/bool.pyi] +[out] + +[case testLiteralMultipleValuesExplicitTuple] +from typing_extensions import Literal +# Unfortunately, it seems like typed_ast is unable to distinguish this from +# Literal[1, 2, 3]. So we treat the two as being equivalent for now. +a: Literal[1, 2, 3] +b: Literal[(1, 2, 3)] +reveal_type(a) # N: Revealed type is "Union[Literal[1], Literal[2], Literal[3]]" +reveal_type(b) # N: Revealed type is "Union[Literal[1], Literal[2], Literal[3]]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralNestedUsage] + +from typing_extensions import Literal +a: Literal[Literal[3], 4, Literal["foo"]] +reveal_type(a) # N: Revealed type is "Union[Literal[3], Literal[4], Literal['foo']]" + +alias_for_literal = Literal[5] +b: Literal[alias_for_literal] +reveal_type(b) # N: Revealed type is "Literal[5]" + +another_alias = Literal[1, None] +c: Literal[alias_for_literal, another_alias, "r"] +reveal_type(c) # N: Revealed type is "Union[Literal[5], Literal[1], None, Literal['r']]" + +basic_mode = Literal["r", "w", "a"] +basic_with_plus = Literal["r+", "w+", "a+"] +combined: Literal[basic_mode, basic_with_plus] +reveal_type(combined) # N: Revealed type is "Union[Literal['r'], Literal['w'], Literal['a'], Literal['r+'], Literal['w+'], Literal['a+']]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralBiasTowardsAssumingForwardReference] +from typing_extensions import Literal + +a: "Foo" +reveal_type(a) # N: Revealed type is "__main__.Foo" + +b: Literal["Foo"] +reveal_type(b) # N: Revealed type is "Literal['Foo']" + +c: "Literal[Foo]" # E: Parameter 1 of Literal[...] is invalid + +d: "Literal['Foo']" +reveal_type(d) # N: Revealed type is "Literal['Foo']" + +class Foo: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralBiasTowardsAssumingForwardReferenceForTypeAliases] +from typing_extensions import Literal + +a: "Foo" +reveal_type(a) # N: Revealed type is "Literal[5]" + +b: Literal["Foo"] +reveal_type(b) # N: Revealed type is "Literal['Foo']" + +c: "Literal[Foo]" +reveal_type(c) # N: Revealed type is "Literal[5]" + +d: "Literal['Foo']" +reveal_type(d) # N: Revealed type is "Literal['Foo']" + +e: Literal[Foo, 'Foo'] +reveal_type(e) # N: Revealed type is "Union[Literal[5], Literal['Foo']]" + +Foo = Literal[5] +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralBiasTowardsAssumingForwardReferencesForTypeComments] +from typing_extensions import Literal + +a: Foo +reveal_type(a) # N: Revealed type is "__main__.Foo" + +b: "Foo" +reveal_type(b) # N: Revealed type is "__main__.Foo" + +c: Literal["Foo"] +reveal_type(c) # N: Revealed type is "Literal['Foo']" + +d: Literal[Foo] # E: Parameter 1 of Literal[...] is invalid + +class Foo: pass +[builtins fixtures/tuple.pyi] +[out] + + +-- +-- Check how we handle very basic subtyping and other useful things +-- + +[case testLiteralCallingFunction] +from typing_extensions import Literal +def foo(x: Literal[3]) -> None: pass + +a: Literal[1] +b: Literal[2] +c: int + +foo(a) # E: Argument 1 to "foo" has incompatible type "Literal[1]"; expected "Literal[3]" +foo(b) # E: Argument 1 to "foo" has incompatible type "Literal[2]"; expected "Literal[3]" +foo(c) # E: Argument 1 to "foo" has incompatible type "int"; expected "Literal[3]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralCallingFunctionWithUnionLiteral] +from typing_extensions import Literal +def foo(x: Literal[1, 2, 3]) -> None: pass + +a: Literal[1] +b: Literal[2, 3] +c: Literal[4, 5] +d: int + +foo(a) +foo(b) +foo(c) # E: Argument 1 to "foo" has incompatible type "Literal[4, 5]"; expected "Literal[1, 2, 3]" +foo(d) # E: Argument 1 to "foo" has incompatible type "int"; expected "Literal[1, 2, 3]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralCallingFunctionWithStandardBase] +from typing_extensions import Literal +def foo(x: int) -> None: pass + +a: Literal[1] +b: Literal[1, -4] +c: Literal[4, 'foo'] + +foo(a) +foo(b) +foo(c) # E: Argument 1 to "foo" has incompatible type "Literal[4, 'foo']"; expected "int" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralCheckSubtypingStrictOptional] +from typing import Any, NoReturn +from typing_extensions import Literal + +lit: Literal[1] +def f_lit(x: Literal[1]) -> None: pass + +def fa(x: Any) -> None: pass +def fb(x: NoReturn) -> None: pass +def fc(x: None) -> None: pass + +a: Any +b: NoReturn +c: None + +fa(lit) +fb(lit) # E: Argument 1 to "fb" has incompatible type "Literal[1]"; expected "Never" +fc(lit) # E: Argument 1 to "fc" has incompatible type "Literal[1]"; expected "None" + +f_lit(a) +f_lit(b) +f_lit(c) # E: Argument 1 to "f_lit" has incompatible type "None"; expected "Literal[1]" +[builtins fixtures/tuple.pyi] + +[case testLiteralCheckSubtypingNoStrictOptional] +# flags: --no-strict-optional +from typing import Any, NoReturn +from typing_extensions import Literal + +lit: Literal[1] +def f_lit(x: Literal[1]) -> None: pass + +def fa(x: Any) -> None: pass +def fb(x: NoReturn) -> None: pass +def fc(x: None) -> None: pass + +a: Any +b: NoReturn +c: None + +fa(lit) +fb(lit) # E: Argument 1 to "fb" has incompatible type "Literal[1]"; expected "Never" +fc(lit) # E: Argument 1 to "fc" has incompatible type "Literal[1]"; expected "None" + +f_lit(a) +f_lit(b) +f_lit(c) +[builtins fixtures/tuple.pyi] + +[case testLiteralCallingOverloadedFunction] +from typing import overload, Generic, TypeVar, Any +from typing_extensions import Literal + +T = TypeVar('T') +class IOLike(Generic[T]): pass + +@overload +def foo(x: Literal[1]) -> IOLike[int]: ... +@overload +def foo(x: Literal[2]) -> IOLike[str]: ... +@overload +def foo(x: int) -> IOLike[Any]: ... +def foo(x: int) -> IOLike[Any]: + if x == 1: + return IOLike[int]() + elif x == 2: + return IOLike[str]() + else: + return IOLike() + +a: Literal[1] +b: Literal[2] +c: int +d: Literal[3] + +reveal_type(foo(a)) # N: Revealed type is "__main__.IOLike[builtins.int]" +reveal_type(foo(b)) # N: Revealed type is "__main__.IOLike[builtins.str]" +reveal_type(foo(c)) # N: Revealed type is "__main__.IOLike[Any]" +foo(d) +[builtins fixtures/ops.pyi] +[out] + +[case testLiteralVariance] +from typing import Generic, TypeVar +from typing_extensions import Literal + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class Invariant(Generic[T]): pass +class Covariant(Generic[T_co]): pass +class Contravariant(Generic[T_contra]): pass + +a1: Invariant[Literal[1]] +a2: Invariant[Literal[1, 2]] +a3: Invariant[Literal[1, 2, 3]] +a2 = a1 # E: Incompatible types in assignment (expression has type "Invariant[Literal[1]]", variable has type "Invariant[Literal[1, 2]]") +a2 = a3 # E: Incompatible types in assignment (expression has type "Invariant[Literal[1, 2, 3]]", variable has type "Invariant[Literal[1, 2]]") + +b1: Covariant[Literal[1]] +b2: Covariant[Literal[1, 2]] +b3: Covariant[Literal[1, 2, 3]] +b2 = b1 +b2 = b3 # E: Incompatible types in assignment (expression has type "Covariant[Literal[1, 2, 3]]", variable has type "Covariant[Literal[1, 2]]") + +c1: Contravariant[Literal[1]] +c2: Contravariant[Literal[1, 2]] +c3: Contravariant[Literal[1, 2, 3]] +c2 = c1 # E: Incompatible types in assignment (expression has type "Contravariant[Literal[1]]", variable has type "Contravariant[Literal[1, 2]]") +c2 = c3 +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInListAndSequence] +from typing import List, Sequence +from typing_extensions import Literal + +def foo(x: List[Literal[1, 2]]) -> None: pass +def bar(x: Sequence[Literal[1, 2]]) -> None: pass + +a: List[Literal[1]] +b: List[Literal[1, 2, 3]] + +foo(a) # E: Argument 1 to "foo" has incompatible type "List[Literal[1]]"; expected "List[Literal[1, 2]]" \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant +foo(b) # E: Argument 1 to "foo" has incompatible type "List[Literal[1, 2, 3]]"; expected "List[Literal[1, 2]]" +bar(a) +bar(b) # E: Argument 1 to "bar" has incompatible type "List[Literal[1, 2, 3]]"; expected "Sequence[Literal[1, 2]]" +[builtins fixtures/list.pyi] +[out] + +[case testLiteralRenamingDoesNotChangeTypeChecking] +from typing_extensions import Literal as Foo +from other_module import Bar1, Bar2, c + +def func(x: Foo[15]) -> None: pass + +a: Bar1 +b: Bar2 +func(a) +func(b) # E: Argument 1 to "func" has incompatible type "Literal[14]"; expected "Literal[15]" +func(c) + +[file other_module.py] +from typing_extensions import Literal + +Bar1 = Literal[15] +Bar2 = Literal[14] +c: Literal[15] +[builtins fixtures/tuple.pyi] + + +-- +-- Check to make sure we handle inference of literal values correctly, +-- especially when doing assignments or calls +-- + +[case testLiteralInferredInAssignment] +from typing_extensions import Literal + +int1: Literal[1] = 1 +int2 = 1 +int3: int = 1 + +str1: Literal["foo"] = "foo" +str2 = "foo" +str3: str = "foo" + +bool1: Literal[True] = True +bool2 = True +bool3: bool = True + +none1: Literal[None] = None +none2 = None +none3: None = None + +reveal_type(int1) # N: Revealed type is "Literal[1]" +reveal_type(int2) # N: Revealed type is "builtins.int" +reveal_type(int3) # N: Revealed type is "builtins.int" +reveal_type(str1) # N: Revealed type is "Literal['foo']" +reveal_type(str2) # N: Revealed type is "builtins.str" +reveal_type(str3) # N: Revealed type is "builtins.str" +reveal_type(bool1) # N: Revealed type is "Literal[True]" +reveal_type(bool2) # N: Revealed type is "builtins.bool" +reveal_type(bool3) # N: Revealed type is "builtins.bool" +reveal_type(none1) # N: Revealed type is "None" +reveal_type(none2) # N: Revealed type is "None" +reveal_type(none3) # N: Revealed type is "None" +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralInferredOnlyForActualLiterals] +from typing_extensions import Literal + +w: Literal[1] +x: Literal["foo"] +y: Literal[True] +z: Literal[None] +combined: Literal[1, "foo", True, None] + +a = 1 +b = "foo" +c = True +d = None + +w = a # E: Incompatible types in assignment (expression has type "int", variable has type "Literal[1]") +x = b # E: Incompatible types in assignment (expression has type "str", variable has type "Literal['foo']") +y = c # E: Incompatible types in assignment (expression has type "bool", variable has type "Literal[True]") +z = d # This is ok: Literal[None] and None are equivalent. + +combined = a # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[Literal[1, 'foo', True]]") +combined = b # E: Incompatible types in assignment (expression has type "str", variable has type "Optional[Literal[1, 'foo', True]]") +combined = c # E: Incompatible types in assignment (expression has type "bool", variable has type "Optional[Literal[1, 'foo', True]]") +combined = d # Also ok, for similar reasons. + +e: Literal[1] = 1 +f: Literal["foo"] = "foo" +g: Literal[True] = True +h: Literal[None] = None + +w = e +x = f +y = g +z = h +combined = e +combined = f +combined = g +combined = h + +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralInferredTypeMustMatchExpected] +from typing_extensions import Literal + +a: Literal[1] = 2 # E: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]") +b: Literal["foo"] = "bar" # E: Incompatible types in assignment (expression has type "Literal['bar']", variable has type "Literal['foo']") +c: Literal[True] = False # E: Incompatible types in assignment (expression has type "Literal[False]", variable has type "Literal[True]") + +d: Literal[1, 2] = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", variable has type "Literal[1, 2]") +e: Literal["foo", "bar"] = "baz" # E: Incompatible types in assignment (expression has type "Literal['baz']", variable has type "Literal['foo', 'bar']") +f: Literal[True, 4] = False # E: Incompatible types in assignment (expression has type "Literal[False]", variable has type "Literal[True, 4]") + +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralInferredInCall] +from typing_extensions import Literal + +def f_int_lit(x: Literal[1]) -> None: pass +def f_int(x: int) -> None: pass + +def f_str_lit(x: Literal["foo"]) -> None: pass +def f_str(x: str) -> None: pass + +def f_bool_lit(x: Literal[True]) -> None: pass +def f_bool(x: bool) -> None: pass + +def f_none_lit(x: Literal[None]) -> None: pass +def f_none(x: None) -> None: pass + +i1: Literal[1] +i2: Literal[2] +f_int_lit(1) +f_int_lit(2) # E: Argument 1 to "f_int_lit" has incompatible type "Literal[2]"; expected "Literal[1]" +f_int(1) +f_int_lit(i1) +f_int_lit(i2) # E: Argument 1 to "f_int_lit" has incompatible type "Literal[2]"; expected "Literal[1]" + +s1: Literal["foo"] +s2: Literal["bar"] +f_str_lit("foo") +f_str_lit("bar") # E: Argument 1 to "f_str_lit" has incompatible type "Literal['bar']"; expected "Literal['foo']" +f_str("baz") +f_str_lit(s1) +f_str_lit(s2) # E: Argument 1 to "f_str_lit" has incompatible type "Literal['bar']"; expected "Literal['foo']" + +b1: Literal[True] +b2: Literal[False] +f_bool_lit(True) +f_bool_lit(False) # E: Argument 1 to "f_bool_lit" has incompatible type "Literal[False]"; expected "Literal[True]" +f_bool(True) +f_bool_lit(b1) +f_bool_lit(b2) # E: Argument 1 to "f_bool_lit" has incompatible type "Literal[False]"; expected "Literal[True]" + +n1: Literal[None] +f_none_lit(None) +f_none(None) +f_none_lit(n1) +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralInferredInReturnContext] +from typing_extensions import Literal + +def f1() -> int: + return 1 + +def f2() -> Literal[1]: + return 1 + +def f3() -> Literal[1]: + return 2 # E: Incompatible return value type (got "Literal[2]", expected "Literal[1]") + +def f4(x: Literal[1]) -> Literal[1]: + return x + +def f5(x: Literal[2]) -> Literal[1]: + return x # E: Incompatible return value type (got "Literal[2]", expected "Literal[1]") +[builtins fixtures/tuple.pyi] + +[out] + +[case testLiteralInferredInListContext] +from typing import List +from typing_extensions import Literal + +a: List[Literal[1]] = [1, 1, 1] +b = [1, 1, 1] +c: List[Literal[1, 2, 3]] = [1, 2, 3] +d = [1, 2, 3] +e: List[Literal[1, "x"]] = [1, "x"] +f = [1, "x"] +g: List[List[List[Literal[1, 2, 3]]]] = [[[1, 2, 3], [3]]] +h: List[Literal[1]] = [] + +reveal_type(a) # N: Revealed type is "builtins.list[Literal[1]]" +reveal_type(b) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(c) # N: Revealed type is "builtins.list[Union[Literal[1], Literal[2], Literal[3]]]" +reveal_type(d) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(e) # N: Revealed type is "builtins.list[Union[Literal[1], Literal['x']]]" +reveal_type(f) # N: Revealed type is "builtins.list[builtins.object]" +reveal_type(g) # N: Revealed type is "builtins.list[builtins.list[builtins.list[Union[Literal[1], Literal[2], Literal[3]]]]]" +reveal_type(h) # N: Revealed type is "builtins.list[Literal[1]]" + +lit1: Literal[1] +lit2: Literal[2] +lit3: Literal["foo"] + +arr1 = [lit1, lit1, lit1] +arr2 = [lit1, lit2] +arr3 = [lit1, 4, 5] +arr4 = [lit1, lit2, lit3] +arr5 = [object(), lit1] + +reveal_type(arr1) # N: Revealed type is "builtins.list[Literal[1]]" +reveal_type(arr2) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(arr3) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(arr4) # N: Revealed type is "builtins.list[builtins.object]" +reveal_type(arr5) # N: Revealed type is "builtins.list[builtins.object]" + +bad: List[Literal[1, 2]] = [1, 2, 3] # E: List item 2 has incompatible type "Literal[3]"; expected "Literal[1, 2]" + +[builtins fixtures/list.pyi] +[out] + +[case testLiteralInferredInTupleContext] +# Note: most of the 'are we handling context correctly' tests should have been +# handled up above, so we keep things comparatively simple for tuples and dicts. +from typing import Tuple +from typing_extensions import Literal + +a: Tuple[Literal[1], Literal[2]] = (1, 2) +b: Tuple[int, Literal[1, 2], Literal[3], Tuple[Literal["foo"]]] = (1, 2, 3, ("foo",)) +c: Tuple[Literal[1], Literal[2]] = (2, 1) # E: Incompatible types in assignment (expression has type "Tuple[Literal[2], Literal[1]]", variable has type "Tuple[Literal[1], Literal[2]]") +d = (1, 2) + +reveal_type(d) # N: Revealed type is "Tuple[builtins.int, builtins.int]" + +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInferredInDictContext] +from typing import Dict +from typing_extensions import Literal + +a = {"x": 1, "y": 2} +b: Dict[str, Literal[1, 2]] = {"x": 1, "y": 2} +c: Dict[Literal["x", "y"], int] = {"x": 1, "y": 2} + +reveal_type(a) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + +[builtins fixtures/dict.pyi] +[out] + +[case testLiteralInferredInOverloadContextBasic] +from typing import overload +from typing_extensions import Literal + +@overload +def func(x: Literal[1]) -> str: ... +@overload +def func(x: Literal[2]) -> int: ... +@overload +def func(x: int) -> object: ... +def func(x: int) -> object: pass + +a: Literal[1] +b: Literal[2] +c: Literal[1, 2] + +reveal_type(func(1)) # N: Revealed type is "builtins.str" +reveal_type(func(2)) # N: Revealed type is "builtins.int" +reveal_type(func(3)) # N: Revealed type is "builtins.object" +reveal_type(func(a)) # N: Revealed type is "builtins.str" +reveal_type(func(b)) # N: Revealed type is "builtins.int" + +# Note: the fact that we don't do union math here is consistent +# with the output we would have gotten if we replaced int and the +# Literal types here with regular classes/subclasses. +reveal_type(func(c)) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralOverloadProhibitUnsafeOverlaps] +from typing import overload +from typing_extensions import Literal + +@overload +def func1(x: Literal[1]) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def func1(x: int) -> int: ... +def func1(x): pass + +@overload +def func2(x: Literal['a']) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def func2(x: str) -> Literal[2]: ... +def func2(x): pass + +# This one is typesafe +@overload +def func3(x: Literal['a']) -> Literal[2]: ... +@overload +def func3(x: str) -> int: ... +def func3(x): pass +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInferredInOverloadContextUnionMath] +from typing import overload, Union +from typing_extensions import Literal + +class A: pass +class B: pass +class C: pass + +@overload +def func(x: Literal[-40]) -> A: ... +@overload +def func(x: Literal[3, 4, 5, 6]) -> B: ... +@overload +def func(x: Literal["foo"]) -> C: ... +def func(x: Union[int, str]) -> Union[A, B, C]: pass + +a: Literal[-40, "foo"] +b: Literal[3] +c: Literal[3, -40] +d: Literal[6, 7] +e: int +f: Literal[7, "bar"] + +reveal_type(func(a)) # N: Revealed type is "Union[__main__.A, __main__.C]" +reveal_type(func(b)) # N: Revealed type is "__main__.B" +reveal_type(func(c)) # N: Revealed type is "Union[__main__.B, __main__.A]" +reveal_type(func(d)) # N: Revealed type is "__main__.B" \ + # E: Argument 1 to "func" has incompatible type "Literal[6, 7]"; expected "Literal[3, 4, 5, 6]" + +reveal_type(func(e)) # E: No overload variant of "func" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def func(x: Literal[-40]) -> A \ + # N: def func(x: Literal[3, 4, 5, 6]) -> B \ + # N: def func(x: Literal['foo']) -> C \ + # N: Revealed type is "Any" + +reveal_type(func(f)) # E: No overload variant of "func" matches argument type "Literal[7, 'bar']" \ + # N: Possible overload variants: \ + # N: def func(x: Literal[-40]) -> A \ + # N: def func(x: Literal[3, 4, 5, 6]) -> B \ + # N: def func(x: Literal['foo']) -> C \ + # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInferredInOverloadContextUnionMathOverloadingReturnsBestType] +# This test is a transliteration of check-overloading::testUnionMathOverloadingReturnsBestType +from typing import overload +from typing_extensions import Literal + +@overload +def f(x: Literal[1, 2]) -> int: ... +@overload +def f(x: int) -> object: ... +def f(x): + pass + +x: Literal[1, 2] +y: Literal[1, 2, 3] +z: Literal[1, 2, "three"] +reveal_type(f(x)) # N: Revealed type is "builtins.int" +reveal_type(f(1)) # N: Revealed type is "builtins.int" +reveal_type(f(2)) # N: Revealed type is "builtins.int" +reveal_type(f(y)) # N: Revealed type is "builtins.object" +reveal_type(f(z)) # N: Revealed type is "builtins.int" \ + # E: Argument 1 to "f" has incompatible type "Literal[1, 2, 'three']"; expected "Literal[1, 2]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInferredInOverloadContextWithTypevars] +from typing import TypeVar, overload, Union +from typing_extensions import Literal + +T = TypeVar('T') + +@overload +def f1(x: T, y: int) -> T: ... +@overload +def f1(x: T, y: str) -> Union[T, str]: ... +def f1(x, y): pass + +a: Literal[1] +reveal_type(f1(1, 1)) # N: Revealed type is "builtins.int" +reveal_type(f1(a, 1)) # N: Revealed type is "Literal[1]" + +@overload +def f2(x: T, y: Literal[3]) -> T: ... +@overload +def f2(x: T, y: str) -> Union[T]: ... +def f2(x, y): pass + +reveal_type(f2(1, 3)) # N: Revealed type is "builtins.int" +reveal_type(f2(a, 3)) # N: Revealed type is "Literal[1]" + +@overload +def f3(x: Literal[3]) -> Literal[3]: ... +@overload +def f3(x: T) -> T: ... +def f3(x): pass + +reveal_type(f3(1)) # N: Revealed type is "builtins.int" +reveal_type(f3(a)) # N: Revealed type is "Literal[1]" + +@overload +def f4(x: str) -> str: ... +@overload +def f4(x: T) -> T: ... +def f4(x): pass + +b: Literal['foo'] +reveal_type(f4(1)) # N: Revealed type is "builtins.int" +reveal_type(f4(a)) # N: Revealed type is "Literal[1]" +reveal_type(f4("foo")) # N: Revealed type is "builtins.str" + +# Note: first overload is selected and prevents the typevar from +# ever inferring a Literal["something"]. +reveal_type(f4(b)) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralInferredInOverloadContextUnionMathTrickyOverload] +# This test is a transliteration of check-overloading::testUnionMathTrickyOverload1 +from typing import overload +from typing_extensions import Literal + +@overload +def f(x: Literal['a'], y: Literal['a']) -> int: ... +@overload +def f(x: str, y: Literal['b']) -> str: ... +def f(x): + pass + +x: Literal['a', 'b'] +y: Literal['a', 'b'] +f(x, y) # E: Argument 1 to "f" has incompatible type "Literal['a', 'b']"; expected "Literal['a']" \ + # E: Argument 2 to "f" has incompatible type "Literal['a', 'b']"; expected "Literal['a']" \ +[builtins fixtures/tuple.pyi] +[out] + + +--- +--- Tests that make sure we're correctly using the fallback +--- + +[case testLiteralFallbackOperatorsWorkCorrectly] +from typing_extensions import Literal + +a: Literal[3] +b: int +c: Literal[4] +d: Literal['foo'] +e: str + +reveal_type(a + a) # N: Revealed type is "builtins.int" +reveal_type(a + b) # N: Revealed type is "builtins.int" +reveal_type(b + a) # N: Revealed type is "builtins.int" +reveal_type(a + 1) # N: Revealed type is "builtins.int" +reveal_type(1 + a) # N: Revealed type is "builtins.int" +reveal_type(a + c) # N: Revealed type is "builtins.int" +reveal_type(c + a) # N: Revealed type is "builtins.int" + +reveal_type(d + d) # N: Revealed type is "builtins.str" +reveal_type(d + e) # N: Revealed type is "builtins.str" +reveal_type(e + d) # N: Revealed type is "builtins.str" +reveal_type(d + 'foo') # N: Revealed type is "builtins.str" +reveal_type('foo' + d) # N: Revealed type is "builtins.str" + +reveal_type(a.__add__(b)) # N: Revealed type is "builtins.int" +reveal_type(b.__add__(a)) # N: Revealed type is "builtins.int" + +a *= b # E: Incompatible types in assignment (expression has type "int", variable has type "Literal[3]") +b *= a + +reveal_type(b) # N: Revealed type is "builtins.int" +[builtins fixtures/primitives.pyi] + +[case testLiteralFallbackInheritedMethodsWorkCorrectly] +from typing_extensions import Literal +a: Literal['foo'] +b: str + +reveal_type(a.startswith(a)) # N: Revealed type is "builtins.bool" +reveal_type(b.startswith(a)) # N: Revealed type is "builtins.bool" +reveal_type(a.startswith(b)) # N: Revealed type is "builtins.bool" +reveal_type(a.strip()) # N: Revealed type is "builtins.str" +[builtins fixtures/ops.pyi] +[out] + +[case testLiteralFallbackMethodsDoNotCoerceToLiteral] +from typing_extensions import Literal + +a: Literal[3] +b: int +c: Literal["foo"] + +if int(): + a = a * a # E: Incompatible types in assignment (expression has type "int", variable has type "Literal[3]") + a = a * b # E: Incompatible types in assignment (expression has type "int", variable has type "Literal[3]") + a = b * a # E: Incompatible types in assignment (expression has type "int", variable has type "Literal[3]") + + b = a * a + b = a * b + b = b * a + + c = c.strip() # E: Incompatible types in assignment (expression has type "str", variable has type "Literal['foo']") +[builtins fixtures/ops.pyi] +[out] + + +-- +-- Tests that check we report errors when we try using Literal[...] +-- in invalid places. +-- + +[case testLiteralErrorsWithIsInstanceAndIsSubclass] +from typing_extensions import Literal +from typing_extensions import Literal as Renamed +import typing_extensions as indirect + +Alias = Literal[3] + +isinstance(3, Literal[3]) # E: Cannot use isinstance() with Literal type +isinstance(3, Alias) # E: Cannot use isinstance() with Literal type \ + # E: Argument 2 to "isinstance" has incompatible type ""; expected "Union[type, Tuple[Any, ...]]" +isinstance(3, Renamed[3]) # E: Cannot use isinstance() with Literal type +isinstance(3, indirect.Literal[3]) # E: Cannot use isinstance() with Literal type + +issubclass(int, Literal[3]) # E: Cannot use issubclass() with Literal type +issubclass(int, Alias) # E: Cannot use issubclass() with Literal type \ + # E: Argument 2 to "issubclass" has incompatible type ""; expected "Union[type, Tuple[Any, ...]]" +issubclass(int, Renamed[3]) # E: Cannot use issubclass() with Literal type +issubclass(int, indirect.Literal[3]) # E: Cannot use issubclass() with Literal type +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] +[out] + +[case testLiteralErrorsWhenSubclassed] + +from typing_extensions import Literal +from typing_extensions import Literal as Renamed +import typing_extensions as indirect + +Alias = Literal[3] + +class Bad1(Literal[3]): pass # E: Invalid base class "Literal" +class Bad2(Renamed[3]): pass # E: Invalid base class "Renamed" +class Bad3(indirect.Literal[3]): pass # E: Invalid base class "indirect.Literal" +class Bad4(Alias): pass # E: Invalid base class "Alias" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralErrorsWhenInvoked-skip] +# TODO: We don't seem to correctly handle invoking types like +# 'Final' and 'Protocol' as well. When fixing this, also fix +# those types? +from typing_extensions import Literal +from typing_extensions import Literal as Renamed +import typing_extensions as indirect + +Alias = Literal[3] + +Literal[3]() # E: The type "Type[Literal]" is not generic and not indexable +Renamed[3]() # E: The type "Type[Literal]" is not generic and not indexable +indirect.Literal[3]() # E: The type "Type[Literal]" is not generic and not indexable +Alias() # E: "object" not callable + +# TODO: Add appropriate error messages to the following lines +Literal() +Renamed() +indirect.Literal() +[builtins fixtures/isinstancelist.pyi] +[out] + + +-- +-- Test to make sure literals interact with generics as expected +-- + +[case testLiteralAndGenericsWithSimpleFunctions] +from typing import TypeVar +from typing_extensions import Literal + +T = TypeVar('T') +def foo(x: T) -> T: pass +def expects_literal(x: Literal[3]) -> None: pass +def expects_int(x: int) -> None: pass + +a: Literal[3] +reveal_type(foo(3)) # N: Revealed type is "builtins.int" +reveal_type(foo(a)) # N: Revealed type is "Literal[3]" + +expects_literal(3) +expects_literal(foo(3)) +expects_literal(foo(foo(3))) + +expects_literal(a) +expects_literal(foo(a)) +expects_literal(foo(foo(a))) + +expects_literal(5) # E: Argument 1 to "expects_literal" has incompatible type "Literal[5]"; expected "Literal[3]" +expects_literal(foo(5)) # E: Argument 1 to "foo" has incompatible type "Literal[5]"; expected "Literal[3]" +expects_literal(foo(foo(5))) # E: Argument 1 to "foo" has incompatible type "Literal[5]"; expected "Literal[3]" + +expects_int(a) +expects_int(foo(a)) +expects_int(foo(foo(a))) +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralAndGenericWithUnion] +from typing import TypeVar, Union +from typing_extensions import Literal + +T = TypeVar('T') +def identity(x: T) -> T: return x + +a: Union[int, Literal['foo']] = identity('foo') +b: Union[int, Literal['foo']] = identity('bar') # E: Argument 1 to "identity" has incompatible type "Literal['bar']"; expected "Union[int, Literal['foo']]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralAndGenericsNoMatch] +from typing import TypeVar, Union, List +from typing_extensions import Literal + +def identity(x: T) -> T: + return x + +Ok1 = Union[List[int], Literal['bad']] +Ok2 = Union[List[Literal[42]], Literal['bad']] +Bad = Union[List[Literal[43]], Literal['bad']] + +x: Ok1 = identity([42]) +y: Ok2 = identity([42]) +z: Bad = identity([42]) # E: List item 0 has incompatible type "Literal[42]"; expected "Literal[43]" +[builtins fixtures/list.pyi] +[out] + +[case testLiteralAndGenericsWithSimpleClasses] +from typing import TypeVar, Generic +from typing_extensions import Literal + +T = TypeVar('T') +class Wrapper(Generic[T]): + def __init__(self, val: T) -> None: + self.val = val + def inner(self) -> T: + return self.val + +def expects_literal(a: Literal[3]) -> None: pass +def expects_literal_wrapper(x: Wrapper[Literal[3]]) -> None: pass + +a: Literal[3] +reveal_type(Wrapper(3)) # N: Revealed type is "__main__.Wrapper[builtins.int]" +reveal_type(Wrapper[Literal[3]](3)) # N: Revealed type is "__main__.Wrapper[Literal[3]]" +reveal_type(Wrapper(a)) # N: Revealed type is "__main__.Wrapper[Literal[3]]" + +expects_literal(Wrapper(a).inner()) + +# Note: the following probably ought to type-check: it's reasonable to infer +# Wrapper[Literal[3]] here. +# TODO: Consider finding a way to handle this edge case better +expects_literal(Wrapper(3).inner()) # E: Argument 1 to "expects_literal" has incompatible type "int"; expected "Literal[3]" + +# Note: if we handle the edge case above, we should make sure this error +# message switches to warning about an incompatible type 'Literal[5]' rather +# then an incompatible type 'int' +expects_literal(Wrapper(5).inner()) # E: Argument 1 to "expects_literal" has incompatible type "int"; expected "Literal[3]" + +expects_literal_wrapper(Wrapper(a)) +expects_literal_wrapper(Wrapper(3)) +expects_literal_wrapper(Wrapper(5)) # E: Argument 1 to "Wrapper" has incompatible type "Literal[5]"; expected "Literal[3]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralAndGenericsRespectsUpperBound] +from typing import TypeVar +from typing_extensions import Literal + +TLiteral = TypeVar('TLiteral', bound=Literal[3]) +TInt = TypeVar('TInt', bound=int) + +def func1(x: TLiteral) -> TLiteral: pass +def func2(x: TInt) -> TInt: pass + +def func3(x: TLiteral) -> TLiteral: + y = func2(x) + return y +def func4(x: TInt) -> TInt: + y = func1(x) # E: Value of type variable "TLiteral" of "func1" cannot be "TInt" + return y + +a: Literal[3] +b: Literal[4] +c: int + +reveal_type(func1) # N: Revealed type is "def [TLiteral <: Literal[3]] (x: TLiteral`-1) -> TLiteral`-1" + +reveal_type(func1(3)) # N: Revealed type is "Literal[3]" +reveal_type(func1(a)) # N: Revealed type is "Literal[3]" +reveal_type(func1(4)) # E: Value of type variable "TLiteral" of "func1" cannot be "Literal[4]" \ + # N: Revealed type is "Literal[4]" +reveal_type(func1(b)) # E: Value of type variable "TLiteral" of "func1" cannot be "Literal[4]" \ + # N: Revealed type is "Literal[4]" +reveal_type(func1(c)) # E: Value of type variable "TLiteral" of "func1" cannot be "int" \ + # N: Revealed type is "builtins.int" + +reveal_type(func2(3)) # N: Revealed type is "builtins.int" +reveal_type(func2(a)) # N: Revealed type is "Literal[3]" +reveal_type(func2(4)) # N: Revealed type is "builtins.int" +reveal_type(func2(b)) # N: Revealed type is "Literal[4]" +reveal_type(func2(c)) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralAndGenericsRespectsValueRestriction] +from typing import TypeVar +from typing_extensions import Literal + +TLiteral = TypeVar('TLiteral', Literal[3], Literal['foo']) +TNormal = TypeVar('TNormal', int, str) + +def func1(x: TLiteral) -> TLiteral: pass +def func2(x: TNormal) -> TNormal: pass + +def func3(x: TLiteral) -> TLiteral: + y = func2(x) + return y # E: Incompatible return value type (got "int", expected "Literal[3]") \ + # E: Incompatible return value type (got "str", expected "Literal['foo']") +def func4(x: TNormal) -> TNormal: + y = func1(x) # E: Value of type variable "TLiteral" of "func1" cannot be "int" \ + # E: Value of type variable "TLiteral" of "func1" cannot be "str" + return y + +i1: Literal[3] +i2: Literal[4] +i: int + +s1: Literal['foo'] +s2: Literal['bar'] +s: str + +reveal_type(func1) # N: Revealed type is "def [TLiteral in (Literal[3], Literal['foo'])] (x: TLiteral`-1) -> TLiteral`-1" + +reveal_type(func1(3)) # N: Revealed type is "Literal[3]" +reveal_type(func1(i1)) # N: Revealed type is "Literal[3]" +reveal_type(func1(4)) # E: Value of type variable "TLiteral" of "func1" cannot be "Literal[4]" \ + # N: Revealed type is "Literal[4]" +reveal_type(func1(i2)) # E: Value of type variable "TLiteral" of "func1" cannot be "Literal[4]" \ + # N: Revealed type is "Literal[4]" +reveal_type(func1(i)) # E: Value of type variable "TLiteral" of "func1" cannot be "int" \ + # N: Revealed type is "builtins.int" +reveal_type(func1("foo")) # N: Revealed type is "Literal['foo']" +reveal_type(func1(s1)) # N: Revealed type is "Literal['foo']" +reveal_type(func1("bar")) # E: Value of type variable "TLiteral" of "func1" cannot be "Literal['bar']" \ + # N: Revealed type is "Literal['bar']" +reveal_type(func1(s2)) # E: Value of type variable "TLiteral" of "func1" cannot be "Literal['bar']" \ + # N: Revealed type is "Literal['bar']" +reveal_type(func1(s)) # E: Value of type variable "TLiteral" of "func1" cannot be "str" \ + # N: Revealed type is "builtins.str" + +reveal_type(func2(3)) # N: Revealed type is "builtins.int" +reveal_type(func2(i1)) # N: Revealed type is "builtins.int" +reveal_type(func2(4)) # N: Revealed type is "builtins.int" +reveal_type(func2(i2)) # N: Revealed type is "builtins.int" +reveal_type(func2("foo")) # N: Revealed type is "builtins.str" +reveal_type(func2(s1)) # N: Revealed type is "builtins.str" +reveal_type(func2("bar")) # N: Revealed type is "builtins.str" +reveal_type(func2(s2)) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralAndGenericsWithOverloads] +from typing import TypeVar, overload, Union +from typing_extensions import Literal + +@overload +def func1(x: Literal[4]) -> Literal[19]: ... +@overload +def func1(x: int) -> int: ... +def func1(x: int) -> int: pass + +T = TypeVar('T') +def identity(x: T) -> T: pass + +a: Literal[4] +b: Literal[5] + +reveal_type(func1(identity(4))) # N: Revealed type is "Literal[19]" +reveal_type(func1(identity(5))) # N: Revealed type is "builtins.int" +reveal_type(func1(identity(a))) # N: Revealed type is "Literal[19]" +reveal_type(func1(identity(b))) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +-- +-- Interactions with meets +-- + +[case testLiteralMeets] +from typing import TypeVar, List, Callable, Union, Optional +from typing_extensions import Literal + +a: Callable[[Literal[1]], int] +b: Callable[[Literal[2]], str] +c: Callable[[int], str] +d: Callable[[object], str] +e: Callable[[Union[Literal[1], Literal[2]]], str] + +arr1 = [a, a] +arr2 = [a, b] +arr3 = [a, c] +arr4 = [a, d] +arr5 = [a, e] + +reveal_type(arr1) # N: Revealed type is "builtins.list[def (Literal[1]) -> builtins.int]" +reveal_type(arr2) # N: Revealed type is "builtins.list[builtins.function]" +reveal_type(arr3) # N: Revealed type is "builtins.list[def (Literal[1]) -> builtins.object]" +reveal_type(arr4) # N: Revealed type is "builtins.list[def (Literal[1]) -> builtins.object]" +reveal_type(arr5) # N: Revealed type is "builtins.list[def (Literal[1]) -> builtins.object]" + +# Inspect just only one interesting one +lit: Literal[1] +reveal_type(arr2[0](lit)) # E: Cannot call function of unknown type \ + # N: Revealed type is "Any" + +T = TypeVar('T') +def unify(func: Callable[[T, T], None]) -> T: pass + +def f1(x: Literal[1], y: Literal[1]) -> None: pass +def f2(x: Literal[1], y: Literal[2]) -> None: pass +def f3(x: Literal[1], y: int) -> None: pass +def f4(x: Literal[1], y: object) -> None: pass +def f5(x: Literal[1], y: Union[Literal[1], Literal[2]]) -> None: pass +def f6(x: Optional[Literal[1]], y: Optional[Literal[2]]) -> None: pass + +reveal_type(unify(f1)) # N: Revealed type is "Literal[1]" +if object(): + reveal_type(unify(f2)) # N: Revealed type is "Never" +reveal_type(unify(f3)) # N: Revealed type is "Literal[1]" +reveal_type(unify(f4)) # N: Revealed type is "Literal[1]" +reveal_type(unify(f5)) # N: Revealed type is "Literal[1]" +reveal_type(unify(f6)) # N: Revealed type is "None" +[builtins fixtures/list.pyi] +[out] + +[case testLiteralMeetsWithStrictOptional] +from typing import TypeVar, Callable, Union +from typing_extensions import Literal + +a: Callable[[Literal[1]], int] +b: Callable[[Literal[2]], str] +lit: Literal[1] + +arr = [a, b] +reveal_type(arr) # N: Revealed type is "builtins.list[builtins.function]" +reveal_type(arr[0](lit)) # E: Cannot call function of unknown type \ + # N: Revealed type is "Any" + +T = TypeVar('T') +def unify(func: Callable[[T, T], None]) -> T: pass +def func(x: Literal[1], y: Literal[2]) -> None: pass + +reveal_type(unify(func)) # N: Revealed type is "Never" +[builtins fixtures/list.pyi] +[out] + + +-- +-- Checks for intelligent indexing +-- + +[case testLiteralIntelligentIndexingTuples] +from typing import Tuple, NamedTuple, Optional, Final +from typing_extensions import Literal + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass + +idx0: Literal[0] +idx1: Literal[1] +idx2: Literal[2] +idx3: Literal[3] +idx4: Literal[4] +idx5: Literal[5] +idx_neg1: Literal[-1] +idx_final: Final = 2 + +tup1: Tuple[A, B, Optional[C], D, E] +reveal_type(tup1[idx0]) # N: Revealed type is "__main__.A" +reveal_type(tup1[idx1]) # N: Revealed type is "__main__.B" +reveal_type(tup1[idx2]) # N: Revealed type is "Union[__main__.C, None]" +reveal_type(tup1[idx_final]) # N: Revealed type is "Union[__main__.C, None]" +reveal_type(tup1[idx3]) # N: Revealed type is "__main__.D" +reveal_type(tup1[idx4]) # N: Revealed type is "__main__.E" +reveal_type(tup1[idx_neg1]) # N: Revealed type is "__main__.E" +tup1[idx5] # E: Tuple index out of range +reveal_type(tup1[idx2:idx4]) # N: Revealed type is "Tuple[Union[__main__.C, None], __main__.D]" +reveal_type(tup1[::idx2]) # N: Revealed type is "Tuple[__main__.A, Union[__main__.C, None], __main__.E]" +if tup1[idx2] is not None: + reveal_type(tup1[idx2]) # N: Revealed type is "Union[__main__.C, None]" +if tup1[idx_final] is not None: + reveal_type(tup1[idx_final]) # N: Revealed type is "__main__.C" + +Tup2Class = NamedTuple('Tup2Class', [('a', A), ('b', B), ('c', C), ('d', D), ('e', E)]) +tup2: Tup2Class +reveal_type(tup2[idx0]) # N: Revealed type is "__main__.A" +reveal_type(tup2[idx1]) # N: Revealed type is "__main__.B" +reveal_type(tup2[idx2]) # N: Revealed type is "__main__.C" +reveal_type(tup2[idx3]) # N: Revealed type is "__main__.D" +reveal_type(tup2[idx4]) # N: Revealed type is "__main__.E" +reveal_type(tup2[idx_neg1]) # N: Revealed type is "__main__.E" +tup2[idx5] # E: Tuple index out of range +reveal_type(tup2[idx2:idx4]) # N: Revealed type is "Tuple[__main__.C, __main__.D]" +reveal_type(tup2[::idx2]) # N: Revealed type is "Tuple[__main__.A, __main__.C, __main__.E]" +tup3: Tup2Class = tup2[:] # E: Incompatible types in assignment (expression has type "Tuple[A, B, C, D, E]", variable has type "Tup2Class") +[builtins fixtures/slice.pyi] + +[case testLiteralIntelligentIndexingTypedDict] +from typing_extensions import Literal +from mypy_extensions import TypedDict + +class Unrelated: pass +u: Unrelated + +class Inner(TypedDict): + a: int +class Outer(Inner, total=False): + b: str + +a_key: Literal["a"] +b_key: Literal["b"] +c_key: Literal["c"] + +d: Outer + +reveal_type(d[a_key]) # N: Revealed type is "builtins.int" +reveal_type(d[b_key]) # N: Revealed type is "builtins.str" +d[c_key] # E: TypedDict "Outer" has no key "c" + +reveal_type(d.get(a_key, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(d.get(b_key, u)) # N: Revealed type is "Union[builtins.str, __main__.Unrelated]" +reveal_type(d.get(c_key, u)) # N: Revealed type is "builtins.object" + +reveal_type(d.pop(a_key)) # E: Key "a" of TypedDict "Outer" cannot be deleted \ + # N: Revealed type is "builtins.int" +reveal_type(d.pop(b_key)) # N: Revealed type is "builtins.str" +d.pop(c_key) # E: TypedDict "Outer" has no key "c" + +del d[a_key] # E: Key "a" of TypedDict "Outer" cannot be deleted +del d[b_key] +del d[c_key] # E: TypedDict "Outer" has no key "c" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] + +[case testLiteralIntelligentIndexingUsingFinal] +from typing import Tuple, NamedTuple +from typing_extensions import Literal, Final +from mypy_extensions import TypedDict + +int_key_good: Final = 0 +int_key_bad: Final = 3 +str_key_good: Final = "foo" +str_key_bad: Final = "missing" + +class Unrelated: pass + +MyTuple = NamedTuple('MyTuple', [ + ('foo', int), + ('bar', str), +]) + +class MyDict(TypedDict): + foo: int + bar: str + +a: Tuple[int, str] +b: MyTuple +c: MyDict +u: Unrelated + +reveal_type(a[int_key_good]) # N: Revealed type is "builtins.int" +reveal_type(b[int_key_good]) # N: Revealed type is "builtins.int" +reveal_type(c[str_key_good]) # N: Revealed type is "builtins.int" +reveal_type(c.get(str_key_good, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(c.get(str_key_bad, u)) # N: Revealed type is "builtins.object" + +a[int_key_bad] # E: Tuple index out of range +b[int_key_bad] # E: Tuple index out of range +c[str_key_bad] # E: TypedDict "MyDict" has no key "missing" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] + +[case testLiteralIntelligentIndexingTupleUnions] +from typing import Tuple, NamedTuple +from typing_extensions import Literal + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass + +idx1: Literal[1, 2] +idx2: Literal[3, 4] +idx_bad: Literal[1, 20] + +tup1: Tuple[A, B, C, D, E] +Tup2Class = NamedTuple('Tup2Class', [('a', A), ('b', B), ('c', C), ('d', D), ('e', E)]) +tup2: Tup2Class + +reveal_type(tup1[idx1]) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(tup1[idx1:idx2]) # N: Revealed type is "Union[Tuple[__main__.B, __main__.C], Tuple[__main__.B, __main__.C, __main__.D], Tuple[__main__.C], Tuple[__main__.C, __main__.D]]" +reveal_type(tup1[0::idx1]) # N: Revealed type is "Union[Tuple[__main__.A, __main__.B, __main__.C, __main__.D, __main__.E], Tuple[__main__.A, __main__.C, __main__.E]]" +tup1[idx_bad] # E: Tuple index out of range + +reveal_type(tup2[idx1]) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(tup2[idx1:idx2]) # N: Revealed type is "Union[Tuple[__main__.B, __main__.C], Tuple[__main__.B, __main__.C, __main__.D], Tuple[__main__.C], Tuple[__main__.C, __main__.D]]" +reveal_type(tup2[0::idx1]) # N: Revealed type is "Union[Tuple[__main__.A, __main__.B, __main__.C, __main__.D, __main__.E], Tuple[__main__.A, __main__.C, __main__.E]]" +tup2[idx_bad] # E: Tuple index out of range +[builtins fixtures/slice.pyi] +[out] + +[case testLiteralIntelligentIndexingTypedDictUnions] +from typing_extensions import Literal, Final +from mypy_extensions import TypedDict + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass + +class Base(TypedDict): + a: A + b: B + c: C + +class Test(Base, total=False): + d: D + e: E + +class AAndB(A, B): pass + +test: Test +good_keys: Literal["a", "b"] +optional_keys: Literal["d", "e"] +bad_keys: Literal["a", "bad"] + +reveal_type(test[good_keys]) # N: Revealed type is "Union[__main__.A, __main__.B]" +reveal_type(test.get(good_keys)) # N: Revealed type is "Union[__main__.A, __main__.B, None]" +reveal_type(test.get(good_keys, 3)) # N: Revealed type is "Union[__main__.A, Literal[3]?, __main__.B]" +reveal_type(test.pop(optional_keys)) # N: Revealed type is "Union[__main__.D, __main__.E]" +reveal_type(test.pop(optional_keys, 3)) # N: Revealed type is "Union[__main__.D, __main__.E, Literal[3]?]" +reveal_type(test.setdefault(good_keys, AAndB())) # N: Revealed type is "Union[__main__.A, __main__.B]" +reveal_type(test.get(bad_keys)) # N: Revealed type is "builtins.object" +reveal_type(test.get(bad_keys, 3)) # N: Revealed type is "builtins.object" +del test[optional_keys] + + +test[bad_keys] # E: TypedDict "Test" has no key "bad" +test.pop(good_keys) # E: Key "a" of TypedDict "Test" cannot be deleted \ + # E: Key "b" of TypedDict "Test" cannot be deleted +test.pop(bad_keys) # E: Key "a" of TypedDict "Test" cannot be deleted \ + # E: TypedDict "Test" has no key "bad" +test.setdefault(good_keys, 3) # E: Argument 2 to "setdefault" of "TypedDict" has incompatible type "int"; expected "A" +test.setdefault(bad_keys, 3 ) # E: Argument 2 to "setdefault" of "TypedDict" has incompatible type "int"; expected "A" +del test[good_keys] # E: Key "a" of TypedDict "Test" cannot be deleted \ + # E: Key "b" of TypedDict "Test" cannot be deleted +del test[bad_keys] # E: Key "a" of TypedDict "Test" cannot be deleted \ + # E: TypedDict "Test" has no key "bad" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] + +[case testLiteralIntelligentIndexingMultiTypedDict] +from typing import Union +from typing_extensions import Literal +from mypy_extensions import TypedDict + +class A: pass +class B: pass +class C: pass +class D: pass + +class D1(TypedDict): + a: A + b: B + c: C + +class D2(TypedDict): + b: B + c: C + d: D + +x: Union[D1, D2] +bad_keys: Literal['a', 'b', 'c', 'd'] +good_keys: Literal['b', 'c'] + +x[bad_keys] # E: TypedDict "D1" has no key "d" \ + # E: TypedDict "D2" has no key "a" + +reveal_type(x[good_keys]) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(x.get(good_keys)) # N: Revealed type is "Union[__main__.B, __main__.C, None]" +reveal_type(x.get(good_keys, 3)) # N: Revealed type is "Union[__main__.B, Literal[3]?, __main__.C]" +reveal_type(x.get(bad_keys)) # N: Revealed type is "builtins.object" +reveal_type(x.get(bad_keys, 3)) # N: Revealed type is "builtins.object" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +-- +-- Interactions with 'Final' +-- + +[case testLiteralFinalInferredAsLiteral] +from typing_extensions import Final, Literal + +var1: Final = 1 +var2: Final = "foo" +var3: Final = True +var4: Final = None + +class Foo: + classvar1: Final = 1 + classvar2: Final = "foo" + classvar3: Final = True + classvar4: Final = None + + def __init__(self) -> None: + self.instancevar1: Final = 1 + self.instancevar2: Final = "foo" + self.instancevar3: Final = True + self.instancevar4: Final = None + +def force1(x: Literal[1]) -> None: pass +def force2(x: Literal["foo"]) -> None: pass +def force3(x: Literal[True]) -> None: pass +def force4(x: Literal[None]) -> None: pass + +reveal_type(var1) # N: Revealed type is "Literal[1]?" +reveal_type(var2) # N: Revealed type is "Literal['foo']?" +reveal_type(var3) # N: Revealed type is "Literal[True]?" +reveal_type(var4) # N: Revealed type is "None" +force1(reveal_type(var1)) # N: Revealed type is "Literal[1]" +force2(reveal_type(var2)) # N: Revealed type is "Literal['foo']" +force3(reveal_type(var3)) # N: Revealed type is "Literal[True]" +force4(reveal_type(var4)) # N: Revealed type is "None" + +reveal_type(Foo.classvar1) # N: Revealed type is "Literal[1]?" +reveal_type(Foo.classvar2) # N: Revealed type is "Literal['foo']?" +reveal_type(Foo.classvar3) # N: Revealed type is "Literal[True]?" +reveal_type(Foo.classvar4) # N: Revealed type is "None" +force1(reveal_type(Foo.classvar1)) # N: Revealed type is "Literal[1]" +force2(reveal_type(Foo.classvar2)) # N: Revealed type is "Literal['foo']" +force3(reveal_type(Foo.classvar3)) # N: Revealed type is "Literal[True]" +force4(reveal_type(Foo.classvar4)) # N: Revealed type is "None" + +f = Foo() +reveal_type(f.instancevar1) # N: Revealed type is "Literal[1]?" +reveal_type(f.instancevar2) # N: Revealed type is "Literal['foo']?" +reveal_type(f.instancevar3) # N: Revealed type is "Literal[True]?" +reveal_type(f.instancevar4) # N: Revealed type is "None" +force1(reveal_type(f.instancevar1)) # N: Revealed type is "Literal[1]" +force2(reveal_type(f.instancevar2)) # N: Revealed type is "Literal['foo']" +force3(reveal_type(f.instancevar3)) # N: Revealed type is "Literal[True]" +force4(reveal_type(f.instancevar4)) # N: Revealed type is "None" +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralFinalDirectInstanceTypesSupercedeInferredLiteral] +from typing_extensions import Final, Literal + +var1: Final[int] = 1 +var2: Final[str] = "foo" +var3: Final[bool] = True +var4: Final[None] = None + +class Foo: + classvar1: Final[int] = 1 + classvar2: Final[str] = "foo" + classvar3: Final[bool] = True + classvar4: Final[None] = None + + def __init__(self) -> None: + self.instancevar1: Final[int] = 1 + self.instancevar2: Final[str] = "foo" + self.instancevar3: Final[bool] = True + self.instancevar4: Final[None] = None + +def force1(x: Literal[1]) -> None: pass +def force2(x: Literal["foo"]) -> None: pass +def force3(x: Literal[True]) -> None: pass +def force4(x: Literal[None]) -> None: pass + +reveal_type(var1) # N: Revealed type is "builtins.int" +reveal_type(var2) # N: Revealed type is "builtins.str" +reveal_type(var3) # N: Revealed type is "builtins.bool" +reveal_type(var4) # N: Revealed type is "None" +force1(var1) # E: Argument 1 to "force1" has incompatible type "int"; expected "Literal[1]" +force2(var2) # E: Argument 1 to "force2" has incompatible type "str"; expected "Literal['foo']" +force3(var3) # E: Argument 1 to "force3" has incompatible type "bool"; expected "Literal[True]" +force4(var4) + +reveal_type(Foo.classvar1) # N: Revealed type is "builtins.int" +reveal_type(Foo.classvar2) # N: Revealed type is "builtins.str" +reveal_type(Foo.classvar3) # N: Revealed type is "builtins.bool" +reveal_type(Foo.classvar4) # N: Revealed type is "None" +force1(Foo.classvar1) # E: Argument 1 to "force1" has incompatible type "int"; expected "Literal[1]" +force2(Foo.classvar2) # E: Argument 1 to "force2" has incompatible type "str"; expected "Literal['foo']" +force3(Foo.classvar3) # E: Argument 1 to "force3" has incompatible type "bool"; expected "Literal[True]" +force4(Foo.classvar4) + +f = Foo() +reveal_type(f.instancevar1) # N: Revealed type is "builtins.int" +reveal_type(f.instancevar2) # N: Revealed type is "builtins.str" +reveal_type(f.instancevar3) # N: Revealed type is "builtins.bool" +reveal_type(f.instancevar4) # N: Revealed type is "None" +force1(f.instancevar1) # E: Argument 1 to "force1" has incompatible type "int"; expected "Literal[1]" +force2(f.instancevar2) # E: Argument 1 to "force2" has incompatible type "str"; expected "Literal['foo']" +force3(f.instancevar3) # E: Argument 1 to "force3" has incompatible type "bool"; expected "Literal[True]" +force4(f.instancevar4) +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralFinalDirectLiteralTypesForceLiteral] +from typing_extensions import Final, Literal + +var1: Final[Literal[1]] = 1 +var2: Final[Literal["foo"]] = "foo" +var3: Final[Literal[True]] = True +var4: Final[Literal[None]] = None + +class Foo: + classvar1: Final[Literal[1]] = 1 + classvar2: Final[Literal["foo"]] = "foo" + classvar3: Final[Literal[True]] = True + classvar4: Final[Literal[None]] = None + + def __init__(self) -> None: + self.instancevar1: Final[Literal[1]] = 1 + self.instancevar2: Final[Literal["foo"]] = "foo" + self.instancevar3: Final[Literal[True]] = True + self.instancevar4: Final[Literal[None]] = None + +def force1(x: Literal[1]) -> None: pass +def force2(x: Literal["foo"]) -> None: pass +def force3(x: Literal[True]) -> None: pass +def force4(x: Literal[None]) -> None: pass + +reveal_type(var1) # N: Revealed type is "Literal[1]" +reveal_type(var2) # N: Revealed type is "Literal['foo']" +reveal_type(var3) # N: Revealed type is "Literal[True]" +reveal_type(var4) # N: Revealed type is "None" +force1(reveal_type(var1)) # N: Revealed type is "Literal[1]" +force2(reveal_type(var2)) # N: Revealed type is "Literal['foo']" +force3(reveal_type(var3)) # N: Revealed type is "Literal[True]" +force4(reveal_type(var4)) # N: Revealed type is "None" + +reveal_type(Foo.classvar1) # N: Revealed type is "Literal[1]" +reveal_type(Foo.classvar2) # N: Revealed type is "Literal['foo']" +reveal_type(Foo.classvar3) # N: Revealed type is "Literal[True]" +reveal_type(Foo.classvar4) # N: Revealed type is "None" +force1(reveal_type(Foo.classvar1)) # N: Revealed type is "Literal[1]" +force2(reveal_type(Foo.classvar2)) # N: Revealed type is "Literal['foo']" +force3(reveal_type(Foo.classvar3)) # N: Revealed type is "Literal[True]" +force4(reveal_type(Foo.classvar4)) # N: Revealed type is "None" + +f = Foo() +reveal_type(f.instancevar1) # N: Revealed type is "Literal[1]" +reveal_type(f.instancevar2) # N: Revealed type is "Literal['foo']" +reveal_type(f.instancevar3) # N: Revealed type is "Literal[True]" +reveal_type(f.instancevar4) # N: Revealed type is "None" +force1(reveal_type(f.instancevar1)) # N: Revealed type is "Literal[1]" +force2(reveal_type(f.instancevar2)) # N: Revealed type is "Literal['foo']" +force3(reveal_type(f.instancevar3)) # N: Revealed type is "Literal[True]" +force4(reveal_type(f.instancevar4)) # N: Revealed type is "None" +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralFinalErasureInMutableDatastructures1] +from typing_extensions import Final + +var1: Final = [0, None] +var2: Final = (0, None) + +reveal_type(var1) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" +reveal_type(var2) # N: Revealed type is "Tuple[Literal[0]?, None]" +[builtins fixtures/tuple.pyi] + +[case testLiteralFinalErasureInMutableDatastructures2] +from typing_extensions import Final, Literal + +var1: Final = [] +var1.append(0) +reveal_type(var1) # N: Revealed type is "builtins.list[builtins.int]" + +var2 = [] +var2.append(0) +reveal_type(var2) # N: Revealed type is "builtins.list[builtins.int]" + +x: Literal[0] = 0 +var3 = [] +var3.append(x) +reveal_type(var3) # N: Revealed type is "builtins.list[Literal[0]]" + +[builtins fixtures/list.pyi] + +[case testLiteralFinalMismatchCausesError] +from typing_extensions import Final, Literal + +var1: Final[Literal[4]] = 1 # E: Incompatible types in assignment (expression has type "Literal[1]", variable has type "Literal[4]") +var2: Final[Literal['bad']] = "foo" # E: Incompatible types in assignment (expression has type "Literal['foo']", variable has type "Literal['bad']") +var3: Final[Literal[False]] = True # E: Incompatible types in assignment (expression has type "Literal[True]", variable has type "Literal[False]") + +class Foo: + classvar1: Final[Literal[4]] = 1 # E: Incompatible types in assignment (expression has type "Literal[1]", variable has type "Literal[4]") + classvar2: Final[Literal['bad']] = "foo" # E: Incompatible types in assignment (expression has type "Literal['foo']", variable has type "Literal['bad']") + classvar3: Final[Literal[False]] = True # E: Incompatible types in assignment (expression has type "Literal[True]", variable has type "Literal[False]") + + def __init__(self) -> None: + self.instancevar1: Final[Literal[4]] = 1 # E: Incompatible types in assignment (expression has type "Literal[1]", variable has type "Literal[4]") + self.instancevar2: Final[Literal['bad']] = "foo" # E: Incompatible types in assignment (expression has type "Literal['foo']", variable has type "Literal['bad']") + self.instancevar3: Final[Literal[False]] = True # E: Incompatible types in assignment (expression has type "Literal[True]", variable has type "Literal[False]") + +# TODO: Fix the order in which these error messages are shown to be more consistent. +var1 = 10 # E: Cannot assign to final name "var1" \ + # E: Incompatible types in assignment (expression has type "Literal[10]", variable has type "Literal[4]") + + +Foo.classvar1 = 10 # E: Cannot assign to final attribute "classvar1" \ + # E: Incompatible types in assignment (expression has type "Literal[10]", variable has type "Literal[4]") + +Foo().instancevar1 = 10 # E: Cannot assign to final attribute "instancevar1" \ + # E: Incompatible types in assignment (expression has type "Literal[10]", variable has type "Literal[4]") +[builtins fixtures/primitives.pyi] +[out] + +[case testLiteralFinalGoesOnlyOneLevelDown] +from typing import Tuple +from typing_extensions import Final, Literal + +a: Final = 1 +b: Final = (1, 2) + +def force1(x: Literal[1]) -> None: pass +def force2(x: Tuple[Literal[1], Literal[2]]) -> None: pass + +reveal_type(a) # N: Revealed type is "Literal[1]?" +reveal_type(b) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?]" + +force1(a) # ok +force2(b) # ok +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralFinalCollectionPropagation] +from typing import List +from typing_extensions import Final, Literal + +a: Final = 1 +implicit = [a] +explicit: List[Literal[1]] = [a] +direct = [1] + +def force1(x: List[Literal[1]]) -> None: pass +def force2(x: Literal[1]) -> None: pass + +reveal_type(implicit) # N: Revealed type is "builtins.list[builtins.int]" +force1(reveal_type(implicit)) # E: Argument 1 to "force1" has incompatible type "List[int]"; expected "List[Literal[1]]" \ + # N: Revealed type is "builtins.list[builtins.int]" +force2(reveal_type(implicit[0])) # E: Argument 1 to "force2" has incompatible type "int"; expected "Literal[1]" \ + # N: Revealed type is "builtins.int" + +reveal_type(explicit) # N: Revealed type is "builtins.list[Literal[1]]" +force1(reveal_type(explicit)) # N: Revealed type is "builtins.list[Literal[1]]" +force2(reveal_type(explicit[0])) # N: Revealed type is "Literal[1]" + +reveal_type(direct) # N: Revealed type is "builtins.list[builtins.int]" +force1(reveal_type(direct)) # E: Argument 1 to "force1" has incompatible type "List[int]"; expected "List[Literal[1]]" \ + # N: Revealed type is "builtins.list[builtins.int]" +force2(reveal_type(direct[0])) # E: Argument 1 to "force2" has incompatible type "int"; expected "Literal[1]" \ + # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] +[out] + +[case testLiteralFinalStringTypesPython3] +from typing_extensions import Final, Literal + +a: Final = u"foo" +b: Final = "foo" +c: Final = b"foo" + +def force_unicode(x: Literal[u"foo"]) -> None: pass +def force_bytes(x: Literal[b"foo"]) -> None: pass + +force_unicode(reveal_type(a)) # N: Revealed type is "Literal['foo']" +force_unicode(reveal_type(b)) # N: Revealed type is "Literal['foo']" +force_unicode(reveal_type(c)) # E: Argument 1 to "force_unicode" has incompatible type "Literal[b'foo']"; expected "Literal['foo']" \ + # N: Revealed type is "Literal[b'foo']" + +force_bytes(reveal_type(a)) # E: Argument 1 to "force_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" \ + # N: Revealed type is "Literal['foo']" +force_bytes(reveal_type(b)) # E: Argument 1 to "force_bytes" has incompatible type "Literal['foo']"; expected "Literal[b'foo']" \ + # N: Revealed type is "Literal['foo']" +force_bytes(reveal_type(c)) # N: Revealed type is "Literal[b'foo']" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralFinalPropagatesThroughGenerics] +from typing import TypeVar, Generic +from typing_extensions import Final, Literal + + +T = TypeVar('T') + +class WrapperClass(Generic[T]): + def __init__(self, data: T) -> None: + self.data = data + +def wrapper_func(x: T) -> T: + return x + +def force(x: Literal[99]) -> None: pass +def over_int(x: WrapperClass[int]) -> None: pass +def over_literal(x: WrapperClass[Literal[99]]) -> None: pass + +var1: Final = 99 +w1 = WrapperClass(var1) +force(reveal_type(w1.data)) # E: Argument 1 to "force" has incompatible type "int"; expected "Literal[99]" \ + # N: Revealed type is "builtins.int" +force(reveal_type(WrapperClass(var1).data)) # E: Argument 1 to "force" has incompatible type "int"; expected "Literal[99]" \ + # N: Revealed type is "builtins.int" +force(reveal_type(wrapper_func(var1))) # N: Revealed type is "Literal[99]" +over_int(reveal_type(w1)) # N: Revealed type is "__main__.WrapperClass[builtins.int]" +over_literal(reveal_type(w1)) # E: Argument 1 to "over_literal" has incompatible type "WrapperClass[int]"; expected "WrapperClass[Literal[99]]" \ + # N: Revealed type is "__main__.WrapperClass[builtins.int]" +over_int(reveal_type(WrapperClass(var1))) # N: Revealed type is "__main__.WrapperClass[builtins.int]" +over_literal(reveal_type(WrapperClass(var1))) # N: Revealed type is "__main__.WrapperClass[Literal[99]]" + +w2 = WrapperClass(99) +force(reveal_type(w2.data)) # E: Argument 1 to "force" has incompatible type "int"; expected "Literal[99]" \ + # N: Revealed type is "builtins.int" +force(reveal_type(WrapperClass(99).data)) # E: Argument 1 to "force" has incompatible type "int"; expected "Literal[99]" \ + # N: Revealed type is "builtins.int" +force(reveal_type(wrapper_func(99))) # N: Revealed type is "Literal[99]" +over_int(reveal_type(w2)) # N: Revealed type is "__main__.WrapperClass[builtins.int]" +over_literal(reveal_type(w2)) # E: Argument 1 to "over_literal" has incompatible type "WrapperClass[int]"; expected "WrapperClass[Literal[99]]" \ + # N: Revealed type is "__main__.WrapperClass[builtins.int]" +over_int(reveal_type(WrapperClass(99))) # N: Revealed type is "__main__.WrapperClass[builtins.int]" +over_literal(reveal_type(WrapperClass(99))) # N: Revealed type is "__main__.WrapperClass[Literal[99]]" + +var3: Literal[99] = 99 +w3 = WrapperClass(var3) +force(reveal_type(w3.data)) # N: Revealed type is "Literal[99]" +force(reveal_type(WrapperClass(var3).data)) # N: Revealed type is "Literal[99]" +force(reveal_type(wrapper_func(var3))) # N: Revealed type is "Literal[99]" +over_int(reveal_type(w3)) # E: Argument 1 to "over_int" has incompatible type "WrapperClass[Literal[99]]"; expected "WrapperClass[int]" \ + # N: Revealed type is "__main__.WrapperClass[Literal[99]]" +over_literal(reveal_type(w3)) # N: Revealed type is "__main__.WrapperClass[Literal[99]]" +over_int(reveal_type(WrapperClass(var3))) # N: Revealed type is "__main__.WrapperClass[builtins.int]" +over_literal(reveal_type(WrapperClass(var3))) # N: Revealed type is "__main__.WrapperClass[Literal[99]]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralFinalUsedInLiteralType] + +from typing_extensions import Literal, Final +a: Final[int] = 3 +b: Final = 3 +c: Final[Literal[3]] = 3 +d: Literal[3] + +a_wrap: Literal[4, a] # E: Parameter 2 of Literal[...] is invalid +b_wrap: Literal[4, b] # E: Parameter 2 of Literal[...] is invalid +c_wrap: Literal[4, c] # E: Parameter 2 of Literal[...] is invalid +d_wrap: Literal[4, d] # E: Parameter 2 of Literal[...] is invalid +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralWithFinalPropagation] +from typing_extensions import Final, Literal + +a: Final = 3 +b: Final = a +c = a + +def expect_3(x: Literal[3]) -> None: pass +expect_3(a) +expect_3(b) +expect_3(c) # E: Argument 1 to "expect_3" has incompatible type "int"; expected "Literal[3]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralWithFinalPropagationIsNotLeaking] +from typing_extensions import Final, Literal + +final_tuple_direct: Final = (2, 3) +final_tuple_indirect: Final = final_tuple_direct +mutable_tuple = final_tuple_direct +final_list_1: Final = [2] +final_list_2: Final = [2, 2] +final_dict: Final = {"foo": 2} +final_set_1: Final = {2} +final_set_2: Final = {2, 2} + +def expect_2(x: Literal[2]) -> None: pass + +expect_2(final_tuple_direct[0]) +expect_2(final_tuple_indirect[0]) + +expect_2(mutable_tuple[0]) # E: Argument 1 to "expect_2" has incompatible type "int"; expected "Literal[2]" +expect_2(final_list_1[0]) # E: Argument 1 to "expect_2" has incompatible type "int"; expected "Literal[2]" +expect_2(final_list_2[0]) # E: Argument 1 to "expect_2" has incompatible type "int"; expected "Literal[2]" +expect_2(final_dict["foo"]) # E: Argument 1 to "expect_2" has incompatible type "int"; expected "Literal[2]" +expect_2(final_set_1.pop()) # E: Argument 1 to "expect_2" has incompatible type "int"; expected "Literal[2]" +expect_2(final_set_2.pop()) # E: Argument 1 to "expect_2" has incompatible type "int"; expected "Literal[2]" +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +-- +-- Tests for Literals and enums +-- + +[case testLiteralWithEnumsBasic] + +from typing_extensions import Literal +from enum import Enum + +class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + __ROUGE = RED + def func(self) -> int: pass + +r: Literal[Color.RED] +g: Literal[Color.GREEN] +b: Literal[Color.BLUE] +bad1: Literal[Color] # E: Parameter 1 of Literal[...] is invalid +bad2: Literal[Color.func] # E: Parameter 1 of Literal[...] is invalid +bad3: Literal[Color.func()] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +# TODO: change the next line to use Color._Color__ROUGE when mypy implements name mangling +bad4: Literal[Color.__ROUGE] # E: Parameter 1 of Literal[...] is invalid + +def expects_color(x: Color) -> None: pass +def expects_red(x: Literal[Color.RED]) -> None: pass +def bad_func(x: Color.RED) -> None: pass # E: Invalid type: try using Literal[Color.RED] instead? + +expects_color(r) +expects_color(g) +expects_color(b) +expects_red(r) +expects_red(g) # E: Argument 1 to "expects_red" has incompatible type "Literal[Color.GREEN]"; expected "Literal[Color.RED]" +expects_red(b) # E: Argument 1 to "expects_red" has incompatible type "Literal[Color.BLUE]"; expected "Literal[Color.RED]" + +reveal_type(expects_red) # N: Revealed type is "def (x: Literal[__main__.Color.RED])" +reveal_type(r) # N: Revealed type is "Literal[__main__.Color.RED]" +reveal_type(r.func()) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralWithEnumsDefinedInClass] +from typing_extensions import Literal +from enum import Enum + +class Wrapper: + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + +def foo(x: Literal[Wrapper.Color.RED]) -> None: pass + +r: Literal[Wrapper.Color.RED] +g: Literal[Wrapper.Color.GREEN] + +foo(r) +foo(g) # E: Argument 1 to "foo" has incompatible type "Literal[Color.GREEN]"; expected "Literal[Color.RED]" + +reveal_type(foo) # N: Revealed type is "def (x: Literal[__main__.Wrapper.Color.RED])" +reveal_type(r) # N: Revealed type is "Literal[__main__.Wrapper.Color.RED]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralWithEnumsSimilarDefinitions] +from typing_extensions import Literal +import mod_a +import mod_b + +def f(x: Literal[mod_a.Test.FOO]) -> None: pass + +a: Literal[mod_a.Test.FOO] +b: Literal[mod_a.Test2.FOO] +c: Literal[mod_b.Test.FOO] + +f(a) +f(b) # E: Argument 1 to "f" has incompatible type "Literal[Test2.FOO]"; expected "Literal[Test.FOO]" +f(c) # E: Argument 1 to "f" has incompatible type "Literal[mod_b.Test.FOO]"; expected "Literal[mod_a.Test.FOO]" + +[file mod_a.py] +from enum import Enum + +class Test(Enum): + FOO = 1 + BAR = 2 + +class Test2(Enum): + FOO = 1 + BAR = 2 + +[file mod_b.py] +from enum import Enum + +class Test(Enum): + FOO = 1 + BAR = 2 +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralWithEnumsDeclaredUsingCallSyntax] +from typing_extensions import Literal +from enum import Enum + +A = Enum('A', 'FOO BAR') +B = Enum('B', ['FOO', 'BAR']) +C = Enum('C', [('FOO', 1), ('BAR', 2)]) +D = Enum('D', {'FOO': 1, 'BAR': 2}) + +a: Literal[A.FOO] +b: Literal[B.FOO] +c: Literal[C.FOO] +d: Literal[D.FOO] + +reveal_type(a) # N: Revealed type is "Literal[__main__.A.FOO]" +reveal_type(b) # N: Revealed type is "Literal[__main__.B.FOO]" +reveal_type(c) # N: Revealed type is "Literal[__main__.C.FOO]" +reveal_type(d) # N: Revealed type is "Literal[__main__.D.FOO]" +[builtins fixtures/dict.pyi] +[out] + +[case testLiteralWithEnumsDerivedEnums] +from typing_extensions import Literal +from enum import Enum, IntEnum, IntFlag, Flag + +def expects_int(x: int) -> None: pass + +class A(Enum): + FOO = 1 + +class B(IntEnum): + FOO = 1 + +class C(IntFlag): + FOO = 1 + +class D(Flag): + FOO = 1 + +a: Literal[A.FOO] +b: Literal[B.FOO] +c: Literal[C.FOO] +d: Literal[D.FOO] + +expects_int(a) # E: Argument 1 to "expects_int" has incompatible type "Literal[A.FOO]"; expected "int" +expects_int(b) +expects_int(c) +expects_int(d) # E: Argument 1 to "expects_int" has incompatible type "Literal[D.FOO]"; expected "int" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralWithEnumsAliases] +from typing_extensions import Literal +from enum import Enum + +class Test(Enum): + FOO = 1 + BAR = 2 + +Alias = Test + +x: Literal[Alias.FOO] +reveal_type(x) # N: Revealed type is "Literal[__main__.Test.FOO]" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralUsingEnumAttributesInLiteralContexts] +from typing_extensions import Literal, Final +from enum import Enum + +class Test1(Enum): + FOO = 1 + BAR = 2 +Test2 = Enum('Test2', [('FOO', 1), ('BAR', 2)]) + +def expects_test1_foo(x: Literal[Test1.FOO]) -> None: ... +def expects_test2_foo(x: Literal[Test2.FOO]) -> None: ... + +expects_test1_foo(Test1.FOO) +expects_test1_foo(Test1.BAR) # E: Argument 1 to "expects_test1_foo" has incompatible type "Literal[Test1.BAR]"; expected "Literal[Test1.FOO]" +expects_test2_foo(Test2.FOO) +expects_test2_foo(Test2.BAR) # E: Argument 1 to "expects_test2_foo" has incompatible type "Literal[Test2.BAR]"; expected "Literal[Test2.FOO]" + +# Make sure the two 'FOO's are not interchangeable +expects_test1_foo(Test2.FOO) # E: Argument 1 to "expects_test1_foo" has incompatible type "Literal[Test2.FOO]"; expected "Literal[Test1.FOO]" +expects_test2_foo(Test1.FOO) # E: Argument 1 to "expects_test2_foo" has incompatible type "Literal[Test1.FOO]"; expected "Literal[Test2.FOO]" + +# Make sure enums follow the same semantics as 'x = 1' vs 'x: Final = 1' +var1 = Test1.FOO +final1: Final = Test1.FOO +expects_test1_foo(var1) # E: Argument 1 to "expects_test1_foo" has incompatible type "Test1"; expected "Literal[Test1.FOO]" +expects_test1_foo(final1) + +var2 = Test2.FOO +final2: Final = Test2.FOO +expects_test2_foo(var2) # E: Argument 1 to "expects_test2_foo" has incompatible type "Test2"; expected "Literal[Test2.FOO]" +expects_test2_foo(final2) +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralUsingEnumAttributeNamesInLiteralContexts] +from typing_extensions import Literal, Final +from enum import Enum + +class Test1(Enum): + FOO = 1 + BAR = 2 +Test2 = Enum('Test2', [('FOO', 1), ('BAR', 2)]) +Test3 = Enum('Test3', 'FOO BAR') +Test4 = Enum('Test4', ['FOO', 'BAR']) +Test5 = Enum('Test5', {'FOO': 1, 'BAR': 2}) + +def expects_foo(x: Literal['FOO']) -> None: ... + +expects_foo(Test1.FOO.name) +expects_foo(Test2.FOO.name) +expects_foo(Test3.FOO.name) +expects_foo(Test4.FOO.name) +expects_foo(Test5.FOO.name) + +expects_foo(Test1.BAR.name) # E: Argument 1 to "expects_foo" has incompatible type "Literal['BAR']"; expected "Literal['FOO']" +expects_foo(Test2.BAR.name) # E: Argument 1 to "expects_foo" has incompatible type "Literal['BAR']"; expected "Literal['FOO']" +expects_foo(Test3.BAR.name) # E: Argument 1 to "expects_foo" has incompatible type "Literal['BAR']"; expected "Literal['FOO']" +expects_foo(Test4.BAR.name) # E: Argument 1 to "expects_foo" has incompatible type "Literal['BAR']"; expected "Literal['FOO']" +expects_foo(Test5.BAR.name) # E: Argument 1 to "expects_foo" has incompatible type "Literal['BAR']"; expected "Literal['FOO']" + +reveal_type(Test1.FOO.name) # N: Revealed type is "Literal['FOO']?" +reveal_type(Test2.FOO.name) # N: Revealed type is "Literal['FOO']?" +reveal_type(Test3.FOO.name) # N: Revealed type is "Literal['FOO']?" +reveal_type(Test4.FOO.name) # N: Revealed type is "Literal['FOO']?" +reveal_type(Test5.FOO.name) # N: Revealed type is "Literal['FOO']?" +[builtins fixtures/tuple.pyi] +[out] + +[case testLiteralBinderLastValueErased] +# mypy: strict-equality + +from typing_extensions import Literal + +def takes_three(x: Literal[3]) -> None: ... +x: object +x = 3 + +takes_three(x) # E: Argument 1 to "takes_three" has incompatible type "int"; expected "Literal[3]" +if x == 2: # OK + ... +[builtins fixtures/bool.pyi] + +[case testLiteralBinderLastValueErasedPartialTypes] +# mypy: strict-equality + +def test() -> None: + x = None + if bool(): + x = 1 + + if x == 2: # OK + ... +[builtins fixtures/bool.pyi] + +[case testUnaryOpLiteral] +from typing_extensions import Literal + +a: Literal[-2] = -2 +b: Literal[-1] = -1 +c: Literal[0] = 0 +d: Literal[1] = 1 +e: Literal[2] = 2 +f: Literal[+1] = 1 +g: Literal[+2] = 2 +h: Literal[1] = +1 +i: Literal[+2] = 2 +j: Literal[+3] = +3 + +x: Literal[+True] = True # E: Invalid type: Literal[...] cannot contain arbitrary expressions +y: Literal[-True] = -1 # E: Invalid type: Literal[...] cannot contain arbitrary expressions +z: Literal[~0] = 0 # E: Invalid type: Literal[...] cannot contain arbitrary expressions +[out] +[builtins fixtures/ops.pyi] + +[case testNegativeIntLiteralWithFinal] +from typing_extensions import Literal, Final + +ONE: Final = 1 +x: Literal[-1] = -ONE +y: Literal[+1] = +ONE + +TWO: Final = 2 +THREE: Final = 3 + +err_code = -TWO +if bool(): + err_code = -THREE +[builtins fixtures/ops.pyi] + +[case testAliasForEnumTypeAsLiteral] +from typing_extensions import Literal +from enum import Enum + +class Foo(Enum): + A = 1 + +F = Foo + +x: Literal[Foo.A] +y: Literal[F.A] +reveal_type(x) # N: Revealed type is "Literal[__main__.Foo.A]" +reveal_type(y) # N: Revealed type is "Literal[__main__.Foo.A]" +[builtins fixtures/tuple.pyi] + +[case testStrictEqualityLiteralTrueVsFalse] +# mypy: strict-equality + +class C: + a = True + + def update(self) -> None: + self.a = False + +c = C() +assert c.a is True +c.update() +assert c.a is False +[builtins fixtures/bool.pyi] + +[case testConditionalBoolLiteralUnionNarrowing] +# flags: --warn-unreachable + +from typing import Union +from typing_extensions import Literal + +class Truth: + def __bool__(self) -> Literal[True]: ... + +class AlsoTruth: + def __bool__(self) -> Literal[True]: ... + +class Lie: + def __bool__(self) -> Literal[False]: ... + +class AnyAnswer: + def __bool__(self) -> bool: ... + +class NoAnswerSpecified: + pass + +x: Union[Truth, Lie] + +if x: + reveal_type(x) # N: Revealed type is "__main__.Truth" +else: + reveal_type(x) # N: Revealed type is "__main__.Lie" + +if not x: + reveal_type(x) # N: Revealed type is "__main__.Lie" +else: + reveal_type(x) # N: Revealed type is "__main__.Truth" + +y: Union[Truth, AlsoTruth, Lie] + +if y: + reveal_type(y) # N: Revealed type is "Union[__main__.Truth, __main__.AlsoTruth]" +else: + reveal_type(y) # N: Revealed type is "__main__.Lie" + +z: Union[Truth, AnyAnswer] + +if z: + reveal_type(z) # N: Revealed type is "Union[__main__.Truth, __main__.AnyAnswer]" +else: + reveal_type(z) # N: Revealed type is "__main__.AnyAnswer" + +q: Union[Truth, NoAnswerSpecified] + +if q: + reveal_type(q) # N: Revealed type is "Union[__main__.Truth, __main__.NoAnswerSpecified]" +else: + reveal_type(q) # N: Revealed type is "__main__.NoAnswerSpecified" + +w: Union[Truth, AlsoTruth] + +if w: + reveal_type(w) # N: Revealed type is "Union[__main__.Truth, __main__.AlsoTruth]" +else: + reveal_type(w) # E: Statement is unreachable + +[builtins fixtures/bool.pyi] + +[case testLiteralAndInstanceSubtyping] +# https://github.com/python/mypy/issues/7399 +# https://github.com/python/mypy/issues/11232 +from typing import Tuple, Union +from typing_extensions import Literal, Final + +x: bool + +def f() -> Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]: + if x: + return (True, 5) + else: + return (False, 'oops') + +reveal_type(f()) # N: Revealed type is "Union[Tuple[Literal[True], builtins.int], Tuple[Literal[False], builtins.str]]" + +def does_work() -> Tuple[Literal[1]]: + x: Final = (1,) + return x + +def also_works() -> Tuple[Literal[1]]: + x: Tuple[Literal[1]] = (1,) + return x + +def invalid_literal_value() -> Tuple[Literal[1]]: + x: Final = (2,) + return x # E: Incompatible return value type (got "Tuple[int]", expected "Tuple[Literal[1]]") + +def invalid_literal_type() -> Tuple[Literal[1]]: + x: Final = (True,) + return x # E: Incompatible return value type (got "Tuple[bool]", expected "Tuple[Literal[1]]") + +def incorrect_return1() -> Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]: + if x: + return (False, 5) # E: Incompatible return value type (got "Tuple[bool, int]", expected "Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]") + else: + return (True, 'oops') # E: Incompatible return value type (got "Tuple[bool, str]", expected "Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]") + +def incorrect_return2() -> Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]: + if x: + return (bool(), 5) # E: Incompatible return value type (got "Tuple[bool, int]", expected "Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]") + else: + return (bool(), 'oops') # E: Incompatible return value type (got "Tuple[bool, str]", expected "Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]") +[builtins fixtures/bool.pyi] + +[case testLiteralSubtypeContext] +from typing_extensions import Literal + +class A: + foo: Literal['bar', 'spam'] +class B(A): + foo = 'spam' + +reveal_type(B().foo) # N: Revealed type is "Literal['spam']" +[builtins fixtures/tuple.pyi] + +[case testLiteralSubtypeContextNested] +from typing import List +from typing_extensions import Literal + +class A: + foo: List[Literal['bar', 'spam']] +class B(A): + foo = ['spam'] + +reveal_type(B().foo) # N: Revealed type is "builtins.list[Union[Literal['bar'], Literal['spam']]]" +[builtins fixtures/tuple.pyi] + +[case testLiteralSubtypeContextGeneric] +from typing_extensions import Literal +from typing import Generic, List, TypeVar + +T = TypeVar("T", bound=str) + +class B(Generic[T]): + collection: List[T] + word: T + +class C(B[Literal["word"]]): + collection = ["word"] + word = "word" + +reveal_type(C().collection) # N: Revealed type is "builtins.list[Literal['word']]" +reveal_type(C().word) # N: Revealed type is "Literal['word']" +[builtins fixtures/tuple.pyi] + +[case testLiteralTernaryUnionNarrowing] +from typing_extensions import Literal +from typing import Optional + +SEP = Literal["a", "b"] + +class Base: + def feed_data( + self, + sep: SEP, + ) -> int: + return 0 + +class C(Base): + def feed_data( + self, + sep: Optional[SEP] = None, + ) -> int: + if sep is None: + sep = "a" if int() else "b" + reveal_type(sep) # N: Revealed type is "Union[Literal['a'], Literal['b']]" + return super().feed_data(sep) +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-lowercase.test b/test-data/unit/check-lowercase.test new file mode 100644 index 000000000000..ab6d68929f8e --- /dev/null +++ b/test-data/unit/check-lowercase.test @@ -0,0 +1,65 @@ + +[case testTupleLowercaseSettingOff] +# flags: --python-version 3.9 --force-uppercase-builtins +x = (3,) +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "Tuple[int]") +[builtins fixtures/tuple.pyi] + +[case testTupleLowercaseSettingOn] +# flags: --python-version 3.9 --no-force-uppercase-builtins +x = (3,) +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "tuple[int]") +[builtins fixtures/tuple.pyi] + +[case testListLowercaseSettingOff] +# flags: --python-version 3.9 --force-uppercase-builtins +x = [3] +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "List[int]") + +[case testListLowercaseSettingOn] +# flags: --python-version 3.9 --no-force-uppercase-builtins +x = [3] +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "list[int]") + +[case testDictLowercaseSettingOff] +# flags: --python-version 3.9 --force-uppercase-builtins +x = {"key": "value"} +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "Dict[str, str]") + +[case testDictLowercaseSettingOn] +# flags: --python-version 3.9 --no-force-uppercase-builtins +x = {"key": "value"} +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "dict[str, str]") + +[case testSetLowercaseSettingOff] +# flags: --python-version 3.9 --force-uppercase-builtins +x = {3} +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "Set[int]") +[builtins fixtures/set.pyi] + +[case testSetLowercaseSettingOn] +# flags: --python-version 3.9 --no-force-uppercase-builtins +x = {3} +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "set[int]") +[builtins fixtures/set.pyi] + +[case testTypeLowercaseSettingOff] +# flags: --python-version 3.9 --no-force-uppercase-builtins +x: type[type] +y: int + +y = x # E: Incompatible types in assignment (expression has type "type[type]", variable has type "int") + +[case testLowercaseSettingOnTypeAnnotationHint] +# flags: --python-version 3.9 --no-force-uppercase-builtins +x = [] # E: Need type annotation for "x" (hint: "x: list[] = ...") +y = {} # E: Need type annotation for "y" (hint: "y: dict[, ] = ...") +z = set() # E: Need type annotation for "z" (hint: "z: set[] = ...") +[builtins fixtures/primitives.pyi] + +[case testLowercaseSettingOnRevealTypeType] +# flags: --python-version 3.9 --no-force-uppercase-builtins +def f(t: type[int]) -> None: + reveal_type(t) # N: Revealed type is "type[builtins.int]" +reveal_type(f) # N: Revealed type is "def (t: type[builtins.int])" +[builtins fixtures/primitives.pyi] diff --git a/test-data/unit/check-modules-case.test b/test-data/unit/check-modules-case.test new file mode 100644 index 000000000000..b9e48888fea3 --- /dev/null +++ b/test-data/unit/check-modules-case.test @@ -0,0 +1,150 @@ +-- Type checker test cases dealing with modules and imports on case-insensitive filesystems. + +[case testCaseSensitivityDir] +# flags: --no-namespace-packages +from a import B # E: Module "a" has no attribute "B" + +[file a/__init__.py] +[file a/b/__init__.py] + +[case testCaseSensitivityDirNamespacePackages] +# flags: --namespace-packages +from a import B # E: Module "a" has no attribute "B" + +[file a/__init__.py] +[file a/b/__init__.py] + +[case testCaseInsensitivityDir] +# flags: --config-file tmp/mypy.ini + +from a import B # E: Module "a" has no attribute "B" +from other import x +reveal_type(x) # N: Revealed type is "builtins.int" + +[file a/__init__.py] +[file a/b/__init__.py] +[file FuNkY_CaSe/other.py] +x = 1 + +[file mypy.ini] +\[mypy] +mypy_path = tmp/funky_case + + +[case testCaseInsensitivityDirPyProjectTOML] +# flags: --config-file tmp/pyproject.toml + +from a import B # E: Module "a" has no attribute "B" +from other import x +reveal_type(x) # N: Revealed type is "builtins.int" + +[file a/__init__.py] + +[file a/b/__init__.py] + +[file FuNkY_CaSe/other.py] +x = 1 + +[file pyproject.toml] +\[tool.mypy] +mypy_path = "tmp/funky_case" + + +[case testPreferPackageOverFileCase] +# flags: --config-file tmp/mypy.ini +import a +[file funky/a.py] +/ # Deliberate syntax error, this file should not be parsed. +[file FuNkY/a/__init__.py] +pass + +[file mypy.ini] +\[mypy] +mypy_path = tmp/funky + + +[case testPreferPackageOverFileCasePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import a + +[file funky/a.py] +/ # Deliberate syntax error, this file should not be parsed. + +[file FuNkY/a/__init__.py] +pass + +[file pyproject.toml] +\[tool.mypy] +mypy_path = "tmp/funky" + + +[case testNotPreferPackageOverFileCase] +import a +[file a.py] +'no'() # E: "str" not callable +[file A/__init__.py] +/ # Deliberate syntax error, this file should not be parsed. + +[case testNamespacePackagePickFirstOnMypyPathCase] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar import x +reveal_type(x) # N: Revealed type is "builtins.int" +[file XX/foo/bar.py] +x = 0 +[file yy/foo/bar.py] +x = '' +[file mypy.ini] +\[mypy] +mypy_path = tmp/xx, tmp/yy + + +[case testNamespacePackagePickFirstOnMypyPathCasePyProjectTOML] +# flags: --namespace-packages --config-file tmp/pyproject.toml +from foo.bar import x +reveal_type(x) # N: Revealed type is "builtins.int" + +[file XX/foo/bar.py] +x = 0 + +[file yy/foo/bar.py] +x = '' + +[file pyproject.toml] +\[tool.mypy] +mypy_path = ["tmp/xx", "tmp/yy"] + + +[case testClassicPackageInsideNamespacePackageCase] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar.baz.boo import x +reveal_type(x) # N: Revealed type is "builtins.int" +[file xx/foo/bar/baz/boo.py] +x = '' +[file xx/foo/bar/baz/__init__.py] +[file yy/foo/bar/baz/boo.py] +x = 0 +[file yy/foo/bar/__init__.py] + +[file mypy.ini] +\[mypy] +mypy_path = TmP/xX, TmP/yY + + +[case testClassicPackageInsideNamespacePackageCasePyProjectTOML] +# flags: --namespace-packages --config-file tmp/pyproject.toml +from foo.bar.baz.boo import x +reveal_type(x) # N: Revealed type is "builtins.int" + +[file xx/foo/bar/baz/boo.py] +x = '' + +[file xx/foo/bar/baz/__init__.py] + +[file yy/foo/bar/baz/boo.py] +x = 0 + +[file yy/foo/bar/__init__.py] + +[file pyproject.toml] +\[tool.mypy] +mypy_path = ["TmP/xX", "TmP/yY"] diff --git a/test-data/unit/check-modules-fast.test b/test-data/unit/check-modules-fast.test new file mode 100644 index 000000000000..875125c6532b --- /dev/null +++ b/test-data/unit/check-modules-fast.test @@ -0,0 +1,136 @@ +-- Type checker test cases dealing with module lookup edge cases +-- to ensure that --fast-module-lookup matches regular lookup behavior + +[case testModuleLookup] +# flags: --fast-module-lookup +import m +reveal_type(m.a) # N: Revealed type is "m.A" + +[file m.py] +class A: pass +a = A() + +[case testModuleLookupStub] +# flags: --fast-module-lookup +import m +reveal_type(m.a) # N: Revealed type is "m.A" + +[file m.pyi] +class A: pass +a = A() + +[case testModuleLookupFromImport] +# flags: --fast-module-lookup +from m import a +reveal_type(a) # N: Revealed type is "m.A" + +[file m.py] +class A: pass +a = A() + +[case testModuleLookupStubFromImport] +# flags: --fast-module-lookup +from m import a +reveal_type(a) # N: Revealed type is "m.A" + +[file m.pyi] +class A: pass +a = A() + + +[case testModuleLookupWeird] +# flags: --fast-module-lookup +from m import a +reveal_type(a) # N: Revealed type is "builtins.object" +reveal_type(a.b) # N: Revealed type is "m.a.B" + +[file m.py] +class A: pass +a = A() + +[file m/__init__.py] +[file m/a.py] +class B: pass +b = B() + + +[case testModuleLookupWeird2] +# flags: --fast-module-lookup +from m.a import b +reveal_type(b) # N: Revealed type is "m.a.B" + +[file m.py] +class A: pass +a = A() + +[file m/__init__.py] +[file m/a.py] +class B: pass +b = B() + + +[case testModuleLookupWeird3] +# flags: --fast-module-lookup +from m.a import b +reveal_type(b) # N: Revealed type is "m.a.B" + +[file m.py] +class A: pass +a = A() +[file m/__init__.py] +class B: pass +a = B() +[file m/a.py] +class B: pass +b = B() + + +[case testModuleLookupWeird4] +# flags: --fast-module-lookup +import m.a +m.a.b # E: "str" has no attribute "b" + +[file m.py] +class A: pass +a = A() +[file m/__init__.py] +class B: pass +a = 'foo' +b = B() +[file m/a.py] +class C: pass +b = C() + + +[case testModuleLookupWeird5] +# flags: --fast-module-lookup +import m.a as ma +reveal_type(ma.b) # N: Revealed type is "m.a.C" + +[file m.py] +class A: pass +a = A() +[file m/__init__.py] +class B: pass +a = 'foo' +b = B() +[file m/a.py] +class C: pass +b = C() + + +[case testModuleLookupWeird6] +# flags: --fast-module-lookup +from m.a import b +reveal_type(b) # N: Revealed type is "m.a.C" + +[file m.py] +class A: pass +a = A() +[file m/__init__.py] +class B: pass +a = 'foo' +b = B() +[file m/a.py] +class C: pass +b = C() diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test new file mode 100644 index 000000000000..68897790e4bf --- /dev/null +++ b/test-data/unit/check-modules.test @@ -0,0 +1,3220 @@ +-- Type checker test cases dealing with modules and imports. +-- Towards the end there are tests for PEP 420 (namespace packages, i.e. __init__.py-less packages). + +[case testAccessImportedDefinitions0] +import m +import typing +m.f() # E: Missing positional argument "a" in call to "f" +m.f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A" +m.x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +m.f(m.A()) +m.x = m.A() +[file m.py] +class A: pass +def f(a: A) -> None: pass +x = A() + +[case testAccessImportedDefinitions1] +import m +import typing +m.f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A" +m.f(m.A()) +[file m.py] +class A: pass +def f(a: A) -> None: pass + +[case testAccessImportedDefinitions2] +from m import f, A +import typing +f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A" +f(A()) +[file m.py] +class A: pass +def f(a: A) -> None: pass + +[case testImportedExceptionType] +import m +import typing +try: + pass +except m.Err: + pass +except m.Bad: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +[file m.py] +class Err(BaseException): pass +class Bad: pass +[builtins fixtures/exception.pyi] + +[case testImportedExceptionType2] +from m import Err, Bad +import typing +try: + pass +except Err: + pass +except Bad: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +[file m.py] +class Err(BaseException): pass +class Bad: pass +[builtins fixtures/exception.pyi] + +[case testImportWithinBlock] +import typing +if 1: + import m + m.a = m.b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + m.a = m.a + m.f() + m.f(m.a) # E: Too many arguments for "f" + m.a = m.A() + m.a = m.B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[file m.py] +class A: pass +class B: pass +a = A() +b = B() +def f() -> None: pass + +[case testImportWithinFunction] +import typing +def f() -> None: + from m import a, b, f, A, B + if int(): + a = b \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = a + f() + f(a) # E: Too many arguments for "f" + a = A() + a = B() \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[file m.py] +class A: pass +class B: pass +a = A() +b = B() +def f() -> None: pass +[out] + +[case testImportWithinMethod] +import typing +class C: + def f(self) -> None: + from m import * + if int(): + a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = a + f() + f(a) # E: Too many arguments for "f" + a = A() + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[file m.py] +class A: pass +class B: pass +a = A() +b = B() +def f() -> None: pass +[out] + +[case testImportWithinClassBody] +import typing +class C: + import m + m.f() + m.f(C) # E: Too many arguments for "f" +[file m.py] +def f() -> None: pass +[out] + +[case testImportWithinClassBody2] +import typing +class C: + from m import f # E: Unsupported class scoped import + f() + # ideally, the following should error: + f(C) +[file m.py] +def f() -> None: pass +[out] + +[case testImportWithStub] +import _m +_m.f("hola") +[file _m.pyi] +def f(c:str) -> None: pass +[out] + +[case testImportWithStubIncompatibleType] +import _m +_m.f("hola") +_m.f(12) # E: Argument 1 to "f" has incompatible type "int"; expected "str" +[file _m.py] +def f(c): + print(c) +[file _m.pyi] +def f(c:str) -> None: pass + +[case testInvalidOperationsOnModules] +import m +import typing + +class A: pass +m() # E: Module not callable +a = m # type: A # E: Incompatible types in assignment (expression has type Module, variable has type "A") +m + None # E: Unsupported left operand type for + (Module) +[file m.py] +[builtins fixtures/module.pyi] + +[case testNameDefinedInDifferentModule] +import m, n +import typing +m.x # E: Module has no attribute "x" +[file m.py] +y = object() +[file n.py] +x = object() +[builtins fixtures/module.pyi] + +[case testChainedAssignmentAndImports] +import m + +i: int +s: str +if int(): + i = m.x +if int(): + i = m.y +if int(): + s = m.x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + s = m.y # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[file m.py] +x = y = 1 +[builtins fixtures/primitives.pyi] + +[case testConditionalFunctionDefinitionAndImports] +import m +import typing +m.f(1) +m.f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +[file m.py] +x = object() +if x: + def f(x: int) -> None: pass +else: + def f(x: int) -> None: pass + +[case testTypeCheckWithUnknownModule] +import nonexistent +None + '' +[out] +main:1: error: Cannot find implementation or library stub for module named "nonexistent" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Unsupported left operand type for + ("None") + +[case testTypeCheckWithUnknownModule2] +import m, nonexistent +None + '' +m.x = 1 +m.x = '' +[file m.py] +x = 1 +[out] +main:1: error: Cannot find implementation or library stub for module named "nonexistent" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Unsupported left operand type for + ("None") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testTypeCheckWithUnknownModule3] +import nonexistent, m +None + '' +m.x = 1 +m.x = '' +[file m.py] +x = 1 +[out] +main:1: error: Cannot find implementation or library stub for module named "nonexistent" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Unsupported left operand type for + ("None") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testTypeCheckWithUnknownModule4] +import nonexistent, another +None + '' +[out] +main:1: error: Cannot find implementation or library stub for module named "nonexistent" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "another" +main:2: error: Unsupported left operand type for + ("None") + +[case testTypeCheckWithUnknownModule5] +import nonexistent as x +None + '' +[out] +main:1: error: Cannot find implementation or library stub for module named "nonexistent" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Unsupported left operand type for + ("None") + +[case testTypeCheckWithUnknownModuleUsingFromImport] +from nonexistent import x +None + '' +[out] +main:1: error: Cannot find implementation or library stub for module named "nonexistent" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Unsupported left operand type for + ("None") + +[case testTypeCheckWithUnknownModuleUsingImportStar] +from nonexistent import * +None + '' +[out] +main:1: error: Cannot find implementation or library stub for module named "nonexistent" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Unsupported left operand type for + ("None") + +[case testAccessingUnknownModule] +import xyz +xyz.foo() +xyz() +[out] +main:1: error: Cannot find implementation or library stub for module named "xyz" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testAccessingUnknownModule2] +import xyz, bar +xyz.foo() +bar() +[out] +main:1: error: Cannot find implementation or library stub for module named "xyz" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "bar" + +[case testAccessingUnknownModule3] +import xyz as z +xyz.foo() +z() +[out] +main:1: error: Cannot find implementation or library stub for module named "xyz" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Name "xyz" is not defined + +[case testAccessingNameImportedFromUnknownModule] +from xyz import y, z +y.foo() +z() +[out] +main:1: error: Cannot find implementation or library stub for module named "xyz" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testAccessingNameImportedFromUnknownModule2] +from xyz import * +y +[out] +main:1: error: Cannot find implementation or library stub for module named "xyz" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Name "y" is not defined + +[case testAccessingNameImportedFromUnknownModule3] +from xyz import y as z +y +z +[out] +main:1: error: Cannot find implementation or library stub for module named "xyz" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Name "y" is not defined + +[case testUnknownModuleRedefinition] +# Error messages differ with the new analyzer + +import xab # E: Cannot find implementation or library stub for module named "xab" # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +def xab(): pass # E: Name "xab" already defined (possibly by an import) + +[case testAccessingUnknownModuleFromOtherModule] +import x +x.nonexistent.foo +x.z +[file x.py] +import nonexistent +[builtins fixtures/module.pyi] +[out] +tmp/x.py:1: error: Cannot find implementation or library stub for module named "nonexistent" +tmp/x.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:3: error: Module has no attribute "z" + +[case testUnknownModuleImportedWithinFunction] +def f(): + import foobar +def foobar(): pass +foobar('') +[out] +main:2: error: Cannot find implementation or library stub for module named "foobar" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:4: error: Too many arguments for "foobar" + +[case testUnknownModuleImportedWithinFunction2] +def f(): + from foobar import x +def x(): pass +x('') +[out] +main:2: error: Cannot find implementation or library stub for module named "foobar" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:4: error: Too many arguments for "x" + +[case testRelativeImports] +import typing +import m.a +m.a.x = m.a.y # Error +[file m/__init__.py] +[file m/a.py] +import typing +from .b import A, B, x, y +z = x +if int(): + z = y # Error +[file m/b.py] +import typing +class A: pass +class B: pass +x = A() +y = B() +[out] +tmp/m/a.py:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:3: error: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testRelativeImports2] +import typing +import m.a +m.a.x = m.a.y # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[file m/__init__.py] +[file m/a.py] +import typing +from .b import A, B, x, y +[file m/b.py] +import typing +class A: pass +class B: pass +x = A() +y = B() + +[case testExportedValuesInImportAll] +import typing +from m import * +_ = a +_ = b +_ = c +_ = d +_ = e # E: Name "e" is not defined +_ = f +_ = _g # E: Name "_g" is not defined +[file m.py] +__all__ = ['a'] +__all__ += ('b',) +__all__.append('c') +__all__.extend(('d', 'e', 'f')) +__all__.remove('e') + +a = b = c = d = e = f = _g = 1 +[builtins fixtures/module_all.pyi] + +[case testAllMustBeSequenceStr] +import typing +__all__ = [1, 2, 3] +[builtins fixtures/module_all.pyi] +[out] +main:2: error: Type of __all__ must be "Sequence[str]", not "List[int]" + +[case testUnderscoreExportedValuesInImportAll] +import typing +from m import * +_ = a +_ = _b +_ = __c__ +_ = ___d +_ = e +_ = f # E: Name "f" is not defined +_ = _g # E: Name "_g" is not defined +[file m.py] +__all__ = ['a'] +__all__ += ('_b',) +__all__.append('__c__') +__all__.extend(('___d', 'e')) + +a = _b = __c__ = ___d = e = f = _g = 1 +[builtins fixtures/module_all.pyi] + +[case testEllipsisInitializerInStubFileWithType] +import m +m.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[file m.pyi] +x = ... # type: int + +[case testEllipsisInitializerInStubFileWithoutType] +import m +m.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "ellipsis") +[file m.pyi] +# Ellipsis is only special with a # type: comment (not sure though if this is great) +x = ... + +[case testEllipsisInitializerInModule] +x = ... # type: int # E: Incompatible types in assignment (expression has type "ellipsis", variable has type "int") + +[case testEllipsisDefaultArgValueInStub] +import m +m.f(1) +m.f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +[file m.pyi] +def f(x: int = ...) -> None: pass + +[case testEllipsisDefaultArgValueInStub2] +import m +def f1(x: int = ...) -> int: return 1 +def f2(x: int = '') -> int: return 1 +[file m.pyi] +def g1(x: int = ...) -> int: pass +def g2(x: int = '') -> int: pass +[out] +tmp/m.pyi:2: error: Incompatible default for argument "x" (default has type "str", argument has type "int") +main:2: error: Incompatible default for argument "x" (default has type "ellipsis", argument has type "int") +main:3: error: Incompatible default for argument "x" (default has type "str", argument has type "int") + +[case testEllipsisDefaultArgValueInNonStub] +def ok_1(x: int = ...) -> None: pass +def ok_2(x: int = ...) -> None: ... +def ok_3(x: int = ...) -> None: raise NotImplementedError +def ok_4(x: int = ...) -> None: raise NotImplementedError() +def ok_5(x: int = ...) -> None: + """Docstring here""" + pass + +def bad_1(x: int = ...) -> None: 1 # E: Incompatible default for argument "x" (default has type "ellipsis", argument has type "int") +def bad_2(x: int = ...) -> None: # E: Incompatible default for argument "x" (default has type "ellipsis", argument has type "int") + """Docstring here""" + ok_1() +def bad_3(x: int = ...) -> None: # E: Incompatible default for argument "x" (default has type "ellipsis", argument has type "int") + raise Exception("Some other exception") +[builtins fixtures/exception.pyi] +[out] + +[case testEllipsisDefaultArgValueInNonStubsOverload] +from typing import overload, Union + +Both = Union[int, str] + +@overload +def foo(x: int, y: int = ...) -> int: ... +@overload +def foo(x: str, y: str = ...) -> str: ... +def foo(x: Both, y: Both = ...) -> Both: # E: Incompatible default for argument "y" (default has type "ellipsis", argument has type "Union[int, str]") + return x + +@overload +def bar(x: int, y: int = ...) -> int: ... +@overload +def bar(x: str, y: str = ...) -> str: ... +def bar(x: Both, y: Both = ...) -> Both: + raise NotImplementedError +[builtins fixtures/exception.pyi] +[out] + +[case testEllipsisDefaultArgValueInNonStubsMethods] +from typing import Generic, TypeVar +from typing_extensions import Protocol +from abc import abstractmethod + +T = TypeVar('T') +class Wrap(Generic[T]): ... + +class MyProtocol(Protocol): + def no_impl(self, x: Wrap[int] = ...) -> int: ... + def default_impl(self, x: Wrap[int] = ...) -> int: return 3 # E: Incompatible default for argument "x" (default has type "ellipsis", argument has type "Wrap[int]") + +class MyAbstractClass: + @abstractmethod + def no_impl(self, x: Wrap[int] = ...) -> int: raise NotImplementedError + + @abstractmethod + def default_impl(self, x: Wrap[int] = ...) -> int: return 3 # E: Incompatible default for argument "x" (default has type "ellipsis", argument has type "Wrap[int]") +[builtins fixtures/exception.pyi] +[out] + +[case testStarImportOverlapping] +from m1 import * +from m2 import * +j = '' +[file m1.py] +x = 1 +[file m2.py] +x = 1 + +[case testStarImportOverlappingMismatch] +from m1 import * +from m2 import * # E: Incompatible import of "x" (imported name has type "int", local name has type "str") +j = '' +[file m1.py] +x = '' +[file m2.py] +x = 1 + +[case testStarImportOverridingLocalImports] +from m1 import * +from m2 import * +x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[file m1.py] +x = 1 +[file m2.py] +x = 1 + +[case testAssignToFuncDefViaImport] + +# Errors differ with the new analyzer. (Old analyzer gave error on the +# input, which is maybe better, but no error about f, which seems +# wrong) +from m import * +f = None # E: Incompatible types in assignment (expression has type "None", variable has type "Callable[[], Any]") +x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[file m.py] +def f(): pass +x = 1+0 +[out] + + +-- Conditional definitions and function redefinitions via module object +-- -------------------------------------------------------------------- + + +[case testConditionalImportAndAssign] +# flags: --no-strict-optional +try: + from m import x +except: + x = None +try: + from m import x as y +except: + y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[file m.py] +x = '' + +[case testAssignAndConditionalImport] +x = '' +try: + from m import x +except: + pass +y = 1 +try: + from m import x as y # E: Incompatible import of "y" (imported name has type "str", local name has type "int") +except: + pass +[file m.py] +x = '' + +[case testAssignAndConditionalStarImport] +x = '' +y = 1 +try: + from m import * # E: Incompatible import of "y" (imported name has type "str", local name has type "int") +except: + pass +[file m.py] +x = '' +y = '' + +[case testRedefineImportedFunctionViaImport] +try: + from m import f, g +except: + def f(x): pass + def g(x): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: Any, y: Any) -> Any \ + # N: Redefinition: \ + # N: def g(x: Any) -> Any +[file m.py] +def f(x): pass +def g(x, y): pass + +[case testImportedVariableViaImport] +try: + from m import x +except: + from n import x # E: Incompatible import of "x" (imported name has type "str", local name has type "int") +[file m.py] +x = 1 +[file n.py] +x = '' + +[case testRedefineFunctionViaImport] +def f(x): pass +def g(x): pass +try: + from m import f, g # E: Incompatible import of "g" (imported name has type "Callable[[Any, Any], Any]", local name has type "Callable[[Any], Any]") +except: + pass + +import m as f # E: Incompatible import of "f" (imported name has type "object", local name has type "Callable[[Any], Any]") + +[file m.py] +def f(x): pass +def g(x, y): pass + +[case testRedefineTypeViaImport] +from typing import Type +import mod + +X: Type[mod.A] +Y: Type[mod.B] +from mod import B as X +from mod import A as Y # E: Incompatible import of "Y" (imported name has type "Type[A]", local name has type "Type[B]") + +import mod as X # E: Incompatible import of "X" (imported name has type "object", local name has type "Type[A]") + +[file mod.py] +class A: ... +class B(A): ... + + +[case testImportVariableAndAssignNone] +# flags: --no-strict-optional +try: + from m import x +except: + x = None +[file m.py] +x = 1 + +[case testImportFunctionAndAssignNone] +# flags: --no-strict-optional +try: + from m import f +except: + f = None +[file m.py] +def f(): pass + +[case testImportFunctionAndAssignFunction] +def g(x): pass +try: + from m import f +except: + f = g +[file m.py] +def f(x): pass + +[case testImportFunctionAndAssignIncompatible] +try: + from m import f +except: + f = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[[], Any]") +[file m.py] +def f(): pass + +[case testAssignToFuncDefViaGlobalDecl2] +# flags: --no-strict-optional +import typing +from m import f +def g() -> None: + global f + f = None + if int(): + f = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[[], Any]") +[file m.py] +def f(): pass +[out] + +[case testAssignToFuncDefViaNestedModules] +# flags: --no-strict-optional +import m.n +m.n.f = None +m.n.f = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[[], Any]") +[file m/__init__.py] +[file m/n.py] +def f(): pass +[out] + +[case testAssignToFuncDefViaModule] +# flags: --no-strict-optional +import m +m.f = None +m.f = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[[], Any]") +[file m.py] +def f(): pass +[out] + +[case testConditionalImportAndAssignNoneToModule] +# flags: --no-strict-optional +if object(): + import m +else: + m = None +m.f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "str" +[file m.py] +def f(x: str) -> None: pass +[builtins fixtures/module.pyi] +[out] + +[case testConditionalImportAndAssignInvalidToModule] +if object(): + import m +else: + m = 1 # E: Incompatible types in assignment (expression has type "int", variable has type Module) +[file m.py] +[builtins fixtures/module.pyi] +[out] + +[case testImportAndAssignToModule] +# flags: --no-strict-optional +import m +m = None +m.f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "str" +[file m.py] +def f(x: str) -> None: pass +[builtins fixtures/module.pyi] +[out] + + +-- Test cases that simulate 'mypy -m modname' +-- +-- The module name to import is encoded in a comment. + +[case testTypeCheckNamedModule] +# cmd: mypy -m m.a +[file m/__init__.py] +None + 1 +[file m/a.py] +[out] +tmp/m/__init__.py:1: error: Unsupported left operand type for + ("None") + +[case testTypeCheckNamedModule2] +# cmd: mypy -m m.a +[file m/__init__.py] +[file m/a.py] +None + 1 +[out] +tmp/m/a.py:1: error: Unsupported left operand type for + ("None") + +[case testTypeCheckNamedModule3] +# cmd: mypy -m m +[file m/__init__.py] +None + 1 +[file m/a.py] +[out] +tmp/m/__init__.py:1: error: Unsupported left operand type for + ("None") + +[case testTypeCheckNamedModule4] +# cmd: mypy -m m +[file m/__init__.py] +[file m/a.py] +None + 1 # Not analyzed. +[out] + +[case testTypeCheckNamedModule5] +# cmd: mypy -m m +None + '' # Not analyzed. +[file m.py] +None + 1 +[out] +tmp/m.py:1: error: Unsupported left operand type for + ("None") + +[case testTypeCheckNamedModuleWithImportCycle] +# cmd: mypy -m m.a +None + 1 # Does not generate error, as this file won't be analyzed. +[file m/__init__.py] +import m.a +[file m/a.py] +[out] + +[case testCheckDecoratedFuncAsAnnotWithImportCycle] + +import a +[file a.py] +from typing import TypeVar +import b + +T = TypeVar('T') +def idf(x: T) -> T: return x + +@idf +def Session() -> None: pass + +[file b.py] +MYPY = False +if MYPY: + from a import Session + +def f(self, session: Session) -> None: # E: Function "a.Session" is not valid as a type \ + # N: Perhaps you need "Callable[...]" or a callback protocol? + pass +[builtins fixtures/bool.pyi] + + +-- Checks dealing with submodules and different kinds of imports +-- ------------------------------------------------------------- + +[case testSubmoduleRegularImportAddsAllParents] +import a.b.c +reveal_type(a.value) # N: Revealed type is "builtins.int" +reveal_type(a.b.value) # N: Revealed type is "builtins.str" +reveal_type(a.b.c.value) # N: Revealed type is "builtins.float" +b.value # E: Name "b" is not defined +c.value # E: Name "c" is not defined + +[file a/__init__.py] +value = 3 +[file a/b/__init__.py] +value = "a" +[file a/b/c.py] +value = 3.2 +[out] + +[case testSubmoduleImportAsDoesNotAddParents] +import a.b.c as foo +reveal_type(foo.value) # N: Revealed type is "builtins.float" +a.value # E: Name "a" is not defined +b.value # E: Name "b" is not defined +c.value # E: Name "c" is not defined + +[file a/__init__.py] +value = 3 +[file a/b/__init__.py] +value = "a" +[file a/b/c.py] +value = 3.2 +[out] + +[case testSubmoduleImportFromDoesNotAddParents] +from a import b +reveal_type(b.value) # N: Revealed type is "builtins.str" +b.c.value # E: Module has no attribute "c" +a.value # E: Name "a" is not defined + +[file a/__init__.py] +value = 3 +[file a/b/__init__.py] +value = "a" +[file a/b/c.py] +value = 3.2 +[builtins fixtures/module.pyi] +[out] + +[case testSubmoduleImportFromDoesNotAddParents2] +from a.b import c +reveal_type(c.value) # N: Revealed type is "builtins.float" +a.value # E: Name "a" is not defined +b.value # E: Name "b" is not defined + +[file a/__init__.py] +value = 3 +[file a/b/__init__.py] +value = "a" +[file a/b/c.py] +value = 3.2 +[out] + +[case testSubmoduleRegularImportNotDirectlyAddedToParent] +import a.b.c +def accept_float(x: float) -> None: pass +accept_float(a.b.c.value) + +[file a/__init__.py] +value = 3 +b.value +a.b.value + +[file a/b/__init__.py] +value = "a" +c.value +a.b.c.value + +[file a/b/c.py] +value = 3.2 +[out] +tmp/a/__init__.py:2: error: Name "b" is not defined +tmp/a/__init__.py:3: error: Name "a" is not defined +tmp/a/b/__init__.py:2: error: Name "c" is not defined +tmp/a/b/__init__.py:3: error: Name "a" is not defined + +[case testSubmoduleMixingLocalAndQualifiedNames] +from a.b import MyClass +val1: a.b.MyClass # E: Name "a" is not defined +val2: MyClass + +[file a/__init__.py] +[file a/b.py] +class MyClass: pass +[out] + +[case testSubmoduleMixingImportFrom] +import parent.child + +[file parent/__init__.py] + +[file parent/common.py] +class SomeClass: pass + +[file parent/child.py] +from parent.common import SomeClass +from parent import common +foo = parent.common.SomeClass() + +[builtins fixtures/module.pyi] +[out] +tmp/parent/child.py:3: error: Name "parent" is not defined + +[case testSubmoduleMixingImportFromAndImport] +import parent.child + +[file parent/__init__.py] + +[file parent/common.py] +class SomeClass: pass + +[file parent/unrelated.py] +class ShouldNotLoad: pass + +[file parent/child.py] +from parent.common import SomeClass +import parent + +# Note, since this might be unintuitive -- when `parent.common` is loaded in any way, +# shape, or form, it's added to `parent`'s namespace, which is why the below line +# succeeds. +foo = parent.common.SomeClass() +reveal_type(foo) +bar = parent.unrelated.ShouldNotLoad() + +[builtins fixtures/module.pyi] +[out] +tmp/parent/child.py:8: note: Revealed type is "parent.common.SomeClass" +tmp/parent/child.py:9: error: Module has no attribute "unrelated" + +[case testSubmoduleMixingImportFromAndImport2] +import parent.child + +[file parent/__init__.py] + +[file parent/common.py] +class SomeClass: pass + +[file parent/child.py] +from parent import common +import parent +foo = parent.common.SomeClass() +reveal_type(foo) + +[builtins fixtures/module.pyi] +[out] +tmp/parent/child.py:4: note: Revealed type is "parent.common.SomeClass" + +-- Tests repeated imports + +[case testIdenticalImportFromTwice] +from a import x, y, z +from b import x, y, z +[file a.py] +from common import x, y, z +[file b.py] +from common import x, y, z +[file common.py] +x = 3 +def y() -> int: return 3 +class z: pass +[out] + +[case testIdenticalImportStarTwice] +from a import * +from b import * +[file a.py] +from common import x, y, z +[file b.py] +from common import x, y, z +[file common.py] +x = 3 +def y() -> int: return 3 +class z: pass +[out] + +[case testDifferentImportSameNameTwice] +from a import x, y, z +from b import x, y, z +[file a.py] +x = 3 +def y() -> int: return 1 +class z: pass +[file b.py] +x = "foo" +def y() -> str: return "foo" +class z: pass +[out] +main:2: error: Incompatible import of "x" (imported name has type "str", local name has type "int") +main:2: error: Incompatible import of "y" (imported name has type "Callable[[], str]", local name has type "Callable[[], int]") +main:2: error: Incompatible import of "z" (imported name has type "Type[b.z]", local name has type "Type[a.z]") + +-- Misc + +[case testInheritFromBadImport] +# cmd: mypy -m bar +[file foo.py] +pass +[file bar.py] +from foo import B +class C(B): + pass +[out] +tmp/bar.py:1: error: Module "foo" has no attribute "B" + +[case testImportSuppressedWhileAlmostSilent] +# cmd: mypy -m main +# flags: --follow-imports=error +[file main.py] +import mod +[file mod.py] +[builtins fixtures/module.pyi] +[out] +tmp/main.py:1: error: Import of "mod" ignored +tmp/main.py:1: note: (Using --follow-imports=error, module not passed on command line) + +[case testAncestorSuppressedWhileAlmostSilent] +# cmd: mypy -m foo.bar +# flags: --follow-imports=error +[file foo/bar.py] +[file foo/__init__.py] +[builtins fixtures/module.pyi] +[out] +tmp/foo/bar.py: error: Ancestor package "foo" ignored +tmp/foo/bar.py: note: (Using --follow-imports=error, submodule passed on command line) + +[case testStubImportNonStubWhileSilent] +# cmd: mypy -m main +# flags: --follow-imports=skip +[file main.py] +from stub import x, z # Followed +from other import y # Not followed +x + '' # No error here +y + '' # No error here +z + '' # Error here +[file stub.pyi] +from non_stub import x as x # this import is not followed + +z = 42 +[file non_stub.py] +x = 42 + +x + '' # no error because file is not analyzed +[file other.py] +y = 42 +[builtins fixtures/module.pyi] +[out] +tmp/main.py:5: error: Unsupported left operand type for + ("int") + +[case testSilentSubmoduleImport] +# cmd: mypy -m foo +# flags: --follow-imports=skip +[file foo/__init__.py] +from foo import bar +[file foo/bar.py] +pass + +[case testImportReExportFromChildrenInCycle1] +# cmd: mypy -m project.root project.study.a project.neighbor +[file project/__init__.py] +from project.study import CustomType +x = 10 +[file project/root.py] +[file project/study/__init__.py] +from project.study.a import CustomType +[file project/study/a.py] +from project import root +# TODO (#4498): This test is basically testing the `all_are_submodules` logic +# in build, which skips generating a dependency to a module if +# everything in it is a submodule. But that is still all just a +# workaround for bugs in cycle handling. If we uncomment the next +# line, we'll still break: +# from project import x +CustomType = str +[file project/neighbor/__init__.py] +from project.study import CustomType +def m(arg: CustomType) -> str: + return 'test' + +[case testImportReExportFromChildrenInCycle2] +# cmd: mypy -m project project.b project.ba project.c +# See comments in above test about this being a workaround. +[file foo.py] +def get_foo() -> int: return 12 + +[file project/ba.py] +from . import b +b.FOO + +[file project/b.py] +import foo +from . import c +FOO = foo.get_foo() + +[file project/c.py] + +[file project/__init__.py] +from . import ba + +[case testSuperclassInImportCycle] +import a +import d +a.A().f(d.D()) +[file a.py] +if 0: + import d +class B: pass +class C(B): pass +class A: + def f(self, x: B) -> None: pass +[file d.py] +import a +class D(a.C): pass + +[case testSuperclassInImportCycleReversedImports] +import d +import a +a.A().f(d.D()) +[file a.py] +if 0: + import d +class B: pass +class C(B): pass +class A: + def f(self, x: B) -> None: pass +[file d.py] +import a +class D(a.C): pass + +[case testPreferPackageOverFile] +import a +[file a.py] +/ # intentional syntax error -- this file shouldn't be parsed +[file a/__init__.py] +pass +[out] + +[case testPreferPackageOverFile2] +from a import x +[file a.py] +/ # intentional syntax error -- this file shouldn't be parsed +[file a/__init__.py] +x = 0 +[out] + +[case testImportInClass] +class C: + import foo +reveal_type(C.foo.bar) # N: Revealed type is "builtins.int" +[file foo.py] +bar = 0 +[builtins fixtures/module.pyi] +[out] + +[case testIfFalseImport] +if False: + import a +def f(x: 'a.A') -> int: + return x.f() +[file a.py] +class A: + def f(self) -> int: + return 0 +[builtins fixtures/bool.pyi] + + +-- Test stability under import cycles +-- ---------------------------------- + +-- The first two tests are identical except one main has 'import x' +-- and the other 'import y'. Previously (before build.order_ascc() +-- was added) one of these would fail because the imports were +-- processed in the (reverse) order in which the files were +-- encountered. + +[case testImportCycleStability1] +import x +[file x.py] +def f() -> str: return '' +class Base: + attr = f() +def foo(): + import y +[file y.py] +import x +class Sub(x.Base): + attr = x.Base.attr +[out] + +[case testImportCycleStability2] +import y +[file x.py] +def f() -> str: return '' +class Base: + attr = f() +def foo(): + import y +[file y.py] +import x +class Sub(x.Base): + attr = x.Base.attr +[out] + +-- This case isn't fixed by order_ascc(), but is fixed by the +-- lightweight type inference added to semanal.py +-- (analyze_simple_literal_type()). + +[case testImportCycleStability3] +import y +[file x.py] +class Base: + pass +def foo() -> int: + import y + reveal_type(y.Sub.attr) + return y.Sub.attr +[file y.py] +import x +class Sub(x.Base): + attr = 0 +[out] +tmp/x.py:5: note: Revealed type is "builtins.int" + +-- This case has a symmetrical cycle, so it doesn't matter in what +-- order the files are processed. It depends on the lightweight type +-- interference. + +[case testImportCycleStability4] +import x +[file x.py] +import y +class C: + attr = '' +def foo() -> int: + return y.D.attr +[file y.py] +import x +class D: + attr = 0 +def bar() -> str: + return x.C.attr + +-- These cases test all supported literal types. + +[case testImportCycleStability5] +import y +[file x.py] +class Base: + pass +def foo() -> None: + import y + i = y.Sub.iattr # type: int + f = y.Sub.fattr # type: float + s = y.Sub.sattr # type: str + b = y.Sub.battr # type: bytes +[file y.py] +import x +class Sub(x.Base): + iattr = 0 + fattr = 0.0 + sattr = '' + battr = b'' +[out] + +-- This case tests module-level variables. + +[case testImportCycleStability7] +import x +[file x.py] +def foo() -> int: + import y + reveal_type(y.value) + return y.value +[file y.py] +import x +value = 12 +[out] +tmp/x.py:3: note: Revealed type is "builtins.int" + +-- This is not really cycle-related but still about the lightweight +-- type checker. + +[case testImportCycleStability8] +x = 1 # type: str +reveal_type(x) +[out] +main:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") +main:2: note: Revealed type is "builtins.str" + +-- Tests for cross-module second_pass checking. + +[case testSymmetricImportCycle1] +import a +[file a.py] +import b +def f() -> int: + return b.x +y = 0 + int() +[file b.py] +import a +def g() -> int: + reveal_type(a.y) + return a.y +x = 1 + int() +[out] +tmp/b.py:3: note: Revealed type is "builtins.int" + +[case testSymmetricImportCycle2] +import b +[file a.py] +import b +def f() -> int: + reveal_type(b.x) + return b.x +y = 0 + int() +[file b.py] +import a +def g() -> int: + return a.y +x = 1 + int() +[out] +tmp/a.py:3: note: Revealed type is "builtins.int" + +[case testThreePassesRequired] +import b +[file a.py] +import b +class C: + def f1(self) -> None: + self.x2 + def f2(self) -> None: + self.x2 = b.b +[file b.py] +import a +b = 1 + int() +[out] +tmp/a.py:4: error: Cannot determine type of "x2" + +[case testErrorInPassTwo1] +import b +[file a.py] +import b +def f() -> None: + a = b.x + 1 + a + '' +[file b.py] +import a +x = 1 + int() +[out] +tmp/a.py:4: error: Unsupported operand types for + ("int" and "str") + +[case testErrorInPassTwo2] +import a +[file a.py] +import b +def f() -> None: + a = b.x + 1 + a + '' +[file b.py] +import a +x = 1 + int() +[out] +tmp/a.py:4: error: Unsupported operand types for + ("int" and "str") + +[case testDeferredDecorator] +import a +[file a.py] +import b +def g() -> None: + f('') +@b.deco +def f(a: str) -> int: pass +reveal_type(f) +x = 1 + int() +[file b.py] +from typing import Callable, TypeVar +import a +T = TypeVar('T') +def deco(f: Callable[[T], int]) -> Callable[[T], int]: + a.x + return f +[out] +tmp/a.py:6: note: Revealed type is "def (builtins.str) -> builtins.int" + +[case testDeferredClassContext] +class A: + def f(self) -> str: return 'foo' +class B(A): + def f(self) -> str: return self.x + def initialize(self) -> None: self.x = 'bar' + +[case testDeferredClassContextUnannotated] +class A: + def f(self) -> str: return 'foo' +class B(A): + def f(self) -> str: return self.x + def initialize(self): self.x = 'bar' + +-- Scripts and __main__ + +[case testScriptsAreModules] +# flags: --scripts-are-modules +[file a] +pass +[file b] +pass + +-- Misc + +[case testScriptsAreNotModules] +# cmd: mypy a b +[file a] +pass +[file b] +pass +[out] + +[case testTypeCheckPrio] +# cmd: mypy -m part1 part2 part3 part4 + +[file part1.py] +from part3 import Thing +class FirstThing: pass + +[file part2.py] +from part4 import part4_thing as Thing + +[file part3.py] +from part2 import Thing +reveal_type(Thing) + +[file part4.py] +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from part1 import FirstThing +def part4_thing(a: int) -> str: pass + +[builtins fixtures/bool.pyi] +[typing fixtures/typing-medium.pyi] +[out] +tmp/part3.py:2: note: Revealed type is "def (a: builtins.int) -> builtins.str" + +[case testImportStarAliasAnyList] +import bar + +[file bar.py] +from foo import * +def bar(y: AnyAlias) -> None: pass + +l: ListAlias[int] +reveal_type(l) + +[file foo.py] +from typing import Any, List +AnyAlias = Any +ListAlias = List +[builtins fixtures/list.pyi] +[out] +tmp/bar.py:5: note: Revealed type is "builtins.list[builtins.int]" + +[case testImportStarAliasSimpleGeneric] +from ex2a import * + +def do_something(dic: Row) -> None: + pass + +def do_another() -> Row: + return {} + +do_something({'good': 'bad'}) # E: Dict entry 0 has incompatible type "str": "str"; expected "str": "int" +reveal_type(do_another()) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + +[file ex2a.py] +from typing import Dict +Row = Dict[str, int] +[builtins fixtures/dict.pyi] +[out] + +[case testImportStarAliasGeneric] +from y import * +notes: G[X] +another = G[X]() +second = XT[str]() +last = XT[G]() + +reveal_type(notes) # N: Revealed type is "y.G[y.G[builtins.int]]" +reveal_type(another) # N: Revealed type is "y.G[y.G[builtins.int]]" +reveal_type(second) # N: Revealed type is "y.G[builtins.str]" +reveal_type(last) # N: Revealed type is "y.G[y.G[Any]]" + +[file y.py] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class G(Generic[T]): + pass + +X = G[int] +XT = G[T] +[out] + +[case testImportStarAliasCallable] +from foo import * +from typing import Any + +def bar(x: Any, y: AnyCallable) -> Any: + return 'foo' + +cb: AnyCallable +reveal_type(cb) # N: Revealed type is "def (*Any, **Any) -> Any" + +[file foo.py] +from typing import Callable, Any +AnyCallable = Callable[..., Any] +[out] + +[case testRevealType] +import types +def f() -> types.ModuleType: + return types +reveal_type(f()) # N: Revealed type is "types.ModuleType" +reveal_type(types) # N: Revealed type is "types.ModuleType" + +[builtins fixtures/module.pyi] + +[case testClassImportAccessedInMethod] +class C: + import m + def foo(self) -> None: + x = self.m.a + reveal_type(x) # N: Revealed type is "builtins.str" + # ensure we distinguish self from other variables + y = 'hello' + z = y.m.a # E: "str" has no attribute "m" + @classmethod + def cmethod(cls) -> None: + y = cls.m.a + reveal_type(y) # N: Revealed type is "builtins.str" + @staticmethod + def smethod(foo: int) -> None: + # we aren't confused by first arg of a staticmethod + y = foo.m.a # E: "int" has no attribute "m" + +[file m.py] +a = 'foo' + +[builtins fixtures/module.pyi] + +[case testModuleAlias] +import m +m2 = m +reveal_type(m2.a) # N: Revealed type is "builtins.str" +m2.b # E: Module has no attribute "b" +m2.c = 'bar' # E: Module has no attribute "c" + +[file m.py] +a = 'foo' + +[builtins fixtures/module.pyi] + +[case testClassModuleAlias] +import m + +class C: + x = m + def foo(self) -> None: + reveal_type(self.x.a) # N: Revealed type is "builtins.str" + +[file m.py] +a = 'foo' + +[builtins fixtures/module.pyi] + +[case testLocalModuleAlias] +import m + +def foo() -> None: + x = m + reveal_type(x.a) # N: Revealed type is "builtins.str" + +class C: + def foo(self) -> None: + x = m + reveal_type(x.a) # N: Revealed type is "builtins.str" + +[file m.py] +a = 'foo' + +[builtins fixtures/module.pyi] + +[case testChainedModuleAlias] +import m +m3 = m2 = m +m4 = m3 +m5 = m4 +reveal_type(m2.a) # N: Revealed type is "builtins.str" +reveal_type(m3.a) # N: Revealed type is "builtins.str" +reveal_type(m4.a) # N: Revealed type is "builtins.str" +reveal_type(m5.a) # N: Revealed type is "builtins.str" + +[file m.py] +a = 'foo' + +[builtins fixtures/module.pyi] + +[case testMultiModuleAlias] +import m, n +m2, n2, (m3, n3) = m, n, [m, n] +reveal_type(m2.a) # N: Revealed type is "builtins.str" +reveal_type(n2.b) # N: Revealed type is "builtins.str" +reveal_type(m3.a) # N: Revealed type is "builtins.str" +reveal_type(n3.b) # N: Revealed type is "builtins.str" + +x, y = m # E: Module object is not iterable +x, y, z = m, n # E: Need more than 2 values to unpack (3 expected) +x, y = m, m, m # E: Too many values to unpack (2 expected, 3 provided) +x, (y, z) = m, n # E: Module object is not iterable +x, (y, z) = m, (n, n, n) # E: Too many values to unpack (2 expected, 3 provided) + +[file m.py] +a = 'foo' + +[file n.py] +b = 'bar' + +[builtins fixtures/module.pyi] + +[case testModuleAliasWithExplicitAnnotation] +from typing import Any +import types +import m +mod_mod: types.ModuleType = m +mod_mod2: types.ModuleType +mod_mod2 = m +mod_mod3 = m # type: types.ModuleType +mod_any: Any = m +mod_int: int = m # E: Incompatible types in assignment (expression has type Module, variable has type "int") + +reveal_type(mod_mod) # N: Revealed type is "types.ModuleType" +reveal_type(mod_mod.a) # N: Revealed type is "Any" +reveal_type(mod_mod2) # N: Revealed type is "types.ModuleType" +reveal_type(mod_mod2.a) # N: Revealed type is "Any" +reveal_type(mod_mod3) # N: Revealed type is "types.ModuleType" +reveal_type(mod_mod3.a) # N: Revealed type is "Any" +reveal_type(mod_any) # N: Revealed type is "Any" + +[file m.py] +a = 'foo' + +[builtins fixtures/module.pyi] + +[case testModuleAliasPassedToFunction] +import types +import m + +def takes_module(x: types.ModuleType): + reveal_type(x.__file__) # N: Revealed type is "builtins.str" + +n = m +takes_module(m) +takes_module(n) + +[file m.py] +a = 'foo' + +[builtins fixtures/module.pyi] + +[case testModuleAliasRepeated] +import m, n + +if bool(): + x = m +else: + x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type Module) + +if bool(): + y = 3 +else: + y = m # E: Incompatible types in assignment (expression has type Module, variable has type "int") + +if bool(): + z = m +else: + z = n # E: Cannot assign multiple modules to name "z" without explicit "types.ModuleType" annotation + +[file m.py] +a = 'foo' + +[file n.py] +a = 3 + +[builtins fixtures/module.pyi] + +[case testModuleAliasRepeatedWithAnnotation] +import types +import m, n + +x: types.ModuleType +if bool(): + x = m +else: + x = n + +reveal_type(x.nope) # N: Revealed type is "Any" +reveal_type(x.__file__) # N: Revealed type is "builtins.str" + +[file m.py] +a = 'foo' + +[file n.py] +a = 3 + +[builtins fixtures/module.pyi] + +[case testModuleAliasRepeatedComplex] +import m, n, o + +x = m +if int(): + x = n # E: Cannot assign multiple modules to name "x" without explicit "types.ModuleType" annotation +if int(): + x = o # E: Cannot assign multiple modules to name "x" without explicit "types.ModuleType" annotation + +y = o +if int(): + y, z = m, n # E: Cannot assign multiple modules to name "y" without explicit "types.ModuleType" annotation + +xx = m +if int(): + xx = m +reveal_type(xx.a) # N: Revealed type is "builtins.str" + +[file m.py] +a = 'foo' + +[file n.py] +a = 3 + +[file o.py] +a = 'bar' + +[builtins fixtures/module.pyi] + +[case testModuleAliasToOtherModule] +import m, n +m = n # E: Cannot assign multiple modules to name "m" without explicit "types.ModuleType" annotation + +[file m.py] + +[file n.py] + +[builtins fixtures/module.pyi] + +[case testNoReExportFromStubs] +from stub import Iterable # E: Module "stub" does not explicitly export attribute "Iterable" +from stub import D # E: Module "stub" does not explicitly export attribute "D" +from stub import C +from stub import foo +from stub import bar # E: Module "stub" does not explicitly export attribute "bar" + +c = C() +reveal_type(c.x) # N: Revealed type is "builtins.int" +it: Iterable[int] +reveal_type(it) # N: Revealed type is "typing.Iterable[builtins.int]" + +[file stub.pyi] +from typing import Iterable +from substub import C as C +from substub import C as D +from package import foo as foo +import package.bar as bar + +def fun(x: Iterable[str]) -> Iterable[int]: pass + +[file substub.pyi] +class C: + x: int + +[file package/foo.pyi] + +[file package/bar.pyi] + +[builtins fixtures/module.pyi] + +[case testNoReExportFromStubsMemberType] +import stub + +c = stub.C() +reveal_type(c.x) # N: Revealed type is "builtins.int" +it: stub.Iterable[int] # E: Name "stub.Iterable" is not defined +reveal_type(it) # N: Revealed type is "Any" + +[file stub.pyi] +from typing import Iterable +from substub import C as C + +def fun(x: Iterable[str]) -> Iterable[int]: pass + +[file substub.pyi] +class C: + x: int + +[builtins fixtures/module.pyi] + +[case testNoReExportFromStubsMemberVar] +import stub + +reveal_type(stub.y) # N: Revealed type is "builtins.int" +reveal_type(stub.z) # E: Module "stub" does not explicitly export attribute "z" \ + # N: Revealed type is "builtins.int" + + +[file stub.pyi] +from substub import y as y +from substub import z + +[file substub.pyi] +y = 42 +z: int + +[builtins fixtures/module.pyi] + +[case testReExportChildStubs] +import mod +from mod import submod + +reveal_type(mod.x) # N: Revealed type is "mod.submod.C" +y = submod.C() +reveal_type(y.a) # N: Revealed type is "builtins.str" + +[file mod/__init__.pyi] +from . import submod +x: submod.C + +[file mod/submod.pyi] +class C: + a: str + +[builtins fixtures/module.pyi] + +[case testReExportChildStubs2] +import mod.submod + +y = mod.submod.C() +reveal_type(y.a) # N: Revealed type is "builtins.str" + +[file mod/__init__.pyi] +from . import submod +x: submod.C + +[file mod/submod.pyi] +class C: + a: str + +[builtins fixtures/module.pyi] + +[case testReExportChildStubs3] +from util import mod +reveal_type(mod) # N: Revealed type is "def () -> package.mod.mod" + +from util import internal_detail # E: Module "util" does not explicitly export attribute "internal_detail" + +[file package/__init__.pyi] +from .mod import mod as mod + +[file package/mod.pyi] +class mod: ... + +[file util.pyi] +from package import mod as mod +# stubs require explicit re-export +from package import mod as internal_detail +[builtins fixtures/module.pyi] + +[case testNoReExportUnrelatedModule] +from mod2 import unrelated # E: Module "mod2" does not explicitly export attribute "unrelated" + +[file mod1/__init__.pyi] +[file mod1/unrelated.pyi] +x: int + +[file mod2.pyi] +from mod1 import unrelated +[builtins fixtures/module.pyi] + +[case testNoReExportUnrelatedSiblingPrefix] +from pkg.unrel import unrelated # E: Module "pkg.unrel" does not explicitly export attribute "unrelated" + +[file pkg/__init__.pyi] +[file pkg/unrelated.pyi] +x: int + +[file pkg/unrel.pyi] +from pkg import unrelated +[builtins fixtures/module.pyi] + +[case testNoReExportChildStubs] +import mod +from mod import C, D # E: Module "mod" does not explicitly export attribute "C" + +reveal_type(mod.x) # N: Revealed type is "mod.submod.C" +mod.C # E: Module "mod" does not explicitly export attribute "C" +y = mod.D() +reveal_type(y.a) # N: Revealed type is "builtins.str" + +[file mod/__init__.pyi] +from .submod import C, D as D +x: C + +[file mod/submod.pyi] +class C: pass +class D: + a: str +[builtins fixtures/module.pyi] + +[case testNoReExportNestedStub] +from stub import substub # E: Module "stub" does not explicitly export attribute "substub" + +[file stub.pyi] +import substub + +[file substub.pyi] +x = 42 + +[file mod/submod.pyi] + +[case testModuleAliasToQualifiedImport] +import package.module +alias = package.module +reveal_type(alias.whatever('/')) # N: Revealed type is "builtins.str" +[file package/__init__.py] + +[file package/module.py] +from typing import TypeVar +T = TypeVar('T') +def whatever(x: T) -> T: pass +[builtins fixtures/module.pyi] + +[case testModuleAliasToQualifiedImport2] +import mod +import othermod +alias = mod.submod +reveal_type(alias.whatever('/')) # N: Revealed type is "builtins.str" +if int(): + alias = othermod # E: Cannot assign multiple modules to name "alias" without explicit "types.ModuleType" annotation +[file mod.py] +import submod +[file submod.py] +from typing import TypeVar +T = TypeVar('T') +def whatever(x: T) -> T: pass +[file othermod.py] + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattr] +import has_getattr + +reveal_type(has_getattr.any_attribute) # N: Revealed type is "Any" + +[file has_getattr.pyi] +from typing import Any + +def __getattr__(name: str) -> Any: ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrReturnType] +import has_getattr + +reveal_type(has_getattr.any_attribute) # N: Revealed type is "builtins.str" + +[file has_getattr.pyi] +def __getattr__(name: str) -> str: ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrInvalidSignature] +import has_getattr + +reveal_type(has_getattr.any_attribute) + +[file has_getattr.pyi] +def __getattr__(x: int, y: str) -> str: ... + +[out] +tmp/has_getattr.pyi:1: error: Invalid signature "Callable[[int, str], str]" for "__getattr__" +main:3: note: Revealed type is "builtins.str" + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrNotCallable] +import has_getattr + +reveal_type(has_getattr.any_attribute) + +[file has_getattr.pyi] +__getattr__ = 3 + +[out] +tmp/has_getattr.pyi:1: error: Invalid signature "int" for "__getattr__" +main:3: note: Revealed type is "Any" + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrUntyped] +import has_getattr +reveal_type(has_getattr.any_attribute) # N: Revealed type is "Any" + +[file has_getattr.pyi] +def __getattr__(name): ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrNotStub] +import has_getattr +reveal_type(has_getattr.any_attribute) # N: Revealed type is "builtins.str" + +[file has_getattr.py] +def __getattr__(name) -> str: ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattribute] + +def __getattribute__(): ... # E: __getattribute__ is not valid at the module level + +[case testModuleLevelGetattrImportFrom] +from has_attr import name +reveal_type(name) # N: Revealed type is "Any" + +[file has_attr.pyi] +from typing import Any +def __getattr__(name: str) -> Any: ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrImportFromRetType] +from has_attr import int_attr +reveal_type(int_attr) # N: Revealed type is "builtins.int" + +[file has_attr.pyi] +def __getattr__(name: str) -> int: ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrImportFromNotStub] +from non_stub import name +reveal_type(name) # N: Revealed type is "Any" + +[file non_stub.py] +from typing import Any +def __getattr__(name: str) -> Any: ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrImportFromAs] +from has_attr import name as n +reveal_type(name) # E: Name "name" is not defined # N: Revealed type is "Any" +reveal_type(n) # N: Revealed type is "Any" + +[file has_attr.pyi] +from typing import Any +def __getattr__(name: str) -> Any: ... + +[builtins fixtures/module.pyi] + +[case testModuleLevelGetattrImportFromAsTwice] + +from has_attr import name +from has_attr import name +from has_attr import x +from has_attr import y as x # E: Name "x" already defined (possibly by an import) +reveal_type(name) # N: Revealed type is "builtins.int" + +[file has_attr.pyi] +from typing import Any +def __getattr__(name: str) -> int: ... + + +[case testModuleLevelGetattrAssignedGood] +import non_stub +reveal_type(non_stub.name) # N: Revealed type is "builtins.int" + +[file non_stub.py] +from typing import Callable + +def make_getattr_good() -> Callable[[str], int]: ... +__getattr__ = make_getattr_good() # OK + +[case testModuleLevelGetattrAssignedBad] +import non_stub +reveal_type(non_stub.name) + +[file non_stub.py] +from typing import Callable + +def make_getattr_bad() -> Callable[[], int]: ... +__getattr__ = make_getattr_bad() + +[out] +tmp/non_stub.py:4: error: Invalid signature "Callable[[], int]" for "__getattr__" +main:2: note: Revealed type is "builtins.int" + +[case testModuleLevelGetattrImportedGood] +import non_stub +reveal_type(non_stub.name) # N: Revealed type is "builtins.int" + +[file non_stub.py] +from has_getattr import __getattr__ + +[file has_getattr.py] +def __getattr__(name: str) -> int: ... + +[case testModuleLevelGetattrImportedBad] +import non_stub +reveal_type(non_stub.name) + +[file non_stub.py] +from has_getattr import __getattr__ + +[file has_getattr.py] +def __getattr__() -> int: ... + +[out] +tmp/has_getattr.py:1: error: Invalid signature "Callable[[], int]" for "__getattr__" +main:2: note: Revealed type is "builtins.int" + +[builtins fixtures/module.pyi] + +[case testFailedImportFromTwoModules] +import c +import b +[file b.py] +import c + +[out] +-- TODO: it would be better for this to be in the other order +tmp/b.py:1: error: Cannot find implementation or library stub for module named "c" +main:1: error: Cannot find implementation or library stub for module named "c" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testIndirectFromImportWithinCycle1] +import a +[file a.py] +from b import f +from c import x +[file b.py] +from c import y +from a import x +def f() -> None: pass +reveal_type(x) # N: Revealed type is "builtins.str" +[file c.py] +x = str() +y = int() + +[case testIndirectFromImportWithinCycle2] +import a +[file a.py] +from c import y +from b import x +def f() -> None: pass +reveal_type(x) # N: Revealed type is "builtins.str" +[file b.py] +from a import f +from c import x +[file c.py] +x = str() +y = int() + +[case testIndirectFromImportWithinCycleInPackage] +import p.a +[file p/__init__.py] +[file p/a.py] +from p.b import f +from p.c import x +[file p/b.py] +from p.c import y +from p.a import x +def f() -> None: pass +reveal_type(x) # N: Revealed type is "builtins.str" +[file p/c.py] +x = str() +y = int() + +[case testIndirectFromImportWithinCycleInPackageIgnoredInit] +# cmd: mypy -m p.a p.b p.c +# flags: --follow-imports=skip --ignore-missing-imports +[file p/__init__.py] +[file p/a.py] +from p.b import f +from p.c import x +[file p/b.py] +from p.c import y +from p.a import x +def f() -> None: pass +reveal_type(x) # N: Revealed type is "builtins.str" +[file p/c.py] +x = str() +y = int() + +[case testForwardReferenceToListAlias] +x: List[int] +reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" +def f() -> 'List[int]': pass +reveal_type(f) # N: Revealed type is "def () -> builtins.list[builtins.int]" +class A: + y: 'List[str]' + def g(self, x: 'List[int]') -> None: pass +reveal_type(A().y) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type(A().g) # N: Revealed type is "def (x: builtins.list[builtins.int])" +from typing import List +[builtins fixtures/list.pyi] + +[case testIndirectStarImportWithinCycle1] +import a +[file a.py] +from b import f +from c import x +[file b.py] +from c import y +from a import * +def f() -> None: pass +reveal_type(x) # N: Revealed type is "builtins.str" +[file c.py] +x = str() +y = int() + +[case testIndirectStarImportWithinCycle2] +import a +[file a.py] +from c import y +from b import * +def f() -> None: pass +reveal_type(x) # N: Revealed type is "builtins.str" +[file b.py] +from a import f +from c import x +[file c.py] +x = str() +y = int() + +[case testModuleGetattrInit1] +from a import b + +x = b.f() +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testModuleGetattrInit2] +import a.b + +x = a.b.f() +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testModuleGetattrInit3] +import a.b + +x = a.b.f() +[file a/__init__.py] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] +main:1: error: Cannot find implementation or library stub for module named "a.b" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testModuleGetattrInit4] +import a.b.c + +x = a.b.c.f() +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testModuleGetattrInit5] +from a.b import f + +x = f() +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testModuleGetattrInit5a] +from a.b import f + +x = f() +[file a/__init__.pyi] +from types import ModuleType +def __getattr__(attr: str) -> ModuleType: ... +[builtins fixtures/module.pyi] +[out] + + +[case testModuleGetattrInit8] +import a.b.c.d + +x = a.b.c.d.f() +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[file a/b/__init__.pyi] +# empty (i.e. complete subpackage) +[builtins fixtures/module.pyi] +[out] +main:1: error: Cannot find implementation or library stub for module named "a.b.c.d" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "a.b.c" + +[case testModuleGetattrInit8a] +import a.b.c # E: Cannot find implementation or library stub for module named "a.b.c" # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +import a.d # OK +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[file a/b/__init__.pyi] +# empty (i.e. complete subpackage) +[builtins fixtures/module.pyi] + +[case testModuleGetattrInit10] +# flags: --config-file tmp/mypy.ini +import a.b.c # silenced +import a.b.d # error + +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[file a/b/__init__.pyi] +# empty (i.e. complete subpackage) + +[file mypy.ini] +\[mypy] +\[mypy-a.b.c] +ignore_missing_imports = True +[builtins fixtures/module.pyi] +[out] +main:3: error: Cannot find implementation or library stub for module named "a.b.d" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + + +[case testModuleGetattrInit10PyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import a.b.c # silenced +import a.b.d # error + +[file a/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... + +[file a/b/__init__.pyi] +# empty (i.e. complete subpackage) + +[file pyproject.toml] +\[tool.mypy] +\[[tool.mypy.overrides]] +module = 'a.b.c' +ignore_missing_imports = true + +[builtins fixtures/module.pyi] + +[out] +main:3: error: Cannot find implementation or library stub for module named "a.b.d" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + + +[case testMultipleModulesInOverridePyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import a +import b + +[file pyproject.toml] +\[tool.mypy] +\[[tool.mypy.overrides]] +module = ['a', 'b'] +ignore_missing_imports = true + + +[case testIndirectFromImportWithinCycleUsedAsBaseClass] +import a +[file a.py] +from b import f +from c import B +[file b.py] +from c import y +class A(B): pass +reveal_type(A().x) # N: Revealed type is "builtins.int" +from a import B +def f() -> None: pass +[file c.py] +class B: + x: int +x = str() +y = int() + +[case testImportFromReExportInCycleUsingRelativeImport1] +from m import One +reveal_type(One) +[file m/__init__.py] +from .one import One +from .two import Two +reveal_type(One) +[file m/one.py] +class One: + pass +[file m/two.py] +from m import One +reveal_type(One) +x: One +reveal_type(x) + +class Two(One): + pass +y: Two +y = x +x = y +[out] +tmp/m/two.py:2: note: Revealed type is "def () -> m.one.One" +tmp/m/two.py:4: note: Revealed type is "m.one.One" +tmp/m/two.py:9: error: Incompatible types in assignment (expression has type "One", variable has type "Two") +tmp/m/__init__.py:3: note: Revealed type is "def () -> m.one.One" +main:2: note: Revealed type is "def () -> m.one.One" + +[case testImportReExportInCycleUsingRelativeImport2] +from m import One +reveal_type(One) +[file m/__init__.py] +from .one import One +from .two import Two +reveal_type(One) +[file m/one.py] +class One: + pass +[file m/two.py] +import m +reveal_type(m.One) +x: m.One +reveal_type(x) +class Two: + pass +[out] +tmp/m/two.py:2: note: Revealed type is "def () -> m.one.One" +tmp/m/two.py:4: note: Revealed type is "m.one.One" +tmp/m/__init__.py:3: note: Revealed type is "def () -> m.one.One" +main:2: note: Revealed type is "def () -> m.one.One" + +[case testImportReExportedNamedTupleInCycle1] +from m import One +[file m/__init__.py] +from .one import One +from .two import Two +[file m/one.py] +from typing import NamedTuple +class One(NamedTuple): + name: str +[file m/two.py] +import m +x = m.One(name="Foo") +reveal_type(x.name) +class Two: + pass +[builtins fixtures/tuple.pyi] +[out] +tmp/m/two.py:3: note: Revealed type is "builtins.str" + +[case testImportReExportedNamedTupleInCycle2] +from m import One +[file m/__init__.py] +from .one import One +from .two import Two +[file m/one.py] +from typing import NamedTuple +One = NamedTuple('One', [('name', str)]) +[file m/two.py] +import m +x = m.One(name="Foo") +reveal_type(x.name) +class Two: + pass +[builtins fixtures/tuple.pyi] +[out] +tmp/m/two.py:3: note: Revealed type is "builtins.str" + +[case testImportReExportedTypeAliasInCycle] +from m import One +[file m/__init__.py] +from .one import One +from .two import Two +[file m/one.py] +from typing import Union +One = Union[int, str] +[file m/two.py] +import m +x: m.One +reveal_type(x) +class Two: + pass +[out] +tmp/m/two.py:3: note: Revealed type is "Union[builtins.int, builtins.str]" + +[case testImportCycleSpecialCase] +import p +[file p/__init__.py] +from . import a +from . import b +reveal_type(a.foo()) +[file p/a.py] +import p +def foo() -> int: pass +[file p/b.py] +import p + +def run() -> None: + reveal_type(p.a.foo()) +[builtins fixtures/module.pyi] +[out] +tmp/p/b.py:4: note: Revealed type is "builtins.int" +tmp/p/__init__.py:3: note: Revealed type is "builtins.int" + +[case testMissingSubmoduleImportedWithIgnoreMissingImports] +# flags: --ignore-missing-imports +import whatever.works +import a.b +x = whatever.works.f() +y = a.b.f() +[file a/__init__.py] +# empty +[out] + +[case testMissingSubmoduleImportedWithIgnoreMissingImportsStub] +# flags: --ignore-missing-imports --follow-imports=skip +import whatever.works +import a.b +x = whatever.works.f() +y = a.b.f() +xx: whatever.works.C +yy: a.b.C +xx2: whatever.works.C.D +yy2: a.b.C.D +[file a/__init__.pyi] +# empty +[out] + +[case testMissingSubmoduleImportedWithIgnoreMissingImportsNested] +# flags: --ignore-missing-imports +import a.b.c.d +y = a.b.c.d.f() +[file a/__init__.py] +# empty +[file a/b/__init__.py] +# empty +[out] + +[case testModuleGetattrBusted] +from a import A +x: A +reveal_type(x) # N: Revealed type is "Any" +[file a.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testModuleGetattrBusted2] +from a import A +def f(x: A.B) -> None: ... +reveal_type(f) # N: Revealed type is "def (x: Any)" +[file a.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[builtins fixtures/module.pyi] +[out] + +[case testNoGetattrInterference] +import testmod as t +def f(x: t.Cls) -> None: + reveal_type(x) # N: Revealed type is "testmod.Cls" +[file testmod.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +class Cls: ... +[builtins fixtures/module.pyi] +[out] + +[case testFunctionWithDunderName] +def __add__(self) -> int: ... + +[case testFunctionWithReversibleDunderName] +def __radd__(self) -> int: ... + +[case testFunctionWithInPlaceDunderName] +def __iadd__(self) -> int: ... + +-- Tests for PEP 420 namespace packages. + +[case testClassicPackage] +from foo.bar import x +[file foo/__init__.py] +# empty +[file foo/bar.py] +x = 0 + +[case testClassicNotPackage] +# flags: --no-namespace-packages +from foo.bar import x +[file foo/bar.py] +x = 0 +[out] +main:2: error: Cannot find implementation or library stub for module named "foo.bar" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testNamespacePackage] +# flags: --namespace-packages +from foo.bar import x +reveal_type(x) # N: Revealed type is "builtins.int" +[file foo/bar.py] +x = 0 + +[case testNamespacePackageWithMypyPath] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bax import x +from foo.bay import y +from foo.baz import z +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.int" +reveal_type(z) # N: Revealed type is "builtins.int" +[file xx/foo/bax.py] +x = 0 +[file yy/foo/bay.py] +y = 0 +[file foo/baz.py] +z = 0 +[file mypy.ini] +\[mypy] +mypy_path = tmp/xx, tmp/yy + + +[case testNamespacePackageWithMypyPathPyProjectTOML] +# flags: --namespace-packages --config-file tmp/pyproject.toml +from foo.bax import x +from foo.bay import y +from foo.baz import z +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.int" +reveal_type(z) # N: Revealed type is "builtins.int" + +[file xx/foo/bax.py] +x = 0 + +[file yy/foo/bay.py] +y = 0 + +[file foo/baz.py] +z = 0 + +[file pyproject.toml] +\[tool.mypy] +mypy_path = ["tmp/xx", "tmp/yy"] + + +[case testClassicPackageIgnoresEarlierNamespacePackage] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar import y +reveal_type(y) # N: Revealed type is "builtins.int" +[file xx/foo/bar.py] +x = '' +[file yy/foo/bar.py] +y = 0 +[file yy/foo/__init__.py] +[file mypy.ini] +\[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testNamespacePackagePickFirstOnMypyPath] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar import x +reveal_type(x) # N: Revealed type is "builtins.int" +[file xx/foo/bar.py] +x = 0 +[file yy/foo/bar.py] +x = '' +[file mypy.ini] +\[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testNamespacePackageInsideClassicPackage] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar.baz import x +reveal_type(x) # N: Revealed type is "builtins.int" +[file xx/foo/bar/baz.py] +x = '' +[file yy/foo/bar/baz.py] +x = 0 +[file yy/foo/__init__.py] +[file mypy.ini] +\[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testClassicPackageInsideNamespacePackage] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar.baz.boo import x +reveal_type(x) # N: Revealed type is "builtins.int" +[file xx/foo/bar/baz/boo.py] +x = '' +[file xx/foo/bar/baz/__init__.py] +[file yy/foo/bar/baz/boo.py] +x = 0 +[file yy/foo/bar/__init__.py] +[file mypy.ini] +\[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testNamespacePackagePlainImport] +# flags: --namespace-packages +import foo.bar.baz +reveal_type(foo.bar.baz.x) # N: Revealed type is "builtins.int" +[file foo/bar/baz.py] +x = 0 + +[case testModuleGetAttrAssignUnannotated] +import roles +# this should not crash +roles.role = 1 + +[file roles.pyi] +def __getattr__(name): ... + +[case testModuleGetAttrAssignUnannotatedDouble] +import roles +# this also should not crash +roles.role.attr = 1 + +[file roles.pyi] +def __getattr__(name): ... + +[case testModuleGetAttrAssignAny] +import roles +roles.role = 1 + +[file roles.pyi] +from typing import Any +def __getattr__(name: str) -> Any: ... + +[case testModuleGetAttrAssignError] +import roles + +roles.role = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[file roles.pyi] +def __getattr__(name: str) -> str: ... + +[case testModuleGetAttrAssignSubmodule] +import roles +roles.role = 1 +roles.missing.attr = 1 + +[file roles/__init__.pyi] +from typing import Any +def __getattr__(name: str) -> Any: ... + +[case testModuleGetAttrAssignSubmoduleStrict] +import roles +roles.role = 1 # E: Incompatible types in assignment (expression has type "int", variable has type Module) + +[file roles/__init__.pyi] +from types import ModuleType +def __getattr__(name: str) -> ModuleType: ... +[builtins fixtures/module.pyi] + +[case testAlwaysReportMissingAttributesOnFoundModules] +# flags: --ignore-missing-imports +import pack.mod as alias + +x: alias.NonExistent # E: Name "alias.NonExistent" is not defined + +[file pack/__init__.py] +[file pack/mod.py] +class Existent: pass + +[case testModuleAttributeTwoSuggestions] +import m +m.aaaa # E: Module has no attribute "aaaa"; maybe "aaaaa" or "aaa"? + +[file m.py] +aaa: int +aaaaa: int +[builtins fixtures/module.pyi] + +[case testModuleAttributeThreeSuggestions] +import m +m.aaaaa # E: Module has no attribute "aaaaa"; maybe "aaaab", "aaaba", or "aabaa"? + +[file m.py] +aaaab: int +aaaba: int +aabaa: int +[builtins fixtures/module.pyi] + +[case testDirectlyImportTypedDictObjectAtTopLevel] +import foo.bar.custom_dict +from foo import bar +from foo.bar import custom_dict +from foo.bar.custom_dict import CustomDict + +foo.bar.custom_dict.CustomDict(foo="abc", bar="def") +bar.custom_dict.CustomDict(foo="abc", bar="def") +custom_dict.CustomDict(foo="abc", bar="def") +CustomDict(foo="abc", bar="def") + +[file foo/__init__.py] +[file foo/bar/__init__.py] +[file foo/bar/custom_dict.py] +from typing_extensions import TypedDict + +CustomDict = TypedDict( + "CustomDict", + { + "foo": str, + "bar": str, + }, +) +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testNoReExportFromMissingStubs] +from stub import a # E: Module "stub" does not explicitly export attribute "a" +from stub import b +from stub import c # E: Module "stub" has no attribute "c" +from stub import d # E: Module "stub" does not explicitly export attribute "d" + +[file stub.pyi] +from mystery import a, b as b, c as d + +[out] +tmp/stub.pyi:1: error: Cannot find implementation or library stub for module named "mystery" +tmp/stub.pyi:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testPackagePath] +import p +reveal_type(p.__path__) # N: Revealed type is "builtins.list[builtins.str]" +p.m.__path__ # E: "object" has no attribute "__path__" + +[file p/__init__.py] +from . import m as m +[file p/m.py] +[builtins fixtures/list.pyi] + +[case testSpecialModulesNameImplicitAttr] +import typing +import builtins +import abc + +reveal_type(abc.__name__) # N: Revealed type is "builtins.str" +reveal_type(builtins.__name__) # N: Revealed type is "builtins.str" +reveal_type(typing.__name__) # N: Revealed type is "builtins.str" + +[case testSpecialAttrsAreAvaliableInClasses] +class Some: + name = __name__ +reveal_type(Some.name) # N: Revealed type is "builtins.str" + +[case testReExportAllInStub] +from m1 import C +from m1 import D # E: Module "m1" has no attribute "D" +C() +C(1) # E: Too many arguments for "C" +[file m1.pyi] +from m2 import * +[file m2.pyi] +from m3 import * +from m3 import __all__ as __all__ +class D: pass +[file m3.pyi] +from m4 import C as C +__all__ = ['C'] +[file m4.pyi] +class C: pass +[builtins fixtures/list.pyi] + +[case testMypyPathAndPython2Dir] +# flags: --config-file tmp/mypy.ini +from m import f +f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "str" +f('x') + +[file xx/@python2/m.pyi] +def f(x: int) -> None: ... + +[file xx/m.pyi] +def f(x: str) -> None: ... + +[file mypy.ini] +\[mypy] +mypy_path = tmp/xx + +[case testImportCycleSpecialCase2] +import m + +[file m.pyi] +from f import F +class M: pass + +[file f.pyi] +from m import M + +from typing import Generic, TypeVar + +T = TypeVar("T") + +class W(Generic[T]): ... + +class F(M): + A = W[int] + x: C + class C(W[F.A]): ... + +[case testImportCycleSpecialCase3] +import f + +[file m.pyi] +from f import F +class M: pass + +[file f.pyi] +from m import M + +from typing import Generic, TypeVar + +T = TypeVar("T") + +class F(M): + x: C + class C: ... + +[case testLimitLegacyStubErrorVolume] +# flags: --disallow-any-expr --soft-error-limit=5 +import certifi # E: Cannot find implementation or library stub for module named "certifi" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # N: (Skipping most remaining errors due to unresolved imports or missing stubs; fix these first) +certifi.x +certifi.x +certifi.x +certifi.x + +[case testDoNotLimitErrorVolumeIfNotImportErrors] +# flags: --disallow-any-expr --soft-error-limit=5 +def f(): pass +certifi = f() # E: Expression has type "Any" +1() # E: "int" not callable +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +1() # E: "int" not callable + +[case testDoNotLimitImportErrorVolume] +# flags: --disallow-any-expr --soft-error-limit=3 +import xyz1 # E: Cannot find implementation or library stub for module named "xyz1" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +import xyz2 # E: Cannot find implementation or library stub for module named "xyz2" +import xyz3 # E: Cannot find implementation or library stub for module named "xyz3" +import xyz4 # E: Cannot find implementation or library stub for module named "xyz4" + +[case testUnlimitedStubErrorVolume] +# flags: --disallow-any-expr --soft-error-limit=-1 +import certifi # E: Cannot find implementation or library stub for module named "certifi" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" +certifi.x # E: Expression has type "Any" + +[case testIgnoreErrorFromMissingStubs1] +# flags: --config-file tmp/pyproject.toml +import certifi +from foobar1 import x +import foobar2 +[file pyproject.toml] +\[tool.mypy] +ignore_missing_imports = true +\[[tool.mypy.overrides]] +module = "certifi" +ignore_missing_imports = true +\[[tool.mypy.overrides]] +module = "foobar1" +ignore_missing_imports = true + +[case testIgnoreErrorFromMissingStubs2] +# flags: --config-file tmp/pyproject.toml +import certifi +from foobar1 import x +import foobar2 # E: Cannot find implementation or library stub for module named "foobar2" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +[file pyproject.toml] +\[tool.mypy] +ignore_missing_imports = false +\[[tool.mypy.overrides]] +module = "certifi" +ignore_missing_imports = true +\[[tool.mypy.overrides]] +module = "foobar1" +ignore_missing_imports = true + +[case testIgnoreErrorFromGoogleCloud] +# flags: --ignore-missing-imports +import google.cloud +from google.cloud import x + +[case testErrorFromGoogleCloud] +import google.cloud # E: Cannot find implementation or library stub for module named "google.cloud" \ + # E: Cannot find implementation or library stub for module named "google" +from google.cloud import x +import google.non_existent # E: Cannot find implementation or library stub for module named "google.non_existent" +from google.non_existent import x + +import google.cloud.ndb # E: Library stubs not installed for "google.cloud.ndb" \ + # N: Hint: "python3 -m pip install types-google-cloud-ndb" \ + # N: (or run "mypy --install-types" to install all missing stub packages) \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +from google.cloud import ndb + +[case testMissingSubmoduleOfInstalledStubPackage] +import bleach.exists +import bleach.xyz # E: Cannot find implementation or library stub for module named "bleach.xyz" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +from bleach.abc import fgh # E: Cannot find implementation or library stub for module named "bleach.abc" +[file bleach/__init__.pyi] +[file bleach/exists.pyi] + +[case testMissingSubmoduleOfInstalledStubPackageIgnored] +# flags: --ignore-missing-imports +import bleach.xyz +from bleach.abc import fgh +[file bleach/__init__.pyi] + +[case testCyclicUndefinedImportWithName] +import a +[file a.py] +from b import no_such_export +[file b.py] +from a import no_such_export # E: Module "a" has no attribute "no_such_export" + +[case testCyclicUndefinedImportWithStar1] +import a +[file a.py] +from b import no_such_export +[file b.py] +from a import * +[out] +tmp/b.py:1: error: Cannot resolve name "no_such_export" (possible cyclic definition) +tmp/a.py:1: error: Module "b" has no attribute "no_such_export" + +[case testCyclicUndefinedImportWithStar2] +import a +[file a.py] +from b import no_such_export +[file b.py] +from c import * +[file c.py] +from a import * +[out] +tmp/c.py:1: error: Cannot resolve name "no_such_export" (possible cyclic definition) +tmp/b.py:1: error: Cannot resolve name "no_such_export" (possible cyclic definition) +tmp/a.py:1: error: Module "b" has no attribute "no_such_export" + +[case testCyclicUndefinedImportWithStar3] +import test1 +[file test1.py] +from dir1 import * +[file dir1/__init__.py] +from .test2 import * +[file dir1/test2.py] +from test1 import aaaa # E: Module "test1" has no attribute "aaaa" + +[case testIncompatibleOverrideFromCachedModuleIncremental] +import b +[file a.py] +class Foo: + def frobnicate(self, x: str, *args, **kwargs): pass +[file b.py] +from a import Foo +class Bar(Foo): + def frobnicate(self) -> None: pass +[file b.py.2] +from a import Foo +class Bar(Foo): + def frobnicate(self, *args: int) -> None: pass +[file b.py.3] +from a import Foo +class Bar(Foo): + def frobnicate(self, *args: int) -> None: pass # type: ignore[override] # I know +[builtins fixtures/dict.pyi] +[out1] +tmp/b.py:3: error: Signature of "frobnicate" incompatible with supertype "Foo" +tmp/b.py:3: note: Superclass: +tmp/b.py:3: note: def frobnicate(self, x: str, *args: Any, **kwargs: Any) -> Any +tmp/b.py:3: note: Subclass: +tmp/b.py:3: note: def frobnicate(self) -> None +[out2] +tmp/b.py:3: error: Signature of "frobnicate" incompatible with supertype "Foo" +tmp/b.py:3: note: Superclass: +tmp/b.py:3: note: def frobnicate(self, x: str, *args: Any, **kwargs: Any) -> Any +tmp/b.py:3: note: Subclass: +tmp/b.py:3: note: def frobnicate(self, *args: int) -> None diff --git a/test-data/unit/check-multiple-inheritance.test b/test-data/unit/check-multiple-inheritance.test new file mode 100644 index 000000000000..d03f2e35e1c4 --- /dev/null +++ b/test-data/unit/check-multiple-inheritance.test @@ -0,0 +1,708 @@ +-- Test cases for multiple inheritance. +-- +-- Related: check-abstract.test + + +-- No name collisions +-- ------------------ + + +[case testSimpleMultipleInheritanceAndMethods] +import typing +class A: + def f(self, x: int) -> None: pass +class B: + def g(self, x: str) -> None: pass +class C(A, B): pass +c = C() +c.f(1) +c.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +c.g('') +c.g(1) # E: Argument 1 to "g" of "B" has incompatible type "int"; expected "str" + +[case testSimpleMultipleInheritanceAndMethods2] +import typing +class A: + def f(self, x: int) -> None: pass +class B: + def g(self, x): pass +class C(A, B): pass +c = C() +c.f(1) +c.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +c.g('') +c.g(1) + +[case testSimpleMultipleInheritanceAndInstanceVariables] +import typing +class A: + def f(self) -> None: + self.x = 1 +class B: + def g(self) -> None: + self.y = '' +class C(A, B): pass +c = C() +c.x = 1 +c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +c.y = '' +c.y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testSimpleMultipleInheritanceAndInstanceVariableInClassBody] +import typing +class A: + x = 1 +class B: + y = '' +class C(A, B): pass +c = C() +c.x = 1 +c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +c.y = '' +c.y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testSimpleMultipleInheritanceAndClassVariable] +import typing +class A: + x = 1 +class B: + y = '' +class C(A, B): pass +C.x = 1 +C.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +C.y = '' +C.y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + +-- Name collisions +-- --------------- + + +[case testMethodNameCollisionInMultipleInheritanceWithValidSigs] +import typing +class A: + def f(self, x: int) -> None: pass +class B: + def f(self, x: int) -> None: pass +class C(A, B): pass +c = C() +c.f(1) +c.f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testInstanceVarNameOverlapInMultipleInheritanceWithCompatibleTypes] +import typing +class A: + def f(self) -> None: + self.x = 1 +class B: + def g(self) -> None: + self.x = 1 +class C(A, B): pass +c = C() +c.x = 1 +c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testClassVarNameOverlapInMultipleInheritanceWithCompatibleTypes] +import typing +class A: + x = 1 +class B: + x = 1 +class C(A, B): pass +c = C() +c.x = 1 +c.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +C.x = 1 +C.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testMethodNameCollisionInMultipleInheritanceWithIncompatibleSigs] +import typing +class A: + def f(self, x: int) -> None: pass +class B: + def f(self, x: str) -> None: pass +class C(A, B): pass +[out] +main:6: error: Definition of "f" in base class "A" is incompatible with definition in base class "B" + +[case testMethodNameCollisionInMultipleInheritanceWithIncompatibleSigs2] +import typing +class A: + def f(self, x: int) -> None: pass +class B: + def f(self, x, y): pass +class C(A, B): pass +class D(B, A): pass +[out] +main:6: error: Definition of "f" in base class "A" is incompatible with definition in base class "B" +main:7: error: Definition of "f" in base class "B" is incompatible with definition in base class "A" + + +[case testMethodOverridingWithBothDynamicallyAndStaticallyTypedMethods] +class A: + def f(self) -> int: pass +class B: + def f(self): pass +class C(B, A): pass +class D(A, B): pass +[out] + +[case testInstanceVarNameOverlapInMultipleInheritanceWithInvalidTypes] +import typing +class A: + def f(self) -> None: + self.x = 1 +class B: + def g(self) -> None: + self.x = '' +class C(A, B): pass +[out] +main:8: error: Definition of "x" in base class "A" is incompatible with definition in base class "B" + +[case testClassVarNameOverlapInMultipleInheritanceWithInvalidTypes] +import typing +class A: + x = 1 +class B: + x = '' +class C(A, B): pass +[out] +main:6: error: Definition of "x" in base class "A" is incompatible with definition in base class "B" + +[case testMethodOverlapsWithClassVariableInMultipleInheritance] +from typing import Callable +class A: + def f(self) -> None: pass +class B: + f = '' +class C(A, B): pass +[out] +main:6: error: Definition of "f" in base class "A" is incompatible with definition in base class "B" + +[case testMethodOverlapsWithInstanceVariableInMultipleInheritance] +from typing import Callable +class A: + def f(self) -> None: pass +class B: + def g(self) -> None: + self.f = '' +class C(A, B): pass +[out] +main:7: error: Definition of "f" in base class "A" is incompatible with definition in base class "B" + +[case testMultipleInheritanceAndInit] +import typing +class A: + def __init__(self, x: int) -> None: pass +class B: + def __init__(self) -> None: pass +class C(A, B): pass + +[case testMultipleInheritanceAndDifferentButCompatibleSignatures] +class A: + def clear(self): pass + +class B: + def clear(self, x=None): pass + +class C(B, A): pass +class D(A, B): pass +[out] +main:8: error: Definition of "clear" in base class "A" is incompatible with definition in base class "B" + + +-- Special cases +-- ------------- + + +[case testGenericInheritanceAndOverridingWithMultipleInheritance] +from typing import Generic, TypeVar +T = TypeVar('T') +class G(Generic[T]): + def f(self, s: int) -> 'G[T]': pass +class A(G[int]): + def f(self, s: int) -> 'A': pass +class B(A, int): pass + +[case testCannotDetermineTypeInMultipleInheritance] +from typing import Callable, TypeVar +T = TypeVar('T') +class A(B, C): + pass +class B: + @dec + def f(self): pass +class C: + @dec + def f(self): pass +def dec(f: Callable[..., T]) -> Callable[..., T]: + return f +[out] +main:3: error: Cannot determine type of "f" in base class "B" +main:3: error: Cannot determine type of "f" in base class "C" + +[case testMultipleInheritance_NestedClassesWithSameName] +class Mixin1: + class Meta: + pass +class Mixin2: + class Meta: + pass +class A(Mixin1, Mixin2): + pass +[out] +main:7: error: Definition of "Meta" in base class "Mixin1" is incompatible with definition in base class "Mixin2" + +[case testMultipleInheritance_NestedClassesWithSameNameCustomMetaclass] +class Metaclass(type): + pass +class Mixin1: + class Meta(metaclass=Metaclass): + pass +class Mixin2: + class Meta(metaclass=Metaclass): + pass +class A(Mixin1, Mixin2): + pass +[out] +main:9: error: Definition of "Meta" in base class "Mixin1" is incompatible with definition in base class "Mixin2" + +[case testMultipleInheritance_NestedClassesWithSameNameOverloadedNew] +from mixins import Mixin1, Mixin2 +class A(Mixin1, Mixin2): + pass +[file mixins.py] +class Mixin1: + class Meta: + pass +class Mixin2: + class Meta: + pass +[file mixins.pyi] +from typing import overload, Any, Mapping, Dict +class Mixin1: + class Meta: + @overload + def __new__(cls, *args, **kwargs: None) -> Mixin1.Meta: + pass + @overload + def __new__(cls, *args, **kwargs: Dict[str, Any]) -> Mixin1.Meta: + pass +class Mixin2: + class Meta: + pass +[builtins fixtures/dict.pyi] +[out] +main:2: error: Definition of "Meta" in base class "Mixin1" is incompatible with definition in base class "Mixin2" + +[case testMultipleInheritance_NestedClassAndAttrHaveSameName] +class Mixin1: + class Nested1: + pass +class Mixin2: + Nested1: str +class A(Mixin1, Mixin2): + pass +[out] +main:6: error: Definition of "Nested1" in base class "Mixin1" is incompatible with definition in base class "Mixin2" + +[case testMultipleInheritance_NestedClassAndFunctionHaveSameName] +class Mixin1: + class Nested1: + pass +class Mixin2: + def Nested1(self) -> str: + pass +class A(Mixin1, Mixin2): + pass +[out] +main:7: error: Definition of "Nested1" in base class "Mixin1" is incompatible with definition in base class "Mixin2" + +[case testMultipleInheritance_NestedClassAndRefToOtherClass] +class Outer: + pass +class Mixin1: + class Nested1: + pass +class Mixin2: + Nested1 = Outer +class A(Mixin2, Mixin1): + pass +[out] +main:8: error: Definition of "Nested1" in base class "Mixin2" is incompatible with definition in base class "Mixin1" + +[case testMultipleInheritance_ReferenceToSubclassesFromSameMRO] +class A: + def __init__(self, arg: str) -> None: + pass +class B(A): + pass +class Base1: + NestedVar = A +class Base2: + NestedVar = B +class Combo(Base2, Base1): ... +[out] + +[case testMultipleInheritance_ReferenceToSubclassesFromSameMROCustomMetaclass] +class Metaclass(type): + pass +class A(metaclass=Metaclass): + pass +class B(A): + pass +class Base1: + NestedVar = A +class Base2: + NestedVar = B +class Combo(Base2, Base1): ... +[out] + +[case testMultipleInheritance_ReferenceToSubclassesFromSameMROOverloadedNew] +from mixins import A, B +class Base1: + NestedVar = A +class Base2: + NestedVar = B +class Combo(Base2, Base1): ... +[file mixins.py] +class A: + pass +class B(A): + pass +[file mixins.pyi] +from typing import overload, Dict, Any +class A: + @overload + def __new__(cls, *args, **kwargs: None) -> A: + pass + @overload + def __new__(cls, *args, **kwargs: Dict[str, Any]) -> A: + pass +class B: + pass +[builtins fixtures/dict.pyi] +[out] +main:6: error: Definition of "NestedVar" in base class "Base2" is incompatible with definition in base class "Base1" + +[case testMultipleInheritance_ReferenceToGenericClasses] +from typing import TypeVar, Generic +T = TypeVar('T') +class Generic1(Generic[T]): + pass +class Generic2(Generic[T]): + pass +class Base1: + Nested = Generic1 +class Base2: + Nested = Generic2 +class A(Base1, Base2): + pass +[out] +main:11: error: Definition of "Nested" in base class "Base1" is incompatible with definition in base class "Base2" + +[case testMultipleInheritance_GenericSubclasses_SuperclassFirst] +from typing import TypeVar, Generic +T = TypeVar('T') +class ParentGeneric(Generic[T]): + pass +class ChildGeneric(ParentGeneric[T]): + pass +class Base1: + Nested = ParentGeneric +class Base2: + Nested = ChildGeneric +class A(Base1, Base2): + pass +[out] +main:11: error: Definition of "Nested" in base class "Base1" is incompatible with definition in base class "Base2" + +[case testMultipleInheritance_GenericSubclasses_SubclassFirst] +from typing import TypeVar, Generic +T = TypeVar('T') +class ParentGeneric(Generic[T]): + pass +class ChildGeneric(ParentGeneric[T]): + pass +class Base1: + Nested = ParentGeneric +class Base2: + Nested = ChildGeneric +class A(Base2, Base1): + pass +[out] + +[case testMultipleInheritance_RefersToNamedTuples] +from typing import NamedTuple +class NamedTuple1: + attr1: int +class NamedTuple2: + attr2: int +class Base1: + Nested = NamedTuple1 +class Base2: + Nested = NamedTuple2 +class A(Base1, Base2): + pass +[out] +main:10: error: Definition of "Nested" in base class "Base1" is incompatible with definition in base class "Base2" + +[case testMultipleInheritance_NestedVariableRefersToSuperlassUnderSubclass] +class A: + def __init__(self, arg: str) -> None: + pass +class B(A): + pass +class Base1: + NestedVar = B +class Base2: + NestedVar = A +class Combo(Base2, Base1): ... +[out] +main:10: error: Definition of "NestedVar" in base class "Base2" is incompatible with definition in base class "Base1" + +[case testMultipleInheritance_NestedVariableOverriddenWithCompatibleType] +from typing import TypeVar, Generic +T = TypeVar('T', covariant=True) +class GenericBase(Generic[T]): + pass +class Base1: + Nested: GenericBase['Base1'] +class Base2: + Nested: GenericBase['Base2'] +class A(Base1, Base2): + Nested: GenericBase['A'] +[out] + +[case testMultipleInheritance_NestedVariableOverriddenWithIncompatibleType1] +from typing import TypeVar, Generic +T = TypeVar('T', covariant=True) +class GenericBase(Generic[T]): + pass +class Base1: + Nested: GenericBase['Base1'] +class Base2: + Nested: GenericBase['Base2'] +class A(Base1, Base2): + Nested: GenericBase['Base1'] +[out] +main:10: error: Incompatible types in assignment (expression has type "GenericBase[Base1]", base class "Base2" defined the type as "GenericBase[Base2]") + +[case testMultipleInheritance_NestedVariableOverriddenWithIncompatibleType2] +from typing import TypeVar, Generic +T = TypeVar('T', covariant=True) +class GenericBase(Generic[T]): + pass +class Base1: + Nested: GenericBase['Base1'] +class Base2: + Nested: GenericBase['Base2'] +class A(Base1, Base2): + Nested: GenericBase['Base2'] +[out] +main:10: error: Incompatible types in assignment (expression has type "GenericBase[Base2]", base class "Base1" defined the type as "GenericBase[Base1]") + +[case testMultipleInheritance_NestedVariableOverriddenWithCompatibleType2] +from typing import TypeVar, Generic +T = TypeVar('T', covariant=True) +class GenericBase(Generic[T]): + pass +class Base1: + Nested: GenericBase['Base1'] +class Base2: + Nested: GenericBase['Base1'] +class A(Base1, Base2): + Nested: GenericBase['Base1'] +[out] + +[case testMultipleInheritance_MethodDefinitionsCompatibleWithOverride] +from typing import TypeVar, Union +_T = TypeVar('_T') + +class Flag: + def __or__(self: _T, other: _T) -> _T: ... + +# int defines __or__ as: +# def __or__(self, n: int) -> int: ... +class IntFlag(int, Flag): + def __or__(self: _T, other: Union[int, _T]) -> _T: ... +[out] + +[case testMultipleInheritance_MethodDefinitionsIncompatibleOverride] +from typing import TypeVar, Union +_T = TypeVar('_T') + +class Flag: + def __or__(self: _T, other: _T) -> _T: ... + +class IntFlag(int, Flag): + def __or__(self: _T, other: str) -> _T: ... +[out] +main:8: error: Argument 1 of "__or__" is incompatible with supertype "Flag"; supertype defines the argument type as "IntFlag" +main:8: note: This violates the Liskov substitution principle +main:8: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + +[case testMultipleInheritance_MethodDefinitionsCompatibleNoOverride] +from typing import TypeVar, Union +_T = TypeVar('_T') + +class Flag: + def __or__(self: _T, other: _T) -> _T: ... + +class IntFlag(int, Flag): + pass +[out] + +[case testMultipleInheritance_MethodsReturningSelfCompatible] +class A(object): + def x(self) -> 'A': + return self + +class B(object): + def x(self) -> 'B': + return self + +class C(A, B): + def x(self) -> 'C': + return self + +[case testMultipleInheritance_MethodsReturningSelfIncompatible] +class A(object): + def x(self) -> 'A': + return self + +class B(object): + def x(self) -> 'B': + return self + +class C(A, B): # E: Definition of "x" in base class "A" is incompatible with definition in base class "B" + pass + +[case testNestedVariableRefersToSubclassOfAnotherNestedClass] +class Mixin1: + class Meta: + pass +class Outer(Mixin1.Meta): + pass +class Mixin2: + NestedVar = Outer +class Combo(Mixin2, Mixin1): ... +[out] + +[case testNestedVariableRefersToCompletelyDifferentClasses] +class A: + pass +class B: + pass +class Base1: + NestedVar = A +class Base2: + NestedVar = B +class Combo(Base2, Base1): ... +[out] +main:9: error: Definition of "NestedVar" in base class "Base2" is incompatible with definition in base class "Base1" + +[case testDoNotFailIfBothNestedClassesInheritFromAny] +from typing import Any +class Mixin1: + class Meta(Any): + pass +class Mixin2: + class Meta(Any): + pass +class A(Mixin1, Mixin2): + pass +[out] + +[case testDoNotFailIfOneOfNestedClassesIsOuterInheritedFromAny] +from typing import Any +class Outer(Any): + pass +class Mixin1: + Meta = Outer +class Mixin2: + class Meta(Any): + pass +class A(Mixin1, Mixin2): + pass +[out] + +[case testGenericMultipleOverrideRemap] +from typing import TypeVar, Generic, Tuple + +K = TypeVar('K') +V = TypeVar('V') +T = TypeVar('T') + +class ItemsView(Generic[K, V]): + def __iter__(self) -> Tuple[K, V]: ... + +class Sequence(Generic[T]): + def __iter__(self) -> T: ... + +# Override compatible between bases. +class OrderedItemsView(ItemsView[K, V], Sequence[Tuple[K, V]]): + def __iter__(self) -> Tuple[K, V]: ... + +class OrderedItemsViewDirect(ItemsView[K, V], Sequence[Tuple[K, V]]): + pass +[builtins fixtures/tuple.pyi] + +[case testGenericMultipleOverrideReplace] +from typing import TypeVar, Generic, Union + +T = TypeVar('T') + +class A(Generic[T]): + def foo(self, x: T) -> None: ... + +class B(A[T]): ... + +class C1: + def foo(self, x: str) -> None: ... + +class C2: + def foo(self, x: Union[str, int]) -> None: ... + +class D1(B[str], C1): ... +class D2(B[Union[int, str]], C2): ... +class D3(C2, B[str]): ... +class D4(B[str], C2): ... # E: Definition of "foo" in base class "A" is incompatible with definition in base class "C2" + + +[case testMultipleInheritanceOverridingOfFunctionsWithCallableInstances] +from typing import Any, Callable + +def dec1(f: Callable[[Any, int], None]) -> Callable[[Any, int], None]: ... + +class F: + def __call__(self, x: int) -> None: ... + +def dec2(f: Callable[[Any, int], None]) -> F: ... + +class B1: + def f(self, x: int) -> None: ... + +class B2: + @dec1 + def f(self, x: int) -> None: ... + +class B3: + @dec2 + def f(self, x: int) -> None: ... + +class B4: + f = F() + +class C12(B1, B2): ... +class C13(B1, B3): ... # E: Definition of "f" in base class "B1" is incompatible with definition in base class "B3" +class C14(B1, B4): ... # E: Definition of "f" in base class "B1" is incompatible with definition in base class "B4" +class C21(B2, B1): ... +class C23(B2, B3): ... # E: Definition of "f" in base class "B2" is incompatible with definition in base class "B3" +class C24(B2, B4): ... # E: Definition of "f" in base class "B2" is incompatible with definition in base class "B4" +class C31(B3, B1): ... +class C32(B3, B2): ... +class C34(B3, B4): ... +class C41(B4, B1): ... +class C42(B4, B2): ... +class C43(B4, B3): ... diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test new file mode 100644 index 000000000000..df2c7ffc8067 --- /dev/null +++ b/test-data/unit/check-namedtuple.test @@ -0,0 +1,1443 @@ +[case testNamedTupleUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', 'x y') +x: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range +[builtins fixtures/tuple.pyi] + +[case testNamedTupleWithTupleFieldNamesUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', ('x', 'y')) +x: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range +[builtins fixtures/tuple.pyi] + +[case testNamedTupleInvalidFields] +from collections import namedtuple + +X = namedtuple('X', 'x, _y') # E: "namedtuple()" field name "_y" starts with an underscore +Y = namedtuple('Y', ['x', '1']) # E: "namedtuple()" field name "1" is not a valid identifier +Z = namedtuple('Z', ['x', 'def']) # E: "namedtuple()" field name "def" is a keyword +A = namedtuple('A', ['x', 'x']) # E: "namedtuple()" has duplicate field name "x" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleAccessingAttributes] +from collections import namedtuple + +X = namedtuple('X', 'x y') +x: X +x.x +x.y +x.z # E: "X" has no attribute "z" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleClassInStub] +import foo + +[file foo.pyi] +from typing import NamedTuple + +class A(NamedTuple): + x: int +[builtins fixtures/tuple.pyi] + +[case testNamedTupleAttributesAreReadOnly] +from collections import namedtuple + +X = namedtuple('X', 'x y') +x: X +x.x = 5 # E: Property "x" defined in "X" is read-only +x.y = 5 # E: Property "y" defined in "X" is read-only +x.z = 5 # E: "X" has no attribute "z" + +class A(X): pass +a: A +a.x = 5 # E: Property "x" defined in "X" is read-only +a.y = 5 # E: Property "y" defined in "X" is read-only +-- a.z = 5 # not supported yet +[builtins fixtures/tuple.pyi] + + +[case testTypingNamedTupleAttributesAreReadOnly] +from typing import NamedTuple +from typing_extensions import Protocol + +class HasX(Protocol): + x: str + +class A(NamedTuple): + x: str + +a: HasX = A("foo") +a.x = "bar" +[builtins fixtures/tuple.pyi] +[out] +main:10: error: Incompatible types in assignment (expression has type "A", variable has type "HasX") +main:10: note: Protocol member HasX.x expected settable variable, got read-only attribute + + +[case testNamedTupleCreateWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', 'x y') +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Missing positional argument "y" in call to "X" +x = X(1, 2, 3) # E: Too many arguments for "X" +[builtins fixtures/tuple.pyi] + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', 'x y') +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleCreateAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', 'x y') +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +[builtins fixtures/tuple.pyi] + +[case testNamedTupleAdditionalArgs] +from collections import namedtuple + +A = namedtuple('A', 'a b') +B = namedtuple('B', 'a b', rename=1) +C = namedtuple('C', 'a b', rename='not a bool') +D = namedtuple('D', 'a b', unrecognized_arg=False) +E = namedtuple('E', 'a b', 0) + +[builtins fixtures/bool.pyi] + +[out] +main:4: error: Boolean literal expected as the "rename" argument to namedtuple() +main:5: error: Boolean literal expected as the "rename" argument to namedtuple() +main:5: error: Argument "rename" to "namedtuple" has incompatible type "str"; expected "int" +main:6: error: Unexpected keyword argument "unrecognized_arg" for "namedtuple" +/test-data/unit/lib-stub/collections.pyi:3: note: "namedtuple" defined here +main:7: error: Too many positional arguments for "namedtuple" + +[case testNamedTupleDefaults] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y'], defaults=(1,)) + +X() # E: Missing positional argument "x" in call to "X" +X(0) # ok +X(0, 1) # ok +X(0, 1, 2) # E: Too many arguments for "X" + +Y = namedtuple('Y', ['x', 'y'], defaults=(1, 2, 3)) # E: Too many defaults given in call to "namedtuple()" +Z = namedtuple('Z', ['x', 'y'], defaults='not a tuple') # E: List or tuple literal expected as the defaults argument to namedtuple() # E: Argument "defaults" to "namedtuple" has incompatible type "str"; expected "Optional[Iterable[Any]]" + +[builtins fixtures/list.pyi] + +[case testNamedTupleRename] +from collections import namedtuple + +X = namedtuple('X', ['abc', 'def'], rename=False) # E: "namedtuple()" field name "def" is a keyword +Y = namedtuple('Y', ['x', 'x', 'def', '42', '_x'], rename=True) +y = Y(x=0, _1=1, _2=2, _3=3, _4=4) +reveal_type(y.x) # N: Revealed type is "Any" +reveal_type(y._1) # N: Revealed type is "Any" +reveal_type(y._2) # N: Revealed type is "Any" +reveal_type(y._3) # N: Revealed type is "Any" +reveal_type(y._4) # N: Revealed type is "Any" +y._0 # E: "Y" has no attribute "_0" +y._5 # E: "Y" has no attribute "_5" +y._x # E: "Y" has no attribute "_x" + +[builtins fixtures/list.pyi] + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +if int(): + x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[targets __main__, __main__.N.__new__, __main__.N._asdict, __main__.N._make, __main__.N._replace] +[builtins fixtures/tuple.pyi] + + +[case testNamedTupleWithTupleFieldNamesWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', (('a', int), + ('b', str))) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +if int(): + x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] + + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument "b" to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) +[builtins fixtures/tuple.pyi] + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +if int(): + s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + i, s = x +if int(): + s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +if int(): + s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + i, s = x +if int(): + s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + + +[case testNamedTuplesTwoAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass +[builtins fixtures/tuple.pyi] + + +[case testNamedTuplesTwoAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass +[builtins fixtures/tuple.pyi] + + +[case testNamedTupleSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + if int(): + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[builtins fixtures/tuple.pyi] + + +[out] + +[case testNamedTupleTypeReferenceToClassDerivedFrom] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + if int(): + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[builtins fixtures/tuple.pyi] + +[out] + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t: Tuple[int, str] +if int(): + b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +if int(): + b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +if int(): + t = a +if int(): + t = (1, '') +if int(): + t = b +if int(): + a = b +[builtins fixtures/tuple.pyi] + + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +if int(): + a = l[0] +(i,) = l[0] +if int(): + i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +if int(): + l = [A(1)] +if int(): + a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.pyi] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "Type[MyNamedTuple]" has no attribute "x" + +[builtins fixtures/list.pyi] + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/tuple.pyi] + + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b + +[builtins fixtures/property.pyi] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x: X +reveal_type(x._asdict()) # N: Revealed type is "builtins.dict[builtins.str, Any]" + +[builtins fixtures/dict.pyi] + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x: X +reveal_type(x._replace()) # N: Revealed type is "Tuple[Any, Any, fallback=__main__.X]" +x._replace(y=5) +x._replace(x=3) +x._replace(x=3, y=5) +x._replace(z=5) # E: Unexpected keyword argument "z" for "_replace" of "X" +x._replace(5) # E: Too many positional arguments for "_replace" of "X" + +[builtins fixtures/list.pyi] + +[case testNamedTupleReplaceAsClass] +# flags: --no-strict-optional +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +X._replace(x, x=1, y=2) +X._replace(x=1, y=2) # E: Missing positional argument "_self" in call to "_replace" of "X" + +[builtins fixtures/list.pyi] + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x: X +reveal_type(x._replace()) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.X]" +x._replace(x=5) +x._replace(y=5) # E: Argument "y" to "_replace" of "X" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleMake] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +reveal_type(X._make([5, 'a'])) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.X]" +X._make('a b') # E: Argument 1 to "_make" of "X" has incompatible type "str"; expected "Iterable[Any]" + +-- # FIX: not a proper class method +-- x: X +-- reveal_type(x._make([5, 'a'])) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.X]" +-- x._make('a b') # E: Argument 1 to "_make" of "X" has incompatible type "str"; expected Iterable[Any] + +[builtins fixtures/list.pyi] + +[case testNamedTupleFields] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +reveal_type(X._fields) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleSource] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +reveal_type(X._source) # N: Revealed type is "builtins.str" +x: X +reveal_type(x._source) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleUnit] +from typing import NamedTuple + +X = NamedTuple('X', []) +x = X() # type: X +x._replace() +x._fields[0] # E: Tuple index out of range +[builtins fixtures/tuple.pyi] + +[case testNamedTupleJoinNamedTuple] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +Y = NamedTuple('Y', [('x', int), ('y', str)]) +reveal_type([X(3, 'b'), Y(1, 'a')]) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" + +[builtins fixtures/list.pyi] + +[case testNamedTupleJoinTuple] +from typing import NamedTuple, Tuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +reveal_type([(3, 'b'), X(1, 'a')]) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" +reveal_type([X(1, 'a'), (3, 'b')]) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" + +[builtins fixtures/list.pyi] + +[case testNamedTupleFieldTypes] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +reveal_type(X._field_types) # N: Revealed type is "builtins.dict[builtins.str, Any]" +x: X +reveal_type(x._field_types) # N: Revealed type is "builtins.dict[builtins.str, Any]" + +[builtins fixtures/dict.pyi] + +[case testNamedTupleAndOtherSuperclass] +from typing import NamedTuple + +class A: pass +def f(x: A) -> None: pass + +class B(NamedTuple('B', []), A): pass +f(B()) +x: A +if int(): + x = B() + +# Sanity check: fail if baseclass does not match +class C: pass +def g(x: C) -> None: pass +class D(NamedTuple('D', []), A): pass + +g(D()) # E: Argument 1 to "g" has incompatible type "D"; expected "C" +y: C +if int(): + y = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C") +[builtins fixtures/tuple.pyi] + +[case testNamedTupleSelfTypeMethod] +from typing import TypeVar, NamedTuple + +T = TypeVar('T', bound='A') + +class A(NamedTuple('A', [('x', str)])): + def member(self: T) -> T: + return self + +class B(A): + pass + +a: A +a = A('').member() +b: B +b = B('').member() +a = B('') +a = B('').member() +[builtins fixtures/tuple.pyi] + +[case testNamedTupleSelfTypeReplace] +from typing import NamedTuple, TypeVar +A = NamedTuple('A', [('x', str)]) +reveal_type(A('hello')._replace(x='')) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.A]" +a: A +a = A('hello')._replace(x='') + +class B(A): + pass + +reveal_type(B('hello')._replace(x='')) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.B]" +b: B +b = B('hello')._replace(x='') +[builtins fixtures/tuple.pyi] + +[case testNamedTupleSelfTypeMake] +from typing import NamedTuple, TypeVar +A = NamedTuple('A', [('x', str)]) +reveal_type(A._make([''])) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.A]" +a = A._make(['']) # type: A + +class B(A): + pass + +reveal_type(B._make([''])) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.B]" +b = B._make(['']) # type: B + +[builtins fixtures/list.pyi] + +[case testNamedTupleIncompatibleRedefinition] +from typing import NamedTuple +class Crash(NamedTuple): + count: int # E: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[int, ...], object], int]") +[builtins fixtures/tuple.pyi] + +[case testNamedTupleInClassNamespace] +# https://github.com/python/mypy/pull/2553#issuecomment-266474341 +from typing import NamedTuple +class C: + def f(self): + A = NamedTuple('A', [('x', int)]) + def g(self): + A = NamedTuple('A', [('y', int)]) +C.A # E: "Type[C]" has no attribute "A" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleInFunction] +from typing import NamedTuple +def f() -> None: + A = NamedTuple('A', [('x', int)]) +A # E: Name "A" is not defined +[builtins fixtures/tuple.pyi] + +[case testNamedTupleForwardAsUpperBound] +# flags: --disable-error-code=used-before-def +from typing import NamedTuple, TypeVar, Generic +T = TypeVar('T', bound='M') +class G(Generic[T]): + x: T + +yb: G[int] # E: Type argument "int" of "G" must be a subtype of "M" +yg: G[M] +reveal_type(G[M]().x.x) # N: Revealed type is "builtins.int" +reveal_type(G[M]().x[0]) # N: Revealed type is "builtins.int" + +M = NamedTuple('M', [('x', int)]) +[builtins fixtures/tuple.pyi] +[out] + +[case testNamedTupleWithImportCycle] +import a +[file a.py] +from collections import namedtuple +from b import f + +N = namedtuple('N', 'a') + +class X(N): pass +[file b.py] +import a + +def f(x: a.X) -> None: + reveal_type(x) + x = a.X(1) + reveal_type(x) +[builtins fixtures/tuple.pyi] +[out] +tmp/b.py:4: note: Revealed type is "Tuple[Any, fallback=a.X]" +tmp/b.py:6: note: Revealed type is "Tuple[Any, fallback=a.X]" + +[case testNamedTupleWithImportCycle2] +import a +[file a.py] +from collections import namedtuple +from b import f + +N = namedtuple('N', 'a') +[file b.py] +import a + +def f(x: a.N) -> None: + reveal_type(x) + if int(): + x = a.N(1) + reveal_type(x) +[builtins fixtures/tuple.pyi] +[out] +tmp/b.py:4: note: Revealed type is "Tuple[Any, fallback=a.N]" +tmp/b.py:7: note: Revealed type is "Tuple[Any, fallback=a.N]" + +[case testSimpleSelfReferentialNamedTuple] +from typing import NamedTuple + +def test() -> None: + class MyNamedTuple(NamedTuple): + parent: 'MyNamedTuple' # E: Cannot resolve name "MyNamedTuple" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + + def bar(nt: MyNamedTuple) -> MyNamedTuple: + return nt + + x: MyNamedTuple + reveal_type(x.parent) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +-- Some crazy self-referential named tuples and types dicts +-- to be sure that everything works + +[case testCrossFileNamedTupleForwardRefs] +import a +[file a.py] +import b +from typing import Any, NamedTuple + +class A: + def a(self, b: 'b.B') -> str: + return 'a' +ATuple = NamedTuple('ATuple', [('a', Any)]) + +[file b.py] +import a + +class B: + def b(self, a: 'a.A') -> str: + return 'b' + def aWithTuple(self, atuple: 'a.ATuple') -> str: + return 'a' +[builtins fixtures/tuple.pyi] +[out] + +[case testSelfRefNT1] +from typing import Tuple, NamedTuple + +def test() -> None: + Node = NamedTuple('Node', [ + ('name', str), + ('children', Tuple['Node', ...]), # E: Cannot resolve name "Node" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + ]) + n: Node + reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Any, ...], fallback=__main__.Node@4]" +[builtins fixtures/tuple.pyi] + +[case testSelfRefNT2] +from typing import Tuple, NamedTuple + +def test() -> None: + A = NamedTuple('A', [ + ('x', str), + ('y', Tuple['B', ...]), # E: Cannot resolve name "B" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + ]) + class B(NamedTuple): + x: A + y: int + + n: A + reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Any, ...], fallback=__main__.A@4]" +[builtins fixtures/tuple.pyi] + +[case testSelfRefNT3] +from typing import NamedTuple, Tuple + +def test() -> None: + class B(NamedTuple): + x: Tuple[A, int] # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + y: int + + A = NamedTuple('A', [ + ('x', str), + ('y', 'B'), + ]) + n: B + m: A + reveal_type(n.x) # N: Revealed type is "Tuple[Any, builtins.int]" + reveal_type(m[0]) # N: Revealed type is "builtins.str" + lst = [m, n] + reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.object]" +[builtins fixtures/tuple.pyi] + +[case testSelfRefNT4] +from typing import NamedTuple + +def test() -> None: + class B(NamedTuple): + x: A # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + y: int + + class A(NamedTuple): + x: str + y: B + + n: A + reveal_type(n.y[0]) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testSelfRefNT5] +from typing import NamedTuple + +def test() -> None: + B = NamedTuple('B', [ + ('x', A), # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope \ + # E: Name "A" is used before definition + ('y', int), + ]) + A = NamedTuple('A', [ + ('x', str), + ('y', 'B'), + ]) + n: A + def f(m: B) -> None: pass + reveal_type(n) # N: Revealed type is "Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.B@4], fallback=__main__.A@8]" + reveal_type(f) # N: Revealed type is "def (m: Tuple[Any, builtins.int, fallback=__main__.B@4])" +[builtins fixtures/tuple.pyi] + +[case testRecursiveNamedTupleInBases] +from typing import List, NamedTuple, Union + +def test() -> None: + Exp = Union['A', 'B'] # E: Cannot resolve name "Exp" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope \ + # E: Cannot resolve name "A" (possible cyclic definition) + class A(NamedTuple('A', [('attr', List[Exp])])): pass + class B(NamedTuple('B', [('val', object)])): pass + + exp: Exp + reveal_type(exp) # N: Revealed type is "Union[Any, Tuple[builtins.object, fallback=__main__.B@6]]" + if isinstance(exp, A): + reveal_type(exp[0][0]) # N: Revealed type is "Union[Any, Tuple[builtins.object, fallback=__main__.B@6]]" + reveal_type(exp.attr[0]) # N: Revealed type is "Union[Any, Tuple[builtins.object, fallback=__main__.B@6]]" + if isinstance(exp, B): + reveal_type(exp.val) # N: Revealed type is "builtins.object" + reveal_type(A([B(1), B(2)])) # N: Revealed type is "Tuple[builtins.list[Union[Any, Tuple[builtins.object, fallback=__main__.B@6]]], fallback=__main__.A@5]" +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testNamedTupleImportCycle] + +import b +[file a.py] +class C: + pass + +from b import tp +x: tp +reveal_type(x.x) # N: Revealed type is "builtins.int" + +reveal_type(tp) # N: Revealed type is "def (x: builtins.int) -> Tuple[builtins.int, fallback=b.tp]" +tp('x') # E: Argument 1 to "tp" has incompatible type "str"; expected "int" + +[file b.py] +from a import C +from typing import NamedTuple + +tp = NamedTuple('tp', [('x', int)]) +[builtins fixtures/tuple.pyi] +[out] + +[case testSubclassOfRecursiveNamedTuple] +from typing import List, NamedTuple + +def test() -> None: + class Command(NamedTuple): + subcommands: List['Command'] # E: Cannot resolve name "Command" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + + class HelpCommand(Command): + pass + + hc = HelpCommand(subcommands=[]) + reveal_type(hc) # N: Revealed type is "Tuple[builtins.list[Any], fallback=__main__.HelpCommand@7]" +[builtins fixtures/list.pyi] +[out] + +[case testUnsafeOverlappingNamedTuple] +from typing import NamedTuple + +class Real(NamedTuple): + def __sub__(self, other: Real) -> str: return "" +class Fraction(Real): + def __rsub__(self, other: Real) -> Real: return other # E: Signatures of "__rsub__" of "Fraction" and "__sub__" of "Real" are unsafely overlapping +[builtins fixtures/tuple.pyi] + +[case testForwardReferenceInNamedTuple] +from typing import NamedTuple + +class A(NamedTuple): + b: 'B' + x: int + +class B: + pass +[builtins fixtures/tuple.pyi] + +[case testTypeNamedTupleClassmethod] +from typing import Type, NamedTuple +class D(NamedTuple): + @classmethod + def f(cls) -> None: pass + +d: Type[D] +d.g() # E: "Type[D]" has no attribute "g" +d.f() +[builtins fixtures/classmethod.pyi] + +[case testTypeNamedTupleCall] + +from typing import NamedTuple + +Thing = NamedTuple('Thing', [('s', str), ('n', int)]) + +class CallableTuple(Thing): + def __call__(self) -> None: + pass + +o = CallableTuple('hello ', 12) +o() +[builtins fixtures/tuple.pyi] + +[case testNamedTupleSubclassMulti] +from typing import NamedTuple + +class Base: + pass +class BaseTuple(NamedTuple): + value: float +class MyTuple(BaseTuple, Base): + pass + +def f(o: Base) -> None: + if isinstance(o, MyTuple): + reveal_type(o.value) # N: Revealed type is "builtins.float" +[builtins fixtures/isinstance.pyi] +[out] + +[case testNamedTupleNew] +from typing import NamedTuple + +Base = NamedTuple('Base', [('param', int)]) + +class Child(Base): + def __new__(cls, param: int = 1) -> 'Child': + return Base.__new__(cls, param) + +Base(param=10) +Child(param=10) +[builtins fixtures/tuple.pyi] + +[case testNamedTupleClassMethodWithGenericReturnValue] +from typing import TypeVar, Type, NamedTuple + +T = TypeVar('T', bound='Parent') + +class Parent(NamedTuple): + x: str + + @classmethod + def class_method(cls: Type[T]) -> T: + return cls(x='text') + +class Child(Parent): + pass + +reveal_type(Child.class_method()) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.Child]" +[builtins fixtures/classmethod.pyi] + +[case testNamedTupleAsConditionalStrictOptionalDisabled] +# flags: --no-strict-optional --warn-unreachable +from typing import NamedTuple + +class C(NamedTuple): + a: int + b: str + +a: C +if not a: + 1() # E: "int" not callable +b = (1, 2) +if not b: + ''() # E: "str" not callable +[builtins fixtures/tuple.pyi] + +[case testNamedTupleDoubleForward] +# flags: --disable-error-code=used-before-def +from typing import Union, Mapping, NamedTuple + +class MyBaseTuple(NamedTuple): + base_field_1: int + base_field_2: int + +MyBaseTupleMapping = Mapping[MyBaseTuple, int] +MyTupleUnion = Union[MyTupleA, MyTupleB] + +class MyTupleA(NamedTuple): + field_1: MyBaseTupleMapping + field_2: MyBaseTuple + + +class MyTupleB(NamedTuple): + field_1: MyBaseTupleMapping + field_2: MyBaseTuple + +u: MyTupleUnion +reveal_type(u.field_1) # N: Revealed type is "typing.Mapping[Tuple[builtins.int, builtins.int, fallback=__main__.MyBaseTuple], builtins.int]" +reveal_type(u.field_2) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.MyBaseTuple]" +reveal_type(u[0]) # N: Revealed type is "typing.Mapping[Tuple[builtins.int, builtins.int, fallback=__main__.MyBaseTuple], builtins.int]" +reveal_type(u[1]) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.MyBaseTuple]" +[builtins fixtures/tuple.pyi] + +[case testAssignNamedTupleAsAttribute] +from typing import NamedTuple + +class A: + def __init__(self) -> None: + self.b = NamedTuple('x', [('s', str), ('n', int)]) # E: NamedTuple type as an attribute is not supported + +reveal_type(A().b) # N: Revealed type is "typing.NamedTuple" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + + +[case testEmptyNamedTupleTypeRepr] +from typing import NamedTuple + +N = NamedTuple('N', []) +n: N +reveal_type(N) # N: Revealed type is "def () -> Tuple[(), fallback=__main__.N]" +reveal_type(n) # N: Revealed type is "Tuple[(), fallback=__main__.N]" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleWrongfile] +from typing import NamedTuple +from b import Type1 +Type2 = NamedTuple('Type2', [('x', Type1)]) +[file b.py] +from typing import NamedTuple + +def foo(): + pass + +Type1 = NamedTuple('Type1', [('foo', foo)]) # E: Function "b.foo" is not valid as a type # N: Perhaps you need "Callable[...]" or a callback protocol? + +[builtins fixtures/tuple.pyi] + +[case testNamedTupleTypeNameMatchesVariableName] +from typing import NamedTuple +from collections import namedtuple + +A = NamedTuple('X', [('a', int)]) # E: First argument to namedtuple() should be "A", not "X" +B = namedtuple('X', ['a']) # E: First argument to namedtuple() should be "B", not "X" + +C = NamedTuple('X', [('a', 'Y')]) # E: First argument to namedtuple() should be "C", not "X" +class Y: ... +[builtins fixtures/tuple.pyi] + +[case testNamedTupleTypeIsASuperTypeOfOtherNamedTuples] +from typing import Tuple, NamedTuple + +class Bar(NamedTuple): + name: str = "Bar" + +class Baz(NamedTuple): + a: str + b: str + +class Biz(Baz): ... +class Other: ... +class Both1(Bar, Other): ... +class Both2(Other, Bar): ... +class Both3(Biz, Other): ... + +def print_namedtuple(obj: NamedTuple) -> None: + reveal_type(obj._fields) # N: Revealed type is "builtins.tuple[builtins.str, ...]" + +b1: Bar +b2: Baz +b3: Biz +b4: Both1 +b5: Both2 +b6: Both3 +print_namedtuple(b1) # ok +print_namedtuple(b2) # ok +print_namedtuple(b3) # ok +print_namedtuple(b4) # ok +print_namedtuple(b5) # ok +print_namedtuple(b6) # ok + +print_namedtuple(1) # E: Argument 1 to "print_namedtuple" has incompatible type "int"; expected "NamedTuple" +print_namedtuple(('bar',)) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[str]"; expected "NamedTuple" +print_namedtuple((1, 2)) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" +print_namedtuple((b1,)) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[Bar]"; expected "NamedTuple" +t: Tuple[str, ...] +print_namedtuple(t) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[str, ...]"; expected "NamedTuple" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleTypeIsASuperTypeOfOtherNamedTuplesReturns] +from typing import Tuple, NamedTuple + +class Bar(NamedTuple): + n: int + +class Baz(NamedTuple): + a: str + b: str + +class Biz(Bar): ... +class Other: ... +class Both1(Bar, Other): ... +class Both2(Other, Bar): ... +class Both3(Biz, Other): ... + +def good1() -> NamedTuple: + b: Bar + return b +def good2() -> NamedTuple: + b: Baz + return b +def good3() -> NamedTuple: + b: Biz + return b +def good4() -> NamedTuple: + b: Both1 + return b +def good5() -> NamedTuple: + b: Both2 + return b +def good6() -> NamedTuple: + b: Both3 + return b + +def bad1() -> NamedTuple: + return 1 # E: Incompatible return value type (got "int", expected "NamedTuple") +def bad2() -> NamedTuple: + return () # E: Incompatible return value type (got "Tuple[()]", expected "NamedTuple") +def bad3() -> NamedTuple: + return (1, 2) # E: Incompatible return value type (got "Tuple[int, int]", expected "NamedTuple") + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testBoolInTuplesRegression] +# https://github.com/python/mypy/issues/11701 +from typing import NamedTuple, Literal, List, Tuple + +C = NamedTuple("C", [("x", Literal[True, False])]) + +T = Tuple[Literal[True, False]] + +# Was error here: +# Incompatible types in assignment (expression has type "List[C]", variable has type "List[C]") +x: List[C] = [C(True)] + +t: T + +# Was error here: +# Incompatible types in assignment (expression has type "List[Tuple[bool]]", +# variable has type "List[Tuple[Union[Literal[True], Literal[False]]]]") +y: List[T] = [t] +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleWithBoolNarrowsToBool] +# flags: --warn-unreachable +from typing import NamedTuple + +class C(NamedTuple): + x: int + + def __bool__(self) -> bool: + pass + +def foo(c: C) -> None: + if c: + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.C]" + else: + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.C]" + +def bar(c: C) -> None: + if not c: + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.C]" + else: + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.C]" + +class C1(NamedTuple): + x: int + +def foo1(c: C1) -> None: + if c: + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.C1]" + else: + c # E: Statement is unreachable + +def bar1(c: C1) -> None: + if not c: + c # E: Statement is unreachable + else: + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.C1]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testInvalidNamedTupleWithinFunction] +from collections import namedtuple + +def f(fields) -> None: + TupleType = namedtuple("TupleType", fields) \ + # E: List or tuple literal expected as the second argument to "namedtuple()" + class InheritFromTuple(TupleType): + pass + t: TupleType + it: InheritFromTuple + NT2 = namedtuple("bad", "x") # E: First argument to namedtuple() should be "NT2", not "bad" + nt2: NT2 = NT2(x=1) +[builtins fixtures/tuple.pyi] + +[case testNamedTupleHasMatchArgs] +# flags: --python-version 3.10 +from typing import NamedTuple +class One(NamedTuple): + bar: int + baz: str +o: One +reveal_type(o.__match_args__) # N: Revealed type is "Tuple[Literal['bar'], Literal['baz']]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleHasNoMatchArgsOldVersion] +# flags: --python-version 3.9 +from typing import NamedTuple +class One(NamedTuple): + bar: int + baz: str +o: One +reveal_type(o.__match_args__) # E: "One" has no attribute "__match_args__" \ + # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleNoBytes] +from collections import namedtuple +from typing import NamedTuple + +NT1 = namedtuple('NT1', b'x y z') # E: List or tuple literal expected as the second argument to "namedtuple()" +NT2 = namedtuple(b'NT2', 'x y z') # E: "namedtuple()" expects a string literal as the first argument \ + # E: Argument 1 to "namedtuple" has incompatible type "bytes"; expected "str" +NT3 = namedtuple('NT3', [b'x', 'y']) # E: String literal expected as "namedtuple()" item + +NT4 = NamedTuple('NT4', [('x', int), (b'y', int)]) # E: Invalid "NamedTuple()" field name +NT5 = NamedTuple(b'NT5', [('x', int), ('y', int)]) # E: "NamedTuple()" expects a string literal as the first argument + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleCreation] +from typing import Generic, NamedTuple, TypeVar + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +nts: NT[str] +reveal_type(nts) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.NT[builtins.str]]" +reveal_type(nts.value) # N: Revealed type is "builtins.str" + +nti = NT(key=0, value=0) +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" +reveal_type(nti.value) # N: Revealed type is "builtins.int" + +NT[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleAlias] +from typing import NamedTuple, Generic, TypeVar, List + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +Alias = NT[List[T]] + +an: Alias[str] +reveal_type(an) # N: Revealed type is "Tuple[builtins.int, builtins.list[builtins.str], fallback=__main__.NT[builtins.list[builtins.str]]]" +Alias[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "int"; expected "List[str]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleMethods] +from typing import Generic, NamedTuple, TypeVar + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T +x: int + +nti: NT[int] +reveal_type(nti * x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +nts: NT[str] +reveal_type(nts * x) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleCustomMethods] +from typing import Generic, NamedTuple, TypeVar + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + def foo(self) -> T: ... + @classmethod + def from_value(cls, value: T) -> NT[T]: ... + +nts: NT[str] +reveal_type(nts.foo()) # N: Revealed type is "builtins.str" + +nti = NT.from_value(1) +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" +NT[str].from_value(1) # E: Argument 1 to "from_value" of "NT" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleSubtyping] +from typing import Generic, NamedTuple, TypeVar, Tuple + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +nts: NT[str] +nti: NT[int] + +def foo(x: Tuple[int, ...]) -> None: ... +foo(nti) +foo(nts) # E: Argument 1 to "foo" has incompatible type "NT[str]"; expected "Tuple[int, ...]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleJoin] +from typing import Generic, NamedTuple, TypeVar, Tuple + +T = TypeVar("T", covariant=True) +class NT(NamedTuple, Generic[T]): + key: int + value: T + +nts: NT[str] +nti: NT[int] +x: Tuple[int, ...] + +S = TypeVar("S") +def foo(x: S, y: S) -> S: ... +reveal_type(foo(nti, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" + +reveal_type(foo(nti, nts)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=__main__.NT[builtins.object]]" +reveal_type(foo(nts, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=__main__.NT[builtins.object]]" + +reveal_type(foo(nti, x)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(foo(nts, x)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" +reveal_type(foo(x, nti)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(foo(x, nts)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleCallSyntax] +from typing import NamedTuple, TypeVar + +T = TypeVar("T") +NT = NamedTuple("NT", [("key", int), ("value", T)]) +reveal_type(NT) # N: Revealed type is "def [T] (key: builtins.int, value: T`1) -> Tuple[builtins.int, T`1, fallback=__main__.NT[T`1]]" + +nts: NT[str] +reveal_type(nts) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.NT[builtins.str]]" + +nti = NT(key=0, value=0) +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" +NT[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleNoLegacySyntax] +from typing import TypeVar, NamedTuple + +T = TypeVar("T") +class C( + NamedTuple("_C", [("x", int), ("y", T)]) # E: Generic named tuples are not supported for legacy class syntax \ + # N: Use either Python 3 class syntax, or the assignment syntax +): ... + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleSelfItemNotAllowed] +from typing import Self, NamedTuple, Optional + +class NT(NamedTuple): + val: int + next: Optional[Self] # E: Self type cannot be used in NamedTuple item type +NTC = NamedTuple("NTC", [("val", int), ("next", Optional[Self])]) # E: Self type cannot be used in NamedTuple item type +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleTypingSelfMethod] +from typing import Self, NamedTuple, TypeVar, Generic + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: str + val: T + def meth(self) -> Self: + nt: NT[int] + if bool(): + return nt._replace() # E: Incompatible return value type (got "NT[int]", expected "Self") + else: + return self._replace() + +class SNT(NT[int]): ... +reveal_type(SNT("test", 42).meth()) # N: Revealed type is "Tuple[builtins.str, builtins.int, fallback=__main__.SNT]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNoCrashUnsupportedNamedTuple] +from typing import NamedTuple +class Test: + def __init__(self, field) -> None: + self.Item = NamedTuple("x", [(field, str)]) # E: NamedTuple type as an attribute is not supported + self.item: self.Item # E: Name "self.Item" is not defined +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNoClassKeywordsForNamedTuple] +from typing import NamedTuple +class Test1(NamedTuple, x=1, y=2): # E: Unexpected keyword argument "x" for "__init_subclass__" of "NamedTuple" \ + # E: Unexpected keyword argument "y" for "__init_subclass__" of "NamedTuple" + ... + +class Meta(type): ... + +class Test2(NamedTuple, metaclass=Meta): # E: Unexpected keyword argument "metaclass" for "__init_subclass__" of "NamedTuple" + ... + +# Technically this would work, but it is just easier for the implementation: +class Test3(NamedTuple, metaclass=type): # E: Unexpected keyword argument "metaclass" for "__init_subclass__" of "NamedTuple" + ... +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + + +[case testNamedTupleDunderReplace] +# flags: --python-version 3.13 +from typing import NamedTuple + +class A(NamedTuple): + x: int + +A(x=0).__replace__(x=1) +A(x=0).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "A" has incompatible type "str"; expected "int" +A(x=0).__replace__(y=1) # E: Unexpected keyword argument "y" for "__replace__" of "A" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testUnpackSelfNamedTuple] +import typing + +class Foo(typing.NamedTuple): + bar: int + def baz(self: typing.Self) -> None: + x, = self + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNameErrorInNamedTupleNestedInFunction1] +from typing import NamedTuple + +def bar() -> None: + class MyNamedTuple(NamedTuple): + a: int + def foo(self) -> None: + ... + int_set: Set[int] # E: Name "Set" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Set") +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNameErrorInNamedTupleNestedInFunction2] +from typing import NamedTuple + +def bar() -> None: + class MyNamedTuple(NamedTuple): + a: int + def foo(self) -> None: + misspelled_var_name # E: Name "misspelled_var_name" is not defined +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test new file mode 100644 index 000000000000..285d56ff7e50 --- /dev/null +++ b/test-data/unit/check-narrowing.test @@ -0,0 +1,2354 @@ +[case testNarrowingParentWithStrsBasic] +from dataclasses import dataclass +from typing import NamedTuple, Tuple, Union +from typing_extensions import Literal, TypedDict + +class Object1: + key: Literal["A"] + foo: int +class Object2: + key: Literal["B"] + bar: str + +@dataclass +class Dataclass1: + key: Literal["A"] + foo: int +@dataclass +class Dataclass2: + key: Literal["B"] + foo: str + +class NamedTuple1(NamedTuple): + key: Literal["A"] + foo: int +class NamedTuple2(NamedTuple): + key: Literal["B"] + foo: str + +Tuple1 = Tuple[Literal["A"], int] +Tuple2 = Tuple[Literal["B"], str] + +class TypedDict1(TypedDict): + key: Literal["A"] + foo: int +class TypedDict2(TypedDict): + key: Literal["B"] + foo: str + +x1: Union[Object1, Object2] +if x1.key == "A": + reveal_type(x1) # N: Revealed type is "__main__.Object1" + reveal_type(x1.key) # N: Revealed type is "Literal['A']" +else: + reveal_type(x1) # N: Revealed type is "__main__.Object2" + reveal_type(x1.key) # N: Revealed type is "Literal['B']" + +x2: Union[Dataclass1, Dataclass2] +if x2.key == "A": + reveal_type(x2) # N: Revealed type is "__main__.Dataclass1" + reveal_type(x2.key) # N: Revealed type is "Literal['A']" +else: + reveal_type(x2) # N: Revealed type is "__main__.Dataclass2" + reveal_type(x2.key) # N: Revealed type is "Literal['B']" + +x3: Union[NamedTuple1, NamedTuple2] +if x3.key == "A": + reveal_type(x3) # N: Revealed type is "Tuple[Literal['A'], builtins.int, fallback=__main__.NamedTuple1]" + reveal_type(x3.key) # N: Revealed type is "Literal['A']" +else: + reveal_type(x3) # N: Revealed type is "Tuple[Literal['B'], builtins.str, fallback=__main__.NamedTuple2]" + reveal_type(x3.key) # N: Revealed type is "Literal['B']" +if x3[0] == "A": + reveal_type(x3) # N: Revealed type is "Tuple[Literal['A'], builtins.int, fallback=__main__.NamedTuple1]" + reveal_type(x3[0]) # N: Revealed type is "Literal['A']" +else: + reveal_type(x3) # N: Revealed type is "Tuple[Literal['B'], builtins.str, fallback=__main__.NamedTuple2]" + reveal_type(x3[0]) # N: Revealed type is "Literal['B']" + +x4: Union[Tuple1, Tuple2] +if x4[0] == "A": + reveal_type(x4) # N: Revealed type is "Tuple[Literal['A'], builtins.int]" + reveal_type(x4[0]) # N: Revealed type is "Literal['A']" +else: + reveal_type(x4) # N: Revealed type is "Tuple[Literal['B'], builtins.str]" + reveal_type(x4[0]) # N: Revealed type is "Literal['B']" + +x5: Union[TypedDict1, TypedDict2] +if x5["key"] == "A": + reveal_type(x5) # N: Revealed type is "TypedDict('__main__.TypedDict1', {'key': Literal['A'], 'foo': builtins.int})" +else: + reveal_type(x5) # N: Revealed type is "TypedDict('__main__.TypedDict2', {'key': Literal['B'], 'foo': builtins.str})" +[builtins fixtures/primitives.pyi] + +[case testNarrowingParentWithEnumsBasic] +from enum import Enum +from dataclasses import dataclass +from typing import NamedTuple, Tuple, Union +from typing_extensions import Literal, TypedDict + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] + foo: int +class Object2: + key: Literal[Key.B] + bar: str + +@dataclass +class Dataclass1: + key: Literal[Key.A] + foo: int +@dataclass +class Dataclass2: + key: Literal[Key.B] + foo: str + +class NamedTuple1(NamedTuple): + key: Literal[Key.A] + foo: int +class NamedTuple2(NamedTuple): + key: Literal[Key.B] + foo: str + +Tuple1 = Tuple[Literal[Key.A], int] +Tuple2 = Tuple[Literal[Key.B], str] + +class TypedDict1(TypedDict): + key: Literal[Key.A] + foo: int +class TypedDict2(TypedDict): + key: Literal[Key.B] + foo: str + +x1: Union[Object1, Object2] +if x1.key is Key.A: + reveal_type(x1) # N: Revealed type is "__main__.Object1" + reveal_type(x1.key) # N: Revealed type is "Literal[__main__.Key.A]" +else: + reveal_type(x1) # N: Revealed type is "__main__.Object2" + reveal_type(x1.key) # N: Revealed type is "Literal[__main__.Key.B]" + +x2: Union[Dataclass1, Dataclass2] +if x2.key is Key.A: + reveal_type(x2) # N: Revealed type is "__main__.Dataclass1" + reveal_type(x2.key) # N: Revealed type is "Literal[__main__.Key.A]" +else: + reveal_type(x2) # N: Revealed type is "__main__.Dataclass2" + reveal_type(x2.key) # N: Revealed type is "Literal[__main__.Key.B]" + +x3: Union[NamedTuple1, NamedTuple2] +if x3.key is Key.A: + reveal_type(x3) # N: Revealed type is "Tuple[Literal[__main__.Key.A], builtins.int, fallback=__main__.NamedTuple1]" + reveal_type(x3.key) # N: Revealed type is "Literal[__main__.Key.A]" +else: + reveal_type(x3) # N: Revealed type is "Tuple[Literal[__main__.Key.B], builtins.str, fallback=__main__.NamedTuple2]" + reveal_type(x3.key) # N: Revealed type is "Literal[__main__.Key.B]" +if x3[0] is Key.A: + reveal_type(x3) # N: Revealed type is "Tuple[Literal[__main__.Key.A], builtins.int, fallback=__main__.NamedTuple1]" + reveal_type(x3[0]) # N: Revealed type is "Literal[__main__.Key.A]" +else: + reveal_type(x3) # N: Revealed type is "Tuple[Literal[__main__.Key.B], builtins.str, fallback=__main__.NamedTuple2]" + reveal_type(x3[0]) # N: Revealed type is "Literal[__main__.Key.B]" + +x4: Union[Tuple1, Tuple2] +if x4[0] is Key.A: + reveal_type(x4) # N: Revealed type is "Tuple[Literal[__main__.Key.A], builtins.int]" + reveal_type(x4[0]) # N: Revealed type is "Literal[__main__.Key.A]" +else: + reveal_type(x4) # N: Revealed type is "Tuple[Literal[__main__.Key.B], builtins.str]" + reveal_type(x4[0]) # N: Revealed type is "Literal[__main__.Key.B]" + +x5: Union[TypedDict1, TypedDict2] +if x5["key"] is Key.A: + reveal_type(x5) # N: Revealed type is "TypedDict('__main__.TypedDict1', {'key': Literal[__main__.Key.A], 'foo': builtins.int})" +else: + reveal_type(x5) # N: Revealed type is "TypedDict('__main__.TypedDict2', {'key': Literal[__main__.Key.B], 'foo': builtins.str})" +[builtins fixtures/narrowing.pyi] + +[case testNarrowingParentWithIsInstanceBasic] +from dataclasses import dataclass +from typing import NamedTuple, Tuple, Union +from typing_extensions import TypedDict + +class Object1: + key: int +class Object2: + key: str + +@dataclass +class Dataclass1: + key: int +@dataclass +class Dataclass2: + key: str + +class NamedTuple1(NamedTuple): + key: int +class NamedTuple2(NamedTuple): + key: str + +Tuple1 = Tuple[int] +Tuple2 = Tuple[str] + +class TypedDict1(TypedDict): + key: int +class TypedDict2(TypedDict): + key: str + +x1: Union[Object1, Object2] +if isinstance(x1.key, int): + reveal_type(x1) # N: Revealed type is "__main__.Object1" +else: + reveal_type(x1) # N: Revealed type is "__main__.Object2" + +x2: Union[Dataclass1, Dataclass2] +if isinstance(x2.key, int): + reveal_type(x2) # N: Revealed type is "__main__.Dataclass1" +else: + reveal_type(x2) # N: Revealed type is "__main__.Dataclass2" + +x3: Union[NamedTuple1, NamedTuple2] +if isinstance(x3.key, int): + reveal_type(x3) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.NamedTuple1]" +else: + reveal_type(x3) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.NamedTuple2]" +if isinstance(x3[0], int): + reveal_type(x3) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.NamedTuple1]" +else: + reveal_type(x3) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.NamedTuple2]" + +x4: Union[Tuple1, Tuple2] +if isinstance(x4[0], int): + reveal_type(x4) # N: Revealed type is "Tuple[builtins.int]" +else: + reveal_type(x4) # N: Revealed type is "Tuple[builtins.str]" + +x5: Union[TypedDict1, TypedDict2] +if isinstance(x5["key"], int): + reveal_type(x5) # N: Revealed type is "TypedDict('__main__.TypedDict1', {'key': builtins.int})" +else: + reveal_type(x5) # N: Revealed type is "TypedDict('__main__.TypedDict2', {'key': builtins.str})" +[builtins fixtures/narrowing.pyi] + +[case testNarrowingParentMultipleKeys] +# flags: --warn-unreachable +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + D = 4 + +class Object1: + key: Literal[Key.A, Key.C] +class Object2: + key: Literal[Key.B, Key.C] + +x: Union[Object1, Object2] +if x.key is Key.A: + reveal_type(x) # N: Revealed type is "__main__.Object1" +else: + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, __main__.Object2]" + +if x.key is Key.C: + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, __main__.Object2]" +else: + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, __main__.Object2]" + +if x.key is Key.D: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, __main__.Object2]" +[builtins fixtures/tuple.pyi] + +[case testNarrowingTypedDictParentMultipleKeys] +# flags: --warn-unreachable +from typing import Union +from typing_extensions import Literal, TypedDict + +class TypedDict1(TypedDict): + key: Literal['A', 'C'] +class TypedDict2(TypedDict): + key: Literal['B', 'C'] + +x: Union[TypedDict1, TypedDict2] +if x['key'] == 'A': + reveal_type(x) # N: Revealed type is "TypedDict('__main__.TypedDict1', {'key': Union[Literal['A'], Literal['C']]})" +else: + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key': Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key': Union[Literal['B'], Literal['C']]})]" + +if x['key'] == 'C': + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key': Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key': Union[Literal['B'], Literal['C']]})]" +else: + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key': Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key': Union[Literal['B'], Literal['C']]})]" + +if x['key'] == 'D': + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key': Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key': Union[Literal['B'], Literal['C']]})]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingPartialTypedDictParentMultipleKeys] +# flags: --warn-unreachable +from typing import Union +from typing_extensions import Literal, TypedDict + +class TypedDict1(TypedDict, total=False): + key: Literal['A', 'C'] +class TypedDict2(TypedDict, total=False): + key: Literal['B', 'C'] + +x: Union[TypedDict1, TypedDict2] +if x['key'] == 'A': + reveal_type(x) # N: Revealed type is "TypedDict('__main__.TypedDict1', {'key'?: Union[Literal['A'], Literal['C']]})" +else: + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key'?: Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key'?: Union[Literal['B'], Literal['C']]})]" + +if x['key'] == 'C': + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key'?: Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key'?: Union[Literal['B'], Literal['C']]})]" +else: + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key'?: Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key'?: Union[Literal['B'], Literal['C']]})]" + +if x['key'] == 'D': + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "Union[TypedDict('__main__.TypedDict1', {'key'?: Union[Literal['A'], Literal['C']]}), TypedDict('__main__.TypedDict2', {'key'?: Union[Literal['B'], Literal['C']]})]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingNestedTypedDicts] +from typing import Union +from typing_extensions import TypedDict, Literal + +class A(TypedDict): + key: Literal['A'] +class B(TypedDict): + key: Literal['B'] +class C(TypedDict): + key: Literal['C'] + +class X(TypedDict): + inner: Union[A, B] +class Y(TypedDict): + inner: Union[B, C] + +unknown: Union[X, Y] +if unknown['inner']['key'] == 'A': + reveal_type(unknown) # N: Revealed type is "TypedDict('__main__.X', {'inner': Union[TypedDict('__main__.A', {'key': Literal['A']}), TypedDict('__main__.B', {'key': Literal['B']})]})" + reveal_type(unknown['inner']) # N: Revealed type is "TypedDict('__main__.A', {'key': Literal['A']})" +if unknown['inner']['key'] == 'B': + reveal_type(unknown) # N: Revealed type is "Union[TypedDict('__main__.X', {'inner': Union[TypedDict('__main__.A', {'key': Literal['A']}), TypedDict('__main__.B', {'key': Literal['B']})]}), TypedDict('__main__.Y', {'inner': Union[TypedDict('__main__.B', {'key': Literal['B']}), TypedDict('__main__.C', {'key': Literal['C']})]})]" + reveal_type(unknown['inner']) # N: Revealed type is "TypedDict('__main__.B', {'key': Literal['B']})" +if unknown['inner']['key'] == 'C': + reveal_type(unknown) # N: Revealed type is "TypedDict('__main__.Y', {'inner': Union[TypedDict('__main__.B', {'key': Literal['B']}), TypedDict('__main__.C', {'key': Literal['C']})]})" + reveal_type(unknown['inner']) # N: Revealed type is "TypedDict('__main__.C', {'key': Literal['C']})" +[builtins fixtures/primitives.pyi] + +[case testNarrowingParentWithMultipleParents] +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] +class Object2: + key: Literal[Key.B] +class Object3: + key: Literal[Key.C] +class Object4: + key: str + +x: Union[Object1, Object2, Object3, Object4] +if x.key is Key.A: + reveal_type(x) # N: Revealed type is "__main__.Object1" +else: + reveal_type(x) # N: Revealed type is "Union[__main__.Object2, __main__.Object3, __main__.Object4]" + +if isinstance(x.key, str): + reveal_type(x) # N: Revealed type is "__main__.Object4" +else: + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, __main__.Object2, __main__.Object3]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingParentsWithGenerics] +from typing import Union, TypeVar, Generic + +T = TypeVar('T') +class Wrapper(Generic[T]): + key: T + +x: Union[Wrapper[int], Wrapper[str]] +if isinstance(x.key, int): + reveal_type(x) # N: Revealed type is "__main__.Wrapper[builtins.int]" +else: + reveal_type(x) # N: Revealed type is "__main__.Wrapper[builtins.str]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingParentWithParentMixtures] +from enum import Enum +from typing import Union, NamedTuple +from typing_extensions import Literal, TypedDict + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class KeyedObject: + key: Literal[Key.A] +class KeyedTypedDict(TypedDict): + key: Literal[Key.B] +class KeyedNamedTuple(NamedTuple): + key: Literal[Key.C] + +ok_mixture: Union[KeyedObject, KeyedNamedTuple] +if ok_mixture.key is Key.A: + reveal_type(ok_mixture) # N: Revealed type is "__main__.KeyedObject" +else: + reveal_type(ok_mixture) # N: Revealed type is "Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]" + +impossible_mixture: Union[KeyedObject, KeyedTypedDict] +if impossible_mixture.key is Key.A: # E: Item "KeyedTypedDict" of "Union[KeyedObject, KeyedTypedDict]" has no attribute "key" + reveal_type(impossible_mixture) # N: Revealed type is "Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]" +else: + reveal_type(impossible_mixture) # N: Revealed type is "Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]" + +if impossible_mixture["key"] is Key.A: # E: Value of type "Union[KeyedObject, KeyedTypedDict]" is not indexable + reveal_type(impossible_mixture) # N: Revealed type is "Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]" +else: + reveal_type(impossible_mixture) # N: Revealed type is "Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]" + +weird_mixture: Union[KeyedTypedDict, KeyedNamedTuple] +if weird_mixture["key"] is Key.B: # E: No overload variant of "__getitem__" of "tuple" matches argument type "str" \ + # N: Possible overload variants: \ + # N: def __getitem__(self, int, /) -> Literal[Key.C] \ + # N: def __getitem__(self, slice, /) -> Tuple[Literal[Key.C], ...] + reveal_type(weird_mixture) # N: Revealed type is "Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]" +else: + reveal_type(weird_mixture) # N: Revealed type is "Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]" + +if weird_mixture[0] is Key.B: # E: TypedDict key must be a string literal; expected one of ("key") + reveal_type(weird_mixture) # N: Revealed type is "Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]" +else: + reveal_type(weird_mixture) # N: Revealed type is "Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testNarrowingParentWithProperties] +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] + +class Object2: + @property + def key(self) -> Literal[Key.A]: ... + +class Object3: + @property + def key(self) -> Literal[Key.B]: ... + +x: Union[Object1, Object2, Object3] +if x.key is Key.A: + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, __main__.Object2]" +else: + reveal_type(x) # N: Revealed type is "__main__.Object3" +[builtins fixtures/property.pyi] + +[case testNarrowingParentWithAny] +from enum import Enum +from typing import Union, Any +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] + +class Object2: + key: Literal[Key.B] + +x: Union[Object1, Object2, Any] +if x.key is Key.A: + reveal_type(x.key) # N: Revealed type is "Literal[__main__.Key.A]" + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, Any]" +else: + # TODO: Is this a bug? Should we skip inferring Any for singleton types? + reveal_type(x.key) # N: Revealed type is "Union[Any, Literal[__main__.Key.B]]" + reveal_type(x) # N: Revealed type is "Union[__main__.Object1, __main__.Object2, Any]" +[builtins fixtures/tuple.pyi] + +[case testNarrowingParentsHierarchy] +from typing import Union +from typing_extensions import Literal +from enum import Enum + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Parent1: + child: Union[Child1, Child2] +class Parent2: + child: Union[Child2, Child3] +class Parent3: + child: Union[Child3, Child1] + +class Child1: + main: Literal[Key.A] + same_for_1_and_2: Literal[Key.A] +class Child2: + main: Literal[Key.B] + same_for_1_and_2: Literal[Key.A] +class Child3: + main: Literal[Key.C] + same_for_1_and_2: Literal[Key.B] + +x: Union[Parent1, Parent2, Parent3] +if x.child.main is Key.A: + reveal_type(x) # N: Revealed type is "Union[__main__.Parent1, __main__.Parent3]" + reveal_type(x.child) # N: Revealed type is "__main__.Child1" +else: + reveal_type(x) # N: Revealed type is "Union[__main__.Parent1, __main__.Parent2, __main__.Parent3]" + reveal_type(x.child) # N: Revealed type is "Union[__main__.Child2, __main__.Child3]" + +if x.child.same_for_1_and_2 is Key.A: + reveal_type(x) # N: Revealed type is "Union[__main__.Parent1, __main__.Parent2, __main__.Parent3]" + reveal_type(x.child) # N: Revealed type is "Union[__main__.Child1, __main__.Child2]" +else: + reveal_type(x) # N: Revealed type is "Union[__main__.Parent2, __main__.Parent3]" + reveal_type(x.child) # N: Revealed type is "__main__.Child3" + +y: Union[Parent1, Parent2] +if y.child.main is Key.A: + reveal_type(y) # N: Revealed type is "__main__.Parent1" + reveal_type(y.child) # N: Revealed type is "__main__.Child1" +else: + reveal_type(y) # N: Revealed type is "Union[__main__.Parent1, __main__.Parent2]" + reveal_type(y.child) # N: Revealed type is "Union[__main__.Child2, __main__.Child3]" + +if y.child.same_for_1_and_2 is Key.A: + reveal_type(y) # N: Revealed type is "Union[__main__.Parent1, __main__.Parent2]" + reveal_type(y.child) # N: Revealed type is "Union[__main__.Child1, __main__.Child2]" +else: + reveal_type(y) # N: Revealed type is "__main__.Parent2" + reveal_type(y.child) # N: Revealed type is "__main__.Child3" +[builtins fixtures/tuple.pyi] + +[case testNarrowingParentsHierarchyGenerics] +from typing import Generic, TypeVar, Union + +T = TypeVar('T') +class Model(Generic[T]): + attr: T +class A: + model: Model[int] +class B: + model: Model[str] + +x: Union[A, B] +if isinstance(x.model.attr, int): + reveal_type(x) # N: Revealed type is "__main__.A" + reveal_type(x.model) # N: Revealed type is "__main__.Model[builtins.int]" +else: + reveal_type(x) # N: Revealed type is "__main__.B" + reveal_type(x.model) # N: Revealed type is "__main__.Model[builtins.str]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingParentsHierarchyTypedDict] +# flags: --warn-unreachable +from typing import Union +from typing_extensions import TypedDict, Literal +from enum import Enum + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Parent1(TypedDict): + model: Model1 + foo: int + +class Parent2(TypedDict): + model: Model2 + bar: str + +class Model1(TypedDict): + key: Literal[Key.A] + +class Model2(TypedDict): + key: Literal[Key.B] + +x: Union[Parent1, Parent2] +if x["model"]["key"] is Key.A: + reveal_type(x) # N: Revealed type is "TypedDict('__main__.Parent1', {'model': TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]}), 'foo': builtins.int})" + reveal_type(x["model"]) # N: Revealed type is "TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]})" +else: + reveal_type(x) # N: Revealed type is "TypedDict('__main__.Parent2', {'model': TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]}), 'bar': builtins.str})" + reveal_type(x["model"]) # N: Revealed type is "TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]})" + +y: Union[Parent1, Parent2] +if y["model"]["key"] is Key.C: + reveal_type(y) # E: Statement is unreachable + reveal_type(y["model"]) +else: + reveal_type(y) # N: Revealed type is "Union[TypedDict('__main__.Parent1', {'model': TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]}), 'foo': builtins.int}), TypedDict('__main__.Parent2', {'model': TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]}), 'bar': builtins.str})]" + reveal_type(y["model"]) # N: Revealed type is "Union[TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]}), TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]})]" +[builtins fixtures/tuple.pyi] + +[case testNarrowingParentsHierarchyTypedDictWithStr] +# flags: --warn-unreachable +from typing import Union +from typing_extensions import TypedDict, Literal + +class Parent1(TypedDict): + model: Model1 + foo: int + +class Parent2(TypedDict): + model: Model2 + bar: str + +class Model1(TypedDict): + key: Literal['A'] + +class Model2(TypedDict): + key: Literal['B'] + +x: Union[Parent1, Parent2] +if x["model"]["key"] == 'A': + reveal_type(x) # N: Revealed type is "TypedDict('__main__.Parent1', {'model': TypedDict('__main__.Model1', {'key': Literal['A']}), 'foo': builtins.int})" + reveal_type(x["model"]) # N: Revealed type is "TypedDict('__main__.Model1', {'key': Literal['A']})" +else: + reveal_type(x) # N: Revealed type is "TypedDict('__main__.Parent2', {'model': TypedDict('__main__.Model2', {'key': Literal['B']}), 'bar': builtins.str})" + reveal_type(x["model"]) # N: Revealed type is "TypedDict('__main__.Model2', {'key': Literal['B']})" + +y: Union[Parent1, Parent2] +if y["model"]["key"] == 'C': + reveal_type(y) # E: Statement is unreachable + reveal_type(y["model"]) +else: + reveal_type(y) # N: Revealed type is "Union[TypedDict('__main__.Parent1', {'model': TypedDict('__main__.Model1', {'key': Literal['A']}), 'foo': builtins.int}), TypedDict('__main__.Parent2', {'model': TypedDict('__main__.Model2', {'key': Literal['B']}), 'bar': builtins.str})]" + reveal_type(y["model"]) # N: Revealed type is "Union[TypedDict('__main__.Model1', {'key': Literal['A']}), TypedDict('__main__.Model2', {'key': Literal['B']})]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingExprPropagation] +from typing import Union +from typing_extensions import Literal + +class A: + tag: Literal['A'] + +class B: + tag: Literal['B'] + +abo: Union[A, B, None] + +if abo is not None and abo.tag == "A": + reveal_type(abo.tag) # N: Revealed type is "Literal['A']" + reveal_type(abo) # N: Revealed type is "__main__.A" + +if not (abo is None or abo.tag != "B"): + reveal_type(abo.tag) # N: Revealed type is "Literal['B']" + reveal_type(abo) # N: Revealed type is "__main__.B" +[builtins fixtures/primitives.pyi] + +[case testNarrowingEqualityFlipFlop] +# flags: --warn-unreachable --strict-equality +from typing_extensions import Literal, Final +from enum import Enum + +class State(Enum): + A = 1 + B = 2 + +class FlipFlopEnum: + def __init__(self) -> None: + self.state = State.A + + def mutate(self) -> None: + self.state = State.B if self.state == State.A else State.A + +class FlipFlopStr: + def __init__(self) -> None: + self.state = "state-1" + + def mutate(self) -> None: + self.state = "state-2" if self.state == "state-1" else "state-1" + + +def test1(switch: FlipFlopStr) -> None: + # Naively, we might assume the 'assert' here would narrow the type to + # Literal["state-1"]. However, doing this ends up breaking a fair number of real-world + # code (usually test cases) that looks similar to this function: e.g. checks + # to make sure a field was mutated to some particular value. + # + # And since mypy can't really reason about state mutation, we take a conservative + # approach and avoid narrowing anything here. + + assert switch.state == "state-1" + reveal_type(switch.state) # N: Revealed type is "builtins.str" + + switch.mutate() + + assert switch.state == "state-2" + reveal_type(switch.state) # N: Revealed type is "builtins.str" + +def test2(switch: FlipFlopEnum) -> None: + # This is the same thing as 'test1', except we use enums, which we allow to be narrowed + # to literals. + + assert switch.state == State.A + reveal_type(switch.state) # N: Revealed type is "Literal[__main__.State.A]" + + switch.mutate() + + assert switch.state == State.B # E: Non-overlapping equality check (left operand type: "Literal[State.A]", right operand type: "Literal[State.B]") + reveal_type(switch.state) # E: Statement is unreachable + +def test3(switch: FlipFlopEnum) -> None: + # Same thing, but using 'is' comparisons. Previously mypy's behaviour differed + # here, narrowing when using 'is', but not when using '=='. + + assert switch.state is State.A + reveal_type(switch.state) # N: Revealed type is "Literal[__main__.State.A]" + + switch.mutate() + + assert switch.state is State.B # E: Non-overlapping identity check (left operand type: "Literal[State.A]", right operand type: "Literal[State.B]") + reveal_type(switch.state) # E: Statement is unreachable +[builtins fixtures/primitives.pyi] + +[case testNarrowingEqualityRequiresExplicitStrLiteral] +from typing_extensions import Literal, Final + +A_final: Final = "A" +A_literal: Literal["A"] + +# Neither the LHS nor the RHS are explicit literals, so regrettably nothing +# is narrowed here -- see 'testNarrowingEqualityFlipFlop' for an example of +# why more precise inference here is problematic. +x_str: str +if x_str == "A": + reveal_type(x_str) # N: Revealed type is "builtins.str" +else: + reveal_type(x_str) # N: Revealed type is "builtins.str" +reveal_type(x_str) # N: Revealed type is "builtins.str" + +if x_str == A_final: + reveal_type(x_str) # N: Revealed type is "builtins.str" +else: + reveal_type(x_str) # N: Revealed type is "builtins.str" +reveal_type(x_str) # N: Revealed type is "builtins.str" + +# But the RHS is a literal, so we can at least narrow the 'if' case now. +if x_str == A_literal: + reveal_type(x_str) # N: Revealed type is "Literal['A']" +else: + reveal_type(x_str) # N: Revealed type is "builtins.str" +reveal_type(x_str) # N: Revealed type is "builtins.str" + +# But in these two cases, the LHS is a literal/literal-like type. So we +# assume the user *does* want literal-based narrowing and narrow accordingly +# regardless of whether the RHS is an explicit literal or not. +x_union: Literal["A", "B", None] +if x_union == A_final: + reveal_type(x_union) # N: Revealed type is "Literal['A']" +else: + reveal_type(x_union) # N: Revealed type is "Union[Literal['B'], None]" +reveal_type(x_union) # N: Revealed type is "Union[Literal['A'], Literal['B'], None]" + +if x_union == A_literal: + reveal_type(x_union) # N: Revealed type is "Literal['A']" +else: + reveal_type(x_union) # N: Revealed type is "Union[Literal['B'], None]" +reveal_type(x_union) # N: Revealed type is "Union[Literal['A'], Literal['B'], None]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingEqualityRequiresExplicitEnumLiteral] +from typing import Union +from typing_extensions import Literal, Final +from enum import Enum + +class Foo(Enum): + A = 1 + B = 2 + +A_final: Final = Foo.A +A_literal: Literal[Foo.A] + +# Note this is unlike testNarrowingEqualityRequiresExplicitStrLiteral +# See also testNarrowingEqualityFlipFlop +x1: Foo +if x1 == Foo.A: + reveal_type(x1) # N: Revealed type is "Literal[__main__.Foo.A]" +else: + reveal_type(x1) # N: Revealed type is "Literal[__main__.Foo.B]" + +x2: Foo +if x2 == A_final: + reveal_type(x2) # N: Revealed type is "Literal[__main__.Foo.A]" +else: + reveal_type(x2) # N: Revealed type is "Literal[__main__.Foo.B]" + +# But we let this narrow since there's an explicit literal in the RHS. +x3: Foo +if x3 == A_literal: + reveal_type(x3) # N: Revealed type is "Literal[__main__.Foo.A]" +else: + reveal_type(x3) # N: Revealed type is "Literal[__main__.Foo.B]" + + +class SingletonFoo(Enum): + A = "A" + +def bar(x: Union[SingletonFoo, Foo], y: SingletonFoo) -> None: + if x == y: + reveal_type(x) # N: Revealed type is "Literal[__main__.SingletonFoo.A]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingEqualityDisabledForCustomEquality] +from typing import Union +from typing_extensions import Literal +from enum import Enum + +class Custom: + def __eq__(self, other: object) -> bool: return True + +class Default: pass + +x1: Union[Custom, Literal[1], Literal[2]] +if x1 == 1: + reveal_type(x1) # N: Revealed type is "Union[__main__.Custom, Literal[1], Literal[2]]" +else: + reveal_type(x1) # N: Revealed type is "Union[__main__.Custom, Literal[1], Literal[2]]" + +x2: Union[Default, Literal[1], Literal[2]] +if x2 == 1: + reveal_type(x2) # N: Revealed type is "Literal[1]" +else: + reveal_type(x2) # N: Revealed type is "Union[__main__.Default, Literal[2]]" + +class CustomEnum(Enum): + A = 1 + B = 2 + + def __eq__(self, other: object) -> bool: return True + +x3: CustomEnum +key: Literal[CustomEnum.A] +if x3 == key: + reveal_type(x3) # N: Revealed type is "__main__.CustomEnum" +else: + reveal_type(x3) # N: Revealed type is "__main__.CustomEnum" + +# For comparison, this narrows since we bypass __eq__ +if x3 is key: + reveal_type(x3) # N: Revealed type is "Literal[__main__.CustomEnum.A]" +else: + reveal_type(x3) # N: Revealed type is "Literal[__main__.CustomEnum.B]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingEqualityDisabledForCustomEqualityChain] +# flags: --strict-equality --warn-unreachable +from typing import Union +from typing_extensions import Literal + +class Custom: + def __eq__(self, other: object) -> bool: return True + +class Default: pass + +x: Literal[1, 2, None] +y: Custom +z: Default + +# We could maybe try doing something clever, but for simplicity we +# treat the whole chain as contaminated and mostly disable narrowing. +# +# The only exception is that we do at least strip away the 'None'. We +# (perhaps optimistically) assume no custom class would be pathological +# enough to declare itself to be equal to None and so permit this narrowing, +# since it's often convenient in practice. +if 1 == x == y: + reveal_type(x) # N: Revealed type is "Union[Literal[1], Literal[2]]" + reveal_type(y) # N: Revealed type is "__main__.Custom" +else: + reveal_type(x) # N: Revealed type is "Union[Literal[1], Literal[2], None]" + reveal_type(y) # N: Revealed type is "__main__.Custom" + +# No contamination here +if 1 == x == z: # E: Non-overlapping equality check (left operand type: "Optional[Literal[1, 2]]", right operand type: "Default") + reveal_type(x) # E: Statement is unreachable + reveal_type(z) +else: + reveal_type(x) # N: Revealed type is "Union[Literal[1], Literal[2], None]" + reveal_type(z) # N: Revealed type is "__main__.Default" +[builtins fixtures/primitives.pyi] + +[case testNarrowingUnreachableCases] +# flags: --strict-equality --warn-unreachable +from typing import Union +from typing_extensions import Literal + +a: Literal[1] +b: Literal[1, 2] +c: Literal[2, 3] + +if a == b == c: + reveal_type(a) # E: Statement is unreachable + reveal_type(b) + reveal_type(c) +else: + reveal_type(a) # N: Revealed type is "Literal[1]" + reveal_type(b) # N: Revealed type is "Union[Literal[1], Literal[2]]" + reveal_type(c) # N: Revealed type is "Union[Literal[2], Literal[3]]" + +if a == a == a: + reveal_type(a) # N: Revealed type is "Literal[1]" +else: + reveal_type(a) # E: Statement is unreachable + +if a == a == b: + reveal_type(a) # N: Revealed type is "Literal[1]" + reveal_type(b) # N: Revealed type is "Literal[1]" +else: + reveal_type(a) # N: Revealed type is "Literal[1]" + reveal_type(b) # N: Revealed type is "Literal[2]" + +# In this case, it's ok for 'b' to narrow down to Literal[1] in the else case +# since that's the only way 'b == 2' can be false +if b == 2: + reveal_type(b) # N: Revealed type is "Literal[2]" +else: + reveal_type(b) # N: Revealed type is "Literal[1]" + +# But in this case, we can't conclude anything about the else case. This expression +# could end up being either '2 == 2 == 3' or '1 == 2 == 2', which means we can't +# conclude anything. +if b == 2 == c: + reveal_type(b) # N: Revealed type is "Literal[2]" + reveal_type(c) # N: Revealed type is "Literal[2]" +else: + reveal_type(b) # N: Revealed type is "Union[Literal[1], Literal[2]]" + reveal_type(c) # N: Revealed type is "Union[Literal[2], Literal[3]]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingUnreachableCases2] +# flags: --strict-equality --warn-unreachable +from typing import Union +from typing_extensions import Literal + +a: Literal[1, 2, 3, 4] +b: Literal[1, 2, 3, 4] + +if a == b == 1: + reveal_type(a) # N: Revealed type is "Literal[1]" + reveal_type(b) # N: Revealed type is "Literal[1]" +elif a == b == 2: + reveal_type(a) # N: Revealed type is "Literal[2]" + reveal_type(b) # N: Revealed type is "Literal[2]" +elif a == b == 3: + reveal_type(a) # N: Revealed type is "Literal[3]" + reveal_type(b) # N: Revealed type is "Literal[3]" +elif a == b == 4: + reveal_type(a) # N: Revealed type is "Literal[4]" + reveal_type(b) # N: Revealed type is "Literal[4]" +else: + # This branch is reachable if a == 1 and b == 2, for example. + reveal_type(a) # N: Revealed type is "Union[Literal[1], Literal[2], Literal[3], Literal[4]]" + reveal_type(b) # N: Revealed type is "Union[Literal[1], Literal[2], Literal[3], Literal[4]]" + +if a == a == 1: + reveal_type(a) # N: Revealed type is "Literal[1]" +elif a == a == 2: + reveal_type(a) # N: Revealed type is "Literal[2]" +elif a == a == 3: + reveal_type(a) # N: Revealed type is "Literal[3]" +elif a == a == 4: + reveal_type(a) # N: Revealed type is "Literal[4]" +else: + # In contrast, this branch must be unreachable: we assume (maybe naively) + # that 'a' won't be mutated in the middle of the expression. + reveal_type(a) # E: Statement is unreachable + reveal_type(b) +[builtins fixtures/primitives.pyi] + +[case testNarrowingLiteralTruthiness] +from typing import Union +from typing_extensions import Literal + +str_or_false: Union[Literal[False], str] + +if str_or_false: + reveal_type(str_or_false) # N: Revealed type is "builtins.str" +else: + reveal_type(str_or_false) # N: Revealed type is "Union[Literal[False], Literal['']]" + +true_or_false: Literal[True, False] + +if true_or_false: + reveal_type(true_or_false) # N: Revealed type is "Literal[True]" +else: + reveal_type(true_or_false) # N: Revealed type is "Literal[False]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingFalseyToLiteral] +from typing import Union + +a: str +b: bytes +c: int +d: Union[str, bytes, int] + +if not a: + reveal_type(a) # N: Revealed type is "Literal['']" +if not b: + reveal_type(b) # N: Revealed type is "Literal[b'']" +if not c: + reveal_type(c) # N: Revealed type is "Literal[0]" +if not d: + reveal_type(d) # N: Revealed type is "Union[Literal[''], Literal[b''], Literal[0]]" + +[case testNarrowingIsInstanceFinalSubclass] +# flags: --warn-unreachable + +from typing import final + +class N: ... +@final +class F1: ... +@final +class F2: ... + +n: N +f1: F1 + +if isinstance(f1, F1): + reveal_type(f1) # N: Revealed type is "__main__.F1" +else: + reveal_type(f1) # E: Statement is unreachable + +if isinstance(n, F1): # E: Subclass of "N" and "F1" cannot exist: "F1" is final + reveal_type(n) # E: Statement is unreachable +else: + reveal_type(n) # N: Revealed type is "__main__.N" + +if isinstance(f1, N): # E: Subclass of "F1" and "N" cannot exist: "F1" is final + reveal_type(f1) # E: Statement is unreachable +else: + reveal_type(f1) # N: Revealed type is "__main__.F1" + +if isinstance(f1, F2): # E: Subclass of "F1" and "F2" cannot exist: "F1" is final \ + # E: Subclass of "F1" and "F2" cannot exist: "F2" is final + reveal_type(f1) # E: Statement is unreachable +else: + reveal_type(f1) # N: Revealed type is "__main__.F1" +[builtins fixtures/isinstance.pyi] + + +[case testNarrowingIsInstanceFinalSubclassWithUnions] +# flags: --warn-unreachable + +from typing import final, Union + +class N: ... +@final +class F1: ... +@final +class F2: ... + +n_f1: Union[N, F1] +n_f2: Union[N, F2] +f1_f2: Union[F1, F2] + +if isinstance(n_f1, F1): + reveal_type(n_f1) # N: Revealed type is "__main__.F1" +else: + reveal_type(n_f1) # N: Revealed type is "__main__.N" + +if isinstance(n_f2, F1): # E: Subclass of "N" and "F1" cannot exist: "F1" is final \ + # E: Subclass of "F2" and "F1" cannot exist: "F2" is final \ + # E: Subclass of "F2" and "F1" cannot exist: "F1" is final + reveal_type(n_f2) # E: Statement is unreachable +else: + reveal_type(n_f2) # N: Revealed type is "Union[__main__.N, __main__.F2]" + +if isinstance(f1_f2, F1): + reveal_type(f1_f2) # N: Revealed type is "__main__.F1" +else: + reveal_type(f1_f2) # N: Revealed type is "__main__.F2" +[builtins fixtures/isinstance.pyi] + + +[case testNarrowingIsSubclassFinalSubclassWithTypeVar] +# flags: --warn-unreachable + +from typing import final, Type, TypeVar + +@final +class A: ... +@final +class B: ... + +T = TypeVar("T", A, B) + +def f(cls: Type[T]) -> T: + if issubclass(cls, A): + reveal_type(cls) # N: Revealed type is "Type[__main__.A]" + x: bool + if x: + return A() + else: + return B() # E: Incompatible return value type (got "B", expected "A") + assert False + +reveal_type(f(A)) # N: Revealed type is "__main__.A" +reveal_type(f(B)) # N: Revealed type is "__main__.B" +[builtins fixtures/isinstance.pyi] + + +[case testNarrowingLiteralIdentityCheck] +from typing import Union +from typing_extensions import Literal + +str_or_false: Union[Literal[False], str] + +if str_or_false is not False: + reveal_type(str_or_false) # N: Revealed type is "builtins.str" +else: + reveal_type(str_or_false) # N: Revealed type is "Literal[False]" + +if str_or_false is False: + reveal_type(str_or_false) # N: Revealed type is "Literal[False]" +else: + reveal_type(str_or_false) # N: Revealed type is "builtins.str" + +str_or_true: Union[Literal[True], str] + +if str_or_true is True: + reveal_type(str_or_true) # N: Revealed type is "Literal[True]" +else: + reveal_type(str_or_true) # N: Revealed type is "builtins.str" + +if str_or_true is not True: + reveal_type(str_or_true) # N: Revealed type is "builtins.str" +else: + reveal_type(str_or_true) # N: Revealed type is "Literal[True]" + +str_or_bool_literal: Union[Literal[False], Literal[True], str] + +if str_or_bool_literal is not True: + reveal_type(str_or_bool_literal) # N: Revealed type is "Union[Literal[False], builtins.str]" +else: + reveal_type(str_or_bool_literal) # N: Revealed type is "Literal[True]" + +if str_or_bool_literal is not True and str_or_bool_literal is not False: + reveal_type(str_or_bool_literal) # N: Revealed type is "builtins.str" +else: + reveal_type(str_or_bool_literal) # N: Revealed type is "builtins.bool" +[builtins fixtures/primitives.pyi] + +[case testNarrowingBooleanIdentityCheck] +from typing import Optional +from typing_extensions import Literal + +bool_val: bool + +if bool_val is not False: + reveal_type(bool_val) # N: Revealed type is "Literal[True]" +else: + reveal_type(bool_val) # N: Revealed type is "Literal[False]" + +opt_bool_val: Optional[bool] + +if opt_bool_val is not None: + reveal_type(opt_bool_val) # N: Revealed type is "builtins.bool" + +if opt_bool_val is not False: + reveal_type(opt_bool_val) # N: Revealed type is "Union[Literal[True], None]" +else: + reveal_type(opt_bool_val) # N: Revealed type is "Literal[False]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingBooleanTruthiness] +from typing import Optional +from typing_extensions import Literal + +bool_val: bool + +if bool_val: + reveal_type(bool_val) # N: Revealed type is "Literal[True]" +else: + reveal_type(bool_val) # N: Revealed type is "Literal[False]" +reveal_type(bool_val) # N: Revealed type is "builtins.bool" + +opt_bool_val: Optional[bool] + +if opt_bool_val: + reveal_type(opt_bool_val) # N: Revealed type is "Literal[True]" +else: + reveal_type(opt_bool_val) # N: Revealed type is "Union[Literal[False], None]" +reveal_type(opt_bool_val) # N: Revealed type is "Union[builtins.bool, None]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingBooleanBoolOp] +from typing import Optional +from typing_extensions import Literal + +bool_a: bool +bool_b: bool + +if bool_a and bool_b: + reveal_type(bool_a) # N: Revealed type is "Literal[True]" + reveal_type(bool_b) # N: Revealed type is "Literal[True]" +else: + reveal_type(bool_a) # N: Revealed type is "builtins.bool" + reveal_type(bool_b) # N: Revealed type is "builtins.bool" + +if not bool_a or bool_b: + reveal_type(bool_a) # N: Revealed type is "builtins.bool" + reveal_type(bool_b) # N: Revealed type is "builtins.bool" +else: + reveal_type(bool_a) # N: Revealed type is "Literal[True]" + reveal_type(bool_b) # N: Revealed type is "Literal[False]" + +if True and bool_b: + reveal_type(bool_b) # N: Revealed type is "Literal[True]" + +x = True and bool_b +reveal_type(x) # N: Revealed type is "builtins.bool" +[builtins fixtures/primitives.pyi] + +[case testNarrowingTypedDictUsingEnumLiteral] +from typing import Union +from typing_extensions import TypedDict, Literal +from enum import Enum + +class E(Enum): + FOO = "a" + BAR = "b" + +class Foo(TypedDict): + tag: Literal[E.FOO] + x: int + +class Bar(TypedDict): + tag: Literal[E.BAR] + y: int + +def f(d: Union[Foo, Bar]) -> None: + assert d['tag'] == E.FOO + d['x'] + reveal_type(d) # N: Revealed type is "TypedDict('__main__.Foo', {'tag': Literal[__main__.E.FOO], 'x': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testNarrowingUsingMetaclass] +from typing import Type + +class M(type): + pass + +class C: pass + +def f(t: Type[C]) -> None: + if type(t) is M: + reveal_type(t) # N: Revealed type is "Type[__main__.C]" + else: + reveal_type(t) # N: Revealed type is "Type[__main__.C]" + if type(t) is not M: + reveal_type(t) # N: Revealed type is "Type[__main__.C]" + else: + reveal_type(t) # N: Revealed type is "Type[__main__.C]" + reveal_type(t) # N: Revealed type is "Type[__main__.C]" + +[case testNarrowingUsingTypeVar] +from typing import Type, TypeVar + +class A: pass +class B(A): pass + +T = TypeVar("T", bound=A) + +def f(t: Type[T], a: A, b: B) -> None: + if type(a) is t: + reveal_type(a) # N: Revealed type is "T`-1" + else: + reveal_type(a) # N: Revealed type is "__main__.A" + + if type(b) is t: + reveal_type(b) # N: Revealed type is "Never" + else: + reveal_type(b) # N: Revealed type is "__main__.B" + +[case testNarrowingNestedUnionOfTypedDicts] +from typing import Union +from typing_extensions import Literal, TypedDict + +class A(TypedDict): + tag: Literal["A"] + a: int + +class B(TypedDict): + tag: Literal["B"] + b: int + +class C(TypedDict): + tag: Literal["C"] + c: int + +AB = Union[A, B] +ABC = Union[AB, C] +abc: ABC + +if abc["tag"] == "A": + reveal_type(abc) # N: Revealed type is "TypedDict('__main__.A', {'tag': Literal['A'], 'a': builtins.int})" +elif abc["tag"] == "C": + reveal_type(abc) # N: Revealed type is "TypedDict('__main__.C', {'tag': Literal['C'], 'c': builtins.int})" +else: + reveal_type(abc) # N: Revealed type is "TypedDict('__main__.B', {'tag': Literal['B'], 'b': builtins.int})" + +[builtins fixtures/primitives.pyi] + + +[case testNarrowingRuntimeCover] +from typing import Dict, List, Union + +def unreachable(x: Union[str, List[str]]) -> None: + if isinstance(x, str): + reveal_type(x) # N: Revealed type is "builtins.str" + elif isinstance(x, list): + reveal_type(x) # N: Revealed type is "builtins.list[builtins.str]" + else: + reveal_type(x) # No output: this branch is unreachable + +def all_parts_covered(x: Union[str, List[str], List[int], int]) -> None: + if isinstance(x, str): + reveal_type(x) # N: Revealed type is "builtins.str" + elif isinstance(x, list): + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.str], builtins.list[builtins.int]]" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + +def two_type_vars(x: Union[str, Dict[str, int], Dict[bool, object], int]) -> None: + if isinstance(x, str): + reveal_type(x) # N: Revealed type is "builtins.str" + elif isinstance(x, dict): + reveal_type(x) # N: Revealed type is "Union[builtins.dict[builtins.str, builtins.int], builtins.dict[builtins.bool, builtins.object]]" + else: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + + +[case testNarrowingWithDef] +from typing import Callable, Optional + +def g() -> None: + foo: Optional[Callable[[], None]] = None + if foo is None: + def foo(): ... + foo() +[builtins fixtures/dict.pyi] + + +[case testNarrowingOptionalEqualsNone] +from typing import Optional + +class A: ... + +val: Optional[A] + +if val == None: + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" +else: + reveal_type(val) # N: Revealed type is "__main__.A" +if val != None: + reveal_type(val) # N: Revealed type is "__main__.A" +else: + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" + +if val in (None,): + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" +else: + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" +if val not in (None,): + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" +else: + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingWithTupleOfTypes] +from typing import Tuple, Type + +class Base: ... + +class Impl1(Base): ... +class Impl2(Base): ... + +impls: Tuple[Type[Base], ...] = (Impl1, Impl2) +some: object + +if isinstance(some, impls): + reveal_type(some) # N: Revealed type is "__main__.Base" +else: + reveal_type(some) # N: Revealed type is "builtins.object" + +raw: Tuple[type, ...] +if isinstance(some, raw): + reveal_type(some) # N: Revealed type is "builtins.object" +else: + reveal_type(some) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] + + +[case testNarrowingWithTupleOfTypesPy310Plus] +# flags: --python-version 3.10 +class Base: ... + +class Impl1(Base): ... +class Impl2(Base): ... + +some: int | Base + +impls: tuple[type[Base], ...] = (Impl1, Impl2) +if isinstance(some, impls): + reveal_type(some) # N: Revealed type is "__main__.Base" +else: + reveal_type(some) # N: Revealed type is "Union[builtins.int, __main__.Base]" + +raw: tuple[type, ...] +if isinstance(some, raw): + reveal_type(some) # N: Revealed type is "Union[builtins.int, __main__.Base]" +else: + reveal_type(some) # N: Revealed type is "Union[builtins.int, __main__.Base]" +[builtins fixtures/dict.pyi] + +[case testNarrowingWithAnyOps] +from typing import Any + +class C: ... +class D(C): ... +tp: Any + +c: C +if isinstance(c, tp) or isinstance(c, D): + reveal_type(c) # N: Revealed type is "Union[Any, __main__.D]" +else: + reveal_type(c) # N: Revealed type is "__main__.C" +reveal_type(c) # N: Revealed type is "__main__.C" + +c1: C +if isinstance(c1, tp) and isinstance(c1, D): + reveal_type(c1) # N: Revealed type is "Any" +else: + reveal_type(c1) # N: Revealed type is "__main__.C" +reveal_type(c1) # N: Revealed type is "__main__.C" + +c2: C +if isinstance(c2, D) or isinstance(c2, tp): + reveal_type(c2) # N: Revealed type is "Union[__main__.D, Any]" +else: + reveal_type(c2) # N: Revealed type is "__main__.C" +reveal_type(c2) # N: Revealed type is "__main__.C" + +c3: C +if isinstance(c3, D) and isinstance(c3, tp): + reveal_type(c3) # N: Revealed type is "Any" +else: + reveal_type(c3) # N: Revealed type is "__main__.C" +reveal_type(c3) # N: Revealed type is "__main__.C" + +t: Any +if isinstance(t, (list, tuple)) and isinstance(t, tuple): + reveal_type(t) # N: Revealed type is "builtins.tuple[Any, ...]" +else: + reveal_type(t) # N: Revealed type is "Any" +reveal_type(t) # N: Revealed type is "Any" +[builtins fixtures/isinstancelist.pyi] + +[case testNarrowingLenItemAndLenCompare] +from typing import Any + +x: Any +if len(x) == x: + reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/len.pyi] + +[case testNarrowingLenTuple] +from typing import Tuple, Union + +VarTuple = Union[Tuple[int, int], Tuple[int, int, int]] + +x: VarTuple +a = b = c = 0 +if len(x) == 3: + a, b, c = x +else: + a, b = x + +if len(x) != 3: + a, b = x +else: + a, b, c = x +[builtins fixtures/len.pyi] + +[case testNarrowingLenHomogeneousTuple] +from typing import Tuple + +x: Tuple[int, ...] +if len(x) == 3: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int]" +else: + reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +if len(x) != 3: + reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenTypeUnaffected] +from typing import Union, List + +x: Union[str, List[int]] +if len(x) == 3: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.list[builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.list[builtins.int]]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenAnyListElseNotAffected] +from typing import Any + +def f(self, value: Any) -> Any: + if isinstance(value, list) and len(value) == 0: + reveal_type(value) # N: Revealed type is "builtins.list[Any]" + return value + reveal_type(value) # N: Revealed type is "Any" + return None +[builtins fixtures/len.pyi] + +[case testNarrowingLenMultiple] +from typing import Tuple, Union + +VarTuple = Union[Tuple[int, int], Tuple[int, int, int]] + +x: VarTuple +y: VarTuple +if len(x) == len(y) == 3: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int]" + reveal_type(y) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenFinal] +from typing import Tuple, Union +from typing_extensions import Final + +VarTuple = Union[Tuple[int, int], Tuple[int, int, int]] + +x: VarTuple +fin: Final = 3 +if len(x) == fin: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenGreaterThan] +from typing import Tuple, Union + +VarTuple = Union[Tuple[int], Tuple[int, int], Tuple[int, int, int]] + +x: VarTuple +if len(x) > 1: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int]" + +if len(x) < 2: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" + +if len(x) >= 2: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int]" + +if len(x) <= 2: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int], Tuple[builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenBothSidesUnionTuples] +from typing import Tuple, Union + +VarTuple = Union[ + Tuple[int], + Tuple[int, int], + Tuple[int, int, int], + Tuple[int, int, int, int], +] + +x: VarTuple +if 2 <= len(x) <= 3: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int], Tuple[builtins.int, builtins.int, builtins.int, builtins.int]]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenGreaterThanHomogeneousTupleShort] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple + +VarTuple = Tuple[int, ...] + +x: VarTuple +if len(x) < 3: + reveal_type(x) # N: Revealed type is "Union[Tuple[()], Tuple[builtins.int], Tuple[builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]" +reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenBiggerThanHomogeneousTupleLong] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple + +VarTuple = Tuple[int, ...] + +x: VarTuple +if len(x) < 30: + reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +else: + reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenBothSidesHomogeneousTuple] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple + +x: Tuple[int, ...] +if 1 < len(x) < 4: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[()], Tuple[builtins.int], Tuple[builtins.int, builtins.int, builtins.int, builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]]" +reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenUnionTupleUnreachable] +# flags: --warn-unreachable +from typing import Tuple, Union + +x: Union[Tuple[int, int], Tuple[int, int, int]] +if len(x) >= 4: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" + +if len(x) < 2: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenMixedTypes] +from typing import Tuple, List, Union + +x: Union[Tuple[int, int], Tuple[int, int, int], List[int]] +a = b = c = 0 +if len(x) == 3: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int, builtins.int], builtins.list[builtins.int]]" + a, b, c = x +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], builtins.list[builtins.int]]" + a, b = x + +if len(x) != 3: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], builtins.list[builtins.int]]" + a, b = x +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int, builtins.int], builtins.list[builtins.int]]" + a, b, c = x +[builtins fixtures/len.pyi] + +[case testNarrowingLenTypeVarTupleEquals] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def foo(x: Tuple[int, Unpack[Ts], str]) -> None: + if len(x) == 5: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + + if len(x) != 5: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenTypeVarTupleGreaterThan] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def foo(x: Tuple[int, Unpack[Ts], str]) -> None: + if len(x) > 5: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + reveal_type(x[5]) # N: Revealed type is "builtins.object" + reveal_type(x[-6]) # N: Revealed type is "builtins.object" + reveal_type(x[-1]) # N: Revealed type is "builtins.str" + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + + if len(x) < 5: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + x[5] # E: Tuple index out of range \ + # N: Variadic tuple can have length 5 + x[-6] # E: Tuple index out of range \ + # N: Variadic tuple can have length 5 + x[2] # E: Tuple index out of range \ + # N: Variadic tuple can have length 2 + x[-3] # E: Tuple index out of range \ + # N: Variadic tuple can have length 2 +[builtins fixtures/len.pyi] + +[case testNarrowingLenTypeVarTupleUnreachable] +# flags: --warn-unreachable +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def foo(x: Tuple[int, Unpack[Ts], str]) -> None: + if len(x) == 1: + reveal_type(x) # E: Statement is unreachable + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + + if len(x) != 1: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + else: + reveal_type(x) # E: Statement is unreachable + +def bar(x: Tuple[int, Unpack[Ts], str]) -> None: + if len(x) >= 2: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + else: + reveal_type(x) # E: Statement is unreachable + + if len(x) < 2: + reveal_type(x) # E: Statement is unreachable + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenVariadicTupleEquals] +from typing import Tuple +from typing_extensions import Unpack + +def foo(x: Tuple[int, Unpack[Tuple[float, ...]], str]) -> None: + if len(x) == 4: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.float, builtins.float, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + + if len(x) != 4: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.float, builtins.float, builtins.str]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenVariadicTupleGreaterThan] +from typing import Tuple +from typing_extensions import Unpack + +def foo(x: Tuple[int, Unpack[Tuple[float, ...]], str]) -> None: + if len(x) > 3: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.float, builtins.float, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.str], Tuple[builtins.int, builtins.float, builtins.str]]" + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + + if len(x) < 3: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.float, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenVariadicTupleUnreachable] +# flags: --warn-unreachable +from typing import Tuple +from typing_extensions import Unpack + +def foo(x: Tuple[int, Unpack[Tuple[float, ...]], str]) -> None: + if len(x) == 1: + reveal_type(x) # E: Statement is unreachable + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + + if len(x) != 1: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + else: + reveal_type(x) # E: Statement is unreachable + +def bar(x: Tuple[int, Unpack[Tuple[float, ...]], str]) -> None: + if len(x) >= 2: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" + else: + reveal_type(x) # E: Statement is unreachable + + if len(x) < 2: + reveal_type(x) # E: Statement is unreachable + else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenBareExpressionPrecise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple + +x: Tuple[int, ...] +assert x +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenBareExpressionTypeVarTuple] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def test(*xs: Unpack[Ts]) -> None: + assert xs + xs[0] # OK +[builtins fixtures/len.pyi] + +[case testNarrowingLenBareExpressionWithNonePrecise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple, Optional + +x: Optional[Tuple[int, ...]] +if x: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[()], None]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenBareExpressionWithNoneImprecise] +from typing import Tuple, Optional + +x: Optional[Tuple[int, ...]] +if x: + reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.tuple[builtins.int, ...], None]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenMixWithAnyPrecise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Any + +x: Any +if isinstance(x, (list, tuple)) and len(x) == 0: + reveal_type(x) # N: Revealed type is "Union[Tuple[()], builtins.list[Any]]" +else: + reveal_type(x) # N: Revealed type is "Any" +reveal_type(x) # N: Revealed type is "Any" + +x1: Any +if isinstance(x1, (list, tuple)) and len(x1) > 1: + reveal_type(x1) # N: Revealed type is "Union[Tuple[Any, Any, Unpack[builtins.tuple[Any, ...]]], builtins.list[Any]]" +else: + reveal_type(x1) # N: Revealed type is "Any" +reveal_type(x1) # N: Revealed type is "Any" +[builtins fixtures/len.pyi] + +[case testNarrowingLenMixWithAnyImprecise] +from typing import Any + +x: Any +if isinstance(x, (list, tuple)) and len(x) == 0: + reveal_type(x) # N: Revealed type is "Union[Tuple[()], builtins.list[Any]]" +else: + reveal_type(x) # N: Revealed type is "Any" +reveal_type(x) # N: Revealed type is "Any" + +x1: Any +if isinstance(x1, (list, tuple)) and len(x1) > 1: + reveal_type(x1) # N: Revealed type is "Union[builtins.tuple[Any, ...], builtins.list[Any]]" +else: + reveal_type(x1) # N: Revealed type is "Any" +reveal_type(x1) # N: Revealed type is "Any" +[builtins fixtures/len.pyi] + +[case testNarrowingLenExplicitLiteralTypes] +from typing import Tuple, Union +from typing_extensions import Literal + +VarTuple = Union[ + Tuple[int], + Tuple[int, int], + Tuple[int, int, int], +] +x: VarTuple + +supported: Literal[2] +if len(x) == supported: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" + +not_supported_yet: Literal[2, 3] +if len(x) == not_supported_yet: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int], Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int], Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenUnionOfVariadicTuples] +from typing import Tuple, Union + +x: Union[Tuple[int, ...], Tuple[str, ...]] +if len(x) == 2: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.str, builtins.str]]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.tuple[builtins.int, ...], builtins.tuple[builtins.str, ...]]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenUnionOfNamedTuples] +from typing import NamedTuple, Union + +class Point2D(NamedTuple): + x: int + y: int +class Point3D(NamedTuple): + x: int + y: int + z: int + +x: Union[Point2D, Point3D] +if len(x) == 2: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.Point2D]" +else: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.int, fallback=__main__.Point3D]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenTupleSubclass] +from typing import Tuple + +class Ints(Tuple[int, ...]): + size: int + +x: Ints +if len(x) == 2: + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.Ints]" + reveal_type(x.size) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "__main__.Ints" + reveal_type(x.size) # N: Revealed type is "builtins.int" + +reveal_type(x) # N: Revealed type is "__main__.Ints" +[builtins fixtures/len.pyi] + +[case testNarrowingLenTupleSubclassCustomNotAllowed] +from typing import Tuple + +class Ints(Tuple[int, ...]): + def __len__(self) -> int: + return 0 + +x: Ints +if len(x) > 2: + reveal_type(x) # N: Revealed type is "__main__.Ints" +else: + reveal_type(x) # N: Revealed type is "__main__.Ints" +[builtins fixtures/len.pyi] + +[case testNarrowingLenTupleSubclassPreciseNotAllowed] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple + +class Ints(Tuple[int, ...]): + size: int + +x: Ints +if len(x) > 2: + reveal_type(x) # N: Revealed type is "__main__.Ints" +else: + reveal_type(x) # N: Revealed type is "__main__.Ints" +[builtins fixtures/len.pyi] + +[case testNarrowingLenUnknownLen] +from typing import Any, Tuple, Union + +x: Union[Tuple[int, int], Tuple[int, int, int]] + +n: int +if len(x) == n: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" + +a: Any +if len(x) == a: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.int, builtins.int, builtins.int]]" +[builtins fixtures/len.pyi] + +[case testNarrowingLenUnionWithUnreachable] +from typing import Union, Sequence + +def f(x: Union[int, Sequence[int]]) -> None: + if ( + isinstance(x, tuple) + and len(x) == 2 + and isinstance(x[0], int) + and isinstance(x[1], int) + ): + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[builtins fixtures/len.pyi] + +[case testNarrowingIsSubclassNoneType1] +from typing import Type, Union + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, int): + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" + else: + reveal_type(cls) # N: Revealed type is "Type[None]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsSubclassNoneType2] +from typing import Type, Union + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, type(None)): + reveal_type(cls) # N: Revealed type is "Type[None]" + else: + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsSubclassNoneType3] +from typing import Type, Union + +NoneType_ = type(None) + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, NoneType_): + reveal_type(cls) # N: Revealed type is "Type[None]" + else: + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsSubclassNoneType4] +# flags: --python-version 3.10 + +from types import NoneType +from typing import Type, Union + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, NoneType): + reveal_type(cls) # N: Revealed type is "Type[None]" + else: + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsInstanceNoIntersectionWithFinalTypeAndNoneType] +# flags: --warn-unreachable --python-version 3.10 + +from types import NoneType +from typing import final + +class X: ... +class Y: ... +@final +class Z: ... + +x: X + +if isinstance(x, (Y, Z)): + reveal_type(x) # N: Revealed type is "__main__." +if isinstance(x, (Y, NoneType)): + reveal_type(x) # N: Revealed type is "__main__.1" +if isinstance(x, (Y, Z, NoneType)): + reveal_type(x) # N: Revealed type is "__main__.2" +if isinstance(x, (Z, NoneType)): # E: Subclass of "X" and "Z" cannot exist: "Z" is final \ + # E: Subclass of "X" and "NoneType" cannot exist: "NoneType" is final + reveal_type(x) # E: Statement is unreachable + +[builtins fixtures/isinstance.pyi] + +[case testTypeNarrowingReachableNegative] +# flags: --warn-unreachable +from typing import Literal + +x: Literal[-1] + +if x == -1: + assert True + +[typing fixtures/typing-medium.pyi] +[builtins fixtures/ops.pyi] + +[case testTypeNarrowingReachableNegativeUnion] +from typing import Literal + +x: Literal[-1, 1] + +if x == -1: + reveal_type(x) # N: Revealed type is "Literal[-1]" +else: + reveal_type(x) # N: Revealed type is "Literal[1]" + +[typing fixtures/typing-medium.pyi] +[builtins fixtures/ops.pyi] + +[case testNarrowingWithIntEnum] +# mypy: strict-equality +from __future__ import annotations +from typing import Any +from enum import IntEnum + +class IE(IntEnum): + X = 1 + Y = 2 + +def f1(x: int) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + if x != IE.X: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + +def f2(x: IE) -> None: + if x == 1: + reveal_type(x) # N: Revealed type is "__main__.IE" + else: + reveal_type(x) # N: Revealed type is "__main__.IE" + +def f3(x: object) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "builtins.object" + else: + reveal_type(x) # N: Revealed type is "builtins.object" + +def f4(x: int | Any) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" + else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" + +def f5(x: int) -> None: + if x is IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + if x is not IE.X: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + +def f6(x: IE) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.Y]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingWithIntEnum2] +# mypy: strict-equality +from __future__ import annotations +from typing import Any +from enum import IntEnum, Enum + +class MyDecimal: ... + +class IE(IntEnum): + X = 1 + Y = 2 + +class IE2(IntEnum): + X = 1 + Y = 2 + +class E(Enum): + X = 1 + Y = 2 + +def f1(x: IE | MyDecimal) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.MyDecimal]" + else: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.MyDecimal]" + +def f2(x: E | bytes) -> None: + if x == E.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.E.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.E.Y], builtins.bytes]" + +def f3(x: IE | IE2) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.IE2]" + else: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.IE2]" + +def f4(x: IE | E) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + elif x == E.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.E.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.IE.Y], Literal[__main__.E.Y]]" + +def f5(x: E | str | int) -> None: + if x == E.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.E.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.E.Y], builtins.str, builtins.int]" + +def f6(x: IE | Any) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, Any]" + else: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, Any]" + +def f7(x: IE | None) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.IE.Y], None]" + +def f8(x: IE | None) -> None: + if x is None: + reveal_type(x) # N: Revealed type is "None" + elif x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.Y]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingWithStrEnum] +# mypy: strict-equality +from enum import StrEnum + +class SE(StrEnum): + A = 'a' + B = 'b' + +def f1(x: str) -> None: + if x == SE.A: + reveal_type(x) # N: Revealed type is "builtins.str" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + +def f2(x: SE) -> None: + if x == 'a': + reveal_type(x) # N: Revealed type is "__main__.SE" + else: + reveal_type(x) # N: Revealed type is "__main__.SE" + +def f3(x: object) -> None: + if x == SE.A: + reveal_type(x) # N: Revealed type is "builtins.object" + else: + reveal_type(x) # N: Revealed type is "builtins.object" + +def f4(x: SE) -> None: + if x == SE.A: + reveal_type(x) # N: Revealed type is "Literal[__main__.SE.A]" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.SE.B]" +[builtins fixtures/primitives.pyi] + +[case testConsistentNarrowingEqAndIn] +# flags: --python-version 3.10 + +# https://github.com/python/mypy/issues/17864 +def f(x: str | int) -> None: + if x == "x": + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + y = x + + if x in ["x"]: + # TODO: we should fix this reveal https://github.com/python/mypy/issues/3229 + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + y = x + z = x + z = y +[builtins fixtures/primitives.pyi] + +[case testConsistentNarrowingInWithCustomEq] +# flags: --python-version 3.10 + +# https://github.com/python/mypy/issues/17864 +class C: + def __init__(self, x: int) -> None: + self.x = x + + def __eq__(self, other: object) -> bool: + raise + # Example implementation: + # if isinstance(other, C) and other.x == self.x: + # return True + # return NotImplemented + +class D(C): + pass + +def f(x: C) -> None: + if x in [D(5)]: + reveal_type(x) # D # N: Revealed type is "__main__.C" + +f(C(5)) +[builtins fixtures/primitives.pyi] + +[case testNarrowingTypeVarNone] +# flags: --warn-unreachable + +# https://github.com/python/mypy/issues/18126 +from typing import TypeVar + +T = TypeVar("T") + +def fn_if(arg: T) -> None: + if arg is None: + return None + return None + +def fn_while(arg: T) -> None: + while arg is None: + return None + return None +[builtins fixtures/primitives.pyi] diff --git a/test-data/unit/check-native-int.test b/test-data/unit/check-native-int.test new file mode 100644 index 000000000000..2f852ca522c5 --- /dev/null +++ b/test-data/unit/check-native-int.test @@ -0,0 +1,232 @@ +[case testNativeIntBasics] +from mypy_extensions import i32, i64 + +def f(x: int) -> i32: + return i32(x) + +def g(x: i32) -> None: + pass + +reveal_type(i32(1) + i32(2)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(i64(1) + i64(2)) # N: Revealed type is "mypy_extensions.i64" +i32(1) + i64(2) # E: Unsupported operand types for + ("i32" and "i64") +i64(1) + i32(2) # E: Unsupported operand types for + ("i64" and "i32") +g(i32(2)) +g(i64(2)) # E: Argument 1 to "g" has incompatible type "i64"; expected "i32" +[builtins fixtures/dict.pyi] + +[case testNativeIntCoercions] +from mypy_extensions import i32, i64 + +def f1(x: int) -> None: pass +def f2(x: i32) -> None: pass + +a: i32 = 1 +b: i64 = 2 +c: i64 = a # E: Incompatible types in assignment (expression has type "i32", variable has type "i64") +d: i64 = i64(a) +e: i32 = b # E: Incompatible types in assignment (expression has type "i64", variable has type "i32") +f: i32 = i32(b) +g: int = a +h: int = b + +f1(1) +f1(a) +f1(b) +f2(1) +f2(g) +f2(h) +f2(a) +f2(b) # E: Argument 1 to "f2" has incompatible type "i64"; expected "i32" +[builtins fixtures/dict.pyi] + +[case testNativeIntJoins] +from typing import TypeVar, Any +from mypy_extensions import i32, i64 + +T = TypeVar('T') + +def join(x: T, y: T) -> T: return x + +n32: i32 = 0 +n64: i64 = 1 +n = 2 + +reveal_type(join(n32, n)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(join(n, n32)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(join(n64, n)) # N: Revealed type is "mypy_extensions.i64" +reveal_type(join(n, n64)) # N: Revealed type is "mypy_extensions.i64" +# i32 and i64 aren't treated as compatible +reveal_type(join(n32, n64)) # N: Revealed type is "builtins.object" +reveal_type(join(n64, n32)) # N: Revealed type is "builtins.object" + +a: Any +reveal_type(join(n, a)) # N: Revealed type is "Any" +reveal_type(join(n32, a)) # N: Revealed type is "Any" +reveal_type(join(a, n64)) # N: Revealed type is "Any" +reveal_type(join(n64, a)) # N: Revealed type is "Any" +reveal_type(join(a, n64)) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] + +[case testNativeIntMeets] +from typing import TypeVar, Callable, Any +from mypy_extensions import i32, i64 + +T = TypeVar('T') + +def f32(x: i32) -> None: pass +def f64(x: i64) -> None: pass +def f(x: int) -> None: pass +def fa(x: Any) -> None: pass + +def meet(c1: Callable[[T], None], c2: Callable[[T], None]) -> T: + pass + +reveal_type(meet(f32, f)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(meet(f, f32)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(meet(f64, f)) # N: Revealed type is "mypy_extensions.i64" +reveal_type(meet(f, f64)) # N: Revealed type is "mypy_extensions.i64" +if object(): + reveal_type(meet(f32, f64)) # N: Revealed type is "Never" +if object(): + reveal_type(meet(f64, f32)) # N: Revealed type is "Never" + +reveal_type(meet(f, fa)) # N: Revealed type is "builtins.int" +reveal_type(meet(f32, fa)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(meet(fa, f32)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(meet(f64, fa)) # N: Revealed type is "mypy_extensions.i64" +reveal_type(meet(fa, f64)) # N: Revealed type is "mypy_extensions.i64" +[builtins fixtures/dict.pyi] + +[case testNativeIntCoerceInArithmetic] +from mypy_extensions import i32, i64 + +reveal_type(i32(1) + 1) # N: Revealed type is "mypy_extensions.i32" +reveal_type(1 + i32(1)) # N: Revealed type is "mypy_extensions.i32" +reveal_type(i64(1) + 1) # N: Revealed type is "mypy_extensions.i64" +reveal_type(1 + i64(1)) # N: Revealed type is "mypy_extensions.i64" +n = int() +reveal_type(i32(1) + n) # N: Revealed type is "mypy_extensions.i32" +reveal_type(n + i32(1)) # N: Revealed type is "mypy_extensions.i32" +[builtins fixtures/dict.pyi] + +[case testNativeIntNoNarrowing] +from mypy_extensions import i32 + +x: i32 = 1 +if int(): + x = 2 + reveal_type(x) # N: Revealed type is "mypy_extensions.i32" +reveal_type(x) # N: Revealed type is "mypy_extensions.i32" + +y = 1 +if int(): + # We don't narrow an int down to i32, since they have different + # representations. + y = i32(1) + reveal_type(y) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testNativeIntFloatConversion] +from typing import TypeVar, Callable +from mypy_extensions import i32 + +x: i32 = 1.1 # E: Incompatible types in assignment (expression has type "float", variable has type "i32") +y: float = i32(1) # E: Incompatible types in assignment (expression has type "i32", variable has type "float") + +T = TypeVar('T') + +def join(x: T, y: T) -> T: return x + +reveal_type(join(x, y)) # N: Revealed type is "builtins.object" +reveal_type(join(y, x)) # N: Revealed type is "builtins.object" + +def meet(c1: Callable[[T], None], c2: Callable[[T], None]) -> T: + pass + +def ff(x: float) -> None: pass +def fi32(x: i32) -> None: pass + +if object(): + reveal_type(meet(ff, fi32)) # N: Revealed type is "Never" +if object(): + reveal_type(meet(fi32, ff)) # N: Revealed type is "Never" +[builtins fixtures/dict.pyi] + +[case testNativeIntForLoopRange] +from mypy_extensions import i64, i32 + +for a in range(i64(5)): + reveal_type(a) # N: Revealed type is "mypy_extensions.i64" + +for b in range(0, i32(5)): + reveal_type(b) # N: Revealed type is "mypy_extensions.i32" + +for c in range(i64(0), 5): + reveal_type(c) # N: Revealed type is "mypy_extensions.i64" + +for d in range(i64(0), i64(5)): + reveal_type(d) # N: Revealed type is "mypy_extensions.i64" + +for e in range(i64(0), i32(5)): + reveal_type(e) # N: Revealed type is "builtins.int" + +for f in range(0, i64(3), 2): + reveal_type(f) # N: Revealed type is "mypy_extensions.i64" + +n = 5 +for g in range(0, n, i64(2)): + reveal_type(g) # N: Revealed type is "mypy_extensions.i64" +[builtins fixtures/primitives.pyi] + +[case testNativeIntComprehensionRange] +from mypy_extensions import i64, i32 + +reveal_type([a for a in range(i64(5))]) # N: Revealed type is "builtins.list[mypy_extensions.i64]" +[reveal_type(a) for a in range(0, i32(5))] # N: Revealed type is "mypy_extensions.i32" +[builtins fixtures/primitives.pyi] + +[case testNativeIntNarrowing] +from typing import Union +from mypy_extensions import i64, i32 + +def narrow_i64(x: Union[str, i64]) -> None: + if isinstance(x, i64): + reveal_type(x) # N: Revealed type is "mypy_extensions.i64" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.str, mypy_extensions.i64]" + + if isinstance(x, str): + reveal_type(x) # N: Revealed type is "builtins.str" + else: + reveal_type(x) # N: Revealed type is "mypy_extensions.i64" + reveal_type(x) # N: Revealed type is "Union[builtins.str, mypy_extensions.i64]" + + if isinstance(x, int): + reveal_type(x) # N: Revealed type is "mypy_extensions.i64" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.str, mypy_extensions.i64]" + +def narrow_i32(x: Union[str, i32]) -> None: + if isinstance(x, i32): + reveal_type(x) # N: Revealed type is "mypy_extensions.i32" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.str, mypy_extensions.i32]" + + if isinstance(x, str): + reveal_type(x) # N: Revealed type is "builtins.str" + else: + reveal_type(x) # N: Revealed type is "mypy_extensions.i32" + reveal_type(x) # N: Revealed type is "Union[builtins.str, mypy_extensions.i32]" + + if isinstance(x, int): + reveal_type(x) # N: Revealed type is "mypy_extensions.i32" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.str, mypy_extensions.i32]" + +[builtins fixtures/primitives.pyi] diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test new file mode 100644 index 000000000000..784b9db9f66e --- /dev/null +++ b/test-data/unit/check-newsemanal.test @@ -0,0 +1,3243 @@ +-- Test cases for the new semantic analyzer + +[case testNewAnalyzerEmpty] + +[case testNewAnalyzerSimpleAssignment] +x = 1 +x.y # E: "int" has no attribute "y" +y # E: Name "y" is not defined + +[case testNewAnalyzerSimpleAnnotation] +x: int = 0 +y: str = 0 \ + # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNewAnalyzerSimpleClass] +class A: + x: int +a: A +a.x +a.y # E: "A" has no attribute "y" + +[case testNewAnalyzerErrorInClassBody] +class A: + x # E: Name "x" is not defined + +[case testNewAnalyzerTypeAnnotationForwardReference] +class A: + b: B +class B: + a: A +a: A +b: B +a.b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a.b = b +b.a = a +b.a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testNewAnalyzerTypeAnnotationCycle1] +import b +[file a.py] +import b +class A: pass +y: b.B +y() # E: "B" not callable +[file b.py] +import a +class B: pass +x: a.A +reveal_type(x) # N: Revealed type is "a.A" + +[case testNewAnalyzerTypeAnnotationCycle2] +import a +[file a.py] +from b import B +class A: pass +y: B +y() +[file b.py] +from a import A +class B: pass +x: A +x() +[out] +tmp/b.py:4: error: "A" not callable +tmp/a.py:4: error: "B" not callable + +[case testNewAnalyzerTypeAnnotationCycle3] +import b +[file a.py] +from b import bad # E: Module "b" has no attribute "bad"; maybe "bad2"? +[file b.py] +from a import bad2 # E: Module "a" has no attribute "bad2"; maybe "bad"? + +[case testNewAnalyzerTypeAnnotationCycle4] +import b +[file a.py] +from b import bad # E: Module "b" has no attribute "bad" +[file b.py] +# TODO: Could we generate an error here as well? +from a import bad +[targets a, b, a, b, a, b, a, b, __main__] + +[case testNewAnalyzerExportedValuesInImportAll] +from m import * +_ = a +_ = b +_ = c +_ = d +_e = e +_f = f # E: Name "f" is not defined +_ = _g # E: Name "_g" is not defined +reveal_type(_e) # N: Revealed type is "m.A" +[file m.py] +__all__ = ['a'] +__all__ += ('b',) +__all__.append('c') +__all__.extend(('d', 'e')) + +a = b = c = d = _g = 1 +e: 'A' +f: 'A' + +class A: ... +[builtins fixtures/module_all.pyi] + +[case testNewAnalyzerSimpleFunction] +def f(x: int) -> str: + return 'x' + +def g(x: int) -> int: + y = f(1) + return y # E: Incompatible return value type (got "str", expected "int") + +[case testNewAnalyzerSimpleMethod] +class A: + def __init__(self, x: int) -> None: + self.x = x + + def f(self) -> str: + return self.x # E: Incompatible return value type (got "int", expected "str") + + def g(self) -> int: + return self.f() # E: Incompatible return value type (got "str", expected "int") + +[case testNewAnalyzerFunctionForwardRef] +def f() -> None: + x = g(1) # E: Argument 1 to "g" has incompatible type "int"; expected "str" + reveal_type(x) # N: Revealed type is "builtins.str" + +def g(x: str) -> str: + return x + +[case testNewAnalyzerExportedImportThreePasses] +import b + +[file a.py] +from b import b1 as a2 +from b import b2 as a3 + +def a1() -> int: pass + +reveal_type(a3()) # N: Revealed type is "builtins.int" + +[file b.py] +from a import a1 as b2 +from a import a2 as b3 + +def b1() -> str: pass + +reveal_type(b3()) # N: Revealed type is "builtins.str" + +[case testNewAnalyzerBool] +reveal_type(True) # N: Revealed type is "Literal[True]?" +reveal_type(False) # N: Revealed type is "Literal[False]?" + +[case testNewAnalyzerNewTypeMultiplePasses] +import b + +[file a.py] +from typing import NewType +import b +class A: pass +N2 = NewType('N2', b.N1) +def f1(x: A) -> None: pass +def f2(x: b.N1) -> None: pass +def f3(x: N2) -> None: pass +a = A() +n1 = b.N1(a) +n2 = N2(n1) +f1(a) +f1(n1) +f1(n2) +f2(a) # E: Argument 1 to "f2" has incompatible type "A"; expected "N1" +f2(n1) +f2(n2) +f3(a) # E: Argument 1 to "f3" has incompatible type "A"; expected "N2" +f3(n1) # E: Argument 1 to "f3" has incompatible type "N1"; expected "N2" +f3(n2) + +# Test N2 etc. + +[file b.py] +from typing import NewType +import a +N1 = NewType('N1', a.A) + +[case testNewAnalyzerInheritanceForwardRef] +class C(B): + pass +class B(A): + pass +class A: + def __init__(self, x: str) -> None: pass + def f(self, x: int) -> None: pass + +C(1) # E: Argument 1 to "C" has incompatible type "int"; expected "str" +B(1) # E: Argument 1 to "B" has incompatible type "int"; expected "str" +C('').f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +B('').f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testNewAnalyzerInheritanceMROInCycle] +import a + +[file a.py] +from b import A +import b + +class B(A): + b: int + +class D(b.C): + d: int + +d = D() +reveal_type(d.a) # N: Revealed type is "builtins.int" +reveal_type(d.b) # N: Revealed type is "builtins.int" +reveal_type(d.c) # N: Revealed type is "builtins.int" +reveal_type(d.d) # N: Revealed type is "builtins.int" + +[file b.py] +from a import B + +class A: + a: int + +class C(B): + c: int + +[targets b, a, b, a, __main__] + +[case testNewAnalyzerTypedDictClass] +from mypy_extensions import TypedDict +import a +class T1(TypedDict): + x: A +class A: pass +reveal_type(T1(x=A())) # E + +[file a.py] +from mypy_extensions import TypedDict +from b import TD1 as TD2, TD3 +class T2(TD3): + x: int +reveal_type(T2(x=2)) # E + +[file b.py] +from a import TypedDict as TD1 +from a import TD2 as TD3 +[builtins fixtures/tuple.pyi] + +[out] +tmp/a.py:5: note: Revealed type is "TypedDict('a.T2', {'x': builtins.int})" +main:6: note: Revealed type is "TypedDict('__main__.T1', {'x': __main__.A})" + + +[case testNewAnalyzerTypedDictClassInheritance] +from mypy_extensions import TypedDict + +class T2(T1): + y: int + +class T1(TypedDict): + x: str + +class T3(TypedDict): + x: str + +class T4(T3): + y: A + +class A: pass + +T2(x=0, y=0) # E: Incompatible types (expression has type "int", TypedDict item "x" has type "str") +x: T2 +reveal_type(x) # N: Revealed type is "TypedDict('__main__.T2', {'x': builtins.str, 'y': builtins.int})" +y: T4 +reveal_type(y) # N: Revealed type is "TypedDict('__main__.T4', {'x': builtins.str, 'y': __main__.A})" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerRedefinitionAndDeferral1a] +import a + +[file a.py] +MYPY = False +if MYPY: + from b import x as y +x = 0 + +def y(): pass # E: Name "y" already defined on line 4 +reveal_type(y) # N: Revealed type is "builtins.int" + +y2 = y +class y2: pass # E: Name "y2" already defined on line 9 +reveal_type(y2) # N: Revealed type is "builtins.int" + +y3, y4 = y, y +if MYPY: # Tweak processing order + from b import f as y3 # E: Incompatible import of "y3" (imported name has type "Callable[[], Any]", local name has type "int") +reveal_type(y3) # N: Revealed type is "builtins.int" + +[file b.py] +from a import x + +def f(): pass + +[targets a, b, a, a.y, b.f, __main__] +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerRedefinitionAndDeferral1b] +import a + +[file a.py] +from b import x as y +x = 0 + +def y(): pass # E: Name "y" already defined on line 2 +reveal_type(y) # N: Revealed type is "builtins.int" + +y2 = y +class y2: pass # E: Name "y2" already defined on line 7 +reveal_type(y2) # N: Revealed type is "builtins.int" + +y3, y4 = y, y +from b import f as y3 # E: Incompatible import of "y3" (imported name has type "Callable[[], Any]", local name has type "int") +reveal_type(y3) # N: Revealed type is "builtins.int" + +[file b.py] +MYPY = False +if MYPY: # Tweak processing order + from a import x + +def f(): pass + +[targets b, a, b, a, b.f, a.y, __main__] + +[case testNewAnalyzerRedefinitionAndDeferral2a] +import a + +[file a.py] +MYPY = False +if MYPY: # Tweak processing order + from b import C as C2 +class C: pass +class C2: pass # E: Name "C2" already defined on line 4 + +[file b.py] +from a import C + +[case testNewAnalyzerRedefinitionAndDeferral2b] +import a + +[file a.py] +from b import C as C2 +class C: pass + +class C2: pass # E: Name "C2" already defined on line 2 +[file b.py] +MYPY = False +if MYPY: # Tweak processing order + from a import C + +[case testNewAnalyzerRedefinitionAndDeferral3] +import a + +[file a.py] +from b import f as g +def f(): pass + +a, *b = g() +class b(): pass # E: Name "b" already defined on line 4 +reveal_type(b) # N: Revealed type is "Any" + +[file b.py] +from a import f + +[case testNewAnalyzerImportStarForwardRef1] +import a + +[file a.py] +x: A +reveal_type(x) # N: Revealed type is "b.A" + +from b import * + +class A: pass # E: Name "A" already defined (possibly by an import) + +[file b.py] +class A: pass +MYPY = False +if MYPY: # Tweak processing order + from a import x + +[case testNewAnalyzerImportStarForwardRef2] +import a + +[file a.py] +x: A +reveal_type(x) # N: Revealed type is "b.A" + +MYPY = False +if MYPY: # Tweak processing order + from b import * + +class A: pass # E: Name "A" already defined (possibly by an import) + +[file b.py] +class A: pass +from a import x + +[case testNewAnalyzerClassInFunction] +def main() -> None: + x: C + class C: + def __init__(self) -> None: + self.x: A + x() # E: "C" not callable + reveal_type(x.x) # N: Revealed type is "__main__.A@8" + class A: pass + +[case testNewAnalyzerMutuallyRecursiveFunctions] +def main() -> None: + def f() -> int: + reveal_type(g()) # N: Revealed type is "builtins.str" + return int() + def g() -> str: + reveal_type(f()) # N: Revealed type is "builtins.int" + return str() + +[case testNewAnalyzerMissingNamesInFunctions] +def main() -> None: + def f() -> None: + x # E: Name "x" is not defined + class C: + x # E: Name "x" is not defined + +[case testNewAnalyzerCyclicDefinitions] +# flags: --disable-error-code used-before-def +gx = gy # E: Cannot resolve name "gy" (possible cyclic definition) +gy = gx +def main() -> None: + class C: + def meth(self) -> None: + lx = ly # E: Cannot resolve name "ly" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + ly = lx + +[case testNewAnalyzerCyclicDefinitionCrossModule] +import b +[file a.py] +import b +x = b.x # E: Cannot resolve attribute "x" (possible cyclic definition) \ + # E: Module has no attribute "x" +[file b.py] +import a +x = a.x +[builtins fixtures/module.pyi] + +[case testNewAnalyzerMutuallyRecursiveOverloadedFunctions] +from typing import overload, Union + +def main() -> None: + @overload + def f(x: int) -> int: ... + @overload + def f(x: str) -> str: ... + def f(x: Union[int, str]) -> Union[int, str]: + reveal_type(g(str())) # N: Revealed type is "builtins.str" + return x + @overload + def g(x: int) -> int: ... + @overload + def g(x: str) -> str: ... + def g(x: Union[int, str]) -> Union[int, str]: + reveal_type(f(int())) # N: Revealed type is "builtins.int" + return float() # E: Incompatible return value type (got "float", expected "Union[int, str]") + +[case testNewAnalyzerNestedClassInMethod] +class C: + class D: + def meth(self) -> None: + x: Out.In + reveal_type(x.t) # N: Revealed type is "builtins.int" + class Out: + class In: + def meth(self) -> None: + self.t: int + +[case testNewAnalyzerDeeplyNestedFunctions] +class Out: + class In: + def meth(self) -> None: + x: C.D + reveal_type(x.t) # N: Revealed type is "__main__.Test@10" + class C: + class D: + def meth(self) -> None: + self.t: Test + class Test: + def test(self) -> None: + def one() -> int: + reveal_type(other()) # N: Revealed type is "builtins.str" + return int() + def other() -> str: + reveal_type(one()) # N: Revealed type is "builtins.int" + return str() + +[case testNewAnalyzerNestedClass1] +class A: + class B: + x: int + + def __init__(self, x: int) -> None: + self.x = x + + def f(self) -> str: + return self.x # E: Incompatible return value type (got "int", expected "str") + +b: A.B +b = A.B('') # E: Argument 1 to "B" has incompatible type "str"; expected "int" +reveal_type(b) # N: Revealed type is "__main__.A.B" +reveal_type(b.x) # N: Revealed type is "builtins.int" +reveal_type(b.f()) # N: Revealed type is "builtins.str" + +[case testNewAnalyzerNestedClass2] +class A: + class B: + x: int + + def __init__(self, x: int) -> None: + self.x = x + + def f(self) -> str: + return self.x # E: Incompatible return value type (got "int", expected "str") + +b: A.B +b = A.B('') # E: Argument 1 to "B" has incompatible type "str"; expected "int" +reveal_type(b) # N: Revealed type is "__main__.A.B" +reveal_type(b.x) # N: Revealed type is "builtins.int" +reveal_type(b.f()) # N: Revealed type is "builtins.str" +[case testNewAnalyzerGenerics] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + + def get(self) -> T: + return self.x + +c: C[int] +c2: C[int, str] # E: "C" expects 1 type argument, but 2 given +c3: C +c = C('') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +reveal_type(c.get()) # N: Revealed type is "builtins.int" +reveal_type(c2) # N: Revealed type is "__main__.C[Any]" +reveal_type(c3) # N: Revealed type is "__main__.C[Any]" +[case testNewAnalyzerGenericsTypeVarForwardRef] +from typing import TypeVar, Generic + +class C(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + + def get(self) -> T: + return self.x + +T = TypeVar('T') + +c: C[int] +reveal_type(c) # N: Revealed type is "__main__.C[builtins.int]" +c = C('') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +reveal_type(c.get()) # N: Revealed type is "builtins.int" + +[case testNewAnalyzerTypeAlias] +from typing import Union, TypeVar, Generic + +T = TypeVar('T') +S = TypeVar('S') +class D(Generic[T, S]): pass + +class C: pass + +C2 = C +U = Union[C, int] +G = D[T, C] + +c: C2 +reveal_type(c) # N: Revealed type is "__main__.C" +u: U +reveal_type(u) # N: Revealed type is "Union[__main__.C, builtins.int]" +g: G[int] +reveal_type(g) # N: Revealed type is "__main__.D[builtins.int, __main__.C]" +[case testNewAnalyzerTypeAlias2] +from typing import Union + +class C(D): pass + +A = Union[C, int] +x: A +reveal_type(x) # N: Revealed type is "Union[__main__.C, builtins.int]" + +class D: pass + +[case testNewAnalyzerBuiltinTypeAliases] +from typing import List + +x: List[C] +reveal_type(x) # N: Revealed type is "builtins.list[__main__.C]" + +class C: pass +[builtins fixtures/list.pyi] + +[case testNewAnalyzerVersionCheck] +import sys + +if sys.version_info[0] < 2: + 1() + import nonexistent +else: + def f(x: int) -> None: pass + +f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +def g() -> None: + if sys.version_info[0] < 3: + import nonexistent2 + else: + 1() # E: "int" not callable +[builtins fixtures/ops.pyi] + +[case testNewAnalyzerVersionCheck2] +import sys + +assert sys.version_info[0] == 3 +1() # E: "int" not callable +assert sys.version_info[0] < 3 +''() +[builtins fixtures/ops.pyi] + +[case testNewAnalyzerOverload] +from typing import overload, Union + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: + return 1.0 # E: Incompatible return value type (got "float", expected "Union[int, str]") + +f(1) +f('') +f(1.0) # E: No overload variant of "f" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def f(x: int) -> int \ + # N: def f(x: str) -> str + +[case testNewAnalyzerOverload2] +from typing import overload, Union + +class A: + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... + def f(self, x: Union[int, str]) -> Union[int, str]: + return 1.0 # E: Incompatible return value type (got "float", expected "Union[int, str]") + +a = A() +a.f(1) +a.f('') +a.f(1.0) # E: No overload variant of "f" of "A" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def f(self, x: int) -> int \ + # N: def f(self, x: str) -> str + +[case testNewAnalyzerPromotion] +def f(x: float) -> None: pass +y: int +f(y) +f(1) +[builtins fixtures/primitives.pyi] + +[case testNewAnalyzerFunctionDecorator] +# flags: --disable-error-code used-before-def +from typing import Callable + +@dec +def f1(x: int) -> int: + return '' # E: Incompatible return value type (got "str", expected "int") + +def dec(f: Callable[[int], int]) -> Callable[[str], str]: ... + +@dec +def f2(x: int) -> int: + return '' # E: Incompatible return value type (got "str", expected "int") + +f1(1) # E: Argument 1 to "f1" has incompatible type "int"; expected "str" +reveal_type(f1('')) # N: Revealed type is "builtins.str" +f2(1) # E: Argument 1 to "f2" has incompatible type "int"; expected "str" + +[case testNewAnalyzerTypeVarForwardReference] +# flags: --disable-error-code used-before-def +from typing import TypeVar, Generic + +T = TypeVar('T') +XY = TypeVar('XY', X, Y) + +class C(Generic[T]): pass + +class D(C[XY], Generic[XY]): pass + +class X: pass +class Y: pass + +x: D[int] # E: Value of type variable "XY" of "D" cannot be "int" +y: D[Y] + +[case testNewAnalyzerTypeVarForwardReference2] +from typing import TypeVar, Generic + +T = TypeVar('T') +XY = TypeVar('XY', 'X', 'Y') + +class C(Generic[T]): pass + +class D(C[XY]): pass + +class X: pass +class Y: pass + +x: D[int] # E: Value of type variable "XY" of "D" cannot be "int" +y: D[Y] + +[case testNewAnalyzerTypeVarForwardReferenceValuesDeferred] +from typing import TypeVar, Generic + +T = TypeVar('T') +XY = TypeVar('XY', 'X', 'Y') + +class C(Generic[T]): pass + +class D(C[XY], Generic[XY]): pass + +class X(Defer): pass +class Y(Defer): pass +class Defer: ... + +x: D[int] # E: Value of type variable "XY" of "D" cannot be "int" +y: D[Y] +[builtins fixtures/list.pyi] + +[case testNewAnalyzerTypeVarForwardReferenceBoundDeferred] +from typing import TypeVar, Generic + +T = TypeVar('T') +TY = TypeVar('TY', bound='Y') + +class C(Generic[T]): pass + +class D(C[TY], Generic[TY]): pass + +class Y(Defer): pass +class Defer: ... + +x: D[int] # E: Type argument "int" of "D" must be a subtype of "Y" +y: D[Y] + +[case testNewAnalyzerTypeVarForwardReferenceErrors] +from typing import TypeVar, Generic + +class C(Generic[T]): + def __init__(self, x: T) -> None: ... +def func(x: U) -> U: ... + +U = TypeVar('U', asdf, asdf) # E: Name "asdf" is not defined +T = TypeVar('T', bound='asdf') # E: Name "asdf" is not defined + +reveal_type(C) # N: Revealed type is "def [T <: Any] (x: T`1) -> __main__.C[T`1]" +reveal_type(func) # N: Revealed type is "def [U in (Any, Any)] (x: U`-1) -> U`-1" + +[case testNewAnalyzerSubModuleInCycle] +import a +[file a.py] +MYPY = False +if MYPY: + from b.c import x +[file b/__init__.pyi] +import b.c +[file b/c.pyi] +x = 0 +import a + +[case testNewAnalyzerBaseClassSelfReference] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): pass + +class C(A[C]): + pass + +class D(A['D']): + pass + +a1: A[C] = C() +a2: A[D] = C() \ + # E: Incompatible types in assignment (expression has type "C", variable has type "A[D]") + +[case testNewAnalyzerTypeVarBoundForwardRef] +from typing import TypeVar + +T = TypeVar('T', bound='C') + +class C: pass +class D(C): pass +class E: pass + +def f(x: T) -> T: + return x + +reveal_type(f(D())) # N: Revealed type is "__main__.D" +f(E()) # E: Value of type variable "T" of "f" cannot be "E" + +[case testNewAnalyzerNameExprRefersToIncompleteType] +import a + +[file a.py] +from b import f + +class C(D): pass +class D: pass + +[file b.py] +from a import C +reveal_type(C()) # N: Revealed type is "a.C" +def f(): pass + +[case testNewAnalyzerMemberExprRefersToIncompleteType] +import a + +[file a.py] +from b import f + +class C(D): pass +class D: pass + +[file b.py] +import a +reveal_type(a.C()) # N: Revealed type is "a.C" +def f(): pass + +[case testNewAnalyzerNamedTupleCall] +from typing import NamedTuple + + +class Other: pass +In = NamedTuple('In', [('s', str), ('t', Other)]) +Out = NamedTuple('Out', [('x', In), ('y', Other)]) +o: Out +i: In +reveal_type(o) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other, fallback=__main__.In], __main__.Other, fallback=__main__.Out]" +reveal_type(o.x) # N: Revealed type is "Tuple[builtins.str, __main__.Other, fallback=__main__.In]" +reveal_type(o.y) # N: Revealed type is "__main__.Other" +reveal_type(o.x.t) # N: Revealed type is "__main__.Other" +reveal_type(i.t) # N: Revealed type is "__main__.Other" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNamedTupleClass] +from typing import NamedTuple + +o: Out +i: In + +class Out(NamedTuple): + x: In + y: Other + +reveal_type(o) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other, fallback=__main__.In], __main__.Other, fallback=__main__.Out]" +reveal_type(o.x) # N: Revealed type is "Tuple[builtins.str, __main__.Other, fallback=__main__.In]" +reveal_type(o.y) # N: Revealed type is "__main__.Other" +reveal_type(o.x.t) # N: Revealed type is "__main__.Other" +reveal_type(i.t) # N: Revealed type is "__main__.Other" + +class In(NamedTuple): + s: str + t: Other +class Other: pass +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNamedTupleCallNested] +from typing import NamedTuple + +o: C.Out +i: C.In + +reveal_type(o) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.C.Other, fallback=__main__.C.In], __main__.C.Other, fallback=__main__.C.Out]" +reveal_type(o.x) # N: Revealed type is "Tuple[builtins.str, __main__.C.Other, fallback=__main__.C.In]" +reveal_type(o.y) # N: Revealed type is "__main__.C.Other" +reveal_type(o.x.t) # N: Revealed type is "__main__.C.Other" +reveal_type(i.t) # N: Revealed type is "__main__.C.Other" + +class C: + In = NamedTuple('In', [('s', str), ('t', Other)]) + Out = NamedTuple('Out', [('x', In), ('y', Other)]) + class Other: pass +[builtins fixtures/tuple.pyi] + + +[case testNewAnalyzerNamedTupleClassNested] +from typing import NamedTuple + +o: C.Out +i: C.In + +reveal_type(o) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.C.Other, fallback=__main__.C.In], __main__.C.Other, fallback=__main__.C.Out]" +reveal_type(o.x) # N: Revealed type is "Tuple[builtins.str, __main__.C.Other, fallback=__main__.C.In]" +reveal_type(o.y) # N: Revealed type is "__main__.C.Other" +reveal_type(o.x.t) # N: Revealed type is "__main__.C.Other" +reveal_type(i.t) # N: Revealed type is "__main__.C.Other" + +class C: + class Out(NamedTuple): + x: C.In + y: C.Other + class In(NamedTuple): + s: str + t: C.Other + class Other: pass +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNamedTupleCallNestedMethod] +from typing import NamedTuple + +class C: + def get_tuple(self) -> None: + Out = NamedTuple('Out', [('x', 'In'), ('y', 'Other')]) + In = NamedTuple('In', [('s', str), ('t', 'Other')]) + class Other: pass + self.o: Out + +c = C() +reveal_type(c.o) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other@7, fallback=__main__.C.In@6], __main__.Other@7, fallback=__main__.C.Out@5]" +reveal_type(c.o.x) # N: Revealed type is "Tuple[builtins.str, __main__.Other@7, fallback=__main__.C.In@6]" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNamedTupleClassNestedMethod] +from typing import NamedTuple + +class C: + def get_tuple(self) -> None: + class Out(NamedTuple): + x: In + y: Other + def method(self) -> In: ... + class In(NamedTuple): + s: str + t: Other + class Other: pass + self.o: Out + +c = C() +reveal_type(c.o) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other@12, fallback=__main__.C.In@9], __main__.Other@12, fallback=__main__.C.Out@5]" +reveal_type(c.o.x) # N: Revealed type is "Tuple[builtins.str, __main__.Other@12, fallback=__main__.C.In@9]" +reveal_type(c.o.method()) # N: Revealed type is "Tuple[builtins.str, __main__.Other@12, fallback=__main__.C.In@9]" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNamedTupleClassForwardMethod] +from typing import NamedTuple + +n: NT +reveal_type(n.get_other()) # N: Revealed type is "Tuple[builtins.str, fallback=__main__.Other]" +reveal_type(n.get_other().s) # N: Revealed type is "builtins.str" + +class NT(NamedTuple): + x: int + y: int + def get_other(self) -> Other: pass + +class Other(NamedTuple): + s: str +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNamedTupleSpecialMethods] +from typing import NamedTuple + +class Other: pass +In = NamedTuple('In', [('s', str), ('t', Other)]) +Out = NamedTuple('Out', [('x', In), ('y', Other)]) +class SubO(Out): pass + +o: SubO + +reveal_type(SubO._make) # N: Revealed type is "def (iterable: typing.Iterable[Any]) -> Tuple[Tuple[builtins.str, __main__.Other, fallback=__main__.In], __main__.Other, fallback=__main__.SubO]" +reveal_type(o._replace(y=Other())) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other, fallback=__main__.In], __main__.Other, fallback=__main__.SubO]" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNamedTupleBaseClass] +from typing import NamedTuple +class Other: pass +class In(NamedTuple): + s: str + t: Other +class Out(NamedTuple('Out', [('x', In), ('y', Other)])): + pass + +o: Out +reveal_type(o) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other, fallback=__main__.In], __main__.Other, fallback=__main__.Out]" +reveal_type(o.x) # N: Revealed type is "Tuple[builtins.str, __main__.Other, fallback=__main__.In]" +reveal_type(o.x.t) # N: Revealed type is "__main__.Other" +reveal_type(Out._make) # N: Revealed type is "def (iterable: typing.Iterable[Any]) -> Tuple[Tuple[builtins.str, __main__.Other, fallback=__main__.In], __main__.Other, fallback=__main__.Out]" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerIncompleteRefShadowsBuiltin1] +import a + +[file a.py] +from typing import TypeVar, Generic + +from b import C as int + +x: int[str] + +reveal_type(x) # N: Revealed type is "a.C[builtins.str]" + +T = TypeVar('T') +class C(Generic[T]): pass + +[file b.py] +from a import C + +[case testNewAnalyzerIncompleteRefShadowsBuiltin2] +import b + +[file a.py] +import b + +int = b.C + +class C: pass + +x: int +reveal_type(x) # N: Revealed type is "b.C" + +[file b.py] +import a + +int = a.C + +class C: pass + +x: int +reveal_type(x) # N: Revealed type is "a.C" + +[case testNewAnalyzerNamespaceCompleteness] +import a + +[file a.py] +import b + +x: b.C + +[file b.py] +from c import * +class C: pass + +[file c.py] +import a +from b import C + +[targets c, b, a, c, b, __main__] + +[case testNewAnalyzerImportOverExistingInCycle] +import a +[file a.py] +C = 1 +from b import C # E: Incompatible import of "C" (imported name has type "Type[C]", local name has type "int") + +[file b.py] +import a + +class C(B): ... +class B: ... + +[case testNewAnalyzerImportOverExistingInCycleStar1] +import a +[file a.py] +C = 1 +MYPY = False +if MYPY: # Tweak processing order + from b import * # E: Incompatible import of "C" (imported name has type "Type[C]", local name has type "int") + +[file b.py] +import a + +class C(B): ... +class B: ... + +[case testNewAnalyzerImportOverExistingInCycleStar2] +import a +[file a.py] +C = 1 +from b import * # E: Incompatible import of "C" (imported name has type "Type[C]", local name has type "int") + +[file b.py] +MYPY = False +if MYPY: # Tweak processing order + import a + +class C(B): ... +class B: ... + +[case testNewAnalyzerIncompleteFixture] +from typing import Tuple + +x: Tuple[int] # E: Name "tuple" is not defined +[builtins fixtures/complex.pyi] + +[case testNewAnalyzerMetaclass1] +class A(metaclass=B): + pass + +class B(type): + def f(cls) -> int: + return 0 + +reveal_type(A.f()) # N: Revealed type is "builtins.int" + +[case testNewAnalyzerMetaclass2] +class B(type): + def f(cls) -> int: + return 0 + +class C: pass + +class A(metaclass=B): + pass + +class AA(metaclass=C): # E: Metaclasses not inheriting from "type" are not supported + pass + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +[case testNewAnalyzerMetaclassPlaceholder] +class B(C): pass + +class A(metaclass=B): + pass + +class C(type): + def f(cls) -> int: + return 0 + +reveal_type(A.f()) # N: Revealed type is "builtins.int" + +[case testNewAnalyzerMetaclassSix1] +import six + +class A(six.with_metaclass(B)): + pass + +class B(type): + def f(cls) -> int: + return 0 + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerMetaclassSix2] +import six + +@six.add_metaclass(B) +class A: + pass + +class B(type): + def f(cls) -> int: + return 0 + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerMetaclassSix3] +import six + +class A(six.with_metaclass(B, Defer)): + pass + +class B(type): + def f(cls) -> int: + return 0 + +class Defer: + x: str + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +reveal_type(A.x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerMetaclassSix4] +import six + +class B(type): + def f(cls) -> int: + return 0 + +class A(six.with_metaclass(B, Defer)): + pass + +class Defer: + x: str + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +reveal_type(A.x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerMetaclassFuture1] +import future.utils + +class A(future.utils.with_metaclass(B)): + pass + +class B(type): + def f(cls) -> int: + return 0 + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerMetaclassFuture3] +import future.utils + +class A(future.utils.with_metaclass(B, Defer)): + pass + +class B(type): + def f(cls) -> int: + return 0 + +class Defer: + x: str + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +reveal_type(A.x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerMetaclassFuture4] +# flags: --disable-error-code used-before-def +import future.utils + +class B(type): + def f(cls) -> int: + return 0 + +reveal_type(A.f()) # N: Revealed type is "builtins.int" +reveal_type(A.x) # N: Revealed type is "builtins.str" + +class A(future.utils.with_metaclass(B, Defer)): + pass + +class Defer: + x: str +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerFinalDefiningModuleVar] +from typing import Final + +class D(C): ... +class C: ... + +x: Final = C() +y: Final[C] = D() +bad: Final[D] = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") + +reveal_type(x) # N: Revealed type is "__main__.C" +reveal_type(y) # N: Revealed type is "__main__.C" +[case testNewAnalyzerFinalDefiningInstanceVar] +from typing import Final + +class D: ... +class E(C): ... + +class C: + def __init__(self, x: D) -> None: + self.x: Final = x + self.y: Final[C] = E(D()) +reveal_type(C(D()).x) # N: Revealed type is "__main__.D" +reveal_type(C(D()).y) # N: Revealed type is "__main__.C" +[case testNewAnalyzerFinalReassignModuleVar] +from typing import Final + +class A: ... + +x: Final = A() +x = A() # E: Cannot assign to final name "x" + +x2: Final = A() +def f2() -> None: + global x2 + def f() -> None: + g() + x2 = A() # E: Cannot assign to final name "x2" + def g() -> None: + f() + +[case testNewAnalyzerFinalReassignModuleReexport] +import a +[file a.py] +from b import ID, A + +class C(A): ... +ID = C() # E: Cannot assign to final name "ID" +[file b.py] +from typing import Final +from a import C + +class A: + x: C + +ID: Final = A() + +[case testNewAnalyzerFinalOverrideInSubclass] +from typing import Final + +class B: + def __init__(self, x: int) -> None: + self.x: Final = x + +class C(B): + x = 1 # E: Cannot assign to final name "x" + +[case testNewAnalyzerAssignmentAfterStarImport] +import a +[file a.py] +from b import * + +x = 1 +def f(): ... +[file b.py] +from a import f +x: int + +[case testNewAnalyzerClassLevelImport] +# flags: --ignore-missing-imports +class Test: + import a + def __init__(self) -> None: + some_module = self.a + +[case testNewAnalyzerAliasToNotReadyClass] +import a +[file a.py] +from b import B + +x: A +A = B +[file b.py] +from typing import List +from a import x + +class B(List[B]): pass + +reveal_type(x[0][0]) # N: Revealed type is "b.B" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClass2] +from typing import List + +x: A + +class A(List[B]): pass +B = A + +reveal_type(x[0][0]) # N: Revealed type is "__main__.A" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClass3] +# flags: --disable-error-code used-before-def +from typing import List + +x: B +B = A +A = C +class C(List[B]): pass + +reveal_type(x[0][0]) # N: Revealed type is "__main__.C" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyNestedClass] +import a +[file a.py] +from b import Out + +x: A +A = Out.B +[file b.py] +from typing import List +from a import x + +class Out: + class B(List[B]): pass + +reveal_type(x[0][0]) # N: Revealed type is "b.Out.B" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyNestedClass2] +from typing import List + +x: Out.A + +class Out: + class A(List[B]): pass +B = Out.A + +reveal_type(x[0][0]) # N: Revealed type is "__main__.Out.A" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClassGeneric] +import a +[file a.py] +from typing import Tuple +from b import B, T + +x: A[int] +A = B[Tuple[T, T]] +[file b.py] +from typing import List, Generic, TypeVar +from a import x + +class B(List[B], Generic[T]): pass +T = TypeVar('T') +reveal_type(x) # N: Revealed type is "b.B[Tuple[builtins.int, builtins.int]]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClassInGeneric] +import a +[file a.py] +from typing import Tuple +from b import B + +x: A +A = Tuple[B, B] +[file b.py] +from typing import List +from a import x + +class B(List[B]): pass + +reveal_type(x) # N: Revealed type is "Tuple[b.B, b.B]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyClassDoubleGeneric] +from typing import List, TypeVar, Union + +T = TypeVar('T') + +x: B[int] +A = Union[int, T] +B = A[List[T]] +class C(List[B[int]]): pass + +y: C +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.list[builtins.int]]" +reveal_type(y[0]) # N: Revealed type is "Union[builtins.int, builtins.list[builtins.int]]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerForwardAliasFromUnion] +from typing import Union, List + +A = Union['B', 'C'] + +class D: + x: List[A] + + def test(self) -> None: + reveal_type(self.x[0].y) # N: Revealed type is "builtins.int" + +class B: + y: int +class C: + y: int +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyTwoDeferrals] +# flags: --disable-error-code used-before-def +from typing import List + +x: B +B = List[C] +A = C +class C(List[A]): pass + +reveal_type(x) # N: Revealed type is "builtins.list[__main__.C]" +reveal_type(x[0][0]) # N: Revealed type is "__main__.C" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyDirectBase] +# flags: --disable-error-code used-before-def +from typing import List + +def test() -> None: + x: B + B = List[C] + class C(B): pass + + reveal_type(x) + reveal_type(x[0][0]) +[builtins fixtures/list.pyi] +[out] +main:5: error: Cannot resolve name "B" (possible cyclic definition) +main:5: note: Recursive types are not allowed at function scope +main:6: error: Cannot resolve name "B" (possible cyclic definition) +main:6: note: Recursive types are not allowed at function scope +main:6: error: Cannot resolve name "C" (possible cyclic definition) +main:9: note: Revealed type is "Any" +main:10: note: Revealed type is "Any" + +[case testNewAnalyzerAliasToNotReadyTwoDeferralsFunction] +# flags: --disable-error-code used-before-def +import a +[file a.py] +from typing import List +from b import D + +def f(x: B) -> List[B]: ... +B = List[C] +A = C +class C(List[A]): pass +[file b.py] +from a import f +class D: ... +reveal_type(f) # N: Revealed type is "def (x: builtins.list[a.C]) -> builtins.list[builtins.list[a.C]]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyDirectBaseFunction] +# flags: --disable-error-code used-before-def +import a +[file a.py] +from typing import List +from b import D + +def f(x: B) -> List[B]: ... +B = List[C] +class C(B): pass + +[file b.py] +from a import f +class D: ... +reveal_type(f) # N: Revealed type is "def (x: builtins.list[a.C]) -> builtins.list[builtins.list[a.C]]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasToNotReadyMixed] +from typing import List, Union +x: A + +class B(List[A]): pass +class C(List[A]): pass + +A = Union[B, C] + +reveal_type(x) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(x[0]) # N: Revealed type is "Union[__main__.B, __main__.C]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerTrickyAliasInFuncDef] +import a +[file a.py] +from b import B +def func() -> B: ... +reveal_type(func()) # N: Revealed type is "builtins.list[Tuple[b.C, b.C]]" + +[file b.py] +from typing import List, Tuple +from a import func + +class A: ... +class C(A): ... +B = List[Tuple[C, C]] +[builtins fixtures/list.pyi] + +[case testNewAnalyzerListComprehension] +from typing import List +class A: pass +class B: pass +a: List[A] +a = [x for x in a] +b: List[B] = [x for x in a] # E: List comprehension has incompatible type List[A]; expected List[B] +[builtins fixtures/for.pyi] + +[case testNewAnalyzerDictionaryComprehension] +from typing import Dict, List, Tuple +abd: Dict[A, B] +abl: List[Tuple[A, B]] +abd = {a: b for a, b in abl} +x: Dict[B, A] = {a: b for a, b in abl} # E: Key expression in dictionary comprehension has incompatible type "A"; expected type "B" \ + # E: Value expression in dictionary comprehension has incompatible type "B"; expected type "A" +y: A = {a: b for a, b in abl} # E: Incompatible types in assignment (expression has type "Dict[A, B]", variable has type "A") +class A: pass +class B: pass +[builtins fixtures/dict.pyi] + +[case testNewAnalyzerTypeArgBoundCheck] +from typing import TypeVar, Generic + +class F(E): pass +class E: pass +T = TypeVar('T', bound=E) +class C(Generic[T]): pass + +class D(B): pass + +x: C[D] # E: Type argument "D" of "C" must be a subtype of "E" +y: C[F] + +class B: pass + +[case testNewAnalyzerTypeArgValueRestriction] +from typing import TypeVar, Generic + +class F(E): pass +class E: pass +T = TypeVar('T', E, str) +class C(Generic[T]): pass + +class D(B): pass + +x: C[D] # E: Value of type variable "T" of "C" cannot be "D" +y: C[E] +z: C[str] + +class B: pass + +[case testNewAnalyzerTypeArgBoundCheckWithContext] +# flags: --show-error-context +import a +[file a.py] +from typing import TypeVar, Generic + +T = TypeVar('T', bound=int) +class C(Generic[T]): pass + +def f(x: C[str]) -> None: # E + y: C[str] # E +class A(C[str]): # E + z: C[str] # E + def g(self, x: C[str]) -> None: # E + a: C[str] # E +[out] +main:2: note: In module imported here: +tmp/a.py: note: In function "f": +tmp/a.py:6: error: Type argument "str" of "C" must be a subtype of "int" +tmp/a.py:7: error: Type argument "str" of "C" must be a subtype of "int" +tmp/a.py: note: In class "A": +tmp/a.py:8: error: Type argument "str" of "C" must be a subtype of "int" +tmp/a.py:9: error: Type argument "str" of "C" must be a subtype of "int" +tmp/a.py: note: In member "g" of class "A": +tmp/a.py:10: error: Type argument "str" of "C" must be a subtype of "int" +tmp/a.py:11: error: Type argument "str" of "C" must be a subtype of "int" + +[case testNewAnalyzerTypeArgBoundCheckDifferentNodes] +from typing import TypeVar, Generic, NamedTuple, NewType, Union, Any, cast, overload +from mypy_extensions import TypedDict + +T = TypeVar('T', bound=int) +class C(Generic[T]): pass +class C2(Generic[T]): pass + +A = C[str] # E: Type argument "str" of "C" must be a subtype of "int" \ + # E: Value of type variable "T" of "C" cannot be "str" +B = Union[C[str], int] # E: Type argument "str" of "C" must be a subtype of "int" +S = TypeVar('S', bound=C[str]) # E: Type argument "str" of "C" must be a subtype of "int" +U = TypeVar('U', C[str], str) # E: Type argument "str" of "C" must be a subtype of "int" +N = NamedTuple('N', [ + ('x', C[str])]) # E: Type argument "str" of "C" must be a subtype of "int" +class N2(NamedTuple): + x: C[str] # E: Type argument "str" of "C" must be a subtype of "int" +class TD(TypedDict): + x: C[str] # E: Type argument "str" of "C" must be a subtype of "int" +class TD2(TD): + y: C2[str] # E: Type argument "str" of "C2" must be a subtype of "int" +NT = NewType('NT', + C[str]) # E: Type argument "str" of "C" must be a subtype of "int" +class D( + C[str]): # E: Type argument "str" of "C" must be a subtype of "int" + pass + +TD3 = TypedDict('TD3', {'x': C[str]}) # E: Type argument "str" of "C" must be a subtype of "int" + +a: Any +for i in a: # type: C[str] # E: Type argument "str" of "C" must be a subtype of "int" + pass + +with a as w: # type: C[str] # E: Type argument "str" of "C" must be a subtype of "int" + pass + +cast(C[str], a) # E: Type argument "str" of "C" must be a subtype of "int" +C[str]() # E: Value of type variable "T" of "C" cannot be "str" + +def f(s: S, y: U) -> None: pass # No error here + +@overload +def g(x: C[str]) -> int: ... # E: Type argument "str" of "C" must be a subtype of "int" +@overload +def g(x: int) -> int: ... +def g(x: Union[C[str], int]) -> int: # E: Type argument "str" of "C" must be a subtype of "int" + y: C[object] # E: Type argument "object" of "C" must be a subtype of "int" + return 0 +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerTypeArgBoundCheckWithStrictOptional] +# flags: --config-file tmp/mypy.ini +import a + +[file b.py] +from typing import TypeVar, Generic + +x: C[None] +y: C[str] # E: Type argument "str" of "C" must be a subtype of "int" +z: C[int] + +T = TypeVar('T', bound=int) +class C(Generic[T]): + pass + +[file a.py] +from b import C + +x: C[None] # E: Type argument "None" of "C" must be a subtype of "int" +y: C[str] # E: Type argument "str" of "C" must be a subtype of "int" +z: C[int] + +[file mypy.ini] +\[mypy-a] +strict_optional = True +\[mypy-b] +strict_optional = False + + +[case testNewAnalyzerTypeArgBoundCheckWithStrictOptionalPyProjectTOML] +# flags: --config-file tmp/pyproject.toml +import a + +[file b.py] +from typing import TypeVar, Generic + +x: C[None] +y: C[str] # E: Type argument "str" of "C" must be a subtype of "int" +z: C[int] + +T = TypeVar('T', bound=int) +class C(Generic[T]): + pass + +[file a.py] +from b import C + +x: C[None] # E: Type argument "None" of "C" must be a subtype of "int" +y: C[str] # E: Type argument "str" of "C" must be a subtype of "int" +z: C[int] + +[file pyproject.toml] +\[[tool.mypy.overrides]] +module = 'a' +strict_optional = true +\[[tool.mypy.overrides]] +module = 'b' +strict_optional = false + + +[case testNewAnalyzerProperty] +class A: + @property + def x(self) -> B: + return 0 # E: Incompatible return value type (got "int", expected "B") + + @property + def y(self) -> B: + pass + + @y.setter + def y(self, x: B) -> None: + pass + +class B: pass + +a = A() +reveal_type(a.x) # N: Revealed type is "__main__.B" +a.y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "B") +[builtins fixtures/property.pyi] + +[case testNewAnalyzerAliasesFixedFew] +from typing import List, Generic, TypeVar +T = TypeVar('T') +class C(Generic[T]): + ... +A = List[C] +x: A + +def func(x: List[C[T]]) -> T: + ... + +reveal_type(x) # N: Revealed type is "builtins.list[__main__.C[Any]]" +reveal_type(func(x)) # N: Revealed type is "Any" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerAliasesFixedMany] +from typing import List, Generic, TypeVar + +T = TypeVar('T') +class C(Generic[T]): + ... + +def func(x: List[C[T]]) -> T: + ... + +x: A +A = List[C[int, str]] # E: "C" expects 1 type argument, but 2 given + +reveal_type(x) # N: Revealed type is "builtins.list[__main__.C[Any]]" +reveal_type(func(x)) # N: Revealed type is "Any" + + +[builtins fixtures/list.pyi] + +[case testNewAnalyzerBuiltinAliasesFixed] +from typing import List, Optional +x: Optional[List] = None +y: List[str] + +reveal_type(x) # N: Revealed type is "Union[builtins.list[Any], None]" +x = ['a', 'b'] +reveal_type(x) # N: Revealed type is "builtins.list[Any]" +x.extend(y) +[builtins fixtures/list.pyi] + +[case testNewAnalyzerImportPriosB] +import b +[file a.py] +from b import x +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[file b.py] +import a +x = (1, 2) +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerImportPriosA] +import a +[file a.py] +from b import x +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[file b.py] +import a +x = (1, 2) +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerConditionalFunc] +if int(): + def f(x: int) -> None: + pass + def g(x: int) -> None: + pass +elif bool(): + def f(x: int) -> None: + 1() # E: "int" not callable + def g(x: str) -> None: # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: int) -> None \ + # N: Redefinition: \ + # N: def g(x: str) -> None + pass +else: + def f(x: int) -> None: + ''() # E: "str" not callable + +reveal_type(g) # N: Revealed type is "def (x: builtins.int)" + +[case testNewAnalyzerConditionalFuncDefer] +if int(): + def f(x: A) -> None: + pass + def g(x: A) -> None: + pass +else: + def f(x: A) -> None: + 1() # E: "int" not callable + def g(x: str) -> None: # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: A) -> None \ + # N: Redefinition: \ + # N: def g(x: str) -> None + + pass + +reveal_type(g) # N: Revealed type is "def (x: __main__.A)" + +class A: pass + +[case testNewAnalyzerConditionalDecoratedFunc] +from typing import Callable + +def dec(f: Callable[[int], None]) -> Callable[[str], None]: + pass + +if int(): + from m import f +else: + @dec + def f(x: int) -> None: + 1() # E: "int" not callable +reveal_type(f) # N: Revealed type is "def (builtins.str)" +[file m.py] +def f(x: str, /) -> None: pass + +[case testNewAnalyzerConditionallyDefineFuncOverVar] +from typing import Callable + +if int(): + f: Callable[[str], None] +else: + def f(x: str) -> None: + ... +reveal_type(f) # N: Revealed type is "def (builtins.str)" + +[case testNewAnalyzerConditionallyDefineFuncOverClass] +class C: + 1() # E: "int" not callable +def C() -> None: # E: Name "C" already defined on line 1 + ''() # E: "str" not callable + +[case testNewAnalyzerTupleIteration] +from typing import Union, Tuple, NamedTuple + +class T(Tuple[B, C]): + pass + +class A: pass +class B(A): pass +class C(A): pass + +class NTInt(NamedTuple): + x: int + y: int + +class NTStr(NamedTuple): + x: str + y: str + +t1: T +reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[Union[__main__.B, __main__.C]]" + +t2: NTInt +reveal_type(t2.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.int]" +nt: Union[NTInt, NTStr] +reveal_type(nt.__iter__) # N: Revealed type is "Union[def () -> typing.Iterator[builtins.int], def () -> typing.Iterator[builtins.str]]" +for nx in nt: + reveal_type(nx) # N: Revealed type is "Union[builtins.int, builtins.str]" + +t: Union[Tuple[int, int], Tuple[str, str]] +for x in t: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/for.pyi] + +[case testNewAnalyzerFallbackUpperBoundCheckAndFallbacks] +from typing import TypeVar, Generic, Tuple + +class A: pass +class B: pass +class C(B): pass + +S = TypeVar('S', bound='Tuple[G[A], ...]') + +class GG(Generic[S]): pass + +g: GG[Tuple[G[B], G[C]]] # E: Type argument "Tuple[G[B], G[C]]" of "GG" must be a subtype of "Tuple[G[A], ...]" \ + # E: Type argument "B" of "G" must be a subtype of "A" \ + # E: Type argument "C" of "G" must be a subtype of "A" + +T = TypeVar('T', bound=A, covariant=True) + +class G(Generic[T]): pass + +t: Tuple[G[B], G[C]] # E: Type argument "B" of "G" must be a subtype of "A" \ + # E: Type argument "C" of "G" must be a subtype of "A" +reveal_type(t.__iter__) # N: Revealed type is "def () -> typing.Iterator[__main__.G[__main__.B]]" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerClassKeywordsForward] +class C(B, other=A): ... +class B: ... +class A: ... + +[case testNewAnalyzerClassKeywordsCyclic] +from typing import List + +class C(List[C], other=C): ... +[builtins fixtures/list.pyi] + +[case testNewAnalyzerClassKeywordsError] +class C(other=asdf): ... # E: Name "asdf" is not defined + +[case testNewAnalyzerMissingImport] +# flags: --ignore-missing-imports +import non_existing + +x: C +class C: ... + +[case testNewAnalyzerMissingImportFrom] +# flags: --ignore-missing-imports +from non_existing import stuff + +x: C +class C: ... + +[case testNewAnalyzerFollowSkip] +# flags: --follow-imports=skip +from other import y +x: C +class C: ... +[file other.py] +y = 1 + +[case testNewAnalyzerMissingImportErrors] +# flags: --ignore-missing-imports +from non_existing import stuff, other_stuff + +stuff = 1 # OK +other_stuff: int = 1 # E: Name "other_stuff" already defined (possibly by an import) + +x: C +class C: ... + +[case testNewAnalyzerMissingImportErrorsRedefinition] +# flags: --ignore-missing-imports + +class Other: ... +from non_existing import Other # E: Name "Other" already defined on line 3 +from non_existing import Cls +class Cls: ... # E: Name "Cls" already defined (possibly by an import) + +x: C +class C: ... + +[case testNewAnalyzerTupleInit] +from typing import Tuple + +c: C +class C(Tuple[int, str]): + def __init__(self) -> None: pass +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNotAnAlias] +class Meta(type): + x = int() + +class C(metaclass=Meta): + pass + +y = C.x +reveal_type(y) # N: Revealed type is "builtins.int" + +[case testNewAnalyzerFunctionError] +def f(x: asdf) -> None: # E: Name "asdf" is not defined + pass + +[case testNewAnalyzerEnumRedefinition] +from enum import Enum + +A = Enum('A', ['x', 'y']) +A = Enum('A', ['z', 't']) # E: Name "A" already defined on line 3 +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerNewTypeRedefinition] +from typing import NewType + +A = NewType('A', int) +A = NewType('A', str) # E: Cannot redefine "A" as a NewType \ + # E: Name "A" already defined on line 3 + +[case testNewAnalyzerNewTypeForwardClass] +from typing import NewType, List + +x: C +reveal_type(x[0]) # N: Revealed type is "__main__.C" + +C = NewType('C', 'B') + +class B(List[C]): + pass +[builtins fixtures/list.pyi] + +[case testNewAnalyzerNewTypeForwardClassAlias] +from typing import NewType, List + +x: D +reveal_type(x[0]) # N: Revealed type is "__main__.C" + +C = NewType('C', 'B') +D = C + +class B(List[D]): + pass +[builtins fixtures/list.pyi] + +[case testNewAnalyzerNewTypeForwardClassAliasReversed] +from typing import NewType, List + +x: D +reveal_type(x[0][0]) # N: Revealed type is "__main__.C" + +D = C # E: Name "C" is used before definition +C = NewType('C', 'List[B]') + +class B(List[C]): + pass +[builtins fixtures/list.pyi] + +[case testNewAnalyzerNewTypeForwardClassAliasDirect] +# flags: --disable-error-code used-before-def +from typing import NewType, List + +def test() -> None: + x: D + reveal_type(x[0][0]) + + D = List[C] + C = NewType('C', 'B') + + class B(D): + pass +[builtins fixtures/list.pyi] +[out] +main:5: error: Cannot resolve name "D" (possible cyclic definition) +main:5: note: Recursive types are not allowed at function scope +main:6: note: Revealed type is "Any" +main:8: error: Cannot resolve name "D" (possible cyclic definition) +main:8: note: Recursive types are not allowed at function scope +main:8: error: Cannot resolve name "C" (possible cyclic definition) +main:9: error: Argument 2 to NewType(...) must be a valid type +main:9: error: Cannot resolve name "B" (possible cyclic definition) +main:9: note: Recursive types are not allowed at function scope + +-- Copied from check-classes.test (tricky corner cases). +[case testNewAnalyzerNoCrashForwardRefToBrokenDoubleNewTypeClass] +from typing import Any, Dict, List, NewType + +Foo = NewType('NotFoo', int) # type: ignore +Foos = NewType('Foos', List[Foo]) + +x: C +class C: + def frob(self, foos: Dict[Any, Foos]) -> None: + foo = foos.get(1) + assert foo + dict(foo) +[builtins fixtures/dict.pyi] + +[case testNewAnalyzerForwardTypeAliasInBase] +from typing import List, Generic, TypeVar, NamedTuple +T = TypeVar('T') + +def test() -> None: + class C(A, B): # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + pass + class G(Generic[T]): pass + A = G[C] # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + class B(NamedTuple): + x: int + + y: C + reveal_type(y.x) # N: Revealed type is "builtins.int" + reveal_type(y[0]) # N: Revealed type is "builtins.int" + x: A + reveal_type(x) # N: Revealed type is "__main__.G@7[Tuple[builtins.int, fallback=__main__.C@5]]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerDuplicateTypeVar] +from typing import TypeVar, Generic, Any + +T = TypeVar('T', bound='B[Any]') +# The "int" error is because of typing fixture. +T = TypeVar('T', bound='C') # E: Cannot redefine "T" as a type variable \ + # E: Invalid assignment target + +class B(Generic[T]): + x: T +class C: ... + +x: B[int] # E: Type argument "int" of "B" must be a subtype of "B[Any]" +y: B[B[Any]] +reveal_type(y.x) # N: Revealed type is "__main__.B[Any]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testNewAnalyzerDuplicateTypeVarImportCycle] +# flags: --disable-error-code used-before-def +import a +[file a.py] +from typing import TypeVar, Any +from b import B, C + +T = TypeVar('T', bound=B[Any]) +T = TypeVar('T', bound=C) + +[file b.py] +from typing import Generic, Any +from a import T + +class B(Generic[T]): + x: T +class C: ... + +x: B[int] +y: B[B[Any]] +reveal_type(y.x) +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] +[out] +tmp/b.py:8: error: Type argument "int" of "B" must be a subtype of "B[Any]" +tmp/b.py:10: note: Revealed type is "b.B[Any]" +tmp/a.py:5: error: Cannot redefine "T" as a type variable +tmp/a.py:5: error: Invalid assignment target + +[case testNewAnalyzerDuplicateTypeVarImportCycleWithAliases] +# flags: --disable-error-code used-before-def +import a +[file a.py] +from typing import TypeVar, Any +from b import BA, C + +T = TypeVar('T', bound=BAA[Any]) +T = TypeVar('T', bound=C) +BAA = BA + +[file b.py] +from typing import Generic, Any +from a import T + +BA = B +class B(Generic[T]): + x: T +class C: ... + +x: B[int] +y: B[B[Any]] +reveal_type(y.x) +[out] +tmp/b.py:9: error: Type argument "int" of "B" must be a subtype of "B[Any]" +tmp/b.py:11: note: Revealed type is "b.B[Any]" +tmp/a.py:5: error: Cannot redefine "T" as a type variable +tmp/a.py:5: error: Invalid assignment target + +[case testNewAnalyzerTypeVarBoundInCycle] +import factory, box + +[file factory.py] +from typing import Generic, Type + +from box import BoxT + +class Factory(Generic[BoxT]): + value: int + + def create(self, boxClass: Type[BoxT]) -> BoxT: + reveal_type(boxClass.create(self)) # N: Revealed type is "BoxT`1" + return boxClass.create(self) + +[file box.py] +from typing import TYPE_CHECKING, Type, TypeVar + +if TYPE_CHECKING: + from factory import Factory + +BoxT = TypeVar('BoxT', bound='Box') + +class Box: + @classmethod + def create(cls: Type[BoxT], f: Factory) -> BoxT: + return cls(f.value) + + def __init__(self, value: int) -> None: ... +[builtins fixtures/classmethod.pyi] +[typing fixtures/typing-medium.pyi] + +[case testNewAnalyzerCastForward1] +from typing import cast + +x = cast('C', None) +class A: + def foo(self) -> None: + self.x = cast('C', None) + +reveal_type(x) # N: Revealed type is "__main__.C" +reveal_type(A().x) # N: Revealed type is "__main__.C" + +class C(A): ... + +[case testNewAnalyzerCastForward2] +from typing import cast + +x = cast('C', None) + +reveal_type(x) # N: Revealed type is "builtins.int" + +C = int + +[case testNewAnalyzerCastForward3] +from typing import cast, NamedTuple + +x = cast('C', None) + +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.C]" +reveal_type(x.x) # N: Revealed type is "builtins.int" + +C = NamedTuple('C', [('x', int)]) +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerApplicationForward1] +# flags: --disable-error-code used-before-def +from typing import Generic, TypeVar + +x = C[int]() +reveal_type(x) # N: Revealed type is "__main__.C[builtins.int]" + +T = TypeVar('T') +class C(Generic[T]): ... + +[case testNewAnalyzerApplicationForward2] +from typing import Generic, TypeVar + +T = TypeVar('T') +class C(Generic[T]): ... + +x = C['A']() +reveal_type(x) # N: Revealed type is "__main__.C[__main__.A]" + +class A: ... + +[case testNewAnalyzerApplicationForward3] +from typing import Generic, TypeVar + +class A: ... +T = TypeVar('T') +class C(Generic[T]): ... +x = C[A]() +reveal_type(x) # N: Revealed type is "__main__.C[__main__.A]" + +[case testNewAnalyzerApplicationForward4] +# flags: --disable-error-code used-before-def +from typing import Generic, TypeVar + +x = C[A]() # E: Value of type variable "T" of "C" cannot be "A" +reveal_type(x) # N: Revealed type is "__main__.C[__main__.A]" + +T = TypeVar('T', bound='D') +class C(Generic[T]): ... + +class A: ... +class D: ... + +[case testNewAnalyzerAddedSubStarImport_incremental] +# TODO: This can be removed once testAddedSubStarImport is enabled in check-incremental.test. +# cmd: mypy -m a pack pack.mod b +# cmd2: mypy -m other +[file a.py] +from pack import * +[file pack/__init__.py] +[file pack/mod.py] +[file b.py] +import pack.mod +[file other.py] +import a +[out] +[out2] + +[case testNewAnalyzerModuleGetattrSerialize_incremental] +import a +[file a.py] +import p +[file a.py.2] +import p +reveal_type(p.y) +[file p.pyi] +from pp import x +y = x +[file pp.pyi] +def __getattr__(attr): ... +[out2] +tmp/a.py:2: note: Revealed type is "Any" + +[case testNewAnanlyzerTrickyImportPackage] +from lib import config +import lib + +reveal_type(lib.config.x) # N: Revealed type is "builtins.int" +reveal_type(config.x) # N: Revealed type is "builtins.int" + +[file lib/__init__.py] +from lib.config import config + +[file lib/config.py] +class Config: + x: int + +config = Config() +[builtins fixtures/module.pyi] + +[case testNewAnanlyzerTrickyImportPackageAlt] +import lib.config +import lib.config as tmp + +reveal_type(lib.config.x) # N: Revealed type is "builtins.int" +# TODO: this actually doesn't match runtime behavior, variable wins. +tmp.x # E: Module has no attribute "x" + +[file lib/__init__.py] +from lib.config import config + +[file lib/config.py] +class Config: + x: int + +config = Config() +[builtins fixtures/module.pyi] + +[case testNewAnanlyzerTrickyImportPackage_incremental] +import a + +[file a.py] +from lib import config +import lib + +[file a.py.2] +from lib import config +import lib + +reveal_type(lib.config.x) +reveal_type(config.x) + +[file lib/__init__.py] +from lib.config import config + +[file lib/config.py] +class Config: + x: int + +config = Config() +[builtins fixtures/module.pyi] +[out2] +tmp/a.py:4: note: Revealed type is "builtins.int" +tmp/a.py:5: note: Revealed type is "builtins.int" + +[case testNewAnalyzerRedefineAsClass] +from typing import Any +from other import C # type: ignore + +y = 'bad' + +class C: # E: Name "C" already defined (possibly by an import) + def meth(self, other: int) -> None: + y() # E: "str" not callable + +[case testNewAnalyzerRedefineAsOverload] +from typing import overload + +y = 'bad' + +if int(): + def f(x: int) -> None: + pass +else: + @overload # E: Name "f" already defined on line 6 + def f(x: int) -> None: ... + @overload + def f(x: str) -> None: ... + def f(x) -> None: + y() # E: "str" not callable + +[case testNewAnalyzerFirstAliasTargetWins] +class DesiredTarget: + attr: int + +if int(): + Alias = DesiredTarget +else: + class DummyTarget: + pass + Alias = DummyTarget # type: ignore + +x: Alias +reveal_type(x.attr) # N: Revealed type is "builtins.int" +[case testNewAnalyzerFirstVarDefinitionWins] +x = y # E: Name "y" is used before definition +x = 1 + +# We want to check that the first definition creates the variable. +def x() -> None: ... # E: Name "x" already defined on line 1 +y = 2 + +[case testNewAnalyzerImportStarSpecialCase] +import unittest +[file unittest/__init__.pyi] +from unittest.suite import * + +def load_tests() -> TestSuite: ... +[file unittest/suite.pyi] +from typing import Union # Iterable not imported +import unittest.case + +_TestType = Union[unittest.case.TestCase] + +class BaseTestSuite(Iterable[_TestType]): + ... + +class TestSuite(BaseTestSuite): + ... + +[file unittest/case.pyi] +class TestCase: ... + +[out] +tmp/unittest/suite.pyi:6: error: Name "Iterable" is not defined +tmp/unittest/suite.pyi:6: note: Did you forget to import it from "typing"? (Suggestion: "from typing import Iterable") + +[case testNewAnalyzerNewTypeSpecialCase] +from typing import NewType +from typing_extensions import Final, Literal + +X = NewType('X', int) + +var1: Final = 1 + +def force1(x: Literal[1]) -> None: pass + +force1(reveal_type(var1)) # N: Revealed type is "Literal[1]" +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerReportLoopInMRO] +class A(A): ... # E: Cannot resolve name "A" (possible cyclic definition) +[out] + +[case testNewSemanticAnalyzerUnimportedSpecialCase] +# flags: --ignore-missing-imports +import other +import p.u + +[file p/__init__.py] + +[file p/u.pyi] +from . import c +x: c.C + +[file other.py] +from p.c import B + +[out] + +[case testNewSemanticAnalyzerQualifiedFunctionAsType] +import m + +x: m.C.a.b # E: Name "m.C.a.b" is not defined + +[file m.py] +def C(): pass + +[case testNewSemanticAnalyzerModulePrivateRefInMiddleOfQualified] +import m + +x: m.n.C # E: Name "m.n.C" is not defined +reveal_type(x) # N: Revealed type is "Any" + +[file m.pyi] +import n + +[file n.pyi] +class C: pass + +[case testNewAnalyzerModuleGetAttr] +import m +import n + +x: m.n.C +y: n.D +[file m.py] +import n +[file n.py] +def __getattr__(x): pass + +[case testNewAnalyzerReportLoopInMRO2] +def f() -> None: + class A(A): ... # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + +[case testNewAnalyzerUnsupportedBaseClassInsideFunction] +class C: + class E: pass + + def f(self) -> None: + # TODO: Error message could be better + class D(self.E): # E: Name "self.E" is not defined + pass + +[case testNewAnalyzerShadowOuterDefinitionBasedOnOrderSinglePass] +# Only one semantic analysis pass +class X: pass +class C: + X = X + reveal_type(X) # N: Revealed type is "def () -> __main__.X" +reveal_type(C.X) # N: Revealed type is "def () -> __main__.X" + +[case testNewAnalyzerShadowOuterDefinitionBasedOnOrderTwoPasses] +c: C # Force second semantic analysis pass +class X: pass +class C: + X = X + reveal_type(X) # N: Revealed type is "def () -> __main__.X" + +reveal_type(C.X) # N: Revealed type is "def () -> __main__.X" + +[case testNewAnalyzerAnnotationConflictsWithAttributeSinglePass] +class C: + def x(self) -> int: + return 0 + + def __init__(self) -> None: + self.int = '' + + def y(self) -> int: + return 0 + + z: str + + def str(self) -> str: + return 0 # E: Incompatible return value type (got "int", expected "str") + + zz: str # E: Function "__main__.C.str" is not valid as a type \ + # N: Perhaps you need "Callable[...]" or a callback protocol? + +reveal_type(C().x()) # N: Revealed type is "builtins.int" +reveal_type(C().y()) # N: Revealed type is "builtins.int" +reveal_type(C().z) # N: Revealed type is "builtins.str" +reveal_type(C().str()) # N: Revealed type is "builtins.str" + +[case testNewAnalyzerAnnotationConflictsWithAttributeTwoPasses] +c: C # Force second semantic analysis pass + +class C: + def x(self) -> int: + return 0 + + def __init__(self) -> None: + self.int = '' + + def y(self) -> int: + return 0 + + z: str + + def str(self) -> str: + return 0 # E: Incompatible return value type (got "int", expected "str") + + zz: str # E: Function "__main__.C.str" is not valid as a type \ + # N: Perhaps you need "Callable[...]" or a callback protocol? + +reveal_type(C().x()) # N: Revealed type is "builtins.int" +reveal_type(C().y()) # N: Revealed type is "builtins.int" +reveal_type(C().z) # N: Revealed type is "builtins.str" +reveal_type(C().str()) # N: Revealed type is "builtins.str" + +[case testNewAnalyzerNameConflictsAndMultiLineDefinition] +c: C # Force second semantic analysis pass + +class X: pass + +class C: + X = ( + X) + + def str(self + ) -> str: + return 0 # E: Incompatible return value type (got "int", expected "str") + +reveal_type(C.X) # E: # N: Revealed type is "def () -> __main__.X" +reveal_type(C().str()) # N: Revealed type is "builtins.str" + +[case testNewAnalyzerNameNotDefinedYetInClassBody] +class C: + X = Y # E: Name "Y" is not defined + Y = 1 + + f = g # E: Name "g" is not defined + + def g(self) -> None: pass + +reveal_type(C.X) # N: Revealed type is "Any" + +[case testNewAnalyzerImportedNameUsedInClassBody] +import m + +[file m.py] +class C: + from mm import f # E: Unsupported class scoped import + @dec(f) + def m(self): pass + +def dec(f): pass +[file mm.py] +# 1 padding to increase line number of 'f' +# 2 padding +# 3 padding +# 4 padding +# 5 padding +# 6 padding +def f(): pass + + +[case testNewAnalyzerImportedNameUsedInClassBody2] +import m + +[file m/__init__.py] +class C: + from m.m import f # E: Unsupported class scoped import + @dec(f) + def m(self): pass + +def dec(f): pass +[file m/m.py] +# 1 padding to increase line number of 'f' +# 2 padding +# 3 padding +# 4 padding +# 5 padding +# 6 padding +def f(): pass + +[case testNewAnalyzerOverrideClassWithTypeAlias] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class C(Generic[T]): + pass +# TODO: Error message is confusing +C = C[int] # E: Cannot assign to a type \ + # E: Incompatible types in assignment (expression has type "Type[C[Any]]", variable has type "Type[C[Any]]") +x: C +reveal_type(x) # N: Revealed type is "__main__.C[Any]" +[out] +[out2] + +[case testNewAnalyzerClassVariableOrdering] +def foo(x: str) -> None: pass + +class Something: + def run(self) -> None: + foo(self.IDS[0]) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" + + IDS = [87] +[builtins fixtures/list.pyi] + +[case testNewAnalyzerPlaceholderFromOuterScope] +import b +[file a.py] +import b +class A(B): ... +class B: ... + +[file b.py] +from a import A + +class C: + A = A # Initially rvalue will be a placeholder + +reveal_type(C.A) # N: Revealed type is "def () -> a.A" + +[case testNewAnalyzerFinalLiteralInferredAsLiteralWithDeferral] +from typing_extensions import Final, Literal + +defer: Yes + +var: Final = 42 +def force(x: Literal[42]) -> None: pass +force(reveal_type(var)) # N: Revealed type is "Literal[42]" + +class Yes: ... +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerImportCycleWithIgnoreMissingImports] +# flags: --ignore-missing-imports +import p +reveal_type(p.get) # N: Revealed type is "def () -> builtins.int" + +[file p/__init__.pyi] +from . import api +get = api.get + +[file p/api.pyi] +import p + +def get() -> int: ... + +[case testUseObsoleteNameForTypeVar3] +import typing +t = typing.typevar('t') # E: Module has no attribute "typevar" +[builtins fixtures/module.pyi] + +[case testNewAnalyzerImportFromTopLevelFunction] +import a.b # This works at runtime +reveal_type(a.b) # N +[file a/__init__.py] +from .b import B +from . import b as c +def b() -> None: pass +reveal_type(b) # N +reveal_type(c.B()) # N +x: Forward +class Forward: + ... + +[file a/b.py] +class B: ... +[builtins fixtures/module.pyi] + +[out] +tmp/a/__init__.py:4: note: Revealed type is "def ()" +tmp/a/__init__.py:5: note: Revealed type is "a.b.B" +main:2: note: Revealed type is "def ()" + +[case testNewAnalyzerImportFromTopLevelAlias] +import a.b # This works at runtime +reveal_type(a.b) # N +[file a/__init__.py] +from .b import B +from . import b as c +b = int +y: b +reveal_type(y) # N +reveal_type(c.B) # N +x: Forward +class Forward: + ... + +[file a/b.py] +class B: ... +[builtins fixtures/module.pyi] + +[out] +tmp/a/__init__.py:5: note: Revealed type is "builtins.int" +tmp/a/__init__.py:6: note: Revealed type is "def () -> a.b.B" +main:2: note: Revealed type is "def () -> builtins.int" + +[case testNewAnalyzerImportAmbiguousWithTopLevelFunction] +import a.b # This works at runtime +x: a.b.B # E +reveal_type(a.b) # N +[file a/__init__.py] +import a.b +import a.b as c +def b() -> None: pass +reveal_type(b) # N +reveal_type(c.B()) # N +x: Forward +class Forward: + ... + +[file a/b.py] +class B: ... +[builtins fixtures/module.pyi] + +[out] +tmp/a/__init__.py:4: note: Revealed type is "def ()" +tmp/a/__init__.py:5: note: Revealed type is "a.b.B" +main:2: error: Name "a.b.B" is not defined +main:3: note: Revealed type is "def ()" + +[case testNewAnalyzerConfusingImportConflictingNames] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy -m other a.b a +[file a/__init__.py] +[file a/b/__init__.py] +import other +import a.b.a +import a.b.c + +[file other.py] +from a.b.a import foo +[builtins fixtures/module.pyi] +[out] + +[case testNewAnalyzerNamedTupleMethod] +from typing import NamedTuple + +g: N + +class N(NamedTuple): + def f(self) -> None: + b = ( + a + for a in [1] + ) + b +[builtins fixtures/tuple.pyi] + +[case testWithMultipleTargetsDeferred] +a: A + +class A: + def __enter__(self) -> int: pass + def __exit__(self, x, y, z): pass + +with A() as x, A() as y: # type: int, int + pass + +[case testNewAnalyzerLessErrorsNeedAnnotation] +from typing import TypeVar, Optional + +T = TypeVar('T') + +def f(x: Optional[T] = None) -> T: ... + +x = f() # E: Need type annotation for "x" +y = x + +def g() -> None: + x = f() # E: Need type annotation for "x" + y = x + +[case testNewAnalyzerLessErrorsNeedAnnotationList] +x = [] # type: ignore +reveal_type(x) # N: Revealed type is "builtins.list[Any]" + +def g() -> None: + x = [] # type: ignore + reveal_type(x) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerLessErrorsNeedAnnotationNested] +from typing import TypeVar, Optional, Generic + +T = TypeVar('T') +class G(Generic[T]): ... + +def f(x: Optional[T] = None) -> G[T]: ... + +x = f() # E: Need type annotation for "x" +y = x +reveal_type(y) # N: Revealed type is "__main__.G[Any]" + +def g() -> None: + x = f() # E: Need type annotation for "x" + y = x + reveal_type(y) # N: Revealed type is "__main__.G[Any]" + +[case testNewAnalyzerRedefinedNonlocal] +# flags: --disable-error-code=annotation-unchecked +import typing + +def f(): + bar = [] # type: typing.List[int] + + def foo(): + nonlocal bar + bar = [] # type: typing.List[int] + +def g() -> None: + bar = [] # type: typing.List[int] + + def foo() -> None: + nonlocal bar + bar = [] # type: typing.List[int] # E: Name "bar" already defined on line 12 +[builtins fixtures/list.pyi] + +[case testNewAnalyzerMoreInvalidTypeVarArgumentsDeferred] +from typing import TypeVar, Generic + +defer: Yes + +S = TypeVar('S', covariant=True, contravariant=True) # E: TypeVar cannot be both covariant and contravariant \ + # E: "int" not callable + +class Yes: ... +[builtins fixtures/bool.pyi] + +[case testNewAnalyzerDisallowAnyGenericsMessages] +# mypy: disallow-any-generics +from a import B +x: B + +[file a.py] +from typing import TypeVar, List + +T = TypeVar('T') + +A = List[T] +B = A + +[builtins fixtures/list.pyi] + +[case testNewAnalyzerVarTypeVarNoCrash] +from typing import Callable, TypeVar + +FooT = TypeVar('FooT', bound='Foo') +class Foo: ... + +f = lambda x: True # type: Callable[[FooT], bool] +reveal_type(f) # N: Revealed type is "def [FooT <: __main__.Foo] (FooT`-1) -> builtins.bool" +[builtins fixtures/bool.pyi] + +[case testNewAnalyzerVarTypeVarNoCrashImportCycle] +import a + +[file a.py] +from b import B +from typing import TypeVar + +FooT = TypeVar('FooT', bound='Foo') +class Foo: ... + +[file b.py] +from a import FooT +from typing import Callable + +f = lambda x: True # type: Callable[[FooT], bool] +reveal_type(f) # N: Revealed type is "def [FooT <: a.Foo] (FooT`-1) -> builtins.bool" + +class B: ... +[builtins fixtures/bool.pyi] + +[case testNewAnalyzerFuncTypeVarNoCrashImportCycle] +import a + +[file a.py] +from b import B +from typing import TypeVar + +FooT = TypeVar('FooT', bound='Foo') +class Foo: ... + +[file b.py] +from a import FooT +from typing import Callable + +def f(x: FooT) -> bool: ... +reveal_type(f) # N: Revealed type is "def [FooT <: a.Foo] (x: FooT`-1) -> builtins.bool" + +class B: ... +[builtins fixtures/bool.pyi] + +[case testNewAnalyzerNoCrashOnStarInference] +from typing import Tuple + +def f() -> None: + t: Tuple[str, Tuple[str, str, str]] + x, (y, *z) = t + reveal_type(z) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/list.pyi] + +[case testNewAnalyzerIdentityAssignment1] +from foo import * + +try: + X = X +except: + class X: # E: Name "X" already defined (possibly by an import) + pass + +reveal_type(X()) # N: Revealed type is "foo.X" + +[file foo.py] +class X: pass + +[case testNewAnalyzerIdentityAssignment2] +try: + int = int + reveal_type(int()) # N: Revealed type is "builtins.int" +except: + class int: # E: Name "int" already defined (possibly by an import) + pass + +reveal_type(int()) # N: Revealed type is "builtins.int" + +[case testNewAnalyzerIdentityAssignment3] +forwardref: C + +try: + int = int + reveal_type(int()) # N: Revealed type is "builtins.int" +except: + class int: # E: Name "int" already defined (possibly by an import) + pass + +reveal_type(int()) # N: Revealed type is "builtins.int" + +class C: pass + +[case testNewAnalyzerIdentityAssignment4] +try: + C = C + C +except: + class C: + pass + +reveal_type(C()) # N: Revealed type is "__main__.C" + +[case testNewAnalyzerIdentityAssignment5] +forwardref: D + +try: + C = C + C +except: + class C: + pass + +class D: pass + +reveal_type(C()) # N: Revealed type is "__main__.C" + +[case testNewAnalyzerIdentityAssignment6] +x: C +class C: + pass +C = C + +reveal_type(C()) # N: Revealed type is "__main__.C" +reveal_type(x) # N: Revealed type is "__main__.C" + +[case testNewAnalyzerIdentityAssignment7] +C = C # E: Name "C" is not defined + +reveal_type(C) # E: Name "C" is not defined \ + # N: Revealed type is "Any" + +[case testNewAnalyzerIdentityAssignment8] +from typing import Final +x: Final = 0 +x = x # E: Cannot assign to final name "x" + +[case testNewAnalyzerClassPropertiesInAllScopes] +from abc import abstractmethod, ABCMeta + +class TopLevel(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + +TopLevel() # E: Cannot instantiate abstract class "TopLevel" with abstract attribute "f" + +def func() -> None: + class Function(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + + Function() # E: Cannot instantiate abstract class "Function" with abstract attribute "f" + +class C: + def meth(self) -> None: + class Method(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + + Method() # E: Cannot instantiate abstract class "Method" with abstract attribute "f" + +[case testModulesAndFuncsTargetsInCycle] +import a +[file a.py] +import b +defer: Yes +def func() -> int: ... +class Yes: ... +[file b.py] +import a + +def func() -> int: ... +[targets b, a, a, b.func, a.func, __main__] + +[case testNewAnalyzerForwardReferenceInFunction] +def f(x: 'A') -> 'A': + return A() + +class A: + pass +[targets __main__, __main__.f] + +[case testNewAnalyzerSimpleImportStarNoDeferral] +from m import * + +x: A +f() + +[file m.py] +class A: pass +def f() -> None: pass + +[targets m, m.f, __main__] + +[case testNewAnalyzerNoCrashOnCustomProperty] +# flags: --ignore-missing-imports +from unimported import custom + +class User: + first_name: str + + @custom + def name(self) -> str: + return self.first_name + + @name.setter # type: ignore + def name(self, value: str) -> None: + self.first_name = value + + def __init__(self, name: str) -> None: + self.name = name # E: Cannot assign to a method + +[case testNewAnalyzerMemberNameMatchesTypedDict] +from typing import Union, Any +from typing_extensions import TypedDict + +class T(TypedDict): + b: b.T + +class b: + T: Union[Any] +[builtins fixtures/tuple.pyi] + +[case testNewAnalyzerMemberNameMatchesNamedTuple] +from typing import Union, Any, NamedTuple + +class T(NamedTuple): + b: b.T + +class b: + T = Union[Any] + +[builtins fixtures/tuple.pyi] + +[case testSelfReferentialSubscriptExpression] +x = x[1] # E: Cannot resolve name "x" (possible cyclic definition) +y = 1[y] # E: Value of type "int" is not indexable \ + # E: Cannot determine type of "y" diff --git a/test-data/unit/check-newsyntax.test b/test-data/unit/check-newsyntax.test new file mode 100644 index 000000000000..3ed4c6d3d8e2 --- /dev/null +++ b/test-data/unit/check-newsyntax.test @@ -0,0 +1,129 @@ +[case testNewSyntaxSyntaxError] +x: int: int # E: invalid syntax +[out] + +[case testNewSyntaxBasics] +x: int +x = 5 +y: int = 5 + +a: str +a = 5 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +b: str = 5 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +zzz: int +zzz: str # E: Name "zzz" already defined on line 9 +[out] + +[case testNewSyntaxWithDict] +from typing import Dict, Any + +d: Dict[int, str] = {} +d[42] = 'ab' +d[42] = 42 # E: Incompatible types in assignment (expression has type "int", target has type "str") +d['ab'] = 'ab' # E: Invalid index type "str" for "Dict[int, str]"; expected type "int" +[builtins fixtures/dict.pyi] +[out] + +[case testNewSyntaxWithRevealType] +from typing import Dict + +def tst_local(dct: Dict[int, T]) -> Dict[T, int]: + ret: Dict[T, int] = {} + return ret + +reveal_type(tst_local({1: 'a'})) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +[builtins fixtures/dict.pyi] +[out] + +[case testNewSyntaxWithInstanceVars] +class TstInstance: + a: str + def __init__(self) -> None: + self.x: int + +TstInstance().x = 5 +TstInstance().x = 'ab' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +TstInstance().a = 5 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +TstInstance().a = 'ab' +[out] + +[case testNewSyntaxWithClassVars] +class CCC: + a: str = None # E: Incompatible types in assignment (expression has type "None", variable has type "str") +[out] + +[case testNewSyntaxWithStrictOptional] +strict: int +strict = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") +strict2: int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") +[out] + +[case testNewSyntaxWithStrictOptionalFunctions] +def f() -> None: + x: int + if int(): + x = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") +[out] + +[case testNewSyntaxWithStrictOptionalClasses] +class C: + def meth(self) -> None: + x: int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + self.x: int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") +[out] + +[case testNewSyntaxSpecialAssign] +class X: + x: str + x[0]: int + x.x: int + +[out] +main:3: error: Unexpected type declaration +main:3: error: Unsupported target for indexed assignment ("str") +main:4: error: Type cannot be declared in assignment to non-self attribute +main:4: error: "str" has no attribute "x" + +[case testNewSyntaxFStringBasics] +f'foobar' +f'{"foobar"}' +f'foo{"bar"}' +f'.{1}.' +f'{type(1)}' +a: str +a = f'foobar' +a = f'{"foobar"}' +[builtins fixtures/f_string.pyi] + +[case testNewSyntaxFStringExpressionsOk] +f'.{1 + 1}.' +f'.{1 + 1}.{"foo" + "bar"}' +[builtins fixtures/f_string.pyi] + +[case testNewSyntaxFStringExpressionsErrors] +f'{1 + ""}' +f'.{1 + ""}' +[builtins fixtures/f_string.pyi] +[out] +main:1: error: Unsupported operand types for + ("int" and "str") +main:2: error: Unsupported operand types for + ("int" and "str") + +[case testNewSyntaxFStringParseFormatOptions] +value = 10.5142 +width = 10 +precision = 4 +f'result: {value:{width}.{precision}}' +[builtins fixtures/f_string.pyi] + +[case testNewSyntaxFStringSingleField] +v = 1 +reveal_type(f'{v}') # N: Revealed type is "builtins.str" +reveal_type(f'{1}') # N: Revealed type is "builtins.str" +[builtins fixtures/f_string.pyi] + +[case testFeatureVersionSuggestion] +# flags: --python-version 3.99 +x *** x this is what future python looks like public static void main String[] args await goto exit +[out] +main:2: error: invalid syntax; you likely need to run mypy using Python 3.99 or newer diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test new file mode 100644 index 000000000000..a0a30079f062 --- /dev/null +++ b/test-data/unit/check-newtype.test @@ -0,0 +1,387 @@ +-- Checks NewType(...) + +-- Checks for basic functionality + +[case testNewTypePEP484Example1] +from typing import NewType + +UserId = NewType('UserId', int) + +def name_by_id(user_id: UserId) -> str: + return "foo" + +UserId('user') # E: Argument 1 to "UserId" has incompatible type "str"; expected "int" +name_by_id(42) # E: Argument 1 to "name_by_id" has incompatible type "int"; expected "UserId" +name_by_id(UserId(42)) + +id = UserId(5) +num = id + 1 + +reveal_type(id) # N: Revealed type is "__main__.UserId" +reveal_type(num) # N: Revealed type is "builtins.int" + +[targets __main__, __main__.UserId.__init__, __main__.name_by_id] + +[case testNewTypePEP484Example2] +from typing import NewType + +class PacketId: + def __init__(self, major: int, minor: int) -> None: + self._major = major + self._minor = minor + +TcpPacketId = NewType('TcpPacketId', PacketId) + +packet = PacketId(100, 100) +tcp_packet = TcpPacketId(packet) +tcp_packet = TcpPacketId(127, 0) + +[out] +main:12: error: Too many arguments for "TcpPacketId" +main:12: error: Argument 1 to "TcpPacketId" has incompatible type "int"; expected "PacketId" + +[case testNewTypeWithTuples] +from typing import NewType, Tuple +TwoTuple = NewType('TwoTuple', Tuple[int, str]) +a = TwoTuple((3, "a")) +b = TwoTuple(("a", 3)) # E: Argument 1 to "TwoTuple" has incompatible type "Tuple[str, int]"; expected "Tuple[int, str]" + +reveal_type(a[0]) # N: Revealed type is "builtins.int" +reveal_type(a[1]) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[out] + +[case testNewTypeWithLists] +from typing import NewType, List +UserId = NewType('UserId', int) +IdList = NewType('IdList', List[UserId]) + +bad1 = IdList([1]) # E: List item 0 has incompatible type "int"; expected "UserId" + +foo = IdList([]) +foo.append(3) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "UserId" +foo.append(UserId(3)) +foo.extend([UserId(1), UserId(2), UserId(3)]) +foo.extend(IdList([UserId(1), UserId(2), UserId(3)])) +bar = IdList([UserId(2)]) + +baz = foo + bar +reveal_type(foo) # N: Revealed type is "__main__.IdList" +reveal_type(bar) # N: Revealed type is "__main__.IdList" +reveal_type(baz) # N: Revealed type is "builtins.list[__main__.UserId]" + +[builtins fixtures/list.pyi] +[out] + +[case testNewTypeWithGenerics] +from typing import TypeVar, Generic, NewType, Any + +T = TypeVar('T') + +class Base(Generic[T]): + def __init__(self, item: T) -> None: + self.item = item + + def getter(self) -> T: + return self.item + +Derived1 = NewType('Derived1', Base[str]) +Derived2 = NewType('Derived2', Base) # Implicit 'Any' +Derived3 = NewType('Derived3', Base[Any]) # Explicit 'Any' + +Derived1(Base(1)) # E: Argument 1 to "Base" has incompatible type "int"; expected "str" +Derived1(Base('a')) +Derived2(Base(1)) +Derived2(Base('a')) +Derived3(Base(1)) +Derived3(Base('a')) + +reveal_type(Derived1(Base('a')).getter()) # N: Revealed type is "builtins.str" +reveal_type(Derived3(Base('a')).getter()) # N: Revealed type is "Any" +[out] + +[case testNewTypeWithNamedTuple] +from collections import namedtuple +from typing import NewType, NamedTuple + +Vector1 = namedtuple('Vector1', ['x', 'y']) +Point1 = NewType('Point1', Vector1) +p1 = Point1(Vector1(1, 2)) +reveal_type(p1.x) # N: Revealed type is "Any" +reveal_type(p1.y) # N: Revealed type is "Any" + +Vector2 = NamedTuple('Vector2', [('x', int), ('y', int)]) +Point2 = NewType('Point2', Vector2) +p2 = Point2(Vector2(1, 2)) +reveal_type(p2.x) # N: Revealed type is "builtins.int" +reveal_type(p2.y) # N: Revealed type is "builtins.int" + +class Vector3: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y +Point3 = NewType('Point3', Vector3) +p3 = Point3(Vector3(1, 3)) +reveal_type(p3.x) # N: Revealed type is "builtins.int" +reveal_type(p3.y) # N: Revealed type is "builtins.int" + +[builtins fixtures/list.pyi] + +[out] + +[case testNewTypeWithCasts] +from typing import NewType, cast +UserId = NewType('UserId', int) +foo = UserId(3) +foo = cast(UserId, 3) +foo = cast(UserId, "foo") +foo = cast(UserId, UserId(4)) +[out] + +[case testNewTypeWithTypeAliases] +from typing import NewType +Foo = int +Bar = NewType('Bar', Foo) +Bar2 = Bar + +def func1(x: Foo) -> Bar: + return Bar(x) + +def func2(x: int) -> Bar: + return Bar(x) + +def func3(x: Bar2) -> Bar: + return x + +x = Bar(42) +y = Bar2(42) + +y = func3(x) +[out] + +[case testNewTypeWithNewType] +from typing import NewType +A = NewType('A', int) +B = NewType('B', A) +C = A +D = C +E = NewType('E', D) + +a = A(1) +b = B(a) +e = E(a) + +def funca(a: A) -> None: ... +def funcb(b: B) -> None: ... + +funca(a) +funca(b) +funca(e) +funcb(a) # E: Argument 1 to "funcb" has incompatible type "A"; expected "B" +funcb(b) +funcb(e) # E: Argument 1 to "funcb" has incompatible type "E"; expected "B" + +[out] + +-- Make sure NewType works as expected in a variety of different scopes/across files + +[case testNewTypeInLocalScope] +from typing import NewType +A = NewType('A', int) +a = A(3) + +def func() -> None: + A = NewType('A', str) + B = NewType('B', str) + + a = A(3) # E: Argument 1 to "A@6" has incompatible type "int"; expected "str" + a = A('xyz') + b = B('xyz') + +class MyClass: + C = NewType('C', float) + + def foo(self) -> 'MyClass.C': + return MyClass.C(3.2) + +b = A(3) +c = MyClass.C(3.5) +[out] + +[case testNewTypeInMultipleFiles] +import a +import b +list1 = [a.UserId(1), a.UserId(2)] +list1.append(b.UserId(3)) # E: Argument 1 to "append" of "list" has incompatible type "b.UserId"; expected "a.UserId" + +[file a.py] +from typing import NewType +UserId = NewType('UserId', int) + +[file b.py] +from typing import NewType +UserId = NewType('UserId', int) + +[builtins fixtures/list.pyi] +[out] + +[case testNewTypeWithIncremental] +import m + +[file m.py] +from typing import NewType + +UserId = NewType('UserId', int) + +def name_by_id(user_id: UserId) -> str: + return "foo" + +name_by_id(UserId(42)) + +id = UserId(5) +num = id + 1 + +[file m.py.2] +from typing import NewType + +UserId = NewType('UserId', int) + +def name_by_id(user_id: UserId) -> str: + return "foo" + +name_by_id(UserId(42)) + +id = UserId(5) +num = id + 1 + +reveal_type(id) +reveal_type(num) +[rechecked m] +[stale] +[out1] +[out2] +tmp/m.py:13: note: Revealed type is "m.UserId" +tmp/m.py:14: note: Revealed type is "builtins.int" + + +-- Check misuses of NewType fail + +[case testNewTypeBadInitializationFails] +from typing import NewType + +a = NewType('b', int) # E: String argument 1 "b" to NewType(...) does not match variable name "a" +b = NewType('b', 3) # E: Argument 2 to NewType(...) must be a valid type +c = NewType(2, int) # E: Argument 1 to NewType(...) must be a string literal +d = NewType(b'f', int) # E: Argument 1 to NewType(...) must be a string literal +foo = "d" +e = NewType(foo, int) # E: Argument 1 to NewType(...) must be a string literal +f = NewType(name='e', tp=int) # E: NewType(...) expects exactly two positional arguments +g = NewType('f', tp=int) # E: NewType(...) expects exactly two positional arguments +[out] + +[case testNewTypeWithAnyFails] +from typing import NewType, Any +A = NewType('A', Any) # E: Argument 2 to NewType(...) must be subclassable (got "Any") +[out] + +[case testNewTypeWithUnionsFails] +from typing import NewType, Union +Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got "Union[int, float]") +[out] + +[case testNewTypeWithTypeTypeFails] +from typing import NewType, Type +Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got "Type[int]") +a = Foo(type(3)) +[builtins fixtures/args.pyi] +[out] + +[case testNewTypeWithTypeVarsFails] + +from typing import NewType, TypeVar, List +T = TypeVar('T') +A = NewType('A', T) +B = NewType('B', List[T]) +[builtins fixtures/list.pyi] +[out] +main:4: error: Argument 2 to NewType(...) must be subclassable (got T?) +main:4: error: Type variable "__main__.T" is unbound +main:4: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) +main:4: note: (Hint: Use "T" in function signature to bind "T" inside a function) +main:5: error: Type variable "__main__.T" is unbound +main:5: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) +main:5: note: (Hint: Use "T" in function signature to bind "T" inside a function) + +[case testNewTypeRedefiningVariablesFails] + +from typing import NewType + +a = 3 +def f(): a +a = NewType('a', int) # E: Cannot redefine "a" as a NewType \ + # E: Name "a" already defined on line 4 + +b = NewType('b', int) +def g(): b +b = NewType('b', float) # E: Cannot redefine "b" as a NewType \ + # E: Name "b" already defined on line 8 + +c = NewType('c', str) # type: str # E: Cannot declare the type of a NewType declaration + +[case testNewTypeAddingExplicitTypesFails] +from typing import NewType +UserId = NewType('UserId', int) + +a = 3 # type: UserId # E: Incompatible types in assignment (expression has type "int", variable has type "UserId") +[out] + +[case testNewTypeTestSubclassingFails] +from typing import NewType +class A: pass +B = NewType('B', A) +class C(B): pass # E: Cannot subclass "NewType" +[out] + +[case testCannotUseNewTypeWithProtocols] +from typing import Protocol, NewType + +class P(Protocol): + attr: int = 0 +class D: + attr: int + +C = NewType('C', P) # E: NewType cannot be used with protocol classes + +x: C = C(D()) # We still accept this, treating 'C' as non-protocol subclass. +reveal_type(x.attr) # N: Revealed type is "builtins.int" +x.bad_attr # E: "C" has no attribute "bad_attr" +C(1) # E: Argument 1 to "C" has incompatible type "int"; expected "P" +[out] + +[case testNewTypeAny] +from typing import NewType +Any = NewType('Any', int) +Any(5) + +[case testNewTypeWithIsInstanceAndIsSubclass] +from typing import NewType +T = NewType('T', int) +d: object +if isinstance(d, T): # E: Cannot use isinstance() with NewType type + reveal_type(d) # N: Revealed type is "__main__.T" +issubclass(object, T) # E: Cannot use issubclass() with NewType type +[builtins fixtures/isinstancelist.pyi] + +[case testInvalidNewTypeCrash] + +from typing import List, NewType, Union +N = NewType('N', XXX) # E: Argument 2 to NewType(...) must be subclassable (got "Any") \ + # E: Name "XXX" is not defined +x: List[Union[N, int]] +[builtins fixtures/list.pyi] + +[case testTypingExtensionsNewType] +from typing_extensions import NewType +N = NewType("N", int) +x: N +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-optional.test b/test-data/unit/check-optional.test new file mode 100644 index 000000000000..683ce0446915 --- /dev/null +++ b/test-data/unit/check-optional.test @@ -0,0 +1,1345 @@ +-- Tests for strict Optional behavior + +[case testImplicitNoneType] +x = None +x() # E: "None" not callable + +[case testImplicitNoneTypeInNestedFunction] +def f() -> None: + def g() -> None: + x = None + x() # E: "None" not callable + +[case testExplicitNoneType] +x = None # type: None +x() # E: "None" not callable + +[case testNoneMemberOfOptional] +from typing import Optional +x = None # type: Optional[int] + +[case testTypeMemberOfOptional] +from typing import Optional +x = 0 # type: Optional[int] + +[case testNoneNotMemberOfType] +x = None # type: int +[out] +main:1: error: Incompatible types in assignment (expression has type "None", variable has type "int") + +[case testTypeNotMemberOfNone] +x = 0 # type: None +[out] +main:1: error: Incompatible types in assignment (expression has type "int", variable has type "None") + +[case testOptionalNotMemberOfType] +from typing import Optional +def f(a: int) -> None: pass +x = None # type: Optional[int] +f(x) # E: Argument 1 to "f" has incompatible type "Optional[int]"; expected "int" + +[case testIsinstanceCases] +from typing import Optional +x = None # type: Optional[int] +if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "None" +[builtins fixtures/isinstance.pyi] + +[case testIfCases] +from typing import Optional +x = None # type: Optional[int] +if x: + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "Union[Literal[0], None]" +[builtins fixtures/bool.pyi] + +[case testIfNotCases] +from typing import Optional +x = None # type: Optional[int] +if not x: + reveal_type(x) # N: Revealed type is "Union[Literal[0], None]" +else: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/bool.pyi] + +[case testIsNotNoneCases] +from typing import Optional +x = None # type: Optional[int] +if x is not None: + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "None" +[builtins fixtures/bool.pyi] + +[case testIsNoneCases] +from typing import Optional +x = None # type: Optional[int] +if x is None: + reveal_type(x) # N: Revealed type is "None" +else: + reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/bool.pyi] + +[case testAnyCanBeNone] +from typing import Optional, Any +x = None # type: Any +if x is None: + reveal_type(x) # N: Revealed type is "None" +else: + reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/bool.pyi] + +[case testOrCases] +from typing import Optional +x = None # type: Optional[str] +y1 = x or 'a' +reveal_type(y1) # N: Revealed type is "builtins.str" +y2 = x or 1 +reveal_type(y2) # N: Revealed type is "Union[builtins.str, builtins.int]" +z1 = 'a' or x +reveal_type(z1) # N: Revealed type is "Union[builtins.str, None]" +z2 = int() or x +reveal_type(z2) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + +[case testAndCases] +from typing import Optional +x = None # type: Optional[str] +y1 = x and 'b' +reveal_type(y1) # N: Revealed type is "Union[Literal[''], None, builtins.str]" +y2 = x and 1 # x could be '', so... +reveal_type(y2) # N: Revealed type is "Union[Literal[''], None, builtins.int]" +z1 = 'b' and x +reveal_type(z1) # N: Revealed type is "Union[builtins.str, None]" +z2 = int() and x +reveal_type(z2) # N: Revealed type is "Union[Literal[0], builtins.str, None]" + +[case testLambdaReturningNone] +f = lambda: None +x = f() +reveal_type(x) # N: Revealed type is "None" + +[case testNoneArgumentType] +def f(x: None) -> None: pass +f(None) + +[case testInferOptionalFromDefaultNone] +# flags: --implicit-optional +def f(x: int = None) -> None: + x + 1 # E: Unsupported left operand type for + ("None") \ + # N: Left operand is of type "Optional[int]" +f(None) +[out] + +[case testNoInferOptionalFromDefaultNone] +# flags: --no-implicit-optional +def f(x: int = None) -> None: # E: Incompatible default for argument "x" (default has type "None", argument has type "int") \ + # N: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True \ + # N: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase + pass +[out] + +[case testInferOptionalFromDefaultNoneComment] +# flags: --implicit-optional +def f(x=None): + # type: (int) -> None + x + 1 # E: Unsupported left operand type for + ("None") \ + # N: Left operand is of type "Optional[int]" +f(None) +[out] + +[case testNoInferOptionalFromDefaultNoneComment] +# flags: --no-implicit-optional +def f(x=None): # E: Incompatible default for argument "x" (default has type "None", argument has type "int") \ + # N: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True \ + # N: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase + # type: (int) -> None + pass +[out] + +[case testInferOptionalType] +x = None +if bool(): + # scope limit assignment + x = 1 + # in scope of the assignment, x is an int + reveal_type(x) # N: Revealed type is "builtins.int" +# out of scope of the assignment, it's an Optional[int] +reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/bool.pyi] + +[case testInferOptionalTypeLocallyBound] +x = None +x = 1 +reveal_type(x) # N: Revealed type is "builtins.int" + +[case testInferOptionalAnyType] +from typing import Any +x = None +a = None # type: Any +if bool(): + x = a + reveal_type(x) # N: Revealed type is "Any" +reveal_type(x) # N: Revealed type is "Union[Any, None]" +[builtins fixtures/bool.pyi] + +[case testInferOptionalTypeFromOptional] +from typing import Optional +y = None # type: Optional[int] +x = None +x = y +reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + +[case testInferOptionalListType] +x = [None] +x.append(1) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "None" +[builtins fixtures/list.pyi] + +[case testInferNonOptionalListType] +x = [] +x.append(1) +x() # E: "List[int]" not callable +[builtins fixtures/list.pyi] + +[case testInferOptionalDictKeyValueTypes] +x = {None: None} +x["bar"] = 1 +[builtins fixtures/dict.pyi] +[out] +main:2: error: Invalid index type "str" for "Dict[None, None]"; expected type "None" +main:2: error: Incompatible types in assignment (expression has type "int", target has type "None") + +[case testInferNonOptionalDictType] +x = {} +x["bar"] = 1 +x() # E: "Dict[str, int]" not callable +[builtins fixtures/dict.pyi] + +[case testNoneClassVariable] +from typing import Optional +class C: + x = None # type: int + def __init__(self) -> None: + self.x = 0 + +[case testNoneClassVariableInInit] +from typing import Optional +class C: + x = None # type: int + def __init__(self) -> None: + self.x = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") +[out] + +[case testMultipleAssignmentNoneClassVariableInInit] +from typing import Optional +class C: + x, y = None, None # type: int, str + def __init__(self) -> None: + self.x = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + self.y = None # E: Incompatible types in assignment (expression has type "None", variable has type "str") +[builtins fixtures/tuple.pyi] +[out] + +[case testOverloadWithNone] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: None) -> str: pass +@overload +def f(x: int) -> int: pass +reveal_type(f(None)) # N: Revealed type is "builtins.str" +reveal_type(f(0)) # N: Revealed type is "builtins.int" + +[case testOptionalTypeOrTypePlain] +from typing import Optional +def f(a: Optional[int]) -> int: + return a or 0 +[out] + +[case testOptionalTypeOrTypeTypeVar] +from typing import Optional, TypeVar +T = TypeVar('T') +def f(a: Optional[T], b: T) -> T: + return a or b +[out] + +[case testOptionalTypeOrTypeBothOptional] +from typing import Optional +def f(a: Optional[int], b: Optional[int]) -> None: + reveal_type(a or b) +def g(a: int, b: Optional[int]) -> None: + reveal_type(a or b) +[out] +main:3: note: Revealed type is "Union[builtins.int, None]" +main:5: note: Revealed type is "Union[builtins.int, None]" + +[case testOptionalTypeOrTypeComplexUnion] +from typing import Union +def f(a: Union[int, str, None]) -> None: + reveal_type(a or 'default') +[out] +main:3: note: Revealed type is "Union[builtins.int, builtins.str]" + +[case testOptionalTypeOrTypeNoTriggerPlain] +from typing import Optional +def f(a: Optional[int], b: int) -> int: + return b or a +[out] +main:3: error: Incompatible return value type (got "Optional[int]", expected "int") + +[case testOptionalTypeOrTypeNoTriggerTypeVar] +from typing import Optional, TypeVar +T = TypeVar('T') +def f(a: Optional[T], b: T) -> T: + return b or a +[out] +main:4: error: Incompatible return value type (got "Optional[T]", expected "T") + +[case testNoneOrStringIsString] +def f() -> str: + a = None + b = '' + return a or b +[out] + +[case testNoneOrTypeVarIsTypeVar] +from typing import TypeVar +T = TypeVar('T') +def f(b: T) -> T: + a = None + return a or b +[out] + +[case testYieldNothingInFunctionReturningGenerator] +from typing import Generator +def f() -> Generator[None, None, None]: + yield +[out] + +[case testNoneAndStringIsNone] +a = None +b = "foo" +reveal_type(a and b) # N: Revealed type is "None" + +[case testNoneMatchesObjectInOverload] +import a +a.f(None) + +[file a.pyi] +from typing import overload +@overload +def f() -> None: ... +@overload +def f(o: object) -> None: ... + +[case testGenericSubclassReturningNone] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class Base(Generic[T]): + def f(self) -> T: + pass + +class SubNone(Base[None]): + def f(self) -> None: + pass + +class SubInt(Base[int]): + def f(self) -> int: + return 1 + +[case testUseOfNoneReturningFunction] +from typing import Optional +def f() -> None: + pass + +def g(x: Optional[int]) -> int: + pass + +x = f() # E: "f" does not return a value (it only ever returns None) +f() + 1 # E: "f" does not return a value (it only ever returns None) +g(f()) # E: "f" does not return a value (it only ever returns None) + +[case testEmptyReturn] +def f() -> None: + return + +[case testReturnNone] +def f() -> None: + return None + +[case testNoneCallable] +from typing import Callable +def f() -> None: pass +x = f # type: Callable[[], None] + +[case testOptionalCallable] +from typing import Callable, Optional +T = Optional[Callable[..., None]] + +[case testAnyTypeInPartialTypeList] +# flags: --check-untyped-defs +def f(): ... + +def lookup_field(name, obj): + try: + pass + except: + attr = f() + else: + attr = None + +[case testTernaryWithNone] +reveal_type(None if bool() else 0) # N: Revealed type is "Union[None, Literal[0]?]" +[builtins fixtures/bool.pyi] + +[case testListWithNone] +reveal_type([0, None, 0]) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" +[builtins fixtures/list.pyi] + +[case testNoneContextInference] +from typing import Dict, List +def f() -> List[None]: + return [] +def g() -> Dict[None, None]: + return {} +[builtins fixtures/dict.pyi] + +[case testRaiseFromNone] +raise BaseException from None +[builtins fixtures/exception.pyi] + +[case testOptionalNonPartialTypeWithNone] +from typing import Generator +def f() -> Generator[str, None, None]: pass +x = f() +reveal_type(x) # N: Revealed type is "typing.Generator[builtins.str, None, None]" +l = [f()] +reveal_type(l) # N: Revealed type is "builtins.list[typing.Generator[builtins.str, None, None]]" +[builtins fixtures/list.pyi] + +[case testNoneListTernary] +x = [None] if "" else [1] # E: List item 0 has incompatible type "int"; expected "None" +[builtins fixtures/list.pyi] + +[case testListIncompatibleErrorMessage] +from typing import List, Callable + +def foo(l: List[Callable[[], str]]) -> None: pass +def f() -> int: + return 42 + +foo([f]) # E: List item 0 has incompatible type "Callable[[], int]"; expected "Callable[[], str]" +[builtins fixtures/list.pyi] + +[case testInferEqualsNotOptional] +from typing import Optional +x = '' # type: Optional[str] +if x == '': + reveal_type(x) # N: Revealed type is "builtins.str" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" +if x is '': + reveal_type(x) # N: Revealed type is "builtins.str" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/ops.pyi] + +[case testInferEqualsNotOptionalWithUnion] +from typing import Union +x = '' # type: Union[str, int, None] +if x == '': + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +if x is '': + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +[builtins fixtures/ops.pyi] + +[case testInferEqualsNotOptionalWithOverlap] +from typing import Union +x = '' # type: Union[str, int, None] +if x == object(): + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +if x is object(): + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +[builtins fixtures/ops.pyi] + +[case testInferEqualsStillOptionalWithNoOverlap] +from typing import Optional +x = '' # type: Optional[str] +if x == 0: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" +if x is 0: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/ops.pyi] + +[case testInferEqualsStillOptionalWithBothOptional] +from typing import Union +x = '' # type: Union[str, int, None] +y = '' # type: Union[str, None] +if x == y: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +if x is y: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None]" +[builtins fixtures/ops.pyi] + +[case testInferEqualsNotOptionalWithMultipleArgs] +from typing import Optional +x: Optional[int] +y: Optional[int] +if x == y == 1: + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(y) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" + +class A: pass +a: Optional[A] +b: Optional[A] +if a == b == object(): + reveal_type(a) # N: Revealed type is "__main__.A" + reveal_type(b) # N: Revealed type is "__main__.A" +else: + reveal_type(a) # N: Revealed type is "Union[__main__.A, None]" + reveal_type(b) # N: Revealed type is "Union[__main__.A, None]" +[builtins fixtures/ops.pyi] + +[case testInferInWithErasedTypes] +from typing import TypeVar, Callable + +T = TypeVar('T') +def foo(f: Callable[[T], bool], it: T) -> None: ... + +foo(lambda x: x in [1, 2] and bool(), 3) +[builtins fixtures/list.pyi] + +[case testWarnNoReturnWorksWithStrictOptional] +# flags: --warn-no-return +def f() -> None: + 1 + 1 # no error + +def g() -> int: + 1 + 1 # +[out] +main:5: error: Missing return statement + +[case testGenericTypeAliasesOptional] +from typing import TypeVar, Generic, Optional +T = TypeVar('T') +class Node(Generic[T]): + def __init__(self, x: T) -> None: + self.x = x + +ONode = Optional[Node[T]] +def f(x: T) -> ONode[T]: + if 1 > 0: + return Node(x) + else: + return None + +x = None # type: ONode[int] +if int(): + x = f(1) +if int(): + x = f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +x.x = 1 # E: Item "None" of "Optional[Node[int]]" has no attribute "x" +if x is not None: + x.x = 1 # OK here + +[builtins fixtures/ops.pyi] + +[case testOptionalTypeNarrowedInBooleanStatement] +from typing import Optional + +x: Optional[int] = None +x is not None and x + 42 +x is not None and x + '42' # E: Unsupported operand types for + ("int" and "str") +[builtins fixtures/isinstance.pyi] + +[case testInvalidBooleanBranchIgnored] +from typing import Optional + +x = None +x is not None and x + 42 +[builtins fixtures/isinstance.pyi] + +[case testOptionalLambdaInference] +from typing import Optional, Callable +f = None # type: Optional[Callable[[int], None]] +f = lambda x: None +f(0) +[builtins fixtures/function.pyi] + +[case testDontSimplifyNoneUnionsWithStrictOptional] +from typing import Any, TypeVar, Union +A = None # type: Any +class C(A): pass +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass +a = None # type: Any + +# Test both orders +reveal_type(u(C(), None)) # N: Revealed type is "Union[None, __main__.C]" +reveal_type(u(None, C())) # N: Revealed type is "Union[__main__.C, None]" + +reveal_type(u(a, None)) # N: Revealed type is "Union[None, Any]" +reveal_type(u(None, a)) # N: Revealed type is "Union[Any, None]" + +reveal_type(u(1, None)) # N: Revealed type is "Union[None, builtins.int]" +reveal_type(u(None, 1)) # N: Revealed type is "Union[builtins.int, None]" + +[case testOptionalAndAnyBaseClass] +from typing import Any, Optional +A = None # type: Any +class C(A): + pass +x = None # type: Optional[C] +x.foo() # E: Item "None" of "Optional[C]" has no attribute "foo" + +[case testIsinstanceAndOptionalAndAnyBase] +from typing import Any, Optional + +B = None # type: Any +class A(B): pass + +def f(a: Optional[A]): + reveal_type(a) # N: Revealed type is "Union[__main__.A, None]" + if a is not None: + reveal_type(a) # N: Revealed type is "__main__.A" + else: + reveal_type(a) # N: Revealed type is "None" + reveal_type(a) # N: Revealed type is "Union[__main__.A, None]" +[builtins fixtures/isinstance.pyi] + +[case testFlattenOptionalUnion] +from typing import Optional, Union + +x: Optional[Union[int, str]] +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +y: Optional[Union[int, None]] +reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" + +[case testOverloadWithNoneAndOptional] +from typing import overload, Optional + +@overload +def f(x: int) -> str: ... +@overload +def f(x: Optional[int]) -> Optional[str]: ... +def f(x): return x + +reveal_type(f(1)) # N: Revealed type is "builtins.str" +reveal_type(f(None)) # N: Revealed type is "Union[builtins.str, None]" +x: Optional[int] +reveal_type(f(x)) # N: Revealed type is "Union[builtins.str, None]" + +[case testUnionTruthinessTracking] +from typing import Optional, Any +def test_or_shortcut(value: Optional[Any]) -> None: + if not value: + pass + if not value or value.get('foo') == 'hello': + pass +[builtins fixtures/bool.pyi] + +[case testNarrowingFromObjectToOptional] +from typing import Optional +x: object +y: Optional[int] +x = y +reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" +[out] + +[case testNarrowOptionalOutsideLambda] +from typing import Optional + +class A: + a: int + +def f(x: Optional[A]) -> None: + assert x + lambda: x.a +[builtins fixtures/isinstancelist.pyi] + +[case testNarrowOptionalOutsideLambdaWithDeferred] +from typing import Optional + +class A: + a: int + + def f(self, x: Optional['A']) -> None: + assert x + lambda: (self.y, x.a) # E: Cannot determine type of "y" + self.y = int() +[builtins fixtures/isinstancelist.pyi] + +[case testDeferredAndOptionalInferenceSpecialCase] +def f() -> str: + y + x = None + if int(): + x = '' + if x is None: + x = '' + g(x) + return x + +def g(x: str) -> None: pass + +y = int() +[builtins fixtures/bool.pyi] + +[case testOptionalAssignAny1] +from typing import Optional +def f(): + return 0 + +def g(x: Optional[int]) -> int: + if x is None: + reveal_type(x) # N: Revealed type is "None" + # As a special case for Unions containing None, during + x = f() + reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" + return x + +[builtins fixtures/bool.pyi] + +[case testOptionalAssignAny2] +from typing import Optional +def f(): + return 0 + +def g(x: Optional[int]) -> int: + if x is None: + reveal_type(x) # N: Revealed type is "None" + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + # Since we've assigned to x, the special case None behavior shouldn't happen + x = f() + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + return x # E: Incompatible return value type (got "Optional[int]", expected "int") + +[builtins fixtures/bool.pyi] + +[case testOptionalAssignAny3] +from typing import Optional +def f(): + return 0 + +def g(x: Optional[int]) -> int: + if x is not None: + return x + reveal_type(x) # N: Revealed type is "None" + if 1: + x = f() + reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" + return x + +[builtins fixtures/bool.pyi] + +[case testStrictOptionalCovarianceCrossModule] +# flags: --config-file tmp/mypy.ini +from a import asdf +x = ["lol"] +asdf(x) + +[file a.py] +from typing import List, Optional + +def asdf(x: List[Optional[str]]) -> None: + pass + +x = ["lol"] +asdf(x) + +[file mypy.ini] +\[mypy] +\[mypy-a] +strict_optional = False +[out] +main:4: error: Argument 1 to "asdf" has incompatible type "List[str]"; expected "List[Optional[str]]" +main:4: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance +main:4: note: Consider using "Sequence" instead, which is covariant +[builtins fixtures/list.pyi] + +[case testOptionalBackwards1] +from typing import Any, Optional + +def f1(b: bool) -> Optional[int]: + if b: + z = 10 + reveal_type(z) # N: Revealed type is "builtins.int" + else: + z = None + reveal_type(z) # N: Revealed type is "None" + reveal_type(z) # N: Revealed type is "Union[builtins.int, None]" + return z + +def f2(b: bool) -> int: + if b: + z = 10 + else: + z = None + return z # E: Incompatible return value type (got "Optional[int]", expected "int") + +def f3(b: bool) -> int: + # XXX: This one is a little questionable! Maybe we *do* want to allow this? + z = 10 + if b: + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + return z + +def f4() -> Optional[int]: + z = 10 + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + + return z + +def f5() -> None: + z = 10 + + def f() -> None: + nonlocal z + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +def f6(b: bool) -> None: + if b: + z = 10 + else: + z = 11 + + def f() -> None: + nonlocal z + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +def f7(b: bool) -> None: + if b: + z = 10 + else: + z = 11 + + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +def f8(b: bool, c: bool) -> Optional[int]: + if b: + if c: + z = 10 + else: + z = 11 + else: + z = None + return z + +def f9(b: bool) -> None: + if b: + z: int = 10 + else: + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +def f10(b: bool) -> None: + z: int + if b: + z = 10 + else: + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +def f11(b: bool, c: bool) -> None: + if b: + z = 10 + elif c: + z = 30 + else: + z = None + +def f12(b: bool, a: Any) -> None: + if b: + z = a + else: + z = None + reveal_type(z) # N: Revealed type is "Any" + +def f13(b: bool, a: Any) -> None: + if b: + try: + z = f2(True) + except Exception: + raise RuntimeError + else: + z = None + +def f14(b: bool, a: Any) -> None: + if b: + with a: + z = 10 + else: + z = None + +def f15() -> None: + try: + z = f2(True) + except Exception: + z = None + reveal_type(z) # N: Revealed type is "Union[builtins.int, None]" + +def f16(z: Any) -> None: + for x in z: + if x == 0: + y = 50 + break + else: + y = None + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" + +def f17(b: bool, c: bool, d: bool) -> None: + if b: + z = 2 + elif c: + z = None + elif d: + z = 3 + reveal_type(z) # N: Revealed type is "Union[builtins.int, None]" + +def f18(b: bool, c: bool, d: bool) -> None: + if b: + z = 4 + else: + if c: + z = 5 + else: + z = None + reveal_type(z) # N: Revealed type is "Union[builtins.int, None]" + +def f19(b: bool, c: bool, d: bool) -> None: + if b: + z = 5 + else: + z = None + if c: + z = 6 + reveal_type(z) # N: Revealed type is "Union[builtins.int, None]" + +def f20(b: bool) -> None: + if b: + x: Any = 5 + else: + x = None + reveal_type(x) # N: Revealed type is "Any" + +def f_unannot(): pass + +def f21(b: bool) -> None: + if b: + x = f_unannot() + else: + x = None + reveal_type(x) # N: Revealed type is "Any" + +def f22(b: bool) -> None: + if b: + z = 10 + if not b: + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +def f23(b: bool) -> None: + if b: + z = 10 + if b: + z = 11 + else: + z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +[builtins fixtures/exception.pyi] + +[case testOptionalBackwards2] + +def f1(b: bool) -> None: + if b: + x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") + else: + x = None + +def f2(b: bool) -> None: + if b: + x = [] + x.append(1) + else: + x = None + reveal_type(x) # N: Revealed type is "Union[builtins.list[builtins.int], None]" + + +[builtins fixtures/list.pyi] + +[case testOptionalBackwards3] + +# We don't allow this sort of updating for globals or attributes currently. +gb: bool +if gb: + Z = 10 +else: + Z = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + +class Foo: + def __init__(self, b: bool) -> None: + if b: + self.x = 5 + else: + self.x = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") + + def foo(self) -> None: + reveal_type(self.x) # N: Revealed type is "builtins.int" + +[case testOptionalBackwards4] +from typing import Any, Optional + +def f1(b: bool) -> Optional[int]: + if b: + z = 10 + reveal_type(z) # N: Revealed type is "builtins.int" + else: + # Force the node to get deferred between the two assignments + Defer().defer + z = None + reveal_type(z) # N: Revealed type is "None" + reveal_type(z) # N: Revealed type is "Union[builtins.int, None]" + return z + +class Defer: + def __init__(self) -> None: + self.defer = 10 + +[case testOptionalIterator] +# mypy: no-strict-optional +from typing import Optional, List + +x: Optional[List[int]] +if 3 in x: + pass + +[case testNarrowedVariableInNestedFunctionBasic] +from typing import Optional + +def can_narrow(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return reveal_type(x) # N: Revealed type is "builtins.str" + nested() + +def foo(a): pass + +class C: + def can_narrow_in_method(self, x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return reveal_type(x) # N: Revealed type is "builtins.str" + # Reading the variable is fine + y = x + with foo(x): + foo(x) + for a in foo(x): + foo(x) + nested() + +def can_narrow_lambda(x: Optional[str]) -> None: + if x is None: + x = "a" + nested = lambda: x + reveal_type(nested()) # N: Revealed type is "builtins.str" + +def cannot_narrow_if_reassigned(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + if int(): + x = None + nested() + +x: Optional[str] = "x" + +def narrow_global_in_func() -> None: + global x + if x is None: + x = "a" + def nested() -> str: + # This should perhaps not be narrowed, since the nested function could outlive + # the outer function, and since other functions could also assign to x, but + # this seems like a minor issue. + return x + nested() + +x = "y" + +def narrowing_global_at_top_level_not_propagated() -> str: + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + +[case testNarrowedVariableInNestedFunctionMore1] +from typing import Optional, overload + +class C: + a: Optional[str] + +def attribute_narrowing(c: C) -> None: + # This case is not supported, since we can't keep track of assignments to attributes. + c.a = "x" + def nested() -> str: + return c.a # E: Incompatible return value type (got "Optional[str]", expected "str") + nested() + +def assignment_in_for(x: Optional[str]) -> None: + if x is None: + x = "e" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + for x in ["x"]: + pass + +def foo(): pass + +def assignment_in_with(x: Optional[str]) -> None: + if x is None: + x = "e" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + with foo() as x: + pass + +g: Optional[str] + +def assign_to_global() -> None: + global g + g = "x" + # This is unsafe, but we don't generate an error, for convenience. Besides, + # this is probably a very rare case. + def nested() -> str: + return g + +def assign_to_nonlocal(x: Optional[str]) -> None: + def nested() -> str: + nonlocal x + + if x is None: + x = "a" + + def nested2() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + + return nested2() + nested() + x = None + +def dec(f): + return f + +@dec +def decorated_outer(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x + nested() + +@dec +def decorated_outer_bad(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + x = None + nested() + +def decorated_inner(x: Optional[str]) -> None: + if x is None: + x = "a" + @dec + def nested() -> str: + return x + nested() + +def decorated_inner_bad(x: Optional[str]) -> None: + if x is None: + x = "a" + @dec + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + x = None + nested() + +@overload +def overloaded_outer(x: None) -> None: ... +@overload +def overloaded_outer(x: str) -> None: ... +def overloaded_outer(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x + nested() + +@overload +def overloaded_outer_bad(x: None) -> None: ... +@overload +def overloaded_outer_bad(x: str) -> None: ... +def overloaded_outer_bad(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + x = None + nested() + +[case testNarrowedVariableInNestedFunctionMore2] +from typing import Optional + +def narrow_multiple(x: Optional[str], y: Optional[int]) -> None: + z: Optional[str] = x + if x is None: + x = "" + if y is None: + y = 1 + if int(): + if z is None: + z = "" + def nested() -> None: + a: str = x + b: int = y + c: str = z + nested() + +def narrow_multiple_partial(x: Optional[str], y: Optional[int]) -> None: + z: Optional[str] = x + if x is None: + x = "" + if isinstance(y, int): + if z is None: + z = "" + def nested() -> None: + a: str = x + b: int = y + c: str = z # E: Incompatible types in assignment (expression has type "Optional[str]", variable has type "str") + z = None + nested() + +def multiple_nested_functions(x: Optional[str], y: Optional[str]) -> None: + if x is None: + x = "" + def nested1() -> str: + return x + if y is None: + y = "" + def nested2() -> str: + a: str = y + return x + +class C: + a: str + def __setitem__(self, key, value): pass + +def narrowed_variable_used_in_lvalue_but_not_assigned(c: Optional[C]) -> None: + if c is None: + c = C() + def nested() -> C: + return c + c.a = "x" + c[1] = 2 + cc = C() + cc[c] = 3 + nested() + +def narrow_with_multi_lvalues_1(x: Optional[str]) -> None: + if x is None: + x = "" + + def nested() -> str: + return x + + y = z = None + +def narrow_with_multi_lvalue_2(x: Optional[str]) -> None: + if x is None: + x = "" + + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + + x = y = None + +def narrow_with_multi_lvalue_3(x: Optional[str]) -> None: + if x is None: + x = "" + + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + + y = x = None + +def narrow_with_multi_assign_1(x: Optional[str]) -> None: + if x is None: + x = "" + + def nested() -> str: + return x + + y, z = None, None + +def narrow_with_multi_assign_2(x: Optional[str]) -> None: + if x is None: + x = "" + + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + + x, y = None, None + +def narrow_with_multi_assign_3(x: Optional[str]) -> None: + if x is None: + x = "" + + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + + y, x = None, None + +[builtins fixtures/isinstance.pyi] + +[case testNestedFunctionSpecialCase] +class C: + def __enter__(self, *args): ... + def __exit__(self, *args) -> bool: ... + +def f(x: object) -> None: + if x is not None: + pass + + def nested() -> None: + with C(): + pass +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test new file mode 100644 index 000000000000..9d01ce6bd480 --- /dev/null +++ b/test-data/unit/check-overloading.test @@ -0,0 +1,6770 @@ +-- Test cases for function overloading +[case testOverloadNotImportedNoCrash] + +@overload +def f(a): pass +@overload +def f(a): pass +def f(a): pass +f(0) + +@overload # E: Name "overload" is not defined +def g(a:int): pass +def g(a): pass # E: Name "g" already defined on line 9 +g(0) + +@something # E: Name "something" is not defined +def r(a:int): pass +def r(a): pass # E: Name "r" already defined on line 14 +r(0) +[out] +main:2: error: Name "overload" is not defined +main:4: error: Name "f" already defined on line 2 +main:4: error: Name "overload" is not defined +main:6: error: Name "f" already defined on line 2 + +[case testTypeCheckOverloadWithImplementation] +from typing import overload, Any +class A: pass +class B: pass + +@overload +def f(x: 'A') -> 'B': ... +@overload +def f(x: 'B') -> 'A': ... + +def f(x: Any) -> Any: + pass + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testTypingExtensionsOverload] +from typing import Any +from typing_extensions import overload +class A: pass +class B: pass + +@overload +def f(x: 'A') -> 'B': ... +@overload +def f(x: 'B') -> 'A': ... + +def f(x: Any) -> Any: + pass + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testOverloadNeedsImplementation] +from typing import overload, Any + +class A: pass +class B: pass + +@overload # E: An overloaded function outside a stub file must have an implementation +def f(x: 'A') -> 'B': ... +@overload +def f(x: 'B') -> 'A': ... + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testSingleOverloadNoImplementation] +from typing import overload, Any +@overload # E: Single overload definition, multiple required +def f(x: 'A') -> 'B': ... + +class A: pass +class B: pass +[builtins fixtures/isinstance.pyi] + +[case testOverloadByAnyOtherName] +from typing import overload as rose +from typing import Any +class A: pass +class B: pass + +@rose +def f(x: 'A') -> 'B': ... +@rose +def f(x: 'B') -> 'A': ... + +def f(x: Any) -> Any: + pass + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testTypeCheckOverloadWithDecoratedImplementation] +from typing import overload, Any + +class A: pass +class B: pass + +def deco(fun): ... + +@overload +def f(x: 'A') -> 'B': ... +@overload +def f(x: 'B') -> 'A': ... + +@deco +def f(x: Any) -> Any: + pass + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testOverloadDecoratedImplementationNotLast] +from typing import overload, Any + +def deco(fun): ... + +@overload +def f(x: 'A') -> 'B': ... + +@deco # E: The implementation for an overloaded function must come last +def f(x: Any) -> Any: + pass + +@overload +def f(x: 'B') -> 'A': ... + +class A: pass +class B: pass +[builtins fixtures/isinstance.pyi] + +[case testOverloadImplementationNotLast] +from typing import overload, Any + +@overload +def f(x: 'A') -> 'B': ... + +def f(x: Any) -> Any: # E: The implementation for an overloaded function must come last + pass + +@overload +def f(x: 'B') -> 'A': ... + +class A: pass +class B: pass +[builtins fixtures/isinstance.pyi] + +[case testDecoratedRedefinitionIsNotOverload] +from typing import overload, Any + +def deco(fun): ... + +@deco +def f(x: 'A') -> 'B': ... +@deco # E: Name "f" already defined on line 5 +def f(x: 'B') -> 'A': ... +@deco # E: Name "f" already defined on line 5 +def f(x: Any) -> Any: ... + +class A: pass +class B: pass +[builtins fixtures/isinstance.pyi] + +[case testTypeCheckOverloadWithImplementationError] +from typing import overload, Any + +class A: pass +class B: pass + +@overload +def f(x: 'A') -> 'B': ... +@overload +def f(x: 'B') -> 'A': ... + +def f(x: Any) -> Any: + foo = 1 + if int(): + foo = "bar" # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +@overload +def g(x: 'A') -> 'B': ... +@overload +def g(x: 'B') -> 'A': ... + +def g(x): + foo = 1 + if int(): + foo = "bar" + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testTypeCheckOverloadWithUntypedImplAndMultipleVariants] +from typing import overload + +@overload +def f(x: int) -> str: ... +@overload +def f(x: str) -> int: ... # E: Overloaded function signatures 2 and 3 overlap with incompatible return types +@overload +def f(x: object) -> str: ... +def f(x): ... + +[case testTypeCheckOverloadWithImplTooSpecificArg] +from typing import overload, Any + +class A: pass +class B: pass + +a = A() + +@overload +def f(x: 'A') -> 'B': ... +@overload +def f(x: 'B') -> 'A': ... + +def f(x: 'A') -> Any: # E: Overloaded function implementation does not accept all possible arguments of signature 2 + pass + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" + +[builtins fixtures/isinstance.pyi] + +[case testTypeCheckOverloadWithImplTooSpecificRetType] +from typing import overload, Any + +class A: pass +class B: pass + +a = A() + +@overload +def f(x: 'A') -> 'B': ... +@overload +def f(x: 'B') -> 'A': ... + +def f(x: Any) -> 'B': # E: Overloaded function implementation cannot produce return type of signature 2 + return B() + +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.A" + +[builtins fixtures/isinstance.pyi] + +[case testTypeCheckOverloadWithImplTypeVar] +from typing import overload, Any, TypeVar + +T = TypeVar('T') + +class A: pass +class B: pass + +a = A() + +@overload +def f(x: 'A') -> 'A': ... +@overload +def f(x: 'B') -> 'B': ... + +def f(x: T) -> T: + ... + +reveal_type(f(A())) # N: Revealed type is "__main__.A" +reveal_type(f(B())) # N: Revealed type is "__main__.B" + +[builtins fixtures/isinstance.pyi] + +[case testTypeCheckOverloadWithImplTypeVarProblems] +from typing import overload, Any, TypeVar, Union + +T = TypeVar('T', bound='A') + +class A: pass +class B: pass + +a = A() + +@overload +def f(x: 'A') -> 'A': ... +@overload +def f(x: 'B') -> 'B': ... + +def f(x: Union[T, B]) -> T: # E: Overloaded function implementation cannot satisfy signature 2 due to inconsistencies in how they use type variables + ... + +reveal_type(f(A())) # N: Revealed type is "__main__.A" +reveal_type(f(B())) # N: Revealed type is "__main__.B" + +[builtins fixtures/isinstance.pyi] + +[case testTypeCheckOverloadImplementationTypeVarWithValueRestriction] +from typing import overload, TypeVar, Union + +class A: pass +class B: pass +class C: pass + +T = TypeVar('T', A, B) + +@overload +def foo(x: T) -> T: ... +@overload +def foo(x: C) -> int: ... +def foo(x: Union[A, B, C]) -> Union[A, B, int]: + if isinstance(x, C): + return 3 + else: + return x + +@overload +def bar(x: T) -> T: ... +@overload +def bar(x: C) -> int: ... +def bar(x: Union[T, C]) -> Union[T, int]: + if isinstance(x, C): + return 3 + else: + return x + +[builtins fixtures/isinstancelist.pyi] + +[case testTypeCheckOverloadImplementationTypeVarDifferingUsage1] +from typing import overload, Union, List, TypeVar, Generic + +T = TypeVar('T') + +@overload +def foo(t: List[T]) -> T: ... +@overload +def foo(t: T) -> T: ... +def foo(t: Union[List[T], T]) -> T: + if isinstance(t, list): + return t[0] + else: + return t + +class Wrapper(Generic[T]): + @overload + def foo(self, t: List[T]) -> T: ... + @overload + def foo(self, t: T) -> T: ... + def foo(self, t: Union[List[T], T]) -> T: + if isinstance(t, list): + return t[0] + else: + return t +[builtins fixtures/isinstancelist.pyi] + +[case testTypeCheckOverloadImplementationTypeVarDifferingUsage2] +from typing import overload, Union, List, TypeVar, Generic + +T = TypeVar('T') + +# Note: this is unsafe when T = object +@overload +def foo(t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def foo(t: T, s: T) -> str: ... +def foo(t, s): pass + +class Wrapper(Generic[T]): + @overload + def foo(self, t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error + @overload + def foo(self, t: T, s: T) -> str: ... + def foo(self, t, s): pass + +class Dummy(Generic[T]): pass + +# Same root issue: why does the additional constraint bound T <: T +# cause the constraint solver to not infer T = object like it did in the +# first example? +@overload +def bar(d: Dummy[T], t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error +@overload +def bar(d: Dummy[T], t: T, s: T) -> str: ... +def bar(d: Dummy[T], t, s): pass +[builtins fixtures/isinstancelist.pyi] + +[case testTypeCheckOverloadedFunctionBody] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: 'A'): + if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + x = A() +@overload +def f(x: 'B'): + if int(): + x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + x = B() +class A: pass +class B: pass +[out] + +[case testTypeCheckOverloadedMethodBody] +from foo import * +[file foo.pyi] +from typing import overload +class A: + @overload + def f(self, x: 'A'): + if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + x = A() + @overload + def f(self, x: 'B'): + if int(): + x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + x = B() +class B: pass +[out] + +[case testCallToOverloadedFunction] +from foo import * +[file foo.pyi] +from typing import overload +f(C()) # E: No overload variant of "f" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f(x: A) -> None \ + # N: def f(x: B) -> None +f(A()) +f(B()) + +@overload +def f(x: 'A') -> None: pass +@overload +def f(x: 'B') -> None: pass + +class A: pass +class B: pass +class C: pass + +[case testOverloadedFunctionReturnValue] +from foo import * +[file foo.pyi] +from typing import overload +a: A +b: B +if int(): + b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = f(a) +if int(): + b = f(b) + +@overload +def f(x: 'A') -> 'A': pass +@overload +def f(x: 'B') -> 'B': pass +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testCallToOverloadedMethod] +from foo import * +[file foo.pyi] +from typing import overload +A().f(C()) # E: No overload variant of "f" of "A" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f(self, x: A) -> None \ + # N: def f(self, x: B) -> None +A().f(A()) +A().f(B()) + +class A: + @overload + def f(self, x: 'A') -> None: pass + @overload + def f(self, x: 'B') -> None: pass + +class B: pass +class C: pass + +[case testOverloadedMethodReturnValue] +from foo import * +[file foo.pyi] +from typing import overload +a: A +b: B +if int(): + b = a.f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = a.f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = a.f(a) +if int(): + b = a.f(b) + +class A: + @overload + def f(self, x: 'A') -> 'A': pass + @overload + def f(self, x: 'B') -> 'B': pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testOverloadsWithDifferentArgumentCounts] +from foo import * +[file foo.pyi] +from typing import overload +a: A +b: B +if int(): + a = f(a) +if int(): + b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +f(b) # E: No overload variant of "f" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f(x: A) -> A \ + # N: def f(x: B, y: A) -> B +if int(): + b = f(b, a) +if int(): + a = f(b, a) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +f(a, a) # E: No overload variant of "f" matches argument types "A", "A" \ + # N: Possible overload variants: \ + # N: def f(x: A) -> A \ + # N: def f(x: B, y: A) -> B +f(b, b) # E: No overload variant of "f" matches argument types "B", "B" \ + # N: Possible overload variants: \ + # N: def f(x: A) -> A \ + # N: def f(x: B, y: A) -> B + +@overload +def f(x: 'A') -> 'A': pass +@overload +def f(x: 'B', y: 'A') -> 'B': pass +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testGenericOverloadVariant] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar, Generic +t = TypeVar('t') +ab: A[B] +ac: A[C] +b: B +c: C +if int(): + b = f(ab) + c = f(ac) + b = f(ac) # E: Incompatible types in assignment (expression has type "C", variable has type "B") + b = f(b) + c = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "C") +@overload +def f(x: 'A[t]') -> t: pass +@overload +def f(x: 'B') -> 'B': pass +class A(Generic[t]): pass +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] + +[case testOverloadedInit] +from foo import * +[file foo.pyi] +from typing import overload +a: A +b: B +a = A(a) +a = A(b) +a = A(object()) # E: No overload variant of "A" matches argument type "object" \ + # N: Possible overload variants: \ + # N: def __init__(self, a: A) -> A \ + # N: def __init__(self, b: B) -> A + +class A: + @overload + def __init__(self, a: 'A') -> None: pass + @overload + def __init__(self, b: 'B') -> None: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testIntersectionTypeCompatibility] +from foo import * +[file foo.pyi] +from typing import overload, Callable +o: object +a: A + +if int(): + a = f # E: Incompatible types in assignment (expression has type overloaded function, variable has type "A") +if int(): + o = f + +@overload +def f(a: 'A') -> None: pass +@overload +def f(a: Callable[[], None]) -> None: pass +class A: pass + +[case testCompatibilityOfIntersectionTypeObjectWithStdType] +from foo import * +[file foo.pyi] +from typing import overload +t: type +a: A + +if int(): + a = A # E: Incompatible types in assignment (expression has type "Type[A]", variable has type "A") + t = A + +class A: + @overload + def __init__(self, a: 'A') -> None: pass + @overload + def __init__(self, a: 'B') -> None: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testOverloadedGetitem] +from foo import * +[file foo.pyi] +from typing import overload +a: int +b: str +if int(): + a = A()[a] +if int(): + b = A()[a] # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + b = A()[b] +if int(): + a = A()[b] # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +class A: + @overload + def __getitem__(self, a: int) -> int: pass + @overload + def __getitem__(self, b: str) -> str: pass +[builtins fixtures/tuple.pyi] + +[case testOverloadedGetitemWithGenerics] +from foo import * +[file foo.pyi] +from typing import TypeVar, Generic, overload +t = TypeVar('t') +a: A +b: B +c: C[A] +if int(): + a = c[a] + b = c[a] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = c[b] + b = c[b] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +class C(Generic[t]): + @overload + def __getitem__(self, a: 'A') -> t: pass + @overload + def __getitem__(self, b: 'B') -> t: pass +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testImplementingOverloadedMethod] +from foo import * +[file foo.pyi] +from typing import overload +from abc import abstractmethod, ABCMeta + +class I(metaclass=ABCMeta): + @overload + @abstractmethod + def f(self) -> None: pass + @overload + @abstractmethod + def f(self, a: 'A') -> None: pass +class A(I): + @overload + def f(self) -> None: pass + @overload + def f(self, a: 'A') -> None: pass + +[case testOverloadWithFunctionType] +from foo import * +[file foo.pyi] +from typing import overload, Callable +class A: pass +@overload +def f(x: A) -> None: pass +@overload +def f(x: Callable[[], None]) -> None: pass + +f(A()) +[builtins fixtures/function.pyi] + +[case testVarArgsOverload] +from foo import * +[file foo.pyi] +from typing import overload, Any +@overload +def f(x: 'A', *more: Any) -> 'A': pass +@overload +def f(x: 'B', *more: Any) -> 'A': pass +f(A()) +f(A(), A, A) +f(B()) +f(B(), B) +f(B(), B, B) +f(object()) # E: No overload variant of "f" matches argument type "object" \ + # N: Possible overload variants: \ + # N: def f(x: A, *more: Any) -> A \ + # N: def f(x: B, *more: Any) -> A +class A: pass +class B: pass +[builtins fixtures/list.pyi] + +[case testVarArgsOverload2] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: 'A', *more: 'B') -> 'A': pass +@overload +def f(x: 'B', *more: 'A') -> 'A': pass +f(A(), B()) +f(A(), B(), B()) +f(A(), A(), B()) # E: No overload variant of "f" matches argument types "A", "A", "B" \ + # N: Possible overload variants: \ + # N: def f(x: A, *more: B) -> A \ + # N: def f(x: B, *more: A) -> A +f(A(), B(), A()) # E: No overload variant of "f" matches argument types "A", "B", "A" \ + # N: Possible overload variants: \ + # N: def f(x: A, *more: B) -> A \ + # N: def f(x: B, *more: A) -> A +class A: pass +class B: pass +[builtins fixtures/list.pyi] + +[case testOverloadWithTypeObject] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(a: 'A', t: type) -> None: pass +@overload +def f(a: 'B', t: type) -> None: pass +f(A(), B) +f(B(), A) +class A: pass +class B: pass +[builtins fixtures/function.pyi] + +[case testOverloadedInitAndTypeObjectInOverload] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(t: type) -> 'A': pass +@overload +def f(t: 'A') -> 'B': pass +a: A +b: B +if int(): + a = f(A) +if int(): + b = f(a) +if int(): + b = f(A) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = f(a) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +class A: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, a: 'A') -> None: pass +class B: + pass +[builtins fixtures/tuple.pyi] + +[case testOverlappingErasedSignatures] +from foo import * +[file foo.pyi] +from typing import overload, List +@overload +def f(a: List[int]) -> int: pass +@overload +def f(a: List[str]) -> int: pass +list_int = [] # type: List[int] +list_str = [] # type: List[str] +list_object = [] # type: List[object] +n = f(list_int) +m = f(list_str) +def p(): n, m # Prevent redefinition +n = 1 +m = 1 +n = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +m = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +f(list_object) # E: Argument 1 to "f" has incompatible type "List[object]"; expected "List[int]" +[builtins fixtures/list.pyi] + +[case testOverlappingOverloadSignatures] +from foo import * +[file foo.pyi] +from typing import overload +class A: pass +class B(A): pass +@overload +def f(x: B) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: A) -> str: pass + +[case testContravariantOverlappingOverloadSignatures] +from foo import * +[file foo.pyi] +from typing import overload +class A: pass +class B(A): pass +@overload +def f(x: A) -> A: pass +@overload +def f(x: B) -> B: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +[case testPartiallyCovariantOverlappingOverloadSignatures] +from foo import * +[file foo.pyi] +from typing import overload +class A: pass +class B(A): pass +@overload +def f(x: B) -> A: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: A) -> B: pass + +[case testPartiallyContravariantOverloadSignatures] +from foo import * +[file foo.pyi] +from typing import overload +class A: pass +class B(A): pass +@overload +def g(x: A) -> int: pass +@overload +def g(x: B) -> str: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +[case testCovariantOverlappingOverloadSignatures] +from foo import * +[file foo.pyi] +from typing import overload +class A: pass +class B(A): pass +@overload +def g(x: B) -> B: pass +@overload +def g(x: A) -> A: pass + +[case testCovariantOverlappingOverloadSignaturesWithSomeSameArgTypes] +from foo import * +[file foo.pyi] +from typing import overload +class A: pass +class B(A): pass +@overload +def g(x: int, y: B) -> B: pass +@overload +def g(x: int, y: A) -> A: pass + +[case testCovariantOverlappingOverloadSignaturesWithAnyType] +from foo import * +[file foo.pyi] +from typing import Any, overload +@overload +def g(x: int) -> int: pass +@overload +def g(x: Any) -> Any: pass + +[case testContravariantOverlappingOverloadSignaturesWithAnyType] +from foo import * +[file foo.pyi] +from typing import Any, overload +@overload +def g(x: Any) -> Any: pass +@overload +def g(x: int) -> int: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +[case testOverloadedLtAndGtMethods] +from foo import * +[file foo.pyi] +from typing import overload +class A: + def __lt__(self, x: A) -> int: pass + def __gt__(self, x: A) -> int: pass +class B: + @overload + def __lt__(self, x: B) -> int: pass + @overload + def __lt__(self, x: A) -> int: pass + @overload + def __gt__(self, x: B) -> int: pass + @overload + def __gt__(self, x: A) -> int: pass +A() < A() +A() < B() +B() < A() +B() < B() +A() < object() # E: Unsupported operand types for < ("A" and "object") +B() < object() # E: No overload variant of "__lt__" of "B" matches argument type "object" \ + # N: Possible overload variants: \ + # N: def __lt__(self, B, /) -> int \ + # N: def __lt__(self, A, /) -> int + +[case testOverloadedForwardMethodAndCallingReverseMethod] +from foo import * +[file foo.pyi] +from typing import overload +class A: + @overload + def __add__(self, x: 'A') -> int: pass + @overload + def __add__(self, x: int) -> int: pass +class B: + def __radd__(self, x: A) -> int: pass +A() + A() +A() + 1 +A() + B() +A() + '' # E: No overload variant of "__add__" of "A" matches argument type "str" \ + # N: Possible overload variants: \ + # N: def __add__(self, A, /) -> int \ + # N: def __add__(self, int, /) -> int + +[case testOverrideOverloadSwapped] +from foo import * +[file foo.pyi] +from typing import overload + +class Parent: + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... +class Child(Parent): + @overload # E: Signature of "f" incompatible with supertype "Parent" \ + # N: Overload variants must be defined in the same order as they are in "Parent" + def f(self, x: str) -> str: ... + @overload + def f(self, x: int) -> int: ... + +[case testOverrideOverloadSwappedWithExtraVariants] +from foo import * +[file foo.pyi] +from typing import overload + +class bool: pass + +class Parent: + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... +class Child1(Parent): + @overload # E: Signature of "f" incompatible with supertype "Parent" \ + # N: Overload variants must be defined in the same order as they are in "Parent" + def f(self, x: bool) -> bool: ... + @overload + def f(self, x: str) -> str: ... + @overload + def f(self, x: int) -> int: ... +class Child2(Parent): + @overload # E: Signature of "f" incompatible with supertype "Parent" \ + # N: Overload variants must be defined in the same order as they are in "Parent" + def f(self, x: str) -> str: ... + @overload + def f(self, x: bool) -> bool: ... + @overload + def f(self, x: int) -> int: ... +class Child3(Parent): + @overload # E: Signature of "f" incompatible with supertype "Parent" \ + # N: Overload variants must be defined in the same order as they are in "Parent" + def f(self, x: str) -> str: ... + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: bool) -> bool: ... + +[case testOverrideOverloadSwappedWithAdjustedVariants] +from foo import * +[file foo.pyi] +from typing import overload + +class A: pass +class B(A): pass +class C(B): pass + +class Parent: + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: B) -> B: ... +class Child1(Parent): + @overload # Fail + def f(self, x: A) -> B: ... + @overload + def f(self, x: int) -> int: ... +class Child2(Parent): + @overload # Fail + def f(self, x: B) -> C: ... + @overload + def f(self, x: int) -> int: ... +class Child3(Parent): + @overload # Fail + def f(self, x: B) -> A: ... + @overload + def f(self, x: int) -> int: ... +[out] +tmp/foo.pyi:13: error: Signature of "f" incompatible with supertype "Parent" +tmp/foo.pyi:13: note: Overload variants must be defined in the same order as they are in "Parent" +tmp/foo.pyi:18: error: Signature of "f" incompatible with supertype "Parent" +tmp/foo.pyi:18: note: Overload variants must be defined in the same order as they are in "Parent" +tmp/foo.pyi:23: error: Signature of "f" incompatible with supertype "Parent" +tmp/foo.pyi:23: note: Superclass: +tmp/foo.pyi:23: note: @overload +tmp/foo.pyi:23: note: def f(self, x: int) -> int +tmp/foo.pyi:23: note: @overload +tmp/foo.pyi:23: note: def f(self, x: B) -> B +tmp/foo.pyi:23: note: Subclass: +tmp/foo.pyi:23: note: @overload +tmp/foo.pyi:23: note: def f(self, x: B) -> A +tmp/foo.pyi:23: note: @overload +tmp/foo.pyi:23: note: def f(self, x: int) -> int + +[case testOverrideOverloadedMethodWithMoreGeneralArgumentTypes] +from foo import * +[file foo.pyi] +from typing import overload + +class IntSub(int): pass + +class StrSub(str): pass +class A: + @overload + def f(self, x: IntSub) -> int: return 0 + @overload + def f(self, x: StrSub) -> str: return '' +class B(A): + @overload + def f(self, x: int) -> int: return 0 + @overload + def f(self, x: str) -> str: return '' +[out] + +[case testOverrideOverloadedMethodWithMoreSpecificArgumentTypes] +from foo import * +[file foo.pyi] +from typing import overload + +class IntSub(int): pass + +class StrSub(str): pass +class A: + @overload + def f(self, x: int) -> int: return 0 + @overload + def f(self, x: str) -> str: return '' +class B(A): + @overload # Fail + def f(self, x: IntSub) -> int: return 0 + @overload + def f(self, x: str) -> str: return '' +class C(A): + @overload # Fail + def f(self, x: int) -> int: return 0 + @overload + def f(self, x: StrSub) -> str: return '' +class D(A): + @overload + def f(self, x: int) -> int: return 0 + @overload + def f(self, x: str) -> str: return '' +[out] +tmp/foo.pyi:12: error: Signature of "f" incompatible with supertype "A" +tmp/foo.pyi:12: note: Superclass: +tmp/foo.pyi:12: note: @overload +tmp/foo.pyi:12: note: def f(self, x: int) -> int +tmp/foo.pyi:12: note: @overload +tmp/foo.pyi:12: note: def f(self, x: str) -> str +tmp/foo.pyi:12: note: Subclass: +tmp/foo.pyi:12: note: @overload +tmp/foo.pyi:12: note: def f(self, x: IntSub) -> int +tmp/foo.pyi:12: note: @overload +tmp/foo.pyi:12: note: def f(self, x: str) -> str +tmp/foo.pyi:17: error: Signature of "f" incompatible with supertype "A" +tmp/foo.pyi:17: note: Superclass: +tmp/foo.pyi:17: note: @overload +tmp/foo.pyi:17: note: def f(self, x: int) -> int +tmp/foo.pyi:17: note: @overload +tmp/foo.pyi:17: note: def f(self, x: str) -> str +tmp/foo.pyi:17: note: Subclass: +tmp/foo.pyi:17: note: @overload +tmp/foo.pyi:17: note: def f(self, x: int) -> int +tmp/foo.pyi:17: note: @overload +tmp/foo.pyi:17: note: def f(self, x: StrSub) -> str + +[case testOverloadingAndDucktypeCompatibility] +from foo import * +[file foo.pyi] +from typing import overload, _promote + +class A: pass + +@_promote(A) +class B: pass + +@overload +def f(n: B) -> B: + return n +@overload +def f(n: A) -> A: + return n + +f(B()) + 'x' # E: Unsupported left operand type for + ("B") +f(A()) + 'x' # E: Unsupported left operand type for + ("A") +[typing fixtures/typing-medium.pyi] + +[case testOverloadingAndIntFloatSubtyping] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: float) -> None: pass +@overload +def f(x: str) -> None: pass +f(1.1) +f('') +f(1) +f(()) # E: No overload variant of "f" matches argument type "Tuple[()]" \ + # N: Possible overload variants: \ + # N: def f(x: float) -> None \ + # N: def f(x: str) -> None +[builtins fixtures/primitives.pyi] +[out] + +[case testOverloadingVariableInputs] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: int, y: int) -> None: pass +@overload +def f(x: int) -> None: pass +f(1) +f(1, 2) +z = (1, 2) +f(*z) +[builtins fixtures/primitives.pyi] +[out] + +[case testTypeInferenceSpecialCaseWithOverloading] +from foo import * +[file foo.pyi] +from typing import overload + +class A: + def __add__(self, x: A) -> A: pass +class B: + def __radd__(self, x: A) -> B: pass + +@overload +def f(x: A) -> A: pass +@overload +def f(x: B) -> B: pass + +f(A() + B())() # E: "B" not callable + +[case testKeywordArgOverload] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: int, y: str) -> int: pass +@overload +def f(x: str, y: int) -> str: pass +f(x=1, y='')() # E: "int" not callable +f(y=1, x='')() # E: "str" not callable + +[case testIgnoreOverloadVariantBasedOnKeywordArg] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: int) -> int: pass +@overload +def f(y: int) -> str: pass +f(x=1)() # E: "int" not callable +f(y=1)() # E: "str" not callable + +[case testOverloadWithTupleVarArg] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(x: int, y: str) -> int: pass +@overload +def f(*x: str) -> str: pass +f(*(1,))() # E: No overload variant of "f" matches argument type "Tuple[int]" \ + # N: Possible overload variants: \ + # N: def f(x: int, y: str) -> int \ + # N: def f(*x: str) -> str +f(*('',))() # E: "str" not callable +f(*(1, ''))() # E: "int" not callable +f(*(1, '', 1))() # E: No overload variant of "f" matches argument type "Tuple[int, str, int]" \ + # N: Possible overload variants: \ + # N: def f(x: int, y: str) -> int \ + # N: def f(*x: str) -> str +[builtins fixtures/tuple.pyi] + +[case testPreferExactSignatureMatchInOverload] +# flags: --no-strict-optional +from foo import * +[file foo.pyi] +from typing import overload, List +@overload +def f(x: int, y: List[int] = None) -> int: pass +@overload +def f(x: int, y: List[str] = None) -> int: pass +f(y=[1], x=0)() # E: "int" not callable +f(y=[''], x=0)() # E: "int" not callable +a = f(y=[['']], x=0) # E: List item 0 has incompatible type "List[str]"; expected "int" +reveal_type(a) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testOverloadWithDerivedFromAny] +from foo import * +[file foo.pyi] +from typing import Any, overload +Base = None # type: Any + +class C: + @overload + def __init__(self, a: str) -> None: pass + @overload + def __init__(self, a: int) -> None: pass + +class Derived(Base): + def to_dict(self) -> C: + return C(self) # fails without the fix for #1363 +C(Derived()) # fails without the hack +C(Base()) # Always ok + +[case testOverloadWithBoundedTypeVar] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar +T = TypeVar('T', bound=str) +@overload +def f(x: T) -> T: pass +@overload +def f(x: int) -> bool: pass +class mystr(str): pass + +f('x')() # E: "str" not callable +f(1)() # E: "bool" not callable +f(1.1) # E: No overload variant of "f" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def [T: str] f(x: T) -> T \ + # N: def f(x: int) -> bool +f(mystr())() # E: "mystr" not callable +[builtins fixtures/primitives.pyi] + +[case testOverloadedCallWithVariableTypes] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar, List +T = TypeVar('T', bound=str) +@overload +def f(x: T) -> T: pass +@overload +def f(x: List[T]) -> None: pass +class mystr(str): pass + +U = TypeVar('U', bound=mystr) +V = TypeVar('V') +def g(x: U, y: V) -> None: + f(x)() # E: "mystr" not callable + f(y) # E: No overload variant of "f" matches argument type "V" \ + # N: Possible overload variants: \ + # N: def [T: str] f(x: T) -> T \ + # N: def [T: str] f(x: List[T]) -> None + a = f([x]) + reveal_type(a) # N: Revealed type is "None" + f([y]) # E: Value of type variable "T" of "f" cannot be "V" + f([x, y]) # E: Value of type variable "T" of "f" cannot be "object" +[builtins fixtures/list.pyi] +[out] + +[case testOverloadOverlapWithTypeVars] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar, Sequence, List +T = TypeVar('T', bound=str) +@overload +def f(x: Sequence[T]) -> None: pass +@overload +def f(x: Sequence[int]) -> int: pass + +@overload +def g(x: Sequence[T]) -> None: pass +@overload +def g(x: Sequence[str]) -> int: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +@overload +def h(x: Sequence[str]) -> int: pass +@overload +def h(x: Sequence[T]) -> None: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +# Safety of this highly depends on the implementation, so we lean towards being silent. +@overload +def i(x: List[str]) -> int: pass +@overload +def i(x: List[T]) -> None: pass +[builtins fixtures/list.pyi] + +[case testOverloadOverlapWithTypeVarsWithValues] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar +AnyStr = TypeVar('AnyStr', bytes, str) + +@overload +def f(x: int) -> int: pass +@overload +def f(x: AnyStr) -> str: pass + +f(1)() # E: "int" not callable +f('1')() # E: "str" not callable +f(b'1')() # E: "str" not callable +f(1.0) # E: No overload variant of "f" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def f(x: int) -> int \ + # N: def [AnyStr: (bytes, str)] f(x: AnyStr) -> str + +@overload +def g(x: AnyStr, *a: AnyStr) -> None: pass +@overload +def g(x: int, *a: AnyStr) -> None: pass + +g('foo') +g('foo', 'bar') +g('foo', b'bar') # E: Value of type variable "AnyStr" of "g" cannot be "Sequence[object]" +g(1) +g(1, 'foo') +g(1, 'foo', b'bar') # E: Value of type variable "AnyStr" of "g" cannot be "Sequence[object]" +[builtins fixtures/primitives.pyi] + +[case testOverloadOverlapWithTypeVarsWithValuesOrdering] +from foo import * +[file foo.pyi] +from typing import overload, TypeVar +AnyStr = TypeVar('AnyStr', bytes, str) + +@overload +def f(x: AnyStr) -> AnyStr: pass +@overload +def f(x: str) -> str: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +@overload +def g(x: str) -> str: pass +@overload +def g(x: AnyStr) -> AnyStr: pass +[builtins fixtures/primitives.pyi] + +[case testOverloadsUsingAny] +from typing import overload, List, Any, Union + +@overload +def foo(x: List[int]) -> int: ... +@overload +def foo(x: List[str]) -> str: ... +def foo(x): pass + +a: List[int] +b: List[str] +c: List[Any] +d: Union[List[int], List[str]] +e: List[bool] +f: List[object] +g: List[Union[int, str]] + +reveal_type(foo(a)) +reveal_type(foo(b)) +reveal_type(foo(c)) +reveal_type(foo(d)) +foo(e) +foo(f) +foo(g) + +[builtins fixtures/list.pyi] +[out] +main:17: note: Revealed type is "builtins.int" +main:18: note: Revealed type is "builtins.str" +main:19: note: Revealed type is "Any" +main:20: note: Revealed type is "Union[builtins.int, builtins.str]" +main:21: error: Argument 1 to "foo" has incompatible type "List[bool]"; expected "List[int]" +main:21: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance +main:21: note: Consider using "Sequence" instead, which is covariant +main:22: error: Argument 1 to "foo" has incompatible type "List[object]"; expected "List[int]" +main:23: error: Argument 1 to "foo" has incompatible type "List[Union[int, str]]"; expected "List[int]" + +[case testOverloadAgainstEmptyCollections] +from typing import overload, List + +@overload +def f(x: List[int]) -> int: ... +@overload +def f(x: List[str]) -> str: ... +def f(x): pass + +reveal_type(f([])) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testOverloadAgainstEmptyCovariantCollections] +from typing import overload, TypeVar, Generic + +T = TypeVar('T', covariant=True) +class Wrapper(Generic[T]): pass + +class A: pass +class B(A): pass +class C: pass + +@overload +def f(x: Wrapper[A]) -> int: ... +@overload +def f(x: Wrapper[C]) -> str: ... +def f(x): pass + +reveal_type(f(Wrapper())) # N: Revealed type is "builtins.int" +reveal_type(f(Wrapper[C]())) # N: Revealed type is "builtins.str" +reveal_type(f(Wrapper[B]())) # N: Revealed type is "builtins.int" + +[case testOverlappingOverloadCounting] +from foo import * +[file foo.pyi] +from typing import overload +class A: pass +class B(A): pass +@overload +def f(x: int) -> None: pass +@overload +def f(x: B) -> str: pass # E: Overloaded function signatures 2 and 3 overlap with incompatible return types +@overload +def f(x: A) -> int: pass + +[case testOverloadWithTupleMatchingTypeVar] +from foo import * +[file foo.pyi] +from typing import TypeVar, Generic, Tuple, overload + +T = TypeVar('T') + +class A(Generic[T]): + @overload + def f(self, arg: T) -> None: + pass + @overload + def f(self, arg: T, default: int) -> None: + pass + +b = A() # type: A[Tuple[int, int]] +b.f((0, 0)) +b.f((0, '')) # E: Argument 1 to "f" of "A" has incompatible type "Tuple[int, str]"; expected "Tuple[int, int]" +[builtins fixtures/tuple.pyi] + +[case testSingleOverloadStub] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(a: int) -> None: pass +def f(a: int) -> None: pass +[out] +tmp/foo.pyi:2: error: Single overload definition, multiple required +tmp/foo.pyi:4: error: An implementation for an overloaded function is not allowed in a stub file + +[case testSingleOverload2] +from foo import * +[file foo.pyi] +from typing import overload +def f(a: int) -> None: pass +@overload +def f(a: str) -> None: pass +[out] +tmp/foo.pyi:3: error: Name "f" already defined on line 2 +tmp/foo.pyi:3: error: Single overload definition, multiple required + +[case testNonconsecutiveOverloads] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(a: int) -> None: pass +1 +@overload +def f(a: str) -> None: pass +[out] +tmp/foo.pyi:2: error: Single overload definition, multiple required +tmp/foo.pyi:5: error: Name "f" already defined on line 2 +tmp/foo.pyi:5: error: Single overload definition, multiple required + +[case testNonconsecutiveOverloadsMissingFirstOverload] +from foo import * +[file foo.pyi] +from typing import overload +def f(a: int) -> None: pass +1 +@overload +def f(a: str) -> None: pass +[out] +tmp/foo.pyi:4: error: Name "f" already defined on line 2 +tmp/foo.pyi:4: error: Single overload definition, multiple required + +[case testNonconsecutiveOverloadsMissingLaterOverload] +from foo import * +[file foo.pyi] +from typing import overload +@overload +def f(a: int) -> None: pass +1 +def f(a: str) -> None: pass +[out] +tmp/foo.pyi:2: error: Single overload definition, multiple required +tmp/foo.pyi:5: error: Name "f" already defined on line 2 + +[case testOverloadTuple] +from foo import * +[file foo.pyi] +from typing import overload, Tuple +@overload +def f(x: int, y: Tuple[str, ...]) -> None: pass +@overload +def f(x: int, y: str) -> None: pass +f(1, ('2', '3')) +f(1, (2, '3')) # E: Argument 2 to "f" has incompatible type "Tuple[int, str]"; expected "Tuple[str, ...]" +f(1, ('2',)) +f(1, '2') +f(1, (2, 3)) # E: Argument 2 to "f" has incompatible type "Tuple[int, int]"; expected "Tuple[str, ...]" +x = ('2', '3') # type: Tuple[str, ...] +f(1, x) +y = (2, 3) # type: Tuple[int, ...] +f(1, y) # E: Argument 2 to "f" has incompatible type "Tuple[int, ...]"; expected "Tuple[str, ...]" +[builtins fixtures/tuple.pyi] + +[case testCallableSpecificOverload] +from foo import * +[file foo.pyi] +from typing import overload, Callable +@overload +def f(a: Callable[[], int]) -> None: pass +@overload +def f(a: str) -> None: pass +f(0) # E: No overload variant of "f" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def f(a: Callable[[], int]) -> None \ + # N: def f(a: str) -> None + +[case testCustomRedefinitionDecorator] + +from typing import Any, Callable, Type + +class Chain(object): + def chain(self, function: Callable[[Any], int]) -> 'Chain': + return self + +class Test(object): + do_chain = Chain() + + @do_chain.chain # E: Name "do_chain" already defined on line 9 + def do_chain(self) -> int: + return 2 + + @do_chain.chain # E: Name "do_chain" already defined on line 11 + def do_chain(self) -> int: + return 3 + +t = Test() +reveal_type(t.do_chain) # N: Revealed type is "__main__.Chain" + +[case testOverloadWithOverlappingItemsAndAnyArgument1] +from typing import overload, Any + +@overload +def f(x: int) -> int: ... +@overload +def f(x: object) -> object: ... +def f(x): pass + +a: Any +reveal_type(f(a)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument2] +from typing import overload, Any + +@overload +def f(x: int) -> int: ... +@overload +def f(x: float) -> float: ... +def f(x): pass + +a: Any +reveal_type(f(a)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument3] +from typing import overload, Any + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x): pass + +a: Any +reveal_type(f(a)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument4] +from typing import overload, Any + +@overload +def f(x: int, y: int, z: str) -> int: ... +@overload +def f(x: object, y: int, z: str) -> object: ... +def f(x): pass + +a: Any +# Any causes ambiguity +reveal_type(f(a, 1, '')) # N: Revealed type is "Any" +# Any causes no ambiguity +reveal_type(f(1, a, a)) # N: Revealed type is "builtins.int" +reveal_type(f('', a, a)) # N: Revealed type is "builtins.object" +# Like above, but use keyword arguments. +reveal_type(f(y=1, z='', x=a)) # N: Revealed type is "Any" +reveal_type(f(y=a, z='', x=1)) # N: Revealed type is "builtins.int" +reveal_type(f(z='', x=1, y=a)) # N: Revealed type is "builtins.int" +reveal_type(f(z='', x=a, y=1)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument5] +from typing import overload, Any, Union + +class A: pass +class B(A): pass + +@overload +def f(x: B) -> B: ... +@overload +def f(x: Union[A, B]) -> A: ... +def f(x): pass + +# Note: overloads ignore promotions so we treat 'int' and 'float' as distinct types +@overload +def g(x: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def g(x: Union[int, float]) -> float: ... +def g(x): pass + +a: Any +reveal_type(f(a)) # N: Revealed type is "Any" +reveal_type(g(a)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument6] +from typing import overload, Any + +@overload +def f(x: int, y: int) -> int: ... +@overload +def f(x: float, y: int, z: str) -> float: ... +@overload +def f(x: object, y: int, z: str, a: None) -> str: ... +def f(x): pass + +a: Any +# Any causes ambiguity +reveal_type(f(*a)) # N: Revealed type is "Any" +reveal_type(f(a, *a)) # N: Revealed type is "Any" +reveal_type(f(1, *a)) # N: Revealed type is "Any" +reveal_type(f(1.1, *a)) # N: Revealed type is "Any" +reveal_type(f('', *a)) # N: Revealed type is "builtins.str" + +[case testOverloadWithOverlappingItemsAndAnyArgument7] +from typing import overload, Any + +@overload +def f(x: int, y: int, z: int) -> int: ... +@overload +def f(x: object, y: int, z: int) -> object: ... +def f(x): pass + +@overload +def g(x: int, y: int, z: int) -> int: ... +@overload +def g(x: object, y: int, z: str) -> object: ... +def g(x): pass + +a: Any +reveal_type(f(1, *a)) # N: Revealed type is "builtins.int" +reveal_type(g(1, *a)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument8] +from typing import overload, Any + +@overload +def f(x: int, y: int, z: int) -> str: ... +@overload +def f(x: object, y: int, z: int) -> str: ... +def f(x): pass + +a: Any +# The return type is not ambiguous so Any arguments cause no ambiguity. +reveal_type(f(a, 1, 1)) # N: Revealed type is "builtins.str" +reveal_type(f(1, *a)) # N: Revealed type is "builtins.str" + +[case testOverloadWithOverlappingItemsAndAnyArgument9] +from typing import overload, Any, List + +@overload +def f(x: List[int]) -> List[int]: ... +@overload +def f(x: List[Any]) -> List[Any]: ... +def f(x): pass + +a: Any +b: List[Any] +c: List[str] +d: List[int] +reveal_type(f(a)) # N: Revealed type is "builtins.list[Any]" +reveal_type(f(b)) # N: Revealed type is "builtins.list[Any]" +reveal_type(f(c)) # N: Revealed type is "builtins.list[Any]" +reveal_type(f(d)) # N: Revealed type is "builtins.list[builtins.int]" + +[builtins fixtures/list.pyi] + +[case testOverloadWithOverlappingItemsAndAnyArgument10] +from typing import overload, Any + +@overload +def f(*, x: int = 3, y: int = 3) -> int: ... +@overload +def f(**kwargs: str) -> str: ... +def f(*args, **kwargs): pass + +a: Any +i: int +reveal_type(f(x=a, y=i)) # N: Revealed type is "builtins.int" +reveal_type(f(y=a)) # N: Revealed type is "Any" +reveal_type(f(x=a, y=a)) # N: Revealed type is "Any" + +[builtins fixtures/dict.pyi] + +[case testOverloadWithOverlappingItemsAndAnyArgument11] +from typing import overload, Any, Dict + +@overload +def f(x: int = 3, **kwargs: int) -> int: ... +@overload +def f(**kwargs: str) -> str: ... +def f(*args, **kwargs): pass + +a: Dict[str, Any] +i: int +reveal_type(f(x=i, **a)) # N: Revealed type is "builtins.int" +reveal_type(f(**a)) # N: Revealed type is "Any" + +[builtins fixtures/dict.pyi] + +[case testOverloadWithOverlappingItemsAndAnyArgument12] +from typing import overload, Any + +@overload +def f(x: int) -> Any: ... +@overload +def f(x: str) -> str: ... +def f(x): pass + +a: Any +reveal_type(f(a)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument13] +from typing import Any, overload, TypeVar, Generic + +class slice: pass + +T = TypeVar('T') +class A(Generic[T]): + @overload + def f(self, x: int) -> T: ... + @overload + def f(self, x: slice) -> A[T]: ... + def f(self, x): ... + +i: Any +a: A[Any] +reveal_type(a.f(i)) # N: Revealed type is "Any" + +[case testOverloadWithOverlappingItemsAndAnyArgument14] +from typing import Any, overload, TypeVar, Generic + +T = TypeVar('T') + +class Wrapper(Generic[T]): pass +class slice: pass + +class A(Generic[T]): + @overload + def f(self, x: int) -> Wrapper[T]: ... + @overload + def f(self, x: slice) -> Wrapper[A[T]]: ... + def f(self, x): ... + +i: Any +a: A[Any] +reveal_type(a.f(i)) # N: Revealed type is "__main__.Wrapper[Any]" + +[case testOverloadWithOverlappingItemsAndAnyArgument15] +from typing import overload, Any, Union + +@overload +def f(x: int) -> str: ... +@overload +def f(x: str) -> str: ... +def f(x): pass + +@overload +def g(x: int) -> Union[str, int]: ... +@overload +def g(x: str) -> Union[int, str]: ... +def g(x): pass + +a: Any +reveal_type(f(a)) # N: Revealed type is "builtins.str" +reveal_type(g(a)) # N: Revealed type is "Union[builtins.str, builtins.int]" + +[case testOverloadWithOverlappingItemsAndAnyArgument16] +from typing import overload, Any, Union, Callable + +@overload +def f(x: int) -> Callable[[int, int], int]: ... +@overload +def f(x: str) -> Callable[[str], str]: ... +def f(x): pass + +a: Any +reveal_type(f(a)) # N: Revealed type is "def (*Any, **Any) -> Any" +reveal_type(f(a)(a)) # N: Revealed type is "Any" + +[case testOverloadOnOverloadWithType] +from typing import Any, Type, TypeVar, overload +from mod import MyInt +T = TypeVar('T') + +@overload +def make(cls: Type[T]) -> T: pass +@overload +def make() -> Any: pass + +def make(*args): + pass + +c = make(MyInt) +reveal_type(c) # N: Revealed type is "mod.MyInt" + +[file mod.pyi] +from typing import overload +class MyInt: + @overload + def __init__(self, x: str) -> None: pass + @overload + def __init__(self, x: str, y: int) -> None: pass +[builtins fixtures/tuple.pyi] +[out] + +[case testOverloadTupleInstance] +from typing import overload, Tuple, Any + +class A: ... +class A1(A): ... +class B: ... +class C: ... +class D: ... + +@overload +def f(x: A) -> A: ... +@overload +def f(x: Tuple[C]) -> B: ... +@overload +def f(x: Tuple[A1, int]) -> C: ... # E: Overloaded function signatures 3 and 5 overlap with incompatible return types +@overload +def f(x: Tuple[A, str]) -> D: ... +@overload +def f(x: Tuple[A, int]) -> D: ... +@overload +def f(x: Tuple[()]) -> D: ... +def f(x: Any) -> Any:... +[builtins fixtures/tuple.pyi] + +[case testOverloadTupleEllipsisNumargs] +from typing import overload, Tuple, Any + +class A: ... +class B: ... + +@overload +def r1(x: Tuple[()]) -> B: ... # E: Overloaded function signatures 1 and 4 overlap with incompatible return types +@overload +def r1(x: Tuple[A]) -> B: ... # E: Overloaded function signatures 2 and 4 overlap with incompatible return types +@overload +def r1(x: Tuple[A, A]) -> B: ... # E: Overloaded function signatures 3 and 4 overlap with incompatible return types +@overload +def r1(x: Tuple[A, ...]) -> A: ... +def r1(x: Any) -> Any: ... + +@overload +def r2(x: Tuple[A, ...]) -> A: ... +@overload +def r2(x: Tuple[A, A]) -> B: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +@overload +def r2(x: Tuple[A]) -> B: ... # E: Overloaded function signature 3 will never be matched: signature 1's parameter type(s) are the same or broader +@overload +def r2(x: Tuple[()]) -> B: ... # E: Overloaded function signature 4 will never be matched: signature 1's parameter type(s) are the same or broader +def r2(x: Any) -> Any: ... + +[builtins fixtures/tuple.pyi] + +[case testOverloadTupleEllipsisVariance] +from typing import overload, Tuple, Any + +class A: ... +class A1(A): ... +class B: ... +class C: ... +class D: ... + +@overload +def r(x: Tuple[A1, ...]) -> A: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def r(x: Tuple[A, ...]) -> B: ... +@overload +def r(x: Tuple[B, ...]) -> C: ... +def r(x: Any) -> Any:... + +@overload +def g(x: A) -> A: ... +@overload +def g(x: Tuple[A1, ...]) -> B: ... # E: Overloaded function signatures 2 and 3 overlap with incompatible return types +@overload +def g(x: Tuple[A, A]) -> C: ... +@overload +def g(x: Tuple[A, B]) -> D: ... +def g(x: Any) -> Any:... + +[builtins fixtures/tuple.pyi] + +[case testOverloadWithMethodOverrideAndImplementation] +from typing import overload, Union, Any + +class Parent: + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Any) -> Any: ... + +class Child1(Parent): + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Union[int, str]) -> Union[int, str]: ... + +class Child2(Parent): + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Union[int, str]) -> int: ... # E: Overloaded function implementation cannot produce return type of signature 2 + +class Child3(Parent): + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Any) -> Any: ... + +class Child4(Parent): + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Union[int, str]) -> Union[int, str]: + return b'' # E: Incompatible return value type (got "bytes", expected "Union[int, str]") + +[builtins fixtures/tuple.pyi] + +[case testOverloadWithIncompatibleMethodOverrideAndImplementation] +from typing import overload, Union, Any + +class StrSub: pass + +class ParentWithTypedImpl: + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Union[int, str]) -> Union[int, str]: ... + +class Child1(ParentWithTypedImpl): + @overload # Fail + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: StrSub) -> str: ... + def f(self, arg: Union[int, StrSub]) -> Union[int, str]: ... + +class Child2(ParentWithTypedImpl): + @overload # Fail + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: StrSub) -> str: ... + def f(self, arg: Any) -> Any: ... + +class ParentWithDynamicImpl: + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Any) -> Any: ... + +class Child3(ParentWithDynamicImpl): + @overload # Fail + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: StrSub) -> str: ... + def f(self, arg: Union[int, StrSub]) -> Union[int, str]: ... + +class Child4(ParentWithDynamicImpl): + @overload # Fail + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: StrSub) -> str: ... + def f(self, arg: Any) -> Any: ... + +[builtins fixtures/tuple.pyi] +[out] +main:13: error: Signature of "f" incompatible with supertype "ParentWithTypedImpl" +main:13: note: Superclass: +main:13: note: @overload +main:13: note: def f(self, arg: int) -> int +main:13: note: @overload +main:13: note: def f(self, arg: str) -> str +main:13: note: Subclass: +main:13: note: @overload +main:13: note: def f(self, arg: int) -> int +main:13: note: @overload +main:13: note: def f(self, arg: StrSub) -> str +main:20: error: Signature of "f" incompatible with supertype "ParentWithTypedImpl" +main:20: note: Superclass: +main:20: note: @overload +main:20: note: def f(self, arg: int) -> int +main:20: note: @overload +main:20: note: def f(self, arg: str) -> str +main:20: note: Subclass: +main:20: note: @overload +main:20: note: def f(self, arg: int) -> int +main:20: note: @overload +main:20: note: def f(self, arg: StrSub) -> str +main:34: error: Signature of "f" incompatible with supertype "ParentWithDynamicImpl" +main:34: note: Superclass: +main:34: note: @overload +main:34: note: def f(self, arg: int) -> int +main:34: note: @overload +main:34: note: def f(self, arg: str) -> str +main:34: note: Subclass: +main:34: note: @overload +main:34: note: def f(self, arg: int) -> int +main:34: note: @overload +main:34: note: def f(self, arg: StrSub) -> str +main:41: error: Signature of "f" incompatible with supertype "ParentWithDynamicImpl" +main:41: note: Superclass: +main:41: note: @overload +main:41: note: def f(self, arg: int) -> int +main:41: note: @overload +main:41: note: def f(self, arg: str) -> str +main:41: note: Subclass: +main:41: note: @overload +main:41: note: def f(self, arg: int) -> int +main:41: note: @overload +main:41: note: def f(self, arg: StrSub) -> str + +[case testOverloadAnyIsConsideredValidReturnSubtype] +from typing import Any, overload, Optional + +@overload +def foo(x: None) -> Any: ... +@overload +def foo(x: Optional[str]) -> str: ... +def foo(x): pass + +@overload +def bar(x: None) -> object: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def bar(x: Optional[str]) -> str: ... +def bar(x): pass + +[case testOverloadWithNonPositionalArgs] +from typing import overload + +class A: ... +class B: ... +class C: ... + +@overload +def foo(*, p1: A, p2: B = B()) -> A: ... +@overload +def foo(*, p2: B = B()) -> B: ... +def foo(p1, p2=None): ... + +reveal_type(foo()) # N: Revealed type is "__main__.B" +reveal_type(foo(p2=B())) # N: Revealed type is "__main__.B" +reveal_type(foo(p1=A())) # N: Revealed type is "__main__.A" + +[case testOverloadWithNonPositionalArgsIgnoresOrder] +from typing import overload + +class A: ... +class B(A): ... +class X: ... +class Y: ... + +@overload +def f(*, p1: X, p2: A) -> X: ... +@overload +def f(*, p2: B, p1: X) -> Y: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +def f(*, p1, p2): ... + +@overload +def g(*, p1: X, p2: B) -> X: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def g(*, p2: A, p1: X) -> Y: ... +def g(*, p1, p2): ... + +[case testOverloadWithVariableArgsAreOverlapping] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +# Safety of this highly depends on the implementation, so we lean towards being silent. +@overload +def foo1(*x: int) -> int: ... +@overload +def foo1(x: int, y: int, z: int) -> str: ... + +@overload +def foo2(*x: int) -> int: ... +@overload +def foo2(x: int, y: str, z: int) -> str: ... + +# Note: this is technically unsafe, but we don't report this for now. +@overload +def bar1(x: int, y: int, z: int) -> str: ... +@overload +def bar1(*x: int) -> int: ... + +@overload +def bar2(x: int, y: str, z: int) -> str: ... +@overload +def bar2(*x: int) -> int: ... +[builtins fixtures/tuple.pyi] + +[case testOverloadDetectsPossibleMatchesWithGenerics] +# flags: --strict-optional +from typing import overload, TypeVar, Generic, Optional, List + +T = TypeVar('T') +# The examples below are unsafe, but it is a quite common pattern +# so we ignore the possibility of type variables taking value `None` +# for the purpose of overload overlap checks. + +@overload +def foo(x: None, y: None) -> str: ... +@overload +def foo(x: T, y: T) -> int: ... +def foo(x): ... + +oi: Optional[int] +reveal_type(foo(None, None)) # N: Revealed type is "builtins.str" +reveal_type(foo(None, 42)) # N: Revealed type is "builtins.int" +reveal_type(foo(42, 42)) # N: Revealed type is "builtins.int" +reveal_type(foo(oi, None)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(foo(oi, 42)) # N: Revealed type is "builtins.int" +reveal_type(foo(oi, oi)) # N: Revealed type is "Union[builtins.int, builtins.str]" + +@overload +def foo_list(x: None) -> None: ... +@overload +def foo_list(x: T) -> List[T]: ... +def foo_list(x): ... + +reveal_type(foo_list(oi)) # N: Revealed type is "Union[builtins.list[builtins.int], None]" + +# What if 'T' is 'object'? +@overload +def bar(x: None, y: int) -> str: ... +@overload +def bar(x: T, y: T) -> int: ... +def bar(x, y): ... + +class Wrapper(Generic[T]): + @overload + def foo(self, x: None, y: None) -> str: ... + @overload + def foo(self, x: T, y: None) -> int: ... + def foo(self, x): ... + + @overload + def bar(self, x: None, y: int) -> str: ... + @overload + def bar(self, x: T, y: T) -> int: ... + def bar(self, x, y): ... + +@overload +def baz(x: str, y: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def baz(x: T, y: T) -> int: ... +def baz(x): ... +[builtins fixtures/tuple.pyi] + +[case testOverloadFlagsPossibleMatches] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(x: str) -> str: ... +@overload +def foo1(x: str, y: str = ...) -> int: ... + +@overload +def foo2(x: str, y: str = ...) -> int: ... +@overload +def foo2(x: str) -> str: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +@overload +def foo3(x: str) -> str: ... +@overload +def foo3(x: str, y: str) -> int: ... + +[case testOverloadPossibleOverlapWithArgsAndKwargs] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(*args: int) -> int: ... +@overload +def foo1(**kwargs: int) -> str: ... + +@overload +def foo2(**kwargs: int) -> str: ... +@overload +def foo2(*args: int) -> int: ... +[builtins fixtures/dict.pyi] + +[case testOverloadPossibleOverlapWithVarargs] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(*args: int) -> int: ... +@overload +def foo1(*args2: int) -> str: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +@overload +def foo2(*args: int) -> str: ... +@overload +def foo2(*args2: str) -> int: ... + +@overload +def foo3(*args: int) -> str: ... +@overload +def foo3(*args: str) -> int: ... +[builtins fixtures/tuple.pyi] + +[case testOverloadPossibleOverlapWithVarargs2] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(*args: str) -> int: ... +@overload +def foo1(x: int, *args2: int) -> str: ... + +@overload +def foo2(x: int, *args: int) -> str: ... +@overload +def foo2(*args2: str) -> int: ... + +# The two examples are unsafe, but this is hard to detect. +@overload +def foo3(*args: int) -> int: ... +@overload +def foo3(x: int, *args2: int) -> str: ... + +@overload +def foo4(x: int, *args: int) -> str: ... +@overload +def foo4(*args2: int) -> int: ... +[builtins fixtures/tuple.pyi] + +[case testOverloadPossibleOverlapWithVarargs3] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +class Other: ... + +@overload +def foo1(x: Other, *args: int) -> str: ... +@overload +def foo1(*args: str) -> int: ... + +@overload +def foo2(*args: int) -> str: ... +@overload +def foo2(x: Other, *args: str) -> int: ... + +@overload +def foo3(x: Other = ..., *args: int) -> str: ... +@overload +def foo3(*args: str) -> int: ... + +@overload +def foo4(*args: int) -> str: ... +@overload +def foo4(x: Other = ..., *args: str) -> int: ... +[builtins fixtures/tuple.pyi] + +[case testOverloadPossibleOverlapWithVarargs4] +from typing import overload + +@overload +def foo1(x: int = 0, y: int = 0) -> int: ... +@overload +def foo1(*xs: int) -> str: ... +def foo1(*args): pass + +@overload +def foo2(*xs: int) -> str: ... +@overload +def foo2(x: int = 0, y: int = 0) -> int: ... +def foo2(*args): pass +[builtins fixtures/tuple.pyi] + +[case testOverloadPossibleOverlapWithKwargs] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(**kwargs: int) -> int: ... +@overload +def foo1(**kwargs2: int) -> str: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +@overload +def foo2(**kwargs: int) -> str: ... +@overload +def foo2(**kwargs2: str) -> int: ... + +@overload +def foo(**kwargs: int) -> str: ... +@overload +def foo(**kwargs: str) -> int: ... +[builtins fixtures/dict.pyi] + +[case testOverloadPossibleOverlapMixingNamedArgsWithVarargs] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(x: str, *, y: str) -> str: ... +@overload +def foo1(*x: str) -> int: ... + +@overload +def foo2(*x: str) -> int: ... +@overload +def foo2(x: str, *, y: str) -> str: ... +[builtins fixtures/tuple.pyi] + +[case testOverloadPossibleOverlapMixingOptionalArgsWithVarargs] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(x: str, y: str = ..., z: str = ...) -> str: ... +@overload +def foo1(*x: str) -> int: ... + +@overload +def foo2(*x: str) -> int: ... +@overload +def foo2(x: str, y: str = ..., z: str = ...) -> str: ... + +@overload +def foo3(x: int, y: str = ..., z: str = ...) -> str: ... +@overload +def foo3(*x: str) -> int: ... +[builtins fixtures/tuple.pyi] + +[case testOverloadPossibleOverlapMixingOptionalArgsWithVarargs2] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(x: str, y: str = ..., z: int = ...) -> str: ... +@overload +def foo1(*x: str) -> int: ... + +@overload +def foo2(x: str, y: str = ..., z: int = ...) -> str: ... +@overload +def foo2(*x: str) -> int: ... +[builtins fixtures/tuple.pyi] + +[case testOverloadPossibleOverlapMixingNamedArgsWithKwargs] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(*, x: str, y: str, z: str) -> str: ... +@overload +def foo1(**x: str) -> int: ... + +@overload +def foo2(**x: str) -> int: ... +@overload +def foo2(*, x: str, y: str, z: str) -> str: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader + +@overload +def foo3(*, x: int, y: str, z: str) -> str: ... +@overload +def foo3(*x: str) -> int: ... + +[builtins fixtures/dict.pyi] + +[case testOverloadPossibleOverlapMixingNamedArgsWithKwargs2] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(*, x: str, y: str, z: int) -> str: ... +@overload +def foo1(**x: str) -> int: ... + +@overload +def foo2(**x: str) -> int: ... +@overload +def foo2(*, x: str, y: str, z: int) -> str: ... + +@overload +def foo3(*, x: str, y: str, z: int = ...) -> str: ... +@overload +def foo3(**x: str) -> int: ... + +@overload +def foo4(**x: str) -> int: ... +@overload +def foo4(*, x: str, y: str, z: int = ...) -> str: ... +[builtins fixtures/dict.pyi] + +[case testOverloadPossibleOverlapMixingNamedArgsWithKwargs3] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(x: str, *, y: str, z: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error +@overload +def foo1(**x: str) -> int: ... + +@overload +def foo2(**x: str) -> int: ... +@overload +def foo2(x: str, *, y: str, z: str) -> str: ... + +[builtins fixtures/dict.pyi] + +[case testOverloadVarargInputAndVarargDefinition] +from typing import overload, List + +class A: ... +class B: ... +class C: ... + +@overload +def foo(x: int) -> A: ... +@overload +def foo(x: int, y: int) -> B: ... +@overload +def foo(x: int, y: int, z: int, *args: int) -> C: ... +def foo(*args): pass + +reveal_type(foo(1)) # N: Revealed type is "__main__.A" +reveal_type(foo(1, 2)) # N: Revealed type is "__main__.B" +reveal_type(foo(1, 2, 3)) # N: Revealed type is "__main__.C" + +reveal_type(foo(*[1])) # N: Revealed type is "__main__.C" +reveal_type(foo(*[1, 2])) # N: Revealed type is "__main__.C" +reveal_type(foo(*[1, 2, 3])) # N: Revealed type is "__main__.C" + +x: List[int] +reveal_type(foo(*x)) # N: Revealed type is "__main__.C" + +y: List[str] +foo(*y) # E: No overload variant of "foo" matches argument type "List[str]" \ + # N: Possible overload variants: \ + # N: def foo(x: int) -> A \ + # N: def foo(x: int, y: int) -> B \ + # N: def foo(x: int, y: int, z: int, *args: int) -> C +[builtins fixtures/list.pyi] + +[case testOverloadMultipleVarargDefinition] +from typing import overload, List, Any + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def foo(x: int) -> A: ... +@overload +def foo(x: int, y: int) -> B: ... +@overload +def foo(x: int, y: int, z: int, *args: int) -> C: ... +@overload +def foo(*x: str) -> D: ... +def foo(*args): pass + +reveal_type(foo(*[1, 2])) # N: Revealed type is "__main__.C" +reveal_type(foo(*["a", "b"])) # N: Revealed type is "__main__.D" + +x: List[Any] +reveal_type(foo(*x)) # N: Revealed type is "Any" +[builtins fixtures/list.pyi] + +[case testOverloadMultipleVarargDefinitionComplex] +from typing import TypeVar, overload, Any, Callable + +T1 = TypeVar('T1') +T2 = TypeVar('T2') +T3 = TypeVar('T3') + +@overload +def chain_call(input_value: T1, + f1: Callable[[T1], T2]) -> T2: ... +@overload +def chain_call(input_value: T1, + f1: Callable[[T1], T2], + f2: Callable[[T2], T3]) -> T3: ... +@overload +def chain_call(input_value: T1, + *f_rest: Callable[[T1], T1]) -> T1: ... +@overload +def chain_call(input_value: T1, + f1: Callable[[T1], T2], + f2: Callable[[T2], T3], + f3: Callable[[T3], Any], + *f_rest: Callable[[Any], Any]) -> Any: ... +def chain_call(input_value, *f_rest): + for function in f_rest: + input_value = function(input_value) + return input_value + + +class A: ... +class B: ... +class C: ... +class D: ... + +def f(x: A) -> A: ... +def f1(x: A) -> B: ... +def f2(x: B) -> C: ... +def f3(x: C) -> D: ... + +reveal_type(chain_call(A(), f1, f2)) # N: Revealed type is "__main__.C" +reveal_type(chain_call(A(), f1, f2, f3)) # N: Revealed type is "Any" +reveal_type(chain_call(A(), f, f, f, f)) # N: Revealed type is "__main__.A" +[builtins fixtures/list.pyi] + +[case testOverloadVarargsSelection] +from typing import overload, Tuple +@overload +def f(x: int) -> Tuple[int]: ... +@overload +def f(x: int, y: int) -> Tuple[int, int]: ... +@overload +def f(*xs: int) -> Tuple[int, ...]: ... +def f(*args): pass + +i: int +reveal_type(f(i)) # N: Revealed type is "Tuple[builtins.int]" +reveal_type(f(i, i)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +reveal_type(f(i, i, i)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +reveal_type(f(*[])) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(f(*[i])) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(f(*[i, i])) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(f(*[i, i, i])) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/list.pyi] + +[case testOverloadVarargsSelectionWithTuples] +from typing import overload, Tuple +@overload +def f(x: int) -> Tuple[int]: ... +@overload +def f(x: int, y: int) -> Tuple[int, int]: ... +@overload +def f(*xs: int) -> Tuple[int, ...]: ... +def f(*args): pass + +i: int +reveal_type(f(*())) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(f(*(i,))) # N: Revealed type is "Tuple[builtins.int]" +reveal_type(f(*(i, i))) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +reveal_type(f(*(i, i, i))) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testOverloadVarargsSelectionWithNamedTuples] +from typing import overload, Tuple, NamedTuple +@overload +def f(x: int, y: int) -> Tuple[int, int]: ... +@overload +def f(*xs: int) -> Tuple[int, ...]: ... +def f(*args): pass + +A = NamedTuple('A', [('x', int), ('y', int)]) +B = NamedTuple('B', [('a', int), ('b', int)]) +C = NamedTuple('C', [('a', int), ('b', int), ('c', int)]) + +a: A +b: B +c: C +reveal_type(f(*a)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +reveal_type(f(*b)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +reveal_type(f(*c)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testOverloadKwargsSelectionWithDict] +from typing import overload, Tuple, Dict +@overload +def f(*, x: int) -> Tuple[int]: ... +@overload +def f(*, x: int, y: int) -> Tuple[int, int]: ... +@overload +def f(**xs: int) -> Tuple[int, ...]: ... +def f(**kwargs): pass + +empty: Dict[str, int] +reveal_type(f(**empty)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(f(**{'x': 4})) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(f(**{'x': 4, 'y': 4})) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(f(**{'a': 4, 'b': 4, 'c': 4})) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/dict.pyi] + +[case testOverloadKwargsSelectionWithTypedDict] +from typing import overload, Tuple +from typing_extensions import TypedDict +@overload +def f(*, x: int) -> Tuple[int]: ... +@overload +def f(*, x: int, y: int) -> Tuple[int, int]: ... +@overload +def f(**xs: int) -> Tuple[int, ...]: ... +def f(**args): pass + +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'x': int, 'y': int}) +C = TypedDict('C', {'x': int, 'y': int, 'z': int}) + +a: A +b: B +c: C + +reveal_type(f(**a)) # N: Revealed type is "Tuple[builtins.int]" +reveal_type(f(**b)) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +reveal_type(f(**c)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/dict.pyi] + +[case testOverloadVarargsAndKwargsSelection] +from typing import overload, Any, Tuple, Dict + +class A: pass +class B(A): pass + +@overload +def f(x: int, y: int) -> B: pass +@overload +def f(x: int, y: int, **kwargs: int) -> A: pass +@overload +def f(*args: int, **kwargs: int) -> Any: pass +def f(*args, **kwargs): pass + +a: Tuple[int, int] +b: Tuple[int, ...] +c: Dict[str, int] + +reveal_type(f(*a, **c)) # N: Revealed type is "__main__.A" +reveal_type(f(*b, **c)) # N: Revealed type is "__main__.A" +reveal_type(f(*a)) # N: Revealed type is "__main__.B" +reveal_type(f(*b)) # N: Revealed type is "Any" + +# TODO: Should this be 'Any' instead? +# The first matching overload with a kwarg is f(int, int, **int) -> A, +# but f(*int, **int) -> Any feels like a better fit. +reveal_type(f(**c)) # N: Revealed type is "__main__.A" +[builtins fixtures/args.pyi] + +[case testOverloadWithPartiallyOverlappingUnions] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def f(x: Union[A, B]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: Union[B, C]) -> str: ... +def f(x): ... + +@overload +def g(x: Union[A, B]) -> int: ... +@overload +def g(x: Union[B, C]) -> int: ... +def g(x): ... + +@overload +def h(x: Union[A, B]) -> int: ... +@overload +def h(x: Union[C, D]) -> str: ... +def h(x): ... + +@overload +def i(x: Union[A, B]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def i(x: Union[A, B, C]) -> str: ... +def i(x): ... + +[case testOverloadWithPartiallyOverlappingUnionsNested] +from typing import overload, Union, List + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def f(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: List[Union[B, C]]) -> str: ... +def f(x): ... + +@overload +def g(x: List[Union[A, B]]) -> int: ... +@overload +def g(x: List[Union[B, C]]) -> int: ... +def g(x): ... + +@overload +def h(x: List[Union[A, B]]) -> int: ... +@overload +def h(x: List[Union[C, D]]) -> str: ... +def h(x): ... + +@overload +def i(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error +@overload +def i(x: List[Union[A, B, C]]) -> str: ... +def i(x): ... + +[builtins fixtures/list.pyi] + +[case testOverloadPartialOverlapWithUnrestrictedTypeVar] +from typing import TypeVar, overload + +T = TypeVar('T') + +# Note: this is unsafe, but it is hard to detect. +@overload +def f(x: int) -> str: ... +@overload +def f(x: T) -> T: ... +def f(x): ... + +@overload +def g(x: int) -> int: ... +@overload +def g(x: T) -> T: ... +def g(x): ... + +[case testOverloadPartialOverlapWithUnrestrictedTypeVarNested] +from typing import TypeVar, overload, List + +T = TypeVar('T') + +# Note: first two examples are unsafe, but it is hard to detect. +@overload +def f1(x: List[int]) -> str: ... +@overload +def f1(x: List[T]) -> T: ... +def f1(x): ... + +@overload +def f2(x: List[int]) -> List[str]: ... +@overload +def f2(x: List[T]) -> List[T]: ... +def f2(x): ... + +@overload +def g1(x: List[int]) -> int: ... +@overload +def g1(x: List[T]) -> T: ... +def g1(x): ... + +@overload +def g2(x: List[int]) -> List[int]: ... +@overload +def g2(x: List[T]) -> List[T]: ... +def g2(x): ... + +[builtins fixtures/list.pyi] + +[case testOverloadPartialOverlapWithUnrestrictedTypeVarInClass] +from typing import TypeVar, overload, Generic + +T = TypeVar('T') + +class Wrapper(Generic[T]): + # Similar to above: this is unsafe, but it is hard to detect. + @overload + def f(self, x: int) -> str: ... + @overload + def f(self, x: T) -> T: ... + def f(self, x): ... + + @overload + def g(self, x: int) -> int: ... + @overload + def g(self, x: T) -> T: ... + def g(self, x): ... + +[case testOverloadPartialOverlapWithUnrestrictedTypeVarInClassNested] +from typing import TypeVar, overload, Generic, List + +T = TypeVar('T') + +class Wrapper(Generic[T]): + # Similar to above: first two examples are unsafe, but it is hard to detect. + @overload + def f1(self, x: List[int]) -> str: ... + @overload + def f1(self, x: List[T]) -> T: ... + def f1(self, x): ... + + @overload + def f2(self, x: List[int]) -> List[str]: ... + @overload + def f2(self, x: List[T]) -> List[T]: ... + def f2(self, x): ... + + @overload + def g1(self, x: List[int]) -> int: ... + @overload + def g1(self, x: List[T]) -> T: ... + def g1(self, x): ... + + @overload + def g2(self, x: List[int]) -> List[int]: ... + @overload + def g2(self, x: List[T]) -> List[T]: ... + def g2(self, x): ... + +[builtins fixtures/list.pyi] + +[case testOverloadTypedDictDifferentRequiredKeysMeansDictsAreDisjoint] +from typing import overload +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': int}) +B = TypedDict('B', {'x': int, 'y': str}) + +@overload +def f(x: A) -> int: ... +@overload +def f(x: B) -> str: ... +def f(x): pass +[builtins fixtures/dict.pyi] + +[case testOverloadedTypedDictPartiallyOverlappingRequiredKeys] +from typing import overload, Union +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': Union[int, str]}) +B = TypedDict('B', {'x': int, 'y': Union[str, float]}) + +@overload +def f(x: A) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: B) -> str: ... +def f(x): pass + +@overload +def g(x: A) -> int: ... +@overload +def g(x: B) -> object: ... +def g(x): pass +[builtins fixtures/dict.pyi] + +[case testOverloadedTypedDictFullyNonTotalDictsAreAlwaysPartiallyOverlapping] +from typing import overload +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}, total=False) +B = TypedDict('B', {'a': bool}, total=False) +C = TypedDict('C', {'x': str, 'y': int}, total=False) + +@overload +def f(x: A) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: B) -> str: ... +def f(x): pass + +@overload +def g(x: A) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def g(x: C) -> str: ... +def g(x): pass +[builtins fixtures/dict.pyi] + +[case testOverloadedTotalAndNonTotalTypedDictsCanPartiallyOverlap] +from typing import overload, Union +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) +B = TypedDict('B', {'x': Union[int, str], 'y': str, 'z': int}, total=False) + +@overload +def f1(x: A) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f1(x: B) -> str: ... +def f1(x): pass + +@overload +def f2(x: B) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f2(x: A) -> str: ... +def f2(x): pass + +[builtins fixtures/dict.pyi] + +[case testOverloadedTypedDictsWithSomeOptionalKeysArePartiallyOverlapping] +from typing import overload, Union +from mypy_extensions import TypedDict + +class A(TypedDict): + x: int + y: int + +class B(TypedDict, total=False): + z: str + +class C(TypedDict, total=False): + z: int + +@overload +def f(x: B) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: C) -> str: ... +def f(x): pass + +[builtins fixtures/dict.pyi] + +[case testOverloadedPartiallyOverlappingInheritedTypes1] +from typing import overload, List, Union, TypeVar, Generic + +class A: pass +class B: pass +class C: pass + +T = TypeVar('T') + +class ListSubclass(List[T]): pass +class Unrelated(Generic[T]): pass + +@overload +def f(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: ListSubclass[Union[B, C]]) -> str: ... +def f(x): pass + +@overload +def g(x: List[Union[A, B]]) -> int: ... +@overload +def g(x: Unrelated[Union[B, C]]) -> str: ... +def g(x): pass + +[builtins fixtures/list.pyi] + +[case testOverloadedPartiallyOverlappingInheritedTypes2] +from typing import overload, List, Union + +class A: pass +class B: pass +class C: pass + +class ListSubclass(List[Union[B, C]]): pass + +@overload +def f(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: ListSubclass) -> str: ... +def f(x): pass + +[builtins fixtures/list.pyi] + +[case testOverloadedPartiallyOverlappingInheritedTypes3] +from typing import overload, Union, Dict, TypeVar + +class A: pass +class B: pass +class C: pass + +S = TypeVar('S') + +class DictSubclass(Dict[str, S]): pass + +@overload +def f(x: Dict[str, Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: DictSubclass[Union[B, C]]) -> str: ... +def f(x): pass + +[builtins fixtures/dict.pyi] + +[case testOverloadedPartiallyOverlappingTypeVarsAndUnion] +from typing import overload, TypeVar, Union + +class A: pass +class B: pass +class C: pass + +S = TypeVar('S', A, B) + +@overload +def f(x: S) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error +@overload +def f(x: Union[B, C]) -> str: ... +def f(x): pass + +@overload +def g(x: Union[B, C]) -> int: ... +@overload +def g(x: S) -> str: ... +def g(x): pass + +[case testOverloadPartiallyOverlappingTypeVarsIdentical] +from typing import overload, TypeVar, Union + +T = TypeVar('T') + +class A: pass +class B: pass +class C: pass + +@overload +def f(x: T, y: T, z: Union[A, B]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: T, y: T, z: Union[B, C]) -> str: ... +def f(x, y, z): pass + +[case testOverloadedPartiallyOverlappingCallables] +from typing import overload, Union, Callable + +class A: pass +class B: pass +class C: pass + +@overload +def f(x: Callable[[Union[A, B]], int]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: Callable[[Union[B, C]], int]) -> str: ... +def f(x): pass + +[case testOverloadNotConfusedForProperty] +from typing import overload + +class PropertyClass: + @property + def foo(self) -> str: return "..." + @foo.setter + def foo(self, value: str) -> None: pass + @foo.deleter + def foo(self) -> None: pass + +class OverloadClass: + @overload + def foo(self) -> str: pass + @overload + def foo(self, value: str) -> None: pass + @overload + def foo(self) -> None: pass # E: Overloaded function signature 3 will never be matched: signature 1's parameter type(s) are the same or broader + def foo(self, *args): pass + +[builtins fixtures/property.pyi] + +[case testOverloadInferUnionReturnBasic] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def f1(x: A) -> B: ... +@overload +def f1(x: C) -> D: ... +def f1(x): ... + +arg1: Union[A, C] +reveal_type(f1(arg1)) # N: Revealed type is "Union[__main__.B, __main__.D]" + +arg2: Union[A, B] +f1(arg2) # E: Argument 1 to "f1" has incompatible type "Union[A, B]"; expected "A" + +@overload +def f2(x: A) -> B: ... +@overload +def f2(x: C) -> B: ... +def f2(x): ... + +reveal_type(f2(arg1)) # N: Revealed type is "__main__.B" + +[case testOverloadInferUnionReturnMultipleArguments] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def f1(x: A, y: C) -> B: ... +@overload +def f1(x: C, y: A) -> D: ... +def f1(x, y): ... + +arg1: Union[A, C] +reveal_type(f1(arg1, arg1)) + +@overload +def f2(x: A, y: C) -> B: ... +@overload +def f2(x: C, y: C) -> D: ... +def f2(x, y): ... + +reveal_type(f2(arg1, arg1)) +reveal_type(f2(arg1, C())) + +[out] +main:15: note: Revealed type is "__main__.B" +main:15: error: Argument 1 to "f1" has incompatible type "Union[A, C]"; expected "A" +main:15: error: Argument 2 to "f1" has incompatible type "Union[A, C]"; expected "C" +main:23: note: Revealed type is "__main__.B" +main:23: error: Argument 1 to "f2" has incompatible type "Union[A, C]"; expected "A" +main:23: error: Argument 2 to "f2" has incompatible type "Union[A, C]"; expected "C" +main:24: note: Revealed type is "Union[__main__.B, __main__.D]" + +[case testOverloadInferUnionRespectsVariance] +from typing import overload, TypeVar, Union, Generic + +class A: pass +class B(A): pass +class C(B): pass + +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class WrapperCo(Generic[T_co]): pass +class WrapperContra(Generic[T_contra]): pass + +@overload +def foo(x: WrapperCo[B]) -> int: ... +@overload +def foo(x: WrapperContra[B]) -> str: ... +def foo(x): pass + +compat: Union[WrapperCo[C], WrapperContra[A]] +reveal_type(foo(compat)) # N: Revealed type is "Union[builtins.int, builtins.str]" + +not_compat: Union[WrapperCo[A], WrapperContra[C]] +foo(not_compat) # E: Argument 1 to "foo" has incompatible type "Union[WrapperCo[A], WrapperContra[C]]"; expected "WrapperCo[B]" + +[case testOverloadInferUnionIfParameterNamesAreDifferent] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... + +@overload +def f(x: A) -> B: ... +@overload +def f(y: B) -> C: ... +def f(x): ... + +x: Union[A, B] +reveal_type(f(A())) # N: Revealed type is "__main__.B" +reveal_type(f(B())) # N: Revealed type is "__main__.C" +reveal_type(f(x)) # N: Revealed type is "Union[__main__.B, __main__.C]" + +[case testOverloadInferUnionReturnFunctionsWithKwargs] +from typing import overload, Union, Optional + +class A: ... +class B: ... +class C: ... +class D(B, C): ... + +@overload +def f(x: A) -> D: ... +@overload +def f(x: A, y: Optional[B] = None) -> C: ... +@overload +def f(x: A, z: Optional[C] = None) -> B: ... +def f(x, y=None, z=None): ... + +reveal_type(f(A(), B())) # N: Revealed type is "__main__.C" +reveal_type(f(A(), C())) # N: Revealed type is "__main__.B" + +arg: Union[B, C] +reveal_type(f(A(), arg)) # N: Revealed type is "Union[__main__.C, __main__.B]" +reveal_type(f(A())) # N: Revealed type is "__main__.D" + +[builtins fixtures/tuple.pyi] + +[case testOverloadInferUnionWithDifferingLengths] +from typing import overload, Union + +class Parent: ... +class Child(Parent): ... + +class A: ... +class B: ... + +@overload +def f(x: A) -> Child: ... +@overload +def f(x: B, y: B = B()) -> Parent: ... +def f(*args): ... + +x: Union[A, B] +reveal_type(f(x)) # N: Revealed type is "__main__.Parent" +f(x, B()) # E: Argument 1 to "f" has incompatible type "Union[A, B]"; expected "B" +[builtins fixtures/tuple.pyi] + +[case testOverloadInferUnionWithMixOfPositionalAndOptionalArgs] +from typing import overload, Union, Optional + +class A: ... +class B: ... + +@overload +def f(x: A) -> int: ... +@overload +def f(x: Optional[B] = None) -> str: ... +def f(*args): ... + +x: Union[A, B] +y: Optional[A] +z: Union[A, Optional[B]] +reveal_type(f(x)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(f(y)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(f(z)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(f()) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testOverloadingInferUnionReturnWithTypevarWithValueRestriction] +from typing import overload, Union, TypeVar, Generic + +class A: pass +class B: pass +class C: pass + +T = TypeVar('T', B, C) + +class Wrapper(Generic[T]): + @overload + def f(self, x: T) -> B: ... + + @overload + def f(self, x: A) -> C: ... + + def f(self, x): ... + +obj: Wrapper[B] = Wrapper() +x: Union[A, B] + +reveal_type(obj.f(A())) # N: Revealed type is "__main__.C" +reveal_type(obj.f(B())) # N: Revealed type is "__main__.B" +reveal_type(obj.f(x)) # N: Revealed type is "Union[__main__.C, __main__.B]" + +[case testOverloadingInferUnionReturnWithFunctionTypevarReturn] +from typing import overload, Union, TypeVar, Generic + +T = TypeVar('T') + +class W1(Generic[T]): pass +class W2(Generic[T]): pass +class A: pass +class B: pass + +@overload +def foo(x: W1[T]) -> T: ... +@overload +def foo(x: W2[T]) -> T: ... +def foo(x): ... + +def bar(x: Union[W1[T], W2[T]]) -> T: ... + +def wrapper() -> None: + obj1: Union[W1[A], W2[A]] + + a1: A = foo(obj1) + a2 = foo(obj1) + reveal_type(a1) # N: Revealed type is "__main__.A" + reveal_type(a2) # N: Revealed type is "__main__.A" + + obj2: Union[W1[A], W2[B]] + + reveal_type(foo(obj2)) # N: Revealed type is "Union[__main__.A, __main__.B]" + bar(obj2) # E: Cannot infer type argument 1 of "bar" + + b1_overload: A = foo(obj2) # E: Incompatible types in assignment (expression has type "Union[A, B]", variable has type "A") + b1_union: A = bar(obj2) # E: Cannot infer type argument 1 of "bar" + +[case testOverloadingInferUnionReturnWithObjectTypevarReturn] +from typing import overload, Union, TypeVar, Generic + +T = TypeVar('T') + +class W1(Generic[T]): pass +class W2(Generic[T]): pass +class A: pass +class B: pass + +class SomeType(Generic[T]): + @overload + def foo(self, x: W1[T]) -> T: ... + @overload + def foo(self, x: W2[T]) -> T: ... + def foo(self, x): ... + + def bar(self, x: Union[W1[T], W2[T]]) -> T: ... + +def wrapper() -> None: + obj1: Union[W1[A], W2[A]] + + a1 = SomeType[A]().foo(obj1) + reveal_type(a1) # N: Revealed type is "__main__.A" + + # Note: These should be fine, but mypy has an unrelated bug + # that makes them error out? + a2_overload: A = SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "W1[Never]" + a2_union: A = SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[Never], W2[Never]]" + + SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "W1[Never]" + SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[Never], W2[Never]]" + +[case testOverloadingInferUnionReturnWithBadObjectTypevarReturn] +from typing import overload, Union, TypeVar, Generic + +T = TypeVar('T') + +class W1(Generic[T]): pass +class W2(Generic[T]): pass +class A: pass +class B: pass + +class SomeType(Generic[T]): + @overload + def foo(self, x: W1[T]) -> T: ... + @overload + def foo(self, x: W2[T]) -> T: ... + def foo(self, x): ... + + def bar(self, x: Union[W1[T], W2[T]]) -> T: ... + +def wrapper(mysterious: T) -> T: + obj1: Union[W1[A], W2[B]] + + SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "W1[Never]" + SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[Never], W2[Never]]" + + SomeType[A]().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "W1[A]" + SomeType[A]().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[A], W2[A]]" + + SomeType[T]().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "W1[T]" + SomeType[T]().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[T], W2[T]]" + + return mysterious + +[case testOverloadingInferUnionReturnWithMixedTypevars] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') + +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... + +T1 = TypeVar('T1', bound=A) + +def t_is_same_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: + x1: Union[List[S], List[Tuple[T1, S]]] + y1: S + reveal_type(Dummy[T1]().foo(x1, y1)) # N: Revealed type is "Union[S`-2, T1`-1]" + + x2: Union[List[T1], List[Tuple[T1, T1]]] + y2: T1 + reveal_type(Dummy[T1]().foo(x2, y2)) # N: Revealed type is "T1`-1" + + return arg1, arg2 + +[builtins fixtures/list.pyi] + +[case testOverloadingInferUnionReturnWithMixedTypevarsInnerMismatch] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') + +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... + +T1 = TypeVar('T1', bound=A) + +def t_is_same_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: + # The arguments in the tuple are swapped + x3: Union[List[S], List[Tuple[S, T1]]] + y3: S + Dummy[T1]().foo(x3, y3) # E: Cannot infer type argument 1 of "foo" of "Dummy" \ + # E: Argument 1 to "foo" of "Dummy" has incompatible type "Union[List[S], List[Tuple[S, T1]]]"; expected "List[Tuple[T1, Any]]" + + x4: Union[List[int], List[Tuple[C, int]]] + y4: int + reveal_type(Dummy[C]().foo(x4, y4)) # N: Revealed type is "Union[builtins.int, __main__.C]" + Dummy[A]().foo(x4, y4) # E: Argument 1 to "foo" of "Dummy" has incompatible type "Union[List[int], List[Tuple[C, int]]]"; expected "List[Tuple[A, int]]" + + return arg1, arg2 + +[builtins fixtures/list.pyi] + +[case testOverloadingInferUnionReturnWithMixedTypevarsTighterBound] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') + +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... + +T1 = TypeVar('T1', bound=B) + +def t_is_tighter_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: + x1: Union[List[S], List[Tuple[T1, S]]] + y1: S + reveal_type(Dummy[T1]().foo(x1, y1)) # N: Revealed type is "Union[S`-2, T1`-1]" + + x2: Union[List[T1], List[Tuple[T1, T1]]] + y2: T1 + reveal_type(Dummy[T1]().foo(x2, y2)) # N: Revealed type is "T1`-1" + + return arg1, arg2 + +[builtins fixtures/list.pyi] + +[case testOverloadingInferUnionReturnWithTypevarsAndValueRestrictions] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') + +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... + +T3 = TypeVar('T3', B, C) + +def t_is_compatible_bound(arg1: T3, arg2: S) -> Tuple[T3, S]: + x1: Union[List[S], List[Tuple[T3, S]]] + y1: S + reveal_type(Dummy[T3]().foo(x1, y1)) + + x2: Union[List[T3], List[Tuple[T3, T3]]] + y2: T3 + reveal_type(Dummy[T3]().foo(x2, y2)) + + return arg1, arg2 + +[builtins fixtures/list.pyi] +[out] +main:22: note: Revealed type is "Union[S`-2, __main__.B]" +main:22: note: Revealed type is "Union[S`-2, __main__.C]" +main:26: note: Revealed type is "__main__.B" +main:26: note: Revealed type is "__main__.C" + +[case testOverloadInferUnionReturnWithInconsistentTypevarNames] +from typing import overload, TypeVar, Union + +T = TypeVar('T') +S = TypeVar('S') + +@overload +def consistent(x: T, y: str) -> T: ... +@overload +def consistent(x: T, y: int) -> T: ... +def consistent(x: T, y: Union[str, int]) -> T: + return x + +@overload +def inconsistent(x: T, y: str) -> T: ... +@overload +def inconsistent(x: S, y: int) -> S: ... +def inconsistent(x: T, y: Union[str, int]) -> T: + return x + +def test(x: T) -> T: + y: Union[str, int] + + reveal_type(consistent(x, y)) # N: Revealed type is "T`-1" + + # On one hand, this overload is defined in a weird way; on the other, there's technically nothing wrong with it. + inconsistent(x, y) + + return x + +[case testOverloadsAndNoneWithoutStrictOptional] +# flags: --no-strict-optional +from typing import overload, Optional + +@overload +def f(x: None) -> int: ... +@overload +def f(x: object) -> str: ... +def f(x): ... + +# We pretend strict-optional is enabled for overload definitions, +# even in non-strict optional mode +@overload +def g(x: None) -> int: ... +@overload +def g(x: int) -> str: ... +def g(x): ... + +# Calls are still checked normally though +a: None +b: int +c: Optional[int] +reveal_type(g(a)) # N: Revealed type is "builtins.int" +reveal_type(g(b)) # N: Revealed type is "builtins.str" +reveal_type(g(c)) # N: Revealed type is "builtins.str" + +[case testOverloadsAndNoneWithStrictOptional] +from typing import overload, Optional + +@overload +def f(x: None) -> int: ... +@overload +def f(x: object) -> str: ... +def f(x): ... + +@overload +def g(x: None) -> int: ... +@overload +def g(x: int) -> str: ... +def g(x): ... + +a: None +b: int +c: Optional[int] +reveal_type(g(a)) # N: Revealed type is "builtins.int" +reveal_type(g(b)) # N: Revealed type is "builtins.str" +reveal_type(g(c)) # N: Revealed type is "Union[builtins.str, builtins.int]" + +[case testOverloadsNoneAndTypeVarsWithNoStrictOptional] +# flags: --no-strict-optional +from typing import Callable, Iterable, TypeVar, overload, Optional + +T = TypeVar('T') +S = TypeVar('S') + +@overload +def mymap(func: None, seq: Iterable[T]) -> Iterable[T]: ... +@overload +def mymap(func: Callable[[T], S], seq: Iterable[T]) -> Iterable[S]: ... +def mymap(*args): ... + +seq = [1, 2, 3] +f1: Callable[[int], str] +f2: None +f3: Optional[Callable[[int], str]] + +reveal_type(mymap(f1, seq)) # N: Revealed type is "typing.Iterable[builtins.str]" +reveal_type(mymap(f2, seq)) # N: Revealed type is "typing.Iterable[builtins.int]" +reveal_type(mymap(f3, seq)) # N: Revealed type is "typing.Iterable[builtins.str]" + +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsNoneAndTypeVarsWithStrictOptional] +from typing import Callable, Iterable, TypeVar, overload, Optional + +T = TypeVar('T') +S = TypeVar('S') + +@overload +def mymap(func: None, seq: Iterable[T]) -> Iterable[T]: ... +@overload +def mymap(func: Callable[[T], S], seq: Iterable[T]) -> Iterable[S]: ... +def mymap(*args): ... + +seq = [1, 2, 3] +f1: Callable[[int], str] +f2: None +f3: Optional[Callable[[int], str]] + +reveal_type(mymap(f1, seq)) # N: Revealed type is "typing.Iterable[builtins.str]" +reveal_type(mymap(f2, seq)) # N: Revealed type is "typing.Iterable[builtins.int]" +reveal_type(mymap(f3, seq)) # N: Revealed type is "Union[typing.Iterable[builtins.str], typing.Iterable[builtins.int]]" + +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional1] +# flags: --no-strict-optional +from typing import overload, Union, NoReturn + +@overload +def narrow_int(x: str) -> NoReturn: ... +@overload +def narrow_int(x: int) -> int: ... +def narrow_int(x: Union[int, str]) -> Union[int, NoReturn]: + assert isinstance(x, int) + return x + +def test_narrow_int() -> None: + a: Union[int, str] + if int(): + a = narrow_int(a) + reveal_type(a) # N: Revealed type is "builtins.int" + + b: int + if int(): + b = narrow_int(b) + reveal_type(b) # N: Revealed type is "builtins.int" + + c: str + if int(): + c = narrow_int(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + # TODO: maybe we should make mypy report a warning instead? + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional1] +from typing import overload, Union, NoReturn + +@overload +def narrow_int(x: str) -> NoReturn: ... +@overload +def narrow_int(x: int) -> int: ... +def narrow_int(x: Union[int, str]) -> Union[int, NoReturn]: + assert isinstance(x, int) + return x + +def test_narrow_int() -> None: + a: Union[int, str] + if int(): + a = narrow_int(a) + reveal_type(a) # N: Revealed type is "builtins.int" + + b: int + if int(): + b = narrow_int(b) + reveal_type(b) # N: Revealed type is "builtins.int" + + c: str + if int(): + c = narrow_int(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + # TODO: maybe we should make mypy report a warning instead? + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional2] +# flags: --no-strict-optional +from typing import overload, Union, TypeVar, NoReturn, Optional + +T = TypeVar('T') +@overload +def narrow_none(x: None) -> NoReturn: ... +@overload +def narrow_none(x: T) -> T: ... +def narrow_none(x: Optional[T]) -> Union[NoReturn, T]: + assert x is not None + return x + +def test_narrow_none() -> None: + a: Optional[int] + if int(): + a = narrow_none(a) + reveal_type(a) # N: Revealed type is "builtins.int" + + b: int + if int(): + b = narrow_none(b) + reveal_type(b) # N: Revealed type is "builtins.int" + + c: None + if int(): + c = narrow_none(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional2] +from typing import overload, Union, TypeVar, NoReturn, Optional + +T = TypeVar('T') +@overload +def narrow_none(x: None) -> NoReturn: ... +@overload +def narrow_none(x: T) -> T: ... +def narrow_none(x: Optional[T]) -> Union[NoReturn, T]: + assert x is not None + return x + +def test_narrow_none() -> None: + a: Optional[int] + if int(): + a = narrow_none(a) + reveal_type(a) # N: Revealed type is "builtins.int" + + b: int + if int(): + b = narrow_none(b) + reveal_type(b) # N: Revealed type is "builtins.int" + + c: None + if int(): + c = narrow_none(c) + reveal_type(c) # Branch is now dead + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + + +[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional3] +# flags: --no-strict-optional +from typing import overload, TypeVar, NoReturn, Optional + +@overload +def narrow_none_v2(x: None) -> NoReturn: ... +@overload +def narrow_none_v2(x: T) -> T: ... +def narrow_none_v2(x: Optional[T]) -> T: + assert x is not None + return x + +def test_narrow_none_v2() -> None: + a: Optional[int] + if int(): + a = narrow_none_v2(a) + reveal_type(a) # N: Revealed type is "builtins.int" + + b: int + if int(): + b = narrow_none_v2(b) + reveal_type(b) # N: Revealed type is "builtins.int" + + c: None + if int(): + c = narrow_none_v2(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional3] +from typing import overload, TypeVar, NoReturn, Optional + +@overload +def narrow_none_v2(x: None) -> NoReturn: ... +@overload +def narrow_none_v2(x: T) -> T: ... +def narrow_none_v2(x: Optional[T]) -> T: + assert x is not None + return x + +def test_narrow_none_v2() -> None: + a: Optional[int] + if int(): + a = narrow_none_v2(a) + reveal_type(a) # N: Revealed type is "builtins.int" + + b: int + if int(): + b = narrow_none_v2(b) + reveal_type(b) # N: Revealed type is "builtins.int" + + c: None + if int(): + c = narrow_none_v2(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsAndNoReturnNarrowWhenBlacklistingSubtype] +from typing import TypeVar, NoReturn, Union, overload + +class Parent: ... +class A(Parent): ... +class B(Parent): ... +T = TypeVar('T', bound=Parent) + +@overload +def narrow_to_not_a(x: A) -> NoReturn: ... +@overload +def narrow_to_not_a(x: T) -> T: ... +def narrow_to_not_a(x: T) -> Union[NoReturn, T]: + assert not isinstance(x, A) + return x + +def test() -> None: + val: Union[A, B] + if int(): + val = narrow_to_not_a(val) + reveal_type(val) # N: Revealed type is "__main__.B" + + val2: A + if int(): + val2 = narrow_to_not_a(val2) + reveal_type(val2) # Branch now dead + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsAndNoReturnNarrowWhenBlacklistingSubtype2] +from typing import TypeVar, NoReturn, Union, overload + +class Parent: ... +class A(Parent): ... +class B(Parent): ... +T = TypeVar('T', bound=Parent) + +@overload +def narrow_to_not_a_v2(x: A) -> NoReturn: ... +@overload +def narrow_to_not_a_v2(x: T) -> T: ... +def narrow_to_not_a_v2(x: T) -> T: + assert not isinstance(x, A) + return x + +def test_v2(val: Union[A, B], val2: A) -> None: + if int(): + val = narrow_to_not_a_v2(val) + reveal_type(val) # N: Revealed type is "__main__.B" + + if int(): + val2 = narrow_to_not_a_v2(val2) + reveal_type(val2) # Branch now dead + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadWithNonGenericDescriptor] +from typing import overload, Any, Optional, Union + +class NumberAttribute: + @overload + def __get__(self, instance: None, owner: Any) -> 'NumberAttribute': ... + @overload + def __get__(self, instance: object, owner: Any) -> int: ... + def __get__(self, instance: Optional[object], owner: Any) -> Union['NumberAttribute', int]: + if instance is None: + return self + else: + return 3 + + def foo(self) -> str: ... + +class MyModel: + my_number = NumberAttribute() + +reveal_type(MyModel().my_number) # N: Revealed type is "builtins.int" +MyModel().my_number.foo() # E: "int" has no attribute "foo" + +reveal_type(MyModel.my_number) # N: Revealed type is "__main__.NumberAttribute" +reveal_type(MyModel.my_number.foo()) # N: Revealed type is "builtins.str" + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadWithNonGenericDescriptorLookalike] +from typing import overload, Any, Optional, Union + +class FakeAttribute: + @overload + def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... + @overload + def dummy(self, instance: object, owner: Any) -> int: ... + def dummy(self, instance: Optional[object], owner: Any) -> Union['FakeAttribute', int]: ... + +[case testOverloadWithGenericDescriptor] +from typing import overload, Any, Optional, TypeVar, Type, Union, Generic + +T = TypeVar('T') + +class NumberAttribute(Generic[T]): + @overload + def __get__(self, instance: None, owner: Type[T]) -> 'NumberAttribute[T]': ... + @overload + def __get__(self, instance: T, owner: Type[T]) -> int: ... + def __get__(self, instance: Optional[T], owner: Type[T]) -> Union['NumberAttribute[T]', int]: + if instance is None: + return self + else: + return 3 + + def foo(self) -> str: ... + +class MyModel: + my_number = NumberAttribute[MyModel]() + +reveal_type(MyModel().my_number) # N: Revealed type is "builtins.int" +MyModel().my_number.foo() # E: "int" has no attribute "foo" + +reveal_type(MyModel.my_number) # N: Revealed type is "__main__.NumberAttribute[__main__.MyModel]" +reveal_type(MyModel.my_number.foo()) # N: Revealed type is "builtins.str" + +reveal_type(NumberAttribute[MyModel]().__get__(None, MyModel)) # N: Revealed type is "__main__.NumberAttribute[__main__.MyModel]" +reveal_type(NumberAttribute[str]().__get__(None, str)) # N: Revealed type is "__main__.NumberAttribute[builtins.str]" + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadWithGenericDescriptorLookalike] +from typing import overload, Any, Optional, TypeVar, Type, Union, Generic + +T = TypeVar('T') + +class FakeAttribute(Generic[T]): + @overload + def dummy(self, instance: None, owner: Type[T]) -> 'FakeAttribute[T]': ... + @overload + def dummy(self, instance: T, owner: Type[T]) -> int: ... + def dummy(self, instance: Optional[T], owner: Type[T]) -> Union['FakeAttribute[T]', int]: ... + +[case testOverloadWithClassMethods] +from typing import overload + +class Wrapper: + @overload + @classmethod + def foo(cls, x: int) -> int: ... + @overload + @classmethod + def foo(cls, x: str) -> str: ... + @classmethod + def foo(cls, x): pass + +reveal_type(Wrapper.foo(3)) # N: Revealed type is "builtins.int" +reveal_type(Wrapper.foo("foo")) # N: Revealed type is "builtins.str" + +[builtins fixtures/classmethod.pyi] + +[case testOverloadWithInconsistentClassMethods] +from typing import overload + +class Wrapper1: + @overload # E: Overload does not consistently use the "@classmethod" decorator on all function signatures. + @classmethod + def foo(cls, x: int) -> int: ... + @overload + @classmethod + def foo(cls, x: str) -> str: ... + def foo(cls, x): pass + +class Wrapper2: + @overload # E: Overload does not consistently use the "@classmethod" decorator on all function signatures. + @classmethod + def foo(cls, x: int) -> int: ... + @overload + def foo(cls, x: str) -> str: ... + @classmethod + def foo(cls, x): pass + +class Wrapper3: + @overload # E: Overload does not consistently use the "@classmethod" decorator on all function signatures. + def foo(cls, x: int) -> int: ... + @overload + def foo(cls, x: str) -> str: ... + @classmethod + def foo(cls, x): pass + +[builtins fixtures/classmethod.pyi] + +[case testOverloadWithSwappedDecorators] +from typing import overload + +class Wrapper1: + @classmethod + @overload + def foo(cls, x: int) -> int: ... + + @classmethod + @overload + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x): pass + +class Wrapper2: + @classmethod + @overload + def foo(cls, x: int) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x): pass + +class Wrapper3: + @classmethod # E: Overload does not consistently use the "@classmethod" decorator on all function signatures. + @overload + def foo(cls, x: int) -> int: ... + + @overload + def foo(cls, x: str) -> str: ... + + def foo(cls, x): pass + +reveal_type(Wrapper1.foo(3)) # N: Revealed type is "builtins.int" +reveal_type(Wrapper2.foo(3)) # N: Revealed type is "builtins.int" + +[builtins fixtures/classmethod.pyi] + +[case testOverloadFaultyClassMethodInheritance] +from typing import overload + +class A: pass +class B(A): pass +class C(B): pass + +class Parent: + @overload + @classmethod + def foo(cls, x: B) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x): pass + +class BadChild(Parent): + @overload # Fail + @classmethod + def foo(cls, x: C) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x): pass + +class GoodChild(Parent): + @overload + @classmethod + def foo(cls, x: A) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x): pass + +[builtins fixtures/classmethod.pyi] +[out] +main:20: error: Signature of "foo" incompatible with supertype "Parent" +main:20: note: Superclass: +main:20: note: @overload +main:20: note: @classmethod +main:20: note: def foo(cls, x: B) -> int +main:20: note: @overload +main:20: note: @classmethod +main:20: note: def foo(cls, x: str) -> str +main:20: note: Subclass: +main:20: note: @overload +main:20: note: @classmethod +main:20: note: def foo(cls, x: C) -> int +main:20: note: @overload +main:20: note: @classmethod +main:20: note: def foo(cls, x: str) -> str + +[case testOverloadClassMethodMixingInheritance] +from typing import overload + +class BadParent: + @overload + @classmethod + def foo(cls, x: int) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x): pass + +class BadChild(BadParent): + @overload # Fail + def foo(cls, x: int) -> int: ... + + @overload + def foo(cls, x: str) -> str: ... + + def foo(cls, x): pass + +class GoodParent: + @overload + def foo(cls, x: int) -> int: ... + + @overload + def foo(cls, x: str) -> str: ... + + def foo(cls, x): pass + +class GoodChild(GoodParent): + @overload + @classmethod + def foo(cls, x: int) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x): pass + +[builtins fixtures/classmethod.pyi] +[out] +main:16: error: Signature of "foo" incompatible with supertype "BadParent" +main:16: note: Superclass: +main:16: note: @overload +main:16: note: @classmethod +main:16: note: def foo(cls, x: int) -> int +main:16: note: @overload +main:16: note: @classmethod +main:16: note: def foo(cls, x: str) -> str +main:16: note: Subclass: +main:16: note: @overload +main:16: note: def foo(cls, x: int) -> int +main:16: note: @overload +main:16: note: def foo(cls, x: str) -> str + +[case testOverloadClassMethodImplementation] +from typing import overload, Union + +class Wrapper: + @classmethod + def other(cls) -> str: + return "..." + + @overload + @classmethod + def foo(cls, x: int) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod # E: Overloaded function implementation cannot produce return type of signature 1 + def foo(cls, x: Union[int, str]) -> str: + reveal_type(cls) # N: Revealed type is "Type[__main__.Wrapper]" + reveal_type(cls.other()) # N: Revealed type is "builtins.str" + return "..." + +[builtins fixtures/classmethod.pyi] + +[case testOverloadWithStaticMethods] +from typing import overload + +class Wrapper: + @overload + @staticmethod + def foo(x: int) -> int: ... + @overload + @staticmethod + def foo(x: str) -> str: ... + @staticmethod + def foo(x): pass + +reveal_type(Wrapper.foo(3)) # N: Revealed type is "builtins.int" +reveal_type(Wrapper.foo("foo")) # N: Revealed type is "builtins.str" + +[builtins fixtures/staticmethod.pyi] + +[case testOverloadWithInconsistentStaticMethods] +from typing import overload, Union + +class Wrapper1: + @overload # E: Overload does not consistently use the "@staticmethod" decorator on all function signatures. + @staticmethod + def foo(x: int) -> int: ... + @overload + @staticmethod + def foo(x: str) -> str: ... + def foo(x): pass + +class Wrapper2: + @overload # E: Overload does not consistently use the "@staticmethod" decorator on all function signatures. + @staticmethod + def foo(x: int) -> int: ... + @overload + def foo(x: str) -> str: ... # E: Self argument missing for a non-static method (or an invalid type for self) + @staticmethod + def foo(x): pass + +class Wrapper3: + @overload # E: Overload does not consistently use the "@staticmethod" decorator on all function signatures. + @staticmethod + def foo(x: int) -> int: ... + @overload + @staticmethod + def foo(x: str) -> str: ... + def foo(x: Union[int, str]): pass # E: Self argument missing for a non-static method (or an invalid type for self) +[builtins fixtures/staticmethod.pyi] + +[case testOverloadWithSwappedDecorators2] +from typing import overload + +class Wrapper1: + @staticmethod + @overload + def foo(x: int) -> int: ... + + @staticmethod + @overload + def foo(x: str) -> str: ... + + @staticmethod + def foo(x): pass + +class Wrapper2: + @staticmethod + @overload + def foo(x: int) -> int: ... + + @overload + @staticmethod + def foo(x: str) -> str: ... + + @staticmethod + def foo(x): pass + +class Wrapper3: + @staticmethod # E: Overload does not consistently use the "@staticmethod" decorator on all function signatures. + @overload + def foo(x: int) -> int: ... + + @overload + def foo(x: str) -> str: ... # E: Self argument missing for a non-static method (or an invalid type for self) + + @staticmethod + def foo(x): pass + +reveal_type(Wrapper1.foo(3)) # N: Revealed type is "builtins.int" +reveal_type(Wrapper2.foo(3)) # N: Revealed type is "builtins.int" + +[builtins fixtures/staticmethod.pyi] + +[case testOverloadFaultyStaticMethodInheritance] +from typing import overload + +class A: pass +class B(A): pass +class C(B): pass + +class Parent: + @overload + @staticmethod + def foo(x: B) -> int: ... + + @overload + @staticmethod + def foo(x: str) -> str: ... + + @staticmethod + def foo(x): pass + +class BadChild(Parent): + @overload # Fail + @staticmethod + def foo(x: C) -> int: ... + + @overload + @staticmethod + def foo(x: str) -> str: ... + + @staticmethod + def foo(x): pass + +class GoodChild(Parent): + @overload + @staticmethod + def foo(x: A) -> int: ... + + @overload + @staticmethod + def foo(x: str) -> str: ... + + @staticmethod + def foo(x): pass + +[builtins fixtures/staticmethod.pyi] +[out] +main:20: error: Signature of "foo" incompatible with supertype "Parent" +main:20: note: Superclass: +main:20: note: @overload +main:20: note: @staticmethod +main:20: note: def foo(x: B) -> int +main:20: note: @overload +main:20: note: @staticmethod +main:20: note: def foo(x: str) -> str +main:20: note: Subclass: +main:20: note: @overload +main:20: note: @staticmethod +main:20: note: def foo(x: C) -> int +main:20: note: @overload +main:20: note: @staticmethod +main:20: note: def foo(x: str) -> str + +[case testOverloadStaticMethodMixingInheritance] +from typing import overload + +class BadParent: + @overload + @staticmethod + def foo(x: int) -> int: ... + + @overload + @staticmethod + def foo(x: str) -> str: ... + + @staticmethod + def foo(x): pass + +class BadChild(BadParent): + @overload # Fail + def foo(self, x: int) -> int: ... + + @overload + def foo(self, x: str) -> str: ... + + def foo(self, x): pass + +class GoodParent: + @overload + def foo(self, x: int) -> int: ... + + @overload + def foo(self, x: str) -> str: ... + + def foo(self, x): pass + +class GoodChild(GoodParent): + @overload + @staticmethod + def foo(x: int) -> int: ... + + @overload + @staticmethod + def foo(x: str) -> str: ... + + @staticmethod + def foo(x): pass + +[builtins fixtures/staticmethod.pyi] +[out] +main:16: error: Signature of "foo" incompatible with supertype "BadParent" +main:16: note: Superclass: +main:16: note: @overload +main:16: note: @staticmethod +main:16: note: def foo(x: int) -> int +main:16: note: @overload +main:16: note: @staticmethod +main:16: note: def foo(x: str) -> str +main:16: note: Subclass: +main:16: note: @overload +main:16: note: def foo(self, x: int) -> int +main:16: note: @overload +main:16: note: def foo(self, x: str) -> str + +[case testOverloadStaticMethodImplementation] +from typing import overload, Union + +class Wrapper: + @staticmethod + def other() -> str: + return "..." + + @overload + @staticmethod + def foo(x: int) -> int: ... + + @overload + @staticmethod + def foo(x: str) -> str: ... + + @staticmethod # E: Overloaded function implementation cannot produce return type of signature 1 + def foo(x: Union[int, str]) -> str: + return 3 # E: Incompatible return value type (got "int", expected "str") + +[builtins fixtures/staticmethod.pyi] + +[case testUnionMathOverloadingReturnsBestType] +from typing import Union, overload + +@overload +def f(x: Union[int, str]) -> int: ... +@overload +def f(x: object) -> object: ... +def f(x): + pass + +x: Union[int, str] +reveal_type(f(x)) # N: Revealed type is "builtins.int" +[out] + +[case testOverloadAndSelfTypes] +from typing import overload, Union, TypeVar, Type + +T = TypeVar('T', bound='Parent') +class Parent: + @overload + def foo(self: T, x: int) -> T: pass + + @overload + def foo(self, x: str) -> str: pass + + def foo(self: T, x: Union[int, str]) -> Union[T, str]: + reveal_type(self.bar()) # N: Revealed type is "builtins.str" + return self + + def bar(self) -> str: pass + +class Child(Parent): + def child_only(self) -> int: pass + +x: Union[int, str] +reveal_type(Parent().foo(3)) # N: Revealed type is "__main__.Parent" +reveal_type(Child().foo(3)) # N: Revealed type is "__main__.Child" +reveal_type(Child().foo("...")) # N: Revealed type is "builtins.str" +reveal_type(Child().foo(x)) # N: Revealed type is "Union[__main__.Child, builtins.str]" +reveal_type(Child().foo(3).child_only()) # N: Revealed type is "builtins.int" + +[case testOverloadAndSelfTypesGenericNoOverlap] +from typing import Generic, TypeVar, Any, overload, Self, Union + +T = TypeVar("T") +class C(Generic[T]): + @overload + def get(self, obj: None) -> Self: ... + @overload + def get(self, obj: Any) -> T: ... + def get(self, obj: Union[Any, None]) -> Union[T, Self]: + return self + +class D(C[int]): ... +d: D +reveal_type(d.get(None)) # N: Revealed type is "__main__.D" +reveal_type(d.get("whatever")) # N: Revealed type is "builtins.int" + +[case testOverloadAndClassTypes] +from typing import overload, Union, TypeVar, Type + +T = TypeVar('T', bound='Parent') +class Parent: + @overload + @classmethod + def foo(cls: Type[T], x: int) -> Type[T]: pass + + @overload + @classmethod + def foo(cls, x: str) -> str: pass + + @classmethod + def foo(cls: Type[T], x: Union[int, str]) -> Union[Type[T], str]: + reveal_type(cls.bar()) # N: Revealed type is "builtins.str" + return cls + + @classmethod + def bar(cls) -> str: pass + +class Child(Parent): + def child_only(self) -> int: pass + +x: Union[int, str] +reveal_type(Parent.foo(3)) # N: Revealed type is "Type[__main__.Parent]" +reveal_type(Child.foo(3)) # N: Revealed type is "Type[__main__.Child]" +reveal_type(Child.foo("...")) # N: Revealed type is "builtins.str" +reveal_type(Child.foo(x)) # N: Revealed type is "Union[Type[__main__.Child], builtins.str]" +reveal_type(Child.foo(3)().child_only()) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testOptionalIsNotAUnionIfNoStrictOverload] +# flags: --no-strict-optional +from typing import Optional, overload + +class B: pass +class C(B): pass + +@overload +def rp(x: C) -> C: ... +@overload +def rp(x: B) -> B: ... +def rp(x): + pass + +x: Optional[C] +reveal_type(rp(x)) # N: Revealed type is "__main__.C" +[out] + +[case testUnionMathTrickyOverload1] +from typing import Union, overload + +@overload +def f(x: int, y: int) -> int: ... +@overload +def f(x: object, y: str) -> str: ... +def f(x): + pass + +x: Union[int, str] +y: Union[int, str] +f(x, y) +[out] +main:12: error: Argument 1 to "f" has incompatible type "Union[int, str]"; expected "int" +main:12: error: Argument 2 to "f" has incompatible type "Union[int, str]"; expected "int" + +[case testUnionMathTrickyOverload2] +from typing import overload, Union, Any + +class C: + def f(self, other: C) -> C: ... + +class D(C): + @overload + def f(self, other: D) -> D: ... + @overload + def f(self, other: C) -> C: ... + def f(self, other): ... + +x: D +y: Union[D, Any] +reveal_type(x.f(y)) # N: Revealed type is "Union[__main__.D, Any]" +[out] + +[case testManyUnionsInOverload] +from typing import overload, TypeVar, Union + +T = TypeVar('T') + +@overload +def f(x: int, y: object, z: object, t: object, u: object, w: object, v: object, s: object) -> int: ... +@overload +def f(x: str, y: object, z: object, t: object, u: object, w: object, v: object, s: object) -> str: ... +@overload +def f(x: T, y: object, z: object, t: object, u: object, w: object, v: object, s: object) -> T: ... +def f(*args, **kwargs): + pass + +class A: pass +class B: pass +x: Union[int, str, A, B] +y = f(x, x, x, x, x, x, x, x) # 8 args + +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str, __main__.A, __main__.B]" +[builtins fixtures/dict.pyi] +[out] + +[case testOverloadsWithNoneComingSecondAreAlwaysFlaggedInNoStrictOptional] +# flags: --no-strict-optional +from typing import overload + +@overload +def none_first(x: None) -> None: ... +@overload +def none_first(x: int) -> int: ... +def none_first(x: int) -> int: + return x + +@overload +def none_second(x: int) -> int: ... +@overload +def none_second(x: None) -> None: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +def none_second(x: int) -> int: + return x + +[case testOverloadsWithNoneComingSecondIsOkInStrictOptional] +from typing import overload, Optional + +@overload +def none_first(x: None) -> None: ... +@overload +def none_first(x: int) -> int: ... +def none_first(x: Optional[int]) -> Optional[int]: + return x + +@overload +def none_second(x: int) -> int: ... +@overload +def none_second(x: None) -> None: ... +def none_second(x: Optional[int]) -> Optional[int]: + return x + +@overload +def none_loose_impl(x: None) -> None: ... +@overload +def none_loose_impl(x: int) -> int: ... +def none_loose_impl(x: int) -> int: + return x +[out] +main:21: error: Overloaded function implementation does not accept all possible arguments of signature 1 +main:21: error: Overloaded function implementation cannot produce return type of signature 1 + +[case testTooManyUnionsException] +from typing import overload, Union + +@overload +def f(*args: int) -> int: ... +@overload +def f(*args: str) -> str: ... +def f(*args): + pass + +x: Union[int, str] +f(x, x, x, x, x, x, x, x) +[builtins fixtures/tuple.pyi] +[out] +main:11: error: Not all union combinations were tried because there are too many unions +main:11: error: Argument 1 to "f" has incompatible type "Union[int, str]"; expected "int" +main:11: error: Argument 2 to "f" has incompatible type "Union[int, str]"; expected "int" +main:11: error: Argument 3 to "f" has incompatible type "Union[int, str]"; expected "int" +main:11: error: Argument 4 to "f" has incompatible type "Union[int, str]"; expected "int" +main:11: error: Argument 5 to "f" has incompatible type "Union[int, str]"; expected "int" +main:11: error: Argument 6 to "f" has incompatible type "Union[int, str]"; expected "int" +main:11: error: Argument 7 to "f" has incompatible type "Union[int, str]"; expected "int" +main:11: error: Argument 8 to "f" has incompatible type "Union[int, str]"; expected "int" + +[case testSafeDunderOverlapInSubclass] +from typing import overload + +class A: + def __add__(self, x : 'A') -> 'A': ... + +class B(A): + @overload + def __add__(self, x : 'B') -> 'B': ... + @overload + def __add__(self, x : 'A') -> 'A' : ... + def __add__(self, x): + pass +[out] + +[case testUnsafeDunderOverlapInSubclass] +from typing import overload + +class A: + def __add__(self, x : 'A') -> 'A': + if isinstance(x, A): + return A() + else: + return NotImplemented + +# This is unsafe override because of the problem below +class B(A): + @overload # Fail + def __add__(self, x : 'Other') -> 'B' : ... + @overload + def __add__(self, x : 'A') -> 'A': ... + def __add__(self, x): + if isinstance(x, Other): + return B() + elif isinstance(x, A): + return A() + else: + return NotImplemented + +class Other: + def __radd__(self, x: 'A') -> 'Other': + if isinstance(x, A): + return Other() + else: + return NotImplemented + +actually_b: A = B() +reveal_type(actually_b + Other()) # Note +# Runtime type is B, this is why we report the error on overriding. +[builtins fixtures/isinstance.pyi] +[out] +main:12: error: Signature of "__add__" incompatible with supertype "A" +main:12: note: Superclass: +main:12: note: def __add__(self, A, /) -> A +main:12: note: Subclass: +main:12: note: @overload +main:12: note: def __add__(self, Other, /) -> B +main:12: note: @overload +main:12: note: def __add__(self, A, /) -> A +main:12: note: Overloaded operator methods can't have wider argument types in overrides +main:32: note: Revealed type is "__main__.Other" + +[case testOverloadErrorMessageManyMatches] +from typing import overload + +class A: pass +class B: pass +class C: pass +class D: pass + +@overload +def f(x: A) -> None: ... +@overload +def f(x: B) -> None: ... +@overload +def f(x: C) -> None: ... +@overload +def f(x: D) -> None: ... +@overload +def f(x: int, y: int) -> None: ... +def f(*args): pass + +f(3) # E: No overload variant of "f" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def f(x: A) -> None \ + # N: def f(x: B) -> None \ + # N: def f(x: C) -> None \ + # N: def f(x: D) -> None \ + # N: def f(x: int, y: int) -> None + +@overload +def g(x: A) -> None: ... +@overload +def g(x: B) -> None: ... +@overload +def g(x: C) -> None: ... +def g(*args): pass + +g(3) # E: No overload variant of "g" matches argument type "int" \ + # N: Possible overload variants: \ + # N: def g(x: A) -> None \ + # N: def g(x: B) -> None \ + # N: def g(x: C) -> None +[builtins fixtures/tuple.pyi] + +[case testOverloadedInIter] +from lib import f, g + +for fun in [f, g]: + reveal_type(fun) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.str, def (x: builtins.str) -> builtins.int)" +[file lib.pyi] +from typing import overload + +@overload +def f(x: int) -> str: ... +@overload +def f(x: str) -> int: ... + +@overload +def g(x: int) -> str: ... +@overload +def g(x: str) -> int: ... + +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] +[out] + +[case testNestedOverloadsNoCrash] +from typing import overload + +def f() -> None: + @overload + def g(x: str) -> str: ... + @overload + def g(x: int) -> int: ... + def g(x): + pass + g(str()) +[out] + +[case testNestedOverloadsTypeVar] +from typing import overload, TypeVar + +T = TypeVar('T') + +def f() -> None: + @overload + def g(x: str) -> str: ... + @overload + def g(x: T, y: int) -> T: ... + def g(x): + pass + + g(str(), str()) # E: No overload variant of "g" matches argument types "str", "str" \ + # N: Possible overload variants: \ + # N: def g(x: str) -> str \ + # N: def [T] g(x: T, y: int) -> T + reveal_type(g(str(), int())) # N: Revealed type is "builtins.str" +[out] + +[case testNestedOverloadsTypeVarOverlap] +from typing import overload, TypeVar + +T = TypeVar('T') + +def f() -> None: + @overload + def g(x: str) -> int: ... + @overload + def g(x: T) -> T: ... + def g(x): + pass +[out] + +[case testNestedOverloadsMutuallyRecursive] +from typing import overload, TypeVar, Dict, Any + +class C: ... +T = TypeVar('T') + +def f() -> None: + @overload + def g() -> None: ... + @overload + def g(x: T) -> Dict[int, T]: ... + def g(*args, **kwargs) -> Any: + reveal_type(h(C())) # N: Revealed type is "builtins.dict[builtins.str, __main__.C]" + + @overload + def h() -> None: ... + @overload + def h(x: T) -> Dict[str, T]: ... + def h(*args, **kwargs) -> Any: + reveal_type(g(C())) # N: Revealed type is "builtins.dict[builtins.int, __main__.C]" + +[builtins fixtures/dict.pyi] +[out] + +[case testOverloadConstrainedTypevarNotShadowingAny] +from lib import attr +from typing import Any + +reveal_type(attr(1)) # N: Revealed type is "builtins.int" +reveal_type(attr("hi")) # N: Revealed type is "builtins.int" +x: Any +reveal_type(attr(x)) # N: Revealed type is "Any" +attr("hi", 1) # E: No overload variant of "attr" matches argument types "str", "int" \ + # N: Possible overload variants: \ + # N: def [T: (int, float)] attr(default: T, blah: int = ...) -> T \ + # N: def attr(default: Any = ...) -> int +[file lib.pyi] +from typing import overload, Any, TypeVar + +T = TypeVar('T', int, float) + +@overload +def attr(default: T, blah: int = ...) -> T: ... +@overload +def attr(default: Any = ...) -> int: ... +[out] + +[case testOverloadBoundedTypevarNotShadowingAny] +from lib import attr +from typing import Any + +reveal_type(attr(1)) # N: Revealed type is "builtins.int" +reveal_type(attr("hi")) # N: Revealed type is "builtins.int" +x: Any +reveal_type(attr(x)) # N: Revealed type is "Any" +attr("hi", 1) # E: No overload variant of "attr" matches argument types "str", "int" \ + # N: Possible overload variants: \ + # N: def [T: int] attr(default: T = ..., blah: int = ...) -> T \ + # N: def attr(default: Any = ...) -> int +[file lib.pyi] +from typing import overload, TypeVar, Any + +T = TypeVar('T', bound=int) + +@overload +def attr(default: T = ..., blah: int = ...) -> T: ... +@overload +def attr(default: Any = ...) -> int: ... +[out] + +[case testAnyIsOKAsFallbackInOverloads] +import stub +[file stub.pyi] +from typing import TypeVar, Any, overload + +T = TypeVar('T') + +@overload +def foo(x: T) -> T: ... +@overload +def foo(x: Any) -> Any: ... + +@overload +def bar(x: T) -> T: ... +@overload +def bar(x: Any) -> int: ... +[out] + +[case testOverloadsIgnorePromotions] +from typing import overload, List, Union, _promote + +class Parent: pass +class Child(Parent): pass + +children: List[Child] +parents: List[Parent] + +@overload +def f(x: Child) -> List[Child]: pass +@overload +def f(x: Parent) -> List[Parent]: pass +def f(x: Union[Child, Parent]) -> Union[List[Child], List[Parent]]: + if isinstance(x, Child): + reveal_type(x) # N: Revealed type is "__main__.Child" + return children + else: + reveal_type(x) # N: Revealed type is "__main__.Parent" + return parents + +ints: List[int] +floats: List[float] + +@overload +def g(x: int) -> List[int]: pass +@overload +def g(x: float) -> List[float]: pass +def g(x: Union[int, float]) -> Union[List[int], List[float]]: + if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" + return ints + else: + reveal_type(x) # N: Revealed type is "builtins.float" + return floats + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverloadsTypesAndUnions] +from typing import overload, Type, Union + +class A: pass +class B: pass + +@overload +def f(x: Type[A]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: Union[Type[A], Type[B]]) -> str: ... +def f(x: Union[Type[A], Type[B]]) -> Union[int, str]: + return 1 + +[case testBadOverloadProbableMatch] +from typing import overload, List, Type + +class Other: pass + +@overload +def multiple_plausible(x: int) -> int: ... +@overload +def multiple_plausible(x: str) -> str: ... +def multiple_plausible(x): pass + + +@overload +def single_plausible(x: Type[int]) -> int: ... +@overload +def single_plausible(x: List[str]) -> str: ... +def single_plausible(x): pass + +a = multiple_plausible(Other()) # E: No overload variant of "multiple_plausible" matches argument type "Other" \ + # N: Possible overload variants: \ + # N: def multiple_plausible(x: int) -> int \ + # N: def multiple_plausible(x: str) -> str +reveal_type(a) # N: Revealed type is "Any" + +b = single_plausible(Other) # E: Argument 1 to "single_plausible" has incompatible type "Type[Other]"; expected "Type[int]" +reveal_type(b) # N: Revealed type is "builtins.int" + +c = single_plausible([Other()]) # E: List item 0 has incompatible type "Other"; expected "str" +reveal_type(c) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + +[case testDisallowUntypedDecoratorsOverload] +# flags: --disallow-untyped-decorators +from typing import Any, Callable, overload, TypeVar + +F = TypeVar('F', bound=Callable[..., Any]) + +@overload +def dec(x: F) -> F: ... +@overload +def dec(x: str) -> Callable[[F], F]: ... +def dec(x) -> Any: + pass + +@dec +def f(name: str) -> int: + return 0 + +@dec('abc') +def g(name: str) -> int: + return 0 + +reveal_type(f) # N: Revealed type is "def (name: builtins.str) -> builtins.int" +reveal_type(g) # N: Revealed type is "def (name: builtins.str) -> builtins.int" + +[case testDisallowUntypedDecoratorsOverloadDunderCall] +# flags: --disallow-untyped-decorators +from typing import Any, Callable, overload, TypeVar + +F = TypeVar('F', bound=Callable[..., Any]) + +class Dec: + @overload + def __call__(self, x: F) -> F: ... + @overload + def __call__(self, x: str) -> Callable[[F], F]: ... + def __call__(self, x) -> Any: + pass + +dec = Dec() + +@dec +def f(name: str) -> int: + return 0 + +@dec('abc') +def g(name: str) -> int: + return 0 + +reveal_type(f) # N: Revealed type is "def (name: builtins.str) -> builtins.int" +reveal_type(g) # N: Revealed type is "def (name: builtins.str) -> builtins.int" + +[case testOverloadBadArgumentsInferredToAny1] +from typing import Union, Any, overload + +def bar(x: int) -> Union[int, Any]: ... + +@overload +def foo(x: str) -> None: ... +@overload +def foo(x: int) -> None: ... +def foo(x) -> None: pass + +foo(bar('lol')) # E: Argument 1 to "bar" has incompatible type "str"; expected "int" + +[case testOverloadBadArgumentsInferredToAny2] +from typing import Union, Iterable, Tuple, TypeVar, Generic, overload, Any + +class A: + def foo(self) -> Iterable[int]: pass + +def bar(x: int) -> Union[A, int]: ... + +_T = TypeVar('_T') + +@overload +def foo() -> None: ... +@overload +def foo(iterable: Iterable[_T]) -> None: ... +def foo(iterable = None) -> None: pass + +foo(bar('lol').foo()) # E: Item "int" of "Union[A, int]" has no attribute "foo" \ + # E: Argument 1 to "bar" has incompatible type "str"; expected "int" + + +[case testOverloadInferringArgumentsUsingContext1] +from typing import Optional, List, overload, TypeVar +T = TypeVar('T') + +def g(x: Optional[T] = None) -> List[T]: ... + +@overload +def f(x: int) -> int: ... +@overload +def f(x: List[int]) -> List[int]: ... +def f(x): pass + +reveal_type(f(g())) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testOverloadInferringArgumentsUsingContext2-skip] +# TODO: Overloads only use outer context to infer type variables in a given overload variant, +# but never use outer context to _choose_ a better overload in ambiguous situations +# like empty containers or multiple inheritance, instead just always choosing the first one. + +from typing import Optional, List, overload, TypeVar +T = TypeVar('T') +@overload +def g(x: List[str]) -> List[str]: ... +@overload +def g(x: List[int]) -> List[int]: ... +def g(x): + pass + +@overload +def f(x: int) -> int: ... +@overload +def f(x: List[int]) -> List[int]: ... +def f(x): + pass + +reveal_type(f(g([]))) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testOverloadDeferredNode] +from typing import Callable, TypeVar, Generic, Any, overload + +_S = TypeVar('_S') +_T = TypeVar('_T') +_R = TypeVar('_R') + +@overload +def partial(__func: Callable[[_T], _S], __arg: _T) -> Callable[[], _S]: ... +@overload +def partial(__func: Callable[[_T, _S], _S], __arg: _T) -> Callable[[_S], _R]: ... +def partial(*args: Any) -> Any: + pass + +def f(f: Callable[[int], int]) -> None: + pass + +def dec(f: Callable[[_S, _T], _R]) -> Callable[[_S, _T], _R]: pass + +def asdf() -> None: + f(partial(lol, 0)) + +@dec +def lol(x: int, y: int) -> int: + pass +[builtins fixtures/tuple.pyi] + +[case testVeryBrokenOverload] +import lib +reveal_type(lib.func) +[file lib.pyi] +@overload +def func(x: int) -> int: ... + +def func(x): + return x +[out] +tmp/lib.pyi:1: error: Name "overload" is not defined +tmp/lib.pyi:4: error: Name "func" already defined on line 1 +main:2: note: Revealed type is "Any" + +-- Order of errors is different +[case testVeryBrokenOverload2] + +import lib +reveal_type(lib.func) +[file lib.pyi] +@overload +def func(x: int) -> int: ... +@overload +def func(x: str) -> str: ... +[out] +tmp/lib.pyi:1: error: Name "overload" is not defined +tmp/lib.pyi:3: error: Name "func" already defined on line 1 +tmp/lib.pyi:3: error: Name "overload" is not defined +main:3: note: Revealed type is "Any" + +[case testLiteralSubtypeOverlap] +from typing import overload +from typing_extensions import Literal + +class MyInt(int): ... + +# Strictly speaking we can't prove this is unsafe (this depends on the implementation), +# but such APIs seem like an anti-pattern anyways. +@overload +def foo(x: Literal[0]) -> None: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def foo(x: MyInt) -> int: ... +def foo(x): + ... +[builtins fixtures/tuple.pyi] + +[case testOverloadedToGeneric] +from typing import TypeVar, Callable, NewType, overload, Union + +# int in our stubs isn't overloaded +class fakeint: + @overload + def __init__(self, x: Union[str, bytes] = ...) -> None: ... + @overload + def __init__(self, x: Union[str, bytes], base: int) -> None: ... + def __init__(self, *args) -> None: pass # type: ignore + + +U = TypeVar('U') +V = TypeVar('V') +W = TypeVar('W') +def compose(f: Callable[[U], V], g: Callable[[W], U]) -> Callable[[W], V]: + return lambda x: f(g(x)) + +ID = NewType("ID", fakeint) + +compose(ID, fakeint)("test") +reveal_type(compose(ID, fakeint)) # N: Revealed type is "def (Union[builtins.str, builtins.bytes]) -> __main__.ID" + +[builtins fixtures/tuple.pyi] + +[case testOverloadTwoTypeArgs] +from typing import Generic, overload, TypeVar, Any + +T1 = TypeVar("T1") +T2 = TypeVar("T2") + +class A: ... +class B: ... +class G(Generic[T1, T2]): ... + +@overload +def f1(g: G[A, A]) -> A: ... +@overload +def f1(g: G[A, B]) -> B: ... +def f1(g: Any) -> Any: ... + +@overload +def f2(g: G[A, Any]) -> A: ... +@overload +def f2(g: G[A, B], x: int = ...) -> B: ... +def f2(g: Any, x: int = ...) -> Any: ... + +[case testOverloadTypeVsCallable] +from typing import TypeVar, Type, Callable, Any, overload, Optional +class Foo: + def __init__(self, **kwargs: Any): pass +_T = TypeVar('_T') +@overload +def register(cls: Type[_T]) -> int: ... +@overload +def register(cls: Callable[..., _T]) -> Optional[int]: ... +def register(cls: Any) -> Any: return None + + +x = register(Foo) +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + + +[case testOverloadWithObjectDecorator] +from typing import Any, Callable, Union, overload + +class A: + def __call__(self, *arg, **kwargs) -> None: ... + +def dec_a(f: Callable[..., Any]) -> A: + return A() + +@overload +def f_a(arg: int) -> None: ... +@overload +def f_a(arg: str) -> None: ... +@dec_a +def f_a(arg): ... + +class B: + def __call__(self, arg: Union[int, str]) -> None: ... + +def dec_b(f: Callable[..., Any]) -> B: + return B() + +@overload +def f_b(arg: int) -> None: ... +@overload +def f_b(arg: str) -> None: ... +@dec_b +def f_b(arg): ... + +class C: + def __call__(self, arg: int) -> None: ... + +def dec_c(f: Callable[..., Any]) -> C: + return C() + +@overload +def f_c(arg: int) -> None: ... +@overload +def f_c(arg: str) -> None: ... +@dec_c # E: Overloaded function implementation does not accept all possible arguments of signature 2 +def f_c(arg): ... +[builtins fixtures/dict.pyi] + +[case testOverloadWithErrorDecorator] +from typing import Any, Callable, TypeVar, overload + +def dec_d(f: Callable[..., Any]) -> int: ... + +@overload +def f_d(arg: int) -> None: ... +@overload +def f_d(arg: str) -> None: ... +@dec_d # E: "int" not callable +def f_d(arg): ... + +Bad1 = TypeVar('Good') # type: ignore + +def dec_e(f: Bad1) -> Bad1: ... # type: ignore + +@overload +def f_e(arg: int) -> None: ... +@overload +def f_e(arg: str) -> None: ... +@dec_e # E: Bad1? not callable +def f_e(arg): ... + +class Bad2: + def __getattr__(self, attr): + # __getattr__ is not called for implicit `__call__` + if attr == "__call__": + return lambda *a, **kw: print(a, kw) + raise AttributeError + +@overload +def f_f(arg: int) -> None: ... +@overload +def f_f(arg: str) -> None: ... +@Bad2() # E: "Bad2" not callable +def f_f(arg): ... +[builtins fixtures/dict.pyi] + + +[case testOverloadIfBasic] +# flags: --always-true True --always-false False +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +# ----- +# Test basic overload merging +# ----- + +@overload +def f1(g: A) -> A: ... +if True: + @overload + def f1(g: B) -> B: ... +def f1(g): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" + +@overload +def f2(g: A) -> A: ... +@overload +def f2(g: B) -> B: ... +if False: + @overload + def f2(g: C) -> C: ... +def f2(g): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(C())) # E: No overload variant of "f2" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f2(g: A) -> A \ + # N: def f2(g: B) -> B \ + # N: Revealed type is "Any" + +@overload +def f3(g: A) -> A: ... +@overload +def f3(g: B) -> B: ... +if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f3(g: C) -> C: ... +def f3(g): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" +reveal_type(f3(C())) # E: No overload variant of "f3" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f3(g: A) -> A \ + # N: def f3(g: B) -> B \ + # N: Revealed type is "Any" + +if True: + @overload + def f4(g: A) -> A: ... +if True: + @overload + def f4(g: B) -> B: ... +@overload +def f4(g: C) -> C: ... +def f4(g): ... +reveal_type(f4(A())) # N: Revealed type is "__main__.A" +reveal_type(f4(B())) # N: Revealed type is "__main__.B" +reveal_type(f4(C())) # N: Revealed type is "__main__.C" + +if True: + @overload + def f5(g: A) -> A: ... +@overload +def f5(g: B) -> B: ... +if True: + @overload + def f5(g: C) -> C: ... +@overload +def f5(g: D) -> D: ... +def f5(g): ... +reveal_type(f5(A())) # N: Revealed type is "__main__.A" +reveal_type(f5(B())) # N: Revealed type is "__main__.B" +reveal_type(f5(C())) # N: Revealed type is "__main__.C" +reveal_type(f5(D())) # N: Revealed type is "__main__.D" + +[case testOverloadIfSysVersion] +# flags: --python-version 3.9 +from typing import overload +import sys + +class A: ... +class B: ... +class C: ... + +# ----- +# "Real" world example +# Test overload merging for sys.version_info +# ----- + +@overload +def f1(g: A) -> A: ... +if sys.version_info >= (3, 9): + @overload + def f1(g: B) -> B: ... +def f1(g): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" + +@overload +def f2(g: A) -> A: ... +@overload +def f2(g: B) -> B: ... +if sys.version_info >= (3, 10): + @overload + def f2(g: C) -> C: ... +def f2(g): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(C())) # E: No overload variant of "f2" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f2(g: A) -> A \ + # N: def f2(g: B) -> B \ + # N: Revealed type is "Any" +[builtins fixtures/ops.pyi] + +[case testOverloadIfMerging] +# flags: --always-true True +from typing import overload + +class A: ... +class B: ... +class C: ... + +# ----- +# Test overload merging +# ----- + +@overload +def f1(g: A) -> A: ... +if True: + # Some comment + @overload + def f1(g: B) -> B: ... +def f1(g): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" + +@overload +def f2(g: A) -> A: ... +if True: + @overload + def f2(g: bytes) -> B: ... + @overload + def f2(g: B) -> C: ... +def f2(g): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(B())) # N: Revealed type is "__main__.C" + +@overload +def f3(g: A) -> A: ... +@overload +def f3(g: B) -> B: ... +if True: + def f3(g): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" +reveal_type(f3(B())) # N: Revealed type is "__main__.B" + +if True: + @overload + def f4(g: A) -> A: ... +@overload +def f4(g: B) -> B: ... +def f4(g): ... +reveal_type(f4(A())) # N: Revealed type is "__main__.A" +reveal_type(f4(B())) # N: Revealed type is "__main__.B" + +if True: + # Some comment + @overload + def f5(g: A) -> A: ... + @overload + def f5(g: B) -> B: ... +def f5(g): ... +reveal_type(f5(A())) # N: Revealed type is "__main__.A" +reveal_type(f5(B())) # N: Revealed type is "__main__.B" + +[case testOverloadIfNotMerging] +# flags: --always-true True +from typing import overload + +class A: ... +class B: ... +class C: ... + +# ----- +# Don't merge if IfStmt contains nodes other than overloads +# ----- + +@overload # E: An overloaded function outside a stub file must have an implementation +def f1(g: A) -> A: ... +@overload +def f1(g: B) -> B: ... +if True: + @overload # E: Name "f1" already defined on line 12 \ + # E: Single overload definition, multiple required + def f1(g: C) -> C: ... + pass # Some other action +def f1(g): ... # E: Name "f1" already defined on line 12 +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(C())) # E: No overload variant of "f1" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f1(g: A) -> A \ + # N: def f1(g: B) -> B \ + # N: Revealed type is "Any" + +if True: + pass # Some other action + @overload # E: Single overload definition, multiple required + def f2(g: A) -> A: ... +@overload # E: Name "f2" already defined on line 26 +def f2(g: B) -> B: ... +@overload +def f2(g: C) -> C: ... +def f2(g): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(C())) # N: Revealed type is "__main__.A" \ + # E: Argument 1 to "f2" has incompatible type "C"; expected "A" + +[case testOverloadIfOldStyle] +# flags: --always-false var_false --always-true var_true +from typing import overload + +class A: ... +class B: ... + +# ----- +# Test old style to make sure it still works +# ----- + +var_true = True +var_false = False + +if var_false: + @overload + def f1(g: A) -> A: ... + @overload + def f1(g: B) -> B: ... + def f1(g): ... +elif var_true: + @overload + def f1(g: A) -> A: ... + @overload + def f1(g: B) -> B: ... + def f1(g): ... +else: + @overload + def f1(g: A) -> A: ... + @overload + def f1(g: B) -> B: ... + def f1(g): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" + +[case testOverloadIfElse] +# flags: --always-true True --always-false False +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +# ----- +# Match the first always-true block +# ----- + +@overload +def f1(x: A) -> A: ... +if True: + @overload + def f1(x: B) -> B: ... +elif False: + @overload + def f1(x: C) -> C: ... +else: + @overload + def f1(x: D) -> D: ... +def f1(x): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" +reveal_type(f1(C())) # E: No overload variant of "f1" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f1(x: A) -> A \ + # N: def f1(x: B) -> B \ + # N: Revealed type is "Any" + +@overload +def f2(x: A) -> A: ... +if False: + @overload + def f2(x: B) -> B: ... +elif True: + @overload + def f2(x: C) -> C: ... +else: + @overload + def f2(x: D) -> D: ... +def f2(x): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(B())) # E: No overload variant of "f2" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f2(x: A) -> A \ + # N: def f2(x: C) -> C \ + # N: Revealed type is "Any" +reveal_type(f2(C())) # N: Revealed type is "__main__.C" + +@overload +def f3(x: A) -> A: ... +if False: + @overload + def f3(x: B) -> B: ... +elif False: + @overload + def f3(x: C) -> C: ... +else: + @overload + def f3(x: D) -> D: ... +def f3(x): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" +reveal_type(f3(C())) # E: No overload variant of "f3" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f3(x: A) -> A \ + # N: def f3(x: D) -> D \ + # N: Revealed type is "Any" +reveal_type(f3(D())) # N: Revealed type is "__main__.D" + +[case testOverloadIfElse2] +# flags: --always-true True +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +# ----- +# Match the first always-true block +# Don't merge overloads if can't be certain about execution of block +# ----- + +@overload +def f1(x: A) -> A: ... +if True: + @overload + def f1(x: B) -> B: ... +else: + @overload + def f1(x: D) -> D: ... +def f1(x): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" +reveal_type(f1(D())) # E: No overload variant of "f1" matches argument type "D" \ + # N: Possible overload variants: \ + # N: def f1(x: A) -> A \ + # N: def f1(x: B) -> B \ + # N: Revealed type is "Any" + +@overload +def f2(x: A) -> A: ... +if True: + @overload + def f2(x: B) -> B: ... +elif maybe_true: + @overload + def f2(x: C) -> C: ... +else: + @overload + def f2(x: D) -> D: ... +def f2(x): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(B())) # N: Revealed type is "__main__.B" +reveal_type(f2(C())) # E: No overload variant of "f2" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f2(x: A) -> A \ + # N: def f2(x: B) -> B \ + # N: Revealed type is "Any" + +@overload # E: Single overload definition, multiple required +def f3(x: A) -> A: ... +if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f3(x: B) -> B: ... +elif True: + @overload + def f3(x: C) -> C: ... +else: + @overload + def f3(x: D) -> D: ... +def f3(x): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" +reveal_type(f3(B())) # E: No overload variant of "f3" matches argument type "B" \ + # N: Possible overload variant: \ + # N: def f3(x: A) -> A \ + # N: Revealed type is "Any" + +@overload # E: Single overload definition, multiple required +def f4(x: A) -> A: ... +if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f4(x: B) -> B: ... +else: + @overload + def f4(x: D) -> D: ... +def f4(x): ... +reveal_type(f4(A())) # N: Revealed type is "__main__.A" +reveal_type(f4(B())) # E: No overload variant of "f4" matches argument type "B" \ + # N: Possible overload variant: \ + # N: def f4(x: A) -> A \ + # N: Revealed type is "Any" + + +[case testOverloadIfElse3] +# flags: --always-false False +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... +class E: ... + +# ----- +# Match the first always-true block +# Don't merge overloads if can't be certain about execution of block +# ----- + +@overload +def f1(x: A) -> A: ... +if False: + @overload + def f1(x: B) -> B: ... +else: + @overload + def f1(x: D) -> D: ... +def f1(x): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # E: No overload variant of "f1" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f1(x: A) -> A \ + # N: def f1(x: D) -> D \ + # N: Revealed type is "Any" +reveal_type(f1(D())) # N: Revealed type is "__main__.D" + +@overload # E: Single overload definition, multiple required +def f2(x: A) -> A: ... +if False: + @overload + def f2(x: B) -> B: ... +elif maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f2(x: C) -> C: ... +else: + @overload + def f2(x: D) -> D: ... +def f2(x): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(C())) # E: No overload variant of "f2" matches argument type "C" \ + # N: Possible overload variant: \ + # N: def f2(x: A) -> A \ + # N: Revealed type is "Any" + +@overload # E: Single overload definition, multiple required +def f3(x: A) -> A: ... +if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f3(x: B) -> B: ... +elif False: + @overload + def f3(x: C) -> C: ... +else: + @overload + def f3(x: D) -> D: ... +def f3(x): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" +reveal_type(f3(B())) # E: No overload variant of "f3" matches argument type "B" \ + # N: Possible overload variant: \ + # N: def f3(x: A) -> A \ + # N: Revealed type is "Any" + +def g(bool_var: bool) -> None: + @overload + def f4(x: A) -> A: ... + if bool_var: # E: Condition can't be inferred, unable to merge overloads + @overload + def f4(x: B) -> B: ... + elif maybe_true: # E: Name "maybe_true" is not defined + # No 'Condition cannot be inferred' error here since it's already + # emitted on the first condition, 'bool_var', above. + @overload + def f4(x: C) -> C: ... + else: + @overload + def f4(x: D) -> D: ... + @overload + def f4(x: E) -> E: ... + def f4(x): ... + reveal_type(f4(E())) # N: Revealed type is "__main__.E" + reveal_type(f4(B())) # E: No overload variant of "f4" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f4(x: A) -> A \ + # N: def f4(x: E) -> E \ + # N: Revealed type is "Any" + + +[case testOverloadIfSkipUnknownExecution] +# flags: --always-true True +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +# ----- +# If blocks should be skipped if execution can't be certain +# Overload name must match outer name +# ----- + +@overload # E: Single overload definition, multiple required +def f1(x: A) -> A: ... +if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f1(x: B) -> B: ... +def f1(x): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" + +if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f2(x: A) -> A: ... +@overload +def f2(x: B) -> B: ... +@overload +def f2(x: C) -> C: ... +def f2(x): ... +reveal_type(f2(A())) # E: No overload variant of "f2" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f2(x: B) -> B \ + # N: def f2(x: C) -> C \ + # N: Revealed type is "Any" + +if True: + @overload # E: Single overload definition, multiple required + def f3(x: A) -> A: ... + if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f3(x: B) -> B: ... + def f3(x): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" + +if True: + if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f4(x: A) -> A: ... + @overload + def f4(x: B) -> B: ... + @overload + def f4(x: C) -> C: ... + def f4(x): ... +reveal_type(f4(A())) # E: No overload variant of "f4" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f4(x: B) -> B \ + # N: def f4(x: C) -> C \ + # N: Revealed type is "Any" + +[case testOverloadIfDontSkipUnrelatedOverload] +# flags: --always-true True +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +# ----- +# Don't skip if block if overload name doesn't match outer name +# ----- + +@overload # E: Single overload definition, multiple required +def f1(x: A) -> A: ... +if maybe_true: # E: Name "maybe_true" is not defined + @overload # E: Single overload definition, multiple required + def g1(x: B) -> B: ... +def f1(x): ... # E: Name "f1" already defined on line 13 +reveal_type(f1(A())) # N: Revealed type is "__main__.A" + +if maybe_true: # E: Name "maybe_true" is not defined + @overload # E: Single overload definition, multiple required + def g2(x: A) -> A: ... +@overload +def f2(x: B) -> B: ... +@overload +def f2(x: C) -> C: ... +def f2(x): ... +reveal_type(f2(A())) # E: No overload variant of "f2" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f2(x: B) -> B \ + # N: def f2(x: C) -> C \ + # N: Revealed type is "Any" + +if True: + @overload # E: Single overload definition, multiple required + def f3(x: A) -> A: ... + def f3(x): ... + if maybe_true: # E: Name "maybe_true" is not defined + @overload # E: Single overload definition, multiple required + def g3(x: B) -> B: ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" + +if True: + if maybe_true: # E: Name "maybe_true" is not defined + @overload # E: Single overload definition, multiple required + def g4(x: A) -> A: ... + @overload + def f4(x: B) -> B: ... + @overload + def f4(x: C) -> C: ... + def f4(x): ... +reveal_type(f4(A())) # E: No overload variant of "f4" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f4(x: B) -> B \ + # N: def f4(x: C) -> C \ + # N: Revealed type is "Any" + +[case testOverloadIfNotMergingDifferentNames] +# flags: --always-true True +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +# ----- +# Don't merge overloads if IfStmts contains overload with different name +# ----- + +@overload # E: An overloaded function outside a stub file must have an implementation +def f1(x: A) -> A: ... +@overload +def f1(x: B) -> B: ... +if True: + @overload # E: Single overload definition, multiple required + def g1(x: C) -> C: ... +def f1(x): ... # E: Name "f1" already defined on line 13 +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(C())) # E: No overload variant of "f1" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f1(x: A) -> A \ + # N: def f1(x: B) -> B \ + # N: Revealed type is "Any" + +if True: + @overload # E: Single overload definition, multiple required + def g2(x: A) -> A: ... +@overload +def f2(x: B) -> B: ... +@overload +def f2(x: C) -> C: ... +def f2(x): ... +reveal_type(f2(A())) # E: No overload variant of "f2" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f2(x: B) -> B \ + # N: def f2(x: C) -> C \ + # N: Revealed type is "Any" +reveal_type(f2(B())) # N: Revealed type is "__main__.B" + +if True: + if True: + @overload # E: Single overload definition, multiple required + def g3(x: A) -> A: ... + @overload + def f3(x: B) -> B: ... + @overload + def f3(x: C) -> C: ... + def f3(x): ... +reveal_type(f3(A())) # E: No overload variant of "f3" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f3(x: B) -> B \ + # N: def f3(x: C) -> C \ + # N: Revealed type is "Any" +reveal_type(f3(B())) # N: Revealed type is "__main__.B" + +[case testOverloadIfSplitFunctionDef] +# flags: --always-true True --always-false False +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +# ----- +# Test split FuncDefs +# ----- + +@overload +def f1(x: A) -> A: ... +@overload +def f1(x: B) -> B: ... +if True: + def f1(x): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" + +@overload +def f2(x: A) -> A: ... +@overload +def f2(x: B) -> B: ... +if False: + def f2(x): ... +else: + def f2(x): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" + +@overload # E: An overloaded function outside a stub file must have an implementation +def f3(x: A) -> A: ... +@overload +def f3(x: B) -> B: ... +if True: + def f3(x): ... # E: Name "f3" already defined on line 31 +else: + pass # some other node + def f3(x): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" + +[case testOverloadIfMixed] +# flags: --always-true True --always-false False +from typing import overload, TYPE_CHECKING + +class A: ... +class B: ... +class C: ... +class D: ... + +if maybe_var: # E: Name "maybe_var" is not defined + pass +if True: + @overload + def f1(x: A) -> A: ... +@overload +def f1(x: B) -> B: ... +def f1(x): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" + +if True: + @overload + def f2(x: A) -> A: ... + @overload + def f2(x: B) -> B: ... +def f2(x): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(B())) # N: Revealed type is "__main__.B" + +if True: + @overload + def f3(x: A) -> A: ... + @overload + def f3(x: B) -> B: ... + def f3(x): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" +reveal_type(f3(B())) # N: Revealed type is "__main__.B" + +# Don't crash with AssignmentStmt if elif +@overload # E: Single overload definition, multiple required +def f4(x: A) -> A: ... +if False: + @overload + def f4(x: B) -> B: ... +elif True: + var = 1 +def f4(x): ... # E: Name "f4" already defined on line 39 + +if TYPE_CHECKING: + @overload + def f5(x: A) -> A: ... + @overload + def f5(x: B) -> B: ... +def f5(x): ... +reveal_type(f5(A())) # N: Revealed type is "__main__.A" +reveal_type(f5(B())) # N: Revealed type is "__main__.B" + +# Test from check-functions - testUnconditionalRedefinitionOfConditionalFunction +# Don't merge If blocks if they appear before any overloads +# and don't contain any overloads themselves. +if maybe_true: # E: Name "maybe_true" is not defined + def f6(x): ... +def f6(x): ... # E: Name "f6" already defined on line 61 + +if maybe_true: # E: Name "maybe_true" is not defined + pass # Some other node + def f7(x): ... +def f7(x): ... # E: Name "f7" already defined on line 66 + +@overload +def f8(x: A) -> A: ... +@overload +def f8(x: B) -> B: ... +if False: + def f8(x: C) -> C: ... +def f8(x): ... +reveal_type(f8(A())) # N: Revealed type is "__main__.A" +reveal_type(f8(C())) # E: No overload variant of "f8" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f8(x: A) -> A \ + # N: def f8(x: B) -> B \ + # N: Revealed type is "Any" + +if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f9(x: A) -> A: ... +if another_maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "another_maybe_true" is not defined + @overload + def f9(x: B) -> B: ... +@overload +def f9(x: C) -> C: ... +@overload +def f9(x: D) -> D: ... +def f9(x): ... +reveal_type(f9(A())) # E: No overload variant of "f9" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f9(x: C) -> C \ + # N: def f9(x: D) -> D \ + # N: Revealed type is "Any" +reveal_type(f9(C())) # N: Revealed type is "__main__.C" + +if True: + if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f10(x: A) -> A: ... + if another_maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "another_maybe_true" is not defined + @overload + def f10(x: B) -> B: ... + @overload + def f10(x: C) -> C: ... + @overload + def f10(x: D) -> D: ... + def f10(x): ... +reveal_type(f10(A())) # E: No overload variant of "f10" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f10(x: C) -> C \ + # N: def f10(x: D) -> D \ + # N: Revealed type is "Any" +reveal_type(f10(C())) # N: Revealed type is "__main__.C" + +if some_var: # E: Name "some_var" is not defined + pass +@overload +def f11(x: A) -> A: ... +@overload +def f11(x: B) -> B: ... +def f11(x): ... +reveal_type(f11(A())) # N: Revealed type is "__main__.A" + +if True: + if some_var: # E: Name "some_var" is not defined + pass + @overload + def f12(x: A) -> A: ... + @overload + def f12(x: B) -> B: ... + def f12(x): ... +reveal_type(f12(A())) # N: Revealed type is "__main__.A" + +[typing fixtures/typing-medium.pyi] + +[case testOverloadIfUnconditionalFuncDef] +# flags: --always-true True --always-false False +from typing import overload + +class A: ... +class B: ... + +# ----- +# Don't merge conditional FuncDef after unconditional one +# ----- + +@overload +def f1(x: A) -> A: ... +@overload +def f1(x: B) -> B: ... +def f1(x): ... + +@overload +def f2(x: A) -> A: ... +if True: + @overload + def f2(x: B) -> B: ... +def f2(x): ... +if True: + def f2(x): ... # E: Name "f2" already defined on line 17 + +[case testOverloadItemHasMoreGeneralReturnType] +from typing import overload + +@overload +def f() -> object: ... + +@overload +def f(x: int) -> object: ... + +def f(x: int = 0) -> int: + return x + +@overload +def g() -> object: ... + +@overload +def g(x: int) -> str: ... + +def g(x: int = 0) -> int: # E: Overloaded function implementation cannot produce return type of signature 2 + return x + +[case testOverloadIfNestedOk] +# flags: --always-true True --always-false False +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def f1(g: A) -> A: ... +if True: + @overload + def f1(g: B) -> B: ... + if True: + @overload + def f1(g: C) -> C: ... + @overload + def f1(g: D) -> D: ... +def f1(g): ... +reveal_type(f1(A())) # N: Revealed type is "__main__.A" +reveal_type(f1(B())) # N: Revealed type is "__main__.B" +reveal_type(f1(C())) # N: Revealed type is "__main__.C" +reveal_type(f1(D())) # N: Revealed type is "__main__.D" + +@overload +def f2(g: A) -> A: ... +if True: + @overload + def f2(g: B) -> B: ... + if True: + @overload + def f2(g: C) -> C: ... + if True: + @overload + def f2(g: D) -> D: ... +def f2(g): ... +reveal_type(f2(A())) # N: Revealed type is "__main__.A" +reveal_type(f2(B())) # N: Revealed type is "__main__.B" +reveal_type(f2(C())) # N: Revealed type is "__main__.C" +reveal_type(f2(D())) # N: Revealed type is "__main__.D" + +@overload +def f3(g: A) -> A: ... +if True: + if True: + @overload + def f3(g: B) -> B: ... + if True: + @overload + def f3(g: C) -> C: ... +def f3(g): ... +reveal_type(f3(A())) # N: Revealed type is "__main__.A" +reveal_type(f3(B())) # N: Revealed type is "__main__.B" +reveal_type(f3(C())) # N: Revealed type is "__main__.C" + +@overload +def f4(g: A) -> A: ... +if True: + if False: + @overload + def f4(g: B) -> B: ... + else: + @overload + def f4(g: C) -> C: ... +def f4(g): ... +reveal_type(f4(A())) # N: Revealed type is "__main__.A" +reveal_type(f4(B())) # E: No overload variant of "f4" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f4(g: A) -> A \ + # N: def f4(g: C) -> C \ + # N: Revealed type is "Any" +reveal_type(f4(C())) # N: Revealed type is "__main__.C" + +@overload +def f5(g: A) -> A: ... +if True: + if False: + @overload + def f5(g: B) -> B: ... + elif True: + @overload + def f5(g: C) -> C: ... +def f5(g): ... +reveal_type(f5(A())) # N: Revealed type is "__main__.A" +reveal_type(f5(B())) # E: No overload variant of "f5" matches argument type "B" \ + # N: Possible overload variants: \ + # N: def f5(g: A) -> A \ + # N: def f5(g: C) -> C \ + # N: Revealed type is "Any" +reveal_type(f5(C())) # N: Revealed type is "__main__.C" + +[case testOverloadIfNestedFailure] +# flags: --always-true True --always-false False +from typing import overload + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload # E: Single overload definition, multiple required +def f1(g: A) -> A: ... +if True: + @overload # E: Single overload definition, multiple required + def f1(g: B) -> B: ... # E: Incompatible redefinition (redefinition with type "Callable[[B], B]", original type "Callable[[A], A]") + if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ + # E: Name "maybe_true" is not defined + @overload + def f1(g: C) -> C: ... + @overload + def f1(g: D) -> D: ... +def f1(g): ... # E: Name "f1" already defined on line 9 + +@overload # E: Single overload definition, multiple required +def f2(g: A) -> A: ... +if True: + if False: + @overload + def f2(g: B) -> B: ... + elif maybe_true: # E: Name "maybe_true" is not defined + @overload # E: Single overload definition, multiple required + def f2(g: C) -> C: ... # E: Incompatible redefinition (redefinition with type "Callable[[C], C]", original type "Callable[[A], A]") +def f2(g): ... # E: Name "f2" already defined on line 21 + +@overload # E: Single overload definition, multiple required +def f3(g: A) -> A: ... +if True: + @overload # E: Single overload definition, multiple required + def f3(g: B) -> B: ... # E: Incompatible redefinition (redefinition with type "Callable[[B], B]", original type "Callable[[A], A]") + if True: + pass # Some other node + @overload # E: Name "f3" already defined on line 32 \ + # E: An overloaded function outside a stub file must have an implementation + def f3(g: C) -> C: ... + @overload + def f3(g: D) -> D: ... +def f3(g): ... # E: Name "f3" already defined on line 32 + +[case testOverloadingWithParamSpec] +from typing import TypeVar, Callable, Any, overload +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +R = TypeVar("R") + +@overload +def func(x: Callable[Concatenate[Any, P], R]) -> Callable[P, R]: ... +@overload +def func(x: Callable[P, R]) -> Callable[Concatenate[str, P], R]: ... +def func(x: Callable[..., R]) -> Callable[..., R]: ... + +def foo(arg1: str, arg2: int) -> bytes: ... +reveal_type(func(foo)) # N: Revealed type is "def (arg2: builtins.int) -> builtins.bytes" + +def bar() -> int: ... +reveal_type(func(bar)) # N: Revealed type is "def (builtins.str) -> builtins.int" + +baz: Callable[[str, str], str] = lambda x, y: 'baz' +reveal_type(func(baz)) # N: Revealed type is "def (builtins.str) -> builtins.str" + +eggs = lambda: 'eggs' +reveal_type(func(eggs)) # N: Revealed type is "def (builtins.str) -> builtins.str" + +spam: Callable[..., str] = lambda x, y: 'baz' +reveal_type(func(spam)) # N: Revealed type is "def (*Any, **Any) -> Any" +[builtins fixtures/paramspec.pyi] + +[case testGenericOverloadOverlapWithType] +import m + +[file m.pyi] +from typing import TypeVar, Type, overload, Callable + +T = TypeVar("T", bound=str) +@overload +def foo(x: Type[T] | int) -> int: ... +@overload +def foo(x: Callable[[int], bool]) -> str: ... + +[case testGenericOverloadOverlapWithCollection] +import m + +[file m.pyi] +from typing import TypeVar, Sequence, overload, List + +T = TypeVar("T", bound=str) + +@overload +def foo(x: List[T]) -> str: ... +@overload +def foo(x: Sequence[int]) -> int: ... +[builtins fixtures/list.pyi] + +# Also see `check-python38.test` for similar tests with `/` args: +[case testOverloadPositionalOnlyErrorMessageOldStyle] +from typing import overload + +@overload +def foo(__a: int): ... +@overload +def foo(a: str): ... +def foo(a): ... + +foo(a=1) +[out] +main:9: error: No overload variant of "foo" matches argument type "int" +main:9: note: Possible overload variants: +main:9: note: def foo(int, /) -> Any +main:9: note: def foo(a: str) -> Any + +[case testOverloadUnionGenericBounds] +from typing import overload, TypeVar, Sequence, Union + +class Entity: ... +class Assoc: ... + +E = TypeVar("E", bound=Entity) +A = TypeVar("A", bound=Assoc) + +class Test: + @overload + def foo(self, arg: Sequence[E]) -> None: ... + @overload + def foo(self, arg: Sequence[A]) -> None: ... + def foo(self, arg: Union[Sequence[E], Sequence[A]]) -> None: + ... + +[case testOverloadedStaticMethodOnInstance] +from typing import overload + +class Snafu(object): + @overload + @staticmethod + def snafu(value: bytes) -> bytes: ... + @overload + @staticmethod + def snafu(value: str) -> str: ... + @staticmethod + def snafu(value): + ... +reveal_type(Snafu().snafu('123')) # N: Revealed type is "builtins.str" +reveal_type(Snafu.snafu('123')) # N: Revealed type is "builtins.str" +[builtins fixtures/staticmethod.pyi] + +[case testOverloadedWithInternalTypeVars] +# flags: --new-type-inference +import m + +[file m.pyi] +from typing import Callable, TypeVar, overload + +T = TypeVar("T") +S = TypeVar("S", bound=str) + +@overload +def foo(x: int = ...) -> Callable[[T], T]: ... +@overload +def foo(x: S = ...) -> Callable[[T], T]: ... + +[case testOverloadGenericStarArgOverlap] +from typing import Any, Callable, TypeVar, overload, Union, Tuple, List + +F = TypeVar("F", bound=Callable[..., Any]) +S = TypeVar("S", bound=int) + +def id(f: F) -> F: ... + +@overload +def struct(*cols: S) -> int: ... +@overload +def struct(__cols: Union[List[S], Tuple[S, ...]]) -> int: ... +@id +def struct(*cols: Union[S, Union[List[S], Tuple[S, ...]]]) -> int: + pass +[builtins fixtures/tuple.pyi] + +[case testRegularGenericDecoratorOverload] +from typing import Callable, overload, TypeVar, List + +S = TypeVar("S") +T = TypeVar("T") +def transform(func: Callable[[S], List[T]]) -> Callable[[S], T]: ... + +@overload +def foo(x: int) -> List[float]: ... +@overload +def foo(x: str) -> List[str]: ... +def foo(x): ... + +reveal_type(transform(foo)) # N: Revealed type is "Overload(def (builtins.int) -> builtins.float, def (builtins.str) -> builtins.str)" + +@transform +@overload +def bar(x: int) -> List[float]: ... +@transform +@overload +def bar(x: str) -> List[str]: ... +@transform +def bar(x): ... + +reveal_type(bar) # N: Revealed type is "Overload(def (builtins.int) -> builtins.float, def (builtins.str) -> builtins.str)" +[builtins fixtures/paramspec.pyi] + +[case testOverloadOverlapWithNameOnlyArgs] +from typing import overload + +@overload +def d(x: int) -> int: ... +@overload +def d(f: int, *, x: int) -> str: ... +def d(*args, **kwargs): ... +[builtins fixtures/tuple.pyi] + +[case testOverloadCallableGenericSelf] +from typing import Any, TypeVar, Generic, overload, reveal_type + +T = TypeVar("T") + +class MyCallable(Generic[T]): + def __init__(self, t: T): + self.t = t + + @overload + def __call__(self: "MyCallable[int]") -> str: ... + @overload + def __call__(self: "MyCallable[str]") -> int: ... + def __call__(self): ... + +c = MyCallable(5) +reveal_type(c) # N: Revealed type is "__main__.MyCallable[builtins.int]" +reveal_type(c()) # N: Revealed type is "builtins.str" + +c2 = MyCallable("test") +reveal_type(c2) # N: Revealed type is "__main__.MyCallable[builtins.str]" +reveal_type(c2()) # should be int # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testOverloadWithStarAnyFallback] +from typing import overload, Any + +class A: + @overload + def f(self, e: str) -> str: ... + @overload + def f(self, *args: Any, **kwargs: Any) -> Any: ... + def f(self, *args, **kwargs): + pass + +class B: + @overload + def f(self, e: str, **kwargs: Any) -> str: ... + @overload + def f(self, *args: Any, **kwargs: Any) -> Any: ... + def f(self, *args, **kwargs): + pass +[builtins fixtures/tuple.pyi] + +[case testOverloadsSafeOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload + +@overload +def bar(x: object) -> object: ... +@overload +def bar(x: int = ...) -> int: ... + +[case testOverloadsInvariantOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload, List + +@overload +def bar(x: List[int]) -> List[int]: ... +@overload +def bar(x: List[object]) -> List[object]: ... + +[case testOverloadsNoneAnyOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload, Any + +@overload +def foo(x: None) -> int: ... +@overload +def foo(x: object) -> str: ... + +@overload +def bar(x: int) -> int: ... +@overload +def bar(x: Any) -> str: ... + +[case testOverloadOnInvalidTypeArgument] +from typing import TypeVar, Self, Generic, overload + +class C: pass + +T = TypeVar("T", bound=C) + +class D(Generic[T]): + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... + def f(Self, x): ... + +a: D[str] # E: Type argument "str" of "D" must be a subtype of "C" +reveal_type(a.f(1)) # N: Revealed type is "builtins.int" +reveal_type(a.f("x")) # N: Revealed type is "builtins.str" diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test new file mode 100644 index 000000000000..7f038b811741 --- /dev/null +++ b/test-data/unit/check-parameter-specification.test @@ -0,0 +1,2475 @@ +[case testBasicParamSpec] +from typing_extensions import ParamSpec +P = ParamSpec('P') +[builtins fixtures/tuple.pyi] + +[case testInvalidParamSpecDefinitions] +from typing import ParamSpec + +P1 = ParamSpec("P1", covariant=True) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet +P2 = ParamSpec("P2", contravariant=True) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet +P3 = ParamSpec("P3", bound=int) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet +P4 = ParamSpec("P4", int, str) # E: Too many positional arguments for "ParamSpec" +P5 = ParamSpec("P5", covariant=True, bound=int) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet +[builtins fixtures/paramspec.pyi] + +[case testParamSpecLocations] +from typing import Callable, List +from typing_extensions import ParamSpec, Concatenate +P = ParamSpec('P') + +x: P # E: ParamSpec "P" is unbound + +def foo1(x: Callable[P, int]) -> Callable[P, str]: ... + +def foo2(x: P) -> P: ... # E: Invalid location for ParamSpec "P" \ + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" + +def foo3(x: Concatenate[int, P]) -> int: ... # E: Invalid location for Concatenate \ + # N: You can use Concatenate as the first argument to Callable + +def foo4(x: List[P]) -> None: ... # E: Invalid location for ParamSpec "P" \ + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" + +def foo5(x: Callable[[int, str], P]) -> None: ... # E: Invalid location for ParamSpec "P" \ + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" + +def foo6(x: Callable[[P], int]) -> None: ... # E: Invalid location for ParamSpec "P" \ + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecImports] +import lib +from lib import Base + +class C(Base[[int]]): + def test(self, x: int): ... + +class D(lib.Base[[int]]): + def test(self, x: int): ... + +class E(lib.Base[...]): ... +reveal_type(E().test) # N: Revealed type is "def (*Any, **Any)" + +[file lib.py] +from typing import Generic +from typing_extensions import ParamSpec + +P = ParamSpec("P") +class Base(Generic[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + ... +[builtins fixtures/paramspec.pyi] + +[case testParamSpecEllipsisInAliases] +from typing import Any, Callable, Generic, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec('P') +R = TypeVar('R') +Alias = Callable[P, R] + +class B(Generic[P]): ... +Other = B[P] + +T = TypeVar('T', bound=Alias[..., Any]) +Alias[..., Any] # E: Type application is only supported for generic classes +B[...] +Other[...] +[builtins fixtures/paramspec.pyi] + +[case testParamSpecEllipsisInConcatenate] +from typing import Any, Callable, Generic, TypeVar +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec('P') +R = TypeVar('R') +Alias = Callable[P, R] + +IntFun = Callable[Concatenate[int, ...], None] +f: IntFun +reveal_type(f) # N: Revealed type is "def (builtins.int, *Any, **Any)" + +g: Callable[Concatenate[int, ...], None] +reveal_type(g) # N: Revealed type is "def (builtins.int, *Any, **Any)" + +class B(Generic[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + ... + +x: B[Concatenate[int, ...]] +reveal_type(x.test) # N: Revealed type is "def (builtins.int, *Any, **Any)" + +Bad = Callable[Concatenate[int, [int, str]], None] # E: The last parameter to Concatenate needs to be a ParamSpec \ + # E: Bracketed expression "[...]" is not valid as a type +def bad(fn: Callable[Concatenate[P, int], None]): # E: The last parameter to Concatenate needs to be a ParamSpec + ... +[builtins fixtures/paramspec.pyi] + +[case testParamSpecContextManagerLike] +from typing import Callable, List, Iterator, TypeVar +from typing_extensions import ParamSpec +P = ParamSpec('P') +T = TypeVar('T') + +def tmpcontextmanagerlike(x: Callable[P, Iterator[T]]) -> Callable[P, List[T]]: ... + +@tmpcontextmanagerlike +def whatever(x: int) -> Iterator[int]: + yield x + +reveal_type(whatever) # N: Revealed type is "def (x: builtins.int) -> builtins.list[builtins.int]" +reveal_type(whatever(217)) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/paramspec.pyi] + +[case testInvalidParamSpecType] +# flags: --python-version 3.10 +from typing import ParamSpec + +P = ParamSpec("P") + +class MyFunction(P): # E: Invalid base class "P" + ... + +[case testParamSpecRevealType] +from typing import Callable +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +def f(x: Callable[P, int]) -> None: ... +reveal_type(f) # N: Revealed type is "def [P] (x: def (*P.args, **P.kwargs) -> builtins.int)" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecSimpleFunction] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +def changes_return_type_to_str(x: Callable[P, int]) -> Callable[P, str]: ... + +def returns_int(a: str, b: bool) -> int: ... + +reveal_type(changes_return_type_to_str(returns_int)) # N: Revealed type is "def (a: builtins.str, b: builtins.bool) -> builtins.str" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecSimpleClass] +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +class C(Generic[P]): + def __init__(self, x: Callable[P, None]) -> None: ... + + def m(self, *args: P.args, **kwargs: P.kwargs) -> int: + return 1 + +def f(x: int, y: str) -> None: ... + +reveal_type(C(f)) # N: Revealed type is "__main__.C[[x: builtins.int, y: builtins.str]]" +reveal_type(C(f).m) # N: Revealed type is "def (x: builtins.int, y: builtins.str) -> builtins.int" +[builtins fixtures/dict.pyi] + +[case testParamSpecClassWithPrefixArgument] +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +class C(Generic[P]): + def __init__(self, x: Callable[P, None]) -> None: ... + + def m(self, a: str, *args: P.args, **kwargs: P.kwargs) -> int: + return 1 + +def f(x: int, y: str) -> None: ... + +reveal_type(C(f).m) # N: Revealed type is "def (a: builtins.str, x: builtins.int, y: builtins.str) -> builtins.int" +reveal_type(C(f).m('', 1, '')) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testParamSpecDecorator] +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +P = ParamSpec('P') +R = TypeVar('R') + +class W(Generic[P, R]): + f: Callable[P, R] + x: int + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: + reveal_type(self.f(*args, **kwargs)) # N: Revealed type is "R`2" + return self.f(*args, **kwargs) + +def dec() -> Callable[[Callable[P, R]], W[P, R]]: + pass + +@dec() +def f(a: int, b: str) -> None: ... + +reveal_type(f) # N: Revealed type is "__main__.W[[a: builtins.int, b: builtins.str], None]" +reveal_type(f(1, '')) # N: Revealed type is "None" +reveal_type(f.x) # N: Revealed type is "builtins.int" + +## TODO: How should this work? +# +# class C: +# @dec() +# def m(self, x: int) -> str: ... +# +# reveal_type(C().m(x=1)) +[builtins fixtures/dict.pyi] + +[case testParamSpecFunction] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec('P') +R = TypeVar('R') + +def f(x: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: + return x(*args, **kwargs) + +def g(x: int, y: str) -> None: ... + +reveal_type(f(g, 1, y='x')) # N: Revealed type is "None" +f(g, 'x', y='x') # E: Argument 2 to "f" has incompatible type "str"; expected "int" +f(g, 1, y=1) # E: Argument "y" to "f" has incompatible type "int"; expected "str" +f(g) # E: Missing positional arguments "x", "y" in call to "f" +[builtins fixtures/dict.pyi] + +[case testParamSpecSpecialCase] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec('P') +T = TypeVar('T') + +def register(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> Callable[P, T]: ... + +def f(x: int, y: str, z: int, a: str) -> None: ... + +x = register(f, 1, '', 1, '') +[builtins fixtures/dict.pyi] + +[case testParamSpecInferredFromAny] +from typing import Callable, Any +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +def f(x: Callable[P, int]) -> Callable[P, str]: ... + +g: Any +reveal_type(f(g)) # N: Revealed type is "def (*Any, **Any) -> builtins.str" + +f(g)(1, 3, x=1, y=2) +[builtins fixtures/paramspec.pyi] + +[case testParamSpecDecoratorImplementation] +from typing import Callable, Any, TypeVar, List +from typing_extensions import ParamSpec + +P = ParamSpec('P') +T = TypeVar('T') + +def dec(f: Callable[P, T]) -> Callable[P, List[T]]: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> List[T]: + return [f(*args, **kwargs)] + return wrapper + +@dec +def g(x: int, y: str = '') -> int: ... + +reveal_type(g) # N: Revealed type is "def (x: builtins.int, y: builtins.str =) -> builtins.list[builtins.int]" +[builtins fixtures/dict.pyi] + +[case testParamSpecArgsAndKwargsTypes] +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +class C(Generic[P]): + def __init__(self, x: Callable[P, None]) -> None: ... + + def m(self, *args: P.args, **kwargs: P.kwargs) -> None: + reveal_type(args) # N: Revealed type is "P.args`1" + reveal_type(kwargs) # N: Revealed type is "P.kwargs`1" +[builtins fixtures/dict.pyi] + +[case testParamSpecSubtypeChecking1] +from typing import Callable, TypeVar, Generic, Any +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +class C(Generic[P]): + def __init__(self, x: Callable[P, None]) -> None: ... + + def m(self, *args: P.args, **kwargs: P.kwargs) -> None: + args = args + kwargs = kwargs + o: object + o = args + o = kwargs + o2: object + args = o2 # E: Incompatible types in assignment (expression has type "object", variable has type "P.args") + kwargs = o2 # E: Incompatible types in assignment (expression has type "object", variable has type "P.kwargs") + a: Any + a = args + a = kwargs + args = kwargs # E: Incompatible types in assignment (expression has type "P.kwargs", variable has type "P.args") + kwargs = args # E: Incompatible types in assignment (expression has type "P.args", variable has type "P.kwargs") + args = a + kwargs = a +[builtins fixtures/dict.pyi] + +[case testParamSpecSubtypeChecking2] +from typing import Callable, Generic +from typing_extensions import ParamSpec + +P = ParamSpec('P') +P2 = ParamSpec('P2') + +class C(Generic[P]): + pass + +def f(c1: C[P], c2: C[P2]) -> None: + c1 = c1 + c2 = c2 + c1 = c2 # E: Incompatible types in assignment (expression has type "C[P2]", variable has type "C[P]") + c2 = c1 # E: Incompatible types in assignment (expression has type "C[P]", variable has type "C[P2]") + +def g(f: Callable[P, None], g: Callable[P2, None]) -> None: + f = f + g = g + f = g # E: Incompatible types in assignment (expression has type "Callable[P2, None]", variable has type "Callable[P, None]") + g = f # E: Incompatible types in assignment (expression has type "Callable[P, None]", variable has type "Callable[P2, None]") +[builtins fixtures/dict.pyi] + +[case testParamSpecJoin] +from typing import Callable, Generic, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec('P') +P2 = ParamSpec('P2') +P3 = ParamSpec('P3') +T = TypeVar('T') + +def join(x: T, y: T) -> T: ... + +class C(Generic[P, P2]): + def m(self, f: Callable[P, None], g: Callable[P2, None]) -> None: + reveal_type(join(f, f)) # N: Revealed type is "def (*P.args, **P.kwargs)" + reveal_type(join(f, g)) # N: Revealed type is "builtins.function" + + def m2(self, *args: P.args, **kwargs: P.kwargs) -> None: + reveal_type(join(args, args)) # N: Revealed type is "P.args`1" + reveal_type(join(kwargs, kwargs)) # N: Revealed type is "P.kwargs`1" + reveal_type(join(args, kwargs)) # N: Revealed type is "builtins.object" + def f(*args2: P2.args, **kwargs2: P2.kwargs) -> None: + reveal_type(join(args, args2)) # N: Revealed type is "builtins.object" + reveal_type(join(kwargs, kwargs2)) # N: Revealed type is "builtins.object" + + def m3(self, c: C[P, P3]) -> None: + reveal_type(join(c, c)) # N: Revealed type is "__main__.C[P`1, P3`-1]" + reveal_type(join(self, c)) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] + +[case testParamSpecClassWithAny] +from typing import Callable, Generic, Any +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +class C(Generic[P]): + def __init__(self, x: Callable[P, None]) -> None: ... + + def m(self, *args: P.args, **kwargs: P.kwargs) -> int: + return 1 + +c: C[Any] +reveal_type(c) # N: Revealed type is "__main__.C[Any]" +reveal_type(c.m) # N: Revealed type is "def (*args: Any, **kwargs: Any) -> builtins.int" +c.m(4, 6, y='x') +c = c + +def f() -> None: pass + +c2 = C(f) +c2 = c +c3 = C(f) +c = c3 +[builtins fixtures/dict.pyi] + +[case testParamSpecInferredFromLambda] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec('P') +T = TypeVar('T') + +# Similar to atexit.register +def register(f: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> Callable[P, T]: ... + +def f(x: int) -> None: pass +def g(x: int, y: str) -> None: pass + +reveal_type(register(lambda: f(1))) # N: Revealed type is "def ()" +reveal_type(register(lambda x: f(x), x=1)) # N: Revealed type is "def (x: Literal[1]?)" +register(lambda x: f(x)) # E: Cannot infer type of lambda \ + # E: Argument 1 to "register" has incompatible type "Callable[[Any], None]"; expected "Callable[[], None]" +register(lambda x: f(x), y=1) # E: Argument 1 to "register" has incompatible type "Callable[[Arg(int, 'x')], None]"; expected "Callable[[Arg(int, 'y')], None]" +reveal_type(register(lambda x: f(x), 1)) # N: Revealed type is "def (Literal[1]?)" +reveal_type(register(lambda x, y: g(x, y), 1, "a")) # N: Revealed type is "def (Literal[1]?, Literal['a']?)" +reveal_type(register(lambda x, y: g(x, y), 1, y="a")) # N: Revealed type is "def (Literal[1]?, y: Literal['a']?)" +[builtins fixtures/dict.pyi] + +[case testParamSpecInvalidCalls] +from typing import Callable, Generic +from typing_extensions import ParamSpec + +P = ParamSpec('P') +P2 = ParamSpec('P2') + +class C(Generic[P, P2]): + def m1(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.m1(*args, **kwargs) + self.m2(*args, **kwargs) # E: Argument 1 to "m2" of "C" has incompatible type "*P.args"; expected "P2.args" \ + # E: Argument 2 to "m2" of "C" has incompatible type "**P.kwargs"; expected "P2.kwargs" + self.m1(*kwargs, **args) # E: Argument 1 to "m1" of "C" has incompatible type "*P.kwargs"; expected "P.args" \ + # E: Argument 2 to "m1" of "C" has incompatible type "**P.args"; expected "P.kwargs" + self.m3(*args, **kwargs) # E: Argument 1 to "m3" of "C" has incompatible type "*P.args"; expected "int" \ + # E: Argument 2 to "m3" of "C" has incompatible type "**P.kwargs"; expected "int" + self.m4(*args, **kwargs) # E: Argument 1 to "m4" of "C" has incompatible type "*P.args"; expected "int" \ + # E: Argument 2 to "m4" of "C" has incompatible type "**P.kwargs"; expected "int" + + self.m1(*args, **args) # E: Argument 2 to "m1" of "C" has incompatible type "**P.args"; expected "P.kwargs" + self.m1(*kwargs, **kwargs) # E: Argument 1 to "m1" of "C" has incompatible type "*P.kwargs"; expected "P.args" + + def m2(self, *args: P2.args, **kwargs: P2.kwargs) -> None: + pass + + def m3(self, *args: int, **kwargs: int) -> None: + pass + + def m4(self, x: int) -> None: + pass +[builtins fixtures/dict.pyi] + +[case testParamSpecOverUnannotatedDecorator] +from typing import Callable, Iterator, TypeVar, ContextManager, Any +from typing_extensions import ParamSpec + +from nonexistent import deco2 # type: ignore + +T = TypeVar("T") +P = ParamSpec("P") +T_co = TypeVar("T_co", covariant=True) + +class CM(ContextManager[T_co]): + def __call__(self, func: T) -> T: ... + +def deco1( + func: Callable[P, Iterator[T]]) -> Callable[P, CM[T]]: ... + +@deco1 +@deco2 +def f(): + pass + +reveal_type(f) # N: Revealed type is "def (*Any, **Any) -> __main__.CM[Any]" + +with f() as x: + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testParamSpecLiterals] +from typing_extensions import ParamSpec, TypeAlias +from typing import Generic, TypeVar + +P = ParamSpec("P") +T = TypeVar("T") + +class Z(Generic[P]): ... + +# literals can be applied +n: Z[[int]] + +nt1 = Z[[int]] +nt2: TypeAlias = Z[[int]] + +unt1: nt1 +unt2: nt2 + +# literals actually keep types +reveal_type(n) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(unt1) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(unt2) # N: Revealed type is "__main__.Z[[builtins.int]]" + +# passing into a function keeps the type +def fT(a: T) -> T: ... +def fP(a: Z[P]) -> Z[P]: ... + +reveal_type(fT(n)) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(fP(n)) # N: Revealed type is "__main__.Z[[builtins.int]]" + +# literals can be in function args and return type +def k(a: Z[[int]]) -> Z[[str]]: ... + +# functions work +reveal_type(k(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" + +# literals can be matched in arguments +def kb(a: Z[[bytes]]) -> Z[[str]]: ... + +reveal_type(kb(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" \ + # E: Argument 1 to "kb" has incompatible type "Z[[int]]"; expected "Z[[bytes]]" + + +n2: Z[bytes] + +reveal_type(kb(n2)) # N: Revealed type is "__main__.Z[[builtins.str]]" +[builtins fixtures/tuple.pyi] + +[case testParamSpecConcatenateFromPep] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar, Generic + +P = ParamSpec("P") +R = TypeVar("R") + +# CASE 1 +class Request: + ... + +def with_request(f: Callable[Concatenate[Request, P], R]) -> Callable[P, R]: + def inner(*args: P.args, **kwargs: P.kwargs) -> R: + return f(Request(), *args, **kwargs) + return inner + +@with_request +def takes_int_str(request: Request, x: int, y: str) -> int: + # use request + return x + 7 + +reveal_type(takes_int_str) # N: Revealed type is "def (x: builtins.int, y: builtins.str) -> builtins.int" + +takes_int_str(1, "A") # Accepted +takes_int_str("B", 2) # E: Argument 1 to "takes_int_str" has incompatible type "str"; expected "int" \ + # E: Argument 2 to "takes_int_str" has incompatible type "int"; expected "str" + +# CASE 2 +T = TypeVar("T") +P_2 = ParamSpec("P_2") + +class X(Generic[T, P]): + f: Callable[P, int] + x: T + +def f1(x: X[int, P_2]) -> str: ... # Accepted +def f2(x: X[int, Concatenate[int, P_2]]) -> str: ... # Accepted +def f3(x: X[int, [int, bool]]) -> str: ... # Accepted +# ellipsis only show up here, but I can assume it works like Callable[..., R] +def f4(x: X[int, ...]) -> str: ... # Accepted +def f5(x: X[int, int]) -> str: ... # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" + +# CASE 3 +def bar(x: int, *args: bool) -> int: ... +def add(x: Callable[P, int]) -> Callable[Concatenate[str, P], bool]: ... + +reveal_type(add(bar)) # N: Revealed type is "def (builtins.str, x: builtins.int, *args: builtins.bool) -> builtins.bool" + +def remove(x: Callable[Concatenate[int, P], int]) -> Callable[P, bool]: ... + +reveal_type(remove(bar)) # N: Revealed type is "def (*args: builtins.bool) -> builtins.bool" + +def transform( + x: Callable[Concatenate[int, P], int] +) -> Callable[Concatenate[str, P], bool]: ... + +# In the PEP, "__a" appears. What is that? Autogenerated names? To what spec? +reveal_type(transform(bar)) # N: Revealed type is "def (builtins.str, *args: builtins.bool) -> builtins.bool" + +# CASE 4 +def expects_int_first(x: Callable[Concatenate[int, P], int]) -> None: ... + +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]" \ + # N: This is likely because "one" has named arguments: "x". Consider marking them positional-only +def one(x: str) -> int: ... + +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int, NamedArg(int, 'x')], int]" +def two(*, x: int) -> int: ... + +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[KwArg(int)], int]"; expected "Callable[[int, KwArg(int)], int]" +def three(**kwargs: int) -> int: ... + +@expects_int_first # Accepted +def four(*args: int) -> int: ... +[builtins fixtures/dict.pyi] + +[case testParamSpecTwiceSolving] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") + +def f(one: Callable[Concatenate[int, P], R], two: Callable[Concatenate[str, P], R]) -> Callable[P, R]: ... + +a: Callable[[int, bytes], str] +b: Callable[[str, bytes], str] + +reveal_type(f(a, b)) # N: Revealed type is "def (builtins.bytes) -> builtins.str" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConcatenateInReturn] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, Protocol + +P = ParamSpec("P") + +def f(i: Callable[Concatenate[int, P], str]) -> Callable[Concatenate[int, P], str]: ... + +n: Callable[[int, bytes], str] + +reveal_type(f(n)) # N: Revealed type is "def (builtins.int, builtins.bytes) -> builtins.str" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConcatenateNamedArgs] +# flags: --extra-checks +# this is one noticeable deviation from PEP but I believe it is for the better +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") + +def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Accepted + +def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Rejected + +# reason for rejection: +f2(lambda x: 42)(42, x=42) +[builtins fixtures/paramspec.pyi] +[out] +main:17: error: Incompatible return value type (got "Callable[[Arg(int, 'x'), **P], R]", expected "Callable[[int, **P], R]") +main:17: note: This is likely because "result" has named arguments: "x". Consider marking them positional-only + +[case testNonStrictParamSpecConcatenateNamedArgs] +# this is one noticeable deviation from PEP but I believe it is for the better +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") + +def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Accepted + +def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Rejected -> Accepted + +# reason for rejection: +f2(lambda x: 42)(42, x=42) +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConcatenateWithTypeVar] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") +S = TypeVar("S") + +def f(c: Callable[Concatenate[S, P], R]) -> Callable[Concatenate[S, P], R]: ... + +def a(n: int) -> None: ... + +n = f(a) + +reveal_type(n) # N: Revealed type is "def (builtins.int)" +reveal_type(n(42)) # N: Revealed type is "None" +[builtins fixtures/paramspec.pyi] + +[case testCallablesAsParameters] +# credits to https://github.com/microsoft/pyright/issues/2705 +from typing_extensions import ParamSpec, Concatenate +from typing import Generic, Callable, Any + +P = ParamSpec("P") + +class Foo(Generic[P]): + def __init__(self, func: Callable[P, Any]) -> None: ... +def bar(baz: Foo[Concatenate[int, P]]) -> Foo[P]: ... + +def test(a: int, /, b: str) -> str: ... + +abc = Foo(test) +reveal_type(abc) +bar(abc) +[builtins fixtures/paramspec.pyi] +[out] +main:14: note: Revealed type is "__main__.Foo[[builtins.int, b: builtins.str]]" + +[case testSolveParamSpecWithSelfType] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, Generic + +P = ParamSpec("P") + +class Foo(Generic[P]): + def foo(self: 'Foo[P]', other: Callable[P, None]) -> None: ... + +n: Foo[[int]] +def f(x: int) -> None: ... + +n.foo(f) +[builtins fixtures/paramspec.pyi] + +[case testParamSpecLiteralsTypeApplication] +from typing_extensions import ParamSpec +from typing import Generic, Callable + +P = ParamSpec("P") + +class Z(Generic[P]): + def __init__(self, c: Callable[P, None]) -> None: + ... + +# it allows valid functions +reveal_type(Z[[int]](lambda x: None)) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(Z[[]](lambda: None)) # N: Revealed type is "__main__.Z[[]]" +reveal_type(Z[bytes, str](lambda b, s: None)) # N: Revealed type is "__main__.Z[[builtins.bytes, builtins.str]]" + +# it disallows invalid functions +def f1(n: str) -> None: ... +def f2(b: bytes, i: int) -> None: ... + +Z[[int]](lambda one, two: None) # E: Cannot infer type of lambda \ + # E: Argument 1 to "Z" has incompatible type "Callable[[Any, Any], None]"; expected "Callable[[int], None]" +Z[[int]](f1) # E: Argument 1 to "Z" has incompatible type "Callable[[str], None]"; expected "Callable[[int], None]" + +Z[[]](lambda one: None) # E: Cannot infer type of lambda \ + # E: Argument 1 to "Z" has incompatible type "Callable[[Any], None]"; expected "Callable[[], None]" + +Z[bytes, str](lambda one: None) # E: Cannot infer type of lambda \ + # E: Argument 1 to "Z" has incompatible type "Callable[[Any], None]"; expected "Callable[[bytes, str], None]" +Z[bytes, str](f2) # E: Argument 1 to "Z" has incompatible type "Callable[[bytes, int], None]"; expected "Callable[[bytes, str], None]" + +[builtins fixtures/paramspec.pyi] + +[case testParamSpecLiteralEllipsis] +from typing_extensions import ParamSpec +from typing import Generic, Callable + +P = ParamSpec("P") + +class Z(Generic[P]): + def __init__(self: 'Z[P]', c: Callable[P, None]) -> None: + ... + +def f1() -> None: ... +def f2(*args: int) -> None: ... +def f3(a: int, *, b: bytes) -> None: ... + +def f4(b: bytes) -> None: ... + +argh: Callable[..., None] = f4 + +# check it works +Z[...](f1) +Z[...](f2) +Z[...](f3) + +# check subtyping works +n: Z[...] +n = Z(f1) +n = Z(f2) +n = Z(f3) + +[builtins fixtures/paramspec.pyi] + +[case testParamSpecApplyConcatenateTwice] +from typing_extensions import ParamSpec, Concatenate +from typing import Generic, Callable, Optional + +P = ParamSpec("P") + +class C(Generic[P]): + # think PhantomData from rust + phantom: Optional[Callable[P, None]] + + def add_str(self) -> C[Concatenate[str, P]]: + return C[Concatenate[str, P]]() + + def add_int(self) -> C[Concatenate[int, P]]: + return C[Concatenate[int, P]]() + +def f(c: C[P]) -> None: + reveal_type(c) # N: Revealed type is "__main__.C[P`-1]" + + n1 = c.add_str() + reveal_type(n1) # N: Revealed type is "__main__.C[[builtins.str, **P`-1]]" + n2 = n1.add_int() + reveal_type(n2) # N: Revealed type is "__main__.C[[builtins.int, builtins.str, **P`-1]]" + + p1 = c.add_int() + reveal_type(p1) # N: Revealed type is "__main__.C[[builtins.int, **P`-1]]" + p2 = p1.add_str() + reveal_type(p2) # N: Revealed type is "__main__.C[[builtins.str, builtins.int, **P`-1]]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecLiteralJoin] +from typing import Generic, Callable, Union +from typing_extensions import ParamSpec + + +_P = ParamSpec("_P") + +class Job(Generic[_P]): + def __init__(self, target: Callable[_P, None]) -> None: + self.target = target + +def func( + action: Union[Job[int], Callable[[int], None]], +) -> None: + job = action if isinstance(action, Job) else Job(action) + reveal_type(job) # N: Revealed type is "__main__.Job[[builtins.int]]" +[builtins fixtures/paramspec.pyi] + +[case testApplyParamSpecToParamSpecLiterals] +from typing import TypeVar, Generic, Callable +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_R_co = TypeVar("_R_co", covariant=True) + +class Job(Generic[_P, _R_co]): + def __init__(self, target: Callable[_P, _R_co]) -> None: + self.target = target + +def run_job(job: Job[_P, None], *args: _P.args, **kwargs: _P.kwargs) -> None: # N: "run_job" defined here + ... + + +def func(job: Job[[int, str], None]) -> None: + run_job(job, 42, "Hello") + run_job(job, "Hello", 42) # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" \ + # E: Argument 3 to "run_job" has incompatible type "int"; expected "str" + run_job(job, 42, msg="Hello") # E: Unexpected keyword argument "msg" for "run_job" + run_job(job, "Hello") # E: Too few arguments for "run_job" \ + # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" + +def func2(job: Job[..., None]) -> None: + run_job(job, 42, "Hello") + run_job(job, "Hello", 42) + run_job(job, 42, msg="Hello") + run_job(job, x=42, msg="Hello") +[builtins fixtures/paramspec.pyi] + +[case testExpandNonBareParamSpecAgainstCallable] +from typing import Callable, TypeVar, Any +from typing_extensions import ParamSpec + +CallableT = TypeVar("CallableT", bound=Callable[..., Any]) +_P = ParamSpec("_P") +_R = TypeVar("_R") + +def simple_decorator(callable: CallableT) -> CallableT: + # set some attribute on 'callable' + return callable + + +class A: + @simple_decorator + def func(self, action: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: + ... + +reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`6, *_P.args, **_P.kwargs) -> _R`6" +reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`10, *_P.args, **_P.kwargs) -> _R`10" + +def f(x: int) -> int: + ... + +reveal_type(A().func(f, 42)) # N: Revealed type is "builtins.int" + +reveal_type(A().func(lambda x: x + x, 42)) # N: Revealed type is "builtins.int" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConstraintOnOtherParamSpec] +from typing import Callable, TypeVar, Any, Generic +from typing_extensions import ParamSpec + +CallableT = TypeVar("CallableT", bound=Callable[..., Any]) +_P = ParamSpec("_P") +_R_co = TypeVar("_R_co", covariant=True) + +def simple_decorator(callable: CallableT) -> CallableT: + ... + +class Job(Generic[_P, _R_co]): + def __init__(self, target: Callable[_P, _R_co]) -> None: + ... + + +class A: + @simple_decorator + def func(self, action: Job[_P, None]) -> Job[_P, None]: + ... + +reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" +reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`6, None]) -> __main__.Job[_P`6, None]" +reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[[x: Any], None]" + +def f(x: int, y: int) -> None: ... +reveal_type(A().func(Job(f))) # N: Revealed type is "__main__.Job[[x: builtins.int, y: builtins.int], None]" +[builtins fixtures/paramspec.pyi] + +[case testConstraintBetweenParamSpecFunctions1] +from typing import Callable, TypeVar, Any, Generic +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_R_co = TypeVar("_R_co", covariant=True) + +def simple_decorator(callable: Callable[_P, _R_co]) -> Callable[_P, _R_co]: ... +class Job(Generic[_P]): ... + + +@simple_decorator +def func(__action: Job[_P]) -> Callable[_P, None]: + ... + +reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)" +[builtins fixtures/paramspec.pyi] + +[case testConstraintBetweenParamSpecFunctions2] +from typing import Callable, TypeVar, Any, Generic +from typing_extensions import ParamSpec + +CallableT = TypeVar("CallableT", bound=Callable[..., Any]) +_P = ParamSpec("_P") + +def simple_decorator(callable: CallableT) -> CallableT: ... +class Job(Generic[_P]): ... + + +@simple_decorator +def func(__action: Job[_P]) -> Callable[_P, None]: + ... + +reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)" +[builtins fixtures/paramspec.pyi] + +[case testConstraintsBetweenConcatenatePrefixes] +from typing import Any, Callable, Generic, TypeVar +from typing_extensions import Concatenate, ParamSpec + +_P = ParamSpec("_P") +_T = TypeVar("_T") + +class Awaitable(Generic[_T]): ... + +def adds_await() -> Callable[ + [Callable[Concatenate[_T, _P], None]], + Callable[Concatenate[_T, _P], Awaitable[None]], +]: + def decorator( + func: Callable[Concatenate[_T, _P], None], + ) -> Callable[Concatenate[_T, _P], Awaitable[None]]: + ... + + return decorator # we want `_T` and `_P` to refer to the same things. +[builtins fixtures/paramspec.pyi] + +[case testParamSpecVariance] +from typing import Callable, Generic +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") + +class Job(Generic[_P]): + def __init__(self, target: Callable[_P, None]) -> None: ... + def into_callable(self) -> Callable[_P, None]: ... + +class A: + def func(self, var: int) -> None: ... + def other_func(self, job: Job[[int]]) -> None: ... + + +job = Job(A().func) +reveal_type(job) # N: Revealed type is "__main__.Job[[var: builtins.int]]" +A().other_func(job) # This should NOT error (despite the keyword) + +# and yet the keyword should remain +job.into_callable()(var=42) +job.into_callable()(x=42) # E: Unexpected keyword argument "x" + +# similar for other functions +def f1(n: object) -> None: ... +def f2(n: int) -> None: ... +def f3(n: bool) -> None: ... + +# just like how this is legal... +a1: Callable[[bool], None] +a1 = f3 +a1 = f2 +a1 = f1 + +# ... this is also legal +a2: Job[[bool]] +a2 = Job(f3) +a2 = Job(f2) +a2 = Job(f1) + +# and this is not legal +def f4(n: bytes) -> None: ... +a1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[bytes], None]", variable has type "Callable[[bool], None]") +a2 = Job(f4) # E: Argument 1 to "Job" has incompatible type "Callable[[bytes], None]"; expected "Callable[[bool], None]" + +# nor is this: +a4: Job[[int]] +a4 = Job(f3) # E: Argument 1 to "Job" has incompatible type "Callable[[bool], None]"; expected "Callable[[int], None]" +a4 = Job(f2) +a4 = Job(f1) + +# just like this: +a3: Callable[[int], None] +a3 = f3 # E: Incompatible types in assignment (expression has type "Callable[[bool], None]", variable has type "Callable[[int], None]") +a3 = f2 +a3 = f1 +[builtins fixtures/paramspec.pyi] + +[case testDecoratingClassesThatUseParamSpec] +from typing import Generic, TypeVar, Callable, Any +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_T = TypeVar("_T") +_F = TypeVar("_F", bound=Callable[..., Any]) + +def f(x: _F) -> _F: ... + +@f # Should be ok +class OnlyParamSpec(Generic[_P]): + pass + +@f # Should be ok +class MixedWithTypeVar1(Generic[_P, _T]): + pass + +@f # Should be ok +class MixedWithTypeVar2(Generic[_T, _P]): + pass +[builtins fixtures/dict.pyi] + +[case testGenericsInInferredParamspec] +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_T = TypeVar("_T") + +class Job(Generic[_P]): + def __init__(self, target: Callable[_P, None]) -> None: ... + def into_callable(self) -> Callable[_P, None]: ... + +def generic_f(x: _T) -> None: ... + +j = Job(generic_f) +reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`-1]]" + +jf = j.into_callable() +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`4)" +reveal_type(jf(1)) # N: Revealed type is "None" +[builtins fixtures/paramspec.pyi] + +[case testGenericsInInferredParamspecReturn] +# flags: --new-type-inference +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_T = TypeVar("_T") + +class Job(Generic[_P, _T]): + def __init__(self, target: Callable[_P, _T]) -> None: ... + def into_callable(self) -> Callable[_P, _T]: ... + +def generic_f(x: _T) -> _T: ... + +j = Job(generic_f) +reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`3], _T`3]" + +jf = j.into_callable() +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`4) -> _T`4" +reveal_type(jf(1)) # N: Revealed type is "builtins.int" +[builtins fixtures/paramspec.pyi] + +[case testStackedConcatenateIsIllegal] +from typing_extensions import Concatenate, ParamSpec +from typing import Callable + +P = ParamSpec("P") + +def x(f: Callable[Concatenate[int, Concatenate[int, P]], None]) -> None: ... # E: Nested Concatenates are invalid +[builtins fixtures/paramspec.pyi] + +[case testPropagatedAnyConstraintsAreOK] +from typing import Any, Callable, Generic, TypeVar +from typing_extensions import ParamSpec + +T = TypeVar("T") +P = ParamSpec("P") + +def callback(func: Callable[[Any], Any]) -> None: ... +class Job(Generic[P]): ... + +@callback +def run_job(job: Job[...]) -> T: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar +[builtins fixtures/tuple.pyi] + +[case testTupleAndDictOperationsOnParamSpecArgsAndKwargs] +from typing import Callable, Iterator, Iterable, TypeVar, Tuple +from typing_extensions import ParamSpec + +P = ParamSpec('P') +T = TypeVar('T') +def enumerate(x: Iterable[T]) -> Iterator[Tuple[int, T]]: ... + +def func(callback: Callable[P, str]) -> Callable[P, str]: + def inner(*args: P.args, **kwargs: P.kwargs) -> str: + reveal_type(args[5]) # N: Revealed type is "builtins.object" + for a in args: + reveal_type(a) # N: Revealed type is "builtins.object" + for idx, a in enumerate(args): + reveal_type(idx) # N: Revealed type is "builtins.int" + reveal_type(a) # N: Revealed type is "builtins.object" + b = 'foo' in args + reveal_type(b) # N: Revealed type is "builtins.bool" + reveal_type(args.count(42)) # N: Revealed type is "builtins.int" + reveal_type(len(args)) # N: Revealed type is "builtins.int" + for c, d in kwargs.items(): + reveal_type(c) # N: Revealed type is "builtins.str" + reveal_type(d) # N: Revealed type is "builtins.object" + kwargs.pop('bar') + return 'baz' + return inner +[builtins fixtures/paramspec.pyi] + +[case testUnpackingParamsSpecArgsAndKwargs] +from typing import Callable +from typing_extensions import ParamSpec + +P = ParamSpec("P") + +def func(callback: Callable[P, str]) -> Callable[P, str]: + def inner(*args: P.args, **kwargs: P.kwargs) -> str: + a, *b = args + reveal_type(a) # N: Revealed type is "builtins.object" + reveal_type(b) # N: Revealed type is "builtins.list[builtins.object]" + c, *d = kwargs + reveal_type(c) # N: Revealed type is "builtins.str" + reveal_type(d) # N: Revealed type is "builtins.list[builtins.str]" + e = {**kwargs} + reveal_type(e) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" + return "foo" + return inner +[builtins fixtures/paramspec.pyi] + +[case testParamSpecArgsAndKwargsMissmatch] +from typing import Callable +from typing_extensions import ParamSpec + +P1 = ParamSpec("P1") + +def func(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.kwargs, # E: Use "P1.args" for variadic "*" parameter + **kwargs: P1.args, # E: Use "P1.kwargs" for variadic "**" parameter + ) -> str: + return "foo" + return inner +[builtins fixtures/paramspec.pyi] + +[case testParamSpecTestPropAccess] +from typing import Callable +from typing_extensions import ParamSpec + +P1 = ParamSpec("P1") + +def func1(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.typo, # E: Use "P1.args" for variadic "*" parameter \ + # E: Name "P1.typo" is not defined + **kwargs: P1.kwargs, + ) -> str: + return "foo" + return inner + +def func2(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.args, + **kwargs: P1.__bound__, # E: Use "P1.kwargs" for variadic "**" parameter \ + # E: Name "P1.__bound__" is not defined + ) -> str: + return "foo" + return inner + +def func3(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.__bound__, # E: Use "P1.args" for variadic "*" parameter \ + # E: Name "P1.__bound__" is not defined + **kwargs: P1.invalid, # E: Use "P1.kwargs" for variadic "**" parameter \ + # E: Name "P1.invalid" is not defined + ) -> str: + return "foo" + return inner +[builtins fixtures/paramspec.pyi] + + +[case testInvalidParamSpecDefinitionsWithArgsKwargs] +from typing import Callable, ParamSpec + +P = ParamSpec('P') + +def c1(f: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> int: ... +def c2(f: Callable[P, int]) -> int: ... +def c3(f: Callable[P, int], *args, **kwargs) -> int: ... + +# It is ok to define, +def c4(f: Callable[P, int], *args: int, **kwargs: str) -> int: + # but not ok to call: + f(*args, **kwargs) # E: Argument 1 has incompatible type "*Tuple[int, ...]"; expected "P.args" \ + # E: Argument 2 has incompatible type "**Dict[str, str]"; expected "P.kwargs" + return 1 + +def f1(f: Callable[P, int], *args, **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f2(f: Callable[P, int], *args: P.args, **kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f3(f: Callable[P, int], *args: P.args) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f4(f: Callable[P, int], **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f5(f: Callable[P, int], *args: P.args, extra_keyword_arg: int, **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" + +# Error message test: +P1 = ParamSpec('P1') + +def m1(f: Callable[P1, int], *a, **k: P1.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +[builtins fixtures/paramspec.pyi] + + +[case testInvalidParamSpecAndConcatenateDefinitionsWithArgsKwargs] +from typing import Callable, ParamSpec +from typing_extensions import Concatenate + +P = ParamSpec('P') + +def c1(f: Callable[Concatenate[int, P], int], *args: P.args, **kwargs: P.kwargs) -> int: ... +def c2(f: Callable[Concatenate[int, P], int]) -> int: ... +def c3(f: Callable[Concatenate[int, P], int], *args, **kwargs) -> int: ... + +# It is ok to define, +def c4(f: Callable[Concatenate[int, P], int], *args: int, **kwargs: str) -> int: + # but not ok to call: + f(1, *args, **kwargs) # E: Argument 2 has incompatible type "*Tuple[int, ...]"; expected "P.args" \ + # E: Argument 3 has incompatible type "**Dict[str, str]"; expected "P.kwargs" + return 1 + +def f1(f: Callable[Concatenate[int, P], int], *args, **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f2(f: Callable[Concatenate[int, P], int], *args: P.args, **kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f3(f: Callable[Concatenate[int, P], int], *args: P.args) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f4(f: Callable[Concatenate[int, P], int], **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f5(f: Callable[Concatenate[int, P], int], *args: P.args, extra_keyword_arg: int, **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +[builtins fixtures/paramspec.pyi] + + +[case testValidParamSpecInsideGenericWithoutArgsAndKwargs] +from typing import Callable, ParamSpec, Generic +from typing_extensions import Concatenate + +P = ParamSpec('P') + +class Some(Generic[P]): ... + +def create(s: Some[P], *args: int): ... +def update(s: Some[P], **kwargs: int): ... +def delete(s: Some[P]): ... + +def from_callable1(c: Callable[P, int], *args: int, **kwargs: int) -> Some[P]: ... +def from_callable2(c: Callable[P, int], **kwargs: int) -> Some[P]: ... +def from_callable3(c: Callable[P, int], *args: int) -> Some[P]: ... + +def from_extra1(c: Callable[Concatenate[int, P], int], *args: int, **kwargs: int) -> Some[P]: ... +def from_extra2(c: Callable[Concatenate[int, P], int], **kwargs: int) -> Some[P]: ... +def from_extra3(c: Callable[Concatenate[int, P], int], *args: int) -> Some[P]: ... +[builtins fixtures/paramspec.pyi] + + +[case testUnboundParamSpec] +from typing import Callable, ParamSpec + +P1 = ParamSpec('P1') +P2 = ParamSpec('P2') + +def f0(f: Callable[P1, int], *args: P1.args, **kwargs: P2.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" + +def f1(*args: P1.args): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f2(**kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f3(*args: P1.args, **kwargs: int): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f4(*args: int, **kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" + +# Error message is based on the `args` definition: +def f5(*args: P2.args, **kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P2.args" and "**kwargs" typed as "P2.kwargs" +def f6(*args: P1.args, **kwargs: P2.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" + +# Multiple `ParamSpec` variables can be found, they should not affect error message: +P3 = ParamSpec('P3') + +def f7(first: Callable[P3, int], *args: P1.args, **kwargs: P2.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f8(first: Callable[P3, int], *args: P2.args, **kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P2.args" and "**kwargs" typed as "P2.kwargs" +[builtins fixtures/paramspec.pyi] + + +[case testArgsKwargsWithoutParamSpecVar] +from typing import Generic, Callable, ParamSpec + +P = ParamSpec('P') + +# This must be allowed: +class Some(Generic[P]): + def call(self, *args: P.args, **kwargs: P.kwargs): ... + +def call(*args: P.args, **kwargs: P.kwargs): ... +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInferenceCrash] +from typing import Callable, Generic, ParamSpec, TypeVar + +def foo(x: int) -> int: ... +T = TypeVar("T") +def bar(x: T) -> T: ... + +P = ParamSpec("P") + +class C(Generic[P]): + def __init__(self, fn: Callable[P, int], *args: P.args, **kwargs: P.kwargs): ... + +reveal_type(bar(C(fn=foo, x=1))) # N: Revealed type is "__main__.C[[x: builtins.int]]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecClassConstructor] +from typing import ParamSpec, Callable, TypeVar + +P = ParamSpec("P") + +class SomeClass: + def __init__(self, a: str) -> None: + pass + +def func(t: Callable[P, SomeClass], val: Callable[P, SomeClass]) -> Callable[P, SomeClass]: + pass + +def func_regular(t: Callable[[T], SomeClass], val: Callable[[T], SomeClass]) -> Callable[[T], SomeClass]: + pass + +def constructor(a: str) -> SomeClass: + return SomeClass(a) + +def wrong_constructor(a: bool) -> SomeClass: + return SomeClass("a") + +def wrong_name_constructor(b: bool) -> SomeClass: + return SomeClass("a") + +func(SomeClass, constructor) +reveal_type(func(SomeClass, wrong_constructor)) # N: Revealed type is "def (a: Never) -> __main__.SomeClass" +reveal_type(func_regular(SomeClass, wrong_constructor)) # N: Revealed type is "def (Never) -> __main__.SomeClass" +reveal_type(func(SomeClass, wrong_name_constructor)) # N: Revealed type is "def (Never) -> __main__.SomeClass" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInTypeAliasBasic] +from typing import ParamSpec, Callable + +P = ParamSpec("P") +C = Callable[P, int] +def f(n: C[P]) -> C[P]: ... + +@f +def bar(x: int) -> int: ... +@f # E: Argument 1 to "f" has incompatible type "Callable[[int], str]"; expected "Callable[[int], int]" +def foo(x: int) -> str: ... + +x: C[[int, str]] +reveal_type(x) # N: Revealed type is "def (builtins.int, builtins.str) -> builtins.int" +y: C[int, str] +reveal_type(y) # N: Revealed type is "def (builtins.int, builtins.str) -> builtins.int" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInTypeAliasConcatenate] +from typing import ParamSpec, Callable +from typing_extensions import Concatenate + +P = ParamSpec("P") +C = Callable[Concatenate[int, P], int] +def f(n: C[P]) -> C[P]: ... + +@f # E: Argument 1 to "f" has incompatible type "Callable[[], int]"; expected "Callable[[int], int]" +def bad() -> int: ... + +@f +def bar(x: int) -> int: ... + +@f +def bar2(x: int, y: str) -> int: ... +reveal_type(bar2) # N: Revealed type is "def (builtins.int, y: builtins.str) -> builtins.int" + +@f # E: Argument 1 to "f" has incompatible type "Callable[[int], str]"; expected "Callable[[int], int]" \ + # N: This is likely because "foo" has named arguments: "x". Consider marking them positional-only +def foo(x: int) -> str: ... + +@f # E: Argument 1 to "f" has incompatible type "Callable[[str, int], int]"; expected "Callable[[int, int], int]" \ + # N: This is likely because "foo2" has named arguments: "x". Consider marking them positional-only +def foo2(x: str, y: int) -> int: ... + +x: C[[int, str]] +reveal_type(x) # N: Revealed type is "def (builtins.int, builtins.int, builtins.str) -> builtins.int" +y: C[int, str] +reveal_type(y) # N: Revealed type is "def (builtins.int, builtins.int, builtins.str) -> builtins.int" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInTypeAliasIllegalBare] +from typing import ParamSpec +from typing_extensions import Concatenate, TypeAlias + +P = ParamSpec("P") +Bad1: TypeAlias = P # E: Invalid location for ParamSpec "P" \ + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" +Bad2: TypeAlias = Concatenate[int, P] # E: Invalid location for Concatenate \ + # N: You can use Concatenate as the first argument to Callable +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInTypeAliasRecursive] +from typing import ParamSpec, Callable, Union + +P = ParamSpec("P") +C = Callable[P, Union[int, C[P]]] +def f(n: C[P]) -> C[P]: ... + +@f +def bar(x: int) -> int: ... + +@f +def bar2(__x: int) -> Callable[[int], int]: ... + +@f # E: Argument 1 to "f" has incompatible type "Callable[[int], str]"; expected "C[[int]]" +def foo(x: int) -> str: ... + +@f # E: Argument 1 to "f" has incompatible type "Callable[[int], Callable[[int], str]]"; expected "C[[int]]" +def foo2(__x: int) -> Callable[[int], str]: ... + +x: C[[int, str]] +reveal_type(x) # N: Revealed type is "def (builtins.int, builtins.str) -> Union[builtins.int, ...]" +y: C[int, str] +reveal_type(y) # N: Revealed type is "def (builtins.int, builtins.str) -> Union[builtins.int, ...]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecAliasInRuntimeContext] +from typing import ParamSpec, Generic + +P = ParamSpec("P") +class C(Generic[P]): ... + +c = C[int, str]() +reveal_type(c) # N: Revealed type is "__main__.C[[builtins.int, builtins.str]]" + +A = C[P] +a = A[int, str]() +reveal_type(a) # N: Revealed type is "__main__.C[[builtins.int, builtins.str]]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecAliasInvalidLocations] +from typing import ParamSpec, Generic, List, TypeVar, Callable + +P = ParamSpec("P") +T = TypeVar("T") +A = List[T] +def f(x: A[[int, str]]) -> None: ... # E: Bracketed expression "[...]" is not valid as a type +def g(x: A[P]) -> None: ... # E: Invalid location for ParamSpec "P" \ + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" + +C = Callable[P, T] +x: C[int] # E: Bad number of arguments for type alias, expected 2, given 1 +y: C[int, str] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" +z: C[int, str, bytes] # E: Bad number of arguments for type alias, expected 2, given 3 + +[builtins fixtures/paramspec.pyi] + +[case testTrivialParametersHandledCorrectly] +from typing import ParamSpec, Generic, TypeVar, Callable, Any +from typing_extensions import Concatenate + +P = ParamSpec("P") +T = TypeVar("T") +S = TypeVar("S") + +class C(Generic[S, P, T]): ... + +def foo(f: Callable[P, int]) -> None: + x: C[Any, ..., Any] + x1: C[int, Concatenate[int, str, P], str] + x = x1 # OK +[builtins fixtures/paramspec.pyi] + +[case testParamSpecAliasNested] +from typing import ParamSpec, Callable, List, TypeVar, Generic +from typing_extensions import Concatenate + +P = ParamSpec("P") +A = List[Callable[P, None]] +B = List[Callable[Concatenate[int, P], None]] + +fs: A[int, str] +reveal_type(fs) # N: Revealed type is "builtins.list[def (builtins.int, builtins.str)]" +gs: B[int, str] +reveal_type(gs) # N: Revealed type is "builtins.list[def (builtins.int, builtins.int, builtins.str)]" + +T = TypeVar("T") +class C(Generic[T]): ... +C[Callable[P, int]]() +[builtins fixtures/paramspec.pyi] + +[case testConcatDeferralNoCrash] +from typing import Callable, TypeVar +from typing_extensions import Concatenate, ParamSpec + +P = ParamSpec("P") +T = TypeVar("T", bound="Defer") + +Alias = Callable[P, bool] +Concat = Alias[Concatenate[T, P]] + +def test(f: Concat[T, ...]) -> None: ... + +class Defer: ... +[builtins fixtures/paramspec.pyi] + +[case testNoParamSpecDoubling] +# https://github.com/python/mypy/issues/12734 +from typing import Callable, ParamSpec +from typing_extensions import Concatenate + +P = ParamSpec("P") +Q = ParamSpec("Q") + +def foo(f: Callable[P, int]) -> Callable[P, int]: + return f + +def bar(f: Callable[Concatenate[str, Q], int]) -> Callable[Concatenate[str, Q], int]: + return foo(f) +[builtins fixtures/paramspec.pyi] + +[case testAlreadyExpandedCallableWithParamSpecReplacement] +from typing import Callable, Any, overload +from typing_extensions import Concatenate, ParamSpec + +P = ParamSpec("P") + +@overload +def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: + ... + +@overload +def command( + cls: int = ..., +) -> Callable[[Callable[Concatenate[object, P], object]], None]: + ... + +def command( + cls: int = 42, +) -> Any: + ... +[builtins fixtures/paramspec.pyi] + +[case testCopiedParamSpecComparison] +# minimized from https://github.com/python/mypy/issues/12909 +from typing import Callable +from typing_extensions import ParamSpec + +P = ParamSpec("P") + +def identity(func: Callable[P, None]) -> Callable[P, None]: ... + +@identity +def f(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... +[builtins fixtures/paramspec.pyi] + +[case testParamSpecDecoratorAppliedToGeneric] +# flags: --new-type-inference +from typing import Callable, List, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") +U = TypeVar("U") + +def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... +def test(x: U) -> U: ... +reveal_type(dec) # N: Revealed type is "def [P, T] (f: def (*P.args, **P.kwargs) -> T`-2) -> def (*P.args, **P.kwargs) -> builtins.list[T`-2]" +reveal_type(dec(test)) # N: Revealed type is "def [T] (x: T`3) -> builtins.list[T`3]" + +class A: ... +TA = TypeVar("TA", bound=A) + +def test_with_bound(x: TA) -> TA: ... +reveal_type(dec(test_with_bound)) # N: Revealed type is "def [T <: __main__.A] (x: T`5) -> builtins.list[T`5]" +dec(test_with_bound)(0) # E: Value of type variable "T" of function cannot be "int" +dec(test_with_bound)(A()) # OK +[builtins fixtures/paramspec.pyi] + +[case testParamSpecArgumentParamInferenceRegular] +from typing import TypeVar, Generic +from typing_extensions import ParamSpec + +P = ParamSpec("P") +class Foo(Generic[P]): + def call(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +def test(*args: P.args, **kwargs: P.kwargs) -> Foo[P]: ... + +reveal_type(test(1, 2)) # N: Revealed type is "__main__.Foo[[Literal[1]?, Literal[2]?]]" +reveal_type(test(x=1, y=2)) # N: Revealed type is "__main__.Foo[[x: Literal[1]?, y: Literal[2]?]]" +ints = [1, 2, 3] +reveal_type(test(*ints)) # N: Revealed type is "__main__.Foo[[*builtins.int]]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecArgumentParamInferenceGeneric] +# flags: --new-type-inference +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec("P") +R = TypeVar("R") +def call(f: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: + return f(*args, **kwargs) + +T = TypeVar("T") +def identity(x: T) -> T: + return x + +reveal_type(call(identity, 2)) # N: Revealed type is "builtins.int" +y: int = call(identity, 2) +[builtins fixtures/paramspec.pyi] + +[case testParamSpecNestedApplyNoCrash] +# flags: --new-type-inference +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") + +def apply(fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: ... +def test() -> int: ... +reveal_type(apply(apply, test)) # N: Revealed type is "builtins.int" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecNestedApplyPosVsNamed] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") + +def apply(fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> None: ... + +def test(x: int) -> int: ... +apply(apply, test, x=42) # OK +apply(apply, test, 42) # Also OK (but requires some special casing) +apply(apply, test, "bad") # E: Argument 1 to "apply" has incompatible type "Callable[[Callable[P, T], **P], None]"; expected "Callable[[Callable[[int], int], str], None]" + +def test2(x: int, y: str) -> None: ... +apply(apply, test2, 42, "yes") +apply(apply, test2, "no", 42) # E: Argument 1 to "apply" has incompatible type "Callable[[Callable[P, T], **P], None]"; expected "Callable[[Callable[[int, str], None], str, int], None]" +apply(apply, test2, x=42, y="yes") +apply(apply, test2, y="yes", x=42) +apply(apply, test2, y=42, x="no") # E: Argument 1 to "apply" has incompatible type "Callable[[Callable[P, T], **P], None]"; expected "Callable[[Callable[[int, str], None], int, str], None]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecApplyPosVsNamedOptional] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") + +def apply(fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> None: ... +def test(x: str = ..., y: int = ...) -> int: ... +apply(test, y=42) # OK +[builtins fixtures/paramspec.pyi] + +[case testParamSpecPrefixSubtypingGenericInvalid] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") + +class A(Generic[P]): + def foo(self, *args: P.args, **kwargs: P.kwargs): + ... + +def bar(b: A[P]) -> A[Concatenate[int, P]]: + return b # E: Incompatible return value type (got "A[P]", expected "A[[int, **P]]") +[builtins fixtures/paramspec.pyi] + +[case testParamSpecPrefixSubtypingProtocolInvalid] +from typing import Protocol +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") + +class A(Protocol[P]): + def foo(self, *args: P.args, **kwargs: P.kwargs): + ... + +def bar(b: A[P]) -> A[Concatenate[int, P]]: + return b # E: Incompatible return value type (got "A[P]", expected "A[[int, **P]]") \ + # N: Following member(s) of "A[P]" have conflicts: \ + # N: Expected: \ + # N: def foo(self, int, /, *args: P.args, **kwargs: P.kwargs) -> Any \ + # N: Got: \ + # N: def foo(self, *args: P.args, **kwargs: P.kwargs) -> Any +[builtins fixtures/paramspec.pyi] + +[case testParamSpecPrefixSubtypingValidNonStrict] +from typing import Protocol +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") + +class A(Protocol[P]): + def foo(self, a: int, *args: P.args, **kwargs: P.kwargs): + ... + +class B(Protocol[P]): + def foo(self, a: int, b: int, *args: P.args, **kwargs: P.kwargs): + ... + +def bar(b: B[P]) -> A[Concatenate[int, P]]: + return b +[builtins fixtures/paramspec.pyi] + +[case testParamSpecPrefixSubtypingInvalidStrict] +# flags: --extra-checks +from typing import Protocol +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") + +class A(Protocol[P]): + def foo(self, a: int, *args: P.args, **kwargs: P.kwargs): + ... + +class B(Protocol[P]): + def foo(self, a: int, b: int, *args: P.args, **kwargs: P.kwargs): + ... + +def bar(b: B[P]) -> A[Concatenate[int, P]]: + return b # E: Incompatible return value type (got "B[P]", expected "A[[int, **P]]") \ + # N: Following member(s) of "B[P]" have conflicts: \ + # N: Expected: \ + # N: def foo(self, a: int, int, /, *args: P.args, **kwargs: P.kwargs) -> Any \ + # N: Got: \ + # N: def foo(self, a: int, b: int, *args: P.args, **kwargs: P.kwargs) -> Any +[builtins fixtures/paramspec.pyi] + +[case testParamSpecDecoratorOverload] +from typing import Callable, overload, TypeVar, List +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") +def transform(func: Callable[P, List[T]]) -> Callable[P, T]: ... + +@overload +def foo(x: int) -> List[float]: ... +@overload +def foo(x: str) -> List[str]: ... +def foo(x): ... + +reveal_type(transform(foo)) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.float, def (x: builtins.str) -> builtins.str)" + +@transform +@overload +def bar(x: int) -> List[float]: ... +@transform +@overload +def bar(x: str) -> List[str]: ... +@transform +def bar(x): ... + +reveal_type(bar) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.float, def (x: builtins.str) -> builtins.str)" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecDecoratorOverloadNoCrashOnInvalidTypeVar] +from typing import Any, Callable, List +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = 1 + +Alias = Callable[P, List[T]] # type: ignore +def dec(fn: Callable[P, T]) -> Alias[P, T]: ... # type: ignore +f: Any +dec(f) # No crash +[builtins fixtures/paramspec.pyi] + +[case testParamSpecErrorNestedParams] +from typing import Generic +from typing_extensions import ParamSpec + +P = ParamSpec("P") +class C(Generic[P]): ... +c: C[int, [int, str], str] # E: Nested parameter specifications are not allowed +reveal_type(c) # N: Revealed type is "__main__.C[Any]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInheritNoCrashOnNested] +from typing import Generic +from typing_extensions import ParamSpec + +P = ParamSpec("P") +class C(Generic[P]): ... +class D(C[int, [int, str], str]): ... # E: Nested parameter specifications are not allowed +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConcatenateSelfType] +from typing import Callable +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class A: + def __init__(self, a_param_1: str) -> None: ... + + @classmethod + def add_params(cls: Callable[P, A]) -> Callable[Concatenate[float, P], A]: + def new_constructor(i: float, *args: P.args, **kwargs: P.kwargs) -> A: + return cls(*args, **kwargs) + return new_constructor + + @classmethod + def remove_params(cls: Callable[Concatenate[str, P], A]) -> Callable[P, A]: + def new_constructor(*args: P.args, **kwargs: P.kwargs) -> A: + return cls("my_special_str", *args, **kwargs) + return new_constructor + +reveal_type(A.add_params()) # N: Revealed type is "def (builtins.float, a_param_1: builtins.str) -> __main__.A" +reveal_type(A.remove_params()) # N: Revealed type is "def () -> __main__.A" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConcatenateCallbackProtocol] +from typing import Protocol, TypeVar +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +R = TypeVar("R", covariant=True) + +class Path: ... + +class Function(Protocol[P, R]): + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: ... + +def file_cache(fn: Function[Concatenate[Path, P], R]) -> Function[P, R]: + def wrapper(*args: P.args, **kw: P.kwargs) -> R: + return fn(Path(), *args, **kw) + return wrapper + +@file_cache +def get_thing(path: Path, *, some_arg: int) -> int: ... +reveal_type(get_thing) # N: Revealed type is "__main__.Function[[*, some_arg: builtins.int], builtins.int]" +get_thing(some_arg=1) # OK +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConcatenateKeywordOnly] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +R = TypeVar("R") + +class Path: ... + +def file_cache(fn: Callable[Concatenate[Path, P], R]) -> Callable[P, R]: + def wrapper(*args: P.args, **kw: P.kwargs) -> R: + return fn(Path(), *args, **kw) + return wrapper + +@file_cache +def get_thing(path: Path, *, some_arg: int) -> int: ... +reveal_type(get_thing) # N: Revealed type is "def (*, some_arg: builtins.int) -> builtins.int" +get_thing(some_arg=1) # OK +[builtins fixtures/paramspec.pyi] + +[case testParamSpecConcatenateCallbackApply] +from typing import Callable, Protocol +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") + +class FuncType(Protocol[P]): + def __call__(self, x: int, s: str, *args: P.args, **kw_args: P.kwargs) -> str: ... + +def forwarder1(fp: FuncType[P], *args: P.args, **kw_args: P.kwargs) -> str: + return fp(0, '', *args, **kw_args) + +def forwarder2(fp: Callable[Concatenate[int, str, P], str], *args: P.args, **kw_args: P.kwargs) -> str: + return fp(0, '', *args, **kw_args) + +def my_f(x: int, s: str, d: bool) -> str: ... +forwarder1(my_f, True) # OK +forwarder2(my_f, True) # OK +forwarder1(my_f, 1.0) # E: Argument 2 to "forwarder1" has incompatible type "float"; expected "bool" +forwarder2(my_f, 1.0) # E: Argument 2 to "forwarder2" has incompatible type "float"; expected "bool" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecCallbackProtocolSelf] +from typing import Callable, Protocol, TypeVar +from typing_extensions import ParamSpec, Concatenate + +Params = ParamSpec("Params") +Result = TypeVar("Result", covariant=True) + +class FancyMethod(Protocol): + def __call__(self, arg1: int, arg2: str) -> bool: ... + def return_me(self: Callable[Params, Result]) -> Callable[Params, Result]: ... + def return_part(self: Callable[Concatenate[int, Params], Result]) -> Callable[Params, Result]: ... + +m: FancyMethod +reveal_type(m.return_me()) # N: Revealed type is "def (arg1: builtins.int, arg2: builtins.str) -> builtins.bool" +reveal_type(m.return_part()) # N: Revealed type is "def (arg2: builtins.str) -> builtins.bool" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInferenceCallableAgainstAny] +from typing import Callable, TypeVar, Any +from typing_extensions import ParamSpec, Concatenate + +_P = ParamSpec("_P") +_R = TypeVar("_R") + +class A: ... +a = A() + +def a_func( + func: Callable[Concatenate[A, _P], _R], +) -> Callable[Concatenate[Any, _P], _R]: + def wrapper(__a: Any, *args: _P.args, **kwargs: _P.kwargs) -> _R: + return func(a, *args, **kwargs) + return wrapper + +def test(a, *args): ... +x: Any +y: object + +a_func(test) +x = a_func(test) +y = a_func(test) +[builtins fixtures/paramspec.pyi] + +[case testParamSpecInferenceWithCallbackProtocol] +from typing import Protocol, Callable, ParamSpec + +class CB(Protocol): + def __call__(self, x: str, y: int) -> None: ... + +P = ParamSpec('P') +def g(fn: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... + +cb: CB +g(cb, y=0, x='a') # OK +g(cb, y='a', x=0) # E: Argument "y" to "g" has incompatible type "str"; expected "int" \ + # E: Argument "x" to "g" has incompatible type "int"; expected "str" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecBadRuntimeTypeApplication] +from typing import ParamSpec, TypeVar, Generic, Callable + +R = TypeVar("R") +P = ParamSpec("P") +class C(Generic[P, R]): + x: Callable[P, R] + +bad = C[int, str]() # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" +reveal_type(bad) # N: Revealed type is "__main__.C[Any, Any]" +reveal_type(bad.x) # N: Revealed type is "def (*Any, **Any) -> Any" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecNoCrashOnUnificationAlias] +import mod +[file mod.pyi] +from typing import Callable, Protocol, TypeVar, overload +from typing_extensions import ParamSpec + +P = ParamSpec("P") +R_co = TypeVar("R_co", covariant=True) +Handler = Callable[P, R_co] + +class HandlerDecorator(Protocol): + def __call__(self, handler: Handler[P, R_co]) -> Handler[P, R_co]: ... + +@overload +def event(event_handler: Handler[P, R_co]) -> Handler[P, R_co]: ... +@overload +def event(namespace: str, *args, **kwargs) -> HandlerDecorator: ... +[builtins fixtures/paramspec.pyi] + +[case testParamSpecNoCrashOnUnificationCallable] +import mod +[file mod.pyi] +from typing import Callable, Protocol, TypeVar, overload +from typing_extensions import ParamSpec + +P = ParamSpec("P") +R_co = TypeVar("R_co", covariant=True) + +class HandlerDecorator(Protocol): + def __call__(self, handler: Callable[P, R_co]) -> Callable[P, R_co]: ... + +@overload +def event(event_handler: Callable[P, R_co]) -> Callable[P, R_co]: ... +@overload +def event(namespace: str, *args, **kwargs) -> HandlerDecorator: ... +[builtins fixtures/paramspec.pyi] + +[case testParamSpecNoCrashOnUnificationPrefix] +from typing import Any, Callable, TypeVar, overload +from typing_extensions import ParamSpec, Concatenate + +T = TypeVar("T") +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") +P = ParamSpec("P") + +@overload +def call( + func: Callable[Concatenate[T, P], U], + x: T, + *args: Any, + **kwargs: Any, +) -> U: ... +@overload +def call( + func: Callable[Concatenate[T, U, P], V], + x: T, + y: U, + *args: Any, + **kwargs: Any, +) -> V: ... +def call(*args: Any, **kwargs: Any) -> Any: ... + +def test1(x: int) -> str: ... +def test2(x: int, y: int) -> str: ... +reveal_type(call(test1, 1)) # N: Revealed type is "builtins.str" +reveal_type(call(test2, 1, 2)) # N: Revealed type is "builtins.str" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecCorrectParameterNameInference] +from typing import Callable, Protocol +from typing_extensions import ParamSpec, Concatenate + +def a(i: int) -> None: ... +def b(__i: int) -> None: ... + +class WithName(Protocol): + def __call__(self, i: int) -> None: ... +NoName = Callable[[int], None] + +def f1(__fn: WithName, i: int) -> None: ... +def f2(__fn: NoName, i: int) -> None: ... + +P = ParamSpec("P") +def d(f: Callable[P, None], fn: Callable[Concatenate[Callable[P, None], P], None]) -> Callable[P, None]: + def inner(*args: P.args, **kwargs: P.kwargs) -> None: + fn(f, *args, **kwargs) + return inner + +reveal_type(d(a, f1)) # N: Revealed type is "def (i: builtins.int)" +reveal_type(d(a, f2)) # N: Revealed type is "def (i: builtins.int)" +reveal_type(d(b, f1)) # E: Cannot infer type argument 1 of "d" \ + # N: Revealed type is "def (*Any, **Any)" +reveal_type(d(b, f2)) # N: Revealed type is "def (builtins.int)" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecGenericWithNamedArg1] +from typing import Callable, TypeVar +from typing_extensions import ParamSpec + +R = TypeVar("R") +P = ParamSpec("P") + +def run(func: Callable[[], R], *args: object, backend: str = "asyncio") -> R: ... +class Result: ... +def run_portal() -> Result: ... +def submit(func: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> R: ... + +reveal_type(submit( # N: Revealed type is "__main__.Result" + run, + run_portal, + backend="asyncio", +)) +submit( + run, # E: Argument 1 to "submit" has incompatible type "Callable[[Callable[[], R], VarArg(object), DefaultNamedArg(str, 'backend')], R]"; expected "Callable[[Callable[[], Result], int], Result]" + run_portal, + backend=int(), +) +[builtins fixtures/paramspec.pyi] + +[case testParamSpecGenericWithNamedArg2] +from typing import Callable, TypeVar, Type +from typing_extensions import ParamSpec + +P= ParamSpec("P") +T = TypeVar("T") + +def smoke_testable(*args: P.args, **kwargs: P.kwargs) -> Callable[[Callable[P, T]], Type[T]]: + ... + +@smoke_testable(name="bob", size=512, flt=0.5) +class SomeClass: + def __init__(self, size: int, name: str, flt: float) -> None: + pass + +# Error message is confusing, but this is a known issue, see #4530. +@smoke_testable(name=42, size="bad", flt=0.5) # E: Argument 1 has incompatible type "Type[OtherClass]"; expected "Callable[[int, str, float], OtherClass]" +class OtherClass: + def __init__(self, size: int, name: str, flt: float) -> None: + pass +[builtins fixtures/paramspec.pyi] + +[case testInferenceAgainstGenericCallableUnionParamSpec] +from typing import Callable, TypeVar, List, Union +from typing_extensions import ParamSpec + +T = TypeVar("T") +P = ParamSpec("P") + +def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... +@dec +def func(arg: T) -> Union[T, str]: + ... +reveal_type(func) # N: Revealed type is "def [T] (arg: T`-1) -> builtins.list[Union[T`-1, builtins.str]]" +reveal_type(func(42)) # N: Revealed type is "builtins.list[Union[builtins.int, builtins.str]]" + +def dec2(f: Callable[P, List[T]]) -> Callable[P, T]: ... +@dec2 +def func2(arg: T) -> List[Union[T, str]]: + ... +reveal_type(func2) # N: Revealed type is "def [T] (arg: T`-1) -> Union[T`-1, builtins.str]" +reveal_type(func2(42)) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/paramspec.pyi] + +[case testParamSpecPreciseKindsUsedIfPossible] +from typing import Callable, Generic +from typing_extensions import ParamSpec + +P = ParamSpec('P') + +class Case(Generic[P]): + def __init__(self, *args: P.args, **kwargs: P.kwargs) -> None: + pass + +def _test(a: int, b: int = 0) -> None: ... + +def parametrize( + func: Callable[P, None], *cases: Case[P], **named_cases: Case[P] +) -> Callable[[], None]: + ... + +parametrize(_test, Case(1, 2), Case(3, 4)) +parametrize(_test, Case(1, b=2), Case(3, b=4)) +parametrize(_test, Case(1, 2), Case(3)) +parametrize(_test, Case(1, 2), Case(3, b=4)) +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecInsufficientArgs] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable + +_P = ParamSpec("_P") + +def run(predicate: Callable[_P, None], *args: _P.args, **kwargs: _P.kwargs) -> None: # N: "run" defined here + predicate() # E: Too few arguments + predicate(*args) # E: Too few arguments + predicate(**kwargs) # E: Too few arguments + predicate(*args, **kwargs) + +def fn() -> None: ... +def fn_args(x: int) -> None: ... +def fn_posonly(x: int, /) -> None: ... + +run(fn) +run(fn_args, 1) +run(fn_args, x=1) +run(fn_posonly, 1) +run(fn_posonly, x=1) # E: Unexpected keyword argument "x" for "run" + +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecConcatenateInsufficientArgs] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable + +_P = ParamSpec("_P") + +def run(predicate: Callable[Concatenate[int, _P], None], *args: _P.args, **kwargs: _P.kwargs) -> None: # N: "run" defined here + predicate() # E: Too few arguments + predicate(1) # E: Too few arguments + predicate(1, *args) # E: Too few arguments + predicate(1, *args) # E: Too few arguments + predicate(1, **kwargs) # E: Too few arguments + predicate(*args, **kwargs) # E: Argument 1 has incompatible type "*_P.args"; expected "int" + predicate(1, *args, **kwargs) + +def fn() -> None: ... +def fn_args(x: int, y: str) -> None: ... +def fn_posonly(x: int, /) -> None: ... +def fn_posonly_args(x: int, /, y: str) -> None: ... + +run(fn) # E: Argument 1 to "run" has incompatible type "Callable[[], None]"; expected "Callable[[int], None]" +run(fn_args, 1, 'a') # E: Too many arguments for "run" \ + # E: Argument 2 to "run" has incompatible type "int"; expected "str" +run(fn_args, y='a') +run(fn_args, 'a') +run(fn_posonly) +run(fn_posonly, x=1) # E: Unexpected keyword argument "x" for "run" +run(fn_posonly_args) # E: Missing positional argument "y" in call to "run" +run(fn_posonly_args, 'a') +run(fn_posonly_args, y='a') + +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecConcatenateInsufficientArgsInDecorator] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable + +P = ParamSpec("P") + +def decorator(fn: Callable[Concatenate[str, P], None]) -> Callable[P, None]: + def inner(*args: P.args, **kwargs: P.kwargs) -> None: + fn("value") # E: Too few arguments + fn("value", *args) # E: Too few arguments + fn("value", **kwargs) # E: Too few arguments + fn(*args, **kwargs) # E: Argument 1 has incompatible type "*P.args"; expected "str" + fn("value", *args, **kwargs) + return inner + +@decorator +def foo(s: str, s2: str) -> None: ... + +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecOverload] +from typing_extensions import ParamSpec +from typing import Callable, NoReturn, TypeVar, Union, overload + +P = ParamSpec("P") +T = TypeVar("T") + +@overload +def capture( + sync_fn: Callable[P, NoReturn], + *args: P.args, + **kwargs: P.kwargs, +) -> int: ... +@overload +def capture( + sync_fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, +) -> Union[T, int]: ... +def capture( + sync_fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, +) -> Union[T, int]: + return sync_fn(*args, **kwargs) + +def fn() -> str: return '' +def err() -> NoReturn: ... + +reveal_type(capture(fn)) # N: Revealed type is "Union[builtins.str, builtins.int]" +reveal_type(capture(err)) # N: Revealed type is "builtins.int" + +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecOverlappingOverloadsOrder] +from typing import Any, Callable, overload +from typing_extensions import ParamSpec + +P = ParamSpec("P") + +class Base: + pass +class Child(Base): + def __call__(self) -> str: ... +class NotChild: + def __call__(self) -> str: ... + +@overload +def handle(func: Base) -> int: ... +@overload +def handle(func: Callable[P, str], *args: P.args, **kwargs: P.kwargs) -> str: ... +def handle(func: Any, *args: Any, **kwargs: Any) -> Any: + return func(*args, **kwargs) + +@overload +def handle_reversed(func: Callable[P, str], *args: P.args, **kwargs: P.kwargs) -> str: ... +@overload +def handle_reversed(func: Base) -> int: ... +def handle_reversed(func: Any, *args: Any, **kwargs: Any) -> Any: + return func(*args, **kwargs) + +reveal_type(handle(Child())) # N: Revealed type is "builtins.int" +reveal_type(handle(NotChild())) # N: Revealed type is "builtins.str" + +reveal_type(handle_reversed(Child())) # N: Revealed type is "builtins.str" +reveal_type(handle_reversed(NotChild())) # N: Revealed type is "builtins.str" + +[builtins fixtures/paramspec.pyi] + +[case testBindPartial] +from functools import partial +from typing_extensions import ParamSpec +from typing import Callable, TypeVar + +P = ParamSpec("P") +T = TypeVar("T") + +def run(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, **kwargs) + return func2(*args) + +def run2(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, *args) + return func2(**kwargs) + +def run3(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, *args, **kwargs) + return func2() + +def run4(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, *args, **kwargs) + return func2(**kwargs) + +def run_bad(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, *args, **kwargs) + return func2(*args) # E: Too many arguments + +def run_bad2(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, **kwargs) + return func2(**kwargs) # E: Too few arguments + +def run_bad3(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, *args) + return func2() # E: Too few arguments + +[builtins fixtures/paramspec.pyi] + +[case testBindPartialConcatenate] +from functools import partial +from typing_extensions import Concatenate, ParamSpec +from typing import Callable, TypeVar + +P = ParamSpec("P") +T = TypeVar("T") + +def run(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, 1, **kwargs) + return func2(*args) + +def run2(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, **kwargs) + p = [""] + func2(1, *p) # E: Too few arguments \ + # E: Argument 2 has incompatible type "*List[str]"; expected "P.args" + func2(1, 2, *p) # E: Too few arguments \ + # E: Argument 2 has incompatible type "int"; expected "P.args" \ + # E: Argument 3 has incompatible type "*List[str]"; expected "P.args" + func2(1, *args, *p) # E: Argument 3 has incompatible type "*List[str]"; expected "P.args" + func2(1, *p, *args) # E: Argument 2 has incompatible type "*List[str]"; expected "P.args" + return func2(1, *args) + +def run3(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, 1, *args) + d = {"":""} + func2(**d) # E: Too few arguments \ + # E: Argument 1 has incompatible type "**Dict[str, str]"; expected "P.kwargs" + return func2(**kwargs) + +def run4(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, 1) + return func2(*args, **kwargs) + +def run5(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, 1, *args, **kwargs) + func2() + return func2(**kwargs) + +def run_bad(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, *args) # E: Argument 1 has incompatible type "*P.args"; expected "int" + return func2(1, **kwargs) # E: Argument 1 has incompatible type "int"; expected "P.args" + +def run_bad2(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, 1, *args) + func2() # E: Too few arguments + func2(*args, **kwargs) # E: Too many arguments + return func2(1, **kwargs) # E: Argument 1 has incompatible type "int"; expected "P.args" + +def run_bad3(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, 1, **kwargs) + func2() # E: Too few arguments + return func2(1, *args) # E: Argument 1 has incompatible type "int"; expected "P.args" + +def run_bad4(func: Callable[Concatenate[int, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, 1) + func2() # E: Too few arguments + func2(*args) # E: Too few arguments + func2(1, *args) # E: Too few arguments \ + # E: Argument 1 has incompatible type "int"; expected "P.args" + func2(1, **kwargs) # E: Too few arguments \ + # E: Argument 1 has incompatible type "int"; expected "P.args" + return func2(**kwargs) # E: Too few arguments + +[builtins fixtures/paramspec.pyi] + +[case testOtherVarArgs] +from functools import partial +from typing_extensions import Concatenate, ParamSpec +from typing import Callable, TypeVar, Tuple + +P = ParamSpec("P") +T = TypeVar("T") + +def run(func: Callable[Concatenate[int, str, P], T], *args: P.args, **kwargs: P.kwargs) -> T: + func2 = partial(func, **kwargs) + args_prefix: Tuple[int, str] = (1, 'a') + func2(*args_prefix) # E: Too few arguments + func2(*args, *args_prefix) # E: Argument 1 has incompatible type "*P.args"; expected "int" \ + # E: Argument 1 has incompatible type "*P.args"; expected "str" \ + # E: Argument 2 has incompatible type "*Tuple[int, str]"; expected "P.args" + return func2(*args_prefix, *args) + +[builtins fixtures/paramspec.pyi] diff --git a/test-data/unit/check-plugin-attrs.test b/test-data/unit/check-plugin-attrs.test new file mode 100644 index 000000000000..0c653d608187 --- /dev/null +++ b/test-data/unit/check-plugin-attrs.test @@ -0,0 +1,2497 @@ +[case testAttrsSimple_no_empty] +import attr +@attr.s +class A: + a = attr.ib() + _b = attr.ib() + c = attr.ib(18) + _d = attr.ib(validator=None, default=18) + E = 18 + + def foo(self): + return self.a +reveal_type(A) # N: Revealed type is "def (a: Any, b: Any, c: Any =, d: Any =) -> __main__.A" +A(1, [2]) +A(1, [2], '3', 4) +A(1, 2, 3, 4) +A(1, [2], '3', 4, 5) # E: Too many arguments for "A" +[builtins fixtures/list.pyi] + +[case testAttrsAnnotated] +import attr +from typing import List, ClassVar +@attr.s +class A: + a: int = attr.ib() + _b: List[int] = attr.ib() + c: str = attr.ib('18') + _d: int = attr.ib(validator=None, default=18) + E = 7 + F: ClassVar[int] = 22 +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.list[builtins.int], c: builtins.str =, d: builtins.int =) -> __main__.A" +A(1, [2]) +A(1, [2], '3', 4) +A(1, 2, 3, 4) # E: Argument 2 to "A" has incompatible type "int"; expected "List[int]" # E: Argument 3 to "A" has incompatible type "int"; expected "str" +A(1, [2], '3', 4, 5) # E: Too many arguments for "A" +[builtins fixtures/list.pyi] + +[case testAttrsTypeComments] +import attr +from typing import List, ClassVar +@attr.s +class A: + a = attr.ib() # type: int + _b = attr.ib() # type: List[int] + c = attr.ib('18') # type: str + _d = attr.ib(validator=None, default=18) # type: int + E = 7 + F: ClassVar[int] = 22 +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.list[builtins.int], c: builtins.str =, d: builtins.int =) -> __main__.A" +A(1, [2]) +A(1, [2], '3', 4) +A(1, 2, 3, 4) # E: Argument 2 to "A" has incompatible type "int"; expected "List[int]" # E: Argument 3 to "A" has incompatible type "int"; expected "str" +A(1, [2], '3', 4, 5) # E: Too many arguments for "A" +[builtins fixtures/list.pyi] + +[case testAttrsAutoAttribs] +import attr +from typing import List, ClassVar +@attr.s(auto_attribs=True) +class A: + a: int + _b: List[int] + c: str = '18' + _d: int = attr.ib(validator=None, default=18) + E = 7 + F: ClassVar[int] = 22 +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.list[builtins.int], c: builtins.str =, d: builtins.int =) -> __main__.A" +A(1, [2]) +A(1, [2], '3', 4) +A(1, 2, 3, 4) # E: Argument 2 to "A" has incompatible type "int"; expected "List[int]" # E: Argument 3 to "A" has incompatible type "int"; expected "str" +A(1, [2], '3', 4, 5) # E: Too many arguments for "A" +[builtins fixtures/list.pyi] + +[case testAttrsUntypedNoUntypedDefs] +# flags: --disallow-untyped-defs +import attr +@attr.s +class A: + a = attr.ib() # E: Need type annotation for "a" + _b = attr.ib() # E: Need type annotation for "_b" + c = attr.ib(18) # E: Need type annotation for "c" + _d = attr.ib(validator=None, default=18) # E: Need type annotation for "_d" + E = 18 +[builtins fixtures/bool.pyi] + +[case testAttrsWrongReturnValue] +import attr +@attr.s +class A: + x: int = attr.ib(8) + def foo(self) -> str: + return self.x # E: Incompatible return value type (got "int", expected "str") +@attr.s +class B: + x = attr.ib(8) # type: int + def foo(self) -> str: + return self.x # E: Incompatible return value type (got "int", expected "str") +@attr.dataclass +class C: + x: int = 8 + def foo(self) -> str: + return self.x # E: Incompatible return value type (got "int", expected "str") +@attr.s +class D: + x = attr.ib(8, type=int) + def foo(self) -> str: + return self.x # E: Incompatible return value type (got "int", expected "str") +[builtins fixtures/bool.pyi] + +[case testAttrsSeriousNames] +from attr import attrib, attrs +from typing import List +@attrs(init=True) +class A: + a = attrib() + _b: List[int] = attrib() + c = attrib(18) + _d = attrib(validator=None, default=18) + CLASS_VAR = 18 +reveal_type(A) # N: Revealed type is "def (a: Any, b: builtins.list[builtins.int], c: Any =, d: Any =) -> __main__.A" +A(1, [2]) +A(1, [2], '3', 4) +A(1, 2, 3, 4) # E: Argument 2 to "A" has incompatible type "int"; expected "List[int]" +A(1, [2], '3', 4, 5) # E: Too many arguments for "A" +[builtins fixtures/list.pyi] + +[case testAttrsDefaultErrors] +import attr +@attr.s +class A: + x = attr.ib(default=17) + y = attr.ib() # E: Non-default attributes not allowed after default attributes. +@attr.s(auto_attribs=True) +class B: + x: int = 17 + y: int # E: Non-default attributes not allowed after default attributes. +@attr.s(auto_attribs=True) +class C: + x: int = attr.ib(default=17) + y: int # E: Non-default attributes not allowed after default attributes. +@attr.s +class D: + x = attr.ib() + y = attr.ib() # E: Non-default attributes not allowed after default attributes. + + @x.default + def foo(self): + return 17 +[builtins fixtures/bool.pyi] + +[case testAttrsNotBooleans] +import attr +x = True +@attr.s(cmp=x) # E: "cmp" argument must be a True, False, or None literal +class A: + a = attr.ib(init=x) # E: "init" argument must be a True or False literal +[builtins fixtures/bool.pyi] + +[case testAttrsInitFalse] +from attr import attrib, attrs +@attrs(auto_attribs=True, init=False) +class A: + a: int + _b: int + c: int = 18 + _d: int = attrib(validator=None, default=18) +reveal_type(A) # N: Revealed type is "def () -> __main__.A" +A() +A(1, [2]) # E: Too many arguments for "A" +A(1, [2], '3', 4) # E: Too many arguments for "A" +[builtins fixtures/list.pyi] + +[case testAttrsInitAttribFalse] +from attr import attrib, attrs +@attrs +class A: + a = attrib(init=False) + b = attrib() +reveal_type(A) # N: Revealed type is "def (b: Any) -> __main__.A" +[builtins fixtures/bool.pyi] + +[case testAttrsCmpTrue] +from attr import attrib, attrs +@attrs(auto_attribs=True) +class A: + a: int +reveal_type(A) # N: Revealed type is "def (a: builtins.int) -> __main__.A" +reveal_type(A.__lt__) # N: Revealed type is "def [_AT] (self: _AT`3, other: _AT`3) -> builtins.bool" +reveal_type(A.__le__) # N: Revealed type is "def [_AT] (self: _AT`4, other: _AT`4) -> builtins.bool" +reveal_type(A.__gt__) # N: Revealed type is "def [_AT] (self: _AT`5, other: _AT`5) -> builtins.bool" +reveal_type(A.__ge__) # N: Revealed type is "def [_AT] (self: _AT`6, other: _AT`6) -> builtins.bool" + +A(1) < A(2) +A(1) <= A(2) +A(1) > A(2) +A(1) >= A(2) +A(1) == A(2) +A(1) != A(2) + +A(1) < 1 # E: Unsupported operand types for < ("A" and "int") +A(1) <= 1 # E: Unsupported operand types for <= ("A" and "int") +A(1) > 1 # E: Unsupported operand types for > ("A" and "int") +A(1) >= 1 # E: Unsupported operand types for >= ("A" and "int") +A(1) == 1 +A(1) != 1 + +1 < A(1) # E: Unsupported operand types for > ("A" and "int") +1 <= A(1) # E: Unsupported operand types for >= ("A" and "int") +1 > A(1) # E: Unsupported operand types for < ("A" and "int") +1 >= A(1) # E: Unsupported operand types for <= ("A" and "int") +1 == A(1) +1 != A(1) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsEqFalse] +from attr import attrib, attrs +@attrs(auto_attribs=True, eq=False) +class A: + a: int +reveal_type(A) # N: Revealed type is "def (a: builtins.int) -> __main__.A" +reveal_type(A.__eq__) # N: Revealed type is "def (builtins.object, builtins.object) -> builtins.bool" +reveal_type(A.__ne__) # N: Revealed type is "def (builtins.object, builtins.object) -> builtins.bool" + +A(1) < A(2) # E: Unsupported left operand type for < ("A") +A(1) <= A(2) # E: Unsupported left operand type for <= ("A") +A(1) > A(2) # E: Unsupported left operand type for > ("A") +A(1) >= A(2) # E: Unsupported left operand type for >= ("A") +A(1) == A(2) +A(1) != A(2) + +A(1) < 1 # E: Unsupported left operand type for < ("A") +A(1) <= 1 # E: Unsupported left operand type for <= ("A") +A(1) > 1 # E: Unsupported left operand type for > ("A") +A(1) >= 1 # E: Unsupported left operand type for >= ("A") +A(1) == 1 +A(1) != 1 + +1 < A(1) # E: Unsupported left operand type for < ("int") +1 <= A(1) # E: Unsupported left operand type for <= ("int") +1 > A(1) # E: Unsupported left operand type for > ("int") +1 >= A(1) # E: Unsupported left operand type for >= ("int") +1 == A(1) +1 != A(1) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsOrderFalse] +from attr import attrib, attrs +@attrs(auto_attribs=True, order=False) +class A: + a: int +reveal_type(A) # N: Revealed type is "def (a: builtins.int) -> __main__.A" + +A(1) < A(2) # E: Unsupported left operand type for < ("A") +A(1) <= A(2) # E: Unsupported left operand type for <= ("A") +A(1) > A(2) # E: Unsupported left operand type for > ("A") +A(1) >= A(2) # E: Unsupported left operand type for >= ("A") +A(1) == A(2) +A(1) != A(2) + +A(1) < 1 # E: Unsupported left operand type for < ("A") +A(1) <= 1 # E: Unsupported left operand type for <= ("A") +A(1) > 1 # E: Unsupported left operand type for > ("A") +A(1) >= 1 # E: Unsupported left operand type for >= ("A") +A(1) == 1 +A(1) != 1 + +1 < A(1) # E: Unsupported left operand type for < ("int") +1 <= A(1) # E: Unsupported left operand type for <= ("int") +1 > A(1) # E: Unsupported left operand type for > ("int") +1 >= A(1) # E: Unsupported left operand type for >= ("int") +1 == A(1) +1 != A(1) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsCmpEqOrderValues] +from attr import attrib, attrs +@attrs(cmp=True) +class DeprecatedTrue: + ... + +@attrs(cmp=False) +class DeprecatedFalse: + ... + +@attrs(cmp=False, eq=True) # E: Don't mix "cmp" with "eq" and "order" +class Mixed: + ... + +@attrs(order=True, eq=False) # E: eq must be True if order is True +class Confused: + ... +[builtins fixtures/plugin_attrs.pyi] + + +[case testAttrsInheritance] +import attr +@attr.s +class A: + a: int = attr.ib() +@attr.s +class B: + b: str = attr.ib() +@attr.s +class C(A, B): + c: bool = attr.ib() +reveal_type(C) # N: Revealed type is "def (a: builtins.int, b: builtins.str, c: builtins.bool) -> __main__.C" +[builtins fixtures/bool.pyi] + +[case testAttrsNestedInClasses] +import attr +@attr.s +class C: + y = attr.ib() + @attr.s + class D: + x: int = attr.ib() +reveal_type(C) # N: Revealed type is "def (y: Any) -> __main__.C" +reveal_type(C.D) # N: Revealed type is "def (x: builtins.int) -> __main__.C.D" +[builtins fixtures/bool.pyi] + +[case testAttrsInheritanceOverride] +import attr + +@attr.s +class A: + a: int = attr.ib() + x: int = attr.ib() + +@attr.s +class B(A): + b: str = attr.ib() + x: int = attr.ib(default=22) + +@attr.s +class C(B): + c: bool = attr.ib() # No error here because the x below overwrites the x above. + x: int = attr.ib() + +reveal_type(A) # N: Revealed type is "def (a: builtins.int, x: builtins.int) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (a: builtins.int, b: builtins.str, x: builtins.int =) -> __main__.B" +reveal_type(C) # N: Revealed type is "def (a: builtins.int, b: builtins.str, c: builtins.bool, x: builtins.int) -> __main__.C" +[builtins fixtures/bool.pyi] + +[case testAttrsTypeEquals] +import attr + +@attr.s +class A: + a = attr.ib(type=int) + b = attr.ib(18, type=int) +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.int =) -> __main__.A" +[builtins fixtures/bool.pyi] + +[case testAttrsFrozen] +import attr + +@attr.s(frozen=True) +class A: + a = attr.ib() + +a = A(5) +a.a = 16 # E: Property "a" defined in "A" is read-only +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsNextGenFrozen] +from attr import frozen, field + +@frozen +class A: + a = field() + +a = A(5) +a.a = 16 # E: Property "a" defined in "A" is read-only +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsNextGenDetect] +from attr import define, field + +@define +class A: + a = field() + +@define +class B: + a: int + +@define +class C: + a: int = field() + b = field() + +@define +class D: + a: int + b = field() + +reveal_type(A) # N: Revealed type is "def (a: Any) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (a: builtins.int) -> __main__.B" +reveal_type(C) # N: Revealed type is "def (a: builtins.int, b: Any) -> __main__.C" +reveal_type(D) # N: Revealed type is "def (b: Any) -> __main__.D" + +[builtins fixtures/bool.pyi] + +[case testAttrsOldPackage] +import attr +@attr.s(auto_attribs=True) +class A: + a: int = attr.ib() + b: bool + +@attr.s(auto_attribs=True, frozen=True) +class B: + a: bool + b: int + +@attr.s +class C: + a = attr.ib(type=int) + +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.bool) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (a: builtins.bool, b: builtins.int) -> __main__.B" +reveal_type(C) # N: Revealed type is "def (a: builtins.int) -> __main__.C" + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsDataClass] +import attr +from typing import List, ClassVar +@attr.dataclass +class A: + a: int + _b: List[str] + c: str = '18' + _d: int = attr.ib(validator=None, default=18) + E = 7 + F: ClassVar[int] = 22 +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.list[builtins.str], c: builtins.str =, d: builtins.int =) -> __main__.A" +A(1, ['2']) +[builtins fixtures/list.pyi] + +[case testAttrsTypeAlias] +from typing import List +import attr +Alias = List[int] +@attr.s(auto_attribs=True) +class A: + Alias2 = List[str] + x: Alias + y: Alias2 = attr.ib() +reveal_type(A) # N: Revealed type is "def (x: builtins.list[builtins.int], y: builtins.list[builtins.str]) -> __main__.A" +[builtins fixtures/list.pyi] + +[case testAttrsGeneric] +from typing import TypeVar, Generic, List +import attr +T = TypeVar('T') +@attr.s(auto_attribs=True) +class A(Generic[T]): + x: List[T] + y: T = attr.ib() + def foo(self) -> List[T]: + return [self.y] + def bar(self) -> T: + return self.x[0] + def problem(self) -> T: + return self.x # E: Incompatible return value type (got "List[T]", expected "T") +reveal_type(A) # N: Revealed type is "def [T] (x: builtins.list[T`1], y: T`1) -> __main__.A[T`1]" +a = A([1], 2) +reveal_type(a) # N: Revealed type is "__main__.A[builtins.int]" +reveal_type(a.x) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(a.y) # N: Revealed type is "builtins.int" + +A(['str'], 7) # E: Cannot infer type argument 1 of "A" +A([1], '2') # E: Cannot infer type argument 1 of "A" + +[builtins fixtures/list.pyi] + +[case testAttrsGenericWithConverter] +from typing import TypeVar, Generic, List, Iterable, Iterator, Callable +import attr +T = TypeVar('T') + +def int_gen() -> Iterator[int]: + yield 1 + +def list_converter(x: Iterable[T]) -> List[T]: + return list(x) + +@attr.s(auto_attribs=True) +class A(Generic[T]): + x: List[T] = attr.ib(converter=list_converter) + y: T = attr.ib() + def foo(self) -> List[T]: + return [self.y] + def bar(self) -> T: + return self.x[0] + def problem(self) -> T: + return self.x # E: Incompatible return value type (got "List[T]", expected "T") +reveal_type(A) # N: Revealed type is "def [T] (x: typing.Iterable[T`1], y: T`1) -> __main__.A[T`1]" +a1 = A([1], 2) +reveal_type(a1) # N: Revealed type is "__main__.A[builtins.int]" +reveal_type(a1.x) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(a1.y) # N: Revealed type is "builtins.int" + +a2 = A(int_gen(), 2) +reveal_type(a2) # N: Revealed type is "__main__.A[builtins.int]" +reveal_type(a2.x) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(a2.y) # N: Revealed type is "builtins.int" + + +def get_int() -> int: + return 1 + +class Other(Generic[T]): + def __init__(self, x: T) -> None: + pass + +@attr.s(auto_attribs=True) +class B(Generic[T]): + x: Other[Callable[..., T]] = attr.ib(converter=Other[Callable[..., T]]) + +b1 = B(get_int) +reveal_type(b1) # N: Revealed type is "__main__.B[builtins.int]" +reveal_type(b1.x) # N: Revealed type is "__main__.Other[def (*Any, **Any) -> builtins.int]" + +[builtins fixtures/list.pyi] + + +[case testAttrsUntypedGenericInheritance] +from typing import Generic, TypeVar +import attr + +T = TypeVar("T") + +@attr.s(auto_attribs=True) +class Base(Generic[T]): + attr: T + +@attr.s(auto_attribs=True) +class Sub(Base): + pass + +sub = Sub(attr=1) +reveal_type(sub) # N: Revealed type is "__main__.Sub" +reveal_type(sub.attr) # N: Revealed type is "Any" + +[builtins fixtures/bool.pyi] + + +[case testAttrsGenericInheritance] +from typing import Generic, TypeVar +import attr + +S = TypeVar("S") +T = TypeVar("T") + +@attr.s(auto_attribs=True) +class Base(Generic[T]): + attr: T + +@attr.s(auto_attribs=True) +class Sub(Base[S]): + pass + +sub_int = Sub[int](attr=1) +reveal_type(sub_int) # N: Revealed type is "__main__.Sub[builtins.int]" +reveal_type(sub_int.attr) # N: Revealed type is "builtins.int" + +sub_str = Sub[str](attr='ok') +reveal_type(sub_str) # N: Revealed type is "__main__.Sub[builtins.str]" +reveal_type(sub_str.attr) # N: Revealed type is "builtins.str" + +[builtins fixtures/bool.pyi] + + +[case testAttrsGenericInheritance2] +from typing import Generic, TypeVar +import attr + +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") + +@attr.s(auto_attribs=True) +class Base(Generic[T1, T2, T3]): + one: T1 + two: T2 + three: T3 + +@attr.s(auto_attribs=True) +class Sub(Base[int, str, float]): + pass + +sub = Sub(one=1, two='ok', three=3.14) +reveal_type(sub) # N: Revealed type is "__main__.Sub" +reveal_type(sub.one) # N: Revealed type is "builtins.int" +reveal_type(sub.two) # N: Revealed type is "builtins.str" +reveal_type(sub.three) # N: Revealed type is "builtins.float" + +[builtins fixtures/bool.pyi] + + +[case testAttrsGenericInheritance3] +import attr +from typing import Any, Callable, Generic, TypeVar, List + +T = TypeVar("T") +S = TypeVar("S") + +@attr.s(auto_attribs=True) +class Parent(Generic[T]): + f: Callable[[T], Any] + +@attr.s(auto_attribs=True) +class Child(Parent[T]): ... + +class A: ... +def func(obj: A) -> bool: ... + +reveal_type(Child[A](func).f) # N: Revealed type is "def (__main__.A) -> Any" + +@attr.s(auto_attribs=True) +class Parent2(Generic[T]): + a: List[T] + +@attr.s(auto_attribs=True) +class Child2(Generic[T, S], Parent2[S]): + b: List[T] + +reveal_type(Child2([A()], [1]).a) # N: Revealed type is "builtins.list[__main__.A]" +reveal_type(Child2[int, A]([A()], [1]).b) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testAttrsMultiGenericInheritance] +from typing import Generic, TypeVar +import attr + +T = TypeVar("T") + +@attr.s(auto_attribs=True, eq=False) +class Base(Generic[T]): + base_attr: T + +S = TypeVar("S") + +@attr.s(auto_attribs=True, eq=False) +class Middle(Base[int], Generic[S]): + middle_attr: S + +@attr.s(auto_attribs=True, eq=False) +class Sub(Middle[str]): + pass + +sub = Sub(base_attr=1, middle_attr='ok') +reveal_type(sub) # N: Revealed type is "__main__.Sub" +reveal_type(sub.base_attr) # N: Revealed type is "builtins.int" +reveal_type(sub.middle_attr) # N: Revealed type is "builtins.str" + +[builtins fixtures/bool.pyi] + + +[case testAttrsGenericClassmethod] +from typing import TypeVar, Generic, Optional +import attr +T = TypeVar('T') +@attr.s(auto_attribs=True) +class A(Generic[T]): + x: Optional[T] + @classmethod + def clsmeth(cls) -> None: + reveal_type(cls) # N: Revealed type is "Type[__main__.A[T`1]]" + +[builtins fixtures/classmethod.pyi] + +[case testAttrsForwardReference] +# flags: --no-strict-optional +import attr +@attr.s(auto_attribs=True) +class A: + parent: 'B' + +@attr.s(auto_attribs=True) +class B: + parent: A + +reveal_type(A) # N: Revealed type is "def (parent: __main__.B) -> __main__.A" +reveal_type(B) # N: Revealed type is "def (parent: __main__.A) -> __main__.B" +A(B(None)) +[builtins fixtures/list.pyi] + +[case testAttrsForwardReferenceInClass] +# flags: --no-strict-optional +import attr +@attr.s(auto_attribs=True) +class A: + parent: A.B + + @attr.s(auto_attribs=True) + class B: + parent: A + +reveal_type(A) # N: Revealed type is "def (parent: __main__.A.B) -> __main__.A" +reveal_type(A.B) # N: Revealed type is "def (parent: __main__.A) -> __main__.A.B" +A(A.B(None)) +[builtins fixtures/list.pyi] + +[case testAttrsImporting] +from helper import A +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.str) -> helper.A" +[file helper.py] +import attr +@attr.s(auto_attribs=True) +class A: + a: int + b: str = attr.ib() +[builtins fixtures/list.pyi] + +[case testAttrsOtherMethods] +import attr +@attr.s(auto_attribs=True) +class A: + a: int + b: str = attr.ib() + @classmethod + def new(cls) -> A: + reveal_type(cls) # N: Revealed type is "Type[__main__.A]" + return cls(6, 'hello') + @classmethod + def bad(cls) -> A: + return cls(17) # E: Missing positional argument "b" in call to "A" + def foo(self) -> int: + return self.a +reveal_type(A) # N: Revealed type is "def (a: builtins.int, b: builtins.str) -> __main__.A" +a = A.new() +reveal_type(a.foo) # N: Revealed type is "def () -> builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testAttrsOtherOverloads] +import attr +from typing import overload, Union + +@attr.s +class A: + a = attr.ib() + b = attr.ib(default=3) + + @classmethod + def other(cls) -> str: + return "..." + + @overload + @classmethod + def foo(cls, x: int) -> int: ... + + @overload + @classmethod + def foo(cls, x: str) -> str: ... + + @classmethod + def foo(cls, x: Union[int, str]) -> Union[int, str]: + reveal_type(cls) # N: Revealed type is "Type[__main__.A]" + reveal_type(cls.other()) # N: Revealed type is "builtins.str" + return x + +reveal_type(A.foo(3)) # N: Revealed type is "builtins.int" +reveal_type(A.foo("foo")) # N: Revealed type is "builtins.str" + +[builtins fixtures/classmethod.pyi] + +[case testAttrsDefaultDecorator] +import attr +@attr.s +class C(object): + x: int = attr.ib(default=1) + y: int = attr.ib() + @y.default + def name_does_not_matter(self): + return self.x + 1 +C() +[builtins fixtures/list.pyi] + +[case testAttrsValidatorDecorator] +import attr +@attr.s +class C(object): + x = attr.ib() + @x.validator + def check(self, attribute, value): + if value > 42: + raise ValueError("x must be smaller or equal to 42") +C(42) +C(43) +[builtins fixtures/exception.pyi] + +[case testAttrsLocalVariablesInClassMethod] +import attr +@attr.s(auto_attribs=True) +class A: + a: int + b: int = attr.ib() + @classmethod + def new(cls, foo: int) -> A: + a = foo + b = a + return cls(a, b) +[builtins fixtures/classmethod.pyi] + +[case testAttrsUnionForward] +import attr +from typing import Union, List + +@attr.s(auto_attribs=True) +class A: + frob: List['AOrB'] + +class B: + pass + +AOrB = Union[A, B] + +reveal_type(A) # N: Revealed type is "def (frob: builtins.list[Union[__main__.A, __main__.B]]) -> __main__.A" +reveal_type(B) # N: Revealed type is "def () -> __main__.B" + +A([B()]) +[builtins fixtures/list.pyi] + +[case testAttrsUsingConvert] +import attr + +def convert(s:int) -> str: + return 'hello' + +@attr.s +class C: + x: str = attr.ib(convert=convert) # E: convert is deprecated, use converter + +# Because of the convert the __init__ takes an int, but the variable is a str. +reveal_type(C) # N: Revealed type is "def (x: builtins.int) -> __main__.C" +reveal_type(C(15).x) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + +[case testAttrsUsingConverter] +import attr +import helper + +def converter2(s:int) -> str: + return 'hello' + +@attr.s +class C: + x: str = attr.ib(converter=helper.converter) + y: str = attr.ib(converter=converter2) + +# Because of the converter the __init__ takes an int, but the variable is a str. +reveal_type(C) # N: Revealed type is "def (x: builtins.int, y: builtins.int) -> __main__.C" +reveal_type(C(15, 16).x) # N: Revealed type is "builtins.str" +[file helper.py] +def converter(s:int) -> str: + return 'hello' +[builtins fixtures/list.pyi] + +[case testAttrsUsingConvertAndConverter] +import attr + +def converter(s:int) -> str: + return 'hello' + +@attr.s +class C: + x: str = attr.ib(converter=converter, convert=converter) # E: Can't pass both "convert" and "converter". + +[builtins fixtures/list.pyi] + +[case testAttrsUsingBadConverter] +# flags: --no-strict-optional +import attr +from typing import overload +@overload +def bad_overloaded_converter(x: int, y: int) -> int: + ... +@overload +def bad_overloaded_converter(x: str, y: str) -> str: + ... +def bad_overloaded_converter(x, y=7): + return x +def bad_converter() -> str: + return '' +@attr.dataclass +class A: + bad: str = attr.ib(converter=bad_converter) + bad_overloaded: int = attr.ib(converter=bad_overloaded_converter) +reveal_type(A) +[out] +main:16: error: Cannot determine __init__ type from converter +main:16: error: Argument "converter" has incompatible type "Callable[[], str]"; expected "Callable[[Any], str]" +main:17: error: Cannot determine __init__ type from converter +main:17: error: Argument "converter" has incompatible type overloaded function; expected "Callable[[Any], int]" +main:18: note: Revealed type is "def (bad: Any, bad_overloaded: Any) -> __main__.A" +[builtins fixtures/list.pyi] + +[case testAttrsUsingBadConverterReprocess] +# flags: --no-strict-optional +import attr +from typing import overload +forward: 'A' +@overload +def bad_overloaded_converter(x: int, y: int) -> int: + ... +@overload +def bad_overloaded_converter(x: str, y: str) -> str: + ... +def bad_overloaded_converter(x, y=7): + return x +def bad_converter() -> str: + return '' +@attr.dataclass +class A: + bad: str = attr.ib(converter=bad_converter) + bad_overloaded: int = attr.ib(converter=bad_overloaded_converter) +reveal_type(A) +[out] +main:17: error: Cannot determine __init__ type from converter +main:17: error: Argument "converter" has incompatible type "Callable[[], str]"; expected "Callable[[Any], str]" +main:18: error: Cannot determine __init__ type from converter +main:18: error: Argument "converter" has incompatible type overloaded function; expected "Callable[[Any], int]" +main:19: note: Revealed type is "def (bad: Any, bad_overloaded: Any) -> __main__.A" +[builtins fixtures/list.pyi] + +[case testAttrsUsingUnsupportedConverter] +import attr +class Thing: + def do_it(self, int) -> str: + ... +thing = Thing() +def factory(default: int): + ... +@attr.s +class C: + x: str = attr.ib(converter=thing.do_it) # E: Unsupported converter, only named functions, types and lambdas are currently supported + y: str = attr.ib(converter=lambda x: x) + z: str = attr.ib(converter=factory(8)) # E: Unsupported converter, only named functions, types and lambdas are currently supported +reveal_type(C) # N: Revealed type is "def (x: Any, y: Any, z: Any) -> __main__.C" +[builtins fixtures/list.pyi] + +[case testAttrsUsingConverterAndSubclass] +import attr + +def converter(s:int) -> str: + return 'hello' + +@attr.s +class C: + x: str = attr.ib(converter=converter) + +@attr.s +class A(C): + pass + +# Because of the convert the __init__ takes an int, but the variable is a str. +reveal_type(A) # N: Revealed type is "def (x: builtins.int) -> __main__.A" +reveal_type(A(15).x) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + +[case testAttrsUsingConverterWithTypes] +from typing import overload +import attr + +@attr.dataclass +class A: + x: str + +@attr.s +class C: + x: complex = attr.ib(converter=complex) + y: int = attr.ib(converter=int) + z: A = attr.ib(converter=A) + +o = C("1", "2", "3") +o = C(1, 2, "3") +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsCmpWithSubclasses] +import attr +@attr.s +class A: pass +@attr.s +class B: pass +@attr.s +class C(A, B): pass +@attr.s +class D(A): pass + +reveal_type(A.__lt__) # N: Revealed type is "def [_AT] (self: _AT`5, other: _AT`5) -> builtins.bool" +reveal_type(B.__lt__) # N: Revealed type is "def [_AT] (self: _AT`6, other: _AT`6) -> builtins.bool" +reveal_type(C.__lt__) # N: Revealed type is "def [_AT] (self: _AT`7, other: _AT`7) -> builtins.bool" +reveal_type(D.__lt__) # N: Revealed type is "def [_AT] (self: _AT`8, other: _AT`8) -> builtins.bool" + +A() < A() +B() < B() +A() < B() # E: Unsupported operand types for < ("A" and "B") + +C() > A() +C() > B() +C() > C() +C() > D() # E: Unsupported operand types for > ("C" and "D") + +D() >= A() +D() >= B() # E: Unsupported operand types for >= ("D" and "B") +D() >= C() # E: Unsupported operand types for >= ("D" and "C") +D() >= D() + +A() <= 1 # E: Unsupported operand types for <= ("A" and "int") +B() <= 1 # E: Unsupported operand types for <= ("B" and "int") +C() <= 1 # E: Unsupported operand types for <= ("C" and "int") +D() <= 1 # E: Unsupported operand types for <= ("D" and "int") + +[builtins fixtures/list.pyi] + +[case testAttrsComplexSuperclass] +import attr +@attr.s +class C: + x: int = attr.ib(default=1) + y: int = attr.ib() + @y.default + def name_does_not_matter(self): + return self.x + 1 +@attr.s +class A(C): + z: int = attr.ib(default=18) +reveal_type(C) # N: Revealed type is "def (x: builtins.int =, y: builtins.int =) -> __main__.C" +reveal_type(A) # N: Revealed type is "def (x: builtins.int =, y: builtins.int =, z: builtins.int =) -> __main__.A" +[builtins fixtures/list.pyi] + +[case testAttrsMultiAssign] +import attr +@attr.s +class A: + x, y, z = attr.ib(), attr.ib(type=int), attr.ib(default=17) +reveal_type(A) # N: Revealed type is "def (x: Any, y: builtins.int, z: Any =) -> __main__.A" +[builtins fixtures/list.pyi] + +[case testAttrsMultiAssign2] +import attr +@attr.s +class A: + x = y = z = attr.ib() # E: Too many names for one attribute +[builtins fixtures/list.pyi] + +[case testAttrsPrivateInit] +import attr +@attr.s +class C(object): + _x = attr.ib(init=False, default=42) +C() +C(_x=42) # E: Unexpected keyword argument "_x" for "C" +[builtins fixtures/list.pyi] + +[case testAttrsAliasForInit] +from attrs import define, field + +@define +class C1: + _x: int = field(alias="x1") + +c1 = C1(x1=42) +reveal_type(c1._x) # N: Revealed type is "builtins.int" +c1.x1 # E: "C1" has no attribute "x1" +C1(_x=42) # E: Unexpected keyword argument "_x" for "C1" + +alias = "x2" +@define +class C2: + _x: int = field(alias=alias) # E: "alias" argument to attrs field must be a string literal + +@define +class C3: + _x: int = field(alias="_x") + +c3 = C3(_x=1) +reveal_type(c3._x) # N: Revealed type is "builtins.int" +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsAutoMustBeAll] +import attr +@attr.s(auto_attribs=True) +class A: + a: int + b = 17 + # The following forms are not allowed with auto_attribs=True + c = attr.ib() # E: Need type annotation for "c" + d, e = attr.ib(), attr.ib() # E: Need type annotation for "d" # E: Need type annotation for "e" + f = g = attr.ib() # E: Need type annotation for "f" # E: Need type annotation for "g" +[builtins fixtures/bool.pyi] + +[case testAttrsRepeatedName] +import attr +@attr.s +class A: + a = attr.ib(default=8) + b = attr.ib() + a = attr.ib() +reveal_type(A) # N: Revealed type is "def (b: Any, a: Any) -> __main__.A" +@attr.s +class B: + a: int = attr.ib(default=8) + b: int = attr.ib() + a: int = attr.ib() # E: Name "a" already defined on line 10 +reveal_type(B) # N: Revealed type is "def (b: builtins.int, a: builtins.int) -> __main__.B" +@attr.s(auto_attribs=True) +class C: + a: int = 8 + b: int + a: int = attr.ib() # E: Name "a" already defined on line 16 +reveal_type(C) # N: Revealed type is "def (a: builtins.int, b: builtins.int) -> __main__.C" +[builtins fixtures/bool.pyi] + +[case testAttrsFrozenSubclass] +import attr + +@attr.dataclass +class NonFrozenBase: + a: int + +@attr.dataclass(frozen=True) +class FrozenBase: + a: int + +@attr.dataclass(frozen=True) +class FrozenNonFrozen(NonFrozenBase): + b: int + +@attr.dataclass(frozen=True) +class FrozenFrozen(FrozenBase): + b: int + +@attr.dataclass +class NonFrozenFrozen(FrozenBase): + b: int + +# Make sure these are untouched +non_frozen_base = NonFrozenBase(1) +non_frozen_base.a = 17 +frozen_base = FrozenBase(1) +frozen_base.a = 17 # E: Property "a" defined in "FrozenBase" is read-only + +a = FrozenNonFrozen(1, 2) +a.a = 17 # E: Property "a" defined in "FrozenNonFrozen" is read-only +a.b = 17 # E: Property "b" defined in "FrozenNonFrozen" is read-only + +b = FrozenFrozen(1, 2) +b.a = 17 # E: Property "a" defined in "FrozenFrozen" is read-only +b.b = 17 # E: Property "b" defined in "FrozenFrozen" is read-only + +c = NonFrozenFrozen(1, 2) +c.a = 17 # E: Property "a" defined in "NonFrozenFrozen" is read-only +c.b = 17 # E: Property "b" defined in "NonFrozenFrozen" is read-only + +[builtins fixtures/plugin_attrs.pyi] +[case testAttrsCallableAttributes] +from typing import Callable +import attr +def blah(a: int, b: int) -> bool: + return True + +@attr.s(auto_attribs=True) +class F: + _cb: Callable[[int, int], bool] = blah + def foo(self) -> bool: + return self._cb(5, 6) + +@attr.s +class G: + _cb: Callable[[int, int], bool] = attr.ib(blah) + def foo(self) -> bool: + return self._cb(5, 6) + +@attr.s(auto_attribs=True, frozen=True) +class FFrozen(F): + def bar(self) -> bool: + return self._cb(5, 6) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsWithFactory] +from typing import List +import attr +def my_factory() -> int: + return 7 +@attr.s +class A: + x: List[int] = attr.ib(factory=list) + y: int = attr.ib(factory=my_factory) +A() +[builtins fixtures/list.pyi] + +[case testAttrsFactoryAndDefault] +import attr +@attr.s +class A: + x: int = attr.ib(factory=int, default=7) # E: Can't pass both "default" and "factory". +[builtins fixtures/bool.pyi] + +[case testAttrsFactoryBadReturn] +# flags: --new-type-inference +import attr +def my_factory() -> int: + return 7 +@attr.s +class A: + x: int = attr.ib(factory=list) # E: Incompatible types in assignment (expression has type "List[Never]", variable has type "int") + y: str = attr.ib(factory=my_factory) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/list.pyi] + +[case testAttrsDefaultAndInit] +import attr + +@attr.s +class C: + a = attr.ib(init=False, default=42) + b = attr.ib() # Ok because previous attribute is init=False + c = attr.ib(default=44) + d = attr.ib(init=False) # Ok because this attribute is init=False + e = attr.ib() # E: Non-default attributes not allowed after default attributes. + +[builtins fixtures/bool.pyi] + +[case testAttrsOptionalConverter] +import attr +from attr.converters import optional +from typing import Optional + +def converter(s:int) -> str: + return 'hello' + + +@attr.s +class A: + y: Optional[int] = attr.ib(converter=optional(int)) + z: Optional[str] = attr.ib(converter=optional(converter)) + + +A(None, None) + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsOptionalConverterNewPackage] +import attrs +from attrs.converters import optional +from typing import Optional + +def converter(s:int) -> str: + return 'hello' + + +@attrs.define +class A: + y: Optional[int] = attrs.field(converter=optional(int)) + z: Optional[str] = attrs.field(converter=optional(converter)) + + +A(None, None) + +[builtins fixtures/plugin_attrs.pyi] + + +[case testAttrsTypeVarNoCollision] +from typing import TypeVar, Generic +import attr + +T = TypeVar("T", bytes, str) + +# Make sure the generated __le__ (and friends) don't use T for their arguments. +@attr.s(auto_attribs=True) +class A(Generic[T]): + v: T +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsKwOnlyAttrib] +import attr +@attr.s +class A: + a = attr.ib(kw_only=True) +A() # E: Missing named argument "a" for "A" +A(15) # E: Too many positional arguments for "A" +A(a=15) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsKwOnlyClass] +import attr +@attr.s(kw_only=True, auto_attribs=True) +class A: + a: int + b: bool +A() # E: Missing named argument "a" for "A" # E: Missing named argument "b" for "A" +A(b=True, a=15) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsKwOnlyClassNoInit] +import attr +@attr.s(kw_only=True) +class B: + a = attr.ib(init=False) + b = attr.ib() +B(b=True) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsKwOnlyWithDefault] +import attr +@attr.s +class C: + a = attr.ib(0) + b = attr.ib(kw_only=True) + c = attr.ib(16, kw_only=True) +C(b=17) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsKwOnlyClassWithMixedDefaults] +import attr +@attr.s(kw_only=True) +class D: + a = attr.ib(10) + b = attr.ib() + c = attr.ib(15) +D(b=17) +[builtins fixtures/plugin_attrs.pyi] + + +[case testAttrsKwOnlySubclass] +import attr +@attr.s +class A2: + a = attr.ib(default=0) +@attr.s +class B2(A2): + b = attr.ib(kw_only=True) +B2(b=1) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsNonKwOnlyAfterKwOnly] +import attr +@attr.s(kw_only=True) +class A: + a = attr.ib(default=0) +@attr.s +class B(A): + b = attr.ib() +@attr.s +class C: + a = attr.ib(kw_only=True) + b = attr.ib(15) + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsDisallowUntypedWorksForward] +# flags: --disallow-untyped-defs +import attr +from typing import List + +@attr.s +class B: + x: C = attr.ib() + +class C(List[C]): + pass + +reveal_type(B) # N: Revealed type is "def (x: __main__.C) -> __main__.B" +[builtins fixtures/list.pyi] + +[case testDisallowUntypedWorksForwardBad] +# flags: --disallow-untyped-defs +import attr + +@attr.s +class B: + x = attr.ib() # E: Need type annotation for "x" + +reveal_type(B) # N: Revealed type is "def (x: Any) -> __main__.B" +[builtins fixtures/list.pyi] + +[case testAttrsDefaultDecoratorDeferred] +defer: Yes + +import attr +@attr.s +class C(object): + x: int = attr.ib(default=1) + y: int = attr.ib() + @y.default + def inc(self): + return self.x + 1 + +class Yes: ... +[builtins fixtures/list.pyi] + +[case testAttrsValidatorDecoratorDeferred] +defer: Yes + +import attr +@attr.s +class C(object): + x = attr.ib() + @x.validator + def check(self, attribute, value): + if value > 42: + raise ValueError("x must be smaller or equal to 42") +C(42) +C(43) + +class Yes: ... +[builtins fixtures/exception.pyi] + +[case testTypeInAttrUndefined] +import attr + +@attr.s +class C: + total = attr.ib(type=Bad) # E: Name "Bad" is not defined +[builtins fixtures/bool.pyi] + +[case testTypeInAttrForwardInRuntime] +import attr + +@attr.s +class C: + total = attr.ib(type=Forward) + +reveal_type(C.total) # N: Revealed type is "__main__.Forward" +C('no') # E: Argument 1 to "C" has incompatible type "str"; expected "Forward" +class Forward: ... +[builtins fixtures/bool.pyi] + +[case testDefaultInAttrForward] +import attr + +@attr.s +class C: + total = attr.ib(default=func()) + +def func() -> int: ... + +C() +C(1) +C(1, 2) # E: Too many arguments for "C" +[builtins fixtures/bool.pyi] + +[case testTypeInAttrUndefinedFrozen] +import attr + +@attr.s(frozen=True) +class C: + total = attr.ib(type=Bad) # E: Name "Bad" is not defined + +C(0).total = 1 # E: Property "total" defined in "C" is read-only +[builtins fixtures/plugin_attrs.pyi] + +[case testTypeInAttrDeferredStar] +import lib +[file lib.py] +import attr +MYPY = False +if MYPY: # Force deferral + from other import * + +@attr.s +class C: + total = attr.ib(type=int) + +C() # E: Missing positional argument "total" in call to "C" +C('no') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +[file other.py] +import lib +[builtins fixtures/bool.pyi] + +[case testAttrsDefaultsMroOtherFile] +import a + +[file a.py] +import attr +from b import A1, A2 + +@attr.s +class Asdf(A1, A2): # E: Non-default attributes not allowed after default attributes. + pass + +[file b.py] +import attr + +@attr.s +class A1: + a: str = attr.ib('test') + +@attr.s +class A2: + b: int = attr.ib() + +[builtins fixtures/list.pyi] + +[case testAttrsInheritanceNoAnnotation] +import attr + +@attr.s +class A: + foo = attr.ib() # type: int + +x = 0 +@attr.s +class B(A): + foo = x + +reveal_type(B) # N: Revealed type is "def (foo: builtins.int) -> __main__.B" +[builtins fixtures/bool.pyi] + +[case testAttrsClassHasMagicAttribute] +import attr + +@attr.s +class A: + b: int = attr.ib() + c: str = attr.ib() + +reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A.____main___A_AttrsAttributes__]" +reveal_type(A.__attrs_attrs__[0]) # N: Revealed type is "attr.Attribute[builtins.int]" +reveal_type(A.__attrs_attrs__.b) # N: Revealed type is "attr.Attribute[builtins.int]" +A.__attrs_attrs__.x # E: "____main___A_AttrsAttributes__" has no attribute "x" + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsBareClassHasMagicAttribute] +import attr + +@attr.s +class A: + b = attr.ib() + c = attr.ib() + +reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[Any], attr.Attribute[Any], fallback=__main__.A.____main___A_AttrsAttributes__]" +reveal_type(A.__attrs_attrs__[0]) # N: Revealed type is "attr.Attribute[Any]" +reveal_type(A.__attrs_attrs__.b) # N: Revealed type is "attr.Attribute[Any]" +A.__attrs_attrs__.x # E: "____main___A_AttrsAttributes__" has no attribute "x" + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsNGClassHasMagicAttribute] +import attr + +@attr.define +class A: + b: int + c: str + +reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A.____main___A_AttrsAttributes__]" +reveal_type(A.__attrs_attrs__[0]) # N: Revealed type is "attr.Attribute[builtins.int]" +reveal_type(A.__attrs_attrs__.b) # N: Revealed type is "attr.Attribute[builtins.int]" +A.__attrs_attrs__.x # E: "____main___A_AttrsAttributes__" has no attribute "x" + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsMagicAttributeProtocol] +import attr +from typing import Any, Protocol, Type, ClassVar + +class AttrsInstance(Protocol): + __attrs_attrs__: ClassVar[Any] + +@attr.define +class A: + b: int + c: str + +def takes_attrs_cls(cls: Type[AttrsInstance]) -> None: + pass + +def takes_attrs_instance(inst: AttrsInstance) -> None: + pass + +takes_attrs_cls(A) +takes_attrs_instance(A(1, "")) + +takes_attrs_cls(A(1, "")) # E: Argument 1 to "takes_attrs_cls" has incompatible type "A"; expected "Type[AttrsInstance]" +takes_attrs_instance(A) # E: Argument 1 to "takes_attrs_instance" has incompatible type "Type[A]"; expected "AttrsInstance" # N: ClassVar protocol member AttrsInstance.__attrs_attrs__ can never be matched by a class object +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsFields] +import attr +from attrs import fields as f # Common usage. + +@attr.define +class A: + b: int + c: str + +reveal_type(f(A)) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A.____main___A_AttrsAttributes__]" +reveal_type(f(A)[0]) # N: Revealed type is "attr.Attribute[builtins.int]" +reveal_type(f(A).b) # N: Revealed type is "attr.Attribute[builtins.int]" +f(A).x # E: "____main___A_AttrsAttributes__" has no attribute "x" + +for ff in f(A): + reveal_type(ff) # N: Revealed type is "attr.Attribute[Any]" + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsGenericFields] +from typing import TypeVar + +import attr +from attrs import fields + +@attr.define +class A: + b: int + c: str + +TA = TypeVar('TA', bound=A) + +def f(t: TA) -> None: + reveal_type(fields(t)) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A.____main___A_AttrsAttributes__]" + reveal_type(fields(t)[0]) # N: Revealed type is "attr.Attribute[builtins.int]" + reveal_type(fields(t).b) # N: Revealed type is "attr.Attribute[builtins.int]" + fields(t).x # E: "____main___A_AttrsAttributes__" has no attribute "x" + + +[builtins fixtures/plugin_attrs.pyi] + +[case testNonattrsFields] +from typing import Any, cast, Type +from attrs import fields, has + +class A: + b: int + c: str + +if has(A): + fields(A) +else: + fields(A) # E: Argument 1 to "fields" has incompatible type "Type[A]"; expected "Type[AttrsInstance]" +fields(None) # E: Argument 1 to "fields" has incompatible type "None"; expected "Type[AttrsInstance]" +fields(cast(Any, 42)) +fields(cast(Type[Any], 43)) + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsInitMethodAlwaysGenerates] +from typing import Tuple +import attr + +@attr.define(init=False) +class A: + b: int + c: str + def __init__(self, bc: Tuple[int, str]) -> None: + b, c = bc + self.__attrs_init__(b, c) + +reveal_type(A) # N: Revealed type is "def (bc: Tuple[builtins.int, builtins.str]) -> __main__.A" +reveal_type(A.__init__) # N: Revealed type is "def (self: __main__.A, bc: Tuple[builtins.int, builtins.str])" +reveal_type(A.__attrs_init__) # N: Revealed type is "def (self: __main__.A, b: builtins.int, c: builtins.str)" + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsClassWithSlots] +import attr + +@attr.define +class Define: + b: int = attr.ib() + + def __attrs_post_init__(self) -> None: + self.b = 1 + self.c = 2 # E: Trying to assign name "c" that is not in "__slots__" of type "__main__.Define" + + +@attr.define(slots=False) +class DefineSlotsFalse: + b: int = attr.ib() + + def __attrs_post_init__(self) -> None: + self.b = 1 + self.c = 2 + + +@attr.s(slots=True) +class A: + b: int = attr.ib() + + def __attrs_post_init__(self) -> None: + self.b = 1 + self.c = 2 # E: Trying to assign name "c" that is not in "__slots__" of type "__main__.A" + +@attr.dataclass(slots=True) +class B: + __slots__ = () # would be replaced + b: int + + def __attrs_post_init__(self) -> None: + self.b = 1 + self.c = 2 # E: Trying to assign name "c" that is not in "__slots__" of type "__main__.B" + +@attr.dataclass(slots=False) +class C: + __slots__ = () # would not be replaced + b: int + + def __attrs_post_init__(self) -> None: + self.b = 1 # E: Trying to assign name "b" that is not in "__slots__" of type "__main__.C" + self.c = 2 # E: Trying to assign name "c" that is not in "__slots__" of type "__main__.C" +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsClassWithSlotsDerivedFromNonSlots] +import attrs + +class A: + pass + +@attrs.define(slots=True) +class B(A): + x: int + + def __attrs_post_init__(self) -> None: + self.y = 42 + +[builtins fixtures/plugin_attrs.pyi] + +[case testRuntimeSlotsAttr] +from attr import dataclass + +@dataclass(slots=True) +class Some: + x: int + y: str + z: bool + +reveal_type(Some.__slots__) # N: Revealed type is "Tuple[builtins.str, builtins.str, builtins.str]" + +@dataclass(slots=True) +class Other: + x: int + y: str + +reveal_type(Other.__slots__) # N: Revealed type is "Tuple[builtins.str, builtins.str]" + + +@dataclass +class NoSlots: + x: int + y: str + +NoSlots.__slots__ # E: "Type[NoSlots]" has no attribute "__slots__" +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsWithMatchArgs] +# flags: --python-version 3.10 +import attr + +@attr.s(match_args=True, auto_attribs=True) +class ToMatch: + x: int + y: int + # Not included: + z: int = attr.field(kw_only=True) + i: int = attr.field(init=False) + +reveal_type(ToMatch(x=1, y=2, z=3).__match_args__) # N: Revealed type is "Tuple[Literal['x']?, Literal['y']?]" +reveal_type(ToMatch(1, 2, z=3).__match_args__) # N: Revealed type is "Tuple[Literal['x']?, Literal['y']?]" +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsWithMatchArgsDefaultCase] +# flags: --python-version 3.10 +import attr + +@attr.s(auto_attribs=True) +class ToMatch1: + x: int + y: int + +t1: ToMatch1 +reveal_type(t1.__match_args__) # N: Revealed type is "Tuple[Literal['x']?, Literal['y']?]" + +@attr.define +class ToMatch2: + x: int + y: int + +t2: ToMatch2 +reveal_type(t2.__match_args__) # N: Revealed type is "Tuple[Literal['x']?, Literal['y']?]" +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsWithMatchArgsOverrideExisting] +# flags: --python-version 3.10 +import attr +from typing import Final + +@attr.s(match_args=True, auto_attribs=True) +class ToMatch: + __match_args__: Final = ('a', 'b') + x: int + y: int + +# It works the same way runtime does: +reveal_type(ToMatch(x=1, y=2).__match_args__) # N: Revealed type is "Tuple[Literal['a']?, Literal['b']?]" + +@attr.s(auto_attribs=True) +class WithoutMatch: + __match_args__: Final = ('a', 'b') + x: int + y: int + +reveal_type(WithoutMatch(x=1, y=2).__match_args__) # N: Revealed type is "Tuple[Literal['a']?, Literal['b']?]" +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsWithMatchArgsOldVersion] +# flags: --python-version 3.9 +import attr + +@attr.s(match_args=True) +class NoMatchArgs: + ... + +n: NoMatchArgs + +reveal_type(n.__match_args__) # E: "NoMatchArgs" has no attribute "__match_args__" \ + # N: Revealed type is "Any" +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsMultipleInheritance] +# flags: --python-version 3.10 +import attr + +@attr.s +class A: + x = attr.ib(type=int) + +@attr.s +class B: + y = attr.ib(type=int) + +class AB(A, B): + pass +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsForwardReferenceInTypeVarBound] +from typing import TypeVar, Generic +import attr + +T = TypeVar("T", bound="C") + +@attr.define +class D(Generic[T]): + x: int + +class C: + pass +[builtins fixtures/plugin_attrs.pyi] + +[case testComplexTypeInAttrIb] +import a + +[file a.py] +import attr +import b +from typing import Callable + +@attr.s +class C: + a = attr.ib(type=Lst[int]) + # Note that for this test, the 'Value of type "int" is not indexable' errors are silly, + # and a consequence of Callable etc. being set to an int in the test stub. + b = attr.ib(type=Callable[[], C]) +[file b.py] +import attr +import a +from typing import List as Lst, Optional + +@attr.s +class D: + a = attr.ib(type=Lst[int]) + b = attr.ib(type=Optional[int]) +[builtins fixtures/list.pyi] +[out] +tmp/b.py:8: error: Value of type "int" is not indexable +tmp/a.py:7: error: Name "Lst" is not defined +tmp/a.py:10: error: Value of type "int" is not indexable + +[case testAttrsGenericInheritanceSpecialCase1] +import attr +from typing import Generic, TypeVar, List + +T = TypeVar("T") + +@attr.define +class Parent(Generic[T]): + x: List[T] + +@attr.define +class Child1(Parent["Child2"]): ... + +@attr.define +class Child2(Parent["Child1"]): ... + +def f(c: Child2) -> None: + reveal_type(Child1([c]).x) # N: Revealed type is "builtins.list[__main__.Child2]" + +def g(c: Child1) -> None: + reveal_type(Child2([c]).x) # N: Revealed type is "builtins.list[__main__.Child1]" +[builtins fixtures/list.pyi] + +[case testAttrsGenericInheritanceSpecialCase2] +import attr +from typing import Generic, TypeVar + +T = TypeVar("T") + +# A subclass might be analyzed before base in import cycles. They are +# defined here in reversed order to simulate this. + +@attr.define +class Child1(Parent["Child2"]): + x: int + +@attr.define +class Child2(Parent["Child1"]): + y: int + +@attr.define +class Parent(Generic[T]): + key: str + +Child1(x=1, key='') +Child2(y=1, key='') +[builtins fixtures/list.pyi] + +[case testAttrsUnsupportedConverterWithDisallowUntypedDefs] +# flags: --disallow-untyped-defs +import attr +from typing import Mapping, Any, Union + +def default_if_none(factory: Any) -> Any: pass + +@attr.s(slots=True, frozen=True) +class C: + name: Union[str, None] = attr.ib(default=None) + options: Mapping[str, Mapping[str, Any]] = attr.ib( + default=None, converter=default_if_none(factory=dict) \ + # E: Unsupported converter, only named functions, types and lambdas are currently supported + ) +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsUnannotatedConverter] +import attr + +def foo(value): + return value.split() + +@attr.s +class Bar: + field = attr.ib(default=None, converter=foo) + +reveal_type(Bar) # N: Revealed type is "def (field: Any =) -> __main__.Bar" +bar = Bar("Hello") +reveal_type(bar.field) # N: Revealed type is "Any" + +[builtins fixtures/tuple.pyi] + +[case testAttrsLambdaConverter] +import attr + +@attr.s +class Bar: + name: str = attr.ib(converter=lambda s: s.lower()) + +reveal_type(Bar) # N: Revealed type is "def (name: Any) -> __main__.Bar" +bar = Bar("Hello") +reveal_type(bar.name) # N: Revealed type is "builtins.str" + +[builtins fixtures/tuple.pyi] + +[case testAttrsNestedClass] +from typing import List +import attr + +@attr.s +class C: + @attr.s + class D: + pass + x = attr.ib(type=List[D]) + +c = C(x=[C.D()]) +reveal_type(c.x) # N: Revealed type is "builtins.list[__main__.C.D]" +[builtins fixtures/list.pyi] + +[case testRedefinitionInFrozenClassNoCrash] +import attr + +@attr.s +class MyData: + is_foo: bool = attr.ib() + + @staticmethod # E: Name "is_foo" already defined on line 5 + def is_foo(string: str) -> bool: ... +[builtins fixtures/classmethod.pyi] + +[case testOverrideWithPropertyInFrozenClassNoCrash] +from attrs import frozen + +@frozen(kw_only=True) +class Base: + name: str + +@frozen(kw_only=True) +class Sub(Base): + first_name: str + last_name: str + + @property + def name(self) -> str: ... +[builtins fixtures/plugin_attrs.pyi] + +[case testOverrideWithPropertyInFrozenClassChecked] +from attrs import frozen + +@frozen(kw_only=True) +class Base: + name: str + +@frozen(kw_only=True) +class Sub(Base): + first_name: str + last_name: str + + @property + def name(self) -> int: ... # E: Signature of "name" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: str \ + # N: Subclass: \ + # N: int + +# This matches runtime semantics +reveal_type(Sub) # N: Revealed type is "def (*, name: builtins.str, first_name: builtins.str, last_name: builtins.str) -> __main__.Sub" +[builtins fixtures/plugin_attrs.pyi] + +[case testFinalInstanceAttribute] +from attrs import define +from typing import Final + +@define +class C: + a: Final[int] + +reveal_type(C) # N: Revealed type is "def (a: builtins.int) -> __main__.C" + +C(1).a = 2 # E: Cannot assign to final attribute "a" + +[builtins fixtures/property.pyi] + +[case testFinalInstanceAttributeInheritance] +from attrs import define +from typing import Final + +@define +class C: + a: Final[int] + +@define +class D(C): + b: Final[str] + +reveal_type(D) # N: Revealed type is "def (a: builtins.int, b: builtins.str) -> __main__.D" + +D(1, "").a = 2 # E: Cannot assign to final attribute "a" +D(1, "").b = "2" # E: Cannot assign to final attribute "b" + +[builtins fixtures/property.pyi] + +[case testEvolve] +import attr + +class Base: + pass + +class Derived(Base): + pass + +class Other: + pass + +@attr.s(auto_attribs=True) +class C: + name: str + b: Base + +c = C(name='foo', b=Derived()) +c = attr.evolve(c) +c = attr.evolve(c, name='foo') +c = attr.evolve(c, 'foo') # E: Too many positional arguments for "evolve" of "C" +c = attr.evolve(c, b=Derived()) +c = attr.evolve(c, b=Base()) +c = attr.evolve(c, b=Other()) # E: Argument "b" to "evolve" of "C" has incompatible type "Other"; expected "Base" +c = attr.evolve(c, name=42) # E: Argument "name" to "evolve" of "C" has incompatible type "int"; expected "str" +c = attr.evolve(c, foobar=42) # E: Unexpected keyword argument "foobar" for "evolve" of "C" + +# test passing instance as 'inst' kw +c = attr.evolve(inst=c, name='foo') +c = attr.evolve(not_inst=c, name='foo') # E: Missing positional argument "inst" in call to "evolve" + +# test determining type of first argument's expression from something that's not NameExpr +def f() -> C: + return c + +c = attr.evolve(f(), name='foo') + +[builtins fixtures/plugin_attrs.pyi] + +[case testEvolveFromNonAttrs] +import attr + +attr.evolve(42, name='foo') # E: Argument 1 to "evolve" has incompatible type "int"; expected an attrs class +attr.evolve(None, name='foo') # E: Argument 1 to "evolve" has incompatible type "None"; expected an attrs class +[case testEvolveFromAny] +from typing import Any +import attr + +any: Any = 42 +ret = attr.evolve(any, name='foo') +reveal_type(ret) # N: Revealed type is "Any" + +[typing fixtures/typing-medium.pyi] + +[case testEvolveGeneric] +import attrs +from typing import Generic, TypeVar + +T = TypeVar('T') + +@attrs.define +class A(Generic[T]): + x: T + + +a = A(x=42) +reveal_type(a) # N: Revealed type is "__main__.A[builtins.int]" +a2 = attrs.evolve(a, x=42) +reveal_type(a2) # N: Revealed type is "__main__.A[builtins.int]" +a2 = attrs.evolve(a, x='42') # E: Argument "x" to "evolve" of "A[int]" has incompatible type "str"; expected "int" +reveal_type(a2) # N: Revealed type is "__main__.A[builtins.int]" + +[builtins fixtures/plugin_attrs.pyi] + +[case testEvolveUnion] +# flags: --python-version 3.10 +from typing import Generic, TypeVar +import attrs + +T = TypeVar('T') + + +@attrs.define +class A(Generic[T]): + x: T # exercises meet(T=int, int) = int + y: bool # exercises meet(bool, int) = bool + z: str # exercises meet(str, bytes) = Never + w: dict # exercises meet(dict, Never) = Never + + +@attrs.define +class B: + x: int + y: bool + z: bytes + + +a_or_b: A[int] | B +a2 = attrs.evolve(a_or_b, x=42, y=True) +a2 = attrs.evolve(a_or_b, x=42, y=True, z='42') # E: Argument "z" to "evolve" of "Union[A[int], B]" has incompatible type "str"; expected "Never" +a2 = attrs.evolve(a_or_b, x=42, y=True, w={}) # E: Argument "w" to "evolve" of "Union[A[int], B]" has incompatible type "Dict[Never, Never]"; expected "Never" + +[builtins fixtures/plugin_attrs.pyi] + +[case testEvolveUnionOfTypeVar] +# flags: --python-version 3.10 +import attrs +from typing import TypeVar + +@attrs.define +class A: + x: int + y: int + z: str + w: dict + + +class B: + pass + +TA = TypeVar('TA', bound=A) +TB = TypeVar('TB', bound=B) + +def f(b_or_t: TA | TB | int) -> None: + a2 = attrs.evolve(b_or_t) # E: Argument 1 to "evolve" has type "Union[TA, TB, int]" whose item "TB" is not bound to an attrs class \ + # E: Argument 1 to "evolve" has incompatible type "Union[TA, TB, int]" whose item "int" is not an attrs class + + +[builtins fixtures/plugin_attrs.pyi] + +[case testEvolveTypeVarBound] +import attrs +from typing import TypeVar + +@attrs.define +class A: + x: int + +@attrs.define +class B(A): + pass + +TA = TypeVar('TA', bound=A) + +def f(t: TA) -> TA: + t2 = attrs.evolve(t, x=42) + reveal_type(t2) # N: Revealed type is "TA`-1" + t3 = attrs.evolve(t, x='42') # E: Argument "x" to "evolve" of "TA" has incompatible type "str"; expected "int" + return t2 + +f(A(x=42)) +f(B(x=42)) + +[builtins fixtures/plugin_attrs.pyi] + +[case testEvolveTypeVarBoundNonAttrs] +import attrs +from typing import Union, TypeVar + +TInt = TypeVar('TInt', bound=int) +TAny = TypeVar('TAny') +TNone = TypeVar('TNone', bound=None) +TUnion = TypeVar('TUnion', bound=Union[str, int]) + +def f(t: TInt) -> None: + _ = attrs.evolve(t, x=42) # E: Argument 1 to "evolve" has a variable type "TInt" not bound to an attrs class + +def g(t: TAny) -> None: + _ = attrs.evolve(t, x=42) # E: Argument 1 to "evolve" has a variable type "TAny" not bound to an attrs class + +def h(t: TNone) -> None: + _ = attrs.evolve(t, x=42) # E: Argument 1 to "evolve" has a variable type "TNone" not bound to an attrs class + +def x(t: TUnion) -> None: + _ = attrs.evolve(t, x=42) # E: Argument 1 to "evolve" has incompatible type "TUnion" whose item "str" is not an attrs class \ + # E: Argument 1 to "evolve" has incompatible type "TUnion" whose item "int" is not an attrs class + +[builtins fixtures/plugin_attrs.pyi] + +[case testEvolveTypeVarConstrained] +import attrs +from typing import TypeVar + +@attrs.define +class A: + x: int + +@attrs.define +class B: + x: str # conflicting with A.x + +T = TypeVar('T', A, B) + +def f(t: T) -> T: + t2 = attrs.evolve(t, x=42) # E: Argument "x" to "evolve" of "B" has incompatible type "int"; expected "str" + reveal_type(t2) # N: Revealed type is "__main__.A" # N: Revealed type is "__main__.B" + t2 = attrs.evolve(t, x='42') # E: Argument "x" to "evolve" of "A" has incompatible type "str"; expected "int" + return t2 + +f(A(x=42)) +f(B(x='42')) + +[builtins fixtures/plugin_attrs.pyi] + +[case testEvolveVariants] +from typing import Any +import attr +import attrs + + +@attr.s(auto_attribs=True) +class C: + name: str + +c = C(name='foo') + +c = attr.assoc(c, name='test') +c = attr.assoc(c, name=42) # E: Argument "name" to "assoc" of "C" has incompatible type "int"; expected "str" + +c = attrs.evolve(c, name='test') +c = attrs.evolve(c, name=42) # E: Argument "name" to "evolve" of "C" has incompatible type "int"; expected "str" + +c = attrs.assoc(c, name='test') +c = attrs.assoc(c, name=42) # E: Argument "name" to "assoc" of "C" has incompatible type "int"; expected "str" + +[builtins fixtures/plugin_attrs.pyi] +[typing fixtures/typing-medium.pyi] + +[case testFrozenInheritFromGeneric] +from typing import Generic, TypeVar +from attrs import field, frozen + +T = TypeVar('T') + +def f(s: str) -> int: + ... + +@frozen +class A(Generic[T]): + x: T + y: int = field(converter=f) + +@frozen +class B(A[int]): + pass + +b = B(42, 'spam') +reveal_type(b.x) # N: Revealed type is "builtins.int" +reveal_type(b.y) # N: Revealed type is "builtins.int" + +[builtins fixtures/plugin_attrs.pyi] + +[case testDefaultHashability] +from attrs import define + +@define +class A: + a: int + +reveal_type(A.__hash__) # N: Revealed type is "None" + +[builtins fixtures/plugin_attrs.pyi] + +[case testFrozenHashability] +from attrs import frozen + +@frozen +class A: + a: int + +reveal_type(A.__hash__) # N: Revealed type is "def (self: builtins.object) -> builtins.int" + +[builtins fixtures/plugin_attrs.pyi] + +[case testManualHashHashability] +from attrs import define + +@define(hash=True) +class A: + a: int + +reveal_type(A.__hash__) # N: Revealed type is "def (self: builtins.object) -> builtins.int" + +[builtins fixtures/plugin_attrs.pyi] + +[case testManualUnsafeHashHashability] +from attrs import define + +@define(unsafe_hash=True) +class A: + a: int + +reveal_type(A.__hash__) # N: Revealed type is "def (self: builtins.object) -> builtins.int" + +[builtins fixtures/plugin_attrs.pyi] + +[case testSubclassingHashability] +from attrs import define + +@define(unsafe_hash=True) +class A: + a: int + +@define +class B(A): + pass + +reveal_type(B.__hash__) # N: Revealed type is "None" + +[builtins fixtures/plugin_attrs.pyi] + +[case testManualOwnHashability] +from attrs import define, frozen + +@define +class A: + a: int + def __hash__(self) -> int: + ... + +reveal_type(A.__hash__) # N: Revealed type is "def (self: __main__.A) -> builtins.int" + +[builtins fixtures/plugin_attrs.pyi] + +[case testSubclassDefaultLosesHashability] +from attrs import define, frozen + +@define +class A: + a: int + def __hash__(self) -> int: + ... + +@define +class B(A): + pass + +reveal_type(B.__hash__) # N: Revealed type is "None" + +[builtins fixtures/plugin_attrs.pyi] + +[case testSubclassEqFalseKeepsHashability] +from attrs import define, frozen + +@define +class A: + a: int + def __hash__(self) -> int: + ... + +@define(eq=False) +class B(A): + pass + +reveal_type(B.__hash__) # N: Revealed type is "def (self: __main__.A) -> builtins.int" + +[builtins fixtures/plugin_attrs.pyi] + +[case testSubclassingFrozenHashability] +from attrs import define, frozen + +@define +class A: + a: int + +@frozen +class B(A): + pass + +reveal_type(B.__hash__) # N: Revealed type is "def (self: builtins.object) -> builtins.int" + +[builtins fixtures/plugin_attrs.pyi] + +[case testSubclassingFrozenHashOffHashability] +from attrs import define, frozen + +@define +class A: + a: int + def __hash__(self) -> int: + ... + +@frozen(unsafe_hash=False) +class B(A): + pass + +reveal_type(B.__hash__) # N: Revealed type is "None" + +[builtins fixtures/plugin_attrs.pyi] + +[case testUnsafeHashPrecedence] +from attrs import define, frozen + +@define(unsafe_hash=True, hash=False) +class A: + pass +reveal_type(A.__hash__) # N: Revealed type is "def (self: builtins.object) -> builtins.int" + +@define(unsafe_hash=False, hash=True) +class B: + pass +reveal_type(B.__hash__) # N: Revealed type is "None" + +[builtins fixtures/plugin_attrs.pyi] + +[case testAttrsStrictOptionalSetProperly] +from typing import Generic, Optional, TypeVar + +import attr + +T = TypeVar("T") + +@attr.mutable() +class Parent(Generic[T]): + run_type: Optional[int] = None + +@attr.mutable() +class Child(Parent[float]): + pass + +Parent(run_type = None) +c = Child(run_type = None) +reveal_type(c.run_type) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/plugin_attrs.pyi] diff --git a/test-data/unit/check-possibly-undefined.test b/test-data/unit/check-possibly-undefined.test new file mode 100644 index 000000000000..ae277949c049 --- /dev/null +++ b/test-data/unit/check-possibly-undefined.test @@ -0,0 +1,1045 @@ +[case testDefinedInOneBranch] +# flags: --enable-error-code possibly-undefined +if int(): + a = 1 +else: + x = 2 +z = a + 1 # E: Name "a" may be undefined +z = a + 1 # We only report the error on first occurrence. + +[case testElif] +# flags: --enable-error-code possibly-undefined +if int(): + a = 1 +elif int(): + a = 2 +else: + x = 3 + +z = a + 1 # E: Name "a" may be undefined + +[case testUsedInIf] +# flags: --enable-error-code possibly-undefined +if int(): + y = 1 +if int(): + x = y # E: Name "y" may be undefined + +[case testDefinedInAllBranches] +# flags: --enable-error-code possibly-undefined +if int(): + a = 1 +elif int(): + a = 2 +else: + a = 3 +z = a + 1 + +[case testOmittedElse] +# flags: --enable-error-code possibly-undefined +if int(): + a = 1 +z = a + 1 # E: Name "a" may be undefined + +[case testUpdatedInIf] +# flags: --enable-error-code possibly-undefined +# Variable a is already defined. Just updating it in an "if" is acceptable. +a = 1 +if int(): + a = 2 +z = a + 1 + +[case testNestedIf] +# flags: --enable-error-code possibly-undefined +if int(): + if int(): + a = 1 + x = 1 + x = x + 1 + else: + a = 2 + b = a + x # E: Name "x" may be undefined + b = b + 1 +else: + b = 2 +z = a + b # E: Name "a" may be undefined + +[case testVeryNestedIf] +# flags: --enable-error-code possibly-undefined +if int(): + if int(): + if int(): + a = 1 + else: + a = 2 + x = a + else: + a = 2 + b = a +else: + b = 2 +z = a + b # E: Name "a" may be undefined + +[case testTupleUnpack] +# flags: --enable-error-code possibly-undefined + +if int(): + (x, y) = (1, 2) +else: + [y, z] = [1, 2] +a = y + x # E: Name "x" may be undefined +a = y + z # E: Name "z" may be undefined + +[case testIndexExpr] +# flags: --enable-error-code possibly-undefined + +if int(): + *x, y = (1, 2) +else: + x = [1, 2] +a = x # No error. +b = y # E: Name "y" may be undefined + +[case testRedefined] +# flags: --enable-error-code possibly-undefined +y = 3 +if int(): + if int(): + y = 2 + x = y + 2 +else: + if int(): + y = 2 + x = y + 2 + +x = y + 2 + +[case testFunction] +# flags: --enable-error-code possibly-undefined +def f0() -> None: + if int(): + def some_func() -> None: + pass + + some_func() # E: Name "some_func" may be undefined + +def f1() -> None: + if int(): + def some_func() -> None: + pass + else: + def some_func() -> None: + pass + + some_func() # No error. + +[case testLambda] +# flags: --enable-error-code possibly-undefined +def f0(b: bool) -> None: + if b: + fn = lambda: 2 + y = fn # E: Name "fn" may be undefined + +[case testUsedBeforeDefClass] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f(x: A): # No error here. + pass +y = A() # E: Name "A" is used before definition +class A: pass + +[case testClassScope] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +class C: + x = 0 + def f0(self) -> None: pass + + def f2(self) -> None: + f0() # No error. + self.f0() # No error. + +f0() # E: Name "f0" is used before definition +def f0() -> None: pass +y = x # E: Name "x" is used before definition +x = 1 + +[case testClassInsideFunction] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f() -> None: + class C: pass + +c = C() # E: Name "C" is used before definition +class C: pass + +[case testUsedBeforeDefFunc] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +foo() # E: Name "foo" is used before definition +def foo(): pass +[case testGenerator] +# flags: --enable-error-code possibly-undefined +if int(): + a = 3 +s = [a + 1 for a in [1, 2, 3]] +x = a # E: Name "a" may be undefined + +[case testScope] +# flags: --enable-error-code possibly-undefined +def foo() -> None: + if int(): + y = 2 + +if int(): + y = 3 +x = y # E: Name "y" may be undefined + +[case testVarDefinedInOuterScopeUpdated] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f0() -> None: + global x + y = x + x = 1 # No error. + +x = 2 + +[case testNonlocalVar] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f0() -> None: + x = 2 + + def inner() -> None: + nonlocal x + y = x + x = 1 # No error. + +[case testGlobalDeclarationAfterUsage] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f0() -> None: + y = x # E: Name "x" is used before definition + global x + x = 1 # No error. + +x = 2 + +[case testVarDefinedInOuterScope] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f0() -> None: + global x + y = x # We do not detect such errors right now. + +f0() +x = 1 + +[case testDefinedInOuterScopeNoError] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def foo() -> None: + bar() + +def bar() -> None: + foo() + +[case testClassFromOuterScopeRedefined] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +class c: pass + +def f0() -> None: + s = c() # E: Name "c" is used before definition + class c: pass + +def f1() -> None: + s = c() # No error. + +def f2() -> None: + s = c() # E: Name "c" is used before definition + if int(): + class c: pass + +glob = c() +def f3(x: c = glob) -> None: + glob = 123 + +[case testVarFromOuterScopeRedefined] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +x = 0 + +def f0() -> None: + y = x # E: Name "x" is used before definition + x = 0 + +def f1() -> None: + y = x # No error. + +def f2() -> None: + y = x # E: Name "x" is used before definition + global x + +def f3() -> None: + global x + y = x # No error. + +def f4() -> None: + if int(): + x = 0 + y = x # E: Name "x" may be undefined + +[case testFuncParams] +# flags: --enable-error-code possibly-undefined +def foo(a: int) -> None: + if int(): + a = 2 + x = a + +[case testWhile] +# flags: --enable-error-code possibly-undefined +while int(): + a = 1 + +x = a # E: Name "a" may be undefined + +while int(): + b = 1 +else: + b = 2 + +y = b # No error. + +while True: + c = 1 + if int(): + break +y = c # No error. + +# This while loop doesn't have a `break` inside, so we know that the else must always get executed. +while int(): + pass +else: + d = 1 +y = d # No error. + +while int(): + if int(): + break +else: + e = 1 +# If a while loop has a `break`, it's possible that the else didn't get executed. +y = e # E: Name "e" may be undefined + +while int(): + while int(): + if int(): + break + else: + f = 1 +else: + g = 2 + +y = f # E: Name "f" may be undefined +y = g + +[case testForLoop] +# flags: --enable-error-code possibly-undefined +for x in [1, 2, 3]: + if x: + x = 1 + y = x +else: + z = 2 + +a = z + y # E: Name "y" may be undefined + +[case testReturn] +# flags: --enable-error-code possibly-undefined +def f1() -> int: + if int(): + x = 1 + else: + return 0 + return x + +def f2() -> int: + if int(): + x = 1 + elif int(): + return 0 + else: + x = 2 + return x + +def f3() -> int: + if int(): + x = 1 + elif int(): + return 0 + else: + y = 2 + return x # E: Name "x" may be undefined + +def f4() -> int: + if int(): + x = 1 + elif int(): + return 0 + else: + return 0 + return x + +def f5() -> int: + # This is a test against crashes. + if int(): + return 1 + if int(): + return 2 + else: + return 3 + return 1 + +def f6() -> int: + if int(): + x = 0 + return x + return x # E: Name "x" may be undefined + +[case testDefinedDifferentBranchUsedBeforeDef] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def + +def f0() -> None: + if int(): + x = 0 + else: + y = x # E: Name "x" is used before definition + z = x # E: Name "x" is used before definition + +def f1() -> None: + x = 1 + if int(): + x = 0 + else: + y = x # No error. + +def f2() -> None: + if int(): + x = 0 + elif int(): + y = x # E: Name "x" is used before definition + else: + y = x # E: Name "x" is used before definition + if int(): + z = x # E: Name "x" is used before definition + x = 1 + else: + x = 2 + w = x # No error. + +[case testPossiblyUndefinedLoop] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def + +def f0() -> None: + first_iter = True + for i in [0, 1]: + if first_iter: + first_iter = False + x = 0 + elif int(): + # This is technically a false positive but mypy isn't smart enough for this yet. + y = x # E: Name "x" may be undefined + else: + y = x # E: Name "x" may be undefined + if int(): + z = x # E: Name "x" may be undefined + x = 1 + else: + x = 2 + w = x # No error. + +def f1() -> None: + while True: + if int(): + x = 0 + else: + y = x # E: Name "x" may be undefined + z = x # E: Name "x" may be undefined + +def f2() -> None: + for i in [0, 1]: + x = i + else: + y = x # E: Name "x" may be undefined + +def f3() -> None: + while int(): + x = 1 + else: + y = x # E: Name "x" may be undefined + +def f4() -> None: + while int(): + y = x # E: Name "x" may be undefined + x: int = 1 + +[case testAssert] +# flags: --enable-error-code possibly-undefined +def f1() -> int: + if int(): + x = 1 + else: + assert False, "something something" + return x + +def f2() -> int: + if int(): + x = 1 + elif int(): + assert False + else: + y = 2 + return x # E: Name "x" may be undefined + +[case testRaise] +# flags: --enable-error-code possibly-undefined +def f1() -> int: + if int(): + x = 1 + else: + raise BaseException("something something") + return x + +def f2() -> int: + if int(): + x = 1 + elif int(): + raise BaseException("something something") + else: + y = 2 + return x # E: Name "x" may be undefined +[builtins fixtures/exception.pyi] + +[case testContinue] +# flags: --enable-error-code possibly-undefined +def f1() -> int: + while int(): + if int(): + x = 1 + else: + continue + y = x + else: + x = 2 + return x + +def f2() -> int: + while int(): + if int(): + x = 1 + elif int(): + pass + else: + continue + y = x # E: Name "x" may be undefined + return x # E: Name "x" may be undefined + +def f3() -> None: + while True: + if int(): + x = 2 + elif int(): + continue + else: + continue + y = x + +[case testBreak] +# flags: --enable-error-code possibly-undefined +def f1() -> None: + while int(): + if int(): + x = 1 + else: + break + y = x # No error -- x is always defined. + +def f2() -> None: + while int(): + if int(): + x = 1 + elif int(): + pass + else: + break + y = x # E: Name "x" may be undefined + +def f3() -> None: + while int(): + x = 1 + while int(): + if int(): + x = 2 + else: + break + y = x + z = x # E: Name "x" may be undefined + +[case testTryBasic] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f1() -> int: + try: + x = 1 + except: + pass + return x # E: Name "x" may be undefined + +def f2() -> int: + try: + pass + except: + x = 1 + return x # E: Name "x" may be undefined + +def f3() -> int: + try: + x = 1 + except: + y = x # E: Name "x" may be undefined + return x # E: Name "x" may be undefined + +def f4() -> int: + try: + x = 1 + except: + return 0 + return x + +def f5() -> int: + try: + x = 1 + except: + raise + return x + +def f6() -> None: + try: + pass + except BaseException as exc: + x = exc # No error. + exc = BaseException() + # This case is covered by the other check, not by possibly undefined check. + y = exc # E: Trying to read deleted variable "exc" + +def f7() -> int: + try: + if int(): + x = 1 + assert False + except: + pass + return x # E: Name "x" may be undefined +[builtins fixtures/exception.pyi] + +[case testTryMultiExcept] +# flags: --enable-error-code possibly-undefined +def f1() -> int: + try: + x = 1 + except BaseException: + x = 2 + except: + x = 3 + return x + +def f2() -> int: + try: + x = 1 + except BaseException: + pass + except: + x = 3 + return x # E: Name "x" may be undefined +[builtins fixtures/exception.pyi] + +[case testTryFinally] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +def f1() -> int: + try: + x = 1 + finally: + x = 2 + return x + +def f2() -> int: + try: + pass + except: + pass + finally: + x = 2 + return x + +def f3() -> int: + try: + x = 1 + except: + pass + finally: + y = x # E: Name "x" may be undefined + return x + +def f4() -> int: + try: + x = 0 + except BaseException: + raise + finally: + y = x # E: Name "x" may be undefined + return y + +def f5() -> int: + try: + if int(): + x = 1 + else: + return 0 + finally: + pass + return x # No error. + +def f6() -> int: + try: + if int(): + x = 1 + else: + return 0 + finally: + a = x # E: Name "x" may be undefined + return a +[builtins fixtures/exception.pyi] + +[case testTryElse] +# flags: --enable-error-code possibly-undefined +def f1() -> int: + try: + return 0 + except BaseException: + x = 1 + else: + x = 2 + finally: + y = x + return y + +def f2() -> int: + try: + pass + except: + x = 1 + else: + x = 2 + return x + +def f3() -> int: + try: + pass + except: + x = 1 + else: + pass + return x # E: Name "x" may be undefined + +def f4() -> int: + try: + x = 1 + except: + x = 2 + else: + pass + return x + +def f5() -> int: + try: + pass + except: + x = 1 + else: + return 1 + return x +[builtins fixtures/exception.pyi] + +[case testNoReturn] +# flags: --enable-error-code possibly-undefined + +from typing import NoReturn +def fail() -> NoReturn: + assert False + +def f() -> None: + if int(): + x = 1 + elif int(): + x = 2 + y = 3 + else: + # This has a NoReturn type, so we can skip it. + fail() + z = y # E: Name "y" may be undefined + z = x + +[case testDictComprehension] +# flags: --enable-error-code possibly-undefined + +def f() -> None: + for _ in [1, 2]: + key = 2 + val = 2 + + x = ( + key, # E: Name "key" may be undefined + val, # E: Name "val" may be undefined + ) + + d = [(0, "a"), (1, "b")] + {val: key for key, val in d} +[builtins fixtures/dict.pyi] + +[case testWithStmt] +# flags: --enable-error-code possibly-undefined +from contextlib import contextmanager + +@contextmanager +def ctx(*args): + yield 1 + +def f() -> None: + if int(): + a = b = 1 + x = 1 + + with ctx() as a, ctx(a) as b, ctx(x) as x: # E: Name "x" may be undefined + c = a + c = b + d = a + d = b +[builtins fixtures/tuple.pyi] + +[case testUnreachable] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def +import typing + +def f0() -> None: + if typing.TYPE_CHECKING: + x = 1 + elif int(): + y = 1 + else: + y = 2 + a = x + +def f1() -> None: + if not typing.TYPE_CHECKING: + pass + else: + z = 1 + a = z + +def f2() -> None: + if typing.TYPE_CHECKING: + x = 1 + else: + y = x +[typing fixtures/typing-medium.pyi] + +[case testUsedBeforeDef] +# flags: --enable-error-code used-before-def + +def f0() -> None: + x = y # E: Name "y" is used before definition + y: int = 1 + +def f2() -> None: + if int(): + pass + else: + # No used-before-def error. + y = z # E: Name "z" is not defined + + def inner2() -> None: + z = 0 + +def f3() -> None: + if int(): + pass + else: + y = z # E: Name "z" is used before definition + z: int = 2 + +def f4() -> None: + if int(): + pass + else: + y = z # E: Name "z" is used before definition + x = z # E: Name "z" is used before definition + z: int = 2 + +[case testUsedBeforeDefImportsBasicImportNoError] +# flags: --enable-error-code used-before-def --enable-error-code possibly-undefined --disable-error-code no-redef +import foo # type: ignore + +a = foo # No error. +foo: int = 1 + +[case testUsedBeforeDefImportsDotImport] +# flags: --enable-error-code used-before-def --enable-error-code possibly-undefined --disable-error-code no-redef +import x.y # type: ignore + +a = y # E: Name "y" is used before definition +y: int = 1 + +b = x # No error. +x: int = 1 + +c = x.y # No error. +x: int = 1 + +[case testUsedBeforeDefImportBasicRename] +# flags: --enable-error-code used-before-def --disable-error-code=no-redef +import x.y as z # type: ignore +from typing import Any + +a = z # No error. +z: int = 1 + +a = x # E: Name "x" is used before definition +x: int = 1 + +a = y # E: Name "y" is used before definition +y: int = 1 + +[case testUsedBeforeDefImportFrom] +# flags: --enable-error-code used-before-def --disable-error-code no-redef +from foo import x # type: ignore + +a = x # No error. +x: int = 1 + +[case testUsedBeforeDefImportFromRename] +# flags: --enable-error-code used-before-def --disable-error-code no-redef +from foo import x as y # type: ignore + +a = y # No error. +y: int = 1 + +a = x # E: Name "x" is used before definition +x: int = 1 + +[case testUsedBeforeDefFunctionDeclarations] +# flags: --enable-error-code used-before-def + +def f0() -> None: + def inner() -> None: + pass + + inner() # No error. + inner = lambda: None + +[case testUsedBeforeDefBuiltinsFunc] +# flags: --enable-error-code used-before-def + +def f0() -> None: + s = type(123) # E: Name "type" is used before definition + type = "abc" + a = type + +def f1() -> None: + s = type(123) + +[case testUsedBeforeDefBuiltinsGlobal] +# flags: --enable-error-code used-before-def + +s = type(123) +type = "abc" +a = type + +[case testUsedBeforeDefBuiltinsClass] +# flags: --enable-error-code used-before-def + +class C: + s = type + type = s + +[case testUsedBeforeDefBuiltinsGenerator] +# flags: --enable-error-code used-before-def + +def f0() -> None: + _ = [type for type in [type("a"), type(1)]] + +[case testUsedBeforeDefBuiltinsMultipass] +# flags: --enable-error-code used-before-def + +# When doing multiple passes, mypy resolves references slightly differently. +# In this case, it would refer the earlier `type` call to the range class defined below. +_type = type # No error +_C = C # E: Name "C" is used before definition +class type: pass +class C: pass + +[case testUsedBeforeDefImplicitModuleAttrs] +# flags: --enable-error-code used-before-def +a = __name__ # No error. +__name__ = "abc" + +[case testUntypedDef] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def + +def f(): + if int(): + x = 0 + z = y # No used-before-def error because def is untyped. + y = x # No possibly-undefined error because def is untyped. + +[case testUntypedDefCheckUntypedDefs] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def --check-untyped-defs + +def f(): + if int(): + x = 0 + z = y # E: Name "y" is used before definition + y: int = x # E: Name "x" may be undefined + +[case testClassBody] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def + +class A: + # The following should not only trigger an error from semantic analyzer, but not the used-before-def check. + y = x + 1 # E: Name "x" is not defined + x = 0 + # Same as above but in a loop, which should trigger a possibly-undefined error. + for _ in [1, 2, 3]: + b = a + 1 # E: Name "a" is not defined + a = 0 + + +class B: + if int(): + x = 0 + else: + # This type of check is not caught by the semantic analyzer. If we ever update it to catch such issues, + # we should make sure that errors are not double-reported. + y = x # E: Name "x" is used before definition + for _ in [1, 2, 3]: + if int(): + a = 0 + else: + # Same as above but in a loop. + b = a # E: Name "a" may be undefined + +[case testUnreachableCausingMissingTypeMap] +# flags: --enable-error-code possibly-undefined --enable-error-code used-before-def --no-warn-unreachable +# Regression test for https://github.com/python/mypy/issues/15958 +from typing import Union, NoReturn + +def assert_never(__x: NoReturn) -> NoReturn: ... + +def foo(x: Union[int, str]) -> None: + if isinstance(x, str): + f = "foo" + elif isinstance(x, int): + f = "bar" + else: + assert_never(x) + f # OK +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test new file mode 100644 index 000000000000..dd19eb1f21d6 --- /dev/null +++ b/test-data/unit/check-protocols.test @@ -0,0 +1,4246 @@ +-- Simple protocol types +-- --------------------- + +[case testCannotInstantiateProtocol] +from typing import Protocol + +class P(Protocol): + def meth(self) -> None: + pass + +P() # E: Cannot instantiate protocol class "P" + +[case testSimpleProtocolOneMethod] +from typing import Protocol + +class P(Protocol): + def meth(self) -> None: + pass + +class B: pass +class C: + def meth(self) -> None: + pass + +x: P +def fun(x: P) -> None: + x.meth() + x.meth(x) # E: Too many arguments for "meth" of "P" + x.bad # E: "P" has no attribute "bad" + +x = C() +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") + +fun(C()) +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" + +def fun2() -> P: + return C() +def fun3() -> P: + return B() # E: Incompatible return value type (got "B", expected "P") + +[case testProtocolAttrAccessDecoratedGetAttrDunder] +from typing import Any, Protocol, Callable + +def typed_decorator(fun: Callable) -> Callable[[Any, str], str]: + pass + +def untyped_decorator(fun): + pass + +class P(Protocol): + @property + def x(self) -> int: + pass + +class A: + @untyped_decorator + def __getattr__(self, key: str) -> int: + pass + +class B: + @typed_decorator + def __getattr__(self, key: str) -> int: + pass + +class C: + def __getattr__(self, key: str) -> int: + pass + +def fun(x: P) -> None: + pass + +a: A +reveal_type(a.x) +fun(a) + +b: B +reveal_type(b.x) +fun(b) + +c: C +reveal_type(c.x) +fun(c) +[out] +main:32: note: Revealed type is "Any" +main:36: note: Revealed type is "builtins.str" +main:37: error: Argument 1 to "fun" has incompatible type "B"; expected "P" +main:37: note: Following member(s) of "B" have conflicts: +main:37: note: x: expected "int", got "str" +main:40: note: Revealed type is "builtins.int" +[builtins fixtures/bool.pyi] + +[case testSimpleProtocolOneAbstractMethod] +from typing import Protocol +from abc import abstractmethod + +class P(Protocol): + @abstractmethod + def meth(self) -> None: + pass + +class B: pass +class C: + def meth(self) -> None: + pass +class D(B): + def meth(self) -> None: + pass + +x: P +def fun(x: P) -> None: + x.meth() + x.meth(x) # E: Too many arguments for "meth" of "P" + x.bad # E: "P" has no attribute "bad" + +x = C() +x = D() +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") +fun(C()) +fun(D()) +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" +fun(x) + +[case testProtocolMethodBodies] +from typing import Protocol, List + +class P(Protocol): + def meth(self) -> int: + return 'no way' # E: Incompatible return value type (got "str", expected "int") + +# explicit ellipsis is OK in protocol methods +class P2(Protocol): + def meth2(self) -> List[int]: + ... +[builtins fixtures/list.pyi] + +[case testSimpleProtocolOneMethodOverride] +from typing import Protocol, Union + +class P(Protocol): + def meth(self) -> Union[int, str]: + pass +class SubP(P, Protocol): + def meth(self) -> int: + pass + +class B: pass +class C: + def meth(self) -> int: + pass +z: P +x: SubP +def fun(x: SubP) -> str: + x.bad # E: "SubP" has no attribute "bad" + return x.meth() # E: Incompatible return value type (got "int", expected "str") + +z = x +x = C() +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") + +reveal_type(fun(C())) # N: Revealed type is "builtins.str" +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" + +[case testSimpleProtocolTwoMethodsMerge] +from typing import Protocol + +class P1(Protocol): + def meth1(self) -> int: + pass +class P2(Protocol): + def meth2(self) -> str: + pass +class P(P1, P2, Protocol): pass + +class B: pass +class C1: + def meth1(self) -> int: + pass +class C2(C1): + def meth2(self) -> str: + pass +class C: + def meth1(self) -> int: + pass + def meth2(self) -> str: + pass + +class AnotherP(Protocol): + def meth1(self) -> int: + pass + def meth2(self) -> str: + pass + +x: P +reveal_type(x.meth1()) # N: Revealed type is "builtins.int" +reveal_type(x.meth2()) # N: Revealed type is "builtins.str" + +c: C +c1: C1 +c2: C2 +y: AnotherP + +if int(): + x = c +if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") +if int(): + x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P") \ + # N: "C1" is missing following "P" protocol member: \ + # N: meth2 +if int(): + x = c2 +if int(): + x = y +if int(): + y = x + +[case testSimpleProtocolTwoMethodsExtend] +from typing import Protocol + +class P1(Protocol): + def meth1(self) -> int: + pass +class P2(P1, Protocol): + def meth2(self) -> str: + pass + +class Cbad: + def meth1(self) -> int: + pass + +class C: + def meth1(self) -> int: + pass + def meth2(self) -> str: + pass + +x: P2 +reveal_type(x.meth1()) # N: Revealed type is "builtins.int" +reveal_type(x.meth2()) # N: Revealed type is "builtins.str" + +if int(): + x = C() # OK +if int(): + x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2") \ + # N: "Cbad" is missing following "P2" protocol member: \ + # N: meth2 + +[case testProtocolMethodVsAttributeErrors] +from typing import Protocol + +class P(Protocol): + def meth(self) -> int: + pass +class C: + meth: int +x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ + # N: Following member(s) of "C" have conflicts: \ + # N: meth: expected "Callable[[], int]", got "int" + +[case testProtocolMethodVsAttributeErrors2] +from typing import Protocol + +class P(Protocol): + @property + def meth(self) -> int: + pass +class C: + def meth(self) -> int: + pass +x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ + # N: Following member(s) of "C" have conflicts: \ + # N: meth: expected "int", got "Callable[[], int]" +[builtins fixtures/property.pyi] + +[case testCannotAssignNormalToProtocol] +from typing import Protocol + +class P(Protocol): + def meth(self) -> int: + pass +class C: + def meth(self) -> int: + pass + +x: C +y: P +x = y # E: Incompatible types in assignment (expression has type "P", variable has type "C") + +[case testIndependentProtocolSubtyping] +from typing import Protocol + +class P1(Protocol): + def meth(self) -> int: + pass +class P2(Protocol): + def meth(self) -> int: + pass + +x1: P1 +x2: P2 + +x1 = x2 +x2 = x1 + +def f1(x: P1) -> None: pass +def f2(x: P2) -> None: pass + +f1(x2) +f2(x1) + +[case testNoneDisablesProtocolImplementation] +from typing import Protocol + +class MyHashable(Protocol): + def __my_hash__(self) -> int: + return 0 + +class C: + __my_hash__ = None + +var: MyHashable = C() # E: Incompatible types in assignment (expression has type "C", variable has type "MyHashable") \ + # N: Following member(s) of "C" have conflicts: \ + # N: __my_hash__: expected "Callable[[], int]", got "None" + +[case testNoneDisablesProtocolSubclassingWithStrictOptional] +from typing import Protocol + +class MyHashable(Protocol): + def __my_hash__(self) -> int: + return 0 + +class C(MyHashable): + __my_hash__ = None # E: Incompatible types in assignment \ +(expression has type "None", base class "MyHashable" defined the type as "Callable[[MyHashable], int]") + +[case testProtocolsWithNoneAndStrictOptional] +from typing import Protocol +class P(Protocol): + x = 0 # type: int + +class C: + x = None + +x: P = C() # Error! +def f(x: P) -> None: pass +f(C()) # Error! +[out] +main:8: error: Incompatible types in assignment (expression has type "C", variable has type "P") +main:8: note: Following member(s) of "C" have conflicts: +main:8: note: x: expected "int", got "None" +main:10: error: Argument 1 to "f" has incompatible type "C"; expected "P" +main:10: note: Following member(s) of "C" have conflicts: +main:10: note: x: expected "int", got "None" + +-- Semanal errors in protocol types +-- -------------------------------- + +[case testBasicSemanalErrorsInProtocols] + +from typing import Protocol, Generic, TypeVar, Iterable + +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) + +class P1(Protocol[T, T]): # E: Duplicate type variables in Generic[...] or Protocol[...] + def meth(self) -> T: + pass + +class P2(Protocol[T], Protocol[S]): # E: Only single Generic[...] or Protocol[...] can be in bases + def meth(self) -> T: + pass + +class P3(Protocol[T], Generic[S]): # E: Only single Generic[...] or Protocol[...] can be in bases + def meth(self) -> T: + pass + +class P4(Protocol[T]): + attr: Iterable[S] # E: Type variable "__main__.S" is unbound \ + # N: (Hint: Use "Generic[S]" or "Protocol[S]" base class to bind "S" inside a class) \ + # N: (Hint: Use "S" in function signature to bind "S" inside a function) + +class P5(Iterable[S], Protocol[T]): # E: If Generic[...] or Protocol[...] is present it should list all type variables + def meth(self) -> T: + pass + +[case testProhibitSelfDefinitionInProtocols] +from typing import Protocol + +class P(Protocol): + def __init__(self, a: int) -> None: + self.a = a # E: Protocol members cannot be defined via assignment to self \ + # E: "P" has no attribute "a" + +class B: pass +class C: + def __init__(self, a: int) -> None: + pass + +x: P +x = B() +# The above has an incompatible __init__, but mypy ignores this for nominal subtypes? +x = C(1) + +class P2(Protocol): + a: int + def __init__(self) -> None: + self.a = 1 + +class B2(P2): + a: int + +x2: P2 = B2() # OK + +[case testProtocolAndRuntimeAreDefinedAlsoInTypingExtensions] +from typing_extensions import Protocol, runtime_checkable + +@runtime_checkable +class P(Protocol): + def meth(self) -> int: + pass + +x: object +if isinstance(x, P): + reveal_type(x) # N: Revealed type is "__main__.P" + reveal_type(x.meth()) # N: Revealed type is "builtins.int" + +class C: + def meth(self) -> int: + pass + +z: P = C() +[builtins fixtures/dict.pyi] + +[case testProtocolsCannotInheritFromNormal] +from typing import Protocol + +class C: pass +class D: pass + +class P(C, Protocol): # E: All bases of a protocol must be protocols + attr: int + +class P2(P, D, Protocol): # E: All bases of a protocol must be protocols + pass + +P2() # E: Cannot instantiate protocol class "P2" +p: P2 +reveal_type(p.attr) # N: Revealed type is "builtins.int" + +-- Generic protocol types +-- ---------------------- + +[case testGenericMethodWithProtocol] +from typing import Protocol, TypeVar +T = TypeVar('T') + +class P(Protocol): + def meth(self, x: int) -> int: + return x +class C: + def meth(self, x: T) -> T: + return x + +x: P = C() + +[case testGenericMethodWithProtocol2] +from typing import Protocol, TypeVar +T = TypeVar('T') + +class P(Protocol): + def meth(self, x: T) -> T: + return x +class C: + def meth(self, x: int) -> int: + return x + +x: P = C() +[out] +main:11: error: Incompatible types in assignment (expression has type "C", variable has type "P") +main:11: note: Following member(s) of "C" have conflicts: +main:11: note: Expected: +main:11: note: def [T] meth(self, x: T) -> T +main:11: note: Got: +main:11: note: def meth(self, x: int) -> int + +[case testAutomaticProtocolVariance] +from typing import TypeVar, Protocol + +T = TypeVar('T') + +# In case of these errors we proceed with declared variance. +class Pco(Protocol[T]): # E: Invariant type variable "T" used in protocol where covariant one is expected + def meth(self) -> T: + pass +class Pcontra(Protocol[T]): # E: Invariant type variable "T" used in protocol where contravariant one is expected + def meth(self, x: T) -> None: + pass +class Pinv(Protocol[T]): + attr: T + +class A: pass +class B(A): pass + +x1: Pco[B] +y1: Pco[A] +if int(): + x1 = y1 # E: Incompatible types in assignment (expression has type "Pco[A]", variable has type "Pco[B]") +if int(): + y1 = x1 # E: Incompatible types in assignment (expression has type "Pco[B]", variable has type "Pco[A]") + +x2: Pcontra[B] +y2: Pcontra[A] +if int(): + y2 = x2 # E: Incompatible types in assignment (expression has type "Pcontra[B]", variable has type "Pcontra[A]") +if int(): + x2 = y2 # E: Incompatible types in assignment (expression has type "Pcontra[A]", variable has type "Pcontra[B]") + +x3: Pinv[B] +y3: Pinv[A] +if int(): + y3 = x3 # E: Incompatible types in assignment (expression has type "Pinv[B]", variable has type "Pinv[A]") +if int(): + x3 = y3 # E: Incompatible types in assignment (expression has type "Pinv[A]", variable has type "Pinv[B]") + +[case testProtocolVarianceWithCallableAndList] +from typing import Protocol, TypeVar, Callable, List +T = TypeVar('T') +S = TypeVar('S') +T_co = TypeVar('T_co', covariant=True) + +class P(Protocol[T, S]): # E: Invariant type variable "T" used in protocol where covariant one is expected \ + # E: Invariant type variable "S" used in protocol where contravariant one is expected + def fun(self, callback: Callable[[T], S]) -> None: pass + +class P2(Protocol[T_co]): # E: Covariant type variable "T_co" used in protocol where invariant one is expected + lst: List[T_co] +[builtins fixtures/list.pyi] + + +[case testProtocolConstraintsUnsolvableWithSelfAnnotation1] +# https://github.com/python/mypy/issues/11020 +from typing import overload, Protocol, TypeVar + +I = TypeVar('I', covariant=True) +V_contra = TypeVar('V_contra', contravariant=True) + +class C(Protocol[I]): + def __abs__(self: 'C[V_contra]') -> 'C[V_contra]': + ... + + @overload + def f(self: 'C', q: int) -> int: + ... + @overload + def f(self: 'C[float]', q: float) -> 'C[float]': + ... +[builtins fixtures/bool.pyi] + + +[case testProtocolConstraintsUnsolvableWithSelfAnnotation2] +# https://github.com/python/mypy/issues/11020 +from typing import Protocol, TypeVar + +I = TypeVar('I', covariant=True) +V = TypeVar('V') + +class C(Protocol[I]): + def g(self: 'C[V]') -> 'C[V]': + ... + +class D: + pass + +x: C = D() # E: Incompatible types in assignment (expression has type "D", variable has type "C[Any]") +[builtins fixtures/bool.pyi] + + +[case testProtocolConstraintsUnsolvableWithSelfAnnotation3] +# https://github.com/python/mypy/issues/11020 +from typing import Protocol, TypeVar + +I = TypeVar('I', covariant=True) +V = TypeVar('V') + +class C(Protocol[I]): + def g(self: 'C[V]') -> 'C[V]': + ... + +class D: + def g(self) -> D: + ... + +x: C = D() +[builtins fixtures/bool.pyi] + + +[case testProtocolVarianceWithUnusedVariable] +from typing import Protocol, TypeVar +T = TypeVar('T') + +class P(Protocol[T]): # E: Invariant type variable "T" used in protocol where covariant one is expected + attr: int + +[case testGenericProtocolsInference1] +from typing import Protocol, Sequence, TypeVar + +T = TypeVar('T', covariant=True) + +class Closeable(Protocol[T]): + def close(self) -> T: + pass + +class F: + def close(self) -> int: + return 0 + +def close(arg: Closeable[T]) -> T: + return arg.close() + +def close_all(args: Sequence[Closeable[T]]) -> T: + for arg in args: + arg.close() + return args[0].close() + +arg: Closeable[int] + +reveal_type(close(F())) # N: Revealed type is "builtins.int" +reveal_type(close(arg)) # N: Revealed type is "builtins.int" +reveal_type(close_all([F()])) # N: Revealed type is "builtins.int" +reveal_type(close_all([arg])) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +[case testProtocolGenericInference2] +from typing import Generic, TypeVar, Protocol +T = TypeVar('T') +S = TypeVar('S') + +class P(Protocol[T, S]): + x: T + y: S + +class C: + x: int + y: int + +def fun3(x: P[T, T]) -> T: + pass +reveal_type(fun3(C())) # N: Revealed type is "builtins.int" + +[case testProtocolGenericInferenceCovariant] +from typing import Generic, TypeVar, Protocol +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) +U = TypeVar('U') + +class P(Protocol[T, S]): + def x(self) -> T: pass + def y(self) -> S: pass + +class C: + def x(self) -> int: pass + def y(self) -> int: pass + +def fun4(x: U, y: P[U, U]) -> U: + pass +reveal_type(fun4('a', C())) # N: Revealed type is "builtins.object" + +[case testUnrealtedGenericProtolsEquivalent] +from typing import TypeVar, Protocol +T = TypeVar('T') + +class PA(Protocol[T]): + attr: int + def meth(self) -> T: pass + def other(self, arg: T) -> None: pass +class PB(Protocol[T]): # exactly the same as above + attr: int + def meth(self) -> T: pass + def other(self, arg: T) -> None: pass + +def fun(x: PA[T]) -> PA[T]: + y: PB[T] = x + z: PB[T] + return z + +x: PA +y: PB +x = y +y = x + +xi: PA[int] +yi: PB[int] +xi = yi +yi = xi + +[case testGenericSubProtocols] +from typing import TypeVar, Protocol, Tuple, Generic + +T = TypeVar('T') +S = TypeVar('S') + +class P1(Protocol[T]): + attr1: T +class P2(P1[T], Protocol[T, S]): + attr2: Tuple[T, S] + +class C: + def __init__(self, a1: int, a2: Tuple[int, int]) -> None: + self.attr1 = a1 + self.attr2 = a2 + +c: C +var: P2[int, int] = c +var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type "P2[int, str]") \ + # N: Following member(s) of "C" have conflicts: \ + # N: attr2: expected "Tuple[int, str]", got "Tuple[int, int]" + +class D(Generic[T]): + attr1: T +class E(D[T]): + attr2: Tuple[T, T] + +def f(x: T) -> T: + z: P2[T, T] = E[T]() + y: P2[T, T] = D[T]() # E: Incompatible types in assignment (expression has type "D[T]", variable has type "P2[T, T]") \ + # N: "D" is missing following "P2" protocol member: \ + # N: attr2 + return x +[builtins fixtures/isinstancelist.pyi] + +[case testGenericSubProtocolsExtensionInvariant] +from typing import TypeVar, Protocol, Union + +T = TypeVar('T') +S = TypeVar('S') + +class P1(Protocol[T]): + attr1: T +class P2(Protocol[T]): + attr2: T +class P(P1[T], P2[S], Protocol): + pass + +class C: + attr1: int + attr2: str + +class A: + attr1: A +class B: + attr2: B +class D(A, B): pass + +x: P = D() # Same as P[Any, Any] + +var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P[Union[int, P[Any, Any]], Union[P[Any, Any], str]]") \ + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "Union[int, P[Any, Any]]", got "int" \ + # N: attr2: expected "Union[P[Any, Any], str]", got "str" + +[case testGenericSubProtocolsExtensionCovariant] +from typing import TypeVar, Protocol, Union + +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) + +class P1(Protocol[T]): + def attr1(self) -> T: pass +class P2(Protocol[T]): + def attr2(self) -> T: pass +class P(P1[T], P2[S], Protocol): + pass + +class C: + def attr1(self) -> int: pass + def attr2(self) -> str: pass + +var: P[Union[int, P], Union[P, str]] = C() # OK for covariant +var2: P[Union[str, P], Union[P, int]] = C() +[out] +main:18: error: Incompatible types in assignment (expression has type "C", variable has type "P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]") +main:18: note: Following member(s) of "C" have conflicts: +main:18: note: Expected: +main:18: note: def attr1(self) -> Union[str, P[Any, Any]] +main:18: note: Got: +main:18: note: def attr1(self) -> int +main:18: note: Expected: +main:18: note: def attr2(self) -> Union[P[Any, Any], int] +main:18: note: Got: +main:18: note: def attr2(self) -> str + +[case testSelfTypesWithProtocolsBehaveAsWithNominal] +from typing import Protocol, TypeVar + +T = TypeVar('T', bound='Shape') +class Shape(Protocol): + def combine(self: T, other: T) -> T: + pass + +class NonProtoShape: + def combine(self: T, other: T) -> T: + pass +class Circle: + def combine(self: T, other: Shape) -> T: + pass +class Triangle: + def combine(self, other: Shape) -> Shape: + pass +class Bad: + def combine(self, other: int) -> str: + pass + +def f(s: Shape) -> None: pass +f(NonProtoShape()) +f(Circle()) +s: Shape +if int(): + s = Triangle() + s = Bad() + +n2: NonProtoShape = s +[out] +main:26: error: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") +main:26: note: Following member(s) of "Triangle" have conflicts: +main:26: note: Expected: +main:26: note: def combine(self, other: Triangle) -> Triangle +main:26: note: Got: +main:26: note: def combine(self, other: Shape) -> Shape +main:27: error: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") +main:27: note: Following member(s) of "Bad" have conflicts: +main:27: note: Expected: +main:27: note: def combine(self, other: Bad) -> Bad +main:27: note: Got: +main:27: note: def combine(self, other: int) -> str +main:29: error: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") + +[case testBadVarianceInProtocols] +from typing import Protocol, TypeVar + +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class Proto(Protocol[T_co, T_contra]): # type: ignore + def one(self, x: T_co) -> None: # E: Cannot use a covariant type variable as a parameter + pass + def other(self) -> T_contra: # E: Cannot use a contravariant type variable as return type + pass + +# Check that we respect user overrides of variance after the errors are reported +x: Proto[int, float] +y: Proto[float, int] +y = x # OK +[builtins fixtures/list.pyi] + +[case testSubtleBadVarianceInProtocols] +from typing import Protocol, TypeVar, Iterable, Sequence + +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class Proto(Protocol[T_co, T_contra]): # E: Covariant type variable "T_co" used in protocol where contravariant one is expected \ + # E: Contravariant type variable "T_contra" used in protocol where covariant one is expected + def one(self, x: Iterable[T_co]) -> None: + pass + def other(self) -> Sequence[T_contra]: + pass + +# Check that we respect user overrides of variance after the errors are reported +x: Proto[int, float] +y: Proto[float, int] +y = x # OK +[builtins fixtures/list.pyi] + +-- Recursive protocol types +-- ------------------------ + +[case testRecursiveProtocols1] +from typing import Protocol, Sequence, List, Generic, TypeVar + +T = TypeVar('T') + +class Traversable(Protocol): + @property + def leaves(self) -> Sequence[Traversable]: pass + +class C: pass + +class D(Generic[T]): + leaves: List[D[T]] + +t: Traversable +t = D[int]() # OK +if int(): + t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable") +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] + +[case testRecursiveProtocols2] +from typing import Protocol, TypeVar + +T = TypeVar('T') +class Linked(Protocol[T]): + val: T + def next(self) -> Linked[T]: pass + +class L: + val: int + def next(self) -> L: pass + +def last(seq: Linked[T]) -> T: + pass + +reveal_type(last(L())) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testRecursiveProtocolSubtleMismatch] +from typing import Protocol, TypeVar + +T = TypeVar('T') +class Linked(Protocol[T]): + val: T + def next(self) -> Linked[T]: pass +class L: + val: int + def next(self) -> int: pass + +def last(seq: Linked[T]) -> T: + pass +last(L()) # E: Argument 1 to "last" has incompatible type "L"; expected "Linked[Never]" + +[case testMutuallyRecursiveProtocols] +from typing import Protocol, Sequence, List + +class P1(Protocol): + @property + def attr1(self) -> Sequence[P2]: pass +class P2(Protocol): + @property + def attr2(self) -> Sequence[P1]: pass + +class C: pass +class A: + attr1: List[B] +class B: + attr2: List[A] + +t: P1 +t = A() # OK +if int(): + t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1") + t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1") +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] + +[case testMutuallyRecursiveProtocolsTypesWithSubteMismatch] +from typing import Protocol, Sequence, List + +class P1(Protocol): + @property + def attr1(self) -> Sequence[P2]: pass +class P2(Protocol): + @property + def attr2(self) -> Sequence[P1]: pass + +class C: pass +class A: + attr1: List[B] +class B: + attr2: List[C] + +t: P1 +t = A() # E: Incompatible types in assignment (expression has type "A", variable has type "P1") \ + # N: Following member(s) of "A" have conflicts: \ + # N: attr1: expected "Sequence[P2]", got "List[B]" +[builtins fixtures/list.pyi] + +[case testMutuallyRecursiveProtocolsTypesWithSubteMismatchWriteable] +from typing import Protocol + +class P1(Protocol): + @property + def attr1(self) -> P2: pass +class P2(Protocol): + attr2: P1 + +class A: + attr1: B +class B: + attr2: A + +x: P1 = A() # E: Incompatible types in assignment (expression has type "A", variable has type "P1") \ + # N: Following member(s) of "A" have conflicts: \ + # N: attr1: expected "P2", got "B" +[builtins fixtures/property.pyi] + +[case testTwoUncomfortablyIncompatibleProtocolsWithoutRunningInIssue9771] +from typing import cast, Protocol, TypeVar, Union + +T1 = TypeVar("T1", covariant=True) +T2 = TypeVar("T2") + +class P1(Protocol[T1]): + def b(self) -> int: ... + def a(self, other: "P1[T2]") -> T1: ... + +class P2(Protocol[T1]): + def a(self, other: Union[P1[T2], "P2[T2]"]) -> T1: ... + +p11: P1 = cast(P1, 1) +p12: P1 = cast(P2, 1) # E +p21: P2 = cast(P1, 1) +p22: P2 = cast(P2, 1) # E +[out] +main:14: error: Incompatible types in assignment (expression has type "P2[Any]", variable has type "P1[Any]") +main:14: note: "P2" is missing following "P1" protocol member: +main:14: note: b +main:15: error: Incompatible types in assignment (expression has type "P1[Any]", variable has type "P2[Any]") +main:15: note: Following member(s) of "P1[Any]" have conflicts: +main:15: note: Expected: +main:15: note: def [T2] a(self, other: Union[P1[T2], P2[T2]]) -> Any +main:15: note: Got: +main:15: note: def [T2] a(self, other: P1[T2]) -> Any + +[case testHashable] + +from typing import Hashable, Iterable + +def f(x: Hashable) -> None: + pass + +def g(x: Iterable[str]) -> None: + f(x) # E: Argument 1 to "f" has incompatible type "Iterable[str]"; expected "Hashable" + +[builtins fixtures/object_hashable.pyi] +[typing fixtures/typing-full.pyi] + +-- FIXME: things like this should work +[case testWeirdRecursiveInferenceForProtocols-skip] +from typing import Protocol, TypeVar, Generic +T_co = TypeVar('T_co', covariant=True) +T = TypeVar('T') + +class P(Protocol[T_co]): + def meth(self) -> P[T_co]: pass + +class C(Generic[T]): + def meth(self) -> C[T]: pass + +x: C[int] +def f(arg: P[T]) -> T: pass +reveal_type(f(x)) #E: Revealed type is "builtins.int" + +-- @property, @classmethod and @staticmethod in protocol types +-- ----------------------------------------------------------- + +[case testCannotInstantiateAbstractMethodExplicitProtocolSubtypes] +from typing import Protocol +from abc import abstractmethod + +class P(Protocol): + @abstractmethod + def meth(self) -> int: + pass + +class A(P): + pass + +A() # E: Cannot instantiate abstract class "A" with abstract attribute "meth" + +class C(A): + def meth(self) -> int: + pass +class C2(P): + def meth(self) -> int: + pass + +C() +C2() + +[case testCannotInstantiateAbstractVariableExplicitProtocolSubtypes] +from typing import Protocol + +class P(Protocol): + attr: int + +class A(P): + pass + +A() # E: Cannot instantiate abstract class "A" with abstract attribute "attr" + +class C(A): + attr: int +class C2(P): + def __init__(self) -> None: + self.attr = 1 + +C() +C2() + +class P2(Protocol): + attr: int = 1 + +class B(P2): pass +B() # OK, attr is not abstract + +[case testClassVarsInProtocols] +from typing import Protocol, ClassVar + +class PInst(Protocol): + v: int + +class PClass(Protocol): + v: ClassVar[int] + +class CInst: + v: int + +class CClass: + v: ClassVar[int] + +x: PInst +y: PClass + +x = CInst() +if int(): + x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst") \ + # N: Protocol member PInst.v expected instance variable, got class variable +y = CClass() +if int(): + y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass") \ + # N: Protocol member PClass.v expected class variable, got instance variable + +[case testPropertyInProtocols] +from typing import Protocol + +class PP(Protocol): + @property + def attr(self) -> int: + pass + +class P(Protocol): + attr: int + +x: P +y: PP +y = x + +x2: P +y2: PP +x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P") \ + # N: Protocol member P.attr expected settable variable, got read-only attribute +[builtins fixtures/property.pyi] + +[case testClassVarProtocolImmutable] +from typing import Protocol, ClassVar + +class P(Protocol): + @property + def x(self) -> int: ... + +class C: + x: ClassVar[int] + +class Bad: + x: ClassVar[str] + +x: P = C() +y: P = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "P") \ + # N: Following member(s) of "Bad" have conflicts: \ + # N: x: expected "int", got "str" +[builtins fixtures/property.pyi] + +[case testSettablePropertyInProtocols] +from typing import Protocol + +class PPS(Protocol): + @property + def attr(self) -> int: + pass + @attr.setter + def attr(self, x: int) -> None: + pass + +class PP(Protocol): + @property + def attr(self) -> int: + pass + +class P(Protocol): + attr: int + +x: P +z: PPS +z = x + +x2: P +z2: PPS +x2 = z2 + +y3: PP +z3: PPS +y3 = z3 + +y4: PP +z4: PPS +z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS") \ + # N: Protocol member PPS.attr expected settable variable, got read-only attribute +[builtins fixtures/property.pyi] + +[case testFinalAttributeProtocol] +from typing import Protocol, Final + +class P(Protocol): + x: int + +class C: + def __init__(self, x: int) -> None: + self.x = x +class CF: + def __init__(self, x: int) -> None: + self.x: Final = x + +x: P +y: P +x = C(42) +y = CF(42) # E: Incompatible types in assignment (expression has type "CF", variable has type "P") \ + # N: Protocol member P.x expected settable variable, got read-only attribute + +[case testStaticAndClassMethodsInProtocols] +from typing import Protocol, Type, TypeVar + +class P(Protocol): + def meth(self, x: int) -> str: + pass + +class PC(Protocol): + @classmethod + def meth(cls, x: int) -> str: + pass + +class B: + @staticmethod + def meth(x: int) -> str: + pass + +class C: + def meth(self, x: int) -> str: + pass + +x: P +x = C() +if int(): + x = B() + +y: PC +y = B() +if int(): + y = C() \ + # E: Incompatible types in assignment (expression has type "C", variable has type "PC") \ + # N: Protocol member PC.meth expected class or static method +[builtins fixtures/classmethod.pyi] + +[case testOverloadedMethodsInProtocols] +from typing import overload, Protocol, Union, Optional + +class P(Protocol): + @overload + def f(self, x: int) -> Optional[int]: pass + @overload + def f(self, x: str) -> Optional[str]: pass + +class C: + def f(self, x: Union[int, str]) -> None: + pass +class D: + def f(self, x: int) -> None: + pass + +x: P = C() +if int(): + x = D() +[out] +main:18: error: Incompatible types in assignment (expression has type "D", variable has type "P") +main:18: note: Following member(s) of "D" have conflicts: +main:18: note: Expected: +main:18: note: @overload +main:18: note: def f(self, x: int) -> Optional[int] +main:18: note: @overload +main:18: note: def f(self, x: str) -> Optional[str] +main:18: note: Got: +main:18: note: def f(self, x: int) -> None + +[case testCannotInstantiateProtocolWithOverloadedUnimplementedMethod] +from typing import overload, Protocol + +class P(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass +class C(P): + pass +C() # E: Cannot instantiate abstract class "C" with abstract attribute "meth" + +[case testCanUseOverloadedImplementationsInProtocols] +from typing import overload, Protocol, Union +class P(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bool: pass + def meth(self, x: Union[int, str]): + if isinstance(x, int): + return x + return True + +class C(P): + pass +x = C() +reveal_type(x.meth('hi')) # N: Revealed type is "builtins.bool" +[builtins fixtures/isinstance.pyi] + +[case testProtocolsWithIdenticalOverloads] +from typing import overload, Protocol + +class PA(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass +class PB(Protocol): # identical to above + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass + +x: PA +y: PB +x = y +def fun(arg: PB) -> None: pass +fun(x) + +[case testProtocolsWithIncompatibleOverloads] +from typing import overload, Protocol + +class PA(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass +class PB(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: bytes) -> str: pass + +x: PA +y: PB +x = y +[out] +main:16: error: Incompatible types in assignment (expression has type "PB", variable has type "PA") +main:16: note: Following member(s) of "PB" have conflicts: +main:16: note: Expected: +main:16: note: @overload +main:16: note: def meth(self, x: int) -> int +main:16: note: @overload +main:16: note: def meth(self, x: str) -> bytes +main:16: note: Got: +main:16: note: @overload +main:16: note: def meth(self, x: int) -> int +main:16: note: @overload +main:16: note: def meth(self, x: bytes) -> str + +-- Join and meet with protocol types +-- --------------------------------- + +[case testJoinProtocolWithProtocol] +from typing import Protocol + +class P(Protocol): + attr: int +class P2(Protocol): + attr: int + attr2: str + +x: P +y: P2 + +l0 = [x, x] +l1 = [y, y] +l = [x, y] +reveal_type(l0) # N: Revealed type is "builtins.list[__main__.P]" +reveal_type(l1) # N: Revealed type is "builtins.list[__main__.P2]" +reveal_type(l) # N: Revealed type is "builtins.list[__main__.P]" +[builtins fixtures/list.pyi] + +[case testJoinOfIncompatibleProtocols] +from typing import Protocol + +class P(Protocol): + attr: int +class P2(Protocol): + attr2: str + +x: P +y: P2 +reveal_type([x, y]) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/list.pyi] + +[case testJoinProtocolWithNormal] +from typing import Protocol + +class P(Protocol): + attr: int + +class C: + attr: int + +x: P +y: C + +l = [x, y] + +reveal_type(l) # N: Revealed type is "builtins.list[__main__.P]" +[builtins fixtures/list.pyi] + +[case testMeetProtocolWithProtocol] +from typing import Protocol, Callable, TypeVar + +class P(Protocol): + attr: int +class P2(Protocol): + attr: int + attr2: str + +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: P, y: P2) -> None: pass +reveal_type(f(g)) # N: Revealed type is "__main__.P2" + +[case testMeetOfIncompatibleProtocols] +# flags: --no-strict-optional +from typing import Protocol, Callable, TypeVar + +class P(Protocol): + attr: int +class P2(Protocol): + attr2: str + +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: P, y: P2) -> None: pass +x = f(g) +reveal_type(x) # N: Revealed type is "None" +[case testMeetProtocolWithNormal] +from typing import Protocol, Callable, TypeVar + +class P(Protocol): + attr: int +class C: + attr: int + +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: P, y: C) -> None: pass +reveal_type(f(g)) # N: Revealed type is "__main__.C" + +[case testInferProtocolFromProtocol] +from typing import Protocol, Sequence, TypeVar, Generic + +T = TypeVar('T') +class Box(Protocol[T]): + content: T +class Linked(Protocol[T]): + val: T + def next(self) -> Linked[T]: pass + +class L(Generic[T]): + val: Box[T] + def next(self) -> L[T]: pass + +def last(seq: Linked[T]) -> T: + pass + +reveal_type(last(L[int]())) # N: Revealed type is "__main__.Box[builtins.int]" +reveal_type(last(L[str]()).content) # N: Revealed type is "builtins.str" + +[case testOverloadOnProtocol] +from typing import overload, Protocol, runtime_checkable + +@runtime_checkable +class P1(Protocol): + attr1: int +class P2(Protocol): + attr2: str + +class C1: + attr1: int +class C2: + attr2: str +class C: pass + +@overload +def f(x: P1) -> int: ... +@overload +def f(x: P2) -> str: ... +def f(x): + if isinstance(x, P1): + return P1.attr1 + if isinstance(x, P2): # E: Only @runtime_checkable protocols can be used with instance and class checks + return P1.attr2 + +reveal_type(f(C1())) # N: Revealed type is "builtins.int" +reveal_type(f(C2())) # N: Revealed type is "builtins.str" +class D(C1, C2): pass # Compatible with both P1 and P2 +# TODO: Should this return a union instead? +reveal_type(f(D())) # N: Revealed type is "builtins.int" +f(C()) # E: No overload variant of "f" matches argument type "C" \ + # N: Possible overload variants: \ + # N: def f(x: P1) -> int \ + # N: def f(x: P2) -> str +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +-- Unions of protocol types +-- ------------------------ + +[case testBasicUnionsOfProtocols] +from typing import Union, Protocol + +class P1(Protocol): + attr1: int +class P2(Protocol): + attr2: int + +class C1: + attr1: int +class C2: + attr2: int +class C(C1, C2): + pass + +class B: ... + +x: Union[P1, P2] + +x = C1() +if int(): + x = C2() + x = C() + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "Union[P1, P2]") + +[case testUnionsOfNormalClassesWithProtocols] +from typing import Protocol, Union + +class P1(Protocol): + attr1: int +class P2(Protocol): + attr2: int + +class C1: + attr1: int +class C2: + attr2: int +class C(C1, C2): + pass + +class D1: + attr1: int + +def f1(x: P1) -> None: + pass +def f2(x: P2) -> None: + pass + +x: Union[C1, C2] +y: Union[C1, D1] +z: Union[C, D1] + +f1(x) # E: Argument 1 to "f1" has incompatible type "Union[C1, C2]"; expected "P1" +f1(y) +f1(z) +f2(x) # E: Argument 1 to "f2" has incompatible type "Union[C1, C2]"; expected "P2" +f2(z) # E: Argument 1 to "f2" has incompatible type "Union[C, D1]"; expected "P2" + +-- Type[] with protocol types +-- -------------------------- + +[case testInstantiationProtocolInTypeForFunctions] +from typing import Type, Protocol + +class P(Protocol): + def m(self) -> None: return None +class P1(Protocol): + def m(self) -> None: pass +class Pbad(Protocol): + def mbad(self) -> int: pass +class B(P): pass +class C: + def m(self) -> None: + pass + +def f(cls: Type[P]) -> P: + return cls() # OK +def g() -> P: + return P() # E: Cannot instantiate protocol class "P" + +f(P) # E: Only concrete class can be given where "Type[P]" is expected +f(B) # OK +f(C) # OK +x: Type[P1] +xbad: Type[Pbad] +f(x) # OK +f(xbad) # E: Argument 1 to "f" has incompatible type "Type[Pbad]"; expected "Type[P]" + +[case testInstantiationProtocolInTypeForAliases] +from typing import Type, Protocol + +class P(Protocol): + def m(self) -> None: pass +class C: + def m(self) -> None: + pass + +def f(cls: Type[P]) -> P: + return cls() # OK + +Alias = P +GoodAlias = C +Alias() # E: Cannot instantiate protocol class "P" +GoodAlias() +f(Alias) # E: Only concrete class can be given where "Type[P]" is expected +f(GoodAlias) + +[case testInstantiationProtocolInTypeForVariables] +# flags: --no-strict-optional +from typing import Type, Protocol + +class P(Protocol): + def m(self) -> None: return None +class B(P): pass +class C: + def m(self) -> None: + pass + +var: Type[P] +var() +if int(): + var = P # E: Can only assign concrete classes to a variable of type "Type[P]" + var = B # OK + var = C # OK + +var_old = None # type: Type[P] # Old syntax for variable annotations +var_old() +if int(): + var_old = P # E: Can only assign concrete classes to a variable of type "Type[P]" + var_old = B # OK + var_old = C # OK + +[case testInstantiationProtocolInTypeForClassMethods] +from typing import Type, Protocol + +class Logger: + @staticmethod + def log(a: Type[C]): + pass +class C(Protocol): + @classmethod + def action(cls) -> None: + cls() #OK for classmethods + Logger.log(cls) #OK for classmethods +[builtins fixtures/classmethod.pyi] + +-- isinstance() with @runtime_checkable protocols +-- ---------------------------------------------- + +[case testSimpleRuntimeProtocolCheck] +from typing import Protocol, runtime_checkable + +@runtime_checkable +class C: # E: @runtime_checkable can only be used with protocol classes + pass + +class P(Protocol): + def meth(self) -> None: + pass + +@runtime_checkable +class R(Protocol): + def meth(self) -> int: + pass + +x: object + +if isinstance(x, P): # E: Only @runtime_checkable protocols can be used with instance and class checks + reveal_type(x) # N: Revealed type is "__main__.P" + +if isinstance(x, R): + reveal_type(x) # N: Revealed type is "__main__.R" + reveal_type(x.meth()) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testRuntimeIterableProtocolCheck] +from typing import Iterable, List, Union + +x: Union[int, List[str]] + +if isinstance(x, Iterable): + reveal_type(x) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + +[case testConcreteClassesInProtocolsIsInstance] +from typing import Protocol, runtime_checkable, TypeVar, Generic + +T = TypeVar('T') + +@runtime_checkable +class P1(Protocol): + def meth1(self) -> int: + pass +@runtime_checkable +class P2(Protocol): + def meth2(self) -> int: + pass +@runtime_checkable +class P(P1, P2, Protocol): + pass + +class C1(Generic[T]): + def meth1(self) -> T: + pass +class C2: + def meth2(self) -> int: + pass +class C(C1[int], C2): pass + +c = C() +if isinstance(c, P1): + reveal_type(c) # N: Revealed type is "__main__.C" +else: + reveal_type(c) # Unreachable +if isinstance(c, P): + reveal_type(c) # N: Revealed type is "__main__.C" +else: + reveal_type(c) # Unreachable + +c1i: C1[int] +if isinstance(c1i, P1): + reveal_type(c1i) # N: Revealed type is "__main__.C1[builtins.int]" +else: + reveal_type(c1i) # Unreachable +if isinstance(c1i, P): + reveal_type(c1i) # N: Revealed type is "__main__." +else: + reveal_type(c1i) # N: Revealed type is "__main__.C1[builtins.int]" + +c1s: C1[str] +if isinstance(c1s, P1): + reveal_type(c1s) # Unreachable +else: + reveal_type(c1s) # N: Revealed type is "__main__.C1[builtins.str]" + +c2: C2 +if isinstance(c2, P): + reveal_type(c2) # N: Revealed type is "__main__." +else: + reveal_type(c2) # N: Revealed type is "__main__.C2" + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + +[case testConcreteClassesUnionInProtocolsIsInstance] +from typing import Protocol, runtime_checkable, TypeVar, Generic, Union + +T = TypeVar('T') + +@runtime_checkable +class P1(Protocol): + def meth1(self) -> int: + pass +@runtime_checkable +class P2(Protocol): + def meth2(self) -> int: + pass + +class C1(Generic[T]): + def meth1(self) -> T: + pass +class C2: + def meth2(self) -> int: + pass + +x: Union[C1[int], C2] +if isinstance(x, P1): + reveal_type(x) # N: Revealed type is "__main__.C1[builtins.int]" +else: + reveal_type(x) # N: Revealed type is "__main__.C2" + +if isinstance(x, P2): + reveal_type(x) # N: Revealed type is "__main__.C2" +else: + reveal_type(x) # N: Revealed type is "__main__.C1[builtins.int]" +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + +-- Non-Instances and protocol types (Callable vs __call__ etc.) +-- ------------------------------------------------------------ + +[case testBasicTupleStructuralSubtyping] +from typing import Tuple, TypeVar, Protocol + +T = TypeVar('T', covariant=True) + +class MyProto(Protocol[T]): + def __len__(self) -> T: + pass + +t: Tuple[int, str] +def f(x: MyProto[int]) -> None: + pass +f(t) # OK + +y: MyProto[str] +y = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "MyProto[str]") +[builtins fixtures/isinstancelist.pyi] + +[case testBasicNamedTupleStructuralSubtyping] +from typing import NamedTuple, TypeVar, Protocol + +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) + +class P(Protocol[T, S]): + @property + def x(self) -> T: pass + @property + def y(self) -> S: pass + +class N(NamedTuple): + x: int + y: str +class N2(NamedTuple): + x: int +class N3(NamedTuple): + x: int + y: int + +z: N +z3: N3 + +def fun(x: P[int, str]) -> None: + pass +def fun2(x: P[int, int]) -> None: + pass +def fun3(x: P[T, T]) -> T: + return x.x + +fun(z) +fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected "P[int, int]" \ + # N: Following member(s) of "N" have conflicts: \ + # N: y: expected "int", got "str" + +fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected "P[int, str]" \ + # N: "N2" is missing following "P" protocol member: \ + # N: y + +reveal_type(fun3(z)) # N: Revealed type is "builtins.object" + +reveal_type(fun3(z3)) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testBasicCallableStructuralSubtyping] +from typing import Callable, Generic, TypeVar + +def apply(f: Callable[[int], int], x: int) -> int: + return f(x) + +class Add5: + def __call__(self, x: int) -> int: + return x + 5 + +apply(Add5(), 5) + +T = TypeVar('T') +def apply_gen(f: Callable[[T], T]) -> T: + pass + +reveal_type(apply_gen(Add5())) # N: Revealed type is "builtins.int" +def apply_str(f: Callable[[str], int], x: str) -> int: + return f(x) +apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected "Callable[[str], int]" \ + # N: "Add5.__call__" has type "Callable[[Arg(int, 'x')], int]" +[builtins fixtures/isinstancelist.pyi] + +[case testMoreComplexCallableStructuralSubtyping] +from mypy_extensions import Arg, VarArg +from typing import Protocol, Callable + +def call_soon(cb: Callable[[Arg(int, 'x'), VarArg(str)], int]): pass + +class Good: + def __call__(self, x: int, *rest: str) -> int: pass +class Bad1: + def __call__(self, x: int, *rest: int) -> int: pass +class Bad2: + def __call__(self, y: int, *rest: str) -> int: pass +call_soon(Good()) +call_soon(Bad1()) # E: Argument 1 to "call_soon" has incompatible type "Bad1"; expected "Callable[[int, VarArg(str)], int]" \ + # N: "Bad1.__call__" has type "Callable[[Arg(int, 'x'), VarArg(int)], int]" +call_soon(Bad2()) # E: Argument 1 to "call_soon" has incompatible type "Bad2"; expected "Callable[[int, VarArg(str)], int]" \ + # N: "Bad2.__call__" has type "Callable[[Arg(int, 'y'), VarArg(str)], int]" +[builtins fixtures/isinstancelist.pyi] + +[case testStructuralSupportForPartial] +from typing import Callable, TypeVar, Generic, Any + +T = TypeVar('T') + +class partial(Generic[T]): + def __init__(self, func: Callable[..., T], *args: Any) -> None: ... + def __call__(self, *args: Any) -> T: ... + +def inc(a: int, temp: str) -> int: + pass + +def foo(f: Callable[[int], T]) -> T: + return f(1) + +reveal_type(foo(partial(inc, 'temp'))) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testStructuralInferenceForCallable] +from typing import Callable, TypeVar, Tuple + +T = TypeVar('T') +S = TypeVar('S') + +class Actual: + def __call__(self, arg: int) -> str: pass + +def fun(cb: Callable[[T], S]) -> Tuple[T, S]: pass +reveal_type(fun(Actual())) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +-- Standard protocol types (SupportsInt, Sized, etc.) +-- -------------------------------------------------- + +-- More tests could be added for types from typing converted to protocols + +[case testBasicSizedProtocol] +from typing import Sized + +class Foo: + def __len__(self) -> int: + return 42 + +def bar(a: Sized) -> int: + return a.__len__() + +bar(Foo()) +bar((1, 2)) +bar(1) # E: Argument 1 to "bar" has incompatible type "int"; expected "Sized" + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +[case testBasicSupportsIntProtocol] +from typing import SupportsInt + +class Bar: + def __int__(self): + return 1 + +def foo(a: SupportsInt): + pass + +foo(Bar()) +foo('no way') # E: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +-- Additional tests and corner cases for protocols +-- ---------------------------------------------- + +[case testAnyWithProtocols] +from typing import Protocol, Any, TypeVar + +T = TypeVar('T') + +class P1(Protocol): + attr1: int +class P2(Protocol[T]): + attr2: T +class P3(Protocol): + attr: P3 + +def f1(x: P1) -> None: pass +def f2(x: P2[str]) -> None: pass +def f3(x: P3) -> None: pass + +class C1: + attr1: Any +class C2: + attr2: Any +class C3: + attr: Any + +f1(C1()) +f2(C2()) +f3(C3()) + +f2(C3()) # E: Argument 1 to "f2" has incompatible type "C3"; expected "P2[str]" +a: Any +f1(a) +f2(a) +f3(a) + +[case testErrorsForProtocolsInDifferentPlaces] +from typing import Protocol + +class P(Protocol): + attr1: int + attr2: str + attr3: int + +class C: + attr1: str + @property + def attr2(self) -> int: pass + +x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ + # N: "C" is missing following "P" protocol member: \ + # N: attr3 \ + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "int", got "str" \ + # N: attr2: expected "str", got "int" \ + # N: Protocol member P.attr2 expected settable variable, got read-only attribute + +def f(x: P) -> P: + return C() # E: Incompatible return value type (got "C", expected "P") \ + # N: "C" is missing following "P" protocol member: \ + # N: attr3 \ + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "int", got "str" \ + # N: attr2: expected "str", got "int" \ + # N: Protocol member P.attr2 expected settable variable, got read-only attribute + +f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ + # N: "C" is missing following "P" protocol member: \ + # N: attr3 \ + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "int", got "str" \ + # N: attr2: expected "str", got "int" \ + # N: Protocol member P.attr2 expected settable variable, got read-only attribute +[builtins fixtures/list.pyi] + +[case testIterableProtocolOnClass] +from typing import TypeVar, Iterator +T = TypeVar('T', bound='A') + +class A: + def __iter__(self: T) -> Iterator[T]: pass + +class B(A): pass + +reveal_type(list(b for b in B())) # N: Revealed type is "builtins.list[__main__.B]" +reveal_type(list(B())) # N: Revealed type is "builtins.list[__main__.B]" +[builtins fixtures/list.pyi] + +[case testIterableProtocolOnMetaclass] +from typing import TypeVar, Iterator, Type +T = TypeVar('T') + +class EMeta(type): + def __iter__(self: Type[T]) -> Iterator[T]: pass + +class E(metaclass=EMeta): + pass + +class C(E): + pass + +reveal_type(list(c for c in C)) # N: Revealed type is "builtins.list[__main__.C]" +reveal_type(list(C)) # N: Revealed type is "builtins.list[__main__.C]" +[builtins fixtures/list.pyi] + +[case testClassesGetattrWithProtocols] +from typing import Protocol + +class P(Protocol): + attr: int + +class PP(Protocol): + @property + def attr(self) -> int: + pass + +class C: + def __getattr__(self, attr: str) -> int: + pass +class C2(C): + def __setattr__(self, attr: str, val: int) -> None: + pass + +class D: + def __getattr__(self, attr: str) -> str: + pass + +def fun(x: P) -> None: + reveal_type(P.attr) # N: Revealed type is "builtins.int" +def fun_p(x: PP) -> None: + reveal_type(P.attr) # N: Revealed type is "builtins.int" + +fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P" \ + # N: Protocol member P.attr expected settable variable, got read-only attribute +fun(C2()) +fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" \ + # N: Following member(s) of "D" have conflicts: \ + # N: attr: expected "int", got "str" +fun_p(C()) # OK +[builtins fixtures/list.pyi] + +[case testImplicitTypesInProtocols] +from typing import Protocol + +class P(Protocol): + x = 1 # E: All protocol members must have explicitly declared types + +class C: + x: int + +class D: + x: str + +x: P +x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ + # N: Following member(s) of "D" have conflicts: \ + # N: x: expected "int", got "str" +x = C() # OK +[builtins fixtures/list.pyi] + +[case testProtocolIncompatibilityWithGenericMethod] +from typing import Protocol, TypeVar + +T = TypeVar('T') +S = TypeVar('S') + +class A(Protocol): + def f(self, x: T) -> None: pass +class B: + def f(self, x: S, y: T) -> None: pass + +x: A = B() +[out] +main:11: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:11: note: Following member(s) of "B" have conflicts: +main:11: note: Expected: +main:11: note: def [T] f(self, x: T) -> None +main:11: note: Got: +main:11: note: def [S, T] f(self, x: S, y: T) -> None + +[case testProtocolIncompatibilityWithGenericMethodBounded] +from typing import Protocol, TypeVar + +T = TypeVar('T') +S = TypeVar('S', bound=int) + +class A(Protocol): + def f(self, x: T) -> None: pass +class B: + def f(self, x: S, y: T) -> None: pass + +x: A = B() +[out] +main:11: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:11: note: Following member(s) of "B" have conflicts: +main:11: note: Expected: +main:11: note: def [T] f(self, x: T) -> None +main:11: note: Got: +main:11: note: def [S: int, T] f(self, x: S, y: T) -> None + +[case testProtocolIncompatibilityWithGenericRestricted] +from typing import Protocol, TypeVar + +T = TypeVar('T') +S = TypeVar('S', int, str) + +class A(Protocol): + def f(self, x: T) -> None: pass +class B: + def f(self, x: S, y: T) -> None: pass + +x: A = B() +[out] +main:11: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:11: note: Following member(s) of "B" have conflicts: +main:11: note: Expected: +main:11: note: def [T] f(self, x: T) -> None +main:11: note: Got: +main:11: note: def [S: (int, str), T] f(self, x: S, y: T) -> None + +[case testProtocolIncompatibilityWithManyOverloads] +from typing import Protocol, overload + +class C1: pass +class C2: pass +class A(Protocol): + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass + @overload + def f(self, x: C1) -> C2: pass + @overload + def f(self, x: C2) -> C1: pass + +class B: + def f(self) -> None: pass + +x: A = B() +[out] +main:18: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:18: note: Following member(s) of "B" have conflicts: +main:18: note: Expected: +main:18: note: @overload +main:18: note: def f(self, x: int) -> int +main:18: note: @overload +main:18: note: def f(self, x: str) -> str +main:18: note: @overload +main:18: note: def f(self, x: C1) -> C2 +main:18: note: @overload +main:18: note: def f(self, x: C2) -> C1 +main:18: note: Got: +main:18: note: def f(self) -> None + +[case testProtocolIncompatibilityWithManyConflicts] +from typing import Protocol + +class A(Protocol): + def f(self, x: int) -> None: pass + def g(self, x: int) -> None: pass + def h(self, x: int) -> None: pass + def i(self, x: int) -> None: pass +class B: + def f(self, x: str) -> None: pass + def g(self, x: str) -> None: pass + def h(self, x: str) -> None: pass + def i(self, x: str) -> None: pass + +x: A = B() +[out] +main:14: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:14: note: Following member(s) of "B" have conflicts: +main:14: note: Expected: +main:14: note: def f(self, x: int) -> None +main:14: note: Got: +main:14: note: def f(self, x: str) -> None +main:14: note: Expected: +main:14: note: def g(self, x: int) -> None +main:14: note: Got: +main:14: note: def g(self, x: str) -> None +main:14: note: <2 more conflict(s) not shown> + +[case testProtocolIncompatibilityWithUnionType] +from typing import Any, Optional, Protocol + +class A(Protocol): + def execute(self, statement: Any, *args: Any, **kwargs: Any) -> None: ... + +class B(Protocol): + def execute(self, stmt: Any, *args: Any, **kwargs: Any) -> None: ... + def cool(self) -> None: ... + +def func1(arg: A) -> None: ... +def func2(arg: Optional[A]) -> None: ... + +x: B +func1(x) +func2(x) +[builtins fixtures/dict.pyi] +[out] +main:14: error: Argument 1 to "func1" has incompatible type "B"; expected "A" +main:14: note: Following member(s) of "B" have conflicts: +main:14: note: Expected: +main:14: note: def execute(self, statement: Any, *args: Any, **kwargs: Any) -> None +main:14: note: Got: +main:14: note: def execute(self, stmt: Any, *args: Any, **kwargs: Any) -> None +main:15: error: Argument 1 to "func2" has incompatible type "B"; expected "Optional[A]" +main:15: note: Following member(s) of "B" have conflicts: +main:15: note: Expected: +main:15: note: def execute(self, statement: Any, *args: Any, **kwargs: Any) -> None +main:15: note: Got: +main:15: note: def execute(self, stmt: Any, *args: Any, **kwargs: Any) -> None + +[case testDontShowNotesForTupleAndIterableProtocol] +from typing import Iterable, Sequence, Protocol, NamedTuple + +class N(NamedTuple): + x: int + +def f1(x: Iterable[str]) -> None: pass +def f2(x: Sequence[str]) -> None: pass + +# The errors below should be short +f1(N(1)) # E: Argument 1 to "f1" has incompatible type "N"; expected "Iterable[str]" +f2(N(2)) # E: Argument 1 to "f2" has incompatible type "N"; expected "Sequence[str]" +[builtins fixtures/tuple.pyi] + +[case testNotManyFlagConflitsShownInProtocols] +from typing import Protocol + +class AllSettable(Protocol): + a: int + b: int + c: int + d: int + +class AllReadOnly: + @property + def a(self) -> int: pass + @property + def b(self) -> int: pass + @property + def c(self) -> int: pass + @property + def d(self) -> int: pass + +x: AllSettable = AllReadOnly() +[builtins fixtures/property.pyi] +[out] +main:19: error: Incompatible types in assignment (expression has type "AllReadOnly", variable has type "AllSettable") +main:19: note: Protocol member AllSettable.a expected settable variable, got read-only attribute +main:19: note: Protocol member AllSettable.b expected settable variable, got read-only attribute +main:19: note: <2 more conflict(s) not shown> + +[case testProtocolsMoreConflictsNotShown] +from typing_extensions import Protocol +from typing import Generic, TypeVar + +T = TypeVar('T') + +class MockMapping(Protocol[T]): + def a(self, x: T) -> int: pass + def b(self, x: T) -> int: pass + def c(self, x: T) -> int: pass + d: T + e: T + f: T + +class MockDict(MockMapping[T]): + more: int + +def f(x: MockMapping[int]) -> None: pass +x: MockDict[str] +f(x) # E: Argument 1 to "f" has incompatible type "MockDict[str]"; expected "MockMapping[int]" +[builtins fixtures/tuple.pyi] + +[case testProtocolNotesForComplexSignatures] +from typing import Protocol, Optional + +class P(Protocol): + def meth(self, x: int, *args: str) -> None: pass + def other(self, *args, hint: Optional[str] = None, **kwargs: str) -> None: pass +class C: + def meth(self) -> int: pass + def other(self) -> int: pass + +x: P = C() +[builtins fixtures/dict.pyi] +[out] +main:10: error: Incompatible types in assignment (expression has type "C", variable has type "P") +main:10: note: Following member(s) of "C" have conflicts: +main:10: note: Expected: +main:10: note: def meth(self, x: int, *args: str) -> None +main:10: note: Got: +main:10: note: def meth(self) -> int +main:10: note: Expected: +main:10: note: def other(self, *args: Any, hint: Optional[str] = ..., **kwargs: str) -> None +main:10: note: Got: +main:10: note: def other(self) -> int + +[case testObjectAllowedInProtocolBases] +from typing import Protocol +class P(Protocol, object): + pass +[out] + +[case testNoneSubtypeOfEmptyProtocol] +from typing import Protocol +class P(Protocol): + pass + +x: P = None +[out] + +[case testNoneSubtypeOfAllProtocolsWithoutStrictOptional] +# flags: --no-strict-optional +from typing import Protocol +class P(Protocol): + attr: int + def meth(self, arg: str) -> str: + pass + +x: P = None +[out] + +[case testNoneSubtypeOfEmptyProtocolStrict] +from typing import Protocol +class P(Protocol): + pass +x: P = None + +class PBad(Protocol): + x: int +y: PBad = None # E: Incompatible types in assignment (expression has type "None", variable has type "PBad") +[out] + +[case testOnlyMethodProtocolUsableWithIsSubclass] +from typing import Protocol, runtime_checkable, Union, Type, Sequence, overload +@runtime_checkable +class P(Protocol): + def meth(self) -> int: + pass +@runtime_checkable +class PBad(Protocol): + x: str + +class C: + x: str + def meth(self) -> int: + pass +class E: pass + +cls: Type[Union[C, E]] +issubclass(cls, PBad) # E: Only protocols that don't have non-method members can be used with issubclass() \ + # N: Protocol "PBad" has non-method member(s): x +if issubclass(cls, P): + reveal_type(cls) # N: Revealed type is "Type[__main__.C]" +else: + reveal_type(cls) # N: Revealed type is "Type[__main__.E]" + +@runtime_checkable +class POverload(Protocol): + @overload + def meth(self, a: int) -> float: ... + @overload + def meth(self, a: str) -> Sequence[float]: ... + def meth(self, a): + pass + +reveal_type(issubclass(int, POverload)) # N: Revealed type is "builtins.bool" +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] +[out] + +[case testCallableImplementsProtocol] +from typing import Protocol + +class Caller(Protocol): + def __call__(self, x: str, *args: int) -> None: ... + +def call(x: str, *args: int) -> None: + pass +def bad(x: int, *args: str) -> None: + pass + +def func(caller: Caller) -> None: + pass + +func(call) +func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[int, VarArg(str)], None]"; expected "Caller" \ + # N: "Caller.__call__" has type "Callable[[Arg(str, 'x'), VarArg(int)], None]" +[builtins fixtures/tuple.pyi] +[out] + +[case testCallableImplementsProtocolGeneric] +from typing import Protocol, TypeVar, Tuple + +T = TypeVar('T') +S = TypeVar('S') + +class Caller(Protocol[T, S]): + def __call__(self, x: T, y: S) -> Tuple[T, S]: ... + +def call(x: int, y: str) -> Tuple[int, str]: ... + +def func(caller: Caller[T, S]) -> Tuple[T, S]: + pass + +reveal_type(func(call)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] +[out] + +[case testCallableImplementsProtocolGenericTight] +from typing import Protocol, TypeVar + +T = TypeVar('T') + +class Caller(Protocol): + def __call__(self, x: T) -> T: ... + +def call(x: T) -> T: ... +def bad(x: int) -> int: ... + +def func(caller: Caller) -> None: + pass + +func(call) +func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[int], int]"; expected "Caller" \ + # N: "Caller.__call__" has type "Callable[[Arg(T, 'x')], T]" +[builtins fixtures/tuple.pyi] +[out] + +[case testCallableImplementsProtocolGenericNotGeneric] +from typing import Protocol, TypeVar, Tuple + +T = TypeVar('T') + +class Caller(Protocol): + def __call__(self, x: int) -> int: ... + +def call(x: T) -> T: ... + +def bad(x: T) -> Tuple[T, T]: ... + +def func(caller: Caller) -> None: + pass + +func(call) +func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[T], Tuple[T, T]]"; expected "Caller" \ + # N: "Caller.__call__" has type "Callable[[Arg(int, 'x')], int]" +[builtins fixtures/tuple.pyi] +[out] + +[case testCallableImplementsProtocolOverload] +from typing import Protocol, overload, Union + +class Caller(Protocol): + @overload + def __call__(self, x: int) -> int: ... + @overload + def __call__(self, x: str) -> str: ... + +@overload +def call(x: int) -> int: ... +@overload +def call(x: str) -> str: ... +def call(x: Union[int, str]) -> Union[int, str]: + pass + +def bad(x: Union[int, str]) -> Union[int, str]: + pass + +def func(caller: Caller) -> None: + pass + +func(call) +func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[Union[int, str]], Union[int, str]]"; expected "Caller" \ + # N: "Caller.__call__" has type overloaded function +[out] + +[case testCallableImplementsProtocolExtraNote] +from typing import Protocol + +class Caller(Protocol): + def __call__(self, x: str, *args: int) -> None: ... + +def bad(x: int, *args: str) -> None: + pass + +cb: Caller = bad # E: Incompatible types in assignment (expression has type "Callable[[int, VarArg(str)], None]", variable has type "Caller") \ + # N: "Caller.__call__" has type "Callable[[Arg(str, 'x'), VarArg(int)], None]" +[builtins fixtures/tuple.pyi] +[out] + +[case testCallableImplementsProtocolArgName] +from typing import Protocol + +class Caller(Protocol): + def __call__(self, x: str) -> None: ... + +class CallerAnon(Protocol): + def __call__(self, __x: str) -> None: ... + +def call(x: str) -> None: + pass +def bad(y: str) -> None: + pass + +def func(caller: Caller) -> None: + pass + +def anon(caller: CallerAnon) -> None: + pass + + +func(call) +func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[str], None]"; expected "Caller" \ + # N: "Caller.__call__" has type "Callable[[Arg(str, 'x')], None]" +anon(bad) +[out] + +[case testCallableProtocolVsProtocol] +from typing import Protocol + +class One(Protocol): + def __call__(self, x: str) -> None: ... + +class Other(Protocol): + def __call__(self, x: str) -> None: ... + +class Bad(Protocol): + def __call__(self, zzz: str) -> None: ... + +def func(caller: One) -> None: + pass + +a: Other +b: Bad + +func(a) +func(b) # E: Argument 1 to "func" has incompatible type "Bad"; expected "One" \ + # N: "One.__call__" has type "Callable[[Arg(str, 'x')], None]" +[out] + +[case testJoinProtocolCallback] +from typing import Protocol, Callable + +class A: ... +class B(A): ... +class C(B): ... +class D(B): ... + +class Call(Protocol): + def __call__(self, x: B) -> C: ... +Normal = Callable[[A], D] + +a: Call +b: Normal + +reveal_type([a, b]) # N: Revealed type is "builtins.list[def (__main__.B) -> __main__.B]" +reveal_type([b, a]) # N: Revealed type is "builtins.list[def (__main__.B) -> __main__.B]" +[builtins fixtures/list.pyi] +[out] + +[case testMeetProtocolCallback] +from typing import Protocol, Callable + +class A: ... +class B(A): ... +class C(B): ... +class D(B): ... + +class Call(Protocol): + def __call__(self, __x: C) -> B: ... +Normal = Callable[[D], A] + +def a(x: Call) -> None: ... +def b(x: Normal) -> None: ... + +reveal_type([a, b]) # N: Revealed type is "builtins.list[def (x: def (__main__.B) -> __main__.B)]" +reveal_type([b, a]) # N: Revealed type is "builtins.list[def (x: def (__main__.B) -> __main__.B)]" +[builtins fixtures/list.pyi] +[out] + +[case testCallbackProtocolFunctionAttributesSubtyping] +from typing import Protocol + +class A(Protocol): + __name__: str + def __call__(self) -> str: ... + +class B1(Protocol): + __name__: int + def __call__(self) -> str: ... + +class B2(Protocol): + __name__: str + def __call__(self) -> int: ... + +class B3(Protocol): + __name__: str + extra_stuff: int + def __call__(self) -> str: ... + +def f() -> str: ... + +reveal_type(f.__name__) # N: Revealed type is "builtins.str" +a: A = f # OK +b1: B1 = f # E: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "B1") \ + # N: Following member(s) of "function" have conflicts: \ + # N: __name__: expected "int", got "str" +b2: B2 = f # E: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "B2") \ + # N: "B2.__call__" has type "Callable[[], int]" +b3: B3 = f # E: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "B3") \ + # N: "function" is missing following "B3" protocol member: \ + # N: extra_stuff + +[case testCallbackProtocolFunctionAttributesInference] +from typing import Protocol, TypeVar, Generic, Tuple + +T = TypeVar("T") +S = TypeVar("S", covariant=True) +class A(Protocol[T, S]): + __name__: T + def __call__(self) -> S: ... + +def f() -> int: ... +def test(func: A[T, S]) -> Tuple[T, S]: ... +reveal_type(test(f)) # N: Revealed type is "Tuple[builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testProtocolsAlwaysABCs] +from typing import Protocol + +class P(Protocol): ... +class C(P): ... + +reveal_type(C.register(int)) # N: Revealed type is "def () -> builtins.int" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] +[out] + +[case testProtocolVarianceAfterDecorators] +# The test case is simplified, in reality this caused problems with @abstractmethod +# in stubs and test fixtures. +from typing import Protocol, TypeVar + +T = TypeVar('T') +def dec(x: T) -> T: ... +alias = dec + +class P(Protocol[T]): + @alias + def meth(self, arg: T) -> T: ... +[out] + +[case testNamedTupleWithNoArgsCallableField] +from typing import Callable, NamedTuple, Protocol + +class N(NamedTuple): + func: Callable[[], str] + +class P(Protocol): + @property + def func(self) -> Callable[[], str]: ... + +p: P = N(lambda: 'foo') +[builtins fixtures/property.pyi] + +[case testNamedTupleWithManyArgsCallableField] +from typing import Callable, NamedTuple, Protocol + +class N(NamedTuple): + func: Callable[[str, str, str], str] + +class P(Protocol): + @property + def func(self) -> Callable[[str, str, str], str]: ... + +p: P = N(lambda a, b, c: 'foo') +[builtins fixtures/property.pyi] + +[case testLiteralsAgainstProtocols] +from typing import SupportsInt, SupportsAbs, TypeVar +from typing_extensions import Literal, Final + +T = TypeVar('T') +def abs(x: SupportsAbs[T]) -> T: ... +def foo(x: SupportsInt) -> None: ... + +ONE: Final = 1 +TWO: Literal[2] +ALL: Literal[1, 2, 3] + +foo(ONE) +foo(TWO) +foo(3) + +reveal_type(abs(ONE)) # N: Revealed type is "builtins.int" +reveal_type(abs(TWO)) # N: Revealed type is "builtins.int" +reveal_type(abs(3)) # N: Revealed type is "builtins.int" +reveal_type(abs(ALL)) # N: Revealed type is "builtins.int" +[builtins fixtures/float.pyi] +[typing fixtures/typing-full.pyi] + +[case testProtocolWithSlots] +from typing import Protocol + +class A(Protocol): + __slots__ = () + +[builtins fixtures/tuple.pyi] + +[case testProtocolSlotsIsNotProtocolMember] +# https://github.com/python/mypy/issues/11884 +from typing import Protocol + +class Foo(Protocol): + __slots__ = () +class NoSlots: + pass +class EmptySlots: + __slots__ = () +class TupleSlots: + __slots__ = ('x', 'y') +class StringSlots: + __slots__ = 'x y' +class InitSlots: + __slots__ = ('x',) + def __init__(self) -> None: + self.x = None +def foo(f: Foo): + pass + +# All should pass: +foo(NoSlots()) +foo(EmptySlots()) +foo(TupleSlots()) +foo(StringSlots()) +foo(InitSlots()) +[builtins fixtures/tuple.pyi] + +[case testProtocolSlotsAndRuntimeCheckable] +from typing import Protocol, runtime_checkable + +@runtime_checkable +class Foo(Protocol): + __slots__ = () +class Bar: + pass +issubclass(Bar, Foo) # Used to be an error, when `__slots__` counted as a protocol member +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + + +[case testProtocolWithClassGetItem] +# https://github.com/python/mypy/issues/11886 +from typing import Any, Iterable, Protocol, Union + +class B: + ... + +class C: + def __class_getitem__(cls, __item: Any) -> Any: + ... + +class SupportsClassGetItem(Protocol): + __slots__: Union[str, Iterable[str]] = () + def __class_getitem__(cls, __item: Any) -> Any: + ... + +b1: SupportsClassGetItem = B() +c1: SupportsClassGetItem = C() +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + + +[case testNoneVsProtocol] +# mypy: strict-optional +from typing_extensions import Protocol + +class MyHashable(Protocol): + def __hash__(self) -> int: ... + +def f(h: MyHashable) -> None: pass +f(None) + +class Proto(Protocol): + def __hash__(self) -> int: ... + def method(self) -> None: ... + +def g(h: Proto) -> None: pass +g(None) # E: Argument 1 to "g" has incompatible type "None"; expected "Proto" + +class Proto2(Protocol): + def hash(self) -> None: ... + +def h(h: Proto2) -> None: pass +h(None) # E: Argument 1 to "h" has incompatible type "None"; expected "Proto2" + +class EmptyProto(Protocol): ... + +def hh(h: EmptyProto) -> None: pass +hh(None) + +# See https://github.com/python/mypy/issues/13081 +class SupportsStr(Protocol): + def __str__(self) -> str: ... + +def ss(s: SupportsStr) -> None: pass +ss(None) + +class HashableStr(Protocol): + def __str__(self) -> str: ... + def __hash__(self) -> int: ... + +def hs(n: HashableStr) -> None: pass +hs(None) +[builtins fixtures/tuple.pyi] + + +[case testPartialTypeProtocol] +from typing import Protocol + +class Flapper(Protocol): + def flap(self) -> int: ... + +class Blooper: + flap = None + + def bloop(self, x: Flapper) -> None: + reveal_type([self, x]) # N: Revealed type is "builtins.list[builtins.object]" + +class Gleemer: + flap = [] # E: Need type annotation for "flap" (hint: "flap: List[] = ...") + + def gleem(self, x: Flapper) -> None: + reveal_type([self, x]) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/tuple.pyi] + + +[case testPartialTypeProtocolHashable] +# flags: --no-strict-optional +from typing import Protocol + +class Hashable(Protocol): + def __hash__(self) -> int: ... + +class ObjectHashable: + def __hash__(self) -> int: ... + +class DataArray(ObjectHashable): + __hash__ = None + + def f(self, x: Hashable) -> None: + reveal_type([self, x]) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/tuple.pyi] + + +[case testPartialAttributeNoneType] +# flags: --no-strict-optional +from typing import Optional, Protocol, runtime_checkable + +@runtime_checkable +class MyProtocol(Protocol): + def is_valid(self) -> bool: ... + text: Optional[str] + +class MyClass: + text = None + def is_valid(self) -> bool: + reveal_type(self.text) # N: Revealed type is "None" + assert isinstance(self, MyProtocol) +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + + +[case testPartialAttributeNoneTypeStrictOptional] +from typing import Optional, Protocol, runtime_checkable + +@runtime_checkable +class MyProtocol(Protocol): + def is_valid(self) -> bool: ... + text: Optional[str] + +class MyClass: + text = None + def is_valid(self) -> bool: + reveal_type(self.text) # N: Revealed type is "None" + assert isinstance(self, MyProtocol) +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testProtocolAndTypeVariableSpecialCase] +from typing import TypeVar, Iterable, Optional, Callable, Protocol + +T_co = TypeVar('T_co', covariant=True) + +class SupportsNext(Protocol[T_co]): + def __next__(self) -> T_co: ... + +N = TypeVar("N", bound=SupportsNext, covariant=True) + +class SupportsIter(Protocol[T_co]): + def __iter__(self) -> T_co: ... + +def f(i: SupportsIter[N]) -> N: ... + +I = TypeVar('I', bound=Iterable) + +def g(x: I, y: Iterable) -> None: + f(x) + f(y) + +[case testMatchProtocolAgainstOverloadWithAmbiguity] +from typing import TypeVar, Protocol, Union, Generic, overload + +T = TypeVar("T", covariant=True) + +class slice: pass + +class GetItem(Protocol[T]): + def __getitem__(self, k: int) -> T: ... + +class Str: # Resembles 'str' + def __getitem__(self, k: Union[int, slice]) -> Str: ... + +class Lst(Generic[T]): # Resembles 'list' + def __init__(self, x: T): ... + @overload + def __getitem__(self, k: int) -> T: ... + @overload + def __getitem__(self, k: slice) -> Lst[T]: ... + def __getitem__(self, k): pass + +def f(x: GetItem[GetItem[Str]]) -> None: ... + +a: Lst[Str] +f(Lst(a)) + +class Lst2(Generic[T]): + def __init__(self, x: T): ... + # The overload items are tweaked but still compatible + @overload + def __getitem__(self, k: Str) -> None: ... + @overload + def __getitem__(self, k: slice) -> Lst2[T]: ... + @overload + def __getitem__(self, k: Union[int, str]) -> T: ... + def __getitem__(self, k): pass + +b: Lst2[Str] +f(Lst2(b)) + +class Lst3(Generic[T]): # Resembles 'list' + def __init__(self, x: T): ... + # The overload items are no longer compatible (too narrow argument type) + @overload + def __getitem__(self, k: slice) -> Lst3[T]: ... + @overload + def __getitem__(self, k: bool) -> T: ... + def __getitem__(self, k): pass + +c: Lst3[Str] +f(Lst3(c)) # E: Argument 1 to "f" has incompatible type "Lst3[Lst3[Str]]"; expected "GetItem[GetItem[Str]]" \ +# N: Following member(s) of "Lst3[Lst3[Str]]" have conflicts: \ +# N: Expected: \ +# N: def __getitem__(self, int, /) -> GetItem[Str] \ +# N: Got: \ +# N: @overload \ +# N: def __getitem__(self, slice, /) -> Lst3[Lst3[Str]] \ +# N: @overload \ +# N: def __getitem__(self, bool, /) -> Lst3[Str] + +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + +[case testMatchProtocolAgainstOverloadWithMultipleMatchingItems] +from typing import Protocol, overload, TypeVar, Any + +_T_co = TypeVar("_T_co", covariant=True) +_T = TypeVar("_T") + +class SupportsRound(Protocol[_T_co]): + @overload + def __round__(self) -> int: ... + @overload + def __round__(self, __ndigits: int) -> _T_co: ... + +class C: + # This matches both overload items of SupportsRound + def __round__(self, __ndigits: int = ...) -> int: ... + +def round(number: SupportsRound[_T], ndigits: int) -> _T: ... + +round(C(), 1) + +[case testEmptyBodyImplicitlyAbstractProtocol] +from typing import Protocol, overload, Union + +class P1(Protocol): + def meth(self) -> int: ... +class B1(P1): ... +class C1(P1): + def meth(self) -> int: + return 0 +B1() # E: Cannot instantiate abstract class "B1" with abstract attribute "meth" +C1() + +class P2(Protocol): + @classmethod + def meth(cls) -> int: ... +class B2(P2): ... +class C2(P2): + @classmethod + def meth(cls) -> int: + return 0 +B2() # E: Cannot instantiate abstract class "B2" with abstract attribute "meth" +C2() + +class P3(Protocol): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + @overload + def not_abstract(self, x: int) -> int: ... + @overload + def not_abstract(self, x: str) -> str: ... + def not_abstract(self, x: Union[int, str]) -> Union[int, str]: + return 0 +class B3(P3): ... +class C3(P3): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return 0 +B3() # E: Cannot instantiate abstract class "B3" with abstract attribute "meth" +C3() +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocolProperty] +from typing import Protocol + +class P1(Protocol): + @property + def attr(self) -> int: ... +class B1(P1): ... +class C1(P1): + @property + def attr(self) -> int: + return 0 +B1() # E: Cannot instantiate abstract class "B1" with abstract attribute "attr" +C1() + +class P2(Protocol): + @property + def attr(self) -> int: ... + @attr.setter + def attr(self, value: int) -> None: ... +class B2(P2): ... +class C2(P2): + @property + def attr(self) -> int: return 0 + @attr.setter + def attr(self, value: int) -> None: pass +B2() # E: Cannot instantiate abstract class "B2" with abstract attribute "attr" +C2() +[builtins fixtures/property.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocolStub] +from stub import P1, P2, P3, P4 + +class B1(P1): ... +class B2(P2): ... +class B3(P3): ... +class B4(P4): ... + +B1() +B2() +B3() +B4() # E: Cannot instantiate abstract class "B4" with abstract attribute "meth" + +[file stub.pyi] +from typing import Protocol, overload, Union +from abc import abstractmethod + +class P1(Protocol): + def meth(self) -> int: ... + +class P2(Protocol): + @classmethod + def meth(cls) -> int: ... + +class P3(Protocol): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + +class P4(Protocol): + @abstractmethod + def meth(self) -> int: ... +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyVariationsImplicitlyAbstractProtocol] +from typing import Protocol + +class WithPass(Protocol): + def meth(self) -> int: + pass +class A(WithPass): ... +A() # E: Cannot instantiate abstract class "A" with abstract attribute "meth" + +class WithEllipses(Protocol): + def meth(self) -> int: ... +class B(WithEllipses): ... +B() # E: Cannot instantiate abstract class "B" with abstract attribute "meth" + +class WithDocstring(Protocol): + def meth(self) -> int: + """Docstring for meth. + + This is meth.""" +class C(WithDocstring): ... +C() # E: Cannot instantiate abstract class "C" with abstract attribute "meth" + +class WithRaise(Protocol): + def meth(self) -> int: + """Docstring for meth.""" + raise NotImplementedError +class D(WithRaise): ... +D() # E: Cannot instantiate abstract class "D" with abstract attribute "meth" +[builtins fixtures/exception.pyi] + +[case testEmptyBodyNoneCompatibleProtocol] +from abc import abstractmethod +from typing import Any, Optional, Protocol, Union, overload +from typing_extensions import TypeAlias + +NoneAlias: TypeAlias = None + +class NoneCompatible(Protocol): + def f(self) -> None: ... + def g(self) -> Any: ... + def h(self) -> Optional[int]: ... + def i(self) -> NoneAlias: ... + @classmethod + def j(cls) -> None: ... + +class A(NoneCompatible): ... +A() # E: Cannot instantiate abstract class "A" with abstract attributes "f", "g", "h", "i" and "j" \ + # N: The following methods were marked implicitly abstract because they have empty function bodies: "f", "g", "h", "i" and "j". If they are not meant to be abstract, explicitly `return` or `return None`. + +class NoneCompatible2(Protocol): + def f(self, x: int): ... + +class B(NoneCompatible2): ... +B() # E: Cannot instantiate abstract class "B" with abstract attribute "f" \ + # N: "f" is implicitly abstract because it has an empty function body. If it is not meant to be abstract, explicitly `return` or `return None`. + +class NoneCompatible3(Protocol): + @abstractmethod + def f(self) -> None: ... + @overload + def g(self, x: int) -> int: ... + @overload + def g(self, x: str) -> None: ... + def h(self, x): ... + +class C(NoneCompatible3): ... +C() # E: Cannot instantiate abstract class "C" with abstract attributes "f", "g" and "h" +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyWithFinal] +from typing import Protocol, final + +class P(Protocol): + @final # E: Protocol member cannot be final + def f(self, x: int) -> str: ... + +class A(P): ... +A() # E: Cannot instantiate abstract class "A" with abstract attribute "f" + +[case testProtocolWithNestedClass] +from typing import TypeVar, Protocol + +class Template(Protocol): + var: int + class Meta: ... + +class B: + var: int + class Meta: ... +class C: + var: int + class Meta(Template.Meta): ... + +def foo(t: Template) -> None: ... +foo(B()) # E: Argument 1 to "foo" has incompatible type "B"; expected "Template" \ + # N: Following member(s) of "B" have conflicts: \ + # N: Meta: expected "Type[__main__.Template.Meta]", got "Type[__main__.B.Meta]" +foo(C()) # OK + +[case testProtocolClassObjectAttribute] +from typing import ClassVar, Protocol + +class P(Protocol): + foo: int + +class A: + foo = 42 +class B: + foo: ClassVar[int] +class C: + foo: ClassVar[str] +class D: + foo: int + +def test(arg: P) -> None: ... +test(A) # OK +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: foo: expected "int", got "str" +test(D) # E: Argument 1 to "test" has incompatible type "Type[D]"; expected "P" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "D" + +[case testProtocolClassObjectClassVarRejected] +from typing import ClassVar, Protocol + +class P(Protocol): + foo: ClassVar[int] + +class B: + foo: ClassVar[int] + +def test(arg: P) -> None: ... +test(B) # E: Argument 1 to "test" has incompatible type "Type[B]"; expected "P" \ + # N: ClassVar protocol member P.foo can never be matched by a class object + +[case testProtocolClassObjectPropertyRejected] +from typing import ClassVar, Protocol + +class P(Protocol): + @property + def foo(self) -> int: ... + +class B: + @property + def foo(self) -> int: ... +class C: + foo: int +class D: + foo: ClassVar[int] + +def test(arg: P) -> None: ... +# TODO: skip type mismatch diagnostics in this case. +test(B) # E: Argument 1 to "test" has incompatible type "Type[B]"; expected "P" \ + # N: Following member(s) of "B" have conflicts: \ + # N: foo: expected "int", got "Callable[[B], int]" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "B" +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "C" +test(D) # OK +[builtins fixtures/property.pyi] + +[case testProtocolClassObjectInstanceMethod] +from typing import Any, Protocol + +class P(Protocol): + def foo(self, obj: Any) -> int: ... + +class B: + def foo(self) -> int: ... +class C: + def foo(self) -> str: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(obj: Any) -> int \ + # N: Got: \ + # N: def foo(self: C) -> str + +[case testProtocolClassObjectInstanceMethodArg] +from typing import Any, Protocol + +class P(Protocol): + def foo(self, obj: B) -> int: ... + +class B: + def foo(self) -> int: ... +class C: + def foo(self) -> int: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(obj: B) -> int \ + # N: Got: \ + # N: def foo(self: C) -> int + +[case testProtocolClassObjectInstanceMethodOverloaded] +from typing import Any, Protocol, overload + +class P(Protocol): + @overload + def foo(self, obj: Any, arg: int) -> int: ... + @overload + def foo(self, obj: Any, arg: str) -> str: ... + +class B: + @overload + def foo(self, arg: int) -> int: ... + @overload + def foo(self, arg: str) -> str: ... + def foo(self, arg: Any) -> Any: + ... + +class C: + @overload + def foo(self, arg: int) -> int: ... + @overload + def foo(self, arg: str) -> int: ... + def foo(self, arg: Any) -> Any: + ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: @overload \ + # N: def foo(obj: Any, arg: int) -> int \ + # N: @overload \ + # N: def foo(obj: Any, arg: str) -> str \ + # N: Got: \ + # N: @overload \ + # N: def foo(self: C, arg: int) -> int \ + # N: @overload \ + # N: def foo(self: C, arg: str) -> int + +[case testProtocolClassObjectClassMethod] +from typing import Protocol + +class P(Protocol): + def foo(self) -> int: ... + +class B: + @classmethod + def foo(cls) -> int: ... +class C: + @classmethod + def foo(cls) -> str: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> int \ + # N: Got: \ + # N: def foo() -> str +[builtins fixtures/classmethod.pyi] + +[case testProtocolClassObjectStaticMethod] +from typing import Protocol + +class P(Protocol): + def foo(self) -> int: ... + +class B: + @staticmethod + def foo() -> int: ... +class C: + @staticmethod + def foo() -> str: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> int \ + # N: Got: \ + # N: def foo() -> str +[builtins fixtures/staticmethod.pyi] + +[case testProtocolClassObjectGenericInstanceMethod] +from typing import Any, Protocol, Generic, List, TypeVar + +class P(Protocol): + def foo(self, obj: Any) -> List[int]: ... + +T = TypeVar("T") +class A(Generic[T]): + def foo(self) -> T: ... +class AA(A[List[T]]): ... + +class B(AA[int]): ... +class C(AA[str]): ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(obj: Any) -> List[int] \ + # N: Got: \ + # N: def foo(self: A[List[str]]) -> List[str] +[builtins fixtures/list.pyi] + +[case testProtocolClassObjectGenericClassMethod] +from typing import Any, Protocol, Generic, List, TypeVar + +class P(Protocol): + def foo(self) -> List[int]: ... + +T = TypeVar("T") +class A(Generic[T]): + @classmethod + def foo(self) -> T: ... +class AA(A[List[T]]): ... + +class B(AA[int]): ... +class C(AA[str]): ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> List[int] \ + # N: Got: \ + # N: def foo() -> List[str] +[builtins fixtures/isinstancelist.pyi] + +[case testProtocolClassObjectSelfTypeInstanceMethod] +from typing import Protocol, TypeVar, Union + +T = TypeVar("T") +class P(Protocol): + def foo(self, arg: T) -> T: ... + +class B: + def foo(self: T) -> T: ... +class C: + def foo(self: T) -> Union[T, int]: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def [T] foo(arg: T) -> T \ + # N: Got: \ + # N: def [T] foo(self: T) -> Union[T, int] + +[case testProtocolClassObjectSelfTypeClassMethod] +from typing import Protocol, Type, TypeVar + +T = TypeVar("T") +class P(Protocol): + def foo(self) -> B: ... + +class B: + @classmethod + def foo(cls: Type[T]) -> T: ... +class C: + @classmethod + def foo(cls: Type[T]) -> T: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> B \ + # N: Got: \ + # N: def foo() -> C +[builtins fixtures/classmethod.pyi] + +[case testProtocolClassObjectAttributeAndCall] +from typing import Any, ClassVar, Protocol + +class P(Protocol): + foo: int + def __call__(self, x: int, y: int) -> Any: ... + +class B: + foo: ClassVar[int] + def __init__(self, x: int, y: int) -> None: ... +class C: + foo: ClassVar[int] + def __init__(self, x: int, y: str) -> None: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: "C" has constructor incompatible with "__call__" of "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def __call__(x: int, y: int) -> Any \ + # N: Got: \ + # N: def __init__(x: int, y: str) -> C \ + # N: "P.__call__" has type "Callable[[Arg(int, 'x'), Arg(int, 'y')], Any]" + +[case testProtocolClassObjectPureCallback] +from typing import Any, ClassVar, Protocol + +class P(Protocol): + def __call__(self, x: int, y: int) -> Any: ... + +class B: + def __init__(self, x: int, y: int) -> None: ... +class C: + def __init__(self, x: int, y: str) -> None: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: "C" has constructor incompatible with "__call__" of "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def __call__(x: int, y: int) -> Any \ + # N: Got: \ + # N: def __init__(x: int, y: str) -> C \ + # N: "P.__call__" has type "Callable[[Arg(int, 'x'), Arg(int, 'y')], Any]" +[builtins fixtures/type.pyi] + +[case testProtocolClassObjectCallableError] +from typing import Protocol, Any, Callable + +class P(Protocol): + def __call__(self, app: int) -> Callable[[str], None]: + ... + +class C: + def __init__(self, app: str) -> None: + pass + + def __call__(self, el: str) -> None: + return None + +p: P = C # E: Incompatible types in assignment (expression has type "Type[C]", variable has type "P") \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def __call__(app: int) -> Callable[[str], None] \ + # N: Got: \ + # N: def __init__(app: str) -> C \ + # N: "P.__call__" has type "Callable[[Arg(int, 'app')], Callable[[str], None]]" + +[builtins fixtures/type.pyi] + +[case testProtocolTypeTypeAttribute] +from typing import ClassVar, Protocol, Type + +class P(Protocol): + foo: int + +class A: + foo = 42 +class B: + foo: ClassVar[int] +class C: + foo: ClassVar[str] +class D: + foo: int + +def test(arg: P) -> None: ... +a: Type[A] +b: Type[B] +c: Type[C] +d: Type[D] +test(a) # OK +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: foo: expected "int", got "str" +test(d) # E: Argument 1 to "test" has incompatible type "Type[D]"; expected "P" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "D" + +[case testProtocolTypeTypeInstanceMethod] +from typing import Any, Protocol, Type + +class P(Protocol): + def foo(self, cls: Any) -> int: ... + +class B: + def foo(self) -> int: ... +class C: + def foo(self) -> str: ... + +def test(arg: P) -> None: ... +b: Type[B] +c: Type[C] +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(cls: Any) -> int \ + # N: Got: \ + # N: def foo(self: C) -> str + +[case testProtocolTypeTypeClassMethod] +from typing import Protocol, Type + +class P(Protocol): + def foo(self) -> int: ... + +class B: + @classmethod + def foo(cls) -> int: ... +class C: + @classmethod + def foo(cls) -> str: ... + +def test(arg: P) -> None: ... +b: Type[B] +c: Type[C] +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> int \ + # N: Got: \ + # N: def foo() -> str +[builtins fixtures/classmethod.pyi] + +[case testProtocolTypeTypeSelfTypeInstanceMethod] +from typing import Protocol, Type, TypeVar, Union + +T = TypeVar("T") +class P(Protocol): + def foo(self, arg: T) -> T: ... + +class B: + def foo(self: T) -> T: ... +class C: + def foo(self: T) -> Union[T, int]: ... + +def test(arg: P) -> None: ... +b: Type[B] +c: Type[C] +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def [T] foo(arg: T) -> T \ + # N: Got: \ + # N: def [T] foo(self: T) -> Union[T, int] + +[case testProtocolClassObjectInference] +from typing import Any, Protocol, TypeVar + +T = TypeVar("T", contravariant=True) +class P(Protocol[T]): + def foo(self, obj: T) -> int: ... + +class B: + def foo(self) -> int: ... + +S = TypeVar("S") +def test(arg: P[S]) -> S: ... +reveal_type(test(B)) # N: Revealed type is "__main__.B" + +[case testProtocolTypeTypeInference] +from typing import Any, Protocol, TypeVar, Type + +T = TypeVar("T", contravariant=True) +class P(Protocol[T]): + def foo(self, obj: T) -> int: ... + +class B: + def foo(self) -> int: ... + +S = TypeVar("S") +def test(arg: P[S]) -> S: ... +b: Type[B] +reveal_type(test(b)) # N: Revealed type is "__main__.B" + +[case testTypeAliasInProtocolBody] +from typing import Protocol, List + +class P(Protocol): + x = List[str] # E: Type aliases are prohibited in protocol bodies \ + # N: Use variable annotation syntax to define protocol members + +class C: + x: int +def foo(x: P) -> None: ... +foo(C()) # No extra error here +[builtins fixtures/list.pyi] + +[case testTypeVarInProtocolBody] +from typing import Protocol, TypeVar + +class C(Protocol): + T = TypeVar('T') + def __call__(self, t: T) -> T: ... + +def f_bad(t: int) -> int: + return t + +S = TypeVar("S") +def f_good(t: S) -> S: + return t + +g: C = f_bad # E: Incompatible types in assignment (expression has type "Callable[[int], int]", variable has type "C") \ + # N: "C.__call__" has type "Callable[[Arg(T, 't')], T]" +g = f_good # OK + +[case testModuleAsProtocolImplementation] +import default_config +import bad_config_1 +import bad_config_2 +import bad_config_3 +from typing import Protocol + +class Options(Protocol): + timeout: int + one_flag: bool + other_flag: bool + def update(self) -> bool: ... + +def setup(options: Options) -> None: ... +setup(default_config) # OK +setup(bad_config_1) # E: Argument 1 to "setup" has incompatible type Module; expected "Options" \ + # N: "ModuleType" is missing following "Options" protocol member: \ + # N: timeout +setup(bad_config_2) # E: Argument 1 to "setup" has incompatible type Module; expected "Options" \ + # N: Following member(s) of Module "bad_config_2" have conflicts: \ + # N: one_flag: expected "bool", got "int" +setup(bad_config_3) # E: Argument 1 to "setup" has incompatible type Module; expected "Options" \ + # N: Following member(s) of Module "bad_config_3" have conflicts: \ + # N: Expected: \ + # N: def update() -> bool \ + # N: Got: \ + # N: def update(obj: Any) -> bool + +[file default_config.py] +timeout = 100 +one_flag = True +other_flag = False +def update() -> bool: ... + +[file bad_config_1.py] +one_flag = True +other_flag = False +def update() -> bool: ... + +[file bad_config_2.py] +timeout = 100 +one_flag = 42 +other_flag = False +def update() -> bool: ... + +[file bad_config_3.py] +timeout = 100 +one_flag = True +other_flag = False +def update(obj) -> bool: ... +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationInference] +import default_config +from typing import Protocol, TypeVar + +T = TypeVar("T", covariant=True) +class Options(Protocol[T]): + timeout: int + one_flag: bool + other_flag: bool + def update(self) -> T: ... + +def setup(options: Options[T]) -> T: ... +reveal_type(setup(default_config)) # N: Revealed type is "builtins.str" + +[file default_config.py] +timeout = 100 +one_flag = True +other_flag = False +def update() -> str: ... +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationClassObject] +import runner +import bad_runner +from typing import Callable, Protocol + +class Runner(Protocol): + @property + def Run(self) -> Callable[[int], Result]: ... + +class Result(Protocol): + value: int + +def run(x: Runner) -> None: ... +run(runner) # OK +run(bad_runner) # E: Argument 1 to "run" has incompatible type Module; expected "Runner" \ + # N: Following member(s) of Module "bad_runner" have conflicts: \ + # N: Expected: \ + # N: def (int, /) -> Result \ + # N: Got: \ + # N: def __init__(arg: str) -> Run + +[file runner.py] +class Run: + value: int + def __init__(self, arg: int) -> None: ... + +[file bad_runner.py] +class Run: + value: int + def __init__(self, arg: str) -> None: ... +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationTypeAlias] +import runner +import bad_runner +from typing import Callable, Protocol + +class Runner(Protocol): + @property + def run(self) -> Callable[[int], Result]: ... + +class Result(Protocol): + value: int + +def run(x: Runner) -> None: ... +run(runner) # OK +run(bad_runner) # E: Argument 1 to "run" has incompatible type Module; expected "Runner" \ + # N: Following member(s) of Module "bad_runner" have conflicts: \ + # N: Expected: \ + # N: def (int, /) -> Result \ + # N: Got: \ + # N: def __init__(arg: str) -> Run + +[file runner.py] +class Run: + value: int + def __init__(self, arg: int) -> None: ... +run = Run + +[file bad_runner.py] +class Run: + value: int + def __init__(self, arg: str) -> None: ... +run = Run +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationClassVar] +from typing import ClassVar, Protocol +import mod + +class My(Protocol): + x: ClassVar[int] + +def test(mod: My) -> None: ... +test(mod=mod) # E: Argument "mod" to "test" has incompatible type Module; expected "My" \ + # N: Protocol member My.x expected class variable, got instance variable +[file mod.py] +x: int +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationFinal] +from typing import Protocol +import some_module + +class My(Protocol): + a: int + +def func(arg: My) -> None: ... +func(some_module) # E: Argument 1 to "func" has incompatible type Module; expected "My" \ + # N: Protocol member My.a expected settable variable, got read-only attribute + +[file some_module.py] +from typing_extensions import Final + +a: Final = 1 +[builtins fixtures/module.pyi] + + +[case testModuleAsProtocolRedefinitionTopLevel] +from typing import Protocol + +class P(Protocol): + def f(self) -> str: ... + +cond: bool +t: P +if cond: + import mod1 as t +else: + import mod2 as t + +import badmod as t # E: Incompatible import of "t" (imported name has type Module, local name has type "P") + +[file mod1.py] +def f() -> str: ... + +[file mod2.py] +def f() -> str: ... + +[file badmod.py] +def nothing() -> int: ... +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolRedefinitionImportFrom] +from typing import Protocol + +class P(Protocol): + def f(self) -> str: ... + +cond: bool +t: P +if cond: + from package import mod1 as t +else: + from package import mod2 as t + +from package import badmod as t # E: Incompatible import of "t" (imported name has type Module, local name has type "P") + +package: int = 10 + +import package.mod1 as t +import package.mod1 # E: Incompatible import of "package" (imported name has type Module, local name has type "int") + +[file package/mod1.py] +def f() -> str: ... + +[file package/mod2.py] +def f() -> str: ... + +[file package/badmod.py] +def nothing() -> int: ... +[builtins fixtures/module.pyi] + +[case testProtocolSelfTypeNewSyntax] +from typing import Protocol, Self + +class P(Protocol): + @property + def next(self) -> Self: ... + +class C: + next: C +class S: + next: Self + +x: P = C() +y: P = S() + +z: P +reveal_type(S().next) # N: Revealed type is "__main__.S" +reveal_type(z.next) # N: Revealed type is "__main__.P" +[builtins fixtures/property.pyi] + +[case testProtocolSelfTypeNewSyntaxSubProtocol] +from typing import Protocol, Self + +class P(Protocol): + @property + def next(self) -> Self: ... +class PS(P, Protocol): + @property + def other(self) -> Self: ... + +class C: + next: C + other: C +class S: + next: Self + other: Self + +x: PS = C() +y: PS = S() +[builtins fixtures/property.pyi] + +[case testProtocolClassVarSelfType] +from typing import ClassVar, Self, Protocol + +class P(Protocol): + DEFAULT: ClassVar[Self] +class C: + DEFAULT: ClassVar[C] + +x: P = C() + +[case testInferenceViaTypeTypeMetaclass] +from typing import Iterator, Iterable, TypeVar, Type + +M = TypeVar("M") + +class Meta(type): + def __iter__(self: Type[M]) -> Iterator[M]: ... +class Foo(metaclass=Meta): ... + +T = TypeVar("T") +def test(x: Iterable[T]) -> T: ... + +reveal_type(test(Foo)) # N: Revealed type is "__main__.Foo" +t_foo: Type[Foo] +reveal_type(test(t_foo)) # N: Revealed type is "__main__.Foo" + +TF = TypeVar("TF", bound=Foo) +def outer(cls: Type[TF]) -> TF: + reveal_type(test(cls)) # N: Revealed type is "TF`-1" + return cls() + +[case testProtocolImportNotMember] +import m +import lib + +class Bad: + x: int +class Good: + x: lib.C + +x: m.P = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "P") \ + # N: Following member(s) of "Bad" have conflicts: \ + # N: x: expected "C", got "int" +x = Good() + +[file m.py] +from typing import Protocol + +class P(Protocol): + import lib + x: lib.C + +[file lib.py] +class C: ... + +[case testAllowDefaultConstructorInProtocols] +from typing import Protocol + +class P(Protocol): + x: int + def __init__(self, x: int) -> None: + self.x = x + +class C(P): ... +C(0) # OK + +[case testTypeVarValueConstraintAgainstGenericProtocol] +from typing import TypeVar, Generic, Protocol, overload + +T_contra = TypeVar("T_contra", contravariant=True) +AnyStr = TypeVar("AnyStr", str, bytes) + +class SupportsWrite(Protocol[T_contra]): + def write(self, s: T_contra, /) -> None: ... + +class Buffer: ... + +class IO(Generic[AnyStr]): + @overload + def write(self: IO[bytes], s: Buffer, /) -> None: ... + @overload + def write(self, s: AnyStr, /) -> None: ... + def write(self, s): ... + +def foo(fdst: SupportsWrite[AnyStr]) -> None: ... + +x: IO[str] +foo(x) + +[case testTypeVarValueConstraintAgainstGenericProtocol2] +from typing import Generic, Protocol, TypeVar, overload + +AnyStr = TypeVar("AnyStr", str, bytes) +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) + +class SupportsRead(Generic[T_co]): + def read(self) -> T_co: ... + +class SupportsWrite(Protocol[T_contra]): + def write(self, s: T_contra) -> object: ... + +def copyfileobj(fsrc: SupportsRead[AnyStr], fdst: SupportsWrite[AnyStr]) -> None: ... + +class WriteToMe(Generic[AnyStr]): + @overload + def write(self: WriteToMe[str], s: str) -> int: ... + @overload + def write(self: WriteToMe[bytes], s: bytes) -> int: ... + def write(self, s): ... + +class WriteToMeOrReadFromMe(WriteToMe[AnyStr], SupportsRead[AnyStr]): ... + +copyfileobj(WriteToMeOrReadFromMe[bytes](), WriteToMe[bytes]()) + +[case testOverloadedMethodWithExplictSelfTypes] +from typing import Generic, overload, Protocol, TypeVar, Union + +AnyStr = TypeVar("AnyStr", str, bytes) +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) + +class SupportsRead(Protocol[T_co]): + def read(self) -> T_co: ... + +class SupportsWrite(Protocol[T_contra]): + def write(self, s: T_contra) -> int: ... + +class Input(Generic[AnyStr]): + def read(self) -> AnyStr: ... + +class Output(Generic[AnyStr]): + @overload + def write(self: Output[str], s: str) -> int: ... + @overload + def write(self: Output[bytes], s: bytes) -> int: ... + def write(self, s: Union[str, bytes]) -> int: ... + +def f(src: SupportsRead[AnyStr], dst: SupportsWrite[AnyStr]) -> None: ... + +def g1(a: Input[bytes], b: Output[bytes]) -> None: + f(a, b) + +def g2(a: Input[bytes], b: Output[bytes]) -> None: + f(a, b) + +def g3(a: Input[str], b: Output[bytes]) -> None: + f(a, b) # E: Cannot infer type argument 1 of "f" + +def g4(a: Input[bytes], b: Output[str]) -> None: + f(a, b) # E: Cannot infer type argument 1 of "f" + +[builtins fixtures/tuple.pyi] + +[case testOverloadProtocolSubtyping] +from typing import Protocol, Self, overload + +class NumpyFloat: + __add__: "FloatOP" + +class FloatOP(Protocol): + @overload + def __call__(self, other: float) -> NumpyFloat: ... + @overload + def __call__(self, other: NumpyFloat) -> NumpyFloat: ... + +class SupportsAdd(Protocol): + @overload + def __add__(self, other: float) -> Self: ... + @overload + def __add__(self, other: NumpyFloat) -> Self: ... + +x: SupportsAdd = NumpyFloat() +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test new file mode 100644 index 000000000000..0231b47cf4a0 --- /dev/null +++ b/test-data/unit/check-python310.test @@ -0,0 +1,2441 @@ +-- Capture Pattern -- + +[case testMatchCapturePatternType] +class A: ... +m: A + +match m: + case a: + reveal_type(a) # N: Revealed type is "__main__.A" + +-- Literal Pattern -- + +[case testMatchLiteralPatternNarrows] +m: object + +match m: + case 1: + reveal_type(m) # N: Revealed type is "Literal[1]" + +[case testMatchLiteralPatternAlreadyNarrower-skip] +m: bool + +match m: + case 1: + reveal_type(m) # This should probably be unreachable, but isn't detected as such. +[builtins fixtures/primitives.pyi] + +[case testMatchLiteralPatternUnreachable] +# primitives are needed because otherwise mypy doesn't see that int and str are incompatible +m: int + +match m: + case "str": + reveal_type(m) +[builtins fixtures/primitives.pyi] + +-- Value Pattern -- + +[case testMatchValuePatternNarrows] +import b +m: object + +match m: + case b.b: + reveal_type(m) # N: Revealed type is "builtins.int" +[file b.py] +b: int + +[case testMatchValuePatternAlreadyNarrower] +import b +m: bool + +match m: + case b.b: + reveal_type(m) # N: Revealed type is "builtins.bool" +[file b.py] +b: int + +[case testMatchValuePatternIntersect] +import b + +class A: ... +m: A + +match m: + case b.b: + reveal_type(m) # N: Revealed type is "__main__.1" +[file b.py] +class B: ... +b: B + +[case testMatchValuePatternUnreachable] +# primitives are needed because otherwise mypy doesn't see that int and str are incompatible +import b + +m: int + +match m: + case b.b: + reveal_type(m) +[file b.py] +b: str +[builtins fixtures/primitives.pyi] + +-- Sequence Pattern -- + +[case testMatchSequencePatternCaptures] +from typing import List +m: List[int] + +match m: + case [a]: + reveal_type(a) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternCapturesStarred] +from typing import Sequence +m: Sequence[int] + +match m: + case [a, *b]: + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternNarrowsInner] +from typing import Sequence +m: Sequence[object] + +match m: + case [1, True]: + reveal_type(m) # N: Revealed type is "typing.Sequence[builtins.int]" + +[case testMatchSequencePatternNarrowsOuter] +from typing import Sequence +m: object + +match m: + case [1, True]: + reveal_type(m) # N: Revealed type is "typing.Sequence[builtins.int]" + +[case testMatchSequencePatternAlreadyNarrowerInner] +from typing import Sequence +m: Sequence[bool] + +match m: + case [1, True]: + reveal_type(m) # N: Revealed type is "typing.Sequence[builtins.bool]" + +[case testMatchSequencePatternAlreadyNarrowerOuter] +from typing import Sequence +m: Sequence[object] + +match m: + case [1, True]: + reveal_type(m) # N: Revealed type is "typing.Sequence[builtins.int]" + +[case testMatchSequencePatternAlreadyNarrowerBoth] +from typing import Sequence +m: Sequence[bool] + +match m: + case [1, True]: + reveal_type(m) # N: Revealed type is "typing.Sequence[builtins.bool]" + +[case testMatchNestedSequencePatternNarrowsInner] +from typing import Sequence +m: Sequence[Sequence[object]] + +match m: + case [[1], [True]]: + reveal_type(m) # N: Revealed type is "typing.Sequence[typing.Sequence[builtins.int]]" + +[case testMatchNestedSequencePatternNarrowsOuter] +from typing import Sequence +m: object + +match m: + case [[1], [True]]: + reveal_type(m) # N: Revealed type is "typing.Sequence[typing.Sequence[builtins.int]]" + +[case testMatchSequencePatternDoesntNarrowInvariant] +from typing import List +m: List[object] + +match m: + case [1]: + reveal_type(m) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternMatches] +import array, collections +from typing import Sequence, Iterable + +m1: object +m2: Sequence[int] +m3: array.array[int] +m4: collections.deque[int] +m5: list[int] +m6: memoryview +m7: range +m8: tuple[int] + +m9: str +m10: bytes +m11: bytearray + +match m1: + case [a]: + reveal_type(a) # N: Revealed type is "builtins.object" + +match m2: + case [b]: + reveal_type(b) # N: Revealed type is "builtins.int" + +match m3: + case [c]: + reveal_type(c) # N: Revealed type is "builtins.int" + +match m4: + case [d]: + reveal_type(d) # N: Revealed type is "builtins.int" + +match m5: + case [e]: + reveal_type(e) # N: Revealed type is "builtins.int" + +match m6: + case [f]: + reveal_type(f) # N: Revealed type is "builtins.int" + +match m7: + case [g]: + reveal_type(g) # N: Revealed type is "builtins.int" + +match m8: + case [h]: + reveal_type(h) # N: Revealed type is "builtins.int" + +match m9: + case [i]: + reveal_type(i) + +match m10: + case [j]: + reveal_type(j) + +match m11: + case [k]: + reveal_type(k) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testMatchSequencePatternCapturesTuple] +from typing import Tuple +m: Tuple[int, str, bool] + +match m: + case [a, b, c]: + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: Revealed type is "builtins.str" + reveal_type(c) # N: Revealed type is "builtins.bool" + reveal_type(m) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.bool]" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternTupleTooLong] +from typing import Tuple +m: Tuple[int, str] + +match m: + case [a, b, c]: + reveal_type(a) + reveal_type(b) + reveal_type(c) +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternTupleTooShort] +from typing import Tuple +m: Tuple[int, str, bool] + +match m: + case [a, b]: + reveal_type(a) + reveal_type(b) +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternTupleNarrows] +from typing import Tuple +m: Tuple[object, object] + +match m: + case [1, "str"]: + reveal_type(m) # N: Revealed type is "Tuple[Literal[1], Literal['str']]" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternTupleStarred] +from typing import Tuple +m: Tuple[int, str, bool] + +match m: + case [a, *b, c]: + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(c) # N: Revealed type is "builtins.bool" + reveal_type(m) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.bool]" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternTupleStarredUnion] +from typing import Tuple +m: Tuple[int, str, float, bool] + +match m: + case [a, *b, c]: + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: Revealed type is "builtins.list[Union[builtins.str, builtins.float]]" + reveal_type(c) # N: Revealed type is "builtins.bool" + reveal_type(m) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.float, builtins.bool]" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternTupleStarredTooShort] +from typing import Tuple +m: Tuple[int] +reveal_type(m) # N: Revealed type is "Tuple[builtins.int]" + +match m: + case [a, *b, c]: + reveal_type(a) + reveal_type(b) + reveal_type(c) +[builtins fixtures/list.pyi] + +[case testMatchNonMatchingSequencePattern] +from typing import List + +x: List[int] +match x: + case [str()]: + pass + +[case testMatchSequencePatternWithInvalidClassPattern] +class Example: + __match_args__ = ("value",) + def __init__(self, value: str) -> None: + self.value = value + +SubClass: type[Example] + +match [SubClass("a"), SubClass("b")]: + case [SubClass(value), *rest]: # E: Expected type in class pattern; found "Type[__main__.Example]" + reveal_type(value) # E: Cannot determine type of "value" \ + # N: Revealed type is "Any" + reveal_type(rest) # N: Revealed type is "builtins.list[__main__.Example]" +[builtins fixtures/tuple.pyi] + +[case testMatchSequenceUnion-skip] +from typing import List, Union +m: Union[List[List[str]], str] + +match m: + case [list(['str'])]: + reveal_type(m) # N: Revealed type is "builtins.list[builtins.list[builtins.str]]" +[builtins fixtures/list.pyi] + +[case testMatchSequencePatternNarrowSubjectItems] +m: int +n: str +o: bool + +match m, n, o: + case [3, "foo", True]: + reveal_type(m) # N: Revealed type is "Literal[3]" + reveal_type(n) # N: Revealed type is "Literal['foo']" + reveal_type(o) # N: Revealed type is "Literal[True]" + case [a, b, c]: + reveal_type(m) # N: Revealed type is "builtins.int" + reveal_type(n) # N: Revealed type is "builtins.str" + reveal_type(o) # N: Revealed type is "builtins.bool" + +reveal_type(m) # N: Revealed type is "builtins.int" +reveal_type(n) # N: Revealed type is "builtins.str" +reveal_type(o) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testMatchSequencePatternNarrowSubjectItemsRecursive] +m: int +n: int +o: int +p: int +q: int +r: int + +match m, (n, o), (p, (q, r)): + case [0, [1, 2], [3, [4, 5]]]: + reveal_type(m) # N: Revealed type is "Literal[0]" + reveal_type(n) # N: Revealed type is "Literal[1]" + reveal_type(o) # N: Revealed type is "Literal[2]" + reveal_type(p) # N: Revealed type is "Literal[3]" + reveal_type(q) # N: Revealed type is "Literal[4]" + reveal_type(r) # N: Revealed type is "Literal[5]" +[builtins fixtures/tuple.pyi] + +[case testMatchSequencePatternSequencesLengthMismatchNoNarrowing] +m: int +n: str +o: bool + +match m, n, o: + case [3, "foo"]: + pass + case [3, "foo", True, True]: + pass +[builtins fixtures/tuple.pyi] + +[case testMatchSequencePatternSequencesLengthMismatchNoNarrowingRecursive] +m: int +n: int +o: int + +match m, (n, o): + case [0]: + pass + case [0, 1, [2]]: + pass + case [0, [1]]: + pass + case [0, [1, 2, 3]]: + pass +[builtins fixtures/tuple.pyi] + +-- Mapping Pattern -- + +[case testMatchMappingPatternCaptures] +from typing import Dict +import b +m: Dict[str, int] + +match m: + case {"key": v}: + reveal_type(v) # N: Revealed type is "builtins.int" + case {b.b: v2}: + reveal_type(v2) # N: Revealed type is "builtins.int" +[file b.py] +b: str +[builtins fixtures/dict.pyi] + +[case testMatchMappingPatternCapturesWrongKeyType] +# This is not actually unreachable, as a subclass of dict could accept keys with different types +from typing import Dict +import b +m: Dict[str, int] + +match m: + case {1: v}: + reveal_type(v) # N: Revealed type is "builtins.int" + case {b.b: v2}: + reveal_type(v2) # N: Revealed type is "builtins.int" +[file b.py] +b: int +[builtins fixtures/dict.pyi] + +[case testMatchMappingPatternCapturesTypedDict] +from typing import TypedDict + +class A(TypedDict): + a: str + b: int + +m: A + +match m: + case {"a": v}: + reveal_type(v) # N: Revealed type is "builtins.str" + case {"b": v2}: + reveal_type(v2) # N: Revealed type is "builtins.int" + case {"a": v3, "b": v4}: + reveal_type(v3) # N: Revealed type is "builtins.str" + reveal_type(v4) # N: Revealed type is "builtins.int" + case {"o": v5}: + reveal_type(v5) # N: Revealed type is "builtins.object" +[typing fixtures/typing-typeddict.pyi] + +[case testMatchMappingPatternCapturesTypedDictWithLiteral] +from typing import TypedDict +import b + +class A(TypedDict): + a: str + b: int + +m: A + +match m: + case {b.a: v}: + reveal_type(v) # N: Revealed type is "builtins.str" + case {b.b: v2}: + reveal_type(v2) # N: Revealed type is "builtins.int" + case {b.a: v3, b.b: v4}: + reveal_type(v3) # N: Revealed type is "builtins.str" + reveal_type(v4) # N: Revealed type is "builtins.int" + case {b.o: v5}: + reveal_type(v5) # N: Revealed type is "builtins.object" +[file b.py] +from typing import Final, Literal +a: Final = "a" +b: Literal["b"] = "b" +o: Final[str] = "o" +[typing fixtures/typing-typeddict.pyi] + +[case testMatchMappingPatternCapturesTypedDictWithNonLiteral] +from typing import TypedDict +import b + +class A(TypedDict): + a: str + b: int + +m: A + +match m: + case {b.a: v}: + reveal_type(v) # N: Revealed type is "builtins.object" +[file b.py] +from typing import Final, Literal +a: str +[typing fixtures/typing-typeddict.pyi] + +[case testMatchMappingPatternCapturesTypedDictUnreachable] +# TypedDict keys are always str, so this is actually unreachable +from typing import TypedDict +import b + +class A(TypedDict): + a: str + b: int + +m: A + +match m: + case {1: v}: + reveal_type(v) + case {b.b: v2}: + reveal_type(v2) +[file b.py] +b: int +[typing fixtures/typing-typeddict.pyi] + +[case testMatchMappingPatternCaptureRest] +m: object + +match m: + case {'k': 1, **r}: + reveal_type(r) # N: Revealed type is "builtins.dict[builtins.object, builtins.object]" +[builtins fixtures/dict.pyi] + +[case testMatchMappingPatternCaptureRestFromMapping] +from typing import Mapping + +m: Mapping[str, int] + +match m: + case {'k': 1, **r}: + reveal_type(r) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +[builtins fixtures/dict.pyi] + +-- Mapping patterns currently do not narrow -- + +-- Class Pattern -- + +[case testMatchClassPatternCapturePositional] +from typing import Final + +class A: + __match_args__: Final = ("a", "b") + a: str + b: int + +m: A + +match m: + case A(i, j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternMemberClassCapturePositional] +import b + +m: b.A + +match m: + case b.A(i, j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" +[file b.py] +from typing import Final + +class A: + __match_args__: Final = ("a", "b") + a: str + b: int +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternCaptureKeyword] +class A: + a: str + b: int + +m: A + +match m: + case A(a=i, b=j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" + +[case testMatchClassPatternCaptureSelf] +m: object + +match m: + case bool(a): + reveal_type(a) # N: Revealed type is "builtins.bool" + case bytearray(b): + reveal_type(b) # N: Revealed type is "builtins.bytearray" + case bytes(c): + reveal_type(c) # N: Revealed type is "builtins.bytes" + case dict(d): + reveal_type(d) # N: Revealed type is "builtins.dict[Any, Any]" + case float(e): + reveal_type(e) # N: Revealed type is "builtins.float" + case frozenset(f): + reveal_type(f) # N: Revealed type is "builtins.frozenset[Any]" + case int(g): + reveal_type(g) # N: Revealed type is "builtins.int" + case list(h): + reveal_type(h) # N: Revealed type is "builtins.list[Any]" + case set(i): + reveal_type(i) # N: Revealed type is "builtins.set[Any]" + case str(j): + reveal_type(j) # N: Revealed type is "builtins.str" + case tuple(k): + reveal_type(k) # N: Revealed type is "builtins.tuple[Any, ...]" +[builtins fixtures/primitives.pyi] + +[case testMatchClassPatternNarrowSelfCapture] +m: object + +match m: + case bool(): + reveal_type(m) # N: Revealed type is "builtins.bool" + case bytearray(): + reveal_type(m) # N: Revealed type is "builtins.bytearray" + case bytes(): + reveal_type(m) # N: Revealed type is "builtins.bytes" + case dict(): + reveal_type(m) # N: Revealed type is "builtins.dict[Any, Any]" + case float(): + reveal_type(m) # N: Revealed type is "builtins.float" + case frozenset(): + reveal_type(m) # N: Revealed type is "builtins.frozenset[Any]" + case int(): + reveal_type(m) # N: Revealed type is "builtins.int" + case list(): + reveal_type(m) # N: Revealed type is "builtins.list[Any]" + case set(): + reveal_type(m) # N: Revealed type is "builtins.set[Any]" + case str(): + reveal_type(m) # N: Revealed type is "builtins.str" + case tuple(): + reveal_type(m) # N: Revealed type is "builtins.tuple[Any, ...]" +[builtins fixtures/primitives.pyi] + +[case testMatchClassPatternCaptureSelfSubtype] +class A(str): + pass + +class B(str): + __match_args__ = ("b",) + b: int + +def f1(x: A): + match x: + case A(a): + reveal_type(a) # N: Revealed type is "__main__.A" + +def f2(x: B): + match x: + case B(b): + reveal_type(b) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchInvalidClassPattern] +m: object + +match m: + case xyz(y): # E: Name "xyz" is not defined + reveal_type(m) # N: Revealed type is "Any" + reveal_type(y) # E: Cannot determine type of "y" \ + # N: Revealed type is "Any" + +match m: + case xyz(z=x): # E: Name "xyz" is not defined + reveal_type(x) # E: Cannot determine type of "x" \ + # N: Revealed type is "Any" + +[case testMatchClassPatternCaptureDataclass] +from dataclasses import dataclass + +@dataclass +class A: + a: str + b: int + +m: A + +match m: + case A(i, j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" +[builtins fixtures/dataclasses.pyi] + +[case testMatchClassPatternCaptureDataclassNoMatchArgs] +from dataclasses import dataclass + +@dataclass(match_args=False) +class A: + a: str + b: int + +m: A + +match m: + case A(i, j): # E: Class "__main__.A" doesn't define "__match_args__" + pass +[builtins fixtures/dataclasses.pyi] + +[case testMatchClassPatternCaptureDataclassPartialMatchArgs] +from dataclasses import dataclass, field + +@dataclass +class A: + a: str + b: int = field(init=False) + +m: A + +match m: + case A(i, j): # E: Too many positional patterns for class pattern + pass + case A(k): + reveal_type(k) # N: Revealed type is "builtins.str" +[builtins fixtures/dataclasses.pyi] + +[case testMatchClassPatternCaptureNamedTupleInline] +from collections import namedtuple + +A = namedtuple("A", ["a", "b"]) + +m: A + +match m: + case A(i, j): + reveal_type(i) # N: Revealed type is "Any" + reveal_type(j) # N: Revealed type is "Any" +[builtins fixtures/list.pyi] + +[case testMatchClassPatternCaptureNamedTupleInlineTyped] +from typing import NamedTuple + +A = NamedTuple("A", [("a", str), ("b", int)]) + +m: A + +match m: + case A(i, j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testMatchClassPatternCaptureNamedTupleClass] +from typing import NamedTuple + +class A(NamedTuple): + a: str + b: int + +m: A + +match m: + case A(i, j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternCaptureGeneric] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class A(Generic[T]): + a: T + +m: object + +match m: + case A(a=i): + reveal_type(m) # N: Revealed type is "__main__.A[Any]" + reveal_type(i) # N: Revealed type is "Any" + +[case testMatchClassPatternCaptureVariadicGeneric] +from typing import Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple('Ts') +class A(Generic[Unpack[Ts]]): + a: Tuple[Unpack[Ts]] + +m: object +match m: + case A(a=i): + reveal_type(m) # N: Revealed type is "__main__.A[Unpack[builtins.tuple[Any, ...]]]" + reveal_type(i) # N: Revealed type is "builtins.tuple[Any, ...]" +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternCaptureGenericAlreadyKnown] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class A(Generic[T]): + a: T + +m: A[int] + +match m: + case A(a=i): + reveal_type(m) # N: Revealed type is "__main__.A[builtins.int]" + reveal_type(i) # N: Revealed type is "builtins.int" + +[case testMatchClassPatternCaptureFilledGenericTypeAlias] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class A(Generic[T]): + a: T + +B = A[int] + +m: object + +match m: + case B(a=i): # E: Class pattern class must not be a type alias with type parameters + reveal_type(i) + +[case testMatchClassPatternCaptureGenericTypeAlias] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class A(Generic[T]): + a: T + +B = A + +m: object + +match m: + case B(a=i): + pass + +[case testMatchClassPatternNarrows] +from typing import Final + +class A: + __match_args__: Final = ("a", "b") + a: str + b: int + +m: object + +match m: + case A(): + reveal_type(m) # N: Revealed type is "__main__.A" + case A(i, j): + reveal_type(m) # N: Revealed type is "__main__.A" +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternNarrowsUnion] +from typing import Final, Union + +class A: + __match_args__: Final = ("a", "b") + a: str + b: int + +class B: + __match_args__: Final = ("a", "b") + a: int + b: str + +m: Union[A, B] + +match m: + case A(): + reveal_type(m) # N: Revealed type is "__main__.A" + +match m: + case A(i, j): + reveal_type(m) # N: Revealed type is "__main__.A" + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" + +match m: + case B(): + reveal_type(m) # N: Revealed type is "__main__.B" + +match m: + case B(k, l): + reveal_type(m) # N: Revealed type is "__main__.B" + reveal_type(k) # N: Revealed type is "builtins.int" + reveal_type(l) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternAlreadyNarrower] +from typing import Final + +class A: + __match_args__: Final = ("a", "b") + a: str + b: int +class B(A): ... + +m: B + +match m: + case A(): + reveal_type(m) # N: Revealed type is "__main__.B" + +match m: + case A(i, j): + reveal_type(m) # N: Revealed type is "__main__.B" +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternIntersection] +from typing import Final + +class A: + __match_args__: Final = ("a", "b") + a: str + b: int +class B: ... + +m: B + +match m: + case A(): + reveal_type(m) # N: Revealed type is "__main__.2" + case A(i, j): + reveal_type(m) # N: Revealed type is "__main__.3" +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternNonexistentKeyword] +class A: ... + +m: object + +match m: + case A(a=j): # E: Class "__main__.A" has no attribute "a" + reveal_type(m) # N: Revealed type is "__main__.A" + reveal_type(j) # N: Revealed type is "Any" + +[case testMatchClassPatternDuplicateKeyword] +class A: + a: str + +m: object + +match m: + case A(a=i, a=j): # E: Duplicate keyword pattern "a" + pass + +[case testMatchClassPatternDuplicateImplicitKeyword] +from typing import Final + +class A: + __match_args__: Final = ("a",) + a: str + +m: object + +match m: + case A(i, a=j): # E: Keyword "a" already matches a positional pattern + pass +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternTooManyPositionals] +from typing import Final + +class A: + __match_args__: Final = ("a", "b") + a: str + b: int + +m: object + +match m: + case A(i, j, k): # E: Too many positional patterns for class pattern + pass +[builtins fixtures/tuple.pyi] + +[case testMatchClassPatternIsNotType] +a = 1 +m: object + +match m: + case a(i, j): # E: Expected type in class pattern; found "builtins.int" + reveal_type(i) + reveal_type(j) + +[case testMatchClassPatternAny] +from typing import Any + +Foo: Any +m: object + +match m: + case Foo(): + pass + +[case testMatchClassPatternNestedGenerics] +# From cpython test_patma.py +x = [[{0: 0}]] +match x: + case list([({-0-0j: int(real=0+0j, imag=0-0j) | (1) as z},)]): + y = 0 + +reveal_type(x) # N: Revealed type is "builtins.list[builtins.list[builtins.dict[builtins.int, builtins.int]]]" +reveal_type(y) # N: Revealed type is "builtins.int" +reveal_type(z) # N: Revealed type is "builtins.int" +[builtins fixtures/dict-full.pyi] + +[case testMatchNonFinalMatchArgs] +class A: + __match_args__ = ("a", "b") + a: str + b: int + +m: object + +match m: + case A(i, j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchAnyTupleMatchArgs] +from typing import Tuple, Any + +class A: + __match_args__: Tuple[Any, ...] + a: str + b: int + +m: object + +match m: + case A(i, j, k): + reveal_type(i) # N: Revealed type is "Any" + reveal_type(j) # N: Revealed type is "Any" + reveal_type(k) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testMatchNonLiteralMatchArgs] +from typing import Final + +b: str = "b" +class A: + __match_args__: Final = ("a", b) # N: __match_args__ must be a tuple containing string literals for checking of match statements to work + a: str + b: int + +m: object + +match m: + case A(i, j, k): # E: Too many positional patterns for class pattern + pass + case A(i, j): + reveal_type(i) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testMatchExternalMatchArgs] +from typing import Final, Literal + +args: Final = ("a", "b") +class A: + __match_args__: Final = args + a: str + b: int + +arg: Final = "a" +arg2: Literal["b"] = "b" +class B: + __match_args__: Final = (arg, arg2) + a: str + b: int + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +-- As Pattern -- + +[case testMatchAsPattern] +m: int + +match m: + case x as l: + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(l) # N: Revealed type is "builtins.int" + +[case testMatchAsPatternNarrows] +m: object + +match m: + case int() as l: + reveal_type(l) # N: Revealed type is "builtins.int" + +[case testMatchAsPatternCapturesOr] +m: object + +match m: + case 1 | 2 as n: + reveal_type(n) # N: Revealed type is "Union[Literal[1], Literal[2]]" + +[case testMatchAsPatternAlreadyNarrower] +m: bool + +match m: + case int() as l: + reveal_type(l) # N: Revealed type is "builtins.bool" + +-- Or Pattern -- + +[case testMatchOrPatternNarrows] +m: object + +match m: + case 1 | 2: + reveal_type(m) # N: Revealed type is "Union[Literal[1], Literal[2]]" + +[case testMatchOrPatternNarrowsStr] +m: object + +match m: + case "foo" | "bar": + reveal_type(m) # N: Revealed type is "Union[Literal['foo'], Literal['bar']]" + +[case testMatchOrPatternNarrowsUnion] +m: object + +match m: + case 1 | "foo": + reveal_type(m) # N: Revealed type is "Union[Literal[1], Literal['foo']]" + +[case testMatchOrPatterCapturesMissing] +from typing import List +m: List[int] + +match m: + case [x, y] | list(x): # E: Alternative patterns bind different names + reveal_type(x) # N: Revealed type is "builtins.object" + reveal_type(y) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testMatchOrPatternCapturesJoin] +m: object + +match m: + case list(x) | dict(x): + reveal_type(x) # N: Revealed type is "typing.Iterable[Any]" +[builtins fixtures/dict.pyi] + +-- Interactions -- + +[case testMatchCapturePatternMultipleCases] +m: object + +match m: + case int(x): + reveal_type(x) # N: Revealed type is "builtins.int" + case str(x): + reveal_type(x) # N: Revealed type is "builtins.str" + +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testMatchCapturePatternMultipleCaptures] +from typing import Iterable + +m: Iterable[int] + +match m: + case [x, x]: # E: Multiple assignments to name "x" in pattern + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testMatchCapturePatternPreexistingSame] +a: int +m: int + +match m: + case a: + reveal_type(a) # N: Revealed type is "builtins.int" + +[case testMatchCapturePatternPreexistingNarrows] +a: int +m: bool + +match m: + case a: + reveal_type(a) # N: Revealed type is "builtins.bool" + +reveal_type(a) # N: Revealed type is "builtins.bool" +a = 3 +reveal_type(a) # N: Revealed type is "builtins.int" + +[case testMatchCapturePatternPreexistingIncompatible] +a: str +m: int + +match m: + case a: # E: Incompatible types in capture pattern (pattern captures type "int", variable has type "str") + reveal_type(a) # N: Revealed type is "builtins.str" + +reveal_type(a) # N: Revealed type is "builtins.str" + +[case testMatchCapturePatternPreexistingIncompatibleLater] +a: str +m: object + +match m: + case str(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case int(a): # E: Incompatible types in capture pattern (pattern captures type "int", variable has type "str") + reveal_type(a) # N: Revealed type is "builtins.str" + +reveal_type(a) # N: Revealed type is "builtins.str" + +[case testMatchCapturePatternFromFunctionReturningUnion] +def func1(arg: bool) -> str | int: ... +def func2(arg: bool) -> bytes | int: ... + +def main() -> None: + match func1(True): + case str(a): + match func2(True): + case c: + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(c) # N: Revealed type is "Union[builtins.bytes, builtins.int]" + reveal_type(a) # N: Revealed type is "builtins.str" + case a: + reveal_type(a) # N: Revealed type is "builtins.int" + +[case testMatchCapturePatternFromAsyncFunctionReturningUnion-xfail] +async def func1(arg: bool) -> str | int: ... +async def func2(arg: bool) -> bytes | int: ... + +async def main() -> None: + match await func1(True): + case str(a): + match await func2(True): + case c: + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(c) # N: Revealed type is "Union[builtins.bytes, builtins.int]" + reveal_type(a) # N: Revealed type is "builtins.str" + case a: + reveal_type(a) # N: Revealed type is "builtins.int" + +-- Guards -- + +[case testMatchSimplePatternGuard] +m: str + +def guard() -> bool: ... + +match m: + case a if guard(): + reveal_type(a) # N: Revealed type is "builtins.str" + +[case testMatchAlwaysTruePatternGuard] +m: str + +match m: + case a if True: + reveal_type(a) # N: Revealed type is "builtins.str" + +[case testMatchAlwaysFalsePatternGuard] +m: str + +match m: + case a if False: + reveal_type(a) + +[case testMatchRedefiningPatternGuard] +m: str + +match m: + case a if a := 1: # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(a) # N: Revealed type is "Never" + +[case testMatchAssigningPatternGuard] +m: str + +match m: + case a if a := "test": + reveal_type(a) # N: Revealed type is "builtins.str" + +[case testMatchNarrowingPatternGuard] +m: object + +match m: + case a if isinstance(a, str): + reveal_type(a) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstancelist.pyi] + +[case testMatchIncompatiblePatternGuard] +class A: ... +class B: ... + +m: A + +match m: + case a if isinstance(a, B): + reveal_type(a) # N: Revealed type is "__main__." +[builtins fixtures/isinstancelist.pyi] + +[case testMatchUnreachablePatternGuard] +m: str + +match m: + case a if isinstance(a, int): + reveal_type(a) +[builtins fixtures/isinstancelist.pyi] + +-- Exhaustiveness -- + +[case testMatchUnionNegativeNarrowing] +from typing import Union + +m: Union[str, int] + +match m: + case str(a): + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(m) # N: Revealed type is "builtins.str" + case b: + reveal_type(b) # N: Revealed type is "builtins.int" + reveal_type(m) # N: Revealed type is "builtins.int" + +[case testMatchOrPatternNegativeNarrowing] +from typing import Union + +m: Union[str, bytes, int] + +match m: + case str(a) | bytes(a): + reveal_type(a) # N: Revealed type is "builtins.object" + reveal_type(m) # N: Revealed type is "Union[builtins.str, builtins.bytes]" + case b: + reveal_type(b) # N: Revealed type is "builtins.int" + +[case testMatchExhaustiveReturn] +def foo(value) -> int: + match value: + case "bar": + return 1 + case _: + return 2 + +[case testMatchNonExhaustiveReturn] +def foo(value) -> int: # E: Missing return statement + match value: + case "bar": + return 1 + case 2: + return 2 + +[case testMatchMoreExhaustiveReturnCases] +def g(value: int | None) -> int: + match value: + case int(): + return 0 + case None: + return 1 + +def b(value: bool) -> int: + match value: + case True: + return 2 + case False: + return 3 + +[case testMatchMiscNonExhaustiveReturn] +class C: + a: int | str + +def f1(value: int | str | None) -> int: # E: Missing return statement + match value: + case int(): + return 0 + case None: + return 1 + +def f2(c: C) -> int: # E: Missing return statement + match c: + case C(a=int()): + return 0 + case C(a=str()): + return 1 + +def f3(x: list[str]) -> int: # E: Missing return statement + match x: + case [a]: + return 0 + case [a, b]: + return 1 + +def f4(x: dict[str, int]) -> int: # E: Missing return statement + match x: + case {'x': a}: + return 0 + +def f5(x: bool) -> int: # E: Missing return statement + match x: + case True: + return 0 +[builtins fixtures/dict.pyi] + +[case testMatchNonExhaustiveError] +from typing import NoReturn +def assert_never(x: NoReturn) -> None: ... + +def f(value: int) -> int: # E: Missing return statement + match value: + case 1: + return 0 + case 2: + return 1 + case o: + assert_never(o) # E: Argument 1 to "assert_never" has incompatible type "int"; expected "Never" + +[case testMatchExhaustiveNoError] +from typing import NoReturn, Union, Literal +def assert_never(x: NoReturn) -> None: ... + +def f(value: Literal[1] | Literal[2]) -> int: + match value: + case 1: + return 0 + case 2: + return 1 + case o: + assert_never(o) +[typing fixtures/typing-medium.pyi] + +[case testMatchSequencePatternNegativeNarrowing] +from typing import Union, Sequence, Tuple +from typing_extensions import Literal + +m1: Sequence[int | str] + +match m1: + case [int()]: + reveal_type(m1) # N: Revealed type is "typing.Sequence[builtins.int]" + case r: + reveal_type(m1) # N: Revealed type is "typing.Sequence[Union[builtins.int, builtins.str]]" + +m2: Tuple[int | str] + +match m2: + case (int(),): + reveal_type(m2) # N: Revealed type is "Tuple[builtins.int]" + case r2: + reveal_type(m2) # N: Revealed type is "Tuple[builtins.str]" + +m3: Tuple[Union[int, str]] + +match m3: + case (1,): + reveal_type(m3) # N: Revealed type is "Tuple[Literal[1]]" + case r2: + reveal_type(m3) # N: Revealed type is "Tuple[Union[builtins.int, builtins.str]]" + +m4: Tuple[Literal[1], int] + +match m4: + case (1, 5): + reveal_type(m4) # N: Revealed type is "Tuple[Literal[1], Literal[5]]" + case (1, 6): + reveal_type(m4) # N: Revealed type is "Tuple[Literal[1], Literal[6]]" + case _: + reveal_type(m4) # N: Revealed type is "Tuple[Literal[1], builtins.int]" + +m5: Tuple[Literal[1, 2], Literal["a", "b"]] + +match m5: + case (1, str()): + reveal_type(m5) # N: Revealed type is "Tuple[Literal[1], Union[Literal['a'], Literal['b']]]" + case _: + reveal_type(m5) # N: Revealed type is "Tuple[Literal[2], Union[Literal['a'], Literal['b']]]" + +match m5: + case (1, "a"): + reveal_type(m5) # N: Revealed type is "Tuple[Literal[1], Literal['a']]" + case _: + reveal_type(m5) # N: Revealed type is "Tuple[Union[Literal[1], Literal[2]], Union[Literal['a'], Literal['b']]]" + +[builtins fixtures/tuple.pyi] + +[case testMatchEnumSingleChoice] +from enum import Enum +from typing import NoReturn + +def assert_never(x: NoReturn) -> None: ... + +class Medal(Enum): + gold = 1 + +def f(m: Medal) -> None: + always_assigned: int | None = None + match m: + case Medal.gold: + always_assigned = 1 + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.gold]" + case _: + assert_never(m) + + reveal_type(always_assigned) # N: Revealed type is "builtins.int" +[builtins fixtures/bool.pyi] + +[case testMatchLiteralPatternEnumNegativeNarrowing] +from enum import Enum +class Medal(Enum): + gold = 1 + silver = 2 + bronze = 3 + +def f(m: Medal) -> int: + match m: + case Medal.gold: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.gold]" + return 0 + case _: + reveal_type(m) # N: Revealed type is "Union[Literal[__main__.Medal.silver], Literal[__main__.Medal.bronze]]" + return 1 + +def g(m: Medal) -> int: + match m: + case Medal.gold: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.gold]" + return 0 + case Medal.silver: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.silver]" + return 1 + case Medal.bronze: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.bronze]" + return 2 +[builtins fixtures/enum.pyi] + + +[case testMatchLiteralPatternEnumWithTypedAttribute] +from enum import Enum +from typing import NoReturn +def assert_never(x: NoReturn) -> None: ... + +class int: + def __new__(cls, value: int): pass + +class Medal(int, Enum): + prize: str + + def __new__(cls, value: int, prize: str) -> Medal: + enum = int.__new__(cls, value) + enum._value_ = value + enum.prize = prize + return enum + + gold = (1, 'cash prize') + silver = (2, 'sponsorship') + bronze = (3, 'nothing') + +m: Medal + +match m: + case Medal.gold: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.gold]" + case Medal.silver: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.silver]" + case Medal.bronze: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.bronze]" + case _ as unreachable: + assert_never(unreachable) + +[builtins fixtures/tuple.pyi] + +[case testMatchLiteralPatternFunctionalEnum] +from enum import Enum +from typing import NoReturn +def assert_never(x: NoReturn) -> None: ... + +Medal = Enum('Medal', 'gold silver bronze') +m: Medal + +match m: + case Medal.gold: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.gold]" + case Medal.silver: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.silver]" + case Medal.bronze: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.bronze]" + case _ as unreachable: + assert_never(unreachable) +[builtins fixtures/enum.pyi] + +[case testMatchLiteralPatternEnumCustomEquals-skip] +from enum import Enum +class Medal(Enum): + gold = 1 + silver = 2 + bronze = 3 + + def __eq__(self, other) -> bool: ... + +m: Medal + +match m: + case Medal.gold: + reveal_type(m) # N: Revealed type is "Literal[__main__.Medal.gold]" + case _: + reveal_type(m) # N: Revealed type is "__main__.Medal" +[builtins fixtures/enum.pyi] + +[case testMatchNarrowUsingPatternGuardSpecialCase] +def f(x: int | str) -> int: + match x: + case x if isinstance(x, str): + return 0 + case int(): + return 1 +[builtins fixtures/isinstance.pyi] + +[case testMatchNarrowDownUnionPartially] + +def f(x: int | str) -> None: + match x: + case int(): + return + reveal_type(x) # N: Revealed type is "builtins.str" + +def g(x: int | str | None) -> None: + match x: + case int() | None: + return + reveal_type(x) # N: Revealed type is "builtins.str" + +def h(x: int | str | None) -> None: + match x: + case int() | str(): + return + reveal_type(x) # N: Revealed type is "None" + +[case testMatchNarrowDownUsingLiteralMatch] +from enum import Enum +class Medal(Enum): + gold = 1 + silver = 2 + +def b1(x: bool) -> None: + match x: + case True: + return + reveal_type(x) # N: Revealed type is "Literal[False]" + +def b2(x: bool) -> None: + match x: + case False: + return + reveal_type(x) # N: Revealed type is "Literal[True]" + +def e1(x: Medal) -> None: + match x: + case Medal.gold: + return + reveal_type(x) # N: Revealed type is "Literal[__main__.Medal.silver]" + +def e2(x: Medal) -> None: + match x: + case Medal.silver: + return + reveal_type(x) # N: Revealed type is "Literal[__main__.Medal.gold]" + +def i(x: int) -> None: + match x: + case 1: + return + reveal_type(x) # N: Revealed type is "builtins.int" + +def s(x: str) -> None: + match x: + case 'x': + return + reveal_type(x) # N: Revealed type is "builtins.str" + +def union(x: str | bool) -> None: + match x: + case True: + return + reveal_type(x) # N: Revealed type is "Union[builtins.str, Literal[False]]" +[builtins fixtures/tuple.pyi] + +[case testMatchAssertFalseToSilenceFalsePositives] +class C: + a: int | str + +def f(c: C) -> int: + match c: + case C(a=int()): + return 0 + case C(a=str()): + return 1 + case _: + assert False + +def g(c: C) -> int: + match c: + case C(a=int()): + return 0 + case C(a=str()): + return 1 + assert False + +[case testMatchAsPatternExhaustiveness] +def f(x: int | str) -> int: + match x: + case int() as n: + return n + case str() as s: + return 1 + +[case testMatchOrPatternExhaustiveness] +from typing import NoReturn, Literal +def assert_never(x: NoReturn) -> None: ... + +Color = Literal["blue", "green", "red"] +c: Color + +match c: + case "blue": + reveal_type(c) # N: Revealed type is "Literal['blue']" + case "green" | "notColor": + reveal_type(c) # N: Revealed type is "Literal['green']" + case _: + assert_never(c) # E: Argument 1 to "assert_never" has incompatible type "Literal['red']"; expected "Never" +[typing fixtures/typing-typeddict.pyi] + +[case testMatchAsPatternIntersection-skip] +class A: pass +class B: pass +class C: pass + +def f(x: A) -> None: + match x: + case B() as y: + reveal_type(y) # N: Revealed type is "__main__." + case C() as y: + reveal_type(y) # N: Revealed type is "__main__." + reveal_type(y) # N: Revealed type is "Union[__main__., __main__.]" + +[case testMatchWithBreakAndContinue] +def f(x: int | str | None) -> None: + i = int() + while i: + match x: + case int(): + continue + case str(): + break + reveal_type(x) # N: Revealed type is "None" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + +[case testMatchNarrowDownWithStarred-skip] +from typing import List +def f(x: List[int] | int) -> None: + match x: + case [*y]: + reveal_type(y) # N: Revealed type is "builtins.list[builtins.int]" + return + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +-- Misc + +[case testMatchAndWithStatementScope] +from m import A, B + +with A() as x: + pass +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass + +with A() as y: + pass +with B() as y: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass + +with A() as z: + pass +with B() as z: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass + +with A() as zz: + pass +with B() as zz: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass + +match x: + case str(y) as z: + zz = y + +[file m.pyi] +from typing import Any + +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testOverrideMatchArgs] +class AST: + __match_args__ = () + +class stmt(AST): ... + +class AnnAssign(stmt): + __match_args__ = ('target', 'annotation', 'value', 'simple') + target: str + annotation: int + value: str + simple: int + +reveal_type(AST.__match_args__) # N: Revealed type is "Tuple[()]" +reveal_type(stmt.__match_args__) # N: Revealed type is "Tuple[()]" +reveal_type(AnnAssign.__match_args__) # N: Revealed type is "Tuple[Literal['target']?, Literal['annotation']?, Literal['value']?, Literal['simple']?]" + +AnnAssign.__match_args__ = ('a', 'b', 'c', 'd') # E: Cannot assign to "__match_args__" +__match_args__ = 0 + +def f(x: AST) -> None: + match x: + case AST(): + reveal_type(x) # N: Revealed type is "__main__.AST" + match x: + case stmt(): + reveal_type(x) # N: Revealed type is "__main__.stmt" + match x: + case AnnAssign(a, b, c, d): + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(b) # N: Revealed type is "builtins.int" + reveal_type(c) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testMatchReachableDottedNames] +# flags: --warn-unreachable +class Consts: + BLANK = "" + SPECIAL = "asdf" + +def test_func(test_str: str) -> str: + match test_str: + case Consts.BLANK: + return "blank" + case Consts.SPECIAL: + return "special" + case _: + return "other" + + +[case testNoneTypeWarning] +from types import NoneType + +def foo(x: NoneType): # E: NoneType should not be used as a type, please use None instead + reveal_type(x) # N: Revealed type is "None" + +[builtins fixtures/tuple.pyi] + +[case testMatchTupleInstanceUnionNoCrash] +from typing import Union + +def func(e: Union[str, tuple[str]]) -> None: + match e: + case (a,) if isinstance(a, str): + reveal_type(a) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testMatchTupleOptionalNoCrash] +foo: tuple[int] | None +match foo: + case x,: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchUnionTwoTuplesNoCrash] +var: tuple[int, int] | tuple[str, str] + +# TODO: we can infer better here. +match var: + case (42, a): + reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" + case ("yes", b): + reveal_type(b) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testMatchNamedAndKeywordsAreTheSame] +from typing import Generic, TypeVar, Union +from typing_extensions import Final +from dataclasses import dataclass + +T = TypeVar("T") + +class Regular: + x: str + y: int + __match_args__ = ("x",) +class ReveresedOrder: + x: int + y: str + __match_args__ = ("y",) +class GenericRegular(Generic[T]): + x: T + __match_args__ = ("x",) +class GenericWithFinal(Generic[T]): + x: T + __match_args__: Final = ("x",) +class RegularSubtype(GenericRegular[str]): ... + +@dataclass +class GenericDataclass(Generic[T]): + x: T + +input_arg: Union[ + Regular, + ReveresedOrder, + GenericRegular[str], + GenericWithFinal[str], + RegularSubtype, + GenericDataclass[str], +] + +# Positional: +match input_arg: + case Regular(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case ReveresedOrder(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericWithFinal(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case RegularSubtype(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericRegular(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericDataclass(a): + reveal_type(a) # N: Revealed type is "builtins.str" + +# Keywords: +match input_arg: + case Regular(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case ReveresedOrder(x=b): # Order is different + reveal_type(b) # N: Revealed type is "builtins.int" + case GenericWithFinal(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case RegularSubtype(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericRegular(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericDataclass(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" +[builtins fixtures/dataclasses.pyi] + +[case testMatchValueConstrainedTypeVar] +from typing import TypeVar, Iterable + +S = TypeVar("S", int, str) + +def my_func(pairs: Iterable[tuple[S, S]]) -> None: + for pair in pairs: + reveal_type(pair) # N: Revealed type is "Tuple[builtins.int, builtins.int]" \ + # N: Revealed type is "Tuple[builtins.str, builtins.str]" + match pair: + case _: + reveal_type(pair) # N: Revealed type is "Tuple[builtins.int, builtins.int]" \ + # N: Revealed type is "Tuple[builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testPossiblyUndefinedMatch] +# flags: --enable-error-code possibly-undefined +def f0(x: int | str) -> int: + match x: + case int(): + y = 1 + return y # E: Name "y" may be undefined + +def f1(a: object) -> None: + match a: + case [y]: pass + case _: + y = 1 + x = 2 + z = y + z = x # E: Name "x" may be undefined + +def f2(a: object) -> None: + match a: + case [[y] as x]: pass + case {"k1": 1, "k2": x, "k3": y}: pass + case [0, *x]: + y = 2 + case _: + y = 1 + x = [2] + z = x + z = y + +def f3(a: object) -> None: + y = 1 + match a: + case [x]: + y = 2 + # Note the missing `case _:` + z = x # E: Name "x" may be undefined + z = y + +def f4(a: object) -> None: + y = 1 + match a: + case [x]: + y = 2 + case _: + assert False, "unsupported" + z = x + z = y + +def f5(a: object) -> None: + match a: + case tuple(x): pass + case _: + return + y = x + +def f6(a: object) -> None: + if int(): + y = 1 + match a: + case _ if y is not None: # E: Name "y" may be undefined + pass +[builtins fixtures/tuple.pyi] + +[case testPossiblyUndefinedMatchUnreachable] +# flags: --enable-error-code possibly-undefined +import typing + +def f0(x: int) -> int: + match x: + case 1 if not typing.TYPE_CHECKING: + pass + case 2: + y = 2 + case _: + y = 3 + return y # No error. + +def f1(x: int) -> int: + match x: + case 1 if not typing.TYPE_CHECKING: + pass + case 2: + y = 2 + return y # E: Name "y" may be undefined + +[typing fixtures/typing-medium.pyi] + +[case testUsedBeforeDefMatchWalrus] +# flags: --enable-error-code used-before-def +import typing + +def f0(x: int) -> None: + a = y # E: Cannot determine type of "y" # E: Name "y" is used before definition + match y := x: + case 1: + b = y + case 2: + c = y + d = y + +[case testTypeAliasWithNewUnionSyntaxAndNoneLeftOperand] +from typing import overload +class C: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: int) -> None: pass + def __init__(self, x=0): + pass + +class D: pass + +X = None | C +Y = None | D +[builtins fixtures/type.pyi] + +[case testMatchStatementWalrus] +class A: + a = 1 + +def returns_a_or_none() -> A | None: + return A() + +def returns_a() -> A: + return A() + +def f() -> None: + match x := returns_a_or_none(): + case A(): + reveal_type(x.a) # N: Revealed type is "builtins.int" + match x := returns_a(): + case A(): + reveal_type(x.a) # N: Revealed type is "builtins.int" + y = returns_a_or_none() + match y: + case A(): + reveal_type(y.a) # N: Revealed type is "builtins.int" + +[case testNarrowedVariableInNestedModifiedInMatch] +from typing import Optional + +def match_stmt_error1(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + match object(): + case str(x): + pass + nested() + +def foo(x): pass + +def match_stmt_ok1(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x + match foo(x): + case str(y): + z = x + nested() + +def match_stmt_error2(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + match [None]: + case [x]: + pass + nested() + +def match_stmt_error3(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + match {'a': None}: + case {'a': x}: + pass + nested() + +def match_stmt_error4(x: Optional[list[str]]) -> None: + if x is None: + x = ["a"] + def nested() -> list[str]: + return x # E: Incompatible return value type (got "Optional[List[str]]", expected "List[str]") + match ["a"]: + case [*x]: + pass + nested() + +class C: + a: str + +def match_stmt_error5(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + match C(): + case C(a=x): + pass + nested() +[builtins fixtures/tuple.pyi] + +[case testMatchSubjectRedefinition] +# flags: --allow-redefinition +def transform1(a: str) -> int: + ... + +def transform2(a: int) -> str: + ... + +def redefinition_good(a: str): + a = transform1(a) + + match (a + 1): + case _: + ... + + +def redefinition_bad(a: int): + a = transform2(a) + + match (a + 1): # E: Unsupported operand types for + ("str" and "int") + case _: + ... + +[builtins fixtures/primitives.pyi] + +[case testPatternMatchingClassPatternLocation] +# See https://github.com/python/mypy/issues/15496 +from some_missing_lib import DataFrame, Series # type: ignore[import] +from typing import TypeVar + +T = TypeVar("T", Series, DataFrame) + +def f(x: T) -> None: + match x: + case Series() | DataFrame(): # type: ignore[misc] + pass + +def f2(x: T) -> None: + match x: + case Series(): # type: ignore[misc] + pass + case DataFrame(): # type: ignore[misc] + pass +[builtins fixtures/primitives.pyi] + +[case testMatchGuardReachability] +# flags: --warn-unreachable +def f1(e: int) -> int: + match e: + case x if True: + return x + case _: + return 0 # E: Statement is unreachable + e = 0 # E: Statement is unreachable + + +def f2(e: int) -> int: + match e: + case x if bool(): + return x + case _: + return 0 + e = 0 # E: Statement is unreachable + +def f3(e: int | str | bytes) -> int: + match e: + case x if isinstance(x, int): + return x + case [x]: + return 0 # E: Statement is unreachable + case str(x): + return 0 + reveal_type(e) # N: Revealed type is "builtins.bytes" + return 0 + +def f4(e: int | str | bytes) -> int: + match e: + case int(x): + pass + case [x]: + return 0 # E: Statement is unreachable + case x if isinstance(x, str): + return 0 + reveal_type(e) # N: Revealed type is "Union[builtins.int, builtins.bytes]" + return 0 + +[builtins fixtures/primitives.pyi] + +[case testMatchSequencePatternVariadicTupleNotTooShort] +from typing import Tuple +from typing_extensions import Unpack + +fm1: Tuple[int, int, Unpack[Tuple[str, ...]], int] +match fm1: + case [fa1, fb1, fc1]: + reveal_type(fa1) # N: Revealed type is "builtins.int" + reveal_type(fb1) # N: Revealed type is "builtins.int" + reveal_type(fc1) # N: Revealed type is "builtins.int" + +fm2: Tuple[int, int, Unpack[Tuple[str, ...]], int] +match fm2: + case [fa2, fb2]: + reveal_type(fa2) + reveal_type(fb2) + +fm3: Tuple[int, int, Unpack[Tuple[str, ...]], int] +match fm3: + case [fa3, fb3, fc3, fd3, fe3]: + reveal_type(fa3) # N: Revealed type is "builtins.int" + reveal_type(fb3) # N: Revealed type is "builtins.int" + reveal_type(fc3) # N: Revealed type is "builtins.str" + reveal_type(fd3) # N: Revealed type is "builtins.str" + reveal_type(fe3) # N: Revealed type is "builtins.int" + +m1: Tuple[int, Unpack[Tuple[str, ...]], int] +match m1: + case [a1, *b1, c1]: + reveal_type(a1) # N: Revealed type is "builtins.int" + reveal_type(b1) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(c1) # N: Revealed type is "builtins.int" + +m2: Tuple[int, Unpack[Tuple[str, ...]], int] +match m2: + case [a2, b2, *c2, d2, e2]: + reveal_type(a2) # N: Revealed type is "builtins.int" + reveal_type(b2) # N: Revealed type is "builtins.str" + reveal_type(c2) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(d2) # N: Revealed type is "builtins.str" + reveal_type(e2) # N: Revealed type is "builtins.int" + +m3: Tuple[int, int, Unpack[Tuple[str, ...]], int, int] +match m3: + case [a3, *b3, c3]: + reveal_type(a3) # N: Revealed type is "builtins.int" + reveal_type(b3) # N: Revealed type is "builtins.list[Union[builtins.int, builtins.str]]" + reveal_type(c3) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchSequencePatternTypeVarTupleNotTooShort] +from typing import Tuple +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") +def test(xs: Tuple[Unpack[Ts]]) -> None: + fm1: Tuple[int, int, Unpack[Ts], int] + match fm1: + case [fa1, fb1, fc1]: + reveal_type(fa1) # N: Revealed type is "builtins.int" + reveal_type(fb1) # N: Revealed type is "builtins.int" + reveal_type(fc1) # N: Revealed type is "builtins.int" + + fm2: Tuple[int, int, Unpack[Ts], int] + match fm2: + case [fa2, fb2]: + reveal_type(fa2) + reveal_type(fb2) + + fm3: Tuple[int, int, Unpack[Ts], int] + match fm3: + case [fa3, fb3, fc3, fd3, fe3]: + reveal_type(fa3) # N: Revealed type is "builtins.int" + reveal_type(fb3) # N: Revealed type is "builtins.int" + reveal_type(fc3) # N: Revealed type is "builtins.object" + reveal_type(fd3) # N: Revealed type is "builtins.object" + reveal_type(fe3) # N: Revealed type is "builtins.int" + + m1: Tuple[int, Unpack[Ts], int] + match m1: + case [a1, *b1, c1]: + reveal_type(a1) # N: Revealed type is "builtins.int" + reveal_type(b1) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(c1) # N: Revealed type is "builtins.int" + + m2: Tuple[int, Unpack[Ts], int] + match m2: + case [a2, b2, *c2, d2, e2]: + reveal_type(a2) # N: Revealed type is "builtins.int" + reveal_type(b2) # N: Revealed type is "builtins.object" + reveal_type(c2) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(d2) # N: Revealed type is "builtins.object" + reveal_type(e2) # N: Revealed type is "builtins.int" + + m3: Tuple[int, int, Unpack[Ts], int, int] + match m3: + case [a3, *b3, c3]: + reveal_type(a3) # N: Revealed type is "builtins.int" + reveal_type(b3) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(c3) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchSequencePatternTypeVarBoundNoCrash] +# This was crashing: https://github.com/python/mypy/issues/18089 +from typing import TypeVar, Sequence, Any + +T = TypeVar("T", bound=Sequence[Any]) + +def f(x: T) -> None: + match x: + case [_]: + pass +[builtins fixtures/tuple.pyi] + +[case testMatchSequencePatternTypeVarBoundNarrows] +from typing import TypeVar, Sequence + +T = TypeVar("T", bound=Sequence[int | str]) + +def accept_seq_int(x: Sequence[int]): ... + +def f(x: T) -> None: + match x: + case [1, 2]: + accept_seq_int(x) + case _: + accept_seq_int(x) # E: Argument 1 to "accept_seq_int" has incompatible type "T"; expected "Sequence[int]" +[builtins fixtures/tuple.pyi] + +[case testNarrowingTypeVarMatch] +# flags: --warn-unreachable + +# https://github.com/python/mypy/issues/18126 +from typing import TypeVar + +T = TypeVar("T") + +def fn_case(arg: T) -> None: + match arg: + case None: + return None + return None +[builtins fixtures/primitives.pyi] + +[case testNoneCheckDoesNotMakeTypeVarOptionalMatch] +from typing import TypeVar + +T = TypeVar('T') + +def foo(x: T) -> T: + out = None + out = x + match out: + case None: + pass + return out + +[builtins fixtures/isinstance.pyi] diff --git a/test-data/unit/check-python311.test b/test-data/unit/check-python311.test new file mode 100644 index 000000000000..6f4c540572b0 --- /dev/null +++ b/test-data/unit/check-python311.test @@ -0,0 +1,261 @@ +[case testTryStarSimple] +try: + pass +except* Exception as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[builtins.Exception]" +[builtins fixtures/exception.pyi] + +[case testTryStarMultiple] +try: + pass +except* Exception as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[builtins.Exception]" +except* RuntimeError as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[builtins.RuntimeError]" +[builtins fixtures/exception.pyi] + +[case testTryStarBase] +try: + pass +except* BaseException as e: + reveal_type(e) # N: Revealed type is "builtins.BaseExceptionGroup[builtins.BaseException]" +[builtins fixtures/exception.pyi] + +[case testTryStarTuple] +class Custom(Exception): ... + +try: + pass +except* (RuntimeError, Custom) as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Union[builtins.RuntimeError, __main__.Custom]]" +[builtins fixtures/exception.pyi] + +[case testTryStarInvalidType] +class Bad: ... +try: + pass +except* (RuntimeError, Bad) as e: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Any]" +[builtins fixtures/exception.pyi] + +[case testTryStarGroupInvalid] +try: + pass +except* ExceptionGroup as e: # E: Exception type in except* cannot derive from BaseExceptionGroup + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Any]" +[builtins fixtures/exception.pyi] + +[case testTryStarGroupInvalidTuple] +try: + pass +except* (RuntimeError, ExceptionGroup) as e: # E: Exception type in except* cannot derive from BaseExceptionGroup + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Union[builtins.RuntimeError, Any]]" +[builtins fixtures/exception.pyi] + +[case testBasicTypeVarTupleGeneric] +from typing import Generic, TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +class Variadic(Generic[Unpack[Ts]]): + ... + +variadic: Variadic[int, str] +reveal_type(variadic) # N: Revealed type is "__main__.Variadic[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAsyncGeneratorWithinComprehension] +# flags: --python-version 3.11 +from typing import Any, Generator, List + +async def asynciter(iterable): + for x in iterable: + yield x + +async def coro() -> Generator[List[Any], None, None]: + return ([i async for i in asynciter([0,j])] for j in [3, 5]) +reveal_type(coro) # N: Revealed type is "def () -> typing.Coroutine[Any, Any, typing.Generator[builtins.list[Any], None, None]]" +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testTypeVarTupleNewSyntaxAnnotations] +Ints = tuple[int, int, int] +x: tuple[str, *Ints] +reveal_type(x) # N: Revealed type is "Tuple[builtins.str, builtins.int, builtins.int, builtins.int]" +y: tuple[int, *tuple[int, ...]] +reveal_type(y) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleNewSyntaxGenerics] +from typing import Generic, TypeVar, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +class C(Generic[T, *Ts]): + attr: tuple[int, *Ts, str] + + def test(self) -> None: + reveal_type(self.attr) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`2], builtins.str]" + self.attr = ci # E: Incompatible types in assignment (expression has type "C[*Tuple[int, ...]]", variable has type "Tuple[int, *Ts, str]") + def meth(self, *args: *Ts) -> T: ... + +ci: C[*tuple[int, ...]] +reveal_type(ci) # N: Revealed type is "__main__.C[Unpack[builtins.tuple[builtins.int, ...]]]" +reveal_type(ci.meth) # N: Revealed type is "def (*args: builtins.int) -> builtins.int" +c3: C[str, str, str] +reveal_type(c3) # N: Revealed type is "__main__.C[builtins.str, builtins.str, builtins.str]" + +A = C[int, *Ts] +B = tuple[str, *tuple[str, str], str] +z: A[*B] +reveal_type(z) # N: Revealed type is "__main__.C[builtins.int, builtins.str, builtins.str, builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleNewSyntaxCallables] +from typing import Generic, overload, TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2") +class MyClass(Generic[T1, T2]): + @overload + def __init__(self: MyClass[None, None]) -> None: ... + + @overload + def __init__(self: MyClass[T1, None], *types: *tuple[type[T1]]) -> None: ... + + @overload + def __init__(self: MyClass[T1, T2], *types: *tuple[type[T1], type[T2]]) -> None: ... + + def __init__(self: MyClass[T1, T2], *types: *tuple[type, ...]) -> None: + pass + +myclass = MyClass() +reveal_type(myclass) # N: Revealed type is "__main__.MyClass[None, None]" +myclass1 = MyClass(float) +reveal_type(myclass1) # N: Revealed type is "__main__.MyClass[builtins.float, None]" +myclass2 = MyClass(float, float) +reveal_type(myclass2) # N: Revealed type is "__main__.MyClass[builtins.float, builtins.float]" +myclass3 = MyClass(float, float, float) # E: No overload variant of "MyClass" matches argument types "Type[float]", "Type[float]", "Type[float]" \ + # N: Possible overload variants: \ + # N: def [T1, T2] __init__(self) -> MyClass[None, None] \ + # N: def [T1, T2] __init__(self, Type[T1], /) -> MyClass[T1, None] \ + # N: def [T1, T2] __init__(Type[T1], Type[T2], /) -> MyClass[T1, T2] +reveal_type(myclass3) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testUnpackNewSyntaxInvalidCallableAlias] +from typing import Any, Callable, List, Tuple, TypeVar, Unpack + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") # E: Name "TypeVarTuple" is not defined + +def good(*x: int) -> int: ... +def bad(*x: int, y: int) -> int: ... + +Alias1 = Callable[[*Ts], int] # E: Variable "__main__.Ts" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +x1: Alias1[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(x1) # N: Revealed type is "def (*Any) -> builtins.int" +x1 = good +x1 = bad # E: Incompatible types in assignment (expression has type "Callable[[VarArg(int), NamedArg(int, 'y')], int]", variable has type "Callable[[VarArg(Any)], int]") + +Alias2 = Callable[[*T], int] # E: "T" cannot be unpacked (must be tuple or TypeVarTuple) +x2: Alias2[int] +reveal_type(x2) # N: Revealed type is "def (*Any) -> builtins.int" + +Unknown = Any +Alias3 = Callable[[*Unknown], int] +x3: Alias3[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(x3) # N: Revealed type is "def (*Any) -> builtins.int" + +IntList = List[int] +Alias4 = Callable[[*IntList], int] # E: "List[int]" cannot be unpacked (must be tuple or TypeVarTuple) +x4: Alias4[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(x4) # N: Revealed type is "def (*Any) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testReturnInExceptStarBlock1] +# flags: --python-version 3.11 +def foo() -> None: + try: + pass + except* Exception: + return # E: "return" not allowed in except* block + finally: + return +[builtins fixtures/exception.pyi] + +[case testReturnInExceptStarBlock2] +# flags: --python-version 3.11 +def foo(): + while True: + try: + pass + except* Exception: + while True: + return # E: "return" not allowed in except* block +[builtins fixtures/exception.pyi] + +[case testContinueInExceptBlockNestedInExceptStarBlock] +# flags: --python-version 3.11 +while True: + try: + ... + except* Exception: + try: + ... + except Exception: + continue # E: "continue" not allowed in except* block + continue # E: "continue" not allowed in except* block +[builtins fixtures/exception.pyi] + +[case testReturnInExceptBlockNestedInExceptStarBlock] +# flags: --python-version 3.11 +def foo(): + try: + ... + except* Exception: + try: + ... + except Exception: + return # E: "return" not allowed in except* block + return # E: "return" not allowed in except* block +[builtins fixtures/exception.pyi] + +[case testBreakContinueReturnInExceptStarBlock1] +# flags: --python-version 3.11 +from typing import Iterable +def foo(x: Iterable[int]) -> None: + for _ in x: + try: + pass + except* Exception: + continue # E: "continue" not allowed in except* block + except* Exception: + for _ in x: + continue + break # E: "break" not allowed in except* block + except* Exception: + return # E: "return" not allowed in except* block +[builtins fixtures/exception.pyi] + +[case testBreakContinueReturnInExceptStarBlock2] +# flags: --python-version 3.11 +def foo(): + while True: + try: + pass + except* Exception: + def inner(): + while True: + if 1 < 1: + continue + else: + break + return + if 1 < 2: + break # E: "break" not allowed in except* block + if 1 < 2: + continue # E: "continue" not allowed in except* block + return # E: "return" not allowed in except* block +[builtins fixtures/exception.pyi] diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test new file mode 100644 index 000000000000..8b4d638ecdaa --- /dev/null +++ b/test-data/unit/check-python312.test @@ -0,0 +1,1974 @@ +[case testPEP695TypeAliasBasic] +type MyInt = int + +def f(x: MyInt) -> MyInt: + return reveal_type(x) # N: Revealed type is "builtins.int" + +type MyList[T] = list[T] + +def g(x: MyList[int]) -> MyList[int]: + return reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" + +type MyInt2 = int + +def h(x: MyInt2) -> MyInt2: + return reveal_type(x) # N: Revealed type is "builtins.int" + +[case testPEP695Class] +class MyGen[T]: + def __init__(self, x: T) -> None: + self.x = x + +def f(x: MyGen[int]): + reveal_type(x.x) # N: Revealed type is "builtins.int" + +[case testPEP695Function] +def f[T](x: T) -> T: + return reveal_type(x) # N: Revealed type is "T`-1" + +reveal_type(f(1)) # N: Revealed type is "builtins.int" + +async def g[T](x: T) -> T: + return reveal_type(x) # N: Revealed type is "T`-1" + +reveal_type(g(1)) # E: Value of type "Coroutine[Any, Any, int]" must be used \ + # N: Are you missing an await? \ + # N: Revealed type is "typing.Coroutine[Any, Any, builtins.int]" + +[case testPEP695TypeVarBasic] +from typing import Callable +type Alias1[T: int] = list[T] +type Alias2[**P] = Callable[P, int] +type Alias3[*Ts] = tuple[*Ts] + +class Cls1[T: int]: ... +class Cls2[**P]: ... +class Cls3[*Ts]: ... + +def func1[T: int](x: T) -> T: ... +def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... +def func3[*Ts](x: tuple[*Ts]) -> tuple[int, *Ts]: ... +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeAliasType] +from typing import Callable, TypeAliasType, TypeVar, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") + +TestType = TypeAliasType("TestType", int | str) +x: TestType = 42 +y: TestType = 'a' +z: TestType = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[int, str]") + +BadAlias1 = TypeAliasType("BadAlias1", tuple[*Ts]) # E: TypeVarTuple "Ts" is not included in type_params +ba1: BadAlias1[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(ba1) # N: Revealed type is "builtins.tuple[Any, ...]" + +BadAlias2 = TypeAliasType("BadAlias2", Callable[[*Ts], str]) # E: TypeVarTuple "Ts" is not included in type_params +ba2: BadAlias2[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(ba2) # N: Revealed type is "def (*Any) -> builtins.str" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695IncompleteFeatureIsAcceptedButHasNoEffect] +# mypy: enable-incomplete-feature=NewGenericSyntax +def f[T](x: T) -> T: + return x +reveal_type(f(1)) # N: Revealed type is "builtins.int" + +[case testPEP695GenericFunctionSyntax] +def ident[TV](x: TV) -> TV: + y: TV = x + y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "TV") + return x + +reveal_type(ident(1)) # N: Revealed type is "builtins.int" +reveal_type(ident('x')) # N: Revealed type is "builtins.str" + +a: TV # E: Name "TV" is not defined + +def tup[T, S](x: T, y: S) -> tuple[T, S]: + reveal_type((x, y)) # N: Revealed type is "Tuple[T`-1, S`-2]" + return (x, y) + +reveal_type(tup(1, 'x')) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testPEP695GenericClassSyntax] +class C[T]: + x: T + + def __init__(self, x: T) -> None: + self.x = x + + def ident(self, x: T) -> T: + y: T = x + if int(): + return self.x + else: + return y + +reveal_type(C("x")) # N: Revealed type is "__main__.C[builtins.str]" +c: C[int] = C(1) +reveal_type(c.x) # N: Revealed type is "builtins.int" +reveal_type(c.ident(1)) # N: Revealed type is "builtins.int" + +[case testPEP695GenericMethodInGenericClass] +class C[T]: + def m[S](self, x: S) -> T | S: ... + +a: C[int] = C[object]() # E: Incompatible types in assignment (expression has type "C[object]", variable has type "C[int]") +b: C[object] = C[int]() + +reveal_type(C[str]().m(1)) # N: Revealed type is "Union[builtins.str, builtins.int]" + +[case testPEP695InferVarianceSimpleFromMethod] +class Invariant[T]: + def f(self, x: T) -> None: + pass + + def g(self) -> T | None: + return None + +a: Invariant[object] +b: Invariant[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") + +class Covariant[T]: + def g(self) -> T | None: + return None + +c: Covariant[object] +d: Covariant[int] +if int(): + c = d +if int(): + d = c # E: Incompatible types in assignment (expression has type "Covariant[object]", variable has type "Covariant[int]") + +class Contravariant[T]: + def f(self, x: T) -> None: + pass + +e: Contravariant[object] +f: Contravariant[int] +if int(): + e = f # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[object]") +if int(): + f = e + +[case testPEP695InferVarianceSimpleFromAttribute] +class Invariant1[T]: + def __init__(self, x: T) -> None: + self.x = x + +a: Invariant1[object] +b: Invariant1[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant1[int]", variable has type "Invariant1[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Invariant1[object]", variable has type "Invariant1[int]") + +class Invariant2[T]: + def __init__(self) -> None: + self.x: list[T] = [] + +a2: Invariant2[object] +b2: Invariant2[int] +if int(): + a2 = b2 # E: Incompatible types in assignment (expression has type "Invariant2[int]", variable has type "Invariant2[object]") +if int(): + b2 = a2 # E: Incompatible types in assignment (expression has type "Invariant2[object]", variable has type "Invariant2[int]") + +class Invariant3[T]: + def __init__(self) -> None: + self.x: T | None = None + +a3: Invariant3[object] +b3: Invariant3[int] +if int(): + a3 = b3 # E: Incompatible types in assignment (expression has type "Invariant3[int]", variable has type "Invariant3[object]") +if int(): + b3 = a3 # E: Incompatible types in assignment (expression has type "Invariant3[object]", variable has type "Invariant3[int]") + +[case testPEP695InferVarianceRecursive] +class Invariant[T]: + def f(self, x: Invariant[T]) -> Invariant[T]: + return x + +class Covariant[T]: + def f(self) -> Covariant[T]: + return self + +class Contravariant[T]: + def f(self, x: Contravariant[T]) -> None: + pass + +a: Invariant[object] +b: Invariant[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") +if int(): + b = a + +c: Covariant[object] +d: Covariant[int] +if int(): + c = d +if int(): + d = c # E: Incompatible types in assignment (expression has type "Covariant[object]", variable has type "Covariant[int]") + +e: Contravariant[object] +f: Contravariant[int] +if int(): + e = f # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[object]") +if int(): + f = e + +[case testPEP695InferVarianceInFrozenDataclass] +from dataclasses import dataclass + +@dataclass(frozen=True) +class Covariant[T]: + x: T + +cov1: Covariant[float] = Covariant[int](1) +cov2: Covariant[int] = Covariant[float](1) # E: Incompatible types in assignment (expression has type "Covariant[float]", variable has type "Covariant[int]") + +@dataclass(frozen=True) +class Invariant[T]: + x: list[T] + +inv1: Invariant[float] = Invariant[int]([1]) # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[float]") +inv2: Invariant[int] = Invariant[float]([1]) # E: Incompatible types in assignment (expression has type "Invariant[float]", variable has type "Invariant[int]") +[builtins fixtures/tuple.pyi] + +[case testPEP695InferVarianceCalculateOnDemand] +class Covariant[T]: + def __init__(self) -> None: + self.x = [1] + + def f(self) -> None: + c = Covariant[int]() + # We need to know that T is covariant here + self.g(c) + c2 = Covariant[object]() + self.h(c2) # E: Argument 1 to "h" of "Covariant" has incompatible type "Covariant[object]"; expected "Covariant[int]" + + def g(self, x: Covariant[object]) -> None: pass + def h(self, x: Covariant[int]) -> None: pass + +[case testPEP695InferVarianceNotReadyWhenNeeded] +class Covariant[T]: + def f(self) -> None: + c = Covariant[int]() + # We need to know that T is covariant here + self.g(c) + c2 = Covariant[object]() + self.h(c2) # E: Argument 1 to "h" of "Covariant" has incompatible type "Covariant[object]"; expected "Covariant[int]" + + def g(self, x: Covariant[object]) -> None: pass + def h(self, x: Covariant[int]) -> None: pass + + def __init__(self) -> None: + self.x = [1] + +class Invariant[T]: + def f(self) -> None: + c = Invariant(1) + # We need to know that T is invariant here, and for this we need the type + # of self.x, which won't be available on the first type checking pass, + # since __init__ is defined later in the file. In this case we fall back + # covariance. + self.g(c) + c2 = Invariant(object()) + self.h(c2) # E: Argument 1 to "h" of "Invariant" has incompatible type "Invariant[object]"; expected "Invariant[int]" + + def g(self, x: Invariant[object]) -> None: pass + def h(self, x: Invariant[int]) -> None: pass + + def __init__(self, x: T) -> None: + self.x = x + +# Now we should have the variance correct. +a: Invariant[object] +b: Invariant[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") + +[case testPEP695InferVarianceNotReadyForJoin] +class Invariant[T]: + def f(self) -> None: + # Assume covariance if variance us not ready + reveal_type([Invariant(1), Invariant(object())]) \ + # N: Revealed type is "builtins.list[__main__.Invariant[builtins.object]]" + + def __init__(self, x: T) -> None: + self.x = x + +reveal_type([Invariant(1), Invariant(object())]) # N: Revealed type is "builtins.list[builtins.object]" + +[case testPEP695InferVarianceNotReadyForMeet] +from typing import TypeVar, Callable + +S = TypeVar("S") +def c(a: Callable[[S], None], b: Callable[[S], None]) -> S: ... + +def a1(x: Invariant[int]) -> None: pass +def a2(x: Invariant[object]) -> None: pass + +class Invariant[T]: + def f(self) -> None: + reveal_type(c(a1, a2)) # N: Revealed type is "__main__.Invariant[builtins.int]" + + def __init__(self, x: T) -> None: + self.x = x + +reveal_type(c(a1, a2)) # N: Revealed type is "Never" + +[case testPEP695InferVarianceUnderscorePrefix] +class Covariant1[T]: + def __init__(self, x: T) -> None: + self._x = x + + @property + def x(self) -> T: + return self._x + +co1_1: Covariant1[float] = Covariant1[int](1) +co1_2: Covariant1[int] = Covariant1[float](1) # E: Incompatible types in assignment (expression has type "Covariant1[float]", variable has type "Covariant1[int]") + +class Covariant2[T]: + def __init__(self, x: T) -> None: + self.__foo_bar = x + + @property + def x(self) -> T: + return self.__foo_bar + +co2_1: Covariant2[float] = Covariant2[int](1) +co2_2: Covariant2[int] = Covariant2[float](1) # E: Incompatible types in assignment (expression has type "Covariant2[float]", variable has type "Covariant2[int]") + +class Invariant1[T]: + def __init__(self, x: T) -> None: + self._x = x + + # Methods behave differently from attributes + def _f(self, x: T) -> None: ... + + @property + def x(self) -> T: + return self._x + +inv1_1: Invariant1[float] = Invariant1[int](1) # E: Incompatible types in assignment (expression has type "Invariant1[int]", variable has type "Invariant1[float]") +inv1_2: Invariant1[int] = Invariant1[float](1) # E: Incompatible types in assignment (expression has type "Invariant1[float]", variable has type "Invariant1[int]") + +class Invariant2[T]: + def __init__(self, x: T) -> None: + # Dunders are special + self.__x__ = x + + @property + def x(self) -> T: + return self.__x__ + +inv2_1: Invariant2[float] = Invariant2[int](1) # E: Incompatible types in assignment (expression has type "Invariant2[int]", variable has type "Invariant2[float]") +inv2_2: Invariant2[int] = Invariant2[float](1) # E: Incompatible types in assignment (expression has type "Invariant2[float]", variable has type "Invariant2[int]") + +class Invariant3[T]: + def __init__(self, x: T) -> None: + self._x = Invariant1(x) + + @property + def x(self) -> T: + return self._x._x + +inv3_1: Invariant3[float] = Invariant3[int](1) # E: Incompatible types in assignment (expression has type "Invariant3[int]", variable has type "Invariant3[float]") +inv3_2: Invariant3[int] = Invariant3[float](1) # E: Incompatible types in assignment (expression has type "Invariant3[float]", variable has type "Invariant3[int]") +[builtins fixtures/property.pyi] + +[case testPEP695InferVarianceWithInheritedSelf] +from typing import overload, Self, TypeVar, Generic + +T = TypeVar("T") +S = TypeVar("S") + +class C(Generic[T]): + def f(self, x: T) -> Self: ... + def g(self) -> T: ... + +class D[T1, T2](C[T1]): + def m(self, x: T2) -> None: ... + +a1: D[int, int] = D[int, object]() +a2: D[int, object] = D[int, int]() # E: Incompatible types in assignment (expression has type "D[int, int]", variable has type "D[int, object]") +a3: D[int, int] = D[object, object]() # E: Incompatible types in assignment (expression has type "D[object, object]", variable has type "D[int, int]") +a4: D[object, int] = D[int, object]() # E: Incompatible types in assignment (expression has type "D[int, object]", variable has type "D[object, int]") + +[case testPEP695InferVarianceWithReturnSelf] +from typing import Self, overload + +class Cov[T]: + def f(self) -> Self: ... + +a1: Cov[int] = Cov[float]() # E: Incompatible types in assignment (expression has type "Cov[float]", variable has type "Cov[int]") +a2: Cov[float] = Cov[int]() + +class Contra[T]: + def f(self) -> Self: ... + def g(self, x: T) -> None: ... + +b1: Contra[int] = Contra[float]() +b2: Contra[float] = Contra[int]() # E: Incompatible types in assignment (expression has type "Contra[int]", variable has type "Contra[float]") + +class Cov2[T]: + @overload + def f(self, x): ... + @overload + def f(self) -> Self: ... + def f(self, x=None): ... + +c1: Cov2[int] = Cov2[float]() # E: Incompatible types in assignment (expression has type "Cov2[float]", variable has type "Cov2[int]") +c2: Cov2[float] = Cov2[int]() + +class Contra2[T]: + @overload + def f(self, x): ... + @overload + def f(self) -> Self: ... + def f(self, x=None): ... + + def g(self, x: T) -> None: ... + +d1: Contra2[int] = Contra2[float]() +d2: Contra2[float] = Contra2[int]() # E: Incompatible types in assignment (expression has type "Contra2[int]", variable has type "Contra2[float]") + +[case testPEP695InheritInvariant] +class Invariant[T]: + x: T + +class Subclass[T](Invariant[T]): + pass + +x: Invariant[int] +y: Invariant[object] +if int(): + x = y # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") +if int(): + y = x # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") + +a: Subclass[int] +b: Subclass[object] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Subclass[object]", variable has type "Subclass[int]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Subclass[int]", variable has type "Subclass[object]") + +[case testPEP695InheritanceMakesInvariant] +class Covariant[T]: + def f(self) -> T: + ... + +class Subclass[T](Covariant[list[T]]): + pass + +x: Covariant[int] = Covariant[object]() # E: Incompatible types in assignment (expression has type "Covariant[object]", variable has type "Covariant[int]") +y: Covariant[object] = Covariant[int]() + +a: Subclass[int] = Subclass[object]() # E: Incompatible types in assignment (expression has type "Subclass[object]", variable has type "Subclass[int]") +b: Subclass[object] = Subclass[int]() # E: Incompatible types in assignment (expression has type "Subclass[int]", variable has type "Subclass[object]") + +[case testPEP695InheritCoOrContravariant] +class Contravariant[T]: + def f(self, x: T) -> None: pass + +class CovSubclass[T](Contravariant[T]): + pass + +a: CovSubclass[int] = CovSubclass[object]() +b: CovSubclass[object] = CovSubclass[int]() # E: Incompatible types in assignment (expression has type "CovSubclass[int]", variable has type "CovSubclass[object]") + +class Covariant[T]: + def f(self) -> T: ... + +class CoSubclass[T](Covariant[T]): + pass + +c: CoSubclass[int] = CoSubclass[object]() # E: Incompatible types in assignment (expression has type "CoSubclass[object]", variable has type "CoSubclass[int]") +d: CoSubclass[object] = CoSubclass[int]() + +class InvSubclass[T](Covariant[T]): + def g(self, x: T) -> None: pass + +e: InvSubclass[int] = InvSubclass[object]() # E: Incompatible types in assignment (expression has type "InvSubclass[object]", variable has type "InvSubclass[int]") +f: InvSubclass[object] = InvSubclass[int]() # E: Incompatible types in assignment (expression has type "InvSubclass[int]", variable has type "InvSubclass[object]") + +[case testPEP695FinalAttribute] +from typing import Final + +class C[T]: + def __init__(self, x: T) -> None: + self.x: Final = x + +a: C[int] = C[object](1) # E: Incompatible types in assignment (expression has type "C[object]", variable has type "C[int]") +b: C[object] = C[int](1) + +[case testPEP695TwoTypeVariables] +class C[T, S]: + def f(self, x: T) -> None: ... + def g(self) -> S: ... + +a: C[int, int] = C[object, int]() +b: C[object, int] = C[int, int]() # E: Incompatible types in assignment (expression has type "C[int, int]", variable has type "C[object, int]") +c: C[int, int] = C[int, object]() # E: Incompatible types in assignment (expression has type "C[int, object]", variable has type "C[int, int]") +d: C[int, object] = C[int, int]() + +[case testPEP695Properties] +class R[T]: + @property + def p(self) -> T: ... + +class RW[T]: + @property + def p(self) -> T: ... + @p.setter + def p(self, x: T) -> None: ... + +a: R[int] = R[object]() # E: Incompatible types in assignment (expression has type "R[object]", variable has type "R[int]") +b: R[object] = R[int]() +c: RW[int] = RW[object]() # E: Incompatible types in assignment (expression has type "RW[object]", variable has type "RW[int]") +d: RW[object] = RW[int]() # E: Incompatible types in assignment (expression has type "RW[int]", variable has type "RW[object]") +[builtins fixtures/property.pyi] + +[case testPEP695Protocol] +from typing import Protocol + +class PContra[T](Protocol): + def f(self, x: T) -> None: ... + +PContra() # E: Cannot instantiate protocol class "PContra" +a: PContra[int] +b: PContra[object] +if int(): + a = b +if int(): + b = a # E: Incompatible types in assignment (expression has type "PContra[int]", variable has type "PContra[object]") + +class PCov[T](Protocol): + def f(self) -> T: ... + +PCov() # E: Cannot instantiate protocol class "PCov" +c: PCov[int] +d: PCov[object] +if int(): + c = d # E: Incompatible types in assignment (expression has type "PCov[object]", variable has type "PCov[int]") +if int(): + d = c + +class PInv[T](Protocol): + def f(self, x: T) -> T: ... + +PInv() # E: Cannot instantiate protocol class "PInv" +e: PInv[int] +f: PInv[object] +if int(): + e = f # E: Incompatible types in assignment (expression has type "PInv[object]", variable has type "PInv[int]") +if int(): + f = e # E: Incompatible types in assignment (expression has type "PInv[int]", variable has type "PInv[object]") + +[case testPEP695TypeAlias] +class C[T]: pass +class D[T, S]: pass + +type A[S] = C[S] + +a: A[int] +reveal_type(a) # N: Revealed type is "__main__.C[builtins.int]" + +type A2[T] = C[C[T]] +a2: A2[str] +reveal_type(a2) # N: Revealed type is "__main__.C[__main__.C[builtins.str]]" + +type A3[T, S] = D[S, C[T]] +a3: A3[int, str] +reveal_type(a3) # N: Revealed type is "__main__.D[builtins.str, __main__.C[builtins.int]]" + +type A4 = int | str +a4: A4 +reveal_type(a4) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/type.pyi] + +[case testPEP695TypeAliasNotValidAsBaseClass] +from typing import TypeAlias + +import m + +type A1 = int +class Bad1(A1): # E: Type alias defined using "type" statement not valid as base class + pass + +type A2[T] = list[T] +class Bad2(A2[int]): # E: Type alias defined using "type" statement not valid as base class + pass + +class Bad3(m.A1): # E: Type alias defined using "type" statement not valid as base class + pass + +class Bad4(m.A2[int]): # E: Type alias defined using "type" statement not valid as base class + pass + +B1 = int +B2 = list +B3: TypeAlias = int +class Good1(B1): pass +class Good2(B2[int]): pass +class Good3(list[A1]): pass +class Good4(list[A2[int]]): pass +class Good5(B3): pass + +[file m.py] +type A1 = str +type A2[T] = list[T] +[typing fixtures/typing-medium.pyi] + +[case testPEP695TypeAliasWithUnusedTypeParams] +type A[T] = int +a: A[str] +reveal_type(a) # N: Revealed type is "builtins.int" + +[case testPEP695TypeAliasForwardReference1] +type A[T] = C[T] + +a: A[int] +reveal_type(a) # N: Revealed type is "__main__.C[builtins.int]" + +class C[T]: pass + +[case testPEP695TypeAliasForwardReference2] +type X = C +type A = X + +a: A +reveal_type(a) # N: Revealed type is "__main__.C" + +class C: pass +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasForwardReference3] +type X = D +type A = C[X] + +a: A +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" + +class C[T]: pass +class D: pass + +[case testPEP695TypeAliasForwardReference4] +type A = C + +class D(A): # E: Type alias defined using "type" statement not valid as base class + pass + +class C: pass + +x: C = D() +y: D = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") + +[case testPEP695TypeAliasForwardReference5] +type A = str +type B[T] = C[T] +class C[T]: pass +a: A +b: B[int] +c: C[str] +reveal_type(a) # N: Revealed type is "builtins.str" +reveal_type(b) # N: Revealed type is "__main__.C[builtins.int]" +reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" + +[case testPEP695TypeAliasWithUndefineName] +type A[T] = XXX # E: Name "XXX" is not defined +a: A[int] +reveal_type(a) # N: Revealed type is "Any" + +[case testPEP695TypeAliasInvalidType] +type A = int | 1 # E: Invalid type: try using Literal[1] instead? + +a: A +reveal_type(a) # N: Revealed type is "Union[builtins.int, Any]" +type B = int + str # E: Invalid type alias: expression is not a valid type +b: B +reveal_type(b) # N: Revealed type is "Any" +[builtins fixtures/type.pyi] + +[case testPEP695TypeAliasBoundForwardReference] +type B[T: Foo] = list[T] +class Foo: pass + +[case testPEP695UpperBound] +class D: + x: int +class E(D): pass + +class C[T: D]: pass + +a: C[D] +b: C[E] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E]" + +c: C[int] # E: Type argument "int" of "C" must be a subtype of "D" + +def f[T: D](a: T) -> T: + reveal_type(a.x) # N: Revealed type is "builtins.int" + return a + +reveal_type(f(D())) # N: Revealed type is "__main__.D" +reveal_type(f(E())) # N: Revealed type is "__main__.E" +f(1) # E: Value of type variable "T" of "f" cannot be "int" + +[case testPEP695UpperBoundForwardReference1] +class C[T: D]: pass + +a: C[D] +b: C[E] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E]" + +c: C[int] # E: Type argument "int" of "C" must be a subtype of "D" + +class D: pass +class E(D): pass + +[case testPEP695UpperBoundForwardReference2] +type A = D +class C[T: A]: pass + +class D: pass +class E(D): pass + +a: C[D] +b: C[E] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E]" + +c: C[int] # E: Type argument "int" of "C" must be a subtype of "D" + +[case testPEP695UpperBoundForwardReference3] +class D[T]: pass +class E[T](D[T]): pass + +type A = D[X] + +class C[T: A]: pass + +class X: pass + +a: C[D[X]] +b: C[E[X]] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D[__main__.X]]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E[__main__.X]]" + +c: C[D[int]] # E: Type argument "D[int]" of "C" must be a subtype of "D[X]" + +[case testPEP695UpperBoundForwardReference4] +def f[T: D](a: T) -> T: + reveal_type(a.x) # N: Revealed type is "builtins.int" + return a + +class D: + x: int +class E(D): pass + +reveal_type(f(D())) # N: Revealed type is "__main__.D" +reveal_type(f(E())) # N: Revealed type is "__main__.E" +f(1) # E: Value of type variable "T" of "f" cannot be "int" + +[case testPEP695UpperBoundUndefinedName] +class C[T: XX]: # E: Name "XX" is not defined + pass + +a: C[int] + +def f[T: YY](x: T) -> T: # E: Name "YY" is not defined + return x +reveal_type(f) # N: Revealed type is "def [T <: Any] (x: T`-1) -> T`-1" + +[case testPEP695UpperBoundWithMultipleParams] +class C[T, S: int]: pass +class D[A: int, B]: pass + +def f[T: int, S: int | str](x: T, y: S) -> T | S: + return x + +C[str, int]() +C[str, str]() # E: Value of type variable "S" of "C" cannot be "str" +D[int, str]() +D[str, str]() # E: Value of type variable "A" of "D" cannot be "str" +f(1, 1) +u: int | str +f(1, u) +f('x', None) # E: Value of type variable "T" of "f" cannot be "str" \ + # E: Value of type variable "S" of "f" cannot be "None" + +[case testPEP695InferVarianceOfTupleType] +class Cov[T](tuple[int, str]): + def f(self) -> T: pass + +class Cov2[T](tuple[T, T]): + pass + +class Contra[T](tuple[int, str]): + def f(self, x: T) -> None: pass + +a: Cov[object] = Cov[int]() +b: Cov[int] = Cov[object]() # E: Incompatible types in assignment (expression has type "Cov[object]", variable has type "Cov[int]") + +c: Cov2[object] = Cov2[int]() +d: Cov2[int] = Cov2[object]() # E: Incompatible types in assignment (expression has type "Cov2[object]", variable has type "Cov2[int]") + +e: Contra[int] = Contra[object]() +f: Contra[object] = Contra[int]() # E: Incompatible types in assignment (expression has type "Contra[int]", variable has type "Contra[object]") +[builtins fixtures/tuple-simple.pyi] + +[case testPEP695ValueRestriction] +def f[T: (int, str)](x: T) -> T: + reveal_type(x) # N: Revealed type is "builtins.int" \ + # N: Revealed type is "builtins.str" + return x + +reveal_type(f(1)) # N: Revealed type is "builtins.int" +reveal_type(f('x')) # N: Revealed type is "builtins.str" +f(None) # E: Value of type variable "T" of "f" cannot be "None" + +class C[T: (object, None)]: pass + +a: C[object] +b: C[None] +c: C[int] # E: Value of type variable "T" of "C" cannot be "int" + +[case testPEP695ValueRestrictionForwardReference] +class C[T: (int, D)]: + def __init__(self, x: T) -> None: + a = x + if int(): + a = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") \ + # E: Incompatible types in assignment (expression has type "str", variable has type "D") + self.x: T = x + +reveal_type(C(1).x) # N: Revealed type is "builtins.int" +C(None) # E: Value of type variable "T" of "C" cannot be "None" + +class D: pass + +C(D()) + +[case testPEP695ValueRestrictionUndefinedName] +class C[T: (int, XX)]: # E: Name "XX" is not defined + pass + +def f[S: (int, YY)](x: S) -> S: # E: Name "YY" is not defined + return x + +[case testPEP695ParamSpec] +from typing import Callable + +def g[**P](f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: + f(*args, **kwargs) + f(1, *args, **kwargs) # E: Argument 1 has incompatible type "int"; expected "P.args" + +def h(x: int, y: str) -> None: pass + +g(h, 1, y='x') +g(h, 1, x=1) # E: "g" gets multiple values for keyword argument "x" \ + # E: Missing positional argument "y" in call to "g" + +class C[**P, T]: + def m(self, *args: P.args, **kwargs: P.kwargs) -> T: ... + +a: C[[int, str], None] +reveal_type(a) # N: Revealed type is "__main__.C[[builtins.int, builtins.str], None]" +reveal_type(a.m) # N: Revealed type is "def (builtins.int, builtins.str)" +[builtins fixtures/tuple.pyi] + +[case testPEP695ParamSpecTypeAlias] +from typing import Callable + +type C[**P] = Callable[P, int] + +f: C[[str, int | None]] +reveal_type(f) # N: Revealed type is "def (builtins.str, Union[builtins.int, None]) -> builtins.int" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeVarTuple] +def f[*Ts](t: tuple[*Ts]) -> tuple[*Ts]: + reveal_type(t) # N: Revealed type is "Tuple[Unpack[Ts`-1]]" + return t + +reveal_type(f((1, 'x'))) # N: Revealed type is "Tuple[Literal[1]?, Literal['x']?]" +a: tuple[int, ...] +reveal_type(f(a)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +class C[T, *Ts]: + pass + +b: C[int, str, None] +reveal_type(b) # N: Revealed type is "__main__.C[builtins.int, builtins.str, None]" +c: C[str] +reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" +b = c # E: Incompatible types in assignment (expression has type "C[str]", variable has type "C[int, str, None]") +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeVarTupleAlias] +from typing import Callable + +type C[*Ts] = tuple[*Ts, int] + +a: C[str, None] +reveal_type(a) # N: Revealed type is "Tuple[builtins.str, None, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testPEP695IncrementalFunction] +import a + +[file a.py] +import b + +[file a.py.2] +import b +reveal_type(b.f(1)) +reveal_type(b.g(1, 'x')) +b.g('x', 'x') +b.g(1, 2) + +[file b.py] +def f[T](x: T) -> T: + return x + +def g[T: int, S: (str, None)](x: T, y: S) -> T | S: + return x + +[out2] +tmp/a.py:2: note: Revealed type is "builtins.int" +tmp/a.py:3: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/a.py:4: error: Value of type variable "T" of "g" cannot be "str" +tmp/a.py:5: error: Value of type variable "S" of "g" cannot be "int" + +[case testPEP695IncrementalClass] +import a + +[file a.py] +import b + +[file a.py.2] +from b import C, D +x: C[int] +reveal_type(x) + +class N(int): pass +class SS(str): pass + +y1: D[int, str] +y2: D[N, str] +y3: D[int, None] +y4: D[int, None] +y5: D[int, SS] # Error +y6: D[object, str] # Error + +[file b.py] +class C[T]: pass + +class D[T: int, S: (str, None)]: + pass + +[out2] +tmp/a.py:3: note: Revealed type is "b.C[builtins.int]" +tmp/a.py:12: error: Value of type variable "S" of "D" cannot be "SS" +tmp/a.py:13: error: Type argument "object" of "D" must be a subtype of "int" + +[case testPEP695IncrementalParamSpecAndTypeVarTuple] +import a + +[file a.py] +import b + +[file a.py.2] +from b import C, D +x1: C[()] +x2: C[int] +x3: C[int, str] +y: D[[int, str]] +reveal_type(y.m) + +[file b.py] +class C[*Ts]: pass +class D[**P]: + def m(self, *args: P.args, **kwargs: P.kwargs) -> None: pass + +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:6: note: Revealed type is "def (builtins.int, builtins.str)" + +[case testPEP695IncrementalTypeAlias] +import a + +[file a.py] +import b + +[file a.py.2] +from b import A, B +a: A +reveal_type(a) +b: B[int] +reveal_type(b) + +[file b.py] +type A = str +class Foo[T]: pass +type B[T] = Foo[T] + +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:3: note: Revealed type is "builtins.str" +tmp/a.py:5: note: Revealed type is "b.Foo[builtins.int]" + +[case testPEP695UndefinedNameInGenericFunction] +def f[T](x: T) -> T: + return unknown() # E: Name "unknown" is not defined + +class C: + def m[T](self, x: T) -> T: + return unknown() # E: Name "unknown" is not defined + +[case testPEP695FunctionTypeVarAccessInFunction] +from typing import cast + +class C: + def m[T](self, x: T) -> T: + y: T = x + reveal_type(y) # N: Revealed type is "T`-1" + return cast(T, y) + +reveal_type(C().m(1)) # N: Revealed type is "builtins.int" + +[case testPEP695ScopingBasics] +T = 1 + +def f[T](x: T) -> T: + T = 'a' + reveal_type(T) # N: Revealed type is "builtins.str" + return x + +reveal_type(T) # N: Revealed type is "builtins.int" + +class C[T]: + T = 1.2 + reveal_type(T) # N: Revealed type is "builtins.float" + +reveal_type(T) # N: Revealed type is "builtins.int" + +[case testPEP695ClassScoping] +class C: + class D: pass + + def m[T: D](self, x: T, y: D) -> T: + return x + +C().m(C.D(), C.D()) +C().m(1, C.D()) # E: Value of type variable "T" of "m" of "C" cannot be "int" + +[case testPEP695NestedGenericFunction] +def f[T](x: T) -> T: + reveal_type(f(x)) # N: Revealed type is "T`-1" + reveal_type(f(1)) # N: Revealed type is "builtins.int" + + def ff(x: T) -> T: + y: T = x + return y + reveal_type(ff(x)) # N: Revealed type is "T`-1" + ff(1) # E: Argument 1 to "ff" has incompatible type "int"; expected "T" + + def g[S](a: S) -> S: + ff(a) # E: Argument 1 to "ff" has incompatible type "S"; expected "T" + return a + reveal_type(g(1)) # N: Revealed type is "builtins.int" + reveal_type(g(x)) # N: Revealed type is "T`-1" + + def h[S](a: S) -> S: + return a + reveal_type(h(1)) # N: Revealed type is "builtins.int" + reveal_type(h(x)) # N: Revealed type is "T`-1" + return x + +[case testPEP695NonLocalAndGlobal] +def f() -> None: + T = 1 + def g[T](x: T) -> T: + nonlocal T # E: nonlocal binding not allowed for type parameter "T" + T = 'x' # E: "T" is a type variable and only valid in type context + return x + reveal_type(T) # N: Revealed type is "builtins.int" + +def g() -> None: + a = 1 + def g[T](x: T) -> T: + nonlocal a + a = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + return x + +x = 1 + +def h[T](a: T) -> T: + global x + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + return a + +class C[T]: + def m[S](self, a: S) -> S: + global x + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + return a + +[case testPEP695ArgumentDefault] +from typing import cast + +def f[T]( + x: T = + T # E: Name "T" is not defined \ + # E: Incompatible default for argument "x" (default has type "TypeVar", argument has type "T") +) -> T: + return x + +def g[T](x: T = cast(T, None)) -> T: # E: Name "T" is not defined + return x + +class C: + def m[T](self, x: T = cast(T, None)) -> T: # E: Name "T" is not defined + return x +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695ListComprehension] +from typing import cast + +def f[T](x: T) -> T: + b = [cast(T, a) for a in [1, 2]] + reveal_type(b) # N: Revealed type is "builtins.list[T`-1]" + return x + +[case testPEP695ReuseNameInSameScope] +class C[T]: + def m[S](self, x: S, y: T) -> S | T: + return x + + def m2[S](self, x: S, y: T) -> S | T: + return x + +class D[T]: + pass + +def f[T](x: T) -> T: + return x + +def g[T](x: T) -> T: + def nested[S](y: S) -> S: + return y + def nested2[S](y: S) -> S: + return y + return x + +[case testPEP695NestedScopingSpecialCases] +# This is adapted from PEP 695 +S = 0 + +def outer1[S]() -> None: + S = 1 + T = 1 + + def outer2[T]() -> None: + def inner1() -> None: + nonlocal S + nonlocal T # E: nonlocal binding not allowed for type parameter "T" + + def inner2() -> None: + global S + +[case testPEP695ScopingWithBaseClasses] +# This is adapted from PEP 695 +class Outer: + class Private: + pass + + # If the type parameter scope was like a traditional scope, + # the base class 'Private' would not be accessible here. + class Inner[T](Private, list[T]): + pass + + # Likewise, 'Inner' would not be available in these type annotations. + def method1[T](self, a: Inner[T]) -> Inner[T]: + return a + +[case testPEP695RedefineTypeParameterInScope] +class C[T]: + def m[T](self, x: T) -> T: # E: "T" already defined as a type parameter + return x + def m2(self) -> None: + def nested[T](x: T) -> T: # E: "T" already defined as a type parameter + return x + +def f[S, S](x: S) -> S: # E: "S" already defined as a type parameter + return x + +[case testPEP695ClassDecorator] +from typing import Any + +T = 0 + +def decorator(x: str) -> Any: ... + +@decorator(T) # E: Argument 1 to "decorator" has incompatible type "int"; expected "str" +class C[T]: + pass + +[case testPEP695RecursiceTypeAlias] +type A = str | list[A] +a: A +reveal_type(a) # N: Revealed type is "Union[builtins.str, builtins.list[...]]" + +class C[T]: pass + +type B[T] = C[T] | list[B[T]] +b: B[int] +reveal_type(b) # N: Revealed type is "Union[__main__.C[builtins.int], builtins.list[...]]" +[builtins fixtures/type.pyi] + +[case testPEP695BadRecursiveTypeAlias] +type A = A # E: Cannot resolve name "A" (possible cyclic definition) +type B = B | int # E: Invalid recursive alias: a union item of itself +a: A +reveal_type(a) # N: Revealed type is "Any" +b: B +reveal_type(b) # N: Revealed type is "Any" +[builtins fixtures/type.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695RecursiveTypeAliasForwardReference] +def f(a: A) -> None: + if isinstance(a, str): + reveal_type(a) # N: Revealed type is "builtins.str" + else: + reveal_type(a) # N: Revealed type is "__main__.C[Union[builtins.str, __main__.C[...]]]" + +type A = str | C[A] + +class C[T]: pass + +f('x') +f(C[str]()) +f(C[C[str]]()) +f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "A" +f(C[int]()) # E: Argument 1 to "f" has incompatible type "C[int]"; expected "A" +[builtins fixtures/isinstance.pyi] + +[case testPEP695InvalidGenericOrProtocolBaseClass] +from typing import Generic, Protocol, TypeVar + +S = TypeVar("S") + +class C[T](Generic[T]): # E: Generic[...] base class is redundant + pass +class C2[T](Generic[S]): # E: Generic[...] base class is redundant + pass + +a: C[int] +b: C2[int, str] + +class P[T](Protocol[T]): # E: No arguments expected for "Protocol" base class + pass +class P2[T](Protocol[S]): # E: No arguments expected for "Protocol" base class + pass + +[case testPEP695CannotUseTypeVarFromOuterClass] +class ClassG[V]: + # This used to crash + class ClassD[T: dict[str, V]]: # E: Name "V" is not defined + ... +[builtins fixtures/dict.pyi] + +[case testPEP695MixNewAndOldStyleGenerics] +from typing import TypeVar + +S = TypeVar("S") +U = TypeVar("U") + +def f[T](x: T, y: S) -> T | S: ... # E: All type parameters should be declared ("S" not declared) +def g[T](x: S, y: U) -> T | S | U: ... # E: All type parameters should be declared ("S", "U" not declared) + +def h[S: int](x: S) -> S: + a: int = x + return x + +class C[T]: + def m[X, S](self, x: S, y: U) -> X | S | U: ... # E: All type parameters should be declared ("U" not declared) + def m2(self, x: T, y: S) -> T | S: ... + +class D[T](C[S]): # E: All type parameters should be declared ("S" not declared) + pass + +[case testPEP695MixNewAndOldStyleTypeVarTupleAndParamSpec] +from typing import TypeVarTuple, ParamSpec, Callable +Ts = TypeVarTuple("Ts") +P = ParamSpec("P") + +def f[T](x: T, f: Callable[P, None] # E: All type parameters should be declared ("P" not declared) + ) -> Callable[P, T]: ... +def g[T](x: T, f: tuple[*Ts] # E: All type parameters should be declared ("Ts" not declared) + ) -> tuple[T, *Ts]: ... +[builtins fixtures/tuple.pyi] + +[case testPEP695MixNewAndOldStyleGenericsInTypeAlias] +from typing import TypeVar, ParamSpec, TypeVarTuple, Callable + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +P = ParamSpec("P") + +type A = list[T] # E: All type parameters should be declared ("T" not declared) +a: A[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(a) # N: Revealed type is "builtins.list[Any]" + +type B = tuple[*Ts] # E: All type parameters should be declared ("Ts" not declared) +type C = Callable[P, None] # E: All type parameters should be declared ("P" not declared) +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695NonGenericAliasToGenericClass] +class C[T]: pass +type A = C +x: C +y: A +reveal_type(x) # N: Revealed type is "__main__.C[Any]" +reveal_type(y) # N: Revealed type is "__main__.C[Any]" +z: A[int] # E: Bad number of arguments for type alias, expected 0, given 1 + +[case testPEP695SelfType] +from typing import Self + +class C: + @classmethod + def m[T](cls, x: T) -> tuple[Self, T]: + return cls(), x + +class D(C): + pass + +reveal_type(C.m(1)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" +reveal_type(D.m(1)) # N: Revealed type is "Tuple[__main__.D, builtins.int]" + +class E[T]: + def m(self) -> Self: + return self + + def mm[S](self, x: S) -> tuple[Self, S]: + return self, x + +class F[T](E[T]): + pass + +reveal_type(E[int]().m()) # N: Revealed type is "__main__.E[builtins.int]" +reveal_type(E[int]().mm(b'x')) # N: Revealed type is "Tuple[__main__.E[builtins.int], builtins.bytes]" +reveal_type(F[str]().m()) # N: Revealed type is "__main__.F[builtins.str]" +reveal_type(F[str]().mm(b'x')) # N: Revealed type is "Tuple[__main__.F[builtins.str], builtins.bytes]" +[builtins fixtures/tuple.pyi] + +[case testPEP695CallAlias] +class C: + def __init__(self, x: str) -> None: ... +type A = C + +class D[T]: pass +type B[T] = D[T] + +reveal_type(A) # N: Revealed type is "typing.TypeAliasType" +reveal_type(B) # N: Revealed type is "typing.TypeAliasType" +reveal_type(B[int]) # N: Revealed type is "typing.TypeAliasType" + +A(1) # E: "TypeAliasType" not callable +B[int]() # E: "TypeAliasType" not callable + +A2 = C +B2 = D +A2(1) # E: Argument 1 to "C" has incompatible type "int"; expected "str" +B2[int]() +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695IncrementalTypeAliasKinds] +import a + +[file a.py] +from b import A + +[file a.py.2] +from b import A, B, C +A() +B() +C() + +[file b.py] +from typing_extensions import TypeAlias +type A = int +B = int +C: TypeAlias = int +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] +[out2] +tmp/a.py:2: error: "TypeAliasType" not callable + +[case testPEP695TypeAliasBoundAndValueChecking] +from typing import Any, cast + +class C: pass +class D(C): pass + +type A[T: C] = list[T] +a1: A +reveal_type(a1) # N: Revealed type is "builtins.list[Any]" +a2: A[Any] +a3: A[C] +a4: A[D] +a5: A[object] # E: Type argument "object" of "A" must be a subtype of "C" +a6: A[int] # E: Type argument "int" of "A" must be a subtype of "C" + +x1 = cast(A[C], a1) +x2 = cast(A[None], a1) # E: Type argument "None" of "A" must be a subtype of "C" + +type A2[T: (int, C)] = list[T] +b1: A2 +reveal_type(b1) # N: Revealed type is "builtins.list[Any]" +b2: A2[Any] +b3: A2[int] +b4: A2[C] +b5: A2[D] # E: Value of type variable "T" of "A2" cannot be "D" +b6: A2[object] # E: Value of type variable "T" of "A2" cannot be "object" + +list[A2[int]]() +list[A2[None]]() # E: Invalid type argument value for "A2" + +class N(int): pass + +type A3[T: C, S: (int, str)] = T | S +c1: A3[C, int] +c2: A3[D, str] +c3: A3[C, N] # E: Value of type variable "S" of "A3" cannot be "N" +c4: A3[int, str] # E: Type argument "int" of "A3" must be a subtype of "C" +[builtins fixtures/type.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasInClassBodyOrFunction] +class C: + type A = int + type B[T] = list[T] | None + a: A + b: B[str] + + def method(self) -> None: + v: C.A + reveal_type(v) # N: Revealed type is "builtins.int" + +reveal_type(C.a) # N: Revealed type is "builtins.int" +reveal_type(C.b) # N: Revealed type is "Union[builtins.list[builtins.str], None]" + +C.A = str # E: Incompatible types in assignment (expression has type "Type[str]", variable has type "TypeAliasType") + +x: C.A +y: C.B[int] +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "Union[builtins.list[builtins.int], None]" + +def f() -> None: + type A = int + type B[T] = list[T] | None + a: A + reveal_type(a) # N: Revealed type is "builtins.int" + + def g() -> None: + b: B[int] + reveal_type(b) # N: Revealed type is "Union[builtins.list[builtins.int], None]" + +class D: + def __init__(self) -> None: + type A = int + self.a: A = 0 + type B[T] = list[T] + self.b: B[int] = [1] + +reveal_type(D().a) # N: Revealed type is "builtins.int" +reveal_type(D().b) # N: Revealed type is "builtins.list[builtins.int]" + +class E[T]: + type X = list[T] # E: All type parameters should be declared ("T" not declared) + + def __init__(self) -> None: + type A = list[T] # E: All type parameters should be declared ("T" not declared) + self.a: A + +reveal_type(E[str]().a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/type.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasInvalidGenericConstraint] +class A[T]: + class a[S: (int, list[T])]: pass # E: Name "T" is not defined + type b[S: (int, list[T])] = S # E: TypeVar constraint type cannot be parametrized by type variables + def c[S: (int, list[T])](self) -> None: ... # E: TypeVar constraint type cannot be parametrized by type variables +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasUnboundTypeVarConstraint] +from typing import TypeVar +T = TypeVar("T") +class a[S: (int, list[T])]: pass # E: Type variable "__main__.T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) +type b[S: (int, list[T])] = S # E: Type variable "__main__.T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) +def c[S: (int, list[T])](self) -> None: ... # E: Type variable "__main__.T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695RedefineAsTypeAlias1] +class C: pass +type C = int # E: Name "C" already defined on line 1 + +A = 0 +type A = str # E: Name "A" already defined on line 4 +reveal_type(A) # N: Revealed type is "builtins.int" + +[case testPEP695RedefineAsTypeAlias2] +from m import D +type D = int # E: Name "D" already defined (possibly by an import) +a: D +reveal_type(a) # N: Revealed type is "m.D" +[file m.py] +class D: pass + +[case testPEP695RedefineAsTypeAlias3] +D = list["Forward"] +type D = int # E: Name "D" already defined on line 1 +Forward = str +x: D +reveal_type(x) # N: Revealed type is "builtins.list[builtins.str]" + +[case testPEP695MultiDefinitionsForTypeAlias] +if int(): + type A[T] = list[T] +else: + type A[T] = str # E: Name "A" already defined on line 2 +x: T # E: Name "T" is not defined +a: A[int] +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +[case testPEP695UndefinedNameInAnnotation] +def f[T](x: foobar, y: T) -> T: ... # E: Name "foobar" is not defined +reveal_type(f) # N: Revealed type is "def [T] (x: Any, y: T`-1) -> T`-1" + +[case testPEP695WrongNumberOfConstrainedTypes] +type A[T: ()] = list[T] # E: Type variable must have at least two constrained types +a: A[int] +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +type B[T: (int,)] = list[T] # E: Type variable must have at least two constrained types +b: B[str] +reveal_type(b) # N: Revealed type is "builtins.list[builtins.str]" + +[case testPEP695UsingTypeVariableInOwnBoundOrConstraint] +type A[T: list[T]] = str # E: Name "T" is not defined +type B[S: (list[S], str)] = str # E: Name "S" is not defined +type C[T, S: list[T]] = str # E: Name "T" is not defined + +def f[T: T](x: T) -> T: ... # E: Name "T" is not defined +class D[T: T]: # E: Name "T" is not defined + pass + +[case testPEP695InvalidType] +def f[T: 1](x: T) -> T: ... # E: Invalid type: try using Literal[1] instead? +class C[T: (int, (1 + 2))]: pass # E: Invalid type comment or annotation +type A = list[1] # E: Invalid type: try using Literal[1] instead? +type B = (1 + 2) # E: Invalid type alias: expression is not a valid type +a: A +reveal_type(a) # N: Revealed type is "builtins.list[Any]" +b: B +reveal_type(b) # N: Revealed type is "Any" + +[case testPEP695GenericNamedTuple] +from typing import NamedTuple + +# Invariant because of the signature of the generated _replace method +class N[T](NamedTuple): + x: T + y: int + +a: N[object] +reveal_type(a.x) # N: Revealed type is "builtins.object" +b: N[int] +reveal_type(b.x) # N: Revealed type is "builtins.int" +if int(): + a = b # E: Incompatible types in assignment (expression has type "N[int]", variable has type "N[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "N[object]", variable has type "N[int]") + +class M[T: (int, str)](NamedTuple): + x: T + +c: M[int] +d: M[str] +e: M[bool] # E: Value of type variable "T" of "M" cannot be "bool" + +[builtins fixtures/tuple.pyi] + +[case testPEP695GenericTypedDict] +from typing import TypedDict + +class D[T](TypedDict): + x: T + y: int + +class E[T: str](TypedDict): + x: T + y: int + +a: D[object] +reveal_type(a["x"]) # N: Revealed type is "builtins.object" +b: D[int] +reveal_type(b["x"]) # N: Revealed type is "builtins.int" +c: E[str] +d: E[int] # E: Type argument "int" of "E" must be a subtype of "str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testCurrentClassWorksAsBound] +from typing import Protocol + +class Comparable[T: Comparable](Protocol): + def compare(self, other: T) -> bool: ... + +class Good: + def compare(self, other: Good) -> bool: ... + +x: Comparable[Good] +y: Comparable[int] # E: Type argument "int" of "Comparable" must be a subtype of "Comparable[Any]" + +[case testPEP695TypeAliasWithDifferentTargetTypes] +import types # We need GenericAlias from here, and test stubs don't bring in 'types' +from typing import Any, Callable, List, Literal, TypedDict + +# Test that various type expressions don't generate false positives as type alias +# values, as they are type checked as expressions. There is a similar test case in +# pythoneval.test that uses typeshed stubs. + +class C[T]: pass + +class TD(TypedDict): + x: int + +type A1 = type[int] +type A2 = type[int] | None +type A3 = None | type[int] +type A4 = type[Any] + +type B1[**P, R] = Callable[P, R] | None +type B2[**P, R] = None | Callable[P, R] +type B3 = Callable[[str], int] +type B4 = Callable[..., int] + +type C1 = A1 | None +type C2 = None | A1 + +type D1 = Any | None +type D2 = None | Any + +type E1 = List[int] +type E2 = List[int] | None +type E3 = None | List[int] + +type F1 = Literal[1] +type F2 = Literal['x'] | None +type F3 = None | Literal[True] + +type G1 = tuple[int, Any] +type G2 = tuple[int, Any] | None +type G3 = None | tuple[int, Any] + +type H1 = TD +type H2 = TD | None +type H3 = None | TD + +type I1 = C[int] +type I2 = C[Any] | None +type I3 = None | C[TD] +[builtins fixtures/type.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypedDictInlineYesNewStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +type X[T] = {"item": T, "other": X[T] | None} +x: X[str] +reveal_type(x) # N: Revealed type is "TypedDict({'item': builtins.str, 'other': Union[..., None]})" +if x["other"] is not None: + reveal_type(x["other"]["item"]) # N: Revealed type is "builtins.str" + +type Y[T] = {"item": T, **Y[T]} # E: Overwriting TypedDict field "item" while merging +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695UsingIncorrectExpressionsInTypeVariableBound] +type X[T: (yield 1)] = Any # E: Yield expression cannot be used as a type variable bound +type Y[T: (yield from [])] = Any # E: Yield expression cannot be used as a type variable bound +type Z[T: (a := 1)] = Any # E: Named expression cannot be used as a type variable bound +type K[T: (await 1)] = Any # E: Await expression cannot be used as a type variable bound + +type XNested[T: (1 + (yield 1))] = Any # E: Yield expression cannot be used as a type variable bound +type YNested[T: (1 + (yield from []))] = Any # E: Yield expression cannot be used as a type variable bound +type ZNested[T: (1 + (a := 1))] = Any # E: Named expression cannot be used as a type variable bound +type KNested[T: (1 + (await 1))] = Any # E: Await expression cannot be used as a type variable bound + +class FooX[T: (yield 1)]: pass # E: Yield expression cannot be used as a type variable bound +class FooY[T: (yield from [])]: pass # E: Yield expression cannot be used as a type variable bound +class FooZ[T: (a := 1)]: pass # E: Named expression cannot be used as a type variable bound +class FooK[T: (await 1)]: pass # E: Await expression cannot be used as a type variable bound + +class FooXNested[T: (1 + (yield 1))]: pass # E: Yield expression cannot be used as a type variable bound +class FooYNested[T: (1 + (yield from []))]: pass # E: Yield expression cannot be used as a type variable bound +class FooZNested[T: (1 + (a := 1))]: pass # E: Named expression cannot be used as a type variable bound +class FooKNested[T: (1 + (await 1))]: pass # E: Await expression cannot be used as a type variable bound + +def foox[T: (yield 1)](): pass # E: Yield expression cannot be used as a type variable bound +def fooy[T: (yield from [])](): pass # E: Yield expression cannot be used as a type variable bound +def fooz[T: (a := 1)](): pass # E: Named expression cannot be used as a type variable bound +def fook[T: (await 1)](): pass # E: Await expression cannot be used as a type variable bound + +def foox_nested[T: (1 + (yield 1))](): pass # E: Yield expression cannot be used as a type variable bound +def fooy_nested[T: (1 + (yield from []))](): pass # E: Yield expression cannot be used as a type variable bound +def fooz_nested[T: (1 + (a := 1))](): pass # E: Named expression cannot be used as a type variable bound +def fook_nested[T: (1 +(await 1))](): pass # E: Await expression cannot be used as a type variable bound + +[case testPEP695UsingIncorrectExpressionsInTypeAlias] +type X = (yield 1) # E: Yield expression cannot be used within a type alias +type Y = (yield from []) # E: Yield expression cannot be used within a type alias +type Z = (a := 1) # E: Named expression cannot be used within a type alias +type K = (await 1) # E: Await expression cannot be used within a type alias + +type XNested = (1 + (yield 1)) # E: Yield expression cannot be used within a type alias +type YNested = (1 + (yield from [])) # E: Yield expression cannot be used within a type alias +type ZNested = (1 + (a := 1)) # E: Named expression cannot be used within a type alias +type KNested = (1 + (await 1)) # E: Await expression cannot be used within a type alias + +[case testPEP695TypeAliasAndAnnotated] +from typing_extensions import Annotated, Annotated as _Annotated +import typing_extensions as t + +def ann(*args): ... + +type A = Annotated[int, ann()] +type B = Annotated[int | str, ann((1, 2))] +type C = _Annotated[int, ann()] +type D = t.Annotated[str, ann()] + +x: A +y: B +z: C +zz: D +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(z) # N: Revealed type is "builtins.int" +reveal_type(zz) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testPEP695NestedGenericClass1] +class C[T]: + def f(self) -> T: ... + +class A: + class B[Q]: + def __init__(self, a: Q) -> None: + self.a = a + + def f(self) -> Q: + return self.a + + def g(self, x: Q) -> None: ... + + b: B[str] + +x: A.B[int] +x.g("x") # E: Argument 1 to "g" of "B" has incompatible type "str"; expected "int" +reveal_type(x.a) # N: Revealed type is "builtins.int" +reveal_type(x) # N: Revealed type is "__main__.A.B[builtins.int]" +reveal_type(A.b) # N: Revealed type is "__main__.A.B[builtins.str]" + +[case testPEP695NestedGenericClass2] +class A: + def m(self) -> None: + class B[T]: + def f(self) -> T: ... + x: B[int] + reveal_type(x.f()) # N: Revealed type is "builtins.int" + self.a = B[str]() + +reveal_type(A().a) # N: Revealed type is "__main__.B@3[builtins.str]" +reveal_type(A().a.f()) # N: Revealed type is "builtins.str" + +[case testPEP695NestedGenericClass3] +class C[T]: + def f(self) -> T: ... + class D[S]: + x: T # E: Name "T" is not defined + def g(self) -> S: ... + +a: C[int] +reveal_type(a.f()) # N: Revealed type is "builtins.int" +b: C.D[str] +reveal_type(b.g()) # N: Revealed type is "builtins.str" + +class E[T]: + class F[T]: # E: "T" already defined as a type parameter + x: T + +c: E.F[int] + +[case testPEP695NestedGenericClass4] +class A: + class B[T]: + def __get__(self, instance: A, owner: type[A]) -> T: + return None # E: Incompatible return value type (got "None", expected "T") + f = B[int]() + +a = A() +v = a.f + +[case testPEP695VarianceInheritedFromBaseWithExplicitVariance] +from typing import TypeVar, Generic + +T = TypeVar("T") + +class ParentInvariant(Generic[T]): + pass + +class Invariant1[T](ParentInvariant[T]): + pass + +a1: Invariant1[int] = Invariant1[float]() # E: Incompatible types in assignment (expression has type "Invariant1[float]", variable has type "Invariant1[int]") +a2: Invariant1[float] = Invariant1[int]() # E: Incompatible types in assignment (expression has type "Invariant1[int]", variable has type "Invariant1[float]") + +T_contra = TypeVar("T_contra", contravariant=True) + +class ParentContravariant(Generic[T_contra]): + pass + +class Contravariant[T](ParentContravariant[T]): + pass + +b1: Contravariant[int] = Contravariant[float]() +b2: Contravariant[float] = Contravariant[int]() # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[float]") + +class Invariant2[T](ParentContravariant[T]): + def f(self) -> T: ... + +c1: Invariant2[int] = Invariant2[float]() # E: Incompatible types in assignment (expression has type "Invariant2[float]", variable has type "Invariant2[int]") +c2: Invariant2[float] = Invariant2[int]() # E: Incompatible types in assignment (expression has type "Invariant2[int]", variable has type "Invariant2[float]") + +class Multi[T, S](ParentInvariant[T], ParentContravariant[S]): + pass + +d1: Multi[int, str] = Multi[float, str]() # E: Incompatible types in assignment (expression has type "Multi[float, str]", variable has type "Multi[int, str]") +d2: Multi[float, str] = Multi[int, str]() # E: Incompatible types in assignment (expression has type "Multi[int, str]", variable has type "Multi[float, str]") +d3: Multi[str, int] = Multi[str, float]() +d4: Multi[str, float] = Multi[str, int]() # E: Incompatible types in assignment (expression has type "Multi[str, int]", variable has type "Multi[str, float]") + +[case testPEP695MultipleNestedGenericClass1] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + class B: + class C: + class D[Q]: + def g(self, x: Q): ... + d: D[str] + +x: A.B.C.D[int] +x.g('a') # E: Argument 1 to "g" of "D" has incompatible type "str"; expected "int" +reveal_type(x) # N: Revealed type is "__main__.A.B.C.D[builtins.int]" +reveal_type(A.B.C.d) # N: Revealed type is "__main__.A.B.C.D[builtins.str]" + +[case testPEP695MultipleNestedGenericClass2] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + class B: + def m(self) -> None: + class C[T]: + def f(self) -> T: ... + x: C[int] + reveal_type(x.f()) # N: Revealed type is "builtins.int" + self.a = C[str]() + +reveal_type(A().B().a) # N: Revealed type is "__main__.C@5[builtins.str]" + +[case testPEP695MultipleNestedGenericClass3] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + class C[T]: + def f(self) -> T: ... + class D[S]: + x: T # E: Name "T" is not defined + def g(self) -> S: ... + +a: A.C[int] +reveal_type(a.f()) # N: Revealed type is "builtins.int" +b: A.C.D[str] +reveal_type(b.g()) # N: Revealed type is "builtins.str" + +class B: + class E[T]: + class F[T]: # E: "T" already defined as a type parameter + x: T + +c: B.E.F[int] + +[case testPEP695MultipleNestedGenericClass4] +# flags: --enable-incomplete-feature=NewGenericSyntax +class Z: + class A: + class B[T]: + def __get__(self, instance: Z.A, owner: type[Z.A]) -> T: + return None # E: Incompatible return value type (got "None", expected "T") + f = B[int]() + +a = Z.A() +v = a.f + +[case testPEP695MultipleNestedGenericClass5] +# flags: --enable-incomplete-feature=NewGenericSyntax +from a.b.c import d +x: d.D.E.F.G[int] +x.g('a') # E: Argument 1 to "g" of "G" has incompatible type "str"; expected "int" +reveal_type(x) # N: Revealed type is "a.b.c.d.D.E.F.G[builtins.int]" +reveal_type(d.D.E.F.d) # N: Revealed type is "a.b.c.d.D.E.F.G[builtins.str]" + +[file a/b/c/d.py] +class D: + class E: + class F: + class G[Q]: + def g(self, x: Q): ... + d: G[str] diff --git a/test-data/unit/check-python313.test b/test-data/unit/check-python313.test new file mode 100644 index 000000000000..2729ad3e21d1 --- /dev/null +++ b/test-data/unit/check-python313.test @@ -0,0 +1,257 @@ +[case testPEP695TypeParameterDefaultSupported] +class C[T = None]: ... +def f[T = list[int]]() -> None: ... +def g[**P = [int, str]]() -> None: ... +type A[T, S = int, U = str] = list[T] + +[case testPEP695TypeParameterDefaultBasic] +from typing import Callable + +def f1[T1 = int](a: T1) -> list[T1]: ... +reveal_type(f1) # N: Revealed type is "def [T1 = builtins.int] (a: T1`-1 = builtins.int) -> builtins.list[T1`-1 = builtins.int]" + +def f2[**P1 = [int, str]](a: Callable[P1, None]) -> Callable[P1, None]: ... +reveal_type(f2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] (a: def (*P1.args, **P1.kwargs)) -> def (*P1.args, **P1.kwargs)" + +def f3[*Ts1 = *tuple[int, str]](a: tuple[*Ts1]) -> tuple[*Ts1]: ... +reveal_type(f3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] (a: Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]) -> Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]" + + +class ClassA1[T1 = int]: ... +class ClassA2[**P1 = [int, str]]: ... +class ClassA3[*Ts1 = *tuple[int, str]]: ... + +reveal_type(ClassA1) # N: Revealed type is "def [T1 = builtins.int] () -> __main__.ClassA1[T1`1 = builtins.int]" +reveal_type(ClassA2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] () -> __main__.ClassA2[P1`1 = [builtins.int, builtins.str]]" +reveal_type(ClassA3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] () -> __main__.ClassA3[Unpack[Ts1`1 = Unpack[Tuple[builtins.int, builtins.str]]]]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultValid] +from typing import Any + +class ClassT1[T = int]: ... +class ClassT2[T: float = int]: ... +class ClassT3[T: list[Any] = list[int]]: ... +class ClassT4[T: (int, str) = int]: ... + +class ClassP1[**P = []]: ... +class ClassP2[**P = ...]: ... +class ClassP3[**P = [int, str]]: ... + +class ClassTs1[*Ts = *tuple[int]]: ... +class ClassTs2[*Ts = *tuple[int, ...]]: ... +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultInvalid] +class ClassT1[T = 2]: ... # E: TypeVar "default" must be a type +class ClassT2[T = [int]]: ... # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type +class ClassT3[T: str = int]: ... # E: TypeVar default must be a subtype of the bound type +class ClassT4[T: list[str] = list[int]]: ... # E: TypeVar default must be a subtype of the bound type +class ClassT5[T: (int, str) = bytes]: ... # E: TypeVar default must be one of the constraint types +class ClassT6[T: (int, str) = int | str]: ... # E: TypeVar default must be one of the constraint types +class ClassT7[T: (float, str) = int]: ... # E: TypeVar default must be one of the constraint types + +class ClassP1[**P = int]: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +class ClassP2[**P = 2]: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +class ClassP3[**P = (2, int)]: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +class ClassP4[**P = [2, int]]: ... # E: Argument 0 of ParamSpec default must be a type + +class ClassTs1[*Ts = 2]: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +class ClassTs2[*Ts = int]: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +class ClassTs3[*Ts = tuple[int]]: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultInvalid2] +from typing import overload +def f1[T = 2]() -> None: ... # E: TypeVar "default" must be a type +def f2[T = [int]]() -> None: ... # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type +def f3[T: str = int](x: T) -> T: ... # E: TypeVar default must be a subtype of the bound type +def f4[T: list[str] = list[int]](x: T) -> T: ... # E: TypeVar default must be a subtype of the bound type +def f5[T: (int, str) = bytes](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f6[T: (int, str) = int | str](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f7[T: (float, str) = int](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f8[T: str = int]() -> None: ... # TODO check unused TypeVars +@overload +def f9[T: str = int](x: T) -> T: ... # E: TypeVar default must be a subtype of the bound type +@overload +def f9[T: (int, str) = bytes](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f9() -> None: ... # type: ignore[misc] + +def g1[**P = int]() -> None: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +def g2[**P = 2]() -> None: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +def g3[**P = (2, int)]() -> None: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +def g4[**P = [2, int]]() -> None: ... # E: Argument 0 of ParamSpec default must be a type + +def h1[*Ts = 2]() -> None: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +def h2[*Ts = int]() -> None: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +def h3[*Ts = tuple[int]]() -> None: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultInvalid3] +from typing import Callable + +type TA1[T: str = 1] = list[T] # E: TypeVar "default" must be a type +type TA2[T: str = [int]] = list[T] # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type +type TA3[T: str = int] = list[T] # E: TypeVar default must be a subtype of the bound type +type TA4[T: list[str] = list[int]] = list[T] # E: TypeVar default must be a subtype of the bound type +type TA5[T: (int, str) = bytes] = list[T] # E: TypeVar default must be one of the constraint types +type TA6[T: (int, str) = int | str] = list[T] # E: TypeVar default must be one of the constraint types +type TA7[T: (float, str) = int] = list[T] # E: TypeVar default must be one of the constraint types + +type TB1[**P = int] = Callable[P, None] # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +type TB2[**P = 2] = Callable[P, None] # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +type TB3[**P = (2, int)] = Callable[P, None] # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +type TB4[**P = [2, int]] = Callable[P, None] # E: Argument 0 of ParamSpec default must be a type + +type TC1[*Ts = 2] = tuple[*Ts] # E: The default argument to TypeVarTuple must be an Unpacked tuple +type TC2[*Ts = int] = tuple[*Ts] # E: The default argument to TypeVarTuple must be an Unpacked tuple +type TC3[*Ts = tuple[int]] = tuple[*Ts] # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeParameterDefaultFunctions] +from typing import Callable + +def callback1(x: str) -> None: ... + +def func_a1[T = str](x: int | T) -> T: ... +reveal_type(func_a1(2)) # N: Revealed type is "builtins.str" +reveal_type(func_a1(2.1)) # N: Revealed type is "builtins.float" + +def func_a2[T = str](x: int | T) -> list[T]: ... +reveal_type(func_a2(2)) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type(func_a2(2.1)) # N: Revealed type is "builtins.list[builtins.float]" + + +def func_a3[T: str = str](x: int | T) -> T: ... +reveal_type(func_a3(2)) # N: Revealed type is "builtins.str" + +def func_a4[T: (bytes, str) = str](x: int | T) -> T: ... +reveal_type(func_a4(2)) # N: Revealed type is "builtins.str" + +def func_b1[**P = [int, str]](x: int | Callable[P, None]) -> Callable[P, None]: ... +reveal_type(func_b1(callback1)) # N: Revealed type is "def (x: builtins.str)" +reveal_type(func_b1(2)) # N: Revealed type is "def (builtins.int, builtins.str)" + +def func_c1[*Ts = *tuple[int, str]](x: int | Callable[[*Ts], None]) -> tuple[*Ts]: ... +# reveal_type(func_c1(callback1)) # Revealed type is "Tuple[str]" # TODO +reveal_type(func_c1(2)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultClass1] +# flags: --disallow-any-generics + +class ClassA1[T2 = int, T3 = str]: ... + +def func_a1( + a: ClassA1, + b: ClassA1[float], + c: ClassA1[float, float], + d: ClassA1[float, float, float], # E: "ClassA1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultClass2] +# flags: --disallow-any-generics + +class ClassB1[**P2 = [int, str], **P3 = ...]: ... + +def func_b1( + a: ClassB1, + b: ClassB1[[float]], + c: ClassB1[[float], [float]], + d: ClassB1[[float], [float], [float]], # E: "ClassB1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], ...]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + + k = ClassB1() + reveal_type(k) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + l = ClassB1[[float]]() + reveal_type(l) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + m = ClassB1[[float], [float]]() + reveal_type(m) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + n = ClassB1[[float], [float], [float]]() # E: Type application has too many types (expected between 0 and 2) + reveal_type(n) # N: Revealed type is "Any" + +[case testPEP695TypeParameterDefaultClass3] +# flags: --disallow-any-generics + +class ClassC1[*Ts = *tuple[int, str]]: ... + +def func_c1( + a: ClassC1, + b: ClassC1[float], +) -> None: + # reveal_type(a) # Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "__main__.ClassC1[builtins.float]" + + k = ClassC1() + reveal_type(k) # N: Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" + l = ClassC1[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassC1[builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultTypeAlias1] +# flags: --disallow-any-generics + +type TA1[T2 = int, T3 = str] = dict[T2, T3] + +def func_a1( + a: TA1, + b: TA1[float], + c: TA1[float, float], + d: TA1[float, float, float], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeParameterDefaultTypeAlias2] +# flags: --disallow-any-generics + +class ClassB1[**P2, **P3]: ... +type TB1[**P2 = [int, str], **P3 = ...] = ClassB1[P2, P3] + +def func_b1( + a: TB1, + b: TB1[[float]], + c: TB1[[float], [float]], + d: TB1[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeParameterDefaultTypeAlias3] +# flags: --disallow-any-generics + +type TC1[*Ts = *tuple[int, str]] = tuple[*Ts] + +def func_c1( + a: TC1, + b: TC1[float], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "Tuple[builtins.float]" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-python38.test b/test-data/unit/check-python38.test new file mode 100644 index 000000000000..199014a66fed --- /dev/null +++ b/test-data/unit/check-python38.test @@ -0,0 +1,805 @@ +[case testDecoratedClassLine] +def d(c): ... +@d + +class C: ... +class C: ... # E: Name "C" already defined on line 4 + +[case testDecoratedFunctionLine] +# flags: --disallow-untyped-defs +def d(f): ... # type: ignore +@d + +def f(): ... # E: Function is missing a return type annotation \ + # N: Use "-> None" if function does not return a value + +[case testIgnoreDecoratedFunction1] +# flags: --disallow-untyped-defs --warn-unused-ignores +def d(f): ... # type: ignore +@d +# type: ignore +def f(): ... # type: ignore # E: Unused "type: ignore" comment + +[case testIgnoreDecoratedFunction2] +# flags: --disallow-untyped-defs +def d(f): ... # type: ignore +@d + +def f(): ... # type: ignore + +[case testIgnoreScopeIssue1032] +def f(a: int): ... +f( + 1, + 2, +) # type: ignore + +[case testIgnoreScopeNested1] +def g(x: str) -> str: ... +def f(a: int) -> int: ... +f( + f( + g( + "IGNORE" + ) # type: ignore + ) +) + +[case testIgnoreScopeNested2] +[ + "IGNORE" # type: ignore + & + "IGNORE", +] +[builtins fixtures/list.pyi] + +[case testIgnoreScopeNested3] +{ + "IGNORE" + | # type: ignore + "IGNORE", +} +[builtins fixtures/set.pyi] + +[case testIgnoreScopeNested4] +{ + None: "IGNORE" + ^ + "IGNORE", # type: ignore +} +[builtins fixtures/dict.pyi] + +[case testIgnoreScopeNestedNonOverlapping] +def f(x: int): ... +def g(x: int): ... +( + f("ERROR"), # E: Argument 1 to "f" has incompatible type "str"; expected "int" + g("IGNORE"), # type: ignore + f("ERROR"), # E: Argument 1 to "f" has incompatible type "str"; expected "int" +) +[builtins fixtures/tuple.pyi] + +[case testIgnoreScopeNestedOverlapping] +def f(x: int): ... +def g(x: int): ... +( + f("ERROR"), g( # E: Argument 1 to "f" has incompatible type "str"; expected "int" + "IGNORE" # type: ignore + ), f("ERROR"), # E: Argument 1 to "f" has incompatible type "str"; expected "int" +) +[builtins fixtures/tuple.pyi] + +[case testIgnoreScopeUnused1] +# flags: --warn-unused-ignores +( # type: ignore # E: Unused "type: ignore" comment + "IGNORE" # type: ignore # E: Unused "type: ignore" comment + + # type: ignore # E: Unused "type: ignore" comment + 0 # type: ignore +) # type: ignore # E: Unused "type: ignore" comment +[builtins fixtures/primitives.pyi] + +[case testIgnoreScopeUnused2] +# flags: --warn-unused-ignores +( # type: ignore # E: Unused "type: ignore" comment + "IGNORE" + - # type: ignore + 0 # type: ignore # E: Unused "type: ignore" comment +) # type: ignore # E: Unused "type: ignore" comment + +[case testIgnoreScopeUnused3] +# flags: --warn-unused-ignores +( # type: ignore # E: Unused "type: ignore" comment + "IGNORE" + / + 0 # type: ignore +) # type: ignore # E: Unused "type: ignore" comment + +[case testPEP570ArgTypesMissing] +# flags: --disallow-untyped-defs +def f(arg, /) -> None: ... # E: Function is missing a type annotation for one or more arguments + +[case testPEP570ArgTypesBadDefault] +def f(arg: int = "ERROR", /) -> None: ... # E: Incompatible default for argument "arg" (default has type "str", argument has type "int") + +[case testPEP570ArgTypesDefault] +def f(arg: int = 0, /) -> None: + reveal_type(arg) # N: Revealed type is "builtins.int" + +[case testPEP570ArgTypesRequired] +def f(arg: int, /) -> None: + reveal_type(arg) # N: Revealed type is "builtins.int" + +[case testPEP570Required] +def f(arg: int, /) -> None: ... # N: "f" defined here +f(1) +f("ERROR") # E: Argument 1 to "f" has incompatible type "str"; expected "int" +f(arg=1) # E: Unexpected keyword argument "arg" for "f" +f(arg="ERROR") # E: Unexpected keyword argument "arg" for "f" + +[case testPEP570Default] +def f(arg: int = 0, /) -> None: ... # N: "f" defined here +f() +f(1) +f("ERROR") # E: Argument 1 to "f" has incompatible type "str"; expected "int" +f(arg=1) # E: Unexpected keyword argument "arg" for "f" +f(arg="ERROR") # E: Unexpected keyword argument "arg" for "f" + +[case testPEP570Calls] +# flags: --no-strict-optional +from typing import Any, Dict +def f(p, /, p_or_kw, *, kw) -> None: ... # N: "f" defined here +d = None # type: Dict[Any, Any] +f(0, 0, 0) # E: Too many positional arguments for "f" +f(0, 0, kw=0) +f(0, p_or_kw=0, kw=0) +f(p=0, p_or_kw=0, kw=0) # E: Unexpected keyword argument "p" for "f" +f(0, **d) +f(**d) # E: Missing positional argument "p_or_kw" in call to "f" +[builtins fixtures/dict.pyi] + +[case testPEP570Signatures1] +def f(p1: bytes, p2: float, /, p_or_kw: int, *, kw: str) -> None: + reveal_type(p1) # N: Revealed type is "builtins.bytes" + reveal_type(p2) # N: Revealed type is "builtins.float" + reveal_type(p_or_kw) # N: Revealed type is "builtins.int" + reveal_type(kw) # N: Revealed type is "builtins.str" + +[case testPEP570Signatures2] +def f(p1: bytes, p2: float = 0.0, /, p_or_kw: int = 0, *, kw: str) -> None: + reveal_type(p1) # N: Revealed type is "builtins.bytes" + reveal_type(p2) # N: Revealed type is "builtins.float" + reveal_type(p_or_kw) # N: Revealed type is "builtins.int" + reveal_type(kw) # N: Revealed type is "builtins.str" + +[case testPEP570Signatures3] +def f(p1: bytes, p2: float = 0.0, /, *, kw: int) -> None: + reveal_type(p1) # N: Revealed type is "builtins.bytes" + reveal_type(p2) # N: Revealed type is "builtins.float" + reveal_type(kw) # N: Revealed type is "builtins.int" + +[case testPEP570Signatures4] +def f(p1: bytes, p2: int = 0, /) -> None: + reveal_type(p1) # N: Revealed type is "builtins.bytes" + reveal_type(p2) # N: Revealed type is "builtins.int" + +[case testPEP570Signatures5] +def f(p1: bytes, p2: float, /, p_or_kw: int) -> None: + reveal_type(p1) # N: Revealed type is "builtins.bytes" + reveal_type(p2) # N: Revealed type is "builtins.float" + reveal_type(p_or_kw) # N: Revealed type is "builtins.int" + +[case testPEP570Signatures6] +def f(p1: bytes, p2: float, /) -> None: + reveal_type(p1) # N: Revealed type is "builtins.bytes" + reveal_type(p2) # N: Revealed type is "builtins.float" + +[case testPEP570Unannotated] +def f(arg, /): ... # N: "f" defined here +g = lambda arg, /: arg +def h(arg=0, /): ... # N: "h" defined here +i = lambda arg=0, /: arg + +f(1) +g(1) +h() +h(1) +i() +i(1) +f(arg=0) # E: Unexpected keyword argument "arg" for "f" +g(arg=0) # E: Unexpected keyword argument "arg" +h(arg=0) # E: Unexpected keyword argument "arg" for "h" +i(arg=0) # E: Unexpected keyword argument "arg" + +[case testWalrus] +from typing import NamedTuple, Optional, List +from typing_extensions import Final + +if a := 2: + reveal_type(a) # N: Revealed type is "builtins.int" + +while b := "x": + reveal_type(b) # N: Revealed type is "builtins.str" + +l = [y2 := 1, y2 + 2, y2 + 3] +reveal_type(y2) # N: Revealed type is "builtins.int" +reveal_type(l) # N: Revealed type is "builtins.list[builtins.int]" + +filtered_data = [y3 for x in l if (y3 := a) is not None] +reveal_type(filtered_data) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(y3) # N: Revealed type is "builtins.int" + +d = {'a': (a2 := 1), 'b': a2 + 1, 'c': a2 + 2} +reveal_type(d) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(a2) # N: Revealed type is "builtins.int" + +d2 = {(prefix := 'key_') + 'a': (start_val := 1), prefix + 'b': start_val + 1, prefix + 'c': start_val + 2} +reveal_type(d2) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(prefix) # N: Revealed type is "builtins.str" +reveal_type(start_val) # N: Revealed type is "builtins.int" + +filtered_dict = {k: new_v for k, v in [('a', 1), ('b', 2), ('c', 3)] if (new_v := v + 1) == 2} +reveal_type(filtered_dict) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(new_v) # N: Revealed type is "builtins.int" + +def f(x: int = (c := 4)) -> int: + if a := 2: + reveal_type(a) # N: Revealed type is "builtins.int" + + while b := "x": + reveal_type(b) # N: Revealed type is "builtins.str" + + x = (y := 1) + (z := 2) + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(y) # N: Revealed type is "builtins.int" + reveal_type(z) # N: Revealed type is "builtins.int" + + l = [y2 := 1, y2 + 2, y2 + 3] + reveal_type(y2) # N: Revealed type is "builtins.int" + reveal_type(l) # N: Revealed type is "builtins.list[builtins.int]" + + filtered_data = [y3 for x in l if (y3 := a) is not None] + reveal_type(filtered_data) # N: Revealed type is "builtins.list[builtins.int]" + reveal_type(y3) # N: Revealed type is "builtins.int" + + d = {'a': (a2 := 1), 'b': a2 + 1, 'c': a2 + 2} + reveal_type(d) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + reveal_type(a2) # N: Revealed type is "builtins.int" + + d2 = {(prefix := 'key_') + 'a': (start_val := 1), prefix + 'b': start_val + 1, prefix + 'c': start_val + 2} + reveal_type(d2) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + reveal_type(prefix) # N: Revealed type is "builtins.str" + reveal_type(start_val) # N: Revealed type is "builtins.int" + + filtered_dict = {k: new_v for k, v in [('a', 1), ('b', 2), ('c', 3)] if (new_v := v + 1) == 2} + reveal_type(filtered_dict) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + reveal_type(new_v) # N: Revealed type is "builtins.int" + + # https://www.python.org/dev/peps/pep-0572/#exceptional-cases + (y4 := 3) + reveal_type(y4) # N: Revealed type is "builtins.int" + + y5 = (y6 := 3) + reveal_type(y5) # N: Revealed type is "builtins.int" + reveal_type(y6) # N: Revealed type is "builtins.int" + + f(x=(y7 := 3)) + reveal_type(y7) # N: Revealed type is "builtins.int" + + reveal_type((lambda: (y8 := 3) and y8)()) # N: Revealed type is "builtins.int" + y8 # E: Name "y8" is not defined + + y7 = 1.0 # E: Incompatible types in assignment (expression has type "float", variable has type "int") + if y7 := "x": # E: Incompatible types in assignment (expression has type "str", variable has type "int") + pass + + # Just make sure we don't crash on this sort of thing. + if NT := NamedTuple("NT", [("x", int)]): # E: "int" not callable + z2: NT # E: Variable "NT" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + + if Alias := int: # E: Function "Alias" could always be true in boolean context + z3: Alias # E: Variable "Alias" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + + if (reveal_type(y9 := 3) and # N: Revealed type is "Literal[3]?" + reveal_type(y9)): # N: Revealed type is "builtins.int" + reveal_type(y9) # N: Revealed type is "builtins.int" + + return (y10 := 3) + y10 + +reveal_type(c) # N: Revealed type is "builtins.int" + +def check_final() -> None: + x: Final = 3 + + if x := 4: # E: Cannot assign to final name "x" + pass + +def check_binder(x: Optional[int], y: Optional[int], z: Optional[int], a: Optional[int], + b: Optional[int]) -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + + (x := 1) + reveal_type(x) # N: Revealed type is "builtins.int" + + if x or (y := 1): + reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" + + if x and (y := 1): + reveal_type(y) # N: Revealed type is "builtins.int" + + if (a := 1) and x: + reveal_type(a) # N: Revealed type is "builtins.int" + + if (b := 1) or x: + reveal_type(b) # N: Revealed type is "builtins.int" + + if z := 1: + reveal_type(z) # N: Revealed type is "builtins.int" + +def check_partial() -> None: + x = None + if bool() and (x := 2): + pass + + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + +def check_narrow(x: Optional[int], s: List[int]) -> None: + if (y := x): + reveal_type(y) # N: Revealed type is "builtins.int" + + if (y := x) is not None: + reveal_type(y) # N: Revealed type is "builtins.int" + + if (y := x) == 10: + reveal_type(y) # N: Revealed type is "builtins.int" + + if (y := x) in s: + reveal_type(y) # N: Revealed type is "builtins.int" + + if isinstance((y := x), int): + reveal_type(y) # N: Revealed type is "builtins.int" + +class AssignmentExpressionsClass: + x = (y := 1) + (z := 2) + reveal_type(z) # N: Revealed type is "builtins.int" + + l = [x2 := 1, 2, 3] + reveal_type(x2) # N: Revealed type is "builtins.int" + + def __init__(self) -> None: + reveal_type(self.z) # N: Revealed type is "builtins.int" + + l = [z2 := 1, z2 + 2, z2 + 3] + reveal_type(z2) # N: Revealed type is "builtins.int" + reveal_type(l) # N: Revealed type is "builtins.list[builtins.int]" + + filtered_data = [z3 for x in l if (z3 := 1) is not None] + reveal_type(filtered_data) # N: Revealed type is "builtins.list[builtins.int]" + reveal_type(z3) # N: Revealed type is "builtins.int" + +# Assignment expressions from inside the class should not escape the class scope. +reveal_type(x2) # E: Name "x2" is not defined # N: Revealed type is "Any" +reveal_type(z2) # E: Name "z2" is not defined # N: Revealed type is "Any" + +[builtins fixtures/isinstancelist.pyi] + +[case testWalrusConditionalTypeBinder] +from typing import Tuple, Union +from typing_extensions import Literal + +class Good: + @property + def is_good(self) -> Literal[True]: ... + +class Bad: + @property + def is_good(self) -> Literal[False]: ... + +def get_thing() -> Union[Good, Bad]: ... + +if (thing := get_thing()).is_good: + reveal_type(thing) # N: Revealed type is "__main__.Good" +else: + reveal_type(thing) # N: Revealed type is "__main__.Bad" + +def get_things() -> Union[Tuple[Good], Tuple[Bad]]: ... + +if (things := get_things())[0].is_good: + reveal_type(things) # N: Revealed type is "Tuple[__main__.Good]" +else: + reveal_type(things) # N: Revealed type is "Tuple[__main__.Bad]" +[builtins fixtures/list.pyi] + +[case testWalrusConditionalTypeCheck] +from typing import Optional + +maybe_str: Optional[str] + +if (is_str := maybe_str is not None): + reveal_type(is_str) # N: Revealed type is "Literal[True]" + reveal_type(maybe_str) # N: Revealed type is "builtins.str" +else: + reveal_type(is_str) # N: Revealed type is "Literal[False]" + reveal_type(maybe_str) # N: Revealed type is "None" + +reveal_type(maybe_str) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/bool.pyi] + +[case testWalrusConditionalTypeCheck2] +from typing import Optional + +maybe_str: Optional[str] + +if (x := maybe_str) is not None: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(maybe_str) # N: Revealed type is "Union[builtins.str, None]" +else: + reveal_type(x) # N: Revealed type is "None" + reveal_type(maybe_str) # N: Revealed type is "Union[builtins.str, None]" + +reveal_type(maybe_str) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/bool.pyi] + +[case testWalrusPartialTypes] +from typing import List + +def check_partial_list() -> None: + if (x := []): # E: Need type annotation for "x" (hint: "x: List[] = ...") + pass + + y: List[str] + if (y := []): + pass + + if (z := []): + z.append(3) + reveal_type(z) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] + +[case testWalrusAssignmentAndConditionScopeForLiteral] +# flags: --warn-unreachable + +if (x := 0): + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is "Literal[0]" + +reveal_type(x) # N: Revealed type is "Literal[0]" + +[case testWalrusAssignmentAndConditionScopeForProperty] +# flags: --warn-unreachable + +from typing_extensions import Literal + +class PropertyWrapper: + @property + def f(self) -> str: ... + @property + def always_false(self) -> Literal[False]: ... + +wrapper = PropertyWrapper() + +if x := wrapper.f: + reveal_type(x) # N: Revealed type is "builtins.str" +else: + reveal_type(x) # N: Revealed type is "Literal['']" + +reveal_type(x) # N: Revealed type is "builtins.str" + +if y := wrapper.always_false: + reveal_type(y) # E: Statement is unreachable +else: + reveal_type(y) # N: Revealed type is "Literal[False]" + +reveal_type(y) # N: Revealed type is "Literal[False]" +[builtins fixtures/property.pyi] + +[case testWalrusAssignmentAndConditionScopeForFunction] +# flags: --warn-unreachable + +from typing_extensions import Literal + +def f() -> str: ... + +if x := f(): + reveal_type(x) # N: Revealed type is "builtins.str" +else: + reveal_type(x) # N: Revealed type is "Literal['']" + +reveal_type(x) # N: Revealed type is "builtins.str" + +def always_false() -> Literal[False]: ... + +if y := always_false(): + reveal_type(y) # E: Statement is unreachable +else: + reveal_type(y) # N: Revealed type is "Literal[False]" + +reveal_type(y) # N: Revealed type is "Literal[False]" + +def always_false_with_parameter(x: int) -> Literal[False]: ... + +if z := always_false_with_parameter(5): + reveal_type(z) # E: Statement is unreachable +else: + reveal_type(z) # N: Revealed type is "Literal[False]" + +reveal_type(z) # N: Revealed type is "Literal[False]" +[builtins fixtures/tuple.pyi] + +[case testWalrusExpr] +def func() -> None: + foo = Foo() + if x := foo.x: + pass + +class Foo: + def __init__(self) -> None: + self.x = 123 + +[case testWalrusTypeGuard] +from typing_extensions import TypeGuard +def is_float(a: object) -> TypeGuard[float]: pass +def main(a: object) -> None: + if is_float(x := a): + reveal_type(x) # N: Revealed type is "builtins.float" + reveal_type(a) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] + +[case testWalrusRedefined] +def foo() -> None: + x = 0 + [x := x + y for y in [1, 2, 3]] +[builtins fixtures/dict.pyi] + +[case testWalrusUsedBeforeDef] +class C: + def f(self, c: 'C') -> None: pass + +(x := C()).f(y) # E: Cannot determine type of "y" # E: Name "y" is used before definition +(y := C()).f(y) + +[case testOverloadWithPositionalOnlySelf] +from typing import overload, Optional + +class Foo: + @overload + def f(self, a: str, /) -> None: ... + + @overload + def f(self, *, b: bool = False) -> None: ... + + def f(self, a: Optional[str] = None, /, *, b: bool = False) -> None: # E: Overloaded function implementation does not accept all possible arguments of signature 2 + ... + +class Bar: + @overload + def f(self, a: str, /) -> None: ... + + @overload # Notice `/` in sig below: + def f(self, /, *, b: bool = False) -> None: ... + + def f(self, a: Optional[str] = None, /, *, b: bool = False) -> None: + ... +[builtins fixtures/bool.pyi] + +[case testOverloadPositionalOnlyErrorMessage] +from typing import overload + +@overload +def foo(a: int, /): ... +@overload +def foo(a: str): ... +def foo(a): ... + +foo(a=1) +[out] +main:9: error: No overload variant of "foo" matches argument type "int" +main:9: note: Possible overload variants: +main:9: note: def foo(int, /) -> Any +main:9: note: def foo(a: str) -> Any + +[case testOverloadPositionalOnlyErrorMessageAllTypes] +from typing import overload + +@overload +def foo(a: int, /, b: int, *, c: int): ... +@overload +def foo(a: str, b: int, *, c: int): ... +def foo(a, b, *, c): ... + +foo(a=1) +[out] +main:9: error: No overload variant of "foo" matches argument type "int" +main:9: note: Possible overload variants: +main:9: note: def foo(int, /, b: int, *, c: int) -> Any +main:9: note: def foo(a: str, b: int, *, c: int) -> Any + +[case testOverloadPositionalOnlyErrorMessageMultiplePosArgs] +from typing import overload + +@overload +def foo(a: int, b: int, c: int, /, d: str): ... +@overload +def foo(a: str, b: int, c: int, d: str): ... +def foo(a, b, c, d): ... + +foo(a=1) +[out] +main:9: error: No overload variant of "foo" matches argument type "int" +main:9: note: Possible overload variants: +main:9: note: def foo(int, int, int, /, d: str) -> Any +main:9: note: def foo(a: str, b: int, c: int, d: str) -> Any + +[case testOverloadPositionalOnlyErrorMessageMethod] +from typing import overload + +class Some: + @overload + def foo(self, __a: int): ... + @overload + def foo(self, a: float, /): ... + @overload + def foo(self, a: str): ... + def foo(self, a): ... + +Some().foo(a=1) +[out] +main:12: error: No overload variant of "foo" of "Some" matches argument type "int" +main:12: note: Possible overload variants: +main:12: note: def foo(self, int, /) -> Any +main:12: note: def foo(self, float, /) -> Any +main:12: note: def foo(self, a: str) -> Any + +[case testOverloadPositionalOnlyErrorMessageClassMethod] +from typing import overload + +class Some: + @overload + @classmethod + def foo(cls, __a: int): ... + @overload + @classmethod + def foo(cls, a: float, /): ... + @overload + @classmethod + def foo(cls, a: str): ... + @classmethod + def foo(cls, a): ... + +Some.foo(a=1) +[builtins fixtures/classmethod.pyi] +[out] +main:16: error: No overload variant of "foo" of "Some" matches argument type "int" +main:16: note: Possible overload variants: +main:16: note: def foo(cls, int, /) -> Any +main:16: note: def foo(cls, float, /) -> Any +main:16: note: def foo(cls, a: str) -> Any + +[case testUnpackWithDuplicateNamePositionalOnly] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int +def foo(name: str, /, **kwargs: Unpack[Person]) -> None: # Allowed + ... +[builtins fixtures/dict.pyi] + +[case testPossiblyUndefinedWithAssignmentExpr] +# flags: --enable-error-code possibly-undefined +def f1() -> None: + d = {0: 1} + if int(): + x = 1 + + if (x := d[x]) is None: # E: Name "x" may be undefined + y = x + z = x +[builtins fixtures/dict.pyi] + +[case testNarrowOnSelfInGeneric] +from typing import Generic, TypeVar, Optional + +T = TypeVar("T", int, str) + +class C(Generic[T]): + x: Optional[T] + def meth(self) -> Optional[T]: + if (y := self.x) is not None: + reveal_type(y) + return None +[out] +main:9: note: Revealed type is "builtins.int" +main:9: note: Revealed type is "builtins.str" + +[case testTypeGuardWithPositionalOnlyArg] +from typing_extensions import TypeGuard + +def typeguard(x: object, /) -> TypeGuard[int]: + ... + +n: object +if typeguard(n): + reveal_type(n) +[builtins fixtures/tuple.pyi] +[out] +main:8: note: Revealed type is "builtins.int" + +[case testTypeGuardKeywordFollowingWalrus] +from typing import cast +from typing_extensions import TypeGuard + +def typeguard(x: object) -> TypeGuard[int]: + ... + +if typeguard(x=(n := cast(object, "hi"))): + reveal_type(n) +[builtins fixtures/tuple.pyi] +[out] +main:8: note: Revealed type is "builtins.int" + +[case testNoCrashOnAssignmentExprClass] +class C: + [(j := i) for i in [1, 2, 3]] # E: Assignment expression within a comprehension cannot be used in a class body +[builtins fixtures/list.pyi] + +[case testNarrowedVariableInNestedModifiedInWalrus] +from typing import Optional + +def walrus_with_nested_error(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x # E: Incompatible return value type (got "Optional[str]", expected "str") + if x := None: + pass + nested() + +def walrus_with_nested_ok(x: Optional[str]) -> None: + if x is None: + x = "a" + def nested() -> str: + return x + if y := x: + pass + nested() + +[case testIgnoreWholeModule] +# flags: --warn-unused-ignores +# type: ignore +IGNORE # type: ignore + +[case testUnusedIgnoreVersionCheck] +# flags: --warn-unused-ignores +import sys + +if sys.version_info < (3, 6): + 42 # type: ignore +else: + 42 # type: ignore # E: Unused "type: ignore" comment +[builtins fixtures/ops.pyi] + +[case testDictExpressionErrorLocations] +# flags: --pretty +from typing import Dict + +other: Dict[str, str] +dct: Dict[str, int] = {"a": "b", **other} +[builtins fixtures/dict.pyi] +[out] +main:5: error: Dict entry 0 has incompatible type "str": "str"; expected "str": "int" + dct: Dict[str, int] = {"a": "b", **other} + ^~~~~~~~ +main:5: error: Unpacked dict entry 1 has incompatible type "Dict[str, str]"; expected "SupportsKeysAndGetItem[str, int]" + dct: Dict[str, int] = {"a": "b", **other} + ^~~~~ + +[case testWalrusAssignmentEmptyCollection] +from typing import List + +y: List[int] +if (y := []): + reveal_type(y) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] diff --git a/test-data/unit/check-python39.test b/test-data/unit/check-python39.test new file mode 100644 index 000000000000..e17bf1e7ab5b --- /dev/null +++ b/test-data/unit/check-python39.test @@ -0,0 +1,27 @@ +[case testGivingSameKeywordArgumentTwice] +# This test was originally in check-kwargs.test +# Python 3.9's new parser started producing a different error message here. Since this isn't the +# most important test, to deal with this we'll only run this test with Python 3.9 and later. +import typing +def f(a: 'A', b: 'B') -> None: pass +class A: pass +class B: pass +f(a=A(), b=B(), a=A()) # E: "f" gets multiple values for keyword argument "a" + + +[case testPEP614] +from typing import Callable, List + +decorator_list: List[Callable[..., Callable[[int], str]]] +@decorator_list[0] +def f(x: float) -> float: ... +reveal_type(f) # N: Revealed type is "def (builtins.int) -> builtins.str" +[builtins fixtures/list.pyi] + +[case testStarredExpressionsInForLoop] +# flags: --python-version 3.9 + +a = b = c = [1, 2, 3] +for x in *a, *b, *c: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test new file mode 100644 index 000000000000..4d7af98204fb --- /dev/null +++ b/test-data/unit/check-recursive-types.test @@ -0,0 +1,1016 @@ +-- Tests checking that basic functionality works + +[case testRecursiveAliasBasic] +from typing import Dict, List, Union, TypeVar, Sequence + +JSON = Union[str, List[JSON], Dict[str, JSON]] + +x: JSON = ["foo", {"bar": "baz"}] + +reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.list[...], builtins.dict[builtins.str, ...]]" +if isinstance(x, list): + x = x[0] + +class Bad: ... +x = ["foo", {"bar": [Bad()]}] # E: List item 0 has incompatible type "Bad"; expected "Union[str, List[JSON], Dict[str, JSON]]" +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasBasicGenericSubtype] +from typing import Union, TypeVar, Sequence, List + +T = TypeVar("T") + +Nested = Sequence[Union[T, Nested[T]]] + +class Bad: ... +x: Nested[int] +y: Nested[Bad] +x = y # E: Incompatible types in assignment (expression has type "Nested[Bad]", variable has type "Nested[int]") + +NestedOther = Sequence[Union[T, Nested[T]]] + +xx: Nested[int] +yy: NestedOther[bool] +xx = yy # OK +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasBasicGenericInference] +from typing import Union, TypeVar, Sequence, List + +T = TypeVar("T") + +Nested = Sequence[Union[T, Nested[T]]] + +def flatten(arg: Nested[T]) -> List[T]: + res: List[T] = [] + for item in arg: + if isinstance(item, Sequence): + res.extend(flatten(item)) + else: + res.append(item) + return res + +reveal_type(flatten([1, [2, [3]]])) # N: Revealed type is "builtins.list[builtins.int]" + +class Bad: ... +x: Nested[int] = [1, [2, [3]]] +x = [1, [Bad()]] # E: List item 0 has incompatible type "Bad"; expected "Union[int, Nested[int]]" +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasGenericInferenceNested] +from typing import Union, TypeVar, Sequence, List + +T = TypeVar("T") +class A: ... +class B(A): ... + +Nested = Sequence[Union[T, Nested[T]]] + +def flatten(arg: Nested[T]) -> List[T]: ... +reveal_type(flatten([[B(), B()]])) # N: Revealed type is "builtins.list[__main__.B]" +reveal_type(flatten([[[[B()]]]])) # N: Revealed type is "builtins.list[__main__.B]" +reveal_type(flatten([[B(), [[B()]]]])) # N: Revealed type is "builtins.list[__main__.B]" +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasNewStyleSupported] +from test import A + +x: A +if isinstance(x, list): + reveal_type(x[0]) # N: Revealed type is "Union[builtins.int, builtins.list[Union[builtins.int, builtins.list[...]]]]" +else: + reveal_type(x) # N: Revealed type is "builtins.int" + +[file test.pyi] +A = int | list[A] +[builtins fixtures/isinstancelist.pyi] + +-- Tests duplicating some existing type alias tests with recursive aliases enabled + +[case testRecursiveAliasesMutual] +# flags: --disable-error-code used-before-def +from typing import Type, Callable, Union + +A = Union[B, int] +B = Callable[[C], int] +C = Type[A] +x: A +reveal_type(x) # N: Revealed type is "Union[def (Union[Type[def (...) -> builtins.int], Type[builtins.int]]) -> builtins.int, builtins.int]" + +[case testRecursiveAliasesProhibited-skip] +from typing import Type, Callable, Union + +A = Union[B, int] +B = Union[A, int] +C = Type[C] + +[case testRecursiveAliasImported] +import lib +x: lib.A +reveal_type(x) # N: Revealed type is "builtins.list[builtins.list[...]]" + +[file lib.pyi] +from typing import List +from other import B +A = List[B] + +[file other.pyi] +from typing import List +from lib import A +B = List[A] +[builtins fixtures/list.pyi] + +[case testRecursiveAliasViaBaseClass] +# flags: --disable-error-code used-before-def +from typing import List + +x: B +B = List[C] +class C(B): pass + +reveal_type(x) # N: Revealed type is "builtins.list[__main__.C]" +reveal_type(x[0][0]) # N: Revealed type is "__main__.C" +[builtins fixtures/list.pyi] + +[case testRecursiveAliasViaBaseClass2] +# flags: --disable-error-code used-before-def +from typing import NewType, List + +x: D +reveal_type(x[0][0]) # N: Revealed type is "__main__.C" + +D = List[C] +C = NewType('C', B) + +class B(D): + pass +[builtins fixtures/list.pyi] + +[case testRecursiveAliasViaBaseClass3] +from typing import List, Generic, TypeVar, NamedTuple +T = TypeVar('T') + +class C(A, B): + pass +class G(Generic[T]): pass +A = G[C] +class B(NamedTuple): + x: int + +y: C +reveal_type(y.x) # N: Revealed type is "builtins.int" +reveal_type(y[0]) # N: Revealed type is "builtins.int" +x: A +reveal_type(x) # N: Revealed type is "__main__.G[Tuple[builtins.int, fallback=__main__.C]]" +[builtins fixtures/list.pyi] + +[case testRecursiveAliasViaBaseClassImported] +# flags: --disable-error-code used-before-def +import a +[file a.py] +from typing import List +from b import D + +def f(x: B) -> List[B]: ... +B = List[C] +class C(B): pass + +[file b.py] +from a import f +class D: ... +reveal_type(f) # N: Revealed type is "def (x: builtins.list[a.C]) -> builtins.list[builtins.list[a.C]]" +[builtins fixtures/list.pyi] + +[case testRecursiveAliasViaNamedTuple] +from typing import List, NamedTuple, Union + +Exp = Union['A', 'B'] +class A(NamedTuple('A', [('attr', List[Exp])])): pass +class B(NamedTuple('B', [('val', object)])): pass + +def my_eval(exp: Exp) -> int: + reveal_type(exp) # N: Revealed type is "Union[Tuple[builtins.list[...], fallback=__main__.A], Tuple[builtins.object, fallback=__main__.B]]" + if isinstance(exp, A): + my_eval(exp[0][0]) + return my_eval(exp.attr[0]) + if isinstance(exp, B): + return exp.val # E: Incompatible return value type (got "object", expected "int") + return 0 + +my_eval(A([B(1), B(2)])) +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasesSimplifiedUnion] +from typing import Sequence, TypeVar, Union + +class A: ... +class B(A): ... + +NestedA = Sequence[Union[A, NestedA]] +NestedB = Sequence[Union[B, NestedB]] +a: NestedA +b: NestedB + +T = TypeVar("T") +S = TypeVar("S") +def union(a: T, b: S) -> Union[T, S]: ... + +x: int +y = union(a, b) +x = y # E: Incompatible types in assignment (expression has type "Sequence[Union[A, NestedA]]", variable has type "int") +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasesJoins] +from typing import Sequence, TypeVar, Union + +class A: ... +class B(A): ... + +NestedA = Sequence[Union[A, NestedA]] +NestedB = Sequence[Union[B, NestedB]] +a: NestedA +b: NestedB +la: Sequence[Sequence[A]] +lb: Sequence[Sequence[B]] + +T = TypeVar("T") +def join(a: T, b: T) -> T: ... +x: int + +y1 = join(a, b) +x = y1 # E: Incompatible types in assignment (expression has type "Sequence[Union[A, NestedA]]", variable has type "int") +y2 = join(a, lb) +x = y2 # E: Incompatible types in assignment (expression has type "Sequence[Union[A, NestedA]]", variable has type "int") +y3 = join(la, b) +x = y3 # E: Incompatible types in assignment (expression has type "Sequence[Union[Sequence[A], B, NestedB]]", variable has type "int") +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasesRestrictions] +from typing import Sequence, Mapping, Union + +A = Sequence[Union[int, A]] +B = Mapping[int, Union[int, B]] + +x: int +y: Union[A, B] +if isinstance(y, Sequence): + x = y # E: Incompatible types in assignment (expression has type "Sequence[Union[int, A]]", variable has type "int") +else: + x = y # E: Incompatible types in assignment (expression has type "Mapping[int, Union[int, B]]", variable has type "int") +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasesRestrictions2] +from typing import Sequence, Union + +class A: ... +class B(A): ... + +NestedA = Sequence[Union[A, NestedA]] +NestedB = Sequence[Union[B, NestedB]] + +a: NestedA +b: NestedB +aa: NestedA + +x: int +x = a # E: Incompatible types in assignment (expression has type "NestedA", variable has type "int") +a = b +x = a # E: Incompatible types in assignment (expression has type "Sequence[Union[B, NestedB]]", variable has type "int") +b = aa # E: Incompatible types in assignment (expression has type "NestedA", variable has type "NestedB") +if isinstance(b[0], Sequence): + a = b[0] + x = a # E: Incompatible types in assignment (expression has type "Sequence[Union[B, NestedB]]", variable has type "int") +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasWithRecursiveInstance] +from typing import Sequence, Union, TypeVar + +class A: ... +T = TypeVar("T") +Nested = Sequence[Union[T, Nested[T]]] +class B(Sequence[B]): ... + +a: Nested[A] +aa: Nested[A] +b: B +a = b # OK +a = [[b]] # OK +b = aa # E: Incompatible types in assignment (expression has type "Nested[A]", variable has type "B") + +def join(a: T, b: T) -> T: ... +reveal_type(join(a, b)) # N: Revealed type is "typing.Sequence[Union[__main__.A, typing.Sequence[Union[__main__.A, ...]]]]" +reveal_type(join(b, a)) # N: Revealed type is "typing.Sequence[Union[__main__.A, typing.Sequence[Union[__main__.A, ...]]]]" +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasWithRecursiveInstanceInference] +from typing import Sequence, Union, TypeVar, List + +T = TypeVar("T") +Nested = Sequence[Union[T, Nested[T]]] +class B(Sequence[B]): ... + +nb: Nested[B] = [B(), [B(), [B()]]] +lb: List[B] + +def foo(x: Nested[T]) -> T: ... +reveal_type(foo(lb)) # N: Revealed type is "__main__.B" +reveal_type(foo([B(), [B(), [B()]]])) # N: Revealed type is "__main__.B" + +NestedInv = List[Union[T, NestedInv[T]]] +nib: NestedInv[B] = [B(), [B(), [B()]]] +def bar(x: NestedInv[T]) -> T: ... +reveal_type(bar(nib)) # N: Revealed type is "__main__.B" +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasTopUnion] +from typing import Sequence, Union, TypeVar, List + +class A: ... +class B(A): ... + +T = TypeVar("T") +PlainNested = Union[T, Sequence[PlainNested[T]]] + +x: PlainNested[A] +y: PlainNested[B] = [B(), [B(), [B()]]] +x = y # OK + +xx: PlainNested[B] +yy: PlainNested[A] +xx = yy # E: Incompatible types in assignment (expression has type "PlainNested[A]", variable has type "PlainNested[B]") + +def foo(arg: PlainNested[T]) -> T: ... +lb: List[B] +reveal_type(foo([B(), [B(), [B()]]])) # N: Revealed type is "__main__.B" +reveal_type(foo(lb)) # N: Revealed type is "__main__.B" +reveal_type(foo(xx)) # N: Revealed type is "__main__.B" +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasInferenceExplicitNonRecursive] +from typing import Sequence, Union, TypeVar, List + +T = TypeVar("T") +Nested = Sequence[Union[T, Nested[T]]] +PlainNested = Union[T, Sequence[PlainNested[T]]] + +def foo(x: Nested[T]) -> T: ... +def bar(x: PlainNested[T]) -> T: ... + +class A: ... +a: A +la: List[A] +lla: List[Union[A, List[A]]] +llla: List[Union[A, List[Union[A, List[A]]]]] + +reveal_type(foo(la)) # N: Revealed type is "__main__.A" +reveal_type(foo(lla)) # N: Revealed type is "__main__.A" +reveal_type(foo(llla)) # N: Revealed type is "__main__.A" + +reveal_type(bar(a)) # N: Revealed type is "__main__.A" +reveal_type(bar(la)) # N: Revealed type is "__main__.A" +reveal_type(bar(lla)) # N: Revealed type is "__main__.A" +reveal_type(bar(llla)) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstancelist.pyi] + +[case testRecursiveAliasesWithOptional] +from typing import Optional, Sequence + +A = Sequence[Optional[A]] +x: A +y: str = x[0] # E: Incompatible types in assignment (expression has type "Optional[A]", variable has type "str") + +[case testRecursiveAliasesProhibitBadAliases] +# flags: --disable-error-code used-before-def +from typing import Union, Type, List, TypeVar + +NR = List[int] +NR2 = Union[NR, NR] +NR3 = Union[NR, Union[NR2, NR2]] + +T = TypeVar("T") +NRG = Union[int, T] +NR4 = NRG[str] +NR5 = Union[NRG[int], NR4] + +A = Union[B, int] # E: Invalid recursive alias: a union item of itself +B = Union[int, A] # Error reported above +def f() -> A: ... +reveal_type(f()) # N: Revealed type is "Any" + +G = Union[T, G[T]] # E: Invalid recursive alias: a union item of itself +GL = Union[T, GL[List[T]]] # E: Invalid recursive alias: a union item of itself \ + # E: Invalid recursive alias: type variable nesting on right hand side +def g() -> G[int]: ... +reveal_type(g()) # N: Revealed type is "Any" + +def local() -> None: + L = List[Union[int, L]] # E: Cannot resolve name "L" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + x: L + reveal_type(x) # N: Revealed type is "builtins.list[Union[builtins.int, Any]]" + +S = Type[S] # E: Type[...] can't contain another Type[...] +U = Type[Union[int, U]] # E: Type[...] can't contain another Type[...] +x: U +reveal_type(x) # N: Revealed type is "Type[Any]" + +D = List[F[List[T]]] # E: Invalid recursive alias: type variable nesting on right hand side +F = D[T] # Error reported above +E = List[E[E[T]]] # E: Invalid recursive alias: type variable nesting on right hand side +d: D +reveal_type(d) # N: Revealed type is "Any" +[builtins fixtures/isinstancelist.pyi] + +[case testBasicRecursiveNamedTuple] +from typing import NamedTuple, Optional + +NT = NamedTuple("NT", [("x", Optional[NT]), ("y", int)]) +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[Union[..., None], builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +reveal_type(nt[0]) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testBasicRecursiveNamedTupleSpecial] +from typing import NamedTuple, TypeVar, Tuple + +NT = NamedTuple("NT", [("x", NT), ("y", int)]) +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.NT]" +reveal_type(nt[0]) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallback=__main__.NT], builtins.int, fallback=__main__.NT]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "NT", variable has type "str") + +T = TypeVar("T") +def f(a: T, b: T) -> T: ... +tnt: Tuple[NT] + +# TODO: these should be tuple[object] instead. +reveal_type(f(nt, tnt)) # N: Revealed type is "builtins.tuple[Any, ...]" +reveal_type(f(tnt, nt)) # N: Revealed type is "builtins.tuple[Any, ...]" +[builtins fixtures/tuple.pyi] + +[case testBasicRecursiveNamedTupleClass] +from typing import NamedTuple, Optional + +class NT(NamedTuple): + x: Optional[NT] + y: int + +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[Union[..., None], builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +reveal_type(nt[0]) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testRecursiveRegularTupleClass] +from typing import Tuple + +x: B +class B(Tuple[B, int]): + x: int + +b, _ = x +reveal_type(b.x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleClassesNewType] +from typing import Tuple, NamedTuple, NewType + +x: C +class B(Tuple[B, int]): + x: int +C = NewType("C", B) +b, _ = x +reveal_type(b) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.B]" +reveal_type(b.x) # N: Revealed type is "builtins.int" + +y: CNT +class BNT(NamedTuple): + x: CNT + y: int +CNT = NewType("CNT", BNT) +bnt, _ = y +reveal_type(bnt.y) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +-- Tests duplicating some existing named tuple tests with recursive aliases enabled + +[case testMutuallyRecursiveNamedTuples] +# flags: --disable-error-code used-before-def + +from typing import Tuple, NamedTuple, TypeVar, Union + +A = NamedTuple('A', [('x', str), ('y', Tuple[B, ...])]) +class B(NamedTuple): + x: A + y: int + +n: A +reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Tuple[..., builtins.int, fallback=__main__.B], ...], fallback=__main__.A]" + +T = TypeVar("T") +S = TypeVar("S") +def foo(arg: Tuple[T, S]) -> Union[T, S]: ... +x = foo(n) +y: str = x # E: Incompatible types in assignment (expression has type "Union[str, Tuple[B, ...]]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesJoin] +from typing import NamedTuple, Tuple + +class B(NamedTuple): + x: Tuple[A, int] + y: int + +A = NamedTuple('A', [('x', str), ('y', B)]) +n: B +m: A +s: str = n.x # E: Incompatible types in assignment (expression has type "Tuple[A, int]", variable has type "str") +reveal_type(m[0]) # N: Revealed type is "builtins.str" +lst = [m, n] + +# Unfortunately, join of two recursive types is not very precise. +reveal_type(lst[0]) # N: Revealed type is "builtins.object" + +# These just should not crash +lst1 = [m] +lst2 = [m, m] +lst3 = [m, m, m] +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesClasses] +from typing import NamedTuple, Tuple + +class B(NamedTuple): + x: A + y: int +class A(NamedTuple): + x: str + y: B + +n: A +s: str = n.y[0] # E: Incompatible types in assignment (expression has type "A", variable has type "str") + +m: B +n = m.x +n = n.y.x + +t: Tuple[str, B] +t = n +t = m # E: Incompatible types in assignment (expression has type "B", variable has type "Tuple[str, B]") +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesCalls] +# flags: --disable-error-code used-before-def +from typing import NamedTuple + +B = NamedTuple('B', [('x', A), ('y', int)]) +A = NamedTuple('A', [('x', str), ('y', 'B')]) +n: A +def f(m: B) -> None: pass +reveal_type(n) # N: Revealed type is "Tuple[builtins.str, Tuple[..., builtins.int, fallback=__main__.B], fallback=__main__.A]" +reveal_type(f) # N: Revealed type is "def (m: Tuple[Tuple[builtins.str, ..., fallback=__main__.A], builtins.int, fallback=__main__.B])" +f(n) # E: Argument 1 to "f" has incompatible type "A"; expected "B" +[builtins fixtures/tuple.pyi] + +[case testNoRecursiveTuplesAtFunctionScope] +from typing import NamedTuple, Tuple +def foo() -> None: + class B(NamedTuple): + x: B # E: Cannot resolve name "B" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + y: int + b: B + reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@3]" +[builtins fixtures/tuple.pyi] + +[case testBasicRecursiveGenericNamedTuple] +from typing import Generic, NamedTuple, TypeVar, Union + +T = TypeVar("T", covariant=True) +class NT(NamedTuple, Generic[T]): + key: int + value: Union[T, NT[T]] + +class A: ... +class B(A): ... + +nti: NT[int] = NT(key=0, value=NT(key=1, value=A())) # E: Argument "value" to "NT" has incompatible type "A"; expected "Union[int, NT[int]]" +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, Union[builtins.int, ...], fallback=__main__.NT[builtins.int]]" + +nta: NT[A] +ntb: NT[B] +nta = ntb # OK, covariance +ntb = nti # E: Incompatible types in assignment (expression has type "NT[int]", variable has type "NT[B]") + +def last(arg: NT[T]) -> T: ... +reveal_type(last(ntb)) # N: Revealed type is "__main__.B" +[builtins fixtures/tuple.pyi] + +[case testBasicRecursiveTypedDictClass] +from typing import TypedDict + +class TD(TypedDict): + x: int + y: TD + +td: TD +reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.int, 'y': ...})" +s: str = td["y"] # E: Incompatible types in assignment (expression has type "TD", variable has type "str") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testBasicRecursiveTypedDictCall] +from typing import TypedDict + +TD = TypedDict("TD", {"x": int, "y": TD}) +td: TD +reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.int, 'y': ...})" + +TD2 = TypedDict("TD2", {"x": int, "y": TD2}) +td2: TD2 +TD3 = TypedDict("TD3", {"x": str, "y": TD3}) +td3: TD3 + +td = td2 +td = td3 # E: Incompatible types in assignment (expression has type "TD3", variable has type "TD") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testBasicRecursiveTypedDictExtending] +from typing import TypedDict + +class TDA(TypedDict): + xa: int + ya: TD + +class TDB(TypedDict): + xb: int + yb: TD + +class TD(TDA, TDB): + a: TDA + b: TDB + +td: TD +reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'xb': builtins.int, 'yb': ..., 'xa': builtins.int, 'ya': ..., 'a': TypedDict('__main__.TDA', {'xa': builtins.int, 'ya': ...}), 'b': TypedDict('__main__.TDB', {'xb': builtins.int, 'yb': ...})})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictCreation] +from typing import TypedDict, Optional + +class TD(TypedDict): + x: int + y: Optional[TD] + +td: TD = {"x": 0, "y": None} +td2: TD = {"x": 0, "y": {"x": 1, "y": {"x": 2, "y": None}}} + +itd = TD(x=0, y=None) +itd2 = TD(x=0, y=TD(x=0, y=TD(x=0, y=None))) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictMethods] +from typing import TypedDict + +class TD(TypedDict, total=False): + x: int + y: TD + +td: TD +td["y"] = {"x": 0, "y": {}} +td["y"] = {"x": 0, "y": {"x": 0, "y": 42}} # E: Incompatible types (expression has type "int", TypedDict item "y" has type "TD") + +reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...})}), None]" +s: str = td.get("y") # E: Incompatible types in assignment (expression has type "Optional[TD]", variable has type "str") + +td.update({"x": 0, "y": {"x": 1, "y": {}}}) +td.update({"x": 0, "y": {"x": 1, "y": {"x": 2, "y": 42}}}) # E: Incompatible types (expression has type "int", TypedDict item "y" has type "TD") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictSubtyping] +from typing import TypedDict + +class TDA1(TypedDict): + x: int + y: TDA1 +class TDA2(TypedDict): + x: int + y: TDA2 +class TDB(TypedDict): + x: str + y: TDB + +tda1: TDA1 +tda2: TDA2 +tdb: TDB +def fa1(arg: TDA1) -> None: ... +def fa2(arg: TDA2) -> None: ... +def fb(arg: TDB) -> None: ... + +fa1(tda2) +fa2(tda1) +fb(tda1) # E: Argument 1 to "fb" has incompatible type "TDA1"; expected "TDB" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictJoin] +from typing import TypedDict, TypeVar + +class TDA1(TypedDict): + x: int + y: TDA1 +class TDA2(TypedDict): + x: int + y: TDA2 +class TDB(TypedDict): + x: str + y: TDB + +tda1: TDA1 +tda2: TDA2 +tdb: TDB + +T = TypeVar("T") +def f(x: T, y: T) -> T: ... +# Join for recursive types is very basic, but just add tests that we don't crash. +reveal_type(f(tda1, tda2)) # N: Revealed type is "TypedDict({'x': builtins.int, 'y': TypedDict('__main__.TDA1', {'x': builtins.int, 'y': ...})})" +reveal_type(f(tda1, tdb)) # N: Revealed type is "TypedDict({})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testBasicRecursiveGenericTypedDict] +from typing import TypedDict, TypeVar, Generic, Optional, List + +T = TypeVar("T") +class Tree(TypedDict, Generic[T], total=False): + value: T + left: Tree[T] + right: Tree[T] + +def collect(arg: Tree[T]) -> List[T]: ... + +reveal_type(collect({"left": {"right": {"value": 0}}})) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveGenericTypedDictExtending] +from typing import TypedDict, Generic, TypeVar, List + +T = TypeVar("T") + +class TD(TypedDict, Generic[T]): + val: T + other: STD[T] +class STD(TD[T]): + sval: T + one: TD[T] + +std: STD[str] +reveal_type(std) # N: Revealed type is "TypedDict('__main__.STD', {'val': builtins.str, 'other': ..., 'sval': builtins.str, 'one': TypedDict('__main__.TD', {'val': builtins.str, 'other': ...})})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveClassLevelAlias] +from typing import Union, Sequence + +class A: + Children = Union[Sequence['Children'], 'A', None] +x: A.Children +reveal_type(x) # N: Revealed type is "Union[typing.Sequence[...], __main__.A, None]" + +class B: + Foo = Sequence[Bar] + Bar = Sequence[Foo] +y: B.Foo +reveal_type(y) # N: Revealed type is "typing.Sequence[typing.Sequence[...]]" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnRecursiveTupleFallback] +from typing import Union, Tuple + +Tree1 = Union[str, Tuple[Tree1]] +Tree2 = Union[str, Tuple[Tree2, Tree2]] +Tree3 = Union[str, Tuple[Tree3, Tree3, Tree3]] + +def test1() -> Tree1: + return 42 # E: Incompatible return value type (got "int", expected "Union[str, Tuple[Tree1]]") +def test2() -> Tree2: + return 42 # E: Incompatible return value type (got "int", expected "Union[str, Tuple[Tree2, Tree2]]") +def test3() -> Tree3: + return 42 # E: Incompatible return value type (got "int", expected "Union[str, Tuple[Tree3, Tree3, Tree3]]") +[builtins fixtures/tuple.pyi] + +[case testRecursiveDoubleUnionNoCrash] +from typing import Tuple, Union, Callable, Sequence + +K = Union[int, Tuple[Union[int, K]]] +L = Union[int, Callable[[], Union[int, L]]] +M = Union[int, Sequence[Union[int, M]]] + +x: K +x = x +y: L +y = y +z: M +z = z + +x = y # E: Incompatible types in assignment (expression has type "L", variable has type "K") +z = x # OK +[builtins fixtures/tuple.pyi] + +[case testRecursiveInstanceInferenceNoCrash] +from typing import Sequence, TypeVar, Union + +class C(Sequence[C]): ... + +T = TypeVar("T") +def foo(x: T) -> C: ... + +Nested = Union[C, Sequence[Nested]] +x: Nested = foo(42) + +[case testNoRecursiveExpandInstanceUnionCrash] +from typing import List, Union + +class Tag(List[Union[Tag, List[Tag]]]): ... +Tag() + +[case testNoRecursiveExpandInstanceUnionCrashGeneric] +from typing import Generic, Iterable, TypeVar, Union + +ValueT = TypeVar("ValueT") +class Recursive(Iterable[Union[ValueT, Recursive[ValueT]]]): + pass + +class Base(Generic[ValueT]): + def __init__(self, element: ValueT): + pass +class Sub(Base[Union[ValueT, Recursive[ValueT]]]): + pass + +x: Iterable[str] +reveal_type(Sub) # N: Revealed type is "def [ValueT] (element: Union[ValueT`1, __main__.Recursive[ValueT`1]]) -> __main__.Sub[ValueT`1]" +reveal_type(Sub(x)) # N: Revealed type is "__main__.Sub[typing.Iterable[builtins.str]]" + +[case testNoRecursiveExpandInstanceUnionCrashInference] +# flags: --disable-error-code used-before-def +from typing import TypeVar, Union, Generic, List + +T = TypeVar("T") +InList = Union[T, InListRecurse[T]] +class InListRecurse(Generic[T], List[InList[T]]): ... + +def list_thing(transforming: InList[T]) -> T: + ... +reveal_type(list_thing([5])) # N: Revealed type is "builtins.list[builtins.int]" + +[case testRecursiveTypedDictWithList] +from typing import List +from typing_extensions import TypedDict + +Example = TypedDict("Example", {"rec": List["Example"]}) +e: Example +reveal_type(e) # N: Revealed type is "TypedDict('__main__.Example', {'rec': builtins.list[...]})" +[builtins fixtures/dict.pyi] + +[case testRecursiveNamedTupleWithList] +from typing import List, NamedTuple + +Example = NamedTuple("Example", [("rec", List["Example"])]) +e: Example +reveal_type(e) # N: Revealed type is "Tuple[builtins.list[...], fallback=__main__.Example]" +[builtins fixtures/tuple.pyi] + +[case testRecursiveBoundFunctionScopeNoCrash] +from typing import TypeVar, Union, Dict + +def dummy() -> None: + A = Union[str, Dict[str, "A"]] # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + T = TypeVar("T", bound=A) + + def bar(x: T) -> T: + pass + reveal_type(bar) # N: Revealed type is "def [T <: Union[builtins.str, builtins.dict[builtins.str, Any]]] (x: T`-1) -> T`-1" +[builtins fixtures/dict.pyi] + +[case testForwardBoundFunctionScopeWorks] +from typing import TypeVar, Dict + +def dummy() -> None: + A = Dict[str, "B"] + B = Dict[str, str] + T = TypeVar("T", bound=A) + + def bar(x: T) -> T: + pass + reveal_type(bar) # N: Revealed type is "def [T <: builtins.dict[builtins.str, builtins.dict[builtins.str, builtins.str]]] (x: T`-1) -> T`-1" +[builtins fixtures/dict.pyi] + +[case testAliasRecursiveUnpackMultiple] +from typing import Tuple, TypeVar, Optional + +T = TypeVar("T") +S = TypeVar("S") + +A = Tuple[T, S, Optional[A[T, S]]] +x: A[int, str] + +*_, last = x +if last is not None: + reveal_type(last) # N: Revealed type is "Tuple[builtins.int, builtins.str, Union[Tuple[builtins.int, builtins.str, Union[..., None]], None]]" +[builtins fixtures/tuple.pyi] + +[case testRecursiveAliasLiteral] +from typing import Tuple +from typing_extensions import Literal + +NotFilter = Tuple[Literal["not"], "NotFilter"] +n: NotFilter +reveal_type(n[1][1][0]) # N: Revealed type is "Literal['not']" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnRecursiveAliasWithNone] +# flags: --strict-optional +from typing import Union, Generic, TypeVar, Optional + +T = TypeVar("T") +class A(Generic[T]): ... +class B(Generic[T]): ... + +Z = Union[A[Z], B[Optional[Z]]] +X = Union[A[Optional[X]], B[Optional[X]]] + +z: Z +x: X +reveal_type(z) # N: Revealed type is "Union[__main__.A[...], __main__.B[Union[..., None]]]" +reveal_type(x) # N: Revealed type is "Union[__main__.A[Union[..., None]], __main__.B[Union[..., None]]]" + +[case testRecursiveTupleFallback1] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +T2 = Tuple[T1, "T4", "T4"] +T3 = Tuple[str, "T4", "T4"] +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback2] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +class T2(Tuple[T1, "T4", "T4"]): ... +T3 = Tuple[str, "T4", "T4"] +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback3] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +T2 = Tuple[T1, "T4", "T4"] +class T3(Tuple[str, "T4", "T4"]): ... +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback4] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +class T2(Tuple[T1, "T4", "T4"]): ... +class T3(Tuple[str, "T4", "T4"]): ... +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback5] +from typing import Protocol, Tuple, Union + +class Proto(Protocol): + def __len__(self) -> int: ... + +A = Union[Proto, Tuple[A]] +ta: Tuple[A] +p: Proto +p = ta +[builtins fixtures/tuple.pyi] + +[case testRecursiveAliasesWithAnyUnimported] +# flags: --disallow-any-unimported +from typing import Callable +from bogus import Foo # type: ignore + +A = Callable[[Foo, "B"], Foo] # E: Type alias target becomes "Callable[[Any, B], Any]" due to an unfollowed import +B = Callable[[Foo, A], Foo] # E: Type alias target becomes "Callable[[Any, A], Any]" due to an unfollowed import diff --git a/test-data/unit/check-redefine.test b/test-data/unit/check-redefine.test new file mode 100644 index 000000000000..b7642d30efc8 --- /dev/null +++ b/test-data/unit/check-redefine.test @@ -0,0 +1,575 @@ +-- Test cases for the redefinition of variable with a different type. + + +-- Redefine local variable +-- ----------------------- + + +[case testRedefineLocalWithDifferentType] +# flags: --allow-redefinition +def f() -> None: + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" + +[case testCannotConditionallyRedefineLocalWithDifferentType] +# flags: --allow-redefinition +def f() -> None: + y = 0 + reveal_type(y) # N: Revealed type is "builtins.int" + if int(): + y = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + reveal_type(y) # N: Revealed type is "builtins.int" + reveal_type(y) # N: Revealed type is "builtins.int" + +[case testRedefineFunctionArg] +# flags: --allow-redefinition +def f(x: int) -> None: + g(x) + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" +def g(x: int) -> None: + if int(): + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + reveal_type(x) # N: Revealed type is "builtins.int" + +[case testRedefineAnnotationOnly] +# flags: --allow-redefinition +def f() -> None: + x: int + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + reveal_type(x) # N: Revealed type is "builtins.int" +def g() -> None: + x: int + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" + +[case testRedefineLocalUsingOldValue] +# flags: --allow-redefinition +from typing import TypeVar, Union + +T = TypeVar('T') + +def f(x: int) -> None: + x = g(x) + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + y = 1 + y = g(y) + reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def g(x: T) -> Union[T, str]: pass + +[case testRedefineLocalForLoopIndexVariable] +# flags: --allow-redefinition +from typing import Iterable +def f(a: Iterable[int], b: Iterable[str]) -> None: + for x in a: + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + reveal_type(x) # N: Revealed type is "builtins.int" + for x in b: + x = 1 \ + # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(x) # N: Revealed type is "builtins.str" + +def g(a: Iterable[int]) -> None: + for x in a: pass + x = '' + +def h(a: Iterable[int]) -> None: + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" + for x in a: pass + +[case testCannotRedefineLocalWithinTry] +# flags: --allow-redefinition +def f() -> None: + try: + x = 0 + x + g() # Might raise an exception + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + except: + pass + reveal_type(x) # N: Revealed type is "builtins.int" + y = 0 + y + y = '' + +def g(): pass + +[case testRedefineLocalWithinWith] +# flags: --allow-redefinition +def f() -> None: + with g(): + x = 0 + x + g() # Might raise an exception, but we ignore this + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" + y = 0 + y + y = '' + +def g(): pass + +[case testCannotRedefineAcrossNestedFunction] +# flags: --allow-redefinition +def f() -> None: + x = 0 + x + def g() -> None: + x + g() + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + g() + y = 0 + y + y = '' + +[case testCannotRedefineAcrossNestedDecoratedFunction] +# flags: --allow-redefinition +def dec(f): return f + +def f() -> None: + x = 0 + x + @dec + def g() -> None: + x + g() + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + g() + y = 0 + y + y = '' + +[case testCannotRedefineAcrossNestedOverloadedFunction] +# flags: --allow-redefinition +from typing import overload + +def f() -> None: + x = 0 + x + @overload + def g() -> None: pass + @overload + def g(x: int) -> None: pass + def g(x=0): + pass + g() + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + g() + y = 0 + y + y = '' + +[case testRedefineLocalInMultipleAssignment] +# flags: --allow-redefinition +def f() -> None: + x, x = 1, '' + reveal_type(x) # N: Revealed type is "builtins.str" + x = object() + reveal_type(x) # N: Revealed type is "builtins.object" + +def g() -> None: + x = 1 + if 1: + x, x = '', 1 \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testRedefineUnderscore] +# flags: --allow-redefinition +def f() -> None: + _, _ = 1, '' + if 1: + _, _ = '', 1 + reveal_type(_) # N: Revealed type is "Any" + +[case testRedefineWithBreakAndContinue] +# flags: --allow-redefinition +def f() -> None: + y = 0 + y + while int(): + z = 0 + z + z = '' + x = 0 + if int(): + break + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + reveal_type(x) # N: Revealed type is "builtins.int" + y = '' + +def g() -> None: + y = 0 + y + for a in h(): + z = 0 + z + z = '' + x = 0 + if int(): + continue + x = '' \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + reveal_type(x) # N: Revealed type is "builtins.int" + y = '' + +def h(): pass + +[case testRedefineLocalAndNestedLoops] +# flags: --allow-redefinition +def f() -> None: + z = 0 + z + while int(): + x = 0 + x + while int(): + if 1: + y = 1 + y + if int(): + break + y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + x = '' + z = '' + +[case testCannotRedefineVarAsFunction] +# flags: --allow-redefinition +def f() -> None: + def x(): pass + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Callable[[], Any]") + reveal_type(x) # N: Revealed type is "def () -> Any" + y = 1 + def y(): pass # E: Name "y" already defined on line 6 + +[case testCannotRedefineVarAsClass] +# flags: --allow-redefinition +def f() -> None: + class x: pass + x = 1 # E: Cannot assign to a type \ + # E: Incompatible types in assignment (expression has type "int", variable has type "Type[x]") + y = 1 + class y: pass # E: Name "y" already defined on line 5 + +[case testRedefineVarAsTypeVar] +# flags: --allow-redefinition +from typing import TypeVar +def f() -> None: + x = TypeVar('x') + x = 1 # E: Invalid assignment target \ + # E: Incompatible types in assignment (expression has type "int", variable has type "TypeVar") + reveal_type(x) # N: Revealed type is "typing.TypeVar" + y = 1 + # NOTE: '"int" not callable' is due to test stubs + y = TypeVar('y') # E: Cannot redefine "y" as a type variable \ + # E: Incompatible types in assignment (expression has type "TypeVar", variable has type "int") + def h(a: y) -> y: return a # E: Variable "y" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testCannotRedefineVarAsModule] +# flags: --allow-redefinition +def f() -> None: + import typing as m + m = 1 # E: Incompatible types in assignment (expression has type "int", variable has type Module) + n = 1 + import typing as n # E: Incompatible import of "n" (imported name has type Module, local name has type "int") +[builtins fixtures/module.pyi] + +[case testRedefineLocalWithTypeAnnotation] +# flags: --allow-redefinition +def f() -> None: + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + x = '' # type: object + reveal_type(x) # N: Revealed type is "builtins.object" +def g() -> None: + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + x: object = '' + reveal_type(x) # N: Revealed type is "builtins.object" +def h() -> None: + x: int + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + x: object + x: object = '' # E: Name "x" already defined on line 16 +def farg(x: int) -> None: + x: str = '' # E: Name "x" already defined on line 18 +def farg2(x: int) -> None: + x: str = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testRedefineLocalWithTypeAnnotationSpecialCases] +# flags: --allow-redefinition +def f() -> None: + x: object + x = 1 + if int(): + x = '' + reveal_type(x) # N: Revealed type is "builtins.object" + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" + if int(): + x = 2 \ + # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + +[case testCannotRedefineSelf] +# flags: --allow-redefinition +class A: + x = 0 + + def f(self) -> None: + reveal_type(self.x) # N: Revealed type is "builtins.int" + self = f() + self.y: str = '' + reveal_type(self.y) # N: Revealed type is "builtins.str" + +def f() -> A: return A() + + +-- Redefine global variable +-- ------------------------ + + +[case testRedefineGlobalWithDifferentType] +# flags: --allow-redefinition +import m +reveal_type(m.x) +[file m.py] +x = 0 +reveal_type(x) +x = object() +reveal_type(x) +x = '' +reveal_type(x) +[out] +tmp/m.py:2: note: Revealed type is "builtins.int" +tmp/m.py:4: note: Revealed type is "builtins.object" +tmp/m.py:6: note: Revealed type is "builtins.str" +main:3: note: Revealed type is "builtins.str" + +[case testRedefineGlobalForIndex] +# flags: --allow-redefinition +import m +reveal_type(m.x) +[file m.py] +from typing import Iterable +def f(): pass +it1: Iterable[int] = f() +it2: Iterable[str] = f() +for x in it1: + reveal_type(x) +for x in it2: + reveal_type(x) +reveal_type(x) +[out] +tmp/m.py:6: note: Revealed type is "builtins.int" +tmp/m.py:8: note: Revealed type is "builtins.str" +tmp/m.py:9: note: Revealed type is "builtins.str" +main:3: note: Revealed type is "builtins.str" + +[case testRedefineGlobalBasedOnPreviousValues] +# flags: --allow-redefinition +from typing import TypeVar, Iterable +T = TypeVar('T') +def f(x: T) -> Iterable[T]: pass +a = 0 +a = f(a) +reveal_type(a) # N: Revealed type is "typing.Iterable[builtins.int]" + +[case testRedefineGlobalWithSeparateDeclaration] +# flags: --allow-redefinition +x = '' +reveal_type(x) # N: Revealed type is "builtins.str" +x: int +x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +reveal_type(x) # N: Revealed type is "builtins.int" +x: object +x = 1 +reveal_type(x) # N: Revealed type is "builtins.int" +if int(): + x = object() + +[case testRedefineGlobalUsingForLoop] +# flags: --allow-redefinition +from typing import Iterable, TypeVar, Union +T = TypeVar('T') +def f(x: T) -> Iterable[Union[T, str]]: pass +x = 0 +reveal_type(x) # N: Revealed type is "builtins.int" +for x in f(x): + pass +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testNoRedefinitionIfOnlyInitialized] +# flags: --allow-redefinition --no-strict-optional +x = None # type: int +x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "int") +x # Reference to variable +x = '' + +y = 0 +y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNoRedefinitionIfNoValueAssigned] +# flags: --allow-redefinition +x: int +x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +reveal_type(x) # N: Revealed type is "builtins.int" +x: object + +[case testNoRedefinitionIfExplicitlyDisallowed] +# flags: --disallow-redefinition +x = 0 +x = 2 +x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +def f() -> None: + y = 0 + y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +class C: + y = 0 + y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +def g() -> None: + # _ is a special case + _ = 0 + _ = '' + x, _ = 0, C() +[builtins fixtures/tuple.pyi] + +[case testRedefineAsException] +# flags: --allow-redefinition +e = 1 +reveal_type(e) # N: Revealed type is "builtins.int" +try: + pass +except Exception as e: + reveal_type(e) # N: Revealed type is "builtins.Exception" +e = '' +reveal_type(e) # N: Revealed type is "builtins.str" +[builtins fixtures/exception.pyi] + +[case testRedefineUsingWithStatement] +# flags: --allow-redefinition +class A: + def __enter__(self) -> int: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> str: ... + def __exit__(self, x, y, z) -> None: ... +with A() as x: + reveal_type(x) # N: Revealed type is "builtins.int" +with B() as x: + x = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + + +[case testRedefineModuleAsException] +import typing +try: + pass +except Exception as typing: + pass +[builtins fixtures/exception.pyi] + +[case testRedefiningUnderscoreFunctionIsntAnError] +def _(arg): + pass + +def _(arg): + pass + +[case testTypeErrorsInUnderscoreFunctionsReported] +def _(arg: str): + x = arg + 1 # E: Unsupported left operand type for + ("str") + +def _(arg: int) -> int: + return 'a' # E: Incompatible return value type (got "str", expected "int") + +[case testCallingUnderscoreFunctionIsNotAllowed-skip] +# Skipped because of https://github.com/python/mypy/issues/11774 +def _(arg: str) -> None: + pass + +def _(arg: int) -> int: + return arg + +_('a') # E: Calling function named "_" is not allowed + +y = _(5) # E: Calling function named "_" is not allowed + +[case testFunctionStillTypeCheckedWhenAliasedAsUnderscoreDuringImport] +from a import f as _ + +_(1) # E: Argument 1 to "f" has incompatible type "int"; expected "str" +reveal_type(_('a')) # N: Revealed type is "builtins.str" + +[file a.py] +def f(arg: str) -> str: + return arg + +[case testCallToFunctionStillTypeCheckedWhenAssignedToUnderscoreVariable] +from a import g +_ = g + +_('a') # E: Argument 1 has incompatible type "str"; expected "int" +reveal_type(_(1)) # N: Revealed type is "builtins.int" + +[file a.py] +def g(arg: int) -> int: + return arg + +[case testRedefiningUnderscoreFunctionWithDecoratorWithUnderscoreFunctionsNextToEachOther] +def dec(f): + return f + +@dec +def _(arg): + pass + +@dec +def _(arg): + pass + +[case testRedefiningUnderscoreFunctionWithDecoratorInDifferentPlaces] +def dec(f): + return f + +def dec2(f): + return f + +@dec +def _(arg): + pass + +def f(arg): + pass + +@dec2 +def _(arg): + pass + +[case testOverwritingImportedFunctionThatWasAliasedAsUnderscore] +from a import f as _ + +def _(arg: str) -> str: # E: Name "_" already defined (possibly by an import) + return arg + +[file a.py] +def f(s: str) -> str: + return s diff --git a/test-data/unit/check-reports.test b/test-data/unit/check-reports.test new file mode 100644 index 000000000000..423cbcc49289 --- /dev/null +++ b/test-data/unit/check-reports.test @@ -0,0 +1,345 @@ +[case testReportBasic] +# flags: --xml-report out +def f(): pass + +def g() -> None: pass +[outfile out/index.xml] + + +[case testLinePrecisionBasic] +# flags: --lineprecision-report out +def f(): pass + +def g() -> None: + a = 1 +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 5 2 0 1 2 0 + +[case testLinePrecisionImpreciseType] +# flags: --lineprecision-report out +def f(x: list) -> None: pass +[builtins fixtures/list.pyi] +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 2 0 1 0 1 0 + +[case testLinePrecisionUnanalyzed] +# flags: --lineprecision-report out +import sys +MYPY = False +if not MYPY: + a = 1 + +def f(x: int) -> None: + if isinstance(x, str): + b = 1 + c = 1 +[builtins fixtures/isinstance.pyi] +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 10 5 0 0 2 3 + +[case testLinePrecisionEmptyLines] +# flags: --lineprecision-report out +def f() -> None: + """docstring + + long + """ + x = 0 + + # comment + y = 0 # comment (non-empty) +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 10 3 0 0 7 0 + +[case testLinePrecisionImportFrom] +# flags: --lineprecision-report out --ignore-missing-imports +from m import f +from m import g +from bad import foo +from bad import ( # treated as a single line + foo2, + foo3, +) + +[file m.py] +def f(): pass +def g() -> None: pass + +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 8 2 0 2 4 0 +m 2 1 0 1 0 0 + +[case testLinePrecisionImport] +# flags: --lineprecision-report out --ignore-missing-imports +import m +import bad +import m, bad + +[file m.py] +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 4 1 0 2 1 0 +m 0 0 0 0 0 0 + +[case testLinePrecisionStarImport] +# flags: --lineprecision-report out --ignore-missing-imports +from m import * +from bad import * + +[file m.py] +def f(): pass +def g() -> None: pass +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 3 1 0 1 1 0 +m 2 1 0 1 0 0 + +[case testLinePrecisionRelativeImport] +# flags: --lineprecision-report out --ignore-missing-imports +import a + +[file a/__init__.py] +from .m import f +from .bad import g + +[file a/m.py] +def f(): pass + +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 2 1 0 0 1 0 +a 2 1 0 1 0 0 +a.m 1 0 0 1 0 0 + +[case testLinePrecisionPassStatement] +# flags: --lineprecision-report out +def f() -> None: + pass +def g(): + pass +class C: + pass +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 7 4 0 2 1 0 + +[case testLinePrecisionBreakAndContinueStatement] +# flags: --lineprecision-report out +import a +import b + +[file a.py] +def f() -> int: + while f(): + break + return f() +def g(): + while g(): + break + +[file b.py] +def f() -> int: + while f(): + continue + return f() +def g(): + while g(): + continue + +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 3 2 0 0 1 0 +a 7 4 0 3 0 0 +b 7 4 0 3 0 0 + +[case testLinePrecisionLiterals] +# flags: --lineprecision-report out +import str_lit +import bytes_lit +import int_lit +import float_lit +import true_lit +import false_lit +import none_lit +import complex_lit +import dots_lit + +[file str_lit.py] +def f() -> object: + return '' +def g(): + return '' + +[file bytes_lit.py] +def f() -> object: + return b'' +def g(): + return b'' + +[file int_lit.py] +def f() -> object: + return 1 +def g(): + return 1 + +[file float_lit.py] +def f() -> object: + return 1.1 +def g(): + return 1.1 + +[file true_lit.py] +def f() -> object: + return True +def g(): + return True + +[file false_lit.py] +def f() -> object: + return False +def g(): + return False + +[file none_lit.py] +def f() -> object: + return None +def g(): + return None + +[file complex_lit.py] +def f() -> object: + return None +def g(): + return None + +[file dots_lit.py] +def f() -> object: + return ... +def g(): + return ... + +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +---------------------------------------------------------------- +__main__ 10 9 0 0 1 0 +bytes_lit 4 2 0 2 0 0 +complex_lit 4 2 0 2 0 0 +dots_lit 4 2 0 2 0 0 +false_lit 4 2 0 2 0 0 +float_lit 4 2 0 2 0 0 +int_lit 4 2 0 2 0 0 +none_lit 4 2 0 2 0 0 +str_lit 4 2 0 2 0 0 +true_lit 4 2 0 2 0 0 + +[case testLinePrecisionIfStatement] +# flags: --lineprecision-report out +if int(): + x = 1 +else: # This is treated as empty + x = 2 +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 5 3 0 0 2 0 + +[case testLinePrecisionCallAnyArg] +# flags: --lineprecision-report out +from m import f +def g() -> None: + f(1) # Precise + f(1, 2) # Any +[file m.py] +from typing import Any +def f(x: int, y: Any = 0) -> None: + pass +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 5 3 0 1 1 0 +m 3 2 0 1 0 0 + +[case testLinePrecisionCallImpreciseArg] +# flags: --lineprecision-report out +from m import f +def g() -> None: + f(1) # Precise + f(1, [2]) # Imprecise +[file m.py] +from typing import List, Any +def f(x: int, y: List[Any] = []) -> None: + pass +[builtins fixtures/list.pyi] +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 5 3 1 0 1 0 +m 3 2 1 0 0 0 + +[case testLinePrecisionCallAnyArgWithKeywords] +# flags: --lineprecision-report out +from m import f +def g() -> None: + f(x=1) # Precise + f(x=1, z=1) # Precise + f(z=1, x=1) # Precise + f(y=1) # Any + f(y=1, x=1) # Any +[file m.py] +from typing import Any +def f(x: int = 0, y: Any = 0, z: int = 0) -> None: + pass +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 8 5 0 2 1 0 +m 3 2 0 1 0 0 + +[case testLinePrecisionCallAnyMethodArg] +# flags: --lineprecision-report out +from m import C +def g(c: C) -> None: + c.f(1) # Precise + c.f(1, 2) # Any +[file m.py] +from typing import Any +class C: + def f(self, x: int, y: Any = 0) -> None: + pass +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 5 3 0 1 1 0 +m 4 3 0 1 0 0 + +[case testLinePrecisionCallAnyConstructorArg] +# flags: --lineprecision-report out +from m import C +def g() -> None: + C(1) # Precise + C(1, 2) # Any +[file m.py] +from typing import Any +class C: + def __init__(self, x: int, y: Any = 0) -> None: + pass +[outfile out/lineprecision.txt] +Name Lines Precise Imprecise Any Empty Unanalyzed +------------------------------------------------------------- +__main__ 5 3 0 1 1 0 +m 4 3 0 1 0 0 diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test new file mode 100644 index 000000000000..fa853ac48e5a --- /dev/null +++ b/test-data/unit/check-selftype.test @@ -0,0 +1,2216 @@ +[case testSelfTypeInstance] +from typing import TypeVar + +T = TypeVar('T', bound='A', covariant=True) + +class A: + def copy(self: T) -> T: pass + +class B(A): + pass + +reveal_type(A().copy) # N: Revealed type is "def () -> __main__.A" +reveal_type(B().copy) # N: Revealed type is "def () -> __main__.B" +reveal_type(A().copy()) # N: Revealed type is "__main__.A" +reveal_type(B().copy()) # N: Revealed type is "__main__.B" + +[builtins fixtures/bool.pyi] + +[case testSelfTypeStaticAccess] +from typing import TypeVar + +T = TypeVar('T', bound='A', covariant=True) +class A: + def copy(self: T) -> T: pass + +class B(A): + pass + +# Erased instances appear on reveal_type; unrelated to self type +def f(a: A) -> None: pass +f(A.copy(A())) +f(A.copy(B())) +f(B.copy(B())) + +# TODO: make it an error +# f(B.copy(A())) + +def g(a: B) -> None: pass +g(A.copy(A())) # E: Argument 1 to "g" has incompatible type "A"; expected "B" +g(A.copy(B())) +g(B.copy(B())) + +[builtins fixtures/bool.pyi] + +[case testSelfTypeReturn] +from typing import TypeVar, Type + +R = TypeVar('R') +def _type(self: R) -> Type[R]: pass + +T = TypeVar('T', bound='A', covariant=True) +class A: + def copy(self: T) -> T: + if B(): + return A() # E: Incompatible return value type (got "A", expected "T") + elif A(): + return B() # E: Incompatible return value type (got "B", expected "T") + reveal_type(_type(self)) # N: Revealed type is "Type[T`-1]" + return reveal_type(_type(self)()) # N: Revealed type is "T`-1" + +class B(A): + pass + +Q = TypeVar('Q', bound='C', covariant=True) +class C: + def __init__(self, a: int) -> None: pass + + def copy(self: Q) -> Q: + if self: + return reveal_type(_type(self)(1)) # N: Revealed type is "Q`-1" + else: + return _type(self)() # E: Missing positional argument "a" in call to "C" + + +[builtins fixtures/bool.pyi] + +[case testSelfTypeClass] +from typing import TypeVar, Type + +T = TypeVar('T', bound='A') + +class A: + @classmethod + def new(cls: Type[T]) -> T: + return reveal_type(cls()) # N: Revealed type is "T`-1" + +class B(A): + pass + +Q = TypeVar('Q', bound='C', covariant=True) +class C: + def __init__(self, a: int) -> None: pass + + @classmethod + def new(cls: Type[Q]) -> Q: + if cls: + return cls(1) + else: + return cls() # E: Missing positional argument "a" in call to "C" + + +reveal_type(A.new) # N: Revealed type is "def () -> __main__.A" +reveal_type(B.new) # N: Revealed type is "def () -> __main__.B" +reveal_type(A.new()) # N: Revealed type is "__main__.A" +reveal_type(B.new()) # N: Revealed type is "__main__.B" + +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeOverride] +from typing import TypeVar, cast + +T = TypeVar('T', bound='A', covariant=True) + +class A: + def copy(self: T) -> T: pass + +class B(A): + pass + +Q = TypeVar('Q', bound='C', covariant=True) +class C(A): + def copy(self: Q) -> Q: pass + +reveal_type(C().copy) # N: Revealed type is "def () -> __main__.C" +reveal_type(C().copy()) # N: Revealed type is "__main__.C" +reveal_type(cast(A, C()).copy) # N: Revealed type is "def () -> __main__.A" +reveal_type(cast(A, C()).copy()) # N: Revealed type is "__main__.A" + +[builtins fixtures/bool.pyi] + +[case testSelfTypeOverrideCompatibility] +from typing import overload, TypeVar, Generic + +T = TypeVar("T") + +class A(Generic[T]): + @overload + def f(self: A[int]) -> int: ... + @overload + def f(self: A[str]) -> str: ... + def f(self): ... + +class B(A[T]): + @overload + def f(self: A[int]) -> int: ... + @overload + def f(self: A[str]) -> str: ... + def f(self): ... + +class B2(A[T]): + @overload + def f(self: A[int]) -> int: ... + @overload + def f(self: A[str]) -> str: ... + @overload + def f(self: A[bytes]) -> bytes: ... + def f(self): ... + +class C(A[int]): + def f(self) -> int: ... + +class D(A[str]): + def f(self) -> int: ... # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: @overload \ + # N: def f(self) -> str \ + # N: Subclass: \ + # N: def f(self) -> int + +class E(A[T]): + def f(self) -> int: ... # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: @overload \ + # N: def f(self) -> int \ + # N: @overload \ + # N: def f(self) -> str \ + # N: Subclass: \ + # N: def f(self) -> int + + +class F(A[bytes]): + # Note there's an argument to be made that this is actually compatible with the supertype + def f(self) -> bytes: ... # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: @overload \ + # N: def f(self) -> int \ + # N: @overload \ + # N: def f(self) -> str \ + # N: Subclass: \ + # N: def f(self) -> bytes + +class G(A): + def f(self): ... + +class H(A[int]): + def f(self): ... + +class I(A[int]): + def f(*args): ... + +class J(A[int]): + def f(self, arg) -> int: ... # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: @overload \ + # N: def f(self) -> int \ + # N: Subclass: \ + # N: def f(self, arg: Any) -> int + +[builtins fixtures/tuple.pyi] + +[case testSelfTypeOverrideCompatibilityGeneric] +from typing import TypeVar, Generic, overload + +T = TypeVar("T", str, int, None) + +class A(Generic[T]): + @overload + def f(self, s: T) -> T: ... + @overload + def f(self: A[str], s: bytes) -> str: ... + def f(self, s: object): ... + +class B(A[int]): + def f(self, s: int) -> int: ... + +class C(A[None]): + def f(self, s: int) -> int: ... # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: @overload \ + # N: def f(self, s: None) -> None \ + # N: Subclass: \ + # N: def f(self, s: int) -> int +[builtins fixtures/tuple.pyi] + +[case testSelfTypeOverrideCompatibilityTypeVar] +from typing import overload, TypeVar, Union + +AT = TypeVar("AT", bound="A") + +class A: + @overload + def f(self: AT, x: int) -> AT: ... + @overload + def f(self, x: str) -> None: ... + @overload + def f(self: AT) -> bytes: ... + def f(*a, **kw): ... + +class B(A): + @overload # E: Signature of "f" incompatible with supertype "A" \ + # N: Superclass: \ + # N: @overload \ + # N: def f(self, x: int) -> B \ + # N: @overload \ + # N: def f(self, x: str) -> None \ + # N: @overload \ + # N: def f(self) -> bytes \ + # N: Subclass: \ + # N: @overload \ + # N: def f(self, x: int) -> B \ + # N: @overload \ + # N: def f(self, x: str) -> None + def f(self, x: int) -> B: ... + @overload + def f(self, x: str) -> None: ... + def f(*a, **kw): ... +[builtins fixtures/dict.pyi] + +[case testSelfTypeOverrideCompatibilitySelfTypeVar] +from typing import Any, Generic, Self, TypeVar, overload + +T_co = TypeVar('T_co', covariant=True) + +class Config(Generic[T_co]): + @overload + def get(self, instance: None) -> Self: ... + @overload + def get(self, instance: Any) -> T_co: ... + def get(self, *a, **kw): ... + +class MultiConfig(Config[T_co]): + @overload + def get(self, instance: None) -> Self: ... + @overload + def get(self, instance: Any) -> T_co: ... + def get(self, *a, **kw): ... +[builtins fixtures/dict.pyi] + +[case testSelfTypeSuper] +from typing import TypeVar, cast + +T = TypeVar('T', bound='A', covariant=True) + +class A: + def copy(self: T) -> T: pass + +Q = TypeVar('Q', bound='B', covariant=True) +class B(A): + def copy(self: Q) -> Q: + reveal_type(self) # N: Revealed type is "Q`-1" + reveal_type(super().copy) # N: Revealed type is "def () -> Q`-1" + return super().copy() + +[builtins fixtures/bool.pyi] + +[case testSelfTypeRecursiveBinding] +from typing import TypeVar, Callable, Type + +T = TypeVar('T', bound='A', covariant=True) +class A: + # TODO: This is potentially unsafe, as we use T in an argument type + def copy(self: T, factory: Callable[[T], T]) -> T: + return factory(self) + + @classmethod + def new(cls: Type[T], factory: Callable[[T], T]) -> T: + reveal_type(cls) # N: Revealed type is "Type[T`-1]" + reveal_type(cls()) # N: Revealed type is "T`-1" + cls(2) # E: Too many arguments for "A" + return cls() + +class B(A): + pass + +reveal_type(A().copy) # N: Revealed type is "def (factory: def (__main__.A) -> __main__.A) -> __main__.A" +reveal_type(B().copy) # N: Revealed type is "def (factory: def (__main__.B) -> __main__.B) -> __main__.B" +reveal_type(A.new) # N: Revealed type is "def (factory: def (__main__.A) -> __main__.A) -> __main__.A" +reveal_type(B.new) # N: Revealed type is "def (factory: def (__main__.B) -> __main__.B) -> __main__.B" + +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeBound] +from typing import TypeVar, Callable, cast + +TA = TypeVar('TA', bound='A', covariant=True) + +class A: + def copy(self: TA) -> TA: + pass + +class C(A): + def copy(self: C) -> C: + pass + +class D(A): + def copy(self: A) -> A: # E: Return type "A" of "copy" incompatible with return type "D" in supertype "A" + pass + +TB = TypeVar('TB', bound='B', covariant=True) +class B(A): + x = 1 + def copy(self: TB) -> TB: + reveal_type(self.x) # N: Revealed type is "builtins.int" + return cast(TB, None) + +[builtins fixtures/bool.pyi] + +-- # TODO: fail for this +-- [case testSelfTypeBare] +-- from typing import TypeVar, Type +-- +-- T = TypeVar('T', bound='E') +-- +-- class E: +-- def copy(self: T, other: T) -> T: pass + +[case testSelfTypeClone] +from typing import TypeVar, Type +T = TypeVar('T', bound='C') + +class C: + def copy(self: T) -> T: + return self + + @classmethod + def new(cls: Type[T]) -> T: + return cls() + +class D(C): pass + +reveal_type(D.new) # N: Revealed type is "def () -> __main__.D" +reveal_type(D().new) # N: Revealed type is "def () -> __main__.D" +reveal_type(D.new()) # N: Revealed type is "__main__.D" +reveal_type(D().new()) # N: Revealed type is "__main__.D" + +Q = TypeVar('Q', bound=C) + +def clone(arg: Q) -> Q: + reveal_type(arg.copy) # N: Revealed type is "def () -> Q`-1" + reveal_type(arg.copy()) # N: Revealed type is "Q`-1" + reveal_type(arg.new) # N: Revealed type is "def () -> Q`-1" + reveal_type(arg.new()) # N: Revealed type is "Q`-1" + return arg.copy() + +def make(cls: Type[Q]) -> Q: + reveal_type(cls.new) # N: Revealed type is "def () -> Q`-1" + reveal_type(cls().new) # N: Revealed type is "def () -> Q`-1" + reveal_type(cls().new()) # N: Revealed type is "Q`-1" + return cls.new() + +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeGeneric] +from typing import TypeVar + +T = TypeVar('T', int, str) + +class A: + pass + +class B(A): + def __init__(self, arg: T) -> None: + super(B, self).__init__() + +[case testSelfTypeNonsensical] +from typing import TypeVar, Type + +T = TypeVar('T', bound=str) +class A: + def foo(self: T) -> T: # E: The erased type of self "builtins.str" is not a supertype of its class "__main__.A" + return self + + @classmethod + def cfoo(cls: Type[T]) -> T: # E: The erased type of self "Type[builtins.str]" is not a supertype of its class "Type[__main__.A]" + return cls() + +Q = TypeVar('Q', bound='B') +class B: + def foo(self: Q) -> Q: + return self + + @classmethod + def cfoo(cls: Type[Q]) -> Q: + return cls() + +class C: + def foo(self: C) -> C: return self + + @classmethod + def cfoo(cls: Type[C]) -> C: + return cls() + +class D: + def foo(self: Q) -> Q: # E: The erased type of self "__main__.B" is not a supertype of its class "__main__.D" + return self + + @staticmethod + def bar(self: str) -> str: + return self + + @classmethod + def cfoo(cls: Type[Q]) -> Q: # E: The erased type of self "Type[__main__.B]" is not a supertype of its class "Type[__main__.D]" + return cls() + +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeLambdaDefault] +from typing import Callable +class C: + @classmethod + def foo(cls, + arg: Callable[[int], str] = lambda a: '' + ) -> None: + pass + + def bar(self, + arg: Callable[[int], str] = lambda a: '' + ) -> None: + pass +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeNew] +from typing import TypeVar, Type + +T = TypeVar('T', bound='A') +class A: + def __new__(cls: Type[T]) -> T: + return cls() + + def __init_subclass__(cls: Type[T]) -> None: + pass + +class B: + def __new__(cls: Type[T]) -> T: # E: The erased type of self "Type[__main__.A]" is not a supertype of its class "Type[__main__.B]" + return cls() + + def __init_subclass__(cls: Type[T]) -> None: # E: The erased type of self "Type[__main__.A]" is not a supertype of its class "Type[__main__.B]" + pass + +class C: + def __new__(cls: Type[C]) -> C: + return cls() + + def __init_subclass__(cls: Type[C]) -> None: + pass + +class D: + def __new__(cls: D) -> D: # E: The erased type of self "__main__.D" is not a supertype of its class "Type[__main__.D]" + return cls + + def __init_subclass__(cls: D) -> None: # E: The erased type of self "__main__.D" is not a supertype of its class "Type[__main__.D]" + pass + +class E: + def __new__(cls) -> E: + reveal_type(cls) # N: Revealed type is "Type[__main__.E]" + return cls() + + def __init_subclass__(cls) -> None: + reveal_type(cls) # N: Revealed type is "Type[__main__.E]" + +[case testSelfTypeNew_explicit] +from typing import TypeVar, Type + +T = TypeVar('T', bound='A') +class A: + @staticmethod + def __new__(cls: Type[T]) -> T: + return cls() + + @classmethod + def __init_subclass__(cls: Type[T]) -> None: + pass + +class B: + @staticmethod + def __new__(cls: Type[T]) -> T: # E: The erased type of self "Type[__main__.A]" is not a supertype of its class "Type[__main__.B]" + return cls() + + @classmethod + def __init_subclass__(cls: Type[T]) -> None: # E: The erased type of self "Type[__main__.A]" is not a supertype of its class "Type[__main__.B]" + pass + +class C: + @staticmethod + def __new__(cls: Type[C]) -> C: + return cls() + + @classmethod + def __init_subclass__(cls: Type[C]) -> None: + pass + +class D: + @staticmethod + def __new__(cls: D) -> D: # E: The erased type of self "__main__.D" is not a supertype of its class "Type[__main__.D]" + return cls + + @classmethod + def __init_subclass__(cls: D) -> None: # E: The erased type of self "__main__.D" is not a supertype of its class "Type[__main__.D]" + pass + +class E: + @staticmethod + def __new__(cls) -> E: + reveal_type(cls) # N: Revealed type is "Type[__main__.E]" + return cls() + + @classmethod + def __init_subclass__(cls) -> None: + reveal_type(cls) # N: Revealed type is "Type[__main__.E]" + +[builtins fixtures/classmethod.pyi] + +[case testSelfTypePropertyUnion] +from typing import Union +class A: + @property + def f(self: A) -> int: pass + +class B: + @property + def f(self: B) -> int: pass +x: Union[A, B] +reveal_type(x.f) # N: Revealed type is "builtins.int" + +[builtins fixtures/property.pyi] + +[case testSelfTypeProperSupertypeAttribute] +from typing import Callable, TypeVar, ClassVar +class K: pass +T = TypeVar('T', bound=K) +class A(K): + @property + def g(self: K) -> int: return 0 + @property + def gt(self: T) -> T: return self + f: ClassVar[Callable[[object], int]] + ft: ClassVar[Callable[[T], T]] + +class B(A): + pass + +reveal_type(A().g) # N: Revealed type is "builtins.int" +reveal_type(A().gt) # N: Revealed type is "__main__.A" +reveal_type(A().f()) # N: Revealed type is "builtins.int" +reveal_type(A().ft()) # N: Revealed type is "__main__.A" +reveal_type(B().g) # N: Revealed type is "builtins.int" +reveal_type(B().gt) # N: Revealed type is "__main__.B" +reveal_type(B().f()) # N: Revealed type is "builtins.int" +reveal_type(B().ft()) # N: Revealed type is "__main__.B" + +[builtins fixtures/property.pyi] + +[case testSelfTypeProperSupertypeAttributeTuple] +from typing import Callable, TypeVar, Tuple, ClassVar +T = TypeVar('T') +class A(Tuple[int, int]): + @property + def g(self: object) -> int: return 0 + @property + def gt(self: T) -> T: return self + f: ClassVar[Callable[[object], int]] + ft: ClassVar[Callable[[T], T]] + +class B(A): + pass + +reveal_type(A().g) # N: Revealed type is "builtins.int" +reveal_type(A().gt) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.A]" +reveal_type(A().f()) # N: Revealed type is "builtins.int" +reveal_type(A().ft()) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.A]" +reveal_type(B().g) # N: Revealed type is "builtins.int" +reveal_type(B().gt) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.B]" +reveal_type(B().f()) # N: Revealed type is "builtins.int" +reveal_type(B().ft()) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.B]" + +[builtins fixtures/property.pyi] + +[case testSelfTypeProperSupertypeAttributeMeta] +from typing import Callable, TypeVar, Type, ClassVar +T = TypeVar('T') +class A(type): + @property + def g(cls: object) -> int: return 0 + @property + def gt(cls: T) -> T: return cls + f: ClassVar[Callable[[object], int]] + ft: ClassVar[Callable[[T], T]] + +class B(A): + pass + +class X(metaclass=B): + def __init__(self, x: int) -> None: pass +class Y(X): pass +X1: Type[X] +reveal_type(X.g) # N: Revealed type is "builtins.int" +reveal_type(X.gt) # N: Revealed type is "def (x: builtins.int) -> __main__.X" +reveal_type(X.f()) # N: Revealed type is "builtins.int" +reveal_type(X.ft()) # N: Revealed type is "def (x: builtins.int) -> __main__.X" +reveal_type(Y.g) # N: Revealed type is "builtins.int" +reveal_type(Y.gt) # N: Revealed type is "def (x: builtins.int) -> __main__.Y" +reveal_type(Y.f()) # N: Revealed type is "builtins.int" +reveal_type(Y.ft()) # N: Revealed type is "def (x: builtins.int) -> __main__.Y" +reveal_type(X1.g) # N: Revealed type is "builtins.int" +reveal_type(X1.gt) # N: Revealed type is "Type[__main__.X]" +reveal_type(X1.f()) # N: Revealed type is "builtins.int" +reveal_type(X1.ft()) # N: Revealed type is "Type[__main__.X]" + +[builtins fixtures/property.pyi] + +[case testSelfTypeProperSupertypeAttributeGeneric] +from typing import Callable, TypeVar, Generic, ClassVar +Q = TypeVar('Q', covariant=True) +class K(Generic[Q]): + q: Q +T = TypeVar('T') +class A(K[Q]): + @property + def g(self: K[object]) -> int: return 0 + @property + def gt(self: K[T]) -> T: return self.q + f: ClassVar[Callable[[object], int]] + ft: ClassVar[Callable[[T], T]] + +class B(A[Q]): + pass +a: A[int] +b: B[str] +reveal_type(a.g) # N: Revealed type is "builtins.int" +reveal_type(a.gt) # N: Revealed type is "builtins.int" +reveal_type(a.f()) # N: Revealed type is "builtins.int" +reveal_type(a.ft()) # N: Revealed type is "__main__.A[builtins.int]" +reveal_type(b.g) # N: Revealed type is "builtins.int" +reveal_type(b.gt) # N: Revealed type is "builtins.str" +reveal_type(b.f()) # N: Revealed type is "builtins.int" +reveal_type(b.ft()) # N: Revealed type is "__main__.B[builtins.str]" +[builtins fixtures/property.pyi] + +[case testSelfTypeRestrictedMethod] +from typing import TypeVar, Generic + +T = TypeVar('T') +class C(Generic[T]): + def from_item(self: C[str]) -> None: ... + +i: C[int] +s: C[str] + +i.from_item() # E: Invalid self argument "C[int]" to attribute function "from_item" with type "Callable[[C[str]], None]" +s.from_item() + +[case testSelfTypeRestrictedClassMethod] +from typing import TypeVar, Generic, Type + +T = TypeVar('T') +class C(Generic[T]): + @classmethod + def from_item(cls: Type[C[str]]) -> None: ... + +class DI(C[int]): ... +class DS(C[str]): ... + +DI().from_item() # E: Invalid self argument "Type[DI]" to class attribute function "from_item" with type "Callable[[Type[C[str]]], None]" +DS().from_item() +DI.from_item() # E: Invalid self argument "Type[DI]" to attribute function "from_item" with type "Callable[[Type[C[str]]], None]" +DS.from_item() +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeRestrictedMethodOverload] +from typing import TypeVar, Generic, overload, Tuple + +T = TypeVar('T') +class C(Generic[T]): + @overload + def from_item(self: C[str], item: str) -> None: ... + @overload + def from_item(self: C[int], item: Tuple[int]) -> None: ... + def from_item(self, item): + ... + +ci: C[int] +cs: C[str] +reveal_type(ci.from_item) # N: Revealed type is "def (item: Tuple[builtins.int])" +reveal_type(cs.from_item) # N: Revealed type is "def (item: builtins.str)" +[builtins fixtures/tuple.pyi] + +[case testSelfTypeRestrictedMethodOverloadFallback] +from typing import TypeVar, Generic, overload, Callable + +T = TypeVar('T') +class C(Generic[T]): + @overload + def from_item(self: C[str]) -> str: ... + @overload + def from_item(self, converter: Callable[[T], str]) -> str: ... + def from_item(self, converter): + ... + +ci: C[int] +cs: C[str] +reveal_type(cs.from_item()) # N: Revealed type is "builtins.str" +ci.from_item() # E: Missing positional argument "converter" in call to "from_item" of "C" + +def conv(x: int) -> str: ... +def bad(x: str) -> str: ... +reveal_type(ci.from_item(conv)) # N: Revealed type is "builtins.str" +ci.from_item(bad) # E: Argument 1 to "from_item" of "C" has incompatible type "Callable[[str], str]"; expected "Callable[[int], str]" + +[case testSelfTypeRestrictedMethodOverloadInit] +from typing import TypeVar +from lib import P, C + +reveal_type(P) # N: Revealed type is "Overload(def [T] (use_str: Literal[True]) -> lib.P[builtins.str], def [T] (use_str: Literal[False]) -> lib.P[builtins.int])" +reveal_type(P(use_str=True)) # N: Revealed type is "lib.P[builtins.str]" +reveal_type(P(use_str=False)) # N: Revealed type is "lib.P[builtins.int]" + +reveal_type(C) # N: Revealed type is "Overload(def [T] (item: T`1, use_tuple: Literal[False]) -> lib.C[T`1], def [T] (item: T`1, use_tuple: Literal[True]) -> lib.C[builtins.tuple[T`1, ...]])" +reveal_type(C(0, use_tuple=False)) # N: Revealed type is "lib.C[builtins.int]" +reveal_type(C(0, use_tuple=True)) # N: Revealed type is "lib.C[builtins.tuple[builtins.int, ...]]" + +T = TypeVar('T') +class SubP(P[T]): + pass + +SubP('no') # E: No overload variant of "SubP" matches argument type "str" \ + # N: Possible overload variants: \ + # N: def [T] __init__(self, use_str: Literal[True]) -> SubP[T] \ + # N: def [T] __init__(self, use_str: Literal[False]) -> SubP[T] + +# This is a bit unfortunate: we don't have a way to map the overloaded __init__ to subtype. +x = SubP(use_str=True) # E: Need type annotation for "x" +reveal_type(x) # N: Revealed type is "__main__.SubP[Any]" +y: SubP[str] = SubP(use_str=True) + +[file lib.pyi] +from typing import TypeVar, Generic, overload, Tuple +from typing_extensions import Literal + +T = TypeVar('T') +class P(Generic[T]): + @overload + def __init__(self: P[str], use_str: Literal[True]) -> None: ... + @overload + def __init__(self: P[int], use_str: Literal[False]) -> None: ... + +class C(Generic[T]): + @overload + def __init__(self: C[T], item: T, use_tuple: Literal[False]) -> None: ... + @overload + def __init__(self: C[Tuple[T, ...]], item: T, use_tuple: Literal[True]) -> None: ... +[builtins fixtures/bool.pyi] + +[case testSelfTypeRestrictedMethodOverloadInitFallBacks] +from lib import PFallBack, PFallBackAny + +t: bool +xx = PFallBack(t) # E: Need type annotation for "xx" +yy = PFallBackAny(t) # OK + +[file lib.pyi] +from typing import TypeVar, Generic, overload, Tuple, Any +from typing_extensions import Literal + +class PFallBack(Generic[T]): + @overload + def __init__(self: PFallBack[str], use_str: Literal[True]) -> None: ... + @overload + def __init__(self: PFallBack[int], use_str: Literal[False]) -> None: ... + @overload + def __init__(self, use_str: bool) -> None: ... + +class PFallBackAny(Generic[T]): + @overload + def __init__(self: PFallBackAny[str], use_str: Literal[True]) -> None: ... + @overload + def __init__(self: PFallBackAny[int], use_str: Literal[False]) -> None: ... + @overload + def __init__(self: PFallBackAny[Any], use_str: bool) -> None: ... +[builtins fixtures/bool.pyi] + +[case testSelfTypeRestrictedMethodOverloadInitBadTypeNoCrash] +from lib import P +P(0) +[file lib.pyi] +from typing import overload + +class P: + @overload + def __init__(self: Bad, x: int) -> None: ... # E: Name "Bad" is not defined + @overload + def __init__(self) -> None: ... + +[case testSelfTypeNarrowBinding] +from typing import TypeVar, List, Generic + +T = TypeVar('T') +S = TypeVar('S') + +class Base(Generic[T]): + def get_item(self: Base[List[S]]) -> S: ... + +class Sub(Base[List[int]]): ... +class BadSub(Base[int]): ... + +reveal_type(Sub().get_item()) # N: Revealed type is "builtins.int" +BadSub().get_item() # E: Invalid self argument "BadSub" to attribute function "get_item" with type "Callable[[Base[List[S]]], S]" +[builtins fixtures/list.pyi] + +[case testMixinAllowedWithProtocol] +from typing import TypeVar +from typing_extensions import Protocol + +class Resource(Protocol): + def close(self) -> int: ... + +class AtomicClose: + def atomic_close(self: Resource) -> int: + return self.close() + +T = TypeVar('T', bound=Resource) +class Copyable: + def copy(self: T) -> T: ... + +class File(AtomicClose, Copyable): + def close(self) -> int: + ... + +class Bad(AtomicClose, Copyable): + ... + +f: File +b: Bad +f.atomic_close() # OK +b.atomic_close() # E: Invalid self argument "Bad" to attribute function "atomic_close" with type "Callable[[Resource], int]" + +reveal_type(f.copy()) # N: Revealed type is "__main__.File" +b.copy() # E: Invalid self argument "Bad" to attribute function "copy" with type "Callable[[T], T]" +[builtins fixtures/tuple.pyi] + +[case testMixinProtocolSuper] +from typing import Protocol + +class Base(Protocol): + def func(self) -> int: + ... + +class TweakFunc: + def func(self: Base) -> int: + return reveal_type(super().func()) # E: Call to abstract method "func" of "Base" with trivial body via super() is unsafe \ + # N: Revealed type is "builtins.int" + +class Good: + def func(self) -> int: ... +class C(TweakFunc, Good): pass +C().func() # OK + +class Bad: + def func(self) -> str: ... +class CC(TweakFunc, Bad): pass # E: Definition of "func" in base class "TweakFunc" is incompatible with definition in base class "Bad" + +[case testBadClassLevelDecoratorHack] +from typing_extensions import Protocol +from typing import TypeVar, Any + +class FuncLike(Protocol): + __call__: Any +F = TypeVar('F', bound=FuncLike) + +class Test: + def _deco(func: F) -> F: ... + + @_deco + def meth(self, x: str) -> int: ... + +reveal_type(Test().meth) # N: Revealed type is "def (x: builtins.str) -> builtins.int" +Test()._deco # E: Invalid self argument "Test" to attribute function "_deco" with type "Callable[[F], F]" +[builtins fixtures/tuple.pyi] + +[case testSelfTypeTrickyExample] +from typing import * + +In = TypeVar('In') +Out = TypeVar('Out') +Mid = TypeVar('Mid') +NewOut = TypeVar('NewOut') + +class Lnk(Generic[In, Out]): + def test(self: Lnk[In, Mid], other: Lnk[Mid, NewOut]) -> Lnk[In, NewOut]: ... + +class X: pass +class Y: pass +class Z: pass + +a: Lnk[X, Y] = Lnk() +b: Lnk[Y, Z] = Lnk() + +a.test(b) +b.test(a) # E: Argument 1 to "test" of "Lnk" has incompatible type "Lnk[X, Y]"; expected "Lnk[Z, Y]" + +[case testSelfTypeReallyTrickyExample] +from typing import * + +In = TypeVar('In') +Out = TypeVar('Out') +Other = TypeVar('Other') + +_1 = TypeVar('_1') +_2 = TypeVar('_2') +__1 = TypeVar('__1') +__2 = TypeVar('__2') + +class Lnk(Generic[In, Out]): + @overload + def __rshift__(self, other: Lnk[Out, Other]) -> Lnk[In,Other]: ... + @overload + def __rshift__(self: Lnk[In, Tuple[_1, _2]], + other: Tuple[Lnk[_1, __1], Lnk[_2, __2]]) -> Lnk[In, Tuple[__1, __2]]: ... + def __rshift__(self: Any, other: Any) -> Any: + ... + +a: Lnk[str, Tuple[str, int]] = Lnk() +b: Lnk[str, int] = Lnk() +c: Lnk[int, float] = Lnk() + +d: Lnk[str, float] = b >> c # OK +e: Lnk[str, Tuple[int, float]] = a >> (b, c) # OK +f: Lnk[str, Tuple[float, int]] = a >> (c, b) # E: Unsupported operand types for >> ("Lnk[str, Tuple[str, int]]" and "Tuple[Lnk[int, float], Lnk[str, int]]") +[builtins fixtures/tuple.pyi] + +[case testSelfTypeMutuallyExclusiveRestrictions] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class Foo(Generic[T]): + def f1(self: Foo[str]) -> None: + self.f2() # E: Invalid self argument "Foo[str]" to attribute function "f2" with type "Callable[[Foo[int]], None]" + def f2(self: Foo[int]) -> None: + self.f1() # E: Invalid self argument "Foo[int]" to attribute function "f1" with type "Callable[[Foo[str]], None]" + +[case testSelfTypeStructureMetaclassMatch] +from typing import TypeVar, Type, Generic, cast + +Cls = TypeVar('Cls') +T = TypeVar('T') + +class Manager(Generic[Cls]): + def create(self: Manager[Type[T]]) -> T: ... + +class ModelMeta(type): + @property + def objects(cls: T) -> Manager[T]: ... + +class Model(metaclass=ModelMeta): + pass + +class Dog(Model): ... +class Cat(Model): ... + +c: Cat = Dog.objects.create() # E: Incompatible types in assignment (expression has type "Dog", variable has type "Cat") +d: Dog = Dog.objects.create() +[builtins fixtures/property.pyi] + +[case testSelfTypeProtocolMetaclassMatch] +from typing import Type, TypeVar, Protocol + +class HasX(Protocol): + x: int + +T = TypeVar('T', bound=HasX) + +class Meta(type): + def do_x(cls: Type[T]) -> T: + cls.x + return cls() + +class Good(metaclass=Meta): + x: int +class Bad(metaclass=Meta): + pass + +Good.do_x() +Bad.do_x() # E: Invalid self argument "Type[Bad]" to attribute function "do_x" with type "Callable[[Type[T]], T]" + +[case testSelfTypeProtocolClassmethodMatch] +from typing import Type, TypeVar, Protocol + +T = TypeVar('T') + +class HasDoX(Protocol): + @classmethod + def do_x(cls: Type[T]) -> T: + ... + +class Good: + @classmethod + def do_x(cls) -> 'Good': + ... + +class Bad: + @classmethod + def do_x(cls) -> Good: + ... + +good: HasDoX = Good() +bad: HasDoX = Bad() +[builtins fixtures/classmethod.pyi] +[out] +main:21: error: Incompatible types in assignment (expression has type "Bad", variable has type "HasDoX") +main:21: note: Following member(s) of "Bad" have conflicts: +main:21: note: Expected: +main:21: note: def do_x(cls) -> Bad +main:21: note: Got: +main:21: note: def do_x(cls) -> Good + +[case testSelfTypeNotSelfType] +# Friendlier error messages for common mistakes. See #2950 +class A: + def f(x: int) -> None: ... + def g(self: None) -> None: ... +[out] +main:3: error: Self argument missing for a non-static method (or an invalid type for self) +main:4: error: The erased type of self "None" is not a supertype of its class "__main__.A" + +[case testUnionPropertyField] +from typing import Union + +class A: + x: int + +class B: + @property + def x(self) -> int: return 1 + +class C: + @property + def x(self) -> int: return 1 + +ab: Union[A, B, C] +reveal_type(ab.x) # N: Revealed type is "builtins.int" +[builtins fixtures/property.pyi] + +[case testSelfTypeNoTypeVars] +from typing import Generic, List, Optional, TypeVar, Any + +Q = TypeVar("Q") +T = TypeVar("T", bound='Super[Any]') + +class Super(Generic[Q]): + @classmethod + def meth(cls, arg: List[T]) -> List[T]: + pass + +class Sub(Super[int]): ... + +def test(x: List[Sub]) -> None: + reveal_type(Sub.meth(x)) # N: Revealed type is "builtins.list[__main__.Sub]" +[builtins fixtures/isinstancelist.pyi] + +[case testSelfTypeNoTypeVarsRestrict] +from typing import Generic, TypeVar + +T = TypeVar('T') +S = TypeVar('S') + +class C(Generic[T]): + def limited(self: C[str], arg: S) -> S: ... + +reveal_type(C[str]().limited(0)) # N: Revealed type is "builtins.int" + +[case testSelfTypeMultipleTypeVars] +from typing import Generic, TypeVar, Tuple + +T = TypeVar('T') +S = TypeVar('S') +U = TypeVar('U') +V = TypeVar('V') + +class C(Generic[T]): + def magic(self: C[Tuple[S, U]]) -> Tuple[T, S, U]: ... + +class D(Generic[V]): + def f(self) -> None: + reveal_type(C[Tuple[V, str]]().magic()) # N: Revealed type is "Tuple[Tuple[V`1, builtins.str], V`1, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testSelfTypeOnUnion] +from typing import TypeVar, Union + +T = TypeVar('T') + +class A: + same: int + +class C: + def same(self: T) -> T: ... + +x: Union[A, C] +reveal_type(x.same) # N: Revealed type is "Union[builtins.int, def () -> __main__.C]" + +[case testSelfTypeOnUnionClassMethod] +from typing import TypeVar, Union, Type + +T = TypeVar('T') + +class A: + same: int + +class C: + @classmethod + def same(cls: Type[T]) -> T: ... + +x: Union[A, C] +reveal_type(x.same) # N: Revealed type is "Union[builtins.int, def () -> __main__.C]" +[builtins fixtures/classmethod.pyi] + +[case SelfTypeOverloadedClassMethod] +from lib import Base +from typing import overload, Tuple + +class Sub(Base): + @overload + @classmethod + def make(cls) -> Sub: ... + @overload + @classmethod + def make(cls, num: int) -> Tuple[Sub, ...]: ... + @classmethod + def make(cls, num=1): + ... + +class Other(Base): ... +class Double(Sub): ... + +reveal_type(Other.make()) # N: Revealed type is "__main__.Other" +reveal_type(Other.make(3)) # N: Revealed type is "builtins.tuple[__main__.Other, ...]" +reveal_type(Double.make()) # N: Revealed type is "__main__.Sub" +reveal_type(Double.make(3)) # N: Revealed type is "builtins.tuple[__main__.Sub, ...]" +[file lib.pyi] +from typing import overload, TypeVar, Type, Tuple + +T = TypeVar('T', bound=Base) + +class Base: + @overload + @classmethod + def make(cls: Type[T]) -> T: ... + @overload + @classmethod + def make(cls: Type[T], num: int) -> Tuple[T, ...]: ... +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeClassMethodOnUnion] +from typing import Type, Union, TypeVar + +T = TypeVar('T') + +class A: + @classmethod + def meth(cls: Type[T]) -> T: ... +class B(A): ... +class C(A): ... + +t: Type[Union[B, C]] +reveal_type(t.meth) # N: Revealed type is "Union[def () -> __main__.B, def () -> __main__.C]" +x = t.meth() +reveal_type(x) # N: Revealed type is "Union[__main__.B, __main__.C]" +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeClassMethodOnUnionGeneric] +from typing import Type, Union, TypeVar, Generic + +T = TypeVar('T') +S = TypeVar('S') + +class A(Generic[T]): + @classmethod + def meth(cls: Type[S]) -> S: ... + +t: Type[Union[A[int], A[str]]] +x = t.meth() +reveal_type(x) # N: Revealed type is "Union[__main__.A[builtins.int], __main__.A[builtins.str]]" +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeClassMethodOnUnionList] +from typing import Type, Union, TypeVar, List + +T = TypeVar('T') + +class A: + @classmethod + def meth(cls: Type[T]) -> List[T]: ... +class B(A): ... +class C(A): ... + +t: Type[Union[B, C]] +x = t.meth()[0] +reveal_type(x) # N: Revealed type is "Union[__main__.B, __main__.C]" +[builtins fixtures/isinstancelist.pyi] + +[case testSelfTypeClassMethodOverloadedOnInstance] +from typing import Optional, Type, TypeVar, overload, Union + +Id = int + +A = TypeVar("A", bound='AClass') + +class AClass: + @overload + @classmethod + def delete(cls: Type[A], id: Id, id2: Id) -> Optional[int]: ... + + @overload + @classmethod + def delete(cls: Type[A], id: A, id2: None = None) -> Optional[int]: ... + + @classmethod + def delete(cls: Type[A], id: Union[A, Id], id2: Optional[Id] = None) -> Optional[int]: + ... + +def foo(x: Type[AClass]) -> None: + reveal_type(x.delete) # N: Revealed type is "Overload(def (id: builtins.int, id2: builtins.int) -> Union[builtins.int, None], def (id: __main__.AClass, id2: None =) -> Union[builtins.int, None])" + y = x() + reveal_type(y.delete) # N: Revealed type is "Overload(def (id: builtins.int, id2: builtins.int) -> Union[builtins.int, None], def (id: __main__.AClass, id2: None =) -> Union[builtins.int, None])" + y.delete(10, 20) + y.delete(y) + +def bar(x: AClass) -> None: + reveal_type(x.delete) # N: Revealed type is "Overload(def (id: builtins.int, id2: builtins.int) -> Union[builtins.int, None], def (id: __main__.AClass, id2: None =) -> Union[builtins.int, None])" + x.delete(10, 20) +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeBadTypeIgnoredInConstructor] +class Base: ... +class Sub(Base): + def __init__(self: Base) -> None: ... + +reveal_type(Sub()) # N: Revealed type is "__main__.Sub" + +[case testSelfTypeBadTypeIgnoredInConstructorGeneric] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class Base(Generic[T]): ... +class Sub(Base[T]): + def __init__(self: Base[T], item: T) -> None: ... + +reveal_type(Sub(42)) # N: Revealed type is "__main__.Sub[builtins.int]" + +[case testSelfTypeBadTypeIgnoredInConstructorOverload] +from typing import overload + +class Base: ... +class Sub(Base): + @overload + def __init__(self: Sub, item: int) -> None: ... + @overload + def __init__(self: Base) -> None: ... + def __init__(self, item=None): + ... + +reveal_type(Sub) # N: Revealed type is "Overload(def (item: builtins.int) -> __main__.Sub, def () -> __main__.Sub)" + +[case testSelfTypeBadTypeIgnoredInConstructorAbstract] +from abc import abstractmethod +from typing import Protocol + +class Blah(Protocol): + @abstractmethod + def something(self) -> None: ... + +class Concrete(Blah): + def __init__(self: Blah) -> None: ... + def something(self) -> None: ... + +Concrete() # OK + +[case testSelfTypeGenericClassNoClashInstanceMethod] +from typing import TypeVar, Generic + +M = TypeVar("M") +T = TypeVar("T") +S = TypeVar("S") + +class Descriptor(Generic[M]): ... + +class BaseWrapper(Generic[M]): + def create_wrapper(self: T, metric_descriptor: Descriptor[M]) -> T: ... +class SubWrapper(BaseWrapper[M]): ... + +def build_wrapper(descriptor: Descriptor[M]) -> BaseWrapper[M]: + wrapper: BaseWrapper[M] + return wrapper.create_wrapper(descriptor) + +def build_sub_wrapper(descriptor: Descriptor[S]) -> SubWrapper[S]: + wrapper: SubWrapper[S] + x = wrapper.create_wrapper(descriptor) + reveal_type(x) # N: Revealed type is "__main__.SubWrapper[S`-1]" + return x + +[case testSelfTypeGenericClassNoClashClassMethod] +from typing import TypeVar, Generic, Type + +M = TypeVar("M") +T = TypeVar("T") +S = TypeVar("S") + +class Descriptor(Generic[M]): ... + +class BaseWrapper(Generic[M]): + @classmethod + def create_wrapper(cls: Type[T], metric_descriptor: Descriptor[M]) -> T: ... +class SubWrapper(BaseWrapper[M]): ... + +def build_wrapper(descriptor: Descriptor[M]) -> BaseWrapper[M]: + wrapper_cls: Type[BaseWrapper[M]] + return wrapper_cls.create_wrapper(descriptor) + +def build_sub_wrapper(descriptor: Descriptor[S]) -> SubWrapper[S]: + wrapper_cls: Type[SubWrapper[S]] + x = wrapper_cls.create_wrapper(descriptor) + reveal_type(x) # N: Revealed type is "__main__.SubWrapper[S`-1]" + return x +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeGenericClassNoClashClassMethodClassObject] +from typing import TypeVar, Generic, Type + +M = TypeVar("M") +T = TypeVar("T") + +class Descriptor(Generic[M]): ... + +class BaseWrapper(Generic[M]): + @classmethod + def create_wrapper(cls: Type[T], metric_descriptor: Descriptor[M]) -> T: ... +class SubWrapper(BaseWrapper[M]): ... + +def build_wrapper(descriptor: Descriptor[M]) -> BaseWrapper[M]: + return BaseWrapper.create_wrapper(descriptor) + +def build_sub_wrapper(descriptor: Descriptor[M]) -> SubWrapper[M]: + x = SubWrapper.create_wrapper(descriptor) + reveal_type(x) # N: Revealed type is "__main__.SubWrapper[M`-1]" + return x + +def build_wrapper_non_gen(descriptor: Descriptor[int]) -> BaseWrapper[str]: + return BaseWrapper.create_wrapper(descriptor) # E: Argument 1 to "create_wrapper" of "BaseWrapper" has incompatible type "Descriptor[int]"; expected "Descriptor[str]" + +def build_sub_wrapper_non_gen(descriptor: Descriptor[int]) -> SubWrapper[str]: + return SubWrapper.create_wrapper(descriptor) # E: Argument 1 to "create_wrapper" of "BaseWrapper" has incompatible type "Descriptor[int]"; expected "Descriptor[str]" +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeInGenericClassUsedFromAnotherGenericClass1] +from typing import TypeVar, Generic, Iterator, List, Tuple + +_T_co = TypeVar("_T_co", covariant=True) +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +S = TypeVar("S") + +class Z(Iterator[_T_co]): + def __new__(cls, + __iter1: List[_T1], + __iter2: List[_T2]) -> Z[Tuple[_T1, _T2]]: ... + def __iter__(self: S) -> S: ... + def __next__(self) -> _T_co: ... + +T = TypeVar('T') + +class C(Generic[T]): + a: List[T] + b: List[str] + + def f(self) -> None: + for x, y in Z(self.a, self.b): + reveal_type((x, y)) # N: Revealed type is "Tuple[T`1, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testEnumerateReturningSelfFromIter] +from typing import Generic, Iterable, Iterator, TypeVar, Tuple + +T = TypeVar("T") +KT = TypeVar("KT") +VT = TypeVar("VT") +Self = TypeVar("Self") + +class enumerate(Iterator[Tuple[int, T]], Generic[T]): + def __init__(self, iterable: Iterable[T]) -> None: ... + def __iter__(self: Self) -> Self: ... + def __next__(self) -> Tuple[int, T]: ... + +class Dict(Generic[KT, VT]): + def update(self, __m: Iterable[Tuple[KT, VT]]) -> None: ... + +class ThingCollection(Generic[T]): + collection: Iterable[Tuple[float, T]] + index: Dict[int, T] + + def do_thing(self) -> None: + self.index.update((idx, c) for idx, (k, c) in enumerate(self.collection)) +[builtins fixtures/tuple.pyi] + +[case testDequeReturningSelfFromCopy] +# Tests a bug with generic self types identified in issue #12641 +from typing import Generic, Sequence, TypeVar + +T = TypeVar("T") +Self = TypeVar("Self") + +class deque(Sequence[T]): + def copy(self: Self) -> Self: ... + +class List(Sequence[T]): ... + +class Test(Generic[T]): + def test(self) -> None: + a: deque[List[T]] + # previously this failed with 'Incompatible types in assignment (expression has type "deque[List[List[T]]]", variable has type "deque[List[T]]")' + b: deque[List[T]] = a.copy() + +[case testTypingSelfBasic] +from typing import Self, List + +class C: + attr: List[Self] + def meth(self) -> List[Self]: ... + def test(self) -> Self: + if bool(): + return C() # E: Incompatible return value type (got "C", expected "Self") + else: + return self +class D(C): ... + +reveal_type(C.meth) # N: Revealed type is "def [Self <: __main__.C] (self: Self`1) -> builtins.list[Self`1]" +C.attr # E: Access to generic instance variables via class is ambiguous +reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.D]" +reveal_type(D().attr) # N: Revealed type is "builtins.list[__main__.D]" + +[case testTypingSelfInvalidLocations] +from typing import Self, Callable + +var: Self # E: Self type is only allowed in annotations within class definition +reveal_type(var) # N: Revealed type is "Any" + +def foo() -> Self: ... # E: Self type is only allowed in annotations within class definition +reveal_type(foo) # N: Revealed type is "def () -> Any" + +bad: Callable[[Self], Self] # E: Self type is only allowed in annotations within class definition +reveal_type(bad) # N: Revealed type is "def (Any) -> Any" + +def func() -> None: + var: Self # E: Self type is only allowed in annotations within class definition + +class C(Self): ... # E: Self type is only allowed in annotations within class definition + +[case testTypingSelfInvalidArgs] +from typing import Self, List + +class C: + x: Self[int] # E: Self type cannot have type arguments + def meth(self) -> List[Self[int]]: # E: Self type cannot have type arguments + ... + +[case testTypingSelfConflict] +from typing import Self, TypeVar, Tuple + +T = TypeVar("T") +class C: + def meth(self: T) -> Tuple[Self, T]: ... # E: Method cannot have explicit self annotation and Self type +reveal_type(C().meth()) # N: Revealed type is "Tuple[Never, __main__.C]" +[builtins fixtures/property.pyi] + +[case testTypingSelfProperty] +from typing import Self, Tuple +class C: + @property + def attr(self) -> Tuple[Self, ...]: ... +class D(C): ... + +reveal_type(D().attr) # N: Revealed type is "builtins.tuple[__main__.D, ...]" +[builtins fixtures/property.pyi] + +[case testTypingSelfCallableVar] +from typing import Self, Callable + +class C: + x: Callable[[Self], Self] + def meth(self) -> Callable[[Self], Self]: ... +class D(C): ... + +reveal_type(C().x) # N: Revealed type is "def (__main__.C) -> __main__.C" +reveal_type(D().x) # N: Revealed type is "def (__main__.D) -> __main__.D" +reveal_type(D().meth()) # N: Revealed type is "def (__main__.D) -> __main__.D" + +[case testTypingSelfClassMethod] +from typing import Self + +class C: + @classmethod + def meth(cls) -> Self: ... + @staticmethod + def bad() -> Self: ... # E: Static methods cannot use Self type \ + # E: A function returning TypeVar should receive at least one argument containing the same TypeVar \ + # N: Consider using the upper bound "C" instead + +class D(C): ... +reveal_type(D.meth()) # N: Revealed type is "__main__.D" +reveal_type(D.bad()) # N: Revealed type is "Never" +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfOverload] +from typing import Self, overload, Union + +class C: + @overload + def foo(self, other: Self) -> Self: ... + @overload + def foo(self, other: int) -> int: ... + def foo(self, other: Union[Self, int]) -> Union[Self, int]: + return other +class D(C): ... +reveal_type(D().foo) # N: Revealed type is "Overload(def (other: __main__.D) -> __main__.D, def (other: builtins.int) -> builtins.int)" + +[case testTypingSelfNestedInAlias] +from typing import Generic, Self, TypeVar, List, Tuple + +T = TypeVar("T") +Pairs = List[Tuple[T, T]] + +class C(Generic[T]): + def pairs(self) -> Pairs[Self]: ... +class D(C[T]): ... +reveal_type(D[int]().pairs()) # N: Revealed type is "builtins.list[Tuple[__main__.D[builtins.int], __main__.D[builtins.int]]]" +[builtins fixtures/tuple.pyi] + +[case testTypingSelfOverrideVar] +from typing import Self, TypeVar, Generic + +T = TypeVar("T") +class C(Generic[T]): + x: Self + +class D(C[int]): + x: D +class Bad(C[int]): + x: C[int] # E: Incompatible types in assignment (expression has type "C[int]", base class "C" defined the type as "Bad") + +[case testTypingSelfOverrideVarMulti] +from typing import Self + +class C: + x: Self +class D: + x: C +class E: + x: Good + +class Bad(D, C): # E: Definition of "x" in base class "D" is incompatible with definition in base class "C" + ... +class Good(E, C): + ... + +[case testTypingSelfAlternativeGenericConstructor] +from typing import Self, Generic, TypeVar, Tuple + +T = TypeVar("T") +class C(Generic[T]): + def __init__(self, val: T) -> None: ... + @classmethod + def pair(cls, val: T) -> Tuple[Self, Self]: + return (cls(val), C(val)) # E: Incompatible return value type (got "Tuple[Self, C[T]]", expected "Tuple[Self, Self]") + +class D(C[int]): pass +reveal_type(C.pair(42)) # N: Revealed type is "Tuple[__main__.C[builtins.int], __main__.C[builtins.int]]" +reveal_type(D.pair("no")) # N: Revealed type is "Tuple[__main__.D, __main__.D]" \ + # E: Argument 1 to "pair" of "C" has incompatible type "str"; expected "int" +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfMixedTypeVars] +from typing import Self, TypeVar, Generic, Tuple + +T = TypeVar("T") +S = TypeVar("S") + +class C(Generic[T]): + def meth(self, arg: S) -> Tuple[Self, S, T]: ... + +class D(C[int]): ... + +c: C[int] +d: D +reveal_type(c.meth("test")) # N: Revealed type is "Tuple[__main__.C[builtins.int], builtins.str, builtins.int]" +reveal_type(d.meth("test")) # N: Revealed type is "Tuple[__main__.D, builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypingSelfRecursiveInit] +from typing import Self + +class C: + def __init__(self, other: Self) -> None: ... +class D(C): ... + +reveal_type(C) # N: Revealed type is "def (other: __main__.C) -> __main__.C" +reveal_type(D) # N: Revealed type is "def (other: __main__.D) -> __main__.D" + +[case testTypingSelfCorrectName] +from typing import Self, List + +class C: + Self = List[C] + def meth(self) -> Self: ... +reveal_type(C.meth) # N: Revealed type is "def (self: __main__.C) -> builtins.list[__main__.C]" + +[case testTypingSelfClassVar] +from typing import Self, ClassVar, Generic, TypeVar + +class C: + DEFAULT: ClassVar[Self] +reveal_type(C.DEFAULT) # N: Revealed type is "__main__.C" + +T = TypeVar("T") +class G(Generic[T]): + BAD: ClassVar[Self] # E: ClassVar cannot contain Self type in generic classes +reveal_type(G.BAD) # N: Revealed type is "__main__.G[Any]" + +[case testTypingSelfMetaClassDisabled] +from typing import Self + +class Meta(type): + def meth(cls) -> Self: ... # E: Self type cannot be used in a metaclass + +[case testTypingSelfNonAnnotationUses] +from typing import Self, List, cast + +class C: + A = List[Self] # E: Self type cannot be used in type alias target + B = cast(Self, ...) + def meth(self) -> A: ... + +class D(C): ... +reveal_type(D().meth()) # N: Revealed type is "builtins.list[Any]" +reveal_type(D().B) # N: Revealed type is "__main__.D" + +[case testTypingSelfInternalSafe] +from typing import Self + +class C: + x: Self + def __init__(self, x: C) -> None: + self.x = x # E: Incompatible types in assignment (expression has type "C", variable has type "Self") + +[case testTypingSelfRedundantAllowed] +from typing import Self, Type + +class C: + def f(self: Self) -> Self: + d: Defer + class Defer: ... + return self + + @classmethod + def g(cls: Type[Self]) -> Self: + d: DeferAgain + class DeferAgain: ... + return cls() +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfRedundantAllowed_pep585] +# flags: --python-version 3.9 +from typing import Self + +class C: + def f(self: Self) -> Self: + d: Defer + class Defer: ... + return self + + @classmethod + def g(cls: type[Self]) -> Self: + d: DeferAgain + class DeferAgain: ... + return cls() +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfRedundantWarning] +# mypy: enable-error-code="redundant-self" + +from typing import Self, Type + +class C: + def copy(self: Self) -> Self: # E: Redundant "Self" annotation for the first method argument + d: Defer + class Defer: ... + return self + + @classmethod + def g(cls: Type[Self]) -> Self: # E: Redundant "Self" annotation for the first method argument + d: DeferAgain + class DeferAgain: ... + return cls() +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfRedundantWarning_pep585] +# flags: --python-version 3.9 +# mypy: enable-error-code="redundant-self" + +from typing import Self + +class C: + def copy(self: Self) -> Self: # E: Redundant "Self" annotation for the first method argument + d: Defer + class Defer: ... + return self + + @classmethod + def g(cls: type[Self]) -> Self: # E: Redundant "Self" annotation for the first method argument + d: DeferAgain + class DeferAgain: ... + return cls() +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfAssertType] +from typing import Self, assert_type + +class C: + def foo(self) -> None: + assert_type(self, Self) # E: Expression is of type "C", not "Self" + assert_type(C(), Self) # E: Expression is of type "C", not "Self" + + def bar(self) -> Self: + assert_type(self, Self) # OK + assert_type(C(), Self) # E: Expression is of type "C", not "Self" + return self + +[case testTypingSelfTypeVarClash] +from typing import Self, TypeVar, Tuple + +S = TypeVar("S") +class C: + def bar(self) -> Self: ... + def foo(self, x: S) -> Tuple[Self, S]: ... + +reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`1, x: S`2) -> Tuple[Self`1, S`2]" +reveal_type(C().foo(42)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypingSelfTypeVarClashAttr] +from typing import Self, TypeVar, Tuple, Callable + +class Defer(This): ... + +S = TypeVar("S") +class C: + def bar(self) -> Self: ... + foo: Callable[[S, Self], Tuple[Self, S]] + +reveal_type(C().foo) # N: Revealed type is "def [S] (S`1, __main__.C) -> Tuple[__main__.C, S`1]" +reveal_type(C().foo(42, C())) # N: Revealed type is "Tuple[__main__.C, builtins.int]" +class This: ... +[builtins fixtures/tuple.pyi] + +[case testTypingSelfAttrOldVsNewStyle] +from typing import Self, TypeVar + +T = TypeVar("T", bound='C') +class C: + x: Self + def foo(self: T) -> T: + return self.x + def bar(self: T) -> T: + self.x = self + return self + def baz(self: Self) -> None: + self.x = self + def bad(self) -> None: + # This is unfortunate, but required by PEP 484 + self.x = self # E: Incompatible types in assignment (expression has type "C", variable has type "Self") + +[case testTypingSelfClashInBodies] +from typing import Self, TypeVar + +T = TypeVar("T") +class C: + def very_bad(self, x: T) -> None: + self.x = x # E: Incompatible types in assignment (expression has type "T", variable has type "Self") + x: Self + def baz(self: Self, x: T) -> None: + y: T = x + +[case testTypingSelfClashUnrelated] +from typing import Self, Generic, TypeVar + +class B: ... + +T = TypeVar("T", bound=B) +class C(Generic[T]): + def __init__(self, val: T) -> None: + self.val = val + def foo(self) -> Self: ... + +def test(x: C[T]) -> T: + reveal_type(x.val) # N: Revealed type is "T`-1" + return x.val + +[case testTypingSelfGenericBound] +from typing import Self, Generic, TypeVar + +T = TypeVar("T") +class C(Generic[T]): + val: T + def foo(self) -> Self: + reveal_type(self.val) # N: Revealed type is "T`1" + return self + +[case testTypingSelfDifferentImport] +import typing as t + +class Foo: + def foo(self) -> t.Self: + return self + @classmethod + def bar(cls) -> t.Self: + return cls() +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfAllowAliasUseInFinalClasses] +from typing import Self, final + +@final +class C: + def meth(self) -> Self: + return C() # OK for final classes + +[case testTypingSelfCallableClassVar] +from typing import Self, ClassVar, Callable, TypeVar + +class C: + f: ClassVar[Callable[[Self], Self]] +class D(C): ... + +reveal_type(D.f) # N: Revealed type is "def (__main__.D) -> __main__.D" +reveal_type(D().f) # N: Revealed type is "def () -> __main__.D" + +[case testSelfTypeCallableClassVarOldStyle] +from typing import ClassVar, Callable, TypeVar + +T = TypeVar("T") +class C: + f: ClassVar[Callable[[T], T]] + +class D(C): ... + +reveal_type(D.f) # N: Revealed type is "def [T] (T`1) -> T`1" +reveal_type(D().f) # N: Revealed type is "def () -> __main__.D" + +[case testTypingSelfOnSuperTypeVarValues] +from typing import Self, Generic, TypeVar + +T = TypeVar("T", int, str) + +class B: + def copy(self) -> Self: ... +class C(B, Generic[T]): + def copy(self) -> Self: + inst = super().copy() + reveal_type(inst) # N: Revealed type is "Self`0" + return inst + +[case testTypingSelfWithValuesExpansion] +from typing import Self, Generic, TypeVar + +class A: pass +class B: pass +T = TypeVar("T", A, B) + +class C(Generic[T]): + val: T + def foo(self, x: T) -> None: ... + def bar(self, x: T) -> Self: + reveal_type(self.foo) # N: Revealed type is "def (x: __main__.A)" \ + # N: Revealed type is "def (x: __main__.B)" + self.foo(x) + return self + def baz(self: Self, x: T) -> None: + reveal_type(self.val) # N: Revealed type is "__main__.A" \ + # N: Revealed type is "__main__.B" + self.val = x + +[case testNarrowSelfType] +from typing import Self, Union + +class A: ... +class B: + def f1(self, v: Union[Self, A]) -> A: + if isinstance(v, B): + return A() + else: + return v + def f2(self, v: Union[Self, A]) -> A: + if isinstance(v, B): + return A() + else: + return B() # E: Incompatible return value type (got "B", expected "A") + +[builtins fixtures/isinstancelist.pyi] + +[case testAttributeOnSelfAttributeInSubclass] +from typing import List, Self + +class A: + x: Self + xs: List[Self] + +class B(A): + extra: int + + def meth(self) -> None: + reveal_type(self.x) # N: Revealed type is "Self`0" + reveal_type(self.xs[0]) # N: Revealed type is "Self`0" + reveal_type(self.x.extra) # N: Revealed type is "builtins.int" + reveal_type(self.xs[0].extra) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testSelfTypesWithParamSpecExtract] +from typing import Any, Callable, Generic, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec("P") +F = TypeVar("F", bound=Callable[..., Any]) +class Example(Generic[F]): + def __init__(self, fn: F) -> None: + ... + def __call__(self: Example[Callable[P, Any]], *args: P.args, **kwargs: P.kwargs) -> None: + ... + +def test_fn(a: int, b: str) -> None: + ... + +example = Example(test_fn) +example() # E: Missing positional arguments "a", "b" in call to "__call__" of "Example" +example(1, "b") # OK +[builtins fixtures/list.pyi] + +[case testSelfTypesWithParamSpecInfer] +from typing import TypeVar, Protocol, Type, Callable +from typing_extensions import ParamSpec + +R = TypeVar("R", covariant=True) +P = ParamSpec("P") +class AsyncP(Protocol[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: + ... + +class Async: + @classmethod + def async_func(cls: Type[AsyncP[P]]) -> Callable[P, int]: + ... + +class Add(Async): + def meth(self, x: int, y: int) -> None: ... + +reveal_type(Add.async_func()) # N: Revealed type is "def (x: builtins.int, y: builtins.int) -> builtins.int" +reveal_type(Add().async_func()) # N: Revealed type is "def (x: builtins.int, y: builtins.int) -> builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testSelfTypeMethodOnClassObject] +from typing import Self + +class Object: # Needed to mimic object in typeshed + ref: Self + +class Foo: + def foo(self) -> Self: + return self + +class Ben(Object): + MY_MAP = { + "foo": Foo.foo, + } + @classmethod + def doit(cls) -> Foo: + reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`4) -> Self`4]" + foo_method = cls.MY_MAP["foo"] + return foo_method(Foo()) +[builtins fixtures/isinstancelist.pyi] + +[case testSelfTypeOnGenericClassObjectNewStyleBound] +from typing import Generic, TypeVar, Self + +T = TypeVar("T") +S = TypeVar("S") +class B(Generic[T, S]): + def copy(self) -> Self: ... + +b: B[int, str] +reveal_type(B.copy(b)) # N: Revealed type is "__main__.B[builtins.int, builtins.str]" + +class C(B[T, S]): ... + +c: C[int, str] +reveal_type(C.copy(c)) # N: Revealed type is "__main__.C[builtins.int, builtins.str]" + +B.copy(42) # E: Value of type variable "Self" of "copy" of "B" cannot be "int" +C.copy(42) # E: Value of type variable "Self" of "copy" of "B" cannot be "int" +[builtins fixtures/tuple.pyi] + +[case testRecursiveSelfTypeCallMethodNoCrash] +from typing import Callable, TypeVar + +T = TypeVar("T") +class Partial: + def __call__(self: Callable[..., T]) -> T: ... + +class Partial2: + def __call__(self: Callable[..., T], x: T) -> T: ... + +p: Partial +reveal_type(p()) # N: Revealed type is "Never" +p2: Partial2 +reveal_type(p2(42)) # N: Revealed type is "builtins.int" + +[case testAccessingSelfClassVarInClassMethod] +from typing import Self, ClassVar, Type, TypeVar + +T = TypeVar("T", bound="Foo") + +class Foo: + instance: ClassVar[Self] + @classmethod + def get_instance(cls) -> Self: + return reveal_type(cls.instance) # N: Revealed type is "Self`0" + @classmethod + def get_instance_old(cls: Type[T]) -> T: + return reveal_type(cls.instance) # N: Revealed type is "T`-1" + +class Bar(Foo): + extra: int + + @classmethod + def get_instance(cls) -> Self: + reveal_type(cls.instance.extra) # N: Revealed type is "builtins.int" + return cls.instance + + @classmethod + def other(cls) -> None: + reveal_type(cls.instance) # N: Revealed type is "Self`0" + reveal_type(cls.instance.extra) # N: Revealed type is "builtins.int" + +reveal_type(Bar.instance) # N: Revealed type is "__main__.Bar" +[builtins fixtures/classmethod.pyi] + +[case testAccessingSelfClassVarInClassMethodTuple] +from typing import Self, ClassVar, Tuple + +class C(Tuple[int, str]): + x: Self + y: ClassVar[Self] + + @classmethod + def bar(cls) -> None: + reveal_type(cls.y) # N: Revealed type is "Self`0" + @classmethod + def bar_self(self) -> Self: + return reveal_type(self.y) # N: Revealed type is "Self`0" + +c: C +reveal_type(c.x) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C]" +reveal_type(c.y) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C]" +reveal_type(C.y) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C]" +C.x # E: Access to generic instance variables via class is ambiguous +[builtins fixtures/classmethod.pyi] + +[case testAccessingTypingSelfUnion] +from typing import Self, Union + +class C: + x: Self +class D: + x: int +x: Union[C, D] +reveal_type(x.x) # N: Revealed type is "Union[__main__.C, builtins.int]" + +[case testCallableProtocolTypingSelf] +from typing import Protocol, Self + +class MyProtocol(Protocol): + __name__: str + + def __call__( + self: Self, + ) -> None: ... + +def test() -> None: ... +value: MyProtocol = test + +[case testCallableProtocolOldSelf] +from typing import Protocol, TypeVar + +Self = TypeVar("Self", bound="MyProtocol") + +class MyProtocol(Protocol): + __name__: str + + def __call__( + self: Self, + ) -> None: ... + +def test() -> None: ... +value: MyProtocol = test + +[case testSelfTypeUnionIter] +from typing import Self, Iterator, Generic, TypeVar, Union + +T = TypeVar("T") + +class range(Generic[T]): + def __iter__(self) -> Self: ... + def __next__(self) -> T: ... + +class count: + def __iter__(self) -> Iterator[int]: ... + +def foo(x: Union[range[int], count]) -> None: + for item in x: + reveal_type(item) # N: Revealed type is "builtins.int" + +[case testGenericDescriptorWithSelfTypeAnnotationsAndOverloads] +from __future__ import annotations +from typing import Any, overload, Callable, TypeVar, Generic, ParamSpec +from typing_extensions import Concatenate + +C = TypeVar("C", bound=Callable[..., Any]) +S = TypeVar("S") +P = ParamSpec("P") +R = TypeVar("R") + +class Descriptor(Generic[C]): + def __init__(self, impl: C) -> None: ... + + @overload + def __get__( + self: Descriptor[C], instance: None, owner: type | None + ) -> Descriptor[C]: ... + + @overload + def __get__( + self: Descriptor[Callable[Concatenate[S, P], R]], instance: S, owner: type | None, + ) -> Callable[P, R]: ... + + def __get__(self, *args, **kwargs): ... + +class Test: + @Descriptor + def method(self, foo: int, bar: str) -> bytes: ... + +reveal_type(Test().method) # N: Revealed type is "def (foo: builtins.int, bar: builtins.str) -> builtins.bytes" + +class Test2: + @Descriptor + def method(self, foo: int, *, bar: str) -> bytes: ... + +reveal_type(Test2().method) # N: Revealed type is "def (foo: builtins.int, *, bar: builtins.str) -> builtins.bytes" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-semanal-error.test b/test-data/unit/check-semanal-error.test new file mode 100644 index 000000000000..d7ab272aed6c --- /dev/null +++ b/test-data/unit/check-semanal-error.test @@ -0,0 +1,183 @@ +-- Type checking after an error during semantic analysis +-- ----------------------------------------------------- +-- +-- This tests both the semantic analyzer (that it does not generate +-- corrupt state on error) and the type checker (that it can deal with +-- whatever state the semantic analyzer sets up). + +-- TODO: +-- - invalid type in annotation +-- - invalid function comment type annotation +-- - invalid multiple assignment type annotation +-- - using a type variable as a value +-- - using special names defined in typing as values + +[case testMissingModuleImport1] +import m # E +m.foo() +m.x = m.y +1() # E +[out] +main:1: error: Cannot find implementation or library stub for module named "m" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:4: error: "int" not callable + +[case testMissingModuleImport2] +from m import x # E +x.foo() +x.a = x.b +1() # E +[out] +main:1: error: Cannot find implementation or library stub for module named "m" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:4: error: "int" not callable + +[case testMissingModuleImport3] +from m import * # E +x # E +1() # E +[out] +main:1: error: Cannot find implementation or library stub for module named "m" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Name "x" is not defined +main:3: error: "int" not callable + +[case testInvalidBaseClass1] +class A(X): # E: Name "X" is not defined + x = 1 +A().foo(1) +A().x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testInvalidBaseClass2] + +X = 1 +class A(X): # E + x = 1 +A().foo(1) +A().x = '' # E +[out] +main:3: error: Variable "__main__.X" is not valid as a type +main:3: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +main:3: error: Invalid base class "X" +main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") + + +[case testInvalidNumberOfTypeArgs] +from typing import TypeVar +T = TypeVar('T') +class C: # Forgot to add type params here + def __init__(self, t: T) -> None: pass +c = C(t=3) # type: C[int] # E: "C" expects no type arguments, but 1 given + +[case testBreakOutsideLoop] +break # E: "break" outside loop + +[case testContinueOutsideLoop] +continue # E: "continue" outside loop + +[case testYieldOutsideFunction] +yield # E: "yield" outside function +x = 1 +yield from x # E: "yield from" outside function +[(yield 1) for _ in x] # E: "yield" inside comprehension or generator expression +{(yield 1) for _ in x} # E: "yield" inside comprehension or generator expression +{i: (yield 1) for i in x} # E: "yield" inside comprehension or generator expression +((yield 1) for _ in x) # E: "yield" inside comprehension or generator expression +y = 1 +[(yield from x) for _ in y] # E: "yield from" inside comprehension or generator expression +{(yield from x) for _ in y} # E: "yield from" inside comprehension or generator expression +{i: (yield from x) for i in y} # E: "yield from" inside comprehension or generator expression +((yield from x) for _ in y) # E: "yield from" inside comprehension or generator expression +def f(y): + [x for x in (yield y)] + {x for x in (yield y)} + {x: x for x in (yield y)} + (x for x in (yield y)) + [x for x in (yield from y)] + {x for x in (yield from y)} + {x: x for x in (yield from y)} + (x for x in (yield from y)) +def g(y): + [(yield 1) for _ in y] # E: "yield" inside comprehension or generator expression + {(yield 1) for _ in y} # E: "yield" inside comprehension or generator expression + {i: (yield 1) for i in y} # E: "yield" inside comprehension or generator expression + ((yield 1) for _ in y) # E: "yield" inside comprehension or generator expression + lst = 1 + [(yield from lst) for _ in y] # E: "yield from" inside comprehension or generator expression + {(yield from lst) for _ in y} # E: "yield from" inside comprehension or generator expression + {i: (yield from lst) for i in y} # E: "yield from" inside comprehension or generator expression + ((yield from lst) for _ in y) # E: "yield from" inside comprehension or generator expression +def h(y): + lst = 1 + [x for x in lst if (yield y)] # E: "yield" inside comprehension or generator expression + {x for x in lst if (yield y)} # E: "yield" inside comprehension or generator expression + {x: x for x in lst if (yield y)} # E: "yield" inside comprehension or generator expression + (x for x in lst if (yield y)) # E: "yield" inside comprehension or generator expression + lst = 1 + [x for x in lst if (yield from y)] # E: "yield from" inside comprehension or generator expression + {x for x in lst if (yield from y)} # E: "yield from" inside comprehension or generator expression + {x: x for x in lst if (yield from y)} # E: "yield from" inside comprehension or generator expression + (x for x in lst if (yield from y)) # E: "yield from" inside comprehension or generator expression + +[case testImportFuncDup] + +import m +def m() -> None: ... # E: Name "m" already defined (by an import) + +[file m.py] +[out] + +[case testIgnoredImportDup] + +import m # type: ignore +from m import f # type: ignore +def m() -> None: ... # E: Name "m" already defined (possibly by an import) +def f() -> None: ... # E: Name "f" already defined (possibly by an import) + +[out] + +[case testRuntimeProtoTwoBases] +from typing_extensions import Protocol, runtime_checkable +from typing import TypeVar, Generic + +T = TypeVar('T') + +@runtime_checkable +class P(Protocol, Generic[T]): + attr: T + +class C: + attr: int + +x: P[int] = C() +[builtins fixtures/tuple.pyi] + +[case testSemanalDoesNotLeakSyntheticTypes] +# flags: --cache-fine-grained +from typing import Generic, NamedTuple, TypedDict, TypeVar +from dataclasses import dataclass + +T = TypeVar('T') +class Wrap(Generic[T]): pass + +invalid_1: 1 + 2 # E: Invalid type comment or annotation +invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class A: + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class B(NamedTuple): + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class C(TypedDict): + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +@dataclass +class D: + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test new file mode 100644 index 000000000000..81da94c0591c --- /dev/null +++ b/test-data/unit/check-serialize.test @@ -0,0 +1,1413 @@ +-- Serialization test cases (incremental type checking) +-- +-- These test that modules deserialized from cache files behave +-- identically to modules that have undergone full type checking. +-- +-- These tests are written using the same syntax as test cases in +-- check-incremental.test. Look at the comment at that the top of +-- that file for the details of how these tests work. +-- +-- There is probably some overlap with check-incremental.test, but it +-- is perhaps not worth trying to simplify these, since a few redundant +-- test cases are cheap but accidentally losing test coverage is bad. +-- +-- These are intended to be straightforward, and do not test import +-- cycles and other tricky business. Add test cases for complex things +-- to check-incremental.test. + +-- +-- Basic things +-- + +[case testSerializeModuleAttribute] +import a +[file a.py] +import b +[file a.py.2] +import b +y = b.x # type: int +[file b.py] +x = '' +-- We only do the following two sections once here to avoid repetition. +-- Most other test cases are similar. +[rechecked a] +[stale] +[out2] +tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +-- +-- Functions +-- + +[case testSerializeAnnotatedFunction] +import a +[file a.py] +import b +[file a.py.2] +import b +b.f(1) +x = b.f('') # type: str +[file b.py] +def f(x: str) -> int: pass +[out2] +tmp/a.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" +tmp/a.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testSerializeUnannotatedFunction] +import a +[file a.py] +import b +[file a.py.2] +import b +b.f(x=1) +b.f() +[file b.py] +def f(x): pass +[out2] +tmp/a.py:3: error: Missing positional argument "x" in call to "f" + +[case testSerializeGenericFunction] +import a +[file a.py] +import b +[file a.py.2] +from b import f +reveal_type(f(1)) +reveal_type(f(x='')) +[file b.py] +from typing import TypeVar + +T = TypeVar('T') + +def f(x: T) -> T: return x +[out2] +tmp/a.py:2: note: Revealed type is "builtins.int" +tmp/a.py:3: note: Revealed type is "builtins.str" + +[case testSerializeFunctionReturningGenericFunction] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.f) +reveal_type(b.f()('')) +[file b.py] +from typing import TypeVar, Callable + +T = TypeVar('T') + +def f() -> Callable[[T], T]: pass +[out2] +tmp/a.py:2: note: Revealed type is "def () -> def [T] (T`-1) -> T`-1" +tmp/a.py:3: note: Revealed type is "builtins.str" + +[case testSerializeArgumentKinds] +import a +[file a.py] +import b +[file a.py.2] +from b import f +f(1, z=1) +f(1, '', z=1) +f(1, y='', z=1) +f(1, '', 2, 3, z=1) +f(1, '', zz=1, z=1) +f(1, '', foo='', z=1) +[file b.py] +def f(x: int, + y: str = '', + *args: int, + z: int, + zz: int = 1, + **kw: str) -> None: pass +[builtins fixtures/dict.pyi] +[out2] + +[case testSerializeCallableWithBoundTypeArguments] +import a +[file a.py] +import b +[file a.py.2] +import b +x = b.f +[file b.py] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + def f(self, x: T) -> None: pass + +c: C[int] +f = c.f +[out] +[out2] + +[case testSerializePositionalOnlyArgument] +import a +[file a.py] +import b +[file a.py.2] +import b +b.f(1) +b.f('') +b.f(__x=1) +[file b.py] +def f(__x: int) -> None: pass +[out2] +tmp/a.py:3: error: Argument 1 to "f" has incompatible type "str"; expected "int" +tmp/a.py:4: error: Unexpected keyword argument "__x" for "f" + +[case testSerializeArgumentKindsErrors] +import a +[file a.py] +import b +[file a.py.2] +from b import f +f('', z=1) # Line 2 +f(1, 2, z=1) # 3 +f(1, y=1, z=1) # 4 +f(1, '', 2, '', z=1) # 5 +f(1, '', z='') # 6 +f(1, '', zz='', z=1) # 7 +f(1, '', z=1, foo=1) # 8 +[file b.py] +def f(x: int, + y: str = '', + *args: int, + z: int, + zz: int = 1, + **kw: str) -> None: pass +[builtins fixtures/dict.pyi] +[out2] +tmp/a.py:2: error: Argument 1 to "f" has incompatible type "str"; expected "int" +tmp/a.py:3: error: Argument 2 to "f" has incompatible type "int"; expected "str" +tmp/a.py:4: error: Argument "y" to "f" has incompatible type "int"; expected "str" +tmp/a.py:5: error: Argument 4 to "f" has incompatible type "str"; expected "int" +tmp/a.py:6: error: Argument "z" to "f" has incompatible type "str"; expected "int" +tmp/a.py:7: error: Argument "zz" to "f" has incompatible type "str"; expected "int" +tmp/a.py:8: error: Argument "foo" to "f" has incompatible type "int"; expected "str" + +[case testSerializeOverloadedFunction] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.f(1)) +reveal_type(b.f('')) +[file b.pyi] +from typing import overload +@overload +def f(x: int) -> int: pass +@overload +def f(x: str) -> str: pass +[out2] +tmp/a.py:2: note: Revealed type is "builtins.int" +tmp/a.py:3: note: Revealed type is "builtins.str" + +[case testSerializeDecoratedFunction] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.f('')) +b.f(x=1) +[file b.py] +from typing import Callable +def dec(f: Callable[[int], int]) -> Callable[[str], str]: pass +@dec +def f(x: int) -> int: pass +[out2] +tmp/a.py:2: note: Revealed type is "builtins.str" +tmp/a.py:3: error: Unexpected keyword argument "x" for "f" + +[case testSerializeTypeGuardFunction] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.guard('')) +reveal_type(b.guard) +[file b.py] +from typing_extensions import TypeGuard +def guard(a: object) -> TypeGuard[str]: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:2: note: Revealed type is "builtins.bool" +tmp/a.py:3: note: Revealed type is "def (a: builtins.object) -> TypeGuard[builtins.str]" +-- +-- Classes +-- + +[case testSerializeClassAttribute] +import a +[file a.py] +import b +[file a.py.2] +import b +b.A().x = '' +[file b.py] +class A: + x = 1 +[out2] +tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testSerializeMethod] +import a +[file a.py] +import b +[file a.py.2] +import b +b.A().f('') +[file b.py] +class A: + def f(self, x: int) -> None: pass +[out2] +tmp/a.py:2: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testSerialize__init__] +import a +[file a.py] +import b +[file a.py.2] +from b import A +A('') +class B(A): + def f(self) -> None: + super().__init__('') +[file b.py] +class A: + def __init__(self, x: int) -> None: pass +[out2] +tmp/a.py:2: error: Argument 1 to "A" has incompatible type "str"; expected "int" +tmp/a.py:5: error: Argument 1 to "__init__" of "A" has incompatible type "str"; expected "int" + +[case testSerializeOverloaded__init__] +import a +[file a.py] +import b +[file a.py.2] +from b import A +A(object()) # E +A(x='') +A(0) +class B(A): + def f(self) -> None: + super().__init__(object()) # E + super().__init__('') + super().__init__(0) +[file b.pyi] +from typing import overload +class A: + @overload + def __init__(self, x: int) -> None: pass + @overload + def __init__(self, x: str) -> None: pass +[out2] +tmp/a.py:2: error: No overload variant of "A" matches argument type "object" +tmp/a.py:2: note: Possible overload variants: +tmp/a.py:2: note: def A(self, x: int) -> A +tmp/a.py:2: note: def A(self, x: str) -> A +tmp/a.py:7: error: No overload variant of "__init__" of "A" matches argument type "object" +tmp/a.py:7: note: Possible overload variants: +tmp/a.py:7: note: def __init__(self, x: int) -> None +tmp/a.py:7: note: def __init__(self, x: str) -> None + +[case testSerialize__new__] +import a +[file a.py] +import b +[file a.py.2] +from b import A +A('') +[file b.py] +class A: + def __new__(cls, x: int) -> 'A': pass +[out2] +tmp/a.py:2: error: Argument 1 to "A" has incompatible type "str"; expected "int" + +[case testSerializeClassVar] +import a +[file a.py] +import b +[file a.py.2] +from b import A +A.x = '' +A().x = 1 +[file b.py] +from typing import ClassVar +class A: + x: ClassVar[int] +[out2] +tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/a.py:3: error: Cannot assign to class variable "x" via instance + +[case testSerializeGenericClass] +import a +[file a.py] +import b +[file a.py.2] +from b import A +a1: A[int, str] = A(1) +a2: A[int, str] = A('') +reveal_type(a1.y) +reveal_type(a1.f()) +[file b.py] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T, S]): + x: T + y: S + def __init__(self, x: T) -> None: + self.x = x + def f(self) -> T: + return self.x +[out2] +tmp/a.py:3: error: Argument 1 to "A" has incompatible type "str"; expected "int" +tmp/a.py:4: note: Revealed type is "builtins.str" +tmp/a.py:5: note: Revealed type is "builtins.int" + +[case testSerializeAbstractClass] +import a +[file a.py] +import b +[file a.py.2] +from b import A +A() +class B(A): + def f(self) -> None: pass + x: int +B() +a: A +a.f() +a.x = 1 +[file b.py] +from abc import ABCMeta, abstractmethod, abstractproperty +class A(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + @abstractproperty + def x(self) -> int: return 0 +[typing fixtures/typing-medium.pyi] +[out2] +tmp/a.py:2: error: Cannot instantiate abstract class "A" with abstract attributes "f" and "x" +tmp/a.py:9: error: Property "x" defined in "A" is read-only + +[case testSerializeStaticMethod] +import a +[file a.py] +import b +[file a.py.2] +from b import A +A.f(1) +A.f() +A().f() +[file b.py] +class A: + @staticmethod + def f() -> None: pass +[builtins fixtures/staticmethod.pyi] +[out2] +tmp/a.py:2: error: Too many arguments for "f" of "A" + +[case testSerializeClassMethod] +import a +[file a.py] +import b +[file a.py.2] +from b import A +A.f(1) +A.f() +A().f() +[file b.py] +class A: + @classmethod + def f(cls) -> None: pass +[builtins fixtures/classmethod.pyi] +[out2] +tmp/a.py:2: error: Too many arguments for "f" of "A" + +[case testSerializeReadOnlyProperty] +import a +[file a.py] +import b +[file a.py.2] +from b import A +reveal_type(A().x) +A().x = 0 +[file b.py] +class A: + @property + def x(self) -> int: return 0 +[builtins fixtures/property.pyi] +[out2] +tmp/a.py:2: note: Revealed type is "builtins.int" +tmp/a.py:3: error: Property "x" defined in "A" is read-only + +[case testSerializeReadWriteProperty] +import a +[file a.py] +import b +[file a.py.2] +from b import A +reveal_type(A().x) +A().x = '' +A().x = 0 +[file b.py] +class A: + @property + def x(self) -> int: return 0 + @x.setter + def x(self, v: int) -> None: pass +[builtins fixtures/property.pyi] +[out2] +tmp/a.py:2: note: Revealed type is "builtins.int" +tmp/a.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testSerializeSelfType] +import a +[file a.py] +import b +[file a.py.2] +from b import A +reveal_type(A().f()) +class B(A): pass +reveal_type(B().f()) +[file b.py] +from typing import TypeVar +T = TypeVar('T', bound='A') +class A: + def f(self: T) -> T: return self +[out2] +tmp/a.py:2: note: Revealed type is "b.A" +tmp/a.py:4: note: Revealed type is "a.B" + +[case testSerializeInheritance] +import a +[file a.py] +import b +[file a.py.2] +from b import A, B, C +C().f(1) # E +C().g(1) # E +reveal_type(C().h()) +a: A = C() +b: B = C() +i: int = C() # E +[file b.py] +class A: + def f(self) -> int: pass +class B: + def g(self) -> str: pass + def h(self) -> object: pass +class C(A, B): + def h(self) -> int: pass +[out2] +tmp/a.py:2: error: Too many arguments for "f" of "A" +tmp/a.py:3: error: Too many arguments for "g" of "B" +tmp/a.py:4: note: Revealed type is "builtins.int" +tmp/a.py:7: error: Incompatible types in assignment (expression has type "C", variable has type "int") + +[case testSerializeGenericInheritance] +import a +[file a.py] +import b +[file a.py.2] +from b import B +b: B[int] +reveal_type(b.f()) +[file b.py] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def f(self) -> T: pass +class B(A[A[T]]): + pass +[out2] +tmp/a.py:3: note: Revealed type is "b.A[builtins.int]" + +[case testSerializeFixedLengthTupleBaseClass] +import a +[file a.py] +import b +[file a.py.2] +from b import A +a: A +a.f(1) +reveal_type((a[0], a[1])) +[file b.py] +from typing import Tuple +class A(Tuple[int, str]): + def f(self) -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:3: error: Too many arguments for "f" of "A" +tmp/a.py:4: note: Revealed type is "Tuple[builtins.int, builtins.str]" + +[case testSerializeVariableLengthTupleBaseClass] +import a +[file a.py] +import b +[file a.py.2] +from b import A +a: A +a.f(1) +reveal_type((a[0], a[1])) +[file b.py] +from typing import Tuple +class A(Tuple[int, ...]): + def f(self) -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:3: error: Too many arguments for "f" of "A" +tmp/a.py:4: note: Revealed type is "Tuple[builtins.int, builtins.int]" + +[case testSerializePlainTupleBaseClass] +import a +[file a.py] +import b +[file a.py.2] +from b import A +a: A +a.f(1) +reveal_type((a[0], a[1])) +[file b.py] +from typing import Tuple +class A(tuple): + def f(self) -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:3: error: Too many arguments for "f" of "A" +tmp/a.py:4: note: Revealed type is "Tuple[Any, Any]" + +[case testSerializeNamedTupleBaseClass] +import a +[file a.py] +import b +[file a.py.2] +from b import A +a: A +a.f(1) +reveal_type((a[0], a[1])) +reveal_type((a.x, a.y)) +[file b.py] +from typing import NamedTuple +class A(NamedTuple('N', [('x', int), ('y', str)])): + def f(self) -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:3: error: Too many arguments for "f" of "A" +tmp/a.py:4: note: Revealed type is "Tuple[builtins.int, builtins.str]" +tmp/a.py:5: note: Revealed type is "Tuple[builtins.int, builtins.str]" + +[case testSerializeAnyBaseClass] +import a +[file a.py] +import b +[file a.py.2] +from b import B +B().f(1) +reveal_type(B().xyz) +[file b.py] +from typing import Any +A: Any +class B(A): + def f(self) -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:2: error: Too many arguments for "f" of "B" +tmp/a.py:3: note: Revealed type is "Any" + +[case testSerializeIndirectAnyBaseClass] +import a +[file a.py] +import b +[file a.py.2] +from b import C +C().f(1) +C().g(1) +reveal_type(C().xyz) +[file b.py] +from typing import Any +A: Any +class B(A): + def f(self) -> None: pass +class C(B): + def g(self) -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:2: error: Too many arguments for "f" of "B" +tmp/a.py:3: error: Too many arguments for "g" of "C" +tmp/a.py:4: note: Revealed type is "Any" + +[case testSerializeNestedClass] +import a +[file a.py] +import b +[file a.py.2] +import b +b.A.B().f(1) +b.A.B.C().g(1) +b.b.f(1) +b.c.g(1) +[file b.py] +class A: + class B: + def f(self) -> None: pass + class C: + def g(self) -> None: pass +b: A.B +c: A.B.C +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:2: error: Too many arguments for "f" of "B" +tmp/a.py:3: error: Too many arguments for "g" of "C" +tmp/a.py:4: error: Too many arguments for "f" of "B" +tmp/a.py:5: error: Too many arguments for "g" of "C" + +[case testSerializeCallableVsTypeObjectDistinction] +import a +[file a.py] +import b +[file a.py.2] +import b +t: type +t = b.A +if int(): + t = b.f # E +[file b.py] +class A: pass +def f() -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:5: error: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "type") + +[case testSerializeOverloadedVsTypeObjectDistinction] +import a +[file a.py] +import b +[file a.py.2] +import b +t: type +t = b.A +if int(): + t = b.f # E +[file b.pyi] +from typing import overload +class A: + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: int) -> None: pass +@overload +def f() -> None: pass +@overload +def f(x: int) -> None: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:5: error: Incompatible types in assignment (expression has type overloaded function, variable has type "type") + +[case testSerializeNamedTupleInMethod4] +from ntcrash import C +reveal_type(C().a) +reveal_type(C().b) +reveal_type(C().c) +[file ntcrash.py] +from typing import NamedTuple +class C: + def __init__(self) -> None: + A = NamedTuple('A', [('x', int)]) + self.a = A(0) + self.b = A(0) # type: A + self.c = A +[builtins fixtures/tuple.pyi] +[out1] +main:2: note: Revealed type is "Tuple[builtins.int, fallback=ntcrash.C.A@4]" +main:3: note: Revealed type is "Tuple[builtins.int, fallback=ntcrash.C.A@4]" +main:4: note: Revealed type is "def (x: builtins.int) -> Tuple[builtins.int, fallback=ntcrash.C.A@4]" +[out2] +main:2: note: Revealed type is "Tuple[builtins.int, fallback=ntcrash.C.A@4]" +main:3: note: Revealed type is "Tuple[builtins.int, fallback=ntcrash.C.A@4]" +main:4: note: Revealed type is "def (x: builtins.int) -> Tuple[builtins.int, fallback=ntcrash.C.A@4]" + +-- +-- Strict optional +-- + +[case testSerializeOptionalType] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.x) +b.f(b.x) +[file b.py] +from typing import Optional +x: Optional[int] +def f(x: int) -> None: pass +[out2] +tmp/a.py:2: note: Revealed type is "Union[builtins.int, None]" +tmp/a.py:3: error: Argument 1 to "f" has incompatible type "Optional[int]"; expected "int" + +-- +-- # type: ignore +-- + +[case testSerializeIgnoredUndefinedType] +import b +reveal_type(b.x) +[file b.py] +x: NonExistent # type: ignore +[out1] +main:2: note: Revealed type is "Any" +[out2] +main:2: note: Revealed type is "Any" + +[case testSerializeIgnoredInvalidType] +import b +reveal_type(b.x) +[file b.py] +A = 0 +x: A # type: ignore +[out1] +main:2: note: Revealed type is "A?" +[out2] +main:2: note: Revealed type is "A?" + +[case testSerializeIgnoredMissingBaseClass] +import b +reveal_type(b.B()) +reveal_type(b.B().x) +[file b.py] +class B(A): pass # type: ignore +[out1] +main:2: note: Revealed type is "b.B" +main:3: note: Revealed type is "Any" +[out2] +main:2: note: Revealed type is "b.B" +main:3: note: Revealed type is "Any" + +[case testSerializeIgnoredInvalidBaseClass] +import b +reveal_type(b.B()) +reveal_type(b.B().x) +[file b.py] +A = 0 +class B(A): pass # type: ignore +[out1] +main:2: note: Revealed type is "b.B" +main:3: note: Revealed type is "Any" +[out2] +main:2: note: Revealed type is "b.B" +main:3: note: Revealed type is "Any" + +[case testSerializeIgnoredImport] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.m) +reveal_type(b.x) +[file b.py] +import m # type: ignore +from m import x # type: ignore +[out2] +tmp/a.py:2: note: Revealed type is "Any" +tmp/a.py:3: note: Revealed type is "Any" + +-- +-- TypeVar +-- + +[case testSerializeSimpleTypeVar] +import a +[file a.py] +import b +[file a.py.2] +import b +def f(x: b.T) -> b.T: return x +reveal_type(f) +[file b.py] +from typing import TypeVar +T = TypeVar('T') +[out2] +tmp/a.py:3: note: Revealed type is "def [b.T] (x: b.T`-1) -> b.T`-1" + +[case testSerializeBoundedTypeVar] +import a +[file a.py] +import b +[file a.py.2] +import b +def f(x: b.T) -> b.T: return x +reveal_type(f) +reveal_type(b.g) +[file b.py] +from typing import TypeVar +T = TypeVar('T', bound=int) +def g(x: T) -> T: return x +[out2] +tmp/a.py:3: note: Revealed type is "def [b.T <: builtins.int] (x: b.T`-1) -> b.T`-1" +tmp/a.py:4: note: Revealed type is "def [T <: builtins.int] (x: T`-1) -> T`-1" + +[case testSerializeTypeVarWithValues] +import a +[file a.py] +import b +[file a.py.2] +import b +def f(x: b.T) -> b.T: return x +reveal_type(f) +reveal_type(b.g) +[file b.py] +from typing import TypeVar +T = TypeVar('T', int, str) +def g(x: T) -> T: return x +[out2] +tmp/a.py:3: note: Revealed type is "def [b.T in (builtins.int, builtins.str)] (x: b.T`-1) -> b.T`-1" +tmp/a.py:4: note: Revealed type is "def [T in (builtins.int, builtins.str)] (x: T`-1) -> T`-1" + +[case testSerializeTypeVarInClassBody] +import a +[file a.py] +import b +[file a.py.2] +from b import A +def f(x: A.T) -> A.T: return x +reveal_type(f) +[file b.py] +from typing import TypeVar +class A: + T = TypeVar('T', int, str) +[out2] +tmp/a.py:3: note: Revealed type is "def [A.T in (builtins.int, builtins.str)] (x: A.T`-1) -> A.T`-1" + +-- +-- NewType +-- + +[case testSerializeNewType] +import a +[file a.py] +import b +[file a.py.2] +import b +y: b.N +y = 1 +i = y +b.x = 1 +b.x = y +y = b.N(1) +y = b.N('') +[file b.py] +from typing import NewType +N = NewType('N', int) +x: N +[out2] +tmp/a.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "N") +tmp/a.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "N") +tmp/a.py:8: error: Argument 1 to "N" has incompatible type "str"; expected "int" + +-- +-- Named tuples +-- + +[case testSerializeNamedTuple] +import a +[file a.py] +import b +[file a.py.2] +import b +from typing import Tuple +y: b.N +t: Tuple[int] +y = t +b.x = t +t = y +b.x = t +reveal_type(b.N(x=1)) +reveal_type(y[0]) +b.N(x='') +[file b.py] +from typing import NamedTuple +N = NamedTuple('N', [('x', int)]) +x: N +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:5: error: Incompatible types in assignment (expression has type "Tuple[int]", variable has type "N") +tmp/a.py:6: error: Incompatible types in assignment (expression has type "Tuple[int]", variable has type "N") +tmp/a.py:9: note: Revealed type is "Tuple[builtins.int, fallback=b.N]" +tmp/a.py:10: note: Revealed type is "builtins.int" +tmp/a.py:11: error: Argument "x" to "N" has incompatible type "str"; expected "int" + +-- +-- Types and type aliases +-- + +[case testSerializeTypeAliases] +import a +[file a.py] +import b +[file a.py.2] +import b +d: b.D +a: b.A +u: b.U +l: b.L +t: b.T +c: b.C +ty: b.Ty +reveal_type(d) +reveal_type(a) +reveal_type(u) +reveal_type(l) +reveal_type(t) +reveal_type(c) +reveal_type(ty) +c2: b.C2 +reveal_type(c2) +ty2: b.Ty2 +reveal_type(ty2) +[file b.py] +from typing import Any, Union, List, Tuple, Callable, Type +class DD: pass +D = DD +A = Any +U = Union[int, str] +L = List[int] +T = Tuple[int, str] +C = Callable[[int], str] +C2 = Callable[..., str] +Ty = Type[int] +Ty2 = type +[builtins fixtures/list.pyi] +[out2] +tmp/a.py:9: note: Revealed type is "b.DD" +tmp/a.py:10: note: Revealed type is "Any" +tmp/a.py:11: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/a.py:12: note: Revealed type is "builtins.list[builtins.int]" +tmp/a.py:13: note: Revealed type is "Tuple[builtins.int, builtins.str]" +tmp/a.py:14: note: Revealed type is "def (builtins.int) -> builtins.str" +tmp/a.py:15: note: Revealed type is "Type[builtins.int]" +tmp/a.py:17: note: Revealed type is "def (*Any, **Any) -> builtins.str" +tmp/a.py:19: note: Revealed type is "builtins.type" + +[case testSerializeGenericTypeAlias] +import b +from b import X # Work around https://github.com/python/mypy/issues/2887 +t: b.Y[int] +reveal_type(t) +[file b.py] +from typing import TypeVar, Tuple +X = TypeVar('X') +Y = Tuple[X, str] +[builtins fixtures/tuple.pyi] +[out1] +main:4: note: Revealed type is "Tuple[builtins.int, builtins.str]" +[out2] +main:4: note: Revealed type is "Tuple[builtins.int, builtins.str]" + +[case testSerializeTuple] +# Don't repreat types tested by testSerializeTypeAliases here. +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.x) +reveal_type(b.y) +[file b.py] +from typing import Tuple +x: Tuple[int, ...] +y: tuple +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:2: note: Revealed type is "builtins.tuple[builtins.int, ...]" +tmp/a.py:3: note: Revealed type is "builtins.tuple[Any, ...]" + +[case testSerializeNone] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.x) +[file b.py] +x: None +[out2] +tmp/a.py:2: note: Revealed type is "None" + +-- +-- TypedDict +-- + +[case testSerializeTypedDictInMethod] +from ntcrash import C +reveal_type(C().a) +reveal_type(C().b) +reveal_type(C().c) +[file ntcrash.py] +from mypy_extensions import TypedDict +class C: + def __init__(self) -> None: + A = TypedDict('A', {'x': int}) + self.a = A(x=0) + self.b = A(x=0) # type: A + self.c = A +[builtins fixtures/dict.pyi] +[out1] +main:2: note: Revealed type is "TypedDict('ntcrash.C.A@4', {'x': builtins.int})" +main:3: note: Revealed type is "TypedDict('ntcrash.C.A@4', {'x': builtins.int})" +main:4: note: Revealed type is "def (*, x: builtins.int) -> TypedDict('ntcrash.C.A@4', {'x': builtins.int})" +[out2] +main:2: note: Revealed type is "TypedDict('ntcrash.C.A@4', {'x': builtins.int})" +main:3: note: Revealed type is "TypedDict('ntcrash.C.A@4', {'x': builtins.int})" +main:4: note: Revealed type is "def (*, x: builtins.int) -> TypedDict('ntcrash.C.A@4', {'x': builtins.int})" + +[case testSerializeNonTotalTypedDict] +from m import d +reveal_type(d) +[file m.py] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=False) +d: D +[builtins fixtures/dict.pyi] +[out1] +main:2: note: Revealed type is "TypedDict('m.D', {'x'?: builtins.int, 'y'?: builtins.str})" +[out2] +main:2: note: Revealed type is "TypedDict('m.D', {'x'?: builtins.int, 'y'?: builtins.str})" + +-- +-- Modules +-- + +[case testSerializeImport] +import b +b.c.f() +b.c.g() +[file b.py] +import c +[file c.py] +def f() -> None: pass +def g(x: int) -> None: pass +[out1] +main:3: error: Missing positional argument "x" in call to "g" +[out2] +main:3: error: Missing positional argument "x" in call to "g" + +[case testSerializeImportAs] +import b +b.d.f() +b.d.g() +[file b.py] +import c as d +[file c.py] +def f() -> None: pass +def g(x: int) -> None: pass +[out1] +main:3: error: Missing positional argument "x" in call to "g" +[out2] +main:3: error: Missing positional argument "x" in call to "g" + +[case testSerializeFromImportedClass] +import b +b.A(1) +reveal_type(b.A()) +[file b.py] +from c import A +[file c.py] +class A: pass +[out1] +main:2: error: Too many arguments for "A" +main:3: note: Revealed type is "c.A" +[out2] +main:2: error: Too many arguments for "A" +main:3: note: Revealed type is "c.A" + +[case testSerializeFromImportedClassAs] +import b +b.B(1) +reveal_type(b.B()) +[file b.py] +from c import A as B +[file c.py] +class A: pass +[out1] +main:2: error: Too many arguments for "A" +main:3: note: Revealed type is "c.A" +[out2] +main:2: error: Too many arguments for "A" +main:3: note: Revealed type is "c.A" + +[case testSerializeFromImportedModule] +import b +b.d.f() +b.d.g() +[file b.py] +from c import d +[file c/__init__.py] +[file c/d.py] +def f() -> None: pass +def g(x: int) -> None: pass +[out1] +main:3: error: Missing positional argument "x" in call to "g" +[out2] +main:3: error: Missing positional argument "x" in call to "g" + +[case testSerializeQualifiedImport] +import b +b.c.d.f() +b.c.d.g() +[file b.py] +import c.d +[file c/__init__.py] +[file c/d.py] +def f() -> None: pass +def g(x: int) -> None: pass +[out1] +main:3: error: Missing positional argument "x" in call to "g" +[out2] +main:3: error: Missing positional argument "x" in call to "g" + +[case testSerializeQualifiedImportAs] +import b +b.e.f() +b.e.g() +[file b.py] +import c.d as e +[file c/__init__.py] +[file c/d.py] +def f() -> None: pass +def g(x: int) -> None: pass +[out1] +main:3: error: Missing positional argument "x" in call to "g" +[out2] +main:3: error: Missing positional argument "x" in call to "g" + +[case testSerialize__init__ModuleImport] +import b +b.c.f() +b.c.g() +a: b.c.d.A +reveal_type(a) +[file b.py] +import c +[file c/__init__.py] +import d +def f() -> None: pass +def g(x: int) -> None: pass +[file d.py] +class A: pass +[out1] +main:3: error: Missing positional argument "x" in call to "g" +main:5: note: Revealed type is "d.A" +[out2] +main:3: error: Missing positional argument "x" in call to "g" +main:5: note: Revealed type is "d.A" + +[case testSerializeImportInClassBody] +import b +b.A.c.f() +b.A.c.g() +[file b.py] +class A: + import c +[file c.py] +def f() -> None: pass +def g(x: int) -> None: pass +[out1] +main:3: error: Missing positional argument "x" in call to "g" +[out2] +main:3: error: Missing positional argument "x" in call to "g" + +[case testSerializeImportedTypeAlias] +import b +x: b.B +reveal_type(x) +[file b.py] +from c import B +[file c.py] +from typing import Any +class A: pass +B = A +[out1] +main:3: note: Revealed type is "c.A" +[out2] +main:3: note: Revealed type is "c.A" + +[case testSerializeStarImport] +import a +[file a.py] +import b +[file a.py.2] +import b +b.f(1) +x: b.A +reveal_type(x) +[file b.py] +from c import * +[file c.py] +def f() -> None: pass +class A: pass +[out2] +tmp/a.py:2: error: Too many arguments for "f" +tmp/a.py:4: note: Revealed type is "c.A" + +[case testSerializeRelativeImport] +import b.c +b.c.f(1) +[file b/__init__.py] +[file b/c.py] +from .d import f +[file b/d.py] +def f() -> None: pass +[out1] +main:2: error: Too many arguments for "f" +[out2] +main:2: error: Too many arguments for "f" + +[case testSerializeDummyType] +# flags: --no-strict-optional +import a +[file a.py] +import b +reveal_type(b.Test(None).foo) +[file a.py.2] +import b +reveal_type(b.Test(b.Foo()).foo) +[file b.py] +class Foo(object): pass +class Test: + def __init__(self, o: Foo) -> None: + self.foo = None + if callable(o): + self.foo = o +[builtins fixtures/callable.pyi] +[out1] +tmp/a.py:2: note: Revealed type is "b." +[out2] +tmp/a.py:2: note: Revealed type is "b." + +[case testSerializeForwardReferenceToAliasInProperty] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.A().p) +[file b.py] + +class A: + @property + def p(self) -> C: pass + @p.setter + def p(self, c: C) -> None: pass + @p.deleter + def p(self) -> None: pass + +C = str +[builtins fixtures/property.pyi] +[out2] +tmp/a.py:2: note: Revealed type is "builtins.str" + +[case testSerializeForwardReferenceToImportedAliasInProperty] + +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.A().p) +[file b.py] +class A: + @property + def p(self) -> C: pass + @p.setter + def p(self, c: C) -> None: pass + @p.deleter + def p(self) -> None: pass + +from m import C +[file m.py] +C = str +[builtins fixtures/property.pyi] +[out2] +tmp/a.py:2: note: Revealed type is "builtins.str" + +[case testSerializeNestedClassStuff] +# flags: --verbose +import a +[file a.py] +import b +[file a.py.2] +import b +# +[file b.py] +def foo() -> None: + class Foo: + class Bar: pass + +[case testSerializeGenericClassMethod] +import a +[file a.py] +import b +from typing import TypeVar + +T = TypeVar('T') + +class C: + @classmethod + def f(cls, x: T) -> T: ... + +x = C.f +[file b.py] +x = 1 +[file b.py.2] +x = 'yes' +[builtins fixtures/classmethod.pyi] + +[case testSerializeGenericAbstractMethod] +import a +[file a.py] +import b +from typing import TypeVar +from abc import abstractmethod + +T = TypeVar('T') + +class C: + @abstractmethod + def f(self, x: T) -> T: ... + +c: C +x = c.f +[file b.py] +x = 1 +[file b.py.2] +x = 'yes' + +[case testSerializeGenericNormalMethod] +import a +[file a.py] +import b +from typing import TypeVar +from abc import abstractmethod + +T = TypeVar('T') + +class C: + def f(self, x: T) -> T: ... + +c: C +x = c.f +[file b.py] +x = 1 +[file b.py.2] +x = 'yes' diff --git a/test-data/unit/check-singledispatch.test b/test-data/unit/check-singledispatch.test new file mode 100644 index 000000000000..e63d4c073e86 --- /dev/null +++ b/test-data/unit/check-singledispatch.test @@ -0,0 +1,309 @@ +[case testIncorrectDispatchArgumentWhenDoesntMatchFallback] +from functools import singledispatch + +class A: pass +class B(A): pass + +@singledispatch +def fun(arg: A) -> None: + pass +@fun.register +def fun_b(arg: B) -> None: + pass + +fun(1) # E: Argument 1 to "fun" has incompatible type "int"; expected "A" + +# probably won't be required after singledispatch is special cased +[builtins fixtures/args.pyi] + +[case testMultipleUnderscoreFunctionsIsntError] +from functools import singledispatch + +@singledispatch +def fun(arg) -> None: + pass +@fun.register +def _(arg: str) -> None: + pass +@fun.register +def _(arg: int) -> None: + pass + +[builtins fixtures/args.pyi] + +[case testCheckNonDispatchArgumentsWithTypeAlwaysTheSame] +from functools import singledispatch + +class A: pass +class B(A): pass + +@singledispatch +def f(arg: A, arg2: str) -> None: + pass + +@f.register +def g(arg: B, arg2: str) -> None: + pass + +f(A(), 'a') +f(A(), 5) # E: Argument 2 to "f" has incompatible type "int"; expected "str" + +f(B(), 'a') +f(B(), 1) # E: Argument 2 to "f" has incompatible type "int"; expected "str" + +[builtins fixtures/args.pyi] + +[case testImplementationHasSameDispatchTypeAsFallback-xfail] +from functools import singledispatch + +# TODO: differentiate between fallback and other implementations in error message +@singledispatch +def f(arg: int) -> None: # E: singledispatch implementation 1 will never be used: implementation 2's dispatch type is the same + pass + +@f.register +def g(arg: int) -> None: + pass + +[builtins fixtures/args.pyi] + +[case testRegisterHasDifferentTypeThanTypeSignature-xfail] +from functools import singledispatch + +@singledispatch +def f(arg) -> None: + pass + +@f.register(str) +def g(arg: int) -> None: # E: Argument to register "str" is incompatible with type "int" in function signature + pass + +[builtins fixtures/args.pyi] + +[case testTypePassedAsArgumentToRegister] +from functools import singledispatch + +@singledispatch +def f(arg: int) -> None: + pass +@f.register(str) +def g(arg) -> None: # E: Dispatch type "str" must be subtype of fallback function first argument "int" + pass + +[builtins fixtures/args.pyi] + +[case testCustomClassPassedAsTypeToRegister] +from functools import singledispatch +class A: pass + +@singledispatch +def f(arg: int) -> None: + pass +@f.register(A) +def g(arg) -> None: # E: Dispatch type "A" must be subtype of fallback function first argument "int" + pass + +[builtins fixtures/args.pyi] + +[case testMultiplePossibleImplementationsForKnownType] +from functools import singledispatch +from typing import Union + +class A: pass +class B(A): pass +class C: pass + +@singledispatch +def f(arg: Union[A, C]) -> None: + pass + +@f.register +def g(arg: B) -> None: + pass + +@f.register +def h(arg: C) -> None: + pass + +x: Union[B, C] +f(x) + +[builtins fixtures/args.pyi] + +[case testOnePartOfUnionDoesNotHaveCorrespondingImplementation] +from functools import singledispatch +from typing import Union + +class A: pass +class B(A): pass +class C: pass + +@singledispatch +def f(arg: Union[A, C]) -> None: + pass + +@f.register +def g(arg: B) -> None: + pass + +@f.register +def h(arg: C) -> None: + pass + +x: Union[B, C, int] +f(x) # E: Argument 1 to "f" has incompatible type "Union[B, C, int]"; expected "Union[A, C]" + +[builtins fixtures/args.pyi] + +[case testABCAllowedAsDispatchType] +from functools import singledispatch +from collections.abc import Mapping + +@singledispatch +def f(arg) -> None: + pass + +@f.register +def g(arg: Mapping) -> None: + pass +[builtins fixtures/dict.pyi] + +[case testIncorrectArgumentsInSingledispatchFunctionDefinition] +from functools import singledispatch + +@singledispatch +def f() -> None: # E: Singledispatch function requires at least one argument + pass + +@singledispatch +def g(**kwargs) -> None: # E: First argument to singledispatch function must be a positional argument + pass + +@singledispatch +def h(*, x) -> None: # E: First argument to singledispatch function must be a positional argument + pass + +@singledispatch +def i(*, x=1) -> None: # E: First argument to singledispatch function must be a positional argument + pass + +[builtins fixtures/args.pyi] + +[case testDispatchTypeIsNotASubtypeOfFallbackFirstArgument] +from functools import singledispatch + +class A: pass +class B(A): pass +class C: pass + +@singledispatch +def f(arg: A) -> None: + pass + +@f.register +def g(arg: B) -> None: + pass + +@f.register +def h(arg: C) -> None: # E: Dispatch type "C" must be subtype of fallback function first argument "A" + pass + +[builtins fixtures/args.pyi] + +[case testMultipleSingledispatchFunctionsIntermixed] +from functools import singledispatch + +class A: pass +class B(A): pass +class C: pass + +@singledispatch +def f(arg: A) -> None: + pass + +@singledispatch +def h(arg: C) -> None: + pass + +@f.register +def g(arg: B) -> None: + pass + +[builtins fixtures/args.pyi] + +[case testAnyInConstructorArgsWithClassPassedToRegister] +from functools import singledispatch +from typing import Any + +class Base: pass +class ConstExpr: + def __init__(self, **kwargs: Any) -> None: pass + +@singledispatch +def f(arg: Base) -> ConstExpr: + pass + +@f.register(ConstExpr) +def g(arg: ConstExpr) -> ConstExpr: # E: Dispatch type "ConstExpr" must be subtype of fallback function first argument "Base" + pass + + +[builtins fixtures/args.pyi] + +[case testRegisteredImplementationUsedBeforeDefinition] +from functools import singledispatch +from typing import Union + +class Node: pass +class MypyFile(Node): pass +class Missing: pass + +@singledispatch +def f(a: Union[Node, Missing]) -> None: + pass + +@f.register +def g(a: MypyFile) -> None: + x: Missing + f(x) + +@f.register +def h(a: Missing) -> None: + pass + +[builtins fixtures/args.pyi] + +[case testIncorrectArgumentTypeWhenCallingRegisteredImplDirectly] +from functools import singledispatch + +@singledispatch +def f(arg, arg2: str) -> bool: + return False + +@f.register +def g(arg: int, arg2: str) -> bool: + pass + +@f.register(str) +def h(arg, arg2: str) -> bool: + pass + +g('a', 'a') # E: Argument 1 to "g" has incompatible type "str"; expected "int" +g(1, 1) # E: Argument 2 to "g" has incompatible type "int"; expected "str" + +# don't show errors for incorrect first argument here, because there's no type annotation for the +# first argument +h(1, 'a') +h('a', 1) # E: Argument 2 to "h" has incompatible type "int"; expected "str" + +[builtins fixtures/args.pyi] + +[case testDontCrashWhenRegisteringAfterError] +import functools +a = functools.singledispatch('a') # E: Need type annotation for "a" # E: Argument 1 to "singledispatch" has incompatible type "str"; expected "Callable[..., Never]" + +@a.register(int) +def default(val) -> int: + return 3 + +[builtins fixtures/args.pyi] diff --git a/test-data/unit/check-slots.test b/test-data/unit/check-slots.test new file mode 100644 index 000000000000..b7ce5e596101 --- /dev/null +++ b/test-data/unit/check-slots.test @@ -0,0 +1,523 @@ +[case testSlotsDefinitionWithStrAndListAndTuple] +class A: + __slots__ = "a" + def __init__(self) -> None: + self.a = 1 + self.b = 2 # E: Trying to assign name "b" that is not in "__slots__" of type "__main__.A" + +class B: + __slots__ = ("a", "b") + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.c = 3 # E: Trying to assign name "c" that is not in "__slots__" of type "__main__.B" + +class C: + __slots__ = ['c'] + def __init__(self) -> None: + self.a = 1 # E: Trying to assign name "a" that is not in "__slots__" of type "__main__.C" + self.c = 3 + +class WithVariable: + __fields__ = ['a', 'b'] + __slots__ = __fields__ + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.c = 3 +[builtins fixtures/list.pyi] + + +[case testSlotsDefinitionWithDict] +class D: + __slots__ = {'key': 'docs'} + def __init__(self) -> None: + self.key = 1 + self.missing = 2 # E: Trying to assign name "missing" that is not in "__slots__" of type "__main__.D" +[builtins fixtures/dict.pyi] + + +[case testSlotsDefinitionWithDynamicDict] +slot_kwargs = {'b': 'docs'} +class WithDictKwargs: + __slots__ = {'a': 'docs', **slot_kwargs} + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.c = 3 +[builtins fixtures/dict.pyi] + + +[case testSlotsDefinitionWithSet] +class E: + __slots__ = {'e'} + def __init__(self) -> None: + self.e = 1 + self.missing = 2 # E: Trying to assign name "missing" that is not in "__slots__" of type "__main__.E" +[builtins fixtures/set.pyi] + + +[case testSlotsDefinitionOutsideOfClass] +__slots__ = ("a", "b") +class A: + def __init__(self) -> None: + self.x = 1 + self.y = 2 +[builtins fixtures/tuple.pyi] + + +[case testSlotsDefinitionWithClassVar] +class A: + __slots__ = ('a',) + b = 4 + + def __init__(self) -> None: + self.a = 1 + + # You cannot override class-level variables, but you can use them: + b = self.b + self.b = 2 # E: Trying to assign name "b" that is not in "__slots__" of type "__main__.A" + + self.c = 3 # E: Trying to assign name "c" that is not in "__slots__" of type "__main__.A" + +A.b = 1 +[builtins fixtures/tuple.pyi] + + +[case testSlotsDefinitionMultipleVars1] +class A: + __slots__ = __fields__ = ("a", "b") + def __init__(self) -> None: + self.x = 1 + self.y = 2 +[builtins fixtures/tuple.pyi] + + +[case testSlotsDefinitionMultipleVars2] +class A: + __fields__ = __slots__ = ("a", "b") + def __init__(self) -> None: + self.x = 1 + self.y = 2 +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentEmptySlots] +class A: + __slots__ = () + def __init__(self) -> None: + self.a = 1 + self.b = 2 + +a = A() +a.a = 1 +a.b = 2 +a.missing = 2 +[out] +main:4: error: Trying to assign name "a" that is not in "__slots__" of type "__main__.A" +main:5: error: Trying to assign name "b" that is not in "__slots__" of type "__main__.A" +main:8: error: Trying to assign name "a" that is not in "__slots__" of type "__main__.A" +main:9: error: Trying to assign name "b" that is not in "__slots__" of type "__main__.A" +main:10: error: "A" has no attribute "missing" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithSuper] +class A: + __slots__ = ("a",) +class B(A): + __slots__ = ("b", "c") + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self._one = 1 + +b = B() +b.a = 1 +b.b = 2 +b.c = 3 +b._one = 1 +b._two = 2 +[out] +main:9: error: Trying to assign name "_one" that is not in "__slots__" of type "__main__.B" +main:14: error: "B" has no attribute "c" +main:15: error: Trying to assign name "_one" that is not in "__slots__" of type "__main__.B" +main:16: error: "B" has no attribute "_two" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithSuperDuplicateSlots] +class A: + __slots__ = ("a",) +class B(A): + __slots__ = ("a", "b",) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self._one = 1 # E: Trying to assign name "_one" that is not in "__slots__" of type "__main__.B" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithMixin] +class A: + __slots__ = ("a",) +class Mixin: + __slots__ = ("m",) +class B(A, Mixin): + __slots__ = ("b",) + + def __init__(self) -> None: + self.a = 1 + self.m = 2 + self._one = 1 + +b = B() +b.a = 1 +b.m = 2 +b.b = 2 +b._two = 2 +[out] +main:11: error: Trying to assign name "_one" that is not in "__slots__" of type "__main__.B" +main:16: error: "B" has no attribute "b" +main:17: error: "B" has no attribute "_two" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithSlottedSuperButNoChildSlots] +class A: + __slots__ = ("a",) +class B(A): + def __init__(self) -> None: + self.a = 1 + self.b = 1 + +b = B() +b.a = 1 +b.b = 2 +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithoutSuperSlots] +class A: + pass # no slots +class B(A): + __slots__ = ("a", "b") + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.missing = 3 + +b = B() +b.a = 1 +b.b = 2 +b.missing = 3 +b.extra = 4 # E: "B" has no attribute "extra" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithoutSuperMixingSlots] +class A: + __slots__ = () +class Mixin: + pass # no slots +class B(A, Mixin): + __slots__ = ("a", "b") + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.missing = 3 + +b = B() +b.a = 1 +b.b = 2 +b.missing = 3 +b.extra = 4 # E: "B" has no attribute "extra" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithExplicitSetattr] +class A: + __slots__ = ("a",) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + + def __setattr__(self, k, v) -> None: + ... + +a = A() +a.a = 1 +a.b = 2 +a.c = 3 +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithParentSetattr] +class Parent: + __slots__ = () + + def __setattr__(self, k, v) -> None: + ... + +class A(Parent): + __slots__ = ("a",) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + +a = A() +a.a = 1 +a.b = 2 +a.c = 3 +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithProps] +from typing import Any + +custom_prop: Any + +class A: + __slots__ = ("a",) + + @property + def first(self) -> int: + ... + + @first.setter + def first(self, arg: int) -> None: + ... + +class B(A): + __slots__ = ("b",) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.c = 3 + + @property + def second(self) -> int: + ... + + @second.setter + def second(self, arg: int) -> None: + ... + + def get_third(self) -> int: + ... + + def set_third(self, arg: int) -> None: + ... + + third = custom_prop(get_third, set_third) + +b = B() +b.a = 1 +b.b = 2 +b.c = 3 +b.first = 1 +b.second = 2 +b.third = 3 +b.extra = 'extra' +[out] +main:22: error: Trying to assign name "c" that is not in "__slots__" of type "__main__.B" +main:43: error: Trying to assign name "c" that is not in "__slots__" of type "__main__.B" +main:47: error: "B" has no attribute "extra" +[builtins fixtures/property.pyi] + + +[case testSlotsAssignmentWithUnionProps] +from typing import Any, Callable, Union + +custom_obj: Any + +class custom_property(object): + def __set__(self, *args, **kwargs): + ... + +class A: + __slots__ = ("a",) + + def __init__(self) -> None: + self.a = 1 + + b: custom_property + c: Union[Any, custom_property] + d: Union[Callable, custom_property] + e: Callable + +a = A() +a.a = 1 +a.b = custom_obj +a.c = custom_obj +a.d = custom_obj +a.e = custom_obj +[out] +[builtins fixtures/dict.pyi] + + +[case testSlotsAssignmentWithMethodReassign] +class A: + __slots__ = () + + def __init__(self) -> None: + self.method = lambda: None # E: Cannot assign to a method + + def method(self) -> None: + ... + +a = A() +a.method = lambda: None # E: Cannot assign to a method +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithExplicitDict] +class A: + __slots__ = ("a",) +class B(A): + __slots__ = ("__dict__",) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + +b = B() +b.a = 1 +b.b = 2 +b.c = 3 # E: "B" has no attribute "c" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithExplicitSuperDict] +class A: + __slots__ = ("__dict__",) +class B(A): + __slots__ = ("a",) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + +b = B() +b.a = 1 +b.b = 2 +b.c = 3 # E: "B" has no attribute "c" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentWithVariable] +slot_name = "b" +class A: + __slots__ = ("a", slot_name) + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.c = 3 + +a = A() +a.a = 1 +a.b = 2 +a.c = 3 +a.d = 4 # E: "A" has no attribute "d" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentMultipleLeftValues] +class A: + __slots__ = ("a", "b") + def __init__(self) -> None: + self.a, self.b, self.c = (1, 2, 3) # E: Trying to assign name "c" that is not in "__slots__" of type "__main__.A" +[builtins fixtures/tuple.pyi] + + +[case testSlotsAssignmentMultipleAssignments] +class A: + __slots__ = ("a",) + def __init__(self) -> None: + self.a = self.b = self.c = 1 +[out] +main:4: error: Trying to assign name "b" that is not in "__slots__" of type "__main__.A" +main:4: error: Trying to assign name "c" that is not in "__slots__" of type "__main__.A" +[builtins fixtures/tuple.pyi] + + +[case testSlotsWithTupleCall] +class A: + # TODO: for now this way of writing tuples are not recognised + __slots__ = tuple(("a", "b")) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.missing = 3 +[builtins fixtures/tuple.pyi] + + +[case testSlotsWithListCall] +class A: + # TODO: for now this way of writing lists are not recognised + __slots__ = list(("a", "b")) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.missing = 3 +[builtins fixtures/list.pyi] + + +[case testSlotsWithSetCall] +class A: + # TODO: for now this way of writing sets are not recognised + __slots__ = set(("a", "b")) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.missing = 3 +[builtins fixtures/set.pyi] + + +[case testSlotsWithDictCall] +class A: + # TODO: for now this way of writing dicts are not recognised + __slots__ = dict((("a", "docs"), ("b", "docs"))) + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.missing = 3 +[builtins fixtures/dict.pyi] + + +[case testSlotsWithAny] +from typing import Any + +some_obj: Any + +class A: + # You can do anything with `Any`: + __slots__ = some_obj + + def __init__(self) -> None: + self.a = 1 + self.b = 2 + self.missing = 3 +[builtins fixtures/tuple.pyi] + +[case testSlotsWithClassVar] +from typing import ClassVar +class X: + __slots__ = ('a',) + a: int +x = X() +X.a # E: "a" in __slots__ conflicts with class variable access +x.a +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test new file mode 100644 index 000000000000..44880cf35204 --- /dev/null +++ b/test-data/unit/check-statements.test @@ -0,0 +1,2351 @@ +-- Return statement +-- ---------------- + + +[case testReturnValue] +import typing +def f() -> 'A': + return A() +def g() -> 'B': + return A() +class A: + pass +class B: + pass +[out] +main:5: error: Incompatible return value type (got "A", expected "B") + +[case testReturnSubtype] +import typing +def f() -> 'B': + return A() +def g() -> 'A': + return B() +class A: + pass +class B(A): + pass +[out] +main:3: error: Incompatible return value type (got "A", expected "B") + +[case testReturnWithoutAValue] +import typing +def f() -> 'A': + return +def g() -> None: + return +class A: + pass +[out] +main:3: error: Return value expected + +[case testReturnNoneInFunctionReturningNone] +import typing +def f() -> None: + return None +def g() -> None: + return f() +[out] + +[case testReturnInGenerator] +from typing import Generator +def f() -> Generator[int, None, str]: + yield 1 + return "foo" +[out] + +[case testEmptyReturnInGenerator] +from typing import Generator +def f() -> Generator[int, None, str]: + yield 1 + return # E: Return value expected +[out] + +[case testNoReturnInGenerator] +from typing import Generator +def f() -> Generator[int, None, str]: # E: Missing return statement + yield 1 +[out] + +[case testEmptyReturnInNoneTypedGenerator] +from typing import Generator +def f() -> Generator[int, None, None]: + yield 1 + return +[out] + +[case testNonEmptyReturnInNoneTypedGenerator] +from typing import Generator +def f() -> Generator[int, None, None]: + yield 1 + return 42 # E: No return value expected +[out] + +[case testReturnInIterator] +from typing import Iterator +def f() -> Iterator[int]: + yield 1 + return "foo" # E: No return value expected +[out] + + +-- If statement +-- ------------ + + +[case testIfStatement] + +a: A +a2: A +a3: A +b: bool +if a: + a = b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +elif a2: + a = b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +elif a3: + a = b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +else: + a = b # E: Incompatible types in assignment (expression has type "bool", variable has type "A") +if b: + pass +elif b: + pass +if b: + pass + +class A: pass +[builtins fixtures/bool.pyi] + + +-- Loops +-- ----- + + +[case testWhileStatement] + +a: A +b: bool +while a: + a = b # Fail +else: + a = b # Fail +while b: + b = b + +class A: pass +[builtins fixtures/bool.pyi] +[out] +main:5: error: Incompatible types in assignment (expression has type "bool", variable has type "A") +main:7: error: Incompatible types in assignment (expression has type "bool", variable has type "A") + +[case testForStatement] +class A: pass + +a: A +b: object +for a in [A()]: + a = b # E: Incompatible types in assignment (expression has type "object", variable has type "A") +else: + a = b # E: Incompatible types in assignment (expression has type "object", variable has type "A") +[builtins fixtures/list.pyi] + +[case testBreakStatement] +import typing +while None: + break +[builtins fixtures/bool.pyi] +[out] + +[case testContinueStatement] +import typing +while None: + continue +[builtins fixtures/bool.pyi] +[out] + +[case testForStatementTypeComments] + +from typing import List, Union +x = [] # type: List[int] + +for y in x: # type: str # E: Incompatible types in assignment (expression has type "int", variable has type "str") + pass + +for z in x: # type: int + pass + +for w in x: # type: Union[int, str] + reveal_type(w) # N: Revealed type is "Union[builtins.int, builtins.str]" + +for v in x: # type: int, int # E: Syntax error in type annotation # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + pass +[builtins fixtures/list.pyi] + +[case testForStatementMultipleTypeComments] + +from typing import List, Tuple +x = [] # type: List[Tuple[int, int]] + +for y in x: # type: int, int # E: Syntax error in type annotation # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + pass + +for z in x: # type: Tuple[int, int] + pass + +for w,v in x: # type: int, str # E: Incompatible types in assignment (expression has type "int", variable has type "str") + pass + +for a, b in x: # type: int, int, int # E: Incompatible number of tuple items + pass +[builtins fixtures/list.pyi] + + +-- Operator assignment +-- ------------------- + + +[case testPlusAssign] +a: A +b: B +c: C +a += b # Fail +b += a # Fail +c += a # Fail +a += c + +class A: + def __add__(self, x: 'C') -> 'A': pass + +class B: + def __add__(self, x: A) -> 'C': pass + +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:4: error: Unsupported operand types for + ("A" and "B") +main:5: error: Incompatible types in assignment (expression has type "C", variable has type "B") +main:6: error: Unsupported left operand type for + ("C") + +[case testMinusAssign] +a: A +b: B +c: C +a -= b # Fail +b -= a # Fail +c -= a # Fail +a -= c + +class A: + def __sub__(self, x: 'C') -> 'A': pass + +class B: + def __sub__(self, x: A) -> 'C': pass + +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:4: error: Unsupported operand types for - ("A" and "B") +main:5: error: Incompatible types in assignment (expression has type "C", variable has type "B") +main:6: error: Unsupported left operand type for - ("C") + +[case testMulAssign] +a: A +c: C +a *= a # Fail +c *= a # Fail +a *= c + +class A: + def __mul__(self, x: 'C') -> 'A': pass + +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:3: error: Unsupported operand types for * ("A" and "A") +main:4: error: Unsupported left operand type for * ("C") + +[case testMatMulAssign] +a: A +c: C +a @= a # E: Unsupported operand types for @ ("A" and "A") +c @= a # E: Unsupported left operand type for @ ("C") +a @= c + +class A: + def __matmul__(self, x: 'C') -> 'A': pass + +class C: pass +[builtins fixtures/tuple.pyi] + +[case testDivAssign] +a: A +c: C +a /= a # Fail +c /= a # Fail +a /= c + +class A: + def __truediv__(self, x: 'C') -> 'A': pass + +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:3: error: Unsupported operand types for / ("A" and "A") +main:4: error: Unsupported left operand type for / ("C") + +[case testPowAssign] +a: A +c: C +a **= a # Fail +c **= a # Fail +a **= c + +class A: + def __pow__(self, x: 'C') -> 'A': pass + +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:3: error: Unsupported operand types for ** ("A" and "A") +main:4: error: Unsupported left operand type for ** ("C") + +[case testSubtypesInOperatorAssignment] +a: A +b: B +b += b +b += a +a += b + +class A: + def __add__(self, x: 'A') -> 'B': pass + +class B(A): pass +[builtins fixtures/tuple.pyi] +[out] + +[case testAdditionalOperatorsInOpAssign] +a: A +c: C +a &= a # Fail +a >>= a # Fail +a //= a # Fail +a &= c +a >>= c +a //= c +class A: + def __and__(self, x: 'C') -> 'A': pass + def __rshift__(self, x: 'C') -> 'A': pass + def __floordiv__(self, x: 'C') -> 'A': pass +class C: pass +[builtins fixtures/tuple.pyi] +[out] +main:3: error: Unsupported operand types for & ("A" and "A") +main:4: error: Unsupported operand types for >> ("A" and "A") +main:5: error: Unsupported operand types for // ("A" and "A") + +[case testInplaceOperatorMethods] +import typing +class A: + def __iadd__(self, x: int) -> 'A': pass + def __imul__(self, x: str) -> 'A': pass + def __imatmul__(self, x: str) -> 'A': pass +a = A() +a += 1 +a *= '' +a @= '' +a += '' # E: Argument 1 to "__iadd__" of "A" has incompatible type "str"; expected "int" +a *= 1 # E: Argument 1 to "__imul__" of "A" has incompatible type "int"; expected "str" +a @= 1 # E: Argument 1 to "__imatmul__" of "A" has incompatible type "int"; expected "str" + +[case testInplaceSetitem] +class A(object): + def __init__(self) -> None: + self.a = [1] + + def __iadd__(self, a): + # type: (int) -> A + self.a += [2] + return self + +a = A() +b = [a] +b[0] += 1 +[builtins fixtures/list.pyi] +[out] + + +-- Assert statement +-- ---------------- + + +[case testAssert] +import typing +assert None + None # Fail +assert None +[out] +main:2: error: Unsupported left operand type for + ("None") + + +-- Exception handling +-- ------------------ + + +[case testRaiseStatement] + +e: BaseException +f: MyError +a: A +raise a # Fail +raise e +raise f +class A: pass +class MyError(BaseException): pass +[builtins fixtures/exception.pyi] +[out] +main:5: error: Exception must be derived from BaseException + +[case testRaiseClassObject] +class A: pass +class MyError(BaseException): pass +def f(): pass +if object(): + raise BaseException +if object(): + raise MyError +if object(): + raise A # E: Exception must be derived from BaseException +if object(): + raise object # E: Exception must be derived from BaseException +if object(): + raise f # E: Exception must be derived from BaseException +[builtins fixtures/exception.pyi] + +[case testRaiseClassObjectCustomInit] +class MyBaseError(BaseException): + def __init__(self, required) -> None: + ... +class MyError(Exception): + def __init__(self, required1, required2) -> None: + ... +class MyKwError(Exception): + def __init__(self, *, kwonly) -> None: + ... +class MyErrorWithDefault(Exception): + def __init__(self, optional=1) -> None: + ... +if object(): + raise BaseException +if object(): + raise Exception +if object(): + raise BaseException(1) +if object(): + raise Exception(2) +if object(): + raise MyBaseError(4) +if object(): + raise MyError(5, 6) +if object(): + raise MyKwError(kwonly=7) +if object(): + raise MyErrorWithDefault(8) +if object(): + raise MyErrorWithDefault +if object(): + raise MyBaseError # E: Too few arguments for "MyBaseError" +if object(): + raise MyError # E: Too few arguments for "MyError" +if object(): + raise MyKwError # E: Missing named argument "kwonly" for "MyKwError" +[builtins fixtures/exception.pyi] + +[case testRaiseExceptionType] +import typing +x: typing.Type[BaseException] +raise x +[builtins fixtures/exception.pyi] + +[case testRaiseNonExceptionTypeFails] +import typing +x = int # type: typing.Type[int] +raise x # E: Exception must be derived from BaseException +[builtins fixtures/exception.pyi] + +[case testRaiseUnion] +import typing +x: typing.Union[BaseException, typing.Type[BaseException]] +raise x +[builtins fixtures/exception.pyi] + +[case testRaiseNonExceptionUnionFails] +import typing +x: typing.Union[BaseException, int] +raise x # E: Exception must be derived from BaseException +[builtins fixtures/exception.pyi] + +[case testRaiseFromStatement] +e: BaseException +f: MyError +a: A +x: BaseException +del x +if object(): + raise e from a # E: Exception must be derived from BaseException +if object(): + raise e from e +if object(): + raise e from f +if object(): + raise e from x # E: Trying to read deleted variable "x" +class A: pass +class MyError(BaseException): pass +[builtins fixtures/exception.pyi] + +[case testRaiseFromClassobject] +import typing +class A: pass +class MyError(BaseException): pass +def f(): pass +if object(): + raise BaseException from BaseException +if object(): + raise BaseException from MyError +if object(): + raise BaseException from A # E: Exception must be derived from BaseException +if object(): + raise BaseException from object # E: Exception must be derived from BaseException +if object(): + raise BaseException from f # E: Exception must be derived from BaseException +[builtins fixtures/exception.pyi] + +[case testRaiseNotImplementedFails] +if object(): + raise NotImplemented # E: Exception must be derived from BaseException; did you mean "NotImplementedError"? +if object(): + raise NotImplemented() # E: NotImplemented? not callable +[builtins fixtures/notimplemented.pyi] + +[case testTryFinallyStatement] +import typing +try: + b = object() # type: A # Fail +finally: + c = object() # type: A # Fail +class A: pass +[out] +main:3: error: Incompatible types in assignment (expression has type "object", variable has type "A") +main:5: error: Incompatible types in assignment (expression has type "object", variable has type "A") + +[case testSimpleTryExcept] + +try: + pass +except BaseException as e: + a: BaseException + o: object + e = a + e = o # Fail +class A: pass +class B: pass +[builtins fixtures/exception.pyi] +[out] +main:8: error: Incompatible types in assignment (expression has type "object", variable has type "BaseException") + +[case testTypeErrorInBlock] +class A: pass +class B: pass +while int(): + x: A + if int(): + x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testTypeErrorInvolvingBaseException] +class A: pass + +x: BaseException +a: A +if int(): + a = BaseException() # E: Incompatible types in assignment (expression has type "BaseException", variable has type "A") +if int(): + a = object() # E: Incompatible types in assignment (expression has type "object", variable has type "A") +if int(): + x = object() # E: Incompatible types in assignment (expression has type "object", variable has type "BaseException") +if int(): + x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "BaseException") +if int(): + x = BaseException() +[builtins fixtures/exception.pyi] + +[case testSimpleTryExcept2] +import typing +try: + pass +except BaseException as e: + e = object() # Fail + e = BaseException() +[builtins fixtures/exception.pyi] +[out] +main:5: error: Incompatible types in assignment (expression has type "object", variable has type "BaseException") + +[case testBaseClassAsExceptionTypeInExcept] +import typing +class Err(BaseException): pass +try: + pass +except Err as e: + e = BaseException() # E: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") + e = Err() +[builtins fixtures/exception.pyi] +[case testMultipleExceptHandlers] +import typing +class Err(BaseException): pass +try: + pass +except BaseException as e: + pass +except Err as f: + f = BaseException() # E: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") + f = Err() +[builtins fixtures/exception.pyi] +[case testTryExceptStatement] +import typing +class A: pass +class B: pass +class Err(BaseException): pass +try: + a = B() # type: A # E: Incompatible types in assignment (expression has type "B", variable has type "A") +except BaseException as e: + e = A() # E: Incompatible types in assignment (expression has type "A", variable has type "BaseException") + e = Err() +except Err as f: + f = BaseException() # E: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") + f = Err() +[builtins fixtures/exception.pyi] +[case testTryExceptWithinFunction] +import typing +def f() -> None: + try: pass + except BaseException as e: + e = object() # Fail + e = BaseException() + except Err as f: + f = BaseException() # Fail + f = Err() +class Err(BaseException): pass +[builtins fixtures/exception.pyi] +[out] +main:5: error: Incompatible types in assignment (expression has type "object", variable has type "BaseException") +main:8: error: Incompatible types in assignment (expression has type "BaseException", variable has type "Err") + +[case testTryExceptFlow] +def f() -> None: + x = 1 + try: + pass + except: + raise + x + 'a' # E: Unsupported left operand type for + ("int") +[builtins fixtures/exception.pyi] +[out] + +[case testTryWithElse] +import typing +try: pass +except BaseException: pass +else: + object(None) # E: Too many arguments for "object" +[builtins fixtures/exception.pyi] + +[case testRedefinedFunctionInTryWithElse] +def f() -> None: pass +try: + pass +except BaseException: + f2 = f +else: + def f2() -> str: pass +try: + pass +except BaseException: + f3 = f +else: + def f3() -> None: pass +[builtins fixtures/exception.pyi] +[out] +main:7: error: Incompatible redefinition (redefinition with type "Callable[[], str]", original type "Callable[[], None]") + +[case testExceptWithoutType] +import typing +try: + -None # E: Unsupported operand type for unary - ("None") +except: + ~None # E: Unsupported operand type for ~ ("None") +[builtins fixtures/exception.pyi] + +[case testRaiseWithoutArgument] +import typing +try: + None +except: + raise +[builtins fixtures/exception.pyi] + +[case testExceptWithMultipleTypes] +import typing +class E1(BaseException): pass +class E2(E1): pass +try: + pass +except (E1, E2): pass +except (E1, object): pass # E: Exception type must be derived from BaseException (or be a tuple of exception classes) +except (object, E2): pass # E: Exception type must be derived from BaseException (or be a tuple of exception classes) +except (E1, (E2,)): pass # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + +except (E1, E2): pass +except ((E1, E2)): pass +except (((E1, E2))): pass +[builtins fixtures/exception.pyi] + +[case testExceptWithTypeType] +import typing +E = BaseException # type: typing.Type[BaseException] + +try: + pass +except E: + pass +[builtins fixtures/exception.pyi] + +[case testExceptWithMultipleTypes2] +import typing +class E1(BaseException): pass +class E2(E1): pass +try: + pass +except (E1, E2) as e1: + x = e1 # type: E1 + y = e1 # type: E2 # E: Incompatible types in assignment (expression has type "E1", variable has type "E2") +except (E2, E1) as e2: + a = e2 # type: E1 + b = e2 # type: E2 # E: Incompatible types in assignment (expression has type "E1", variable has type "E2") +except (E1, E2, int) as e3: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +[builtins fixtures/exception.pyi] + +[case testExceptWithMultipleTypes3] +import typing +class E1(BaseException): pass +class E1_1(E1): pass +class E1_2(E1): pass +try: pass +except (E1, E1_1, E1_2) as e1: + x = e1 # type: E1 + y = e1 # type: E1_1 # E: Incompatible types in assignment (expression has type "E1", variable has type "E1_1") + z = e1 # type: E1_2 # E: Incompatible types in assignment (expression has type "E1", variable has type "E1_2") +except (E1_1, E1_2) as e2: + a = e2 # type: E1 + b = e2 # type: E1_1 # E: Incompatible types in assignment (expression has type "Union[E1_1, E1_2]", variable has type "E1_1") + c = e2 # type: E1_2 # E: Incompatible types in assignment (expression has type "Union[E1_1, E1_2]", variable has type "E1_2") +[builtins fixtures/exception.pyi] + +[case testExceptWithMultipleTypes4] +from typing import Tuple, Type, Union + +class E1(BaseException): pass +class E2(BaseException): pass +class E3(BaseException): pass + +def variadic(exc: Tuple[Type[E1], ...]) -> None: + try: + pass + except exc as e: + reveal_type(e) # N: Revealed type is "__main__.E1" + +def union(exc: Union[Type[E1], Type[E2]]) -> None: + try: + pass + except exc as e: + reveal_type(e) # N: Revealed type is "Union[__main__.E1, __main__.E2]" + +def tuple_in_union(exc: Union[Type[E1], Tuple[Type[E2], Type[E3]]]) -> None: + try: + pass + except exc as e: + reveal_type(e) # N: Revealed type is "Union[__main__.E1, __main__.E2, __main__.E3]" + +def variadic_in_union(exc: Union[Type[E1], Tuple[Type[E2], ...]]) -> None: + try: + pass + except exc as e: + reveal_type(e) # N: Revealed type is "Union[__main__.E1, __main__.E2]" + +def nested_union(exc: Union[Type[E1], Union[Type[E2], Type[E3]]]) -> None: + try: + pass + except exc as e: + reveal_type(e) # N: Revealed type is "Union[__main__.E1, __main__.E2, __main__.E3]" + +def error_in_union(exc: Union[Type[E1], int]) -> None: + try: + pass + except exc as e: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass + +def error_in_variadic(exc: Tuple[int, ...]) -> None: + try: + pass + except exc as e: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass + +[builtins fixtures/tuple.pyi] + +[case testExceptWithAnyTypes] +from typing import Any + +E1 = None # type: Any +class E2(BaseException): pass +class NotBaseDerived: pass + +try: + pass +except BaseException as e1: + reveal_type(e1) # N: Revealed type is "builtins.BaseException" +except (E1, BaseException) as e2: + reveal_type(e2) # N: Revealed type is "Union[Any, builtins.BaseException]" +except (E1, E2) as e3: + reveal_type(e3) # N: Revealed type is "Union[Any, __main__.E2]" +except (E1, E2, BaseException) as e4: + reveal_type(e4) # N: Revealed type is "Union[Any, builtins.BaseException]" + +try: pass +except E1 as e1: + reveal_type(e1) # N: Revealed type is "Any" +except E2 as e2: + reveal_type(e2) # N: Revealed type is "__main__.E2" +except NotBaseDerived as e3: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +except (NotBaseDerived, E1) as e4: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +except (NotBaseDerived, E2) as e5: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +except (NotBaseDerived, E1, E2) as e6: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +except (E1, E2, NotBaseDerived) as e6: # E: Exception type must be derived from BaseException (or be a tuple of exception classes) + pass +[builtins fixtures/exception.pyi] + +[case testReuseTryExceptionVariable] +import typing +class E1(BaseException): pass +class E2(BaseException): pass +try: pass +except E1 as e: pass +try: pass +except E1 as e: pass +try: pass +except E2 as e: pass +e + 1 # E: Trying to read deleted variable "e" # E: Name "e" is used before definition +e = E1() # E: Assignment to variable "e" outside except: block +[builtins fixtures/exception.pyi] + +[case testReuseDefinedTryExceptionVariable] +import typing +class E1(BaseException): pass +class E2(BaseException): pass +e = 1 +def f(): e # Prevent redefinition +e = 1 +try: pass +except E1 as e: pass +e = 1 # E: Assignment to variable "e" outside except: block +e = E1() # E: Assignment to variable "e" outside except: block +[builtins fixtures/exception.pyi] + +[case testExceptionVariableReuseInDeferredNode1] +def f(*a: BaseException) -> int: + x + try: pass + except BaseException as err: pass + try: pass + except BaseException as err: f(err) + return 0 +x = f() +[builtins fixtures/exception.pyi] + +[case testExceptionVariableReuseInDeferredNode2] +def f(*a: BaseException) -> int: + try: pass + except BaseException as err: pass + x + try: pass + except BaseException as err: f(err) + return 0 +x = f() +[builtins fixtures/exception.pyi] + +[case testExceptionVariableReuseInDeferredNode3] +def f(*a: BaseException) -> int: + try: pass + except BaseException as err: pass + try: pass + except BaseException as err: f(err) + x + return 0 +x = f() +[builtins fixtures/exception.pyi] + +[case testExceptionVariableReuseInDeferredNode4] +class EA(BaseException): + a = None # type: int +class EB(BaseException): + b = None # type: str +def f(*arg: BaseException) -> int: + x + try: pass + except EA as err: + f(err) + a = err.a + reveal_type(a) + try: pass + except EB as err: + f(err) + b = err.b + reveal_type(b) + return 0 +x = f() +[builtins fixtures/exception.pyi] +[out] +main:11: note: Revealed type is "builtins.int" +main:16: note: Revealed type is "builtins.str" + +[case testExceptionVariableReuseInDeferredNode5] +class EA(BaseException): + a = None # type: int +class EB(BaseException): + b = None # type: str +def f(*arg: BaseException) -> int: + try: pass + except EA as err: + f(err) + a = err.a + reveal_type(a) + x + try: pass + except EB as err: + f(err) + b = err.b + reveal_type(b) + return 0 +x = f() +[builtins fixtures/exception.pyi] +[out] +main:10: note: Revealed type is "builtins.int" +main:16: note: Revealed type is "builtins.str" + +[case testExceptionVariableReuseInDeferredNode6] +class EA(BaseException): + a = None # type: int +class EB(BaseException): + b = None # type: str +def f(*arg: BaseException) -> int: + try: pass + except EA as err: + f(err) + a = err.a + reveal_type(a) + try: pass + except EB as err: + f(err) + b = err.b + reveal_type(b) + x + return 0 +x = f() +[builtins fixtures/exception.pyi] +[out] +main:10: note: Revealed type is "builtins.int" +main:15: note: Revealed type is "builtins.str" + +[case testExceptionVariableWithDisallowAnyExprInDeferredNode] +# flags: --disallow-any-expr +def f() -> int: + x + try: + pass + except Exception as ex: + pass + return 0 +x = f() +[builtins fixtures/exception.pyi] + +[case testArbitraryExpressionAsExceptionType] +import typing +a = BaseException +try: pass +except a as b: + b = BaseException() + b = object() # E: Incompatible types in assignment (expression has type "object", variable has type "BaseException") +[builtins fixtures/exception.pyi] + +[case testInvalidExceptionCallable] +import typing +def exc() -> BaseException: pass +try: pass +except exc as e: pass # E: Exception type must be derived from BaseException (or be a tuple of exception classes) +except BaseException() as b: pass # E: Exception type must be derived from BaseException (or be a tuple of exception classes) +[builtins fixtures/exception.pyi] + +[case testTupleValueAsExceptionType] +import typing +def exc() -> BaseException: pass +class E1(BaseException): pass +class E1_1(E1): pass +class E1_2(E1): pass + +exs1 = (E1, E1_1, E1_2) +try: pass +except exs1 as e1: + x = e1 # type: E1 + y = e1 # type: E1_1 # E: Incompatible types in assignment (expression has type "E1", variable has type "E1_1") + z = e1 # type: E1_2 # E: Incompatible types in assignment (expression has type "E1", variable has type "E1_2") + +exs2 = (E1_1, E1_2) +try: pass +except exs2 as e2: + a = e2 # type: E1 + b = e2 # type: E1_1 # E: Incompatible types in assignment (expression has type "Union[E1_1, E1_2]", variable has type "E1_1") + c = e2 # type: E1_2 # E: Incompatible types in assignment (expression has type "Union[E1_1, E1_2]", variable has type "E1_2") + +exs3 = (E1, (E1_1, (E1_2,))) +try: pass +except exs3 as e3: pass # E: Exception type must be derived from BaseException (or be a tuple of exception classes) +[builtins fixtures/exception.pyi] + +[case testInvalidTupleValueAsExceptionType] +import typing +def exc() -> BaseException: pass +class E1(BaseException): pass +class E2(E1): pass + +exs1 = (E1, E2, int) +try: pass +except exs1 as e: pass # E: Exception type must be derived from BaseException (or be a tuple of exception classes) +[builtins fixtures/exception.pyi] + +[case testOverloadedExceptionType] +from foo import * +[file foo.pyi] +from typing import overload +class E(BaseException): + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x) -> None: pass +try: + pass +except E as e: + e = E() + e = BaseException() # E: Incompatible types in assignment (expression has type "BaseException", variable has type "E") +[builtins fixtures/exception.pyi] + +[case testExceptionWithAnyBaseClass] +from typing import Any +E = None # type: Any +class EE(E): pass +raise EE() +raise EE +[builtins fixtures/exception.pyi] + +[case testExceptionIsType] +from typing import Type +class B(BaseException): pass +def f(e: Type[B]): + try: pass + except e: pass +def g(e: Type[BaseException]): + try: pass + except e as err: + reveal_type(err) +def h(e: Type[int]): + try: pass + except e: pass +[builtins fixtures/exception.pyi] +[out] +main:9: note: Revealed type is "builtins.BaseException" +main:12: error: Exception type must be derived from BaseException (or be a tuple of exception classes) + + +-- Del statement +-- ------------- + + +[case testDelStmtWithIndex] +a: A +b: B +del b[a] +del b[b] # E: Argument 1 to "__delitem__" of "B" has incompatible type "B"; expected "A" +del a[a] # E: "A" has no attribute "__delitem__" +del a[b] # E: "A" has no attribute "__delitem__" +class B: + def __delitem__(self, index: 'A'): pass +class A: pass +[builtins fixtures/tuple.pyi] + +[case testDelStmtWithAttribute] +class A: + def f(self): pass + x = 0 +a = A() +del a.f +del a.x +del a.z # E: "A" has no attribute "z" + +[case testDelStatementWithTuple] +class A: + x = 0 +a = A() +del a.x, a.y # E: "A" has no attribute "y" +[builtins fixtures/tuple.pyi] + +[case testDelStmtWithTypeInfo] +class Foo: ... +del Foo +Foo + 1 # E: Trying to read deleted variable "Foo" + +[case testDelStatementWithAssignmentSimple] +a = 1 +a + 1 +del a +a + 1 # E: Trying to read deleted variable "a" +[builtins fixtures/ops.pyi] + +[case testDelStatementWithAssignmentTuple] +a = 1 +b = 1 +del (a, b) +b + 1 # E: Trying to read deleted variable "b" +[builtins fixtures/ops.pyi] + +[case testDelStatementWithAssignmentList] +a = 1 +b = 1 +del [a, b] +b + 1 # E: Trying to read deleted variable "b" +[builtins fixtures/list.pyi] + +[case testDelStatementWithAssignmentClass] +class C: + a = 1 + +c = C() +c.a = 1 +c.a + 1 +del c.a +c.a + 1 +[builtins fixtures/ops.pyi] + +[case testDelStatementWithConditions] +x = 5 +del x +if x: ... # E: Trying to read deleted variable "x" + +def f(x): + return x + +if 0: ... +elif f(x): ... # E: Trying to read deleted variable "x" + +while x == 5: ... # E: Trying to read deleted variable "x" + +-- Yield statement +-- --------------- + + +[case testSimpleYield] +from typing import Iterator +def f() -> Iterator[int]: + yield 1 + yield '' # E: Incompatible types in "yield" (actual type "str", expected type "int") +[builtins fixtures/for.pyi] +[out] + +[case testYieldInFunctionReturningGenerator] +from typing import Generator +def f() -> Generator[int, None, None]: + yield 1 +[builtins fixtures/for.pyi] +[out] + +[case testYieldInFunctionReturningIterable] +from typing import Iterable +def f() -> Iterable[int]: + yield 1 +[builtins fixtures/for.pyi] +[out] + +[case testYieldInFunctionReturningObject] +def f() -> object: + yield 1 +[builtins fixtures/for.pyi] +[out] + +[case testYieldInFunctionReturningAny] +from typing import Any +def f() -> Any: + yield object() +[out] + +[case testYieldInFunctionReturningFunction] +from typing import Callable +def f() -> Callable[[], None]: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield object() +[out] + +[case testYieldInDynamicallyTypedFunction] +import typing +def f(): + yield f + +[case testWithInvalidInstanceReturnType] +import typing +def f() -> int: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield 1 +[builtins fixtures/for.pyi] +[out] + +[case testTypeInferenceContextAndYield] +from typing import List, Iterator +def f() -> 'Iterator[List[int]]': + yield [] + yield [object()] # E: List item 0 has incompatible type "object"; expected "int" +[builtins fixtures/for.pyi] +[out] + +[case testYieldAndReturnWithoutValue] +from typing import Iterator +def f() -> Iterator[int]: + yield 1 + return +[builtins fixtures/for.pyi] + +[case testYieldWithNoValue] +from typing import Iterator +def f() -> Iterator[None]: + yield +[builtins fixtures/for.pyi] + +[case testYieldWithNoValueWhenValueRequired] +from typing import Iterator +def f() -> Iterator[int]: + yield # E: Yield value expected +[builtins fixtures/for.pyi] +[out] + +[case testYieldWithExplicitNone] +from typing import Iterator +def f() -> Iterator[None]: + yield None +[builtins fixtures/for.pyi] +[out] + + +-- Yield from statement +-- -------------------- +-- +-- (It's not really a statement, but don't want to move the tests.) + +[case testSimpleYieldFromWithIterator] +from typing import Iterator +def g() -> Iterator[str]: + yield '42' +def h() -> Iterator[int]: + yield 42 +def f() -> Iterator[str]: + yield from g() + yield from h() # E: Incompatible types in "yield from" (actual type "int", expected type "str") +[out] + +[case testYieldFromAppliedToAny] +from typing import Any +def g() -> Any: + yield object() +def f() -> Any: + yield from g() +[out] + +[case testYieldFromInFunctionReturningFunction] +from typing import Iterator, Callable +def g() -> Iterator[int]: + yield 42 +def f() -> Callable[[], None]: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield from g() +[out] + +[case testYieldFromNotIterableReturnType] +from typing import Iterator +def g() -> Iterator[int]: + yield 42 +def f() -> int: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield from g() +[out] + +[case testYieldFromNotAppliedIterator] +from typing import Iterator +def g() -> int: + return 42 +def f() -> Iterator[int]: + yield from g() # E: "yield from" can't be applied to "int" +[out] + +[case testYieldFromCheckIncompatibleTypesTwoIterables] +from typing import List, Iterator +def g() -> Iterator[List[int]]: + yield [2, 3, 4] +def f() -> Iterator[List[int]]: + yield from g() + yield from [1, 2, 3] # E: Incompatible types in "yield from" (actual type "int", expected type "List[int]") +[builtins fixtures/for.pyi] +[out] + +[case testYieldFromNotAppliedToNothing] +def h(): + yield from # E: invalid syntax +[out] + +[case testYieldFromAndYieldTogether] +from typing import Iterator +def f() -> Iterator[str]: + yield "g1 ham" + yield from g() + yield "g1 eggs" +def g() -> Iterator[str]: + yield "g2 spam" + yield "g2 more spam" +[out] + +[case testYieldFromAny] +from typing import Iterator +def f(a): + b = yield from a + return b +[out] + +[case testYieldFromGenericCall] +from typing import Generator, TypeVar +T = TypeVar('T') +def f(a: T) -> Generator[int, str, T]: pass +def g() -> Generator[int, str, float]: + r = yield from f('') + reveal_type(r) # N: Revealed type is "builtins.str" + return 3.14 + +[case testYieldFromTupleStatement] +from typing import Generator +def g() -> Generator[int, None, None]: + yield from () + yield from (0, 1, 2) + yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int") + yield from ("ERROR",) # E: Incompatible types in "yield from" (actual type "str", expected type "int") +[builtins fixtures/tuple.pyi] + +-- With statement +-- -------------- + + +[case testSimpleWith] +import typing +class A: + def __enter__(self) -> None: pass + def __exit__(self, x, y, z) -> None: pass +with A(): + object(A) # E: Too many arguments for "object" + +[case testWithStmtAndInvalidExit] +import typing +class A: + def __enter__(self) -> None: pass + def __exit__(self, x, y) -> None: pass +with A(): # E: Too many arguments for "__exit__" of "A" + pass + +[case testWithStmtAndMissingExit] +import typing +class A: + def __enter__(self) -> None: pass +with A(): # E: "A" has no attribute "__exit__" + pass + +[case testWithStmtAndInvalidEnter] +import typing +class A: + def __enter__(self, x) -> None: pass + def __exit__(self, x, y, z) -> None: pass +with A(): # E: Too few arguments for "__enter__" of "A" + pass + +[case testWithStmtAndMissingEnter] +import typing +class A: + def __exit__(self, x, y, z) -> None: pass +with A(): # E: "A" has no attribute "__enter__" + pass + +[case testWithStmtAndMultipleExprs] +import typing +class A: + def __enter__(self) -> None: pass + def __exit__(self, x, y, z) -> None: pass +class B: + def __enter__(self) -> None: pass +with A(), B(): # E: "B" has no attribute "__exit__" + pass +with B(), A(): # E: "B" has no attribute "__exit__" + pass + +[case testWithStmtAndResult] +import typing +class B: pass +class A: + def __enter__(self) -> B: pass + def __exit__(self, x, y, z): pass +with A() as b: + b = B() + b = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testWithStmtAndMultipleResults] +from typing import TypeVar, Generic +t = TypeVar('t') +class B: pass +class C: pass +class A(Generic[t]): + def __enter__(self) -> t: pass + def __exit__(self, x, y, z): pass +a_b = A() # type: A[B] +a_c = A() # type: A[C] +with a_b as b, a_c as c: + b = B() + c = C() + b = c # E: Incompatible types in assignment (expression has type "C", variable has type "B") + c = b # E: Incompatible types in assignment (expression has type "B", variable has type "C") + +[case testWithStmtAndComplexTarget] +from typing import Tuple +class A: + def __enter__(self) -> Tuple[int, str]: pass + def __exit__(self, x, y, z): pass +with A() as (a, b): + a = 1 + b = '' + a = b # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] + +[case testWithStmtTypeComment] + +from typing import Union +class A: + def __enter__(self) -> int: pass + def __exit__(self, x, y, z): pass + +with A(): # type: int # E: Invalid type comment: "with" statement has no targets + pass + +with A() as a: # type: int + pass + +with A() as b: # type: str # E: Incompatible types in assignment (expression has type "int", variable has type "str") + pass + +with A() as c: # type: int, int # E: Syntax error in type annotation # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + pass + +with A() as d: # type: Union[int, str] + reveal_type(d) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testWithStmtTupleTypeComment] + +from typing import Tuple +class A: + def __enter__(self) -> Tuple[int, int]: pass + def __exit__(self, x, y, z): pass + +with A(): + pass + +with A() as a: # type: Tuple[int, int] + pass + +with A() as b: # type: Tuple[int, str] # E: Incompatible types in assignment (expression has type "Tuple[int, int]", variable has type "Tuple[int, str]") + pass + +with A() as (c, d): # type: int, int + pass + +with A() as (e, f): # type: Tuple[int, int] + pass + +with A() as (g, h): # type: int # E: Tuple type expected for multiple variables + pass + +with A() as (i, j): # type: int, int, str # E: Incompatible number of tuple items + pass + +with A() as (k, l): # type: int, str # E: Incompatible types in assignment (expression has type "int", variable has type "str") + pass +[builtins fixtures/tuple.pyi] + +[case testWithStmtComplexTypeComment] + +from typing import Tuple +class A: + def __enter__(self) -> Tuple[int, int]: pass + def __exit__(self, x, y, z): pass + +class B: + def __enter__(self) -> str: pass + def __exit__(self, x, y, z): pass + +with A() as a, A() as (b, c), B() as d: # type: Tuple[int, int], (int, int), str + pass + +with A() as e, A() as (f, g), B() as h: # type: Tuple[int, int], Tuple[int, int], str + pass + +with A() as i, A() as (j, k), B() as l: # type: (int, int), (int, int), str # E: Syntax error in type annotation # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + pass + +with A(), A(), B() as m, A() as n, B(), B() as o: # type: int, Tuple[int, int] # E: Incompatible number of types for "with" targets + pass + +with A(), B(), B() as p, A(), A(): # type: str + pass +[builtins fixtures/tuple.pyi] + +[case testWithStmtBoolExitReturnWithResultFalse] +from typing import Optional + +class InvalidReturn1: + def __exit__(self, x, y, z) -> bool: # E: "bool" is invalid as return type for "__exit__" that always returns False \ +# N: Use "typing_extensions.Literal[False]" as the return type or change it to "None" \ +# N: If return type of "__exit__" implies that it may return True, the context manager may swallow exceptions + return False + +class InvalidReturn2: + def __exit__(self, x, y, z) -> Optional[bool]: # E: "bool" is invalid as return type for "__exit__" that always returns False \ +# N: Use "typing_extensions.Literal[False]" as the return type or change it to "None" \ +# N: If return type of "__exit__" implies that it may return True, the context manager may swallow exceptions + if int(): + return False + else: + return False + +class InvalidReturn3: + def __exit__(self, x, y, z) -> bool: # E: "bool" is invalid as return type for "__exit__" that always returns False \ +# N: Use "typing_extensions.Literal[False]" as the return type or change it to "None" \ +# N: If return type of "__exit__" implies that it may return True, the context manager may swallow exceptions + def nested() -> bool: + return True + return False +[builtins fixtures/bool.pyi] + +[case testWithStmtBoolExitReturnOkay] +from typing_extensions import Literal + +class GoodReturn1: + def __exit__(self, x, y, z) -> bool: + if int(): + return True + else: + return False + +class GoodReturn2: + def __exit__(self, x, y, z) -> bool: + if int(): + return False + else: + return True + +class GoodReturn3: + def __exit__(self, x, y, z) -> bool: + return bool() + +class GoodReturn4: + def __exit__(self, x, y, z) -> None: + return + +class GoodReturn5: + def __exit__(self, x, y, z) -> None: + return None + +class GoodReturn6: + def exit(self, x, y, z) -> bool: + return False + +class GoodReturn7: + def exit(self, x, y, z) -> bool: + pass + +class MissingReturn: + def exit(self, x, y, z) -> bool: # E: Missing return statement + x = 0 + +class LiteralReturn: + def __exit__(self, x, y, z) -> Literal[False]: + return False +[builtins fixtures/bool.pyi] + +[case testWithStmtBoolExitReturnInStub] +import stub + +[file stub.pyi] +from typing import Optional + +class C1: + def __exit__(self, x, y, z) -> bool: ... + +class C2: + def __exit__(self, x, y, z) -> bool: pass + +class C3: + def __exit__(self, x, y, z) -> Optional[bool]: pass +[builtins fixtures/bool.pyi] + +[case testWithStmtScopeBasics] +from m import A, B + +def f1() -> None: + with A() as x: + reveal_type(x) # N: Revealed type is "m.A" + with B() as x: + reveal_type(x) # N: Revealed type is "m.B" + +def f2() -> None: + with A() as x: + reveal_type(x) # N: Revealed type is "m.A" + y = x # Use outside with makes the scope function-level + with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(x) # N: Revealed type is "m.A" + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndFuncDef] +from m import A, B + +with A() as x: + reveal_type(x) # N: Revealed type is "m.A" + +def f() -> None: + pass # Don't support function definition in the middle + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(x) # N: Revealed type is "m.A" + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndFuncDef2] +from m import A, B + +def f() -> None: + pass # function before with is unsupported + +with A() as x: + reveal_type(x) # N: Revealed type is "m.A" + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(x) # N: Revealed type is "m.A" + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndFuncDef3] +from m import A, B + +with A() as x: + reveal_type(x) # N: Revealed type is "m.A" + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(x) # N: Revealed type is "m.A" + +def f() -> None: + pass # function after with is unsupported + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndFuncDef4] +from m import A, B + +with A() as x: + def f() -> None: + pass # Function within with is unsupported + + reveal_type(x) # N: Revealed type is "m.A" + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(x) # N: Revealed type is "m.A" + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndImport1] +from m import A, B, x + +with A() as x: \ + # E: Incompatible types in assignment (expression has type "A", variable has type "B") + reveal_type(x) # N: Revealed type is "m.B" + +with B() as x: + reveal_type(x) # N: Revealed type is "m.B" + +[file m.pyi] +x: B + +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndImport2] +from m import A, B +import m as x + +with A() as x: \ + # E: Incompatible types in assignment (expression has type "A", variable has type Module) + pass + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type Module) + pass + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... +[builtins fixtures/module.pyi] + +[case testWithStmtScopeAndImportStar] +from m import A, B +from m import * + +with A() as x: + pass + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeNestedWith1] +from m import A, B + +with A() as x: + with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(x) # N: Revealed type is "m.A" + +with B() as x: + with A() as x: \ + # E: Incompatible types in assignment (expression has type "A", variable has type "B") + reveal_type(x) # N: Revealed type is "m.B" + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeNestedWith2] +from m import A, B + +with A() as x: + with A() as y: + reveal_type(y) # N: Revealed type is "m.A" + with B() as y: + reveal_type(y) # N: Revealed type is "m.B" + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeInnerAndOuterScopes] +from m import A, B + +x = A() # Outer scope should have no impact + +with A() as x: + pass + +def f() -> None: + with A() as x: + reveal_type(x) # N: Revealed type is "m.A" + with B() as x: + reveal_type(x) # N: Revealed type is "m.B" + +y = x + +with A() as x: + pass + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeMultipleContextManagers] +from m import A, B + +with A() as x, B() as y: + reveal_type(x) # N: Revealed type is "m.A" + reveal_type(y) # N: Revealed type is "m.B" +with B() as x, A() as y: + reveal_type(x) # N: Revealed type is "m.B" + reveal_type(y) # N: Revealed type is "m.A" + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeMultipleAssignment] +from m import A, B + +with A() as (x, y): + reveal_type(x) # N: Revealed type is "m.A" + reveal_type(y) # N: Revealed type is "builtins.int" +with B() as [x, y]: + reveal_type(x) # N: Revealed type is "m.B" + reveal_type(y) # N: Revealed type is "builtins.str" + +[file m.pyi] +from typing import Tuple + +class A: + def __enter__(self) -> Tuple[A, int]: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> Tuple[B, str]: ... + def __exit__(self, x, y, z) -> None: ... +[builtins fixtures/tuple.pyi] + +[case testWithStmtScopeComplexAssignments] +from m import A, B, f + +with A() as x: + pass +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass +with B() as f(x).x: + pass + +with A() as y: + pass +with B() as y: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass +with B() as f(y)[0]: + pass + +[file m.pyi] +def f(x): ... + +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndClass] +from m import A, B + +with A() as x: + pass + +class C: + with A() as y: + pass + with B() as y: + pass + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass + +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeInnerScopeReference] +from m import A, B + +with A() as x: + def f() -> A: + return x + f() + +with B() as x: \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") + pass +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + +[case testWithStmtScopeAndLambda] +from m import A, B + +# This is technically not correct, since the lambda can outlive the with +# statement, but this behavior seems more intuitive. + +with A() as x: + lambda: reveal_type(x) # N: Revealed type is "m.A" + +with B() as x: + pass +[file m.pyi] +class A: + def __enter__(self) -> A: ... + def __exit__(self, x, y, z) -> None: ... +class B: + def __enter__(self) -> B: ... + def __exit__(self, x, y, z) -> None: ... + + +-- Chained assignment +-- ------------------ + + +[case testChainedAssignment] +import typing +class A: pass +class B: pass +x = y = A() +if int(): + x = A() +if int(): + y = A() +if int(): + x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + y = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testChainedAssignment2] +import typing +def f() -> None: + x = 1 + y = 'x' + if int(): + x = y = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + x = y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/primitives.pyi] +[out] + +[case testChainedAssignmentWithType] +# flags: --no-strict-optional +x = y = None # type: int +if int(): + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + y = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + x = 1 +if int(): + y = 1 + + +-- Star assignment +-- --------------- + + +[case testAssignListToStarExpr] +from typing import List +bs: List[A] +cs: List[B] +if int(): + *bs, b = bs +if int(): + *bs, c = cs # E: Incompatible types in assignment (expression has type "List[B]", variable has type "List[A]") + if int(): + *ns, c = cs +if int(): + nc = cs + +class A: pass +class B: pass +[builtins fixtures/list.pyi] + + +-- Type aliases +-- ------------ + + +[case testSimpleTypeAlias] +import typing +foo = int +def f(x: foo) -> None: pass +f(1) +f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testTypeAliasDefinedInAModule] +import typing +import m +def f(x: m.foo) -> None: pass +f(1) +f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +[file m.py] +import typing +foo = int + +[case testTypeAliasDefinedInAModule2] +import typing +from m import foo +def f(x: foo) -> None: pass +f(1) +f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +[file m.py] +import typing +foo = int + + +-- nonlocal and global +-- ------------------- + + +[case testTypeOfGlobalUsed] +import typing +class A(): pass +class B(): pass +g = A() +def f() -> None: + global g + g = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[case testTypeOfNonlocalUsed] +import typing +def f() -> None: + a = A() + def g() -> None: + nonlocal a + a = B() + +class A(): pass +class B(): pass +[out] +main:6: error: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testTypeOfOuterMostNonlocalUsed] +import typing +def f() -> None: + a = A() + def g() -> None: + a = B() + def h() -> None: + nonlocal a + a = A() + a = B() + +class A(): pass +class B(): pass +[out] +main:8: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testAugmentedAssignmentIntFloat] +weight0 = 65.5 +reveal_type(weight0) # N: Revealed type is "builtins.float" +if int(): + weight0 = 65 + reveal_type(weight0) # N: Revealed type is "builtins.int" + weight0 *= 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "float") + weight0 *= 0.5 + reveal_type(weight0) # N: Revealed type is "builtins.float" + weight0 *= object() # E: Unsupported operand types for * ("float" and "object") + reveal_type(weight0) # N: Revealed type is "builtins.float" + +[builtins fixtures/float.pyi] + +[case testAugmentedAssignmentIntFloatMember] +class A: + def __init__(self) -> None: + self.weight0 = 65.5 + reveal_type(self.weight0) # N: Revealed type is "builtins.float" + self.weight0 = 65 + reveal_type(self.weight0) # N: Revealed type is "builtins.int" + self.weight0 *= 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "float") + self.weight0 *= 0.5 + reveal_type(self.weight0) # N: Revealed type is "builtins.float" + self.weight0 *= object() # E: Unsupported operand types for * ("float" and "object") + reveal_type(self.weight0) # N: Revealed type is "builtins.float" + +[builtins fixtures/float.pyi] + +[case testAugmentedAssignmentIntFloatDict] +from typing import Dict +d = {'weight0': 65.5} +reveal_type(d['weight0']) # N: Revealed type is "builtins.float" +d['weight0'] = 65 +reveal_type(d['weight0']) # N: Revealed type is "builtins.float" +d['weight0'] *= 'a' # E: Unsupported operand types for * ("float" and "str") +d['weight0'] *= 0.5 +reveal_type(d['weight0']) # N: Revealed type is "builtins.float" +d['weight0'] *= object() # E: Unsupported operand types for * ("float" and "object") +reveal_type(d['weight0']) # N: Revealed type is "builtins.float" + +[builtins fixtures/floatdict.pyi] + +[case testForwardRefsInForStatementImplicit] +from typing import List, NamedTuple +lst: List[N] + +for i in lst: + reveal_type(i.x) # N: Revealed type is "builtins.int" + a: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +N = NamedTuple('N', [('x', int)]) +[builtins fixtures/list.pyi] +[out] + +[case testForwardRefsInForStatement] +from typing import List, NamedTuple +lst: List[M] + +for i in lst: # type: N + reveal_type(i.x) # N: Revealed type is "builtins.int" + a: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +N = NamedTuple('N', [('x', int)]) +class M(N): pass +[builtins fixtures/list.pyi] +[out] + +[case testForwardRefsInWithStatementImplicit] +from typing import ContextManager, Any +from mypy_extensions import TypedDict +cm: ContextManager[N] + +with cm as g: + a: int = g['x'] + +N = TypedDict('N', {'x': int}) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-medium.pyi] +[out] + +[case testForwardRefsInWithStatement] +from typing import ContextManager, Any +from mypy_extensions import TypedDict +cm: ContextManager[Any] + +with cm as g: # type: N + a: str = g['x'] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +N = TypedDict('N', {'x': int}) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-medium.pyi] +[out] + +[case testGlobalWithoutInitialization] +# flags: --disable-error-code=annotation-unchecked +from typing import List + +def foo() -> None: + global bar + # TODO: Confusing error message + bar = [] # type: List[str] # E: Name "bar" already defined (possibly by an import) + bar # E: Name "bar" is not defined + +def foo2(): + global bar2 + bar2 = [] # type: List[str] + bar2 +[builtins fixtures/list.pyi] + +[case testNoteUncheckedAnnotation] +def foo(): + x: int = "no" # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs + y = "no" # type: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs + z: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs + +[case testGeneratorUnion] +from typing import Generator, Union + +class A: pass +class B: pass + +def foo(x: int) -> Union[Generator[A, None, None], Generator[B, None, None]]: + yield x # E: Incompatible types in "yield" (actual type "int", expected type "Union[A, B]") + +[case testYieldFromUnionOfGenerators] +from typing import Generator, Union + +class T: pass + +def foo(arg: Union[Generator[int, None, T], Generator[str, None, T]]) -> Generator[Union[int, str], None, T]: + return (yield from arg) + +[case testYieldFromInvalidUnionReturn] +from typing import Generator, Union + +class A: pass +class B: pass + +def foo(arg: Union[A, B]) -> Generator[Union[int, str], None, A]: + return (yield from arg) # E: "yield from" can't be applied to "Union[A, B]" + +[case testYieldFromUnionOfGeneratorWithIterableStr] +from typing import Generator, Union, Iterable, Optional + +def foo(arg: Union[Generator[int, None, bytes], Iterable[str]]) -> Generator[Union[int, str], None, Optional[bytes]]: + return (yield from arg) + +def bar(arg: Generator[str, None, str]) -> Generator[str, None, str]: + return foo(arg) # E: Incompatible return value type (got "Generator[Union[int, str], None, Optional[bytes]]", expected "Generator[str, None, str]") + +def launder(arg: Iterable[str]) -> Generator[Union[int, str], None, Optional[bytes]]: + return foo(arg) + +def baz(arg: Generator[str, None, str]) -> Generator[Union[int, str], None, Optional[bytes]]: + # this is unsound, the Generator return type will actually be str + return launder(arg) +[builtins fixtures/tuple.pyi] + +[case testYieldIteratorReturn] +from typing import Iterator + +def get_strings(foo: bool) -> Iterator[str]: + if foo: + return ["foo1", "foo2"] # E: No return value expected + else: + yield "bar1" + yield "bar2" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnStarRightHandSide] +x = *(1, 2, 3) # E: can't use starred expression here +[builtins fixtures/tuple.pyi] + + +[case testTypingExtensionsSuggestion] +from typing import _FutureFeatureFixture + +# This import is only needed in tests. In real life, mypy will always have typing_extensions in its +# build due to its pervasive use in typeshed. This assumption may one day prove False, but when +# that day comes this suggestion will also be less helpful than it is today. +import typing_extensions +[out] +main:1: error: Module "typing" has no attribute "_FutureFeatureFixture" +main:1: note: Use `from typing_extensions import _FutureFeatureFixture` instead +main:1: note: See https://mypy.readthedocs.io/en/stable/runtime_troubles.html#using-new-additions-to-the-typing-module +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnBreakOutsideLoopFunction] +def foo(): + for x in [1, 2]: + def inner(): + break # E: "break" outside loop +[builtins fixtures/list.pyi] + +[case testNoCrashOnBreakOutsideLoopClass] +class Outer: + for x in [1, 2]: + class Inner: + break # E: "break" outside loop +[builtins fixtures/list.pyi] + +[case testCallableInstanceOverlapAllowed] +# flags: --warn-unreachable +from typing import Any, Callable, List + +class CAny: + def __call__(self) -> Any: ... +class CNone: + def __call__(self) -> None: ... +class CWrong: + def __call__(self, x: int) -> None: ... + +def describe(func: Callable[[], None]) -> str: + if isinstance(func, CAny): + return "CAny" + elif isinstance(func, CNone): + return "CNone" + elif isinstance(func, CWrong): + return "CWrong" # E: Statement is unreachable + else: + return "other" + +class C(CAny): + def __call__(self) -> None: ... + +def f(): + pass + +describe(CAny()) +describe(C()) +describe(CNone()) +describe(CWrong()) # E: Argument 1 to "describe" has incompatible type "CWrong"; expected "Callable[[], None]" \ + # N: "CWrong.__call__" has type "Callable[[Arg(int, 'x')], None]" +describe(f) +[builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/check-super.test b/test-data/unit/check-super.test new file mode 100644 index 000000000000..8816322a270a --- /dev/null +++ b/test-data/unit/check-super.test @@ -0,0 +1,434 @@ +-- Test cases for type checker related to super(). + + +-- Supertype member reference +-- -------------------------- + + +[case testAccessingSupertypeMethod] + +class B: + def f(self) -> 'B': pass +class A(B): + def f(self) -> 'A': + a: A + b: B + if int(): + a = super().f() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + a = super().g() # E: "g" undefined in superclass + b = super().f() + return a +[builtins fixtures/tuple.pyi] +[out] + +[case testAccessingSuperTypeMethodWithArgs] +from typing import Any +class B: + def f(self, y: 'A') -> None: pass +class A(B): + def f(self, y: Any) -> None: + a: A + b: B + super().f(b) # E: Argument 1 to "f" of "B" has incompatible type "B"; expected "A" + super().f(a) + self.f(b) + self.f(a) +[builtins fixtures/tuple.pyi] +[out] + +[case testAccessingSuperInit] +# flags: --no-strict-optional +import typing +class B: + def __init__(self, x: A) -> None: pass +class A(B): + def __init__(self) -> None: + super().__init__(B(None)) # E: Argument 1 to "__init__" of "B" has incompatible type "B"; expected "A" + super().__init__() # E: Missing positional argument "x" in call to "__init__" of "B" + super().__init__(A()) +[out] + +[case testAccessingSuperMemberWithDeepHierarchy] +import typing +class C: + def f(self) -> None: pass +class B(C): pass +class A(B): + def f(self) -> None: + super().g() # E: "g" undefined in superclass + super().f() +[out] + +[case testAssignToBaseClassMethod] +import typing +class A: + def f(self) -> None: pass +class B(A): + def g(self) -> None: + super().f = None +[out] +main:6: error: Invalid assignment target + +[case testSuperWithMultipleInheritance] +import typing +class A: + def f(self) -> None: pass +class B: + def g(self, x: int) -> None: pass +class C(A, B): + def f(self) -> None: + super().f() + super().g(1) + super().f(1) # E: Too many arguments for "f" of "A" + super().g() # E: Missing positional argument "x" in call to "g" of "B" + super().not_there() # E: "not_there" undefined in superclass +[out] + +[case testSuperWithNew] +class A: + def __new__(cls, x: int) -> 'A': + return object.__new__(cls) + +class B(A): + def __new__(cls, x: int, y: str = '') -> 'B': + super().__new__(cls, 1) + super().__new__(cls, 1, '') # E: Too many arguments for "__new__" of "A" + return cls(1) +B('') # E: Argument 1 to "B" has incompatible type "str"; expected "int" +B(1) +B(1, 'x') +[builtins fixtures/__new__.pyi] + +reveal_type(C.a) # N: Revealed type is "Any" +[out] + +[case testSuperWithUnknownBase] +from typing import Any +B = None # type: Any +class C(B): + def __init__(self, arg=0): + super(C, self).__init__(arg, arg=arg) +[out] + +[case testSuperSilentInDynamicFunction] +class A: + pass + +class B(A): + def foo(self): + super(B, self).foo() # Not an error +[out] + +[case testSuperWithAny] +class B: + def f(self) -> None: pass +class C(B): + def h(self, x) -> None: + reveal_type(super(x, x).f) # N: Revealed type is "def ()" + reveal_type(super(C, x).f) # N: Revealed type is "def ()" + reveal_type(super(C, type(x)).f) # N: Revealed type is "def (self: __main__.B)" + +[case testSuperInUnannotatedMethod] +class C: + def h(self): + super(C, self).xyz + +[case testSuperWithTypeObjects] +from typing import Type + +class A: + def f(self) -> object: pass + +class B(A): + def f(self) -> int: pass + + @classmethod + def g(cls, x) -> None: + reveal_type(super(cls, x).f) # N: Revealed type is "def () -> builtins.object" + + def h(self, t: Type[B]) -> None: + reveal_type(super(t, self).f) # N: Revealed type is "def () -> builtins.object" +[builtins fixtures/classmethod.pyi] + +[case testSuperWithTypeTypeAsSecondArgument] +class B: + def f(self) -> None: pass + +class C(B): + def __new__(cls) -> 'C': + super(C, cls).f + return C() + +[case testSuperWithGenericSelf] +from typing import TypeVar + +T = TypeVar('T', bound='C') + +class B: + def f(self) -> float: pass + +class C(B): + def f(self) -> int: pass + + def g(self: T) -> T: + reveal_type(super(C, self).f) # N: Revealed type is "def () -> builtins.float" + return self + +[case testSuperWithTypeVarValues1] +from typing import TypeVar + +T = TypeVar('T', 'C', 'D') +S = TypeVar('S', 'B', 'C') + +class B: + def f(self) -> None: pass + +class C(B): + def f(self) -> None: pass + + def g(self, x: T, y: S) -> None: + super(C, x).f + super(C, y).f # E: Argument 2 for "super" not an instance of argument 1 + +class D(C): pass + +[case testSuperWithTypeVarValues2] +from typing import TypeVar, Generic + +T = TypeVar('T', 'C', 'D') +S = TypeVar('S', 'B', 'C') + +class B: + def f(self) -> None: pass + +class C(B, Generic[T, S]): + def f(self) -> None: pass + + def g(self, x: T, y: S) -> None: + super(C, x).f + super(C, y).f # E: Argument 2 for "super" not an instance of argument 1 + +class D(C): pass + +[case testSuperInClassMethod] +from typing import Union + +class A: + def f(self, i: int) -> None: pass + +class B(A): + def f(self, i: Union[int, str]) -> None: pass + + @classmethod + def g(cls, i: int) -> None: + super().f(B(), i) + super(B, cls).f(cls(), i) + super(B, B()).f(i) + + super().f(B(), '') # E: Argument 2 to "f" of "A" has incompatible type "str"; expected "int" + super(B, cls).f(cls(), '') # E: Argument 2 to "f" of "A" has incompatible type "str"; expected "int" + super(B, B()).f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +[builtins fixtures/classmethod.pyi] + +[case testSuperWithUnrelatedTypes] +from typing import Union + +class A: + def f(self, s: str) -> None: pass + +class B(A): + def f(self, i: Union[int, str]) -> None: pass + +class C: + def g(self, b: B) -> None: + super(B, b).f('42') + super(B, b).f(42) # E: Argument 1 to "f" of "A" has incompatible type "int"; expected "str" + +[case testSuperOutsideClass] +from typing import Union + +class A: + def f(self, s: str) -> None: pass + +class B(A): + def f(self, i: Union[int, str]) -> None: pass + +def g(b: B) -> None: + super(B, b).f('42') + super(B, b).f(42) # E: Argument 1 to "f" of "A" has incompatible type "int"; expected "str" + +[case testSuperInInitSubclass] +class A: + def __init_subclass__(cls) -> None: + super().__init_subclass__() +[builtins fixtures/__init_subclass__.pyi] + +[case testSuperClassGetItem] +from typing import TypeVar, Type, Any + +T = TypeVar("T", bound="B") + +class A: + def __class_getitem__(cls, item) -> None: pass + +class B(A): + def __class_getitem__(cls: Type[T], item: Any) -> None: + super(B, cls).__class_getitem__(item) + +-- Invalid uses of super() +-- ----------------------- + + +[case testSuperOutsideMethodNoCrash] +class A: + x = 1 +class B(A): pass +class C(B): + b = super(B, B).x + a = super().whatever # E: "super()" outside of a method is not supported + +[case testSuperWithObjectClassAsFirstArgument] +class A: + def f(self) -> None: + super(object, self).f() # E: Target class has no base class + +[case testSuperWithTypeVarAsFirstArgument] +from typing import TypeVar + +T = TypeVar('T') + +def f(obj: T) -> None: + super(obj.__class__, obj).f() # E: Target class has no base class +[builtins fixtures/__new__.pyi] + +[case testSuperWithSingleArgument] +class B: + def f(self) -> None: pass +class C(B): + def __init__(self) -> None: + super(C).f() # E: "super" with a single argument not supported + +[case testSuperWithThreeArguments] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(C, self, 1).f() # E: Too many arguments for "super" + +[case testSuperWithNonPositionalArguments] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(C, x=self).f() # E: "super" only accepts positional arguments + super(**{}).f() # E: "super" only accepts positional arguments + +[case testSuperWithVarArgs] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(*(C, self)).f() # E: Varargs not supported with "super" + +[case testInvalidSuperArg] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(x, y).f # E: Name "x" is not defined # E: Name "y" is not defined + +[case testTypeErrorInSuperArg] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(1(), self).f # E: "int" not callable + super(C, ''()).f # E: "str" not callable + +[case testFlippedSuperArgs] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(self, C).f # E: Argument 1 for "super" must be a type object; got a non-type instance + +[case testInvalidFirstSuperArg] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(None, C).f # E: Argument 1 for "super" must be a type object; got "None" + +[case testInvalidSecondArgumentToSuper] +class B: + def f(self) -> None: pass +class C(B): + def h(self) -> None: + super(C, 1).f # E: Argument 2 for "super" not an instance of argument 1 + super(C, None).f # E: Unsupported argument 2 for "super" + +[case testSuperInMethodWithNoArguments] +class A: + def f(self) -> None: pass + @staticmethod + def st() -> int: + return 1 + +class B(A): + def g() -> None: # E: Method must have at least one argument. Did you forget the "self" argument? + super().f() # E: "super()" requires one or two positional arguments in enclosing function + def h(self) -> None: + def a() -> None: + super().f() # E: "super()" requires one or two positional arguments in enclosing function + @staticmethod + def st() -> int: + reveal_type(super(B, B).st()) # N: Revealed type is "builtins.int" + super().st() # E: "super()" requires one or two positional arguments in enclosing function + return 2 +[builtins fixtures/staticmethod.pyi] + +[case testSuperWithUnsupportedTypeObject] +from typing import Type + +class A: + def f(self) -> int: pass + +class B(A): + def h(self, t: Type[None]) -> None: + super(t, self).f # E: Unsupported argument 1 for "super" + +[case testSuperSelfTypeInstanceMethod] +from typing import TypeVar, Type + +T = TypeVar("T", bound="A") + +class A: + def foo(self: T) -> T: ... + +class B(A): + def foo(self: T) -> T: + reveal_type(super().foo()) # N: Revealed type is "T`-1" + return super().foo() + +[case testSuperSelfTypeClassMethod] +from typing import TypeVar, Type + +T = TypeVar("T", bound="A") + +class A: + @classmethod + def foo(cls: Type[T]) -> T: ... + +class B(A): + @classmethod + def foo(cls: Type[T]) -> T: + reveal_type(super().foo()) # N: Revealed type is "T`-1" + return super().foo() +[builtins fixtures/classmethod.pyi] + +[case testWrongSuperOutsideMethodNoCrash] +class B: + x: int +class C1(B): ... +class C2(B): ... +super(C1, C2).x # E: Argument 2 for "super" not an instance of argument 1 diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test new file mode 100644 index 000000000000..d675a35c4aae --- /dev/null +++ b/test-data/unit/check-tuples.test @@ -0,0 +1,1740 @@ +-- Normal assignment and subtyping +-- ------------------------------- + + +[case testTupleAssignmentWithTupleTypes] +from typing import Tuple +t1: Tuple[A] +t2: Tuple[B] +t3: Tuple[A, A] +t4: Tuple[A, B] +t5: Tuple[B, A] + +if int(): + t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") +if int(): + t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +if int(): + t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") +if int(): + t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") +if int(): + t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") + +# Ok +if int(): + t1 = t1 +if int(): + t2 = t2 +if int(): + t3 = t3 +if int(): + t4 = t4 +if int(): + t5 = t5 + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testTupleSubtyping] +from typing import Tuple +t1: Tuple[A, A] +t2: Tuple[A, B] +t3: Tuple[B, A] + +if int(): + t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") + t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") + t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") + t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") + + t1 = t2 + t1 = t3 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.pyi] + +[case testTupleCompatibilityWithOtherTypes] +# flags: --no-strict-optional +from typing import Tuple +a, o = None, None # type: (A, object) +t = None # type: Tuple[A, A] + +if int(): + a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") +if int(): + t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") +if int(): + t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") +# TODO: callable types + tuples + +# Ok +if int(): + o = t +if int(): + t = None + +class A: pass +[builtins fixtures/tuple.pyi] + +[case testNestedTupleTypes] +from typing import Tuple +t1: Tuple[A, Tuple[A, A]] +t2: Tuple[B, Tuple[B, B]] + +if int(): + t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +if int(): + t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.pyi] + +[case testNestedTupleTypes2] +from typing import Tuple +t1: Tuple[A, Tuple[A, A]] +t2: Tuple[B, Tuple[B, B]] + +if int(): + t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +if int(): + t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.pyi] + +[case testSubtypingWithTupleType] +from __future__ import annotations +from typing import Any, Tuple + +tuple_aa: tuple[A, A] +Tuple_aa: Tuple[A, A] + +tuple_obj: tuple[object, ...] +Tuple_obj: Tuple[object, ...] + +tuple_obj_one: tuple[object] +Tuple_obj_one: Tuple[object] + +tuple_obj_two: tuple[object, object] +Tuple_obj_two: Tuple[object, object] + +tuple_any_implicit: tuple +Tuple_any_implicit: Tuple + +tuple_any: tuple[Any, ...] +Tuple_any: Tuple[Any, ...] + +tuple_any_one: tuple[Any] +Tuple_any_one: Tuple[Any] + +tuple_any_two: tuple[Any, Any] +Tuple_any_two: Tuple[Any, Any] + +def takes_tuple_aa(t: tuple[A, A]): ... + +takes_tuple_aa(tuple_aa) +takes_tuple_aa(Tuple_aa) +takes_tuple_aa(tuple_obj) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[object, ...]"; expected "Tuple[A, A]" +takes_tuple_aa(Tuple_obj) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[object, ...]"; expected "Tuple[A, A]" +takes_tuple_aa(tuple_obj_one) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[object]"; expected "Tuple[A, A]" +takes_tuple_aa(Tuple_obj_one) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[object]"; expected "Tuple[A, A]" +takes_tuple_aa(tuple_obj_two) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[object, object]"; expected "Tuple[A, A]" +takes_tuple_aa(Tuple_obj_two) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[object, object]"; expected "Tuple[A, A]" +takes_tuple_aa(tuple_any_implicit) +takes_tuple_aa(Tuple_any_implicit) +takes_tuple_aa(tuple_any) +takes_tuple_aa(Tuple_any) +takes_tuple_aa(tuple_any_one) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[Any]"; expected "Tuple[A, A]" +takes_tuple_aa(Tuple_any_one) # E: Argument 1 to "takes_tuple_aa" has incompatible type "Tuple[Any]"; expected "Tuple[A, A]" +takes_tuple_aa(tuple_any_two) +takes_tuple_aa(Tuple_any_two) + +def takes_tuple_any_implicit(t: tuple): ... + +takes_tuple_any_implicit(tuple_aa) +takes_tuple_any_implicit(Tuple_aa) +takes_tuple_any_implicit(tuple_obj) +takes_tuple_any_implicit(Tuple_obj) +takes_tuple_any_implicit(tuple_obj_one) +takes_tuple_any_implicit(Tuple_obj_one) +takes_tuple_any_implicit(tuple_obj_two) +takes_tuple_any_implicit(Tuple_obj_two) +takes_tuple_any_implicit(tuple_any_implicit) +takes_tuple_any_implicit(Tuple_any_implicit) +takes_tuple_any_implicit(tuple_any) +takes_tuple_any_implicit(Tuple_any) +takes_tuple_any_implicit(tuple_any_one) +takes_tuple_any_implicit(Tuple_any_one) +takes_tuple_any_implicit(tuple_any_two) +takes_tuple_any_implicit(Tuple_any_two) + +def takes_tuple_any_one(t: tuple[Any]): ... + +takes_tuple_any_one(tuple_aa) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[A, A]"; expected "Tuple[Any]" +takes_tuple_any_one(Tuple_aa) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[A, A]"; expected "Tuple[Any]" +takes_tuple_any_one(tuple_obj) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[object, ...]"; expected "Tuple[Any]" +takes_tuple_any_one(Tuple_obj) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[object, ...]"; expected "Tuple[Any]" +takes_tuple_any_one(tuple_obj_one) +takes_tuple_any_one(Tuple_obj_one) +takes_tuple_any_one(tuple_obj_two) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[object, object]"; expected "Tuple[Any]" +takes_tuple_any_one(Tuple_obj_two) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[object, object]"; expected "Tuple[Any]" +takes_tuple_any_one(tuple_any_implicit) +takes_tuple_any_one(Tuple_any_implicit) +takes_tuple_any_one(tuple_any) +takes_tuple_any_one(Tuple_any) +takes_tuple_any_one(tuple_any_one) +takes_tuple_any_one(Tuple_any_one) +takes_tuple_any_one(tuple_any_two) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[Any, Any]"; expected "Tuple[Any]" +takes_tuple_any_one(Tuple_any_two) # E: Argument 1 to "takes_tuple_any_one" has incompatible type "Tuple[Any, Any]"; expected "Tuple[Any]" + +class A: pass +[builtins fixtures/tuple.pyi] + +[case testSubtypingWithTupleTypeSubclass] +from __future__ import annotations +from typing import Any, Tuple + +class A: ... + +inst_tuple_aa: Tuple[A, A] + +class tuple_aa_subclass(Tuple[A, A]): ... +inst_tuple_aa_subclass: tuple_aa_subclass + +class tuple_any_subclass(Tuple[Any, ...]): ... +inst_tuple_any_subclass: tuple_any_subclass + +class tuple_any_one_subclass(Tuple[Any]): ... +inst_tuple_any_one_subclass: tuple_any_one_subclass + +class tuple_any_two_subclass(Tuple[Any, Any]): ... +inst_tuple_any_two_subclass: tuple_any_two_subclass + +class tuple_obj_subclass(Tuple[object, ...]): ... +inst_tuple_obj_subclass: tuple_obj_subclass + +class tuple_obj_one_subclass(Tuple[object]): ... +inst_tuple_obj_one_subclass: tuple_obj_one_subclass + +class tuple_obj_two_subclass(Tuple[object, object]): ... +inst_tuple_obj_two_subclass: tuple_obj_two_subclass + +def takes_tuple_aa(t: Tuple[A, A]): ... + +takes_tuple_aa(inst_tuple_aa) +takes_tuple_aa(inst_tuple_aa_subclass) +takes_tuple_aa(inst_tuple_any_subclass) +takes_tuple_aa(inst_tuple_any_one_subclass) # E: Argument 1 to "takes_tuple_aa" has incompatible type "tuple_any_one_subclass"; expected "Tuple[A, A]" +takes_tuple_aa(inst_tuple_any_two_subclass) +takes_tuple_aa(inst_tuple_obj_subclass) # E: Argument 1 to "takes_tuple_aa" has incompatible type "tuple_obj_subclass"; expected "Tuple[A, A]" +takes_tuple_aa(inst_tuple_obj_one_subclass) # E: Argument 1 to "takes_tuple_aa" has incompatible type "tuple_obj_one_subclass"; expected "Tuple[A, A]" +takes_tuple_aa(inst_tuple_obj_two_subclass) # E: Argument 1 to "takes_tuple_aa" has incompatible type "tuple_obj_two_subclass"; expected "Tuple[A, A]" + +def takes_tuple_aa_subclass(t: tuple_aa_subclass): ... + +takes_tuple_aa_subclass(inst_tuple_aa) # E: Argument 1 to "takes_tuple_aa_subclass" has incompatible type "Tuple[A, A]"; expected "tuple_aa_subclass" +takes_tuple_aa_subclass(inst_tuple_aa_subclass) +takes_tuple_aa_subclass(inst_tuple_any_subclass) # E: Argument 1 to "takes_tuple_aa_subclass" has incompatible type "tuple_any_subclass"; expected "tuple_aa_subclass" +takes_tuple_aa_subclass(inst_tuple_any_one_subclass) # E: Argument 1 to "takes_tuple_aa_subclass" has incompatible type "tuple_any_one_subclass"; expected "tuple_aa_subclass" +takes_tuple_aa_subclass(inst_tuple_any_two_subclass) # E: Argument 1 to "takes_tuple_aa_subclass" has incompatible type "tuple_any_two_subclass"; expected "tuple_aa_subclass" +takes_tuple_aa_subclass(inst_tuple_obj_subclass) # E: Argument 1 to "takes_tuple_aa_subclass" has incompatible type "tuple_obj_subclass"; expected "tuple_aa_subclass" +takes_tuple_aa_subclass(inst_tuple_obj_one_subclass) # E: Argument 1 to "takes_tuple_aa_subclass" has incompatible type "tuple_obj_one_subclass"; expected "tuple_aa_subclass" +takes_tuple_aa_subclass(inst_tuple_obj_two_subclass) # E: Argument 1 to "takes_tuple_aa_subclass" has incompatible type "tuple_obj_two_subclass"; expected "tuple_aa_subclass" + +[builtins fixtures/tuple.pyi] + +[case testTupleInitializationWithNone] +# flags: --no-strict-optional +from typing import Tuple +t = None # type: Tuple[A, A] +t = None +class A: pass +[builtins fixtures/tuple.pyi] + + +-- Tuple expressions +-- ----------------- + + +[case testTupleExpressions] +# flags: --no-strict-optional +from typing import Tuple +t1 = None # type: tuple +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B] + +a, b, c = None, None, None # type: (A, B, C) + +if int(): + t2 = () # E: Incompatible types in assignment (expression has type "Tuple[()]", variable has type "Tuple[A]") +if int(): + t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +if int(): + t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +if int(): + t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") +if int(): + t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") + +t1 = () +t1 = (a,) +t2 = (a,) +t3 = (a, b) +t3 = (a, c) +t3 = (None, None) + +class A: pass +class B: pass +class C(B): pass +[builtins fixtures/tuple.pyi] + +[case testVoidValueInTuple] +import typing +def f() -> None: pass + +(None, f()) # E: "f" does not return a value (it only ever returns None) +(f(), None) # E: "f" does not return a value (it only ever returns None) +[builtins fixtures/tuple.pyi] + + +-- Indexing +-- -------- + + +[case testIndexingTuples] +from typing import Tuple +t1: Tuple[A, B] +t2: Tuple[A] +t3: Tuple[A, B, C, D, E] +a: A +b: B +x: Tuple[A, B, C] +y: Tuple[A, C, E] +n = 0 + +if int(): + a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[2] # E: Tuple index out of range +t1[3] # E: Tuple index out of range +t2[1] # E: Tuple index out of range +reveal_type(t1[n]) # N: Revealed type is "Union[__main__.A, __main__.B]" +reveal_type(t3[n:]) # N: Revealed type is "builtins.tuple[Union[__main__.A, __main__.B, __main__.C, __main__.D, __main__.E], ...]" +if int(): + b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a = t1[0] +if int(): + b = t1[1] +if int(): + b = t1[-1] +if int(): + a = t1[(0)] +if int(): + b = t1[+1] +if int(): + x = t3[0:3] # type (A, B, C) +if int(): + y = t3[0:+5:2] # type (A, C, E) +if int(): + x = t3[:-2] # type (A, B, C) + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass +[builtins fixtures/tuple.pyi] + +[case testIndexingTuplesWithNegativeIntegers] +from typing import Tuple +t1: Tuple[A, B] +t2: Tuple[A] +a: A +b: B + +if int(): + a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[-3] # E: Tuple index out of range +t1[-4] # E: Tuple index out of range +if int(): + b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a = t1[-2] +if int(): + b = t1[-1] +if int(): + a = t2[(-1)] + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testAssigningToTupleItems] +from typing import Tuple + +class A: pass +class B: pass + +t: Tuple[A, B] +n = 0 + +t[0] = A() # E: Unsupported target for indexed assignment ("Tuple[A, B]") +t[2] = A() # E: Unsupported target for indexed assignment ("Tuple[A, B]") +t[n] = A() # E: Unsupported target for indexed assignment ("Tuple[A, B]") +[builtins fixtures/tuple.pyi] + + +-- Multiple assignment +-- ------------------- + + +[case testMultipleAssignmentWithTuples] +# flags: --no-strict-optional +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A, B, A] +a, b = None, None # type: (A, B) +(a1, b1) = None, None # type: Tuple[A, B] + +reveal_type(a1) # N: Revealed type is "__main__.A" +reveal_type(b1) # N: Revealed type is "__main__.B" + +if int(): + a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a, b = t1 +if int(): + a, b, a1 = t2 + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithSquareBracketTuples] +# flags: --no-strict-optional +from typing import Tuple + +def avoid_confusing_test_parser() -> None: + t1 = None # type: Tuple[A, B] + t2 = None # type: Tuple[A, B, A] + [a, b] = None, None # type: (A, B) + [a1, b1] = None, None # type: Tuple[A, B] + + reveal_type(a) # N: Revealed type is "__main__.A" + reveal_type(b) # N: Revealed type is "__main__.B" + reveal_type(a1) # N: Revealed type is "__main__.A" + reveal_type(b1) # N: Revealed type is "__main__.B" + + if int(): + [a, a] = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") + [b, b] = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") + [a, b, b] = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") + + [a, b] = t1 + [a, b, a1] = t2 + + [a2, b2] = t1 + reveal_type(a2) # N: Revealed type is "__main__.A" + reveal_type(b2) # N: Revealed type is "__main__.B" + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithInvalidNumberOfValues] +from typing import Tuple +t1: Tuple[A, A, A] +a: A + +a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) +a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) + +a, a, a = t1 + +class A: pass +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithTupleExpressionRvalue] +a: A +b: B + +if int(): + a, b = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a, b = b, a \ + # E: Incompatible types in assignment (expression has type "B", variable has type "A") \ + # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a, b = a, b +if int(): + a, a = a, a + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testSubtypingInMultipleAssignment] +a: A +b: B + +if int(): + b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + a, b = b, b +if int(): + b, a = b, b + +class A: pass +class B(A): pass +[builtins fixtures/tuple.pyi] + +[case testInitializationWithMultipleValues] +# flags: --no-strict-optional +a, b = None, None # type: (A, B) + +a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a3, b3 = a # type: (A, B) # E: "A" object is not iterable +a4, b4 = None # type: (A, B) # E: "None" object is not iterable +a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) + +ax, bx = a, b # type: (A, B) + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithNonTupleRvalue] +a: A +b: B +def f(): pass + +a, b = None # E: "None" object is not iterable +a, b = a # E: "A" object is not iterable +a, b = f # E: "Callable[[], Any]" object is not iterable + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithIndexedLvalues] +a: A +b: B +aa: AA +bb: BB + +a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") +a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") +a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A"; expected type "A" +a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B"; expected type "B" +a[a], b[b] = aa, bb + +class A: + def __setitem__(self, x: 'A', y: 'AA') -> None: pass +class B: + def __setitem__(self, x: 'B', y: 'BB') -> None: pass + +class AA: pass +class BB: pass +[builtins fixtures/tuple.pyi] + +[case testMultipleDeclarationWithParentheses] +# flags: --no-strict-optional +(a, b) = (None, None) # type: int, str +if int(): + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + a = 1 + b = '' +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithExtraParentheses] +a: A +b: B + +if int(): + (a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + (a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + ((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + ((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + [a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + [a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +if int(): + (a, b) = (a, b) +if int(): + ((a), (b)) = ((a), (b)) +if int(): + [a, b] = a, b + +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentUsingSingleTupleType] +# flags: --no-strict-optional +from typing import Tuple +a, b = None, None # type: Tuple[int, str] +if int(): + a = 1 +if int(): + b = '' +if int(): + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithMixedVariables] +a = b, c = 1, 1 +x, y = p, q = 1, 1 +u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) +d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) +[builtins fixtures/tuple.pyi] + + +-- Assignment to starred expressions +-- --------------------------------- + + +[case testAssignmentToStarMissingAnnotation] +from typing import List +t = 1, 2 +a, b, *c = 1, 2 # E: Need type annotation for "c" (hint: "c: List[] = ...") +aa, bb, *cc = t # E: Need type annotation for "cc" (hint: "cc: List[] = ...") +[builtins fixtures/list.pyi] + +[case testAssignmentToStarAnnotation] +# flags: --no-strict-optional +from typing import List +li, lo = None, None # type: List[int], List[object] +a, b, *c = 1, 2 # type: int, int, List[int] +if int(): + c = lo # E: Incompatible types in assignment (expression has type "List[object]", variable has type "List[int]") +if int(): + c = li +[builtins fixtures/list.pyi] + +[case testAssignmentToStarCount1] +from typing import List +ca: List[int] +c = [1] +if int(): + a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) +if int(): + a, b, *c = 1, 2 +if int(): + a, b, *c = 1, 2, 3 +if int(): + a, b, *c = 1, 2, 3, 4 +[builtins fixtures/list.pyi] + +[case testAssignmentToStarCount2] +from typing import List +ca: List[int] +t1 = 1, +t2 = 1, 2 +t3 = 1, 2, 3 +t4 = 1, 2, 3, 4 +c = [1] +if int(): + a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) +if int(): + a, b, *c = t2 +if int(): + a, b, *c = t3 +if int(): + a, b, *c = t4 +[builtins fixtures/list.pyi] + +[case testAssignmentToStarFromAny] +from typing import Any, cast +class C: pass + +a, c = cast(Any, 1), C() +p, *q = a +c = a +c = q +[case testAssignmentToComplexStar] +from typing import List +li: List[int] +if int(): + a, *(li) = 1, +a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) +if int(): + a, *(b, c) = 1, 2, 3 +if int(): + a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) +[builtins fixtures/list.pyi] + +[case testAssignmentToStarFromTupleType] +from typing import List, Tuple +li: List[int] +la: List[A] +ta: Tuple[A, A, A] +if int(): + a, *la = ta +if int(): + a, *li = ta # E: List item 0 has incompatible type "A"; expected "int" \ + # E: List item 1 has incompatible type "A"; expected "int" +if int(): + a, *na = ta + if int(): + na = la + na = a # E: Incompatible types in assignment (expression has type "A", variable has type "List[A]") + +class A: pass +[builtins fixtures/list.pyi] + +[case testAssignmentToStarFromTupleInference] +from typing import List +class A: pass +li: List[int] +la: List[A] +a, *l = A(), A() +if int(): + l = li # E: Incompatible types in assignment (expression has type "List[int]", variable has type "List[A]") +if int(): + l = la +[builtins fixtures/list.pyi] +[out] + +[case testAssignmentToStarFromListInference] +from typing import List + +class A: pass + +li: List[int] +la: List[A] +a, *l = [A(), A()] +if int(): + l = li # E: Incompatible types in assignment (expression has type "List[int]", variable has type "List[A]") +if int(): + l = la +[builtins fixtures/list.pyi] +[out] + +[case testAssignmentToStarFromTupleTypeInference] +from typing import List, Tuple +li: List[int] +la: List[A] +ta: Tuple[A, A, A] +a, *l = ta +if int(): + l = li # E: Incompatible types in assignment (expression has type "List[int]", variable has type "List[A]") +if int(): + l = la + +class A: pass +[builtins fixtures/list.pyi] +[out] + +[case testAssignmentToStarFromListTypeInference] +from typing import List +li: List[int] +la: List[A] +a, *l = la +if int(): + l = li # E: Incompatible types in assignment (expression has type "List[int]", variable has type "List[A]") +if int(): + l = la + +class A: pass +[builtins fixtures/list.pyi] +[out] + +[case testAssignmentToStarFromIterable] +from typing import List, Tuple, Iterable + +class CustomIterable(Iterable[int]): pass + +a: List[int] +b: Tuple[int, ...] +c: Tuple[int, int, int] +d: Iterable[int] +e: CustomIterable + +a1, *a2 = a +b1, *b2 = b +c1, *c2 = c +d1, *d2 = d +e1, *e2 = e + +reveal_type(a2) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(b2) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(c2) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(d2) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(e2) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/tuple.pyi] + +-- Nested tuple assignment +-- ---------------------------- + + +[case testNestedTupleAssignment1] +a1: A +a2: A +b1: B +b2: B +c1: C +c2: C + +if int(): + a1, (b1, c1) = a2, (b2, c2) +if int(): + a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) +if int(): + a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # E: Incompatible types in assignment (expression has type "C", variable has type "B") + +class A: pass +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] + +[case testNestedTupleAssignment2] +a1: A +a2: A +b1: B +b2: B +c1: C +c2: C +t = a1, b1 + +if int(): + a2, b2 = t +if int(): + (a2, b2), c2 = t, c1 +if int(): + (a2, c2), c2 = t, c1 # E: Incompatible types in assignment (expression has type "B", variable has type "C") +if int(): + t, c2 = (a2, b2), c2 +if int(): + t, c2 = (a2, a2), c2 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +if int(): + t = a1, a1, a1 # E: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") +if int(): + t = a1 # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") +if int(): + a2, a2, a2 = t # E: Need more than 2 values to unpack (3 expected) +if int(): + a2, = t # E: Too many values to unpack (1 expected, 2 provided) +if int(): + a2 = t # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") + +class A: pass +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] + + +-- Error messages +-- -------------- + + +[case testTupleErrorMessages] +class A: + def __add__(self, x: 'A') -> 'A': pass +def f(x: 'A') -> None: pass + +a: A + +(a, a) + a # E: Unsupported operand types for + ("Tuple[A, A]" and "A") +a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") +f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" +(a, a).foo # E: "Tuple[A, A]" has no attribute "foo" +[builtins fixtures/tuple.pyi] + +[case testLargeTuplesInErrorMessages] + +a: LongTypeName +a + (a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a) # Fail + +class LongTypeName: + def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass +[builtins fixtures/tuple.pyi] +[out] +main:3: error: Unsupported operand types for + ("LongTypeName" and "Tuple[LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName, LongTypeName]") + + +-- Tuple methods +-- ------------- + + +[case testTupleMethods] +from typing import Tuple +t: Tuple[int, str] +i = 0 +s = '' +b = bool() + +if int(): + s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") +t.foo # E: "Tuple[int, str]" has no attribute "foo" + +if int(): + i = t.__len__() +if int(): + s = t.__str__() +if int(): + b = s in t + +[file builtins.py] +from typing import TypeVar, Generic +_T = TypeVar('_T') +class object: + def __init__(self) -> None: pass +class tuple(Generic[_T]): + def __len__(self) -> int: pass + def __str__(self) -> str: pass + def __contains__(self, o: object) -> bool: pass +class int: pass +class str: pass +class bool: pass +class type: pass +class function: pass +class dict: pass + + +-- For loop over tuple +-- ------------------- + + +[case testForLoopOverTuple] +import typing +t = 1, 2 +for x in t: + x = 1 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/for.pyi] + +[case testForLoopOverEmptyTuple] +import typing +t = () +for x in t: pass # E: Need type annotation for "x" +[builtins fixtures/for.pyi] + +[case testForLoopOverNoneValuedTuple] +import typing +for x in None, None: pass +[builtins fixtures/for.pyi] + +[case testForLoopOverTupleAndSubtyping] +import typing +class A: pass +class B(A): pass +for x in B(), A(): + x = A() + x = B() + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +[builtins fixtures/for.pyi] + +[case testTupleIterable] +from typing import Iterable, Optional, TypeVar + +T = TypeVar("T") + +def sum(iterable: Iterable[T], start: Optional[T] = None) -> T: pass + +y = 'a' +x = sum((1,2)) +if int(): + y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + + +-- Tuple as a base type +-- -------------------- + + +[case testTupleBaseClass] +import m +[file m.pyi] +from typing import Tuple +class A(Tuple[int, str]): + def f(self, x: int) -> None: + a, b = 1, '' + if int(): + a, b = self + b, a = self # Error + self.f('') # Error +[builtins fixtures/tuple.pyi] +[out] +tmp/m.pyi:7: error: Incompatible types in assignment (expression has type "int", variable has type "str") +tmp/m.pyi:7: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/m.pyi:8: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testValidTupleBaseClass2] +from typing import Tuple +class A(Tuple[int, str]): pass + +x, y = A() +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.str" + +x1 = A()[0] # type: int +x2 = A()[1] # type: int # E: Incompatible types in assignment (expression has type "str", variable has type "int") +A()[2] # E: Tuple index out of range + +class B(Tuple[int, ...]): pass + +z1 = B()[0] # type: int +z2 = B()[1] # type: str # E: Incompatible types in assignment (expression has type "int", variable has type "str") +B()[100] +[builtins fixtures/tuple.pyi] +[out] + +[case testValidTupleBaseClass] +from typing import Tuple +class A(tuple): pass +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleBaseClass2] +import m +[file m.pyi] +from typing import Tuple +a: A +class A(Tuple[int, str]): pass +x, y = a +x() # E: "int" not callable +y() # E: "str" not callable +[builtins fixtures/tuple.pyi] +[out] + +[case testGenericClassWithTupleBaseClass] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +class Test(Generic[T], Tuple[T]): pass +x = Test() # type: Test[int] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.Test[builtins.int]]" +[builtins fixtures/tuple.pyi] +[out] + + +-- Variable-length tuples (Tuple[t, ...] with literal '...') +-- --------------------------------------------------------- + + +[case testIndexingVariableLengthTuple] +from typing import Tuple +x = () # type: Tuple[str, ...] +n = 5 +x[n]() # E: "str" not callable +x[3]() # E: "str" not callable +[builtins fixtures/tuple.pyi] + +[case testSubtypingVariableLengthTuple] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +ta = () # type: Tuple[A, ...] +tb = () # type: Tuple[B, ...] +fa(ta) +fa(tb) +fb(tb) +fb(ta) # E: Argument 1 to "fb" has incompatible type "Tuple[A, ...]"; expected "Tuple[B, ...]" +[builtins fixtures/tuple.pyi] + +[case testSubtypingFixedAndVariableLengthTuples] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +aa = (A(), A()) +ab = (A(), B()) +bb = (B(), B()) +fa(aa) +fa(ab) +fa(bb) +fb(bb) +fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected "Tuple[B, ...]" +fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected "Tuple[B, ...]" +[builtins fixtures/tuple.pyi] + +[case testSubtypingTupleIsContainer] +from typing import Container +a: Container[str] +a = () +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testSubtypingTupleIsSized] +from typing import Sized +a: Sized +a = () +[typing fixtures/typing-medium.pyi] +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr1] + +a = (1, 2) +b = (*a, '') +reveal_type(b) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr2] +a = [1] +b = (0, *a) +reveal_type(b) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr2Precise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +a = [1] +b = (0, *a) +reveal_type(b) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr3] +a = [''] +b = (0, *a) +reveal_type(b) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +c = (*a, '') +reveal_type(c) # N: Revealed type is "builtins.tuple[builtins.str, ...]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr3Precise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +a = [''] +b = (0, *a) +reveal_type(b) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.str, ...]]]" +c = (*a, '') +reveal_type(c) # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.str, ...]], builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr4] +a = (1, 1, 'x', 'x') +b = (1, 'x') +a = (0, *b, '') +[builtins fixtures/tuple.pyi] + +[case testUnpackSyntaxError] +*foo # E: can't use starred expression here +[builtins fixtures/tuple.pyi] + +[case testUnpackBases] +class A: ... +class B: ... +bases = (A, B) +class C(*bases): ... # E: Invalid base class +[builtins fixtures/tuple.pyi] + +[case testTupleMeetTupleAny] +from typing import Union, Tuple +class A: pass +class B: pass + +def f(x: Union[B, Tuple[A, A]]) -> None: + if isinstance(x, tuple): + reveal_type(x) # N: Revealed type is "Tuple[__main__.A, __main__.A]" + else: + reveal_type(x) # N: Revealed type is "__main__.B" + +def g(x: Union[str, Tuple[str, str]]) -> None: + if isinstance(x, tuple): + reveal_type(x) # N: Revealed type is "Tuple[builtins.str, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleMeetTupleAnyComplex] +from typing import Tuple, Union + +Pair = Tuple[int, int] +Variant = Union[int, Pair] +def tuplify(v: Variant) -> None: + reveal_type(v) # N: Revealed type is "Union[builtins.int, Tuple[builtins.int, builtins.int]]" + if not isinstance(v, tuple): + reveal_type(v) # N: Revealed type is "builtins.int" + v = (v, v) + reveal_type(v) # N: Revealed type is "Tuple[builtins.int, builtins.int]" + reveal_type(v) # N: Revealed type is "Tuple[builtins.int, builtins.int]" + reveal_type(v[0]) # N: Revealed type is "builtins.int" + +Pair2 = Tuple[int, str] +Variant2 = Union[int, Pair2] +def tuplify2(v: Variant2) -> None: + if isinstance(v, tuple): + reveal_type(v) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + else: + reveal_type(v) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleMeetTupleAnyAfter] +from typing import Tuple, Union + +def good(blah: Union[Tuple[int, int], int]) -> None: + reveal_type(blah) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], builtins.int]" + if isinstance(blah, tuple): + reveal_type(blah) # N: Revealed type is "Tuple[builtins.int, builtins.int]" + reveal_type(blah) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], builtins.int]" +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleMeetTupleVariable] +from typing import Tuple, TypeVar, Generic, Union +T = TypeVar('T') + +class A: pass +class B1(A): pass +class B2(A): pass +class C: pass + +x: Tuple[A, ...] +y: Tuple[Union[B1, C], Union[B2, C]] + +def g(x: T) -> Tuple[T, T]: + return (x, x) + +z = 1 +x, y = g(z) # E: Argument 1 to "g" has incompatible type "int"; expected "Tuple[B1, B2]" +[builtins fixtures/tuple.pyi] +[out] + +[case testFixedTupleJoinVarTuple] +from typing import Tuple, TypeVar + +class A: pass +class B(A): pass + +fixtup: Tuple[B, B] + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +vartup_b: Tuple[B, ...] +reveal_type(join(fixtup, vartup_b)) # N: Revealed type is "builtins.tuple[__main__.B, ...]" +reveal_type(join(vartup_b, fixtup)) # N: Revealed type is "builtins.tuple[__main__.B, ...]" + +vartup_a: Tuple[A, ...] +reveal_type(join(fixtup, vartup_a)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(vartup_a, fixtup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" + +[builtins fixtures/tuple.pyi] +[out] + +[case testFixedTupleJoinList] +from typing import Tuple, List, TypeVar + +class A: pass +class B(A): pass + +fixtup: Tuple[B, B] + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +lst_b: List[B] +reveal_type(join(fixtup, lst_b)) # N: Revealed type is "typing.Sequence[__main__.B]" +reveal_type(join(lst_b, fixtup)) # N: Revealed type is "typing.Sequence[__main__.B]" + +lst_a: List[A] +reveal_type(join(fixtup, lst_a)) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(lst_a, fixtup)) # N: Revealed type is "typing.Sequence[__main__.A]" + +[builtins fixtures/tuple.pyi] +[out] + +[case testEmptyTupleJoin] +from typing import Tuple, List, TypeVar + +class A: pass + +empty = () + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +fixtup: Tuple[A] +reveal_type(join(fixtup, empty)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(empty, fixtup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" + +vartup: Tuple[A, ...] +reveal_type(join(vartup, empty)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(empty, vartup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" + +lst: List[A] +reveal_type(join(empty, lst)) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(lst, empty)) # N: Revealed type is "typing.Sequence[__main__.A]" + +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleSubclassJoin] +from typing import Tuple, NamedTuple, TypeVar + +class NTup(NamedTuple): + a: bool + b: bool + +class SubTuple(Tuple[bool]): ... +class SubVarTuple(Tuple[int, ...]): ... + +ntup: NTup +subtup: SubTuple +vartup: SubVarTuple + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(ntup, vartup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, vartup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleJoinIrregular] +from typing import Tuple, TypeVar + +tup1: Tuple[bool, int] +tup2: Tuple[bool] + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(tup1, tup2)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup2, tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +reveal_type(join(tup1, ())) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join((), tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +reveal_type(join(tup2, ())) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join((), tup2)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" + +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleSubclassJoinIrregular] +from typing import Tuple, NamedTuple, TypeVar + +class NTup1(NamedTuple): + a: bool + +class NTup2(NamedTuple): + a: bool + b: bool + +class SubTuple(Tuple[bool, int, int]): ... + +tup1: NTup1 +tup2: NTup2 +subtup: SubTuple + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(tup1, tup2)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join(tup2, tup1)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" + +reveal_type(join(tup1, subtup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +reveal_type(join(tup2, subtup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, tup2)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +[builtins fixtures/tuple.pyi] +[out] + +[case testTupleWithUndersizedContext] +a = ([1], 'x') +if int(): + a = ([], 'x', 1) # E: Incompatible types in assignment (expression has type "Tuple[List[Never], str, int]", variable has type "Tuple[List[int], str]") +[builtins fixtures/tuple.pyi] + +[case testTupleWithOversizedContext] +a = (1, [1], 'x') +if int(): + a = (1, []) # E: Incompatible types in assignment (expression has type "Tuple[int, List[int]]", variable has type "Tuple[int, List[int], str]") +[builtins fixtures/tuple.pyi] + +[case testTupleWithoutContext] +a = (1, []) # E: Need type annotation for "a" +[builtins fixtures/tuple.pyi] + +[case testTupleWithUnionContext] +from typing import List, Union, Tuple +def f() -> Union[int, Tuple[List[str]]]: + return ([],) +[builtins fixtures/tuple.pyi] + +[case testTupleWithVariableSizedTupleContext] +from typing import List, Tuple +def f() -> Tuple[List[str], ...]: + return ([],) +[builtins fixtures/tuple.pyi] + +[case testTupleWithoutArgs] +from typing import Tuple +def f(a: Tuple) -> None: pass +f(()) +f((1,)) +f(('', '')) +f(0) # E: Argument 1 to "f" has incompatible type "int"; expected "Tuple[Any, ...]" +[builtins fixtures/tuple.pyi] + +[case testTupleSingleton] + +from typing import Tuple +def f(a: Tuple[()]) -> None: pass +f(()) +f((1,)) # E: Argument 1 to "f" has incompatible type "Tuple[int]"; expected "Tuple[()]" +f(('', '')) # E: Argument 1 to "f" has incompatible type "Tuple[str, str]"; expected "Tuple[()]" +f(0) # E: Argument 1 to "f" has incompatible type "int"; expected "Tuple[()]" +[builtins fixtures/tuple.pyi] + +[case testNonliteralTupleIndex] +t = (0, "") +x = 0 +y = "" +reveal_type(t[x]) # N: Revealed type is "Union[builtins.int, builtins.str]" +t[y] # E: No overload variant of "__getitem__" of "tuple" matches argument type "str" \ + # N: Possible overload variants: \ + # N: def __getitem__(self, int, /) -> Union[int, str] \ + # N: def __getitem__(self, slice, /) -> Tuple[Union[int, str], ...] + +[builtins fixtures/tuple.pyi] + +[case testNonliteralTupleSlice] +t = (0, "") +x = 0 +y = "" +reveal_type(t[x:]) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" +t[y:] # E: Slice index must be an integer, SupportsIndex or None +[builtins fixtures/tuple.pyi] + +[case testTupleSliceStepZeroNoCrash] +# This was crashing: https://github.com/python/mypy/issues/18062 +# TODO: emit better error when 0 is used for step +()[::0] # E: Ambiguous slice of a variadic tuple +[builtins fixtures/tuple.pyi] + +[case testInferTupleTypeFallbackAgainstInstance] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') + +class Base(Generic[T]): pass +def f(x: Base[T]) -> T: pass + +class DT(Tuple[str, str], Base[int]): + pass + +reveal_type(f(DT())) # N: Revealed type is "builtins.int" + +[builtins fixtures/tuple.pyi] +[out] + +[case testTypeTupleClassmethod] +from typing import Tuple, Type + +class C(Tuple[int, str]): + @classmethod + def f(cls) -> None: pass + +t: Type[C] +t.g() # E: "Type[C]" has no attribute "g" +t.f() +[builtins fixtures/classmethod.pyi] + +[case testTypeTupleCall] +from typing import Tuple + +def foo(o: CallableTuple) -> int: + reveal_type(o) # N: Revealed type is "Tuple[builtins.str, builtins.int, fallback=__main__.CallableTuple]" + return o(1, 2) + +class CallableTuple(Tuple[str, int]): + def __call__(self, n: int, m: int) -> int: + return n +[builtins fixtures/tuple.pyi] + +[case testTypeTupleGenericCall] +from typing import Generic, Tuple, TypeVar + +T = TypeVar('T') + +def foo(o: CallableTuple[int]) -> int: + reveal_type(o) # N: Revealed type is "Tuple[builtins.str, builtins.int, fallback=__main__.CallableTuple[builtins.int]]" + reveal_type(o.count(3)) # N: Revealed type is "builtins.int" + return o(1, 2) + +class CallableTuple(Tuple[str, T]): + def __call__(self, n: int, m: int) -> int: + return n +[builtins fixtures/tuple.pyi] + +[case testTupleCompatibleWithSequence] +from typing import Sequence +s: Sequence[str] +s = tuple() +reveal_type(s) # N: Revealed type is "builtins.tuple[builtins.str, ...]" + +[builtins fixtures/tuple.pyi] + +[case testTupleInstanceCompatibleWithIterable] +from typing import Iterable, Tuple +x: Iterable[int] = () +y: Tuple[int, ...] = (1, 2, 3) +x = y +reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +[builtins fixtures/tuple.pyi] + +[case testTupleTypeCompatibleWithIterable] +from typing import Iterable, Tuple +x: Iterable[int] = () +y: Tuple[int, int] = (1, 2) +x = y +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTupleOverlapDifferentTuples] +from typing import Optional, Tuple +class A: pass +class B: pass + +possibles: Tuple[int, Tuple[A]] +x: Optional[Tuple[B]] + +if x in possibles: + reveal_type(x) # N: Revealed type is "Tuple[__main__.B]" +else: + reveal_type(x) # N: Revealed type is "Union[Tuple[__main__.B], None]" + +[builtins fixtures/tuple.pyi] + +[case testUnionOfTupleIndex] +from typing import Union, Tuple + +tup: Union[Tuple[int, str], Tuple[int, int, str]] +reveal_type(tup[0]) # N: Revealed type is "builtins.int" +reveal_type(tup[1]) # N: Revealed type is "Union[builtins.str, builtins.int]" +reveal_type(tup[2]) # E: Tuple index out of range \ + # N: Revealed type is "Union[Any, builtins.str]" +reveal_type(tup[:]) # N: Revealed type is "Union[Tuple[builtins.int, builtins.str], Tuple[builtins.int, builtins.int, builtins.str]]" + +[builtins fixtures/tuple.pyi] + +[case testUnionOfTupleIndexMixed] +from typing import Union, Tuple, List + +tup: Union[Tuple[int, str], List[int]] +reveal_type(tup[0]) # N: Revealed type is "builtins.int" +reveal_type(tup[1]) # N: Revealed type is "Union[builtins.str, builtins.int]" +reveal_type(tup[2]) # E: Tuple index out of range \ + # N: Revealed type is "Union[Any, builtins.int]" +reveal_type(tup[:]) # N: Revealed type is "Union[Tuple[builtins.int, builtins.str], builtins.list[builtins.int]]" + +[builtins fixtures/tuple.pyi] + +[case testFixedLengthTupleConcatenation] +a = (1, "foo", 3) +b = ("bar", 7) + +reveal_type(a + b) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.int, builtins.str, builtins.int]" + +[builtins fixtures/tuple.pyi] + +[case testAssigningWithLongTupleInitializer] +from typing import Tuple + +# long initializer assignment with few mismatches +t: Tuple[int, ...] = (1, 2, 3, 4, 5, 6, 7, 8, "str", "str", "str", 11) # E: Incompatible types in assignment (3 tuple items are incompatible) \ + # N: Expression tuple item 8 has type "str"; "int" expected; \ + # N: Expression tuple item 9 has type "str"; "int" expected; \ + # N: Expression tuple item 10 has type "str"; "int" expected; + +# long initializer assignment with more mismatches +t1: Tuple[int, ...] = (1, 2, 3, 4, 5, 6, 7, 8, "str", "str", "str", "str") # E: Incompatible types in assignment (4 tuple items are incompatible; 1 items are omitted) \ + # N: Expression tuple item 8 has type "str"; "int" expected; \ + # N: Expression tuple item 9 has type "str"; "int" expected; \ + # N: Expression tuple item 10 has type "str"; "int" expected; + +# short tuple initializer assignment +t2: Tuple[int, ...] = (1, 2, "s", 4) # E: Incompatible types in assignment (expression has type "Tuple[int, int, str, int]", variable has type "Tuple[int, ...]") + +# long initializer assignment with few mismatches, no ellipsis +t3: Tuple[int, int, int, int, int, int, int, int, int, int, int, int] = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, "str", "str") # E: Incompatible types in assignment (2 tuple items are incompatible) \ + # N: Expression tuple item 10 has type "str"; "int" expected; \ + # N: Expression tuple item 11 has type "str"; "int" expected; + +# long initializer assignment with more mismatches, no ellipsis +t4: Tuple[int, int, int, int, int, int, int, int, int, int, int, int] = (1, 2, 3, 4, 5, 6, 7, 8, "str", "str", "str", "str") # E: Incompatible types in assignment (4 tuple items are incompatible; 1 items are omitted) \ + # N: Expression tuple item 8 has type "str"; "int" expected; \ + # N: Expression tuple item 9 has type "str"; "int" expected; \ + # N: Expression tuple item 10 has type "str"; "int" expected; + +# short tuple initializer assignment, no ellipsis +t5: Tuple[int, int] = (1, 2, "s", 4) # E: Incompatible types in assignment (expression has type "Tuple[int, int, str, int]", variable has type "Tuple[int, int]") + +# long initializer assignment with mismatched pairs +t6: Tuple[int, int, int, int, int, int, int, int, int, int, int, int] = (1, 2, 3, 4, 5, 6, 7, 8, "str", "str", "str", "str", 1, 1, 1, 1, 1) # E: Incompatible types in assignment (expression has type Tuple[int, int, ... <15 more items>], variable has type Tuple[int, int, ... <10 more items>]) + +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr] +from typing import Tuple, List +points = (1, "test") # type: Tuple[int, str] +x, y, z = *points, 0 +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "builtins.str" +reveal_type(z) # N: Revealed type is "builtins.int" + +points2 = [1,2] +x2, y2, z2= *points2, "test" + +reveal_type(x2) # N: Revealed type is "builtins.int" +reveal_type(y2) # N: Revealed type is "builtins.int" +reveal_type(z2) # N: Revealed type is "builtins.str" + +x3, x4, y3, y4, z3 = *points, *points2, "test" + +reveal_type(x3) # N: Revealed type is "builtins.int" +reveal_type(x4) # N: Revealed type is "builtins.str" +reveal_type(y3) # N: Revealed type is "builtins.int" +reveal_type(y4) # N: Revealed type is "builtins.int" +reveal_type(z3) # N: Revealed type is "builtins.str" + +x5, x6, y5, y6, z4 = *points2, *points2, "test" + +reveal_type(x5) # N: Revealed type is "builtins.int" +reveal_type(x6) # N: Revealed type is "builtins.int" +reveal_type(y5) # N: Revealed type is "builtins.int" +reveal_type(y6) # N: Revealed type is "builtins.int" +reveal_type(z4) # N: Revealed type is "builtins.str" + +points3 = ["test1", "test2"] +x7, x8, y7, y8 = *points2, *points3 # E: Contiguous iterable with same type expected + +x9, y9, x10, y10, z5 = *points2, 1, *points2 # E: Contiguous iterable with same type expected +[builtins fixtures/tuple.pyi] + +[case testAssignEmpty] +() = [] + +[case testAssignEmptyBogus] +() = 1 # E: "int" object is not iterable +[builtins fixtures/tuple.pyi] + +[case testMultiplyTupleByIntegerLiteral] +from typing import Tuple +t = ('',) * 2 +reveal_type(t) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +t2 = ('',) * -1 +reveal_type(t2) # N: Revealed type is "Tuple[()]" +t3 = ('', 1) * 2 +reveal_type(t3) # N: Revealed type is "Tuple[builtins.str, builtins.int, builtins.str, builtins.int]" +def f() -> Tuple[str, ...]: + return ('', ) +reveal_type(f() * 2) # N: Revealed type is "builtins.tuple[builtins.str, ...]" +[builtins fixtures/tuple.pyi] + +[case testEmptyTupleTypeRepr] +from typing import Tuple + +def f() -> Tuple[()]: ... + +reveal_type(f) # N: Revealed type is "def () -> Tuple[()]" +reveal_type(f()) # N: Revealed type is "Tuple[()]" +[builtins fixtures/tuple.pyi] + +[case testMultiplyTupleByIntegerLiteralReverse] +from typing import Tuple +t = 2 * ('',) +reveal_type(t) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +t2 = -1 * ('',) +reveal_type(t2) # N: Revealed type is "Tuple[()]" +t3 = 2 * ('', 1) +reveal_type(t3) # N: Revealed type is "Tuple[builtins.str, builtins.int, builtins.str, builtins.int]" +def f() -> Tuple[str, ...]: + return ('', ) +reveal_type(2 * f()) # N: Revealed type is "builtins.tuple[builtins.str, ...]" +[builtins fixtures/tuple.pyi] + +[case testSingleUndefinedTypeAndTuple] +from typing import Tuple + +class Foo: + ... + +class Bar(aaaaaaaaaa): # E: Name "aaaaaaaaaa" is not defined + ... + +class FooBarTuple(Tuple[Foo, Bar]): + ... +[builtins fixtures/tuple.pyi] + +[case testMultipleUndefinedTypeAndTuple] +from typing import Tuple + +class Foo(aaaaaaaaaa): # E: Name "aaaaaaaaaa" is not defined + ... + +class Bar(aaaaaaaaaa): # E: Name "aaaaaaaaaa" is not defined + ... + +class FooBarTuple(Tuple[Foo, Bar]): + ... +[builtins fixtures/tuple.pyi] + + +[case testTupleOverloadZipAny] +from typing import Any, Iterable, Iterator, Tuple, TypeVar, overload + +T = TypeVar("T") + +@overload +def zip(__i: Iterable[T]) -> Iterator[Tuple[T]]: ... +@overload +def zip(*i: Iterable[Any]) -> Iterator[Tuple[Any, ...]]: ... +def zip(i): ... + +def g(t: Tuple): + reveal_type(zip(*t)) # N: Revealed type is "typing.Iterator[builtins.tuple[Any, ...]]" + reveal_type(zip(t)) # N: Revealed type is "typing.Iterator[Tuple[Any]]" +[builtins fixtures/tuple.pyi] + +[case testTupleSubclassSlice] +from typing import Tuple + +class A: ... + +class tuple_aa_subclass(Tuple[A, A]): ... + +inst_tuple_aa_subclass: tuple_aa_subclass = tuple_aa_subclass((A(), A()))[:] # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "tuple_aa_subclass") +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test new file mode 100644 index 000000000000..c7b9694a9188 --- /dev/null +++ b/test-data/unit/check-type-aliases.test @@ -0,0 +1,1240 @@ +[case testSimpleTypeAlias] +import typing +i = int +def f(x: i) -> None: pass +f(1) +f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +[targets __main__, __main__.f] + +[case testUnionTypeAlias] +from typing import Union +U = Union[int, str] +def f(x: U) -> None: pass +f(1) +f('') +f(()) # E: Argument 1 to "f" has incompatible type "Tuple[()]"; expected "Union[int, str]" +[targets __main__, __main__.f] +[builtins fixtures/tuple.pyi] + +[case testTupleTypeAlias] +from typing import Tuple +T = Tuple[int, str] +def f(x: T) -> None: pass +f((1, 'x')) +f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "Tuple[int, str]" +[targets __main__, __main__.f] +[builtins fixtures/tuple.pyi] + +[case testCallableTypeAlias] +from typing import Callable +A = Callable[[int], None] +f: A +f(1) +f('') # E: Argument 1 has incompatible type "str"; expected "int" +[targets __main__] + +[case testListTypeAlias] +from typing import List +A = List[int] +def f(x: A) -> None: pass +f([1]) +f(['x']) # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] +[targets __main__, __main__.f] + +[case testAnyTypeAlias] +from typing import Any +A = Any +def f(x: A) -> None: + x.foo() +f(1) +f('x') + +[case testNoReturnTypeAlias] +# https://github.com/python/mypy/issues/11903 +from typing import NoReturn +Never = NoReturn +a: Never # Used to be an error here + +def f(a: Never): ... +f(5) # E: Argument 1 to "f" has incompatible type "int"; expected "Never" +[case testImportUnionAlias] +import typing +from _m import U +def f(x: U) -> None: pass +f(1) +f('x') +f(()) # E: Argument 1 to "f" has incompatible type "Tuple[()]"; expected "Union[int, str]" +[file _m.py] +from typing import Union +U = Union[int, str] +[builtins fixtures/tuple.pyi] + +[case testProhibitReassigningAliases] +A = float +if int(): + A = int # E: Cannot assign multiple types to name "A" without an explicit "Type[...]" annotation +[out] + +[case testProhibitReassigningSubscriptedAliases] +from typing import Callable +A = Callable[[], float] +if int(): + A = Callable[[], int] \ + # E: Cannot assign multiple types to name "A" without an explicit "Type[...]" annotation \ + # E: Value of type "int" is not indexable + # the second error is because of `Callable = 0` in lib-stub/typing.pyi +[builtins fixtures/list.pyi] +[out] + +[case testProhibitReassigningGenericAliases] +from typing import TypeVar, Union, Tuple +T = TypeVar('T') + +A = Tuple[T, T] +if int(): + A = Union[T, int] # E: Cannot assign multiple types to name "A" without an explicit "Type[...]" annotation +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testProhibitUsingVariablesAsTypesAndAllowAliasesAsTypes] + +from typing import TypeVar, Sequence, Type +T = TypeVar('T') + +A: Type[float] = int +if int(): + A = float # OK +x: A # E: Variable "__main__.A" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +def bad(tp: A) -> None: # E: Variable "__main__.A" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + pass + +Alias = int +GenAlias = Sequence[T] +def fun(x: Alias) -> GenAlias[int]: pass +[out] + +[case testCorrectQualifiedAliasesAlsoInFunctions] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') + +class X(Generic[T]): + A = X[S] + def f(self) -> X[T]: + pass + + a: X[T] + b: A = a + c: A[T] = a + d: A[int] = a # E: Incompatible types in assignment (expression has type "X[T]", variable has type "X[int]") + + def g(self) -> None: + a: X[T] + b: X.A = a + c: X.A[T] = a + d: X.A[int] = a # E: Incompatible types in assignment (expression has type "X[T]", variable has type "X[int]") + +def g(arg: X[int]) -> None: + p: X[int] = arg.f() + q: X.A = arg.f() + r: X.A[str] = arg.f() # E: Incompatible types in assignment (expression has type "X[int]", variable has type "X[str]") +[out] + +[case testProhibitBoundTypeVariableReuseForAliases] +from typing import TypeVar, Generic, List +T = TypeVar('T') +class C(Generic[T]): + A = List[T] # E: Can't use bound type variable "T" to define generic alias + +x: C.A +reveal_type(x) # N: Revealed type is "builtins.list[Any]" + +def f(x: T) -> T: + A = List[T] # E: Can't use bound type variable "T" to define generic alias + return x +[builtins fixtures/list.pyi] +[out] + +[case testTypeAliasInBuiltins] +def f(x: bytes): pass +bytes +f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "str" +[builtins fixtures/alias.pyi] + +[case testEmptyTupleTypeAlias] +from typing import Tuple, Callable +EmptyTuple = Tuple[()] +x: EmptyTuple +reveal_type(x) # N: Revealed type is "Tuple[()]" + +EmptyTupleCallable = Callable[[Tuple[()]], None] +f: EmptyTupleCallable +reveal_type(f) # N: Revealed type is "def (Tuple[()])" +[builtins fixtures/list.pyi] + +[case testForwardTypeAlias] +def f(p: 'Alias') -> None: + pass + +reveal_type(f) # N: Revealed type is "def (p: builtins.int)" +Alias = int +[out] + +[case testForwardTypeAliasGeneric] +from typing import TypeVar, Tuple +def f(p: 'Alias[str]') -> None: + pass + +reveal_type(f) # N: Revealed type is "def (p: Tuple[builtins.int, builtins.str])" +T = TypeVar('T') +Alias = Tuple[int, T] +[builtins fixtures/tuple.pyi] +[out] + +[case testRecursiveAliasesErrors1] +from typing import Type, Callable, Union + +def test() -> None: + A = Union[A, int] # E: Cannot resolve name "A" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + B = Callable[[B], int] # E: Cannot resolve name "B" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + C = Type[C] # E: Cannot resolve name "C" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + +[case testRecursiveAliasesErrors2] +# flags: --disable-error-code=used-before-def +from typing import Type, Callable, Union + +def test() -> None: + A = Union[B, int] + B = Callable[[C], int] + C = Type[A] + x: A + reveal_type(x) +[out] +main:5: error: Cannot resolve name "A" (possible cyclic definition) +main:5: note: Recursive types are not allowed at function scope +main:5: error: Cannot resolve name "B" (possible cyclic definition) +main:6: error: Cannot resolve name "B" (possible cyclic definition) +main:6: note: Recursive types are not allowed at function scope +main:6: error: Cannot resolve name "C" (possible cyclic definition) +main:7: error: Cannot resolve name "C" (possible cyclic definition) +main:7: note: Recursive types are not allowed at function scope +main:9: note: Revealed type is "Union[Any, builtins.int]" + +[case testDoubleForwardAlias] +# flags: --disable-error-code=used-before-def +from typing import List +x: A +A = List[B] +B = List[int] +reveal_type(x) # N: Revealed type is "builtins.list[builtins.list[builtins.int]]" +[builtins fixtures/list.pyi] +[out] + +[case testDoubleForwardAliasWithNamedTuple] +# flags: --disable-error-code=used-before-def +from typing import List, NamedTuple +x: A +A = List[B] +class B(NamedTuple): + x: str +reveal_type(x[0].x) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] +[out] + +[case testJSONAliasApproximation] +from typing import List, Union, Dict + +def test() -> None: + x: JSON # E: Cannot resolve name "JSON" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + JSON = Union[int, str, List[JSON], Dict[str, JSON]] # E: Cannot resolve name "JSON" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + reveal_type(x) # N: Revealed type is "Any" + if isinstance(x, list): + reveal_type(x) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testForwardRefToTypeVar] +# flags: --disable-error-code=used-before-def + +from typing import TypeVar, List +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" +a: A[int] +A = List[T] +T = TypeVar('T') +[builtins fixtures/list.pyi] +[out] + +[case testFunctionForwardRefAlias] + +from typing import List, TypeVar + +T = TypeVar('T') + +def f(x: T) -> List[T]: + y: A[T] + reveal_type(y) # N: Revealed type is "builtins.list[T`-1]" + return [x] + y + +A = List[T] +[builtins fixtures/list.pyi] +[out] + +[case testFunctionForwardRefAlias2] + +from typing import List, TypeVar + +def f() -> None: + X = List[int] + x: A[X] + reveal_type(x) # N: Revealed type is "builtins.list[builtins.list[builtins.int]]" + +T = TypeVar('T') +A = List[T] +[builtins fixtures/list.pyi] +[out] + +[case testNoneAlias] +from typing import Union +void = type(None) +x: void +reveal_type(x) # N: Revealed type is "None" +y: Union[int, void] +reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/bool.pyi] + +[case testNoneAliasStrict] +from typing import Optional, Union +void = type(None) +x: int +y: Union[int, void] +z: Optional[int] +x = y # E: Incompatible types in assignment (expression has type "Optional[int]", variable has type "int") +y = z +[builtins fixtures/bool.pyi] + +[case testAliasToTupleAndCallable] +from typing import Callable, Tuple +C = Callable +T = Tuple +c: C +t: T +reveal_type(c) # N: Revealed type is "def (*Any, **Any) -> Any" +reveal_type(t) # N: Revealed type is "builtins.tuple[Any, ...]" +bad: C[int] # E: Bad number of arguments for type alias, expected 0, given 1 +also_bad: T[int] # E: Bad number of arguments for type alias, expected 0, given 1 +[builtins fixtures/tuple.pyi] + +[case testAliasRefOnClass] +from typing import Generic, TypeVar, Type + +T = TypeVar('T') +class C(Generic[T]): + pass + +class N: + A = C[T] + B = C[int] + +x: N.A[C] +reveal_type(x) # N: Revealed type is "__main__.C[__main__.C[Any]]" + +xx = N.A[C]() +reveal_type(xx) # N: Revealed type is "__main__.C[__main__.C[Any]]" + +y = N.A() +reveal_type(y) # N: Revealed type is "__main__.C[Any]" + +M = N +b = M.A[int]() +reveal_type(b) # N: Revealed type is "__main__.C[builtins.int]" + +n: Type[N] +w = n.B() +reveal_type(w) # N: Revealed type is "__main__.C[builtins.int]" +[out] + +[case testTypeAliasesToNamedTuple] +from nt import C, D, E + +A1 = C +A2 = D +A3 = E + +class Cls: + A1 = C + A2 = D + A3 = E + +A1('no') # E: Argument 1 to "C" has incompatible type "str"; expected "int" +a1 = A1(1) +reveal_type(a1) # N: Revealed type is "Tuple[builtins.int, fallback=nt.C]" + +A2(0) # E: Argument 1 to "D" has incompatible type "int"; expected "str" +a2 = A2('yes') +reveal_type(a2) # N: Revealed type is "Tuple[builtins.str, fallback=nt.D]" + +a3 = A3() +reveal_type(a3) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=nt.E]" + +Cls.A1('no') # E: Argument 1 has incompatible type "str"; expected "int" +ca1 = Cls.A1(1) +reveal_type(ca1) # N: Revealed type is "Tuple[builtins.int, fallback=nt.C]" + +Cls.A2(0) # E: Argument 1 has incompatible type "int"; expected "str" +ca2 = Cls.A2('yes') +reveal_type(ca2) # N: Revealed type is "Tuple[builtins.str, fallback=nt.D]" + +ca3 = Cls.A3() +reveal_type(ca3) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=nt.E]" +[file nt.pyi] +from typing import NamedTuple, Tuple + +class C(NamedTuple): + x: int + +D = NamedTuple('D', [('y', str)]) + +class E(Tuple[int, str]): + pass +[builtins fixtures/tuple.pyi] +[out] + +[case testTypeAliasesToAny] +from typing import Any + +A = Any +A # this should not fail +[out] + +[case testDoubleImportsOfAnAlias] +from b import * +from c import * +[file b.py] +from a import A +[file c.py] +from a import A +[file a.py] +A = int +[out] + +[case testDoubleImportsOfAnAlias2] +from b import A +from c import A +[file b.py] +from a import A +[file c.py] +from a import A +[file a.py] +A = int +[out] + +[case testDoubleImportsOfAnAlias3] +from b import * +from c import * +[file b.py] +from a import A +[file c.py] +from a import A +[file a.py] +from typing import Union +A = Union[None] +[out] + +[case testAliasToClassMethod] +from typing import TypeVar, Generic, Union, Type + +T = TypeVar('T', bound='C') + +MYPY = False +if MYPY: + test = classmethod + +class C: + @test + def meth(cls: Type[T], arg: int) -> Union[T, int]: ... + +class D(C): ... + +reveal_type(D.meth(1)) # N: Revealed type is "Union[__main__.D, builtins.int]" +reveal_type(D().meth(1)) # N: Revealed type is "Union[__main__.D, builtins.int]" +[builtins fixtures/classmethod.pyi] +[out] + +[case testAliasInImportCycle] +# cmd: mypy -m t t2 +[file t.py] +MYPY = False +if MYPY: + from t2 import A +x: A +[file t2.py] +import t +from typing import Callable +A = Callable[[], None] +[builtins fixtures/bool.pyi] +[out] + +[case testAliasInImportCycle2] +import a +[file a.pyi] +from b import Parameter + +class _ParamType: + p: Parameter + +_ConvertibleType = _ParamType + +def convert_type(ty: _ConvertibleType): + ... + +[file b.pyi] +from a import _ConvertibleType + +class Parameter: + type: _ConvertibleType +[out] + +[case testAliasInImportCycle3] +# cmd: mypy -m t t2 + +[file t.py] +MYPY = False +if MYPY: + from t2 import A +x: A +reveal_type(x) # N: Revealed type is "t2.D" + +reveal_type(A) # N: Revealed type is "def () -> t2.D" +A() +[file t2.py] +import t +class D: pass +A = D +[builtins fixtures/bool.pyi] +[out] + +[case testFlexibleAlias1] +from typing import TypeVar, List, Tuple +from mypy_extensions import FlexibleAlias + +T = TypeVar('T') +U = TypeVar('U') + +AnInt = FlexibleAlias[T, int] + +x: AnInt[str] +reveal_type(x) # N: Revealed type is "builtins.int" + +TwoArgs = FlexibleAlias[Tuple[T, U], bool] +TwoArgs2 = FlexibleAlias[Tuple[T, U], List[U]] + +def welp(x: TwoArgs[str, int]) -> None: + reveal_type(x) # N: Revealed type is "builtins.bool" + +def welp2(x: TwoArgs2[str, int]) -> None: + reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" + + +Id = FlexibleAlias[T, T] + +def take_id(x: Id[int]) -> None: + reveal_type(x) # N: Revealed type is "builtins.int" + +def id(x: Id[T]) -> T: + return x + +# TODO: This doesn't work and maybe it should? +# Indirection = AnInt[T] +# y: Indirection[str] +# reveal_type(y) # E : Revealed type is "builtins.int" + +# But this does +Indirection2 = FlexibleAlias[T, AnInt[T]] +z: Indirection2[str] +reveal_type(z) # N: Revealed type is "builtins.int" + +Indirection3 = FlexibleAlias[Tuple[T, U], AnInt[T]] +w: Indirection3[str, int] +reveal_type(w) # N: Revealed type is "builtins.int" + +[builtins fixtures/dict.pyi] + +[case testFlexibleAlias2] +# flags: --always-true=BOGUS +from typing import TypeVar, Any +from mypy_extensions import FlexibleAlias + +T = TypeVar('T') + +BOGUS = True +if BOGUS: + Bogus = FlexibleAlias[T, Any] +else: + Bogus = FlexibleAlias[T, T] + +class A: + x: Bogus[str] + +reveal_type(A().x) # N: Revealed type is "Any" + +def foo(x: Bogus[int]) -> None: + reveal_type(x) # N: Revealed type is "Any" + +[builtins fixtures/dict.pyi] + +[case testFlexibleAlias3] +# flags: --always-false=BOGUS +from typing import TypeVar, Any +from mypy_extensions import FlexibleAlias + +T = TypeVar('T') + +BOGUS = True +if BOGUS: + Bogus = FlexibleAlias[T, Any] +else: + Bogus = FlexibleAlias[T, T] + +class A: + x: Bogus[str] + +reveal_type(A().x) # N: Revealed type is "builtins.str" + +def foo(x: Bogus[int]) -> None: + reveal_type(x) # N: Revealed type is "builtins.int" + +[builtins fixtures/dict.pyi] + +[case testOverrideByIdemAliasCorrectType] +C = C +class C: # type: ignore + pass +x: C +reveal_type(x) # N: Revealed type is "__main__.C" +[out] + +[case testOverrideByIdemAliasCorrectTypeReversed] +class C: + pass +C = C # type: ignore +x: C +reveal_type(x) # N: Revealed type is "__main__.C" +[out] + +[case testOverrideByIdemAliasCorrectTypeImported] +from other import C as B +C = B +x: C +reveal_type(x) # N: Revealed type is "other.C" +[file other.py] +class C: + pass +[out] + +[case testConditionalExceptionAlias] + +try: + E = E +except BaseException: + class E(BaseException): pass # type: ignore + +try: + pass +except E as e: + reveal_type(e) # N: Revealed type is "__main__.E" +[builtins fixtures/exception.pyi] +[out] + +[case testNestedClassOnAliasAsType] +class Out: + class In: + class Inner: + pass + +O = Out +I = Out.In +OI = O.In +A = Out +B = A + +w: O.In +x: I.Inner +y: OI.Inner +z: B.In +reveal_type(w) # N: Revealed type is "__main__.Out.In" +reveal_type(x) # N: Revealed type is "__main__.Out.In.Inner" +reveal_type(y) # N: Revealed type is "__main__.Out.In.Inner" +reveal_type(z) # N: Revealed type is "__main__.Out.In" + + +[case testSimplePep613] +from typing_extensions import TypeAlias +x: TypeAlias = str +a: x +reveal_type(a) # N: Revealed type is "builtins.str" + +y: TypeAlias = "str" +b: y +reveal_type(b) # N: Revealed type is "builtins.str" + +z: TypeAlias = "int | str" +c: z +reveal_type(c) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testForwardRefPep613] +from typing_extensions import TypeAlias + +x: TypeAlias = "MyClass" +a: x +reveal_type(a) # N: Revealed type is "__main__.MyClass" + +class MyClass: ... +[builtins fixtures/tuple.pyi] + +[case testInvalidPep613] +from typing_extensions import TypeAlias + +x: TypeAlias = list(int) # E: Invalid type alias: expression is not a valid type \ + # E: Too many arguments for "list" +a: x +[builtins fixtures/tuple.pyi] + +[case testAliasedImportPep613] +import typing as tpp +import typing_extensions as tpx +from typing import TypeAlias as TPA +from typing_extensions import TypeAlias as TXA +import typing +import typing_extensions + +Int1: tpp.TypeAlias = int +Int2: tpx.TypeAlias = int +Int3: TPA = int +Int4: TXA = int +Int5: typing.TypeAlias = int +Int6: typing_extensions.TypeAlias = int + +x1: Int1 = "str" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +x2: Int2 = "str" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +x3: Int3 = "str" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +x4: Int4 = "str" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +x5: Int5 = "str" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +x6: Int6 = "str" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testFunctionScopePep613] +from typing_extensions import TypeAlias + +def f() -> None: + x: TypeAlias = str + a: x + reveal_type(a) # N: Revealed type is "builtins.str" + + y: TypeAlias = "str" + b: y + reveal_type(b) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testImportCyclePep613] +# cmd: mypy -m t t2 +[file t.py] +MYPY = False +if MYPY: + from t2 import A +x: A +reveal_type(x) # N: Revealed type is "builtins.str" +[file t2.py] +from typing_extensions import TypeAlias +A: TypeAlias = str +[builtins fixtures/bool.pyi] +[out] + + +[case testLiteralStringPep675] +# flags: --python-version 3.11 +from typing import LiteralString as tpLS +from typing_extensions import LiteralString as tpxLS + +def f(a: tpLS, b: tpxLS) -> None: + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(b) # N: Revealed type is "builtins.str" + +# This isn't the correct behaviour, but should unblock use of LiteralString in typeshed +f("asdf", "asdf") +string: str +f(string, string) + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testForwardTypeVarRefWithRecursiveFlag] +import c +[file a.py] +from typing import TypeVar, List, Any, Generic +from b import Alias + +T = TypeVar("T", bound=Alias[Any]) +def foo(x: T) -> T: ... + +[file b.py] +from c import C +from typing import TypeVar, List + +S = TypeVar("S") +Alias = List[C[S]] + +[file c.py] +from typing import TypeVar, List, Generic +import a + +S = TypeVar("S") +class C(Generic[S], List[Defer]): ... +class Defer: ... +[builtins fixtures/list.pyi] + +[case testClassLevelTypeAliasesInUnusualContexts] +from typing import Union +from typing_extensions import TypeAlias + +class Foo: pass + +NormalImplicit = Foo +NormalExplicit: TypeAlias = Foo +SpecialImplicit = Union[int, str] +SpecialExplicit: TypeAlias = Union[int, str] + +class Parent: + NormalImplicit = Foo + NormalExplicit: TypeAlias = Foo + SpecialImplicit = Union[int, str] + SpecialExplicit: TypeAlias = Union[int, str] + +class Child(Parent): pass + +p = Parent() +c = Child() + +# Use type aliases in a runtime context + +reveal_type(NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(SpecialImplicit) # N: Revealed type is "typing._SpecialForm" +reveal_type(SpecialExplicit) # N: Revealed type is "typing._SpecialForm" + +reveal_type(Parent.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Parent.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Parent.SpecialImplicit) # N: Revealed type is "typing._SpecialForm" +reveal_type(Parent.SpecialExplicit) # N: Revealed type is "typing._SpecialForm" + +reveal_type(Child.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Child.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Child.SpecialImplicit) # N: Revealed type is "typing._SpecialForm" +reveal_type(Child.SpecialExplicit) # N: Revealed type is "typing._SpecialForm" + +reveal_type(p.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(p.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(p.SpecialImplicit) # N: Revealed type is "typing._SpecialForm" +reveal_type(p.SpecialExplicit) # N: Revealed type is "typing._SpecialForm" + +reveal_type(c.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(p.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(c.SpecialImplicit) # N: Revealed type is "typing._SpecialForm" +reveal_type(c.SpecialExplicit) # N: Revealed type is "typing._SpecialForm" + +# Use type aliases in a type alias context in a plausible way + +def plausible_top_1() -> NormalImplicit: pass +def plausible_top_2() -> NormalExplicit: pass +def plausible_top_3() -> SpecialImplicit: pass +def plausible_top_4() -> SpecialExplicit: pass +reveal_type(plausible_top_1) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_top_2) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_top_3) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" +reveal_type(plausible_top_4) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" + +def plausible_parent_1() -> Parent.NormalImplicit: pass # E: Variable "__main__.Parent.NormalImplicit" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +def plausible_parent_2() -> Parent.NormalExplicit: pass +def plausible_parent_3() -> Parent.SpecialImplicit: pass +def plausible_parent_4() -> Parent.SpecialExplicit: pass +reveal_type(plausible_parent_1) # N: Revealed type is "def () -> Parent.NormalImplicit?" +reveal_type(plausible_parent_2) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_parent_3) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" +reveal_type(plausible_parent_4) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" + +def plausible_child_1() -> Child.NormalImplicit: pass # E: Variable "__main__.Parent.NormalImplicit" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +def plausible_child_2() -> Child.NormalExplicit: pass +def plausible_child_3() -> Child.SpecialImplicit: pass +def plausible_child_4() -> Child.SpecialExplicit: pass +reveal_type(plausible_child_1) # N: Revealed type is "def () -> Child.NormalImplicit?" +reveal_type(plausible_child_2) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_child_3) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" +reveal_type(plausible_child_4) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" + +# Use type aliases in a type alias context in an implausible way + +def weird_parent_1() -> p.NormalImplicit: pass # E: Name "p.NormalImplicit" is not defined +def weird_parent_2() -> p.NormalExplicit: pass # E: Name "p.NormalExplicit" is not defined +def weird_parent_3() -> p.SpecialImplicit: pass # E: Name "p.SpecialImplicit" is not defined +def weird_parent_4() -> p.SpecialExplicit: pass # E: Name "p.SpecialExplicit" is not defined +reveal_type(weird_parent_1) # N: Revealed type is "def () -> Any" +reveal_type(weird_parent_2) # N: Revealed type is "def () -> Any" +reveal_type(weird_parent_3) # N: Revealed type is "def () -> Any" +reveal_type(weird_parent_4) # N: Revealed type is "def () -> Any" + +def weird_child_1() -> c.NormalImplicit: pass # E: Name "c.NormalImplicit" is not defined +def weird_child_2() -> c.NormalExplicit: pass # E: Name "c.NormalExplicit" is not defined +def weird_child_3() -> c.SpecialImplicit: pass # E: Name "c.SpecialImplicit" is not defined +def weird_child_4() -> c.SpecialExplicit: pass # E: Name "c.SpecialExplicit" is not defined +reveal_type(weird_child_1) # N: Revealed type is "def () -> Any" +reveal_type(weird_child_2) # N: Revealed type is "def () -> Any" +reveal_type(weird_child_3) # N: Revealed type is "def () -> Any" +reveal_type(weird_child_4) # N: Revealed type is "def () -> Any" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testMalformedTypeAliasRuntimeReassignments] +from typing import Union +from typing_extensions import TypeAlias + +class Foo: pass + +NormalImplicit = Foo +NormalExplicit: TypeAlias = Foo +SpecialImplicit = Union[int, str] +SpecialExplicit: TypeAlias = Union[int, str] + +class Parent: + NormalImplicit = Foo + NormalExplicit: TypeAlias = Foo + SpecialImplicit = Union[int, str] + SpecialExplicit: TypeAlias = Union[int, str] + +class Child(Parent): pass + +p = Parent() +c = Child() + +NormalImplicit = 4 # E: Cannot assign multiple types to name "NormalImplicit" without an explicit "Type[...]" annotation \ + # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +NormalExplicit = 4 # E: Cannot assign multiple types to name "NormalExplicit" without an explicit "Type[...]" annotation \ + # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +SpecialImplicit = 4 # E: Cannot assign multiple types to name "SpecialImplicit" without an explicit "Type[...]" annotation +SpecialExplicit = 4 # E: Cannot assign multiple types to name "SpecialExplicit" without an explicit "Type[...]" annotation + +Parent.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Parent.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Parent.SpecialImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "") +Parent.SpecialExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "") + +Child.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Child.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Child.SpecialImplicit = 4 +Child.SpecialExplicit = 4 + +p.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +p.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +p.SpecialImplicit = 4 +p.SpecialExplicit = 4 + +c.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +c.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +c.SpecialImplicit = 4 +c.SpecialExplicit = 4 +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testNewStyleUnionInTypeAliasWithMalformedInstance] +# flags: --python-version 3.10 +from typing import List + +A = List[int, str] | int # E: "list" expects 1 type argument, but 2 given +B = int | list[int, str] # E: "list" expects 1 type argument, but 2 given +a: A +b: B +reveal_type(a) # N: Revealed type is "Union[builtins.list[Any], builtins.int]" +reveal_type(b) # N: Revealed type is "Union[builtins.int, builtins.list[Any]]" +[builtins fixtures/type.pyi] + +[case testValidTypeAliasValues] +from typing import TypeVar, Generic, List + +T = TypeVar("T", int, str) +S = TypeVar("S", int, bytes) + +class C(Generic[T]): ... +class D(C[S]): ... # E: Invalid type argument value for "C" + +U = TypeVar("U") +A = List[C[U]] +x: A[bytes] # E: Value of type variable "T" of "C" cannot be "bytes" + +V = TypeVar("V", bound=int) +class E(Generic[V]): ... +B = List[E[U]] +y: B[str] # E: Type argument "str" of "E" must be a subtype of "int" + +[case testValidTypeAliasValuesMoreRestrictive] +from typing import TypeVar, Generic, List + +T = TypeVar("T") +S = TypeVar("S", int, str) +U = TypeVar("U", bound=int) + +class C(Generic[T]): ... + +A = List[C[S]] +x: A[int] +x_bad: A[bytes] # E: Value of type variable "S" of "A" cannot be "bytes" + +B = List[C[U]] +y: B[int] +y_bad: B[str] # E: Type argument "str" of "B" must be a subtype of "int" + +[case testTupleWithDifferentArgsPy38] +# flags: --python-version 3.8 +NotYet1 = tuple[float] # E: "tuple" is not subscriptable +NotYet2 = tuple[float, float] # E: "tuple" is not subscriptable +NotYet3 = tuple[float, ...] # E: Unexpected "..." \ + # E: "tuple" is not subscriptable +NotYet4 = tuple[float, float, ...] # E: Unexpected "..." \ + # E: "tuple" is not subscriptable +[builtins fixtures/tuple.pyi] + +[case testTupleWithDifferentArgsStub] +# https://github.com/python/mypy/issues/11098 +import tup + +[file tup.pyi] +Correct1 = str | tuple[float, float, str] +Correct2 = tuple[float] | str +Correct3 = tuple[float, ...] | str +Correct4 = tuple[float, str] | str +Correct5 = tuple[int, str] +Correct6 = tuple[int, ...] + +RHSAlias1: type = tuple[int, int] +RHSAlias2: type = tuple[int] +RHSAlias3: type = tuple[int, ...] + +# Wrong: + +WrongTypeElement = str | tuple[float, 1] # E: Invalid type: try using Literal[1] instead? +WrongEllipsis = str | tuple[float, float, ...] # E: Unexpected "..." +[builtins fixtures/tuple.pyi] + +[case testCompiledNoCrashOnSingleItemUnion] +# flags: --no-strict-optional +from typing import Callable, Union, Generic, TypeVar + +Alias = Callable[[], int] + +T = TypeVar("T") +class C(Generic[T]): + attr: Union[Alias, None] = None + + @classmethod + def test(cls) -> None: + cls.attr +[builtins fixtures/classmethod.pyi] + +[case testRecursiveAliasTuple] +from typing_extensions import Literal, TypeAlias +from typing import Tuple, Union + +Expr: TypeAlias = Union[ + Tuple[Literal[123], int], + Tuple[Literal[456], "Expr"], +] + +def eval(e: Expr) -> int: + if e[0] == 123: + return e[1] + elif e[0] == 456: + return -eval(e[1]) +[builtins fixtures/dict-full.pyi] + +[case testTypeAliasType] +from typing import Union +from typing_extensions import TypeAliasType + +TestType = TypeAliasType("TestType", Union[int, str]) +x: TestType = 42 +y: TestType = 'a' +z: TestType = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[int, str]") + +reveal_type(TestType) # N: Revealed type is "typing_extensions.TypeAliasType" +TestType() # E: "TypeAliasType" not callable + +class A: + ClassAlias = TypeAliasType("ClassAlias", int) +xc: A.ClassAlias = 1 +yc: A.ClassAlias = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeAliasTypePython311] +# flags: --python-version 3.11 +# Pinning to 3.11, because 3.12 has `TypeAliasType` +from typing_extensions import TypeAliasType + +TestType = TypeAliasType("TestType", int) +x: TestType = 1 +[builtins fixtures/tuple.pyi] + +[case testTypeAliasTypeInvalid] +from typing_extensions import TypeAliasType + +TestType = TypeAliasType("T", int) # E: String argument 1 "T" to TypeAliasType(...) does not match variable name "TestType" + +T1 = T2 = TypeAliasType("T", int) +t1: T1 # E: Variable "__main__.T1" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +T3 = TypeAliasType("T3", -1) # E: Invalid type: try using Literal[-1] instead? +t3: T3 +reveal_type(t3) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeAliasTypeGeneric] +from typing import Callable, Dict, Generic, TypeVar, Tuple +from typing_extensions import TypeAliasType, TypeVarTuple, ParamSpec, Unpack + +K = TypeVar('K') +V = TypeVar('V') +T = TypeVar('T') +Ts = TypeVarTuple("Ts") +Ts1 = TypeVarTuple("Ts1") +P = ParamSpec("P") + +TestType = TypeAliasType("TestType", Dict[K, V], type_params=(K, V)) +x: TestType[int, str] = {1: 'a'} +y: TestType[str, int] = {'a': 1} +z: TestType[str, int] = {1: 'a'} # E: Dict entry 0 has incompatible type "int": "str"; expected "str": "int" +w: TestType[int] # E: Bad number of arguments for type alias, expected 2, given 1 + +InvertedDict = TypeAliasType("InvertedDict", Dict[K, V], type_params=(V, K)) +xi: InvertedDict[str, int] = {1: 'a'} +yi: InvertedDict[str, int] = {'a': 1} # E: Dict entry 0 has incompatible type "str": "int"; expected "int": "str" +zi: InvertedDict[int, str] = {1: 'a'} # E: Dict entry 0 has incompatible type "int": "str"; expected "str": "int" +reveal_type(xi) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +VariadicAlias1 = TypeAliasType("VariadicAlias1", Tuple[Unpack[Ts]], type_params=(Ts,)) +VariadicAlias2 = TypeAliasType("VariadicAlias2", Tuple[Unpack[Ts], K], type_params=(Ts, K)) +VariadicAlias3 = TypeAliasType("VariadicAlias3", Callable[[Unpack[Ts]], int], type_params=(Ts,)) +xv: VariadicAlias1[int, str] = (1, 'a') +yv: VariadicAlias1[str, int] = (1, 'a') # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "Tuple[str, int]") +zv: VariadicAlias2[int, str] = (1, 'a') +def int_in_int_out(x: int) -> int: return x +wv: VariadicAlias3[int] = int_in_int_out +reveal_type(wv) # N: Revealed type is "def (builtins.int) -> builtins.int" + +ParamAlias = TypeAliasType("ParamAlias", Callable[P, int], type_params=(P,)) +def f(x: str, y: float) -> int: return 1 +def g(x: int, y: float) -> int: return 1 +xp1: ParamAlias[str, float] = f +xp2: ParamAlias[str, float] = g # E: Incompatible types in assignment (expression has type "Callable[[int, float], int]", variable has type "Callable[[str, float], int]") +xp3: ParamAlias[str, float] = lambda x, y: 1 + +class G(Generic[P, T]): ... +ParamAlias2 = TypeAliasType("ParamAlias2", G[P, T], type_params=(P, T)) +xp: ParamAlias2[[int], str] +reveal_type(xp) # N: Revealed type is "__main__.G[[builtins.int], builtins.str]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeAliasTypeInvalidGeneric] +from typing_extensions import TypeAliasType, TypeVarTuple, ParamSpec +from typing import Callable, Dict, Generic, TypeVar, Tuple, Unpack + +K = TypeVar('K') +V = TypeVar('V') +T = TypeVar('T') +Ts = TypeVarTuple("Ts") +Ts1 = TypeVarTuple("Ts1") +P = ParamSpec("P") + +Ta0 = TypeAliasType("Ta0", int, type_params=(T, T)) # E: Duplicate type variable "T" in type_params argument to TypeAliasType + +Ta1 = TypeAliasType("Ta1", int, type_params=K) # E: Tuple literal expected as the type_params argument to TypeAliasType + +Ta2 = TypeAliasType("Ta2", int, type_params=(None,)) # E: Free type variable expected in type_params argument to TypeAliasType + +Ta3 = TypeAliasType("Ta3", Dict[K, V], type_params=(V,)) # E: Type variable "K" is not included in type_params +partially_generic1: Ta3[int] = {"a": 1} +reveal_type(partially_generic1) # N: Revealed type is "builtins.dict[Any, builtins.int]" +partially_generic2: Ta3[int] = {1: "a"} # E: Dict entry 0 has incompatible type "int": "str"; expected "Any": "int" + +Ta4 = TypeAliasType("Ta4", Tuple[Unpack[Ts]], type_params=(Ts, Ts1)) # E: Can only use one TypeVarTuple in type_params argument to TypeAliasType + +Ta5 = TypeAliasType("Ta5", Dict) # Unlike old style aliases, this is not generic +non_generic_dict: Ta5[int, str] # E: Bad number of arguments for type alias, expected 0, given 2 +reveal_type(non_generic_dict) # N: Revealed type is "builtins.dict[Any, Any]" + +Ta6 = TypeAliasType("Ta6", Tuple[Unpack[Ts]]) # E: TypeVarTuple "Ts" is not included in type_params +unbound_tvt_alias: Ta6[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(unbound_tvt_alias) # N: Revealed type is "builtins.tuple[Any, ...]" + +class G(Generic[P, T]): ... +Ta7 = TypeAliasType("Ta7", G[P, T]) # E: ParamSpec "P" is not included in type_params \ + # E: Type variable "T" is not included in type_params +unbound_ps_alias: Ta7[[int], str] # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: Bad number of arguments for type alias, expected 0, given 2 +reveal_type(unbound_ps_alias) # N: Revealed type is "__main__.G[Any, Any]" + +Ta8 = TypeAliasType("Ta8", Callable[P, int]) # E: ParamSpec "P" is not included in type_params +unbound_ps_alias2: Ta8[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(unbound_ps_alias2) # N: Revealed type is "def [P] (*Any, **Any) -> builtins.int" + +Ta9 = TypeAliasType("Ta9", Callable[P, T]) # E: ParamSpec "P" is not included in type_params \ + # E: Type variable "T" is not included in type_params +unbound_ps_alias3: Ta9[int, str] # E: Bad number of arguments for type alias, expected 0, given 2 +reveal_type(unbound_ps_alias3) # N: Revealed type is "def [P] (*Any, **Any) -> Any" + +Ta10 = TypeAliasType("Ta10", Callable[[Unpack[Ts]], str]) # E: TypeVarTuple "Ts" is not included in type_params +unbound_tvt_alias2: Ta10[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(unbound_tvt_alias2) # N: Revealed type is "def (*Any) -> builtins.str" + +class A(Generic[T]): + Ta11 = TypeAliasType("Ta11", Dict[str, T], type_params=(T,)) # E: Can't use bound type variable "T" to define generic alias +x: A.Ta11 = {"a": 1} +reveal_type(x) # N: Revealed type is "builtins.dict[builtins.str, Any]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeAliasTypeNoUnpackInTypeParams311] +# flags: --python-version 3.11 +from typing_extensions import TypeAliasType, TypeVar, TypeVarTuple, Unpack + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") + +Ta1 = TypeAliasType("Ta1", None, type_params=(*Ts,)) # E: can't use starred expression here +Ta2 = TypeAliasType("Ta2", None, type_params=(Unpack[Ts],)) # E: Free type variable expected in type_params argument to TypeAliasType \ + # N: Don't Unpack type variables in type_params + +[builtins fixtures/tuple.pyi] + +[case testAliasInstanceNameClash] +from lib import func +class A: ... +func(A()) # E: Argument 1 to "func" has incompatible type "__main__.A"; expected "lib.A" +[file lib.py] +from typing import List, Union + +A = Union[int, List[A]] +def func(x: A) -> int: ... +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-type-checks.test b/test-data/unit/check-type-checks.test new file mode 100644 index 000000000000..03c8de4177f3 --- /dev/null +++ b/test-data/unit/check-type-checks.test @@ -0,0 +1,117 @@ +-- Conditional type checks. + + +[case testSimpleIsinstance] +x: object +n: int +s: str +if int(): + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") +if isinstance(x, int): + n = x + s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") +[builtins fixtures/isinstance.pyi] + +[case testSimpleIsinstance2] +import typing +def f(x: object, n: int, s: str) -> None: + if int(): + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") + if isinstance(x, int): + n = x + s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") +[builtins fixtures/isinstance.pyi] +[out] + +[case testSimpleIsinstance3] +class A: + x = None # type: object + n = None # type: int + s = None # type: str + if int(): + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") + if isinstance(x, int): + n = x + s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + else: + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") +[builtins fixtures/isinstance.pyi] +[out] + +[case testMultipleIsinstanceTests] +import typing +class A: pass +class B(A): pass +def f(x: object, a: A, b: B, c: int) -> None: + if isinstance(x, A): + if isinstance(x, B): + b = x + x = a + a = x + c = x # E: Incompatible types in assignment (expression has type "A", variable has type "int") +[builtins fixtures/isinstance.pyi] +[out] + +[case testMultipleIsinstanceTests2] +import typing +class A: pass +class B(A): pass +def f(x: object, y: object, n: int, s: str) -> None: + if isinstance(x, int): + if isinstance(y, str): + n = x + s = y + s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + n = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + s = y # E: Incompatible types in assignment (expression has type "object", variable has type "str") + n = y # E: Incompatible types in assignment (expression has type "object", variable has type "int") + n = x +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceAndElif] +import typing +def f(x: object, n: int, s: str) -> None: + if int(): + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") + if isinstance(x, int): + n = x + s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + elif isinstance(x, str): + s = x + n = x # E: Incompatible types in assignment (expression has type "str", variable has type "int") + else: + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") + s = x # E: Incompatible types in assignment (expression has type "object", variable has type "str") + n = x # E: Incompatible types in assignment (expression has type "object", variable has type "int") +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceAndAnyType] +from typing import Any +def f(x: Any, n: int, s: str) -> None: + if int(): + s = x + if isinstance(x, int): + n = x + s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + s = x +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceAndGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +class C(Generic[T]): + def f(self, x: T) -> None: pass +def f(x: object) -> None: + if isinstance(x, C): + x.f(1) + x.f('') + x.g() # E: "C[Any]" has no attribute "g" + x.g() # E: "object" has no attribute "g" +[builtins fixtures/isinstance.pyi] +[out] diff --git a/test-data/unit/check-type-object-type-inference.test b/test-data/unit/check-type-object-type-inference.test new file mode 100644 index 000000000000..5a4afa0c9248 --- /dev/null +++ b/test-data/unit/check-type-object-type-inference.test @@ -0,0 +1,42 @@ +[case testInferTupleType] +# flags: --python-version 3.9 +from typing import TypeVar, Generic, Type +from abc import abstractmethod +import types # Explicitly bring in stubs for 'types' + +T = TypeVar('T') +class E(Generic[T]): + @abstractmethod + def e(self, t: T) -> str: + ... + +class F: + @abstractmethod + def f(self, tp: Type[T]) -> E[T]: + ... + +def g(f: F): + f.f(int).e(7) + f.f(tuple[int,str]) + f.f(tuple[int,str]).e('x') # E: Argument 1 to "e" of "E" has incompatible type "str"; expected "Tuple[int, str]" + f.f(tuple[int,str]).e( (7,8) ) # E: Argument 1 to "e" of "E" has incompatible type "Tuple[int, int]"; expected "Tuple[int, str]" + f.f(tuple[int,str]).e( (7,'x') ) # OK + reveal_type(f.f(tuple[int,str]).e) # N: Revealed type is "def (t: Tuple[builtins.int, builtins.str]) -> builtins.str" + +def h(f: F): + f.f(int).e(7) + f.f(tuple) + f.f(tuple).e('y') # E: Argument 1 to "e" of "E" has incompatible type "str"; expected "Tuple[Any, ...]" + f.f(tuple).e( (8,'y') ) # OK + reveal_type(f.f(tuple).e) # N: Revealed type is "def (t: builtins.tuple[Any, ...]) -> builtins.str" + +def i(f: F): + f.f(tuple[int,tuple[int,str]]) + f.f(tuple[int,tuple[int,str]]).e('z') # E: Argument 1 to "e" of "E" has incompatible type "str"; expected "Tuple[int, Tuple[int, str]]" + f.f(tuple[int,tuple[int,str]]).e( (8,9) ) # E: Argument 1 to "e" of "E" has incompatible type "Tuple[int, int]"; expected "Tuple[int, Tuple[int, str]]" + f.f(tuple[int,tuple[int,str]]).e( (17, (28, 29)) ) # E: Argument 1 to "e" of "E" has incompatible type "Tuple[int, Tuple[int, int]]"; expected "Tuple[int, Tuple[int, str]]" + f.f(tuple[int,tuple[int,str]]).e( (27,(28,'z')) ) # OK + reveal_type(f.f(tuple[int,tuple[int,str]]).e) # N: Revealed type is "def (t: Tuple[builtins.int, Tuple[builtins.int, builtins.str]]) -> builtins.str" + +x = tuple[int,str][str] # False negative +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-type-promotion.test b/test-data/unit/check-type-promotion.test new file mode 100644 index 000000000000..d98d0c60e164 --- /dev/null +++ b/test-data/unit/check-type-promotion.test @@ -0,0 +1,189 @@ +-- Test cases for type promotion (e.g. int -> float). + + +[case testPromoteIntToFloat] +def f(x: float) -> None: pass +f(1) +[builtins fixtures/primitives.pyi] + +[case testCantPromoteFloatToInt] +def f(x: int) -> None: pass +f(1.1) # E: Argument 1 to "f" has incompatible type "float"; expected "int" +[builtins fixtures/primitives.pyi] + +[case testPromoteFloatToComplex] +def f(x: complex) -> None: pass +f(1) +[builtins fixtures/primitives.pyi] + +[case testPromoteIntToComplex] +def f(x: complex) -> None: pass +f(1) +[builtins fixtures/primitives.pyi] + +[case testPromoteBytearrayToByte] +def f(x: bytes) -> None: pass +f(bytearray(b'')) +[builtins fixtures/primitives.pyi] + +[case testPromoteMemoryviewToBytes] +def f(x: bytes) -> None: pass +f(memoryview(b'')) +[builtins fixtures/primitives.pyi] + +[case testNarrowingDownFromPromoteTargetType] +y = 0.0 +y = 1 +y() # E: "int" not callable +[builtins fixtures/primitives.pyi] + +[case testNarrowingDownFromPromoteTargetType2] +y = 0.0 +y = 1 +y.x # E: "int" has no attribute "x" +[builtins fixtures/primitives.pyi] + +[case testTypePromotionsDontInterfereWithProtocols] +from typing import TypeVar, Union, Protocol + +class SupportsFloat(Protocol): + def __float__(self) -> float: pass + +T = TypeVar('T') +def f(x: Union[SupportsFloat, T]) -> Union[SupportsFloat, T]: pass +f(0) # should not crash +[builtins fixtures/primitives.pyi] +[out] + +[case testIntersectionUsingPromotion1] +# flags: --warn-unreachable +from typing import Union + +x: complex = 1 +reveal_type(x) # N: Revealed type is "builtins.complex" +if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "builtins.complex" +reveal_type(x) # N: Revealed type is "builtins.complex" + +y: Union[int, float] +if isinstance(y, float): + reveal_type(y) # N: Revealed type is "builtins.float" +else: + reveal_type(y) # N: Revealed type is "builtins.int" + +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.float]" + +if isinstance(y, int): + reveal_type(y) # N: Revealed type is "builtins.int" +else: + reveal_type(y) # N: Revealed type is "builtins.float" +[builtins fixtures/primitives.pyi] + +[case testIntersectionUsingPromotion2] +# flags: --warn-unreachable +x: complex = 1 +reveal_type(x) # N: Revealed type is "builtins.complex" +if isinstance(x, (int, float)): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" +else: + reveal_type(x) # N: Revealed type is "builtins.complex" + +# Note we make type precise, since type promotions are involved +reveal_type(x) # N: Revealed type is "builtins.complex" +[builtins fixtures/primitives.pyi] + +[case testIntersectionUsingPromotion3] +# flags: --warn-unreachable +x: object +if isinstance(x, int) and isinstance(x, complex): + reveal_type(x) # N: Revealed type is "builtins.int" +if isinstance(x, complex) and isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/primitives.pyi] + +[case testIntersectionUsingPromotion4] +# flags: --warn-unreachable +x: object +if isinstance(x, int): + if isinstance(x, complex): + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.int" +if isinstance(x, complex): + if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.complex" +[builtins fixtures/primitives.pyi] + +[case testIntersectionUsingPromotion5] +# flags: --warn-unreachable +from typing import Union + +x: Union[float, complex] +if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.float, builtins.complex]" +reveal_type(x) # N: Revealed type is "Union[builtins.float, builtins.complex]" +[builtins fixtures/primitives.pyi] + +[case testIntersectionUsingPromotion6] +# flags: --warn-unreachable +from typing import Union + +x: Union[str, complex] +if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.complex]" +reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.complex]" +[builtins fixtures/primitives.pyi] + +[case testIntersectionUsingPromotion7] +# flags: --warn-unreachable +from typing import Union + +x: Union[int, float, complex] +if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.float, builtins.complex]" + +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.complex]" + +if isinstance(x, float): + reveal_type(x) # N: Revealed type is "builtins.float" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.complex]" + +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.complex]" + +if isinstance(x, complex): + reveal_type(x) # N: Revealed type is "builtins.complex" +else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" + +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.complex]" +[builtins fixtures/primitives.pyi] + +[case testIntersectionUsingPromotion8] +# flags: --warn-unreachable +from typing import Union + +x: Union[int, float, complex] +if isinstance(x, (int, float)): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" +else: + reveal_type(x) # N: Revealed type is "builtins.complex" +if isinstance(x, (int, complex)): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.complex]" +else: + reveal_type(x) # N: Revealed type is "builtins.float" +if isinstance(x, (float, complex)): + reveal_type(x) # N: Revealed type is "Union[builtins.float, builtins.complex]" +else: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/primitives.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test new file mode 100644 index 000000000000..a30fec1b9422 --- /dev/null +++ b/test-data/unit/check-typeddict.test @@ -0,0 +1,4055 @@ +-- Create Instance + +[case testCanCreateTypedDictInstanceWithKeywordArguments] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(x=42, y=1337) +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})" +# Use values() to check fallback value type. +reveal_type(p.values()) # N: Revealed type is "typing.Iterable[builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[targets __main__] + +[case testCanCreateTypedDictInstanceWithDictCall] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})" +# Use values() to check fallback value type. +reveal_type(p.values()) # N: Revealed type is "typing.Iterable[builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCanCreateTypedDictInstanceWithDictLiteral] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point({'x': 42, 'y': 1337}) +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})" +# Use values() to check fallback value type. +reveal_type(p.values()) # N: Revealed type is "typing.Iterable[builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCanCreateTypedDictInstanceWithNoArguments] +from typing import TypeVar, Union +from mypy_extensions import TypedDict +EmptyDict = TypedDict('EmptyDict', {}) +p = EmptyDict() +reveal_type(p) # N: Revealed type is "TypedDict('__main__.EmptyDict', {})" +reveal_type(p.values()) # N: Revealed type is "typing.Iterable[builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +-- Create Instance (Errors) + +[case testCannotCreateTypedDictInstanceWithUnknownArgumentPattern] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(42, 1337) # E: Expected keyword arguments, {...}, or dict(...) in TypedDict constructor +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictInstanceNonLiteralItemName] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +x = 'x' +p = Point({x: 42, 'y': 1337}) # E: Expected TypedDict key to be string literal +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictInstanceWithExtraItems] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(x=42, y=1337, z=666) # E: Extra key "z" for TypedDict "Point" +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictInstanceWithMissingItems] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(x=42) # E: Missing key "y" for TypedDict "Point" +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictInstanceWithIncompatibleItemType] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(x='meaning_of_life', y=1337) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictInstanceWithInlineTypedDict] +from mypy_extensions import TypedDict +D = TypedDict('D', { + 'x': TypedDict('E', { # E: Use dict literal for nested TypedDict + 'y': int + }) +}) +[builtins fixtures/dict.pyi] + +-- Define TypedDict (Class syntax) + +[case testCanCreateTypedDictWithClass] +from mypy_extensions import TypedDict + +class Point(TypedDict): + x: int + y: int + +p = Point(x=42, y=1337) +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCanCreateTypedDictWithSubclass] +from mypy_extensions import TypedDict + +class Point1D(TypedDict): + x: int +class Point2D(Point1D): + y: int +r: Point1D +p: Point2D +reveal_type(r) # N: Revealed type is "TypedDict('__main__.Point1D', {'x': builtins.int})" +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point2D', {'x': builtins.int, 'y': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCanCreateTypedDictWithSubclass2] +from mypy_extensions import TypedDict + +class Point1D(TypedDict): + x: int +class Point2D(TypedDict, Point1D): # We also allow to include TypedDict in bases, it is simply ignored at runtime + y: int + +p: Point2D +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point2D', {'x': builtins.int, 'y': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCanCreateTypedDictClassEmpty] +from mypy_extensions import TypedDict + +class EmptyDict(TypedDict): + pass + +p = EmptyDict() +reveal_type(p) # N: Revealed type is "TypedDict('__main__.EmptyDict', {})" +[builtins fixtures/dict.pyi] + + +[case testCanCreateTypedDictWithClassOldVersion] +# Test that we can use class-syntax to merge function-based TypedDicts + +from mypy_extensions import TypedDict + +MovieBase1 = TypedDict( + 'MovieBase1', {'name': str, 'year': int}) +MovieBase2 = TypedDict( + 'MovieBase2', {'based_on': str}, total=False) + +class Movie(MovieBase1, MovieBase2): + pass + +def foo(x): + # type: (Movie) -> None + pass + +foo({}) # E: Missing keys ("name", "year") for TypedDict "Movie" +foo({'name': 'lol', 'year': 2009, 'based_on': 0}) # E: Incompatible types (expression has type "int", TypedDict item "based_on" has type "str") + +[builtins fixtures/dict.pyi] + +-- Define TypedDict (Class syntax errors) + +[case testCannotCreateTypedDictWithClassOtherBases] +from mypy_extensions import TypedDict + +class A: pass + +class Point1D(TypedDict, A): # E: All bases of a new TypedDict must be TypedDict types + x: int +class Point2D(Point1D, A): # E: All bases of a new TypedDict must be TypedDict types + y: int + +p: Point2D +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point2D', {'x': builtins.int, 'y': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictWithDuplicateBases] +# https://github.com/python/mypy/issues/3673 +from typing import TypedDict + +class A(TypedDict): + x: str + y: int + +class B(A, A): # E: Duplicate base class "A" + z: str + +class C(TypedDict, TypedDict): # E: Duplicate base class "TypedDict" + c1: int +[typing fixtures/typing-typeddict.pyi] + +[case testCannotCreateTypedDictWithClassWithOtherStuff] +from mypy_extensions import TypedDict + +class Point(TypedDict): + x: int + y: int = 1 # E: Right hand side values are not supported in TypedDict + def f(): pass # E: Invalid statement in TypedDict definition; expected "field_name: field_type" + z = int # E: Invalid statement in TypedDict definition; expected "field_name: field_type" + +p = Point(x=42, y=1337, z='whatever') +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int, 'z': Any})" +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictWithClassWithFunctionUsedToCrash] +# https://github.com/python/mypy/issues/11079 +from typing import TypedDict +class D(TypedDict): + y: int + def x(self, key: int): # E: Invalid statement in TypedDict definition; expected "field_name: field_type" + pass + +d = D(y=1) +reveal_type(d) # N: Revealed type is "TypedDict('__main__.D', {'y': builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCannotCreateTypedDictWithDecoratedFunction] +# flags: --disallow-any-expr +# https://github.com/python/mypy/issues/13066 +from typing import TypedDict +class D(TypedDict): + @classmethod # E: Invalid statement in TypedDict definition; expected "field_name: field_type" + def m(self) -> D: + pass +d = D() +reveal_type(d) # N: Revealed type is "TypedDict('__main__.D', {})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictWithClassmethodAlternativeConstructorDoesNotCrash] +# https://github.com/python/mypy/issues/5653 +from typing import TypedDict + +class Foo(TypedDict): + bar: str + @classmethod # E: Invalid statement in TypedDict definition; expected "field_name: field_type" + def baz(cls) -> "Foo": ... +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCanCreateTypedDictTypeWithUnderscoreItemName] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int, '_fallback': object}) +[builtins fixtures/dict.pyi] + +[case testCanCreateTypedDictWithClassUnderscores] +from mypy_extensions import TypedDict + +class Point(TypedDict): + x: int + _y: int + +p: Point +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, '_y': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictWithDuplicateKey1] +from mypy_extensions import TypedDict + +class Bad(TypedDict): + x: int + x: str # E: Duplicate TypedDict key "x" + +b: Bad +reveal_type(b) # N: Revealed type is "TypedDict('__main__.Bad', {'x': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictWithDuplicateKey2] +from typing import TypedDict + +D1 = TypedDict("D1", { + "x": int, + "x": int, # E: Duplicate TypedDict key "x" +}) +D2 = TypedDict("D2", {"x": int, "x": str}) # E: Duplicate TypedDict key "x" + +d1: D1 +d2: D2 +reveal_type(d1) # N: Revealed type is "TypedDict('__main__.D1', {'x': builtins.int})" +reveal_type(d2) # N: Revealed type is "TypedDict('__main__.D2', {'x': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCanCreateTypedDictWithClassOverwriting] +from mypy_extensions import TypedDict + +class Point1(TypedDict): + x: int +class Point2(TypedDict): + x: float +class Bad(Point1, Point2): # E: Overwriting TypedDict field "x" while merging + pass + +b: Bad +reveal_type(b) # N: Revealed type is "TypedDict('__main__.Bad', {'x': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCanCreateTypedDictWithClassOverwriting2] +from mypy_extensions import TypedDict + +class Point1(TypedDict): + x: int +class Point2(Point1): + x: float # E: Overwriting TypedDict field "x" while extending + +p2: Point2 +reveal_type(p2) # N: Revealed type is "TypedDict('__main__.Point2', {'x': builtins.float})" +[builtins fixtures/dict.pyi] + + +-- Subtyping + +[case testCanConvertTypedDictToItself] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +def identity(p: Point) -> Point: + return p +[builtins fixtures/dict.pyi] + +[case testCanConvertTypedDictToEquivalentTypedDict] +from mypy_extensions import TypedDict +PointA = TypedDict('PointA', {'x': int, 'y': int}) +PointB = TypedDict('PointB', {'x': int, 'y': int}) +def identity(p: PointA) -> PointB: + return p +[builtins fixtures/dict.pyi] + +[case testCannotConvertTypedDictToSimilarTypedDictWithNarrowerItemTypes] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +ObjectPoint = TypedDict('ObjectPoint', {'x': object, 'y': object}) +def convert(op: ObjectPoint) -> Point: + return op # E: Incompatible return value type (got "ObjectPoint", expected "Point") +[builtins fixtures/dict.pyi] + +[case testCannotConvertTypedDictToSimilarTypedDictWithWiderItemTypes] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +ObjectPoint = TypedDict('ObjectPoint', {'x': object, 'y': object}) +def convert(p: Point) -> ObjectPoint: + return p # E: Incompatible return value type (got "Point", expected "ObjectPoint") +[builtins fixtures/dict.pyi] + +[case testCannotConvertTypedDictToSimilarTypedDictWithIncompatibleItemTypes] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +Chameleon = TypedDict('Chameleon', {'x': str, 'y': str}) +def convert(p: Point) -> Chameleon: + return p # E: Incompatible return value type (got "Point", expected "Chameleon") +[builtins fixtures/dict.pyi] + +[case testCanConvertTypedDictToNarrowerTypedDict] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +Point1D = TypedDict('Point1D', {'x': int}) +def narrow(p: Point) -> Point1D: + return p +[builtins fixtures/dict.pyi] + +[case testCannotConvertTypedDictToWiderTypedDict] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +Point3D = TypedDict('Point3D', {'x': int, 'y': int, 'z': int}) +def widen(p: Point) -> Point3D: + return p # E: Incompatible return value type (got "Point", expected "Point3D") +[builtins fixtures/dict.pyi] + +[case testCanConvertTypedDictToCompatibleMapping] +from mypy_extensions import TypedDict +from typing import Mapping +Point = TypedDict('Point', {'x': int, 'y': int}) +def as_mapping(p: Point) -> Mapping[str, object]: + return p +[builtins fixtures/dict.pyi] + +[case testCannotConvertTypedDictToIncompatibleMapping] +from mypy_extensions import TypedDict +from typing import Mapping +Point = TypedDict('Point', {'x': int, 'y': int}) +def as_mapping(p: Point) -> Mapping[str, int]: + return p # E: Incompatible return value type (got "Point", expected "Mapping[str, int]") +[builtins fixtures/dict.pyi] + +[case testTypedDictAcceptsIntForFloatDuckTypes] +from mypy_extensions import TypedDict +from typing import Any, Mapping +Point = TypedDict('Point', {'x': float, 'y': float}) +def create_point() -> Point: + return Point(x=1, y=2) +reveal_type(Point(x=1, y=2)) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.float, 'y': builtins.float})" +[builtins fixtures/dict.pyi] + +[case testTypedDictDoesNotAcceptsFloatForInt] +from mypy_extensions import TypedDict +from typing import Any, Mapping +Point = TypedDict('Point', {'x': int, 'y': int}) +def create_point() -> Point: + return Point(x=1.2, y=2.5) +[out] +main:5: error: Incompatible types (expression has type "float", TypedDict item "x" has type "int") +main:5: error: Incompatible types (expression has type "float", TypedDict item "y" has type "int") +[builtins fixtures/dict.pyi] + +[case testTypedDictAcceptsAnyType] +from mypy_extensions import TypedDict +from typing import Any, Mapping +Point = TypedDict('Point', {'x': float, 'y': float}) +def create_point(something: Any) -> Point: + return Point({ + 'x': something.x, + 'y': something.y + }) +[builtins fixtures/dict.pyi] + +[case testTypedDictValueTypeContext] +from mypy_extensions import TypedDict +from typing import List +D = TypedDict('D', {'x': List[int]}) +reveal_type(D(x=[])) # N: Revealed type is "TypedDict('__main__.D', {'x': builtins.list[builtins.int]})" +[builtins fixtures/dict.pyi] + +[case testCannotConvertTypedDictToDictOrMutableMapping] +from mypy_extensions import TypedDict +from typing import Dict, MutableMapping +Point = TypedDict('Point', {'x': int, 'y': int}) +def as_dict(p: Point) -> Dict[str, int]: + return p # E: Incompatible return value type (got "Point", expected "Dict[str, int]") +def as_mutable_mapping(p: Point) -> MutableMapping[str, object]: + return p # E: Incompatible return value type (got "Point", expected "MutableMapping[str, object]") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanConvertTypedDictToAny] +from mypy_extensions import TypedDict +from typing import Any +Point = TypedDict('Point', {'x': int, 'y': int}) +def unprotect(p: Point) -> Any: + return p +[builtins fixtures/dict.pyi] + +[case testAnonymousTypedDictInErrorMessages] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) +B = TypedDict('B', {'x': int, 'z': str, 'a': int}) +C = TypedDict('C', {'x': int, 'z': str, 'a': str}) +a: A +b: B +c: C + +def f(a: A) -> None: pass + +l = [a, b] # Join generates an anonymous TypedDict +f(l) # E: Argument 1 to "f" has incompatible type "List[TypedDict({'x': int})]"; expected "A" +ll = [b, c] +f(ll) # E: Argument 1 to "f" has incompatible type "List[TypedDict({'x': int, 'z': str})]"; expected "A" +[builtins fixtures/dict.pyi] + +[case testTypedDictWithSimpleProtocol] +from typing_extensions import Protocol, TypedDict + +class StrObjectMap(Protocol): + def __getitem__(self, key: str) -> object: ... +class StrIntMap(Protocol): + def __getitem__(self, key: str) -> int: ... + +A = TypedDict('A', {'x': int, 'y': int}) +B = TypedDict('B', {'x': int, 'y': str}) + +def fun(arg: StrObjectMap) -> None: ... +def fun2(arg: StrIntMap) -> None: ... +a: A +b: B +fun(a) +fun(b) +fun2(a) # Error +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +main:17: error: Argument 1 to "fun2" has incompatible type "A"; expected "StrIntMap" +main:17: note: Following member(s) of "A" have conflicts: +main:17: note: Expected: +main:17: note: def __getitem__(self, str, /) -> int +main:17: note: Got: +main:17: note: def __getitem__(self, str, /) -> object + +[case testTypedDictWithSimpleProtocolInference] +from typing_extensions import Protocol, TypedDict +from typing import TypeVar + +T_co = TypeVar('T_co', covariant=True) +T = TypeVar('T') + +class StrMap(Protocol[T_co]): + def __getitem__(self, key: str) -> T_co: ... + +A = TypedDict('A', {'x': int, 'y': int}) +B = TypedDict('B', {'x': int, 'y': str}) + +def fun(arg: StrMap[T]) -> T: + return arg['whatever'] +a: A +b: B +reveal_type(fun(a)) # N: Revealed type is "builtins.object" +reveal_type(fun(b)) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +-- Join + +[case testJoinOfTypedDictHasOnlyCommonKeysAndNewFallback] +from mypy_extensions import TypedDict +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +Point3D = TypedDict('Point3D', {'x': int, 'y': int, 'z': int}) +p1 = TaggedPoint(type='2d', x=0, y=0) +p2 = Point3D(x=1, y=1, z=1) +joined_points = [p1, p2][0] +reveal_type(p1.values()) # N: Revealed type is "typing.Iterable[builtins.object]" +reveal_type(p2.values()) # N: Revealed type is "typing.Iterable[builtins.object]" +reveal_type(joined_points) # N: Revealed type is "TypedDict({'x': builtins.int, 'y': builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testJoinOfTypedDictRemovesNonequivalentKeys] +from mypy_extensions import TypedDict +CellWithInt = TypedDict('CellWithInt', {'value': object, 'meta': int}) +CellWithObject = TypedDict('CellWithObject', {'value': object, 'meta': object}) +c1 = CellWithInt(value=1, meta=42) +c2 = CellWithObject(value=2, meta='turtle doves') +joined_cells = [c1, c2] +reveal_type(c1) # N: Revealed type is "TypedDict('__main__.CellWithInt', {'value': builtins.object, 'meta': builtins.int})" +reveal_type(c2) # N: Revealed type is "TypedDict('__main__.CellWithObject', {'value': builtins.object, 'meta': builtins.object})" +reveal_type(joined_cells) # N: Revealed type is "builtins.list[TypedDict({'value': builtins.object})]" +[builtins fixtures/dict.pyi] + +[case testJoinOfDisjointTypedDictsIsEmptyTypedDict] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +Cell = TypedDict('Cell', {'value': object}) +d1 = Point(x=0, y=0) +d2 = Cell(value='pear tree') +joined_dicts = [d1, d2] +reveal_type(d1) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})" +reveal_type(d2) # N: Revealed type is "TypedDict('__main__.Cell', {'value': builtins.object})" +reveal_type(joined_dicts) # N: Revealed type is "builtins.list[TypedDict({})]" +[builtins fixtures/dict.pyi] + +[case testJoinOfTypedDictWithCompatibleMappingIsMapping] +from mypy_extensions import TypedDict +from typing import Mapping +Cell = TypedDict('Cell', {'value': int}) +left = Cell(value=42) +right = {'score': 999} # type: Mapping[str, int] +joined1 = [left, right] +joined2 = [right, left] +reveal_type(joined1) # N: Revealed type is "builtins.list[typing.Mapping[builtins.str, builtins.object]]" +reveal_type(joined2) # N: Revealed type is "builtins.list[typing.Mapping[builtins.str, builtins.object]]" +[builtins fixtures/dict.pyi] + +[case testJoinOfTypedDictWithCompatibleMappingSupertypeIsSupertype] +from mypy_extensions import TypedDict +from typing import Sized +Cell = TypedDict('Cell', {'value': int}) +left = Cell(value=42) +right = {'score': 999} # type: Sized +joined1 = [left, right] +joined2 = [right, left] +reveal_type(joined1) # N: Revealed type is "builtins.list[typing.Sized]" +reveal_type(joined2) # N: Revealed type is "builtins.list[typing.Sized]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testJoinOfTypedDictWithIncompatibleTypeIsObject] +from mypy_extensions import TypedDict +from typing import Mapping +Cell = TypedDict('Cell', {'value': int}) +left = Cell(value=42) +right = 42 +joined1 = [left, right] +joined2 = [right, left] +reveal_type(joined1) # N: Revealed type is "builtins.list[builtins.object]" +reveal_type(joined2) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/dict.pyi] + + +-- Meet + +[case testMeetOfTypedDictsWithCompatibleCommonKeysHasAllKeysAndNewFallback] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XY = TypedDict('XY', {'x': int, 'y': int}) +YZ = TypedDict('YZ', {'y': int, 'z': int}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypedDict({'x': builtins.int, 'y': builtins.int, 'z': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictsWithIncompatibleCommonKeysIsUninhabited] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XYa = TypedDict('XYa', {'x': int, 'y': int}) +YbZ = TypedDict('YbZ', {'y': object, 'z': int}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XYa, y: YbZ) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Never" +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictsWithNoCommonKeysHasAllKeysAndNewFallback] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +X = TypedDict('X', {'x': int}) +Z = TypedDict('Z', {'z': int}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: X, y: Z) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypedDict({'x': builtins.int, 'z': builtins.int})" +[builtins fixtures/dict.pyi] + +# TODO: It would be more accurate for the meet to be TypedDict instead. +[case testMeetOfTypedDictWithCompatibleMappingIsUninhabitedForNow] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable, Mapping +X = TypedDict('X', {'x': int}) +M = Mapping[str, int] +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: X, y: M) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Never" +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictWithIncompatibleMappingIsUninhabited] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable, Mapping +X = TypedDict('X', {'x': int}) +M = Mapping[str, str] +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: X, y: M) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Never" +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictWithCompatibleMappingSuperclassIsUninhabitedForNow] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable, Iterable +X = TypedDict('X', {'x': int}) +I = Iterable[str] +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: X, y: I) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypedDict('__main__.X', {'x': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictsWithNonTotal] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XY = TypedDict('XY', {'x': int, 'y': int}, total=False) +YZ = TypedDict('YZ', {'y': int, 'z': int}, total=False) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypedDict({'x'?: builtins.int, 'y'?: builtins.int, 'z'?: builtins.int})" +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictsWithNonTotalAndTotal] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XY = TypedDict('XY', {'x': int}, total=False) +YZ = TypedDict('YZ', {'y': int, 'z': int}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypedDict({'x'?: builtins.int, 'y': builtins.int, 'z': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictsWithIncompatibleNonTotalAndTotal] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XY = TypedDict('XY', {'x': int, 'y': int}, total=False) +YZ = TypedDict('YZ', {'y': int, 'z': int}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # N: Revealed type is "Never" +[builtins fixtures/dict.pyi] + + +-- Constraint Solver + +[case testTypedDictConstraintsAgainstIterable] +from typing import TypeVar, Iterable +from mypy_extensions import TypedDict +T = TypeVar('T') +def f(x: Iterable[T]) -> T: pass +A = TypedDict('A', {'x': int}) +a: A +reveal_type(f(a)) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +-- TODO: Figure out some way to trigger the ConstraintBuilderVisitor.visit_typeddict_type() path. + + +-- Special Method: __getitem__ + +[case testCanGetItemOfTypedDictWithValidStringLiteralKey] +from mypy_extensions import TypedDict +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p = TaggedPoint(type='2d', x=42, y=1337) +reveal_type(p['type']) # N: Revealed type is "builtins.str" +reveal_type(p['x']) # N: Revealed type is "builtins.int" +reveal_type(p['y']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testCannotGetItemOfTypedDictWithInvalidStringLiteralKey] +from mypy_extensions import TypedDict +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p: TaggedPoint +p['typ'] # E: TypedDict "TaggedPoint" has no key "typ" \ + # N: Did you mean "type"? +[builtins fixtures/dict.pyi] + +[case testCannotGetItemOfAnonymousTypedDictWithInvalidStringLiteralKey] +from typing import TypeVar +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': str, 'y': int, 'z': str}) +B = TypedDict('B', {'x': str, 'z': int}) +C = TypedDict('C', {'x': str, 'y': int, 'z': int}) +T = TypeVar('T') +def join(x: T, y: T) -> T: return x +ab = join(A(x='', y=1, z=''), B(x='', z=1)) +ac = join(A(x='', y=1, z=''), C(x='', y=0, z=1)) +ab['y'] # E: "y" is not a valid TypedDict key; expected one of ("x") +ac['a'] # E: "a" is not a valid TypedDict key; expected one of ("x", "y") +[builtins fixtures/dict.pyi] + +[case testCannotGetItemOfTypedDictWithNonLiteralKey] +from mypy_extensions import TypedDict +from typing import Union +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p = TaggedPoint(type='2d', x=42, y=1337) +def get_coordinate(p: TaggedPoint, key: str) -> Union[str, int]: + return p[key] # E: TypedDict key must be a string literal; expected one of ("type", "x", "y") +[builtins fixtures/dict.pyi] + + +-- Special Method: __setitem__ + +[case testCanSetItemOfTypedDictWithValidStringLiteralKeyAndCompatibleValueType] +from mypy_extensions import TypedDict +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p = TaggedPoint(type='2d', x=42, y=1337) +p['type'] = 'two_d' +p['x'] = 1 +[builtins fixtures/dict.pyi] + +[case testCannotSetItemOfTypedDictWithIncompatibleValueType] +from mypy_extensions import TypedDict +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p = TaggedPoint(type='2d', x=42, y=1337) +p['x'] = 'y' # E: Value of "x" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCannotSetItemOfTypedDictWithInvalidStringLiteralKey] +from mypy_extensions import TypedDict +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p = TaggedPoint(type='2d', x=42, y=1337) +p['z'] = 1 # E: TypedDict "TaggedPoint" has no key "z" +[builtins fixtures/dict.pyi] + +[case testCannotSetItemOfTypedDictWithNonLiteralKey] +from mypy_extensions import TypedDict +from typing import Union +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p = TaggedPoint(type='2d', x=42, y=1337) +def set_coordinate(p: TaggedPoint, key: str, value: int) -> None: + p[key] = value # E: TypedDict key must be a string literal; expected one of ("type", "x", "y") +[builtins fixtures/dict.pyi] + + +-- isinstance + +[case testTypedDictWithIsInstanceAndIsSubclass] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int}) +d: object +if isinstance(d, D): # E: Cannot use isinstance() with TypedDict type + reveal_type(d) # N: Revealed type is "TypedDict('__main__.D', {'x': builtins.int})" +issubclass(object, D) # E: Cannot use issubclass() with TypedDict type +[builtins fixtures/isinstancelist.pyi] + + +-- Scoping + +[case testTypedDictInClassNamespace] +# https://github.com/python/mypy/pull/2553#issuecomment-266474341 +from mypy_extensions import TypedDict +class C: + def f(self): + A = TypedDict('A', {'x': int}) + def g(self): + A = TypedDict('A', {'y': int}) +C.A # E: "Type[C]" has no attribute "A" +[builtins fixtures/dict.pyi] + +[case testTypedDictInFunction] +from mypy_extensions import TypedDict +def f() -> None: + A = TypedDict('A', {'x': int}) +A # E: Name "A" is not defined +[builtins fixtures/dict.pyi] + + +-- Union simplification / proper subtype checks + +[case testTypedDictUnionSimplification] +from typing import TypeVar, Union, Any, cast +from mypy_extensions import TypedDict + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +C = TypedDict('C', {'a': int}) +D = TypedDict('D', {'a': int, 'b': int}) +E = TypedDict('E', {'a': str}) +F = TypedDict('F', {'x': int}) +G = TypedDict('G', {'a': Any}) + +c = C(a=1) +d = D(a=1, b=1) +e = E(a='') +f = F(x=1) +g = G(a=cast(Any, 1)) # Work around #2610 + +reveal_type(u(d, d)) # N: Revealed type is "TypedDict('__main__.D', {'a': builtins.int, 'b': builtins.int})" +reveal_type(u(c, d)) # N: Revealed type is "TypedDict('__main__.C', {'a': builtins.int})" +reveal_type(u(d, c)) # N: Revealed type is "TypedDict('__main__.C', {'a': builtins.int})" +reveal_type(u(c, e)) # N: Revealed type is "Union[TypedDict('__main__.E', {'a': builtins.str}), TypedDict('__main__.C', {'a': builtins.int})]" +reveal_type(u(e, c)) # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), TypedDict('__main__.E', {'a': builtins.str})]" +reveal_type(u(c, f)) # N: Revealed type is "Union[TypedDict('__main__.F', {'x': builtins.int}), TypedDict('__main__.C', {'a': builtins.int})]" +reveal_type(u(f, c)) # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), TypedDict('__main__.F', {'x': builtins.int})]" +reveal_type(u(c, g)) # N: Revealed type is "Union[TypedDict('__main__.G', {'a': Any}), TypedDict('__main__.C', {'a': builtins.int})]" +reveal_type(u(g, c)) # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), TypedDict('__main__.G', {'a': Any})]" +[builtins fixtures/dict.pyi] + +[case testTypedDictUnionSimplification2] +from typing import TypeVar, Union, Mapping, Any +from mypy_extensions import TypedDict + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +C = TypedDict('C', {'a': int, 'b': int}) + +c = C(a=1, b=1) +m_s_o: Mapping[str, object] +m_s_s: Mapping[str, str] +m_i_i: Mapping[int, int] +m_s_a: Mapping[str, Any] + +reveal_type(u(c, m_s_o)) # N: Revealed type is "typing.Mapping[builtins.str, builtins.object]" +reveal_type(u(m_s_o, c)) # N: Revealed type is "typing.Mapping[builtins.str, builtins.object]" +reveal_type(u(c, m_s_s)) # N: Revealed type is "Union[typing.Mapping[builtins.str, builtins.str], TypedDict('__main__.C', {'a': builtins.int, 'b': builtins.int})]" +reveal_type(u(c, m_i_i)) # N: Revealed type is "Union[typing.Mapping[builtins.int, builtins.int], TypedDict('__main__.C', {'a': builtins.int, 'b': builtins.int})]" +reveal_type(u(c, m_s_a)) # N: Revealed type is "Union[typing.Mapping[builtins.str, Any], TypedDict('__main__.C', {'a': builtins.int, 'b': builtins.int})]" +[builtins fixtures/dict.pyi] + +[case testTypedDictUnionUnambiguousCase] +from typing import Union, Mapping, Any, cast +from typing_extensions import TypedDict, Literal + +A = TypedDict('A', {'@type': Literal['a-type'], 'a': str}) +B = TypedDict('B', {'@type': Literal['b-type'], 'b': int}) + +c: Union[A, B] = {'@type': 'a-type', 'a': 'Test'} +reveal_type(c) # N: Revealed type is "Union[TypedDict('__main__.A', {'@type': Literal['a-type'], 'a': builtins.str}), TypedDict('__main__.B', {'@type': Literal['b-type'], 'b': builtins.int})]" +[builtins fixtures/dict.pyi] + +[case testTypedDictUnionAmbiguousCaseBothMatch] +from typing import Union, Mapping, Any, cast +from typing_extensions import TypedDict, Literal + +A = TypedDict('A', {'@type': Literal['a-type'], 'value': str}) +B = TypedDict('B', {'@type': Literal['b-type'], 'value': str}) + +c: Union[A, B] = {'@type': 'a-type', 'value': 'Test'} +[builtins fixtures/dict.pyi] + +[case testTypedDictUnionAmbiguousCaseNoMatch] +from typing import Union, Mapping, Any, cast +from typing_extensions import TypedDict, Literal + +A = TypedDict('A', {'@type': Literal['a-type'], 'value': int}) +B = TypedDict('B', {'@type': Literal['b-type'], 'value': int}) + +c: Union[A, B] = {'@type': 'a-type', 'value': 'Test'} # E: Type of TypedDict is ambiguous, none of ("A", "B") matches cleanly \ + # E: Incompatible types in assignment (expression has type "Dict[str, str]", variable has type "Union[A, B]") +[builtins fixtures/dict.pyi] + +-- Use dict literals + +[case testTypedDictDictLiterals] +from mypy_extensions import TypedDict + +Point = TypedDict('Point', {'x': int, 'y': int}) + +def f(p: Point) -> None: + if int(): + p = {'x': 2, 'y': 3} + p = {'x': 2} # E: Missing key "y" for TypedDict "Point" + p = dict(x=2, y=3) + +f({'x': 1, 'y': 3}) +f({'x': 1, 'y': 'z'}) # E: Incompatible types (expression has type "str", TypedDict item "y" has type "int") + +f(dict(x=1, y=3)) +f(dict(x=1, y=3, z=4)) # E: Extra key "z" for TypedDict "Point" +f(dict(x=1, y=3, z=4, a=5)) # E: Extra keys ("z", "a") for TypedDict "Point" + +[builtins fixtures/dict.pyi] + +[case testTypedDictExplicitTypes] +from mypy_extensions import TypedDict + +Point = TypedDict('Point', {'x': int, 'y': int}) + +p1a: Point = {'x': 'hi'} # E: Missing key "y" for TypedDict "Point" +p1b: Point = {} # E: Missing keys ("x", "y") for TypedDict "Point" + +p2: Point +p2 = dict(x='bye') # E: Missing key "y" for TypedDict "Point" + +p3 = Point(x=1, y=2) +if int(): + p3 = {'x': 'hi'} # E: Missing key "y" for TypedDict "Point" + +p4: Point = {'x': 1, 'y': 2} + +[builtins fixtures/dict.pyi] + +[case testCannotCreateAnonymousTypedDictInstanceUsingDictLiteralWithExtraItems] +from mypy_extensions import TypedDict +from typing import TypeVar +A = TypedDict('A', {'x': int, 'y': int}) +B = TypedDict('B', {'x': int, 'y': str}) +T = TypeVar('T') +def join(x: T, y: T) -> T: return x +ab = join(A(x=1, y=1), B(x=1, y='')) +if int(): + ab = {'x': 1, 'z': 1} # E: Expected TypedDict key "x" but found keys ("x", "z") +[builtins fixtures/dict.pyi] + +[case testCannotCreateAnonymousTypedDictInstanceUsingDictLiteralWithMissingItems] +from mypy_extensions import TypedDict +from typing import TypeVar +A = TypedDict('A', {'x': int, 'y': int, 'z': int}) +B = TypedDict('B', {'x': int, 'y': int, 'z': str}) +T = TypeVar('T') +def join(x: T, y: T) -> T: return x +ab = join(A(x=1, y=1, z=1), B(x=1, y=1, z='')) +if int(): + ab = {} # E: Expected TypedDict keys ("x", "y") but found no keys +[builtins fixtures/dict.pyi] + + +-- Other TypedDict methods + +[case testTypedDictGetMethod] +from mypy_extensions import TypedDict +class A: pass +D = TypedDict('D', {'x': int, 'y': str}) +d: D +reveal_type(d.get('x')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get('y')) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(d.get('x', A())) # N: Revealed type is "Union[builtins.int, __main__.A]" +reveal_type(d.get('x', 1)) # N: Revealed type is "builtins.int" +reveal_type(d.get('y', None)) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictGetMethodTypeContext] +from typing import List +from mypy_extensions import TypedDict +class A: pass +D = TypedDict('D', {'x': List[int], 'y': int}) +d: D +reveal_type(d.get('x', [])) # N: Revealed type is "builtins.list[builtins.int]" +d.get('x', ['x']) # E: List item 0 has incompatible type "str"; expected "int" +a = [''] +reveal_type(d.get('x', a)) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictGetMethodInvalidArgs] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}) +d: D +d.get() # E: All overload variants of "get" of "Mapping" require at least one argument \ + # N: Possible overload variants: \ + # N: def get(self, k: str) -> object \ + # N: def [V] get(self, k: str, default: object) -> object +d.get('x', 1, 2) # E: No overload variant of "get" of "Mapping" matches argument types "str", "int", "int" \ + # N: Possible overload variants: \ + # N: def get(self, k: str) -> object \ + # N: def [V] get(self, k: str, default: Union[int, V]) -> object +x = d.get('z') +reveal_type(x) # N: Revealed type is "builtins.object" +s = '' +y = d.get(s) +reveal_type(y) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictMissingMethod] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}) +d: D +d.bad(1) # E: "D" has no attribute "bad" +[builtins fixtures/dict.pyi] + +[case testTypedDictChainedGetMethodWithDictFallback] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}) +E = TypedDict('E', {'d': D}) +p = E(d=D(x=0, y='')) +reveal_type(p.get('d', {'x': 1, 'y': ''})) # N: Revealed type is "TypedDict('__main__.D', {'x': builtins.int, 'y': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictGetDefaultParameterStillTypeChecked] +from mypy_extensions import TypedDict +TaggedPoint = TypedDict('TaggedPoint', {'type': str, 'x': int, 'y': int}) +p = TaggedPoint(type='2d', x=42, y=1337) +p.get('x', 1 + 'y') # E: Unsupported operand types for + ("int" and "str") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictChainedGetWithEmptyDictDefault] +from mypy_extensions import TypedDict +C = TypedDict('C', {'a': int}) +D = TypedDict('D', {'x': C, 'y': str}) +d: D +reveal_type(d.get('x', {})) \ + # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', None)) \ + # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), None]" +reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +-- Totality (the "total" keyword argument) + +[case testTypedDictWithTotalTrue] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=True) +d: D +reveal_type(d) \ + # N: Revealed type is "TypedDict('__main__.D', {'x': builtins.int, 'y': builtins.str})" +[builtins fixtures/dict.pyi] + +[case testTypedDictWithInvalidTotalArgument] +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': int}, total=0) # E: "total" argument must be a True or False literal +B = TypedDict('B', {'x': int}, total=bool) # E: "total" argument must be a True or False literal +C = TypedDict('C', {'x': int}, x=False) # E: Unexpected keyword argument "x" for "TypedDict" +D = TypedDict('D', {'x': int}, False) # E: Unexpected arguments to TypedDict() +[builtins fixtures/dict.pyi] + +[case testTypedDictWithTotalFalse] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=False) +def f(d: D) -> None: + reveal_type(d) # N: Revealed type is "TypedDict('__main__.D', {'x'?: builtins.int, 'y'?: builtins.str})" +f({}) +f({'x': 1}) +f({'y': ''}) +f({'x': 1, 'y': ''}) +f({'x': 1, 'z': ''}) # E: Extra key "z" for TypedDict "D" +f({'x': ''}) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +[builtins fixtures/dict.pyi] + +[case testTypedDictConstructorWithTotalFalse] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=False) +def f(d: D) -> None: pass +reveal_type(D()) # N: Revealed type is "TypedDict('__main__.D', {'x'?: builtins.int, 'y'?: builtins.str})" +reveal_type(D(x=1)) # N: Revealed type is "TypedDict('__main__.D', {'x'?: builtins.int, 'y'?: builtins.str})" +f(D(y='')) +f(D(x=1, y='')) +f(D(x=1, z='')) # E: Extra key "z" for TypedDict "D" +f(D(x='')) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +[builtins fixtures/dict.pyi] + +[case testTypedDictIndexingWithNonRequiredKey] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=False) +d: D +reveal_type(d['x']) # N: Revealed type is "builtins.int" +reveal_type(d['y']) # N: Revealed type is "builtins.str" +reveal_type(d.get('x')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get('y')) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictSubtypingWithTotalFalse] +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'x': int}, total=False) +C = TypedDict('C', {'x': int, 'y': str}, total=False) +def fa(a: A) -> None: pass +def fb(b: B) -> None: pass +def fc(c: C) -> None: pass +a: A +b: B +c: C +fb(b) +fc(c) +fb(c) +fb(a) # E: Argument 1 to "fb" has incompatible type "A"; expected "B" +fa(b) # E: Argument 1 to "fa" has incompatible type "B"; expected "A" +fc(b) # E: Argument 1 to "fc" has incompatible type "B"; expected "C" +[builtins fixtures/dict.pyi] + +[case testTypedDictJoinWithTotalFalse] +from typing import TypeVar +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'x': int}, total=False) +C = TypedDict('C', {'x': int, 'y': str}, total=False) +T = TypeVar('T') +def j(x: T, y: T) -> T: return x +a: A +b: B +c: C +reveal_type(j(a, b)) \ + # N: Revealed type is "TypedDict({})" +reveal_type(j(b, b)) \ + # N: Revealed type is "TypedDict({'x'?: builtins.int})" +reveal_type(j(c, c)) \ + # N: Revealed type is "TypedDict({'x'?: builtins.int, 'y'?: builtins.str})" +reveal_type(j(b, c)) \ + # N: Revealed type is "TypedDict({'x'?: builtins.int})" +reveal_type(j(c, b)) \ + # N: Revealed type is "TypedDict({'x'?: builtins.int})" +[builtins fixtures/dict.pyi] + +[case testTypedDictClassWithTotalArgument] +from mypy_extensions import TypedDict +class D(TypedDict, total=False): + x: int + y: str +d: D +reveal_type(d) # N: Revealed type is "TypedDict('__main__.D', {'x'?: builtins.int, 'y'?: builtins.str})" +[builtins fixtures/dict.pyi] + +[case testTypedDictClassWithInvalidTotalArgument] +from mypy_extensions import TypedDict +class D(TypedDict, total=1): # E: "total" argument must be a True or False literal + x: int +class E(TypedDict, total=bool): # E: "total" argument must be a True or False literal + x: int +class F(TypedDict, total=xyz): # E: Name "xyz" is not defined \ + # E: "total" argument must be a True or False literal + x: int +[builtins fixtures/dict.pyi] + +[case testTypedDictClassInheritanceWithTotalArgument] +from mypy_extensions import TypedDict +class A(TypedDict): + x: int +class B(TypedDict, A, total=False): + y: int +class C(TypedDict, B, total=True): + z: str +c: C +reveal_type(c) # N: Revealed type is "TypedDict('__main__.C', {'x': builtins.int, 'y'?: builtins.int, 'z': builtins.str})" +[builtins fixtures/dict.pyi] + +[case testNonTotalTypedDictInErrorMessages] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}, total=False) +B = TypedDict('B', {'x': int, 'z': str, 'a': int}, total=False) +C = TypedDict('C', {'x': int, 'z': str, 'a': str}, total=False) +a: A +b: B +c: C + +def f(a: A) -> None: pass + +l = [a, b] # Join generates an anonymous TypedDict +f(l) # E: Argument 1 to "f" has incompatible type "List[TypedDict({'x'?: int})]"; expected "A" +ll = [b, c] +f(ll) # E: Argument 1 to "f" has incompatible type "List[TypedDict({'x'?: int, 'z'?: str})]"; expected "A" +[builtins fixtures/dict.pyi] + +[case testNonTotalTypedDictCanBeEmpty] +# flags: --warn-unreachable +from mypy_extensions import TypedDict + +class A(TypedDict): + ... + +class B(TypedDict, total=False): + x: int + +a: A = {} +b: B = {} + +if not a: + reveal_type(a) # N: Revealed type is "TypedDict('__main__.A', {})" + +if not b: + reveal_type(b) # N: Revealed type is "TypedDict('__main__.B', {'x'?: builtins.int})" +[builtins fixtures/dict.pyi] + +-- Create Type (Errors) + +[case testCannotCreateTypedDictTypeWithTooFewArguments] +from mypy_extensions import TypedDict +Point = TypedDict('Point') # E: Too few arguments for TypedDict() +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithTooManyArguments] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}, dict) # E: Unexpected arguments to TypedDict() +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidName] +from mypy_extensions import TypedDict +Point = TypedDict(dict, {'x': int, 'y': int}) # E: TypedDict() expects a string literal as the first argument +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidItems] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x'}) # E: TypedDict() expects a dictionary literal as the second argument +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithKwargs] +from mypy_extensions import TypedDict +d = {'x': int, 'y': int} +Point = TypedDict('Point', {**d}) # E: Invalid TypedDict() field name +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithBytes] +from mypy_extensions import TypedDict +Point = TypedDict(b'Point', {'x': int, 'y': int}) # E: TypedDict() expects a string literal as the first argument +# This technically works at runtime but doesn't make sense. +Point2 = TypedDict('Point2', {b'x': int}) # E: Invalid TypedDict() field name +[builtins fixtures/dict.pyi] + +-- NOTE: The following code works at runtime but is not yet supported by mypy. +-- Keyword arguments may potentially be supported in the future. +[case testCannotCreateTypedDictTypeWithNonpositionalArgs] +from mypy_extensions import TypedDict +Point = TypedDict(typename='Point', fields={'x': int, 'y': int}) # E: Unexpected arguments to TypedDict() +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidItemName] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {int: int, int: int}) # E: Invalid TypedDict() field name +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidItemType] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': 1, 'y': 1}) # E: Invalid type: try using Literal[1] instead? +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidName2] +from mypy_extensions import TypedDict +X = TypedDict('Y', {'x': int}) # E: First argument "Y" to TypedDict() does not match variable name "X" +[builtins fixtures/dict.pyi] + + +-- Overloading + +[case testTypedDictOverloading] +from typing import overload, Iterable +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int}) + +@overload +def f(x: Iterable[str]) -> str: ... +@overload +def f(x: int) -> int: ... +def f(x): pass + +a: A +reveal_type(f(a)) # N: Revealed type is "builtins.str" +reveal_type(f(1)) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverloading2] +from typing import overload, Iterable +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int}) + +@overload +def f(x: Iterable[int]) -> None: ... +@overload +def f(x: int) -> None: ... +def f(x): pass + +a: A +f(a) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +main:13: error: Argument 1 to "f" has incompatible type "A"; expected "Iterable[int]" +main:13: note: Following member(s) of "A" have conflicts: +main:13: note: Expected: +main:13: note: def __iter__(self) -> Iterator[int] +main:13: note: Got: +main:13: note: def __iter__(self) -> Iterator[str] + +[case testTypedDictOverloading3] +from typing import overload +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int}) + +@overload +def f(x: str) -> None: ... +@overload +def f(x: int) -> None: ... +def f(x): pass + +a: A +f(a) # E: No overload variant of "f" matches argument type "A" \ + # N: Possible overload variants: \ + # N: def f(x: str) -> None \ + # N: def f(x: int) -> None +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverloading4] +from typing import overload +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'x': str}) + +@overload +def f(x: A) -> int: ... +@overload +def f(x: int) -> str: ... +def f(x): pass + +a: A +b: B +reveal_type(f(a)) # N: Revealed type is "builtins.int" +reveal_type(f(1)) # N: Revealed type is "builtins.str" +f(b) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverloading5] +from typing import overload +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'y': str}) +C = TypedDict('C', {'y': int}) + +@overload +def f(x: A) -> None: ... +@overload +def f(x: B) -> None: ... +def f(x): pass + +a: A +b: B +c: C +f(a) +f(b) +f(c) # E: Argument 1 to "f" has incompatible type "C"; expected "A" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverloading6] +from typing import overload +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'y': str}) + +@overload +def f(x: A) -> int: ... +@overload +def f(x: B) -> str: ... +def f(x): pass + +a: A +b: B +reveal_type(f(a)) # N: Revealed type is "builtins.int" +reveal_type(f(b)) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +-- Special cases + +[case testForwardReferenceInTypedDict] +from typing import Mapping +from mypy_extensions import TypedDict +X = TypedDict('X', {'b': 'B', 'c': 'C'}) +class B: pass +class C(B): pass +x: X +reveal_type(x) # N: Revealed type is "TypedDict('__main__.X', {'b': __main__.B, 'c': __main__.C})" +m1: Mapping[str, object] = x +m2: Mapping[str, B] = x # E: Incompatible types in assignment (expression has type "X", variable has type "Mapping[str, B]") +[builtins fixtures/dict.pyi] + +[case testForwardReferenceInClassTypedDict] +from typing import Mapping +from mypy_extensions import TypedDict +class X(TypedDict): + b: 'B' + c: 'C' +class B: pass +class C(B): pass +x: X +reveal_type(x) # N: Revealed type is "TypedDict('__main__.X', {'b': __main__.B, 'c': __main__.C})" +m1: Mapping[str, object] = x +m2: Mapping[str, B] = x # E: Incompatible types in assignment (expression has type "X", variable has type "Mapping[str, B]") +[builtins fixtures/dict.pyi] + +[case testForwardReferenceToTypedDictInTypedDict] +from typing import Mapping +from mypy_extensions import TypedDict +X = TypedDict('X', {'a': 'A'}) +A = TypedDict('A', {'b': int}) +x: X +reveal_type(x) # N: Revealed type is "TypedDict('__main__.X', {'a': TypedDict('__main__.A', {'b': builtins.int})})" +reveal_type(x['a']['b']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testSelfRecursiveTypedDictInheriting] +from mypy_extensions import TypedDict + +def test() -> None: + class MovieBase(TypedDict): + name: str + year: int + + class Movie(MovieBase): + director: 'Movie' # E: Cannot resolve name "Movie" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + m: Movie + reveal_type(m['director']['name']) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] + +[case testSubclassOfRecursiveTypedDict] +from typing import List +from mypy_extensions import TypedDict + +def test() -> None: + class Command(TypedDict): + subcommands: List['Command'] # E: Cannot resolve name "Command" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + + class HelpCommand(Command): + pass + + hc = HelpCommand(subcommands=[]) + reveal_type(hc) # N: Revealed type is "TypedDict('__main__.HelpCommand@8', {'subcommands': builtins.list[Any]})" +[builtins fixtures/list.pyi] +[out] + +[case testTypedDictForwardAsUpperBound] +from typing import TypeVar, Generic +from mypy_extensions import TypedDict +T = TypeVar('T', bound='M') +class G(Generic[T]): + x: T + +yb: G[int] # E: Type argument "int" of "G" must be a subtype of "M" +yg: G[M] +z: int = G[M]().x['x'] # type: ignore[used-before-def] + +class M(TypedDict): + x: int +[builtins fixtures/dict.pyi] +[out] + +[case testTypedDictWithImportCycleForward] +import a +[file a.py] +from mypy_extensions import TypedDict +from b import f + +N = TypedDict('N', {'a': str}) +[file b.py] +import a + +def f(x: a.N) -> None: + reveal_type(x) + reveal_type(x['a']) +[builtins fixtures/dict.pyi] +[out] +tmp/b.py:4: note: Revealed type is "TypedDict('a.N', {'a': builtins.str})" +tmp/b.py:5: note: Revealed type is "builtins.str" + +[case testTypedDictImportCycle] + +import b +[file a.py] +class C: + pass + +from b import tp +x: tp +reveal_type(x['x']) # N: Revealed type is "builtins.int" + +reveal_type(tp) # N: Revealed type is "def (*, x: builtins.int) -> TypedDict('b.tp', {'x': builtins.int})" +tp(x='no') # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") + +[file b.py] +from a import C +from mypy_extensions import TypedDict + +tp = TypedDict('tp', {'x': int}) +[builtins fixtures/dict.pyi] +[out] + +[case testTypedDictAsStarStarArg] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) +class B: pass + +def f1(x: int, y: str) -> None: ... +def f2(x: int, y: int) -> None: ... +def f3(x: B, y: str) -> None: ... +def f4(x: int) -> None: pass +def f5(x: int, y: str, z: int) -> None: pass +def f6(x: int, z: str) -> None: pass + +a: A +f1(**a) +f2(**a) # E: Argument "y" to "f2" has incompatible type "str"; expected "int" +f3(**a) # E: Argument "x" to "f3" has incompatible type "int"; expected "B" +f4(**a) # E: Extra argument "y" from **args for "f4" +f5(**a) # E: Missing positional arguments "y", "z" in call to "f5" +f6(**a) # E: Extra argument "y" from **args for "f6" +f1(1, **a) # E: "f1" gets multiple values for keyword argument "x" +[builtins fixtures/tuple.pyi] + +[case testTypedDictAsStarStarArgConstraints] +from typing import TypeVar, Union +from mypy_extensions import TypedDict + +T = TypeVar('T') +S = TypeVar('S') +def f1(x: T, y: S) -> Union[T, S]: ... + +A = TypedDict('A', {'y': int, 'x': str}) +a: A +reveal_type(f1(**a)) # N: Revealed type is "Union[builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypedDictAsStarStarArgCalleeKwargs] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) +B = TypedDict('B', {'x': str, 'y': str}) + +def f(**kwargs: str) -> None: ... +def g(x: int, **kwargs: str) -> None: ... + +a: A +b: B +f(**a) # E: Argument 1 to "f" has incompatible type "**A"; expected "str" +f(**b) +g(**a) +g(**b) # E: Argument "x" to "g" has incompatible type "str"; expected "int" +g(1, **a) # E: "g" gets multiple values for keyword argument "x" +g(1, **b) # E: "g" gets multiple values for keyword argument "x" \ + # E: Argument "x" to "g" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testTypedDictAsStarStarTwice] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) +B = TypedDict('B', {'z': bytes}) +C = TypedDict('C', {'x': str, 'z': bytes}) + +def f1(x: int, y: str, z: bytes) -> None: ... +def f2(x: int, y: float, z: bytes) -> None: ... +def f3(x: int, y: str, z: float) -> None: ... + +a: A +b: B +c: C +f1(**a, **b) +f1(**b, **a) +f2(**a, **b) # E: Argument "y" to "f2" has incompatible type "str"; expected "float" +f3(**a, **b) # E: Argument "z" to "f3" has incompatible type "bytes"; expected "float" +f3(**b, **a) # E: Argument "z" to "f3" has incompatible type "bytes"; expected "float" +f1(**a, **c) # E: "f1" gets multiple values for keyword argument "x" \ + # E: Argument "x" to "f1" has incompatible type "str"; expected "int" +f1(**c, **a) # E: "f1" gets multiple values for keyword argument "x" \ + # E: Argument "x" to "f1" has incompatible type "str"; expected "int" +[builtins fixtures/tuple.pyi] + +[case testTypedDictAsStarStarAndDictAsStarStar] +from mypy_extensions import TypedDict +from typing import Any, Dict + +TD = TypedDict('TD', {'x': int, 'y': str}) + +def f1(x: int, y: str, z: bytes) -> None: ... +def f2(x: int, y: str) -> None: ... + +td: TD +d: Dict[Any, Any] + +f1(**td, **d) +f1(**d, **td) +f2(**td, **d) +f2(**d, **td) +[builtins fixtures/dict.pyi] + +[case testTypedDictNonMappingMethods] +from typing import List +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': List[int]}) +a: A + +reveal_type(a.copy()) # N: Revealed type is "TypedDict('__main__.A', {'x': builtins.int, 'y': builtins.list[builtins.int]})" +a.has_key('x') # E: "A" has no attribute "has_key" +# TODO: Better error message +a.clear() # E: "A" has no attribute "clear" + +a.setdefault('invalid', 1) # E: TypedDict "A" has no key "invalid" +reveal_type(a.setdefault('x', 1)) # N: Revealed type is "builtins.int" +reveal_type(a.setdefault('y', [])) # N: Revealed type is "builtins.list[builtins.int]" +a.setdefault('y', '') # E: Argument 2 to "setdefault" of "TypedDict" has incompatible type "str"; expected "List[int]" +x = '' +a.setdefault(x, 1) # E: Expected TypedDict key to be string literal +alias = a.setdefault +alias(x, 1) # E: Argument 1 has incompatible type "str"; expected "Never" + +a.update({}) +a.update({'x': 1}) +a.update({'x': ''}) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +a.update({'x': 1, 'y': []}) +a.update({'x': 1, 'y': [1]}) +a.update({'z': 1}) # E: Unexpected TypedDict key "z" +a.update({'z': 1, 'zz': 1}) # E: Unexpected TypedDict keys ("z", "zz") +a.update({'z': 1, 'x': 1}) # E: Expected TypedDict key "x" but found keys ("z", "x") +d = {'x': 1} +a.update(d) # E: Argument 1 to "update" of "TypedDict" has incompatible type "Dict[str, int]"; expected "TypedDict({'x'?: int, 'y'?: List[int]})" +[builtins fixtures/dict.pyi] + +[case testTypedDictPopMethod] +from typing import List +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': List[int]}, total=False) +B = TypedDict('B', {'x': int}) +a: A +b: B + +reveal_type(a.pop('x')) # N: Revealed type is "builtins.int" +reveal_type(a.pop('y', [])) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(a.pop('x', '')) # N: Revealed type is "Union[builtins.int, Literal['']?]" +reveal_type(a.pop('x', (1, 2))) # N: Revealed type is "Union[builtins.int, Tuple[Literal[1]?, Literal[2]?]]" +a.pop('invalid', '') # E: TypedDict "A" has no key "invalid" +b.pop('x') # E: Key "x" of TypedDict "B" cannot be deleted +x = '' +b.pop(x) # E: Expected TypedDict key to be string literal +pop = b.pop +pop('x') # E: Argument 1 has incompatible type "str"; expected "Never" +pop('invalid') # E: Argument 1 has incompatible type "str"; expected "Never" +[builtins fixtures/dict.pyi] + +[case testTypedDictDel] +from typing import List +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': List[int]}, total=False) +B = TypedDict('B', {'x': int}) +a: A +b: B + +del a['x'] +del a['invalid'] # E: TypedDict "A" has no key "invalid" +del b['x'] # E: Key "x" of TypedDict "B" cannot be deleted +s = '' +del a[s] # E: Expected TypedDict key to be string literal +del b[s] # E: Expected TypedDict key to be string literal +alias = b.__delitem__ +alias('x') +alias(s) +[builtins fixtures/dict.pyi] + +[case testPluginUnionsOfTypedDicts] +from typing import Union +from mypy_extensions import TypedDict + +class TDA(TypedDict): + a: int + b: str + +class TDB(TypedDict): + a: int + b: int + c: int + +td: Union[TDA, TDB] + +reveal_type(td.get('a')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(td.get('b')) # N: Revealed type is "Union[builtins.str, None, builtins.int]" +reveal_type(td.get('c')) # N: Revealed type is "builtins.object" + +reveal_type(td['a']) # N: Revealed type is "builtins.int" +reveal_type(td['b']) # N: Revealed type is "Union[builtins.str, builtins.int]" +reveal_type(td['c']) # N: Revealed type is "Union[Any, builtins.int]" \ + # E: TypedDict "TDA" has no key "c" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testPluginUnionsOfTypedDictsNonTotal] +from typing import Union +from mypy_extensions import TypedDict + +class TDA(TypedDict, total=False): + a: int + b: str + +class TDB(TypedDict, total=False): + a: int + b: int + c: int + +td: Union[TDA, TDB] + +reveal_type(td.pop('a')) # N: Revealed type is "builtins.int" +reveal_type(td.pop('b')) # N: Revealed type is "Union[builtins.str, builtins.int]" +reveal_type(td.pop('c')) # E: TypedDict "TDA" has no key "c" \ + # N: Revealed type is "Union[Any, builtins.int]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCanCreateTypedDictWithTypingExtensions] +from typing_extensions import TypedDict + +class Point(TypedDict): + x: int + y: int + +p = Point(x=42, y=1337) +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testCanCreateTypedDictWithTypingProper] +from typing import TypedDict + +class Point(TypedDict): + x: int + y: int + +p = Point(x=42, y=1337) +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOptionalUpdate] +from typing import Union +from mypy_extensions import TypedDict + +class A(TypedDict): + x: int + +d: A +d.update({'x': 1}) +[builtins fixtures/dict.pyi] + +[case testTypedDictOverlapWithDict] +# mypy: strict-equality +from typing import TypedDict, Dict + +class Config(TypedDict): + a: str + b: str + +x: Dict[str, str] +y: Config + +x == y +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictNonOverlapping] +# mypy: strict-equality +from typing import TypedDict, Dict + +class Config(TypedDict): + a: str + b: int + +x: Dict[str, str] +y: Config + +x == y # E: Non-overlapping equality check (left operand type: "Dict[str, str]", right operand type: "Config") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictNonTotal] +# mypy: strict-equality +from typing import TypedDict, Dict + +class Config(TypedDict, total=False): + a: str + b: int + +x: Dict[str, str] +y: Config + +x == y +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictNonTotalNonOverlapping] +# mypy: strict-equality +from typing import TypedDict, Dict + +class Config(TypedDict, total=False): + a: int + b: int + +x: Dict[str, str] +y: Config + +x == y # E: Non-overlapping equality check (left operand type: "Dict[str, str]", right operand type: "Config") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictEmpty] +# mypy: strict-equality +from typing import TypedDict + +class Config(TypedDict): + a: str + b: str + +x: Config +x == {} # E: Non-overlapping equality check (left operand type: "Config", right operand type: "Dict[Never, Never]") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictNonTotalEmpty] +# mypy: strict-equality +from typing import TypedDict + +class Config(TypedDict, total=False): + a: str + b: str + +x: Config +x == {} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictNonStrKey] +# mypy: strict-equality +from typing import TypedDict, Dict, Union + +class Config(TypedDict): + a: str + b: str + +x: Config +y: Dict[Union[str, int], str] +x == y +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictOverload] +from typing import overload, TypedDict, Dict + +class Map(TypedDict): + x: int + y: str + +@overload +def func(x: Map) -> int: ... +@overload +def func(x: Dict[str, str]) -> str: ... +def func(x): + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictOverloadBad] +from typing import overload, TypedDict, Dict + +class Map(TypedDict, total=False): + x: int + y: str + +@overload +def func(x: Map) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def func(x: Dict[str, str]) -> str: ... +def func(x): + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictOverloadMappingBad] +from typing import overload, TypedDict, Mapping + +class Map(TypedDict, total=False): + x: int + y: str + +@overload +def func(x: Map) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def func(x: Mapping[str, str]) -> str: ... +def func(x): + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictOverlapWithDictOverloadNonStrKey] +from typing import overload, TypedDict, Dict + +class Map(TypedDict): + x: str + y: str + +@overload +def func(x: Map) -> int: ... +@overload +def func(x: Dict[int, str]) -> str: ... +def func(x): + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictIsInstance] +from typing import TypedDict, Union + +class User(TypedDict): + id: int + name: str + +u: Union[str, User] +u2: User + +if isinstance(u, dict): + reveal_type(u) # N: Revealed type is "TypedDict('__main__.User', {'id': builtins.int, 'name': builtins.str})" +else: + reveal_type(u) # N: Revealed type is "builtins.str" + +assert isinstance(u2, dict) +reveal_type(u2) # N: Revealed type is "TypedDict('__main__.User', {'id': builtins.int, 'name': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictIsInstanceABCs] +from typing import TypedDict, Union, Mapping, Iterable + +class User(TypedDict): + id: int + name: str + +u: Union[int, User] +u2: User + +if isinstance(u, Iterable): + reveal_type(u) # N: Revealed type is "TypedDict('__main__.User', {'id': builtins.int, 'name': builtins.str})" +else: + reveal_type(u) # N: Revealed type is "builtins.int" + +assert isinstance(u2, Mapping) +reveal_type(u2) # N: Revealed type is "TypedDict('__main__.User', {'id': builtins.int, 'name': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypedDictLiteralTypeKeyInCreation] +from typing import TypedDict, Final, Literal + +class Value(TypedDict): + num: int + +num: Final = 'num' +v: Value = {num: 5} +v = {num: ''} # E: Incompatible types (expression has type "str", TypedDict item "num" has type "int") + +bad: Final = 2 +v = {bad: 3} # E: Expected TypedDict key to be string literal +union: Literal['num', 'foo'] +v = {union: 2} # E: Expected TypedDict key to be string literal +num2: Literal['num'] +v = {num2: 2} +bad2: Literal['bad'] +v = {bad2: 2} # E: Missing key "num" for TypedDict "Value" \ + # E: Extra key "bad" for TypedDict "Value" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testOperatorContainsNarrowsTypedDicts_unionWithList] +from __future__ import annotations +from typing import assert_type, TypedDict, Union +from typing_extensions import final + +@final +class D(TypedDict): + foo: int + + +d_or_list: D | list[str] + +if 'foo' in d_or_list: + assert_type(d_or_list, Union[D, list[str]]) +elif 'bar' in d_or_list: + assert_type(d_or_list, list[str]) +else: + assert_type(d_or_list, list[str]) + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testOperatorContainsNarrowsTypedDicts_total] +from __future__ import annotations +from typing import assert_type, Literal, TypedDict, TypeVar, Union +from typing_extensions import final + +@final +class D1(TypedDict): + foo: int + + +@final +class D2(TypedDict): + bar: int + + +d: D1 | D2 + +if 'foo' in d: + assert_type(d, D1) +else: + assert_type(d, D2) + +foo_or_bar: Literal['foo', 'bar'] +if foo_or_bar in d: + assert_type(d, Union[D1, D2]) +else: + assert_type(d, Union[D1, D2]) + +foo_or_invalid: Literal['foo', 'invalid'] +if foo_or_invalid in d: + assert_type(d, D1) + # won't narrow 'foo_or_invalid' + assert_type(foo_or_invalid, Literal['foo', 'invalid']) +else: + assert_type(d, Union[D1, D2]) + # won't narrow 'foo_or_invalid' + assert_type(foo_or_invalid, Literal['foo', 'invalid']) + +TD = TypeVar('TD', D1, D2) + +def f(arg: TD) -> None: + value: int + if 'foo' in arg: + assert_type(arg['foo'], int) + else: + assert_type(arg['bar'], int) + + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testOperatorContainsNarrowsTypedDicts_final] +# flags: --warn-unreachable +from __future__ import annotations +from typing import assert_type, TypedDict, Union +from typing_extensions import final + +@final +class DFinal(TypedDict): + foo: int + + +class DNotFinal(TypedDict): + bar: int + + +d_not_final: DNotFinal + +if 'bar' in d_not_final: + assert_type(d_not_final, DNotFinal) +else: + spam = 'ham' # E: Statement is unreachable + +if 'spam' in d_not_final: + assert_type(d_not_final, DNotFinal) +else: + assert_type(d_not_final, DNotFinal) + +d_final: DFinal + +if 'spam' in d_final: + spam = 'ham' # E: Statement is unreachable +else: + assert_type(d_final, DFinal) + +d_union: DFinal | DNotFinal + +if 'foo' in d_union: + assert_type(d_union, Union[DFinal, DNotFinal]) +else: + assert_type(d_union, DNotFinal) + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testOperatorContainsNarrowsTypedDicts_partialThroughTotalFalse] +from __future__ import annotations +from typing import assert_type, Literal, TypedDict, Union +from typing_extensions import final + +@final +class DTotal(TypedDict): + required_key: int + + +@final +class DNotTotal(TypedDict, total=False): + optional_key: int + + +d: DTotal | DNotTotal + +if 'required_key' in d: + assert_type(d, DTotal) +else: + assert_type(d, DNotTotal) + +if 'optional_key' in d: + assert_type(d, DNotTotal) +else: + assert_type(d, Union[DTotal, DNotTotal]) + +key: Literal['optional_key', 'required_key'] +if key in d: + assert_type(d, Union[DTotal, DNotTotal]) +else: + assert_type(d, Union[DTotal, DNotTotal]) + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testOperatorContainsNarrowsTypedDicts_partialThroughNotRequired] +from __future__ import annotations +from typing import assert_type, Required, NotRequired, TypedDict, Union +from typing_extensions import final + +@final +class D1(TypedDict): + required_key: Required[int] + optional_key: NotRequired[int] + + +@final +class D2(TypedDict): + abc: int + xyz: int + + +d: D1 | D2 + +if 'required_key' in d: + assert_type(d, D1) +else: + assert_type(d, D2) + +if 'optional_key' in d: + assert_type(d, D1) +else: + assert_type(d, Union[D1, D2]) + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCannotSubclassFinalTypedDict] +from typing import TypedDict +from typing_extensions import final + +@final +class DummyTypedDict(TypedDict): + int_val: int + float_val: float + str_val: str + +class SubType(DummyTypedDict): # E: Cannot inherit from final class "DummyTypedDict" + pass + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCannotSubclassFinalTypedDictWithForwardDeclarations] +from typing import TypedDict +from typing_extensions import final + +@final +class DummyTypedDict(TypedDict): + forward_declared: "ForwardDeclared" + +class SubType(DummyTypedDict): # E: Cannot inherit from final class "DummyTypedDict" + pass + +class ForwardDeclared: pass + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictTypeNarrowingWithFinalKey] +from typing import Final, Optional, TypedDict + +KEY_NAME: Final = "bar" +class Foo(TypedDict): + bar: Optional[str] + +foo = Foo(bar="hello") +if foo["bar"] is not None: + reveal_type(foo["bar"]) # N: Revealed type is "builtins.str" + reveal_type(foo[KEY_NAME]) # N: Revealed type is "builtins.str" +if foo[KEY_NAME] is not None: + reveal_type(foo["bar"]) # N: Revealed type is "builtins.str" + reveal_type(foo[KEY_NAME]) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictDoubleForwardClass] +from mypy_extensions import TypedDict +from typing import Any, List + +class Foo(TypedDict): + bar: Bar + baz: Bar + +Bar = List[Any] + +foo: Foo +reveal_type(foo['bar']) # N: Revealed type is "builtins.list[Any]" +reveal_type(foo['baz']) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictDoubleForwardFunc] +from mypy_extensions import TypedDict +from typing import Any, List + +Foo = TypedDict('Foo', {'bar': 'Bar', 'baz': 'Bar'}) + +Bar = List[Any] + +foo: Foo +reveal_type(foo['bar']) # N: Revealed type is "builtins.list[Any]" +reveal_type(foo['baz']) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictDoubleForwardMixed] +from mypy_extensions import TypedDict +from typing import Any, List + +Bar = List[Any] + +class Foo(TypedDict): + foo: Toto + bar: Bar + baz: Bar + +Toto = int + +foo: Foo +reveal_type(foo['foo']) # N: Revealed type is "builtins.int" +reveal_type(foo['bar']) # N: Revealed type is "builtins.list[Any]" +reveal_type(foo['baz']) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testAssignTypedDictAsAttribute] +from typing import TypedDict + +class A: + def __init__(self) -> None: + self.b = TypedDict('b', {'x': int, 'y': str}) # E: TypedDict type as attribute is not supported + +reveal_type(A().b) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictAsUpperBoundAndIndexedAssign] +from typing import TypeVar, Generic, TypedDict + + +class BaseDict(TypedDict, total=False): + foo: int + + +_DICT_T = TypeVar('_DICT_T', bound=BaseDict) + + +class SomeGeneric(Generic[_DICT_T]): + def __init__(self, data: _DICT_T) -> None: + self._data: _DICT_T = data + + def set_state(self) -> None: + self._data['foo'] = 1 +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictCreatedWithEmptyDict] +from typing import TypedDict + +class TD(TypedDict, total=False): + foo: int + bar: int + +d: TD = dict() +d2: TD = dict(foo=1) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictBytesKey] +from typing import TypedDict + +class TD(TypedDict): + foo: int + +d: TD = {b'foo': 2} # E: Expected TypedDict key to be string literal +d[b'foo'] = 3 # E: TypedDict key must be a string literal; expected one of ("foo") \ + # E: Argument 1 to "__setitem__" has incompatible type "bytes"; expected "str" +d[b'foo'] # E: TypedDict key must be a string literal; expected one of ("foo") +d[3] # E: TypedDict key must be a string literal; expected one of ("foo") +d[True] # E: TypedDict key must be a string literal; expected one of ("foo") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUppercaseKey] +from mypy_extensions import TypedDict + +Foo = TypedDict('Foo', {'camelCaseKey': str}) +value: Foo = {} # E: Missing key "camelCaseKey" for TypedDict "Foo" +[builtins fixtures/dict.pyi] + +[case testTypedDictWithDeferredFieldTypeEval] +from typing import Generic, TypeVar, TypedDict, NotRequired + +class Foo(TypedDict): + y: NotRequired[int] + x: Outer[Inner[ForceDeferredEval]] + +var: Foo +reveal_type(var) # N: Revealed type is "TypedDict('__main__.Foo', {'y'?: builtins.int, 'x': __main__.Outer[__main__.Inner[__main__.ForceDeferredEval]]})" + +T1 = TypeVar("T1") +class Outer(Generic[T1]): pass + +T2 = TypeVar("T2", bound="ForceDeferredEval") +class Inner(Generic[T2]): pass + +class ForceDeferredEval: pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictRequiredUnimportedAny] +# flags: --disallow-any-unimported +from typing import NotRequired, TypedDict, ReadOnly +from nonexistent import Foo # type: ignore[import-not-found] +class Bar(TypedDict): + foo: NotRequired[Foo] # E: Type of variable becomes "Any" due to an unfollowed import + bar: ReadOnly[Foo] # E: Type of variable becomes "Any" due to an unfollowed import + baz: NotRequired[ReadOnly[Foo]] # E: Type of variable becomes "Any" due to an unfollowed import +[typing fixtures/typing-typeddict.pyi] + +-- Required[] + +[case testDoesRecognizeRequiredInTypedDictWithClass] +from typing import TypedDict +from typing import Required +class Movie(TypedDict, total=False): + title: Required[str] + year: int +m = Movie(title='The Matrix') +m = Movie() # E: Missing key "title" for TypedDict "Movie" +[typing fixtures/typing-typeddict.pyi] + +[case testDoesRecognizeRequiredInTypedDictWithAssignment] +from typing import TypedDict +from typing import Required +Movie = TypedDict('Movie', { + 'title': Required[str], + 'year': int, +}, total=False) +m = Movie(title='The Matrix') +m = Movie() # E: Missing key "title" for TypedDict "Movie" +[typing fixtures/typing-typeddict.pyi] + +[case testDoesDisallowRequiredOutsideOfTypedDict] +from typing import Required +x: Required[int] = 42 # E: Required[] can be only used in a TypedDict definition +[typing fixtures/typing-typeddict.pyi] + +[case testDoesOnlyAllowRequiredInsideTypedDictAtTopLevel] +from typing import TypedDict +from typing import Union +from typing import Required +Movie = TypedDict('Movie', { + 'title': Union[ + Required[str], # E: Required[] can be only used in a TypedDict definition + bytes + ], + 'year': int, +}, total=False) +[typing fixtures/typing-typeddict.pyi] + +[case testDoesDisallowRequiredInsideRequired] +from typing import TypedDict +from typing import Union +from typing import Required +Movie = TypedDict('Movie', { + 'title': Required[Union[ + Required[str], # E: Required[] can be only used in a TypedDict definition + bytes + ]], + 'year': int, +}, total=False) +[typing fixtures/typing-typeddict.pyi] + +[case testRequiredOnlyAllowsOneItem] +from typing import TypedDict +from typing import Required +class Movie(TypedDict, total=False): + title: Required[str, bytes] # E: Required[] must have exactly one type argument + year: int +[typing fixtures/typing-typeddict.pyi] + +[case testRequiredExplicitAny] +# flags: --disallow-any-explicit +from typing import TypedDict +from typing import Required +Foo = TypedDict("Foo", {"a.x": Required[int]}) +[typing fixtures/typing-typeddict.pyi] + +-- NotRequired[] + +[case testDoesRecognizeNotRequiredInTypedDictWithClass] +from typing import TypedDict +from typing import NotRequired +class Movie(TypedDict): + title: str + year: NotRequired[int] +m = Movie(title='The Matrix') +m = Movie() # E: Missing key "title" for TypedDict "Movie" +[typing fixtures/typing-typeddict.pyi] + +[case testDoesRecognizeNotRequiredInTypedDictWithAssignment] +from typing import TypedDict +from typing import NotRequired +Movie = TypedDict('Movie', { + 'title': str, + 'year': NotRequired[int], +}) +m = Movie(title='The Matrix') +m = Movie() # E: Missing key "title" for TypedDict "Movie" +[typing fixtures/typing-typeddict.pyi] + +[case testDoesDisallowNotRequiredOutsideOfTypedDict] +from typing import NotRequired +x: NotRequired[int] = 42 # E: NotRequired[] can be only used in a TypedDict definition +[typing fixtures/typing-typeddict.pyi] + +[case testDoesOnlyAllowNotRequiredInsideTypedDictAtTopLevel] +from typing import TypedDict +from typing import Union +from typing import NotRequired +Movie = TypedDict('Movie', { + 'title': Union[ + NotRequired[str], # E: NotRequired[] can be only used in a TypedDict definition + bytes + ], + 'year': int, +}) +[typing fixtures/typing-typeddict.pyi] + +[case testDoesDisallowNotRequiredInsideNotRequired] +from typing import TypedDict +from typing import Union +from typing import NotRequired +Movie = TypedDict('Movie', { + 'title': NotRequired[Union[ + NotRequired[str], # E: NotRequired[] can be only used in a TypedDict definition + bytes + ]], + 'year': int, +}) +[typing fixtures/typing-typeddict.pyi] + +[case testNotRequiredOnlyAllowsOneItem] +from typing import TypedDict +from typing import NotRequired +class Movie(TypedDict): + title: NotRequired[str, bytes] # E: NotRequired[] must have exactly one type argument + year: int +[typing fixtures/typing-typeddict.pyi] + +[case testNotRequiredExplicitAny] +# flags: --disallow-any-explicit +from typing import TypedDict +from typing import NotRequired +Foo = TypedDict("Foo", {"a.x": NotRequired[int]}) +[typing fixtures/typing-typeddict.pyi] + +-- Union dunders + +[case testTypedDictUnionGetItem] +from typing import TypedDict, Union + +class Foo1(TypedDict): + z: str + a: int +class Foo2(TypedDict): + z: str + b: int + +def func(foo: Union[Foo1, Foo2]) -> str: + reveal_type(foo["z"]) # N: Revealed type is "builtins.str" + # ok, but type is incorrect: + reveal_type(foo.__getitem__("z")) # N: Revealed type is "builtins.object" + + reveal_type(foo["a"]) # N: Revealed type is "Union[builtins.int, Any]" \ + # E: TypedDict "Foo2" has no key "a" + reveal_type(foo["b"]) # N: Revealed type is "Union[Any, builtins.int]" \ + # E: TypedDict "Foo1" has no key "b" + reveal_type(foo["missing"]) # N: Revealed type is "Any" \ + # E: TypedDict "Foo1" has no key "missing" \ + # E: TypedDict "Foo2" has no key "missing" + reveal_type(foo[1]) # N: Revealed type is "Any" \ + # E: TypedDict key must be a string literal; expected one of ("z", "a") \ + # E: TypedDict key must be a string literal; expected one of ("z", "b") + + return foo["z"] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnionSetItem] +from typing import TypedDict, Union + +class Foo1(TypedDict): + z: str + a: int +class Foo2(TypedDict): + z: str + b: int + +def func(foo: Union[Foo1, Foo2]): + foo["z"] = "a" # ok + foo.__setitem__("z", "a") # ok + + foo["z"] = 1 # E: Value of "z" has incompatible type "int"; expected "str" + + foo["a"] = 1 # E: TypedDict "Foo2" has no key "a" + foo["b"] = 2 # E: TypedDict "Foo1" has no key "b" + + foo["missing"] = 1 # E: TypedDict "Foo1" has no key "missing" \ + # E: TypedDict "Foo2" has no key "missing" + foo[1] = "m" # E: TypedDict key must be a string literal; expected one of ("z", "a") \ + # E: TypedDict key must be a string literal; expected one of ("z", "b") \ + # E: Argument 1 to "__setitem__" has incompatible type "int"; expected "str" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnionDelItem] +from typing import TypedDict, Union + +class Foo1(TypedDict): + z: str + a: int +class Foo2(TypedDict): + z: str + b: int + +def func(foo: Union[Foo1, Foo2]): + del foo["z"] # E: Key "z" of TypedDict "Foo1" cannot be deleted \ + # E: Key "z" of TypedDict "Foo2" cannot be deleted + foo.__delitem__("z") # E: Key "z" of TypedDict "Foo1" cannot be deleted \ + # E: Key "z" of TypedDict "Foo2" cannot be deleted + + del foo["a"] # E: Key "a" of TypedDict "Foo1" cannot be deleted \ + # E: TypedDict "Foo2" has no key "a" + del foo["b"] # E: TypedDict "Foo1" has no key "b" \ + # E: Key "b" of TypedDict "Foo2" cannot be deleted + + del foo["missing"] # E: TypedDict "Foo1" has no key "missing" \ + # E: TypedDict "Foo2" has no key "missing" + del foo[1] # E: Expected TypedDict key to be string literal \ + # E: Argument 1 to "__delitem__" has incompatible type "int"; expected "str" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictTypeVarUnionSetItem] +from typing import TypedDict, Union, TypeVar + +F1 = TypeVar('F1', bound='Foo1') +F2 = TypeVar('F2', bound='Foo2') + +class Foo1(TypedDict): + z: str + a: int +class Foo2(TypedDict): + z: str + b: int + +def func(foo: Union[F1, F2]): + foo["z"] = "a" # ok + foo["z"] = 1 # E: Value of "z" has incompatible type "int"; expected "str" + + foo["a"] = 1 # E: TypedDict "Foo2" has no key "a" + foo["b"] = 2 # E: TypedDict "Foo1" has no key "b" + + foo["missing"] = 1 # E: TypedDict "Foo1" has no key "missing" \ + # E: TypedDict "Foo2" has no key "missing" + foo[1] = "m" # E: TypedDict key must be a string literal; expected one of ("z", "a") \ + # E: TypedDict key must be a string literal; expected one of ("z", "b") \ + # E: Argument 1 to "__setitem__" has incompatible type "int"; expected "str" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testGenericTypedDictCreation] +from typing import TypedDict, Generic, TypeVar + +T = TypeVar("T") + +class TD(TypedDict, Generic[T]): + key: int + value: T + +tds: TD[str] +reveal_type(tds) # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.str})" + +tdi = TD(key=0, value=0) +reveal_type(tdi) # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.int})" +TD[str](key=0, value=0) # E: Incompatible types (expression has type "int", TypedDict item "value" has type "str") +TD[str]({"key": 0, "value": 0}) # E: Incompatible types (expression has type "int", TypedDict item "value" has type "str") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testGenericTypedDictInference] +from typing import TypedDict, Generic, TypeVar, List + +T = TypeVar("T") + +class TD(TypedDict, Generic[T]): + key: int + value: T + +def foo(x: TD[T]) -> List[T]: ... + +reveal_type(foo(TD(key=1, value=2))) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(foo({"key": 1, "value": 2})) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(foo(dict(key=1, value=2))) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testGenericTypedDictExtending] +from typing import TypedDict, Generic, TypeVar, List + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + key: int + value: T + +S = TypeVar("S") +class STD(TD[List[S]]): + other: S + +std: STD[str] +reveal_type(std) # N: Revealed type is "TypedDict('__main__.STD', {'key': builtins.int, 'value': builtins.list[builtins.str], 'other': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testGenericTypedDictExtendingErrors] +from typing import TypedDict, Generic, TypeVar + +T = TypeVar("T") +class Base(TypedDict, Generic[T]): + x: T +class Sub(Base[{}]): # E: Invalid TypedDict type argument \ + # E: Type expected within [...] \ + # E: Invalid base class "Base" + y: int +s: Sub +reveal_type(s) # N: Revealed type is "TypedDict('__main__.Sub', {'y': builtins.int})" + +class Sub2(Base[int, str]): # E: Invalid number of type arguments for "Base" \ + # E: "Base" expects 1 type argument, but 2 given + y: int +s2: Sub2 +reveal_type(s2) # N: Revealed type is "TypedDict('__main__.Sub2', {'x': Any, 'y': builtins.int})" + +class Sub3(Base): # OK + y: int +s3: Sub3 +reveal_type(s3) # N: Revealed type is "TypedDict('__main__.Sub3', {'x': Any, 'y': builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictAttributeOnClassObject] +from typing import TypedDict + +class TD(TypedDict): + x: str + y: str + +reveal_type(TD.__iter__) # N: Revealed type is "def (typing._TypedDict) -> typing.Iterator[builtins.str]" +reveal_type(TD.__annotations__) # N: Revealed type is "typing.Mapping[builtins.str, builtins.object]" +reveal_type(TD.values) # N: Revealed type is "def (self: typing.Mapping[T`1, T_co`2]) -> typing.Iterable[T_co`2]" +[builtins fixtures/dict-full.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testGenericTypedDictAlias] +# flags: --disallow-any-generics +from typing import TypedDict, Generic, TypeVar, List + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + key: int + value: T + +Alias = TD[List[T]] + +ad: Alias[str] +reveal_type(ad) # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.list[builtins.str]})" +Alias[str](key=0, value=0) # E: Incompatible types (expression has type "int", TypedDict item "value" has type "List[str]") + +# Generic aliases are *always* filled with Any, so this is different from TD(...) call. +Alias(key=0, value=0) # E: Missing type parameters for generic type "Alias" \ + # E: Incompatible types (expression has type "int", TypedDict item "value" has type "List[Any]") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testGenericTypedDictMultipleGenerics] +# See https://github.com/python/mypy/issues/13755 +from typing import Generic, TypeVar, TypedDict + +T = TypeVar("T") +Foo = TypedDict("Foo", {"bar": T}) +class Stack(Generic[T]): pass + +a = Foo[str] +b = Foo[int] +reveal_type(a) # N: Revealed type is "def (*, bar: builtins.str) -> TypedDict('__main__.Foo', {'bar': builtins.str})" +reveal_type(b) # N: Revealed type is "def (*, bar: builtins.int) -> TypedDict('__main__.Foo', {'bar': builtins.int})" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testGenericTypedDictCallSyntax] +from typing import TypedDict, TypeVar + +T = TypeVar("T") +TD = TypedDict("TD", {"key": int, "value": T}) +reveal_type(TD) # N: Revealed type is "def [T] (*, key: builtins.int, value: T`1) -> TypedDict('__main__.TD', {'key': builtins.int, 'value': T`1})" + +tds: TD[str] +reveal_type(tds) # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.str})" + +tdi = TD(key=0, value=0) +reveal_type(tdi) # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.int})" +TD[str](key=0, value=0) # E: Incompatible types (expression has type "int", TypedDict item "value" has type "str") +TD[str]({"key": 0, "value": 0}) # E: Incompatible types (expression has type "int", TypedDict item "value" has type "str") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictSelfItemNotAllowed] +from typing import Self, TypedDict, Optional + +class TD(TypedDict): + val: int + next: Optional[Self] # E: Self type cannot be used in TypedDict item type +TDC = TypedDict("TDC", {"val": int, "next": Optional[Self]}) # E: Self type cannot be used in TypedDict item type + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testUnionOfEquivalentTypedDictsInferred] +from typing import TypedDict, Dict + +D = TypedDict("D", {"foo": int}, total=False) + +def f(d: Dict[str, D]) -> None: + args = d["a"] + args.update(d.get("b", {})) # OK +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testUnionOfEquivalentTypedDictsDeclared] +from typing import TypedDict, Union + +class A(TypedDict, total=False): + name: str +class B(TypedDict, total=False): + name: str + +def foo(data: Union[A, B]) -> None: ... +foo({"name": "Robert"}) # OK +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testUnionOfEquivalentTypedDictsEmpty] +from typing import TypedDict, Union + +class Foo(TypedDict, total=False): + foo: str +class Bar(TypedDict, total=False): + bar: str + +def foo(body: Union[Foo, Bar] = {}) -> None: # OK + ... +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testUnionOfEquivalentTypedDictsDistinct] +from typing import TypedDict, Union, Literal + +class A(TypedDict): + type: Literal['a'] + value: bool +class B(TypedDict): + type: Literal['b'] + value: str + +Response = Union[A, B] +def method(message: Response) -> None: ... + +method({'type': 'a', 'value': True}) # OK +method({'type': 'b', 'value': 'abc'}) # OK +method({'type': 'a', 'value': 'abc'}) # E: Type of TypedDict is ambiguous, none of ("A", "B") matches cleanly \ + # E: Argument 1 to "method" has incompatible type "Dict[str, str]"; expected "Union[A, B]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testUnionOfEquivalentTypedDictsNested] +from typing import TypedDict, Union + +class A(TypedDict, total=False): + foo: C +class B(TypedDict, total=False): + foo: D +class C(TypedDict, total=False): + c: str +class D(TypedDict, total=False): + d: str + +def foo(data: Union[A, B]) -> None: ... +foo({"foo": {"c": "foo"}}) # OK +foo({"foo": {"e": "foo"}}) # E: Type of TypedDict is ambiguous, none of ("A", "B") matches cleanly \ + # E: Argument 1 to "foo" has incompatible type "Dict[str, Dict[str, str]]"; expected "Union[A, B]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictMissingEmptyKey] +from typing_extensions import TypedDict + +class A(TypedDict): + my_attr_1: str + my_attr_2: int + +d: A +d[''] # E: TypedDict "A" has no key "" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFlexibleUpdate] +from mypy_extensions import TypedDict + +A = TypedDict("A", {"foo": int, "bar": int}) +B = TypedDict("B", {"foo": int}) + +a = A({"foo": 1, "bar": 2}) +b = B({"foo": 2}) +a.update({"foo": 2}) +a.update(b) +a.update(a) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictStrictUpdate] +# flags: --extra-checks +from mypy_extensions import TypedDict + +A = TypedDict("A", {"foo": int, "bar": int}) +B = TypedDict("B", {"foo": int}) + +a = A({"foo": 1, "bar": 2}) +b = B({"foo": 2}) +a.update({"foo": 2}) # OK +a.update(b) # E: Argument 1 to "update" of "TypedDict" has incompatible type "B"; expected "TypedDict({'foo': int, 'bar'?: int})" +a.update(a) # OK +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFlexibleUpdateUnion] +from typing import Union +from mypy_extensions import TypedDict + +A = TypedDict("A", {"foo": int, "bar": int}) +B = TypedDict("B", {"foo": int}) +C = TypedDict("C", {"bar": int}) + +a = A({"foo": 1, "bar": 2}) +u: Union[B, C] +a.update(u) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFlexibleUpdateUnionExtra] +from typing import Union +from mypy_extensions import TypedDict + +A = TypedDict("A", {"foo": int, "bar": int}) +B = TypedDict("B", {"foo": int, "extra": int}) +C = TypedDict("C", {"bar": int, "extra": int}) + +a = A({"foo": 1, "bar": 2}) +u: Union[B, C] +a.update(u) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFlexibleUpdateUnionStrict] +# flags: --extra-checks +from typing import Union, NotRequired +from mypy_extensions import TypedDict + +A = TypedDict("A", {"foo": int, "bar": int}) +A1 = TypedDict("A1", {"foo": int, "bar": NotRequired[int]}) +A2 = TypedDict("A2", {"foo": NotRequired[int], "bar": int}) +B = TypedDict("B", {"foo": int}) +C = TypedDict("C", {"bar": int}) + +a = A({"foo": 1, "bar": 2}) +u: Union[B, C] +a.update(u) # E: Argument 1 to "update" of "TypedDict" has incompatible type "Union[B, C]"; expected "Union[TypedDict({'foo': int, 'bar'?: int}), TypedDict({'foo'?: int, 'bar': int})]" +u2: Union[A1, A2] +a.update(u2) # OK +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackSame] +# flags: --extra-checks +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +foo1: Foo = {"a": 1, "b": 1} +foo2: Foo = {**foo1, "b": 2} +foo3 = Foo(**foo1, b=2) +foo4 = Foo({**foo1, "b": 2}) +foo5 = Foo(dict(**foo1, b=2)) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackCompatible] +# flags: --extra-checks +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {"a": 1} +bar: Bar = {**foo, "b": 2} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackIncompatible] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: str + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {"a": 1, "b": "a"} +bar1: Bar = {**foo, "b": 2} # Incompatible item is overriden +bar2: Bar = {**foo, "a": 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackNotRequiredKeyIncompatible] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: NotRequired[str] + +class Bar(TypedDict): + a: NotRequired[int] + +foo: Foo = {} +bar: Bar = {**foo} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackMissingOrExtraKey] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: int + +foo1: Foo = {"a": 1} +bar1: Bar = {"a": 1, "b": 1} +foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" +bar2: Bar = {**foo1} # E: Missing key "b" for TypedDict "Bar" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackNotRequiredKeyExtra] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: NotRequired[int] + +foo1: Foo = {"a": 1} +bar1: Bar = {"a": 1} +foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" +bar2: Bar = {**foo1} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackRequiredKeyMissing] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: NotRequired[int] + +class Bar(TypedDict): + a: int + +foo: Foo = {"a": 1} +bar: Bar = {**foo} # E: Missing key "a" for TypedDict "Bar" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackMultiple] +# flags: --extra-checks +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + b: int + +class Baz(TypedDict): + a: int + b: int + c: int + +foo: Foo = {"a": 1} +bar: Bar = {"b": 1} +baz: Baz = {**foo, **bar, "c": 1} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackNested] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + c: Foo + d: int + +foo: Foo = {"a": 1, "b": 1} +bar: Bar = {"c": foo, "d": 1} +bar2: Bar = {**bar, "c": {**bar["c"], "b": 2}, "d": 2} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackNestedError] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + c: Foo + d: int + +foo: Foo = {"a": 1, "b": 1} +bar: Bar = {"c": foo, "d": 1} +bar2: Bar = {**bar, "c": {**bar["c"], "b": "wrong"}, "d": 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackOverrideRequired] +from mypy_extensions import TypedDict + +Details = TypedDict('Details', {'first_name': str, 'last_name': str}) +DetailsSubset = TypedDict('DetailsSubset', {'first_name': str, 'last_name': str}, total=False) +defaults: Details = {'first_name': 'John', 'last_name': 'Luther'} + +def generate(data: DetailsSubset) -> Details: + return {**defaults, **data} # OK +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackUntypedDict] +from typing import Any, Dict, TypedDict + +class Bar(TypedDict): + pass + +foo: Dict[str, Any] = {} +bar: Bar = {**foo} # E: Unsupported type "Dict[str, Any]" for ** expansion in TypedDict +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackIntoUnion] +from typing import TypedDict, Union + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + b: int + +foo: Foo = {'a': 1} +foo_or_bar: Union[Foo, Bar] = {**foo} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackFromUnion] +from typing import TypedDict, Union + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + b: int + +foo_or_bar: Union[Foo, Bar] = {'b': 1} +foo: Bar = {**foo_or_bar} # E: Extra key "a" for TypedDict "Bar" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackUnionRequiredMissing] +from typing import TypedDict, NotRequired, Union + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + a: int + b: NotRequired[int] + +foo_or_bar: Union[Foo, Bar] = {"a": 1} +foo: Foo = {**foo_or_bar} # E: Missing key "b" for TypedDict "Foo" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackInference] +from typing import TypedDict, Generic, TypeVar + +class Foo(TypedDict): + a: int + b: str + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + a: T + b: str + +foo: Foo +bar = TD(**foo) +reveal_type(bar) # N: Revealed type is "TypedDict('__main__.TD', {'a': builtins.int, 'b': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackStrictMode] +# flags: --extra-checks +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: NotRequired[int] + +foo: Foo +bar: Bar = {**foo} # E: Non-required key "b" not explicitly found in any ** item +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackAny] +from typing import Any, TypedDict, NotRequired, Dict, Union + +class Foo(TypedDict): + a: int + b: NotRequired[int] + +x: Any +y: Dict[Any, Any] +z: Union[Any, Dict[Any, Any]] +t1: Foo = {**x} # E: Missing key "a" for TypedDict "Foo" +t2: Foo = {**y} # E: Missing key "a" for TypedDict "Foo" +t3: Foo = {**z} # E: Missing key "a" for TypedDict "Foo" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictUnpackError] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +def foo(x: int) -> Foo: ... + +f: Foo = {**foo("no")} # E: Argument 1 to "foo" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictWith__or__method] +from typing import Dict +from mypy_extensions import TypedDict + +class Foo(TypedDict): + key: int + +foo1: Foo = {'key': 1} +foo2: Foo = {'key': 2} + +reveal_type(foo1 | foo2) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})" +reveal_type(foo1 | {'key': 1}) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})" +reveal_type(foo1 | {'key': 'a'}) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +reveal_type(foo1 | {}) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})" + +d1: Dict[str, int] +d2: Dict[int, str] + +reveal_type(foo1 | d1) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +foo1 | d2 # E: Unsupported operand types for | ("Foo" and "Dict[int, str]") + + +class Bar(TypedDict): + key: int + value: str + +bar: Bar +reveal_type(bar | {}) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type(bar | {'key': 1, 'value': 'v'}) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type(bar | {'key': 1}) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type(bar | {'value': 'v'}) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type(bar | {'key': 'a'}) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +reveal_type(bar | {'value': 1}) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +reveal_type(bar | {'key': 'a', 'value': 1}) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" + +reveal_type(bar | foo1) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type(bar | d1) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +bar | d2 # E: Unsupported operand types for | ("Bar" and "Dict[int, str]") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testTypedDictWith__or__method_error] +from mypy_extensions import TypedDict + +class Foo(TypedDict): + key: int + +foo: Foo = {'key': 1} +foo | 1 + +class SubDict(dict): ... +foo | SubDict() +[out] +main:7: error: No overload variant of "__or__" of "TypedDict" matches argument type "int" +main:7: note: Possible overload variants: +main:7: note: def __or__(self, TypedDict({'key'?: int}), /) -> Foo +main:7: note: def __or__(self, Dict[str, Any], /) -> Dict[str, object] +main:10: error: No overload variant of "__ror__" of "dict" matches argument type "Foo" +main:10: note: Possible overload variants: +main:10: note: def __ror__(self, Dict[Any, Any], /) -> Dict[Any, Any] +main:10: note: def [T, T2] __ror__(self, Dict[T, T2], /) -> Dict[Union[Any, T], Union[Any, T2]] +[builtins fixtures/dict-full.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testTypedDictWith__ror__method] +from typing import Dict +from mypy_extensions import TypedDict + +class Foo(TypedDict): + key: int + +foo: Foo = {'key': 1} + +reveal_type({'key': 1} | foo) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})" +reveal_type({'key': 'a'} | foo) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +reveal_type({} | foo) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})" +{1: 'a'} | foo # E: Dict entry 0 has incompatible type "int": "str"; expected "str": "Any" + +d1: Dict[str, int] +d2: Dict[int, str] + +reveal_type(d1 | foo) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +d2 | foo # E: Unsupported operand types for | ("Dict[int, str]" and "Foo") +1 | foo # E: Unsupported left operand type for | ("int") + + +class Bar(TypedDict): + key: int + value: str + +bar: Bar +reveal_type({} | bar) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type({'key': 1, 'value': 'v'} | bar) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type({'key': 1} | bar) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type({'value': 'v'} | bar) # N: Revealed type is "TypedDict('__main__.Bar', {'key': builtins.int, 'value': builtins.str})" +reveal_type({'key': 'a'} | bar) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +reveal_type({'value': 1} | bar) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +reveal_type({'key': 'a', 'value': 1} | bar) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" + +reveal_type(d1 | bar) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" +d2 | bar # E: Unsupported operand types for | ("Dict[int, str]" and "Bar") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testTypedDictWith__ior__method] +from typing import Dict +from mypy_extensions import TypedDict + +class Foo(TypedDict): + key: int + +foo: Foo = {'key': 1} +foo |= {'key': 2} + +foo |= {} +foo |= {'key': 'a', 'b': 'a'} # E: Expected TypedDict key "key" but found keys ("key", "b") \ + # E: Incompatible types (expression has type "str", TypedDict item "key" has type "int") +foo |= {'b': 2} # E: Unexpected TypedDict key "b" + +d1: Dict[str, int] +d2: Dict[int, str] + +foo |= d1 # E: Argument 1 to "__ior__" of "TypedDict" has incompatible type "Dict[str, int]"; expected "TypedDict({'key'?: int})" +foo |= d2 # E: Argument 1 to "__ior__" of "TypedDict" has incompatible type "Dict[int, str]"; expected "TypedDict({'key'?: int})" + + +class Bar(TypedDict): + key: int + value: str + +bar: Bar +bar |= {} +bar |= {'key': 1, 'value': 'a'} +bar |= {'key': 'a', 'value': 'a', 'b': 'a'} # E: Expected TypedDict keys ("key", "value") but found keys ("key", "value", "b") \ + # E: Incompatible types (expression has type "str", TypedDict item "key" has type "int") + +bar |= foo +bar |= d1 # E: Argument 1 to "__ior__" of "TypedDict" has incompatible type "Dict[str, int]"; expected "TypedDict({'key'?: int, 'value'?: str})" +bar |= d2 # E: Argument 1 to "__ior__" of "TypedDict" has incompatible type "Dict[int, str]"; expected "TypedDict({'key'?: int, 'value'?: str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testGenericTypedDictStrictOptionalExtending] +from typing import Generic, TypeVar, TypedDict, Optional + +T = TypeVar("T") +class Foo(TypedDict, Generic[T], total=False): + a: Optional[str] + g: Optional[T] + +class Bar(Foo[T], total=False): + other: str + +b: Bar[int] +reveal_type(b["a"]) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(b["g"]) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testNoCrashOnUnImportedAnyNotRequired] +# flags: --disallow-any-unimported +from typing import NotRequired, Required, TypedDict +from thismoduledoesntexist import T # type: ignore[import] + +B = TypedDict("B", { # E: Type of a TypedDict key becomes "Any" due to an unfollowed import + "T1": NotRequired[T], + "T2": Required[T], +}) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictWithClassLevelKeywords] +from typing import TypedDict, Generic, TypeVar + +T = TypeVar('T') + +class Meta(type): ... + +class WithMetaKeyword(TypedDict, metaclass=Meta): # E: Unexpected keyword argument "metaclass" for "__init_subclass__" of "TypedDict" + ... + +class GenericWithMetaKeyword(TypedDict, Generic[T], metaclass=Meta): # E: Unexpected keyword argument "metaclass" for "__init_subclass__" of "TypedDict" + ... + +# We still don't allow this, because the implementation is much easier +# and it does not make any practical sense to do it: +class WithTypeMeta(TypedDict, metaclass=type): # E: Unexpected keyword argument "metaclass" for "__init_subclass__" of "TypedDict" + ... + +class OtherKeywords(TypedDict, a=1, b=2, c=3, total=True): # E: Unexpected keyword argument "a" for "__init_subclass__" of "TypedDict" \ + # E: Unexpected keyword argument "b" for "__init_subclass__" of "TypedDict" \ + # E: Unexpected keyword argument "c" for "__init_subclass__" of "TypedDict" + ... + +class TotalInTheMiddle(TypedDict, a=1, total=True, b=2, c=3): # E: Unexpected keyword argument "a" for "__init_subclass__" of "TypedDict" \ + # E: Unexpected keyword argument "b" for "__init_subclass__" of "TypedDict" \ + # E: Unexpected keyword argument "c" for "__init_subclass__" of "TypedDict" + ... +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCanCreateClassWithFunctionBasedTypedDictBase] +from mypy_extensions import TypedDict + +class Params(TypedDict("Params", {'x': int})): + pass + +p: Params = {'x': 2} +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Params', {'x': builtins.int})" +[builtins fixtures/dict.pyi] + +[case testInitTypedDictFromType] +from typing import TypedDict, Type +from typing_extensions import Required + +class Point(TypedDict, total=False): + x: Required[int] + y: int + +def func(cls: Type[Point]) -> None: + reveal_type(cls) # N: Revealed type is "Type[TypedDict('__main__.Point', {'x': builtins.int, 'y'?: builtins.int})]" + cls(x=1, y=2) + cls(1, 2) # E: Too many positional arguments + cls(x=1) + cls(y=2) # E: Missing named argument "x" + cls(x=1, y=2, error="") # E: Unexpected keyword argument "error" +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testInitTypedDictFromTypeGeneric] +from typing import Generic, TypedDict, Type, TypeVar +from typing_extensions import Required + +class Point(TypedDict, total=False): + x: Required[int] + y: int + +T = TypeVar("T", bound=Point) + +class A(Generic[T]): + def __init__(self, a: Type[T]) -> None: + self.a = a + + def func(self) -> T: + reveal_type(self.a) # N: Revealed type is "Type[T`1]" + self.a(x=1, y=2) + self.a(y=2) # E: Missing named argument "x" + return self.a(x=1) +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testNameUndefinedErrorDoesNotLoseUnpackedKWArgsInformation] +from typing import overload +from typing_extensions import TypedDict, Unpack + +class TD(TypedDict, total=False): + x: int + y: str + +@overload +def f(self, *, x: int) -> None: ... +@overload +def f(self, *, y: str) -> None: ... +def f(self, **kwargs: Unpack[TD]) -> None: + z # E: Name "z" is not defined + +@overload +def g(self, *, x: float) -> None: ... +@overload +def g(self, *, y: str) -> None: ... +def g(self, **kwargs: Unpack[TD]) -> None: # E: Overloaded function implementation does not accept all possible arguments of signature 1 + z # E: Name "z" is not defined + +class A: + def f(self, *, x: int) -> None: ... + def g(self, *, x: float) -> None: ... +class B(A): + def f(self, **kwargs: Unpack[TD]) -> None: + z # E: Name "z" is not defined + def g(self, **kwargs: Unpack[TD]) -> None: # E: Signature of "g" incompatible with supertype "A" \ + # N: Superclass: \ + # N: def g(self, *, x: float) -> None \ + # N: Subclass: \ + # N: def g(*, x: int = ..., y: str = ...) -> None + z # E: Name "z" is not defined +reveal_type(B.f) # N: Revealed type is "def (self: __main__.B, **kwargs: Unpack[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: builtins.str})])" +B().f(x=1.0) # E: Argument "x" to "f" of "B" has incompatible type "float"; expected "int" +[builtins fixtures/primitives.pyi] + +[case testTypedDictUnpackWithParamSpecInference] +from typing import TypeVar, ParamSpec, Callable +from typing_extensions import TypedDict, Unpack + +P = ParamSpec("P") +R = TypeVar("R") + +def run(func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: ... + +class Params(TypedDict): + temperature: float + +def test(temperature: int) -> None: ... +def test2(temperature: float, other: str) -> None: ... + +class Test: + def f(self, c: Callable[..., None], **params: Unpack[Params]) -> None: + run(c, **params) + def g(self, **params: Unpack[Params]) -> None: + run(test, **params) # E: Argument "temperature" to "run" has incompatible type "float"; expected "int" + def h(self, **params: Unpack[Params]) -> None: + run(test2, other="yes", **params) + run(test2, other=0, **params) # E: Argument "other" to "run" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] + +[case testTypedDictUnpackSingleWithSubtypingNoCrash] +from typing import Callable +from typing_extensions import TypedDict, Unpack + +class Kwargs(TypedDict): + name: str + +def f(**kwargs: Unpack[Kwargs]) -> None: + pass + +class C: + d: Callable[[Unpack[Kwargs]], None] + +# TODO: it is an old question whether we should allow this, for now simply don't crash. +class D(C): + d = f +[builtins fixtures/tuple.pyi] + +[case testTypedDictInlineNoOldStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +X = {"int": int, "str": str} +reveal_type(X) # N: Revealed type is "builtins.dict[builtins.str, def () -> builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineYesMidStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing_extensions import TypeAlias +X: TypeAlias = {"int": int, "str": str} +x: X +reveal_type(x) # N: # N: Revealed type is "TypedDict({'int': builtins.int, 'str': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNoEmpty] +# flags: --enable-incomplete-feature=InlineTypedDict +x: {} # E: Invalid type comment or annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNotRequired] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import NotRequired + +x: {"one": int, "other": NotRequired[int]} +x = {"one": 1} # OK +y: {"one": int, "other": int} +y = {"one": 1} # E: Expected TypedDict keys ("one", "other") but found only key "one" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineReadOnly] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import ReadOnly + +x: {"one": int, "other": ReadOnly[int]} +x["one"] = 1 # ok +x["other"] = 1 # E: ReadOnly TypedDict key "other" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNestedSchema] +# flags: --enable-incomplete-feature=InlineTypedDict +def nested() -> {"one": str, "other": {"a": int, "b": int}}: + if bool(): + return {"one": "yes", "other": {"a": 1, "b": 2}} # OK + else: + return {"one": "no", "other": {"a": 1, "b": "2"}} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineMergeAnother] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import TypeVar +from typing_extensions import TypeAlias + +T = TypeVar("T") +X: TypeAlias = {"item": T} +x: {"a": int, **X[str], "b": int} +reveal_type(x) # N: Revealed type is "TypedDict({'a': builtins.int, 'b': builtins.int, 'item': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + + +# ReadOnly +# See: https://peps.python.org/pep-0705 + +[case testTypedDictReadOnly] +# flags: --show-error-codes +from typing import ReadOnly, TypedDict + +class TP(TypedDict): + one: int + other: ReadOnly[str] + +x: TP +reveal_type(x["one"]) # N: Revealed type is "builtins.int" +reveal_type(x["other"]) # N: Revealed type is "builtins.str" +x["one"] = 1 # ok +x["other"] = "a" # E: ReadOnly TypedDict key "other" TypedDict is mutated [typeddict-readonly-mutated] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyCreation] +from typing import ReadOnly, TypedDict + +class TD(TypedDict): + x: ReadOnly[int] + y: int + +# Ok: +x = TD({"x": 1, "y": 2}) +y = TD(x=1, y=2) +z: TD = {"x": 1, "y": 2} + +# Error: +x2 = TD({"x": "a", "y": 2}) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +y2 = TD(x="a", y=2) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +z2: TD = {"x": "a", "y": 2} # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyDel] +from typing import ReadOnly, TypedDict, NotRequired + +class TP(TypedDict): + required_key: ReadOnly[str] + optional_key: ReadOnly[NotRequired[str]] + +x: TP +del x["required_key"] # E: Key "required_key" of TypedDict "TP" cannot be deleted +del x["optional_key"] # E: Key "optional_key" of TypedDict "TP" cannot be deleted +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyMutateMethods] +from typing import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + other: ReadOnly[int] + mutable: bool + +x: TP +reveal_type(x.pop("key")) # E: Key "key" of TypedDict "TP" cannot be deleted \ + # N: Revealed type is "builtins.str" + +x.update({"key": "abc", "other": 1, "mutable": True}) # E: ReadOnly TypedDict keys ("key", "other") TypedDict are mutated +x.setdefault("key", "abc") # E: ReadOnly TypedDict key "key" TypedDict is mutated +x.setdefault("other", 1) # E: ReadOnly TypedDict key "other" TypedDict is mutated +x.setdefault("mutable", False) # ok +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFromTypingExtensionsReadOnlyMutateMethods] +from typing_extensions import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + +x: TP +x.update({"key": "abc"}) # E: ReadOnly TypedDict key "key" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFromMypyExtensionsReadOnlyMutateMethods] +from mypy_extensions import TypedDict +from typing_extensions import ReadOnly + +class TP(TypedDict): + key: ReadOnly[str] + +x: TP +x.update({"key": "abc"}) # E: ReadOnly TypedDict key "key" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyMutate__ior__Statements] +from typing_extensions import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + other: ReadOnly[int] + mutable: bool + +x: TP +x |= {"mutable": True} # ok +x |= {"key": "a"} # E: ReadOnly TypedDict key "key" TypedDict is mutated +x |= {"key": "a", "other": 1, "mutable": True} # E: ReadOnly TypedDict keys ("key", "other") TypedDict are mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testTypedDictReadOnlyMutate__or__Statements] +from typing_extensions import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + other: ReadOnly[int] + mutable: bool + +x: TP +# These are new objects, not mutation: +x = x | {"mutable": True} +x = x | {"key": "a"} +x = x | {"key": "a", "other": 1, "mutable": True} +y1 = x | {"mutable": True} +y2 = x | {"key": "a"} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testTypedDictReadOnlyMutateWithOtherDicts] +from typing import ReadOnly, TypedDict, Dict + +class TP(TypedDict): + key: ReadOnly[str] + mutable: bool + +class Mutable(TypedDict): + mutable: bool + +class Regular(TypedDict): + key: str + +m: Mutable +r: Regular +d: Dict[str, object] + +# Creating new objects is ok: +tp: TP = {**r, **m} +tp1: TP = {**tp, **m} +tp2: TP = {**r, **m} +tp3: TP = {**tp, **r} +tp4: TP = {**tp, **d} # E: Unsupported type "Dict[str, object]" for ** expansion in TypedDict +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictGenericReadOnly] +from typing import ReadOnly, TypedDict, TypeVar, Generic + +T = TypeVar('T') + +class TP(TypedDict, Generic[T]): + key: ReadOnly[T] + +x: TP[int] +reveal_type(x["key"]) # N: Revealed type is "builtins.int" +x["key"] = 1 # E: ReadOnly TypedDict key "key" TypedDict is mutated +x["key"] = "a" # E: ReadOnly TypedDict key "key" TypedDict is mutated \ + # E: Value of "key" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyOtherTypedDict] +from typing import ReadOnly, TypedDict + +class First(TypedDict): + field: int + +class TP(TypedDict): + key: ReadOnly[First] + +x: TP +reveal_type(x["key"]["field"]) # N: Revealed type is "builtins.int" +x["key"]["field"] = 1 # ok +x["key"] = {"field": 2} # E: ReadOnly TypedDict key "key" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyInheritance] +from typing import ReadOnly, TypedDict + +class Base(TypedDict): + a: ReadOnly[str] + +class Child(Base): + b: ReadOnly[int] + +base: Base +reveal_type(base["a"]) # N: Revealed type is "builtins.str" +base["a"] = "x" # E: ReadOnly TypedDict key "a" TypedDict is mutated +base["b"] # E: TypedDict "Base" has no key "b" + +child: Child +reveal_type(child["a"]) # N: Revealed type is "builtins.str" +reveal_type(child["b"]) # N: Revealed type is "builtins.int" +child["a"] = "x" # E: ReadOnly TypedDict key "a" TypedDict is mutated +child["b"] = 1 # E: ReadOnly TypedDict key "b" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlySubtyping] +from typing import ReadOnly, TypedDict + +class A(TypedDict): + key: ReadOnly[str] + +class B(TypedDict): + key: str + +a: A +b: B + +def accepts_A(d: A): ... +def accepts_B(d: B): ... + +accepts_A(a) +accepts_A(b) +accepts_B(a) # E: Argument 1 to "accepts_B" has incompatible type "A"; expected "B" +accepts_B(b) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictRequiredConsistentWithNotRequiredReadOnly] +from typing import NotRequired, ReadOnly, Required, TypedDict + +class A(TypedDict): + x: NotRequired[ReadOnly[str]] + +class B(TypedDict): + x: Required[str] + +def f(b: B): + a: A = b # ok +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyCall] +from typing import ReadOnly, TypedDict + +TP = TypedDict("TP", {"one": int, "other": ReadOnly[str]}) + +x: TP +reveal_type(x["one"]) # N: Revealed type is "builtins.int" +reveal_type(x["other"]) # N: Revealed type is "builtins.str" +x["one"] = 1 # ok +x["other"] = "a" # E: ReadOnly TypedDict key "other" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyABCSubtypes] +from typing import ReadOnly, TypedDict, Mapping, Dict, MutableMapping + +class TP(TypedDict): + one: int + other: ReadOnly[int] + +def accepts_mapping(m: Mapping[str, object]): ... +def accepts_mutable_mapping(mm: MutableMapping[str, object]): ... +def accepts_dict(d: Dict[str, object]): ... + +x: TP +accepts_mapping(x) +accepts_mutable_mapping(x) # E: Argument 1 to "accepts_mutable_mapping" has incompatible type "TP"; expected "MutableMapping[str, object]" +accepts_dict(x) # E: Argument 1 to "accepts_dict" has incompatible type "TP"; expected "Dict[str, object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyAndNotRequired] +from typing import ReadOnly, TypedDict, NotRequired + +class TP(TypedDict): + one: ReadOnly[NotRequired[int]] + two: NotRequired[ReadOnly[str]] + +x: TP +reveal_type(x) # N: Revealed type is "TypedDict('__main__.TP', {'one'?=: builtins.int, 'two'?=: builtins.str})" +reveal_type(x.get("one")) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(x.get("two")) # N: Revealed type is "Union[builtins.str, None]" +x["one"] = 1 # E: ReadOnly TypedDict key "one" TypedDict is mutated +x["two"] = "a" # E: ReadOnly TypedDict key "two" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testMeetOfTypedDictsWithReadOnly] +from typing import TypeVar, Callable, TypedDict, ReadOnly +XY = TypedDict('XY', {'x': ReadOnly[int], 'y': int}) +YZ = TypedDict('YZ', {'y': int, 'z': ReadOnly[int]}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypedDict({'x'=: builtins.int, 'y': builtins.int, 'z'=: builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyUnpack] +from typing_extensions import TypedDict, Unpack, ReadOnly + +class TD(TypedDict): + x: ReadOnly[int] + y: str + +def func(**kwargs: Unpack[TD]): + kwargs["x"] = 1 # E: ReadOnly TypedDict key "x" TypedDict is mutated + kwargs["y" ] = "a" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testIncorrectTypedDictSpecialFormsUsage] +from typing import ReadOnly, TypedDict, NotRequired, Required + +x: ReadOnly[int] # E: ReadOnly[] can be only used in a TypedDict definition +y: Required[int] # E: Required[] can be only used in a TypedDict definition +z: NotRequired[int] # E: NotRequired[] can be only used in a TypedDict definition + +class TP(TypedDict): + a: ReadOnly[ReadOnly[int]] # E: "ReadOnly[]" type cannot be nested + b: ReadOnly[NotRequired[ReadOnly[str]]] # E: "ReadOnly[]" type cannot be nested + c: NotRequired[Required[int]] # E: "Required[]" type cannot be nested + d: Required[NotRequired[int]] # E: "NotRequired[]" type cannot be nested + e: Required[ReadOnly[NotRequired[int]]] # E: "NotRequired[]" type cannot be nested + f: ReadOnly[ReadOnly[ReadOnly[int]]] # E: "ReadOnly[]" type cannot be nested + g: Required[Required[int]] # E: "Required[]" type cannot be nested + h: NotRequired[NotRequired[int]] # E: "NotRequired[]" type cannot be nested + + j: NotRequired[ReadOnly[Required[ReadOnly[int]]]] # E: "Required[]" type cannot be nested \ + # E: "ReadOnly[]" type cannot be nested + + k: ReadOnly # E: "ReadOnly[]" must have exactly one type argument +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictAnnotatedWithSpecialForms] +from typing import NotRequired, ReadOnly, Required, TypedDict +from typing_extensions import Annotated + +class A(TypedDict): + a: Annotated[NotRequired[ReadOnly[int]], ""] # ok + b: NotRequired[ReadOnly[Annotated[int, ""]]] # ok +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyCovariant] +from typing import ReadOnly, TypedDict, Union + +class A(TypedDict): + a: ReadOnly[Union[int, str]] + +class A2(TypedDict): + a: ReadOnly[int] + +class B(TypedDict): + a: int + +class B2(TypedDict): + a: Union[int, str] + +class B3(TypedDict): + a: int + +def fa(a: A) -> None: ... +def fa2(a: A2) -> None: ... + +b: B = {"a": 1} +fa(b) +fa2(b) +b2: B2 = {"a": 1} +fa(b2) +fa2(b2) # E: Argument 1 to "fa2" has incompatible type "B2"; expected "A2" + +class C(TypedDict): + a: ReadOnly[Union[int, str]] + b: Union[str, bytes] + +class D(TypedDict): + a: int + b: str + +d: D = {"a": 1, "b": "x"} +c: C = d # E: Incompatible types in assignment (expression has type "D", variable has type "C") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test new file mode 100644 index 000000000000..e7a8eac4f043 --- /dev/null +++ b/test-data/unit/check-typeguard.test @@ -0,0 +1,732 @@ +[case testTypeGuardBasic] +from typing_extensions import TypeGuard +class Point: pass +def is_point(a: object) -> TypeGuard[Point]: pass +def main(a: object) -> None: + if is_point(a): + reveal_type(a) # N: Revealed type is "__main__.Point" + else: + reveal_type(a) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardTypeArgsNone] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard: # E: TypeGuard must have exactly one type argument + pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardTypeArgsTooMany] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard[int, int]: # E: TypeGuard must have exactly one type argument + pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardTypeArgType] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard[42]: # E: Invalid type: try using Literal[42] instead? + pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardRepr] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard[int]: + pass +reveal_type(foo) # N: Revealed type is "def (a: builtins.object) -> TypeGuard[builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardCallArgsNone] +from typing_extensions import TypeGuard +class Point: pass + +def is_point() -> TypeGuard[Point]: pass # E: TypeGuard functions must have a positional argument +def main(a: object) -> None: + if is_point(): + reveal_type(a) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardCallArgsMultiple] +from typing_extensions import TypeGuard +class Point: pass +def is_point(a: object, b: object) -> TypeGuard[Point]: pass +def main(a: object, b: object) -> None: + if is_point(a, b): + reveal_type(a) # N: Revealed type is "__main__.Point" + reveal_type(b) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardTypeVarReturn] +from typing import Callable, Optional, TypeVar +from typing_extensions import TypeGuard +T = TypeVar('T') +def is_str(x: object) -> TypeGuard[str]: pass +def main(x: object, type_check_func: Callable[[object], TypeGuard[T]]) -> T: + if not type_check_func(x): + raise Exception() + return x +reveal_type(main("a", is_str)) # N: Revealed type is "builtins.str" +[builtins fixtures/exception.pyi] + +[case testTypeGuardIsBool] +from typing_extensions import TypeGuard +def f(a: TypeGuard[int]) -> None: pass +reveal_type(f) # N: Revealed type is "def (a: builtins.bool)" +a: TypeGuard[int] +reveal_type(a) # N: Revealed type is "builtins.bool" +class C: + a: TypeGuard[int] +reveal_type(C().a) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWithTypeVar] +from typing import TypeVar, Tuple +from typing_extensions import TypeGuard +T = TypeVar('T') +def is_two_element_tuple(a: Tuple[T, ...]) -> TypeGuard[Tuple[T, T]]: pass +def main(a: Tuple[T, ...]): + if is_two_element_tuple(a): + reveal_type(a) # N: Revealed type is "Tuple[T`-1, T`-1]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardPassedAsTypeVarIsBool] +from typing import Callable, TypeVar +from typing_extensions import TypeGuard +T = TypeVar('T') +def is_str(x: object) -> TypeGuard[str]: ... +def main(f: Callable[[object], T]) -> T: ... +reveal_type(main(is_str)) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNonOverlapping] +from typing import List +from typing_extensions import TypeGuard +def is_str_list(a: List[object]) -> TypeGuard[List[str]]: pass +def main(a: List[object]): + if is_str_list(a): + reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(a) # N: Revealed type is "builtins.list[builtins.object]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardUnionIn] +from typing import Union +from typing_extensions import TypeGuard +def is_foo(a: Union[int, str]) -> TypeGuard[str]: pass +def main(a: Union[str, int]) -> None: + if is_foo(a): + reveal_type(a) # N: Revealed type is "builtins.str" + reveal_type(a) # N: Revealed type is "Union[builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardUnionOut] +from typing import Union +from typing_extensions import TypeGuard +def is_foo(a: object) -> TypeGuard[Union[int, str]]: pass +def main(a: object) -> None: + if is_foo(a): + reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNonzeroFloat] +from typing_extensions import TypeGuard +def is_nonzero(a: object) -> TypeGuard[float]: pass +def main(a: int): + if is_nonzero(a): + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardHigherOrder] +from typing import Callable, TypeVar, Iterable, List +from typing_extensions import TypeGuard +T = TypeVar('T') +R = TypeVar('R') +def filter(f: Callable[[T], TypeGuard[R]], it: Iterable[T]) -> Iterable[R]: pass +def is_float(a: object) -> TypeGuard[float]: pass +a: List[object] = ["a", 0, 0.0] +b = filter(is_float, a) +reveal_type(b) # N: Revealed type is "typing.Iterable[builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardMethod] +from typing_extensions import TypeGuard +class C: + def main(self, a: object) -> None: + if self.is_float(a): + reveal_type(self) # N: Revealed type is "__main__.C" + reveal_type(a) # N: Revealed type is "builtins.float" + def is_float(self, a: object) -> TypeGuard[float]: pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardCrossModule] +import guard +from points import Point +def main(a: object) -> None: + if guard.is_point(a): + reveal_type(a) # N: Revealed type is "points.Point" +[file guard.py] +from typing_extensions import TypeGuard +import points +def is_point(a: object) -> TypeGuard[points.Point]: pass +[file points.py] +class Point: pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardBodyRequiresBool] +from typing_extensions import TypeGuard +def is_float(a: object) -> TypeGuard[float]: + return "not a bool" # E: Incompatible return value type (got "str", expected "bool") +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNarrowToTypedDict] +from typing import Dict, TypedDict +from typing_extensions import TypeGuard +class User(TypedDict): + name: str + id: int +def is_user(a: Dict[str, object]) -> TypeGuard[User]: + return isinstance(a.get("name"), str) and isinstance(a.get("id"), int) +def main(a: Dict[str, object]) -> None: + if is_user(a): + reveal_type(a) # N: Revealed type is "TypedDict('__main__.User', {'name': builtins.str, 'id': builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypeGuardInAssert] +from typing_extensions import TypeGuard +def is_float(a: object) -> TypeGuard[float]: pass +def main(a: object) -> None: + assert is_float(a) + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardFromAny] +from typing import Any +from typing_extensions import TypeGuard +def is_objfloat(a: object) -> TypeGuard[float]: pass +def is_anyfloat(a: Any) -> TypeGuard[float]: pass +def objmain(a: object) -> None: + if is_objfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" + if is_anyfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" +def anymain(a: Any) -> None: + if is_objfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" + if is_anyfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNegatedAndElse] +from typing import Union +from typing_extensions import TypeGuard +def is_int(a: object) -> TypeGuard[int]: pass +def is_str(a: object) -> TypeGuard[str]: pass +def intmain(a: Union[int, str]) -> None: + if not is_int(a): + reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(a) # N: Revealed type is "builtins.int" +def strmain(a: Union[int, str]) -> None: + if is_str(a): + reveal_type(a) # N: Revealed type is "builtins.str" + else: + reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardClassMethod] +from typing_extensions import TypeGuard +class C: + @classmethod + def is_float(cls, a: object) -> TypeGuard[float]: pass + def method(self, a: object) -> None: + if self.is_float(a): + reveal_type(a) # N: Revealed type is "builtins.float" +def main(a: object) -> None: + if C.is_float(a): + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/classmethod.pyi] + +[case testTypeGuardRequiresPositionalArgs] +from typing_extensions import TypeGuard +def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass +def main1(a: object) -> None: + if is_float(a=a, b=1): + reveal_type(a) # N: Revealed type is "builtins.float" + + if is_float(b=1, a=a): + reveal_type(a) # N: Revealed type is "builtins.float" + + # This is debatable -- should we support these cases? + + ta = (a,) + if is_float(*ta): # E: Type guard requires positional argument + reveal_type(ta) # N: Revealed type is "Tuple[builtins.object]" + reveal_type(a) # N: Revealed type is "builtins.object" + + la = [a] + if is_float(*la): # E: Type guard requires positional argument + reveal_type(la) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(a) # N: Revealed type is "builtins.object" + +[builtins fixtures/tuple.pyi] + +[case testTypeGuardOverload] +from typing import overload, Any, Callable, Iterable, Iterator, List, Optional, TypeVar +from typing_extensions import TypeGuard + +T = TypeVar("T") +R = TypeVar("R") + +@overload +def filter(f: Callable[[T], TypeGuard[R]], it: Iterable[T]) -> Iterator[R]: ... +@overload +def filter(f: Callable[[T], bool], it: Iterable[T]) -> Iterator[T]: ... +def filter(*args): pass + +def is_int_typeguard(a: object) -> TypeGuard[int]: pass +def is_int_bool(a: object) -> bool: pass + +def main(a: List[Optional[int]]) -> None: + bb = filter(lambda x: x is not None, a) + reveal_type(bb) # N: Revealed type is "typing.Iterator[Union[builtins.int, None]]" + # Also, if you replace 'bool' with 'Any' in the second overload, bb is Iterator[Any] + cc = filter(is_int_typeguard, a) + reveal_type(cc) # N: Revealed type is "typing.Iterator[builtins.int]" + dd = filter(is_int_bool, a) + reveal_type(dd) # N: Revealed type is "typing.Iterator[Union[builtins.int, None]]" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeGuardDecorated] +from typing import TypeVar +from typing_extensions import TypeGuard +T = TypeVar("T") +def decorator(f: T) -> T: pass +@decorator +def is_float(a: object) -> TypeGuard[float]: + pass +def main(a: object) -> None: + if is_float(a): + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardMethodOverride] +from typing_extensions import TypeGuard +class C: + def is_float(self, a: object) -> TypeGuard[float]: pass +class D(C): + def is_float(self, a: object) -> bool: pass # Fail +[builtins fixtures/tuple.pyi] +[out] +main:5: error: Signature of "is_float" incompatible with supertype "C" +main:5: note: Superclass: +main:5: note: def is_float(self, a: object) -> TypeGuard[float] +main:5: note: Subclass: +main:5: note: def is_float(self, a: object) -> bool + +[case testTypeGuardInAnd] +from typing import Any +from typing_extensions import TypeGuard +import types +def isclass(a: object) -> bool: + pass +def ismethod(a: object) -> TypeGuard[float]: + pass +def isfunction(a: object) -> TypeGuard[str]: + pass +def isclassmethod(obj: Any) -> bool: + if ismethod(obj) and obj.__self__ is not None and isclass(obj.__self__): # E: "float" has no attribute "__self__" + return True + + return False +def coverage(obj: Any) -> bool: + if not (ismethod(obj) or isfunction(obj)): + return True + return False +[builtins fixtures/classmethod.pyi] + +[case testAssignToTypeGuardedVariable1] +from typing_extensions import TypeGuard + +class A: pass +class B(A): pass + +def guard(a: A) -> TypeGuard[B]: + pass + +a = A() +if not guard(a): + a = A() +[builtins fixtures/tuple.pyi] + +[case testAssignToTypeGuardedVariable2] +from typing_extensions import TypeGuard + +class A: pass +class B: pass + +def guard(a: A) -> TypeGuard[B]: + pass + +a = A() +if not guard(a): + a = A() +[builtins fixtures/tuple.pyi] + +[case testAssignToTypeGuardedVariable3] +from typing_extensions import TypeGuard + +class A: pass +class B: pass + +def guard(a: A) -> TypeGuard[B]: + pass + +a = A() +if guard(a): + reveal_type(a) # N: Revealed type is "__main__.B" + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(a) # N: Revealed type is "__main__.B" + a = A() + reveal_type(a) # N: Revealed type is "__main__.A" +reveal_type(a) # N: Revealed type is "__main__.A" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNestedRestrictionAny] +from typing_extensions import TypeGuard +from typing import Any + +class A: ... +def f(x: object) -> TypeGuard[A]: ... +def g(x: object) -> None: ... + +def test(x: Any) -> None: + if not(f(x) or x): + return + g(reveal_type(x)) # N: Revealed type is "Union[__main__.A, Any]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNestedRestrictionUnionOther] +from typing_extensions import TypeGuard +from typing import Any + +class A: ... +class B: ... +def f(x: object) -> TypeGuard[A]: ... +def f2(x: object) -> TypeGuard[B]: ... +def g(x: object) -> None: ... + +def test(x: object) -> None: + if not(f(x) or f2(x)): + return + g(reveal_type(x)) # N: Revealed type is "Union[__main__.A, __main__.B]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardComprehensionSubtype] +from typing import List +from typing_extensions import TypeGuard + +class Base: ... +class Foo(Base): ... +class Bar(Base): ... + +def is_foo(item: object) -> TypeGuard[Foo]: + return isinstance(item, Foo) + +def is_bar(item: object) -> TypeGuard[Bar]: + return isinstance(item, Bar) + +def foobar(items: List[object]): + a: List[Base] = [x for x in items if is_foo(x) or is_bar(x)] + b: List[Base] = [x for x in items if is_foo(x)] + c: List[Bar] = [x for x in items if is_foo(x)] # E: List comprehension has incompatible type List[Foo]; expected List[Bar] +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNestedRestrictionUnionIsInstance] +from typing_extensions import TypeGuard +from typing import Any, List + +class A: ... +def f(x: List[object]) -> TypeGuard[List[str]]: ... +def g(x: object) -> None: ... + +def test(x: List[object]) -> None: + if not(f(x) or isinstance(x, A)): + return + g(reveal_type(x)) # N: Revealed type is "Union[builtins.list[builtins.str], __main__.]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardMultipleCondition-xfail] +from typing_extensions import TypeGuard +from typing import Any, List + +class Foo: ... +class Bar: ... + +def is_foo(item: object) -> TypeGuard[Foo]: + return isinstance(item, Foo) + +def is_bar(item: object) -> TypeGuard[Bar]: + return isinstance(item, Bar) + +def foobar(x: object): + if not isinstance(x, Foo) or not isinstance(x, Bar): + return + reveal_type(x) # N: Revealed type is "__main__." + +def foobar_typeguard(x: object): + if not is_foo(x) or not is_bar(x): + return + reveal_type(x) # N: Revealed type is "__main__." +[builtins fixtures/tuple.pyi] + +[case testTypeGuardAsFunctionArgAsBoolSubtype] +from typing import Callable +from typing_extensions import TypeGuard + +def accepts_bool(f: Callable[[object], bool]): pass + +def with_bool_typeguard(o: object) -> TypeGuard[bool]: pass +def with_str_typeguard(o: object) -> TypeGuard[str]: pass +def with_bool(o: object) -> bool: pass + +accepts_bool(with_bool_typeguard) +accepts_bool(with_str_typeguard) +accepts_bool(with_bool) +[builtins fixtures/tuple.pyi] + +[case testTypeGuardAsFunctionArg] +from typing import Callable +from typing_extensions import TypeGuard + +def accepts_typeguard(f: Callable[[object], TypeGuard[bool]]): pass +def different_typeguard(f: Callable[[object], TypeGuard[str]]): pass + +def with_typeguard(o: object) -> TypeGuard[bool]: pass +def with_bool(o: object) -> bool: pass + +accepts_typeguard(with_typeguard) +accepts_typeguard(with_bool) # E: Argument 1 to "accepts_typeguard" has incompatible type "Callable[[object], bool]"; expected "Callable[[object], TypeGuard[bool]]" + +different_typeguard(with_typeguard) # E: Argument 1 to "different_typeguard" has incompatible type "Callable[[object], TypeGuard[bool]]"; expected "Callable[[object], TypeGuard[str]]" +different_typeguard(with_bool) # E: Argument 1 to "different_typeguard" has incompatible type "Callable[[object], bool]"; expected "Callable[[object], TypeGuard[str]]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardAsGenericFunctionArg] +from typing import Callable, TypeVar +from typing_extensions import TypeGuard + +T = TypeVar('T') + +def accepts_typeguard(f: Callable[[object], TypeGuard[T]]): pass + +def with_bool_typeguard(o: object) -> TypeGuard[bool]: pass +def with_str_typeguard(o: object) -> TypeGuard[str]: pass +def with_bool(o: object) -> bool: pass + +accepts_typeguard(with_bool_typeguard) +accepts_typeguard(with_str_typeguard) +accepts_typeguard(with_bool) # E: Argument 1 to "accepts_typeguard" has incompatible type "Callable[[object], bool]"; expected "Callable[[object], TypeGuard[Never]]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardAsOverloadedFunctionArg] +# https://github.com/python/mypy/issues/11307 +from typing import Callable, TypeVar, Generic, Any, overload +from typing_extensions import TypeGuard + +_T = TypeVar('_T') + +class filter(Generic[_T]): + @overload + def __init__(self, function: Callable[[object], TypeGuard[_T]]) -> None: pass + @overload + def __init__(self, function: Callable[[_T], Any]) -> None: pass + def __init__(self, function): pass + +def is_int_typeguard(a: object) -> TypeGuard[int]: pass +def returns_bool(a: object) -> bool: pass + +reveal_type(filter(is_int_typeguard)) # N: Revealed type is "__main__.filter[builtins.int]" +reveal_type(filter(returns_bool)) # N: Revealed type is "__main__.filter[builtins.object]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardSubtypingVariance] +from typing import Callable +from typing_extensions import TypeGuard + +class A: pass +class B(A): pass +class C(B): pass + +def accepts_typeguard(f: Callable[[object], TypeGuard[B]]): pass + +def with_typeguard_a(o: object) -> TypeGuard[A]: pass +def with_typeguard_b(o: object) -> TypeGuard[B]: pass +def with_typeguard_c(o: object) -> TypeGuard[C]: pass + +accepts_typeguard(with_typeguard_a) # E: Argument 1 to "accepts_typeguard" has incompatible type "Callable[[object], TypeGuard[A]]"; expected "Callable[[object], TypeGuard[B]]" +accepts_typeguard(with_typeguard_b) +accepts_typeguard(with_typeguard_c) +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWithIdentityGeneric] +from typing import TypeVar +from typing_extensions import TypeGuard + +_T = TypeVar("_T") + +def identity(val: _T) -> TypeGuard[_T]: + pass + +def func1(name: _T): + reveal_type(name) # N: Revealed type is "_T`-1" + if identity(name): + reveal_type(name) # N: Revealed type is "_T`-1" + +def func2(name: str): + reveal_type(name) # N: Revealed type is "builtins.str" + if identity(name): + reveal_type(name) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWithGenericInstance] +from typing import TypeVar, List +from typing_extensions import TypeGuard + +_T = TypeVar("_T") + +def is_list_of_str(val: _T) -> TypeGuard[List[_T]]: + pass + +def func(name: str): + reveal_type(name) # N: Revealed type is "builtins.str" + if is_list_of_str(name): + reveal_type(name) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWithTupleGeneric] +from typing import TypeVar, Tuple +from typing_extensions import TypeGuard + +_T = TypeVar("_T") + +def is_two_element_tuple(val: Tuple[_T, ...]) -> TypeGuard[Tuple[_T, _T]]: + pass + +def func(names: Tuple[str, ...]): + reveal_type(names) # N: Revealed type is "builtins.tuple[builtins.str, ...]" + if is_two_element_tuple(names): + reveal_type(names) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardErroneousDefinitionFails] +from typing_extensions import TypeGuard + +class Z: + def typeguard1(self, *, x: object) -> TypeGuard[int]: # line 4 + ... + + @staticmethod + def typeguard2(x: object) -> TypeGuard[int]: + ... + + @staticmethod # line 11 + def typeguard3(*, x: object) -> TypeGuard[int]: + ... + +def bad_typeguard(*, x: object) -> TypeGuard[int]: # line 15 + ... + +# In Python 3.8 the line number associated with FunctionDef nodes changed +[builtins fixtures/classmethod.pyi] +[out] +main:4: error: TypeGuard functions must have a positional argument +main:12: error: TypeGuard functions must have a positional argument +main:15: error: TypeGuard functions must have a positional argument + +[case testTypeGuardWithKeywordArg] +from typing_extensions import TypeGuard + +class Z: + def typeguard(self, x: object) -> TypeGuard[int]: + ... + +def typeguard(x: object) -> TypeGuard[int]: + ... + +n: object +if typeguard(x=n): + reveal_type(n) # N: Revealed type is "builtins.int" + +if Z().typeguard(x=n): + reveal_type(n) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testStaticMethodTypeGuard] +from typing_extensions import TypeGuard + +class Y: + @staticmethod + def typeguard(h: object) -> TypeGuard[int]: + ... + +x: object +if Y().typeguard(x): + reveal_type(x) # N: Revealed type is "builtins.int" +if Y.typeguard(x): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testTypeGuardKwargFollowingThroughOverloaded] +from typing import overload, Union +from typing_extensions import TypeGuard + +@overload +def typeguard(x: object, y: str) -> TypeGuard[str]: + ... + +@overload +def typeguard(x: object, y: int) -> TypeGuard[int]: + ... + +def typeguard(x: object, y: Union[int, str]) -> Union[TypeGuard[int], TypeGuard[str]]: + ... + +x: object +if typeguard(x=x, y=42): + reveal_type(x) # N: Revealed type is "builtins.int" + +if typeguard(y=42, x=x): + reveal_type(x) # N: Revealed type is "builtins.int" + +if typeguard(x=x, y="42"): + reveal_type(x) # N: Revealed type is "builtins.str" + +if typeguard(y="42", x=x): + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testGenericAliasWithTypeGuard] +from typing import Callable, List, TypeVar +from typing_extensions import TypeGuard, TypeAlias + +A = Callable[[object], TypeGuard[List[T]]] +def foo(x: object) -> TypeGuard[List[str]]: ... + +def test(f: A[T]) -> T: ... +reveal_type(test(foo)) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + +[case testNoCrashOnDunderCallTypeGuard] +from typing_extensions import TypeGuard + +class A: + def __call__(self, x) -> TypeGuard[int]: + return True + +a: A +assert a(x=1) + +x: object +assert a(x=x) +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typeis.test b/test-data/unit/check-typeis.test new file mode 100644 index 000000000000..2372f990fda1 --- /dev/null +++ b/test-data/unit/check-typeis.test @@ -0,0 +1,819 @@ +[case testTypeIsBasic] +from typing_extensions import TypeIs +class Point: pass +def is_point(a: object) -> TypeIs[Point]: pass +def main(a: object) -> None: + if is_point(a): + reveal_type(a) # N: Revealed type is "__main__.Point" + else: + reveal_type(a) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] + +[case testTypeIsElif] +from typing_extensions import TypeIs +from typing import Union +class Point: pass +def is_point(a: object) -> TypeIs[Point]: pass +class Line: pass +def is_line(a: object) -> TypeIs[Line]: pass +def main(a: Union[Point, Line, int]) -> None: + if is_point(a): + reveal_type(a) # N: Revealed type is "__main__.Point" + elif is_line(a): + reveal_type(a) # N: Revealed type is "__main__.Line" + else: + reveal_type(a) # N: Revealed type is "builtins.int" + +[builtins fixtures/tuple.pyi] + +[case testTypeIsTypeArgsNone] +from typing_extensions import TypeIs +def foo(a: object) -> TypeIs: # E: TypeIs must have exactly one type argument + pass +[builtins fixtures/tuple.pyi] + +[case testTypeIsTypeArgsTooMany] +from typing_extensions import TypeIs +def foo(a: object) -> TypeIs[int, int]: # E: TypeIs must have exactly one type argument + pass +[builtins fixtures/tuple.pyi] + +[case testTypeIsTypeArgType] +from typing_extensions import TypeIs +def foo(a: object) -> TypeIs[42]: # E: Invalid type: try using Literal[42] instead? + pass +[builtins fixtures/tuple.pyi] + +[case testTypeIsRepr] +from typing_extensions import TypeIs +def foo(a: object) -> TypeIs[int]: + pass +reveal_type(foo) # N: Revealed type is "def (a: builtins.object) -> TypeIs[builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsCallArgsNone] +from typing_extensions import TypeIs +class Point: pass + +def is_point() -> TypeIs[Point]: pass # E: "TypeIs" functions must have a positional argument +def main(a: object) -> None: + if is_point(): + reveal_type(a) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] + +[case testTypeIsCallArgsMultiple] +from typing_extensions import TypeIs +class Point: pass +def is_point(a: object, b: object) -> TypeIs[Point]: pass +def main(a: object, b: object) -> None: + if is_point(a, b): + reveal_type(a) # N: Revealed type is "__main__.Point" + reveal_type(b) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] + +[case testTypeIsIsBool] +from typing_extensions import TypeIs +def f(a: TypeIs[int]) -> None: pass +reveal_type(f) # N: Revealed type is "def (a: builtins.bool)" +a: TypeIs[int] +reveal_type(a) # N: Revealed type is "builtins.bool" +class C: + a: TypeIs[int] +reveal_type(C().a) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testTypeIsWithTypeVar] +from typing import TypeVar, Tuple, Type +from typing_extensions import TypeIs +T = TypeVar('T') +def is_tuple_of_type(a: Tuple[object, ...], typ: Type[T]) -> TypeIs[Tuple[T, ...]]: pass +def main(a: Tuple[object, ...]): + if is_tuple_of_type(a, int): + reveal_type(a) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsTypeVarReturn] +from typing import Callable, Optional, TypeVar +from typing_extensions import TypeIs +T = TypeVar('T') +def is_str(x: object) -> TypeIs[str]: pass +def main(x: object, type_check_func: Callable[[object], TypeIs[T]]) -> T: + if not type_check_func(x): + raise Exception() + return x +reveal_type(main("a", is_str)) # N: Revealed type is "builtins.str" +[builtins fixtures/exception.pyi] + +[case testTypeIsPassedAsTypeVarIsBool] +from typing import Callable, TypeVar +from typing_extensions import TypeIs +T = TypeVar('T') +def is_str(x: object) -> TypeIs[str]: pass +def main(f: Callable[[object], T]) -> T: pass +reveal_type(main(is_str)) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testTypeIsUnionIn] +from typing import Union +from typing_extensions import TypeIs +def is_foo(a: Union[int, str]) -> TypeIs[str]: pass +def main(a: Union[str, int]) -> None: + if is_foo(a): + reveal_type(a) # N: Revealed type is "builtins.str" + else: + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(a) # N: Revealed type is "Union[builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsUnionOut] +from typing import Union +from typing_extensions import TypeIs +def is_foo(a: object) -> TypeIs[Union[int, str]]: pass +def main(a: object) -> None: + if is_foo(a): + reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsNonzeroFloat] +from typing_extensions import TypeIs +def is_nonzero(a: object) -> TypeIs[float]: pass +def main(a: int): + if is_nonzero(a): + reveal_type(a) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeIsHigherOrder] +from typing import Callable, TypeVar, Iterable, List +from typing_extensions import TypeIs +T = TypeVar('T') +R = TypeVar('R') +def filter(f: Callable[[T], TypeIs[R]], it: Iterable[T]) -> Iterable[R]: pass +def is_float(a: object) -> TypeIs[float]: pass +a: List[object] = ["a", 0, 0.0] +b = filter(is_float, a) +reveal_type(b) # N: Revealed type is "typing.Iterable[builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsMethod] +from typing_extensions import TypeIs +class C: + def main(self, a: object) -> None: + if self.is_float(a): + reveal_type(self) # N: Revealed type is "__main__.C" + reveal_type(a) # N: Revealed type is "builtins.float" + def is_float(self, a: object) -> TypeIs[float]: pass +[builtins fixtures/tuple.pyi] + +[case testTypeIsCrossModule] +import guard +from points import Point +def main(a: object) -> None: + if guard.is_point(a): + reveal_type(a) # N: Revealed type is "points.Point" +[file guard.py] +from typing_extensions import TypeIs +import points +def is_point(a: object) -> TypeIs[points.Point]: pass +[file points.py] +class Point: pass +[builtins fixtures/tuple.pyi] + +[case testTypeIsBodyRequiresBool] +from typing_extensions import TypeIs +def is_float(a: object) -> TypeIs[float]: + return "not a bool" # E: Incompatible return value type (got "str", expected "bool") +[builtins fixtures/tuple.pyi] + +[case testTypeIsNarrowToTypedDict] +from typing import Mapping, TypedDict +from typing_extensions import TypeIs +class User(TypedDict): + name: str + id: int +def is_user(a: Mapping[str, object]) -> TypeIs[User]: + return isinstance(a.get("name"), str) and isinstance(a.get("id"), int) +def main(a: Mapping[str, object]) -> None: + if is_user(a): + reveal_type(a) # N: Revealed type is "TypedDict('__main__.User', {'name': builtins.str, 'id': builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypeIsInAssert] +from typing_extensions import TypeIs +def is_float(a: object) -> TypeIs[float]: pass +def main(a: object) -> None: + assert is_float(a) + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] + +[case testTypeIsFromAny] +from typing import Any +from typing_extensions import TypeIs +def is_objfloat(a: object) -> TypeIs[float]: pass +def is_anyfloat(a: Any) -> TypeIs[float]: pass +def objmain(a: object) -> None: + if is_objfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" + if is_anyfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" +def anymain(a: Any) -> None: + if is_objfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" + if is_anyfloat(a): + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] + +[case testTypeIsNegatedAndElse] +from typing import Union +from typing_extensions import TypeIs +def is_int(a: object) -> TypeIs[int]: pass +def is_str(a: object) -> TypeIs[str]: pass +def intmain(a: Union[int, str]) -> None: + if not is_int(a): + reveal_type(a) # N: Revealed type is "builtins.str" + else: + reveal_type(a) # N: Revealed type is "builtins.int" +def strmain(a: Union[int, str]) -> None: + if is_str(a): + reveal_type(a) # N: Revealed type is "builtins.str" + else: + reveal_type(a) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeIsClassMethod] +from typing_extensions import TypeIs +class C: + @classmethod + def is_float(cls, a: object) -> TypeIs[float]: pass + def method(self, a: object) -> None: + if self.is_float(a): + reveal_type(a) # N: Revealed type is "builtins.float" +def main(a: object) -> None: + if C.is_float(a): + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/classmethod.pyi] + +[case testTypeIsRequiresPositionalArgs] +from typing_extensions import TypeIs +def is_float(a: object, b: object = 0) -> TypeIs[float]: pass +def main1(a: object) -> None: + if is_float(a=a, b=1): + reveal_type(a) # N: Revealed type is "builtins.float" + + if is_float(b=1, a=a): + reveal_type(a) # N: Revealed type is "builtins.float" + +[builtins fixtures/tuple.pyi] + +[case testTypeIsOverload] +from typing import overload, Any, Callable, Iterable, Iterator, List, Optional, TypeVar +from typing_extensions import TypeIs + +T = TypeVar("T") +R = TypeVar("R") + +@overload +def filter(f: Callable[[T], TypeIs[R]], it: Iterable[T]) -> Iterator[R]: ... +@overload +def filter(f: Callable[[T], bool], it: Iterable[T]) -> Iterator[T]: ... +def filter(*args): pass + +def is_int_typeis(a: object) -> TypeIs[int]: pass +def is_int_bool(a: object) -> bool: pass + +def main(a: List[Optional[int]]) -> None: + bb = filter(lambda x: x is not None, a) + reveal_type(bb) # N: Revealed type is "typing.Iterator[Union[builtins.int, None]]" + # Also, if you replace 'bool' with 'Any' in the second overload, bb is Iterator[Any] + cc = filter(is_int_typeis, a) + reveal_type(cc) # N: Revealed type is "typing.Iterator[builtins.int]" + dd = filter(is_int_bool, a) + reveal_type(dd) # N: Revealed type is "typing.Iterator[Union[builtins.int, None]]" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeIsDecorated] +from typing import TypeVar +from typing_extensions import TypeIs +T = TypeVar("T") +def decorator(f: T) -> T: pass +@decorator +def is_float(a: object) -> TypeIs[float]: + pass +def main(a: object) -> None: + if is_float(a): + reveal_type(a) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] + +[case testTypeIsMethodOverride] +from typing_extensions import TypeIs +class C: + def is_float(self, a: object) -> TypeIs[float]: pass +class D(C): + def is_float(self, a: object) -> bool: pass # Fail +[builtins fixtures/tuple.pyi] +[out] +main:5: error: Signature of "is_float" incompatible with supertype "C" +main:5: note: Superclass: +main:5: note: def is_float(self, a: object) -> TypeIs[float] +main:5: note: Subclass: +main:5: note: def is_float(self, a: object) -> bool + +[case testTypeIsInAnd] +from typing import Any +from typing_extensions import TypeIs +def isclass(a: object) -> bool: + pass +def isfloat(a: object) -> TypeIs[float]: + pass +def isstr(a: object) -> TypeIs[str]: + pass + +def coverage1(obj: Any) -> bool: + if isfloat(obj) and obj.__self__ is not None and isclass(obj.__self__): # E: "float" has no attribute "__self__" + reveal_type(obj) # N: Revealed type is "builtins.float" + return True + reveal_type(obj) # N: Revealed type is "Any" + return False + +def coverage2(obj: Any) -> bool: + if not (isfloat(obj) or isstr(obj)): + reveal_type(obj) # N: Revealed type is "Any" + return True + reveal_type(obj) # N: Revealed type is "Union[builtins.float, builtins.str]" + return False +[builtins fixtures/classmethod.pyi] + +[case testAssignToTypeIsedVariable1] +from typing_extensions import TypeIs + +class A: pass +class B(A): pass + +def guard(a: A) -> TypeIs[B]: + pass + +a = A() +if not guard(a): + a = A() +[builtins fixtures/tuple.pyi] + +[case testAssignToTypeIsedVariable2] +from typing_extensions import TypeIs + +class A: pass +class B: pass + +def guard(a: object) -> TypeIs[B]: + pass + +a = A() +if not guard(a): + a = A() +[builtins fixtures/tuple.pyi] + +[case testAssignToTypeIsedVariable3] +from typing_extensions import TypeIs + +class A: pass +class B: pass + +def guard(a: object) -> TypeIs[B]: + pass + +a = A() +if guard(a): + reveal_type(a) # N: Revealed type is "__main__." + a = B() # E: Incompatible types in assignment (expression has type "B", variable has type "A") + reveal_type(a) # N: Revealed type is "__main__." + a = A() + reveal_type(a) # N: Revealed type is "__main__.A" +reveal_type(a) # N: Revealed type is "__main__.A" +[builtins fixtures/tuple.pyi] + +[case testTypeIsNestedRestrictionAny] +from typing_extensions import TypeIs +from typing import Any + +class A: ... +def f(x: object) -> TypeIs[A]: ... +def g(x: object) -> None: ... + +def test(x: Any) -> None: + if not(f(x) or x): + return + g(reveal_type(x)) # N: Revealed type is "Union[__main__.A, Any]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsNestedRestrictionUnionOther] +from typing_extensions import TypeIs +from typing import Any + +class A: ... +class B: ... +def f(x: object) -> TypeIs[A]: ... +def f2(x: object) -> TypeIs[B]: ... +def g(x: object) -> None: ... + +def test(x: object) -> None: + if not(f(x) or f2(x)): + return + g(reveal_type(x)) # N: Revealed type is "Union[__main__.A, __main__.B]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsComprehensionSubtype] +from typing import List +from typing_extensions import TypeIs + +class Base: ... +class Foo(Base): ... +class Bar(Base): ... + +def is_foo(item: object) -> TypeIs[Foo]: + return isinstance(item, Foo) + +def is_bar(item: object) -> TypeIs[Bar]: + return isinstance(item, Bar) + +def foobar(items: List[object]): + a: List[Base] = [x for x in items if is_foo(x) or is_bar(x)] + b: List[Base] = [x for x in items if is_foo(x)] + c: List[Foo] = [x for x in items if is_foo(x)] + d: List[Bar] = [x for x in items if is_foo(x)] # E: List comprehension has incompatible type List[Foo]; expected List[Bar] +[builtins fixtures/tuple.pyi] + +[case testTypeIsNestedRestrictionUnionIsInstance] +from typing_extensions import TypeIs +from typing import Any, List + +class A: ... +def f(x: List[Any]) -> TypeIs[List[str]]: ... +def g(x: object) -> None: ... + +def test(x: List[Any]) -> None: + if not(f(x) or isinstance(x, A)): + return + g(reveal_type(x)) # N: Revealed type is "Union[builtins.list[builtins.str], __main__.]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsMultipleCondition] +from typing_extensions import TypeIs +from typing import Any, List + +class Foo: ... +class Bar: ... + +def is_foo(item: object) -> TypeIs[Foo]: + return isinstance(item, Foo) + +def is_bar(item: object) -> TypeIs[Bar]: + return isinstance(item, Bar) + +def foobar(x: object): + if not isinstance(x, Foo) or not isinstance(x, Bar): + return + reveal_type(x) # N: Revealed type is "__main__." + +def foobar_typeis(x: object): + if not is_foo(x) or not is_bar(x): + return + # Looks like a typo but this is what our unique name generation produces + reveal_type(x) # N: Revealed type is "__main__.1" +[builtins fixtures/tuple.pyi] + +[case testTypeIsAsFunctionArgAsBoolSubtype] +from typing import Callable +from typing_extensions import TypeIs + +def accepts_bool(f: Callable[[object], bool]): pass + +def with_bool_typeis(o: object) -> TypeIs[bool]: pass +def with_str_typeis(o: object) -> TypeIs[str]: pass +def with_bool(o: object) -> bool: pass + +accepts_bool(with_bool_typeis) +accepts_bool(with_str_typeis) +accepts_bool(with_bool) +[builtins fixtures/tuple.pyi] + +[case testTypeIsAsFunctionArg] +from typing import Callable +from typing_extensions import TypeIs + +def accepts_typeis(f: Callable[[object], TypeIs[bool]]): pass +def different_typeis(f: Callable[[object], TypeIs[str]]): pass + +def with_typeis(o: object) -> TypeIs[bool]: pass +def with_bool(o: object) -> bool: pass + +accepts_typeis(with_typeis) +accepts_typeis(with_bool) # E: Argument 1 to "accepts_typeis" has incompatible type "Callable[[object], bool]"; expected "Callable[[object], TypeIs[bool]]" + +different_typeis(with_typeis) # E: Argument 1 to "different_typeis" has incompatible type "Callable[[object], TypeIs[bool]]"; expected "Callable[[object], TypeIs[str]]" +different_typeis(with_bool) # E: Argument 1 to "different_typeis" has incompatible type "Callable[[object], bool]"; expected "Callable[[object], TypeIs[str]]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsAsGenericFunctionArg] +from typing import Callable, TypeVar +from typing_extensions import TypeIs + +T = TypeVar('T') + +def accepts_typeis(f: Callable[[object], TypeIs[T]]): pass + +def with_bool_typeis(o: object) -> TypeIs[bool]: pass +def with_str_typeis(o: object) -> TypeIs[str]: pass +def with_bool(o: object) -> bool: pass + +accepts_typeis(with_bool_typeis) +accepts_typeis(with_str_typeis) +accepts_typeis(with_bool) # E: Argument 1 to "accepts_typeis" has incompatible type "Callable[[object], bool]"; expected "Callable[[object], TypeIs[Never]]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsAsOverloadedFunctionArg] +# https://github.com/python/mypy/issues/11307 +from typing import Callable, TypeVar, Generic, Any, overload +from typing_extensions import TypeIs + +_T = TypeVar('_T') + +class filter(Generic[_T]): + @overload + def __init__(self, function: Callable[[object], TypeIs[_T]]) -> None: pass + @overload + def __init__(self, function: Callable[[_T], Any]) -> None: pass + def __init__(self, function): pass + +def is_int_typeis(a: object) -> TypeIs[int]: pass +def returns_bool(a: object) -> bool: pass + +reveal_type(filter(is_int_typeis)) # N: Revealed type is "__main__.filter[builtins.int]" +reveal_type(filter(returns_bool)) # N: Revealed type is "__main__.filter[builtins.object]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsSubtypingVariance] +from typing import Callable +from typing_extensions import TypeIs + +class A: pass +class B(A): pass +class C(B): pass + +def accepts_typeis(f: Callable[[object], TypeIs[B]]): pass + +def with_typeis_a(o: object) -> TypeIs[A]: pass +def with_typeis_b(o: object) -> TypeIs[B]: pass +def with_typeis_c(o: object) -> TypeIs[C]: pass + +accepts_typeis(with_typeis_a) # E: Argument 1 to "accepts_typeis" has incompatible type "Callable[[object], TypeIs[A]]"; expected "Callable[[object], TypeIs[B]]" +accepts_typeis(with_typeis_b) +accepts_typeis(with_typeis_c) # E: Argument 1 to "accepts_typeis" has incompatible type "Callable[[object], TypeIs[C]]"; expected "Callable[[object], TypeIs[B]]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsWithIdentityGeneric] +from typing import TypeVar +from typing_extensions import TypeIs + +_T = TypeVar("_T") + +def identity(val: _T) -> TypeIs[_T]: + pass + +def func1(name: _T): + reveal_type(name) # N: Revealed type is "_T`-1" + if identity(name): + reveal_type(name) # N: Revealed type is "_T`-1" + +def func2(name: str): + reveal_type(name) # N: Revealed type is "builtins.str" + if identity(name): + reveal_type(name) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testTypeIsWithGenericOnSecondParam] +from typing import TypeVar +from typing_extensions import TypeIs + +_R = TypeVar("_R") + +def guard(val: object, param: _R) -> TypeIs[_R]: + pass + +def func1(name: object): + reveal_type(name) # N: Revealed type is "builtins.object" + if guard(name, name): + reveal_type(name) # N: Revealed type is "builtins.object" + if guard(name, 1): + reveal_type(name) # N: Revealed type is "builtins.int" + +def func2(name: int): + reveal_type(name) # N: Revealed type is "builtins.int" + if guard(name, True): + reveal_type(name) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testTypeIsWithGenericInstance] +from typing import TypeVar, List, Iterable +from typing_extensions import TypeIs + +_T = TypeVar("_T") + +def is_list_of_str(val: Iterable[_T]) -> TypeIs[List[_T]]: + pass + +def func(name: Iterable[str]): + reveal_type(name) # N: Revealed type is "typing.Iterable[builtins.str]" + if is_list_of_str(name): + reveal_type(name) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsWithTupleGeneric] +from typing import TypeVar, Tuple +from typing_extensions import TypeIs + +_T = TypeVar("_T") + +def is_two_element_tuple(val: Tuple[_T, ...]) -> TypeIs[Tuple[_T, _T]]: + pass + +def func(names: Tuple[str, ...]): + reveal_type(names) # N: Revealed type is "builtins.tuple[builtins.str, ...]" + if is_two_element_tuple(names): + reveal_type(names) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsErroneousDefinitionFails] +from typing_extensions import TypeIs + +class Z: + def typeis1(self, *, x: object) -> TypeIs[int]: # E: "TypeIs" functions must have a positional argument + ... + + @staticmethod + def typeis2(x: object) -> TypeIs[int]: + ... + + @staticmethod + def typeis3(*, x: object) -> TypeIs[int]: # E: "TypeIs" functions must have a positional argument + ... + +def bad_typeis(*, x: object) -> TypeIs[int]: # E: "TypeIs" functions must have a positional argument + ... + +[builtins fixtures/classmethod.pyi] + +[case testTypeIsWithKeywordArg] +from typing_extensions import TypeIs + +class Z: + def typeis(self, x: object) -> TypeIs[int]: + ... + +def typeis(x: object) -> TypeIs[int]: + ... + +n: object +if typeis(x=n): + reveal_type(n) # N: Revealed type is "builtins.int" + +if Z().typeis(x=n): + reveal_type(n) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testStaticMethodTypeIs] +from typing_extensions import TypeIs + +class Y: + @staticmethod + def typeis(h: object) -> TypeIs[int]: + ... + +x: object +if Y().typeis(x): + reveal_type(x) # N: Revealed type is "builtins.int" +if Y.typeis(x): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/classmethod.pyi] + +[case testTypeIsKwargFollowingThroughOverloaded] +from typing import overload, Union +from typing_extensions import TypeIs + +@overload +def typeis(x: object, y: str) -> TypeIs[str]: + ... + +@overload +def typeis(x: object, y: int) -> TypeIs[int]: + ... + +def typeis(x: object, y: Union[int, str]) -> Union[TypeIs[int], TypeIs[str]]: + ... + +x: object +if typeis(x=x, y=42): + reveal_type(x) # N: Revealed type is "builtins.int" + +if typeis(y=42, x=x): + reveal_type(x) # N: Revealed type is "builtins.int" + +if typeis(x=x, y="42"): + reveal_type(x) # N: Revealed type is "builtins.str" + +if typeis(y="42", x=x): + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testGenericAliasWithTypeIs] +from typing import Callable, List, TypeVar +from typing_extensions import TypeIs + +T = TypeVar('T') +A = Callable[[object], TypeIs[List[T]]] +def foo(x: object) -> TypeIs[List[str]]: ... + +def test(f: A[T]) -> T: ... +reveal_type(test(foo)) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + +[case testNoCrashOnDunderCallTypeIs] +from typing_extensions import TypeIs + +class A: + def __call__(self, x) -> TypeIs[int]: + return True + +a: A +assert a(x=1) + +x: object +assert a(x=x) +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeIsMustBeSubtypeFunctions] +from typing_extensions import TypeIs +from typing import List, Sequence, TypeVar + +def f(x: str) -> TypeIs[int]: # E: Narrowed type "int" is not a subtype of input type "str" + pass + +T = TypeVar('T') + +def g(x: List[T]) -> TypeIs[Sequence[T]]: # E: Narrowed type "Sequence[T]" is not a subtype of input type "List[T]" + pass + +[builtins fixtures/tuple.pyi] + +[case testTypeIsMustBeSubtypeMethods] +from typing_extensions import TypeIs + +class NarrowHolder: + @classmethod + def cls_narrower_good(cls, x: object) -> TypeIs[int]: + pass + + @classmethod + def cls_narrower_bad(cls, x: str) -> TypeIs[int]: # E: Narrowed type "int" is not a subtype of input type "str" + pass + + @staticmethod + def static_narrower_good(x: object) -> TypeIs[int]: + pass + + @staticmethod + def static_narrower_bad(x: str) -> TypeIs[int]: # E: Narrowed type "int" is not a subtype of input type "str" + pass + + def inst_narrower_good(self, x: object) -> TypeIs[int]: + pass + + def inst_narrower_bad(self, x: str) -> TypeIs[int]: # E: Narrowed type "int" is not a subtype of input type "str" + pass + + +[builtins fixtures/classmethod.pyi] + +[case testTypeIsTypeGuardNoSubtyping] +from typing_extensions import TypeGuard, TypeIs +from typing import Callable + +def accept_typeis(x: Callable[[object], TypeIs[str]]): + pass + +def accept_typeguard(x: Callable[[object], TypeGuard[str]]): + pass + +def typeis(x: object) -> TypeIs[str]: + pass + +def typeguard(x: object) -> TypeGuard[str]: + pass + +accept_typeis(typeis) +accept_typeis(typeguard) # E: Argument 1 to "accept_typeis" has incompatible type "Callable[[object], TypeGuard[str]]"; expected "Callable[[object], TypeIs[str]]" +accept_typeguard(typeis) # E: Argument 1 to "accept_typeguard" has incompatible type "Callable[[object], TypeIs[str]]"; expected "Callable[[object], TypeGuard[str]]" +accept_typeguard(typeguard) + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test new file mode 100644 index 000000000000..93d20eb26f6e --- /dev/null +++ b/test-data/unit/check-typevar-defaults.test @@ -0,0 +1,751 @@ +[case testTypeVarDefaultsBasic] +from typing import Generic, TypeVar, ParamSpec, Callable, Tuple, List +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1", default=int) +P1 = ParamSpec("P1", default=[int, str]) +Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int, str]]) + +def f1(a: T1) -> List[T1]: ... +reveal_type(f1) # N: Revealed type is "def [T1 = builtins.int] (a: T1`-1 = builtins.int) -> builtins.list[T1`-1 = builtins.int]" + +def f2(a: Callable[P1, None]) -> Callable[P1, None]: ... +reveal_type(f2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] (a: def (*P1.args, **P1.kwargs)) -> def (*P1.args, **P1.kwargs)" + +def f3(a: Tuple[Unpack[Ts1]]) -> Tuple[Unpack[Ts1]]: ... +reveal_type(f3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] (a: Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]) -> Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]" + + +class ClassA1(Generic[T1]): ... +class ClassA2(Generic[P1]): ... +class ClassA3(Generic[Unpack[Ts1]]): ... + +reveal_type(ClassA1) # N: Revealed type is "def [T1 = builtins.int] () -> __main__.ClassA1[T1`1 = builtins.int]" +reveal_type(ClassA2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] () -> __main__.ClassA2[P1`1 = [builtins.int, builtins.str]]" +reveal_type(ClassA3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] () -> __main__.ClassA3[Unpack[Ts1`1 = Unpack[Tuple[builtins.int, builtins.str]]]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsValid] +from typing import TypeVar, ParamSpec, Any, List, Tuple +from typing_extensions import TypeVarTuple, Unpack + +S0 = TypeVar("S0") +S1 = TypeVar("S1", bound=int) + +P0 = ParamSpec("P0") +Ts0 = TypeVarTuple("Ts0") + +T1 = TypeVar("T1", default=int) +T2 = TypeVar("T2", bound=float, default=int) +T3 = TypeVar("T3", bound=List[Any], default=List[int]) +T4 = TypeVar("T4", int, str, default=int) +T5 = TypeVar("T5", default=S0) +T6 = TypeVar("T6", bound=float, default=S1) +# T7 = TypeVar("T7", bound=List[Any], default=List[S0]) # TODO + +P1 = ParamSpec("P1", default=[]) +P2 = ParamSpec("P2", default=...) +P3 = ParamSpec("P3", default=[int, str]) +P4 = ParamSpec("P4", default=P0) + +Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int]]) +Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, ...]]) +# Ts3 = TypeVarTuple("Ts3", default=Unpack[Ts0]) # TODO +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsInvalid] +from typing import TypeVar, ParamSpec, Tuple +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1", default=2) # E: TypeVar "default" must be a type +T2 = TypeVar("T2", default=[int]) # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type + +P1 = ParamSpec("P1", default=int) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +P2 = ParamSpec("P2", default=2) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +P3 = ParamSpec("P3", default=(2, int)) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +P4 = ParamSpec("P4", default=[2, int]) # E: Argument 0 of ParamSpec default must be a type + +Ts1 = TypeVarTuple("Ts1", default=2) # E: The default argument to TypeVarTuple must be an Unpacked tuple +Ts2 = TypeVarTuple("Ts2", default=int) # E: The default argument to TypeVarTuple must be an Unpacked tuple +Ts3 = TypeVarTuple("Ts3", default=Tuple[int]) # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsInvalid2] +from typing import TypeVar, List, Union + +T1 = TypeVar("T1", bound=str, default=int) # E: TypeVar default must be a subtype of the bound type +T2 = TypeVar("T2", bound=List[str], default=List[int]) # E: TypeVar default must be a subtype of the bound type +T3 = TypeVar("T3", int, str, default=bytes) # E: TypeVar default must be one of the constraint types +T4 = TypeVar("T4", int, str, default=Union[int, str]) # E: TypeVar default must be one of the constraint types +T5 = TypeVar("T5", float, str, default=int) # E: TypeVar default must be one of the constraint types + +[case testTypeVarDefaultsInvalid3] +from typing import Dict, Generic, TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=T3) # E: Name "T3" is used before definition +T3 = TypeVar("T3", default=str) +T4 = TypeVar("T4", default=T3) + +class ClassError1(Generic[T3, T1]): ... # E: "T1" cannot appear after "T3" in type parameter list because it has no default type + +def func_error1( + a: ClassError1, + b: ClassError1[int], + c: ClassError1[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassError1[builtins.str, Any]" + reveal_type(b) # N: Revealed type is "__main__.ClassError1[builtins.int, Any]" + reveal_type(c) # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]" + + k = ClassError1() + reveal_type(k) # N: Revealed type is "__main__.ClassError1[builtins.str, Any]" + l = ClassError1[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassError1[builtins.int, Any]" + m = ClassError1[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]" + +class ClassError2(Generic[T4, T3]): ... # E: Type parameter "T4" has a default type that refers to one or more type variables that are out of scope + +def func_error2( + a: ClassError2, + b: ClassError2[int], + c: ClassError2[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassError2[Any, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]" + + k = ClassError2() + reveal_type(k) # N: Revealed type is "__main__.ClassError2[Any, builtins.str]" + l = ClassError2[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]" + m = ClassError2[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]" + +TERR1 = Dict[T3, T1] # E: "T1" cannot appear after "T3" in type parameter list because it has no default type + +def func_error_alias1( + a: TERR1, + b: TERR1[int], + c: TERR1[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.str, Any]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, Any]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]" + +TERR2 = Dict[T4, T3] # TODO should be an error \ + # Type parameter "T4" has a default type that refers to one or more type variables that are out of scope + +def func_error_alias2( + a: TERR2, + b: TERR2[int], + c: TERR2[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[Any, builtins.str]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]" +[builtins fixtures/dict.pyi] + +[case testTypeVarDefaultsFunctions] +from typing import TypeVar, ParamSpec, List, Union, Callable, Tuple +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1", default=str) +T2 = TypeVar("T2", bound=str, default=str) +T3 = TypeVar("T3", bytes, str, default=str) +P1 = ParamSpec("P1", default=[int, str]) +Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int, str]]) + +def callback1(x: str) -> None: ... + +def func_a1(x: Union[int, T1]) -> T1: ... +reveal_type(func_a1(2)) # N: Revealed type is "builtins.str" +reveal_type(func_a1(2.1)) # N: Revealed type is "builtins.float" + +def func_a2(x: Union[int, T1]) -> List[T1]: ... +reveal_type(func_a2(2)) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type(func_a2(2.1)) # N: Revealed type is "builtins.list[builtins.float]" + +def func_a3(x: Union[int, T2]) -> T2: ... +reveal_type(func_a3(2)) # N: Revealed type is "builtins.str" + +def func_a4(x: Union[int, T3]) -> T3: ... +reveal_type(func_a4(2)) # N: Revealed type is "builtins.str" + +def func_b1(x: Union[int, Callable[P1, None]]) -> Callable[P1, None]: ... +reveal_type(func_b1(callback1)) # N: Revealed type is "def (x: builtins.str)" +reveal_type(func_b1(2)) # N: Revealed type is "def (builtins.int, builtins.str)" + +def func_c1(x: Union[int, Callable[[Unpack[Ts1]], None]]) -> Tuple[Unpack[Ts1]]: ... +# reveal_type(func_c1(callback1)) # Revealed type is "Tuple[str]" # TODO +reveal_type(func_c1(2)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsClass1] +# flags: --disallow-any-generics +from typing import Generic, TypeVar, Union, overload + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=int) +T3 = TypeVar("T3", default=str) +T4 = TypeVar("T4", default=Union[int, None]) + +class ClassA1(Generic[T2, T3]): ... + +def func_a1( + a: ClassA1, + b: ClassA1[float], + c: ClassA1[float, float], + d: ClassA1[float, float, float], # E: "ClassA1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + + k = ClassA1() + reveal_type(k) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + l = ClassA1[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.str]" + m = ClassA1[float, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.float]" + n = ClassA1[float, float, float]() # E: Type application has too many types (expected between 0 and 2) + reveal_type(n) # N: Revealed type is "Any" + +class ClassA2(Generic[T1, T2, T3]): ... + +def func_a2( + a: ClassA2, # E: Missing type parameters for generic type "ClassA2" + b: ClassA2[float], + c: ClassA2[float, float], + d: ClassA2[float, float, float], + e: ClassA2[float, float, float, float], # E: "ClassA2" expects between 1 and 3 type arguments, but 4 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA2[Any, builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.str]" + reveal_type(d) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.float]" + reveal_type(e) # N: Revealed type is "__main__.ClassA2[Any, builtins.int, builtins.str]" + + k = ClassA2() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassA2[Any, builtins.int, builtins.str]" + l = ClassA2[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.int, builtins.str]" + m = ClassA2[float, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.str]" + n = ClassA2[float, float, float]() + reveal_type(n) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.float]" + o = ClassA2[float, float, float, float]() # E: Type application has too many types (expected between 1 and 3) + reveal_type(o) # N: Revealed type is "Any" + +class ClassA3(Generic[T1, T2]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, var: int) -> None: ... + def __init__(self, var: Union[int, None] = None) -> None: ... + +def func_a3( + a: ClassA3, # E: Missing type parameters for generic type "ClassA3" + b: ClassA3[float], + c: ClassA3[float, float], + d: ClassA3[float, float, float], # E: "ClassA3" expects between 1 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA3[Any, builtins.int]" + reveal_type(b) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.int]" + reveal_type(c) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "__main__.ClassA3[Any, builtins.int]" + + k = ClassA3() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassA3[Any, builtins.int]" + l = ClassA3[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.int]" + m = ClassA3[float, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.float]" + n = ClassA3[float, float, float]() # E: Type application has too many types (expected between 1 and 2) + reveal_type(n) # N: Revealed type is "Any" + +class ClassA4(Generic[T4]): ... + +def func_a4( + a: ClassA4, + b: ClassA4[float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA4[Union[builtins.int, None]]" + reveal_type(b) # N: Revealed type is "__main__.ClassA4[builtins.float]" + + k = ClassA4() + reveal_type(k) # N: Revealed type is "__main__.ClassA4[Union[builtins.int, None]]" + l = ClassA4[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassA4[builtins.float]" + +[case testTypeVarDefaultsClass2] +# flags: --disallow-any-generics +from typing import Generic, ParamSpec + +P1 = ParamSpec("P1") +P2 = ParamSpec("P2", default=[int, str]) +P3 = ParamSpec("P3", default=...) + +class ClassB1(Generic[P2, P3]): ... + +def func_b1( + a: ClassB1, + b: ClassB1[[float]], + c: ClassB1[[float], [float]], + d: ClassB1[[float], [float], [float]], # E: "ClassB1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], ...]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + + k = ClassB1() + reveal_type(k) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + l = ClassB1[[float]]() + reveal_type(l) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + m = ClassB1[[float], [float]]() + reveal_type(m) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + n = ClassB1[[float], [float], [float]]() # E: Type application has too many types (expected between 0 and 2) + reveal_type(n) # N: Revealed type is "Any" + +class ClassB2(Generic[P1, P2]): ... + +def func_b2( + a: ClassB2, # E: Missing type parameters for generic type "ClassB2" + b: ClassB2[[float]], + c: ClassB2[[float], [float]], + d: ClassB2[[float], [float], [float]], # E: "ClassB2" expects between 1 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.int, builtins.str]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + + k = ClassB2() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + l = ClassB2[[float]]() + reveal_type(l) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.int, builtins.str]]" + m = ClassB2[[float], [float]]() + reveal_type(m) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.float]]" + n = ClassB2[[float], [float], [float]]() # E: Type application has too many types (expected between 1 and 2) + reveal_type(n) # N: Revealed type is "Any" + +[case testTypeVarDefaultsClass3] +# flags: --disallow-any-generics +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1") +T3 = TypeVar("T3", default=str) + +Ts1 = TypeVarTuple("Ts1") +Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, str]]) +Ts3 = TypeVarTuple("Ts3", default=Unpack[Tuple[float, ...]]) +Ts4 = TypeVarTuple("Ts4", default=Unpack[Tuple[()]]) + +class ClassC1(Generic[Unpack[Ts2]]): ... + +def func_c1( + a: ClassC1, + b: ClassC1[float], +) -> None: + # reveal_type(a) # Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "__main__.ClassC1[builtins.float]" + + k = ClassC1() + reveal_type(k) # N: Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" + l = ClassC1[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassC1[builtins.float]" + +class ClassC2(Generic[T3, Unpack[Ts3]]): ... + +def func_c2( + a: ClassC2, + b: ClassC2[int], + c: ClassC2[int, Unpack[Tuple[()]]], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassC2[builtins.str, Unpack[builtins.tuple[builtins.float, ...]]]" + # reveal_type(b) # Revealed type is "__main__.ClassC2[builtins.int, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + reveal_type(c) # N: Revealed type is "__main__.ClassC2[builtins.int]" + + k = ClassC2() + reveal_type(k) # N: Revealed type is "__main__.ClassC2[builtins.str, Unpack[builtins.tuple[builtins.float, ...]]]" + l = ClassC2[int]() + # reveal_type(l) # Revealed type is "__main__.ClassC2[builtins.int, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + m = ClassC2[int, Unpack[Tuple[()]]]() + reveal_type(m) # N: Revealed type is "__main__.ClassC2[builtins.int]" + +class ClassC3(Generic[T3, Unpack[Ts4]]): ... + +def func_c3( + a: ClassC3, + b: ClassC3[int], + c: ClassC3[int, Unpack[Tuple[float]]] +) -> None: + # reveal_type(a) # Revealed type is "__main__.ClassC3[builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "__main__.ClassC3[builtins.int]" + reveal_type(c) # N: Revealed type is "__main__.ClassC3[builtins.int, builtins.float]" + + k = ClassC3() + reveal_type(k) # N: Revealed type is "__main__.ClassC3[builtins.str]" + l = ClassC3[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassC3[builtins.int]" + m = ClassC3[int, Unpack[Tuple[float]]]() + reveal_type(m) # N: Revealed type is "__main__.ClassC3[builtins.int, builtins.float]" + +class ClassC4(Generic[T1, Unpack[Ts1], T3]): ... + +def func_c4( + a: ClassC4, # E: Missing type parameters for generic type "ClassC4" + b: ClassC4[int], + c: ClassC4[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassC4[Any, Unpack[builtins.tuple[Any, ...]], builtins.str]" + # reveal_type(b) # Revealed type is "__main__.ClassC4[builtins.int, builtins.str]" # TODO + reveal_type(c) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" + + k = ClassC4() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassC4[Any, Unpack[builtins.tuple[Any, ...]], builtins.str]" + l = ClassC4[int]() + # reveal_type(l) # Revealed type is "__main__.ClassC4[builtins.int, builtins.str]" # TODO + m = ClassC4[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsClassRecursive1] +# flags: --disallow-any-generics +from typing import Generic, TypeVar, List + +T1 = TypeVar("T1", default=str) +T2 = TypeVar("T2", default=T1) +T3 = TypeVar("T3", default=T2) +T4 = TypeVar("T4", default=List[T1]) + +class ClassD1(Generic[T1, T2]): ... + +def func_d1( + a: ClassD1, + b: ClassD1[int], + c: ClassD1[int, float] +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassD1[builtins.str, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassD1[builtins.int, builtins.int]" + reveal_type(c) # N: Revealed type is "__main__.ClassD1[builtins.int, builtins.float]" + + k = ClassD1() + reveal_type(k) # N: Revealed type is "__main__.ClassD1[builtins.str, builtins.str]" + l = ClassD1[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassD1[builtins.int, builtins.int]" + m = ClassD1[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassD1[builtins.int, builtins.float]" + +class ClassD2(Generic[T1, T2, T3]): ... + +def func_d2( + a: ClassD2, + b: ClassD2[int], + c: ClassD2[int, float], + d: ClassD2[int, float, str], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassD2[builtins.str, builtins.str, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.int, builtins.int]" + reveal_type(c) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.float, builtins.str]" + + k = ClassD2() + reveal_type(k) # N: Revealed type is "__main__.ClassD2[builtins.str, builtins.str, builtins.str]" + l = ClassD2[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.int, builtins.int]" + m = ClassD2[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.float, builtins.float]" + n = ClassD2[int, float, str]() + reveal_type(n) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.float, builtins.str]" + +class ClassD3(Generic[T1, T4]): ... + +def func_d3( + a: ClassD3, + b: ClassD3[int], + c: ClassD3[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]" + reveal_type(b) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]" + reveal_type(c) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]" + + # k = ClassD3() + # reveal_type(k) # Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]" # TODO + l = ClassD3[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]" + m = ClassD3[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]" + +[case testTypeVarDefaultsClassRecursiveMultipleFiles] +# flags: --disallow-any-generics +from typing import Generic, TypeVar +from file2 import T as T2 + +T = TypeVar("T", default=T2) + +class ClassG1(Generic[T2, T]): + pass + +def func( + a: ClassG1, + b: ClassG1[str], + c: ClassG1[str, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassG1[builtins.int, builtins.int]" + reveal_type(b) # N: Revealed type is "__main__.ClassG1[builtins.str, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassG1[builtins.str, builtins.float]" + + k = ClassG1() + reveal_type(k) # N: Revealed type is "__main__.ClassG1[builtins.int, builtins.int]" + l = ClassG1[str]() + reveal_type(l) # N: Revealed type is "__main__.ClassG1[builtins.str, builtins.str]" + m = ClassG1[str, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassG1[builtins.str, builtins.float]" + +[file file2.py] +from typing import TypeVar +T = TypeVar('T', default=int) + +[case testTypeVarDefaultsTypeAlias1] +# flags: --disallow-any-generics +from typing import Any, Dict, List, Tuple, TypeVar, Union + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=int) +T3 = TypeVar("T3", default=str) +T4 = TypeVar("T4") + +TA1 = Dict[T2, T3] + +def func_a1( + a: TA1, + b: TA1[float], + c: TA1[float, float], + d: TA1[float, float, float], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +TA2 = Tuple[T1, T2, T3] + +def func_a2( + a: TA2, # E: Missing type parameters for generic type "TA2" + b: TA2[float], + c: TA2[float, float], + d: TA2[float, float, float], + e: TA2[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given 4 +) -> None: + reveal_type(a) # N: Revealed type is "Tuple[Any, builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "Tuple[builtins.float, builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.str]" + reveal_type(d) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.float]" + reveal_type(e) # N: Revealed type is "Tuple[Any, builtins.int, builtins.str]" + +TA3 = Union[Dict[T1, T2], List[T3]] + +def func_a3( + a: TA3, # E: Missing type parameters for generic type "TA3" + b: TA3[float], + c: TA3[float, float], + d: TA3[float, float, float], + e: TA3[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given 4 +) -> None: + reveal_type(a) # N: Revealed type is "Union[builtins.dict[Any, builtins.int], builtins.list[builtins.str]]" + reveal_type(b) # N: Revealed type is "Union[builtins.dict[builtins.float, builtins.int], builtins.list[builtins.str]]" + reveal_type(c) # N: Revealed type is "Union[builtins.dict[builtins.float, builtins.float], builtins.list[builtins.str]]" + reveal_type(d) # N: Revealed type is "Union[builtins.dict[builtins.float, builtins.float], builtins.list[builtins.float]]" + reveal_type(e) # N: Revealed type is "Union[builtins.dict[Any, builtins.int], builtins.list[builtins.str]]" + +TA4 = Tuple[T1, T4, T2] + +def func_a4( + a: TA4, # E: Missing type parameters for generic type "TA4" + b: TA4[float], # E: Bad number of arguments for type alias, expected between 2 and 3, given 1 + c: TA4[float, float], + d: TA4[float, float, float], + e: TA4[float, float, float, float], # E: Bad number of arguments for type alias, expected between 2 and 3, given 4 +) -> None: + reveal_type(a) # N: Revealed type is "Tuple[Any, Any, builtins.int]" + reveal_type(b) # N: Revealed type is "Tuple[Any, Any, builtins.int]" + reveal_type(c) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.int]" + reveal_type(d) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.float]" + reveal_type(e) # N: Revealed type is "Tuple[Any, Any, builtins.int]" +[builtins fixtures/dict.pyi] + +[case testTypeVarDefaultsTypeAlias2] +# flags: --disallow-any-generics +from typing import Any, Generic, ParamSpec + +P1 = ParamSpec("P1") +P2 = ParamSpec("P2", default=[int, str]) +P3 = ParamSpec("P3", default=...) + +class ClassB1(Generic[P2, P3]): ... +TB1 = ClassB1[P2, P3] + +def func_b1( + a: TB1, + b: TB1[[float]], + c: TB1[[float], [float]], + d: TB1[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + +class ClassB2(Generic[P1, P2]): ... +TB2 = ClassB2[P1, P2] + +def func_b2( + a: TB2, # E: Missing type parameters for generic type "TB2" + b: TB2[[float]], + c: TB2[[float], [float]], + d: TB2[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 1 and 2, given 3 +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.int, builtins.str]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsTypeAlias3] +# flags: --disallow-any-generics +from typing import Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1") +T3 = TypeVar("T3", default=str) + +Ts1 = TypeVarTuple("Ts1") +Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, str]]) +Ts3 = TypeVarTuple("Ts3", default=Unpack[Tuple[float, ...]]) +Ts4 = TypeVarTuple("Ts4", default=Unpack[Tuple[()]]) + +TC1 = Tuple[Unpack[Ts2]] + +def func_c1( + a: TC1, + b: TC1[float], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "Tuple[builtins.float]" + +TC2 = Tuple[T3, Unpack[Ts3]] + +def func_c2( + a: TC2, + b: TC2[int], + c: TC2[int, Unpack[Tuple[()]]], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.str, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + # reveal_type(b) # Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + reveal_type(c) # N: Revealed type is "Tuple[builtins.int]" + +TC3 = Tuple[T3, Unpack[Ts4]] + +def func_c3( + a: TC3, + b: TC3[int], + c: TC3[int, Unpack[Tuple[float]]], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "Tuple[builtins.int]" + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, builtins.float]" + +TC4 = Tuple[T1, Unpack[Ts1], T3] + +def func_c4( + a: TC4, # E: Missing type parameters for generic type "TC4" + b: TC4[int], + c: TC4[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "Tuple[Any, Unpack[builtins.tuple[Any, ...]], builtins.str]" + # reveal_type(b) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsTypeAliasRecursive1] +# flags: --disallow-any-generics +from typing import Dict, List, TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=T1) + +TD1 = Dict[T1, T2] + +def func_d1( + a: TD1, # E: Missing type parameters for generic type "TD1" + b: TD1[int], + c: TD1[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[Any, Any]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, builtins.int]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]" +[builtins fixtures/dict.pyi] + +[case testTypeVarDefaultsTypeAliasRecursive2] +from typing import Any, Dict, Generic, TypeVar + +T1 = TypeVar("T1", default=str) +T2 = TypeVar("T2", default=T1) +Alias1 = Dict[T1, T2] +T3 = TypeVar("T3") +class A(Generic[T3]): ... + +T4 = TypeVar("T4", default=A[Alias1]) +class B(Generic[T4]): ... + +def func_d3( + a: B, + b: B[A[Alias1[int]]], + c: B[A[Alias1[int, float]]], + d: B[int], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.str, builtins.str]]]" + reveal_type(b) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.int]]]" + reveal_type(c) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.float]]]" + reveal_type(d) # N: Revealed type is "__main__.B[builtins.int]" +[builtins fixtures/dict.pyi] + +[case testTypeVarDefaultsAndTypeObjectTypeInUnion] +from __future__ import annotations +from typing import Generic +from typing_extensions import TypeVar + +_I = TypeVar("_I", default=int) + +class C(Generic[_I]): pass + +t: type[C] | int = C +[builtins fixtures/tuple.pyi] + + + +[case testGenericTypeAliasWithDefaultTypeVarPreservesNoneInDefault] +from typing_extensions import TypeVar +from typing import Generic, Union + +T1 = TypeVar("T1", default=Union[int, None]) +T2 = TypeVar("T2", default=Union[int, None]) + + +class A(Generic[T1, T2]): + def __init__(self, a: T1, b: T2) -> None: + self.a = a + self.b = b + + +MyA = A[T1, int] +a: MyA = A(None, 10) +reveal_type(a.a) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test new file mode 100644 index 000000000000..f49e1b3c6613 --- /dev/null +++ b/test-data/unit/check-typevar-tuple.test @@ -0,0 +1,2489 @@ +[case testTypeVarTupleBasic] +from typing import Any, Tuple +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def f(a: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + return a + +any: Any +args: Tuple[int, str] = (1, 'x') +args2: Tuple[bool, str] = (False, 'y') +args3: Tuple[int, str, bool] = (2, 'z', True) +varargs: Tuple[int, ...] = (1, 2, 3) + +reveal_type(f(args)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +reveal_type(f(varargs)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +f(0) # E: Argument 1 to "f" has incompatible type "int"; expected "Tuple[Never, ...]" + +def g(a: Tuple[Unpack[Ts]], b: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + return a + +reveal_type(g(args, args)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +reveal_type(g(args, args2)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +reveal_type(g(args, args3)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" +reveal_type(g(any, any)) # N: Revealed type is "builtins.tuple[Any, ...]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleMixed] +from typing import Tuple +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def to_str(i: int) -> str: + ... + +def f(a: Tuple[int, Unpack[Ts]]) -> Tuple[str, Unpack[Ts]]: + return (to_str(a[0]),) + a[1:] + +def g(a: Tuple[Unpack[Ts], int]) -> Tuple[Unpack[Ts], str]: + return a[:-1] + (to_str(a[-1]),) + +def h(a: Tuple[bool, int, Unpack[Ts], str, object]) -> Tuple[Unpack[Ts]]: + return a[2:-2] + +empty = () +bad_args: Tuple[str, str] +var_len_tuple: Tuple[int, ...] + +f_args: Tuple[int, str] +f_args2: Tuple[int] +f_args3: Tuple[int, str, bool] + +reveal_type(f(f_args)) # N: Revealed type is "Tuple[builtins.str, builtins.str]" +reveal_type(f(f_args2)) # N: Revealed type is "Tuple[builtins.str]" +reveal_type(f(f_args3)) # N: Revealed type is "Tuple[builtins.str, builtins.str, builtins.bool]" +f(empty) # E: Argument 1 to "f" has incompatible type "Tuple[()]"; expected "Tuple[int]" +f(bad_args) # E: Argument 1 to "f" has incompatible type "Tuple[str, str]"; expected "Tuple[int, str]" + +# The reason for error in subtle: actual can be empty, formal cannot. +reveal_type(f(var_len_tuple)) # N: Revealed type is "Tuple[builtins.str, Unpack[builtins.tuple[builtins.int, ...]]]" \ + # E: Argument 1 to "f" has incompatible type "Tuple[int, ...]"; expected "Tuple[int, Unpack[Tuple[int, ...]]]" + +g_args: Tuple[str, int] +reveal_type(g(g_args)) # N: Revealed type is "Tuple[builtins.str, builtins.str]" + +h_args: Tuple[bool, int, str, int, str, object] +reveal_type(h(h_args)) # N: Revealed type is "Tuple[builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleChaining] +from typing import Tuple +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def to_str(i: int) -> str: + ... + +def f(a: Tuple[int, Unpack[Ts]]) -> Tuple[str, Unpack[Ts]]: + return (to_str(a[0]),) + a[1:] + +def g(a: Tuple[bool, int, Unpack[Ts], str, object]) -> Tuple[str, Unpack[Ts]]: + return f(a[1:-2]) + +def h(a: Tuple[bool, int, Unpack[Ts], str, object]) -> Tuple[str, Unpack[Ts]]: + x = f(a[1:-2]) + return x + +args: Tuple[bool, int, str, int, str, object] +reveal_type(g(args)) # N: Revealed type is "Tuple[builtins.str, builtins.str, builtins.int]" +reveal_type(h(args)) # N: Revealed type is "Tuple[builtins.str, builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleGenericClassDefn] +from typing import Generic, TypeVar, Tuple, Union +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") + +class Variadic(Generic[Unpack[Ts]]): + pass + +class Mixed1(Generic[T, Unpack[Ts]]): + pass + +class Mixed2(Generic[Unpack[Ts], T]): + pass + +variadic: Variadic[int, str] +reveal_type(variadic) # N: Revealed type is "__main__.Variadic[builtins.int, builtins.str]" + +variadic_single: Variadic[int] +reveal_type(variadic_single) # N: Revealed type is "__main__.Variadic[builtins.int]" + +empty: Variadic[()] +reveal_type(empty) # N: Revealed type is "__main__.Variadic[()]" + +omitted: Variadic +reveal_type(omitted) # N: Revealed type is "__main__.Variadic[Unpack[builtins.tuple[Any, ...]]]" + +bad: Variadic[Unpack[Tuple[int, ...]], str, Unpack[Tuple[bool, ...]]] # E: More than one Unpack in a type is not allowed +reveal_type(bad) # N: Revealed type is "__main__.Variadic[Unpack[builtins.tuple[builtins.int, ...]], builtins.str]" + +bad2: Unpack[Tuple[int, ...]] # E: Unpack is only valid in a variadic position + +m1: Mixed1[int, str, bool] +reveal_type(m1) # N: Revealed type is "__main__.Mixed1[builtins.int, builtins.str, builtins.bool]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleGenericClassWithFunctions] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +T = TypeVar("T") +S = TypeVar("S") + +class Variadic(Generic[T, Unpack[Ts], S]): + pass + +def foo(t: Variadic[int, Unpack[Ts], object]) -> Tuple[int, Unpack[Ts]]: + ... + +v: Variadic[int, str, bool, object] +reveal_type(foo(v)) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.bool]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleGenericClassWithMethods] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +T = TypeVar("T") +S = TypeVar("S") + +class Variadic(Generic[T, Unpack[Ts], S]): + def __init__(self, t: Tuple[Unpack[Ts]]) -> None: + ... + + def foo(self, t: int) -> Tuple[int, Unpack[Ts]]: + ... + +v: Variadic[float, str, bool, object] +reveal_type(v.foo(0)) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.bool]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleIsNotValidAliasTarget] +from typing_extensions import TypeVarTuple + +Ts = TypeVarTuple("Ts") +B = Ts # E: Type variable "__main__.Ts" is invalid as target for type alias +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646ArrayExample] +from typing import Generic, Tuple, TypeVar, Protocol, NewType +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Height = NewType('Height', int) +Width = NewType('Width', int) + +T_co = TypeVar("T_co", covariant=True) +T = TypeVar("T") + +class SupportsAbs(Protocol[T_co]): + def __abs__(self) -> T_co: pass + +def abs(a: SupportsAbs[T]) -> T: + ... + +class Array(Generic[Unpack[Shape]]): + def __init__(self, shape: Tuple[Unpack[Shape]]): + self._shape: Tuple[Unpack[Shape]] = shape + + def get_shape(self) -> Tuple[Unpack[Shape]]: + return self._shape + + def __abs__(self) -> Array[Unpack[Shape]]: ... + + def __add__(self, other: Array[Unpack[Shape]]) -> Array[Unpack[Shape]]: ... + +shape = (Height(480), Width(640)) +x: Array[Height, Width] = Array(shape) +reveal_type(abs(x)) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" +reveal_type(x + x) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646ArrayExampleWithDType] +from typing import Generic, Tuple, TypeVar, Protocol, NewType +from typing_extensions import TypeVarTuple, Unpack + +DType = TypeVar("DType") +Shape = TypeVarTuple('Shape') + +Height = NewType('Height', int) +Width = NewType('Width', int) + +T_co = TypeVar("T_co", covariant=True) +T = TypeVar("T") + +class SupportsAbs(Protocol[T_co]): + def __abs__(self) -> T_co: pass + +def abs(a: SupportsAbs[T]) -> T: + ... + +class Array(Generic[DType, Unpack[Shape]]): + def __init__(self, shape: Tuple[Unpack[Shape]]): + self._shape: Tuple[Unpack[Shape]] = shape + + def get_shape(self) -> Tuple[Unpack[Shape]]: + return self._shape + + def __abs__(self) -> Array[DType, Unpack[Shape]]: ... + + def __add__(self, other: Array[DType, Unpack[Shape]]) -> Array[DType, Unpack[Shape]]: ... + +shape = (Height(480), Width(640)) +x: Array[float, Height, Width] = Array(shape) +reveal_type(abs(x)) # N: Revealed type is "__main__.Array[builtins.float, __main__.Height, __main__.Width]" +reveal_type(x + x) # N: Revealed type is "__main__.Array[builtins.float, __main__.Height, __main__.Width]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646ArrayExampleInfer] +from typing import Generic, Tuple, TypeVar, NewType +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Height = NewType('Height', int) +Width = NewType('Width', int) + +class Array(Generic[Unpack[Shape]]): + pass + +x: Array[float, Height, Width] = Array() +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646TypeConcatenation] +from typing import Generic, TypeVar, NewType +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Channels = NewType("Channels", int) +Batch = NewType("Batch", int) +Height = NewType('Height', int) +Width = NewType('Width', int) + +class Array(Generic[Unpack[Shape]]): + pass + + +def add_batch_axis(x: Array[Unpack[Shape]]) -> Array[Batch, Unpack[Shape]]: ... +def del_batch_axis(x: Array[Batch, Unpack[Shape]]) -> Array[Unpack[Shape]]: ... +def add_batch_channels( + x: Array[Unpack[Shape]] +) -> Array[Batch, Unpack[Shape], Channels]: ... + +a: Array[Height, Width] +b = add_batch_axis(a) +reveal_type(b) # N: Revealed type is "__main__.Array[__main__.Batch, __main__.Height, __main__.Width]" +c = del_batch_axis(b) +reveal_type(c) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" +d = add_batch_channels(a) +reveal_type(d) # N: Revealed type is "__main__.Array[__main__.Batch, __main__.Height, __main__.Width, __main__.Channels]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646TypeVarConcatenation] +from typing import Generic, TypeVar, NewType, Tuple +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar('T') +Ts = TypeVarTuple('Ts') + +def prefix_tuple( + x: T, + y: Tuple[Unpack[Ts]], +) -> Tuple[T, Unpack[Ts]]: + ... + +z = prefix_tuple(x=0, y=(True, 'a')) +reveal_type(z) # N: Revealed type is "Tuple[builtins.int, builtins.bool, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646TypeVarTupleUnpacking] +from typing import Generic, TypeVar, NewType, Any, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Channels = NewType("Channels", int) +Batch = NewType("Batch", int) +Height = NewType('Height', int) +Width = NewType('Width', int) + +class Array(Generic[Unpack[Shape]]): + pass + +def process_batch_channels( + x: Array[Batch, Unpack[Tuple[Any, ...]], Channels] +) -> None: + ... + +x: Array[Batch, Height, Width, Channels] +process_batch_channels(x) +y: Array[Batch, Channels] +process_batch_channels(y) +z: Array[Batch] +process_batch_channels(z) # E: Argument 1 to "process_batch_channels" has incompatible type "Array[Batch]"; expected "Array[Batch, Unpack[Tuple[Any, ...]], Channels]" + +u: Array[Unpack[Tuple[Any, ...]]] + +def expect_variadic_array( + x: Array[Batch, Unpack[Shape]] +) -> None: + ... + +def expect_variadic_array_2( + x: Array[Batch, Height, Width, Channels] +) -> None: + ... + +expect_variadic_array(u) +expect_variadic_array_2(u) + +Ts = TypeVarTuple("Ts") +Ts2 = TypeVarTuple("Ts2") + +def bad(x: Tuple[int, Unpack[Ts], str, Unpack[Ts2]]) -> None: # E: More than one Unpack in a type is not allowed + + ... +reveal_type(bad) # N: Revealed type is "def [Ts, Ts2] (x: Tuple[builtins.int, Unpack[Ts`-1], builtins.str])" + +def bad2(x: Tuple[int, Unpack[Tuple[int, ...]], str, Unpack[Tuple[str, ...]]]) -> None: # E: More than one Unpack in a type is not allowed + ... +reveal_type(bad2) # N: Revealed type is "def (x: Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.str])" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646TypeVarStarArgsBasic] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +def args_to_tuple(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: + reveal_type(args) # N: Revealed type is "Tuple[Unpack[Ts`-1]]" + reveal_type(args_to_tuple(1, *args)) # N: Revealed type is "Tuple[Literal[1]?, Unpack[Ts`-1]]" + reveal_type(args_to_tuple(*args, 'a')) # N: Revealed type is "Tuple[Unpack[Ts`-1], Literal['a']?]" + reveal_type(args_to_tuple(1, *args, 'a')) # N: Revealed type is "Tuple[Literal[1]?, Unpack[Ts`-1], Literal['a']?]" + args_to_tuple(*args, *args) # E: Passing multiple variadic unpacks in a call is not supported + ok = (1, 'a') + reveal_type(args_to_tuple(*ok, *ok)) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.int, builtins.str]" + if int(): + return args + else: + return args_to_tuple(*args) + +reveal_type(args_to_tuple(1, 'a')) # N: Revealed type is "Tuple[Literal[1]?, Literal['a']?]" +vt: Tuple[int, ...] +reveal_type(args_to_tuple(1, *vt)) # N: Revealed type is "Tuple[Literal[1]?, Unpack[builtins.tuple[builtins.int, ...]]]" +reveal_type(args_to_tuple(*vt, 'a')) # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.int, ...]], Literal['a']?]" +reveal_type(args_to_tuple(1, *vt, 'a')) # N: Revealed type is "Tuple[Literal[1]?, Unpack[builtins.tuple[builtins.int, ...]], Literal['a']?]" +args_to_tuple(*vt, *vt) # E: Passing multiple variadic unpacks in a call is not supported +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646TypeVarStarArgs] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +def args_to_tuple(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: + with_prefix_suffix(*args) # E: Too few arguments for "with_prefix_suffix" \ + # E: Argument 1 to "with_prefix_suffix" has incompatible type "*Tuple[Unpack[Ts]]"; expected "bool" + new_args = (True, "foo", *args, 5) + with_prefix_suffix(*new_args) + return args + +def with_prefix_suffix(*args: Unpack[Tuple[bool, str, Unpack[Ts], int]]) -> Tuple[bool, str, Unpack[Ts], int]: + reveal_type(args) # N: Revealed type is "Tuple[builtins.bool, builtins.str, Unpack[Ts`-1], builtins.int]" + reveal_type(args_to_tuple(*args)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, Unpack[Ts`-1], builtins.int]" + reveal_type(args_to_tuple(1, *args, 'a')) # N: Revealed type is "Tuple[Literal[1]?, builtins.bool, builtins.str, Unpack[Ts`-1], builtins.int, Literal['a']?]" + return args + +reveal_type(with_prefix_suffix(True, "bar", "foo", 5)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, Literal['foo']?, builtins.int]" +reveal_type(with_prefix_suffix(True, "bar", 5)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, builtins.int]" + +with_prefix_suffix(True, "bar", "foo", 1.0) # E: Argument 4 to "with_prefix_suffix" has incompatible type "float"; expected "int" +with_prefix_suffix(True, "bar") # E: Too few arguments for "with_prefix_suffix" + +t = (True, "bar", "foo", 5) +reveal_type(with_prefix_suffix(*t)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, builtins.str, builtins.int]" +reveal_type(with_prefix_suffix(True, *("bar", "foo"), 5)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, Literal['foo']?, builtins.int]" + +reveal_type(with_prefix_suffix(True, "bar", *["foo1", "foo2"], 5)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, Unpack[builtins.tuple[builtins.str, ...]], builtins.int]" + +bad_t = (True, "bar") +with_prefix_suffix(*bad_t) # E: Too few arguments for "with_prefix_suffix" + +def foo(*args: Unpack[Ts]) -> None: + reveal_type(with_prefix_suffix(True, "bar", *args, 5)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, Unpack[Ts`-1], builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646TypeVarStarArgsFixedLengthTuple] +from typing import Tuple +from typing_extensions import Unpack + +def foo(*args: Unpack[Tuple[int, str]]) -> None: + reveal_type(args) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +foo(0, "foo") +foo(0, 1) # E: Argument 2 to "foo" has incompatible type "int"; expected "str" +foo("foo", "bar") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" +foo(0, "foo", 1) # E: Too many arguments for "foo" +foo(0) # E: Too few arguments for "foo" +foo() # E: Too few arguments for "foo" +foo(*(0, "foo")) + +def foo2(*args: Unpack[Tuple[bool, Unpack[Tuple[int, str]], bool]]) -> None: + reveal_type(args) # N: Revealed type is "Tuple[builtins.bool, builtins.int, builtins.str, builtins.bool]" + +# It is hard to normalize callable types in definition, because there is deep relation between `FuncDef.type` +# and `FuncDef.arguments`, therefore various typeops need to be sure to normalize Callable types before using them. +reveal_type(foo2) # N: Revealed type is "def (*args: Unpack[Tuple[builtins.bool, builtins.int, builtins.str, builtins.bool]])" + +class C: + def foo2(self, *args: Unpack[Tuple[bool, Unpack[Tuple[int, str]], bool]]) -> None: ... +reveal_type(C().foo2) # N: Revealed type is "def (*args: Unpack[Tuple[builtins.bool, builtins.int, builtins.str, builtins.bool]])" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646TypeVarStarArgsVariableLengthTuple] +from typing import Tuple +from typing_extensions import Unpack, TypeVarTuple + +def foo(*args: Unpack[Tuple[int, ...]]) -> None: + reveal_type(args) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +foo(0, 1, 2) +foo(0, 1, "bar") # E: Argument 3 to "foo" has incompatible type "str"; expected "int" + +def foo2(*args: Unpack[Tuple[str, Unpack[Tuple[int, ...]], bool, bool]]) -> None: + reveal_type(args) # N: Revealed type is "Tuple[builtins.str, Unpack[builtins.tuple[builtins.int, ...]], builtins.bool, builtins.bool]" + reveal_type(args[1]) # N: Revealed type is "builtins.int" + +def foo3(*args: Unpack[Tuple[str, Unpack[Tuple[int, ...]], str, float]]) -> None: + reveal_type(args[0]) # N: Revealed type is "builtins.str" + reveal_type(args[1]) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(args[2]) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.float]" + args[3] # E: Tuple index out of range \ + # N: Variadic tuple can have length 3 + reveal_type(args[-1]) # N: Revealed type is "builtins.float" + reveal_type(args[-2]) # N: Revealed type is "builtins.str" + reveal_type(args[-3]) # N: Revealed type is "Union[builtins.str, builtins.int]" + args[-4] # E: Tuple index out of range \ + # N: Variadic tuple can have length 3 + reveal_type(args[::-1]) # N: Revealed type is "Tuple[builtins.float, builtins.str, Unpack[builtins.tuple[builtins.int, ...]], builtins.str]" + args[::2] # E: Ambiguous slice of a variadic tuple + args[:2] # E: Ambiguous slice of a variadic tuple + +Ts = TypeVarTuple("Ts") +def foo4(*args: Unpack[Tuple[str, Unpack[Ts], bool, bool]]) -> None: + reveal_type(args[1]) # N: Revealed type is "builtins.object" + +foo2("bar", 1, 2, 3, False, True) +foo2(0, 1, 2, 3, False, True) # E: Argument 1 to "foo2" has incompatible type "int"; expected "str" +foo2("bar", "bar", 2, 3, False, True) # E: Argument 2 to "foo2" has incompatible type "str"; expected "Unpack[Tuple[Unpack[Tuple[int, ...]], bool, bool]]" +foo2("bar", 1, 2, 3, 4, True) # E: Argument 5 to "foo2" has incompatible type "int"; expected "Unpack[Tuple[Unpack[Tuple[int, ...]], bool, bool]]" +foo2(*("bar", 1, 2, 3, False, True)) +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646Callable] +from typing import Tuple, Callable +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def call( + target: Callable[[Unpack[Ts]], None], + args: Tuple[Unpack[Ts]], +) -> None: + pass + +def func(arg1: int, arg2: str) -> None: ... +def func2(arg1: int, arg2: int) -> None: ... +def func3(*args: int) -> None: ... + +vargs: Tuple[int, ...] +vargs_str: Tuple[str, ...] + +call(target=func, args=(0, 'foo')) +call(target=func, args=('bar', 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[str, str], None]" +call(target=func, args=(True, 'foo', 0)) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[bool, str, int], None]" +call(target=func, args=(0, 0, 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, int, str], None]" +call(target=func, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(int)], None]" + +# NOTE: This behavior may be a bit contentious, it is maybe inconsistent with our handling of +# PEP646 but consistent with our handling of callable constraints. +call(target=func2, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, int], None]"; expected "Callable[[VarArg(int)], None]" +call(target=func3, args=vargs) +call(target=func3, args=(0,1)) +call(target=func3, args=(0,'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[int, str], None]" +call(target=func3, args=vargs_str) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[VarArg(str)], None]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646CallableWithPrefixSuffix] +from typing import Tuple, Callable +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def call_prefix( + target: Callable[[bytes, Unpack[Ts]], None], + args: Tuple[Unpack[Ts]], +) -> None: + pass + +def func_prefix(arg0: bytes, arg1: int, arg2: str) -> None: ... +def func2_prefix(arg0: str, arg1: int, arg2: str) -> None: ... + +call_prefix(target=func_prefix, args=(0, 'foo')) +call_prefix(target=func2_prefix, args=(0, 'foo')) # E: Argument "target" to "call_prefix" has incompatible type "Callable[[str, int, str], None]"; expected "Callable[[bytes, int, str], None]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646CallableSuffixSyntax] +from typing import Callable, Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +x: Callable[[str, Unpack[Tuple[int, ...]], bool], None] +reveal_type(x) # N: Revealed type is "def (builtins.str, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.bool]])" + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +A = Callable[[T, Unpack[Ts], S], int] +y: A[int, str, bool] +reveal_type(y) # N: Revealed type is "def (builtins.int, builtins.str, builtins.bool) -> builtins.int" +z: A[Unpack[Tuple[int, ...]]] +reveal_type(z) # N: Revealed type is "def (builtins.int, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.int]]) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646CallableInvalidSyntax] +from typing import Callable, Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") +a: Callable[[Unpack[Ts], Unpack[Us]], int] # E: More than one Unpack in a type is not allowed +reveal_type(a) # N: Revealed type is "def [Ts, Us] (*Unpack[Ts`-1]) -> builtins.int" +b: Callable[[Unpack], int] # E: Unpack[...] requires exactly one type argument +reveal_type(b) # N: Revealed type is "def (*Any) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646CallableNewSyntax] +from typing import Callable, Generic, Tuple +from typing_extensions import ParamSpec + +x: Callable[[str, *Tuple[int, ...]], None] +reveal_type(x) # N: Revealed type is "def (builtins.str, *builtins.int)" +y: Callable[[str, *Tuple[int, ...], bool], None] +reveal_type(y) # N: Revealed type is "def (builtins.str, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.bool]])" + +P = ParamSpec("P") +class C(Generic[P]): ... +bad: C[[int, *Tuple[int, ...], int]] # E: Unpack is only valid in a variadic position +reveal_type(bad) # N: Revealed type is "__main__.C[[builtins.int, *Any]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646UnspecifiedParameters] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +class Array(Generic[Unpack[Ts]]): + ... + +def takes_any_array(arr: Array) -> None: + ... + +x: Array[int, bool] +takes_any_array(x) + +T = TypeVar("T") + +class Array2(Generic[T, Unpack[Ts]]): + ... + +def takes_empty_array2(arr: Array2[int]) -> None: + ... + +y: Array2[int] +takes_empty_array2(y) +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646CallableStarArgs] +from typing import Tuple, Callable +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def call( + target: Callable[[Unpack[Ts]], None], + *args: Unpack[Ts], +) -> None: + ... + target(*args) + +class A: + def func(self, arg1: int, arg2: str) -> None: ... + def func2(self, arg1: int, arg2: int) -> None: ... + def func3(self, *args: int) -> None: ... + +vargs: Tuple[int, ...] +vargs_str: Tuple[str, ...] + +call(A().func) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[], None]" +call(A().func, 0, 'foo') +call(A().func, 0, 'foo', 0) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, str, int], None]" +call(A().func, 0) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int], None]" +call(A().func, 0, 1) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, int], None]" +call(A().func2, 0, 0) +call(A().func3, 0, 1, 2) +call(A().func3) +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasBasicTuple] +from typing import Tuple, List, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") + +A = List[Tuple[T, Unpack[Ts], T]] +x: A[int, str, str] +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str, builtins.str, builtins.int]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasBasicCallable] +from typing import TypeVar, Callable +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") + +A = Callable[[T, Unpack[Ts]], S] +x: A[int, str, int, str] +reveal_type(x) # N: Revealed type is "def (builtins.int, builtins.str, builtins.int) -> builtins.str" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasBasicInstance] +from typing import TypeVar, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") + +class G(Generic[Unpack[Ts], T]): ... + +A = G[T, Unpack[Ts], T] +x: A[int, str, str] +reveal_type(x) # N: Revealed type is "__main__.G[builtins.int, builtins.str, builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasUnpackFixedTupleArgs] +from typing import Tuple, List, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") + +Start = Tuple[int, str] +A = List[Tuple[T, Unpack[Ts], S]] +x: A[Unpack[Start], int] +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str, builtins.int]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasUnpackFixedTupleTarget] +from typing import Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") + +Prefix = Tuple[int, int] +A = Tuple[Unpack[Prefix], Unpack[Ts]] +x: A[str, str] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasMultipleUnpacks] +from typing import Tuple, Generic, Callable +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") +class G(Generic[Unpack[Ts]]): ... + +A = Tuple[Unpack[Ts], Unpack[Us]] # E: More than one Unpack in a type is not allowed +x: A[int, str] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +B = Callable[[Unpack[Ts], Unpack[Us]], int] # E: More than one Unpack in a type is not allowed +y: B[int, str] +reveal_type(y) # N: Revealed type is "def (builtins.int, builtins.str) -> builtins.int" + +C = G[Unpack[Ts], Unpack[Us]] # E: More than one Unpack in a type is not allowed +z: C[int, str] +reveal_type(z) # N: Revealed type is "__main__.G[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasNoArgs] +from typing import Tuple, TypeVar, Generic, Callable, List +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +class G(Generic[Unpack[Ts]]): ... + +A = List[Tuple[T, Unpack[Ts], T]] +x: A +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[Any, Unpack[builtins.tuple[Any, ...]], Any]]" + +B = Callable[[T, Unpack[Ts]], int] +y: B +reveal_type(y) # N: Revealed type is "def (Any, *Any) -> builtins.int" + +C = G[T, Unpack[Ts], T] +z: C +reveal_type(z) # N: Revealed type is "__main__.G[Any, Unpack[builtins.tuple[Any, ...]], Any]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasFewArgs] +from typing import Tuple, List, TypeVar, Generic, Callable +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +class G(Generic[Unpack[Ts]]): ... + +A = List[Tuple[T, Unpack[Ts], S]] +x: A[int] # E: Bad number of arguments for type alias, expected at least 2, given 1 +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[Any, Unpack[builtins.tuple[Any, ...]], Any]]" + +B = Callable[[T, S, Unpack[Ts]], int] +y: B[int] # E: Bad number of arguments for type alias, expected at least 2, given 1 +reveal_type(y) # N: Revealed type is "def (Any, Any, *Any) -> builtins.int" + +C = G[T, Unpack[Ts], S] +z: C[int] # E: Bad number of arguments for type alias, expected at least 2, given 1 +reveal_type(z) # N: Revealed type is "__main__.G[Any, Unpack[builtins.tuple[Any, ...]], Any]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasRecursiveUnpack] +from typing import Tuple, Optional +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +A = Tuple[Unpack[Ts], Optional[A[Unpack[Ts]]]] +x: A[int, str] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.str, Union[..., None]]" + +*_, last = x +if last is not None: + reveal_type(last) # N: Revealed type is "Tuple[builtins.int, builtins.str, Union[Tuple[builtins.int, builtins.str, Union[..., None]], None]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasUpperBoundCheck] +from typing import Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +class A: ... +class B: ... +class C: ... +class D: ... + +T = TypeVar("T", bound=int) +S = TypeVar("S", bound=str) +Ts = TypeVarTuple("Ts") + +Alias = Tuple[T, Unpack[Ts], S] +First = Tuple[A, B] +Second = Tuple[C, D] +x: Alias[Unpack[First], Unpack[Second]] # E: Type argument "A" of "Alias" must be a subtype of "int" \ + # E: Type argument "D" of "Alias" must be a subtype of "str" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasEmptyArg] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +A = Tuple[int, Unpack[Ts], str] +x: A[()] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasVariadicTupleArg] +from typing import Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +A = Tuple[int, Unpack[Ts]] +B = A[str, Unpack[Ts]] +C = B[Unpack[Tuple[bool, ...]]] +x: C +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.str, Unpack[builtins.tuple[builtins.bool, ...]]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasVariadicTupleArgGeneric] +from typing import Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") + +A = Tuple[int, Unpack[Ts]] +B = A[Unpack[Tuple[T, ...]]] +x: B[str] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.str, ...]]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasVariadicTupleArgSplit] +from typing import Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") + +A = Tuple[T, Unpack[Ts], S, T] + +x: A[int, Unpack[Tuple[bool, ...]], str] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.bool, ...]], builtins.str, builtins.int]" + +y: A[Unpack[Tuple[bool, ...]]] +reveal_type(y) # N: Revealed type is "Tuple[builtins.bool, Unpack[builtins.tuple[builtins.bool, ...]], builtins.bool, builtins.bool]" +[builtins fixtures/tuple.pyi] + +[case testBanPathologicalRecursiveTuples] +from typing import Tuple +from typing_extensions import Unpack +A = Tuple[int, Unpack[A]] # E: Invalid recursive alias: a tuple item of itself +B = Tuple[int, Unpack[C]] # E: Invalid recursive alias: a tuple item of itself \ + # E: Name "C" is used before definition +C = Tuple[int, Unpack[B]] +x: A +y: B +z: C +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "Any" +reveal_type(z) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[Any, ...]]]" + +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicWithBadType] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... + +def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], T]: ... +def f(*args: Unpack[Us]) -> Foo[Us]: ... # E: TypeVarTuple "Us" is only valid with an unpack +dec(f) # No crash +[builtins fixtures/tuple.pyi] + +[case testHomogeneousGenericTupleUnpackInferenceNoCrash1] +from typing import Any, TypeVar, Tuple, Type, Optional +from typing_extensions import Unpack + +T = TypeVar("T") +def convert(obj: Any, *to_classes: Unpack[Tuple[Type[T], ...]]) -> Optional[T]: + ... + +x = convert(1, int, float) +reveal_type(x) # N: Revealed type is "Union[builtins.float, None]" +[builtins fixtures/tuple.pyi] + +[case testHomogeneousGenericTupleUnpackInferenceNoCrash2] +from typing import TypeVar, Tuple, Callable, Iterable +from typing_extensions import Unpack + +T = TypeVar("T") +def combine(x: T, y: T) -> T: ... +def reduce(fn: Callable[[T, T], T], xs: Iterable[T]) -> T: ... + +def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[str, ...]], bool]]) -> None: + reduce(combine, xs) +[builtins fixtures/tuple.pyi] + +[case testVariadicStarArgsCallNoCrash] +from typing import TypeVar, Callable, Tuple +from typing_extensions import TypeVarTuple, Unpack + +X = TypeVar("X") +Y = TypeVar("Y") +Xs = TypeVarTuple("Xs") +Ys = TypeVarTuple("Ys") + +def nil() -> Tuple[()]: + return () + +def cons( + f: Callable[[X], Y], + g: Callable[[Unpack[Xs]], Tuple[Unpack[Ys]]], +) -> Callable[[X, Unpack[Xs]], Tuple[Y, Unpack[Ys]]]: + def wrapped(x: X, *xs: Unpack[Xs]) -> Tuple[Y, Unpack[Ys]]: + y, ys = f(x), g(*xs) + return y, *ys + return wrapped + +def star(f: Callable[[X], Y]) -> Callable[[Unpack[Tuple[X, ...]]], Tuple[Y, ...]]: + def wrapped(*xs: X) -> Tuple[Y, ...]: + if not xs: + return nil() + return cons(f, star(f))(*xs) + return wrapped +[builtins fixtures/tuple.pyi] + +[case testInvalidTypeVarTupleUseNoCrash] +from typing_extensions import TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def f(x: Ts) -> Ts: # E: TypeVarTuple "Ts" is only valid with an unpack + return x + +v = f(1, 2, "A") # E: Too many arguments for "f" +reveal_type(v) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleSimpleDecoratorWorks] +from typing import TypeVar, Callable +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +T = TypeVar("T") + +def decorator(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], T]: + def wrapper(*args: Unpack[Ts]) -> T: + return f(*args) + return wrapper + +@decorator +def f(a: int, b: int) -> int: ... +reveal_type(f) # N: Revealed type is "def (builtins.int, builtins.int) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTupleWithUnpackIterator] +from typing import Tuple +from typing_extensions import Unpack + +def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[float, ...]], bool]]) -> None: + for x in xs: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testFixedUnpackItemInInstanceArguments] +from typing import TypeVar, Callable, Tuple, Generic +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") + +class C(Generic[T, Unpack[Ts], S]): + prefix: T + suffix: S + middle: Tuple[Unpack[Ts]] + +Ints = Tuple[int, int] +c: C[Unpack[Ints]] +reveal_type(c.prefix) # N: Revealed type is "builtins.int" +reveal_type(c.suffix) # N: Revealed type is "builtins.int" +reveal_type(c.middle) # N: Revealed type is "Tuple[()]" +[builtins fixtures/tuple.pyi] + +[case testVariadicUnpackItemInInstanceArguments] +from typing import TypeVar, Callable, Tuple, Generic +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") + +class Other(Generic[Unpack[Ts]]): ... +class C(Generic[T, Unpack[Ts], S]): + prefix: T + suffix: S + x: Tuple[Unpack[Ts]] + y: Callable[[Unpack[Ts]], None] + z: Other[Unpack[Ts]] + +Ints = Tuple[int, ...] +c: C[Unpack[Ints]] +reveal_type(c.prefix) # N: Revealed type is "builtins.int" +reveal_type(c.suffix) # N: Revealed type is "builtins.int" +reveal_type(c.x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(c.y) # N: Revealed type is "def (*builtins.int)" +reveal_type(c.z) # N: Revealed type is "__main__.Other[Unpack[builtins.tuple[builtins.int, ...]]]" +[builtins fixtures/tuple.pyi] + +[case testTooFewItemsInInstanceArguments] +from typing import Generic, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +class C(Generic[T, Unpack[Ts], S]): ... + +c: C[int] # E: Bad number of arguments, expected: at least 2, given: 1 +reveal_type(c) # N: Revealed type is "__main__.C[Any, Unpack[builtins.tuple[Any, ...]], Any]" +[builtins fixtures/tuple.pyi] + +[case testVariadicClassUpperBoundCheck] +from typing import Tuple, TypeVar, Generic +from typing_extensions import Unpack, TypeVarTuple + +class A: ... +class B: ... +class C: ... +class D: ... + +T = TypeVar("T", bound=int) +S = TypeVar("S", bound=str) +Ts = TypeVarTuple("Ts") + +class G(Generic[T, Unpack[Ts], S]): ... +First = Tuple[A, B] +Second = Tuple[C, D] +x: G[Unpack[First], Unpack[Second]] # E: Type argument "A" of "G" must be a subtype of "int" \ + # E: Type argument "D" of "G" must be a subtype of "str" +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleType] +from typing import Tuple, Callable +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class A(Tuple[Unpack[Ts]]): + fn: Callable[[Unpack[Ts]], None] + +x: A[int] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.A[builtins.int]]" +reveal_type(x[0]) # N: Revealed type is "builtins.int" +reveal_type(x.fn) # N: Revealed type is "def (builtins.int)" + +y: A[int, str] +reveal_type(y) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.A[builtins.int, builtins.str]]" +reveal_type(y[0]) # N: Revealed type is "builtins.int" +reveal_type(y.fn) # N: Revealed type is "def (builtins.int, builtins.str)" + +z: A[Unpack[Tuple[int, ...]]] +reveal_type(z) # N: Revealed type is "__main__.A[Unpack[builtins.tuple[builtins.int, ...]]]" +reveal_type(z[0]) # N: Revealed type is "builtins.int" +reveal_type(z.fn) # N: Revealed type is "def (*builtins.int)" + +t: A[int, Unpack[Tuple[int, str]], str] +reveal_type(t) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str, builtins.str, fallback=__main__.A[builtins.int, builtins.int, builtins.str, builtins.str]]" +reveal_type(t[0]) # N: Revealed type is "builtins.int" +reveal_type(t.fn) # N: Revealed type is "def (builtins.int, builtins.int, builtins.str, builtins.str)" +[builtins fixtures/tuple.pyi] + +[case testVariadicNamedTuple] +from typing import Tuple, Callable, NamedTuple, Generic, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +class A(NamedTuple, Generic[Unpack[Ts], T]): + fn: Callable[[Unpack[Ts]], None] + val: T + +y: A[int, str] +reveal_type(y) # N: Revealed type is "Tuple[def (builtins.int), builtins.str, fallback=__main__.A[builtins.int, builtins.str]]" +reveal_type(y[0]) # N: Revealed type is "def (builtins.int)" +reveal_type(y.fn) # N: Revealed type is "def (builtins.int)" + +z: A[Unpack[Tuple[int, ...]]] +reveal_type(z) # N: Revealed type is "Tuple[def (*builtins.int), builtins.int, fallback=__main__.A[Unpack[builtins.tuple[builtins.int, ...]], builtins.int]]" +reveal_type(z.fn) # N: Revealed type is "def (*builtins.int)" + +t: A[int, Unpack[Tuple[int, str]], str] +reveal_type(t) # N: Revealed type is "Tuple[def (builtins.int, builtins.int, builtins.str), builtins.str, fallback=__main__.A[builtins.int, builtins.int, builtins.str, builtins.str]]" + +def test(x: int, y: str) -> None: ... +nt = A(fn=test, val=42) +reveal_type(nt) # N: Revealed type is "Tuple[def (builtins.int, builtins.str), builtins.int, fallback=__main__.A[builtins.int, builtins.str, builtins.int]]" + +def bad() -> int: ... +nt2 = A(fn=bad, val=42) # E: Argument "fn" to "A" has incompatible type "Callable[[], int]"; expected "Callable[[], None]" +[builtins fixtures/tuple.pyi] + +[case testVariadicTypedDict] +from typing import Tuple, Callable, Generic, TypeVar +from typing_extensions import TypeVarTuple, Unpack, TypedDict + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +class A(TypedDict, Generic[Unpack[Ts], T]): + fn: Callable[[Unpack[Ts]], None] + val: T + +y: A[int, str] +reveal_type(y) # N: Revealed type is "TypedDict('__main__.A', {'fn': def (builtins.int), 'val': builtins.str})" +reveal_type(y["fn"]) # N: Revealed type is "def (builtins.int)" + +z: A[Unpack[Tuple[int, ...]]] +reveal_type(z) # N: Revealed type is "TypedDict('__main__.A', {'fn': def (*builtins.int), 'val': builtins.int})" +reveal_type(z["fn"]) # N: Revealed type is "def (*builtins.int)" + +t: A[int, Unpack[Tuple[int, str]], str] +reveal_type(t) # N: Revealed type is "TypedDict('__main__.A', {'fn': def (builtins.int, builtins.int, builtins.str), 'val': builtins.str})" + +def test(x: int, y: str) -> None: ... +td = A({"fn": test, "val": 42}) +reveal_type(td) # N: Revealed type is "TypedDict('__main__.A', {'fn': def (builtins.int, builtins.str), 'val': builtins.int})" + +def bad() -> int: ... +td2 = A({"fn": bad, "val": 42}) # E: Incompatible types (expression has type "Callable[[], int]", TypedDict item "fn" has type "Callable[[], None]") +[builtins fixtures/tuple.pyi] + +[case testFixedUnpackWithRegularInstance] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack + +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") +T4 = TypeVar("T4") + +class C(Generic[T1, T2, T3, T4]): ... +x: C[int, Unpack[Alias], str] +Alias = Tuple[int, str] +reveal_type(x) # N: Revealed type is "__main__.C[builtins.int, builtins.int, builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicUnpackWithRegularInstance] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack + +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") +T4 = TypeVar("T4") + +class C(Generic[T1, T2, T3, T4]): ... +x: C[int, Unpack[Alias], str, str] # E: Unpack is only valid in a variadic position +Alias = Tuple[int, ...] +reveal_type(x) # N: Revealed type is "__main__.C[Any, Any, Any, Any]" +y: C[int, Unpack[Undefined]] # E: Name "Undefined" is not defined +reveal_type(y) # N: Revealed type is "__main__.C[Any, Any, Any, Any]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasInvalidUnpackNoCrash] +from typing import Tuple, Generic, Union, List +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") +Alias = Tuple[int, Unpack[Ts], str] + +A = Union[int, str] +x: List[Alias[int, Unpack[A], str]] # E: "Union[int, str]" cannot be unpacked (must be tuple or TypeVarTuple) +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.int, Unpack[builtins.tuple[Any, ...]], builtins.str, builtins.str]]" +y: List[Alias[int, Unpack[Undefined], str]] # E: Name "Undefined" is not defined +reveal_type(y) # N: Revealed type is "builtins.list[Tuple[builtins.int, Unpack[builtins.tuple[Any, ...]], builtins.str]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasForwardRefToFixedUnpack] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Alias = Tuple[T, Unpack[Ts], S] +x: Alias[int, Unpack[Other]] +Other = Tuple[int, str] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasForwardRefToVariadicUnpack] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Alias = Tuple[T, Unpack[Ts], S] +x: Alias[int, Unpack[Other]] +Other = Tuple[int, ...] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testVariadicInstanceStrictPrefixSuffixCheck] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +class C(Generic[T, Unpack[Ts], S]): ... + +def foo(x: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + y: C[int, Unpack[Ts]] # E: TypeVarTuple cannot be split + z: C[Unpack[Ts], int] # E: TypeVarTuple cannot be split + return x +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasStrictPrefixSuffixCheck] +from typing import Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Alias = Tuple[T, Unpack[Ts], S] + +def foo(x: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + y: Alias[int, Unpack[Ts]] # E: TypeVarTuple cannot be split + z: Alias[Unpack[Ts], int] # E: TypeVarTuple cannot be split + return x +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleWithIsInstance] +# flags: --warn-unreachable +from typing import Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +TP = TypeVarTuple("TP") +class A(Tuple[Unpack[TP]]): ... + +def test(d: A[int, str]) -> None: + if isinstance(d, A): + reveal_type(d) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.A[builtins.int, builtins.str]]" + else: + reveal_type(d) # E: Statement is unreachable + +class B(Generic[Unpack[TP]]): ... + +def test2(d: B[int, str]) -> None: + if isinstance(d, B): + reveal_type(d) # N: Revealed type is "__main__.B[builtins.int, builtins.str]" + else: + reveal_type(d) # E: Statement is unreachable +[builtins fixtures/isinstancelist.pyi] + +[case testVariadicTupleSubtyping] +from typing import Tuple +from typing_extensions import Unpack + +def f1(x: Tuple[float, ...]) -> None: ... +def f2(x: Tuple[float, Unpack[Tuple[float, ...]]]) -> None: ... +def f3(x: Tuple[Unpack[Tuple[float, ...]], float]) -> None: ... +def f4(x: Tuple[float, Unpack[Tuple[float, ...]], float]) -> None: ... + +t1: Tuple[int, int] +t2: Tuple[int, Unpack[Tuple[int, ...]]] +t3: Tuple[Unpack[Tuple[int, ...]], int] +t4: Tuple[int, Unpack[Tuple[int, ...]], int] +t5: Tuple[int, ...] + +tl: Tuple[int, int, Unpack[Tuple[int, ...]]] +tr: Tuple[Unpack[Tuple[int, ...]], int, int] + +f1(t1) +f1(t2) +f1(t3) +f1(t4) +f1(t5) + +f1(tl) +f1(tr) + +f2(t1) +f2(t2) +f2(t3) +f2(t4) +f2(t5) # E: Argument 1 to "f2" has incompatible type "Tuple[int, ...]"; expected "Tuple[float, Unpack[Tuple[float, ...]]]" + +f2(tl) +f2(tr) + +f3(t1) +f3(t2) +f3(t3) +f3(t4) +f3(t5) # E: Argument 1 to "f3" has incompatible type "Tuple[int, ...]"; expected "Tuple[Unpack[Tuple[float, ...]], float]" + +f3(tl) +f3(tr) + +f4(t1) +f4(t2) # E: Argument 1 to "f4" has incompatible type "Tuple[int, Unpack[Tuple[int, ...]]]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" +f4(t3) # E: Argument 1 to "f4" has incompatible type "Tuple[Unpack[Tuple[int, ...]], int]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" +f4(t4) +f4(t5) # E: Argument 1 to "f4" has incompatible type "Tuple[int, ...]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" + +f4(tl) +f4(tr) + +t5_verbose: Tuple[Unpack[Tuple[int, ...]]] +t5 = t5_verbose # OK +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleInference] +from typing import List, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +def f(x: Tuple[int, Unpack[Tuple[T, ...]]]) -> T: ... + +vt0: Tuple[int, ...] +f(vt0) # E: Argument 1 to "f" has incompatible type "Tuple[int, ...]"; expected "Tuple[int, Unpack[Tuple[int, ...]]]" + +vt1: Tuple[Unpack[Tuple[int, ...]], int] +reveal_type(f(vt1)) # N: Revealed type is "builtins.int" + +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +def g(x: Tuple[T, Unpack[Ts], S]) -> Tuple[T, Unpack[Ts], S]: ... +g(vt0) # E: Argument 1 to "g" has incompatible type "Tuple[int, ...]"; expected "Tuple[int, Unpack[Tuple[int, ...]], int]" + +U = TypeVar("U") +def h(x: List[Tuple[T, S, U]]) -> Tuple[T, S, U]: ... +vt2: Tuple[Unpack[Tuple[int, ...]], int] +vt2 = h(reveal_type([])) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.int, builtins.int]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicSelfTypeErasure] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class Array(Generic[Unpack[Ts]]): + def _close(self) -> None: ... + + def close(self) -> None: + self._close() +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassFixed] +from typing import Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +class C(B[int, str]): ... +class D(B[Unpack[Tuple[int, ...]]]): ... + +def fii(x: B[int, int]) -> None: ... +def fis(x: B[int, str]) -> None: ... +def fiv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +fii(C()) # E: Argument 1 to "fii" has incompatible type "C"; expected "B[int, int]" +fii(D()) # E: Argument 1 to "fii" has incompatible type "D"; expected "B[int, int]" +fis(C()) +fis(D()) # E: Argument 1 to "fis" has incompatible type "D"; expected "B[int, str]" +fiv(C()) # E: Argument 1 to "fiv" has incompatible type "C"; expected "B[Unpack[Tuple[int, ...]]]" +fiv(D()) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassSame] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +class C(B[Unpack[Ts]]): ... + +def fii(x: B[int, int]) -> None: ... +def fis(x: B[int, str]) -> None: ... +def fiv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +cii: C[int, int] +cis: C[int, str] +civ: C[Unpack[Tuple[int, ...]]] + +fii(cii) +fii(cis) # E: Argument 1 to "fii" has incompatible type "C[int, str]"; expected "B[int, int]" +fii(civ) # E: Argument 1 to "fii" has incompatible type "C[Unpack[Tuple[int, ...]]]"; expected "B[int, int]" + +fis(cii) # E: Argument 1 to "fis" has incompatible type "C[int, int]"; expected "B[int, str]" +fis(cis) +fis(civ) # E: Argument 1 to "fis" has incompatible type "C[Unpack[Tuple[int, ...]]]"; expected "B[int, str]" + +fiv(cii) +fiv(cis) # E: Argument 1 to "fiv" has incompatible type "C[int, str]"; expected "B[Unpack[Tuple[int, ...]]]" +fiv(civ) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassExtra] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... + +T = TypeVar("T") +class C(B[int, Unpack[Ts], T]): ... + +def ff(x: B[int, int, int]) -> None: ... +def fv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +cii: C[int, int] +cis: C[int, str] +civ: C[Unpack[Tuple[int, ...]]] + +ff(cii) +ff(cis) # E: Argument 1 to "ff" has incompatible type "C[int, str]"; expected "B[int, int, int]" +ff(civ) # E: Argument 1 to "ff" has incompatible type "C[Unpack[Tuple[int, ...]]]"; expected "B[int, int, int]" + +fv(cii) +fv(cis) # E: Argument 1 to "fv" has incompatible type "C[int, str]"; expected "B[Unpack[Tuple[int, ...]]]" +fv(civ) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassVariadic] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +T = TypeVar("T") +class C(B[Unpack[Tuple[T, ...]]]): ... + +def ff(x: B[int, int]) -> None: ... +def fv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +ci: C[int] +ff(ci) # E: Argument 1 to "ff" has incompatible type "C[int]"; expected "B[int, int]" +fv(ci) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassMethodAccess] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): + def meth(self) -> Tuple[Unpack[Ts]]: ... + +class C1(B[int, str]): ... +class C2(B[Unpack[Ts]]): ... +T = TypeVar("T") +class C3(B[int, Unpack[Ts], T]): ... +class C4(B[Unpack[Tuple[T, ...]]]): ... + +c1: C1 +reveal_type(c1.meth()) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +c2f: C2[int, str] +c2v: C2[Unpack[Tuple[int, ...]]] +reveal_type(c2f.meth()) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +reveal_type(c2v.meth()) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +c3f: C3[int, str] +c3v: C3[Unpack[Tuple[int, ...]]] +reveal_type(c3f.meth()) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str]" +reveal_type(c3v.meth()) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.int]" + +c4: C4[int] +reveal_type(c4.meth()) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleAnySubtype] +from typing import Any, Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +class C1(B[Unpack[Tuple[Any, ...]]]): ... +c1 = C1() +class C2(B): ... +c2 = C2() +x: B[int, str] +x = c1 +x = c2 +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleAnySubtypeTupleType] +from typing import Any, Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Tuple[Unpack[Ts]]): ... +class C1(B[Unpack[Tuple[Any, ...]]]): ... +c1 = C1() +class C2(B): ... +c2 = C2() +x: B[int, str] +x = c1 +x = c2 +[builtins fixtures/tuple.pyi] + +[case testUnpackingVariadicTuplesTypeVar] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: + x1, y1, z1 = arg # E: Variadic tuple unpacking requires a star target + reveal_type(x1) # N: Revealed type is "Any" + reveal_type(y1) # N: Revealed type is "Any" + reveal_type(z1) # N: Revealed type is "Any" + x2, *y2, z2 = arg + reveal_type(x2) # N: Revealed type is "builtins.int" + reveal_type(y2) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z2) # N: Revealed type is "builtins.str" + x3, *y3 = arg + reveal_type(x3) # N: Revealed type is "builtins.int" + reveal_type(y3) # N: Revealed type is "builtins.list[builtins.object]" + *y4, z4 = arg + reveal_type(y4) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z4) # N: Revealed type is "builtins.str" + x5, xx5, *y5, z5, zz5 = arg # E: Too many assignment targets for variadic unpack + reveal_type(x5) # N: Revealed type is "Any" + reveal_type(xx5) # N: Revealed type is "Any" + reveal_type(y5) # N: Revealed type is "builtins.list[Any]" + reveal_type(z5) # N: Revealed type is "Any" + reveal_type(zz5) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testUnpackingVariadicTuplesHomogeneous] +from typing import Tuple +from typing_extensions import Unpack + +def bar(arg: Tuple[int, Unpack[Tuple[float, ...]], str]) -> None: + x1, y1, z1 = arg # E: Variadic tuple unpacking requires a star target + reveal_type(x1) # N: Revealed type is "Any" + reveal_type(y1) # N: Revealed type is "Any" + reveal_type(z1) # N: Revealed type is "Any" + x2, *y2, z2 = arg + reveal_type(x2) # N: Revealed type is "builtins.int" + reveal_type(y2) # N: Revealed type is "builtins.list[builtins.float]" + reveal_type(z2) # N: Revealed type is "builtins.str" + x3, *y3 = arg + reveal_type(x3) # N: Revealed type is "builtins.int" + reveal_type(y3) # N: Revealed type is "builtins.list[builtins.object]" + *y4, z4 = arg + reveal_type(y4) # N: Revealed type is "builtins.list[builtins.float]" + reveal_type(z4) # N: Revealed type is "builtins.str" + x5, xx5, *y5, z5, zz5 = arg # E: Too many assignment targets for variadic unpack + reveal_type(x5) # N: Revealed type is "Any" + reveal_type(xx5) # N: Revealed type is "Any" + reveal_type(y5) # N: Revealed type is "builtins.list[Any]" + reveal_type(z5) # N: Revealed type is "Any" + reveal_type(zz5) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testRepackingVariadicTuplesTypeVar] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: + x1, *y1, z1 = *arg, + reveal_type(x1) # N: Revealed type is "builtins.int" + reveal_type(y1) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z1) # N: Revealed type is "builtins.str" + x2, *y2, z2 = 1, *arg, 2 + reveal_type(x2) # N: Revealed type is "builtins.int" + reveal_type(y2) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z2) # N: Revealed type is "builtins.int" + x3, *y3 = *arg, 42 + reveal_type(x3) # N: Revealed type is "builtins.int" + reveal_type(y3) # N: Revealed type is "builtins.list[builtins.object]" + *y4, z4 = 42, *arg + reveal_type(y4) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z4) # N: Revealed type is "builtins.str" + x5, xx5, *y5, z5, zz5 = 1, *arg, 2 + reveal_type(x5) # N: Revealed type is "builtins.int" + reveal_type(xx5) # N: Revealed type is "builtins.int" + reveal_type(y5) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z5) # N: Revealed type is "builtins.str" + reveal_type(zz5) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testRepackingVariadicTuplesHomogeneous] +from typing import Tuple +from typing_extensions import Unpack + +def foo(arg: Tuple[int, Unpack[Tuple[float, ...]], str]) -> None: + x1, *y1, z1 = *arg, + reveal_type(x1) # N: Revealed type is "builtins.int" + reveal_type(y1) # N: Revealed type is "builtins.list[builtins.float]" + reveal_type(z1) # N: Revealed type is "builtins.str" + x2, *y2, z2 = 1, *arg, 2 + reveal_type(x2) # N: Revealed type is "builtins.int" + reveal_type(y2) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type(z2) # N: Revealed type is "builtins.int" + x3, *y3 = *arg, 42 + reveal_type(x3) # N: Revealed type is "builtins.int" + reveal_type(y3) # N: Revealed type is "builtins.list[builtins.object]" + *y4, z4 = 42, *arg + reveal_type(y4) # N: Revealed type is "builtins.list[builtins.float]" + reveal_type(z4) # N: Revealed type is "builtins.str" + x5, xx5, *y5, z5, zz5 = 1, *arg, 2 + reveal_type(x5) # N: Revealed type is "builtins.int" + reveal_type(xx5) # N: Revealed type is "builtins.int" + reveal_type(y5) # N: Revealed type is "builtins.list[builtins.float]" + reveal_type(z5) # N: Revealed type is "builtins.str" + reveal_type(zz5) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testPackingVariadicTuplesTypeVar] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: + x = *arg, + reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str]" + y = 1, *arg, 2 + reveal_type(y) # N: Revealed type is "Tuple[builtins.int, builtins.int, Unpack[Ts`-1], builtins.str, builtins.int]" + z = (*arg, *arg) + reveal_type(z) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +[builtins fixtures/tuple.pyi] + +[case testPackingVariadicTuplesHomogeneous] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple +from typing_extensions import Unpack + +a: Tuple[float, ...] +b: Tuple[int, Unpack[Tuple[float, ...]], str] + +x = *a, +reveal_type(x) # N: Revealed type is "builtins.tuple[builtins.float, ...]" +y = 1, *a, 2 +reveal_type(y) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int]" +z = (*a, *a) +reveal_type(z) # N: Revealed type is "builtins.tuple[builtins.float, ...]" + +x2 = *b, +reveal_type(x2) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str]" +y2 = 1, *b, 2 +reveal_type(y2) # N: Revealed type is "Tuple[builtins.int, builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.str, builtins.int]" +z2 = (*b, *b) +reveal_type(z2) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleInListSetExpr] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +vt: Tuple[int, Unpack[Tuple[float, ...]], int] +reveal_type([1, *vt]) # N: Revealed type is "builtins.list[builtins.float]" +reveal_type({1, *vt}) # N: Revealed type is "builtins.set[builtins.float]" + +Ts = TypeVarTuple("Ts") +def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: + reveal_type([1, *arg]) # N: Revealed type is "builtins.list[builtins.object]" + reveal_type({1, *arg}) # N: Revealed type is "builtins.set[builtins.object]" +[builtins fixtures/isinstancelist.pyi] + +[case testVariadicTupleInTupleContext] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple, Optional +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def test(x: Optional[Tuple[Unpack[Ts]]] = None) -> Tuple[Unpack[Ts]]: ... + +vt: Tuple[int, Unpack[Tuple[float, ...]], int] +vt = 1, *test(), 2 # OK, type context is used +vt2 = 1, *test(), 2 # E: Need type annotation for "vt2" +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleConcatenation] +# flags: --enable-incomplete-feature=PreciseTupleTypes +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +vtf: Tuple[float, ...] +vt: Tuple[int, Unpack[Tuple[float, ...]], int] + +reveal_type(vt + (1, 2)) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int, Literal[1]?, Literal[2]?]" +reveal_type((1, 2) + vt) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int]" +reveal_type(vt + vt) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.float], ...]" +reveal_type(vtf + (1, 2)) # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.float, ...]], Literal[1]?, Literal[2]?]" +reveal_type((1, 2) + vtf) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, Unpack[builtins.tuple[builtins.float, ...]]]" + +Ts = TypeVarTuple("Ts") +def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: + reveal_type(arg + (1, 2)) # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`-1], builtins.str, Literal[1]?, Literal[2]?]" + reveal_type((1, 2) + arg) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, builtins.int, Unpack[Ts`-1], builtins.str]" + reveal_type(arg + arg) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAnyOverload] +from typing import Any, Generic, overload, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class Array(Generic[Unpack[Ts]]): ... + +class A: + @overload + def f(self, x: Tuple[Unpack[Ts]]) -> Array[Unpack[Ts]]: ... + @overload + def f(self, x: Any) -> Any: ... + def f(self, x: Any) -> Any: + ... +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleInferAgainstAny] +from typing import Any, Tuple, TypeVar +from typing_extensions import Unpack + +T = TypeVar("T") + +def test(x: int, t: Tuple[T, ...]) -> Tuple[int, Unpack[Tuple[T, ...]]]: + ... +a: Any = test(42, ()) +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleIndexTypeVar] +from typing import Any, List, Sequence, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +def f(data: Sequence[Tuple[Unpack[Ts]]]) -> List[Any]: + return [d[0] for d in data] # E: Tuple index out of range \ + # N: Variadic tuple can have length 0 + +T = TypeVar("T") +def g(data: Sequence[Tuple[T, Unpack[Ts]]]) -> List[T]: + return [d[0] for d in data] # OK +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleOverloadMatch] +from typing import Any, Generic, overload, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +_Ts = TypeVarTuple("_Ts") +_T = TypeVar("_T") +_T2 = TypeVar("_T2") + +class Container(Generic[_T]): ... +class Array(Generic[Unpack[_Ts]]): ... + +@overload +def build(entity: Container[_T], /) -> Array[_T]: ... +@overload +def build(entity: Container[_T], entity2: Container[_T2], /) -> Array[_T, _T2]: ... +@overload +def build(*entities: Container[Any]) -> Array[Unpack[Tuple[Any, ...]]]: ... +def build(*entities: Container[Any]) -> Array[Unpack[Tuple[Any, ...]]]: + ... + +def test(a: Container[Any], b: Container[int], c: Container[str]): + reveal_type(build(a, b)) # N: Revealed type is "__main__.Array[Any, builtins.int]" + reveal_type(build(b, c)) # N: Revealed type is "__main__.Array[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleOverloadArbitraryLength] +from typing import Any, Tuple, TypeVar, TypeVarTuple, Unpack, overload + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +@overload +def add(self: Tuple[Unpack[Ts]], other: Tuple[T]) -> Tuple[Unpack[Ts], T]: + ... +@overload +def add(self: Tuple[T, ...], other: Tuple[T, ...]) -> Tuple[T, ...]: + ... +def add(self: Any, other: Any) -> Any: + ... +def test(a: Tuple[int, str], b: Tuple[bool], c: Tuple[bool, ...]): + reveal_type(add(a, b)) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.bool]" + reveal_type(add(b, c)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleOverloadOverlap] +from typing import Union, overload, Tuple +from typing_extensions import Unpack + +class Int(int): ... + +A = Tuple[int, Unpack[Tuple[int, ...]]] +B = Tuple[int, Unpack[Tuple[str, ...]]] + +@overload +def f(arg: A) -> int: ... +@overload +def f(arg: B) -> str: ... +def f(arg: Union[A, B]) -> Union[int, str]: + ... + +A1 = Tuple[int, Unpack[Tuple[Int, ...]]] +B1 = Tuple[Unpack[Tuple[Int, ...]], int] + +@overload +def f1(arg: A1) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f1(arg: B1) -> str: ... +def f1(arg: Union[A1, B1]) -> Union[int, str]: + ... + +A2 = Tuple[int, int, int] +B2 = Tuple[int, Unpack[Tuple[int, ...]]] + +@overload +def f2(arg: A2) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f2(arg: B2) -> str: ... +def f2(arg: Union[A2, B2]) -> Union[int, str]: + ... + +A3 = Tuple[int, int, int] +B3 = Tuple[int, Unpack[Tuple[str, ...]]] + +@overload +def f3(arg: A3) -> int: ... +@overload +def f3(arg: B3) -> str: ... +def f3(arg: Union[A3, B3]) -> Union[int, str]: + ... + +A4 = Tuple[int, int, Unpack[Tuple[int, ...]]] +B4 = Tuple[int] + +@overload +def f4(arg: A4) -> int: ... +@overload +def f4(arg: B4) -> str: ... +def f4(arg: Union[A4, B4]) -> Union[int, str]: + ... +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleIndexOldStyleNonNormalizedAndNonLiteral] +from typing import Any, Tuple +from typing_extensions import Unpack + +t: Tuple[Unpack[Tuple[int, ...]]] +reveal_type(t[42]) # N: Revealed type is "builtins.int" +i: int +reveal_type(t[i]) # N: Revealed type is "builtins.int" +t1: Tuple[int, Unpack[Tuple[int, ...]]] +reveal_type(t1[i]) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleNotConcreteCallable] +from typing_extensions import Unpack, TypeVarTuple +from typing import Callable, TypeVar, Tuple + +T = TypeVar("T") +Args = TypeVarTuple("Args") +Args2 = TypeVarTuple("Args2") + +def submit(fn: Callable[[Unpack[Args]], T], *args: Unpack[Args]) -> T: + ... + +def submit2(fn: Callable[[int, Unpack[Args]], T], *args: Unpack[Tuple[int, Unpack[Args]]]) -> T: + ... + +def foo(func: Callable[[Unpack[Args]], T], *args: Unpack[Args]) -> T: + return submit(func, *args) + +def foo2(func: Callable[[Unpack[Args2]], T], *args: Unpack[Args2]) -> T: + return submit(func, *args) + +def foo3(func: Callable[[int, Unpack[Args2]], T], *args: Unpack[Args2]) -> T: + return submit2(func, 1, *args) + +def foo_bad(func: Callable[[Unpack[Args2]], T], *args: Unpack[Args2]) -> T: + return submit2(func, 1, *args) # E: Argument 1 to "submit2" has incompatible type "Callable[[VarArg(Unpack[Args2])], T]"; expected "Callable[[int, VarArg(Unpack[Args2])], T]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleParamSpecInteraction] +from typing_extensions import Unpack, TypeVarTuple, ParamSpec +from typing import Callable, TypeVar + +T = TypeVar("T") +Args = TypeVarTuple("Args") +Args2 = TypeVarTuple("Args2") +P = ParamSpec("P") + +def submit(fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + ... + +def foo(func: Callable[[Unpack[Args]], T], *args: Unpack[Args]) -> T: + return submit(func, *args) + +def foo2(func: Callable[[Unpack[Args]], T], *args: Unpack[Args2]) -> T: + return submit(func, *args) # E: Argument 2 to "submit" has incompatible type "*Tuple[Unpack[Args2]]"; expected "Unpack[Args]" + +def foo3(func: Callable[[int, Unpack[Args2]], T], *args: Unpack[Args2]) -> T: + return submit(func, 1, *args) +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleEmptySpecialCase] +from typing import Any, Callable, Generic +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") +class MyClass(Generic[Unpack[Ts]]): + func: Callable[[Unpack[Ts]], object] + + def __init__(self, func: Callable[[Unpack[Ts]], object]) -> None: + self.func = func + +explicit: MyClass[()] +reveal_type(explicit) # N: Revealed type is "__main__.MyClass[()]" +reveal_type(explicit.func) # N: Revealed type is "def () -> builtins.object" + +a: Any +explicit_2 = MyClass[()](a) +reveal_type(explicit_2) # N: Revealed type is "__main__.MyClass[()]" +reveal_type(explicit_2.func) # N: Revealed type is "def () -> builtins.object" + +Alias = MyClass[()] +explicit_3: Alias +reveal_type(explicit_3) # N: Revealed type is "__main__.MyClass[()]" +reveal_type(explicit_3.func) # N: Revealed type is "def () -> builtins.object" + +explicit_4 = Alias(a) +reveal_type(explicit_4) # N: Revealed type is "__main__.MyClass[()]" +reveal_type(explicit_4.func) # N: Revealed type is "def () -> builtins.object" + +def no_args() -> None: ... +implicit = MyClass(no_args) +reveal_type(implicit) # N: Revealed type is "__main__.MyClass[()]" +reveal_type(implicit.func) # N: Revealed type is "def () -> builtins.object" + +def one_arg(__a: int) -> None: ... +x = MyClass(one_arg) +x = explicit # E: Incompatible types in assignment (expression has type "MyClass[()]", variable has type "MyClass[int]") + +# Consistently handle special case for no argument aliases +Direct = MyClass +y = Direct(one_arg) +reveal_type(y) # N: Revealed type is "__main__.MyClass[builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleRuntimeTypeApplication] +from typing import Generic, TypeVar, Tuple +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +class C(Generic[T, Unpack[Ts], S]): ... + +Ints = Tuple[int, int] +x = C[Unpack[Ints]]() +reveal_type(x) # N: Revealed type is "__main__.C[builtins.int, builtins.int]" + +y = C[Unpack[Tuple[int, ...]]]() +reveal_type(y) # N: Revealed type is "__main__.C[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.int]" + +z = C[int]() # E: Bad number of arguments, expected: at least 2, given: 1 +reveal_type(z) # N: Revealed type is "__main__.C[Any, Unpack[builtins.tuple[Any, ...]], Any]" +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleTupleSubclassPrefixSuffix] +from typing import Tuple +from typing_extensions import Unpack + +i: int + +class A(Tuple[int, Unpack[Tuple[int, ...]]]): ... +a: A +reveal_type(a[i]) # N: Revealed type is "builtins.int" + +class B(Tuple[Unpack[Tuple[int, ...]], int]): ... +b: B +reveal_type(b[i]) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testVariadicClassSubclassInit] +from typing import Tuple, Generic, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): + def __init__(self, x: Tuple[Unpack[Ts]], *args: Unpack[Ts]) -> None: ... +reveal_type(B) # N: Revealed type is "def [Ts] (x: Tuple[Unpack[Ts`1]], *args: Unpack[Ts`1]) -> __main__.B[Unpack[Ts`1]]" + +T = TypeVar("T") +S = TypeVar("S") +class C(B[T, S]): ... +reveal_type(C) # N: Revealed type is "def [T, S] (x: Tuple[T`1, S`2], T`1, S`2) -> __main__.C[T`1, S`2]" +[builtins fixtures/tuple.pyi] + +[case testVariadicClassGenericSelf] +from typing import Tuple, Generic, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): + def copy(self: T) -> T: ... + def on_pair(self: B[T, S]) -> Tuple[T, S]: ... + +b1: B[int] +reveal_type(b1.on_pair()) # E: Invalid self argument "B[int]" to attribute function "on_pair" with type "Callable[[B[T, S]], Tuple[T, S]]" \ + # N: Revealed type is "Tuple[Never, Never]" +b2: B[int, str] +reveal_type(b2.on_pair()) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +b3: B[int, str, int] +reveal_type(b3.on_pair()) # E: Invalid self argument "B[int, str, int]" to attribute function "on_pair" with type "Callable[[B[T, S]], Tuple[T, S]]" \ + # N: Revealed type is "Tuple[Never, Never]" + +class C(B[T, S]): ... +c: C[int, str] +reveal_type(c.copy()) # N: Revealed type is "__main__.C[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicClassNewStyleSelf] +from typing import Generic, TypeVar, Self +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): + next: Self + def copy(self) -> Self: + return self.next + +b: B[int, str, int] +reveal_type(b.next) # N: Revealed type is "__main__.B[builtins.int, builtins.str, builtins.int]" +reveal_type(b.copy()) # N: Revealed type is "__main__.B[builtins.int, builtins.str, builtins.int]" +reveal_type(B.copy(b)) # N: Revealed type is "__main__.B[builtins.int, builtins.str, builtins.int]" + +T = TypeVar("T") +S = TypeVar("S") +class C(B[T, S]): ... +c: C[int, str] + +reveal_type(c.next) # N: Revealed type is "__main__.C[builtins.int, builtins.str]" +reveal_type(c.copy()) # N: Revealed type is "__main__.C[builtins.int, builtins.str]" +reveal_type(C.copy(c)) # N: Revealed type is "__main__.C[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleDataclass] +from dataclasses import dataclass +from typing import Generic, TypeVar, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +@dataclass +class B(Generic[Unpack[Ts]]): + items: Tuple[Unpack[Ts]] + +reveal_type(B) # N: Revealed type is "def [Ts] (items: Tuple[Unpack[Ts`1]]) -> __main__.B[Unpack[Ts`1]]" +b = B((1, "yes")) +reveal_type(b.items) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +T = TypeVar("T") +S = TypeVar("S") + +@dataclass +class C(B[T, S]): + first: T + second: S + +reveal_type(C) # N: Revealed type is "def [T, S] (items: Tuple[T`1, S`2], first: T`1, second: S`2) -> __main__.C[T`1, S`2]" +c = C((1, "yes"), 2, "no") +reveal_type(c.items) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +reveal_type(c.first) # N: Revealed type is "builtins.int" +reveal_type(c.second) # N: Revealed type is "builtins.str" +[builtins fixtures/dataclasses.pyi] +[typing fixtures/typing-medium.pyi] + +[case testVariadicTupleInProtocol] +from typing import Protocol, Tuple, List +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class P(Protocol[Unpack[Ts]]): + def items(self) -> Tuple[Unpack[Ts]]: ... + +class PC(Protocol[Unpack[Ts]]): + def meth(self, *args: Unpack[Ts]) -> None: ... + +def get_items(x: P[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: ... +def match(x: PC[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: ... + +class Bad: + def items(self) -> List[int]: ... + def meth(self, *, named: int) -> None: ... + +class Good: + def items(self) -> Tuple[int, str]: ... + def meth(self, __x: int, y: str) -> None: ... + +g: Good +reveal_type(get_items(g)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +reveal_type(match(g)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +b: Bad +get_items(b) # E: Argument 1 to "get_items" has incompatible type "Bad"; expected "P[Unpack[Tuple[Never, ...]]]" \ + # N: Following member(s) of "Bad" have conflicts: \ + # N: Expected: \ + # N: def items(self) -> Tuple[Never, ...] \ + # N: Got: \ + # N: def items(self) -> List[int] +match(b) # E: Argument 1 to "match" has incompatible type "Bad"; expected "PC[Unpack[Tuple[Never, ...]]]" \ + # N: Following member(s) of "Bad" have conflicts: \ + # N: Expected: \ + # N: def meth(self, *args: Never) -> None \ + # N: Got: \ + # N: def meth(self, *, named: int) -> None +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleCollectionCheck] +from typing import Tuple, Optional +from typing_extensions import Unpack + +allowed: Tuple[int, Unpack[Tuple[int, ...]]] + +x: Optional[int] +if x in allowed: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testJoinOfVariadicTupleCallablesNoCrash] +from typing import Callable, Tuple + +f: Callable[[int, *Tuple[str, ...], int], None] +g: Callable[[int, *Tuple[str, ...], int], None] +reveal_type([f, g]) # N: Revealed type is "builtins.list[def (builtins.int, *Unpack[Tuple[Unpack[builtins.tuple[builtins.str, ...]], builtins.int]])]" + +h: Callable[[int, *Tuple[str, ...], str], None] +reveal_type([f, h]) # N: Revealed type is "builtins.list[def (builtins.int, *Unpack[Tuple[Unpack[builtins.tuple[builtins.str, ...]], Never]])]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleBothUnpacksSimple] +from typing import Tuple +from typing_extensions import Unpack, TypeVarTuple, TypedDict + +class Keywords(TypedDict): + a: str + b: str + +Ints = Tuple[int, ...] + +def f(*args: Unpack[Ints], other: str = "no", **kwargs: Unpack[Keywords]) -> None: ... +reveal_type(f) # N: Revealed type is "def (*args: builtins.int, other: builtins.str =, **kwargs: Unpack[TypedDict('__main__.Keywords', {'a': builtins.str, 'b': builtins.str})])" +f(1, 2, a="a", b="b") # OK +f(1, 2, 3) # E: Missing named argument "a" for "f" \ + # E: Missing named argument "b" for "f" + +Ts = TypeVarTuple("Ts") +def g(*args: Unpack[Ts], other: str = "no", **kwargs: Unpack[Keywords]) -> None: ... +reveal_type(g) # N: Revealed type is "def [Ts] (*args: Unpack[Ts`-1], other: builtins.str =, **kwargs: Unpack[TypedDict('__main__.Keywords', {'a': builtins.str, 'b': builtins.str})])" +g(1, 2, a="a", b="b") # OK +g(1, 2, 3) # E: Missing named argument "a" for "g" \ + # E: Missing named argument "b" for "g" + +def bad( + *args: Unpack[Keywords], # E: "Keywords" cannot be unpacked (must be tuple or TypeVarTuple) + **kwargs: Unpack[Ints], # E: Unpack item in ** argument must be a TypedDict +) -> None: ... +reveal_type(bad) # N: Revealed type is "def (*args: Any, **kwargs: Any)" + +def bad2( + one: int, + *args: Unpack[Keywords], # E: "Keywords" cannot be unpacked (must be tuple or TypeVarTuple) + other: str = "no", + **kwargs: Unpack[Ints], # E: Unpack item in ** argument must be a TypedDict +) -> None: ... +reveal_type(bad2) # N: Revealed type is "def (one: builtins.int, *args: Any, other: builtins.str =, **kwargs: Any)" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleBothUnpacksCallable] +from typing import Callable, Tuple +from typing_extensions import Unpack, TypedDict + +class Keywords(TypedDict): + a: str + b: str +Ints = Tuple[int, ...] + +cb: Callable[[Unpack[Ints], Unpack[Keywords]], None] +reveal_type(cb) # N: Revealed type is "def (*builtins.int, **Unpack[TypedDict('__main__.Keywords', {'a': builtins.str, 'b': builtins.str})])" + +cb2: Callable[[int, Unpack[Ints], int, Unpack[Keywords]], None] +reveal_type(cb2) # N: Revealed type is "def (builtins.int, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.int]], **Unpack[TypedDict('__main__.Keywords', {'a': builtins.str, 'b': builtins.str})])" +cb2(1, 2, 3, a="a", b="b") +cb2(1, a="a", b="b") # E: Too few arguments +cb2(1, 2, 3, a="a") # E: Missing named argument "b" + +bad1: Callable[[Unpack[Ints], Unpack[Ints]], None] # E: More than one Unpack in a type is not allowed +reveal_type(bad1) # N: Revealed type is "def (*builtins.int)" +bad2: Callable[[Unpack[Keywords], Unpack[Keywords]], None] # E: "Keywords" cannot be unpacked (must be tuple or TypeVarTuple) +reveal_type(bad2) # N: Revealed type is "def (*Any, **Unpack[TypedDict('__main__.Keywords', {'a': builtins.str, 'b': builtins.str})])" +bad3: Callable[[Unpack[Keywords], Unpack[Ints]], None] # E: "Keywords" cannot be unpacked (must be tuple or TypeVarTuple) \ + # E: More than one Unpack in a type is not allowed +reveal_type(bad3) # N: Revealed type is "def (*Any)" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleBothUnpacksApplication] +from typing import Callable, TypeVar, Optional +from typing_extensions import Unpack, TypeVarTuple, TypedDict + +class Keywords(TypedDict): + a: str + b: str + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +def test( + x: int, + func: Callable[[Unpack[Ts]], T], + *args: Unpack[Ts], + other: Optional[str] = None, + **kwargs: Unpack[Keywords], +) -> T: + if bool(): + func(*args, **kwargs) # E: Extra argument "a" from **args + return func(*args) +def test2( + x: int, + func: Callable[[Unpack[Ts], Unpack[Keywords]], T], + *args: Unpack[Ts], + other: Optional[str] = None, + **kwargs: Unpack[Keywords], +) -> T: + if bool(): + func(*args) # E: Missing named argument "a" \ + # E: Missing named argument "b" + return func(*args, **kwargs) +[builtins fixtures/tuple.pyi] + +[case testUnpackTupleSpecialCaseNoCrash] +from typing import Tuple, TypeVar +from typing_extensions import Unpack + +T = TypeVar("T") + +def foo(*x: object) -> None: ... +def bar(*x: int) -> None: ... +def baz(*x: T) -> T: ... + +keys: Tuple[Unpack[Tuple[int, ...]]] + +foo(keys, 1) +foo(*keys, 1) + +bar(keys, 1) # E: Argument 1 to "bar" has incompatible type "Tuple[Unpack[Tuple[int, ...]]]"; expected "int" +bar(*keys, 1) # OK + +reveal_type(baz(keys, 1)) # N: Revealed type is "builtins.object" +reveal_type(baz(*keys, 1)) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleContextNoCrash] +from typing import Tuple, Unpack + +x: Tuple[int, Unpack[Tuple[int, ...]]] = () # E: Incompatible types in assignment (expression has type "Tuple[()]", variable has type "Tuple[int, Unpack[Tuple[int, ...]]]") +y: Tuple[int, Unpack[Tuple[int, ...]]] = (1, 2) +z: Tuple[int, Unpack[Tuple[int, ...]]] = (1,) +w: Tuple[int, Unpack[Tuple[int, ...]]] = (1, *[2, 3, 4]) +t: Tuple[int, Unpack[Tuple[int, ...]]] = (1, *(2, 3, 4)) +[builtins fixtures/tuple.pyi] + +[case testAliasToCallableWithUnpack] +from typing import Any, Callable, Tuple, Unpack + +_CallableValue = Callable[[Unpack[Tuple[Any, ...]]], Any] +def higher_order(f: _CallableValue) -> None: ... + +def good1(*args: int) -> None: ... +def good2(*args: str) -> int: ... + +def bad1(a: str, b: int, /) -> None: ... +def bad2(c: bytes, *args: int) -> str: ... +def bad3(*, d: str) -> int: ... +def bad4(**kwargs: None) -> None: ... + +higher_order(good1) +higher_order(good2) + +higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[str, int], None]"; expected "Callable[[VarArg(Any)], Any]" +higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "Callable[[bytes, VarArg(int)], str]"; expected "Callable[[VarArg(Any)], Any]" +higher_order(bad3) # E: Argument 1 to "higher_order" has incompatible type "Callable[[NamedArg(str, 'd')], int]"; expected "Callable[[VarArg(Any)], Any]" +higher_order(bad4) # E: Argument 1 to "higher_order" has incompatible type "Callable[[KwArg(None)], None]"; expected "Callable[[VarArg(Any)], Any]" +[builtins fixtures/tuple.pyi] + +[case testAliasToCallableWithUnpack2] +from typing import Any, Callable, Tuple, Unpack + +_CallableValue = Callable[[int, str, Unpack[Tuple[Any, ...]], int], Any] +def higher_order(f: _CallableValue) -> None: ... + +def good(a: int, b: str, *args: Unpack[Tuple[Unpack[Tuple[Any, ...]], int]]) -> int: ... +def bad1(a: str, b: int, /) -> None: ... +def bad2(c: bytes, *args: int) -> str: ... +def bad3(*, d: str) -> int: ... +def bad4(**kwargs: None) -> None: ... + +higher_order(good) +higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[str, int], None]"; expected "Callable[[int, str, VarArg(Unpack[Tuple[Unpack[Tuple[Any, ...]], int]])], Any]" +higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "Callable[[bytes, VarArg(int)], str]"; expected "Callable[[int, str, VarArg(Unpack[Tuple[Unpack[Tuple[Any, ...]], int]])], Any]" +higher_order(bad3) # E: Argument 1 to "higher_order" has incompatible type "Callable[[NamedArg(str, 'd')], int]"; expected "Callable[[int, str, VarArg(Unpack[Tuple[Unpack[Tuple[Any, ...]], int]])], Any]" +higher_order(bad4) # E: Argument 1 to "higher_order" has incompatible type "Callable[[KwArg(None)], None]"; expected "Callable[[int, str, VarArg(Unpack[Tuple[Unpack[Tuple[Any, ...]], int]])], Any]" +[builtins fixtures/tuple.pyi] + +[case testAliasToCallableWithUnpackInvalid] +from typing import Any, Callable, List, Tuple, TypeVar, Unpack + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") # E: Name "TypeVarTuple" is not defined + +def good(*x: int) -> int: ... +def bad(*x: int, y: int) -> int: ... + +Alias = Callable[[Unpack[T]], int] # E: "T" cannot be unpacked (must be tuple or TypeVarTuple) +x: Alias[int] +reveal_type(x) # N: Revealed type is "def (*Any) -> builtins.int" +x = good +x = bad # E: Incompatible types in assignment (expression has type "Callable[[VarArg(int), NamedArg(int, 'y')], int]", variable has type "Callable[[VarArg(Any)], int]") +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleInvariant] +from typing import Generic, Tuple +from typing_extensions import Unpack, TypeVarTuple +Ts = TypeVarTuple("Ts") + +class Array(Generic[Unpack[Ts]]): ... + +def pointwise_multiply(x: Array[Unpack[Ts]], y: Array[Unpack[Ts]]) -> Array[Unpack[Ts]]: ... + +def a1(x: Array[int], y: Array[str], z: Array[int, str]) -> None: + reveal_type(pointwise_multiply(x, x)) # N: Revealed type is "__main__.Array[builtins.int]" + reveal_type(pointwise_multiply(x, y)) # E: Cannot infer type argument 1 of "pointwise_multiply" \ + # N: Revealed type is "__main__.Array[Unpack[builtins.tuple[Any, ...]]]" + reveal_type(pointwise_multiply(x, z)) # E: Cannot infer type argument 1 of "pointwise_multiply" \ + # N: Revealed type is "__main__.Array[Unpack[builtins.tuple[Any, ...]]]" + +def func(x: Array[Unpack[Ts]], *args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: + ... + +def a2(x: Array[int, str]) -> None: + reveal_type(func(x, 2, "Hello")) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + reveal_type(func(x, 2)) # E: Cannot infer type argument 1 of "func" \ + # N: Revealed type is "builtins.tuple[Any, ...]" + reveal_type(func(x, 2, "Hello", True)) # E: Cannot infer type argument 1 of "func" \ + # N: Revealed type is "builtins.tuple[Any, ...]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleTypeApplicationOverload] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, overload, Callable + +T = TypeVar("T") +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") +Ts = TypeVarTuple("Ts") + +class C(Generic[T, Unpack[Ts]]): + @overload + def __init__(self, f: Callable[[Unpack[Ts]], T]) -> None: ... + @overload + def __init__(self, f: Callable[[T1, T2, T3, Unpack[Ts]], T], a: T1, b: T2, c: T3) -> None: ... + def __init__(self, f, *args, **kwargs) -> None: + ... + +reveal_type(C[int, str]) # N: Revealed type is "Overload(def (f: def (builtins.str) -> builtins.int) -> __main__.C[builtins.int, builtins.str], def [T1, T2, T3] (f: def (T1`-1, T2`-2, T3`-3, builtins.str) -> builtins.int, a: T1`-1, b: T2`-2, c: T3`-3) -> __main__.C[builtins.int, builtins.str])" +Alias = C[int, str] + +def f(x: int, y: int, z: int, t: int) -> str: ... +x = C(f, 0, 0, "hm") # E: Argument 1 to "C" has incompatible type "Callable[[int, int, int, int], str]"; expected "Callable[[int, int, str, int], str]" +reveal_type(x) # N: Revealed type is "__main__.C[builtins.str, builtins.int]" +reveal_type(C(f)) # N: Revealed type is "__main__.C[builtins.str, builtins.int, builtins.int, builtins.int, builtins.int]" +C[()] # E: At least 1 type argument(s) expected, none given +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAgainstParamSpecActualSuccess] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, Callable, Tuple, List +from typing_extensions import ParamSpec + +R = TypeVar("R") +P = ParamSpec("P") + +class CM(Generic[R]): ... +def cm(fn: Callable[P, R]) -> Callable[P, CM[R]]: ... + +Ts = TypeVarTuple("Ts") +@cm +def test(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: ... + +reveal_type(test) # N: Revealed type is "def [Ts] (*args: Unpack[Ts`-1]) -> __main__.CM[Tuple[Unpack[Ts`-1]]]" +reveal_type(test(1, 2, 3)) # N: Revealed type is "__main__.CM[Tuple[Literal[1]?, Literal[2]?, Literal[3]?]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAgainstParamSpecActualFailedNoCrash] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, Callable, Tuple, List +from typing_extensions import ParamSpec + +R = TypeVar("R") +P = ParamSpec("P") + +class CM(Generic[R]): ... +def cm(fn: Callable[P, List[R]]) -> Callable[P, CM[R]]: ... + +Ts = TypeVarTuple("Ts") +@cm # E: Argument 1 to "cm" has incompatible type "Callable[[VarArg(Unpack[Ts])], Tuple[Unpack[Ts]]]"; expected "Callable[[VarArg(Never)], List[Never]]" +def test(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: ... + +reveal_type(test) # N: Revealed type is "def (*args: Never) -> __main__.CM[Never]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAgainstParamSpecActualPrefix] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, Callable, Tuple, List +from typing_extensions import ParamSpec, Concatenate + +R = TypeVar("R") +P = ParamSpec("P") +T = TypeVar("T") + +class CM(Generic[R]): ... +def cm(fn: Callable[Concatenate[T, P], R]) -> Callable[Concatenate[List[T], P], CM[R]]: ... + +Ts = TypeVarTuple("Ts") +@cm +def test(x: T, *args: Unpack[Ts]) -> Tuple[T, Unpack[Ts]]: ... + +reveal_type(test) # N: Revealed type is "def [T, Ts] (builtins.list[T`2], *args: Unpack[Ts`-2]) -> __main__.CM[Tuple[T`2, Unpack[Ts`-2]]]" +[builtins fixtures/tuple.pyi] + +[case testMixingTypeVarTupleAndParamSpec] +from typing import Generic, ParamSpec, TypeVarTuple, Unpack, Callable, TypeVar + +P = ParamSpec("P") +Ts = TypeVarTuple("Ts") + +class A(Generic[P, Unpack[Ts]]): ... +class B(Generic[Unpack[Ts], P]): ... + +a: A[[int, str], int, str] +reveal_type(a) # N: Revealed type is "__main__.A[[builtins.int, builtins.str], builtins.int, builtins.str]" +b: B[int, str, [int, str]] +reveal_type(b) # N: Revealed type is "__main__.B[builtins.int, builtins.str, [builtins.int, builtins.str]]" + +x: A[int, str, [int, str]] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" +reveal_type(x) # N: Revealed type is "__main__.A[Any, Unpack[builtins.tuple[Any, ...]]]" +y: B[[int, str], int, str] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str" +reveal_type(y) # N: Revealed type is "__main__.B[Unpack[builtins.tuple[Any, ...]], Any]" + +R = TypeVar("R") +class C(Generic[P, R]): + fn: Callable[P, None] + +c: C[int, str] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" +reveal_type(c.fn) # N: Revealed type is "def (*Any, **Any)" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-unbound.test b/test-data/unit/check-typevar-unbound.test new file mode 100644 index 000000000000..ed6beaa100db --- /dev/null +++ b/test-data/unit/check-typevar-unbound.test @@ -0,0 +1,71 @@ +[case testUnboundTypeVar] +from typing import TypeVar + +T = TypeVar('T') + +def f() -> T: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + ... +f() + +U = TypeVar('U', bound=int) + +def g() -> U: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar \ + # N: Consider using the upper bound "int" instead + ... + +V = TypeVar('V', int, str) + +def h() -> V: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + ... + +[case testInnerFunctionTypeVar] + +from typing import TypeVar + +T = TypeVar('T') + +def g(a: T) -> T: + def f() -> T: + ... + return f() + +[case testUnboundIterableOfTypeVars] +from typing import Iterable, TypeVar + +T = TypeVar('T') + +def f() -> Iterable[T]: + ... +f() + +[case testBoundTypeVar] +from typing import TypeVar + +T = TypeVar('T') + +def f(a: T, b: T, c: int) -> T: + ... + +[case testNestedBoundTypeVar] +from typing import Callable, List, Union, Tuple, TypeVar + +T = TypeVar('T') + +def f(a: Union[int, T], b: str) -> T: + ... + +def g(a: Callable[..., T], b: str) -> T: + ... + +def h(a: List[Union[Callable[..., T]]]) -> T: + ... + +def j(a: List[Union[Callable[..., Tuple[T, T]], int]]) -> T: + ... +[builtins fixtures/tuple.pyi] + +[case testUnboundedTypevarUnpacking] +from typing import TypeVar +T = TypeVar("T") +def f(t: T) -> None: + a, *b = t # E: "object" object is not iterable diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test new file mode 100644 index 000000000000..8b961d88d23d --- /dev/null +++ b/test-data/unit/check-typevar-values.test @@ -0,0 +1,734 @@ +-- Test cases for type variables with values restriction. + + +[case testCallGenericFunctionWithTypeVarValueRestriction] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> None: pass +f(1) +f('x') +f(object()) # E: Value of type variable "T" of "f" cannot be "object" + +[case testCallGenericFunctionWithTypeVarValueRestrictionUsingContext] +from typing import TypeVar, List +T = TypeVar('T', int, str) +def f(x: T) -> List[T]: pass +i = [1] +s = ['x'] +o = [object()] +if int(): + i = f(1) + s = f('') + o = f(1) \ + # E: Incompatible types in assignment (expression has type "List[int]", variable has type "List[object]") \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant +[builtins fixtures/list.pyi] + +[case testCallGenericFunctionWithTypeVarValueRestrictionAndAnyArgs] +from typing import TypeVar, Any, cast +T = TypeVar('T', int, str) +def f(x: T) -> None: pass +f(cast(Any, object())) +[out] + +[case testCallGenericFunctionWithTypeVarValueRestrictionInDynamicFunc] +from typing import TypeVar, Any +T = TypeVar('T', int, str) +def f(x: T) -> None: pass +def g(): + f(object()) +[out] + +[case testCallGenericFunctionWithTypeVarValueRestrictionUsingSubtype] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> None: pass +class S(str): pass +f(S()) +[out] + +[case testCheckGenericFunctionBodyWithTypeVarValues] +from typing import TypeVar +class A: + def f(self, x: int) -> A: return self +class B: + def f(self, x: int) -> B: return self +AB = TypeVar('AB', A, B) +def f(x: AB) -> AB: + x = x.f(1) + return x.f(1) + +[case testCheckGenericFunctionBodyWithTypeVarValues2] +from typing import TypeVar +class A: + def f(self) -> A: return A() + def g(self) -> B: return B() +class B: + def f(self) -> A: return A() + def g(self) -> B: return B() +AB = TypeVar('AB', A, B) +def f(x: AB) -> AB: + return x.f() # Error +def g(x: AB) -> AB: + return x.g() # Error +[out] +main:10: error: Incompatible return value type (got "A", expected "B") +main:12: error: Incompatible return value type (got "B", expected "A") + +[case testTypeInferenceAndTypeVarValues] +from typing import TypeVar +class A: + def f(self) -> A: return self + def g(self) -> B: return B() +class B: + def f(self) -> B: return self + def g(self) -> B: return B() +AB = TypeVar('AB', A, B) +def f(x: AB) -> AB: + y = x + if y: + return y.f() + else: + return y.g() # E: Incompatible return value type (got "B", expected "A") +[out] + +[case testTypeDeclaredBasedOnTypeVarWithValues] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: + a: T + b: T + if 1: + a = x + b = x + a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + return x +[out] + +[case testIsinstanceAndTypeVarValues] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: + if isinstance(x, int): + return 2 + return x +def g(x: T) -> T: + if isinstance(x, str): + return '' + return x +def h(x: T) -> T: + if isinstance(x, int): + return '' # E: Incompatible return value type (got "str", expected "int") + return x +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceAndTypeVarValues2] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: + if isinstance(x, int): + return 2 + else: + return '' +def g(x: T) -> T: + if isinstance(x, int): + return '' # E: Incompatible return value type (got "str", expected "int") + else: + return 2 # E: Incompatible return value type (got "int", expected "str") + return x +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceAndTypeVarValues3] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: + if isinstance(x, int): + y = 1 + else: + y = '' + return y +[builtins fixtures/isinstance.pyi] + +[case testIsinstanceAndTypeVarValues4] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: + if isinstance(x, int): + y = 1 + else: + y = object() + return y # E: Incompatible return value type (got "object", expected "str") +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceAndTypeVarValues5] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: + if isinstance(x, int): + y = object() + else: + y = '' + return y # E: Incompatible return value type (got "object", expected "int") +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceWithUserDefinedTypeAndTypeVarValues] +# flags: --warn-unreachable +from typing import TypeVar +class A: pass +class B: pass +T1 = TypeVar('T1', A, B) +def f1(x: T1) -> None: + y = x + if isinstance(x, A): + x = y + x = A() # E: Incompatible types in assignment (expression has type "A", variable has type "B") + else: + x = B() + x = y + x.foo() # E: "B" has no attribute "foo" + +class C: + field: int +class D: + field: str +T2 = TypeVar('T2', C, D) +def f2(x: T2) -> None: + y = x + if isinstance(x, C): + # C and D are non-overlapping, so this branch is never checked + x = y + x = C() + else: + x = D() + x = y + x.foo() # E: "D" has no attribute "foo" + +S = TypeVar('S', int, str) +def g(x: S) -> None: + y = x + if isinstance(x, int): + x = y +[builtins fixtures/isinstance.pyi] +[out] + +[case testIsinstanceWithUserDefinedTypeAndTypeVarValues2] +from typing import TypeVar +class S(str): pass +T = TypeVar('T', S, int) +def f(x: T) -> None: + y = x + if isinstance(x, S): + # This is checked only when type of x is str. + x = y + x = S() + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "S") + else: + x = y + x = 1 + x = S() # E: Incompatible types in assignment (expression has type "S", variable has type "int") +[builtins fixtures/isinstance.pyi] +[out] + +[case testTypeVarValuesAndNestedCalls] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(m: T) -> int: pass +def h(x: int) -> int: pass +def g(a: T) -> None: + h(f(a)) +[out] + +[case testGenericTypeWithTypevarValues] +from typing import TypeVar, Generic, Any +X = TypeVar('X', int, str) +class A(Generic[X]): pass +a: A[int] +b: A[str] +d: A[object] # E: Value of type variable "X" of "A" cannot be "object" +c: A[Any] + +[case testConstructGenericTypeWithTypevarValuesAndTypeInference] +from typing import TypeVar, Generic, Any, cast +X = TypeVar('X', int, str) +class A(Generic[X]): + def __init__(self, x: X) -> None: pass +A(1) +A('x') +A(cast(Any, object())) +A(object()) # E: Value of type variable "X" of "A" cannot be "object" + +[case testGenericTypeWithTypevarValuesAndTypevarArgument] +from typing import TypeVar, Generic +class C: pass +X = TypeVar('X', int, str) +Y = TypeVar('Y', int, C) +Z = TypeVar('Z') +class D(Generic[X]): + def __init__(self, x: X) -> None: pass +def f(x: X) -> None: + a: D[X] +def g(x: Y) -> None: + a: D[Y] +def h(x: Z) -> None: + a: D[Z] +[out] +main:11: error: Invalid type argument value for "D" +main:13: error: Type variable "Z" not valid as type argument value for "D" + +[case testGenericTypeWithTypevarValuesAndSubtypePromotion] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class S(str): pass +class C(Generic[X]): + def __init__(self, x: X) -> None: pass +x: C[str] +y = C(S()) +if int(): + x = y + y = x +c_int = C(1) # type: C[int] +if int(): + y = c_int # E: Incompatible types in assignment (expression has type "C[int]", variable has type "C[str]") + +[case testGenericTypeBodyWithTypevarValues] +from typing import TypeVar, Generic +class A: + def f(self, x: int) -> None: pass + def g(self, x: int) -> None: pass + def h(self, x: str) -> None: pass +class B: + def f(self, x: int) -> None: pass + def g(self, x: str) -> None: pass + def h(self, x: int) -> None: pass +X = TypeVar('X', A, B) +class C(Generic[X]): + def f(self, x: X) -> None: + x.f(1) + x.g(1) # E: Argument 1 to "g" of "B" has incompatible type "int"; expected "str" + x.h(1) # E: Argument 1 to "h" of "A" has incompatible type "int"; expected "str" +[out] + +[case testAttributeInGenericTypeWithTypevarValues1] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class C(Generic[X]): + x = None # type: X + def f(self, x: X) -> None: + self.x = x + self.x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[out] + +[case testAttributeInGenericTypeWithTypevarValues2] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class C(Generic[X]): + x = None # type: X +cn = C() # type: C[int] +cn.x = 1 +cn.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +cs = C() # type: C[str] +cs.x = '' +cs.x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAttributeInGenericTypeWithTypevarValues3] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class C(Generic[X]): + def f(self, x: X) -> None: + self.x = x # type: X +ci: C[int] +cs: C[str] +reveal_type(ci.x) # N: Revealed type is "builtins.int" +reveal_type(cs.x) # N: Revealed type is "builtins.str" + +[case testAttributeInGenericTypeWithTypevarValuesUsingInference1] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class C(Generic[X]): + def f(self, x: X) -> None: + self.x = x # E: Need type annotation for "x" +ci: C[int] +cs: C[str] +reveal_type(ci.x) # N: Revealed type is "Any" +reveal_type(cs.x) # N: Revealed type is "Any" + +[case testAttributeInGenericTypeWithTypevarValuesUsingInference2] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class C(Generic[X]): + def f(self, x: X) -> None: + self.x = 1 + reveal_type(self.x) # N: Revealed type is "builtins.int" +ci: C[int] +cs: C[str] +reveal_type(ci.x) # N: Revealed type is "builtins.int" +reveal_type(cs.x) # N: Revealed type is "builtins.int" + +[case testAttributeInGenericTypeWithTypevarValuesUsingInference3] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class C(Generic[X]): + x: X + def f(self) -> None: + self.y = self.x # E: Need type annotation for "y" +ci: C[int] +cs: C[str] +reveal_type(ci.y) # N: Revealed type is "Any" +reveal_type(cs.y) # N: Revealed type is "Any" + +[case testInferredAttributeInGenericClassBodyWithTypevarValues] +from typing import TypeVar, Generic +X = TypeVar('X', int, str) +class C(Generic[X]): + x = 1 +C.x = 1 +C.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testMultipleClassTypevarsWithValues1] +from typing import TypeVar, Generic +class A: + def f(self, x: int) -> None: pass +class B: + def f(self, x: str) -> None: pass +X = TypeVar('X', A, B) +Y = TypeVar('Y', int, str) +class C(Generic[X, Y]): + def f(self, x: X, y: Y) -> None: + x.f(y) +[out] +main:10: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" +main:10: error: Argument 1 to "f" of "B" has incompatible type "int"; expected "str" + +[case testMultipleClassTypevarsWithValues2] +from typing import TypeVar, Generic +class A: pass +class B: pass +X = TypeVar('X', A, B) +Y = TypeVar('Y', int, str) +class C(Generic[X, Y]): pass +a: C[A, int] +b: C[B, str] +c: C[int, int] # E: Value of type variable "X" of "C" cannot be "int" +d: C[A, A] # E: Value of type variable "Y" of "C" cannot be "A" + +[case testCallGenericFunctionUsingMultipleTypevarsWithValues] +from typing import TypeVar +class A: pass +class B: pass +X = TypeVar('X', A, B) +Y = TypeVar('Y', int, str) +def f(x: X, y: Y) -> None: pass +f(A(), '') +f(B(), 1) +f(A(), A()) # E: Value of type variable "Y" of "f" cannot be "A" +f(1, 1) # E: Value of type variable "X" of "f" cannot be "int" + +[case testGenericFunctionWithNormalAndRestrictedTypevar] +from typing import TypeVar, Generic +X = TypeVar('X') +Y = TypeVar('Y', int, str) +class C(Generic[Y]): + def __init__(self, y: Y) -> None: pass +def f(x: X, y: Y, z: int) -> None: + C(y) + C(x) # Error + if int(): + z = x # Error + z = y # Error + y.foo # Error +[out] +main:8: error: Value of type variable "Y" of "C" cannot be "X" +main:10: error: Incompatible types in assignment (expression has type "X", variable has type "int") +main:11: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:12: error: "int" has no attribute "foo" +main:12: error: "str" has no attribute "foo" + +[case testTypeVarWithValueInferredFromObjectReturnTypeContext] +from typing import TypeVar +T = TypeVar('T', int, str) +def c1(x: object) -> None: pass +def c2(x: int) -> None: pass +def c3(x: str) -> None: pass +def g(x: T) -> T: pass +c1(g('')) +c2(g(1)) +c3(g('')) +c2(g('')) # E: Argument 1 to "c2" has incompatible type "str"; expected "int" +c3(g(1)) # E: Argument 1 to "c3" has incompatible type "int"; expected "str" + +[case testTypeVarWithValueInferredFromObjectReturnTypeContext2] +from typing import TypeVar +T = TypeVar('T', int, str) +class ss(str): pass +def c(x: ss) -> None: pass +def g(x: T) -> T: pass +c(g('')) +c(g(1)) +[out] +main:6: error: Argument 1 to "c" has incompatible type "str"; expected "ss" +main:7: error: Argument 1 to "c" has incompatible type "int"; expected "ss" + +[case testDefineAttributeInGenericMethodUsingTypeVarWithValues] +from typing import TypeVar +T = TypeVar('T', int, str) +class A: + def f(self, x: T) -> None: + self.x = x # E: Need type annotation for "x" + self.y = [x] # E: Need type annotation for "y" + self.z = 1 +reveal_type(A().x) # N: Revealed type is "Any" +reveal_type(A().y) # N: Revealed type is "Any" +reveal_type(A().z) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + + +-- Special cases +-- ------------- + + +[case testTypevarValuesSpecialCase1] +from typing import TypeVar, Generic +from abc import abstractmethod +T = TypeVar('T', int, str) +class A(Generic[T]): + @abstractmethod + def f(self) -> 'A[T]': pass +class B(A[str]): + @abstractmethod + def f(self) -> 'B': pass +class C(A[str]): + @abstractmethod + def f(self) -> int: # E: Return type "int" of "f" incompatible with return type "A[str]" in supertype "A" + pass +[out] + +[case testDefaultArgumentValueInGenericClassWithTypevarValues] +from typing import TypeVar, Generic +T = TypeVar('T', int, str) +class C(Generic[T]): + def f(self, x: int = 2) -> None: pass + +[case testTypevarValuesWithOverloadedFunctionSpecialCase] +from foo import * +[file foo.pyi] +from typing import TypeVar, overload, Callable + +T = TypeVar('T', int, str) +def f(x: T) -> None: + y = m(g, x) + if int(): + x = y + y = object() # Error + +A = TypeVar('A') +R = TypeVar('R') +def m(f: Callable[[A], R], it: A) -> A: pass + +@overload +def g(x: int) -> int: return x +@overload +def g(x: str) -> str: return x +[out] +tmp/foo.pyi:8: error: Incompatible types in assignment (expression has type "object", variable has type "int") +tmp/foo.pyi:8: error: Incompatible types in assignment (expression has type "object", variable has type "str") + +[case testGenericFunctionSubtypingWithTypevarValues] +from typing import TypeVar +class A: pass +T = TypeVar('T', int, str) +U = TypeVar('U', str, A, int) +def f(x: T) -> T: pass +def g(x: U) -> U: pass +a = f +if int(): + a = f +if int(): + a = g +b = g +if int(): + b = g +if int(): + b = f # E: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[U], U]") + +[case testInnerFunctionWithTypevarValues] +from typing import TypeVar +T = TypeVar('T', int, str) +U = TypeVar('U', int, str) +def outer(x: T) -> T: + def inner(y: T) -> T: + return x + def inner2(y: U) -> U: + return y + inner(x) + inner(3) # E: Argument 1 to "inner" has incompatible type "int"; expected "str" + inner2(x) + inner2(3) + outer(3) + return x +[out] + +[case testInnerFunctionMutualRecursionWithTypevarValues] +from typing import TypeVar +T = TypeVar('T', int, str) +def outer(x: T) -> T: + def inner1(y: T) -> T: + return inner2(y) + def inner2(y: T) -> T: + return inner1('a') # E: Argument 1 to "inner1" has incompatible type "str"; expected "int" + return inner1(x) +[out] + +[case testClassMemberTypeVarInFunctionBody] +from typing import TypeVar, List +S = TypeVar('S') +class C: + T = TypeVar('T', bound=int) + def f(self, x: T) -> T: + L = List[S] + y: L[C.T] = [x] + C.T # E: Type variable "C.T" cannot be used as an expression + A = C.T # E: Type variable "C.T" cannot be used as an expression + return y[0] + +[builtins fixtures/list.pyi] + +[case testTypeVarWithAnyTypeBound] +# flags: --follow-imports=skip +from typing import Type, TypeVar +from a import A +T = TypeVar('T', bound=A) +def method(t: Type[T]) -> None: + t.a +[file a.py] +class A: + a: int = 7 +[out] + +[case testParameterLessGenericAsRestriction] +from typing import Sequence, Iterable, TypeVar +S = TypeVar('S', Sequence, Iterable) +def my_len(s: S) -> None: pass +def crash() -> None: my_len((0,)) +[builtins fixtures/tuple.pyi] + +[case testReferenceToDecoratedFunctionAndTypeVarValues] +from typing import TypeVar, Callable + +T = TypeVar('T') +S = TypeVar('S', int, str) + +def dec(f: Callable[..., T]) -> Callable[..., T]: ... + +@dec +def g(s: S) -> Callable[[S], None]: ... + +def f(x: S) -> None: + h = g(x) + h(x) + +[case testTypeVarWithTypedDictBoundInIndexExpression] +from typing import TypeVar +from typing_extensions import TypedDict + +class Data(TypedDict): + x: int + + +T = TypeVar("T", bound=Data) + + +def f(data: T) -> None: + reveal_type(data["x"]) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeVarWithUnionTypedDictBoundInIndexExpression] +from typing import TypeVar, Union, Dict +from typing_extensions import TypedDict + +class Data(TypedDict): + x: int + + +T = TypeVar("T", bound=Union[Data, Dict[str, str]]) + + +def f(data: T) -> None: + reveal_type(data["x"]) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/dict.pyi] + +[case testTypeVarWithTypedDictValueInIndexExpression] +from typing import TypeVar, Union, Dict +from typing_extensions import TypedDict + +class Data(TypedDict): + x: int + + +T = TypeVar("T", Data, Dict[str, str]) + + +def f(data: T) -> None: + _: Union[str, int] = data["x"] +[builtins fixtures/dict.pyi] + +[case testSelfTypeVarIndexExpr] +from typing import TypeVar, Union, Type +from typing_extensions import TypedDict + +T = TypeVar("T", bound="Indexable") + +class Indexable: + def __init__(self, index: str) -> None: + self.index = index + + def __getitem__(self: T, index: str) -> T: + return self._new_instance(index) + + @classmethod + def _new_instance(cls: Type[T], index: str) -> T: + return cls("foo") + + def m(self: T) -> T: + return self["foo"] +[builtins fixtures/classmethod.pyi] + +[case testTypeVarWithValueDeferral] +from typing import TypeVar, Callable + +T = TypeVar("T", "A", "B") +Func = Callable[[], T] + +class A: ... +class B: ... + +[case testTypeCommentInGenericTypeWithConstrainedTypeVar] +from typing import Generic, TypeVar + +NT = TypeVar("NT", int, float) + +class Foo1(Generic[NT]): + p = 1 # type: int + +class Foo2(Generic[NT]): + p, q = 1, 2.0 # type: (int, float) + +class Foo3(Generic[NT]): + def bar(self) -> None: + p = 1 # type: int + +class Foo4(Generic[NT]): + def bar(self) -> None: + p, q = 1, 2.0 # type: (int, float) + +def foo3(x: NT) -> None: + p = 1 # type: int + +def foo4(x: NT) -> None: + p, q = 1, 2.0 # type: (int, float) +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-underscores.test b/test-data/unit/check-underscores.test new file mode 100644 index 000000000000..2a789b3314f3 --- /dev/null +++ b/test-data/unit/check-underscores.test @@ -0,0 +1,5 @@ +[case testUnderscoresBasics] +x: int +x = 1000_000 +x = 0x_FF_FF_FF_FF +y: str = 1000_000.000_001 # E: Incompatible types in assignment (expression has type "float", variable has type "str") diff --git a/test-data/unit/check-union-error-syntax.test b/test-data/unit/check-union-error-syntax.test new file mode 100644 index 000000000000..2928cc312709 --- /dev/null +++ b/test-data/unit/check-union-error-syntax.test @@ -0,0 +1,61 @@ +[case testUnionErrorSyntax] +# flags: --python-version 3.10 --no-force-union-syntax +from typing import Union +x : Union[bool, str] +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "bool | str") + +[case testOrErrorSyntax] +# flags: --python-version 3.10 --force-union-syntax +from typing import Union +x : Union[bool, str] +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "Union[bool, str]") + +[case testOrNoneErrorSyntax] +# flags: --python-version 3.10 --no-force-union-syntax +from typing import Union +x : Union[bool, None] +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "bool | None") + +[case testOptionalErrorSyntax] +# flags: --python-version 3.10 --force-union-syntax +from typing import Union +x : Union[bool, None] +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[bool]") + +[case testNoneAsFinalItem] +# flags: --python-version 3.10 --no-force-union-syntax +from typing import Union +x : Union[bool, None, str] +x = 3 # E: Incompatible types in assignment (expression has type "int", variable has type "bool | str | None") + +[case testLiteralOrErrorSyntax] +# flags: --python-version 3.10 --no-force-union-syntax +from typing import Union +from typing_extensions import Literal +x : Union[Literal[1], Literal[2], str] +x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", variable has type "Literal[1, 2] | str") +[builtins fixtures/tuple.pyi] + +[case testLiteralUnionErrorSyntax] +# flags: --python-version 3.10 --force-union-syntax +from typing import Union +from typing_extensions import Literal +x : Union[Literal[1], Literal[2], str] +x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", variable has type "Union[str, Literal[1, 2]]") +[builtins fixtures/tuple.pyi] + +[case testLiteralOrNoneErrorSyntax] +# flags: --python-version 3.10 --no-force-union-syntax +from typing import Union +from typing_extensions import Literal +x : Union[Literal[1], None] +x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", variable has type "Literal[1] | None") +[builtins fixtures/tuple.pyi] + +[case testLiteralOptionalErrorSyntax] +# flags: --python-version 3.10 --force-union-syntax +from typing import Union +from typing_extensions import Literal +x : Union[Literal[1], None] +x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", variable has type "Optional[Literal[1]]") +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-union-or-syntax.test b/test-data/unit/check-union-or-syntax.test new file mode 100644 index 000000000000..fcf679fff401 --- /dev/null +++ b/test-data/unit/check-union-or-syntax.test @@ -0,0 +1,253 @@ +-- Type checking of union types with '|' syntax + +[case testUnionOrSyntaxWithTwoBuiltinsTypes] +# flags: --python-version 3.10 +from __future__ import annotations +def f(x: int | str) -> int | str: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + z: int | str = 0 + reveal_type(z) # N: Revealed type is "Union[builtins.int, builtins.str]" + return x +reveal_type(f) # N: Revealed type is "def (x: Union[builtins.int, builtins.str]) -> Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testUnionOrSyntaxWithThreeBuiltinsTypes] +# flags: --python-version 3.10 +def f(x: int | str | float) -> int | str | float: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.float]" + z: int | str | float = 0 + reveal_type(z) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.float]" + return x +reveal_type(f) # N: Revealed type is "def (x: Union[builtins.int, builtins.str, builtins.float]) -> Union[builtins.int, builtins.str, builtins.float]" + +[case testUnionOrSyntaxWithTwoTypes] +# flags: --python-version 3.10 +class A: pass +class B: pass +def f(x: A | B) -> A | B: + reveal_type(x) # N: Revealed type is "Union[__main__.A, __main__.B]" + z: A | B = A() + reveal_type(z) # N: Revealed type is "Union[__main__.A, __main__.B]" + return x +reveal_type(f) # N: Revealed type is "def (x: Union[__main__.A, __main__.B]) -> Union[__main__.A, __main__.B]" + +[case testUnionOrSyntaxWithThreeTypes] +# flags: --python-version 3.10 +class A: pass +class B: pass +class C: pass +def f(x: A | B | C) -> A | B | C: + reveal_type(x) # N: Revealed type is "Union[__main__.A, __main__.B, __main__.C]" + z: A | B | C = A() + reveal_type(z) # N: Revealed type is "Union[__main__.A, __main__.B, __main__.C]" + return x +reveal_type(f) # N: Revealed type is "def (x: Union[__main__.A, __main__.B, __main__.C]) -> Union[__main__.A, __main__.B, __main__.C]" + +[case testUnionOrSyntaxWithLiteral] +# flags: --python-version 3.10 +from typing_extensions import Literal +reveal_type(Literal[4] | str) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testUnionOrSyntaxWithBadOperator] +# flags: --python-version 3.10 +x: 1 + 2 # E: Invalid type comment or annotation + +[case testUnionOrSyntaxWithBadOperands] +# flags: --python-version 3.10 +x: int | 42 # E: Invalid type: try using Literal[42] instead? +y: 42 | int # E: Invalid type: try using Literal[42] instead? +z: str | 42 | int # E: Invalid type: try using Literal[42] instead? + +[case testUnionOrSyntaxWithGenerics] +# flags: --python-version 3.10 +from typing import List +x: List[int | str] +reveal_type(x) # N: Revealed type is "builtins.list[Union[builtins.int, builtins.str]]" +[builtins fixtures/list.pyi] + +[case testUnionOrSyntaxWithQuotedFunctionTypesPre310] +# flags: --python-version 3.9 +from typing import Union +def f(x: 'Union[int, str, None]') -> 'Union[int, None]': + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + return 42 +reveal_type(f) # N: Revealed type is "def (x: Union[builtins.int, builtins.str, None]) -> Union[builtins.int, None]" + +def g(x: "int | str | None") -> "int | None": + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + return 42 +reveal_type(g) # N: Revealed type is "def (x: Union[builtins.int, builtins.str, None]) -> Union[builtins.int, None]" + +[case testUnionOrSyntaxWithQuotedVariableTypesPre310] +# flags: --python-version 3.9 +y: "int | str" = 42 +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testUnionOrSyntaxWithTypeAliasWorking] +# flags: --python-version 3.10 +T = int | str +x: T +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +S = list[int] | str | None +y: S +reveal_type(y) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.str, None]" +U = str | None +z: U +reveal_type(z) # N: Revealed type is "Union[builtins.str, None]" + +def f(): pass + +X = int | str | f() +b: X # E: Variable "__main__.X" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/type.pyi] + +[case testUnionOrSyntaxWithinRuntimeContextNotAllowed] +# flags: --python-version 3.9 +from __future__ import annotations +from typing import List +T = int | str # E: Invalid type alias: expression is not a valid type \ + # E: Unsupported left operand type for | ("Type[int]") +class C(List[int | str]): # E: Type expected within [...] \ + # E: Invalid base class "List" + pass +C() +[builtins fixtures/tuple.pyi] + +[case testUnionOrSyntaxWithinRuntimeContextNotAllowed2] +# flags: --python-version 3.9 +from __future__ import annotations +from typing import cast +cast(str | int, 'x') # E: Cast target is not a type +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testUnionOrSyntaxInComment] +x = 1 # type: int | str + +[case testUnionOrSyntaxFutureImport] +from __future__ import annotations +x: int | None +[builtins fixtures/tuple.pyi] + +[case testUnionOrSyntaxMissingFutureImport] +# flags: --python-version 3.9 +x: int | None # E: X | Y syntax for unions requires Python 3.10 + +[case testUnionOrSyntaxInStubFile] +# flags: --python-version 3.9 +from lib import x +[file lib.pyi] +x: int | None + +[case testUnionOrSyntaxInMiscRuntimeContexts] +# flags: --python-version 3.10 +from typing import cast + +class C(list[int | None]): + pass + +def f() -> object: pass + +reveal_type(cast(str | None, f())) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(list[str | None]()) # N: Revealed type is "builtins.list[Union[builtins.str, None]]" +[builtins fixtures/type.pyi] + +[case testUnionOrSyntaxRuntimeContextInStubFile] +import lib +reveal_type(lib.x) # N: Revealed type is "Union[builtins.int, builtins.list[builtins.str], None]" +reveal_type(lib.y) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" + +[file lib.pyi] +A = int | list[str] | None +x: A +B = list[int | None] +y: B +class C(list[int | None]): + pass +[builtins fixtures/list.pyi] + +[case testUnionOrSyntaxInIsinstance] +# flags: --python-version 3.10 +class C: pass + +def f(x: int | str | C) -> None: + if isinstance(x, int | str): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "__main__.C" + +def g(x: int | str | tuple[int, str] | C) -> None: + if isinstance(x, int | str | tuple): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, Tuple[builtins.int, builtins.str]]" + else: + reveal_type(x) # N: Revealed type is "__main__.C" +[builtins fixtures/isinstance_python3_10.pyi] + +[case testUnionOrSyntaxInIsinstanceNotSupported] +# flags: --python-version 3.9 +from typing import Union +def f(x: Union[int, str, None]) -> None: + if isinstance(x, int | str): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "None" +[builtins fixtures/isinstance.pyi] + +[case testImplicit604TypeAliasWithCyclicImportInStub] +# flags: --python-version 3.10 +from was_builtins import foo +reveal_type(foo) # N: Revealed type is "Union[builtins.str, was_mmap.mmap]" +[file was_builtins.pyi] +import was_mmap +WriteableBuffer = was_mmap.mmap +ReadableBuffer = str | WriteableBuffer +foo: ReadableBuffer +[file was_mmap.pyi] +from was_builtins import * +class mmap: ... +[builtins fixtures/type.pyi] + +[case testTypeAliasWithNewUnionIsInstance] +# flags: --python-version 3.10 +SimpleAlias = int | str + +def foo(x: int | str | tuple): + if isinstance(x, SimpleAlias): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "builtins.tuple[Any, ...]" + +ParameterizedAlias = str | list[str] + +# these are false negatives: +isinstance(5, str | list[str]) +isinstance(5, ParameterizedAlias) +[builtins fixtures/type.pyi] + +[case testIsInstanceUnionNone] +# flags: --python-version 3.10 +def foo(value: str | bool | None): + assert not isinstance(value, str | None) + reveal_type(value) # N: Revealed type is "builtins.bool" + +def bar(value: object): + assert isinstance(value, str | None) + reveal_type(value) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/type.pyi] + + +# TODO: Get this test to pass +[case testImplicit604TypeAliasWithCyclicImportNotInStub-xfail] +# flags: --python-version 3.10 +from was_builtins import foo +reveal_type(foo) # N: Revealed type is "Union[builtins.str, was_mmap.mmap]" +[file was_builtins.py] +import was_mmap +WriteableBuffer = was_mmap.mmap +ReadableBuffer = str | WriteableBuffer +foo: ReadableBuffer +[file was_mmap.py] +from was_builtins import * +class mmap: ... diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test new file mode 100644 index 000000000000..329896f7a1a7 --- /dev/null +++ b/test-data/unit/check-unions.test @@ -0,0 +1,1348 @@ +-- Type checking of union types + +[case testUnion1] +from typing import Union +def f(x: Union[int, str]) -> None: + if isinstance(x, int): + y = 1 + y = x + elif isinstance(x, str): + z = 'a' + z = x +[builtins fixtures/isinstance.pyi] + +[case testUnion2] +from typing import Union +def f(x: Union[int, str]) -> None: + if isinstance(x, int): + y = 1 + y = x + else: + z = 'a' + z = x +[builtins fixtures/isinstance.pyi] + +[case testUnion3] +from typing import Union +def f(x: Union[int, str]) -> None: + if isinstance(x, int): + y = 1 + if int(): + y = x + else: + z = 2 + if int(): + z = x # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/isinstance.pyi] +[out] + +[case testUnionAnyIsInstance] +from typing import Any, Union + +def func(v: Union[int, Any]) -> None: + if isinstance(v, int): + reveal_type(v) # N: Revealed type is "builtins.int" + else: + reveal_type(v) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] +[out] + +[case testUnionAttributeAccess] +from typing import Union + +class A: y = 1 +class B: y = 2 +class C: pass +class D: pass + +u: Union[A, C, D] +v: Union[C, D] +w: Union[A, B] +x: Union[A, C] +y: int +z: str + +if int(): + y = w.y +v.y # E: Item "C" of "Union[C, D]" has no attribute "y" \ + # E: Item "D" of "Union[C, D]" has no attribute "y" +u.y # E: Item "C" of "Union[A, C, D]" has no attribute "y" \ + # E: Item "D" of "Union[A, C, D]" has no attribute "y" +if int(): + z = w.y # E: Incompatible types in assignment (expression has type "int", variable has type "str") +w.y = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +if int(): + y = x.y # E: Item "C" of "Union[A, C]" has no attribute "y" +zz = x.y # E: Item "C" of "Union[A, C]" has no attribute "y" +if int(): + z = zz # E: Incompatible types in assignment (expression has type "Union[int, Any]", variable has type "str") + +[builtins fixtures/isinstance.pyi] + +[case testUnionMethodCalls] +from typing import Union + +class A: + def foo(self) -> int: pass +class B: + def foo(self) -> int: pass +class C: + def foo(self) -> str: pass + +x: Union[A, B] +y: Union[A, C] +i: int + +x.foo() +y.foo() +i = x.foo() +if int(): + i = y.foo() # E: Incompatible types in assignment (expression has type "Union[int, str]", variable has type "int") + +[builtins fixtures/isinstance.pyi] + +[case testUnionIndexing] +from typing import Union, List +x: Union[List[int], str] +x[2] +x[2] + 1 # E: Unsupported operand types for + ("str" and "int") \ + # N: Left operand is of type "Union[int, str]" +[builtins fixtures/isinstancelist.pyi] + +[case testUnionAsOverloadArg] +from foo import * +x = 0 +if int(): + x = f(1) +if int(): + x = f('') +s = '' +if int(): + s = f(int) +if int(): + s = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +if int(): + x = f(int) # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[file foo.pyi] +from typing import Union, overload +@overload +def f(x: Union[int, str]) -> int: pass +@overload +def f(x: type) -> str: pass + +[case testUnionWithNoneItem] +# flags: --no-strict-optional +from typing import Union +def f() -> Union[int, None]: pass +x = 1 +x = f() + +[case testUnionWithEllipsis] +from typing import Union +def f(x: Union[int, EllipsisType]) -> int: + if x is Ellipsis: + reveal_type(x) # N: Revealed type is "builtins.ellipsis" + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + return x +[builtins fixtures/isinstancelist.pyi] + +[case testOptional] +from typing import Optional +def f(x: Optional[int]) -> None: pass +f(1) +f(None) +f('') # E: Argument 1 to "f" has incompatible type "str"; expected "Optional[int]" + +[case testUnionWithNoReturn] +from typing import Union, NoReturn +def f() -> Union[int, NoReturn]: ... +reveal_type(f()) # N: Revealed type is "builtins.int" + +[case testUnionSimplificationGenericFunction] +from typing import TypeVar, Union, List +T = TypeVar('T') +def f(x: List[T]) -> Union[T, int]: pass +def g(y: str) -> None: pass +a = f([1]) +g(a) # E: Argument 1 to "g" has incompatible type "int"; expected "str" +[builtins fixtures/list.pyi] + +[case testUnionSimplificationGenericClass] +from typing import TypeVar, Union, Generic +T = TypeVar('T') +U = TypeVar('U') +class C(Generic[T, U]): + def f(self, x: str) -> Union[T, U]: pass +a = C() # type: C[int, int] +b = a.f('a') +a.f(b) # E: Argument 1 to "f" of "C" has incompatible type "int"; expected "str" + +[case testUnionOrderEquivalence] +from typing import Union + +def foo(): pass + +S = str +T = int + +if foo(): + def f(x: Union[int, str]) -> None: pass +elif foo(): + def f(x: Union[str, int]) -> None: pass +elif foo(): + def f(x: Union[int, str, int, int, str]) -> None: pass +elif foo(): + def f(x: Union[int, str, float]) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: Union[int, str]) -> None \ + # N: Redefinition: \ + # N: def f(x: Union[int, str, float]) -> None +elif foo(): + def f(x: Union[S, T]) -> None: pass +elif foo(): + def f(x: Union[str]) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: Union[int, str]) -> None \ + # N: Redefinition: \ + # N: def f(x: str) -> None +else: + def f(x: Union[Union[int, T], Union[S, T], str]) -> None: pass + +# Checks bidirectionality of testing. The first definition of g is consistent with +# the second, but not vice-versa. +if foo(): + def g(x: Union[int, str, bytes]) -> None: pass +else: + def g(x: Union[int, str]) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: Union[int, str, bytes]) -> None \ + # N: Redefinition: \ + # N: def g(x: Union[int, str]) -> None + +[case testUnionSimplificationSpecialCases] +# flags: --no-strict-optional +from typing import Any, TypeVar, Union + +class C(Any): pass + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +a = None # type: Any + +reveal_type(u(C(), None)) # N: Revealed type is "__main__.C" +reveal_type(u(None, C())) # N: Revealed type is "__main__.C" + +reveal_type(u(C(), a)) # N: Revealed type is "Union[Any, __main__.C]" +reveal_type(u(a, C())) # N: Revealed type is "Union[__main__.C, Any]" + +reveal_type(u(C(), C())) # N: Revealed type is "__main__.C" +reveal_type(u(a, a)) # N: Revealed type is "Any" + +[case testUnionSimplificationSpecialCase2] +from typing import Any, TypeVar, Union + +class C(Any): pass + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +def f(x: T) -> None: + reveal_type(u(C(), x)) # N: Revealed type is "Union[T`-1, __main__.C]" + reveal_type(u(x, C())) # N: Revealed type is "Union[__main__.C, T`-1]" + +[case testUnionSimplificationSpecialCase3] +from typing import Any, TypeVar, Generic, Union + +class C(Any): pass + +V = TypeVar('V') +T = TypeVar('T') + +class M(Generic[V]): + def get(self, default: T) -> Union[V, T]: ... + +def f(x: M[C]) -> None: + y = x.get(None) + reveal_type(y) # N: Revealed type is "Union[__main__.C, None]" + +[case testUnionSimplificationSpecialCases2] +# flags: --no-strict-optional +from typing import Any, TypeVar, Union + +class C(Any): pass + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +a = None # type: Any + +# Base-class-Any and None, simplify +reveal_type(u(C(), None)) # N: Revealed type is "__main__.C" +reveal_type(u(None, C())) # N: Revealed type is "__main__.C" + +# Normal instance type and None, simplify +reveal_type(u(1, None)) # N: Revealed type is "builtins.int" +reveal_type(u(None, 1)) # N: Revealed type is "builtins.int" + +# Normal instance type and base-class-Any, no simplification +reveal_type(u(C(), 1)) # N: Revealed type is "Union[builtins.int, __main__.C]" +reveal_type(u(1, C())) # N: Revealed type is "Union[__main__.C, builtins.int]" + +# Normal instance type and Any, no simplification +reveal_type(u(1, a)) # N: Revealed type is "Union[Any, builtins.int]" +reveal_type(u(a, 1)) # N: Revealed type is "Union[builtins.int, Any]" + +# Any and base-class-Any, no simplification +reveal_type(u(C(), a)) # N: Revealed type is "Union[Any, __main__.C]" +reveal_type(u(a, C())) # N: Revealed type is "Union[__main__.C, Any]" + +# Two normal instance types, simplify +reveal_type(u(1, object())) # N: Revealed type is "builtins.object" +reveal_type(u(object(), 1)) # N: Revealed type is "builtins.object" + +# Two normal instance types, no simplification +reveal_type(u(1, '')) # N: Revealed type is "Union[builtins.str, builtins.int]" +reveal_type(u('', 1)) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testUnionSimplificationWithDuplicateItems] +from typing import Any, TypeVar, Union + +class C(Any): pass + +T = TypeVar('T') +S = TypeVar('S') +R = TypeVar('R') +def u(x: T, y: S, z: R) -> Union[R, S, T]: pass + +a: Any + +reveal_type(u(1, 1, 1)) # N: Revealed type is "builtins.int" +reveal_type(u(C(), C(), None)) # N: Revealed type is "Union[None, __main__.C]" +reveal_type(u(a, a, 1)) # N: Revealed type is "Union[builtins.int, Any]" +reveal_type(u(a, C(), a)) # N: Revealed type is "Union[Any, __main__.C]" +reveal_type(u('', 1, 1)) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testUnionAndBinaryOperation] +from typing import Union +class A: pass +def f(x: Union[int, str, A]): + x + object() # E: Unsupported left operand type for + ("A") \ + # N: Left operand is of type "Union[int, str, A]" \ + # E: Unsupported operand types for + ("int" and "object") \ + # E: Unsupported operand types for + ("str" and "object") +[builtins fixtures/primitives.pyi] + +[case testNarrowingDownNamedTupleUnion] +from typing import NamedTuple, Union + +A = NamedTuple('A', [('y', int)]) +B = NamedTuple('B', [('x', int)]) +C = NamedTuple('C', [('x', int)]) + +def foo(a: Union[A, B, C]): + if isinstance(a, (B, C)): + reveal_type(a) # N: Revealed type is "Union[Tuple[builtins.int, fallback=__main__.B], Tuple[builtins.int, fallback=__main__.C]]" + a.x + a.y # E: Item "B" of "Union[B, C]" has no attribute "y" \ + # E: Item "C" of "Union[B, C]" has no attribute "y" + b = a # type: Union[B, C] +[builtins fixtures/isinstance.pyi] + +[case testSimplifyingUnionAndTypePromotions] +from typing import TypeVar, Union +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[T, S]: pass + +reveal_type(u(1, 2.3)) # N: Revealed type is "Union[builtins.int, builtins.float]" +reveal_type(u(2.3, 1)) # N: Revealed type is "Union[builtins.float, builtins.int]" +reveal_type(u(False, 2.2)) # N: Revealed type is "Union[builtins.bool, builtins.float]" +reveal_type(u(2.2, False)) # N: Revealed type is "Union[builtins.float, builtins.bool]" +[builtins fixtures/primitives.pyi] + +[case testSimplifyingUnionWithTypeTypes1] +from typing import TypeVar, Union, Type, Any + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +t_o: Type[object] +t_s: Type[str] +t_a: Type[Any] + +# Two identical items +reveal_type(u(t_o, t_o)) # N: Revealed type is "Type[builtins.object]" +reveal_type(u(t_s, t_s)) # N: Revealed type is "Type[builtins.str]" +reveal_type(u(t_a, t_a)) # N: Revealed type is "Type[Any]" +reveal_type(u(type, type)) # N: Revealed type is "def (x: builtins.object) -> builtins.type" + +# One type, other non-type +reveal_type(u(t_s, 1)) # N: Revealed type is "Union[builtins.int, Type[builtins.str]]" +reveal_type(u(1, t_s)) # N: Revealed type is "Union[Type[builtins.str], builtins.int]" +reveal_type(u(type, 1)) # N: Revealed type is "Union[builtins.int, def (x: builtins.object) -> builtins.type]" +reveal_type(u(1, type)) # N: Revealed type is "Union[def (x: builtins.object) -> builtins.type, builtins.int]" +reveal_type(u(t_a, 1)) # N: Revealed type is "Union[builtins.int, Type[Any]]" +reveal_type(u(1, t_a)) # N: Revealed type is "Union[Type[Any], builtins.int]" +reveal_type(u(t_o, 1)) # N: Revealed type is "Union[builtins.int, Type[builtins.object]]" +reveal_type(u(1, t_o)) # N: Revealed type is "Union[Type[builtins.object], builtins.int]" + +[case testSimplifyingUnionWithTypeTypes2] +from typing import TypeVar, Union, Type, Any + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +t_o: Type[object] +t_s: Type[str] +t_a: Type[Any] +t: type + +# Union with object +reveal_type(u(t_o, object())) # N: Revealed type is "builtins.object" +reveal_type(u(object(), t_o)) # N: Revealed type is "builtins.object" +reveal_type(u(t_s, object())) # N: Revealed type is "builtins.object" +reveal_type(u(object(), t_s)) # N: Revealed type is "builtins.object" +reveal_type(u(t_a, object())) # N: Revealed type is "builtins.object" +reveal_type(u(object(), t_a)) # N: Revealed type is "builtins.object" + +# Union between type objects +reveal_type(u(t_o, t_a)) # N: Revealed type is "Union[Type[Any], Type[builtins.object]]" +reveal_type(u(t_a, t_o)) # N: Revealed type is "Union[Type[builtins.object], Type[Any]]" +reveal_type(u(t_s, t_o)) # N: Revealed type is "Type[builtins.object]" +reveal_type(u(t_o, t_s)) # N: Revealed type is "Type[builtins.object]" +reveal_type(u(t_o, type)) # N: Revealed type is "Type[builtins.object]" +reveal_type(u(type, t_o)) # N: Revealed type is "Type[builtins.object]" +reveal_type(u(t_a, t)) # N: Revealed type is "builtins.type" +reveal_type(u(t, t_a)) # N: Revealed type is "builtins.type" +# The following should arguably not be simplified, but it's unclear how to fix then +# without causing regressions elsewhere. +reveal_type(u(t_o, t)) # N: Revealed type is "builtins.type" +reveal_type(u(t, t_o)) # N: Revealed type is "builtins.type" + +[case testNotSimplifyingUnionWithMetaclass] +from typing import TypeVar, Union, Type, Any + +class M(type): pass +class M2(M): pass +class A(metaclass=M): pass + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +a: Any +t_a: Type[A] + +reveal_type(u(M(*a), t_a)) # N: Revealed type is "__main__.M" +reveal_type(u(t_a, M(*a))) # N: Revealed type is "__main__.M" + +reveal_type(u(M2(*a), t_a)) # N: Revealed type is "Union[Type[__main__.A], __main__.M2]" +reveal_type(u(t_a, M2(*a))) # N: Revealed type is "Union[__main__.M2, Type[__main__.A]]" + +[case testSimplifyUnionWithCallable] +from typing import TypeVar, Union, Any, Callable + +T = TypeVar('T') +S = TypeVar('S') +def u(x: T, y: S) -> Union[S, T]: pass + +class C: pass +class D(C): pass + +D_C: Callable[[D], C] +A_C: Callable[[Any], C] +D_A: Callable[[D], Any] +C_C: Callable[[C], C] +D_D: Callable[[D], D] +i_C: Callable[[int], C] + +# TODO: Test argument names and kinds once we have flexible callable types. + +reveal_type(u(D_C, D_C)) # N: Revealed type is "def (__main__.D) -> __main__.C" + +reveal_type(u(A_C, D_C)) # N: Revealed type is "Union[def (__main__.D) -> __main__.C, def (Any) -> __main__.C]" +reveal_type(u(D_C, A_C)) # N: Revealed type is "Union[def (Any) -> __main__.C, def (__main__.D) -> __main__.C]" + +reveal_type(u(D_A, D_C)) # N: Revealed type is "Union[def (__main__.D) -> __main__.C, def (__main__.D) -> Any]" +reveal_type(u(D_C, D_A)) # N: Revealed type is "Union[def (__main__.D) -> Any, def (__main__.D) -> __main__.C]" + +reveal_type(u(D_C, C_C)) # N: Revealed type is "def (__main__.D) -> __main__.C" +reveal_type(u(C_C, D_C)) # N: Revealed type is "def (__main__.D) -> __main__.C" + +reveal_type(u(D_C, D_D)) # N: Revealed type is "def (__main__.D) -> __main__.C" +reveal_type(u(D_D, D_C)) # N: Revealed type is "def (__main__.D) -> __main__.C" + +reveal_type(u(D_C, i_C)) # N: Revealed type is "Union[def (builtins.int) -> __main__.C, def (__main__.D) -> __main__.C]" + +[case testUnionOperatorMethodSpecialCase] +from typing import Union +class C: + def __le__(self, x: 'C') -> int: ... +class D: + def __le__(self, other) -> int: ... +class E: + def __ge__(self, other: Union[C, D]) -> int: ... + +[case testUnionSimplificationWithBoolIntAndFloat] +from typing import List, Union +l = reveal_type([]) # type: List[Union[bool, int, float]] \ + # N: Revealed type is "builtins.list[Union[builtins.int, builtins.float]]" +reveal_type(l) \ + # N: Revealed type is "builtins.list[Union[builtins.bool, builtins.int, builtins.float]]" +[builtins fixtures/list.pyi] + +[case testUnionSimplificationWithBoolIntAndFloat2] +from typing import List, Union +l = reveal_type([]) # type: List[Union[bool, int, float, str]] \ + # N: Revealed type is "builtins.list[Union[builtins.int, builtins.float, builtins.str]]" +reveal_type(l) \ + # N: Revealed type is "builtins.list[Union[builtins.bool, builtins.int, builtins.float, builtins.str]]" +[builtins fixtures/list.pyi] + +[case testNestedUnionsProcessedCorrectly] +from typing import Union + +class A: pass +class B: pass +class C: pass + +def foo(bar: Union[Union[A, B], C]) -> None: + if isinstance(bar, A): + reveal_type(bar) # N: Revealed type is "__main__.A" + else: + reveal_type(bar) # N: Revealed type is "Union[__main__.B, __main__.C]" +[builtins fixtures/isinstance.pyi] +[out] + +[case testAssignAnyToUnion] +from typing import Union, Any +x: Union[int, str] +a: Any +if bool(): + x = a + # TODO: Maybe we should infer Any as the type instead. + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/bool.pyi] + +[case testAssignAnyToUnionWithAny] +from typing import Union, Any +x: Union[int, Any] +a: Any +if bool(): + x = a + reveal_type(x) # N: Revealed type is "Any" +reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" +[builtins fixtures/bool.pyi] + +[case testUnionMultiassignSingle] +from typing import Union, Tuple, Any + +a: Union[Tuple[int], Tuple[float]] +(a1,) = a +reveal_type(a1) # N: Revealed type is "Union[builtins.int, builtins.float]" + +b: Union[Tuple[int], Tuple[str]] +(b1,) = b +reveal_type(b1) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testUnionMultiassignDouble] +from typing import Union, Tuple + +c: Union[Tuple[int, int], Tuple[int, float]] +(c1, c2) = c +reveal_type(c1) # N: Revealed type is "builtins.int" +reveal_type(c2) # N: Revealed type is "Union[builtins.int, builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testUnionMultiassignGeneric] +from typing import Union, Tuple, TypeVar +T = TypeVar('T') +S = TypeVar('S') + +def pack_two(x: T, y: S) -> Union[Tuple[T, T], Tuple[S, S]]: + pass + +(x, y) = pack_two(1, 'a') +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testUnionMultiassignAny] +from typing import Union, Tuple, Any + +d: Union[Any, Tuple[float, float]] +(d1, d2) = d +reveal_type(d1) # N: Revealed type is "Union[Any, builtins.float]" +reveal_type(d2) # N: Revealed type is "Union[Any, builtins.float]" + +e: Union[Any, Tuple[float, float], int] +(e1, e2) = e # E: "int" object is not iterable +[builtins fixtures/tuple.pyi] + +[case testUnionMultiassignNotJoin] +from typing import Union, List + +class A: pass +class B(A): pass +class C(A): pass +a: Union[List[B], List[C]] +x, y = a +reveal_type(x) # N: Revealed type is "Union[__main__.B, __main__.C]" +[builtins fixtures/list.pyi] + +[case testUnionMultiassignRebind] +from typing import Union, List + +class A: pass +class B(A): pass +class C(A): pass +obj: object +a: Union[List[B], List[C]] +obj, new = a +reveal_type(obj) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(new) # N: Revealed type is "Union[__main__.B, __main__.C]" + +obj = 1 +reveal_type(obj) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testUnionMultiassignAlreadyDeclared] +from typing import Union, Tuple + +a: Union[Tuple[int, int], Tuple[int, float]] +a1: object +a2: int +(a1, a2) = a # E: Incompatible types in assignment (expression has type "float", variable has type "int") + +b: Union[Tuple[float, int], Tuple[int, int]] +b1: object +b2: int +(b1, b2) = b +reveal_type(b1) # N: Revealed type is "Union[builtins.float, builtins.int]" +reveal_type(b2) # N: Revealed type is "builtins.int" + +c: Union[Tuple[int, int], Tuple[int, int]] +c1: object +c2: int +(c1, c2) = c +reveal_type(c1) # N: Revealed type is "builtins.int" +reveal_type(c2) # N: Revealed type is "builtins.int" + +d: Union[Tuple[int, int], Tuple[int, float]] +d1: object +(d1, d2) = d +reveal_type(d1) # N: Revealed type is "builtins.int" +reveal_type(d2) # N: Revealed type is "Union[builtins.int, builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testUnionMultiassignIndexed] +from typing import Union, Tuple, List + +class B: + x: object + +x: List[int] +b: B + +a: Union[Tuple[int, int], Tuple[int, object]] +(x[0], b.x) = a +reveal_type(x[0]) # N: Revealed type is "builtins.int" +reveal_type(b.x) # N: Revealed type is "builtins.object" +[builtins fixtures/list.pyi] + +[case testUnionMultiassignIndexedWithError] +from typing import Union, Tuple, List + +class A: pass +class B: + x: int + +x: List[A] +b: B + +a: Union[Tuple[int, int], Tuple[int, object]] +(x[0], b.x) = a # E: Incompatible types in assignment (expression has type "int", target has type "A") \ + # E: Incompatible types in assignment (expression has type "object", variable has type "int") +reveal_type(x[0]) # N: Revealed type is "__main__.A" +reveal_type(b.x) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + +[case testUnionMultiassignPacked] +from typing import Union, Tuple, List + +a: Union[Tuple[int, int, int], Tuple[int, int, str]] +a1: int +a2: object +(a1, *xs, a2) = a + +reveal_type(a1) # N: Revealed type is "builtins.int" +reveal_type(xs) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(a2) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/list.pyi] + +[case testUnpackingUnionOfListsInFunction] +from typing import Union, List + +def f(x: bool) -> Union[List[int], List[str]]: + if x: + return [1, 1] + else: + return ['a', 'a'] + +def g(x: bool) -> None: + a, b = f(x) + reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/list.pyi] + +[case testUnionOfVariableLengthTupleUnpacking] +from typing import Tuple, Union +VarTuple = Union[Tuple[int, int], Tuple[int, int, int]] + +def make_tuple() -> VarTuple: + pass +x = make_tuple() + +a, b = x # E: Too many values to unpack (2 expected, 3 provided) +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +c, *d = x +reveal_type(c) # N: Revealed type is "builtins.int" +reveal_type(d) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testUnionOfNonIterableUnpacking] +from typing import Union +bad: Union[int, str] + +x, y = bad # E: "int" object is not iterable \ + # E: Unpacking a string is disallowed +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "Any" +[out] + +[case testStringDisallowedUnpacking] +from typing import Dict + +d: Dict[str, str] + +for a, b in d: # E: Unpacking a string is disallowed + pass + +s = "foo" +a, b = s # E: Unpacking a string is disallowed + +[builtins fixtures/dict.pyi] +[out] + +[case testUnionAlwaysTooMany] +from typing import Union, Tuple +bad: Union[Tuple[int, int, int], Tuple[str, str, str]] + +x, y = bad # E: Too many values to unpack (2 expected, 3 provided) +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[out] + +[case testUnionAlwaysTooFew] +from typing import Union, Tuple +bad: Union[Tuple[int, int, int], Tuple[str, str, str]] + +x, y, z, w = bad # E: Need more than 3 values to unpack (4 expected) +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "Any" +reveal_type(z) # N: Revealed type is "Any" +reveal_type(w) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[out] + +[case testUnionUnpackingChainedTuple] +from typing import Union, Tuple +good: Union[Tuple[int, int], Tuple[str, str]] + +x, y = t = good +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(t) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.str, builtins.str]]" +[builtins fixtures/tuple.pyi] +[out] + +[case testUnionUnpackingChainedTuple2] +from typing import Union, Tuple +good: Union[Tuple[int, int], Tuple[str, str]] + +t = x, y = good +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(t) # N: Revealed type is "Union[Tuple[builtins.int, builtins.int], Tuple[builtins.str, builtins.str]]" +[builtins fixtures/tuple.pyi] +[out] + +[case testUnionUnpackingChainedTuple3] +from typing import Union, Tuple +good: Union[Tuple[int, int], Tuple[str, str]] + +x, y = a, b = good +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(b) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] +[out] + +[case testUnionUnpackingChainedList] +from typing import Union, List +good: Union[List[int], List[str]] + +lst = x, y = good +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(lst) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" +[builtins fixtures/list.pyi] +[out] + +[case testUnionUnpackingChainedList2] +from typing import Union, List +good: Union[List[int], List[str]] + +x, *y, z = lst = good +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" +reveal_type(z) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(lst) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" +[builtins fixtures/list.pyi] +[out] + +[case testUnionUnpackingInForTuple] +from typing import Union, Tuple, NamedTuple +class NTInt(NamedTuple): + x: int + y: int +class NTStr(NamedTuple): + x: str + y: str + +t1: NTInt +reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.int]" +nt: Union[NTInt, NTStr] +reveal_type(nt.__iter__) # N: Revealed type is "Union[def () -> typing.Iterator[builtins.int], def () -> typing.Iterator[builtins.str]]" +for nx in nt: + reveal_type(nx) # N: Revealed type is "Union[builtins.int, builtins.str]" + +t: Union[Tuple[int, int], Tuple[str, str]] +for x in t: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/for.pyi] +[out] + +[case testUnionUnpackingInForList] +from typing import Union, List, Tuple + +t: Union[List[Tuple[int, int]], List[Tuple[str, str]]] +for x, y in t: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" + +t2: List[Union[Tuple[int, int], Tuple[str, str]]] +for x2, y2 in t2: + reveal_type(x2) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(y2) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/for.pyi] +[out] + +[case testUnionUnpackingDoubleBinder] +from typing import Union, Tuple + +x: object +y: object +class A: pass +class B: pass + +t1: Union[Tuple[A, A], Tuple[B, B]] +t2: Union[Tuple[int, int], Tuple[str, str]] + +x, y = t1 +reveal_type(x) # N: Revealed type is "Union[__main__.A, __main__.B]" +reveal_type(y) # N: Revealed type is "Union[__main__.A, __main__.B]" + +x, y = t2 +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" + +x, y = object(), object() +reveal_type(x) # N: Revealed type is "builtins.object" +reveal_type(y) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] +[out] + +[case testUnionUnpackingFromNestedTuples] +from typing import Union, Tuple + +t: Union[Tuple[int, Tuple[int, int]], Tuple[str, Tuple[str, str]]] +x, (y, z) = t +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(z) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] +[out] + +[case testNestedUnionUnpackingFromNestedTuples] +from typing import Union, Tuple + +class A: pass +class B: pass + +t: Union[Tuple[int, Union[Tuple[int, int], Tuple[A, A]]], Tuple[str, Union[Tuple[str, str], Tuple[B, B]]]] +x, (y, z) = t +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, __main__.A, builtins.str, __main__.B]" +reveal_type(z) # N: Revealed type is "Union[builtins.int, __main__.A, builtins.str, __main__.B]" +[builtins fixtures/tuple.pyi] +[out] + +[case testNestedUnionUnpackingFromNestedTuplesBinder] +from typing import Union, Tuple + +class A: pass +class B: pass + +x: object +y: object +z: object + +t: Union[Tuple[int, Union[Tuple[int, int], Tuple[A, A]]], Tuple[str, Union[Tuple[str, str], Tuple[B, B]]]] +x, (y, z) = t +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(y) # N: Revealed type is "Union[builtins.int, __main__.A, builtins.str, __main__.B]" +reveal_type(z) # N: Revealed type is "Union[builtins.int, __main__.A, builtins.str, __main__.B]" +[builtins fixtures/tuple.pyi] +[out] + +[case testUnpackUnionNoCrashOnPartialNone] +from typing import Dict, Tuple, List, Any + +a: Any +d: Dict[str, Tuple[List[Tuple[str, str]], str]] +x, _ = d.get(a, (None, None)) + +for y in x: pass # E: Item "None" of "Optional[List[Tuple[str, str]]]" has no attribute "__iter__" (not iterable) +if x: + for s, t in x: + reveal_type(s) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] +[out] + +[case testUnpackUnionNoCrashOnPartialNone2] +from typing import Dict, Tuple, List, Any + +a: Any +x = None +d: Dict[str, Tuple[List[Tuple[str, str]], str]] +x, _ = d.get(a, (None, None)) + +for y in x: pass # E: Item "None" of "Optional[List[Tuple[str, str]]]" has no attribute "__iter__" (not iterable) +if x: + for s, t in x: + reveal_type(s) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] +[out] + +[case testUnpackUnionNoCrashOnPartialNoneBinder] +from typing import Dict, Tuple, List, Any + +x: object +a: Any +d: Dict[str, Tuple[List[Tuple[str, str]], str]] +x, _ = d.get(a, (None, None)) +reveal_type(x) # N: Revealed type is "Union[builtins.list[Tuple[builtins.str, builtins.str]], None]" + +if x: + for y in x: pass +[builtins fixtures/dict.pyi] +[out] + +[case testUnpackUnionNoCrashOnPartialList] +from typing import Dict, Tuple, List, Any + +a: Any +d: Dict[str, Tuple[List[Tuple[str, str]], str]] +x, _ = d.get(a, ([], "")) +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[builtins.str, builtins.str]]" + +for y in x: pass +[builtins fixtures/dict.pyi] +[out] + +[case testLongUnionFormatting] +from typing import Any, Generic, TypeVar, Union + +T = TypeVar('T') + +class ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes(Generic[T]): + pass + +x: Union[ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[int], + ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[object], + ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[float], + ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[str], + ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[Any], + ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[bytes]] + +def takes_int(arg: int) -> None: pass + +takes_int(x) # E: Argument 1 to "takes_int" has incompatible type "Union[ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[int], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[object], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[float], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[str], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[Any], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[bytes]]"; expected "int" + +[case testRecursiveForwardReferenceInUnion] +from typing import List, Union + +def test() -> None: + MYTYPE = List[Union[str, "MYTYPE"]] # E: Cannot resolve name "MYTYPE" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope +[builtins fixtures/list.pyi] + +[case testNonStrictOptional] +# flags: --no-strict-optional +from typing import Optional, List + +def union_test1(x): + # type: (Optional[List[int]]) -> Optional[int] + if x is None: + return x + else: + return x[0] + +def union_test2(x): + # type: (Optional[List[int]]) -> Optional[int] + if isinstance(x, type(None)): + return x + else: + return x[0] + +def f(): return 0 + +def union_test3(): + # type: () -> int + x = f() + assert x is None + x = f() + return x + 1 + +[builtins fixtures/isinstancelist.pyi] + +[case testInvariance] +from typing import List, Union +from enum import Enum + +class Boop(Enum): + FOO = 1 + +def do_thing_with_enums(enums: Union[List[Enum], Enum]) -> None: ... + +boop: List[Boop] = [] +do_thing_with_enums(boop) # E: Argument 1 to "do_thing_with_enums" has incompatible type "List[Boop]"; expected "Union[List[Enum], Enum]" \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant +[builtins fixtures/isinstancelist.pyi] + +[case testAssignUnionWithTenaryExprWithEmptyCollection] +from typing import Dict, List, Union +x: Union[int, List[int]] = 1 if bool() else [] +y: Union[int, Dict[int, int]] = 1 if bool() else {} + +u: Union[int, List[int]] = [] if bool() else 1 +v: Union[int, Dict[int, int]] = {} if bool() else 1 +[builtins fixtures/isinstancelist.pyi] + +[case testFlattenTypeAliasWhenAliasedAsUnion] +from typing import Union + +T1 = int +T2 = Union[T1, float] +T3 = Union[T2, complex] +T4 = Union[T3, int] + +def foo(a: T2, b: T2) -> T2: + return a + b + +def bar(a: T4, b: T4) -> T4: # test multi-level alias + return a + b +[builtins fixtures/ops.pyi] + +[case testJoinUnionWithUnionAndAny] +from typing import TypeVar, Union, Any +T = TypeVar("T") +def f(x: T, y: T) -> T: + return x +x: Union[None, Any] +y: Union[int, None] +reveal_type(f(x, y)) # N: Revealed type is "Union[None, Any, builtins.int]" +reveal_type(f(y, x)) # N: Revealed type is "Union[builtins.int, None, Any]" + +[case testNestedProtocolUnions] +from typing import Union, Iterator, Iterable +def foo( + values: Union[ + Iterator[Union[ + Iterator[Union[Iterator[int], Iterable[int]]], + Iterable[Union[Iterator[int], Iterable[int]]], + ]], + Iterable[Union[ + Iterator[Union[Iterator[int], Iterable[int]]], + Iterable[Union[Iterator[int], Iterable[int]]], + ]], + ] +) -> Iterator[int]: + for i in values: + for j in i: + for k in j: + yield k +foo([[[1]]]) +[builtins fixtures/list.pyi] + +[case testNestedProtocolGenericUnions] +from typing import Union, Iterator, List +def foo( + values: Union[ + Iterator[Union[ + Iterator[Union[Iterator[int], List[int]]], + List[Union[Iterator[int], List[int]]], + ]], + List[Union[ + Iterator[Union[Iterator[int], List[int]]], + List[Union[Iterator[int], List[int]]], + ]], + ] +) -> Iterator[int]: + for i in values: + for j in i: + for k in j: + yield k +foo([[[1]]]) +[builtins fixtures/list.pyi] + +[case testNestedProtocolGenericUnionsDeep] +from typing import TypeVar, Union, Iterator, List +T = TypeVar("T") +Iter = Union[Iterator[T], List[T]] +def foo( + values: Iter[Iter[Iter[Iter[Iter[int]]]]], +) -> Iterator[int]: + for i in values: + for j in i: + for k in j: + for l in k: + for m in l: + yield m +foo([[[[[1]]]]]) +[builtins fixtures/list.pyi] + +[case testNestedInstanceUnsimplifiedUnion] +from typing import TypeVar, Union, Iterator, List, Any +T = TypeVar("T") + +Iter = Union[Iterator[T], List[T]] +def foo( + values: Iter[Union[Any, Any]], +) -> Iterator[Any]: + for i in values: + yield i +foo([1]) +[builtins fixtures/list.pyi] + +[case testNestedInstanceTypeAlias] +from typing import TypeVar, Union, Iterator, List, Any +T = TypeVar("T") + +Iter = Union[Iterator[T], List[T]] +def foo( + values: Iter["Any"], +) -> Iterator[Any]: + for i in values: + yield i +foo([1]) +[builtins fixtures/list.pyi] + +[case testGenericUnionMemberWithTypeVarConstraints] + +from typing import Generic, TypeVar, Union + +T = TypeVar('T', str, int) + +class C(Generic[T]): ... + +def f(s: Union[T, C[T]]) -> T: ... + +ci: C[int] +cs: C[str] + +reveal_type(f(1)) # N: Revealed type is "builtins.int" +reveal_type(f('')) # N: Revealed type is "builtins.str" +reveal_type(f(ci)) # N: Revealed type is "builtins.int" +reveal_type(f(cs)) # N: Revealed type is "builtins.str" + + +[case testNestedInstanceTypeAliasUnsimplifiedUnion] +from typing import TypeVar, Union, Iterator, List, Any +T = TypeVar("T") + +Iter = Union[Iterator[T], List[T]] +def foo( + values: Iter["Union[Any, Any]"], +) -> Iterator[Any]: + for i in values: + yield i +foo([1]) +[builtins fixtures/list.pyi] + +[case testUnionIterableContainer] +from typing import Iterable, Container, Union + +i: Iterable[str] +c: Container[str] +u: Union[Iterable[str], Container[str]] +ni: Union[Iterable[str], int] +nc: Union[Container[str], int] + +'x' in i +'x' in c +'x' in u +'x' in ni # E: Unsupported right operand type for in ("Union[Iterable[str], int]") +'x' in nc # E: Unsupported right operand type for in ("Union[Container[str], int]") +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testDescriptorAccessForUnionOfTypes] +from typing import overload, Generic, Any, TypeVar, List, Optional, Union, Type + +_T_co = TypeVar("_T_co", bound=Any, covariant=True) + +class Mapped(Generic[_T_co]): + def __init__(self, value: _T_co): + self.value = value + + @overload + def __get__( + self, instance: None, owner: Any + ) -> List[_T_co]: + ... + + @overload + def __get__(self, instance: object, owner: Any) -> _T_co: + ... + + def __get__( + self, instance: Optional[object], owner: Any + ) -> Union[List[_T_co], _T_co]: + return self.value + +class A: + field_1: Mapped[int] = Mapped(1) + field_2: Mapped[str] = Mapped('1') + +class B: + field_1: Mapped[int] = Mapped(2) + field_2: Mapped[str] = Mapped('2') + +mix: Union[Type[A], Type[B]] = A +reveal_type(mix) # N: Revealed type is "Union[Type[__main__.A], Type[__main__.B]]" +reveal_type(mix.field_1) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(mix().field_1) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + + +[case testDescriptorAccessForUnionOfTypesWithNoStrictOptional] +# mypy: no-strict-optional +from typing import overload, Generic, Any, TypeVar, List, Optional, Union, Type + +class Descriptor: + @overload + def __get__( + self, instance: None, owner: type + ) -> str: + ... + + @overload + def __get__(self, instance: object, owner: type) -> int: + ... + + def __get__( + self, instance: Optional[object], owner: type + ) -> Union[str, int]: + ... + +class A: + field = Descriptor() + +a_class_or_none: Optional[Type[A]] +x: str = a_class_or_none.field + +a_or_none: Optional[A] +y: int = a_or_none.field +[builtins fixtures/list.pyi] + +[case testLargeUnionsShort] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... +class C11: ... + +u: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11] +x: int = u # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, <6 more items>]", variable has type "int") + +[case testLargeUnionsLongIfNeeded] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... + +x: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, int] +y: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, str] +x = y # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, str]", variable has type "Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, int]") \ + # N: Item in the first union not in the second: "str" + +[case testLargeUnionsNoneShown] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... +class C11: ... + +x: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11] +y: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, None] +x = y # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, <6 more items>, None]", variable has type "Union[C1, C2, C3, C4, C5, <6 more items>]") \ + # N: Item in the first union not in the second: "None" diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test new file mode 100644 index 000000000000..e6818ab5c3c7 --- /dev/null +++ b/test-data/unit/check-unreachable-code.test @@ -0,0 +1,1519 @@ +-- Type checker test cases for conditional checks that result in some +-- blocks classified as unreachable (they are not type checked or semantically +-- analyzed). +-- +-- For example, we skip blocks that will not be executed on the active +-- Python version. + +[case testConditionalTypeAliasPY3] +import typing +def f(): pass +PY3 = f() +if PY3: + t = int + x = object() + 'x' # E: Unsupported left operand type for + ("object") +else: + t = str + y = 'x' / 1 +x +z = 1 # type: t + +[case testConditionalAssignmentPY2] +import typing +def f(): pass +PY2 = f() +if PY2: + x = object() + 'x' +else: + y = 'x' / 1 # E: Unsupported left operand type for / ("str") +y + +[case testConditionalImport] +import typing +def f(): pass +PY2 = f() +if PY2: + import fuzzybar + from barbar import * + from pawwaw import a, bc +else: + import m +[file m.py] +import typing +x = 1 +if int(): + x = 'a' +[out] +tmp/m.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNegatedMypyConditional] +import typing +MYPY = 0 +if not MYPY: + import xyz753 +else: + import pow123 # E +[builtins fixtures/bool.pyi] +[out] +main:6: error: Cannot find implementation or library stub for module named "pow123" +main:6: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testMypyConditional] +import typing +MYPY = 0 +if MYPY: + None + 1 # E: Unsupported left operand type for + ("None") +else: + None + '' +[builtins fixtures/bool.pyi] + +[case testTypeCheckingConditional] +import typing +if typing.TYPE_CHECKING: + import pow123 # E +else: + import xyz753 +[typing fixtures/typing-medium.pyi] +[out] +main:3: error: Cannot find implementation or library stub for module named "pow123" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testTypeCheckingConditionalFromImport] +from typing import TYPE_CHECKING +if TYPE_CHECKING: + import pow123 # E +else: + import xyz753 +[typing fixtures/typing-medium.pyi] +[out] +main:3: error: Cannot find implementation or library stub for module named "pow123" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testNegatedTypeCheckingConditional] +import typing +if not typing.TYPE_CHECKING: + import pow123 # E +else: + import xyz753 +[builtins fixtures/bool.pyi] +[typing fixtures/typing-medium.pyi] +[out] +main:5: error: Cannot find implementation or library stub for module named "xyz753" +main:5: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testUndefinedTypeCheckingConditional] +if not TYPE_CHECKING: # E + import pow123 +else: + import xyz753 +[builtins fixtures/bool.pyi] +[out] +main:1: error: Name "TYPE_CHECKING" is not defined +main:4: error: Cannot find implementation or library stub for module named "xyz753" +main:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testConditionalClassDefPY3] +def f(): pass +PY3 = f() +if PY3: + pass +else: + class X(object): + pass + +[case testUnreachabilityAndElifPY3] +def f(): pass +PY3 = f() +if PY3: + pass +elif bool(): + import nonexistent + 1 + '' +else: + import bad_name + 1 + '' +[builtins fixtures/bool.pyi] +[out] + +[case testSysVersionInfo] +import sys +if sys.version_info[0] >= 3: + def foo() -> int: return 0 +else: + def foo() -> str: return '' +reveal_type(foo()) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoReversedOperandsOrder] +import sys +if (3,) <= sys.version_info: + def foo() -> int: return 0 +else: + def foo() -> str: return '' +reveal_type(foo()) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoNegated] +import sys +if not (sys.version_info[0] < 3): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +reveal_type(foo()) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced1] +import sys +if sys.version_info[:1] >= (3,): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced2] +import sys +if sys.version_info[:2] >= (3, 0): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced3] +import sys +if sys.version_info[:] >= (3, 0): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced4] +import sys +if sys.version_info[0:2] >= (3, 0): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced5] +import sys +if sys.version_info[0:] >= (3,): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced6] +import sys +if sys.version_info[1:] >= (5,): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced7] +import sys +if sys.version_info >= (3, 5): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced8] +# Our pyversion only has (major, minor), +# so testing for (major, minor, bugfix) is unsupported. +import sys +if sys.version_info >= (3, 5, 0): + def foo() -> int: return 0 +else: + def foo() -> str: return '' # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def foo() -> int \ + # N: Redefinition: \ + # N: def foo() -> str +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoSliced9] +# Our pyversion only has (major, minor), +# so testing for (minor, bugfix) is unsupported (also it's silly :-). +import sys +if sys.version_info[1:] >= (5, 0): + def foo() -> int: return 0 +else: + def foo() -> str: return '' # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def foo() -> int \ + # N: Redefinition: \ + # N: def foo() -> str +[builtins fixtures/ops.pyi] +[out] + +[case testSysPlatform1] +import sys +if sys.platform == 'fictional': + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + '' +[builtins fixtures/ops.pyi] +[out] + +[case testSysPlatform2] +import sys +if sys.platform != 'fictional': + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysPlatformNegated] +import sys +if not (sys.platform == 'fictional'): + def foo() -> int: return 0 +else: + def foo() -> str: return '' +foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoClass] +import sys +if sys.version_info < (3, 5): + class C: + pass +else: + class C: + def foo(self) -> int: return 0 +C().foo() + 0 +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoImport] +import sys +if sys.version_info >= (3, 5): + import collections +else: + collections = None +Pt = collections.namedtuple('Pt', 'x y z') +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoVariable] +import sys +if sys.version_info >= (3, 5): + x = '' +else: + x = 0 +x + '' +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoInClass] +import sys +class C: + if sys.version_info >= (3, 5): + def foo(self) -> int: return 0 + else: + def foo(self) -> str: return '' +reveal_type(C().foo()) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] +[out] + +[case testSysVersionInfoInFunction] +import sys +def foo() -> None: + if sys.version_info >= (3, 5): + x = '' + else: + x = 0 + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/ops.pyi] +[out] + +[case testSysPlatformInMethod] +import sys +class C: + def foo(self) -> None: + if sys.platform != 'fictional': + x = '' + else: + x = 0 + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/ops.pyi] +[out] + +[case testSysPlatformInFunctionImport1] +import sys +def foo() -> None: + if sys.platform != 'fictional': + import a + else: + import b as a + a.x +[file a.py] +x = 1 +[builtins fixtures/ops.pyi] +[out] + +[case testSysPlatformInFunctionImport2] +import sys +def foo() -> None: + if sys.platform == 'fictional': + import b as a + else: + import a + a.x +[file a.py] +x = 1 +[builtins fixtures/ops.pyi] +[out] + +[case testSysPlatformInFunctionImport3] +from typing import Callable +import sys + +def idf(x: Callable[[], None]) -> Callable[[], None]: return x + +@idf +def foo() -> None: + if sys.platform == 'fictional': + import b as a + else: + import a + a.x +[file a.py] +x = 1 +[builtins fixtures/ops.pyi] +[out] + + +[case testSysPlatformInMethodImport2] + +import sys +class A: + def foo(self) -> None: + if sys.platform == 'fictional': + import b as a + else: + import a + a.x +[file a.py] +x = 1 +[builtins fixtures/ops.pyi] +[out] + +[case testCustomSysVersionInfo] +# flags: --python-version 3.11 +import sys +if sys.version_info == (3, 11): + x = "foo" +else: + x = 3 +reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/ops.pyi] +[out] + +[case testCustomSysVersionInfo2] +# flags: --python-version 3.11 +import sys +if sys.version_info == (3, 6): + x = "foo" +else: + x = 3 +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] +[out] + +[case testCustomSysPlatform] +# flags: --platform linux +import sys +if sys.platform == 'linux': + x = "foo" +else: + x = 3 +reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/ops.pyi] +[out] + +[case testCustomSysPlatform2] +# flags: --platform win32 +import sys +if sys.platform == 'linux': + x = "foo" +else: + x = 3 +reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] +[out] + +[case testCustomSysPlatformStartsWith] +# flags: --platform win32 +import sys +if sys.platform.startswith('win'): + x = "foo" +else: + x = 3 +reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/ops.pyi] +[out] + +[case testShortCircuitInExpression] +import typing +def make() -> bool: pass +PY2 = PY3 = make() + +a = PY2 and 's' +b = PY3 and 's' +c = PY2 or 's' +d = PY3 or 's' +e = (PY2 or PY3) and 's' +f = (PY3 or PY2) and 's' +g = (PY2 or PY3) or 's' +h = (PY3 or PY2) or 's' +reveal_type(a) # N: Revealed type is "builtins.bool" +reveal_type(b) # N: Revealed type is "Literal['s']" +reveal_type(c) # N: Revealed type is "Literal['s']" +reveal_type(d) # N: Revealed type is "builtins.bool" +reveal_type(e) # N: Revealed type is "Literal['s']" +reveal_type(f) # N: Revealed type is "Literal['s']" +reveal_type(g) # N: Revealed type is "builtins.bool" +reveal_type(h) # N: Revealed type is "builtins.bool" +[builtins fixtures/ops.pyi] +[out] + +[case testShortCircuitAndWithConditionalAssignment] +# flags: --platform linux +import sys + +def f(): pass +PY2 = f() +if PY2 and sys.platform == 'linux': + x = 'foo' +else: + x = 3 +reveal_type(x) # N: Revealed type is "builtins.int" +if sys.platform == 'linux' and PY2: + y = 'foo' +else: + y = 3 +reveal_type(y) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] + +[case testShortCircuitOrWithConditionalAssignment] +# flags: --platform linux +import sys + +def f(): pass +PY2 = f() +if PY2 or sys.platform == 'linux': + x = 'foo' +else: + x = 3 +reveal_type(x) # N: Revealed type is "builtins.str" +if sys.platform == 'linux' or PY2: + y = 'foo' +else: + y = 3 +reveal_type(y) # N: Revealed type is "builtins.str" +[builtins fixtures/ops.pyi] + +[case testShortCircuitNoEvaluation] +# flags: --platform linux --always-false COMPILE_TIME_FALSE +import sys + +if sys.platform == 'darwin': + mac_only = 'junk' + +# `mac_only` should not be evaluated +if sys.platform == 'darwin' and mac_only: + pass +if sys.platform == 'linux' or mac_only: + pass + +COMPILE_TIME_FALSE = 'junk' + +if COMPILE_TIME_FALSE: + compile_time_false_only = 'junk' + +# `compile_time_false_only` should not be evaluated +if COMPILE_TIME_FALSE and compile_time_false_only: + pass +if not COMPILE_TIME_FALSE or compile_time_false_only: + pass + +MYPY = False + +if not MYPY: + mypy_only = 'junk' + +# `mypy_only` should not be evaluated +if not MYPY and mypy_only: + pass +if MYPY or mypy_only: + pass +[builtins fixtures/ops.pyi] + +[case testSemanticAnalysisFalseButTypeNarrowingTrue] +# flags: --always-false COMPILE_TIME_FALSE +from typing import Literal + +indeterminate: str +COMPILE_TIME_FALSE: Literal[True] # type-narrowing: mapped in 'if' only +a = COMPILE_TIME_FALSE or indeterminate +reveal_type(a) # N: Revealed type is "builtins.str" +b = indeterminate or COMPILE_TIME_FALSE +reveal_type(b) # N: Revealed type is "Union[builtins.str, Literal[True]]" +[typing fixtures/typing-medium.pyi] + +[case testSemanticAnalysisTrueButTypeNarrowingFalse] +# flags: --always-true COMPILE_TIME_TRUE +from typing import Literal + +indeterminate: str +COMPILE_TIME_TRUE: Literal[False] # type narrowed to `else` only +a = COMPILE_TIME_TRUE or indeterminate +reveal_type(a) # N: Revealed type is "Literal[False]" +b = indeterminate or COMPILE_TIME_TRUE +reveal_type(b) # N: Revealed type is "Union[builtins.str, Literal[False]]" + +[typing fixtures/typing-medium.pyi] +[case testConditionalAssertWithoutElse] +import typing + +class A: pass +class B(A): pass + +x = A() +reveal_type(x) # N: Revealed type is "__main__.A" + +if typing.TYPE_CHECKING: + assert isinstance(x, B) + reveal_type(x) # N: Revealed type is "__main__.B" + +reveal_type(x) # N: Revealed type is "__main__.B" + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-medium.pyi] + +[case testUnreachableWhenSuperclassIsAny] +from typing import Any + +# This can happen if we're importing a class from a missing module +Parent: Any +class Child(Parent): + def foo(self) -> int: + reveal_type(self) # N: Revealed type is "__main__.Child" + if self is None: + reveal_type(self) + return None + reveal_type(self) # N: Revealed type is "__main__.Child" + return 3 + + def bar(self) -> int: + if 1: + self = super(Child, self).something() + reveal_type(self) # N: Revealed type is "__main__.Child" + if self is None: + reveal_type(self) + return None + reveal_type(self) # N: Revealed type is "__main__.Child" + return 3 +[builtins fixtures/isinstance.pyi] + +[case testUnreachableWhenSuperclassIsAnyNoStrictOptional] +# flags: --no-strict-optional +from typing import Any + +Parent: Any +class Child(Parent): + def foo(self) -> int: + reveal_type(self) # N: Revealed type is "__main__.Child" + if self is None: + reveal_type(self) # N: Revealed type is "None" + return None + reveal_type(self) # N: Revealed type is "__main__.Child" + return 3 +[builtins fixtures/isinstance.pyi] + +[case testUnreachableAfterToplevelAssert] +import sys +reveal_type(0) # N: Revealed type is "Literal[0]?" +assert sys.platform == 'lol' +reveal_type('') # No error here :-) +[builtins fixtures/ops.pyi] + +[case testUnreachableAfterToplevelAssert2] +import sys +reveal_type(0) # N: Revealed type is "Literal[0]?" +assert sys.version_info[0] == 1 +reveal_type('') # No error here :-) +[builtins fixtures/ops.pyi] + +[case testUnreachableAfterToplevelAssert3] +reveal_type(0) # N: Revealed type is "Literal[0]?" +MYPY = False +assert not MYPY +reveal_type('') # No error here :-) +[builtins fixtures/ops.pyi] + +[case testUnreachableAfterToplevelAssert4] +# flags: --always-false NOPE +reveal_type(0) # N: Revealed type is "Literal[0]?" +NOPE = False +assert NOPE +reveal_type('') # No error here :-) +[builtins fixtures/ops.pyi] + +[case testUnreachableAfterToplevelAssertImport] +import foo +foo.bar() # E: "object" has no attribute "bar" +[file foo.py] +import sys +assert sys.platform == 'lol' +def bar() -> None: pass +[builtins fixtures/ops.pyi] + +[case testUnreachableAfterToplevelAssertImport2] +# flags: --platform lol +import foo +foo.bar() # No error :-) +[file foo.py] +import sys +assert sys.platform == 'lol' +def bar() -> None: pass +[builtins fixtures/ops.pyi] + +[case testUnreachableAfterToplevelAssertNotInsideIf] +import sys +if sys.version_info[0] >= 2: + assert sys.platform == 'lol' + reveal_type('') # N: Revealed type is "Literal['']?" +reveal_type('') # N: Revealed type is "Literal['']?" +[builtins fixtures/ops.pyi] + +[case testUnreachableFlagWithBadControlFlow1] +# flags: --warn-unreachable +a: int +if isinstance(a, int): + reveal_type(a) # N: Revealed type is "builtins.int" +else: + reveal_type(a) # E: Statement is unreachable +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableFlagWithBadControlFlow2] +# flags: --warn-unreachable +b: int +while isinstance(b, int): + reveal_type(b) # N: Revealed type is "builtins.int" +else: + reveal_type(b) # E: Statement is unreachable +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableFlagWithBadControlFlow3] +# flags: --warn-unreachable +def foo(c: int) -> None: + reveal_type(c) # N: Revealed type is "builtins.int" + assert not isinstance(c, int) + reveal_type(c) # E: Statement is unreachable +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableFlagWithBadControlFlow4] +# flags: --warn-unreachable +d: int +if False: + reveal_type(d) # E: Statement is unreachable +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableFlagWithBadControlFlow5] +# flags: --warn-unreachable +e: int +if True: + reveal_type(e) # N: Revealed type is "builtins.int" +else: + reveal_type(e) # E: Statement is unreachable +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableFlagStatementAfterReturn] +# flags: --warn-unreachable +def foo(x: int) -> None: + reveal_type(x) # N: Revealed type is "builtins.int" + return + reveal_type(x) # E: Statement is unreachable + +[case testUnreachableFlagTryBlocks] +# flags: --warn-unreachable + +def foo(x: int) -> int: + try: + reveal_type(x) # N: Revealed type is "builtins.int" + return x + reveal_type(x) # E: Statement is unreachable + finally: + reveal_type(x) # N: Revealed type is "builtins.int" + if True: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # E: Statement is unreachable + +def bar(x: int) -> int: + try: + if True: + raise Exception() + reveal_type(x) # E: Statement is unreachable + except: + reveal_type(x) # N: Revealed type is "builtins.int" + return x + else: + reveal_type(x) # E: Statement is unreachable + +def baz(x: int) -> int: + try: + reveal_type(x) # N: Revealed type is "builtins.int" + except: + # Mypy assumes all lines could throw an exception + reveal_type(x) # N: Revealed type is "builtins.int" + return x + else: + reveal_type(x) # N: Revealed type is "builtins.int" + return x +[builtins fixtures/exception.pyi] + +[case testUnreachableFlagIgnoresSemanticAnalysisUnreachable] +# flags: --warn-unreachable --python-version 3.8 --platform win32 --always-false FOOBAR +import sys +from typing import TYPE_CHECKING + +x: int +if TYPE_CHECKING: + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) + +if not TYPE_CHECKING: + reveal_type(x) +else: + reveal_type(x) # N: Revealed type is "builtins.int" + +if sys.platform == 'darwin': + reveal_type(x) +else: + reveal_type(x) # N: Revealed type is "builtins.int" + +if sys.platform == 'win32': + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) + +if sys.version_info == (2, 7): + reveal_type(x) +else: + reveal_type(x) # N: Revealed type is "builtins.int" + +if sys.version_info == (3, 8): + reveal_type(x) # N: Revealed type is "builtins.int" +else: + reveal_type(x) + +FOOBAR = "" +if FOOBAR: + reveal_type(x) +else: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/ops.pyi] +[typing fixtures/typing-medium.pyi] + +[case testUnreachableFlagIgnoresSemanticAnalysisExprUnreachable] +# flags: --warn-unreachable --always-false FOOBAR +import sys +from typing import TYPE_CHECKING + +FOOBAR = "" + +def foo() -> bool: ... + +lst = [1, 2, 3] + +a = FOOBAR and foo() +b = (not FOOBAR) or foo() +c = 1 if FOOBAR else 2 +d = [x for x in lst if FOOBAR] +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] + +[case testUnreachableFlagOkWithDeadStatements] +# flags: --warn-unreachable +from typing import NoReturn +def assert_never(x: NoReturn) -> NoReturn: + assert False + +def nonthrowing_assert_never(x: NoReturn) -> None: ... + +def expect_str(x: str) -> str: pass + +x: int +if False: + assert False + reveal_type(x) # E: Statement is unreachable + +if False: + raise Exception() + reveal_type(x) # E: Statement is unreachable + +if False: + assert_never(x) + reveal_type(x) # E: Statement is unreachable + +if False: + nonthrowing_assert_never(x) # E: Statement is unreachable + reveal_type(x) + +if False: + # Ignore obvious type errors + assert_never(expect_str(x)) + reveal_type(x) # E: Statement is unreachable +[builtins fixtures/exception.pyi] + +[case testNeverVariants] +from typing import Never +from typing_extensions import Never as TENever +from typing import NoReturn +from typing_extensions import NoReturn as TENoReturn +from mypy_extensions import NoReturn as MENoReturn + +bottom1: Never +reveal_type(bottom1) # N: Revealed type is "Never" +bottom2: TENever +reveal_type(bottom2) # N: Revealed type is "Never" +bottom3: NoReturn +reveal_type(bottom3) # N: Revealed type is "Never" +bottom4: TENoReturn +reveal_type(bottom4) # N: Revealed type is "Never" +bottom5: MENoReturn +reveal_type(bottom5) # N: Revealed type is "Never" + +[builtins fixtures/tuple.pyi] + +[case testUnreachableFlagExpressions] +# flags: --warn-unreachable +def foo() -> bool: ... + +lst = [1, 2, 3, 4] + +a = True or foo() # E: Right operand of "or" is never evaluated +b = 42 or False # E: Right operand of "or" is never evaluated +d = False and foo() # E: Right operand of "and" is never evaluated +e = True or (True or (True or foo())) # E: Right operand of "or" is never evaluated +f = (True or foo()) or (True or foo()) # E: Right operand of "or" is never evaluated + +k = [x for x in lst if isinstance(x, int) or foo()] # E: Right operand of "or" is never evaluated +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableFlagMiscTestCaseMissingMethod] +# flags: --warn-unreachable + +class Case1: + def test1(self) -> bool: + return False and self.missing() # E: Right operand of "and" is never evaluated + + def test2(self) -> bool: + return not self.property_decorator_missing and self.missing() # E: Function "property_decorator_missing" could always be true in boolean context \ + # E: Right operand of "and" is never evaluated + + def property_decorator_missing(self) -> bool: + return True +[builtins fixtures/bool.pyi] + +[case testUnreachableFlagWithGenerics] +# flags: --warn-unreachable +from typing import TypeVar, Generic + +T1 = TypeVar('T1', bound=int) +T2 = TypeVar('T2', int, str) +T3 = TypeVar('T3', None, str) + +def test1(x: T1) -> T1: + if isinstance(x, int): + reveal_type(x) # N: Revealed type is "T1`-1" + else: + reveal_type(x) # E: Statement is unreachable + return x + +def test2(x: T2) -> T2: + if isinstance(x, int): + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + + if False: + # This is unreachable, but we don't report an error, unfortunately. + # The presence of the TypeVar with values unfortunately currently shuts + # down type-checking for this entire function. + # TODO: Find a way of removing this limitation + reveal_type(x) + + return x + +class Test3(Generic[T2]): + x: T2 + + def func(self) -> None: + if isinstance(self.x, int): + reveal_type(self.x) # N: Revealed type is "builtins.int" + else: + reveal_type(self.x) # N: Revealed type is "builtins.str" + + if False: + # Same issue as above + reveal_type(self.x) + + +class Test4(Generic[T3]): + def __init__(self, x: T3): + # https://github.com/python/mypy/issues/9456 + # On TypeVars with value restrictions, we currently have no way + # of checking a statement for all the type expansions. + # Thus unreachable warnings are disabled + if x and False: + pass + # This test should fail after this limitation is removed. + if False and x: + pass + +[builtins fixtures/isinstancelist.pyi] + +[case testUnreachableBlockStaysUnreachableWithTypeVarConstraints] +# flags: --always-false COMPILE_TIME_FALSE +from typing import TypeVar +COMPILE_TIME_FALSE = False +T = TypeVar("T", int, str) +def foo(x: T) -> T: + if COMPILE_TIME_FALSE: + return "bad" + return x + +[case testUnreachableFlagContextManagersNoSuppress] +# flags: --warn-unreachable +from contextlib import contextmanager +from typing import Optional, Iterator, Any +from typing_extensions import Literal +class DoesNotSuppress1: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> Optional[bool]: ... + +class DoesNotSuppress2: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> Literal[False]: ... + +class DoesNotSuppress3: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> Any: ... + +class DoesNotSuppress4: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> None: ... + +@contextmanager +def simple() -> Iterator[int]: + yield 3 + +def cond() -> bool: ... + +def noop() -> None: ... + +def f_no_suppress_1a() -> int: + with DoesNotSuppress1(): + return 3 + noop() # E: Statement is unreachable + +def f_no_suppress_1b() -> int: + with DoesNotSuppress1(): + if cond(): + return 3 + else: + return 3 + noop() # E: Statement is unreachable + +def f_no_suppress_2() -> int: + with DoesNotSuppress2(): + return 3 + noop() # E: Statement is unreachable + +def f_no_suppress_3() -> int: + with DoesNotSuppress3(): + return 3 + noop() # E: Statement is unreachable + +def f_no_suppress_4() -> int: + with DoesNotSuppress4(): + return 3 + noop() # E: Statement is unreachable + +def f_no_suppress_5() -> int: + with simple(): + return 3 + noop() # E: Statement is unreachable + +[typing fixtures/typing-medium.pyi] +[builtins fixtures/tuple.pyi] + +[case testUnreachableFlagContextManagersSuppressed] +# flags: --warn-unreachable +from contextlib import contextmanager +from typing import Optional, Iterator, Any +from typing_extensions import Literal + +class DoesNotSuppress: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> Optional[bool]: ... + +class Suppresses1: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> bool: ... + +class Suppresses2: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> Literal[True]: ... + +def cond() -> bool: ... + +def noop() -> None: ... + +def f_suppress_1a() -> int: # E: Missing return statement + with Suppresses1(): + return 3 + noop() + +def f_suppress_1b() -> int: # E: Missing return statement + with Suppresses1(): + if cond(): + return 3 + else: + return 3 + noop() + +def f_suppress_2() -> int: # E: Missing return statement + with Suppresses2(): + return 3 + noop() + +def f_mix() -> int: # E: Missing return statement + with DoesNotSuppress(), Suppresses1(), DoesNotSuppress(): + return 3 + noop() +[typing fixtures/typing-medium.pyi] +[builtins fixtures/tuple.pyi] + +[case testUnreachableFlagContextManagersSuppressedNoStrictOptional] +# flags: --warn-unreachable --no-strict-optional +from contextlib import contextmanager +from typing import Optional, Iterator, Any +from typing_extensions import Literal + +class DoesNotSuppress1: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> Optional[bool]: ... + +# Normally, this should suppress. But when strict-optional mode is disabled, we can't +# necessarily distinguish between bool and Optional[bool]. So we default to assuming +# no suppression, since that's what most context managers will do. +class DoesNotSuppress2: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> bool: ... + +# But if we see Literal[True], it's pretty unlikely the return type is actually meant to +# be 'Optional[Literal[True]]'. So, we optimistically assume this is meant to be suppressing. +class Suppresses: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> Literal[True]: ... + +def noop() -> None: ... + +def f_no_suppress_1() -> int: + with DoesNotSuppress1(): + return 3 + noop() # E: Statement is unreachable + +def f_no_suppress_2() -> int: + with DoesNotSuppress1(): + return 3 + noop() # E: Statement is unreachable + +def f_suppress() -> int: # E: Missing return statement + with Suppresses(): + return 3 + noop() +[typing fixtures/typing-medium.pyi] +[builtins fixtures/tuple.pyi] + +[case testUnreachableFlagContextAsyncManagersNoSuppress] +# flags: --warn-unreachable +from contextlib import asynccontextmanager +from typing import Optional, AsyncIterator, Any +from typing_extensions import Literal + +class DoesNotSuppress1: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> Optional[bool]: ... + +class DoesNotSuppress2: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> Literal[False]: ... + +class DoesNotSuppress3: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> Any: ... + +class DoesNotSuppress4: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> None: ... + +@asynccontextmanager +async def simple() -> AsyncIterator[int]: + yield 3 + +def cond() -> bool: ... + +def noop() -> None: ... + +async def f_no_suppress_1a() -> int: + async with DoesNotSuppress1(): + return 3 + noop() # E: Statement is unreachable + +async def f_no_suppress_1b() -> int: + async with DoesNotSuppress1(): + if cond(): + return 3 + else: + return 3 + noop() # E: Statement is unreachable + +async def f_no_suppress_2() -> int: + async with DoesNotSuppress2(): + return 3 + noop() # E: Statement is unreachable + +async def f_no_suppress_3() -> int: + async with DoesNotSuppress3(): + return 3 + noop() # E: Statement is unreachable + +async def f_no_suppress_4() -> int: + async with DoesNotSuppress4(): + return 3 + noop() # E: Statement is unreachable + +async def f_no_suppress_5() -> int: + async with simple(): + return 3 + noop() # E: Statement is unreachable + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testUnreachableFlagContextAsyncManagersSuppressed] +# flags: --warn-unreachable +from contextlib import asynccontextmanager +from typing import Optional, AsyncIterator, Any +from typing_extensions import Literal + +class DoesNotSuppress: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> Optional[bool]: ... + +class Suppresses1: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> bool: ... + +class Suppresses2: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> Literal[True]: ... + +def cond() -> bool: ... + +def noop() -> None: ... + +async def f_suppress_1() -> int: # E: Missing return statement + async with Suppresses1(): + return 3 + noop() + +async def f_suppress_2() -> int: # E: Missing return statement + async with Suppresses1(): + if cond(): + return 3 + else: + return 3 + noop() + +async def f_suppress_3() -> int: # E: Missing return statement + async with Suppresses2(): + return 3 + noop() + +async def f_mix() -> int: # E: Missing return statement + async with DoesNotSuppress(), Suppresses1(), DoesNotSuppress(): + return 3 + noop() +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testUnreachableFlagContextAsyncManagersAbnormal] +# flags: --warn-unreachable +from contextlib import asynccontextmanager +from typing import Optional, AsyncIterator, Any +from typing_extensions import Literal + +class RegularManager: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> bool: ... + +class AsyncManager: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> bool: ... + +def noop() -> None: ... + +async def f_bad_1() -> int: + async with RegularManager(): # E: "RegularManager" has no attribute "__aenter__"; maybe "__enter__"? \ + # E: "RegularManager" has no attribute "__aexit__"; maybe "__exit__"? + return 3 + noop() # E: Statement is unreachable + +def f_bad_2() -> int: + with AsyncManager(): # E: "AsyncManager" has no attribute "__enter__"; maybe "__aenter__"? \ + # E: "AsyncManager" has no attribute "__exit__"; maybe "__aexit__"? + return 3 + noop() # E: Statement is unreachable + +# TODO: We should consider reporting an error when the user tries using +# context manager with malformed signatures instead of silently continuing. + +class RegularManagerMalformedSignature: + def __enter__(self) -> int: ... + def __exit__(self, exctype: object, excvalue: object, traceback: object) -> object: ... + +class AsyncManagerMalformedSignature: + async def __aenter__(self) -> int: ... + async def __aexit__(self, exctype: object, excvalue: object, traceback: object) -> object: ... + +def f_malformed_1() -> int: + with RegularManagerMalformedSignature(): + return 3 + noop() # E: Statement is unreachable + +async def f_malformed_2() -> int: + async with AsyncManagerMalformedSignature(): + return 3 + noop() # E: Statement is unreachable + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testUnreachableUntypedFunction] +# flags: --warn-unreachable + +def test_untyped_fn(obj): + assert obj.prop is True + + obj.update(prop=False) + obj.reload() + + assert obj.prop is False + reveal_type(obj.prop) + +def test_typed_fn(obj) -> None: + assert obj.prop is True + + obj.update(prop=False) + obj.reload() + + assert obj.prop is False + reveal_type(obj.prop) # E: Statement is unreachable + +[case testUnreachableCheckedUntypedFunction] +# flags: --warn-unreachable --check-untyped-defs + +def test_untyped_fn(obj): + assert obj.prop is True + + obj.update(prop=False) + obj.reload() + + assert obj.prop is False + reveal_type(obj.prop) # E: Statement is unreachable + +[case testConditionalTypeVarException] +# every part of this test case was necessary to trigger the crash +import sys +from typing import TypeVar + +T = TypeVar("T", int, str) + +def f(t: T) -> None: + if sys.platform == "lol": + try: + pass + except BaseException as e: + pass +[builtins fixtures/dict.pyi] + + +[case testUnreachableLiteral] +# flags: --warn-unreachable +from typing_extensions import Literal + +def nope() -> Literal[False]: ... + +def f() -> None: + if nope(): + x = 1 # E: Statement is unreachable +[builtins fixtures/dict.pyi] + +[case testUnreachableLiteralFrom__bool__] +# flags: --warn-unreachable +from typing_extensions import Literal + +class Truth: + def __bool__(self) -> Literal[True]: ... + +class Lie: + def __bool__(self) -> Literal[False]: ... + +class Maybe: + def __bool__(self) -> Literal[True | False]: ... + +t = Truth() +if t: + x = 1 +else: + x = 2 # E: Statement is unreachable + +if Lie(): + x = 3 # E: Statement is unreachable + +if Maybe(): + x = 4 + + +def foo() -> bool: ... + +y = Truth() or foo() # E: Right operand of "or" is never evaluated +z = Lie() and foo() # E: Right operand of "and" is never evaluated +[builtins fixtures/dict.pyi] + +[case testUnreachableModuleBody1] +# flags: --warn-unreachable +from typing import NoReturn +def foo() -> NoReturn: + raise Exception("foo") +foo() +x = 1 # E: Statement is unreachable +[builtins fixtures/exception.pyi] + +[case testUnreachableModuleBody2] +# flags: --warn-unreachable +raise Exception +x = 1 # E: Statement is unreachable +[builtins fixtures/exception.pyi] + +[case testUnreachableNoReturnBinaryOps] +# flags: --warn-unreachable +from typing import NoReturn + +a: NoReturn +a and 1 # E: Right operand of "and" is never evaluated +a or 1 # E: Right operand of "or" is never evaluated +a or a # E: Right operand of "or" is never evaluated +1 and a and 1 # E: Right operand of "and" is never evaluated +a and a # E: Right operand of "and" is never evaluated +[builtins fixtures/exception.pyi] + +[case testUnreachableFlagWithTerminalBranchInDeferredNode] +# flags: --warn-unreachable +from typing import NoReturn + +def assert_never(x: NoReturn) -> NoReturn: ... + +def force_forward_ref() -> int: + return 4 + +def f(value: None) -> None: + x + if value is not None: + assert_never(value) + +x = force_forward_ref() +[builtins fixtures/exception.pyi] + +[case testSetitemNoReturn] +# flags: --warn-unreachable +from typing import NoReturn +class Foo: + def __setitem__(self, key: str, value: str) -> NoReturn: + raise Exception +Foo()['a'] = 'a' +x = 0 # E: Statement is unreachable +[builtins fixtures/exception.pyi] + +[case TestNoImplicNoReturnFromError] +# flags: --warn-unreachable +from typing import TypeVar + +T = TypeVar("T") +class Foo: + def __setitem__(self, key: str, value: str) -> T: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + raise Exception + +def f() -> None: + Foo()['a'] = 'a' + x = 0 # This should not be reported as unreachable +[builtins fixtures/exception.pyi] + +[case testIntentionallyEmptyGeneratorFunction] +# flags: --warn-unreachable +from typing import Generator + +def f() -> Generator[None, None, None]: + return + yield + +[case testIntentionallyEmptyGeneratorFunction_None] +# flags: --warn-unreachable +from typing import Generator + +def f() -> Generator[None, None, None]: + return None + yield None + +[case testLambdaNoReturn] +# flags: --warn-unreachable +from typing import Callable, NoReturn + +def foo() -> NoReturn: + raise + +f1 = lambda: foo() +x = 0 # not unreachable + +f2: Callable[[], NoReturn] = lambda: foo() +x = 0 # not unreachable diff --git a/test-data/unit/check-unsupported.test b/test-data/unit/check-unsupported.test new file mode 100644 index 000000000000..f8de533dc5e1 --- /dev/null +++ b/test-data/unit/check-unsupported.test @@ -0,0 +1,17 @@ +-- Tests for unsupported features + + +[case testDecorateOverloadedFunction] +from foo import * +[file foo.pyi] +# The error messages are not the most informative ever. +def d(x): pass +@d +def f(): pass +def f(x): pass # E +def g(): pass +@d # E +def g(x): pass +[out] +tmp/foo.pyi:5: error: Name "f" already defined on line 3 +tmp/foo.pyi:7: error: Name "g" already defined on line 6 diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test new file mode 100644 index 000000000000..bb0e80acee1e --- /dev/null +++ b/test-data/unit/check-varargs.test @@ -0,0 +1,1082 @@ +-- Test cases for the type checker related to varargs. + + +-- Varargs within body +-- ------------------- + + +[case testVarArgsWithinFunction] +from typing import Tuple +def f( *b: 'B') -> None: + ab: Tuple[B, ...] + ac: Tuple[C, ...] + if int(): + b = ac # E: Incompatible types in assignment (expression has type "Tuple[C, ...]", variable has type "Tuple[B, ...]") + ac = b # E: Incompatible types in assignment (expression has type "Tuple[B, ...]", variable has type "Tuple[C, ...]") + b = ab + ab = b + +class B: pass +class C: pass +[builtins fixtures/tuple.pyi] +[out] + + +[case testVarArgsAreTuple] +from typing import Tuple, Sequence +def want_tuple(types: Tuple[type, ...]): pass +def want_sequence(types: Sequence[type]): pass +def test(*t: type) -> None: + want_tuple(t) + want_sequence(t) +[builtins fixtures/tuple.pyi] +[out] + + +-- Calling varargs function +-- ------------------------ + + +[case testCallingVarArgsFunction] +def f( *a: 'A') -> None: pass + +def g() -> None: pass + +class A: pass +class B(A): pass +class C: pass + +a: A +b: B +c: C + +f(c) # E: Argument 1 to "f" has incompatible type "C"; expected "A" +f(a, b, c) # E: Argument 3 to "f" has incompatible type "C"; expected "A" +f(g()) # E: "g" does not return a value (it only ever returns None) +f(a, g()) # E: "g" does not return a value (it only ever returns None) +f() +f(a) +f(b) +f(a, b, a, b) +[builtins fixtures/list.pyi] + +[case testCallingVarArgsFunctionWithAlsoNormalArgs] +def f(a: 'C', *b: 'A') -> None: pass + +class A: pass +class B(A): pass +class C: pass + +a: A +b: B +c: C + +f(a) # E: Argument 1 to "f" has incompatible type "A"; expected "C" +f(c, c) # E: Argument 2 to "f" has incompatible type "C"; expected "A" +f(c, a, b, c) # E: Argument 4 to "f" has incompatible type "C"; expected "A" +f(c) +f(c, a) +f(c, b, b, a, b) +[builtins fixtures/list.pyi] + +[case testCallingVarArgsFunctionWithDefaultArgs] +# flags: --implicit-optional --no-strict-optional +def f(a: 'C' = None, *b: 'A') -> None: + pass + +class A: pass +class B(A): pass +class C: pass + +a: A +b: B +c: C + +f(a) # E: Argument 1 to "f" has incompatible type "A"; expected "Optional[C]" +f(c, c) # E: Argument 2 to "f" has incompatible type "C"; expected "A" +f(c, a, b, c) # E: Argument 4 to "f" has incompatible type "C"; expected "A" +f() +f(c) +f(c, a) +f(c, b, b, a, b) +[builtins fixtures/list.pyi] + +[case testCallVarargsFunctionWithIterable] +from typing import Iterable +it1: Iterable[int] +it2: Iterable[str] +def f(*x: int) -> None: pass +f(*it1) +f(*it2) # E: Argument 1 to "f" has incompatible type "*Iterable[str]"; expected "int" +[builtins fixtures/for.pyi] + +[case testCallVarargsFunctionWithTwoTupleStarArgs] +from typing import TypeVar, Tuple + +T1 = TypeVar('T1') +T2 = TypeVar('T2') +T3 = TypeVar('T3') +T4 = TypeVar('T4') + +def f(a: T1, b: T2, c: T3, d: T4) -> Tuple[T1, T2, T3, T4]: ... +x: Tuple[int, str] +y: Tuple[float, bool] +reveal_type(f(*x, *y)) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.float, builtins.bool]" +[builtins fixtures/list.pyi] + +[case testCallVarargsFunctionWithIterableAndPositional] + +from typing import Iterable +it1: Iterable[int] +def f(*x: int) -> None: pass +f(*it1, 1, 2) +f(*it1, 1, *it1, 2) +f(*it1, '') # E: Argument 2 to "f" has incompatible type "str"; expected "int" +[builtins fixtures/for.pyi] + +[case testCallVarargsFunctionWithTupleAndPositional] + +def f(*x: int) -> None: pass +it1 = (1, 2) +it2 = ('',) +f(*it1, 1, 2) +f(*it1, 1, *it1, 2) +f(*it1, 1, *it2, 2) # E: Argument 3 to "f" has incompatible type "*Tuple[str]"; expected "int" +f(*it1, '') # E: Argument 2 to "f" has incompatible type "str"; expected "int" +[builtins fixtures/for.pyi] + + +-- Calling varargs function + type inference +-- ----------------------------------------- + + +[case testTypeInferenceWithCalleeVarArgs] +from typing import TypeVar +T = TypeVar('T') + +def f( *a: T) -> T: + pass + +class A: pass +class B(A): pass +class C: pass + +a: A +b: B +c: C +o: object + +if int(): + a = f(o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") +if int(): + b = f(b, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = f(a, b) # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +if int(): + o = f() +if int(): + a = f(a) +if int(): + a = f(b) +if int(): + a = f(a, b, a) +if int(): + o = f(a, b, o) +if int(): + c = f(c) +[builtins fixtures/list.pyi] + +[case testTypeInferenceWithCalleeVarArgsAndDefaultArgs] +# flags: --no-strict-optional +from typing import TypeVar +T = TypeVar('T') +a = None # type: A +o = None # type: object + +def f(a: T, b: T = None, *c: T) -> T: + pass + +class A: pass + +if int(): + a = f(o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") +if int(): + a = f(a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") +if int(): + a = f(a, a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") +if int(): + a = f(a, a, a, o) # E: Incompatible types in assignment (expression has type "object", variable has type "A") + +if int(): + a = f(a) +if int(): + a = f(a, a) +if int(): + a = f(a, a, a) +[builtins fixtures/list.pyi] + + +-- Calling normal function with varargs +-- ------------------------------------ + + +[case testCallingWithListVarArgs] +from typing import List, Any, cast + +def f(a: 'A', b: 'B') -> None: + pass + +class A: pass +class B: pass + +aa: List[A] +ab: List[B] +a: A +b: B + +f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" +f(a, *ab) # Ok +f(a, b) +(cast(Any, f))(*aa) # IDEA: Move to check-dynamic? +(cast(Any, f))(a, *ab) # IDEA: Move to check-dynamic? +[builtins fixtures/list.pyi] +[case testCallingWithTupleVarArgs] +def f(a: 'A', b: 'B', c: 'C') -> None: pass + +class A: pass +class B: pass +class C: pass +class CC(C): pass + +a: A +b: B +c: C +cc: CC + +f(*(a, b, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B, B]"; expected "C" +f(*(b, b, c)) # E: Argument 1 to "f" has incompatible type "*Tuple[B, B, C]"; expected "A" +f(a, *(b, b)) # E: Argument 2 to "f" has incompatible type "*Tuple[B, B]"; expected "C" +f(b, *(b, c)) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +f(*(a, b)) # E: Missing positional arguments "b", "c" in call to "f" +f(*(a, b, c, c)) # E: Too many arguments for "f" +f(a, *(b, c, c)) # E: Too many arguments for "f" +f(*(a, b, c)) +f(a, *(b, c)) +f(a, b, *(c,)) +f(a, *(b, cc)) +[builtins fixtures/tuple.pyi] + +[case testInvalidVarArg] +def f(a: 'A') -> None: + pass + +class A: pass + +a = A() + +f(*None) # E: Expected iterable as variadic argument +f(*a) # E: Expected iterable as variadic argument +f(*(a,)) + +f(*4) # E: Expected iterable as variadic argument +f(a, *4) # E: Expected iterable as variadic argument +[builtins fixtures/tuple.pyi] + + +-- Calling varargs function with varargs +-- ------------------------------------- + + +[case testCallingVarArgsFunctionWithListVarArgs] +from typing import List + +def f(a: 'A', *b: 'B') -> None: pass +def g(a: 'A', *b: 'A') -> None: pass +class A: pass +class B: pass + +aa: List[A] +ab: List[B] +a: A +b: B +f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" +f(a, *aa) # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "B" +f(b, *ab) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +f(a, a, *ab) # E: Argument 2 to "f" has incompatible type "A"; expected "B" +f(a, b, *aa) # E: Argument 3 to "f" has incompatible type "*List[A]"; expected "B" +f(b, b, *ab) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +g(*ab) # E: Argument 1 to "g" has incompatible type "*List[B]"; expected "A" +f(a, *ab) +f(a, b, *ab) +f(a, b, b, *ab) +g(*aa) +[builtins fixtures/list.pyi] +[case testCallingVarArgsFunctionWithTupleVarArgs] +def f(a: 'A', *b: 'B') -> None: + pass + +class A: pass +class B: pass +class C: pass +class CC(C): pass + +a: A +b: B +c: C +cc: CC + +f(*(b, b, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[B, B, B]"; expected "A" +f(*(a, a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, A, B]"; expected "B" +f(*(a, b, a)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B, A]"; expected "B" +f(a, *(a, b)) # E: Argument 2 to "f" has incompatible type "*Tuple[A, B]"; expected "B" +f(b, *(b, b)) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +f(b, b, *(b,)) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +f(a, a, *(b,)) # E: Argument 2 to "f" has incompatible type "A"; expected "B" +f(a, b, *(a,)) # E: Argument 3 to "f" has incompatible type "*Tuple[A]"; expected "B" +f(*()) # E: Too few arguments for "f" +f(*(a, b, b)) +f(a, *(b, b)) +f(a, b, *(b,)) +[builtins fixtures/list.pyi] + + +-- Varargs special cases +-- --------------------- + + +[case testDynamicVarArg] +from typing import Any +def f(a: 'A') -> None: pass +def g(a: 'A', *b: 'A') -> None: pass +class A: pass + +d: Any +a: A +f(a, a, *d) # E: Too many arguments for "f" +f(a, *d) # Ok +f(*d) # Ok + +g(*d) +g(a, *d) +g(a, a, *d) +[builtins fixtures/list.pyi] + +[case testListVarArgsAndSubtyping] +from typing import List +def f( *a: 'A') -> None: + pass + +def g( *a: 'B') -> None: + pass + +class A: pass +class B(A): pass + +aa: List[A] +ab: List[B] + +g(*aa) # E: Argument 1 to "g" has incompatible type "*List[A]"; expected "B" +f(*aa) +f(*ab) +g(*ab) +[builtins fixtures/list.pyi] + +[case testCallerVarArgsAndDefaultArgs] +# flags: --implicit-optional --no-strict-optional + +def f(a: 'A', b: 'B' = None, *c: 'B') -> None: + pass + +class A: pass +class B: pass + +a, b = None, None # type: (A, B) +f(*()) # E: Too few arguments for "f" +f(a, *[a]) # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "Optional[B]" \ + # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "B" +f(a, b, *[a]) # E: Argument 3 to "f" has incompatible type "*List[A]"; expected "B" +f(*(a, a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, A, B]"; expected "Optional[B]" +f(*(a,)) +f(*(a, b)) +f(*(a, b, b, b)) +f(a, *[]) +f(a, *[b]) +f(a, *[b, b]) +[builtins fixtures/list.pyi] + +[case testVarArgsAfterKeywordArgInCall1] +# see: mypy issue #2729 +def f(x: int, y: str) -> None: pass +f(x=1, *[2]) +[builtins fixtures/list.pyi] +[out] +main:3: error: "f" gets multiple values for keyword argument "x" +main:3: error: Argument 1 to "f" has incompatible type "*List[int]"; expected "str" + +[case testVarArgsAfterKeywordArgInCall2] +# see: mypy issue #2729 +def f(x: int, y: str) -> None: pass +f(y='x', *[1]) +[builtins fixtures/list.pyi] +[out] +main:3: error: "f" gets multiple values for keyword argument "y" +main:3: error: Argument 1 to "f" has incompatible type "*List[int]"; expected "str" + +[case testVarArgsAfterKeywordArgInCall3] +def f(x: int, y: str) -> None: pass +f(y='x', *(1,)) +[builtins fixtures/list.pyi] + +[case testVarArgsAfterKeywordArgInCall4] +def f(x: int, *, y: str) -> None: pass +f(y='x', *[1]) +[builtins fixtures/list.pyi] + +[case testVarArgsAfterKeywordArgInCall5] +def f(x: int, *, y: str) -> None: pass +f(y='x', *(1,)) +[builtins fixtures/list.pyi] + +[case testVarArgsEmptyList] +from typing import List + +def foo() -> None: + pass + +lst: List[int] = [] +foo(*lst) +[builtins fixtures/list.pyi] + +[case testVarArgsEmptyTuple] +def foo() -> None: + pass + +foo(*()) +[builtins fixtures/tuple.pyi] + +-- Overloads + varargs +-- ------------------- + + +[case testIntersectionTypesAndVarArgs] +# flags: --no-strict-optional +from foo import * +[file foo.pyi] +from typing import overload +a, b = None, None # type: (A, B) + +if int(): + b = f() # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = f(a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = f(a, b) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = f(b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = f(b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + b = f(a, *[b]) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = f(*()) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = f(*(a,)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + b = f(*(a, b)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +if int(): + a = f(*(b,)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = f(*(b, b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +if int(): + a = f(*[b]) # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +a = f() +a = f(a) +a = f(a, b) +b = f(b) +b = f(b, b) +a = f(a, *[b]) +a = f(*()) +a = f(*(a,)) +a = f(*(a, b)) +b = f(*(b,)) +b = f(*(b, b)) +b = f(*[b]) + +class A: pass +class B: pass + +@overload +def f(a: A = None, *b: B) -> A: pass + +@overload +def f(a: B, *b: B) -> B: pass +[builtins fixtures/list.pyi] + + +-- Caller varargs + type inference +-- ------------------------------- + + +[case testCallerVarArgsListWithTypeInference] +from typing import List, TypeVar, Tuple +S = TypeVar('S') +T = TypeVar('T') + +def f(a: S, *b: T) -> Tuple[S, T]: + pass + +class A: pass +class B: pass + +a: A +b: B +aa: List[A] + +if int(): + a, b = f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" +if int(): + b, b = f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" +if int(): + a, a = f(b, *aa) # E: Argument 1 to "f" has incompatible type "B"; expected "A" +if int(): + b, b = f(b, *aa) # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "B" +if int(): + b, b = f(b, b, *aa) # E: Argument 3 to "f" has incompatible type "*List[A]"; expected "B" +if int(): + a, b = f(a, *a) # E: Expected iterable as variadic argument +if int(): + a, b = f(*a) # E: Expected iterable as variadic argument + +if int(): + a, a = f(*aa) +if int(): + b, a = f(b, *aa) +if int(): + b, a = f(b, a, *aa) +[builtins fixtures/list.pyi] + +[case testCallerVarArgsTupleWithTypeInference] +from typing import TypeVar, Tuple +S = TypeVar('S') +T = TypeVar('T') + +def f(a: S, b: T) -> Tuple[S, T]: pass + +class A: pass +class B: pass + +a: A +b: B + +if int(): + a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B]"; expected "A" +if int(): + b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" +if int(): + a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B]"; expected "A" +if int(): + b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" +if int(): + a, b = f(*(a, b, b)) # E: Too many arguments for "f" +if int(): + a, b = f(*(a, b)) +if int(): + a, b = f(a, *(b,)) +[builtins fixtures/list.pyi] + +[case testCallerVarargsAndComplexTypeInference] +from typing import List, TypeVar, Generic, Tuple +T = TypeVar('T') +S = TypeVar('S') +a: A +b: B +ao: List[object] +aa: List[A] +ab: List[B] + +class G(Generic[T]): + def f(self, *a: S) -> Tuple[List[S], List[T]]: + pass + +class A: pass +class B: pass + +if int(): + a, aa = G().f(*[a]) # E: Incompatible types in assignment (expression has type "List[A]", variable has type "A") +if int(): + aa, a = G().f(*[a]) # E: Incompatible types in assignment (expression has type "List[Never]", variable has type "A") +if int(): + ab, aa = G().f(*[a]) # E: Argument 1 to "f" of "G" has incompatible type "*List[A]"; expected "B" +if int(): + ao, ao = G().f(*[a]) +if int(): + aa, aa = G().f(*[a]) +[builtins fixtures/list.pyi] + +[case testCallerTupleVarArgsAndGenericCalleeVarArg] +from typing import TypeVar + +T = TypeVar('T') + +def f(*args: T) -> T: ... +reveal_type(f(*(1, None))) # N: Revealed type is "Union[Literal[1]?, None]" +reveal_type(f(1, *(None, 1))) # N: Revealed type is "Union[Literal[1]?, None]" +reveal_type(f(1, *(1, None))) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/tuple.pyi] + + +-- Comment signatures +-- ------------------ + + +[case testVarArgsAndCommentSignature] +import typing +def f(*x): # type: (*int) -> None + pass +f(1) +f(1, 2) +f('') # E: Argument 1 to "f" has incompatible type "str"; expected "int" +f(1, '') # E: Argument 2 to "f" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + + +-- Subtyping +-- --------- + + +[case testVarArgsFunctionSubtyping] +from typing import Callable +x: Callable[[int], None] +def f(*x: int) -> None: pass +def g(*x: str) -> None: pass +x = f +x = g # E: Incompatible types in assignment (expression has type "Callable[[VarArg(str)], None]", variable has type "Callable[[int], None]") +[builtins fixtures/list.pyi] +[out] + + +-- Decorated method where self is implied by *args +-- ----------------------------------------------- + +[case testVarArgsCallableSelf] +from typing import Callable +def cm(func) -> Callable[..., None]: pass +class C: + @cm + def foo(self) -> None: pass +C().foo() +C().foo(1) # The decorator's return type says this should be okay + +[case testInvariantDictArgNote] +from typing import Dict, Sequence +def f(x: Dict[str, Sequence[int]]) -> None: pass +def g(x: Dict[str, float]) -> None: pass +def h(x: Dict[str, int]) -> None: pass +a = {'a': [1, 2]} +b = {'b': ['c', 'd']} +c = {'c': 1.0} +d = {'d': 1} +f(a) # E: Argument 1 to "f" has incompatible type "Dict[str, List[int]]"; expected "Dict[str, Sequence[int]]" \ + # N: "Dict" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Mapping" instead, which is covariant in the value type +f(b) # E: Argument 1 to "f" has incompatible type "Dict[str, List[str]]"; expected "Dict[str, Sequence[int]]" +g(c) +g(d) # E: Argument 1 to "g" has incompatible type "Dict[str, int]"; expected "Dict[str, float]" \ + # N: "Dict" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Mapping" instead, which is covariant in the value type +h(c) # E: Argument 1 to "h" has incompatible type "Dict[str, float]"; expected "Dict[str, int]" +h(d) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-medium.pyi] + +[case testInvariantListArgNote] +from typing import List, Union +def f(numbers: List[Union[int, float]]) -> None: pass +a = [1, 2] +f(a) # E: Argument 1 to "f" has incompatible type "List[int]"; expected "List[Union[int, float]]" \ + # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ + # N: Consider using "Sequence" instead, which is covariant +x = [1] +y = ['a'] +if int(): + x = y # E: Incompatible types in assignment (expression has type "List[str]", variable has type "List[int]") +[builtins fixtures/list.pyi] + +[case testInvariantTypeConfusingNames] +from typing import TypeVar +class Listener: pass +class DictReader: pass +def f(x: Listener) -> None: pass +def g(y: DictReader) -> None: pass +a = [1, 2] +b = {'b': 1} +f(a) # E: Argument 1 to "f" has incompatible type "List[int]"; expected "Listener" +g(b) # E: Argument 1 to "g" has incompatible type "Dict[str, int]"; expected "DictReader" +[builtins fixtures/dict.pyi] + +[case testInvariantTypeConfusingNames2] +from typing import Iterable, Generic, TypeVar, List + +T = TypeVar('T') + +class I(Iterable[T]): + ... + +class Bad(Generic[T]): + ... + +def bar(*args: float) -> float: + ... + +good1: Iterable[float] +good2: List[float] +good3: I[float] +bad1: I[str] +bad2: Bad[float] +bar(*good1) +bar(*good2) +bar(*good3) +bar(*bad1) # E: Argument 1 to "bar" has incompatible type "*I[str]"; expected "float" +bar(*bad2) # E: Expected iterable as variadic argument +[builtins fixtures/dict.pyi] + +-- Keyword arguments unpacking + +[case testUnpackKwargsReveal] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int +def foo(arg: bool, **kwargs: Unpack[Person]) -> None: ... + +reveal_type(foo) # N: Revealed type is "def (arg: builtins.bool, **kwargs: Unpack[TypedDict('__main__.Person', {'name': builtins.str, 'age': builtins.int})])" +[builtins fixtures/dict.pyi] + +[case testUnpackOutsideOfKwargs] +from typing_extensions import Unpack, TypedDict +class Person(TypedDict): + name: str + age: int + +def foo(x: Unpack[Person]) -> None: # E: Unpack is only valid in a variadic position + ... +def bar(x: int, *args: Unpack[Person]) -> None: # E: "Person" cannot be unpacked (must be tuple or TypeVarTuple) + ... +def baz(**kwargs: Unpack[Person]) -> None: # OK + ... +[builtins fixtures/dict.pyi] + +[case testUnpackWithoutTypedDict] +from typing_extensions import Unpack + +def foo(**kwargs: Unpack[dict]) -> None: # E: Unpack item in ** argument must be a TypedDict + ... +[builtins fixtures/dict.pyi] + +[case testUnpackWithDuplicateKeywords] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int +def foo(name: str, **kwargs: Unpack[Person]) -> None: # E: Overlap between argument names and ** TypedDict items: "name" + ... +[builtins fixtures/dict.pyi] + +[case testUnpackWithDuplicateKeywordKwargs] +from typing_extensions import Unpack, TypedDict +from typing import Dict, List + +class Spec(TypedDict): + args: List[int] + kwargs: Dict[int, int] +def foo(**kwargs: Unpack[Spec]) -> None: # Allowed + ... +foo(args=[1], kwargs={"2": 3}) # E: Dict entry 0 has incompatible type "str": "int"; expected "int": "int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsNonIdentifier] +from typing_extensions import Unpack, TypedDict + +Weird = TypedDict("Weird", {"@": int}) + +def foo(**kwargs: Unpack[Weird]) -> None: + reveal_type(kwargs["@"]) # N: Revealed type is "builtins.int" +foo(**{"@": 42}) +foo(**{"no": "way"}) # E: Argument 1 to "foo" has incompatible type "**Dict[str, str]"; expected "int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsEmpty] +from typing_extensions import Unpack, TypedDict + +Empty = TypedDict("Empty", {}) + +def foo(**kwargs: Unpack[Empty]) -> None: # N: "foo" defined here + reveal_type(kwargs) # N: Revealed type is "TypedDict('__main__.Empty', {})" +foo() +foo(x=1) # E: Unexpected keyword argument "x" for "foo" +[builtins fixtures/dict.pyi] + +[case testUnpackTypedDictTotality] +from typing_extensions import Unpack, TypedDict + +class Circle(TypedDict, total=True): + radius: int + color: str + x: int + y: int + +def foo(**kwargs: Unpack[Circle]): + ... +foo(x=0, y=0, color='orange') # E: Missing named argument "radius" for "foo" + +class Square(TypedDict, total=False): + side: int + color: str + +def bar(**kwargs: Unpack[Square]): + ... +bar(side=12) +[builtins fixtures/dict.pyi] + +[case testUnpackUnexpectedKeyword] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict, total=False): + name: str + age: int + +def foo(**kwargs: Unpack[Person]) -> None: # N: "foo" defined here + ... +foo(name='John', age=42, department='Sales') # E: Unexpected keyword argument "department" for "foo" +foo(name='Jennifer', age=38) +[builtins fixtures/dict.pyi] + +[case testUnpackKeywordTypes] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]): + ... +foo(name='John', age='42') # E: Argument "age" to "foo" has incompatible type "str"; expected "int" +foo(name='Jennifer', age=38) +[builtins fixtures/dict.pyi] + +[case testUnpackKeywordTypesTypedDict] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class LegacyPerson(TypedDict): + name: str + age: str + +def foo(**kwargs: Unpack[Person]) -> None: + ... +lp = LegacyPerson(name="test", age="42") +foo(**lp) # E: Argument "age" to "foo" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testFunctionBodyWithUnpackedKwargs] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]) -> int: + name: str = kwargs['name'] + age: str = kwargs['age'] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + department: str = kwargs['department'] # E: TypedDict "Person" has no key "department" + return kwargs['age'] +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsOverrides] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class Base: + def foo(self, **kwargs: Unpack[Person]) -> None: ... +class SubGood(Base): + def foo(self, *, name: str, age: int, extra: bool = False) -> None: ... +class SubBad(Base): + def foo(self, *, name: str, age: str) -> None: ... # E: Argument 2 of "foo" is incompatible with supertype "Base"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsOverridesTypedDict] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class PersonExtra(Person, total=False): + extra: bool + +class Unrelated(TypedDict): + baz: int + +class Base: + def foo(self, **kwargs: Unpack[Person]) -> None: ... +class SubGood(Base): + def foo(self, **kwargs: Unpack[PersonExtra]) -> None: ... +class SubBad(Base): + def foo(self, **kwargs: Unpack[Unrelated]) -> None: ... # E: Signature of "foo" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: def foo(*, name: str, age: int) -> None \ + # N: Subclass: \ + # N: def foo(self, *, baz: int) -> None +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsGeneric] +from typing import Generic, TypeVar +from typing_extensions import Unpack, TypedDict + +T = TypeVar("T") +class Person(TypedDict, Generic[T]): + name: str + value: T + +def foo(**kwargs: Unpack[Person[T]]) -> T: ... +reveal_type(foo(name="test", value=42)) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsInference] +from typing import Generic, TypeVar, Protocol +from typing_extensions import Unpack, TypedDict + +T_contra = TypeVar("T_contra", contravariant=True) +class CBPerson(Protocol[T_contra]): + def __call__(self, **kwargs: Unpack[Person[T_contra]]) -> None: ... + +T = TypeVar("T") +class Person(TypedDict, Generic[T]): + name: str + value: T + +def test(cb: CBPerson[T]) -> T: ... + +def foo(*, name: str, value: int) -> None: ... +reveal_type(test(foo)) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsOverload] +from typing import Any, overload +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class Fruit(TypedDict): + sort: str + taste: int + +@overload +def foo(**kwargs: Unpack[Person]) -> int: ... +@overload +def foo(**kwargs: Unpack[Fruit]) -> str: ... +def foo(**kwargs: Any) -> Any: + ... + +reveal_type(foo(sort="test", taste=999)) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsJoin] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(*, name: str, age: int) -> None: ... +def bar(**kwargs: Unpack[Person]) -> None: ... + +reveal_type([foo, bar]) # N: Revealed type is "builtins.list[def (*, name: builtins.str, age: builtins.int)]" +reveal_type([bar, foo]) # N: Revealed type is "builtins.list[def (*, name: builtins.str, age: builtins.int)]" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsParamSpec] +from typing import Callable, Any, TypeVar, List +from typing_extensions import ParamSpec, Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +P = ParamSpec('P') +T = TypeVar('T') + +def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... + +@dec +def g(**kwargs: Unpack[Person]) -> int: ... + +reveal_type(g) # N: Revealed type is "def (*, name: builtins.str, age: builtins.int) -> builtins.list[builtins.int]" +[builtins fixtures/dict.pyi] + +[case testUnpackGenericTypedDictImplicitAnyEnabled] +from typing import Generic, TypeVar +from typing_extensions import Unpack, TypedDict + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + key: str + value: T + +def foo(**kwds: Unpack[TD]) -> None: ... # Same as `TD[Any]` +foo(key="yes", value=42) +foo(key="yes", value="ok") +[builtins fixtures/dict.pyi] + +[case testUnpackGenericTypedDictImplicitAnyDisabled] +# flags: --disallow-any-generics +from typing import Generic, TypeVar +from typing_extensions import Unpack, TypedDict + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + key: str + value: T + +def foo(**kwds: Unpack[TD]) -> None: ... # E: Missing type parameters for generic type "TD" +foo(key="yes", value=42) +foo(key="yes", value="ok") +[builtins fixtures/dict.pyi] + +[case testUnpackNoCrashOnEmpty] +from typing_extensions import Unpack + +class C: + def __init__(self, **kwds: Unpack) -> None: ... # E: Unpack[...] requires exactly one type argument +class D: + def __init__(self, **kwds: Unpack[int, str]) -> None: ... # E: Unpack[...] requires exactly one type argument +[builtins fixtures/dict.pyi] + +[case testUnpackInCallableType] +from typing import Callable +from typing_extensions import Unpack, TypedDict + +class TD(TypedDict): + key: str + value: str + +foo: Callable[[Unpack[TD]], None] +foo(key="yes", value=42) # E: Argument "value" has incompatible type "int"; expected "str" +foo(key="yes", value="ok") + +bad: Callable[[*TD], None] # E: "TD" cannot be unpacked (must be tuple or TypeVarTuple) +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-warnings.test b/test-data/unit/check-warnings.test new file mode 100644 index 000000000000..90f40777d6b7 --- /dev/null +++ b/test-data/unit/check-warnings.test @@ -0,0 +1,236 @@ +-- Test cases for warning generation. + +-- Redundant casts +-- --------------- + +[case testRedundantCast] +# flags: --warn-redundant-casts +from typing import cast +a = 1 +b = cast(str, a) +c = cast(int, a) +[out] +main:5: error: Redundant cast to "int" + +[case testRedundantCastWithIsinstance] +# flags: --warn-redundant-casts +from typing import cast, Union +x = 1 # type: Union[int, str] +if isinstance(x, str): + cast(str, x) +[builtins fixtures/isinstance.pyi] +[out] +main:5: error: Redundant cast to "str" + +[case testCastToSuperclassNotRedundant] +# flags: --warn-redundant-casts +from typing import cast, TypeVar, List +T = TypeVar('T') +def add(xs: List[T], ys: List[T]) -> List[T]: pass +class A: pass +class B(A): pass +a = A() +b = B() +# Without the cast, the following line would fail to type check. +c = add([cast(A, b)], [a]) +[builtins fixtures/list.pyi] + +[case testCastToAnyTypeNotRedundant] +# flags: --warn-redundant-casts +from typing import cast, Any +a: Any +b = cast(Any, a) +[builtins fixtures/list.pyi] + +-- Unused 'type: ignore' comments +-- ------------------------------ + +[case testUnusedTypeIgnore] +# flags: --warn-unused-ignores +a = 1 +if int(): + a = 'a' # type: ignore +if int(): + a = 2 # type: ignore # E: Unused "type: ignore" comment +if int(): + a = 'b' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testUnusedTypeIgnoreImport] +# flags: --warn-unused-ignores +import banana # type: ignore +import m # type: ignore +from m import * # type: ignore +[file m.py] +pass +[out] +main:3: error: Unused "type: ignore" comment +main:4: error: Unused "type: ignore" comment + + +-- No return +-- --------- + +[case testNoReturn] +# flags: --warn-no-return +def f() -> int: + pass + +def g() -> int: + if bool(): + return 1 +[builtins fixtures/list.pyi] +[out] +main:5: error: Missing return statement + +[case testNoReturnWhile] +# flags: --warn-no-return +def h() -> int: + while True: + if bool(): + return 1 + +def i() -> int: + while 1: + if bool(): + return 1 + if bool(): + break + +def j() -> int: + while 1: + if bool(): + return 1 + if bool(): + continue +[builtins fixtures/list.pyi] +[out] +main:7: error: Missing return statement + +[case testNoReturnExcept] +# flags: --warn-no-return +def f() -> int: + try: + return 1 + except: + pass +def g() -> int: + try: + pass + except: + return 1 + else: + return 1 +def h() -> int: + try: + pass + except: + pass + else: + pass + finally: + return 1 +[builtins fixtures/exception.pyi] +[out] +main:2: error: Missing return statement + +[case testNoReturnEmptyBodyWithDocstring] +def f() -> int: + """Return the number of peppers.""" + # This might be an @abstractmethod, for example + pass +[out] + + +-- Returning Any +-- ------------- + +[case testReturnAnyFromTypedFunction] +# flags: --warn-return-any +from typing import Any +def g() -> Any: pass +def f() -> int: return g() +[out] +main:4: error: Returning Any from function declared to return "int" + +[case testReturnAnyForNotImplementedInBinaryMagicMethods] +# flags: --warn-return-any +class A: + def __eq__(self, other: object) -> bool: return NotImplemented +[builtins fixtures/notimplemented.pyi] +[out] + +[case testReturnAnyForNotImplementedInNormalMethods] +# flags: --warn-return-any +class A: + def some(self) -> bool: return NotImplemented +[builtins fixtures/notimplemented.pyi] +[out] +main:3: error: Returning Any from function declared to return "bool" + +[case testReturnAnyFromTypedFunctionWithSpecificFormatting] +# flags: --warn-return-any +from typing import Any, Tuple +typ = Tuple[int, int, int, int, int, int, int, int, int, int, int, int, int, + int, int, int, int, int, int, int, int, int, int, int, int, int, + int, int, int, int, int, int, int, int, int, int, int, int, int, + int, int, int, int, int, int, int, int, int, int, int, int, int, + int, int, int, int, int, int, int, int, int, int, int, int, int, + int, int, int, int, int, int, int, int, int, int, int, int, int, + int, int, int, int, int, int, int, int, int, int, int, int, int] +def g() -> Any: pass +def f() -> typ: return g() +[builtins fixtures/tuple.pyi] +[out] +main:11: error: Returning Any from function declared to return "Tuple[int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int]" + +[case testReturnAnySilencedFromTypedFunction] +# flags: --warn-return-any +from typing import Any +def g() -> Any: pass +def f() -> int: + result = g() # type: int + return result +[out] + +[case testReturnAnyFromUntypedFunction] +# flags: --warn-return-any +from typing import Any +def g() -> Any: pass +def f(): return g() +[out] + +[case testReturnAnyFromAnyTypedFunction] +# flags: --warn-return-any +from typing import Any +def g() -> Any: pass +def f() -> Any: return g() +[out] + +[case testOKReturnAnyIfProperSubtype] +# flags: --warn-return-any +from typing import Any, Optional + +class Test(object): + + def __init__(self) -> None: + self.attr = "foo" # type: Any + + def foo(self, do_it: bool) -> Optional[Any]: + if do_it: + return self.attr # Should not warn here + else: + return None +[builtins fixtures/list.pyi] +[out] + +[case testReturnAnyDeferred] +# flags: --warn-return-any +def foo(a1: A) -> int: + if a1._x: + return 1 + n = 1 + return n + +class A: + def __init__(self, x: int) -> None: + self._x = x diff --git a/test-data/unit/cmdline.pyproject.test b/test-data/unit/cmdline.pyproject.test new file mode 100644 index 000000000000..831bce2eb63d --- /dev/null +++ b/test-data/unit/cmdline.pyproject.test @@ -0,0 +1,135 @@ +-- Tests for command line parsing +-- ------------------------------ +-- +-- The initial line specifies the command line, in the format +-- +-- # cmd: mypy +-- +-- Note that # flags: --some-flag IS NOT SUPPORTED. +-- Use # cmd: mypy --some-flag ... +-- +-- '== Return code: ' is added to the output when the process return code +-- is "nonobvious" -- that is, when it is something other than 0 if there are no +-- messages and 1 if there are. + +-- Directories/packages on the command line +-- ---------------------------------------- + +[case testNonArrayOverridesPyprojectTOML] +# cmd: mypy x.py +[file pyproject.toml] +\[tool.mypy] +\[tool.mypy.overrides] +module = "x" +disallow_untyped_defs = false +[file x.py] +def f(a): + pass +def g(a: int) -> int: + return f(a) +[out] +pyproject.toml: tool.mypy.overrides sections must be an array. Please make sure you are using double brackets like so: [[tool.mypy.overrides]] +== Return code: 0 + +[case testNoModuleInOverridePyprojectTOML] +# cmd: mypy x.py +[file pyproject.toml] +\[tool.mypy] +\[[tool.mypy.overrides]] +disallow_untyped_defs = false +[file x.py] +def f(a): + pass +def g(a: int) -> int: + return f(a) +[out] +pyproject.toml: toml config file contains a [[tool.mypy.overrides]] section, but no module to override was specified. +== Return code: 0 + +[case testInvalidModuleInOverridePyprojectTOML] +# cmd: mypy x.py +[file pyproject.toml] +\[tool.mypy] +\[[tool.mypy.overrides]] +module = 0 +disallow_untyped_defs = false +[file x.py] +def f(a): + pass +def g(a: int) -> int: + return f(a) +[out] +pyproject.toml: toml config file contains a [[tool.mypy.overrides]] section with a module value that is not a string or a list of strings +== Return code: 0 + +[case testConflictingModuleInOverridesPyprojectTOML] +# cmd: mypy x.py +[file pyproject.toml] +\[tool.mypy] +\[[tool.mypy.overrides]] +module = 'x' +disallow_untyped_defs = false +\[[tool.mypy.overrides]] +module = ['x'] +disallow_untyped_defs = true +[file x.py] +def f(a): + pass +def g(a: int) -> int: + return f(a) +[out] +pyproject.toml: toml config file contains [[tool.mypy.overrides]] sections with conflicting values. Module 'x' has two different values for 'disallow_untyped_defs' +== Return code: 0 + +[case testMultilineLiteralExcludePyprojectTOML] +# cmd: mypy x +[file pyproject.toml] +\[tool.mypy] +exclude = '''(?x)( + (^|/)[^/]*skipme_\.py$ + |(^|/)_skipme[^/]*\.py$ +)''' +[file x/__init__.py] +i: int = 0 +[file x/_skipme_please.py] +This isn't even syntatically valid! +[file x/please_skipme_.py] +Neither is this! + +[case testMultilineBasicExcludePyprojectTOML] +# cmd: mypy x +[file pyproject.toml] +\[tool.mypy] +exclude = """(?x)( + (^|/)[^/]*skipme_\\.py$ + |(^|/)_skipme[^/]*\\.py$ +)""" +[file x/__init__.py] +i: int = 0 +[file x/_skipme_please.py] +This isn't even syntatically valid! +[file x/please_skipme_.py] +Neither is this! + +[case testSequenceExcludePyprojectTOML] +# cmd: mypy x +[file pyproject.toml] +\[tool.mypy] +exclude = [ + '(^|/)[^/]*skipme_\.py$', # literal (no escaping) + "(^|/)_skipme[^/]*\\.py$", # basic (backslash needs escaping) +] +[file x/__init__.py] +i: int = 0 +[file x/_skipme_please.py] +This isn't even syntatically valid! +[file x/please_skipme_.py] +Neither is this! + +[case testPyprojectTOMLUnicode] +# cmd: mypy x.py +[file pyproject.toml] +\[project] +description = "Factory ⸻ A code generator 🏭" +\[tool.mypy] +[file x.py] diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test new file mode 100644 index 000000000000..2bab19e0d42f --- /dev/null +++ b/test-data/unit/cmdline.test @@ -0,0 +1,1337 @@ +-- Tests for command line parsing +-- ------------------------------ +-- +-- The initial line specifies the command line, in the format +-- +-- # cmd: mypy +-- +-- Note that # flags: --some-flag IS NOT SUPPORTED. +-- Use # cmd: mypy --some-flag ... +-- +-- '== Return code: ' is added to the output when the process return code +-- is "nonobvious" -- that is, when it is something other than 0 if there are no +-- messages and 1 if there are. + +-- Directories/packages on the command line +-- ---------------------------------------- + +[case testCmdlinePackage] +# cmd: mypy pkg +[file pkg/__init__.py] +[file pkg/a.py] +undef +[file pkg/subpkg/__init__.py] +[file pkg/subpkg/a.py] +undef +import pkg.subpkg.a +[out] +pkg/a.py:1: error: Name "undef" is not defined +pkg/subpkg/a.py:1: error: Name "undef" is not defined + +[case testCmdlinePackageSlash] +# cmd: mypy pkg/ +[file pkg/__init__.py] +[file pkg/a.py] +undef +[file pkg/subpkg/__init__.py] +[file pkg/subpkg/a.py] +undef +import pkg.subpkg.a +[out] +pkg/a.py:1: error: Name "undef" is not defined +pkg/subpkg/a.py:1: error: Name "undef" is not defined + +[case testCmdlineNonPackage] +# cmd: mypy dir +[file dir/a.py] +undef +[file dir/subdir/b.py] +undef +[out] +dir/a.py:1: error: Name "undef" is not defined +dir/subdir/b.py:1: error: Name "undef" is not defined + +[case testCmdlineNonPackageDuplicate] +# cmd: mypy dir +[file dir/a.py] +undef +[file dir/subdir/a.py] +undef +[out] +dir/a.py: error: Duplicate module named "a" (also at "dir/subdir/a.py") +dir/a.py: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules for more info +dir/a.py: note: Common resolutions include: a) using `--exclude` to avoid checking one of them, b) adding `__init__.py` somewhere, c) using `--explicit-package-bases` or adjusting MYPYPATH +== Return code: 2 + +[case testCmdlineNonPackageSlash] +# cmd: mypy dir/ +[file dir/a.py] +undef +import b +[file dir/subdir/b.py] +undef +import a +[out] +dir/a.py:1: error: Name "undef" is not defined +dir/subdir/b.py:1: error: Name "undef" is not defined + +[case testCmdlinePackageContainingSubdir] +# cmd: mypy pkg +[file pkg/__init__.py] +[file pkg/a.py] +undef +import pkg.a +[file pkg/subdir/a.py] +undef +import pkg.a +[out] +pkg/a.py:1: error: Name "undef" is not defined +pkg/subdir/a.py:1: error: Name "undef" is not defined + +[case testCmdlineNonPackageContainingPackage] +# cmd: mypy dir +[file dir/a.py] +undef +import subpkg.a +[file dir/subpkg/__init__.py] +[file dir/subpkg/a.py] +undef +[out] +dir/subpkg/a.py:1: error: Name "undef" is not defined +dir/a.py:1: error: Name "undef" is not defined + +[case testCmdlineInvalidPackageName] +# cmd: mypy dir/sub.pkg/a.py +[file dir/sub.pkg/__init__.py] +[file dir/sub.pkg/a.py] +undef +[out] +sub.pkg is not a valid Python package name +== Return code: 2 + +[case testBadFileEncoding] +# cmd: mypy a.py +[file a.py] +# coding: uft-8 +[out] +mypy: can't decode file 'a.py': unknown encoding: uft-8 +== Return code: 2 + +-- ' +[case testCannotIgnoreDuplicateModule] +# cmd: mypy one/mod/__init__.py two/mod/__init__.py +[file one/mod/__init__.py] +# type: ignore +[file two/mod/__init__.py] +# type: ignore +[out] +two/mod/__init__.py: error: Duplicate module named "mod" (also at "one/mod/__init__.py") +two/mod/__init__.py: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules for more info +two/mod/__init__.py: note: Common resolutions include: a) using `--exclude` to avoid checking one of them, b) adding `__init__.py` somewhere, c) using `--explicit-package-bases` or adjusting MYPYPATH +== Return code: 2 + +-- Note that we use `----`, because this is how `--` is escaped while `--` is a comment starter. +[case testFlagsFile] +# cmd: mypy @flagsfile +[file flagsfile] +----always-true=FLAG +main.py +[file main.py] +x: int +FLAG = False +if not FLAG: + x = "unreachable" + +[case testConfigFile] +# cmd: mypy main.py +[file mypy.ini] +\[mypy] +always_true = FLAG +[file main.py] +x: int +FLAG = False +if not FLAG: + x = "unreachable" + +[case testAltConfigFile] +# cmd: mypy --config-file config.ini main.py +[file config.ini] +\[mypy] +always_true = FLAG +[file main.py] +x: int +FLAG = False +if not FLAG: + x = "unreachable" + +[case testPerFileConfigSectionMultipleMatchesDisallowed] +# cmd: mypy xx.py xy.py yx.py yy.py +[file mypy.ini] +\[mypy] +\[mypy-*x*] +disallow_untyped_defs = True +\[mypy-*y*] +disallow_untyped_calls = True +[file xx.py] +def f(a): pass +def g(a: int) -> int: return f(a) +[file xy.py] +def f(a): pass +def g(a: int) -> int: return f(a) +[file yx.py] +def f(a): pass +def g(a: int) -> int: return f(a) +[file yy.py] +def f(a): pass +def g(a: int) -> int: return f(a) +[out] +mypy.ini: [mypy-*x*]: Patterns must be fully-qualified module names, optionally with '*' in some components (e.g spam.*.eggs.*) +mypy.ini: [mypy-*y*]: Patterns must be fully-qualified module names, optionally with '*' in some components (e.g spam.*.eggs.*) +== Return code: 0 + +[case testMultipleGlobConfigSection] +# cmd: mypy x.py y.py z.py +[file mypy.ini] +\[mypy] +\[mypy-x.*,z.*] +disallow_untyped_defs = True +[file x.py] +def f(a): pass +[file y.py] +def f(a): pass +[file z.py] +def f(a): pass +[out] +z.py:1: error: Function is missing a type annotation +x.py:1: error: Function is missing a type annotation + +[case testConfigErrorNoSection] +# cmd: mypy -c pass +[file mypy.ini] +[out] +mypy.ini: No [mypy] section in config file +== Return code: 0 + +[case testConfigErrorUnknownFlag] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +bad = 0 +[out] +mypy.ini: [mypy]: Unrecognized option: bad = 0 +== Return code: 0 + +[case testConfigErrorBadFlag] +# cmd: mypy a.py +[file mypy.ini] +\[mypy] +disallow-untyped-defs = True +[file a.py] +def f(): + pass +[out] +mypy.ini: [mypy]: Unrecognized option: disallow-untyped-defs = True +== Return code: 0 + +[case testConfigErrorBadBoolean] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +ignore_missing_imports = nah +[out] +mypy.ini: [mypy]: ignore_missing_imports: Not a boolean: nah +== Return code: 0 + +[case testConfigErrorNotPerFile] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +\[mypy-*] +python_version = 3.11 +[out] +mypy.ini: [mypy-*]: Per-module sections should only specify per-module flags (python_version) +== Return code: 0 + +[case testConfigMypyPath] +# cmd: mypy file.py +[file mypy.ini] +\[mypy] +mypy_path = + foo_dir:bar_dir + , baz_dir +[file foo_dir/foo.pyi] +def foo(x: int) -> str: ... +[file bar_dir/bar.pyi] +def bar(x: str) -> list: ... +[file baz_dir/baz.pyi] +def baz(x: list) -> dict: ... +[file file.py] +import no_stubs +from foo import foo +from bar import bar +from baz import baz +baz(bar(foo(42))) +baz(bar(foo('oof'))) +[out] +file.py:1: error: Cannot find implementation or library stub for module named "no_stubs" +file.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +file.py:6: error: Argument 1 to "foo" has incompatible type "str"; expected "int" + +[case testConfigFollowImportsSysPath] +# cmd: mypy main.py +[file main.py] +from a import x +x + 0 +x + '' # E +import a +a.x + 0 +a.x + '' # E +a.y # E +a + 0 # E +[file mypy.ini] +\[mypy] +follow_imports = normal +no_silence_site_packages = True +[file pypath/a/__init__.py] +x = 0 +x += '' # Error reported here +[file pypath/a/py.typed] +[out] +pypath/a/__init__.py:2: error: Unsupported operand types for + ("int" and "str") +main.py:3: error: Unsupported operand types for + ("int" and "str") +main.py:6: error: Unsupported operand types for + ("int" and "str") +main.py:7: error: Module has no attribute "y" +main.py:8: error: Unsupported operand types for + (Module and "int") + +[case testConfigFollowImportsInvalid] +# cmd: mypy main.py +[file mypy.ini] +\[mypy] +follow_imports =True +[file main.py] +[out] +mypy.ini: [mypy]: follow_imports: invalid choice 'True' (choose from 'normal', 'silent', 'skip', 'error') +== Return code: 0 + +[case testFailedImportOnWrongCWD] +# cmd: mypy main.py +# cwd: main/subdir1/subdir2 +[file main/subdir1/subdir2/main.py] +import parent +import grandparent +import missing +[file main/subdir1/subdir2/__init__.py] +[file main/subdir1/parent.py] +[file main/subdir1/__init__.py] +[file main/grandparent.py] +[file main/__init__.py] +[out] +main.py:1: error: Cannot find implementation or library stub for module named "parent" +main.py:1: note: You may be running mypy in a subpackage, mypy should be run on the package root +main.py:2: error: Cannot find implementation or library stub for module named "grandparent" +main.py:3: error: Cannot find implementation or library stub for module named "missing" +main.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testImportInParentButNoInit] +# cmd: mypy main.py +# cwd: main/not_a_package +[file main/not_a_package/main.py] +import needs_init +[file main/needs_init.py] +[file main/__init__.py] +[out] +main.py:1: error: Cannot find implementation or library stub for module named "needs_init" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testConfigNoErrorForUnknownXFlagInSubsection] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +\[mypy-foo] +x_bad = 0 +[out] + +[case testDotInFilenameOKScript] +# cmd: mypy a.b.py c.d.pyi +[file a.b.py] +undef +[file c.d.pyi] +whatever +[out] +c.d.pyi:1: error: Name "whatever" is not defined +a.b.py:1: error: Name "undef" is not defined + +[case testDotInFilenameOKFolder] +# cmd: mypy my.folder +[file my.folder/tst.py] +undef +[out] +my.folder/tst.py:1: error: Name "undef" is not defined + +[case testDotInFilenameNoImport] +# cmd: mypy main.py +[file main.py] +import a.b +[file a.b.py] +whatever +[out] +main.py:1: error: Cannot find implementation or library stub for module named "a.b" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main.py:1: error: Cannot find implementation or library stub for module named "a" + +[case testPythonVersionWrongFormatPyProjectTOML] +# cmd: mypy -c pass +[file pyproject.toml] +\[tool.mypy] +python_version = 3.10 +[out] +pyproject.toml: [mypy]: python_version: Python 3.1 is not supported (must be 3.8 or higher). You may need to put quotes around your Python version +== Return code: 0 + +[case testPythonVersionTooOld10] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +python_version = 1.0 +[out] +mypy.ini: [mypy]: python_version: Python major version '1' out of range (must be 3) +== Return code: 0 + +[case testPythonVersionTooOld37] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +python_version = 3.7 +[out] +mypy.ini: [mypy]: python_version: Python 3.7 is not supported (must be 3.8 or higher) +== Return code: 0 + +[case testPythonVersionTooNew40] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +python_version = 4.0 +[out] +mypy.ini: [mypy]: python_version: Python major version '4' out of range (must be 3) +== Return code: 0 + +[case testPythonVersionTooDead27] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +python_version = 2.7 +[out] +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: Mypy no longer supports checking Python 2 code. Consider pinning to mypy<0.980 if you need to check Python 2 code. +== Return code: 2 + +[case testPythonVersionAccepted38] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +python_version = 3.8 +[out] + +[case testPythonVersionAccepted311] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +python_version = 3.11 +[out] + +-- This should be a dumping ground for tests of plugins that are sensitive to +-- typeshed changes. +[case testTypeshedSensitivePlugins] +# cmd: mypy int_pow.py + +[file int_pow.py] +a = 1 +b = a + 2 +reveal_type(a**0) +reveal_type(a**1) +reveal_type(a**2) +reveal_type(a**-0) +reveal_type(a**-1) +reveal_type(a**(-2)) +reveal_type(a**b) +reveal_type(a.__pow__(2)) +reveal_type(a.__pow__(a)) +[out] +int_pow.py:3: note: Revealed type is "Literal[1]" +int_pow.py:4: note: Revealed type is "builtins.int" +int_pow.py:5: note: Revealed type is "builtins.int" +int_pow.py:6: note: Revealed type is "Literal[1]" +int_pow.py:7: note: Revealed type is "builtins.float" +int_pow.py:8: note: Revealed type is "builtins.float" +int_pow.py:9: note: Revealed type is "Any" +int_pow.py:10: note: Revealed type is "builtins.int" +int_pow.py:11: note: Revealed type is "Any" +== Return code: 0 + +[case testDisallowAnyGenericsBuiltinCollectionsPre39] +# cmd: mypy m.py +[file mypy.ini] +\[mypy] +python_version = 3.8 +\[mypy-m] +disallow_any_generics = True +[file m.py] +def j(s: frozenset) -> None: pass +[out] +m.py:1: error: Implicit generic "Any". Use "typing.FrozenSet" and specify generic parameters + +[case testDisallowAnyGenericsTypingCollections] +# cmd: mypy m.py +[file mypy.ini] +\[mypy] +\[mypy-m] +disallow_any_generics = True +[file m.py] +from typing import FrozenSet +def j(s: FrozenSet) -> None: pass +[out] +m.py:2: error: Missing type parameters for generic type "FrozenSet" + +[case testSectionInheritance] +# cmd: mypy a +[file a/__init__.py] +0() +[file a/foo.py] +0() +[file a/b/__init__.py] +[file a/b/c/__init__.py] +0() +[file a/b/c/d/__init__.py] +[file a/b/c/d/e/__init__.py] +from typing import List +def g(x: List) -> None: pass +g(None) +[file mypy.ini] +\[mypy] +allow_any_generics = True +\[mypy-a.*] +ignore_errors = True +\[mypy-a.b.*] +disallow_any_generics = True +ignore_errors = True +\[mypy-a.b.c.*] +ignore_errors = True +\[mypy-a.b.c.d.*] +ignore_errors = True +\[mypy-a.b.c.d.e.*] +ignore_errors = True +strict_optional = True +\[mypy-a.b.c.d.e] +ignore_errors = False +[out] +a/b/c/d/e/__init__.py:2: error: Missing type parameters for generic type "List" +a/b/c/d/e/__init__.py:3: error: Argument 1 to "g" has incompatible type "None"; expected "List[Any]" + +[case testMissingFile] +# cmd: mypy nope.py +[out] +mypy: can't read file 'nope.py': No such file or directory +== Return code: 2 +--' + +[case testModulesAndPackages] +# cmd: mypy --package p.a --package p.b --module c +[file p/__init__.py] +[file p/a.py] +def foo(x): + # type: (int) -> str + return "x" +foo("wrong") +[file p/b/__init__.py] +from ..a import foo +def bar(a): + # type: (int) -> str + return foo(a) +bar("wrong") +[file c.py] +import p.b +p.b.bar("wrong") +[out] +p/a.py:4: error: Argument 1 to "foo" has incompatible type "str"; expected "int" +p/b/__init__.py:5: error: Argument 1 to "bar" has incompatible type "str"; expected "int" +c.py:2: error: Argument 1 to "bar" has incompatible type "str"; expected "int" + +[case testSrcPEP420Packages] +# cmd: mypy -p anamespace --namespace-packages +[file mypy.ini] +\[mypy] +mypy_path = src +[file src/setup.cfg] +[file src/anamespace/foo/__init__.py] +[file src/anamespace/foo/bar.py] +def bar(a: int, b: int) -> str: + return a + b +[out] +src/anamespace/foo/bar.py:2: error: Incompatible return value type (got "int", expected "str") + +[case testNestedPEP420Packages] +# cmd: mypy -p pkg --namespace-packages +[file pkg/a1/b/c/d/e.py] +x = 0 # type: str +[file pkg/a1/b/f.py] +from pkg.a1.b.c.d.e import x +x() + +[file pkg/a2/__init__.py] +[file pkg/a2/b/c/d/e.py] +x = 0 # type: str +[file pkg/a2/b/f.py] +from pkg.a2.b.c.d.e import x +x() +[out] +pkg/a2/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") +pkg/a1/b/c/d/e.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") +pkg/a2/b/f.py:2: error: "str" not callable +pkg/a1/b/f.py:2: error: "str" not callable + +[case testFollowImportStubs1] +# cmd: mypy main.py +[file mypy.ini] +\[mypy] +\[mypy-math.*] +follow_imports = error +follow_imports_for_stubs = True +[file main.py] +import math +math.frobnicate() +[out] +main.py:1: error: Import of "math" ignored +main.py:1: note: (Using --follow-imports=error, module not passed on command line) + +[case testFollowImportStubs2] +# cmd: mypy main.py +[file mypy.ini] +\[mypy] +\[mypy-math.*] +follow_imports = skip +follow_imports_for_stubs = True +[file main.py] +import math +math.frobnicate() + +[case testShadowFile1] +# cmd: mypy --shadow-file source.py shadow.py source.py +[file source.py] +def foo() -> str: + return "bar" +[file shadow.py] +def bar() -> str: + return 14 +[out] +source.py:2: error: Incompatible return value type (got "int", expected "str") + +[case testShadowFile2] +# cmd: mypy --shadow-file s1.py shad1.py --shadow-file s2.py shad2.py --shadow-file s3.py shad3.py s1.py s2.py s3.py s4.py +[file s1.py] +def foo() -> str: + return "bar" +[file shad1.py] +def bar() -> str: + return 14 +[file s2.py] +def baz() -> str: + return 14 +[file shad2.py] +def baz() -> int: + return 14 +[file s3.py] +def qux() -> str: + return "bar" +[file shad3.py] +def foo() -> int: + return [42] +[file s4.py] +def foo() -> str: + return 9 +[out] +s4.py:2: error: Incompatible return value type (got "int", expected "str") +s3.py:2: error: Incompatible return value type (got "List[int]", expected "int") +s1.py:2: error: Incompatible return value type (got "int", expected "str") + +[case testShadowFileWithPretty] +# cmd: mypy a.py --pretty --shadow-file a.py b.py +[file a.py] +b: bytes +[file b.py] +a: int = "" +b: bytes = 1 +[out] +a.py:1: error: Incompatible types in assignment (expression has type "str", +variable has type "int") + a: int = "" + ^~ +a.py:2: error: Incompatible types in assignment (expression has type "int", +variable has type "bytes") + b: bytes = 1 + ^ + +[case testConfigWarnUnusedSection1] +# cmd: mypy foo.py quux.py spam/eggs.py +[file mypy.ini] +\[mypy] +warn_unused_configs = True +incremental = False +\[mypy-bar] +\[mypy-foo] +\[mypy-baz.*] +\[mypy-quux.*] +\[mypy-spam.*] +\[mypy-spam.eggs] +\[mypy-emarg.*] +\[mypy-emarg.hatch] +-- Currently we don't treat an unstructured pattern like a.*.b as unused +-- if it matches another section (like a.x.b). This would be reasonable +-- to change. ' +\[mypy-a.*.b] +\[mypy-a.*.c] +\[mypy-a.x.b] +[file foo.py] +[file quux.py] +[file spam/__init__.py] +[file spam/eggs.py] +[out] +Warning: unused section(s) in mypy.ini: [mypy-bar], [mypy-baz.*], [mypy-emarg.*], [mypy-emarg.hatch], [mypy-a.*.c], [mypy-a.x.b] +== Return code: 0 + +[case testConfigUnstructuredGlob] +# cmd: mypy emarg foo +[file mypy.ini] +\[mypy] +ignore_errors = true +\[mypy-*.lol] +ignore_errors = false +\[mypy-emarg.*] +ignore_errors = false +\[mypy-emarg.*.villip.*] +ignore_errors = true +\[mypy-emarg.hatch.villip.mankangulisk] +ignore_errors = false +[file emarg/__init__.py] +[file emarg/foo.py] +fail +[file emarg/villip.py] +fail +[file emarg/hatch/__init__.py] +[file emarg/hatch/villip/__init__.py] +[file emarg/hatch/villip/nus.py] +fail +[file emarg/hatch/villip/mankangulisk.py] +fail +[file foo/__init__.py] +[file foo/lol.py] +fail +[out] +foo/lol.py:1: error: Name "fail" is not defined +emarg/foo.py:1: error: Name "fail" is not defined +emarg/hatch/villip/mankangulisk.py:1: error: Name "fail" is not defined + +[case testPackageRootEmpty] +# cmd: mypy --no-namespace-packages --package-root= a/b/c.py main.py +[file a/b/c.py] +[file main.py] +import a.b.c + +[case testPackageRootEmptyNamespacePackage] +# cmd: mypy --namespace-packages --package-root= a/b/c.py main.py +[file a/b/c.py] +[file main.py] +import a.b.c + +[case testPackageRootNonEmpty] +# cmd: mypy --package-root=a/ a/b/c.py main.py +[file a/b/c.py] +[file main.py] +import b.c + +[case testPackageRootMultiple1] +# cmd: mypy --package-root=. --package-root=a a/b/c.py d.py main.py +[file a/b/c.py] +[file d.py] +[file main.py] +import b.c +import d + +[case testPackageRootMultiple2] +# cmd: mypy --package-root=a/ --package-root=./ a/b/c.py d.py main.py +[file a/b/c.py] +[file d.py] +[file main.py] +import b.c +import d + +[case testCacheMap] +-- This just checks that a valid --cache-map triple is accepted. +-- (Errors are too verbose to check.) +# cmd: mypy a.py --no-sqlite-cache --cache-map a.py a.meta.json a.data.json +[file a.py] +[out] + +[case testIniFiles] +# cmd: mypy +[file mypy.ini] +\[mypy] +files = a.py, b.py +[file a.py] +fail +[file b.py] +fail +[out] +b.py:1: error: Name "fail" is not defined +a.py:1: error: Name "fail" is not defined + +[case testIniFilesGlobbing] +# cmd: mypy +[file mypy.ini] +\[mypy] +files = **/*.py +[file a/b.py] +fail +[file c.py] +fail +[out] +a/b.py:1: error: Name "fail" is not defined +c.py:1: error: Name "fail" is not defined + +[case testIniFilesCmdlineOverridesConfig] +# cmd: mypy override.py +[file mypy.ini] +\[mypy] +files = config.py +[out] +mypy: can't read file 'override.py': No such file or directory +== Return code: 2 + +[case testErrorSummaryOnSuccess] +# cmd: mypy --error-summary good.py +[file good.py] +x = 2 + 2 +[out] +Success: no issues found in 1 source file +== Return code: 0 + +[case testErrorSummaryOnFail] +# cmd: mypy --error-summary bad.py +[file bad.py] +42 + 'no' +[out] +bad.py:1: error: Unsupported operand types for + ("int" and "str") +Found 1 error in 1 file (checked 1 source file) + +[case testErrorSummaryOnFailNotes] +# cmd: mypy --error-summary bad.py +[file bad.py] +from typing import List +x = [] # type: List[float] +y = [] # type: List[int] +x = y +[out] +bad.py:4: error: Incompatible types in assignment (expression has type "List[int]", variable has type "List[float]") +bad.py:4: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance +bad.py:4: note: Consider using "Sequence" instead, which is covariant +Found 1 error in 1 file (checked 1 source file) + +[case testErrorSummaryOnFailTwoErrors] +# cmd: mypy --error-summary bad.py foo.py +[file bad.py] +42 + 'no' +42 + 'no' +[file foo.py] +[out] +bad.py:1: error: Unsupported operand types for + ("int" and "str") +bad.py:2: error: Unsupported operand types for + ("int" and "str") +Found 2 errors in 1 file (checked 2 source files) + +[case testErrorSummaryOnFailTwoFiles] +# cmd: mypy --error-summary bad.py bad2.py +[file bad.py] +42 + 'no' +[file bad2.py] +42 + 'no' +[out] +bad2.py:1: error: Unsupported operand types for + ("int" and "str") +bad.py:1: error: Unsupported operand types for + ("int" and "str") +Found 2 errors in 2 files (checked 2 source files) + +[case testErrorSummaryOnBadUsage] +# cmd: mypy --error-summary missing.py +[out] +mypy: can't read file 'missing.py': No such file or directory +== Return code: 2 + +[case testShowSourceCodeSnippetsWrappedFormatting] +# cmd: mypy --pretty some_file.py +[file some_file.py] +from typing import Union + +42 + 'no way' + +class OneCustomClassName: + def some_interesting_method(self, arg: AnotherCustomClassDefinedBelow) -> AnotherCustomClassDefinedBelow: + ... + +class AnotherCustomClassDefinedBelow: + def another_even_more_interesting_method(self, arg: Union[int, str, float]) -> None: + self.very_important_attribute_with_long_name: OneCustomClassName = OneCustomClassName().some_interesting_method(arg) +[out] +some_file.py:3: error: Unsupported operand types for + ("int" and "str") + 42 + 'no way' + ^~~~~~~~ +some_file.py:11: error: Incompatible types in assignment (expression has type +"AnotherCustomClassDefinedBelow", variable has type "OneCustomClassName") + ...t_attribute_with_long_name: OneCustomClassName = OneCustomClassName().... + ^~~~~~~~~~~~~~~~~~~~~... +some_file.py:11: error: Argument 1 to "some_interesting_method" of +"OneCustomClassName" has incompatible type "Union[int, str, float]"; expected +"AnotherCustomClassDefinedBelow" + ...OneCustomClassName = OneCustomClassName().some_interesting_method(arg) + ^~~ +[case testShowSourceCodeSnippetsBlockingError] +# cmd: mypy --pretty --show-error-codes some_file.py +[file some_file.py] +it_looks_like_we_started_typing_something_but_then. = did_not_notice(an_extra_dot) +[out] +some_file.py:1: error: invalid syntax [syntax] + ...ooks_like_we_started_typing_something_but_then. = did_not_notice(an_ex... + ^ +== Return code: 2 + +[case testTabRenderingUponError] +# cmd: mypy --pretty tabs.py +[file tabs.py] +def test_tabs() -> str: + return None +def test_between(x: str) -> None: ... +test_between(1 + 1) +[out] +tabs.py:2: error: Incompatible return value type (got "None", expected "str") + return None + ^~~~ +tabs.py:4: error: Argument 1 to "test_between" has incompatible type "int"; +expected "str" + test_between(1 + 1) + ^~~~~~~~~~~~ + +[case testErrorMessageWhenOpenPydFile] +# cmd: mypy a.pyd +[file a.pyd] +# coding: uft-8 +[out] +mypy: stubgen does not support .pyd files: 'a.pyd' +== Return code: 2 + +[case testDuplicateModules] +# cmd: mypy src +[file mypy.ini] +\[mypy] +mypy_path = src +[file src/__init__.py] +[file src/a.py] +import foo.bar +[file src/foo/__init__.py] +[file src/foo/bar.py] +1+'x' +[out] +src/foo/bar.py: error: Source file found twice under different module names: "src.foo.bar" and "foo.bar" +src/foo/bar.py: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules for more info +src/foo/bar.py: note: Common resolutions include: a) adding `__init__.py` somewhere, b) using `--explicit-package-bases` or adjusting MYPYPATH +== Return code: 2 + +[case testEnableInvalidErrorCode] +# cmd: mypy --enable-error-code YOLO test.py +[file test.py] +x = 1 +[out] +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: Invalid error code(s): YOLO +== Return code: 2 + +[case testDisableInvalidErrorCode] +# cmd: mypy --disable-error-code YOLO test.py +[file test.py] +x = 1 +[out] +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: Invalid error code(s): YOLO +== Return code: 2 + +[case testEnableAndDisableInvalidErrorCode] +# cmd: mypy --disable-error-code YOLO --enable-error-code YOLO2 test.py +[file test.py] +x = 1 +[out] +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: Invalid error code(s): YOLO, YOLO2 +== Return code: 2 + +[case testEnableValidAndInvalidErrorCode] +# cmd: mypy --enable-error-code attr-defined --enable-error-code YOLO test.py +[file test.py] +x = 1 +[out] +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: Invalid error code(s): YOLO +== Return code: 2 + +[case testDisableValidAndInvalidErrorCode] +# cmd: mypy --disable-error-code attr-defined --disable-error-code YOLO test.py +[file test.py] +x = 1 +[out] +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: Invalid error code(s): YOLO +== Return code: 2 + +[case testStubsDirectory] +# cmd: mypy --error-summary pkg-stubs +[file pkg-stubs/__init__.pyi] +[file pkg-stubs/thing.pyi] +class Thing: ... +[out] +Success: no issues found in 2 source files +== Return code: 0 + +[case testStubsDirectoryFile] +# cmd: mypy --error-summary pkg-stubs/thing.pyi +[file pkg-stubs/__init__.pyi] +[file pkg-stubs/thing.pyi] +class Thing: ... +[out] +Success: no issues found in 1 source file +== Return code: 0 + +[case testStubsSubDirectory] +# cmd: mypy --error-summary src/pkg-stubs +[file src/pkg-stubs/__init__.pyi] +[file src/pkg-stubs/thing.pyi] +class Thing: ... +[out] +Success: no issues found in 2 source files +== Return code: 0 + +[case testStubsSubDirectoryFile] +# cmd: mypy --error-summary src/pkg-stubs/thing.pyi +[file src/pkg-stubs/__init__.pyi] +[file src/pkg-stubs/thing.pyi] +class Thing: ... +[out] +Success: no issues found in 1 source file +== Return code: 0 + +[case testBlocker] +# cmd: mypy pkg --error-summary --disable-error-code syntax +[file pkg/x.py] +public static void main(String[] args) +[file pkg/y.py] +x: str = 0 +[out] +pkg/x.py:1: error: invalid syntax +Found 1 error in 1 file (errors prevented further checking) +== Return code: 2 +[out version>=3.10] +pkg/x.py:1: error: invalid syntax. Perhaps you forgot a comma? +Found 1 error in 1 file (errors prevented further checking) +== Return code: 2 +[out version>=3.10.3] +pkg/x.py:1: error: invalid syntax +Found 1 error in 1 file (errors prevented further checking) +== Return code: 2 + +[case testCmdlinePackageAndFile] +# cmd: mypy -p pkg file +[out] +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: May only specify one of: module/package, files, or command. +== Return code: 2 + +[case testCmdlinePackageAndIniFiles] +# cmd: mypy -p pkg +[file mypy.ini] +\[mypy] +files=file +[file pkg.py] +x = 0 # type: str +[file file.py] +y = 0 # type: str +[out] +pkg.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + + +[case testCmdlineModuleAndIniFiles] +# cmd: mypy -m pkg +[file mypy.ini] +\[mypy] +files=file +[file pkg.py] +x = 0 # type: str +[file file.py] +y = 0 # type: str +[out] +pkg.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testCmdlineNonInteractiveWithoutInstallTypes] +# cmd: mypy --non-interactive -m pkg +[out] +error: --non-interactive is only supported with --install-types +== Return code: 2 + +[case testCmdlineNonInteractiveInstallTypesNothingToDo] +# cmd: mypy --install-types --non-interactive -m pkg +[file pkg.py] +1() +[out] +pkg.py:1: error: "int" not callable + +[case testCmdlineNonInteractiveInstallTypesNothingToDoNoError] +# cmd: mypy --install-types --non-interactive -m pkg +[file pkg.py] +1 + 2 +[out] + +[case testCmdlineNonInteractiveInstallTypesNoSitePackages] +# cmd: mypy --install-types --non-interactive --no-site-packages -m pkg +[out] +error: --install-types not supported without python executable or site packages +== Return code: 2 + +[case testCmdlineInteractiveInstallTypesNothingToDo] +# cmd: mypy --install-types -m pkg +[file pkg.py] +1() +[out] +pkg.py:1: error: "int" not callable + +[case testCmdlineExclude] +# cmd: mypy --exclude abc . +[file abc/apkg.py] +1() +[file b/bpkg.py] +1() +[file c/cpkg.py] +1() +[out] +c/cpkg.py:1: error: "int" not callable +b/bpkg.py:1: error: "int" not callable + +[case testCmdlineMultipleExclude] +# cmd: mypy --exclude abc --exclude b/ . +[file abc/apkg.py] +1() +[file b/bpkg.py] +1() +[file c/cpkg.py] +1() +[out] +c/cpkg.py:1: error: "int" not callable + +[case testCmdlineCfgExclude] +# cmd: mypy . +[file mypy.ini] +\[mypy] +exclude = abc +[file abc/apkg.py] +1() +[file b/bpkg.py] +1() +[file c/cpkg.py] +1() +[out] +c/cpkg.py:1: error: "int" not callable +b/bpkg.py:1: error: "int" not callable + +[case testCmdlineCfgMultipleExclude] +# cmd: mypy . +[file mypy.ini] +\[mypy] +exclude = (?x)( + ^abc/ + |^b/ + ) +[file abc/apkg.py] +1() +[file b/bpkg.py] +1() +[file c/cpkg.py] +1() +[out] +c/cpkg.py:1: error: "int" not callable + +[case testCmdlineTimingStats] +# cmd: mypy --timing-stats timing.txt . +[file b/__init__.py] +[file b/c.py] +class C: pass +[outfile-re timing.txt] +.* +b \d+ +b\.c \d+ +.* + +[case testShadowTypingModuleEarlyLoad] +# cmd: mypy dir +[file dir/__init__.py] +from typing import Union + +def foo(a: Union[int, str]) -> str: + return str +[file typing.py] +# Since this file will be picked by mypy itself, we need it to be a fully-working typing +# A bare minimum would be NamedTuple and TypedDict, which are used in runtime, +# everything else technically can be just mocked. +import sys +import os +del sys.modules["typing"] +path = sys.path +try: + sys.path.remove(os.getcwd()) +except ValueError: + sys.path.remove("") # python 3.6 +from typing import * +sys.path = path +[out] +mypy: "typing.py" shadows library module "typing" +note: A user-defined top-level module with name "typing" is not supported +== Return code: 2 + +[case testCustomTypeshedDirWithRelativePathDoesNotCrash] +# cmd: mypy --custom-typeshed-dir dir dir/typing.pyi +[file dir/stdlib/abc.pyi] +[file dir/stdlib/builtins.pyi] +[file dir/stdlib/sys.pyi] +[file dir/stdlib/types.pyi] +[file dir/stdlib/typing.pyi] +[file dir/stdlib/typing_extensions.pyi] +[file dir/stdlib/_typeshed.pyi] +[file dir/stdlib/_collections_abc.pyi] +[file dir/stdlib/collections/abc.pyi] +[file dir/stdlib/collections/__init__.pyi] +[file dir/stdlib/VERSIONS] +[out] +Failed to find builtin module mypy_extensions, perhaps typeshed is broken? +== Return code: 2 + +[case testNewTypeInferenceFlagDeprecated] +# cmd: mypy --new-type-inference a.py +[file a.py] +pass +[out] +Warning: --new-type-inference flag is deprecated; new type inference algorithm is already enabled by default +== Return code: 0 + +[case testCustomTypeshedDirFilePassedExplicitly] +# cmd: mypy --custom-typeshed-dir dir m.py dir/stdlib/foo.pyi +[file m.py] +1() +[file dir/stdlib/abc.pyi] +1() # Errors are not reported from typeshed by default +[file dir/stdlib/builtins.pyi] +class object: pass +class str(object): pass +class int(object): pass +class list: pass +class dict: pass +[file dir/stdlib/sys.pyi] +[file dir/stdlib/types.pyi] +[file dir/stdlib/typing.pyi] +[file dir/stdlib/mypy_extensions.pyi] +[file dir/stdlib/typing_extensions.pyi] +[file dir/stdlib/_typeshed.pyi] +[file dir/stdlib/_collections_abc.pyi] +[file dir/stdlib/collections/abc.pyi] +[file dir/stdlib/collections/__init__.pyi] +[file dir/stdlib/foo.pyi] +1() # Errors are reported if the file was explicitly passed on the command line +[file dir/stdlib/VERSIONS] +[out] +dir/stdlib/foo.pyi:1: error: "int" not callable +m.py:1: error: "int" not callable + +[case testFileInPythonPathPassedExplicitly1] +# cmd: mypy $CWD/pypath/foo.py +[file pypath/foo.py] +1() +[out] +pypath/foo.py:1: error: "int" not callable + +[case testFileInPythonPathPassedExplicitly2] +# cmd: mypy pypath/foo.py +[file pypath/foo.py] +1() +[out] +pypath/foo.py:1: error: "int" not callable + +[case testFileInPythonPathPassedExplicitly3] +# cmd: mypy -p foo +# cwd: pypath +[file pypath/foo/__init__.py] +1() +[file pypath/foo/m.py] +1() +[out] +foo/m.py:1: error: "int" not callable +foo/__init__.py:1: error: "int" not callable + +[case testFileInPythonPathPassedExplicitly4] +# cmd: mypy -m foo +# cwd: pypath +[file pypath/foo.py] +1() +[out] +foo.py:1: error: "int" not callable + +[case testFileInPythonPathPassedExplicitly5] +# cmd: mypy -m foo.m +# cwd: pypath +[file pypath/foo/__init__.py] +1() # TODO: Maybe this should generate errors as well? But how would we decide? +[file pypath/foo/m.py] +1() +[out] +foo/m.py:1: error: "int" not callable + +[case testCmdlineCfgEnableErrorCodeTrailingComma] +# cmd: mypy . +[file mypy.ini] +\[mypy] +enable_error_code = + truthy-bool, + redundant-expr, +[out] + +[case testCmdlineCfgDisableErrorCodeTrailingComma] +# cmd: mypy . +[file mypy.ini] +\[mypy] +disable_error_code = + misc, + override, +[out] + +[case testCmdlineCfgAlwaysTrueTrailingComma] +# cmd: mypy . +[file mypy.ini] +\[mypy] +always_true = + MY_VAR, +[out] + +[case testTypeVarTupleUnpackEnabled] +# cmd: mypy --enable-incomplete-feature=TypeVarTuple --enable-incomplete-feature=Unpack a.py +[file a.py] +from typing_extensions import TypeVarTuple +Ts = TypeVarTuple("Ts") +[out] +Warning: TypeVarTuple is already enabled by default +Warning: Unpack is already enabled by default +== Return code: 0 diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test new file mode 100644 index 000000000000..7dfddd8f74df --- /dev/null +++ b/test-data/unit/daemon.test @@ -0,0 +1,649 @@ +-- End-to-end test cases for the daemon (dmypy). +-- These are special because they run multiple shell commands. + +[case testDaemonStartStop] +$ dmypy start -- --follow-imports=error +Daemon started +$ dmypy stop +Daemon stopped + +[case testDaemonBasic] +$ dmypy start -- --follow-imports=error +Daemon started +$ dmypy check -- foo.py +Success: no issues found in 1 source file +$ dmypy recheck +Success: no issues found in 1 source file +$ dmypy stop +Daemon stopped +[file foo.py] +def f(): pass + +[case testDaemonRun] +$ dmypy run -- foo.py --follow-imports=error +Daemon started +Success: no issues found in 1 source file +$ dmypy stop +Daemon stopped +[file foo.py] +def f(): pass + +[case testDaemonRunIgnoreMissingImports] +$ dmypy run -- foo.py --follow-imports=error --ignore-missing-imports +Daemon started +Success: no issues found in 1 source file +$ dmypy stop +Daemon stopped +[file foo.py] +def f(): pass + +[case testDaemonRunErrorCodes] +$ dmypy run -- foo.py --follow-imports=error --disable-error-code=type-abstract +Daemon started +Success: no issues found in 1 source file +$ dmypy stop +Daemon stopped +[file foo.py] +def f(): pass + +[case testDaemonRunCombinedOptions] +$ dmypy run -- foo.py --follow-imports=error --ignore-missing-imports --disable-error-code=type-abstract +Daemon started +Success: no issues found in 1 source file +$ dmypy stop +Daemon stopped +[file foo.py] +def f(): pass + +[case testDaemonIgnoreConfigFiles] +$ dmypy start -- --follow-imports=error +Daemon started +[file mypy.ini] +\[mypy] +files = ./foo.py + +[case testDaemonRunMultipleStrict] +$ dmypy run -- foo.py --strict --follow-imports=error +Daemon started +foo.py:1: error: Function is missing a return type annotation +foo.py:1: note: Use "-> None" if function does not return a value +Found 1 error in 1 file (checked 1 source file) +== Return code: 1 +$ dmypy run -- bar.py --strict --follow-imports=error +bar.py:1: error: Function is missing a return type annotation +bar.py:1: note: Use "-> None" if function does not return a value +Found 1 error in 1 file (checked 1 source file) +== Return code: 1 +$ dmypy run -- foo.py --strict --follow-imports=error +foo.py:1: error: Function is missing a return type annotation +foo.py:1: note: Use "-> None" if function does not return a value +Found 1 error in 1 file (checked 1 source file) +== Return code: 1 +[file foo.py] +def f(): pass +[file bar.py] +def f(): pass + +[case testDaemonRunRestart] +$ dmypy run -- foo.py --follow-imports=error +Daemon started +Success: no issues found in 1 source file +$ dmypy run -- foo.py --follow-imports=error +Success: no issues found in 1 source file +$ {python} -c "print('[mypy]')" >mypy.ini +$ {python} -c "print('disallow_untyped_defs = True')" >>mypy.ini +$ dmypy run -- foo.py --follow-imports=error +Restarting: configuration changed +Daemon stopped +Daemon started +foo.py:1: error: Function is missing a return type annotation +foo.py:1: note: Use "-> None" if function does not return a value +Found 1 error in 1 file (checked 1 source file) +== Return code: 1 +$ {python} -c "print('def f() -> None: pass')" >foo.py +$ dmypy run -- foo.py --follow-imports=error +Success: no issues found in 1 source file +$ dmypy stop +Daemon stopped +[file foo.py] +def f(): pass + +[case testDaemonRunRestartPretty] +$ dmypy run -- foo.py --follow-imports=error --pretty +Daemon started +Success: no issues found in 1 source file +$ dmypy run -- foo.py --follow-imports=error --pretty +Success: no issues found in 1 source file +$ {python} -c "print('[mypy]')" >mypy.ini +$ {python} -c "print('disallow_untyped_defs = True')" >>mypy.ini +$ dmypy run -- foo.py --follow-imports=error --pretty +Restarting: configuration changed +Daemon stopped +Daemon started +foo.py:1: error: Function is missing a return type annotation + def f(): + ^ +foo.py:1: note: Use "-> None" if function does not return a value +Found 1 error in 1 file (checked 1 source file) +== Return code: 1 +$ {python} -c "print('def f() -> None: pass')" >foo.py +$ dmypy run -- foo.py --follow-imports=error --pretty +Success: no issues found in 1 source file +$ dmypy stop +Daemon stopped +[file foo.py] +def f(): + pass + +[case testDaemonRunRestartPluginVersion] +$ dmypy run -- foo.py --no-error-summary +Daemon started +$ {python} -c "print(' ')" >> plug.py +$ dmypy run -- foo.py --no-error-summary +Restarting: plugins changed +Daemon stopped +Daemon started +$ dmypy stop +Daemon stopped +[file mypy.ini] +\[mypy] +follow_imports = error +plugins = plug.py +[file foo.py] +pass +[file plug.py] +from mypy.plugin import Plugin +class Dummy(Plugin): pass +def plugin(version): return Dummy + +[case testDaemonRunRestartGlobs] +-- Ensure dmypy is not restarted if the configuration doesn't change and it contains globs +-- Note: Backslash path separator in output is replaced with forward slash so the same test succeeds on Windows as well +$ dmypy run -- foo --follow-imports=error +Daemon started +foo/lol.py:1: error: Name "fail" is not defined +Found 1 error in 1 file (checked 3 source files) +== Return code: 1 +$ dmypy run -- foo --follow-imports=error +foo/lol.py:1: error: Name "fail" is not defined +Found 1 error in 1 file (checked 3 source files) +== Return code: 1 +$ {python} -c "print('[mypy]')" >mypy.ini +$ {python} -c "print('ignore_errors=True')" >>mypy.ini +$ dmypy run -- foo --follow-imports=error +Restarting: configuration changed +Daemon stopped +Daemon started +Success: no issues found in 3 source files +$ dmypy stop +Daemon stopped +[file mypy.ini] +\[mypy] +ignore_errors = True +\[mypy-*.lol] +ignore_errors = False + +[file foo/__init__.py] +[file foo/lol.py] +fail +[file foo/ok.py] +a: int = 1 + +[case testDaemonStatusKillRestartRecheck] +$ dmypy status +No status file found +== Return code: 2 +$ dmypy stop +No status file found +== Return code: 2 +$ dmypy kill +No status file found +== Return code: 2 +$ dmypy recheck +No status file found +== Return code: 2 +$ dmypy start -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy status +Daemon is up and running +$ dmypy start +Daemon is still alive +== Return code: 2 +$ dmypy restart -- --follow-imports=error --no-error-summary +Daemon stopped +Daemon started +$ dmypy stop +Daemon stopped +$ dmypy status +No status file found +== Return code: 2 +$ dmypy restart -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy recheck +Command 'recheck' is only valid after a 'check' command +== Return code: 2 +$ dmypy kill +Daemon killed +$ dmypy status +Daemon has died +== Return code: 2 + +[case testDaemonRecheck] +$ dmypy start -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy check foo.py bar.py +$ dmypy recheck +$ dmypy recheck --update foo.py --remove bar.py sir_not_appearing_in_this_film.py +foo.py:1: error: Import of "bar" ignored [misc] +foo.py:1: note: (Using --follow-imports=error, module not passed on command line) +== Return code: 1 +$ dmypy recheck --update bar.py +$ dmypy recheck --update sir_not_appearing_in_this_film.py +$ dmypy recheck --update --remove +$ dmypy stop +Daemon stopped +[file foo.py] +import bar +[file bar.py] +pass + +[case testDaemonTimeout] +$ dmypy start --timeout 1 -- --follow-imports=error +Daemon started +$ {python} -c "import time;time.sleep(1)" +$ dmypy status +No status file found +== Return code: 2 + +[case testDaemonRunNoTarget] +$ dmypy run -- --follow-imports=error +Daemon started +mypy-daemon: error: Missing target module, package, files, or command. +== Return code: 2 +$ dmypy stop +Daemon stopped + +[case testDaemonRunTwoFilesFullTypeshed] +$ dmypy run x.py +Daemon started +Success: no issues found in 1 source file +$ dmypy run y.py +Success: no issues found in 1 source file +$ dmypy run x.py +Success: no issues found in 1 source file +[file x.py] +[file y.py] + +[case testDaemonCheckTwoFilesFullTypeshed] +$ dmypy start +Daemon started +$ dmypy check foo.py +foo.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] +Found 1 error in 1 file (checked 1 source file) +== Return code: 1 +$ dmypy check bar.py +Success: no issues found in 1 source file +$ dmypy check foo.py +foo.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] +Found 1 error in 1 file (checked 1 source file) +== Return code: 1 +[file foo.py] +from bar import add +x: str = add("a", "b") +x_error: int = add("a", "b") +[file bar.py] +def add(a, b) -> str: + return a + b + +[case testDaemonWarningSuccessExitCode-posix] +$ dmypy run -- foo.py --follow-imports=error --python-version=3.11 +Daemon started +foo.py:2: note: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs +Success: no issues found in 1 source file +$ echo $? +0 +$ dmypy stop +Daemon stopped +[file foo.py] +def foo(): + a: int = 1 + print(a + "2") + +-- this is carefully constructed to be able to break if the quickstart system lets +-- something through incorrectly. in particular, the files need to have the same size +[case testDaemonQuickstart] +$ {python} -c "print('x=1')" >foo.py +$ {python} -c "print('x=1')" >bar.py +$ mypy --local-partial-types --cache-fine-grained --follow-imports=error --no-sqlite-cache --python-version=3.11 -- foo.py bar.py +Success: no issues found in 2 source files +$ {python} -c "import shutil; shutil.copy('.mypy_cache/3.11/bar.meta.json', 'asdf.json')" +-- update bar's timestamp but don't change the file +$ {python} -c "import time;time.sleep(1)" +$ {python} -c "print('x=1')" >bar.py +$ dmypy run -- foo.py bar.py --follow-imports=error --use-fine-grained-cache --no-sqlite-cache --python-version=3.11 +Daemon started +Success: no issues found in 2 source files +$ dmypy status --fswatcher-dump-file test.json +Daemon is up and running +$ dmypy stop +Daemon stopped +-- copy the original bar cache file back so that the mtime mismatches +$ {python} -c "import shutil; shutil.copy('asdf.json', '.mypy_cache/3.11/bar.meta.json')" +-- sleep guarantees timestamp changes +$ {python} -c "import time;time.sleep(1)" +$ {python} -c "print('lol')" >foo.py +$ dmypy run --log-file=log -- foo.py bar.py --follow-imports=error --use-fine-grained-cache --no-sqlite-cache --python-version=3.11 --quickstart-file test.json +Daemon started +foo.py:1: error: Name "lol" is not defined +Found 1 error in 1 file (checked 2 source files) +== Return code: 1 +-- make sure no errors made it to the log file +$ {python} -c "import sys; sys.stdout.write(open('log').read())" +-- make sure the meta file didn't get updated. we use this as an imperfect proxy for +-- whether the source file got rehashed, which we don't want it to have been. +$ {python} -c "x = open('.mypy_cache/3.11/bar.meta.json').read(); y = open('asdf.json').read(); assert x == y" + +[case testDaemonSuggest] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy suggest foo:foo +Command 'suggest' is only valid after a 'check' command (that produces no parse errors) +== Return code: 2 +$ dmypy check foo.py bar.py +$ dmypy suggest foo.bar +Unknown function foo.bar +== Return code: 2 +$ dmypy suggest foo.var +Object foo.var is not a function +== Return code: 2 +$ dmypy suggest foo.Foo.var +Unknown class foo.Foo +== Return code: 2 +$ dmypy suggest foo.Bar.baz +Unknown method foo.Bar.baz +== Return code: 2 +$ dmypy suggest foo.foo.baz +Object foo.foo is not a class +== Return code: 2 +$ dmypy suggest --callsites foo.foo +bar.py:3: (str) +bar.py:4: (arg=str) +$ dmypy suggest foo.foo +(str) -> int +$ {python} -c "import shutil; shutil.copy('foo2.py', 'foo.py')" +$ dmypy check foo.py bar.py +bar.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] +== Return code: 1 +[file foo.py] +def foo(arg): + return 12 +class Bar: + def bar(self): pass +var = 0 +[file foo2.py] +def foo(arg: str) -> int: + return 12 +class Bar: + def bar(self) -> None: pass +var = 0 +[file bar.py] +from foo import foo +def bar() -> None: + x = foo('abc') # type: str + foo(arg='xyz') + +[case testDaemonInspectCheck] +$ dmypy start +Daemon started +$ dmypy check foo.py +Success: no issues found in 1 source file +$ dmypy check foo.py --export-types +Success: no issues found in 1 source file +$ dmypy inspect foo.py:1:1 +"int" +[file foo.py] +x = 1 + +[case testDaemonInspectRun] +$ dmypy run test1.py +Daemon started +Success: no issues found in 1 source file +$ dmypy run test2.py +Success: no issues found in 1 source file +$ dmypy run test1.py --export-types +Success: no issues found in 1 source file +$ dmypy inspect test1.py:1:1 +"int" +[file test1.py] +a: int +[file test2.py] +a: str + +[case testDaemonGetType] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary --python-version 3.8 +Daemon started +$ dmypy inspect foo:1:2:3:4 +Command "inspect" is only valid after a "check" command (that produces no parse errors) +== Return code: 2 +$ dmypy check foo.py --export-types +foo.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] +== Return code: 1 +$ dmypy inspect foo:1 +Format should be file:line:column[:end_line:end_column] +== Return code: 2 +$ dmypy inspect foo:1:2:3 +Source file is not a Python file +== Return code: 2 +$ dmypy inspect foo.py:1:2:a:b +invalid literal for int() with base 10: 'a' +== Return code: 2 +$ dmypy inspect foo.pyc:1:1:2:2 +Source file is not a Python file +== Return code: 2 +$ dmypy inspect bar/baz.py:1:1:2:2 +Unknown module: bar/baz.py +== Return code: 1 +$ dmypy inspect foo.py:3:1:1:1 +"end_line" must not be before "line" +== Return code: 2 +$ dmypy inspect foo.py:3:3:3:1 +"end_column" must be after "column" +== Return code: 2 +$ dmypy inspect foo.py:3:10:3:17 +"str" +$ dmypy inspect foo.py:3:10:3:17 -vv +"builtins.str" +$ dmypy inspect foo.py:9:9:9:11 +"int" +$ dmypy inspect foo.py:11:1:11:3 +"Callable[[Optional[int]], None]" +$ dmypy inspect foo.py:11:1:13:1 +"None" +$ dmypy inspect foo.py:1:2:3:4 +Can't find expression at span 1:2:3:4 +== Return code: 1 +$ dmypy inspect foo.py:17:5:17:5 +No known type available for "NameExpr" (maybe unreachable or try --force-reload) +== Return code: 1 + +[file foo.py] +from typing import Optional + +x: int = "no way" # line 3 + +def foo(arg: Optional[int] = None) -> None: + if arg is None: + arg + else: + arg # line 9 + +foo( + # multiline +) + +def unreachable(x: int) -> None: + return + x # line 17 + +[case testDaemonGetTypeInexact] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy check foo.py --export-types +$ dmypy inspect foo.py:1:a +invalid literal for int() with base 10: 'a' +== Return code: 2 +$ dmypy inspect foo.pyc:1:2 +Source file is not a Python file +== Return code: 2 +$ dmypy inspect bar/baz.py:1:2 +Unknown module: bar/baz.py +== Return code: 1 +$ dmypy inspect foo.py:7:5 --include-span +7:5:7:5 -> "int" +7:5:7:11 -> "int" +7:1:7:12 -> "None" +$ dmypy inspect foo.py:7:5 --include-kind +NameExpr -> "int" +OpExpr -> "int" +CallExpr -> "None" +$ dmypy inspect foo.py:7:5 --include-span --include-kind +NameExpr:7:5:7:5 -> "int" +OpExpr:7:5:7:11 -> "int" +CallExpr:7:1:7:12 -> "None" +$ dmypy inspect foo.py:7:5 -vv +"builtins.int" +"builtins.int" +"None" +$ dmypy inspect foo.py:7:5 -vv --limit=1 +"builtins.int" +$ dmypy inspect foo.py:7:3 +"Callable[[int], None]" +"None" +$ dmypy inspect foo.py:1:2 +Can't find any expressions at position 1:2 +== Return code: 1 +$ dmypy inspect foo.py:11:5 --force-reload +No known type available for "NameExpr" (maybe unreachable) +No known type available for "OpExpr" (maybe unreachable) +== Return code: 1 + +[file foo.py] +from typing import Optional + +def foo(x: int) -> None: ... + +a: int +b: int +foo(a and b) # line 7 + +def unreachable(x: int, y: int) -> None: + return + x and y # line 11 + +[case testDaemonGetAttrs] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy check foo.py bar.py --export-types +$ dmypy inspect foo.py:9:1 --show attrs --include-span --include-kind -vv +NameExpr:9:1:9:1 -> {"foo.C": ["a", "x", "y"], "foo.B": ["a", "b"]} +$ dmypy inspect foo.py:11:10 --show attrs +No known type available for "StrExpr" (maybe unreachable or try --force-reload) +== Return code: 1 +$ dmypy inspect foo.py:1:1 --show attrs +Can't find any expressions at position 1:1 +== Return code: 1 +$ dmypy inspect --show attrs bar.py:10:1 +{"A": ["z"], "B": ["z"]} +$ dmypy inspect --show attrs bar.py:10:1 --union-attrs +{"A": ["x", "z"], "B": ["y", "z"]} + +[file foo.py] +class B: + def b(self) -> int: return 0 + a: int +class C(B): + a: int + y: int + def x(self) -> int: return 0 + +v: C # line 9 +if False: + "unreachable" + +[file bar.py] +from typing import Union + +class A: + x: int + z: int +class B: + y: int + z: int +var: Union[A, B] +var # line 10 + +[case testDaemonGetDefinition] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy check foo.py bar/baz.py bar/__init__.py --export-types +$ dmypy inspect foo.py:5:1 --show definition +foo.py:4:1:y +$ dmypy inspect foo.py:2:3 --show definition --include-span --include-kind -vv +MemberExpr:2:1:2:7 -> bar/baz.py:3:5:Alias +$ dmypy inspect foo.py:3:1 --show definition +Cannot find definition for "NameExpr" at 3:1:3:1 +== Return code: 1 +$ dmypy inspect foo.py:4:6 --show definition +No name or member expressions at 4:6 +== Return code: 1 +$ dmypy inspect foo.py:7:1:7:6 --show definition +bar/baz.py:4:5:attr +$ dmypy inspect foo.py:10:10 --show definition --include-span +10:1:10:12 -> bar/baz.py:6:1:test +$ dmypy inspect foo.py:14:6 --show definition --include-span --include-kind +NameExpr:14:5:14:7 -> foo.py:13:9:arg +MemberExpr:14:5:14:9 -> bar/baz.py:9:5:x, bar/baz.py:11:5:x + +[file foo.py] +from bar.baz import A, B, C +C.Alias +x # type: ignore +y = 42 +y # line 5 +z = C() +z.attr + +import bar +bar.baz.test() # line 10 + +from typing import Union +def foo(arg: Union[A, B]) -> None: + arg.x + +[file bar/__init__.py] +[file bar/baz.py] +from typing import Union +class C: + Alias = Union[int, str] + attr = 42 + +def test() -> None: ... # line 6 + +class A: + x: int +class B: + x: int + +[case testDaemonInspectSelectCorrectFile] +$ dmypy run test.py --export-types +Daemon started +Success: no issues found in 1 source file +$ dmypy inspect demo/test.py:1:1 +"int" +$ dmypy inspect test.py:1:1 +"str" +[file test.py] +b: str +from demo.test import a +[file demo/test.py] +a: int diff --git a/test-data/unit/deps-classes.test b/test-data/unit/deps-classes.test new file mode 100644 index 000000000000..a8fc5d629491 --- /dev/null +++ b/test-data/unit/deps-classes.test @@ -0,0 +1,250 @@ +-- Test cases for generating fine-grained dependencies for classes. +-- +-- The dependencies are used for fined-grained incremental checking. +-- +-- See the comment at the top of deps.test for more documentation. + +-- TODO: Move class related test cases from deps.test to here + +[case testNamedTuple] +from typing import NamedTuple, Any +from a import A +N = NamedTuple('N', [('a', 'A')]) + +def f(a: Any) -> None: + n = N(a) + n.a +[file a.py] +class A: pass +[builtins fixtures/tuple.pyi] +[out] + -> m.f + -> m.f + -> m.f + -> m.f + -> , , m + -> m + +[case testNamedTuple2] +from typing import NamedTuple, Any, Tuple +from a import A, B +N = NamedTuple('N', [('a', 'Tuple[A, B]')]) + +def f(a: Any) -> None: + n = N(a) + n.a +[file a.py] +class A: pass +class B: pass +[builtins fixtures/tuple.pyi] +[out] + -> m.f + -> m.f + -> m.f + -> m.f + -> , , m + -> , , m + -> m + +[case testNamedTuple3] +from typing import NamedTuple +N = NamedTuple('N', [('x', int)]) +x = N(1) +M = NamedTuple('M', [('z', 'N')]) +y = M(x) +[builtins fixtures/tuple.pyi] +[out] + -> m + -> m + -> , m + -> m + -> m + -> , , , , m + -> m + -> m + +[case testNamedTuple4] +from typing import NamedTuple, Any +from a import A +class N(NamedTuple): + a: A + +def f(a: Any) -> None: + n = N(a) + n.a +[file a.py] +class A: pass +[builtins fixtures/tuple.pyi] +[out] + -> m.f + -> m.f + -> m.f + -> m.N, m.f + -> , , m, m.N + -> m + +[case testIfFalseInClassBody] +class A: + if False: + x = None # type: str + x.foo() +[builtins fixtures/bool.pyi] +[out] + -> m.A + +[case testAlwaysFalseIsinstanceInClassBody] +class A: + x: int + if isinstance(x, str): + y: str = None + y.foo() +[builtins fixtures/isinstance.pyi] +[out] + -> m.A + +[case testDoubleAttributeInitializationToNone] +class C: + def __init__(self) -> None: + self.x = None + self.x = None +[out] + -> m.C.__init__ + -> m.C + +[case testClassNestedWithinFunction] +class C: pass + +def f() -> None: + class S1(C): pass + +class D: + def g(self) -> None: + class S2(C): pass +[out] +-- TODO: Is it okay to have targets like m.S1@4.__init__? + -> , , m.D.g, m.f + -> , + -> , + -> m.C, m.D.g, m.f + -> m.D.g + -> m.D + -> m.f + +[case testClassSuper] +class C: + def __init__(self, x: int) -> None: pass + def foo(self) -> None: pass + +class D(C): + def __init__(self, x: int) -> None: + super().__init__(x) + super().foo() +[out] + -> , m + -> , m.D.__init__ + -> + -> , m.D.__init__ + -> m, m.C, m.D + -> m.D + +[case testClassMissingInit] +class C: + def __init__(self, x: int) -> None: pass + +class D(C): + pass + +def foo() -> None: + D(6) +[out] + -> , m + -> + -> + -> m, m.C, m.D + -> m.foo + -> m.foo + -> m.D, m.foo + +[case testClassBasedEnum] +from enum import Enum +from m import B + +class A(Enum): + X = B() + +def f(a: A) -> None: + pass +def g() -> None: + A.X +[file m.py] +class B: pass +[builtins fixtures/enum.pyi] +[out] + -> m.g + -> , m.A, m.f, m.g + -> m + -> m +-- The dependency target is superfluous but benign + -> , m + -> m + +[case testClassAttribute] +class C: + x = 0 + +def f() -> None: + C.x + +def g() -> None: + C.x = 1 +[out] + -> m.f, m.g + -> m.C, m.f, m.g + +[case testStaticAndClassMethods] +class C: + @staticmethod + def foo() -> None: + h() + + @classmethod + def bar(cls) -> None: + h() + +def fstatic() -> None: + C.foo() + +def fclass() -> None: + C.bar() + +cc = C() + +def gstatic() -> None: + cc.foo() + +def gclass() -> None: + cc.bar() + +def h() -> None: pass +[builtins fixtures/classmethod.pyi] +[out] + -> m + -> m + -> m, m.fclass, m.gclass + -> m, m.fstatic, m.gstatic + -> , m, m.C, m.fclass, m.fstatic + -> m, m.gclass, m.gstatic + -> m.C.bar, m.C.foo + +[case testClassAttributeWithMetaclass] +class M(type): + x = 1 +class C(metaclass=M): + pass +def f() -> None: + C.x +[out] + -> m.f + -> m.C, m.f + -> m.f + -> , m, m.M diff --git a/test-data/unit/deps-expressions.test b/test-data/unit/deps-expressions.test new file mode 100644 index 000000000000..ff8c875b66f0 --- /dev/null +++ b/test-data/unit/deps-expressions.test @@ -0,0 +1,436 @@ +-- Test cases for generating fine-grained dependencies for expressions. +-- +-- The dependencies are used for fined-grained incremental checking. +-- +-- See the comment at the top of deps.test for more documentation. + +[case testListExpr] +def f() -> int: pass +def g() -> None: + a = [f()] +[builtins fixtures/list.pyi] +[out] + -> m.g + +[case testDictExpr] +def f1() -> int: pass +def f2() -> int: pass +def g() -> None: + a = {f1(): 1, 2: f2()} +[builtins fixtures/dict.pyi] +[out] + -> m.g + -> m.g + +[case testSetExpr] +def f() -> int: pass +def g() -> None: + a = {f()} +[builtins fixtures/set.pyi] +[out] + -> m.g + +[case testTupleExpr] +def f1() -> int: pass +def f2() -> int: pass +def g() -> None: + a = (f1(), f2()) +[builtins fixtures/tuple.pyi] +[out] + -> m.g + -> m.g + +[case testListComprehension] +from typing import Iterator + +class A: + def __iter__(self) -> Iterator[int]: pass + +def f1() -> int: pass +def f2() -> int: pass +def g() -> None: + a = [f1() for x in A() if f2()] +[builtins fixtures/list.pyi] +[out] + -> m.g + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.g + +[case testSetComprehension] +from typing import Set +def f1() -> int: pass +def f2() -> Set[int]: pass +def f3() -> int: pass +def g() -> None: + a = {f1() for x in f2() if f3()} +[builtins fixtures/set.pyi] +[out] + -> m.g + -> m.g + -> m.g + +[case testDictComprehension] +from typing import Iterator + +class A: + def __iter__(self) -> Iterator[int]: pass + +def f1() -> int: pass +def f2() -> int: pass +def f3() -> int: pass +def g() -> None: + a = {f1(): f2() for x in A() if f3()} +[builtins fixtures/dict.pyi] +[out] + -> m.g + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.g + -> m.g + +[case testGeneratorExpr] +from typing import List +def f1() -> int: pass +def f2() -> List[int]: pass +def f3() -> int: pass +def g() -> None: + a = (f1() for x in f2() if f3()) +[builtins fixtures/list.pyi] +[out] + -> m.g + -> m.g + -> m.g + +[case testConditionalExpr] +def f1() -> int: pass +def f2() -> int: pass +def f3() -> int: pass +def g() -> None: + a = f1() if f2() else f3() +[out] + -> m.g + -> m.g + -> m.g + +[case testAwaitExpr] +def f(): pass +async def g() -> None: + x = await f() +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] + -> m.g + +[case testStarExpr] +from typing import Iterator + +class A: + def __iter__(self) -> Iterator[int]: pass + +def g() -> None: + a = [*A()] +[builtins fixtures/list.pyi] +[out] + -> m.g + -> , m.g + -> m.g + -> m.A, m.g, typing.Iterable + +[case testCast] +from typing import cast +class A: pass +def f() -> object: pass +def g() -> None: + x = cast(A, f()) +[out] + -> m.A, m.g + -> m.g + +[case testTypeApplication] +from typing import TypeVar, Generic + +T = TypeVar('T') +S = TypeVar('S') + +class A(Generic[T, S]): + def __init__(self, x): pass +class B: pass +class C: pass + +def f() -> int: pass + +def g() -> None: + x = A[B, C](f()) +[out] + -> m.g + -> m.g + -> m.A, m.g + -> m.B, m.g + -> m.C, m.g + -> m.A + -> m.A + -> m.g + +[case testIndexExpr] +class A: + def __getitem__(self, x: int) -> int: pass + +def f1() -> A: pass +def f2() -> int: pass + +def g(a: A) -> int: + return f1()[f2()] +[out] + -> m.g + -> , , m.A, m.f1, m.g + -> m.g + -> m.g + +[case testIndexExpr2] +class A: + def __getitem__(self, x: int) -> int: pass + +def f1() -> A: pass +def f2() -> int: pass + +def g(a: A) -> int: + return f1()[f2()] +[out] + -> m.g + -> , , m.A, m.f1, m.g + -> m.g + -> m.g + +[case testIndexExprLvalue] +class A: + def __setitem__(self, x: int, y: int) -> None: pass + +def f1() -> A: pass +def f2() -> int: pass +def f3() -> int: pass + +def g(a: A) -> None: + f1()[f2()] = f3() +[out] +-- __getitem__ dependency is redundant but harmless + -> m.g + -> m.g + -> , , m.A, m.f1, m.g + -> m.g + -> m.g + -> m.g + +[case testUnaryExpr] +class A: + def __neg__(self) -> int: pass + def __pos__(self) -> int: pass + def __invert__(self) -> int: pass + +def f1() -> A: pass +def f2() -> A: pass +def f3() -> A: pass + +def g1() -> int: + return +f1() +def g2() -> int: + return -f2() +def g3() -> int: + return ~f3() +[out] + -> m.g3 + -> m.g2 + -> m.g1 + -> , , , m.A, m.f1, m.f2, m.f3 + -> m.g1 + -> m.g2 + -> m.g3 + +[case testOpExpr] +class A: + def __add__(self, x: 'B') -> int: pass +class B: pass + +def f() -> int: + a: A + b: B + return a + b +[out] + -> m.f + -> m.A, m.f + -> m.f + -> , m.A.__add__, m.B, m.f + +[case testComparisonExpr] +class A: + def __lt__(self, x: 'B') -> int: pass +class B: pass + +def f() -> int: + return A() < B() +[out] + -> m.f + -> m.f + -> m.f + -> m.A, m.f + -> m.f + -> m.f + -> m.f + -> , m.A.__lt__, m.B, m.f + +[case testIsOp] +class A: pass +class B: pass + +def f() -> bool: + return A() is B() +[builtins fixtures/bool.pyi] +[out] + -> m.f + -> m.f + -> m.A, m.f + -> m.f + -> m.f + -> m.B, m.f + +[case testInOp] +class A: + def __contains__(self, x: B) -> int: pass +class B: pass + +def f() -> bool: + return B() in A() +[builtins fixtures/bool.pyi] +[out] + -> m.f + -> m.f + -> m.f + -> m.A, m.f + -> m.f + -> m.f + -> , m.A.__contains__, m.B, m.f + +[case testComparisonExprWithMultipleOperands] +class A: + def __lt__(self, x: 'B') -> int: pass +class B: pass +class C: + def __ge__(self, x: 'B') -> int: pass + +def f() -> int: + return A() < B() <= C() +[out] + -> m.f + -> m.f + -> m.f + -> m.A, m.f + -> m.f + -> m.f + -> m.f + -> m.f + -> , , m.A.__lt__, m.B, m.C.__ge__, m.f + -> m.f + -> m.f + -> m.f + -> m.C, m.f + +[case testOperatorWithTupleOperand] +from typing import Tuple + +class C(Tuple[int, str]): + def __and__(self, x: D) -> int: pass + def __neg__(self) -> int: pass +class D: pass + +def f() -> None: + c: C + d: D + x = c & d + y = -c +[builtins fixtures/tuple.pyi] +[out] + -> m.f + -> m.f + -> m.C, m.f + -> m.f + -> , m.C.__and__, m.D, m.f + +[case testUnionTypeOperation] +from typing import Union + +class A: + def __add__(self, x: str) -> int: pass +class B: + def __add__(self, x: str) -> int: pass + +def f(a: Union[A, B]) -> int: + return a + '' +[out] + -> m.f + -> , m.A, m.f + -> m.f + -> , m.B, m.f + +[case testSliceExpr] +class A: + def __getitem__(self, x) -> None: pass + +def f1() -> int: pass +def f2() -> int: pass +def f3() -> int: pass +def f4() -> int: pass +def f5() -> int: pass + +def f() -> None: + a: A + a[f1():f2():f3()] + a[f4():] + a[::f5()] +[builtins fixtures/slice.pyi] +[out] + -> m.f + -> m.A, m.f + -> m.f + -> m.f + -> m.f + -> m.f + -> m.f + +[case testRevealTypeExpr] +def f1() -> int: pass +def f() -> None: + reveal_type(f1()) # type: ignore +[out] + -> m.f + +[case testLambdaExpr] +from typing import Callable + +def f1(c: Callable[[int], str]) -> None: pass +def f2() -> str: pass + +def g() -> None: + f1(lambda x: f2()) +[out] + -> m.g + -> m.g + +[case testLiteralDepsExpr] +from typing_extensions import Literal + +Alias = Literal[1] + +a: Alias +b = a +def f(x: Alias) -> None: pass +def g() -> Literal[1]: + return b +[builtins fixtures/tuple.pyi] +[out] + -> , m, m.f + -> m + -> m, m.g diff --git a/test-data/unit/deps-generics.test b/test-data/unit/deps-generics.test new file mode 100644 index 000000000000..6baa57266d2f --- /dev/null +++ b/test-data/unit/deps-generics.test @@ -0,0 +1,204 @@ +-- Test cases for generating fine-grained dependencies involving generics. +-- +-- The dependencies are used for fined-grained incremental checking. +-- +-- See the comment at the top of deps.test for more documentation. + +[case testGenericFunction] +from typing import TypeVar + +T = TypeVar('T') + +class A: pass + +def f(x: T) -> T: + y: T + z: A + return x +[out] + -> m.A, m.f + -> , m.f + +[case testGenericClass] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): pass +class B: pass + +def f() -> None: + a: A[B] +[out] + -> m.A, m.f + -> m.B, m.f + -> m.A + +[case testGenericClassWithMembers] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): + def g(self, a: T) -> None: + self.x = a + def f(self) -> T: + return self.x +[out] + -> m.A.f, m.A.g + -> m.A + -> , , , m.A, m.A.f, m.A.g + +[case testGenericClassInit] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): + def __init__(self, a: T) -> None: + self.x = a + +class B: pass + +def f() -> None: + a = A(B()) +[out] + -> m.f + -> m.f + -> m.A.__init__ + -> m.A, m.f + -> m.f + -> m.f + -> m.B, m.f + -> , , m.A, m.A.__init__ + +[case testGenericMethod] +from typing import TypeVar + +T = TypeVar('T') + +class A: + def f(self, x: T) -> T: + return x +[out] + -> m.A + -> , m.A.f + +[case testGenericBaseClass] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): pass +class B(A[C]): pass +class C: pass +[out] + -> , m + -> + -> + -> m, m.A, m.B + -> m.B + -> m, m.B, m.C + -> m.A + +[case testGenericBaseClass2] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class A(Generic[T]): pass +class B(A[T]): pass +[out] + -> , m + -> + -> + -> m, m.A, m.B + -> m.B + -> m, m.A, m.B + +[case testTypeVarBound] +from typing import TypeVar, Tuple + +class A: pass +class B: pass + +T = TypeVar('T', bound=Tuple[A, B]) + +def f(x: T) -> T: + return x +[builtins fixtures/tuple.pyi] +[out] + -> , , m, m.A, m.f + -> , , m, m.B, m.f + -> , m.f + +[case testTypeVarBoundOperations] +from typing import TypeVar, Tuple + +class A: + def f(self) -> None: pass + def __add__(self, other: int) -> int: pass + +T = TypeVar('T', bound=A) + +def f(x: T) -> None: + x.f() + x + 1 +[out] + -> m.f + -> m.f + -> , , m, m.A, m.f + -> , m.f + +[case testTypeVarValues] +from typing import TypeVar +class A: pass +class B: pass +class C: pass +class D: pass +T = TypeVar('T', A, B) +S = TypeVar('S', C, D) + +def f(x: T, y: S) -> S: + pass +[out] + -> , , m, m.A, m.f + -> , , m, m.B, m.f + -> , , m, m.C, m.f + -> , , m, m.D, m.f + -> , m.f + -> , m.f + +[case testTypeVarValuesMethod] +from typing import TypeVar, Generic +class C: pass +class D: pass +S = TypeVar('S', C, D) + +class G(Generic[S]): + def f(self) -> S: + pass +[out] + -> , , m, m.C, m.G.f + -> , , m, m.D, m.G.f + -> m.G + -> , m.G, m.G.f + +[case testTypeVarValuesMethodAttr] +from typing import TypeVar, Generic +class A: + x: int +class B: + x: int +T = TypeVar('T', A, B) + +class G(Generic[T]): + def f(self, x: T) -> None: + x.x +[out] + -> m.G.f + -> , , m, m.A, m.G.f + -> m.G.f + -> , , m, m.B, m.G.f + -> m.G + -> , m.G, m.G.f diff --git a/test-data/unit/deps-statements.test b/test-data/unit/deps-statements.test new file mode 100644 index 000000000000..a67f9c762009 --- /dev/null +++ b/test-data/unit/deps-statements.test @@ -0,0 +1,634 @@ +-- Test cases for generating fine-grained dependencies for statements. +-- +-- The dependencies are used for fined-grained incremental checking. +-- +-- See the comment at the top of deps.test for more documentation. + +[case testIfStmt] +def f1() -> int: pass +def f2() -> None: pass +def f3() -> int: pass +def f4() -> None: pass +def f5() -> None: pass + +def g() -> None: + if f1(): + f2() + elif f3(): + f4() + else: + f5() +[out] + -> m.g + -> m.g + -> m.g + -> m.g + -> m.g + +[case testWhileStmt] +def f1() -> int: pass +def f2() -> None: pass +def f3() -> None: pass + +def g() -> None: + while f1(): + f2() + else: + f3() +[out] + -> m.g + -> m.g + -> m.g + +[case testAssertStmt] +def f1() -> int: pass +def f2() -> str: pass +def f3() -> int: pass + +def g() -> None: + assert f1(), f2() + assert f3() +[out] + -> m.g + -> m.g + -> m.g + +[case testRaiseStmt] +def f1() -> BaseException: pass +def f2() -> BaseException: pass + +def g1() -> None: + raise f1() + +def g2() -> None: + raise f1() from f2() +[builtins fixtures/exception.pyi] +[out] + -> m.g1, m.g2 + -> m.g2 + +[case testTryFinallyStmt] +def f1() -> None: pass +def f2() -> None: pass + +def g() -> None: + try: + f1() + finally: + f2() +[out] + -> m.g + -> m.g + +[case testForStmt] +from typing import Iterator + +class A: + def __iter__(self) -> Iterator[int]: pass + +def f1() -> None: pass +def f2() -> None: pass + +def g() -> None: + a: A + for x in a: + f1() + else: + f2() +[builtins fixtures/list.pyi] +[out] + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.g + +[case testTryExceptStmt] +class A(BaseException): pass +class B(BaseException): + def f(self) -> None: pass + +def f1() -> None: pass +def f2() -> None: pass +def f3() -> None: pass + +def g() -> None: + try: + f1() + except A: + f2() + except B as e: + e.f() + else: + f3() +[builtins fixtures/exception.pyi] +[out] +-- The dependencies on the ctor are basically spurious but not a problem + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.g + -> m.g + -> m.B, m.g + -> m.g + -> m.g + -> m.g + +[case testTryExceptStmt2] +class A(BaseException): pass +class B(BaseException): + def f(self) -> None: pass + +def f1() -> None: pass +def f2() -> None: pass + +def g() -> None: + try: + f1() + except (A, B): + f2() +[builtins fixtures/exception.pyi] +[out] +-- The dependencies on the ctor are basically spurious but not a problem + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.g + -> m.B, m.g + -> m.g + -> m.g + +[case testWithStmt] +from typing import Any +class A: + def __enter__(self) -> 'B': pass + def __exit__(self, a, b, c) -> None: pass + +class B: + def f(self) -> None: pass + +def g() -> None: + a: A + with a as x: + x.f() +[out] + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> , m.A.__enter__, m.B + +[case testWithStmt2] +from typing import Any +class A: + def __enter__(self) -> 'C': pass + def __exit__(self, a, b, c) -> None: pass +class B: + def __enter__(self) -> 'D': pass + def __exit__(self, a, b, c) -> None: pass + +class C: pass +class D: pass + +def g() -> None: + a: A + b: B + with a as x, b as y: + pass +[out] + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.g + -> m.B, m.g + -> , m.A.__enter__, m.C + -> , m.B.__enter__, m.D + +[case testWithStmtAnnotation] +from typing import Any +class A: + def __enter__(self) -> Any: pass + def __exit__(self, a, b, c) -> None: pass + +class B: pass + +def f(b: B) -> None: pass + +def g() -> None: + a: A + with a as x: # type: B + f(x) +[out] + -> m.g + -> m.g + -> m.A, m.g + -> , m.B, m.f, m.g + -> m.g + +[case testForStmtAnnotation] +class A: + def __iter__(self): pass + +class B: + def f(self) -> None: pass + +def g() -> None: + a: A + for x in a: # type: B + x.f() +[builtins fixtures/list.pyi] +[out] + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.B, m.g + +[case testMultipleAssignment] +from typing import Iterator +class A: + def __iter__(self) -> Iterator[int]: pass + +def f() -> None: + a: A + x, y = a +[out] + -> , m.f + -> m.A, m.f, typing.Iterable + +[case testMultipleLvalues] +class A: + def f(self) -> None: + self.x = 1 + self.y = 1 + +def g() -> None: + a: A + a.x = a.y = 1 +[out] + -> m.A.f, m.g + -> m.A.f, m.g + -> m.A, m.g + +[case testNestedLvalues] +class A: + def f(self) -> None: + self.x = 1 + self.y = '' + +def g() -> None: + a: A + a.x, a.y = 1, '' +[out] + -> m.A.f, m.g + -> m.A.f, m.g + -> m.A, m.g + +[case testForAndSetItem] +class A: + def __setitem__(self, x: int, y: int) -> None: pass + +def f(): pass + +def g() -> None: + a: A + for a[0] in f(): + pass +[builtins fixtures/list.pyi] +[out] + -> m.g + -> m.g + -> m.A, m.g + -> m.g + +[case testMultipleAssignmentAndFor] +from typing import Iterator, Iterable + +class A: + def f(self) -> None: + self.x = 1 + self.y = 1 + +class B: + def __iter__(self) -> Iterator[int]: pass + +def f() -> Iterable[B]: pass + +def g() -> None: + a: A + for a.x, a.y in f(): + pass +[builtins fixtures/list.pyi] +[out] + -> m.A.f, m.g + -> m.A.f, m.g + -> m.A, m.g + -> m.g + -> , m.g + -> , m.B, m.f, typing.Iterable + -> m.g + +[case testNestedSetItem] +class A: + def __setitem__(self, x: int, y: int) -> None: pass +class B: + def __setitem__(self, x: int, y: int) -> None: pass + +def f(): pass + +def g() -> None: + a: A + b: B + a[0], b[0] = f() +[out] + -> m.g + -> m.g + -> m.A, m.g + -> m.g + -> m.g + -> m.B, m.g + -> m.g + +[case testOperatorAssignmentStmt] +class A: + def __add__(self, other: 'B') -> 'A': pass + +class B: pass + +def f() -> B: pass + +def g() -> None: + a: A + a += f() +[out] + -> m.g + -> m.g + -> , m.A, m.A.__add__, m.g + -> , , m.A.__add__, m.B, m.f + -> m.g + +[case testOperatorAssignmentStmtSetItem] +class A: + def __add__(self, other: 'B') -> 'A': pass + +class B: pass + +class C: + def __getitem__(self, x: int) -> A: pass + def __setitem__(self, x: int, y: A) -> None: pass + +def f() -> int: pass + +def g() -> None: + b: B + c: C + c[f()] += b +[out] + -> m.g + -> m.g + -> , , , m.A, m.A.__add__, m.C.__getitem__, m.C.__setitem__ + -> , m.A.__add__, m.B, m.g + -> m.g + -> m.g + -> m.C, m.g + -> m.g + +[case testYieldStmt] +from typing import Iterator + +class A: pass + +def f1() -> A: pass + +def g() -> Iterator[A]: + yield f1() +[builtins fixtures/list.pyi] +[out] + -> , , m.A, m.f1, m.g + -> m.g + +[case testDelStmt] +class A: + def f(self) -> None: + self.x = 1 + +def f() -> A: pass + +def g() -> None: + del f().x +[out] + -> m.A.f, m.g + -> , m.A, m.f + -> m.g + +[case testDelStmtWithIndexing] +class A: + def __delitem__(self, x: int) -> None: pass + +def f1() -> A: pass +def f2() -> int: pass + +def g() -> None: + del f1()[f2()] +[out] + -> m.g +-- __getitem__ is redundant but harmless + -> m.g + -> , m.A, m.f1 + -> m.g + -> m.g + +[case testYieldFrom] +from typing import Iterator +class A: + def __iter__(self) -> Iterator[int]: pass +def f() -> Iterator[int]: + yield from A() +[out] + -> m.f + -> , m.f + -> m.f + -> m.A, m.f, typing.Iterable + +[case testFunctionDecorator] +from typing import Callable +def dec(f: Callable[[int], int]) -> Callable[[str], str]: + pass + +def f() -> int: pass + +@dec +def g(x: int) -> int: + return f() +[out] + -> m + -> m.g + -> m + +[case testMethodDecorator] +from typing import Callable, Any +def dec(f: Callable[[Any, int], int]) -> Callable[[Any, str], str]: + pass + +def f() -> int: pass + +class A: + @dec + def g(self, x: int) -> int: + return f() +[out] + -> m + -> m.A + -> m + -> m.A.g + +[case testNestedFunction] +class A: pass + +def h() -> None: pass + +def f() -> None: + def g(x: A) -> None: + h() +[out] + -> , m.A, m.f + -> m.f + +[case testPlatformCheck] +import a +import sys + +def f() -> int: + if sys.platform == 'nonexistent': + return a.g() + else: + return 1 +[file a.py] +[builtins fixtures/ops.pyi] +[out] + -> m + -> m.f + -> m, m.f + +[case testOverload] +from typing import overload + +class A: pass +class B(A): pass + +@overload +def f(x: B) -> B: ... +@overload +def f(x: A) -> A: ... + +def f(x: A) -> A: + g() + return x + +def g() -> None: pass +[builtins fixtures/isinstancelist.pyi] +[out] + -> , m + -> + -> + -> , m, m.A, m.B, m.f + -> , m.B, m.f + -> m.f + +[case testOverloadedMethod] +from typing import overload + +class A: pass +class B(A): pass + +class C: + @overload + def f(self, x: B) -> B: ... + @overload + def f(self, x: A) -> A: ... + + def f(self, x: A) -> A: + self.g() + return x + + def g(self) -> None: pass +[builtins fixtures/isinstancelist.pyi] +[out] + -> , m + -> + -> + -> , m, m.A, m.B, m.C.f + -> , m.B, m.C.f + -> m.C.f + -> m.C + +[case testConditionalFunctionDefinition] +import sys + +class A: pass +class B: pass + +if sys.platform == 'nonexistent': + def f(x: A) -> None: + g() +else: + def f(x: B) -> None: + h() + +def g() -> None: pass +def h() -> None: pass +[builtins fixtures/ops.pyi] +[out] + -> m.A + -> , m.B, m.f + -> m.f + -> m + -> m + +[case testConditionalMethodDefinition] +import sys + +class A: pass +class B: pass + +class C: + if sys.platform == 'nonexistent': + def f(self, x: A) -> None: + self.g() + else: + def f(self, x: B) -> None: + self.h() + + def g(self) -> None: pass + def h(self) -> None: pass +[builtins fixtures/ops.pyi] +[out] + -> m.A + -> , m.B, m.C.f + -> m.C.f + -> m.C + -> m + -> m + +[case testNewType] +from typing import NewType +from m import C + +N = NewType('N', C) + +def f(n: N) -> None: + pass +[file m.py] +class C: + x: int +[out] + -> , m, m.f + -> , m + -> + -> + -> + -> m, m.N + -> m diff --git a/test-data/unit/deps-types.test b/test-data/unit/deps-types.test new file mode 100644 index 000000000000..6992a5bdec00 --- /dev/null +++ b/test-data/unit/deps-types.test @@ -0,0 +1,1026 @@ +-- Test cases for generating fine-grained dependencies between types. +-- +-- The dependencies are used for fined-grained incremental checking. +-- +-- See the comment at the top of deps.test for more documentation. + +[case testFilterOutBuiltInTypes] +class A: pass + +def f(x: int, y: str, z: A) -> None: + pass +[out] + -> , m.A, m.f + +[case testTupleType] +from typing import Tuple + +class A: pass +class B: pass + +def f(x: Tuple[A, B]) -> None: + pass +[builtins fixtures/tuple.pyi] +[out] + -> , m.A, m.f + -> , m.B, m.f + +[case testUnionType] +from typing import Union + +class A: pass +class B: pass + +def f() -> None: + x: Union[int, A, B] +[out] + -> m.A, m.f + -> m.B, m.f + +[case testCallableType] +from typing import Callable + +class A: pass +class B: pass + +def f() -> None: + x: Callable[[int, A], None] + y: Callable[[int, str], B] +[out] + -> m.A, m.f + -> m.B, m.f + +[case testTypeType] +from typing import Type + +class A: pass + +def f() -> None: + x: Type[A] + y: Type[int] +[out] + -> m.f + -> m.f + -> m.A, m.f + +[case testTypeTypeAttribute] +from typing import Type + +class A: + @staticmethod + def f() -> None: pass + +def f(x: Type[A]) -> None: + x.f() +[builtins fixtures/staticmethod.pyi] +[out] + -> , m.f + -> , m.f + -> m, m.f + -> , m.A, m.f + +[case testComplexNestedType] +from typing import Union, Callable, Type + +class A: pass +class B: pass +class C: pass + +def f() -> None: + x: Union[int, Callable[[Type[A]], B], C] +[out] + -> m.f + -> m.f + -> m.A, m.f + -> m.B, m.f + -> m.C, m.f + +[case testUnionTypeAttributeAccess] +from typing import Union + +class A: + def f(self) -> None: + self.x = 0 + +class B: + def f(self) -> None: + self.x = '' + +def f(a: Union[A, B]) -> None: + a.x + a.f() +[out] + -> m.f + -> m.A.f, m.f + -> , m.A, m.f + -> m.f + -> m.B.f, m.f + -> , m.B, m.f + +[case testTupleTypeAttributeAccess] +from typing import Tuple + +class C(Tuple[int, str]): + def f(self) -> None: pass + +def f(c: C) -> None: + c.f() +[builtins fixtures/tuple.pyi] +[out] + -> m.f + -> , m.C, m.f + +[case testOverloaded] +from typing import overload + +class A: pass +class B: pass + +def g() -> None: pass + +@overload +def f(x: A) -> A: pass +@overload +def f(x: B) -> B: pass + +def f(x): + g() + +ff = f + +def h() -> None: + f(A()) + ff(A()) +[out] + -> m.h + -> m.h + -> , , m.A, m.f, m.h + -> , , m.B, m.f + -> m, m.h + -> m, m.h + -> m.f + +[case testMetaclassAttributes] +from mod import C +from typing import Type +def f(arg: Type[C]) -> None: + arg.x +[file mod.py] +class M(type): + x: int +class C(metaclass=M): + pass +[out] + -> , m.f + -> , m.f + -> m.f + -> , m, m.f + -> m.f + -> m + +[case testMetaclassAttributesDirect] +from mod import C +def f() -> None: + C.x +[file mod.py] +class M(type): + x: int +class C(metaclass=M): + pass +[out] + -> m.f + -> m, m.f + -> m.f + -> m + +[case testMetaclassOperators] +from mod import C +from typing import Type +def f(arg: Type[C]) -> None: + arg + arg +[file mod.py] +class M(type): + def __add__(self, other: M) -> M: + pass +class C(metaclass=M): + pass +[out] + -> , m.f + -> , m.f + -> , m, m.f + -> m.f + -> m.f + -> m + +[case testMetaclassOperatorsDirect] +from mod import C +def f() -> None: + C + C +[file mod.py] +class M(type): + def __add__(self, other: M) -> M: + pass +class C(metaclass=M): + pass +[out] + -> m.f + -> m.f + -> m, m.f + -> m.f + -> m.f + -> m + +[case testMetaclassDepsDeclared] +import mod +class C(metaclass=mod.M): + pass +[file mod.py] +class M(type): + pass +[out] + -> m.C + -> , m + -> m + +[case testMetaclassDepsDeclaredNested] +import mod + +def func() -> None: + class C(metaclass=mod.M): + pass +[file mod.py] +class M(type): + pass +[out] + -> m.func + -> , m.func + -> m, m.func + +-- Type aliases + +[case testAliasDepsNormalMod] +from mod import I +A = I +x: A +[file mod.py] +class I: pass +[out] + -> m + -> m + -> m + -> m + -> , m + -> m + +[case testAliasDepsNormalModExtended] +# __dump_all__ +import a +x: a.A +[file a.py] +from mod import I +A = I +[file mod.py] +class I: pass +[out] + -> m + -> m + -> m + -> a + -> a + -> , m, a, mod.I + -> a + +[case testAliasDepsNormalFunc] +from mod import I +A = I +def f(x: A) -> None: + pass +[file mod.py] +class I: pass +[out] + -> m.f + -> m + -> m + -> , m, m.f + -> m + +[case testAliasDepsNormalFuncExtended] +# __dump_all__ +import a +def f(x: a.A) -> None: + pass +[file a.py] +from mod import I +A = I +[file mod.py] +class I: pass +[out] + -> m.f + -> m + -> a + -> a + -> , m.f, a, mod.I + -> a + +[case testAliasDepsNormalClass] +from a import A +class C: + x: A +[file a.py] +from mod import I +A = I +[file mod.py] +class I: pass +[out] + -> m.C + -> m + -> m + -> , m + +[case testAliasDepsNormalClassBases] +from a import A +class C(A): + pass +[file a.py] +from mod import I +A = I +[file mod.py] +class I: pass +[out] + -> m.C + -> m + -> m + -> , m + -> + -> + -> m.C + +[case testAliasDepsGenericMod] +from mod import I, S, D +A = D[I, S] +x: A +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> m + -> m + -> m + -> m + -> , m + -> , m + -> , m + -> m + +[case testAliasDepsGenericFunc] +from mod import I, S, D +A = D[S, I] +def f(x: A) -> None: + pass +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> , m.f + -> m + -> m + -> , m, m.f + -> , m, m.f + -> , m, m.f + -> m + +[case testAliasDepsGenericFuncExtended] +import a +def f(x: a.A) -> None: + pass +[file a.py] +from mod import I, S, D +A = D[S, I] +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> , m.f + -> m + -> , m.f + -> , m.f + -> , m.f + +[case testAliasDepsGenericClass] +from mod import I, D, S, T +A = D[S, T] +class C: + x: A[I] +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> m + -> m.C + -> m + -> m + -> , m + -> , m + -> , m + -> m + -> m + +[case testAliasDepsForwardMod] +from mod import I +x: A +A = I +[file mod.py] +from typing import TypeVar, Generic +class I: pass +[out] + -> m + -> m + -> m + -> m + -> , m + -> m + +[case testAliasDepsForwardFunc] +from mod import I +def f(x: A) -> None: + pass +A = I +[file mod.py] +class I: pass +[out] + -> m.f + -> m + -> m + -> , m, m.f + -> m + +[case testAliasDepsForwardClass] +from mod import I +class C: + x: A +A = I +[file mod.py] +class I: pass +[out] + -> m + -> m.C + -> m + -> m + -> , m + -> m + +[case testAliasDepsChainedMod] +from mod import I +A = I +B = A +x: B +[file mod.py] +class I: pass +[out] + -> m + -> m + -> m + -> m + -> m + -> , m + -> m + +[case testAliasDepsChainedFunc] +from mod import I +A = I +B = A +def f(x: B) -> None: + pass +[file mod.py] +class I: pass +[out] + -> m + -> m.f + -> m + -> m + -> , m, m.f + -> m + +[case testAliasDepsChainedFuncExtended] +import a +B = a.A +def f(x: B) -> None: + pass +[file a.py] +from mod import I +A = I +[file mod.py] +class I: pass +[out] + -> m.f + -> m + -> m + -> m + -> m + -> , m, m.f + +[case testAliasDepsChainedClass] +from mod import I +A = I +B = A +class C(B): + pass +[file mod.py] +class I: pass +[out] + -> m + -> m + -> m.C + -> , m + -> , m + -> , m + -> m, m.C + -> m + +[case testAliasDepsNestedMod] +from mod import I, S, D +A = D[S, I] +B = D[S, A] +x: B +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> , m + -> m + -> m + -> m + -> m + -> , m + -> , m + -> , m + -> m + +[case testAliasDepsNestedModExtended] +# __dump_all__ +from mod import S, D +import a +B = D[S, a.A] +x: B +[file a.py] +from mod import I, S, D +A = D[S, I] +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> m + -> m + -> , m + -> m + -> m, a + -> m, a + -> , m, a, mod.D + -> , m, a, mod.I + -> , m, a, mod.S + -> mod.D + -> mod.D + -> m, a + +[case testAliasDepsNestedFunc] +from mod import I, S, D +A = D[S, I] +B = D[S, A] +def f(x: B) -> None: + pass +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> , m, m.f + -> , m.f + -> m + -> m + -> , m, m.f + -> , m, m.f + -> , m, m.f + -> m + +[case testAliasDepsNestedFuncExtended] +# __dump_all__ +from mod import S, D +import a +B = D[S, a.A] +def f(x: B) -> None: + pass +[file a.py] +from mod import I, S, D +A = D[S, I] +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> , m.f + -> , m, m.f + -> m + -> m, a + -> m, a + -> , m, m.f, a, mod.D + -> , m, m.f, a, mod.I + -> , m, m.f, a, mod.S + -> mod.D + -> mod.D + -> m, a + +[case testAliasDepsNestedFuncDirect] +from mod import I, S, D +A = D[S, I] +E = D +def f(x: E[S, A]) -> None: + pass +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> , m.f + -> m.f + -> m + -> m + -> , m, m.f + -> , m, m.f + -> , m, m.f + -> m + -> m + -> m + +[case testAliasDepsNestedClass] +from mod import I, S, D +A = D[S, I] +B = D[S, A] +class C: + x: B +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> , m + -> m + -> m.C + -> m + -> m + -> , m + -> , m + -> , m + -> m + +[case testAliasDepsCast] +from typing import cast +from mod import I +A = I +def fun() -> None: + x = cast(A, 42) +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> m.fun + -> m + -> m + -> m, m.fun + -> m + +[case testAliasDepsRuntime] +from mod import I, S, D +A = I +x = D[S, A]() +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> m + -> m + -> m + -> m + -> , m + -> m + -> m + -> , m + -> , m + -> m + +[case testAliasDepsRuntimeExtended] +# __dump_all__ +from mod import I, S, D +import a +x = D[S, a.A]() +[file a.py] +from mod import I +A = I +[file mod.py] +from typing import TypeVar, Generic +T = TypeVar('T') +U = TypeVar('U') +class D(Generic[T, U]): pass +class I: pass +class S: pass +[out] + -> m + -> m + -> m + -> m + -> m + -> , m, mod.D + -> a + -> a + -> , m, a, mod.I + -> , m, mod.S + -> mod.D + -> mod.D + -> m, a + +[case testAliasDepsNamedTuple] +from typing import NamedTuple +from mod import I +A = I +class P(NamedTuple): + x: A +[file mod.py] +class I: pass +[builtins fixtures/tuple.pyi] +[out] + -> m + -> m.P + -> m + -> m + -> , , m, m.P + -> m + +[case testAliasDepsNamedTupleFunctional] +# __dump_all__ +from typing import NamedTuple +import a +P = NamedTuple('P', [('x', a.A)]) +[file a.py] +from mod import I +A = I +[file mod.py] +class I: pass +[builtins fixtures/tuple.pyi] +[out] + -> m + -> m + -> a + -> a + -> , , m, a, mod.I + -> a + +[case testAliasDepsTypedDict] +from mypy_extensions import TypedDict +from mod import I +A = I +class P(TypedDict): + x: A +[file mod.py] +class I: pass +[builtins fixtures/dict.pyi] +[out] + -> m + -> m.P + -> m + -> m + -> , m + -> m + +[case testAliasDepsTypedDictFunctional] +# __dump_all__ +from mypy_extensions import TypedDict +import a +P = TypedDict('P', {'x': a.A}) +[file a.py] +from mod import I +A = I +[file mod.py] +class I: pass +[builtins fixtures/dict.pyi] +[out] + -> m + -> m + -> a + -> a + -> , a, mod.I + -> a + +[case testAliasDepsClassInFunction] +from mod import I +A = I +def f() -> None: + class C: + x: A +[file mod.py] +class I: pass +[out] + -> m.f + -> m.f + -> m + -> m + -> , m, m.f + -> m + +[case testAliasDepsFromImportUnqualified] +from a import C +x: C +def f() -> None: + C() +class A: + def meth(self) -> None: + def inner() -> C: + pass +[file a.py] +from b import D +C = D +[file b.py] +class D: + pass +[out] + -> m.A + -> m + -> m, m.A.meth, m.f + -> m + -> m.f + -> m.f + -> , , m, m.A.meth + +[case testFuncBasedEnum] +from enum import Enum +from mod import B + +A = Enum('A', [('X', B())]) + +def f(a: A) -> None: + pass +def g() -> None: + A.X +[file mod.py] +class B: pass +[builtins fixtures/tuple.pyi] +[out] + -> m.g + -> , m.f, m.g + -> m + -> m + +[case testProtocolDepsWildcard] +# __dump_all__ +import mod +x: mod.P +[file mod.py] +from typing import Protocol +class P(Protocol): + x: int +[out] + -> m + -> , m, mod.P + -> + -> m + +[case testProtocolDepsWildcardSupertype] +# __dump_all__ +import mod +x: mod.P +[file mod.py] +from typing import Protocol +class PBase(Protocol): + x: int +class P(PBase, Protocol): + y: int +[out] + -> m + -> mod + -> , m, mod.P + -> , mod + -> + -> + -> + -> + -> mod, mod.P, mod.PBase + -> , + -> + -> m + +[case testProtocolDepsPositive] +# __dump_all__ +import mod +class C: + x: int +x: mod.P = C() +[file mod.py] +from typing import Protocol +class P(Protocol): + x: int +[out] + -> m + -> m + -> + -> m, m.C, mod.P + -> m + -> , m, mod.P + -> + -> m + +[case testProtocolDepsNegative] +# __dump_all__ +import mod +from typing import overload +class C: + y: int +@overload +def func(x: C) -> int: ... +@overload +def func(x: mod.P) -> str: ... +def func(x): + pass + +func(C()) +[file mod.py] +from typing import Protocol +class P(Protocol): + x: int +[out] + -> m + -> m + -> + -> , m, m.C, m.func, mod.P + -> m + -> , m.func, mod.P + -> + -> m + +[case testProtocolDepsConcreteSuperclass] +# __dump_all__ +import mod +class B: + x: int +class C(B): + pass +x: mod.P = C() +[file mod.py] +from typing import Protocol +class P(Protocol): + x: int +[out] + -> , m + -> + -> + -> , + -> m, m.B, m.C + -> m + -> m + -> + -> m, m.C, mod.P + -> m + -> , m, mod.P + -> + -> m diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test new file mode 100644 index 000000000000..84cea99bf2f6 --- /dev/null +++ b/test-data/unit/deps.test @@ -0,0 +1,1455 @@ +-- Test cases for generating dependencies between ASTs nodes. +-- +-- The dependencies are used for fined-grained incremental checking and +-- the daemon mode. +-- +-- The output of each test case includes the dependency map for whitelisted +-- modules (includes the main module and the modules 'pkg' and 'pkg.mod' at +-- least). +-- +-- Dependencies are formatted as " -> affected locations". +-- +-- Look at the docstring of mypy.server.deps for an explanation of +-- how fine-grained dependencies are represented. + +[case testCallFunction] +def f() -> None: + g() +def g() -> None: + pass +[out] + -> m.f + +[case testCallMethod] +def f(a: A) -> None: + a.g() +class A: + def g(self) -> None: pass +[out] + -> m.f + -> , m.A, m.f + +[case testAccessAttribute] +def f(a: A) -> None: + a.x +class A: + def g(self) -> None: + self.x = 1 +[out] + -> m.A.g, m.f + -> , m.A, m.f + +[case testConstructInstance] +def f() -> None: + A() +class A: pass +[out] + -> m.f + -> m.f + -> m.A, m.f + +[case testAccessModuleAttribute] +class A: pass +x = A() +def f() -> None: + x +[out] + -> m + -> m + -> , m, m.A + -> m, m.f + +[case testAccessModuleAttribute2] +import n +def f() -> None: + n.x +[file n.py] +x = 1 +[out] + -> m.f + -> m, m.f + +[case testImport] +import n +[file n.py] +x = 1 +[out] + -> m + +[case testCallImportedFunction] +import n +n.f() +[file n.py] +def f() -> None: pass +[out] + -> m + -> m + +[case testImportModuleAs] +import n as x +x.f() +[file n.py] +def f() -> None: pass +[out] + -> m + -> m + +[case testCallImportedFunctionInFunction] +import n +def g() -> None: + n.f() +[file n.py] +def f() -> None: pass +[out] + -> m.g + -> m, m.g + +[case testInheritanceSimple] +class A: + pass +class B(A): + pass +[out] + -> , m + -> + -> + -> m, m.A, m.B + -> m.B + +[case testInheritanceWithMethodAndAttribute] +class A: + pass +class B(A): + def f(self) -> None: + self.x = 1 +[out] + -> , m + -> + -> + -> m.B.f + -> + -> m, m.A, m.B + -> m.B.f + -> m.B + +[case testInheritanceWithMethodAndAttributeAndDeepHierarchy] +class A: + pass +class B(A): + pass +class C(B): + def f(self) -> None: + self.x = 1 +[out] + -> , , m + -> , + -> , + -> m.C.f + -> + -> m, m.A, m.B + -> , m + -> + -> + -> m.C.f + -> + -> m, m.B, m.C + -> m.C.f + -> m.C + +[case testInheritAttribute] +import n +class B(n.A): + def f(self) -> None: + a = 1 + a = self.x +[file n.py] +class A: + def g(self) -> None: + self.x = 1 +[out] + -> m.B.f + -> m.B + -> , m + -> + -> + -> m.B.f + -> + -> + -> m, m.B + -> m + +[case testInheritMethod] +class A: + def g(self) -> None: pass +class B(A): + def f(self) -> None: + self.g() +[out] + -> , m + -> + -> + -> m.B.f + -> + -> m, m.A, m.B + -> m.B.f + -> m.B + +[case testPackage] +import a.b +def f() -> None: + a.b.g() +[file a/__init__.py] +[file a/b.py] +def g() -> None: pass +[out] + -> m.f + -> m, m.f + -> m.f + +[case testClassInPackage] +import a.b +def f(x: a.b.A) -> None: + x.g() + x.y +[file a/__init__.py] +[file a/b.py] +class A: + def g(self) -> None: + self.y = 1 +[out] + -> m.f + -> m.f + -> , m.f + -> m + +[case testPackage__init__] +import a +def f() -> None: + a.g() +[file a/__init__.py] +def g() -> None: pass +[out] + -> m.f + -> m, m.f + +[case testClassInPackage__init__] +import a +def f(x: a.A) -> None: + x.g() + x.y +[file a/__init__.py] +class A: + def g(self) -> None: + self.y = 1 +[out] + -> m.f + -> m.f + -> , m.f + -> m + +[case testConstructor] +class A: + def __init__(self, x: C) -> None: pass +class C: pass +def f() -> None: + A(C()) +[out] + -> m.f + -> m.f + -> m.A, m.f + -> m.f + -> m.f + -> , m.A.__init__, m.C, m.f + +[case testNonTrivialConstructor] +class C: + def __init__(self) -> None: + self.x = 1 +[out] + -> m.C.__init__ + -> m.C + +[case testImportFrom] +from n import f + +def g() -> None: + f() +[file n.py] +def f() -> None: pass +[out] + -> m, m.g + -> m + +[case testImportFromAs] +from n import f as ff + +def g() -> None: + ff() +[file n.py] +def f() -> None: pass +[out] + -> m, m.g + -> m + +[case testNestedClass] +def f() -> None: + b = A.B() + b.f() +class A: + class B: + def f(self) -> None: pass +[out] + -> m.f + -> m.f + -> m.f + -> m.A.B, m.f + -> m.A, m.f + +[case testNestedClassAttribute] +def f() -> None: + b = A.B() + b.x +class A: + class B: + def f(self) -> None: + self.x = 1 +[out] + -> m.f + -> m.f + -> m.A.B.f, m.f + -> m.A.B, m.f + -> m.A, m.f + +[case testNestedClassInAnnotation] +def f(x: A.B) -> None: + pass +class A: + class B: pass +[out] + -> , m.A.B, m.f + -> m.A + +[case testNestedClassInAnnotation2] +def f(x: A.B) -> None: + x.f() +class A: + class B: + def f(self) -> None: pass +[out] + -> m.f + -> , m.A.B, m.f + -> m.A + +[case NestedFunctionType] +from mod import A, B, C, D +def outer() -> None: + def inner(x: A, *args: B, **kwds: C) -> D: + pass +[file mod.py] +class A: pass +class B: pass +class C: pass +class D: pass +[builtins fixtures/dict.pyi] +[out] + -> , m, m.outer + -> , m, m.outer + -> , m, m.outer + -> , m, m.outer + -> m + +[case NestedFunctionBody] +from mod import A, B, C +def outer() -> None: + def inner() -> None: + A() + x: B + y: C + y.x +[file mod.py] +class A: pass +class B: pass +class C: + x: int +[builtins fixtures/dict.pyi] +[out] + -> m.outer + -> m.outer + -> m, m.outer + -> m, m.outer + -> m.outer + -> m, m.outer + -> m + +[case testDefaultArgValue] +def f1(x: int) -> int: pass +def f2() -> int: pass +def g(x: int = f1(f2())) -> None: pass +[out] + -> m.g + -> m.g + +[case testIsInstance] +class A: + def g(self) -> None: pass + +def f(x: object) -> None: + if isinstance(x, A): + x.g() +def ff(x: object) -> None: + if isinstance(x, A): + pass +[builtins fixtures/isinstancelist.pyi] +[out] +-- The dependencies on the ctor are basically spurious but not a problem + -> m.f + -> m.A, m.f, m.ff + +[case testUnreachableIsInstance] +class A: + x: int + +class B: + x: str + +def f(x: A) -> None: + if isinstance(x, B): + x.y +[builtins fixtures/isinstancelist.pyi] +[out] + -> , m.A, m.f + -> m.B, m.f + +[case testIsInstanceAdHocIntersectionDeps] +class A: + x: int + +class B: + y: int + +def f(x: A) -> None: + if isinstance(x, B): + x.y +[builtins fixtures/isinstancelist.pyi] +[out] +.y> -> m.f + -> , m.A, m.f + -> m.B, m.f + +[case testAttributeWithClassType1] +from n import A + +class B: + def h(self, z: A) -> None: + self.z = z +[file n.py] +class A: pass +[out] + -> m.B.h + -> m.B + -> , , m, m.B.h + -> m + +[case testAttributeWithClassType2] +from m import A + +class B: + def f(self) -> None: + self.x = A() +[file m.py] +class A: pass +[out] + -> m.B.f + -> m.B + -> m.B.f + -> m.B.f + -> , m, m.B.f + -> m + +[case testAttributeWithClassType3] +from n import A, x + +class B: + def g(self) -> None: + self.x = x +[file n.py] +class A: pass + +x = A() +[out] + -> m.B.g + -> m.B + -> , m + -> m, m.B.g + -> m + +[case testAttributeWithClassType4] +from n import A + +class B: + def g(self) -> None: + self.x: A +[file n.py] +class A: pass +[out] + -> m.B.g + -> m.B + -> , m, m.B.g + -> m + +[case testClassBody] +def f() -> int: pass +def g() -> int: pass +def h() -> int: pass + +class A: + h() + if f(): + g() +[out] + -> m.A + -> m + -> m + -> m + +[case testVariableInitializedInClass] +from n import A + +class B: + x = None # type: A +[file n.py] +class A: pass +[out] + -> m.B + -> , m + -> m + +[case testVariableAnnotationInClass] +from n import A + +class B: + x: A + + def f(self) -> None: + y = self.x +[file n.py] +class A: pass +[out] + -> m.B.f + -> m.B + -> , m + -> m + +[case testGlobalVariableInitialized] +from n import A + +x = A() +[file n.py] +class A: pass +[out] + -> m + -> m + -> m + -> , m + -> m + +[case testGlobalVariableAnnotation] +from n import A + +x: A +[file n.py] +class A: pass +[out] + -> m + -> , m + -> m + +[case testProperty] +class B: pass + +class A: + @property + def x(self) -> B: pass + +def f(a: A) -> None: + b = a.x +[builtins fixtures/property.pyi] +[out] + -> m, m.f + -> , m.A, m.f + -> , m.A.x, m.B + +[case testUnreachableAssignment] +from typing import List, Tuple + +def f() -> None: pass + +class C: + def __init__(self, x: int) -> None: + if isinstance(x, int): + self.y = 1 + else: + self.y = f() +[builtins fixtures/isinstancelist.pyi] +[out] + -> m.C.__init__ + -> m.C + -> m.C.__init__ + +[case testPartialNoneTypeAttributeCrash1] +class C: pass + +class A: + x = None + + def f(self) -> None: + self.x = C() +[out] + -> m.A.f + -> m.A + -> m.A.f + -> m.A.f + -> , m.A.f, m.C + +[case testPartialNoneTypeAttributeCrash2] +class C: pass + +class A: + x = None + + def f(self) -> None: + self.x = C() +[out] + -> m.A.f + -> m.A + -> m.A.f + -> m.A.f + -> , m.A.f, m.C + +[case testRelativeImport] +import pkg # Magic package name in test runner +[file pkg/__init__.py] +from . import mod +from .a import x +[file pkg/mod.py] +from . import a +[file pkg/a.py] +x = 1 +[out] + -> pkg + -> pkg, pkg.mod + -> pkg + -> m, pkg, pkg.mod + +[case testTypedDict] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) +def foo(x: Point) -> int: + return x['x'] + x['y'] +[builtins fixtures/dict.pyi] +[out] + -> m + -> m + -> , , m, m.foo + -> m + +[case testTypedDict2] +from mypy_extensions import TypedDict +class A: pass +Point = TypedDict('Point', {'x': int, 'y': A}) +p = Point(dict(x=42, y=A())) +def foo(x: Point) -> int: + return x['x'] +[builtins fixtures/dict.pyi] +[out] + -> m + -> m + -> , , , m, m.A, m.foo + -> m + -> m + -> , , m, m.foo + -> m + +[case testTypedDict3] +from mypy_extensions import TypedDict +class A: pass +class Point(TypedDict): + x: int + y: A +p = Point(dict(x=42, y=A())) +def foo(x: Point) -> int: + return x['x'] +[builtins fixtures/dict.pyi] +[out] + -> m + -> m + -> , , , m, m.A, m.foo + -> m + -> m + -> , , m, m.Point, m.foo + -> m + +[case testImportStar] +from a import * +[file a.py] +x = 0 +[out] + -> m + +[case testDecoratorDepsMod] +import mod + +@mod.deca +@mod.decb(mod.C()) +def func(x: int) -> int: + pass +[file mod.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[int], int]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +[out] + -> m + -> m + -> m + -> m + -> m + -> m + -> m + +[case testDecoratorDepsFunc] +import mod + +def outer() -> None: + @mod.deca + @mod.decb(mod.C()) + def func(x: int) -> int: + pass +[file mod.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[int], int]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +[out] + -> m.outer + -> m.outer + -> m.outer + -> m.outer + -> m.outer + -> m, m.outer + +[case testDecoratorDepsDeeepNested] +import mod + +def outer() -> None: + def inner() -> None: + @mod.dec + def func(x: int) -> int: + pass +[file mod.py] +from typing import Callable +def dec(func: Callable[[int], int]) -> Callable[[str], str]: + pass +[out] + -> m.outer + -> m, m.outer + +[case testDecoratorDepsNestedClass] +import mod + +class Outer: + class Inner: + c = mod.C() + @c.dec + def func(self, x: int) -> int: + pass +[file mod.py] +from typing import Callable +class C: + def dec(self, func: Callable[..., int]) -> Callable[..., str]: + pass +[out] + -> m + -> m.Outer.Inner + -> m.Outer + -> m + -> m + -> m + -> , m + -> m + +[case testDecoratorDepsClassInFunction] +import mod + +def outer() -> None: + class Inner: + c = mod.C() + @c.dec + def func(self, x: int) -> int: + pass +[file mod.py] +from typing import Callable +class C: + def dec(self, func: Callable[..., int]) -> Callable[..., str]: + pass +[out] + -> m.outer + -> m.outer + -> m.outer + -> m.outer + -> , m.outer + -> m, m.outer + +[case DecoratorDepsMethod] +import mod + +class D: + @mod.deca + @mod.decb(mod.C()) + def func(self, x: int) -> int: + pass +[file mod.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[..., int]) -> Callable[..., str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +[out] + -> m + -> m.D + -> m + -> m + -> m + -> m + -> m + -> m + +[case testMissingModuleClass1] +from b import A # type: ignore +def f(x: A) -> None: + x.foo() +[out] + -> , m.f + -> m + -> m + +[case testMissingModuleClass2] +from p.b import A # type: ignore +def f(x: A) -> None: + x.foo() +[out] + -> , m.f + -> m + -> m + +[case testIgnoredMissingModuleAttribute] +import a +a.x # type: ignore + +import b.c +b.c.x # type: ignore + +from b import c +c.y # type: ignore +[file a.py] +[file b/__init__.py] +[file b/c.py] +[builtins fixtures/module.pyi] +[out] + -> m + -> m + -> m + -> m + -> m + -> m + -> + -> typing.Awaitable + +[case testIgnoredMissingInstanceAttribute] +from a import C +C().x # type: ignore +[file a.py] +class C: pass +[out] + -> + -> m + -> m + -> m + -> m, typing.Awaitable + -> m + +[case testIgnoredMissingClassAttribute] +from a import C +C.x # type: ignore +[file a.py] +class C: pass +[out] + -> m + -> m + -> m + +[case testDepsFromOverloadMod] +# __dump_all__ +import mod +x = mod.f +[file mod.py] +from typing import overload, Any +import submod +@overload +def f(x: int) -> submod.A: pass +@overload +def f(x: str) -> submod.B: pass +def f(x) -> Any: + y: submod.C + y.x +[file submod.py] +class A: pass +class B: pass +class C: + x: D +class D: pass +[out] + -> m + -> m + -> m + -> , , mod.f, submod.A + -> , , mod.f, submod.B + -> mod.f + -> mod.f, submod.C + -> , submod, submod.D + -> mod + +[case testDepsFromOverloadFunc] +import mod +def f() -> None: + x = mod.f +[file mod.py] +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> str: pass +def f(x): + pass +[out] + -> m.f + -> m, m.f + +[case testDepsToOverloadMod] +from typing import overload, Any +import mod +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> str: pass +def f(x: Any) -> Any: + mod.g() +[file mod.py] +def g() -> int: + pass +[out] + -> m.f + -> m, m.f + +[case testDepsToOverloadFunc] +from typing import overload, Any +import mod +def outer() -> None: + @overload + def f(x: int) -> mod.A: pass + @overload + def f(x: str) -> mod.B: pass + def f(x: Any) -> Any: + mod.g() +[file mod.py] +def g() -> int: + pass +class A: pass +class B: pass +[out] + -> , m.outer + -> , m.outer + -> m.outer + -> m, m.outer + +[case testDepsFromOverloadToOverload] +from typing import overload, Any +import mod +def outer() -> None: + @overload + def f(x: int) -> None: pass + @overload + def f(x: str) -> str: pass + def f(x: Any) -> Any: + mod.g(str()) +[file mod.py] +from typing import overload +@overload +def g(x: int) -> None: pass +@overload +def g(x: str) -> str: pass +def g(x): + pass +[out] + -> m.outer + -> m, m.outer + +[case testDepsFromOverloadToOverloadDefaultClass] +from typing import overload, Any +import mod +class Outer: + @overload + def f(self, x: int) -> None: pass + @overload + def f(self, x: str, cb=mod.g) -> str: pass + def f(self, *args: Any, **kwargs: Any) -> Any: + pass +[file mod.py] +from typing import overload +@overload +def g(x: int) -> None: pass +@overload +def g(x: str) -> str: pass +def g(x): + pass +[builtins fixtures/dict.pyi] +[out] + -> m.Outer + -> m.Outer.f + -> m, m.Outer.f + +[case testDepsFromOverloadToOverloadDefaultNested] +from typing import overload, Any +import mod +def outer() -> None: + @overload + def f(x: int) -> None: pass + @overload + def f(x: str, cb=mod.g) -> str: pass + def f(*args: Any, **kwargs: Any) -> Any: + pass +[file mod.py] +from typing import overload +@overload +def g(x: int) -> None: pass +@overload +def g(x: str) -> str: pass +def g(x): + pass +[builtins fixtures/dict.pyi] +[out] + -> m.outer + -> m, m.outer + +[case testDepsToOverloadGeneric] +# __dump_all__ +import mod +from typing import overload, Any +@overload +def f(x: mod.TA) -> mod.TA: pass +@overload +def f(x: mod.TB, y: int) -> mod.TB: pass +def f(*args: Any, **kwargs: Any) -> Any: + pass +[file mod.py] +from typing import TypeVar +import submod +TA = TypeVar('TA', submod.A, submod.B) +TB = TypeVar('TB', bound=submod.C) +[file submod.py] +class A: pass +class B: pass +class C: pass +[builtins fixtures/dict.pyi] +[out] + -> , m.f + -> , m.f + -> m + -> , , m.f, mod, submod.A + -> , , m.f, mod, submod.B + -> , , m.f, mod, submod.C + -> mod + +[case testDepsOverloadBothExternalAndImplementationType] +import mod +from typing import overload +@overload +def f(x: mod.A) -> mod.A: pass +@overload +def f(x: mod.B) -> mod.B: pass +def f(x: mod.Base) -> mod.Base: + pass +[file mod.py] +class Base: + pass +class A(Base): + pass +class B(Base): + pass +[out] + -> , m.f + -> , m.f + -> , m.f + -> m + +[case testCustomIterator] +class A: + def __iter__(self) -> B: pass +class B: + def __iter__(self) -> B: pass + def __next__(self) -> C: pass +class C: + pass +def f() -> None: + for x in A(): pass +[out] + -> m.f + -> m.f + -> m.f + -> m.f + -> m.A, m.f + -> m.f + -> , , m.A.__iter__, m.B, m.B.__iter__ + -> , m.B.__next__, m.C + +[case testDepsLiskovClass] +from mod import A, C +class D(C): + x: A +[file mod.py] +class C: + x: B +class B: + pass +class A(B): + pass +[out] + -> m + -> m.D + -> , m + -> , m + -> + -> + -> + -> m, m.D + -> m + +[case testDepsLiskovMethod] +from mod import A, C +class D(C): + def __init__(self) -> None: + self.x: A +[file mod.py] +class C: + def __init__(self) -> None: + self.x: B +class B: + pass +class A(B): + pass +[out] + -> m.D.__init__ + -> m.D + -> , m, m.D.__init__ + -> , m + -> , m.D.__init__ + -> + -> + -> m, m.D + -> m + +[case testIndexedStarLvalue] +from typing import List, Tuple + +class B: + def __setitem__(self, i: int, v: List[str]) -> None: pass + +def g() -> Tuple[int, str, str]: pass + +def f(b: B) -> None: + a, *b[0] = g() +[builtins fixtures/list.pyi] +[out] + -> m.f + -> m.f + -> , m.B, m.f + -> m.f + +[case testLogicalDecorator] +# flags: --logical-deps +from mod import dec +@dec +def f() -> None: + pass +[file mod.py] +from typing import Callable +def dec(f: Callable[[], None]) -> Callable[[], None]: + pass +[out] + -> , m + +[case testLogicalDecoratorWithArgs] +# flags: --logical-deps +from mod import dec +@dec(str()) +def f() -> None: + pass +[file mod.py] +from typing import Callable +def dec(arg: str) -> Callable[[Callable[[], None]], Callable[[], None]]: + pass +[out] + -> , m + +[case testLogicalDecoratorMember] +# flags: --logical-deps +import mod +@mod.dec +def f() -> None: + pass +[file mod.py] +from typing import Callable +def dec(f: Callable[[], None]) -> Callable[[], None]: + pass +[out] + -> , m + -> m + +[case testLogicalDefinition] +# flags: --logical-deps +from mod import func +b = func() +[file mod.py] +def func() -> int: + pass +[out] + -> m + -> , m + +[case testLogicalDefinitionIrrelevant] +# flags: --logical-deps +from mod import func +def outer() -> None: + a = func() +b: int = func() +c = int() +c = func() +[file mod.py] +def func() -> int: + pass +[out] + -> m + -> m + -> m, m.outer + +[case testLogicalDefinitionMember] +# flags: --logical-deps +import mod +b = mod.func() +[file mod.py] +def func() -> int: + pass +[out] + -> m + -> , m + -> m + +[case testLogicalDefinitionClass] +# flags: --logical-deps +from mod import Cls +b = Cls() +[file mod.py] +class Base: + def __init__(self) -> None: pass +class Cls(Base): pass +[out] + -> m + -> + -> m + -> m + -> , m + +[case testLogicalBaseAttribute] +# flags: --logical-deps +from mod import C +class D(C): + x: int +[file mod.py] +class C: + x: int + y: int +[out] + -> m + -> m.D + -> + -> m, m.D + +[case testLogicalIgnoredImport1] +# flags: --logical-deps --ignore-missing-imports +import foo +def f() -> None: + foo.x +[out] + -> m.f + -> m, m.f + +[case testLogicalIgnoredImport2] +# flags: --logical-deps --ignore-missing-imports +import foo.bar +import a.b.c.d +def f() -> None: + foo.bar.x + foo.bar.x.y + a.b.c.d.e +[out] + -> m.f + -> m, m.f + -> m.f + -> m.f + -> m.f + -> m.f + -> m.f + -> m, m.f + -> m.f + +[case testLogicalIgnoredFromImport] +# flags: --logical-deps --ignore-missing-imports +from foo.bar import f, C +def g() -> None: + f() + C.ff() +def gg(x: C) -> None: + z = x.y + z.zz +[out] + -> m.g + -> m.gg + -> , m.g, m.gg + -> m.g + +[case testLogical__init__] +# flags: --logical-deps +class A: + def __init__(self) -> None: pass +class B(A): pass +class C(B): pass +class D(A): + def __init__(self, x: int) -> None: pass +def f() -> None: + A() +def g() -> None: + C() +def h() -> None: + D(1) +[out] + -> m.f + -> m.f + -> m, m.A, m.B, m.D, m.f + -> m, m.B, m.C + -> m.g + -> m.g + -> m.C, m.g + -> m.h + -> m.h + -> m.D, m.h + +[case testDataclassDepsOldVersion] +from dataclasses import dataclass + +Z = int + +@dataclass +class A: + x: Z + +@dataclass +class B(A): + y: int +[builtins fixtures/dataclasses.pyi] + +[out] + -> , m + -> + -> , m.B.__init__ + -> , m, m.B.__mypy-replace + -> + -> + -> + -> m, m.A, m.B + -> m + -> m + -> m + -> m.B + -> m + -> m + -> m + +[case testDataclassDeps] +# flags: --python-version 3.10 +from dataclasses import dataclass + +Z = int + +@dataclass +class A: + x: Z + +@dataclass +class B(A): + y: int +[builtins fixtures/dataclasses.pyi] + +[out] + -> , m + -> + -> , m.B.__init__ + -> + -> , m, m.B.__mypy-replace + -> + -> + -> + -> m, m.A, m.B + -> m + -> m + -> m + -> m.B + -> m + -> m + -> m + +[case testPEP695TypeAliasDeps] +# flags: --python-version=3.12 +from a import C, E +type A = C +type A2 = A +type A3 = E +[file a.py] +class C: pass +class D: pass +type E = D +[out] + -> m + -> m + -> m + -> m + -> m + -> m + -> m + -> m +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test new file mode 100644 index 000000000000..4acf451e2c34 --- /dev/null +++ b/test-data/unit/diff.test @@ -0,0 +1,1633 @@ +-- Test cases for taking a diff of two module ASTs/symbol tables. +-- The diffs are used for fined-grained incremental checking. + +[case testChangeTypeOfModuleAttribute] +x = 1 +y = 1 +[file next.py] +x = '' +y = 1 +[out] +__main__.x + +[case testChangeSignatureOfModuleFunction] +def f(x: int) -> None: + pass +def g(y: str) -> None: + pass +[file next.py] +def f(x: str) -> None: + x = '' +def g(y: str) -> None: + y = '' +[out] +__main__.f + +[case testAddModuleAttribute] +x = 1 +[file next.py] +x = 1 +y = 1 +[out] +__main__.y + +[case testRemoveModuleAttribute] +x = 1 +y = 1 +[file next.py] +x = 1 +[out] +__main__.y + +-- +-- Classes +-- + +[case testChangeMethodSignature] +class A: + def f(self) -> None: pass + def g(self) -> None: pass +[file next.py] +class A: + def f(self, x: int) -> None: pass + def g(self) -> None: pass +[out] +__main__.A.f + +[case testChangeAttributeType] +class A: + def f(self) -> None: + self.x = 1 + self.y = 1 +[file next.py] +class A: + def f(self) -> None: + self.x = 1 + self.y = '' +[out] +__main__.A.y + +[case testAddAttribute] +class A: pass +[file next.py] +class A: + def f(self) -> None: + self.x = 1 +[out] +__main__.A.f +__main__.A.x + +[case testAddAttribute2] +class A: + def f(self) -> None: pass +[file next.py] +class A: + def f(self) -> None: + self.x = 1 +[out] +__main__.A.x + +[case testRemoveAttribute] +class A: + def f(self) -> None: + self.x = 1 +[file next.py] +class A: pass +[out] +__main__.A.f +__main__.A.x + +[case testAddMethod] +class A: + def f(self) -> None: pass +[file next.py] +class A: + def f(self) -> None: pass + def g(self) -> None: pass +[out] +__main__.A.g + +[case testRemoveMethod] +class A: + def f(self) -> None: pass + def g(self) -> None: pass +[file next.py] +class A: + def f(self) -> None: pass +[out] +__main__.A.g + +[case testAddImport] +import nn +[file next.py] +import n +import nn +[file n.py] +x = 1 +[file nn.py] +y = 1 +[out] +__main__.n + +[case testRemoveImport] +import n +[file next.py] +[file n.py] +x = 1 +[out] +__main__.n + +[case testChangeClassIntoFunction] +class A: pass +[file next.py] +def A() -> None: pass +[out] +__main__.A + +[case testDeleteClass] +class A: pass +[file next.py] +[out] +__main__.A + +[case testAddBaseClass] +class A: pass +[file next.py] +class B: pass +class A(B): pass +[out] +__main__.A +__main__.B + +[case testChangeBaseClass] +class A: pass +class B: pass +class C(A): pass +[file next.py] +class A: pass +class B: pass +class C(B): pass +[out] +__main__.C + +[case testRemoveBaseClass] +class A: pass +class B(A): pass +[file next.py] +class A: pass +class B: pass +[out] +__main__.B + +[case testRemoveClassFromMiddleOfMro] +class A: pass +class B(A): pass +class C(B): pass +[file next.py] +class A: pass +class B: pass +class C(B): pass +[out] +__main__.B +__main__.C + +[case testDifferenceInConstructor] +class A: + def __init__(self) -> None: pass +[file next.py] +class A: + def __init__(self, x: int) -> None: pass +[out] +__main__.A.__init__ + +[case testChangeSignatureOfMethodInNestedClass] +class A: + class B: + def f(self) -> int: pass +[file next.py] +class A: + class B: + def f(self) -> str: pass +[out] +__main__.A.B.f + +[case testChangeTypeOfAttributeInNestedClass] +class A: + class B: + def f(self) -> None: + self.x = 1 +[file next.py] +class A: + class B: + def f(self) -> None: + self.x = '' +[out] +__main__.A.B.x + +[case testAddMethodToNestedClass] +class A: + class B: pass +[file next.py] +class A: + class B: + def f(self) -> str: pass +[out] +__main__.A.B.f + +[case testAddNestedClass] +class A: pass +[file next.py] +class A: + class B: + def f(self) -> None: pass +[out] +__main__.A.B + +[case testRemoveNestedClass] +class A: + class B: + def f(self) -> None: pass +[file next.py] +class A: pass +[out] +__main__.A.B + +[case testChangeNestedClassToMethod] +class A: + class B: pass +[file next.py] +class A: + def B(self) -> None: pass + +[out] +__main__.A.B + +[case testChangeNamedTupleAttribute] + +from typing import NamedTuple +class A: + x: str +N = NamedTuple('N', [('x', int), ('y', str)]) +M = NamedTuple('M', [('x', int), ('y', str)]) +[file next.py] +from typing import NamedTuple +N = NamedTuple('N', [('x', int), ('y', int)]) +M = NamedTuple('M', [('x', int), ('y', str)]) +[builtins fixtures/tuple.pyi] +[out] +__main__.A +__main__.N +__main__.N._NT +__main__.N.__new__ +__main__.N._asdict +__main__.N._make +__main__.N._replace +__main__.N.y + +[case testSimpleDecoratedFunction] +from a import dec +@dec +def f() -> None: pass +@dec +def g() -> None: pass +[file next.py] +from a import dec +@dec +def f(x: int) -> None: pass +@dec +def g() -> None: pass +[file a.py] +from typing import TypeVar + +T = TypeVar('T') + +def dec(f: T) -> T: + return f +[out] +__main__.f + +[case testSimpleDecoratedMethod] +from a import dec +class A: + @dec + def f(self) -> None: + self.g() + @dec + def g(self) -> None: pass +[file next.py] +from a import dec +class A: + @dec + def f(self, x: int) -> None: + self.g() + @dec + def g(self) -> None: pass +[file a.py] +from typing import TypeVar + +T = TypeVar('T') + +def dec(f: T) -> T: + return f +[out] +__main__.A.f + +[case testTypeVarBound] +from typing import TypeVar +T = TypeVar('T') +S = TypeVar('S') +[file next.py] +from typing import TypeVar +T = TypeVar('T', bound=int) +S = TypeVar('S') +[out] +__main__.T + +[case testTypeVarVariance] +from typing import TypeVar +A = TypeVar('A', covariant=True) +B = TypeVar('B', covariant=True) +C = TypeVar('C', covariant=True) +[file next.py] +from typing import TypeVar +A = TypeVar('A', covariant=True) +B = TypeVar('B', contravariant=True) +C = TypeVar('C') +[out] +__main__.B +__main__.C + +[case testTypeVarValues] +from typing import TypeVar +A = TypeVar('A', int, str) +B = TypeVar('B', int, str) +C = TypeVar('C', int, str) +[file next.py] +from typing import TypeVar +A = TypeVar('A', int, str) +B = TypeVar('B', int, str, object) +C = TypeVar('C') +[out] +__main__.B +__main__.C + +[case testGenericFunction] +from typing import TypeVar +T = TypeVar('T') +S = TypeVar('S') +def f(x: T) -> T: pass +def g(x: S) -> S: pass +[file next.py] +from typing import TypeVar +T = TypeVar('T', int, str) +S = TypeVar('S') +def f(x: T) -> T: pass +def g(x: S) -> S: pass +[out] +__main__.T +__main__.f + +[case testGenericTypes] +from typing import List +x: List[int] +y: List[int] +[file next.py] +from typing import List +x: List[int] +y: List[str] +[builtins fixtures/list.pyi] +[out] +__main__.y + +[case testTypeAliasOfList] +from typing import List +X = List[int] +Y = List[int] +[file next.py] +from typing import List +X = List[str] +Y = List[int] +[builtins fixtures/list.pyi] +[out] +__main__.X + +[case testTypeAliasOfCallable] +from typing import Callable +A = Callable[[int], str] +B = Callable[[int], str] +C = Callable[[int], str] +[file next.py] +from typing import Callable +A = Callable[[int], str] +B = Callable[[], str] +C = Callable[[int], int] +[out] +__main__.B +__main__.C + +[case testGenericTypeAlias] +from typing import Callable, TypeVar +T = TypeVar('T') +A = Callable[[T], T] +B = Callable[[T], T] +[file next.py] +from typing import Callable, TypeVar +T = TypeVar('T') +S = TypeVar('S') +A = Callable[[T], T] +B = Callable[[T], S] +[out] +__main__.B +__main__.S + +[case testDifferentListTypes] +from typing import List +A = List +B = list +C = List +[file next.py] +from typing import List +A = List +B = list +C = list +[builtins fixtures/list.pyi] +[out] +__main__.C + +[case testDecoratorChangesSignature] +from contextlib import contextmanager +from typing import Iterator, List, Tuple + +@contextmanager +def f(x: List[Tuple[int]]) -> Iterator[None]: + yield + +@contextmanager +def g(x: str) -> Iterator[None]: + yield +[file next.py] +from contextlib import contextmanager +from typing import Iterator, List, Tuple + +@contextmanager +def f(x: List[Tuple[int]]) -> Iterator[None]: + yield + +@contextmanager +def g(x: object) -> Iterator[None]: + yield +[typing fixtures/typing-medium.pyi] +[builtins fixtures/list.pyi] +[out] +__main__.g + +[case testOverloadedMethod] +from typing import overload + +class A: + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass + def f(self, x): pass + + @overload + def g(self, x: int) -> int: pass + @overload + def g(self, x: str) -> str: pass + def g(self, x): pass +[file next.py] +from typing import overload + +class A: + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass + def f(self, x): pass + + @overload + def g(self, x: int) -> int: pass + @overload + def g(self, x: object) -> object: pass + def g(self, x): pass +[out] +__main__.A.g + +[case testPropertyWithSetter] +class A: + @property + def x(self) -> int: + pass + + @x.setter + def x(self, o: int) -> None: + pass + +class B: + @property + def x(self) -> int: + pass + + @x.setter + def x(self, o: int) -> None: + pass +[file next.py] +class A: + @property + def x(self) -> int: + pass + + @x.setter + def x(self, o: int) -> None: + pass + +class B: + @property + def x(self) -> str: + pass + + @x.setter + def x(self, o: str) -> None: + pass +[builtins fixtures/property.pyi] +[out] +__main__.B.x + +[case testFunctionalEnum] +from enum import Enum, IntEnum +A = Enum('A', 'x') +B = Enum('B', 'x') +C = Enum('C', 'x') +D = IntEnum('D', 'x') +[file next.py] +from enum import Enum, IntEnum +A = Enum('A', 'x') +B = Enum('B', 'y') +C = IntEnum('C', 'x') +D = IntEnum('D', 'x y') +[builtins fixtures/enum.pyi] +[out] +__main__.B.x +__main__.B.y +__main__.C +__main__.D.y + +[case testClassBasedEnum] +from enum import Enum +class A(Enum): + X = 0 + Y = 1 +class B(Enum): + X = 0 + Y = 1 +class C(Enum): + X = 0 + Y = 1 +class D(Enum): + X = 0 + Y = 1 +class E(Enum): + X = 0 +[file next.py] +from enum import Enum +class A(Enum): + X = 0 + Y = 1 +class B(Enum): + X = 0 + Z = 1 +class C(Enum): + X = 0 + Y = 1 + Z = 2 +class D(Enum): + X = 'a' + Y = 'b' +class F(Enum): + X = 0 +[builtins fixtures/enum.pyi] +[out] +__main__.B.Y +__main__.B.Z +__main__.C.Z +__main__.D.X +__main__.D.Y +__main__.E +__main__.F + +[case testTypedDict] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) +[file next.py] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': str}) +p = Point(dict(x=42, y='lurr')) +[builtins fixtures/dict.pyi] +[out] +__main__.Point +__main__.p + +[case testTypedDict2] +from mypy_extensions import TypedDict +class Point(TypedDict): + x: int + y: int +p = Point(dict(x=42, y=1337)) +[file next.py] +from mypy_extensions import TypedDict +class Point(TypedDict): + x: int + y: str +p = Point(dict(x=42, y='lurr')) +[builtins fixtures/dict.pyi] +[out] +__main__.Point +__main__.p + +[case testTypedDict3] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) +[file next.py] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int}) +p = Point(dict(x=42)) +[builtins fixtures/dict.pyi] +[out] +__main__.Point +__main__.p + +[case testTypedDict4] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) +[file next.py] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}, total=False) +p = Point(dict(x=42, y=1337)) +[builtins fixtures/dict.pyi] +[out] +__main__.Point +__main__.p + +[case testTypeAliasSimple] +A = int +B = int +[file next.py] +A = str +B = int +[out] +__main__.A + +[case testTypeAliasGeneric] +from typing import List +A = List[int] +B = List[int] +[file next.py] +from typing import List +A = List[str] +B = List[int] +[builtins fixtures/list.pyi] +[out] +__main__.A + +[case testTypeAliasGenToNonGen] +from typing import List +A = List[str] +B = List +[file next.py] +from typing import List +A = List +B = List +[builtins fixtures/list.pyi] +[out] +__main__.A + +[case testTypeAliasNonGenToGen] +from typing import List +A = List +B = List +[file next.py] +from typing import List +A = List[str] +B = List +[builtins fixtures/list.pyi] +[out] +__main__.A + +[case testTypeAliasGenericTypeVar] +from typing import TypeVar, Dict +T = TypeVar('T') +S = TypeVar('S') +A = Dict[str, T] +B = Dict[str, S] +[file next.py] +from typing import TypeVar, Dict +class T: pass +S = TypeVar('S') +A = Dict[str, T] +B = Dict[str, S] +[builtins fixtures/dict.pyi] +[out] +__main__.A +__main__.T + +[case testNewType] +from typing import NewType +class C: pass +class D: pass +N1 = NewType('N1', C) +N2 = NewType('N2', D) +N3 = NewType('N3', C) +class N4(C): pass +[file next.py] +from typing import NewType +class C: pass +class D(C): pass +N1 = NewType('N1', C) +N2 = NewType('N2', D) +class N3(C): pass +N4 = NewType('N4', C) +[out] +__main__.D +__main__.N2 +__main__.N3 +__main__.N3.__init__ +__main__.N4 +__main__.N4.__init__ + +[case testChangeGenericBaseClassOnly] +from typing import List +class C(List[int]): pass +[file next.py] +from typing import List +class C(List[str]): pass +[builtins fixtures/list.pyi] +[out] +__main__.C + +[case testOverloads] +from typing import overload +class C: pass +@overload +def a(x: int) -> None: pass +@overload +def a(x: str) -> str: pass +def a(x): + pass +@overload +def b(x: int) -> None: pass +@overload +def b(x: str) -> str: pass +def b(x): + pass +@overload +def c(x: int) -> None: pass +@overload +def c(x: str) -> str: pass +def c(x): + pass +@overload +def d(x: int) -> None: pass +@overload +def d(x: str) -> str: pass +def d(x): + pass +[file next.py] +from typing import overload +class C: pass +@overload +def a(x: int) -> None: pass +@overload +def a(x: str) -> str: pass +def a(x): + pass +@overload +def b(x: str) -> str: pass +@overload +def b(x: int) -> None: pass +def b(x): + pass +@overload +def c(x: int) -> None: pass +@overload +def c(x: str) -> str: pass +@overload +def c(x: C) -> C: pass +def c(x): + pass +def d(x: int) -> None: + pass +@overload +def e(x: int) -> None: pass +@overload +def e(x: str) -> str: pass +def e(x): + pass +[out] +__main__.b +__main__.c +__main__.d +__main__.e + +[case testOverloadsExternalOnly] +from typing import overload +class Base: pass +class A(Base): pass +class B(Base): pass +class C(Base): pass +@overload +def f(x: A) -> A: pass +@overload +def f(x: B) -> B: pass +def f(x: Base) -> Base: + pass +@overload +def g(x: A) -> A: pass +@overload +def g(x: B) -> B: pass +def g(x: Base) -> Base: + pass +[file next.py] +from typing import overload +class Base: pass +class A(Base): pass +class B(Base): pass +class C(Base): pass +@overload +def f(x: A) -> A: pass +@overload +def f(x: B) -> B: pass +def f(x: object) -> object: + pass +@overload +def g(x: A) -> A: pass +@overload +def g(x: C) -> C: pass +def g(x: Base) -> Base: + pass +[out] +__main__.g + +[case testNestedFunctionDoesntMatter] +class A: pass +class B: pass +def outer() -> None: + def inner(x: A) -> B: + pass +[file next.py] +class A: pass +class B: pass +def outer() -> None: + def inner(y: B) -> A: + pass +[out] + +[case testProtocolVsNominal] +from typing import Protocol +class A(Protocol): + x: int +class B(Protocol): + x: int +class C(Protocol): + x: int +class D(Protocol): + x: int +[file next.py] +from typing import Protocol +class A(Protocol): + x: int +class B(Protocol): + x: str +class C(Protocol): + x: int + y: int +class D: + x: int +[out] +__main__.B.x +__main__.C.(abstract) +__main__.C.y +__main__.D +__main__.D.(abstract) + +[case testProtocolNormalVsGeneric] +from typing import Protocol, TypeVar +T = TypeVar('T') +class P(Protocol[T]): + x: T +class P2(Protocol[T]): + x: T + y: T +[file next.py] +from typing import Protocol, TypeVar +T = TypeVar('T') +class P(Protocol): + x: int +class P2(Protocol[T]): + x: T +[out] +__main__.P +__main__.P.x +__main__.P2.(abstract) +__main__.P2.y + +[case testAddAbstractMethod] +from abc import abstractmethod +class A: + @abstractmethod + def f(self) -> None: pass +[file next.py] +from abc import abstractmethod +class A: + @abstractmethod + def f(self) -> None: pass + @abstractmethod + def g(self) -> None: pass +[out] +__main__.A.(abstract) +__main__.A.g + +[case testFinalFlagsTriggerVar] +from typing import Final + +v: Final = 1 +w: Final = 1 +x: Final = 1 +y: Final[int] = 1 +z: Final[int] = 1 +same1: Final = 1 +same2: Final[int] = 1 +class C: + v: Final = 1 + w: Final = 1 + x: Final = 1 + y: Final[int] = 1 + z: Final[int] = 1 + same1: Final = 1 + same2: Final[int] = 1 + def __init__(self) -> None: + self.vi: Final = 1 + self.wi: Final = 1 + self.xi: Final = 1 + self.yi: Final[int] = 1 + self.zi: Final[int] = 1 + self.same1_instance: Final = 1 + self.same2_instance: Final[int] = 1 + +[file next.py] +from typing import Final + +v: Final = 0 +w = 1 +x: Final[int] = 1 +y: int = 1 +z: Final = 1 +same1: Final = 1 +same2: Final[int] = 0 +class C: + v: Final = 0 + w = 1 + x: Final[int] = 1 + y: int = 1 + z: Final = 1 + same1: Final = 1 + same2: Final[int] = 0 + def __init__(self) -> None: + self.vi: Final = 0 + self.wi = 1 + self.xi: Final[int] = 1 + self.yi: int = 1 + self.zi: Final = 1 + self.same1_instance: Final = 1 + self.same2_instance: Final[int] = 0 +[out] +__main__.C.v +__main__.C.vi +__main__.C.w +__main__.C.wi +__main__.C.x +__main__.C.xi +__main__.C.y +__main__.C.yi +__main__.C.z +__main__.C.zi +__main__.v +__main__.w +__main__.x +__main__.y +__main__.z + +[case testFinalFlagsTriggerMethod] +from typing import final + +class C: + def meth(self) -> int: pass + @final + def same(self) -> int: pass + @classmethod + def cmeth(cls) -> int: pass + +[file next.py] +from typing import final + +class C: + @final + def meth(self) -> int: pass + @final + def same(self) -> int: pass + @final + @classmethod + def cmeth(cls) -> int: pass +[builtins fixtures/classmethod.pyi] +[out] +__main__.C.cmeth +__main__.C.meth + +[case testFinalFlagsTriggerProperty] +from typing import final + +class C: + @final + @property + def p(self) -> int: pass + @final + @property + def same(self) -> str: pass + +[file next.py] +from typing import final + +class C: + @property + def p(self) -> int: pass + @final + @property + def same(self) -> str: pass +[builtins fixtures/property.pyi] +[out] +__main__.C.p + +[case testFinalFlagsTriggerMethodOverload] +from typing import final, overload + +class C: + @overload + def m(self, x: int) -> int: ... + @overload + def m(self, x: str) -> str: ... + @final + def m(self, x): + pass + @overload + def same(self, x: int) -> int: ... + @overload + def same(self, x: str) -> str: ... + @final + def same(self, x): + pass + +[file next.py] +from typing import final, overload + +class C: + @overload + def m(self, x: int) -> int: ... + @overload + def m(self, x: str) -> str: ... + def m(self, x): + pass + @overload + def same(self, x: int) -> int: ... + @overload + def same(self, x: str) -> str: ... + @final + def same(self, x): + pass +[out] +__main__.C.m + +[case testDynamicBasePluginDiff] +# flags: --config-file tmp/mypy.ini +from mod import declarative_base, Column, Instr + +Base = declarative_base() + +class Model(Base): + x: Column[int] +class Other: + x: Column[int] +class Diff: + x: Column[int] +[file next.py] +from mod import declarative_base, Column, Instr + +Base = declarative_base() + +class Model(Base): + x: Column[int] +class Other: + x: Column[int] +class Diff(Base): + x: Column[int] +[file mod.py] +from typing import Generic, TypeVar +def declarative_base(): ... + +T = TypeVar('T') + +class Column(Generic[T]): ... +class Instr(Generic[T]): ... + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/dyn_class.py +[out] +__main__.Diff +__main__.Diff.x + +[case testLiteralTriggersVar] +from typing_extensions import Literal + +x: Literal[1] = 1 +y = 1 +z: Literal[1] = 1 +same: Literal[1] = 1 +class C: + x_class: Literal[1] = 1 + y_class = 1 + z_class: Literal[1] = 1 + same_class: Literal[1] = 1 + def __init__(self) -> None: + self.x_instance: Literal[1] = 1 + self.y_instance = 1 + self.z_instance: Literal[1] = 1 + self.same_instance: Literal[1] = 1 + +[file next.py] +from typing_extensions import Literal + +x = 1 +y: Literal[1] = 1 +z: Literal[2] = 2 +same: Literal[1] = 1 +class C: + x_class = 1 + y_class: Literal[1] = 1 + z_class: Literal[2] = 2 + same_class: Literal[1] = 1 + def __init__(self) -> None: + self.x_instance = 1 + self.y_instance: Literal[1] = 1 + self.z_instance: Literal[2] = 2 + self.same_instance: Literal[1] = 1 +[builtins fixtures/tuple.pyi] +[out] +__main__.C.x_class +__main__.C.x_instance +__main__.C.y_class +__main__.C.y_instance +__main__.C.z_class +__main__.C.z_instance +__main__.x +__main__.y +__main__.z + +[case testLiteralTriggersFunctions] +from typing_extensions import Literal + +def function_1() -> int: pass +def function_2() -> Literal[1]: pass +def function_3() -> Literal[1]: pass + +def function_4(x: int) -> None: pass +def function_5(x: Literal[1]) -> None: pass +def function_6(x: Literal[1]) -> None: pass + +def function_same_1() -> Literal[1]: pass +def function_same_2(x: Literal[1]) -> None: pass + +class C: + def method_1(self) -> int: pass + def method_2(self) -> Literal[1]: pass + def method_3(self) -> Literal[1]: pass + + def method_4(self, x: int) -> None: pass + def method_5(self, x: Literal[1]) -> None: pass + def method_6(self, x: Literal[1]) -> None: pass + + def method_same_1(self) -> Literal[1]: pass + def method_same_2(self, x: Literal[1]) -> None: pass + + @classmethod + def classmethod_1(cls) -> int: pass + @classmethod + def classmethod_2(cls) -> Literal[1]: pass + @classmethod + def classmethod_3(cls) -> Literal[1]: pass + + @classmethod + def classmethod_4(cls, x: int) -> None: pass + @classmethod + def classmethod_5(cls, x: Literal[1]) -> None: pass + @classmethod + def classmethod_6(cls, x: Literal[1]) -> None: pass + + @classmethod + def classmethod_same_1(cls) -> Literal[1]: pass + @classmethod + def classmethod_same_2(cls, x: Literal[1]) -> None: pass + + @staticmethod + def staticmethod_1() -> int: pass + @staticmethod + def staticmethod_2() -> Literal[1]: pass + @staticmethod + def staticmethod_3() -> Literal[1]: pass + + @staticmethod + def staticmethod_4(x: int) -> None: pass + @staticmethod + def staticmethod_5(x: Literal[1]) -> None: pass + @staticmethod + def staticmethod_6(x: Literal[1]) -> None: pass + + @staticmethod + def staticmethod_same_1() -> Literal[1]: pass + @staticmethod + def staticmethod_same_2(x: Literal[1]) -> None: pass + +[file next.py] +from typing_extensions import Literal + +def function_1() -> Literal[1]: pass +def function_2() -> int: pass +def function_3() -> Literal[2]: pass + +def function_4(x: Literal[1]) -> None: pass +def function_5(x: int) -> None: pass +def function_6(x: Literal[2]) -> None: pass + +def function_same_1() -> Literal[1]: pass +def function_same_2(x: Literal[1]) -> None: pass + +class C: + def method_1(self) -> Literal[1]: pass + def method_2(self) -> int: pass + def method_3(self) -> Literal[2]: pass + + def method_4(self, x: Literal[1]) -> None: pass + def method_5(self, x: int) -> None: pass + def method_6(self, x: Literal[2]) -> None: pass + + def method_same_1(self) -> Literal[1]: pass + def method_same_2(self, x: Literal[1]) -> None: pass + + @classmethod + def classmethod_1(cls) -> Literal[1]: pass + @classmethod + def classmethod_2(cls) -> int: pass + @classmethod + def classmethod_3(cls) -> Literal[2]: pass + + @classmethod + def classmethod_4(cls, x: Literal[1]) -> None: pass + @classmethod + def classmethod_5(cls, x: int) -> None: pass + @classmethod + def classmethod_6(cls, x: Literal[2]) -> None: pass + + @classmethod + def classmethod_same_1(cls) -> Literal[1]: pass + @classmethod + def classmethod_same_2(cls, x: Literal[1]) -> None: pass + + @staticmethod + def staticmethod_1() -> Literal[1]: pass + @staticmethod + def staticmethod_2() -> int: pass + @staticmethod + def staticmethod_3() -> Literal[2]: pass + + @staticmethod + def staticmethod_4(x: Literal[1]) -> None: pass + @staticmethod + def staticmethod_5(x: int) -> None: pass + @staticmethod + def staticmethod_6(x: Literal[2]) -> None: pass + + @staticmethod + def staticmethod_same_1() -> Literal[1]: pass + @staticmethod + def staticmethod_same_2(x: Literal[1]) -> None: pass +[builtins fixtures/classmethod.pyi] +[out] +__main__.C.classmethod_1 +__main__.C.classmethod_2 +__main__.C.classmethod_3 +__main__.C.classmethod_4 +__main__.C.classmethod_5 +__main__.C.classmethod_6 +__main__.C.method_1 +__main__.C.method_2 +__main__.C.method_3 +__main__.C.method_4 +__main__.C.method_5 +__main__.C.method_6 +__main__.C.staticmethod_1 +__main__.C.staticmethod_2 +__main__.C.staticmethod_3 +__main__.C.staticmethod_4 +__main__.C.staticmethod_5 +__main__.C.staticmethod_6 +__main__.function_1 +__main__.function_2 +__main__.function_3 +__main__.function_4 +__main__.function_5 +__main__.function_6 + +[case testLiteralTriggersProperty] +from typing_extensions import Literal + +class C: + @property + def p1(self) -> Literal[1]: pass + + @property + def p2(self) -> int: pass + + @property + def same(self) -> Literal[1]: pass + +[file next.py] +from typing_extensions import Literal + +class C: + @property + def p1(self) -> int: pass + + @property + def p2(self) -> Literal[1]: pass + + @property + def same(self) -> Literal[1]: pass +[builtins fixtures/property.pyi] +[out] +__main__.C.p1 +__main__.C.p2 + +[case testLiteralsTriggersOverload] +from typing import overload +from typing_extensions import Literal + +@overload +def func(x: str) -> str: ... +@overload +def func(x: Literal[1]) -> int: ... +def func(x): + pass + +@overload +def func_same(x: str) -> str: ... +@overload +def func_same(x: Literal[1]) -> int: ... +def func_same(x): + pass + +class C: + @overload + def method(self, x: str) -> str: ... + @overload + def method(self, x: Literal[1]) -> int: ... + def method(self, x): + pass + + @overload + def method_same(self, x: str) -> str: ... + @overload + def method_same(self, x: Literal[1]) -> int: ... + def method_same(self, x): + pass + +[file next.py] +from typing import overload +from typing_extensions import Literal + +@overload +def func(x: str) -> str: ... +@overload +def func(x: Literal[2]) -> int: ... +def func(x): + pass + +@overload +def func_same(x: str) -> str: ... +@overload +def func_same(x: Literal[1]) -> int: ... +def func_same(x): + pass + +class C: + @overload + def method(self, x: str) -> str: ... + @overload + def method(self, x: Literal[2]) -> int: ... + def method(self, x): + pass + + @overload + def method_same(self, x: str) -> str: ... + @overload + def method_same(self, x: Literal[1]) -> int: ... + def method_same(self, x): + pass +[builtins fixtures/tuple.pyi] +[out] +__main__.C.method +__main__.func + +[case testUnionOfLiterals] +from typing_extensions import Literal +x: Literal[1, '2'] +[file next.py] +from typing_extensions import Literal +x: Literal[1, 2] +[builtins fixtures/tuple.pyi] +[out] +__main__.x + +[case testUnionOfCallables] +from typing import Callable, Union +from mypy_extensions import Arg +x: Union[Callable[[Arg(int, 'x')], None], + Callable[[int], None]] +[file next.py] +from typing import Callable, Union +from mypy_extensions import Arg +x: Union[Callable[[Arg(int, 'y')], None], + Callable[[int], None]] +[builtins fixtures/tuple.pyi] +[out] +__main__.x + +[case testChangeParamSpec] +from typing import ParamSpec, TypeVar +A = ParamSpec('A') +B = ParamSpec('B') +C = TypeVar('C') +[file next.py] +from typing import ParamSpec, TypeVar +A = ParamSpec('A') +B = TypeVar('B') +C = ParamSpec('C') +[out] +__main__.B +__main__.C + +[case testEmptyBodySuper] +from abc import abstractmethod +class C: + @abstractmethod + def meth(self) -> int: ... +[file next.py] +from abc import abstractmethod +class C: + @abstractmethod + def meth(self) -> int: return 0 +[out] +__main__.C.meth + +[case testGenericFunctionWithOptionalReturnType] +from typing import Type, TypeVar + +T = TypeVar("T") + +class C: + @classmethod + def get_by_team_and_id( + cls: Type[T], + raw_member_id: int, + include_removed: bool = False, + ) -> T: + pass + +[file next.py] +from typing import Type, TypeVar, Optional + +T = TypeVar("T") + +class C: + @classmethod + def get_by_team_and_id( + cls: Type[T], + raw_member_id: int, + include_removed: bool = False, + ) -> Optional[T]: + pass + +[builtins fixtures/classmethod.pyi] +[out] +__main__.C.get_by_team_and_id +__main__.Optional + +[case testPEP695TypeAlias] +# flags: --python-version=3.12 +from typing_extensions import TypeAlias, TypeAliasType +type A = int +type B = str +type C = int +D = int +E: TypeAlias = int +F = TypeAliasType("F", int) +G = TypeAliasType("G", int) +type H = int + +[file next.py] +# flags: --python-version=3.12 +from typing_extensions import TypeAlias, TypeAliasType +type A = str +type B = str +type C[T] = int +type D = int +type E = int +type F = int +type G = str +type H[T] = int + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] +[out] +__main__.A +__main__.C +__main__.D +__main__.E +__main__.G +__main__.H + +[case testPEP695TypeAlias2] +# flags: --python-version=3.12 +type A[T: int] = list[T] +type B[T: int] = list[T] +type C[T: (int, str)] = list[T] +type D[T: (int, str)] = list[T] +type E[T: int] = list[T] +type F[T: (int, str)] = list[T] + +[file next.py] +# flags: --python-version=3.12 +type A[T] = list[T] +type B[T: str] = list[T] +type C[T: (int, None)] = list[T] +type D[T] = list[T] +type E[T: int] = list[T] +type F[T: (int, str)] = list[T] + +[out] +__main__.A +__main__.B +__main__.C +__main__.D + +[case testPEP695GenericFunction] +# flags: --python-version=3.12 +def f[T](x: T) -> T: + return x +def g[T](x: T, y: T) -> T: + return x +[file next.py] +# flags: --python-version=3.12 +def f[T](x: T) -> T: + return x +def g[T, S](x: T, y: S) -> S: + return y +[out] +__main__.g + +[case testPEP695GenericClass] +# flags: --python-version=3.12 +class C[T]: + pass +class D[T]: + pass +class E[T]: + pass +class F[T]: + def f(self, x: object) -> T: ... +[file next.py] +# flags: --python-version=3.12 +class C[T]: + pass +class D[T: int]: + pass +class E: + pass +class F[T]: + def f(self, x: T) -> T: ... +[out] +__main__.D +__main__.E +__main__.F +__main__.F.f diff --git a/test-data/unit/envvars.test b/test-data/unit/envvars.test new file mode 100644 index 000000000000..8832f80cff3c --- /dev/null +++ b/test-data/unit/envvars.test @@ -0,0 +1,10 @@ +# Test cases related to environment variables +[case testEnvvar_MYPY_CONFIG_FILE_DIR] +# cmd: mypy --config-file=subdir/mypy.ini +[file bogus.py] +FOO = 'x' # type: int +[file subdir/good.py] +BAR = 0 # type: int +[file subdir/mypy.ini] +\[mypy] +files=$MYPY_CONFIG_FILE_DIR/good.py diff --git a/test-data/unit/errorstream.test b/test-data/unit/errorstream.test new file mode 100644 index 000000000000..46af433f8916 --- /dev/null +++ b/test-data/unit/errorstream.test @@ -0,0 +1,54 @@ +-- Test cases for incremental error streaming. +-- Each time errors are reported, '==== Errors flushed ====' is printed. + +[case testErrorStream] +import b +[file a.py] +1 + '' +[file b.py] +import a +'' / 2 +[out] +==== Errors flushed ==== +a.py:1: error: Unsupported operand types for + ("int" and "str") +==== Errors flushed ==== +b.py:2: error: Unsupported operand types for / ("str" and "int") + +[case testBlockers] +import b +[file a.py] +1 + '' +[file b.py] +import a +break +1 / '' # won't get reported, after a blocker +[out] +==== Errors flushed ==== +a.py:1: error: Unsupported operand types for + ("int" and "str") +==== Errors flushed ==== +b.py:2: error: "break" outside loop + +[case testCycles] +import a +[file a.py] +import b +1 + '' +def f() -> int: + reveal_type(b.x) + return b.x +y = 0 + int() +[file b.py] +import a +def g() -> int: + reveal_type(a.y) + return a.y +1 / '' +x = 1 + int() + +[out] +==== Errors flushed ==== +b.py:3: note: Revealed type is "builtins.int" +b.py:5: error: Unsupported operand types for / ("int" and "str") +==== Errors flushed ==== +a.py:2: error: Unsupported operand types for + ("int" and "str") +a.py:4: note: Revealed type is "builtins.int" diff --git a/test-data/unit/fine-grained-attr.test b/test-data/unit/fine-grained-attr.test new file mode 100644 index 000000000000..8606fea15849 --- /dev/null +++ b/test-data/unit/fine-grained-attr.test @@ -0,0 +1,82 @@ +[case updateMagicField] +from attrs import Attribute +import m + +def g() -> Attribute[int]: + return m.A.__attrs_attrs__[0] + +[file m.py] +from attrs import define + +@define +class A: + a: int +[file m.py.2] +from attrs import define + +@define +class A: + a: float +[builtins fixtures/plugin_attrs.pyi] +[out] +== +main:5: error: Incompatible return value type (got "Attribute[float]", expected "Attribute[int]") + +[case magicAttributeConsistency] +import m + +[file c.py] +from attrs import define + +@define +class A: + a: float + b: int +[builtins fixtures/plugin_attrs.pyi] + +[file m.py] +from c import A + +A.__attrs_attrs__.a + +[file m.py.2] +from c import A + +A.__attrs_attrs__.b + +[out] +== + +[case magicAttributeConsistency2-only_when_cache] +[file c.py] +import attrs + +@attrs.define +class Entry: + var: int +[builtins fixtures/plugin_attrs.pyi] + +[file m.py] +from typing import Any, ClassVar, Protocol +from c import Entry + +class AttrsInstance(Protocol): + __attrs_attrs__: ClassVar[Any] + +def func(e: AttrsInstance) -> None: ... +func(Entry(2)) + +[file m.py.2] +from typing import Any, ClassVar, Protocol +from c import Entry + +class AttrsInstance(Protocol): + __attrs_attrs__: ClassVar[Any] + +def func(e: AttrsInstance) -> int: + return 2 # Change return type to force reanalysis + +func(Entry(2)) + +[out] +== diff --git a/test-data/unit/fine-grained-blockers.test b/test-data/unit/fine-grained-blockers.test new file mode 100644 index 000000000000..33dedd887114 --- /dev/null +++ b/test-data/unit/fine-grained-blockers.test @@ -0,0 +1,568 @@ +-- Test cases for fine-grained incremental mode and blocking errors +-- +-- The comments in fine-grained.test explain how these tests work. + +-- TODO: +-- - blocking error while other existing errors as well (that get preserved) +-- - differences in other modules + blocking error + +[case testParseError] +import a +a.f() +[file a.py] +def f() -> None: pass +[file a.py.2] +def f(x: int) -> +[file a.py.3] +def f(x: int) -> None: pass +[file a.py.4] +def f() -> None: pass +[out] +== +a.py:1: error: invalid syntax +== +main:2: error: Missing positional argument "x" in call to "f" +== +[out version>=3.10] +== +a.py:1: error: expected ':' +== +main:2: error: Missing positional argument "x" in call to "f" +== + +[case testParseErrorShowSource] +# flags: --pretty --show-error-codes +import a +a.f() +[file a.py] +def f() -> None: pass +[file a.py.2] +def f(x: int) -> +[file a.py.3] +def f(x: int) -> None: pass +[file a.py.4] +def f() -> None: pass +[out] +== +a.py:1: error: invalid syntax [syntax] + def f(x: int) -> + ^ +== +main:3: error: Missing positional argument "x" in call to "f" [call-arg] + a.f() + ^~~~~ +== +[out version>=3.10] +== +a.py:1: error: expected ':' [syntax] + def f(x: int) -> + ^ +== +main:3: error: Missing positional argument "x" in call to "f" [call-arg] + a.f() + ^~~~~ +== + +[case testParseErrorMultipleTimes] +import a +a.f() +[file a.py] +def f() -> None: pass +[file a.py.2] +def f(x: int) -> +[file a.py.3] +def f(x: int + ) -> None +[file a.py.4] +def f(x: int) -> None: pass +[out] +== +a.py:1: error: invalid syntax +== +a.py:2: error: invalid syntax +== +main:2: error: Missing positional argument "x" in call to "f" +[out version>=3.10] +== +a.py:1: error: expected ':' +== +a.py:2: error: expected ':' +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testSemanticAnalysisBlockingError] +import a +a.f() +[file a.py] +def f() -> None: pass +[file a.py.2] +def f() -> None: pass +break +[file a.py.3] +def f(x: int) -> None: pass +[out] +== +a.py:2: error: "break" outside loop +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testBlockingErrorWithPreviousError] +import a +import b +a.f(1) +def g() -> None: + b.f(1) +[file a.py] +def f() -> None: pass +[file b.py] +def f() -> None: pass +[file a.py.2] +def f() -> None +[file a.py.3] +def f() -> None: pass +[out] +main:3: error: Too many arguments for "f" +main:5: error: Too many arguments for "f" +== +a.py:1: error: invalid syntax +== +main:3: error: Too many arguments for "f" +main:5: error: Too many arguments for "f" +[out version>=3.10] +main:3: error: Too many arguments for "f" +main:5: error: Too many arguments for "f" +== +a.py:1: error: expected ':' +== +main:3: error: Too many arguments for "f" +main:5: error: Too many arguments for "f" + +[case testUpdateClassReferenceAcrossBlockingError] +import a +c: a.C + +def f() -> None: + c.f() +[file a.py] +class C: + def f(self) -> None: pass +[file a.py.2] +error error +[file a.py.3] +class C: + def f(self, x: int) -> None: pass +[out] +== +a.py:1: error: invalid syntax +== +main:5: error: Missing positional argument "x" in call to "f" of "C" +[out version==3.10.0] +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +main:5: error: Missing positional argument "x" in call to "f" of "C" + +[case testAddFileWithBlockingError] +import a +a.f(1) +[file a.py.2] +x x +[file a.py.3] +def f() -> None: pass +[out] +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +a.py:1: error: invalid syntax +== +main:2: error: Too many arguments for "f" +[out version==3.10.0] +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +main:2: error: Too many arguments for "f" + +[case testModifyTwoFilesOneWithBlockingError1] +import a +[file a.py] +import b +def f() -> None: pass +b.g() +[file b.py] +import a +a.f() +def g() -> None: pass +[file a.py.2] +import b # Dummy edit +def f() -> None: pass +b.g() +[file b.py.2] +import a a # Syntax error +a.f() +def g() -> None: pass +[file b.py.3] +import a +a.f() +def g() -> None: pass +[out] +== +b.py:1: error: invalid syntax +== + +[case testModifyTwoFilesOneWithBlockingError2] +import a +[file a.py] +import b +def f() -> None: pass +b.g() +[file b.py] +import a +a.f() +def g() -> None: pass +[file a.py.2] +import b b +def f() -> None: pass +b.g() +[file b.py.2] +import a # Dummy edit +a.f() +def g() -> None: pass +[file a.py.3] +import b +def f() -> None: pass +b.g() +[out] +== +a.py:1: error: invalid syntax +== + +[case testBlockingErrorRemainsUnfixed] +import a +[file a.py] +import b +b.f() +[file b.py] +def f() -> None: pass +[file a.py.2] +x x +[file b.py.3] +def f(x: int) -> None: pass +[file a.py.4] +import b +b.f() +[out] +== +a.py:1: error: invalid syntax +== +a.py:1: error: invalid syntax +== +a.py:2: error: Missing positional argument "x" in call to "f" +[out version==3.10.0] +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +a.py:2: error: Missing positional argument "x" in call to "f" + +[case testModifyTwoFilesIntroduceTwoBlockingErrors] +import a +[file a.py] +import b +def f() -> None: pass +b.g() +[file b.py] +import a +a.f() +def g() -> None: pass +[file a.py.2] +import b b +def f() -> None: pass +b.g() +[file b.py.2] +import a a +a.f() +def g() -> None: pass +[file a.py.3] +import b b +def f() -> None: pass +b.g() +[file b.py.3] +import a a +a.f() +def g() -> None: pass +[file a.py.4] +import b +def f() -> None: pass +b.g(1) +[file b.py.4] +import a +def g() -> None: pass +a.f(1) +[out] +== +a.py:1: error: invalid syntax +== +a.py:1: error: invalid syntax +== +a.py:3: error: Too many arguments for "g" +b.py:3: error: Too many arguments for "f" + +[case testDeleteFileWithBlockingError-only_when_nocache] +-- Different cache/no-cache tests because: +-- Error message ordering differs +import a +import b +[file a.py] +def f() -> None: pass +[file b.py] +import a +a.f() +[file a.py.2] +x x +[delete a.py.3] +[out] +== +a.py:1: error: invalid syntax +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +b.py:1: error: Cannot find implementation or library stub for module named "a" +[out version==3.10.0] +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +b.py:1: error: Cannot find implementation or library stub for module named "a" + +[case testDeleteFileWithBlockingError2-only_when_cache] +-- Different cache/no-cache tests because: +-- Error message ordering differs +import a +import b +[file a.py] +def f() -> None: pass +[file b.py] +import a +a.f() +[file a.py.2] +x x +[delete a.py.3] +[out] +== +a.py:1: error: invalid syntax +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "a" +[out version==3.10.0] +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "a" + +[case testModifyFileWhileBlockingErrorElsewhere] +import a +import b +[file a.py] +[file b.py] +import a +[file a.py.2] +x x +[file b.py.3] +import a +a.f() +1() +[file a.py.4] +[builtins fixtures/module.pyi] +[out] +== +a.py:1: error: invalid syntax +== +a.py:1: error: invalid syntax +== +b.py:2: error: Module has no attribute "f" +b.py:3: error: "int" not callable +[out version==3.10.0] +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +b.py:2: error: Module has no attribute "f" +b.py:3: error: "int" not callable + +[case testImportBringsAnotherFileWithBlockingError1] +import a +[file a.py] +[file a.py.2] +import blocker +1() +[file a.py.3] +1() +def f() -> None: pass +[out] +== +/test-data/unit/lib-stub/blocker.pyi:2: error: invalid syntax +== +a.py:1: error: "int" not callable +[out version==3.10.0] +== +/test-data/unit/lib-stub/blocker.pyi:2: error: invalid syntax. Perhaps you forgot a comma? +== +a.py:1: error: "int" not callable + +[case testImportBringsAnotherFileWithSemanticAnalysisBlockingError] +import a +[file a.py] +[file a.py.2] +import blocker2 +1() +[file a.py.3] +1() +[out] +== +/test-data/unit/lib-stub/blocker2.pyi:2: error: "continue" outside loop +== +a.py:1: error: "int" not callable + +[case testFixingBlockingErrorTriggersDeletion1-only_when_nocache] +-- Disabled in cache mdode: +-- Cache mode fails to produce the error in the final step, but this is +-- a manifestation of a bug that can occur in no-cache mode also. +import a + +def g(x: a.A) -> None: + x.f() +[file a.py] +class A: + def f(self) -> None: pass +[delete a.py.2] +[file a.py.3] +class A: pass +[builtins fixtures/module.pyi] +[out] +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main:4: error: "A" has no attribute "f" + +[case testFixingBlockingErrorTriggersDeletion2] +from a import A + +def g(x: A) -> None: + x.f() +[file a.py] +class A: + def f(self) -> None: pass +[delete a.py.2] +[file a.py.3] +[builtins fixtures/module.pyi] +[out] +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main:1: error: Module "a" has no attribute "A" + +[case testFixingBlockingErrorBringsInAnotherModuleWithBlocker] +import a +[file a.py] +[file a.py.2] +x y +[file a.py.3] +import blocker +1() +[file a.py.4] +import sys +1() +[builtins fixtures/tuple.pyi] +[out] +== +a.py:1: error: invalid syntax +== +/test-data/unit/lib-stub/blocker.pyi:2: error: invalid syntax +== +a.py:2: error: "int" not callable +[out version==3.10.0] +== +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +/test-data/unit/lib-stub/blocker.pyi:2: error: invalid syntax. Perhaps you forgot a comma? +== +a.py:2: error: "int" not callable + +[case testInitialBlocker] +# cmd: mypy a.py b.py +[file a.py] +1 1 +[file b.py] +def f() -> int: + return '' +[file a.py.2] +x = 1 +[file b.py.3] +def f() -> int: + return 0 +[out] +a.py:1: error: invalid syntax +== +b.py:2: error: Incompatible return value type (got "str", expected "int") +== +[out version==3.10.0] +a.py:1: error: invalid syntax. Perhaps you forgot a comma? +== +b.py:2: error: Incompatible return value type (got "str", expected "int") +== + +[case testDecodeErrorBlocker1-posix] +import a +a.f(1) +[file a.py] +def f(x: int) -> None: ... +[file a.py.2] +# coding: ascii +ä = 1 +[file a.py.3] +def f(x: str) -> None: ... +[out] +== +mypy: can't decode file 'tmp/a.py': 'ascii' codec can't decode byte 0xc3 in position 16: ordinal not in range(128) +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testDecodeErrorBlocker2-windows] +import a +a.f(1) +[file a.py] +def f(x: int) -> None: ... +[file a.py.2] +# coding: ascii +ä = 1 +[file a.py.3] +def f(x: str) -> None: ... +[out] +== +mypy: can't decode file 'tmp/a.py': 'ascii' codec can't decode byte 0xc3 in position 17: ordinal not in range(128) +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testDecodeErrorBlockerOnInitialRun-posix] +# Note that there's no test variant for Windows, since the above Windows test case is good enough. +import a +a.f(1) +[file a.py] +# coding: ascii +ä = 1 +[file a.py.2] +def f(x: str) -> None: ... +[out] +mypy: can't decode file 'tmp/a.py': 'ascii' codec can't decode byte 0xc3 in position 16: ordinal not in range(128) +== +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" diff --git a/test-data/unit/fine-grained-cache-incremental.test b/test-data/unit/fine-grained-cache-incremental.test new file mode 100644 index 000000000000..00157333efd7 --- /dev/null +++ b/test-data/unit/fine-grained-cache-incremental.test @@ -0,0 +1,241 @@ +-- Test cases for building caches for fine-grained mode using incremental +-- builds. +-- +-- These tests are run only with the cache. +-- +-- '# num_build_steps: N' specifies how many regular build steps (without the daemon) +-- to do before switching to running using the daemon. Default is 1. + + +-- Add file +-- -------- + + +[case testIncrCacheBasic1] +# num_build_steps: 2 +import a +[file a.py] +from b import x +def f() -> int: + return 0 +[file b.py] +x = 1 +[file a.py.2] +from b import x +def f() -> int: + return 0 + x +[file b.py.3] +x = 'hi' +[out] +== +== +a.py:3: error: Unsupported operand types for + ("int" and "str") + +[case testIncrCacheBasic2] +# num_build_steps: 2 +import a +[file a.py] +from b import x +def f() -> int: + return 0+x +[file b.py] +x = 1 +[file b.py.2] +from c import x +[file c.py.2] +x = 1 +[file c.py.3] +x = 'hi' +[out] +== +== +a.py:3: error: Unsupported operand types for + ("int" and "str") + +[case testIncrCacheDoubleChange1] +# num_build_steps: 2 +import b +import c +[file a.py] +def f(x: int) -> None: + pass +[file b.py] +from a import f +f(10) +[file c.py] +from a import f +f(10) + +[file a.py.2] +def f(x: int) -> None: + pass +# nothing changed + +[file a.py.3] +def f(x: str) -> None: + pass +[out] +== +== +c.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" +b.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testIncrCacheProtocol1] +# num_build_steps: 2 +import a +[file a.py] +import b +from typing import Protocol +class P(Protocol): + x: int +def f() -> None: + def g(x: P) -> None: + pass + g(b.C()) +[file c.py] +[file c.py.2] +# yo +[file b.py] +class C: + x: int +[file b.py.3] +class C: + x: str +-- If we did a *full* reload (because the proto cache failed to load), +-- nothing would show up as stale +[stale2 b] +[rechecked2 a, b] +[out] +== +== +a.py:8: error: Argument 1 to "g" has incompatible type "C"; expected "P" +a.py:8: note: Following member(s) of "C" have conflicts: +a.py:8: note: x: expected "int", got "str" + +[case testIncrCacheProtocol2] +# num_build_steps: 3 +import a +[file a.py] +import b +from typing import Protocol +class P(Protocol): + x: int +class Q(Protocol): + x: int +def f() -> None: + def g(x: P) -> None: + pass + g(b.C()) +[file c.py] +[file c.py.2] +# uh +[file c.py.3] +from a import Q +import b +def f() -> None: + def g(x: Q) -> None: + pass + g(b.C()) +[file b.py] +class C: + x: int +[file b.py.4] +class C: + x: str +-- If we did a *full* reload (because the proto cache failed to load), +-- nothing would show up as stale +[stale3 b] +[rechecked3 a, b, c] +[out] +== +== +== +c.py:6: error: Argument 1 to "g" has incompatible type "C"; expected "Q" +c.py:6: note: Following member(s) of "C" have conflicts: +c.py:6: note: x: expected "int", got "str" +a.py:10: error: Argument 1 to "g" has incompatible type "C"; expected "P" +a.py:10: note: Following member(s) of "C" have conflicts: +a.py:10: note: x: expected "int", got "str" + +[case testIncrCacheProtocol3] +# num_build_steps: 2 +import a +[file a.py] +import b +from typing import Protocol +class P(Protocol): + x: int +def f() -> None: + def g(x: P) -> None: + pass + g(b.C()) +[file c.py] +[file c.py.2] +# yo +[file b.py] +class C: + x: int +[file b.py.3] +class C: + x: int + y: int +[file b.py.4] +class C: + x: str + y: int +-- If we did a *full* reload (because the proto cache failed to load), +-- nothing would show up as stale +[stale2 b] +[rechecked2 b] +[stale3 b] +[rechecked3 a, b] +[out] +== +== +== +a.py:8: error: Argument 1 to "g" has incompatible type "C"; expected "P" +a.py:8: note: Following member(s) of "C" have conflicts: +a.py:8: note: x: expected "int", got "str" + +[case testIncrCacheBustedProtocol] +# flags: --no-sqlite-cache +[file a.py] +[file b.py] +-- This is a heinous hack, but we simulate having a invalid cache by clobbering +-- the proto deps file with something with mtime mismatches. +[file ../.mypy_cache/3.8/@deps.meta.json.2] +{"snapshot": {"__main__": "a7c958b001a45bd6a2a320f4e53c4c16", "a": "d41d8cd98f00b204e9800998ecf8427e", "b": "d41d8cd98f00b204e9800998ecf8427e", "builtins": "c532c89da517a4b779bcf7a964478d67"}, "deps_meta": {"@root": {"path": "@root.deps.json", "mtime": 0}, "__main__": {"path": "__main__.deps.json", "mtime": 0}, "a": {"path": "a.deps.json", "mtime": 0}, "b": {"path": "b.deps.json", "mtime": 0}, "builtins": {"path": "builtins.deps.json", "mtime": 0}}} + +[file b.py.2] +# uh +-- A full reload shows up as nothing getting rechecked by fine-grained mode. +-- If we did not do a full reload, b would be stale and checked in fine-grained mode +[stale] +[rechecked] +[out] +== + +[case testInvalidateCachePart] +# flags: --no-sqlite-cache +# cmd: mypy a1.py a2.py b.py p/__init__.py p/c.py +[file a1.py] +import p +from b import x +from p.c import C + +[file a2.py] +import p +from b import x +from p.c import C + +[file b.py] +x = 10 + +[file p/__init__.py] +[file p/c.py] +class C: pass + +[delete ../.mypy_cache/3.8/b.meta.json.2] +[delete ../.mypy_cache/3.8/p/c.meta.json.2] + +[out] +== diff --git a/test-data/unit/fine-grained-cycles.test b/test-data/unit/fine-grained-cycles.test new file mode 100644 index 000000000000..16915423e472 --- /dev/null +++ b/test-data/unit/fine-grained-cycles.test @@ -0,0 +1,221 @@ +-- Test cases for fine-grained incremental checking and import cycles +-- +-- The comment at the top of fine-grained.test explains how these tests +-- work. + +[case testFunctionSelfReferenceThroughImportCycle] +import a +[file a.py] +from b import f +[file b.py] +import a + +def f() -> None: + a.f() +[file b.py.2] +import a + +def f(x: int) -> None: + a.f() +[out] +== +b.py:4: error: Missing positional argument "x" in call to "f" + +[case testClassSelfReferenceThroughImportCycle] +import a +[file a.py] +from b import A +[file b.py] +import a + +class A: + def g(self) -> None: pass + +def f() -> None: + a.A().g() +[file b.py.2] +import a + +class A: + def g(self, x: int) -> None: pass + +def f() -> None: + a.A().g() +[out] +== +b.py:7: error: Missing positional argument "x" in call to "g" of "A" + +[case testAnnotationSelfReferenceThroughImportCycle] +import a +[file a.py] +from b import A +[file b.py] +import a + +x: a.A + +class A: + def g(self) -> None: pass + +def f() -> None: + x.g() +[file b.py.2] +import a + +x: a.A + +class A: + def g(self, x: int) -> None: pass + +def f() -> None: + x.g() +[out] +== +b.py:9: error: Missing positional argument "x" in call to "g" of "A" + +[case testModuleSelfReferenceThroughImportCycle] +import a +[file a.py] +import b +[file b.py] +import a + +def f() -> None: + a.b.f() +[file b.py.2] +import a + +def f(x: int) -> None: + a.b.f() +[out] +== +b.py:4: error: Missing positional argument "x" in call to "f" + +[case testVariableSelfReferenceThroughImportCycle] +import a +[file a.py] +from b import x +[file b.py] +import a + +x: int + +def f() -> None: + a.x = 1 +[file b.py.2] +import a + +x: str + +def f() -> None: + a.x = 1 +[out] +== +b.py:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testReferenceToTypeThroughCycle] +import a + +[file a.py] +from b import C + +def f() -> C: pass + +[file b.py] +import a + +class C: + def g(self) -> None: pass + +def h() -> None: + c = a.f() + c.g() + +[file b.py.2] +import a + +class C: + def g(self, x: int) -> None: pass + +def h() -> None: + c = a.f() + c.g() + +[out] +== +b.py:8: error: Missing positional argument "x" in call to "g" of "C" + +[case testReferenceToTypeThroughCycleAndDeleteType] +import a + +[file a.py] +from b import C + +def f() -> C: pass + +[file b.py] +import a + +class C: + def g(self) -> None: pass + +def h() -> None: + c = a.f() + c.g() + +[file b.py.2] +import a + +def h() -> None: + c = a.f() + c.g() + +[out] +== +a.py:1: error: Module "b" has no attribute "C" + +[case testReferenceToTypeThroughCycleAndReplaceWithFunction] + +import a + +[file a.py] +from b import C + +def f() -> C: pass + +[file b.py] +import a + +class C: + def g(self) -> None: pass + +def h() -> None: + c = a.f() + c.g() + +[file b.py.2] +import a + +def C() -> int: pass + +def h() -> None: + c = a.f() + c.g() + +[out] +== +a.py:3: error: Function "b.C" is not valid as a type +a.py:3: note: Perhaps you need "Callable[...]" or a callback protocol? +b.py:7: error: C? has no attribute "g" + +-- TODO: More import cycle: +-- +-- * "from x import y" through cycle +-- * "from x import *" through cycle +-- * "Cls.module" though cycle +-- * TypeVar +-- * type alias +-- * all kinds of reference deleted +-- * all kinds of reference rebound to different kind +-- +-- etc. diff --git a/test-data/unit/fine-grained-dataclass-transform.test b/test-data/unit/fine-grained-dataclass-transform.test new file mode 100644 index 000000000000..89628256fda5 --- /dev/null +++ b/test-data/unit/fine-grained-dataclass-transform.test @@ -0,0 +1,141 @@ +[case updateDataclassTransformParameterViaDecorator] +# flags: --python-version 3.11 +from m import my_dataclass + +@my_dataclass +class Foo: + x: int + +foo = Foo(1) +foo.x = 2 + +[file m.py] +from typing import dataclass_transform + +@dataclass_transform(frozen_default=False) +def my_dataclass(cls): return cls + +[file m.py.2] +from typing import dataclass_transform + +@dataclass_transform(frozen_default=True) +def my_dataclass(cls): return cls + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[out] +== +main:9: error: Property "x" defined in "Foo" is read-only + +[case updateDataclassTransformParameterViaParentClass] +# flags: --python-version 3.11 +from m import Dataclass + +class Foo(Dataclass): + x: int + +foo = Foo(1) +foo.x = 2 + +[file m.py] +from typing import dataclass_transform + +@dataclass_transform(frozen_default=False) +class Dataclass: ... + +[file m.py.2] +from typing import dataclass_transform + +@dataclass_transform(frozen_default=True) +class Dataclass: ... + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[out] +== +main:8: error: Property "x" defined in "Foo" is read-only + +[case updateBaseClassToUseDataclassTransform] +# flags: --python-version 3.11 +from m import A + +class B(A): + y: int + +B(x=1, y=2) + +[file m.py] +class Dataclass: ... + +class A(Dataclass): + x: int + +[file m.py.2] +from typing import dataclass_transform + +@dataclass_transform() +class Dataclass: ... + +class A(Dataclass): + x: int + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +[out] +main:7: error: Unexpected keyword argument "x" for "B" +builtins.pyi:14: note: "B" defined here +main:7: error: Unexpected keyword argument "y" for "B" +builtins.pyi:14: note: "B" defined here +== + +[case frozenInheritanceViaDefault] +# flags: --python-version 3.11 +from foo import Foo + +foo = Foo(base=0, foo=1) + +[file transform.py] +from typing import dataclass_transform, Type + +@dataclass_transform(frozen_default=True) +def dataclass(cls: Type) -> Type: return cls + +[file base.py] +from transform import dataclass + +@dataclass +class Base: + base: int + +[file foo.py] +from base import Base +from transform import dataclass + +@dataclass +class Foo(Base): + foo: int + +[file foo.py.2] +from base import Base +from transform import dataclass + +@dataclass +class Foo(Base): + foo: int + bar: int = 0 + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dataclasses.pyi] + +# If the frozen parameter is being maintained correctly, we *don't* expect to see issues; if it's +# broken in incremental mode, then we'll see an error about inheriting a non-frozen class from a +# frozen one. +# +# Ideally we'd also add a `foo.foo = 2` to confirm that frozen semantics are actually being +# enforced, but incremental tests currently can't start with an error, which makes it tricky to +# write such a test case. +[out] +== diff --git a/test-data/unit/fine-grained-dataclass.test b/test-data/unit/fine-grained-dataclass.test new file mode 100644 index 000000000000..036d858ddf69 --- /dev/null +++ b/test-data/unit/fine-grained-dataclass.test @@ -0,0 +1,25 @@ +[case testReplace] +[file model.py] +from dataclasses import dataclass + +@dataclass +class Model: + x: int = 0 +[file replace.py] +from dataclasses import replace +from model import Model + +m = Model() +replace(m, x=42) + +[file model.py.2] +from dataclasses import dataclass + +@dataclass +class Model: + x: str = 'hello' + +[builtins fixtures/dataclasses.pyi] +[out] +== +replace.py:5: error: Argument "x" to "replace" of "Model" has incompatible type "int"; expected "str" diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test new file mode 100644 index 000000000000..22f2a7895cf9 --- /dev/null +++ b/test-data/unit/fine-grained-follow-imports.test @@ -0,0 +1,848 @@ +-- Test cases for --follow-imports=normal in fine-grained incremental mode. +-- +-- Note that the implementation is mostly separate from normal incremental mode. + +[case testFollowImportsNormalBasic] +# flags: --follow-imports=normal +# cmd: mypy main.py a.py + +[file main.py] +import a +a.f() + +[file a.py] +def f() -> None: pass + +[file a.py.2] +def f(x: str) -> None: pass + +[file a.py.3] +def f() -> None: pass + +[out] +== +main.py:2: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalAddSuppressed] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a +a.f() + +[file a.py.2] +def f(x: str) -> None: pass + +[file a.py.3] +def f() -> None: pass + +[out] +main.py:1: error: Cannot find implementation or library stub for module named "a" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main.py:2: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalAddSuppressed2] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a # type: ignore +a.f() + +[file a.py.2] +def f(x: str) -> None: pass + +[file a.py.3] +def f() -> None: pass + +[out] +== +main.py:2: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalAddSuppressed3] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a +a.b.f() + +[file a.py.2] +import b + +[file b.py.2] +def f(x: str) -> None: pass + +[file b.py.3] +def f() -> None: pass + +[out] +main.py:1: error: Cannot find implementation or library stub for module named "a" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main.py:2: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalEditingFileBringNewModule] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] + +[file main.py.2] +import a +a.f() + +[file a.py.2] +def f(x: str) -> None: pass + +[file a.py.3] +def f() -> None: pass + +[out] +== +main.py:2: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalEditingFileBringNewModules] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] + +[file main.py.2] +import a +a.b.f() + +[file a.py.2] +import b + +[file b.py.2] +def f(x: str) -> None: pass + +[file b.py.3] +def f() -> None: pass + +[out] +== +main.py:2: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalDuringStartup] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a +a.f() + +[file a.py] +def f() -> None: pass + +[file a.py.2] +def f(x: str) -> None: pass + +[out] +== +main.py:2: error: Missing positional argument "x" in call to "f" + +[case testFollowImportsNormalDuringStartup2] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a +a.f() + +[file a.py] +def f(x: str) -> None: pass + +[file a.py.2] +def f() -> None: pass + +[out] +main.py:2: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalDuringStartup3] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a +a.g() + +[file a.py] +import b +g = b.f + +[file b.py] +def f(x: str) -> None: pass + +[file b.py.2] +def f() -> None: pass + +[file a.py.3] +g = 0 + +[out] +main.py:2: error: Too few arguments +== +== +main.py:2: error: "int" not callable + +[case testFollowImportsNormalDeleteFile1] +# flags: --follow-imports=normal +# cmd: mypy main.py a.py +# cmd2: mypy main.py +# cmd3: mypy main.py a.py + +[file main.py] +import a +a.f() + +[file a.py] +def f() -> None: pass + +[delete a.py.2] + +[file a.py.3] +def f(x: str) -> None: pass + +[out] +== +main.py:1: error: Cannot find implementation or library stub for module named "a" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main.py:2: error: Missing positional argument "x" in call to "f" + +[case testFollowImportsNormalDeleteFile2] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a +a.f() + +[file a.py] +def f() -> None: pass + +[delete a.py.2] + +[file a.py.3] +def f(x: str) -> None: pass + +[out] +== +main.py:1: error: Cannot find implementation or library stub for module named "a" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main.py:2: error: Missing positional argument "x" in call to "f" + +[case testFollowImportsNormalDeleteFile3] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a # type: ignore +a.f() + +[file a.py] +def f() -> None: pass + +[delete a.py.2] + +[file a.py.3] +def f(x: str) -> None: pass + +[out] +== +== +main.py:2: error: Missing positional argument "x" in call to "f" + +[case testFollowImportsNormalDeleteFile4] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a + +[file a.py] +1() + +[file main.py.2] +# don't import a + +[file main.py.3] +import a + +[out] +a.py:1: error: "int" not callable +== +== +a.py:1: error: "int" not callable + +[case testFollowImportsNormalDeleteFile5] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a + +[file a.py] +import b + +[file b.py] +1() + +[file a.py.2] +# don't import b + +[file a.py.3] +import b + +[out] +b.py:1: error: "int" not callable +== +== +b.py:1: error: "int" not callable + +[case testFollowImportsNormalDeleteFile6] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a + +[file a.py] +import b + +[file b.py] +1() + +[delete a.py.2] + +[file a.py.3] +import b + +[out] +b.py:1: error: "int" not callable +== +main.py:1: error: Cannot find implementation or library stub for module named "a" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +b.py:1: error: "int" not callable + +[case testFollowImportsNormalDeleteFile7] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a + +[file a.py] +import b +from c import f + +[file b.py] +from d import * + +[file c.py] +import a +def f(): pass +1() + +[file d.py] +1() + +[file main.py.2] + +[file main.py.3] +import d + +[out] +d.py:1: error: "int" not callable +c.py:3: error: "int" not callable +== +== +d.py:1: error: "int" not callable + +[case testFollowImportsNormalKeepAliveViaNewFile] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import a + +[file a.py] +import b + +[file b.py] +1() + +[file a.py.2] +import c + +[file c.py.2] +import d + +[file d.py.2] +import b + +[out] +b.py:1: error: "int" not callable +== +b.py:1: error: "int" not callable + +[case testFollowImportsNormalPackage-only_when_nocache] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import p.m # type: ignore + +p.m.f() + +[file p/__init__.py.2] + +[file p/m.py.2] +def f(x: str) -> None: pass + +1() + +[file p/m.py.3] +def f(x: str) -> None: pass + +[delete p.4] + +[out] +== +p/m.py:3: error: "int" not callable +main.py:3: error: Missing positional argument "x" in call to "f" +== +main.py:3: error: Missing positional argument "x" in call to "f" +== + +[case testFollowImportsNormalPackage2-only_when_cache] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import p.m # type: ignore + +p.m.f() + +[file p/__init__.py.2] + +[file p/m.py.2] +def f(x: str) -> None: pass + +1() + +[delete p.3] + +[out] +== +main.py:3: error: Missing positional argument "x" in call to "f" +p/m.py:3: error: "int" not callable +== + +[case testFollowImportsNormalPackageInitFile1] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import p1.m +from p2 import m + +[file p1/__init__.py.2] +1() + +[file p1/m.py.2] + +[file p2/__init__.py.3] +''() + +[file p2/m.py.3] + +[out] +main.py:1: error: Cannot find implementation or library stub for module named "p1.m" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main.py:1: error: Cannot find implementation or library stub for module named "p1" +main.py:2: error: Cannot find implementation or library stub for module named "p2" +== +main.py:2: error: Cannot find implementation or library stub for module named "p2" +main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +p1/__init__.py:1: error: "int" not callable +== +p1/__init__.py:1: error: "int" not callable +p2/__init__.py:1: error: "str" not callable + +[case testFollowImportsNormalPackageInitFile2] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +from p import m + +[file p/__init__.py.2] + +[file p/m.py.3] +''() + +[out] +main.py:1: error: Cannot find implementation or library stub for module named "p" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main.py:1: error: Module "p" has no attribute "m" +== +p/m.py:1: error: "str" not callable + +[case testFollowImportsNormalPackageInitFile3] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import p1.s1.m +from p2.s2 import m +p1.s1.m.f() +m.f(1) + +[file p1/__init__.py.2] + +[file p1/s1/__init__.py.2] + +[file p1/s1/m.py.2] +def f(x: str) -> None: pass + +[file p2/__init__.py.3] + +[file p2/s2/__init__.py.3] + +[file p2/s2/m.py.3] +def f() -> None: pass + +[out] +main.py:1: error: Cannot find implementation or library stub for module named "p1.s1.m" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main.py:1: error: Cannot find implementation or library stub for module named "p1.s1" +main.py:1: error: Cannot find implementation or library stub for module named "p1" +main.py:2: error: Cannot find implementation or library stub for module named "p2.s2" +== +main.py:2: error: Cannot find implementation or library stub for module named "p2.s2" +main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main.py:3: error: Missing positional argument "x" in call to "f" +== +main.py:3: error: Missing positional argument "x" in call to "f" +main.py:4: error: Too many arguments for "f" + +[case testFollowImportsNormalPackageInitFile4-only_when_cache] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import p1.m # type: ignore +from p2 import m # type: ignore + +[file p1/__init__.py.2] +1() + +[file p1/m.py.2] +''() + +[file p2/__init__.py.3] +''() + +[file p2/m.py.3] + +[out] +== +p1/__init__.py:1: error: "int" not callable +p1/m.py:1: error: "str" not callable +== +p1/__init__.py:1: error: "int" not callable +p1/m.py:1: error: "str" not callable +p2/__init__.py:1: error: "str" not callable + +[case testFollowImportsNormalSubmoduleCreatedWithImportInFunction] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +def f() -> None: + from p import m + +[file p/__init__.py.2] +1() + +[file p/m.py.2] +''() + +[out] +main.py:2: error: Cannot find implementation or library stub for module named "p" +main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +p/__init__.py:1: error: "int" not callable +p/m.py:1: error: "str" not callable + +[case testFollowImportsNormalPackageInitFileStub] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +from p import m + +[file p/__init__.pyi.2] +1() + +[file p/m.pyi.2] +''() + +[file p/mm.pyi.3] +x x x + +[out] +main.py:1: error: Cannot find implementation or library stub for module named "p" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +p/__init__.pyi:1: error: "int" not callable +p/m.pyi:1: error: "str" not callable +== +p/__init__.pyi:1: error: "int" not callable +p/m.pyi:1: error: "str" not callable + +[case testFollowImportsNormalNamespacePackages] +# flags: --follow-imports=normal --namespace-packages +# cmd: mypy main.py + +[file main.py] +import p1.m1 +import p2.m2 + +[file p1/m1.py] +1() + +[file p2/m2.py.2] +''() + +[delete p2/m2.py.3] + +[out] +p1/m1.py:1: error: "int" not callable +main.py:2: error: Cannot find implementation or library stub for module named "p2.m2" +main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main.py:2: error: Cannot find implementation or library stub for module named "p2" +== +p1/m1.py:1: error: "int" not callable +p2/m2.py:1: error: "str" not callable +== +p1/m1.py:1: error: "int" not callable +main.py:2: error: Cannot find implementation or library stub for module named "p2.m2" +main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testFollowImportsNormalNewFileOnCommandLine] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy main.py x.py + +[file main.py] +1() + +[file x.py.2] +''() + +[out] +main.py:1: error: "int" not callable +== +main.py:1: error: "int" not callable +x.py:1: error: "str" not callable + +[case testFollowImportsNormalSearchPathUpdate-only_when_nocache] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy main.py src/foo.py + +[file main.py] + +[file src/foo.py.2] +import bar +''() + +[file src/bar.py.2] +1() + +[out] +== +src/foo.py:2: error: "str" not callable +src/bar.py:1: error: "int" not callable + +[case testFollowImportsNormalSearchPathUpdate2-only_when_cache] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy main.py src/foo.py + +[file main.py] + +[file src/foo.py.2] +import bar +''() + +[file src/bar.py.2] +1() + +[out] +== +src/foo.py:2: error: "str" not callable +src/bar.py:1: error: "int" not callable + +[case testFollowImportsNormalSuppressedAffectsCachedFile-only_when_cache] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +from p import m # type: ignore +m.f(1) + +[file p/__init__.py] + +[file p/m.py.2] +# This change will trigger a cached file (main.py) through a suppressed +# submodule, resulting in additional errors in main.py. +def f() -> None: pass + +[out] +== +main.py:2: error: Too many arguments for "f" + +[case testFollowImportsNormalMultipleImportedModulesSpecialCase] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import pkg + +[file pkg/__init__.py.2] +from . import mod1 + +[file pkg/mod1.py.2] +from . import mod2 + +[file pkg/mod2.py.2] + +[out] +main.py:1: error: Cannot find implementation or library stub for module named "pkg" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== + +[case testFollowImportsNormalDeletePackage] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import pkg + +[file pkg/__init__.py] +from . import mod + +[file pkg/mod.py] +from . import mod2 +import pkg2 + +[file pkg/mod2.py] +from . import mod2 +import pkg2 + +[file pkg2/__init__.py] +from . import mod3 + +[file pkg2/mod3.py] + +[delete pkg/.2] +[delete pkg2/.2] + +[out] +== +main.py:1: error: Cannot find implementation or library stub for module named "pkg" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testNewImportCycleTypeVarBound] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy other.py + +[file main.py] +# empty + +[file other.py.2] +import trio + +[file trio/__init__.py.2] +from typing import TypeVar +import trio +from . import abc as abc + +T = TypeVar("T", bound=trio.abc.A) + +[file trio/abc.py.2] +import trio +class A: ... +[out] +== + +[case testNewImportCycleTupleBase] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy other.py + +[file main.py] +# empty + +[file other.py.2] +import trio + +[file trio/__init__.py.2] +from typing import TypeVar, Tuple +import trio +from . import abc as abc + +class C(Tuple[trio.abc.A, trio.abc.A]): ... + +[file trio/abc.py.2] +import trio +class A: ... +[builtins fixtures/tuple.pyi] +[out] +== + +[case testNewImportCycleTypedDict] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy other.py + +[file main.py] +# empty + +[file other.py.2] +import trio + +[file trio/__init__.py.2] +from typing import TypeVar +from typing_extensions import TypedDict +import trio +from . import abc as abc + +class C(TypedDict): + x: trio.abc.A + y: trio.abc.A + +[file trio/abc.py.2] +import trio +class A: ... +[builtins fixtures/dict.pyi] +[out] +== diff --git a/test-data/unit/fine-grained-inspect.test b/test-data/unit/fine-grained-inspect.test new file mode 100644 index 000000000000..ed89f2f099f9 --- /dev/null +++ b/test-data/unit/fine-grained-inspect.test @@ -0,0 +1,269 @@ +[case testInspectTypeBasic] +# inspect2: --include-kind tmp/foo.py:10:13 +# inspect2: --show=type --include-kind tmp/foo.py:10:13 +# inspect2: --include-span -vv tmp/foo.py:12:5 +# inspect2: --include-span --include-kind tmp/foo.py:12:5:12:9 +import foo +[file foo.py] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, x: T) -> None: ... + x: T + +def foo(arg: C[T]) -> T: + return arg.x + +foo(C(42)) +[out] +== +NameExpr -> "C[T]" +MemberExpr -> "T" +NameExpr -> "C[T]" +MemberExpr -> "T" +12:5:12:5 -> "Type[foo.C[Any]]" +12:5:12:9 -> "foo.C[builtins.int]" +12:1:12:10 -> "builtins.int" +CallExpr:12:5:12:9 -> "C[int]" + +[case testInspectAttrsBasic] +# inspect2: --show=attrs tmp/foo.py:6:1 +# inspect2: --show=attrs tmp/foo.py:7:1 +# inspect2: --show=attrs tmp/foo.py:10:1 +# inspect2: --show=attrs --include-object-attrs tmp/foo.py:10:1 +import foo +[file foo.py] +from bar import Meta +class C(metaclass=Meta): + x: int + def meth(self) -> None: ... + +c: C +C + +def foo() -> int: ... +foo +[file bar.py] +class Meta(type): + y: int +[out] +== +{"C": ["meth", "x"]} +{"C": ["meth", "x"], "Meta": ["y"], "type": ["__init__"]} +{"function": ["__name__"]} +{"function": ["__name__"], "object": ["__init__"]} + +[case testInspectDefBasic] +# inspect2: --show=definition tmp/foo.py:5:5 +# inspect2: --show=definition --include-kind tmp/foo.py:6:3 +# inspect2: --show=definition --include-span tmp/foo.py:7:5 +# inspect2: --show=definition tmp/foo.py:8:1:8:4 +# inspect2: --show=definition tmp/foo.py:8:6:8:8 +# inspect2: --show=definition tmp/foo.py:9:3 +import foo +[file foo.py] +from bar import var, test, A +from baz import foo + +a: A +a.meth() +a.x +A.B.y +test(var) +foo +[file bar.py] +class A: + x: int + @classmethod + def meth(cls) -> None: ... + class B: + y: int + +var = 42 +def test(x: int) -> None: ... +[file baz.py] +from typing import overload, Union + +@overload +def foo(x: int) -> None: ... +@overload +def foo(x: str) -> None: ... +def foo(x: Union[int, str]) -> None: + pass +[builtins fixtures/classmethod.pyi] +[out] +== +tmp/bar.py:4:0:meth +MemberExpr -> tmp/bar.py:2:5:x +7:1:7:5 -> tmp/bar.py:6:9:y +tmp/bar.py:9:1:test +tmp/bar.py:8:1:var +tmp/baz.py:3:2:foo + +[case testInspectFallbackAttributes] +# inspect2: --show=attrs --include-object-attrs tmp/foo.py:5:1 +# inspect2: --show=attrs tmp/foo.py:8:1 +# inspect2: --show=attrs --include-kind tmp/foo.py:10:1 +# inspect2: --show=attrs --include-kind --include-object-attrs tmp/foo.py:10:1 +import foo +[file foo.py] +class B: ... +class C(B): + x: int +c: C +c # line 5 + +t = 42, "foo" +t # line 8 + +None +[builtins fixtures/args.pyi] +[out] +== +{"C": ["x"], "object": ["__eq__", "__init__", "__ne__"]} +{"Iterable": ["__iter__"]} +NameExpr -> {} +NameExpr -> {"object": ["__eq__", "__init__", "__ne__"]} + +[case testInspectTypeVarBoundAttrs] +# inspect2: --show=attrs tmp/foo.py:8:13 +import foo +[file foo.py] +from typing import TypeVar + +class C: + x: int + +T = TypeVar('T', bound=C) +def foo(arg: T) -> T: + return arg +[out] +== +{"C": ["x"]} + +[case testInspectTypeVarValuesAttrs] +# inspect2: --show=attrs --force-reload tmp/foo.py:13:13 +# inspect2: --show=attrs --force-reload --union-attrs tmp/foo.py:13:13 +# inspect2: --show=attrs tmp/foo.py:16:5 +# inspect2: --show=attrs --union-attrs tmp/foo.py:16:5 +import foo +[file foo.py] +from typing import TypeVar, Generic + +class A: + x: int + z: int + +class B: + y: int + z: int + +T = TypeVar('T', A, B) +def foo(arg: T) -> T: + return arg + +class C(Generic[T]): + x: T +[out] +== +{"A": ["z"], "B": ["z"]} +{"A": ["x", "z"], "B": ["y", "z"]} +{"A": ["z"], "B": ["z"]} +{"A": ["x", "z"], "B": ["y", "z"]} + +[case testInspectTypeVarBoundDef] +# inspect2: --show=definition tmp/foo.py:9:13 +# inspect2: --show=definition tmp/foo.py:8:9 +import foo +[file foo.py] +from typing import TypeVar + +class C: + x: int + +T = TypeVar('T', bound=C) +def foo(arg: T) -> T: + arg.x + return arg +[out] +== +tmp/foo.py:7:9:arg +tmp/foo.py:4:5:x + +[case testInspectTypeVarValuesDef] +# inspect2: --show=definition --force-reload tmp/foo.py:13:9 +# inspect2: --show=definition --force-reload tmp/foo.py:14:13 +# inspect2: --show=definition tmp/foo.py:18:7 +import foo +[file foo.py] +from typing import TypeVar, Generic + +class A: + x: int + z: int + +class B: + y: int + z: int + +T = TypeVar('T', A, B) +def foo(arg: T) -> T: + arg.z + return arg + +class C(Generic[T]): + x: T + x.z +[out] +== +tmp/foo.py:5:5:z, tmp/foo.py:9:5:z +tmp/foo.py:12:9:arg +tmp/foo.py:5:5:z, tmp/foo.py:9:5:z + +[case testInspectModuleAttrs] +# inspect2: --show=attrs tmp/foo.py:2:1 +import foo +[file foo.py] +from pack import bar +bar +[file pack/__init__.py] +[file pack/bar.py] +x: int +def bar() -> None: ... +class C: ... +[builtins fixtures/module.pyi] +[out] +== +{"": ["C", "__annotations__", "__doc__", "__file__", "__name__", "__package__", "__spec__", "bar", "x"], "ModuleType": ["__file__", "__getattr__"]} + +[case testInspectModuleDef] +# inspect2: --show=definition --include-kind tmp/foo.py:2:1 +import foo +[file foo.py] +from pack import bar +bar.x +[file pack/__init__.py] +[file pack/bar.py] +pass +if True: + x: int +[out] +== +NameExpr -> tmp/pack/bar.py:1:1:bar +MemberExpr -> tmp/pack/bar.py:3:5:x + +[case testInspectFunctionArgDef] +# inspect2: --show=definition --include-span tmp/foo.py:4:13 +# TODO: for now all arguments have line/column set to function definition. +import foo +[file foo.py] +def foo(arg: int) -> int: + pass + pass + return arg + +[out] +== +4:12:4:14 -> tmp/foo.py:1:9:arg diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test new file mode 100644 index 000000000000..f28dbaa1113b --- /dev/null +++ b/test-data/unit/fine-grained-modules.test @@ -0,0 +1,2246 @@ +-- Test cases for fine-grained incremental mode related to modules +-- +-- Covers adding and deleting modules, changes to multiple modules, +-- changes to import graph, and changes to module references. +-- +-- The comments in fine-grained.test explain how these tests work. + + +-- Add file +-- -------- + + +[case testAddFile] +import b +[file b.py] +[file a.py.2] +def f() -> None: pass +[file b.py.3] +import a +a.f(1) +[out] +== +== +b.py:2: error: Too many arguments for "f" + +[case testAddFileWithErrors] +import b +[file b.py] +[file a.py.2] +def f() -> str: + return 1 +[file b.py.3] +import a +a.f(1) +[file a.py.4] +def f(x: int) -> None: pass +[out] +== +a.py:2: error: Incompatible return value type (got "int", expected "str") +== +a.py:2: error: Incompatible return value type (got "int", expected "str") +b.py:2: error: Too many arguments for "f" +== + +[case testAddFileFixesError] +import b +[file b.py] +[file b.py.2] +from a import f +f() +[file a.py.3] +def f() -> None: pass +[out] +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== + +[case testAddFileFixesAndGeneratesError1] +import b +[file b.py] +[file b.py.2] +from a import f +[file b.py.3] +from a import f +f(1) +[file a.py.4] +def f() -> None: pass +[out] +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +b.py:2: error: Too many arguments for "f" + +[case testAddFileFixesAndGeneratesError2] +import b +[file b.py] +[file b.py.2] +from a import f +f(1) +[file c.py.3] +x = 'whatever' +[file a.py.4] +def f() -> None: pass +[out] +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +b.py:2: error: Too many arguments for "f" + +[case testAddFileGeneratesError1] +# flags: --ignore-missing-imports +import a +[file a.py] +from b import f +f(1) +[file b.py.2] +def f() -> None: pass +[out] +== +a.py:2: error: Too many arguments for "f" + +[case testAddFilePreservesError1] +import b +[file b.py] +[file b.py.2] +from a import f +f(1) +[file x.py.3] +# unrelated change +[out] +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +b.py:1: error: Cannot find implementation or library stub for module named "a" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testAddFilePreservesError2] +import b +[file b.py] +f() +[file a.py.2] +[out] +b.py:1: error: Name "f" is not defined +== +b.py:1: error: Name "f" is not defined + +[case testRemoveSubmoduleFromBuild1] +# cmd1: mypy a.py b/__init__.py b/c.py +# cmd2: mypy a.py b/__init__.py +# flags: --follow-imports=skip --ignore-missing-imports +[file a.py] +from b import c +x=1 +[file a.py.2] +from b import c +x=2 +[file a.py.3] +from b import c +x=3 +[file b/__init__.py] +[file b/c.py] +[out] +== +== + +[case testImportLineNumber1] +import b +[file b.py] +[file b.py.2] +x = 1 +import a +[out] +== +b.py:2: error: Cannot find implementation or library stub for module named "a" +b.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testImportLineNumber2] +import b +[file b.py] +[file b.py.2] +x = 1 +import a +from c import f +[file x.py.3] +[out] +== +b.py:2: error: Cannot find implementation or library stub for module named "a" +b.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +b.py:3: error: Cannot find implementation or library stub for module named "c" +== +b.py:2: error: Cannot find implementation or library stub for module named "a" +b.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +b.py:3: error: Cannot find implementation or library stub for module named "c" + +[case testAddPackage1] +import p.a +p.a.f(1) +[file p/__init__.py.2] +[file p/a.py.2] +def f(x: str) -> None: pass +[out] +main:1: error: Cannot find implementation or library stub for module named "p.a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "p" +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddPackage2] +import p +p.f(1) +[file p/__init__.py.2] +from p.a import f +[file p/a.py.2] +def f(x: str) -> None: pass +[out] +main:1: error: Cannot find implementation or library stub for module named "p" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddPackage3] +import p.a +p.a.f(1) +[file p/__init__.py.2] +[file p/a.py.3] +def f(x: str) -> None: pass +[out] +main:1: error: Cannot find implementation or library stub for module named "p.a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "p" +== +main:1: error: Cannot find implementation or library stub for module named "p.a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" +[builtins fixtures/module.pyi] + +[case testAddPackage4] +import p.a +p.a.f(1) +[file p/a.py.2] +def f(x: str) -> None: pass +[file p/__init__.py.3] +[out] +main:1: error: Cannot find implementation or library stub for module named "p.a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "p" +== +main:1: error: Cannot find implementation or library stub for module named "p.a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "p" +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddNonPackageSubdir] +# cmd: mypy x.py +# cmd2: mypy x.py foo/a.py foo/b.py +[file x.py] +[file foo/a.py.2] +import b +b.foo(5) +[file foo/b.py.2] +def foo(x: str) -> None: pass +[out] +== +foo/a.py:2: error: Argument 1 to "foo" has incompatible type "int"; expected "str" + +[case testAddPackage5] +# cmd: mypy main p/a.py +# cmd2: mypy main p/a.py +# cmd3: mypy main p/a.py p/__init__.py +import p.a +p.a.f(1) +[file p/a.py] +[file p/a.py.2] +def f(x: str) -> None: pass +[file p/__init__.py.3] +[out] +main:4: error: Cannot find implementation or library stub for module named "p.a" +main:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:4: error: Cannot find implementation or library stub for module named "p" +== +main:4: error: Cannot find implementation or library stub for module named "p.a" +main:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:4: error: Cannot find implementation or library stub for module named "p" +== +main:5: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testAddPackage6] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy x.py +# cmd2: mypy x.py p/a.py +[file x.py] +import p.a +p.a.f(1) +[file p/a.py.2] +def f(x: str) -> None: pass +[file p/__init__.py.2] +[out] +== +-- It is a bug (#4797) that this isn't an error, but not a fine-grained specific one + +[case testAddPackage7] +# flags: --follow-imports=skip +# cmd: mypy x.py +# cmd2: mypy x.py p/a.py +[file x.py] +from p.a import f +f(1) +[file p/a.py.2] +def f(x: str) -> None: pass +[file p/__init__.py.2] +[out] +x.py:1: error: Cannot find implementation or library stub for module named "p.a" +x.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +x.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + + +[case testAddPackage8] +# cmd: mypy x.py p/a.py +# cmd2: mypy x.py p/a.py p/__init__.py +# cmd3: mypy x.py p/a.py p/__init__.py +[file x.py] +[file p/a.py] +1+'hi' +[file p/__init__.py.2] +[file p/a.py.3] +1+3 +[out] +p/a.py:1: error: Unsupported operand types for + ("int" and "str") +== +p/a.py:1: error: Unsupported operand types for + ("int" and "str") +== + + +-- Delete file +-- ----------- + + +[case testDeleteBasic] +import a +[file a.py] +import b +[file b.py] +def f() -> None: pass +[file a.py.2] +[delete b.py.3] +[out] +== +== + +[case testDeleteDepOfDunderInit1] +[file p/__init__.py] +from .foo import Foo +[file p/foo.py] +class Foo: pass +[file p/__init__.py.2] +[delete p/foo.py.2] +[out] +== + +[case testDeleteDepOfDunderInit2] +[file p/__init__.py] +from p.foo import Foo +[file p/foo.py] +class Foo: pass +[file p/__init__.py.2] +[delete p/foo.py.2] +[out] +== + +[case testDeletionTriggersImportFrom] +import a +[file a.py] +from b import f + +def g() -> None: + f() +[file b.py] +def f() -> None: pass +[delete b.py.2] +[file b.py.3] +def f(x: int) -> None: pass +[out] +== +a.py:1: error: Cannot find implementation or library stub for module named "b" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +a.py:4: error: Missing positional argument "x" in call to "f" + +[case testDeletionTriggersImport] +import a +[file a.py] +def f() -> None: pass +[delete a.py.2] +[file a.py.3] +def f() -> None: pass +[out] +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== + +[case testDeletionOfSubmoduleTriggersImportFrom1-only_when_nocache] +-- Different cache/no-cache tests because: +-- missing module error message mismatch +from p import q +[file p/__init__.py] +[file p/q.py] +[delete p/q.py.2] +[file p/q.py.3] +[out] +== +main:1: error: Module "p" has no attribute "q" +-- TODO: The following messages are different compared to non-incremental mode +main:1: error: Cannot find implementation or library stub for module named "p.q" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== + +-- TODO: Fix this bug. It is a real bug that was been papered over +-- by the test harness. +[case testDeletionOfSubmoduleTriggersImportFrom1_2-only_when_cache-skip] +-- Different cache/no-cache tests because: +-- missing module error message mismatch +from p import q +[file p/__init__.py] +[file p/q.py] +[delete p/q.py.2] +[file p/q.py.3] +[out] +== +main:1: error: Cannot find implementation or library stub for module named "p.q" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== + +[case testDeletionOfSubmoduleTriggersImportFrom2] +from p.q import f +f() +[file p/__init__.py] +[file p/q.py] +def f() -> None: pass +[delete p/q.py.2] +[file p/q.py.3] +def f(x: int) -> None: pass +[out] +== +main:1: error: Cannot find implementation or library stub for module named "p.q" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testDeletionOfSubmoduleTriggersImport] +import p.q +[file p/__init__.py] +[file p/q.py] +def f() -> None: pass +[delete p/q.py.2] +[file p/q.py.3] +def f(x: int) -> None: pass +[out] +== +main:1: error: Cannot find implementation or library stub for module named "p.q" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== + +[case testDeleteSubpackageWithNontrivialParent1] +[file p/__init__.py] +def g() -> None: + pass +[file p/b.py.2] +def foo() -> None: pass +foo() +[delete p/b.py.3] +[out] +== +== + +[case testDeleteModuleWithError] +import a +[file a.py] +def f() -> int: + return 1 +[file a.py.2] +def f() -> str: + return 1 +[delete a.py.3] +def f() -> str: + return 1 +[out] +== +a.py:2: error: Incompatible return value type (got "int", expected "str") +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testDeleteModuleWithErrorInsidePackage] +import a.b +[file a/__init__.py] +[file a/b.py] +def f() -> int: + return '' +[delete a/b.py.2] +def f() -> str: + return 1 +[out] +a/b.py:2: error: Incompatible return value type (got "str", expected "int") +== +main:1: error: Cannot find implementation or library stub for module named "a.b" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testModifyTwoFilesNoError1] +import a +[file a.py] +import b +b.f() +[file b.py] +def f() -> None: pass +[file a.py.2] +import b +b.f(1) +[file b.py.2] +def f(x: int) -> None: pass +[out] +== + +[case testDeleteSubpackageInit1] +# cmd: mypy q/r/s.py +# flags: --follow-imports=skip --ignore-missing-imports +[file q/__init__.py] +[file q/r/__init__.py] +[file q/r/s.py] +[delete q/__init__.py.2] +[out] +== + +[case testAddSubpackageInit2] +# cmd: mypy q/r/s.py +# flags: --follow-imports=skip --ignore-missing-imports +[file q/r/__init__.py] +[file q/r/s.py] +1 +[file q/r/s.py.2] +2 +[file q/__init__.py.2] +[out] +== + +[case testModifyTwoFilesNoError2] +import a +[file a.py] +from b import g +def f() -> None: pass +[file b.py] +import a +def g() -> None: pass +a.f() +[file a.py.2] +from b import g +def f(x: int) -> None: pass +[file b.py.2] +import a +def g() -> None: pass +a.f(1) +[out] +== + +[case testModifyTwoFilesErrorsElsewhere] +import a +import b +a.f() +b.g(1) +[file a.py] +def f() -> None: pass +[file b.py] +def g(x: int) -> None: pass +[file a.py.2] +def f(x: int) -> None: pass +[file b.py.2] +def g() -> None: pass +[out] +== +main:3: error: Missing positional argument "x" in call to "f" +main:4: error: Too many arguments for "g" + +[case testModifyTwoFilesErrorsInBoth] +import a +[file a.py] +import b +def f() -> None: pass +b.g(1) +[file b.py] +import a +def g(x: int) -> None: pass +a.f() +[file a.py.2] +import b +def f(x: int) -> None: pass +b.g(1) +[file b.py.2] +import a +def g() -> None: pass +a.f() +[out] +== +b.py:3: error: Missing positional argument "x" in call to "f" +a.py:3: error: Too many arguments for "g" + +[case testModifyTwoFilesFixErrorsInBoth] +import a +[file a.py] +import b +def f(x: int) -> None: pass +b.g(1) +[file b.py] +import a +def g() -> None: pass +a.f() +[file a.py.2] +import b +def f() -> None: pass +b.g(1) +[file b.py.2] +import a +def g(x: int) -> None: pass +a.f() +[out] +b.py:3: error: Missing positional argument "x" in call to "f" +a.py:3: error: Too many arguments for "g" +== + +[case testAddTwoFilesNoError] +import a +[file a.py] +import b +import c +b.f() +c.g() +[file b.py.2] +import c +def f() -> None: pass +c.g() +[file c.py.2] +import b +def g() -> None: pass +b.f() +[out] +a.py:1: error: Cannot find implementation or library stub for module named "b" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +a.py:2: error: Cannot find implementation or library stub for module named "c" +== + +[case testAddTwoFilesErrorsInBoth] +import a +[file a.py] +import b +import c +b.f() +c.g() +[file b.py.2] +import c +def f() -> None: pass +c.g(1) +[file c.py.2] +import b +def g() -> None: pass +b.f(1) +[out] +a.py:1: error: Cannot find implementation or library stub for module named "b" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +a.py:2: error: Cannot find implementation or library stub for module named "c" +== +c.py:3: error: Too many arguments for "f" +b.py:3: error: Too many arguments for "g" + +[case testAddTwoFilesErrorsElsewhere] +import a +import b +a.f(1) +b.g(1) +[file a.py.2] +def f() -> None: pass +[file b.py.2] +def g() -> None: pass +[out] +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "b" +== +main:3: error: Too many arguments for "f" +main:4: error: Too many arguments for "g" + +[case testDeleteTwoFilesErrorsElsewhere] +import a +import b +a.f() +b.g() +[file a.py] +def f() -> None: pass +[file b.py] +def g() -> None: pass +[delete a.py.2] +[delete b.py.2] +[out] +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "b" + +[case testDeleteTwoFilesNoErrors] +import a +[file a.py] +import b +import c +b.f() +c.g() +[file b.py] +def f() -> None: pass +[file c.py] +def g() -> None: pass +[file a.py.2] +[delete b.py.3] +[delete c.py.3] +[out] +== +== + +[case testDeleteTwoFilesFixErrors] +import a +import b +a.f() +b.g() +[file a.py] +import b +def f() -> None: pass +b.g(1) +[file b.py] +import a +def g() -> None: pass +a.f(1) +[delete a.py.2] +[delete b.py.2] +[out] +b.py:3: error: Too many arguments for "f" +a.py:3: error: Too many arguments for "g" +== +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "b" + +[case testAddFileWhichImportsLibModule] +import a +a.x = 0 +[file a.py.2] +import sys +x = sys.platform +[builtins fixtures/tuple.pyi] +[out] +main:1: error: Cannot find implementation or library stub for module named "a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main:2: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAddFileWhichImportsLibModuleWithErrors] +# flags: --no-silence-site-packages +import a +a.x = 0 +[file a.py.2] +import broken +x = broken.x +z +[out] +main:2: error: Cannot find implementation or library stub for module named "a" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +a.py:3: error: Name "z" is not defined +/test-data/unit/lib-stub/broken.pyi:2: error: Name "y" is not defined + +[case testRenameModule] +import a +[file a.py] +import b +b.f() +[file b.py] +def f() -> None: pass +[file a.py.2] +import c +c.f() +[file c.py.2] +def f() -> None: pass +[file a.py.3] +import c +c.f(1) +[out] +== +== +a.py:2: error: Too many arguments for "f" + +[case testDeleteFileWSuperClass] +# flags: --ignore-missing-imports +[file a.py] +from c import Bar +from b import Foo +z = (1, Foo()) +[file b.py] +from e import Quux +from d import Baz +class Foo(Baz, Quux): + pass + +[file e.py] +from c import Bar +class Quux(Bar): + pass +[file c.py] +class Bar: + pass +[file d.py] +class Baz: + pass +[delete c.py.2] +[builtins fixtures/tuple.pyi] +[out] +== + +[case testDeleteFileWithinPackage] + +import a +[file a.py] +import m.x +m.x.g(1) +[file m/__init__.py] +[file m/x.py] +def g() -> None: pass +[delete m/x.py.2] +[builtins fixtures/module.pyi] +[out] +a.py:2: error: Too many arguments for "g" +== +a.py:1: error: Cannot find implementation or library stub for module named "m.x" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +a.py:2: error: Module has no attribute "x" + +[case testDeletePackage1] +import p.a +p.a.f(1) +[file p/__init__.py] +[file p/a.py] +def f(x: str) -> None: pass +[delete p.2] +[out] +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" +== +main:1: error: Cannot find implementation or library stub for module named "p.a" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:1: error: Cannot find implementation or library stub for module named "p" + +[case testDeletePackage2] +import p +p.f(1) +[file p/__init__.py] +from p.a import f +[file p/a.py] +def f(x: str) -> None: pass +[delete p.2] +[out] +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" +== +main:1: error: Cannot find implementation or library stub for module named "p" +main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testDeletePackage3] + +import p.a +p.a.f(1) +[file p/__init__.py] +[file p/a.py] +def f(x: str) -> None: pass +[delete p/a.py.2] +[delete p.3] +[builtins fixtures/module.pyi] +[out] +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" +== +main:2: error: Cannot find implementation or library stub for module named "p.a" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:3: error: Module has no attribute "a" +== +main:2: error: Cannot find implementation or library stub for module named "p.a" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "p" + +[case testDeletePackage4] +# flags: --no-namespace-packages +import p.a +p.a.f(1) +[file p/a.py] +def f(x: str) -> None: pass +[file p/__init__.py] +[delete p/__init__.py.2] +[delete p.3] +[out] +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" +== +main:2: error: Cannot find implementation or library stub for module named "p.a" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "p" +== +main:2: error: Cannot find implementation or library stub for module named "p.a" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "p" + +[case testDeletePackage5] +# flags: --no-namespace-packages +# cmd1: mypy -m main -m p.a -m p.__init__ +# cmd2: mypy -m main -m p.a +# cmd3: mypy -m main + +import p.a +p.a.f(1) +[file p/a.py] +def f(x: str) -> None: pass +[file p/__init__.py] +[delete p/__init__.py.2] +[delete p.3] +[out] +main:7: error: Argument 1 to "f" has incompatible type "int"; expected "str" +== +main:6: error: Cannot find implementation or library stub for module named "p.a" +main:6: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:6: error: Cannot find implementation or library stub for module named "p" +== +main:6: error: Cannot find implementation or library stub for module named "p.a" +main:6: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:6: error: Cannot find implementation or library stub for module named "p" + + +[case testDeletePackage6] +# flags: --no-namespace-packages +# cmd1: mypy -m p.a -m p.b -m p.__init__ +# cmd2: mypy -m p.a -m p.b +# cmd3: mypy -m p.a -m p.b +[file p/a.py] +def f(x: str) -> None: pass +[file p/b.py] +from p.a import f +f(12) +[file p/__init__.py] +[delete p/__init__.py.2] +[file p/b.py.3] +from a import f +f(12) +[out] +p/b.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" +== +p/b.py:1: error: Cannot find implementation or library stub for module named "p.a" +p/b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +p/b.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + + +-- TODO: +-- - add one file which imports another new file, blocking error in new file +-- - arbitrary blocking errors +-- - order of processing makes a difference +-- - mix of modify, add and delete in one iteration + + +-- Controlling imports using command line options +-- ---------------------------------------------- + + +[case testIgnoreMissingImports] +# flags: --ignore-missing-imports +import a +[file a.py] +import b +import c +[file c.py] +[delete c.py.2] +[file b.py.3] +import d +1 + '' +[out] +== +== +b.py:2: error: Unsupported operand types for + ("int" and "str") + +[case testSkipImports] +# cmd: mypy main a.py +# flags: --follow-imports=skip --ignore-missing-imports +import a +[file a.py] +import b +[file b.py] +1 + '' +class A: pass +[file a.py.2] +import b +reveal_type(b) +reveal_type(b.A) +[file a.py.3] +import b +reveal_type(b) +reveal_type(b.A) +[file b.py.3] +1 + '' +class A: pass +[out] +== +a.py:2: note: Revealed type is "Any" +a.py:3: note: Revealed type is "Any" +== +a.py:2: note: Revealed type is "Any" +a.py:3: note: Revealed type is "Any" + +[case testSkipImportsWithinPackage] +# cmd: mypy a/b.py +# flags: --follow-imports=skip --ignore-missing-imports +[file a/__init__.py] +1 + '' +[file a/b.py] +import a.c +[file a/b.py.2] +import a.c +import x +reveal_type(a.c) +[file a/b.py.3] +import a.c +import x +1 + '' +[out] +== +a/b.py:3: note: Revealed type is "Any" +== +a/b.py:3: error: Unsupported operand types for + ("int" and "str") + +[case testDeleteModuleWithinPackageInitIgnored-only_when_nocache] +-- Disabled in cache mode because incorrect behavior: +-- Having deleted files specified on command line seems dodgy, though. +# cmd: mypy x.py a/b.py +# flags: --follow-imports=skip --ignore-missing-imports +[file x.py] +import a.b +[file a/__init__.py] +[file a/b.py] +x = 1 +[delete a/b.py.2] +[out] +== + +[case testAddImport] +import what.b +[file aaa/__init__.py] +[file aaa/z.py] +def foo(x: int) -> None: + pass +[file aaa/z.py.2] +import config +def foo() -> None: + pass +[file what/__init__.py] +[file what/b.py] +import config +import aaa.z +def main() -> None: + aaa.z.foo(5) +[file what/b.py.2] +import aaa.z +def main() -> None: + aaa.z.foo() +[file config.py] +[out] +== + +[case testAddImport2] +import what.b +[file aaa/__init__.py] +[file aaa/z.py] +def foo(x: int) -> None: + pass +[file aaa/z.py.2] +def foo() -> None: + pass +[file what/__init__.py] +[file what/b.py] +import aaa.z +def main() -> None: + aaa.z.foo(5) +[file what/b.py.2] +import config +import aaa.z +def main() -> None: + aaa.z.foo() +[file config.py] +[out] +== + + +-- Misc +-- ---- + + +[case testChangeModuleToVariable] +from a import m +m.x +[file a.py] +from b import m +[file b.py] +import m +[file b.py.2] +m = '' +[file m.py] +x = 1 +[file m2.py] +[out] +== +main:2: error: "str" has no attribute "x" + +[case testChangeVariableToModule] +from a import m +y: str = m +[file a.py] +from b import m +[file b.py] +m = '' +[file b.py.2] +import m +[file m.py] +x = 1 +[file m2.py] +[builtins fixtures/module.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type Module, variable has type "str") + +[case testRefreshImportOfIgnoredModule1] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy c.py a/__init__.py b.py +[file c.py] +from a import a2 +import b +b.x +[file a/__init__.py] +[file b.py] +x = 0 +[file b.py.2] +x = '' +[file b.py.3] +x = 0 +[file a/a2.py] +[out] +== +== + +[case testRefreshImportOfIgnoredModule2] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy c.py a/__init__.py b.py +[file c.py] +from a import a2 +import b +b.x +[file a/__init__.py] +[file b.py] +x = 0 +[file b.py.2] +x = '' +[file b.py.3] +x = 0 +[file a/a2/__init__.py] +[out] +== +== + +[case testIncrementalWithIgnoresTwice] +import a +[file a.py] +import b +import foo # type: ignore +[file b.py] +x = 1 +[file b.py.2] +x = 'hi' +[file b.py.3] +x = 1 +[out] +== +== + +[case testIgnoredImport2] +import x +[file y.py] +import xyz # type: ignore +B = 0 +from x import A +[file x.py] +A = 0 +from y import B +[file x.py.2] +A = 1 +from y import B +[file x.py.3] +A = 2 +from y import B +[out] +== +== + +[case testDeleteIndirectDependency] +import b +b.x.foo() +[file b.py] +import c +x = c.Foo() +[file c.py] +class Foo: + def foo(self) -> None: pass +[delete c.py.2] +[file b.py.2] +class Foo: + def foo(self) -> None: pass +x = Foo() +[file b.py.3] +class Foo: + def foo(self, x: int) -> None: pass +x = Foo() +[out] +== +== +main:2: error: Missing positional argument "x" in call to "foo" of "Foo" + +-- This series of tests is designed to test adding a new module that +-- does not appear in the cache, for cache mode. They are run in +-- cache mode only because stale and rechecked differ heavily between +-- the modes. +[case testAddModuleAfterCache1-only_when_cache] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +import a +[file a.py] +pass +[file a.py.2] +import b +b.foo(0) +[file b.py.2] +def foo() -> None: pass + +[stale a, b] +[rechecked a, b] + +[file b.py.3] +def foo(x: int) -> None: pass + +[stale2 b] +[rechecked2 b] + +[out] +== +a.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache2-only_when_cache] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +[file a.py] +import b +b.foo(0) +[file b.py.2] +def foo() -> None: pass + +[stale b] +[rechecked a, b] + +[file b.py.3] +def foo(x: int) -> None: pass + +[stale2 b] + +[out] +== +a.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache3-only_when_cache] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py c.py d.py e.py f.py g.py h.py i.py j.py +# cmd3: mypy main a.py b.py c.py d.py e.py f.py g.py h.py i.py j.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +[file a.py] +import b, c, d, e, f, g, h, i, j +b.foo(10) +[file b.py.2] +def foo() -> None: pass +[file c.py.2] +[file d.py.2] +[file e.py.2] +[file f.py.2] +[file g.py.2] +[file h.py.2] +[file i.py.2] +[file j.py.2] + +-- No files should be stale or reprocessed in the first step since the large number +-- of missing files will force build to give up on cache loading. +[stale] + +[file b.py.3] +def foo(x: int) -> None: pass +[stale2 b] + +[out] +== +a.py:2: error: Too many arguments for "foo" +== + + +[case testAddModuleAfterCache4-only_when_cache] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +import b +[file a.py] +def foo() -> None: pass +[file b.py.2] +import a +a.foo(10) +[file a.py.3] +def foo(x: int) -> None: pass +[out] +== +b.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache5-only_when_cache] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +import b +[file a.py] +def foo(x: int) -> None: pass +[file a.py.2] +def foo() -> None: pass +[file b.py.2] +import a +a.foo(10) + +[stale a, b] + +[file a.py.3] +def foo(x: int) -> None: pass + +[stale2 a] + +[out] +== +b.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache6-only_when_cache] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +[file a.py] +import b +b.foo() + +[file a.py.2] +import b +b.foo(0) +[file b.py.2] +def foo() -> None: pass + +[stale a, b] + +[file b.py.3] +def foo(x: int) -> None: pass + +[stale2 b] + +[out] +== +a.py:2: error: Too many arguments for "foo" +== + +[case testRenameAndDeleteModuleAfterCache-only_when_cache] +import a +[file a.py] +from b1 import f +f() +[file b1.py] +def f() -> None: pass +[file b2.py.2] +def f() -> None: pass +[delete b1.py.2] +[file a.py.2] +from b2 import f +f() + +-- in cache mode, there is no way to know about b1 yet +[stale a, b2] + +[out] +== + +[case testDeleteModuleAfterCache-only_when_cache] +import a +[file a.py] +from b import f +f() +[file b.py] +def f() -> None: pass +[delete b.py.2] + +-- in cache mode, there is no way to know about b yet, +-- but a should get flagged as changed by the initial cache +-- check, since one of its dependencies is missing. +[stale a] + +[out] +== +a.py:1: error: Cannot find implementation or library stub for module named "b" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + + +[case testRefreshImportIfMypyElse1] +import a +[file a.py] +from b import foo + +MYPY = False +if MYPY: + x = 0 +else: + from fictional import x + +x = 1 +[file b/__init__.py] +[file b/foo.py] +[file b/__init__.py.2] +# Dummy change +[builtins fixtures/primitives.pyi] +[out] +== + +[case testImportStarPropagateChange1] +from b import f +f() +[file b.py] +from c import * +[file c.py] +def f() -> None: pass +[file c.py.2] +def f(x: int) -> None: pass +[out] +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testImportStarPropagateChange2] +from b import * +f() +[file b.py] +def f() -> None: pass +[file b.py.2] +def f(x: int) -> None: pass +[out] +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testImportStarAddMissingDependency1] +from b import f +f() +[file b.py] +from c import * +[file c.py] +[file c.py.2] +def f(x: int) -> None: pass +[out] +main:1: error: Module "b" has no attribute "f" +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testImportStarAddMissingDependency2] +from b import * +f() +[file b.py] +[file b.py.2] +def f(x: int) -> None: pass +[out] +main:2: error: Name "f" is not defined +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testImportStarAddMissingDependencyWithinClass] +class A: + from b import * + f() + x: C +[file b.py] +[file b.py.2] +def f(x: int) -> None: pass +[file b.py.3] +def f(x: int) -> None: pass +class C: pass +[file b.py.4] +def f() -> None: pass +class C: pass +[out] +main:3: error: Name "f" is not defined +main:4: error: Name "C" is not defined +== +main:2: error: Unsupported class scoped import +main:4: error: Name "C" is not defined +== +main:2: error: Unsupported class scoped import +== +main:2: error: Unsupported class scoped import + +[case testImportStarAddMissingDependencyInsidePackage1] +from p.b import f +f() +[file p/__init__.py] +[file p/b.py] +from p.c import * +[file p/c.py] +[file p/c.py.2] +def f(x: int) -> None: pass +[out] +main:1: error: Module "p.b" has no attribute "f" +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testImportStarAddMissingDependencyInsidePackage2] +import p.a +[file p/__init__.py] +[file p/a.py] +from p.b import * +f() +[file p/b.py] +[file p/b.py.2] +def f(x: int) -> None: pass +[out] +p/a.py:2: error: Name "f" is not defined +== +p/a.py:2: error: Missing positional argument "x" in call to "f" + +[case testImportStarRemoveDependency1] +from b import f +f() +[file b.py] +from c import * +[file c.py] +def f() -> None: pass +[file c.py.2] +[out] +== +main:1: error: Module "b" has no attribute "f" + +[case testImportStarRemoveDependency2] +from b import * +f() +[file b.py] +def f() -> None: pass +[file b.py.2] +[out] +== +main:2: error: Name "f" is not defined + +[case testImportStarWithinFunction] +def f() -> None: + from m import * + f() +[file m.py] +[file m.py.2] +def f(x: int) -> None: pass +[file m.py.3] +def f() -> None: pass +[out] +== +main:3: error: Missing positional argument "x" in call to "f" +== + +[case testImportStarMutuallyRecursive-skip] +# FIXME: busted with new analyzer? +import a +[file a.py] +from b import * +[file b.py] +from a import * +[file b.py.2] +from a import * +x = 0 +[file b.py.3] +from a import * +x = '' +[out] +== +== + +[case testImportStarSomethingMoved] +import p +[file p.py] +from r2 import * +[file r2.py] +class A: pass + +[file p.py.2] +from r1 import * +from r2 import * +[file r2.py.2] + +[file r1.py.2] +class A: pass + +[out] +== + +[case testImportStarOverlap] +from b import * +from c import * # type: ignore +[file b.py] +from d import T +[file c.py] +from d import T +[file c.py.2] +from d import T +z = 10 +[file d.py] +from typing import TypeVar +T = TypeVar('T') +[out] +== + +[case testImportStarOverlap2] +from b import * +import typing +def foo(x: typing.List[int]) -> int: + return x[0] +[file b.py] +import typing +z = 10 +[file b.py.2] +import typing +z = '10' +[builtins fixtures/list.pyi] +[out] +== + + +[case testImportStarOverlap3] +from b import * +from c import typing +def foo(x: typing.List[int]) -> int: + return x[0] +[file b.py] +import typing +z = 10 +[file b.py.2] +import typing +z = '10' +[file c.py] +import typing +z = 10 +[builtins fixtures/list.pyi] +[out] +== + + +[case testImportPartialAssign] +import a +[file a.py] +from c import * +from b import A, x +[file b.py] +A = 10 +x = 1 +[file b.py.2] +class A: pass +x = 1 +[file c.py] +x = 10 +[out] +== + +[case testDeleteFileWithErrors] +# cmd: mypy main a.py +# cmd2: mypy main +# flags: --follow-imports=skip --ignore-missing-imports +import a +[file a.py] +def f() -> None: + 1() +''() +[file b.py.2] +# unrelated change +[out] +a.py:2: error: "int" not callable +a.py:3: error: "str" not callable +== + +[case testAddAndUseClass1] +[file a.py] +[file a.py.2] +from b import Foo +def bar(f: Foo) -> None: + f.foo(12) +[file b.py.2] +class Foo: + def foo(self, s: str) -> None: pass +[out] +== +a.py:3: error: Argument 1 to "foo" of "Foo" has incompatible type "int"; expected "str" + +[case testAddAndUseClass2] +[file a.py] +[file a.py.3] +from b import Foo +def bar(f: Foo) -> None: + f.foo(12) +[file b.py.2] +class Foo: + def foo(self, s: str) -> None: pass +[out] +== +== +a.py:3: error: Argument 1 to "foo" of "Foo" has incompatible type "int"; expected "str" + +[case testAddAndUseClass3] +# flags: --ignore-missing-imports +[file a.py] +[file a.py.2] +from b import Foo +def bar(f: Foo) -> None: + f.foo(12) +[file b.py.3] +class Foo: + def foo(self, s: str) -> None: pass +[out] +== +== +a.py:3: error: Argument 1 to "foo" of "Foo" has incompatible type "int"; expected "str" + +[case testAddAndUseClass4] +[file a.py] +[file a.py.2] +from b import * +def bar(f: Foo) -> None: + f.foo(12) +[file b.py.2] +class Foo: + def foo(self, s: str) -> None: pass +[out] +== +a.py:3: error: Argument 1 to "foo" of "Foo" has incompatible type "int"; expected "str" + +[case testAddAndUseClass4_2] +[file a.py] +[file a.py.2] +from p.b import * +def bar(f: Foo) -> None: + f.foo(12) +[file p/__init__.py] +[file p/b.py.2] +class Foo: + def foo(self, s: str) -> None: pass +[out] +== +a.py:3: error: Argument 1 to "foo" of "Foo" has incompatible type "int"; expected "str" + +[case testAddAndUseClass5] +[file a.py] +[file a.py.2] +from b import * +def bar(f: Foo) -> None: + f.foo(12) +[file b.py.2] +class Foo: + def foo(self, s: str) -> None: pass +[out] +== +a.py:3: error: Argument 1 to "foo" of "Foo" has incompatible type "int"; expected "str" + +[case testSkipButDontIgnore1] +# cmd: mypy a.py c.py +# flags: --follow-imports=skip +[file a.py] +import b +from c import x +[file b.py] +1+'lol' +[file c.py] +x = 1 +[file c.py.2] +x = '2' +[file b.py.3] +[out] +== +== + +[case testSkipButDontIgnore2] +# cmd: mypy a.py c.py +# flags: --follow-imports=skip +[file a.py] +from c import x +import b +[file b.py] +[file c.py] +x = 1 +[file b.py.2] +1+'x' +[file c.py.2] +x = '2' +[file c.py.3] +x = 2 +[delete b.py.3] +[out] +== +== +a.py:2: error: Cannot find implementation or library stub for module named "b" +a.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testErrorButDontIgnore1] +# cmd: mypy a.py c.py +# flags: --follow-imports=error +[file a.py] +from c import x +import b +[file b.py] +[file c.py] +x = 1 +[file c.py.2] +x = '2' +[out] +a.py:2: error: Import of "b" ignored +a.py:2: note: (Using --follow-imports=error, module not passed on command line) +== +a.py:2: error: Import of "b" ignored +a.py:2: note: (Using --follow-imports=error, module not passed on command line) + +[case testErrorButDontIgnore2] +# cmd1: mypy a.py c.py b.py +# cmd2: mypy a.py c.py +# flags: --follow-imports=error +[file a.py] +from c import x +import b +[file b.py] +[file c.py] +x = 1 +[file c.py.2] +x = '2' +[out] +== +a.py:2: error: Import of "b" ignored +a.py:2: note: (Using --follow-imports=error, module not passed on command line) + +-- TODO: This test fails because p.b does not depend on p (#4847) +[case testErrorButDontIgnore3-skip] +# cmd1: mypy a.py c.py p/b.py p/__init__.py +# cmd2: mypy a.py c.py p/b.py +# flags: --follow-imports=error +[file a.py] +from c import x +from p.b import y +[file p/b.py] +y = 12 +[file p/__init__.py] +[file c.py] +x = 1 +[file c.py.2] +x = '2' +[out] +== +p/b.py: error: Ancestor package "p" ignored +p/b.py: note: (Using --follow-imports=error, submodule passed on command line) + +[case testErrorButDontIgnore4] +# cmd: mypy a.py z.py p/b.py p/__init__.py +# cmd2: mypy a.py p/b.py +# flags: --follow-imports=error +[file a.py] +from p.b import y +[file p/b.py] +from z import x +y = 12 +[file p/__init__.py] +[file z.py] +x = 1 +[delete z.py.2] +[out] +== +p/b.py: error: Ancestor package "p" ignored +p/b.py: note: (Using --follow-imports=error, submodule passed on command line) +p/b.py:1: error: Cannot find implementation or library stub for module named "z" +p/b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testTurnPackageToModule] +[file a.py] +[file b.py] +import p +x = p.f() +[file p/__init__.py] +def f() -> int: pass +[delete p/__init__.py.2] +[file p.py.2] +def f() -> str: pass +[file a.py.3] +import b +reveal_type(b.x) +[out] +== +== +a.py:2: note: Revealed type is "builtins.str" + +[case testModuleToPackage] +[file a.py] +[file b.py] +import p +x = p.f() +[file p.py] +def f() -> str: pass +[delete p.py.2] +[file p/__init__.py.2] +def f() -> int: pass +[file a.py.3] +import b +reveal_type(b.x) +[out] +== +== +a.py:2: note: Revealed type is "builtins.int" + +[case testQualifiedSubpackage1] +[file c/__init__.py] +[file c/a.py] +from lurr import x +from c.d import f + +[file c/d.py] +def f() -> None: pass +def g(x: int) -> None: pass +[file lurr.py] +x = 10 +[file lurr.py.2] +x = '10' +[out] +== + +[case testImportedMissingSubpackage] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy a.py b/__init__.py +[file a.py] +from b.foo import bar +x = 10 +[file b/__init__.py] +[file a.py.2] +from b.foo import bar +x = '10' +[out] +== + +[case testFineAddedMissingStubs] +# flags: --ignore-missing-imports +from missing import f +f(int()) +[file missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsPackage] +# flags: --ignore-missing-imports +import package.missing +package.missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsPackageFrom] +# flags: --ignore-missing-imports +from package import missing +missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsPackagePartial] +# flags: --ignore-missing-imports +import package.missing +package.missing.f(int()) +[file package/__init__.pyi] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsPackagePartialGetAttr] +import package.missing +package.missing.f(int()) +[file package/__init__.pyi] +from typing import Any +def __getattr__(attr: str) -> Any: ... +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsIgnore] +from missing import f # type: ignore +f(int()) +[file missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsIgnorePackage] +import package.missing # type: ignore +package.missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsIgnorePackageFrom] +from package import missing # type: ignore +missing.f(int()) +[file package/__init__.pyi.2] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineAddedMissingStubsIgnorePackagePartial] +import package.missing # type: ignore +package.missing.f(int()) +[file package/__init__.pyi] +[file package/missing.pyi.2] +def f(x: str) -> None: pass +[out] +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testFineFollowImportSkipNotInvalidatedOnPresent] +# flags: --follow-imports=skip +# cmd: mypy main.py +[file main.py] +import other +[file other.py] +x = 1 +[file other.py.2] +x = 'hi' +[stale] +[rechecked] +[out] +== + +[case testFineFollowImportSkipNotInvalidatedOnPresentPackage] +# flags: --follow-imports=skip +# cmd: mypy main.py +[file main.py] +import other +[file other/__init__.py] +x = 1 +[file other/__init__.py.2] +x = 'hi' +[stale] +[rechecked] +[out] +== + +[case testFineFollowImportSkipNotInvalidatedOnAdded] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy main.py +[file main.py] +import other +[file other.py.2] +x = 1 +[stale] +[rechecked] +[out] +== + +-- TODO: Fix this: stubs should be followed normally even with follow-imports=skip +[case testFineFollowImportSkipInvalidatedOnAddedStub-skip] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy main.py +[file main.py] +import other +x: str = other.x +[file other.pyi.2] +x = 1 +[stale main, other] +[rechecked main, other] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testFineAddedSkippedStubsPackageFrom] +# flags: --follow-imports=skip --ignore-missing-imports +# cmd: mypy main.py +# cmd2: mypy main.py package/__init__.py package/missing.py +[file main.py] +from package import missing +missing.f(int()) +[file package/__init__.py] +[file package/missing.py] +def f(x: str) -> None: pass +[out] +== +main.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testIgnoredAttrReprocessedModule] +import a +[file a.py] +import b +x = b.x # type: ignore +y: int = x +[file b.py] +import c +[file b.py.2] +import c +x = c.x +[file c.py] +x: str +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testMissingStubAdded1] +# flags: --follow-imports=skip +# cmd: mypy main.py + +[file main.py] +import foo +foo.x = 1 +[file foo.pyi.2] +x = 'x' +[file main.py.3] +import foo +foo.x = 'y' +[out] +main.py:1: error: Cannot find implementation or library stub for module named "foo" +main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main.py:2: error: Incompatible types in assignment (expression has type "int", variable has type "str") +== + +[case testMissingStubAdded2] +# flags: --follow-imports=skip +# cmd: mypy main.py + +[file main.py] +import foo # type: ignore +foo.x = 1 +[file foo.pyi.2] +x = 'x' +[file main.py.3] +import foo +foo.x = 'y' +[out] +== +main.py:2: error: Incompatible types in assignment (expression has type "int", variable has type "str") +== + +[case testDoNotFollowImportToNonStubFile] +# flags: --follow-imports=skip +# cmd: mypy main.py + +[file main.py] +import foo # type: ignore +foo.x = 1 +[file foo.py.2] +x = 'x' +1 + 'x' + +[out] +== + +[case testLibraryStubsNotInstalled] +import a +[file a.py] +import requests +[file a.py.2] +# nothing +[file a.py.3] +import jack +[out] +a.py:1: error: Library stubs not installed for "requests" +a.py:1: note: Hint: "python3 -m pip install types-requests" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +== +a.py:1: error: Library stubs not installed for "jack" +a.py:1: note: Hint: "python3 -m pip install types-JACK-Client" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testIgnoreErrorsFromTypeshed] +# flags: --custom-typeshed-dir tmp/ts --follow-imports=normal +# cmd1: mypy a.py +# cmd2: mypy a.py + +[file a.py] +import foobar + +[file ts/stdlib/abc.pyi] +[file ts/stdlib/builtins.pyi] +class object: pass +class str: pass +class ellipsis: pass +[file ts/stdlib/sys.pyi] +[file ts/stdlib/types.pyi] +[file ts/stdlib/typing.pyi] +def cast(x): ... +[file ts/stdlib/typing_extensions.pyi] +[file ts/stdlib/VERSIONS] +[file ts/stubs/mypy_extensions/mypy_extensions.pyi] + +[file ts/stdlib/foobar.pyi.2] +# We report no errors from typeshed. It would be better to test ignoring +# errors from PEP 561 packages, but it's harder to test and uses the +# same code paths, so we are using typeshed instead. +import baz +import zar +undefined + +[file ts/stdlib/baz.pyi.2] +import whatever +undefined + +[out] +a.py:1: error: Cannot find implementation or library stub for module named "foobar" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== diff --git a/test-data/unit/fine-grained-python312.test b/test-data/unit/fine-grained-python312.test new file mode 100644 index 000000000000..2cb2148a66fe --- /dev/null +++ b/test-data/unit/fine-grained-python312.test @@ -0,0 +1,117 @@ +[case testPEP695TypeAliasDep] +import m +def g() -> m.C: + return m.f() +[file m.py] +type C = int + +def f() -> int: + pass +[file m.py.2] +type C = str + +def f() -> int: + pass +[out] +== +main:3: error: Incompatible return value type (got "int", expected "str") + +[case testPEP695ChangeOldStyleToNewStyleTypeAlias] +from m import A +A() + +[file m.py] +A = int + +[file m.py.2] +type A = int +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: "TypeAliasType" not callable + +[case testPEP695VarianceChangesDueToDependency] +from a import C + +x: C[object] = C[int]() + +[file a.py] +from b import A + +class C[T]: + def f(self) -> A[T]: ... + +[file b.py] +class A[T]: + def f(self) -> T: ... + +[file b.py.2] +class A[T]: + def f(self) -> list[T]: ... + +[out] +== +main:3: error: Incompatible types in assignment (expression has type "C[int]", variable has type "C[object]") + +[case testPEP695TypeAliasChangesDueToDependency] +from a import A +x: A +x = 0 +x = '' + +[file a.py] +from b import B +type A = B[int, str] + +[file b.py] +from typing import Union as B + +[file b.py.2] +from builtins import tuple as B + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] +[out] +== +main:3: error: Incompatible types in assignment (expression has type "int", variable has type "Tuple[int, str]") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "Tuple[int, str]") + +[case testPEP695NestedGenericClassMethodUpdated] +from a import f + +class C: + class D[T]: + x: T + def m(self) -> T: + f() + return self.x + +[file a.py] +def f() -> None: pass + +[file a.py.2] +def f(x: int) -> None: pass +[out] +== +main:7: error: Missing positional argument "x" in call to "f" + +[case testPEP695MultipleNestedGenericClassMethodUpdated] +from a import f + +class A: + class C: + class D[T]: + x: T + def m(self) -> T: + f() + return self.x + +[file a.py] +def f() -> None: pass + +[file a.py.2] +def f(x: int) -> None: pass +[out] +== +main:8: error: Missing positional argument "x" in call to "f" diff --git a/test-data/unit/fine-grained-suggest.test b/test-data/unit/fine-grained-suggest.test new file mode 100644 index 000000000000..02373091ad54 --- /dev/null +++ b/test-data/unit/fine-grained-suggest.test @@ -0,0 +1,1143 @@ +[case testSuggestCallsites1] +# suggest: --callsites foo.foo +[file foo.py] +def foo(arg): + return 12 +var = 0 +[file bar.py] +from foo import foo +def bar() -> None: + foo('abc') + foo(arg='xyz') + args = [''] + foo(*args) + kwds = {'arg': ''} + foo(**kwds) +[builtins fixtures/dict.pyi] +[out] +bar.py:3: (str) +bar.py:4: (arg=str) +bar.py:6: (*typing.List[str]) +bar.py:8: (**typing.Dict[str, str]) +== + +[case testSuggestCallsitesStep2] +# suggest2: --callsites foo.foo +[file foo.py] +def foo(arg): + return 12 +var = 0 +[file bar.py] +from foo import foo +def bar() -> None: + foo('abc') + foo(arg='xyz') + args = [''] + foo(*args) + kwds = {'arg': ''} + foo(**kwds) +[builtins fixtures/dict.pyi] +[out] +== +bar.py:3: (str) +bar.py:4: (arg=str) +bar.py:6: (*typing.List[str]) +bar.py:8: (**typing.Dict[str, str]) + +[case testMaxGuesses] +# suggest: foo.foo +# suggest: --max-guesses=2 foo.foo +[file foo.py] +# The idea here is that we can only find the union type with more guesses. +def foo(x, y): + if not isinstance(x, int): + x+'1' + +foo(1, 2) +foo('3', '4') +[builtins fixtures/isinstancelist.pyi] +[out] +(Union[int, str], object) -> None +(object, object) -> None +== + +[case testSuggestInferFunc1] +# suggest: foo.foo +[file foo.py] +def foo(arg, lol=None): + if isinstance(arg, int): + arg+1 + else: + assert arg + arg+'1' +[file bar.py] +from foo import foo +def bar() -> None: + foo('abc') + foo(lol=10, arg=10) + foo(None) +def untyped(x) -> None: + foo(x) +[builtins fixtures/isinstancelist.pyi] +[out] +(Union[str, int, None], Optional[int]) -> None +== + +[case testSuggestInferFunc2] +# suggest: foo.foo +[file foo.py] +def foo(arg): + return arg +[file bar.py] +from foo import foo +def thing() -> str: + return '' +def bar() -> None: + # We stick it in a list so that the argument type is marked as "inferred", + # which we want to make sure doesn't show up. + x = ['hello'] + foo(x[0]) +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + +[case testSuggestInferFuncAny1] +# suggest: foo.foo +# suggest: foo.bar +[file foo.py] +def foo(arg): + return arg.x +def bar(arg): + pass +[file bar.py] +from foo import bar +bar(None) +[out] +(Any) -> Any +(Optional[Any]) -> None +== + +[case testSuggestInferFuncAny2] +# suggest: --no-any foo.foo +# suggest: --no-any foo.bar +[file foo.py] +def foo(arg): + return arg.x +def bar(arg): + pass +[file bar.py] +from foo import bar +bar(None) +[out] +No guesses that match criteria! +No guesses that match criteria! +== + +[case testSuggestInferTuple] +# suggest: --no-any foo.foo +[file foo.py] +def foo(): + return 1, "1" +[builtins fixtures/tuple.pyi] +[out] +() -> Tuple[int, str] +== + +[case testSuggestInferNamedTuple] +# suggest: foo.foo +[file foo.py] +from typing import NamedTuple +N = NamedTuple('N', [('x', int)]) +def foo(): + return N(1) +[builtins fixtures/tuple.pyi] +[out] +() -> foo.N +== + +[case testSuggestInferTypedDict] +# suggest: foo.foo +[file foo.py] +from typing_extensions import TypedDict +TD = TypedDict('TD', {'x': int}) +def foo(): + return bar() + +def bar() -> TD: ... +[builtins fixtures/dict.pyi] +[out] +() -> foo.TD +== + +[case testSuggestWithNested] +# suggest: foo.foo +[file foo.py] +def foo(): + def bar(): + return 1 + return 'lol' + +[out] +() -> str +== + +[case testSuggestReexportNaming] +# suggest: foo.foo +[file foo.py] +from bar import A +def foo(x): + return A(), A.C() +[file bar.py] +from baz import A +[file baz.py] +class A: + class C: + ... +class B: ... + +[file caller.py] +from foo import foo +from baz import B +foo(B()) +[builtins fixtures/tuple.pyi] + +[out] +(baz.B) -> Tuple[foo.A, foo:A.C] +== + +[case testSuggestInferInit] +# suggest: foo.Foo.__init__ +[file foo.py] +class Foo: + def __init__(self, arg): + self.arg = arg +[file bar.py] +from foo import Foo +Foo('lol') +[out] +(str) -> None +== + +[case testSuggestInferMethod1] +# suggest: --no-any foo.Foo.foo +[file foo.py] +class Foo: + def __init__(self) -> None: + self.y = '10' + + def foo(self, arg, lol=None): + if isinstance(arg, int): + return arg+1 + else: + assert arg + return arg+self.y +[file bar.py] +from foo import Foo +def bar() -> None: + x = Foo() + x.foo('abc') + x.foo(lol=10, arg=10) + x.foo(None) +[builtins fixtures/isinstancelist.pyi] +[out] +(Union[str, int, None], Optional[int]) -> object +== + +[case testSuggestInferMethod2] +# suggest: foo.Foo.foo +[file foo.py] +class Foo: + def i(self, x: int) -> int: return x + def s(self, x: str) -> str: return x + + def foo(self, arg, lol=None): + if isinstance(arg, int): + return self.i(arg) + else: + assert arg + return self.s(arg) +[file bar.py] +from typing import Union +from foo import Foo +def bar() -> None: + x = Foo() + x.foo('abc') + x.foo(lol=10, arg=10) + a: Union[str, int] = x.foo(None) +[builtins fixtures/isinstancelist.pyi] +[out] +(Union[str, int, None], Optional[int]) -> Union[int, str] +== + +[case testSuggestInferMethod3] +# suggest2: foo.Foo.foo +[file foo.py] +class Foo: + def foo(self, lol = None): + pass + + def lol(self) -> None: + self.foo('lol') +[file bar.py] +from foo import Foo +def bar() -> None: + x = Foo() + x.foo('abc') +[builtins fixtures/isinstancelist.pyi] +[out] +== +(Optional[str]) -> None + +[case testSuggestBackflow] +# suggest: foo.foo +# suggest: foo.bar +# suggest: foo.baz +# suggest: foo.Cls.spam +# suggest: foo.method +# suggest: foo.meet +# suggest: foo.has_nested +[file foo.py] +from typing import Any, List, Optional + +class A: pass +class B(A): + def test(self, x: A) -> None: + pass + +def take_optional_a(x: Optional[A]) -> None: + pass + +def take_a(x: A) -> None: + pass + +def take_b(x: B) -> None: + pass + +def take_any(x: Any) -> None: + pass + +def take_kws(a: A, b: B) -> None: + pass + +def take_star(*a: A) -> None: + pass + +def foo(x): + take_b(x) + +def bar(x): + take_b(x) + +bar(A()) + +def baz(x, y): + take_kws(a=y, b=x) + +class Cls: + def spam(self, x, y): + take_star(x, y) + +def method(x): + b = B() + b.test(x) + +def meet(x, y): + take_a(x) + take_b(x) + + take_a(y) + take_optional_a(y) + take_any(y) + foo(y) # unannotated + +def has_nested(x): + def nested(): + take_b(x) + +[builtins fixtures/isinstancelist.pyi] +[out] +(foo.B) -> None +(foo.A) -> None +(foo.B, foo.A) -> None +(foo.A, foo.A) -> None +(foo.A) -> None +(foo.B, foo.A) -> None +(foo.B) -> None +== + +[case testSuggestInferFunctionUnreachable] +# suggest: foo.foo +[file foo.py] +import sys +def foo(lol): + if sys.platform == 'nothing': + return lol + else: + return lol + lol +[file bar.py] +from foo import foo +foo('test') +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + +[case testSuggestInferMethodStep2] +# suggest2: foo.Foo.foo +[file foo.py] +class Foo: + def i(self, x: int) -> int: return x + def s(self, x: str) -> str: return x + + def foo(self, arg, lol=None): + if isinstance(arg, int): + return self.i(arg) + else: + assert arg + return self.s(arg) +[file bar.py] +from typing import Union +from foo import Foo +def bar() -> None: + x = Foo() + x.foo('abc') + x.foo(lol=10, arg=10) + a: Union[str, int] = x.foo(None) +[builtins fixtures/isinstancelist.pyi] +[out] +== +(Union[str, int, None], Optional[int]) -> Union[int, str] + +[case testSuggestInferNestedMethod] +# suggest: foo.Foo.Bar.baz +[file foo.py] +class Foo: + class Bar: + def baz(self, lol): + return lol +[file bar.py] +from foo import Foo +def bar() -> None: + x = Foo.Bar() + x.baz('abc') +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + +[case testSuggestCallable] +# suggest: foo.foo +# suggest: foo.bar +# suggest: --flex-any=0.9 foo.bar +# suggest: foo.baz +# suggest: foo.quux +[file foo.py] +def foo(f): + return f(0, "lol") +def bar(f): + return f(0, "lol", 100) +def baz(f): + return f(y=1) + f(x=10, y=1) +def quux(f): + return f(1) +[file bar.py] +from typing import Any +from foo import foo, bar, baz, quux + +def whatever(x: int, y: str) -> int: + return 0 + +def starargs(*args: Any) -> int: + return 0 + +def named(*, x: int = 0, y: int) -> str: + return '0' + +# we don't properly handle default really. we just assume it is +# actually required. +def default(x: int = 0) -> str: + return '0' + +def test() -> None: + foo(whatever) + bar(starargs) + baz(named) + quux(default) +[builtins fixtures/primitives.pyi] +[out] +(Callable[[int, str], int]) -> int +(Callable[..., int]) -> int +No guesses that match criteria! +(Callable[..., str]) -> str +(Callable[[int], str]) -> str +== + +[case testSuggestNewSemanal] +# suggest: foo.Foo.foo +# suggest: foo.foo +[file foo.py] +class Foo: + def __init__(self) -> None: + self.y = '10' + + def foo(self, arg, lol=None): + if isinstance(arg, int): + return arg+1 + else: + assert arg + return arg+self.y +def foo(arg, lol=None): + if isinstance(arg, int): + arg+1 + else: + assert arg + arg+'1' +[file bar.py] +from foo import Foo, foo +def bar() -> None: + x = Foo() + x.foo('abc') + x.foo(lol=10, arg=10) + x.foo(None) +def baz() -> None: + foo('abc') + foo(lol=10, arg=10) + foo(None) +[builtins fixtures/isinstancelist.pyi] +[out] +(Union[str, int, None], Optional[int]) -> object +(Union[str, int, None], Optional[int]) -> None +== + +[case testSuggestInferFuncDecorator1] +# suggest: foo.foo +[file foo.py] +from typing import TypeVar +F = TypeVar('F') + +def dec(x: F) -> F: + return x + +@dec +def foo(arg): + return arg +[file bar.py] +from foo import foo +def bar() -> None: + foo('abc') +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + +[case testSuggestInferFuncDecorator2] +# suggest: foo.foo +[file foo.py] +from typing import TypeVar, Callable, Any +F = TypeVar('F', bound=Callable[..., Any]) + +def dec(x: F) -> F: + return x + +@dec +def foo(arg): + return arg +[file bar.py] +from foo import foo +def bar() -> None: + foo('abc') +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + +[case testSuggestInferFuncDecorator3] +# suggest: foo.foo +[file foo.py] +from typing import TypeVar, Callable, Any +F = TypeVar('F', bound=Callable[..., Any]) + +def dec(s: str) -> Callable[[F], F]: + def f(x: F) -> F: + return x + return f + +@dec('lol') +def foo(arg): + return arg +[file bar.py] +from foo import foo +def bar() -> None: + foo('abc') +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + +[case testSuggestInferFuncDecorator4] +# suggest: foo.foo +[file dec.py] +from typing import TypeVar, Callable, Any +F = TypeVar('F', bound=Callable[..., Any]) + +def dec(s: str) -> Callable[[F], F]: + def f(x: F) -> F: + return x + return f + +[file foo.py] +import dec + +@dec.dec('lol') +def foo(arg): + return arg +[file bar.py] +from foo import foo +def bar() -> None: + foo('abc') +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + +[case testSuggestFlexAny1] +# suggest: --flex-any=0.4 m.foo +# suggest: --flex-any=0.7 m.foo +# suggest: --flex-any=0.4 m.bar +# suggest: --flex-any=0.6 m.bar +# suggest2: --flex-any=0.4 m.foo +# suggest2: --flex-any=0.7 m.foo +[file m.py] +from typing import Any +any: Any + +def foo(arg): + return 0 +def bar(x, y): + return any + +[file n.py] +from typing import Any +any: Any + +from m import foo, bar +def wtvr() -> None: + foo(any) + bar(1, 2) + +[file n.py.2] +from typing import Any +any: Any + +from m import foo, bar +def wtvr() -> None: + foo([any]) + +[builtins fixtures/isinstancelist.pyi] +[out] +(Any) -> int +No guesses that match criteria! +(int, int) -> Any +No guesses that match criteria! +== +(typing.List[Any]) -> int +(typing.List[Any]) -> int + + +[case testSuggestFlexAny2] +# suggest: --flex-any=0.5 m.baz +# suggest: --flex-any=0.0 m.baz +# suggest: --flex-any=0.5 m.F.foo +# suggest: --flex-any=0.7 m.F.foo +# suggest: --flex-any=0.7 m.noargs +[file m.py] +# Test mostly corner cases + +# Test that a None return doesn't get counted +def baz(x): + pass + +class F: + # Test that self doesn't get counted + def foo(self, x): + return 0 + +# Make sure we don't crash on noarg functions +def noargs(): + pass + +[builtins fixtures/isinstancelist.pyi] +[out] +No guesses that match criteria! +(Any) -> None +(Any) -> int +No guesses that match criteria! +() -> None +== + +[case testSuggestClassMethod] +# suggest: foo.F.bar +# suggest: foo.F.baz +# suggest: foo.F.eggs +[file foo.py] +class F: + @classmethod + def bar(cls, x, y): + return x + + @staticmethod + def baz(x, y): + return x + + @classmethod + def spam(cls): + # type: () -> None + cls.eggs(4) + + @classmethod + def eggs(cls, x): + pass + +[file bar.py] +from foo import F +def bar(iany) -> None: + F.bar(0, iany) + F().bar(0, 5) + F.baz('lol', iany) + F().baz('lol', 10) +[builtins fixtures/classmethod.pyi] +[out] +(int, int) -> int +(str, int) -> str +(int) -> None +== + +[case testSuggestColonBasic] +# suggest: tmp/foo.py:1 +# suggest: tmp/bar/baz.py:2 +[file foo.py] +def func(arg): + return 0 +func('test') +from bar.baz import C +C().method('test') +[file bar/__init__.py] +[file bar/baz.py] +class C: + def method(self, x): + return 0 +[out] +(str) -> int +(str) -> int +== + +[case testSuggestColonAfter] +# suggest: tmp/foo.py:6 +# suggest: tmp/foo.py:15 +# suggest: tmp/foo.py:16 +# suggest: tmp/foo.py:18 +[file foo.py] +from typing import TypeVar +F = TypeVar('F') + +def foo(): + # hi + return 1 + +def dec(x: F) -> F: + return x + +class A: + @dec + def bar(self): + return 1.0 + +@dec +def baz(): + return 'test' + +[out] +() -> int +() -> float +() -> str +() -> str +== + +[case testSuggestParent] +# suggest: foo.B.foo +# suggest: foo.B.bar +# suggest: foo.C.foo +[file foo.py] +from typing import TypeVar, Callable, Any +F = TypeVar('F', bound=Callable[..., Any]) +def deco(f: F) -> F: ... + +class A: + def foo(self, x: int) -> float: + return 0.0 + + @deco + def bar(self, x: int) -> float: + return 0.0 + + +class B(A): + def foo(self, x): + return 0.0 + + @deco + def bar(self, x): + return 0.0 + +class C(B): + def foo(self, x): + return 0.0 + +[out] +(int) -> float +(int) -> float +(int) -> float +== + +[case testSuggestColonBadLocation] +# suggest: tmp/foo.py:7:8:9 +[file foo.py] +[out] +Malformed location for function: tmp/foo.py:7:8:9. Must be either package.module.Class.method or path/to/file.py:line +== + +[case testSuggestColonBadLine] +# suggest: tmp/foo.py:bad +[file foo.py] +[out] +Line number must be a number. Got bad +== + +[case testSuggestColonBadFile] +# suggest: tmp/foo.txt:1 +[file foo.txt] +def f(): pass +[out] +Source file is not a Python file +== + +[case testSuggestColonClass] +# suggest: tmp/foo.py:1 +[file foo.py] +class C: + pass +[out] +Cannot find a function at line 1 +== + +[case testSuggestColonDecorator] +# suggest: tmp/foo.py:6 +[file foo.py] +from typing import TypeVar, Callable, Any +F = TypeVar('F', bound=Callable[..., Any]) +def deco(f: F) -> F: ... + +@deco +def func(arg): + return 0 +func('test') +[out] +(str) -> int +== + +[case testSuggestColonMethod] +# suggest: tmp/foo.py:3 +[file foo.py] +class Out: + class In: + def method(self, x): + return Out() +x: Out.In +x.method(x) +[out] +(foo:Out.In) -> foo.Out +== + +[case testSuggestColonMethodJSON] +# suggest: --json tmp/foo.py:3 +[file foo.py] +class Out: + class In: + def method(self, x): + return Out() +x: Out.In +x.method(x) +[out] +\[{"func_name": "Out.In.method", "line": 3, "path": "tmp/foo.py", "samples": 0, "signature": {"arg_types": ["foo:Out.In"], "return_type": "foo.Out"}}] +== + +[case testSuggestColonNonPackageDir] +# cmd: mypy foo/bar/baz.py +# suggest: tmp/foo/bar/baz.py:1 +[file foo/bar/baz.py] +def func(arg): + return 0 +func('test') +[out] +(str) -> int +== + +[case testSuggestUseFixmeBasic] +# suggest: --use-fixme=UNKNOWN foo.foo +# suggest: --use-fixme=UNKNOWN foo.bar +[file foo.py] + +def foo(): + return g() + +def bar(x): + return None + +def g(): ... +x = bar(g()) +[out] +() -> UNKNOWN +(UNKNOWN) -> None +== + +[case testSuggestUseFixmeNoNested] +# suggest: --use-fixme=UNKNOWN foo.foo +[file foo.py] +from typing import List, Any + +def foo(x, y): + return x, y + +def f() -> List[Any]: ... +def g(): ... + +z = foo(f(), g()) +[builtins fixtures/isinstancelist.pyi] +[out] +(foo.List[Any], UNKNOWN) -> Tuple[foo.List[Any], Any] +== + +[case testSuggestBadImport] +# suggest: foo.foo +[file foo.py] +from nothing import Foo # type: ignore + +def foo(x: Foo): + return 10 +[out] +(foo.Foo) -> int +== + +[case testSuggestDict] +# suggest: foo.foo +# suggest: foo.bar +# suggest: foo.baz +# suggest: foo.quux +# suggest: foo.spam +[file foo.py] +from typing import List, Any + +def foo(): + return {'x': 5} + +def bar(): + return {} + +def baz() -> List[Any]: + return [{'x': 5}] + +def quux() -> List[Any]: + return [1] + +def spam(x): + pass + +spam({'x': 5}) + +[builtins fixtures/dict.pyi] +[out] +() -> typing.Dict[str, int] +() -> typing.Dict[Any, Any] +() -> foo:List[typing.Dict[str, int]] +() -> foo.List[int] +(typing.Dict[str, int]) -> None +== + +[case testSuggestWithErrors] +# suggest: foo.foo +[file foo.py] +1+'no' + +def foo(): + return 10 +[out] +foo.py:1: error: Unsupported operand types for + ("int" and "str") +() -> int +== +foo.py:1: error: Unsupported operand types for + ("int" and "str") + +[case testSuggestWithBlockingError] +# suggest: foo.foo +[file foo.py] +def foo(): + return 10 + +( +[out] +foo.py:4: error: unexpected EOF while parsing +Command 'suggest' is only valid after a 'check' command (that produces no parse errors) +== +foo.py:4: error: unexpected EOF while parsing +[out version>=3.10] +foo.py:4: error: '(' was never closed +Command 'suggest' is only valid after a 'check' command (that produces no parse errors) +== +foo.py:4: error: '(' was never closed +-- ) + +[case testSuggestRefine] +# suggest: foo.foo +# suggest: foo.spam +# suggest: foo.eggs +# suggest: foo.take_l +# suggest: foo.union +# suggest: foo.callable1 +# suggest: foo.callable2 +# suggest: foo.optional1 +# suggest: foo.optional2 +# suggest: foo.optional3 +# suggest: foo.optional4 +# suggest: foo.optional5 +# suggest: foo.optional_any +# suggest: foo.dict1 +# suggest: foo.tuple1 +[file foo.py] +from typing import Any, List, Union, Callable, Optional, Set, Dict, Tuple + +def bar(): + return 10 + +def foo(x: int, y): + return x + y + +foo(bar(), 10) + +def spam(x: int, y: Any) -> Any: + return x + y + +spam(bar(), 20) + +def eggs(x: int) -> List[Any]: + a = [x] + return a + +def take_l(x: List[Any]) -> Any: + return x[0] + +test = [10, 20] +take_l(test) + +def union(x: Union[int, str]): + pass + +union(10) + +def add1(x: float) -> int: + pass + +def callable1(f: Callable[[int], Any]): + return f(10) + +callable1(add1) + +def callable2(f: Callable[..., Any]): + return f(10) + +callable2(add1) + +def optional1(x: Optional[Any]): + pass + +optional1(10) + +def optional2(x: Union[None, int, Any]): + if x is None: + pass + elif isinstance(x, str): + pass + else: + add1(x) + +optional2(10) +optional2('test') + +def optional3(x: Optional[List[Any]]): + assert x + return x[0] + +optional3(test) + +set_test = {1, 2} + +def optional4(x: Union[Set[Any], List[Any]]): + pass + +optional4(test) +optional4(set_test) + +def optional5(x: Optional[Any]): + pass + +optional5(10) +optional5(None) + +def optional_any(x: Optional[Any] = None): + pass + +def dict1(d: Dict[int, Any]): + pass + +d: Dict[Any, int] +dict1(d) + +def tuple1(d: Tuple[int, Any]): + pass + +t: Tuple[Any, int] +tuple1(t) + +[builtins fixtures/isinstancelist.pyi] +[out] +(int, int) -> int +(int, int) -> int +(int) -> foo.List[int] +(foo.List[int]) -> int +(Union[int, str]) -> None +(Callable[[int], int]) -> int +(Callable[[float], int]) -> int +(Optional[int]) -> None +(Union[None, int, str]) -> None +(Optional[foo.List[int]]) -> int +(Union[foo.Set[int], foo.List[int]]) -> None +(Optional[int]) -> None +(Optional[Any]) -> None +(foo.Dict[int, int]) -> None +(Tuple[int, int]) -> None +== + +[case testSuggestRefine2] +# suggest: foo.optional5 +[file foo.py] +from typing import Optional, Any + +def optional5(x: Optional[Any]): + pass + +optional5(10) +optional5(None) + +[builtins fixtures/isinstancelist.pyi] +[out] +(Optional[int]) -> None +== diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test new file mode 100644 index 000000000000..9ff8a37ae9ae --- /dev/null +++ b/test-data/unit/fine-grained.test @@ -0,0 +1,11163 @@ +-- Test cases for fine-grained incremental checking +-- +-- Test cases may define multiple versions of a file +-- (e.g. m.py, m.py.2). There is always an initial batch +-- pass that processes all files present initially, followed +-- by one or more fine-grained incremental passes that use +-- alternative versions of files, if available. If a file +-- just has a single .py version, it is used for all passes. + +-- TODO: what if version for some passes but not all + +-- Output is laid out like this: +-- +-- [out] +-- +-- == +-- +-- +-- +-- Modules that are expected to be detected as changed by dmypy_server +-- can be checked with [stale ...] +-- Generally this should mean added, deleted, or changed files, though there +-- are important edge cases related to the cache: deleted files won't be detected +-- as changed in the initial run with the cache while modules that depended on them +-- should be. +-- +-- Modules that require a full-module reprocessing by update can be checked with +-- [rechecked ...]. This should include any files detected as having changed as well +-- as any files that contain targets that need to be reprocessed but which haven't +-- been loaded yet. If there is no [rechecked...] directive, it inherits the value of +-- [stale ...]. +-- +-- Specifications for later runs can be given with [stale2 ...], [stale3 ...], etc. +-- +-- Test runner can parse options from mypy.ini file. Updating this file in between +-- incremental runs is not yet supported. +-- +-- Each test case run without caching and with caching (if the initial run passes), +-- unless it has one a -only_when_cache or -only_when_nocache arguments. We sometimes +-- skip caching test cases to speed up tests, if the caching variant is not useful. +-- The caching test case variants get an implicit _cached suffix. + +[case testReprocessFunction] +import m +def g() -> int: + return m.f() +[file m.py] +def f() -> int: + pass +[file m.py.2] +def f() -> str: + pass +[out] +== +main:3: error: Incompatible return value type (got "str", expected "int") + +[case testReprocessTopLevel] +import m +m.f(1) +def g() -> None: pass +[file m.py] +def f(x: int) -> None: pass +[file m.py.2] +def f(x: str) -> None: pass +[out] +== +main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testReprocessMethod] +import m +class B: + def f(self, a: m.A) -> None: + a.g() # E +[file m.py] +class A: + def g(self) -> None: pass +[file m.py.2] +class A: + def g(self, a: A) -> None: pass +[out] +== +main:4: error: Missing positional argument "a" in call to "g" of "A" + +[case testReprocessMethodShowSource] +# flags: --pretty --show-error-codes +import m +class B: + def f(self, a: m.A) -> None: + a.g() # E +[file m.py] +class A: + def g(self) -> None: pass +[file m.py.2] +class A: + def g(self, a: A) -> None: pass +[out] +== +main:5: error: Missing positional argument "a" in call to "g" of "A" [call-arg] + a.g() # E + ^~~~~ + +[case testFunctionMissingModuleAttribute] +import m +def h() -> None: + m.f(1) +[file m.py] +def f(x: int) -> None: pass +[file m.py.2] +def g(x: str) -> None: pass +[builtins fixtures/fine_grained.pyi] +[out] +== +main:3: error: Module has no attribute "f" + +[case testTopLevelMissingModuleAttribute] +import m +m.f(1) +def g() -> None: pass +[file m.py] +def f(x: int) -> None: pass +[file m.py.2] +def g(x: int) -> None: pass +[builtins fixtures/fine_grained.pyi] +[out] +== +main:2: error: Module has no attribute "f" + +[case testClassChangedIntoFunction] + +import m +def f(a: m.A) -> None: + pass +[file m.py] +class A: pass +[file m.py.2] +def A() -> None: pass +[out] +== +main:3: error: Function "m.A" is not valid as a type +main:3: note: Perhaps you need "Callable[...]" or a callback protocol? + +[case testClassChangedIntoFunction2] + +import m +class B: + def f(self, a: m.A) -> None: pass +[file m.py] +class A: pass +[file m.py.2] +def A() -> None: pass +[file n.py.3] +[out] +== +main:4: error: Function "m.A" is not valid as a type +main:4: note: Perhaps you need "Callable[...]" or a callback protocol? +== +main:4: error: Function "m.A" is not valid as a type +main:4: note: Perhaps you need "Callable[...]" or a callback protocol? + +[case testAttributeTypeChanged] +import m +def f(a: m.A) -> int: + return a.x +[file m.py] +class A: + def f(self) -> None: + self.x = 1 +[file m.py.2] +class A: + def f(self) -> None: + self.x = 'x' +[out] +== +main:3: error: Incompatible return value type (got "str", expected "int") + +[case testAttributeRemoved] +import m +def f(a: m.A) -> int: + return a.x +[file m.py] +class A: + def f(self) -> None: + self.x = 1 +[file m.py.2] +class A: + def f(self) -> None: pass +[out] +== +main:3: error: "A" has no attribute "x" + +[case testVariableTypeBecomesInvalid] +import m +def f() -> None: + a: m.A +[file m.py] +class A: pass +[file m.py.2] +[out] +== +main:3: error: Name "m.A" is not defined + +[case testTwoIncrementalSteps] +import m +import n +[file m.py] +def f() -> None: pass +[file n.py] +import m +def g() -> None: + m.f() # E +[file m.py.2] +import n +def f(x: int) -> None: + n.g() # E +[file n.py.3] +import m +def g(a: str) -> None: + m.f('') # E +[out] +== +n.py:3: error: Missing positional argument "x" in call to "f" +== +n.py:3: error: Argument 1 to "f" has incompatible type "str"; expected "int" +m.py:3: error: Missing positional argument "a" in call to "g" + +[case testTwoRounds] +import m +def h(a: m.A) -> int: + return a.x +[file m.py] +import n +class A: + def g(self, b: n.B) -> None: + self.x = b.f() +[file n.py] +class B: + def f(self) -> int: pass +[file n.py.2] +class B: + def f(self) -> str: pass +[out] +== +main:3: error: Incompatible return value type (got "str", expected "int") + +[case testFixTypeError] +import m +def f(a: m.A) -> None: + a.f(a) +[file m.py] +class A: + def f(self, a: 'A') -> None: pass +[file m.py.2] +class A: + def f(self) -> None: pass +[file m.py.3] +class A: + def f(self, a: 'A') -> None: pass +[out] +== +main:3: error: Too many arguments for "f" of "A" +== + +[case testFixTypeError2] +import m +def f(a: m.A) -> None: + a.f() +[file m.py] +class A: + def f(self) -> None: pass +[file m.py.2] +class A: + def g(self) -> None: pass +[file m.py.3] +class A: + def f(self) -> None: pass +[out] +== +main:3: error: "A" has no attribute "f" +== + +[case testFixSemanticAnalysisError] +import m +def f() -> None: + m.A() +[file m.py] +class A: pass +[file m.py.2] +class B: pass +[file m.py.3] +class A: pass +[builtins fixtures/fine_grained.pyi] +[out] +== +main:3: error: Module has no attribute "A" +== + +[case testContinueToReportTypeCheckError] +import m +def f(a: m.A) -> None: + a.f() +def g(a: m.A) -> None: + a.g() +[file m.py] +class A: + def f(self) -> None: pass + def g(self) -> None: pass +[file m.py.2] +class A: pass +[file m.py.3] +class A: + def f(self) -> None: pass +[out] +== +main:3: error: "A" has no attribute "f" +main:5: error: "A" has no attribute "g" +== +main:5: error: "A" has no attribute "g" + +[case testContinueToReportSemanticAnalysisError] +import m +def f() -> None: + m.A() +def g() -> None: + m.B() +[file m.py] +class A: pass +class B: pass +[file m.py.2] +[file m.py.3] +class A: pass +[builtins fixtures/fine_grained.pyi] +[out] +== +main:3: error: Module has no attribute "A" +main:5: error: Module has no attribute "B" +== +main:5: error: Module has no attribute "B" + +[case testContinueToReportErrorAtTopLevel-only_when_nocache] +-- Different cache/no-cache tests because: +-- Error message ordering differs +import n +import m +m.A().f() +[file n.py] +import m +m.A().g() +[file m.py] +class A: + def f(self) -> None: pass + def g(self) -> None: pass +[file m.py.2] +class A: pass +[file m.py.3] +class A: + def f(self) -> None: pass +[out] +== +main:3: error: "A" has no attribute "f" +n.py:2: error: "A" has no attribute "g" +== +n.py:2: error: "A" has no attribute "g" + +[case testContinueToReportErrorAtTopLevel2-only_when_cache] +-- Different cache/no-cache tests because: +-- Error message ordering differs +import n +import m +m.A().f() +[file n.py] +import m +m.A().g() +[file m.py] +class A: + def f(self) -> None: pass + def g(self) -> None: pass +[file m.py.2] +class A: pass +[file m.py.3] +class A: + def f(self) -> None: pass +[out] +== +n.py:2: error: "A" has no attribute "g" +main:3: error: "A" has no attribute "f" +== +n.py:2: error: "A" has no attribute "g" + +[case testContinueToReportErrorInMethod] +import m +class C: + def f(self, a: m.A) -> None: + a.f() + def g(self, a: m.A) -> None: + a.g() +[file m.py] +class A: + def f(self) -> None: pass + def g(self) -> None: pass +[file m.py.2] +class A: pass +[file m.py.3] +class A: + def f(self) -> None: pass +[out] +== +main:4: error: "A" has no attribute "f" +main:6: error: "A" has no attribute "g" +== +main:6: error: "A" has no attribute "g" + +[case testInitialBatchGeneratedError] +import m +def g() -> None: + m.f() +def h() -> None: + m.g() +[file m.py] +def f(x: object) -> None: pass +[file m.py.2] +def f() -> None: pass +[file m.py.3] +def f() -> None: pass +def g() -> None: pass +[builtins fixtures/fine_grained.pyi] +[out] +main:3: error: Missing positional argument "x" in call to "f" +main:5: error: Module has no attribute "g" +== +main:5: error: Module has no attribute "g" +== + +[case testKeepReportingErrorIfNoChanges] +import m +def h() -> None: + m.g() +[file m.py] +[file m.py.2] +[builtins fixtures/fine_grained.pyi] +[out] +main:3: error: Module has no attribute "g" +== +main:3: error: Module has no attribute "g" + +[case testFixErrorAndReintroduce] +import m +def h() -> None: + m.g() +[file m.py] +[file m.py.2] +def g() -> None: pass +[file m.py.3] +[builtins fixtures/fine_grained.pyi] +[out] +main:3: error: Module has no attribute "g" +== +== +main:3: error: Module has no attribute "g" + +[case testIgnoreWorksAfterUpdate] +import a +[file a.py] +import b +int() + str() # type: ignore +[file b.py] +x = 1 +[file b.py.2] +x = 2 +[file b.py.3] +x = 3 +[delete b.py.4] +[out] +== +== +== +a.py:1: error: Cannot find implementation or library stub for module named "b" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testIgnoreWorksWithMissingImports] +import a +[file a.py] +import b +import xyz # type: ignore +xyz.whatever +[file b.py] +x = 1 +[file b.py.2] +x = 2 +[file b.py.3] +x = 3 +[file xyz.py.4] +[out] +== +== +== +a.py:3: error: "object" has no attribute "whatever" + +[case testAddedIgnoreWithMissingImports] +import a +[file a.py] +from b import x +y: int = x +[file b.py] +from xyz import x +[file b.py.2] +from xyz import x # type: ignore +[file xyz.py.3] +x = str() +[out] +b.py:1: error: Cannot find implementation or library stub for module named "xyz" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +== +a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testRemovedIgnoreWithMissingImport] +import a +[file a.py] +from b import x +y: int = x +[file b.py] +from xyz import x # type: ignore +[file b.py.2] +from xyz import x +[file xyz.py.3] +x = str() +[out] +== +b.py:1: error: Cannot find implementation or library stub for module named "xyz" +b.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testRemovedModuleUnderIgnore] +import a +[file a.py] +import c +from b import x # type: ignore +y: int = x +[file b.py] +x = str() +[file c.py] +x = 1 +[delete b.py.2] +[file c.py.3] +x = 3 +[out] +a.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") +== +== + +[case AddedModuleUnderIgnore] +import a +[file a.py] +import c +from b import x # type: ignore +y: int = x +[file c.py] +x = 1 +[file c.py.2] +x = 2 +[file b.py.3] +# empty +[out] +== +== + +[case testIgnoreInBetween] +import a +[file a.py] +import b +x: int = b.x +[file b.py] +import c +x = c.C.x # type: ignore +[file c.py] +class C: + pass +[file c.py.2] +class C: + x: int +[file c.py.3] +# empty +[file c.py.4] +class C: + x: str +[out] +== +== +== +a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testIgnoredAttrReprocessedModule] +import a +[file a.py] +import b +x = b.x # type: ignore +y: int = x +[file b.py] +import c +[file b.py.2] +import c +x = c.x +[file c.py] +x: str +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testIgnoredAttrReprocessedBase] +import a +[file a.py] +import b +def fun() -> None: + x = b.C.x # type: ignore + y: int = x +[file b.py] +import c +class C: + pass +[file b.py.2] +import c +class C(c.B): + pass +[file c.py] +class B: + x: str +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testIgnoredAttrReprocessedMeta] +import a +[file a.py] +import b +def fun() -> None: + x = b.C.x # type: ignore + y: int = x +[file b.py] +import c +class C: + pass +[file b.py.2] +import c +class C(metaclass=c.M): + pass +[file c.py] +class M(type): + x: str +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testDataclassUpdate1] +[file a.py] +from dataclasses import dataclass + +@dataclass +class A: + x: int + +[file b.py] +from dataclasses import dataclass + +from a import A +@dataclass +class B(A): + y: int + +B(1, 2) + +[file a.py.2] +from dataclasses import dataclass + +@dataclass +class A: + x: str + +[file a.py.3] +from dataclasses import dataclass + +@dataclass +class A: + x: int + +[out] +== +b.py:8: error: Argument 1 to "B" has incompatible type "int"; expected "str" +== +[builtins fixtures/dataclasses.pyi] + +[case testDataclassUpdate2] +[file c.py] +Foo = int + +[file c.py.2] +Foo = str + +[file a.py] +from dataclasses import dataclass +from c import Foo + +@dataclass +class A: + x: Foo + +[file b.py] +from dataclasses import dataclass + +from a import A +@dataclass +class B(A): + y: int + +B(1, 2) + +[out] +== +b.py:8: error: Argument 1 to "B" has incompatible type "int"; expected "str" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassUpdate3] +from b import B +B(1, 2) +[file b.py] +from a import A +from dataclasses import dataclass +@dataclass +class B(A): + b: int +[file a.py] +from dataclasses import dataclass +@dataclass +class A: + a: int + +[file a.py.2] +from dataclasses import dataclass +@dataclass +class A: + a: int + other: int +[builtins fixtures/dataclasses.pyi] +[out] +== +main:2: error: Missing positional argument "b" in call to "B" + +[case testDataclassUpdate4] +from b import B +B(1, 2) +[file b.py] +from a import A +from dataclasses import dataclass +@dataclass(frozen=True) +class B(A): + b: int +[file a.py] +from dataclasses import dataclass +@dataclass(frozen=True) +class A: + a: int + +[file a.py.2] +from dataclasses import dataclass +@dataclass(frozen=True) +class A: + a: int + other: int +[builtins fixtures/dataclasses.pyi] +[out] +== +main:2: error: Missing positional argument "b" in call to "B" + +[case testDataclassUpdate5] +from b import B +B(1, 2) +[file b.py] +from a import A +from dataclasses import dataclass +@dataclass +class B(A): + b: int +[file a.py] +from dataclasses import dataclass +@dataclass(init=False) +class A: + a: int + +[file a.py.2] +from dataclasses import dataclass +@dataclass(init=False) +class A: + a: int + other: int + +[file a.py.3] +from dataclasses import dataclass +@dataclass(init=False) +class A: + a: int + +[builtins fixtures/dataclasses.pyi] +[out] +== +main:2: error: Missing positional argument "b" in call to "B" +== + +[case testDataclassUpdate6] +from b import B +B(1, 2) < B(1, 2) +[file b.py] +from a import A +from dataclasses import dataclass +@dataclass +class B(A): + b: int +[file a.py] +from dataclasses import dataclass +@dataclass(order=True) +class A: + a: int + +[file a.py.2] +from dataclasses import dataclass +@dataclass +class A: + a: int +[builtins fixtures/dataclasses.pyi] +[out] +== +main:2: error: Unsupported left operand type for < ("B") + +[case testDataclassUpdate8] +from c import C +C(1, 2, 3) +[file c.py] +from b import B +from dataclasses import dataclass +@dataclass +class C(B): + c: int +[file b.py] +from a import A +from dataclasses import dataclass +@dataclass +class B(A): + b: int +[file a.py] +from dataclasses import dataclass +@dataclass +class A: + a: int + +[file a.py.2] +from dataclasses import dataclass +@dataclass +class A: + a: int + other: int +[builtins fixtures/dataclasses.pyi] +[out] +== +main:2: error: Missing positional argument "c" in call to "C" + +[case testDataclassUpdate9] +from c import C +C(1, 2, 3) +[file c.py] +from b import B +from dataclasses import dataclass +@dataclass +class C(B): + c: int +[file b.py] +from a import A +from dataclasses import dataclass +@dataclass +class B(A): + b: int +[file a.py] +from dataclasses import dataclass +@dataclass(init=False) +class A: + a: int + +[file a.py.2] +from dataclasses import dataclass +@dataclass(init=False) +class A: + a: int + other: int + +[file a.py.3] +from dataclasses import dataclass +@dataclass(init=False) +class A: + a: int + +[builtins fixtures/dataclasses.pyi] +[out] +== +main:2: error: Missing positional argument "c" in call to "C" +== + +[case testAttrsUpdate1] +[file a.py] +import attr +@attr.s +class A: + a = attr.ib() # type: int + +[file b.py] +from a import A +import attr +@attr.s +class B(A): + b = attr.ib() # type: int + +B(1, 2) + +[file a.py.2] +import attr +@attr.s +class A: + a = attr.ib() # type: int + other = attr.ib() # type: int +[builtins fixtures/list.pyi] +[out] +== +b.py:7: error: Missing positional argument "b" in call to "B" + +[case testAttrsUpdate2] +from b import B +B(1, 2) +[file b.py] +from a import A +import attr +@attr.s +class B(A): + b = attr.ib() # type: int +[file a.py] +import attr +@attr.s(init=False) +class A: + a = attr.ib() # type: int + +[file a.py.2] +import attr +@attr.s(init=False) +class A: + a = attr.ib() # type: int + other = attr.ib() # type: int +[builtins fixtures/list.pyi] +[out] +== +main:2: error: Missing positional argument "b" in call to "B" + +[case testAttrsUpdate3] +from b import B +B(1, 2) +[file b.py] +from a import A +import attr +@attr.s(auto_attribs=True) +class B(A): + x: int +[file a.py] +import attr +@attr.s(auto_attribs=True, init=False) +class A: + a: int + +[file a.py.2] +import attr +@attr.s(auto_attribs=True, init=False) +class A: + a: int + other: int + +[file a.py.3] +import attr +@attr.s(auto_attribs=True, init=False) +class A: + a: int +[builtins fixtures/list.pyi] + +[out] +== +main:2: error: Missing positional argument "x" in call to "B" +== + +[case testAttrsUpdate4] +from b import B +B(1, 2) < B(1, 2) +[file b.py] +from a import A +import attr +@attr.s(eq=False) +class B(A): + b = attr.ib() # type: int +[file a.py] +import attr +@attr.s(init=False) +class A: + a = attr.ib() # type: int + +[file a.py.2] +import attr +@attr.s(eq=False, init=False) +class A: + a = attr.ib() # type: int +[builtins fixtures/list.pyi] +[out] +== +main:2: error: Unsupported left operand type for < ("B") + +[case testAttrsUpdateBaseKwOnly] +from b import B +B(5) +[file a.py] +import attr +@attr.s() +class A: + a = attr.ib(15) # type: int +[file b.py] +from a import A +import attr +@attr.s(kw_only=True) +class B(A): + b = attr.ib("16") # type: str + +[file a.py.2] +import attr +@attr.s(kw_only=True) +class A: + a = attr.ib(15) # type: int +[builtins fixtures/plugin_attrs.pyi] +[out] +== +main:2: error: Too many positional arguments for "B" + +[case testAddBaseClassMethodCausingInvalidOverride] +import m +class B(m.A): + def f(self) -> str: pass +[file m.py] +class A: pass +[file m.py.2] +class A: + def f(self) -> int: pass +[file n.py.3] +[out] +== +main:3: error: Return type "str" of "f" incompatible with return type "int" in supertype "A" +== +main:3: error: Return type "str" of "f" incompatible with return type "int" in supertype "A" + +[case testModifyBaseClassMethodCausingInvalidOverride] +import m +class B(m.A): + def f(self) -> str: pass +[file m.py] +class A: + def f(self) -> str: pass +[file m.py.2] +class A: + def f(self) -> int: pass +[out] +== +main:3: error: Return type "str" of "f" incompatible with return type "int" in supertype "A" + +[case testAddBaseClassAttributeCausingErrorInSubclass] +import m +class B(m.A): + def a(self) -> None: + x = 1 + if int(): + x = self.x + + def f(self) -> None: + self.x = 1 + + def z(self) -> None: + x = 1 + if int(): + x = self.x +[file m.py] +class A: pass +[file m.py.2] +class A: + def g(self) -> None: + self.x = 'a' +[out] +== +main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") +main:14: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testChangeBaseClassAttributeType] +import m +class B(m.A): + def f(sel) -> None: + sel.x = 1 +[file m.py] +class A: + def g(self) -> None: + self.x = 1 +[file m.py.2] +class A: + def g(self) -> None: + self.x = 'a' +[out] +== +main:4: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testRemoveAttributeInBaseClass] +import m +class B(m.A): + def f(self) -> None: + a = 1 + a = self.x +[file m.py] +class A: + def g(self) -> None: + self.x = 1 +[file m.py.2] +class A: pass +[out] +== +main:5: error: "B" has no attribute "x" + +[case testTestSignatureOfInheritedMethod] +import m +class B(m.A): + def f(self) -> None: + self.g() +[file m.py] +class A: + def g(self) -> None: pass +[file m.py.2] +class A: + def g(self, a: 'A') -> None: pass +[out] +== +main:4: error: Missing positional argument "a" in call to "g" of "A" + +[case testRemoveBaseClass] +import m +class A(m.B): + def f(self) -> None: + self.g() + self.x + self.y = 1 +[file m.py] +class C: + def g(self) -> None: + self.x = 1 +class B(C): pass +[file m.py.2] +class C: pass +class B: pass +[out] +== +main:4: error: "A" has no attribute "g" +main:5: error: "A" has no attribute "x" + +[case testRemoveBaseClass2] +import m +class A(m.B): + def f(self) -> None: + self.g() + self.x + self.y = 1 +[file m.py] +class C: + def g(self) -> None: + self.x = 1 +class B(C): pass +[file m.py.2] +class C: + def g(self) -> None: + self.x = 1 +class B: pass +[out] +== +main:4: error: "A" has no attribute "g" +main:5: error: "A" has no attribute "x" + +[case testChangeInPackage] +import m.n +def f() -> None: + m.n.g() +[file m/__init__.py] +[file m/n.py] +def g() -> None: pass +[file m/n.py.2] +def g(x: int) -> None: pass +[out] +== +main:3: error: Missing positional argument "x" in call to "g" + +[case testTriggerTargetInPackage] +import m.n +[file m/__init__.py] +[file m/n.py] +import a +def f() -> None: + a.g() +[file a.py] +def g() -> None: pass +[file a.py.2] +def g(x: int) -> None: pass +[out] +== +m/n.py:3: error: Missing positional argument "x" in call to "g" + +[case testChangeInPackage__init__] +import m +import m.n +def f() -> None: + m.g() +[file m/__init__.py] +def g() -> None: pass +[file m/__init__.py.2] +def g(x: int) -> None: pass +[file m/n.py] +[out] +== +main:4: error: Missing positional argument "x" in call to "g" + +[case testTriggerTargetInPackage__init__] +import m +import m.n +[file m/__init__.py] +import a +def f() -> None: + a.g() +[file a.py] +def g() -> None: pass +[file a.py.2] +def g(x: int) -> None: pass +[file m/n.py] +[out] +== +m/__init__.py:3: error: Missing positional argument "x" in call to "g" + +[case testModuleAttributeTypeChanges] +import m +def f() -> None: + x = 1 + if int(): + x = m.x +[file m.py] +x = 1 +[file m.py.2] +x = '' +[out] +== +main:5: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testTwoStepsDueToModuleAttribute] +import m +x = m.f() + +def g() -> None: + y = 1 + if int(): + y = x # E +[file m.py] +def f() -> int: pass +[file m.py.2] +def f() -> str: pass +[out] +== +main:7: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testTwoStepsDueToMultipleNamespaces] +import m +x = m.f() + +def g() -> None: + xx = 1 + if int(): + xx = x # E + +class A: + def a(self) -> None: + self.y = m.f() + def b(self) -> None: + yy = 1 + if int(): + yy = self.y + +class B: + def c(self) -> None: + self.z = m.f() + def b(self) -> None: + zz = 1 + if int(): + zz = self.z +[file m.py] +def f() -> int: pass +[file m.py.2] +def f() -> str: pass +[out] +== +main:7: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:15: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:23: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testConstructorSignatureChanged] +import m + +def f() -> None: + m.A() +[file m.py] +class A: + def __init__(self) -> None: pass +[file m.py.2] +class A: + def __init__(self, x: int) -> None: pass +[out] +== +main:4: error: Missing positional argument "x" in call to "A" + +[case testConstructorSignatureChanged2] +from typing import Callable +import m + +def use(x: Callable[[], m.A]) -> None: + x() +def f() -> None: + use(m.A) +[file m.py] +class A: + def __init__(self) -> None: pass +[file m.py.2] +class A: + def __init__(self, x: int) -> None: pass +[out] +== +-- This is a bad error message +main:7: error: Argument 1 to "use" has incompatible type "Type[A]"; expected "Callable[[], A]" + +[case testConstructorSignatureChanged3] +from a import C +class D(C): + def g(self) -> None: + super().__init__() +D() +[file a.py] +class C: + def __init__(self) -> None: pass +[file a.py.2] +class C: + def __init__(self, x: int) -> None: pass +[out] +== +main:4: error: Missing positional argument "x" in call to "__init__" of "C" +main:5: error: Missing positional argument "x" in call to "D" + +[case testConstructorAdded] +import m + +def f() -> None: + m.A() +[file m.py] +class A: pass +[file m.py.2] +class A: + def __init__(self, x: int) -> None: pass +[out] +== +main:4: error: Missing positional argument "x" in call to "A" + +[case testConstructorDeleted] +import m + +def f() -> None: + m.A(1) +[file m.py] +class A: + def __init__(self, x: int) -> None: pass +[file m.py.2] +class A: pass +[out] +== +main:4: error: Too many arguments for "A" + +[case testBaseClassConstructorChanged] +import m + +def f() -> None: + m.B() +[file m.py] +class A: + def __init__(self) -> None: pass +class B(A): pass +[file m.py.2] +class A: + def __init__(self, x: int) -> None: pass +class B(A): pass +[out] +== +main:4: error: Missing positional argument "x" in call to "B" + +[case testSuperField] +from a import C +class D(C): + def g(self) -> int: + return super().x +[file a.py] +class C: + def __init__(self) -> None: self.x = 12 +[file a.py.2] +class C: + def __init__(self) -> None: self.x = 'ar' +[out] +== +main:4: error: Incompatible return value type (got "str", expected "int") + +[case testImportFrom] +from m import f + +def g() -> None: + f() +[file m.py] +def f() -> None: pass +[file m.py.2] +def f(x: int) -> None: pass +[builtins fixtures/fine_grained.pyi] +[out] +== +main:4: error: Missing positional argument "x" in call to "f" + +[case testImportFrom2] +from m import f +f() +[file m.py] +def f() -> None: pass +[file m.py.2] +def f(x: int) -> None: pass +[out] +== +main:2: error: Missing positional argument "x" in call to "f" + +[case testImportFromTargetsClass] +from m import C + +def f(c: C) -> None: + c.g() +[file m.py] +class C: + def g(self) -> None: pass +[file m.py.2] +class C: + def g(self, x: int) -> None: pass +[out] +== +main:4: error: Missing positional argument "x" in call to "g" of "C" + +[case testImportFromTargetsVariable] +from m import x + +def f() -> None: + y = 1 + if int(): + y = x +[file m.py] +x = 1 +[file m.py.2] +x = '' +[out] +== +main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testImportFromSubmoduleOfPackage] +from m import n + +def f() -> None: + n.g() +[file m/__init__.py] +[file m/n.py] +def g() -> None: pass +[file m/n.py.2] +def g(x: int) -> None: pass +[out] +== +main:4: error: Missing positional argument "x" in call to "g" + +[case testImportedFunctionGetsImported] +from m import f + +def g() -> None: + f() +[file m.py] +from n import f +[file n.py] +def f() -> None: pass +[file n.py.2] +def f(x: int) -> None: pass +[out] +== +main:4: error: Missing positional argument "x" in call to "f" + +[case testNestedClassMethodSignatureChanges] +from m import A + +def f(x: A.B) -> None: + x.g() +[file m.py] +class A: + class B: + def g(self) -> None: pass +[file m.py.2] +class A: + class B: + def g(self, x: int) -> None: pass +[out] +== +main:4: error: Missing positional argument "x" in call to "g" of "B" + +[case testNestedClassAttributeTypeChanges] +from m import A + +def f(x: A.B) -> None: + z = 1 + if int(): + z = x.y +[file m.py] +class A: + class B: + def g(self) -> None: + self.y = 1 +[file m.py.2] +class A: + class B: + def g(self) -> None: + self.y = '' +[out] +== +main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReprocessMethodInNestedClass] +from m import f + +class A: + class B: + def g(self) -> None: + x = 1 + if int(): + x = f() +[file m.py] +def f() -> int: pass +[file m.py.2] +def f() -> str: pass +[file n.py.3] +[out] +== +main:8: error: Incompatible types in assignment (expression has type "str", variable has type "int") +== +main:8: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReprocessMethodInNestedClassSemanal] +import a +[file a.py] +class A: + class B: + def g() -> None: pass + def foo(self) -> int: return 12 +[file b.py.2] +[file b.py.3] +2 +[out] +a.py:3: error: Method must have at least one argument. Did you forget the "self" argument? +== +a.py:3: error: Method must have at least one argument. Did you forget the "self" argument? +== +a.py:3: error: Method must have at least one argument. Did you forget the "self" argument? + +[case testBaseClassDeleted] +import m + +class A(m.C): + def f(self) -> None: + self.g() # No error here because m.C becomes an Any base class + def g(self) -> None: + self.x +[file m.py] +class C: + def g(self) -> None: pass +[file m.py.2] +[out] +main:7: error: "A" has no attribute "x" +== +main:3: error: Name "m.C" is not defined + +[case testBaseClassOfNestedClassDeleted] +import m + +class A: + class B(m.C): + def f(self) -> None: + self.g() # No error here because m.C becomes an Any base class + def g(self) -> None: + self.x +[file m.py] +class C: + def g(self) -> None: pass +[file m.py.2] +[out] +main:8: error: "B" has no attribute "x" +== +main:4: error: Name "m.C" is not defined + +[case testImportQualifiedModuleName] +import a +[file a.py] +import b.c +b.c.f() +[file a.py.2] +import b.c +b.c.f() # dummy change +[file b/__init__.py] +[file b/c.py] +def f() -> None: pass +[out] +== + +[case testTypeAliasRefresh] +from typing import Callable +from a import f +C = Callable[[int], str] +[file a.py] +def f() -> None: pass +[file a.py.2] +[out] +== +main:2: error: Module "a" has no attribute "f" + +[case testTypeVarRefresh] +from typing import TypeVar +from a import f +T = TypeVar('T') +[file a.py] +def f() -> None: pass +[file a.py.2] +[out] +== +main:2: error: Module "a" has no attribute "f" + +[case testRefreshTyping] +from typing import Sized +from c import A +import z + +# Force typing to get refreshed by using a protocol from it +x: Sized = A() + +[file c.py] +class D: + def __len__(self) -> int: return 0 +A = D +[file c.py.2] +class C: + def __len__(self) -> int: return 0 +A = C +[file z.py] +from typing import List +T = List[int] +[file z.py.2] +from typing import List +T = List[int] # yo +[builtins fixtures/list.pyi] +[typing fixtures/typing-medium.pyi] +[out] +== + +[case testNamedTupleRefresh] +from typing import NamedTuple +from a import f +N = NamedTuple('N', [('x', int)]) +[file a.py] +def f() -> None: pass +[file a.py.2] +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Module "a" has no attribute "f" + +[case testModuleLevelAttributeRefresh] +from typing import Callable +from a import f +x = 1 +y = '' # type: str +[file a.py] +def f() -> None: pass +[file a.py.2] +[out] +== +main:2: error: Module "a" has no attribute "f" + +[case testClassBodyRefresh] +from a import f +class A: + x = 1 + y = '' # type: str + + def f(self) -> None: + self.x = 1 +[file a.py] +f = 1 +[file a.py.2] +[out] +== +main:1: error: Module "a" has no attribute "f" + +[case testDecoratedMethodRefresh] +from typing import Iterator, Callable, List, Optional +from a import f +import a + +def dec(f: Callable[['A'], Optional[Iterator[int]]]) -> Callable[[int], int]: pass + +class A: + @dec + def f(self) -> Optional[Iterator[int]]: + self.x = a.g() # type: int + return None +[builtins fixtures/list.pyi] +[file a.py] +f = 1 +def g() -> int: pass +[file a.py.2] +def f() -> None: pass +def g() -> int: pass +[file a.py.3] +def f() -> None: pass +def g() -> str: pass +[out] +== +== +main:10: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testTwoPassTypeChecking] +import a +[file a.py] +[file a.py.2] +class A: + def __init__(self, b: B) -> None: + self.a = b.a + +class B: + def __init__(self) -> None: + self.a = int() +[file a.py.3] +class A: + def __init__(self, b: B) -> None: + self.a = b.a + reveal_type(self.a) # E + +class B: + def __init__(self) -> None: + self.a = int() +[out] +== +== +a.py:4: note: Revealed type is "builtins.int" + +[case testStripRevealType] +import a +reveal_type(a.f()) +[file a.py] +def f() -> int: pass +[file a.py.2] +def f() -> str: pass +[out] +main:2: note: Revealed type is "builtins.int" +== +main:2: note: Revealed type is "builtins.str" + +[case testDecoratorTypeAfterReprocessing] +import a +reveal_type(a.f()) +[file a.py] +from contextlib import contextmanager +from typing import Iterator +import b +@contextmanager +def f() -> Iterator[None]: + yield +[file b.py] +[delete b.py.2] +[file b.py.3] +[typing fixtures/typing-medium.pyi] +[builtins fixtures/list.pyi] +[triggered] +2: , , __main__ +3: , , __main__, a +[out] +main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]" +== +main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]" +a.py:3: error: Cannot find implementation or library stub for module named "b" +a.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]" + +[case testDecoratorSpecialCase1] +import a +[file a.py] +import contextlib +from typing import List, Iterator + +@contextlib.contextmanager +def f(x: List[int]) -> Iterator[None]: + x.append(1) + yield + +def g() -> None: + import b + b.h(1) +[file b.py] +def h() -> None: pass +[delete b.py.2] +[file b.py.3] +def h() -> None: pass +[file a.py.4] +import contextlib +from typing import List, Iterator + +@contextlib.contextmanager +def f(x: List[int]) -> Iterator[None]: + x.append(1) + yield + +def g() -> None: + import b + b.h(1) + pass +[typing fixtures/typing-medium.pyi] +[builtins fixtures/list.pyi] +[triggered] +2: , , , a.g +3: , , , a +4: a.g +[out] +a.py:11: error: Too many arguments for "h" +== +a.py:10: error: Cannot find implementation or library stub for module named "b" +a.py:10: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +a.py:11: error: Too many arguments for "h" +== +a.py:11: error: Too many arguments for "h" + +[case testDecoratorSpecialCase2] +import a +[file a.py] +from contextlib import contextmanager +from typing import Iterator, List +import b + +@contextmanager +def f(x: List[int]) -> Iterator[None]: + x.append(1) + yield +[file b.py] +[delete b.py.2] +[file b.py.3] +[file a.py.4] +from contextlib import contextmanager +from typing import Iterator, List +import b + +@contextmanager +def f(x: List[int]) -> Iterator[None]: + x.append(1) + yield +[typing fixtures/typing-medium.pyi] +[builtins fixtures/list.pyi] +[out] +== +a.py:3: error: Cannot find implementation or library stub for module named "b" +a.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== +== + +[case testDecoratorMethodCompat] +from typing import Callable, List, TypeVar +import x + +class Base: pass +_Base = TypeVar('_Base', bound=Base) + +def dec(f: Callable[[_Base], int]) -> Callable[[_Base], List[int]]: pass + +class B(Base): + def foo(self) -> List[int]: pass + +class A(B): + @dec + def foo(self) -> int: + x.lol() + return 12 + +[file x.py] +def lol() -> str: pass +[file x.py.2] +def lol() -> int: pass +[file x.py.3] +def lol() -> str: pass + +[builtins fixtures/list.pyi] +[out] +== +== + +[case testPreviousErrorInDecoratedFunction] +import a +[file a.py] +from typing import Callable +import b + +def dec(x: Callable[[], None]) -> Callable[[], None]: + return x + +@dec +def f() -> None: + 1 + '' +[file b.py] +[file b.py.2] +1 +[file b.py.3] +2 +[file a.py.4] +from typing import Callable +import b + +def dec(f: Callable[[], None]) -> Callable[[], None]: + return f + +@dec +def f() -> None: + 1 + 2 +[out] +a.py:9: error: Unsupported operand types for + ("int" and "str") +== +a.py:9: error: Unsupported operand types for + ("int" and "str") +== +a.py:9: error: Unsupported operand types for + ("int" and "str") +== + +[case testPreviousErrorInDecoratedMethodOverride] +import a +[file a.py] +from typing import Callable +from b import B + +def dec(x: Callable[['A'], int]) -> Callable[['A'], int]: + return x + +class A(B): + @dec + def foo(self) -> int: return 12 + +[file b.py] +class B: + def foo(self) -> str: return 'hi' +[file c.py.2] +[file c.py.3] +1 +[file b.py.4] +class B: + def foo(self) -> int: return 12 +[out] +a.py:9: error: Return type "int" of "foo" incompatible with return type "str" in supertype "B" +== +a.py:9: error: Return type "int" of "foo" incompatible with return type "str" in supertype "B" +== +a.py:9: error: Return type "int" of "foo" incompatible with return type "str" in supertype "B" +== + +[case testPreviousErrorInMethodSemanal1] +import a +[file a.py] +class A: + def foo() -> int: pass +[file c.py.2] +[file c.py.3] +1 +[file a.py.4] +class A: + def foo(self) -> int: pass +[out] +a.py:2: error: Method must have at least one argument. Did you forget the "self" argument? +== +a.py:2: error: Method must have at least one argument. Did you forget the "self" argument? +== +a.py:2: error: Method must have at least one argument. Did you forget the "self" argument? +== + +[case testPreviousErrorInMethodSemanal2] +import a +[file a.py] +class A: + def foo(self) -> None: + nothing +[file c.py.2] +[file c.py.3] +1 +[file a.py.4] +class A: + def foo(self) -> int: pass +[out] +a.py:3: error: Name "nothing" is not defined +== +a.py:3: error: Name "nothing" is not defined +== +a.py:3: error: Name "nothing" is not defined +== + +[case testPreviousErrorInMethodSemanalPass3] +import a +[file a.py] +from typing import List +class A: + def __init__(self) -> None: + self.x = [] # type: List[int, str] +[file c.py.2] +[file c.py.3] +1 +[file a.py.4] +from typing import List +class A: + def __init__(self) -> None: + self.x = [] # type: List[int] +[builtins fixtures/list.pyi] +[out] +a.py:4: error: "list" expects 1 type argument, but 2 given +== +a.py:4: error: "list" expects 1 type argument, but 2 given +== +a.py:4: error: "list" expects 1 type argument, but 2 given +== + +[case testPreviousErrorInOverloadedFunctionSemanalPass3] +import a +[file a.py] +from typing import overload, List +@overload +def f(x: str) -> None: ... +@overload +def f(x: int) -> List[int, str]: ... +def f(x: object) -> object: + pass +[file c.py.2] +[file c.py.3] +1 +[file a.py.4] +from typing import overload, List +@overload +def f(x: str) -> None: ... +@overload +def f(x: int) -> List[int]: ... +def f(x: object) -> object: + pass +[builtins fixtures/list.pyi] +[out] +a.py:5: error: "list" expects 1 type argument, but 2 given +== +a.py:5: error: "list" expects 1 type argument, but 2 given +== +a.py:5: error: "list" expects 1 type argument, but 2 given +== + +[case testPreviousErrorInOverloadedFunction] +import a +[file a.py] +from typing import overload +@overload +def f(x: str) -> None: ... +@overload +def f(x: int) -> int: ... +def f(x: object) -> None: + pass +[file b.py] +[file b.py.2] +1 +[file b.py.3] +2 +[file a.py.4] +from typing import overload +@overload +def f(x: str) -> None: ... +@overload +def f(x: int) -> None: ... +def f(x: object) -> None: + pass +[out] +a.py:6: error: Overloaded function implementation cannot produce return type of signature 2 +== +a.py:6: error: Overloaded function implementation cannot produce return type of signature 2 +== +a.py:6: error: Overloaded function implementation cannot produce return type of signature 2 +== + +[case testPreviousErrorInOverloadedFunctionSemanal] +import a +[file a.py] +from typing import overload +@overload +def f(x: str) -> None: ... +@overload +def f(x: int) -> None: ... +[file b.py] +[file b.py.2] +1 +[file b.py.3] +2 +[file a.py.4] +from typing import overload +@overload +def f(x: str) -> None: ... +@overload +def f(x: int) -> None: ... +def f(x: object) -> None: + pass +[out] +a.py:2: error: An overloaded function outside a stub file must have an implementation +== +a.py:2: error: An overloaded function outside a stub file must have an implementation +== +a.py:2: error: An overloaded function outside a stub file must have an implementation +== + +[case testPreviousErrorInDecoratedMethodSemanalPass3] +import a +[file a.py] +from typing import Callable, TypeVar, Any, List + +T = TypeVar('T', bound=Callable) +def dec(x: T) -> T: + return x + +@dec +def foo(self) -> List[str, int]: return [] + +[file c.py.2] +[file c.py.3] +[file a.py.4] +from typing import Callable, TypeVar, Any, List + +T = TypeVar('T', bound=Callable[..., Any]) +def dec(x: T) -> T: + return x + +@dec +def foo(self) -> List[str]: return [] +[builtins fixtures/list.pyi] +[out] +a.py:8: error: "list" expects 1 type argument, but 2 given +== +a.py:8: error: "list" expects 1 type argument, but 2 given +== +a.py:8: error: "list" expects 1 type argument, but 2 given +== + +[case testDecoratorUpdateMod] +import a +[file a.py] +import mod + +@mod.deca +@mod.decb(mod.C()) +def func(x: mod.B) -> mod.B: + x.x + return x +[file mod.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[B], B]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + x: int +[file mod.py.2] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[str], str]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + x: int +[file mod.py.3] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[B], B]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + y: int +[file mod.py.4] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[B], B]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + def __init__(self, x: int) -> None: + pass +class B: + x: int +[out] +== +a.py:3: error: Argument 1 to "deca" has incompatible type "Callable[[B], B]"; expected "Callable[[str], str]" +== +a.py:6: error: "B" has no attribute "x" +== +a.py:4: error: Missing positional argument "x" in call to "C" + +[case testDecoratorUpdateFunc] +import a +[file a.py] +import mod + +def outer() -> None: + @mod.deca + @mod.decb(mod.C()) + def func(x: mod.B) -> mod.B: + x.x + return x +[file mod.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[B], B]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + x: int +[file mod.py.2] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[str], str]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + x: int +[file mod.py.3] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[B], B]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + y: int +[file mod.py.4] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[[B], B]) -> Callable[[str], str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + def __init__(self, x: int) -> None: + pass +class B: + x: int +[out] +== +a.py:4: error: Argument 1 to "deca" has incompatible type "Callable[[B], B]"; expected "Callable[[str], str]" +== +a.py:7: error: "B" has no attribute "x" +== +a.py:5: error: Missing positional argument "x" in call to "C" + +[case DecoratorUpdateMethod] +import a +[file a.py] +import mod + +class D: + @mod.deca + @mod.decb(mod.C()) + def func(self, x: mod.B) -> mod.B: + x.x + return x +[file mod.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[..., B]) -> Callable[..., str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + x: int +[file mod.py.2] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[..., str]) -> Callable[..., str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + x: int +[file mod.py.3] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[..., B]) -> Callable[..., str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + pass +class B: + y: int +[file mod.py.4] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) + +def deca(func: Callable[..., B]) -> Callable[..., str]: + pass +def decb(arg: C) -> Callable[[F], F]: + pass +class C: + def __init__(self, x: int) -> None: + pass +class B: + x: int +[out] +== +a.py:4: error: Argument 1 to "deca" has incompatible type "Callable[[D, B], B]"; expected "Callable[..., str]" +== +a.py:7: error: "B" has no attribute "x" +== +a.py:5: error: Missing positional argument "x" in call to "C" + +[case testDecoratorUpdateDeeepNested] +import a +[file a.py] +import mod + +def outer() -> None: + def inner() -> None: + @mod.dec + def func(x: int) -> int: + pass +[file mod.py] +from typing import Callable +def dec(func: Callable[[int], int]) -> Callable[[str], str]: + pass +[file mod.py.2] +from typing import Callable +def dec(func: Callable[[str], str]) -> Callable[[str], str]: + pass +[out] +== +a.py:5: error: Argument 1 to "dec" has incompatible type "Callable[[int], int]"; expected "Callable[[str], str]" + +[case testDecoratorUpdateNestedClass] +import a +[file a.py] +import mod + +class Outer: + class Inner: + c = mod.C() + @c.dec + def func(self, x: int) -> int: + pass +[file mod.py] +from typing import Callable +class C: + def dec(self, func: Callable[..., int]) -> Callable[..., str]: + pass +[file mod.py.2] +from typing import Callable +class C: + def dec(self, func: Callable[..., str]) -> Callable[..., str]: + pass +[out] +== +a.py:6: error: Argument 1 to "dec" of "C" has incompatible type "Callable[[Inner, int], int]"; expected "Callable[..., str]" + +[case testDecoratorUpdateClassInFunction] +import a +[file a.py] +import mod + +def outer() -> None: + class Inner: + c = mod.C() + @c.dec + def func(self, x: mod.B) -> int: + return x.x +[file mod.py] +from typing import Callable +class C: + def dec(self, func: Callable[..., int]) -> Callable[..., str]: + pass +class B: + x: int +[file mod.py.2] +from typing import Callable +class C: + def dec(self, func: Callable[..., str]) -> Callable[..., str]: + pass +class B: + x: int +[file mod.py.3] +from typing import Callable +class C: + def dec(self, func: Callable[..., int]) -> Callable[..., str]: + pass +class B: + x: str +[out] +== +a.py:6: error: Argument 1 to "dec" of "C" has incompatible type "Callable[[Inner, B], int]"; expected "Callable[..., str]" +== +a.py:8: error: Incompatible return value type (got "str", expected "int") + +[case testDecoratorUpdateMROUpdated] +import a +[file a.py] +import mod + +@mod.dec +def func(x: mod.B) -> int: + pass +[file mod.py] +from typing import Callable +class B: + pass +class C(B): + pass +def dec(f: Callable[[C], int]) -> Callable[[int], int]: + pass +[file mod.py.2] +from typing import Callable +class B: + pass +class C: + pass +def dec(f: Callable[[C], int]) -> Callable[[int], int]: + pass +[out] +== +a.py:3: error: Argument 1 to "dec" has incompatible type "Callable[[B], int]"; expected "Callable[[C], int]" + +[case testOverloadRefresh] +from typing import overload +import m + +@overload +def f(x: m.A) -> None: ... +@overload +def f(x: int) -> None: ... +def f(x: object) -> None: + from n import g +[file m.py] +class A: pass +[file n.py] +def g() -> None: pass +[delete m.py.2] +[delete n.py.2] +[out] +== +main:2: error: Cannot find implementation or library stub for module named "m" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:7: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +main:9: error: Cannot find implementation or library stub for module named "n" + +[case testOverloadSpecialCase] +from typing import overload +import m +import sys + +class C: + if sys.platform == 'nonexistent': + def f(self, x): pass + else: + @overload + def f(self, x: m.A) -> None: pass + @overload + def f(self, x: int) -> None: pass + def f(self, x: object) -> None: + from n import g +[file m.py] +class A: pass +[file n.py] +def g() -> None: pass +[delete m.py.2] +[delete n.py.2] +[builtins fixtures/ops.pyi] +[out] +== +main:2: error: Cannot find implementation or library stub for module named "m" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:12: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +main:14: error: Cannot find implementation or library stub for module named "n" + +[case testOverloadClassmethodDisappears] +from typing import overload +from m import Wrapper +reveal_type(Wrapper.foo(3)) +[file m.pyi] +from typing import overload +class Wrapper: + @overload + @classmethod + def foo(self, x: int) -> int: ... + @overload + @classmethod + def foo(self, x: str) -> str: ... +[file m.pyi.2] +from typing import overload +class Wrapper: + @overload + def foo(cls, x: int) -> int: ... + @overload + def foo(cls, x: str) -> str: ... +[builtins fixtures/classmethod.pyi] +[out] +main:3: note: Revealed type is "builtins.int" +== +main:3: error: No overload variant of "foo" of "Wrapper" matches argument type "int" +main:3: note: Possible overload variants: +main:3: note: def foo(cls: Wrapper, x: int) -> int +main:3: note: def foo(cls: Wrapper, x: str) -> str +main:3: note: Revealed type is "Any" + +[case testRefreshGenericClass] +from typing import TypeVar, Generic +from a import A + +X = TypeVar('X') + +class C(Generic[X]): + def f(self, x: A) -> X: ... +[file a.py] +class A: pass +[file a.py.2] +[file a.py.3] +class A: pass +[out] +== +main:2: error: Module "a" has no attribute "A" +== + +[case testRefreshGenericAndFailInPass3] +# Failure in semantic analysis pass 3 +from a import C +a: C[int] +[file a.py] +from typing import TypeVar, Generic +T = TypeVar('T') +class C(Generic[T]): pass +[file a.py.2] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +class C(Generic[T, S]): pass +[file a.py.3] +from typing import TypeVar, Generic +T = TypeVar('T') +class C(Generic[T]): pass +[out] +== +main:3: error: "C" expects 2 type arguments, but 1 given +== + +[case testUnannotatedClass] +import a +[file a.py] +class A: + def f(self, x): + self.y = x + self.g() + + def g(self): pass +[file a.py.2] +class A: + def f(self, x, y): + self.y = x + self.z = y + self.g() + + def g(self): pass +[triggered] +2: , , +[out] +== + +[case testSuperBasics] +import a +[file a.py] +class A: + def f(self) -> None: pass +class B(A): + def f(self) -> None: + super(B, self).f() +[file a.py.2] +class A: + def f(self) -> None: pass +class B(A): + def f(self) -> None: + super(B, self).f() +[out] +== + +[case testErrorInTypeCheckSecondPassThroughPropagation] +import a + +def f() -> None: + x = a.C() +[file a.py] +[file a.py.2] +from typing import Generic, TypeVar +T = TypeVar('T') +class C(Generic[T]): pass +[out] +main:4: error: "object" has no attribute "C" +== +main:4: error: Need type annotation for "x" + +[case testPartialTypeInNestedClass] +import a +class C: + def f(self) -> None: + a.g() + class D: + def __init__(self) -> None: + self.x = {} + def meth(self) -> None: + self.x['a'] = 'b' +[file a.py] +def g() -> None: pass +[file a.py.2] +def g() -> int: pass +[builtins fixtures/dict.pyi] +[out] +main:7: error: Need type annotation for "x" (hint: "x: Dict[, ] = ...") +== +main:7: error: Need type annotation for "x" (hint: "x: Dict[, ] = ...") + +[case testRefreshPartialTypeInClass] +import a +class D: + def __init__(self) -> None: + a.g() + self.x = {} + def meth(self) -> None: + self.x['a'] = 'b' +[file a.py] +def g() -> None: pass +[file a.py.2] +def g() -> int: pass +[builtins fixtures/dict.pyi] +[out] +main:5: error: Need type annotation for "x" (hint: "x: Dict[, ] = ...") +== +main:5: error: Need type annotation for "x" (hint: "x: Dict[, ] = ...") + +[case testRefreshPartialTypeInferredAttributeIndex] +from c import C +reveal_type(C().a) +[file c.py] +from b import f +class C: + def __init__(self) -> None: + self.a = {} + if bool(): + self.a[0] = f() +[file b.py] +def f() -> int: ... +[file b.py.2] +from typing import List +def f() -> str: ... +[builtins fixtures/dict.pyi] +[out] +main:2: note: Revealed type is "builtins.dict[builtins.int, builtins.int]" +== +main:2: note: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +[case testRefreshPartialTypeInferredAttributeAssign] +from c import C +reveal_type(C().a) +[file c.py] +from b import f +class C: + def __init__(self) -> None: + self.a = [] + if bool(): + self.a = f() +[file b.py] +from typing import List +def f() -> List[int]: ... +[file b.py.2] +from typing import List +def f() -> List[str]: ... +[builtins fixtures/list.pyi] +[out] +main:2: note: Revealed type is "builtins.list[builtins.int]" +== +main:2: note: Revealed type is "builtins.list[builtins.str]" + +[case testRefreshPartialTypeInferredAttributeAppend] +from c import C +reveal_type(C().a) +[file c.py] +from b import f +class C: + def __init__(self) -> None: + self.a = [] + if bool(): + self.a.append(f()) +[file b.py] +def f() -> int: ... +[file b.py.2] +def f() -> str: ... +[builtins fixtures/list.pyi] +[out] +main:2: note: Revealed type is "builtins.list[builtins.int]" +== +main:2: note: Revealed type is "builtins.list[builtins.str]" + +[case testRefreshTryExcept] +import a +def f() -> None: + a.g() + try: + pass + except BaseException as e: + e +[file a.py] +def g() -> int: pass +[file a.py.2] +def g() -> str: pass +[builtins fixtures/exception.pyi] +[out] +== + +[case testMroSpecialCase] +import b +import a + +[file a.py] +class C: pass +class D(C): + 1() +class E(D): pass + +[file b.py] +import a + +[file a.py.2] +class C: pass +class D(C): + 1() +class E(D): pass +# Something needs to change + +[file b.py.2] +import a +# Something needs to change + +[triggered] +2: a, a +[out] +a.py:3: error: "int" not callable +== +a.py:3: error: "int" not callable + +[case testMetaclassAttributes] +import a +[file a.py] +from mod import C +from typing import Type +def f(arg: Type[C]) -> None: + arg.x = int() +[file mod.py] +import submod +class C(metaclass=submod.M): + pass +[file submod.py] +class M(type): + x: int +[file submod.py.2] +class M(type): + x: str +[file submod.py.3] +class M(type): + y: str +[file submod.py.4] +class M(type): + x: int +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "int", variable has type "str") +== +a.py:4: error: "Type[C]" has no attribute "x" +== + +[case testMetaclassAttributesDirect] +import a +[file a.py] +from mod import C +def f() -> None: + C.x = int() +[file mod.py] +import submod +class C(metaclass=submod.M): + pass +[file submod.py] +class M(type): + x: int +[file submod.py.2] +class M(type): + x: str +[file submod.py.3] +class M(type): + y: str +[file submod.py.4] +class M(type): + x: int +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") +== +a.py:3: error: "Type[C]" has no attribute "x" +== + +[case testMetaclassOperators] +import a +[file a.py] +from mod import C +from typing import Type +def f(arg: Type[C]) -> None: + arg + arg +[file mod.py] +import submod +class C(metaclass=submod.M): + pass +[file submod.py] +class M(type): + def __add__(self, other: M) -> M: + pass +[file submod.py.2] +class M(type): + def __add__(self, other: int) -> M: + pass +[out] +== +a.py:4: error: Unsupported operand types for + ("Type[C]" and "Type[C]") + +[case testMetaclassOperatorsDirect] +import a +[file a.py] +from mod import C +def f() -> None: + C + C +[file mod.py] +import submod +class C(metaclass=submod.M): + pass +[file submod.py] +class M(type): + def __add__(self, other: int) -> M: + pass +[file submod.py.2] +class M(type): + def __add__(self, other: M) -> M: + pass +[out] +a.py:3: error: Unsupported operand types for + ("Type[C]" and "Type[C]") +== + +[case testFineMetaclassUpdate] +import a +[file a.py] +from c import M +import b +def f(arg: M) -> None: + pass + +f(b.B) +[file b.py] +import c +class B: pass + +[file b.py.2] +import c +class B(metaclass=c.M): pass + +[file c.py] +class M(type): + pass +[out] +a.py:6: error: Argument 1 to "f" has incompatible type "Type[B]"; expected "M" +== + +[case testFineMetaclassRecalculation] +import a +[file a.py] +from b import B +class M2(type): pass +class D(B, metaclass=M2): pass +[file b.py] +import c +class B: pass + +[file b.py.2] +import c +class B(metaclass=c.M): pass + +[file c.py] +class M(type): + pass +[out] +== +a.py:3: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +[case testFineMetaclassDeclaredUpdate] +import a +[file a.py] +import b +class B(metaclass=b.M): pass +class D(B, metaclass=b.M2): pass +[file b.py] +class M(type): pass +class M2(M): pass +[file b.py.2] +class M(type): pass +class M2(type): pass +[out] +== +a.py:3: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +[case testFineMetaclassRemoveFromClass] +import a +[file a.py] +import b +def func() -> int: + return b.B.x +[file b.py] +from c import M +class B(metaclass=M): + pass +[file b.py.2] +from c import M +class B: + pass +[file c.py] +class M(type): + x: int +[out] +== +a.py:3: error: "Type[B]" has no attribute "x" + +[case testFineMetaclassRemoveFromClass2] +import a +[file a.py] +import b +def func() -> None: + b.test(b.B) +[file b.py] +import c +def test(cls: c.M) -> None: + pass +class B(metaclass=c.M): + pass +[file b.py.2] +import c +def test(cls: c.M) -> None: + pass +class B: + pass +[file c.py] +class M(type): + x: int +[out] +== +a.py:3: error: Argument 1 to "test" has incompatible type "Type[B]"; expected "M" + +[case testBadMetaclassCorrected] +import a +[file a.py] +import b +class C(metaclass=b.M): + pass +[file b.py] +from c import M +[file c.py] +M = 1 +[file c.py.2] +class M(type): + pass +[out] +a.py:2: error: Invalid metaclass "b.M" +== + +[case testFixedAttrOnAddedMetaclass] +import a +[file a.py] +import b +def fun() -> None: + x: int = b.C.x +[file b.py] +import c +class C: + pass +[file b.py.2] +import c +class C(metaclass=c.M): + pass +[file c.py] +class M(type): + x: int +[out] +a.py:3: error: "Type[C]" has no attribute "x" +== + +[case testIndirectSubclassReferenceMetaclass] +import a +[file a.py] +import b +def f() -> None: + b.x = int() +[file b.py] +import bb +x = bb.D.x +[file bb.py] +import mod +class D(mod.C): + pass +[file mod.py] +import submod +class C(metaclass=submod.M): + pass +[file submod.py] +class M(type): + x: int +[file submod.py.2] +class M(type): + x: str +[file submod.py.3] +class M(type): + y: str +[file submod.py.4] +class M(type): + x: int +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") +== +b.py:2: error: "Type[D]" has no attribute "x" +== + +[case testMetaclassDeletion] +import a +[file a.py] +import b +def func() -> None: + b.B.x +[file b.py] +import c +class B(metaclass=c.M): + pass +[file c.py] +class M(type): + x: int +[file c.py.2] +whatever: int +[out] +== +b.py:2: error: Name "c.M" is not defined + +[case testFixMissingMetaclass] +import a +[file a.py] +import b +def func() -> None: + b.B.x +[file b.py] +import c +class B(metaclass=c.M): + pass +[file c.py] +whatever: int +[file c.py.2] +class M(type): + x: int +[out] +b.py:2: error: Name "c.M" is not defined +== + +[case testGoodMetaclassSpoiled] +import a +[file a.py] +import b +class C(metaclass=b.M): + pass +[file b.py] +class M(type): + pass +[file b.py.2] +M = 1 +[out] +== +a.py:2: error: Invalid metaclass "b.M" + +[case testRefreshGenericSubclass] +from typing import Generic, TypeVar +import m +m.x + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, x: T) -> None: + pass + +class D(C[T]): + def __init__(self, x: T) -> None: + m.x + super(D, self).__init__(x) +[file m.py] +x = 0 +[file m.py.2] +x = '' +[out] +== + +[case testRefreshNamedTupleSubclass] +from typing import NamedTuple +import m +m.x + +N = NamedTuple('N', [('x', int)]) + +class C(N): + pass +[file m.py] +x = 0 +[file m.py.2] +x = '' +[builtins fixtures/tuple.pyi] +[out] +== + +[case testNewTypeRefresh] +import a + +[file a.py] +from typing import Dict, NewType + +class A: pass +N = NewType('N', A) + +a: Dict[N, int] + +def f(self, x: N) -> None: + a.get(x) + +[file a.py.2] +from typing import Dict, NewType # dummy change + +class A: pass +N = NewType('N', A) + +a: Dict[N, int] + +def f(self, x: N) -> None: + a.get(x) + +[builtins fixtures/dict.pyi] +[out] +== + +[case testRefreshFunctionalEnum] +import a + +[file a.py] +from typing import Dict +from enum import Enum + +N = Enum('N', 'x') +a: Dict[N, int] + +def f(self, x: N) -> None: + a.get(x) + +[file a.py.2] +from typing import Dict +from enum import Enum + +N = Enum('N', 'x') +a: Dict[N, int] + +def f(self, x: N) -> None: + a.get(x) +[builtins fixtures/dict.pyi] +[out] +== + +[case testFineGrainedCallable] +import a +[file a.py] +def f(o: object) -> None: + if callable(o): + o() +[file a.py.2] +def f(o: object) -> None: + if callable(o): + o() +[builtins fixtures/callable.pyi] +[out] +== + +[case testRefreshFunctionalNamedTuple] +import a + +[file a.py] +from typing import NamedTuple +from b import L + +A = NamedTuple('A', []) +a: A + +def g() -> None: + x = L(A()) + x.f(a) + +[file b.pyi] +from typing import TypeVar, Generic, overload + +T = TypeVar('T') + +class L(Generic[T]): + def __init__(self, x: T) -> None: pass + @overload + def f(self) -> None: pass + @overload + def f(self, a: T) -> None: pass + +[file a.py.2] +from typing import NamedTuple +from b import L + +A = NamedTuple('A', []) +a: A + +def g() -> None: + x = L(A()) + x.f(a) +[builtins fixtures/tuple.pyi] +[out] +== + +[case testRefreshSubclassNestedInFunction1] +from a import C +def f() -> None: + class D(C): pass +[file a.py] +class C: pass +[file a.py.2] +[out] +== +main:1: error: Module "a" has no attribute "C" + +[case testRefreshSubclassNestedInFunction2] +from a import C +def f() -> None: + class D(C): + def g(self) -> None: + super().__init__() + d = D() +[file a.py] +class C: + def __init__(self) -> None: pass +[file a.py.2] +class C: + def __init__(self, x: int) -> None: pass +[out] +== +main:5: error: Missing positional argument "x" in call to "__init__" of "C" +main:6: error: Missing positional argument "x" in call to "D" + +[case testInferAttributeTypeAndMultipleStaleTargets] +import a + +class A: + def g(self) -> None: + a.x + self.x = 1 + + def f(self) -> None: + a.x + b = self.x + self.x = 1 + +[file a.py] +x = 0 +[file a.py.2] +x = '' +[out] +== + +[case testNamedTupleUpdate] +import b +[file a.py] +from typing import NamedTuple +N = NamedTuple('N', [('x', int)]) +x = N(1) +[file a.py.2] +from typing import NamedTuple +N = NamedTuple('N', [('x', str)]) +x = N('hi') +[file b.py] +import a +def f(x: a.N) -> None: + pass +f(a.x) +[builtins fixtures/tuple.pyi] +[out] +== + +[case testNamedTupleUpdate2] +import b +[file a.py] +from typing import NamedTuple +N = NamedTuple('N', [('x', int)]) +x = N(1) +[file a.py.2] +from typing import NamedTuple +N = NamedTuple('N', [('y', int)]) +x = N(2) +[file b.py] +import a +def f(x: a.N) -> None: + pass +f(a.x) +[builtins fixtures/tuple.pyi] +[out] +== + +[case testNamedTupleUpdate3] +import c +[file a.py] +from typing import NamedTuple +N = NamedTuple('N', [('x', int)]) +x = N(1) +[file a.py.2] +from typing import NamedTuple +N = NamedTuple('N', [('x', str)]) +x = N('hi') +[file b.py] +import a +from typing import NamedTuple +M = NamedTuple('M', [('z', 'a.N')]) +x = M(a.x) +[file c.py] +import a +import b +from typing import Tuple +def lol(n: Tuple[Tuple[int]]) -> None: + pass +def f(x: b.M) -> None: + lol(x) +f(b.x) +lol(b.x) +[builtins fixtures/tuple.pyi] +[out] +== +c.py:7: error: Argument 1 to "lol" has incompatible type "M"; expected "Tuple[Tuple[int]]" +c.py:9: error: Argument 1 to "lol" has incompatible type "M"; expected "Tuple[Tuple[int]]" + +[case testNamedTupleUpdate4] +import b +[file a.py] +from typing import NamedTuple +class N(NamedTuple): + x: int +x = N(1) +[file a.py.2] +from typing import NamedTuple +class N(NamedTuple): + x: str +x = N('hi') +[file b.py] +import a +def f(x: a.N) -> None: + pass +f(a.x) +[builtins fixtures/tuple.pyi] +[out] +== + +[case testNamedTupleUpdate5] +import b +[file a.py] +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[N] + x: int +x = N(None, 1) +[file a.py.2] +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[N] + x: str +x = N(None, 'hi') +[file b.py] +import a +def f(x: a.N) -> None: + pass +f(a.x) +[builtins fixtures/tuple.pyi] +[out] +== + +[case testNamedTupleUpdateGeneric] +import b +[file a.py] +from typing import NamedTuple +class Point(NamedTuple): + x: int + y: int +[file a.py.2] +from typing import Generic, TypeVar, NamedTuple + +T = TypeVar("T") +class Point(NamedTuple, Generic[T]): + x: int + y: T +[file b.py] +from a import Point +def foo() -> None: + p = Point(x=0, y=1) + i: int = p.y +[file b.py.3] +from a import Point +def foo() -> None: + p = Point(x=0, y="no") + i: int = p.y +[builtins fixtures/tuple.pyi] +[out] +== +== +b.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleUpdateNonRecursiveToRecursiveFine] +import c +[file a.py] +from b import M +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[M] + x: int +n: N +[file b.py] +from a import N +from typing import NamedTuple +class M(NamedTuple): + r: None + x: int +[file b.py.2] +from a import N +from typing import NamedTuple, Optional +class M(NamedTuple): + r: Optional[N] + x: int +[file c.py] +import a +def f(x: a.N) -> None: + if x.r is not None: + s: int = x.r.x +[file c.py.3] +import a +def f(x: a.N) -> None: + if x.r is not None and x.r.r is not None and x.r.r.r is not None: + reveal_type(x) + s: int = x.r.r.r.r +f(a.n) +reveal_type(a.n) +[builtins fixtures/tuple.pyi] +[out] +== +== +c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") +c.py:7: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" + +[case testTupleTypeUpdateNonRecursiveToRecursiveFine] +import c +[file a.py] +from b import M +from typing import Tuple, Optional +class N(Tuple[Optional[M], int]): ... +[file b.py] +from a import N +from typing import Tuple +class M(Tuple[None, int]): ... +[file b.py.2] +from a import N +from typing import Tuple, Optional +class M(Tuple[Optional[N], int]): ... +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +== +== +c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTypeAliasUpdateNonRecursiveToRecursiveFine] +import c +[file a.py] +from b import M +from typing import Tuple, Optional +N = Tuple[Optional[M], int] +[file b.py] +from a import N +from typing import Tuple +M = Tuple[None, int] +[file b.py.2] +from a import N +from typing import Tuple, Optional +M = Tuple[Optional[N], int] +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +== +== +c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" +c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTypedDictRefresh] +[builtins fixtures/dict.pyi] +import a +[file a.py] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) +[file a.py.2] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) # dummy change +[out] +== + +[case testTypedDictUpdate] +import b +[file a.py] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(dict(x=42, y=1337)) +[file a.py.2] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': str}) +p = Point(dict(x=42, y='lurr')) +[file b.py] +from a import Point +def foo(x: Point) -> int: + return x['x'] + x['y'] +[builtins fixtures/dict.pyi] +[out] +== +b.py:3: error: Unsupported operand types for + ("int" and "str") + +[case testTypedDictUpdate2] +import b +[file a.py] +from mypy_extensions import TypedDict +class Point(TypedDict): + x: int + y: int +p = Point(dict(x=42, y=1337)) +[file a.py.2] +from mypy_extensions import TypedDict +class Point(TypedDict): + x: int + y: str +p = Point(dict(x=42, y='lurr')) +[file b.py] +from a import Point +def foo(x: Point) -> int: + return x['x'] + x['y'] +[builtins fixtures/dict.pyi] +[out] +== +b.py:3: error: Unsupported operand types for + ("int" and "str") + +[case testTypedDictUpdate3] +import b +[file a.py] +from mypy_extensions import TypedDict +from typing import Optional +class Point(TypedDict): + x: Optional[Point] + y: int + z: int +p = Point(dict(x=None, y=1337, z=0)) +[file a.py.2] +from mypy_extensions import TypedDict +from typing import Optional +class Point(TypedDict): + x: Optional[Point] + y: str + z: int +p = Point(dict(x=None, y='lurr', z=0)) +[file b.py] +from a import Point +def foo(x: Point) -> int: + assert x['x'] is not None + return x['x']['z'] + x['x']['y'] +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: Unsupported operand types for + ("int" and "str") + +[case testTypedDictUpdateGeneric] +import b +[file a.py] +from mypy_extensions import TypedDict +class Point(TypedDict): + x: int + y: int +[file a.py.2] +from mypy_extensions import TypedDict +from typing import Generic, TypeVar + +T = TypeVar("T") +class Point(TypedDict, Generic[T]): + x: int + y: T +[file b.py] +from a import Point +def foo() -> None: + p = Point(x=0, y=1) + i: int = p["y"] +[file b.py.3] +from a import Point +def foo() -> None: + p = Point(x=0, y="no") + i: int = p["y"] +[builtins fixtures/dict.pyi] +[out] +== +== +b.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testTypedDictUpdateReadOnly] +import b +[file a.py] +from typing_extensions import TypedDict, ReadOnly +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(x=1, y=2) +[file a.py.2] +from typing_extensions import TypedDict, ReadOnly +class Point(TypedDict): + x: int + y: ReadOnly[int] +p = Point(x=1, y=2) +[file a.py.3] +from typing_extensions import TypedDict, ReadOnly +Point = TypedDict('Point', {'x': ReadOnly[int], 'y': int}) +p = Point(x=1, y=2) +[file b.py] +from a import Point +def foo(x: Point) -> None: + x['x'] = 1 + x['y'] = 2 +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: ReadOnly TypedDict key "y" TypedDict is mutated +== +b.py:3: error: ReadOnly TypedDict key "x" TypedDict is mutated + +[case testBasicAliasUpdate] +import b +[file a.py] +N = int +x = 1 +[file a.py.2] +N = str +x = 'hi' +[file b.py] +import a +def f(x: a.N) -> None: + pass +f(a.x) +[out] +== + +[case testBasicAliasUpdateGeneric] +import b +[file a.py] +from typing import Dict, TypeVar +T = TypeVar('T') +D = Dict[int, T] +x = {1: 1} +[file a.py.2] +from typing import Dict, TypeVar +T = TypeVar('T') +D = Dict[str, T] +x = {'hi': 1} +[file b.py] +import a +def f(x: a.D[int]) -> None: + pass +f(a.x) +[builtins fixtures/dict.pyi] +[out] +== + +[case testAliasFineNormalMod] +import b +[file a.py] +A = int +[file a.py.2] +A = str +[file b.py] +import a +x: a.A = int() +[out] +== +b.py:2: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineNormalFunc] +import b +[file a.py] +A = int +[file a.py.2] +A = str +[file b.py] +import a +def f(x: a.A): + if int(): + x = int() +[out] +== +b.py:4: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineNormalClass] +import b +[file a.py] +A = int +[file a.py.2] +A = str +[file b.py] +import a +class C: + x: a.A +c = C() +c.x = int() +[out] +== +b.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineNormalClassBases] +import b +[file a.py] +import c +A = c.BaseI +[file a.py.2] +import c +A = c.BaseS +[file b.py] +import a +class C(a.A): + x = int() +[file c.py] +class BaseI: + x: int +class BaseS: + x: str +[out] +== +b.py:3: error: Incompatible types in assignment (expression has type "int", base class "BaseS" defined the type as "str") + +[case testAliasFineGenericMod] +import b +[file a.py] +from typing import Dict +A = Dict[str, int] +[file a.py.2] +from typing import Dict +A = Dict[str, str] +[file b.py] +import a +x: a.A = {str(): int()} +[builtins fixtures/dict.pyi] +[out] +== +b.py:2: error: Dict entry 0 has incompatible type "str": "int"; expected "str": "str" + +[case testAliasFineGenericFunc] +import b +[file a.py] +from typing import Dict +A = Dict[str, int] +[file a.py.2] +from typing import Dict +A = Dict[str, str] +[file b.py] +import a +def f(x: a.A): + pass +f({str(): int()}) +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: Dict entry 0 has incompatible type "str": "int"; expected "str": "str" + +[case testAliasFineForwardMod] +import b +[file b.py] +x: A = int() +A = int +[file b.py.2] +x: A = int() +A = str +[out] +== +b.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineForwardFunc] +import b +[file b.py] +def f(x: A): + x = int() +A = int +[file b.py.2] +def f(x: A): + if int(): + x = int() +A = str +[out] +== +b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineChainedFunc] +import b +[file a.py] +A = int +[file a.py.2] +A = str +[file aa.py] +import a +B = a.A +[file b.py] +import aa +def f(x: aa.B): + if int(): + x = int() +[out] +== +b.py:4: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineChainedClass] +import b +[file a.py] +A = int +[file a.py.2] +A = str +[file aa.py] +import a +B = a.A +[file b.py] +import aa +class C: + x: aa.B +c = C() +c.x = int() +[out] +== +b.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineNestedMod] +import b +[file a.py] +from typing import Dict +A = Dict[str, int] +[file a.py.2] +from typing import Dict +A = Dict[str, str] +[file aa.py] +from typing import Dict +import a +B = Dict[str, a.A] +[file b.py] +import aa + +x: aa.B = {'first': {str(): int()}} +[builtins fixtures/dict.pyi] +[out] +== +b.py:3: error: Dict entry 0 has incompatible type "str": "int"; expected "str": "str" + +[case testAliasFineNestedFunc] +import b +[file a.py] +from typing import Dict +A = Dict[str, int] +[file a.py.2] +from typing import Dict +A = Dict[str, str] +[file aa.py] +from typing import Dict +import a +B = Dict[str, a.A] +[file b.py] +import aa +def f(x: aa.B): + if int(): + x = {'first': {str(): int()}} +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: Dict entry 0 has incompatible type "str": "int"; expected "str": "str" + +[case testAliasFineNestedFuncDirect] +import b +[file a.py] +from typing import Dict +A = Dict[str, int] +[file a.py.2] +from typing import Dict +A = Dict[str, str] +[file aa.py] +from typing import Dict +import a +E = Dict +[file b.py] +import aa +def f(x: aa.E[str, aa.a.A]): + if int(): + x = {'first': {str(): int()}} +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: Dict entry 0 has incompatible type "str": "int"; expected "str": "str" + +[case testAliasFineNonGenericToGeneric] +import b +[file a.py] +from typing import Dict, TypeVar +T = TypeVar('T') +A = Dict[T, int] +[file a.py.2] +A = str +[file b.py] +import a +def f(x: a.A[str]): + pass +[builtins fixtures/dict.pyi] +[out] +== +b.py:2: error: "str" expects no type arguments, but 1 given + +[case testAliasFineGenericToNonGeneric] +import b +[file a.py] +A = str +[file a.py.2] +from typing import Dict, TypeVar +T = TypeVar('T') +A = Dict[T, int] +[file b.py] +import a +def f(x: a.A): + pass +reveal_type(f) +[builtins fixtures/dict.pyi] +[out] +b.py:4: note: Revealed type is "def (x: builtins.str) -> Any" +== +b.py:4: note: Revealed type is "def (x: builtins.dict[Any, builtins.int]) -> Any" + +[case testAliasFineChangedNumberOfTypeVars] +import b +[file a.py] +from typing import Dict, TypeVar +T = TypeVar('T') +A = Dict[T, int] +[file a.py.2] +from typing import Dict, TypeVar +T = TypeVar('T') +S = TypeVar('S') +A = Dict[T, S] +[file b.py] +import a +def f(x: a.A[str]): + pass +[builtins fixtures/dict.pyi] +[out] +== +b.py:2: error: Bad number of arguments for type alias, expected 2, given 1 + +[case testAliasFineAdded] +import b +[file a.py] +[file a.py.2] +A = int +[file b.py] +import a +x: a.A +[out] +b.py:2: error: Name "a.A" is not defined +== + +[case testAliasFineDeleted] +import b +[file a.py] +A = int +[file a.py.2] +[file b.py] +import a +x: a.A +[out] +== +b.py:2: error: Name "a.A" is not defined + +[case testAliasFineClassToAlias] +import b +[file a.py] +class A: pass +[file a.py.2] +A = int +[file b.py] +import a +x: a.A +x = 1 +[out] +b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "A") +== + +[case testAliasFineAliasToClass] +import b +[file a.py] +A = int +[file a.py.2] +class A: pass +[file b.py] +import a +x: a.A +x = 1 +[out] +== +b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "A") + +[case testAliasFineComponentDeleted] +import b +[file a.py] +class B: pass +[file a.py.2] +x = 1 +[file b.py] +import a +from typing import Dict, TypeVar +T = TypeVar('T') +A = Dict[T, a.B] +def f(x: A[int]): + pass +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: Name "a.B" is not defined + +[case testAliasFineTargetDeleted] +import c +[file a.py] +A = int +[file b.py] +import a +B = a.A +[file b.py.2] +x = 1 +[file c.py] +import b +def f(x: b.B): + pass +[out] +== +c.py:2: error: Name "b.B" is not defined + +[case testAliasFineClassInFunction] +import b +[file a.py] +A = int +[file a.py.2] +A = str +[file b.py] +import a +def f() -> None: + class C: + x: a.A = int() +[out] +== +b.py:4: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAliasFineInitNormalMod] +import c +[file a.py] +class A: + def __init__(self, x: int) -> None: + pass +[file a.py.2] +class A: + def __init__(self, x: str) -> None: + pass +[file b.py] +import a +B = a.A +[file c.py] +from b import B +B(int()) +[out] +== +c.py:2: error: Argument 1 to "A" has incompatible type "int"; expected "str" + +[case testAliasFineInitNormalFunc] +import c +[file a.py] +class A: + def __init__(self, x: int) -> None: + pass +[file a.py.2] +class A: + def __init__(self, x: str) -> None: + pass +[file b.py] +import a +B = a.A +[file c.py] +from b import B +def f() -> None: + B(int()) +[out] +== +c.py:3: error: Argument 1 to "A" has incompatible type "int"; expected "str" + +[case testAliasFineInitGenericMod] +import c +[file a.py] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T, S]): + def __init__(self, x: T) -> None: + pass +[file a.py.2] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T, S]): + def __init__(self, x: S) -> None: + pass +[file b.py] +import a +B = a.A[int, str] +[file c.py] +from b import B +B(int()) +[out] +== +c.py:2: error: Argument 1 to "A" has incompatible type "int"; expected "str" + +[case testAliasFineInitGenericFunc] +import c +[file a.py] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T, S]): + def __init__(self, x: T) -> None: + pass +[file a.py.2] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T, S]): + def __init__(self, x: S) -> None: + pass +[file b.py] +import a +B = a.A[int, str] +[file c.py] +from b import B +def f() -> None: + B(str()) +[out] +c.py:3: error: Argument 1 to "A" has incompatible type "str"; expected "int" +== + +[case testAliasFineInitChainedMod] +import d +[file a.py] +class A: + def __init__(self, x: int) -> None: + pass +[file a.py.2] +class A: + def __init__(self, x: str) -> None: + pass +[file b.py] +import a +B = a.A +[file c.py] +import b +C = b.B +[file d.py] +from c import C +C(int()) +[out] +== +d.py:2: error: Argument 1 to "A" has incompatible type "int"; expected "str" + +[case testAliasFineInitChainedFunc] +import d +[file a.py] +class A: + def __init__(self, x: int) -> None: + pass +[file a.py.2] +class A: + def __init__(self, x: str) -> None: + pass +[file b.py] +import a +B = a.A +[file c.py] +import b +C = b.B +[file d.py] +from c import C +def f() -> None: + C(str()) +[out] +d.py:3: error: Argument 1 to "A" has incompatible type "str"; expected "int" +== + +[case testNonePartialType1] +import a +a.y + +x = None + +def f() -> None: + global x + x = 1 +[file a.py] +y = 0 +[file a.py.2] +y = '' +[out] +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") +== +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") + +[case testNonePartialType2] +import a +a.y + +x = None + +def f(): + global x + x = 1 +[file a.py] +y = 0 +[file a.py.2] +y = '' +[out] +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") +== +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") + +[case testNonePartialType3] +import a +[file a.py] +[file a.py.2] +y = None +def f() -> None: + global y + y = '' +[out] +== +a.py:1: error: Need type annotation for "y" (hint: "y: Optional[] = ...") + +[case testNonePartialType4] +import a +[file a.py] +y = None +def f() -> None: + global y + y = '' +[file a.py.2] +from typing import Optional +y: Optional[str] = None +def f() -> None: + global y + y = '' +[out] +a.py:1: error: Need type annotation for "y" (hint: "y: Optional[] = ...") +== + +[case testSkippedClass1] +import a +[file a.py] +class A: pass +[file a.py.2] +import sys +if sys.platform == 'xyz': + class A: pass +[builtins fixtures/ops.pyi] +[out] +== + +[case testSkippedClass2] +import a +[file a.py] +import sys +if sys.platform == 'xyz': + class A: pass +[file a.py.2] +import sys +if sys.platform == 'xyz': + class A: pass +[builtins fixtures/ops.pyi] +[out] +== + +[case testSkippedClass3] +import a +[file a.py] +import sys +if sys.platform == 'xyz': + class A: pass +[file a.py.2] +class A: pass +[builtins fixtures/ops.pyi] +[out] +== + +[case testSkippedClass4] +import a +[file a.py] +import sys +if sys.platform == 'xyz': + class A: pass +else: + class A: pass +[file a.py.2] +import sys +if sys.platform == 'xyz': + class A: pass +else: + class A: pass +[builtins fixtures/ops.pyi] +[out] +== + +[case testNewTypeDependencies1] +from a import N + +def f(x: N) -> None: + x.y = 1 +[file a.py] +from typing import NewType +from b import C + +N = NewType('N', C) +[file b.py] +class C: + y: int +[file b.py.2] +class C: + y: str +[out] +== +main:4: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNewTypeDependencies2] +from a import N +from b import C, D + +def f(x: C) -> None: pass + +def g(x: N) -> None: + f(x) +[file a.py] +from typing import NewType +from b import D + +N = NewType('N', D) +[file b.py] +class C: pass +class D(C): pass +[file b.py.2] +class C: pass +class D: pass +[out] +== +main:7: error: Argument 1 to "f" has incompatible type "N"; expected "C" + +[case testNewTypeDependencies3] +from a import N + +def f(x: N) -> None: + x.y +[file a.py] +from typing import NewType +from b import C +N = NewType('N', C) +[file a.py.2] +from typing import NewType +from b import D +N = NewType('N', D) +[file b.py] +class C: + y: int +class D: + pass +[out] +== +main:4: error: "N" has no attribute "y" + +[case testNamedTupleWithinFunction] +from typing import NamedTuple +import b +def f() -> None: + b.x + n = NamedTuple('n', []) +[file b.py] +x = 0 +[file b.py.2] +x = '' +[builtins fixtures/tuple.pyi] +[out] +== + +[case testNamedTupleFallback] +# This test will fail without semantic analyzer pass 2 patches +import a +[file a.py] +import b +[file b.py] +from typing import NamedTuple +import c +c.x +class N(NamedTuple): + count: int +[file c.py] +x = 0 +[file c.py.2] +x = '' +[builtins fixtures/tuple.pyi] +[out] +b.py:5: error: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[int, ...], object], int]") +== +b.py:5: error: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[int, ...], object], int]") + +[case testReprocessEllipses1] +import a +[file a.py] +from typing import Tuple +def foo(x: Tuple[int, ...]) -> None: pass +[file a.py.2] +from typing import Tuple +def foo(x: Tuple[int, ...]) -> None: pass +[builtins fixtures/tuple.pyi] +[out] +== + +[case testReprocessEllipses2] +import a +[file a.py] +from typing import Callable +def foo(x: Callable[..., int]) -> None: pass +[file a.py.2] +from typing import Callable +def foo(x: Callable[..., int]) -> None: pass +[out] +== + +[case testReprocessCallableArg] +import a +[file a.py] +from typing import Callable +from mypy_extensions import Arg +def a(f: Callable[[Arg(int, 'x')], int]) -> None: pass +[file a.py.2] +from typing import Callable +from mypy_extensions import Arg +def a(f: Callable[[Arg(int, 'x')], int]) -> None: pass +[builtins fixtures/dict.pyi] +[out] +== + +[case testImplicitTuple1] +import a +[file a.py] +# Bogus annotation in nested function masked because outer function +# isn't annotated +def unchecked(): + def inner(): + # type: () -> (str, int) + return 'lol', 10 +[file a.py.2] +# dummy change +def unchecked(): + def inner(): + # type: () -> (str, int) + return 'lol', 10 +[builtins fixtures/tuple.pyi] +[out] +== + +[case testImplicitTuple2] +import a +[file a.py] +def inner(): + # type: () -> (str, int) + return 'lol', 10 +[file a.py.2] +# dummy change +def inner(): + # type: () -> (str, int) + return 'lol', 10 +[builtins fixtures/tuple.pyi] +[out] +a.py:1: error: Syntax error in type annotation +a.py:1: note: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) +== +a.py:2: error: Syntax error in type annotation +a.py:2: note: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + +[case testImplicitTuple3] +import a +[file a.py] +(x, y) = 1, 'hi' # type: (int, str) +[file a.py.2] +# dummy change +(x, y) = 1, 'hi' # type: (int, str) +[builtins fixtures/tuple.pyi] +[out] +== + +[case testCastConfusion] +import b +[file a.py] +from typing import cast +class Thing: + def foo(self) -> None: pass + +thing = cast(Thing, Thing()) + +[file b.py] +from typing import Optional +from a import Thing, thing +class User: + def __init__(self, x: Optional[Thing]) -> None: + self.x = x if x else thing + def use(self) -> None: self.x.foo() + +[file a.py.2] +from typing import cast +class Thing: + def foo(self) -> None: pass + +thing = cast(Thing, Thing()) +# update + +[file b.py.2] +from typing import Optional +from a import Thing, thing +class User: + def __init__(self, x: Optional[Thing]) -> None: + self.x = x if x else thing + def use(self) -> None: self.x.foo() +# update +[builtins fixtures/ops.pyi] +[out] +== + +[case testNoStrictOptionalModule] +# flags: --no-strict-optional +import a +a.y = a.x +[file a.py] +from typing import Optional +x: int +y: int +[file a.py.2] +from typing import Optional +x: Optional[int] +y: int +[file a.py.3] +from typing import Optional +x: Optional[str] +y: int +[out] +== +== +main:3: error: Incompatible types in assignment (expression has type "Optional[str]", variable has type "int") + +[case testNoStrictOptionalFunction] +# flags: --no-strict-optional +import a +from typing import Optional +def f() -> None: + x: Optional[int] + a.g(x) +[file a.py] +from typing import Optional +def g(x: Optional[int]) -> None: + pass +[file a.py.2] +from typing import Optional +def g(x: int) -> None: + pass +[file a.py.3] +from typing import Optional +def g(x: str) -> None: + pass +[out] +== +== +main:6: error: Argument 1 to "g" has incompatible type "Optional[int]"; expected "str" + +[case testNoStrictOptionalMethod] +# flags: --no-strict-optional +import a +from typing import Optional +class C: + def f(self) -> None: + x: Optional[int] + a.B().g(x) +[file a.py] +from typing import Optional +class B: + def g(self, x: Optional[int]) -> None: + pass +[file a.py.2] +from typing import Optional +class B: + def g(self, x: int) -> None: + pass +[file a.py.3] +from typing import Optional +class B: + def g(self, x: str) -> None: + pass +[out] +== +== +main:7: error: Argument 1 to "g" of "B" has incompatible type "Optional[int]"; expected "str" + +[case testStrictOptionalModule] +import a +a.y = a.x +[file a.py] +from typing import Optional +x: int +y: int +[file a.py.2] +from typing import Optional +x: Optional[int] +y: int +[out] +== +main:2: error: Incompatible types in assignment (expression has type "Optional[int]", variable has type "int") + +[case testStrictOptionalFunction] +import a +from typing import Optional +def f() -> None: + x: Optional[int] + a.g(x) +[file a.py] +from typing import Optional +def g(x: Optional[int]) -> None: + pass +[file a.py.2] +from typing import Optional +def g(x: int) -> None: + pass +[out] +== +main:5: error: Argument 1 to "g" has incompatible type "Optional[int]"; expected "int" + +[case testStrictOptionalMethod] +import a +from typing import Optional +class C: + def f(self) -> None: + x: Optional[int] + a.B().g(x) +[file a.py] +from typing import Optional +class B: + def g(self, x: Optional[int]) -> None: + pass +[file a.py.2] +from typing import Optional +class B: + def g(self, x: int) -> None: + pass +[out] +== +main:6: error: Argument 1 to "g" of "B" has incompatible type "Optional[int]"; expected "int" + +[case testPerFileStrictOptionalModule] +import a +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-a.*] +strict_optional = True +[file a.py] +from typing import Optional +import b +x: int +y: int = x +[file b.py] +from typing import Optional +x: int +y: int = x +[file b.py.2] +from typing import Optional +x: Optional[int] +y: int = x +[file a.py.3] +from typing import Optional +import b +x: Optional[int] +y: int = x +[out] +== +== +a.py:4: error: Incompatible types in assignment (expression has type "Optional[int]", variable has type "int") + +[case testPerFileStrictOptionalModuleOnly] +import a +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-a.*] +strict_optional = True +[file a.py] +from typing import Optional +import b +y: int = b.x +class Dummy: + def f(self) -> None: + pass +[file b.py] +from typing import Optional +import c +x: int +y: int = c.x +class Dummy: + def f(self) -> None: + pass +[file c.py] +from typing import Optional +x: int +[file c.py.2] +from typing import Optional +x: Optional[int] +[file b.py.3] +from typing import Optional +import c +x: Optional[int] +y: int = c.x +[file a.py.4] +from typing import Optional +import b +y: Optional[int] = b.x +class Dummy: + def f(self) -> None: + pass +[out] +== +== +a.py:3: error: Incompatible types in assignment (expression has type "Optional[int]", variable has type "int") +== + +[case testPerFileStrictOptionalFunction] +import a +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-b.*] +strict_optional = True +[file a.py] +from typing import Optional +import b +def f() -> None: + x: int + x = b.g(x) +[file b.py] +from typing import Optional +import c +def g(x: Optional[int]) -> Optional[int]: + return c.h(x) +[file c.py] +from typing import Optional +def h(x: Optional[int]) -> int: + pass +[file c.py.2] +from typing import Optional +def h(x: int) -> int: + pass +[file b.py.3] +from typing import Optional +import c +def g(x: int) -> Optional[int]: + return c.h(x) +[out] +== +b.py:4: error: Argument 1 to "h" has incompatible type "Optional[int]"; expected "int" +== + +[case testPerFileStrictOptionalMethod] +import a +[file mypy.ini] +\[mypy] +strict_optional = False +\[mypy-b.*] +strict_optional = True +[file a.py] +from typing import Optional +import b +class A: + def f(self) -> None: + x: int + x = b.B().g(x) +[file b.py] +from typing import Optional +import c +class B: + def g(self, x: Optional[int]) -> Optional[int]: + return c.C().h(x) +[file c.py] +from typing import Optional +class C: + def h(self, x: Optional[int]) -> int: + pass +[file c.py.2] +from typing import Optional +class C: + def h(self, x: int) -> int: + pass +[file b.py.3] +from typing import Optional +import c +class B: + def g(self, x: int) -> Optional[int]: + return c.C().h(x) +[out] +== +b.py:5: error: Argument 1 to "h" of "C" has incompatible type "Optional[int]"; expected "int" +== + +[case testTypeVarValuesFunction] +import a +[file a.py] +from typing import TypeVar +from c import A, B +T = TypeVar('T', A, B) + +def f(x: T) -> T: + x.x = int() + return x +[file c.py] +class A: + x: int +class B: + x: int +[file c.py.2] +class A: + x: int +class B: + x: str +[out] +== +a.py:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTypeVarValuesClass] +import a +[file a.py] +import c +class C: + x: c.D[c.A] +[file c.py] +from typing import TypeVar, Generic +class A: pass +class B: pass +class C: pass +T = TypeVar('T', A, B, C) +class D(Generic[T]): + pass +[file c.py.2] +from typing import TypeVar, Generic +class A: pass +class B: pass +class C: pass +T = TypeVar('T', B, C) +class D(Generic[T]): + pass +[out] +== +a.py:3: error: Value of type variable "T" of "D" cannot be "A" + +[case testTypeVarValuesMethod1] +import a +[file a.py] +from typing import Generic +import c +class G(Generic[c.T]): + def f(self, x: c.T) -> None: + x.x = int() +[file c.py] +from typing import TypeVar +class A: + x: int +class B: + x: int +class C: + x: str +T = TypeVar('T', A, B, C) +[file c.py.2] +from typing import TypeVar +class A: + x: int +class B: + x: int +class C: + x: str +T = TypeVar('T', A, B) +[out] +a.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "str") +== + +[case testTypeVarValuesMethod2] +import a +[file a.py] +from typing import Generic +import c +class G(Generic[c.T]): + def f(self, x: c.T) -> None: + x.x = int() +[file c.py] +from typing import TypeVar +class A: + x: int +class B: + x: int +T = TypeVar('T', A, B) +[file c.py.2] +from typing import TypeVar +class A: + x: int +class B: + x: str +T = TypeVar('T', A, B) +[out] +== +a.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTypeVarBoundFunction] +import a +[file a.py] +from typing import TypeVar +from c import B +T = TypeVar('T', bound=B) + +def f(x: T) -> T: + x.x = int() + return x +[file c.py] +class B: + x: int +[file c.py.2] +class B: + x: str +[out] +== +a.py:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTypeVarBoundClass] +import a +[file a.py] +import c +class C: + x: c.D[c.A] +[file c.py] +from typing import TypeVar, Generic +class A: pass +class B: pass +T = TypeVar('T', bound=A) +class D(Generic[T]): + pass +[file c.py.2] +from typing import TypeVar, Generic +class A: pass +class B: pass +T = TypeVar('T', bound=B) +class D(Generic[T]): + pass +[out] +== +a.py:3: error: Type argument "A" of "D" must be a subtype of "B" + +[case testTypeVarValuesRuntime] +from mod import I, S, D +A = I +x = D[S, A]() +[file mod.py] +import submod +from typing import Generic +class D(Generic[submod.T, submod.U]): pass +class I: pass +class S: pass +[file submod.py] +from typing import TypeVar +T = TypeVar('T') +U = TypeVar('U') +[file submod.py.2] +from typing import TypeVar +T = TypeVar('T', int, str) +U = TypeVar('U', int, str) +[out] +== +main:3: error: Value of type variable "submod.T" of "D" cannot be "S" +main:3: error: Value of type variable "submod.U" of "D" cannot be "I" + +[case testTypeVarBoundRuntime] +from mod import I, S, D +A = I +x = D[S, A]() +[file mod.py] +import submod +from typing import Generic +class D(Generic[submod.T, submod.U]): pass +class I: pass +class S: pass +[file submod.py] +from typing import TypeVar +T = TypeVar('T', bound=int) +U = TypeVar('U', bound=int) +[file submod.py.2] +from typing import TypeVar +T = TypeVar('T') +U = TypeVar('U') +[out] +main:3: error: Value of type variable "submod.T" of "D" cannot be "S" +main:3: error: Value of type variable "submod.U" of "D" cannot be "I" +== + +[case testGenericFineCallableNormal] +import a +[file a.py] +import b +x: int = b.f(int()) +[file b.py] +from c import g +f = g +[file c.py] +from typing import TypeVar +class B: pass +T = TypeVar('T') +def g(x: T) -> T: + pass +[file c.py.2] +from typing import TypeVar +class B: pass +T = TypeVar('T', str, B) +def g(x: T) -> T: + pass +[out] +== +a.py:2: error: Value of type variable "T" of function cannot be "int" + +[case testGenericFineCallableNamed] +import a +[file a.py] +import b +x: int = b.f(x=int()) +[file b.py] +from c import g +f = g +[file c.py] +from typing import TypeVar +class B: pass +T = TypeVar('T') +def g(x: T) -> T: + pass +[file c.py.2] +from typing import TypeVar +class B: pass +T = TypeVar('T') +def g(y: T) -> T: + pass +[out] +== +a.py:2: error: Unexpected keyword argument "x" +c.py:4: note: Called function defined here + +[case testGenericFineCallableInBound] +import a +[file a.py] +import b +x: int = b.f()(int()) +[file b.py] +from c import g +f = g +[file c.py] +from typing import Callable, TypeVar +class B: pass +T = TypeVar('T') +def g() -> Callable[[T], T]: + pass +[file c.py.2] +from typing import Callable, TypeVar +class B: pass +T = TypeVar('T', str, B) +def g() -> Callable[[T], T]: + pass +[out] +== +a.py:2: error: Value of type variable "T" of function cannot be "int" + +[case testGenericFineCallableAddedBound] +import a +[file a.py] +import b +x: int = b.f(int()) +[file b.py] +from c import g +f = g +[file c.py] +from typing import TypeVar +class B: pass +T = TypeVar('T') +def g(x: T) -> T: + pass +[file c.py.2] +from typing import TypeVar +class B: pass +T = TypeVar('T', bound=B) +def g(x: T) -> T: + pass +[out] +== +a.py:2: error: Value of type variable "T" of function cannot be "int" + +[case testGenericFineCallableBoundDeleted-only_when_cache] +# See https://github.com/python/mypy/issues/4783 +import a +[file a.py] +import b +x: int = b.f(int()) +[file b.py] +from c import g +f = g +[file c.py] +from typing import TypeVar +import d +T = TypeVar('T', bound=d.B) +def g(x: T) -> T: + pass +[file d.py] +class B: + pass +[file d.py.2] +# empty +[out] +a.py:2: error: Value of type variable "T" of function cannot be "int" +== +c.py:3: error: Name "d.B" is not defined + +[case testGenericFineCallableToNonGeneric] +import a +[file a.py] +import b +x: int = b.f(x=int()) +[file b.py] +from c import g +f = g +[file c.py] +from typing import TypeVar +T = TypeVar('T') +def g(x: T) -> T: + pass +[file c.py.2] +from typing import TypeVar +class T: pass +def g(x: T) -> T: + pass +[out] +== +a.py:2: error: Incompatible types in assignment (expression has type "T", variable has type "int") +a.py:2: error: Argument "x" has incompatible type "int"; expected "T" + +[case testGenericFineCallableToGenericClass] +import a +[file a.py] +import b +x: int = b.f(x=int()) +[file b.py] +from c import g +f = g +[file c.py] +from typing import TypeVar, Generic +T = TypeVar('T') +def g(x: T) -> T: + pass +[file c.py.2] +from typing import TypeVar, Generic +T = TypeVar('T') +class g(Generic[T]): + def __init__(self, x: T) -> None: + pass +[out] +== +a.py:2: error: Incompatible types in assignment (expression has type "g[int]", variable has type "int") + +[case testMakeClassNoLongerAbstract1] +[file z.py] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass +[file b.py] +from typing import Optional +from z import I +class Foo(I): + pass +def x() -> Optional[Foo]: return None +[file z.py.2] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + pass +[file b.py.2] +from z import I +class Foo(I): + pass +def x() -> Foo: return Foo() +[out] +== + +[case testMakeClassNoLongerAbstract2] +-- this version never failed, but it is just a file-renaming +-- away from the above test that did +[file a.py] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass +[file b.py] +from typing import Optional +from a import I +class Foo(I): + pass +def x() -> Optional[Foo]: return None +[file a.py.2] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + pass +[file b.py.2] +from a import I +class Foo(I): + pass +def x() -> Foo: return Foo() +[out] +== + +[case testRefreshClassBasedEnum] +import aa +[file aa.py] +import a +[file a.py] +from enum import Enum +import b +b.x +class C(Enum): + X = 0 +[file b.py] +x = 0 +[file b.py.2] +x = '' +[file aa.py.3] +from a import C +c: C +c = C.X +if int(): + c = 1 +[builtins fixtures/enum.pyi] +[out] +== +== +aa.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "C") + +[case testRefreshClassBasedIntEnum] +import aa +[file aa.py] +import a +[file a.py] +from enum import IntEnum +import b +b.x +class C(IntEnum): + X = 0 +x: int +x = C.X +[file b.py] +x = 0 +[file b.py.2] +x = '' +[file aa.py.3] +from a import C +c: C +c = C.X +if int(): + c = 1 + n: int + n = C.X + if int(): + n = c +[builtins fixtures/enum.pyi] +[out] +== +== +aa.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "C") + +[case testClassBasedEnumPropagation1] +import a +[file a.py] +from b import C + +def f(x: C) -> None: pass +f(C.X) +f(C.Y) +[file b.py] +from enum import Enum + +class C(Enum): + X = 0 + Y = 1 +[file b.py.2] +from enum import Enum + +class C(Enum): + X = 0 +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] +[out] +== +a.py:5: error: "Type[C]" has no attribute "Y" + +[case testClassBasedEnumPropagation2] +import a +[file a.py] +from b import C + +def f(x: int) -> None: pass +f(C.X) +f(C.Y) +[file b.py] +class C: + X = 0 + Y = 1 +[file b.py.2] +from enum import Enum +class C(Enum): + X = 0 + Y = 1 +[builtins fixtures/enum.pyi] +[out] +== +a.py:4: error: Argument 1 to "f" has incompatible type "C"; expected "int" +a.py:5: error: Argument 1 to "f" has incompatible type "C"; expected "int" + +[case testRefreshFuncBasedEnum] +import aa +[file aa.py] +import a +[file a.py] +from enum import Enum +import b +b.x +C = Enum('C', [('X', 0)]) +[file b.py] +x = 0 +[file b.py.2] +x = '' +[file aa.py.3] +from a import C +c: C +c = C.X +if int(): + c = 1 +[builtins fixtures/tuple.pyi] +[out] +== +== +aa.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "C") + +[case testRefreshFuncBasedIntEnum] +import aa +[file aa.py] +import a +[file a.py] +from enum import IntEnum +import b +b.x +C = IntEnum('C', 'X') +x: int +x = C.X +[file b.py] +x = 0 +[file b.py.2] +x = '' +[file aa.py.3] +from a import C +c: C +c = C.X +if int(): + c = 1 # Error + n: int + n = C.X +n = c +[builtins fixtures/enum.pyi] +[out] +== +== +aa.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "C") + +[case testFuncBasedEnumPropagation1] +import a +[file a.py] +from b import C + +def f(x: C) -> None: pass +f(C.X) +f(C.Y) +[file b.py] +from enum import Enum + +C = Enum('C', 'X Y') +[file b.py.2] +from enum import Enum + +C = Enum('C', 'X') +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] +[out] +== +a.py:5: error: "Type[C]" has no attribute "Y" + +[case testFuncBasedEnumPropagation2] +import a +[file a.py] +from b import C + +def f(x: int) -> None: pass +f(C.X) +f(C.Y) +[file b.py] +class C: + X = 0 + Y = 1 +[file b.py.2] +from enum import Enum +C = Enum('C', [('X', 0), ('Y', 1)]) +[builtins fixtures/tuple.pyi] +[out] +== +a.py:4: error: Argument 1 to "f" has incompatible type "C"; expected "int" +a.py:5: error: Argument 1 to "f" has incompatible type "C"; expected "int" + +[case testChangeTypeVarToFunction] + +import a +from typing import Generic + +class C(Generic[a.T]): + def meth(self, x: a.T) -> None: + pass + +Alias = C[C[a.T]] + +def outer() -> None: + def func(x: a.T) -> Alias[a.T]: + pass +[file a.py] +from typing import TypeVar +T = TypeVar('T') +[file a.py.2] +from typing import TypeVar +def T() -> None: + pass +[out] +== +main:5: error: Free type variable expected in Generic[...] +main:6: error: Function "a.T" is not valid as a type +main:6: note: Perhaps you need "Callable[...]" or a callback protocol? +main:9: error: "C" expects no type arguments, but 1 given +main:9: error: Function "a.T" is not valid as a type +main:9: note: Perhaps you need "Callable[...]" or a callback protocol? +main:12: error: Function "a.T" is not valid as a type +main:12: note: Perhaps you need "Callable[...]" or a callback protocol? +main:12: error: Bad number of arguments for type alias, expected 0, given 1 + +[case testChangeTypeVarToModule] + +import a +from typing import Generic + +class C(Generic[a.T]): + def meth(self, x: a.T) -> None: + pass + +Alias = C[C[a.T]] + +def outer() -> None: + def func(x: a.T) -> Alias[a.T]: + pass +[file a.py] +from typing import TypeVar +T = TypeVar('T') +[file T.py.2] +[file a.py.3] +from typing import TypeVar +import T +[out] +== +== +main:5: error: Free type variable expected in Generic[...] +main:6: error: Module "T" is not valid as a type +main:6: note: Perhaps you meant to use a protocol matching the module structure? +main:9: error: "C" expects no type arguments, but 1 given +main:9: error: Module "T" is not valid as a type +main:9: note: Perhaps you meant to use a protocol matching the module structure? +main:12: error: Module "T" is not valid as a type +main:12: note: Perhaps you meant to use a protocol matching the module structure? +main:12: error: Bad number of arguments for type alias, expected 0, given 1 + +[case testChangeClassToModule] + +import a +x: a.C +def f() -> None: + a.C() +class A: + def meth(self) -> None: + def inner() -> a.C: + pass +[file a.py] +class C: + pass +[file C.py.2] +[file a.py.3] +import C +[builtins fixtures/module.pyi] +[out] +== +== +main:3: error: Module "C" is not valid as a type +main:3: note: Perhaps you meant to use a protocol matching the module structure? +main:5: error: Module not callable +main:8: error: Module "C" is not valid as a type +main:8: note: Perhaps you meant to use a protocol matching the module structure? + +[case testChangeTypeVarToTypeAlias] + +import a +from typing import Generic + +class C(Generic[a.T]): + def meth(self, x: a.T) -> None: + pass + +Alias = C[C[a.T]] + +def outer() -> None: + def func(x: a.T) -> Alias[a.T]: + pass +[file a.py] +from typing import TypeVar +T = TypeVar('T') +[file a.py.2] +from typing import TypeVar +T = int +[out] +== +main:5: error: Free type variable expected in Generic[...] +main:9: error: "C" expects no type arguments, but 1 given +main:12: error: Bad number of arguments for type alias, expected 0, given 1 + +[case testChangeTypeAliasToModule] + +import a +x: a.C +def f() -> None: + a.C() +class A: + def meth(self) -> None: + def inner() -> a.C: + pass +[file a.py] +import b +C = b.D +[file b.py] +class D: + pass +[file D.py.2] +[file b.py.3] +import D +[builtins fixtures/module.pyi] +[out] +== +== +main:3: error: Module "D" is not valid as a type +main:3: note: Perhaps you meant to use a protocol matching the module structure? +main:5: error: Module not callable +main:8: error: Module "D" is not valid as a type +main:8: note: Perhaps you meant to use a protocol matching the module structure? + +[case testChangeTypeAliasToModuleUnqualified] + +from a import C +x: C +def f() -> None: + C() +class A: + def meth(self) -> None: + def inner() -> C: + pass +[file a.py] +from b import D +C = D +[file b.py] +class D: + pass +[file D.py.2] +[file b.py.3] +import D +[builtins fixtures/module.pyi] +[out] +== +== +main:3: error: Module "D" is not valid as a type +main:3: note: Perhaps you meant to use a protocol matching the module structure? +main:5: error: Module not callable +main:8: error: Module "D" is not valid as a type +main:8: note: Perhaps you meant to use a protocol matching the module structure? + +[case testChangeFunctionToVariableAndRefreshUsingStaleDependency] +import a +import c +[file a.py] +import c +def f() -> c.A: pass +[file a.py.2] +f = 1 +[file c.py] +class A: pass +[file c.py.3] +[out] +== +== + +[case testChangeFunctionToTypeVarAndRefreshUsingStaleDependency] +import a +import c +[file a.py] +import c +def f() -> c.A: pass +[file a.py.2] +from typing import TypeVar +f = TypeVar('f') +[file c.py] +class A: pass +[file c.py.3] +[out] +== +== + +[case testChangeFunctionToModuleAndRefreshUsingStaleDependency] +import a +import c +[file a.py] +import c +def f() -> c.A: pass +[file a.py.2] +import c as f +[file c.py] +class A: pass +[file c.py.3] +[out] +== +== + +[case testChangeFunctionToTypeAliasAndRefreshUsingStaleDependency1] +import a +import c +[file a.py] +import c +def f() -> c.A: pass +[file a.py.2] +f = int +[file c.py] +class A: pass +[file c.py.3] +[out] +== +== + +[case testChangeFunctionToTypeAliasAndRefreshUsingStaleDependency2] +import a +import c +[file a.py] +import c +def f() -> c.A: pass +[file a.py.2] +from typing import List +f = List[int] +[file c.py] +class A: pass +[file c.py.3] +[builtins fixtures/list.pyi] +[out] +== +== + +[case testChangeFunctionToClassAndRefreshUsingStaleDependency] +import a +import c +[file a.py] +import c +def f() -> c.A: pass +[file a.py.2] +class f: pass +[file c.py] +class A: pass +[file c.py.3] +[out] +== +== + +[case testClassToVariableAndRefreshUsingStaleDependency] +import a +import c +[file a.py] +import c +class A: + def f(self) -> c.A: pass +[file a.py.2] +A = 0 +[file c.py] +class A: pass +[file c.py.3] +[out] +== +== + +[case testFunctionToImportedFunctionAndRefreshUsingStaleDependency] +import a +import c +[file a.py] +import c +def f() -> c.A: pass +[file a.py.2] +from d import f +[file c.py] +class A: pass +[file c.py.3] +[file d.py] +def g() -> None: pass +def f() -> None: + g() +[out] +== +== + +[case testMethodToVariableAndRefreshUsingStaleDependency] +import a +import c +[file a.py] +import c +class B: + def f(self) -> c.A: pass +[file a.py.2] +class B: + f = 0 +[file c.py] +class A: pass +[file c.py.3] +[out] +== +== + +[case testChangeGenericFunctionToVariable] +import a +x: int +y: int = a.f(x) +class Dummy: + def g(self) -> None: + a.f(x) +[file a.py] +from typing import TypeVar +T = TypeVar('T') +def f(x: T) -> T: + pass +[file a.py.2] +from typing import TypeVar +T = TypeVar('T') +f = 42 +[out] +== +main:3: error: "int" not callable +main:6: error: "int" not callable + +[case testChangeGenericClassToVariable] +import a +x: int +a.A(x) +class Dummy: + def g(self) -> None: + a.A(x) +[file a.py] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def __init__(self, x: T) -> None: + pass +[file a.py.2] +from typing import TypeVar, Generic +T = TypeVar('T') +A = 'no way' +[out] +== +main:3: error: "str" not callable +main:6: error: "str" not callable + +[case testChangeGenericMethodToVariable] +import a +x: int +y: int = a.A(x).f() +class Dummy: + def g(self) -> None: + a.A(x).f() +[file a.py] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def __init__(self, x: T) -> None: + pass + def f(self) -> T: + pass +[file a.py.2] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + f: T + def __init__(self, x: T) -> None: + pass +[out] +== +main:3: error: "int" not callable +main:6: error: "int" not callable + +[case testRefreshNestedClassWithSelfReference] +import a +[file a.py] +import b + +def f(self) -> None: + b.y + class C: + z: C +[file b.py] +y = 0 +[file b.py.2] +y = '' +[file b.py.3] +y = 0 +[out] +== +== + +[case testMultipleAssignment] +import a +[file a.py] +from b import f + +def h(x: str) -> None: pass + +class C: + def __init__(self) -> None: + self.a, self.b = f() + + def g(self) -> None: + h(self.a) +[file b.py] +from typing import Tuple +def f() -> Tuple[str, int]: pass +[file b.py.2] +from typing import Tuple +def f() -> Tuple[int, object]: pass +[file b.py.3] +from typing import Tuple +def f() -> Tuple[str, int]: pass +[builtins fixtures/tuple.pyi] +[out] +== +a.py:10: error: Argument 1 to "h" has incompatible type "int"; expected "str" +== + +[case testMultipleLvalues] +import a +[file a.py] +from b import f + +def h(x: str) -> None: pass + +class C: + def __init__(self) -> None: + self.a = self.b = f() + + def g(self) -> None: + h(self.a) + h(self.b) +[file b.py] +def f() -> str: pass +[file b.py.2] +def f() -> int: pass +[file b.py.3] +def f() -> str: pass +[out] +== +a.py:10: error: Argument 1 to "h" has incompatible type "int"; expected "str" +a.py:11: error: Argument 1 to "h" has incompatible type "int"; expected "str" +== + +[case testNoOpUpdateFineGrainedIncremental1] +# cmd: mypy a.py +[file a.py] +1() +[file b.py.2] +# Note: this file is not part of the build +[file a.py.3] +x = 1 +[out] +a.py:1: error: "int" not callable +== +a.py:1: error: "int" not callable +== + +[case testNoOpUpdateFineGrainedIncremental2] +# cmd: mypy a.py +[file a.py] +1() +[file a.py.2] +1() +[file a.py.3] +x = 1 +[file a.py.4] +x = 1 +[out] +a.py:1: error: "int" not callable +== +a.py:1: error: "int" not callable +== +== + +[case testNonExistentFileOnCommandLine1] +# cmd: mypy a.py nonexistent.py +[file a.py] +[file a.py.2] +1() +[out] +mypy: can't read file 'tmp/nonexistent.py': No such file or directory +== +mypy: can't read file 'tmp/nonexistent.py': No such file or directory + +[case testNonExistentFileOnCommandLine2] +# cmd: mypy a.py +# cmd2: mypy a.py nonexistent.py +[file a.py] +[file a.py.2] +1() +[out] +== +a.py:1: error: "int" not callable + +[case testNonExistentFileOnCommandLine3] +# cmd: mypy a.py +# cmd2: mypy a.py nonexistent.py +[file a.py] +[file nonexistent.py] +[delete nonexistent.py.2] +[out] +== + +[case testNonExistentFileOnCommandLine4] +# cmd: mypy a.py nonexistent.py +[file a.py] +[file nonexistent.py] +[delete nonexistent.py.2] +[out] +== + +[case testNonExistentFileOnCommandLine5] +# cmd: mypy a.py nonexistent_stub.pyi +# TODO: Should generate an error for missing file +[file a.py] +[file nonexistent_stub.pyi] +[delete nonexistent_stub.pyi.2] +[out] +== + +[case testDunderNewUpdatedMethod] +import a +[file a.py] +import b +class A: + def func(self) -> None: + b.C(int()) +[file b.py] +class C: + def __new__(cls, x: str) -> C: + pass +[file b.py.2] +class C: + def __new__(cls, x: int) -> C: + pass +[out] +a.py:4: error: Argument 1 to "C" has incompatible type "int"; expected "str" +== + +[case testDunderNewUpdatedSubclass] +import a +[file a.py] +import b + +b.D(int()) +[file b.py] +from c import C +class D(C): pass +[file c.py] +class C: + def __new__(cls, x: str) -> C: + pass +[file c.py.2] +class C: + def __new__(cls, x: int) -> C: + pass +[out] +a.py:3: error: Argument 1 to "D" has incompatible type "int"; expected "str" +== + +[case testDunderNewUpdatedAlias] +import a +[file a.py] +import b + +b.D(int()) +[file b.py] +from c import C +D = C +[file c.py] +class C: + def __new__(cls, x: int) -> C: + pass +[file c.py.2] +class C: + def __new__(cls, x: str) -> C: + pass +[out] +== +a.py:3: error: Argument 1 to "C" has incompatible type "int"; expected "str" + +[case testDunderNewUpdatedCallable] +import a +[file a.py] +from typing import Callable, Any +import b + +def func(arg: Callable[[int], Any]) -> None: + pass +func(b.C) +[file b.py] +class C: + def __new__(cls, x: int) -> C: + pass +[file b.py.2] +class C: + def __new__(cls, x: str) -> C: + pass +[out] +== +a.py:6: error: Argument 1 to "func" has incompatible type "Type[C]"; expected "Callable[[int], Any]" + +[case testDunderNewDefine] +import a +[file a.py] +import b +class A: + def func(self) -> None: + b.C() +[file b.py] +class C: + pass +[file b.py.2] +class C: + def __new__(cls, x: int) -> C: + pass +[out] +== +a.py:4: error: Missing positional argument "x" in call to "C" + +[case testDunderNewInsteadOfInit] +import a +[file a.py] +import b +class A: + def func(self) -> None: + b.C(int()) +[file b.py] +class C: + def __init__(cls, x: int) -> None: + pass +[file b.py.2] +class C: + def __new__(cls, x: int) -> C: + pass +[file b.py.3] +class C: + pass +[out] +== +== +a.py:4: error: Too many arguments for "C" + +-- Protocol tests + +[case testProtocolUpdateTypeInVariable] +import a +[file a.py] +import b +class C: + x: int +x: b.P = C() +[file b.py] +from typing import Protocol +class P(Protocol): + x: int +[file b.py.2] +from typing import Protocol +class P(Protocol): + x: str +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "C", variable has type "P") +a.py:4: note: Following member(s) of "C" have conflicts: +a.py:4: note: x: expected "str", got "int" + +[case testProtocolUpdateTypeInFunction] +import a +[file a.py] +import b +class C: + x: int +c: C +def f() -> None: + def g(x: b.P) -> None: + pass + g(c) +[file b.py] +from typing import Protocol +class P(Protocol): + x: int +[file b.py.2] +from typing import Protocol +class P(Protocol): + x: str +[out] +== +a.py:8: error: Argument 1 to "g" has incompatible type "C"; expected "P" +a.py:8: note: Following member(s) of "C" have conflicts: +a.py:8: note: x: expected "str", got "int" + +[case testProtocolUpdateTypeInClass] +import a +[file a.py] +import b +class C: + x: int +class A: + class B: + x: b.P + y: B +A().y.x = C() +[file b.py] +from typing import Protocol +class P(Protocol): + x: int +[file b.py.2] +from typing import Protocol +class P(Protocol): + x: str +[out] +== +a.py:8: error: Incompatible types in assignment (expression has type "C", variable has type "P") +a.py:8: note: Following member(s) of "C" have conflicts: +a.py:8: note: x: expected "str", got "int" + +[case testProtocolAddAttrInFunction] +import a +[file a.py] +import b +class C: + x: int +def f() -> None: + c: C + def g(x: b.P) -> None: + pass + g(c) +[file b.py] +from typing import Protocol +class P(Protocol): + x: int +[file b.py.2] +from typing import Protocol +class P(Protocol): + x: int + y: str +[out] +== +a.py:8: error: Argument 1 to "g" has incompatible type "C"; expected "P" +a.py:8: note: "C" is missing following "P" protocol member: +a.py:8: note: y + +[case testProtocolRemoveAttrInClass] +import a +[file a.py] +import b +class C: + x: int +class A: + class B: + x: b.P + y: B +A().y.x = C() +[file b.py] +from typing import Protocol +class P(Protocol): + x: int + y: str +[file b.py.2] +from typing import Protocol +class P(Protocol): + x: int +[out] +a.py:8: error: Incompatible types in assignment (expression has type "C", variable has type "P") +a.py:8: note: "C" is missing following "P" protocol member: +a.py:8: note: y +== + +[case testProtocolConcreteUpdateTypeFunction] +import a +[file a.py] +import b +from typing import Protocol +class P(Protocol): + x: int +def f() -> None: + def g(x: P) -> None: + pass + g(b.C()) +[file b.py] +class C: + x: int +[file b.py.2] +class C: + x: str +[out] +== +a.py:8: error: Argument 1 to "g" has incompatible type "C"; expected "P" +a.py:8: note: Following member(s) of "C" have conflicts: +a.py:8: note: x: expected "int", got "str" + +[case testProtocolConcreteUpdateTypeMethodGeneric] +import a +[file a.py] +import b +from typing import Protocol, TypeVar +T = TypeVar('T') +class P(Protocol[T]): + x: T +class C: + def g(self, x: P[int]) -> None: + pass + def do(self) -> None: + self.g(b.C()) +[file b.py] +class C: + x: int +[file b.py.2] +class C: + x: str +[out] +== +a.py:10: error: Argument 1 to "g" of "C" has incompatible type "C"; expected "P[int]" +a.py:10: note: Following member(s) of "C" have conflicts: +a.py:10: note: x: expected "int", got "str" + +[case testProtocolConcreteRemoveAttrVariable] +import a +[file a.py] +import b, c +cc: c.C +x: b.P = cc +[file b.py] +from typing import Protocol +class P(Protocol): + x: int +[file c.py] +class C: + x: int +[file c.py.2] +class C: + pass +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "C", variable has type "P") + +[case testProtocolUpdateBaseGeneric] +import a +[file a.py] +import b, c +def g(x: c.P) -> None: + pass +g(b.C()) +[file b.py] +class C: + x: int +[file c.py] +from typing import Protocol +import d +class P(d.PBase[int], Protocol): + pass +[file c.py.2] +from typing import Protocol +import d +class P(d.PBase[str], Protocol): + pass +[file d.py] +from typing import Protocol, TypeVar +T = TypeVar('T') +class PBase(Protocol[T]): + x: T +[out] +== +a.py:4: error: Argument 1 to "g" has incompatible type "C"; expected "P" +a.py:4: note: Following member(s) of "C" have conflicts: +a.py:4: note: x: expected "str", got "int" + +[case testProtocolConcreteUpdateBaseGeneric] +import a +[file a.py] +import b +from typing import Protocol +class P(Protocol): + x: int +def f(x: P) -> None: + pass +f(b.B()) +[file b.py] +import c +class B(c.C[int]): + pass +[file b.py.2] +import c +class B(c.C[str]): + pass +[file c.py] +from typing import TypeVar, Generic +T = TypeVar('T') +class C(Generic[T]): + x: T +[out] +== +a.py:7: error: Argument 1 to "f" has incompatible type "B"; expected "P" +a.py:7: note: Following member(s) of "B" have conflicts: +a.py:7: note: x: expected "int", got "str" + +[case testProtocolChangeGeneric] +import a +[file a.py] +import b, c +x: b.P = c.C() +[file b.py] +import b2 +from typing import Protocol +class P(b2.P2[str], Protocol): + pass +[file b2.py] +from typing import Protocol, TypeVar +T = TypeVar('T') +class P2(Protocol[T]): + x: T +[file b2.py.2] +from typing import Protocol, TypeVar +T = TypeVar('T') +class P2(Protocol): + x: int +[file c.py] +class C: + x: int +[out] +a.py:2: error: Incompatible types in assignment (expression has type "C", variable has type "P") +a.py:2: note: Following member(s) of "C" have conflicts: +a.py:2: note: x: expected "str", got "int" +== +b.py:3: error: "P2" expects no type arguments, but 1 given + +[case testProtocolToNonProtocol] +import a +[file a.py] +import b, c +b.f(c.C()) +[file b.py] +import d +def f(x: d.D) -> None: + pass +[file c.py] +import d +class C: + x: int +[file d.py] +from typing import Protocol +class D(Protocol): + x: int +[file d.py.2] +class D: + x: int +[file c.py.3] +import d +class C(d.D): + pass +[out] +== +a.py:2: error: Argument 1 to "f" has incompatible type "C"; expected "D" +== + +[case testNonProtocolToProtocol] +import a +[file a.py] +import b, c +b.f(c.C()) +[file b.py] +import d +def f(x: d.D) -> None: + pass +[file c.py] +import d +class C(d.D): + pass +[file d.py] +class D: + x: int +[file d.py.2] +from typing import Protocol +class D(Protocol): + x: int +[file c.py.3] +import d +class C: + x: int +[out] +== +a.py:2: error: Cannot instantiate abstract class "C" with abstract attribute "x" +== + +[case testInvalidateProtocolViaSuperClass] +import a +[file a.py] +import b, c +def func(x: c.P) -> None: + pass +func(b.B()) +[file b.py] +class B: + x: int + y: str +[file c.py] +from typing import Protocol +import d +class P(d.PBase, Protocol): + x: int +[file d.py] +from typing import Protocol +class PBase(Protocol): + y: str +[file d.py.2] +from typing import Protocol +class PBase(Protocol): + y: int +[out] +== +a.py:4: error: Argument 1 to "func" has incompatible type "B"; expected "P" +a.py:4: note: Following member(s) of "B" have conflicts: +a.py:4: note: y: expected "int", got "str" + +[case testProtocolInvalidateConcreteViaSuperClassUpdateType] +import a +[file a.py] +import b +def func(x: b.P) -> None: + pass +func(b.B()) +[file b.py] +from typing import Protocol +import c +class P(Protocol): + x: int +class B(c.C): + pass +[file c.py] +class C: + x: int +[file c.py.2] +class C: + x: str +[out] +== +a.py:4: error: Argument 1 to "func" has incompatible type "B"; expected "P" +a.py:4: note: Following member(s) of "B" have conflicts: +a.py:4: note: x: expected "int", got "str" + +[case testProtocolInvalidateConcreteViaSuperClassAddAttr] +import a +[file a.py] +import b +def func(x: b.P) -> None: + pass +bb: b.B +func(bb) +[file b.py] +from typing import Protocol +import c +class P(Protocol): + x: int +class B(c.C): + pass +[file c.py] +class C: + pass +[file c.py.2] +class C: + x: int +[out] +a.py:5: error: Argument 1 to "func" has incompatible type "B"; expected "P" +== + +[case testProtocolInvalidateConcreteViaSuperClassRemoveAttr] +import a +[file a.py] +import b +def func(x: b.P) -> None: + pass +func(b.B()) +[file b.py] +from typing import Protocol +import c +class P(Protocol): + x: int +class B(c.C): + pass +[file c.py] +class C: + x: int +[file c.py.2] +class C: + pass +[out] +== +a.py:4: error: Argument 1 to "func" has incompatible type "B"; expected "P" + +[case testTwoProtocolsTwoFilesCrossedUpdateType-only_when_nocache] +# this test and the next one (TwoProtocolsTwoFilesCrossedDeleteAttr) has errors ordered +# opposite way with and without cache, therefore skip one of each. +import a +[file a.py] +import b1 +import b2 +[file b1.py] +import b2, d +from typing import Protocol +class P1(Protocol): + x: int +def f(x: b2.P2) -> None: + pass +f(d.D()) +[file b2.py] +import b1, d +from typing import Protocol +class P2(Protocol): + x: int +def f(x: b1.P1) -> None: + pass +f(d.D()) +[file d.py] +class D: + x: int +[file d.py.2] +class D: + x: str +[out] +== +b1.py:7: error: Argument 1 to "f" has incompatible type "D"; expected "P2" +b1.py:7: note: Following member(s) of "D" have conflicts: +b1.py:7: note: x: expected "int", got "str" +b2.py:7: error: Argument 1 to "f" has incompatible type "D"; expected "P1" +b2.py:7: note: Following member(s) of "D" have conflicts: +b2.py:7: note: x: expected "int", got "str" + +[case testTwoProtocolsTwoFilesCrossedDeleteAttr-only_when_cache] +import a +[file a.py] +import b1 +import b2 +[file b1.py] +import b2, d +from typing import Protocol +class P1(Protocol): + x: int +def f(x: b2.P2) -> None: + pass +f(d.D()) +[file b2.py] +import b1, d +from typing import Protocol +class P2(Protocol): + x: int +def f(x: b1.P1) -> None: + pass +f(d.D()) +[file d.py] +class D: + x: int +[file d.py.2] +class D: + y: int +[out] +b2.py:7: error: Argument 1 to "f" has incompatible type "D"; expected "P2" (diff) +b1.py:7: error: Argument 1 to "f" has incompatible type "D"; expected "P1" + +[case testProtocolsInvalidateByRemovingBase] +import a +[file a.py] +import b +def func(x: b.P) -> None: + pass +func(b.B()) +[file b.py] +from typing import Protocol +import c +class P(Protocol): + x: int +class B(c.C): + pass +[file c.py] +import d +class C(d.D): + pass +[file c.py.2] +import d +class C: + pass +[file d.py] +class D: + x: int +[out] +== +a.py:4: error: Argument 1 to "func" has incompatible type "B"; expected "P" + +[case testProtocolsInvalidateByRemovingMetaclass] +import a +[file a.py] +import b +def func(x: b.P) -> None: + pass +func(b.B) +[file b.py] +from typing import Protocol +import c +class P(Protocol): + x: int +class B(c.C): + pass +[file c.py] +import d +class C(metaclass=d.M): + pass +[file c.py.2] +import d +class C: + pass +[file d.py] +class M(type): + x: int +[out] +== +a.py:4: error: Argument 1 to "func" has incompatible type "Type[B]"; expected "P" + +[case testProtocolVsProtocolSubUpdated] +import a +[file a.py] +import b, c +x: b.SuperP +y: c.SubP +x = y +[file b.py] +from typing import Protocol +class SuperP(Protocol): + x: int +[file c.py] +from typing import Protocol +import d +class SubP(d.PBase, Protocol): + y: str +[file d.py] +from typing import Protocol +class PBase(Protocol): + x: int +[file d.py.2] +from typing import Protocol +class PBase(Protocol): + x: str +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "SubP", variable has type "SuperP") +a.py:4: note: Following member(s) of "SubP" have conflicts: +a.py:4: note: x: expected "int", got "str" + +[case testProtocolVsProtocolSuperUpdated] +import a +[file a.py] +import b, c +x: b.SuperP +y: c.SubP +x = y +[file b.py] +from typing import Protocol +import d +class SuperP(d.PBase, Protocol): + pass +[file c.py] +from typing import Protocol +class SubP(Protocol): + x: int +[file d.py] +from typing import Protocol +class PBase(Protocol): + x: int +[file d.py.2] +from typing import Protocol +class PBase(Protocol): + y: int +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "SubP", variable has type "SuperP") + +[case testProtocolVsProtocolSuperUpdated2] +import a +[file a.py] +import b, c +x: b.SuperP +y: c.SubP +x = y +[file b.py] +from typing import Protocol +import d +class SuperP(d.PBase, Protocol): + x: int +[file c.py] +from typing import Protocol +class SubP(Protocol): + x: int + y: int +[file d.py] +from typing import Protocol +class PBase(Protocol): + y: int +[file d.py.2] +from typing import Protocol +class PBase(Protocol): + y: int + z: int +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "SubP", variable has type "SuperP") +a.py:4: note: "SubP" is missing following "SuperP" protocol member: +a.py:4: note: z + +[case testProtocolVsProtocolSuperUpdated3] +import a +[file a.py] +import b, c +x: b.SuperP +y: c.SubP +x = y +[file b.py] +from typing import Protocol +import d +class SuperP(d.PBase, Protocol): + x: int +[file c.py] +from typing import Protocol +class SubP(Protocol): + x: int + y: int +[file d.py] +from typing import Protocol +import e +class PBase(Protocol): + y: int +[file d.py.2] +from typing import Protocol +import e +class PBase(e.NewP, Protocol): + y: int +[file e.py] +from typing import Protocol +class NewP(Protocol): + z: int +[out] +== +a.py:4: error: Incompatible types in assignment (expression has type "SubP", variable has type "SuperP") +a.py:4: note: "SubP" is missing following "SuperP" protocol member: +a.py:4: note: z + +[case testProtocolMultipleUpdates] +import a +[file a.py] +import b, c +x: b.P = c.C() +[file b.py] +from typing import Protocol +import b2 +class P(b2.P2, Protocol): + x: int +[file b2.py] +from typing import Protocol +class P2(Protocol): + y: int +[file c.py] +import c2 +class C(c2.C2): + x: int +[file c2.py] +class C2: + y: int +[file b2.py.2] +from typing import Protocol +class P2(Protocol): + y: int + z: int +[file c2.py.3] +class C2: + y: int + z: int +[file c2.py.4] +class C2: + y: int + z: str +[out] +== +a.py:2: error: Incompatible types in assignment (expression has type "C", variable has type "P") +a.py:2: note: "C" is missing following "P" protocol member: +a.py:2: note: z +== +== +a.py:2: error: Incompatible types in assignment (expression has type "C", variable has type "P") +a.py:2: note: Following member(s) of "C" have conflicts: +a.py:2: note: z: expected "int", got "str" + +[case testWeAreCarefulWithBuiltinProtocols] +import a +x: a.A +for i in x: + pass +[file a.py] +from typing import Iterator +class A: + def __iter__(self) -> Iterator[int]: + pass +[file a.py.2] +class A: + pass +[out] +== +main:3: error: "A" has no attribute "__iter__" (not iterable) + +[case testWeAreCarefullWithBuiltinProtocolsBase] +import a +x: a.A +for i in x: + pass +[file a.py] +import b +class A(b.B): + pass +[file a.py.2] +class A: + pass +[file b.py] +from typing import Iterator +class B: + def __iter__(self) -> Iterator[int]: + pass +[out] +== +main:3: error: "A" has no attribute "__iter__" (not iterable) + +[case testOverloadsSimpleFrom] +import a +[file a.py] +import mod +def f() -> None: + x: str = mod.f(str()) +[file mod.py] +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> str: pass +def f(x): + pass +[file mod.py.2] +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> int: pass +def f(x): + pass +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testOverloadsSimpleToNested] +from typing import overload, Any +import mod +def outer() -> None: + @overload + def f(x: int) -> None: pass + @overload + def f(x: str) -> str: pass + def f(x: Any) -> Any: + y: int = mod.f() +[file mod.py] +def f() -> int: + pass +[file mod.py.2] +def f() -> str: + pass +[out] +== +main:9: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testOverloadsRemovedOverload] +import mod +def f() -> None: + x: str = mod.f(str()) +[file mod.py] +class C: pass +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> str: pass +@overload +def f(x: C) -> int: pass +def f(x): + pass +[file mod.py.2] +class C: pass +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: C) -> int: pass +def f(x): + pass +[out] +== +main:3: error: No overload variant of "f" matches argument type "str" +main:3: note: Possible overload variants: +main:3: note: def f(x: int) -> None +main:3: note: def f(x: C) -> int + +[case testOverloadsDeleted] +import mod +def f() -> None: + x: str = mod.f(str()) +[file mod.py] +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> str: pass +def f(x): + pass +[file mod.py.2] +from typing import overload +[builtins fixtures/module.pyi] +[out] +== +main:3: error: Module has no attribute "f" + +[case testOverloadsUpdatedTypeRecheckImplementation] +from typing import overload +import mod +class Outer: + @overload + def f(self, x: mod.D) -> mod.D: pass + @overload + def f(self, x: mod.E) -> mod.E: pass + def f(self, x: mod.C) -> mod.C: + x.x = int() + return x +[file mod.py] +import submod +class C(submod.B): + pass +class D(C): + pass +class E(C): + pass +[file submod.py] +import base +class B(base.AI): + pass +[file submod.py.2] +import base +class B(base.AS): + pass +[file base.py] +class AI: + x: int +class AS: + x: str +[out] +== +main:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testOverloadsUpdatedTypeRechekConsistency] +from typing import overload +import mod +class Outer: + @overload + def f(self, x: mod.D) -> mod.D: pass + @overload + def f(self, x: mod.E) -> mod.E: pass + def f(self, x: mod.C) -> mod.C: + pass +[file mod.py] +class C: + pass +class D(C): + pass +class E(C): + pass +[file mod.py.2] +class C: + pass +class D(C): + pass +class E: + pass +[out] +== +main:8: error: Overloaded function implementation does not accept all possible arguments of signature 2 +main:8: error: Overloaded function implementation cannot produce return type of signature 2 + +[case testOverloadsGenericTypevarUpdated] +import a +[file a.py] +import b +b.f(int()) +[file b.py] +from typing import overload +import c +class C: pass +@overload +def f(x: C) -> None: pass +@overload +def f(x: c.T) -> c.T: pass +def f(x): + pass +[file c.py] +from typing import TypeVar +T = TypeVar('T', int, str) +[file c.py.2] +from typing import TypeVar +T = TypeVar('T', bound=str) +[out] +== +a.py:2: error: No overload variant of "f" matches argument type "int" +a.py:2: note: Possible overload variants: +a.py:2: note: def f(x: C) -> None +a.py:2: note: def [c.T: str] f(x: c.T) -> c.T + +[case testOverloadsGenericToNonGeneric] +import a +[file a.py] +import b +b.f(int()) +[file b.py] +from typing import overload +import c +class C: pass +@overload +def f(x: C) -> None: pass +@overload +def f(x: c.T) -> c.T: pass +def f(x): + pass +[file c.py] +from typing import TypeVar +T = TypeVar('T', bound=int) +[file c.py.2] +from typing import TypeVar +class T: pass +[out] +== +a.py:2: error: No overload variant of "f" matches argument type "int" +a.py:2: note: Possible overload variants: +a.py:2: note: def f(x: C) -> None +a.py:2: note: def f(x: T) -> T + +[case testOverloadsToNonOverloaded] +import a +[file a.py] +import mod +def f() -> None: + x: str = mod.f(str()) +[file mod.py] +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> str: pass +def f(x): + pass +[file mod.py.2] +from typing import overload +def f(x: int) -> int: + pass +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") +a.py:3: error: Argument 1 to "f" has incompatible type "str"; expected "int" + +[case testOverloadsUpdateFunctionToOverloaded] +import a +[file a.py] +import mod +def f() -> None: + x: str = mod.f(str()) +[file mod.py] +from typing import overload +def f(x: str) -> str: + pass +[file mod.py.2] +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> int: pass +def f(x): + pass +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testOverloadedUpdateToClass] +import a +[file a.py] +import mod +def f() -> None: + x: str = mod.f(str()) +[file mod.py] +from typing import overload +@overload +def f(x: int) -> None: pass +@overload +def f(x: str) -> str: pass +def f(x): + pass +[file mod.py.2] +from typing import overload +class f: + def __init__(self, x: str) -> None: + pass +[out] +== +a.py:3: error: Incompatible types in assignment (expression has type "f", variable has type "str") + +[case testDepsFromOverloadUpdatedAttrRecheckImpl] +import mod +x = mod.f +[file mod.py] +from typing import overload, Any +import submod +@overload +def f(x: int) -> submod.A: pass +@overload +def f(x: str) -> submod.B: pass +def f(x) -> Any: + y: submod.C + y.x = int() +[file submod.py] +import other +class A: pass +class B: pass +C = other.C +[file other.py] +class C: + x: int +[file other.py.2] +class C: + x: str +[out] +== +mod.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testOverloadedMethodSupertype-only_when_cache] +-- Different cache/no-cache tests because +-- CallableType.def_extras.first_arg differs ("self"/None) +from typing import overload, Any +import b +class Child(b.Parent): + @overload # Fail + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Any) -> Any: ... +[file b.py] +from typing import overload, Any +class C: pass +class Parent: + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Any) -> Any: ... +[file b.py.2] +from typing import overload, Any +class C: pass +class Parent: + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> C: ... + def f(self, arg: Any) -> Any: ... +[out] +== +main:4: error: Signature of "f" incompatible with supertype "Parent" +main:4: note: Superclass: +main:4: note: @overload +main:4: note: def f(self, arg: int) -> int +main:4: note: @overload +main:4: note: def f(self, arg: str) -> C +main:4: note: Subclass: +main:4: note: @overload +main:4: note: def f(self, arg: int) -> int +main:4: note: @overload +main:4: note: def f(self, arg: str) -> str + +[case testOverloadedMethodSupertype2-only_when_nocache] +-- Different cache/no-cache tests because +-- CallableType.def_extras.first_arg differs ("self"/None) +from typing import overload, Any +import b +class Child(b.Parent): + @overload # Fail + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Any) -> Any: ... +[file b.py] +from typing import overload, Any +class C: pass +class Parent: + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> str: ... + def f(self, arg: Any) -> Any: ... +[file b.py.2] +from typing import overload, Any +class C: pass +class Parent: + @overload + def f(self, arg: int) -> int: ... + @overload + def f(self, arg: str) -> C: ... + def f(self, arg: Any) -> Any: ... +[out] +== +main:4: error: Signature of "f" incompatible with supertype "Parent" +main:4: note: Superclass: +main:4: note: @overload +main:4: note: def f(self, arg: int) -> int +main:4: note: @overload +main:4: note: def f(self, arg: str) -> C +main:4: note: Subclass: +main:4: note: @overload +main:4: note: def f(arg: int) -> int +main:4: note: @overload +main:4: note: def f(arg: str) -> str + +[case testOverloadedInitSupertype] +import a +[file a.py] +from b import B +B(int()) +[file b.py] +import c +class B(c.C): + pass +[file c.py] +from typing import overload +class C: + def __init__(self, x: int) -> None: + pass +[file c.py.2] +from typing import overload +class C: + @overload + def __init__(self, x: str) -> None: pass + @overload + def __init__(self, x: str, y: int) -> None: pass + def __init__(self, *args, **kwargs) -> None: + pass +[builtins fixtures/dict.pyi] +[out] +== +a.py:2: error: No overload variant of "B" matches argument type "int" +a.py:2: note: Possible overload variants: +a.py:2: note: def __init__(self, x: str) -> B +a.py:2: note: def __init__(self, x: str, y: int) -> B + +[case testOverloadedToNormalMethodMetaclass] +import a +[file a.py] +import b +b.B.f(int()) +[file b.py] +import c +class B(metaclass=c.M): + pass +[file c.py] +from typing import overload +class M(type): + @overload + def f(cls, x: str) -> str: pass + @overload + def f(cls, x: int) -> None: pass + def f(cls, x): + pass +[file c.py.2] +from typing import overload +class M(type): + def f(cls, x: str) -> str: + pass +[out] +== +a.py:2: error: Argument 1 to "f" of "M" has incompatible type "int"; expected "str" + +[case testYieldFrom] +from typing import Iterator +from a import f + +def g() -> Iterator[int]: + a = "string" + if int(): + a = yield from f() + +[file a.py] +from typing import Generator + +def f() -> Generator[int, None, str]: + yield 5 + return "ham" + +[file a.py.2] +from typing import Generator + +class A: pass + +def f() -> Generator[int, None, A]: + yield 5 + return A() + +[out] +== +main:7: error: Incompatible types in assignment (expression has type "A", variable has type "str") + +[case testFString] +from a import g +f'{g(1)}' +[file a.py] +def g(x: int) -> str: pass +[file a.py.2] +def g(x: str) -> str: pass +[builtins fixtures/f_string.pyi] +[out] +== +main:2: error: Argument 1 to "g" has incompatible type "int"; expected "str" + +[case testExtendedUnpacking-only_when_nocache] +from typing import List +from a import g +def f() -> List[int]: + a, *b = g() + return b + +[file a.py] +from typing import Tuple +def g() -> Tuple[str, int, int]: pass + +[file a.py.2] +from typing import Tuple +def g() -> Tuple[str, str]: pass + +[builtins fixtures/tuple.pyi] +[out] +== +main:5: error: Incompatible return value type (got "List[str]", expected "List[int]") + +[case testUnpackInExpression1-only_when_nocache] +from typing import Tuple, List +from a import t + +def f() -> Tuple[int, int]: + return (1, *t()) + +def g() -> List[int]: + return [1, *t()] + +[file a.py] +from typing import Tuple +def t() -> Tuple[int]: ... + +[file a.py.2] +from typing import Tuple +def t() -> Tuple[str]: ... + +[builtins fixtures/list.pyi] +[out] +== +main:5: error: Incompatible return value type (got "Tuple[int, str]", expected "Tuple[int, int]") +main:8: error: List item 1 has incompatible type "Tuple[str]"; expected "int" + +[case testUnpackInExpression2-only_when_nocache] +from typing import Set +from a import t + +def f() -> Set[int]: + return {1, *t()} + +[file a.py] +from typing import Tuple +def t() -> Tuple[int]: pass + +[file a.py.2] +from typing import Tuple +def t() -> Tuple[str]: pass + +[builtins fixtures/set.pyi] +[out] +== +main:5: error: Argument 2 to has incompatible type "*Tuple[str]"; expected "int" + +[case testUnpackInExpression3-only_when_nocache] +from typing import Dict +from a import d + +def f() -> Dict[int, str]: + return {1: '', **d()} + +[file a.py] +from typing import Dict +def d() -> Dict[int, str]: pass + +[file a.py.2] +from typing import Dict +def d() -> Dict[int, int]: pass + +[builtins fixtures/dict.pyi] +[out] +== +main:5: error: Unpacked dict entry 1 has incompatible type "Dict[int, int]"; expected "SupportsKeysAndGetItem[int, str]" + +[case testAwaitAndAsyncDef-only_when_nocache] +from a import g + +async def f() -> int: + return await g() + +[file a.py] +async def g() -> int: + return 0 + +[file a.py.2] +async def g() -> str: + return '' + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +== +main:4: error: Incompatible return value type (got "str", expected "int") + +[case testAwaitAnd__await__-only_when_nocache] +from a import C + +async def f(c: C) -> int: + return await c + +[file a.py] +from typing import Any, Generator +class C: + def __await__(self) -> Generator[Any, None, int]: + yield + return 0 + +[file a.py.2] +from typing import Any, Generator +class C: + def __await__(self) -> Generator[Any, None, str]: + yield + return '' + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +== +main:4: error: Incompatible return value type (got "str", expected "int") + +[case test__aiter__and__anext__] +from a import C + +async def f() -> int: + async for x in C(): + pass + return x + +[file a.py] +class C: + def __aiter__(self) -> D: pass +class D: + def __aiter__(self) -> D: pass + async def __anext__(self) -> int: return 0 + +[file a.py.2] +class C: + def __aiter__(self) -> D: pass +class D: + def __aiter__(self) -> D: pass + async def __anext__(self) -> str: return '' + +[file a.py.3] +class C: + def __aiter__(self) -> E: pass +class E: + def __aiter__(self) -> E: pass + async def __anext__(self) -> object: return 0 + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +== +main:6: error: Incompatible return value type (got "str", expected "int") +== +main:6: error: Incompatible return value type (got "object", expected "int") + +[case testAsyncWith2-only_when_nocache] +from a import C + +async def f() -> int: + async with C() as x: + return x + +async def g() -> None: + async with C(): pass + +[file a.py] +class C: + async def __aenter__(self) -> int: pass + async def __aexit__(self, x, y, z) -> None: pass + +[file a.py.2] +class C: + async def __aenter__(self) -> str: pass + async def __aexit__(self, x, y, z) -> None: pass + +[file a.py.3] +from typing import Awaitable +class C: + async def __aenter__(self) -> int: pass + async def __aexit__(self, x, y, z) -> None: pass + +[file a.py.4] +from typing import Awaitable +class C: + async def __aenter__(self) -> int: pass + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] +[out] +== +main:5: error: Incompatible return value type (got "str", expected "int") +== +== +main:4: error: "C" has no attribute "__aexit__" +main:8: error: "C" has no attribute "__aexit__" + +[case testLiskovFineVariable] +import b +class A(b.B): + x: str +def f(x: b.B) -> None: + x.x + int() +f(A()) +[file b.py] +class B: + x: str +[file b.py.2] +class B: + x: int +[builtins fixtures/primitives.pyi] +[out] +main:5: error: Unsupported operand types for + ("str" and "int") +== +main:3: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testLiskovFineVariableInFunction] +from b import B +def outer() -> None: + class A(B): + x: str + def f(x: B) -> None: + x.x + int() +[file b.py] +class B: + x: str +[file b.py.2] +class B: + x: int +[builtins fixtures/primitives.pyi] +[out] +main:6: error: Unsupported operand types for + ("str" and "int") +== +main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testLiskovFineDecorator] +import b +from c import deco +class A(b.B): + @deco + def m(self) -> str: pass +def f(x: b.B) -> None: + x.m() + int() +f(A()) +[file b.py] +from c import deco +class B: + @deco + def m(self) -> str: pass +[file b.py.2] +from c import deco +class B: + @deco + def m(self) -> int: pass +[file c.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) +def deco(f: F) -> F: + pass +[builtins fixtures/primitives.pyi] +[out] +main:7: error: Unsupported operand types for + ("str" and "int") +== +main:5: error: Return type "str" of "m" incompatible with return type "int" in supertype "B" + +[case testLiskovFineVariableClean-only_when_nocache] +import b +class A(b.B): + x: str +[file b.py] +class B: + x: str +[file b.py.2] +class B: + x: int +[out] +== +main:3: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testLiskovFineVariableCleanDefInMethod-only_when_nocache] +import b +class A(b.B): + def meth(self) -> None: + self.x: str +[file b.py] +class B: + x: str +[file b.py.2] +class B: + x: int +[out] +== +main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testLiskovFineVariableCleanDefInMethodStar-only_when_nocache] +from typing import List +import b +class A(b.B): + def meth(self) -> None: + self.x: str + self.y: List[str] +[file b.py] +from typing import List +class B: + y: List[str] +[file b.py.2] +from typing import List +class B: + y: List[int] +[builtins fixtures/list.pyi] +[out] +== +main:6: error: Incompatible types in assignment (expression has type "List[str]", base class "B" defined the type as "List[int]") + +[case testLiskovFineVariableCleanDefInMethodNested-only_when_nocache] +from b import B +def outer() -> None: + class A(B): + def meth(self) -> None: + self.x: str +[file b.py] +class B: + x: str +[file b.py.2] +class B: + x: int +[out] +== +main:5: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testLiskovFineVariableInFunctionClean-only_when_nocache] +from b import B +def outer() -> None: + class A(B): + x: str +[file b.py] +class B: + x: str +[file b.py.2] +class B: + x: int +[out] +== +main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int") + +[case testLiskovFineDecoratorClean-only_when_nocache] +import b +from c import deco +class A(b.B): + @deco + def m(self) -> str: pass +[file b.py] +from c import deco +class B: + @deco + def m(self) -> str: pass +[file b.py.2] +from c import deco +class B: + @deco + def m(self) -> int: pass +[file c.py] +from typing import Callable, TypeVar +F = TypeVar('F', bound=Callable) +def deco(f: F) -> F: + pass +[out] +== +main:5: error: Return type "str" of "m" incompatible with return type "int" in supertype "B" + +[case testAddAbstractMethod] +from b import D +D() +[file b.py] +from a import C +class D(C): + def f(self) -> None: pass +[file a.py] +from abc import abstractmethod +class C: + @abstractmethod + def f(self) -> None: pass +[file a.py.2] +from abc import abstractmethod +class C: + @abstractmethod + def f(self) -> None: pass + @abstractmethod + def g(self) -> None: pass +[file a.py.3] +from abc import abstractmethod +class C: + @abstractmethod + def f(self) -> None: pass + def g(self) -> None: pass +[out] +== +main:2: error: Cannot instantiate abstract class "D" with abstract attribute "g" +== + +[case testMakeClassAbstract] +from a import C +c = C() +[file a.py] +from abc import abstractmethod +class C: pass +[file a.py.2] +from abc import abstractmethod +class C: + @abstractmethod + def f(self) -> None: pass +[out] +== +main:2: error: Cannot instantiate abstract class "C" with abstract attribute "f" + +[case testMakeMethodNoLongerAbstract1] +[file z.py] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + @abstractmethod + def g(self) -> None: pass +[file b.py] +import z +def x() -> Foo: return Foo() +class Foo(z.I): + def f(self) -> None: pass + def g(self) -> None: pass + +[file z.py.2] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + def f(self) -> None: pass + @abstractmethod + def g(self) -> None: pass + +[file b.py.2] +import z +def x() -> Foo: return Foo() +class Foo(z.I): + def g(self) -> None: pass +[out] +== + +[case testMakeMethodNoLongerAbstract2] +-- this version never failed, but it is just a file-renaming +-- away from the above test that did +[file a.py] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + @abstractmethod + def g(self) -> None: pass +[file b.py] +import a +def x() -> Foo: return Foo() +class Foo(a.I): + def f(self) -> None: pass + def g(self) -> None: pass + +[file a.py.2] +from abc import abstractmethod, ABCMeta +class I(metaclass=ABCMeta): + def f(self) -> None: pass + @abstractmethod + def g(self) -> None: pass + +[file b.py.2] +import a +def x() -> Foo: return Foo() +class Foo(a.I): + def g(self) -> None: pass +[out] +== + +[case testImplicitOptionalRefresh1] +# flags: --implicit-optional +from x import f +def foo(x: int = None) -> None: + f() +[file x.py] +def f() -> int: return 0 +[file x.py.2] +def f() -> str: return '0' +[out] +== + +[case testRefreshIgnoreErrors1] +[file mypy.ini] +\[mypy] +\[mypy-b] +ignore_errors = True +[file a.py] +y = '1' +[file a.py.2] +y = 1 +[file b.py] +from a import y +def fu() -> None: + 1+'lurr' + y +[out] +== + +[case testRefreshIgnoreErrors2] +[file mypy.ini] +\[mypy] +\[mypy-b] +ignore_errors = True +[file b.py] +def fu() -> int: + 1+'lurr' + return 1 +[file b.py.2] +def fu() -> int: + 1+'lurr' + return 2 +[out] +== + +[case testRefreshOptions] +[file mypy.ini] +\[mypy] +disallow_any_generics = True +\[mypy-b] +disallow_any_generics = False +[file a.py] +y = '1' +[file a.py.2] +y = 1 +[file b.py] +from typing import List +from a import y +x = [] # type: List +[builtins fixtures/list.pyi] +[out] +== + +[case testNamedTupleFallbackModule] +import b +[file b.py] +from a import A +def f(a: A): + pass +[file b.py.2] +from a import A +def f(a: A): + reveal_type(a) +[file a.py] +from typing import NamedTuple + +F = [('x', int)] +A = NamedTuple('A', F) # type: ignore +[builtins fixtures/list.pyi] +[out] +== +b.py:3: note: Revealed type is "Tuple[(), fallback=a.A]" + +[case testImportOnTopOfAlias1] +from a import A +x: A +[file a.py] +from typing import TypeVar, List +T = TypeVar('T') +A = List[T] +[file a.py.2] +from typing import TypeVar, List +T = TypeVar('T') +A = List[T] +from b import A +[file b.py] +# empty +[builtins fixtures/list.pyi] +[out] +== +a.py:4: error: Module "b" has no attribute "A" +a.py:4: error: Name "A" already defined on line 3 + +-- the order of errors is different with cache +[case testImportOnTopOfAlias2] +from a import A +x: A +[file a.py] +from typing import TypeVar, List +T = TypeVar('T') +A = List[T] +[file a.py.2] +from typing import TypeVar, List +T = TypeVar('T') +A = List[T] +from b import A as A +[file b.py] +def A(x: str) -> str: pass +[builtins fixtures/list.pyi] +[out] +== +a.py:4: error: Incompatible import of "A" (imported name has type "Callable[[str], str]", local name has type "Type[List[Any]]") + +[case testFakeOverloadCrash] +import b +[file a.py] +def dec(fun): + pass +a = 1 +[file a.py.2] +def dec(fun): + pass +a = 2 +[file b.py] +from a import dec +@dec +def a(): + pass +@dec +def a(): + pass +[out] +b.py:5: error: Name "a" already defined on line 2 +== +b.py:5: error: Name "a" already defined on line 2 + +[case testFakeOverloadCrash2] + +# this test just should not crash +import a +[file a.py] +T = TypeVar("T") + +def foo(func): + return func + +@foo +def bar(x: T) -> T: + pass + +@foo +def bar(x: T) -> T: + pass +[file a.py.2] +T = TypeVar("T") + +def foo(func): + return func + +@foo +def bar(x: T) -> T: + pass + +@foo +def bar(x: T) -> T: + pass +x = 1 +[out] +a.py:1: error: Name "TypeVar" is not defined +a.py:1: note: Did you forget to import it from "typing"? (Suggestion: "from typing import TypeVar") +a.py:7: error: Variable "a.T" is not valid as a type +a.py:7: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +a.py:10: error: Name "bar" already defined on line 6 +a.py:11: error: Variable "a.T" is not valid as a type +a.py:11: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +== +a.py:1: error: Name "TypeVar" is not defined +a.py:1: note: Did you forget to import it from "typing"? (Suggestion: "from typing import TypeVar") +a.py:7: error: Variable "a.T" is not valid as a type +a.py:7: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +a.py:10: error: Name "bar" already defined on line 6 +a.py:11: error: Variable "a.T" is not valid as a type +a.py:11: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testRefreshForWithTypeComment1] +[file a.py] +from typing import List +import b +def foo(l: List[int]) -> None: + for x in l: # type: object + pass + x = object() + b.x +[file b.py] +x = 1 +[file b.py.2] +x = '1' +[builtins fixtures/list.pyi] +[out] +== + +[case testRefreshForWithTypeComment2] +from typing import List, Any +import m +def f(x: List[Any]) -> None: + for a in x: # type: m.A + pass +[file m.py] +class A: pass +[file m.py.2] +[builtins fixtures/list.pyi] +[out] +== +main:4: error: Name "m.A" is not defined + +[case testIdLikeDecoForwardCrash] +import b +[file b.py] +from typing import Callable, Any, TypeVar + +F = TypeVar('F_BadName', bound=Callable[..., Any]) # type: ignore +def deco(func: F) -> F: # type: ignore + pass + +@deco +def test(x: int, y: int) -> str: + pass +[file b.py.2] +from typing import Callable, Any, TypeVar + +F = TypeVar('F_BadName', bound=Callable[..., Any]) # type: ignore +def deco(func: F) -> F: # type: ignore + pass + +@deco +def test(x: int, y: int) -> str: + pass +x = 1 +[builtins fixtures/tuple.pyi] +[out] +== + +[case testIdLikeDecoForwardCrashAlias] +# flags: --disable-error-code used-before-def +import b +[file b.py] +from typing import Callable, Any, TypeVar + +F = TypeVar('F', bound=Func) +def deco(func: F) -> F: + pass + +@deco +def test(x: int, y: int) -> str: + pass +Func = Callable[..., Any] +[file b.py.2] +from typing import Callable, Any, TypeVar + +F = TypeVar('F', bound=Func) +def deco(func: F) -> F: + pass + +@deco +def test(x: int, y: int) -> str: + pass +x = 1 +Func = Callable[..., Any] +[out] +== + +-- Test cases for final qualifier + +[case testFinalAddFinalVarAssignFine] +import mod +from a import D +from mod import x + +x = 2 +def outer() -> None: + mod.x = 2 + x = 2 # This is OK because it creates a local variable + d: D + d.y = 2 + d.z = 2 + D.y = 2 +[file a.py] +import mod + +class D(mod.C): + pass +[file mod.py] +x = 1 +class C: + y = 1 + def __init__(self) -> None: + self.z = 1 + +[file mod.py.2] +from typing import Final + +x: Final = 1 +class C: + y: Final = 1 + def __init__(self) -> None: + self.z: Final = 1 +[out] +== +main:5: error: Cannot assign to final name "x" +main:7: error: Cannot assign to final name "x" +main:10: error: Cannot assign to final attribute "y" +main:11: error: Cannot assign to final attribute "z" +main:12: error: Cannot assign to final attribute "y" + +[case testFinalAddFinalVarOverrideFine] +from mod import C + +class D(C): + x = 2 + def __init__(self) -> None: + self.y = 2 +class E(C): + y = 2 + def __init__(self) -> None: + self.x = 2 + +[file mod.py] +class C: + x = 1 + def __init__(self) -> None: + self.y = 1 + +[file mod.py.2] +from typing import Final + +class C: + x: Final = 1 + def __init__(self) -> None: + self.y: Final = 1 +[out] +== +main:4: error: Cannot assign to final name "x" +main:6: error: Cannot assign to final attribute "y" +main:8: error: Cannot assign to final name "y" +main:10: error: Cannot assign to final attribute "x" + +[case testFinalAddFinalMethodOverrideFine] +from mod import C + +class D(C): + def meth(self) -> int: ... + +[file mod.py] +class C: + def meth(self) -> int: ... + +[file mod.py.2] +from typing import final + +class C: + @final + def meth(self) -> int: ... +[out] +== +main:4: error: Cannot override final attribute "meth" (previously declared in base class "C") + +[case testFinalAddFinalMethodOverrideWithVarFine] +from mod import C +from typing import Any + +class D(C): + meth: Any = 2 + def __init__(self) -> None: + self.other: Any = 2 + +[file mod.py] +class C: + def meth(self) -> int: ... + def other(self) -> int: ... + +[file mod.py.2] +from typing import final + +class C: + @final + def meth(self) -> int: ... + @final + def other(self) -> int: ... +[out] +== +main:5: error: Cannot override final attribute "meth" (previously declared in base class "C") +main:7: error: Cannot assign to final attribute "other" +main:7: error: Cannot override final attribute "other" (previously declared in base class "C") + +[case testFinalAddFinalMethodOverrideOverloadFine] +from typing import overload +from mod import C + +def outer() -> None: + class D(C): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x): + pass + +[file mod.pyi] +from typing import overload +class C: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + +[file mod.pyi.2] +from typing import final, overload + +class C: + @final + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +[out] +== +main:6: error: Cannot override final attribute "meth" (previously declared in base class "C") + +[case testFinalAddFinalPropertyWithVarFine] +from mod import C + +def outer() -> None: + class D(C): + p = 2 + class E(C): + def __init__(self) -> None: + self.p: int = 2 + +[file mod.py] +class C: + @property + def p(self) -> int: + pass + +[file mod.py.2] +from typing import final + +class C: + @final + @property + def p(self) -> int: + pass +[builtins fixtures/property.pyi] +[out] +== +main:5: error: Cannot override final attribute "p" (previously declared in base class "C") +main:8: error: Cannot assign to final attribute "p" +main:8: error: Cannot override final attribute "p" (previously declared in base class "C") + +[case testFinalBodyReprocessedAndStillFinal] +import a +[file a.py] +from c import C +class A: + def meth(self) -> None: ... + +[file a.py.3] +from c import C +class A(C): + def meth(self) -> None: ... + +[file c.py] +from typing import final +from d import D + +class C: + @final + def meth(self) -> None: + D(int()) +[file d.py] +class D: + def __init__(self, x: int) -> None: ... +[file d.py.2] +from typing import Optional +class D: + def __init__(self, x: Optional[int]) -> None: ... +[out] +== +== +a.py:3: error: Cannot override final attribute "meth" (previously declared in base class "C") + +[case testFinalBodyReprocessedAndStillFinalOverloaded-only_when_cache] +-- Different cache/no-cache tests because +-- CallableType.def_extras.first_arg differs ("self"/None) +import a +[file a.py] +from c import C +class A: + def meth(self) -> None: ... + +[file a.py.3] +from c import C +class A(C): + def meth(self) -> None: ... + +[file c.py] +from typing import final, overload, Union +from d import D + +class C: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + @final + def meth(self, x: Union[int, str]) -> Union[int, str]: + D(int()) + return x +[file d.py] +class D: + def __init__(self, x: int) -> None: ... +[file d.py.2] +from typing import Optional +class D: + def __init__(self, x: Optional[int]) -> None: ... +[out] +== +== +a.py:3: error: Cannot override final attribute "meth" (previously declared in base class "C") +a.py:3: error: Signature of "meth" incompatible with supertype "C" +a.py:3: note: Superclass: +a.py:3: note: @overload +a.py:3: note: def meth(self, x: int) -> int +a.py:3: note: @overload +a.py:3: note: def meth(self, x: str) -> str +a.py:3: note: Subclass: +a.py:3: note: def meth(self) -> None + +[case testFinalBodyReprocessedAndStillFinalOverloaded2-only_when_nocache] +-- Different cache/no-cache tests because +-- CallableType.def_extras.first_arg differs ("self"/None) +import a +[file a.py] +from c import C +class A: + def meth(self) -> None: ... + +[file a.py.3] +from c import C +class A(C): + def meth(self) -> None: ... + +[file c.py] +from typing import final, overload, Union +from d import D + +class C: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + @final + def meth(self, x: Union[int, str]) -> Union[int, str]: + D(int()) + return x +[file d.py] +class D: + def __init__(self, x: int) -> None: ... +[file d.py.2] +from typing import Optional +class D: + def __init__(self, x: Optional[int]) -> None: ... +[out] +== +== +a.py:3: error: Cannot override final attribute "meth" (previously declared in base class "C") +a.py:3: error: Signature of "meth" incompatible with supertype "C" +a.py:3: note: Superclass: +a.py:3: note: @overload +a.py:3: note: def meth(x: int) -> int +a.py:3: note: @overload +a.py:3: note: def meth(x: str) -> str +a.py:3: note: Subclass: +a.py:3: note: def meth(self) -> None + +[case testIfMypyUnreachableClass] +from a import x + +MYPY = False +if MYPY: + pass +else: + class A: + pass +y: int = x +[file a.py] +x = 1 +[file a.py.2] +x = 2 +[builtins fixtures/bool.pyi] +[out] +== + +[case testIfTypeCheckingUnreachableClass] +from a import x +from typing import TYPE_CHECKING + +if not TYPE_CHECKING: + class A(int): + pass +else: + A = int + +y: A = x +[file a.py] +x = 1 +[file a.py.2] +x = 2 +[file a.py.3] +x = 'no way' +[builtins fixtures/bool.pyi] +[typing fixtures/typing-medium.pyi] +[out] +== +== +main:10: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleForwardFunctionDirect] +# flags: --ignore-missing-imports +from typing import NamedTuple +from b import B + +NT = NamedTuple('NT', [('x', B)]) +[file b.py.2] +def func(x): pass +B = func +[builtins fixtures/tuple.pyi] +[out] +== +main:5: error: Variable "b.B" is not valid as a type +main:5: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testNamedTupleForwardFunctionIndirect] +# flags: --ignore-missing-imports +from typing import NamedTuple +from a import A + +NT = NamedTuple('NT', [('x', A)]) +[file a.py] +from b import B +A = B +[file b.py.2] +def func(x): pass +B = func +[builtins fixtures/tuple.pyi] +[out] +== +main:5: error: Variable "a.A" is not valid as a type +main:5: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testNamedTupleForwardFunctionIndirectReveal] +# flags: --ignore-missing-imports +import m +[file m.py] +from typing import NamedTuple +from a import A + +NT = NamedTuple('NT', [('x', A)]) +[file m.py.3] +from typing import NamedTuple +from a import A + +NT = NamedTuple('NT', [('x', A)]) +reveal_type(NT.x) +x: NT +reveal_type(x.x) +[file a.py] +from b import B +A = B +[file b.py.2] +def func(x): pass +B = func +[builtins fixtures/tuple.pyi] +[out] +== +m.py:4: error: Variable "a.A" is not valid as a type +m.py:4: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +== +m.py:4: error: Variable "a.A" is not valid as a type +m.py:4: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +m.py:5: note: Revealed type is "Any" +m.py:7: note: Revealed type is "Any" + +[case testAliasForwardFunctionDirect] +# flags: --ignore-missing-imports +from typing import Optional +from b import B + +Alias = Optional[B] +[file b.py.2] +def func(x): pass +B = int() +[out] +== +main:5: error: Variable "b.B" is not valid as a type +main:5: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testAliasForwardFunctionIndirect] +# flags: --ignore-missing-imports +from typing import Optional +from a import A + +Alias = Optional[A] +[file a.py] +from b import B +A = B +[file b.py.2] +def func(x): pass +B = func +[out] +== +main:5: error: Variable "a.A" is not valid as a type +main:5: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testLiteralFineGrainedVarConversion] +import mod +reveal_type(mod.x) +[file mod.py] +x = 1 +[file mod.py.2] +from typing_extensions import Literal +x: Literal[1] = 1 +[file mod.py.3] +from typing_extensions import Literal +x: Literal[1] = 2 +[builtins fixtures/tuple.pyi] +[out] +main:2: note: Revealed type is "builtins.int" +== +main:2: note: Revealed type is "Literal[1]" +== +main:2: note: Revealed type is "Literal[1]" +mod.py:2: error: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]") + +[case testLiteralFineGrainedFunctionConversion] +from mod import foo +foo(3) +[file mod.py] +def foo(x: int) -> None: pass +[file mod.py.2] +from typing_extensions import Literal +def foo(x: Literal[3]) -> None: pass +[file mod.py.3] +from typing_extensions import Literal +def foo(x: Literal[4]) -> None: pass +[builtins fixtures/tuple.pyi] +[out] +== +== +main:2: error: Argument 1 to "foo" has incompatible type "Literal[3]"; expected "Literal[4]" + +[case testLiteralFineGrainedAlias] +from mod import Alias +a: Alias = 1 +[file mod.py] +Alias = int +[file mod.py.2] +from typing_extensions import Literal +Alias = Literal[1] +[file mod.py.3] +from typing_extensions import Literal +Alias = Literal[2] +[builtins fixtures/tuple.pyi] +[out] +== +== +main:2: error: Incompatible types in assignment (expression has type "Literal[1]", variable has type "Literal[2]") + +[case testLiteralFineGrainedOverload] +from mod import foo +reveal_type(foo(4)) +[file mod.py] +from typing import overload +from typing_extensions import Literal +@overload +def foo(x: int) -> str: ... +@overload +def foo(x: Literal['bar']) -> int: ... +def foo(x): pass +[file mod.py.2] +from typing import overload +from typing_extensions import Literal +@overload +def foo(x: Literal[4]) -> Literal['foo']: ... +@overload +def foo(x: int) -> str: ... +@overload +def foo(x: Literal['bar']) -> int: ... +def foo(x): pass +[builtins fixtures/tuple.pyi] +[out] +main:2: note: Revealed type is "builtins.str" +== +main:2: note: Revealed type is "Literal['foo']" + +[case testLiteralFineGrainedChainedDefinitions] +from mod1 import foo +from typing_extensions import Literal +def expect_3(x: Literal[3]) -> None: pass +expect_3(foo) +[file mod1.py] +from mod2 import bar +foo = bar +[file mod2.py] +from mod3 import qux as bar +[file mod3.py] +from typing_extensions import Literal +qux: Literal[3] +[file mod3.py.2] +from typing_extensions import Literal +qux: Literal[4] +[builtins fixtures/tuple.pyi] +[out] +== +main:4: error: Argument 1 to "expect_3" has incompatible type "Literal[4]"; expected "Literal[3]" + +[case testLiteralFineGrainedChainedAliases] +from mod1 import Alias1 +from typing_extensions import Literal +x: Alias1 +def expect_3(x: Literal[3]) -> None: pass +expect_3(x) +[file mod1.py] +from mod2 import Alias2 +Alias1 = Alias2 +[file mod2.py] +from mod3 import Alias3 +Alias2 = Alias3 +[file mod3.py] +from typing_extensions import Literal +Alias3 = Literal[3] +[file mod3.py.2] +from typing_extensions import Literal +Alias3 = Literal[4] +[builtins fixtures/tuple.pyi] +[out] +== +main:5: error: Argument 1 to "expect_3" has incompatible type "Literal[4]"; expected "Literal[3]" + +[case testLiteralFineGrainedChainedFunctionDefinitions] +from mod1 import func1 +from typing_extensions import Literal +def expect_3(x: Literal[3]) -> None: pass +expect_3(func1()) +[file mod1.py] +from mod2 import func2 as func1 +[file mod2.py] +from mod3 import func3 +func2 = func3 +[file mod3.py] +from typing_extensions import Literal +def func3() -> Literal[3]: pass +[file mod3.py.2] +from typing_extensions import Literal +def func3() -> Literal[4]: pass +[builtins fixtures/tuple.pyi] +[out] +== +main:4: error: Argument 1 to "expect_3" has incompatible type "Literal[4]"; expected "Literal[3]" + +[case testLiteralFineGrainedChainedTypeVarInference] +from mod1 import foo +reveal_type(foo) +[file mod1.py] +from typing import TypeVar +from mod2 import bar +T = TypeVar('T', bound=int) +def func(x: T) -> T: return x +foo = func(bar) +[file mod2.py] +bar = 3 +[file mod2.py.2] +from typing_extensions import Literal +bar: Literal[3] = 3 +[builtins fixtures/tuple.pyi] +[out] +main:2: note: Revealed type is "builtins.int" +== +main:2: note: Revealed type is "Literal[3]" + +[case testLiteralFineGrainedChainedViaFinal] +from mod1 import foo +from typing_extensions import Literal +def expect_3(x: Literal[3]) -> None: pass +expect_3(foo) +[file mod1.py] +from typing_extensions import Final +from mod2 import bar +foo: Final = bar +[file mod2.py] +from mod3 import qux as bar +[file mod3.py] +from typing_extensions import Final +qux: Final = 3 +[file mod3.py.2] +from typing_extensions import Final +qux: Final = 4 +[file mod3.py.3] +from typing_extensions import Final +qux: Final[int] = 4 +[builtins fixtures/tuple.pyi] +[out] +== +main:4: error: Argument 1 to "expect_3" has incompatible type "Literal[4]"; expected "Literal[3]" +== +main:4: error: Argument 1 to "expect_3" has incompatible type "int"; expected "Literal[3]" + +[case testLiteralFineGrainedStringConversionPython3] +from mod1 import foo +reveal_type(foo) +[file mod1.py] +from mod2 import bar +foo = bar() +[file mod2.py] +from typing_extensions import Literal +def bar() -> Literal["foo"]: pass +[file mod2.py.2] +from typing_extensions import Literal +def bar() -> Literal[u"foo"]: pass +[file mod2.py.3] +from typing_extensions import Literal +def bar() -> Literal[b"foo"]: pass +[builtins fixtures/tuple.pyi] +[out] +main:2: note: Revealed type is "Literal['foo']" +== +main:2: note: Revealed type is "Literal['foo']" +== +main:2: note: Revealed type is "Literal[b'foo']" + +[case testReprocessModuleTopLevelWhileMethodDefinesAttr] +import a +[file a.py] +from b import B + +B().x + +[file a.py.3] +from b import B + +x = B().x + +[file b.py] +from c import f +f(int()) + +class B: + def meth(self) -> None: + self.x: int + +[file c.py] +def f(x: int) -> None: ... + +[file c.py.2] +from typing import Union +def f(x: Union[int, str]) -> None: ... + +[targets2 c, b] +[targets3 a] +[builtins fixtures/tuple.pyi] +[out] +== +== + +[case testCheckReprocessedTargets-only_when_nocache] +from b import B + +class C(B): + def meth(self) -> None: + from b import f + f() + +[file b.py] +class B: ... +def f() -> int: ... + +[file b.py.2] +class A: ... +class B(A): ... +def f() -> int: ... + +[file b.py.3] +class A: ... +class B(A): ... +def f() -> str: ... + +[targets2 b, __main__, __main__.C.meth, __main__, __main__.C.meth] +[targets3 b, __main__.C.meth] +[out] +== +== + +[case testReprocessModuleTopLevelWhileMethodDefinesAttrExplicit] +import a +[file a.py] +from b import B + +B().x + +[file b.py] +from c import f +f(int()) + +class A: + x: int + +class B(A): + def meth(self) -> None: + self.x: int + +[file c.py] +def f(x: int) -> None: ... + +[file c.py.2] +from typing import Union +def f(x: Union[int, str]) -> None: ... + +[file a.py.3] +from b import B + +# Double-check the variable is still accessible. +B().x + +[targets2 c, b] +[targets3 a] +[builtins fixtures/tuple.pyi] +[out] +== +== + +[case testReprocessModuleTopLevelWhileMethodDefinesAttrBothExplicitAndInClass] +import a +[file a.py] +from b import B + +B().x + +[file b.py] +from c import f +f(int()) + +class A: + x: int + +class B(A): + x: int + def meth(self) -> None: + self.x: int + +[file c.py] +def f(x: int) -> None: ... + +[file c.py.2] +from typing import Union +def f(x: Union[int, str]) -> None: ... + +[file a.py.3] +from b import B + +# Double-check the variable is still accessible. +B().x + +[targets2 c, b] +[targets3 a] +[builtins fixtures/tuple.pyi] +[out] +== +== + +[case testReprocessModuleTopLevelWhileMethodDefinesAttrProtocol] +import a +[file a.py] +from b import B + +B().x + +[file b.py] +from typing import Protocol +from c import f +f(int()) + +class A(Protocol): + x: int + +class B(A): + def meth(self) -> None: + self.x = 42 + +[file c.py] +def f(x: int) -> None: ... + +[file c.py.2] +from typing import Union +def f(x: Union[int, str]) -> None: ... + +[file a.py.3] +from b import B + +# Double-check the variable is still accessible. +B().x + +[targets2 c, b] +[targets3 a] +[out] +== +== + +[case testNewSemanticAnalyzerUpdateMethodAndClass] + +import m + +m.x + +class A: + def f(self) -> None: + self.x = 0 + m.y + + def g(self) -> None: + m.x + +[file m.py] +x = 0 +y = 0 + +[file m.py.2] +x = '' +y = 0 + +[file m.py.3] +x = '' +y = '' + +[out] +== +== + +[case testInlineConfigFineGrained1] + +import a +[file a.py] +# mypy: no-warn-no-return + +from typing import List, Optional +def foo() -> Optional[List]: + 20 + +[file a.py.2] +# mypy: disallow-any-generics, no-warn-no-return + +from typing import List, Optional +def foo() -> Optional[List]: + 20 + +[file a.py.3] +# mypy: no-warn-no-return + +from typing import List, Optional +def foo() -> Optional[List]: + 20 + +[file a.py.4] +from typing import List, Optional +def foo() -> Optional[List]: + 20 +[out] +== +a.py:4: error: Missing type parameters for generic type "List" +== +== +a.py:2: error: Missing return statement + +[builtins fixtures/list.pyi] + +[case testInlineConfigFineGrained2] +import a +[file a.py] +# mypy: bogus-flag + +[file b.py.2] + +[out] +a.py:1: error: Unrecognized option: bogus_flag = True +== +a.py:1: error: Unrecognized option: bogus_flag = True + +[case testWrongNumberOfArguments] + +[file a.py] +def bar(x): + # type: () -> None + pass + +def baz(x): + # type: () -> None + pass + +def f(): + # type: () -> None + def g(x): + # type: () -> None + pass + +[file c.py] +def bar(x): + # type: () -> None + pass + +[file b.py] +x = '' + +[file b.py.2] +x = 1 + +[out] +c.py:1: error: Type signature has too few arguments +a.py:1: error: Type signature has too few arguments +a.py:5: error: Type signature has too few arguments +a.py:11: error: Type signature has too few arguments +== +c.py:1: error: Type signature has too few arguments +a.py:1: error: Type signature has too few arguments +a.py:5: error: Type signature has too few arguments +a.py:11: error: Type signature has too few arguments + +[case testErrorReportingNewAnalyzer] +# flags: --disallow-any-generics +from a import A + +def f() -> None: + x: A +[file a.py] +class A: ... + +[file a.py.2] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): ... +[out] +== +main:5: error: Missing type parameters for generic type "A" + +[case testStripNewAnalyzer] +# flags: --ignore-missing-imports +import a +[file a.py] +from typing import List +from b import B + +class A: + def __init__(self) -> None: + self.x: List[int] = [] + + def method(self) -> None: + B() + self.x = [] + +[file b.py] +class B: ... +[delete b.py.2] + +[builtins fixtures/list.pyi] +[out] +== + +[case testClassVariableOrderingRefresh] + +from b import bar + +def foo(x: str) -> None: pass + +class Something: + def run(self) -> None: + bar() + foo(self.IDS[0]) + + IDS = [87] + +[file b.py] +def bar() -> int: return 0 +[file b.py.2] +def bar() -> str: return '0' +[builtins fixtures/list.pyi] +[out] +main:9: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +== +main:9: error: Argument 1 to "foo" has incompatible type "int"; expected "str" + +[case testInfiniteLoop] + +[file a.py] +from b import f +from typing import Callable, TypeVar + +F = TypeVar('F', bound=Callable) +def dec(x: F) -> F: return x + +@dec +def foo(self): + class A: + @classmethod + def asdf(cls, x: 'A') -> None: pass + +@dec +def bar(self): + class B: + @classmethod + def asdf(cls, x: 'B') -> None: pass + f() + +[file b.py] +def f() -> int: pass +[file b.py.2] +def f() -> str: pass +[builtins fixtures/classmethod.pyi] +[out] +== + +[case testInfiniteLoop2] + +[file a.py] +from b import f +from typing import Callable, TypeVar, NamedTuple + +F = TypeVar('F', bound=Callable) +def dec(x: F) -> F: return x + +@dec +def foo(self): + N = NamedTuple('N', [('x', int)]) + def g(x: N) -> None: pass + +@dec +def bar(self): + N = NamedTuple('N', [('x', int)]) + def g(x: N) -> None: pass + f() + +[file b.py] +def f() -> int: pass +[file b.py.2] +def f() -> str: pass +[builtins fixtures/classmethod.pyi] +[out] +== + +[case testFileAddedAndImported] +# flags: --ignore-missing-imports --follow-imports=skip +# cmd: mypy a.py +# cmd2: mypy a.py b.py + +[file a.py] +from b import bad +x = 42 + +[file b.py.2] +def good() -> None: ... +[out] +== +a.py:1: error: Module "b" has no attribute "bad" + +[case testFileAddedAndImported2] +# flags: --ignore-missing-imports --follow-imports=skip +# cmd: mypy -m a +# cmd2: mypy -m a b +[file a.py] +x = 42 + +[file a.py.2] +from b import bad +x = 42 + +[file b.py.2] +def good() -> None: ... +[out] +== +a.py:1: error: Module "b" has no attribute "bad" + +[case testTypedDictCrashFallbackAfterDeletedMeet] +# flags: --ignore-missing-imports + +from z import get_data +from a import Data + +for it in get_data()['things']: + it['id'] + +[file z.py] +from a import Data + +def get_data() -> Data: ... + +[file a.py] +from typing import TypedDict, List, Union + +class File(TypedDict): + id: int + name: str + +class User(TypedDict): + id: int + path: str + +class Data(TypedDict): + things: List[Union[File, User]] + +[delete a.py.2] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +== + +[case testTypedDictCrashFallbackAfterDeletedJoin] +# flags: --ignore-missing-imports + +from z import get_data +from a import Data + +x = [get_data()[0], get_data()[1]] + +[file z.py] +from a import Data + +def get_data() -> Data: ... + +[file a.py] +from typing import TypedDict, Tuple + +class File(TypedDict): + id: int + name: str + +class User(TypedDict): + id: int + path: str + +Data = Tuple[User, File] + +[delete a.py.2] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +== + +[case testClassRedef] +# An issue involved serializing these caused crashes in the past + +[file a.py] +class A: + pass + +x = 0 + +[file a.py.2] +class A: + a = A() + +x = '0' + +[file b.py] +from a import A, x + +class A: # type: ignore + pass + +[out] +== + +[case testAddAttributeThroughNewBaseClass] +import a + +[file a.py] +class C: + def __init__(self) -> None: + self.x = 0 + +[file a.py.2] +from b import B + +class C(B): + def __init__(self) -> None: + self.x = 0 + +[file b.py.2] +class B: + def __init__(self) -> None: + self.x = 0 +[out] +== + +[case testGenericChange1] +import a +[file a.py] +import b +def f() -> b.C: pass +[file b.py] +import a +class C: pass +[file b.py.2] +from typing import TypeVar, Generic, List +import a + +T = TypeVar('T') +class C(Generic[T]): pass + +reveal_type(a.f) +c: C[int] +l = a.f() if True else c +d = a.f() +d = c +c = d + +x: List[C] = [a.f(), a.f()] + +[out] +== +b.py:7: note: Revealed type is "def () -> b.C[Any]" +[builtins fixtures/list.pyi] + +[case testGenericChange2] +import a +[file a.py] +import b +def f() -> b.C[int]: pass +[file b.py] +from typing import TypeVar, Generic +import a +T = TypeVar('T') +class C(Generic[T]): pass +[file b.py.2] +from typing import List +import a + +class C(): pass + +c: C +l = a.f() if True else c +d = a.f() +d = c +c = d + +x: List[C] = [a.f(), a.f()] + +[builtins fixtures/list.pyi] +[out] +== +a.py:2: error: "C" expects no type arguments, but 1 given + +[case testGenericChange3] +import a +[file a.py] +import b +def f() -> b.C[int]: pass +[file b.py] +from typing import TypeVar, Generic +import a +T = TypeVar('T') +class C(Generic[T]): pass +[file b.py.2] +from typing import TypeVar, Generic, List +import a + +T = TypeVar('T') +S = TypeVar('S') +class C(Generic[S, T]): pass + +c: C[int, str] +l = a.f() if True else c +d = a.f() +d = c +c = d + +x: List[C] = [a.f(), a.f()] + +[out] +== +a.py:2: error: "C" expects 2 type arguments, but 1 given +[builtins fixtures/list.pyi] + +[case testIsInstanceAdHocIntersectionFineGrainedIncrementalNoChange] +import b +[file a.py] +class A: pass +class B: pass + +class Foo: + def __init__(self) -> None: + x: A + assert isinstance(x, B) + self.x = x +[file b.py] +from a import Foo +[file b.py.2] +from a import Foo +reveal_type(Foo().x) +[builtins fixtures/isinstance.pyi] +[out] +== +b.py:2: note: Revealed type is "a." + +[case testIsInstanceAdHocIntersectionFineGrainedIncrementalIsInstanceChange] +import c +[file a.py] +class A: pass +class B: pass +class C: pass + +class Foo: + def __init__(self) -> None: + x: A + assert isinstance(x, B) + self.x = x +[file a.py.2] +class A: pass +class B: pass +class C: pass + +class Foo: + def __init__(self) -> None: + x: A + assert isinstance(x, C) + self.x = x + +[file b.py] +from a import Foo +y = Foo().x + +[file c.py] +from b import y +reveal_type(y) +[builtins fixtures/isinstance.pyi] +[out] +c.py:2: note: Revealed type is "a." +== +c.py:2: note: Revealed type is "a." + +[case testIsInstanceAdHocIntersectionFineGrainedIncrementalUnderlyingObjChang] +import c +[file a.py] +class A: pass +class B: pass +class C: pass +Extra = B +[file a.py.2] +class A: pass +class B: pass +class C: pass +Extra = C + +[file b.py] +from a import A, Extra +x: A +if isinstance(x, Extra): + y = x + +[file c.py] +from b import y +reveal_type(y) +[builtins fixtures/isinstance.pyi] +[out] +c.py:2: note: Revealed type is "b." +== +c.py:2: note: Revealed type is "b." + +[case testIsInstanceAdHocIntersectionFineGrainedIncrementalIntersectionToUnreachable] +import c +[file a.py] +class A: + x: int +class B: + x: int +x: A +assert isinstance(x, B) +y = x + +[file a.py.2] +class A: + x: int +class B: + x: str +x: A +assert isinstance(x, B) +y = x + +[file b.py] +from a import y +z = y + +[file c.py] +from b import z +reveal_type(z) +[builtins fixtures/isinstance.pyi] +[out] +c.py:2: note: Revealed type is "a." +== +c.py:2: note: Revealed type is "Any" +b.py:2: error: Cannot determine type of "y" + +[case testIsInstanceAdHocIntersectionFineGrainedIncrementalUnreachaableToIntersection] +import c +[file a.py] +class A: + x: int +class B: + x: str +x: A +assert isinstance(x, B) +y = x + +[file a.py.2] +class A: + x: int +class B: + x: int +x: A +assert isinstance(x, B) +y = x + +[file b.py] +from a import y +z = y + +[file c.py] +from b import z +reveal_type(z) +[builtins fixtures/isinstance.pyi] +[out] +b.py:2: error: Cannot determine type of "y" +c.py:2: note: Revealed type is "Any" +== +c.py:2: note: Revealed type is "a." + +[case testStubFixupIssues] +[file a.py] +import p +[file a.py.2] +import p +# a change + +[file p/__init__.pyi] +from p.util import * + +[file p/util.pyi] +from p.params import N +class Test: ... + +[file p/params.pyi] +import p.util +class N(p.util.Test): + ... + +[builtins fixtures/list.pyi] +[out] +== + +[case testDunderCall1] +from a import C + +c = C() +c(1) + +[file a.py] +class C: + def __call__(self, x: int) -> None: ... + +[file a.py.2] +class C: + def __call__(self, x: str) -> None: ... + +[out] +== +main:4: error: Argument 1 to "__call__" of "C" has incompatible type "int"; expected "str" + +[case testDunderCall2] +from a import C + +C()(1) + +[file a.py] +class C: + def __call__(self, x: int) -> None: ... + +[file a.py.2] +class C: + def __call__(self, x: str) -> None: ... + +[out] +== +main:3: error: Argument 1 to "__call__" of "C" has incompatible type "int"; expected "str" + +[case testDunderCallAddition] +from a import C + +c = C() +x = c() # type: ignore +x + 42 + +[file a.py] +class C: ... + +[file a.py.2] +class C: + def __call__(self) -> str: ... + +[out] +== +main:5: error: Unsupported left operand type for + ("str") + +[case testNoneAttribute] +from typing import Generic, TypeVar + +T = TypeVar('T', int, str) + +class ExampleClass(Generic[T]): + def __init__( + self + ) -> None: + self.example_attribute = None +[out] +== +[case testStrictNoneAttribute] +from typing import Generic, TypeVar + +T = TypeVar('T', int, str) + +class ExampleClass(Generic[T]): + def __init__( + self + ) -> None: + self.example_attribute = None +[out] +== + +[case testDataclassCheckTypeVarBoundsInReprocess] +from dataclasses import dataclass +from typing import ClassVar, Protocol, Dict, TypeVar, Generic +from m import x + +class DataclassProtocol(Protocol): + __dataclass_fields__: ClassVar[Dict] + +T = TypeVar("T", bound=DataclassProtocol) + +@dataclass +class MyDataclass: + x: int = 1 + +class MyGeneric(Generic[T]): ... +class MyClass(MyGeneric[MyDataclass]): ... + +[file m.py] +x: int +[file m.py.2] +x: str + +[builtins fixtures/dataclasses.pyi] +[out] +== + +[case testParamSpecCached] +import a + +[file a.py] +import b + +def f(x: int) -> str: return 'x' + +b.foo(f) + +[file a.py.2] +import b + +def f(x: int) -> str: return 'x' + +reveal_type(b.foo(f)) + +[file b.py] +from typing import TypeVar, Callable, Union +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") + +def foo(f: Callable[P, T]) -> Callable[P, Union[T, None]]: + return f + +[file b.py.2] +from typing import TypeVar, Callable, Union +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") + +def foo(f: Callable[P, T]) -> Callable[P, Union[T, None]]: + return f + +x = 0 # Arbitrary change to trigger reprocessing + +[builtins fixtures/dict.pyi] +[out] +== +a.py:5: note: Revealed type is "def (x: builtins.int) -> Union[builtins.str, None]" + +[case testTypeVarTupleCached] +import a + +[file a.py] +import b + +def f(x: int) -> str: return 'x' + +b.foo((1, 'x')) + +[file a.py.2] +import b + +reveal_type(b.foo((1, 'x'))) + +[file b.py] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +def foo(t: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + return t + +[file b.py.2] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +def foo(t: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + return t + +x = 0 # Arbitrary change to trigger reprocessing +[builtins fixtures/dict.pyi] +[out] +== +a.py:3: note: Revealed type is "Tuple[Literal[1]?, Literal['x']?]" + +[case testVariadicClassFineUpdateRegularToVariadic] +from typing import Any +from lib import C + +x: C[int, str] + +[file lib.py] +from typing import Generic, TypeVar + +T = TypeVar("T") +S = TypeVar("S") +class C(Generic[T, S]): ... + +[file lib.py.2] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): ... +[builtins fixtures/tuple.pyi] +[out] +== + +[case testVariadicClassFineUpdateVariadicToRegular] +from typing import Any +from lib import C + +x: C[int, str, int] + +[file lib.py] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): ... +[file lib.py.2] +from typing import Generic, TypeVar + +T = TypeVar("T") +S = TypeVar("S") +class C(Generic[T, S]): ... +[builtins fixtures/tuple.pyi] +[out] +== +main:4: error: "C" expects 2 type arguments, but 3 given + +-- Order of error messages is different, so we repeat the test twice. +[case testVariadicClassFineUpdateValidToInvalidCached-only_when_cache] +from typing import Any +from lib import C + +x: C[int, str] + +[file lib.py] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): ... + +[file lib.py.2] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Ts]): ... +[builtins fixtures/tuple.pyi] +[out] +== +main:4: error: "C" expects no type arguments, but 2 given +lib.py:5: error: Free type variable expected in Generic[...] + +[case testVariadicClassFineUpdateValidToInvalid-only_when_nocache] +from typing import Any +from lib import C + +x: C[int, str] + +[file lib.py] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): ... + +[file lib.py.2] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Ts]): ... +[builtins fixtures/tuple.pyi] +[out] +== +lib.py:5: error: Free type variable expected in Generic[...] +main:4: error: "C" expects no type arguments, but 2 given + +[case testVariadicClassFineUpdateInvalidToValid] +from typing import Any +from lib import C + +x: C[int, str] + +[file lib.py] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Ts]): ... + +[file lib.py.2] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): ... +[builtins fixtures/tuple.pyi] +[out] +lib.py:5: error: Free type variable expected in Generic[...] +main:4: error: "C" expects no type arguments, but 2 given +== + +[case testUnpackKwargsUpdateFine] +import m +[file shared.py] +from typing_extensions import TypedDict + +class Person(TypedDict): + name: str + age: int + +[file shared.py.2] +from typing_extensions import TypedDict + +class Person(TypedDict): + name: str + age: str + +[file lib.py] +from typing_extensions import Unpack +from shared import Person + +def foo(**kwargs: Unpack[Person]): + ... +[file m.py] +from lib import foo +foo(name='Jennifer', age=38) + +[builtins fixtures/dict.pyi] +[out] +== +m.py:2: error: Argument "age" to "foo" has incompatible type "int"; expected "str" + +[case testModuleAsProtocolImplementationFine] +import m +[file m.py] +from typing import Protocol +from lib import C + +class Options(Protocol): + timeout: int + def update(self) -> bool: ... + +def setup(options: Options) -> None: ... +setup(C().config) + +[file lib.py] +import default_config + +class C: + config = default_config + +[file default_config.py] +timeout = 100 +def update() -> bool: ... + +[file default_config.py.2] +timeout = 100 +def update() -> str: ... +[builtins fixtures/module.pyi] +[out] +== +m.py:9: error: Argument 1 to "setup" has incompatible type Module; expected "Options" +m.py:9: note: Following member(s) of Module "default_config" have conflicts: +m.py:9: note: Expected: +m.py:9: note: def update() -> bool +m.py:9: note: Got: +m.py:9: note: def update() -> str + +[case testBoundGenericMethodFine] +import main +[file main.py] +import lib +[file main.py.3] +import lib +reveal_type(lib.foo(42)) +[file lib/__init__.pyi] +from lib import context +foo = context.test.foo +[file lib/context.pyi] +from typing import TypeVar +import lib.other + +T = TypeVar("T") +class Test: + def foo(self, x: T, n: lib.other.C = ...) -> T: ... +test: Test + +[file lib/other.pyi] +class C: ... +[file lib/other.pyi.2] +class B: ... +class C(B): ... +[out] +== +== +main.py:2: note: Revealed type is "builtins.int" + +[case testBoundGenericMethodParamSpecFine] +import main +[file main.py] +import lib +[file main.py.3] +from typing import Callable +import lib +f: Callable[[], int] +reveal_type(lib.foo(f)) +[file lib/__init__.pyi] +from lib import context +foo = context.test.foo +[file lib/context.pyi] +from typing_extensions import ParamSpec +from typing import Callable +import lib.other + +P = ParamSpec("P") +class Test: + def foo(self, x: Callable[P, int], n: lib.other.C = ...) -> Callable[P, str]: ... +test: Test + +[file lib/other.pyi] +class C: ... +[file lib/other.pyi.2] +class B: ... +class C(B): ... +[builtins fixtures/dict.pyi] +[out] +== +== +main.py:4: note: Revealed type is "def () -> builtins.str" + +[case testAbstractBodyTurnsEmpty] +from b import Base + +class Sub(Base): + def meth(self) -> int: + return super().meth() + +[file b.py] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: return 0 + +[file b.py.2] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: ... +[out] +== +main:5: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe + +[case testAbstractBodyTurnsEmptyProtocol] +from b import Base + +class Sub(Base): + def meth(self) -> int: + return super().meth() + +[file b.py] +from typing import Protocol +class Base(Protocol): + def meth(self) -> int: return 0 +[file b.py.2] +from typing import Protocol +class Base(Protocol): + def meth(self) -> int: ... +[out] +== +main:5: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe + +[case testPrettyMessageSorting] +# flags: --pretty +import a + +[file a.py] +1 + '' +import b + +[file b.py] +object + 1 + +[file b.py.2] +object + 1 +1() + +[out] +b.py:1: error: Unsupported left operand type for + ("Type[object]") + object + 1 + ^~~~~~~~~~ +a.py:1: error: Unsupported operand types for + ("int" and "str") + 1 + '' + ^~ +== +b.py:1: error: Unsupported left operand type for + ("Type[object]") + object + 1 + ^~~~~~~~~~ +b.py:2: error: "int" not callable + 1() + ^~~ +a.py:1: error: Unsupported operand types for + ("int" and "str") + 1 + '' + ^~ + +[case testTypingSelfFine] +import m +[file lib.py] +from typing import Any + +class C: + def meth(self, other: Any) -> C: ... +[file lib.py.2] +from typing import Self + +class C: + def meth(self, other: Self) -> Self: ... + +[file n.py] +import lib +class D(lib.C): ... +[file m.py] +from n import D +d = D() +def test() -> None: + d.meth(42) +[out] +== +m.py:4: error: Argument 1 to "meth" of "C" has incompatible type "int"; expected "D" + +[case testNoNestedDefinitionCrash] +import m +[file m.py] +from typing import Any, TYPE_CHECKING + +class C: + if TYPE_CHECKING: + def __init__(self, **kw: Any): ... + +C +[file m.py.2] +from typing import Any, TYPE_CHECKING + +class C: + if TYPE_CHECKING: + def __init__(self, **kw: Any): ... + +C +# change +[builtins fixtures/dict.pyi] +[out] +== + +[case testNoNestedDefinitionCrash2] +import m +[file m.py] +from typing import Any + +class C: + try: + def __init__(self, **kw: Any): ... + except: + pass + +C +[file m.py.2] +from typing import Any + +class C: + try: + def __init__(self, **kw: Any): ... + except: + pass + +C +# change +[builtins fixtures/dict.pyi] +[out] +== + +[case testNamedTupleNestedCrash] +import m +[file m.py] +from typing import NamedTuple + +class NT(NamedTuple): + class C: ... + x: int + y: int + +[file m.py.2] +from typing import NamedTuple + +class NT(NamedTuple): + class C: ... + x: int + y: int +# change +[builtins fixtures/tuple.pyi] +[out] +m.py:4: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" +== +m.py:4: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" + +[case testNamedTupleNestedClassRecheck] +import n +[file n.py] +import m +x: m.NT +[file m.py] +from typing import NamedTuple +from f import A + +class NT(NamedTuple): + class C: ... + x: int + y: A + +[file f.py] +A = int +[file f.py.2] +A = str +[builtins fixtures/tuple.pyi] +[out] +m.py:5: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" +== +m.py:5: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" + +[case testTypedDictNestedClassRecheck] +import n +[file n.py] +import m +x: m.TD +[file m.py] +from typing_extensions import TypedDict +from f import A + +class TD(TypedDict): + class C: ... + x: int + y: A + +[file f.py] +A = int +[file f.py.2] +A = str +[builtins fixtures/dict.pyi] +[out] +m.py:5: error: Invalid statement in TypedDict definition; expected "field_name: field_type" +== +m.py:5: error: Invalid statement in TypedDict definition; expected "field_name: field_type" + +[case testTypeAliasWithNewStyleUnionChangedToVariable] +# flags: --python-version 3.10 +import a + +[file a.py] +from b import C, D +A = C | D +a: A +reveal_type(a) +[builtins fixtures/type.pyi] + +[file b.py] +C = int +D = str + +[file b.py.2] +C = "x" +D = "y" + +[file b.py.3] +C = str +D = int +[out] +a.py:4: note: Revealed type is "Union[builtins.int, builtins.str]" +== +a.py:2: error: Unsupported left operand type for | ("str") +a.py:3: error: Variable "a.A" is not valid as a type +a.py:3: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +a.py:4: note: Revealed type is "A?" +== +a.py:4: note: Revealed type is "Union[builtins.str, builtins.int]" + +[case testUnionOfSimilarCallablesCrash] +import b + +[file b.py] +from a import x + +[file m.py] +from typing import Union, TypeVar + +T = TypeVar("T") +S = TypeVar("S") +def foo(x: T, y: S) -> Union[T, S]: ... +def f(x: int) -> int: ... +def g(*x: int) -> int: ... + +[file a.py] +from m import f, g, foo +x = foo(f, g) + +[file a.py.2] +from m import f, g, foo +x = foo(f, g) +reveal_type(x) +[builtins fixtures/tuple.pyi] +[out] +== +a.py:3: note: Revealed type is "Union[def (x: builtins.int) -> builtins.int, def (*x: builtins.int) -> builtins.int]" + +[case testErrorInReAddedModule] +# flags: --disallow-untyped-defs --follow-imports=error +# cmd: mypy a.py +# cmd2: mypy b.py +# cmd3: mypy a.py + +[file a.py] +def f(): pass +[file b.py] +def f(): pass +[file unrelated.txt.3] +[out] +a.py:1: error: Function is missing a return type annotation +a.py:1: note: Use "-> None" if function does not return a value +== +b.py:1: error: Function is missing a return type annotation +b.py:1: note: Use "-> None" if function does not return a value +== +a.py:1: error: Function is missing a return type annotation +a.py:1: note: Use "-> None" if function does not return a value + +[case testModuleLevelGetAttrInStub] +import stub +import a +import b + +[file stub/__init__.pyi] +s: str +def __getattr__(self): pass + +[file a.py] + +[file a.py.2] +from stub import x +from stub.pkg import y +from stub.pkg.sub import z + +[file b.py] + +[file b.py.3] +from stub import s +reveal_type(s) + +[out] +== +== +b.py:2: note: Revealed type is "builtins.str" + +[case testRenameSubModule] +import a + +[file a.py] +import pkg.sub + +[file pkg/__init__.py] +[file pkg/sub/__init__.py] +from pkg.sub import mod +[file pkg/sub/mod.py] + +[file pkg/sub/__init__.py.2] +from pkg.sub import modb +[delete pkg/sub/mod.py.2] +[file pkg/sub/modb.py.2] + +[out] +== + +[case testFineGrainedFunctoolsPartial] +import m + +[file m.py] +from typing import Callable +from partial import p1 + +reveal_type(p1) +p1("a") +p1("a", 3) +p1("a", c=3) +p1(1, 3) +p1(1, "a", 3) +p1(a=1, b="a", c=3) +[builtins fixtures/dict.pyi] + +[file partial.py] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... +p1 = foo + +[file partial.py.2] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... +p1 = functools.partial(foo, 1) + +[out] +m.py:4: note: Revealed type is "def (a: builtins.int, b: builtins.str, c: builtins.int =) -> builtins.int" +m.py:5: error: Too few arguments +m.py:5: error: Argument 1 has incompatible type "str"; expected "int" +m.py:6: error: Argument 1 has incompatible type "str"; expected "int" +m.py:6: error: Argument 2 has incompatible type "int"; expected "str" +m.py:7: error: Too few arguments +m.py:7: error: Argument 1 has incompatible type "str"; expected "int" +m.py:8: error: Argument 2 has incompatible type "int"; expected "str" +== +m.py:4: note: Revealed type is "functools.partial[builtins.int]" +m.py:8: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +m.py:9: error: Too many arguments for "foo" +m.py:9: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +m.py:9: error: Argument 2 to "foo" has incompatible type "str"; expected "int" +m.py:10: error: Unexpected keyword argument "a" for "foo" +partial.py:4: note: "foo" defined here + +[case testReplaceFunctionWithDecoratedFunctionIndirect] +from b import f +x: int = f() +import b +y: int = b.f() + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +from typing import Callable +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReplaceFunctionWithDecoratedFunctionIndirect2] +from c import f +x: int = f() +import c +y: int = c.f() + +[file c.py] +from b import f + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +from typing import Callable +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReplaceFunctionWithClassIndirect] +from b import f +x: int = f() +import b +y: int = b.f() + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +class f: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "f", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "f", variable has type "int") + +[case testReplaceFunctionWithClassIndirect2] +from c import f +x: int = f() +import c +y: int = c.f() + +[file c.py] +from b import f + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +class f: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "f", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "f", variable has type "int") + + +[case testDeprecatedAddKeepChangeAndRemoveFunctionDeprecation] +# flags: --enable-error-code=deprecated + +from a import f +f() +import a +a.f() + +[file a.py] +def f() -> None: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.3] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.4] +from typing_extensions import deprecated +@deprecated("use f3 instead") +def f() -> None: ... + +[file a.py.5] +def f() -> None: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:3: error: function a.f is deprecated: use f2 instead +main:6: error: function a.f is deprecated: use f2 instead +== +main:3: error: function a.f is deprecated: use f2 instead +main:6: error: function a.f is deprecated: use f2 instead +== +main:3: error: function a.f is deprecated: use f3 instead +main:6: error: function a.f is deprecated: use f3 instead +== + + +[case testDeprecatedRemoveFunctionDeprecation] +# flags: --enable-error-code=deprecated +from a import f +f() +import a +a.f() + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.2] +def f() -> None: ... + +[builtins fixtures/tuple.pyi] +[out] +main:2: error: function a.f is deprecated: use f2 instead +main:5: error: function a.f is deprecated: use f2 instead +== + +[case testDeprecatedKeepFunctionDeprecation] +# flags: --enable-error-code=deprecated +from a import f +f() +import a +a.f() + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[builtins fixtures/tuple.pyi] +[out] +main:2: error: function a.f is deprecated: use f2 instead +main:5: error: function a.f is deprecated: use f2 instead +== +main:2: error: function a.f is deprecated: use f2 instead +main:5: error: function a.f is deprecated: use f2 instead + + +[case testDeprecatedAddFunctionDeprecationIndirectImport] +# flags: --enable-error-code=deprecated +from b import f +f() +import b +b.f() + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> int: ... + +[builtins fixtures/tuple.pyi] +[out] +== +b.py:1: error: function a.f is deprecated: use f2 instead +main:2: error: function a.f is deprecated: use f2 instead +main:5: error: function a.f is deprecated: use f2 instead + + +[case testDeprecatedChangeFunctionDeprecationIndirectImport] +# flags: --enable-error-code=deprecated +from b import f +f() +import b +b.f() + +[file b.py] +from a import f + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f1 instead") +def f() -> int: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> int: ... + +[builtins fixtures/tuple.pyi] +[out] +b.py:1: error: function a.f is deprecated: use f1 instead +main:2: error: function a.f is deprecated: use f1 instead +main:5: error: function a.f is deprecated: use f1 instead +== +b.py:1: error: function a.f is deprecated: use f2 instead +main:2: error: function a.f is deprecated: use f2 instead +main:5: error: function a.f is deprecated: use f2 instead + +[case testDeprecatedRemoveFunctionDeprecationIndirectImport] +# flags: --enable-error-code=deprecated +from b import f +f() +import b +b.f() + +[file b.py] +from a import f + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f1 instead") +def f() -> int: ... + +[file a.py.2] +def f() -> int: ... + +[builtins fixtures/tuple.pyi] +[out] +b.py:1: error: function a.f is deprecated: use f1 instead +main:2: error: function a.f is deprecated: use f1 instead +main:5: error: function a.f is deprecated: use f1 instead +== + + +[case testDeprecatedFunctionAlreadyDecorated1-only_when_cache] +# flags: --enable-error-code=deprecated +from b import f +x: str = f() +import b +y: str = b.f() + +[file b.py] +from a import f + +[file a.py] +from typing import Callable + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[file a.py.2] +from typing import Callable +from typing_extensions import deprecated + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@deprecated("deprecated decorated function") +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +b.py:1: error: function a.f is deprecated: deprecated decorated function +main:2: error: function a.f is deprecated: deprecated decorated function +main:5: error: function a.f is deprecated: deprecated decorated function + + +[case testDeprecatedFunctionAlreadyDecorated2-only_when_nocache] +# flags: --enable-error-code=deprecated +from b import f +x: str = f() +import b +y: str = b.f() + +[file b.py] +from a import f + +[file a.py] +from typing import Callable + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[file a.py.2] +from typing import Callable +from typing_extensions import deprecated + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@deprecated("deprecated decorated function") +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: function a.f is deprecated: deprecated decorated function +main:5: error: function a.f is deprecated: deprecated decorated function +b.py:1: error: function a.f is deprecated: deprecated decorated function + + +[case testDeprecatedAddClassDeprecationIndirectImport1-only_when_cache] +# flags: --enable-error-code=deprecated +from b import C +x: C +C() +import b +y: b.D +b.D() + +[file b.py] +from a import C +from a import D + +[file a.py] +class C: ... +class D: ... + +[file a.py.2] +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +@deprecated("use D2 instead") +class D: ... + +[builtins fixtures/tuple.pyi] +[out] +== +b.py:1: error: class a.C is deprecated: use C2 instead +b.py:2: error: class a.D is deprecated: use D2 instead +main:2: error: class a.C is deprecated: use C2 instead +main:6: error: class a.D is deprecated: use D2 instead +main:7: error: class a.D is deprecated: use D2 instead + + +[case testDeprecatedAddClassDeprecationIndirectImport2-only_when_nocache] +# flags: --enable-error-code=deprecated +from b import C +x: C +C() +import b +y: b.D +b.D() + +[file b.py] +from a import C +from a import D + +[file a.py] +class C: ... +class D: ... + +[file a.py.2] +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +@deprecated("use D2 instead") +class D: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: class a.C is deprecated: use C2 instead +main:6: error: class a.D is deprecated: use D2 instead +main:7: error: class a.D is deprecated: use D2 instead +b.py:1: error: class a.C is deprecated: use C2 instead +b.py:2: error: class a.D is deprecated: use D2 instead + + +[case testDeprecatedChangeClassDeprecationIndirectImport] +# flags: --enable-error-code=deprecated +from b import C +x: C +C() +import b +y: b.D +b.D() + +[file b.py] +from a import C +from a import D + +[file a.py] +from typing_extensions import deprecated + +@deprecated("use C1 instead") +class C: ... +@deprecated("use D1 instead") +class D: ... + +[file a.py.2] +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +@deprecated("use D2 instead") +class D: ... + +[builtins fixtures/tuple.pyi] +[out] +b.py:1: error: class a.C is deprecated: use C1 instead +b.py:2: error: class a.D is deprecated: use D1 instead +main:2: error: class a.C is deprecated: use C1 instead +main:6: error: class a.D is deprecated: use D1 instead +main:7: error: class a.D is deprecated: use D1 instead +== +b.py:1: error: class a.C is deprecated: use C2 instead +b.py:2: error: class a.D is deprecated: use D2 instead +main:2: error: class a.C is deprecated: use C2 instead +main:6: error: class a.D is deprecated: use D2 instead +main:7: error: class a.D is deprecated: use D2 instead + + +[case testDeprecatedRemoveClassDeprecationIndirectImport] +# flags: --enable-error-code=deprecated +from b import C +x: C +C() +import b +y: b.D +b.D() + +[file b.py] +from a import C +from a import D + +[file a.py] +from typing_extensions import deprecated + +@deprecated("use C1 instead") +class C: ... +@deprecated("use D1 instead") +class D: ... + +[file a.py.2] +class C: ... +class D: ... + +[builtins fixtures/tuple.pyi] +[out] +b.py:1: error: class a.C is deprecated: use C1 instead +b.py:2: error: class a.D is deprecated: use D1 instead +main:2: error: class a.C is deprecated: use C1 instead +main:6: error: class a.D is deprecated: use D1 instead +main:7: error: class a.D is deprecated: use D1 instead +== + + +[case testDeprecatedAddClassDeprecationIndirectImportAlreadyDecorated1-only_when_cache] +# flags: --enable-error-code=deprecated +from b import C +x: C +C() +import b +y: b.D +b.D() + +[file b.py] +from a import C +from a import D + +[file a.py] +from typing import TypeVar + +T = TypeVar("T") +def dec(x: T) -> T: ... + +@dec +class C: ... +@dec +class D: ... + +[file a.py.2] +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +@deprecated("use D2 instead") +class D: ... + +[builtins fixtures/tuple.pyi] +[out] +== +b.py:1: error: class a.C is deprecated: use C2 instead +b.py:2: error: class a.D is deprecated: use D2 instead +main:2: error: class a.C is deprecated: use C2 instead +main:6: error: class a.D is deprecated: use D2 instead +main:7: error: class a.D is deprecated: use D2 instead + + +[case testDeprecatedAddClassDeprecationIndirectImportAlreadyDecorated2-only_when_nocache] +# flags: --enable-error-code=deprecated +from b import C +x: C +C() +import b +y: b.D +b.D() + +[file b.py] +from a import C +from a import D + +[file a.py] +from typing import TypeVar + +T = TypeVar("T") +def dec(x: T) -> T: ... + +@dec +class C: ... +@dec +class D: ... + +[file a.py.2] +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +@deprecated("use D2 instead") +class D: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: class a.C is deprecated: use C2 instead +main:6: error: class a.D is deprecated: use D2 instead +main:7: error: class a.D is deprecated: use D2 instead +b.py:1: error: class a.C is deprecated: use C2 instead +b.py:2: error: class a.D is deprecated: use D2 instead diff --git a/test-data/unit/fixtures/__init_subclass__.pyi b/test-data/unit/fixtures/__init_subclass__.pyi new file mode 100644 index 000000000000..b4618c28249e --- /dev/null +++ b/test-data/unit/fixtures/__init_subclass__.pyi @@ -0,0 +1,14 @@ +# builtins stub with object.__init_subclass__ + +from typing import Mapping, Iterable # needed for ArgumentInferContext + +class object: + def __init_subclass__(cls) -> None: pass + +class type: pass + +class int: pass +class bool: pass +class str: pass +class function: pass +class dict: pass diff --git a/test-data/unit/fixtures/__new__.pyi b/test-data/unit/fixtures/__new__.pyi new file mode 100644 index 000000000000..401de6fb9cd1 --- /dev/null +++ b/test-data/unit/fixtures/__new__.pyi @@ -0,0 +1,19 @@ +# builtins stub with object.__new__ + +from typing import Any + +class object: + def __init__(self) -> None: pass + + __class__ = object + + def __new__(cls) -> Any: pass + +class type: + def __init__(self, x) -> None: pass + +class int: pass +class bool: pass +class str: pass +class function: pass +class dict: pass diff --git a/test-data/unit/fixtures/alias.pyi b/test-data/unit/fixtures/alias.pyi new file mode 100644 index 000000000000..2ec7703f00c4 --- /dev/null +++ b/test-data/unit/fixtures/alias.pyi @@ -0,0 +1,16 @@ +# Builtins test fixture with a type alias 'bytes' + +from typing import Mapping, Iterable # needed for `ArgumentInferContext` + +class object: + def __init__(self) -> None: pass +class type: + def __init__(self, x) -> None: pass + +class int: pass +class str: pass +class function: pass + +bytes = str + +class dict: pass diff --git a/test-data/unit/fixtures/any.pyi b/test-data/unit/fixtures/any.pyi new file mode 100644 index 000000000000..b1f8d83bf524 --- /dev/null +++ b/test-data/unit/fixtures/any.pyi @@ -0,0 +1,10 @@ +from typing import TypeVar, Iterable + +T = TypeVar('T') + +class int: pass +class str: pass + +def any(i: Iterable[T]) -> bool: pass + +class dict: pass diff --git a/test-data/unit/fixtures/args.pyi b/test-data/unit/fixtures/args.pyi new file mode 100644 index 000000000000..0020d9ceff46 --- /dev/null +++ b/test-data/unit/fixtures/args.pyi @@ -0,0 +1,35 @@ +# Builtins stub used to support *args, **kwargs. + +import _typeshed +from typing import TypeVar, Generic, Iterable, Sequence, Tuple, Dict, Any, overload, Mapping + +Tco = TypeVar('Tco', covariant=True) +T = TypeVar('T') +S = TypeVar('S') + +class object: + def __init__(self) -> None: pass + def __eq__(self, o: object) -> bool: pass + def __ne__(self, o: object) -> bool: pass + +class type: + @overload + def __init__(self, o: object) -> None: pass + @overload + def __init__(self, name: str, bases: Tuple[type, ...], dict: Dict[str, Any]) -> None: pass + def __call__(self, *args: Any, **kwargs: Any) -> Any: pass + +class tuple(Iterable[Tco], Generic[Tco]): pass + +class dict(Mapping[T, S], Generic[T, S]): pass + +class list(Sequence[T], Generic[T]): pass + +class int: + def __eq__(self, o: object) -> bool: pass +class float: pass +class str: pass +class bytes: pass +class bool: pass +class function: pass +class ellipsis: pass diff --git a/test-data/unit/fixtures/async_await.pyi b/test-data/unit/fixtures/async_await.pyi new file mode 100644 index 000000000000..96ade881111b --- /dev/null +++ b/test-data/unit/fixtures/async_await.pyi @@ -0,0 +1,26 @@ +import typing + +T = typing.TypeVar('T') +U = typing.TypeVar('U') +class list(typing.Sequence[T]): + def __iter__(self) -> typing.Iterator[T]: ... + def __getitem__(self, i: int) -> T: ... + def __contains__(self, item: object) -> bool: ... + +class object: + def __init__(self) -> None: pass +class type: pass +class function: pass +class int: pass +class float: pass +class str: pass +class bool(int): pass +class dict(typing.Generic[T, U]): pass +class set(typing.Generic[T]): pass +class tuple(typing.Generic[T]): pass +class BaseException: pass +class StopIteration(BaseException): pass +class StopAsyncIteration(BaseException): pass +def iter(obj: typing.Any) -> typing.Any: pass +def next(obj: typing.Any) -> typing.Any: pass +class ellipsis: ... diff --git a/test-data/unit/fixtures/bool.pyi b/test-data/unit/fixtures/bool.pyi new file mode 100644 index 000000000000..bc58a22b952b --- /dev/null +++ b/test-data/unit/fixtures/bool.pyi @@ -0,0 +1,20 @@ +# builtins stub used in boolean-related test cases. +from typing import Generic, TypeVar +T = TypeVar('T') + +class object: + def __init__(self) -> None: pass + def __eq__(self, other: object) -> bool: pass + def __ne__(self, other: object) -> bool: pass + +class type: pass +class tuple(Generic[T]): pass +class function: pass +class int: pass +class bool(int): pass +class float: pass +class str: pass +class ellipsis: pass +class list(Generic[T]): pass +class property: pass +class dict: pass diff --git a/test-data/unit/fixtures/callable.pyi b/test-data/unit/fixtures/callable.pyi new file mode 100644 index 000000000000..44abf0691ceb --- /dev/null +++ b/test-data/unit/fixtures/callable.pyi @@ -0,0 +1,31 @@ +from typing import Generic, Tuple, TypeVar, Union + +T = TypeVar('T') + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + +class tuple(Generic[T]): pass + +class classmethod: pass +class staticmethod: pass +class function: pass + +def isinstance(x: object, t: Union[type, Tuple[type, ...]]) -> bool: pass + +def callable(x: object) -> bool: pass + +class int: + def __add__(self, other: 'int') -> 'int': pass + def __eq__(self, other: 'int') -> 'bool': pass +class float: pass +class bool(int): pass +class str: + def __add__(self, other: 'str') -> 'str': pass + def __eq__(self, other: 'str') -> bool: pass +class ellipsis: pass +class list: ... +class dict: pass diff --git a/test-data/unit/fixtures/classmethod.pyi b/test-data/unit/fixtures/classmethod.pyi new file mode 100644 index 000000000000..97e018b1dc1c --- /dev/null +++ b/test-data/unit/fixtures/classmethod.pyi @@ -0,0 +1,31 @@ +import typing + +_T = typing.TypeVar('_T') + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + def mro(self) -> typing.Any: pass + +class function: pass + +# Dummy definitions. +class classmethod: pass +class staticmethod: pass + +class int: + @classmethod + def from_bytes(cls, bytes: bytes, byteorder: str) -> int: pass + +class float: pass +class str: pass +class bytes: pass +class bool: pass +class ellipsis: pass + +class tuple(typing.Generic[_T]): pass + +class list: pass +class dict: pass diff --git a/test-data/unit/fixtures/complex.pyi b/test-data/unit/fixtures/complex.pyi new file mode 100644 index 000000000000..880ec3dd4d9d --- /dev/null +++ b/test-data/unit/fixtures/complex.pyi @@ -0,0 +1,13 @@ +# Builtins stub used for some float/complex test cases. +# Please don't add tuple to this file, it is used to test incomplete fixtures. + +class object: + def __init__(self): pass + +class type: pass +class function: pass +class int: pass +class float: pass +class complex: pass +class str: pass +class dict: pass diff --git a/test-data/unit/fixtures/complex_tuple.pyi b/test-data/unit/fixtures/complex_tuple.pyi new file mode 100644 index 000000000000..81f1d33d1207 --- /dev/null +++ b/test-data/unit/fixtures/complex_tuple.pyi @@ -0,0 +1,16 @@ +from typing import Generic, TypeVar +_T = TypeVar('_T') + +class object: + def __init__(self): pass + +class tuple(Generic[_T]): pass + +class type: pass +class function: pass +class int: pass +class float: pass +class complex: pass +class str: pass +class ellipsis: pass +class dict: pass diff --git a/test-data/unit/fixtures/dataclasses.pyi b/test-data/unit/fixtures/dataclasses.pyi new file mode 100644 index 000000000000..29f87ae97e62 --- /dev/null +++ b/test-data/unit/fixtures/dataclasses.pyi @@ -0,0 +1,56 @@ +import _typeshed +from typing import ( + Generic, Iterator, Iterable, Mapping, Optional, Sequence, Tuple, + TypeVar, Union, overload, +) +from typing_extensions import override + +_T = TypeVar('_T') +_U = TypeVar('_U') +KT = TypeVar('KT') +VT = TypeVar('VT') + +class object: + def __init__(self) -> None: pass + def __init_subclass__(cls) -> None: pass + def __eq__(self, o: object) -> bool: pass + def __ne__(self, o: object) -> bool: pass + +class type: pass +class ellipsis: pass +class tuple(Generic[_T]): pass +class int: pass +class float: pass +class bytes: pass +class str: pass +class bool(int): pass + +class dict(Mapping[KT, VT]): + @overload + def __init__(self, **kwargs: VT) -> None: pass + @overload + def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: pass + @override + def __getitem__(self, key: KT) -> VT: pass + def __setitem__(self, k: KT, v: VT) -> None: pass + @override + def __iter__(self) -> Iterator[KT]: pass + def __contains__(self, item: object) -> int: pass + def update(self, a: Mapping[KT, VT]) -> None: pass + @overload + def get(self, k: KT) -> Optional[VT]: pass + @overload + def get(self, k: KT, default: Union[KT, _T]) -> Union[VT, _T]: pass + def __len__(self) -> int: ... + +class list(Generic[_T], Sequence[_T]): + def __contains__(self, item: object) -> int: pass + @override + def __getitem__(self, key: int) -> _T: pass + @override + def __iter__(self) -> Iterator[_T]: pass + +class function: pass +class classmethod: pass +class staticmethod: pass +property = object() diff --git a/test-data/unit/fixtures/dict-full.pyi b/test-data/unit/fixtures/dict-full.pyi new file mode 100644 index 000000000000..f20369ce9332 --- /dev/null +++ b/test-data/unit/fixtures/dict-full.pyi @@ -0,0 +1,83 @@ +# Builtins stub used in dictionary-related test cases (more complete). + +from _typeshed import SupportsKeysAndGetItem +import _typeshed +from typing import ( + TypeVar, Generic, Iterable, Iterator, Mapping, Tuple, overload, Optional, Union, Sequence, + Self, +) + +T = TypeVar('T') +T2 = TypeVar('T2') +KT = TypeVar('KT') +VT = TypeVar('VT') + +class object: + def __init__(self) -> None: pass + def __init_subclass__(cls) -> None: pass + def __eq__(self, other: object) -> bool: pass + +class type: + __annotations__: Mapping[str, object] + +class dict(Mapping[KT, VT]): + @overload + def __init__(self, **kwargs: VT) -> None: pass + @overload + def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: pass + def __getitem__(self, key: KT) -> VT: pass + def __setitem__(self, k: KT, v: VT) -> None: pass + def __iter__(self) -> Iterator[KT]: pass + def __contains__(self, item: object) -> int: pass + def update(self, a: SupportsKeysAndGetItem[KT, VT]) -> None: pass + @overload + def get(self, k: KT) -> Optional[VT]: pass + @overload + def get(self, k: KT, default: Union[VT, T]) -> Union[VT, T]: pass + def __len__(self) -> int: ... + + # This was actually added in 3.9: + @overload + def __or__(self, __value: dict[KT, VT]) -> dict[KT, VT]: ... + @overload + def __or__(self, __value: dict[T, T2]) -> dict[Union[KT, T], Union[VT, T2]]: ... + @overload + def __ror__(self, __value: dict[KT, VT]) -> dict[KT, VT]: ... + @overload + def __ror__(self, __value: dict[T, T2]) -> dict[Union[KT, T], Union[VT, T2]]: ... + # dict.__ior__ should be kept roughly in line with MutableMapping.update() + @overload # type: ignore[misc] + def __ior__(self, __value: _typeshed.SupportsKeysAndGetItem[KT, VT]) -> Self: ... + @overload + def __ior__(self, __value: Iterable[Tuple[KT, VT]]) -> Self: ... + +class int: # for convenience + def __add__(self, x: Union[int, complex]) -> int: pass + def __radd__(self, x: int) -> int: pass + def __sub__(self, x: Union[int, complex]) -> int: pass + def __neg__(self) -> int: pass + real: int + imag: int + +class str: pass # for keyword argument key type +class bytes: pass + +class list(Sequence[T]): # needed by some test cases + def __getitem__(self, x: int) -> T: pass + def __iter__(self) -> Iterator[T]: pass + def __mul__(self, x: int) -> list[T]: pass + def __contains__(self, item: object) -> bool: pass + def append(self, item: T) -> None: pass + +class tuple(Generic[T]): pass +class function: pass +class float: pass +class complex: pass +class bool(int): pass + +class ellipsis: + __class__: object +def isinstance(x: object, t: Union[type, Tuple[type, ...]]) -> bool: pass +class BaseException: pass + +def iter(__iterable: Iterable[T]) -> Iterator[T]: pass diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi new file mode 100644 index 000000000000..ed2287511161 --- /dev/null +++ b/test-data/unit/fixtures/dict.pyi @@ -0,0 +1,64 @@ +# Builtins stub used in dictionary-related test cases (stripped down). +# +# NOTE: Use dict-full.pyi if you need more builtins instead of adding here, +# if feasible. + +from _typeshed import SupportsKeysAndGetItem +import _typeshed +from typing import ( + TypeVar, Generic, Iterable, Iterator, Mapping, Tuple, overload, Optional, Union, Sequence, + Self, +) + +T = TypeVar('T') +T2 = TypeVar('T2') +KT = TypeVar('KT') +VT = TypeVar('VT') + +class object: + def __init__(self) -> None: pass + def __eq__(self, other: object) -> bool: pass + +class type: pass + +class dict(Mapping[KT, VT]): + @overload + def __init__(self, **kwargs: VT) -> None: pass + @overload + def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: pass + def __getitem__(self, key: KT) -> VT: pass + def __setitem__(self, k: KT, v: VT) -> None: pass + def __iter__(self) -> Iterator[KT]: pass + def __contains__(self, item: object) -> int: pass + def update(self, a: SupportsKeysAndGetItem[KT, VT]) -> None: pass + @overload + def get(self, k: KT) -> Optional[VT]: pass + @overload + def get(self, k: KT, default: Union[VT, T]) -> Union[VT, T]: pass + def __len__(self) -> int: ... + +class int: # for convenience + def __add__(self, x: Union[int, complex]) -> int: pass + def __radd__(self, x: int) -> int: pass + def __sub__(self, x: Union[int, complex]) -> int: pass + +class str: pass # for keyword argument key type +class bytes: pass + +class list(Sequence[T]): # needed by some test cases + def __getitem__(self, x: int) -> T: pass + def __iter__(self) -> Iterator[T]: pass + def __mul__(self, x: int) -> list[T]: pass + def __contains__(self, item: object) -> bool: pass + def append(self, item: T) -> None: pass + +class tuple(Generic[T]): pass +class function: pass +class float: pass +class complex: pass +class bool(int): pass +class ellipsis: pass +class BaseException: pass + +def isinstance(x: object, t: Union[type, Tuple[type, ...]]) -> bool: pass +def iter(__iterable: Iterable[T]) -> Iterator[T]: pass diff --git a/test-data/unit/fixtures/divmod.pyi b/test-data/unit/fixtures/divmod.pyi new file mode 100644 index 000000000000..4d81d8fb47a2 --- /dev/null +++ b/test-data/unit/fixtures/divmod.pyi @@ -0,0 +1,23 @@ +from typing import TypeVar, Tuple, SupportsInt +class object: + def __init__(self): pass + +class int(SupportsInt): + def __divmod__(self, other: int) -> Tuple[int, int]: pass + def __rdivmod__(self, other: int) -> Tuple[int, int]: pass + +class float(SupportsInt): + def __divmod__(self, other: float) -> Tuple[float, float]: pass + def __rdivmod__(self, other: float) -> Tuple[float, float]: pass + + +class tuple: pass +class function: pass +class str: pass +class type: pass +class ellipsis: pass + +_N = TypeVar('_N', int, float) +def divmod(_x: _N, _y: _N) -> Tuple[_N, _N]: ... + +class dict: pass diff --git a/test-data/unit/fixtures/enum.pyi b/test-data/unit/fixtures/enum.pyi new file mode 100644 index 000000000000..debffacf8f32 --- /dev/null +++ b/test-data/unit/fixtures/enum.pyi @@ -0,0 +1,16 @@ +# Minimal set of builtins required to work with Enums +from typing import TypeVar, Generic + +T = TypeVar('T') + +class object: + def __init__(self): pass + +class type: pass +class tuple(Generic[T]): + def __getitem__(self, x: int) -> T: pass + +class int: pass +class str: pass +class dict: pass +class ellipsis: pass diff --git a/test-data/unit/fixtures/exception.pyi b/test-data/unit/fixtures/exception.pyi new file mode 100644 index 000000000000..08496e4e5934 --- /dev/null +++ b/test-data/unit/fixtures/exception.pyi @@ -0,0 +1,29 @@ +import sys +from typing import Generic, TypeVar +T = TypeVar('T') + +class object: + def __init__(self): pass + +class type: pass +class tuple(Generic[T]): + def __ge__(self, other: object) -> bool: ... +class list: pass +class dict: pass +class function: pass +class int: pass +class str: pass +class bool: pass +class ellipsis: pass + +class BaseException: + def __init__(self, *args: object) -> None: ... +class Exception(BaseException): pass +class RuntimeError(Exception): pass +class NotImplementedError(RuntimeError): pass + +if sys.version_info >= (3, 11): + _BT_co = TypeVar("_BT_co", bound=BaseException, covariant=True) + _T_co = TypeVar("_T_co", bound=Exception, covariant=True) + class BaseExceptionGroup(BaseException, Generic[_BT_co]): ... + class ExceptionGroup(BaseExceptionGroup[_T_co], Exception): ... diff --git a/test-data/unit/fixtures/f_string.pyi b/test-data/unit/fixtures/f_string.pyi new file mode 100644 index 000000000000..328c666b7ece --- /dev/null +++ b/test-data/unit/fixtures/f_string.pyi @@ -0,0 +1,38 @@ +# Builtins stub used for format-string-related test cases. +# We need str and list, and str needs join and format methods. + +from typing import TypeVar, Generic, Iterable, Iterator, List, overload + +T = TypeVar('T') + +class object: + def __init__(self): pass + +class type: + def __init__(self, x) -> None: pass + +class ellipsis: pass + +class list(Iterable[T], Generic[T]): + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: Iterable[T]) -> None: pass + def append(self, x: T) -> None: pass + +class tuple(Generic[T]): pass + +class function: pass +class int: + def __add__(self, i: int) -> int: pass + +class float: pass +class bool(int): pass + +class str: + def __add__(self, s: str) -> str: pass + def format(self, *args) -> str: pass + def join(self, l: List[str]) -> str: pass + + +class dict: pass diff --git a/test-data/unit/fixtures/fine_grained.pyi b/test-data/unit/fixtures/fine_grained.pyi new file mode 100644 index 000000000000..e454a27a5ebd --- /dev/null +++ b/test-data/unit/fixtures/fine_grained.pyi @@ -0,0 +1,30 @@ +# Small stub for fine-grained incremental checking test cases +# +# TODO: Migrate to regular stubs once fine-grained incremental is robust +# enough to handle them. + +import types +from typing import TypeVar, Generic + +T = TypeVar('T') + +class Any: pass + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x: Any) -> None: pass + +class int: + def __add__(self, other: 'int') -> 'int': pass +class str: + def __add__(self, other: 'str') -> 'str': pass + +class float: pass +class bytes: pass +class tuple(Generic[T]): pass +class function: pass +class ellipsis: pass +class list(Generic[T]): pass +class dict: pass diff --git a/test-data/unit/fixtures/float.pyi b/test-data/unit/fixtures/float.pyi new file mode 100644 index 000000000000..5db4525849c0 --- /dev/null +++ b/test-data/unit/fixtures/float.pyi @@ -0,0 +1,38 @@ +from typing import Generic, TypeVar +T = TypeVar('T') + +Any = 0 + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x: Any) -> None: pass + +class str: + def __add__(self, other: 'str') -> 'str': pass + def __rmul__(self, n: int) -> str: ... + +class bytes: pass + +class tuple(Generic[T]): pass +class function: pass + +class ellipsis: pass + + +class int: + def __abs__(self) -> int: ... + def __float__(self) -> float: ... + def __int__(self) -> int: ... + def __mul__(self, x: int) -> int: ... + def __neg__(self) -> int: ... + def __rmul__(self, x: int) -> int: ... + +class float: + def __float__(self) -> float: ... + def __int__(self) -> int: ... + def __mul__(self, x: float) -> float: ... + def __rmul__(self, x: float) -> float: ... + +class dict: pass diff --git a/test-data/unit/fixtures/floatdict.pyi b/test-data/unit/fixtures/floatdict.pyi new file mode 100644 index 000000000000..7baa7ca9206f --- /dev/null +++ b/test-data/unit/fixtures/floatdict.pyi @@ -0,0 +1,68 @@ +from typing import TypeVar, Generic, Iterable, Iterator, Mapping, Tuple, overload, Optional, Union + +T = TypeVar('T') +KT = TypeVar('KT') +VT = TypeVar('VT') + +Any = 0 + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x: Any) -> None: pass + +class str: + def __add__(self, other: 'str') -> 'str': pass + def __rmul__(self, n: int) -> str: ... + +class bytes: pass + +class tuple(Generic[T]): pass +class slice: pass +class function: pass + +class ellipsis: pass + +class list(Iterable[T], Generic[T]): + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: Iterable[T]) -> None: pass + def __iter__(self) -> Iterator[T]: pass + def __add__(self, x: list[T]) -> list[T]: pass + def __mul__(self, x: int) -> list[T]: pass + def __getitem__(self, x: int) -> T: pass + def append(self, x: T) -> None: pass + def extend(self, x: Iterable[T]) -> None: pass + +class dict(Mapping[KT, VT], Generic[KT, VT]): + @overload + def __init__(self, **kwargs: VT) -> None: pass + @overload + def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: pass + def __setitem__(self, k: KT, v: VT) -> None: pass + def __getitem__(self, k: KT) -> VT: pass + def __iter__(self) -> Iterator[KT]: pass + def update(self, a: Mapping[KT, VT]) -> None: pass + @overload + def get(self, k: KT) -> Optional[VT]: pass + @overload + def get(self, k: KT, default: Union[KT, T]) -> Union[VT, T]: pass + + +class int: + def __float__(self) -> float: ... + def __int__(self) -> int: ... + def __mul__(self, x: int) -> int: ... + def __rmul__(self, x: int) -> int: ... + def __truediv__(self, x: int) -> int: ... + def __rtruediv__(self, x: int) -> int: ... + +class float: + def __float__(self) -> float: ... + def __int__(self) -> int: ... + def __mul__(self, x: float) -> float: ... + def __rmul__(self, x: float) -> float: ... + def __truediv__(self, x: float) -> float: ... + def __rtruediv__(self, x: float) -> float: ... diff --git a/test-data/unit/fixtures/for.pyi b/test-data/unit/fixtures/for.pyi new file mode 100644 index 000000000000..10f45e68cd7d --- /dev/null +++ b/test-data/unit/fixtures/for.pyi @@ -0,0 +1,23 @@ +# builtins stub used in for statement test cases + +from typing import TypeVar, Generic, Iterable, Iterator, Generator +from abc import abstractmethod, ABCMeta + +t = TypeVar('t') + +class object: + def __init__(self) -> None: pass + +class type: pass +class tuple(Generic[t]): + def __iter__(self) -> Iterator[t]: pass +class function: pass +class ellipsis: pass +class bool: pass +class int: pass # for convenience +class str: # for convenience + def upper(self) -> str: ... + +class list(Iterable[t], Generic[t]): + def __iter__(self) -> Iterator[t]: pass +class dict: pass diff --git a/test-data/unit/fixtures/function.pyi b/test-data/unit/fixtures/function.pyi new file mode 100644 index 000000000000..697d0d919d98 --- /dev/null +++ b/test-data/unit/fixtures/function.pyi @@ -0,0 +1,8 @@ +class object: + def __init__(self): pass + +class type: pass +class function: pass +class int: pass +class str: pass +class dict: pass diff --git a/test-data/unit/fixtures/isinstance.pyi b/test-data/unit/fixtures/isinstance.pyi new file mode 100644 index 000000000000..12cef2035c2b --- /dev/null +++ b/test-data/unit/fixtures/isinstance.pyi @@ -0,0 +1,30 @@ +from typing import Tuple, TypeVar, Generic, Union, cast, Any, Type + +T = TypeVar('T') + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + def __or__(self, other: type) -> type: pass + +class tuple(Generic[T]): pass + +class function: pass + +def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass +def issubclass(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass +def hasattr(x: object, name: str) -> bool: pass + +class int: + def __add__(self, other: 'int') -> 'int': pass +class float: pass +class bool(int): pass +class str: + def __add__(self, other: 'str') -> 'str': pass +class ellipsis: pass + +NotImplemented = cast(Any, None) + +class dict: pass diff --git a/test-data/unit/fixtures/isinstance_python3_10.pyi b/test-data/unit/fixtures/isinstance_python3_10.pyi new file mode 100644 index 000000000000..0918d10ab1ef --- /dev/null +++ b/test-data/unit/fixtures/isinstance_python3_10.pyi @@ -0,0 +1,31 @@ +# For Python 3.10+ only +from typing import Tuple, TypeVar, Generic, Union, cast, Any, Type +import types + +T = TypeVar('T') + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + def __or__(self, x) -> types.UnionType: pass + +class tuple(Generic[T]): pass + +class function: pass + +def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...], types.UnionType]) -> bool: pass +def issubclass(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass + +class int: + def __add__(self, other: 'int') -> 'int': pass +class float: pass +class bool(int): pass +class str: + def __add__(self, other: 'str') -> 'str': pass +class ellipsis: pass + +NotImplemented = cast(Any, None) + +class dict: pass diff --git a/test-data/unit/fixtures/isinstancelist.pyi b/test-data/unit/fixtures/isinstancelist.pyi new file mode 100644 index 000000000000..0ee5258ff74b --- /dev/null +++ b/test-data/unit/fixtures/isinstancelist.pyi @@ -0,0 +1,61 @@ +from typing import ( + Iterable, Iterator, TypeVar, List, Mapping, overload, Tuple, Set, Union, Generic, Sequence +) + +class object: + def __init__(self) -> None: pass + def __eq__(self, other: object) -> bool: pass + +class type: + def __init__(self, x) -> None: pass + +class function: pass +class classmethod: pass + +class ellipsis: pass +EllipsisType = ellipsis +Ellipsis = ellipsis() + +def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass +def issubclass(x: object, t: Union[type, Tuple]) -> bool: pass + +class int: + def __add__(self, x: int) -> int: pass +class float: pass +class bool(int): pass +class str: + def __add__(self, x: str) -> str: pass + def __getitem__(self, x: int) -> str: pass + +T = TypeVar('T') +KT = TypeVar('KT') +VT = TypeVar('VT') + +class tuple(Generic[T]): + def __len__(self) -> int: pass + +class list(Sequence[T]): + def __iter__(self) -> Iterator[T]: pass + def __mul__(self, x: int) -> list[T]: pass + def __setitem__(self, x: int, v: T) -> None: pass + def __getitem__(self, x: int) -> T: pass + def __add__(self, x: List[T]) -> T: pass + def __contains__(self, item: object) -> bool: pass + def append(self, x: T) -> None: pass + def extend(self, x: Iterable[T]) -> None: pass + +class dict(Mapping[KT, VT]): + @overload + def __init__(self, **kwargs: VT) -> None: pass + @overload + def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: pass + def __setitem__(self, k: KT, v: VT) -> None: pass + def __iter__(self) -> Iterator[KT]: pass + def update(self, a: Mapping[KT, VT]) -> None: pass + +class set(Generic[T]): + def __iter__(self) -> Iterator[T]: pass + def add(self, x: T) -> None: pass + def discard(self, x: T) -> None: pass + def update(self, x: Set[T]) -> None: pass + def pop(self) -> T: pass diff --git a/test-data/unit/fixtures/len.pyi b/test-data/unit/fixtures/len.pyi new file mode 100644 index 000000000000..ee39d952701f --- /dev/null +++ b/test-data/unit/fixtures/len.pyi @@ -0,0 +1,39 @@ +from typing import Tuple, TypeVar, Generic, Union, Type, Sequence, Mapping +from typing_extensions import Protocol + +T = TypeVar("T") +V = TypeVar("V") + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + +class tuple(Sequence[T]): + def __len__(self) -> int: pass + +class list(Sequence[T]): pass +class dict(Mapping[T, V]): pass + +class function: pass + +class Sized(Protocol): + def __len__(self) -> int: pass + +def len(__obj: Sized) -> int: ... +def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass + +class int: + def __add__(self, other: int) -> int: pass + def __eq__(self, other: int) -> bool: pass + def __ne__(self, other: int) -> bool: pass + def __lt__(self, n: int) -> bool: pass + def __gt__(self, n: int) -> bool: pass + def __le__(self, n: int) -> bool: pass + def __ge__(self, n: int) -> bool: pass + def __neg__(self) -> int: pass +class float: pass +class bool(int): pass +class str(Sequence[str]): pass +class ellipsis: pass diff --git a/test-data/unit/fixtures/list.pyi b/test-data/unit/fixtures/list.pyi new file mode 100644 index 000000000000..3dcdf18b2faa --- /dev/null +++ b/test-data/unit/fixtures/list.pyi @@ -0,0 +1,41 @@ +# Builtins stub used in list-related test cases. + +from typing import TypeVar, Generic, Iterable, Iterator, Sequence, overload + +T = TypeVar('T') + +class object: + def __init__(self) -> None: pass + def __eq__(self, other: object) -> bool: pass + +class type: pass +class ellipsis: pass + +class list(Sequence[T]): + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: Iterable[T]) -> None: pass + def __iter__(self) -> Iterator[T]: pass + def __len__(self) -> int: pass + def __contains__(self, item: object) -> bool: pass + def __add__(self, x: list[T]) -> list[T]: pass + def __mul__(self, x: int) -> list[T]: pass + def __getitem__(self, x: int) -> T: pass + def __setitem__(self, x: int, v: T) -> None: pass + def append(self, x: T) -> None: pass + def extend(self, x: Iterable[T]) -> None: pass + +class tuple(Generic[T]): pass +class function: pass +class int: + def __bool__(self) -> bool: pass +class float: + def __bool__(self) -> bool: pass +class str: + def __len__(self) -> bool: pass +class bool(int): pass + +property = object() # Dummy definition. + +class dict: pass diff --git a/test-data/unit/fixtures/module.pyi b/test-data/unit/fixtures/module.pyi new file mode 100644 index 000000000000..47408befd5ce --- /dev/null +++ b/test-data/unit/fixtures/module.pyi @@ -0,0 +1,23 @@ +from typing import Any, Dict, Generic, TypeVar, Sequence +from types import ModuleType + +T = TypeVar('T') +S = TypeVar('S') + +class list(Generic[T], Sequence[T]): pass + +class object: + def __init__(self) -> None: pass +class type: pass +class function: pass +class int: pass +class str: pass +class bool: pass +class tuple(Generic[T]): pass +class dict(Generic[T, S]): pass +class ellipsis: pass + +classmethod = object() +staticmethod = object() +property = object() +def hasattr(x: object, name: str) -> bool: pass diff --git a/test-data/unit/fixtures/module_all.pyi b/test-data/unit/fixtures/module_all.pyi new file mode 100644 index 000000000000..d6060583b20e --- /dev/null +++ b/test-data/unit/fixtures/module_all.pyi @@ -0,0 +1,20 @@ +from typing import Generic, Sequence, TypeVar +from types import ModuleType + +_T = TypeVar('_T') + +class object: + def __init__(self) -> None: pass +class type: pass +class function: pass +class int: pass +class str: pass +class bool: pass +class list(Generic[_T], Sequence[_T]): + def append(self, x: _T): pass + def extend(self, x: Sequence[_T]): pass + def remove(self, x: _T): pass + def __add__(self, rhs: Sequence[_T]) -> list[_T]: pass +class tuple(Generic[_T]): pass +class ellipsis: pass +class dict: pass diff --git a/test-data/unit/fixtures/narrowing.pyi b/test-data/unit/fixtures/narrowing.pyi new file mode 100644 index 000000000000..89ee011c1c80 --- /dev/null +++ b/test-data/unit/fixtures/narrowing.pyi @@ -0,0 +1,20 @@ +# Builtins stub used in check-narrowing test cases. +from typing import Generic, Sequence, Tuple, Type, TypeVar, Union + + +Tco = TypeVar('Tco', covariant=True) +KT = TypeVar("KT") +VT = TypeVar("VT") + +class object: + def __init__(self) -> None: pass + +class type: pass +class tuple(Sequence[Tco], Generic[Tco]): pass +class function: pass +class ellipsis: pass +class int: pass +class str: pass +class dict(Generic[KT, VT]): pass + +def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass diff --git a/test-data/unit/fixtures/notimplemented.pyi b/test-data/unit/fixtures/notimplemented.pyi new file mode 100644 index 000000000000..92edf84a7fd1 --- /dev/null +++ b/test-data/unit/fixtures/notimplemented.pyi @@ -0,0 +1,18 @@ +# builtins stub used in NotImplemented related cases. +from typing import Any + +class object: + def __init__(self) -> None: pass + +class type: pass +class function: pass +class bool: pass +class int: pass +class str: pass +class dict: pass + +class _NotImplementedType(Any): + __call__: NotImplemented # type: ignore +NotImplemented: _NotImplementedType + +class BaseException: pass diff --git a/test-data/unit/fixtures/object_hashable.pyi b/test-data/unit/fixtures/object_hashable.pyi new file mode 100644 index 000000000000..49b17991f01c --- /dev/null +++ b/test-data/unit/fixtures/object_hashable.pyi @@ -0,0 +1,10 @@ +class object: + def __hash__(self) -> int: ... + +class type: ... +class int: ... +class float: ... +class str: ... +class ellipsis: ... +class tuple: ... +class dict: pass diff --git a/test-data/unit/fixtures/object_with_init_subclass.pyi b/test-data/unit/fixtures/object_with_init_subclass.pyi new file mode 100644 index 000000000000..da062349a204 --- /dev/null +++ b/test-data/unit/fixtures/object_with_init_subclass.pyi @@ -0,0 +1,61 @@ +from typing import Sequence, Iterator, TypeVar, Mapping, Iterable, Optional, Union, overload, Tuple, Generic, List + +class object: + def __init__(self) -> None: ... + def __init_subclass__(cls) -> None: ... + +T = TypeVar('T') +KT = TypeVar('KT') +VT = TypeVar('VT') +# copy pasted from primitives.pyi +class type: + def __init__(self, x) -> None: pass + +class int: + # Note: this is a simplification of the actual signature + def __init__(self, x: object = ..., base: int = ...) -> None: pass + def __add__(self, i: int) -> int: pass +class float: + def __float__(self) -> float: pass +class complex: pass +class bool(int): pass +class str(Sequence[str]): + def __add__(self, s: str) -> str: pass + def __iter__(self) -> Iterator[str]: pass + def __contains__(self, other: object) -> bool: pass + def __getitem__(self, item: int) -> str: pass + def format(self, *args) -> str: pass +class bytes(Sequence[int]): + def __iter__(self) -> Iterator[int]: pass + def __contains__(self, other: object) -> bool: pass + def __getitem__(self, item: int) -> int: pass +class bytearray: pass +class tuple(Generic[T]): pass +class function: pass +class ellipsis: pass + +# copy-pasted from list.pyi +class list(Sequence[T]): + def __iter__(self) -> Iterator[T]: pass + def __mul__(self, x: int) -> list[T]: pass + def __setitem__(self, x: int, v: T) -> None: pass + def __getitem__(self, x: int) -> T: pass + def __add__(self, x: List[T]) -> T: pass + def __contains__(self, item: object) -> bool: pass + +# copy-pasted from dict.pyi +class dict(Mapping[KT, VT]): + @overload + def __init__(self, **kwargs: VT) -> None: pass + @overload + def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: pass + def __getitem__(self, key: KT) -> VT: pass + def __setitem__(self, k: KT, v: VT) -> None: pass + def __iter__(self) -> Iterator[KT]: pass + def __contains__(self, item: object) -> int: pass + def update(self, a: Mapping[KT, VT]) -> None: pass + @overload + def get(self, k: KT) -> Optional[VT]: pass + @overload + def get(self, k: KT, default: Union[KT, T]) -> Union[VT, T]: pass + def __len__(self) -> int: ... diff --git a/test-data/unit/fixtures/ops.pyi b/test-data/unit/fixtures/ops.pyi new file mode 100644 index 000000000000..df3b163166ad --- /dev/null +++ b/test-data/unit/fixtures/ops.pyi @@ -0,0 +1,76 @@ +from typing import overload, Any, Generic, Sequence, Tuple, TypeVar, Optional + +Tco = TypeVar('Tco', covariant=True) + +# This is an extension of transform builtins with additional operations. + +class object: + def __init__(self) -> None: pass + def __eq__(self, o: 'object') -> 'bool': pass + def __ne__(self, o: 'object') -> 'bool': pass + +class type: pass + +class slice: pass + +class tuple(Sequence[Tco]): + def __getitem__(self, x: int) -> Tco: pass + def __eq__(self, x: object) -> bool: pass + def __ne__(self, x: object) -> bool: pass + def __lt__(self, x: Tuple[Tco, ...]) -> bool: pass + def __le__(self, x: Tuple[Tco, ...]) -> bool: pass + def __gt__(self, x: Tuple[Tco, ...]) -> bool: pass + def __ge__(self, x: Tuple[Tco, ...]) -> bool: pass + +class function: pass + +class str: + def __init__(self, x: 'int') -> None: pass + def __add__(self, x: 'str') -> 'str': pass + def __eq__(self, x: object) -> bool: pass + def startswith(self, x: 'str') -> bool: pass + def strip(self) -> 'str': pass + +class int: + def __add__(self, x: 'int') -> 'int': pass + def __radd__(self, x: 'int') -> 'int': pass + def __sub__(self, x: 'int') -> 'int': pass + def __mul__(self, x: 'int') -> 'int': pass + def __div__(self, x: 'int') -> 'int': pass + def __rdiv__(self, x: 'int') -> 'int': pass + def __truediv__(self, x: 'int') -> 'int': pass + def __rtruediv__(self, x: 'int') -> 'int': pass + def __mod__(self, x: 'int') -> 'int': pass + def __floordiv__(self, x: 'int') -> 'int': pass + def __pow__(self, x: 'int', __modulo: Optional[int] = ...) -> Any: pass + def __pos__(self) -> 'int': pass + def __neg__(self) -> 'int': pass + def __eq__(self, x: object) -> bool: pass + def __ne__(self, x: object) -> bool: pass + def __lt__(self, x: 'int') -> bool: pass + def __le__(self, x: 'int') -> bool: pass + def __gt__(self, x: 'int') -> bool: pass + def __ge__(self, x: 'int') -> bool: pass + +class bool(int): pass + +class float: + def __add__(self, x: 'float') -> 'float': pass + def __radd__(self, x: 'float') -> 'float': pass + def __div__(self, x: 'float') -> 'float': pass + def __rdiv__(self, x: 'float') -> 'float': pass + def __truediv__(self, x: 'float') -> 'float': pass + def __rtruediv__(self, x: 'float') -> 'float': pass + +class complex: + def __add__(self, x: complex) -> complex: pass + def __radd__(self, x: complex) -> complex: pass + +class BaseException: pass + +def __print(a1: object = None, a2: object = None, a3: object = None, + a4: object = None) -> None: pass + +class ellipsis: pass + +class dict: pass diff --git a/test-data/unit/fixtures/paramspec.pyi b/test-data/unit/fixtures/paramspec.pyi new file mode 100644 index 000000000000..dfb5e126f242 --- /dev/null +++ b/test-data/unit/fixtures/paramspec.pyi @@ -0,0 +1,79 @@ +# builtins stub for paramspec-related test cases + +import _typeshed +from typing import ( + Sequence, Generic, TypeVar, Iterable, Iterator, Tuple, Mapping, Optional, Union, Type, overload, + Protocol +) + +T = TypeVar("T") +T_co = TypeVar('T_co', covariant=True) +KT = TypeVar("KT") +VT = TypeVar("VT") + +class object: + def __init__(self) -> None: ... + +class function: ... +class ellipsis: ... +class classmethod: ... + +class type: + def __init__(self, *a: object) -> None: ... + def __call__(self, *a: object) -> object: ... + +class list(Sequence[T], Generic[T]): + @overload + def __getitem__(self, i: int) -> T: ... + @overload + def __getitem__(self, s: slice) -> list[T]: ... + def __contains__(self, item: object) -> bool: ... + def __iter__(self) -> Iterator[T]: ... + +class int: + def __neg__(self) -> int: ... + def __add__(self, other: int) -> int: ... + +class bool(int): ... +class float: ... +class slice: ... +class str: ... +class bytes: ... + +class tuple(Sequence[T_co], Generic[T_co]): + def __new__(cls: Type[T], iterable: Iterable[T_co] = ...) -> T: ... + def __iter__(self) -> Iterator[T_co]: ... + def __contains__(self, item: object) -> bool: ... + def __getitem__(self, x: int) -> T_co: ... + def __mul__(self, n: int) -> Tuple[T_co, ...]: ... + def __rmul__(self, n: int) -> Tuple[T_co, ...]: ... + def __add__(self, x: Tuple[T_co, ...]) -> Tuple[T_co, ...]: ... + def __len__(self) -> int: ... + def count(self, obj: object) -> int: ... + +class _ItemsView(Iterable[Tuple[KT, VT]]): ... + +class dict(Mapping[KT, VT]): + @overload + def __init__(self, **kwargs: VT) -> None: ... + @overload + def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: ... + def __getitem__(self, key: KT) -> VT: ... + def __setitem__(self, k: KT, v: VT) -> None: ... + def __iter__(self) -> Iterator[KT]: ... + def __contains__(self, item: object) -> int: ... + def update(self, a: Mapping[KT, VT]) -> None: ... + @overload + def get(self, k: KT) -> Optional[VT]: ... + @overload + def get(self, k: KT, default: Union[KT, T]) -> Union[VT, T]: ... + def __len__(self) -> int: ... + def pop(self, k: KT) -> VT: ... + def items(self) -> _ItemsView[KT, VT]: ... + +def isinstance(x: object, t: type) -> bool: ... + +class _Sized(Protocol): + def __len__(self) -> int: ... + +def len(x: _Sized) -> int: ... diff --git a/test-data/unit/fixtures/plugin_attrs.pyi b/test-data/unit/fixtures/plugin_attrs.pyi new file mode 100644 index 000000000000..7fd641727253 --- /dev/null +++ b/test-data/unit/fixtures/plugin_attrs.pyi @@ -0,0 +1,39 @@ +# Builtins stub used to support attrs plugin tests. +from typing import Union, overload, Generic, Sequence, TypeVar, Type, Iterable, Iterator + +class object: + def __init__(self) -> None: pass + def __eq__(self, o: object) -> bool: pass + def __ne__(self, o: object) -> bool: pass + def __hash__(self) -> int: ... + +class type: pass +class bytes: pass +class function: pass +class float: pass +class int: + @overload + def __init__(self, x: Union[str, bytes, int] = ...) -> None: ... + @overload + def __init__(self, x: Union[str, bytes], base: int) -> None: ... +class bool(int): pass +class complex: + @overload + def __init__(self, real: float = ..., im: float = ...) -> None: ... + @overload + def __init__(self, real: str = ...) -> None: ... + +class str: pass +class ellipsis: pass +class list: pass +class dict: pass + +T = TypeVar("T") +Tco = TypeVar('Tco', covariant=True) +class tuple(Sequence[Tco], Generic[Tco]): + def __new__(cls: Type[T], iterable: Iterable[Tco] = ...) -> T: ... + def __iter__(self) -> Iterator[Tco]: pass + def __contains__(self, item: object) -> bool: pass + def __getitem__(self, x: int) -> Tco: pass + +property = object() # Dummy definition diff --git a/test-data/unit/fixtures/primitives.pyi b/test-data/unit/fixtures/primitives.pyi new file mode 100644 index 000000000000..63128a8ae03d --- /dev/null +++ b/test-data/unit/fixtures/primitives.pyi @@ -0,0 +1,71 @@ +# builtins stub with non-generic primitive types +import _typeshed +from typing import Generic, TypeVar, Sequence, Iterator, Mapping, Iterable, Tuple, Union + +T = TypeVar('T') +V = TypeVar('V') + +class object: + def __init__(self) -> None: pass + def __str__(self) -> str: pass + def __eq__(self, other: object) -> bool: pass + def __ne__(self, other: object) -> bool: pass + +class type: + def __init__(self, x: object) -> None: pass + +class int: + # Note: this is a simplification of the actual signature + def __init__(self, x: object = ..., base: int = ...) -> None: pass + def __add__(self, i: int) -> int: pass + def __rmul__(self, x: int) -> int: pass +class float: + def __float__(self) -> float: pass + def __add__(self, x: float) -> float: pass +class complex: + def __add__(self, x: complex) -> complex: pass +class bool(int): pass +class str(Sequence[str]): + def __add__(self, s: str) -> str: pass + def __iter__(self) -> Iterator[str]: pass + def __contains__(self, other: object) -> bool: pass + def __getitem__(self, item: int) -> str: pass + def format(self, *args: object, **kwargs: object) -> str: pass +class bytes(Sequence[int]): + def __iter__(self) -> Iterator[int]: pass + def __contains__(self, other: object) -> bool: pass + def __getitem__(self, item: int) -> int: pass +class bytearray(Sequence[int]): + def __init__(self, x: bytes) -> None: pass + def __iter__(self) -> Iterator[int]: pass + def __contains__(self, other: object) -> bool: pass + def __getitem__(self, item: int) -> int: pass +class memoryview(Sequence[int]): + def __init__(self, x: bytes) -> None: pass + def __iter__(self) -> Iterator[int]: pass + def __contains__(self, other: object) -> bool: pass + def __getitem__(self, item: int) -> int: pass +class tuple(Generic[T]): + def __contains__(self, other: object) -> bool: pass +class list(Sequence[T]): + def __iter__(self) -> Iterator[T]: pass + def __contains__(self, other: object) -> bool: pass + def __getitem__(self, item: int) -> T: pass +class dict(Mapping[T, V]): + def __iter__(self) -> Iterator[T]: pass +class set(Iterable[T]): + def __iter__(self) -> Iterator[T]: pass +class frozenset(Iterable[T]): + def __iter__(self) -> Iterator[T]: pass +class function: pass +class ellipsis: pass + +class range(Sequence[int]): + def __init__(self, __x: int, __y: int = ..., __z: int = ...) -> None: pass + def count(self, value: int) -> int: pass + def index(self, value: int) -> int: pass + def __getitem__(self, i: int) -> int: pass + def __iter__(self) -> Iterator[int]: pass + def __contains__(self, other: object) -> bool: pass + +def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass diff --git a/test-data/unit/fixtures/property.pyi b/test-data/unit/fixtures/property.pyi new file mode 100644 index 000000000000..667bdc02d0f5 --- /dev/null +++ b/test-data/unit/fixtures/property.pyi @@ -0,0 +1,25 @@ +import typing + +_T = typing.TypeVar('_T') + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x: typing.Any) -> None: pass + +class function: pass + +property = object() # Dummy definition +class classmethod: pass + +class list: pass +class dict: pass +class int: pass +class float: pass +class str: pass +class bytes: pass +class bool: pass +class ellipsis: pass + +class tuple(typing.Generic[_T]): pass diff --git a/test-data/unit/fixtures/set.pyi b/test-data/unit/fixtures/set.pyi new file mode 100644 index 000000000000..71d3bd2eee18 --- /dev/null +++ b/test-data/unit/fixtures/set.pyi @@ -0,0 +1,29 @@ +# Builtins stub used in set-related test cases. + +from typing import TypeVar, Generic, Iterator, Iterable, Set + +T = TypeVar('T') + +class object: + def __init__(self) -> None: pass + def __eq__(self, other): pass + +class type: pass +class tuple(Generic[T]): pass +class function: pass + +class int: pass +class str: pass +class bool: pass +class ellipsis: pass + +class set(Iterable[T], Generic[T]): + def __init__(self, iterable: Iterable[T] = ...) -> None: ... + def __iter__(self) -> Iterator[T]: pass + def __contains__(self, item: object) -> bool: pass + def __ior__(self, x: Set[T]) -> None: pass + def add(self, x: T) -> None: pass + def discard(self, x: T) -> None: pass + def update(self, x: Set[T]) -> None: pass + +class dict: pass diff --git a/test-data/unit/fixtures/slice.pyi b/test-data/unit/fixtures/slice.pyi new file mode 100644 index 000000000000..b22a12b5213f --- /dev/null +++ b/test-data/unit/fixtures/slice.pyi @@ -0,0 +1,19 @@ +# Builtins stub used in slicing test cases. +from typing import Generic, TypeVar +T = TypeVar('T') + +class object: + def __init__(self): pass + +class type: pass +class tuple(Generic[T]): pass +class function: pass + +class int: pass +class str: pass + +class slice: pass +class ellipsis: pass +class dict: pass +class list(Generic[T]): + def __getitem__(self, x: slice) -> list[T]: pass diff --git a/test-data/unit/fixtures/staticmethod.pyi b/test-data/unit/fixtures/staticmethod.pyi new file mode 100644 index 000000000000..a0ca831c7527 --- /dev/null +++ b/test-data/unit/fixtures/staticmethod.pyi @@ -0,0 +1,22 @@ +import typing + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + +class function: pass + +staticmethod = object() # Dummy definition. +property = object() # Dummy definition + +class int: + @staticmethod + def from_bytes(bytes: bytes, byteorder: str) -> int: pass + +class str: pass +class bytes: pass +class ellipsis: pass +class dict: pass +class tuple: pass diff --git a/test-data/unit/fixtures/transform.pyi b/test-data/unit/fixtures/transform.pyi new file mode 100644 index 000000000000..7dbb8fa90dbe --- /dev/null +++ b/test-data/unit/fixtures/transform.pyi @@ -0,0 +1,32 @@ +# Builtins stubs used implicitly in program transformation test cases. + +class object: + def __init__(self) -> None: pass + +class type: pass + +# str is handy for debugging; allows outputting messages. +class str: pass + +# Primitive types int/float have special coercion behaviour (they may have +# a different representation from ordinary values). + +class int: pass + +class float: pass + + +# The functions below are special functions used in test cases; their +# implementations are actually in the __dynchk module, but they are defined +# here so that the semantic analyzer and the type checker are happy without +# having to analyze the entire __dynchk module all the time. +# +# The transformation implementation has special case handling for these +# functions; it's a bit ugly but it works for now. + +def __print(a1=None, a2=None, a3=None, a4=None): + # Do not use *args since this would require list and break many test + # cases. + pass + +class dict: pass diff --git a/test-data/unit/fixtures/tuple-simple.pyi b/test-data/unit/fixtures/tuple-simple.pyi new file mode 100644 index 000000000000..07f9edf63cdd --- /dev/null +++ b/test-data/unit/fixtures/tuple-simple.pyi @@ -0,0 +1,21 @@ +# Builtins stub used in some tuple-related test cases. +# +# This is a simpler version of tuple.py which is useful +# and makes some test cases easier to write/debug. + +from typing import Iterable, TypeVar, Generic + +T = TypeVar('T', covariant=True) + +class object: + def __init__(self): pass + +class type: pass +class tuple(Generic[T]): + def __getitem__(self, x: int) -> T: pass +class function: pass + +# We need int for indexing tuples. +class int: pass +class str: pass # For convenience +class dict: pass diff --git a/test-data/unit/fixtures/tuple.pyi b/test-data/unit/fixtures/tuple.pyi new file mode 100644 index 000000000000..3b62d7fc1513 --- /dev/null +++ b/test-data/unit/fixtures/tuple.pyi @@ -0,0 +1,55 @@ +# Builtins stub used in tuple-related test cases. + +import _typeshed +from typing import Iterable, Iterator, TypeVar, Generic, Sequence, Optional, overload, Tuple, Type + +_T = TypeVar("_T") +_Tco = TypeVar('_Tco', covariant=True) + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, *a: object) -> None: pass + def __call__(self, *a: object) -> object: pass +class tuple(Sequence[_Tco], Generic[_Tco]): + def __new__(cls: Type[_T], iterable: Iterable[_Tco] = ...) -> _T: ... + def __iter__(self) -> Iterator[_Tco]: pass + def __contains__(self, item: object) -> bool: pass + @overload + def __getitem__(self, x: int) -> _Tco: pass + @overload + def __getitem__(self, x: slice) -> Tuple[_Tco, ...]: ... + def __mul__(self, n: int) -> Tuple[_Tco, ...]: pass + def __rmul__(self, n: int) -> Tuple[_Tco, ...]: pass + def __add__(self, x: Tuple[_Tco, ...]) -> Tuple[_Tco, ...]: pass + def count(self, obj: object) -> int: pass +class function: + __name__: str +class ellipsis: pass +class classmethod: pass + +# We need int and slice for indexing tuples. +class int: + def __neg__(self) -> 'int': pass + def __pos__(self) -> 'int': pass +class float: pass +class slice: pass +class bool(int): pass +class str: pass # For convenience +class bytes: pass +class bytearray: pass + +class list(Sequence[_T], Generic[_T]): + @overload + def __getitem__(self, i: int) -> _T: ... + @overload + def __getitem__(self, s: slice) -> list[_T]: ... + def __contains__(self, item: object) -> bool: ... + def __iter__(self) -> Iterator[_T]: ... + +def isinstance(x: object, t: type) -> bool: pass + +class BaseException: pass + +class dict: pass diff --git a/test-data/unit/fixtures/type.pyi b/test-data/unit/fixtures/type.pyi new file mode 100644 index 000000000000..0d93b2e1fcd6 --- /dev/null +++ b/test-data/unit/fixtures/type.pyi @@ -0,0 +1,35 @@ +# builtins stub used in type-related test cases. + +from typing import Any, Generic, TypeVar, List, Union +import sys +import types + +T = TypeVar("T") +S = TypeVar("S") + +class object: + def __init__(self) -> None: pass + def __str__(self) -> 'str': pass + +class list(Generic[T]): pass + +class type: + __name__: str + def __call__(self, *args: Any, **kwargs: Any) -> Any: pass + def __or__(self, other: Union[type, None]) -> type: pass + def __ror__(self, other: Union[type, None]) -> type: pass + def mro(self) -> List['type']: pass + +class tuple(Generic[T]): pass +class dict(Generic[T, S]): pass +class function: pass +class bool: pass +class int: pass +class str: pass +class ellipsis: pass +class float: pass + +if sys.version_info >= (3, 10): # type: ignore + def isinstance(obj: object, class_or_tuple: type | types.UnionType, /) -> bool: ... +else: + def isinstance(obj: object, class_or_tuple: type, /) -> bool: ... diff --git a/test-data/unit/fixtures/typing-async.pyi b/test-data/unit/fixtures/typing-async.pyi new file mode 100644 index 000000000000..03728f822316 --- /dev/null +++ b/test-data/unit/fixtures/typing-async.pyi @@ -0,0 +1,129 @@ +# Test stub for typing module, with features for async/await related tests. +# +# Use [typing fixtures/typing-async.pyi] to use this instead of lib-stub/typing.pyi +# in a particular test case. +# +# Many of the definitions have special handling in the type checker, so they +# can just be initialized to anything. + +from abc import abstractmethod, ABCMeta + +cast = 0 +overload = 0 +Any = object() +Union = 0 +Optional = 0 +TypeVar = 0 +Generic = 0 +Protocol = 0 +Tuple = 0 +Callable = 0 +NamedTuple = 0 +Type = 0 +ClassVar = 0 +Final = 0 +Literal = 0 +NoReturn = 0 +Self = 0 + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) +U = TypeVar('U') +V = TypeVar('V') +S = TypeVar('S') + +# Note: definitions below are different from typeshed, variances are declared +# to silence the protocol variance checks. Maybe it is better to use type: ignore? + +class Container(Protocol[T_co]): + @abstractmethod + # Use int because bool isn't in the default test builtins + def __contains__(self, arg: object) -> int: pass + +class Iterable(Protocol[T_co]): + @abstractmethod + def __iter__(self) -> 'Iterator[T_co]': pass + +class Iterator(Iterable[T_co], Protocol): + @abstractmethod + def __next__(self) -> T_co: pass + +class Generator(Iterator[T], Generic[T, U, V]): + @abstractmethod + def send(self, value: U) -> T: pass + + @abstractmethod + def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass + + @abstractmethod + def close(self) -> None: pass + + @abstractmethod + def __iter__(self) -> 'Generator[T, U, V]': pass + +class AsyncGenerator(AsyncIterator[T], Generic[T, U]): + @abstractmethod + def __anext__(self) -> Awaitable[T]: pass + + @abstractmethod + def asend(self, value: U) -> Awaitable[T]: pass + + @abstractmethod + def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T]: pass + + @abstractmethod + def aclose(self) -> Awaitable[T]: pass + + @abstractmethod + def __aiter__(self) -> 'AsyncGenerator[T, U]': pass + +class Awaitable(Protocol[T]): + @abstractmethod + def __await__(self) -> Generator[Any, Any, T]: pass + +class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S], metaclass=ABCMeta): + pass + +class Coroutine(Awaitable[V], Generic[T, U, V]): + @abstractmethod + def send(self, value: U) -> T: pass + + @abstractmethod + def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass + + @abstractmethod + def close(self) -> None: pass + +class AsyncIterable(Protocol[T]): + @abstractmethod + def __aiter__(self) -> 'AsyncIterator[T]': pass + +class AsyncIterator(AsyncIterable[T], Protocol): + def __aiter__(self) -> 'AsyncIterator[T]': return self + @abstractmethod + def __anext__(self) -> Awaitable[T]: pass + +class Sequence(Iterable[T_co], Container[T_co]): + @abstractmethod + def __getitem__(self, n: Any) -> T_co: pass + +class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + @overload + def get(self, k: T) -> Optional[T_co]: pass + @overload + def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass + +class ContextManager(Generic[T]): + def __enter__(self) -> T: pass + # Use Any because not all the precise types are in the fixtures. + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass + +class AsyncContextManager(Generic[T]): + def __aenter__(self) -> Awaitable[T]: pass + # Use Any because not all the precise types are in the fixtures. + def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Awaitable[Any]: pass + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi new file mode 100644 index 000000000000..8e0116aab1c2 --- /dev/null +++ b/test-data/unit/fixtures/typing-full.pyi @@ -0,0 +1,221 @@ +# More complete stub for typing module. +# +# Use [typing fixtures/typing-full.pyi] to use this instead of lib-stub/typing.pyi +# in a particular test case. +# +# Many of the definitions have special handling in the type checker, so they +# can just be initialized to anything. + +from abc import abstractmethod, ABCMeta + +class GenericMeta(type): pass + +class _SpecialForm: + def __getitem__(self, index: Any) -> Any: ... + def __or__(self, other): ... + def __ror__(self, other): ... +class TypeVar: + def __init__(self, name, *args, bound=None): ... + def __or__(self, other): ... +class ParamSpec: ... +class TypeVarTuple: ... + +def cast(t, o): ... +def assert_type(o, t): ... +overload = 0 +Any = object() +Optional = 0 +Generic = 0 +Protocol = 0 +Tuple = 0 +_promote = 0 +Type = 0 +no_type_check = 0 +ClassVar = 0 +Final = 0 +TypedDict = 0 +NoReturn = 0 +NewType = 0 +Self = 0 +Unpack = 0 +Callable: _SpecialForm +Union: _SpecialForm +Literal: _SpecialForm + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) +U = TypeVar('U') +V = TypeVar('V') +S = TypeVar('S') + +def final(x: T) -> T: ... + +class NamedTuple(tuple[Any, ...]): ... + +# Note: definitions below are different from typeshed, variances are declared +# to silence the protocol variance checks. Maybe it is better to use type: ignore? + +@runtime_checkable +class Hashable(Protocol, metaclass=ABCMeta): + @abstractmethod + def __hash__(self) -> int: pass + +@runtime_checkable +class Container(Protocol[T_co]): + @abstractmethod + # Use int because bool isn't in the default test builtins + def __contains__(self, arg: object) -> int: pass + +@runtime_checkable +class Sized(Protocol): + @abstractmethod + def __len__(self) -> int: pass + +@runtime_checkable +class Iterable(Protocol[T_co]): + @abstractmethod + def __iter__(self) -> 'Iterator[T_co]': pass + +@runtime_checkable +class Iterator(Iterable[T_co], Protocol): + @abstractmethod + def __next__(self) -> T_co: pass + +class Generator(Iterator[T], Generic[T, U, V]): + @abstractmethod + def send(self, value: U) -> T: pass + + @abstractmethod + def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass + + @abstractmethod + def close(self) -> None: pass + + @abstractmethod + def __iter__(self) -> 'Generator[T, U, V]': pass + +class AsyncGenerator(AsyncIterator[T], Generic[T, U]): + @abstractmethod + def __anext__(self) -> Awaitable[T]: pass + + @abstractmethod + def asend(self, value: U) -> Awaitable[T]: pass + + @abstractmethod + def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T]: pass + + @abstractmethod + def aclose(self) -> Awaitable[T]: pass + + @abstractmethod + def __aiter__(self) -> 'AsyncGenerator[T, U]': pass + +@runtime_checkable +class Awaitable(Protocol[T]): + @abstractmethod + def __await__(self) -> Generator[Any, Any, T]: pass + +class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S], metaclass=ABCMeta): + pass + +class Coroutine(Awaitable[V], Generic[T, U, V]): + @abstractmethod + def send(self, value: U) -> T: pass + + @abstractmethod + def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass + + @abstractmethod + def close(self) -> None: pass + +@runtime_checkable +class AsyncIterable(Protocol[T]): + @abstractmethod + def __aiter__(self) -> 'AsyncIterator[T]': pass + +@runtime_checkable +class AsyncIterator(AsyncIterable[T], Protocol): + def __aiter__(self) -> 'AsyncIterator[T]': return self + @abstractmethod + def __anext__(self) -> Awaitable[T]: pass + +class Sequence(Iterable[T_co], Container[T_co]): + @abstractmethod + def __getitem__(self, n: Any) -> T_co: pass + +class MutableSequence(Sequence[T]): + @abstractmethod + def __setitem__(self, n: Any, o: T) -> None: pass + +class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + @overload + def get(self, k: T) -> Optional[T_co]: pass + @overload + def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass + def values(self) -> Iterable[T_co]: pass # Approximate return type + def __len__(self) -> int: ... + def __contains__(self, arg: object) -> int: pass + +class MutableMapping(Mapping[T, U], metaclass=ABCMeta): + def __setitem__(self, k: T, v: U) -> None: pass + +class SupportsInt(Protocol): + def __int__(self) -> int: pass + +class SupportsFloat(Protocol): + def __float__(self) -> float: pass + +class SupportsAbs(Protocol[T_co]): + def __abs__(self) -> T_co: pass + +def runtime_checkable(cls: T) -> T: + return cls + +class ContextManager(Generic[T_co]): + def __enter__(self) -> T_co: pass + # Use Any because not all the precise types are in the fixtures. + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass + +TYPE_CHECKING = 1 + +# Fallback type for all typed dicts (does not exist at runtime). +class _TypedDict(Mapping[str, object]): + # Needed to make this class non-abstract. It is explicitly declared abstract in + # typeshed, but we don't want to import abc here, as it would slow down the tests. + def __iter__(self) -> Iterator[str]: ... + def copy(self: T) -> T: ... + # Using NoReturn so that only calls using the plugin hook can go through. + def setdefault(self, k: NoReturn, default: object) -> object: ... + # Mypy expects that 'default' has a type variable type. + def pop(self, k: NoReturn, default: T = ...) -> object: ... + def update(self: T, __m: T) -> None: ... + def __delitem__(self, k: NoReturn) -> None: ... + +def dataclass_transform( + *, + eq_default: bool = ..., + order_default: bool = ..., + kw_only_default: bool = ..., + field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = ..., + **kwargs: Any, +) -> Callable[[T], T]: ... +def override(__arg: T) -> T: ... + +# Was added in 3.11 +def reveal_type(__obj: T) -> T: ... + +# Only exists in type checking time: +def type_check_only(__func_or_class: T) -> T: ... + +# Was added in 3.12 +@final +class TypeAliasType: + def __init__( + self, name: str, value: Any, *, type_params: Tuple[Union[TypeVar, ParamSpec, TypeVarTuple], ...] = () + ) -> None: ... + + def __or__(self, other: Any) -> Any: ... + def __ror__(self, other: Any) -> Any: ... diff --git a/test-data/unit/fixtures/typing-medium.pyi b/test-data/unit/fixtures/typing-medium.pyi new file mode 100644 index 000000000000..c722a9ddb12c --- /dev/null +++ b/test-data/unit/fixtures/typing-medium.pyi @@ -0,0 +1,75 @@ +# More complete stub for typing module. +# +# Use [typing fixtures/typing-medium.pyi] to use this instead of lib-stub/typing.pyi +# in a particular test case. +# +# Many of the definitions have special handling in the type checker, so they +# can just be initialized to anything. + +cast = 0 +overload = 0 +Any = object() +Union = 0 +Optional = 0 +TypeVar = 0 +Generic = 0 +Protocol = 0 +Tuple = 0 +Callable = 0 +_promote = 0 +NamedTuple = 0 +Type = 0 +no_type_check = 0 +ClassVar = 0 +Final = 0 +Literal = 0 +TypedDict = 0 +NoReturn = 0 +NewType = 0 +TypeAlias = 0 +LiteralString = 0 +Self = 0 + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) +U = TypeVar('U') +V = TypeVar('V') +S = TypeVar('S') + +# Note: definitions below are different from typeshed, variances are declared +# to silence the protocol variance checks. Maybe it is better to use type: ignore? + +class Sized(Protocol): + def __len__(self) -> int: pass + +class Iterable(Protocol[T_co]): + def __iter__(self) -> 'Iterator[T_co]': pass + +class Iterator(Iterable[T_co], Protocol): + def __next__(self) -> T_co: pass + +class Generator(Iterator[T], Generic[T, U, V]): + def __iter__(self) -> 'Generator[T, U, V]': pass + +class Sequence(Iterable[T_co]): + def __getitem__(self, n: Any) -> T_co: pass + +class Mapping(Iterable[T], Generic[T, T_co]): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + +class SupportsInt(Protocol): + def __int__(self) -> int: pass + +class SupportsFloat(Protocol): + def __float__(self) -> float: pass + +class ContextManager(Generic[T]): + def __enter__(self) -> T: pass + # Use Any because not all the precise types are in the fixtures. + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass + +class _SpecialForm: pass + +TYPE_CHECKING = 1 diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi new file mode 100644 index 000000000000..bcdcfc44c3d2 --- /dev/null +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -0,0 +1,30 @@ +TypeVar = 0 +Generic = 0 +Any = object() +overload = 0 +Type = 0 +Literal = 0 +Optional = 0 +Self = 0 +Tuple = 0 +ClassVar = 0 + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +KT = TypeVar('KT') + +class Iterable(Generic[T_co]): pass +class Iterator(Iterable[T_co]): pass +class Sequence(Iterable[T_co]): pass +class Mapping(Iterable[KT], Generic[KT, T_co]): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + +class NamedTuple(tuple[Any, ...]): + _fields: ClassVar[tuple[str, ...]] + @overload + def __init__(self, typename: str, fields: Iterable[tuple[str, Any]] = ...) -> None: ... + @overload + def __init__(self, typename: str, fields: None = None, **kwargs: Any) -> None: ... + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-override.pyi b/test-data/unit/fixtures/typing-override.pyi new file mode 100644 index 000000000000..e9d2dfcf55c4 --- /dev/null +++ b/test-data/unit/fixtures/typing-override.pyi @@ -0,0 +1,26 @@ +TypeVar = 0 +Generic = 0 +Any = object() +overload = 0 +Type = 0 +Literal = 0 +Optional = 0 +Self = 0 +Tuple = 0 +ClassVar = 0 +Callable = 0 + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +KT = TypeVar('KT') + +class Iterable(Generic[T_co]): pass +class Iterator(Iterable[T_co]): pass +class Sequence(Iterable[T_co]): pass +class Mapping(Iterable[KT], Generic[KT, T_co]): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + +def override(__arg: T) -> T: ... + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-typeddict-iror.pyi b/test-data/unit/fixtures/typing-typeddict-iror.pyi new file mode 100644 index 000000000000..845ac6cf208f --- /dev/null +++ b/test-data/unit/fixtures/typing-typeddict-iror.pyi @@ -0,0 +1,68 @@ +# Test stub for typing module that includes TypedDict `|` operator. +# It only covers `__or__`, `__ror__`, and `__ior__`. +# +# We cannot define these methods in `typing-typeddict.pyi`, +# because they need `dict` with two type args, +# and not all tests using `[typing typing-typeddict.pyi]` have the proper +# `dict` stub. +# +# Keep in sync with `typeshed`'s definition. +from abc import ABCMeta + +cast = 0 +assert_type = 0 +overload = 0 +Any = object() +Union = 0 +Optional = 0 +TypeVar = 0 +Generic = 0 +Protocol = 0 +Tuple = 0 +Callable = 0 +NamedTuple = 0 +Final = 0 +Literal = 0 +TypedDict = 0 +NoReturn = 0 +Required = 0 +NotRequired = 0 +Self = 0 + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +V = TypeVar('V') + +# Note: definitions below are different from typeshed, variances are declared +# to silence the protocol variance checks. Maybe it is better to use type: ignore? + +class Sized(Protocol): + def __len__(self) -> int: pass + +class Iterable(Protocol[T_co]): + def __iter__(self) -> 'Iterator[T_co]': pass + +class Iterator(Iterable[T_co], Protocol): + def __next__(self) -> T_co: pass + +class Sequence(Iterable[T_co]): + # misc is for explicit Any. + def __getitem__(self, n: Any) -> T_co: pass # type: ignore[misc] + +class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): + pass + +# Fallback type for all typed dicts (does not exist at runtime). +class _TypedDict(Mapping[str, object]): + @overload + def __or__(self, __value: Self) -> Self: ... + @overload + def __or__(self, __value: dict[str, Any]) -> dict[str, object]: ... + @overload + def __ror__(self, __value: Self) -> Self: ... + @overload + def __ror__(self, __value: dict[str, Any]) -> dict[str, object]: ... + # supposedly incompatible definitions of __or__ and __ior__ + def __ior__(self, __value: Self) -> Self: ... # type: ignore[misc] + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi new file mode 100644 index 000000000000..7e9c642cf261 --- /dev/null +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -0,0 +1,80 @@ +# Test stub for typing module that includes TypedDict related things. +# +# Use [typing fixtures/typing-typeddict.pyi] to use this instead of lib-stub/typing.pyi +# in a particular test case. +# +# Many of the definitions have special handling in the type checker, so they +# can just be initialized to anything. + +from abc import ABCMeta + +cast = 0 +assert_type = 0 +overload = 0 +Any = object() +Union = 0 +Optional = 0 +TypeVar = 0 +Generic = 0 +Protocol = 0 +Tuple = 0 +Callable = 0 +NamedTuple = 0 +Final = 0 +Literal = 0 +TypedDict = 0 +NoReturn = 0 +Required = 0 +NotRequired = 0 +ReadOnly = 0 +Self = 0 + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +V = TypeVar('V') + +# Note: definitions below are different from typeshed, variances are declared +# to silence the protocol variance checks. Maybe it is better to use type: ignore? + +class Sized(Protocol): + def __len__(self) -> int: pass + +class Iterable(Protocol[T_co]): + def __iter__(self) -> 'Iterator[T_co]': pass + +class Iterator(Iterable[T_co], Protocol): + def __next__(self) -> T_co: pass + +class Sequence(Iterable[T_co]): + # misc is for explicit Any. + def __getitem__(self, n: Any) -> T_co: pass # type: ignore[misc] + +class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + @overload + def get(self, k: T) -> Optional[T_co]: pass + @overload + def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass + def values(self) -> Iterable[T_co]: pass # Approximate return type + def __len__(self) -> int: ... + def __contains__(self, arg: object) -> int: pass + +class MutableMapping(Mapping[T, T_co], Generic[T, T_co], metaclass=ABCMeta): + # Other methods are not used in tests. + def clear(self) -> None: ... + +# Fallback type for all typed dicts (does not exist at runtime). +class _TypedDict(Mapping[str, object]): + # Needed to make this class non-abstract. It is explicitly declared abstract in + # typeshed, but we don't want to import abc here, as it would slow down the tests. + def __iter__(self) -> Iterator[str]: ... + def copy(self: T) -> T: ... + # Using NoReturn so that only calls using the plugin hook can go through. + def setdefault(self, k: NoReturn, default: object) -> object: ... + # Mypy expects that 'default' has a type variable type. + def pop(self, k: NoReturn, default: T = ...) -> object: ... + def update(self: T, __m: T) -> None: ... + def __delitem__(self, k: NoReturn) -> None: ... + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/union.pyi b/test-data/unit/fixtures/union.pyi new file mode 100644 index 000000000000..350e145a6f8f --- /dev/null +++ b/test-data/unit/fixtures/union.pyi @@ -0,0 +1,18 @@ +# Builtins stub used in tuple-related test cases. + +from isinstance import isinstance +from typing import Iterable, TypeVar, Generic +T = TypeVar('T') + +class object: + def __init__(self): pass + +class type: pass +class function: pass + +class tuple(Generic[T]): pass + +# We need int for indexing tuples. +class int: pass +class str: pass # For convenience +class dict: pass diff --git a/test-data/unit/hacks.txt b/test-data/unit/hacks.txt new file mode 100644 index 000000000000..15b1065cb7a9 --- /dev/null +++ b/test-data/unit/hacks.txt @@ -0,0 +1,99 @@ +Weird legacy stuff in test cases +================================ + +Due to historical reasons, test cases contain things that may appear +baffling without extra context. This file attempts to describe most of +them. + +Dummy if statements to prevent redefinition +------------------------------------------- + +Many test cases use if statements to prevent an assignment from creating +a new variable. This in anticipation of allowing assignments to redefine +variables by default. Conditional assignments will continue to refine +a previously defined variable instead of defining a new one. When the +test cases were written, we didn't anticipate that variables could be +allowed to be redefined, and adding if statements was the easiest way +to migrate these tests. + +Example: + +``` +x = 0 +if int(): + x = '' # Always generates an error since this is not a redefinition + +y = 0 +y = '' # This could be valid if a new 'y' is defined here +``` + +Note that some of the checks may turn out to be redundant, as the +exact rules for what constitutes a redefinition are still up for +debate. This is okay since the extra if statements generally don't +otherwise affect semantics. + +There are a few ways this is used, depending on the context: + +* `if int():` is the most common one. Assignments in the if body won't + redefine variables defined before the if statement. +* `if 1:` is used if the body of the if statement returns a value, and + mypy would complain about a missing return statement otherwise. This + works since `if 1:` is treated as an always taken condition, whereas + `if int():` is not recognized as such. +* `if str():` is used if the builtins fixture doesn't define `int` for + some reason. + +Function definition to prevent redefinition +------------------------------------------- + +Sometimes test cases assume that a variable is not redefined, and we +insert a dummy function definition to prevent this, since variables won't +be able to be redefined across a function definition. Example: + +``` +x = 0 + +def f(): pass + +x = '' # Does not redefine x because of the definition of f() above +``` + +Dummy variable reference to allow redefinition +---------------------------------------------- + +The plan is to only allow a variable to be redefined if the value has +been accessed. This wouldn't count as redefinition, since `x` is never +read: + +``` +x = 0 +x = '' # Not a redefinition +``` + +Sometimes we add a dummy variable access to allow redefinition in the +future, or to trigger the redefinition machinery even if redefinition +should not be okay: + +``` +x = 0 +x +x = '' # Could be a redefinition +``` + +The reason for this special case is type comments with dummy +initializers, where the second assignment should never be treated +as a redefinition: + +``` +x = None # type: int +x = '' # Should not redefine x, since it has only been declared +``` + +Similarly, if there is only a variable annotation, the first +assignment won't redefine the variable, as this would override +the declared type: + +``` +x: int +x = '' # Should not redefine x +``` diff --git a/test-data/unit/lib-stub/_decimal.pyi b/test-data/unit/lib-stub/_decimal.pyi new file mode 100644 index 000000000000..2c2c5bff11f7 --- /dev/null +++ b/test-data/unit/lib-stub/_decimal.pyi @@ -0,0 +1,4 @@ +# Very simplified decimal stubs for use in tests + +class Decimal: + def __new__(cls, value: str = ...) -> Decimal: ... diff --git a/test-data/unit/lib-stub/_typeshed.pyi b/test-data/unit/lib-stub/_typeshed.pyi new file mode 100644 index 000000000000..054ad0ec0c46 --- /dev/null +++ b/test-data/unit/lib-stub/_typeshed.pyi @@ -0,0 +1,8 @@ +from typing import Protocol, TypeVar, Iterable + +_KT = TypeVar("_KT") +_VT_co = TypeVar("_VT_co", covariant=True) + +class SupportsKeysAndGetItem(Protocol[_KT, _VT_co]): + def keys(self) -> Iterable[_KT]: pass + def __getitem__(self, __key: _KT) -> _VT_co: pass diff --git a/test-data/unit/lib-stub/abc.pyi b/test-data/unit/lib-stub/abc.pyi new file mode 100644 index 000000000000..e60f709a5187 --- /dev/null +++ b/test-data/unit/lib-stub/abc.pyi @@ -0,0 +1,9 @@ +from typing import Type, Any, TypeVar + +T = TypeVar('T', bound=Type[Any]) + +class ABCMeta(type): + def register(cls, tp: T) -> T: pass +class ABC(metaclass=ABCMeta): pass +abstractmethod = object() +abstractproperty = object() diff --git a/test-data/unit/lib-stub/attr/__init__.pyi b/test-data/unit/lib-stub/attr/__init__.pyi new file mode 100644 index 000000000000..466c6913062d --- /dev/null +++ b/test-data/unit/lib-stub/attr/__init__.pyi @@ -0,0 +1,253 @@ +from typing import TypeVar, overload, Callable, Any, Type, Optional, Union, Sequence, Mapping, Generic + +_T = TypeVar('_T') +_C = TypeVar('_C', bound=type) + +_ValidatorType = Callable[[Any, Any, _T], Any] +_ConverterType = Callable[[Any], _T] +_FilterType = Callable[[Any, Any], bool] +_ValidatorArgType = Union[_ValidatorType[_T], Sequence[_ValidatorType[_T]]] + +# This form catches explicit None or no default but with no other arguments returns Any. +@overload +def attrib(default: None = ..., + validator: None = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + convert: None = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: None = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + ) -> Any: ... +# This form catches an explicit None or no default and infers the type from the other arguments. +@overload +def attrib(default: None = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + convert: Optional[_ConverterType[_T]] = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: Optional[Type[_T]] = ..., + converter: Optional[_ConverterType[_T]] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + ) -> _T: ... +# This form catches an explicit default argument. +@overload +def attrib(default: _T, + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + convert: Optional[_ConverterType[_T]] = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: Optional[Type[_T]] = ..., + converter: Optional[_ConverterType[_T]] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + ) -> _T: ... +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def attrib(default: Optional[_T] = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + convert: Optional[_ConverterType[_T]] = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: object = ..., + converter: Optional[_ConverterType[_T]] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + ) -> Any: ... + +@overload +def attrs(maybe_cls: _C, + these: Optional[Mapping[str, Any]] = ..., + repr_ns: Optional[str] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + match_args: bool = ..., + ) -> _C: ... +@overload +def attrs(maybe_cls: None = ..., + these: Optional[Mapping[str, Any]] = ..., + repr_ns: Optional[str] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + match_args: bool = ..., + ) -> Callable[[_C], _C]: ... + + +class Attribute(Generic[_T]): pass + +# aliases +s = attributes = attrs +ib = attr = attrib +dataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;) + +# Next Generation API +@overload +def define( + maybe_cls: _C, + *, + these: Optional[Mapping[str, Any]] = ..., + repr: bool = ..., + unsafe_hash: Optional[bool]=None, + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[object] = ..., +) -> _C: ... +@overload +def define( + maybe_cls: None = ..., + *, + these: Optional[Mapping[str, Any]] = ..., + repr: bool = ..., + unsafe_hash: Optional[bool]=None, + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[object] = ..., +) -> Callable[[_C], _C]: ... + +mutable = define +frozen = define # they differ only in their defaults + +@overload +def field( + *, + default: None = ..., + validator: None = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def field( + *, + default: None = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[object] = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def field( + *, + default: _T, + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[object] = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def field( + *, + default: Optional[_T] = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[object] = ..., +) -> Any: ... + +def evolve(inst: _T, **changes: Any) -> _T: ... +def assoc(inst: _T, **changes: Any) -> _T: ... + +def fields(cls: type) -> Any: ... diff --git a/test-data/unit/lib-stub/attr/converters.pyi b/test-data/unit/lib-stub/attr/converters.pyi new file mode 100644 index 000000000000..63b2a3866e31 --- /dev/null +++ b/test-data/unit/lib-stub/attr/converters.pyi @@ -0,0 +1,12 @@ +from typing import TypeVar, Optional, Callable, overload +from . import _ConverterType + +_T = TypeVar("_T") + +def optional( + converter: _ConverterType[_T] +) -> _ConverterType[Optional[_T]]: ... +@overload +def default_if_none(default: _T) -> _ConverterType[_T]: ... +@overload +def default_if_none(*, factory: Callable[[], _T]) -> _ConverterType[_T]: ... diff --git a/test-data/unit/lib-stub/attrs/__init__.pyi b/test-data/unit/lib-stub/attrs/__init__.pyi new file mode 100644 index 000000000000..d0a65c84d9d8 --- /dev/null +++ b/test-data/unit/lib-stub/attrs/__init__.pyi @@ -0,0 +1,148 @@ +from typing import TypeVar, overload, Callable, Any, Optional, Union, Sequence, Mapping, \ + Protocol, ClassVar, Type +from typing_extensions import TypeGuard + +from attr import Attribute as Attribute + + +class AttrsInstance(Protocol): + __attrs_attrs__: ClassVar[Any] + + +_T = TypeVar('_T') +_C = TypeVar('_C', bound=type) + +_ValidatorType = Callable[[Any, Any, _T], Any] +_ConverterType = Callable[[Any], _T] +_ValidatorArgType = Union[_ValidatorType[_T], Sequence[_ValidatorType[_T]]] + +@overload +def define( + maybe_cls: _C, + *, + these: Optional[Mapping[str, Any]] = ..., + repr: bool = ..., + unsafe_hash: Optional[bool]=None, + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[object] = ..., +) -> _C: ... +@overload +def define( + maybe_cls: None = ..., + *, + these: Optional[Mapping[str, Any]] = ..., + repr: bool = ..., + unsafe_hash: Optional[bool]=None, + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[object] = ..., +) -> Callable[[_C], _C]: ... + +mutable = define +frozen = define # they differ only in their defaults + +@overload +def field( + *, + default: None = ..., + validator: None = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., + alias: Optional[str] = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def field( + *, + default: None = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[object] = ..., + alias: Optional[str] = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def field( + *, + default: _T, + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[object] = ..., + alias: Optional[str] = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def field( + *, + default: Optional[_T] = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: object = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[object] = ..., + alias: Optional[str] = ..., +) -> Any: ... + +def evolve(inst: _T, **changes: Any) -> _T: ... +def assoc(inst: _T, **changes: Any) -> _T: ... +def has(cls: type) -> TypeGuard[Type[AttrsInstance]]: ... +def fields(cls: Type[AttrsInstance]) -> Any: ... diff --git a/test-data/unit/lib-stub/attrs/converters.pyi b/test-data/unit/lib-stub/attrs/converters.pyi new file mode 100644 index 000000000000..33800490894d --- /dev/null +++ b/test-data/unit/lib-stub/attrs/converters.pyi @@ -0,0 +1,12 @@ +from typing import TypeVar, Optional, Callable, overload +from attr import _ConverterType + +_T = TypeVar("_T") + +def optional( + converter: _ConverterType[_T] +) -> _ConverterType[Optional[_T]]: ... +@overload +def default_if_none(default: _T) -> _ConverterType[_T]: ... +@overload +def default_if_none(*, factory: Callable[[], _T]) -> _ConverterType[_T]: ... diff --git a/test-data/unit/lib-stub/blocker.pyi b/test-data/unit/lib-stub/blocker.pyi new file mode 100644 index 000000000000..01bd734353a7 --- /dev/null +++ b/test-data/unit/lib-stub/blocker.pyi @@ -0,0 +1,2 @@ +# Stub file that generates a blocking parse error +x y diff --git a/test-data/unit/lib-stub/blocker2.pyi b/test-data/unit/lib-stub/blocker2.pyi new file mode 100644 index 000000000000..dfacc34c2d47 --- /dev/null +++ b/test-data/unit/lib-stub/blocker2.pyi @@ -0,0 +1,2 @@ +# Stub file that generates a blocking semantic analysis error +continue diff --git a/test-data/unit/lib-stub/broken.pyi b/test-data/unit/lib-stub/broken.pyi new file mode 100644 index 000000000000..22cfc72ed744 --- /dev/null +++ b/test-data/unit/lib-stub/broken.pyi @@ -0,0 +1,2 @@ +# Stub file that generates an error +x = y diff --git a/test-data/unit/lib-stub/builtins.pyi b/test-data/unit/lib-stub/builtins.pyi new file mode 100644 index 000000000000..17d519cc8eea --- /dev/null +++ b/test-data/unit/lib-stub/builtins.pyi @@ -0,0 +1,35 @@ +# DO NOT ADD TO THIS FILE AS IT WILL SLOW DOWN TESTS! +# +# Use [builtins fixtures/...pyi] if you need more features. + +import _typeshed + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x: object) -> None: pass + +# These are provided here for convenience. +class int: + def __add__(self, other: int) -> int: pass +class bool(int): pass +class float: pass + +class str: pass +class bytes: pass + +class function: + __name__: str +class ellipsis: pass + +from typing import Generic, Iterator, Sequence, TypeVar +_T = TypeVar('_T') +class list(Generic[_T], Sequence[_T]): + def __contains__(self, item: object) -> bool: pass + def __getitem__(self, key: int) -> _T: pass + def __iter__(self) -> Iterator[_T]: pass + +class dict: pass + +# Definition of None is implicit diff --git a/test-data/unit/lib-stub/collections.pyi b/test-data/unit/lib-stub/collections.pyi new file mode 100644 index 000000000000..7ea264f764ee --- /dev/null +++ b/test-data/unit/lib-stub/collections.pyi @@ -0,0 +1,25 @@ +from typing import Any, Iterable, Union, Dict, TypeVar, Optional, Callable, Generic, Sequence, MutableMapping + +def namedtuple( + typename: str, + field_names: Union[str, Iterable[str]], + *, + # really bool but many tests don't have bool available + rename: int = ..., + module: Optional[str] = ..., + defaults: Optional[Iterable[Any]] = ... +) -> Any: ... + +KT = TypeVar('KT') +VT = TypeVar('VT') + +class OrderedDict(Dict[KT, VT]): ... + +class defaultdict(Dict[KT, VT]): + def __init__(self, default_factory: Optional[Callable[[], VT]]) -> None: ... + +class Counter(Dict[KT, int], Generic[KT]): ... + +class deque(Sequence[KT], Generic[KT]): ... + +class ChainMap(MutableMapping[KT, VT], Generic[KT, VT]): ... diff --git a/test-data/unit/lib-stub/contextlib.pyi b/test-data/unit/lib-stub/contextlib.pyi new file mode 100644 index 000000000000..ca9e91cf4d65 --- /dev/null +++ b/test-data/unit/lib-stub/contextlib.pyi @@ -0,0 +1,13 @@ +from typing import AsyncIterator, Generic, TypeVar, Callable, Iterator +from typing import ContextManager as ContextManager, AsyncContextManager as AsyncContextManager + +_T = TypeVar('_T') + +class GeneratorContextManager(ContextManager[_T], Generic[_T]): + def __call__(self, func: Callable[..., _T]) -> Callable[..., _T]: ... + +# This does not match `typeshed` definition, needs `ParamSpec`: +def contextmanager(func: Callable[..., Iterator[_T]]) -> Callable[..., GeneratorContextManager[_T]]: + ... + +def asynccontextmanager(func: Callable[..., AsyncIterator[_T]]) -> Callable[..., AsyncContextManager[_T]]: ... diff --git a/test-data/unit/lib-stub/dataclasses.pyi b/test-data/unit/lib-stub/dataclasses.pyi new file mode 100644 index 000000000000..cf43747757bd --- /dev/null +++ b/test-data/unit/lib-stub/dataclasses.pyi @@ -0,0 +1,52 @@ +from typing import Any, Callable, Generic, Literal, Mapping, Optional, TypeVar, overload, Type, \ + Protocol, ClassVar +from typing_extensions import TypeGuard + +# DataclassInstance is in _typeshed.pyi normally, but alas we can't do the same for lib-stub +# due to test-data/unit/lib-stub/builtins.pyi not having 'tuple'. +class DataclassInstance(Protocol): + __dataclass_fields__: ClassVar[dict[str, Field[Any]]] + +_T = TypeVar('_T') +_DataclassT = TypeVar("_DataclassT", bound=DataclassInstance) + +class InitVar(Generic[_T]): + ... + +class KW_ONLY: ... + +@overload +def dataclass(_cls: Type[_T]) -> Type[_T]: ... + +@overload +def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., + unsafe_hash: bool = ..., frozen: bool = ..., match_args: bool = ..., + kw_only: bool = ..., slots: bool = ...) -> Callable[[Type[_T]], Type[_T]]: ... + +@overload +def field(*, default: _T, + init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., + metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...,) -> _T: ... + +@overload +def field(*, default_factory: Callable[[], _T], + init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., + metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...,) -> _T: ... + +@overload +def field(*, + init: bool = ..., repr: bool = ..., hash: Optional[bool] = ..., compare: bool = ..., + metadata: Optional[Mapping[str, Any]] = ..., kw_only: bool = ...,) -> Any: ... + + +class Field(Generic[_T]): pass + +@overload +def is_dataclass(obj: DataclassInstance) -> Literal[True]: ... +@overload +def is_dataclass(obj: type) -> TypeGuard[type[DataclassInstance]]: ... +@overload +def is_dataclass(obj: object) -> TypeGuard[DataclassInstance | type[DataclassInstance]]: ... + + +def replace(__obj: _DataclassT, **changes: Any) -> _DataclassT: ... diff --git a/test-data/unit/lib-stub/datetime.pyi b/test-data/unit/lib-stub/datetime.pyi new file mode 100644 index 000000000000..7d71682d051d --- /dev/null +++ b/test-data/unit/lib-stub/datetime.pyi @@ -0,0 +1,16 @@ +# Very simplified datetime stubs for use in tests + +class datetime: + def __new__( + cls, + year: int, + month: int, + day: int, + hour: int = ..., + minute: int = ..., + second: int = ..., + microsecond: int = ..., + *, + fold: int = ..., + ) -> datetime: ... + def __format__(self, __fmt: str) -> str: ... diff --git a/test-data/unit/lib-stub/decimal.pyi b/test-data/unit/lib-stub/decimal.pyi new file mode 100644 index 000000000000..d2ab6eda9ff1 --- /dev/null +++ b/test-data/unit/lib-stub/decimal.pyi @@ -0,0 +1,3 @@ +# Very simplified decimal stubs for use in tests + +from _decimal import * diff --git a/test-data/unit/lib-stub/enum.pyi b/test-data/unit/lib-stub/enum.pyi new file mode 100644 index 000000000000..ccb3818b9d25 --- /dev/null +++ b/test-data/unit/lib-stub/enum.pyi @@ -0,0 +1,62 @@ +from typing import Any, TypeVar, Union, Type, Sized, Iterator +from typing_extensions import Literal + +_T = TypeVar('_T') + +class EnumMeta(type, Sized): + def __len__(self) -> int: pass # to make it non-abstract + def __iter__(self: Type[_T]) -> Iterator[_T]: pass + def __reversed__(self: Type[_T]) -> Iterator[_T]: pass + def __getitem__(self: Type[_T], name: str) -> _T: pass + def __bool__(self) -> Literal[True]: pass + +class Enum(metaclass=EnumMeta): + def __new__(cls: Type[_T], value: object) -> _T: pass + def __repr__(self) -> str: pass + def __str__(self) -> str: pass + def __format__(self, format_spec: str) -> str: pass + def __hash__(self) -> Any: pass + def __reduce_ex__(self, proto: Any) -> Any: pass + + name: str + value: Any + _name_: str + _value_: Any + + # In reality, _generate_next_value_ is python3.6 only and has a different signature. + # However, this should be quick and doesn't require additional stubs (e.g. `staticmethod`) + def _generate_next_value_(self) -> Any: pass + +class IntEnum(int, Enum): + value: int + def __new__(cls: Type[_T], value: Union[int, _T]) -> _T: ... + +def unique(enumeration: _T) -> _T: pass + +# In reality Flag and IntFlag are 3.6 only + +class Flag(Enum): + def __or__(self: _T, other: Union[int, _T]) -> _T: pass + + +class IntFlag(int, Flag): + def __and__(self: _T, other: Union[int, _T]) -> _T: pass + + +class auto(IntFlag): + value: Any + + +# It is python-3.11+ only: +class StrEnum(str, Enum): + def __new__(cls: Type[_T], value: str | _T) -> _T: ... + +# It is python-3.11+ only: +class nonmember(Generic[_T]): + value: _T + def __init__(self, value: _T) -> None: ... + +# It is python-3.11+ only: +class member(Generic[_T]): + value: _T + def __init__(self, value: _T) -> None: ... diff --git a/test-data/unit/lib-stub/functools.pyi b/test-data/unit/lib-stub/functools.pyi new file mode 100644 index 000000000000..b8d47e1da2b5 --- /dev/null +++ b/test-data/unit/lib-stub/functools.pyi @@ -0,0 +1,39 @@ +from typing import Generic, TypeVar, Callable, Any, Mapping, Self, overload + +_T = TypeVar("_T") + +class _SingleDispatchCallable(Generic[_T]): + registry: Mapping[Any, Callable[..., _T]] + def dispatch(self, cls: Any) -> Callable[..., _T]: ... + # @fun.register(complex) + # def _(arg, verbose=False): ... + @overload + def register(self, cls: type[Any], func: None = ...) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... + # @fun.register + # def _(arg: int, verbose=False): + @overload + def register(self, cls: Callable[..., _T], func: None = ...) -> Callable[..., _T]: ... + # fun.register(int, lambda x: x) + @overload + def register(self, cls: type[Any], func: Callable[..., _T]) -> Callable[..., _T]: ... + def _clear_cache(self) -> None: ... + def __call__(__self, *args: Any, **kwargs: Any) -> _T: ... + +def singledispatch(func: Callable[..., _T]) -> _SingleDispatchCallable[_T]: ... + +def total_ordering(cls: type[_T]) -> type[_T]: ... + +class cached_property(Generic[_T]): + func: Callable[[Any], _T] + attrname: str | None + def __init__(self, func: Callable[[Any], _T]) -> None: ... + @overload + def __get__(self, instance: None, owner: type[Any] | None = ...) -> cached_property[_T]: ... + @overload + def __get__(self, instance: object, owner: type[Any] | None = ...) -> _T: ... + def __set_name__(self, owner: type[Any], name: str) -> None: ... + def __class_getitem__(cls, item: Any) -> Any: ... + +class partial(Generic[_T]): + def __new__(cls, __func: Callable[..., _T], *args: Any, **kwargs: Any) -> Self: ... + def __call__(__self, *args: Any, **kwargs: Any) -> _T: ... diff --git a/test-data/unit/lib-stub/future/__init__.pyi b/test-data/unit/lib-stub/future/__init__.pyi new file mode 100644 index 000000000000..b0860a4bae9c --- /dev/null +++ b/test-data/unit/lib-stub/future/__init__.pyi @@ -0,0 +1 @@ +from __future__ import absolute_import, print_function diff --git a/test-data/unit/lib-stub/future/utils.pyi b/test-data/unit/lib-stub/future/utils.pyi new file mode 100644 index 000000000000..1f1e4ec7e597 --- /dev/null +++ b/test-data/unit/lib-stub/future/utils.pyi @@ -0,0 +1,2 @@ +from typing import Type +def with_metaclass(meta: Type[type], *bases: type) -> type: pass diff --git a/test-data/unit/lib-stub/math.pyi b/test-data/unit/lib-stub/math.pyi new file mode 100644 index 000000000000..06f8878a563e --- /dev/null +++ b/test-data/unit/lib-stub/math.pyi @@ -0,0 +1,20 @@ +pi: float +e: float +tau: float +inf: float +nan: float +def sqrt(__x: float) -> float: ... +def sin(__x: float) -> float: ... +def cos(__x: float) -> float: ... +def tan(__x: float) -> float: ... +def exp(__x: float) -> float: ... +def log(__x: float) -> float: ... +def floor(__x: float) -> int: ... +def ceil(__x: float) -> int: ... +def fabs(__x: float) -> float: ... +def pow(__x: float, __y: float) -> float: ... +def copysign(__x: float, __y: float) -> float: ... +def isinf(__x: float) -> bool: ... +def isnan(__x: float) -> bool: ... +def isfinite(__x: float) -> bool: ... +def nextafter(__x: float, __y: float) -> float: ... diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi new file mode 100644 index 000000000000..4295c33f81ad --- /dev/null +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -0,0 +1,173 @@ +# NOTE: Requires fixtures/dict.pyi +from typing import ( + Any, Dict, Type, TypeVar, Optional, Any, Generic, Mapping, NoReturn as NoReturn, Iterator, + Union, Protocol +) + +_T = TypeVar('_T') +_U = TypeVar('_U') + + +def Arg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def DefaultArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def NamedArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def DefaultNamedArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def VarArg(type: _T = ...) -> _T: ... + +def KwArg(type: _T = ...) -> _T: ... + + +# Fallback type for all typed dicts (does not exist at runtime). +class _TypedDict(Mapping[str, object]): + # Needed to make this class non-abstract. It is explicitly declared abstract in + # typeshed, but we don't want to import abc here, as it would slow down the tests. + def __iter__(self) -> Iterator[str]: ... + def copy(self: _T) -> _T: ... + # Using NoReturn so that only calls using the plugin hook can go through. + def setdefault(self, k: NoReturn, default: object) -> object: ... + # Mypy expects that 'default' has a type variable type. + def pop(self, k: NoReturn, default: _T = ...) -> object: ... + def update(self: _T, __m: _T) -> None: ... + def __delitem__(self, k: NoReturn) -> None: ... + +def TypedDict(typename: str, fields: Dict[str, Type[_T]], *, total: Any = ...) -> Type[dict]: ... + +# This is intended as a class decorator, but mypy rejects abstract classes +# when a Type[_T] is expected, so we can't give it the type we want. +def trait(cls: Any) -> Any: ... + +# The real type is in the comment but it isn't safe to use **kwargs in +# a lib-stub because the fixtures might not have dict. Argh! +# def mypyc_attr(*attrs: str, **kwattrs: object) -> Callable[[_T], _T]: ... +mypyc_attr: Any + +class FlexibleAlias(Generic[_T, _U]): ... + +class __SupportsInt(Protocol[T_co]): + def __int__(self) -> int: pass + +_Int = Union[int, u8, i16, i32, i64] + +class u8: + def __init__(self, x: Union[_Int, str, bytes, SupportsInt], base: int = 10) -> None: ... + def __add__(self, x: u8) -> u8: ... + def __radd__(self, x: u8) -> u8: ... + def __sub__(self, x: u8) -> u8: ... + def __rsub__(self, x: u8) -> u8: ... + def __mul__(self, x: u8) -> u8: ... + def __rmul__(self, x: u8) -> u8: ... + def __floordiv__(self, x: u8) -> u8: ... + def __rfloordiv__(self, x: u8) -> u8: ... + def __mod__(self, x: u8) -> u8: ... + def __rmod__(self, x: u8) -> u8: ... + def __and__(self, x: u8) -> u8: ... + def __rand__(self, x: u8) -> u8: ... + def __or__(self, x: u8) -> u8: ... + def __ror__(self, x: u8) -> u8: ... + def __xor__(self, x: u8) -> u8: ... + def __rxor__(self, x: u8) -> u8: ... + def __lshift__(self, x: u8) -> u8: ... + def __rlshift__(self, x: u8) -> u8: ... + def __rshift__(self, x: u8) -> u8: ... + def __rrshift__(self, x: u8) -> u8: ... + def __neg__(self) -> u8: ... + def __invert__(self) -> u8: ... + def __pos__(self) -> u8: ... + def __lt__(self, x: u8) -> bool: ... + def __le__(self, x: u8) -> bool: ... + def __ge__(self, x: u8) -> bool: ... + def __gt__(self, x: u8) -> bool: ... + +class i16: + def __init__(self, x: Union[_Int, str, bytes, SupportsInt], base: int = 10) -> None: ... + def __add__(self, x: i16) -> i16: ... + def __radd__(self, x: i16) -> i16: ... + def __sub__(self, x: i16) -> i16: ... + def __rsub__(self, x: i16) -> i16: ... + def __mul__(self, x: i16) -> i16: ... + def __rmul__(self, x: i16) -> i16: ... + def __floordiv__(self, x: i16) -> i16: ... + def __rfloordiv__(self, x: i16) -> i16: ... + def __mod__(self, x: i16) -> i16: ... + def __rmod__(self, x: i16) -> i16: ... + def __and__(self, x: i16) -> i16: ... + def __rand__(self, x: i16) -> i16: ... + def __or__(self, x: i16) -> i16: ... + def __ror__(self, x: i16) -> i16: ... + def __xor__(self, x: i16) -> i16: ... + def __rxor__(self, x: i16) -> i16: ... + def __lshift__(self, x: i16) -> i16: ... + def __rlshift__(self, x: i16) -> i16: ... + def __rshift__(self, x: i16) -> i16: ... + def __rrshift__(self, x: i16) -> i16: ... + def __neg__(self) -> i16: ... + def __invert__(self) -> i16: ... + def __pos__(self) -> i16: ... + def __lt__(self, x: i16) -> bool: ... + def __le__(self, x: i16) -> bool: ... + def __ge__(self, x: i16) -> bool: ... + def __gt__(self, x: i16) -> bool: ... + +class i32: + def __init__(self, x: Union[_Int, str, bytes, SupportsInt], base: int = 10) -> None: ... + def __add__(self, x: i32) -> i32: ... + def __radd__(self, x: i32) -> i32: ... + def __sub__(self, x: i32) -> i32: ... + def __rsub__(self, x: i32) -> i32: ... + def __mul__(self, x: i32) -> i32: ... + def __rmul__(self, x: i32) -> i32: ... + def __floordiv__(self, x: i32) -> i32: ... + def __rfloordiv__(self, x: i32) -> i32: ... + def __mod__(self, x: i32) -> i32: ... + def __rmod__(self, x: i32) -> i32: ... + def __and__(self, x: i32) -> i32: ... + def __rand__(self, x: i32) -> i32: ... + def __or__(self, x: i32) -> i32: ... + def __ror__(self, x: i32) -> i32: ... + def __xor__(self, x: i32) -> i32: ... + def __rxor__(self, x: i32) -> i32: ... + def __lshift__(self, x: i32) -> i32: ... + def __rlshift__(self, x: i32) -> i32: ... + def __rshift__(self, x: i32) -> i32: ... + def __rrshift__(self, x: i32) -> i32: ... + def __neg__(self) -> i32: ... + def __invert__(self) -> i32: ... + def __pos__(self) -> i32: ... + def __lt__(self, x: i32) -> bool: ... + def __le__(self, x: i32) -> bool: ... + def __ge__(self, x: i32) -> bool: ... + def __gt__(self, x: i32) -> bool: ... + +class i64: + def __init__(self, x: Union[_Int, str, bytes, SupportsInt], base: int = 10) -> None: ... + def __add__(self, x: i64) -> i64: ... + def __radd__(self, x: i64) -> i64: ... + def __sub__(self, x: i64) -> i64: ... + def __rsub__(self, x: i64) -> i64: ... + def __mul__(self, x: i64) -> i64: ... + def __rmul__(self, x: i64) -> i64: ... + def __floordiv__(self, x: i64) -> i64: ... + def __rfloordiv__(self, x: i64) -> i64: ... + def __mod__(self, x: i64) -> i64: ... + def __rmod__(self, x: i64) -> i64: ... + def __and__(self, x: i64) -> i64: ... + def __rand__(self, x: i64) -> i64: ... + def __or__(self, x: i64) -> i64: ... + def __ror__(self, x: i64) -> i64: ... + def __xor__(self, x: i64) -> i64: ... + def __rxor__(self, x: i64) -> i64: ... + def __lshift__(self, x: i64) -> i64: ... + def __rlshift__(self, x: i64) -> i64: ... + def __rshift__(self, x: i64) -> i64: ... + def __rrshift__(self, x: i64) -> i64: ... + def __neg__(self) -> i64: ... + def __invert__(self) -> i64: ... + def __pos__(self) -> i64: ... + def __lt__(self, x: i64) -> bool: ... + def __le__(self, x: i64) -> bool: ... + def __ge__(self, x: i64) -> bool: ... + def __gt__(self, x: i64) -> bool: ... diff --git a/test-data/unit/lib-stub/numbers.pyi b/test-data/unit/lib-stub/numbers.pyi new file mode 100644 index 000000000000..fad173c9a8b6 --- /dev/null +++ b/test-data/unit/lib-stub/numbers.pyi @@ -0,0 +1,10 @@ +# Test fixture for numbers +# +# The numbers module isn't properly supported, but we want to test that mypy +# can tell that it doesn't work as expected. + +class Number: pass +class Complex: pass +class Real: pass +class Rational: pass +class Integral: pass diff --git a/test-data/unit/lib-stub/six.pyi b/test-data/unit/lib-stub/six.pyi new file mode 100644 index 000000000000..97dbeabf861a --- /dev/null +++ b/test-data/unit/lib-stub/six.pyi @@ -0,0 +1,3 @@ +from typing import Type, Callable +def with_metaclass(mcls: Type[type], *args: type) -> type: pass +def add_metaclass(mcls: Type[type]) -> Callable[[type], type]: pass diff --git a/test-data/unit/lib-stub/sys.pyi b/test-data/unit/lib-stub/sys.pyi new file mode 100644 index 000000000000..3959cb035ca1 --- /dev/null +++ b/test-data/unit/lib-stub/sys.pyi @@ -0,0 +1,2 @@ +version_info = (0, 0, 0, '', 0) +platform = '' diff --git a/test-data/unit/lib-stub/traceback.pyi b/test-data/unit/lib-stub/traceback.pyi new file mode 100644 index 000000000000..83c1891f80f5 --- /dev/null +++ b/test-data/unit/lib-stub/traceback.pyi @@ -0,0 +1,3 @@ +# Very simplified traceback stubs for use in tests + +def print_tb(*args, **kwargs) -> None: ... diff --git a/test-data/unit/lib-stub/types.pyi b/test-data/unit/lib-stub/types.pyi new file mode 100644 index 000000000000..3f713c31e417 --- /dev/null +++ b/test-data/unit/lib-stub/types.pyi @@ -0,0 +1,21 @@ +from typing import Any, TypeVar +import sys + +_T = TypeVar('_T') + +def coroutine(func: _T) -> _T: pass + +class ModuleType: + __file__: str + def __getattr__(self, name: str) -> Any: pass + +class GenericAlias: + def __or__(self, o): ... + def __ror__(self, o): ... + +if sys.version_info >= (3, 10): + class NoneType: + ... + + class UnionType: + def __or__(self, x) -> UnionType: ... diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi new file mode 100644 index 000000000000..3cb164140883 --- /dev/null +++ b/test-data/unit/lib-stub/typing.pyi @@ -0,0 +1,67 @@ +# Stub for typing module. Many of the definitions have special handling in +# the type checker, so they can just be initialized to anything. +# +# DO NOT ADD TO THIS FILE UNLESS YOU HAVE A GOOD REASON! Additional definitions +# will slow down tests. +# +# Use [typing fixtures/typing-{medium,full,async,...}.pyi] in a test case for +# a more complete stub for typing. If you need to add things, add to one of +# the stubs under fixtures/. + +cast = 0 +assert_type = 0 +overload = 0 +Any = object() +Union = 0 +Optional = 0 +TypeVar = 0 +Generic = 0 +Protocol = 0 +Tuple = 0 +Callable = 0 +NamedTuple = 0 +Type = 0 +ClassVar = 0 +Final = 0 +NoReturn = 0 +Never = 0 +NewType = 0 +ParamSpec = 0 +TypeVarTuple = 0 +Unpack = 0 +Self = 0 +TYPE_CHECKING = 0 + +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +U = TypeVar('U') +V = TypeVar('V') + +class Iterable(Protocol[T_co]): + def __iter__(self) -> Iterator[T_co]: pass + +class Iterator(Iterable[T_co], Protocol): + def __next__(self) -> T_co: pass + +class Generator(Iterator[T], Generic[T, U, V]): + def __iter__(self) -> Generator[T, U, V]: pass + +class Sequence(Iterable[T_co]): + def __getitem__(self, n: Any) -> T_co: pass + def __len__(self) -> int: pass + +# Mapping type is oversimplified intentionally. +class Mapping(Iterable[T], Generic[T, T_co]): + def keys(self) -> Iterable[T]: pass # Approximate return type + def __getitem__(self, key: T) -> T_co: pass + +class Awaitable(Protocol[T]): + def __await__(self) -> Generator[Any, Any, T]: pass + +class Coroutine(Awaitable[V], Generic[T, U, V]): pass + +def final(meth: T) -> T: pass + +def reveal_type(__obj: T) -> T: pass + +class _SpecialForm: pass diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi new file mode 100644 index 000000000000..cb054b0e6b4f --- /dev/null +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -0,0 +1,97 @@ +import typing +from typing import Any, Callable, Mapping, Iterable, Iterator, NoReturn as NoReturn, Dict, Tuple, Type, Union +from typing import TYPE_CHECKING as TYPE_CHECKING +from typing import NewType as NewType, overload as overload + +import sys + +_T = typing.TypeVar('_T') + +class _SpecialForm: + def __getitem__(self, typeargs: Any) -> Any: + pass + + def __call__(self, arg: Any) -> Any: + pass + +NamedTuple = 0 +Protocol: _SpecialForm = ... +def runtime_checkable(x: _T) -> _T: pass +runtime = runtime_checkable + +Final: _SpecialForm = ... +def final(x: _T) -> _T: pass + +Literal: _SpecialForm = ... + +Annotated: _SpecialForm = ... + +TypeVar: _SpecialForm + +ParamSpec: _SpecialForm +Concatenate: _SpecialForm + +TypeAlias: _SpecialForm + +TypeGuard: _SpecialForm +TypeIs: _SpecialForm +Never: _SpecialForm + +TypeVarTuple: _SpecialForm +Unpack: _SpecialForm +Required: _SpecialForm +NotRequired: _SpecialForm +ReadOnly: _SpecialForm + +Self: _SpecialForm + +@final +class TypeAliasType: + def __init__( + self, name: str, value: Any, *, type_params: Tuple[Union[TypeVar, ParamSpec, TypeVarTuple], ...] = () + ) -> None: ... + +# Fallback type for all typed dicts (does not exist at runtime). +class _TypedDict(Mapping[str, object]): + # Needed to make this class non-abstract. It is explicitly declared abstract in + # typeshed, but we don't want to import abc here, as it would slow down the tests. + def __iter__(self) -> Iterator[str]: ... + def copy(self: _T) -> _T: ... + # Using NoReturn so that only calls using the plugin hook can go through. + def setdefault(self, k: NoReturn, default: object) -> object: ... + # Mypy expects that 'default' has a type variable type. + def pop(self, k: NoReturn, default: _T = ...) -> object: ... + def update(self: _T, __m: _T) -> None: ... + def items(self) -> Iterable[Tuple[str, object]]: ... + def keys(self) -> Iterable[str]: ... + def values(self) -> Iterable[object]: ... + if sys.version_info < (3, 0): + def has_key(self, k: str) -> bool: ... + def __delitem__(self, k: NoReturn) -> None: ... + # Stubtest's tests need the following items: + __required_keys__: frozenset[str] + __optional_keys__: frozenset[str] + __readonly_keys__: frozenset[str] + __mutable_keys__: frozenset[str] + __closed__: bool + __extra_items__: Any + __total__: bool + +def TypedDict(typename: str, fields: Dict[str, Type[_T]], *, total: Any = ...) -> Type[dict]: ... + +def reveal_type(__obj: _T) -> _T: pass +def assert_type(__val: _T, __typ: Any) -> _T: pass + +def dataclass_transform( + *, + eq_default: bool = ..., + order_default: bool = ..., + kw_only_default: bool = ..., + field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = ..., + **kwargs: Any, +) -> Callable[[_T], _T]: ... + +def override(__arg: _T) -> _T: ... +def deprecated(__msg: str) -> Callable[[_T], _T]: ... + +_FutureFeatureFixture = 0 diff --git a/test-data/unit/lib-stub/unannotated_lib.pyi b/test-data/unit/lib-stub/unannotated_lib.pyi new file mode 100644 index 000000000000..90bfb6fa47d6 --- /dev/null +++ b/test-data/unit/lib-stub/unannotated_lib.pyi @@ -0,0 +1 @@ +def f(x): ... diff --git a/test-data/unit/merge.test b/test-data/unit/merge.test new file mode 100644 index 000000000000..a6a64c75b2a3 --- /dev/null +++ b/test-data/unit/merge.test @@ -0,0 +1,1558 @@ +-- Test cases for AST merge (used for fine-grained incremental checking) +-- +-- Each test case has two versions of the module 'target' (target.py and +-- target.py.next). A test cases type checks both of them, merges the ASTs, +-- and finally dumps certain parts of the ASTs for both versions (==> +-- separates the first and second versions). A test case passes if the +-- dumped output is as expected. +-- +-- The dumped output uses to denote identities of objects. Objects +-- suffixed by the same refer to the same object; and (if +-- N != M) refer to different objects. The objective of these test cases +-- is to verify that identities of publicly visible AST nodes is +-- preserved across merge. Other AST nodes may get new identities. +-- +-- Each test case dumps one of four kinds of information: +-- +-- 1) ASTs (test case has no magic suffix) +-- 2) Symbol tables (_symtable test case name suffix) +-- 3) TypeInfos (_typeinfo suffix) +-- 4) Inferred types (_types suffix) +-- +-- If you need to dump multiple different kinds of information, write +-- multiple test cases. + +[case testFunction] +import target +[file target.py] +def f() -> int: + pass +[file target.py.next] +def f() -> int: + pass +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + FuncDef:1<2>( + f + def () -> builtins.int<3> + Block:2<4>( + PassStmt:2<5>()))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + FuncDef:1<2>( + f + def () -> builtins.int<3> + Block:2<6>( + PassStmt:2<7>()))) + +[case testClass] +import target +[file target.py] +class A: + def f(self, x: str) -> int: + pass +[file target.py.next] +class A: + def f(self, x: int) -> str: + pass +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<2>( + A + FuncDef:2<3>( + f + Args( + Var(self) + Var(x)) + def (self: target.A<4>, x: builtins.str<5>) -> builtins.int<6> + Block:3<7>( + PassStmt:3<8>())))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<9>( + A + FuncDef:2<3>( + f + Args( + Var(self) + Var(x)) + def (self: target.A<4>, x: builtins.int<6>) -> builtins.str<5> + Block:3<10>( + PassStmt:3<11>())))) + +[case testClass_typeinfo] +import target +[file target.py] +class A: + def f(self, x: str) -> int: pass + def g(self, x: str) -> int: pass +[file target.py.next] +class A: + def f(self, x: int) -> str: pass + def h(self, x: int) -> str: pass +[out] +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names( + f<2> + g<3>)) +==> +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names( + f<2> + h<4>)) + +[case testConstructInstance] +import target +[file target.py] +class A: + def f(self) -> B: + return B() +class B: pass +[file target.py.next] +class B: pass +class A: + def f(self) -> B: + 1 + return B() +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<2>( + A + FuncDef:2<3>( + f + Args( + Var(self)) + def (self: target.A<4>) -> target.B<5> + Block:3<6>( + ReturnStmt:3<7>( + CallExpr:3<8>( + NameExpr(B [target.B<5>]) + Args()))))) + ClassDef:4<9>( + B + PassStmt:4<10>())) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<11>( + B + PassStmt:1<12>()) + ClassDef:2<13>( + A + FuncDef:3<3>( + f + Args( + Var(self)) + def (self: target.A<4>) -> target.B<5> + Block:4<14>( + ExpressionStmt:4<15>( + IntExpr(1)) + ReturnStmt:5<16>( + CallExpr:5<17>( + NameExpr(B [target.B<5>]) + Args())))))) + +[case testCallMethod] +import target +[file target.py] +class A: + def f(self) -> None: + self.f() +[file target.py.next] +class A: + def f(self) -> None: + self.f() +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<2>( + A + FuncDef:2<3>( + f + Args( + Var(self)) + def (self: target.A<4>) + Block:3<5>( + ExpressionStmt:3<6>( + CallExpr:3<7>( + MemberExpr:3<8>( + NameExpr(self [l<9>]) + f) + Args())))))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<10>( + A + FuncDef:2<3>( + f + Args( + Var(self)) + def (self: target.A<4>) + Block:3<11>( + ExpressionStmt:3<12>( + CallExpr:3<13>( + MemberExpr:3<14>( + NameExpr(self [l<15>]) + f) + Args())))))) + +[case testClassAttribute] +import target +[file target.py] +class A: + def f(self) -> None: + self.x = 1 + self.x +[file target.py.next] +class A: + def f(self) -> None: + self.x = 1 + self.x +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<2>( + A + FuncDef:2<3>( + f + Args( + Var(self)) + def (self: target.A<4>) + Block:3<5>( + AssignmentStmt:3<6>( + MemberExpr:3<8>( + NameExpr(self [l<9>]) + x*<7>) + IntExpr(1)) + ExpressionStmt:4<10>( + MemberExpr:4<11>( + NameExpr(self [l<9>]) + x)))))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<12>( + A + FuncDef:2<3>( + f + Args( + Var(self)) + def (self: target.A<4>) + Block:3<13>( + AssignmentStmt:3<14>( + MemberExpr:3<15>( + NameExpr(self [l<16>]) + x*<7>) + IntExpr(1)) + ExpressionStmt:4<17>( + MemberExpr:4<18>( + NameExpr(self [l<16>]) + x)))))) + +[case testClassAttribute_typeinfo] +import target +[file target.py] +class A: + def f(self) -> None: + self.x = 1 + self.x + self.y = A() +[file target.py.next] +class A: + def f(self) -> None: + self.x = 1 + self.x + self.y = A() +[out] +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names( + f<2> + x<3> (builtins.int<4>) + y<5> (target.A<0>))) +==> +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names( + f<2> + x<3> (builtins.int<4>) + y<5> (target.A<0>))) + +[case testFunction_symtable] +import target +[file target.py] +def f() -> int: + pass +[file target.py.next] +def f() -> int: + pass +[out] +__main__: + target: MypyFile<0> +target: + f: FuncDef<1> +==> +__main__: + target: MypyFile<0> +target: + f: FuncDef<1> + +[case testClass_symtable] +import target +[file target.py] +class A: pass +class B: pass +[file target.py.next] +class A: pass +class C: pass +[out] +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + B: TypeInfo<2> +==> +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + C: TypeInfo<3> + +[case testTopLevelExpression] +import target +[file target.py] +class A: pass +A() +[file target.py.next] +class A: pass +class B: pass +A() +B() +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<2>( + A + PassStmt:1<3>()) + ExpressionStmt:2<4>( + CallExpr:2<5>( + NameExpr(A [target.A<6>]) + Args()))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ClassDef:1<7>( + A + PassStmt:1<8>()) + ClassDef:2<9>( + B + PassStmt:2<10>()) + ExpressionStmt:3<11>( + CallExpr:3<12>( + NameExpr(A [target.A<6>]) + Args())) + ExpressionStmt:4<13>( + CallExpr:4<14>( + NameExpr(B [target.B<15>]) + Args()))) + +[case testExpression_types] +import target +[file target.py] +class A: pass +def f(a: A) -> None: + 1 + a +[file target.py.next] +class A: pass +def f(a: A) -> None: + a + 1 +[out] +## target +IntExpr:3: Literal[1]?<0> +NameExpr:4: target.A<1> +==> +## target +NameExpr:3: target.A<1> +IntExpr:4: Literal[1]?<0> + +[case testClassAttribute_types] +import target +[file target.py] +class A: + def f(self) -> None: + self.x = A() + self.x + self.y = 1 + self.y +[file target.py.next] +class A: + def f(self) -> None: + self.y = 1 + self.y + self.x = A() + self.x +[out] +## target +CallExpr:3: target.A<0> +MemberExpr:3: target.A<0> +NameExpr:3: def () -> target.A<0> +NameExpr:3: target.A<0> +MemberExpr:4: target.A<0> +NameExpr:4: target.A<0> +IntExpr:5: Literal[1]?<1> +MemberExpr:5: builtins.int<1> +NameExpr:5: target.A<0> +MemberExpr:6: builtins.int<1> +NameExpr:6: target.A<0> +==> +## target +IntExpr:3: Literal[1]?<1> +MemberExpr:3: builtins.int<1> +NameExpr:3: target.A<0> +MemberExpr:4: builtins.int<1> +NameExpr:4: target.A<0> +CallExpr:5: target.A<0> +MemberExpr:5: target.A<0> +NameExpr:5: def () -> target.A<0> +NameExpr:5: target.A<0> +MemberExpr:6: target.A<0> +NameExpr:6: target.A<0> + +[case testMethod_types] +import target +[file target.py] +class A: + def f(self) -> A: + return self.f() +[file target.py.next] +class A: + # Extra line to change line numbers + def f(self) -> A: + return self.f() +[out] +## target +CallExpr:3: target.A<0> +MemberExpr:3: def () -> target.A<0> +NameExpr:3: target.A<0> +==> +## target +CallExpr:4: target.A<0> +MemberExpr:4: def () -> target.A<0> +NameExpr:4: target.A<0> + +[case testRenameFunction] +import target +[file target.py] +def f() -> int: pass +[file target.py.next] +def g() -> int: pass +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + FuncDef:1<2>( + f + def () -> builtins.int<3> + Block:1<4>( + PassStmt:1<5>()))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + FuncDef:1<6>( + g + def () -> builtins.int<3> + Block:1<7>( + PassStmt:1<8>()))) + +[case testRenameFunction_symtable] +import target +[file target.py] +def f() -> int: pass +[file target.py.next] +def g() -> int: pass +[out] +__main__: + target: MypyFile<0> +target: + f: FuncDef<1> +==> +__main__: + target: MypyFile<0> +target: + g: FuncDef<2> + +[case testMergeWithBaseClass_typeinfo] +import target +[file target.py] +class A: pass +class B(A): + def f(self) -> None: pass +[file target.py.next] +class C: pass +class A: pass +class B(A): + def f(self) -> None: pass +[out] +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names()) +TypeInfo<2>( + Name(target.B) + Bases(target.A<0>) + Mro(target.B<2>, target.A<0>, builtins.object<1>) + Names( + f<3>)) +==> +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names()) +TypeInfo<2>( + Name(target.B) + Bases(target.A<0>) + Mro(target.B<2>, target.A<0>, builtins.object<1>) + Names( + f<3>)) +TypeInfo<4>( + Name(target.C) + Bases(builtins.object<1>) + Mro(target.C<4>, builtins.object<1>) + Names()) + +[case testModuleAttribute] +import target +[file target.py] +x = 1 +[file target.py.next] +x = 2 +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + AssignmentStmt:1<2>( + NameExpr(x [target.x<3>]) + IntExpr(1) + builtins.int<4>)) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + AssignmentStmt:1<5>( + NameExpr(x [target.x<3>]) + IntExpr(2) + builtins.int<4>)) + +[case testNestedClassMethod_typeinfo] +import target +[file target.py] +class A: + class B: + def f(self) -> None: pass +[file target.py.next] +class A: + class B: + def f(self) -> None: pass +[out] +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names( + B<2>)) +TypeInfo<2>( + Name(target.A.B) + Bases(builtins.object<1>) + Mro(target.A.B<2>, builtins.object<1>) + Names( + f<3>)) +==> +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names( + B<2>)) +TypeInfo<2>( + Name(target.A.B) + Bases(builtins.object<1>) + Mro(target.A.B<2>, builtins.object<1>) + Names( + f<3>)) + +[case testNamedTuple_typeinfo] +# flags: --python-version 3.10 +import target +[file target.py] +from typing import NamedTuple +class A: pass +N = NamedTuple('N', [('x', A)]) +[file target.py.next] +from typing import NamedTuple +class A: pass +N = NamedTuple('N', [('x', A), ('y', A)]) +[builtins fixtures/tuple.pyi] +[out] +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names()) +TypeInfo<2>( + Name(target.N) + Bases(builtins.tuple[target.A<0>, ...]<3>) + Mro(target.N<2>, builtins.tuple<3>, typing.Sequence<4>, typing.Iterable<5>, builtins.object<1>) + Names( + _NT<6> + __annotations__<7> (builtins.dict[builtins.str<8>, Any]<9>) + __doc__<10> (builtins.str<8>) + __match_args__<11> (Tuple[Literal['x']]) + __new__<12> + _asdict<13> + _field_defaults<14> (builtins.dict[builtins.str<8>, Any]<9>) + _field_types<15> (builtins.dict[builtins.str<8>, Any]<9>) + _fields<16> (Tuple[builtins.str<8>]) + _make<17> + _replace<18> + _source<19> (builtins.str<8>) + x<20> (target.A<0>))) +==> +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names()) +TypeInfo<2>( + Name(target.N) + Bases(builtins.tuple[target.A<0>, ...]<3>) + Mro(target.N<2>, builtins.tuple<3>, typing.Sequence<4>, typing.Iterable<5>, builtins.object<1>) + Names( + _NT<6> + __annotations__<7> (builtins.dict[builtins.str<8>, Any]<9>) + __doc__<10> (builtins.str<8>) + __match_args__<11> (Tuple[Literal['x'], Literal['y']]) + __new__<12> + _asdict<13> + _field_defaults<14> (builtins.dict[builtins.str<8>, Any]<9>) + _field_types<15> (builtins.dict[builtins.str<8>, Any]<9>) + _fields<16> (Tuple[builtins.str<8>, builtins.str<8>]) + _make<17> + _replace<18> + _source<19> (builtins.str<8>) + x<20> (target.A<0>) + y<21> (target.A<0>))) + +[case testNamedTupleOldVersion_typeinfo] +import target +[file target.py] +from typing import NamedTuple +class A: pass +N = NamedTuple('N', [('x', A)]) +[file target.py.next] +from typing import NamedTuple +class A: pass +N = NamedTuple('N', [('x', A), ('y', A)]) +[builtins fixtures/tuple.pyi] +[out] +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names()) +TypeInfo<2>( + Name(target.N) + Bases(builtins.tuple[target.A<0>, ...]<3>) + Mro(target.N<2>, builtins.tuple<3>, typing.Sequence<4>, typing.Iterable<5>, builtins.object<1>) + Names( + _NT<6> + __annotations__<7> (builtins.dict[builtins.str<8>, Any]<9>) + __doc__<10> (builtins.str<8>) + __new__<11> + _asdict<12> + _field_defaults<13> (builtins.dict[builtins.str<8>, Any]<9>) + _field_types<14> (builtins.dict[builtins.str<8>, Any]<9>) + _fields<15> (Tuple[builtins.str<8>]) + _make<16> + _replace<17> + _source<18> (builtins.str<8>) + x<19> (target.A<0>))) +==> +TypeInfo<0>( + Name(target.A) + Bases(builtins.object<1>) + Mro(target.A<0>, builtins.object<1>) + Names()) +TypeInfo<2>( + Name(target.N) + Bases(builtins.tuple[target.A<0>, ...]<3>) + Mro(target.N<2>, builtins.tuple<3>, typing.Sequence<4>, typing.Iterable<5>, builtins.object<1>) + Names( + _NT<6> + __annotations__<7> (builtins.dict[builtins.str<8>, Any]<9>) + __doc__<10> (builtins.str<8>) + __new__<11> + _asdict<12> + _field_defaults<13> (builtins.dict[builtins.str<8>, Any]<9>) + _field_types<14> (builtins.dict[builtins.str<8>, Any]<9>) + _fields<15> (Tuple[builtins.str<8>, builtins.str<8>]) + _make<16> + _replace<17> + _source<18> (builtins.str<8>) + x<19> (target.A<0>) + y<20> (target.A<0>))) + +[case testUnionType_types] +import target +[file target.py] +from typing import Union +class A: pass +a: A +[file target.py.next] +from typing import Union +class A: pass +a: Union[A, int] +[out] +## target +NameExpr:3: target.A<0> +==> +## target +NameExpr:3: Union[target.A<0>, builtins.int<1>] + +[case testTypeType_types] +import target +[file target.py] +from typing import Type +class A: pass +a: Type[A] +[file target.py.next] +from typing import Type +class A: pass +a: Type[A] +[out] +## target +NameExpr:3: Type[target.A<0>] +==> +## target +NameExpr:3: Type[target.A<0>] + +[case testTypeVar_types] +import target +[file target.py] +from typing import TypeVar, Generic +T = TypeVar('T', bound=int) +class A(Generic[T]): + x: T +[file target.py.next] +from typing import TypeVar +T = TypeVar('T', bound='A') +class A(Generic[T]): + x: T +[out] +## target +CallExpr:2: Any +NameExpr:2: Any +TypeVarExpr:2: Any +NameExpr:4: T`1(upper_bound=builtins.int<0>) +==> +## target +CallExpr:2: Any +NameExpr:2: Any +TypeVarExpr:2: Any +NameExpr:4: T`1(upper_bound=target.A[Any]<1>) + +[case testUnboundType_types] + +import target +[file target.py] +from typing import TypeVar, Generic +class A: pass +foo: int +x: foo[A] +[file target.py.next] +from typing import TypeVar, Generic +class A: pass +foo: int +x: foo[A] +[out] +tmp/target.py:4: error: Variable "target.foo" is not valid as a type +tmp/target.py:4: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +## target +NameExpr:3: builtins.int<0> +NameExpr:4: foo?[target.A<1>] +==> +## target +NameExpr:3: builtins.int<0> +NameExpr:4: foo?[target.A<1>] + +[case testOverloaded_types] +import target +[file target.py] +from typing import overload +class A: pass + +@overload +def f(x: A) -> A: pass +@overload +def f(x: int) -> int: pass + +def f(x): pass + +g = f +[file target.py.next] +from typing import overload + +class A: pass + +@overload +def f(x: A) -> A: pass +@overload +def f(x: str) -> str: pass + +def f(x): pass + +g = f +[out] +-- TODO: It is unclear why this works correctly... +## target +NameExpr:11: Overload(def (x: target.A<0>) -> target.A<0>, def (x: builtins.int<1>) -> builtins.int<1>) +NameExpr:11: Overload(def (x: target.A<0>) -> target.A<0>, def (x: builtins.int<1>) -> builtins.int<1>) +==> +## target +NameExpr:12: Overload(def (x: target.A<0>) -> target.A<0>, def (x: builtins.str<2>) -> builtins.str<2>) +NameExpr:12: Overload(def (x: target.A<0>) -> target.A<0>, def (x: builtins.str<2>) -> builtins.str<2>) + +[case testOverloaded] +import target +[file target.py] +from typing import overload +class A: pass + +@overload +def f(x: A) -> A: pass +@overload +def f(x: int) -> int: pass + +def f(x): pass +[file target.py.next] +from typing import overload + +class A: pass +class B: pass + +@overload +def f(x: A) -> B: pass +@overload +def f(x: str) -> str: pass + +def f(x): pass +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ImportFrom:1(typing, [overload]) + ClassDef:2<2>( + A + PassStmt:2<3>()) + OverloadedFuncDef:4<4>( + FuncDef:9<5>( + f + Args( + Var(x)) + Block:9<6>( + PassStmt:9<7>())) + Overload(def (x: target.A<8>) -> target.A<8>, def (x: builtins.int<9>) -> builtins.int<9>) + Decorator:4<10>( + Var(f) + NameExpr(overload [typing.overload<11>]) + FuncDef:5<12>( + f + Args( + Var(x)) + def (x: target.A<8>) -> target.A<8> + Block:5<13>( + PassStmt:5<14>()))) + Decorator:6<15>( + Var(f) + NameExpr(overload [typing.overload<11>]) + FuncDef:7<16>( + f + Args( + Var(x)) + def (x: builtins.int<9>) -> builtins.int<9> + Block:7<17>( + PassStmt:7<18>()))))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ImportFrom:1(typing, [overload]) + ClassDef:3<19>( + A + PassStmt:3<20>()) + ClassDef:4<21>( + B + PassStmt:4<22>()) + OverloadedFuncDef:6<4>( + FuncDef:11<23>( + f + Args( + Var(x)) + Block:11<24>( + PassStmt:11<25>())) + Overload(def (x: target.A<8>) -> target.B<26>, def (x: builtins.str<27>) -> builtins.str<27>) + Decorator:6<28>( + Var(f) + NameExpr(overload [typing.overload<11>]) + FuncDef:7<29>( + f + Args( + Var(x)) + def (x: target.A<8>) -> target.B<26> + Block:7<30>( + PassStmt:7<31>()))) + Decorator:8<32>( + Var(f) + NameExpr(overload [typing.overload<11>]) + FuncDef:9<33>( + f + Args( + Var(x)) + def (x: builtins.str<27>) -> builtins.str<27> + Block:9<34>( + PassStmt:9<35>()))))) + +[case testTypeVar_symtable] +import target + +[file target.py] +from typing import TypeVar +T = TypeVar('T') + +[file target.py.next] +from typing import TypeVar +T = TypeVar('T', bound=int) + +[out] +__main__: + target: MypyFile<0> +target: + T: TypeVarExpr<1> + TypeVar: Var<2> +==> +__main__: + target: MypyFile<0> +target: + T: TypeVarExpr<1> + TypeVar: Var<2> + +[case testTypeAlias_symtable] +import target + +[file target.py] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): pass +X = A[int] + +[file target.py.next] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): pass +X = A[str] + +[out] +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + Generic: Var<2> + T: TypeVarExpr<3> + TypeVar: Var<4> + X: TypeAlias<5> +==> +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + Generic: Var<2> + T: TypeVarExpr<3> + TypeVar: Var<4> + X: TypeAlias<5> + +[case testGenericFunction_types] +import target + +[file target.py] +from typing import TypeVar +class A: pass +T = TypeVar('T', bound=A) +def f(x: T) -> T: pass +f + +[file target.py.next] +from typing import TypeVar +class A: pass +T = TypeVar('T', bound=A) +def f(x: T, y: A) -> T: pass +f + +[out] +## target +CallExpr:3: Any +NameExpr:3: Any +TypeVarExpr:3: Any +NameExpr:5: def [T <: target.A<0>] (x: T`-1(upper_bound=target.A<0>)) -> T`-1(upper_bound=target.A<0>) +==> +## target +CallExpr:3: Any +NameExpr:3: Any +TypeVarExpr:3: Any +NameExpr:5: def [T <: target.A<0>] (x: T`-1(upper_bound=target.A<0>), y: target.A<0>) -> T`-1(upper_bound=target.A<0>) + +[case testMergeOverloaded_types] +import target +[file target.py] +from _x import A +a: A +[file target.py.next] +from _x import A +a: A +[fixture _x.pyi] +from typing import Generic, TypeVar, overload + +T = TypeVar('T') + +class C(Generic[T]): + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: int) -> None: pass +A = C[int] +[out] +## target +NameExpr:2: _x.C[builtins.int<0>]<1> +==> +## target +NameExpr:2: _x.C[builtins.int<0>]<1> + +[case testRefreshVar_symtable] +from typing import TypeVar +from target import f +x = 1 +y = '' # type: str +[file target.py] +f = 1 +[file target.py.next] +[out] +__main__: + TypeVar: Var<0> + f: Var<1>(builtins.int<2>) + x: Var<3>(builtins.int<2>) + y: Var<4>(builtins.str<5>) +target: + f: Var<1>(builtins.int<2>) +==> +__main__: + TypeVar: Var<0> + f: Var<6>(Any) + x: Var<3>(builtins.int<2>) + y: Var<4>(builtins.str<5>) +target: + +[case testRefreshTypeVar_symtable] +from typing import TypeVar +from target import f +T = TypeVar('T') +[file target.py] +f = 1 +[file target.py.next] +[out] +__main__: + T: TypeVarExpr<0> + TypeVar: Var<1> + f: Var<2>(builtins.int<3>) +target: + f: Var<2>(builtins.int<3>) +==> +__main__: + T: TypeVarExpr<0> + TypeVar: Var<1> + f: Var<4>(Any) +target: + +[case testRefreshNamedTuple_symtable] +from typing import NamedTuple +from target import f +N = NamedTuple('N', [('x', int)]) +[file target.py] +f = 1 +[file target.py.next] +[builtins fixtures/tuple.pyi] +[out] +__main__: + N: TypeInfo<0> + NamedTuple: Var<1> + f: Var<2>(builtins.int<3>) +target: + f: Var<2>(builtins.int<3>) +==> +__main__: + N: TypeInfo<0> + NamedTuple: Var<1> + f: Var<4>(Any) +target: + +[case testRefreshAttributeDefinedInClassBody_typeinfo] +from target import f +class A: + a = 1 + b = '' # type: str +[file target.py] +f = 1 +[file target.py.next] +[out] +TypeInfo<0>( + Name(__main__.A) + Bases(builtins.object<1>) + Mro(__main__.A<0>, builtins.object<1>) + Names( + a<2> (builtins.int<3>) + b<4> (builtins.str<5>))) +==> +TypeInfo<0>( + Name(__main__.A) + Bases(builtins.object<1>) + Mro(__main__.A<0>, builtins.object<1>) + Names( + a<2> (builtins.int<3>) + b<4> (builtins.str<5>))) + +[case testDecorator_symtable] +import target +[file target.py] +from contextlib import contextmanager +from typing import Iterator, List, Tuple + +@contextmanager +def f(x: List[Tuple[int]]) -> Iterator[None]: + yield +[file target.py.next] +from contextlib import contextmanager +from typing import Iterator, List, Tuple + +@contextmanager +def f(x: List[Tuple[int]]) -> Iterator[None]: + yield +[typing fixtures/typing-medium.pyi] +[builtins fixtures/list.pyi] +[out] +__main__: + target: MypyFile<0> +target: + Iterator: TypeInfo<1> + List: TypeAlias<2> + Tuple: Var<3> + contextmanager: FuncDef<4> + f: Decorator<5> +==> +__main__: + target: MypyFile<0> +target: + Iterator: TypeInfo<1> + List: TypeAlias<2> + Tuple: Var<3> + contextmanager: FuncDef<4> + f: Decorator<5> + +[case testConditionalFunctionDefinition] +import target +[file target.py] +import sys +class A: pass +class B: pass +if sys.platform == 'nonexistent': + def f(x: A) -> None: pass +else: + def f(x: B) -> None: pass +[file target.py.next] +import sys +class A: pass +class B: pass +if sys.platform == 'nonexistent': + def f(x: A, y: int) -> None: pass +else: + def f(x: B, y: int) -> None: pass +[builtins fixtures/ops.pyi] +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + Import:1(sys) + ClassDef:2<2>( + A + PassStmt:2<3>()) + ClassDef:3<4>( + B + PassStmt:3<5>()) + IfStmt:4<6>( + If( + ComparisonExpr:4<7>( + == + MemberExpr:4<9>( + NameExpr(sys<10>) + platform [sys.platform<8>]) + StrExpr(nonexistent))) + Then( + FuncDef:5<11>( + f + Args( + Var(x)) + def (x: A?) -> None? + Block:5<12>( + PassStmt:5<13>()))) + Else( + FuncDef:7<14>( + f + Args( + Var(x)) + def (x: target.B<15>) + Block:7<16>( + PassStmt:7<17>()))))) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + Import:1(sys) + ClassDef:2<18>( + A + PassStmt:2<19>()) + ClassDef:3<20>( + B + PassStmt:3<21>()) + IfStmt:4<22>( + If( + ComparisonExpr:4<23>( + == + MemberExpr:4<24>( + NameExpr(sys<10>) + platform [sys.platform<8>]) + StrExpr(nonexistent))) + Then( + FuncDef:5<25>( + f + Args( + Var(x) + Var(y)) + def (x: A?, y: int?) -> None? + Block:5<26>( + PassStmt:5<27>()))) + Else( + FuncDef:7<14>( + f + Args( + Var(x) + Var(y)) + def (x: target.B<15>, y: builtins.int<28>) + Block:7<29>( + PassStmt:7<30>()))))) + +[case testMergeTypedDict_symtable] +import target +[file target.py] +from mypy_extensions import TypedDict +class A: pass +D = TypedDict('D', {'a': A}) +d: D +[file target.py.next] +from mypy_extensions import TypedDict +class A: pass +D = TypedDict('D', {'a': A, 'b': int}) +d: D +[builtins fixtures/dict.pyi] +[out] +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + D: TypeInfo<2> + TypedDict: FuncDef<3> + d: Var<4>(TypedDict('target.D', {'a': target.A<1>})) +==> +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + D: TypeInfo<2> + TypedDict: FuncDef<3> + d: Var<4>(TypedDict('target.D', {'a': target.A<1>, 'b': builtins.int<5>})) + +[case testNewType_symtable] +import target +[file target.py] +from typing import NewType +class A: pass +B = NewType('B', A) +C = NewType('C', A) +[file target.py.next] +from typing import NewType +class A: pass +B = NewType('B', A) +C = NewType('C', B) +D = NewType('D', int) +[out] +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + B: TypeInfo<2> + C: TypeInfo<3> + NewType: Var<4> +==> +__main__: + target: MypyFile<0> +target: + A: TypeInfo<1> + B: TypeInfo<2> + C: TypeInfo<3> + D: TypeInfo<5> + NewType: Var<4> + +[case testCallable_symtable-skip] +# The TypeInfo is currently not being merged correctly +import target +[file target.py] +def g(o: object) -> None: + if callable(o): + pass +[file target.py.next] +def g(o: object) -> None: + if callable(o): + o() +[builtins fixtures/callable.pyi] +[out] +__main__: + target: MypyFile<0> +target: + : TypeInfo<1> + g: FuncDef<2> +==> +__main__: + target: MypyFile<0> +target: + : TypeInfo<1> + g: FuncDef<2> + +[case testMetaclass_typeinfo] +import target +[file target.py] +class M(type): pass +class C(metaclass=M): + pass +[file target.py.next] +class M(type): pass +class C(metaclass=M): + pass # dummy change +[out] +TypeInfo<0>( + Name(target.C) + Bases(builtins.object<1>) + Mro(target.C<0>, builtins.object<1>) + Names() + DeclaredMetaclass(target.M<2>) + MetaclassType(target.M<2>)) +TypeInfo<2>( + Name(target.M) + Bases(builtins.type<3>) + Mro(target.M<2>, builtins.type<3>, builtins.object<1>) + Names()) +==> +TypeInfo<0>( + Name(target.C) + Bases(builtins.object<1>) + Mro(target.C<0>, builtins.object<1>) + Names() + DeclaredMetaclass(target.M<2>) + MetaclassType(target.M<2>)) +TypeInfo<2>( + Name(target.M) + Bases(builtins.type<3>) + Mro(target.M<2>, builtins.type<3>, builtins.object<1>) + Names()) + +[case testCast_symtable] +import target +[file target.py] +from typing import cast +class Thing: + pass +thing = cast(Thing, Thing()) +[file target.py.next] +from typing import cast +class Thing: + pass +thing = cast(Thing, Thing()) +[out] +__main__: + target: MypyFile<0> +target: + Thing: TypeInfo<1> + cast: Var<2> + thing: Var<3>(target.Thing<1>) +==> +__main__: + target: MypyFile<0> +target: + Thing: TypeInfo<1> + cast: Var<2> + thing: Var<3>(target.Thing<1>) + +[case testClassBasedEnum_typeinfo] +import target +[file target.py] +from enum import Enum +class A(Enum): + X = 0 +[file target.py.next] +from enum import Enum +class A(Enum): + X = 0 + Y = 1 +[builtins fixtures/enum.pyi] +[out] +TypeInfo<0>( + Name(target.A) + Bases(enum.Enum<1>) + Mro(target.A<0>, enum.Enum<1>, builtins.object<2>) + Names( + X<3> (Literal[0]?<4>)) + MetaclassType(enum.EnumMeta<5>)) +==> +TypeInfo<0>( + Name(target.A) + Bases(enum.Enum<1>) + Mro(target.A<0>, enum.Enum<1>, builtins.object<2>) + Names( + X<3> (Literal[0]?<4>) + Y<6> (Literal[1]?<4>)) + MetaclassType(enum.EnumMeta<5>)) + +[case testLiteralMerge] +import target +[file target.py] +from typing_extensions import Literal +def foo(x: Literal[3]) -> Literal['a']: pass +bar: Literal[4] = 4 +[file target.py.next] +from typing_extensions import Literal +def foo(x: Literal['3']) -> Literal['b']: pass +bar: Literal[5] = 5 +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ImportFrom:1(typing_extensions, [Literal]) + FuncDef:2<2>( + foo + Args( + Var(x)) + def (x: Literal[3]) -> Literal['a'] + Block:2<3>( + PassStmt:2<4>())) + AssignmentStmt:3<5>( + NameExpr(bar [target.bar<6>]) + IntExpr(4) + Literal[4])) +==> +MypyFile:1<0>( + tmp/main + Import:1(target)) +MypyFile:1<1>( + tmp/target.py + ImportFrom:1(typing_extensions, [Literal]) + FuncDef:2<2>( + foo + Args( + Var(x)) + def (x: Literal['3']) -> Literal['b'] + Block:2<7>( + PassStmt:2<8>())) + AssignmentStmt:3<9>( + NameExpr(bar [target.bar<6>]) + IntExpr(5) + Literal[5])) diff --git a/test-data/unit/outputjson.test b/test-data/unit/outputjson.test new file mode 100644 index 000000000000..43649b7b781d --- /dev/null +++ b/test-data/unit/outputjson.test @@ -0,0 +1,44 @@ +-- Test cases for `--output=json`. +-- These cannot be run by the usual unit test runner because of the backslashes +-- in the output, which get normalized to forward slashes by the test suite on +-- Windows. + +[case testOutputJsonNoIssues] +# flags: --output=json +def foo() -> None: + pass + +foo() +[out] + +[case testOutputJsonSimple] +# flags: --output=json +def foo() -> None: + pass + +foo(1) +[out] +{"file": "main", "line": 5, "column": 0, "message": "Too many arguments for \"foo\"", "hint": null, "code": "call-arg", "severity": "error"} + +[case testOutputJsonWithHint] +# flags: --output=json +from typing import Optional, overload + +@overload +def foo() -> None: ... +@overload +def foo(x: int) -> None: ... + +def foo(x: Optional[int] = None) -> None: + ... + +reveal_type(foo) + +foo('42') + +def bar() -> None: ... +bar('42') +[out] +{"file": "main", "line": 12, "column": 12, "message": "Revealed type is \"Overload(def (), def (x: builtins.int))\"", "hint": null, "code": "misc", "severity": "note"} +{"file": "main", "line": 14, "column": 0, "message": "No overload variant of \"foo\" matches argument type \"str\"", "hint": "Possible overload variants:\n def foo() -> None\n def foo(x: int) -> None", "code": "call-overload", "severity": "error"} +{"file": "main", "line": 17, "column": 0, "message": "Too many arguments for \"bar\"", "hint": null, "code": "call-arg", "severity": "error"} diff --git a/test-data/unit/parse-errors.test b/test-data/unit/parse-errors.test new file mode 100644 index 000000000000..7b1078d3fa2f --- /dev/null +++ b/test-data/unit/parse-errors.test @@ -0,0 +1,445 @@ +-- Test cases for parser errors. Each test case consists of two sections. +-- The first section contains [case NAME] followed by the input code, while +-- the second section contains [out] followed by the output from the parser. +-- +-- The input file name in errors is "file". +-- +-- Comments starting with "--" in this file will be ignored, except for lines +-- starting with "----" that are not ignored. The first two dashes of these +-- lines are interpreted as escapes and removed. + +[case testInvalidFunction] +def f() + pass +[out] +file:1: error: invalid syntax + +[case testUnexpectedIndent] +1 + 2 +[out] +file:2: error: unexpected indent + +[case testInconsistentIndent] +if x: + 1 + 1 +[out] +file:3: error: unexpected indent + +[case testInconsistentIndent2] +if x: + 1 + 1 +[out] +file:3: error: unindent does not match any outer indentation level + +[case testInvalidBinaryOp] +1> +a* +a+1* +[out] +file:1: error: invalid syntax + +[case testDoubleStar] +**a +[out] +file:1: error: invalid syntax + +[case testMissingSuperClass] +class A(: + pass +[out] +file:1: error: invalid syntax + +[case testUnexpectedEof] +if 1: +[out] +file:1: error: unexpected EOF while parsing + +[case testInvalidKeywordArguments1] +f(x=y, z) +[out] +file:1: error: positional argument follows keyword argument + +[case testInvalidKeywordArguments2] +f(**x, y) +[out] +file:1: error: positional argument follows keyword argument unpacking + +[case testInvalidBareAsteriskAndVarArgs2] +def f(*x: A, *) -> None: pass +[out] +file:1: error: invalid syntax + +[case testInvalidBareAsteriskAndVarArgs3] +def f(*, *x: A) -> None: pass +[out] +file:1: error: invalid syntax + +[case testInvalidBareAsteriskAndVarArgs4] +def f(*, **x: A) -> None: pass +[out] +file:1: error: named arguments must follow bare * + +[case testInvalidBareAsterisk1] +def f(*) -> None: pass +[out] +file:1: error: named arguments must follow bare * + +[case testInvalidBareAsterisk2] +def f(x, *) -> None: pass +[out] +file:1: error: named arguments must follow bare * + +[case testInvalidFuncDefArgs1] +def f(x = y, x): pass +[out] +file:1: error: non-default argument follows default argument + +[case testInvalidFuncDefArgs3] +def f(**x, y): + pass +[out] +file:1: error: invalid syntax + +[case testInvalidFuncDefArgs4] +def f(**x, y=x): + pass +[out] +file:1: error: invalid syntax + +[case testInvalidTypeComment] +0 +x = 0 # type: A A +[out] +file:2: error: Syntax error in type comment "A A" + +[case testInvalidTypeComment2] +0 +x = 0 # type: A[ +[out] +file:2: error: Syntax error in type comment "A[" + +[case testInvalidTypeComment3] +0 +x = 0 # type: +[out] +file:2: error: Syntax error in type comment "" + +[case testInvalidTypeComment4] +0 +x = 0 # type: * +[out] +file:2: error: Syntax error in type comment "*" + +[case testInvalidTypeComment5] +0 +x = 0 # type:# some comment +[out] +file:2: error: Syntax error in type comment "" + +[case testInvalidTypeComment6] +0 +x = 0 # type: *# comment #6 +[out] +file:2: error: Syntax error in type comment "*" + +[case testInvalidTypeComment7] +0 +x = 0 # type: A B #comment #7 +[out] +file:2: error: Syntax error in type comment "A B" + +[case testMissingBracket] +def foo( +[out] +file:1: error: unexpected EOF while parsing +[out version>=3.10] +file:1: error: '(' was never closed + +[case testInvalidSignatureInComment1] +def f(): # type: x + pass +[out] +file:1: error: Syntax error in type comment "x" +file:1: note: Suggestion: wrap argument types in parentheses + +[case testInvalidSignatureInComment2] +def f(): # type: + pass +[out] +file:1: error: Syntax error in type comment "" + +[case testInvalidSignatureInComment3] +def f(): # type: ( + pass +[out] +file:1: error: Syntax error in type comment "(" + +[case testInvalidSignatureInComment4] +def f(): # type: (. + pass +[out] +file:1: error: Syntax error in type comment "(." + +[case testInvalidSignatureInComment5] +def f(): # type: (x + pass +[out] +file:1: error: Syntax error in type comment "(x" + +[case testInvalidSignatureInComment6] +def f(): # type: (x) + pass +[out] +file:1: error: Syntax error in type comment "(x)" + +[case testInvalidSignatureInComment7] +def f(): # type: (x) - + pass +[out] +file:1: error: Syntax error in type comment "(x) -" + +[case testInvalidSignatureInComment8] +def f(): # type: (x) -> + pass +[out] +file:1: error: Syntax error in type comment "(x) ->" + +[case testInvalidSignatureInComment9] +def f(): # type: (x) -> . + pass +[out] +file:1: error: Syntax error in type comment "(x) -> ." + +[case testInvalidSignatureInComment10] +def f(): # type: (x) -> x x + pass +[out] +file:1: error: Syntax error in type comment "(x) -> x x" + +[case testInvalidSignatureInComment11] +def f(): # type: # abc comment + pass +[out] +file:1: error: Syntax error in type comment "" + +[case testInvalidSignatureInComment12] +def f(): # type: (x) -> x x # comment #2 + pass +[out] +file:1: error: Syntax error in type comment "(x) -> x x" + + +[case testDuplicateSignatures1] +def f() -> None: # type: () -> None + pass +def f(): # type: () -> None + pass +[out] +file:1: error: Function has duplicate type signatures + +[case testDuplicateSignatures2] +def f(x, y: Z): # type: (x, y) -> z + pass +[out] +file:1: error: Function has duplicate type signatures + +[case testTooManyTypes] +def f(x, y): # type: (X, Y, Z) -> z + pass +[out] +file:1: error: Type signature has too many arguments + +[case testTooFewTypes] +def f(x, y): # type: (X) -> z + pass +[out] +file:1: error: Type signature has too few arguments + +[case testCommentFunctionAnnotationVarArgMispatch-skip] +# see mypy issue #1997 +def f(x): # type: (*X) -> Y + pass +def g(*x): # type: (X) -> Y + pass +[out] +file:1: error: Inconsistent use of "*" in function signature +file:3: error: Inconsistent use of "*" in function signature + +[case testCommentFunctionAnnotationVarArgMispatch2-skip] +# see mypy issue #1997 +def f(*x, **y): # type: (**X, *Y) -> Z + pass +def g(*x, **y): # type: (*X, *Y) -> Z + pass +[out] +file:1: error: Inconsistent use of "*" in function signature +file:3: error: Syntax error in type comment +file:3: error: Inconsistent use of "*" in function signature +file:3: error: Inconsistent use of "**" in function signature + +[case testPrintStatementInPython3] +print 1 +[out] +file:1: error: Missing parentheses in call to 'print'. Did you mean print(1)? + +[case testInvalidConditionInConditionalExpression] +1 if 2, 3 else 4 +[out] +file:1: error: invalid syntax + +[case testInvalidConditionInConditionalExpression2] +1 if x for y in z else 4 +[out] +file:1: error: invalid syntax + +[case testInvalidConditionInConditionalExpression3] +1 if x else for y in z +[out] +file:1: error: invalid syntax + +[case testYieldFromNotRightParameter] +def f(): + yield from +[out] +file:2: error: invalid syntax + +[case testYieldFromAfterReturn] +def f(): + return yield from h() +[out] +file:2: error: invalid syntax + +[case testImportDotModule] +import .x +[out] +file:1: error: invalid syntax + +[case testImportDot] +import . +[out] +file:1: error: invalid syntax + +[case testInvalidFunctionName] +def while(): pass +[out] +file:1: error: invalid syntax + +[case testInvalidEllipsis1] +...0 +..._ +...a +[out] +file:1: error: invalid syntax + +[case testBlockStatementInSingleLineIf] +if 1: if 2: pass +[out] +file:1: error: invalid syntax + +[case testBlockStatementInSingleLineIf2] +if 1: while 2: pass +[out] +file:1: error: invalid syntax + +[case testBlockStatementInSingleLineIf3] +if 1: for x in y: pass +[out] +file:1: error: invalid syntax + +[case testUnexpectedEllipsis] +a = a... +[out] +file:1: error: invalid syntax + +[case testParseErrorBeforeUnicodeLiteral] +x u'y' +[out] +file:1: error: invalid syntax + +[case testParseErrorInExtendedSlicing] +x[:, +[out] +file:1: error: unexpected EOF while parsing + +[case testParseErrorInExtendedSlicing2] +x[:,:: +[out] +file:1: error: unexpected EOF while parsing + +[case testParseErrorInExtendedSlicing3] +x[:,: +[out] +file:1: error: unexpected EOF while parsing + +[case testInvalidEncoding] +# foo +# coding: uft-8 +[out] +file:0: error: unknown encoding: uft-8 + +[case testInvalidEncoding2] +# coding=Uft.8 +[out] +file:0: error: unknown encoding: Uft.8 + +[case testInvalidEncoding3] +#!/usr/bin python +# vim: set fileencoding=uft8 : +[out] +file:0: error: unknown encoding: uft8 + +[case testDoubleEncoding] +# coding: uft8 +# coding: utf8 +# The first coding cookie should be used and fail. +[out] +file:0: error: unknown encoding: uft8 + +[case testDoubleEncoding2] +# Again the first cookie should be used and fail. +# coding: uft8 +# coding: utf8 +[out] +file:0: error: unknown encoding: uft8 + +[case testLongLiteralInPython3] +2L +0x2L +[out] +file:1: error: invalid syntax + +[case testPython2LegacyInequalityInPython3] +1 <> 2 +[out] +file:1: error: invalid syntax + +[case testLambdaInListComprehensionInPython3] +([ 0 for x in 1, 2 if 3 ]) +[out] +file:1: error: invalid syntax + +[case testTupleArgListInPython3] +def f(x, (y, z)): pass +[out] +file:1: error: invalid syntax + +[case testBackquoteInPython3] +`1 + 2` +[out] +file:1: error: invalid syntax + +[case testSmartQuotes] +foo = ‘bar’ +[out] +file:1: error: invalid character in identifier + +[case testExceptCommaInPython3] +try: + pass +except KeyError, IndexError: + pass +[out] +file:3: error: invalid syntax diff --git a/test-data/unit/parse-python310.test b/test-data/unit/parse-python310.test new file mode 100644 index 000000000000..87e0e9d5d283 --- /dev/null +++ b/test-data/unit/parse-python310.test @@ -0,0 +1,603 @@ +-- Test cases for parser -- Python 3.10 syntax (match statement) +-- +-- See parse.test for a description of this file format. + +[case testSimpleMatch] +match a: + case 1: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + ValuePattern:2( + IntExpr(1))) + Body( + PassStmt:3()))) + + +[case testTupleMatch] +match a, b: + case 1: + pass +[out] +MypyFile:1( + MatchStmt:1( + TupleExpr:1( + NameExpr(a) + NameExpr(b)) + Pattern( + ValuePattern:2( + IntExpr(1))) + Body( + PassStmt:3()))) + +[case testMatchWithGuard] +match a: + case 1 if f(): + pass + case d if d > 5: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + ValuePattern:2( + IntExpr(1))) + Guard( + CallExpr:2( + NameExpr(f) + Args())) + Body( + PassStmt:3()) + Pattern( + AsPattern:4( + NameExpr(d))) + Guard( + ComparisonExpr:4( + > + NameExpr(d) + IntExpr(5))) + Body( + PassStmt:5()))) + +[case testAsPattern] +match a: + case 1 as b: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + AsPattern:2( + ValuePattern:2( + IntExpr(1)) + NameExpr(b))) + Body( + PassStmt:3()))) + + +[case testLiteralPattern] +match a: + case 1: + pass + case -1: + pass + case 1+2j: + pass + case -1+2j: + pass + case 1-2j: + pass + case -1-2j: + pass + case "str": + pass + case b"bytes": + pass + case r"raw_string": + pass + case None: + pass + case True: + pass + case False: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + ValuePattern:2( + IntExpr(1))) + Body( + PassStmt:3()) + Pattern( + ValuePattern:4( + UnaryExpr:4( + - + IntExpr(1)))) + Body( + PassStmt:5()) + Pattern( + ValuePattern:6( + OpExpr:6( + + + IntExpr(1) + ComplexExpr(2j)))) + Body( + PassStmt:7()) + Pattern( + ValuePattern:8( + OpExpr:8( + + + UnaryExpr:8( + - + IntExpr(1)) + ComplexExpr(2j)))) + Body( + PassStmt:9()) + Pattern( + ValuePattern:10( + OpExpr:10( + - + IntExpr(1) + ComplexExpr(2j)))) + Body( + PassStmt:11()) + Pattern( + ValuePattern:12( + OpExpr:12( + - + UnaryExpr:12( + - + IntExpr(1)) + ComplexExpr(2j)))) + Body( + PassStmt:13()) + Pattern( + ValuePattern:14( + StrExpr(str))) + Body( + PassStmt:15()) + Pattern( + ValuePattern:16( + BytesExpr(bytes))) + Body( + PassStmt:17()) + Pattern( + ValuePattern:18( + StrExpr(raw_string))) + Body( + PassStmt:19()) + Pattern( + SingletonPattern:20()) + Body( + PassStmt:21()) + Pattern( + SingletonPattern:22( + True)) + Body( + PassStmt:23()) + Pattern( + SingletonPattern:24( + False)) + Body( + PassStmt:25()))) + +[case testCapturePattern] +match a: + case x: + pass + case longName: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + AsPattern:2( + NameExpr(x))) + Body( + PassStmt:3()) + Pattern( + AsPattern:4( + NameExpr(longName))) + Body( + PassStmt:5()))) + +[case testWildcardPattern] +match a: + case _: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + AsPattern:2()) + Body( + PassStmt:3()))) + +[case testValuePattern] +match a: + case b.c: + pass + case b.c.d.e.f: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + ValuePattern:2( + MemberExpr:2( + NameExpr(b) + c))) + Body( + PassStmt:3()) + Pattern( + ValuePattern:4( + MemberExpr:4( + MemberExpr:4( + MemberExpr:4( + MemberExpr:4( + NameExpr(b) + c) + d) + e) + f))) + Body( + PassStmt:5()))) + +[case testGroupPattern] +# This is optimized out by the compiler. It doesn't appear in the ast +match a: + case (1): + pass +[out] +MypyFile:1( + MatchStmt:2( + NameExpr(a) + Pattern( + ValuePattern:3( + IntExpr(1))) + Body( + PassStmt:4()))) + +[case testSequencePattern] +match a: + case []: + pass + case (): + pass + case [1]: + pass + case (1,): + pass + case 1,: + pass + case [1, 2, 3]: + pass + case (1, 2, 3): + pass + case 1, 2, 3: + pass + case [1, *a, 2]: + pass + case (1, *a, 2): + pass + case 1, *a, 2: + pass + case [1, *_, 2]: + pass + case (1, *_, 2): + pass + case 1, *_, 2: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + SequencePattern:2()) + Body( + PassStmt:3()) + Pattern( + SequencePattern:4()) + Body( + PassStmt:5()) + Pattern( + SequencePattern:6( + ValuePattern:6( + IntExpr(1)))) + Body( + PassStmt:7()) + Pattern( + SequencePattern:8( + ValuePattern:8( + IntExpr(1)))) + Body( + PassStmt:9()) + Pattern( + SequencePattern:10( + ValuePattern:10( + IntExpr(1)))) + Body( + PassStmt:11()) + Pattern( + SequencePattern:12( + ValuePattern:12( + IntExpr(1)) + ValuePattern:12( + IntExpr(2)) + ValuePattern:12( + IntExpr(3)))) + Body( + PassStmt:13()) + Pattern( + SequencePattern:14( + ValuePattern:14( + IntExpr(1)) + ValuePattern:14( + IntExpr(2)) + ValuePattern:14( + IntExpr(3)))) + Body( + PassStmt:15()) + Pattern( + SequencePattern:16( + ValuePattern:16( + IntExpr(1)) + ValuePattern:16( + IntExpr(2)) + ValuePattern:16( + IntExpr(3)))) + Body( + PassStmt:17()) + Pattern( + SequencePattern:18( + ValuePattern:18( + IntExpr(1)) + StarredPattern:18( + NameExpr(a)) + ValuePattern:18( + IntExpr(2)))) + Body( + PassStmt:19()) + Pattern( + SequencePattern:20( + ValuePattern:20( + IntExpr(1)) + StarredPattern:20( + NameExpr(a)) + ValuePattern:20( + IntExpr(2)))) + Body( + PassStmt:21()) + Pattern( + SequencePattern:22( + ValuePattern:22( + IntExpr(1)) + StarredPattern:22( + NameExpr(a)) + ValuePattern:22( + IntExpr(2)))) + Body( + PassStmt:23()) + Pattern( + SequencePattern:24( + ValuePattern:24( + IntExpr(1)) + StarredPattern:24() + ValuePattern:24( + IntExpr(2)))) + Body( + PassStmt:25()) + Pattern( + SequencePattern:26( + ValuePattern:26( + IntExpr(1)) + StarredPattern:26() + ValuePattern:26( + IntExpr(2)))) + Body( + PassStmt:27()) + Pattern( + SequencePattern:28( + ValuePattern:28( + IntExpr(1)) + StarredPattern:28() + ValuePattern:28( + IntExpr(2)))) + Body( + PassStmt:29()))) + +[case testMappingPattern] +match a: + case {'k': v}: + pass + case {a.b: v}: + pass + case {1: v}: + pass + case {a.c: v}: + pass + case {'k': v1, a.b: v2, 1: v3, a.c: v4}: + pass + case {'k1': 1, 'k2': "str", 'k3': b'bytes', 'k4': None}: + pass + case {'k': v, **r}: + pass + case {**r}: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + MappingPattern:2( + Key( + StrExpr(k)) + Value( + AsPattern:2( + NameExpr(v))))) + Body( + PassStmt:3()) + Pattern( + MappingPattern:4( + Key( + MemberExpr:4( + NameExpr(a) + b)) + Value( + AsPattern:4( + NameExpr(v))))) + Body( + PassStmt:5()) + Pattern( + MappingPattern:6( + Key( + IntExpr(1)) + Value( + AsPattern:6( + NameExpr(v))))) + Body( + PassStmt:7()) + Pattern( + MappingPattern:8( + Key( + MemberExpr:8( + NameExpr(a) + c)) + Value( + AsPattern:8( + NameExpr(v))))) + Body( + PassStmt:9()) + Pattern( + MappingPattern:10( + Key( + StrExpr(k)) + Value( + AsPattern:10( + NameExpr(v1))) + Key( + MemberExpr:10( + NameExpr(a) + b)) + Value( + AsPattern:10( + NameExpr(v2))) + Key( + IntExpr(1)) + Value( + AsPattern:10( + NameExpr(v3))) + Key( + MemberExpr:10( + NameExpr(a) + c)) + Value( + AsPattern:10( + NameExpr(v4))))) + Body( + PassStmt:11()) + Pattern( + MappingPattern:12( + Key( + StrExpr(k1)) + Value( + ValuePattern:12( + IntExpr(1))) + Key( + StrExpr(k2)) + Value( + ValuePattern:12( + StrExpr(str))) + Key( + StrExpr(k3)) + Value( + ValuePattern:12( + BytesExpr(bytes))) + Key( + StrExpr(k4)) + Value( + SingletonPattern:12()))) + Body( + PassStmt:13()) + Pattern( + MappingPattern:14( + Key( + StrExpr(k)) + Value( + AsPattern:14( + NameExpr(v))) + Rest( + NameExpr(r)))) + Body( + PassStmt:15()) + Pattern( + MappingPattern:16( + Rest( + NameExpr(r)))) + Body( + PassStmt:17()))) + +[case testClassPattern] +match a: + case A(): + pass + case B(1, 2): + pass + case B(1, b=2): + pass + case B(a=1, b=2): + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(a) + Pattern( + ClassPattern:2( + NameExpr(A))) + Body( + PassStmt:3()) + Pattern( + ClassPattern:4( + NameExpr(B) + Positionals( + ValuePattern:4( + IntExpr(1)) + ValuePattern:4( + IntExpr(2))))) + Body( + PassStmt:5()) + Pattern( + ClassPattern:6( + NameExpr(B) + Positionals( + ValuePattern:6( + IntExpr(1))) + Keyword( + b + ValuePattern:6( + IntExpr(2))))) + Body( + PassStmt:7()) + Pattern( + ClassPattern:8( + NameExpr(B) + Keyword( + a + ValuePattern:8( + IntExpr(1))) + Keyword( + b + ValuePattern:8( + IntExpr(2))))) + Body( + PassStmt:9()))) diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test new file mode 100644 index 000000000000..2b1f9b42e0f7 --- /dev/null +++ b/test-data/unit/parse-python312.test @@ -0,0 +1,90 @@ +[case testPEP695TypeAlias] +# comment +type A[T] = C[T] +[out] +MypyFile:1( + TypeAliasStmt:2( + NameExpr(A) + TypeParam( + T) + LambdaExpr:2( + Block:-1( + ReturnStmt:2( + IndexExpr:2( + NameExpr(C) + NameExpr(T))))))) + +[case testPEP695GenericFunction] +# comment + +def f[T](): pass +def g[T: str](): pass +def h[T: (int, str)](): pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + T) + Block:3( + PassStmt:3())) + FuncDef:4( + g + TypeParam( + T + str?) + Block:4( + PassStmt:4())) + FuncDef:5( + h + TypeParam( + T + Values( + int? + str?)) + Block:5( + PassStmt:5()))) + +[case testPEP695ParamSpec] +# comment + +def f[**P](): pass +class C[T: int, **P]: pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + **P) + Block:3( + PassStmt:3())) + ClassDef:4( + C + TypeParam( + T + int?) + TypeParam( + **P) + PassStmt:4())) + +[case testPEP695TypeVarTuple] +# comment + +def f[*Ts](): pass +class C[T: int, *Ts]: pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + *Ts) + Block:3( + PassStmt:3())) + ClassDef:4( + C + TypeParam( + T + int?) + TypeParam( + *Ts) + PassStmt:4())) diff --git a/test-data/unit/parse-python313.test b/test-data/unit/parse-python313.test new file mode 100644 index 000000000000..efbafb0766f5 --- /dev/null +++ b/test-data/unit/parse-python313.test @@ -0,0 +1,80 @@ +[case testPEP696TypeAlias] +type A[T = int] = C[T] +[out] +MypyFile:1( + TypeAliasStmt:1( + NameExpr(A) + TypeParam( + T + Default( + int?)) + LambdaExpr:1( + Block:-1( + ReturnStmt:1( + IndexExpr:1( + NameExpr(C) + NameExpr(T))))))) + +[case testPEP696GenericFunction] +def f[T = int](): pass +class C[T = int]: pass +[out] +MypyFile:1( + FuncDef:1( + f + TypeParam( + T + Default( + int?)) + Block:1( + PassStmt:1())) + ClassDef:2( + C + TypeParam( + T + Default( + int?)) + PassStmt:2())) + +[case testPEP696ParamSpec] +def f[**P = [int, str]](): pass +class C[**P = [int, str]]: pass +[out] +[out] +MypyFile:1( + FuncDef:1( + f + TypeParam( + **P + Default( + )) + Block:1( + PassStmt:1())) + ClassDef:2( + C + TypeParam( + **P + Default( + )) + PassStmt:2())) + +[case testPEP696TypeVarTuple] +def f[*Ts = *tuple[str, int]](): pass +class C[*Ts = *tuple[str, int]]: pass +[out] +MypyFile:1( + FuncDef:1( + f + TypeParam( + *Ts + Default( + Unpack[tuple?[str?, int?]])) + Block:1( + PassStmt:1())) + ClassDef:2( + C + TypeParam( + *Ts + Default( + Unpack[tuple?[str?, int?]])) + PassStmt:2())) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test new file mode 100644 index 000000000000..10ceaa947fd4 --- /dev/null +++ b/test-data/unit/parse.test @@ -0,0 +1,3839 @@ +-- Test cases for parser. Each test case consists of two sections. +-- The first section contains [case NAME] followed by the input code, while +-- the second section contains [out] followed by the output from the parser. +-- +-- Lines starting with "--" in this file will be ignored, except for lines +-- starting with "----" that are not ignored. The first two dashes of these +-- lines are interpreted as escapes and removed. + +[case testEmptyFile] +[out] +MypyFile:1() + +[case testExpressionStatement] +1 +[out] +MypyFile:1( + ExpressionStmt:1( + IntExpr(1))) + +[case testAssignment] +x = 1 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1))) + +[case testExpressionBasics] +x = f(1, None) +123 * (2 + x) +"hello".lower() +-1.23 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + CallExpr:1( + NameExpr(f) + Args( + IntExpr(1) + NameExpr(None)))) + ExpressionStmt:2( + OpExpr:2( + * + IntExpr(123) + OpExpr:2( + + + IntExpr(2) + NameExpr(x)))) + ExpressionStmt:3( + CallExpr:3( + MemberExpr:3( + StrExpr(hello) + lower) + Args())) + ExpressionStmt:4( + UnaryExpr:4( + - + FloatExpr(1.23)))) + +[case testSingleQuotedStr] +'' +'foo' +'foo\ +bar' +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr()) + ExpressionStmt:2( + StrExpr(foo)) + ExpressionStmt:3( + StrExpr(foobar))) + +[case testDoubleQuotedStr] +"" +"foo" +"foo\ +bar" +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr()) + ExpressionStmt:2( + StrExpr(foo)) + ExpressionStmt:3( + StrExpr(foobar))) + +[case testRawStr] +r'x\n\'' +r"x\n\"" +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr(x\n\')) + ExpressionStmt:2( + StrExpr(x\n\"))) + +[case testBytes] +b'foo' +b"foo\ +bar" +br'x\n\'' +[out] +MypyFile:1( + ExpressionStmt:1( + BytesExpr(foo)) + ExpressionStmt:2( + BytesExpr(foobar)) + ExpressionStmt:3( + BytesExpr(x\\n\\'))) + +[case testEscapesInStrings] +'\r\n\t\x2f\u123f' +b'\r\n\t\x2f\u123f' +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr(\u000d\u000a\u0009/\u123f)) + ExpressionStmt:2( + BytesExpr(\r\n\t/\\\u123f))) +-- Note \\u in the b'...' case (\u sequence not translated) + +[case testEscapedQuote] +'\'' +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr('))) + +[case testOctalEscapes] +'\0\1\177\1234' +b'\1\476' +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr(\u0000\u0001\u007fS4)) + ExpressionStmt:2( + BytesExpr(\x01>))) + +[case testUnicodeLiteralInPython3] +u'foo' +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr(foo))) + +[case testArrays] +a = [] +a = [1, 2] +a[[1]] = a[2] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + ListExpr:1()) + AssignmentStmt:2( + NameExpr(a) + ListExpr:2( + IntExpr(1) + IntExpr(2))) + AssignmentStmt:3( + IndexExpr:3( + NameExpr(a) + ListExpr:3( + IntExpr(1))) + IndexExpr:3( + NameExpr(a) + IntExpr(2)))) + +[case testTuples] +() +(1,) +(1, foo) +a, b = 1, (2, 3) +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1()) + ExpressionStmt:2( + TupleExpr:2( + IntExpr(1))) + ExpressionStmt:3( + TupleExpr:3( + IntExpr(1) + NameExpr(foo))) + AssignmentStmt:4( + TupleExpr:4( + NameExpr(a) + NameExpr(b)) + TupleExpr:4( + IntExpr(1) + TupleExpr:4( + IntExpr(2) + IntExpr(3))))) + +[case testSimpleFunction] +def main(): + 1 +[out] +MypyFile:1( + FuncDef:1( + main + Block:2( + ExpressionStmt:2( + IntExpr(1))))) + +[case testPass] +def f(): + pass +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + PassStmt:2()))) + +[case testIf] +if 1: + 2 +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + ExpressionStmt:2( + IntExpr(2))))) + +[case testIfElse] +if 1: + 2 +else: + 3 +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + ExpressionStmt:2( + IntExpr(2))) + Else( + ExpressionStmt:4( + IntExpr(3))))) + +[case testIfElif] +if 1: + 2 +elif 3: + 4 +elif 5: + 6 +else: + 7 +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + ExpressionStmt:2( + IntExpr(2))) + Else( + IfStmt:3( + If( + IntExpr(3)) + Then( + ExpressionStmt:4( + IntExpr(4))) + Else( + IfStmt:5( + If( + IntExpr(5)) + Then( + ExpressionStmt:6( + IntExpr(6))) + Else( + ExpressionStmt:8( + IntExpr(7))))))))) + +[case testWhile] +while 1: + pass +[out] +MypyFile:1( + WhileStmt:1( + IntExpr(1) + Block:2( + PassStmt:2()))) + +[case testReturn] +def f(): + return 1 +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ReturnStmt:2( + IntExpr(1))))) + + +[case testReturnWithoutValue] +def f(): + return +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ReturnStmt:2()))) + +[case testBreak] +while 1: + break +[out] +MypyFile:1( + WhileStmt:1( + IntExpr(1) + Block:2( + BreakStmt:2()))) + +[case testLargeBlock] +if 1: + x = 1 + while 2: + pass + y = 2 +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + AssignmentStmt:2( + NameExpr(x) + IntExpr(1)) + WhileStmt:3( + IntExpr(2) + Block:4( + PassStmt:4())) + AssignmentStmt:5( + NameExpr(y) + IntExpr(2))))) + +[case testSimpleClass] +class A: + def f(self): + pass +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + f + Args( + Var(self)) + Block:3( + PassStmt:3())))) + +[case testGlobalVarWithType] +x = 0 # type: int +y = False # type: bool +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(0) + int?) + AssignmentStmt:2( + NameExpr(y) + NameExpr(False) + bool?)) + +[case testLocalVarWithType] +def f(): + x = 0 # type: int + y = False # type: bool + a = None # type: Any +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + AssignmentStmt:2( + NameExpr(x) + IntExpr(0) + int?) + AssignmentStmt:3( + NameExpr(y) + NameExpr(False) + bool?) + AssignmentStmt:4( + NameExpr(a) + NameExpr(None) + Any?)))) + +[case testFunctionDefWithType] +def f(y: str) -> int: + return +class A: + def f(self, a: int, b: Any) -> x: + pass + def g(self) -> Any: + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(y)) + def (y: str?) -> int? + Block:2( + ReturnStmt:2())) + ClassDef:3( + A + FuncDef:4( + f + Args( + Var(self) + Var(a) + Var(b)) + def (self: Any, a: int?, b: Any?) -> x? + Block:5( + PassStmt:5())) + FuncDef:6( + g + Args( + Var(self)) + def (self: Any) -> Any? + Block:7( + PassStmt:7())))) + +[case testFuncWithNoneReturn] +def f() -> None: + pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> None? + Block:2( + PassStmt:2()))) + +[case testVarDefWithGenericType] +x = None # type: List[str] +y = None # type: Dict[int, Any] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + NameExpr(None) + List?[str?]) + AssignmentStmt:2( + NameExpr(y) + NameExpr(None) + Dict?[int?, Any?])) + +[case testSignatureWithGenericTypes] +def f(y: t[Any, x]) -> a[b[c], d]: + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(y)) + def (y: t?[Any?, x?]) -> a?[b?[c?], d?] + Block:2( + PassStmt:2()))) + +[case testParsingExpressionsWithLessAndGreaterThan] +# The operators < > can sometimes be confused with generic types. +x = a < b > c +f(x < b, y > c) +a < b > 1 +x < b, y > 2 +(a < b > c) +[out] +MypyFile:1( + AssignmentStmt:2( + NameExpr(x) + ComparisonExpr:2( + < + > + NameExpr(a) + NameExpr(b) + NameExpr(c))) + ExpressionStmt:3( + CallExpr:3( + NameExpr(f) + Args( + ComparisonExpr:3( + < + NameExpr(x) + NameExpr(b)) + ComparisonExpr:3( + > + NameExpr(y) + NameExpr(c))))) + ExpressionStmt:4( + ComparisonExpr:4( + < + > + NameExpr(a) + NameExpr(b) + IntExpr(1))) + ExpressionStmt:5( + TupleExpr:5( + ComparisonExpr:5( + < + NameExpr(x) + NameExpr(b)) + ComparisonExpr:5( + > + NameExpr(y) + IntExpr(2)))) + ExpressionStmt:6( + ComparisonExpr:6( + < + > + NameExpr(a) + NameExpr(b) + NameExpr(c)))) + +[case testLineContinuation] +if (1 + + 2): + pass +[out] +MypyFile:1( + IfStmt:1( + If( + OpExpr:1( + + + IntExpr(1) + IntExpr(2))) + Then( + PassStmt:3()))) + +[case testMultipleVarDef] +x, y = z # type: int, a[c] +[out] +MypyFile:1( + AssignmentStmt:1( + TupleExpr:1( + NameExpr(x) + NameExpr(y)) + NameExpr(z) + Tuple[int?, a?[c?]])) + +[case testMultipleVarDef2] +(xx, z, i) = 1 # type: (a[c], Any, int) +[out] +MypyFile:1( + AssignmentStmt:1( + TupleExpr:1( + NameExpr(xx) + NameExpr(z) + NameExpr(i)) + IntExpr(1) + Tuple[a?[c?], Any?, int?])) + +[case testMultipleVarDef3] +(xx, (z, i)) = 1 # type: (a[c], (Any, int)) +[out] +MypyFile:1( + AssignmentStmt:1( + TupleExpr:1( + NameExpr(xx) + TupleExpr:1( + NameExpr(z) + NameExpr(i))) + IntExpr(1) + Tuple[a?[c?], Tuple[Any?, int?]])) + +[case testAnnotateAssignmentViaSelf] +class A: + def __init__(self): + self.x = 1 # type: int +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + __init__ + Args( + Var(self)) + Block:3( + AssignmentStmt:3( + MemberExpr:3( + NameExpr(self) + x) + IntExpr(1) + int?))))) + +[case testCommentAfterTypeComment] +x = 0 # type: int # bar! +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(0) + int?)) + +[case testMultilineAssignmentAndAnnotations] +(x, + y) = (1, + 2) # type: foo, bar +[out] +MypyFile:1( + AssignmentStmt:1( + TupleExpr:1( + NameExpr(x) + NameExpr(y)) + TupleExpr:2( + IntExpr(1) + IntExpr(2)) + Tuple[foo?, bar?])) + +[case testWhitespaceAndCommentAnnotation] +x = 1#type:int +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1) + int?)) + +[case testWhitespaceAndCommentAnnotation2] +x = 1# type: int +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1) + int?)) + +[case testWhitespaceAndCommentAnnotation3] +x = 1# type : int # not recognized! +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1))) + +[case testInvalidAnnotation] +x=1 ##type: int +y=1 #.type: int +z=1 # Type: int +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1)) + AssignmentStmt:2( + NameExpr(y) + IntExpr(1)) + AssignmentStmt:3( + NameExpr(z) + IntExpr(1))) + +[case testEmptyClass] +class C: + pass +[out] +MypyFile:1( + ClassDef:1( + C + PassStmt:2())) + +[case testOperatorPrecedence] +a | b ^ c +a & b << c +[out] +MypyFile:1( + ExpressionStmt:1( + OpExpr:1( + | + NameExpr(a) + OpExpr:1( + ^ + NameExpr(b) + NameExpr(c)))) + ExpressionStmt:2( + OpExpr:2( + & + NameExpr(a) + OpExpr:2( + << + NameExpr(b) + NameExpr(c))))) + +[case testOperatorAssociativity] +1 - 2 + 3 +1 << 2 << 3 +[out] +MypyFile:1( + ExpressionStmt:1( + OpExpr:1( + + + OpExpr:1( + - + IntExpr(1) + IntExpr(2)) + IntExpr(3))) + ExpressionStmt:2( + OpExpr:2( + << + OpExpr:2( + << + IntExpr(1) + IntExpr(2)) + IntExpr(3)))) + +[case testUnaryOperators] +-2 * +3 * ~3 * 2 +~3**2 +[out] +MypyFile:1( + ExpressionStmt:1( + OpExpr:1( + * + OpExpr:1( + * + OpExpr:1( + * + UnaryExpr:1( + - + IntExpr(2)) + UnaryExpr:1( + + + IntExpr(3))) + UnaryExpr:1( + ~ + IntExpr(3))) + IntExpr(2))) + ExpressionStmt:2( + UnaryExpr:2( + ~ + OpExpr:2( + ** + IntExpr(3) + IntExpr(2))))) + +[case testSingleLineBodies] +if 1: pass +while 1: pass +def f(): pass +def g() -> int: return 1 +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + PassStmt:1())) + WhileStmt:2( + IntExpr(1) + Block:2( + PassStmt:2())) + FuncDef:3( + f + Block:3( + PassStmt:3())) + FuncDef:4( + g + def () -> int? + Block:4( + ReturnStmt:4( + IntExpr(1))))) + +[case testForStatement] +for x in y: + pass +for x, (y, w) in z: + 1 +for [x, (y, w)] in z: + 1 +[out] +MypyFile:1( + ForStmt:1( + NameExpr(x) + NameExpr(y) + Block:2( + PassStmt:2())) + ForStmt:3( + TupleExpr:3( + NameExpr(x) + TupleExpr:3( + NameExpr(y) + NameExpr(w))) + NameExpr(z) + Block:4( + ExpressionStmt:4( + IntExpr(1)))) + ForStmt:5( + TupleExpr:5( + NameExpr(x) + TupleExpr:5( + NameExpr(y) + NameExpr(w))) + NameExpr(z) + Block:6( + ExpressionStmt:6( + IntExpr(1))))) + +[case testGlobalDecl] +global x +def f(): + global x, y +[out] +MypyFile:1( + GlobalDecl:1( + x) + FuncDef:2( + f + Block:3( + GlobalDecl:3( + x + y)))) + +[case testNonlocalDecl] +def f(): + def g(): + nonlocal x, y +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + FuncDef:2( + g + Block:3( + NonlocalDecl:3( + x + y)))))) + +[case testRaiseStatement] +raise foo +[out] +MypyFile:1( + RaiseStmt:1( + NameExpr(foo))) + +[case testRaiseWithoutArg] +try: + pass +except: + raise +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + Block:4( + RaiseStmt:4()))) + +[case testRaiseFrom] +raise e from x +[out] +MypyFile:1( + RaiseStmt:1( + NameExpr(e) + NameExpr(x))) + +[case testBaseclasses] +class A(B): + pass +class A(B[T], C[Any, d[x]]): + pass +[out] +MypyFile:1( + ClassDef:1( + A + BaseTypeExpr( + NameExpr(B)) + PassStmt:2()) + ClassDef:3( + A + BaseTypeExpr( + IndexExpr:3( + NameExpr(B) + NameExpr(T)) + IndexExpr:3( + NameExpr(C) + TupleExpr:3( + NameExpr(Any) + IndexExpr:3( + NameExpr(d) + NameExpr(x))))) + PassStmt:4())) + +[case testIsNot] +x is not y +[out] +MypyFile:1( + ExpressionStmt:1( + ComparisonExpr:1( + is not + NameExpr(x) + NameExpr(y)))) + +[case testNotIn] +x not in y +not x not in y +x not in y | z +[out] +MypyFile:1( + ExpressionStmt:1( + ComparisonExpr:1( + not in + NameExpr(x) + NameExpr(y))) + ExpressionStmt:2( + UnaryExpr:2( + not + ComparisonExpr:2( + not in + NameExpr(x) + NameExpr(y)))) + ExpressionStmt:3( + ComparisonExpr:3( + not in + NameExpr(x) + OpExpr:3( + | + NameExpr(y) + NameExpr(z))))) + +[case testNotAsBinaryOp] +x not y +[out] +main:1: error: invalid syntax +[out version==3.10.0] +main:1: error: invalid syntax. Perhaps you forgot a comma? + +[case testNotIs] +x not is y # E: invalid syntax +[out] + +[case testBinaryNegAsBinaryOp] +1 ~ 2 +[out] +main:1: error: invalid syntax +[out version==3.10.0] +main:1: error: invalid syntax. Perhaps you forgot a comma? + +[case testSliceInList39] +# flags: --python-version 3.9 +x = [1, 2][1:2] +[out] +MypyFile:1( + AssignmentStmt:2( + NameExpr(x) + IndexExpr:2( + ListExpr:2( + IntExpr(1) + IntExpr(2)) + SliceExpr:2( + IntExpr(1) + IntExpr(2))))) + +[case testDictionaryExpression] +{} +{1:x} +{1:x, 2 or 1:2 and 3} +[out] +MypyFile:1( + ExpressionStmt:1( + DictExpr:1()) + ExpressionStmt:2( + DictExpr:2( + IntExpr(1) + NameExpr(x))) + ExpressionStmt:3( + DictExpr:3( + IntExpr(1) + NameExpr(x) + OpExpr:3( + or + IntExpr(2) + IntExpr(1)) + OpExpr:3( + and + IntExpr(2) + IntExpr(3))))) + +[case testImport] +import x +import y.z.foo, __foo__.bar +[out] +MypyFile:1( + Import:1(x) + Import:2(y.z.foo, __foo__.bar)) + +[case testVariableTypeWithQualifiedName] +x = None # type: x.y +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + NameExpr(None) + x.y?)) + +[case testTypeInSignatureWithQualifiedName] +def f() -> x.y[a.b.c]: pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> x.y?[a.b.c?] + Block:1( + PassStmt:1()))) + +[case testImportFrom] +from m import x +from m.n import x, y, z +[out] +MypyFile:1( + ImportFrom:1(m, [x]) + ImportFrom:2(m.n, [x, y, z])) + +[case testImportFromAs] +from m import x as y +from x import y, z as a, c as c +[out] +MypyFile:1( + ImportFrom:1(m, [x : y]) + ImportFrom:2(x, [y, z : a, c : c])) + +[case testImportStar] +from x import * +[out] +MypyFile:1( + ImportAll:1(x)) + +[case testImportsInDifferentPlaces] +1 +import x +def f(): + from x import y + from z import * +[out] +MypyFile:1( + ExpressionStmt:1( + IntExpr(1)) + Import:2(x) + FuncDef:3( + f + Block:4( + ImportFrom:4(x, [y]) + ImportAll:5(z)))) + +[case testImportWithExtraComma] +from x import (y, z,) +[out] +MypyFile:1( + ImportFrom:1(x, [y, z])) + +[case testDefaultArgs] +def f(x=1): + pass +def g(x, y=1+2, z=(1, 2)): + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + default( + Var(x) + IntExpr(1))) + Block:2( + PassStmt:2())) + FuncDef:3( + g + Args( + Var(x) + default( + Var(y) + OpExpr:3( + + + IntExpr(1) + IntExpr(2))) + default( + Var(z) + TupleExpr:3( + IntExpr(1) + IntExpr(2)))) + Block:4( + PassStmt:4()))) + +[case testTryFinally] +try: + 1 +finally: + 2 +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + IntExpr(1))) + Finally( + ExpressionStmt:4( + IntExpr(2))))) + +[case testTry] +try: + 1 +except x: + 2 +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + IntExpr(1))) + NameExpr(x) + Block:4( + ExpressionStmt:4( + IntExpr(2))))) + +[case testComplexTry] +try: + 1 +except x as y: + 2 +except x.y: + 3 +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + IntExpr(1))) + NameExpr(x) + NameExpr(y) + Block:4( + ExpressionStmt:4( + IntExpr(2))) + MemberExpr:5( + NameExpr(x) + y) + Block:6( + ExpressionStmt:6( + IntExpr(3))))) + +[case testGeneratorExpression] +(x for y in z) +[out] +MypyFile:1( + ExpressionStmt:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(y) + NameExpr(z)))) + +[case testGeneratorExpressionNested] +(x for y, (p, q) in z) +[out] +MypyFile:1( + ExpressionStmt:1( + GeneratorExpr:1( + NameExpr(x) + TupleExpr:1( + NameExpr(y) + TupleExpr:1( + NameExpr(p) + NameExpr(q))) + NameExpr(z)))) + +[case testListComprehension] +x=[x for y in z] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListComprehension:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(y) + NameExpr(z))))) + +[case testComplexListComprehension] +x=[(x, y) for y, z in (1, 2)] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListComprehension:1( + GeneratorExpr:1( + TupleExpr:1( + NameExpr(x) + NameExpr(y)) + TupleExpr:1( + NameExpr(y) + NameExpr(z)) + TupleExpr:1( + IntExpr(1) + IntExpr(2)))))) + +[case testListComprehension2] +([x + 1 for x in a]) +[out] +MypyFile:1( + ExpressionStmt:1( + ListComprehension:1( + GeneratorExpr:1( + OpExpr:1( + + + NameExpr(x) + IntExpr(1)) + NameExpr(x) + NameExpr(a))))) + +[case testSlices] +x[1:2] +x[:1] +x[1:] +x[:] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(x) + SliceExpr:1( + IntExpr(1) + IntExpr(2)))) + ExpressionStmt:2( + IndexExpr:2( + NameExpr(x) + SliceExpr:2( + + IntExpr(1)))) + ExpressionStmt:3( + IndexExpr:3( + NameExpr(x) + SliceExpr:3( + IntExpr(1) + ))) + ExpressionStmt:4( + IndexExpr:4( + NameExpr(x) + SliceExpr:4( + + )))) + +[case testSliceWithStride] +x[1:2:3] +x[1::2] +x[:1:2] +x[::2] +x[1:2:] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(x) + SliceExpr:1( + IntExpr(1) + IntExpr(2) + IntExpr(3)))) + ExpressionStmt:2( + IndexExpr:2( + NameExpr(x) + SliceExpr:2( + IntExpr(1) + + IntExpr(2)))) + ExpressionStmt:3( + IndexExpr:3( + NameExpr(x) + SliceExpr:3( + + IntExpr(1) + IntExpr(2)))) + ExpressionStmt:4( + IndexExpr:4( + NameExpr(x) + SliceExpr:4( + + + IntExpr(2)))) + ExpressionStmt:5( + IndexExpr:5( + NameExpr(x) + SliceExpr:5( + IntExpr(1) + IntExpr(2))))) + +[case testYield] +def f(): + yield x + 1 +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ExpressionStmt:2( + YieldExpr:2( + OpExpr:2( + + + NameExpr(x) + IntExpr(1))))))) + +[case testYieldFrom] +def f(): + yield from h() +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ExpressionStmt:2( + YieldFromExpr:2( + CallExpr:2( + NameExpr(h) + Args())))))) + +[case testYieldFromAssignment] +def f(): + a = yield from h() +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + AssignmentStmt:2( + NameExpr(a) + YieldFromExpr:2( + CallExpr:2( + NameExpr(h) + Args())))))) + +[case testDel] +del x +del x[0], y[1] +[out] +MypyFile:1( + DelStmt:1( + NameExpr(x)) + DelStmt:2( + TupleExpr:2( + IndexExpr:2( + NameExpr(x) + IntExpr(0)) + IndexExpr:2( + NameExpr(y) + IntExpr(1))))) + +[case testExtraCommas] +1, 2, ++[1, 2,] +f(1,) +{1:2,} +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1( + IntExpr(1) + IntExpr(2))) + ExpressionStmt:2( + UnaryExpr:2( + + + ListExpr:2( + IntExpr(1) + IntExpr(2)))) + ExpressionStmt:3( + CallExpr:3( + NameExpr(f) + Args( + IntExpr(1)))) + ExpressionStmt:4( + DictExpr:4( + IntExpr(1) + IntExpr(2)))) + +[case testExtraCommaInFunc] +def f(x,): + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + Block:2( + PassStmt:2()))) + +[case testLambda] +lambda: 1 +lambda x: y + 1 +lambda x, y: 1 +[out] +MypyFile:1( + ExpressionStmt:1( + LambdaExpr:1( + Block:1( + ReturnStmt:1( + IntExpr(1))))) + ExpressionStmt:2( + LambdaExpr:2( + Args( + Var(x)) + Block:2( + ReturnStmt:2( + OpExpr:2( + + + NameExpr(y) + IntExpr(1)))))) + ExpressionStmt:3( + LambdaExpr:3( + Args( + Var(x) + Var(y)) + Block:3( + ReturnStmt:3( + IntExpr(1)))))) + +[case testComplexLambda] +lambda x=2: x +[out] +MypyFile:1( + ExpressionStmt:1( + LambdaExpr:1( + Args( + default( + Var(x) + IntExpr(2))) + Block:1( + ReturnStmt:1( + NameExpr(x)))))) + +[case testLambdaPrecedence] +lambda x: 1, 2 +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1( + LambdaExpr:1( + Args( + Var(x)) + Block:1( + ReturnStmt:1( + IntExpr(1)))) + IntExpr(2)))) + +[case testForIndicesInParens] +for (i, j) in x: + pass +[out] +MypyFile:1( + ForStmt:1( + TupleExpr:1( + NameExpr(i) + NameExpr(j)) + NameExpr(x) + Block:2( + PassStmt:2()))) + +[case testForAndTrailingCommaAfterIndexVar] +for i, in x: + pass +[out] +MypyFile:1( + ForStmt:1( + TupleExpr:1( + NameExpr(i)) + NameExpr(x) + Block:2( + PassStmt:2()))) + +[case testListComprehensionAndTrailingCommaAfterIndexVar] +x = [a for b, in c] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListComprehension:1( + GeneratorExpr:1( + NameExpr(a) + TupleExpr:1( + NameExpr(b)) + NameExpr(c))))) + +[case testForAndTrailingCommaAfterIndexVars] +for i, j, in x: + pass +[out] +MypyFile:1( + ForStmt:1( + TupleExpr:1( + NameExpr(i) + NameExpr(j)) + NameExpr(x) + Block:2( + PassStmt:2()))) + +[case testGeneratorWithCondition] +(x for y in z if 0) +[out] +MypyFile:1( + ExpressionStmt:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(y) + NameExpr(z) + IntExpr(0)))) + +[case testListComprehensionWithCondition] +raise [x for y in z if 0] +[out] +MypyFile:1( + RaiseStmt:1( + ListComprehension:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(y) + NameExpr(z) + IntExpr(0))))) + +[case testListComprehensionWithConditions] +raise [x for y in z if 0 if 1] +[out] +MypyFile:1( + RaiseStmt:1( + ListComprehension:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(y) + NameExpr(z) + IntExpr(0) + IntExpr(1))))) + +[case testListComprehensionWithCrazyConditions] +raise [x for y in z if (1 if 2 else 3) if 1] +[out] +MypyFile:1( + RaiseStmt:1( + ListComprehension:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(y) + NameExpr(z) + ConditionalExpr:1( + Condition( + IntExpr(2)) + IntExpr(1) + IntExpr(3)) + IntExpr(1))))) + +[case testDictionaryComprehension] +a = {x: y for x, y in xys} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + DictionaryComprehension:1( + NameExpr(x) + NameExpr(y) + TupleExpr:1( + NameExpr(x) + NameExpr(y)) + NameExpr(xys)))) + +[case testDictionaryComprehensionComplex] +a = {x: y for x, y in xys for p, q in pqs if c} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + DictionaryComprehension:1( + NameExpr(x) + NameExpr(y) + TupleExpr:1( + NameExpr(x) + NameExpr(y)) + TupleExpr:1( + NameExpr(p) + NameExpr(q)) + NameExpr(xys) + NameExpr(pqs) + NameExpr(c)))) + +[case testSetComprehension] +a = {i for i in l} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + SetComprehension:1( + GeneratorExpr:1( + NameExpr(i) + NameExpr(i) + NameExpr(l))))) + +[case testSetComprehensionComplex] +a = {x + p for x in xys for p in pqs if c} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + SetComprehension:1( + GeneratorExpr:1( + OpExpr:1( + + + NameExpr(x) + NameExpr(p)) + NameExpr(x) + NameExpr(p) + NameExpr(xys) + NameExpr(pqs) + NameExpr(c))))) + +[case testWithStatement] +with open('foo') as f: + pass +[out] +MypyFile:1( + WithStmt:1( + Expr( + CallExpr:1( + NameExpr(open) + Args( + StrExpr(foo)))) + Target( + NameExpr(f)) + Block:2( + PassStmt:2()))) + +[case testWithStatementWithoutTarget] +with foo: + pass +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(foo)) + Block:2( + PassStmt:2()))) + +[case testHexOctBinLiterals] +0xa, 0Xaf, 0o7, 0O12, 0b1, 0B101 +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1( + IntExpr(10) + IntExpr(175) + IntExpr(7) + IntExpr(10) + IntExpr(1) + IntExpr(5)))) + +[case testImportFromWithParens] +from x import (y) +from x import (y, + z) +[out] +MypyFile:1( + ImportFrom:1(x, [y]) + ImportFrom:2(x, [y, z])) + +[case testContinueStmt] +while 1: + continue +[out] +MypyFile:1( + WhileStmt:1( + IntExpr(1) + Block:2( + ContinueStmt:2()))) + +[case testStrLiteralConcatenate] +'f' 'bar' +('x' + 'y' + 'z') +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr(fbar)) + ExpressionStmt:2( + StrExpr(xyz))) + +[case testCatchAllExcept] +try: + 1 +except: + pass +try: + 1 +except x: + pass +except: + 2 +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + IntExpr(1))) + Block:4( + PassStmt:4())) + TryStmt:5( + Block:6( + ExpressionStmt:6( + IntExpr(1))) + NameExpr(x) + Block:8( + PassStmt:8()) + Block:10( + ExpressionStmt:10( + IntExpr(2))))) + +[case testTryElse] +try: + pass +except x: + 1 +else: + 2 +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + NameExpr(x) + Block:4( + ExpressionStmt:4( + IntExpr(1))) + Else( + ExpressionStmt:6( + IntExpr(2))))) + +[case testExceptWithMultipleTypes] +try: + pass +except (x, y): + pass +except (a, b, c) as e: + pass +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + TupleExpr:3( + NameExpr(x) + NameExpr(y)) + Block:4( + PassStmt:4()) + TupleExpr:5( + NameExpr(a) + NameExpr(b) + NameExpr(c)) + NameExpr(e) + Block:6( + PassStmt:6()))) + +[case testNestedFunctions] +def f(): + def g(): + pass +def h() -> int: + def g() -> int: + pass +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + FuncDef:2( + g + Block:3( + PassStmt:3())))) + FuncDef:4( + h + def () -> int? + Block:5( + FuncDef:5( + g + def () -> int? + Block:6( + PassStmt:6()))))) + +[case testStatementsAndDocStringsInClassBody] +class A: + "doc string" + x = y + def f(self): + pass +[out] +MypyFile:1( + ClassDef:1( + A + ExpressionStmt:2( + StrExpr(doc string)) + AssignmentStmt:3( + NameExpr(x) + NameExpr(y)) + FuncDef:4( + f + Args( + Var(self)) + Block:5( + PassStmt:5())))) + +[case testSingleLineClass] +class a: pass +[out] +MypyFile:1( + ClassDef:1( + a + PassStmt:1())) + +[case testDecorator] +@property +def f(): + pass +[out] +MypyFile:1( + Decorator:1( + Var(f) + NameExpr(property) + FuncDef:2( + f + Block:3( + PassStmt:3())))) + +[case testComplexDecorator] +@foo(bar, 1) +@zar +def f() -> int: + pass +[out] +MypyFile:1( + Decorator:1( + Var(f) + CallExpr:1( + NameExpr(foo) + Args( + NameExpr(bar) + IntExpr(1))) + NameExpr(zar) + FuncDef:3( + f + def () -> int? + Block:4( + PassStmt:4())))) + +[case testKeywordArgInCall] +f(x=1) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args() + KwArgs( + x + IntExpr(1))))) + +[case testComplexKeywordArgs] +f(x, y=1 or 2, z=y) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args( + NameExpr(x)) + KwArgs( + y + OpExpr:1( + or + IntExpr(1) + IntExpr(2))) + KwArgs( + z + NameExpr(y))))) + +[case testChainedAssignment] +x = z = 1 +[out] +MypyFile:1( + AssignmentStmt:1( + Lvalues( + NameExpr(x) + NameExpr(z)) + IntExpr(1))) + +[case testVarArgs] +def f(x, *a): pass +f(1, *2) +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + VarArg( + Var(a)) + Block:1( + PassStmt:1())) + ExpressionStmt:2( + CallExpr:2( + NameExpr(f) + Args( + IntExpr(1) + IntExpr(2)) + VarArg))) + +[case testVarArgWithType] +def f(x: str, *a: int): pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + def (x: str?, *a: int?) -> Any + VarArg( + Var(a)) + Block:1( + PassStmt:1()))) + +[case testDictVarArgs] +def f(x, **a): pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + DictVarArg( + Var(a)) + Block:1( + PassStmt:1()))) + +[case testBothVarArgs] +def f(x, *a, **b): pass +def g(*a, **b): pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + VarArg( + Var(a)) + DictVarArg( + Var(b)) + Block:1( + PassStmt:1())) + FuncDef:2( + g + VarArg( + Var(a)) + DictVarArg( + Var(b)) + Block:2( + PassStmt:2()))) + +[case testDictVarArgsWithType] +def f(x: X, **a: A) -> None: pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + def (x: X?, **a: A?) -> None? + DictVarArg( + Var(a)) + Block:1( + PassStmt:1()))) + +[case testCallDictVarArgs] +f(**x) +f(x, **y) +f(*x, **y) +f(x, *y, **z) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args() + DictVarArg( + NameExpr(x)))) + ExpressionStmt:2( + CallExpr:2( + NameExpr(f) + Args( + NameExpr(x)) + DictVarArg( + NameExpr(y)))) + ExpressionStmt:3( + CallExpr:3( + NameExpr(f) + Args( + NameExpr(x)) + VarArg + DictVarArg( + NameExpr(y)))) + ExpressionStmt:4( + CallExpr:4( + NameExpr(f) + Args( + NameExpr(x) + NameExpr(y)) + VarArg + DictVarArg( + NameExpr(z))))) + +[case testAssert] +assert x == y +[out] +MypyFile:1( + AssertStmt:1( + ComparisonExpr:1( + == + NameExpr(x) + NameExpr(y)))) + +[case testYieldWithoutExpressions] +def f(): + yield +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ExpressionStmt:2( + YieldExpr:2())))) + +[case testConditionalExpression] +x if y else z +[out] +MypyFile:1( + ExpressionStmt:1( + ConditionalExpr:1( + Condition( + NameExpr(y)) + NameExpr(x) + NameExpr(z)))) + +[case testConditionalExpressionInListComprehension] +a = [x if y else z for a in b] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + ListComprehension:1( + GeneratorExpr:1( + ConditionalExpr:1( + Condition( + NameExpr(y)) + NameExpr(x) + NameExpr(z)) + NameExpr(a) + NameExpr(b))))) + +[case testConditionalExpressionInTuple] +1 if 2 else 3, 4 +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1( + ConditionalExpr:1( + Condition( + IntExpr(2)) + IntExpr(1) + IntExpr(3)) + IntExpr(4)))) + +[case testSetLiteral] +{x or y} +{1, 2} +[out] +MypyFile:1( + ExpressionStmt:1( + SetExpr:1( + OpExpr:1( + or + NameExpr(x) + NameExpr(y)))) + ExpressionStmt:2( + SetExpr:2( + IntExpr(1) + IntExpr(2)))) + +[case testSetLiteralWithExtraComma] +{x,} +[out] +MypyFile:1( + ExpressionStmt:1( + SetExpr:1( + NameExpr(x)))) + +[case testImportAs] +import x as y +import x, z as y, a.b as c, d as d +[out] +MypyFile:1( + Import:1(x : y) + Import:2(x, z : y, a.b : c, d : d)) + +[case testForAndElse] +for x in y: + pass +else: + x +[out] +MypyFile:1( + ForStmt:1( + NameExpr(x) + NameExpr(y) + Block:2( + PassStmt:2()) + Else( + ExpressionStmt:4( + NameExpr(x))))) + +[case testWhileAndElse] +while x: + pass +else: + y +[out] +MypyFile:1( + WhileStmt:1( + NameExpr(x) + Block:2( + PassStmt:2()) + Else( + ExpressionStmt:4( + NameExpr(y))))) + +[case testWithAndMultipleOperands] +with x as y, a as b: + pass +with x(), y(): + pass +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(x)) + Target( + NameExpr(y)) + Expr( + NameExpr(a)) + Target( + NameExpr(b)) + Block:2( + PassStmt:2())) + WithStmt:3( + Expr( + CallExpr:3( + NameExpr(x) + Args())) + Expr( + CallExpr:3( + NameExpr(y) + Args())) + Block:4( + PassStmt:4()))) + +[case testOperatorAssignment] +x += 1 +x -= 1 +x *= 1 +x /= 1 +x //= 1 +x %= 1 +x **= 1 +x |= 1 +x &= 1 +x ^= 1 +x >>= 1 +x <<= 1 +[out] +MypyFile:1( + OperatorAssignmentStmt:1( + + + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:2( + - + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:3( + * + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:4( + / + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:5( + // + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:6( + % + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:7( + ** + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:8( + | + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:9( + & + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:10( + ^ + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:11( + >> + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:12( + << + NameExpr(x) + IntExpr(1))) + +[case testNestedClasses] +class A: + class B: + pass + class C: + pass +[out] +MypyFile:1( + ClassDef:1( + A + ClassDef:2( + B + PassStmt:3()) + ClassDef:4( + C + PassStmt:5()))) + +[case testTryWithExceptAndFinally] +try: + pass +except x: + x +finally: + y +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + NameExpr(x) + Block:4( + ExpressionStmt:4( + NameExpr(x))) + Finally( + ExpressionStmt:6( + NameExpr(y))))) + +[case testBareAsteriskInFuncDef] +def f(x, *, y=1): pass +[out] +MypyFile:1( + FuncDef:1( + f + MaxPos(1) + Args( + Var(x) + default( + Var(y) + IntExpr(1))) + Block:1( + PassStmt:1()))) + +[case testBareAsteriskInFuncDefWithSignature] +def f(x: A, *, y: B = 1) -> None: pass +[out] +MypyFile:1( + FuncDef:1( + f + MaxPos(1) + Args( + Var(x) + default( + Var(y) + IntExpr(1))) + def (x: A?, *, y: B? =) -> None? + Block:1( + PassStmt:1()))) + +[case testBareAsteriskNamedDefault] +def f(*, y: B = 1) -> None: pass +[out] +MypyFile:1( + FuncDef:1( + f + MaxPos(0) + Args( + default( + Var(y) + IntExpr(1))) + def (*, y: B? =) -> None? + Block:1( + PassStmt:1()))) + +[case testBareAsteriskNamedNoDefault] +def f(*, y: B) -> None: pass +[out] +MypyFile:1( + FuncDef:1( + f + MaxPos(0) + Args( + Var(y)) + def (*, y: B?) -> None? + Block:1( + PassStmt:1()))) + +[case testSuperExpr] +super().x +[out] +MypyFile:1( + ExpressionStmt:1( + SuperExpr:1( + x + CallExpr:1( + NameExpr(super) + Args())))) + +[case testKeywordAndDictArgs] +f(x = y, **kwargs) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args() + KwArgs( + x + NameExpr(y)) + DictVarArg( + NameExpr(kwargs))))) + +[case testSimpleFunctionType] +f = None # type: Callable[[], None] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(f) + NameExpr(None) + Callable?[, None?])) + +[case testFunctionTypeWithArgument] +f = None # type: Callable[[str], int] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(f) + NameExpr(None) + Callable?[, int?])) + +[case testFunctionTypeWithTwoArguments] +f = None # type: Callable[[a[b], x.y], List[int]] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(f) + NameExpr(None) + Callable?[, List?[int?]])) + +[case testFunctionTypeWithExtraComma] +def f(x: Callable[[str,], int]): pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + def (x: Callable?[, int?]) -> Any + Block:1( + PassStmt:1()))) + +[case testSimpleStringLiteralType] +def f() -> 'A': pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> A? + Block:1( + PassStmt:1()))) + +[case testGenericStringLiteralType] +def f() -> 'A[B, C]': pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> A?[B?, C?] + Block:1( + PassStmt:1()))) + +[case testPartialStringLiteralType] +def f() -> A['B', C]: pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> A?[B?, C?] + Block:1( + PassStmt:1()))) + +[case testWhitespaceInStringLiteralType] +def f() -> ' A [ X ] ': pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> A?[X?] + Block:1( + PassStmt:1()))) + +[case testEscapeInStringLiteralType] +def f() -> '\x41': pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> A? + Block:1( + PassStmt:1()))) + +[case testMetaclass] +class Foo(metaclass=Bar): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + Metaclass(NameExpr(Bar)) + PassStmt:1())) + +[case testQualifiedMetaclass] +class Foo(metaclass=foo.Bar): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + Metaclass(MemberExpr:1( + NameExpr(foo) + Bar)) + PassStmt:1())) + +[case testBaseAndMetaclass] +class Foo(foo.bar[x], metaclass=Bar): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + Metaclass(NameExpr(Bar)) + BaseTypeExpr( + IndexExpr:1( + MemberExpr:1( + NameExpr(foo) + bar) + NameExpr(x))) + PassStmt:1())) + +[case testClassKeywordArgs] +class Foo(_root=None): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + PassStmt:1())) + +[case testClassKeywordArgsBeforeMeta] +class Foo(_root=None, metaclass=Bar): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + Metaclass(NameExpr(Bar)) + PassStmt:1())) + +[case testClassKeywordArgsAfterMeta] +class Foo(metaclass=Bar, _root=None): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + Metaclass(NameExpr(Bar)) + PassStmt:1())) + +[case testNamesThatAreNoLongerKeywords] +any = interface +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(any) + NameExpr(interface))) + +[case testFunctionOverload] +@overload +def f() -> x: pass +@overload +def f() -> y: pass +[out] +MypyFile:1( + OverloadedFuncDef:1( + Decorator:1( + Var(f) + NameExpr(overload) + FuncDef:2( + f + def () -> x? + Block:2( + PassStmt:2()))) + Decorator:3( + Var(f) + NameExpr(overload) + FuncDef:4( + f + def () -> y? + Block:4( + PassStmt:4()))))) + +[case testFunctionOverloadAndOtherStatements] +x +@overload +def f() -> x: pass +@overload +def f() -> y: pass +x +[out] +MypyFile:1( + ExpressionStmt:1( + NameExpr(x)) + OverloadedFuncDef:2( + Decorator:2( + Var(f) + NameExpr(overload) + FuncDef:3( + f + def () -> x? + Block:3( + PassStmt:3()))) + Decorator:4( + Var(f) + NameExpr(overload) + FuncDef:5( + f + def () -> y? + Block:5( + PassStmt:5())))) + ExpressionStmt:6( + NameExpr(x))) + +[case testFunctionOverloadWithThreeVariants] +@overload +def f() -> x: pass +@overload +def f() -> y: pass +@overload +def f(y): pass +[out] +MypyFile:1( + OverloadedFuncDef:1( + Decorator:1( + Var(f) + NameExpr(overload) + FuncDef:2( + f + def () -> x? + Block:2( + PassStmt:2()))) + Decorator:3( + Var(f) + NameExpr(overload) + FuncDef:4( + f + def () -> y? + Block:4( + PassStmt:4()))) + Decorator:5( + Var(f) + NameExpr(overload) + FuncDef:6( + f + Args( + Var(y)) + Block:6( + PassStmt:6()))))) + +[case testDecoratorsThatAreNotOverloads] +@foo +def f() -> x: pass +@foo +def g() -> y: pass +[out] +MypyFile:1( + Decorator:1( + Var(f) + NameExpr(foo) + FuncDef:2( + f + def () -> x? + Block:2( + PassStmt:2()))) + Decorator:3( + Var(g) + NameExpr(foo) + FuncDef:4( + g + def () -> y? + Block:4( + PassStmt:4())))) + +[case testFunctionOverloadWithinFunction] +def f(): + @overload + def g(): pass + @overload + def g() -> x: pass +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + OverloadedFuncDef:2( + Decorator:2( + Var(g) + NameExpr(overload) + FuncDef:3( + g + Block:3( + PassStmt:3()))) + Decorator:4( + Var(g) + NameExpr(overload) + FuncDef:5( + g + def () -> x? + Block:5( + PassStmt:5()))))))) + +[case testCommentFunctionAnnotation] +def f(): # type: () -> A + pass +def g(x): # type: (A) -> B + pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> A? + Block:2( + PassStmt:2())) + FuncDef:3( + g + Args( + Var(x)) + def (x: A?) -> B? + Block:4( + PassStmt:4()))) + +[case testCommentMethodAnnotation] +class A: + def f(self): # type: () -> A + pass + def g(xself, x): # type: (A) -> B + pass +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + f + Args( + Var(self)) + def (self: Any) -> A? + Block:3( + PassStmt:3())) + FuncDef:4( + g + Args( + Var(xself) + Var(x)) + def (xself: Any, x: A?) -> B? + Block:5( + PassStmt:5())))) + +[case testCommentMethodAnnotationAndNestedFunction] +class A: + def f(self): # type: () -> A + def g(x): # type: (A) -> B + pass +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + f + Args( + Var(self)) + def (self: Any) -> A? + Block:3( + FuncDef:3( + g + Args( + Var(x)) + def (x: A?) -> B? + Block:4( + PassStmt:4())))))) + +[case testCommentFunctionAnnotationOnSeparateLine] +def f(x): + # type: (X) -> Y + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + def (x: X?) -> Y? + Block:3( + PassStmt:3()))) + +[case testCommentFunctionAnnotationOnSeparateLine2] +def f(x): + + # type: (X) -> Y # bar + + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + def (x: X?) -> Y? + Block:5( + PassStmt:5()))) + +[case testCommentFunctionAnnotationAndVarArg] +def f(x, *y): # type: (X, *Y) -> Z + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + def (x: X?, *y: Y?) -> Z? + VarArg( + Var(y)) + Block:2( + PassStmt:2()))) + +[case testCommentFunctionAnnotationAndAllVarArgs] +def f(x, *y, **z): # type: (X, *Y, **Z) -> A + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + def (x: X?, *y: Y?, **z: Z?) -> A? + VarArg( + Var(y)) + DictVarArg( + Var(z)) + Block:2( + PassStmt:2()))) + +[case testClassDecorator] +@foo +class X: pass +@foo(bar) +@x.y +class Z: pass +[out] +MypyFile:1( + ClassDef:2( + X + Decorators( + NameExpr(foo)) + PassStmt:2()) + ClassDef:5( + Z + Decorators( + CallExpr:3( + NameExpr(foo) + Args( + NameExpr(bar))) + MemberExpr:4( + NameExpr(x) + y)) + PassStmt:5())) + +[case testTrailingSemicolon] +def x(): + pass; + +def y(): + pass +[out] +MypyFile:1( + FuncDef:1( + x + Block:2( + PassStmt:2())) + FuncDef:4( + y + Block:5( + PassStmt:5()))) + +[case testEmptySuperClass] +class A(): + pass +[out] +MypyFile:1( + ClassDef:1( + A + PassStmt:2())) + +[case testStarExpression] +*a +*a, b +a, *b +a, (*x, y) +a, (x, *y) +[out] +MypyFile:1( + ExpressionStmt:1( + StarExpr:1( + NameExpr(a))) + ExpressionStmt:2( + TupleExpr:2( + StarExpr:2( + NameExpr(a)) + NameExpr(b))) + ExpressionStmt:3( + TupleExpr:3( + NameExpr(a) + StarExpr:3( + NameExpr(b)))) + ExpressionStmt:4( + TupleExpr:4( + NameExpr(a) + TupleExpr:4( + StarExpr:4( + NameExpr(x)) + NameExpr(y)))) + ExpressionStmt:5( + TupleExpr:5( + NameExpr(a) + TupleExpr:5( + NameExpr(x) + StarExpr:5( + NameExpr(y)))))) + +[case testStarExpressionParenthesis] +*(a) +*(a,b) +[out] +MypyFile:1( + ExpressionStmt:1( + StarExpr:1( + NameExpr(a))) + ExpressionStmt:2( + StarExpr:2( + TupleExpr:2( + NameExpr(a) + NameExpr(b))))) + +[case testStarExpressionInFor] +for *a in b: + pass + +for a, *b in c: + pass + +for *a, b in c: + pass +[out] +MypyFile:1( + ForStmt:1( + StarExpr:1( + NameExpr(a)) + NameExpr(b) + Block:2( + PassStmt:2())) + ForStmt:4( + TupleExpr:4( + NameExpr(a) + StarExpr:4( + NameExpr(b))) + NameExpr(c) + Block:5( + PassStmt:5())) + ForStmt:7( + TupleExpr:7( + StarExpr:7( + NameExpr(a)) + NameExpr(b)) + NameExpr(c) + Block:8( + PassStmt:8()))) + +[case testStarExprInGeneratorExpr] +(x for y, *p in z) +(x for *p, y in z) +(x for y, *p, q in z) +[out] +MypyFile:1( + ExpressionStmt:1( + GeneratorExpr:1( + NameExpr(x) + TupleExpr:1( + NameExpr(y) + StarExpr:1( + NameExpr(p))) + NameExpr(z))) + ExpressionStmt:2( + GeneratorExpr:2( + NameExpr(x) + TupleExpr:2( + StarExpr:2( + NameExpr(p)) + NameExpr(y)) + NameExpr(z))) + ExpressionStmt:3( + GeneratorExpr:3( + NameExpr(x) + TupleExpr:3( + NameExpr(y) + StarExpr:3( + NameExpr(p)) + NameExpr(q)) + NameExpr(z)))) + +[case testParseNamedtupleBaseclass] +class A(namedtuple('x', ['y'])): pass +[out] +MypyFile:1( + ClassDef:1( + A + BaseTypeExpr( + CallExpr:1( + NameExpr(namedtuple) + Args( + StrExpr(x) + ListExpr:1( + StrExpr(y))))) + PassStmt:1())) + +[case testEllipsis] +... +a[1,...,2] +....__class__ +[out] +MypyFile:1( + ExpressionStmt:1( + Ellipsis) + ExpressionStmt:2( + IndexExpr:2( + NameExpr(a) + TupleExpr:2( + IntExpr(1) + Ellipsis + IntExpr(2)))) + ExpressionStmt:3( + MemberExpr:3( + Ellipsis + __class__))) + +[case testFunctionWithManyKindsOfArgs] +def f(x, *args, y=None, **kw): pass +[out] +MypyFile:1( + FuncDef:1( + f + MaxPos(1) + Args( + Var(x) + default( + Var(y) + NameExpr(None))) + VarArg( + Var(args)) + DictVarArg( + Var(kw)) + Block:1( + PassStmt:1()))) + +[case testIfWithSemicolons] +if 1: a; b +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + ExpressionStmt:1( + NameExpr(a)) + ExpressionStmt:1( + NameExpr(b))))) + +[case testIfWithSemicolonsNested] +while 2: + if 1: a; b +[out] +MypyFile:1( + WhileStmt:1( + IntExpr(2) + Block:2( + IfStmt:2( + If( + IntExpr(1)) + Then( + ExpressionStmt:2( + NameExpr(a)) + ExpressionStmt:2( + NameExpr(b))))))) + +[case testIfElseWithSemicolons] +if 1: global x; y = 1 +else: x = 1; return 3 +4 +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + GlobalDecl:1( + x) + AssignmentStmt:1( + NameExpr(y) + IntExpr(1))) + Else( + AssignmentStmt:2( + NameExpr(x) + IntExpr(1)) + ReturnStmt:2( + IntExpr(3)))) + ExpressionStmt:3( + IntExpr(4))) + +[case testIfElseWithSemicolonsNested] +while 2: + if 1: global x; y = 1 + else: x = 1; return 3 +4 +[out] +MypyFile:1( + WhileStmt:1( + IntExpr(2) + Block:2( + IfStmt:2( + If( + IntExpr(1)) + Then( + GlobalDecl:2( + x) + AssignmentStmt:2( + NameExpr(y) + IntExpr(1))) + Else( + AssignmentStmt:3( + NameExpr(x) + IntExpr(1)) + ReturnStmt:3( + IntExpr(3)))))) + ExpressionStmt:4( + IntExpr(4))) + +[case testKeywordArgumentAfterStarArgumentInCall] +f(x=1, *y) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args( + NameExpr(y)) + VarArg + KwArgs( + x + IntExpr(1))))) + +[case testConditionalExpressionInSetComprehension] +{ 1 if x else 2 for x in y } +[out] +MypyFile:1( + ExpressionStmt:1( + SetComprehension:1( + GeneratorExpr:1( + ConditionalExpr:1( + Condition( + NameExpr(x)) + IntExpr(1) + IntExpr(2)) + NameExpr(x) + NameExpr(y))))) + +[case testConditionalExpressionInListComprehension2] +a = [ 1 if x else 2 for x in y ] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + ListComprehension:1( + GeneratorExpr:1( + ConditionalExpr:1( + Condition( + NameExpr(x)) + IntExpr(1) + IntExpr(2)) + NameExpr(x) + NameExpr(y))))) + +[case testComplexWithLvalue] +with x as y.z: pass +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(x)) + Target( + MemberExpr:1( + NameExpr(y) + z)) + Block:1( + PassStmt:1()))) + +[case testRelativeImportWithEllipsis] +from ... import x +[out] +MypyFile:1( + ImportFrom:1(..., [x])) + +[case testRelativeImportWithEllipsis2] +from .... import x +[out] +MypyFile:1( + ImportFrom:1(...., [x])) + +[case testParseExtendedSlicing] +a[:, :] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(a) + TupleExpr:1( + SliceExpr:-1( + + ) + SliceExpr:-1( + + ))))) + +[case testParseExtendedSlicing2] +a[1:2:, :,] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(a) + TupleExpr:1( + SliceExpr:-1( + IntExpr(1) + IntExpr(2)) + SliceExpr:-1( + + ))))) + +[case testParseExtendedSlicing3] +a[1:2:3, ..., 1] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(a) + TupleExpr:1( + SliceExpr:-1( + IntExpr(1) + IntExpr(2) + IntExpr(3)) + Ellipsis + IntExpr(1))))) + +[case testParseIfExprInDictExpr] +test = { 'spam': 'eggs' if True else 'bacon' } +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(test) + DictExpr:1( + StrExpr(spam) + ConditionalExpr:1( + Condition( + NameExpr(True)) + StrExpr(eggs) + StrExpr(bacon))))) + +[case testIgnoreLine] +import x # type: ignore +[out] +MypyFile:1( + Import:1(x) + IgnoredLines(1)) + +[case testIgnore2Lines] +x +y # type: ignore +z # type: ignore +[out] +MypyFile:1( + ExpressionStmt:1( + NameExpr(x)) + ExpressionStmt:2( + NameExpr(y)) + ExpressionStmt:3( + NameExpr(z)) + IgnoredLines(2, 3)) + +[case testCommentedOutIgnoreAnnotation] +y ## type: ignore +[out] +MypyFile:1( + ExpressionStmt:1( + NameExpr(y))) + +[case testSpaceInIgnoreAnnotations] +y # type: ignore # foo +y #type:ignore +[out] +MypyFile:1( + ExpressionStmt:1( + NameExpr(y)) + ExpressionStmt:2( + NameExpr(y)) + IgnoredLines(1, 2)) + +[case testIgnoreAnnotationAndMultilineStatement] +x = { + 1: 2 # type: ignore +} +y = { # type: ignore + 1: 2 +} # type: ignore +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + DictExpr:1( + IntExpr(1) + IntExpr(2))) + AssignmentStmt:4( + NameExpr(y) + DictExpr:4( + IntExpr(1) + IntExpr(2))) + IgnoredLines(2, 4, 6)) + +[case testIgnoreAnnotationAndMultilineStatement2] +from m import ( # type: ignore + x, y +) +[out] +MypyFile:1( + ImportFrom:1(m, [x, y]) + IgnoredLines(1)) + +[case testYieldExpression] +def f(): + x = yield f() +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + AssignmentStmt:2( + NameExpr(x) + YieldExpr:2( + CallExpr:2( + NameExpr(f) + Args())))))) + +[case testForWithSingleItemTuple] +for x in 1,: pass +[out] +MypyFile:1( + ForStmt:1( + NameExpr(x) + TupleExpr:1( + IntExpr(1)) + Block:1( + PassStmt:1()))) + +[case testIsoLatinUnixEncoding] +# coding: iso-latin-1-unix +[out] +MypyFile:1() + +[case testLatinUnixEncoding] +# coding: latin-1-unix +[out] +MypyFile:1() + +[case testLatinUnixEncoding2] +# coding: iso-latin-1 +[out] +MypyFile:1() + +[case testYieldExpressionInParens] +def f(): + (yield) +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ExpressionStmt:2( + YieldExpr:2())))) + +[case testFStringSimple] +x = 'mypy' +f'Hello {x}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + ExpressionStmt:2( + CallExpr:2( + MemberExpr:2( + StrExpr() + join) + Args( + ListExpr:2( + StrExpr(Hello ) + CallExpr:2( + MemberExpr:2( + StrExpr({:{}}) + format) + Args( + NameExpr(x) + StrExpr()))))))) + +[case testFStringWithConversion] +x = 'mypy' +F'Hello {x!r}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + ExpressionStmt:2( + CallExpr:2( + MemberExpr:2( + StrExpr() + join) + Args( + ListExpr:2( + StrExpr(Hello ) + CallExpr:2( + MemberExpr:2( + StrExpr({!r:{}}) + format) + Args( + NameExpr(x) + StrExpr()))))))) + +[case testFStringWithOnlyFormatSpecifier] +x = 'mypy' +f'Hello {x:<30}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + ExpressionStmt:2( + CallExpr:2( + MemberExpr:2( + StrExpr() + join) + Args( + ListExpr:2( + StrExpr(Hello ) + CallExpr:2( + MemberExpr:2( + StrExpr({:{}}) + format) + Args( + NameExpr(x) + StrExpr(<30)))))))) + +[case testFStringWithFormatSpecifierAndConversion] +x = 'mypy' +f'Hello {x!s:<30}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + ExpressionStmt:2( + CallExpr:2( + MemberExpr:2( + StrExpr() + join) + Args( + ListExpr:2( + StrExpr(Hello ) + CallExpr:2( + MemberExpr:2( + StrExpr({!s:{}}) + format) + Args( + NameExpr(x) + StrExpr(<30)))))))) + +[case testFStringWithFormatSpecifierExpression] +x = 'mypy' +y = 30 +f'Hello {x!s:<{y+y}}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + AssignmentStmt:2( + NameExpr(y) + IntExpr(30)) + ExpressionStmt:3( + CallExpr:3( + MemberExpr:3( + StrExpr() + join) + Args( + ListExpr:3( + StrExpr(Hello ) + CallExpr:3( + MemberExpr:3( + StrExpr({!s:{}}) + format) + Args( + NameExpr(x) + CallExpr:3( + MemberExpr:3( + StrExpr() + join) + Args( + ListExpr:3( + StrExpr(<) + CallExpr:3( + MemberExpr:3( + StrExpr({:{}}) + format) + Args( + OpExpr:3( + + + NameExpr(y) + NameExpr(y)) + StrExpr())))))))))))) + +[case testStripFunctionBodiesIfIgnoringErrors] +# mypy: ignore-errors=True +def f(self): + self.x = 1 # Cannot define an attribute + return 1 +[out] +MypyFile:1( + FuncDef:2( + f + Args( + Var(self)) + Block:3())) + +[case testStripMethodBodiesIfIgnoringErrors] +# mypy: ignore-errors=True +class C: + def f(self): + x = self.x + for x in y: + pass + with a as y: + pass + while self.foo(): + self.bah() + a[self.x] = 1 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + f + Args( + Var(self)) + Block:4()))) + +[case testDoNotStripModuleTopLevelOrClassBody] +# mypy: ignore-errors=True +f() +class C: + x = 5 +[out] +MypyFile:1( + ExpressionStmt:2( + CallExpr:2( + NameExpr(f) + Args())) + ClassDef:3( + C + AssignmentStmt:4( + NameExpr(x) + IntExpr(5)))) + +[case testDoNotStripMethodThatAssignsToAttribute] +# mypy: ignore-errors=True +class C: + def m1(self): + self.x = 0 + def m2(self): + a, self.y = 0 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:4( + AssignmentStmt:4( + MemberExpr:4( + NameExpr(self) + x) + IntExpr(0)))) + FuncDef:5( + m2 + Args( + Var(self)) + Block:6( + AssignmentStmt:6( + TupleExpr:6( + NameExpr(a) + MemberExpr:6( + NameExpr(self) + y)) + IntExpr(0)))))) + +[case testDoNotStripMethodThatAssignsToAttributeWithinStatement] +# mypy: ignore-errors=True +class C: + def m1(self): + for x in y: + self.x = 0 + def m2(self): + with x: + self.y = 0 + def m3(self): + if x: + self.y = 0 + else: + x = 4 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:4( + ForStmt:4( + NameExpr(x) + NameExpr(y) + Block:5( + AssignmentStmt:5( + MemberExpr:5( + NameExpr(self) + x) + IntExpr(0)))))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:7( + WithStmt:7( + Expr( + NameExpr(x)) + Block:8( + AssignmentStmt:8( + MemberExpr:8( + NameExpr(self) + y) + IntExpr(0)))))) + FuncDef:9( + m3 + Args( + Var(self)) + Block:10( + IfStmt:10( + If( + NameExpr(x)) + Then( + AssignmentStmt:11( + MemberExpr:11( + NameExpr(self) + y) + IntExpr(0))) + Else( + AssignmentStmt:13( + NameExpr(x) + IntExpr(4)))))))) + +[case testDoNotStripMethodThatDefinesAttributeWithoutAssignment] +# mypy: ignore-errors=True +class C: + def m1(self): + with y as self.x: + pass + def m2(self): + for self.y in x: + pass +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:4( + WithStmt:4( + Expr( + NameExpr(y)) + Target( + MemberExpr:4( + NameExpr(self) + x)) + Block:5( + PassStmt:5())))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:7( + ForStmt:7( + MemberExpr:7( + NameExpr(self) + y) + NameExpr(x) + Block:8( + PassStmt:8())))))) + +[case testStripDecoratedFunctionOrMethod] +# mypy: ignore-errors=True +@deco +def f(): + x = 0 + +class C: + @deco + def m1(self): + x = 0 + + @deco + def m2(self): + self.x = 0 +[out] +MypyFile:1( + Decorator:2( + Var(f) + NameExpr(deco) + FuncDef:3( + f + Block:4())) + ClassDef:6( + C + Decorator:7( + Var(m1) + NameExpr(deco) + FuncDef:8( + m1 + Args( + Var(self)) + Block:9())) + Decorator:11( + Var(m2) + NameExpr(deco) + FuncDef:12( + m2 + Args( + Var(self)) + Block:13( + AssignmentStmt:13( + MemberExpr:13( + NameExpr(self) + x) + IntExpr(0))))))) + +[case testStripOverloadedMethod] +# mypy: ignore-errors=True +class C: + @overload + def m1(self, x: int) -> None: ... + @overload + def m1(self, x: str) -> None: ... + def m1(self, x): + x = 0 + + @overload + def m2(self, x: int) -> None: ... + @overload + def m2(self, x: str) -> None: ... + def m2(self, x): + self.x = 0 +[out] +MypyFile:1( + ClassDef:2( + C + OverloadedFuncDef:3( + Decorator:3( + Var(m1) + NameExpr(overload) + FuncDef:4( + m1 + Args( + Var(self) + Var(x)) + def (self: Any, x: int?) -> None? + Block:4( + ExpressionStmt:4( + Ellipsis)))) + Decorator:5( + Var(m1) + NameExpr(overload) + FuncDef:6( + m1 + Args( + Var(self) + Var(x)) + def (self: Any, x: str?) -> None? + Block:6( + ExpressionStmt:6( + Ellipsis)))) + FuncDef:7( + m1 + Args( + Var(self) + Var(x)) + Block:8())) + OverloadedFuncDef:10( + Decorator:10( + Var(m2) + NameExpr(overload) + FuncDef:11( + m2 + Args( + Var(self) + Var(x)) + def (self: Any, x: int?) -> None? + Block:11( + ExpressionStmt:11( + Ellipsis)))) + Decorator:12( + Var(m2) + NameExpr(overload) + FuncDef:13( + m2 + Args( + Var(self) + Var(x)) + def (self: Any, x: str?) -> None? + Block:13( + ExpressionStmt:13( + Ellipsis)))) + FuncDef:14( + m2 + Args( + Var(self) + Var(x)) + Block:15( + AssignmentStmt:15( + MemberExpr:15( + NameExpr(self) + x) + IntExpr(0))))))) + +[case testStripMethodInNestedClass] +# mypy: ignore-errors=True +class C: + class D: + def m1(self): + self.x = 1 + def m2(self): + return self.x +[out] +MypyFile:1( + ClassDef:2( + C + ClassDef:3( + D + FuncDef:4( + m1 + Args( + Var(self)) + Block:5( + AssignmentStmt:5( + MemberExpr:5( + NameExpr(self) + x) + IntExpr(1)))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:7())))) diff --git a/test-data/unit/pep561.test b/test-data/unit/pep561.test new file mode 100644 index 000000000000..fb303a8fb5ec --- /dev/null +++ b/test-data/unit/pep561.test @@ -0,0 +1,215 @@ +[case testTypedPkgNoSitePkgsIgnoredImports] +# pkgs: typedpkg +# flags: --no-site-packages +from typedpkg.sample import ex +from typedpkg import dne +a = ex(['']) +reveal_type(a) +[file mypy.ini] +\[mypy] +ignore_missing_imports = True +[out] +testTypedPkgNoSitePkgsIgnoredImports.py:6: note: Revealed type is "Any" + +[case testTypedPkgSimple] +# pkgs: typedpkg +from typedpkg.sample import ex +from typedpkg import dne +a = ex(['']) +reveal_type(a) +[out] +testTypedPkgSimple.py:5: note: Revealed type is "builtins.tuple[builtins.str, ...]" + +[case testTypedPkgSimplePackageSearchPath] +# pkgs: typedpkg +# flags: -p typedpkg.pkg +[out] + +[case testTypedPkg_config_nositepackages] +# pkgs: typedpkg +from typedpkg.sample import ex +from typedpkg import dne +a = ex(['']) +reveal_type(a) +[file mypy.ini] +\[mypy] +no_site_packages=True +[out] +testTypedPkg_config_nositepackages.py:2: error: Cannot find implementation or library stub for module named "typedpkg.sample" +testTypedPkg_config_nositepackages.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +testTypedPkg_config_nositepackages.py:3: error: Cannot find implementation or library stub for module named "typedpkg" +testTypedPkg_config_nositepackages.py:5: note: Revealed type is "Any" + +[case testTypedPkg_args_nositepackages] +# pkgs: typedpkg +# flags: --no-site-packages +from typedpkg.sample import ex +from typedpkg import dne +a = ex(['']) +reveal_type(a) +[out] +testTypedPkg_args_nositepackages.py:3: error: Cannot find implementation or library stub for module named "typedpkg.sample" +testTypedPkg_args_nositepackages.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +testTypedPkg_args_nositepackages.py:4: error: Cannot find implementation or library stub for module named "typedpkg" +testTypedPkg_args_nositepackages.py:6: note: Revealed type is "Any" + +[case testTypedPkgStubs] +# pkgs: typedpkg-stubs +from typedpkg.sample import ex +from typedpkg import dne +a = ex(['']) +reveal_type(a) +[out] +testTypedPkgStubs.py:3: error: Module "typedpkg" has no attribute "dne" +testTypedPkgStubs.py:5: note: Revealed type is "builtins.list[builtins.str]" + +[case testStubPrecedence] +# pkgs: typedpkg, typedpkg-stubs +from typedpkg.sample import ex +from typedpkg import dne +a = ex(['']) +reveal_type(a) +[out] +testStubPrecedence.py:5: note: Revealed type is "builtins.list[builtins.str]" + +[case testTypedPkgSimpleEditable] +# pkgs: typedpkg; editable +from typedpkg.sample import ex +from typedpkg import dne +a = ex(['']) +reveal_type(a) +[out] +testTypedPkgSimpleEditable.py:5: note: Revealed type is "builtins.tuple[builtins.str, ...]" + +[case testTypedPkgNamespaceImportFrom] +# pkgs: typedpkg, typedpkg_ns_a +from typedpkg.pkg.aaa import af +from typedpkg_ns.a.bbb import bf +from typedpkg_ns.a.dne import dne + +af("abc") +bf(False) +dne(123) + +af(False) +bf(2) +dne("abc") +[out] +testTypedPkgNamespaceImportFrom.py:4: error: Cannot find implementation or library stub for module named "typedpkg_ns.a.dne" +testTypedPkgNamespaceImportFrom.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +testTypedPkgNamespaceImportFrom.py:10: error: Argument 1 to "af" has incompatible type "bool"; expected "str" +testTypedPkgNamespaceImportFrom.py:11: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" + +[case testTypedPkgNamespaceImportAs] +# pkgs: typedpkg, typedpkg_ns_a +import typedpkg.pkg.aaa as nm; af = nm.af +import typedpkg_ns.a.bbb as am; bf = am.bf +from typedpkg_ns.a.dne import dne + +af("abc") +bf(False) +dne(123) + +af(False) +bf(2) +dne("abc") +[out] +testTypedPkgNamespaceImportAs.py:4: error: Cannot find implementation or library stub for module named "typedpkg_ns.a.dne" +testTypedPkgNamespaceImportAs.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +testTypedPkgNamespaceImportAs.py:10: error: Argument 1 has incompatible type "bool"; expected "str" +testTypedPkgNamespaceImportAs.py:11: error: Argument 1 has incompatible type "int"; expected "bool" + +[case testTypedPkgNamespaceRegImport] +# pkgs: typedpkg, typedpkg_ns_a +import typedpkg.pkg.aaa; af = typedpkg.pkg.aaa.af +import typedpkg_ns.a.bbb; bf = typedpkg_ns.a.bbb.bf +from typedpkg_ns.a.dne import dne + +af("abc") +bf(False) +dne(123) + +af(False) +bf(2) +dne("abc") + +[out] +testTypedPkgNamespaceRegImport.py:4: error: Cannot find implementation or library stub for module named "typedpkg_ns.a.dne" +testTypedPkgNamespaceRegImport.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +testTypedPkgNamespaceRegImport.py:10: error: Argument 1 has incompatible type "bool"; expected "str" +testTypedPkgNamespaceRegImport.py:11: error: Argument 1 has incompatible type "int"; expected "bool" + +-- This is really testing the test framework to make sure incremental works +[case testPep561TestIncremental] +# pkgs: typedpkg +import a +[file a.py] +[file a.py.2] +1 + 'no' +[out] +[out2] +a.py:1: error: Unsupported operand types for + ("int" and "str") + +[case testTypedPkgNamespaceRegFromImportTwice] +# pkgs: typedpkg_ns_a +from typedpkg_ns import a +-- dummy should trigger a second iteration +[file dummy.py.2] +[out] +[out2] + +[case testNamespacePkgWStubs] +# pkgs: typedpkg_ns_a, typedpkg_ns_b, typedpkg_ns_b-stubs +# flags: --no-namespace-packages +import typedpkg_ns.a.bbb as a +import typedpkg_ns.b.bbb as b +a.bf(False) +b.bf(False) +a.bf(1) +b.bf(1) +import typedpkg_ns.whatever as c # type: ignore[import-untyped] +[out] +testNamespacePkgWStubs.py:4: error: Skipping analyzing "typedpkg_ns.b.bbb": module is installed, but missing library stubs or py.typed marker +testNamespacePkgWStubs.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +testNamespacePkgWStubs.py:4: error: Skipping analyzing "typedpkg_ns.b": module is installed, but missing library stubs or py.typed marker +testNamespacePkgWStubs.py:7: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" + +[case testNamespacePkgWStubsWithNamespacePackagesFlag] +# pkgs: typedpkg_ns_a, typedpkg_ns_b, typedpkg_ns_b-stubs +# flags: --namespace-packages +import typedpkg_ns.a.bbb as a +import typedpkg_ns.b.bbb as b +a.bf(False) +b.bf(False) +a.bf(1) +b.bf(1) +[out] +testNamespacePkgWStubsWithNamespacePackagesFlag.py:7: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" +testNamespacePkgWStubsWithNamespacePackagesFlag.py:8: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" + +[case testMissingPytypedFlag] +# pkgs: typedpkg_ns_b +# flags: --namespace-packages --follow-untyped-imports +import typedpkg_ns.b.bbb as b +b.bf("foo", "bar") +[out] +testMissingPytypedFlag.py:4: error: Too many arguments for "bf" + +[case testTypedPkgNamespaceRegFromImportTwiceMissing] +# pkgs: typedpkg_ns_a +from typedpkg_ns import does_not_exist # type: ignore +from typedpkg_ns import a +-- dummy should trigger a second iteration +[file dummy.py.2] +[out] +[out2] + + +[case testTypedPkgNamespaceRegFromImportTwiceMissing2] +# pkgs: typedpkg_ns_a +from typedpkg_ns import does_not_exist # type: ignore +from typedpkg_ns.a.bbb import bf +-- dummy should trigger a second iteration +[file dummy.py.2] +[out] +[out2] diff --git a/test-data/unit/plugins/add_classmethod.py b/test-data/unit/plugins/add_classmethod.py new file mode 100644 index 000000000000..9bc2c4e079dd --- /dev/null +++ b/test-data/unit/plugins/add_classmethod.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import ARG_POS, Argument, Var +from mypy.plugin import ClassDefContext, Plugin +from mypy.plugins.common import add_method +from mypy.types import NoneType + + +class ClassMethodPlugin(Plugin): + def get_base_class_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + if "BaseAddMethod" in fullname: + return add_extra_methods_hook + return None + + +def add_extra_methods_hook(ctx: ClassDefContext) -> None: + add_method(ctx, "foo_classmethod", [], NoneType(), is_classmethod=True) + add_method( + ctx, + "foo_staticmethod", + [Argument(Var(""), ctx.api.named_type("builtins.int"), None, ARG_POS)], + ctx.api.named_type("builtins.str"), + is_staticmethod=True, + ) + + +def plugin(version: str) -> type[ClassMethodPlugin]: + return ClassMethodPlugin diff --git a/test-data/unit/plugins/add_method.py b/test-data/unit/plugins/add_method.py new file mode 100644 index 000000000000..f3a7ebdb95ed --- /dev/null +++ b/test-data/unit/plugins/add_method.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import ClassDefContext, Plugin +from mypy.plugins.common import add_method +from mypy.types import NoneType + + +class AddOverrideMethodPlugin(Plugin): + def get_class_decorator_hook_2(self, fullname: str) -> Callable[[ClassDefContext], bool] | None: + if fullname == "__main__.inject_foo": + return add_extra_methods_hook + return None + + +def add_extra_methods_hook(ctx: ClassDefContext) -> bool: + add_method(ctx, "foo_implicit", [], NoneType()) + return True + + +def plugin(version: str) -> type[AddOverrideMethodPlugin]: + return AddOverrideMethodPlugin diff --git a/test-data/unit/plugins/add_overloaded_method.py b/test-data/unit/plugins/add_overloaded_method.py new file mode 100644 index 000000000000..efda848f790c --- /dev/null +++ b/test-data/unit/plugins/add_overloaded_method.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import ARG_POS, Argument, Var +from mypy.plugin import ClassDefContext, Plugin +from mypy.plugins.common import MethodSpec, add_overloaded_method_to_class + + +class OverloadedMethodPlugin(Plugin): + def get_base_class_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + if "AddOverloadedMethod" in fullname: + return add_overloaded_method_hook + return None + + +def add_overloaded_method_hook(ctx: ClassDefContext) -> None: + add_overloaded_method_to_class(ctx.api, ctx.cls, "method", _generate_method_specs(ctx)) + add_overloaded_method_to_class( + ctx.api, ctx.cls, "clsmethod", _generate_method_specs(ctx), is_classmethod=True + ) + add_overloaded_method_to_class( + ctx.api, ctx.cls, "stmethod", _generate_method_specs(ctx), is_staticmethod=True + ) + + +def _generate_method_specs(ctx: ClassDefContext) -> list[MethodSpec]: + return [ + MethodSpec( + args=[Argument(Var("arg"), ctx.api.named_type("builtins.int"), None, ARG_POS)], + return_type=ctx.api.named_type("builtins.str"), + ), + MethodSpec( + args=[Argument(Var("arg"), ctx.api.named_type("builtins.str"), None, ARG_POS)], + return_type=ctx.api.named_type("builtins.int"), + ), + ] + + +def plugin(version: str) -> type[OverloadedMethodPlugin]: + return OverloadedMethodPlugin diff --git a/test-data/unit/plugins/arg_kinds.py b/test-data/unit/plugins/arg_kinds.py new file mode 100644 index 000000000000..388a3c738b62 --- /dev/null +++ b/test-data/unit/plugins/arg_kinds.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import FunctionContext, MethodContext, Plugin +from mypy.types import Type + + +class ArgKindsPlugin(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if "func" in fullname: + return extract_arg_kinds_from_function + return None + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + if "Class.method" in fullname: + return extract_arg_kinds_from_method + return None + + +def extract_arg_kinds_from_function(ctx: FunctionContext) -> Type: + ctx.api.fail(str([[x.value for x in y] for y in ctx.arg_kinds]), ctx.context) + return ctx.default_return_type + + +def extract_arg_kinds_from_method(ctx: MethodContext) -> Type: + ctx.api.fail(str([[x.value for x in y] for y in ctx.arg_kinds]), ctx.context) + return ctx.default_return_type + + +def plugin(version: str) -> type[ArgKindsPlugin]: + return ArgKindsPlugin diff --git a/test-data/unit/plugins/arg_names.py b/test-data/unit/plugins/arg_names.py new file mode 100644 index 000000000000..981c1a2eb12d --- /dev/null +++ b/test-data/unit/plugins/arg_names.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import StrExpr +from mypy.plugin import FunctionContext, MethodContext, Plugin +from mypy.types import Type + + +class ArgNamesPlugin(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname in { + "mod.func", + "mod.func_unfilled", + "mod.func_star_expr", + "mod.ClassInit", + "mod.Outer.NestedClassInit", + }: + return extract_classname_and_set_as_return_type_function + return None + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + if fullname in { + "mod.Class.method", + "mod.Class.myclassmethod", + "mod.Class.mystaticmethod", + "mod.ClassUnfilled.method", + "mod.ClassStarExpr.method", + "mod.ClassChild.method", + "mod.ClassChild.myclassmethod", + }: + return extract_classname_and_set_as_return_type_method + return None + + +def extract_classname_and_set_as_return_type_function(ctx: FunctionContext) -> Type: + arg = ctx.args[ctx.callee_arg_names.index("classname")][0] + if not isinstance(arg, StrExpr): + return ctx.default_return_type + return ctx.api.named_generic_type(arg.value, []) + + +def extract_classname_and_set_as_return_type_method(ctx: MethodContext) -> Type: + arg = ctx.args[ctx.callee_arg_names.index("classname")][0] + if not isinstance(arg, StrExpr): + return ctx.default_return_type + return ctx.api.named_generic_type(arg.value, []) + + +def plugin(version: str) -> type[ArgNamesPlugin]: + return ArgNamesPlugin diff --git a/test-data/unit/plugins/attrhook.py b/test-data/unit/plugins/attrhook.py new file mode 100644 index 000000000000..9500734daa6c --- /dev/null +++ b/test-data/unit/plugins/attrhook.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import AttributeContext, Plugin +from mypy.types import Instance, Type + + +class AttrPlugin(Plugin): + def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: + if fullname == "m.Signal.__call__": + return signal_call_callback + return None + + +def signal_call_callback(ctx: AttributeContext) -> Type: + if isinstance(ctx.type, Instance): + return ctx.type.args[0] + return ctx.default_attr_type + + +def plugin(version: str) -> type[AttrPlugin]: + return AttrPlugin diff --git a/test-data/unit/plugins/attrhook2.py b/test-data/unit/plugins/attrhook2.py new file mode 100644 index 000000000000..1ce318d2057b --- /dev/null +++ b/test-data/unit/plugins/attrhook2.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import AttributeContext, Plugin +from mypy.types import AnyType, Type, TypeOfAny + + +class AttrPlugin(Plugin): + def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: + if fullname == "m.Magic.magic_field": + return magic_field_callback + if fullname == "m.Magic.nonexistent_field": + return nonexistent_field_callback + if fullname == "m.Magic.no_assignment_field": + return no_assignment_field_callback + return None + + +def magic_field_callback(ctx: AttributeContext) -> Type: + return ctx.api.named_generic_type("builtins.str", []) + + +def nonexistent_field_callback(ctx: AttributeContext) -> Type: + ctx.api.fail("Field does not exist", ctx.context) + return AnyType(TypeOfAny.from_error) + + +def no_assignment_field_callback(ctx: AttributeContext) -> Type: + if ctx.is_lvalue: + ctx.api.fail(f"Cannot assign to field", ctx.context) + return AnyType(TypeOfAny.from_error) + return ctx.default_attr_type + + +def plugin(version: str) -> type[AttrPlugin]: + return AttrPlugin diff --git a/test-data/unit/plugins/badreturn.py b/test-data/unit/plugins/badreturn.py new file mode 100644 index 000000000000..9dce3b3e99c2 --- /dev/null +++ b/test-data/unit/plugins/badreturn.py @@ -0,0 +1,2 @@ +def plugin(version: str) -> None: + pass diff --git a/test-data/unit/plugins/badreturn2.py b/test-data/unit/plugins/badreturn2.py new file mode 100644 index 000000000000..1ae551ecbf20 --- /dev/null +++ b/test-data/unit/plugins/badreturn2.py @@ -0,0 +1,9 @@ +from __future__ import annotations + + +class MyPlugin: + pass + + +def plugin(version: str) -> type[MyPlugin]: + return MyPlugin diff --git a/test-data/unit/plugins/callable_instance.py b/test-data/unit/plugins/callable_instance.py new file mode 100644 index 000000000000..a9f562effb34 --- /dev/null +++ b/test-data/unit/plugins/callable_instance.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import MethodContext, Plugin +from mypy.types import Instance, Type + + +class CallableInstancePlugin(Plugin): + def get_function_hook(self, fullname: str) -> None: + assert not fullname.endswith(" of Foo") + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + # Ensure that all names are fully qualified + assert not fullname.endswith(" of Foo") + + if fullname == "__main__.Class.__call__": + return my_hook + + return None + + +def my_hook(ctx: MethodContext) -> Type: + if isinstance(ctx.type, Instance) and len(ctx.type.args) == 1: + return ctx.type.args[0] + return ctx.default_return_type + + +def plugin(version: str) -> type[CallableInstancePlugin]: + return CallableInstancePlugin diff --git a/test-data/unit/plugins/class_attr_hook.py b/test-data/unit/plugins/class_attr_hook.py new file mode 100644 index 000000000000..5d6a87df48bb --- /dev/null +++ b/test-data/unit/plugins/class_attr_hook.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import AttributeContext, Plugin +from mypy.types import Type as MypyType + + +class ClassAttrPlugin(Plugin): + def get_class_attribute_hook( + self, fullname: str + ) -> Callable[[AttributeContext], MypyType] | None: + if fullname == "__main__.Cls.attr": + return my_hook + return None + + +def my_hook(ctx: AttributeContext) -> MypyType: + return ctx.api.named_generic_type("builtins.int", []) + + +def plugin(_version: str) -> type[ClassAttrPlugin]: + return ClassAttrPlugin diff --git a/test-data/unit/plugins/class_callable.py b/test-data/unit/plugins/class_callable.py new file mode 100644 index 000000000000..9fab30e60458 --- /dev/null +++ b/test-data/unit/plugins/class_callable.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import NameExpr +from mypy.plugin import FunctionContext, Plugin +from mypy.types import Instance, NoneType, Type, UnionType, get_proper_type + + +class AttrPlugin(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname.startswith("mod.Attr"): + return attr_hook + return None + + +def attr_hook(ctx: FunctionContext) -> Type: + default = get_proper_type(ctx.default_return_type) + assert isinstance(default, Instance) + if default.type.fullname == "mod.Attr": + attr_base = default + else: + attr_base = None + for base in default.type.bases: + if base.type.fullname == "mod.Attr": + attr_base = base + break + assert attr_base is not None + last_arg_exprs = ctx.args[-1] + if any(isinstance(expr, NameExpr) and expr.name == "True" for expr in last_arg_exprs): + return attr_base + assert len(attr_base.args) == 1 + arg_type = attr_base.args[0] + return Instance( + attr_base.type, + [UnionType([arg_type, NoneType()])], + line=default.line, + column=default.column, + ) + + +def plugin(version: str) -> type[AttrPlugin]: + return AttrPlugin diff --git a/test-data/unit/plugins/common_api_incremental.py b/test-data/unit/plugins/common_api_incremental.py new file mode 100644 index 000000000000..b14b2f92073e --- /dev/null +++ b/test-data/unit/plugins/common_api_incremental.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import GDEF, MDEF, Block, ClassDef, SymbolTable, SymbolTableNode, TypeInfo, Var +from mypy.plugin import ClassDefContext, DynamicClassDefContext, Plugin + + +class DynPlugin(Plugin): + def get_dynamic_class_hook( + self, fullname: str + ) -> Callable[[DynamicClassDefContext], None] | None: + if fullname == "lib.declarative_base": + return add_info_hook + return None + + def get_base_class_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + sym = self.lookup_fully_qualified(fullname) + if sym and isinstance(sym.node, TypeInfo): + if sym.node.metadata.get("magic"): + return add_magic_hook + return None + + +def add_info_hook(ctx: DynamicClassDefContext) -> None: + class_def = ClassDef(ctx.name, Block([])) + class_def.fullname = ctx.api.qualified_name(ctx.name) + + info = TypeInfo(SymbolTable(), class_def, ctx.api.cur_mod_id) + class_def.info = info + obj = ctx.api.named_type("builtins.object", []) + info.mro = [info, obj.type] + info.bases = [obj] + ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) + info.metadata["magic"] = {"value": True} + + +def add_magic_hook(ctx: ClassDefContext) -> None: + info = ctx.cls.info + str_type = ctx.api.named_type_or_none("builtins.str", []) + assert str_type is not None + var = Var("__magic__", str_type) + var.info = info + info.names["__magic__"] = SymbolTableNode(MDEF, var) + + +def plugin(version: str) -> type[DynPlugin]: + return DynPlugin diff --git a/test-data/unit/plugins/config_data.py b/test-data/unit/plugins/config_data.py new file mode 100644 index 000000000000..9b828bc9ac0a --- /dev/null +++ b/test-data/unit/plugins/config_data.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import json +import os +from typing import Any + +from mypy.plugin import Plugin, ReportConfigContext + + +class ConfigDataPlugin(Plugin): + def report_config_data(self, ctx: ReportConfigContext) -> Any: + path = os.path.join("tmp/test.json") + with open(path) as f: + data = json.load(f) + return data.get(ctx.id) + + +def plugin(version: str) -> type[ConfigDataPlugin]: + return ConfigDataPlugin diff --git a/test-data/unit/plugins/custom_errorcode.py b/test-data/unit/plugins/custom_errorcode.py new file mode 100644 index 000000000000..0af87658e59f --- /dev/null +++ b/test-data/unit/plugins/custom_errorcode.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.errorcodes import ErrorCode +from mypy.plugin import FunctionContext, Plugin +from mypy.types import AnyType, Type, TypeOfAny + +CUSTOM_ERROR = ErrorCode(code="custom", description="", category="Custom") + + +class CustomErrorCodePlugin(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname.endswith(".main"): + return self.emit_error + return None + + def emit_error(self, ctx: FunctionContext) -> Type: + ctx.api.fail("Custom error", ctx.context, code=CUSTOM_ERROR) + return AnyType(TypeOfAny.from_error) + + +def plugin(version: str) -> type[CustomErrorCodePlugin]: + return CustomErrorCodePlugin diff --git a/test-data/unit/plugins/customentry.py b/test-data/unit/plugins/customentry.py new file mode 100644 index 000000000000..1a7ed3348e12 --- /dev/null +++ b/test-data/unit/plugins/customentry.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import FunctionContext, Plugin +from mypy.types import Type + + +class MyPlugin(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname == "__main__.f": + return my_hook + assert fullname + return None + + +def my_hook(ctx: FunctionContext) -> Type: + return ctx.api.named_generic_type("builtins.int", []) + + +def register(version: str) -> type[MyPlugin]: + return MyPlugin diff --git a/test-data/unit/plugins/customize_mro.py b/test-data/unit/plugins/customize_mro.py new file mode 100644 index 000000000000..3b13b2e9d998 --- /dev/null +++ b/test-data/unit/plugins/customize_mro.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import ClassDefContext, Plugin + + +class DummyPlugin(Plugin): + def get_customize_class_mro_hook(self, fullname: str) -> Callable[[ClassDefContext], None]: + def analyze(classdef_ctx: ClassDefContext) -> None: + pass + + return analyze + + +def plugin(version: str) -> type[DummyPlugin]: + return DummyPlugin diff --git a/test-data/unit/plugins/decimal_to_int.py b/test-data/unit/plugins/decimal_to_int.py new file mode 100644 index 000000000000..2318b2367d33 --- /dev/null +++ b/test-data/unit/plugins/decimal_to_int.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import AnalyzeTypeContext, Plugin +from mypy.types import Type + + +class MyPlugin(Plugin): + def get_type_analyze_hook(self, fullname: str) -> Callable[[AnalyzeTypeContext], Type] | None: + if fullname in ("decimal.Decimal", "_decimal.Decimal"): + return decimal_to_int_hook + return None + + +def decimal_to_int_hook(ctx: AnalyzeTypeContext) -> Type: + return ctx.api.named_type("builtins.int", []) + + +def plugin(version: str) -> type[MyPlugin]: + return MyPlugin diff --git a/test-data/unit/plugins/depshook.py b/test-data/unit/plugins/depshook.py new file mode 100644 index 000000000000..bb2460de1196 --- /dev/null +++ b/test-data/unit/plugins/depshook.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from mypy.nodes import MypyFile +from mypy.plugin import Plugin + + +class DepsPlugin(Plugin): + def get_additional_deps(self, file: MypyFile) -> list[tuple[int, str, int]]: + if file.fullname == "__main__": + return [(10, "err", -1)] + return [] + + +def plugin(version: str) -> type[DepsPlugin]: + return DepsPlugin diff --git a/test-data/unit/plugins/descriptor.py b/test-data/unit/plugins/descriptor.py new file mode 100644 index 000000000000..d38853367906 --- /dev/null +++ b/test-data/unit/plugins/descriptor.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import MethodContext, MethodSigContext, Plugin +from mypy.types import CallableType, NoneType, Type, get_proper_type + + +class DescriptorPlugin(Plugin): + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + if fullname == "__main__.Desc.__get__": + return get_hook + return None + + def get_method_signature_hook( + self, fullname: str + ) -> Callable[[MethodSigContext], CallableType] | None: + if fullname == "__main__.Desc.__set__": + return set_hook + return None + + +def get_hook(ctx: MethodContext) -> Type: + arg = get_proper_type(ctx.arg_types[0][0]) + if isinstance(arg, NoneType): + return ctx.api.named_generic_type("builtins.str", []) + return ctx.api.named_generic_type("builtins.int", []) + + +def set_hook(ctx: MethodSigContext) -> CallableType: + return CallableType( + [ + ctx.api.named_generic_type("__main__.Cls", []), + ctx.api.named_generic_type("builtins.int", []), + ], + ctx.default_signature.arg_kinds, + ctx.default_signature.arg_names, + ctx.default_signature.ret_type, + ctx.default_signature.fallback, + ) + + +def plugin(version: str) -> type[DescriptorPlugin]: + return DescriptorPlugin diff --git a/test-data/unit/plugins/dyn_class.py b/test-data/unit/plugins/dyn_class.py new file mode 100644 index 000000000000..18e948e3dd2a --- /dev/null +++ b/test-data/unit/plugins/dyn_class.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import GDEF, Block, ClassDef, SymbolTable, SymbolTableNode, TypeInfo, Var +from mypy.plugin import ClassDefContext, DynamicClassDefContext, Plugin +from mypy.types import Instance, get_proper_type + +DECL_BASES = set() + + +class DynPlugin(Plugin): + def get_dynamic_class_hook( + self, fullname: str + ) -> Callable[[DynamicClassDefContext], None] | None: + if fullname == "mod.declarative_base": + return add_info_hook + return None + + def get_base_class_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + if fullname in DECL_BASES: + return replace_col_hook + return None + + +def add_info_hook(ctx: DynamicClassDefContext) -> None: + class_def = ClassDef(ctx.name, Block([])) + class_def.fullname = ctx.api.qualified_name(ctx.name) + + info = TypeInfo(SymbolTable(), class_def, ctx.api.cur_mod_id) + class_def.info = info + obj = ctx.api.named_type("builtins.object") + info.mro = [info, obj.type] + info.bases = [obj] + ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) + DECL_BASES.add(class_def.fullname) + + +def replace_col_hook(ctx: ClassDefContext) -> None: + info = ctx.cls.info + for sym in info.names.values(): + node = sym.node + if isinstance(node, Var) and isinstance( + (node_type := get_proper_type(node.type)), Instance + ): + if node_type.type.fullname == "mod.Column": + new_sym = ctx.api.lookup_fully_qualified_or_none("mod.Instr") + if new_sym: + new_info = new_sym.node + assert isinstance(new_info, TypeInfo) + node.type = Instance( + new_info, node_type.args, node_type.line, node_type.column + ) + + +def plugin(version: str) -> type[DynPlugin]: + return DynPlugin diff --git a/test-data/unit/plugins/dyn_class_from_method.py b/test-data/unit/plugins/dyn_class_from_method.py new file mode 100644 index 000000000000..2630b16be66e --- /dev/null +++ b/test-data/unit/plugins/dyn_class_from_method.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.nodes import ( + GDEF, + Block, + ClassDef, + IndexExpr, + MemberExpr, + NameExpr, + RefExpr, + SymbolTable, + SymbolTableNode, + TypeApplication, + TypeInfo, +) +from mypy.plugin import DynamicClassDefContext, Plugin +from mypy.types import Instance + + +class DynPlugin(Plugin): + def get_dynamic_class_hook( + self, fullname: str + ) -> Callable[[DynamicClassDefContext], None] | None: + if "from_queryset" in fullname: + return add_info_hook + if "as_manager" in fullname: + return as_manager_hook + return None + + +def add_info_hook(ctx: DynamicClassDefContext) -> None: + class_def = ClassDef(ctx.name, Block([])) + class_def.fullname = ctx.api.qualified_name(ctx.name) + + info = TypeInfo(SymbolTable(), class_def, ctx.api.cur_mod_id) + class_def.info = info + assert isinstance(ctx.call.args[0], RefExpr) + queryset_type_fullname = ctx.call.args[0].fullname + queryset_node = ctx.api.lookup_fully_qualified_or_none(queryset_type_fullname) + assert queryset_node is not None + queryset_info = queryset_node.node + assert isinstance(queryset_info, TypeInfo) + obj = ctx.api.named_type("builtins.object") + info.mro = [info, queryset_info, obj.type] + info.bases = [Instance(queryset_info, [])] + ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) + + +def as_manager_hook(ctx: DynamicClassDefContext) -> None: + class_def = ClassDef(ctx.name, Block([])) + class_def.fullname = ctx.api.qualified_name(ctx.name) + + info = TypeInfo(SymbolTable(), class_def, ctx.api.cur_mod_id) + class_def.info = info + assert isinstance(ctx.call.callee, MemberExpr) + assert isinstance(ctx.call.callee.expr, IndexExpr) + assert isinstance(ctx.call.callee.expr.analyzed, TypeApplication) + assert isinstance(ctx.call.callee.expr.analyzed.expr, NameExpr) + + queryset_type_fullname = ctx.call.callee.expr.analyzed.expr.fullname + queryset_node = ctx.api.lookup_fully_qualified_or_none(queryset_type_fullname) + assert queryset_node is not None + queryset_info = queryset_node.node + assert isinstance(queryset_info, TypeInfo) + parameter_type = ctx.call.callee.expr.analyzed.types[0] + + obj = ctx.api.named_type("builtins.object") + info.mro = [info, queryset_info, obj.type] + info.bases = [Instance(queryset_info, [parameter_type])] + ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) + + +def plugin(version: str) -> type[DynPlugin]: + return DynPlugin diff --git a/test-data/unit/plugins/fnplugin.py b/test-data/unit/plugins/fnplugin.py new file mode 100644 index 000000000000..a5a7e57101c2 --- /dev/null +++ b/test-data/unit/plugins/fnplugin.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import FunctionContext, Plugin +from mypy.types import Type + + +class MyPlugin(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname == "__main__.f": + return my_hook + assert fullname is not None + return None + + +def my_hook(ctx: FunctionContext) -> Type: + return ctx.api.named_generic_type("builtins.int", []) + + +def plugin(version: str) -> type[MyPlugin]: + return MyPlugin diff --git a/test-data/unit/plugins/fully_qualified_test_hook.py b/test-data/unit/plugins/fully_qualified_test_hook.py new file mode 100644 index 000000000000..9230091bba1a --- /dev/null +++ b/test-data/unit/plugins/fully_qualified_test_hook.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import MethodSigContext, Plugin +from mypy.types import CallableType + + +class FullyQualifiedTestPlugin(Plugin): + def get_method_signature_hook( + self, fullname: str + ) -> Callable[[MethodSigContext], CallableType] | None: + # Ensure that all names are fully qualified + if "FullyQualifiedTest" in fullname: + assert fullname.startswith("__main__.") and " of " not in fullname, fullname + return my_hook + + return None + + +def my_hook(ctx: MethodSigContext) -> CallableType: + return ctx.default_signature.copy_modified( + ret_type=ctx.api.named_generic_type("builtins.int", []) + ) + + +def plugin(version: str) -> type[FullyQualifiedTestPlugin]: + return FullyQualifiedTestPlugin diff --git a/test-data/unit/plugins/function_sig_hook.py b/test-data/unit/plugins/function_sig_hook.py new file mode 100644 index 000000000000..a8d3cf058062 --- /dev/null +++ b/test-data/unit/plugins/function_sig_hook.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import FunctionSigContext, Plugin +from mypy.types import CallableType + + +class FunctionSigPlugin(Plugin): + def get_function_signature_hook( + self, fullname: str + ) -> Callable[[FunctionSigContext], CallableType] | None: + if fullname == "__main__.dynamic_signature": + return my_hook + return None + + +def my_hook(ctx: FunctionSigContext) -> CallableType: + arg1_args = ctx.args[0] + if len(arg1_args) != 1: + return ctx.default_signature + arg1_type = ctx.api.get_expression_type(arg1_args[0]) + return ctx.default_signature.copy_modified(arg_types=[arg1_type], ret_type=arg1_type) + + +def plugin(version: str) -> type[FunctionSigPlugin]: + return FunctionSigPlugin diff --git a/test-data/unit/plugins/method_in_decorator.py b/test-data/unit/plugins/method_in_decorator.py new file mode 100644 index 000000000000..3fba7692266c --- /dev/null +++ b/test-data/unit/plugins/method_in_decorator.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import MethodContext, Plugin +from mypy.types import CallableType, Type, get_proper_type + + +class MethodDecoratorPlugin(Plugin): + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + if "Foo.a" in fullname: + return method_decorator_callback + return None + + +def method_decorator_callback(ctx: MethodContext) -> Type: + default = get_proper_type(ctx.default_return_type) + if isinstance(default, CallableType): + str_type = ctx.api.named_generic_type("builtins.str", []) + return default.copy_modified(ret_type=str_type) + return ctx.default_return_type + + +def plugin(version: str) -> type[MethodDecoratorPlugin]: + return MethodDecoratorPlugin diff --git a/test-data/unit/plugins/method_sig_hook.py b/test-data/unit/plugins/method_sig_hook.py new file mode 100644 index 000000000000..b78831cc45d5 --- /dev/null +++ b/test-data/unit/plugins/method_sig_hook.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import CheckerPluginInterface, MethodSigContext, Plugin +from mypy.types import CallableType, Instance, Type, get_proper_type + + +class MethodSigPlugin(Plugin): + def get_method_signature_hook( + self, fullname: str + ) -> Callable[[MethodSigContext], CallableType] | None: + # Ensure that all names are fully qualified + assert not fullname.endswith(" of Foo") + + if fullname.startswith("__main__.Foo."): + return my_hook + + return None + + +def _str_to_int(api: CheckerPluginInterface, typ: Type) -> Type: + typ = get_proper_type(typ) + if isinstance(typ, Instance): + if typ.type.fullname == "builtins.str": + return api.named_generic_type("builtins.int", []) + elif typ.args: + return typ.copy_modified(args=[_str_to_int(api, t) for t in typ.args]) + + return typ + + +def my_hook(ctx: MethodSigContext) -> CallableType: + return ctx.default_signature.copy_modified( + arg_types=[_str_to_int(ctx.api, t) for t in ctx.default_signature.arg_types], + ret_type=_str_to_int(ctx.api, ctx.default_signature.ret_type), + ) + + +def plugin(version: str) -> type[MethodSigPlugin]: + return MethodSigPlugin diff --git a/test-data/unit/plugins/named_callable.py b/test-data/unit/plugins/named_callable.py new file mode 100644 index 000000000000..c37e11c32125 --- /dev/null +++ b/test-data/unit/plugins/named_callable.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import FunctionContext, Plugin +from mypy.types import CallableType, Type, get_proper_type + + +class MyPlugin(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname == "m.decorator1": + return decorator_call_hook + if fullname == "m._decorated": # This is a dummy name generated by the plugin + return decorate_hook + return None + + +def decorator_call_hook(ctx: FunctionContext) -> Type: + default = get_proper_type(ctx.default_return_type) + if isinstance(default, CallableType): + return default.copy_modified(name="m._decorated") + return ctx.default_return_type + + +def decorate_hook(ctx: FunctionContext) -> Type: + default = get_proper_type(ctx.default_return_type) + if isinstance(default, CallableType): + return default.copy_modified(ret_type=ctx.api.named_generic_type("builtins.str", [])) + return ctx.default_return_type + + +def plugin(version: str) -> type[MyPlugin]: + return MyPlugin diff --git a/test-data/unit/plugins/noentry.py b/test-data/unit/plugins/noentry.py new file mode 100644 index 000000000000..c591ad11fd64 --- /dev/null +++ b/test-data/unit/plugins/noentry.py @@ -0,0 +1 @@ +# empty plugin diff --git a/test-data/unit/plugins/plugin2.py b/test-data/unit/plugins/plugin2.py new file mode 100644 index 000000000000..e486d96ea8bf --- /dev/null +++ b/test-data/unit/plugins/plugin2.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import FunctionContext, Plugin +from mypy.types import Type + + +class Plugin2(Plugin): + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + if fullname in ("__main__.f", "__main__.g"): + return str_hook + return None + + +def str_hook(ctx: FunctionContext) -> Type: + return ctx.api.named_generic_type("builtins.str", []) + + +def plugin(version: str) -> type[Plugin2]: + return Plugin2 diff --git a/test-data/unit/plugins/type_anal_hook.py b/test-data/unit/plugins/type_anal_hook.py new file mode 100644 index 000000000000..c380bbe873fe --- /dev/null +++ b/test-data/unit/plugins/type_anal_hook.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import AnalyzeTypeContext, Plugin + +# The official name changed to NoneType but we have an alias for plugin compat reasons +# so we'll keep testing that here. +from mypy.types import AnyType, CallableType, NoneTyp, Type, TypeList, TypeOfAny + + +class TypeAnalyzePlugin(Plugin): + def get_type_analyze_hook(self, fullname: str) -> Callable[[AnalyzeTypeContext], Type] | None: + if fullname == "m.Signal": + return signal_type_analyze_callback + return None + + +def signal_type_analyze_callback(ctx: AnalyzeTypeContext) -> Type: + if len(ctx.type.args) != 1 or not isinstance(ctx.type.args[0], TypeList): + ctx.api.fail('Invalid "Signal" type (expected "Signal[[t, ...]]")', ctx.context) + return AnyType(TypeOfAny.from_error) + + args = ctx.type.args[0] + assert isinstance(args, TypeList) + analyzed = ctx.api.analyze_callable_args(args) + if analyzed is None: + return AnyType(TypeOfAny.from_error) # Error generated elsewhere + arg_types, arg_kinds, arg_names = analyzed + arg_types = [ctx.api.analyze_type(arg) for arg in arg_types] + type_arg = CallableType( + arg_types, arg_kinds, arg_names, NoneTyp(), ctx.api.named_type("builtins.function", []) + ) + return ctx.api.named_type("m.Signal", [type_arg]) + + +def plugin(version: str) -> type[TypeAnalyzePlugin]: + return TypeAnalyzePlugin diff --git a/test-data/unit/plugins/union_method.py b/test-data/unit/plugins/union_method.py new file mode 100644 index 000000000000..7c62ffb8c0cc --- /dev/null +++ b/test-data/unit/plugins/union_method.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import CheckerPluginInterface, MethodContext, MethodSigContext, Plugin +from mypy.types import CallableType, Instance, Type, get_proper_type + + +class MethodPlugin(Plugin): + def get_method_signature_hook( + self, fullname: str + ) -> Callable[[MethodSigContext], CallableType] | None: + if fullname.startswith("__main__.Foo."): + return my_meth_sig_hook + return None + + def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: + if fullname.startswith("__main__.Bar."): + return my_meth_hook + return None + + +def _str_to_int(api: CheckerPluginInterface, typ: Type) -> Type: + typ = get_proper_type(typ) + if isinstance(typ, Instance): + if typ.type.fullname == "builtins.str": + return api.named_generic_type("builtins.int", []) + elif typ.args: + return typ.copy_modified(args=[_str_to_int(api, t) for t in typ.args]) + return typ + + +def _float_to_int(api: CheckerPluginInterface, typ: Type) -> Type: + typ = get_proper_type(typ) + if isinstance(typ, Instance): + if typ.type.fullname == "builtins.float": + return api.named_generic_type("builtins.int", []) + elif typ.args: + return typ.copy_modified(args=[_float_to_int(api, t) for t in typ.args]) + return typ + + +def my_meth_sig_hook(ctx: MethodSigContext) -> CallableType: + return ctx.default_signature.copy_modified( + arg_types=[_str_to_int(ctx.api, t) for t in ctx.default_signature.arg_types], + ret_type=_str_to_int(ctx.api, ctx.default_signature.ret_type), + ) + + +def my_meth_hook(ctx: MethodContext) -> Type: + return _float_to_int(ctx.api, ctx.default_return_type) + + +def plugin(version: str) -> type[MethodPlugin]: + return MethodPlugin diff --git a/test-data/unit/pythoneval-asyncio.test b/test-data/unit/pythoneval-asyncio.test new file mode 100644 index 000000000000..4a185557495b --- /dev/null +++ b/test-data/unit/pythoneval-asyncio.test @@ -0,0 +1,478 @@ +-- Test cases for type checking mypy programs using full stubs and running +-- using CPython. +-- +-- These are mostly regression tests -- no attempt is made to make these +-- complete. +-- +-- This test file checks Asyncio and await interaction + +[case testImportAsyncio] +import asyncio +print('Imported') +[out] +Imported + +[case testSimpleCoroutineSleep] +from typing import Any, Generator +import asyncio +from asyncio import Future + +async def greet_every_two_seconds() -> None: + n = 0 + while n < 5: + print('Prev', n) + await asyncio.sleep(0.01) + print('After', n) + n += 1 + +loop = asyncio.get_event_loop() +try: + loop.run_until_complete(greet_every_two_seconds()) +finally: + loop.close() +[out] +Prev 0 +After 0 +Prev 1 +After 1 +Prev 2 +After 2 +Prev 3 +After 3 +Prev 4 +After 4 + +[case testCoroutineCallingOtherCoroutine] +from typing import Any +import asyncio +from asyncio import Future + +async def compute(x: int, y: int) -> int: + print("Compute %s + %s ..." % (x, y)) + await asyncio.sleep(0.01) + return x + y # Here the int is wrapped in Future[int] + +async def print_sum(x: int, y: int) -> None: + result = await compute(x, y) # The type of result will be int (is extracted from Future[int] + print("%s + %s = %s" % (x, y, result)) + +loop = asyncio.get_event_loop() +loop.run_until_complete(print_sum(1, 2)) +loop.close() +[out] +Compute 1 + 2 ... +1 + 2 = 3 + +[case testCoroutineChangingFuture] +from typing import Any +import asyncio +from asyncio import Future + +async def slow_operation(future: 'Future[str]') -> None: + await asyncio.sleep(0.01) + future.set_result('Future is done!') + +loop = asyncio.get_event_loop() +future = asyncio.Future() # type: Future[str] +asyncio.Task(slow_operation(future)) +loop.run_until_complete(future) +print(future.result()) +loop.close() +[out] +Future is done! + +[case testFunctionAssignedAsCallback] +import typing +from typing import Any +import asyncio +from asyncio import Future, AbstractEventLoop + +async def slow_operation(future: 'Future[str]') -> None: + await asyncio.sleep(1) + future.set_result('Callback works!') + +def got_result(future: 'Future[str]') -> None: + print(future.result()) + loop.stop() + +loop = asyncio.get_event_loop() # type: AbstractEventLoop +future = asyncio.Future() # type: Future[str] +asyncio.Task(slow_operation(future)) # Here create a task with the function. (The Task need a Future[T] as first argument) +future.add_done_callback(got_result) # and assign the callback to the future +try: + loop.run_forever() +finally: + loop.close() +[out] +Callback works! + +[case testMultipleTasks] +import typing +from typing import Any +import asyncio +from asyncio import Task, Future +async def factorial(name, number) -> None: + f = 1 + for i in range(2, number+1): + print("Task %s: Compute factorial(%s)..." % (name, i)) + await asyncio.sleep(0.01) + f *= i + print("Task %s: factorial(%s) = %s" % (name, number, f)) + +loop = asyncio.get_event_loop() +tasks = [ + asyncio.Task(factorial("A", 2)), + asyncio.Task(factorial("B", 3)), + asyncio.Task(factorial("C", 4))] +loop.run_until_complete(asyncio.wait(tasks)) +loop.close() +[out] +Task A: Compute factorial(2)... +Task B: Compute factorial(2)... +Task C: Compute factorial(2)... +Task A: factorial(2) = 2 +Task B: Compute factorial(3)... +Task C: Compute factorial(3)... +Task B: factorial(3) = 6 +Task C: Compute factorial(4)... +Task C: factorial(4) = 24 + + +[case testConcatenatedCoroutines] +import typing +from typing import Any +import asyncio +from asyncio import Future + +async def h4() -> int: + x = await future + return x + +async def h3() -> int: + x = await h4() + print("h3: %s" % x) + return x + +async def h2() -> int: + x = await h3() + print("h2: %s" % x) + return x + +async def h() -> None: + x = await h2() + print("h: %s" % x) + +loop = asyncio.get_event_loop() +future = asyncio.Future() # type: Future[int] +future.set_result(42) +loop.run_until_complete(h()) +print("Outside %s" % future.result()) +loop.close() +[out] +h3: 42 +h2: 42 +h: 42 +Outside 42 + +[case testConcatenatedCoroutinesReturningFutures] +import typing +from typing import Any +import asyncio +from asyncio import Future + +async def h4() -> "Future[int]": + await asyncio.sleep(0.01) + f = asyncio.Future() #type: Future[int] + return f + +async def h3() -> "Future[Future[int]]": + x = await h4() + x.set_result(42) + f = asyncio.Future() #type: Future[Future[int]] + f.set_result(x) + return f + +async def h() -> None: + print("Before") + x = await h3() + y = await x + z = await y + print(z) + def normalize(future): + # The str conversion seems inconsistent; not sure exactly why. Normalize + # the result. + return str(future).replace(' +Future> + + +[case testCoroutineWithOwnClass] +import typing +from typing import Any +import asyncio +from asyncio import Future + +class A: + def __init__(self, x: int) -> None: + self.x = x + +async def h() -> None: + x = await future + print("h: %s" % x.x) + +loop = asyncio.get_event_loop() +future = asyncio.Future() # type: Future[A] +future.set_result(A(42)) +loop.run_until_complete(h()) +print("Outside %s" % future.result().x) +loop.close() +[out] +h: 42 +Outside 42 + + +-- Errors + +[case testErrorAssigningCoroutineThatDontReturn] +from typing import Any +import asyncio +from asyncio import Future + +async def greet() -> None: + await asyncio.sleep(0.2) + print('Hello World') + +async def test() -> None: + await greet() + x = await greet() # Error + +loop = asyncio.get_event_loop() +try: + loop.run_until_complete(test()) +finally: + loop.close() +[out] +_program.py:11: error: Function does not return a value (it only ever returns None) + +[case testErrorReturnIsNotTheSameType] +from typing import Any +import asyncio +from asyncio import Future + +async def compute(x: int, y: int) -> int: + print("Compute %s + %s ..." % (x, y)) + await asyncio.sleep(0.01) + return str(x + y) # Error + +async def print_sum(x: int, y: int) -> None: + result = await compute(x, y) + print("%s + %s = %s" % (x, y, result)) + +loop = asyncio.get_event_loop() +loop.run_until_complete(print_sum(1, 2)) +loop.close() + +[out] +_program.py:8: error: Incompatible return value type (got "str", expected "int") + +[case testErrorSetFutureDifferentInternalType] +from typing import Any +import asyncio +from asyncio import Future + +async def slow_operation(future: 'Future[str]') -> None: + await asyncio.sleep(1) + future.set_result(42) # Error + +loop = asyncio.get_event_loop() +future = asyncio.Future() # type: Future[str] +asyncio.Task(slow_operation(future)) +loop.run_until_complete(future) +print(future.result()) +loop.close() +[out] +_program.py:7: error: Argument 1 to "set_result" of "Future" has incompatible type "int"; expected "str" + + +[case testErrorUsingDifferentFutureType] +from typing import Any +import asyncio +from asyncio import Future + +async def slow_operation(future: 'Future[int]') -> None: + await asyncio.sleep(1) + future.set_result(42) + +loop = asyncio.get_event_loop() +future = asyncio.Future() # type: Future[str] +asyncio.Task(slow_operation(future)) # Error +loop.run_until_complete(future) +print(future.result()) +loop.close() +[out] +_program.py:11: error: Argument 1 to "slow_operation" has incompatible type "Future[str]"; expected "Future[int]" + +[case testErrorUsingDifferentFutureTypeAndSetFutureDifferentInternalType] +from typing import Any +import asyncio +from asyncio import Future + +async def slow_operation(future: 'Future[int]') -> None: + await asyncio.sleep(1) + future.set_result('42') #Try to set an str as result to a Future[int] + +loop = asyncio.get_event_loop() +future = asyncio.Future() # type: Future[str] +asyncio.Task(slow_operation(future)) # Error +loop.run_until_complete(future) +print(future.result()) +loop.close() +[out] +_program.py:7: error: Argument 1 to "set_result" of "Future" has incompatible type "str"; expected "int" +_program.py:11: error: Argument 1 to "slow_operation" has incompatible type "Future[str]"; expected "Future[int]" + +[case testErrorSettingCallbackWithDifferentFutureType] +import typing +from typing import Any +import asyncio +from asyncio import Future, AbstractEventLoop + +async def slow_operation(future: 'Future[str]') -> None: + await asyncio.sleep(1) + future.set_result('Future is done!') + +def got_result(future: 'Future[int]') -> None: + print(future.result()) + loop.stop() + +loop = asyncio.get_event_loop() # type: AbstractEventLoop +future = asyncio.Future() # type: Future[str] +asyncio.Task(slow_operation(future)) +future.add_done_callback(got_result) # Error + +try: + loop.run_forever() +finally: + loop.close() +[out] +_program.py:17: error: Argument 1 to "add_done_callback" of "Future" has incompatible type "Callable[[Future[int]], None]"; expected "Callable[[Future[str]], object]" + +[case testErrorOneMoreFutureInReturnType] +import typing +from typing import Any, Generator +import asyncio +from asyncio import Future + +async def h4() -> Future[int]: + await asyncio.sleep(1) + f = asyncio.Future() #type: Future[int] + return f + +async def h3() -> Future[Future[Future[int]]]: + x = await h4() + x.set_result(42) + f = asyncio.Future() #type: Future[Future[int]] + f.set_result(x) + return f + +async def h() -> None: + print("Before") + x = await h3() + y = await x + z = await y + print(z) + print(y) + print(x) + +loop = asyncio.get_event_loop() +loop.run_until_complete(h()) +loop.close() +[out] +_program.py:16: error: Incompatible return value type (got "Future[Future[int]]", expected "Future[Future[Future[int]]]") + +[case testErrorOneLessFutureInReturnType] +import typing +from typing import Any +import asyncio +from asyncio import Future + +async def h4() -> Future[int]: + await asyncio.sleep(1) + f = asyncio.Future() #type: Future[int] + return f + +async def h3() -> Future[int]: + x = await h4() + x.set_result(42) + f = asyncio.Future() #type: Future[Future[int]] + f.set_result(x) + return f + +async def h() -> None: + print("Before") + x = await h3() + y = await x + print(y) + print(x) + +loop = asyncio.get_event_loop() +loop.run_until_complete(h()) +loop.close() +[out] +_program.py:16: error: Incompatible return value type (got "Future[Future[int]]", expected "Future[int]") +_program.py:16: note: Maybe you forgot to use "await"? + +[case testErrorAssignmentDifferentType] +import typing +from typing import Any +import asyncio +from asyncio import Future + +class A: + def __init__(self, x: int) -> None: + self.x = x + +class B: + def __init__(self, x: int) -> None: + self.x = x + +async def h() -> None: + x = await future # type: B # Error + print("h: %s" % x.x) + +loop = asyncio.get_event_loop() +future = asyncio.Future() # type: Future[A] +future.set_result(A(42)) +loop.run_until_complete(h()) +loop.close() +[out] +_program.py:15: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testForwardRefToBadAsyncShouldNotCrash_newsemanal] +from typing import TypeVar +import asyncio + +T = TypeVar('T') +P = whatever # type: ignore + +def test() -> None: + reveal_type(bad) + bad(0) + +async def bad(arg: P) -> T: + pass +[out] +_program.py:8: note: Revealed type is "def [T] (arg: P?) -> typing.Coroutine[Any, Any, T`-1]" +_program.py:9: error: Value of type "Coroutine[Any, Any, Never]" must be used +_program.py:9: note: Are you missing an await? +_program.py:11: error: Variable "_testForwardRefToBadAsyncShouldNotCrash_newsemanal.P" is not valid as a type +_program.py:11: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test new file mode 100644 index 000000000000..70003545754c --- /dev/null +++ b/test-data/unit/pythoneval.test @@ -0,0 +1,2183 @@ +-- Test cases for type checking mypy programs using full stubs and running +-- using CPython. +-- +-- These are mostly regression tests -- no attempt is made to make these +-- complete. + + +[case testHello] +import typing +print('hello, world') +[out] +hello, world + +[case testMiscStdlibFeatures] +# Various legacy tests merged together to speed up test runtimes. + +def f(x: object) -> None: pass + +# testReversed +from typing import Reversible +class R(Reversible): + def __iter__(self): return iter('oof') + def __reversed__(self): return iter('foo') +f(list(reversed(range(5)))) +f(list(reversed([1,2,3]))) +f(list(reversed('abc'))) +f(list(reversed(R()))) + +# testIntAndFloatConversion +from typing import SupportsInt, SupportsFloat +class A(SupportsInt): + def __int__(self): return 5 +class B(SupportsFloat): + def __float__(self): return 1.2 +f(int(1)) +f(int(6.2)) +f(int('3')) +f(int(b'4')) +f(int(A())) +f(float(-9)) +f(float(B())) + +# testAbs +from typing import SupportsAbs +class Ab(SupportsAbs[float]): + def __abs__(self) -> float: return 5.5 + +f(abs(-1)) +f(abs(-1.2)) +f(abs(Ab())) + +# testRound +from typing import SupportsRound +class Ro(SupportsRound): + def __round__(self, ndigits=0): return 'x%d' % ndigits +f(round(1.6)) +f(round(Ro())) +f(round(Ro(), 2)) + +# testCallMethodViaTypeObject +list.__add__([1, 2], [3, 4]) + +# testInheritedClassAttribute +import typing +class AA: + x = 1 + def f(self: typing.Optional["AA"]) -> None: pass +class BB(AA): + pass +BB.f(None) +f(BB.x) + +# testSpecialAttributes +class Doc: + """A docstring!""" +f(Doc().__doc__) +f(Doc().__class__) + +# testFunctionAttributes +f(ord.__class__) +f(type(ord.__doc__ or '' + '')) +f(ord.__name__) +f(ord.__module__) + +# testModuleAttributes +import math +f(type(__spec__)) +f(math.__name__) +f(math.__spec__.name) +f(type(math.__dict__)) +f(type(math.__doc__ or '')) +f(type(math.__spec__).__name__) +f(math.__class__) + +[case testAbs2] +n: int +f: float +n = abs(1) +abs(1) + 'x' # Error +f = abs(1.1) +abs(1.1) + 'x' # Error +[out] +_program.py:4: error: Unsupported operand types for + ("int" and "str") +_program.py:6: error: Unsupported operand types for + ("float" and "str") + +[case testTypeAttributes] +import typing +print(str.__class__) +print(type(str.__doc__)) +print(str.__name__) +print(str.__module__) +print(str.__dict__ is not None) +[out] + + +str +builtins +True + +[case testBoolCompatibilityWithInt] +import typing +x = 0 +x = True +print(bool('x')) +print(bool('')) +[out] +True +False + +[case testCannotExtendBoolUnlessIgnored] +class A(bool): pass +class B(bool): pass # type: ignore +[out] +_program.py:1: error: Cannot inherit from final class "bool" + +[case testCallBuiltinTypeObjectsWithoutArguments] +import typing +print(int()) +print(repr(str())) +print(repr(bytes())) +print(float()) +print(bool()) +[out] +0 +'' +b'' +0.0 +False + +[case testIntegerDivision] +import typing +x = 1 / 2 +x = 1.5 +[out] + +[case testIntMethods] +import typing +print(int.from_bytes(b'ab', 'big')) +n = 0 +print(n.from_bytes(b'ac', 'big')) +print(n.from_bytes([2, 3], 'big')) +print(n.to_bytes(2, 'big')) +[out] +24930 +24931 +515 +b'\x00\x00' + +[case testFloatMethods] +import typing +print(1.5.as_integer_ratio()) +print(1.5.hex()) +print(2.0.is_integer()) +print(float.fromhex('0x1.8')) +[out] +(3, 2) +0x1.8000000000000p+0 +True +1.5 + +[case testDictFromkeys] +import typing +d = dict.fromkeys('foo') +d['x'] = 2 +d2 = dict.fromkeys([1, 2], b'') +d2[2] = b'foo' +[out] + +[case testIsinstanceWithTuple] +from typing import cast, Any +x = cast(Any, (1, 'x')) +if isinstance(x, tuple): + print(x[0], x[1]) +[out] +1 x + +[case testAnyStr] +from typing import AnyStr +def f(x: AnyStr) -> AnyStr: + if isinstance(x, str): + return 'foo' + else: + return b'zar' +print(f('')) +print(f(b'')) +[out] +foo +b'zar' + +[case testNameNotImportedFromTyping] +import typing +cast(int, 2) +[out] +_program.py:2: error: Name "cast" is not defined +_program.py:2: note: Did you forget to import it from "typing"? (Suggestion: "from typing import cast") + +[case testBinaryIOType] +from typing import BinaryIO +def f(f: BinaryIO) -> None: + f.write(b'foo') + f.write(bytearray(b'foo')) +[out] + +[case testIOTypes] +from typing import IO +import sys +def txt(f: IO[str]) -> None: + f.write('foo') + f.write(b'foo') +def bin(f: IO[bytes]) -> None: + f.write(b'foo') + f.write(bytearray(b'foo')) +txt(sys.stdout) +bin(sys.stdout) +[out] +_program.py:5: error: Argument 1 to "write" of "IO" has incompatible type "bytes"; expected "str" +_program.py:10: error: Argument 1 to "bin" has incompatible type "Union[TextIO, Any]"; expected "IO[bytes]" + +[case testBuiltinOpen] +f = open('x') +f.write('x') +f.write(b'x') +f.foobar() +[out] +_program.py:3: error: Argument 1 to "write" of "_TextIOBase" has incompatible type "bytes"; expected "str" +_program.py:4: error: "TextIOWrapper[_WrappedBuffer]" has no attribute "foobar" + +[case testOpenReturnTypeInference] +reveal_type(open('x')) +reveal_type(open('x', 'r')) +reveal_type(open('x', 'rb')) +mode = 'rb' +reveal_type(open('x', mode)) +[out] +_program.py:1: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:2: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:3: note: Revealed type is "_io.BufferedReader" +_program.py:5: note: Revealed type is "typing.IO[Any]" + +[case testOpenReturnTypeInferenceSpecialCases] +reveal_type(open(mode='rb', file='x')) +reveal_type(open(file='x', mode='rb')) +mode = 'rb' +reveal_type(open(mode=mode, file='r')) +[out] +_testOpenReturnTypeInferenceSpecialCases.py:1: note: Revealed type is "_io.BufferedReader" +_testOpenReturnTypeInferenceSpecialCases.py:2: note: Revealed type is "_io.BufferedReader" +_testOpenReturnTypeInferenceSpecialCases.py:4: note: Revealed type is "typing.IO[Any]" + +[case testPathOpenReturnTypeInference] +from pathlib import Path +p = Path("x") +reveal_type(p.open()) +reveal_type(p.open('r')) +reveal_type(p.open('rb')) +mode = 'rb' +reveal_type(p.open(mode)) +[out] +_program.py:3: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:4: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:5: note: Revealed type is "_io.BufferedReader" +_program.py:7: note: Revealed type is "typing.IO[Any]" + +[case testPathOpenReturnTypeInferenceSpecialCases] +from pathlib import Path +p = Path("x") +reveal_type(p.open(mode='r', errors='replace')) +reveal_type(p.open(errors='replace', mode='r')) +mode = 'r' +reveal_type(p.open(mode=mode, errors='replace')) +[out] +_program.py:3: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:4: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:6: note: Revealed type is "typing.IO[Any]" + +[case testGenericPatterns] +from typing import Pattern +import re +p: Pattern[str] +p = re.compile('foo*') +b: Pattern[bytes] +b = re.compile(b'foo*') +m = p.match('fooo') +assert m +print(m.group(0)) +[out] +fooo + +[case testGenericMatch] +from typing import Match, Optional +import re +def f(m: Optional[Match[bytes]]) -> None: + assert m + print(m.group(0)) +f(re.match(b'x*', b'xxy')) +[out] +b'xx' + +[case testIntFloatDucktyping] +x: float +x = 2.2 +x = 2 +def f(x: float) -> None: pass +f(1.1) +f(1) +[out] + +[case testsFloatOperations] +import typing +print(1.5 + 1.5) +print(1.5 + 1) +[out] +3.0 +2.5 + +[case testMathFunctionWithIntArgument] +import typing +import math +math.sin(2) +math.sin(2.2) + +[case testAbsReturnType] +f: float +n: int +n = abs(2) +f = abs(2.2) +abs(2.2) + 'x' +[out] +_program.py:5: error: Unsupported operand types for + ("float" and "str") + +[case testROperatorMethods] +b: bytes +s: str +if int(): + s = b'foo' * 5 # Error +if int(): + b = 5 * b'foo' +if int(): + b = b'foo' * 5 +if int(): + s = 5 * 'foo' +if int(): + s = 'foo' * 5 +[out] +_program.py:4: error: Incompatible types in assignment (expression has type "bytes", variable has type "str") + +[case testROperatorMethods2] +import typing +print(2 / 0.5) +print(' ', 2 * [3, 4]) +[out] +4.0 + [3, 4, 3, 4] + +[case testNotImplemented] +import typing +class A: + def __add__(self, x: int) -> int: + if isinstance(x, int): + return x + 1 + return NotImplemented +class B: + def __radd__(self, x: A) -> str: + return 'x' +print(A() + 1) +print(A() + B()) +[out] +2 +x + +[case testMappingMethods] +# Regression test +from typing import Mapping +x = {'x': 'y'} # type: Mapping[str, str] +print('x' in x) +print('y' in x) +[out] +True +False + +[case testOverlappingOperatorMethods] +class X: pass +class A: + def __add__(self, x: object) -> int: + if isinstance(x, X): + return 1 + return NotImplemented +class B: + def __radd__(self, x: A) -> str: return 'x' +class C(X, B): pass +b: B +b = C() +print(A() + b) +[out] +_program.py:8: error: Signatures of "__radd__" of "B" and "__add__" of "A" are unsafely overlapping + +[case testBytesAndBytearrayComparisons] +import typing +print(b'ab' < bytearray(b'b')) +print(bytearray(b'ab') < b'a') +[out] +True +False + +[case testBytesAndBytearrayComparisons2] +import typing +'' < b'' +b'' < '' +'' < bytearray() +bytearray() < '' +[out] +_program.py:2: error: Unsupported operand types for < ("str" and "bytes") +_program.py:3: error: Unsupported operand types for < ("bytes" and "str") +_program.py:4: error: Unsupported operand types for < ("str" and "bytearray") +_program.py:5: error: Unsupported operand types for < ("bytearray" and "str") + +[case testInplaceOperatorMethod] +import typing +a = [1] +print('', a.__iadd__([2])) +print('', a) +[out] + [1, 2] + [1, 2] + +[case testListInplaceAdd] +import typing +a = [1] +a += iter([2, 3]) +print(tuple(a)) +[out] +(1, 2, 3) + +[case testInferHeterogeneousListOfIterables] +from typing import Sequence +s = ['x', 'y'] # type: Sequence[str] +a = [['x', 'x'], 'fo', s, iter('foo'), {'aa'}] +for i, x in enumerate(a): + print(i, next(iter(x))) +[out] +0 x +1 f +2 x +3 f +4 aa + +[case testTextIOProperties] +import typing +import sys +print(type(sys.stdin.encoding)) +print(type(sys.stdin.errors)) +sys.stdin.line_buffering +sys.stdin.buffer +sys.stdin.newlines +[out] + + + +[case testIOProperties] +import typing +import sys +print(sys.stdin.name) +print(sys.stdin.buffer.mode) +[out] + +rb + +[case testFromFuturePrintFunction] +from __future__ import print_function +print('a', 'b') +[out] +a b + +[case testListMethods] +import typing +import sys +l = [0, 1, 2, 3, 4] +if sys.version >= '3.3': + l.clear() +else: + l = [] +l.append(0) +print('>', l) +if sys.version >= '3.3': + m = l.copy() +else: + m = l[:] +m.extend([1, 2, 3, 4]) +print('>', m) +print(l.index(0)) +print(l.index(0, 0)) +print(l.index(0, 0, 1)) +try: + print(l.index(1)) + print('expected ValueError') +except ValueError: + pass +l.insert(0, 1) +print('>', l) +print(l.pop(0)) +print(l.pop()) +m.remove(0) +try: + m.remove(0) + print('expected ValueError') +except ValueError: + pass +m.reverse() +m.sort() +m.sort(key=lambda x: -x) +m.sort(reverse=False) +m.sort(key=lambda x: -x, reverse=True) +print('>', m) +[out] +> [0] +> [0, 1, 2, 3, 4] +0 +0 +0 +> [1, 0] +1 +0 +> [1, 2, 3, 4] + +[case testListOperators] +import typing +l = [0, 1] +print('+', l + [2]) +print('*', l * 2) +print('*', 2 * l) +print('in', 1 in l) +print('==', l == [1, 2]) +print('!=', l != [1, 2]) +print('>', l > [1, 2, 3]) +print('>=', l >= [1, 2, 3]) +print('<', l < [1, 2, 3]) +print('<=', l <= [1, 2, 3]) +print('>[0]', l[0]) +l += [2] +print('+=', l) +l *= 2 +print('*=', l) +print('iter', list(iter(l))) +print('len', len(l)) +print('repr', repr(l)) +l[:3] = [] +print('setslice', l) +print('reversed', list(reversed(l))) +[out] ++ [0, 1, 2] +* [0, 1, 0, 1] +* [0, 1, 0, 1] +in True +== False +!= True +> False +>= False +< True +<= True +>[0] 0 ++= [0, 1, 2] +*= [0, 1, 2, 0, 1, 2] +iter [0, 1, 2, 0, 1, 2] +len 6 +repr [0, 1, 2, 0, 1, 2] +setslice [0, 1, 2] +reversed [2, 1, 0] + +[case testTupleAsSubtypeOfSequence] +from typing import TypeVar, Sequence +T = TypeVar('T') +def f(a: Sequence[T]) -> None: print(a) +f(tuple()) +[out] +() + +[case testMapWithLambdaSpecialCase] +from typing import List, Iterator +a = [[1], [3]] +b = map(lambda y: y[0], a) +print('>', list(b)) +[out] +> [1, 3] + +[case testInternalBuiltinDefinition] +import typing +def f(x: _T) -> None: pass +s: FrozenSet +[out] +_program.py:2: error: Name "_T" is not defined +_program.py:3: error: Name "FrozenSet" is not defined + +[case testVarArgsFunctionSubtyping] +import typing +def f(*args: str) -> str: return args[0] +map(f, ['x']) +map(f, [1]) +[out] +_program.py:4: error: Argument 1 to "map" has incompatible type "Callable[[VarArg(str)], str]"; expected "Callable[[int], str]" + +[case testMapStr] +import typing +x = range(3) +a = list(map(str, x)) +a + 1 +[out] +_testMapStr.py:4: error: No overload variant of "__add__" of "list" matches argument type "int" +_testMapStr.py:4: note: Possible overload variants: +_testMapStr.py:4: note: def __add__(self, List[str], /) -> List[str] +_testMapStr.py:4: note: def [_S] __add__(self, List[_S], /) -> List[Union[_S, str]] + +[case testRelativeImport] +import typing +from m import x +print(x) +[file m/__init__.py] +from .n import x +[file m/n.py] +x = 1 +[out] +1 + +[case testRelativeImport2] +import typing +from m.n import x +print(x) +[file m/__init__.py] +[file m/n.py] +from .nn import x +[file m/nn.py] +x = 2 +[out] +2 + +[case testPyiTakesPrecedenceOverPy] +import m +m.f(1) +[file m.py] +def f(x): + print(x) +[file m.pyi] +import typing +def f(x: str) -> None: pass +[out] +_program.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" + +[case testComplexArithmetic] +import typing +print(5 + 8j) +print(3j * 2.0) +print(4J / 2.0) +[out] +(5+8j) +6j +2j + +[case testComplexArithmetic2] +x = 5 + 8j +if int(): + x = '' # E +y = 3j * 2.0 +if int(): + y = '' # E +[out] +_program.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "complex") +_program.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "complex") + +[case testSuperNew] +from typing import Dict, Any +class MyType(type): + def __new__(cls, name: str, bases: tuple, namespace: Dict[str, Any]) -> Any: + return super().__new__(cls, name + 'x', bases, namespace) +class A(metaclass=MyType): pass +print(type(A()).__name__) +[out] +Ax + +[case testSubclassBothGenericAndNonGenericABC] +from typing import Generic, TypeVar +from abc import ABCMeta +T = TypeVar('T') +class A(metaclass=ABCMeta): pass +class B(Generic[T]): pass +class C(A, B): pass +class D(B, A): pass +class E(A, B[T], Generic[T]): pass +class F(B[T], A, Generic[T]): pass +def f(e: E[int], f: F[int]) -> None: pass +[out] + +[case testTypeVariableTypeComparability] +from typing import TypeVar +T = TypeVar('T') +def eq(x: T, y: T, z: T) -> T: + if x == y: + return y + else: + return z +print(eq(1, 2, 3)) +print(eq('x', 'x', 'z')) +[out] +3 +x + +[case testIntDecimalCompatibility] +import typing +from decimal import Decimal +print(Decimal(1) + 2) +print(Decimal(1) - 2) +print(1 + Decimal('2.34')) +print(1 - Decimal('2.34')) +print(2 * Decimal('2.34')) +[out] +3 +-1 +3.34 +-1.34 +4.68 + +[case testInstantiateBuiltinTypes] +from typing import Dict, Set, List +d = dict() # type: Dict[int, str] +s = set() # type: Set[int] +l = list() # type: List[int] +str() +bytes() +bytearray() +int() +float() +complex() +slice(1) +bool() + +[case testVariableLengthTupleError] +from typing import Tuple +def p(t: Tuple[str, ...]) -> None: + n = 5 + print(t[n]) + for s in t: + s() +''.startswith(('x', 'y')) +''.startswith(('x', b'y')) +[out] +_program.py:6: error: "str" not callable +_program.py:8: error: Argument 1 to "startswith" of "str" has incompatible type "Tuple[str, bytes]"; expected "Union[str, Tuple[str, ...]]" + +[case testMultiplyTupleByInteger] +n = 4 +t = ('',) * n +t + 1 +[out] +_program.py:3: error: No overload variant of "__add__" of "tuple" matches argument type "int" +_program.py:3: note: Possible overload variants: +_program.py:3: note: def __add__(self, Tuple[str, ...], /) -> Tuple[str, ...] +_program.py:3: note: def [_T] __add__(self, Tuple[_T, ...], /) -> Tuple[Union[str, _T], ...] + +[case testMultiplyTupleByIntegerReverse] +n = 4 +t = n * ('',) +t + 1 +[out] +_program.py:3: error: No overload variant of "__add__" of "tuple" matches argument type "int" +_program.py:3: note: Possible overload variants: +_program.py:3: note: def __add__(self, Tuple[str, ...], /) -> Tuple[str, ...] +_program.py:3: note: def [_T] __add__(self, Tuple[_T, ...], /) -> Tuple[Union[str, _T], ...] + +[case testDictWithKeywordArgs] +from typing import Dict, Any, List +d1 = dict(a=1, b=2) # type: Dict[str, int] +d2 = dict(a=1, b='') # type: Dict[str, int] # E +d3 = dict(a=1, b=1) +d3.xyz # E +d4 = dict(a=1, b='') # type: Dict[str, Any] +result = dict(x=[], y=[]) # type: Dict[str, List[str]] +[out] +_program.py:3: error: Dict entry 1 has incompatible type "str": "str"; expected "str": "int" +_program.py:5: error: "Dict[str, int]" has no attribute "xyz" + +[case testDefaultDict] +# flags: --new-type-inference +import typing as t +from collections import defaultdict + +T = t.TypeVar('T') + +d1 = defaultdict(list) # type: t.DefaultDict[int, str] +d2 = defaultdict() # type: t.DefaultDict[int, str] +d2[0] = '0' +d2['0'] = 0 + +def tst(dct: t.DefaultDict[int, T]) -> T: + return dct[0] + +collections = ['coins', 'stamps', 'comics'] # type: t.List[str] +d3 = defaultdict(str) # type: t.DefaultDict[int, str] +collections[2] + +tst(defaultdict(list, {0: []})) +tst(defaultdict(list, {'0': []})) + +class MyDDict(t.DefaultDict[int,T], t.Generic[T]): + pass +MyDDict(dict)['0'] +MyDDict(dict)[0] +[out] +_program.py:7: error: Argument 1 to "defaultdict" has incompatible type "Type[List[Any]]"; expected "Optional[Callable[[], str]]" +_program.py:10: error: Invalid index type "str" for "defaultdict[int, str]"; expected type "int" +_program.py:10: error: Incompatible types in assignment (expression has type "int", target has type "str") +_program.py:20: error: Argument 1 to "tst" has incompatible type "defaultdict[str, List[Never]]"; expected "defaultdict[int, List[Never]]" +_program.py:24: error: Invalid index type "str" for "MyDDict[Dict[Never, Never]]"; expected type "int" + +[case testCollectionsAliases] +import typing as t +import collections as c + +o1 = c.Counter() # type: t.Counter[int] +reveal_type(o1) +o1['string'] + +o2 = c.ChainMap() # type: t.ChainMap[int, str] +reveal_type(o2) + +o3 = c.deque() # type: t.Deque[int] +reveal_type(o3) + +o4 = t.Counter[int]() +reveal_type(o4) + +o5 = t.ChainMap[int, str]() +reveal_type(o5) + +o6 = t.Deque[int]() +reveal_type(o6) + +[out] +_testCollectionsAliases.py:5: note: Revealed type is "collections.Counter[builtins.int]" +_testCollectionsAliases.py:6: error: Invalid index type "str" for "Counter[int]"; expected type "int" +_testCollectionsAliases.py:9: note: Revealed type is "collections.ChainMap[builtins.int, builtins.str]" +_testCollectionsAliases.py:12: note: Revealed type is "collections.deque[builtins.int]" +_testCollectionsAliases.py:15: note: Revealed type is "collections.Counter[builtins.int]" +_testCollectionsAliases.py:18: note: Revealed type is "collections.ChainMap[builtins.int, builtins.str]" +_testCollectionsAliases.py:21: note: Revealed type is "collections.deque[builtins.int]" + +[case testChainMapUnimported] +ChainMap[int, str]() + +[out] +_testChainMapUnimported.py:1: error: Name "ChainMap" is not defined + +[case testDequeWrongCase] +import collections +import typing + +collections.Deque() +typing.deque() + +[out] +_testDequeWrongCase.py:4: error: Module has no attribute "Deque"; maybe "deque"? +_testDequeWrongCase.py:5: error: Module has no attribute "deque"; maybe "Deque"? + +[case testDictUpdateInference] +from typing import Dict, Optional +d = {} # type: Dict[str, Optional[int]] +d.update({str(i): None for i in range(4)}) + +[case testSuperAndSetattr] +class A: + def __init__(self) -> None: + super().__setattr__('a', 1) + super().__setattr__(1, 'a') +[out] +_program.py:4: error: Argument 1 to "__setattr__" of "object" has incompatible type "int"; expected "str" + +[case testMetaclassAndSuper] +from typing import Any + +class A(type): + def __new__(cls, name, bases, namespace) -> Any: + return super().__new__(cls, '', (object,), {'x': 7}) + +class B(metaclass=A): + pass + +print(getattr(B(), 'x')) +[out] +7 + +[case testSortedNoError] +from typing import Iterable, Callable, TypeVar, List, Dict, Optional +T = TypeVar('T') +def sorted(x: Iterable[T], *, key: Optional[Callable[[T], object]] = None) -> None: ... +a = [] # type: List[Dict[str, str]] +sorted(a, key=lambda y: y['']) + +[case testAbstractProperty] +from abc import abstractproperty, ABCMeta # type: ignore[deprecated] +class A(metaclass=ABCMeta): + @abstractproperty + def x(self) -> int: pass +class B(A): + @property + def x(self) -> int: + return 3 +b = B() +print(b.x + 1) +[out] +4 + +[case testInferenceWithLambda] +from typing import TypeVar, Iterable, Iterator, List +import itertools + +_T = TypeVar('_T') + +def f(iterable): # type: (Iterable[_T]) -> Iterator[List[_T]] + grouped = itertools.groupby(enumerate(iterable), lambda pair: pair[0] // 2) + return ([elem for _, elem in group] for _, group in grouped) + +[case testReModuleBytes] +# Regression tests for various overloads in the re module -- bytes version +import re +bre = b'a+' +bpat = re.compile(bre) +bpat = re.compile(bpat) +s1 = re.search(bre, b'') +assert s1 +s1.groups() +re.search(bre, u'') # Error +s2 = re.search(bpat, b'') +assert s2 +s2.groups() +re.search(bpat, u'') # Error +# match(), split(), findall(), finditer() are much the same, so skip those. +# sub(), subn() have more overloads and we are checking these: +re.sub(bre, b'', b'') + b'' +re.sub(bpat, b'', b'') + b'' +re.sub(bre, lambda m: b'', b'') + b'' +re.sub(bpat, lambda m: b'', b'') + b'' +re.subn(bre, b'', b'')[0] + b'' +re.subn(bpat, b'', b'')[0] + b'' +re.subn(bre, lambda m: b'', b'')[0] + b'' +re.subn(bpat, lambda m: b'', b'')[0] + b'' +[out] +_testReModuleBytes.py:9: error: No overload variant of "search" matches argument types "bytes", "str" +_testReModuleBytes.py:9: note: Possible overload variants: +_testReModuleBytes.py:9: note: def search(pattern: Union[str, Pattern[str]], string: str, flags: Union[int, RegexFlag] = ...) -> Optional[Match[str]] +_testReModuleBytes.py:9: note: def search(pattern: Union[bytes, Pattern[bytes]], string: Buffer, flags: Union[int, RegexFlag] = ...) -> Optional[Match[bytes]] +_testReModuleBytes.py:13: error: Argument 1 to "search" has incompatible type "Pattern[bytes]"; expected "Union[str, Pattern[str]]" + +[case testReModuleString] +# Regression tests for various overloads in the re module -- string version +import re +sre = 'a+' +spat = re.compile(sre) +spat = re.compile(spat) +s1 = re.search(sre, '') +assert s1 +s1.groups() +re.search(sre, b'') # Error +s2 = re.search(spat, '') +assert s2 +s2.groups() +re.search(spat, b'') # Error +# match(), split(), findall(), finditer() are much the same, so skip those. +# sus(), susn() have more overloads and we are checking these: +re.sub(sre, '', '') + '' +re.sub(spat, '', '') + '' +re.sub(sre, lambda m: '', '') + '' +re.sub(spat, lambda m: '', '') + '' +re.subn(sre, '', '')[0] + '' +re.subn(spat, '', '')[0] + '' +re.subn(sre, lambda m: '', '')[0] + '' +re.subn(spat, lambda m: '', '')[0] + '' +[out] +_testReModuleString.py:9: error: No overload variant of "search" matches argument types "str", "bytes" +_testReModuleString.py:9: note: Possible overload variants: +_testReModuleString.py:9: note: def search(pattern: Union[str, Pattern[str]], string: str, flags: Union[int, RegexFlag] = ...) -> Optional[Match[str]] +_testReModuleString.py:9: note: def search(pattern: Union[bytes, Pattern[bytes]], string: Buffer, flags: Union[int, RegexFlag] = ...) -> Optional[Match[bytes]] +_testReModuleString.py:13: error: Argument 1 to "search" has incompatible type "Pattern[str]"; expected "Union[bytes, Pattern[bytes]]" + +[case testListSetitemTuple] +from typing import List, Tuple +a = [] # type: List[Tuple[str, int]] +a[0] = 'x', 1 +a[1] = 2, 'y' +a[:] = [('z', 3)] +[out] +_program.py:4: error: Incompatible types in assignment (expression has type "Tuple[int, str]", target has type "Tuple[str, int]") + +[case testContextManager] +import contextlib +from contextlib import contextmanager +from typing import Iterator + +@contextmanager +def f(x: int) -> Iterator[str]: + yield 'foo' + +@contextlib.contextmanager +def g(*x: str) -> Iterator[int]: + yield 1 + +reveal_type(f) +reveal_type(g) + +with f('') as s: + reveal_type(s) +[out] +_program.py:13: note: Revealed type is "def (x: builtins.int) -> contextlib._GeneratorContextManager[builtins.str]" +_program.py:14: note: Revealed type is "def (*x: builtins.str) -> contextlib._GeneratorContextManager[builtins.int]" +_program.py:16: error: Argument 1 to "f" has incompatible type "str"; expected "int" +_program.py:17: note: Revealed type is "builtins.str" + +[case testTypedDictGet] +# Test that TypedDict get plugin works with typeshed stubs +from mypy_extensions import TypedDict +class A: pass +D = TypedDict('D', {'x': int, 'y': str}) +d: D +reveal_type(d.get('x')) +reveal_type(d.get('y')) +reveal_type(d.get('z')) +d.get() +s = '' +reveal_type(d.get(s)) +[out] +_testTypedDictGet.py:6: note: Revealed type is "Union[builtins.int, None]" +_testTypedDictGet.py:7: note: Revealed type is "Union[builtins.str, None]" +_testTypedDictGet.py:8: note: Revealed type is "builtins.object" +_testTypedDictGet.py:9: error: All overload variants of "get" of "Mapping" require at least one argument +_testTypedDictGet.py:9: note: Possible overload variants: +_testTypedDictGet.py:9: note: def get(self, str, /) -> object +_testTypedDictGet.py:9: note: def [_T] get(self, str, /, default: object) -> object +_testTypedDictGet.py:11: note: Revealed type is "builtins.object" + +[case testTypedDictMappingMethods] +from mypy_extensions import TypedDict +Cell = TypedDict('Cell', {'value': int}) +c = Cell(value=42) +for x in c: + reveal_type(x) +reveal_type(iter(c)) +reveal_type(len(c)) +reveal_type('value' in c) +reveal_type(c.keys()) +reveal_type(c.items()) +reveal_type(c.values()) +reveal_type(c.copy()) +reveal_type(c.setdefault('value', False)) +c.update({'value': 2}) +c.update({'invalid': 2}) +c.pop('value') +c == c +c != c +Cell2 = TypedDict('Cell2', {'value': int}, total=False) +c2 = Cell2() +reveal_type(c2.pop('value')) +[out] +_testTypedDictMappingMethods.py:5: note: Revealed type is "builtins.str" +_testTypedDictMappingMethods.py:6: note: Revealed type is "typing.Iterator[builtins.str]" +_testTypedDictMappingMethods.py:7: note: Revealed type is "builtins.int" +_testTypedDictMappingMethods.py:8: note: Revealed type is "builtins.bool" +_testTypedDictMappingMethods.py:9: note: Revealed type is "_collections_abc.dict_keys[builtins.str, builtins.object]" +_testTypedDictMappingMethods.py:10: note: Revealed type is "_collections_abc.dict_items[builtins.str, builtins.object]" +_testTypedDictMappingMethods.py:11: note: Revealed type is "_collections_abc.dict_values[builtins.str, builtins.object]" +_testTypedDictMappingMethods.py:12: note: Revealed type is "TypedDict('_testTypedDictMappingMethods.Cell', {'value': builtins.int})" +_testTypedDictMappingMethods.py:13: note: Revealed type is "builtins.int" +_testTypedDictMappingMethods.py:15: error: Unexpected TypedDict key "invalid" +_testTypedDictMappingMethods.py:16: error: Key "value" of TypedDict "Cell" cannot be deleted +_testTypedDictMappingMethods.py:21: note: Revealed type is "builtins.int" + +[case testCrashOnComplexCheckWithNamedTupleNext] +from typing import NamedTuple, Optional + +MyNamedTuple = NamedTuple('MyNamedTuple', [('parent', 'MyNamedTuple')]) # type: ignore +def foo(mymap) -> Optional[MyNamedTuple]: + return next((mymap[key] for key in mymap), None) +[out] + +[case testCanConvertTypedDictToAnySuperclassOfMapping] +from mypy_extensions import TypedDict +from typing import Sized, Iterable, Container + +Point = TypedDict('Point', {'x': int, 'y': int}) + +p: Point +s: Sized = p +it: Iterable[str] = p +c: Container[str] = p +o: object = p +it2: Iterable[int] = p +[out] +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: error: Incompatible types in assignment (expression has type "Point", variable has type "Iterable[int]") +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Following member(s) of "Point" have conflicts: +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Expected: +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: def __iter__(self) -> Iterator[int] +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Got: +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: def __iter__(self) -> Iterator[str] + +[case testAsyncioGatherPreciseType-xfail] +# Mysteriously regressed in #11905 +import asyncio +from typing import Tuple + +async def get_location(arg: str) -> Tuple[str, str]: + return arg, arg + +async def main() -> None: + ((a_x, a_y),) = await asyncio.gather(get_location('start')) + reveal_type(a_x) + reveal_type(a_y) +reveal_type(asyncio.gather(*[asyncio.sleep(1), asyncio.sleep(1)])) +[out] +_testAsyncioGatherPreciseType.py:9: note: Revealed type is "builtins.str" +_testAsyncioGatherPreciseType.py:10: note: Revealed type is "builtins.str" +_testAsyncioGatherPreciseType.py:11: note: Revealed type is "asyncio.futures.Future[builtins.list[Any]]" + +[case testMultipleInheritanceWorksWithTupleTypeGeneric] +from typing import SupportsAbs, NamedTuple + +class Point(NamedTuple('Point', [('x', int), ('y', int)]), SupportsAbs[int]): + def __abs__(p) -> int: + return abs(p.x) + abs(p.y) + +def test(a: Point) -> bool: + return abs(a) == 2 +[out] + +[case testNoCrashOnGenericUnionUnpacking] +from typing import Union, Dict + +TEST = {'key': ('a', 'b')} +def test() -> None: + a, b = TEST.get('foo', ('x', 'y')) + reveal_type(a) + reveal_type(b) +def test2() -> None: + a, b = TEST.get('foo', (1, 2)) + reveal_type(a) + reveal_type(b) + +x: Union[Dict[int, int], Dict[str, str]] = dict(a='b') +for a, b in x.items(): + reveal_type(a) + reveal_type(b) +[out] +_testNoCrashOnGenericUnionUnpacking.py:6: note: Revealed type is "builtins.str" +_testNoCrashOnGenericUnionUnpacking.py:7: note: Revealed type is "builtins.str" +_testNoCrashOnGenericUnionUnpacking.py:10: note: Revealed type is "Union[builtins.str, builtins.int]" +_testNoCrashOnGenericUnionUnpacking.py:11: note: Revealed type is "Union[builtins.str, builtins.int]" +_testNoCrashOnGenericUnionUnpacking.py:15: note: Revealed type is "Union[builtins.int, builtins.str]" +_testNoCrashOnGenericUnionUnpacking.py:16: note: Revealed type is "Union[builtins.int, builtins.str]" + +[case testMetaclassOpAccess] +from typing import Type +class A: + pass + +class Meta(type): + def __mul__(self, other: int) -> Type[A]: + pass + def __add__(self, other: int) -> Type[C]: + pass + def __radd__(self, other: int) -> Type[C]: + pass +class C(metaclass=Meta): + pass + +bar: Type[C] +def get_c_type() -> Type[C]: + pass + +res = bar * 4 +other = 4 + get_c_type() + 5 +reveal_type(res) +reveal_type(other) +[out] +_testMetaclassOpAccess.py:21: note: Revealed type is "Type[_testMetaclassOpAccess.A]" +_testMetaclassOpAccess.py:22: note: Revealed type is "Type[_testMetaclassOpAccess.C]" + +[case testMetaclassOpAccessUnion] +from typing import Type, Union + +class MetaA(type): + def __mul__(self, other: int) -> str: + pass +class A(metaclass=MetaA): + pass +class MetaB(type): + def __mul__(self, other: int) -> int: + pass +class B(metaclass=MetaB): + pass + +bar: Type[Union[A, B]] +res = bar * 4 +reveal_type(res) +[out] +_testMetaclassOpAccessUnion.py:16: note: Revealed type is "Union[builtins.str, builtins.int]" + +[case testMetaclassOpAccessAny] +from typing import Type +from nonexistent import C +bar: Type[C] + +bar * 4 + bar + 3 # should not produce more errors +[out] +_testMetaclassOpAccessAny.py:2: error: Cannot find implementation or library stub for module named "nonexistent" +_testMetaclassOpAccessAny.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testEnumIterationAndPreciseElementType] +# Regression test for #2305 +from enum import Enum +class E(Enum): + A = 'a' +(reveal_type(e) for e in E) +for e in E: + reveal_type(e) +[out] +_testEnumIterationAndPreciseElementType.py:5: note: Revealed type is "_testEnumIterationAndPreciseElementType.E" +_testEnumIterationAndPreciseElementType.py:7: note: Revealed type is "_testEnumIterationAndPreciseElementType.E" + +[case testEnumIterable] +from enum import Enum +from typing import Iterable +class E(Enum): + A = 'a' +def f(ie: Iterable[E]): + pass +f(E) + +[case testIntEnumIterable] +from enum import IntEnum +from typing import Iterable +class N(IntEnum): + X = 1 +def f(ni: Iterable[N]): + pass +def g(ii: Iterable[int]): + pass +f(N) +g(N) +reveal_type(list(N)) +[out] +_testIntEnumIterable.py:11: note: Revealed type is "builtins.list[_testIntEnumIterable.N]" + +[case testDerivedEnumIterable] +from enum import Enum +from typing import Iterable +class E(str, Enum): + A = 'foo' +def f(ei: Iterable[E]): + pass +def g(si: Iterable[str]): + pass +f(E) +g(E) + +[case testInvalidSlots] +from typing import List +class A: + __slots__ = 1 +class B: + __slots__ = (1, 2) +class C: + __slots__: List[int] = [] +[out] +_testInvalidSlots.py:3: error: Invalid type for "__slots__" (actual type "int", expected type "Union[str, Iterable[str]]") +_testInvalidSlots.py:5: error: Invalid type for "__slots__" (actual type "Tuple[int, int]", expected type "Union[str, Iterable[str]]") +_testInvalidSlots.py:7: error: Invalid type for "__slots__" (actual type "List[int]", expected type "Union[str, Iterable[str]]") + +[case testDictWithStarStarSpecialCase] +from typing import Dict + +def f() -> Dict[int, str]: + return {1: '', **d()} + +def d() -> Dict[int, int]: + return {} +[out] +_testDictWithStarStarSpecialCase.py:4: error: Unpacked dict entry 1 has incompatible type "Dict[int, int]"; expected "SupportsKeysAndGetItem[int, str]" + +[case testLoadsOfOverloads] +from typing import overload, Any, TypeVar, Iterable, List, Dict, Callable, Union + +S = TypeVar('S') +T = TypeVar('T') + +@overload +def simple_map() -> None: ... +@overload +def simple_map(func: Callable[[T], S], one: Iterable[T]) -> S: ... +@overload +def simple_map(func: Callable[..., S], *iterables: Iterable[Any]) -> S: ... +def simple_map(*args): pass + +def format_row(*entries: object) -> str: pass + +class DateTime: pass +JsonBlob = Dict[str, Any] +Column = Union[List[str], List[int], List[bool], List[float], List[DateTime], List[JsonBlob]] + +def print_custom_table() -> None: + a: Column + + for row in simple_map(format_row, a, a, a, a, a, a, a, a): # 8 columns + reveal_type(row) +[out] +_testLoadsOfOverloads.py:24: note: Revealed type is "builtins.str" + +[case testReduceWithAnyInstance] +from typing import Iterable +from functools import reduce +M = Iterable +def f(m1: M, m2): + ... +def g(ms: 'T[M]') -> None: + reduce(f, ms) +T = Iterable +[out] + +[case testNamedTupleNew] +# This is an eval test because there was a snag found only with full stubs +from typing import NamedTuple + +Base = NamedTuple('Base', [('param', int)]) + +class Child(Base): + def __new__(cls, param: int = 1) -> 'Child': + return Base.__new__(cls, param) + +Base(param=10) +Child(param=10) +reveal_type(Child()) + +from collections import namedtuple +X = namedtuple('X', ['a', 'b']) +x = X(a=1, b='s') + +[out] +_testNamedTupleNew.py:12: note: Revealed type is "Tuple[builtins.int, fallback=_testNamedTupleNew.Child]" + +[case testNamedTupleTypeInheritanceSpecialCase] +from typing import NamedTuple, Tuple +from collections import namedtuple + +A = NamedTuple('A', [('param', int)]) +B = namedtuple('B', ['param']) + +def accepts_named_tuple(arg: NamedTuple): + reveal_type(arg._asdict()) + reveal_type(arg._fields) + reveal_type(arg._field_defaults) + +a = A(1) +b = B(1) + +accepts_named_tuple(a) +accepts_named_tuple(b) +accepts_named_tuple(1) +accepts_named_tuple((1, 2)) +[out] +_testNamedTupleTypeInheritanceSpecialCase.py:8: note: Revealed type is "builtins.dict[builtins.str, Any]" +_testNamedTupleTypeInheritanceSpecialCase.py:9: note: Revealed type is "builtins.tuple[builtins.str, ...]" +_testNamedTupleTypeInheritanceSpecialCase.py:10: note: Revealed type is "builtins.dict[builtins.str, Any]" +_testNamedTupleTypeInheritanceSpecialCase.py:17: error: Argument 1 to "accepts_named_tuple" has incompatible type "int"; expected "NamedTuple" +_testNamedTupleTypeInheritanceSpecialCase.py:18: error: Argument 1 to "accepts_named_tuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" + +[case testNewAnalyzerBasicTypeshed_newsemanal] +from typing import Dict, List, Tuple + +x: Dict[str, List[int]] +reveal_type(x['test'][0]) +[out] +_testNewAnalyzerBasicTypeshed_newsemanal.py:4: note: Revealed type is "builtins.int" + +[case testNewAnalyzerTypedDictInStub_newsemanal] +import stub +reveal_type(stub.thing) + +[file stub.pyi] +from typing_extensions import TypedDict + +class StuffDict(TypedDict): + foo: str + bar: int + +def thing(stuff: StuffDict) -> int: ... + +[out] +_testNewAnalyzerTypedDictInStub_newsemanal.py:2: note: Revealed type is "def (stuff: TypedDict('stub.StuffDict', {'foo': builtins.str, 'bar': builtins.int})) -> builtins.int" + +[case testStrictEqualityAllowlist] +# mypy: strict-equality +{1} == frozenset({1}) +frozenset({1}) == {1} + +frozenset({1}) == [1] # Error + +{1: 2}.keys() == {1} +{1: 2}.keys() == frozenset({1}) +{1: 2}.items() == {(1, 2)} + +{1: 2}.keys() == {'no'} # OK +{1: 2}.values() == {2} # Error +{1: 2}.keys() == [1] # OK +[out] +_testStrictEqualityAllowlist.py:5: error: Non-overlapping equality check (left operand type: "FrozenSet[int]", right operand type: "List[int]") +_testStrictEqualityAllowlist.py:12: error: Non-overlapping equality check (left operand type: "dict_values[int, int]", right operand type: "Set[int]") + +[case testUnreachableWithStdlibContextManagers] +# mypy: warn-unreachable, strict-optional + +from contextlib import suppress + +# This test overlaps with some of the warn-unreachable tests in check-unreachable-code, +# but 'open(...)' is a very common function so we want to make sure we don't regress +# against it specifically +def f_open() -> str: + with open("foo.txt", "r") as f: + return f.read() + print("noop") + +# contextlib.suppress is less common, but it's a fairly prominent example of an +# exception-suppressing context manager, so it'd be good to double-check. +def f_suppresses() -> int: + with suppress(Exception): + return 3 + print("noop") +[out] +_testUnreachableWithStdlibContextManagers.py:11: error: Statement is unreachable +_testUnreachableWithStdlibContextManagers.py:15: error: Missing return statement + +[case testUnreachableWithStdlibContextManagersNoStrictOptional] +# mypy: warn-unreachable, no-strict-optional + +from contextlib import suppress + +# When strict-optional is disabled, 'open' should still behave in the same way as before +def f_open() -> str: + with open("foo.txt", "r") as f: + return f.read() + print("noop") + +# ...but unfortunately, we can't +def f_suppresses() -> int: + with suppress(Exception): + return 3 + print("noop") +[out] +_testUnreachableWithStdlibContextManagersNoStrictOptional.py:9: error: Statement is unreachable +_testUnreachableWithStdlibContextManagersNoStrictOptional.py:15: error: Statement is unreachable + +[case testIsInstanceAdHocIntersectionWithStrAndBytes] +# mypy: warn-unreachable +x: str +if isinstance(x, bytes): + reveal_type(x) +y: str +if isinstance(x, int): + reveal_type(x) +[out] +_testIsInstanceAdHocIntersectionWithStrAndBytes.py:3: error: Subclass of "str" and "bytes" cannot exist: would have incompatible method signatures +_testIsInstanceAdHocIntersectionWithStrAndBytes.py:4: error: Statement is unreachable +_testIsInstanceAdHocIntersectionWithStrAndBytes.py:6: error: Subclass of "str" and "int" cannot exist: would have incompatible method signatures +_testIsInstanceAdHocIntersectionWithStrAndBytes.py:7: error: Statement is unreachable + +[case testAsyncioFutureWait] +# mypy: strict-optional +from asyncio import Future, wait +from typing import List + +async def foo() -> None: + f = [] # type: List[Future[None]] + await wait(f) + +[case testShadowTypingModule] +1 + '' +[file typing.py] +x = 0 +1 + '' +[out] +mypy: "tmp/typing.py" shadows library module "typing" +note: A user-defined top-level module with name "typing" is not supported + +[case testIgnoreImportIfNoPython3StubAvailable] +# flags: --ignore-missing-imports +import scribe # No Python 3 stubs available for scribe +from scribe import x +import python2 # Python 3 stubs available for python2 +import foobar_asdf +import jack # This has a stubs package but was never bundled with mypy, so ignoring works +[out] +_testIgnoreImportIfNoPython3StubAvailable.py:4: error: Library stubs not installed for "python2" +_testIgnoreImportIfNoPython3StubAvailable.py:4: note: Hint: "python3 -m pip install types-six" +_testIgnoreImportIfNoPython3StubAvailable.py:4: note: (or run "mypy --install-types" to install all missing stub packages) +_testIgnoreImportIfNoPython3StubAvailable.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testNoPython3StubAvailable] +import scribe +from scribe import x +import python2 +[out] +_testNoPython3StubAvailable.py:1: error: Cannot find implementation or library stub for module named "scribe" +_testNoPython3StubAvailable.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +_testNoPython3StubAvailable.py:3: error: Library stubs not installed for "python2" +_testNoPython3StubAvailable.py:3: note: Hint: "python3 -m pip install types-six" +_testNoPython3StubAvailable.py:3: note: (or run "mypy --install-types" to install all missing stub packages) + + +[case testTypingOrderedDictAlias] +from typing import OrderedDict +x: OrderedDict[str, int] = OrderedDict({}) +reveal_type(x) +[out] +_testTypingOrderedDictAlias.py:3: note: Revealed type is "collections.OrderedDict[builtins.str, builtins.int]" + +[case testTypingExtensionsOrderedDictAlias] +from typing_extensions import OrderedDict +x: OrderedDict[str, str] = OrderedDict({}) +reveal_type(x) # Revealed type is "collections.OrderedDict[builtins.str, builtins.int]" +[out] +_testTypingExtensionsOrderedDictAlias.py:3: note: Revealed type is "collections.OrderedDict[builtins.str, builtins.str]" + +[case testSpecialTypingProtocols] +# flags: --warn-unreachable +from typing import Awaitable, Hashable, Union, Tuple, List + +obj: Union[Tuple[int], List[int]] +if isinstance(obj, Hashable): + reveal_type(obj) +if isinstance(obj, Awaitable): + reveal_type(obj) +[out] +_testSpecialTypingProtocols.py:6: note: Revealed type is "Tuple[builtins.int]" +_testSpecialTypingProtocols.py:8: error: Statement is unreachable + +[case testTypeshedRecursiveTypesExample] +from typing import List, Union + +Recursive = Union[str, List["Recursive"]] + +def foo(r: Recursive) -> None: + if not isinstance(r, str): + if r: + foo(r[0]) + if not isinstance(r, list): + r.casefold() + +foo("") +foo(list("")) +foo(list((list(""), ""))) +[out] + +[case testNarrowTypeForDictKeys] +from typing import Dict, KeysView, Optional + +d: Dict[str, int] +key: Optional[str] +if key in d.keys(): + reveal_type(key) +else: + reveal_type(key) + +kv: KeysView[str] +k: Optional[str] +if k in kv: + reveal_type(k) +else: + reveal_type(k) + +[out] +_testNarrowTypeForDictKeys.py:6: note: Revealed type is "builtins.str" +_testNarrowTypeForDictKeys.py:8: note: Revealed type is "Union[builtins.str, None]" +_testNarrowTypeForDictKeys.py:13: note: Revealed type is "builtins.str" +_testNarrowTypeForDictKeys.py:15: note: Revealed type is "Union[builtins.str, None]" + +[case testTypeAliasWithNewStyleUnion] +# flags: --python-version 3.10 +from typing import Literal, Type, TypeAlias, TypeVar + +Foo = Literal[1, 2] +reveal_type(Foo) +Bar1 = Foo | Literal[3] +Bar2 = Literal[3] | Foo +Bar3 = Foo | Foo | Literal[3] | Foo + +U1 = int | str +U2 = U1 | bytes +U3 = bytes | U1 + +Opt1 = None | int +Opt2 = None | float +Opt3 = int | None +Opt4 = float | None + +A = Type[int] | str +B: TypeAlias = Type[int] | str +C = type[int] | str + +D = type[int] | str +x: D +reveal_type(x) +E: TypeAlias = type[int] | str +y: E +reveal_type(y) +F = list[type[int] | str] + +T = TypeVar("T", int, str) +def foo(x: T) -> T: + A = type[int] | str + a: A + return x +[out] +_testTypeAliasWithNewStyleUnion.py:5: note: Revealed type is "typing._SpecialForm" +_testTypeAliasWithNewStyleUnion.py:25: note: Revealed type is "Union[Type[builtins.int], builtins.str]" +_testTypeAliasWithNewStyleUnion.py:28: note: Revealed type is "Union[Type[builtins.int], builtins.str]" + +[case testTypeAliasWithNewStyleUnionInStub] +import m +a: m.A +reveal_type(a) +b: m.B +reveal_type(b) +c: m.C +reveal_type(c) +d: m.D +reveal_type(d) +e: m.E +reveal_type(e) +f: m.F +reveal_type(f) + +[file m.pyi] +from typing import Type, Callable +from typing_extensions import Literal, TypeAlias + +Foo = Literal[1, 2] +reveal_type(Foo) +Bar1 = Foo | Literal[3] +Bar2 = Literal[3] | Foo +Bar3 = Foo | Foo | Literal[3] | Foo + +U1 = int | str +U2 = U1 | bytes +U3 = bytes | U1 + +Opt1 = None | int +Opt2 = None | float +Opt3 = int | None +Opt4 = float | None + +A = Type[int] | str +B: TypeAlias = Type[int] | str +C = type[int] | str +reveal_type(C) +D: TypeAlias = type[int] | str +E = str | type[int] +F: TypeAlias = str | type[int] +G = list[type[int] | str] +H = list[str | type[int]] + +CU1 = int | Callable[[], str | bool] +CU2: TypeAlias = int | Callable[[], str | bool] +CU3 = int | Callable[[str | bool], str] +CU4: TypeAlias = int | Callable[[str | bool], str] +[out] +m.pyi:5: note: Revealed type is "typing._SpecialForm" +m.pyi:22: note: Revealed type is "typing._SpecialForm" +_testTypeAliasWithNewStyleUnionInStub.py:3: note: Revealed type is "Union[Type[builtins.int], builtins.str]" +_testTypeAliasWithNewStyleUnionInStub.py:5: note: Revealed type is "Union[Type[builtins.int], builtins.str]" +_testTypeAliasWithNewStyleUnionInStub.py:7: note: Revealed type is "Union[Type[builtins.int], builtins.str]" +_testTypeAliasWithNewStyleUnionInStub.py:9: note: Revealed type is "Union[Type[builtins.int], builtins.str]" +_testTypeAliasWithNewStyleUnionInStub.py:11: note: Revealed type is "Union[builtins.str, Type[builtins.int]]" +_testTypeAliasWithNewStyleUnionInStub.py:13: note: Revealed type is "Union[builtins.str, Type[builtins.int]]" + +[case testEnumNameWorkCorrectlyOn311] +# flags: --python-version 3.11 +import enum + +class E(enum.Enum): + X = 1 + Y = 2 + @enum.property + def foo(self) -> int: ... + +e: E +reveal_type(e.name) +reveal_type(e.value) +reveal_type(E.X.name) +reveal_type(e.foo) +reveal_type(E.Y.foo) +[out] +_testEnumNameWorkCorrectlyOn311.py:11: note: Revealed type is "builtins.str" +_testEnumNameWorkCorrectlyOn311.py:12: note: Revealed type is "Union[Literal[1]?, Literal[2]?]" +_testEnumNameWorkCorrectlyOn311.py:13: note: Revealed type is "Literal['X']?" +_testEnumNameWorkCorrectlyOn311.py:14: note: Revealed type is "builtins.int" +_testEnumNameWorkCorrectlyOn311.py:15: note: Revealed type is "builtins.int" + +[case testTypeAliasNotSupportedWithNewStyleUnion] +# flags: --python-version 3.9 +from typing_extensions import TypeAlias +A = type[int] | str +B = str | type[int] +C = str | int +D: TypeAlias = str | int +[out] +_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Invalid type alias: expression is not a valid type +_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Unsupported left operand type for | ("GenericAlias") +_testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Invalid type alias: expression is not a valid type +_testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Unsupported left operand type for | ("Type[str]") +_testTypeAliasNotSupportedWithNewStyleUnion.py:5: error: Invalid type alias: expression is not a valid type +_testTypeAliasNotSupportedWithNewStyleUnion.py:5: error: Unsupported left operand type for | ("Type[str]") +_testTypeAliasNotSupportedWithNewStyleUnion.py:6: error: Invalid type alias: expression is not a valid type +_testTypeAliasNotSupportedWithNewStyleUnion.py:6: error: Unsupported left operand type for | ("Type[str]") + +[case testTypedDictUnionGetFull] +from typing import Dict +from typing_extensions import TypedDict + +class TD(TypedDict, total=False): + x: int + y: int + +A = Dict[str, TD] +x: A +def foo(k: str) -> TD: + reveal_type(x.get(k, {})) + return x.get(k, {}) +[out] +_testTypedDictUnionGetFull.py:11: note: Revealed type is "TypedDict('_testTypedDictUnionGetFull.TD', {'x'?: builtins.int, 'y'?: builtins.int})" + +[case testTupleWithDifferentArgsPy310] +# https://github.com/python/mypy/issues/11098 +# flags: --python-version 3.10 +Correct1 = str | tuple[float, float, str] +Correct2 = tuple[float] | str +Correct3 = tuple[float, ...] | str +Correct4 = tuple[float, str] +Correct5 = tuple[float, ...] +Correct6 = list[tuple[int, str]] +c1: Correct1 +c2: Correct2 +c3: Correct3 +c4: Correct4 +c5: Correct5 +c6: Correct6 +reveal_type(c1) +reveal_type(c2) +reveal_type(c3) +reveal_type(c4) +reveal_type(c5) +reveal_type(c6) + +RHSAlias1: type = tuple[int, int] +RHSAlias2: type = tuple[int] +RHSAlias3: type = tuple[int, ...] + +WrongTypeElement = str | tuple[float, 1] # Error +WrongEllipsis = tuple[float, float, ...] | str # Error + +reveal_type(tuple[int, str]((1, "x"))) +[out] +_testTupleWithDifferentArgsPy310.py:15: note: Revealed type is "Union[builtins.str, Tuple[builtins.float, builtins.float, builtins.str]]" +_testTupleWithDifferentArgsPy310.py:16: note: Revealed type is "Union[Tuple[builtins.float], builtins.str]" +_testTupleWithDifferentArgsPy310.py:17: note: Revealed type is "Union[builtins.tuple[builtins.float, ...], builtins.str]" +_testTupleWithDifferentArgsPy310.py:18: note: Revealed type is "Tuple[builtins.float, builtins.str]" +_testTupleWithDifferentArgsPy310.py:19: note: Revealed type is "builtins.tuple[builtins.float, ...]" +_testTupleWithDifferentArgsPy310.py:20: note: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" +_testTupleWithDifferentArgsPy310.py:26: error: Invalid type: try using Literal[1] instead? +_testTupleWithDifferentArgsPy310.py:27: error: Unexpected "..." +_testTupleWithDifferentArgsPy310.py:29: note: Revealed type is "Tuple[builtins.int, builtins.str]" + +[case testEnumIterMetaInference] +import socket +from enum import Enum +from typing import Iterable, Iterator, Type, TypeVar + +_E = TypeVar("_E", bound=Enum) + +def enum_iter(cls: Type[_E]) -> Iterable[_E]: + reveal_type(iter(cls)) + reveal_type(next(iter(cls))) + return iter(cls) + +for value in enum_iter(socket.SocketKind): + reveal_type(value) +[out] +_testEnumIterMetaInference.py:8: note: Revealed type is "typing.Iterator[_E`-1]" +_testEnumIterMetaInference.py:9: note: Revealed type is "_E`-1" +_testEnumIterMetaInference.py:13: note: Revealed type is "socket.SocketKind" + +[case testEnumUnpackedViaMetaclass] +from enum import Enum + +class FooEnum(Enum): + A = 1 + B = 2 + C = 3 + +a, b, c = FooEnum +reveal_type(a) +reveal_type(b) +reveal_type(c) +[out] +_testEnumUnpackedViaMetaclass.py:9: note: Revealed type is "_testEnumUnpackedViaMetaclass.FooEnum" +_testEnumUnpackedViaMetaclass.py:10: note: Revealed type is "_testEnumUnpackedViaMetaclass.FooEnum" +_testEnumUnpackedViaMetaclass.py:11: note: Revealed type is "_testEnumUnpackedViaMetaclass.FooEnum" + +[case testNativeIntTypes] +# Spot check various native int operations with full stubs. +from mypy_extensions import i64, i32 + +x: i64 = 0 +y: int = x +x = i64(0) +y = int(x) +i64() +i64("12") +i64("ab", 16) +i64(1.2) +float(i64(1)) + +i64(1) + i32(2) # Error +reveal_type(x + y) +reveal_type(y + x) +a = [0] +a[x] +[out] +_testNativeIntTypes.py:14: error: Unsupported operand types for + ("i64" and "i32") +_testNativeIntTypes.py:15: note: Revealed type is "mypy_extensions.i64" +_testNativeIntTypes.py:16: note: Revealed type is "mypy_extensions.i64" + +[case testStarUnpackNestedUnderscore] +from typing import Tuple, Dict, List + +def crash() -> None: + d: Dict[int, Tuple[str, int, str]] = {} + k, (v1, *_) = next(iter(d.items())) + +def test1() -> None: + vs: List[str] + d: Dict[int, Tuple[str, int, int]] = {} + k, (v1, *vs) = next(iter(d.items())) + reveal_type(vs) + +def test2() -> None: + d: Dict[int, Tuple[str, int, str]] = {} + k, (v1, *vs) = next(iter(d.items())) + reveal_type(vs) +[out] +_testStarUnpackNestedUnderscore.py:10: error: List item 0 has incompatible type "int"; expected "str" +_testStarUnpackNestedUnderscore.py:10: error: List item 1 has incompatible type "int"; expected "str" +_testStarUnpackNestedUnderscore.py:11: note: Revealed type is "builtins.list[builtins.str]" +_testStarUnpackNestedUnderscore.py:16: note: Revealed type is "builtins.list[builtins.object]" + +[case testStrictEqualitywithParamSpec] +# flags: --strict-equality +from typing import Generic +from typing_extensions import Concatenate, ParamSpec + +P = ParamSpec("P") + +class Foo(Generic[P]): ... +class Bar(Generic[P]): ... + +def bad(foo: Foo[[int]], bar: Bar[[int]]) -> bool: + return foo == bar + +def bad1(foo1: Foo[[int]], foo2: Foo[[str]]) -> bool: + return foo1 == foo2 + +def bad2(foo1: Foo[[int, str]], foo2: Foo[[int, bytes]]) -> bool: + return foo1 == foo2 + +def bad3(foo1: Foo[[int]], foo2: Foo[[int, int]]) -> bool: + return foo1 == foo2 + +def good4(foo1: Foo[[int]], foo2: Foo[[int]]) -> bool: + return foo1 == foo2 + +def good5(foo1: Foo[[int]], foo2: Foo[[bool]]) -> bool: + return foo1 == foo2 + +def good6(foo1: Foo[[int, int]], foo2: Foo[[bool, bool]]) -> bool: + return foo1 == foo2 + +def good7(foo1: Foo[[int]], foo2: Foo[P], *args: P.args, **kwargs: P.kwargs) -> bool: + return foo1 == foo2 + +def good8(foo1: Foo[P], foo2: Foo[[int, str, bytes]], *args: P.args, **kwargs: P.kwargs) -> bool: + return foo1 == foo2 + +def good9(foo1: Foo[Concatenate[int, P]], foo2: Foo[[int, str, bytes]], *args: P.args, **kwargs: P.kwargs) -> bool: + return foo1 == foo2 + +[out] +_testStrictEqualitywithParamSpec.py:11: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Bar[[int]]") +_testStrictEqualitywithParamSpec.py:14: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Foo[[str]]") +_testStrictEqualitywithParamSpec.py:17: error: Non-overlapping equality check (left operand type: "Foo[[int, str]]", right operand type: "Foo[[int, bytes]]") +_testStrictEqualitywithParamSpec.py:20: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Foo[[int, int]]") + +[case testInferenceOfDunderDictOnClassObjects] +class Foo: ... +reveal_type(Foo.__dict__) +reveal_type(Foo().__dict__) +Foo.__dict__ = {} +Foo().__dict__ = {} + +[out] +_testInferenceOfDunderDictOnClassObjects.py:2: note: Revealed type is "types.MappingProxyType[builtins.str, Any]" +_testInferenceOfDunderDictOnClassObjects.py:3: note: Revealed type is "builtins.dict[builtins.str, Any]" +_testInferenceOfDunderDictOnClassObjects.py:4: error: Property "__dict__" defined in "type" is read-only +_testInferenceOfDunderDictOnClassObjects.py:4: error: Incompatible types in assignment (expression has type "Dict[Never, Never]", variable has type "MappingProxyType[str, Any]") + +[case testTypeVarTuple] +# flags: --python-version=3.11 +from typing import Any, Callable, Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def foo(callback: Callable[[], Any]) -> None: + call(callback) + +def call(callback: Callable[[Unpack[Ts]], Any], *args: Unpack[Ts]) -> Any: + ... + +[case testTypeVarTupleTypingExtensions] +from typing_extensions import Unpack, TypeVarTuple +from typing import Any, Callable + +Ts = TypeVarTuple("Ts") + +def foo(callback: Callable[[], Any]) -> None: + call(callback) + +def call(callback: Callable[[Unpack[Ts]], Any], *args: Unpack[Ts]) -> Any: + ... + +[case testDataclassReplace] +from dataclasses import dataclass, replace + +@dataclass +class A: + x: int + +a = A(x=42) +a2 = replace(a, x=42) +reveal_type(a2) +a2 = replace() +a2 = replace(a, x='spam') +a2 = replace(a, x=42, q=42) +[out] +_testDataclassReplace.py:9: note: Revealed type is "_testDataclassReplace.A" +_testDataclassReplace.py:10: error: Too few arguments for "replace" +_testDataclassReplace.py:11: error: Argument "x" to "replace" of "A" has incompatible type "str"; expected "int" +_testDataclassReplace.py:12: error: Unexpected keyword argument "q" for "replace" of "A" + +[case testGenericInferenceWithTuple] +# flags: --new-type-inference +from typing import TypeVar, Callable, Tuple + +T = TypeVar("T") + +def f(x: Callable[..., T]) -> T: + return x() + +x: Tuple[str, ...] = f(tuple) +[out] + +[case testGenericInferenceWithDataclass] +# flags: --new-type-inference +from typing import Any, Collection, List +from dataclasses import dataclass, field + +class Foo: + pass + +@dataclass +class A: + items: Collection[Foo] = field(default_factory=list) +[out] + +[case testGenericInferenceWithItertools] +# flags: --new-type-inference +from typing import TypeVar, Tuple +from itertools import groupby +K = TypeVar("K") +V = TypeVar("V") + +def fst(kv: Tuple[K, V]) -> K: + k, v = kv + return k + +pairs = [(len(s), s) for s in ["one", "two", "three"]] +grouped = groupby(pairs, key=fst) +[out] + +[case testDataclassReplaceOptional] +from dataclasses import dataclass, replace +from typing import Optional + +@dataclass +class A: + x: Optional[int] + +a = A(x=42) +reveal_type(a) +a2 = replace(a, x=None) # OK +reveal_type(a2) +[out] +_testDataclassReplaceOptional.py:9: note: Revealed type is "_testDataclassReplaceOptional.A" +_testDataclassReplaceOptional.py:11: note: Revealed type is "_testDataclassReplaceOptional.A" + +[case testDataclassStrictOptionalAlwaysSet] +from dataclasses import dataclass +from typing import Callable, Optional + +@dataclass +class Description: + name_fn: Callable[[Optional[int]], Optional[str]] + +def f(d: Description) -> None: + reveal_type(d.name_fn) +[out] +_testDataclassStrictOptionalAlwaysSet.py:9: note: Revealed type is "def (Union[builtins.int, None]) -> Union[builtins.str, None]" + +[case testPEP695VarianceInference] +# flags: --python-version=3.12 +from typing import Callable, Final + +class Job[_R_co]: + def __init__(self, target: Callable[[], _R_co]) -> None: + self.target: Final = target + +def func( + action: Job[int | None], + a1: Job[int | None], + a2: Job[int], + a3: Job[None], +) -> None: + action = a1 + action = a2 + action = a3 + a2 = action # Error +[out] +_testPEP695VarianceInference.py:17: error: Incompatible types in assignment (expression has type "Job[None]", variable has type "Job[int]") + +[case testPEP695TypeAliasWithDifferentTargetTypes] +# flags: --python-version=3.12 +from typing import Any, Callable, List, Literal, TypedDict, overload, TypeAlias, TypeVar, Never + +class C[T]: pass + +class O[T]: + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, x: int) -> None: ... + def __init__(self, x: int = 0) -> None: + pass + +class TD(TypedDict): + x: int + +S = TypeVar("S") +A = list[S] +B: TypeAlias = list[S] + +type A1 = type[int] +type A2 = type[int] | None +type A3 = None | type[int] +type A4 = type[Any] +type A5 = type[C] | None +type A6 = None | type[C] +type A7 = type[O] | None +type A8 = None | type[O] + +type B1[**P, R] = Callable[P, R] | None +type B2[**P, R] = None | Callable[P, R] +type B3 = Callable[[str], int] +type B4 = Callable[..., int] + +type C1 = A1 | None +type C2 = None | A1 + +type D1 = Any | None +type D2 = None | Any + +type E1 = List[int] +type E2 = List[int] | None +type E3 = None | List[int] + +type F1 = Literal[1] +type F2 = Literal['x'] | None +type F3 = None | Literal[True] + +type G1 = tuple[int, Any] +type G2 = tuple[int, Any] | None +type G3 = None | tuple[int, Any] + +type H1 = TD +type H2 = TD | None +type H3 = None | TD + +type I1 = C[int] +type I2 = C[Any] | None +type I3 = None | C[TD] +type I4 = O[int] | None +type I5 = None | O[int] + +type J1[T] = T | None +type J2[T] = None | T +type J3[*Ts] = tuple[*Ts] +type J4[T] = J1[T] | None +type J5[T] = None | J1[T] +type J6[*Ts] = J3[*Ts] | None + +type K1 = A[int] | None +type K2 = None | A[int] +type K3 = B[int] | None +type K4 = None | B[int] + +type L1 = Never +type L2 = list[Never] + +[case testPEP695VarianceInferenceSpecialCaseWithTypeshed] +# flags: --python-version=3.12 +class C1[T1, T2](list[T1]): + def m(self, a: T2) -> None: ... + +def func1(p: C1[int, object]): + x: C1[int, int] = p + +class C2[T1, T2, T3](dict[T2, T3]): + def m(self, a: T1) -> None: ... + +def func2(p: C2[object, int, int]): + x: C2[int, int, int] = p + +class C3[T1, T2](tuple[T1, ...]): + def m(self, a: T2) -> None: ... + +def func3(p: C3[int, object]): + x: C3[int, int] = p + + +[case testDynamicClassAttribute] +# Some things that can break if DynamicClassAttribute isn't handled properly +from types import DynamicClassAttribute +from enum import Enum + +class TestClass: + @DynamicClassAttribute + def name(self) -> str: ... + +class TestClass2(TestClass, Enum): ... + +class Status(Enum): + ABORTED = -1 + +def imperfect(status: Status) -> str: + return status.name.lower() diff --git a/test-data/unit/ref-info.test b/test-data/unit/ref-info.test new file mode 100644 index 000000000000..05426130d272 --- /dev/null +++ b/test-data/unit/ref-info.test @@ -0,0 +1,83 @@ +[case testCallGlobalFunction] +def f() -> None: + g() + +def g() -> None: + pass +[out] +2:4:__main__.g + +[case testCallMethod] +def f() -> None: + c = C() + if int(): + c.method() + +class C: + def method(self) -> None: pass +[out] +2:8:__main__.C +3:7:builtins.int +4:8:__main__.C.method + +[case testCallStaticMethod] +class C: + def f(self) -> None: + C.static() + self.static() + + @classmethod + def cm(cls) -> None: + cls.static() + + @staticmethod + def static() -> None: pass +[builtins fixtures/classmethod.pyi] +[out] +3:8:__main__.C +3:8:__main__.C.static +4:8:__main__.C.static +8:8:__main__.C.static + +[case testCallClassMethod] +class C: + def f(self) -> None: + C.cm() + self.cm() + + @classmethod + def cm(cls) -> None: + cls.cm() +[builtins fixtures/classmethod.pyi] +[out] +3:8:__main__.C +3:8:__main__.C.cm +4:8:__main__.C.cm +8:8:__main__.C.cm + +[case testTypeVarWithValueRestriction] +from typing import TypeVar + +T = TypeVar("T", "C", "D") + +def f(o: T) -> None: + f(o) + o.m() + o.x + +class C: + x: int + def m(self) -> None: pass + +class D: + x: str + def m(self) -> None: pass +[out] +3:4:typing.TypeVar +3:0:__main__.T +6:4:__main__.f +7:4:__main__.C.m +8:4:__main__.C.x +6:4:__main__.f +7:4:__main__.D.m +8:4:__main__.D.x diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test new file mode 100644 index 000000000000..6e0fdba8aaa3 --- /dev/null +++ b/test-data/unit/reports.test @@ -0,0 +1,552 @@ +-- Tests for reports +-- +-- This file follows syntax of cmdline.test. + +[case testConfigErrorUnknownReport] +# cmd: mypy -c pass +[file mypy.ini] +\[mypy] +bad_report = . +[out] +mypy.ini: [mypy]: Unrecognized report type: bad_report +== Return code: 0 + +[case testCoberturaParser] +# cmd: mypy --cobertura-xml-report build pkg +[file pkg/__init__.py] +[file pkg/a.py] +from typing import Dict + +def foo() -> Dict: + z = {'hello': 'world'} + return z +[file pkg/subpkg/__init__.py] +[file pkg/subpkg/a.py] +def bar() -> str: + return 'world' +def untyped_function(): + return 42 +[outfile build/cobertura.xml] + + + $PWD + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +[case testCoberturaStarUnpacking] +# cmd: mypy --cobertura-xml-report build a.py +[file a.py] +from typing import TypedDict + +class MyDict(TypedDict): + a: int + +def foo(a: int) -> MyDict: + return {"a": a} +md: MyDict = MyDict(**foo(42)) +[outfile build/cobertura.xml] + + + $PWD + + + + + + + + + + + + + + + + + + + + +[case testAnyExprReportDivisionByZero] +# cmd: mypy --any-exprs-report=out -c 'pass' + +[case testClassDefIsNotTreatedAsEmpty] +# cmd: mypy --html-report report n.py +[file n.py] +class A(object): + pass # line indented with tab; hex 1f here: () + +[file report/mypy-html.css] +[file report/index.html] +[outfile report/html/n.py.html] + + + + + + +

n

+ + + + + + +
n.py
1
+2
+
class A(object):
+	pass  # line indented with tab; hex 1f here: (?)
+
+ + + +[case testNoCrashRecursiveAliasInReport] +# cmd: mypy --any-exprs-report report n.py + +[file n.py] +from typing import Union, List, Any, TypeVar + +Nested = List[Union[Any, Nested]] +T = TypeVar("T") +NestedGen = List[Union[T, NestedGen[T]]] + +x: Nested +y: NestedGen[int] +z: NestedGen[Any] + +[file report/any-exprs.txt] +[outfile report/types-of-anys.txt] + Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact +----------------------------------------------------------------------------------------------------------------- + n 0 2 0 8 0 0 0 +----------------------------------------------------------------------------------------------------------------- +Total 0 2 0 8 0 0 0 + +[case testTypeVarTreatedAsEmptyLine] +# cmd: mypy --html-report report n.py + +[file n.py] +from typing import TypeVar + +T = TypeVar('T') + +[file report/mypy-html.css] +[file report/index.html] +[outfile report/html/n.py.html] + + + + + + +

n

+ + + + + + +
n.py
1
+2
+3
+
from typing import TypeVar
+
+T = TypeVar('T')
+
+ + + +[case testUnreachableCodeMarkedAsAny] +# cmd: mypy --html-report report n.py + +[file any.py] +from typing import Any +def any_f(x: Any) -> None: + pass + +[file n.py] +from any import any_f +def bar(x): + # type: (str) -> None + any_f(x) + assert False + any_f(x) + +[file report/mypy-html.css] +[file report/index.html] +[outfile report/html/n.py.html] + + + + + + +

n

+ + + + + + +
n.py
1
+2
+3
+4
+5
+6
+
from any import any_f
+def bar(x):
+    # type: (str) -> None
+    any_f(x)
+    assert False
+    any_f(x)
+
+ + + +[case testHtmlReportMemberExprNoUnanalyzed] +# cmd: mypy --html-report report n.py + +[file n.py] +import sys + +old_stdout = sys.stdout + +[file report/mypy-html.css] +[file report/index.html] +[outfile report/html/n.py.html] + + + + + + +

n

+ + + + + + +
n.py
1
+2
+3
+
import sys
+
+old_stdout = sys.stdout
+
+ + + +[case testAnyExprReportIncludesDeadCode] +# cmd: mypy --any-exprs-report report i.py j.py + +[file i.py] +def bar(x): + # type: (str) -> None + print(x) + assert False + print(x) # dead code! + +[file j.py] +def bar(x): + # type: (str) -> None + print(x) + assert False + +[file report/types-of-anys.txt] +[outfile report/any-exprs.txt] + Name Anys Exprs Coverage +--------------------------------- + i 1 6 83.33% + j 0 5 100.00% +--------------------------------- +Total 1 11 90.91% + +[case testAnyExprReportHigherKindedTypesAreNotAny] +# cmd: mypy --any-exprs-report report i.py + +[file i.py] +from enum import Enum +from mypy_extensions import TypedDict +from typing import NewType, NamedTuple, TypeVar + +from typing import TypeVar + +T = TypeVar('T') # no error + +def f(t: T) -> T: + return t + +Point = NamedTuple('Point', [('x', int), ('y', int)]) # no error + +def origin() -> Point: + return Point(x=0, y=0) + +NT = NewType('NT', int) # no error + +def nt() -> NT: + return NT(1) + +E = Enum('E', '1, 2, 3') # no error + +def k(s: E) -> None: pass + +Movie = TypedDict('Movie', {'name': str, 'year': int}) + +def g(m: Movie) -> Movie: + return m + +[file report/types-of-anys.txt] +[outfile report/any-exprs.txt] + Name Anys Exprs Coverage +--------------------------------- + i 0 14 100.00% +--------------------------------- +Total 0 14 100.00% + + +[case testAnyExpressionsReportTypesOfAny] +# cmd: mypy --any-exprs-report report n.py + +[file n.py] +from typing import Any, List +from nonexistent import C # type: ignore + +def any_f(x: Any) -> None: # Explicit + pass + +def a(x) -> None: # Unannotated + any_f(x) + +x: Any = 2 # Explicit +y: C = None # Unimported + +def b() -> List: # Omitted Generics + return [1, 2, 3] + +g = 1 +z = g.does_not_exist() # type: ignore # Error + + +[file report/any-exprs.txt] +[outfile report/types-of-anys.txt] + Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact +----------------------------------------------------------------------------------------------------------------- + n 2 3 1 1 3 0 0 +----------------------------------------------------------------------------------------------------------------- +Total 2 3 1 1 3 0 0 + +[case testAnyExpressionsReportUnqualifiedError] +# cmd: mypy --any-exprs-report report n.py + +[file n.py] +z = does_not_exist() # type: ignore # Error + +[file report/any-exprs.txt] +[outfile report/types-of-anys.txt] + Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact +----------------------------------------------------------------------------------------------------------------- + n 0 0 0 0 3 0 0 +----------------------------------------------------------------------------------------------------------------- +Total 0 0 0 0 3 0 0 + +[case testAnyExpressionsReportUntypedDef] +# cmd: mypy --any-exprs-report report n.py + +[file n.py] +def foo(): + x = 0 + f = 0 + +[file report/any-exprs.txt] +[outfile report/types-of-anys.txt] + Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact +----------------------------------------------------------------------------------------------------------------- + n 0 0 0 0 0 0 0 +----------------------------------------------------------------------------------------------------------------- +Total 0 0 0 0 0 0 0 +[case testTrickyCoverage] +# cmd: mypy --linecoverage-report=report n.py +[file n.py] +def blah(x): return x + +@blah + +def f(x: int) -> None: pass + +class Foo: + @blah + #hi + def f(self, x: int) -> None: + pass + +@blah +class Z(object): + pass + +[case testCoverageIgnoresCache] +-- Performs two runs to verify that cached information does not prevent +-- modules from being included in reports. +# cmd: mypy --linecount-report report a.py +[file a.py] +empty = False +[out] +[out2] +[outfile report/linecount.txt] + 1 1 0 0 total + 1 1 0 0 a + +[case testAnyExprReportIgnoresSpecialForms] +# cmd: mypy --any-exprs-report report i.py j.py k.py l.py + +[file i.py] +async def some_function() -> None: + pass + +[file j.py] +from typing import Any + +async def some_function() -> Any: + pass + +[file k.py] +from typing import NamedTuple + +def a() -> None: + _FuzzyMatch(0, 0) + +_FuzzyMatch = NamedTuple('_FuzzyMatch', [ + ('match_length', int), + ('start_pos', int), +]) + +def b() -> None: + _FuzzyMatch(0, 0) + +[file l.py] +async def some_function(x) -> None: + pass + +[file report/any-exprs.txt] +[outfile report/types-of-anys.txt] + Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact +----------------------------------------------------------------------------------------------------------------- + i 0 0 0 0 0 0 0 + j 0 1 0 0 0 0 0 + k 0 0 0 0 0 0 0 + l 1 0 0 0 0 0 0 +----------------------------------------------------------------------------------------------------------------- +Total 1 1 0 0 0 0 0 + +[case testSpecialAnyHtmlReport] +# cmd: mypy --html-report report n.py +[file n.py] +from typing import Callable + +SourceToDisplay = Callable[[int], int] +DisplayToSource = Callable[[int], int] + +[file report/mypy-html.css] +[file report/index.html] +[outfile report/html/n.py.html] + + + + + + +

n

+ + + + + + +
n.py
1
+2
+3
+4
+
from typing import Callable
+
+SourceToDisplay = Callable[[int], int]
+DisplayToSource = Callable[[int], int]
+
+ + + +[case testHtmlReportOnNamespacePackagesWithExplicitBases] +# cmd: mypy --html-report report -p folder +[file folder/subfolder/something.py] +class Something: + pass +[file folder/main.py] +from .subfolder.something import Something +print(Something()) +[file folder/__init__.py] +[file mypy.ini] +\[mypy] +explicit_package_bases = True +namespace_packages = True + +[file report/mypy-html.css] +[file report/index.html] +[outfile report/html/folder/subfolder/something.py.html] + + + + + + +

folder.subfolder.something

+ + + + + + +
folder/subfolder/something.py
1
+2
+
class Something:
+    pass
+
+ + diff --git a/mypy/test/data/semanal-abstractclasses.test b/test-data/unit/semanal-abstractclasses.test similarity index 75% rename from mypy/test/data/semanal-abstractclasses.test rename to test-data/unit/semanal-abstractclasses.test index 59e83fb3db34..b0cb00e82106 100644 --- a/mypy/test/data/semanal-abstractclasses.test +++ b/test-data/unit/semanal-abstractclasses.test @@ -9,11 +9,11 @@ class A(metaclass=ABCMeta): def f(self) -> 'A': return self [out] MypyFile:1( - ImportFrom:1(abc, [abstractmethod : abstractmethod, ABCMeta : ABCMeta]) - Import:2(typing : typing) + ImportFrom:1(abc, [abstractmethod, ABCMeta]) + Import:2(typing) ClassDef:4( A - Metaclass(ABCMeta) + Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) Decorator:5( Var(g) FuncDef:6( @@ -45,15 +45,15 @@ class B(metaclass=ABCMeta): pass class C(A, B): pass [out] MypyFile:1( - ImportFrom:1(abc, [abstractmethod : abstractmethod, ABCMeta : ABCMeta]) - Import:2(typing : typing) + ImportFrom:1(abc, [abstractmethod, ABCMeta]) + Import:2(typing) ClassDef:4( A - Metaclass(ABCMeta) + Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) PassStmt:4()) ClassDef:5( B - Metaclass(ABCMeta) + Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) PassStmt:5()) ClassDef:6( C @@ -64,22 +64,22 @@ MypyFile:1( [case testAbstractGenericClass] from abc import abstractmethod -from typing import AbstractGeneric, typevar -T = typevar('T') -class A(AbstractGeneric[T]): +from typing import Generic, TypeVar +T = TypeVar('T') +class A(Generic[T]): @abstractmethod def f(self) -> 'A[T]': pass [out] MypyFile:1( - ImportFrom:1(abc, [abstractmethod : abstractmethod]) - ImportFrom:2(typing, [AbstractGeneric : AbstractGeneric, typevar : typevar]) + ImportFrom:1(abc, [abstractmethod]) + ImportFrom:2(typing, [Generic, TypeVar]) AssignmentStmt:3( NameExpr(T* [__main__.T]) TypeVarExpr:3()) ClassDef:4( A TypeVars( - T) + T`1) Decorator:5( Var(f) FuncDef:6( @@ -101,12 +101,12 @@ class A(metaclass=ABCMeta): def g(self) -> 'A': pass [out] MypyFile:1( - Import:1(abc : abc) - ImportFrom:2(abc, [ABCMeta : ABCMeta]) - Import:3(typing : typing) + Import:1(abc) + ImportFrom:2(abc, [ABCMeta]) + Import:3(typing) ClassDef:5( A - Metaclass(ABCMeta) + Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) Decorator:6( Var(g) FuncDef:7( diff --git a/test-data/unit/semanal-basic.test b/test-data/unit/semanal-basic.test new file mode 100644 index 000000000000..169769f06a00 --- /dev/null +++ b/test-data/unit/semanal-basic.test @@ -0,0 +1,527 @@ +[case testEmptyFile] +[out] +MypyFile:1() + +[case testGlobalVariable] +x = 1 +x +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + ExpressionStmt:2( + NameExpr(x [__main__.x]))) + +[case testMultipleGlobals] +x = y = 2 +z = 3 +(x, y, z) +[out] +MypyFile:1( + AssignmentStmt:1( + Lvalues( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + IntExpr(2)) + AssignmentStmt:2( + NameExpr(z [__main__.z]) + IntExpr(3) + builtins.int) + ExpressionStmt:3( + TupleExpr:3( + NameExpr(x [__main__.x]) + NameExpr(y [__main__.y]) + NameExpr(z [__main__.z])))) + +[case testEmptyFunction] +def f(): pass +f() +[out] +MypyFile:1( + FuncDef:1( + f + Block:1( + PassStmt:1())) + ExpressionStmt:2( + CallExpr:2( + NameExpr(f [__main__.f]) + Args()))) + +[case testAccessingGlobalNameBeforeDefinition] +# flags: --disable-error-code used-before-def +x +f() +x = 1 +def f(): pass +[out] +MypyFile:1( + ExpressionStmt:2( + NameExpr(x [__main__.x])) + ExpressionStmt:3( + CallExpr:3( + NameExpr(f [__main__.f]) + Args())) + AssignmentStmt:4( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + FuncDef:5( + f + Block:5( + PassStmt:5()))) + +[case testFunctionArgs] +def f(x, y): + (x, y) +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x) + Var(y)) + Block:2( + ExpressionStmt:2( + TupleExpr:2( + NameExpr(x [l]) + NameExpr(y [l])))))) + +[case testLocalVar] +def f(): + x = 1 + x +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + AssignmentStmt:2( + NameExpr(x* [l]) + IntExpr(1)) + ExpressionStmt:3( + NameExpr(x [l]))))) + +[case testAccessGlobalInFn] +def f(): + x + g() +x = 1 +def g(): pass +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ExpressionStmt:2( + NameExpr(x [__main__.x])) + ExpressionStmt:3( + CallExpr:3( + NameExpr(g [__main__.g]) + Args())))) + AssignmentStmt:4( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + FuncDef:5( + g + Block:5( + PassStmt:5()))) + +[case testAssignmentAfterInit] +x = 1 +x = 2 +def f(y): + y = 1 + z = 1 + z = 2 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + IntExpr(2)) + FuncDef:3( + f + Args( + Var(y)) + Block:4( + AssignmentStmt:4( + NameExpr(y [l]) + IntExpr(1)) + AssignmentStmt:5( + NameExpr(z* [l]) + IntExpr(1)) + AssignmentStmt:6( + NameExpr(z [l]) + IntExpr(2))))) + +[case testLocalAndGlobalAliasing] +x = 1 +def f(): + x = 2 + x +x +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + FuncDef:2( + f + Block:3( + AssignmentStmt:3( + NameExpr(x* [l]) + IntExpr(2)) + ExpressionStmt:4( + NameExpr(x [l])))) + ExpressionStmt:5( + NameExpr(x [__main__.x]))) + +[case testArgumentInitializers] +def f(x = f, y = object): + x, y +[out] +MypyFile:1( + FuncDef:1( + f + Args( + default( + Var(x) + NameExpr(f [__main__.f])) + default( + Var(y) + NameExpr(object [builtins.object]))) + Block:2( + ExpressionStmt:2( + TupleExpr:2( + NameExpr(x [l]) + NameExpr(y [l])))))) + +[case testVarArgs] +def f(x, *y): + x, y +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + VarArg( + Var(y)) + Block:2( + ExpressionStmt:2( + TupleExpr:2( + NameExpr(x [l]) + NameExpr(y [l])))))) + +[case testGlobalDecl] +x = None +def f(): + global x + x = None + x +class A: pass +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x* [__main__.x]) + NameExpr(None [builtins.None])) + FuncDef:2( + f + Block:3( + GlobalDecl:3( + x) + AssignmentStmt:4( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None])) + ExpressionStmt:5( + NameExpr(x [__main__.x])))) + ClassDef:6( + A + PassStmt:6())) + +[case testMultipleNamesInGlobalDecl] +x, y = None, None +def f(): + global x, y + x = y +[out] +MypyFile:1( + AssignmentStmt:1( + TupleExpr:1( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + TupleExpr:1( + NameExpr(None [builtins.None]) + NameExpr(None [builtins.None]))) + FuncDef:2( + f + Block:3( + GlobalDecl:3( + x + y) + AssignmentStmt:4( + NameExpr(x [__main__.x]) + NameExpr(y [__main__.y]))))) + +[case testGlobalDeclScope] +x = None +def f(): + global x +def g(): + x = None +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x* [__main__.x]) + NameExpr(None [builtins.None])) + FuncDef:2( + f + Block:3( + GlobalDecl:3( + x))) + FuncDef:4( + g + Block:5( + AssignmentStmt:5( + NameExpr(x* [l]) + NameExpr(None [builtins.None]))))) + +[case testGlobalDeclScope2] +x = None +def f(): + global x +def g(): + x = None +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x* [__main__.x]) + NameExpr(None [builtins.None])) + FuncDef:2( + f + Block:3( + GlobalDecl:3( + x))) + FuncDef:4( + g + Block:5( + AssignmentStmt:5( + NameExpr(x* [l]) + NameExpr(None [builtins.None]))))) + +[case testGlobaWithinMethod] +x = None +class A: + def f(self): + global x + x = self +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x* [__main__.x]) + NameExpr(None [builtins.None])) + ClassDef:2( + A + FuncDef:3( + f + Args( + Var(self)) + Block:4( + GlobalDecl:4( + x) + AssignmentStmt:5( + NameExpr(x [__main__.x]) + NameExpr(self [l])))))) + +[case testGlobalDefinedInBlock] +# flags: --allow-redefinition +if object: + x = object() + x = x +x +[out] +MypyFile:1( + IfStmt:2( + If( + NameExpr(object [builtins.object])) + Then( + AssignmentStmt:3( + NameExpr(x'* [__main__.x']) + CallExpr:3( + NameExpr(object [builtins.object]) + Args())) + AssignmentStmt:4( + NameExpr(x* [__main__.x]) + NameExpr(x' [__main__.x'])))) + ExpressionStmt:5( + NameExpr(x [__main__.x]))) + +[case testNonlocalDecl] +def g(): + x = None + def f(): + nonlocal x + x = None + x +[out] +MypyFile:1( + FuncDef:1( + g + Block:2( + AssignmentStmt:2( + NameExpr(x* [l]) + NameExpr(None [builtins.None])) + FuncDef:3( + f + Block:4( + NonlocalDecl:4( + x) + AssignmentStmt:5( + NameExpr(x [l]) + NameExpr(None [builtins.None])) + ExpressionStmt:6( + NameExpr(x [l]))))))) + +[case testNonlocalClass] +def f() -> None: + a = 0 + class C: + nonlocal a + a = 1 +[out] +MypyFile:1( + FuncDef:1( + f + def () + Block:2( + AssignmentStmt:2( + NameExpr(a* [l]) + IntExpr(0)) + ClassDef:3( + C + NonlocalDecl:4( + a) + AssignmentStmt:5( + NameExpr(a* [m]) + IntExpr(1)))))) + +[case testMultipleNamesInNonlocalDecl] +def g(): + x, y = None, None + def f(z): + nonlocal x, y + x = y +[out] +MypyFile:1( + FuncDef:1( + g + Block:2( + AssignmentStmt:2( + TupleExpr:2( + NameExpr(x* [l]) + NameExpr(y* [l])) + TupleExpr:2( + NameExpr(None [builtins.None]) + NameExpr(None [builtins.None]))) + FuncDef:3( + f + Args( + Var(z)) + Block:4( + NonlocalDecl:4( + x + y) + AssignmentStmt:5( + NameExpr(x [l]) + NameExpr(y [l]))))))) + +[case testNestedFunctions] +def f(x): + def g(y): + z = y + x + return g +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + Block:2( + FuncDef:2( + g + Args( + Var(y)) + Block:3( + AssignmentStmt:3( + NameExpr(z* [l]) + OpExpr:3( + + + NameExpr(y [l]) + NameExpr(x [l]))))) + ReturnStmt:4( + NameExpr(g [l]))))) + +[case testNestedFunctionWithOverlappingName] +def f(x): + def g(): + x = 1 +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + Block:2( + FuncDef:2( + g + Block:3( + AssignmentStmt:3( + NameExpr(x* [l]) + IntExpr(1))))))) + +[case testFinalValuesOnVar] +from typing import Final, Any + +def func() -> Any: ... +x: Final = 1 +y: Final = 1.0 +s: Final = "hi" +t: Final = True +n: Final = func() +[out] +MypyFile:1( + ImportFrom:1(typing, [Final, Any]) + FuncDef:3( + func + def () -> Any + Block:3( + ExpressionStmt:3( + Ellipsis))) + AssignmentStmt:4( + NameExpr(x [__main__.x] = 1) + IntExpr(1) + Literal[1]?) + AssignmentStmt:5( + NameExpr(y [__main__.y] = 1.0) + FloatExpr(1.0) + Literal[1.0]?) + AssignmentStmt:6( + NameExpr(s [__main__.s] = hi) + StrExpr(hi) + Literal['hi']?) + AssignmentStmt:7( + NameExpr(t [__main__.t] = True) + NameExpr(True [builtins.True]) + Literal[True]?) + AssignmentStmt:8( + NameExpr(n* [__main__.n] = None) + CallExpr:8( + NameExpr(func [__main__.func]) + Args()))) diff --git a/test-data/unit/semanal-classes.test b/test-data/unit/semanal-classes.test new file mode 100644 index 000000000000..b14358509f85 --- /dev/null +++ b/test-data/unit/semanal-classes.test @@ -0,0 +1,634 @@ +-- Test cases related to classes for the semantic analyzer. + +[case testSimpleClass] +class A: pass +x = A +[out] +MypyFile:1( + ClassDef:1( + A + PassStmt:1()) + AssignmentStmt:2( + NameExpr(x* [__main__.x]) + NameExpr(A [__main__.A]))) + +[case testMethods] +class A: + def __init__(self, x): + y = x + def f(self): + y = self +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + __init__ + Args( + Var(self) + Var(x)) + Block:3( + AssignmentStmt:3( + NameExpr(y* [l]) + NameExpr(x [l])))) + FuncDef:4( + f + Args( + Var(self)) + Block:5( + AssignmentStmt:5( + NameExpr(y* [l]) + NameExpr(self [l])))))) + +[case testMemberDefinitionInInit] +class A: + def __init__(self): + self.x = 1 + self.y = 2 +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + __init__ + Args( + Var(self)) + Block:3( + AssignmentStmt:3( + MemberExpr:3( + NameExpr(self [l]) + x*) + IntExpr(1)) + AssignmentStmt:4( + MemberExpr:4( + NameExpr(self [l]) + y*) + IntExpr(2)))))) + +[case testMemberAssignmentViaSelfOutsideInit] +class A: + def f(self): + self.x = 1 +def __init__(self): + self.y = 1 +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + f + Args( + Var(self)) + Block:3( + AssignmentStmt:3( + MemberExpr:3( + NameExpr(self [l]) + x*) + IntExpr(1))))) + FuncDef:4( + __init__ + Args( + Var(self)) + Block:5( + AssignmentStmt:5( + MemberExpr:5( + NameExpr(self [l]) + y) + IntExpr(1))))) + +[case testMemberAssignmentNotViaSelf] +class A: + def __init__(x, self): + self.y = 1 # not really self +class B: + def __init__(x): + self = x + self.z = 1 +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + __init__ + Args( + Var(x) + Var(self)) + Block:3( + AssignmentStmt:3( + MemberExpr:3( + NameExpr(self [l]) + y) + IntExpr(1))))) + ClassDef:4( + B + FuncDef:5( + __init__ + Args( + Var(x)) + Block:6( + AssignmentStmt:6( + NameExpr(self* [l]) + NameExpr(x [l])) + AssignmentStmt:7( + MemberExpr:7( + NameExpr(self [l]) + z) + IntExpr(1)))))) + +[case testNonStandardNameForSelfAndInit] +class A: + def __init__(x): + x.y = 1 +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + __init__ + Args( + Var(x)) + Block:3( + AssignmentStmt:3( + MemberExpr:3( + NameExpr(x [l]) + y*) + IntExpr(1)))))) + +[case testAssignmentAfterAttributeInit] +class A: + def __init__(self): + self.x = 1 + self.x = 2 +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + __init__ + Args( + Var(self)) + Block:3( + AssignmentStmt:3( + MemberExpr:3( + NameExpr(self [l]) + x*) + IntExpr(1)) + AssignmentStmt:4( + MemberExpr:4( + NameExpr(self [l]) + x) + IntExpr(2)))))) + +[case testOverloadedMethod] +from typing import overload +class A: + @overload + def f(self) -> None: self + @overload + def f(self, x: 'A') -> None: self + def f(self, *args): self +[out] +MypyFile:1( + ImportFrom:1(typing, [overload]) + ClassDef:2( + A + OverloadedFuncDef:3( + FuncDef:7( + f + Args( + Var(self)) + VarArg( + Var(args)) + Block:7( + ExpressionStmt:7( + NameExpr(self [l])))) + Overload(def (self: __main__.A), \ + def (self: __main__.A, x: __main__.A)) + Decorator:3( + Var(f) + NameExpr(overload [typing.overload]) + FuncDef:4( + f + Args( + Var(self)) + def (self: __main__.A) + Block:4( + ExpressionStmt:4( + NameExpr(self [l]))))) + Decorator:5( + Var(f) + NameExpr(overload [typing.overload]) + FuncDef:6( + f + Args( + Var(self) + Var(x)) + def (self: __main__.A, x: __main__.A) + Block:6( + ExpressionStmt:6( + NameExpr(self [l])))))))) + +[case testAttributeWithoutType] +class A: + a = object +[out] +MypyFile:1( + ClassDef:1( + A + AssignmentStmt:2( + NameExpr(a* [m]) + NameExpr(object [builtins.object])))) + +[case testDataAttributeRefInClassBody] +class A: + x = 1 + y = x +[out] +MypyFile:1( + ClassDef:1( + A + AssignmentStmt:2( + NameExpr(x [m]) + IntExpr(1) + builtins.int) + AssignmentStmt:3( + NameExpr(y* [m]) + NameExpr(x [__main__.A.x])))) + +[case testMethodRefInClassBody] +class A: + def f(self): pass + g = f +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + f + Args( + Var(self)) + Block:2( + PassStmt:2())) + AssignmentStmt:3( + NameExpr(g* [m]) + NameExpr(f [__main__.A.f])))) + +[case testIfStatementInClassBody] +class A: + if A: + x = 1 + else: + x = 2 +[out] +MypyFile:1( + ClassDef:1( + A + IfStmt:2( + If( + NameExpr(A [__main__.A])) + Then( + AssignmentStmt:3( + NameExpr(x [m]) + IntExpr(1) + builtins.int)) + Else( + AssignmentStmt:5( + NameExpr(x [__main__.A.x]) + IntExpr(2)))))) + +[case testForStatementInClassBody] +class A: + for x in [1, 2]: + y = x +[out] +MypyFile:1( + ClassDef:1( + A + ForStmt:2( + NameExpr(x* [m]) + ListExpr:2( + IntExpr(1) + IntExpr(2)) + Block:3( + AssignmentStmt:3( + NameExpr(y* [m]) + NameExpr(x [__main__.A.x])))))) + +[case testReferenceToClassWithinFunction] +def f(): + class A: pass + A +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ClassDef:2( + A + PassStmt:2()) + ExpressionStmt:3( + NameExpr(A [__main__.A@2]))))) + +[case testReferenceToClassWithinClass] +class A: + class B: pass + B +[out] +MypyFile:1( + ClassDef:1( + A + ClassDef:2( + B + PassStmt:2()) + ExpressionStmt:3( + NameExpr(B [__main__.A.B])))) + +[case testClassWithBaseClassWithinClass] +class A: + class B: pass + class C(B): pass +[out] +MypyFile:1( + ClassDef:1( + A + ClassDef:2( + B + PassStmt:2()) + ClassDef:3( + C + BaseType( + __main__.A.B) + PassStmt:3()))) + +[case testDeclarationReferenceToNestedClass] +def f() -> None: + class A: pass + x = None # type: A +[out] +MypyFile:1( + FuncDef:1( + f + def () + Block:2( + ClassDef:2( + A + PassStmt:2()) + AssignmentStmt:3( + NameExpr(x [l]) + NameExpr(None [builtins.None]) + __main__.A@2)))) + +[case testAccessToLocalInOuterScopeWithinNestedClass] +def f(x): + class A: + y = x + def g(self): + z = x +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + Block:2( + ClassDef:2( + A + AssignmentStmt:3( + NameExpr(y* [m]) + NameExpr(x [l])) + FuncDef:4( + g + Args( + Var(self)) + Block:5( + AssignmentStmt:5( + NameExpr(z* [l]) + NameExpr(x [l])))))))) + +[case testQualifiedMetaclass] +import abc +class A(metaclass=abc.ABCMeta): pass +[out] +MypyFile:1( + Import:1(abc) + ClassDef:2( + A + Metaclass(MemberExpr:2( + NameExpr(abc) + ABCMeta [abc.ABCMeta])) + PassStmt:2())) + +[case testStaticMethod] +class A: + @staticmethod + def f(z: int) -> str: pass +[builtins fixtures/staticmethod.pyi] +[out] +MypyFile:1( + ClassDef:1( + A + Decorator:2( + Var(f) + FuncDef:3( + f + Args( + Var(z)) + def (z: builtins.int) -> builtins.str + Static + Block:3( + PassStmt:3()))))) + +[case testStaticMethodWithNoArgs] +class A: + @staticmethod + def f() -> str: pass +[builtins fixtures/staticmethod.pyi] +[out] +MypyFile:1( + ClassDef:1( + A + Decorator:2( + Var(f) + FuncDef:3( + f + def () -> builtins.str + Static + Block:3( + PassStmt:3()))))) + +[case testClassMethod] +class A: + @classmethod + def f(cls, z: int) -> str: pass +[builtins fixtures/classmethod.pyi] +[out] +MypyFile:1( + ClassDef:1( + A + Decorator:2( + Var(f) + FuncDef:3( + f + Args( + Var(cls) + Var(z)) + def (cls: Type[__main__.A], z: builtins.int) -> builtins.str + Class + Block:3( + PassStmt:3()))))) + +[case testClassMethodWithNoArgs] +class A: + @classmethod + def f(cls) -> str: pass +[builtins fixtures/classmethod.pyi] +[out] +MypyFile:1( + ClassDef:1( + A + Decorator:2( + Var(f) + FuncDef:3( + f + Args( + Var(cls)) + def (cls: Type[__main__.A]) -> builtins.str + Class + Block:3( + PassStmt:3()))))) + +[case testProperty] +import typing +class A: + @property + def f(self) -> str: pass +[builtins fixtures/property.pyi] +[out] +MypyFile:1( + Import:1(typing) + ClassDef:2( + A + Decorator:3( + Var(f) + FuncDef:4( + f + Args( + Var(self)) + def (self: __main__.A) -> builtins.str + Property + Block:4( + PassStmt:4()))))) + +[case testClassDecorator] +import typing +@object +class A: pass +[out] +MypyFile:1( + Import:1(typing) + ClassDef:3( + A + Decorators( + NameExpr(object [builtins.object])) + PassStmt:3())) + +[case testClassAttributeAsMethodDefaultArgumentValue] +import typing +class A: + X = 1 + def f(self, x : int = X) -> None: pass +[out] +MypyFile:1( + Import:1(typing) + ClassDef:2( + A + AssignmentStmt:3( + NameExpr(X [m]) + IntExpr(1) + builtins.int) + FuncDef:4( + f + Args( + Var(self) + default( + Var(x) + NameExpr(X [__main__.A.X]))) + def (self: __main__.A, x: builtins.int =) + Block:4( + PassStmt:4())))) + +[case testInvalidBaseClass] + +from typing import Any, Callable +class A(None): pass +class B(Any): pass +class C(Callable[[], int]): pass +[out] +main:3: error: Invalid base class "None" +main:5: error: Invalid base class "Callable" + +[case testTupleAsBaseClass] + +import m +[file m.pyi] +from typing import Tuple +class A(Tuple[int, str]): pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + Import:2(m)) +MypyFile:1( + tmp/m.pyi + ImportFrom:1(typing, [Tuple]) + ClassDef:2( + A + TupleType( + Tuple[builtins.int, builtins.str]) + BaseType( + builtins.tuple[Union[builtins.int, builtins.str], ...]) + PassStmt:2())) + +[case testBaseClassFromIgnoredModule] +import m # type: ignore +class B(m.A): + pass +[out] +MypyFile:1( + Import:1(m) + ClassDef:2( + B + FallbackToAny + PassStmt:3()) + IgnoredLines(1)) + +[case testBaseClassFromIgnoredModuleUsingImportFrom] +from m import A # type: ignore +class B(A, int): + pass +[out] +MypyFile:1( + ImportFrom:1(m, [A]) + ClassDef:2( + B + FallbackToAny + BaseType( + builtins.int) + PassStmt:3()) + IgnoredLines(1)) + +[case testBaseClassWithExplicitAnyType] +from typing import Any +A = 1 # type: Any +class B(A): + pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Any]) + AssignmentStmt:2( + NameExpr(A [__main__.A]) + IntExpr(1) + Any) + ClassDef:3( + B + FallbackToAny + PassStmt:4())) diff --git a/test-data/unit/semanal-classvar.test b/test-data/unit/semanal-classvar.test new file mode 100644 index 000000000000..a7bcec0324dc --- /dev/null +++ b/test-data/unit/semanal-classvar.test @@ -0,0 +1,220 @@ +[case testClassVarDef] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int] +[out] +MypyFile:1( + ImportFrom:1(typing, [ClassVar]) + ClassDef:2( + A + AssignmentStmt:3( + NameExpr(x [m]) + IntExpr(1) + builtins.int))) + +[case testClassVarDefInModuleScope] +from typing import ClassVar +x = None # type: ClassVar[int] +[out] +main:2: error: ClassVar can only be used for assignments in class body + +[case testClassVarDefInFuncScope] +from typing import ClassVar +def f() -> None: + x = None # type: ClassVar[int] +[out] +main:3: error: ClassVar can only be used for assignments in class body + +[case testClassVarDefInMethod] +from typing import ClassVar +class A: + def f(self) -> None: + x = None # type: ClassVar +[out] +main:4: error: ClassVar can only be used for assignments in class body + +[case testClassVarTooManyArguments] +from typing import ClassVar +class A: + x = 1 # type: ClassVar[int, str] +[out] +main:3: error: ClassVar[...] must have at most one type argument + +[case testClassVarWithoutArguments] +from typing import ClassVar +class A: + x = 1 # type: ClassVar +[out] +MypyFile:1( + ImportFrom:1(typing, [ClassVar]) + ClassDef:2( + A + AssignmentStmt:3( + NameExpr(x [m]) + IntExpr(1) + Any))) + +[case testClassVarWithTypeVar] + +from typing import ClassVar, TypeVar +T = TypeVar('T') +class A: + x = None # type: ClassVar[T] +[out] +main:5: error: Type variable "__main__.T" is unbound +main:5: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) +main:5: note: (Hint: Use "T" in function signature to bind "T" inside a function) + +[case testClassVarInFunctionArgs] +from typing import ClassVar +def f(x: str, y: ClassVar) -> None: pass +[out] +main:2: error: ClassVar can only be used for assignments in class body + +[case testClassVarInMethodArgs] +from typing import ClassVar +class A: + def f(x: str, y: ClassVar) -> None: pass +[out] +main:3: error: ClassVar can only be used for assignments in class body + +[case testClassVarFunctionRetType] +from typing import ClassVar +def f() -> ClassVar: pass +[out] +main:2: error: ClassVar can only be used for assignments in class body + +[case testClassVarMethodRetType] +from typing import ClassVar +class A: + def f(self) -> ClassVar: pass +[out] +main:3: error: ClassVar can only be used for assignments in class body + +[case testMultipleClassVarInFunctionSig] +from typing import ClassVar +def f(x: ClassVar, y: ClassVar) -> ClassVar: pass +[out] +main:2: error: ClassVar can only be used for assignments in class body + +[case testClassVarInCallableArgs] +from typing import Callable, ClassVar, Any +f = None # type: Callable[[int, ClassVar], Any] +[out] +main:2: error: Invalid type: ClassVar nested inside other type + +[case testClassVarInCallableRet] +from typing import Callable, ClassVar +f = None # type: Callable[..., ClassVar] +[out] +main:2: error: Invalid type: ClassVar nested inside other type + +[case testClassVarInUnion] +from typing import ClassVar, Union +x = None # type: Union[ClassVar, str] +[out] +main:2: error: Invalid type: ClassVar nested inside other type + +[case testClassVarInUnionAsAttribute] +from typing import ClassVar, Union +class A: + x = None # type: Union[ClassVar, str] +[out] +main:3: error: Invalid type: ClassVar nested inside other type + +[case testListWithClassVars] +from typing import ClassVar, List +x = [] # type: List[ClassVar] +[builtins fixtures/list.pyi] +[out] +main:2: error: Invalid type: ClassVar nested inside other type + +[case testTupleClassVar] +from typing import ClassVar, Tuple +x = None # type: Tuple[ClassVar, int] +[builtins fixtures/tuple.pyi] +[out] +main:2: error: Invalid type: ClassVar nested inside other type + +[case testMultipleLvaluesWithList] +from typing import ClassVar, Tuple +class A: + [x, y] = None, None # type: Tuple[ClassVar, ClassVar] +[builtins fixtures/tuple.pyi] +[out] +main:3: error: Invalid type: ClassVar nested inside other type + +[case testDeeplyNested] +from typing import Callable, ClassVar, Union +class A: pass +class B: + x = None # type: Union[str, Callable[[A, ClassVar], int]] +[out] +main:4: error: Invalid type: ClassVar nested inside other type + +[case testClassVarInClassVar] +from typing import ClassVar +class A: + x = None # type: ClassVar[ClassVar[int]] +[out] +main:3: error: Invalid type: ClassVar nested inside other type + +[case testInsideGeneric] +from typing import ClassVar, Generic, TypeVar +T = TypeVar('T') +class A(Generic[T]): pass +class B: + x = None # type: A[ClassVar] +[out] +main:5: error: Invalid type: ClassVar nested inside other type + +[case testDefineOnSelf] +from typing import ClassVar +class A: + def __init__(self) -> None: + self.x = None # type: ClassVar +[out] +main:4: error: ClassVar can only be used for assignments in class body + +[case testForIndex] +from typing import ClassVar +for i in []: # type: ClassVar + pass +[out] +main:2: error: ClassVar can only be used for assignments in class body + +[case testForIndexInClassBody] +from typing import ClassVar +class A: + for i in []: # type: ClassVar + pass +[out] +main:3: error: ClassVar can only be used for assignments in class body + +[case testWithStmt] +from typing import ClassVar +class A: pass +with A() as x: # type: ClassVar + pass +[out] +main:3: error: ClassVar can only be used for assignments in class body + +[case testWithStmtInClassBody] +from typing import ClassVar +class A: pass +class B: + with A() as x: # type: ClassVar + pass +[out] +main:4: error: ClassVar can only be used for assignments in class body + +[case testClassVarWithTypeVariable] +from typing import ClassVar, TypeVar, Generic, List + +T = TypeVar('T') + +class Some(Generic[T]): + error: ClassVar[T] # E: ClassVar cannot contain type variables + nested: ClassVar[List[List[T]]] # E: ClassVar cannot contain type variables + ok: ClassVar[int] +[out] diff --git a/test-data/unit/semanal-errors-python310.test b/test-data/unit/semanal-errors-python310.test new file mode 100644 index 000000000000..68c158cddae6 --- /dev/null +++ b/test-data/unit/semanal-errors-python310.test @@ -0,0 +1,43 @@ +[case testMatchUndefinedSubject] +import typing +match x: + case _: + pass +[out] +main:2: error: Name "x" is not defined + +[case testMatchUndefinedValuePattern] +import typing +x = 1 +match x: + case a.b: + pass +[out] +main:4: error: Name "a" is not defined + +[case testMatchUndefinedClassPattern] +import typing +x = 1 +match x: + case A(): + pass +[out] +main:4: error: Name "A" is not defined + +[case testNoneBindingWildcardPattern] +import typing +x = 1 +match x: + case _: + _ +[out] +main:5: error: Name "_" is not defined + +[case testNoneBindingStarredWildcardPattern] +import typing +x = 1 +match x: + case [*_]: + _ +[out] +main:5: error: Name "_" is not defined diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test new file mode 100644 index 000000000000..2f0a4c140915 --- /dev/null +++ b/test-data/unit/semanal-errors.test @@ -0,0 +1,1508 @@ +[case testUndefinedVariableInGlobalStatement] +import typing +x +y +[out] +main:2: error: Name "x" is not defined +main:3: error: Name "y" is not defined + +[case testUndefinedVariableWithinFunctionContext] +import typing +def f() -> None: + x +y +[out] +main:3: error: Name "x" is not defined +main:4: error: Name "y" is not defined + +[case testMethodScope] +import typing +class A: + def f(self): pass +f +[out] +main:4: error: Name "f" is not defined + +[case testMethodScope2] +import typing +class A: + def f(self): pass +class B: + def g(self) -> None: + f # error + g # error +[out] +main:6: error: Name "f" is not defined +main:7: error: Name "g" is not defined + +[case testInvalidType] +import typing +x = None # type: X +[out] +main:2: error: Name "X" is not defined + +[case testInvalidGenericArg] +from typing import TypeVar, Generic +t = TypeVar('t') +class A(Generic[t]): pass +x = 0 # type: A[y] +[out] +main:4: error: Name "y" is not defined + +[case testInvalidNumberOfGenericArgsInTypeDecl] +from typing import TypeVar, Generic +t = TypeVar('t') +class A: pass +class B(Generic[t]): pass +x = 0 # type: B[A, A] +y = 0 # type: A[A] +[out] +main:5: error: "B" expects 1 type argument, but 2 given +main:6: error: "A" expects no type arguments, but 1 given + +[case testInvalidNumberOfGenericArgsInUndefinedArg] + +class A: pass +x = None # type: A[int] # E: "A" expects no type arguments, but 1 given +[out] + +[case testInvalidNumberOfGenericArgsInNestedBlock] + +class A: pass +class B: + def f(self) -> None: + while 1: + x = None # type: A[int] \ + # E: "A" expects no type arguments, but 1 given +[out] + +[case testInvalidNumberOfGenericArgsInSignature] +import typing +class A: pass +def f() -> A[int]: pass # E: "A" expects no type arguments, but 1 given +[out] + +[case testInvalidNumberOfGenericArgsInOverloadedSignature] +from typing import overload +class A: pass +@overload +def f(): pass +@overload +def f(x: A[int]) -> None: pass # E: "A" expects no type arguments, but 1 given +def f(*args): pass +[out] + +[case testInvalidNumberOfGenericArgsInBaseType] +import typing +class A: pass +class B(A[int]): pass # E: "A" expects no type arguments, but 1 given +[out] + +[case testInvalidNumberOfGenericArgsInCast] +from typing import cast +class A: pass +x = cast(A[int], 1) # E: "A" expects no type arguments, but 1 given +[out] + +[case testInvalidNumberOfGenericArgsInNestedGenericType] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): pass +class B: pass +def f() -> A[B[int]]: pass # E: "B" expects no type arguments, but 1 given +[out] + +[case testInvalidNumberOfGenericArgsInTupleType] +from typing import Tuple +class A: pass +x = None # type: Tuple[A[int]] # E: "A" expects no type arguments, but 1 given +[builtins fixtures/tuple.pyi] +[out] + +[case testInvalidNumberOfGenericArgsInFunctionType] +from typing import Callable +class A: pass +x = None # type: Callable[[A[int]], int] # E: "A" expects no type arguments, but 1 given +y = None # type: Callable[[], A[int]] # E: "A" expects no type arguments, but 1 given +[out] + +[case testVarOrFuncAsType] + +import typing +def f(): pass +x = 1 +y = 0 # type: f +z = 0 # type: x +[out] +main:5: error: Function "__main__.f" is not valid as a type +main:5: note: Perhaps you need "Callable[...]" or a callback protocol? +main:6: error: Variable "__main__.x" is not valid as a type +main:6: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testGlobalVarRedefinition] +import typing +class A: pass +x = 0 # type: A +x = 0 # type: A +[out] +main:4: error: Name "x" already defined on line 3 + +[case testLocalVarRedefinition] +import typing +class A: pass +def f() -> None: + x = 0 # type: A + x = 0 # type: A +[out] +main:5: error: Name "x" already defined on line 4 + +[case testClassVarRedefinition] +import typing +class A: + x = 0 # type: object + x = 0 # type: object +[out] +main:4: error: Name "x" already defined on line 3 + +[case testMultipleClassDefinitions] +import typing +class A: pass +class A: pass +[out] +main:3: error: Name "A" already defined on line 2 + +[case testMultipleMixedDefinitions] +import typing +x = 1 +def x(): pass +class x: pass +[out] +main:3: error: Name "x" already defined on line 2 +main:4: error: Name "x" already defined on line 2 + +[case testNameNotImported] +import typing +from m import y +x +[file m.py] +x = y = 1 +[out] +main:3: error: Name "x" is not defined + +[case testMissingNameInImportFrom] +import typing +from m import y +[file m.py] +x = 1 +[out] +main:2: error: Module "m" has no attribute "y" + +[case testMissingModule] +import typing +import m +[out] +main:2: error: Cannot find implementation or library stub for module named "m" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testMissingModule2] +import typing +from m import x +[out] +main:2: error: Cannot find implementation or library stub for module named "m" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testMissingModule3] +import typing +from m import * +[out] +main:2: error: Cannot find implementation or library stub for module named "m" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testMissingModuleRelativeImport] +import typing +import m +[file m/__init__.py] +from .x import y +[out] +tmp/m/__init__.py:1: error: Cannot find implementation or library stub for module named "m.x" +tmp/m/__init__.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testMissingModuleRelativeImport2] +import typing +import m.a +[file m/__init__.py] +[file m/a.py] +from .x import y +[out] +tmp/m/a.py:1: error: Cannot find implementation or library stub for module named "m.x" +tmp/m/a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testModuleNotImported] +import typing +import _m +_n.x +[file _m.py] +import _n +[file _n.py] +x = 1 +[out] +main:3: error: Name "_n" is not defined + +[case testImportAsteriskPlusUnderscore] +import typing +from _m import * +_x +__x__ +[file _m.py] +_x = __x__ = 1 +[out] +main:3: error: Name "_x" is not defined +main:4: error: Name "__x__" is not defined + +[case testRelativeImportAtTopLevelModule] +from . import m +[out] +main:1: error: No parent module -- cannot perform relative import + +[case testRelativeImportAtTopLevelModule2] +from .. import m +[out] +main:1: error: No parent module -- cannot perform relative import + +[case testUndefinedTypeWithQualifiedName] +import typing +import m +def f() -> m.c: pass +def g() -> n.c: pass +[file m.py] +[out] +main:3: error: Name "m.c" is not defined +main:4: error: Name "n" is not defined + +[case testMissingPackage] +import typing +import m.n +[out] +main:2: error: Cannot find implementation or library stub for module named "m.n" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "m" + +[case testMissingPackage2] +import typing +from m.n import x +from a.b import * +[out] +main:2: error: Cannot find implementation or library stub for module named "m.n" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:3: error: Cannot find implementation or library stub for module named "a.b" + +[case testErrorInImportedModule] +import m +[file m.py] +import typing +x = y +[out] +tmp/m.py:2: error: Name "y" is not defined + +[case testErrorInImportedModule2] +import m.n +[file m/__init__.py] +[file m/n.py] +import k +[file k.py] +import typing +x = y +[out] +tmp/k.py:2: error: Name "y" is not defined + +[case testPackageWithoutInitFile] +# flags: --no-namespace-packages +import typing +import m.n +m.n.x +[file m/n.py] +x = 1 +[out] +main:3: error: Cannot find implementation or library stub for module named "m.n" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:3: error: Cannot find implementation or library stub for module named "m" + +[case testBreakOutsideLoop] +break +def f(): + break +[out] +main:1: error: "break" outside loop +main:3: error: "break" outside loop + +[case testContinueOutsideLoop] +continue +def f(): + continue +[out] +main:1: error: "continue" outside loop +main:3: error: "continue" outside loop + +[case testReturnOutsideFunction] +def f(): pass +return +return 1 +[out] +main:2: error: "return" outside function +main:3: error: "return" outside function + +[case testYieldOutsideFunction] +yield 1 +yield +[out] +main:1: error: "yield" outside function +main:2: error: "yield" outside function + +[case testInvalidLvalues1] +1 = 1 +[out] +main:1: error: cannot assign to literal +[out version>=3.10] +main:1: error: cannot assign to literal here. Maybe you meant '==' instead of '='? + +[case testInvalidLvalues2] +(1) = 1 +[out] +main:1: error: cannot assign to literal +[out version>=3.10] +main:1: error: cannot assign to literal here. Maybe you meant '==' instead of '='? + +[case testInvalidLvalues3] +(1, 1) = 1 +[out] +main:1: error: cannot assign to literal + +[case testInvalidLvalues4] +[1, 1] = 1 +[out] +main:1: error: cannot assign to literal + +[case testInvalidLvalues6] +x = y = z = 1 # ok +x, (y, 1) = 1 +[out] +main:2: error: cannot assign to literal + +[case testInvalidLvalues7] +x, [y, 1] = 1 +[out] +main:1: error: cannot assign to literal + +[case testInvalidLvalues8] +x, [y, [z, 1]] = 1 +[out] +main:1: error: cannot assign to literal + +[case testInvalidLvalues9] +x, (y) = 1 # ok +x, (y, (z, z)) = 1 # ok +x, (y, (z, 1)) = 1 +[out] +main:3: error: cannot assign to literal + +[case testInvalidLvalues10] +x + x = 1 +[out] +main:1: error: cannot assign to operator +[out version>=3.10] +main:1: error: cannot assign to expression here. Maybe you meant '==' instead of '='? + +[case testInvalidLvalues11] +-x = 1 +[out] +main:1: error: cannot assign to operator +[out version>=3.10] +main:1: error: cannot assign to expression here. Maybe you meant '==' instead of '='? + +[case testInvalidLvalues12] +1.1 = 1 +[out] +main:1: error: cannot assign to literal +[out version>=3.10] +main:1: error: cannot assign to literal here. Maybe you meant '==' instead of '='? + +[case testInvalidLvalues13] +'x' = 1 +[out] +main:1: error: cannot assign to literal +[out version>=3.10] +main:1: error: cannot assign to literal here. Maybe you meant '==' instead of '='? + +[case testInvalidLvalues14] +x() = 1 +[out] +main:1: error: cannot assign to function call +[out version>=3.10] +main:1: error: cannot assign to function call here. Maybe you meant '==' instead of '='? + +[case testTwoStarExpressions] +a, *b, *c = 1 +*a, (*b, c) = 1 +a, (*b, *c) = 1 +[*a, *b] = 1 +[out] +main:1: error: Two starred expressions in assignment +main:3: error: Two starred expressions in assignment +main:4: error: Two starred expressions in assignment + +[case testTwoStarExpressionsInForStmt] +z = 1 +for a, *b, *c in z: + pass +for *a, (*b, c) in z: + pass +for a, (*b, *c) in z: + pass +for [*a, *b] in z: + pass +[out] +main:2: error: Two starred expressions in assignment +main:6: error: Two starred expressions in assignment +main:8: error: Two starred expressions in assignment + +[case testTwoStarExpressionsInGeneratorExpr] +(a for a, *b, *c in []) +(a for *a, (*b, c) in []) +(a for a, (*b, *c) in []) +[out] +main:1: error: Name "a" is not defined +main:1: error: Two starred expressions in assignment +main:3: error: Two starred expressions in assignment + +[case testStarExpressionRhs] +b = 1 +c = 1 +d = 1 +a = *b +[out] +main:4: error: can't use starred expression here + +[case testStarExpressionInExp] +a = 1 +*a + 1 +[out] +main:2: error: can't use starred expression here + +[case testInvalidDel1] +x = 1 +del x(1) +[out] +main:2: error: cannot delete function call + +[case testInvalidDel2] +x = 1 +del x + 1 +[out] +main:2: error: cannot delete operator +[out version>=3.10] +main:2: error: cannot delete expression + +[case testInvalidDel3] +del z # E: Name "z" is not defined + +[case testFunctionTvarScope] + +from typing import TypeVar +t = TypeVar('t') +def f(x: t) -> t: pass +x = 0 # type: t +[out] +main:5: error: Type variable "__main__.t" is unbound +main:5: note: (Hint: Use "Generic[t]" or "Protocol[t]" base class to bind "t" inside a class) +main:5: note: (Hint: Use "t" in function signature to bind "t" inside a function) + +[case testClassTvarScope] + +from typing import Generic, TypeVar +t = TypeVar('t') +class c(Generic[t]): pass +x = 0 # type: t +[out] +main:5: error: Type variable "__main__.t" is unbound +main:5: note: (Hint: Use "Generic[t]" or "Protocol[t]" base class to bind "t" inside a class) +main:5: note: (Hint: Use "t" in function signature to bind "t" inside a function) + +[case testExpressionRefersToTypeVariable] +from typing import TypeVar, Generic +t = TypeVar('t') +class c(Generic[t]): + def f(self) -> None: x = t +def f(y: t): x = t +[out] +main:4: error: "t" is a type variable and only valid in type context +main:5: error: "t" is a type variable and only valid in type context + +[case testMissingSelf] +import typing +class A: + def f(): pass +[out] +main:3: error: Method must have at least one argument. Did you forget the "self" argument? + +[case testInvalidBaseClass] +import typing +class A(B): pass +[out] +main:2: error: Name "B" is not defined + +[case testSuperOutsideClass] +class A: pass +super().x +def f() -> None: super().y +[out] +main:2: error: "super" used outside class +main:3: error: "super" used outside class + +[case testMissingSelfInMethod] +import typing +class A: + def f() -> None: pass + def g(): pass +[out] +main:3: error: Method must have at least one argument. Did you forget the "self" argument? +main:4: error: Method must have at least one argument. Did you forget the "self" argument? + +[case testMultipleMethodDefinition] +import typing +class A: + def f(self) -> None: pass + def g(self) -> None: pass + def f(self, x: object) -> None: pass +[out] +main:5: error: Name "f" already defined on line 3 + +[case testInvalidGlobalDecl] +import typing +def f() -> None: + global x + x = None +[out] +main:4: error: Name "x" is not defined + +[case testInvalidNonlocalDecl] +import typing +def f(): + def g() -> None: + nonlocal x + x = None +[out] +main:4: error: No binding for nonlocal "x" found +main:5: error: Name "x" is not defined + +[case testNonlocalDeclNotMatchingGlobal] +import typing +x = None +def f() -> None: + nonlocal x + x = None +[out] +main:4: error: No binding for nonlocal "x" found +main:5: error: Name "x" is not defined + +[case testNonlocalDeclConflictingWithParameter] +import typing +def g(): + x = None + def f(x) -> None: + nonlocal x + x = None +[out] +main:5: error: Name "x" is already defined in local scope before nonlocal declaration + +[case testNonlocalDeclOutsideFunction] +x = 2 +nonlocal x +[out] +main:2: error: nonlocal declaration not allowed at module level + +[case testGlobalAndNonlocalDecl] +import typing +x = 1 +def f(): + x = 1 + def g() -> None: + global x + nonlocal x + x = None +[out] +main:7: error: Name "x" is nonlocal and global + +[case testNonlocalAndGlobalDecl] +import typing +x = 1 +def f(): + x = 1 + def g() -> None: + nonlocal x + global x + x = None +[out] +main:7: error: Name "x" is nonlocal and global + +[case testNestedFunctionAndScoping] +import typing +def f(x) -> None: + def g(y): + z = x + z + y + x +[out] +main:5: error: Name "z" is not defined +main:6: error: Name "y" is not defined + +[case testMultipleNestedFunctionDef] +import typing +def f(x) -> None: + def g(): pass + x = 1 + def g(): pass +[out] +main:5: error: Name "g" already defined on line 3 + +[case testRedefinedOverloadedFunction] +from typing import overload, Any +def f() -> None: + @overload + def p(o: object) -> None: pass # no error + @overload + def p(o: Any) -> None: pass # no error + x = 1 + def p(): pass # fail +[out] +main:3: error: An overloaded function outside a stub file must have an implementation +main:8: error: Name "p" already defined on line 3 + +[case testNestedFunctionInMethod] +import typing +class A: + def f(self) -> None: + def g() -> None: + x + y +[out] +main:5: error: Name "x" is not defined +main:6: error: Name "y" is not defined + +[case testImportScope] +import typing +def f() -> None: + import x +x.y # E: Name "x" is not defined +[file x.py] +y = 1 +[out] + +[case testImportScope2] +import typing +def f() -> None: + from x import y + y +y # E: Name "y" is not defined +[file x.py] +y = 1 +[out] + +[case testImportScope3] +import typing +def f() -> None: + from x import * + y +y # E: Name "y" is not defined +[file x.py] +y = 1 +[out] + +[case testImportScope4] +import typing +class A: + from x import * + y +y # E: Name "y" is not defined +[file x.py] +y = 1 +[out] + +[case testScopeOfNestedClass] +import typing +def f(): + class A: pass + A +A # E: Name "A" is not defined +[out] + +[case testScopeOfNestedClass2] +import typing +class A: + class B: pass +B # E: Name "B" is not defined +[out] + +[case testScopeOfNestedClass3] +import typing +class A: + def f(self): + class B: pass + B # E: Name "B" is not defined +B # E: Name "B" is not defined +[out] + +[case testInvalidNestedClassReferenceInDecl] +import typing +class A: pass +foo = 0 # type: A.x # E: Name "A.x" is not defined +[out] + +[case testTvarScopingWithNestedClass] + +from typing import TypeVar, Generic +t = TypeVar('t') +s = TypeVar('s') +class A(Generic[t]): + class B(Generic[s]): + x = 0 # type: A[s] + y = 0 # type: A[t] # E: Type variable "__main__.t" is unbound \ + # N: (Hint: Use "Generic[t]" or "Protocol[t]" base class to bind "t" inside a class) \ + # N: (Hint: Use "t" in function signature to bind "t" inside a function) + z = 0 # type: A[s] # E: Type variable "__main__.s" is unbound \ + # N: (Hint: Use "Generic[s]" or "Protocol[s]" base class to bind "s" inside a class) \ + # N: (Hint: Use "s" in function signature to bind "s" inside a function) + a = 0 # type: A[t] +[out] + +[case testTestExtendPrimitives] +# Extending bool is not checked here as it should be typed +# as final meaning the type checker will detect it. +class C(bool): pass # ok +class A(int): pass # ok +class B(float): pass # ok +class D(str): pass # ok +[builtins fixtures/primitives.pyi] +[out] + +[case testCyclicInheritance1] + +class A(A): pass # E: Cannot resolve name "A" (possible cyclic definition) +[out] + +[case testCyclicInheritance2] + +class A(B): pass # E: Cannot resolve name "B" (possible cyclic definition) +class B(A): pass +[out] + +[case testAssignToTypeDef] +import typing +class A: pass +A = None # E: Cannot assign to a type +[out] + +[case testInvalidCastTargetSyntax] + +from typing import cast, TypeVar, Generic +t = TypeVar('t') +class C(Generic[t]): pass +cast(str + str, None) # E: Cast target is not a type +cast(C[str][str], None) # E: Cast target is not a type +cast(C[str + str], None) # E: Cast target is not a type +cast([int], None) # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? +[out] + +[case testInvalidCastTargetType] + +from typing import cast +x = 0 +cast(x, None) # E: Variable "__main__.x" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +cast(t, None) # E: Name "t" is not defined +cast(__builtins__.x, None) # E: Name "__builtins__.x" is not defined +[out] + +[case testInvalidCastTargetType2] +from typing import cast +x = 0 +cast(str[str], None) # E: "str" expects no type arguments, but 1 given +[out] + +[case testInvalidNumberOfArgsToCast] +from typing import cast +cast(str) # E: "cast" expects 2 arguments +cast(str, None, None) # E: "cast" expects 2 arguments +[out] + +[case testInvalidKindsOfArgsToCast] +from typing import cast +cast(str, *None) # E: "cast" must be called with 2 positional arguments +cast(str, target=None) # E: "cast" must be called with 2 positional arguments +[out] + +[case testInvalidAssertType] +from typing import assert_type +assert_type(1, type=int) # E: "assert_type" must be called with 2 positional arguments +assert_type(1, *int) # E: "assert_type" must be called with 2 positional arguments +assert_type() # E: "assert_type" expects 2 arguments +assert_type(1, int, "hello") # E: "assert_type" expects 2 arguments +assert_type(int, 1) # E: Invalid type: try using Literal[1] instead? +assert_type(1, int[int]) # E: "int" expects no type arguments, but 1 given + +[case testInvalidAnyCall] +from typing import Any +Any(str, None) # E: Any(...) is no longer supported. Use cast(Any, ...) instead +Any(arg=str) # E: Any(...) is no longer supported. Use cast(Any, ...) instead +[out] + +[case testTypeListAsType] + +def f(x: [int]) -> None: # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? + pass +[out] + +[case testInvalidFunctionType] +from typing import Callable +x = None # type: Callable[int, str] +y = None # type: Callable[int] +z = None # type: Callable[int, int, int] +[out] +main:2: error: The first argument to Callable must be a list of types, parameter specification, or "..." +main:2: note: See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas +main:3: error: Please use "Callable[[], ]" or "Callable" +main:4: error: Please use "Callable[[], ]" or "Callable" + +[case testAbstractGlobalFunction] +import typing +from abc import abstractmethod +@abstractmethod +def foo(): pass +[out] +main:3: error: "abstractmethod" used with a non-method + +[case testAbstractNestedFunction] +import typing +from abc import abstractmethod +def g() -> None: + @abstractmethod + def foo(): pass +[out] +main:4: error: "abstractmethod" used with a non-method + +[case testInvalidTypeDeclaration] +import typing +def f(): pass +f() = 1 # type: int +[out] +main:3: error: cannot assign to function call +[out version>=3.10] +main:3: error: cannot assign to function call here. Maybe you meant '==' instead of '='? + +[case testIndexedAssignmentWithTypeDeclaration] +import typing +None[1] = 1 # type: int +[out] +main:2: error: Unexpected type declaration + +[case testNonSelfMemberAssignmentWithTypeDeclaration] +import typing +None.x = 1 # type: int +[out] +main:2: error: Type cannot be declared in assignment to non-self attribute + +[case testNonSelfMemberAssignmentWithTypeDeclarationInMethod] +import typing +class A: + def f(self, x) -> None: + x.y = 1 # type: int +[out] +main:4: error: Type cannot be declared in assignment to non-self attribute + +[case testInvalidTypeInTypeApplication] + +from typing import TypeVar, Generic +t = TypeVar('t') +class A(Generic[t]): pass +A[TypeVar] # E: Variable "typing.TypeVar" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[out] + +[case testInvalidTypeInTypeApplication2] +from typing import TypeVar, Generic +t = TypeVar('t') +class A(Generic[t]): pass +A[1] # E: Invalid type: try using Literal[1] instead? +[out] + +[case testVariableDeclWithInvalidNumberOfTypes] +x, y = 1, 2 # type: int, str, int # E: Incompatible number of tuple items +[builtins fixtures/tuple.pyi] +[out] + +[case testVariableDeclWithInvalidNumberOfTypesNested] +x, (y, z) = 1, (2, 3) # type: int, (str, int, int) # E: Incompatible number of tuple items +[builtins fixtures/tuple.pyi] +[out] + +[case testVariableDeclWithInvalidNumberOfTypesNested2] +x, (y, z) = 1, (2, 3) # type: int, (str, ) # E: Incompatible number of tuple items +[builtins fixtures/tuple.pyi] +[out] + +[case testVariableDeclWithInvalidNumberOfTypesNested3] +x, (y, z) = 1, (2, 3) # type: int, str # E: Tuple type expected for multiple variables +[builtins fixtures/tuple.pyi] +[out] + +[case testVariableDeclWithInvalidNumberOfTypesNested4] +x, (y, z) = 1, (2, 3) # type: int, str, int # E: Incompatible number of tuple items +[builtins fixtures/tuple.pyi] +[out] + +[case testVariableDeclWithInvalidNumberOfTypesNested5] +x, (y, ) = 1, (2, ) # type: int, str # E: Tuple type expected for multiple variables +[builtins fixtures/tuple.pyi] +[out] + +[case testVariableDeclWithInvalidType] +x, y = 1, 2 # type: int # E: Tuple type expected for multiple variables +[out] + +[case testInvalidLvalueWithExplicitType] +a = 1 +a() = None # type: int +[out] +main:2: error: cannot assign to function call +[out version>=3.10] +main:2: error: cannot assign to function call here. Maybe you meant '==' instead of '='? + +[case testInvalidLvalueWithExplicitType2] +a = 1 +a[1] = None # type: int # E: Unexpected type declaration +a.x = None # type: int \ + # E: Type cannot be declared in assignment to non-self attribute +[out] + +[case testInvalidLvalueWithExplicitType3] +a = 1 +a.y, a.x = None, None # type: int, int \ + # E: Type cannot be declared in assignment to non-self attribute +a[1], a[2] = None, None # type: int, int \ + # E: Unexpected type declaration +[builtins fixtures/tuple.pyi] +[out] + +[case testMissingGenericImport] +from typing import TypeVar +T = TypeVar('T') +class A(Generic[T]): pass +[out] +main:3: error: Name "Generic" is not defined + +[case testInvalidTypeWithinGeneric] +from typing import Generic +class A(Generic[int]): pass # E: Free type variable expected in Generic[...] +[out] + +[case testInvalidTypeWithinNestedGenericClass] +from typing import Generic, TypeVar +T = TypeVar('T') +class A(Generic[T]): + class B(Generic[T]): pass \ + # E: Free type variable expected in Generic[...] +[out] + +[case testIncludingGenericTwiceInBaseClassList] +from typing import Generic, TypeVar +T = TypeVar('T') +S = TypeVar('S') +class A(Generic[T], Generic[S]): pass \ + # E: Only single Generic[...] or Protocol[...] can be in bases +[out] + +[case testInvalidMetaclass] +class A(metaclass=x): pass # E: Name "x" is not defined +[out] + +[case testInvalidQualifiedMetaclass] +import abc +class A(metaclass=abc.Foo): pass # E: Name "abc.Foo" is not defined +[out] + +[case testNonClassMetaclass] +def f(): pass +class A(metaclass=f): pass # E: Invalid metaclass "f" +[out] + +[case testInvalidTypevarArguments] +from typing import TypeVar +a = TypeVar() # E: Too few arguments for TypeVar() +b = TypeVar(x='b') # E: TypeVar() expects a string literal as first argument +c = TypeVar(1) # E: TypeVar() expects a string literal as first argument +T = TypeVar(b'T') # E: TypeVar() expects a string literal as first argument +d = TypeVar('D') # E: String argument 1 "D" to TypeVar(...) does not match variable name "d" +e = TypeVar('e', int, str, x=1) # E: Unexpected argument to "TypeVar()": "x" +f = TypeVar('f', (int, str), int) # E: Type expected +g = TypeVar('g', int) # E: Type variable must have at least two constrained types +h = TypeVar('h', x=(int, str)) # E: Unexpected argument to "TypeVar()": "x" +i = TypeVar('i', bound=1) # E: TypeVar "bound" must be a type +j = TypeVar('j', covariant=None) # E: TypeVar "covariant" may only be a literal bool +k = TypeVar('k', contravariant=1) # E: TypeVar "contravariant" may only be a literal bool +[out] + +[case testMoreInvalidTypevarArguments] +from typing import TypeVar +T = TypeVar('T', int, str, bound=bool) # E: TypeVar cannot have both values and an upper bound +S = TypeVar('S', covariant=True, contravariant=True) \ + # E: TypeVar cannot be both covariant and contravariant +[builtins fixtures/bool.pyi] + +[case testInvalidTypevarArgumentsGenericConstraint] +from typing import Generic, List, TypeVar +from typing_extensions import Self + +T = TypeVar("T") + +def f(x: T) -> None: + Bad = TypeVar("Bad", int, List[T]) # E: TypeVar constraint type cannot be parametrized by type variables +class C(Generic[T]): + Bad = TypeVar("Bad", int, List[T]) # E: TypeVar constraint type cannot be parametrized by type variables +class D: + Bad = TypeVar("Bad", int, List[Self]) # E: TypeVar constraint type cannot be parametrized by type variables +S = TypeVar("S", int, List[T]) # E: Type variable "__main__.T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) + +[case testInvalidTypevarValues] +from typing import TypeVar +b = TypeVar('b', *[int]) # E: Unexpected argument to "TypeVar()" +c = TypeVar('c', int, 2) # E: Invalid type: try using Literal[2] instead? +[out] + +[case testObsoleteTypevarValuesSyntax] +from typing import TypeVar +a = TypeVar('a', values=(int, str)) +[out] +main:2: error: TypeVar "values" argument not supported +main:2: error: Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...)) + +[case testLocalTypevarScope] +from typing import TypeVar +def f() -> None: + T = TypeVar('T') +def g(x: T) -> None: pass # E: Name "T" is not defined +[out] + +[case testClassTypevarScope] +from typing import TypeVar +class A: + T = TypeVar('T') +def g(x: T) -> None: pass # E: Name "T" is not defined +[out] + +[case testRedefineVariableAsTypevar] +from typing import TypeVar +x = 0 +x = TypeVar('x') # E: Cannot redefine "x" as a type variable +[out] + +[case testTypevarWithType] +from typing import TypeVar +x = TypeVar('x') # type: int # E: Cannot declare the type of a TypeVar or similar construct +[out] + +[case testRedefineTypevar] +from typing import TypeVar +t = TypeVar('t') +t = 1 # E: Invalid assignment target +[out] + +[case testRedefineTypevar2] +from typing import TypeVar +t = TypeVar('t') +def t(): pass # E: Name "t" already defined on line 2 +[out] + +[case testRedefineTypevar3] +from typing import TypeVar +t = TypeVar('t') +class t: pass # E: Name "t" already defined on line 2 +[out] + +[case testRedefineTypevar4] +from typing import TypeVar +t = TypeVar('t') +from typing import Generic as t # E: Name "t" already defined on line 2 +[out] + +[case testInvalidStrLiteralType] +def f(x: 'foo'): pass # E: Name "foo" is not defined +[out] + +[case testInvalidStrLiteralStrayBrace] +def f(x: 'int['): pass # E: Invalid type comment or annotation +[out] + +[case testInvalidStrLiteralSpaces] +def f(x: 'A B'): pass # E: Invalid type comment or annotation +[out] + +[case testInvalidMultilineLiteralType] +def f() -> "A\nB": pass # E: Invalid type comment or annotation +[out] + +[case testInconsistentOverload] +from typing import overload +def dec(x): pass +@dec # E: The implementation for an overloaded function must come last +def f(): pass +@overload +def f(): pass +[out] + +[case testInconsistentOverload2] +from typing import overload +def dec(x): pass +@dec # E: The implementation for an overloaded function must come last +def f(): pass +@overload +def f(): pass +[out] + +[case testMissingOverloadDecorator] +from typing import overload +def dec(x): pass +@dec +def f(): pass +@dec # E: Name "f" already defined on line 3 +def f(): pass +[out] + +[case testIncompatibleSignatureInComment] +import typing +def f(): # type: (int) -> int + pass +def g(x): # type: () -> int + pass +[out] +main:2: error: Type signature has too many arguments +main:4: error: Type signature has too few arguments + +[case testStaticmethodAndNonMethod] +import typing +@staticmethod +def f(): pass +class A: + def g(self) -> None: + @staticmethod + def h(): pass +[builtins fixtures/staticmethod.pyi] +[out] +main:2: error: "staticmethod" used with a non-method +main:6: error: "staticmethod" used with a non-method + +[case testClassmethodAndNonMethod] +import typing +@classmethod +def f(): pass +class A: + def g(self) -> None: + @classmethod + def h(): pass +[builtins fixtures/classmethod.pyi] +[out] +main:2: error: "classmethod" used with a non-method +main:6: error: "classmethod" used with a non-method + +[case testNonMethodProperty] +import typing +@property # E: "property" used with a non-method +def f() -> int: pass +[builtins fixtures/property.pyi] +[out] + +[case testOverloadedProperty] +from typing import overload +class A: + @overload # E: Decorators on top of @property are not supported + @property + def f(self) -> int: pass + @property # E: Only supported top decorator is @f.setter + @overload + def f(self) -> int: pass +[builtins fixtures/property.pyi] +[out] + +[case testOverloadedProperty2] +from typing import overload +class A: + @overload # E: An overloaded function outside a stub file must have an implementation + def f(self) -> int: pass + @property # E: An overload can not be a property + @overload + def f(self) -> int: pass +[builtins fixtures/property.pyi] +[out] + +[case testDecoratedProperty] +import typing +def dec(f): pass +class A: + @dec # E: Decorators on top of @property are not supported + @property + def f(self) -> int: pass + @property # OK + @dec + def g(self) -> int: pass + @dec # type: ignore[misc] + @property + def h(self) -> int: pass + @dec # type: ignore[prop-decorator] + @property + def i(self) -> int: pass +[builtins fixtures/property.pyi] +[out] + +[case testImportTwoModulesWithSameNameInFunction] +import typing +def f() -> None: + import x + import y as x # E: Name "x" already defined (by an import) + x.y +[file x.py] +y = 1 +[file y.py] +[out] + +[case testImportTwoModulesWithSameNameInGlobalContext] +import typing +import x +import y as x # E: Name "x" already defined (by an import) +x.y +[file x.py] +y = 1 +[file y.py] +[out] + +[case testListTypeAliasWithoutImport] +import typing +def f() -> List[int]: pass +[builtins fixtures/list.pyi] +[out] +main:2: error: Name "List" is not defined +main:2: note: Did you forget to import it from "typing"? (Suggestion: "from typing import List") + +[case testInvalidWithTarget] +def f(): pass +with f() as 1: pass +[out] +main:2: error: cannot assign to literal + +[case testInvalidTypeAnnotation] +import typing +def f() -> None: + 1[2] = 1 # type: int +[out] +main:3: error: Unexpected type declaration + +[case testInvalidTypeAnnotation2] +import typing +def f() -> None: + f() = 1 # type: int +[out] +main:3: error: cannot assign to function call +[out version>=3.10] +main:3: error: cannot assign to function call here. Maybe you meant '==' instead of '='? + +[case testInvalidReferenceToAttributeOfOuterClass] +class A: + class X: pass + class B: + y = X # E: Name "X" is not defined +[out] + +[case testStubPackage] +from m import x +from m import y # E: Module "m" has no attribute "y" +[file m/__init__.pyi] +x = 1 +[out] + +[case testStubPackageSubModule] +from m import x +from m import y # E: Module "m" has no attribute "y" +from m.m2 import y +from m.m2 import z # E: Module "m.m2" has no attribute "z" +[file m/__init__.pyi] +x = 1 +[file m/m2.pyi] +y = 1 +[out] + +[case testListComprehensionSpecialScoping] +class A: + x = 1 + y = 1 + z = 1 + [x for i in z if y] +[out] +main:5: error: Name "x" is not defined +main:5: error: Name "y" is not defined + +[case testTypeRedeclarationNoSpuriousWarnings] +from typing import Tuple +a = 1 # type: int +a = 's' # type: str +a = ('spam', 'spam', 'eggs', 'spam') # type: Tuple[str] +[builtins fixtures/tuple.pyi] + +[out] +main:3: error: Name "a" already defined on line 2 +main:4: error: Name "a" already defined on line 2 + +[case testDuplicateDefFromImport] +from m import A +class A: # E: Name "A" already defined (possibly by an import) + pass +[file m.py] +class A: + pass +[out] + +[case testDuplicateDefDec] +from typing import Any +def dec(x: Any) -> Any: + return x +@dec +def f() -> None: + pass +@dec # E: Name "f" already defined on line 4 +def f() -> None: + pass +[out] + +[case testDuplicateDefOverload] +from typing import overload, Any +if 1: + @overload + def f(x: int) -> None: + pass + @overload + def f(x: str) -> None: + pass + def f(x: Any) -> None: + pass +else: + def f(x: str) -> None: # E: Name "f" already defined on line 3 + pass +[out] + +[case testDuplicateDefNT] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) + +class N: # E: Name "N" already defined on line 2 + pass +[builtins fixtures/tuple.pyi] +[out] + +[case testDuplicateDefTypedDict] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) + +class Point: # E: Name "Point" already defined on line 2 + pass +[builtins fixtures/dict.pyi] + +[out] + +[case testTypeVarClassDup] +from typing import TypeVar +T = TypeVar('T') +class T: ... # E: Name "T" already defined on line 2 + +[out] + +[case testAliasDup] +from typing import List +A = List[int] +class A: ... # E: Name "A" already defined on line 2 + +[builtins fixtures/list.pyi] +[out] + +[case testNoInvalidTypeInDynamicFunctions] + +from typing import Dict, TypeVar +T = TypeVar('T') + +def f(): # Note no annotation + x: Dict[str, T] = {} + y: T + z: x + def nested(): pass + t: nested + +def g() -> None: + x: Dict[str, T] = {} # E: Type variable "__main__.T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) +[builtins fixtures/dict.pyi] +[out] + +[case testParamSpec] +from typing_extensions import ParamSpec + +TParams = ParamSpec('TParams') +TP = ParamSpec('?') # E: String argument 1 "?" to ParamSpec(...) does not match variable name "TP" +TP2: int = ParamSpec('TP2') # E: Cannot declare the type of a TypeVar or similar construct + +[out] + + +[case testBaseClassAnnotatedWithoutArgs] +# https://github.com/python/mypy/issues/11808 +from typing_extensions import Annotated +# Next line should not crash: +class A(Annotated): pass # E: Annotated[...] must have exactly one type argument and at least one annotation + +[case testInvalidUnpackTypes] +from typing_extensions import Unpack +from typing import Tuple + +heterogenous_tuple: Tuple[Unpack[Tuple[int, str]]] +homogenous_tuple: Tuple[Unpack[Tuple[int, ...]]] +bad: Tuple[Unpack[int]] # E: "int" cannot be unpacked (must be tuple or TypeVarTuple) +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleErrors] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +TVariadic = TypeVarTuple('TVariadic') +TVariadic2 = TypeVarTuple('TVariadic2') +TP = TypeVarTuple('?') # E: String argument 1 "?" to TypeVarTuple(...) does not match variable name "TP" +TP2: int = TypeVarTuple('TP2') # E: Cannot declare the type of a TypeVar or similar construct +TP3 = TypeVarTuple() # E: Too few arguments for TypeVarTuple() +TP4 = TypeVarTuple('TP4', 'TP4') # E: Too many positional arguments for "TypeVarTuple" +TP5 = TypeVarTuple(t='TP5') # E: TypeVarTuple() expects a string literal as first argument +TP6 = TypeVarTuple('TP6', bound=int) # E: Unexpected keyword argument "bound" for "TypeVarTuple" + +x: TVariadic # E: TypeVarTuple "TVariadic" is unbound +y: Unpack[TVariadic] # E: Unpack is only valid in a variadic position + + +class Variadic(Generic[Unpack[TVariadic], Unpack[TVariadic2]]): # E: Can only use one type var tuple in a class def + pass + +def bad_args(*args: TVariadic): # E: TypeVarTuple "TVariadic" is only valid with an unpack + pass + +def bad_kwargs(**kwargs: Unpack[TVariadic]): # E: Unpack item in ** argument must be a TypedDict + pass + +[builtins fixtures/dict.pyi] diff --git a/mypy/test/data/semanal-expressions.test b/test-data/unit/semanal-expressions.test similarity index 76% rename from mypy/test/data/semanal-expressions.test rename to test-data/unit/semanal-expressions.test index 1413eadb6913..4c9baf6b1b75 100644 --- a/mypy/test/data/semanal-expressions.test +++ b/test-data/unit/semanal-expressions.test @@ -3,12 +3,11 @@ [out] MypyFile:1( ExpressionStmt:1( - ParenExpr:1( - TupleExpr:1( - IntExpr(1) - StrExpr(x) - FloatExpr(1.1) - ComplexExpr(1.1j))))) + TupleExpr:1( + IntExpr(1) + StrExpr(x) + FloatExpr(1.1) + ComplexExpr(1.1j)))) [case testMemberExpr] x = 1 @@ -16,8 +15,9 @@ x.y [out] MypyFile:1( AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) ExpressionStmt:2( MemberExpr:2( NameExpr(x [__main__.x]) @@ -81,8 +81,9 @@ not x [out] MypyFile:1( AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(1)) + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) ExpressionStmt:2( UnaryExpr:2( - @@ -159,12 +160,11 @@ MypyFile:1( NameExpr(y* [__main__.y])) IntExpr(1)) ExpressionStmt:2( - ParenExpr:2( - TupleExpr:2( - ListExpr:2() - ListExpr:2( - NameExpr(x [__main__.x]) - NameExpr(y [__main__.y])))))) + TupleExpr:2( + ListExpr:2() + ListExpr:2( + NameExpr(x [__main__.x]) + NameExpr(y [__main__.y]))))) [case testDictLiterals] x = y = 1 @@ -189,18 +189,18 @@ a = 0 [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) + NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) ExpressionStmt:2( - ParenExpr:2( - ListComprehension:2( - GeneratorExpr:2( - OpExpr:2( - + - NameExpr(x [l]) - IntExpr(1)) - NameExpr(x* [l]) - NameExpr(a [__main__.a])))))) + ListComprehension:2( + GeneratorExpr:2( + OpExpr:2( + + + NameExpr(x [l]) + IntExpr(1)) + NameExpr(x* [l]) + NameExpr(a [__main__.a]))))) [case testListComprehensionInFunction] def f(a) -> None: @@ -212,7 +212,7 @@ MypyFile:1( Args( Var(a)) def (a: Any) - Block:1( + Block:2( ExpressionStmt:2( ListComprehension:2( GeneratorExpr:2( @@ -222,14 +222,15 @@ MypyFile:1( [case testListComprehensionWithCondition] a = 0 -a = [x for x in a if x] +b = [x for x in a if x] [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) - AssignmentStmt:2( NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) + AssignmentStmt:2( + NameExpr(b* [__main__.b]) ListComprehension:2( GeneratorExpr:2( NameExpr(x [l]) @@ -243,29 +244,30 @@ a = 0 [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) + NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) ExpressionStmt:2( - ParenExpr:2( - SetComprehension:2( - GeneratorExpr:2( - OpExpr:2( - + - NameExpr(x [l]) - IntExpr(1)) - NameExpr(x* [l]) - NameExpr(a [__main__.a])))))) + SetComprehension:2( + GeneratorExpr:2( + OpExpr:2( + + + NameExpr(x [l]) + IntExpr(1)) + NameExpr(x* [l]) + NameExpr(a [__main__.a]))))) [case testSetComprehensionWithCondition] a = 0 -a = {x for x in a if x} +b = {x for x in a if x} [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) - AssignmentStmt:2( NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) + AssignmentStmt:2( + NameExpr(b* [__main__.b]) SetComprehension:2( GeneratorExpr:2( NameExpr(x [l]) @@ -279,29 +281,30 @@ a = 0 [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) + NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) ExpressionStmt:2( - ParenExpr:2( - DictionaryComprehension:2( + DictionaryComprehension:2( + NameExpr(x [l]) + OpExpr:2( + + NameExpr(x [l]) - OpExpr:2( - + - NameExpr(x [l]) - IntExpr(1)) - NameExpr(x* [l]) - NameExpr(a [__main__.a]))))) + IntExpr(1)) + NameExpr(x* [l]) + NameExpr(a [__main__.a])))) [case testDictionaryComprehensionWithCondition] a = 0 -a = {x: x + 1 for x in a if x} +b = {x: x + 1 for x in a if x} [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) - AssignmentStmt:2( NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) + AssignmentStmt:2( + NameExpr(b* [__main__.b]) DictionaryComprehension:2( NameExpr(x [l]) OpExpr:2( @@ -314,12 +317,13 @@ MypyFile:1( [case testGeneratorExpression] a = 0 -x for x in a +(x for x in a) [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) + NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) ExpressionStmt:2( GeneratorExpr:2( NameExpr(x [l]) @@ -328,21 +332,21 @@ MypyFile:1( [case testGeneratorExpressionNestedIndex] a = 0 -x for x, (y, z) in a +(x for x, (y, z) in a) [out] MypyFile:1( AssignmentStmt:1( - NameExpr(a* [__main__.a]) - IntExpr(0)) + NameExpr(a [__main__.a]) + IntExpr(0) + builtins.int) ExpressionStmt:2( GeneratorExpr:2( NameExpr(x [l]) TupleExpr:2( NameExpr(x* [l]) - ParenExpr:2( - TupleExpr:2( - NameExpr(y* [l]) - NameExpr(z* [l])))) + TupleExpr:2( + NameExpr(y* [l]) + NameExpr(z* [l]))) NameExpr(a [__main__.a])))) [case testLambda] @@ -351,10 +355,11 @@ lambda: x [out] MypyFile:1( AssignmentStmt:1( - NameExpr(x* [__main__.x]) - IntExpr(0)) + NameExpr(x [__main__.x]) + IntExpr(0) + builtins.int) ExpressionStmt:2( - FuncExpr:2( + LambdaExpr:2( Block:2( ReturnStmt:2( NameExpr(x [__main__.x])))))) @@ -364,7 +369,7 @@ lambda x, y: x + y [out] MypyFile:1( ExpressionStmt:1( - FuncExpr:1( + LambdaExpr:1( Args( Var(x) Var(y)) @@ -385,3 +390,17 @@ MypyFile:1( NameExpr(None [builtins.None])) NameExpr(int [builtins.int]) NameExpr(str [builtins.str])))) + +[case testDictWithKeywordArgs] +dict(a=1, b=str()) +[builtins fixtures/dict.pyi] +[out] +MypyFile:1( + ExpressionStmt:1( + DictExpr:1( + StrExpr(a) + IntExpr(1) + StrExpr(b) + CallExpr:1( + NameExpr(str [builtins.str]) + Args())))) diff --git a/test-data/unit/semanal-lambda.test b/test-data/unit/semanal-lambda.test new file mode 100644 index 000000000000..cc2307b97217 --- /dev/null +++ b/test-data/unit/semanal-lambda.test @@ -0,0 +1,94 @@ +[case testLambdaInheritsCheckedContextFromFunc] +def g(): + return lambda x: UNDEFINED in x +[out] +MypyFile:1( + FuncDef:1( + g + Block:2( + ReturnStmt:2( + LambdaExpr:2( + Args( + Var(x)) + Block:2( + ReturnStmt:2( + ComparisonExpr:2( + in + NameExpr(UNDEFINED) + NameExpr(x [l]))))))))) + +[case testLambdaInheritsCheckedContextFromFuncForced] +# flags: --check-untyped-defs +def g(): + return lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromTypedFunc] +def g() -> None: + return lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromTypedFuncForced] +# flags: --check-untyped-defs +def g() -> None: + return lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromModule] +g = lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromModuleForce] +# flags: --check-untyped-defs +g = lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromModuleLambdaStack] +g = lambda: lambda: lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromModuleLambdaStackForce] +# flags: --check-untyped-defs +g = lambda: lambda: lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromFuncLambdaStack] +def g(): + return lambda: lambda: lambda x: UNDEFINED in x +[out] +MypyFile:1( + FuncDef:1( + g + Block:2( + ReturnStmt:2( + LambdaExpr:2( + Block:2( + ReturnStmt:2( + LambdaExpr:2( + Block:2( + ReturnStmt:2( + LambdaExpr:2( + Args( + Var(x)) + Block:2( + ReturnStmt:2( + ComparisonExpr:2( + in + NameExpr(UNDEFINED) + NameExpr(x [l]))))))))))))))) + +[case testLambdaInheritsCheckedContextFromFuncLambdaStackForce] +# flags: --check-untyped-defs +def g(): + return lambda: lambda: lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromTypedFuncLambdaStack] +def g() -> None: + return lambda: lambda: lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromTypedFuncLambdaStackForce] +# flags: --check-untyped-defs +def g() -> None: + return lambda: lambda: lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromClassLambdaStack] +class A: + g = lambda: lambda: lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined + +[case testLambdaInheritsCheckedContextFromClassLambdaStackForce] +# flags: --check-untyped-defs +class A: + g = lambda: lambda: lambda x: UNDEFINED in x # E: Name "UNDEFINED" is not defined diff --git a/test-data/unit/semanal-literal.test b/test-data/unit/semanal-literal.test new file mode 100644 index 000000000000..4c100add6ec0 --- /dev/null +++ b/test-data/unit/semanal-literal.test @@ -0,0 +1,27 @@ +[case testLiteralSemanalBasicAssignment] +from typing_extensions import Literal +foo: Literal[3] +[out] +MypyFile:1( + ImportFrom:1(typing_extensions, [Literal]) + AssignmentStmt:2( + NameExpr(foo [__main__.foo]) + TempNode:2( + Any) + Literal[3])) + +[case testLiteralSemanalInFunction] +from typing_extensions import Literal +def foo(a: Literal[1], b: Literal[" foo "]) -> Literal[True]: pass +[builtins fixtures/bool.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing_extensions, [Literal]) + FuncDef:2( + foo + Args( + Var(a) + Var(b)) + def (a: Literal[1], b: Literal[' foo ']) -> Literal[True] + Block:2( + PassStmt:2()))) diff --git a/test-data/unit/semanal-modules.test b/test-data/unit/semanal-modules.test new file mode 100644 index 000000000000..d52dd953aea2 --- /dev/null +++ b/test-data/unit/semanal-modules.test @@ -0,0 +1,930 @@ +-- NOTE: If a module has a name starting or ending with _, it is skipped in +-- output. + +[case testImport] +import x +x.y +[file x.py] +y = 1 +[out] +MypyFile:1( + Import:1(x) + ExpressionStmt:2( + MemberExpr:2( + NameExpr(x) + y [x.y]))) +MypyFile:1( + tmp/x.py + AssignmentStmt:1( + NameExpr(y [x.y]) + IntExpr(1) + builtins.int)) + +[case testImportedNameInType] +import m +x = None # type: m.c +[file m.py] +class c: pass +[out] +MypyFile:1( + Import:1(m) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + m.c)) +MypyFile:1( + tmp/m.py + ClassDef:1( + c + PassStmt:1())) + +[case testImportFrom] +from m import y +x = y +[file m.py] +y = 1 +[out] +MypyFile:1( + ImportFrom:1(m, [y]) + AssignmentStmt:2( + NameExpr(x* [__main__.x]) + NameExpr(y [m.y]))) +MypyFile:1( + tmp/m.py + AssignmentStmt:1( + NameExpr(y [m.y]) + IntExpr(1) + builtins.int)) + +[case testImportFromType] +from m import c +x = None # type: c +[file m.py] +class c: pass +[out] +MypyFile:1( + ImportFrom:1(m, [c]) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + m.c)) +MypyFile:1( + tmp/m.py + ClassDef:1( + c + PassStmt:1())) + +[case testImportMultiple] +import _m, _n +_m.x, _n.y +[fixture _m.py] +x = 1 +[fixture _n.py] +y = 2 +[out] +MypyFile:1( + Import:1(_m, _n) + ExpressionStmt:2( + TupleExpr:2( + MemberExpr:2( + NameExpr(_m) + x [_m.x]) + MemberExpr:2( + NameExpr(_n) + y [_n.y])))) + +[case testImportAs] +import _m as n +n.x +[fixture _m.py] +x = 1 +[out] +MypyFile:1( + Import:1(_m : n) + ExpressionStmt:2( + MemberExpr:2( + NameExpr(n [_m]) + x [_m.x]))) + +[case testImportFromMultiple] +from _m import x, y +x, y +[fixture _m.py] +x = y = 1 +[out] +MypyFile:1( + ImportFrom:1(_m, [x, y]) + ExpressionStmt:2( + TupleExpr:2( + NameExpr(x [_m.x]) + NameExpr(y [_m.y])))) + +[case testImportFromAs] +from _m import y as z +z +[fixture _m.py] +y = 1 +[out] +MypyFile:1( + ImportFrom:1(_m, [y : z]) + ExpressionStmt:2( + NameExpr(z [_m.y]))) + +[case testAccessImportedName] +from m import x +y = x +[file m.py] +from _n import x +[fixture _n.py] +x = 1 +[out] +MypyFile:1( + ImportFrom:1(m, [x]) + AssignmentStmt:2( + NameExpr(y* [__main__.y]) + NameExpr(x [_n.x]))) +MypyFile:1( + tmp/m.py + ImportFrom:1(_n, [x])) + +[case testAccessImportedName2] +import _m +y = _m.x +[fixture _m.py] +from _n import x +[fixture _n.py] +x = 1 +[out] +MypyFile:1( + Import:1(_m) + AssignmentStmt:2( + NameExpr(y* [__main__.y]) + MemberExpr:2( + NameExpr(_m) + x [_n.x]))) + +[case testAccessingImportedNameInType] +from _m import c +x = None # type: c +[fixture _m.py] +from _n import c +[fixture _n.py] +class c: pass +[out] +MypyFile:1( + ImportFrom:1(_m, [c]) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + _n.c)) + +[case testAccessingImportedNameInType2] +import _m +x = None # type: _m.c +[fixture _m.py] +from _n import c +[fixture _n.py] +class c: pass +[out] +MypyFile:1( + Import:1(_m) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + _n.c)) + +[case testAccessingImportedModule] +from _m import _n +_n.x +[fixture _m.py] +import _n +[fixture _n.py] +x = 1 +[out] +MypyFile:1( + ImportFrom:1(_m, [_n]) + ExpressionStmt:2( + MemberExpr:2( + NameExpr(_n) + x [_n.x]))) + +[case testAccessingImportedModule2] +import _m +_m._n.x +[fixture _m.py] +import _n +[fixture _n.py] +x = 1 +[out] +MypyFile:1( + Import:1(_m) + ExpressionStmt:2( + MemberExpr:2( + MemberExpr:2( + NameExpr(_m) + _n) + x [_n.x]))) + +[case testAccessTypeViaDoubleIndirection] +from _m import c +a = None # type: c +[fixture _m.py] +from _n import c +[fixture _n.py] +class c: pass +[out] +MypyFile:1( + ImportFrom:1(_m, [c]) + AssignmentStmt:2( + NameExpr(a [__main__.a]) + NameExpr(None [builtins.None]) + _n.c)) + +[case testAccessTypeViaDoubleIndirection2] +import _m +a = None # type: _m.c +[fixture _m.py] +from _n import c +[fixture _n.py] +class c: pass +[out] +MypyFile:1( + Import:1(_m) + AssignmentStmt:2( + NameExpr(a [__main__.a]) + NameExpr(None [builtins.None]) + _n.c)) + +[case testImportAsterisk] +from _m import * +x, y +[fixture _m.py] +x = y = 1 +[out] +MypyFile:1( + ImportAll:1(_m) + ExpressionStmt:2( + TupleExpr:2( + NameExpr(x [_m.x]) + NameExpr(y [_m.y])))) + +[case testImportAsteriskAndImportedNames] +from _m import * +n_.x, y +[fixture _m.py] +import n_ +from n_ import y +[fixture n_.py] +x = y = 1 +[out] +MypyFile:1( + ImportAll:1(_m) + ExpressionStmt:2( + TupleExpr:2( + MemberExpr:2( + NameExpr(n_) + x [n_.x]) + NameExpr(y [n_.y])))) + +[case testImportAsteriskAndImportedNamesInTypes] +from _m import * +x = None # type: n_.c +y = None # type: d +[fixture _m.py] +import n_ +from n_ import d +[fixture n_.py] +class c: pass +class d: pass +[out] +MypyFile:1( + ImportAll:1(_m) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + n_.c) + AssignmentStmt:3( + NameExpr(y [__main__.y]) + NameExpr(None [builtins.None]) + n_.d)) + +[case testModuleInSubdir] +import _m +_m.x +[fixture _m/__init__.py] +x = 1 +[out] +MypyFile:1( + Import:1(_m) + ExpressionStmt:2( + MemberExpr:2( + NameExpr(_m) + x [_m.x]))) + +[case testNestedModules] +import m.n +m.n.x, m.y +[fixture m/__init__.py] +y = 1 +[file m/n.py] +x = 1 +[out] +MypyFile:1( + Import:1(m.n) + ExpressionStmt:2( + TupleExpr:2( + MemberExpr:2( + MemberExpr:2( + NameExpr(m) + n [m.n]) + x [m.n.x]) + MemberExpr:2( + NameExpr(m) + y [m.y])))) +MypyFile:1( + tmp/m/n.py + AssignmentStmt:1( + NameExpr(x [m.n.x]) + IntExpr(1) + builtins.int)) + +[case testImportFromSubmodule] +from m._n import x +x +[fixture m/__init__.py] +[fixture m/_n.py] +x = 1 +[out] +MypyFile:1( + ImportFrom:1(m._n, [x]) + ExpressionStmt:2( + NameExpr(x [m._n.x]))) + +[case testImportAllFromSubmodule] +from m._n import * +x, y +[fixture m/__init__.py] +[fixture m/_n.py] +x = y = 1 +[out] +MypyFile:1( + ImportAll:1(m._n) + ExpressionStmt:2( + TupleExpr:2( + NameExpr(x [m._n.x]) + NameExpr(y [m._n.y])))) + +[case testSubmodulesAndTypes] +import m._n +x = None # type: m._n.c +[fixture m/__init__.py] +[fixture m/_n.py] +class c: pass +[out] +MypyFile:1( + Import:1(m._n) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + m._n.c)) + +[case testSubmodulesAndTypes2] +from m._n import c +x = None # type: c +[fixture m/__init__.py] +[fixture m/_n.py] +class c: pass +[out] +MypyFile:1( + ImportFrom:1(m._n, [c]) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + m._n.c)) + +[case testFromPackageImportModule] +from m import _n +_n.x +[fixture m/__init__.py] +[fixture m/_n.py] +x = 1 +[out] +MypyFile:1( + ImportFrom:1(m, [_n]) + ExpressionStmt:2( + MemberExpr:2( + NameExpr(_n [m._n]) + x [m._n.x]))) + +[case testDeeplyNestedModule] +import m.n.k +m.n.k.x +m.n.b +m.a +[fixture m/__init__.py] +a = 1 +[fixture m/n/__init__.py] +b = 1 +[file m/n/k.py] +x = 1 +[out] +MypyFile:1( + Import:1(m.n.k) + ExpressionStmt:2( + MemberExpr:2( + MemberExpr:2( + MemberExpr:2( + NameExpr(m) + n [m.n]) + k [m.n.k]) + x [m.n.k.x])) + ExpressionStmt:3( + MemberExpr:3( + MemberExpr:3( + NameExpr(m) + n [m.n]) + b [m.n.b])) + ExpressionStmt:4( + MemberExpr:4( + NameExpr(m) + a [m.a]))) +MypyFile:1( + tmp/m/n/k.py + AssignmentStmt:1( + NameExpr(x [m.n.k.x]) + IntExpr(1) + builtins.int)) + +[case testImportInSubmodule] +import m._n +y = m._n.x +[fixture m/__init__.py] +[fixture m/_n.py] +from m._k import x +[fixture m/_k.py] +x = 1 +[out] +MypyFile:1( + Import:1(m._n) + AssignmentStmt:2( + NameExpr(y* [__main__.y]) + MemberExpr:2( + MemberExpr:2( + NameExpr(m) + _n [m._n]) + x [m._k.x]))) + +[case testBuiltinsUsingModule] +o = None # type: __builtins__.object +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(o [__main__.o]) + NameExpr(None [builtins.None]) + builtins.object)) + +[case testImplicitAccessToBuiltins] +object +[out] +MypyFile:1( + ExpressionStmt:1( + NameExpr(object [builtins.object]))) + +[case testAssignmentToModuleAttribute] +import _m +_m.x = ( + _m.x) +[fixture _m.py] +x = None +[out] +MypyFile:1( + Import:1(_m) + AssignmentStmt:2( + MemberExpr:2( + NameExpr(_m) + x [_m.x]) + MemberExpr:3( + NameExpr(_m) + x [_m.x]))) + +[case testAssignmentThatRefersToModule] +import _m +_m.x[None] = None +[fixture _m.py] +x = None +[out] +MypyFile:1( + Import:1(_m) + AssignmentStmt:2( + IndexExpr:2( + MemberExpr:2( + NameExpr(_m) + x [_m.x]) + NameExpr(None [builtins.None])) + NameExpr(None [builtins.None]))) + +[case testImportInBlock] +if 1: + import _x + _x.y +[fixture _x.py] +y = 1 +[out] +MypyFile:1( + IfStmt:1( + If( + IntExpr(1)) + Then( + Import:2(_x) + ExpressionStmt:3( + MemberExpr:3( + NameExpr(_x) + y [_x.y]))))) + +[case testImportInFunction] +def f() -> None: + import _x + _x.y +[fixture _x.py] +y = 1 +[out] +MypyFile:1( + FuncDef:1( + f + def () + Block:2( + Import:2(_x) + ExpressionStmt:3( + MemberExpr:3( + NameExpr(_x) + y [_x.y]))))) + +[case testImportInClassBody] +class A: + from _x import y + z = y +[fixture _x.py] +y = 1 +[out] +MypyFile:1( + ClassDef:1( + A + ImportFrom:2(_x, [y]) + AssignmentStmt:3( + NameExpr(z* [m]) + NameExpr(y [__main__.A.y])))) + +[case testImportInClassBody2] +class A: + import _x + z = _x.y +[fixture _x.py] +y = 1 +[out] +MypyFile:1( + ClassDef:1( + A + Import:2(_x) + AssignmentStmt:3( + NameExpr(z* [m]) + MemberExpr:3( + NameExpr(_x) + y [_x.y])))) + +[case testImportModuleTwice] +def f() -> None: + import x + import x + x.y +[file x.py] +y = 1 +[out] +MypyFile:1( + FuncDef:1( + f + def () + Block:2( + Import:2(x) + Import:3(x) + ExpressionStmt:4( + MemberExpr:4( + NameExpr(x) + y [x.y]))))) +MypyFile:1( + tmp/x.py + AssignmentStmt:1( + NameExpr(y [x.y]) + IntExpr(1) + builtins.int)) + +[case testRelativeImport0] +import m.x +m.x.z.y +[fixture m/__init__.py] +[file m/x.py] +from . import z +[file m/z.py] +y = 1 +[out] +MypyFile:1( + Import:1(m.x) + ExpressionStmt:2( + MemberExpr:2( + MemberExpr:2( + MemberExpr:2( + NameExpr(m) + x [m.x]) + z [m.z]) + y [m.z.y]))) +MypyFile:1( + tmp/m/x.py + ImportFrom:1(., [z])) +MypyFile:1( + tmp/m/z.py + AssignmentStmt:1( + NameExpr(y [m.z.y]) + IntExpr(1) + builtins.int)) + +[case testRelativeImport1] +import m.t.b as b +b.x.y +b.z.y +[fixture m/__init__.py] +[file m/x.py] +y = 1 +[file m/z.py] +y = 3 +[fixture m/t/__init__.py] +[file m/t/b.py] +from .. import x, z +[out] +MypyFile:1( + Import:1(m.t.b : b) + ExpressionStmt:2( + MemberExpr:2( + MemberExpr:2( + NameExpr(b [m.t.b]) + x [m.x]) + y [m.x.y])) + ExpressionStmt:3( + MemberExpr:3( + MemberExpr:3( + NameExpr(b [m.t.b]) + z [m.z]) + y [m.z.y]))) +MypyFile:1( + tmp/m/t/b.py + ImportFrom:1(.., [x, z])) +MypyFile:1( + tmp/m/x.py + AssignmentStmt:1( + NameExpr(y [m.x.y]) + IntExpr(1) + builtins.int)) +MypyFile:1( + tmp/m/z.py + AssignmentStmt:1( + NameExpr(y [m.z.y]) + IntExpr(3) + builtins.int)) + +[case testRelativeImport2] +import m.t.b as b +b.xy +b.zy +[fixture m/__init__.py] +[file m/x.py] +y = 1 +[file m/z.py] +y = 3 +[fixture m/t/__init__.py] +[file m/t/b.py] +from ..x import y as xy +from ..z import y as zy +[out] +MypyFile:1( + Import:1(m.t.b : b) + ExpressionStmt:2( + MemberExpr:2( + NameExpr(b [m.t.b]) + xy [m.x.y])) + ExpressionStmt:3( + MemberExpr:3( + NameExpr(b [m.t.b]) + zy [m.z.y]))) +MypyFile:1( + tmp/m/t/b.py + ImportFrom:1(..x, [y : xy]) + ImportFrom:2(..z, [y : zy])) +MypyFile:1( + tmp/m/x.py + AssignmentStmt:1( + NameExpr(y [m.x.y]) + IntExpr(1) + builtins.int)) +MypyFile:1( + tmp/m/z.py + AssignmentStmt:1( + NameExpr(y [m.z.y]) + IntExpr(3) + builtins.int)) + +[case testRelativeImport3] +import m.t +m.zy +m.xy +m.t.y +[fixture m/__init__.py] +from .x import * +from .z import * +[file m/x.py] +from .z import zy as xy +[file m/z.py] +zy = 3 +[fixture m/t/__init__.py] +from .b import * +[file m/t/b.py] +from .. import xy as y +[out] +MypyFile:1( + Import:1(m.t) + ExpressionStmt:2( + MemberExpr:2( + NameExpr(m) + zy [m.z.zy])) + ExpressionStmt:3( + MemberExpr:3( + NameExpr(m) + xy [m.z.zy])) + ExpressionStmt:4( + MemberExpr:4( + MemberExpr:4( + NameExpr(m) + t [m.t]) + y [m.z.zy]))) +MypyFile:1( + tmp/m/t/b.py + ImportFrom:1(.., [xy : y])) +MypyFile:1( + tmp/m/x.py + ImportFrom:1(.z, [zy : xy])) +MypyFile:1( + tmp/m/z.py + AssignmentStmt:1( + NameExpr(zy [m.z.zy]) + IntExpr(3) + builtins.int)) + +[case testRelativeImportFromSameModule] +import m.x +[fixture m/__init__.py] +[file m/x.py] +from .x import nonexistent +[out] +tmp/m/x.py:1: error: Module "m.x" has no attribute "nonexistent" + +[case testImportFromSameModule] +import m.x +[fixture m/__init__.py] +[file m/x.py] +from m.x import nonexistent +[out] +tmp/m/x.py:1: error: Module "m.x" has no attribute "nonexistent" + +[case testImportMisspellingSingleCandidate] +import f +[fixture m/__init__.py] +[file m/x.py] +def some_function(): + pass +[file f.py] +from m.x import somefunction +[out] +tmp/f.py:1: error: Module "m.x" has no attribute "somefunction"; maybe "some_function"? + +[case testImportMisspellingMultipleCandidates] +import f +[fixture m/__init__.py] +[file m/x.py] +def some_function(): + pass +def somef_unction(): + pass +[file f.py] +from m.x import somefunction +[out] +tmp/f.py:1: error: Module "m.x" has no attribute "somefunction"; maybe "some_function" or "somef_unction"? + +[case testImportMisspellingMultipleCandidatesTruncated] +import f +[fixture m/__init__.py] +[file m/x.py] +def some_function(): + pass +def somef_unction(): + pass +def somefu_nction(): + pass +def somefun_ction(): + pass +[file f.py] +from m.x import somefunction +[out] +tmp/f.py:1: error: Module "m.x" has no attribute "somefunction"; maybe "some_function", "somef_unction", or "somefu_nction"? + +[case testFromImportAsInStub] +from m import * +x +y # E: Name "y" is not defined +[file m.pyi] +from m2 import x as x +from m2 import y +[file m2.py] +x = 1 +y = 2 +[out] + +[case testFromImportAsInNonStub] +from m_ import * +x +y +[fixture m_.py] +from m2_ import x as x +from m2_ import y +[fixture m2_.py] +x = 1 +y = 2 +[out] +MypyFile:1( + ImportAll:1(m_) + ExpressionStmt:2( + NameExpr(x [m2_.x])) + ExpressionStmt:3( + NameExpr(y [m2_.y]))) + +[case testImportAsInStub] +from m import * +m2 +m3 # E: Name "m3" is not defined +[file m.pyi] +import m2 as m2 +import m3 +[file m2.py] +[file m3.py] +[out] + +[case testImportAsInNonStub] +from m_ import * +m2_ +m3_ +[fixture m_.py] +import m2_ as m2_ +import m3_ +[fixture m2_.py] +[fixture m3_.py] +[out] +MypyFile:1( + ImportAll:1(m_) + ExpressionStmt:2( + NameExpr(m2_)) + ExpressionStmt:3( + NameExpr(m3_))) + +[case testErrorsInMultipleModules] +import m +x +[file m.py] +y +[out] +tmp/m.py:1: error: Name "y" is not defined +main:2: error: Name "x" is not defined + +[case testImportTwice] +import typing +from x import a, a # ok (we could give a warning, but this is valid) +def f() -> None: + from x import a + from x import a # ok +import x +import x # ok, since we may import multiple submodules of a package +[file x.py] +a = 1 +[out] +MypyFile:1( + Import:1(typing) + ImportFrom:2(x, [a, a]) + FuncDef:3( + f + def () + Block:4( + ImportFrom:4(x, [a]) + ImportFrom:5(x, [a]))) + Import:6(x) + Import:7(x)) +MypyFile:1( + tmp/x.py + AssignmentStmt:1( + NameExpr(a [x.a]) + IntExpr(1) + builtins.int)) diff --git a/test-data/unit/semanal-namedtuple.test b/test-data/unit/semanal-namedtuple.test new file mode 100644 index 000000000000..16944391da86 --- /dev/null +++ b/test-data/unit/semanal-namedtuple.test @@ -0,0 +1,247 @@ +-- Semantic analysis of named tuples + +[case testSimpleNamedtuple] +from collections import namedtuple +N = namedtuple('N', ['a']) +def f() -> N: pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(collections, [namedtuple]) + AssignmentStmt:2( + NameExpr(N* [__main__.N]) + NamedTupleExpr:2(N, Tuple[Any])) + FuncDef:3( + f + def () -> Tuple[Any, fallback=__main__.N] + Block:3( + PassStmt:3()))) + +[case testTwoItemNamedtuple] +from collections import namedtuple +N = namedtuple('N', ['a', 'xyz']) +def f() -> N: pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(collections, [namedtuple]) + AssignmentStmt:2( + NameExpr(N* [__main__.N]) + NamedTupleExpr:2(N, Tuple[Any, Any])) + FuncDef:3( + f + def () -> Tuple[Any, Any, fallback=__main__.N] + Block:3( + PassStmt:3()))) + +[case testTwoItemNamedtupleWithTupleFieldNames] +from collections import namedtuple +N = namedtuple('N', ('a', 'xyz')) +def f() -> N: pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(collections, [namedtuple]) + AssignmentStmt:2( + NameExpr(N* [__main__.N]) + NamedTupleExpr:2(N, Tuple[Any, Any])) + FuncDef:3( + f + def () -> Tuple[Any, Any, fallback=__main__.N] + Block:3( + PassStmt:3()))) + +[case testTwoItemNamedtupleWithShorthandSyntax] +from collections import namedtuple +N = namedtuple('N', ' a xyz ') +def f() -> N: pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(collections, [namedtuple]) + AssignmentStmt:2( + NameExpr(N* [__main__.N]) + NamedTupleExpr:2(N, Tuple[Any, Any])) + FuncDef:3( + f + def () -> Tuple[Any, Any, fallback=__main__.N] + Block:3( + PassStmt:3()))) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [NamedTuple]) + AssignmentStmt:2( + NameExpr(N* [__main__.N]) + NamedTupleExpr:2(N, Tuple[builtins.int, builtins.str]))) + +[case testNamedTupleWithTupleFieldNamesWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', (('a', int), + ('b', str))) +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [NamedTuple]) + AssignmentStmt:2( + NameExpr(N* [__main__.N]) + NamedTupleExpr:2(N, Tuple[builtins.int, builtins.str]))) + +[case testNamedTupleBaseClass] +from collections import namedtuple +N = namedtuple('N', ['x']) +class A(N): pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(collections, [namedtuple]) + AssignmentStmt:2( + NameExpr(N* [__main__.N]) + NamedTupleExpr:2(N, Tuple[Any])) + ClassDef:3( + A + TupleType( + Tuple[Any, fallback=__main__.N]) + BaseType( + __main__.N) + PassStmt:3())) + +[case testNamedTupleBaseClass2] +from collections import namedtuple +class A(namedtuple('N', ['x'])): pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(collections, [namedtuple]) + ClassDef:2( + A + TupleType( + Tuple[Any, fallback=__main__.N@2]) + BaseType( + __main__.N@2) + PassStmt:2())) + +[case testNamedTupleBaseClassWithItemTypes] +from typing import NamedTuple +class A(NamedTuple('N', [('x', int)])): pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [NamedTuple]) + ClassDef:2( + A + TupleType( + Tuple[builtins.int, fallback=__main__.N@2]) + BaseType( + __main__.N@2) + PassStmt:2())) + +-- Errors + +[case testNamedTupleWithTooFewArguments] +from collections import namedtuple +N = namedtuple('N') # E: Too few arguments for "namedtuple()" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleWithInvalidName] +from collections import namedtuple +N = namedtuple(1, ['x']) # E: "namedtuple()" expects a string literal as the first argument +[builtins fixtures/tuple.pyi] + +[case testNamedTupleWithInvalidItems] +from collections import namedtuple +N = namedtuple('N', 1) # E: List or tuple literal expected as the second argument to "namedtuple()" +[builtins fixtures/tuple.pyi] + +[case testNamedTupleWithInvalidItems2] +from collections import namedtuple +N = namedtuple('N', ['x', 1]) # E: String literal expected as "namedtuple()" item +[builtins fixtures/tuple.pyi] + +[case testNamedTupleWithUnderscoreItemName] +from collections import namedtuple +N = namedtuple('N', ['_fallback']) # E: "namedtuple()" field name "_fallback" starts with an underscore +[builtins fixtures/tuple.pyi] + +-- NOTE: The following code works at runtime but is not yet supported by mypy. +-- Keyword arguments may potentially be supported in the future. +[case testNamedTupleWithNonpositionalArgs] +from collections import namedtuple +N = namedtuple(typename='N', field_names=['x']) # E: Unexpected arguments to "namedtuple()" +[builtins fixtures/tuple.pyi] + +[case testTypingNamedTupleWithTooFewArguments] +from typing import NamedTuple +N = NamedTuple('N') # E: Too few arguments for "NamedTuple()" +[builtins fixtures/tuple.pyi] + +[case testTypingNamedTupleWithManyArguments] +from typing import NamedTuple +N = NamedTuple('N', [], []) # E: Too many arguments for "NamedTuple()" +[builtins fixtures/tuple.pyi] + +[case testTypingNamedTupleWithInvalidName] +from typing import NamedTuple +N = NamedTuple(1, ['x']) # E: "NamedTuple()" expects a string literal as the first argument +[builtins fixtures/tuple.pyi] + +[case testTypingNamedTupleWithInvalidItems] +from typing import NamedTuple +N = NamedTuple('N', 1) # E: List or tuple literal expected as the second argument to "NamedTuple()" +[builtins fixtures/tuple.pyi] + +[case testTypingNamedTupleWithUnderscoreItemName] +from typing import NamedTuple +N = NamedTuple('N', [('_fallback', int)]) # E: "NamedTuple()" field name "_fallback" starts with an underscore +[builtins fixtures/tuple.pyi] + +[case testTypingNamedTupleWithUnexpectedNames] +from typing import NamedTuple +N = NamedTuple(name='N', fields=[]) # E: Unexpected arguments to "NamedTuple()" +[builtins fixtures/tuple.pyi] + +-- NOTE: The following code works at runtime but is not yet supported by mypy. +-- Keyword arguments may potentially be supported in the future. +[case testNamedTupleWithNonpositionalArgs2] +from collections import namedtuple +N = namedtuple(typename='N', field_names=['x']) # E: Unexpected arguments to "namedtuple()" +[builtins fixtures/tuple.pyi] + +[case testInvalidNamedTupleBaseClass] +from typing import NamedTuple +class A(NamedTuple('N', [1])): pass # E: Tuple expected as "NamedTuple()" field +class B(A): pass +[builtins fixtures/tuple.pyi] + +[case testInvalidNamedTupleBaseClass2] + +class A(NamedTuple('N', [1])): pass +class B(A): pass +[out] +main:2: error: Unsupported dynamic base class "NamedTuple" +main:2: error: Name "NamedTuple" is not defined + +[case testNamedTupleWithDecorator] +from typing import final, NamedTuple + +@final +class A(NamedTuple("N", [("x", int)])): + pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [final, NamedTuple]) + ClassDef:4( + A + TupleType( + Tuple[builtins.int, fallback=__main__.N@4]) + Decorators( + NameExpr(final [typing.final])) + BaseType( + __main__.N@4) + PassStmt:5())) diff --git a/test-data/unit/semanal-python310.test b/test-data/unit/semanal-python310.test new file mode 100644 index 000000000000..e96a3ca9d777 --- /dev/null +++ b/test-data/unit/semanal-python310.test @@ -0,0 +1,214 @@ +-- Python 3.10 semantic analysis test cases. + +[case testCapturePattern] +x = 1 +match x: + case a: + a +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:2( + NameExpr(x [__main__.x]) + Pattern( + AsPattern:3( + NameExpr(a* [__main__.a]))) + Body( + ExpressionStmt:4( + NameExpr(a [__main__.a]))))) + +[case testCapturePatternOutliving] +x = 1 +match x: + case a: + pass +a +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:2( + NameExpr(x [__main__.x]) + Pattern( + AsPattern:3( + NameExpr(a* [__main__.a]))) + Body( + PassStmt:4())) + ExpressionStmt:5( + NameExpr(a [__main__.a]))) + +[case testNestedCapturePatterns] +x = 1 +match x: + case ([a], {'k': b}): + a + b +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:2( + NameExpr(x [__main__.x]) + Pattern( + SequencePattern:3( + SequencePattern:3( + AsPattern:3( + NameExpr(a* [__main__.a]))) + MappingPattern:3( + Key( + StrExpr(k)) + Value( + AsPattern:3( + NameExpr(b* [__main__.b])))))) + Body( + ExpressionStmt:4( + NameExpr(a [__main__.a])) + ExpressionStmt:5( + NameExpr(b [__main__.b]))))) + +[case testMappingPatternRest] +x = 1 +match x: + case {**r}: + r +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:2( + NameExpr(x [__main__.x]) + Pattern( + MappingPattern:3( + Rest( + NameExpr(r* [__main__.r])))) + Body( + ExpressionStmt:4( + NameExpr(r [__main__.r]))))) + + +[case testAsPattern] +x = 1 +match x: + case 1 as a: + a +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:2( + NameExpr(x [__main__.x]) + Pattern( + AsPattern:3( + ValuePattern:3( + IntExpr(1)) + NameExpr(a* [__main__.a]))) + Body( + ExpressionStmt:4( + NameExpr(a [__main__.a]))))) + +[case testGuard] +x = 1 +a = 1 +match x: + case 1 if a: + pass +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + AssignmentStmt:2( + NameExpr(a [__main__.a]) + IntExpr(1) + builtins.int) + MatchStmt:3( + NameExpr(x [__main__.x]) + Pattern( + ValuePattern:4( + IntExpr(1))) + Guard( + NameExpr(a [__main__.a])) + Body( + PassStmt:5()))) + +[case testCapturePatternInGuard] +x = 1 +match x: + case a if a: + pass +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:2( + NameExpr(x [__main__.x]) + Pattern( + AsPattern:3( + NameExpr(a* [__main__.a]))) + Guard( + NameExpr(a [__main__.a])) + Body( + PassStmt:4()))) + +[case testAsPatternInGuard] +x = 1 +match x: + case 1 as a if a: + pass +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:2( + NameExpr(x [__main__.x]) + Pattern( + AsPattern:3( + ValuePattern:3( + IntExpr(1)) + NameExpr(a* [__main__.a]))) + Guard( + NameExpr(a [__main__.a])) + Body( + PassStmt:4()))) + +[case testValuePattern] +import _a + +x = 1 +match x: + case _a.b: + pass +[fixture _a.py] +b = 1 +[out] +MypyFile:1( + Import:1(_a) + AssignmentStmt:3( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + MatchStmt:4( + NameExpr(x [__main__.x]) + Pattern( + ValuePattern:5( + MemberExpr:5( + NameExpr(_a) + b [_a.b]))) + Body( + PassStmt:6()))) diff --git a/test-data/unit/semanal-statements.test b/test-data/unit/semanal-statements.test new file mode 100644 index 000000000000..c143805f4564 --- /dev/null +++ b/test-data/unit/semanal-statements.test @@ -0,0 +1,1315 @@ +[case testReturn] +def f(x): return x +def g(): return +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + Block:1( + ReturnStmt:1( + NameExpr(x [l])))) + FuncDef:2( + g + Block:2( + ReturnStmt:2()))) + +[case testRaise] +raise object() +[out] +MypyFile:1( + RaiseStmt:1( + CallExpr:1( + NameExpr(object [builtins.object]) + Args()))) + +[case testYield] +def f(): yield f +[out] +MypyFile:1( + FuncDef:1( + f + Generator + Block:1( + ExpressionStmt:1( + YieldExpr:1( + NameExpr(f [__main__.f])))))) + +[case testAssert] +assert object +[out] +MypyFile:1( + AssertStmt:1( + NameExpr(object [builtins.object]))) + +[case testOperatorAssignment] +x = y = 1 +x += y +y |= x +[out] +MypyFile:1( + AssignmentStmt:1( + Lvalues( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + IntExpr(1)) + OperatorAssignmentStmt:2( + + + NameExpr(x [__main__.x]) + NameExpr(y [__main__.y])) + OperatorAssignmentStmt:3( + | + NameExpr(y [__main__.y]) + NameExpr(x [__main__.x]))) + +[case testWhile] +x = y = 1 +while x: + y +[out] +MypyFile:1( + AssignmentStmt:1( + Lvalues( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + IntExpr(1)) + WhileStmt:2( + NameExpr(x [__main__.x]) + Block:3( + ExpressionStmt:3( + NameExpr(y [__main__.y]))))) + +[case testFor] +for x in object: + x +[out] +MypyFile:1( + ForStmt:1( + NameExpr(x* [__main__.x]) + NameExpr(object [builtins.object]) + Block:2( + ExpressionStmt:2( + NameExpr(x [__main__.x]))))) + +[case testForInFunction] +def f(): + for x in f: + x +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ForStmt:2( + NameExpr(x* [l]) + NameExpr(f [__main__.f]) + Block:3( + ExpressionStmt:3( + NameExpr(x [l]))))))) + +[case testMultipleForIndexVars] +for x, y in []: + x, y +[out] +MypyFile:1( + ForStmt:1( + TupleExpr:1( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + ListExpr:1() + Block:2( + ExpressionStmt:2( + TupleExpr:2( + NameExpr(x [__main__.x]) + NameExpr(y [__main__.y])))))) + +[case testForIndexVarScope] +for x in []: + pass +x +[out] +MypyFile:1( + ForStmt:1( + NameExpr(x* [__main__.x]) + ListExpr:1() + Block:2( + PassStmt:2())) + ExpressionStmt:3( + NameExpr(x [__main__.x]))) + +[case testForIndexVarScope2] +def f(): + for x in []: + pass + x +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + ForStmt:2( + NameExpr(x* [l]) + ListExpr:2() + Block:3( + PassStmt:3())) + ExpressionStmt:4( + NameExpr(x [l]))))) + +[case testReusingForLoopIndexVariable] +# flags: --allow-redefinition +for x in None: + pass +for x in None: + pass +[out] +MypyFile:1( + ForStmt:2( + NameExpr(x'* [__main__.x']) + NameExpr(None [builtins.None]) + Block:3( + PassStmt:3())) + ForStmt:4( + NameExpr(x* [__main__.x]) + NameExpr(None [builtins.None]) + Block:5( + PassStmt:5()))) + +[case testReusingForLoopIndexVariable2] +# flags: --allow-redefinition +def f(): + for x in None: + pass + for x in None: + pass +[out] +MypyFile:1( + FuncDef:2( + f + Block:3( + ForStmt:3( + NameExpr(x* [l]) + NameExpr(None [builtins.None]) + Block:4( + PassStmt:4())) + ForStmt:5( + NameExpr(x'* [l]) + NameExpr(None [builtins.None]) + Block:6( + PassStmt:6()))))) + +[case testLoopWithElse] +for x in []: + pass +else: + x +while 1: + pass +else: + x +[out] +MypyFile:1( + ForStmt:1( + NameExpr(x* [__main__.x]) + ListExpr:1() + Block:2( + PassStmt:2()) + Else( + ExpressionStmt:4( + NameExpr(x [__main__.x])))) + WhileStmt:5( + IntExpr(1) + Block:6( + PassStmt:6()) + Else( + ExpressionStmt:8( + NameExpr(x [__main__.x]))))) + +[case testBreak] +while 1: + break +for x in []: + break +[out] +MypyFile:1( + WhileStmt:1( + IntExpr(1) + Block:2( + BreakStmt:2())) + ForStmt:3( + NameExpr(x* [__main__.x]) + ListExpr:3() + Block:4( + BreakStmt:4()))) + +[case testContinue] +while 1: + continue +for x in []: + continue +[out] +MypyFile:1( + WhileStmt:1( + IntExpr(1) + Block:2( + ContinueStmt:2())) + ForStmt:3( + NameExpr(x* [__main__.x]) + ListExpr:3() + Block:4( + ContinueStmt:4()))) + +[case testIf] +x = 1 +if x: + x +elif x: + x +elif x: + x +else: + x +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + IfStmt:2( + If( + NameExpr(x [__main__.x])) + Then( + ExpressionStmt:3( + NameExpr(x [__main__.x]))) + Else( + IfStmt:4( + If( + NameExpr(x [__main__.x])) + Then( + ExpressionStmt:5( + NameExpr(x [__main__.x]))) + Else( + IfStmt:6( + If( + NameExpr(x [__main__.x])) + Then( + ExpressionStmt:7( + NameExpr(x [__main__.x]))) + Else( + ExpressionStmt:9( + NameExpr(x [__main__.x]))))))))) + +[case testSimpleIf] +if object: + object +[out] +MypyFile:1( + IfStmt:1( + If( + NameExpr(object [builtins.object])) + Then( + ExpressionStmt:2( + NameExpr(object [builtins.object]))))) + +[case testLvalues] +x = y = 1 +xx = 1 +x.m = 1 +x[y] = 1 +x2, y2 = 1 +[x3, y3] = 1 +(x4, y4) = 1 +[out] +MypyFile:1( + AssignmentStmt:1( + Lvalues( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + IntExpr(1)) + AssignmentStmt:2( + NameExpr(xx [__main__.xx]) + IntExpr(1) + builtins.int) + AssignmentStmt:3( + MemberExpr:3( + NameExpr(x [__main__.x]) + m) + IntExpr(1)) + AssignmentStmt:4( + IndexExpr:4( + NameExpr(x [__main__.x]) + NameExpr(y [__main__.y])) + IntExpr(1)) + AssignmentStmt:5( + TupleExpr:5( + NameExpr(x2* [__main__.x2]) + NameExpr(y2* [__main__.y2])) + IntExpr(1)) + AssignmentStmt:6( + TupleExpr:6( + NameExpr(x3* [__main__.x3]) + NameExpr(y3* [__main__.y3])) + IntExpr(1)) + AssignmentStmt:7( + TupleExpr:7( + NameExpr(x4* [__main__.x4]) + NameExpr(y4* [__main__.y4])) + IntExpr(1))) + +[case testStarLvalues] +# flags: --allow-redefinition +*x, y = 1 +*x, (y, *z) = 1 +*(x, q), r = 1 +[out] +MypyFile:1( + AssignmentStmt:2( + TupleExpr:2( + StarExpr:2( + NameExpr(x'* [__main__.x'])) + NameExpr(y'* [__main__.y'])) + IntExpr(1)) + AssignmentStmt:3( + TupleExpr:3( + StarExpr:3( + NameExpr(x''* [__main__.x''])) + TupleExpr:3( + NameExpr(y* [__main__.y]) + StarExpr:3( + NameExpr(z* [__main__.z])))) + IntExpr(1)) + AssignmentStmt:4( + TupleExpr:4( + StarExpr:4( + TupleExpr:4( + NameExpr(x* [__main__.x]) + NameExpr(q* [__main__.q]))) + NameExpr(r* [__main__.r])) + IntExpr(1))) + +[case testMultipleDefinition] +# flags: --allow-redefinition +x, y = 1 +x, y = 2 +[out] +MypyFile:1( + AssignmentStmt:2( + TupleExpr:2( + NameExpr(x'* [__main__.x']) + NameExpr(y'* [__main__.y'])) + IntExpr(1)) + AssignmentStmt:3( + TupleExpr:3( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + IntExpr(2))) + +[case testComplexDefinitions] +(x) = 1 +([y]) = 2 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + AssignmentStmt:2( + TupleExpr:2( + NameExpr(y* [__main__.y])) + IntExpr(2))) + +[case testLocalComplexDefinition] +def f(): + (x) = 1 + x +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + AssignmentStmt:2( + NameExpr(x* [l]) + IntExpr(1)) + ExpressionStmt:3( + NameExpr(x [l]))))) + +[case testMultipleDefOnlySomeNew] +x = 1 +y, x = 1 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + AssignmentStmt:2( + TupleExpr:2( + NameExpr(y* [__main__.y]) + NameExpr(x [__main__.x])) + IntExpr(1))) + +[case testMultipleDefOnlySomeNewNestedTuples] +x = 1 +y, (x, z) = 1 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + AssignmentStmt:2( + TupleExpr:2( + NameExpr(y* [__main__.y]) + TupleExpr:2( + NameExpr(x [__main__.x]) + NameExpr(z* [__main__.z]))) + IntExpr(1))) + +[case testMultipleDefOnlySomeNewNestedLists] +x = 1 +if x: + y, [x, z] = 1 + [p, [x, r]] = 1 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + IfStmt:2( + If( + NameExpr(x [__main__.x])) + Then( + AssignmentStmt:3( + TupleExpr:3( + NameExpr(y* [__main__.y]) + TupleExpr:3( + NameExpr(x [__main__.x]) + NameExpr(z* [__main__.z]))) + IntExpr(1)) + AssignmentStmt:4( + TupleExpr:4( + NameExpr(p* [__main__.p]) + TupleExpr:4( + NameExpr(x [__main__.x]) + NameExpr(r* [__main__.r]))) + IntExpr(1))))) + +[case testIndexedDel] +x = y = 1 +del x[y] +[out] +MypyFile:1( + AssignmentStmt:1( + Lvalues( + NameExpr(x* [__main__.x]) + NameExpr(y* [__main__.y])) + IntExpr(1)) + DelStmt:2( + IndexExpr:2( + NameExpr(x [__main__.x]) + NameExpr(y [__main__.y])))) + +[case testDelGlobalName] +x = 1 +del x +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x [__main__.x]) + IntExpr(1) + builtins.int) + DelStmt:2( + NameExpr(x [__main__.x]))) + +[case testDelLocalName] +def f(x): + del x +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x)) + Block:2( + DelStmt:2( + NameExpr(x [l]))))) + +[case testDelMultipleThings] +def f(x, y): + del x, y[0] +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x) + Var(y)) + Block:2( + DelStmt:2( + TupleExpr:2( + NameExpr(x [l]) + IndexExpr:2( + NameExpr(y [l]) + IntExpr(0))))))) + +[case testDelMultipleThingsInvalid] +def f(x, y) -> None: + del x, y + 1 +[out] +main:2: error: cannot delete operator +[out version>=3.10] +main:2: error: cannot delete expression + +[case testTry] +class c: pass +try: + c +except object: + c +except c as e: + e +except: + c +finally: + c +[out] +MypyFile:1( + ClassDef:1( + c + PassStmt:1()) + TryStmt:2( + Block:3( + ExpressionStmt:3( + NameExpr(c [__main__.c]))) + NameExpr(object [builtins.object]) + Block:5( + ExpressionStmt:5( + NameExpr(c [__main__.c]))) + NameExpr(c [__main__.c]) + NameExpr(e* [__main__.e]) + Block:7( + ExpressionStmt:7( + NameExpr(e [__main__.e]))) + Block:9( + ExpressionStmt:9( + NameExpr(c [__main__.c]))) + Finally( + ExpressionStmt:11( + NameExpr(c [__main__.c]))))) + +[case testTryElse] +try: + pass +except: + pass +else: + object +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + Block:4( + PassStmt:4()) + Else( + ExpressionStmt:6( + NameExpr(object [builtins.object]))))) + +[case testTryWithOnlyFinally] +try: + pass +finally: + pass +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + Finally( + PassStmt:4()))) + +[case testExceptWithMultipleTypes] +class c: pass +try: + pass +except (c, object) as e: + e +[out] +MypyFile:1( + ClassDef:1( + c + PassStmt:1()) + TryStmt:2( + Block:3( + PassStmt:3()) + TupleExpr:4( + NameExpr(c [__main__.c]) + NameExpr(object [builtins.object])) + NameExpr(e* [__main__.e]) + Block:5( + ExpressionStmt:5( + NameExpr(e [__main__.e]))))) + +[case testRaiseWithoutExpr] +raise +[out] +MypyFile:1( + RaiseStmt:1()) + +[case testWith] +with object: + object +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(object [builtins.object])) + Block:2( + ExpressionStmt:2( + NameExpr(object [builtins.object]))))) + +[case testWithAndVariable] +with object as x: + x +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(object [builtins.object])) + Target( + NameExpr(x* [__main__.x])) + Block:2( + ExpressionStmt:2( + NameExpr(x [__main__.x]))))) + +[case testWithInFunction] +def f(): + with f as x: + x +[out] +MypyFile:1( + FuncDef:1( + f + Block:2( + WithStmt:2( + Expr( + NameExpr(f [__main__.f])) + Target( + NameExpr(x* [l])) + Block:3( + ExpressionStmt:3( + NameExpr(x [l]))))))) + +[case testComplexWith] +with object, object: + pass +with object as a, object as b: + pass +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(object [builtins.object])) + Expr( + NameExpr(object [builtins.object])) + Block:2( + PassStmt:2())) + WithStmt:3( + Expr( + NameExpr(object [builtins.object])) + Target( + NameExpr(a* [__main__.a])) + Expr( + NameExpr(object [builtins.object])) + Target( + NameExpr(b* [__main__.b])) + Block:4( + PassStmt:4()))) + +[case testVariableInBlock] +while object: + x = None + if x: + x = x +[out] +MypyFile:1( + WhileStmt:1( + NameExpr(object [builtins.object]) + Block:2( + AssignmentStmt:2( + NameExpr(x* [__main__.x]) + NameExpr(None [builtins.None])) + IfStmt:3( + If( + NameExpr(x [__main__.x])) + Then( + AssignmentStmt:4( + NameExpr(x [__main__.x]) + NameExpr(x [__main__.x]))))))) + +[case testVariableInExceptHandler] +try: + pass +except object as o: + x = None + o = x +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + NameExpr(object [builtins.object]) + NameExpr(o* [__main__.o]) + Block:4( + AssignmentStmt:4( + NameExpr(x* [__main__.x]) + NameExpr(None [builtins.None])) + AssignmentStmt:5( + NameExpr(o [__main__.o]) + NameExpr(x [__main__.x]))))) + +[case testCallInExceptHandler] +try: + pass +except object as o: + o = object() +[out] +MypyFile:1( + TryStmt:1( + Block:2( + PassStmt:2()) + NameExpr(object [builtins.object]) + NameExpr(o* [__main__.o]) + Block:4( + AssignmentStmt:4( + NameExpr(o [__main__.o]) + CallExpr:4( + NameExpr(object [builtins.object]) + Args()))))) + +[case testTryExceptWithMultipleHandlers] +class Err(BaseException): pass +try: + pass +except BaseException as e: + pass +except Err as f: + f = BaseException() # Fail + f = Err() +[builtins fixtures/exception.pyi] +[out] +MypyFile:1( + ClassDef:1( + Err + BaseType( + builtins.BaseException) + PassStmt:1()) + TryStmt:2( + Block:3( + PassStmt:3()) + NameExpr(BaseException [builtins.BaseException]) + NameExpr(e* [__main__.e]) + Block:5( + PassStmt:5()) + NameExpr(Err [__main__.Err]) + NameExpr(f* [__main__.f]) + Block:7( + AssignmentStmt:7( + NameExpr(f [__main__.f]) + CallExpr:7( + NameExpr(BaseException [builtins.BaseException]) + Args())) + AssignmentStmt:8( + NameExpr(f [__main__.f]) + CallExpr:8( + NameExpr(Err [__main__.Err]) + Args()))))) +[case testMultipleAssignmentWithPartialNewDef] +# flags: --allow-redefinition +o = None +x, o = o, o +[out] +MypyFile:1( + AssignmentStmt:2( + NameExpr(o'* [__main__.o']) + NameExpr(None [builtins.None])) + AssignmentStmt:3( + TupleExpr:3( + NameExpr(x* [__main__.x]) + NameExpr(o* [__main__.o])) + TupleExpr:3( + NameExpr(o' [__main__.o']) + NameExpr(o' [__main__.o'])))) + +[case testFunctionDecorator] +def decorate(f): pass +@decorate +def g(): + g() +[out] +MypyFile:1( + FuncDef:1( + decorate + Args( + Var(f)) + Block:1( + PassStmt:1())) + Decorator:2( + Var(g) + NameExpr(decorate [__main__.decorate]) + FuncDef:3( + g + Block:4( + ExpressionStmt:4( + CallExpr:4( + NameExpr(g [__main__.g]) + Args())))))) + +[case testTryWithinFunction] +def f() -> None: + try: + pass + except object as o: + pass +[out] +MypyFile:1( + FuncDef:1( + f + def () + Block:2( + TryStmt:2( + Block:3( + PassStmt:3()) + NameExpr(object [builtins.object]) + NameExpr(o* [l]) + Block:5( + PassStmt:5()))))) + +[case testReuseExceptionVariable] +def f() -> None: + try: + pass + except object as o: + pass + except object as o: + pass +[out] +MypyFile:1( + FuncDef:1( + f + def () + Block:2( + TryStmt:2( + Block:3( + PassStmt:3()) + NameExpr(object [builtins.object]) + NameExpr(o* [l]) + Block:5( + PassStmt:5()) + NameExpr(object [builtins.object]) + NameExpr(o [l]) + Block:7( + PassStmt:7()))))) + +[case testWithMultiple] +def f(a): + pass +def main(): + with f(0) as a, f(a) as b: + x = a, b +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(a)) + Block:2( + PassStmt:2())) + FuncDef:3( + main + Block:4( + WithStmt:4( + Expr( + CallExpr:4( + NameExpr(f [__main__.f]) + Args( + IntExpr(0)))) + Target( + NameExpr(a* [l])) + Expr( + CallExpr:4( + NameExpr(f [__main__.f]) + Args( + NameExpr(a [l])))) + Target( + NameExpr(b* [l])) + Block:5( + AssignmentStmt:5( + NameExpr(x* [l]) + TupleExpr:5( + NameExpr(a [l]) + NameExpr(b [l])))))))) + +[case testRenameGlobalVariable] +# flags: --allow-redefinition +def f(a): pass +x = 0 +f(x) +x = '' +f(x) +[out] +MypyFile:1( + FuncDef:2( + f + Args( + Var(a)) + Block:2( + PassStmt:2())) + AssignmentStmt:3( + NameExpr(x' [__main__.x']) + IntExpr(0) + builtins.int) + ExpressionStmt:4( + CallExpr:4( + NameExpr(f [__main__.f]) + Args( + NameExpr(x' [__main__.x'])))) + AssignmentStmt:5( + NameExpr(x [__main__.x]) + StrExpr() + builtins.str) + ExpressionStmt:6( + CallExpr:6( + NameExpr(f [__main__.f]) + Args( + NameExpr(x [__main__.x]))))) + +[case testNoRenameGlobalVariable] +# flags: --disallow-redefinition +def f(a): pass +x = 0 +f(x) +x = '' +f(x) +[out] +MypyFile:1( + FuncDef:2( + f + Args( + Var(a)) + Block:2( + PassStmt:2())) + AssignmentStmt:3( + NameExpr(x [__main__.x]) + IntExpr(0) + builtins.int) + ExpressionStmt:4( + CallExpr:4( + NameExpr(f [__main__.f]) + Args( + NameExpr(x [__main__.x])))) + AssignmentStmt:5( + NameExpr(x [__main__.x]) + StrExpr()) + ExpressionStmt:6( + CallExpr:6( + NameExpr(f [__main__.f]) + Args( + NameExpr(x [__main__.x]))))) + +[case testRenameLocalVariable] +# flags: --allow-redefinition +def f(a): + f(a) + a = '' + f(a) +[out] +MypyFile:1( + FuncDef:2( + f + Args( + Var(a)) + Block:3( + ExpressionStmt:3( + CallExpr:3( + NameExpr(f [__main__.f]) + Args( + NameExpr(a [l])))) + AssignmentStmt:4( + NameExpr(a'* [l]) + StrExpr()) + ExpressionStmt:5( + CallExpr:5( + NameExpr(f [__main__.f]) + Args( + NameExpr(a' [l]))))))) + +[case testCannotRenameExternalVarWithinClass] +# flags: --allow-redefinition +x = 0 +x +class A: + x = 1 +x = '' +[out] +MypyFile:1( + AssignmentStmt:2( + NameExpr(x [__main__.x]) + IntExpr(0) + builtins.int) + ExpressionStmt:3( + NameExpr(x [__main__.x])) + ClassDef:4( + A + AssignmentStmt:5( + NameExpr(x [m]) + IntExpr(1) + builtins.int)) + AssignmentStmt:6( + NameExpr(x [__main__.x]) + StrExpr())) + +[case testSimpleWithRenaming] +with 0 as y: + z = y +with 1 as y: + y = 1 +[out] +MypyFile:1( + WithStmt:1( + Expr( + IntExpr(0)) + Target( + NameExpr(y'* [__main__.y'])) + Block:2( + AssignmentStmt:2( + NameExpr(z* [__main__.z]) + NameExpr(y' [__main__.y'])))) + WithStmt:3( + Expr( + IntExpr(1)) + Target( + NameExpr(y* [__main__.y])) + Block:4( + AssignmentStmt:4( + NameExpr(y [__main__.y]) + IntExpr(1))))) + +[case testSimpleWithRenamingFailure] +with 0 as y: + z = y +zz = y +with 1 as y: + y = 1 +[out] +MypyFile:1( + WithStmt:1( + Expr( + IntExpr(0)) + Target( + NameExpr(y* [__main__.y])) + Block:2( + AssignmentStmt:2( + NameExpr(z* [__main__.z]) + NameExpr(y [__main__.y])))) + AssignmentStmt:3( + NameExpr(zz* [__main__.zz]) + NameExpr(y [__main__.y])) + WithStmt:4( + Expr( + IntExpr(1)) + Target( + NameExpr(y [__main__.y])) + Block:5( + AssignmentStmt:5( + NameExpr(y [__main__.y]) + IntExpr(1))))) + +[case testConstantFold1] +from typing_extensions import Final +add: Final = 15 + 47 +add_mul: Final = (2 + 3) * 5 +sub: Final = 7 - 11 +bit_and: Final = 6 & 10 +bit_or: Final = 6 | 10 +bit_xor: Final = 6 ^ 10 +lshift: Final = 5 << 2 +rshift: Final = 13 >> 2 +lshift0: Final = 5 << 0 +rshift0: Final = 13 >> 0 +[out] +MypyFile:1( + ImportFrom:1(typing_extensions, [Final]) + AssignmentStmt:2( + NameExpr(add [__main__.add] = 62) + OpExpr:2( + + + IntExpr(15) + IntExpr(47)) + Literal[62]?) + AssignmentStmt:3( + NameExpr(add_mul [__main__.add_mul] = 25) + OpExpr:3( + * + OpExpr:3( + + + IntExpr(2) + IntExpr(3)) + IntExpr(5)) + Literal[25]?) + AssignmentStmt:4( + NameExpr(sub [__main__.sub] = -4) + OpExpr:4( + - + IntExpr(7) + IntExpr(11)) + Literal[-4]?) + AssignmentStmt:5( + NameExpr(bit_and [__main__.bit_and] = 2) + OpExpr:5( + & + IntExpr(6) + IntExpr(10)) + Literal[2]?) + AssignmentStmt:6( + NameExpr(bit_or [__main__.bit_or] = 14) + OpExpr:6( + | + IntExpr(6) + IntExpr(10)) + Literal[14]?) + AssignmentStmt:7( + NameExpr(bit_xor [__main__.bit_xor] = 12) + OpExpr:7( + ^ + IntExpr(6) + IntExpr(10)) + Literal[12]?) + AssignmentStmt:8( + NameExpr(lshift [__main__.lshift] = 20) + OpExpr:8( + << + IntExpr(5) + IntExpr(2)) + Literal[20]?) + AssignmentStmt:9( + NameExpr(rshift [__main__.rshift] = 3) + OpExpr:9( + >> + IntExpr(13) + IntExpr(2)) + Literal[3]?) + AssignmentStmt:10( + NameExpr(lshift0 [__main__.lshift0] = 5) + OpExpr:10( + << + IntExpr(5) + IntExpr(0)) + Literal[5]?) + AssignmentStmt:11( + NameExpr(rshift0 [__main__.rshift0] = 13) + OpExpr:11( + >> + IntExpr(13) + IntExpr(0)) + Literal[13]?)) + +[case testConstantFold2] +from typing_extensions import Final +neg1: Final = -5 +neg2: Final = --1 +neg3: Final = -0 +pos: Final = +5 +inverted1: Final = ~0 +inverted2: Final = ~5 +inverted3: Final = ~3 +p0: Final = 3**0 +p1: Final = 3**5 +p2: Final = (-5)**3 +p3: Final = 0**0 +s: Final = 'x' + 'y' +[out] +MypyFile:1( + ImportFrom:1(typing_extensions, [Final]) + AssignmentStmt:2( + NameExpr(neg1 [__main__.neg1] = -5) + UnaryExpr:2( + - + IntExpr(5)) + Literal[-5]?) + AssignmentStmt:3( + NameExpr(neg2 [__main__.neg2] = 1) + UnaryExpr:3( + - + UnaryExpr:3( + - + IntExpr(1))) + Literal[1]?) + AssignmentStmt:4( + NameExpr(neg3 [__main__.neg3] = 0) + UnaryExpr:4( + - + IntExpr(0)) + Literal[0]?) + AssignmentStmt:5( + NameExpr(pos [__main__.pos] = 5) + UnaryExpr:5( + + + IntExpr(5)) + Literal[5]?) + AssignmentStmt:6( + NameExpr(inverted1 [__main__.inverted1] = -1) + UnaryExpr:6( + ~ + IntExpr(0)) + Literal[-1]?) + AssignmentStmt:7( + NameExpr(inverted2 [__main__.inverted2] = -6) + UnaryExpr:7( + ~ + IntExpr(5)) + Literal[-6]?) + AssignmentStmt:8( + NameExpr(inverted3 [__main__.inverted3] = -4) + UnaryExpr:8( + ~ + IntExpr(3)) + Literal[-4]?) + AssignmentStmt:9( + NameExpr(p0 [__main__.p0] = 1) + OpExpr:9( + ** + IntExpr(3) + IntExpr(0)) + Literal[1]?) + AssignmentStmt:10( + NameExpr(p1 [__main__.p1] = 243) + OpExpr:10( + ** + IntExpr(3) + IntExpr(5)) + Literal[243]?) + AssignmentStmt:11( + NameExpr(p2 [__main__.p2] = -125) + OpExpr:11( + ** + UnaryExpr:11( + - + IntExpr(5)) + IntExpr(3)) + Literal[-125]?) + AssignmentStmt:12( + NameExpr(p3 [__main__.p3] = 1) + OpExpr:12( + ** + IntExpr(0) + IntExpr(0)) + Literal[1]?) + AssignmentStmt:13( + NameExpr(s [__main__.s] = xy) + OpExpr:13( + + + StrExpr(x) + StrExpr(y)) + Literal['xy']?)) diff --git a/test-data/unit/semanal-symtable.test b/test-data/unit/semanal-symtable.test new file mode 100644 index 000000000000..1622fd1f1ad4 --- /dev/null +++ b/test-data/unit/semanal-symtable.test @@ -0,0 +1,96 @@ +[case testEmptyFile] +[out] +-- Note that builtins are ignored to simplify output. +__main__: + SymbolTable() + +[case testVarDef] +x = 1 +[out] +__main__: + SymbolTable( + x : Gdef/Var (__main__.x) : builtins.int) + +[case testFuncDef] +def f(): pass +[out] +__main__: + SymbolTable( + f : Gdef/FuncDef (__main__.f)) + +[case testEmptyClassDef] +class c: pass +[out] +__main__: + SymbolTable( + c : Gdef/TypeInfo (__main__.c)) + +[case testImport] +import m +[file m.py] +x = 1 +[out] +__main__: + SymbolTable( + m : Gdef/MypyFile (m)) +m: + SymbolTable( + x : Gdef/Var (m.x) : builtins.int) + +[case testImportFromModule] +from m import x +[file m.py] +class x: pass +y = 1 +[out] +__main__: + SymbolTable( + x : Gdef/TypeInfo (m.x)) +m: + SymbolTable( + x : Gdef/TypeInfo (m.x) + y : Gdef/Var (m.y) : builtins.int) + +[case testImportAs] +from m import x as xx +[file m.py] +class x: pass +y = 1 +[out] +__main__: + SymbolTable( + xx : Gdef/TypeInfo (m.x)) +m: + SymbolTable( + x : Gdef/TypeInfo (m.x) + y : Gdef/Var (m.y) : builtins.int) + +[case testFailingImports] +from sys import non_existing1 # type: ignore +from xyz import non_existing2 # type: ignore +if int(): + from sys import non_existing3 # type: ignore +import non_existing4 # type: ignore +[out] +__main__: + SymbolTable( + non_existing1 : Gdef/Var (__main__.non_existing1) : Any + non_existing2 : Gdef/Var (__main__.non_existing2) : Any + non_existing3 : Gdef/Var (__main__.non_existing3) : Any + non_existing4 : Gdef/Var (__main__.non_existing4) : Any) + +[case testDecorator] +from typing import Callable + +def dec(f: Callable[[], None]) -> Callable[[], None]: + return f + +@dec +def g() -> None: + pass +[out] +__main__: + SymbolTable( + Callable : Gdef/Var (typing.Callable) : builtins.int + dec : Gdef/FuncDef (__main__.dec) : def (f: def ()) -> def () + g : Gdef/Decorator (__main__.g) : def ()) diff --git a/test-data/unit/semanal-typealiases.test b/test-data/unit/semanal-typealiases.test new file mode 100644 index 000000000000..88d234134350 --- /dev/null +++ b/test-data/unit/semanal-typealiases.test @@ -0,0 +1,446 @@ +[case testListTypeAlias] +from typing import List +def f() -> List[int]: pass +[builtins fixtures/list.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [List]) + FuncDef:2( + f + def () -> builtins.list[builtins.int] + Block:2( + PassStmt:2()))) + +[case testDictTypeAlias] +from typing import Dict +def f() -> Dict[int, str]: pass +[builtins fixtures/dict.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [Dict]) + FuncDef:2( + f + def () -> builtins.dict[builtins.int, builtins.str] + Block:2( + PassStmt:2()))) + +[case testQualifiedTypeAlias] +import typing +def f() -> typing.List[int]: pass +[builtins fixtures/list.pyi] +[out] +MypyFile:1( + Import:1(typing) + FuncDef:2( + f + def () -> builtins.list[builtins.int] + Block:2( + PassStmt:2()))) + +[case testTypeApplicationWithTypeAlias] +from typing import List +List[List[int]] +[builtins fixtures/list.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [List]) + ExpressionStmt:2( + TypeApplication:2( + NameExpr(List [typing.List]) + Types( + builtins.list[builtins.int])))) + +[case testTypeApplicationWithQualifiedTypeAlias] +import typing +typing.List[typing.List[int]] +[builtins fixtures/list.pyi] +[out] +MypyFile:1( + Import:1(typing) + ExpressionStmt:2( + TypeApplication:2( + MemberExpr:2( + NameExpr(typing) + List [typing.List]) + Types( + builtins.list[builtins.int])))) + +[case testSimpleTypeAlias] +import typing +class A: pass +A2 = A +def f(x: A2) -> A: pass +[out] +MypyFile:1( + Import:1(typing) + ClassDef:2( + A + PassStmt:2()) + AssignmentStmt:3( + NameExpr(A2* [__main__.A2]) + NameExpr(A [__main__.A])) + FuncDef:4( + f + Args( + Var(x)) + def (x: __main__.A) -> __main__.A + Block:4( + PassStmt:4()))) + +[case testQualifiedSimpleTypeAlias] +import typing +import _m +A2 = _m.A +x = 1 # type: A2 +[fixture _m.py] +import typing +class A: pass +[out] +MypyFile:1( + Import:1(typing) + Import:2(_m) + AssignmentStmt:3( + NameExpr(A2* [__main__.A2]) + MemberExpr:3( + NameExpr(_m) + A [_m.A])) + AssignmentStmt:4( + NameExpr(x [__main__.x]) + IntExpr(1) + _m.A)) + +[case testUnionTypeAlias] +from typing import Union +U = Union[int, str] +def f(x: U) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + AssignmentStmt:2( + NameExpr(U* [__main__.U]) + TypeAliasExpr(Union[builtins.int, builtins.str])) + FuncDef:3( + f + Args( + Var(x)) + def (x: Union[builtins.int, builtins.str]) + Block:3( + PassStmt:3()))) + +[case testUnionTypeAlias2] +from typing import Union +class A: pass +U = Union[int, A] +def f(x: U) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + ClassDef:2( + A + PassStmt:2()) + AssignmentStmt:3( + NameExpr(U* [__main__.U]) + TypeAliasExpr(Union[builtins.int, __main__.A])) + FuncDef:4( + f + Args( + Var(x)) + def (x: Union[builtins.int, __main__.A]) + Block:4( + PassStmt:4()))) + +[case testUnionTypeAliasWithQualifiedUnion] +import typing +U = typing.Union[int, str] +def f(x: U) -> None: pass +[out] +MypyFile:1( + Import:1(typing) + AssignmentStmt:2( + NameExpr(U* [__main__.U]) + TypeAliasExpr(Union[builtins.int, builtins.str])) + FuncDef:3( + f + Args( + Var(x)) + def (x: Union[builtins.int, builtins.str]) + Block:3( + PassStmt:3()))) + +[case testTupleTypeAlias] +from typing import Tuple +T = Tuple[int, str] +def f(x: T) -> None: pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [Tuple]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeAliasExpr(Tuple[builtins.int, builtins.str])) + FuncDef:3( + f + Args( + Var(x)) + def (x: Tuple[builtins.int, builtins.str]) + Block:3( + PassStmt:3()))) + +[case testCallableTypeAlias] +from typing import Callable +C = Callable[[int], None] +def f(x: C) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Callable]) + AssignmentStmt:2( + NameExpr(C* [__main__.C]) + TypeAliasExpr(def (builtins.int))) + FuncDef:3( + f + Args( + Var(x)) + def (x: def (builtins.int)) + Block:3( + PassStmt:3()))) + +[case testGenericTypeAlias] +from typing import Generic, TypeVar +T = TypeVar('T') +class G(Generic[T]): pass +A = G[int] +def f(x: A) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Generic, TypeVar]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2()) + ClassDef:3( + G + TypeVars( + T`1) + PassStmt:3()) + AssignmentStmt:4( + NameExpr(A* [__main__.A]) + TypeAliasExpr(__main__.G[builtins.int])) + FuncDef:5( + f + Args( + Var(x)) + def (x: __main__.G[builtins.int]) + Block:5( + PassStmt:5()))) + +[case testGenericTypeAlias2] +from typing import List +A = List[int] +def f(x: A) -> None: pass +[builtins fixtures/list.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [List]) + AssignmentStmt:2( + NameExpr(A* [__main__.A]) + TypeAliasExpr(builtins.list[builtins.int])) + FuncDef:3( + f + Args( + Var(x)) + def (x: builtins.list[builtins.int]) + Block:3( + PassStmt:3()))) + +[case testImportUnionTypeAlias] +import typing +from _m import U +def f(x: U) -> None: pass +[fixture _m.py] +from typing import Union +class A: pass +U = Union[int, A] +[out] +MypyFile:1( + Import:1(typing) + ImportFrom:2(_m, [U]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Union[builtins.int, _m.A]) + Block:3( + PassStmt:3()))) + +[case testImportUnionTypeAlias2] +import typing +import _m +def f(x: _m.U) -> None: pass +[fixture _m.py] +from typing import Union +class A: pass +U = Union[int, A] +[out] +MypyFile:1( + Import:1(typing) + Import:2(_m) + FuncDef:3( + f + Args( + Var(x)) + def (x: Union[builtins.int, _m.A]) + Block:3( + PassStmt:3()))) + +[case testImportSimpleTypeAlias] +import typing +from _m import A +def f(x: A) -> None: pass +[fixture _m.py] +import typing +A = int +[out] +MypyFile:1( + Import:1(typing) + ImportFrom:2(_m, [A]) + FuncDef:3( + f + Args( + Var(x)) + def (x: builtins.int) + Block:3( + PassStmt:3()))) + +[case testImportSimpleTypeAlias2] +import typing +import _m +def f(x: _m.A) -> None: pass +[fixture _m.py] +import typing +A = int +[out] +MypyFile:1( + Import:1(typing) + Import:2(_m) + FuncDef:3( + f + Args( + Var(x)) + def (x: builtins.int) + Block:3( + PassStmt:3()))) + +[case testAnyTypeAlias] +from typing import Any +A = Any +a = 1 # type: A +[out] +MypyFile:1( + ImportFrom:1(typing, [Any]) + AssignmentStmt:2( + NameExpr(A* [__main__.A]) + NameExpr(Any [typing.Any])) + AssignmentStmt:3( + NameExpr(a [__main__.a]) + IntExpr(1) + Any)) + +[case testAnyTypeAlias2] +import typing +A = typing.Any +a = 1 # type: A +[out] +MypyFile:1( + Import:1(typing) + AssignmentStmt:2( + NameExpr(A* [__main__.A]) + MemberExpr:2( + NameExpr(typing) + Any [typing.Any])) + AssignmentStmt:3( + NameExpr(a [__main__.a]) + IntExpr(1) + Any)) + +[case testTypeAliasAlias] +from typing import Union +U = Union[int, str] +U2 = U +x = 1 # type: U2 +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + AssignmentStmt:2( + NameExpr(U* [__main__.U]) + TypeAliasExpr(Union[builtins.int, builtins.str])) + AssignmentStmt:3( + NameExpr(U2* [__main__.U2]) + NameExpr(U [__main__.U])) + AssignmentStmt:4( + NameExpr(x [__main__.x]) + IntExpr(1) + Union[builtins.int, builtins.str])) + +[case testTypeAliasOfImportedAlias] +from typing import Union +from _m import U +U2 = U +x = 1 # type: U2 +[fixture _m.py] +from typing import Union +U = Union[int, str] +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + ImportFrom:2(_m, [U]) + AssignmentStmt:3( + NameExpr(U2* [__main__.U2]) + NameExpr(U [_m.U])) + AssignmentStmt:4( + NameExpr(x [__main__.x]) + IntExpr(1) + Union[builtins.int, builtins.str])) + +[case testListTypeDoesNotGenerateAlias] + +import typing +A = [int, str] +a = 1 # type: A # E: Variable "__main__.A" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testCantUseStringLiteralAsTypeAlias] + +from typing import Union +A = 'Union[int, str]' +a = 1 # type: A # E: Variable "__main__.A" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +[case testStringLiteralTypeAsAliasComponent] +from typing import Union +A = Union['int', str] +a = 1 # type: A +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + AssignmentStmt:2( + NameExpr(A* [__main__.A]) + TypeAliasExpr(Union[builtins.int, builtins.str])) + AssignmentStmt:3( + NameExpr(a [__main__.a]) + IntExpr(1) + Union[builtins.int, builtins.str])) + +[case testComplexTypeAlias] +from typing import Union, Tuple, Any +A = Union['int', Tuple[int, Any]] +a = 1 # type: A +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [Union, Tuple, Any]) + AssignmentStmt:2( + NameExpr(A* [__main__.A]) + TypeAliasExpr(Union[builtins.int, Tuple[builtins.int, Any]])) + AssignmentStmt:3( + NameExpr(a [__main__.a]) + IntExpr(1) + Union[builtins.int, Tuple[builtins.int, Any]])) diff --git a/test-data/unit/semanal-typeddict.test b/test-data/unit/semanal-typeddict.test new file mode 100644 index 000000000000..9ce89155c308 --- /dev/null +++ b/test-data/unit/semanal-typeddict.test @@ -0,0 +1,45 @@ +-- Create Type + +-- TODO: Implement support for this syntax. +--[case testCanCreateTypedDictTypeWithDictCall] +--from mypy_extensions import TypedDict +--Point = TypedDict('Point', dict(x=int, y=int)) +--[builtins fixtures/dict.pyi] +--[out] +--MypyFile:1( +-- ImportFrom:1(mypy_extensions, [TypedDict]) +-- AssignmentStmt:2( +-- NameExpr(Point* [__main__.Point]) +-- TypedDictExpr:2(Point))) + +[case testCanCreateTypedDictTypeWithDictLiteral] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}) +[builtins fixtures/dict.pyi] +[out] +MypyFile:1( + ImportFrom:1(mypy_extensions, [TypedDict]) + AssignmentStmt:2( + NameExpr(Point* [__main__.Point]) + TypedDictExpr:2(Point))) + +[case testTypedDictWithDocString] +from mypy_extensions import TypedDict +class A(TypedDict): + """foo""" + x: str +[builtins fixtures/dict.pyi] +[out] +MypyFile:1( + ImportFrom:1(mypy_extensions, [TypedDict]) + ClassDef:2( + A + BaseType( + mypy_extensions._TypedDict) + ExpressionStmt:3( + StrExpr(foo)) + AssignmentStmt:4( + NameExpr(x) + TempNode:4( + Any) + builtins.str))) diff --git a/test-data/unit/semanal-typeinfo.test b/test-data/unit/semanal-typeinfo.test new file mode 100644 index 000000000000..f4b92457df09 --- /dev/null +++ b/test-data/unit/semanal-typeinfo.test @@ -0,0 +1,91 @@ +[case testEmptyFile] +[out] +TypeInfoMap() + +[case testEmptyClass] +class c: pass +[out] +TypeInfoMap( + __main__.c : TypeInfo( + Name(__main__.c) + Bases(builtins.object) + Mro(__main__.c, builtins.object) + Names())) + +[case testClassWithMethod] +class c: + def f(self): pass +[out] +TypeInfoMap( + __main__.c : TypeInfo( + Name(__main__.c) + Bases(builtins.object) + Mro(__main__.c, builtins.object) + Names( + f))) + +[case testClassWithAttributes] +class c: + def __init__(self, x): + self.y = x + self.z = 1 +[out] +TypeInfoMap( + __main__.c : TypeInfo( + Name(__main__.c) + Bases(builtins.object) + Mro(__main__.c, builtins.object) + Names( + __init__ + y + z))) + +[case testBaseClass] +class base: pass +class c(base): pass +[out] +TypeInfoMap( + __main__.base : TypeInfo( + Name(__main__.base) + Bases(builtins.object) + Mro(__main__.base, builtins.object) + Names()) + __main__.c : TypeInfo( + Name(__main__.c) + Bases(__main__.base) + Mro(__main__.c, __main__.base, builtins.object) + Names())) + +[case testClassAndAbstractClass] +from abc import abstractmethod, ABCMeta +import typing + +class i(metaclass=ABCMeta): pass +class c(i): pass +[out] +TypeInfoMap( + __main__.c : TypeInfo( + Name(__main__.c) + Bases(__main__.i) + Mro(__main__.c, __main__.i, builtins.object) + Names() + MetaclassType(abc.ABCMeta)) + __main__.i : TypeInfo( + Name(__main__.i) + Bases(builtins.object) + Mro(__main__.i, builtins.object) + Names() + DeclaredMetaclass(abc.ABCMeta) + MetaclassType(abc.ABCMeta))) + +[case testAttributeWithoutType] +class A: + a = A +[out] +TypeInfoMap( + __main__.A : TypeInfo( + Name(__main__.A) + Bases(builtins.object) + Mro(__main__.A, builtins.object) + Names( + a))) diff --git a/test-data/unit/semanal-types.test b/test-data/unit/semanal-types.test new file mode 100644 index 000000000000..83c44738f055 --- /dev/null +++ b/test-data/unit/semanal-types.test @@ -0,0 +1,1589 @@ +[case testVarWithType] +import typing +class A: pass +x = A() # type: A +y = x +[out] +MypyFile:1( + Import:1(typing) + ClassDef:2( + A + PassStmt:2()) + AssignmentStmt:3( + NameExpr(x [__main__.x]) + CallExpr:3( + NameExpr(A [__main__.A]) + Args()) + __main__.A) + AssignmentStmt:4( + NameExpr(y* [__main__.y]) + NameExpr(x [__main__.x]))) + +[case testLocalVarWithType] +class A: pass +def f(): + x = None # type: A + y = x +[out] +MypyFile:1( + ClassDef:1( + A + PassStmt:1()) + FuncDef:2( + f + Block:3( + AssignmentStmt:3( + NameExpr(x [l]) + NameExpr(None [builtins.None]) + __main__.A) + AssignmentStmt:4( + NameExpr(y* [l]) + NameExpr(x [l]))))) + +[case testAnyType] +from typing import Any +x = None # type: Any +y = x +[out] +MypyFile:1( + ImportFrom:1(typing, [Any]) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + Any) + AssignmentStmt:3( + NameExpr(y* [__main__.y]) + NameExpr(x [__main__.x]))) + +[case testMemberVarWithType] +import typing +class A: + def __init__(self): + self.x = None # type: int +[out] +MypyFile:1( + Import:1(typing) + ClassDef:2( + A + FuncDef:3( + __init__ + Args( + Var(self)) + Block:4( + AssignmentStmt:4( + MemberExpr:4( + NameExpr(self [l]) + x) + NameExpr(None [builtins.None]) + builtins.int))))) + +[case testClassVarWithType] +import typing +class A: + x = None # type: int + x = 1 +[out] +MypyFile:1( + Import:1(typing) + ClassDef:2( + A + AssignmentStmt:3( + NameExpr(x [m]) + NameExpr(None [builtins.None]) + builtins.int) + AssignmentStmt:4( + NameExpr(x [__main__.A.x]) + IntExpr(1)))) + +[case testFunctionSig] +from typing import Any +class A: pass +def f(x: A) -> A: pass +def g(x: Any, y: A) -> None: + z = x, y +[out] +MypyFile:1( + ImportFrom:1(typing, [Any]) + ClassDef:2( + A + PassStmt:2()) + FuncDef:3( + f + Args( + Var(x)) + def (x: __main__.A) -> __main__.A + Block:3( + PassStmt:3())) + FuncDef:4( + g + Args( + Var(x) + Var(y)) + def (x: Any, y: __main__.A) + Block:5( + AssignmentStmt:5( + NameExpr(z* [l]) + TupleExpr:5( + NameExpr(x [l]) + NameExpr(y [l])))))) + +[case testBaseclass] +class A: pass +class B(A): pass +[out] +MypyFile:1( + ClassDef:1( + A + PassStmt:1()) + ClassDef:2( + B + BaseType( + __main__.A) + PassStmt:2())) + +[case testMultipleVarDef] + +class A: pass +class B: pass +a, b = None, None # type: (A, B) +x = a, b +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ClassDef:2( + A + PassStmt:2()) + ClassDef:3( + B + PassStmt:3()) + AssignmentStmt:4( + TupleExpr:4( + NameExpr(a [__main__.a]) + NameExpr(b [__main__.b])) + TupleExpr:4( + NameExpr(None [builtins.None]) + NameExpr(None [builtins.None])) + Tuple[__main__.A, __main__.B]) + AssignmentStmt:5( + NameExpr(x* [__main__.x]) + TupleExpr:5( + NameExpr(a [__main__.a]) + NameExpr(b [__main__.b])))) + +[case testGenericType] +from typing import TypeVar, Generic, Any + +t = TypeVar('t') + +class A(Generic[t]): pass +class B: pass +x = None # type: A[B] +y = None # type: A[Any] +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic, Any]) + AssignmentStmt:3( + NameExpr(t* [__main__.t]) + TypeVarExpr:3()) + ClassDef:5( + A + TypeVars( + t`1) + PassStmt:5()) + ClassDef:6( + B + PassStmt:6()) + AssignmentStmt:7( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + __main__.A[__main__.B]) + AssignmentStmt:8( + NameExpr(y [__main__.y]) + NameExpr(None [builtins.None]) + __main__.A[Any])) + +[case testGenericType2] +from typing import TypeVar, Generic, Any +t = TypeVar('t') +s = TypeVar('s') +class A(Generic[t, s]): pass +class B: pass +x = None # type: A[B, Any] +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic, Any]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + AssignmentStmt:3( + NameExpr(s* [__main__.s]) + TypeVarExpr:3()) + ClassDef:4( + A + TypeVars( + t`1 + s`2) + PassStmt:4()) + ClassDef:5( + B + PassStmt:5()) + AssignmentStmt:6( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + __main__.A[__main__.B, Any])) + +[case testAssignmentAfterDef] + + +class A: pass +a = None # type: A +a = 1 +def f(): + b = None # type: A + b = 1 +[out] +MypyFile:1( + ClassDef:3( + A + PassStmt:3()) + AssignmentStmt:4( + NameExpr(a [__main__.a]) + NameExpr(None [builtins.None]) + __main__.A) + AssignmentStmt:5( + NameExpr(a [__main__.a]) + IntExpr(1)) + FuncDef:6( + f + Block:7( + AssignmentStmt:7( + NameExpr(b [l]) + NameExpr(None [builtins.None]) + __main__.A) + AssignmentStmt:8( + NameExpr(b [l]) + IntExpr(1))))) + +[case testCast] +from typing import TypeVar, Generic, Any, cast +t = TypeVar('t') +class c: pass +class d(Generic[t]): pass +cast(Any, 1) +cast(c, 1) +cast(d[c], c) +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic, Any, cast]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + c + PassStmt:3()) + ClassDef:4( + d + TypeVars( + t`1) + PassStmt:4()) + ExpressionStmt:5( + CastExpr:5( + IntExpr(1) + Any)) + ExpressionStmt:6( + CastExpr:6( + IntExpr(1) + __main__.c)) + ExpressionStmt:7( + CastExpr:7( + NameExpr(c [__main__.c]) + __main__.d[__main__.c]))) + +[case testCastToQualifiedTypeAndCast] +import typing +import _m +typing.cast(_m.C, object) +[fixture _m.py] +class C: pass +[out] +MypyFile:1( + Import:1(typing) + Import:2(_m) + ExpressionStmt:3( + CastExpr:3( + NameExpr(object [builtins.object]) + _m.C))) + +[case testLongQualifiedCast] +import typing +import _m._n +typing.cast(_m._n.C, object) +[fixture _m/__init__.py] +[fixture _m/_n.py] +class C: pass +[out] +MypyFile:1( + Import:1(typing) + Import:2(_m._n) + ExpressionStmt:3( + CastExpr:3( + NameExpr(object [builtins.object]) + _m._n.C))) + +[case testCastTargetWithTwoTypeArgs] +from typing import TypeVar, Generic, cast +t = TypeVar('t') +s = TypeVar('s') +class C(Generic[t, s]): pass +cast(C[str, int], C) +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic, cast]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + AssignmentStmt:3( + NameExpr(s* [__main__.s]) + TypeVarExpr:3()) + ClassDef:4( + C + TypeVars( + t`1 + s`2) + PassStmt:4()) + ExpressionStmt:5( + CastExpr:5( + NameExpr(C [__main__.C]) + __main__.C[builtins.str, builtins.int]))) + +[case testCastToTupleType] +from typing import Tuple, cast +cast(Tuple[int, str], None) +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [Tuple, cast]) + ExpressionStmt:2( + CastExpr:2( + NameExpr(None [builtins.None]) + Tuple[builtins.int, builtins.str]))) + +[case testCastToFunctionType] +from typing import Callable, cast +cast(Callable[[int], str], None) +[out] +MypyFile:1( + ImportFrom:1(typing, [Callable, cast]) + ExpressionStmt:2( + CastExpr:2( + NameExpr(None [builtins.None]) + def (builtins.int) -> builtins.str))) + +[case testCastToStringLiteralType] +from typing import cast +cast('int', 1) +[out] +MypyFile:1( + ImportFrom:1(typing, [cast]) + ExpressionStmt:2( + CastExpr:2( + IntExpr(1) + builtins.int))) + +[case testAssertType] +from typing import assert_type +assert_type(1, int) +[out] +MypyFile:1( + ImportFrom:1(typing, [assert_type]) + ExpressionStmt:2( + AssertTypeExpr:2( + IntExpr(1) + builtins.int))) + +[case testFunctionTypeVariable] +from typing import TypeVar +t = TypeVar('t') +def f(x: t) -> None: + y = None # type: t +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + FuncDef:3( + f + Args( + Var(x)) + def [t] (x: t`-1) + Block:4( + AssignmentStmt:4( + NameExpr(y [l]) + NameExpr(None [builtins.None]) + t`-1)))) + +[case testTwoFunctionTypeVariables] +from typing import TypeVar +t = TypeVar('t') +u = TypeVar('u') +def f(x: t, y: u, z: t) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + AssignmentStmt:3( + NameExpr(u* [__main__.u]) + TypeVarExpr:3()) + FuncDef:4( + f + Args( + Var(x) + Var(y) + Var(z)) + def [t, u] (x: t`-1, y: u`-2, z: t`-1) + Block:4( + PassStmt:4()))) + +[case testNestedGenericFunctionTypeVariable] +from typing import TypeVar, Generic +t = TypeVar('t') +class A(Generic[t]): pass +def f(x: A[t], y) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + A + TypeVars( + t`1) + PassStmt:3()) + FuncDef:4( + f + Args( + Var(x) + Var(y)) + def [t] (x: __main__.A[t`-1], y: Any) + Block:4( + PassStmt:4()))) + +[case testNestedGenericFunctionTypeVariable2] +from typing import TypeVar, Tuple, Generic +t = TypeVar('t') +class A(Generic[t]): pass +def f(x: Tuple[int, t]) -> None: pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Tuple, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + A + TypeVars( + t`1) + PassStmt:3()) + FuncDef:4( + f + Args( + Var(x)) + def [t] (x: Tuple[builtins.int, t`-1]) + Block:4( + PassStmt:4()))) + +[case testNestedGenericFunctionTypeVariable3] +from typing import TypeVar, Callable, Generic +t = TypeVar('t') +class A(Generic[t]): pass +def f(x: Callable[[int, t], int]) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Callable, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + A + TypeVars( + t`1) + PassStmt:3()) + FuncDef:4( + f + Args( + Var(x)) + def [t] (x: def (builtins.int, t`-1) -> builtins.int) + Block:4( + PassStmt:4()))) + +[case testNestedGenericFunctionTypeVariable4] +from typing import TypeVar, Callable, Generic +t = TypeVar('t') +class A(Generic[t]): pass +def f(x: Callable[[], t]) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Callable, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + A + TypeVars( + t`1) + PassStmt:3()) + FuncDef:4( + f + Args( + Var(x)) + def [t] (x: def () -> t`-1) + Block:4( + PassStmt:4()))) + +[case testGenericFunctionTypeVariableInReturnType] +from typing import TypeVar +t = TypeVar('t') +def f() -> t: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + FuncDef:3( + f + def [t] () -> t`-1 + Block:3( + PassStmt:3()))) + +[case testSelfType] +class A: + def f(self, o: object) -> None: pass +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + f + Args( + Var(self) + Var(o)) + def (self: __main__.A, o: builtins.object) + Block:2( + PassStmt:2())))) + +[case testNestedGenericFunction] +from typing import TypeVar +t = TypeVar('t') +def f() -> None: + def g() -> t: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + FuncDef:3( + f + def () + Block:4( + FuncDef:4( + g + def [t] () -> t`-1 + Block:4( + PassStmt:4()))))) + +[case testClassTvar] +from typing import TypeVar, Generic + +t = TypeVar('t') + +class c(Generic[t]): + def f(self) -> t: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:3( + NameExpr(t* [__main__.t]) + TypeVarExpr:3()) + ClassDef:5( + c + TypeVars( + t`1) + FuncDef:6( + f + Args( + Var(self)) + def (self: __main__.c[t`1]) -> t`1 + Block:6( + PassStmt:6())))) + +[case testClassTvar2] +from typing import TypeVar, Generic + +t = TypeVar('t') +s = TypeVar('s') + +class c(Generic[t, s]): + def f(self, x: s) -> t: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:3( + NameExpr(t* [__main__.t]) + TypeVarExpr:3()) + AssignmentStmt:4( + NameExpr(s* [__main__.s]) + TypeVarExpr:4()) + ClassDef:6( + c + TypeVars( + t`1 + s`2) + FuncDef:7( + f + Args( + Var(self) + Var(x)) + def (self: __main__.c[t`1, s`2], x: s`2) -> t`1 + Block:7( + PassStmt:7())))) + +[case testGenericBaseClass] +from typing import TypeVar, Generic +t = TypeVar('t') +class d(Generic[t]): pass +class c(d[t], Generic[t]): pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + d + TypeVars( + t`1) + PassStmt:3()) + ClassDef:4( + c + TypeVars( + t`1) + BaseType( + __main__.d[t`1]) + PassStmt:4())) + +[case testTupleType] +from typing import Tuple +t = None # type: tuple +t1 = None # type: Tuple[object] +t2 = None # type: Tuple[int, object] +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [Tuple]) + AssignmentStmt:2( + NameExpr(t [__main__.t]) + NameExpr(None [builtins.None]) + builtins.tuple[Any, ...]) + AssignmentStmt:3( + NameExpr(t1 [__main__.t1]) + NameExpr(None [builtins.None]) + Tuple[builtins.object]) + AssignmentStmt:4( + NameExpr(t2 [__main__.t2]) + NameExpr(None [builtins.None]) + Tuple[builtins.int, builtins.object])) + +[case testVariableLengthTuple] +from typing import Tuple +t = None # type: Tuple[int, ...] +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [Tuple]) + AssignmentStmt:2( + NameExpr(t [__main__.t]) + NameExpr(None [builtins.None]) + builtins.tuple[builtins.int, ...])) + +[case testInvalidTupleType] +from typing import Tuple +t = None # type: Tuple[int, str, ...] # E: Unexpected "..." +[builtins fixtures/tuple.pyi] +[out] + +[case testFunctionTypes] +from typing import Callable +f = None # type: Callable[[object, int], str] +g = None # type: Callable[[], None] +[out] +MypyFile:1( + ImportFrom:1(typing, [Callable]) + AssignmentStmt:2( + NameExpr(f [__main__.f]) + NameExpr(None [builtins.None]) + def (builtins.object, builtins.int) -> builtins.str) + AssignmentStmt:3( + NameExpr(g [__main__.g]) + NameExpr(None [builtins.None]) + def ())) + +[case testOverloadedFunction] +from typing import overload, Any +@overload +def f(a: object) -> int: a +@overload +def f(a: str) -> object: a + +def f(a: Any) -> Any: return a + +[out] +MypyFile:1( + ImportFrom:1(typing, [overload, Any]) + OverloadedFuncDef:2( + FuncDef:7( + f + Args( + Var(a)) + def (a: Any) -> Any + Block:7( + ReturnStmt:7( + NameExpr(a [l])))) + Overload(def (a: builtins.object) -> builtins.int, \ + def (a: builtins.str) -> builtins.object) + Decorator:2( + Var(f) + NameExpr(overload [typing.overload]) + FuncDef:3( + f + Args( + Var(a)) + def (a: builtins.object) -> builtins.int + Block:3( + ExpressionStmt:3( + NameExpr(a [l]))))) + Decorator:4( + Var(f) + NameExpr(overload [typing.overload]) + FuncDef:5( + f + Args( + Var(a)) + def (a: builtins.str) -> builtins.object + Block:5( + ExpressionStmt:5( + NameExpr(a [l]))))))) + +[case testReferenceToOverloadedFunction] +from typing import overload +@overload +def f() -> None: pass +@overload +def f(x: int) -> None: pass + +def f(*args) -> None: pass + +x = f +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [overload]) + OverloadedFuncDef:2( + FuncDef:7( + f + def (*args: Any) + VarArg( + Var(args)) + Block:7( + PassStmt:7())) + Overload(def (), def (x: builtins.int)) + Decorator:2( + Var(f) + NameExpr(overload [typing.overload]) + FuncDef:3( + f + def () + Block:3( + PassStmt:3()))) + Decorator:4( + Var(f) + NameExpr(overload [typing.overload]) + FuncDef:5( + f + Args( + Var(x)) + def (x: builtins.int) + Block:5( + PassStmt:5())))) + AssignmentStmt:9( + NameExpr(x* [__main__.x]) + NameExpr(f [__main__.f]))) + +[case testNestedOverloadedFunction] +from typing import overload +def f(): + @overload + def g(): pass + @overload + def g(x): pass + + def g(*args): pass + + y = g +[out] +MypyFile:1( + ImportFrom:1(typing, [overload]) + FuncDef:2( + f + Block:3( + OverloadedFuncDef:3( + FuncDef:8( + g + VarArg( + Var(args)) + Block:8( + PassStmt:8())) + Overload(def () -> Any, def (x: Any) -> Any) + Decorator:3( + Var(g) + NameExpr(overload [typing.overload]) + FuncDef:4( + g + Block:4( + PassStmt:4()))) + Decorator:5( + Var(g) + NameExpr(overload [typing.overload]) + FuncDef:6( + g + Args( + Var(x)) + Block:6( + PassStmt:6())))) + AssignmentStmt:10( + NameExpr(y* [l]) + NameExpr(g [l]))))) + +[case testImplicitGenericTypeArgs] +from typing import TypeVar, Generic +t = TypeVar('t') +s = TypeVar('s') +class A(Generic[t, s]): pass +x = None # type: A +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + AssignmentStmt:3( + NameExpr(s* [__main__.s]) + TypeVarExpr:3()) + ClassDef:4( + A + TypeVars( + t`1 + s`2) + PassStmt:4()) + AssignmentStmt:5( + NameExpr(x [__main__.x]) + NameExpr(None [builtins.None]) + __main__.A[Any, Any])) + +[case testImplicitTypeArgsAndGenericBaseClass] +from typing import TypeVar, Generic +t = TypeVar('t') +s = TypeVar('s') +class B(Generic[s]): pass +class A(B, Generic[t]): pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + AssignmentStmt:3( + NameExpr(s* [__main__.s]) + TypeVarExpr:3()) + ClassDef:4( + B + TypeVars( + s`1) + PassStmt:4()) + ClassDef:5( + A + TypeVars( + t`1) + BaseType( + __main__.B[Any]) + PassStmt:5())) + +[case testTypeApplication] +from typing import TypeVar, Generic +t = TypeVar('t') +class A(Generic[t]): pass +x = A[int]() +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + A + TypeVars( + t`1) + PassStmt:3()) + AssignmentStmt:4( + NameExpr(x* [__main__.x]) + CallExpr:4( + TypeApplication:4( + NameExpr(A [__main__.A]) + Types( + builtins.int)) + Args()))) + +[case testTypeApplicationWithTwoTypeArgs] +from typing import TypeVar, Generic, Any +t = TypeVar('t') +s = TypeVar('s') +class A(Generic[t, s]): pass +x = A[int, Any]() +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic, Any]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + AssignmentStmt:3( + NameExpr(s* [__main__.s]) + TypeVarExpr:3()) + ClassDef:4( + A + TypeVars( + t`1 + s`2) + PassStmt:4()) + AssignmentStmt:5( + NameExpr(x* [__main__.x]) + CallExpr:5( + TypeApplication:5( + NameExpr(A [__main__.A]) + Types( + builtins.int + Any)) + Args()))) + +[case testFunctionTypeApplication] +from typing import TypeVar +t = TypeVar('t') +def f(x: t) -> None: pass +f[int](1) +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + FuncDef:3( + f + Args( + Var(x)) + def [t] (x: t`-1) + Block:3( + PassStmt:3())) + ExpressionStmt:4( + CallExpr:4( + TypeApplication:4( + NameExpr(f [__main__.f]) + Types( + builtins.int)) + Args( + IntExpr(1))))) + +[case testTypeApplicationWithStringLiteralType] +from typing import TypeVar, Generic +t = TypeVar('t') +class A(Generic[t]): pass +A['int']() +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(t* [__main__.t]) + TypeVarExpr:2()) + ClassDef:3( + A + TypeVars( + t`1) + PassStmt:3()) + ExpressionStmt:4( + CallExpr:4( + TypeApplication:4( + NameExpr(A [__main__.A]) + Types( + builtins.int)) + Args()))) + +[case testVarArgsAndKeywordArgs] +def g(*x: int, y: str = ''): pass +[builtins fixtures/tuple.pyi] +[out] +MypyFile:1( + FuncDef:1( + g + MaxPos(0) + Args( + default( + Var(y) + StrExpr())) + def (*x: builtins.int, y: builtins.str =) -> Any + VarArg( + Var(x)) + Block:1( + PassStmt:1()))) + +[case testQualifiedGeneric] +from typing import TypeVar +import typing +T = TypeVar('T') +class A(typing.Generic[T]): pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + Import:2(typing) + AssignmentStmt:3( + NameExpr(T* [__main__.T]) + TypeVarExpr:3()) + ClassDef:4( + A + TypeVars( + T`1) + PassStmt:4())) + +[case testQualifiedTypevar] +import typing +T = typing.TypeVar('T') +def f(x: T) -> T: pass +[out] +MypyFile:1( + Import:1(typing) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2()) + FuncDef:3( + f + Args( + Var(x)) + def [T] (x: T`-1) -> T`-1 + Block:3( + PassStmt:3()))) + +[case testAliasedTypevar] +from typing import TypeVar as tv +T = tv('T') +def f(x: T) -> T: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar : tv]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2()) + FuncDef:3( + f + Args( + Var(x)) + def [T] (x: T`-1) -> T`-1 + Block:3( + PassStmt:3()))) + +[case testLocalTypevar] +from typing import TypeVar +def f(): + T = TypeVar('T') + def g(x: T) -> T: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + FuncDef:2( + f + Block:3( + AssignmentStmt:3( + NameExpr(T* [l]) + TypeVarExpr:3()) + FuncDef:4( + g + Args( + Var(x)) + def [T] (x: T`-1) -> T`-1 + Block:4( + PassStmt:4()))))) + +[case testClassLevelTypevar] +from typing import TypeVar +class A: + T = TypeVar('T') + def g(self, x: T) -> T: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + ClassDef:2( + A + AssignmentStmt:3( + NameExpr(T* [m]) + TypeVarExpr:3()) + FuncDef:4( + g + Args( + Var(self) + Var(x)) + def [T] (self: __main__.A, x: T`-1) -> T`-1 + Block:4( + PassStmt:4())))) + +[case testImportTypevar] +from typing import Generic +from _m import T +class A(Generic[T]): + y = None # type: T +[fixture _m.py] +from typing import TypeVar +T = TypeVar('T') +[out] +MypyFile:1( + ImportFrom:1(typing, [Generic]) + ImportFrom:2(_m, [T]) + ClassDef:3( + A + TypeVars( + T`1) + AssignmentStmt:4( + NameExpr(y [m]) + NameExpr(None [builtins.None]) + T`1))) + +[case testQualifiedReferenceToTypevarInClass] +from typing import Generic +import _m +class A(Generic[_m.T]): + a = None # type: _m.T + def f(self, x: _m.T): + b = None # type: _m.T +[fixture _m.py] +from typing import TypeVar +T = TypeVar('T') +[out] +MypyFile:1( + ImportFrom:1(typing, [Generic]) + Import:2(_m) + ClassDef:3( + A + TypeVars( + _m.T`1) + AssignmentStmt:4( + NameExpr(a [m]) + NameExpr(None [builtins.None]) + _m.T`1) + FuncDef:5( + f + Args( + Var(self) + Var(x)) + def (self: __main__.A[_m.T`1], x: _m.T`1) -> Any + Block:6( + AssignmentStmt:6( + NameExpr(b [l]) + NameExpr(None [builtins.None]) + _m.T`1))))) + +[case testQualifiedReferenceToTypevarInFunctionSignature] +import _m +def f(x: _m.T) -> None: + a = None # type: _m.T +[fixture _m.py] +from typing import TypeVar +T = TypeVar('T') +[out] +MypyFile:1( + Import:1(_m) + FuncDef:2( + f + Args( + Var(x)) + def [_m.T] (x: _m.T`-1) + Block:3( + AssignmentStmt:3( + NameExpr(a [l]) + NameExpr(None [builtins.None]) + _m.T`-1)))) + +[case testFunctionCommentAnnotation] +from typing import Any +def f(x): # type: (int) -> Any + x = 1 +[out] +MypyFile:1( + ImportFrom:1(typing, [Any]) + FuncDef:2( + f + Args( + Var(x)) + def (x: builtins.int) -> Any + Block:3( + AssignmentStmt:3( + NameExpr(x [l]) + IntExpr(1))))) + +[case testMethodCommentAnnotation] +import typing +class A: + def f(self, x): # type: (int) -> str + x = 1 +[out] +MypyFile:1( + Import:1(typing) + ClassDef:2( + A + FuncDef:3( + f + Args( + Var(self) + Var(x)) + def (self: __main__.A, x: builtins.int) -> builtins.str + Block:4( + AssignmentStmt:4( + NameExpr(x [l]) + IntExpr(1)))))) + +[case testTypevarWithValues] +from typing import TypeVar, Any +T = TypeVar('T', int, str) +S = TypeVar('S', Any, int, str) +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Any]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + Values( + builtins.int + builtins.str))) + AssignmentStmt:3( + NameExpr(S* [__main__.S]) + TypeVarExpr:3( + Values( + Any + builtins.int + builtins.str)))) + +[case testTypevarWithValuesAndVariance] +from typing import TypeVar +T = TypeVar('T', int, str, covariant=True) +[builtins fixtures/bool.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + Variance(COVARIANT) + Values( + builtins.int + builtins.str)))) + +[case testTypevarWithFalseVariance] +from typing import TypeVar +T1 = TypeVar('T1', covariant=False) +T2 = TypeVar('T2', covariant=False, contravariant=False) +T3 = TypeVar('T3', contravariant=False) +T4 = TypeVar('T4', covariant=True, contravariant=False) +T5 = TypeVar('T5', covariant=False, contravariant=True) +[builtins fixtures/bool.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(T1* [__main__.T1]) + TypeVarExpr:2()) + AssignmentStmt:3( + NameExpr(T2* [__main__.T2]) + TypeVarExpr:3()) + AssignmentStmt:4( + NameExpr(T3* [__main__.T3]) + TypeVarExpr:4()) + AssignmentStmt:5( + NameExpr(T4* [__main__.T4]) + TypeVarExpr:5( + Variance(COVARIANT))) + AssignmentStmt:6( + NameExpr(T5* [__main__.T5]) + TypeVarExpr:6( + Variance(CONTRAVARIANT)))) + +[case testTypevarWithBound] +from typing import TypeVar +T = TypeVar('T', bound=int) +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + UpperBound(builtins.int)))) + +[case testGenericFunctionWithValueSet] +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + Values( + builtins.int + builtins.str))) + FuncDef:3( + f + Args( + Var(x)) + def [T in (builtins.int, builtins.str)] (x: T`-1) -> T`-1 + Block:3( + PassStmt:3()))) + +[case testGenericClassWithValueSet] +from typing import TypeVar, Generic +T = TypeVar('T', int, str) +class C(Generic[T]): pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + Values( + builtins.int + builtins.str))) + ClassDef:3( + C + TypeVars( + T`1) + PassStmt:3())) + +[case testGenericFunctionWithBound] +from typing import TypeVar +T = TypeVar('T', bound=int) +def f(x: T) -> T: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + UpperBound(builtins.int))) + FuncDef:3( + f + Args( + Var(x)) + def [T <: builtins.int] (x: T`-1) -> T`-1 + Block:3( + PassStmt:3()))) + +[case testGenericClassWithBound] +from typing import TypeVar, Generic +T = TypeVar('T', bound=int) +class C(Generic[T]): pass +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar, Generic]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + UpperBound(builtins.int))) + ClassDef:3( + C + TypeVars( + T`1) + PassStmt:3())) + +[case testSimpleDucktypeDecorator] +from typing import _promote +@_promote(str) +class S: pass +[typing fixtures/typing-medium.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [_promote]) + ClassDef:3( + S + Promote([builtins.str]) + Decorators( + PromoteExpr:2(builtins.str)) + PassStmt:3())) + +[case testUnionType] +from typing import Union +def f(x: Union[int, str]) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + FuncDef:2( + f + Args( + Var(x)) + def (x: Union[builtins.int, builtins.str]) + Block:2( + PassStmt:2()))) + +[case testUnionTypeWithNoneItem] +from typing import Union +def f(x: Union[int, None]) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + FuncDef:2( + f + Args( + Var(x)) + def (x: Union[builtins.int, None]) + Block:2( + PassStmt:2()))) + +[case testUnionTypeWithNoneItemAndTwoItems] +from typing import Union +def f(x: Union[int, None, str]) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + FuncDef:2( + f + Args( + Var(x)) + def (x: Union[builtins.int, None, builtins.str]) + Block:2( + PassStmt:2()))) + +[case testUnionTypeWithSingleItem] +from typing import Union +def f(x: Union[int]) -> None: pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Union]) + FuncDef:2( + f + Args( + Var(x)) + def (x: builtins.int) + Block:2( + PassStmt:2()))) + +[case testOptionalTypes] +from typing import Optional +x = 1 # type: Optional[int] +[out] +MypyFile:1( + ImportFrom:1(typing, [Optional]) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + IntExpr(1) + Union[builtins.int, None])) + +[case testInvalidOptionalType] +from typing import Optional +x = 1 # type: Optional[int, str] # E: Optional[...] must have exactly one type argument +y = 1 # type: Optional # E: Optional[...] must have exactly one type argument +[out] + +[case testCoAndContravariantTypeVar] +from typing import TypeVar +T = TypeVar('T', covariant=True) +S = TypeVar('S', contravariant=True) +[builtins fixtures/bool.pyi] +[out] +MypyFile:1( + ImportFrom:1(typing, [TypeVar]) + AssignmentStmt:2( + NameExpr(T* [__main__.T]) + TypeVarExpr:2( + Variance(COVARIANT))) + AssignmentStmt:3( + NameExpr(S* [__main__.S]) + TypeVarExpr:3( + Variance(CONTRAVARIANT)))) + +[case testTupleExpressionAsType] +def f(x: (int, int)) -> None: pass +[out] +main:1: error: Syntax error in type annotation +main:1: note: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) + +[case testQualifiedTypeNameBasedOnAny] +from typing import Any +x = 0 # type: Any +z = 0 # type: x.y +[out] +MypyFile:1( + ImportFrom:1(typing, [Any]) + AssignmentStmt:2( + NameExpr(x [__main__.x]) + IntExpr(0) + Any) + AssignmentStmt:3( + NameExpr(z [__main__.z]) + IntExpr(0) + Any)) + + +[case testParamSpec] +from typing import ParamSpec +P = ParamSpec("P") +[out] +MypyFile:1( + ImportFrom:1(typing, [ParamSpec]) + AssignmentStmt:2( + NameExpr(P* [__main__.P]) + ParamSpecExpr:2())) + +[case testTypeVarTuple] +from typing_extensions import TypeVarTuple +TV = TypeVarTuple("TV") +[out] +MypyFile:1( + ImportFrom:1(typing_extensions, [TypeVarTuple]) + AssignmentStmt:2( + NameExpr(TV* [__main__.TV]) + TypeVarTupleExpr:2( + UpperBound(builtins.tuple[builtins.object, ...])))) +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleCallable] +from typing_extensions import TypeVarTuple, Unpack +from typing import Callable +Ts = TypeVarTuple("Ts") + +def foo(x: Callable[[Unpack[Ts]], None]) -> None: + pass +[out] +MypyFile:1( + ImportFrom:1(typing_extensions, [TypeVarTuple, Unpack]) + ImportFrom:2(typing, [Callable]) + AssignmentStmt:3( + NameExpr(Ts* [__main__.Ts]) + TypeVarTupleExpr:3( + UpperBound(builtins.tuple[builtins.object, ...]))) + FuncDef:5( + foo + Args( + Var(x)) + def [Ts] (x: def (*Unpack[Ts`-1])) + Block:6( + PassStmt:6()))) + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test new file mode 100644 index 000000000000..0801d9a27011 --- /dev/null +++ b/test-data/unit/stubgen.test @@ -0,0 +1,4528 @@ +-- Test cases for stubgen that generate stubs from Python code + +[case testEmptyFile] +[out] + +[case testSingleFunction] +def f(): + x = 1 +[out] +def f() -> None: ... + +[case testTwoFunctions] +def f(a, b): + """ + this is a docstring + + more. + """ + x = 1 +def g(arg): + pass +[out] +def f(a, b) -> None: ... +def g(arg) -> None: ... + +[case testDefaultArgInt] +def f(a, b=2): ... +def g(b=-1, c=0): ... +[out] +def f(a, b: int = 2) -> None: ... +def g(b: int = -1, c: int = 0) -> None: ... + +[case testDefaultArgNone] +def f(x=None): ... +[out] +from _typeshed import Incomplete + +def f(x: Incomplete | None = None) -> None: ... + +[case testDefaultArgBool] +def f(x=True, y=False): ... +[out] +def f(x: bool = True, y: bool = False) -> None: ... + +[case testDefaultArgBool_inspect] +def f(x=True, y=False): ... +[out] +def f(x: bool = ..., y: bool = ...): ... + +[case testDefaultArgStr] +def f(x='foo',y="how's quotes"): ... +[out] +def f(x: str = 'foo', y: str = "how's quotes") -> None: ... + +[case testDefaultArgStr_inspect] +def f(x='foo'): ... +[out] +def f(x: str = ...): ... + +[case testDefaultArgBytes] +def f(x=b'foo',y=b"what's up",z=b'\xc3\xa0 la une'): ... +[out] +def f(x: bytes = b'foo', y: bytes = b"what's up", z: bytes = b'\xc3\xa0 la une') -> None: ... + +[case testDefaultArgFloat] +def f(x=1.2,y=1e-6,z=0.0,w=-0.0,v=+1.0): ... +def g(x=float("nan"), y=float("inf"), z=float("-inf")): ... +[out] +def f(x: float = 1.2, y: float = 1e-06, z: float = 0.0, w: float = -0.0, v: float = +1.0) -> None: ... +def g(x=..., y=..., z=...) -> None: ... + +[case testDefaultArgOther] +def f(x=ord): ... +[out] +def f(x=...) -> None: ... + +[case testPreserveFunctionAnnotation] +def f(x: Foo) -> Bar: ... +def g(x: Foo = Foo()) -> Bar: ... +[out] +def f(x: Foo) -> Bar: ... +def g(x: Foo = ...) -> Bar: ... + +[case testPreserveFunctionAnnotationWithArgs] +def f(x: foo['x']) -> bar: ... +def g(x: foo[x]) -> bar: ... +def h(x: foo['x', 'y']) -> bar: ... +def i(x: foo[x, y]) -> bar: ... +def j(x: foo['x', y]) -> bar: ... +def k(x: foo[x, 'y']) -> bar: ... +def lit_str(x: Literal['str']) -> Literal['str']: ... +def lit_int(x: Literal[1]) -> Literal[1]: ... +[out] +def f(x: foo['x']) -> bar: ... +def g(x: foo[x]) -> bar: ... +def h(x: foo['x', 'y']) -> bar: ... +def i(x: foo[x, y]) -> bar: ... +def j(x: foo['x', y]) -> bar: ... +def k(x: foo[x, 'y']) -> bar: ... +def lit_str(x: Literal['str']) -> Literal['str']: ... +def lit_int(x: Literal[1]) -> Literal[1]: ... + +[case testPreserveVarAnnotation] +x: Foo +[out] +x: Foo + +[case testPreserveVarAnnotationWithoutQuotes] +x: 'Foo' +[out] +x: Foo + +[case testVarArgs] +def f(x, *y): ... +[out] +def f(x, *y) -> None: ... + +[case testKwVarArgs] +def f(x, **y): ... +[out] +def f(x, **y) -> None: ... + +[case testVarArgsWithKwVarArgs] +def f(a, *b, **c): ... +def g(a, *b, c=1): ... +def h(a, *b, c=1, **d): ... +def i(a, *, b=1): ... +def j(a, *, b=1, **c): ... +[out] +def f(a, *b, **c) -> None: ... +def g(a, *b, c: int = 1) -> None: ... +def h(a, *b, c: int = 1, **d) -> None: ... +def i(a, *, b: int = 1) -> None: ... +def j(a, *, b: int = 1, **c) -> None: ... + +[case testClass] +class A: + def f(self, x): + x = 1 +def g(): ... +[out] +class A: + def f(self, x) -> None: ... + +def g() -> None: ... + +[case testVariables] +i = 1 +s = 'a' +f = 1.5 +c1 = 1j +c2 = 0j + 1 +bl1 = True +bl2 = False +bts = b'' +[out] +i: int +s: str +f: float +c1: complex +c2: complex +bl1: bool +bl2: bool +bts: bytes + +[case testVariablesWithUnary] +i = +-1 +f = -1.5 +c1 = -1j +c2 = -1j + 1 +bl1 = not True +bl2 = not not False +[out] +i: int +f: float +c1: complex +c2: complex +bl1: bool +bl2: bool + +[case testVariablesWithUnaryWrong] +i = not +1 +bl1 = -True +bl2 = not -False +bl3 = -(not False) +[out] +from _typeshed import Incomplete + +i: Incomplete +bl1: Incomplete +bl2: Incomplete +bl3: Incomplete + +[case testAnnotatedVariable] +x: int = 1 +[out] +x: int + +[case testAnnotatedVariableGeneric] +x: Foo[int, str] = ... +[out] +x: Foo[int, str] + +[case testAnnotatedVariableOldSyntax] +x = 1 # type: int +[out] +x: int + +[case testAnnotatedVariableNone] +x: None +[out] +x: None + +[case testAnnotatedVariableNoneOldSyntax] +x = None # type: None +[out] +x: None + +[case testMultipleVariable] +x = y = 1 +[out] +x: int +y: int + +[case testClassVariable] +class C: + x = 1 +[out] +class C: + x: int + +[case testInitTypeAnnotationPreserved] +class C: + def __init__(self, x: str): + pass +[out] +class C: + def __init__(self, x: str) -> None: ... + +[case testSelfAssignment] +class C: + def __init__(self): + self.x = 1 + x.y = 2 +[out] +class C: + x: int + def __init__(self) -> None: ... + +[case testSelfAndClassBodyAssignment] +x = 1 +class C: + x = 1 + def __init__(self): + self.x = 1 + self.x = 1 +[out] +x: int + +class C: + x: int + def __init__(self) -> None: ... + +[case testEmptyClass] +class A: ... +[out] +class A: ... + +[case testSkipPrivateFunction] +def _f(): ... +def g(): ... +[out] +def g() -> None: ... + +[case testIncludePrivateFunction] +# flags: --include-private +def _f(): ... +def g(): ... +[out] +def _f() -> None: ... +def g() -> None: ... + +[case testSkipPrivateMethod] +class A: + def _f(self): ... +[out] +class A: ... + +[case testIncludePrivateMethod] +# flags: --include-private +class A: + def _f(self): ... +[out] +class A: + def _f(self) -> None: ... + +[case testSkipPrivateVar] +_x = 1 +class A: + _y = 1 +[out] +class A: ... + +[case testIncludePrivateVar] +# flags: --include-private +_x = 1 +class A: + _y = 1 +[out] +_x: int + +class A: + _y: int + +[case testSpecialInternalVar] +__all__ = [] +__author__ = '' +__version__ = '' +[out] +__version__: str + +[case testBaseClass] +class A: ... +class B(A): ... +[out] +class A: ... +class B(A): ... + +[case testDecoratedFunction] +@decorator +def foo(x): ... +[out] +def foo(x) -> None: ... + +[case testMultipleAssignment] +x, y = 1, 2 +[out] +from _typeshed import Incomplete + +x: Incomplete +y: Incomplete + +[case testMultipleAssignmentAnnotated] +x, y = 1, "2" # type: int, str +[out] +x: int +y: str + +[case testMultipleAssignment2] +[x, y] = 1, 2 +[out] +from _typeshed import Incomplete + +x: Incomplete +y: Incomplete + +[case testKeywordOnlyArg] +def f(x, *, y=1): ... +def g(x, *, y=1, z=2): ... +[out] +def f(x, *, y: int = 1) -> None: ... +def g(x, *, y: int = 1, z: int = 2) -> None: ... + +[case testKeywordOnlyArg_inspect] +def f(x, *, y=1): ... +def g(x, *, y=1, z=2): ... +def h(x, *, y, z=2): ... +[out] +def f(x, *, y: int = ...): ... +def g(x, *, y: int = ..., z: int = ...): ... +def h(x, *, y, z: int = ...): ... + +[case testProperty] +class A: + @property + def f(self): + return 1 + @f.setter + def f(self, x): ... + @f.deleter + def f(self): ... + + def h(self): + self.f = 1 +[out] +class A: + @property + def f(self): ... + @f.setter + def f(self, x) -> None: ... + @f.deleter + def f(self) -> None: ... + def h(self) -> None: ... + +[case testProperty_semanal] +class A: + @property + def f(self): + return 1 + @f.setter + def f(self, x): ... + @f.deleter + def f(self): ... + + def h(self): + self.f = 1 +[out] +class A: + @property + def f(self): ... + @f.setter + def f(self, x) -> None: ... + @f.deleter + def f(self) -> None: ... + def h(self) -> None: ... + +-- a read/write property is treated the same as an attribute +[case testProperty_inspect] +class A: + @property + def f(self): + return 1 + @f.setter + def f(self, x): ... + + def h(self): + self.f = 1 +[out] +from _typeshed import Incomplete + +class A: + f: Incomplete + def h(self): ... + +[case testFunctoolsCachedProperty] +import functools + +class A: + @functools.cached_property + def x(self): + return 'x' +[out] +import functools + +class A: + @functools.cached_property + def x(self): ... + +[case testFunctoolsCachedPropertyAlias] +import functools as ft + +class A: + @ft.cached_property + def x(self): + return 'x' +[out] +import functools as ft + +class A: + @ft.cached_property + def x(self): ... + +[case testCachedProperty] +from functools import cached_property + +class A: + @cached_property + def x(self): + return 'x' +[out] +from functools import cached_property + +class A: + @cached_property + def x(self): ... + +[case testCachedPropertyAlias] +from functools import cached_property as cp + +class A: + @cp + def x(self): + return 'x' +[out] +from functools import cached_property as cp + +class A: + @cp + def x(self): ... + +[case testStaticMethod] +class A: + @staticmethod + def f(x): ... +[out] +class A: + @staticmethod + def f(x) -> None: ... + +[case testClassMethod] +class A: + @classmethod + def f(cls): ... +[out] +class A: + @classmethod + def f(cls) -> None: ... + +[case testClassMethod_inspect] +class A: + @classmethod + def f(cls): ... +[out] +class A: + @classmethod + def f(cls): ... + +[case testIfMainCheck] +def a(): ... +if __name__ == '__main__': + x = 1 + def f(): ... +def b(): ... +[out] +def a() -> None: ... +def b() -> None: ... + +[case testImportStar] +from x import * +from a.b import * +def f(): ... +[out] +from x import * +from a.b import * + +def f() -> None: ... + +[case testNoSpacesBetweenEmptyClasses] +class X: + def g(self): ... +class A: ... +class B: ... +class C: + def f(self): ... +[out] +class X: + def g(self) -> None: ... + +class A: ... +class B: ... + +class C: + def f(self) -> None: ... + +[case testNoSpacesBetweenEmptyClasses_inspect] +class X: + def g(self): ... +class A: ... +class B: ... +class C: + def f(self): ... +[out] +class X: + def g(self): ... + +class A: ... +class B: ... + +class C: + def f(self): ... + +[case testExceptionBaseClasses] +class A(Exception): ... +class B(ValueError): ... +[out] +class A(Exception): ... +class B(ValueError): ... + +[case testOmitSomeSpecialMethods] +class A: + def __str__(self): ... + def __repr__(self): ... + def __eq__(self): ... + def __getstate__(self): ... + def __setstate__(self, state): ... +[out] +class A: + def __eq__(self): ... + +[case testOmitSomeSpecialMethods_inspect] +class A: + def __str__(self): ... + def __repr__(self): ... + def __eq__(self): ... + def __getstate__(self): ... + def __setstate__(self, state): ... +[out] +class A: + def __eq__(self) -> bool: ... + +-- Tests that will perform runtime imports of modules. +-- Don't use `_import` suffix if there are unquoted forward references. + +[case testOmitDefsNotInAll_import] +__all__ = [] + ['f'] +def f(): ... +def g(): ... +[out] +__all__ = ['f'] + +def f() -> None: ... + +[case testOmitDefsNotInAll_semanal] +__all__ = ['f'] +def f(): ... +def g(): ... +[out] +__all__ = ['f'] + +def f() -> None: ... + +[case testOmitDefsNotInAll_inspect] +__all__ = [] + ['f'] +def f(): ... +def g(): ... +[out] +__all__ = ['f'] + +def f(): ... + +[case testVarDefsNotInAll_import] +__all__ = [] + ['f', 'g'] +def f(): ... +x = 1 +y = 1 +def g(): ... +[out] +__all__ = ['f', 'g'] + +def f() -> None: ... +def g() -> None: ... + +[case testVarDefsNotInAll_inspect] +__all__ = [] + ['f', 'g'] +def f(): ... +x = 1 +y = 1 +def g(): ... +[out] +__all__ = ['f', 'g'] + +def f(): ... +def g(): ... + +[case testIncludeClassNotInAll_import] +__all__ = [] + ['f'] +def f(): ... +class A: ... +[out] +__all__ = ['f'] + +def f() -> None: ... + +class A: ... + +[case testIncludeClassNotInAll_inspect] +__all__ = [] + ['f'] +def f(): ... +class A: ... +[out] +__all__ = ['f'] + +def f(): ... + +class A: ... + +[case testAllAndClass_import] +__all__ = ['A'] +class A: + x = 1 + def f(self): ... +[out] +__all__ = ['A'] + +class A: + x: int + def f(self) -> None: ... + +[case testSkipMultiplePrivateDefs] +class A: ... +_x = 1 +_y = 1 +_z = 1 +class C: ... +[out] +class A: ... +class C: ... + +[case testIncludeMultiplePrivateDefs] +# flags: --include-private +class A: ... +_x = 1 +_y = 1 +_z = 1 +class C: ... +[out] +class A: ... + +_x: int +_y: int +_z: int + +class C: ... + +[case testIncludeFromImportIfInAll_import] +from re import match, search, sub +__all__ = ['match', 'sub', 'x'] +x = 1 +[out] +from re import match as match, sub as sub + +__all__ = ['match', 'sub', 'x'] + +x: int + +[case testExportModule_import] +import re +__all__ = ['re', 'x'] +x = 1 +y = 2 +[out] +import re as re + +__all__ = ['re', 'x'] + +x: int + +[case testExportModule2_import] +import re +__all__ = ['re', 'x'] +x = 1 +y = 2 +[out] +import re as re + +__all__ = ['re', 'x'] + +x: int + +[case testExportModuleAs_import] +import re as rex +__all__ = ['rex', 'x'] +x = 1 +y = 2 +[out] +import re as rex + +__all__ = ['rex', 'x'] + +x: int + +[case testExportModuleInPackage_import] +import urllib.parse as p +__all__ = ['p'] +[out] +import urllib.parse as p + +__all__ = ['p'] + +[case testExportPackageOfAModule_import] +import urllib.parse +__all__ = ['urllib'] + +[out] +import urllib as urllib + +__all__ = ['urllib'] + +[case testRelativeImportAll] +from .x import * +[out] +from .x import * + +[case testCommentForUndefinedName_import] +__all__ = ['f', 'x', 'C', 'g'] +def f(): ... +x = 1 +class C: + def g(self): ... +[out] +__all__ = ['f', 'x', 'C', 'g'] + +def f() -> None: ... + +x: int + +class C: + def g(self) -> None: ... + +# Names in __all__ with no definition: +# g + +[case testCommentForUndefinedName_inspect] +__all__ = ['f', 'x', 'C', 'g'] +def f(): ... +x = 1 +class C: + def g(self): ... +[out] +__all__ = ['f', 'x', 'C', 'g'] + +def f(): ... + +x: int + +class C: + def g(self): ... + +# Names in __all__ with no definition: +# g + +[case testIgnoreSlots] +class A: + __slots__ = () +[out] +class A: ... + +[case testSkipPrivateProperty] +class A: + @property + def _foo(self): ... +[out] +class A: ... + +[case testSkipPrivateProperty_inspect] +class A: + @property + def _foo(self): ... +[out] +class A: ... + +[case testIncludePrivateProperty] +# flags: --include-private +class A: + @property + def _foo(self): ... +[out] +class A: + @property + def _foo(self) -> None: ... + +[case testIncludePrivateProperty_inspect] +# flags: --include-private +class A: + @property + def _foo(self): ... +[out] +class A: + @property + def _foo(self): ... + +[case testSkipPrivateStaticAndClassMethod] +class A: + @staticmethod + def _foo(): ... + @classmethod + def _bar(cls): ... +[out] +class A: ... + +[case testSkipPrivateStaticAndClassMethod_inspect] +class A: + @staticmethod + def _foo(): ... + @classmethod + def _bar(cls): ... +[out] +class A: ... + +[case testIncludePrivateStaticAndClassMethod] +# flags: --include-private +class A: + @staticmethod + def _foo(): ... + @classmethod + def _bar(cls): ... +[out] +class A: + @staticmethod + def _foo() -> None: ... + @classmethod + def _bar(cls) -> None: ... + +[case testIncludePrivateStaticAndClassMethod_inspect] +# flags: --include-private +class A: + @staticmethod + def _foo(): ... + @classmethod + def _bar(cls): ... +[out] +class A: + @staticmethod + def _foo(): ... + @classmethod + def _bar(cls): ... + +[case testNamedtuple] +import collections, typing, x +X = collections.namedtuple('X', ['a', 'b']) +Y = typing.NamedTuple('Y', [('a', int), ('b', str)]) +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class X(NamedTuple): + a: Incomplete + b: Incomplete + +class Y(NamedTuple): + a: int + b: str + +[case testNamedTupleClassSyntax_semanal] +from typing import NamedTuple + +class A(NamedTuple): + x: int + y: str = 'a' + +class B(A): + z1: str + z2 = 1 + z3: str = 'b' + +class RegularClass: + x: int + y: str = 'a' + class NestedNamedTuple(NamedTuple): + x: int + y: str = 'a' + z: str = 'b' +[out] +from typing import NamedTuple + +class A(NamedTuple): + x: int + y: str = ... + +class B(A): + z1: str + z2: int + z3: str + +class RegularClass: + x: int + y: str + class NestedNamedTuple(NamedTuple): + x: int + y: str = ... + z: str + + +[case testNestedClassInNamedTuple_semanal-xfail] +from typing import NamedTuple + +# TODO: make sure that nested classes in `NamedTuple` are supported: +class NamedTupleWithNestedClass(NamedTuple): + class Nested: + x: int + y: str = 'a' +[out] +from typing import NamedTuple + +class NamedTupleWithNestedClass(NamedTuple): + class Nested: + x: int + y: str + +[case testEmptyNamedtuple] +import collections, typing +X = collections.namedtuple('X', []) +Y = typing.NamedTuple('Y', []) +[out] +from typing import NamedTuple + +class X(NamedTuple): ... +class Y(NamedTuple): ... + +[case testNamedtupleAltSyntax] +from collections import namedtuple, xx +X = namedtuple('X', 'a b') +xx +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class X(NamedTuple): + a: Incomplete + b: Incomplete + +[case testNamedtupleAltSyntaxUsingComma] +from collections import namedtuple, xx +X = namedtuple('X', 'a, b') +xx +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class X(NamedTuple): + a: Incomplete + b: Incomplete + +[case testNamedtupleAltSyntaxUsingMultipleCommas] +from collections import namedtuple, xx +X = namedtuple('X', 'a,, b') +xx +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class X(NamedTuple): + a: Incomplete + b: Incomplete + +[case testNamedtupleWithUnderscore] +from collections import namedtuple as _namedtuple +from typing import NamedTuple as _NamedTuple +def f(): ... +X = _namedtuple('X', 'a b') +Y = _NamedTuple('Y', [('a', int), ('b', str)]) +def g(): ... +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +def f() -> None: ... + +class X(NamedTuple): + a: Incomplete + b: Incomplete + +class Y(NamedTuple): + a: int + b: str + +def g() -> None: ... + +[case testNamedtupleBaseClass] +import collections, x +_X = collections.namedtuple('_X', ['a', 'b']) +class Y(_X): ... +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class _X(NamedTuple): + a: Incomplete + b: Incomplete + +class Y(_X): ... + +[case testNamedtupleAltSyntaxFieldsTuples] +from collections import namedtuple, xx +from typing import NamedTuple +X = namedtuple('X', ()) +Y = namedtuple('Y', ('a',)) +Z = namedtuple('Z', ('a', 'b', 'c', 'd', 'e')) +xx +R = NamedTuple('R', ()) +S = NamedTuple('S', (('a', int),)) +T = NamedTuple('T', (('a', int), ('b', str))) +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class X(NamedTuple): ... + +class Y(NamedTuple): + a: Incomplete + +class Z(NamedTuple): + a: Incomplete + b: Incomplete + c: Incomplete + d: Incomplete + e: Incomplete + +class R(NamedTuple): ... + +class S(NamedTuple): + a: int + +class T(NamedTuple): + a: int + b: str + +[case testDynamicNamedTuple] +from collections import namedtuple +from typing import NamedTuple +N = namedtuple('N', ['x', 'y'] + ['z']) +M = NamedTuple('M', [('x', int), ('y', str)] + [('z', float)]) +class X(namedtuple('X', ['a', 'b'] + ['c'])): ... +[out] +from _typeshed import Incomplete + +N: Incomplete +M: Incomplete + +class X(Incomplete): ... + +[case testNamedTupleInClassBases] +import collections, typing +from collections import namedtuple +from typing import NamedTuple +class X(namedtuple('X', ['a', 'b'])): ... +class Y(NamedTuple('Y', [('a', int), ('b', str)])): ... +class R(collections.namedtuple('R', ['a', 'b'])): ... +class S(typing.NamedTuple('S', [('a', int), ('b', str)])): ... +[out] +import typing +from _typeshed import Incomplete +from typing import NamedTuple + +class X(NamedTuple('X', [('a', Incomplete), ('b', Incomplete)])): ... +class Y(NamedTuple('Y', [('a', int), ('b', str)])): ... +class R(NamedTuple('R', [('a', Incomplete), ('b', Incomplete)])): ... +class S(typing.NamedTuple('S', [('a', int), ('b', str)])): ... + +[case testNotNamedTuple] +from not_collections import namedtuple +from not_typing import NamedTuple +from collections import notnamedtuple +from typing import NotNamedTuple +X = namedtuple('X', ['a', 'b']) +Y = notnamedtuple('Y', ['a', 'b']) +Z = NamedTuple('Z', [('a', int), ('b', str)]) +W = NotNamedTuple('W', [('a', int), ('b', str)]) +[out] +from _typeshed import Incomplete + +X: Incomplete +Y: Incomplete +Z: Incomplete +W: Incomplete + +[case testNamedTupleFromImportAlias] +import collections as c +import typing as t +import typing_extensions as te +X = c.namedtuple('X', ['a', 'b']) +Y = t.NamedTuple('Y', [('a', int), ('b', str)]) +Z = te.NamedTuple('Z', [('a', int), ('b', str)]) +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class X(NamedTuple): + a: Incomplete + b: Incomplete + +class Y(NamedTuple): + a: int + b: str + +class Z(NamedTuple): + a: int + b: str + +[case testArbitraryBaseClass] +import x +class D(x.C): ... +[out] +import x + +class D(x.C): ... + +[case testArbitraryBaseClass2] +import x.y +class D(x.y.C): ... +[out] +import x.y + +class D(x.y.C): ... + +[case testUnqualifiedArbitraryBaseClassWithNoDef] +class A(int): ... +[out] +class A(int): ... + +[case testUnqualifiedArbitraryBaseClass] +from x import X +class A(X): ... +[out] +from x import X + +class A(X): ... + +[case testUnqualifiedArbitraryBaseClassWithImportAs] +from x import X as _X +class A(_X): ... +[out] +from x import X as _X + +class A(_X): ... + +[case testGenericClass] +class D(Generic[T]): ... +[out] +class D(Generic[T]): ... + +[case testGenericClass_semanal] +from typing import Generic, TypeVar +T = TypeVar('T') +class D(Generic[T]): ... +[out] +from typing import Generic, TypeVar + +T = TypeVar('T') + +class D(Generic[T]): ... + +[case testGenericClassTypeVarTuple] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack +Ts = TypeVarTuple('Ts') +class D(Generic[Unpack[Ts]]): ... +[out] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple('Ts') + +class D(Generic[Unpack[Ts]]): ... + +[case testGenericClassTypeVarTuple_semanal] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack +Ts = TypeVarTuple('Ts') +class D(Generic[Unpack[Ts]]): ... +[out] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple('Ts') + +class D(Generic[Unpack[Ts]]): ... + +[case testGenericClassTypeVarTuplePy311] +# flags: --python-version=3.11 +from typing import Generic, TypeVarTuple +Ts = TypeVarTuple('Ts') +class D(Generic[*Ts]): ... +[out] +from typing import Generic, TypeVarTuple + +Ts = TypeVarTuple('Ts') + +class D(Generic[*Ts]): ... + +[case testGenericClassTypeVarTuplePy311_semanal] +# flags: --python-version=3.11 +from typing import Generic, TypeVarTuple +Ts = TypeVarTuple('Ts') +class D(Generic[*Ts]): ... +[out] +from typing import Generic, TypeVarTuple + +Ts = TypeVarTuple('Ts') + +class D(Generic[*Ts]): ... + +[case testObjectBaseClass] +class A(object): ... +[out] +class A: ... + +[case testObjectBaseClassWithImport] +import builtins as b +class A(b.object): ... +[out] +class A: ... + +[case testEmptyLines] +def x(): ... +def f(): + class A: + def f(self): + self.x = 1 +def g(): ... +[out] +def x() -> None: ... +def f() -> None: ... +def g() -> None: ... + +[case testNestedClass] +class A: + class B: + x = 1 + def f(self): ... + def g(self): ... +[out] +class A: + class B: + x: int + def f(self) -> None: ... + def g(self) -> None: ... + +[case testExportViaRelativeImport] +from .api import get +[out] +from .api import get as get + +[case testExportViaRelativePackageImport] +from .packages.urllib3.contrib import parse +[out] +from .packages.urllib3.contrib import parse as parse + +[case testNoExportViaRelativeImport] +from . import get +get() +[out] + +[case testRelativeImportAndBase] +from .x import X +class A(X): + pass +[out] +from .x import X + +class A(X): ... + +[case testDuplicateDef] +def syslog(a): pass +def syslog(a): pass +[out] +def syslog(a) -> None: ... + +[case testAsyncAwait_fast_parser] +async def f(a): + x = await y +[out] +async def f(a) -> None: ... + +[case testInferOptionalOnlyFunc] +class A: + x = None + def __init__(self, a=None): + self.x = [] + def method(self, a=None): + self.x = [] +[out] +from _typeshed import Incomplete + +class A: + x: Incomplete + def __init__(self, a: Incomplete | None = None) -> None: ... + def method(self, a: Incomplete | None = None) -> None: ... + +[case testAnnotationImportsFrom] +import foo +from collections import defaultdict +x: defaultdict + +[out] +from collections import defaultdict + +x: defaultdict + +[case testAnnotationImports] +import foo +import collections +x: collections.defaultdict + +[out] +import collections + +x: collections.defaultdict + + +[case testAnnotationImports2] +from typing import List +import collections +x: List[collections.defaultdict] + +[out] +import collections + +x: list[collections.defaultdict] + + +[case testAnnotationFwRefs] +x: C + +class C: + attr: C + +y: C +[out] +x: C + +class C: + attr: C + +y: C + +[case testTypeVarPreserved] +tv = TypeVar('tv') +ps = ParamSpec('ps') +tvt = TypeVarTuple('tvt') + +[out] +from typing import TypeVar +from typing_extensions import ParamSpec, TypeVarTuple + +tv = TypeVar('tv') +ps = ParamSpec('ps') +tvt = TypeVarTuple('tvt') + +[case testTypeVarArgsPreserved] +tv = TypeVar('tv', int, str) + +[out] +from typing import TypeVar + +tv = TypeVar('tv', int, str) + +[case testTypeVarNamedArgsPreserved] +tv = TypeVar('tv', bound=bool, covariant=True) +ps = ParamSpec('ps', bound=bool, covariant=True) + +[out] +from typing import TypeVar +from typing_extensions import ParamSpec + +tv = TypeVar('tv', bound=bool, covariant=True) +ps = ParamSpec('ps', bound=bool, covariant=True) + +[case TypeVarImportAlias] +from typing import TypeVar as t_TV, ParamSpec as t_PS +from typing_extensions import TypeVar as te_TV, TypeVarTuple as te_TVT +from x import TypeVar as x_TV + +T = t_TV('T') +U = te_TV('U') +V = x_TV('V') + +PS = t_PS('PS') +TVT = te_TVT('TVT') + +[out] +from _typeshed import Incomplete +from typing import ParamSpec as t_PS, TypeVar as t_TV +from typing_extensions import TypeVar as te_TV, TypeVarTuple as te_TVT + +T = t_TV('T') +U = te_TV('U') +V: Incomplete +PS = t_PS('PS') +TVT = te_TVT('TVT') + +[case testTypeVarFromImportAlias] +import typing as t +import typing_extensions as te +import x + +T = t.TypeVar('T') +U = te.TypeVar('U') +V = x.TypeVar('V') + +PS = t.ParamSpec('PS') +TVT = te.TypeVarTuple('TVT') + +[out] +import typing as t +import typing_extensions as te +from _typeshed import Incomplete + +T = t.TypeVar('T') +U = te.TypeVar('U') +V: Incomplete +PS = t.ParamSpec('PS') +TVT = te.TypeVarTuple('TVT') + +[case testTypeAliasPreserved] +alias = str + +[out] +alias = str + +[case testDeepTypeAliasPreserved] + +alias = Dict[str, List[str]] + +[out] +alias = Dict[str, List[str]] + +[case testDeepGenericTypeAliasPreserved] +from typing import TypeVar + +T = TypeVar('T') +alias = Union[T, List[T]] + +[out] +from typing import TypeVar + +T = TypeVar('T') +alias = Union[T, List[T]] + +[case testEllipsisAliasPreserved] + +alias = Tuple[int, ...] + +[out] +alias = Tuple[int, ...] + +[case testCallableAliasPreserved] + +alias1 = Callable[..., int] +alias2 = Callable[[str, bool], None] + +[out] +alias1 = Callable[..., int] +alias2 = Callable[[str, bool], None] + +[case testAliasPullsImport] +from module import Container + +alias = Container[Any] + +[out] +from module import Container +from typing import Any + +alias = Container[Any] + +[case testAliasExceptions] +noalias1 = None +noalias2 = ... +noalias3 = True + +[out] +from _typeshed import Incomplete + +noalias1: Incomplete +noalias2: Incomplete +noalias3: bool + +[case testComplexAlias] +# modules: main a + +from a import valid + +def func() -> int: + return 2 + +aliased_func = func +int_value = 1 + +class A: + cls_var = valid + + def __init__(self, arg: str) -> None: + self.self_var = arg + + def meth(self) -> None: + func_value = int_value + + alias_meth = meth + alias_func = func + alias_alias_func = aliased_func + int_value = int_value + +[file a.py] +valid : list[int] = [1, 2, 3] + + +[out] +# main.pyi +from _typeshed import Incomplete +from a import valid + +def func() -> int: ... +aliased_func = func +int_value: int + +class A: + cls_var = valid + self_var: Incomplete + def __init__(self, arg: str) -> None: ... + def meth(self) -> None: ... + alias_meth = meth + alias_func = func + alias_alias_func = aliased_func + int_value = int_value +# a.pyi +valid: list[int] + +-- More features/fixes: +-- do not export deleted names + +[case testFunctionNoReturnInfersReturnNone] +def f(): + x = 1 +[out] +def f() -> None: ... + +[case testFunctionReturnNoReturnType] +def f(): + return 1 +def g(): + return +[out] +def f(): ... +def g() -> None: ... + +[case testFunctionEllipsisInfersReturnNone] +def f(): ... +[out] +def f() -> None: ... + +[case testFunctionYields] +def f(): + yield 123 +def g(): + x = yield +def h1(): + yield + return +def h2(): + yield + return "abc" +def h3(): + yield + return None +def all(): + x = yield 123 + return "abc" +[out] +from _typeshed import Incomplete +from collections.abc import Generator + +def f() -> Generator[Incomplete]: ... +def g() -> Generator[None, Incomplete]: ... +def h1() -> Generator[None]: ... +def h2() -> Generator[None, None, Incomplete]: ... +def h3() -> Generator[None]: ... +def all() -> Generator[Incomplete, Incomplete, Incomplete]: ... + +[case testFunctionYieldsNone] +def f(): + yield +def g(): + yield None + +[out] +from collections.abc import Generator + +def f() -> Generator[None]: ... +def g() -> Generator[None]: ... + +[case testGeneratorAlreadyDefined] +class Generator: + pass + +def f(): + yield 123 +[out] +from _typeshed import Incomplete +from collections.abc import Generator as _Generator + +class Generator: ... + +def f() -> _Generator[Incomplete]: ... + +[case testGeneratorYieldFrom] +def g1(): + yield from x +def g2(): + y = yield from x +def g3(): + yield from x + return +def g4(): + yield from x + return None +def g5(): + yield from x + return z + +[out] +from _typeshed import Incomplete +from collections.abc import Generator + +def g1() -> Generator[Incomplete, Incomplete]: ... +def g2() -> Generator[Incomplete, Incomplete]: ... +def g3() -> Generator[Incomplete, Incomplete]: ... +def g4() -> Generator[Incomplete, Incomplete]: ... +def g5() -> Generator[Incomplete, Incomplete, Incomplete]: ... + +[case testGeneratorYieldAndYieldFrom] +def g1(): + yield x1 + yield from x2 +def g2(): + yield x1 + y = yield from x2 +def g3(): + y = yield x1 + yield from x2 +def g4(): + yield x1 + yield from x2 + return +def g5(): + yield x1 + yield from x2 + return None +def g6(): + yield x1 + yield from x2 + return z +def g7(): + yield None + yield from x2 + +[out] +from _typeshed import Incomplete +from collections.abc import Generator + +def g1() -> Generator[Incomplete, Incomplete]: ... +def g2() -> Generator[Incomplete, Incomplete]: ... +def g3() -> Generator[Incomplete, Incomplete]: ... +def g4() -> Generator[Incomplete, Incomplete]: ... +def g5() -> Generator[Incomplete, Incomplete]: ... +def g6() -> Generator[Incomplete, Incomplete, Incomplete]: ... +def g7() -> Generator[Incomplete, Incomplete]: ... + +[case testCallable] +from typing import Callable + +x: Callable[[int, int], int] +[out] +from typing import Callable + +x: Callable[[int, int], int] + +[case testAwaitDef] +class F: + async def f(self): + return 1 + +async def g(): + return 2 +[out] +class F: + async def f(self): ... + +async def g(): ... + +[case testCoroutineImportAsyncio] +import asyncio + +class F: + @asyncio.coroutine + def f(self): + return 1 + +@asyncio.coroutine +def g(): + return 2 + +@asyncio.coroutine +def h(): + return 3 +[out] +import asyncio + +class F: + @asyncio.coroutine + def f(self): ... + +@asyncio.coroutine +def g(): ... +@asyncio.coroutine +def h(): ... + +[case testCoroutineImportAsyncioCoroutines] +import asyncio.coroutines + +class F: + @asyncio.coroutines.coroutine + def f(self): + return 1 + +@asyncio.coroutines.coroutine +def g(): + return 2 +[out] +import asyncio.coroutines + +class F: + @asyncio.coroutines.coroutine + def f(self): ... + +@asyncio.coroutines.coroutine +def g(): ... + +[case testCoroutineImportAsyncioCoroutinesSub] +import asyncio + +class F: + @asyncio.coroutines.coroutine + def f(self): + return 1 + +@asyncio.coroutines.coroutine +def g(): + return 2 +[out] +import asyncio + +class F: + @asyncio.coroutines.coroutine + def f(self): ... + +@asyncio.coroutines.coroutine +def g(): ... + +[case testCoroutineImportTypes] +import types + +class F: + @types.coroutine + def f(self): + return 1 + +@types.coroutine +def g(): + return 2 +[out] +import types + +class F: + @types.coroutine + def f(self): ... + +@types.coroutine +def g(): ... + +[case testCoroutineFromAsyncioImportCoroutine] +from asyncio import coroutine + +class F: + @coroutine + def f(self): + return 1 + +@coroutine +def g(): + return 2 +[out] +from asyncio import coroutine + +class F: + @coroutine + def f(self): ... + +@coroutine +def g(): ... + +[case testCoroutineFromAsyncioCoroutinesImportCoroutine] +from asyncio.coroutines import coroutine + +class F: + @coroutine + def f(self): + return 1 + +@coroutine +def g(): + return 2 +[out] +from asyncio.coroutines import coroutine + +class F: + @coroutine + def f(self): ... + +@coroutine +def g(): ... + +[case testCoroutineFromTypesImportCoroutine] +from types import coroutine + +class F: + @coroutine + def f(self): + return 1 + +@coroutine +def g(): + return 2 +[out] +from types import coroutine + +class F: + @coroutine + def f(self): ... + +@coroutine +def g(): ... + +[case testCoroutineFromAsyncioImportCoroutineAsC] +from asyncio import coroutine as c + +class F: + @c + def f(self): + return 1 + +@c +def g(): + return 2 +[out] +from asyncio import coroutine as c + +class F: + @c + def f(self): ... + +@c +def g(): ... + +[case testCoroutineFromAsyncioCoroutinesImportCoroutineAsC] +from asyncio.coroutines import coroutine as c + +class F: + @c + def f(self): + return 1 + +@c +def g(): + return 2 +[out] +from asyncio.coroutines import coroutine as c + +class F: + @c + def f(self): ... + +@c +def g(): ... + +[case testCoroutineFromTypesImportCoroutineAsC] +from types import coroutine as c + +class F: + @c + def f(self): + return 1 + +@c +def g(): + return 2 +[out] +from types import coroutine as c + +class F: + @c + def f(self): ... + +@c +def g(): ... + +[case testCoroutineImportAsyncioAsA] +import asyncio as a + +class F: + @a.coroutine + def f(self): + return 1 + +@a.coroutine +def g(): + return 2 +[out] +import asyncio as a + +class F: + @a.coroutine + def f(self): ... + +@a.coroutine +def g(): ... + +[case testCoroutineImportAsyncioCoroutinesAsC] +import asyncio.coroutines as c + +class F: + @c.coroutine + def f(self): + return 1 + +@c.coroutine +def g(): + return 2 +[out] +import asyncio.coroutines as c + +class F: + @c.coroutine + def f(self): ... + +@c.coroutine +def g(): ... + +[case testCoroutineImportAsyncioCoroutinesSubAsA] +import asyncio as a + +class F: + @a.coroutines.coroutine + def f(self): + return 1 + +@a.coroutines.coroutine +def g(): + return 2 +[out] +import asyncio as a + +class F: + @a.coroutines.coroutine + def f(self): ... + +@a.coroutines.coroutine +def g(): ... + +[case testCoroutineImportTypesAsT] +import types as t + +class F: + @t.coroutine + def f(self): + return 1 + +@t.coroutine +def g(): + return 2 +[out] +import types as t + +class F: + @t.coroutine + def f(self): ... + +@t.coroutine +def g(): ... + + +-- Tests for stub generation from semantically analyzed trees. +-- These tests are much slower, so use the `_semanal` suffix only when needed. + +[case testNestedClass_semanal] +class Outer: + class Inner: + pass + +A = Outer.Inner +[out] +class Outer: + class Inner: ... +A = Outer.Inner + +-- needs improvement +[case testNestedClass_inspect] +class Outer: + class Inner: + pass + +A = Outer.Inner +[out] +class Outer: + class Inner: ... + +class A: ... + +[case testFunctionAlias_semanal] +from asyncio import coroutine + +@coroutine +def start_server(): + ... + +start = start_server +[out] +from asyncio import coroutine + +@coroutine +def start_server() -> None: ... +start = start_server + +[case testModuleAlias_semanal] +import a + +b = a +[file a.py] +x = 1 +[out] +import a + +b = a + +[case testBadAliasNested_semanal] +import a + +x = registry[a.f] +[file a.py] +def f(): ... +[out] +from _typeshed import Incomplete + +x: Incomplete + +[case testCrossModuleClass_semanal] +import a + +class C: + x: A + def f(self) -> A: ... +A = a.A +[file a.py] +class A: ... +[out] +import a + +class C: + x: A + def f(self) -> A: ... +A = a.A + +[case testCrossModuleFunction_semanal] +import a +g = a.f +[file a.py] +def f(): ... +[out] +import a + +g = a.f + +[case testPrivateAliasesExcluded_semanal] +import a, _a + +class C: ... + +A = a._A +B = _a.f +_C = C +[file a.py] +class _A: ... +[file _a.py] +def f(): ... +[out] +from _typeshed import Incomplete + +class C: ... + +A: Incomplete +B: Incomplete + +[case testPrivateAliasesIncluded_semanal] +# flags: --include-private +import a, _a + +class C: ... + +A = a._A +B = _a.f +_C = C +[file a.py] +class _A: ... +[file _a.py] +def f(): ... +[out] +import _a +import a + +class C: ... +A = a._A +B = _a.f +_C = C + +[case testFinalWrapped_semanal] +from typing import Final + +x: Final = 1 +y: Final = x +z: Final[object] +t: Final +[out] +from _typeshed import Incomplete +from typing import Final + +x: Final[int] +y: Final[Incomplete] +z: Final[object] +t: Final[Incomplete] + +[case testFinalInvalid_semanal] +Final = 'boom' + +x: Final = 1 +[out] +Final: str +x: Final + +[case testNoFunctionNested_semanal] +import a +from typing import Dict, Any + +funcs: Dict[Any, Any] +f = funcs[a.f] +[out] +from _typeshed import Incomplete +from typing import Any + +funcs: dict[Any, Any] +f: Incomplete + +[case testAbstractMethodNameExpr] +from abc import ABCMeta, abstractmethod + +class A(metaclass=ABCMeta): + @abstractmethod + def meth(self): + pass +[out] +from abc import ABCMeta, abstractmethod + +class A(metaclass=ABCMeta): + @abstractmethod + def meth(self): ... + +[case testAbstractMethodMemberExpr] +import abc + +class A(metaclass=abc.ABCMeta): + @abc.abstractmethod + def meth(self): + pass +[out] +import abc + +class A(metaclass=abc.ABCMeta): + @abc.abstractmethod + def meth(self): ... + +[case testAbstractMethodMemberExpr2] +import abc as _abc + +class A(metaclass=abc.ABCMeta): + @_abc.abstractmethod + def meth(self): + pass +[out] +import abc as _abc + +class A(metaclass=abc.ABCMeta): + @_abc.abstractmethod + def meth(self): ... + +[case testABCMeta_semanal] +from base import Base +from abc import abstractmethod + +class C(Base): + @abstractmethod + def other(self): + pass + +[file base.py] +from abc import abstractmethod, ABCMeta + +class Base(metaclass=ABCMeta): + @abstractmethod + def meth(self): + pass +[out] +import abc +from abc import abstractmethod +from base import Base + +class C(Base, metaclass=abc.ABCMeta): + @abstractmethod + def other(self): ... + +[case testInvalidNumberOfArgsInAnnotation] +def f(x): + # type: () -> int + return '' + +[out] +def f(x): ... + +[case testFunctionPartiallyAnnotated] +def f(x) -> None: + pass + +def g(x, y: str): + pass + +class A: + def f(self, x) -> None: + pass + +[out] +def f(x) -> None: ... +def g(x, y: str): ... + +class A: + def f(self, x) -> None: ... + +-- Same as above +[case testFunctionPartiallyAnnotated_inspect] +def f(x) -> None: + pass + +def g(x, y: str): + pass + +class A: + def f(self, x) -> None: + pass + +[out] +def f(x) -> None: ... +def g(x, y: str): ... + +class A: + def f(self, x) -> None: ... + +[case testExplicitAnyArg] +from typing import Any + +def f(x: Any): + pass +def g(x, y: Any) -> str: + pass +def h(x: Any) -> str: + pass + +[out] +from typing import Any + +def f(x: Any): ... +def g(x, y: Any) -> str: ... +def h(x: Any) -> str: ... + +-- Same as above +[case testExplicitAnyArg_inspect] +from typing import Any + +def f(x: Any): + pass +def g(x, y: Any) -> str: + pass +def h(x: Any) -> str: + pass + +[out] +from typing import Any + +def f(x: Any): ... +def g(x, y: Any) -> str: ... +def h(x: Any) -> str: ... + +[case testExplicitReturnedAny] +from typing import Any + +def f(x: str) -> Any: + pass +def g(x, y: str) -> Any: + pass +def h(x) -> Any: + pass + +[out] +from typing import Any + +def f(x: str) -> Any: ... +def g(x, y: str) -> Any: ... +def h(x) -> Any: ... + +[case testPlacementOfDecorators] +class A: + @property + def x(self): + self.y = 'y' + return 'x' + +class B: + @property + def x(self): + return 'x' + @x.setter + def x(self, value): + self.y = 'y' + @x.deleter + def x(self): + del self.y + +[out] +class A: + y: str + @property + def x(self): ... + +class B: + @property + def x(self): ... + y: str + @x.setter + def x(self, value) -> None: ... + @x.deleter + def x(self) -> None: ... + +[case testMisplacedTypeComment] +def f(): + x = 0 + + # type: str + y = '' + +[out] +def f() -> None: ... + +[case testConditionalImportAll_semanal] +__all__ = ['cookielib'] + +if object(): + from http import cookiejar as cookielib +else: + import cookielib + +[out] +import cookielib as cookielib + +__all__ = ['cookielib'] + +[case testCannotCalculateMRO_semanal] +class X: pass + +class int(int, X): # Cycle + pass + +class A: pass +class B(A): pass +class C(B): pass +class D(A, B): pass # No consistent method resolution order +class E(C, D): pass # Ditto + +[out] +class X: ... +class int(int, X): ... +class A: ... +class B(A): ... +class C(B): ... +class D(A, B): ... +class E(C, D): ... + +[case testUnreachableCode_semanal] +MYPY = False +class A: pass +if MYPY: + class C(A): + def f(self) -> None: pass +else: + def f(i): + return i + + class C(A): + def g(self) -> None: pass +[out] +MYPY: bool + +class A: ... + +class C(A): + def f(self) -> None: ... + +[case testAbstractPropertyImportAlias] +import abc as abc_alias + +class A: + @abc_alias.abstractproperty + def x(self): pass + +[out] +import abc as abc_alias + +class A: + @property + @abc_alias.abstractmethod + def x(self): ... + +[case testAbstractPropertyFromImportAlias] +from abc import abstractproperty as ap + +class A: + @ap + def x(self): pass + +[out] +import abc + +class A: + @property + @abc.abstractmethod + def x(self): ... + +[case testAbstractProperty1_semanal] +import other +import abc + +class A: + @abc.abstractproperty + def x(self): pass + +[out] +import abc + +class A(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def x(self): ... + +[case testAbstractProperty2_semanal] +import other +from abc import abstractproperty + +class A: + @abstractproperty + def x(self): pass + +[out] +import abc + +class A(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def x(self): ... + +[case testAbstractProperty3_semanal] +import other +from abc import abstractproperty as alias_name + +class A: + @alias_name + def x(self): pass + +[out] +import abc + +class A(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def x(self): ... + +[case testClassWithNameIncompleteOrOptional] +Y = object() + +def g(x=None): pass + +x = g() + +class Incomplete: + pass + +def Optional(): + return 0 + +[out] +from _typeshed import Incomplete as _Incomplete + +Y: _Incomplete + +def g(x: _Incomplete | None = None) -> None: ... + +x: _Incomplete + +class Incomplete: ... + +def Optional(): ... + +[case testExportedNameImported] +# modules: main a b +from a import C + +class D(C): pass + +[file a.py] +from b import C + +[file b.py] +class C: pass + +[out] +# main.pyi +from a import C + +class D(C): ... +# a.pyi +from b import C as C +# b.pyi +class C: ... + +[case testVendoredSix] +from p1.vendored import six +from p1.vendor.six import foobar +from p1.packages.six.moves import http_client +from .packages.six.moves import queue +from p1.vendored.six.moves.http_client import foo +from p1.vendored.six.moves.urllib.parse import bar + +class C(http_client.HTTPMessage): pass +class D(six.Iterator): pass + +[out] +import six +from six import foobar as foobar +from six.moves import http_client, queue as queue +from six.moves.http_client import foo as foo +from six.moves.urllib.parse import bar as bar + +class C(http_client.HTTPMessage): ... +class D(six.Iterator): ... + +[case testVendoredPackage] +# modules: main p.vendored.requests p.sub.requests +from p.vendored.requests import Request +from p.sub.requests import Request2 + +x = Request() +y = Request2() + +[file p/__init__.py] + +[file p/vendored/__init__.py] + +[file p/vendored/requests.py] +class Request: + pass + +[file p/sub/__init__.py] + +[file p/sub/requests.py] +class Request2: + pass + +[out] +# main.pyi +from _typeshed import Incomplete + +x: Incomplete +y: Incomplete + +# p/sub/requests.pyi +class Request2: ... + +[case testTestFiles] +# modules: p p.x p.tests p.tests.test_foo + +[file p/__init__.py] +def f(): pass + +[file p/x.py] +def g(): pass + +[file p/tests/__init__.py] + +[file p/tests/test_foo.py] +def test_thing(): pass + +[out] +# p/__init__.pyi +def f() -> None: ... +# p/x.pyi +def g() -> None: ... + + + +[case testTestFiles_import] +# modules: p p.x p.tests p.tests.test_foo + +[file p/__init__.py] +def f(): pass + +[file p/x.py] +def g(): pass + +[file p/tests/__init__.py] + +[file p/tests/test_foo.py] +def test_thing(): pass + +[out] +# p/__init__.pyi +def f() -> None: ... +# p/x.pyi +def g() -> None: ... + + + +[case testTestFiles_inspect] +# modules: p p.x p.tests p.tests.test_foo + +[file p/__init__.py] +def f(): pass + +[file p/x.py] +def g(): pass + +[file p/tests/__init__.py] + +[file p/tests/test_foo.py] +def test_thing(): pass + +[out] +# p/__init__.pyi +def f(): ... +# p/x.pyi +def g(): ... + + + +[case testVerboseFlag] +# Just test that --verbose does not break anything in a basic test case. +# flags: --verbose + +def f(x, y): pass +[out] +def f(x, y) -> None: ... + +[case testImportedModuleExits_import] +# modules: a b c + +[file a.py] +def g(): pass + +[file b.py] +import sys +def f(): pass +sys.exit(1) + +[file c.py] +x = 0 + +[out] +# a.pyi +def g() -> None: ... +# b.pyi +def f() -> None: ... +# c.pyi +x: int + +[case testImportedModuleHardExits_import] +# modules: a b c + +[file a.py] +def g(): pass + +[file b.py] +import os +def f(): pass +os._exit(1) # Kill process + +[file c.py] +x = 0 + +[out] +# a.pyi +def g() -> None: ... +# b.pyi +def f() -> None: ... +# c.pyi +x: int + +[case testImportedModuleHardExits2_import] +# modules: p/a p/b p/c + +[file p/__init__.py] + +[file p/a.py] +def g(): pass + +[file p/b.py] +import os +def f(): pass +os._exit(1) # Kill process + +[file p/c.py] +x = 0 + +[out] +# p/a.pyi +def g() -> None: ... +# p/b.pyi +def f() -> None: ... +# p/c.pyi +x: int + +[case testImportedModuleHardExits3_import] +# modules: p p/a + +[file p/__init__.py] +import os +def f(): pass +os._exit(1) # Kill process + +[file p/a.py] +def g(): pass + +[out] +# p/__init__.pyi +def f() -> None: ... +# p/a.pyi +def g() -> None: ... + +[case testImportedModuleHardExits4_import] +# flags: -p p +# modules: p p/a + +[file p/__init__.py] +def ff(): pass + +[file p/a.py] +import os +def gg(): pass +os._exit(1) # Kill process + +[out] +# p/__init__.pyi +def ff() -> None: ... +# p/a.pyi +def gg() -> None: ... + +[case testExportInternalImportsByDefault] +# modules: p p/a + +[file p/__init__.py] +from p.a import A, f +from m import C + +a: A +c: C +f() + +[file p/a.py] +class A: pass +def f(): pass + +[file m.py] +class C: pass + +[out] +# p/__init__.pyi +from m import C +from p.a import A as A, f as f + +a: A +c: C +# p/a.pyi +class A: ... + +def f() -> None: ... + +[case testNoExportOfInternalImportsIfAll_import] +# modules: p p/a + +[file p/__init__.py] +from p.a import A + +__all__ = ['a'] + +a = None # type: A +b = 0 # type: int + +[file p/a.py] +class A: pass + +[out] +# p/__init__.pyi +from p.a import A + +__all__ = ['a'] + +a: A +# p/a.pyi +class A: ... + +[case testExportInternalImportsByDefaultFromUnderscorePackage] +# modules: p + +[file p.py] +from _p import A +from _m import B +from _pm import C + +a: A +b: B +c: C + +[file _p.py] +class A: pass + +[file _m.py] +class B: pass + +[file _pm.py] +class C: pass + +[out] +from _m import B +from _p import A as A +from _pm import C + +a: A +b: B +c: C + +[case testDisableExportOfInternalImports] +# flags: --export-less +# modules: p p/a + +[file p/__init__.py] +from p.a import A, B +from m import C + +a: A +c: C + +[file p/a.py] +class A: pass +class B: pass + +[file m.py] +class C: pass + +[out] +# p/__init__.pyi +from m import C +from p.a import A, B as B + +a: A +c: C +# p/a.pyi +class A: ... +class B: ... + +[case testExportInternalImportsByDefaultUsingRelativeImport] +# modules: p.a + +[file p/__init__.py] + +[file p/a.py] +from .b import f +f() + +[file p/b.py] +def f(): pass + +[out] +from .b import f as f + +[case testExportInternalImportsByDefaultSkipPrivate] +# modules: p.a + +[file p/__init__.py] + +[file p/a.py] +from .b import _f, _g as _g, _i +from p.b import _h +_f() +_h() + +[file p/b.py] +def _f(): pass +def _g(): pass +def _h(): pass +def _i(): pass +x = 0 + +[out] + +[case testExportInternalImportsByDefaultIncludePrivate] +# flags: --include-private +# modules: p.a + +[file p/__init__.py] + +[file p/a.py] +from .b import _f +_f() + +[file p/b.py] +def _f(): pass + +[out] +from .b import _f as _f + +[case testHideDunderModuleAttributes] +from m import ( + __about__, + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__ +) + +class A: + __uri__ = 0 + +[file m.py] +__about__ = '' +__author__ = '' +__copyright__ = '' +__email__ = '' +__license__ = '' +__summary__ = '' +__title__ = '' +__uri__ = '' +__version__ = '' + +[out] +from m import __version__ as __version__ + +class A: ... + +[case testHideDunderModuleAttributesWithAll_import] +from m import ( + __about__, + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__ +) + +__all__ = ['__about__', '__author__', '__version__'] + +[file m.py] +__about__ = '' +__author__ = '' +__copyright__ = '' +__email__ = '' +__license__ = '' +__summary__ = '' +__title__ = '' +__uri__ = '' +__version__ = '' + +[out] +from m import __about__ as __about__, __author__ as __author__, __version__ as __version__ + +__all__ = ['__about__', '__author__', '__version__'] + +[case testAttrsClass_semanal] +import attrs + +@attrs.define +class C: + x = attrs.field() + +[out] +from _typeshed import Incomplete + +class C: + x: Incomplete + def __init__(self, x) -> None: ... + def __lt__(self, other): ... + def __le__(self, other): ... + def __gt__(self, other): ... + def __ge__(self, other): ... + +[case testNamedTupleInClass] +from collections import namedtuple + +class C: + N = namedtuple('N', ['x', 'y']) +[out] +from _typeshed import Incomplete +from typing import NamedTuple + +class C: + class N(NamedTuple): + x: Incomplete + y: Incomplete + +[case testImports_directImportsWithAlias] +import p.a as a +import p.b as b + +x: a.X +y: b.Y + +[out] +import p.a as a +import p.b as b + +x: a.X +y: b.Y + +[case testImports_directImportsMixed] +import p.a +import p.a as a +import p.b as b + +x: a.X +y: b.Y +z: p.a.X + +[out] +import p.a +import p.a as a +import p.b as b + +x: a.X +y: b.Y +z: p.a.X + +[case testImport_overwrites_directWithAlias_from] +import p.a as a +from p import a + +x: a.X + +[out] +from p import a + +x: a.X + +[case testImport_overwrites_directWithAlias_fromWithAlias] +import p.a as a +from p import b as a + +x: a.X + +[out] +from p import b as a + +x: a.X + +[case testImports_overwrites_direct_from] +import a +from p import a + +x: a.X + +[out] +from p import a + +x: a.X + +[case testImports_overwrites_direct_fromWithAlias] +import a +from p import b as a + +x: a.X + +[out] +from p import b as a + +x: a.X + +[case testImports_overwrites_from_directWithAlias] +from p import a +import p.a as a + +x: a.X + +[out] +import p.a as a + +x: a.X + +[case testImports_overwrites_fromWithAlias_direct] +import a +from p import b as a + +x: a.X + +[out] +from p import b as a + +x: a.X + +[case testImports_direct] +import p.a +import pp + +x: a.X +y: p.a.Y + +[out] +import p.a + +x: a.X +y: p.a.Y + +[case testNestedImports] +import p +import p.m1 +import p.m2 + +x: p.X +y: p.m1.Y +z: p.m2.Z + +[out] +import p +import p.m1 +import p.m2 + +x: p.X +y: p.m1.Y +z: p.m2.Z + +[case testNestedImportsAliased] +import p as t +import p.m1 as pm1 +import p.m2 as pm2 + +x: t.X +y: pm1.Y +z: pm2.Z + +[out] +import p as t +import p.m1 as pm1 +import p.m2 as pm2 + +x: t.X +y: pm1.Y +z: pm2.Z + +[case testNestedFromImports] +from p import m1 +from p.m1 import sm1 +from p.m2 import sm2 + +x: m1.X +y: sm1.Y +z: sm2.Z + +[out] +from p import m1 +from p.m1 import sm1 +from p.m2 import sm2 + +x: m1.X +y: sm1.Y +z: sm2.Z + +[case testOverload_fromTypingImport] +from typing import Tuple, Union, overload + +class A: + @overload + def f(self, x: int, y: int) -> int: + ... + + @overload + def f(self, x: Tuple[int, int]) -> int: + ... + + def f(self, *args: Union[int, Tuple[int, int]]) -> int: + pass + +@overload +def f(x: int, y: int) -> int: + ... + +@overload +def f(x: Tuple[int, int]) -> int: + ... + +def f(*args: Union[int, Tuple[int, int]]) -> int: + pass + + +[out] +from typing import overload + +class A: + @overload + def f(self, x: int, y: int) -> int: ... + @overload + def f(self, x: tuple[int, int]) -> int: ... + +@overload +def f(x: int, y: int) -> int: ... +@overload +def f(x: tuple[int, int]) -> int: ... + +[case testOverload_fromTypingExtensionsImport] +from typing import Tuple, Union +from typing_extensions import overload + +class A: + @overload + def f(self, x: int, y: int) -> int: + ... + + @overload + def f(self, x: Tuple[int, int]) -> int: + ... + + def f(self, *args: Union[int, Tuple[int, int]]) -> int: + pass + +@overload +def f(x: int, y: int) -> int: + ... + +@overload +def f(x: Tuple[int, int]) -> int: + ... + +def f(*args: Union[int, Tuple[int, int]]) -> int: + pass + + +[out] +from typing_extensions import overload + +class A: + @overload + def f(self, x: int, y: int) -> int: ... + @overload + def f(self, x: tuple[int, int]) -> int: ... + +@overload +def f(x: int, y: int) -> int: ... +@overload +def f(x: tuple[int, int]) -> int: ... + +[case testOverload_importTyping] +import typing +import typing_extensions + +class A: + @typing.overload + def f(self, x: int, y: int) -> int: + ... + + @typing.overload + def f(self, x: typing.Tuple[int, int]) -> int: + ... + + def f(self, *args: typing.Union[int, typing.Tuple[int, int]]) -> int: + pass + + @typing.overload + @classmethod + def g(cls, x: int, y: int) -> int: + ... + + @typing.overload + @classmethod + def g(cls, x: typing.Tuple[int, int]) -> int: + ... + + @classmethod + def g(self, *args: typing.Union[int, typing.Tuple[int, int]]) -> int: + pass + +@typing.overload +def f(x: int, y: int) -> int: + ... + +@typing.overload +def f(x: typing.Tuple[int, int]) -> int: + ... + +def f(*args: typing.Union[int, typing.Tuple[int, int]]) -> int: + pass + +@typing_extensions.overload +def g(x: int, y: int) -> int: + ... + +@typing_extensions.overload +def g(x: typing.Tuple[int, int]) -> int: + ... + +def g(*args: typing.Union[int, typing.Tuple[int, int]]) -> int: + pass + + +[out] +import typing +import typing_extensions + +class A: + @typing.overload + def f(self, x: int, y: int) -> int: ... + @typing.overload + def f(self, x: tuple[int, int]) -> int: ... + @typing.overload + @classmethod + def g(cls, x: int, y: int) -> int: ... + @typing.overload + @classmethod + def g(cls, x: tuple[int, int]) -> int: ... + +@typing.overload +def f(x: int, y: int) -> int: ... +@typing.overload +def f(x: tuple[int, int]) -> int: ... +@typing_extensions.overload +def g(x: int, y: int) -> int: ... +@typing_extensions.overload +def g(x: tuple[int, int]) -> int: ... + +[case testOverload_importTypingAs] +import typing as t +import typing_extensions as te + +class A: + @t.overload + def f(self, x: int, y: int) -> int: + ... + + @t.overload + def f(self, x: t.Tuple[int, int]) -> int: + ... + + def f(self, *args: typing.Union[int, t.Tuple[int, int]]) -> int: + pass + + @t.overload + @classmethod + def g(cls, x: int, y: int) -> int: + ... + + @t.overload + @classmethod + def g(cls, x: t.Tuple[int, int]) -> int: + ... + + @classmethod + def g(self, *args: t.Union[int, t.Tuple[int, int]]) -> int: + pass + +@t.overload +def f(x: int, y: int) -> int: + ... + +@t.overload +def f(x: t.Tuple[int, int]) -> int: + ... + +def f(*args: t.Union[int, t.Tuple[int, int]]) -> int: + pass + + +@te.overload +def g(x: int, y: int) -> int: + ... + +@te.overload +def g(x: t.Tuple[int, int]) -> int: + ... + +def g(*args: t.Union[int, t.Tuple[int, int]]) -> int: + pass + +[out] +import typing as t +import typing_extensions as te + +class A: + @t.overload + def f(self, x: int, y: int) -> int: ... + @t.overload + def f(self, x: tuple[int, int]) -> int: ... + @t.overload + @classmethod + def g(cls, x: int, y: int) -> int: ... + @t.overload + @classmethod + def g(cls, x: tuple[int, int]) -> int: ... + +@t.overload +def f(x: int, y: int) -> int: ... +@t.overload +def f(x: tuple[int, int]) -> int: ... +@te.overload +def g(x: int, y: int) -> int: ... +@te.overload +def g(x: tuple[int, int]) -> int: ... + +[case testOverloadFromImportAlias] +from typing import overload as t_overload +from typing_extensions import overload as te_overload + +@t_overload +def f(x: int, y: int) -> int: + ... + +@te_overload +def g(x: int, y: int) -> int: + ... + +[out] +from typing import overload as t_overload +from typing_extensions import overload as te_overload + +@t_overload +def f(x: int, y: int) -> int: ... +@te_overload +def g(x: int, y: int) -> int: ... + +[case testProtocol_semanal] +from typing import Protocol, TypeVar + +class P(Protocol): + def f(self, x: int, y: int) -> str: + ... + +T = TypeVar('T') +T2 = TypeVar('T2') +class PT(Protocol[T, T2]): + def f(self, x: T) -> T2: + ... + +[out] +from typing import Protocol, TypeVar + +class P(Protocol): + def f(self, x: int, y: int) -> str: ... +T = TypeVar('T') +T2 = TypeVar('T2') + +class PT(Protocol[T, T2]): + def f(self, x: T) -> T2: ... + +[case testProtocolAbstractMethod_semanal] +from abc import abstractmethod +from typing import Protocol + +class P(Protocol): + @abstractmethod + def f(self, x: int, y: int) -> str: + ... + +[out] +from abc import abstractmethod +from typing import Protocol + +class P(Protocol): + @abstractmethod + def f(self, x: int, y: int) -> str: ... + +[case testNonDefaultKeywordOnlyArgAfterAsterisk] +def func(*, non_default_kwarg: bool, default_kwarg: bool = True): ... +[out] +def func(*, non_default_kwarg: bool, default_kwarg: bool = True): ... + +[case testNestedGenerator] +def f1(): + def g(): + yield 0 + return 0 +def f2(): + def g(): + yield from [0] + return 0 +[out] +def f1(): ... +def f2(): ... + +[case testIncludeDocstrings] +# flags: --include-docstrings +class A: + """class docstring + + a multiline 😊 docstring""" + def func(): + """func docstring + don't forget to indent""" + ... + def nodoc(): + ... +class B: + def quoteA(): + '''func docstring with quotes"""\\n + and an end quote\'''' + ... + def quoteB(): + '''func docstring with quotes""" + \'\'\' + and an end quote\\"''' + ... + def quoteC(): + """func docstring with end quote\\\"""" + ... + def quoteD(): + r'''raw with quotes\"''' + ... +[out] +class A: + """class docstring + + a multiline 😊 docstring""" + def func() -> None: + """func docstring + don't forget to indent""" + def nodoc() -> None: ... + +class B: + def quoteA() -> None: + '''func docstring with quotes"""\\n + and an end quote\'''' + def quoteB() -> None: + '''func docstring with quotes""" + \'\'\' + and an end quote\\"''' + def quoteC() -> None: + '''func docstring with end quote\\"''' + def quoteD() -> None: + '''raw with quotes\\"''' + +[case testIgnoreDocstrings] +class A: + """class docstring + + a multiline docstring""" + def func(): + """func docstring + + don't forget to indent""" + def nodoc(): + ... + +class B: + def func(): + """func docstring""" + ... + def nodoc(): + ... + +[out] +class A: + def func() -> None: ... + def nodoc() -> None: ... + +class B: + def func() -> None: ... + def nodoc() -> None: ... + +[case testKnownMagicMethodsReturnTypes] +class Some: + def __len__(self): ... + def __length_hint__(self): ... + def __init__(self): ... + def __del__(self): ... + def __bool__(self): ... + def __bytes__(self): ... + def __format__(self, spec): ... + def __contains__(self, obj): ... + def __complex__(self): ... + def __int__(self): ... + def __float__(self): ... + def __index__(self): ... +[out] +class Some: + def __len__(self) -> int: ... + def __length_hint__(self) -> int: ... + def __init__(self) -> None: ... + def __del__(self) -> None: ... + def __bool__(self) -> bool: ... + def __bytes__(self) -> bytes: ... + def __format__(self, spec) -> str: ... + def __contains__(self, obj) -> bool: ... + def __complex__(self) -> complex: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __index__(self) -> int: ... + +-- Same as above +[case testKnownMagicMethodsReturnTypes_inspect] +class Some: + def __len__(self): ... + def __length_hint__(self): ... + def __init__(self): ... + def __del__(self): ... + def __bool__(self): ... + def __bytes__(self): ... + def __format__(self, spec): ... + def __contains__(self, obj): ... + def __complex__(self): ... + def __int__(self): ... + def __float__(self): ... + def __index__(self): ... +[out] +class Some: + def __len__(self) -> int: ... + def __length_hint__(self) -> int: ... + def __init__(self) -> None: ... + def __del__(self) -> None: ... + def __bool__(self) -> bool: ... + def __bytes__(self) -> bytes: ... + def __format__(self, spec) -> str: ... + def __contains__(self, obj) -> bool: ... + def __complex__(self) -> complex: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __index__(self) -> int: ... + + +[case testKnownMagicMethodsArgTypes] +class MismatchNames: + def __exit__(self, tp, val, tb): ... + +class MatchNames: + def __exit__(self, type, value, traceback): ... + +[out] +class MismatchNames: + def __exit__(self, tp: type[BaseException] | None, val: BaseException | None, tb: types.TracebackType | None) -> None: ... + +class MatchNames: + def __exit__(self, type: type[BaseException] | None, value: BaseException | None, traceback: types.TracebackType | None) -> None: ... + +-- Same as above (but can generate import statements) +[case testKnownMagicMethodsArgTypes_inspect] +class MismatchNames: + def __exit__(self, tp, val, tb): ... + +class MatchNames: + def __exit__(self, type, value, traceback): ... + +[out] +import types + +class MismatchNames: + def __exit__(self, tp: type[BaseException] | None, val: BaseException | None, tb: types.TracebackType | None): ... + +class MatchNames: + def __exit__(self, type: type[BaseException] | None, value: BaseException | None, traceback: types.TracebackType | None): ... + +[case testTypeVarPEP604Bound] +from typing import TypeVar +T = TypeVar("T", bound=str | None) +[out] +from typing import TypeVar + +T = TypeVar('T', bound=str | None) + + +[case testPEP604UnionType] +a: str | int + +def f(x: str | None) -> None: ... +[out] +a: str | int + +def f(x: str | None) -> None: ... + +[case testTypeddict] +import typing, x +X = typing.TypedDict('X', {'a': int, 'b': str}) +Y = typing.TypedDict('X', {'a': int, 'b': str}, total=False) +[out] +from typing_extensions import TypedDict + +class X(TypedDict): + a: int + b: str + +class Y(TypedDict, total=False): + a: int + b: str + +[case testTypeddictClassWithKeyword] +from typing import TypedDict +class MyDict(TypedDict, total=False): + foo: str + bar: int +[out] +from typing import TypedDict + +class MyDict(TypedDict, total=False): + foo: str + bar: int + +[case testTypeddictKeywordSyntax] +from typing import TypedDict + +X = TypedDict('X', a=int, b=str) +Y = TypedDict('X', a=int, b=str, total=False) +[out] +from typing_extensions import TypedDict + +class X(TypedDict): + a: int + b: str + +class Y(TypedDict, total=False): + a: int + b: str + +[case testTypeddictWithNonIdentifierOrKeywordKeys] +from typing import TypedDict +X = TypedDict('X', {'a-b': int, 'c': str}) +Y = TypedDict('X', {'a-b': int, 'c': str}, total=False) +Z = TypedDict('X', {'a': int, 'in': str}) +[out] +from typing import TypedDict + +X = TypedDict('X', {'a-b': int, 'c': str}) +Y = TypedDict('X', {'a-b': int, 'c': str}, total=False) +Z = TypedDict('X', {'a': int, 'in': str}) + +[case testEmptyTypeddict] +import typing +X = typing.TypedDict('X', {}) +Y = typing.TypedDict('Y', {}, total=False) +Z = typing.TypedDict('Z') +W = typing.TypedDict('W', total=False) +[out] +from typing_extensions import TypedDict + +class X(TypedDict): ... +class Y(TypedDict, total=False): ... +class Z(TypedDict): ... +class W(TypedDict, total=False): ... + +[case testTypeddictAliased] +from typing import TypedDict as t_TypedDict +from typing_extensions import TypedDict as te_TypedDict +def f(): ... +X = t_TypedDict('X', {'a': int, 'b': str}) +Y = te_TypedDict('Y', {'a': int, 'b': str}) +def g(): ... +[out] +from typing_extensions import TypedDict + +def f() -> None: ... + +class X(TypedDict): + a: int + b: str + +class Y(TypedDict): + a: int + b: str + +def g() -> None: ... + +[case testTypeddictFromImportAlias] +import typing as t +import typing_extensions as te +X = t.TypedDict('X', {'a': int, 'b': str}) +Y = te.TypedDict('Y', {'a': int, 'b': str}) +[out] +from typing_extensions import TypedDict + +class X(TypedDict): + a: int + b: str + +class Y(TypedDict): + a: int + b: str + +[case testNotTypeddict] +from x import TypedDict +import y +X = TypedDict('X', {'a': int, 'b': str}) +Y = y.TypedDict('Y', {'a': int, 'b': str}) +[out] +from _typeshed import Incomplete + +X: Incomplete +Y: Incomplete + +[case testTypeddictWithWrongAttributesType] +from typing import TypedDict +R = TypedDict("R", {"a": int, **{"b": str, "c": bytes}}) +S = TypedDict("S", [("b", str), ("c", bytes)]) +T = TypedDict("T", {"a": int}, b=str, total=False) +U = TypedDict("U", {"a": int}, totale=False) +V = TypedDict("V", {"a": int}, {"b": str}) +W = TypedDict("W", **{"a": int, "b": str}) +[out] +from _typeshed import Incomplete + +R: Incomplete +S: Incomplete +T: Incomplete +U: Incomplete +V: Incomplete +W: Incomplete + +[case testUseTypingName] +import collections +import typing +from typing import NamedTuple, TypedDict + +class Incomplete: ... +class Generator: ... +class NamedTuple: ... +class TypedDict: ... + +nt = collections.namedtuple("nt", "a b") +NT = typing.NamedTuple("NT", [("a", int), ("b", str)]) +NT1 = typing.NamedTuple("NT1", [("a", int)] + [("b", str)]) +NT2 = typing.NamedTuple("NT2", [(xx, int), ("b", str)]) +NT3 = typing.NamedTuple(xx, [("a", int), ("b", str)]) +TD = typing.TypedDict("TD", {"a": int, "b": str}) +TD1 = typing.TypedDict("TD1", {"a": int, "b": str}, totale=False) +TD2 = typing.TypedDict("TD2", {xx: int, "b": str}) +TD3 = typing.TypedDict(xx, {"a": int, "b": str}) + +def gen(): + y = yield x + return z + +def gen2(): + y = yield from x + return z + +class X(unknown_call("X", "a b")): ... +class Y(collections.namedtuple("Y", xx)): ... +[out] +from _typeshed import Incomplete as _Incomplete +from collections.abc import Generator as _Generator +from typing import NamedTuple as _NamedTuple +from typing_extensions import TypedDict as _TypedDict + +class Incomplete: ... +class Generator: ... +class NamedTuple: ... +class TypedDict: ... + +class nt(_NamedTuple): + a: _Incomplete + b: _Incomplete + +class NT(_NamedTuple): + a: int + b: str + +NT1: _Incomplete +NT2: _Incomplete +NT3: _Incomplete + +class TD(_TypedDict): + a: int + b: str + +TD1: _Incomplete +TD2: _Incomplete +TD3: _Incomplete + +def gen() -> _Generator[_Incomplete, _Incomplete, _Incomplete]: ... +def gen2() -> _Generator[_Incomplete, _Incomplete, _Incomplete]: ... + +class X(_Incomplete): ... +class Y(_Incomplete): ... + +[case testIgnoreLongDefaults] +def f(x='abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\ +abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\ +abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\ +abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz'): ... + +def g(x=b'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\ +abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\ +abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\ +abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz'): ... + +def h(x=123456789012345678901234567890123456789012345678901234567890\ +123456789012345678901234567890123456789012345678901234567890\ +123456789012345678901234567890123456789012345678901234567890\ +123456789012345678901234567890123456789012345678901234567890): ... + +[out] +def f(x: str = ...) -> None: ... +def g(x: bytes = ...) -> None: ... +def h(x: int = ...) -> None: ... + +[case testDefaultsOfBuiltinContainers] +def f(x=(), y=(1,), z=(1, 2)): ... +def g(x=[], y=[1, 2]): ... +def h(x={}, y={1: 2, 3: 4}): ... +def i(x={1, 2, 3}): ... +def j(x=[(1,"a"), (2,"b")]): ... + +[out] +def f(x=(), y=(1,), z=(1, 2)) -> None: ... +def g(x=[], y=[1, 2]) -> None: ... +def h(x={}, y={1: 2, 3: 4}) -> None: ... +def i(x={1, 2, 3}) -> None: ... +def j(x=[(1, 'a'), (2, 'b')]) -> None: ... + +[case testDefaultsOfBuiltinContainersWithNonTrivialContent] +def f(x=(1, u.v), y=(k(),), z=(w,)): ... +def g(x=[1, u.v], y=[k()], z=[w]): ... +def h(x={1: u.v}, y={k(): 2}, z={m: m}, w={**n}): ... +def i(x={u.v, 2}, y={3, k()}, z={w}): ... + +[out] +def f(x=..., y=..., z=...) -> None: ... +def g(x=..., y=..., z=...) -> None: ... +def h(x=..., y=..., z=..., w=...) -> None: ... +def i(x=..., y=..., z=...) -> None: ... + +[case testDataclass] +import dataclasses +import dataclasses as dcs +from dataclasses import dataclass, InitVar, KW_ONLY +from dataclasses import dataclass as dc +from typing import ClassVar + +@dataclasses.dataclass +class X: + a: int + b: str = "hello" + c: ClassVar + d: ClassVar = 200 + f: list[int] = field(init=False, default_factory=list) + g: int = field(default=2, kw_only=True) + _: KW_ONLY + h: int = 1 + i: InitVar[str] + j: InitVar = 100 + non_field = None + +@dcs.dataclass +class Y: ... + +@dataclass +class Z: ... + +@dc +class W: ... + +@dataclass(init=False, repr=False) +class V: ... + +[out] +import dataclasses +import dataclasses as dcs +from dataclasses import InitVar, KW_ONLY, dataclass, dataclass as dc +from typing import ClassVar + +@dataclasses.dataclass +class X: + a: int + b: str = ... + c: ClassVar + d: ClassVar = ... + f: list[int] = ... + g: int = ... + _: KW_ONLY + h: int = ... + i: InitVar[str] + j: InitVar = ... + non_field = ... + +@dcs.dataclass +class Y: ... +@dataclass +class Z: ... +@dc +class W: ... +@dataclass(init=False, repr=False) +class V: ... + +[case testDataclass_semanal] +from dataclasses import InitVar, dataclass, field +from typing import ClassVar + +@dataclass +class X: + a: int + b: InitVar[str] + c: str = "hello" + d: ClassVar + e: ClassVar = 200 + f: list[int] = field(init=False, default_factory=list) + g: int = field(default=2, kw_only=True) + h: int = 1 + i: InitVar = 100 + j: list[int] = field(default_factory=list) + non_field = None + +@dataclass(init=False, repr=False, frozen=True) +class Y: ... + +[out] +from dataclasses import InitVar, dataclass +from typing import ClassVar + +@dataclass +class X: + a: int + b: InitVar[str] + c: str = ... + d: ClassVar + e: ClassVar = ... + f: list[int] = ... + g: int = ... + h: int = ... + i: InitVar = ... + j: list[int] = ... + non_field = ... + def __init__(self, a, b, c=..., *, g=..., h=..., i=..., j=...) -> None: ... + +@dataclass(init=False, repr=False, frozen=True) +class Y: ... + +[case testDataclassWithKwOnlyField_semanal] +# flags: --python-version=3.10 +from dataclasses import dataclass, field, InitVar, KW_ONLY +from typing import ClassVar + +@dataclass +class X: + a: int + b: str = "hello" + c: ClassVar + d: ClassVar = 200 + f: list[int] = field(init=False, default_factory=list) + g: int = field(default=2, kw_only=True) + _: KW_ONLY + h: int = 1 + i: InitVar[str] + j: InitVar = 100 + non_field = None + +@dataclass(init=False, repr=False, frozen=True) +class Y: ... + +[out] +from dataclasses import InitVar, KW_ONLY, dataclass +from typing import ClassVar + +@dataclass +class X: + a: int + b: str = ... + c: ClassVar + d: ClassVar = ... + f: list[int] = ... + g: int = ... + _: KW_ONLY + h: int = ... + i: InitVar[str] + j: InitVar = ... + non_field = ... + def __init__(self, a, b=..., *, g=..., h=..., i, j=...) -> None: ... + +@dataclass(init=False, repr=False, frozen=True) +class Y: ... + +[case testDataclassWithExplicitGeneratedMethodsOverrides_semanal] +from dataclasses import dataclass + +@dataclass +class X: + a: int + def __init__(self, a: int, b: str = ...) -> None: ... + def __post_init__(self) -> None: ... + +[out] +from dataclasses import dataclass + +@dataclass +class X: + a: int + def __init__(self, a: int, b: str = ...) -> None: ... + def __post_init__(self) -> None: ... + +[case testDataclassInheritsFromAny_semanal] +from dataclasses import dataclass +import missing + +@dataclass +class X(missing.Base): + a: int + +@dataclass +class Y(missing.Base): + generated_args: str + generated_args_: str + generated_kwargs: float + generated_kwargs_: float + +[out] +import missing +from dataclasses import dataclass + +@dataclass +class X(missing.Base): + a: int + def __init__(self, *generated_args, a, **generated_kwargs) -> None: ... + +@dataclass +class Y(missing.Base): + generated_args: str + generated_args_: str + generated_kwargs: float + generated_kwargs_: float + def __init__(self, *generated_args__, generated_args, generated_args_, generated_kwargs, generated_kwargs_, **generated_kwargs__) -> None: ... + +[case testAlwaysUsePEP604Union] +import typing +import typing as t +from typing import Optional, Union, Optional as O, Union as U +import x + +union = Union[int, str] +bad_union = Union[int] +nested_union = Optional[Union[int, str]] +not_union = x.Union[int, str] +u = U[int, str] +o = O[int] + +def f1(a: Union["int", Optional[tuple[int, t.Optional[int]]]]) -> int: ... +def f2(a: typing.Union[int | x.Union[int, int], O[float]]) -> int: ... + +[out] +import x +from _typeshed import Incomplete + +union = int | str +bad_union = int +nested_union = int | str | None +not_union: Incomplete +u = int | str +o = int | None + +def f1(a: int | tuple[int, int | None] | None) -> int: ... +def f2(a: int | x.Union[int, int] | float | None) -> int: ... + +[case testTypingBuiltinReplacements] +import typing +import typing as t +from typing import Tuple +import typing_extensions +import typing_extensions as te +from typing_extensions import List, Type + +# builtins are not builtins +tuple = int +[list,] = float +dict, set, frozenset = str, float, int + +x: Tuple[t.Text, t.FrozenSet[typing.Type[float]]] +y: typing.List[int] +z: t.Dict[str, float] +v: typing.Set[int] +w: List[typing_extensions.Dict[te.FrozenSet[Type[int]], te.Tuple[te.Set[te.Text], ...]]] + +x_alias = Tuple[str, ...] +y_alias = typing.List[int] +z_alias = t.Dict[str, float] +v_alias = typing.Set[int] +w_alias = List[typing_extensions.Dict[str, te.Tuple[int, ...]]] + +[out] +from _typeshed import Incomplete +from builtins import dict as _dict, frozenset as _frozenset, list as _list, set as _set, tuple as _tuple + +tuple = int +list: Incomplete +dict: Incomplete +set: Incomplete +frozenset: Incomplete +x: _tuple[str, _frozenset[type[float]]] +y: _list[int] +z: _dict[str, float] +v: _set[int] +w: _list[_dict[_frozenset[type[int]], _tuple[_set[str], ...]]] +x_alias = _tuple[str, ...] +y_alias = _list[int] +z_alias = _dict[str, float] +v_alias = _set[int] +w_alias = _list[_dict[str, _tuple[int, ...]]] + +[case testHandlingNameCollisions] +# flags: --include-private +from typing import Tuple +tuple = int +_tuple = range +__tuple = map +x: Tuple[int, str] +[out] +from builtins import tuple as ___tuple + +tuple = int +_tuple = range +__tuple = map +x: ___tuple[int, str] + +[case testPEP570PosOnlyParams] +def f(x=0, /): ... +def f1(x: int, /): ... +def f2(x: int, y: float = 1, /): ... +def f3(x: int, /, y: float): ... +def f4(x: int, /, y: float = 1): ... +def f5(x: int, /, *, y: float): ... +def f6(x: int = 0, /, *, y: float): ... +def f7(x: int, /, *, y: float = 1): ... +def f8(x: int = 0, /, *, y: float = 1): ... + +[out] +def f(x: int = 0, /) -> None: ... +def f1(x: int, /): ... +def f2(x: int, y: float = 1, /): ... +def f3(x: int, /, y: float): ... +def f4(x: int, /, y: float = 1): ... +def f5(x: int, /, *, y: float): ... +def f6(x: int = 0, /, *, y: float): ... +def f7(x: int, /, *, y: float = 1): ... +def f8(x: int = 0, /, *, y: float = 1): ... + +[case testPreserveEmptyTuple] +ann: tuple[()] +alias = tuple[()] +def f(x: tuple[()]): ... +class C(tuple[()]): ... + +[out] +ann: tuple[()] +alias = tuple[()] + +def f(x: tuple[()]): ... + +class C(tuple[()]): ... + +[case testPreserveEnumValue_semanal] +from enum import Enum + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +class Bar(Enum): + A = object() + B = "a" + "b" + +[out] +from enum import Enum + +class Foo(Enum): + A = 1 + B = 2 + C = 3 + +class Bar(Enum): + A = ... + B = ... + +[case testGracefullyHandleInvalidOptionalUsage] +from typing import Optional + +x: Optional # invalid +y: Optional[int] # valid +z: Optional[int, str] # invalid +w: Optional[int | str] # valid +r: Optional[type[int | str]] + +X = Optional +Y = Optional[int] +Z = Optional[int, str] +W = Optional[int | str] +R = Optional[type[int | str]] + +[out] +from _typeshed import Incomplete +from typing import Optional + +x: Incomplete +y: int | None +z: Incomplete +w: int | str | None +r: type[int | str] | None +X = Optional +Y = int | None +Z = Incomplete +W = int | str | None +R = type[int | str] | None + +[case testClassInheritanceWithKeywordsConstants] +class Test(Whatever, a=1, b='b', c=True, d=1.5, e=None, f=1j, g=b'123'): ... +[out] +class Test(Whatever, a=1, b='b', c=True, d=1.5, e=None, f=1j, g=b'123'): ... + +[case testClassInheritanceWithKeywordsDynamic] +class Test(Whatever, keyword=SomeName * 2, attr=SomeName.attr): ... +[out] +class Test(Whatever, keyword=SomeName * 2, attr=SomeName.attr): ... + +[case testPEP695GenericClass] +# flags: --python-version=3.12 + +class C[T]: ... +class C1[T1](int): ... +class C2[T2: int]: ... +class C3[T3: str | bytes]: ... +class C4[T4: (str, bytes)]: ... + +class Outer: + class Inner[T]: ... + +[out] +class C[T]: ... +class C1[T1](int): ... +class C2[T2: int]: ... +class C3[T3: str | bytes]: ... +class C4[T4: (str, bytes)]: ... + +class Outer: + class Inner[T]: ... + +[case testPEP695GenericFunction] +# flags: --python-version=3.12 + +def f1[T1](): ... +def f2[T2: int](): ... +def f3[T3: str | bytes](): ... +def f4[T4: (str, bytes)](): ... + +class C: + def m[T](self, x: T) -> T: ... + +[out] +def f1[T1]() -> None: ... +def f2[T2: int]() -> None: ... +def f3[T3: str | bytes]() -> None: ... +def f4[T4: (str, bytes)]() -> None: ... + +class C: + def m[T](self, x: T) -> T: ... + +[case testPEP695TypeAlias] +# flags: --python-version=3.12 + +type Alias = int | str +type Alias1[T1] = list[T1] | set[T1] +type Alias2[T2: int] = list[T2] | set[T2] +type Alias3[T3: str | bytes] = list[T3] | set[T3] +type Alias4[T4: (str, bytes)] = list[T4] | set[T4] + +class C: + type IndentedAlias[T] = list[T] + +[out] +type Alias = int | str +type Alias1[T1] = list[T1] | set[T1] +type Alias2[T2: int] = list[T2] | set[T2] +type Alias3[T3: str | bytes] = list[T3] | set[T3] +type Alias4[T4: (str, bytes)] = list[T4] | set[T4] +class C: + type IndentedAlias[T] = list[T] + +[case testPEP695Syntax_semanal] +# flags: --python-version=3.12 + +class C[T]: ... +def f[S](): ... +type A[R] = list[R] + +[out] +class C[T]: ... + +def f[S]() -> None: ... +type A[R] = list[R] + +[case testPEP696Syntax] +# flags: --python-version=3.13 + +type Alias1[T1 = int] = list[T1] | set[T1] +type Alias2[T2: int | float = int] = list[T2] | set[T2] +class C3[T3 = int]: ... +class C4[T4: int | float = int](list[T4]): ... +def f5[T5 = int](): ... + +[out] +type Alias1[T1 = int] = list[T1] | set[T1] +type Alias2[T2: int | float = int] = list[T2] | set[T2] +class C3[T3 = int]: ... +class C4[T4: int | float = int](list[T4]): ... + +def f5[T5 = int]() -> None: ... + +[case testIgnoreMypyGeneratedMethods_semanal] +# flags: --include-private --python-version=3.13 +from typing_extensions import dataclass_transform + +# TODO: preserve dataclass_transform decorator +@dataclass_transform() +class DCMeta(type): ... +class DC(metaclass=DCMeta): + x: str + +[out] +class DCMeta(type): ... + +class DC(metaclass=DCMeta): + x: str + def __init__(self, x) -> None: ... + def __replace__(self, *, x) -> None: ... diff --git a/test-data/unit/typexport-basic.test b/test-data/unit/typexport-basic.test new file mode 100644 index 000000000000..d78cf0f179f2 --- /dev/null +++ b/test-data/unit/typexport-basic.test @@ -0,0 +1,1279 @@ +-- Test cases for exporting node types from the type checker. +-- +-- Each test case consists of at least two sections. +-- The first section contains [case NAME] followed by the input code, +-- while the second section contains [out] followed by the output from the type +-- checker. +-- +-- The first line of input code should be a regexp in comment that describes +-- the information to dump (prefix with ##). The regexp is matched against +-- the following items: +-- +-- * each name of an expression node +-- * each type string of a node (e.g. OpExpr) +-- +-- Lines starting with "--" in this file will be ignored. + + +-- Expressions +-- ----------- + + +[case testConstructorCall] +import typing +class A: pass +class B: pass +A() +B() +[out] +CallExpr(4) : A +NameExpr(4) : def () -> A +CallExpr(5) : B +NameExpr(5) : def () -> B + +[case testLiterals] +import typing +5 +2.3 +'foo' +[builtins fixtures/primitives.pyi] +[out] +IntExpr(2) : Literal[5]? +FloatExpr(3) : builtins.float +StrExpr(4) : Literal['foo']? + +[case testNameExpression] + +a = None # type: A +a # node +def f(aa: 'A') -> None: + b = None # type: B + aa # node + b # node +class A: + def g(self) -> None: + self # node +class B: pass +[out] +NameExpr(3) : A +NameExpr(6) : A +NameExpr(7) : B +NameExpr(10) : A + +[case testEllipsis] +import typing +... +[out] +EllipsisExpr(2) : builtins.ellipsis + +[case testMemberAccess] +## MemberExpr|CallExpr + +a = None # type: A +a.m +a.f +a.f() +class A: + m = None # type: A + def f(self) -> 'B': pass +class B: pass +[out] +MemberExpr(4) : A +MemberExpr(5) : def () -> B +CallExpr(6) : B +MemberExpr(6) : def () -> B + +[case testCastExpression] +## CastExpr|[a-z] +from typing import Any, cast +d = None # type: Any +b = None # type: B +class A: pass +class B(A): pass +cast(A, d) +cast(A, b) +cast(B, b) +[out] +CastExpr(7) : A +NameExpr(7) : Any +CastExpr(8) : A +NameExpr(8) : B +CastExpr(9) : B +NameExpr(9) : B + +[case testAssertTypeExpr] +## AssertTypeExpr|[a-z] +from typing import Any, assert_type +d = None # type: Any +a = None # type: A +b = None # type: B +class A: pass +class B(A): pass +assert_type(d, Any) +assert_type(a, A) +assert_type(b, B) +[out] +AssertTypeExpr(8) : Any +NameExpr(8) : Any +AssertTypeExpr(9) : A +NameExpr(9) : A +AssertTypeExpr(10) : B +NameExpr(10) : B + +[case testArithmeticOps] +## OpExpr +import typing +a = 1 + 2 +1.2 * 3 +2.2 - 3 +1 / 2 +[file builtins.py] +class object: + def __init__(self) -> None: pass +class function: pass +class int: + def __add__(self, x: int) -> int: pass + def __truediv__(self, x: int) -> float: pass +class float: + def __mul__(self, x: int) -> float: pass + def __sub__(self, x: int) -> float: pass +class type: pass +class str: pass +class list: pass +class dict: pass +[out] +OpExpr(3) : builtins.int +OpExpr(4) : builtins.float +OpExpr(5) : builtins.float +OpExpr(6) : builtins.float + +[case testComparisonOps] +## ComparisonExpr +import typing +1 == object() +1 == 2 +2 < 3 +1 < 2 < 3 +8 > 3 +4 < 6 > 2 +[file builtins.py] +class object: + def __init__(self) -> None: pass +class int: + def __eq__(self, x: object) -> bool: pass + def __lt__(self, x: int) -> bool: pass + def __gt__(self, x: int) -> int: pass +class bool: pass +class type: pass +class function: pass +class str: pass +class list: pass +class dict: pass +[out] +ComparisonExpr(3) : builtins.bool +ComparisonExpr(4) : builtins.bool +ComparisonExpr(5) : builtins.bool +ComparisonExpr(6) : builtins.bool +ComparisonExpr(7) : builtins.int +ComparisonExpr(8) : builtins.object + +[case testBooleanOps] +## OpExpr|UnaryExpr +import typing +a = 1 +a and a +a or a +not a +[builtins fixtures/bool.pyi] +[out] +OpExpr(4) : builtins.int +OpExpr(5) : builtins.int +UnaryExpr(6) : builtins.bool + +[case testBooleanOpsOnBools] +## OpExpr|UnaryExpr +import typing +a = bool() +a and a +a or a +not a +[builtins fixtures/bool.pyi] +[out] +OpExpr(4) : builtins.bool +OpExpr(5) : builtins.bool +UnaryExpr(6) : builtins.bool + +[case testFunctionCall] +## CallExpr +from typing import Tuple +class A: pass +class B: pass +def f(a: A, b: B) -> Tuple[A, B]: pass +f( + A(), + B()) +[builtins fixtures/tuple-simple.pyi] +[out] +CallExpr(6) : Tuple[A, B] +CallExpr(7) : A +CallExpr(8) : B + + +-- Statements +-- ---------- + + +[case testSimpleAssignment] +from typing import Any +a = None # type: A +b = a # type: Any +if b: + b = a + a = b + +class A: pass +[out] +NameExpr(3) : A +NameExpr(4) : Any +NameExpr(5) : A +NameExpr(5) : Any +NameExpr(6) : A +NameExpr(6) : Any + +[case testMemberAssignment] +from typing import Any +class A: + a = None # type: A + b = None # type: Any + def f(self) -> None: + self.b = self.a + self.a.a = self.b +[out] +MemberExpr(6) : A +MemberExpr(6) : Any +NameExpr(6) : A +NameExpr(6) : A +MemberExpr(7) : A +MemberExpr(7) : A +MemberExpr(7) : A +NameExpr(7) : A +NameExpr(7) : A + +[case testIf] + +a = None # type: bool +if a: + 1 +elif not a: + 1 +[builtins fixtures/bool.pyi] +[out] +NameExpr(3) : builtins.bool +IntExpr(4) : Literal[1]? +NameExpr(5) : Literal[False] +UnaryExpr(5) : builtins.bool +IntExpr(6) : Literal[1]? + +[case testWhile] + +a = None # type: bool +while a: + a +[builtins fixtures/bool.pyi] +[out] +NameExpr(3) : builtins.bool +NameExpr(4) : Literal[True] + + +-- Simple type inference +-- --------------------- + + +[case testInferSingleType] +import typing +x = () +[builtins fixtures/primitives.pyi] +[out] +NameExpr(2) : Tuple[()] +TupleExpr(2) : Tuple[()] + +[case testInferTwoTypes] +## NameExpr +import typing +(s, +i) = 'x', 1 +[builtins fixtures/primitives.pyi] +[out] +NameExpr(3) : builtins.str +NameExpr(4) : builtins.int + +[case testInferSingleLocalVarType] +import typing +def f() -> None: + x = () +[builtins fixtures/primitives.pyi] +[out] +NameExpr(3) : Tuple[()] +TupleExpr(3) : Tuple[()] + + +-- Basic generics +-- -------------- + + +[case testImplicitBoundTypeVarsForMethod] +## MemberExpr +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def f(self) -> T: pass +class B: pass +def g() -> None: + a = None # type: A[B] + f = a.f +[out] +MemberExpr(9) : def () -> B + +[case testImplicitBoundTypeVarsForSelfMethodReference] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def f(self) -> T: + return self.f() +[out] +CallExpr(5) : T`1 +MemberExpr(5) : def () -> T`1 +NameExpr(5) : A[T`1] + +[case testGenericFunctionCallWithTypeApp-skip] +## CallExpr|TypeApplication|NameExpr +from typing import Any, TypeVar, Tuple +T = TypeVar('T') +class A: pass +f[A](A()) +f[Any](A()) +def f(a: T) -> Tuple[T, T]: pass +[builtins fixtures/tuple.pyi] +[out] +CallExpr(5) : A +CallExpr(5) : Tuple[A, A] +NameExpr(5) : def () -> A +NameExpr(5) : def (a: A) -> Tuple[A, A] +TypeApplication(5) : def (a: A) -> Tuple[A, A] +CallExpr(6) : A +CallExpr(6) : Tuple[Any, Any] +NameExpr(6) : def () -> A +NameExpr(6) : def (a: Any) -> Tuple[Any, Any] +TypeApplication(6) : def (a: Any) -> Tuple[Any, Any] + +-- NOTE: Type applications are not supported for generic methods, so the +-- following test cases are commented out. + +--[case testGenericMethodCallWithTypeApp] +--## CallExpr|MemberExpr|TypeApplication +--from typing import Any, TypeVar, Tuple +--T = TypeVar('T') +--class A: +-- def f(self, a: T) -> Tuple[T, T]: pass +--a.f[A](a) +--a.f[Any](a) +--a = None # type: A +--[builtins fixtures/tuple.py] +--[out] +--CallExpr(2) : Tuple[A, A] +--MemberExpr(2) : def (A a) -> Tuple[A, A] +--TypeApplication(2) : def (A a) -> Tuple[A, A] +--CallExpr(3) : Tuple[Any, Any] +--MemberExpr(3) : def (any a) -> Tuple[Any, Any] +--TypeApplication(3) : def (any a) -> Tuple[Any, Any] + +--[case testGenericMethodCallInGenericTypeWithTypeApp] +--## CallExpr|MemberExpr|TypeApplication +--from typing import Any, TypeVar, Generic, Tuple +--T = TypeVar('T') +--S = TypeVar('S') +--class B: pass +--class C: pass +--a.f[B](b) +--a.f[Any](b) +--class A(Generic[T]): +-- def f(self, a: S) -> Tuple[T, S]: pass +--a = None # type: A[C] +--b = None # type: B +--[builtins fixtures/tuple.py] +--[out] +--CallExpr(6) : Tuple[C, B] +--MemberExpr(6) : def (B a) -> Tuple[C, B] +--TypeApplication(6) : def (B a) -> Tuple[C, B] +--CallExpr(7) : Tuple[C, Any] +--MemberExpr(7) : def (any a) -> Tuple[C, Any] +--TypeApplication(7) : def (any a) -> Tuple[C, Any] + +[case testGenericTypeVariableInference] +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): + def __init__(self, a: T) -> None: pass +class B: pass +A(A(B())) +[out] +CallExpr(6) : A[A[B]] +CallExpr(6) : A[B] +CallExpr(6) : B +NameExpr(6) : def (a: A[B]) -> A[A[B]] +NameExpr(6) : def (a: B) -> A[B] +NameExpr(6) : def () -> B + + +-- Generic inheritance +-- ------------------- + + +[case testInheritedMethodReferenceWithGenericInheritance] +from typing import TypeVar, Generic +T = TypeVar('T') +class C: pass +class A(Generic[T]): + def f(self, a: T) -> None: pass +class B(A[C]): + def g(self, c: C) -> None: + self.f(c) +[out] +CallExpr(8) : None +MemberExpr(8) : def (a: C) +NameExpr(8) : C +NameExpr(8) : B + +[case testInheritedMethodReferenceWithGenericSubclass] +from typing import TypeVar, Generic +S = TypeVar('S') +T = TypeVar('T') +class C: pass +class A(Generic[S, T]): + def f(self, a: C) -> None: pass +class B(A[C, T], Generic[T]): + def g(self, c: C) -> None: + self.f(c) +[out] +CallExpr(9) : None +MemberExpr(9) : def (a: C) +NameExpr(9) : C +NameExpr(9) : B[T`1] + +[case testExternalReferenceWithGenericInheritance] +from typing import TypeVar, Generic +T = TypeVar('T') +class C: pass +class A(Generic[T]): + def f(self, a: T) -> None: pass +class B(A[C]): pass +b = None # type: B +c = None # type: C +b.f(c) +[out] +CallExpr(9) : None +MemberExpr(9) : def (a: C) +NameExpr(9) : B +NameExpr(9) : C + + +-- Implicit Any types +-- ------------------ + + +[case testDynamicallyTypedFunction] + +def f(x): + y = x + o + z = o + z +o = None # type: object +[out] +NameExpr(3) : builtins.object +NameExpr(3) : Any +NameExpr(3) : Any +OpExpr(3) : Any +NameExpr(4) : builtins.object +NameExpr(4) : Any +NameExpr(5) : Any + +[case testDynamicallyTypedMethod] + +class A: + def f(self, x): + y = ( + o) # Place y and o on separate lines + x + y +o = None # type: object +[out] +NameExpr(4) : Any +NameExpr(5) : builtins.object +NameExpr(6) : Any +NameExpr(7) : Any + +[case testDynamicallyTypedConstructor] + +class A: + def __init__(self, x): + y = o + x + y +o = None # type: object +[out] +NameExpr(4) : builtins.object +NameExpr(4) : Any +NameExpr(5) : Any +NameExpr(6) : Any + +[case testCallInDynamicallyTypedFunction] + +def f(): + g(o) +def g(a: object) -> object: pass +o = None # type: object +[out] +CallExpr(3) : Any +NameExpr(3) : def (a: builtins.object) -> builtins.object +NameExpr(3) : builtins.object + +[case testExpressionInDynamicallyTypedFn] +import typing +def f(): + x = None + x.f() +[out] +CallExpr(4) : Any +MemberExpr(4) : Any +NameExpr(4) : Any + +[case testGenericCall] +from typing import TypeVar, Generic +T = TypeVar('T') +def f() -> None: + a1 = A(b) # type: A[B] + a2 = A(b) # type: A[object] +class A(Generic[T]): + def __init__(self, a: T) -> None: pass +class B: pass +b = None # type: B +[out] +CallExpr(4) : A[B] +NameExpr(4) : def (a: B) -> A[B] +NameExpr(4) : B +CallExpr(5) : A[builtins.object] +NameExpr(5) : def (a: builtins.object) -> A[builtins.object] +NameExpr(5) : B + +[case testGenericCallInDynamicallyTypedFunction] +from typing import TypeVar, Generic +T = TypeVar('T') +def f(): + A() +class A(Generic[T]): pass +[out] +CallExpr(4) : Any +NameExpr(4) : def [T] () -> A[T`1] + +[case testGenericCallInDynamicallyTypedFunction2] +from typing import TypeVar, Generic +T = TypeVar('T') +def f(): + A(f) +class A(Generic[T]): + def __init__(self, x: T) -> None: pass +[out] +CallExpr(4) : Any +NameExpr(4) : def [T] (x: T`1) -> A[T`1] +NameExpr(4) : def () -> Any + +[case testGenericCallInDynamicallyTypedFunction3] +from typing import TypeVar +t = TypeVar('t') +def f(): + g(None) +def g(x: t) -> t: pass +[out] +CallExpr(4) : Any +NameExpr(4) : def [t] (x: t`-1) -> t`-1 + + +-- Generic types and type inference +-- -------------------------------- + + +[case testInferenceInArgumentContext] +## CallExpr +from typing import TypeVar, Generic +T = TypeVar('T') +class A(Generic[T]): pass +class B: pass +class C(B): pass +def f(a: 'A[B]') -> None: pass +def g() -> 'A[T]': pass +def h(a: T) -> 'A[T]': pass + +b = None # type: B +c = None # type: C + +f(g()) +f(h(b)) +f(h(c)) +[out] +CallExpr(14) : None +CallExpr(14) : A[B] +CallExpr(15) : None +CallExpr(15) : A[B] +CallExpr(16) : None +CallExpr(16) : A[B] + +[case testInferGenericTypeForLocalVariable] +from typing import TypeVar, Generic +T = TypeVar('T') +def f() -> None: + a = A(b) + a + a2, a3 = A(b), A(c) + a2 + a3 +b = None # type: B +c = None # type: C +class A(Generic[T]): + def __init__(self, x: T) -> None: pass +class B: pass +class C: pass +[out] +CallExpr(4) : A[B] +NameExpr(4) : def (x: B) -> A[B] +NameExpr(4) : A[B] +NameExpr(4) : B +NameExpr(5) : A[B] +CallExpr(6) : A[B] +CallExpr(6) : A[C] +NameExpr(6) : def (x: B) -> A[B] +NameExpr(6) : def (x: C) -> A[C] +NameExpr(6) : A[B] +NameExpr(6) : A[C] +NameExpr(6) : B +NameExpr(6) : C +NameExpr(7) : A[B] +NameExpr(8) : A[C] + +[case testNestedGenericCalls] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +def h() -> None: + g(f(c)) + +c = None # type: C + +class A(Generic[T]): pass +class B(Generic[T]): pass +class C: pass +def f(a: T) -> A[T]: pass +def g(a: S) -> B[S]: pass +[out] +CallExpr(5) : A[C] +CallExpr(5) : B[A[C]] +NameExpr(5) : C +NameExpr(5) : def (a: C) -> A[C] +NameExpr(5) : def (a: A[C]) -> B[A[C]] + +[case testInferListLiterals] +from typing import List +a = [] # type: List[A] +class A: pass +[builtins fixtures/list.pyi] +[out] +ListExpr(2) : builtins.list[A] + +[case testInferGenericTypeInTypeAnyContext] +from typing import Any +a = [] # type: Any +[builtins fixtures/list.pyi] +[out] +ListExpr(2) : builtins.list[Any] + +[case testHigherOrderFunction] +from typing import TypeVar, Callable, List +t = TypeVar('t') +s = TypeVar('s') +def map(f: Callable[[t], s], a: List[t]) -> List[s]: pass +class A: pass +class B: pass +def f(a: A) -> B: pass +map( + f, + [A()]) +[builtins fixtures/list.pyi] +[out] +CallExpr(8) : builtins.list[B] +NameExpr(8) : def (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] +NameExpr(9) : def (a: A) -> B +CallExpr(10) : A +ListExpr(10) : builtins.list[A] +NameExpr(10) : def () -> A + + +-- Lambdas +-- ------- + + +[case testLambdaWithTypeInferredFromContext] +from typing import Callable +f = lambda x: x.a # type: Callable[[B], A] +class A: pass +class B: + a = None # type: A +[out] +LambdaExpr(2) : def (x: B) -> A +MemberExpr(2) : A +NameExpr(2) : B + +[case testLambdaWithInferredType] +## LambdaExpr|NameExpr +import typing +f = lambda: 1 +[out] +LambdaExpr(3) : def () -> Literal[1]? +NameExpr(3) : def () -> builtins.int + +[case testLambdaWithInferredType2] +## LambdaExpr|NameExpr +import typing +f = lambda: [1] +[builtins fixtures/list.pyi] +[out] +LambdaExpr(3) : def () -> builtins.list[builtins.int] +NameExpr(3) : def () -> builtins.list[builtins.int] + +[case testLambdaWithInferredType3] +from typing import List, Callable +f = lambda x: [] # type: Callable[[B], List[A]] +class A: pass +class B: + a = None # type: A +[builtins fixtures/list.pyi] +[out] +LambdaExpr(2) : def (x: B) -> builtins.list[A] +ListExpr(2) : builtins.list[A] + +[case testLambdaAndHigherOrderFunction] +from typing import TypeVar, Callable, List +t = TypeVar('t') +s = TypeVar('s') +def map(f: Callable[[t], s], a: List[t]) -> List[s]: pass +class A: pass +class B: pass +def f(a: A) -> B: pass +l = None # type: List[A] +map( + lambda x: f(x), l) +[builtins fixtures/list.pyi] +[out] +CallExpr(9) : builtins.list[B] +NameExpr(9) : def (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] +CallExpr(10) : B +LambdaExpr(10) : def (x: A) -> B +NameExpr(10) : def (a: A) -> B +NameExpr(10) : builtins.list[A] +NameExpr(10) : A + +[case testLambdaAndHigherOrderFunction2] +## LambdaExpr|NameExpr|ListExpr +from typing import TypeVar, List, Callable +t = TypeVar('t') +s = TypeVar('s') +def map(f: Callable[[t], List[s]], a: List[t]) -> List[s]: pass +class A: pass +class B: pass +def f(a: A) -> B: pass +l = None # type: List[A] +map( + lambda x: [f(x)], l) +[builtins fixtures/list.pyi] +[out] +NameExpr(10) : def (f: def (A) -> builtins.list[B], a: builtins.list[A]) -> builtins.list[B] +LambdaExpr(11) : def (x: A) -> builtins.list[B] +ListExpr(11) : builtins.list[B] +NameExpr(11) : def (a: A) -> B +NameExpr(11) : builtins.list[A] +NameExpr(11) : A + +[case testLambdaInListAndHigherOrderFunction] +from typing import TypeVar, Callable, List +t = TypeVar('t') +s = TypeVar('s') +def map(f: List[Callable[[t], s]], a: List[t]) -> List[s]: pass +class A: pass +l = None # type: List[A] +map( + [lambda x: x], + l) +[builtins fixtures/list.pyi] +[out] +-- TODO We probably should not silently infer 'Any' types in statically typed +-- context. Perhaps just fail instead? +CallExpr(7) : builtins.list[Any] +NameExpr(7) : def (f: builtins.list[def (A) -> Any], a: builtins.list[A]) -> builtins.list[Any] +LambdaExpr(8) : def (x: A) -> A +ListExpr(8) : builtins.list[def (A) -> Any] +NameExpr(8) : A +NameExpr(9) : builtins.list[A] + +[case testLambdaAndHigherOrderFunction3] +from typing import TypeVar, Callable, List +t = TypeVar('t') +s = TypeVar('s') +def map(f: Callable[[t], s], a: List[t]) -> List[s]: pass +class A: + b = None # type: B +class B: pass +l = None # type: List[A] +map( + lambda x: x.b, + l) +[builtins fixtures/list.pyi] +[out] +CallExpr(9) : builtins.list[B] +NameExpr(9) : def (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] +LambdaExpr(10) : def (x: A) -> B +MemberExpr(10) : B +NameExpr(10) : A +NameExpr(11) : builtins.list[A] + +[case testLambdaAndHigherOrderFunctionAndKeywordArgs] +from typing import TypeVar, Callable, List +t = TypeVar('t') +s = TypeVar('s') +def map(f: Callable[[t], s], a: List[t]) -> List[s]: pass +class A: + b = None # type: B +class B: pass +l = None # type: List[A] +map( + a=l, + f=lambda x: x.b) +[builtins fixtures/list.pyi] +[out] +CallExpr(9) : builtins.list[B] +NameExpr(9) : def (f: def (A) -> B, a: builtins.list[A]) -> builtins.list[B] +NameExpr(10) : builtins.list[A] +LambdaExpr(11) : def (x: A) -> B +MemberExpr(11) : B +NameExpr(11) : A + + +-- Boolean operations +-- ------------------ + + +[case testBooleanOr] +from typing import List +a = None # type: List[A] +a or [] +a = a or [] +if int(): + a = [] or a +class A: pass +[builtins fixtures/list.pyi] +[out] +ListExpr(3) : builtins.list[A] +NameExpr(3) : builtins.list[A] +OpExpr(3) : builtins.list[A] +ListExpr(4) : builtins.list[A] +NameExpr(4) : builtins.list[A] +NameExpr(4) : builtins.list[A] +OpExpr(4) : builtins.list[A] +CallExpr(5) : builtins.int +NameExpr(5) : def () -> builtins.int +ListExpr(6) : builtins.list[A] +NameExpr(6) : builtins.list[A] +NameExpr(6) : builtins.list[A] +OpExpr(6) : builtins.list[A] + + +-- Class attributes +-- ---------------- + + +[case testUnboundMethod] +## MemberExpr +import typing +class A: + def f(self) -> None: pass +A.f +[out] +MemberExpr(5) : def (self: A) + +[case testUnboundMethodWithImplicitSig] +## MemberExpr +import typing +class A: + def f(self): pass +A.f +[out] +MemberExpr(5) : def (self: A) -> Any + +[case testOverloadedUnboundMethod] +## MemberExpr +from typing import overload +class A: + @overload + def f(self) -> None: pass + @overload + def f(self, __x: object) -> None: pass + + def f(self, *args) -> None: pass +A.f +[builtins fixtures/tuple.pyi] +[out] +MemberExpr(10) : Overload(def (self: A), def (self: A, builtins.object)) + +[case testOverloadedUnboundMethodWithImplicitSig] +## MemberExpr +from typing import overload +class A: + @overload + def f(self): pass + @overload + def f(self, __x): pass + + def f(self, *args): pass +A.f +[builtins fixtures/tuple.pyi] +[out] +MemberExpr(10) : Overload(def (self: A) -> Any, def (self: A, Any) -> Any) + +[case testUnboundMethodWithInheritance] +## MemberExpr +import typing +class A: + def __init__(self) -> None: pass + def f(self) -> None: pass +class B(A): + pass +B.f +[out] +MemberExpr(8) : def (self: A) + +[case testUnboundGenericMethod] +## MemberExpr +from typing import TypeVar +t = TypeVar('t') +class B: pass +class A: + def f(self, x: t) -> None: pass +A.f(A(), B()) +[out] +MemberExpr(7) : def (self: A, x: B) + +[case testUnboundMethodOfGenericClass] +## MemberExpr +from typing import TypeVar, Generic +t = TypeVar('t') +class B: pass +class A(Generic[t]): + def f(self, x: t) -> None: pass +A.f +a_b = A() # type: A[B] +A.f(a_b, B()) +[out] +MemberExpr(7) : def [t] (self: A[t`1], x: t`1) +MemberExpr(9) : def (self: A[B], x: B) + +[case testUnboundOverloadedMethodOfGenericClass] +## CallExpr +from typing import TypeVar, Generic, overload +t = TypeVar('t') +class B: pass +class A(Generic[t]): + @overload + def f(self, x: t) -> t: pass + @overload + def f(self) -> object: pass + def f(self, *args): pass + +ab, b = None, None # type: (A[B], B) +A.f(ab, b) +[builtins fixtures/tuple.pyi] +[out] +CallExpr(13) : B + +[case testUnboundMethodOfGenericClassWithImplicitSig] +## MemberExpr +from typing import TypeVar, Generic +t = TypeVar('t') +class B: pass +class A(Generic[t]): + def f(self, x): pass +A.f(None, None) +[out] +MemberExpr(7) : def (self: A[t`1], x: Any) -> Any + +[case testGenericMethodOfGenericClass] +## MemberExpr +from typing import TypeVar, Generic +t = TypeVar('t') +s = TypeVar('s') +class B: pass +class A(Generic[t]): + def f(self, y: s) -> None: pass +ab = None # type: A[B] +o = None # type: object +A.f(ab, o) +[out] +MemberExpr(10) : def (self: A[B], y: builtins.object) + + +-- Type variables with value restriction +-- ------------------------------------- + + +[case testTypeVariableWithValueRestriction] +## NameExpr +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> None: pass +f(1) +f('x') +[out] +NameExpr(5) : def (x: builtins.int) +NameExpr(6) : def (x: builtins.str) + +[case testTypeVariableWithValueRestrictionAndSubtype] +## NameExpr|CallExpr +from typing import TypeVar +T = TypeVar('T', int, str) +def f(x: T) -> T: pass +class S(str): pass +s = None # type: S +f(s) +[out] +CallExpr(7) : builtins.str +NameExpr(7) : def (x: builtins.str) -> builtins.str +NameExpr(7) : S + +[case testTypeVariableWithValueRestrictionInFunction] +## NameExpr +from typing import TypeVar + +T = TypeVar("T", int, str) + +def f(x: T) -> T: + y = 1 + return x +[out] +NameExpr(7) : builtins.int +NameExpr(7) : builtins.int +NameExpr(8) : builtins.int +NameExpr(8) : builtins.str + + +-- Binary operations +-- ----------------- + + +[case testBinaryOperatorWithAnyLeftOperand] +## OpExpr +from typing import Any, cast +class B: + def __add__(self, x: int) -> str: pass +class A: + def __radd__(self, x: B) -> int: pass +cast(Any, 1) + A() +B() + A() +[out] +OpExpr(7) : Any +OpExpr(8) : builtins.int + +[case testBinaryOperatorWithAnyRightOperand] +## OpExpr +from typing import Any, cast +class A: + def __add__(self, x: str) -> int: pass +A() + cast(Any, 1) +[out] +OpExpr(5) : Any + + +-- Callable overloading +-- -------------------- + + +[case testOverloadedFunctionType] +## CallExpr +from typing import overload +@overload +def f(x: int) -> str: pass +@overload +def f(x: str) -> int: pass +def f(x): pass +f(1) +f('') +[out] +CallExpr(8) : builtins.str +CallExpr(9) : builtins.int + +[case testOverlappingOverloadedFunctionType] +## CallExpr +from typing import overload, Any +class A: pass +class B(A): pass +@overload +def f(x: B) -> B: pass +@overload +def f(x: A) -> A: pass + +def f(x) -> Any: pass + +a = None # type: A +b = None # type: B +f(a) +f(b) +[out] +CallExpr(14) : A +CallExpr(15) : B + + + +[case testOverloadedErasedType] +from typing import Callable +from typing import List +from typing import overload +from typing import TypeVar + +T = TypeVar("T") +V = TypeVar("V") + +def fun(s: int) -> int: pass + +def m(fun: Callable[[T], V], iter: List[T]) -> None: pass + +nums = [1] # type: List[int] +m(fun, + nums) +[builtins fixtures/list.pyi] +[out] +IntExpr(13) : Literal[1]? +ListExpr(13) : builtins.list[builtins.int] +CallExpr(14) : None +NameExpr(14) : def (s: builtins.int) -> builtins.int +NameExpr(14) : def (fun: def (builtins.int) -> builtins.int, iter: builtins.list[builtins.int]) +NameExpr(15) : builtins.list[builtins.int] + + +-- Special cases +-- ------------- + + +[case testImplicitDataAttributeInit] +## NameExpr +import typing +class A: + def __init__(self) -> None: + self.x = ( + A()) +[out] +NameExpr(5) : A +NameExpr(6) : def () -> A + +[case testListMultiplicationInContext] +## ListExpr|OpExpr|IntExpr +from typing import List +a = [None] * 3 # type: List[str] +[builtins fixtures/list.pyi] +[out] +IntExpr(3) : Literal[3]? +ListExpr(3) : builtins.list[builtins.str] +OpExpr(3) : builtins.list[builtins.str] + +[case testStringFormatting] +## IntExpr|OpExpr|StrExpr +'%d' % 1 +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] +[out] +IntExpr(2) : Literal[1]? +OpExpr(2) : builtins.str +StrExpr(2) : Literal['%d']? + +[case testExportOverloadArgType] +## LambdaExpr|NameExpr +from typing import List, overload, Callable +@overload +def f(x: int, f: Callable[[int], int]) -> None: ... +@overload +def f(x: str, f: Callable[[str], str]) -> None: ... +def f(x): ... +f( + 1, lambda x: x) +[builtins fixtures/list.pyi] +[out] +NameExpr(8) : Overload(def (x: builtins.int, f: def (builtins.int) -> builtins.int), def (x: builtins.str, f: def (builtins.str) -> builtins.str)) +LambdaExpr(9) : def (x: builtins.int) -> builtins.int +NameExpr(9) : builtins.int + +[case testExportOverloadArgTypeNested] +## LambdaExpr +from typing import overload, Callable +@overload +def f(x: int, f: Callable[[int], int]) -> int: ... +@overload +def f(x: str, f: Callable[[str], str]) -> str: ... +def f(x): ... +f( + f(1, lambda y: y), + lambda x: x) +f( + f('x', lambda y: y), + lambda x: x) +[builtins fixtures/list.pyi] +[out] +LambdaExpr(9) : def (y: builtins.int) -> builtins.int +LambdaExpr(10) : def (x: builtins.int) -> builtins.int +LambdaExpr(12) : def (y: builtins.str) -> builtins.str +LambdaExpr(13) : def (x: builtins.str) -> builtins.str + +[case testExportOverloadArgTypeDict] +## DictExpr +from typing import TypeVar, Generic, Any, overload, Dict +T = TypeVar("T") +class Key(Generic[T]): ... +@overload +def f(x: Key[T], y: T) -> T: ... +@overload +def f(x: int, y: Any) -> Any: ... +def f(x, y): ... +d: Dict = {} +d.get( + "", {}) +f( + 2, {}) +[builtins fixtures/dict.pyi] +[out] +DictExpr(10) : builtins.dict[Any, Any] +DictExpr(12) : builtins.dict[Any, Any] +DictExpr(14) : builtins.dict[Any, Any] + +-- TODO +-- +-- test expressions +-- list literal +-- tuple literal +-- unary minus +-- indexing +-- super expression +-- more complex lambda (multiple arguments etc.) +-- list comprehension +-- generator expression +-- other things +-- type inference +-- default argument value +-- for loop variable +-- exception variable +-- varargs +-- generics +-- explicit types +-- type of 'None' (currently stripped, but sometimes we may want to dump it) diff --git a/test-requirements.in b/test-requirements.in new file mode 100644 index 000000000000..4e53c63cc36b --- /dev/null +++ b/test-requirements.in @@ -0,0 +1,16 @@ +# If you change this file (or mypy-requirements.txt or build-requirements.txt), please run: +# pip-compile --output-file=test-requirements.txt --strip-extras --allow-unsafe test-requirements.in + +-r mypy-requirements.txt +-r build-requirements.txt +attrs>=18.0 +filelock>=3.3.0 +# lxml 4.9.3 switched to manylinux_2_28, the wheel builder still uses manylinux2014 +lxml>=4.9.1,<4.9.3; (python_version<'3.11' or sys_platform!='win32') and python_version<'3.12' +psutil>=4.0 +pytest>=8.1.0 +pytest-xdist>=1.34.0 +pytest-cov>=2.10.0 +setuptools>=75.1.0 +tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.8 +pre_commit>=3.5.0 diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 000000000000..6eb6f6a95ac8 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,65 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in +# +attrs==24.2.0 + # via -r test-requirements.in +cfgv==3.4.0 + # via pre-commit +coverage==7.6.1 + # via pytest-cov +distlib==0.3.9 + # via virtualenv +execnet==2.1.1 + # via pytest-xdist +filelock==3.16.1 + # via + # -r test-requirements.in + # virtualenv +identify==2.6.1 + # via pre-commit +iniconfig==2.0.0 + # via pytest +lxml==4.9.2 ; (python_version < "3.11" or sys_platform != "win32") and python_version < "3.12" + # via -r test-requirements.in +mypy-extensions==1.0.0 + # via -r mypy-requirements.txt +nodeenv==1.9.1 + # via pre-commit +packaging==24.1 + # via pytest +platformdirs==4.3.6 + # via virtualenv +pluggy==1.5.0 + # via pytest +pre-commit==3.5.0 + # via -r test-requirements.in +psutil==6.0.0 + # via -r test-requirements.in +pytest==8.3.3 + # via + # -r test-requirements.in + # pytest-cov + # pytest-xdist +pytest-cov==5.0.0 + # via -r test-requirements.in +pytest-xdist==3.6.1 + # via -r test-requirements.in +pyyaml==6.0.2 + # via pre-commit +tomli==2.0.2 + # via -r test-requirements.in +types-psutil==6.0.0.20241011 + # via -r build-requirements.txt +types-setuptools==75.1.0.20241014 + # via -r build-requirements.txt +typing-extensions==4.12.2 + # via -r mypy-requirements.txt +virtualenv==20.26.6 + # via pre-commit + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 + # via -r test-requirements.in diff --git a/tests.py b/tests.py deleted file mode 100644 index 60db26785331..000000000000 --- a/tests.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys - -import typing - -from mypy.myunit import Suite, run_test -from mypy.test import testtypes -from mypy.test import testsubtypes -from mypy.test import testsolve -from mypy.test import testinfer -from mypy.test import testlex -from mypy.test import testparse -from mypy.test import testsemanal -from mypy.test import testtransform -from mypy.test import testcheck -from mypy.test import testtypegen -from mypy.test import testoutput - - -class AllSuite(Suite): - def __init__(self): - self.test_types = testtypes.TypesSuite() - self.test_typeops = testtypes.TypeOpsSuite() - self.test_join = testtypes.JoinSuite() - self.test_meet = testtypes.MeetSuite() - self.test_subtypes = testsubtypes.SubtypingSuite() - self.test_solve = testsolve.SolveSuite() - self.test_infer = testinfer.MapActualsToFormalsSuite() - self.test_lex = testlex.LexerSuite() - self.test_parse = testparse.ParserSuite() - self.test_parse_errors = testparse.ParseErrorSuite() - self.test_semanal = testsemanal.SemAnalSuite() - self.test_semanal_errors = testsemanal.SemAnalErrorSuite() - self.test_semanal_symtable = testsemanal.SemAnalSymtableSuite() - self.test_semanal_typeinfos = testsemanal.SemAnalTypeInfoSuite() - self.test_transform = testtransform.TransformSuite() - self.test_check = testcheck.TypeCheckSuite() - self.test_typegen = testtypegen.TypeExportSuite() - self.test_output = testoutput.OutputSuite() - super().__init__() - - -if __name__ == '__main__': - run_test(AllSuite(), sys.argv[1:]) diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000000..a505950521fa --- /dev/null +++ b/tox.ini @@ -0,0 +1,64 @@ +[tox] +minversion = 4.4.4 +skip_missing_interpreters = {env:TOX_SKIP_MISSING_INTERPRETERS:True} +envlist = + py38, + py39, + py310, + py311, + py312, + py313, + docs, + lint, + type, +isolated_build = true + +[testenv] +description = run the test driver with {basepython} +passenv = + PYTEST_XDIST_WORKER_COUNT + PROGRAMDATA + PROGRAMFILES(X86) + PYTEST_ADDOPTS +deps = + -r test-requirements.txt + # This is a bit of a hack, but ensures the faster-cache path is tested in CI + orjson;python_version=='3.12' +commands = python -m pytest {posargs} + +[testenv:dev] +description = generate a DEV environment, that has all project libraries +usedevelop = True +deps = + -r test-requirements.txt + -r docs/requirements-docs.txt +commands = + python -m pip list --format=columns + python -c 'import sys; print(sys.executable)' + {posargs} + +[testenv:docs] +description = invoke sphinx-build to build the HTML docs +passenv = + VERIFY_MYPY_ERROR_CODES +deps = -r docs/requirements-docs.txt +commands = + sphinx-build -n -d "{toxworkdir}/docs_doctree" docs/source "{toxworkdir}/docs_out" --color -W -bhtml {posargs} + python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' + +[testenv:lint] +description = check the code style +skip_install = true +deps = pre-commit +commands = pre-commit run --all-files --show-diff-on-failure + +[testenv:type] +description = type check ourselves +passenv = + TERM + MYPY_FORCE_COLOR + MYPY_FORCE_TERMINAL_WIDTH +commands = + python runtests.py self + python -m mypy --config-file mypy_self_check.ini misc --exclude misc/sync-typeshed.py + python -m mypy --config-file mypy_self_check.ini test-data/unit/plugins diff --git a/travis.sh b/travis.sh deleted file mode 100755 index 2104cbe54670..000000000000 --- a/travis.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -PYTHON=python - -result=0 - -fail() -{ - result=1 -} - -# Setup stuff - -DRIVER=$PWD/scripts/mypy -export PYTHONPATH=`pwd`/lib-typing/3.2:`pwd` - -# Basic tests - -echo Running tests... -echo -echo tests.py -"$PYTHON" "$DRIVER" tests.py || fail -"$PYTHON" tests.py || fail -for t in mypy.test.testpythoneval; do - echo $t - "$PYTHON" "$DRIVER" -m $t || fail - "$PYTHON" -m $t || fail -done - -# Stub checks - -STUBTEST=_test_stubs.py -echo "import typing" > $STUBTEST -cd stubs/3.2 -ls *.py | sed s/\\.py//g | sed "s/^/import /g" >> ../../$STUBTEST -for m in os os.path; do - echo "import $m" >> ../../$STUBTEST -done -cd ../.. - -NUMSTUBS=$(( `wc -l $STUBTEST | cut -d' ' -f1` - 1 )) - -echo Type checking $NUMSTUBS stubs... -echo -"$PYTHON" "$DRIVER" $STUBTEST || fail -rm $STUBTEST - -# Sample checks - -# Only run under 3.2 - -if [ "`"$PYTHON" -c 'from sys import version_info as vi; print(vi.major, vi.minor)'`" == "3 2" ]; then - echo Type checking lib-python... - echo - cd lib-python/3.2 - for f in test/test_*.py; do - mod=test.`basename "$f" .py` - echo $mod - "$PYTHON" "$DRIVER" -m $mod || fail - done -else - echo "Skipping lib-python type checks (not Python 3.2!)" -fi - -exit $result